{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 23.24228866922803,
  "global_step": 400000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "activations/layer0_attention_weight_max": 2.7026662826538086,
      "activations/layer0_attention_weight_min": -2.3810482025146484,
      "activations/layer10_attention_weight_max": 0.2262684553861618,
      "activations/layer10_attention_weight_min": -0.2422683984041214,
      "activations/layer11_attention_weight_max": 0.2098274677991867,
      "activations/layer11_attention_weight_min": -0.18241320550441742,
      "activations/layer12_attention_weight_max": 0.17375782132148743,
      "activations/layer12_attention_weight_min": -0.18595850467681885,
      "activations/layer13_attention_weight_max": 0.16491027176380157,
      "activations/layer13_attention_weight_min": -0.17215639352798462,
      "activations/layer14_attention_weight_max": 0.17021788656711578,
      "activations/layer14_attention_weight_min": -0.1523570716381073,
      "activations/layer15_attention_weight_max": 0.15184876322746277,
      "activations/layer15_attention_weight_min": -0.14781402051448822,
      "activations/layer16_attention_weight_max": 0.1319195181131363,
      "activations/layer16_attention_weight_min": -0.137898787856102,
      "activations/layer17_attention_weight_max": 0.12976433336734772,
      "activations/layer17_attention_weight_min": -0.1446106731891632,
      "activations/layer18_attention_weight_max": 0.12024380266666412,
      "activations/layer18_attention_weight_min": -0.12917347252368927,
      "activations/layer19_attention_weight_max": 0.12277376651763916,
      "activations/layer19_attention_weight_min": -0.12836526334285736,
      "activations/layer1_attention_weight_max": 1.4453972578048706,
      "activations/layer1_attention_weight_min": -1.2956246137619019,
      "activations/layer20_attention_weight_max": 0.10586348176002502,
      "activations/layer20_attention_weight_min": -0.12099900841712952,
      "activations/layer21_attention_weight_max": 0.09773679822683334,
      "activations/layer21_attention_weight_min": -0.1159888356924057,
      "activations/layer22_attention_weight_max": 0.1019996702671051,
      "activations/layer22_attention_weight_min": -0.10794715583324432,
      "activations/layer23_attention_weight_max": 0.09683002531528473,
      "activations/layer23_attention_weight_min": -0.10010390728712082,
      "activations/layer2_attention_weight_max": 0.8508017063140869,
      "activations/layer2_attention_weight_min": -0.8351360559463501,
      "activations/layer3_attention_weight_max": 0.6267672181129456,
      "activations/layer3_attention_weight_min": -0.643147349357605,
      "activations/layer4_attention_weight_max": 0.49353525042533875,
      "activations/layer4_attention_weight_min": -0.4694107472896576,
      "activations/layer5_attention_weight_max": 0.46040865778923035,
      "activations/layer5_attention_weight_min": -0.45318397879600525,
      "activations/layer6_attention_weight_max": 0.41001904010772705,
      "activations/layer6_attention_weight_min": -0.34710004925727844,
      "activations/layer7_attention_weight_max": 0.310708612203598,
      "activations/layer7_attention_weight_min": -0.3016427457332611,
      "activations/layer8_attention_weight_max": 0.26590919494628906,
      "activations/layer8_attention_weight_min": -0.2674899995326996,
      "activations/layer9_attention_weight_max": 0.260041207075119,
      "activations/layer9_attention_weight_min": -0.2471604198217392,
      "epoch": 0.0,
      "learning_rate": 0,
      "loss": 11.0459,
      "step": 1
    },
    {
      "activations/layer0_attention_weight_max": 2.904405117034912,
      "activations/layer0_attention_weight_min": -2.682366132736206,
      "activations/layer10_attention_weight_max": 0.22359690070152283,
      "activations/layer10_attention_weight_min": -0.20482604205608368,
      "activations/layer11_attention_weight_max": 0.1812920868396759,
      "activations/layer11_attention_weight_min": -0.22947359085083008,
      "activations/layer12_attention_weight_max": 0.22935603559017181,
      "activations/layer12_attention_weight_min": -0.2640909254550934,
      "activations/layer13_attention_weight_max": 0.15680783987045288,
      "activations/layer13_attention_weight_min": -0.1357249617576599,
      "activations/layer14_attention_weight_max": 0.13764610886573792,
      "activations/layer14_attention_weight_min": -0.1330937296152115,
      "activations/layer15_attention_weight_max": 0.15938904881477356,
      "activations/layer15_attention_weight_min": -0.1266685277223587,
      "activations/layer16_attention_weight_max": 0.11993252485990524,
      "activations/layer16_attention_weight_min": -0.16508294641971588,
      "activations/layer17_attention_weight_max": 0.14373666048049927,
      "activations/layer17_attention_weight_min": -0.16870518028736115,
      "activations/layer18_attention_weight_max": 0.0810789093375206,
      "activations/layer18_attention_weight_min": -0.10230337083339691,
      "activations/layer19_attention_weight_max": 0.10985022783279419,
      "activations/layer19_attention_weight_min": -0.13715027272701263,
      "activations/layer1_attention_weight_max": 1.3766191005706787,
      "activations/layer1_attention_weight_min": -1.5893818140029907,
      "activations/layer20_attention_weight_max": 0.0706535056233406,
      "activations/layer20_attention_weight_min": -0.11197640001773834,
      "activations/layer21_attention_weight_max": 0.1138077974319458,
      "activations/layer21_attention_weight_min": -0.07224638760089874,
      "activations/layer22_attention_weight_max": 0.08635015785694122,
      "activations/layer22_attention_weight_min": -0.09457103908061981,
      "activations/layer23_attention_weight_max": 0.06860679388046265,
      "activations/layer23_attention_weight_min": -0.09201296418905258,
      "activations/layer2_attention_weight_max": 0.9559236764907837,
      "activations/layer2_attention_weight_min": -1.0733520984649658,
      "activations/layer3_attention_weight_max": 0.603473424911499,
      "activations/layer3_attention_weight_min": -0.7268968820571899,
      "activations/layer4_attention_weight_max": 0.6037544012069702,
      "activations/layer4_attention_weight_min": -0.7725066542625427,
      "activations/layer5_attention_weight_max": 0.46971797943115234,
      "activations/layer5_attention_weight_min": -0.49839672446250916,
      "activations/layer6_attention_weight_max": 0.5314158797264099,
      "activations/layer6_attention_weight_min": -0.6791671514511108,
      "activations/layer7_attention_weight_max": 0.30841588973999023,
      "activations/layer7_attention_weight_min": -0.36456045508384705,
      "activations/layer8_attention_weight_max": 0.3519696891307831,
      "activations/layer8_attention_weight_min": -0.43067052960395813,
      "activations/layer9_attention_weight_max": 0.24673549830913544,
      "activations/layer9_attention_weight_min": -0.2678564488887787,
      "epoch": 0.0,
      "learning_rate": 6.480885262242102e-05,
      "loss": 9.3232,
      "step": 50
    },
    {
      "activations/layer0_attention_weight_max": 4.6382527351379395,
      "activations/layer0_attention_weight_min": -3.789039134979248,
      "activations/layer10_attention_weight_max": 0.47380495071411133,
      "activations/layer10_attention_weight_min": -0.5558009147644043,
      "activations/layer11_attention_weight_max": 0.850273072719574,
      "activations/layer11_attention_weight_min": -0.8639842867851257,
      "activations/layer12_attention_weight_max": 0.8887906670570374,
      "activations/layer12_attention_weight_min": -0.8404444456100464,
      "activations/layer13_attention_weight_max": 0.4355781376361847,
      "activations/layer13_attention_weight_min": -0.4875165820121765,
      "activations/layer14_attention_weight_max": 0.5226117372512817,
      "activations/layer14_attention_weight_min": -0.5407835841178894,
      "activations/layer15_attention_weight_max": 0.571049153804779,
      "activations/layer15_attention_weight_min": -0.5471934080123901,
      "activations/layer16_attention_weight_max": 0.8296215534210205,
      "activations/layer16_attention_weight_min": -0.8068585991859436,
      "activations/layer17_attention_weight_max": 1.3996765613555908,
      "activations/layer17_attention_weight_min": -1.3839231729507446,
      "activations/layer18_attention_weight_max": 0.5716568827629089,
      "activations/layer18_attention_weight_min": -0.49613821506500244,
      "activations/layer19_attention_weight_max": 0.2935236692428589,
      "activations/layer19_attention_weight_min": -0.3687724769115448,
      "activations/layer1_attention_weight_max": 2.4003124237060547,
      "activations/layer1_attention_weight_min": -1.6470203399658203,
      "activations/layer20_attention_weight_max": 0.4083443284034729,
      "activations/layer20_attention_weight_min": -0.46501782536506653,
      "activations/layer21_attention_weight_max": 0.38405606150627136,
      "activations/layer21_attention_weight_min": -0.40641266107559204,
      "activations/layer22_attention_weight_max": 0.46353790163993835,
      "activations/layer22_attention_weight_min": -0.3983507752418518,
      "activations/layer23_attention_weight_max": 0.3520139157772064,
      "activations/layer23_attention_weight_min": -0.4195142388343811,
      "activations/layer2_attention_weight_max": 1.7701438665390015,
      "activations/layer2_attention_weight_min": -1.467237949371338,
      "activations/layer3_attention_weight_max": 2.120082139968872,
      "activations/layer3_attention_weight_min": -1.6443289518356323,
      "activations/layer4_attention_weight_max": 4.321783542633057,
      "activations/layer4_attention_weight_min": -3.158844470977783,
      "activations/layer5_attention_weight_max": 1.7101995944976807,
      "activations/layer5_attention_weight_min": -1.3895900249481201,
      "activations/layer6_attention_weight_max": 2.075596570968628,
      "activations/layer6_attention_weight_min": -2.0104780197143555,
      "activations/layer7_attention_weight_max": 0.6737711429595947,
      "activations/layer7_attention_weight_min": -0.841904878616333,
      "activations/layer8_attention_weight_max": 1.3309812545776367,
      "activations/layer8_attention_weight_min": -1.1773808002471924,
      "activations/layer9_attention_weight_max": 0.8724977374076843,
      "activations/layer9_attention_weight_min": -0.870671272277832,
      "epoch": 0.01,
      "learning_rate": 8.055800524842374e-05,
      "loss": 7.0476,
      "step": 100
    },
    {
      "activations/layer0_attention_weight_max": 6.5583319664001465,
      "activations/layer0_attention_weight_min": -4.369915008544922,
      "activations/layer10_attention_weight_max": 0.8138318061828613,
      "activations/layer10_attention_weight_min": -0.7547721862792969,
      "activations/layer11_attention_weight_max": 2.1868796348571777,
      "activations/layer11_attention_weight_min": -1.7422837018966675,
      "activations/layer12_attention_weight_max": 1.5099438428878784,
      "activations/layer12_attention_weight_min": -1.1619179248809814,
      "activations/layer13_attention_weight_max": 0.7874138355255127,
      "activations/layer13_attention_weight_min": -0.7572888135910034,
      "activations/layer14_attention_weight_max": 1.1829092502593994,
      "activations/layer14_attention_weight_min": -0.9102572798728943,
      "activations/layer15_attention_weight_max": 1.2231383323669434,
      "activations/layer15_attention_weight_min": -1.0150235891342163,
      "activations/layer16_attention_weight_max": 2.391324996948242,
      "activations/layer16_attention_weight_min": -2.086395025253296,
      "activations/layer17_attention_weight_max": 8.209704399108887,
      "activations/layer17_attention_weight_min": -4.895308017730713,
      "activations/layer18_attention_weight_max": 1.4537509679794312,
      "activations/layer18_attention_weight_min": -1.18174147605896,
      "activations/layer19_attention_weight_max": 0.6554092764854431,
      "activations/layer19_attention_weight_min": -0.6799978613853455,
      "activations/layer1_attention_weight_max": 2.9874634742736816,
      "activations/layer1_attention_weight_min": -2.006842613220215,
      "activations/layer20_attention_weight_max": 1.0603140592575073,
      "activations/layer20_attention_weight_min": -0.941720187664032,
      "activations/layer21_attention_weight_max": 0.9753615260124207,
      "activations/layer21_attention_weight_min": -0.914339005947113,
      "activations/layer22_attention_weight_max": 1.4977107048034668,
      "activations/layer22_attention_weight_min": -1.1855363845825195,
      "activations/layer23_attention_weight_max": 0.8689247965812683,
      "activations/layer23_attention_weight_min": -0.827002763748169,
      "activations/layer2_attention_weight_max": 1.854895830154419,
      "activations/layer2_attention_weight_min": -1.7090747356414795,
      "activations/layer3_attention_weight_max": 3.0810089111328125,
      "activations/layer3_attention_weight_min": -2.2034785747528076,
      "activations/layer4_attention_weight_max": 14.077879905700684,
      "activations/layer4_attention_weight_min": -10.366164207458496,
      "activations/layer5_attention_weight_max": 3.683563232421875,
      "activations/layer5_attention_weight_min": -2.126936674118042,
      "activations/layer6_attention_weight_max": 5.724435329437256,
      "activations/layer6_attention_weight_min": -3.8848118782043457,
      "activations/layer7_attention_weight_max": 0.9531285166740417,
      "activations/layer7_attention_weight_min": -1.0114924907684326,
      "activations/layer8_attention_weight_max": 2.877864122390747,
      "activations/layer8_attention_weight_min": -1.8768669366836548,
      "activations/layer9_attention_weight_max": 1.7509812116622925,
      "activations/layer9_attention_weight_min": -1.1818077564239502,
      "epoch": 0.01,
      "learning_rate": 8.884661472213505e-05,
      "loss": 6.5072,
      "step": 150
    },
    {
      "activations/layer0_attention_weight_max": 7.588281631469727,
      "activations/layer0_attention_weight_min": -4.101579666137695,
      "activations/layer10_attention_weight_max": 0.9269364476203918,
      "activations/layer10_attention_weight_min": -0.814174473285675,
      "activations/layer11_attention_weight_max": 3.0909996032714844,
      "activations/layer11_attention_weight_min": -2.509136915206909,
      "activations/layer12_attention_weight_max": 1.9543296098709106,
      "activations/layer12_attention_weight_min": -1.372875690460205,
      "activations/layer13_attention_weight_max": 0.9085217118263245,
      "activations/layer13_attention_weight_min": -0.9216160774230957,
      "activations/layer14_attention_weight_max": 1.5885069370269775,
      "activations/layer14_attention_weight_min": -1.176330804824829,
      "activations/layer15_attention_weight_max": 1.4837613105773926,
      "activations/layer15_attention_weight_min": -1.1983473300933838,
      "activations/layer16_attention_weight_max": 4.1105427742004395,
      "activations/layer16_attention_weight_min": -3.506025552749634,
      "activations/layer17_attention_weight_max": 11.769139289855957,
      "activations/layer17_attention_weight_min": -6.056501388549805,
      "activations/layer18_attention_weight_max": 1.9937177896499634,
      "activations/layer18_attention_weight_min": -1.5481959581375122,
      "activations/layer19_attention_weight_max": 0.8028788566589355,
      "activations/layer19_attention_weight_min": -0.8185343146324158,
      "activations/layer1_attention_weight_max": 3.38971209526062,
      "activations/layer1_attention_weight_min": -2.450448513031006,
      "activations/layer20_attention_weight_max": 1.776340365409851,
      "activations/layer20_attention_weight_min": -1.6125065088272095,
      "activations/layer21_attention_weight_max": 1.3249202966690063,
      "activations/layer21_attention_weight_min": -1.2561136484146118,
      "activations/layer22_attention_weight_max": 2.660797595977783,
      "activations/layer22_attention_weight_min": -2.155984878540039,
      "activations/layer23_attention_weight_max": 1.38972806930542,
      "activations/layer23_attention_weight_min": -1.2889528274536133,
      "activations/layer2_attention_weight_max": 1.943663477897644,
      "activations/layer2_attention_weight_min": -1.509529948234558,
      "activations/layer3_attention_weight_max": 3.058320999145508,
      "activations/layer3_attention_weight_min": -2.261181116104126,
      "activations/layer4_attention_weight_max": 12.11979866027832,
      "activations/layer4_attention_weight_min": -9.345473289489746,
      "activations/layer5_attention_weight_max": 4.3587646484375,
      "activations/layer5_attention_weight_min": -2.7699718475341797,
      "activations/layer6_attention_weight_max": 5.904179096221924,
      "activations/layer6_attention_weight_min": -4.111140251159668,
      "activations/layer7_attention_weight_max": 1.2439258098602295,
      "activations/layer7_attention_weight_min": -1.0289862155914307,
      "activations/layer8_attention_weight_max": 3.405038356781006,
      "activations/layer8_attention_weight_min": -2.124652624130249,
      "activations/layer9_attention_weight_max": 2.2251365184783936,
      "activations/layer9_attention_weight_min": -1.3827711343765259,
      "epoch": 0.01,
      "learning_rate": 9.45089594432832e-05,
      "loss": 6.272,
      "step": 200
    },
    {
      "activations/layer0_attention_weight_max": 7.794164657592773,
      "activations/layer0_attention_weight_min": -3.5676004886627197,
      "activations/layer10_attention_weight_max": 1.0398439168930054,
      "activations/layer10_attention_weight_min": -1.0087403059005737,
      "activations/layer11_attention_weight_max": 2.880354642868042,
      "activations/layer11_attention_weight_min": -2.5442535877227783,
      "activations/layer12_attention_weight_max": 2.2162837982177734,
      "activations/layer12_attention_weight_min": -1.7734206914901733,
      "activations/layer13_attention_weight_max": 1.115546703338623,
      "activations/layer13_attention_weight_min": -1.0971355438232422,
      "activations/layer14_attention_weight_max": 1.6679834127426147,
      "activations/layer14_attention_weight_min": -1.3112751245498657,
      "activations/layer15_attention_weight_max": 1.776437759399414,
      "activations/layer15_attention_weight_min": -1.5731234550476074,
      "activations/layer16_attention_weight_max": 4.284211158752441,
      "activations/layer16_attention_weight_min": -3.8352105617523193,
      "activations/layer17_attention_weight_max": 10.790874481201172,
      "activations/layer17_attention_weight_min": -5.341800689697266,
      "activations/layer18_attention_weight_max": 2.3412764072418213,
      "activations/layer18_attention_weight_min": -1.8114664554595947,
      "activations/layer19_attention_weight_max": 0.9471486210823059,
      "activations/layer19_attention_weight_min": -1.0720903873443604,
      "activations/layer1_attention_weight_max": 4.145844459533691,
      "activations/layer1_attention_weight_min": -2.5873889923095703,
      "activations/layer20_attention_weight_max": 2.214034080505371,
      "activations/layer20_attention_weight_min": -2.2597546577453613,
      "activations/layer21_attention_weight_max": 1.5840319395065308,
      "activations/layer21_attention_weight_min": -1.5609790086746216,
      "activations/layer22_attention_weight_max": 3.1943957805633545,
      "activations/layer22_attention_weight_min": -2.722472667694092,
      "activations/layer23_attention_weight_max": 1.8488572835922241,
      "activations/layer23_attention_weight_min": -1.9281120300292969,
      "activations/layer2_attention_weight_max": 1.754751205444336,
      "activations/layer2_attention_weight_min": -1.3876116275787354,
      "activations/layer3_attention_weight_max": 3.070103406906128,
      "activations/layer3_attention_weight_min": -1.929181694984436,
      "activations/layer4_attention_weight_max": 10.501733779907227,
      "activations/layer4_attention_weight_min": -8.75660228729248,
      "activations/layer5_attention_weight_max": 4.430740833282471,
      "activations/layer5_attention_weight_min": -2.756565570831299,
      "activations/layer6_attention_weight_max": 5.026372909545898,
      "activations/layer6_attention_weight_min": -3.9458487033843994,
      "activations/layer7_attention_weight_max": 1.418135166168213,
      "activations/layer7_attention_weight_min": -1.2106322050094604,
      "activations/layer8_attention_weight_max": 4.00205135345459,
      "activations/layer8_attention_weight_min": -2.3969409465789795,
      "activations/layer9_attention_weight_max": 1.9484379291534424,
      "activations/layer9_attention_weight_min": -1.3446897268295288,
      "epoch": 0.01,
      "learning_rate": 9.88147896765855e-05,
      "loss": 6.1004,
      "step": 250
    },
    {
      "activations/layer0_attention_weight_max": 8.662186622619629,
      "activations/layer0_attention_weight_min": -4.014915943145752,
      "activations/layer10_attention_weight_max": 1.5055618286132812,
      "activations/layer10_attention_weight_min": -1.4915990829467773,
      "activations/layer11_attention_weight_max": 4.200786590576172,
      "activations/layer11_attention_weight_min": -3.854830503463745,
      "activations/layer12_attention_weight_max": 2.3930017948150635,
      "activations/layer12_attention_weight_min": -2.2144951820373535,
      "activations/layer13_attention_weight_max": 1.296647071838379,
      "activations/layer13_attention_weight_min": -1.4888492822647095,
      "activations/layer14_attention_weight_max": 1.8697479963302612,
      "activations/layer14_attention_weight_min": -1.8348605632781982,
      "activations/layer15_attention_weight_max": 1.987438678741455,
      "activations/layer15_attention_weight_min": -2.1415514945983887,
      "activations/layer16_attention_weight_max": 4.576725482940674,
      "activations/layer16_attention_weight_min": -4.73150110244751,
      "activations/layer17_attention_weight_max": 10.95364761352539,
      "activations/layer17_attention_weight_min": -5.638565540313721,
      "activations/layer18_attention_weight_max": 2.508495330810547,
      "activations/layer18_attention_weight_min": -2.058589220046997,
      "activations/layer19_attention_weight_max": 1.1238034963607788,
      "activations/layer19_attention_weight_min": -1.293201208114624,
      "activations/layer1_attention_weight_max": 6.20230770111084,
      "activations/layer1_attention_weight_min": -3.7371673583984375,
      "activations/layer20_attention_weight_max": 2.5430967807769775,
      "activations/layer20_attention_weight_min": -3.001228094100952,
      "activations/layer21_attention_weight_max": 1.8024009466171265,
      "activations/layer21_attention_weight_min": -1.8882578611373901,
      "activations/layer22_attention_weight_max": 3.6234240531921387,
      "activations/layer22_attention_weight_min": -3.3710975646972656,
      "activations/layer23_attention_weight_max": 2.27382230758667,
      "activations/layer23_attention_weight_min": -2.588541030883789,
      "activations/layer2_attention_weight_max": 1.9899343252182007,
      "activations/layer2_attention_weight_min": -1.6116547584533691,
      "activations/layer3_attention_weight_max": 3.0735552310943604,
      "activations/layer3_attention_weight_min": -2.488590717315674,
      "activations/layer4_attention_weight_max": 10.605170249938965,
      "activations/layer4_attention_weight_min": -9.069860458374023,
      "activations/layer5_attention_weight_max": 4.123439788818359,
      "activations/layer5_attention_weight_min": -3.037808418273926,
      "activations/layer6_attention_weight_max": 6.758458137512207,
      "activations/layer6_attention_weight_min": -5.689098834991455,
      "activations/layer7_attention_weight_max": 1.6379677057266235,
      "activations/layer7_attention_weight_min": -1.481164813041687,
      "activations/layer8_attention_weight_max": 4.422539234161377,
      "activations/layer8_attention_weight_min": -2.594597101211548,
      "activations/layer9_attention_weight_max": 2.17856502532959,
      "activations/layer9_attention_weight_min": -1.582688808441162,
      "epoch": 0.02,
      "learning_rate": 0.00010229005231906732,
      "loss": 5.9785,
      "step": 300
    },
    {
      "activations/layer0_attention_weight_max": 9.528251647949219,
      "activations/layer0_attention_weight_min": -4.634669780731201,
      "activations/layer10_attention_weight_max": 2.765221357345581,
      "activations/layer10_attention_weight_min": -2.119596242904663,
      "activations/layer11_attention_weight_max": 9.715044975280762,
      "activations/layer11_attention_weight_min": -6.92982816696167,
      "activations/layer12_attention_weight_max": 2.740903854370117,
      "activations/layer12_attention_weight_min": -2.774505376815796,
      "activations/layer13_attention_weight_max": 2.5601119995117188,
      "activations/layer13_attention_weight_min": -2.153477907180786,
      "activations/layer14_attention_weight_max": 3.4660584926605225,
      "activations/layer14_attention_weight_min": -2.4928784370422363,
      "activations/layer15_attention_weight_max": 4.69410514831543,
      "activations/layer15_attention_weight_min": -3.449099540710449,
      "activations/layer16_attention_weight_max": 5.403378009796143,
      "activations/layer16_attention_weight_min": -5.83054780960083,
      "activations/layer17_attention_weight_max": 11.08220100402832,
      "activations/layer17_attention_weight_min": -5.468169212341309,
      "activations/layer18_attention_weight_max": 2.7044546604156494,
      "activations/layer18_attention_weight_min": -2.2281887531280518,
      "activations/layer19_attention_weight_max": 1.4830572605133057,
      "activations/layer19_attention_weight_min": -1.586300253868103,
      "activations/layer1_attention_weight_max": 9.446074485778809,
      "activations/layer1_attention_weight_min": -7.638803482055664,
      "activations/layer20_attention_weight_max": 2.916053533554077,
      "activations/layer20_attention_weight_min": -3.6239500045776367,
      "activations/layer21_attention_weight_max": 1.9186197519302368,
      "activations/layer21_attention_weight_min": -2.0105385780334473,
      "activations/layer22_attention_weight_max": 3.8502273559570312,
      "activations/layer22_attention_weight_min": -3.7618227005004883,
      "activations/layer23_attention_weight_max": 4.1343584060668945,
      "activations/layer23_attention_weight_min": -3.7128329277038574,
      "activations/layer2_attention_weight_max": 3.3285796642303467,
      "activations/layer2_attention_weight_min": -3.545807361602783,
      "activations/layer3_attention_weight_max": 2.8721773624420166,
      "activations/layer3_attention_weight_min": -2.676539421081543,
      "activations/layer4_attention_weight_max": 13.449902534484863,
      "activations/layer4_attention_weight_min": -13.743733406066895,
      "activations/layer5_attention_weight_max": 4.052948951721191,
      "activations/layer5_attention_weight_min": -3.6286745071411133,
      "activations/layer6_attention_weight_max": 9.046945571899414,
      "activations/layer6_attention_weight_min": -8.811007499694824,
      "activations/layer7_attention_weight_max": 2.7834548950195312,
      "activations/layer7_attention_weight_min": -2.0626003742218018,
      "activations/layer8_attention_weight_max": 5.816594123840332,
      "activations/layer8_attention_weight_min": -3.4452927112579346,
      "activations/layer9_attention_weight_max": 3.141267776489258,
      "activations/layer9_attention_weight_min": -2.3467440605163574,
      "epoch": 0.02,
      "learning_rate": 0.00010520393678084459,
      "loss": 5.875,
      "step": 350
    },
    {
      "activations/layer0_attention_weight_max": 7.6206536293029785,
      "activations/layer0_attention_weight_min": -6.034051895141602,
      "activations/layer10_attention_weight_max": 6.124885082244873,
      "activations/layer10_attention_weight_min": -3.7132301330566406,
      "activations/layer11_attention_weight_max": 10.97359848022461,
      "activations/layer11_attention_weight_min": -8.366109848022461,
      "activations/layer12_attention_weight_max": 2.9997823238372803,
      "activations/layer12_attention_weight_min": -2.9270575046539307,
      "activations/layer13_attention_weight_max": 3.9839696884155273,
      "activations/layer13_attention_weight_min": -2.962982654571533,
      "activations/layer14_attention_weight_max": 6.707509517669678,
      "activations/layer14_attention_weight_min": -3.7918694019317627,
      "activations/layer15_attention_weight_max": 6.541144847869873,
      "activations/layer15_attention_weight_min": -3.986912727355957,
      "activations/layer16_attention_weight_max": 6.102004528045654,
      "activations/layer16_attention_weight_min": -5.844771385192871,
      "activations/layer17_attention_weight_max": 10.406394958496094,
      "activations/layer17_attention_weight_min": -5.3394036293029785,
      "activations/layer18_attention_weight_max": 3.033371686935425,
      "activations/layer18_attention_weight_min": -2.6287972927093506,
      "activations/layer19_attention_weight_max": 2.077008008956909,
      "activations/layer19_attention_weight_min": -1.7792549133300781,
      "activations/layer1_attention_weight_max": 8.970973014831543,
      "activations/layer1_attention_weight_min": -8.885750770568848,
      "activations/layer20_attention_weight_max": 3.5176308155059814,
      "activations/layer20_attention_weight_min": -4.378518104553223,
      "activations/layer21_attention_weight_max": 2.1432747840881348,
      "activations/layer21_attention_weight_min": -2.2649195194244385,
      "activations/layer22_attention_weight_max": 4.086216926574707,
      "activations/layer22_attention_weight_min": -3.907179832458496,
      "activations/layer23_attention_weight_max": 5.497235298156738,
      "activations/layer23_attention_weight_min": -4.128891468048096,
      "activations/layer2_attention_weight_max": 7.129245281219482,
      "activations/layer2_attention_weight_min": -6.518105983734131,
      "activations/layer3_attention_weight_max": 3.7880380153656006,
      "activations/layer3_attention_weight_min": -4.0702738761901855,
      "activations/layer4_attention_weight_max": 15.903398513793945,
      "activations/layer4_attention_weight_min": -21.72988510131836,
      "activations/layer5_attention_weight_max": 5.431901931762695,
      "activations/layer5_attention_weight_min": -7.015369415283203,
      "activations/layer6_attention_weight_max": 16.89824867248535,
      "activations/layer6_attention_weight_min": -16.656295776367188,
      "activations/layer7_attention_weight_max": 3.9755852222442627,
      "activations/layer7_attention_weight_min": -2.8518118858337402,
      "activations/layer8_attention_weight_max": 5.9525370597839355,
      "activations/layer8_attention_weight_min": -4.44265604019165,
      "activations/layer9_attention_weight_max": 8.430089950561523,
      "activations/layer9_attention_weight_min": -5.329278469085693,
      "epoch": 0.02,
      "learning_rate": 0.00010771283560930495,
      "loss": 5.7866,
      "step": 400
    },
    {
      "activations/layer0_attention_weight_max": 9.473158836364746,
      "activations/layer0_attention_weight_min": -6.854032039642334,
      "activations/layer10_attention_weight_max": 10.064343452453613,
      "activations/layer10_attention_weight_min": -8.015098571777344,
      "activations/layer11_attention_weight_max": 16.60988426208496,
      "activations/layer11_attention_weight_min": -15.529105186462402,
      "activations/layer12_attention_weight_max": 4.303391933441162,
      "activations/layer12_attention_weight_min": -4.473808288574219,
      "activations/layer13_attention_weight_max": 6.530306339263916,
      "activations/layer13_attention_weight_min": -3.8401763439178467,
      "activations/layer14_attention_weight_max": 8.523355484008789,
      "activations/layer14_attention_weight_min": -5.242117404937744,
      "activations/layer15_attention_weight_max": 6.7326436042785645,
      "activations/layer15_attention_weight_min": -4.217902660369873,
      "activations/layer16_attention_weight_max": 7.058464050292969,
      "activations/layer16_attention_weight_min": -6.035431861877441,
      "activations/layer17_attention_weight_max": 10.269896507263184,
      "activations/layer17_attention_weight_min": -6.197365760803223,
      "activations/layer18_attention_weight_max": 3.770717144012451,
      "activations/layer18_attention_weight_min": -3.030947208404541,
      "activations/layer19_attention_weight_max": 3.3546340465545654,
      "activations/layer19_attention_weight_min": -1.9890785217285156,
      "activations/layer1_attention_weight_max": 16.067018508911133,
      "activations/layer1_attention_weight_min": -13.182310104370117,
      "activations/layer20_attention_weight_max": 4.8373847007751465,
      "activations/layer20_attention_weight_min": -5.264472484588623,
      "activations/layer21_attention_weight_max": 2.7393553256988525,
      "activations/layer21_attention_weight_min": -2.5150513648986816,
      "activations/layer22_attention_weight_max": 4.1700439453125,
      "activations/layer22_attention_weight_min": -3.741130828857422,
      "activations/layer23_attention_weight_max": 5.900680065155029,
      "activations/layer23_attention_weight_min": -4.4586873054504395,
      "activations/layer2_attention_weight_max": 12.957015037536621,
      "activations/layer2_attention_weight_min": -10.923308372497559,
      "activations/layer3_attention_weight_max": 11.46579647064209,
      "activations/layer3_attention_weight_min": -13.134882926940918,
      "activations/layer4_attention_weight_max": 34.42509841918945,
      "activations/layer4_attention_weight_min": -37.17766571044922,
      "activations/layer5_attention_weight_max": 25.25299835205078,
      "activations/layer5_attention_weight_min": -26.847885131835938,
      "activations/layer6_attention_weight_max": 32.24427795410156,
      "activations/layer6_attention_weight_min": -34.13764190673828,
      "activations/layer7_attention_weight_max": 9.878022193908691,
      "activations/layer7_attention_weight_min": -10.772380828857422,
      "activations/layer8_attention_weight_max": 11.663342475891113,
      "activations/layer8_attention_weight_min": -12.485559463500977,
      "activations/layer9_attention_weight_max": 15.819600105285645,
      "activations/layer9_attention_weight_min": -14.115355491638184,
      "epoch": 0.03,
      "learning_rate": 0.00010991570777374085,
      "loss": 5.7001,
      "step": 450
    },
    {
      "activations/layer0_attention_weight_max": 10.311253547668457,
      "activations/layer0_attention_weight_min": -7.503159523010254,
      "activations/layer10_attention_weight_max": 14.223176956176758,
      "activations/layer10_attention_weight_min": -12.255475044250488,
      "activations/layer11_attention_weight_max": 17.398021697998047,
      "activations/layer11_attention_weight_min": -19.345333099365234,
      "activations/layer12_attention_weight_max": 7.108431339263916,
      "activations/layer12_attention_weight_min": -6.587177276611328,
      "activations/layer13_attention_weight_max": 7.325260639190674,
      "activations/layer13_attention_weight_min": -4.990907192230225,
      "activations/layer14_attention_weight_max": 9.530763626098633,
      "activations/layer14_attention_weight_min": -5.558290004730225,
      "activations/layer15_attention_weight_max": 7.8568267822265625,
      "activations/layer15_attention_weight_min": -5.375546932220459,
      "activations/layer16_attention_weight_max": 6.7526774406433105,
      "activations/layer16_attention_weight_min": -6.222873210906982,
      "activations/layer17_attention_weight_max": 10.404608726501465,
      "activations/layer17_attention_weight_min": -5.819116115570068,
      "activations/layer18_attention_weight_max": 4.246829032897949,
      "activations/layer18_attention_weight_min": -3.001516580581665,
      "activations/layer19_attention_weight_max": 3.4655263423919678,
      "activations/layer19_attention_weight_min": -2.0819430351257324,
      "activations/layer1_attention_weight_max": 15.576374053955078,
      "activations/layer1_attention_weight_min": -14.340909004211426,
      "activations/layer20_attention_weight_max": 5.048481464385986,
      "activations/layer20_attention_weight_min": -5.0061445236206055,
      "activations/layer21_attention_weight_max": 2.7647361755371094,
      "activations/layer21_attention_weight_min": -2.555293560028076,
      "activations/layer22_attention_weight_max": 4.154145240783691,
      "activations/layer22_attention_weight_min": -3.489027738571167,
      "activations/layer23_attention_weight_max": 5.7331390380859375,
      "activations/layer23_attention_weight_min": -4.041703224182129,
      "activations/layer2_attention_weight_max": 15.613753318786621,
      "activations/layer2_attention_weight_min": -14.178728103637695,
      "activations/layer3_attention_weight_max": 18.175289154052734,
      "activations/layer3_attention_weight_min": -19.142484664916992,
      "activations/layer4_attention_weight_max": 58.7555046081543,
      "activations/layer4_attention_weight_min": -63.57655715942383,
      "activations/layer5_attention_weight_max": 58.275962829589844,
      "activations/layer5_attention_weight_min": -63.223541259765625,
      "activations/layer6_attention_weight_max": 45.65190505981445,
      "activations/layer6_attention_weight_min": -53.40430450439453,
      "activations/layer7_attention_weight_max": 24.5752010345459,
      "activations/layer7_attention_weight_min": -26.26990509033203,
      "activations/layer8_attention_weight_max": 33.86506271362305,
      "activations/layer8_attention_weight_min": -36.56904220581055,
      "activations/layer9_attention_weight_max": 28.34406852722168,
      "activations/layer9_attention_weight_min": -28.9138126373291,
      "epoch": 0.03,
      "learning_rate": 0.00011187915841478604,
      "loss": 5.5704,
      "step": 500
    },
    {
      "activations/layer0_attention_weight_max": 12.398470878601074,
      "activations/layer0_attention_weight_min": -7.590048313140869,
      "activations/layer10_attention_weight_max": 18.143543243408203,
      "activations/layer10_attention_weight_min": -17.29739761352539,
      "activations/layer11_attention_weight_max": 22.13541030883789,
      "activations/layer11_attention_weight_min": -23.36122703552246,
      "activations/layer12_attention_weight_max": 11.892438888549805,
      "activations/layer12_attention_weight_min": -10.564926147460938,
      "activations/layer13_attention_weight_max": 7.722334861755371,
      "activations/layer13_attention_weight_min": -6.152440071105957,
      "activations/layer14_attention_weight_max": 9.80295467376709,
      "activations/layer14_attention_weight_min": -5.637521743774414,
      "activations/layer15_attention_weight_max": 7.921204566955566,
      "activations/layer15_attention_weight_min": -6.284313678741455,
      "activations/layer16_attention_weight_max": 7.829885959625244,
      "activations/layer16_attention_weight_min": -6.424705505371094,
      "activations/layer17_attention_weight_max": 9.843437194824219,
      "activations/layer17_attention_weight_min": -6.4754438400268555,
      "activations/layer18_attention_weight_max": 4.720455169677734,
      "activations/layer18_attention_weight_min": -4.035129547119141,
      "activations/layer19_attention_weight_max": 3.8796212673187256,
      "activations/layer19_attention_weight_min": -2.614412546157837,
      "activations/layer1_attention_weight_max": 16.02227210998535,
      "activations/layer1_attention_weight_min": -13.356679916381836,
      "activations/layer20_attention_weight_max": 6.251129150390625,
      "activations/layer20_attention_weight_min": -5.064456462860107,
      "activations/layer21_attention_weight_max": 3.2009029388427734,
      "activations/layer21_attention_weight_min": -2.7414796352386475,
      "activations/layer22_attention_weight_max": 4.547125339508057,
      "activations/layer22_attention_weight_min": -4.138940334320068,
      "activations/layer23_attention_weight_max": 6.162423610687256,
      "activations/layer23_attention_weight_min": -4.482598304748535,
      "activations/layer2_attention_weight_max": 18.559123992919922,
      "activations/layer2_attention_weight_min": -14.843204498291016,
      "activations/layer3_attention_weight_max": 24.333782196044922,
      "activations/layer3_attention_weight_min": -25.026317596435547,
      "activations/layer4_attention_weight_max": 81.7308120727539,
      "activations/layer4_attention_weight_min": -86.64704132080078,
      "activations/layer5_attention_weight_max": 93.64818572998047,
      "activations/layer5_attention_weight_min": -99.50039672851562,
      "activations/layer6_attention_weight_max": 65.86935424804688,
      "activations/layer6_attention_weight_min": -72.60661315917969,
      "activations/layer7_attention_weight_max": 41.882450103759766,
      "activations/layer7_attention_weight_min": -42.012535095214844,
      "activations/layer8_attention_weight_max": 56.108238220214844,
      "activations/layer8_attention_weight_min": -59.05104064941406,
      "activations/layer9_attention_weight_max": 38.46044158935547,
      "activations/layer9_attention_weight_min": -41.1978645324707,
      "epoch": 0.03,
      "learning_rate": 0.00011365016669336267,
      "loss": 5.4467,
      "step": 550
    },
    {
      "activations/layer0_attention_weight_max": 10.79498291015625,
      "activations/layer0_attention_weight_min": -9.117390632629395,
      "activations/layer10_attention_weight_max": 21.58013916015625,
      "activations/layer10_attention_weight_min": -20.928951263427734,
      "activations/layer11_attention_weight_max": 22.830764770507812,
      "activations/layer11_attention_weight_min": -23.991069793701172,
      "activations/layer12_attention_weight_max": 15.414073944091797,
      "activations/layer12_attention_weight_min": -16.113622665405273,
      "activations/layer13_attention_weight_max": 8.543460845947266,
      "activations/layer13_attention_weight_min": -5.874390125274658,
      "activations/layer14_attention_weight_max": 9.087576866149902,
      "activations/layer14_attention_weight_min": -5.356240749359131,
      "activations/layer15_attention_weight_max": 7.775601387023926,
      "activations/layer15_attention_weight_min": -7.470871925354004,
      "activations/layer16_attention_weight_max": 9.7940034866333,
      "activations/layer16_attention_weight_min": -7.368291854858398,
      "activations/layer17_attention_weight_max": 10.469380378723145,
      "activations/layer17_attention_weight_min": -6.773175239562988,
      "activations/layer18_attention_weight_max": 3.7849009037017822,
      "activations/layer18_attention_weight_min": -3.654573678970337,
      "activations/layer19_attention_weight_max": 3.3310964107513428,
      "activations/layer19_attention_weight_min": -2.1415975093841553,
      "activations/layer1_attention_weight_max": 17.11696434020996,
      "activations/layer1_attention_weight_min": -14.83555793762207,
      "activations/layer20_attention_weight_max": 5.980906963348389,
      "activations/layer20_attention_weight_min": -4.755160331726074,
      "activations/layer21_attention_weight_max": 3.4315502643585205,
      "activations/layer21_attention_weight_min": -2.78728985786438,
      "activations/layer22_attention_weight_max": 4.673935890197754,
      "activations/layer22_attention_weight_min": -3.853973150253296,
      "activations/layer23_attention_weight_max": 5.9681196212768555,
      "activations/layer23_attention_weight_min": -4.052714824676514,
      "activations/layer2_attention_weight_max": 19.23576545715332,
      "activations/layer2_attention_weight_min": -15.013895988464355,
      "activations/layer3_attention_weight_max": 29.116186141967773,
      "activations/layer3_attention_weight_min": -28.666528701782227,
      "activations/layer4_attention_weight_max": 98.20500183105469,
      "activations/layer4_attention_weight_min": -108.1594467163086,
      "activations/layer5_attention_weight_max": 114.22904968261719,
      "activations/layer5_attention_weight_min": -132.16925048828125,
      "activations/layer6_attention_weight_max": 83.45419311523438,
      "activations/layer6_attention_weight_min": -96.78291320800781,
      "activations/layer7_attention_weight_max": 56.74108123779297,
      "activations/layer7_attention_weight_min": -62.35886001586914,
      "activations/layer8_attention_weight_max": 71.6820068359375,
      "activations/layer8_attention_weight_min": -77.45392608642578,
      "activations/layer9_attention_weight_max": 47.61417007446289,
      "activations/layer9_attention_weight_min": -52.47127151489258,
      "epoch": 0.03,
      "learning_rate": 0.00011526311149794271,
      "loss": 5.3472,
      "step": 600
    },
    {
      "activations/layer0_attention_weight_max": 11.466835975646973,
      "activations/layer0_attention_weight_min": -7.916305065155029,
      "activations/layer10_attention_weight_max": 23.704753875732422,
      "activations/layer10_attention_weight_min": -23.381895065307617,
      "activations/layer11_attention_weight_max": 25.215627670288086,
      "activations/layer11_attention_weight_min": -23.311399459838867,
      "activations/layer12_attention_weight_max": 17.15167236328125,
      "activations/layer12_attention_weight_min": -14.535419464111328,
      "activations/layer13_attention_weight_max": 7.886971950531006,
      "activations/layer13_attention_weight_min": -5.907167434692383,
      "activations/layer14_attention_weight_max": 9.665602684020996,
      "activations/layer14_attention_weight_min": -4.956082820892334,
      "activations/layer15_attention_weight_max": 9.019020080566406,
      "activations/layer15_attention_weight_min": -6.920917510986328,
      "activations/layer16_attention_weight_max": 12.035118103027344,
      "activations/layer16_attention_weight_min": -7.373156547546387,
      "activations/layer17_attention_weight_max": 9.900260925292969,
      "activations/layer17_attention_weight_min": -6.42446756362915,
      "activations/layer18_attention_weight_max": 6.279181957244873,
      "activations/layer18_attention_weight_min": -3.8583157062530518,
      "activations/layer19_attention_weight_max": 3.4662582874298096,
      "activations/layer19_attention_weight_min": -2.334014654159546,
      "activations/layer1_attention_weight_max": 18.17792320251465,
      "activations/layer1_attention_weight_min": -12.971983909606934,
      "activations/layer20_attention_weight_max": 6.166626930236816,
      "activations/layer20_attention_weight_min": -4.331793308258057,
      "activations/layer21_attention_weight_max": 3.6158366203308105,
      "activations/layer21_attention_weight_min": -2.822284460067749,
      "activations/layer22_attention_weight_max": 5.087156295776367,
      "activations/layer22_attention_weight_min": -3.660703659057617,
      "activations/layer23_attention_weight_max": 6.360903739929199,
      "activations/layer23_attention_weight_min": -3.9182326793670654,
      "activations/layer2_attention_weight_max": 17.125080108642578,
      "activations/layer2_attention_weight_min": -14.06934928894043,
      "activations/layer3_attention_weight_max": 37.01602554321289,
      "activations/layer3_attention_weight_min": -31.952301025390625,
      "activations/layer4_attention_weight_max": 113.56501770019531,
      "activations/layer4_attention_weight_min": -117.94426727294922,
      "activations/layer5_attention_weight_max": 135.78823852539062,
      "activations/layer5_attention_weight_min": -145.87960815429688,
      "activations/layer6_attention_weight_max": 102.65294647216797,
      "activations/layer6_attention_weight_min": -107.97505950927734,
      "activations/layer7_attention_weight_max": 69.8953857421875,
      "activations/layer7_attention_weight_min": -74.02208709716797,
      "activations/layer8_attention_weight_max": 80.01546478271484,
      "activations/layer8_attention_weight_min": -85.21615600585938,
      "activations/layer9_attention_weight_max": 54.78121566772461,
      "activations/layer9_attention_weight_min": -58.368568420410156,
      "epoch": 0.04,
      "learning_rate": 0.00011674390997364552,
      "loss": 5.2436,
      "step": 650
    },
    {
      "activations/layer0_attention_weight_max": 11.937602043151855,
      "activations/layer0_attention_weight_min": -8.471624374389648,
      "activations/layer10_attention_weight_max": 24.10441780090332,
      "activations/layer10_attention_weight_min": -24.347618103027344,
      "activations/layer11_attention_weight_max": 23.24353790283203,
      "activations/layer11_attention_weight_min": -21.717466354370117,
      "activations/layer12_attention_weight_max": 14.613009452819824,
      "activations/layer12_attention_weight_min": -12.234801292419434,
      "activations/layer13_attention_weight_max": 9.107656478881836,
      "activations/layer13_attention_weight_min": -5.682669162750244,
      "activations/layer14_attention_weight_max": 10.575069427490234,
      "activations/layer14_attention_weight_min": -6.09990930557251,
      "activations/layer15_attention_weight_max": 8.28147029876709,
      "activations/layer15_attention_weight_min": -7.394184589385986,
      "activations/layer16_attention_weight_max": 12.307720184326172,
      "activations/layer16_attention_weight_min": -7.791110992431641,
      "activations/layer17_attention_weight_max": 10.95450210571289,
      "activations/layer17_attention_weight_min": -6.379919052124023,
      "activations/layer18_attention_weight_max": 6.0486159324646,
      "activations/layer18_attention_weight_min": -4.079848766326904,
      "activations/layer19_attention_weight_max": 3.6496684551239014,
      "activations/layer19_attention_weight_min": -2.614027738571167,
      "activations/layer1_attention_weight_max": 16.599855422973633,
      "activations/layer1_attention_weight_min": -15.624217987060547,
      "activations/layer20_attention_weight_max": 7.604006767272949,
      "activations/layer20_attention_weight_min": -4.634222984313965,
      "activations/layer21_attention_weight_max": 3.8675808906555176,
      "activations/layer21_attention_weight_min": -2.910829782485962,
      "activations/layer22_attention_weight_max": 4.928873062133789,
      "activations/layer22_attention_weight_min": -3.508877992630005,
      "activations/layer23_attention_weight_max": 5.7370195388793945,
      "activations/layer23_attention_weight_min": -4.1816582679748535,
      "activations/layer2_attention_weight_max": 17.841371536254883,
      "activations/layer2_attention_weight_min": -14.056333541870117,
      "activations/layer3_attention_weight_max": 37.643402099609375,
      "activations/layer3_attention_weight_min": -36.30359649658203,
      "activations/layer4_attention_weight_max": 121.0782241821289,
      "activations/layer4_attention_weight_min": -131.38473510742188,
      "activations/layer5_attention_weight_max": 142.80044555664062,
      "activations/layer5_attention_weight_min": -160.64622497558594,
      "activations/layer6_attention_weight_max": 110.89949798583984,
      "activations/layer6_attention_weight_min": -125.21023559570312,
      "activations/layer7_attention_weight_max": 79.17842864990234,
      "activations/layer7_attention_weight_min": -87.2645492553711,
      "activations/layer8_attention_weight_max": 83.64530181884766,
      "activations/layer8_attention_weight_min": -91.48971557617188,
      "activations/layer9_attention_weight_max": 54.833858489990234,
      "activations/layer9_attention_weight_min": -60.400665283203125,
      "epoch": 0.04,
      "learning_rate": 0.0001181125851263692,
      "loss": 5.1207,
      "step": 700
    },
    {
      "activations/layer0_attention_weight_max": 11.879554748535156,
      "activations/layer0_attention_weight_min": -8.13956069946289,
      "activations/layer10_attention_weight_max": 24.242948532104492,
      "activations/layer10_attention_weight_min": -22.547245025634766,
      "activations/layer11_attention_weight_max": 30.619808197021484,
      "activations/layer11_attention_weight_min": -23.802778244018555,
      "activations/layer12_attention_weight_max": 15.142293930053711,
      "activations/layer12_attention_weight_min": -12.804701805114746,
      "activations/layer13_attention_weight_max": 9.719267845153809,
      "activations/layer13_attention_weight_min": -6.205324649810791,
      "activations/layer14_attention_weight_max": 10.936918258666992,
      "activations/layer14_attention_weight_min": -6.422321319580078,
      "activations/layer15_attention_weight_max": 9.28706169128418,
      "activations/layer15_attention_weight_min": -8.068405151367188,
      "activations/layer16_attention_weight_max": 12.331178665161133,
      "activations/layer16_attention_weight_min": -9.098657608032227,
      "activations/layer17_attention_weight_max": 12.155108451843262,
      "activations/layer17_attention_weight_min": -7.608741760253906,
      "activations/layer18_attention_weight_max": 6.529702186584473,
      "activations/layer18_attention_weight_min": -4.581976890563965,
      "activations/layer19_attention_weight_max": 3.9764297008514404,
      "activations/layer19_attention_weight_min": -3.002470016479492,
      "activations/layer1_attention_weight_max": 18.797021865844727,
      "activations/layer1_attention_weight_min": -12.360353469848633,
      "activations/layer20_attention_weight_max": 5.818281173706055,
      "activations/layer20_attention_weight_min": -4.713034629821777,
      "activations/layer21_attention_weight_max": 4.443727493286133,
      "activations/layer21_attention_weight_min": -2.6572985649108887,
      "activations/layer22_attention_weight_max": 5.335484504699707,
      "activations/layer22_attention_weight_min": -4.090886116027832,
      "activations/layer23_attention_weight_max": 6.810635566711426,
      "activations/layer23_attention_weight_min": -3.9898641109466553,
      "activations/layer2_attention_weight_max": 19.04169273376465,
      "activations/layer2_attention_weight_min": -14.740129470825195,
      "activations/layer3_attention_weight_max": 44.03192138671875,
      "activations/layer3_attention_weight_min": -38.80455017089844,
      "activations/layer4_attention_weight_max": 131.48435974121094,
      "activations/layer4_attention_weight_min": -134.17625427246094,
      "activations/layer5_attention_weight_max": 158.67138671875,
      "activations/layer5_attention_weight_min": -167.7064971923828,
      "activations/layer6_attention_weight_max": 125.04692077636719,
      "activations/layer6_attention_weight_min": -128.88949584960938,
      "activations/layer7_attention_weight_max": 88.05620574951172,
      "activations/layer7_attention_weight_min": -90.60234832763672,
      "activations/layer8_attention_weight_max": 89.66361236572266,
      "activations/layer8_attention_weight_min": -92.64026641845703,
      "activations/layer9_attention_weight_max": 56.583038330078125,
      "activations/layer9_attention_weight_min": -58.24715805053711,
      "epoch": 0.04,
      "learning_rate": 0.00011938492784293858,
      "loss": 5.0528,
      "step": 750
    },
    {
      "activations/layer0_attention_weight_max": 12.356308937072754,
      "activations/layer0_attention_weight_min": -8.056446075439453,
      "activations/layer10_attention_weight_max": 25.131982803344727,
      "activations/layer10_attention_weight_min": -21.739368438720703,
      "activations/layer11_attention_weight_max": 26.73609733581543,
      "activations/layer11_attention_weight_min": -20.2894229888916,
      "activations/layer12_attention_weight_max": 17.461074829101562,
      "activations/layer12_attention_weight_min": -12.9952974319458,
      "activations/layer13_attention_weight_max": 9.104063034057617,
      "activations/layer13_attention_weight_min": -5.741340160369873,
      "activations/layer14_attention_weight_max": 9.99901294708252,
      "activations/layer14_attention_weight_min": -5.317931175231934,
      "activations/layer15_attention_weight_max": 9.512750625610352,
      "activations/layer15_attention_weight_min": -7.867480278015137,
      "activations/layer16_attention_weight_max": 12.124495506286621,
      "activations/layer16_attention_weight_min": -8.668829917907715,
      "activations/layer17_attention_weight_max": 9.466954231262207,
      "activations/layer17_attention_weight_min": -6.768296718597412,
      "activations/layer18_attention_weight_max": 13.443872451782227,
      "activations/layer18_attention_weight_min": -4.576132297515869,
      "activations/layer19_attention_weight_max": 3.833251714706421,
      "activations/layer19_attention_weight_min": -2.7728781700134277,
      "activations/layer1_attention_weight_max": 18.631099700927734,
      "activations/layer1_attention_weight_min": -14.779463768005371,
      "activations/layer20_attention_weight_max": 5.911574363708496,
      "activations/layer20_attention_weight_min": -3.9643678665161133,
      "activations/layer21_attention_weight_max": 4.240455627441406,
      "activations/layer21_attention_weight_min": -3.1109619140625,
      "activations/layer22_attention_weight_max": 5.375754356384277,
      "activations/layer22_attention_weight_min": -4.385553359985352,
      "activations/layer23_attention_weight_max": 6.655093193054199,
      "activations/layer23_attention_weight_min": -3.6250548362731934,
      "activations/layer2_attention_weight_max": 17.89679718017578,
      "activations/layer2_attention_weight_min": -13.491888999938965,
      "activations/layer3_attention_weight_max": 44.232093811035156,
      "activations/layer3_attention_weight_min": -42.38285827636719,
      "activations/layer4_attention_weight_max": 127.15157318115234,
      "activations/layer4_attention_weight_min": -134.25022888183594,
      "activations/layer5_attention_weight_max": 154.78933715820312,
      "activations/layer5_attention_weight_min": -177.3299560546875,
      "activations/layer6_attention_weight_max": 122.3836669921875,
      "activations/layer6_attention_weight_min": -134.98854064941406,
      "activations/layer7_attention_weight_max": 86.97447204589844,
      "activations/layer7_attention_weight_min": -96.38848876953125,
      "activations/layer8_attention_weight_max": 81.86680603027344,
      "activations/layer8_attention_weight_min": -90.96422576904297,
      "activations/layer9_attention_weight_max": 55.94234085083008,
      "activations/layer9_attention_weight_min": -57.35300827026367,
      "epoch": 0.05,
      "learning_rate": 0.00012057361147552512,
      "loss": 4.9782,
      "step": 800
    },
    {
      "activations/layer0_attention_weight_max": 13.205276489257812,
      "activations/layer0_attention_weight_min": -8.850564002990723,
      "activations/layer10_attention_weight_max": 28.33652114868164,
      "activations/layer10_attention_weight_min": -23.28217124938965,
      "activations/layer11_attention_weight_max": 25.056678771972656,
      "activations/layer11_attention_weight_min": -19.9549503326416,
      "activations/layer12_attention_weight_max": 16.498554229736328,
      "activations/layer12_attention_weight_min": -11.862582206726074,
      "activations/layer13_attention_weight_max": 8.825139999389648,
      "activations/layer13_attention_weight_min": -6.275982856750488,
      "activations/layer14_attention_weight_max": 10.309700012207031,
      "activations/layer14_attention_weight_min": -4.9424848556518555,
      "activations/layer15_attention_weight_max": 10.371323585510254,
      "activations/layer15_attention_weight_min": -6.829031944274902,
      "activations/layer16_attention_weight_max": 12.935221672058105,
      "activations/layer16_attention_weight_min": -8.234253883361816,
      "activations/layer17_attention_weight_max": 10.140122413635254,
      "activations/layer17_attention_weight_min": -5.967450141906738,
      "activations/layer18_attention_weight_max": 8.786133766174316,
      "activations/layer18_attention_weight_min": -5.101161479949951,
      "activations/layer19_attention_weight_max": 4.407769203186035,
      "activations/layer19_attention_weight_min": -3.0012362003326416,
      "activations/layer1_attention_weight_max": 19.355201721191406,
      "activations/layer1_attention_weight_min": -14.012358665466309,
      "activations/layer20_attention_weight_max": 5.005923271179199,
      "activations/layer20_attention_weight_min": -4.559403419494629,
      "activations/layer21_attention_weight_max": 4.627974033355713,
      "activations/layer21_attention_weight_min": -3.004831552505493,
      "activations/layer22_attention_weight_max": 5.674736976623535,
      "activations/layer22_attention_weight_min": -3.9474027156829834,
      "activations/layer23_attention_weight_max": 6.765842437744141,
      "activations/layer23_attention_weight_min": -3.884286880493164,
      "activations/layer2_attention_weight_max": 19.06509780883789,
      "activations/layer2_attention_weight_min": -13.613409996032715,
      "activations/layer3_attention_weight_max": 49.541202545166016,
      "activations/layer3_attention_weight_min": -41.9085693359375,
      "activations/layer4_attention_weight_max": 132.0589141845703,
      "activations/layer4_attention_weight_min": -136.7825164794922,
      "activations/layer5_attention_weight_max": 164.07275390625,
      "activations/layer5_attention_weight_min": -176.0132293701172,
      "activations/layer6_attention_weight_max": 134.36795043945312,
      "activations/layer6_attention_weight_min": -141.83277893066406,
      "activations/layer7_attention_weight_max": 97.58668518066406,
      "activations/layer7_attention_weight_min": -95.59308624267578,
      "activations/layer8_attention_weight_max": 88.38311767578125,
      "activations/layer8_attention_weight_min": -88.45025634765625,
      "activations/layer9_attention_weight_max": 61.805843353271484,
      "activations/layer9_attention_weight_min": -56.46212387084961,
      "epoch": 0.05,
      "learning_rate": 0.00012168896205813997,
      "loss": 4.8881,
      "step": 850
    },
    {
      "activations/layer0_attention_weight_max": 12.692328453063965,
      "activations/layer0_attention_weight_min": -8.687980651855469,
      "activations/layer10_attention_weight_max": 29.242834091186523,
      "activations/layer10_attention_weight_min": -25.389455795288086,
      "activations/layer11_attention_weight_max": 30.01881217956543,
      "activations/layer11_attention_weight_min": -23.500158309936523,
      "activations/layer12_attention_weight_max": 20.91122817993164,
      "activations/layer12_attention_weight_min": -14.02214241027832,
      "activations/layer13_attention_weight_max": 10.46628189086914,
      "activations/layer13_attention_weight_min": -5.52949333190918,
      "activations/layer14_attention_weight_max": 10.73595142364502,
      "activations/layer14_attention_weight_min": -5.835901737213135,
      "activations/layer15_attention_weight_max": 11.732135772705078,
      "activations/layer15_attention_weight_min": -9.099979400634766,
      "activations/layer16_attention_weight_max": 14.705150604248047,
      "activations/layer16_attention_weight_min": -9.048494338989258,
      "activations/layer17_attention_weight_max": 11.523313522338867,
      "activations/layer17_attention_weight_min": -6.07580041885376,
      "activations/layer18_attention_weight_max": 13.153626441955566,
      "activations/layer18_attention_weight_min": -5.593629837036133,
      "activations/layer19_attention_weight_max": 5.415008544921875,
      "activations/layer19_attention_weight_min": -3.4313743114471436,
      "activations/layer1_attention_weight_max": 20.38526725769043,
      "activations/layer1_attention_weight_min": -12.640838623046875,
      "activations/layer20_attention_weight_max": 5.661876201629639,
      "activations/layer20_attention_weight_min": -3.968700647354126,
      "activations/layer21_attention_weight_max": 4.326144218444824,
      "activations/layer21_attention_weight_min": -2.7595863342285156,
      "activations/layer22_attention_weight_max": 5.907586097717285,
      "activations/layer22_attention_weight_min": -4.0301432609558105,
      "activations/layer23_attention_weight_max": 6.722929000854492,
      "activations/layer23_attention_weight_min": -3.942272663116455,
      "activations/layer2_attention_weight_max": 19.00958251953125,
      "activations/layer2_attention_weight_min": -14.524798393249512,
      "activations/layer3_attention_weight_max": 51.7343864440918,
      "activations/layer3_attention_weight_min": -46.5776252746582,
      "activations/layer4_attention_weight_max": 133.3911895751953,
      "activations/layer4_attention_weight_min": -139.94635009765625,
      "activations/layer5_attention_weight_max": 170.62689208984375,
      "activations/layer5_attention_weight_min": -181.64315795898438,
      "activations/layer6_attention_weight_max": 137.99864196777344,
      "activations/layer6_attention_weight_min": -148.7233123779297,
      "activations/layer7_attention_weight_max": 97.9241943359375,
      "activations/layer7_attention_weight_min": -106.21546936035156,
      "activations/layer8_attention_weight_max": 93.40116119384766,
      "activations/layer8_attention_weight_min": -97.20343017578125,
      "activations/layer9_attention_weight_max": 64.65203094482422,
      "activations/layer9_attention_weight_min": -62.15595245361328,
      "epoch": 0.05,
      "learning_rate": 0.0001227395044356449,
      "loss": 4.8421,
      "step": 900
    },
    {
      "activations/layer0_attention_weight_max": 12.764649391174316,
      "activations/layer0_attention_weight_min": -9.299858093261719,
      "activations/layer10_attention_weight_max": 28.350400924682617,
      "activations/layer10_attention_weight_min": -22.12668228149414,
      "activations/layer11_attention_weight_max": 31.442001342773438,
      "activations/layer11_attention_weight_min": -18.85668182373047,
      "activations/layer12_attention_weight_max": 16.09602165222168,
      "activations/layer12_attention_weight_min": -13.808738708496094,
      "activations/layer13_attention_weight_max": 10.184688568115234,
      "activations/layer13_attention_weight_min": -5.93681526184082,
      "activations/layer14_attention_weight_max": 12.089803695678711,
      "activations/layer14_attention_weight_min": -6.035016059875488,
      "activations/layer15_attention_weight_max": 11.972122192382812,
      "activations/layer15_attention_weight_min": -7.366903781890869,
      "activations/layer16_attention_weight_max": 11.174391746520996,
      "activations/layer16_attention_weight_min": -8.67784595489502,
      "activations/layer17_attention_weight_max": 11.625874519348145,
      "activations/layer17_attention_weight_min": -6.303761959075928,
      "activations/layer18_attention_weight_max": 12.382505416870117,
      "activations/layer18_attention_weight_min": -5.457592964172363,
      "activations/layer19_attention_weight_max": 4.591909408569336,
      "activations/layer19_attention_weight_min": -3.2341973781585693,
      "activations/layer1_attention_weight_max": 18.294111251831055,
      "activations/layer1_attention_weight_min": -13.084188461303711,
      "activations/layer20_attention_weight_max": 5.925565719604492,
      "activations/layer20_attention_weight_min": -3.894665479660034,
      "activations/layer21_attention_weight_max": 4.401113986968994,
      "activations/layer21_attention_weight_min": -2.7287824153900146,
      "activations/layer22_attention_weight_max": 6.0216965675354,
      "activations/layer22_attention_weight_min": -3.7469887733459473,
      "activations/layer23_attention_weight_max": 6.941377639770508,
      "activations/layer23_attention_weight_min": -3.8903777599334717,
      "activations/layer2_attention_weight_max": 18.943763732910156,
      "activations/layer2_attention_weight_min": -14.42392349243164,
      "activations/layer3_attention_weight_max": 52.307369232177734,
      "activations/layer3_attention_weight_min": -46.5726432800293,
      "activations/layer4_attention_weight_max": 131.99136352539062,
      "activations/layer4_attention_weight_min": -143.09970092773438,
      "activations/layer5_attention_weight_max": 164.79776000976562,
      "activations/layer5_attention_weight_min": -188.14810180664062,
      "activations/layer6_attention_weight_max": 137.03298950195312,
      "activations/layer6_attention_weight_min": -153.4052276611328,
      "activations/layer7_attention_weight_max": 98.5837631225586,
      "activations/layer7_attention_weight_min": -107.0848159790039,
      "activations/layer8_attention_weight_max": 86.56019592285156,
      "activations/layer8_attention_weight_min": -93.00350952148438,
      "activations/layer9_attention_weight_max": 56.28852081298828,
      "activations/layer9_attention_weight_min": -57.3350944519043,
      "epoch": 0.06,
      "learning_rate": 0.0001237323582571438,
      "loss": 4.7505,
      "step": 950
    },
    {
      "activations/layer0_attention_weight_max": 11.428360939025879,
      "activations/layer0_attention_weight_min": -8.507604598999023,
      "activations/layer10_attention_weight_max": 27.491819381713867,
      "activations/layer10_attention_weight_min": -23.294614791870117,
      "activations/layer11_attention_weight_max": 26.719661712646484,
      "activations/layer11_attention_weight_min": -17.96415901184082,
      "activations/layer12_attention_weight_max": 16.299386978149414,
      "activations/layer12_attention_weight_min": -12.898581504821777,
      "activations/layer13_attention_weight_max": 11.60728645324707,
      "activations/layer13_attention_weight_min": -6.302979469299316,
      "activations/layer14_attention_weight_max": 12.42727279663086,
      "activations/layer14_attention_weight_min": -6.2826337814331055,
      "activations/layer15_attention_weight_max": 12.917823791503906,
      "activations/layer15_attention_weight_min": -7.59099006652832,
      "activations/layer16_attention_weight_max": 12.57252025604248,
      "activations/layer16_attention_weight_min": -8.677305221557617,
      "activations/layer17_attention_weight_max": 10.591196060180664,
      "activations/layer17_attention_weight_min": -5.5933518409729,
      "activations/layer18_attention_weight_max": 11.809510231018066,
      "activations/layer18_attention_weight_min": -5.466246128082275,
      "activations/layer19_attention_weight_max": 5.092158794403076,
      "activations/layer19_attention_weight_min": -3.8148865699768066,
      "activations/layer1_attention_weight_max": 18.887094497680664,
      "activations/layer1_attention_weight_min": -13.1312894821167,
      "activations/layer20_attention_weight_max": 6.292405128479004,
      "activations/layer20_attention_weight_min": -4.032434940338135,
      "activations/layer21_attention_weight_max": 4.582860946655273,
      "activations/layer21_attention_weight_min": -2.7233855724334717,
      "activations/layer22_attention_weight_max": 6.018063068389893,
      "activations/layer22_attention_weight_min": -3.9799461364746094,
      "activations/layer23_attention_weight_max": 7.058108806610107,
      "activations/layer23_attention_weight_min": -3.882126808166504,
      "activations/layer2_attention_weight_max": 18.224971771240234,
      "activations/layer2_attention_weight_min": -14.772749900817871,
      "activations/layer3_attention_weight_max": 52.08539962768555,
      "activations/layer3_attention_weight_min": -48.18484115600586,
      "activations/layer4_attention_weight_max": 133.7869415283203,
      "activations/layer4_attention_weight_min": -148.23106384277344,
      "activations/layer5_attention_weight_max": 170.20028686523438,
      "activations/layer5_attention_weight_min": -196.77413940429688,
      "activations/layer6_attention_weight_max": 139.61277770996094,
      "activations/layer6_attention_weight_min": -160.4193115234375,
      "activations/layer7_attention_weight_max": 103.2330551147461,
      "activations/layer7_attention_weight_min": -114.27436065673828,
      "activations/layer8_attention_weight_max": 86.63917541503906,
      "activations/layer8_attention_weight_min": -95.11199951171875,
      "activations/layer9_attention_weight_max": 55.840553283691406,
      "activations/layer9_attention_weight_min": -58.219783782958984,
      "epoch": 0.06,
      "learning_rate": 0.00012467353079136803,
      "loss": 4.7064,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_loss": 4.61328125,
      "eval_runtime": 8.0535,
      "eval_samples_per_second": 533.185,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_openwebtext_loss": 4.61328125,
      "eval_openwebtext_ppl": 100.81440478091984,
      "eval_openwebtext_runtime": 8.0535,
      "eval_openwebtext_samples_per_second": 533.185,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_wikitext_loss": 5.16796875,
      "eval_wikitext_ppl": 175.55787313507827,
      "eval_wikitext_runtime": 1.434,
      "eval_wikitext_samples_per_second": 317.989,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_lambada_loss": 8.5078125,
      "eval_lambada_ppl": 4953.315850060822,
      "eval_lambada_runtime": 9.3688,
      "eval_lambada_samples_per_second": 519.701,
      "step": 1000
    },
    {
      "activations/layer0_attention_weight_max": 12.3285551071167,
      "activations/layer0_attention_weight_min": -8.953996658325195,
      "activations/layer10_attention_weight_max": 30.96438980102539,
      "activations/layer10_attention_weight_min": -24.280887603759766,
      "activations/layer11_attention_weight_max": 27.070289611816406,
      "activations/layer11_attention_weight_min": -19.248851776123047,
      "activations/layer12_attention_weight_max": 18.476062774658203,
      "activations/layer12_attention_weight_min": -15.023755073547363,
      "activations/layer13_attention_weight_max": 11.88180923461914,
      "activations/layer13_attention_weight_min": -8.674560546875,
      "activations/layer14_attention_weight_max": 10.26976203918457,
      "activations/layer14_attention_weight_min": -8.008109092712402,
      "activations/layer15_attention_weight_max": 13.477107048034668,
      "activations/layer15_attention_weight_min": -7.2644267082214355,
      "activations/layer16_attention_weight_max": 13.145781517028809,
      "activations/layer16_attention_weight_min": -8.41585922241211,
      "activations/layer17_attention_weight_max": 10.00036907196045,
      "activations/layer17_attention_weight_min": -6.574371337890625,
      "activations/layer18_attention_weight_max": 11.664502143859863,
      "activations/layer18_attention_weight_min": -6.566011905670166,
      "activations/layer19_attention_weight_max": 4.965137004852295,
      "activations/layer19_attention_weight_min": -4.324231147766113,
      "activations/layer1_attention_weight_max": 17.60589027404785,
      "activations/layer1_attention_weight_min": -13.197868347167969,
      "activations/layer20_attention_weight_max": 6.512622833251953,
      "activations/layer20_attention_weight_min": -4.142072677612305,
      "activations/layer21_attention_weight_max": 4.332154273986816,
      "activations/layer21_attention_weight_min": -3.224742889404297,
      "activations/layer22_attention_weight_max": 6.425971031188965,
      "activations/layer22_attention_weight_min": -3.752441167831421,
      "activations/layer23_attention_weight_max": 6.180291652679443,
      "activations/layer23_attention_weight_min": -3.8516833782196045,
      "activations/layer2_attention_weight_max": 19.194774627685547,
      "activations/layer2_attention_weight_min": -14.368171691894531,
      "activations/layer3_attention_weight_max": 56.631568908691406,
      "activations/layer3_attention_weight_min": -49.37062454223633,
      "activations/layer4_attention_weight_max": 139.66273498535156,
      "activations/layer4_attention_weight_min": -144.63034057617188,
      "activations/layer5_attention_weight_max": 182.98696899414062,
      "activations/layer5_attention_weight_min": -194.6904296875,
      "activations/layer6_attention_weight_max": 150.28250122070312,
      "activations/layer6_attention_weight_min": -155.8528289794922,
      "activations/layer7_attention_weight_max": 108.56034851074219,
      "activations/layer7_attention_weight_min": -107.89334106445312,
      "activations/layer8_attention_weight_max": 88.02591705322266,
      "activations/layer8_attention_weight_min": -89.30760955810547,
      "activations/layer9_attention_weight_max": 63.160274505615234,
      "activations/layer9_attention_weight_min": -56.24675369262695,
      "epoch": 0.06,
      "learning_rate": 0.0001255506719926155,
      "loss": 4.6594,
      "step": 1050
    },
    {
      "activations/layer0_attention_weight_max": 12.58371639251709,
      "activations/layer0_attention_weight_min": -8.955232620239258,
      "activations/layer10_attention_weight_max": 34.586063385009766,
      "activations/layer10_attention_weight_min": -25.706491470336914,
      "activations/layer11_attention_weight_max": 26.520902633666992,
      "activations/layer11_attention_weight_min": -20.07893180847168,
      "activations/layer12_attention_weight_max": 18.563810348510742,
      "activations/layer12_attention_weight_min": -17.065521240234375,
      "activations/layer13_attention_weight_max": 12.040246963500977,
      "activations/layer13_attention_weight_min": -6.778027057647705,
      "activations/layer14_attention_weight_max": 11.70482349395752,
      "activations/layer14_attention_weight_min": -7.32018518447876,
      "activations/layer15_attention_weight_max": 12.774862289428711,
      "activations/layer15_attention_weight_min": -7.114012241363525,
      "activations/layer16_attention_weight_max": 13.577101707458496,
      "activations/layer16_attention_weight_min": -7.82256555557251,
      "activations/layer17_attention_weight_max": 11.691624641418457,
      "activations/layer17_attention_weight_min": -5.861202716827393,
      "activations/layer18_attention_weight_max": 12.79591178894043,
      "activations/layer18_attention_weight_min": -5.84513521194458,
      "activations/layer19_attention_weight_max": 4.627098083496094,
      "activations/layer19_attention_weight_min": -4.3653883934021,
      "activations/layer1_attention_weight_max": 17.549196243286133,
      "activations/layer1_attention_weight_min": -13.607532501220703,
      "activations/layer20_attention_weight_max": 6.124117374420166,
      "activations/layer20_attention_weight_min": -4.301521301269531,
      "activations/layer21_attention_weight_max": 4.522934913635254,
      "activations/layer21_attention_weight_min": -3.6307246685028076,
      "activations/layer22_attention_weight_max": 6.303305625915527,
      "activations/layer22_attention_weight_min": -4.358415603637695,
      "activations/layer23_attention_weight_max": 7.256133079528809,
      "activations/layer23_attention_weight_min": -3.6150922775268555,
      "activations/layer2_attention_weight_max": 18.2807559967041,
      "activations/layer2_attention_weight_min": -16.866535186767578,
      "activations/layer3_attention_weight_max": 55.10178756713867,
      "activations/layer3_attention_weight_min": -53.821067810058594,
      "activations/layer4_attention_weight_max": 139.03602600097656,
      "activations/layer4_attention_weight_min": -156.68508911132812,
      "activations/layer5_attention_weight_max": 172.97848510742188,
      "activations/layer5_attention_weight_min": -200.00283813476562,
      "activations/layer6_attention_weight_max": 142.68276977539062,
      "activations/layer6_attention_weight_min": -167.8059844970703,
      "activations/layer7_attention_weight_max": 105.6367416381836,
      "activations/layer7_attention_weight_min": -113.68083953857422,
      "activations/layer8_attention_weight_max": 87.82146453857422,
      "activations/layer8_attention_weight_min": -90.85537719726562,
      "activations/layer9_attention_weight_max": 55.87825393676758,
      "activations/layer9_attention_weight_min": -56.0015869140625,
      "epoch": 0.06,
      "learning_rate": 0.00012640390827022916,
      "loss": 4.5846,
      "step": 1100
    },
    {
      "activations/layer0_attention_weight_max": 11.423898696899414,
      "activations/layer0_attention_weight_min": -8.62095832824707,
      "activations/layer10_attention_weight_max": 30.495559692382812,
      "activations/layer10_attention_weight_min": -26.295040130615234,
      "activations/layer11_attention_weight_max": 19.759965896606445,
      "activations/layer11_attention_weight_min": -18.249345779418945,
      "activations/layer12_attention_weight_max": 17.74921417236328,
      "activations/layer12_attention_weight_min": -16.165437698364258,
      "activations/layer13_attention_weight_max": 13.613783836364746,
      "activations/layer13_attention_weight_min": -8.750129699707031,
      "activations/layer14_attention_weight_max": 11.355033874511719,
      "activations/layer14_attention_weight_min": -8.553688049316406,
      "activations/layer15_attention_weight_max": 12.178502082824707,
      "activations/layer15_attention_weight_min": -8.174430847167969,
      "activations/layer16_attention_weight_max": 10.67488956451416,
      "activations/layer16_attention_weight_min": -8.545469284057617,
      "activations/layer17_attention_weight_max": 9.580381393432617,
      "activations/layer17_attention_weight_min": -8.151365280151367,
      "activations/layer18_attention_weight_max": 11.737456321716309,
      "activations/layer18_attention_weight_min": -7.234676361083984,
      "activations/layer19_attention_weight_max": 4.680693626403809,
      "activations/layer19_attention_weight_min": -4.960750102996826,
      "activations/layer1_attention_weight_max": 16.379438400268555,
      "activations/layer1_attention_weight_min": -12.18242359161377,
      "activations/layer20_attention_weight_max": 5.066810607910156,
      "activations/layer20_attention_weight_min": -5.506798267364502,
      "activations/layer21_attention_weight_max": 5.02694845199585,
      "activations/layer21_attention_weight_min": -4.376893520355225,
      "activations/layer22_attention_weight_max": 5.956439018249512,
      "activations/layer22_attention_weight_min": -4.295236587524414,
      "activations/layer23_attention_weight_max": 6.9019775390625,
      "activations/layer23_attention_weight_min": -3.8857994079589844,
      "activations/layer2_attention_weight_max": 17.52051544189453,
      "activations/layer2_attention_weight_min": -15.133533477783203,
      "activations/layer3_attention_weight_max": 55.05469512939453,
      "activations/layer3_attention_weight_min": -49.74098205566406,
      "activations/layer4_attention_weight_max": 137.00682067871094,
      "activations/layer4_attention_weight_min": -151.34388732910156,
      "activations/layer5_attention_weight_max": 169.5527801513672,
      "activations/layer5_attention_weight_min": -192.37811279296875,
      "activations/layer6_attention_weight_max": 147.2290802001953,
      "activations/layer6_attention_weight_min": -156.90322875976562,
      "activations/layer7_attention_weight_max": 105.75662231445312,
      "activations/layer7_attention_weight_min": -109.94225311279297,
      "activations/layer8_attention_weight_max": 85.53128051757812,
      "activations/layer8_attention_weight_min": -84.8357162475586,
      "activations/layer9_attention_weight_max": 54.3123779296875,
      "activations/layer9_attention_weight_min": -50.73960494995117,
      "epoch": 0.07,
      "learning_rate": 0.00012721869717123439,
      "loss": 4.5401,
      "step": 1150
    },
    {
      "activations/layer0_attention_weight_max": 12.054183959960938,
      "activations/layer0_attention_weight_min": -9.024470329284668,
      "activations/layer10_attention_weight_max": 33.323238372802734,
      "activations/layer10_attention_weight_min": -24.73064613342285,
      "activations/layer11_attention_weight_max": 24.726041793823242,
      "activations/layer11_attention_weight_min": -17.38020896911621,
      "activations/layer12_attention_weight_max": 17.607402801513672,
      "activations/layer12_attention_weight_min": -17.723941802978516,
      "activations/layer13_attention_weight_max": 13.388843536376953,
      "activations/layer13_attention_weight_min": -9.19552230834961,
      "activations/layer14_attention_weight_max": 10.8548002243042,
      "activations/layer14_attention_weight_min": -8.647924423217773,
      "activations/layer15_attention_weight_max": 13.480815887451172,
      "activations/layer15_attention_weight_min": -9.090088844299316,
      "activations/layer16_attention_weight_max": 12.500097274780273,
      "activations/layer16_attention_weight_min": -8.575870513916016,
      "activations/layer17_attention_weight_max": 11.17212963104248,
      "activations/layer17_attention_weight_min": -6.60564661026001,
      "activations/layer18_attention_weight_max": 14.118584632873535,
      "activations/layer18_attention_weight_min": -7.504929542541504,
      "activations/layer19_attention_weight_max": 5.489232540130615,
      "activations/layer19_attention_weight_min": -4.796963214874268,
      "activations/layer1_attention_weight_max": 15.622138023376465,
      "activations/layer1_attention_weight_min": -12.089715003967285,
      "activations/layer20_attention_weight_max": 6.180902004241943,
      "activations/layer20_attention_weight_min": -4.18787956237793,
      "activations/layer21_attention_weight_max": 7.365537166595459,
      "activations/layer21_attention_weight_min": -5.0527472496032715,
      "activations/layer22_attention_weight_max": 5.599186420440674,
      "activations/layer22_attention_weight_min": -3.778630495071411,
      "activations/layer23_attention_weight_max": 6.883040904998779,
      "activations/layer23_attention_weight_min": -3.6096949577331543,
      "activations/layer2_attention_weight_max": 19.266324996948242,
      "activations/layer2_attention_weight_min": -15.666129112243652,
      "activations/layer3_attention_weight_max": 59.52519989013672,
      "activations/layer3_attention_weight_min": -51.896461486816406,
      "activations/layer4_attention_weight_max": 156.09771728515625,
      "activations/layer4_attention_weight_min": -160.62948608398438,
      "activations/layer5_attention_weight_max": 196.179931640625,
      "activations/layer5_attention_weight_min": -205.16844177246094,
      "activations/layer6_attention_weight_max": 152.83811950683594,
      "activations/layer6_attention_weight_min": -158.4014434814453,
      "activations/layer7_attention_weight_max": 114.92353820800781,
      "activations/layer7_attention_weight_min": -114.27826690673828,
      "activations/layer8_attention_weight_max": 90.72262573242188,
      "activations/layer8_attention_weight_min": -88.8509292602539,
      "activations/layer9_attention_weight_max": 58.29639434814453,
      "activations/layer9_attention_weight_min": -52.0461311340332,
      "epoch": 0.07,
      "learning_rate": 0.0001279983547364355,
      "loss": 4.4756,
      "step": 1200
    },
    {
      "activations/layer0_attention_weight_max": 13.171306610107422,
      "activations/layer0_attention_weight_min": -8.650096893310547,
      "activations/layer10_attention_weight_max": 36.015541076660156,
      "activations/layer10_attention_weight_min": -24.94930076599121,
      "activations/layer11_attention_weight_max": 27.433216094970703,
      "activations/layer11_attention_weight_min": -15.795890808105469,
      "activations/layer12_attention_weight_max": 20.591999053955078,
      "activations/layer12_attention_weight_min": -19.38365936279297,
      "activations/layer13_attention_weight_max": 13.533792495727539,
      "activations/layer13_attention_weight_min": -8.567567825317383,
      "activations/layer14_attention_weight_max": 11.540277481079102,
      "activations/layer14_attention_weight_min": -9.395855903625488,
      "activations/layer15_attention_weight_max": 13.925780296325684,
      "activations/layer15_attention_weight_min": -8.085261344909668,
      "activations/layer16_attention_weight_max": 17.724952697753906,
      "activations/layer16_attention_weight_min": -8.237296104431152,
      "activations/layer17_attention_weight_max": 13.10562515258789,
      "activations/layer17_attention_weight_min": -6.857481479644775,
      "activations/layer18_attention_weight_max": 14.61122989654541,
      "activations/layer18_attention_weight_min": -7.364081859588623,
      "activations/layer19_attention_weight_max": 5.638190746307373,
      "activations/layer19_attention_weight_min": -5.01857852935791,
      "activations/layer1_attention_weight_max": 17.721290588378906,
      "activations/layer1_attention_weight_min": -14.026561737060547,
      "activations/layer20_attention_weight_max": 6.505087852478027,
      "activations/layer20_attention_weight_min": -4.838364124298096,
      "activations/layer21_attention_weight_max": 7.90701961517334,
      "activations/layer21_attention_weight_min": -5.744621753692627,
      "activations/layer22_attention_weight_max": 7.0330023765563965,
      "activations/layer22_attention_weight_min": -4.602507591247559,
      "activations/layer23_attention_weight_max": 7.030150890350342,
      "activations/layer23_attention_weight_min": -4.183252811431885,
      "activations/layer2_attention_weight_max": 19.907245635986328,
      "activations/layer2_attention_weight_min": -16.375709533691406,
      "activations/layer3_attention_weight_max": 58.61558532714844,
      "activations/layer3_attention_weight_min": -52.50056076049805,
      "activations/layer4_attention_weight_max": 154.54017639160156,
      "activations/layer4_attention_weight_min": -161.10923767089844,
      "activations/layer5_attention_weight_max": 193.30735778808594,
      "activations/layer5_attention_weight_min": -206.8271942138672,
      "activations/layer6_attention_weight_max": 150.7830047607422,
      "activations/layer6_attention_weight_min": -157.15745544433594,
      "activations/layer7_attention_weight_max": 111.12970733642578,
      "activations/layer7_attention_weight_min": -113.5290298461914,
      "activations/layer8_attention_weight_max": 89.68212127685547,
      "activations/layer8_attention_weight_min": -88.49951171875,
      "activations/layer9_attention_weight_max": 57.899688720703125,
      "activations/layer9_attention_weight_min": -49.77585983276367,
      "epoch": 0.07,
      "learning_rate": 0.0001287457856770406,
      "loss": 4.4257,
      "step": 1250
    },
    {
      "activations/layer0_attention_weight_max": 12.309895515441895,
      "activations/layer0_attention_weight_min": -9.038019180297852,
      "activations/layer10_attention_weight_max": 26.30084228515625,
      "activations/layer10_attention_weight_min": -27.955522537231445,
      "activations/layer11_attention_weight_max": 24.319019317626953,
      "activations/layer11_attention_weight_min": -17.76446533203125,
      "activations/layer12_attention_weight_max": 16.827423095703125,
      "activations/layer12_attention_weight_min": -18.854122161865234,
      "activations/layer13_attention_weight_max": 13.73282241821289,
      "activations/layer13_attention_weight_min": -8.571123123168945,
      "activations/layer14_attention_weight_max": 12.139365196228027,
      "activations/layer14_attention_weight_min": -8.116874694824219,
      "activations/layer15_attention_weight_max": 12.537988662719727,
      "activations/layer15_attention_weight_min": -7.874493598937988,
      "activations/layer16_attention_weight_max": 11.81472396850586,
      "activations/layer16_attention_weight_min": -9.723794937133789,
      "activations/layer17_attention_weight_max": 10.570343971252441,
      "activations/layer17_attention_weight_min": -7.055929660797119,
      "activations/layer18_attention_weight_max": 9.64889144897461,
      "activations/layer18_attention_weight_min": -7.193841457366943,
      "activations/layer19_attention_weight_max": 5.2283220291137695,
      "activations/layer19_attention_weight_min": -5.520119667053223,
      "activations/layer1_attention_weight_max": 17.84398078918457,
      "activations/layer1_attention_weight_min": -11.842716217041016,
      "activations/layer20_attention_weight_max": 6.655030727386475,
      "activations/layer20_attention_weight_min": -4.748116970062256,
      "activations/layer21_attention_weight_max": 8.482205390930176,
      "activations/layer21_attention_weight_min": -6.641587257385254,
      "activations/layer22_attention_weight_max": 5.994919776916504,
      "activations/layer22_attention_weight_min": -4.260828971862793,
      "activations/layer23_attention_weight_max": 7.424705505371094,
      "activations/layer23_attention_weight_min": -4.15148401260376,
      "activations/layer2_attention_weight_max": 19.25615692138672,
      "activations/layer2_attention_weight_min": -15.38923168182373,
      "activations/layer3_attention_weight_max": 56.42483901977539,
      "activations/layer3_attention_weight_min": -54.653072357177734,
      "activations/layer4_attention_weight_max": 152.1627655029297,
      "activations/layer4_attention_weight_min": -162.3546142578125,
      "activations/layer5_attention_weight_max": 186.1156463623047,
      "activations/layer5_attention_weight_min": -209.0834197998047,
      "activations/layer6_attention_weight_max": 153.49815368652344,
      "activations/layer6_attention_weight_min": -164.1317901611328,
      "activations/layer7_attention_weight_max": 113.87095642089844,
      "activations/layer7_attention_weight_min": -118.53858947753906,
      "activations/layer8_attention_weight_max": 83.10060119628906,
      "activations/layer8_attention_weight_min": -89.13560485839844,
      "activations/layer9_attention_weight_max": 52.090858459472656,
      "activations/layer9_attention_weight_min": -50.12733840942383,
      "epoch": 0.08,
      "learning_rate": 0.00012946354871179542,
      "loss": 4.3821,
      "step": 1300
    },
    {
      "activations/layer0_attention_weight_max": 12.44751262664795,
      "activations/layer0_attention_weight_min": -8.896771430969238,
      "activations/layer10_attention_weight_max": 33.02177810668945,
      "activations/layer10_attention_weight_min": -28.34030532836914,
      "activations/layer11_attention_weight_max": 25.349788665771484,
      "activations/layer11_attention_weight_min": -16.444332122802734,
      "activations/layer12_attention_weight_max": 19.630294799804688,
      "activations/layer12_attention_weight_min": -18.227764129638672,
      "activations/layer13_attention_weight_max": 15.872041702270508,
      "activations/layer13_attention_weight_min": -8.182591438293457,
      "activations/layer14_attention_weight_max": 14.056644439697266,
      "activations/layer14_attention_weight_min": -9.433187484741211,
      "activations/layer15_attention_weight_max": 14.430262565612793,
      "activations/layer15_attention_weight_min": -8.068124771118164,
      "activations/layer16_attention_weight_max": 17.182615280151367,
      "activations/layer16_attention_weight_min": -8.281060218811035,
      "activations/layer17_attention_weight_max": 12.417374610900879,
      "activations/layer17_attention_weight_min": -7.320688724517822,
      "activations/layer18_attention_weight_max": 10.344062805175781,
      "activations/layer18_attention_weight_min": -7.355686664581299,
      "activations/layer19_attention_weight_max": 6.306626796722412,
      "activations/layer19_attention_weight_min": -5.353870391845703,
      "activations/layer1_attention_weight_max": 17.50579833984375,
      "activations/layer1_attention_weight_min": -11.81466007232666,
      "activations/layer20_attention_weight_max": 5.919763565063477,
      "activations/layer20_attention_weight_min": -5.175825119018555,
      "activations/layer21_attention_weight_max": 9.100996017456055,
      "activations/layer21_attention_weight_min": -6.684940814971924,
      "activations/layer22_attention_weight_max": 6.7829437255859375,
      "activations/layer22_attention_weight_min": -4.453554153442383,
      "activations/layer23_attention_weight_max": 6.629924297332764,
      "activations/layer23_attention_weight_min": -4.0080718994140625,
      "activations/layer2_attention_weight_max": 19.247177124023438,
      "activations/layer2_attention_weight_min": -16.796398162841797,
      "activations/layer3_attention_weight_max": 61.28425979614258,
      "activations/layer3_attention_weight_min": -57.27466583251953,
      "activations/layer4_attention_weight_max": 156.37193298339844,
      "activations/layer4_attention_weight_min": -163.83718872070312,
      "activations/layer5_attention_weight_max": 192.8134765625,
      "activations/layer5_attention_weight_min": -202.77743530273438,
      "activations/layer6_attention_weight_max": 160.3544464111328,
      "activations/layer6_attention_weight_min": -166.45333862304688,
      "activations/layer7_attention_weight_max": 120.84803009033203,
      "activations/layer7_attention_weight_min": -119.01974487304688,
      "activations/layer8_attention_weight_max": 90.44425201416016,
      "activations/layer8_attention_weight_min": -89.61882019042969,
      "activations/layer9_attention_weight_max": 57.585445404052734,
      "activations/layer9_attention_weight_min": -55.33574295043945,
      "epoch": 0.08,
      "learning_rate": 0.0001301539094251036,
      "loss": 4.3172,
      "step": 1350
    },
    {
      "activations/layer0_attention_weight_max": 11.974580764770508,
      "activations/layer0_attention_weight_min": -8.956799507141113,
      "activations/layer10_attention_weight_max": 36.07300567626953,
      "activations/layer10_attention_weight_min": -25.273094177246094,
      "activations/layer11_attention_weight_max": 27.8112850189209,
      "activations/layer11_attention_weight_min": -17.317800521850586,
      "activations/layer12_attention_weight_max": 24.04633140563965,
      "activations/layer12_attention_weight_min": -19.96551513671875,
      "activations/layer13_attention_weight_max": 16.146514892578125,
      "activations/layer13_attention_weight_min": -11.662863731384277,
      "activations/layer14_attention_weight_max": 11.618724822998047,
      "activations/layer14_attention_weight_min": -10.327783584594727,
      "activations/layer15_attention_weight_max": 16.19000244140625,
      "activations/layer15_attention_weight_min": -9.858382225036621,
      "activations/layer16_attention_weight_max": 17.4964656829834,
      "activations/layer16_attention_weight_min": -9.131309509277344,
      "activations/layer17_attention_weight_max": 11.758099555969238,
      "activations/layer17_attention_weight_min": -7.68044900894165,
      "activations/layer18_attention_weight_max": 10.896683692932129,
      "activations/layer18_attention_weight_min": -7.727293491363525,
      "activations/layer19_attention_weight_max": 6.847250461578369,
      "activations/layer19_attention_weight_min": -5.874014377593994,
      "activations/layer1_attention_weight_max": 18.093339920043945,
      "activations/layer1_attention_weight_min": -11.3096284866333,
      "activations/layer20_attention_weight_max": 6.393719673156738,
      "activations/layer20_attention_weight_min": -4.931920051574707,
      "activations/layer21_attention_weight_max": 9.977357864379883,
      "activations/layer21_attention_weight_min": -7.588622093200684,
      "activations/layer22_attention_weight_max": 6.266363620758057,
      "activations/layer22_attention_weight_min": -4.367433547973633,
      "activations/layer23_attention_weight_max": 7.96309757232666,
      "activations/layer23_attention_weight_min": -3.3297183513641357,
      "activations/layer2_attention_weight_max": 18.639005661010742,
      "activations/layer2_attention_weight_min": -16.056121826171875,
      "activations/layer3_attention_weight_max": 60.34043884277344,
      "activations/layer3_attention_weight_min": -56.35186767578125,
      "activations/layer4_attention_weight_max": 149.9651336669922,
      "activations/layer4_attention_weight_min": -163.07162475585938,
      "activations/layer5_attention_weight_max": 184.85366821289062,
      "activations/layer5_attention_weight_min": -204.514892578125,
      "activations/layer6_attention_weight_max": 148.67022705078125,
      "activations/layer6_attention_weight_min": -156.08013916015625,
      "activations/layer7_attention_weight_max": 111.42618560791016,
      "activations/layer7_attention_weight_min": -111.9698715209961,
      "activations/layer8_attention_weight_max": 80.97654724121094,
      "activations/layer8_attention_weight_min": -83.69654083251953,
      "activations/layer9_attention_weight_max": 54.305049896240234,
      "activations/layer9_attention_weight_min": -49.11701583862305,
      "epoch": 0.08,
      "learning_rate": 0.0001308188834013138,
      "loss": 4.2591,
      "step": 1400
    },
    {
      "activations/layer0_attention_weight_max": 11.846546173095703,
      "activations/layer0_attention_weight_min": -8.856575965881348,
      "activations/layer10_attention_weight_max": 35.19044876098633,
      "activations/layer10_attention_weight_min": -25.77663230895996,
      "activations/layer11_attention_weight_max": 30.15213394165039,
      "activations/layer11_attention_weight_min": -16.564983367919922,
      "activations/layer12_attention_weight_max": 23.844499588012695,
      "activations/layer12_attention_weight_min": -23.25454330444336,
      "activations/layer13_attention_weight_max": 17.013397216796875,
      "activations/layer13_attention_weight_min": -11.672829627990723,
      "activations/layer14_attention_weight_max": 12.617599487304688,
      "activations/layer14_attention_weight_min": -9.616034507751465,
      "activations/layer15_attention_weight_max": 17.317256927490234,
      "activations/layer15_attention_weight_min": -9.5414457321167,
      "activations/layer16_attention_weight_max": 16.052581787109375,
      "activations/layer16_attention_weight_min": -8.904335975646973,
      "activations/layer17_attention_weight_max": 11.404149055480957,
      "activations/layer17_attention_weight_min": -6.706099033355713,
      "activations/layer18_attention_weight_max": 12.920777320861816,
      "activations/layer18_attention_weight_min": -7.183493614196777,
      "activations/layer19_attention_weight_max": 8.101214408874512,
      "activations/layer19_attention_weight_min": -7.268458843231201,
      "activations/layer1_attention_weight_max": 15.518820762634277,
      "activations/layer1_attention_weight_min": -12.05685043334961,
      "activations/layer20_attention_weight_max": 6.746155738830566,
      "activations/layer20_attention_weight_min": -4.532533168792725,
      "activations/layer21_attention_weight_max": 13.710047721862793,
      "activations/layer21_attention_weight_min": -7.715117931365967,
      "activations/layer22_attention_weight_max": 7.365633964538574,
      "activations/layer22_attention_weight_min": -4.473280429840088,
      "activations/layer23_attention_weight_max": 7.077296733856201,
      "activations/layer23_attention_weight_min": -3.9619364738464355,
      "activations/layer2_attention_weight_max": 19.79645538330078,
      "activations/layer2_attention_weight_min": -16.593379974365234,
      "activations/layer3_attention_weight_max": 62.15187072753906,
      "activations/layer3_attention_weight_min": -54.78144836425781,
      "activations/layer4_attention_weight_max": 160.25743103027344,
      "activations/layer4_attention_weight_min": -159.2201385498047,
      "activations/layer5_attention_weight_max": 195.4031982421875,
      "activations/layer5_attention_weight_min": -200.2183837890625,
      "activations/layer6_attention_weight_max": 155.8997039794922,
      "activations/layer6_attention_weight_min": -156.7642364501953,
      "activations/layer7_attention_weight_max": 120.39234161376953,
      "activations/layer7_attention_weight_min": -116.01008605957031,
      "activations/layer8_attention_weight_max": 90.15231323242188,
      "activations/layer8_attention_weight_min": -84.66156768798828,
      "activations/layer9_attention_weight_max": 60.122711181640625,
      "activations/layer9_attention_weight_min": -49.12202835083008,
      "epoch": 0.08,
      "learning_rate": 0.00013146027170578227,
      "loss": 4.2176,
      "step": 1450
    },
    {
      "activations/layer0_attention_weight_max": 11.77742862701416,
      "activations/layer0_attention_weight_min": -9.120384216308594,
      "activations/layer10_attention_weight_max": 38.350303649902344,
      "activations/layer10_attention_weight_min": -26.786924362182617,
      "activations/layer11_attention_weight_max": 28.976974487304688,
      "activations/layer11_attention_weight_min": -18.005210876464844,
      "activations/layer12_attention_weight_max": 20.129819869995117,
      "activations/layer12_attention_weight_min": -21.91675567626953,
      "activations/layer13_attention_weight_max": 17.075361251831055,
      "activations/layer13_attention_weight_min": -10.60750961303711,
      "activations/layer14_attention_weight_max": 15.511276245117188,
      "activations/layer14_attention_weight_min": -8.632514953613281,
      "activations/layer15_attention_weight_max": 17.923757553100586,
      "activations/layer15_attention_weight_min": -10.490761756896973,
      "activations/layer16_attention_weight_max": 13.14682674407959,
      "activations/layer16_attention_weight_min": -10.207292556762695,
      "activations/layer17_attention_weight_max": 11.09465503692627,
      "activations/layer17_attention_weight_min": -7.199210166931152,
      "activations/layer18_attention_weight_max": 15.201117515563965,
      "activations/layer18_attention_weight_min": -7.7865214347839355,
      "activations/layer19_attention_weight_max": 8.156718254089355,
      "activations/layer19_attention_weight_min": -6.466959476470947,
      "activations/layer1_attention_weight_max": 18.072154998779297,
      "activations/layer1_attention_weight_min": -11.168548583984375,
      "activations/layer20_attention_weight_max": 6.690770626068115,
      "activations/layer20_attention_weight_min": -5.275120735168457,
      "activations/layer21_attention_weight_max": 12.980979919433594,
      "activations/layer21_attention_weight_min": -6.742329120635986,
      "activations/layer22_attention_weight_max": 7.0368781089782715,
      "activations/layer22_attention_weight_min": -4.1341776847839355,
      "activations/layer23_attention_weight_max": 7.257916450500488,
      "activations/layer23_attention_weight_min": -4.329096794128418,
      "activations/layer2_attention_weight_max": 19.85122299194336,
      "activations/layer2_attention_weight_min": -17.159202575683594,
      "activations/layer3_attention_weight_max": 63.45228576660156,
      "activations/layer3_attention_weight_min": -56.80757522583008,
      "activations/layer4_attention_weight_max": 162.7505340576172,
      "activations/layer4_attention_weight_min": -159.67733764648438,
      "activations/layer5_attention_weight_max": 205.8194580078125,
      "activations/layer5_attention_weight_min": -208.23263549804688,
      "activations/layer6_attention_weight_max": 158.0881805419922,
      "activations/layer6_attention_weight_min": -159.466552734375,
      "activations/layer7_attention_weight_max": 126.40951538085938,
      "activations/layer7_attention_weight_min": -117.18614196777344,
      "activations/layer8_attention_weight_max": 95.71485137939453,
      "activations/layer8_attention_weight_min": -87.8517837524414,
      "activations/layer9_attention_weight_max": 63.58736038208008,
      "activations/layer9_attention_weight_min": -49.00802230834961,
      "epoch": 0.09,
      "learning_rate": 0.00013207969028649846,
      "loss": 4.1884,
      "step": 1500
    },
    {
      "activations/layer0_attention_weight_max": 13.290587425231934,
      "activations/layer0_attention_weight_min": -9.263803482055664,
      "activations/layer10_attention_weight_max": 31.408384323120117,
      "activations/layer10_attention_weight_min": -26.21773910522461,
      "activations/layer11_attention_weight_max": 23.957670211791992,
      "activations/layer11_attention_weight_min": -15.569976806640625,
      "activations/layer12_attention_weight_max": 18.649192810058594,
      "activations/layer12_attention_weight_min": -23.3316650390625,
      "activations/layer13_attention_weight_max": 17.497020721435547,
      "activations/layer13_attention_weight_min": -11.578268051147461,
      "activations/layer14_attention_weight_max": 11.978789329528809,
      "activations/layer14_attention_weight_min": -12.160026550292969,
      "activations/layer15_attention_weight_max": 16.205116271972656,
      "activations/layer15_attention_weight_min": -10.604228019714355,
      "activations/layer16_attention_weight_max": 14.760503768920898,
      "activations/layer16_attention_weight_min": -10.156143188476562,
      "activations/layer17_attention_weight_max": 11.333104133605957,
      "activations/layer17_attention_weight_min": -7.097525119781494,
      "activations/layer18_attention_weight_max": 14.788139343261719,
      "activations/layer18_attention_weight_min": -7.929518222808838,
      "activations/layer19_attention_weight_max": 9.946005821228027,
      "activations/layer19_attention_weight_min": -7.402388095855713,
      "activations/layer1_attention_weight_max": 15.292427062988281,
      "activations/layer1_attention_weight_min": -10.476009368896484,
      "activations/layer20_attention_weight_max": 6.917688369750977,
      "activations/layer20_attention_weight_min": -5.580428600311279,
      "activations/layer21_attention_weight_max": 14.405876159667969,
      "activations/layer21_attention_weight_min": -8.847061157226562,
      "activations/layer22_attention_weight_max": 8.463985443115234,
      "activations/layer22_attention_weight_min": -4.635809898376465,
      "activations/layer23_attention_weight_max": 7.166901588439941,
      "activations/layer23_attention_weight_min": -4.280709266662598,
      "activations/layer2_attention_weight_max": 18.263938903808594,
      "activations/layer2_attention_weight_min": -16.48236083984375,
      "activations/layer3_attention_weight_max": 61.8857536315918,
      "activations/layer3_attention_weight_min": -55.77780532836914,
      "activations/layer4_attention_weight_max": 152.25022888183594,
      "activations/layer4_attention_weight_min": -162.75482177734375,
      "activations/layer5_attention_weight_max": 187.3665008544922,
      "activations/layer5_attention_weight_min": -198.17430114746094,
      "activations/layer6_attention_weight_max": 147.05270385742188,
      "activations/layer6_attention_weight_min": -152.7542266845703,
      "activations/layer7_attention_weight_max": 112.19740295410156,
      "activations/layer7_attention_weight_min": -113.26222229003906,
      "activations/layer8_attention_weight_max": 80.39501190185547,
      "activations/layer8_attention_weight_min": -83.54924011230469,
      "activations/layer9_attention_weight_max": 53.6716423034668,
      "activations/layer9_attention_weight_min": -48.6082763671875,
      "epoch": 0.09,
      "learning_rate": 0.00013267859450497062,
      "loss": 4.1572,
      "step": 1550
    },
    {
      "activations/layer0_attention_weight_max": 13.094820976257324,
      "activations/layer0_attention_weight_min": -8.504049301147461,
      "activations/layer10_attention_weight_max": 35.616432189941406,
      "activations/layer10_attention_weight_min": -25.337514877319336,
      "activations/layer11_attention_weight_max": 29.03643798828125,
      "activations/layer11_attention_weight_min": -15.910490989685059,
      "activations/layer12_attention_weight_max": 18.11506462097168,
      "activations/layer12_attention_weight_min": -20.44858741760254,
      "activations/layer13_attention_weight_max": 16.97049331665039,
      "activations/layer13_attention_weight_min": -12.758698463439941,
      "activations/layer14_attention_weight_max": 13.666238784790039,
      "activations/layer14_attention_weight_min": -10.750191688537598,
      "activations/layer15_attention_weight_max": 16.123199462890625,
      "activations/layer15_attention_weight_min": -11.034708023071289,
      "activations/layer16_attention_weight_max": 12.750494956970215,
      "activations/layer16_attention_weight_min": -9.95661449432373,
      "activations/layer17_attention_weight_max": 10.525774955749512,
      "activations/layer17_attention_weight_min": -8.289158821105957,
      "activations/layer18_attention_weight_max": 13.638812065124512,
      "activations/layer18_attention_weight_min": -9.272818565368652,
      "activations/layer19_attention_weight_max": 8.395902633666992,
      "activations/layer19_attention_weight_min": -7.480602264404297,
      "activations/layer1_attention_weight_max": 15.597672462463379,
      "activations/layer1_attention_weight_min": -10.733091354370117,
      "activations/layer20_attention_weight_max": 6.344570636749268,
      "activations/layer20_attention_weight_min": -5.861880302429199,
      "activations/layer21_attention_weight_max": 11.757927894592285,
      "activations/layer21_attention_weight_min": -7.5301737785339355,
      "activations/layer22_attention_weight_max": 9.348514556884766,
      "activations/layer22_attention_weight_min": -4.51109504699707,
      "activations/layer23_attention_weight_max": 7.883856773376465,
      "activations/layer23_attention_weight_min": -4.145967960357666,
      "activations/layer2_attention_weight_max": 19.584735870361328,
      "activations/layer2_attention_weight_min": -16.679325103759766,
      "activations/layer3_attention_weight_max": 63.71295166015625,
      "activations/layer3_attention_weight_min": -55.182151794433594,
      "activations/layer4_attention_weight_max": 165.82736206054688,
      "activations/layer4_attention_weight_min": -159.98153686523438,
      "activations/layer5_attention_weight_max": 198.71533203125,
      "activations/layer5_attention_weight_min": -198.12457275390625,
      "activations/layer6_attention_weight_max": 161.0238494873047,
      "activations/layer6_attention_weight_min": -155.78582763671875,
      "activations/layer7_attention_weight_max": 123.44161987304688,
      "activations/layer7_attention_weight_min": -116.01960754394531,
      "activations/layer8_attention_weight_max": 95.65758514404297,
      "activations/layer8_attention_weight_min": -81.41251373291016,
      "activations/layer9_attention_weight_max": 65.9605484008789,
      "activations/layer9_attention_weight_min": -49.58675765991211,
      "epoch": 0.09,
      "learning_rate": 0.0001332582997336764,
      "loss": 4.1331,
      "step": 1600
    },
    {
      "activations/layer0_attention_weight_max": 13.04206371307373,
      "activations/layer0_attention_weight_min": -8.848270416259766,
      "activations/layer10_attention_weight_max": 34.50457763671875,
      "activations/layer10_attention_weight_min": -24.32842445373535,
      "activations/layer11_attention_weight_max": 22.428550720214844,
      "activations/layer11_attention_weight_min": -16.842037200927734,
      "activations/layer12_attention_weight_max": 21.885028839111328,
      "activations/layer12_attention_weight_min": -23.908903121948242,
      "activations/layer13_attention_weight_max": 17.760732650756836,
      "activations/layer13_attention_weight_min": -13.655503273010254,
      "activations/layer14_attention_weight_max": 13.807914733886719,
      "activations/layer14_attention_weight_min": -11.956942558288574,
      "activations/layer15_attention_weight_max": 17.543991088867188,
      "activations/layer15_attention_weight_min": -12.182626724243164,
      "activations/layer16_attention_weight_max": 13.652576446533203,
      "activations/layer16_attention_weight_min": -9.794073104858398,
      "activations/layer17_attention_weight_max": 13.229231834411621,
      "activations/layer17_attention_weight_min": -9.539240837097168,
      "activations/layer18_attention_weight_max": 15.504497528076172,
      "activations/layer18_attention_weight_min": -9.443739891052246,
      "activations/layer19_attention_weight_max": 10.028346061706543,
      "activations/layer19_attention_weight_min": -8.544502258300781,
      "activations/layer1_attention_weight_max": 15.195651054382324,
      "activations/layer1_attention_weight_min": -10.68449878692627,
      "activations/layer20_attention_weight_max": 7.462731838226318,
      "activations/layer20_attention_weight_min": -6.025651931762695,
      "activations/layer21_attention_weight_max": 14.665176391601562,
      "activations/layer21_attention_weight_min": -8.829251289367676,
      "activations/layer22_attention_weight_max": 10.014204025268555,
      "activations/layer22_attention_weight_min": -4.870544910430908,
      "activations/layer23_attention_weight_max": 7.1727213859558105,
      "activations/layer23_attention_weight_min": -4.007579803466797,
      "activations/layer2_attention_weight_max": 19.650053024291992,
      "activations/layer2_attention_weight_min": -17.359954833984375,
      "activations/layer3_attention_weight_max": 60.428436279296875,
      "activations/layer3_attention_weight_min": -56.2430419921875,
      "activations/layer4_attention_weight_max": 152.66815185546875,
      "activations/layer4_attention_weight_min": -155.76707458496094,
      "activations/layer5_attention_weight_max": 186.07479858398438,
      "activations/layer5_attention_weight_min": -191.493896484375,
      "activations/layer6_attention_weight_max": 145.5721435546875,
      "activations/layer6_attention_weight_min": -154.57574462890625,
      "activations/layer7_attention_weight_max": 109.3934326171875,
      "activations/layer7_attention_weight_min": -111.58985900878906,
      "activations/layer8_attention_weight_max": 84.93596649169922,
      "activations/layer8_attention_weight_min": -79.72676849365234,
      "activations/layer9_attention_weight_max": 57.196685791015625,
      "activations/layer9_attention_weight_min": -49.73360061645508,
      "epoch": 0.1,
      "learning_rate": 0.00013381999875347378,
      "loss": 4.1157,
      "step": 1650
    },
    {
      "activations/layer0_attention_weight_max": 12.667469024658203,
      "activations/layer0_attention_weight_min": -8.848937034606934,
      "activations/layer10_attention_weight_max": 32.46505355834961,
      "activations/layer10_attention_weight_min": -26.104745864868164,
      "activations/layer11_attention_weight_max": 20.700286865234375,
      "activations/layer11_attention_weight_min": -14.667564392089844,
      "activations/layer12_attention_weight_max": 18.35670280456543,
      "activations/layer12_attention_weight_min": -24.530298233032227,
      "activations/layer13_attention_weight_max": 17.000751495361328,
      "activations/layer13_attention_weight_min": -13.967403411865234,
      "activations/layer14_attention_weight_max": 15.124107360839844,
      "activations/layer14_attention_weight_min": -11.175935745239258,
      "activations/layer15_attention_weight_max": 15.902576446533203,
      "activations/layer15_attention_weight_min": -10.7865571975708,
      "activations/layer16_attention_weight_max": 13.722289085388184,
      "activations/layer16_attention_weight_min": -12.593351364135742,
      "activations/layer17_attention_weight_max": 9.794487953186035,
      "activations/layer17_attention_weight_min": -7.281363010406494,
      "activations/layer18_attention_weight_max": 12.771950721740723,
      "activations/layer18_attention_weight_min": -9.477421760559082,
      "activations/layer19_attention_weight_max": 10.335640907287598,
      "activations/layer19_attention_weight_min": -8.481571197509766,
      "activations/layer1_attention_weight_max": 16.191137313842773,
      "activations/layer1_attention_weight_min": -10.22513484954834,
      "activations/layer20_attention_weight_max": 7.17963981628418,
      "activations/layer20_attention_weight_min": -4.886030197143555,
      "activations/layer21_attention_weight_max": 13.106132507324219,
      "activations/layer21_attention_weight_min": -7.3786773681640625,
      "activations/layer22_attention_weight_max": 8.582121849060059,
      "activations/layer22_attention_weight_min": -4.7156662940979,
      "activations/layer23_attention_weight_max": 7.5375075340271,
      "activations/layer23_attention_weight_min": -3.991177558898926,
      "activations/layer2_attention_weight_max": 19.941225051879883,
      "activations/layer2_attention_weight_min": -16.813400268554688,
      "activations/layer3_attention_weight_max": 63.361515045166016,
      "activations/layer3_attention_weight_min": -56.48099899291992,
      "activations/layer4_attention_weight_max": 153.7529754638672,
      "activations/layer4_attention_weight_min": -159.1322479248047,
      "activations/layer5_attention_weight_max": 185.94467163085938,
      "activations/layer5_attention_weight_min": -193.84149169921875,
      "activations/layer6_attention_weight_max": 147.32940673828125,
      "activations/layer6_attention_weight_min": -153.89695739746094,
      "activations/layer7_attention_weight_max": 113.45594787597656,
      "activations/layer7_attention_weight_min": -110.9833984375,
      "activations/layer8_attention_weight_max": 84.51480865478516,
      "activations/layer8_attention_weight_min": -80.36725616455078,
      "activations/layer9_attention_weight_max": 57.29518508911133,
      "activations/layer9_attention_weight_min": -51.746185302734375,
      "epoch": 0.1,
      "learning_rate": 0.00013436477652963025,
      "loss": 4.0781,
      "step": 1700
    },
    {
      "activations/layer0_attention_weight_max": 12.678704261779785,
      "activations/layer0_attention_weight_min": -9.210870742797852,
      "activations/layer10_attention_weight_max": 40.12781524658203,
      "activations/layer10_attention_weight_min": -28.925939559936523,
      "activations/layer11_attention_weight_max": 28.408275604248047,
      "activations/layer11_attention_weight_min": -16.91461944580078,
      "activations/layer12_attention_weight_max": 21.199384689331055,
      "activations/layer12_attention_weight_min": -24.769987106323242,
      "activations/layer13_attention_weight_max": 18.8243350982666,
      "activations/layer13_attention_weight_min": -15.626177787780762,
      "activations/layer14_attention_weight_max": 14.623167037963867,
      "activations/layer14_attention_weight_min": -9.726061820983887,
      "activations/layer15_attention_weight_max": 22.148765563964844,
      "activations/layer15_attention_weight_min": -10.981040954589844,
      "activations/layer16_attention_weight_max": 17.6817684173584,
      "activations/layer16_attention_weight_min": -9.833561897277832,
      "activations/layer17_attention_weight_max": 12.687789916992188,
      "activations/layer17_attention_weight_min": -7.869948387145996,
      "activations/layer18_attention_weight_max": 13.552325248718262,
      "activations/layer18_attention_weight_min": -9.212936401367188,
      "activations/layer19_attention_weight_max": 9.763782501220703,
      "activations/layer19_attention_weight_min": -7.645404815673828,
      "activations/layer1_attention_weight_max": 16.677227020263672,
      "activations/layer1_attention_weight_min": -9.973492622375488,
      "activations/layer20_attention_weight_max": 6.9535064697265625,
      "activations/layer20_attention_weight_min": -5.841658115386963,
      "activations/layer21_attention_weight_max": 13.57069206237793,
      "activations/layer21_attention_weight_min": -7.270079612731934,
      "activations/layer22_attention_weight_max": 10.192349433898926,
      "activations/layer22_attention_weight_min": -4.530439853668213,
      "activations/layer23_attention_weight_max": 8.408382415771484,
      "activations/layer23_attention_weight_min": -4.191778182983398,
      "activations/layer2_attention_weight_max": 21.26093864440918,
      "activations/layer2_attention_weight_min": -17.658737182617188,
      "activations/layer3_attention_weight_max": 61.1821403503418,
      "activations/layer3_attention_weight_min": -56.38602828979492,
      "activations/layer4_attention_weight_max": 157.0956268310547,
      "activations/layer4_attention_weight_min": -159.9010009765625,
      "activations/layer5_attention_weight_max": 191.61497497558594,
      "activations/layer5_attention_weight_min": -192.1111297607422,
      "activations/layer6_attention_weight_max": 148.70115661621094,
      "activations/layer6_attention_weight_min": -147.8155975341797,
      "activations/layer7_attention_weight_max": 116.0925521850586,
      "activations/layer7_attention_weight_min": -110.8528060913086,
      "activations/layer8_attention_weight_max": 90.50052642822266,
      "activations/layer8_attention_weight_min": -85.14295959472656,
      "activations/layer9_attention_weight_max": 63.06039047241211,
      "activations/layer9_attention_weight_min": -54.94301223754883,
      "epoch": 0.1,
      "learning_rate": 0.00013489362282661488,
      "loss": 4.052,
      "step": 1750
    },
    {
      "activations/layer0_attention_weight_max": 13.378680229187012,
      "activations/layer0_attention_weight_min": -9.543742179870605,
      "activations/layer10_attention_weight_max": 34.605934143066406,
      "activations/layer10_attention_weight_min": -27.253538131713867,
      "activations/layer11_attention_weight_max": 29.695457458496094,
      "activations/layer11_attention_weight_min": -18.63870620727539,
      "activations/layer12_attention_weight_max": 23.833349227905273,
      "activations/layer12_attention_weight_min": -23.441925048828125,
      "activations/layer13_attention_weight_max": 23.67438316345215,
      "activations/layer13_attention_weight_min": -15.796613693237305,
      "activations/layer14_attention_weight_max": 14.952447891235352,
      "activations/layer14_attention_weight_min": -12.215717315673828,
      "activations/layer15_attention_weight_max": 19.44476318359375,
      "activations/layer15_attention_weight_min": -12.853209495544434,
      "activations/layer16_attention_weight_max": 18.465232849121094,
      "activations/layer16_attention_weight_min": -10.746963500976562,
      "activations/layer17_attention_weight_max": 14.875202178955078,
      "activations/layer17_attention_weight_min": -8.495488166809082,
      "activations/layer18_attention_weight_max": 14.702646255493164,
      "activations/layer18_attention_weight_min": -10.032691955566406,
      "activations/layer19_attention_weight_max": 12.842744827270508,
      "activations/layer19_attention_weight_min": -9.148679733276367,
      "activations/layer1_attention_weight_max": 15.586494445800781,
      "activations/layer1_attention_weight_min": -10.40346622467041,
      "activations/layer20_attention_weight_max": 7.715030670166016,
      "activations/layer20_attention_weight_min": -6.618916034698486,
      "activations/layer21_attention_weight_max": 17.160064697265625,
      "activations/layer21_attention_weight_min": -8.514911651611328,
      "activations/layer22_attention_weight_max": 8.984118461608887,
      "activations/layer22_attention_weight_min": -4.691169261932373,
      "activations/layer23_attention_weight_max": 7.131495475769043,
      "activations/layer23_attention_weight_min": -4.020003795623779,
      "activations/layer2_attention_weight_max": 20.366722106933594,
      "activations/layer2_attention_weight_min": -17.875545501708984,
      "activations/layer3_attention_weight_max": 62.93540954589844,
      "activations/layer3_attention_weight_min": -53.57766342163086,
      "activations/layer4_attention_weight_max": 163.09117126464844,
      "activations/layer4_attention_weight_min": -156.1326904296875,
      "activations/layer5_attention_weight_max": 190.98934936523438,
      "activations/layer5_attention_weight_min": -190.1197509765625,
      "activations/layer6_attention_weight_max": 155.40072631835938,
      "activations/layer6_attention_weight_min": -148.44461059570312,
      "activations/layer7_attention_weight_max": 124.37052154541016,
      "activations/layer7_attention_weight_min": -112.53657531738281,
      "activations/layer8_attention_weight_max": 94.66770935058594,
      "activations/layer8_attention_weight_min": -83.94960021972656,
      "activations/layer9_attention_weight_max": 65.54286193847656,
      "activations/layer9_attention_weight_min": -53.667762756347656,
      "epoch": 0.1,
      "learning_rate": 0.00013540744303023553,
      "loss": 4.0312,
      "step": 1800
    },
    {
      "activations/layer0_attention_weight_max": 12.56243896484375,
      "activations/layer0_attention_weight_min": -8.894275665283203,
      "activations/layer10_attention_weight_max": 43.92151641845703,
      "activations/layer10_attention_weight_min": -25.05052375793457,
      "activations/layer11_attention_weight_max": 28.46815299987793,
      "activations/layer11_attention_weight_min": -15.347537994384766,
      "activations/layer12_attention_weight_max": 21.498218536376953,
      "activations/layer12_attention_weight_min": -23.09100341796875,
      "activations/layer13_attention_weight_max": 21.811460494995117,
      "activations/layer13_attention_weight_min": -15.530121803283691,
      "activations/layer14_attention_weight_max": 20.361248016357422,
      "activations/layer14_attention_weight_min": -12.185623168945312,
      "activations/layer15_attention_weight_max": 22.32468605041504,
      "activations/layer15_attention_weight_min": -10.546677589416504,
      "activations/layer16_attention_weight_max": 17.375452041625977,
      "activations/layer16_attention_weight_min": -9.669257164001465,
      "activations/layer17_attention_weight_max": 11.326122283935547,
      "activations/layer17_attention_weight_min": -8.233267784118652,
      "activations/layer18_attention_weight_max": 14.741255760192871,
      "activations/layer18_attention_weight_min": -10.542140007019043,
      "activations/layer19_attention_weight_max": 12.431992530822754,
      "activations/layer19_attention_weight_min": -9.346818923950195,
      "activations/layer1_attention_weight_max": 17.357498168945312,
      "activations/layer1_attention_weight_min": -11.511553764343262,
      "activations/layer20_attention_weight_max": 7.240789413452148,
      "activations/layer20_attention_weight_min": -6.480316162109375,
      "activations/layer21_attention_weight_max": 15.066869735717773,
      "activations/layer21_attention_weight_min": -8.136604309082031,
      "activations/layer22_attention_weight_max": 9.474689483642578,
      "activations/layer22_attention_weight_min": -4.9418816566467285,
      "activations/layer23_attention_weight_max": 7.6013689041137695,
      "activations/layer23_attention_weight_min": -3.7380967140197754,
      "activations/layer2_attention_weight_max": 21.0467529296875,
      "activations/layer2_attention_weight_min": -18.15386390686035,
      "activations/layer3_attention_weight_max": 64.53272247314453,
      "activations/layer3_attention_weight_min": -54.5406379699707,
      "activations/layer4_attention_weight_max": 166.47471618652344,
      "activations/layer4_attention_weight_min": -154.55755615234375,
      "activations/layer5_attention_weight_max": 198.3010711669922,
      "activations/layer5_attention_weight_min": -187.18553161621094,
      "activations/layer6_attention_weight_max": 158.03924560546875,
      "activations/layer6_attention_weight_min": -141.59132385253906,
      "activations/layer7_attention_weight_max": 121.86616516113281,
      "activations/layer7_attention_weight_min": -108.88865661621094,
      "activations/layer8_attention_weight_max": 92.41500854492188,
      "activations/layer8_attention_weight_min": -76.55242919921875,
      "activations/layer9_attention_weight_max": 68.2881088256836,
      "activations/layer9_attention_weight_min": -47.836143493652344,
      "epoch": 0.11,
      "learning_rate": 0.00013590706747439169,
      "loss": 4.0012,
      "step": 1850
    },
    {
      "activations/layer0_attention_weight_max": 13.402277946472168,
      "activations/layer0_attention_weight_min": -8.961444854736328,
      "activations/layer10_attention_weight_max": 36.4301643371582,
      "activations/layer10_attention_weight_min": -26.64256477355957,
      "activations/layer11_attention_weight_max": 28.41806411743164,
      "activations/layer11_attention_weight_min": -16.630489349365234,
      "activations/layer12_attention_weight_max": 19.87446403503418,
      "activations/layer12_attention_weight_min": -20.61444091796875,
      "activations/layer13_attention_weight_max": 16.058408737182617,
      "activations/layer13_attention_weight_min": -16.8242244720459,
      "activations/layer14_attention_weight_max": 15.474535942077637,
      "activations/layer14_attention_weight_min": -13.812087059020996,
      "activations/layer15_attention_weight_max": 18.8182315826416,
      "activations/layer15_attention_weight_min": -12.154704093933105,
      "activations/layer16_attention_weight_max": 18.828411102294922,
      "activations/layer16_attention_weight_min": -13.335333824157715,
      "activations/layer17_attention_weight_max": 12.602986335754395,
      "activations/layer17_attention_weight_min": -8.579455375671387,
      "activations/layer18_attention_weight_max": 14.921552658081055,
      "activations/layer18_attention_weight_min": -9.858723640441895,
      "activations/layer19_attention_weight_max": 12.933581352233887,
      "activations/layer19_attention_weight_min": -9.506709098815918,
      "activations/layer1_attention_weight_max": 14.691067695617676,
      "activations/layer1_attention_weight_min": -10.148564338684082,
      "activations/layer20_attention_weight_max": 9.317032814025879,
      "activations/layer20_attention_weight_min": -7.0435285568237305,
      "activations/layer21_attention_weight_max": 14.011785507202148,
      "activations/layer21_attention_weight_min": -10.138249397277832,
      "activations/layer22_attention_weight_max": 10.182317733764648,
      "activations/layer22_attention_weight_min": -5.313981533050537,
      "activations/layer23_attention_weight_max": 7.803683280944824,
      "activations/layer23_attention_weight_min": -4.438259124755859,
      "activations/layer2_attention_weight_max": 20.402069091796875,
      "activations/layer2_attention_weight_min": -18.48122215270996,
      "activations/layer3_attention_weight_max": 64.57625579833984,
      "activations/layer3_attention_weight_min": -55.696144104003906,
      "activations/layer4_attention_weight_max": 156.99266052246094,
      "activations/layer4_attention_weight_min": -161.89259338378906,
      "activations/layer5_attention_weight_max": 184.26251220703125,
      "activations/layer5_attention_weight_min": -197.37286376953125,
      "activations/layer6_attention_weight_max": 145.85821533203125,
      "activations/layer6_attention_weight_min": -152.67559814453125,
      "activations/layer7_attention_weight_max": 106.57112884521484,
      "activations/layer7_attention_weight_min": -111.37906646728516,
      "activations/layer8_attention_weight_max": 79.19819641113281,
      "activations/layer8_attention_weight_min": -74.6218032836914,
      "activations/layer9_attention_weight_max": 62.37661361694336,
      "activations/layer9_attention_weight_min": -50.4483757019043,
      "epoch": 0.11,
      "learning_rate": 0.00013639325951374467,
      "loss": 3.9822,
      "step": 1900
    },
    {
      "activations/layer0_attention_weight_max": 13.914315223693848,
      "activations/layer0_attention_weight_min": -8.895401000976562,
      "activations/layer10_attention_weight_max": 34.81068420410156,
      "activations/layer10_attention_weight_min": -26.227542877197266,
      "activations/layer11_attention_weight_max": 25.755260467529297,
      "activations/layer11_attention_weight_min": -17.82447052001953,
      "activations/layer12_attention_weight_max": 22.47024154663086,
      "activations/layer12_attention_weight_min": -21.0488338470459,
      "activations/layer13_attention_weight_max": 22.364341735839844,
      "activations/layer13_attention_weight_min": -17.4041805267334,
      "activations/layer14_attention_weight_max": 14.019335746765137,
      "activations/layer14_attention_weight_min": -12.263051986694336,
      "activations/layer15_attention_weight_max": 22.19160270690918,
      "activations/layer15_attention_weight_min": -12.572233200073242,
      "activations/layer16_attention_weight_max": 22.779726028442383,
      "activations/layer16_attention_weight_min": -10.838863372802734,
      "activations/layer17_attention_weight_max": 13.228618621826172,
      "activations/layer17_attention_weight_min": -6.879729747772217,
      "activations/layer18_attention_weight_max": 15.057551383972168,
      "activations/layer18_attention_weight_min": -11.464455604553223,
      "activations/layer19_attention_weight_max": 16.125133514404297,
      "activations/layer19_attention_weight_min": -10.755328178405762,
      "activations/layer1_attention_weight_max": 16.472227096557617,
      "activations/layer1_attention_weight_min": -11.608444213867188,
      "activations/layer20_attention_weight_max": 7.142879009246826,
      "activations/layer20_attention_weight_min": -6.222373008728027,
      "activations/layer21_attention_weight_max": 14.566548347473145,
      "activations/layer21_attention_weight_min": -8.733110427856445,
      "activations/layer22_attention_weight_max": 9.684683799743652,
      "activations/layer22_attention_weight_min": -4.929810047149658,
      "activations/layer23_attention_weight_max": 7.477126598358154,
      "activations/layer23_attention_weight_min": -4.220441818237305,
      "activations/layer2_attention_weight_max": 20.855045318603516,
      "activations/layer2_attention_weight_min": -17.10291862487793,
      "activations/layer3_attention_weight_max": 61.81782150268555,
      "activations/layer3_attention_weight_min": -55.51060104370117,
      "activations/layer4_attention_weight_max": 161.53895568847656,
      "activations/layer4_attention_weight_min": -159.8329315185547,
      "activations/layer5_attention_weight_max": 195.45220947265625,
      "activations/layer5_attention_weight_min": -192.00242614746094,
      "activations/layer6_attention_weight_max": 145.4315948486328,
      "activations/layer6_attention_weight_min": -149.7541961669922,
      "activations/layer7_attention_weight_max": 118.55488586425781,
      "activations/layer7_attention_weight_min": -114.56584930419922,
      "activations/layer8_attention_weight_max": 82.97224426269531,
      "activations/layer8_attention_weight_min": -80.21195983886719,
      "activations/layer9_attention_weight_max": 61.3388557434082,
      "activations/layer9_attention_weight_min": -50.418392181396484,
      "epoch": 0.11,
      "learning_rate": 0.00013686672253936715,
      "loss": 3.9635,
      "step": 1950
    },
    {
      "activations/layer0_attention_weight_max": 13.15992259979248,
      "activations/layer0_attention_weight_min": -9.793643951416016,
      "activations/layer10_attention_weight_max": 32.9127082824707,
      "activations/layer10_attention_weight_min": -25.019628524780273,
      "activations/layer11_attention_weight_max": 28.132213592529297,
      "activations/layer11_attention_weight_min": -16.59457778930664,
      "activations/layer12_attention_weight_max": 19.63792610168457,
      "activations/layer12_attention_weight_min": -21.84394073486328,
      "activations/layer13_attention_weight_max": 17.73321533203125,
      "activations/layer13_attention_weight_min": -16.948461532592773,
      "activations/layer14_attention_weight_max": 16.15515899658203,
      "activations/layer14_attention_weight_min": -9.8309965133667,
      "activations/layer15_attention_weight_max": 15.804035186767578,
      "activations/layer15_attention_weight_min": -11.703651428222656,
      "activations/layer16_attention_weight_max": 15.48073959350586,
      "activations/layer16_attention_weight_min": -11.321881294250488,
      "activations/layer17_attention_weight_max": 13.342970848083496,
      "activations/layer17_attention_weight_min": -8.344789505004883,
      "activations/layer18_attention_weight_max": 13.979924201965332,
      "activations/layer18_attention_weight_min": -9.487058639526367,
      "activations/layer19_attention_weight_max": 14.10138988494873,
      "activations/layer19_attention_weight_min": -10.282746315002441,
      "activations/layer1_attention_weight_max": 15.752839088439941,
      "activations/layer1_attention_weight_min": -10.79068374633789,
      "activations/layer20_attention_weight_max": 7.759391784667969,
      "activations/layer20_attention_weight_min": -6.8122358322143555,
      "activations/layer21_attention_weight_max": 11.445189476013184,
      "activations/layer21_attention_weight_min": -7.743039608001709,
      "activations/layer22_attention_weight_max": 11.631814002990723,
      "activations/layer22_attention_weight_min": -4.965576648712158,
      "activations/layer23_attention_weight_max": 7.422357559204102,
      "activations/layer23_attention_weight_min": -4.285472869873047,
      "activations/layer2_attention_weight_max": 19.16168785095215,
      "activations/layer2_attention_weight_min": -17.974790573120117,
      "activations/layer3_attention_weight_max": 60.52266311645508,
      "activations/layer3_attention_weight_min": -52.835166931152344,
      "activations/layer4_attention_weight_max": 158.44114685058594,
      "activations/layer4_attention_weight_min": -154.0638885498047,
      "activations/layer5_attention_weight_max": 189.10308837890625,
      "activations/layer5_attention_weight_min": -184.01388549804688,
      "activations/layer6_attention_weight_max": 149.69793701171875,
      "activations/layer6_attention_weight_min": -148.19464111328125,
      "activations/layer7_attention_weight_max": 114.34062957763672,
      "activations/layer7_attention_weight_min": -107.40653228759766,
      "activations/layer8_attention_weight_max": 87.51053619384766,
      "activations/layer8_attention_weight_min": -75.70386505126953,
      "activations/layer9_attention_weight_max": 60.05705642700195,
      "activations/layer9_attention_weight_min": -51.760066986083984,
      "epoch": 0.12,
      "learning_rate": 0.0001373281060992278,
      "loss": 3.9391,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_loss": 3.87109375,
      "eval_runtime": 8.2155,
      "eval_samples_per_second": 522.667,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_openwebtext_loss": 3.87109375,
      "eval_openwebtext_ppl": 47.99485175252902,
      "eval_openwebtext_runtime": 8.2155,
      "eval_openwebtext_samples_per_second": 522.667,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_wikitext_loss": 4.26953125,
      "eval_wikitext_ppl": 71.48811770889668,
      "eval_wikitext_runtime": 1.6932,
      "eval_wikitext_samples_per_second": 269.318,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_lambada_loss": 5.04296875,
      "eval_lambada_ppl": 154.92927926604594,
      "eval_lambada_runtime": 9.2837,
      "eval_lambada_samples_per_second": 524.47,
      "step": 2000
    },
    {
      "activations/layer0_attention_weight_max": 13.710124969482422,
      "activations/layer0_attention_weight_min": -8.713746070861816,
      "activations/layer10_attention_weight_max": 35.006717681884766,
      "activations/layer10_attention_weight_min": -27.4268856048584,
      "activations/layer11_attention_weight_max": 29.35987663269043,
      "activations/layer11_attention_weight_min": -19.520137786865234,
      "activations/layer12_attention_weight_max": 22.52364730834961,
      "activations/layer12_attention_weight_min": -25.674434661865234,
      "activations/layer13_attention_weight_max": 20.200729370117188,
      "activations/layer13_attention_weight_min": -16.886871337890625,
      "activations/layer14_attention_weight_max": 15.610573768615723,
      "activations/layer14_attention_weight_min": -11.661930084228516,
      "activations/layer15_attention_weight_max": 19.563495635986328,
      "activations/layer15_attention_weight_min": -12.166768074035645,
      "activations/layer16_attention_weight_max": 16.504547119140625,
      "activations/layer16_attention_weight_min": -11.827136993408203,
      "activations/layer17_attention_weight_max": 13.915919303894043,
      "activations/layer17_attention_weight_min": -8.511876106262207,
      "activations/layer18_attention_weight_max": 16.8443660736084,
      "activations/layer18_attention_weight_min": -9.158370971679688,
      "activations/layer19_attention_weight_max": 13.5128173828125,
      "activations/layer19_attention_weight_min": -9.175021171569824,
      "activations/layer1_attention_weight_max": 17.225849151611328,
      "activations/layer1_attention_weight_min": -11.29086971282959,
      "activations/layer20_attention_weight_max": 8.25432300567627,
      "activations/layer20_attention_weight_min": -8.400700569152832,
      "activations/layer21_attention_weight_max": 12.183761596679688,
      "activations/layer21_attention_weight_min": -7.410185813903809,
      "activations/layer22_attention_weight_max": 9.352377891540527,
      "activations/layer22_attention_weight_min": -4.842890739440918,
      "activations/layer23_attention_weight_max": 7.3256988525390625,
      "activations/layer23_attention_weight_min": -3.9679839611053467,
      "activations/layer2_attention_weight_max": 20.355018615722656,
      "activations/layer2_attention_weight_min": -18.61149787902832,
      "activations/layer3_attention_weight_max": 60.679603576660156,
      "activations/layer3_attention_weight_min": -54.005123138427734,
      "activations/layer4_attention_weight_max": 152.572998046875,
      "activations/layer4_attention_weight_min": -155.7728271484375,
      "activations/layer5_attention_weight_max": 177.37864685058594,
      "activations/layer5_attention_weight_min": -182.88546752929688,
      "activations/layer6_attention_weight_max": 136.55599975585938,
      "activations/layer6_attention_weight_min": -139.12525939941406,
      "activations/layer7_attention_weight_max": 106.73088073730469,
      "activations/layer7_attention_weight_min": -104.33197021484375,
      "activations/layer8_attention_weight_max": 78.6908187866211,
      "activations/layer8_attention_weight_min": -76.4863052368164,
      "activations/layer9_attention_weight_max": 57.7863655090332,
      "activations/layer9_attention_weight_min": -52.3626708984375,
      "epoch": 0.12,
      "learning_rate": 0.0001377691219698583,
      "loss": 3.9432,
      "step": 2050
    },
    {
      "activations/layer0_attention_weight_max": 14.25937557220459,
      "activations/layer0_attention_weight_min": -9.25207233428955,
      "activations/layer10_attention_weight_max": 36.314693450927734,
      "activations/layer10_attention_weight_min": -24.433395385742188,
      "activations/layer11_attention_weight_max": 25.003658294677734,
      "activations/layer11_attention_weight_min": -17.341533660888672,
      "activations/layer12_attention_weight_max": 21.530725479125977,
      "activations/layer12_attention_weight_min": -22.959877014160156,
      "activations/layer13_attention_weight_max": 22.834806442260742,
      "activations/layer13_attention_weight_min": -15.759854316711426,
      "activations/layer14_attention_weight_max": 15.288108825683594,
      "activations/layer14_attention_weight_min": -10.57365608215332,
      "activations/layer15_attention_weight_max": 18.753477096557617,
      "activations/layer15_attention_weight_min": -10.81553840637207,
      "activations/layer16_attention_weight_max": 17.08627700805664,
      "activations/layer16_attention_weight_min": -10.722350120544434,
      "activations/layer17_attention_weight_max": 13.367847442626953,
      "activations/layer17_attention_weight_min": -7.28140926361084,
      "activations/layer18_attention_weight_max": 16.775875091552734,
      "activations/layer18_attention_weight_min": -9.300148010253906,
      "activations/layer19_attention_weight_max": 15.081160545349121,
      "activations/layer19_attention_weight_min": -8.73155689239502,
      "activations/layer1_attention_weight_max": 16.554534912109375,
      "activations/layer1_attention_weight_min": -10.230793952941895,
      "activations/layer20_attention_weight_max": 9.584508895874023,
      "activations/layer20_attention_weight_min": -6.6983137130737305,
      "activations/layer21_attention_weight_max": 11.796299934387207,
      "activations/layer21_attention_weight_min": -7.251735687255859,
      "activations/layer22_attention_weight_max": 8.594325065612793,
      "activations/layer22_attention_weight_min": -5.125678062438965,
      "activations/layer23_attention_weight_max": 5.620063781738281,
      "activations/layer23_attention_weight_min": -4.246960639953613,
      "activations/layer2_attention_weight_max": 20.60588836669922,
      "activations/layer2_attention_weight_min": -18.661972045898438,
      "activations/layer3_attention_weight_max": 61.85427474975586,
      "activations/layer3_attention_weight_min": -53.85987091064453,
      "activations/layer4_attention_weight_max": 155.81581115722656,
      "activations/layer4_attention_weight_min": -152.21511840820312,
      "activations/layer5_attention_weight_max": 180.78465270996094,
      "activations/layer5_attention_weight_min": -182.49099731445312,
      "activations/layer6_attention_weight_max": 139.7788543701172,
      "activations/layer6_attention_weight_min": -140.72755432128906,
      "activations/layer7_attention_weight_max": 108.99836730957031,
      "activations/layer7_attention_weight_min": -103.96428680419922,
      "activations/layer8_attention_weight_max": 85.35945129394531,
      "activations/layer8_attention_weight_min": -79.76944732666016,
      "activations/layer9_attention_weight_max": 66.97433471679688,
      "activations/layer9_attention_weight_min": -52.15813064575195,
      "epoch": 0.12,
      "learning_rate": 0.00013820831923751152,
      "loss": 3.9134,
      "step": 2100
    },
    {
      "activations/layer0_attention_weight_max": 12.419702529907227,
      "activations/layer0_attention_weight_min": -8.688989639282227,
      "activations/layer10_attention_weight_max": 39.408843994140625,
      "activations/layer10_attention_weight_min": -26.476741790771484,
      "activations/layer11_attention_weight_max": 32.79896545410156,
      "activations/layer11_attention_weight_min": -20.443038940429688,
      "activations/layer12_attention_weight_max": 23.277067184448242,
      "activations/layer12_attention_weight_min": -21.42789649963379,
      "activations/layer13_attention_weight_max": 22.003215789794922,
      "activations/layer13_attention_weight_min": -18.13038444519043,
      "activations/layer14_attention_weight_max": 21.448928833007812,
      "activations/layer14_attention_weight_min": -11.882498741149902,
      "activations/layer15_attention_weight_max": 22.874767303466797,
      "activations/layer15_attention_weight_min": -13.581625938415527,
      "activations/layer16_attention_weight_max": 25.19434356689453,
      "activations/layer16_attention_weight_min": -11.140091896057129,
      "activations/layer17_attention_weight_max": 15.754460334777832,
      "activations/layer17_attention_weight_min": -7.7254157066345215,
      "activations/layer18_attention_weight_max": 19.122282028198242,
      "activations/layer18_attention_weight_min": -10.806157112121582,
      "activations/layer19_attention_weight_max": 18.233728408813477,
      "activations/layer19_attention_weight_min": -11.371037483215332,
      "activations/layer1_attention_weight_max": 18.234329223632812,
      "activations/layer1_attention_weight_min": -11.35349178314209,
      "activations/layer20_attention_weight_max": 10.554656028747559,
      "activations/layer20_attention_weight_min": -8.906121253967285,
      "activations/layer21_attention_weight_max": 12.554275512695312,
      "activations/layer21_attention_weight_min": -9.228642463684082,
      "activations/layer22_attention_weight_max": 11.446233749389648,
      "activations/layer22_attention_weight_min": -5.236886024475098,
      "activations/layer23_attention_weight_max": 7.858452796936035,
      "activations/layer23_attention_weight_min": -4.542399883270264,
      "activations/layer2_attention_weight_max": 21.753559112548828,
      "activations/layer2_attention_weight_min": -18.11566925048828,
      "activations/layer3_attention_weight_max": 66.89469909667969,
      "activations/layer3_attention_weight_min": -55.435585021972656,
      "activations/layer4_attention_weight_max": 165.07908630371094,
      "activations/layer4_attention_weight_min": -154.5908966064453,
      "activations/layer5_attention_weight_max": 204.99746704101562,
      "activations/layer5_attention_weight_min": -190.07098388671875,
      "activations/layer6_attention_weight_max": 160.40354919433594,
      "activations/layer6_attention_weight_min": -143.48887634277344,
      "activations/layer7_attention_weight_max": 130.3466033935547,
      "activations/layer7_attention_weight_min": -113.40460205078125,
      "activations/layer8_attention_weight_max": 93.0755386352539,
      "activations/layer8_attention_weight_min": -77.77333068847656,
      "activations/layer9_attention_weight_max": 70.88082885742188,
      "activations/layer9_attention_weight_min": -57.06038284301758,
      "epoch": 0.12,
      "learning_rate": 0.00013863710306478343,
      "loss": 3.9026,
      "step": 2150
    },
    {
      "activations/layer0_attention_weight_max": 13.420913696289062,
      "activations/layer0_attention_weight_min": -9.006584167480469,
      "activations/layer10_attention_weight_max": 33.228004455566406,
      "activations/layer10_attention_weight_min": -26.802852630615234,
      "activations/layer11_attention_weight_max": 26.496868133544922,
      "activations/layer11_attention_weight_min": -18.110870361328125,
      "activations/layer12_attention_weight_max": 25.829025268554688,
      "activations/layer12_attention_weight_min": -22.125812530517578,
      "activations/layer13_attention_weight_max": 24.33608627319336,
      "activations/layer13_attention_weight_min": -17.3719425201416,
      "activations/layer14_attention_weight_max": 20.748701095581055,
      "activations/layer14_attention_weight_min": -11.284394264221191,
      "activations/layer15_attention_weight_max": 23.6978759765625,
      "activations/layer15_attention_weight_min": -12.129791259765625,
      "activations/layer16_attention_weight_max": 24.285429000854492,
      "activations/layer16_attention_weight_min": -10.585000038146973,
      "activations/layer17_attention_weight_max": 14.805253982543945,
      "activations/layer17_attention_weight_min": -8.101941108703613,
      "activations/layer18_attention_weight_max": 18.766613006591797,
      "activations/layer18_attention_weight_min": -11.519405364990234,
      "activations/layer19_attention_weight_max": 18.272367477416992,
      "activations/layer19_attention_weight_min": -11.625192642211914,
      "activations/layer1_attention_weight_max": 17.246742248535156,
      "activations/layer1_attention_weight_min": -10.064326286315918,
      "activations/layer20_attention_weight_max": 12.456212043762207,
      "activations/layer20_attention_weight_min": -9.691370964050293,
      "activations/layer21_attention_weight_max": 16.468395233154297,
      "activations/layer21_attention_weight_min": -9.716150283813477,
      "activations/layer22_attention_weight_max": 15.086593627929688,
      "activations/layer22_attention_weight_min": -5.783683776855469,
      "activations/layer23_attention_weight_max": 6.883759498596191,
      "activations/layer23_attention_weight_min": -4.772132873535156,
      "activations/layer2_attention_weight_max": 20.187030792236328,
      "activations/layer2_attention_weight_min": -17.341079711914062,
      "activations/layer3_attention_weight_max": 59.17866134643555,
      "activations/layer3_attention_weight_min": -51.211238861083984,
      "activations/layer4_attention_weight_max": 149.13377380371094,
      "activations/layer4_attention_weight_min": -149.47021484375,
      "activations/layer5_attention_weight_max": 176.71177673339844,
      "activations/layer5_attention_weight_min": -178.57302856445312,
      "activations/layer6_attention_weight_max": 132.07058715820312,
      "activations/layer6_attention_weight_min": -132.3011474609375,
      "activations/layer7_attention_weight_max": 105.85002136230469,
      "activations/layer7_attention_weight_min": -99.08519744873047,
      "activations/layer8_attention_weight_max": 78.7157974243164,
      "activations/layer8_attention_weight_min": -76.00194549560547,
      "activations/layer9_attention_weight_max": 59.17700958251953,
      "activations/layer9_attention_weight_min": -52.44511795043945,
      "epoch": 0.13,
      "learning_rate": 0.00013905595584493412,
      "loss": 3.8885,
      "step": 2200
    },
    {
      "activations/layer0_attention_weight_max": 14.031021118164062,
      "activations/layer0_attention_weight_min": -8.830187797546387,
      "activations/layer10_attention_weight_max": 35.88745880126953,
      "activations/layer10_attention_weight_min": -25.60613250732422,
      "activations/layer11_attention_weight_max": 30.910961151123047,
      "activations/layer11_attention_weight_min": -17.567224502563477,
      "activations/layer12_attention_weight_max": 26.561445236206055,
      "activations/layer12_attention_weight_min": -19.370485305786133,
      "activations/layer13_attention_weight_max": 18.421550750732422,
      "activations/layer13_attention_weight_min": -18.724042892456055,
      "activations/layer14_attention_weight_max": 20.903980255126953,
      "activations/layer14_attention_weight_min": -10.906682014465332,
      "activations/layer15_attention_weight_max": 18.733661651611328,
      "activations/layer15_attention_weight_min": -12.140798568725586,
      "activations/layer16_attention_weight_max": 20.59000587463379,
      "activations/layer16_attention_weight_min": -10.388710021972656,
      "activations/layer17_attention_weight_max": 12.468026161193848,
      "activations/layer17_attention_weight_min": -9.354754447937012,
      "activations/layer18_attention_weight_max": 17.66062355041504,
      "activations/layer18_attention_weight_min": -11.094823837280273,
      "activations/layer19_attention_weight_max": 16.758947372436523,
      "activations/layer19_attention_weight_min": -10.843512535095215,
      "activations/layer1_attention_weight_max": 17.456195831298828,
      "activations/layer1_attention_weight_min": -11.253925323486328,
      "activations/layer20_attention_weight_max": 12.962911605834961,
      "activations/layer20_attention_weight_min": -9.326443672180176,
      "activations/layer21_attention_weight_max": 13.947054862976074,
      "activations/layer21_attention_weight_min": -9.87414836883545,
      "activations/layer22_attention_weight_max": 13.965222358703613,
      "activations/layer22_attention_weight_min": -6.192057132720947,
      "activations/layer23_attention_weight_max": 8.135263442993164,
      "activations/layer23_attention_weight_min": -5.181788921356201,
      "activations/layer2_attention_weight_max": 21.409801483154297,
      "activations/layer2_attention_weight_min": -17.575801849365234,
      "activations/layer3_attention_weight_max": 61.90864944458008,
      "activations/layer3_attention_weight_min": -49.052818298339844,
      "activations/layer4_attention_weight_max": 158.4545440673828,
      "activations/layer4_attention_weight_min": -146.8929901123047,
      "activations/layer5_attention_weight_max": 183.73233032226562,
      "activations/layer5_attention_weight_min": -174.34027099609375,
      "activations/layer6_attention_weight_max": 137.88076782226562,
      "activations/layer6_attention_weight_min": -135.39764404296875,
      "activations/layer7_attention_weight_max": 110.6357192993164,
      "activations/layer7_attention_weight_min": -104.50507354736328,
      "activations/layer8_attention_weight_max": 85.98831176757812,
      "activations/layer8_attention_weight_min": -75.3955307006836,
      "activations/layer9_attention_weight_max": 63.474273681640625,
      "activations/layer9_attention_weight_min": -50.88969039916992,
      "epoch": 0.13,
      "learning_rate": 0.00013946532720898237,
      "loss": 3.8806,
      "step": 2250
    },
    {
      "activations/layer0_attention_weight_max": 14.00019645690918,
      "activations/layer0_attention_weight_min": -8.787960052490234,
      "activations/layer10_attention_weight_max": 37.963741302490234,
      "activations/layer10_attention_weight_min": -25.779375076293945,
      "activations/layer11_attention_weight_max": 33.93482208251953,
      "activations/layer11_attention_weight_min": -19.31470489501953,
      "activations/layer12_attention_weight_max": 28.048938751220703,
      "activations/layer12_attention_weight_min": -22.121212005615234,
      "activations/layer13_attention_weight_max": 23.39745330810547,
      "activations/layer13_attention_weight_min": -17.93256378173828,
      "activations/layer14_attention_weight_max": 22.535797119140625,
      "activations/layer14_attention_weight_min": -11.972403526306152,
      "activations/layer15_attention_weight_max": 23.22879409790039,
      "activations/layer15_attention_weight_min": -11.539901733398438,
      "activations/layer16_attention_weight_max": 25.585529327392578,
      "activations/layer16_attention_weight_min": -10.68366527557373,
      "activations/layer17_attention_weight_max": 14.735028266906738,
      "activations/layer17_attention_weight_min": -7.874594688415527,
      "activations/layer18_attention_weight_max": 17.781320571899414,
      "activations/layer18_attention_weight_min": -11.01288890838623,
      "activations/layer19_attention_weight_max": 18.094058990478516,
      "activations/layer19_attention_weight_min": -10.546513557434082,
      "activations/layer1_attention_weight_max": 16.749835968017578,
      "activations/layer1_attention_weight_min": -11.751620292663574,
      "activations/layer20_attention_weight_max": 12.239529609680176,
      "activations/layer20_attention_weight_min": -10.041081428527832,
      "activations/layer21_attention_weight_max": 12.599992752075195,
      "activations/layer21_attention_weight_min": -8.964312553405762,
      "activations/layer22_attention_weight_max": 12.525691986083984,
      "activations/layer22_attention_weight_min": -6.433376789093018,
      "activations/layer23_attention_weight_max": 8.52670669555664,
      "activations/layer23_attention_weight_min": -4.930276870727539,
      "activations/layer2_attention_weight_max": 21.25041389465332,
      "activations/layer2_attention_weight_min": -18.84424591064453,
      "activations/layer3_attention_weight_max": 65.57708740234375,
      "activations/layer3_attention_weight_min": -55.35588073730469,
      "activations/layer4_attention_weight_max": 163.3521728515625,
      "activations/layer4_attention_weight_min": -157.27818298339844,
      "activations/layer5_attention_weight_max": 191.94515991210938,
      "activations/layer5_attention_weight_min": -182.30526733398438,
      "activations/layer6_attention_weight_max": 144.473876953125,
      "activations/layer6_attention_weight_min": -135.90200805664062,
      "activations/layer7_attention_weight_max": 110.12269592285156,
      "activations/layer7_attention_weight_min": -101.8794174194336,
      "activations/layer8_attention_weight_max": 83.17601013183594,
      "activations/layer8_attention_weight_min": -72.1287612915039,
      "activations/layer9_attention_weight_max": 68.48429107666016,
      "activations/layer9_attention_weight_min": -54.487545013427734,
      "epoch": 0.13,
      "learning_rate": 0.0001398656369269228,
      "loss": 3.855,
      "step": 2300
    },
    {
      "activations/layer0_attention_weight_max": 13.256848335266113,
      "activations/layer0_attention_weight_min": -8.689003944396973,
      "activations/layer10_attention_weight_max": 34.89870071411133,
      "activations/layer10_attention_weight_min": -25.876420974731445,
      "activations/layer11_attention_weight_max": 27.9720516204834,
      "activations/layer11_attention_weight_min": -18.140438079833984,
      "activations/layer12_attention_weight_max": 22.9725341796875,
      "activations/layer12_attention_weight_min": -19.857059478759766,
      "activations/layer13_attention_weight_max": 23.11353874206543,
      "activations/layer13_attention_weight_min": -18.24742889404297,
      "activations/layer14_attention_weight_max": 20.69068717956543,
      "activations/layer14_attention_weight_min": -12.559355735778809,
      "activations/layer15_attention_weight_max": 20.484914779663086,
      "activations/layer15_attention_weight_min": -12.229679107666016,
      "activations/layer16_attention_weight_max": 20.388267517089844,
      "activations/layer16_attention_weight_min": -11.931086540222168,
      "activations/layer17_attention_weight_max": 15.23818302154541,
      "activations/layer17_attention_weight_min": -9.49535083770752,
      "activations/layer18_attention_weight_max": 17.029308319091797,
      "activations/layer18_attention_weight_min": -10.953047752380371,
      "activations/layer19_attention_weight_max": 18.727067947387695,
      "activations/layer19_attention_weight_min": -11.926603317260742,
      "activations/layer1_attention_weight_max": 16.16809844970703,
      "activations/layer1_attention_weight_min": -11.111467361450195,
      "activations/layer20_attention_weight_max": 13.592744827270508,
      "activations/layer20_attention_weight_min": -10.296480178833008,
      "activations/layer21_attention_weight_max": 11.802177429199219,
      "activations/layer21_attention_weight_min": -10.375572204589844,
      "activations/layer22_attention_weight_max": 13.750398635864258,
      "activations/layer22_attention_weight_min": -6.356493949890137,
      "activations/layer23_attention_weight_max": 8.18589973449707,
      "activations/layer23_attention_weight_min": -5.832283973693848,
      "activations/layer2_attention_weight_max": 20.173776626586914,
      "activations/layer2_attention_weight_min": -17.66546630859375,
      "activations/layer3_attention_weight_max": 61.228240966796875,
      "activations/layer3_attention_weight_min": -54.0527229309082,
      "activations/layer4_attention_weight_max": 160.0371551513672,
      "activations/layer4_attention_weight_min": -159.963623046875,
      "activations/layer5_attention_weight_max": 184.0927734375,
      "activations/layer5_attention_weight_min": -181.71051025390625,
      "activations/layer6_attention_weight_max": 144.6466064453125,
      "activations/layer6_attention_weight_min": -139.15011596679688,
      "activations/layer7_attention_weight_max": 107.79049682617188,
      "activations/layer7_attention_weight_min": -104.02555084228516,
      "activations/layer8_attention_weight_max": 81.01983642578125,
      "activations/layer8_attention_weight_min": -77.57130432128906,
      "activations/layer9_attention_weight_max": 61.818603515625,
      "activations/layer9_attention_weight_min": -56.51107406616211,
      "epoch": 0.14,
      "learning_rate": 0.0001402572774947444,
      "loss": 3.8324,
      "step": 2350
    },
    {
      "activations/layer0_attention_weight_max": 12.738066673278809,
      "activations/layer0_attention_weight_min": -8.761218070983887,
      "activations/layer10_attention_weight_max": 38.50330352783203,
      "activations/layer10_attention_weight_min": -26.590511322021484,
      "activations/layer11_attention_weight_max": 30.511974334716797,
      "activations/layer11_attention_weight_min": -20.150253295898438,
      "activations/layer12_attention_weight_max": 27.15335464477539,
      "activations/layer12_attention_weight_min": -20.01813507080078,
      "activations/layer13_attention_weight_max": 24.1607723236084,
      "activations/layer13_attention_weight_min": -18.668296813964844,
      "activations/layer14_attention_weight_max": 18.491592407226562,
      "activations/layer14_attention_weight_min": -12.086682319641113,
      "activations/layer15_attention_weight_max": 20.890928268432617,
      "activations/layer15_attention_weight_min": -11.656917572021484,
      "activations/layer16_attention_weight_max": 20.9476318359375,
      "activations/layer16_attention_weight_min": -10.995895385742188,
      "activations/layer17_attention_weight_max": 13.503108978271484,
      "activations/layer17_attention_weight_min": -9.438499450683594,
      "activations/layer18_attention_weight_max": 18.754043579101562,
      "activations/layer18_attention_weight_min": -12.205187797546387,
      "activations/layer19_attention_weight_max": 18.4826717376709,
      "activations/layer19_attention_weight_min": -11.688812255859375,
      "activations/layer1_attention_weight_max": 16.07705307006836,
      "activations/layer1_attention_weight_min": -10.151162147521973,
      "activations/layer20_attention_weight_max": 11.813679695129395,
      "activations/layer20_attention_weight_min": -10.952852249145508,
      "activations/layer21_attention_weight_max": 13.364130973815918,
      "activations/layer21_attention_weight_min": -11.296399116516113,
      "activations/layer22_attention_weight_max": 12.80785083770752,
      "activations/layer22_attention_weight_min": -6.533126354217529,
      "activations/layer23_attention_weight_max": 7.653109550476074,
      "activations/layer23_attention_weight_min": -5.419719696044922,
      "activations/layer2_attention_weight_max": 20.86635971069336,
      "activations/layer2_attention_weight_min": -18.32520294189453,
      "activations/layer3_attention_weight_max": 60.70341491699219,
      "activations/layer3_attention_weight_min": -53.55429458618164,
      "activations/layer4_attention_weight_max": 156.8970184326172,
      "activations/layer4_attention_weight_min": -154.63427734375,
      "activations/layer5_attention_weight_max": 177.21685791015625,
      "activations/layer5_attention_weight_min": -178.404052734375,
      "activations/layer6_attention_weight_max": 139.67181396484375,
      "activations/layer6_attention_weight_min": -133.91387939453125,
      "activations/layer7_attention_weight_max": 108.55016326904297,
      "activations/layer7_attention_weight_min": -104.74944305419922,
      "activations/layer8_attention_weight_max": 85.42676544189453,
      "activations/layer8_attention_weight_min": -76.08679962158203,
      "activations/layer9_attention_weight_max": 62.80153274536133,
      "activations/layer9_attention_weight_min": -52.36761474609375,
      "epoch": 0.14,
      "learning_rate": 0.00014064061644721923,
      "loss": 3.8259,
      "step": 2400
    },
    {
      "activations/layer0_attention_weight_max": 13.605423927307129,
      "activations/layer0_attention_weight_min": -8.723668098449707,
      "activations/layer10_attention_weight_max": 35.53839874267578,
      "activations/layer10_attention_weight_min": -26.24538803100586,
      "activations/layer11_attention_weight_max": 36.579795837402344,
      "activations/layer11_attention_weight_min": -19.552209854125977,
      "activations/layer12_attention_weight_max": 22.822750091552734,
      "activations/layer12_attention_weight_min": -22.908485412597656,
      "activations/layer13_attention_weight_max": 24.509593963623047,
      "activations/layer13_attention_weight_min": -21.313459396362305,
      "activations/layer14_attention_weight_max": 28.349403381347656,
      "activations/layer14_attention_weight_min": -14.256476402282715,
      "activations/layer15_attention_weight_max": 26.71294593811035,
      "activations/layer15_attention_weight_min": -14.626662254333496,
      "activations/layer16_attention_weight_max": 20.81529998779297,
      "activations/layer16_attention_weight_min": -12.204045295715332,
      "activations/layer17_attention_weight_max": 15.950688362121582,
      "activations/layer17_attention_weight_min": -8.882827758789062,
      "activations/layer18_attention_weight_max": 19.85053062438965,
      "activations/layer18_attention_weight_min": -13.103021621704102,
      "activations/layer19_attention_weight_max": 16.97772216796875,
      "activations/layer19_attention_weight_min": -13.22739315032959,
      "activations/layer1_attention_weight_max": 16.45697021484375,
      "activations/layer1_attention_weight_min": -11.236467361450195,
      "activations/layer20_attention_weight_max": 12.538296699523926,
      "activations/layer20_attention_weight_min": -10.445560455322266,
      "activations/layer21_attention_weight_max": 14.815177917480469,
      "activations/layer21_attention_weight_min": -11.152229309082031,
      "activations/layer22_attention_weight_max": 10.518597602844238,
      "activations/layer22_attention_weight_min": -6.478540420532227,
      "activations/layer23_attention_weight_max": 6.750278472900391,
      "activations/layer23_attention_weight_min": -5.642238616943359,
      "activations/layer2_attention_weight_max": 21.567319869995117,
      "activations/layer2_attention_weight_min": -18.513671875,
      "activations/layer3_attention_weight_max": 64.9997787475586,
      "activations/layer3_attention_weight_min": -55.28762435913086,
      "activations/layer4_attention_weight_max": 157.28514099121094,
      "activations/layer4_attention_weight_min": -152.3290557861328,
      "activations/layer5_attention_weight_max": 195.43321228027344,
      "activations/layer5_attention_weight_min": -186.78875732421875,
      "activations/layer6_attention_weight_max": 135.29644775390625,
      "activations/layer6_attention_weight_min": -133.13238525390625,
      "activations/layer7_attention_weight_max": 108.28323364257812,
      "activations/layer7_attention_weight_min": -100.43907165527344,
      "activations/layer8_attention_weight_max": 79.9639663696289,
      "activations/layer8_attention_weight_min": -73.0916519165039,
      "activations/layer9_attention_weight_max": 61.4678955078125,
      "activations/layer9_attention_weight_min": -52.06718063354492,
      "epoch": 0.14,
      "learning_rate": 0.00014101599843062106,
      "loss": 3.814,
      "step": 2450
    },
    {
      "activations/layer0_attention_weight_max": 13.462882995605469,
      "activations/layer0_attention_weight_min": -9.568720817565918,
      "activations/layer10_attention_weight_max": 36.91954803466797,
      "activations/layer10_attention_weight_min": -28.505041122436523,
      "activations/layer11_attention_weight_max": 31.351234436035156,
      "activations/layer11_attention_weight_min": -24.387296676635742,
      "activations/layer12_attention_weight_max": 26.980915069580078,
      "activations/layer12_attention_weight_min": -25.653287887573242,
      "activations/layer13_attention_weight_max": 22.65473747253418,
      "activations/layer13_attention_weight_min": -19.325305938720703,
      "activations/layer14_attention_weight_max": 24.487655639648438,
      "activations/layer14_attention_weight_min": -13.47114372253418,
      "activations/layer15_attention_weight_max": 22.328155517578125,
      "activations/layer15_attention_weight_min": -11.46015453338623,
      "activations/layer16_attention_weight_max": 25.481348037719727,
      "activations/layer16_attention_weight_min": -12.4175386428833,
      "activations/layer17_attention_weight_max": 16.05863380432129,
      "activations/layer17_attention_weight_min": -8.983222961425781,
      "activations/layer18_attention_weight_max": 20.235815048217773,
      "activations/layer18_attention_weight_min": -10.982850074768066,
      "activations/layer19_attention_weight_max": 21.905029296875,
      "activations/layer19_attention_weight_min": -11.479575157165527,
      "activations/layer1_attention_weight_max": 16.16975212097168,
      "activations/layer1_attention_weight_min": -11.34224796295166,
      "activations/layer20_attention_weight_max": 16.784440994262695,
      "activations/layer20_attention_weight_min": -10.740240097045898,
      "activations/layer21_attention_weight_max": 14.268965721130371,
      "activations/layer21_attention_weight_min": -11.65691089630127,
      "activations/layer22_attention_weight_max": 16.15532875061035,
      "activations/layer22_attention_weight_min": -7.583024978637695,
      "activations/layer23_attention_weight_max": 8.780385971069336,
      "activations/layer23_attention_weight_min": -6.66785192489624,
      "activations/layer2_attention_weight_max": 20.851612091064453,
      "activations/layer2_attention_weight_min": -18.28989028930664,
      "activations/layer3_attention_weight_max": 61.90680694580078,
      "activations/layer3_attention_weight_min": -53.37748718261719,
      "activations/layer4_attention_weight_max": 157.56781005859375,
      "activations/layer4_attention_weight_min": -149.14036560058594,
      "activations/layer5_attention_weight_max": 185.37106323242188,
      "activations/layer5_attention_weight_min": -175.6860809326172,
      "activations/layer6_attention_weight_max": 146.1603240966797,
      "activations/layer6_attention_weight_min": -136.045166015625,
      "activations/layer7_attention_weight_max": 115.37934112548828,
      "activations/layer7_attention_weight_min": -106.34039306640625,
      "activations/layer8_attention_weight_max": 88.15715789794922,
      "activations/layer8_attention_weight_min": -71.650634765625,
      "activations/layer9_attention_weight_max": 68.26546478271484,
      "activations/layer9_attention_weight_min": -54.752193450927734,
      "epoch": 0.15,
      "learning_rate": 0.00014138374706467038,
      "loss": 3.7835,
      "step": 2500
    },
    {
      "activations/layer0_attention_weight_max": 13.548212051391602,
      "activations/layer0_attention_weight_min": -9.401025772094727,
      "activations/layer10_attention_weight_max": 36.448448181152344,
      "activations/layer10_attention_weight_min": -27.490182876586914,
      "activations/layer11_attention_weight_max": 33.92330551147461,
      "activations/layer11_attention_weight_min": -26.811729431152344,
      "activations/layer12_attention_weight_max": 22.109880447387695,
      "activations/layer12_attention_weight_min": -25.36366081237793,
      "activations/layer13_attention_weight_max": 19.566808700561523,
      "activations/layer13_attention_weight_min": -18.400924682617188,
      "activations/layer14_attention_weight_max": 19.706884384155273,
      "activations/layer14_attention_weight_min": -13.624982833862305,
      "activations/layer15_attention_weight_max": 22.63808822631836,
      "activations/layer15_attention_weight_min": -11.764549255371094,
      "activations/layer16_attention_weight_max": 17.11253547668457,
      "activations/layer16_attention_weight_min": -11.680627822875977,
      "activations/layer17_attention_weight_max": 14.550877571105957,
      "activations/layer17_attention_weight_min": -9.966462135314941,
      "activations/layer18_attention_weight_max": 19.08646011352539,
      "activations/layer18_attention_weight_min": -13.590287208557129,
      "activations/layer19_attention_weight_max": 17.957372665405273,
      "activations/layer19_attention_weight_min": -12.565227508544922,
      "activations/layer1_attention_weight_max": 16.86669158935547,
      "activations/layer1_attention_weight_min": -11.869755744934082,
      "activations/layer20_attention_weight_max": 14.199213027954102,
      "activations/layer20_attention_weight_min": -11.44829273223877,
      "activations/layer21_attention_weight_max": 13.027949333190918,
      "activations/layer21_attention_weight_min": -11.692790985107422,
      "activations/layer22_attention_weight_max": 12.096065521240234,
      "activations/layer22_attention_weight_min": -7.538712978363037,
      "activations/layer23_attention_weight_max": 7.4801435470581055,
      "activations/layer23_attention_weight_min": -5.813140392303467,
      "activations/layer2_attention_weight_max": 21.948848724365234,
      "activations/layer2_attention_weight_min": -20.725318908691406,
      "activations/layer3_attention_weight_max": 62.75611877441406,
      "activations/layer3_attention_weight_min": -52.830589294433594,
      "activations/layer4_attention_weight_max": 161.53099060058594,
      "activations/layer4_attention_weight_min": -148.30209350585938,
      "activations/layer5_attention_weight_max": 187.91246032714844,
      "activations/layer5_attention_weight_min": -173.9012908935547,
      "activations/layer6_attention_weight_max": 141.220703125,
      "activations/layer6_attention_weight_min": -135.90841674804688,
      "activations/layer7_attention_weight_max": 109.97382354736328,
      "activations/layer7_attention_weight_min": -99.63352966308594,
      "activations/layer8_attention_weight_max": 80.98296356201172,
      "activations/layer8_attention_weight_min": -71.58003997802734,
      "activations/layer9_attention_weight_max": 61.67292022705078,
      "activations/layer9_attention_weight_min": -52.32059860229492,
      "epoch": 0.15,
      "learning_rate": 0.00014174416661891364,
      "loss": 3.7955,
      "step": 2550
    },
    {
      "activations/layer0_attention_weight_max": 12.967033386230469,
      "activations/layer0_attention_weight_min": -9.741798400878906,
      "activations/layer10_attention_weight_max": 36.27691650390625,
      "activations/layer10_attention_weight_min": -26.967594146728516,
      "activations/layer11_attention_weight_max": 32.85498809814453,
      "activations/layer11_attention_weight_min": -20.305763244628906,
      "activations/layer12_attention_weight_max": 20.4661865234375,
      "activations/layer12_attention_weight_min": -21.72171974182129,
      "activations/layer13_attention_weight_max": 21.53290367126465,
      "activations/layer13_attention_weight_min": -17.698911666870117,
      "activations/layer14_attention_weight_max": 20.367359161376953,
      "activations/layer14_attention_weight_min": -13.774371147155762,
      "activations/layer15_attention_weight_max": 19.575313568115234,
      "activations/layer15_attention_weight_min": -13.739045143127441,
      "activations/layer16_attention_weight_max": 20.167882919311523,
      "activations/layer16_attention_weight_min": -12.515305519104004,
      "activations/layer17_attention_weight_max": 14.477506637573242,
      "activations/layer17_attention_weight_min": -10.063769340515137,
      "activations/layer18_attention_weight_max": 18.832571029663086,
      "activations/layer18_attention_weight_min": -13.595757484436035,
      "activations/layer19_attention_weight_max": 21.208608627319336,
      "activations/layer19_attention_weight_min": -13.123808860778809,
      "activations/layer1_attention_weight_max": 17.513208389282227,
      "activations/layer1_attention_weight_min": -11.649731636047363,
      "activations/layer20_attention_weight_max": 15.946945190429688,
      "activations/layer20_attention_weight_min": -12.142474174499512,
      "activations/layer21_attention_weight_max": 14.551397323608398,
      "activations/layer21_attention_weight_min": -12.342140197753906,
      "activations/layer22_attention_weight_max": 14.964388847351074,
      "activations/layer22_attention_weight_min": -8.23147964477539,
      "activations/layer23_attention_weight_max": 7.944175720214844,
      "activations/layer23_attention_weight_min": -6.351649761199951,
      "activations/layer2_attention_weight_max": 22.459938049316406,
      "activations/layer2_attention_weight_min": -19.77241325378418,
      "activations/layer3_attention_weight_max": 61.2554817199707,
      "activations/layer3_attention_weight_min": -53.76428985595703,
      "activations/layer4_attention_weight_max": 158.09036254882812,
      "activations/layer4_attention_weight_min": -151.6150360107422,
      "activations/layer5_attention_weight_max": 184.2189178466797,
      "activations/layer5_attention_weight_min": -175.4681854248047,
      "activations/layer6_attention_weight_max": 141.54046630859375,
      "activations/layer6_attention_weight_min": -129.9976043701172,
      "activations/layer7_attention_weight_max": 111.05073547363281,
      "activations/layer7_attention_weight_min": -100.5318832397461,
      "activations/layer8_attention_weight_max": 81.45470428466797,
      "activations/layer8_attention_weight_min": -72.53531646728516,
      "activations/layer9_attention_weight_max": 64.45806884765625,
      "activations/layer9_attention_weight_min": -53.02216720581055,
      "epoch": 0.15,
      "learning_rate": 0.00014209754352529724,
      "loss": 3.783,
      "step": 2600
    },
    {
      "activations/layer0_attention_weight_max": 13.672086715698242,
      "activations/layer0_attention_weight_min": -8.909741401672363,
      "activations/layer10_attention_weight_max": 44.223228454589844,
      "activations/layer10_attention_weight_min": -29.054840087890625,
      "activations/layer11_attention_weight_max": 36.809329986572266,
      "activations/layer11_attention_weight_min": -19.782894134521484,
      "activations/layer12_attention_weight_max": 22.36102294921875,
      "activations/layer12_attention_weight_min": -20.846542358398438,
      "activations/layer13_attention_weight_max": 23.21906852722168,
      "activations/layer13_attention_weight_min": -17.523237228393555,
      "activations/layer14_attention_weight_max": 19.377914428710938,
      "activations/layer14_attention_weight_min": -12.39807415008545,
      "activations/layer15_attention_weight_max": 19.649181365966797,
      "activations/layer15_attention_weight_min": -13.490753173828125,
      "activations/layer16_attention_weight_max": 20.157100677490234,
      "activations/layer16_attention_weight_min": -13.033227920532227,
      "activations/layer17_attention_weight_max": 14.831207275390625,
      "activations/layer17_attention_weight_min": -9.533576965332031,
      "activations/layer18_attention_weight_max": 14.994525909423828,
      "activations/layer18_attention_weight_min": -12.389185905456543,
      "activations/layer19_attention_weight_max": 16.259138107299805,
      "activations/layer19_attention_weight_min": -13.59873104095459,
      "activations/layer1_attention_weight_max": 16.445642471313477,
      "activations/layer1_attention_weight_min": -12.287477493286133,
      "activations/layer20_attention_weight_max": 13.3643159866333,
      "activations/layer20_attention_weight_min": -11.11626148223877,
      "activations/layer21_attention_weight_max": 18.236312866210938,
      "activations/layer21_attention_weight_min": -11.289041519165039,
      "activations/layer22_attention_weight_max": 13.867508888244629,
      "activations/layer22_attention_weight_min": -8.326803207397461,
      "activations/layer23_attention_weight_max": 6.569421291351318,
      "activations/layer23_attention_weight_min": -6.541033744812012,
      "activations/layer2_attention_weight_max": 22.469165802001953,
      "activations/layer2_attention_weight_min": -20.827539443969727,
      "activations/layer3_attention_weight_max": 60.48124313354492,
      "activations/layer3_attention_weight_min": -54.25954055786133,
      "activations/layer4_attention_weight_max": 159.48837280273438,
      "activations/layer4_attention_weight_min": -152.5643768310547,
      "activations/layer5_attention_weight_max": 177.85284423828125,
      "activations/layer5_attention_weight_min": -172.75718688964844,
      "activations/layer6_attention_weight_max": 129.46653747558594,
      "activations/layer6_attention_weight_min": -127.20814514160156,
      "activations/layer7_attention_weight_max": 103.9542007446289,
      "activations/layer7_attention_weight_min": -102.11736297607422,
      "activations/layer8_attention_weight_max": 77.37020874023438,
      "activations/layer8_attention_weight_min": -75.83116912841797,
      "activations/layer9_attention_weight_max": 62.2458610534668,
      "activations/layer9_attention_weight_min": -53.02511215209961,
      "epoch": 0.15,
      "learning_rate": 0.00014244414774577482,
      "loss": 3.7914,
      "step": 2650
    },
    {
      "activations/layer0_attention_weight_max": 13.07490348815918,
      "activations/layer0_attention_weight_min": -8.892959594726562,
      "activations/layer10_attention_weight_max": 34.8454704284668,
      "activations/layer10_attention_weight_min": -25.75456428527832,
      "activations/layer11_attention_weight_max": 31.895591735839844,
      "activations/layer11_attention_weight_min": -23.056194305419922,
      "activations/layer12_attention_weight_max": 20.545852661132812,
      "activations/layer12_attention_weight_min": -19.304616928100586,
      "activations/layer13_attention_weight_max": 18.59347915649414,
      "activations/layer13_attention_weight_min": -17.601919174194336,
      "activations/layer14_attention_weight_max": 18.57115936279297,
      "activations/layer14_attention_weight_min": -13.884905815124512,
      "activations/layer15_attention_weight_max": 18.625934600830078,
      "activations/layer15_attention_weight_min": -12.222514152526855,
      "activations/layer16_attention_weight_max": 23.74163818359375,
      "activations/layer16_attention_weight_min": -11.96430778503418,
      "activations/layer17_attention_weight_max": 16.228193283081055,
      "activations/layer17_attention_weight_min": -8.941152572631836,
      "activations/layer18_attention_weight_max": 16.270889282226562,
      "activations/layer18_attention_weight_min": -13.6766357421875,
      "activations/layer19_attention_weight_max": 20.11684226989746,
      "activations/layer19_attention_weight_min": -11.795644760131836,
      "activations/layer1_attention_weight_max": 15.594063758850098,
      "activations/layer1_attention_weight_min": -10.617219924926758,
      "activations/layer20_attention_weight_max": 16.885414123535156,
      "activations/layer20_attention_weight_min": -11.231467247009277,
      "activations/layer21_attention_weight_max": 12.867162704467773,
      "activations/layer21_attention_weight_min": -11.097763061523438,
      "activations/layer22_attention_weight_max": 12.5206937789917,
      "activations/layer22_attention_weight_min": -7.812560081481934,
      "activations/layer23_attention_weight_max": 8.56795883178711,
      "activations/layer23_attention_weight_min": -6.343891143798828,
      "activations/layer2_attention_weight_max": 23.255895614624023,
      "activations/layer2_attention_weight_min": -21.169429779052734,
      "activations/layer3_attention_weight_max": 60.4275016784668,
      "activations/layer3_attention_weight_min": -52.83078384399414,
      "activations/layer4_attention_weight_max": 156.7965850830078,
      "activations/layer4_attention_weight_min": -152.04286193847656,
      "activations/layer5_attention_weight_max": 177.8447723388672,
      "activations/layer5_attention_weight_min": -172.45089721679688,
      "activations/layer6_attention_weight_max": 135.24794006347656,
      "activations/layer6_attention_weight_min": -133.8808135986328,
      "activations/layer7_attention_weight_max": 105.27529907226562,
      "activations/layer7_attention_weight_min": -105.82442474365234,
      "activations/layer8_attention_weight_max": 84.3429946899414,
      "activations/layer8_attention_weight_min": -72.56015014648438,
      "activations/layer9_attention_weight_max": 68.40088653564453,
      "activations/layer9_attention_weight_min": -53.08449935913086,
      "epoch": 0.16,
      "learning_rate": 0.00014278423401130834,
      "loss": 3.762,
      "step": 2700
    },
    {
      "activations/layer0_attention_weight_max": 13.433781623840332,
      "activations/layer0_attention_weight_min": -8.900493621826172,
      "activations/layer10_attention_weight_max": 36.35483169555664,
      "activations/layer10_attention_weight_min": -26.448137283325195,
      "activations/layer11_attention_weight_max": 39.302616119384766,
      "activations/layer11_attention_weight_min": -21.932334899902344,
      "activations/layer12_attention_weight_max": 24.6042423248291,
      "activations/layer12_attention_weight_min": -20.509016036987305,
      "activations/layer13_attention_weight_max": 24.12970542907715,
      "activations/layer13_attention_weight_min": -16.50205421447754,
      "activations/layer14_attention_weight_max": 24.1131649017334,
      "activations/layer14_attention_weight_min": -13.993025779724121,
      "activations/layer15_attention_weight_max": 23.646522521972656,
      "activations/layer15_attention_weight_min": -11.928865432739258,
      "activations/layer16_attention_weight_max": 24.194622039794922,
      "activations/layer16_attention_weight_min": -11.647083282470703,
      "activations/layer17_attention_weight_max": 18.771343231201172,
      "activations/layer17_attention_weight_min": -9.74742603302002,
      "activations/layer18_attention_weight_max": 22.695140838623047,
      "activations/layer18_attention_weight_min": -13.33958911895752,
      "activations/layer19_attention_weight_max": 20.714813232421875,
      "activations/layer19_attention_weight_min": -11.966841697692871,
      "activations/layer1_attention_weight_max": 17.81293296813965,
      "activations/layer1_attention_weight_min": -12.185832977294922,
      "activations/layer20_attention_weight_max": 16.882469177246094,
      "activations/layer20_attention_weight_min": -10.705175399780273,
      "activations/layer21_attention_weight_max": 18.91950035095215,
      "activations/layer21_attention_weight_min": -12.648536682128906,
      "activations/layer22_attention_weight_max": 17.33106803894043,
      "activations/layer22_attention_weight_min": -9.528367042541504,
      "activations/layer23_attention_weight_max": 7.766864776611328,
      "activations/layer23_attention_weight_min": -7.646862983703613,
      "activations/layer2_attention_weight_max": 24.925495147705078,
      "activations/layer2_attention_weight_min": -20.919254302978516,
      "activations/layer3_attention_weight_max": 66.66978454589844,
      "activations/layer3_attention_weight_min": -51.62762451171875,
      "activations/layer4_attention_weight_max": 166.8305206298828,
      "activations/layer4_attention_weight_min": -153.18492126464844,
      "activations/layer5_attention_weight_max": 199.031982421875,
      "activations/layer5_attention_weight_min": -176.39059448242188,
      "activations/layer6_attention_weight_max": 141.4888153076172,
      "activations/layer6_attention_weight_min": -121.24158477783203,
      "activations/layer7_attention_weight_max": 122.11387634277344,
      "activations/layer7_attention_weight_min": -101.0201416015625,
      "activations/layer8_attention_weight_max": 85.98689270019531,
      "activations/layer8_attention_weight_min": -69.95303344726562,
      "activations/layer9_attention_weight_max": 72.60901641845703,
      "activations/layer9_attention_weight_min": -53.32540512084961,
      "epoch": 0.16,
      "learning_rate": 0.00014311804294650713,
      "loss": 3.7435,
      "step": 2750
    },
    {
      "activations/layer0_attention_weight_max": 13.12337875366211,
      "activations/layer0_attention_weight_min": -9.592951774597168,
      "activations/layer10_attention_weight_max": 43.17563247680664,
      "activations/layer10_attention_weight_min": -28.7509822845459,
      "activations/layer11_attention_weight_max": 38.84207534790039,
      "activations/layer11_attention_weight_min": -23.035884857177734,
      "activations/layer12_attention_weight_max": 23.99844741821289,
      "activations/layer12_attention_weight_min": -21.219547271728516,
      "activations/layer13_attention_weight_max": 22.936412811279297,
      "activations/layer13_attention_weight_min": -21.172607421875,
      "activations/layer14_attention_weight_max": 23.211990356445312,
      "activations/layer14_attention_weight_min": -13.673622131347656,
      "activations/layer15_attention_weight_max": 23.554443359375,
      "activations/layer15_attention_weight_min": -13.287631034851074,
      "activations/layer16_attention_weight_max": 20.164758682250977,
      "activations/layer16_attention_weight_min": -12.23344898223877,
      "activations/layer17_attention_weight_max": 15.231375694274902,
      "activations/layer17_attention_weight_min": -10.512578010559082,
      "activations/layer18_attention_weight_max": 21.95427131652832,
      "activations/layer18_attention_weight_min": -14.186025619506836,
      "activations/layer19_attention_weight_max": 19.749794006347656,
      "activations/layer19_attention_weight_min": -12.524985313415527,
      "activations/layer1_attention_weight_max": 19.046363830566406,
      "activations/layer1_attention_weight_min": -11.576767921447754,
      "activations/layer20_attention_weight_max": 17.862579345703125,
      "activations/layer20_attention_weight_min": -12.240117073059082,
      "activations/layer21_attention_weight_max": 18.015745162963867,
      "activations/layer21_attention_weight_min": -12.547340393066406,
      "activations/layer22_attention_weight_max": 13.48764705657959,
      "activations/layer22_attention_weight_min": -8.292379379272461,
      "activations/layer23_attention_weight_max": 7.816697597503662,
      "activations/layer23_attention_weight_min": -7.045307159423828,
      "activations/layer2_attention_weight_max": 24.751651763916016,
      "activations/layer2_attention_weight_min": -21.861557006835938,
      "activations/layer3_attention_weight_max": 64.29911804199219,
      "activations/layer3_attention_weight_min": -52.79383850097656,
      "activations/layer4_attention_weight_max": 166.9470672607422,
      "activations/layer4_attention_weight_min": -149.81944274902344,
      "activations/layer5_attention_weight_max": 188.6512908935547,
      "activations/layer5_attention_weight_min": -173.79473876953125,
      "activations/layer6_attention_weight_max": 139.96823120117188,
      "activations/layer6_attention_weight_min": -133.57675170898438,
      "activations/layer7_attention_weight_max": 108.80704498291016,
      "activations/layer7_attention_weight_min": -98.19630432128906,
      "activations/layer8_attention_weight_max": 83.86209106445312,
      "activations/layer8_attention_weight_min": -70.4451675415039,
      "activations/layer9_attention_weight_max": 71.66526794433594,
      "activations/layer9_attention_weight_min": -53.176021575927734,
      "epoch": 0.16,
      "learning_rate": 0.00014344580209234327,
      "loss": 3.7352,
      "step": 2800
    },
    {
      "activations/layer0_attention_weight_max": 13.430370330810547,
      "activations/layer0_attention_weight_min": -8.712321281433105,
      "activations/layer10_attention_weight_max": 38.92280578613281,
      "activations/layer10_attention_weight_min": -29.07604217529297,
      "activations/layer11_attention_weight_max": 26.010778427124023,
      "activations/layer11_attention_weight_min": -20.900970458984375,
      "activations/layer12_attention_weight_max": 24.110618591308594,
      "activations/layer12_attention_weight_min": -19.411020278930664,
      "activations/layer13_attention_weight_max": 23.737295150756836,
      "activations/layer13_attention_weight_min": -16.639081954956055,
      "activations/layer14_attention_weight_max": 24.077390670776367,
      "activations/layer14_attention_weight_min": -14.573822975158691,
      "activations/layer15_attention_weight_max": 21.661136627197266,
      "activations/layer15_attention_weight_min": -13.621636390686035,
      "activations/layer16_attention_weight_max": 26.88298225402832,
      "activations/layer16_attention_weight_min": -14.12757396697998,
      "activations/layer17_attention_weight_max": 17.057857513427734,
      "activations/layer17_attention_weight_min": -10.981144905090332,
      "activations/layer18_attention_weight_max": 19.426372528076172,
      "activations/layer18_attention_weight_min": -15.296773910522461,
      "activations/layer19_attention_weight_max": 24.98945426940918,
      "activations/layer19_attention_weight_min": -12.821596145629883,
      "activations/layer1_attention_weight_max": 15.848090171813965,
      "activations/layer1_attention_weight_min": -10.882149696350098,
      "activations/layer20_attention_weight_max": 19.324344635009766,
      "activations/layer20_attention_weight_min": -13.539562225341797,
      "activations/layer21_attention_weight_max": 17.843860626220703,
      "activations/layer21_attention_weight_min": -12.91589069366455,
      "activations/layer22_attention_weight_max": 15.985733032226562,
      "activations/layer22_attention_weight_min": -9.951539993286133,
      "activations/layer23_attention_weight_max": 8.151185989379883,
      "activations/layer23_attention_weight_min": -8.127656936645508,
      "activations/layer2_attention_weight_max": 22.50621795654297,
      "activations/layer2_attention_weight_min": -20.040924072265625,
      "activations/layer3_attention_weight_max": 60.7632942199707,
      "activations/layer3_attention_weight_min": -52.12124252319336,
      "activations/layer4_attention_weight_max": 158.96205139160156,
      "activations/layer4_attention_weight_min": -150.57186889648438,
      "activations/layer5_attention_weight_max": 175.9322509765625,
      "activations/layer5_attention_weight_min": -168.36965942382812,
      "activations/layer6_attention_weight_max": 129.58775329589844,
      "activations/layer6_attention_weight_min": -124.21682739257812,
      "activations/layer7_attention_weight_max": 110.9037094116211,
      "activations/layer7_attention_weight_min": -96.90318298339844,
      "activations/layer8_attention_weight_max": 77.61205291748047,
      "activations/layer8_attention_weight_min": -73.12076568603516,
      "activations/layer9_attention_weight_max": 70.71025085449219,
      "activations/layer9_attention_weight_min": -59.455867767333984,
      "epoch": 0.17,
      "learning_rate": 0.0001437677268378304,
      "loss": 3.7277,
      "step": 2850
    },
    {
      "activations/layer0_attention_weight_max": 14.11849594116211,
      "activations/layer0_attention_weight_min": -9.124540328979492,
      "activations/layer10_attention_weight_max": 35.957733154296875,
      "activations/layer10_attention_weight_min": -25.660634994506836,
      "activations/layer11_attention_weight_max": 31.85699462890625,
      "activations/layer11_attention_weight_min": -21.557044982910156,
      "activations/layer12_attention_weight_max": 24.155235290527344,
      "activations/layer12_attention_weight_min": -21.893476486206055,
      "activations/layer13_attention_weight_max": 21.65935516357422,
      "activations/layer13_attention_weight_min": -17.712783813476562,
      "activations/layer14_attention_weight_max": 22.32746696472168,
      "activations/layer14_attention_weight_min": -14.331345558166504,
      "activations/layer15_attention_weight_max": 22.311494827270508,
      "activations/layer15_attention_weight_min": -11.355297088623047,
      "activations/layer16_attention_weight_max": 23.63532257080078,
      "activations/layer16_attention_weight_min": -13.694052696228027,
      "activations/layer17_attention_weight_max": 17.711381912231445,
      "activations/layer17_attention_weight_min": -10.791638374328613,
      "activations/layer18_attention_weight_max": 25.013126373291016,
      "activations/layer18_attention_weight_min": -13.395493507385254,
      "activations/layer19_attention_weight_max": 22.984127044677734,
      "activations/layer19_attention_weight_min": -13.754565238952637,
      "activations/layer1_attention_weight_max": 18.225820541381836,
      "activations/layer1_attention_weight_min": -11.617091178894043,
      "activations/layer20_attention_weight_max": 21.895580291748047,
      "activations/layer20_attention_weight_min": -12.496569633483887,
      "activations/layer21_attention_weight_max": 17.29429054260254,
      "activations/layer21_attention_weight_min": -13.145302772521973,
      "activations/layer22_attention_weight_max": 15.514654159545898,
      "activations/layer22_attention_weight_min": -9.336268424987793,
      "activations/layer23_attention_weight_max": 8.310026168823242,
      "activations/layer23_attention_weight_min": -8.673492431640625,
      "activations/layer2_attention_weight_max": 23.6712703704834,
      "activations/layer2_attention_weight_min": -21.35914421081543,
      "activations/layer3_attention_weight_max": 61.328433990478516,
      "activations/layer3_attention_weight_min": -52.69099807739258,
      "activations/layer4_attention_weight_max": 157.9003143310547,
      "activations/layer4_attention_weight_min": -152.9048309326172,
      "activations/layer5_attention_weight_max": 177.23797607421875,
      "activations/layer5_attention_weight_min": -175.58580017089844,
      "activations/layer6_attention_weight_max": 128.51683044433594,
      "activations/layer6_attention_weight_min": -122.90850830078125,
      "activations/layer7_attention_weight_max": 102.79135131835938,
      "activations/layer7_attention_weight_min": -95.74266052246094,
      "activations/layer8_attention_weight_max": 76.0964584350586,
      "activations/layer8_attention_weight_min": -68.24772644042969,
      "activations/layer9_attention_weight_max": 58.23503494262695,
      "activations/layer9_attention_weight_min": -52.35246658325195,
      "epoch": 0.17,
      "learning_rate": 0.00014408402127022058,
      "loss": 3.7076,
      "step": 2900
    },
    {
      "activations/layer0_attention_weight_max": 14.137210845947266,
      "activations/layer0_attention_weight_min": -10.113663673400879,
      "activations/layer10_attention_weight_max": 36.64921569824219,
      "activations/layer10_attention_weight_min": -25.32879066467285,
      "activations/layer11_attention_weight_max": 40.45020294189453,
      "activations/layer11_attention_weight_min": -23.553226470947266,
      "activations/layer12_attention_weight_max": 28.69745445251465,
      "activations/layer12_attention_weight_min": -25.1790828704834,
      "activations/layer13_attention_weight_max": 23.58930778503418,
      "activations/layer13_attention_weight_min": -15.276832580566406,
      "activations/layer14_attention_weight_max": 23.88372802734375,
      "activations/layer14_attention_weight_min": -13.95869255065918,
      "activations/layer15_attention_weight_max": 24.818084716796875,
      "activations/layer15_attention_weight_min": -12.339730262756348,
      "activations/layer16_attention_weight_max": 24.10440444946289,
      "activations/layer16_attention_weight_min": -12.010231971740723,
      "activations/layer17_attention_weight_max": 19.44473648071289,
      "activations/layer17_attention_weight_min": -10.39803695678711,
      "activations/layer18_attention_weight_max": 24.343276977539062,
      "activations/layer18_attention_weight_min": -11.368767738342285,
      "activations/layer19_attention_weight_max": 20.202342987060547,
      "activations/layer19_attention_weight_min": -10.869688987731934,
      "activations/layer1_attention_weight_max": 17.822296142578125,
      "activations/layer1_attention_weight_min": -10.717334747314453,
      "activations/layer20_attention_weight_max": 19.825469970703125,
      "activations/layer20_attention_weight_min": -12.062071800231934,
      "activations/layer21_attention_weight_max": 17.72903823852539,
      "activations/layer21_attention_weight_min": -11.377418518066406,
      "activations/layer22_attention_weight_max": 16.672714233398438,
      "activations/layer22_attention_weight_min": -8.121467590332031,
      "activations/layer23_attention_weight_max": 8.872220993041992,
      "activations/layer23_attention_weight_min": -8.732183456420898,
      "activations/layer2_attention_weight_max": 24.459701538085938,
      "activations/layer2_attention_weight_min": -20.065990447998047,
      "activations/layer3_attention_weight_max": 62.824180603027344,
      "activations/layer3_attention_weight_min": -50.622337341308594,
      "activations/layer4_attention_weight_max": 161.66822814941406,
      "activations/layer4_attention_weight_min": -148.47190856933594,
      "activations/layer5_attention_weight_max": 181.6490936279297,
      "activations/layer5_attention_weight_min": -169.55126953125,
      "activations/layer6_attention_weight_max": 136.07571411132812,
      "activations/layer6_attention_weight_min": -124.84120178222656,
      "activations/layer7_attention_weight_max": 106.2696533203125,
      "activations/layer7_attention_weight_min": -93.68978881835938,
      "activations/layer8_attention_weight_max": 81.98639678955078,
      "activations/layer8_attention_weight_min": -69.307861328125,
      "activations/layer9_attention_weight_max": 59.35007858276367,
      "activations/layer9_attention_weight_min": -52.40576171875,
      "epoch": 0.17,
      "learning_rate": 0.0001443948789521239,
      "loss": 3.7194,
      "step": 2950
    },
    {
      "activations/layer0_attention_weight_max": 12.73944091796875,
      "activations/layer0_attention_weight_min": -8.610001564025879,
      "activations/layer10_attention_weight_max": 35.954490661621094,
      "activations/layer10_attention_weight_min": -26.890336990356445,
      "activations/layer11_attention_weight_max": 37.27819061279297,
      "activations/layer11_attention_weight_min": -23.986236572265625,
      "activations/layer12_attention_weight_max": 20.87027931213379,
      "activations/layer12_attention_weight_min": -20.698270797729492,
      "activations/layer13_attention_weight_max": 22.753854751586914,
      "activations/layer13_attention_weight_min": -19.759864807128906,
      "activations/layer14_attention_weight_max": 22.466564178466797,
      "activations/layer14_attention_weight_min": -15.319480895996094,
      "activations/layer15_attention_weight_max": 21.0964298248291,
      "activations/layer15_attention_weight_min": -12.996329307556152,
      "activations/layer16_attention_weight_max": 19.886335372924805,
      "activations/layer16_attention_weight_min": -13.436575889587402,
      "activations/layer17_attention_weight_max": 19.042613983154297,
      "activations/layer17_attention_weight_min": -12.064197540283203,
      "activations/layer18_attention_weight_max": 19.801254272460938,
      "activations/layer18_attention_weight_min": -13.084330558776855,
      "activations/layer19_attention_weight_max": 22.17129135131836,
      "activations/layer19_attention_weight_min": -14.1301908493042,
      "activations/layer1_attention_weight_max": 18.551565170288086,
      "activations/layer1_attention_weight_min": -11.59732437133789,
      "activations/layer20_attention_weight_max": 18.505895614624023,
      "activations/layer20_attention_weight_min": -12.74280071258545,
      "activations/layer21_attention_weight_max": 16.73594093322754,
      "activations/layer21_attention_weight_min": -12.321375846862793,
      "activations/layer22_attention_weight_max": 14.882792472839355,
      "activations/layer22_attention_weight_min": -8.842321395874023,
      "activations/layer23_attention_weight_max": 8.928099632263184,
      "activations/layer23_attention_weight_min": -8.574618339538574,
      "activations/layer2_attention_weight_max": 24.098173141479492,
      "activations/layer2_attention_weight_min": -19.644853591918945,
      "activations/layer3_attention_weight_max": 61.469852447509766,
      "activations/layer3_attention_weight_min": -50.81597137451172,
      "activations/layer4_attention_weight_max": 161.3725128173828,
      "activations/layer4_attention_weight_min": -147.40513610839844,
      "activations/layer5_attention_weight_max": 189.76284790039062,
      "activations/layer5_attention_weight_min": -164.48953247070312,
      "activations/layer6_attention_weight_max": 142.76890563964844,
      "activations/layer6_attention_weight_min": -120.50352478027344,
      "activations/layer7_attention_weight_max": 119.2403335571289,
      "activations/layer7_attention_weight_min": -94.70045471191406,
      "activations/layer8_attention_weight_max": 83.88311004638672,
      "activations/layer8_attention_weight_min": -69.257568359375,
      "activations/layer9_attention_weight_max": 68.79110717773438,
      "activations/layer9_attention_weight_min": -51.2241096496582,
      "epoch": 0.17,
      "learning_rate": 0.00014470048363296077,
      "loss": 3.6946,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_loss": 3.630859375,
      "eval_runtime": 8.2635,
      "eval_samples_per_second": 519.633,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_openwebtext_loss": 3.630859375,
      "eval_openwebtext_ppl": 37.74523999888851,
      "eval_openwebtext_runtime": 8.2635,
      "eval_openwebtext_samples_per_second": 519.633,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_wikitext_loss": 4.015625,
      "eval_wikitext_ppl": 55.457945789238345,
      "eval_wikitext_runtime": 1.6933,
      "eval_wikitext_samples_per_second": 269.294,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_lambada_loss": 4.7421875,
      "eval_lambada_ppl": 114.68480048885151,
      "eval_lambada_runtime": 9.3766,
      "eval_lambada_samples_per_second": 519.271,
      "step": 3000
    },
    {
      "activations/layer0_attention_weight_max": 13.13752269744873,
      "activations/layer0_attention_weight_min": -8.76103401184082,
      "activations/layer10_attention_weight_max": 34.562713623046875,
      "activations/layer10_attention_weight_min": -25.994365692138672,
      "activations/layer11_attention_weight_max": 35.77580642700195,
      "activations/layer11_attention_weight_min": -27.662555694580078,
      "activations/layer12_attention_weight_max": 22.640790939331055,
      "activations/layer12_attention_weight_min": -21.921260833740234,
      "activations/layer13_attention_weight_max": 19.590625762939453,
      "activations/layer13_attention_weight_min": -18.038312911987305,
      "activations/layer14_attention_weight_max": 23.633604049682617,
      "activations/layer14_attention_weight_min": -15.414169311523438,
      "activations/layer15_attention_weight_max": 25.323720932006836,
      "activations/layer15_attention_weight_min": -16.752283096313477,
      "activations/layer16_attention_weight_max": 19.249116897583008,
      "activations/layer16_attention_weight_min": -13.878152847290039,
      "activations/layer17_attention_weight_max": 17.08392906188965,
      "activations/layer17_attention_weight_min": -10.504219055175781,
      "activations/layer18_attention_weight_max": 17.40072250366211,
      "activations/layer18_attention_weight_min": -12.835348129272461,
      "activations/layer19_attention_weight_max": 19.459245681762695,
      "activations/layer19_attention_weight_min": -12.074533462524414,
      "activations/layer1_attention_weight_max": 17.09290313720703,
      "activations/layer1_attention_weight_min": -11.28188705444336,
      "activations/layer20_attention_weight_max": 16.938261032104492,
      "activations/layer20_attention_weight_min": -12.277071952819824,
      "activations/layer21_attention_weight_max": 16.90868377685547,
      "activations/layer21_attention_weight_min": -12.201228141784668,
      "activations/layer22_attention_weight_max": 15.311296463012695,
      "activations/layer22_attention_weight_min": -9.929701805114746,
      "activations/layer23_attention_weight_max": 7.85218620300293,
      "activations/layer23_attention_weight_min": -8.4920015335083,
      "activations/layer2_attention_weight_max": 23.62366485595703,
      "activations/layer2_attention_weight_min": -21.535221099853516,
      "activations/layer3_attention_weight_max": 58.768394470214844,
      "activations/layer3_attention_weight_min": -51.34061050415039,
      "activations/layer4_attention_weight_max": 157.88575744628906,
      "activations/layer4_attention_weight_min": -150.61631774902344,
      "activations/layer5_attention_weight_max": 176.36981201171875,
      "activations/layer5_attention_weight_min": -170.3836669921875,
      "activations/layer6_attention_weight_max": 127.70266723632812,
      "activations/layer6_attention_weight_min": -121.92044830322266,
      "activations/layer7_attention_weight_max": 102.2798843383789,
      "activations/layer7_attention_weight_min": -93.57756042480469,
      "activations/layer8_attention_weight_max": 76.5541763305664,
      "activations/layer8_attention_weight_min": -70.29752349853516,
      "activations/layer9_attention_weight_max": 58.3681755065918,
      "activations/layer9_attention_weight_min": -54.165401458740234,
      "epoch": 0.18,
      "learning_rate": 0.00014499504805694574,
      "loss": 3.687,
      "step": 3050
    },
    {
      "activations/layer0_attention_weight_max": 14.145526885986328,
      "activations/layer0_attention_weight_min": -8.999471664428711,
      "activations/layer10_attention_weight_max": 37.55641555786133,
      "activations/layer10_attention_weight_min": -28.687082290649414,
      "activations/layer11_attention_weight_max": 34.08597946166992,
      "activations/layer11_attention_weight_min": -22.877788543701172,
      "activations/layer12_attention_weight_max": 20.83850860595703,
      "activations/layer12_attention_weight_min": -17.41042709350586,
      "activations/layer13_attention_weight_max": 19.34124183654785,
      "activations/layer13_attention_weight_min": -16.731876373291016,
      "activations/layer14_attention_weight_max": 22.291791915893555,
      "activations/layer14_attention_weight_min": -14.761940002441406,
      "activations/layer15_attention_weight_max": 19.191280364990234,
      "activations/layer15_attention_weight_min": -11.273821830749512,
      "activations/layer16_attention_weight_max": 19.86884880065918,
      "activations/layer16_attention_weight_min": -12.694581031799316,
      "activations/layer17_attention_weight_max": 19.361343383789062,
      "activations/layer17_attention_weight_min": -10.544676780700684,
      "activations/layer18_attention_weight_max": 18.173397064208984,
      "activations/layer18_attention_weight_min": -13.088618278503418,
      "activations/layer19_attention_weight_max": 21.02631950378418,
      "activations/layer19_attention_weight_min": -11.638443946838379,
      "activations/layer1_attention_weight_max": 17.539657592773438,
      "activations/layer1_attention_weight_min": -11.01107406616211,
      "activations/layer20_attention_weight_max": 17.670461654663086,
      "activations/layer20_attention_weight_min": -12.370611190795898,
      "activations/layer21_attention_weight_max": 16.38695526123047,
      "activations/layer21_attention_weight_min": -12.054055213928223,
      "activations/layer22_attention_weight_max": 15.232359886169434,
      "activations/layer22_attention_weight_min": -9.544184684753418,
      "activations/layer23_attention_weight_max": 8.287833213806152,
      "activations/layer23_attention_weight_min": -8.488462448120117,
      "activations/layer2_attention_weight_max": 25.624874114990234,
      "activations/layer2_attention_weight_min": -21.220726013183594,
      "activations/layer3_attention_weight_max": 61.440589904785156,
      "activations/layer3_attention_weight_min": -50.91205978393555,
      "activations/layer4_attention_weight_max": 160.3861541748047,
      "activations/layer4_attention_weight_min": -148.16773986816406,
      "activations/layer5_attention_weight_max": 177.70677185058594,
      "activations/layer5_attention_weight_min": -164.72854614257812,
      "activations/layer6_attention_weight_max": 137.27301025390625,
      "activations/layer6_attention_weight_min": -123.38810729980469,
      "activations/layer7_attention_weight_max": 108.87088775634766,
      "activations/layer7_attention_weight_min": -94.44677734375,
      "activations/layer8_attention_weight_max": 88.23362731933594,
      "activations/layer8_attention_weight_min": -66.63688659667969,
      "activations/layer9_attention_weight_max": 71.2160873413086,
      "activations/layer9_attention_weight_min": -53.43227005004883,
      "epoch": 0.18,
      "learning_rate": 0.00014529075861283353,
      "loss": 3.6807,
      "step": 3100
    },
    {
      "activations/layer0_attention_weight_max": 13.712576866149902,
      "activations/layer0_attention_weight_min": -9.656902313232422,
      "activations/layer10_attention_weight_max": 40.989559173583984,
      "activations/layer10_attention_weight_min": -29.42807388305664,
      "activations/layer11_attention_weight_max": 34.642845153808594,
      "activations/layer11_attention_weight_min": -23.793964385986328,
      "activations/layer12_attention_weight_max": 21.08255958557129,
      "activations/layer12_attention_weight_min": -19.997146606445312,
      "activations/layer13_attention_weight_max": 18.866596221923828,
      "activations/layer13_attention_weight_min": -18.247081756591797,
      "activations/layer14_attention_weight_max": 23.470735549926758,
      "activations/layer14_attention_weight_min": -15.611681938171387,
      "activations/layer15_attention_weight_max": 18.343765258789062,
      "activations/layer15_attention_weight_min": -12.727376937866211,
      "activations/layer16_attention_weight_max": 22.09613037109375,
      "activations/layer16_attention_weight_min": -13.473134994506836,
      "activations/layer17_attention_weight_max": 19.04656219482422,
      "activations/layer17_attention_weight_min": -11.215994834899902,
      "activations/layer18_attention_weight_max": 23.027862548828125,
      "activations/layer18_attention_weight_min": -16.275480270385742,
      "activations/layer19_attention_weight_max": 20.2823486328125,
      "activations/layer19_attention_weight_min": -12.637846946716309,
      "activations/layer1_attention_weight_max": 17.98929786682129,
      "activations/layer1_attention_weight_min": -11.56220531463623,
      "activations/layer20_attention_weight_max": 16.748435974121094,
      "activations/layer20_attention_weight_min": -12.317992210388184,
      "activations/layer21_attention_weight_max": 16.263303756713867,
      "activations/layer21_attention_weight_min": -12.773247718811035,
      "activations/layer22_attention_weight_max": 15.366037368774414,
      "activations/layer22_attention_weight_min": -9.156548500061035,
      "activations/layer23_attention_weight_max": 7.5686798095703125,
      "activations/layer23_attention_weight_min": -9.115039825439453,
      "activations/layer2_attention_weight_max": 24.622207641601562,
      "activations/layer2_attention_weight_min": -21.68312644958496,
      "activations/layer3_attention_weight_max": 62.4974250793457,
      "activations/layer3_attention_weight_min": -51.34059524536133,
      "activations/layer4_attention_weight_max": 155.41236877441406,
      "activations/layer4_attention_weight_min": -150.71226501464844,
      "activations/layer5_attention_weight_max": 171.01339721679688,
      "activations/layer5_attention_weight_min": -169.87582397460938,
      "activations/layer6_attention_weight_max": 123.15889739990234,
      "activations/layer6_attention_weight_min": -121.7934341430664,
      "activations/layer7_attention_weight_max": 100.95193481445312,
      "activations/layer7_attention_weight_min": -93.8010025024414,
      "activations/layer8_attention_weight_max": 77.7220458984375,
      "activations/layer8_attention_weight_min": -69.19061279296875,
      "activations/layer9_attention_weight_max": 62.86725997924805,
      "activations/layer9_attention_weight_min": -53.706295013427734,
      "epoch": 0.18,
      "learning_rate": 0.00014558171171350442,
      "loss": 3.6609,
      "step": 3150
    },
    {
      "activations/layer0_attention_weight_max": 12.708553314208984,
      "activations/layer0_attention_weight_min": -9.888618469238281,
      "activations/layer10_attention_weight_max": 35.462120056152344,
      "activations/layer10_attention_weight_min": -26.614715576171875,
      "activations/layer11_attention_weight_max": 30.946155548095703,
      "activations/layer11_attention_weight_min": -24.161184310913086,
      "activations/layer12_attention_weight_max": 24.260358810424805,
      "activations/layer12_attention_weight_min": -19.22136878967285,
      "activations/layer13_attention_weight_max": 18.91632843017578,
      "activations/layer13_attention_weight_min": -16.809263229370117,
      "activations/layer14_attention_weight_max": 19.368886947631836,
      "activations/layer14_attention_weight_min": -14.657170295715332,
      "activations/layer15_attention_weight_max": 18.113740921020508,
      "activations/layer15_attention_weight_min": -15.050871849060059,
      "activations/layer16_attention_weight_max": 19.476009368896484,
      "activations/layer16_attention_weight_min": -12.986936569213867,
      "activations/layer17_attention_weight_max": 17.400415420532227,
      "activations/layer17_attention_weight_min": -12.89144515991211,
      "activations/layer18_attention_weight_max": 24.01397705078125,
      "activations/layer18_attention_weight_min": -11.890233039855957,
      "activations/layer19_attention_weight_max": 19.47407341003418,
      "activations/layer19_attention_weight_min": -11.821624755859375,
      "activations/layer1_attention_weight_max": 17.943775177001953,
      "activations/layer1_attention_weight_min": -11.500483512878418,
      "activations/layer20_attention_weight_max": 20.173959732055664,
      "activations/layer20_attention_weight_min": -12.390974044799805,
      "activations/layer21_attention_weight_max": 19.702434539794922,
      "activations/layer21_attention_weight_min": -11.659634590148926,
      "activations/layer22_attention_weight_max": 14.647586822509766,
      "activations/layer22_attention_weight_min": -9.502433776855469,
      "activations/layer23_attention_weight_max": 8.643478393554688,
      "activations/layer23_attention_weight_min": -8.828849792480469,
      "activations/layer2_attention_weight_max": 26.189159393310547,
      "activations/layer2_attention_weight_min": -21.70520782470703,
      "activations/layer3_attention_weight_max": 62.342533111572266,
      "activations/layer3_attention_weight_min": -50.82085418701172,
      "activations/layer4_attention_weight_max": 162.05421447753906,
      "activations/layer4_attention_weight_min": -145.5616455078125,
      "activations/layer5_attention_weight_max": 178.49583435058594,
      "activations/layer5_attention_weight_min": -172.02838134765625,
      "activations/layer6_attention_weight_max": 138.9316864013672,
      "activations/layer6_attention_weight_min": -122.87970733642578,
      "activations/layer7_attention_weight_max": 112.35783386230469,
      "activations/layer7_attention_weight_min": -99.06070709228516,
      "activations/layer8_attention_weight_max": 85.11369323730469,
      "activations/layer8_attention_weight_min": -69.22471618652344,
      "activations/layer9_attention_weight_max": 66.03170013427734,
      "activations/layer9_attention_weight_min": -53.7598876953125,
      "epoch": 0.19,
      "learning_rate": 0.0001458680580162806,
      "loss": 3.6639,
      "step": 3200
    },
    {
      "activations/layer0_attention_weight_max": 13.598609924316406,
      "activations/layer0_attention_weight_min": -9.275860786437988,
      "activations/layer10_attention_weight_max": 46.68229293823242,
      "activations/layer10_attention_weight_min": -32.13981246948242,
      "activations/layer11_attention_weight_max": 38.745819091796875,
      "activations/layer11_attention_weight_min": -22.967458724975586,
      "activations/layer12_attention_weight_max": 25.571266174316406,
      "activations/layer12_attention_weight_min": -18.850173950195312,
      "activations/layer13_attention_weight_max": 22.135936737060547,
      "activations/layer13_attention_weight_min": -19.392141342163086,
      "activations/layer14_attention_weight_max": 23.62050437927246,
      "activations/layer14_attention_weight_min": -12.704462051391602,
      "activations/layer15_attention_weight_max": 22.425142288208008,
      "activations/layer15_attention_weight_min": -13.277461051940918,
      "activations/layer16_attention_weight_max": 19.76646614074707,
      "activations/layer16_attention_weight_min": -14.550159454345703,
      "activations/layer17_attention_weight_max": 17.3818302154541,
      "activations/layer17_attention_weight_min": -12.357938766479492,
      "activations/layer18_attention_weight_max": 27.868694305419922,
      "activations/layer18_attention_weight_min": -14.42878532409668,
      "activations/layer19_attention_weight_max": 20.05622100830078,
      "activations/layer19_attention_weight_min": -12.57641887664795,
      "activations/layer1_attention_weight_max": 16.713048934936523,
      "activations/layer1_attention_weight_min": -10.30994987487793,
      "activations/layer20_attention_weight_max": 18.571556091308594,
      "activations/layer20_attention_weight_min": -12.022736549377441,
      "activations/layer21_attention_weight_max": 20.365798950195312,
      "activations/layer21_attention_weight_min": -13.996353149414062,
      "activations/layer22_attention_weight_max": 15.389750480651855,
      "activations/layer22_attention_weight_min": -11.521622657775879,
      "activations/layer23_attention_weight_max": 10.40157699584961,
      "activations/layer23_attention_weight_min": -8.946617126464844,
      "activations/layer2_attention_weight_max": 25.865087509155273,
      "activations/layer2_attention_weight_min": -22.806455612182617,
      "activations/layer3_attention_weight_max": 61.00685119628906,
      "activations/layer3_attention_weight_min": -52.98788070678711,
      "activations/layer4_attention_weight_max": 154.7344207763672,
      "activations/layer4_attention_weight_min": -149.4134063720703,
      "activations/layer5_attention_weight_max": 176.36361694335938,
      "activations/layer5_attention_weight_min": -168.4404754638672,
      "activations/layer6_attention_weight_max": 126.72493743896484,
      "activations/layer6_attention_weight_min": -120.33362579345703,
      "activations/layer7_attention_weight_max": 105.71537780761719,
      "activations/layer7_attention_weight_min": -95.62349700927734,
      "activations/layer8_attention_weight_max": 77.72721099853516,
      "activations/layer8_attention_weight_min": -68.91960144042969,
      "activations/layer9_attention_weight_max": 69.24708557128906,
      "activations/layer9_attention_weight_min": -52.20693588256836,
      "epoch": 0.19,
      "learning_rate": 0.00014614994113344894,
      "loss": 3.6511,
      "step": 3250
    },
    {
      "activations/layer0_attention_weight_max": 14.135945320129395,
      "activations/layer0_attention_weight_min": -9.374509811401367,
      "activations/layer10_attention_weight_max": 39.340179443359375,
      "activations/layer10_attention_weight_min": -28.52179527282715,
      "activations/layer11_attention_weight_max": 37.339378356933594,
      "activations/layer11_attention_weight_min": -23.60165023803711,
      "activations/layer12_attention_weight_max": 24.338905334472656,
      "activations/layer12_attention_weight_min": -21.933317184448242,
      "activations/layer13_attention_weight_max": 21.130399703979492,
      "activations/layer13_attention_weight_min": -17.898136138916016,
      "activations/layer14_attention_weight_max": 29.803241729736328,
      "activations/layer14_attention_weight_min": -13.389451026916504,
      "activations/layer15_attention_weight_max": 21.35099983215332,
      "activations/layer15_attention_weight_min": -12.801657676696777,
      "activations/layer16_attention_weight_max": 26.21023178100586,
      "activations/layer16_attention_weight_min": -13.002701759338379,
      "activations/layer17_attention_weight_max": 23.54534339904785,
      "activations/layer17_attention_weight_min": -9.8135404586792,
      "activations/layer18_attention_weight_max": 23.535730361938477,
      "activations/layer18_attention_weight_min": -12.002314567565918,
      "activations/layer19_attention_weight_max": 24.949737548828125,
      "activations/layer19_attention_weight_min": -12.360639572143555,
      "activations/layer1_attention_weight_max": 17.6359920501709,
      "activations/layer1_attention_weight_min": -11.239263534545898,
      "activations/layer20_attention_weight_max": 22.266536712646484,
      "activations/layer20_attention_weight_min": -12.334766387939453,
      "activations/layer21_attention_weight_max": 22.998132705688477,
      "activations/layer21_attention_weight_min": -11.272185325622559,
      "activations/layer22_attention_weight_max": 16.48133087158203,
      "activations/layer22_attention_weight_min": -10.48773193359375,
      "activations/layer23_attention_weight_max": 9.904277801513672,
      "activations/layer23_attention_weight_min": -9.036462783813477,
      "activations/layer2_attention_weight_max": 26.0983943939209,
      "activations/layer2_attention_weight_min": -22.248517990112305,
      "activations/layer3_attention_weight_max": 61.13706588745117,
      "activations/layer3_attention_weight_min": -50.68418502807617,
      "activations/layer4_attention_weight_max": 153.4631805419922,
      "activations/layer4_attention_weight_min": -143.75865173339844,
      "activations/layer5_attention_weight_max": 172.89279174804688,
      "activations/layer5_attention_weight_min": -160.47982788085938,
      "activations/layer6_attention_weight_max": 128.29251098632812,
      "activations/layer6_attention_weight_min": -117.44551086425781,
      "activations/layer7_attention_weight_max": 104.42240142822266,
      "activations/layer7_attention_weight_min": -92.72382354736328,
      "activations/layer8_attention_weight_max": 78.28971099853516,
      "activations/layer8_attention_weight_min": -69.45294189453125,
      "activations/layer9_attention_weight_max": 63.76864242553711,
      "activations/layer9_attention_weight_min": -53.3520393371582,
      "epoch": 0.19,
      "learning_rate": 0.0001464274980647812,
      "loss": 3.6452,
      "step": 3300
    },
    {
      "activations/layer0_attention_weight_max": 13.854440689086914,
      "activations/layer0_attention_weight_min": -9.312207221984863,
      "activations/layer10_attention_weight_max": 39.50727081298828,
      "activations/layer10_attention_weight_min": -25.96634292602539,
      "activations/layer11_attention_weight_max": 36.56584167480469,
      "activations/layer11_attention_weight_min": -23.63705062866211,
      "activations/layer12_attention_weight_max": 28.217348098754883,
      "activations/layer12_attention_weight_min": -24.672361373901367,
      "activations/layer13_attention_weight_max": 22.808998107910156,
      "activations/layer13_attention_weight_min": -17.840373992919922,
      "activations/layer14_attention_weight_max": 29.986907958984375,
      "activations/layer14_attention_weight_min": -13.921030044555664,
      "activations/layer15_attention_weight_max": 20.57563591003418,
      "activations/layer15_attention_weight_min": -13.041938781738281,
      "activations/layer16_attention_weight_max": 24.458276748657227,
      "activations/layer16_attention_weight_min": -12.052497863769531,
      "activations/layer17_attention_weight_max": 18.11557960510254,
      "activations/layer17_attention_weight_min": -11.592264175415039,
      "activations/layer18_attention_weight_max": 18.362136840820312,
      "activations/layer18_attention_weight_min": -11.855149269104004,
      "activations/layer19_attention_weight_max": 20.732406616210938,
      "activations/layer19_attention_weight_min": -11.56666088104248,
      "activations/layer1_attention_weight_max": 18.360864639282227,
      "activations/layer1_attention_weight_min": -11.29297161102295,
      "activations/layer20_attention_weight_max": 19.284915924072266,
      "activations/layer20_attention_weight_min": -12.97456169128418,
      "activations/layer21_attention_weight_max": 17.025545120239258,
      "activations/layer21_attention_weight_min": -13.370522499084473,
      "activations/layer22_attention_weight_max": 13.15187931060791,
      "activations/layer22_attention_weight_min": -9.484075546264648,
      "activations/layer23_attention_weight_max": 9.435125350952148,
      "activations/layer23_attention_weight_min": -9.248945236206055,
      "activations/layer2_attention_weight_max": 27.014699935913086,
      "activations/layer2_attention_weight_min": -22.0893611907959,
      "activations/layer3_attention_weight_max": 63.01491928100586,
      "activations/layer3_attention_weight_min": -50.69589614868164,
      "activations/layer4_attention_weight_max": 160.51023864746094,
      "activations/layer4_attention_weight_min": -145.19509887695312,
      "activations/layer5_attention_weight_max": 188.35679626464844,
      "activations/layer5_attention_weight_min": -162.941162109375,
      "activations/layer6_attention_weight_max": 134.1193084716797,
      "activations/layer6_attention_weight_min": -113.5445556640625,
      "activations/layer7_attention_weight_max": 104.53384399414062,
      "activations/layer7_attention_weight_min": -89.4061279296875,
      "activations/layer8_attention_weight_max": 83.46273040771484,
      "activations/layer8_attention_weight_min": -67.78450775146484,
      "activations/layer9_attention_weight_max": 61.00343704223633,
      "activations/layer9_attention_weight_min": -59.336181640625,
      "epoch": 0.19,
      "learning_rate": 0.0001467008595973623,
      "loss": 3.6497,
      "step": 3350
    },
    {
      "activations/layer0_attention_weight_max": 13.91159725189209,
      "activations/layer0_attention_weight_min": -9.09423828125,
      "activations/layer10_attention_weight_max": 37.22119903564453,
      "activations/layer10_attention_weight_min": -28.6224422454834,
      "activations/layer11_attention_weight_max": 37.29448699951172,
      "activations/layer11_attention_weight_min": -30.23639678955078,
      "activations/layer12_attention_weight_max": 27.39018440246582,
      "activations/layer12_attention_weight_min": -20.79612159729004,
      "activations/layer13_attention_weight_max": 23.94167137145996,
      "activations/layer13_attention_weight_min": -19.921125411987305,
      "activations/layer14_attention_weight_max": 23.98101043701172,
      "activations/layer14_attention_weight_min": -15.459924697875977,
      "activations/layer15_attention_weight_max": 25.408348083496094,
      "activations/layer15_attention_weight_min": -14.122596740722656,
      "activations/layer16_attention_weight_max": 20.31855583190918,
      "activations/layer16_attention_weight_min": -13.387215614318848,
      "activations/layer17_attention_weight_max": 20.836990356445312,
      "activations/layer17_attention_weight_min": -12.14786148071289,
      "activations/layer18_attention_weight_max": 19.559040069580078,
      "activations/layer18_attention_weight_min": -14.364312171936035,
      "activations/layer19_attention_weight_max": 20.003807067871094,
      "activations/layer19_attention_weight_min": -11.225565910339355,
      "activations/layer1_attention_weight_max": 16.83624839782715,
      "activations/layer1_attention_weight_min": -11.025776863098145,
      "activations/layer20_attention_weight_max": 18.75090217590332,
      "activations/layer20_attention_weight_min": -11.883197784423828,
      "activations/layer21_attention_weight_max": 19.749954223632812,
      "activations/layer21_attention_weight_min": -12.815452575683594,
      "activations/layer22_attention_weight_max": 19.192405700683594,
      "activations/layer22_attention_weight_min": -10.768081665039062,
      "activations/layer23_attention_weight_max": 8.079463005065918,
      "activations/layer23_attention_weight_min": -9.845975875854492,
      "activations/layer2_attention_weight_max": 25.560665130615234,
      "activations/layer2_attention_weight_min": -22.24880599975586,
      "activations/layer3_attention_weight_max": 60.31753921508789,
      "activations/layer3_attention_weight_min": -50.711219787597656,
      "activations/layer4_attention_weight_max": 153.9139862060547,
      "activations/layer4_attention_weight_min": -143.4351348876953,
      "activations/layer5_attention_weight_max": 172.8717041015625,
      "activations/layer5_attention_weight_min": -163.13821411132812,
      "activations/layer6_attention_weight_max": 127.92208099365234,
      "activations/layer6_attention_weight_min": -118.45911407470703,
      "activations/layer7_attention_weight_max": 104.99988555908203,
      "activations/layer7_attention_weight_min": -94.74634552001953,
      "activations/layer8_attention_weight_max": 77.36520385742188,
      "activations/layer8_attention_weight_min": -69.45904541015625,
      "activations/layer9_attention_weight_max": 65.95452880859375,
      "activations/layer9_attention_weight_min": -52.43654251098633,
      "epoch": 0.2,
      "learning_rate": 0.00014697015067564886,
      "loss": 3.6365,
      "step": 3400
    },
    {
      "activations/layer0_attention_weight_max": 14.531328201293945,
      "activations/layer0_attention_weight_min": -9.820300102233887,
      "activations/layer10_attention_weight_max": 39.59912109375,
      "activations/layer10_attention_weight_min": -29.550472259521484,
      "activations/layer11_attention_weight_max": 39.318153381347656,
      "activations/layer11_attention_weight_min": -25.36913299560547,
      "activations/layer12_attention_weight_max": 25.24363136291504,
      "activations/layer12_attention_weight_min": -20.365564346313477,
      "activations/layer13_attention_weight_max": 22.78264045715332,
      "activations/layer13_attention_weight_min": -18.728757858276367,
      "activations/layer14_attention_weight_max": 27.622678756713867,
      "activations/layer14_attention_weight_min": -14.756665229797363,
      "activations/layer15_attention_weight_max": 21.92946434020996,
      "activations/layer15_attention_weight_min": -12.011148452758789,
      "activations/layer16_attention_weight_max": 27.69975471496582,
      "activations/layer16_attention_weight_min": -13.810967445373535,
      "activations/layer17_attention_weight_max": 23.527250289916992,
      "activations/layer17_attention_weight_min": -13.372315406799316,
      "activations/layer18_attention_weight_max": 21.810382843017578,
      "activations/layer18_attention_weight_min": -11.820467948913574,
      "activations/layer19_attention_weight_max": 23.14351463317871,
      "activations/layer19_attention_weight_min": -12.142844200134277,
      "activations/layer1_attention_weight_max": 15.96601390838623,
      "activations/layer1_attention_weight_min": -10.14580249786377,
      "activations/layer20_attention_weight_max": 21.79718780517578,
      "activations/layer20_attention_weight_min": -11.831860542297363,
      "activations/layer21_attention_weight_max": 23.569103240966797,
      "activations/layer21_attention_weight_min": -12.381423950195312,
      "activations/layer22_attention_weight_max": 20.73744773864746,
      "activations/layer22_attention_weight_min": -11.125410079956055,
      "activations/layer23_attention_weight_max": 8.970039367675781,
      "activations/layer23_attention_weight_min": -9.963573455810547,
      "activations/layer2_attention_weight_max": 27.533361434936523,
      "activations/layer2_attention_weight_min": -22.28807830810547,
      "activations/layer3_attention_weight_max": 62.642948150634766,
      "activations/layer3_attention_weight_min": -49.563045501708984,
      "activations/layer4_attention_weight_max": 152.23597717285156,
      "activations/layer4_attention_weight_min": -140.9405975341797,
      "activations/layer5_attention_weight_max": 173.86802673339844,
      "activations/layer5_attention_weight_min": -159.6536407470703,
      "activations/layer6_attention_weight_max": 125.51725006103516,
      "activations/layer6_attention_weight_min": -114.70152282714844,
      "activations/layer7_attention_weight_max": 110.86769104003906,
      "activations/layer7_attention_weight_min": -93.72199249267578,
      "activations/layer8_attention_weight_max": 79.0888671875,
      "activations/layer8_attention_weight_min": -65.46796417236328,
      "activations/layer9_attention_weight_max": 64.60350799560547,
      "activations/layer9_attention_weight_min": -55.2191276550293,
      "epoch": 0.2,
      "learning_rate": 0.00014723549074437798,
      "loss": 3.636,
      "step": 3450
    },
    {
      "activations/layer0_attention_weight_max": 14.11982250213623,
      "activations/layer0_attention_weight_min": -9.639925003051758,
      "activations/layer10_attention_weight_max": 34.293006896972656,
      "activations/layer10_attention_weight_min": -27.97538185119629,
      "activations/layer11_attention_weight_max": 37.62609100341797,
      "activations/layer11_attention_weight_min": -25.15558433532715,
      "activations/layer12_attention_weight_max": 20.98086166381836,
      "activations/layer12_attention_weight_min": -21.924116134643555,
      "activations/layer13_attention_weight_max": 19.85105323791504,
      "activations/layer13_attention_weight_min": -17.428573608398438,
      "activations/layer14_attention_weight_max": 25.917680740356445,
      "activations/layer14_attention_weight_min": -15.861576080322266,
      "activations/layer15_attention_weight_max": 18.254377365112305,
      "activations/layer15_attention_weight_min": -13.380534172058105,
      "activations/layer16_attention_weight_max": 24.799711227416992,
      "activations/layer16_attention_weight_min": -13.636609077453613,
      "activations/layer17_attention_weight_max": 18.050743103027344,
      "activations/layer17_attention_weight_min": -10.858407020568848,
      "activations/layer18_attention_weight_max": 20.243202209472656,
      "activations/layer18_attention_weight_min": -10.98230266571045,
      "activations/layer19_attention_weight_max": 16.06780433654785,
      "activations/layer19_attention_weight_min": -10.891802787780762,
      "activations/layer1_attention_weight_max": 17.881933212280273,
      "activations/layer1_attention_weight_min": -10.20447063446045,
      "activations/layer20_attention_weight_max": 17.819801330566406,
      "activations/layer20_attention_weight_min": -11.585251808166504,
      "activations/layer21_attention_weight_max": 16.49346160888672,
      "activations/layer21_attention_weight_min": -10.905909538269043,
      "activations/layer22_attention_weight_max": 13.837686538696289,
      "activations/layer22_attention_weight_min": -11.011258125305176,
      "activations/layer23_attention_weight_max": 8.49636173248291,
      "activations/layer23_attention_weight_min": -9.659141540527344,
      "activations/layer2_attention_weight_max": 27.556976318359375,
      "activations/layer2_attention_weight_min": -23.042110443115234,
      "activations/layer3_attention_weight_max": 60.42313003540039,
      "activations/layer3_attention_weight_min": -52.55527877807617,
      "activations/layer4_attention_weight_max": 152.19740295410156,
      "activations/layer4_attention_weight_min": -147.27435302734375,
      "activations/layer5_attention_weight_max": 168.64077758789062,
      "activations/layer5_attention_weight_min": -161.39447021484375,
      "activations/layer6_attention_weight_max": 125.6216812133789,
      "activations/layer6_attention_weight_min": -118.1103286743164,
      "activations/layer7_attention_weight_max": 103.39785766601562,
      "activations/layer7_attention_weight_min": -90.87906646728516,
      "activations/layer8_attention_weight_max": 78.10079956054688,
      "activations/layer8_attention_weight_min": -68.91385650634766,
      "activations/layer9_attention_weight_max": 58.78120803833008,
      "activations/layer9_attention_weight_min": -53.56673049926758,
      "epoch": 0.2,
      "learning_rate": 0.00014749699406668265,
      "loss": 3.6141,
      "step": 3500
    },
    {
      "activations/layer0_attention_weight_max": 13.546856880187988,
      "activations/layer0_attention_weight_min": -8.909780502319336,
      "activations/layer10_attention_weight_max": 35.51528549194336,
      "activations/layer10_attention_weight_min": -27.96590232849121,
      "activations/layer11_attention_weight_max": 39.74360656738281,
      "activations/layer11_attention_weight_min": -25.928855895996094,
      "activations/layer12_attention_weight_max": 22.352155685424805,
      "activations/layer12_attention_weight_min": -23.94379425048828,
      "activations/layer13_attention_weight_max": 21.085777282714844,
      "activations/layer13_attention_weight_min": -18.965436935424805,
      "activations/layer14_attention_weight_max": 28.562042236328125,
      "activations/layer14_attention_weight_min": -14.942298889160156,
      "activations/layer15_attention_weight_max": 20.949207305908203,
      "activations/layer15_attention_weight_min": -13.876139640808105,
      "activations/layer16_attention_weight_max": 22.002601623535156,
      "activations/layer16_attention_weight_min": -14.107297897338867,
      "activations/layer17_attention_weight_max": 20.6838321685791,
      "activations/layer17_attention_weight_min": -11.413063049316406,
      "activations/layer18_attention_weight_max": 19.40139389038086,
      "activations/layer18_attention_weight_min": -12.446839332580566,
      "activations/layer19_attention_weight_max": 24.184053421020508,
      "activations/layer19_attention_weight_min": -13.88933277130127,
      "activations/layer1_attention_weight_max": 14.926763534545898,
      "activations/layer1_attention_weight_min": -10.695308685302734,
      "activations/layer20_attention_weight_max": 20.61204719543457,
      "activations/layer20_attention_weight_min": -14.154998779296875,
      "activations/layer21_attention_weight_max": 19.26755142211914,
      "activations/layer21_attention_weight_min": -14.863456726074219,
      "activations/layer22_attention_weight_max": 18.40494728088379,
      "activations/layer22_attention_weight_min": -10.819539070129395,
      "activations/layer23_attention_weight_max": 9.694265365600586,
      "activations/layer23_attention_weight_min": -8.911552429199219,
      "activations/layer2_attention_weight_max": 25.968482971191406,
      "activations/layer2_attention_weight_min": -21.203508377075195,
      "activations/layer3_attention_weight_max": 61.44735336303711,
      "activations/layer3_attention_weight_min": -51.90504455566406,
      "activations/layer4_attention_weight_max": 152.13357543945312,
      "activations/layer4_attention_weight_min": -141.1159210205078,
      "activations/layer5_attention_weight_max": 171.45077514648438,
      "activations/layer5_attention_weight_min": -160.22348022460938,
      "activations/layer6_attention_weight_max": 121.69512939453125,
      "activations/layer6_attention_weight_min": -118.29330444335938,
      "activations/layer7_attention_weight_max": 99.10514068603516,
      "activations/layer7_attention_weight_min": -92.27403259277344,
      "activations/layer8_attention_weight_max": 74.1181411743164,
      "activations/layer8_attention_weight_min": -68.18527221679688,
      "activations/layer9_attention_weight_max": 57.756683349609375,
      "activations/layer9_attention_weight_min": -57.31460189819336,
      "epoch": 0.21,
      "learning_rate": 0.00014775477001953373,
      "loss": 3.6067,
      "step": 3550
    },
    {
      "activations/layer0_attention_weight_max": 14.439385414123535,
      "activations/layer0_attention_weight_min": -10.3212890625,
      "activations/layer10_attention_weight_max": 44.792842864990234,
      "activations/layer10_attention_weight_min": -32.21345901489258,
      "activations/layer11_attention_weight_max": 39.8873291015625,
      "activations/layer11_attention_weight_min": -26.59476089477539,
      "activations/layer12_attention_weight_max": 24.40311622619629,
      "activations/layer12_attention_weight_min": -21.700077056884766,
      "activations/layer13_attention_weight_max": 26.15153694152832,
      "activations/layer13_attention_weight_min": -18.044445037841797,
      "activations/layer14_attention_weight_max": 28.582242965698242,
      "activations/layer14_attention_weight_min": -15.855403900146484,
      "activations/layer15_attention_weight_max": 22.051786422729492,
      "activations/layer15_attention_weight_min": -13.99014663696289,
      "activations/layer16_attention_weight_max": 31.90398406982422,
      "activations/layer16_attention_weight_min": -15.236035346984863,
      "activations/layer17_attention_weight_max": 24.760921478271484,
      "activations/layer17_attention_weight_min": -12.61254596710205,
      "activations/layer18_attention_weight_max": 22.623992919921875,
      "activations/layer18_attention_weight_min": -13.80444049835205,
      "activations/layer19_attention_weight_max": 24.896974563598633,
      "activations/layer19_attention_weight_min": -14.51091480255127,
      "activations/layer1_attention_weight_max": 16.371795654296875,
      "activations/layer1_attention_weight_min": -10.358427047729492,
      "activations/layer20_attention_weight_max": 21.364036560058594,
      "activations/layer20_attention_weight_min": -12.870596885681152,
      "activations/layer21_attention_weight_max": 24.535831451416016,
      "activations/layer21_attention_weight_min": -14.290144920349121,
      "activations/layer22_attention_weight_max": 21.07611083984375,
      "activations/layer22_attention_weight_min": -12.01093864440918,
      "activations/layer23_attention_weight_max": 10.803057670593262,
      "activations/layer23_attention_weight_min": -10.150724411010742,
      "activations/layer2_attention_weight_max": 26.899696350097656,
      "activations/layer2_attention_weight_min": -22.174922943115234,
      "activations/layer3_attention_weight_max": 62.97101974487305,
      "activations/layer3_attention_weight_min": -51.372249603271484,
      "activations/layer4_attention_weight_max": 156.79576110839844,
      "activations/layer4_attention_weight_min": -144.18553161621094,
      "activations/layer5_attention_weight_max": 172.78067016601562,
      "activations/layer5_attention_weight_min": -158.96600341796875,
      "activations/layer6_attention_weight_max": 125.86763763427734,
      "activations/layer6_attention_weight_min": -112.57551574707031,
      "activations/layer7_attention_weight_max": 107.48470306396484,
      "activations/layer7_attention_weight_min": -95.34490966796875,
      "activations/layer8_attention_weight_max": 80.2798080444336,
      "activations/layer8_attention_weight_min": -66.02469635009766,
      "activations/layer9_attention_weight_max": 66.03221893310547,
      "activations/layer9_attention_weight_min": -53.560237884521484,
      "epoch": 0.21,
      "learning_rate": 0.00014800892336842126,
      "loss": 3.6111,
      "step": 3600
    },
    {
      "activations/layer0_attention_weight_max": 13.671051979064941,
      "activations/layer0_attention_weight_min": -10.667058944702148,
      "activations/layer10_attention_weight_max": 35.19600296020508,
      "activations/layer10_attention_weight_min": -29.21385955810547,
      "activations/layer11_attention_weight_max": 35.318138122558594,
      "activations/layer11_attention_weight_min": -28.211990356445312,
      "activations/layer12_attention_weight_max": 20.001440048217773,
      "activations/layer12_attention_weight_min": -21.506526947021484,
      "activations/layer13_attention_weight_max": 18.702051162719727,
      "activations/layer13_attention_weight_min": -17.426979064941406,
      "activations/layer14_attention_weight_max": 21.909339904785156,
      "activations/layer14_attention_weight_min": -15.7512788772583,
      "activations/layer15_attention_weight_max": 21.138460159301758,
      "activations/layer15_attention_weight_min": -13.563559532165527,
      "activations/layer16_attention_weight_max": 21.180988311767578,
      "activations/layer16_attention_weight_min": -13.479302406311035,
      "activations/layer17_attention_weight_max": 20.353879928588867,
      "activations/layer17_attention_weight_min": -11.868396759033203,
      "activations/layer18_attention_weight_max": 18.752944946289062,
      "activations/layer18_attention_weight_min": -12.361577987670898,
      "activations/layer19_attention_weight_max": 19.583871841430664,
      "activations/layer19_attention_weight_min": -13.121615409851074,
      "activations/layer1_attention_weight_max": 17.57320785522461,
      "activations/layer1_attention_weight_min": -10.682974815368652,
      "activations/layer20_attention_weight_max": 22.713838577270508,
      "activations/layer20_attention_weight_min": -11.787758827209473,
      "activations/layer21_attention_weight_max": 18.851638793945312,
      "activations/layer21_attention_weight_min": -12.391818046569824,
      "activations/layer22_attention_weight_max": 17.1246337890625,
      "activations/layer22_attention_weight_min": -11.185088157653809,
      "activations/layer23_attention_weight_max": 10.75729751586914,
      "activations/layer23_attention_weight_min": -10.59335708618164,
      "activations/layer2_attention_weight_max": 26.570514678955078,
      "activations/layer2_attention_weight_min": -23.98406982421875,
      "activations/layer3_attention_weight_max": 60.99394607543945,
      "activations/layer3_attention_weight_min": -51.90204620361328,
      "activations/layer4_attention_weight_max": 153.88880920410156,
      "activations/layer4_attention_weight_min": -143.55638122558594,
      "activations/layer5_attention_weight_max": 169.50746154785156,
      "activations/layer5_attention_weight_min": -161.9220733642578,
      "activations/layer6_attention_weight_max": 118.86763763427734,
      "activations/layer6_attention_weight_min": -113.03272247314453,
      "activations/layer7_attention_weight_max": 104.8682861328125,
      "activations/layer7_attention_weight_min": -89.96259307861328,
      "activations/layer8_attention_weight_max": 76.42496490478516,
      "activations/layer8_attention_weight_min": -63.79657745361328,
      "activations/layer9_attention_weight_max": 66.13841247558594,
      "activations/layer9_attention_weight_min": -52.474365234375,
      "epoch": 0.21,
      "learning_rate": 0.00014825955452300063,
      "loss": 3.6074,
      "step": 3650
    },
    {
      "activations/layer0_attention_weight_max": 13.282486915588379,
      "activations/layer0_attention_weight_min": -8.484786033630371,
      "activations/layer10_attention_weight_max": 40.21132278442383,
      "activations/layer10_attention_weight_min": -30.94879150390625,
      "activations/layer11_attention_weight_max": 42.66593933105469,
      "activations/layer11_attention_weight_min": -26.35683822631836,
      "activations/layer12_attention_weight_max": 29.0493106842041,
      "activations/layer12_attention_weight_min": -25.327316284179688,
      "activations/layer13_attention_weight_max": 23.41440773010254,
      "activations/layer13_attention_weight_min": -19.609973907470703,
      "activations/layer14_attention_weight_max": 31.259571075439453,
      "activations/layer14_attention_weight_min": -16.062175750732422,
      "activations/layer15_attention_weight_max": 21.8956298828125,
      "activations/layer15_attention_weight_min": -11.914999008178711,
      "activations/layer16_attention_weight_max": 26.192668914794922,
      "activations/layer16_attention_weight_min": -14.444210052490234,
      "activations/layer17_attention_weight_max": 27.69471549987793,
      "activations/layer17_attention_weight_min": -12.375120162963867,
      "activations/layer18_attention_weight_max": 23.61086082458496,
      "activations/layer18_attention_weight_min": -11.1676025390625,
      "activations/layer19_attention_weight_max": 24.035429000854492,
      "activations/layer19_attention_weight_min": -11.791800498962402,
      "activations/layer1_attention_weight_max": 14.91836929321289,
      "activations/layer1_attention_weight_min": -10.491920471191406,
      "activations/layer20_attention_weight_max": 24.735912322998047,
      "activations/layer20_attention_weight_min": -12.76534366607666,
      "activations/layer21_attention_weight_max": 20.01524543762207,
      "activations/layer21_attention_weight_min": -13.485536575317383,
      "activations/layer22_attention_weight_max": 17.699893951416016,
      "activations/layer22_attention_weight_min": -9.735520362854004,
      "activations/layer23_attention_weight_max": 9.054879188537598,
      "activations/layer23_attention_weight_min": -10.77033805847168,
      "activations/layer2_attention_weight_max": 27.760135650634766,
      "activations/layer2_attention_weight_min": -23.13608169555664,
      "activations/layer3_attention_weight_max": 61.82414245605469,
      "activations/layer3_attention_weight_min": -50.145660400390625,
      "activations/layer4_attention_weight_max": 148.89871215820312,
      "activations/layer4_attention_weight_min": -138.5037078857422,
      "activations/layer5_attention_weight_max": 173.7379608154297,
      "activations/layer5_attention_weight_min": -156.39169311523438,
      "activations/layer6_attention_weight_max": 117.23700714111328,
      "activations/layer6_attention_weight_min": -113.21287536621094,
      "activations/layer7_attention_weight_max": 100.1136245727539,
      "activations/layer7_attention_weight_min": -88.8553466796875,
      "activations/layer8_attention_weight_max": 73.90677642822266,
      "activations/layer8_attention_weight_min": -65.77938079833984,
      "activations/layer9_attention_weight_max": 58.94649887084961,
      "activations/layer9_attention_weight_min": -56.04413986206055,
      "epoch": 0.21,
      "learning_rate": 0.0001485067597752656,
      "loss": 3.5926,
      "step": 3700
    },
    {
      "activations/layer0_attention_weight_max": 13.522000312805176,
      "activations/layer0_attention_weight_min": -8.646195411682129,
      "activations/layer10_attention_weight_max": 35.19812774658203,
      "activations/layer10_attention_weight_min": -30.259037017822266,
      "activations/layer11_attention_weight_max": 34.164215087890625,
      "activations/layer11_attention_weight_min": -25.4669189453125,
      "activations/layer12_attention_weight_max": 23.021915435791016,
      "activations/layer12_attention_weight_min": -23.24976921081543,
      "activations/layer13_attention_weight_max": 22.396821975708008,
      "activations/layer13_attention_weight_min": -19.793964385986328,
      "activations/layer14_attention_weight_max": 22.447437286376953,
      "activations/layer14_attention_weight_min": -15.671090126037598,
      "activations/layer15_attention_weight_max": 23.554061889648438,
      "activations/layer15_attention_weight_min": -12.579917907714844,
      "activations/layer16_attention_weight_max": 19.718360900878906,
      "activations/layer16_attention_weight_min": -13.207520484924316,
      "activations/layer17_attention_weight_max": 17.44113540649414,
      "activations/layer17_attention_weight_min": -11.743441581726074,
      "activations/layer18_attention_weight_max": 17.709716796875,
      "activations/layer18_attention_weight_min": -11.430984497070312,
      "activations/layer19_attention_weight_max": 17.033422470092773,
      "activations/layer19_attention_weight_min": -11.818224906921387,
      "activations/layer1_attention_weight_max": 16.58907699584961,
      "activations/layer1_attention_weight_min": -10.577069282531738,
      "activations/layer20_attention_weight_max": 19.27153968811035,
      "activations/layer20_attention_weight_min": -12.336706161499023,
      "activations/layer21_attention_weight_max": 18.785968780517578,
      "activations/layer21_attention_weight_min": -11.22217845916748,
      "activations/layer22_attention_weight_max": 14.746184349060059,
      "activations/layer22_attention_weight_min": -11.77303409576416,
      "activations/layer23_attention_weight_max": 9.257716178894043,
      "activations/layer23_attention_weight_min": -11.3292236328125,
      "activations/layer2_attention_weight_max": 27.679471969604492,
      "activations/layer2_attention_weight_min": -22.949575424194336,
      "activations/layer3_attention_weight_max": 60.2313232421875,
      "activations/layer3_attention_weight_min": -50.176490783691406,
      "activations/layer4_attention_weight_max": 142.8148956298828,
      "activations/layer4_attention_weight_min": -143.3394317626953,
      "activations/layer5_attention_weight_max": 170.68112182617188,
      "activations/layer5_attention_weight_min": -155.94287109375,
      "activations/layer6_attention_weight_max": 115.1313705444336,
      "activations/layer6_attention_weight_min": -113.8480453491211,
      "activations/layer7_attention_weight_max": 95.36566162109375,
      "activations/layer7_attention_weight_min": -89.96640014648438,
      "activations/layer8_attention_weight_max": 72.42034912109375,
      "activations/layer8_attention_weight_min": -63.5279541015625,
      "activations/layer9_attention_weight_max": 67.3948974609375,
      "activations/layer9_attention_weight_min": -57.00965118408203,
      "epoch": 0.22,
      "learning_rate": 0.0001487506315216617,
      "loss": 3.5885,
      "step": 3750
    },
    {
      "activations/layer0_attention_weight_max": 14.033404350280762,
      "activations/layer0_attention_weight_min": -8.987237930297852,
      "activations/layer10_attention_weight_max": 36.027530670166016,
      "activations/layer10_attention_weight_min": -30.968833923339844,
      "activations/layer11_attention_weight_max": 41.941932678222656,
      "activations/layer11_attention_weight_min": -26.11573600769043,
      "activations/layer12_attention_weight_max": 28.40886878967285,
      "activations/layer12_attention_weight_min": -21.194719314575195,
      "activations/layer13_attention_weight_max": 20.56068992614746,
      "activations/layer13_attention_weight_min": -18.649494171142578,
      "activations/layer14_attention_weight_max": 25.031274795532227,
      "activations/layer14_attention_weight_min": -15.201506614685059,
      "activations/layer15_attention_weight_max": 20.881628036499023,
      "activations/layer15_attention_weight_min": -13.69249439239502,
      "activations/layer16_attention_weight_max": 23.11247444152832,
      "activations/layer16_attention_weight_min": -13.39156436920166,
      "activations/layer17_attention_weight_max": 18.612287521362305,
      "activations/layer17_attention_weight_min": -11.631367683410645,
      "activations/layer18_attention_weight_max": 21.93058204650879,
      "activations/layer18_attention_weight_min": -11.810782432556152,
      "activations/layer19_attention_weight_max": 18.81684112548828,
      "activations/layer19_attention_weight_min": -11.133313179016113,
      "activations/layer1_attention_weight_max": 17.015941619873047,
      "activations/layer1_attention_weight_min": -11.321723937988281,
      "activations/layer20_attention_weight_max": 20.608091354370117,
      "activations/layer20_attention_weight_min": -11.268495559692383,
      "activations/layer21_attention_weight_max": 19.65946388244629,
      "activations/layer21_attention_weight_min": -10.99786376953125,
      "activations/layer22_attention_weight_max": 14.03818416595459,
      "activations/layer22_attention_weight_min": -10.810970306396484,
      "activations/layer23_attention_weight_max": 10.79602336883545,
      "activations/layer23_attention_weight_min": -10.629250526428223,
      "activations/layer2_attention_weight_max": 27.684120178222656,
      "activations/layer2_attention_weight_min": -23.217161178588867,
      "activations/layer3_attention_weight_max": 61.836631774902344,
      "activations/layer3_attention_weight_min": -49.61758041381836,
      "activations/layer4_attention_weight_max": 150.79638671875,
      "activations/layer4_attention_weight_min": -139.5157928466797,
      "activations/layer5_attention_weight_max": 174.83432006835938,
      "activations/layer5_attention_weight_min": -158.22450256347656,
      "activations/layer6_attention_weight_max": 122.60057067871094,
      "activations/layer6_attention_weight_min": -108.86290740966797,
      "activations/layer7_attention_weight_max": 95.99469757080078,
      "activations/layer7_attention_weight_min": -87.91838836669922,
      "activations/layer8_attention_weight_max": 75.44828033447266,
      "activations/layer8_attention_weight_min": -64.03929138183594,
      "activations/layer9_attention_weight_max": 64.7503662109375,
      "activations/layer9_attention_weight_min": -57.1759147644043,
      "epoch": 0.22,
      "learning_rate": 0.00014899125847042232,
      "loss": 3.5758,
      "step": 3800
    },
    {
      "activations/layer0_attention_weight_max": 13.503931999206543,
      "activations/layer0_attention_weight_min": -11.079621315002441,
      "activations/layer10_attention_weight_max": 41.69830322265625,
      "activations/layer10_attention_weight_min": -29.143646240234375,
      "activations/layer11_attention_weight_max": 44.27471160888672,
      "activations/layer11_attention_weight_min": -28.58282470703125,
      "activations/layer12_attention_weight_max": 21.13935661315918,
      "activations/layer12_attention_weight_min": -23.189085006713867,
      "activations/layer13_attention_weight_max": 23.24610137939453,
      "activations/layer13_attention_weight_min": -19.594348907470703,
      "activations/layer14_attention_weight_max": 21.359724044799805,
      "activations/layer14_attention_weight_min": -14.717657089233398,
      "activations/layer15_attention_weight_max": 23.907968521118164,
      "activations/layer15_attention_weight_min": -15.408287048339844,
      "activations/layer16_attention_weight_max": 23.680633544921875,
      "activations/layer16_attention_weight_min": -12.771590232849121,
      "activations/layer17_attention_weight_max": 18.94500732421875,
      "activations/layer17_attention_weight_min": -11.369845390319824,
      "activations/layer18_attention_weight_max": 20.749629974365234,
      "activations/layer18_attention_weight_min": -12.378250122070312,
      "activations/layer19_attention_weight_max": 18.39240074157715,
      "activations/layer19_attention_weight_min": -11.207901954650879,
      "activations/layer1_attention_weight_max": 17.905336380004883,
      "activations/layer1_attention_weight_min": -11.652853012084961,
      "activations/layer20_attention_weight_max": 19.154630661010742,
      "activations/layer20_attention_weight_min": -12.838808059692383,
      "activations/layer21_attention_weight_max": 18.29144859313965,
      "activations/layer21_attention_weight_min": -11.955361366271973,
      "activations/layer22_attention_weight_max": 14.535318374633789,
      "activations/layer22_attention_weight_min": -10.611587524414062,
      "activations/layer23_attention_weight_max": 8.95898723602295,
      "activations/layer23_attention_weight_min": -10.562898635864258,
      "activations/layer2_attention_weight_max": 28.441913604736328,
      "activations/layer2_attention_weight_min": -23.491134643554688,
      "activations/layer3_attention_weight_max": 60.13650894165039,
      "activations/layer3_attention_weight_min": -50.64608383178711,
      "activations/layer4_attention_weight_max": 147.97068786621094,
      "activations/layer4_attention_weight_min": -135.07632446289062,
      "activations/layer5_attention_weight_max": 172.72879028320312,
      "activations/layer5_attention_weight_min": -159.5749969482422,
      "activations/layer6_attention_weight_max": 120.51175689697266,
      "activations/layer6_attention_weight_min": -109.4180679321289,
      "activations/layer7_attention_weight_max": 98.6072769165039,
      "activations/layer7_attention_weight_min": -86.89488220214844,
      "activations/layer8_attention_weight_max": 78.56172943115234,
      "activations/layer8_attention_weight_min": -68.04949951171875,
      "activations/layer9_attention_weight_max": 61.41191482543945,
      "activations/layer9_attention_weight_min": -54.20398712158203,
      "epoch": 0.22,
      "learning_rate": 0.00014922872583529145,
      "loss": 3.5556,
      "step": 3850
    },
    {
      "activations/layer0_attention_weight_max": 14.426044464111328,
      "activations/layer0_attention_weight_min": -8.803321838378906,
      "activations/layer10_attention_weight_max": 35.972476959228516,
      "activations/layer10_attention_weight_min": -28.086875915527344,
      "activations/layer11_attention_weight_max": 37.52781677246094,
      "activations/layer11_attention_weight_min": -27.068395614624023,
      "activations/layer12_attention_weight_max": 30.70677947998047,
      "activations/layer12_attention_weight_min": -21.311426162719727,
      "activations/layer13_attention_weight_max": 23.059091567993164,
      "activations/layer13_attention_weight_min": -18.598800659179688,
      "activations/layer14_attention_weight_max": 31.26063346862793,
      "activations/layer14_attention_weight_min": -15.26728630065918,
      "activations/layer15_attention_weight_max": 20.445716857910156,
      "activations/layer15_attention_weight_min": -12.967592239379883,
      "activations/layer16_attention_weight_max": 24.069412231445312,
      "activations/layer16_attention_weight_min": -12.52298641204834,
      "activations/layer17_attention_weight_max": 22.55733871459961,
      "activations/layer17_attention_weight_min": -11.221380233764648,
      "activations/layer18_attention_weight_max": 21.388639450073242,
      "activations/layer18_attention_weight_min": -12.200345993041992,
      "activations/layer19_attention_weight_max": 25.41668128967285,
      "activations/layer19_attention_weight_min": -11.86691951751709,
      "activations/layer1_attention_weight_max": 14.890583038330078,
      "activations/layer1_attention_weight_min": -10.156052589416504,
      "activations/layer20_attention_weight_max": 22.153343200683594,
      "activations/layer20_attention_weight_min": -11.879189491271973,
      "activations/layer21_attention_weight_max": 20.997976303100586,
      "activations/layer21_attention_weight_min": -10.368545532226562,
      "activations/layer22_attention_weight_max": 15.095894813537598,
      "activations/layer22_attention_weight_min": -10.433357238769531,
      "activations/layer23_attention_weight_max": 10.710872650146484,
      "activations/layer23_attention_weight_min": -10.13149642944336,
      "activations/layer2_attention_weight_max": 27.55113983154297,
      "activations/layer2_attention_weight_min": -22.494407653808594,
      "activations/layer3_attention_weight_max": 60.778629302978516,
      "activations/layer3_attention_weight_min": -49.08015441894531,
      "activations/layer4_attention_weight_max": 154.5919647216797,
      "activations/layer4_attention_weight_min": -133.48455810546875,
      "activations/layer5_attention_weight_max": 173.70257568359375,
      "activations/layer5_attention_weight_min": -151.2471923828125,
      "activations/layer6_attention_weight_max": 126.90316009521484,
      "activations/layer6_attention_weight_min": -108.10243225097656,
      "activations/layer7_attention_weight_max": 103.80947875976562,
      "activations/layer7_attention_weight_min": -89.08917999267578,
      "activations/layer8_attention_weight_max": 74.93812561035156,
      "activations/layer8_attention_weight_min": -63.43193817138672,
      "activations/layer9_attention_weight_max": 61.35034942626953,
      "activations/layer9_attention_weight_min": -55.20979690551758,
      "epoch": 0.23,
      "learning_rate": 0.00014946311551669234,
      "loss": 3.5744,
      "step": 3900
    },
    {
      "activations/layer0_attention_weight_max": 15.0493745803833,
      "activations/layer0_attention_weight_min": -10.351091384887695,
      "activations/layer10_attention_weight_max": 37.105751037597656,
      "activations/layer10_attention_weight_min": -31.542442321777344,
      "activations/layer11_attention_weight_max": 38.47883605957031,
      "activations/layer11_attention_weight_min": -26.593603134155273,
      "activations/layer12_attention_weight_max": 21.819013595581055,
      "activations/layer12_attention_weight_min": -22.96278190612793,
      "activations/layer13_attention_weight_max": 22.057104110717773,
      "activations/layer13_attention_weight_min": -16.456581115722656,
      "activations/layer14_attention_weight_max": 27.77077293395996,
      "activations/layer14_attention_weight_min": -15.829211235046387,
      "activations/layer15_attention_weight_max": 18.306623458862305,
      "activations/layer15_attention_weight_min": -13.5596284866333,
      "activations/layer16_attention_weight_max": 22.913969039916992,
      "activations/layer16_attention_weight_min": -14.175152778625488,
      "activations/layer17_attention_weight_max": 19.857318878173828,
      "activations/layer17_attention_weight_min": -11.479076385498047,
      "activations/layer18_attention_weight_max": 18.984325408935547,
      "activations/layer18_attention_weight_min": -12.795065879821777,
      "activations/layer19_attention_weight_max": 22.237586975097656,
      "activations/layer19_attention_weight_min": -12.198190689086914,
      "activations/layer1_attention_weight_max": 15.18511962890625,
      "activations/layer1_attention_weight_min": -10.589653968811035,
      "activations/layer20_attention_weight_max": 20.07435417175293,
      "activations/layer20_attention_weight_min": -12.782434463500977,
      "activations/layer21_attention_weight_max": 19.671573638916016,
      "activations/layer21_attention_weight_min": -13.468693733215332,
      "activations/layer22_attention_weight_max": 16.360963821411133,
      "activations/layer22_attention_weight_min": -12.561634063720703,
      "activations/layer23_attention_weight_max": 10.912328720092773,
      "activations/layer23_attention_weight_min": -11.19809627532959,
      "activations/layer2_attention_weight_max": 27.94324493408203,
      "activations/layer2_attention_weight_min": -22.929039001464844,
      "activations/layer3_attention_weight_max": 59.987937927246094,
      "activations/layer3_attention_weight_min": -50.98988342285156,
      "activations/layer4_attention_weight_max": 145.55630493164062,
      "activations/layer4_attention_weight_min": -137.8963165283203,
      "activations/layer5_attention_weight_max": 159.4783477783203,
      "activations/layer5_attention_weight_min": -150.33819580078125,
      "activations/layer6_attention_weight_max": 116.77674865722656,
      "activations/layer6_attention_weight_min": -105.69803619384766,
      "activations/layer7_attention_weight_max": 93.47232055664062,
      "activations/layer7_attention_weight_min": -86.36415100097656,
      "activations/layer8_attention_weight_max": 72.80382537841797,
      "activations/layer8_attention_weight_min": -64.89404296875,
      "activations/layer9_attention_weight_max": 62.71589279174805,
      "activations/layer9_attention_weight_min": -58.82752227783203,
      "epoch": 0.23,
      "learning_rate": 0.00014969450627130542,
      "loss": 3.5441,
      "step": 3950
    },
    {
      "activations/layer0_attention_weight_max": 13.469805717468262,
      "activations/layer0_attention_weight_min": -10.311330795288086,
      "activations/layer10_attention_weight_max": 41.35734558105469,
      "activations/layer10_attention_weight_min": -32.74192810058594,
      "activations/layer11_attention_weight_max": 42.660457611083984,
      "activations/layer11_attention_weight_min": -30.571208953857422,
      "activations/layer12_attention_weight_max": 24.42685317993164,
      "activations/layer12_attention_weight_min": -23.296676635742188,
      "activations/layer13_attention_weight_max": 23.853967666625977,
      "activations/layer13_attention_weight_min": -20.473648071289062,
      "activations/layer14_attention_weight_max": 29.179027557373047,
      "activations/layer14_attention_weight_min": -16.78298568725586,
      "activations/layer15_attention_weight_max": 20.53485107421875,
      "activations/layer15_attention_weight_min": -15.690275192260742,
      "activations/layer16_attention_weight_max": 22.496774673461914,
      "activations/layer16_attention_weight_min": -13.90240478515625,
      "activations/layer17_attention_weight_max": 20.032062530517578,
      "activations/layer17_attention_weight_min": -13.154367446899414,
      "activations/layer18_attention_weight_max": 19.677993774414062,
      "activations/layer18_attention_weight_min": -12.313965797424316,
      "activations/layer19_attention_weight_max": 19.637866973876953,
      "activations/layer19_attention_weight_min": -12.57342529296875,
      "activations/layer1_attention_weight_max": 14.692360877990723,
      "activations/layer1_attention_weight_min": -10.539973258972168,
      "activations/layer20_attention_weight_max": 20.238840103149414,
      "activations/layer20_attention_weight_min": -15.013598442077637,
      "activations/layer21_attention_weight_max": 22.054569244384766,
      "activations/layer21_attention_weight_min": -14.597637176513672,
      "activations/layer22_attention_weight_max": 17.29281997680664,
      "activations/layer22_attention_weight_min": -13.207599639892578,
      "activations/layer23_attention_weight_max": 10.828264236450195,
      "activations/layer23_attention_weight_min": -10.390426635742188,
      "activations/layer2_attention_weight_max": 29.4888916015625,
      "activations/layer2_attention_weight_min": -23.58414077758789,
      "activations/layer3_attention_weight_max": 64.38495635986328,
      "activations/layer3_attention_weight_min": -50.23679733276367,
      "activations/layer4_attention_weight_max": 155.69676208496094,
      "activations/layer4_attention_weight_min": -138.85604858398438,
      "activations/layer5_attention_weight_max": 178.65069580078125,
      "activations/layer5_attention_weight_min": -154.7274627685547,
      "activations/layer6_attention_weight_max": 120.64846801757812,
      "activations/layer6_attention_weight_min": -110.75959777832031,
      "activations/layer7_attention_weight_max": 98.59683990478516,
      "activations/layer7_attention_weight_min": -89.34680938720703,
      "activations/layer8_attention_weight_max": 78.36856842041016,
      "activations/layer8_attention_weight_min": -65.83560943603516,
      "activations/layer9_attention_weight_max": 66.48492431640625,
      "activations/layer9_attention_weight_min": -59.20795440673828,
      "epoch": 0.23,
      "learning_rate": 0.00014992297387093407,
      "loss": 3.5484,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_loss": 3.498046875,
      "eval_runtime": 8.2918,
      "eval_samples_per_second": 517.863,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_openwebtext_loss": 3.498046875,
      "eval_openwebtext_ppl": 33.05083646319603,
      "eval_openwebtext_runtime": 8.2918,
      "eval_openwebtext_samples_per_second": 517.863,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_wikitext_loss": 3.861328125,
      "eval_wikitext_ppl": 47.52843316962603,
      "eval_wikitext_runtime": 1.714,
      "eval_wikitext_samples_per_second": 266.047,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_lambada_loss": 4.51953125,
      "eval_lambada_ppl": 91.79256012938836,
      "eval_lambada_runtime": 9.2875,
      "eval_lambada_samples_per_second": 524.252,
      "step": 4000
    },
    {
      "activations/layer0_attention_weight_max": 13.985187530517578,
      "activations/layer0_attention_weight_min": -9.80319881439209,
      "activations/layer10_attention_weight_max": 35.9988899230957,
      "activations/layer10_attention_weight_min": -28.616117477416992,
      "activations/layer11_attention_weight_max": 41.271080017089844,
      "activations/layer11_attention_weight_min": -29.583106994628906,
      "activations/layer12_attention_weight_max": 25.849632263183594,
      "activations/layer12_attention_weight_min": -20.979782104492188,
      "activations/layer13_attention_weight_max": 22.022993087768555,
      "activations/layer13_attention_weight_min": -16.63528060913086,
      "activations/layer14_attention_weight_max": 23.703535079956055,
      "activations/layer14_attention_weight_min": -15.916817665100098,
      "activations/layer15_attention_weight_max": 26.092506408691406,
      "activations/layer15_attention_weight_min": -13.467781066894531,
      "activations/layer16_attention_weight_max": 20.95082664489746,
      "activations/layer16_attention_weight_min": -13.488088607788086,
      "activations/layer17_attention_weight_max": 20.28988265991211,
      "activations/layer17_attention_weight_min": -11.253746032714844,
      "activations/layer18_attention_weight_max": 15.413837432861328,
      "activations/layer18_attention_weight_min": -11.43777084350586,
      "activations/layer19_attention_weight_max": 19.973318099975586,
      "activations/layer19_attention_weight_min": -10.897003173828125,
      "activations/layer1_attention_weight_max": 14.344143867492676,
      "activations/layer1_attention_weight_min": -10.39432430267334,
      "activations/layer20_attention_weight_max": 16.704402923583984,
      "activations/layer20_attention_weight_min": -11.734694480895996,
      "activations/layer21_attention_weight_max": 17.04084587097168,
      "activations/layer21_attention_weight_min": -10.923500061035156,
      "activations/layer22_attention_weight_max": 17.432777404785156,
      "activations/layer22_attention_weight_min": -11.560379028320312,
      "activations/layer23_attention_weight_max": 9.742598533630371,
      "activations/layer23_attention_weight_min": -11.621538162231445,
      "activations/layer2_attention_weight_max": 28.56343650817871,
      "activations/layer2_attention_weight_min": -24.020858764648438,
      "activations/layer3_attention_weight_max": 63.880741119384766,
      "activations/layer3_attention_weight_min": -50.785037994384766,
      "activations/layer4_attention_weight_max": 148.12295532226562,
      "activations/layer4_attention_weight_min": -138.13499450683594,
      "activations/layer5_attention_weight_max": 164.1888427734375,
      "activations/layer5_attention_weight_min": -154.64080810546875,
      "activations/layer6_attention_weight_max": 117.76377868652344,
      "activations/layer6_attention_weight_min": -104.60327911376953,
      "activations/layer7_attention_weight_max": 97.76911926269531,
      "activations/layer7_attention_weight_min": -89.89813232421875,
      "activations/layer8_attention_weight_max": 76.23219299316406,
      "activations/layer8_attention_weight_min": -67.59202575683594,
      "activations/layer9_attention_weight_max": 66.64704895019531,
      "activations/layer9_attention_weight_min": -59.96152877807617,
      "epoch": 0.24,
      "learning_rate": 0.00014998825757575756,
      "loss": 3.5509,
      "step": 4050
    },
    {
      "activations/layer0_attention_weight_max": 13.687479972839355,
      "activations/layer0_attention_weight_min": -9.04620361328125,
      "activations/layer10_attention_weight_max": 42.52668762207031,
      "activations/layer10_attention_weight_min": -34.875675201416016,
      "activations/layer11_attention_weight_max": 52.91734313964844,
      "activations/layer11_attention_weight_min": -30.143592834472656,
      "activations/layer12_attention_weight_max": 26.769535064697266,
      "activations/layer12_attention_weight_min": -20.57627296447754,
      "activations/layer13_attention_weight_max": 29.299251556396484,
      "activations/layer13_attention_weight_min": -20.162668228149414,
      "activations/layer14_attention_weight_max": 28.509960174560547,
      "activations/layer14_attention_weight_min": -17.256805419921875,
      "activations/layer15_attention_weight_max": 22.020442962646484,
      "activations/layer15_attention_weight_min": -14.40626335144043,
      "activations/layer16_attention_weight_max": 26.770463943481445,
      "activations/layer16_attention_weight_min": -15.783912658691406,
      "activations/layer17_attention_weight_max": 24.10182762145996,
      "activations/layer17_attention_weight_min": -14.4817476272583,
      "activations/layer18_attention_weight_max": 20.73228645324707,
      "activations/layer18_attention_weight_min": -15.66333293914795,
      "activations/layer19_attention_weight_max": 26.77533531188965,
      "activations/layer19_attention_weight_min": -15.220349311828613,
      "activations/layer1_attention_weight_max": 17.32650375366211,
      "activations/layer1_attention_weight_min": -11.20948314666748,
      "activations/layer20_attention_weight_max": 23.9560489654541,
      "activations/layer20_attention_weight_min": -16.7386474609375,
      "activations/layer21_attention_weight_max": 20.610382080078125,
      "activations/layer21_attention_weight_min": -17.317739486694336,
      "activations/layer22_attention_weight_max": 15.824735641479492,
      "activations/layer22_attention_weight_min": -15.16848373413086,
      "activations/layer23_attention_weight_max": 11.008193016052246,
      "activations/layer23_attention_weight_min": -11.727348327636719,
      "activations/layer2_attention_weight_max": 27.436254501342773,
      "activations/layer2_attention_weight_min": -23.954511642456055,
      "activations/layer3_attention_weight_max": 62.735191345214844,
      "activations/layer3_attention_weight_min": -50.17133331298828,
      "activations/layer4_attention_weight_max": 151.28639221191406,
      "activations/layer4_attention_weight_min": -137.55084228515625,
      "activations/layer5_attention_weight_max": 167.14517211914062,
      "activations/layer5_attention_weight_min": -153.04055786132812,
      "activations/layer6_attention_weight_max": 116.82040405273438,
      "activations/layer6_attention_weight_min": -109.91681671142578,
      "activations/layer7_attention_weight_max": 96.51227569580078,
      "activations/layer7_attention_weight_min": -91.01038360595703,
      "activations/layer8_attention_weight_max": 77.95635986328125,
      "activations/layer8_attention_weight_min": -67.54061126708984,
      "activations/layer9_attention_weight_max": 63.805599212646484,
      "activations/layer9_attention_weight_min": -55.7121467590332,
      "epoch": 0.24,
      "learning_rate": 0.00014996931818181817,
      "loss": 3.5475,
      "step": 4100
    },
    {
      "activations/layer0_attention_weight_max": 14.299026489257812,
      "activations/layer0_attention_weight_min": -8.857845306396484,
      "activations/layer10_attention_weight_max": 38.088966369628906,
      "activations/layer10_attention_weight_min": -30.566387176513672,
      "activations/layer11_attention_weight_max": 42.70028305053711,
      "activations/layer11_attention_weight_min": -25.922780990600586,
      "activations/layer12_attention_weight_max": 24.438186645507812,
      "activations/layer12_attention_weight_min": -20.171260833740234,
      "activations/layer13_attention_weight_max": 23.51827049255371,
      "activations/layer13_attention_weight_min": -17.29286003112793,
      "activations/layer14_attention_weight_max": 27.303035736083984,
      "activations/layer14_attention_weight_min": -15.459999084472656,
      "activations/layer15_attention_weight_max": 22.0123291015625,
      "activations/layer15_attention_weight_min": -12.962275505065918,
      "activations/layer16_attention_weight_max": 23.59937286376953,
      "activations/layer16_attention_weight_min": -13.733925819396973,
      "activations/layer17_attention_weight_max": 25.173839569091797,
      "activations/layer17_attention_weight_min": -11.91171932220459,
      "activations/layer18_attention_weight_max": 21.06671714782715,
      "activations/layer18_attention_weight_min": -12.907763481140137,
      "activations/layer19_attention_weight_max": 26.343509674072266,
      "activations/layer19_attention_weight_min": -13.11210823059082,
      "activations/layer1_attention_weight_max": 15.44546890258789,
      "activations/layer1_attention_weight_min": -11.424164772033691,
      "activations/layer20_attention_weight_max": 20.366252899169922,
      "activations/layer20_attention_weight_min": -13.894335746765137,
      "activations/layer21_attention_weight_max": 21.03684425354004,
      "activations/layer21_attention_weight_min": -12.860634803771973,
      "activations/layer22_attention_weight_max": 17.40199089050293,
      "activations/layer22_attention_weight_min": -11.960515975952148,
      "activations/layer23_attention_weight_max": 11.850963592529297,
      "activations/layer23_attention_weight_min": -11.260340690612793,
      "activations/layer2_attention_weight_max": 29.115234375,
      "activations/layer2_attention_weight_min": -23.949094772338867,
      "activations/layer3_attention_weight_max": 61.80501937866211,
      "activations/layer3_attention_weight_min": -49.688053131103516,
      "activations/layer4_attention_weight_max": 147.74853515625,
      "activations/layer4_attention_weight_min": -135.5583038330078,
      "activations/layer5_attention_weight_max": 161.25418090820312,
      "activations/layer5_attention_weight_min": -146.46099853515625,
      "activations/layer6_attention_weight_max": 115.25447845458984,
      "activations/layer6_attention_weight_min": -106.1211166381836,
      "activations/layer7_attention_weight_max": 105.49381256103516,
      "activations/layer7_attention_weight_min": -89.1658935546875,
      "activations/layer8_attention_weight_max": 82.07681274414062,
      "activations/layer8_attention_weight_min": -67.3991470336914,
      "activations/layer9_attention_weight_max": 70.83807373046875,
      "activations/layer9_attention_weight_min": -54.90409469604492,
      "epoch": 0.24,
      "learning_rate": 0.00014995037878787876,
      "loss": 3.5497,
      "step": 4150
    },
    {
      "activations/layer0_attention_weight_max": 14.336769104003906,
      "activations/layer0_attention_weight_min": -9.065387725830078,
      "activations/layer10_attention_weight_max": 37.20797348022461,
      "activations/layer10_attention_weight_min": -30.473217010498047,
      "activations/layer11_attention_weight_max": 58.61468505859375,
      "activations/layer11_attention_weight_min": -30.251220703125,
      "activations/layer12_attention_weight_max": 24.062150955200195,
      "activations/layer12_attention_weight_min": -22.460763931274414,
      "activations/layer13_attention_weight_max": 25.946645736694336,
      "activations/layer13_attention_weight_min": -18.030134201049805,
      "activations/layer14_attention_weight_max": 25.055837631225586,
      "activations/layer14_attention_weight_min": -16.08523178100586,
      "activations/layer15_attention_weight_max": 29.739151000976562,
      "activations/layer15_attention_weight_min": -13.124486923217773,
      "activations/layer16_attention_weight_max": 26.64720916748047,
      "activations/layer16_attention_weight_min": -14.191629409790039,
      "activations/layer17_attention_weight_max": 19.443557739257812,
      "activations/layer17_attention_weight_min": -11.826471328735352,
      "activations/layer18_attention_weight_max": 17.64354133605957,
      "activations/layer18_attention_weight_min": -10.724456787109375,
      "activations/layer19_attention_weight_max": 20.091550827026367,
      "activations/layer19_attention_weight_min": -11.796940803527832,
      "activations/layer1_attention_weight_max": 16.058082580566406,
      "activations/layer1_attention_weight_min": -10.547609329223633,
      "activations/layer20_attention_weight_max": 19.376737594604492,
      "activations/layer20_attention_weight_min": -12.43896770477295,
      "activations/layer21_attention_weight_max": 17.48012351989746,
      "activations/layer21_attention_weight_min": -10.902023315429688,
      "activations/layer22_attention_weight_max": 15.742568969726562,
      "activations/layer22_attention_weight_min": -11.273164749145508,
      "activations/layer23_attention_weight_max": 10.33934211730957,
      "activations/layer23_attention_weight_min": -10.716817855834961,
      "activations/layer2_attention_weight_max": 29.702167510986328,
      "activations/layer2_attention_weight_min": -21.024860382080078,
      "activations/layer3_attention_weight_max": 63.7357292175293,
      "activations/layer3_attention_weight_min": -48.27507400512695,
      "activations/layer4_attention_weight_max": 145.2452392578125,
      "activations/layer4_attention_weight_min": -134.06304931640625,
      "activations/layer5_attention_weight_max": 182.2256317138672,
      "activations/layer5_attention_weight_min": -147.04052734375,
      "activations/layer6_attention_weight_max": 114.96834564208984,
      "activations/layer6_attention_weight_min": -104.10466766357422,
      "activations/layer7_attention_weight_max": 91.22920227050781,
      "activations/layer7_attention_weight_min": -82.42683410644531,
      "activations/layer8_attention_weight_max": 71.9625244140625,
      "activations/layer8_attention_weight_min": -63.49832534790039,
      "activations/layer9_attention_weight_max": 69.28226470947266,
      "activations/layer9_attention_weight_min": -63.4399299621582,
      "epoch": 0.24,
      "learning_rate": 0.00014993143939393938,
      "loss": 3.5352,
      "step": 4200
    },
    {
      "activations/layer0_attention_weight_max": 13.586446762084961,
      "activations/layer0_attention_weight_min": -10.318123817443848,
      "activations/layer10_attention_weight_max": 36.93972396850586,
      "activations/layer10_attention_weight_min": -29.657867431640625,
      "activations/layer11_attention_weight_max": 45.86949920654297,
      "activations/layer11_attention_weight_min": -28.429794311523438,
      "activations/layer12_attention_weight_max": 21.67638397216797,
      "activations/layer12_attention_weight_min": -20.836627960205078,
      "activations/layer13_attention_weight_max": 20.498620986938477,
      "activations/layer13_attention_weight_min": -19.774999618530273,
      "activations/layer14_attention_weight_max": 23.954448699951172,
      "activations/layer14_attention_weight_min": -16.40330696105957,
      "activations/layer15_attention_weight_max": 19.159643173217773,
      "activations/layer15_attention_weight_min": -13.766802787780762,
      "activations/layer16_attention_weight_max": 26.57149314880371,
      "activations/layer16_attention_weight_min": -14.321695327758789,
      "activations/layer17_attention_weight_max": 20.645740509033203,
      "activations/layer17_attention_weight_min": -11.904730796813965,
      "activations/layer18_attention_weight_max": 18.562641143798828,
      "activations/layer18_attention_weight_min": -11.926477432250977,
      "activations/layer19_attention_weight_max": 19.185720443725586,
      "activations/layer19_attention_weight_min": -12.561156272888184,
      "activations/layer1_attention_weight_max": 15.403565406799316,
      "activations/layer1_attention_weight_min": -9.887099266052246,
      "activations/layer20_attention_weight_max": 17.42605972290039,
      "activations/layer20_attention_weight_min": -13.23897933959961,
      "activations/layer21_attention_weight_max": 17.772720336914062,
      "activations/layer21_attention_weight_min": -13.710088729858398,
      "activations/layer22_attention_weight_max": 17.8472900390625,
      "activations/layer22_attention_weight_min": -11.76599407196045,
      "activations/layer23_attention_weight_max": 11.79651927947998,
      "activations/layer23_attention_weight_min": -11.087218284606934,
      "activations/layer2_attention_weight_max": 27.578685760498047,
      "activations/layer2_attention_weight_min": -22.240781784057617,
      "activations/layer3_attention_weight_max": 58.80665969848633,
      "activations/layer3_attention_weight_min": -49.59724426269531,
      "activations/layer4_attention_weight_max": 142.3127899169922,
      "activations/layer4_attention_weight_min": -127.86628723144531,
      "activations/layer5_attention_weight_max": 154.92295837402344,
      "activations/layer5_attention_weight_min": -145.687255859375,
      "activations/layer6_attention_weight_max": 111.1299057006836,
      "activations/layer6_attention_weight_min": -106.54763793945312,
      "activations/layer7_attention_weight_max": 93.07077026367188,
      "activations/layer7_attention_weight_min": -86.4644546508789,
      "activations/layer8_attention_weight_max": 70.2554702758789,
      "activations/layer8_attention_weight_min": -63.59593963623047,
      "activations/layer9_attention_weight_max": 68.393310546875,
      "activations/layer9_attention_weight_min": -61.624114990234375,
      "epoch": 0.25,
      "learning_rate": 0.0001499125,
      "loss": 3.5273,
      "step": 4250
    },
    {
      "activations/layer0_attention_weight_max": 15.668451309204102,
      "activations/layer0_attention_weight_min": -9.763626098632812,
      "activations/layer10_attention_weight_max": 41.3006477355957,
      "activations/layer10_attention_weight_min": -33.35648727416992,
      "activations/layer11_attention_weight_max": 38.51439666748047,
      "activations/layer11_attention_weight_min": -29.13417625427246,
      "activations/layer12_attention_weight_max": 25.754199981689453,
      "activations/layer12_attention_weight_min": -24.271121978759766,
      "activations/layer13_attention_weight_max": 25.446186065673828,
      "activations/layer13_attention_weight_min": -19.063858032226562,
      "activations/layer14_attention_weight_max": 28.41643714904785,
      "activations/layer14_attention_weight_min": -17.043350219726562,
      "activations/layer15_attention_weight_max": 21.01862335205078,
      "activations/layer15_attention_weight_min": -15.35168743133545,
      "activations/layer16_attention_weight_max": 17.958086013793945,
      "activations/layer16_attention_weight_min": -14.614182472229004,
      "activations/layer17_attention_weight_max": 20.001405715942383,
      "activations/layer17_attention_weight_min": -12.825202941894531,
      "activations/layer18_attention_weight_max": 19.325777053833008,
      "activations/layer18_attention_weight_min": -12.6908597946167,
      "activations/layer19_attention_weight_max": 18.637285232543945,
      "activations/layer19_attention_weight_min": -13.734152793884277,
      "activations/layer1_attention_weight_max": 16.37208366394043,
      "activations/layer1_attention_weight_min": -11.154767036437988,
      "activations/layer20_attention_weight_max": 19.703859329223633,
      "activations/layer20_attention_weight_min": -14.741379737854004,
      "activations/layer21_attention_weight_max": 18.43355941772461,
      "activations/layer21_attention_weight_min": -13.180598258972168,
      "activations/layer22_attention_weight_max": 15.037447929382324,
      "activations/layer22_attention_weight_min": -12.784056663513184,
      "activations/layer23_attention_weight_max": 10.42241096496582,
      "activations/layer23_attention_weight_min": -12.280657768249512,
      "activations/layer2_attention_weight_max": 27.878143310546875,
      "activations/layer2_attention_weight_min": -23.69586181640625,
      "activations/layer3_attention_weight_max": 64.67691040039062,
      "activations/layer3_attention_weight_min": -50.04977035522461,
      "activations/layer4_attention_weight_max": 139.8143310546875,
      "activations/layer4_attention_weight_min": -131.61651611328125,
      "activations/layer5_attention_weight_max": 158.28555297851562,
      "activations/layer5_attention_weight_min": -151.51495361328125,
      "activations/layer6_attention_weight_max": 110.97663879394531,
      "activations/layer6_attention_weight_min": -102.84278106689453,
      "activations/layer7_attention_weight_max": 90.44501495361328,
      "activations/layer7_attention_weight_min": -88.3715591430664,
      "activations/layer8_attention_weight_max": 75.57689666748047,
      "activations/layer8_attention_weight_min": -66.30740356445312,
      "activations/layer9_attention_weight_max": 66.50028228759766,
      "activations/layer9_attention_weight_min": -55.517494201660156,
      "epoch": 0.25,
      "learning_rate": 0.00014989356060606058,
      "loss": 3.5285,
      "step": 4300
    },
    {
      "activations/layer0_attention_weight_max": 14.521156311035156,
      "activations/layer0_attention_weight_min": -10.11100959777832,
      "activations/layer10_attention_weight_max": 41.4548454284668,
      "activations/layer10_attention_weight_min": -29.4017333984375,
      "activations/layer11_attention_weight_max": 37.76115798950195,
      "activations/layer11_attention_weight_min": -26.863506317138672,
      "activations/layer12_attention_weight_max": 24.49117660522461,
      "activations/layer12_attention_weight_min": -22.948638916015625,
      "activations/layer13_attention_weight_max": 21.927949905395508,
      "activations/layer13_attention_weight_min": -18.136734008789062,
      "activations/layer14_attention_weight_max": 25.807559967041016,
      "activations/layer14_attention_weight_min": -15.776267051696777,
      "activations/layer15_attention_weight_max": 24.187593460083008,
      "activations/layer15_attention_weight_min": -14.280619621276855,
      "activations/layer16_attention_weight_max": 27.93013572692871,
      "activations/layer16_attention_weight_min": -14.8715238571167,
      "activations/layer17_attention_weight_max": 23.19121742248535,
      "activations/layer17_attention_weight_min": -11.956674575805664,
      "activations/layer18_attention_weight_max": 24.052536010742188,
      "activations/layer18_attention_weight_min": -12.852314949035645,
      "activations/layer19_attention_weight_max": 25.51890754699707,
      "activations/layer19_attention_weight_min": -12.201024055480957,
      "activations/layer1_attention_weight_max": 16.138410568237305,
      "activations/layer1_attention_weight_min": -10.298686981201172,
      "activations/layer20_attention_weight_max": 23.904516220092773,
      "activations/layer20_attention_weight_min": -14.801862716674805,
      "activations/layer21_attention_weight_max": 24.370498657226562,
      "activations/layer21_attention_weight_min": -14.575636863708496,
      "activations/layer22_attention_weight_max": 23.28589630126953,
      "activations/layer22_attention_weight_min": -12.255423545837402,
      "activations/layer23_attention_weight_max": 11.653648376464844,
      "activations/layer23_attention_weight_min": -11.173506736755371,
      "activations/layer2_attention_weight_max": 26.76611328125,
      "activations/layer2_attention_weight_min": -22.315731048583984,
      "activations/layer3_attention_weight_max": 61.53958511352539,
      "activations/layer3_attention_weight_min": -48.10487365722656,
      "activations/layer4_attention_weight_max": 145.74530029296875,
      "activations/layer4_attention_weight_min": -127.4229736328125,
      "activations/layer5_attention_weight_max": 158.96878051757812,
      "activations/layer5_attention_weight_min": -142.76937866210938,
      "activations/layer6_attention_weight_max": 109.25645446777344,
      "activations/layer6_attention_weight_min": -98.83319854736328,
      "activations/layer7_attention_weight_max": 94.54444885253906,
      "activations/layer7_attention_weight_min": -87.32771301269531,
      "activations/layer8_attention_weight_max": 75.09889221191406,
      "activations/layer8_attention_weight_min": -64.327392578125,
      "activations/layer9_attention_weight_max": 63.171478271484375,
      "activations/layer9_attention_weight_min": -52.27608108520508,
      "epoch": 0.25,
      "learning_rate": 0.0001498746212121212,
      "loss": 3.513,
      "step": 4350
    },
    {
      "activations/layer0_attention_weight_max": 15.389748573303223,
      "activations/layer0_attention_weight_min": -10.488265991210938,
      "activations/layer10_attention_weight_max": 44.25088119506836,
      "activations/layer10_attention_weight_min": -32.575897216796875,
      "activations/layer11_attention_weight_max": 41.493896484375,
      "activations/layer11_attention_weight_min": -32.62300491333008,
      "activations/layer12_attention_weight_max": 25.31073570251465,
      "activations/layer12_attention_weight_min": -21.606826782226562,
      "activations/layer13_attention_weight_max": 24.099491119384766,
      "activations/layer13_attention_weight_min": -18.783132553100586,
      "activations/layer14_attention_weight_max": 25.43811798095703,
      "activations/layer14_attention_weight_min": -15.509679794311523,
      "activations/layer15_attention_weight_max": 22.456514358520508,
      "activations/layer15_attention_weight_min": -14.822218894958496,
      "activations/layer16_attention_weight_max": 24.75475311279297,
      "activations/layer16_attention_weight_min": -17.490800857543945,
      "activations/layer17_attention_weight_max": 22.80327033996582,
      "activations/layer17_attention_weight_min": -15.041898727416992,
      "activations/layer18_attention_weight_max": 19.658164978027344,
      "activations/layer18_attention_weight_min": -16.92555809020996,
      "activations/layer19_attention_weight_max": 22.08424949645996,
      "activations/layer19_attention_weight_min": -13.694464683532715,
      "activations/layer1_attention_weight_max": 15.247858047485352,
      "activations/layer1_attention_weight_min": -10.763195991516113,
      "activations/layer20_attention_weight_max": 22.620290756225586,
      "activations/layer20_attention_weight_min": -15.678119659423828,
      "activations/layer21_attention_weight_max": 22.813079833984375,
      "activations/layer21_attention_weight_min": -13.171867370605469,
      "activations/layer22_attention_weight_max": 18.38524055480957,
      "activations/layer22_attention_weight_min": -13.505980491638184,
      "activations/layer23_attention_weight_max": 11.182120323181152,
      "activations/layer23_attention_weight_min": -11.829788208007812,
      "activations/layer2_attention_weight_max": 29.06644058227539,
      "activations/layer2_attention_weight_min": -22.26417350769043,
      "activations/layer3_attention_weight_max": 63.54716110229492,
      "activations/layer3_attention_weight_min": -49.90116882324219,
      "activations/layer4_attention_weight_max": 149.89480590820312,
      "activations/layer4_attention_weight_min": -130.37132263183594,
      "activations/layer5_attention_weight_max": 172.474853515625,
      "activations/layer5_attention_weight_min": -144.84219360351562,
      "activations/layer6_attention_weight_max": 117.8432846069336,
      "activations/layer6_attention_weight_min": -103.23299407958984,
      "activations/layer7_attention_weight_max": 104.9614486694336,
      "activations/layer7_attention_weight_min": -89.63224792480469,
      "activations/layer8_attention_weight_max": 84.2253189086914,
      "activations/layer8_attention_weight_min": -66.58519744873047,
      "activations/layer9_attention_weight_max": 70.87984466552734,
      "activations/layer9_attention_weight_min": -57.87158966064453,
      "epoch": 0.26,
      "learning_rate": 0.0001498556818181818,
      "loss": 3.5035,
      "step": 4400
    },
    {
      "activations/layer0_attention_weight_max": 14.17226505279541,
      "activations/layer0_attention_weight_min": -10.14905834197998,
      "activations/layer10_attention_weight_max": 44.2579460144043,
      "activations/layer10_attention_weight_min": -31.99580955505371,
      "activations/layer11_attention_weight_max": 43.0290641784668,
      "activations/layer11_attention_weight_min": -35.523887634277344,
      "activations/layer12_attention_weight_max": 25.88957977294922,
      "activations/layer12_attention_weight_min": -21.557661056518555,
      "activations/layer13_attention_weight_max": 24.098058700561523,
      "activations/layer13_attention_weight_min": -20.681772232055664,
      "activations/layer14_attention_weight_max": 26.89327049255371,
      "activations/layer14_attention_weight_min": -17.571714401245117,
      "activations/layer15_attention_weight_max": 21.395009994506836,
      "activations/layer15_attention_weight_min": -14.383492469787598,
      "activations/layer16_attention_weight_max": 23.253517150878906,
      "activations/layer16_attention_weight_min": -17.107913970947266,
      "activations/layer17_attention_weight_max": 23.526342391967773,
      "activations/layer17_attention_weight_min": -13.19093132019043,
      "activations/layer18_attention_weight_max": 22.946468353271484,
      "activations/layer18_attention_weight_min": -12.163835525512695,
      "activations/layer19_attention_weight_max": 23.097265243530273,
      "activations/layer19_attention_weight_min": -12.843461990356445,
      "activations/layer1_attention_weight_max": 14.89555549621582,
      "activations/layer1_attention_weight_min": -10.710267066955566,
      "activations/layer20_attention_weight_max": 21.840373992919922,
      "activations/layer20_attention_weight_min": -12.54580307006836,
      "activations/layer21_attention_weight_max": 22.97171974182129,
      "activations/layer21_attention_weight_min": -11.869429588317871,
      "activations/layer22_attention_weight_max": 20.64950942993164,
      "activations/layer22_attention_weight_min": -12.69385051727295,
      "activations/layer23_attention_weight_max": 11.865470886230469,
      "activations/layer23_attention_weight_min": -11.353198051452637,
      "activations/layer2_attention_weight_max": 29.345176696777344,
      "activations/layer2_attention_weight_min": -22.606609344482422,
      "activations/layer3_attention_weight_max": 58.15686798095703,
      "activations/layer3_attention_weight_min": -50.12135314941406,
      "activations/layer4_attention_weight_max": 137.17318725585938,
      "activations/layer4_attention_weight_min": -129.2354278564453,
      "activations/layer5_attention_weight_max": 164.4781036376953,
      "activations/layer5_attention_weight_min": -143.74009704589844,
      "activations/layer6_attention_weight_max": 115.68830108642578,
      "activations/layer6_attention_weight_min": -100.0803451538086,
      "activations/layer7_attention_weight_max": 98.48015594482422,
      "activations/layer7_attention_weight_min": -83.33546447753906,
      "activations/layer8_attention_weight_max": 72.14119720458984,
      "activations/layer8_attention_weight_min": -66.62023162841797,
      "activations/layer9_attention_weight_max": 61.76418685913086,
      "activations/layer9_attention_weight_min": -54.65700149536133,
      "epoch": 0.26,
      "learning_rate": 0.00014983674242424242,
      "loss": 3.5061,
      "step": 4450
    },
    {
      "activations/layer0_attention_weight_max": 13.94973373413086,
      "activations/layer0_attention_weight_min": -9.454911231994629,
      "activations/layer10_attention_weight_max": 39.59148025512695,
      "activations/layer10_attention_weight_min": -34.192081451416016,
      "activations/layer11_attention_weight_max": 38.535316467285156,
      "activations/layer11_attention_weight_min": -33.4639892578125,
      "activations/layer12_attention_weight_max": 33.91617202758789,
      "activations/layer12_attention_weight_min": -19.196680068969727,
      "activations/layer13_attention_weight_max": 21.685148239135742,
      "activations/layer13_attention_weight_min": -17.376195907592773,
      "activations/layer14_attention_weight_max": 26.479007720947266,
      "activations/layer14_attention_weight_min": -16.92182159423828,
      "activations/layer15_attention_weight_max": 21.37024688720703,
      "activations/layer15_attention_weight_min": -16.60857582092285,
      "activations/layer16_attention_weight_max": 21.501956939697266,
      "activations/layer16_attention_weight_min": -13.941088676452637,
      "activations/layer17_attention_weight_max": 21.08783531188965,
      "activations/layer17_attention_weight_min": -13.446258544921875,
      "activations/layer18_attention_weight_max": 19.06102180480957,
      "activations/layer18_attention_weight_min": -11.734970092773438,
      "activations/layer19_attention_weight_max": 22.247013092041016,
      "activations/layer19_attention_weight_min": -12.067188262939453,
      "activations/layer1_attention_weight_max": 15.044002532958984,
      "activations/layer1_attention_weight_min": -10.291498184204102,
      "activations/layer20_attention_weight_max": 20.656904220581055,
      "activations/layer20_attention_weight_min": -12.061922073364258,
      "activations/layer21_attention_weight_max": 19.838254928588867,
      "activations/layer21_attention_weight_min": -11.250544548034668,
      "activations/layer22_attention_weight_max": 20.01163101196289,
      "activations/layer22_attention_weight_min": -11.282341957092285,
      "activations/layer23_attention_weight_max": 12.355854034423828,
      "activations/layer23_attention_weight_min": -11.948184967041016,
      "activations/layer2_attention_weight_max": 26.55322265625,
      "activations/layer2_attention_weight_min": -22.497146606445312,
      "activations/layer3_attention_weight_max": 61.44704818725586,
      "activations/layer3_attention_weight_min": -49.744632720947266,
      "activations/layer4_attention_weight_max": 140.15785217285156,
      "activations/layer4_attention_weight_min": -130.28700256347656,
      "activations/layer5_attention_weight_max": 154.07949829101562,
      "activations/layer5_attention_weight_min": -142.78321838378906,
      "activations/layer6_attention_weight_max": 112.99664306640625,
      "activations/layer6_attention_weight_min": -102.37572479248047,
      "activations/layer7_attention_weight_max": 95.53662872314453,
      "activations/layer7_attention_weight_min": -88.38067626953125,
      "activations/layer8_attention_weight_max": 73.98358154296875,
      "activations/layer8_attention_weight_min": -66.90189361572266,
      "activations/layer9_attention_weight_max": 62.50575637817383,
      "activations/layer9_attention_weight_min": -62.24208450317383,
      "epoch": 0.26,
      "learning_rate": 0.000149817803030303,
      "loss": 3.5162,
      "step": 4500
    },
    {
      "activations/layer0_attention_weight_max": 14.170581817626953,
      "activations/layer0_attention_weight_min": -9.137837409973145,
      "activations/layer10_attention_weight_max": 39.73866271972656,
      "activations/layer10_attention_weight_min": -33.04632568359375,
      "activations/layer11_attention_weight_max": 38.48619842529297,
      "activations/layer11_attention_weight_min": -29.832122802734375,
      "activations/layer12_attention_weight_max": 28.046005249023438,
      "activations/layer12_attention_weight_min": -21.364765167236328,
      "activations/layer13_attention_weight_max": 28.261272430419922,
      "activations/layer13_attention_weight_min": -20.1280517578125,
      "activations/layer14_attention_weight_max": 30.304197311401367,
      "activations/layer14_attention_weight_min": -18.05986976623535,
      "activations/layer15_attention_weight_max": 22.34140396118164,
      "activations/layer15_attention_weight_min": -14.180717468261719,
      "activations/layer16_attention_weight_max": 25.01644515991211,
      "activations/layer16_attention_weight_min": -15.068737983703613,
      "activations/layer17_attention_weight_max": 23.807659149169922,
      "activations/layer17_attention_weight_min": -12.684372901916504,
      "activations/layer18_attention_weight_max": 20.870214462280273,
      "activations/layer18_attention_weight_min": -11.741905212402344,
      "activations/layer19_attention_weight_max": 22.92987632751465,
      "activations/layer19_attention_weight_min": -12.651183128356934,
      "activations/layer1_attention_weight_max": 14.071968078613281,
      "activations/layer1_attention_weight_min": -11.153489112854004,
      "activations/layer20_attention_weight_max": 20.725370407104492,
      "activations/layer20_attention_weight_min": -12.787622451782227,
      "activations/layer21_attention_weight_max": 23.385265350341797,
      "activations/layer21_attention_weight_min": -12.893681526184082,
      "activations/layer22_attention_weight_max": 20.320301055908203,
      "activations/layer22_attention_weight_min": -12.74510669708252,
      "activations/layer23_attention_weight_max": 13.340507507324219,
      "activations/layer23_attention_weight_min": -11.263713836669922,
      "activations/layer2_attention_weight_max": 27.340152740478516,
      "activations/layer2_attention_weight_min": -22.984176635742188,
      "activations/layer3_attention_weight_max": 59.55887222290039,
      "activations/layer3_attention_weight_min": -48.72087860107422,
      "activations/layer4_attention_weight_max": 141.24769592285156,
      "activations/layer4_attention_weight_min": -125.36690521240234,
      "activations/layer5_attention_weight_max": 152.23507690429688,
      "activations/layer5_attention_weight_min": -140.801025390625,
      "activations/layer6_attention_weight_max": 103.52527618408203,
      "activations/layer6_attention_weight_min": -101.55973052978516,
      "activations/layer7_attention_weight_max": 91.95881652832031,
      "activations/layer7_attention_weight_min": -86.34844970703125,
      "activations/layer8_attention_weight_max": 72.15679931640625,
      "activations/layer8_attention_weight_min": -65.80180358886719,
      "activations/layer9_attention_weight_max": 66.71919250488281,
      "activations/layer9_attention_weight_min": -59.71514892578125,
      "epoch": 0.26,
      "learning_rate": 0.00014979886363636363,
      "loss": 3.5127,
      "step": 4550
    },
    {
      "activations/layer0_attention_weight_max": 13.526905059814453,
      "activations/layer0_attention_weight_min": -10.725086212158203,
      "activations/layer10_attention_weight_max": 40.793701171875,
      "activations/layer10_attention_weight_min": -32.60700607299805,
      "activations/layer11_attention_weight_max": 42.55181884765625,
      "activations/layer11_attention_weight_min": -31.663251876831055,
      "activations/layer12_attention_weight_max": 22.739885330200195,
      "activations/layer12_attention_weight_min": -19.40981101989746,
      "activations/layer13_attention_weight_max": 24.496776580810547,
      "activations/layer13_attention_weight_min": -17.280704498291016,
      "activations/layer14_attention_weight_max": 26.44816017150879,
      "activations/layer14_attention_weight_min": -19.10504722595215,
      "activations/layer15_attention_weight_max": 22.872234344482422,
      "activations/layer15_attention_weight_min": -14.138176918029785,
      "activations/layer16_attention_weight_max": 29.047353744506836,
      "activations/layer16_attention_weight_min": -13.643315315246582,
      "activations/layer17_attention_weight_max": 22.666643142700195,
      "activations/layer17_attention_weight_min": -12.152214050292969,
      "activations/layer18_attention_weight_max": 28.22004508972168,
      "activations/layer18_attention_weight_min": -13.22092342376709,
      "activations/layer19_attention_weight_max": 23.72249412536621,
      "activations/layer19_attention_weight_min": -12.958773612976074,
      "activations/layer1_attention_weight_max": 16.978530883789062,
      "activations/layer1_attention_weight_min": -10.838733673095703,
      "activations/layer20_attention_weight_max": 22.756059646606445,
      "activations/layer20_attention_weight_min": -12.932358741760254,
      "activations/layer21_attention_weight_max": 23.58055305480957,
      "activations/layer21_attention_weight_min": -11.233242988586426,
      "activations/layer22_attention_weight_max": 20.344297409057617,
      "activations/layer22_attention_weight_min": -10.760119438171387,
      "activations/layer23_attention_weight_max": 14.317797660827637,
      "activations/layer23_attention_weight_min": -10.162931442260742,
      "activations/layer2_attention_weight_max": 28.244047164916992,
      "activations/layer2_attention_weight_min": -21.65325927734375,
      "activations/layer3_attention_weight_max": 60.90782165527344,
      "activations/layer3_attention_weight_min": -49.39517593383789,
      "activations/layer4_attention_weight_max": 146.8376922607422,
      "activations/layer4_attention_weight_min": -128.7669677734375,
      "activations/layer5_attention_weight_max": 161.10092163085938,
      "activations/layer5_attention_weight_min": -142.84970092773438,
      "activations/layer6_attention_weight_max": 113.62081909179688,
      "activations/layer6_attention_weight_min": -101.70878601074219,
      "activations/layer7_attention_weight_max": 94.07263946533203,
      "activations/layer7_attention_weight_min": -86.4939193725586,
      "activations/layer8_attention_weight_max": 74.1936264038086,
      "activations/layer8_attention_weight_min": -61.88727569580078,
      "activations/layer9_attention_weight_max": 61.34149932861328,
      "activations/layer9_attention_weight_min": -57.31873321533203,
      "epoch": 0.27,
      "learning_rate": 0.00014977992424242424,
      "loss": 3.498,
      "step": 4600
    },
    {
      "activations/layer0_attention_weight_max": 14.425139427185059,
      "activations/layer0_attention_weight_min": -9.806253433227539,
      "activations/layer10_attention_weight_max": 38.39185333251953,
      "activations/layer10_attention_weight_min": -32.5825080871582,
      "activations/layer11_attention_weight_max": 47.92022705078125,
      "activations/layer11_attention_weight_min": -30.799762725830078,
      "activations/layer12_attention_weight_max": 22.7624454498291,
      "activations/layer12_attention_weight_min": -20.778949737548828,
      "activations/layer13_attention_weight_max": 35.52407455444336,
      "activations/layer13_attention_weight_min": -19.163476943969727,
      "activations/layer14_attention_weight_max": 25.198774337768555,
      "activations/layer14_attention_weight_min": -18.652597427368164,
      "activations/layer15_attention_weight_max": 24.89155387878418,
      "activations/layer15_attention_weight_min": -15.26882266998291,
      "activations/layer16_attention_weight_max": 21.965227127075195,
      "activations/layer16_attention_weight_min": -15.93740177154541,
      "activations/layer17_attention_weight_max": 21.493305206298828,
      "activations/layer17_attention_weight_min": -14.159860610961914,
      "activations/layer18_attention_weight_max": 22.933483123779297,
      "activations/layer18_attention_weight_min": -12.590950965881348,
      "activations/layer19_attention_weight_max": 22.77577018737793,
      "activations/layer19_attention_weight_min": -13.034997940063477,
      "activations/layer1_attention_weight_max": 14.514341354370117,
      "activations/layer1_attention_weight_min": -11.27602767944336,
      "activations/layer20_attention_weight_max": 24.050622940063477,
      "activations/layer20_attention_weight_min": -14.090697288513184,
      "activations/layer21_attention_weight_max": 20.347684860229492,
      "activations/layer21_attention_weight_min": -11.905364036560059,
      "activations/layer22_attention_weight_max": 20.020933151245117,
      "activations/layer22_attention_weight_min": -13.055415153503418,
      "activations/layer23_attention_weight_max": 13.701231956481934,
      "activations/layer23_attention_weight_min": -12.673487663269043,
      "activations/layer2_attention_weight_max": 27.736793518066406,
      "activations/layer2_attention_weight_min": -23.297163009643555,
      "activations/layer3_attention_weight_max": 61.41644287109375,
      "activations/layer3_attention_weight_min": -49.91581344604492,
      "activations/layer4_attention_weight_max": 141.0465087890625,
      "activations/layer4_attention_weight_min": -125.776611328125,
      "activations/layer5_attention_weight_max": 155.9442138671875,
      "activations/layer5_attention_weight_min": -143.76205444335938,
      "activations/layer6_attention_weight_max": 105.59209442138672,
      "activations/layer6_attention_weight_min": -99.99095153808594,
      "activations/layer7_attention_weight_max": 89.36817932128906,
      "activations/layer7_attention_weight_min": -85.2348861694336,
      "activations/layer8_attention_weight_max": 72.4789810180664,
      "activations/layer8_attention_weight_min": -70.41374969482422,
      "activations/layer9_attention_weight_max": 68.0929946899414,
      "activations/layer9_attention_weight_min": -62.71273422241211,
      "epoch": 0.27,
      "learning_rate": 0.00014976098484848483,
      "loss": 3.4947,
      "step": 4650
    },
    {
      "activations/layer0_attention_weight_max": 13.154271125793457,
      "activations/layer0_attention_weight_min": -9.662586212158203,
      "activations/layer10_attention_weight_max": 43.382568359375,
      "activations/layer10_attention_weight_min": -31.59879493713379,
      "activations/layer11_attention_weight_max": 47.86754608154297,
      "activations/layer11_attention_weight_min": -31.995885848999023,
      "activations/layer12_attention_weight_max": 25.858091354370117,
      "activations/layer12_attention_weight_min": -24.538253784179688,
      "activations/layer13_attention_weight_max": 23.568199157714844,
      "activations/layer13_attention_weight_min": -20.438159942626953,
      "activations/layer14_attention_weight_max": 25.60935401916504,
      "activations/layer14_attention_weight_min": -17.467397689819336,
      "activations/layer15_attention_weight_max": 23.78302574157715,
      "activations/layer15_attention_weight_min": -14.329142570495605,
      "activations/layer16_attention_weight_max": 20.92888069152832,
      "activations/layer16_attention_weight_min": -14.928295135498047,
      "activations/layer17_attention_weight_max": 22.850818634033203,
      "activations/layer17_attention_weight_min": -12.522814750671387,
      "activations/layer18_attention_weight_max": 20.805431365966797,
      "activations/layer18_attention_weight_min": -13.792717933654785,
      "activations/layer19_attention_weight_max": 21.09733009338379,
      "activations/layer19_attention_weight_min": -13.301749229431152,
      "activations/layer1_attention_weight_max": 14.986987113952637,
      "activations/layer1_attention_weight_min": -10.240408897399902,
      "activations/layer20_attention_weight_max": 23.464998245239258,
      "activations/layer20_attention_weight_min": -12.580050468444824,
      "activations/layer21_attention_weight_max": 20.053070068359375,
      "activations/layer21_attention_weight_min": -13.356608390808105,
      "activations/layer22_attention_weight_max": 17.59938621520996,
      "activations/layer22_attention_weight_min": -13.2550630569458,
      "activations/layer23_attention_weight_max": 12.501752853393555,
      "activations/layer23_attention_weight_min": -11.151714324951172,
      "activations/layer2_attention_weight_max": 27.78343963623047,
      "activations/layer2_attention_weight_min": -21.958173751831055,
      "activations/layer3_attention_weight_max": 57.94186782836914,
      "activations/layer3_attention_weight_min": -48.88104248046875,
      "activations/layer4_attention_weight_max": 139.91001892089844,
      "activations/layer4_attention_weight_min": -127.466064453125,
      "activations/layer5_attention_weight_max": 153.52767944335938,
      "activations/layer5_attention_weight_min": -140.47715759277344,
      "activations/layer6_attention_weight_max": 108.48280334472656,
      "activations/layer6_attention_weight_min": -92.64134979248047,
      "activations/layer7_attention_weight_max": 95.49440002441406,
      "activations/layer7_attention_weight_min": -80.73452758789062,
      "activations/layer8_attention_weight_max": 80.19989776611328,
      "activations/layer8_attention_weight_min": -66.73909759521484,
      "activations/layer9_attention_weight_max": 69.71385955810547,
      "activations/layer9_attention_weight_min": -55.75307846069336,
      "epoch": 0.27,
      "learning_rate": 0.00014974204545454545,
      "loss": 3.4797,
      "step": 4700
    },
    {
      "activations/layer0_attention_weight_max": 13.716052055358887,
      "activations/layer0_attention_weight_min": -9.597650527954102,
      "activations/layer10_attention_weight_max": 42.43306350708008,
      "activations/layer10_attention_weight_min": -32.828529357910156,
      "activations/layer11_attention_weight_max": 43.35151290893555,
      "activations/layer11_attention_weight_min": -34.51130676269531,
      "activations/layer12_attention_weight_max": 25.96582794189453,
      "activations/layer12_attention_weight_min": -21.416030883789062,
      "activations/layer13_attention_weight_max": 23.7441463470459,
      "activations/layer13_attention_weight_min": -20.777067184448242,
      "activations/layer14_attention_weight_max": 31.80124855041504,
      "activations/layer14_attention_weight_min": -15.839158058166504,
      "activations/layer15_attention_weight_max": 24.208677291870117,
      "activations/layer15_attention_weight_min": -13.548859596252441,
      "activations/layer16_attention_weight_max": 24.67110252380371,
      "activations/layer16_attention_weight_min": -13.482340812683105,
      "activations/layer17_attention_weight_max": 27.099170684814453,
      "activations/layer17_attention_weight_min": -12.817779541015625,
      "activations/layer18_attention_weight_max": 22.610565185546875,
      "activations/layer18_attention_weight_min": -11.141761779785156,
      "activations/layer19_attention_weight_max": 22.534420013427734,
      "activations/layer19_attention_weight_min": -13.271855354309082,
      "activations/layer1_attention_weight_max": 14.87696361541748,
      "activations/layer1_attention_weight_min": -10.73187255859375,
      "activations/layer20_attention_weight_max": 22.12735939025879,
      "activations/layer20_attention_weight_min": -11.287495613098145,
      "activations/layer21_attention_weight_max": 23.053752899169922,
      "activations/layer21_attention_weight_min": -11.32094669342041,
      "activations/layer22_attention_weight_max": 21.76963233947754,
      "activations/layer22_attention_weight_min": -11.302221298217773,
      "activations/layer23_attention_weight_max": 14.38448715209961,
      "activations/layer23_attention_weight_min": -10.900623321533203,
      "activations/layer2_attention_weight_max": 27.35814666748047,
      "activations/layer2_attention_weight_min": -22.30732536315918,
      "activations/layer3_attention_weight_max": 59.198760986328125,
      "activations/layer3_attention_weight_min": -48.439544677734375,
      "activations/layer4_attention_weight_max": 141.6135711669922,
      "activations/layer4_attention_weight_min": -124.4796142578125,
      "activations/layer5_attention_weight_max": 162.591552734375,
      "activations/layer5_attention_weight_min": -140.21932983398438,
      "activations/layer6_attention_weight_max": 114.10590362548828,
      "activations/layer6_attention_weight_min": -101.56278228759766,
      "activations/layer7_attention_weight_max": 98.58984375,
      "activations/layer7_attention_weight_min": -82.3147201538086,
      "activations/layer8_attention_weight_max": 78.48472595214844,
      "activations/layer8_attention_weight_min": -64.25071716308594,
      "activations/layer9_attention_weight_max": 74.09878540039062,
      "activations/layer9_attention_weight_min": -60.4398307800293,
      "epoch": 0.28,
      "learning_rate": 0.00014972310606060606,
      "loss": 3.479,
      "step": 4750
    },
    {
      "activations/layer0_attention_weight_max": 13.868612289428711,
      "activations/layer0_attention_weight_min": -9.956493377685547,
      "activations/layer10_attention_weight_max": 39.57795715332031,
      "activations/layer10_attention_weight_min": -33.01982498168945,
      "activations/layer11_attention_weight_max": 45.48167419433594,
      "activations/layer11_attention_weight_min": -29.930770874023438,
      "activations/layer12_attention_weight_max": 25.900602340698242,
      "activations/layer12_attention_weight_min": -21.75754737854004,
      "activations/layer13_attention_weight_max": 26.0784969329834,
      "activations/layer13_attention_weight_min": -19.239959716796875,
      "activations/layer14_attention_weight_max": 30.962772369384766,
      "activations/layer14_attention_weight_min": -17.079864501953125,
      "activations/layer15_attention_weight_max": 26.476715087890625,
      "activations/layer15_attention_weight_min": -14.82490348815918,
      "activations/layer16_attention_weight_max": 27.68473243713379,
      "activations/layer16_attention_weight_min": -17.87691879272461,
      "activations/layer17_attention_weight_max": 24.593183517456055,
      "activations/layer17_attention_weight_min": -19.330642700195312,
      "activations/layer18_attention_weight_max": 19.778125762939453,
      "activations/layer18_attention_weight_min": -11.954590797424316,
      "activations/layer19_attention_weight_max": 20.218307495117188,
      "activations/layer19_attention_weight_min": -15.805376052856445,
      "activations/layer1_attention_weight_max": 15.91889762878418,
      "activations/layer1_attention_weight_min": -10.592247009277344,
      "activations/layer20_attention_weight_max": 25.31248664855957,
      "activations/layer20_attention_weight_min": -14.759021759033203,
      "activations/layer21_attention_weight_max": 24.220853805541992,
      "activations/layer21_attention_weight_min": -14.208641052246094,
      "activations/layer22_attention_weight_max": 22.972415924072266,
      "activations/layer22_attention_weight_min": -13.98160171508789,
      "activations/layer23_attention_weight_max": 15.37722396850586,
      "activations/layer23_attention_weight_min": -10.609213829040527,
      "activations/layer2_attention_weight_max": 28.848194122314453,
      "activations/layer2_attention_weight_min": -22.13616180419922,
      "activations/layer3_attention_weight_max": 61.392295837402344,
      "activations/layer3_attention_weight_min": -48.949092864990234,
      "activations/layer4_attention_weight_max": 139.7754364013672,
      "activations/layer4_attention_weight_min": -124.15228271484375,
      "activations/layer5_attention_weight_max": 159.42874145507812,
      "activations/layer5_attention_weight_min": -141.57745361328125,
      "activations/layer6_attention_weight_max": 108.92755889892578,
      "activations/layer6_attention_weight_min": -95.05586242675781,
      "activations/layer7_attention_weight_max": 94.17042541503906,
      "activations/layer7_attention_weight_min": -83.40476989746094,
      "activations/layer8_attention_weight_max": 75.31814575195312,
      "activations/layer8_attention_weight_min": -64.74449157714844,
      "activations/layer9_attention_weight_max": 70.40681457519531,
      "activations/layer9_attention_weight_min": -61.14944076538086,
      "epoch": 0.28,
      "learning_rate": 0.00014970416666666665,
      "loss": 3.4697,
      "step": 4800
    },
    {
      "activations/layer0_attention_weight_max": 13.91628646850586,
      "activations/layer0_attention_weight_min": -10.333776473999023,
      "activations/layer10_attention_weight_max": 44.60341262817383,
      "activations/layer10_attention_weight_min": -35.71421432495117,
      "activations/layer11_attention_weight_max": 45.146724700927734,
      "activations/layer11_attention_weight_min": -29.861862182617188,
      "activations/layer12_attention_weight_max": 29.214130401611328,
      "activations/layer12_attention_weight_min": -22.792036056518555,
      "activations/layer13_attention_weight_max": 25.93415641784668,
      "activations/layer13_attention_weight_min": -20.232254028320312,
      "activations/layer14_attention_weight_max": 32.123958587646484,
      "activations/layer14_attention_weight_min": -19.60360336303711,
      "activations/layer15_attention_weight_max": 25.82895851135254,
      "activations/layer15_attention_weight_min": -16.288333892822266,
      "activations/layer16_attention_weight_max": 25.782970428466797,
      "activations/layer16_attention_weight_min": -14.466300964355469,
      "activations/layer17_attention_weight_max": 24.464080810546875,
      "activations/layer17_attention_weight_min": -12.215039253234863,
      "activations/layer18_attention_weight_max": 22.99156951904297,
      "activations/layer18_attention_weight_min": -12.134366035461426,
      "activations/layer19_attention_weight_max": 22.80047035217285,
      "activations/layer19_attention_weight_min": -13.966490745544434,
      "activations/layer1_attention_weight_max": 16.147762298583984,
      "activations/layer1_attention_weight_min": -11.220770835876465,
      "activations/layer20_attention_weight_max": 26.72936248779297,
      "activations/layer20_attention_weight_min": -14.29372501373291,
      "activations/layer21_attention_weight_max": 24.145376205444336,
      "activations/layer21_attention_weight_min": -13.255526542663574,
      "activations/layer22_attention_weight_max": 19.497251510620117,
      "activations/layer22_attention_weight_min": -12.296324729919434,
      "activations/layer23_attention_weight_max": 14.618814468383789,
      "activations/layer23_attention_weight_min": -10.466038703918457,
      "activations/layer2_attention_weight_max": 27.737720489501953,
      "activations/layer2_attention_weight_min": -22.123180389404297,
      "activations/layer3_attention_weight_max": 59.886714935302734,
      "activations/layer3_attention_weight_min": -47.9389762878418,
      "activations/layer4_attention_weight_max": 138.09140014648438,
      "activations/layer4_attention_weight_min": -126.982177734375,
      "activations/layer5_attention_weight_max": 156.31402587890625,
      "activations/layer5_attention_weight_min": -141.72271728515625,
      "activations/layer6_attention_weight_max": 102.67536163330078,
      "activations/layer6_attention_weight_min": -98.55701446533203,
      "activations/layer7_attention_weight_max": 88.57002258300781,
      "activations/layer7_attention_weight_min": -80.59529876708984,
      "activations/layer8_attention_weight_max": 71.70804595947266,
      "activations/layer8_attention_weight_min": -63.83274459838867,
      "activations/layer9_attention_weight_max": 67.10884094238281,
      "activations/layer9_attention_weight_min": -56.805885314941406,
      "epoch": 0.28,
      "learning_rate": 0.00014968522727272726,
      "loss": 3.4776,
      "step": 4850
    },
    {
      "activations/layer0_attention_weight_max": 14.247112274169922,
      "activations/layer0_attention_weight_min": -10.599998474121094,
      "activations/layer10_attention_weight_max": 40.63611602783203,
      "activations/layer10_attention_weight_min": -32.53804016113281,
      "activations/layer11_attention_weight_max": 41.86506652832031,
      "activations/layer11_attention_weight_min": -30.80173110961914,
      "activations/layer12_attention_weight_max": 26.89069938659668,
      "activations/layer12_attention_weight_min": -18.885757446289062,
      "activations/layer13_attention_weight_max": 25.709728240966797,
      "activations/layer13_attention_weight_min": -17.011627197265625,
      "activations/layer14_attention_weight_max": 29.66627311706543,
      "activations/layer14_attention_weight_min": -16.37114715576172,
      "activations/layer15_attention_weight_max": 21.723552703857422,
      "activations/layer15_attention_weight_min": -16.091541290283203,
      "activations/layer16_attention_weight_max": 26.6695556640625,
      "activations/layer16_attention_weight_min": -17.865741729736328,
      "activations/layer17_attention_weight_max": 22.265573501586914,
      "activations/layer17_attention_weight_min": -14.692482948303223,
      "activations/layer18_attention_weight_max": 23.662071228027344,
      "activations/layer18_attention_weight_min": -11.694818496704102,
      "activations/layer19_attention_weight_max": 23.183883666992188,
      "activations/layer19_attention_weight_min": -12.541770935058594,
      "activations/layer1_attention_weight_max": 13.390663146972656,
      "activations/layer1_attention_weight_min": -10.840611457824707,
      "activations/layer20_attention_weight_max": 23.607807159423828,
      "activations/layer20_attention_weight_min": -12.695645332336426,
      "activations/layer21_attention_weight_max": 23.525405883789062,
      "activations/layer21_attention_weight_min": -10.329261779785156,
      "activations/layer22_attention_weight_max": 20.324668884277344,
      "activations/layer22_attention_weight_min": -14.01736831665039,
      "activations/layer23_attention_weight_max": 15.564239501953125,
      "activations/layer23_attention_weight_min": -10.887495994567871,
      "activations/layer2_attention_weight_max": 27.56532096862793,
      "activations/layer2_attention_weight_min": -23.76198959350586,
      "activations/layer3_attention_weight_max": 58.563636779785156,
      "activations/layer3_attention_weight_min": -49.28629684448242,
      "activations/layer4_attention_weight_max": 140.70022583007812,
      "activations/layer4_attention_weight_min": -127.8305435180664,
      "activations/layer5_attention_weight_max": 153.93695068359375,
      "activations/layer5_attention_weight_min": -140.25265502929688,
      "activations/layer6_attention_weight_max": 102.53844451904297,
      "activations/layer6_attention_weight_min": -98.83132934570312,
      "activations/layer7_attention_weight_max": 90.51812744140625,
      "activations/layer7_attention_weight_min": -85.8973388671875,
      "activations/layer8_attention_weight_max": 70.5483169555664,
      "activations/layer8_attention_weight_min": -61.47780990600586,
      "activations/layer9_attention_weight_max": 63.119117736816406,
      "activations/layer9_attention_weight_min": -55.33455276489258,
      "epoch": 0.28,
      "learning_rate": 0.00014966628787878788,
      "loss": 3.4679,
      "step": 4900
    },
    {
      "activations/layer0_attention_weight_max": 13.98063850402832,
      "activations/layer0_attention_weight_min": -10.869291305541992,
      "activations/layer10_attention_weight_max": 40.333229064941406,
      "activations/layer10_attention_weight_min": -34.302207946777344,
      "activations/layer11_attention_weight_max": 45.56269073486328,
      "activations/layer11_attention_weight_min": -37.06290054321289,
      "activations/layer12_attention_weight_max": 21.87247085571289,
      "activations/layer12_attention_weight_min": -20.713790893554688,
      "activations/layer13_attention_weight_max": 25.79914665222168,
      "activations/layer13_attention_weight_min": -21.535655975341797,
      "activations/layer14_attention_weight_max": 25.250293731689453,
      "activations/layer14_attention_weight_min": -20.145944595336914,
      "activations/layer15_attention_weight_max": 22.9383487701416,
      "activations/layer15_attention_weight_min": -15.656569480895996,
      "activations/layer16_attention_weight_max": 22.70720863342285,
      "activations/layer16_attention_weight_min": -17.138927459716797,
      "activations/layer17_attention_weight_max": 20.301456451416016,
      "activations/layer17_attention_weight_min": -15.85869312286377,
      "activations/layer18_attention_weight_max": 19.32071304321289,
      "activations/layer18_attention_weight_min": -14.695443153381348,
      "activations/layer19_attention_weight_max": 23.760557174682617,
      "activations/layer19_attention_weight_min": -16.795873641967773,
      "activations/layer1_attention_weight_max": 14.457402229309082,
      "activations/layer1_attention_weight_min": -11.701695442199707,
      "activations/layer20_attention_weight_max": 29.017288208007812,
      "activations/layer20_attention_weight_min": -16.818279266357422,
      "activations/layer21_attention_weight_max": 19.06521987915039,
      "activations/layer21_attention_weight_min": -16.065195083618164,
      "activations/layer22_attention_weight_max": 18.804054260253906,
      "activations/layer22_attention_weight_min": -17.453737258911133,
      "activations/layer23_attention_weight_max": 15.664131164550781,
      "activations/layer23_attention_weight_min": -12.604738235473633,
      "activations/layer2_attention_weight_max": 26.88193130493164,
      "activations/layer2_attention_weight_min": -24.10993194580078,
      "activations/layer3_attention_weight_max": 59.36647415161133,
      "activations/layer3_attention_weight_min": -49.15376281738281,
      "activations/layer4_attention_weight_max": 136.04026794433594,
      "activations/layer4_attention_weight_min": -124.91741180419922,
      "activations/layer5_attention_weight_max": 150.56832885742188,
      "activations/layer5_attention_weight_min": -137.44482421875,
      "activations/layer6_attention_weight_max": 97.55001068115234,
      "activations/layer6_attention_weight_min": -97.05567169189453,
      "activations/layer7_attention_weight_max": 90.49404907226562,
      "activations/layer7_attention_weight_min": -83.23373413085938,
      "activations/layer8_attention_weight_max": 73.62699127197266,
      "activations/layer8_attention_weight_min": -64.13368225097656,
      "activations/layer9_attention_weight_max": 69.57917785644531,
      "activations/layer9_attention_weight_min": -62.33028030395508,
      "epoch": 0.29,
      "learning_rate": 0.00014964734848484847,
      "loss": 3.4661,
      "step": 4950
    },
    {
      "activations/layer0_attention_weight_max": 14.832632064819336,
      "activations/layer0_attention_weight_min": -9.709539413452148,
      "activations/layer10_attention_weight_max": 38.33709716796875,
      "activations/layer10_attention_weight_min": -32.03416061401367,
      "activations/layer11_attention_weight_max": 45.96046447753906,
      "activations/layer11_attention_weight_min": -32.59434509277344,
      "activations/layer12_attention_weight_max": 26.131332397460938,
      "activations/layer12_attention_weight_min": -22.369129180908203,
      "activations/layer13_attention_weight_max": 28.09830093383789,
      "activations/layer13_attention_weight_min": -19.31171989440918,
      "activations/layer14_attention_weight_max": 27.53217315673828,
      "activations/layer14_attention_weight_min": -15.556988716125488,
      "activations/layer15_attention_weight_max": 20.837970733642578,
      "activations/layer15_attention_weight_min": -14.609780311584473,
      "activations/layer16_attention_weight_max": 20.89406967163086,
      "activations/layer16_attention_weight_min": -14.818265914916992,
      "activations/layer17_attention_weight_max": 18.34825325012207,
      "activations/layer17_attention_weight_min": -12.183284759521484,
      "activations/layer18_attention_weight_max": 19.16586685180664,
      "activations/layer18_attention_weight_min": -12.509510040283203,
      "activations/layer19_attention_weight_max": 24.663846969604492,
      "activations/layer19_attention_weight_min": -13.454867362976074,
      "activations/layer1_attention_weight_max": 14.471931457519531,
      "activations/layer1_attention_weight_min": -11.062031745910645,
      "activations/layer20_attention_weight_max": 24.728286743164062,
      "activations/layer20_attention_weight_min": -13.5780029296875,
      "activations/layer21_attention_weight_max": 21.70351219177246,
      "activations/layer21_attention_weight_min": -13.624608039855957,
      "activations/layer22_attention_weight_max": 19.083295822143555,
      "activations/layer22_attention_weight_min": -13.45443344116211,
      "activations/layer23_attention_weight_max": 14.329858779907227,
      "activations/layer23_attention_weight_min": -11.522726058959961,
      "activations/layer2_attention_weight_max": 27.751646041870117,
      "activations/layer2_attention_weight_min": -23.405052185058594,
      "activations/layer3_attention_weight_max": 59.12879180908203,
      "activations/layer3_attention_weight_min": -47.41215133666992,
      "activations/layer4_attention_weight_max": 137.5617218017578,
      "activations/layer4_attention_weight_min": -122.14900207519531,
      "activations/layer5_attention_weight_max": 153.932373046875,
      "activations/layer5_attention_weight_min": -137.15357971191406,
      "activations/layer6_attention_weight_max": 101.49674987792969,
      "activations/layer6_attention_weight_min": -93.33280944824219,
      "activations/layer7_attention_weight_max": 91.11746215820312,
      "activations/layer7_attention_weight_min": -81.5041732788086,
      "activations/layer8_attention_weight_max": 75.29642486572266,
      "activations/layer8_attention_weight_min": -68.2488021850586,
      "activations/layer9_attention_weight_max": 69.0296630859375,
      "activations/layer9_attention_weight_min": -59.1517219543457,
      "epoch": 0.29,
      "learning_rate": 0.00014962840909090908,
      "loss": 3.4535,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_loss": 3.40625,
      "eval_runtime": 8.2504,
      "eval_samples_per_second": 520.457,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_openwebtext_loss": 3.40625,
      "eval_openwebtext_ppl": 30.151962130172098,
      "eval_openwebtext_runtime": 8.2504,
      "eval_openwebtext_samples_per_second": 520.457,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_wikitext_loss": 3.755859375,
      "eval_wikitext_ppl": 42.77096031537661,
      "eval_wikitext_runtime": 1.7121,
      "eval_wikitext_samples_per_second": 266.336,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_lambada_loss": 4.3828125,
      "eval_lambada_ppl": 80.06289393593435,
      "eval_lambada_runtime": 9.3092,
      "eval_lambada_samples_per_second": 523.032,
      "step": 5000
    },
    {
      "activations/layer0_attention_weight_max": 14.65009880065918,
      "activations/layer0_attention_weight_min": -9.81868839263916,
      "activations/layer10_attention_weight_max": 42.744964599609375,
      "activations/layer10_attention_weight_min": -35.87015151977539,
      "activations/layer11_attention_weight_max": 50.47193908691406,
      "activations/layer11_attention_weight_min": -32.437992095947266,
      "activations/layer12_attention_weight_max": 26.3217830657959,
      "activations/layer12_attention_weight_min": -19.26307487487793,
      "activations/layer13_attention_weight_max": 28.687541961669922,
      "activations/layer13_attention_weight_min": -20.266685485839844,
      "activations/layer14_attention_weight_max": 30.565034866333008,
      "activations/layer14_attention_weight_min": -17.882963180541992,
      "activations/layer15_attention_weight_max": 21.385812759399414,
      "activations/layer15_attention_weight_min": -13.844715118408203,
      "activations/layer16_attention_weight_max": 27.41739845275879,
      "activations/layer16_attention_weight_min": -15.588930130004883,
      "activations/layer17_attention_weight_max": 27.199169158935547,
      "activations/layer17_attention_weight_min": -13.327643394470215,
      "activations/layer18_attention_weight_max": 22.494909286499023,
      "activations/layer18_attention_weight_min": -12.295207977294922,
      "activations/layer19_attention_weight_max": 23.65102767944336,
      "activations/layer19_attention_weight_min": -14.621991157531738,
      "activations/layer1_attention_weight_max": 15.282378196716309,
      "activations/layer1_attention_weight_min": -10.864737510681152,
      "activations/layer20_attention_weight_max": 26.2795467376709,
      "activations/layer20_attention_weight_min": -12.501065254211426,
      "activations/layer21_attention_weight_max": 23.416275024414062,
      "activations/layer21_attention_weight_min": -13.105376243591309,
      "activations/layer22_attention_weight_max": 19.242799758911133,
      "activations/layer22_attention_weight_min": -11.537559509277344,
      "activations/layer23_attention_weight_max": 14.74740982055664,
      "activations/layer23_attention_weight_min": -10.2111177444458,
      "activations/layer2_attention_weight_max": 28.263050079345703,
      "activations/layer2_attention_weight_min": -21.886951446533203,
      "activations/layer3_attention_weight_max": 59.484371185302734,
      "activations/layer3_attention_weight_min": -48.108097076416016,
      "activations/layer4_attention_weight_max": 140.8570556640625,
      "activations/layer4_attention_weight_min": -121.72212982177734,
      "activations/layer5_attention_weight_max": 157.61619567871094,
      "activations/layer5_attention_weight_min": -138.838623046875,
      "activations/layer6_attention_weight_max": 107.25809478759766,
      "activations/layer6_attention_weight_min": -95.58790588378906,
      "activations/layer7_attention_weight_max": 97.62382507324219,
      "activations/layer7_attention_weight_min": -84.5980224609375,
      "activations/layer8_attention_weight_max": 72.70631408691406,
      "activations/layer8_attention_weight_min": -62.93135070800781,
      "activations/layer9_attention_weight_max": 66.60530853271484,
      "activations/layer9_attention_weight_min": -60.52497482299805,
      "epoch": 0.29,
      "learning_rate": 0.00014960984848484847,
      "loss": 3.4571,
      "step": 5050
    },
    {
      "activations/layer0_attention_weight_max": 14.63014030456543,
      "activations/layer0_attention_weight_min": -10.463767051696777,
      "activations/layer10_attention_weight_max": 43.20804214477539,
      "activations/layer10_attention_weight_min": -36.34432601928711,
      "activations/layer11_attention_weight_max": 51.86796188354492,
      "activations/layer11_attention_weight_min": -30.71406364440918,
      "activations/layer12_attention_weight_max": 28.61115837097168,
      "activations/layer12_attention_weight_min": -23.34382438659668,
      "activations/layer13_attention_weight_max": 26.953195571899414,
      "activations/layer13_attention_weight_min": -22.97038459777832,
      "activations/layer14_attention_weight_max": 26.90274429321289,
      "activations/layer14_attention_weight_min": -18.788455963134766,
      "activations/layer15_attention_weight_max": 22.207380294799805,
      "activations/layer15_attention_weight_min": -16.20039176940918,
      "activations/layer16_attention_weight_max": 26.48468017578125,
      "activations/layer16_attention_weight_min": -16.054855346679688,
      "activations/layer17_attention_weight_max": 20.7725772857666,
      "activations/layer17_attention_weight_min": -13.161126136779785,
      "activations/layer18_attention_weight_max": 20.679826736450195,
      "activations/layer18_attention_weight_min": -13.942399024963379,
      "activations/layer19_attention_weight_max": 20.881837844848633,
      "activations/layer19_attention_weight_min": -12.312182426452637,
      "activations/layer1_attention_weight_max": 15.479012489318848,
      "activations/layer1_attention_weight_min": -10.46456241607666,
      "activations/layer20_attention_weight_max": 21.968286514282227,
      "activations/layer20_attention_weight_min": -14.875041961669922,
      "activations/layer21_attention_weight_max": 20.684463500976562,
      "activations/layer21_attention_weight_min": -11.44334602355957,
      "activations/layer22_attention_weight_max": 19.81435203552246,
      "activations/layer22_attention_weight_min": -12.641275405883789,
      "activations/layer23_attention_weight_max": 14.278430938720703,
      "activations/layer23_attention_weight_min": -11.95080852508545,
      "activations/layer2_attention_weight_max": 27.958179473876953,
      "activations/layer2_attention_weight_min": -23.118810653686523,
      "activations/layer3_attention_weight_max": 62.56949234008789,
      "activations/layer3_attention_weight_min": -48.303001403808594,
      "activations/layer4_attention_weight_max": 149.57164001464844,
      "activations/layer4_attention_weight_min": -120.15377044677734,
      "activations/layer5_attention_weight_max": 164.75245666503906,
      "activations/layer5_attention_weight_min": -135.51702880859375,
      "activations/layer6_attention_weight_max": 105.54157257080078,
      "activations/layer6_attention_weight_min": -90.85556030273438,
      "activations/layer7_attention_weight_max": 93.57746124267578,
      "activations/layer7_attention_weight_min": -79.61451721191406,
      "activations/layer8_attention_weight_max": 79.29512786865234,
      "activations/layer8_attention_weight_min": -64.17152404785156,
      "activations/layer9_attention_weight_max": 65.90898895263672,
      "activations/layer9_attention_weight_min": -56.92597579956055,
      "epoch": 0.3,
      "learning_rate": 0.0001495909090909091,
      "loss": 3.4523,
      "step": 5100
    },
    {
      "activations/layer0_attention_weight_max": 14.430399894714355,
      "activations/layer0_attention_weight_min": -9.612822532653809,
      "activations/layer10_attention_weight_max": 41.32395935058594,
      "activations/layer10_attention_weight_min": -33.072975158691406,
      "activations/layer11_attention_weight_max": 46.810646057128906,
      "activations/layer11_attention_weight_min": -33.32985305786133,
      "activations/layer12_attention_weight_max": 24.283416748046875,
      "activations/layer12_attention_weight_min": -24.413619995117188,
      "activations/layer13_attention_weight_max": 24.280298233032227,
      "activations/layer13_attention_weight_min": -18.839170455932617,
      "activations/layer14_attention_weight_max": 30.149999618530273,
      "activations/layer14_attention_weight_min": -17.187999725341797,
      "activations/layer15_attention_weight_max": 29.07382583618164,
      "activations/layer15_attention_weight_min": -14.608851432800293,
      "activations/layer16_attention_weight_max": 25.253700256347656,
      "activations/layer16_attention_weight_min": -14.246243476867676,
      "activations/layer17_attention_weight_max": 24.928171157836914,
      "activations/layer17_attention_weight_min": -15.570395469665527,
      "activations/layer18_attention_weight_max": 18.269062042236328,
      "activations/layer18_attention_weight_min": -11.671618461608887,
      "activations/layer19_attention_weight_max": 23.57645034790039,
      "activations/layer19_attention_weight_min": -15.608896255493164,
      "activations/layer1_attention_weight_max": 16.81757354736328,
      "activations/layer1_attention_weight_min": -11.646620750427246,
      "activations/layer20_attention_weight_max": 21.68817710876465,
      "activations/layer20_attention_weight_min": -15.355293273925781,
      "activations/layer21_attention_weight_max": 21.260622024536133,
      "activations/layer21_attention_weight_min": -13.956920623779297,
      "activations/layer22_attention_weight_max": 18.751707077026367,
      "activations/layer22_attention_weight_min": -13.088558197021484,
      "activations/layer23_attention_weight_max": 18.64419937133789,
      "activations/layer23_attention_weight_min": -10.329946517944336,
      "activations/layer2_attention_weight_max": 27.056072235107422,
      "activations/layer2_attention_weight_min": -22.456151962280273,
      "activations/layer3_attention_weight_max": 58.594661712646484,
      "activations/layer3_attention_weight_min": -47.31672286987305,
      "activations/layer4_attention_weight_max": 135.37709045410156,
      "activations/layer4_attention_weight_min": -122.0475082397461,
      "activations/layer5_attention_weight_max": 151.7952117919922,
      "activations/layer5_attention_weight_min": -135.3251495361328,
      "activations/layer6_attention_weight_max": 106.10391998291016,
      "activations/layer6_attention_weight_min": -95.25981140136719,
      "activations/layer7_attention_weight_max": 90.3153305053711,
      "activations/layer7_attention_weight_min": -77.93899536132812,
      "activations/layer8_attention_weight_max": 72.93321990966797,
      "activations/layer8_attention_weight_min": -61.22195816040039,
      "activations/layer9_attention_weight_max": 69.80657196044922,
      "activations/layer9_attention_weight_min": -59.69009017944336,
      "epoch": 0.3,
      "learning_rate": 0.00014957196969696968,
      "loss": 3.44,
      "step": 5150
    },
    {
      "activations/layer0_attention_weight_max": 14.43544864654541,
      "activations/layer0_attention_weight_min": -9.7821044921875,
      "activations/layer10_attention_weight_max": 45.006317138671875,
      "activations/layer10_attention_weight_min": -35.311241149902344,
      "activations/layer11_attention_weight_max": 47.091365814208984,
      "activations/layer11_attention_weight_min": -33.700523376464844,
      "activations/layer12_attention_weight_max": 24.416290283203125,
      "activations/layer12_attention_weight_min": -21.43471336364746,
      "activations/layer13_attention_weight_max": 21.484827041625977,
      "activations/layer13_attention_weight_min": -21.079666137695312,
      "activations/layer14_attention_weight_max": 29.267318725585938,
      "activations/layer14_attention_weight_min": -19.84400749206543,
      "activations/layer15_attention_weight_max": 25.01402473449707,
      "activations/layer15_attention_weight_min": -16.83325958251953,
      "activations/layer16_attention_weight_max": 22.662633895874023,
      "activations/layer16_attention_weight_min": -14.633295059204102,
      "activations/layer17_attention_weight_max": 28.34868812561035,
      "activations/layer17_attention_weight_min": -13.659510612487793,
      "activations/layer18_attention_weight_max": 22.64925765991211,
      "activations/layer18_attention_weight_min": -11.992313385009766,
      "activations/layer19_attention_weight_max": 22.065967559814453,
      "activations/layer19_attention_weight_min": -14.665350914001465,
      "activations/layer1_attention_weight_max": 16.067729949951172,
      "activations/layer1_attention_weight_min": -11.139044761657715,
      "activations/layer20_attention_weight_max": 23.854141235351562,
      "activations/layer20_attention_weight_min": -11.620845794677734,
      "activations/layer21_attention_weight_max": 21.2235164642334,
      "activations/layer21_attention_weight_min": -11.327603340148926,
      "activations/layer22_attention_weight_max": 19.587034225463867,
      "activations/layer22_attention_weight_min": -14.066057205200195,
      "activations/layer23_attention_weight_max": 16.976015090942383,
      "activations/layer23_attention_weight_min": -12.237247467041016,
      "activations/layer2_attention_weight_max": 27.119367599487305,
      "activations/layer2_attention_weight_min": -21.944387435913086,
      "activations/layer3_attention_weight_max": 58.78828048706055,
      "activations/layer3_attention_weight_min": -46.783145904541016,
      "activations/layer4_attention_weight_max": 131.0020294189453,
      "activations/layer4_attention_weight_min": -120.77398681640625,
      "activations/layer5_attention_weight_max": 147.83224487304688,
      "activations/layer5_attention_weight_min": -136.4588165283203,
      "activations/layer6_attention_weight_max": 101.94453430175781,
      "activations/layer6_attention_weight_min": -91.59807586669922,
      "activations/layer7_attention_weight_max": 89.58062744140625,
      "activations/layer7_attention_weight_min": -78.23480987548828,
      "activations/layer8_attention_weight_max": 74.73551177978516,
      "activations/layer8_attention_weight_min": -65.4964599609375,
      "activations/layer9_attention_weight_max": 65.58380889892578,
      "activations/layer9_attention_weight_min": -59.07139205932617,
      "epoch": 0.3,
      "learning_rate": 0.0001495530303030303,
      "loss": 3.4563,
      "step": 5200
    },
    {
      "activations/layer0_attention_weight_max": 14.779890060424805,
      "activations/layer0_attention_weight_min": -9.894743919372559,
      "activations/layer10_attention_weight_max": 44.935672760009766,
      "activations/layer10_attention_weight_min": -34.34274673461914,
      "activations/layer11_attention_weight_max": 42.97062301635742,
      "activations/layer11_attention_weight_min": -33.60270309448242,
      "activations/layer12_attention_weight_max": 25.58356285095215,
      "activations/layer12_attention_weight_min": -20.489892959594727,
      "activations/layer13_attention_weight_max": 27.45720100402832,
      "activations/layer13_attention_weight_min": -21.511877059936523,
      "activations/layer14_attention_weight_max": 27.50673484802246,
      "activations/layer14_attention_weight_min": -19.519691467285156,
      "activations/layer15_attention_weight_max": 23.905925750732422,
      "activations/layer15_attention_weight_min": -16.043020248413086,
      "activations/layer16_attention_weight_max": 21.77315330505371,
      "activations/layer16_attention_weight_min": -16.693437576293945,
      "activations/layer17_attention_weight_max": 21.317527770996094,
      "activations/layer17_attention_weight_min": -14.109821319580078,
      "activations/layer18_attention_weight_max": 20.387027740478516,
      "activations/layer18_attention_weight_min": -11.474469184875488,
      "activations/layer19_attention_weight_max": 21.89504623413086,
      "activations/layer19_attention_weight_min": -13.024375915527344,
      "activations/layer1_attention_weight_max": 16.34079360961914,
      "activations/layer1_attention_weight_min": -10.875406265258789,
      "activations/layer20_attention_weight_max": 21.82775115966797,
      "activations/layer20_attention_weight_min": -12.225674629211426,
      "activations/layer21_attention_weight_max": 21.197105407714844,
      "activations/layer21_attention_weight_min": -11.19961929321289,
      "activations/layer22_attention_weight_max": 19.966323852539062,
      "activations/layer22_attention_weight_min": -12.672067642211914,
      "activations/layer23_attention_weight_max": 16.368635177612305,
      "activations/layer23_attention_weight_min": -12.161028861999512,
      "activations/layer2_attention_weight_max": 29.04866600036621,
      "activations/layer2_attention_weight_min": -21.999494552612305,
      "activations/layer3_attention_weight_max": 57.22080993652344,
      "activations/layer3_attention_weight_min": -46.65696716308594,
      "activations/layer4_attention_weight_max": 134.4426727294922,
      "activations/layer4_attention_weight_min": -119.23326873779297,
      "activations/layer5_attention_weight_max": 145.54632568359375,
      "activations/layer5_attention_weight_min": -133.6243438720703,
      "activations/layer6_attention_weight_max": 94.30195617675781,
      "activations/layer6_attention_weight_min": -88.85803985595703,
      "activations/layer7_attention_weight_max": 85.66956329345703,
      "activations/layer7_attention_weight_min": -77.75025177001953,
      "activations/layer8_attention_weight_max": 72.77528381347656,
      "activations/layer8_attention_weight_min": -61.5086555480957,
      "activations/layer9_attention_weight_max": 65.82020568847656,
      "activations/layer9_attention_weight_min": -58.452266693115234,
      "epoch": 0.31,
      "learning_rate": 0.0001495340909090909,
      "loss": 3.449,
      "step": 5250
    },
    {
      "activations/layer0_attention_weight_max": 14.51034164428711,
      "activations/layer0_attention_weight_min": -10.181219100952148,
      "activations/layer10_attention_weight_max": 43.349857330322266,
      "activations/layer10_attention_weight_min": -36.339351654052734,
      "activations/layer11_attention_weight_max": 50.639556884765625,
      "activations/layer11_attention_weight_min": -38.8759651184082,
      "activations/layer12_attention_weight_max": 25.087024688720703,
      "activations/layer12_attention_weight_min": -19.736061096191406,
      "activations/layer13_attention_weight_max": 26.134084701538086,
      "activations/layer13_attention_weight_min": -18.5295467376709,
      "activations/layer14_attention_weight_max": 35.04308319091797,
      "activations/layer14_attention_weight_min": -20.05696678161621,
      "activations/layer15_attention_weight_max": 22.377946853637695,
      "activations/layer15_attention_weight_min": -16.202781677246094,
      "activations/layer16_attention_weight_max": 22.313650131225586,
      "activations/layer16_attention_weight_min": -14.118317604064941,
      "activations/layer17_attention_weight_max": 22.23272705078125,
      "activations/layer17_attention_weight_min": -14.094061851501465,
      "activations/layer18_attention_weight_max": 21.120712280273438,
      "activations/layer18_attention_weight_min": -12.132593154907227,
      "activations/layer19_attention_weight_max": 23.689895629882812,
      "activations/layer19_attention_weight_min": -12.434438705444336,
      "activations/layer1_attention_weight_max": 15.633452415466309,
      "activations/layer1_attention_weight_min": -11.606220245361328,
      "activations/layer20_attention_weight_max": 24.625898361206055,
      "activations/layer20_attention_weight_min": -11.637552261352539,
      "activations/layer21_attention_weight_max": 21.414209365844727,
      "activations/layer21_attention_weight_min": -10.669143676757812,
      "activations/layer22_attention_weight_max": 20.303529739379883,
      "activations/layer22_attention_weight_min": -12.864715576171875,
      "activations/layer23_attention_weight_max": 18.395374298095703,
      "activations/layer23_attention_weight_min": -11.002544403076172,
      "activations/layer2_attention_weight_max": 28.56943130493164,
      "activations/layer2_attention_weight_min": -20.6192569732666,
      "activations/layer3_attention_weight_max": 58.914180755615234,
      "activations/layer3_attention_weight_min": -46.888572692871094,
      "activations/layer4_attention_weight_max": 136.47938537597656,
      "activations/layer4_attention_weight_min": -118.3474349975586,
      "activations/layer5_attention_weight_max": 155.70631408691406,
      "activations/layer5_attention_weight_min": -134.31674194335938,
      "activations/layer6_attention_weight_max": 98.6922378540039,
      "activations/layer6_attention_weight_min": -93.7954330444336,
      "activations/layer7_attention_weight_max": 92.00001525878906,
      "activations/layer7_attention_weight_min": -81.35533142089844,
      "activations/layer8_attention_weight_max": 76.66666412353516,
      "activations/layer8_attention_weight_min": -65.24950408935547,
      "activations/layer9_attention_weight_max": 80.5796127319336,
      "activations/layer9_attention_weight_min": -74.96943664550781,
      "epoch": 0.31,
      "learning_rate": 0.0001495151515151515,
      "loss": 3.4572,
      "step": 5300
    },
    {
      "activations/layer0_attention_weight_max": 13.987022399902344,
      "activations/layer0_attention_weight_min": -10.074043273925781,
      "activations/layer10_attention_weight_max": 42.763938903808594,
      "activations/layer10_attention_weight_min": -33.00868225097656,
      "activations/layer11_attention_weight_max": 54.02699279785156,
      "activations/layer11_attention_weight_min": -37.80995178222656,
      "activations/layer12_attention_weight_max": 24.399574279785156,
      "activations/layer12_attention_weight_min": -23.35582160949707,
      "activations/layer13_attention_weight_max": 25.99732208251953,
      "activations/layer13_attention_weight_min": -25.084957122802734,
      "activations/layer14_attention_weight_max": 29.062726974487305,
      "activations/layer14_attention_weight_min": -21.09629249572754,
      "activations/layer15_attention_weight_max": 26.21569061279297,
      "activations/layer15_attention_weight_min": -17.396692276000977,
      "activations/layer16_attention_weight_max": 24.228439331054688,
      "activations/layer16_attention_weight_min": -16.80959129333496,
      "activations/layer17_attention_weight_max": 23.022180557250977,
      "activations/layer17_attention_weight_min": -14.821240425109863,
      "activations/layer18_attention_weight_max": 19.017108917236328,
      "activations/layer18_attention_weight_min": -12.883501052856445,
      "activations/layer19_attention_weight_max": 21.607336044311523,
      "activations/layer19_attention_weight_min": -12.214961051940918,
      "activations/layer1_attention_weight_max": 16.018564224243164,
      "activations/layer1_attention_weight_min": -11.285711288452148,
      "activations/layer20_attention_weight_max": 23.146703720092773,
      "activations/layer20_attention_weight_min": -13.68952465057373,
      "activations/layer21_attention_weight_max": 20.776042938232422,
      "activations/layer21_attention_weight_min": -11.47278118133545,
      "activations/layer22_attention_weight_max": 19.06574249267578,
      "activations/layer22_attention_weight_min": -12.292282104492188,
      "activations/layer23_attention_weight_max": 14.943258285522461,
      "activations/layer23_attention_weight_min": -11.658769607543945,
      "activations/layer2_attention_weight_max": 28.472412109375,
      "activations/layer2_attention_weight_min": -20.842899322509766,
      "activations/layer3_attention_weight_max": 57.209346771240234,
      "activations/layer3_attention_weight_min": -45.624534606933594,
      "activations/layer4_attention_weight_max": 131.22181701660156,
      "activations/layer4_attention_weight_min": -119.05181884765625,
      "activations/layer5_attention_weight_max": 150.59170532226562,
      "activations/layer5_attention_weight_min": -134.55628967285156,
      "activations/layer6_attention_weight_max": 98.14839172363281,
      "activations/layer6_attention_weight_min": -92.44694519042969,
      "activations/layer7_attention_weight_max": 88.7630615234375,
      "activations/layer7_attention_weight_min": -80.55470275878906,
      "activations/layer8_attention_weight_max": 71.416015625,
      "activations/layer8_attention_weight_min": -64.44381713867188,
      "activations/layer9_attention_weight_max": 66.00343322753906,
      "activations/layer9_attention_weight_min": -61.680885314941406,
      "epoch": 0.31,
      "learning_rate": 0.0001494962121212121,
      "loss": 3.4451,
      "step": 5350
    },
    {
      "activations/layer0_attention_weight_max": 13.802728652954102,
      "activations/layer0_attention_weight_min": -9.806487083435059,
      "activations/layer10_attention_weight_max": 40.72681427001953,
      "activations/layer10_attention_weight_min": -35.3203010559082,
      "activations/layer11_attention_weight_max": 51.30282211303711,
      "activations/layer11_attention_weight_min": -34.9826545715332,
      "activations/layer12_attention_weight_max": 32.23884201049805,
      "activations/layer12_attention_weight_min": -26.78469467163086,
      "activations/layer13_attention_weight_max": 33.27543258666992,
      "activations/layer13_attention_weight_min": -27.293439865112305,
      "activations/layer14_attention_weight_max": 25.480905532836914,
      "activations/layer14_attention_weight_min": -18.082870483398438,
      "activations/layer15_attention_weight_max": 19.588029861450195,
      "activations/layer15_attention_weight_min": -16.309080123901367,
      "activations/layer16_attention_weight_max": 22.393327713012695,
      "activations/layer16_attention_weight_min": -14.071632385253906,
      "activations/layer17_attention_weight_max": 20.94094467163086,
      "activations/layer17_attention_weight_min": -13.785480499267578,
      "activations/layer18_attention_weight_max": 18.436294555664062,
      "activations/layer18_attention_weight_min": -11.674878120422363,
      "activations/layer19_attention_weight_max": 18.27977752685547,
      "activations/layer19_attention_weight_min": -13.536786079406738,
      "activations/layer1_attention_weight_max": 13.820657730102539,
      "activations/layer1_attention_weight_min": -10.95143985748291,
      "activations/layer20_attention_weight_max": 21.243003845214844,
      "activations/layer20_attention_weight_min": -13.139121055603027,
      "activations/layer21_attention_weight_max": 21.585208892822266,
      "activations/layer21_attention_weight_min": -11.633153915405273,
      "activations/layer22_attention_weight_max": 17.1991024017334,
      "activations/layer22_attention_weight_min": -14.034122467041016,
      "activations/layer23_attention_weight_max": 14.269488334655762,
      "activations/layer23_attention_weight_min": -11.269071578979492,
      "activations/layer2_attention_weight_max": 28.345542907714844,
      "activations/layer2_attention_weight_min": -22.004150390625,
      "activations/layer3_attention_weight_max": 57.72613525390625,
      "activations/layer3_attention_weight_min": -47.4697380065918,
      "activations/layer4_attention_weight_max": 132.2952117919922,
      "activations/layer4_attention_weight_min": -118.18115234375,
      "activations/layer5_attention_weight_max": 155.130859375,
      "activations/layer5_attention_weight_min": -134.35140991210938,
      "activations/layer6_attention_weight_max": 100.61911010742188,
      "activations/layer6_attention_weight_min": -93.23712921142578,
      "activations/layer7_attention_weight_max": 90.02607727050781,
      "activations/layer7_attention_weight_min": -83.59214782714844,
      "activations/layer8_attention_weight_max": 73.98668670654297,
      "activations/layer8_attention_weight_min": -66.12638854980469,
      "activations/layer9_attention_weight_max": 73.37557983398438,
      "activations/layer9_attention_weight_min": -60.95415115356445,
      "epoch": 0.31,
      "learning_rate": 0.0001494772727272727,
      "loss": 3.4446,
      "step": 5400
    },
    {
      "activations/layer0_attention_weight_max": 13.85538101196289,
      "activations/layer0_attention_weight_min": -10.062813758850098,
      "activations/layer10_attention_weight_max": 41.27570724487305,
      "activations/layer10_attention_weight_min": -38.833106994628906,
      "activations/layer11_attention_weight_max": 40.49787902832031,
      "activations/layer11_attention_weight_min": -32.314971923828125,
      "activations/layer12_attention_weight_max": 24.9316463470459,
      "activations/layer12_attention_weight_min": -21.40397834777832,
      "activations/layer13_attention_weight_max": 24.857547760009766,
      "activations/layer13_attention_weight_min": -20.473857879638672,
      "activations/layer14_attention_weight_max": 32.74521255493164,
      "activations/layer14_attention_weight_min": -22.18634033203125,
      "activations/layer15_attention_weight_max": 25.585800170898438,
      "activations/layer15_attention_weight_min": -17.212751388549805,
      "activations/layer16_attention_weight_max": 24.15599822998047,
      "activations/layer16_attention_weight_min": -16.03965187072754,
      "activations/layer17_attention_weight_max": 24.404277801513672,
      "activations/layer17_attention_weight_min": -14.976431846618652,
      "activations/layer18_attention_weight_max": 21.483163833618164,
      "activations/layer18_attention_weight_min": -12.457538604736328,
      "activations/layer19_attention_weight_max": 22.581207275390625,
      "activations/layer19_attention_weight_min": -12.78950023651123,
      "activations/layer1_attention_weight_max": 14.537781715393066,
      "activations/layer1_attention_weight_min": -11.139418601989746,
      "activations/layer20_attention_weight_max": 24.21343994140625,
      "activations/layer20_attention_weight_min": -14.056962013244629,
      "activations/layer21_attention_weight_max": 23.343185424804688,
      "activations/layer21_attention_weight_min": -12.00660228729248,
      "activations/layer22_attention_weight_max": 20.861364364624023,
      "activations/layer22_attention_weight_min": -12.097149848937988,
      "activations/layer23_attention_weight_max": 16.1509952545166,
      "activations/layer23_attention_weight_min": -11.854631423950195,
      "activations/layer2_attention_weight_max": 27.362659454345703,
      "activations/layer2_attention_weight_min": -23.212947845458984,
      "activations/layer3_attention_weight_max": 54.91664123535156,
      "activations/layer3_attention_weight_min": -48.74415969848633,
      "activations/layer4_attention_weight_max": 128.78245544433594,
      "activations/layer4_attention_weight_min": -119.8321304321289,
      "activations/layer5_attention_weight_max": 143.0886993408203,
      "activations/layer5_attention_weight_min": -132.76829528808594,
      "activations/layer6_attention_weight_max": 99.44384002685547,
      "activations/layer6_attention_weight_min": -94.62162017822266,
      "activations/layer7_attention_weight_max": 84.47405242919922,
      "activations/layer7_attention_weight_min": -80.38590240478516,
      "activations/layer8_attention_weight_max": 66.28571319580078,
      "activations/layer8_attention_weight_min": -62.86333465576172,
      "activations/layer9_attention_weight_max": 62.18380355834961,
      "activations/layer9_attention_weight_min": -57.716796875,
      "epoch": 0.32,
      "learning_rate": 0.0001494583333333333,
      "loss": 3.4466,
      "step": 5450
    },
    {
      "activations/layer0_attention_weight_max": 14.68539810180664,
      "activations/layer0_attention_weight_min": -10.323029518127441,
      "activations/layer10_attention_weight_max": 42.198116302490234,
      "activations/layer10_attention_weight_min": -37.50971984863281,
      "activations/layer11_attention_weight_max": 46.320594787597656,
      "activations/layer11_attention_weight_min": -34.44898986816406,
      "activations/layer12_attention_weight_max": 31.954708099365234,
      "activations/layer12_attention_weight_min": -22.55417823791504,
      "activations/layer13_attention_weight_max": 23.515583038330078,
      "activations/layer13_attention_weight_min": -20.201698303222656,
      "activations/layer14_attention_weight_max": 29.160818099975586,
      "activations/layer14_attention_weight_min": -19.98569107055664,
      "activations/layer15_attention_weight_max": 22.196908950805664,
      "activations/layer15_attention_weight_min": -15.470073699951172,
      "activations/layer16_attention_weight_max": 16.985225677490234,
      "activations/layer16_attention_weight_min": -14.79897689819336,
      "activations/layer17_attention_weight_max": 16.27556037902832,
      "activations/layer17_attention_weight_min": -15.543736457824707,
      "activations/layer18_attention_weight_max": 17.234411239624023,
      "activations/layer18_attention_weight_min": -11.154908180236816,
      "activations/layer19_attention_weight_max": 17.818973541259766,
      "activations/layer19_attention_weight_min": -12.680879592895508,
      "activations/layer1_attention_weight_max": 15.739767074584961,
      "activations/layer1_attention_weight_min": -11.922839164733887,
      "activations/layer20_attention_weight_max": 19.09183692932129,
      "activations/layer20_attention_weight_min": -12.036920547485352,
      "activations/layer21_attention_weight_max": 17.3806095123291,
      "activations/layer21_attention_weight_min": -12.566276550292969,
      "activations/layer22_attention_weight_max": 18.402368545532227,
      "activations/layer22_attention_weight_min": -13.097189903259277,
      "activations/layer23_attention_weight_max": 13.38922119140625,
      "activations/layer23_attention_weight_min": -12.619699478149414,
      "activations/layer2_attention_weight_max": 27.64963150024414,
      "activations/layer2_attention_weight_min": -23.31869888305664,
      "activations/layer3_attention_weight_max": 57.98627853393555,
      "activations/layer3_attention_weight_min": -48.454345703125,
      "activations/layer4_attention_weight_max": 131.12506103515625,
      "activations/layer4_attention_weight_min": -115.1407699584961,
      "activations/layer5_attention_weight_max": 146.4215087890625,
      "activations/layer5_attention_weight_min": -130.53634643554688,
      "activations/layer6_attention_weight_max": 97.30286407470703,
      "activations/layer6_attention_weight_min": -92.95296478271484,
      "activations/layer7_attention_weight_max": 87.58262634277344,
      "activations/layer7_attention_weight_min": -81.78750610351562,
      "activations/layer8_attention_weight_max": 70.8801498413086,
      "activations/layer8_attention_weight_min": -58.45677947998047,
      "activations/layer9_attention_weight_max": 66.41751861572266,
      "activations/layer9_attention_weight_min": -67.53697204589844,
      "epoch": 0.32,
      "learning_rate": 0.00014943939393939393,
      "loss": 3.4377,
      "step": 5500
    },
    {
      "activations/layer0_attention_weight_max": 14.501983642578125,
      "activations/layer0_attention_weight_min": -9.354161262512207,
      "activations/layer10_attention_weight_max": 44.8113899230957,
      "activations/layer10_attention_weight_min": -35.02806091308594,
      "activations/layer11_attention_weight_max": 56.090087890625,
      "activations/layer11_attention_weight_min": -39.978092193603516,
      "activations/layer12_attention_weight_max": 25.358489990234375,
      "activations/layer12_attention_weight_min": -18.839040756225586,
      "activations/layer13_attention_weight_max": 22.449731826782227,
      "activations/layer13_attention_weight_min": -19.15462303161621,
      "activations/layer14_attention_weight_max": 27.763992309570312,
      "activations/layer14_attention_weight_min": -20.660297393798828,
      "activations/layer15_attention_weight_max": 21.449974060058594,
      "activations/layer15_attention_weight_min": -14.578754425048828,
      "activations/layer16_attention_weight_max": 22.453222274780273,
      "activations/layer16_attention_weight_min": -14.41944694519043,
      "activations/layer17_attention_weight_max": 23.424915313720703,
      "activations/layer17_attention_weight_min": -15.30594253540039,
      "activations/layer18_attention_weight_max": 18.456222534179688,
      "activations/layer18_attention_weight_min": -10.938835144042969,
      "activations/layer19_attention_weight_max": 21.211719512939453,
      "activations/layer19_attention_weight_min": -11.439034461975098,
      "activations/layer1_attention_weight_max": 14.463663101196289,
      "activations/layer1_attention_weight_min": -11.587287902832031,
      "activations/layer20_attention_weight_max": 22.760066986083984,
      "activations/layer20_attention_weight_min": -13.530447006225586,
      "activations/layer21_attention_weight_max": 18.653736114501953,
      "activations/layer21_attention_weight_min": -11.78966999053955,
      "activations/layer22_attention_weight_max": 18.50275993347168,
      "activations/layer22_attention_weight_min": -13.034260749816895,
      "activations/layer23_attention_weight_max": 15.504013061523438,
      "activations/layer23_attention_weight_min": -11.748759269714355,
      "activations/layer2_attention_weight_max": 28.31268882751465,
      "activations/layer2_attention_weight_min": -22.35584259033203,
      "activations/layer3_attention_weight_max": 58.762474060058594,
      "activations/layer3_attention_weight_min": -48.16044616699219,
      "activations/layer4_attention_weight_max": 131.81649780273438,
      "activations/layer4_attention_weight_min": -117.0861587524414,
      "activations/layer5_attention_weight_max": 148.67733764648438,
      "activations/layer5_attention_weight_min": -130.04051208496094,
      "activations/layer6_attention_weight_max": 97.89613342285156,
      "activations/layer6_attention_weight_min": -90.7479019165039,
      "activations/layer7_attention_weight_max": 88.76087951660156,
      "activations/layer7_attention_weight_min": -76.5304183959961,
      "activations/layer8_attention_weight_max": 73.53242492675781,
      "activations/layer8_attention_weight_min": -61.89315414428711,
      "activations/layer9_attention_weight_max": 66.68449401855469,
      "activations/layer9_attention_weight_min": -59.81875228881836,
      "epoch": 0.32,
      "learning_rate": 0.00014942045454545451,
      "loss": 3.4143,
      "step": 5550
    },
    {
      "activations/layer0_attention_weight_max": 14.249253273010254,
      "activations/layer0_attention_weight_min": -9.427878379821777,
      "activations/layer10_attention_weight_max": 44.914283752441406,
      "activations/layer10_attention_weight_min": -37.84135818481445,
      "activations/layer11_attention_weight_max": 51.35157012939453,
      "activations/layer11_attention_weight_min": -36.906158447265625,
      "activations/layer12_attention_weight_max": 25.72470474243164,
      "activations/layer12_attention_weight_min": -23.043119430541992,
      "activations/layer13_attention_weight_max": 25.261173248291016,
      "activations/layer13_attention_weight_min": -24.611873626708984,
      "activations/layer14_attention_weight_max": 30.005748748779297,
      "activations/layer14_attention_weight_min": -19.1861629486084,
      "activations/layer15_attention_weight_max": 29.354938507080078,
      "activations/layer15_attention_weight_min": -17.01898765563965,
      "activations/layer16_attention_weight_max": 21.96788215637207,
      "activations/layer16_attention_weight_min": -14.77894401550293,
      "activations/layer17_attention_weight_max": 19.977035522460938,
      "activations/layer17_attention_weight_min": -14.135618209838867,
      "activations/layer18_attention_weight_max": 21.775959014892578,
      "activations/layer18_attention_weight_min": -11.611983299255371,
      "activations/layer19_attention_weight_max": 22.632722854614258,
      "activations/layer19_attention_weight_min": -11.983267784118652,
      "activations/layer1_attention_weight_max": 14.981494903564453,
      "activations/layer1_attention_weight_min": -10.782326698303223,
      "activations/layer20_attention_weight_max": 20.22452163696289,
      "activations/layer20_attention_weight_min": -11.559365272521973,
      "activations/layer21_attention_weight_max": 20.903804779052734,
      "activations/layer21_attention_weight_min": -12.452281951904297,
      "activations/layer22_attention_weight_max": 20.537431716918945,
      "activations/layer22_attention_weight_min": -13.93385124206543,
      "activations/layer23_attention_weight_max": 15.328056335449219,
      "activations/layer23_attention_weight_min": -11.937826156616211,
      "activations/layer2_attention_weight_max": 28.65020751953125,
      "activations/layer2_attention_weight_min": -21.552642822265625,
      "activations/layer3_attention_weight_max": 57.341243743896484,
      "activations/layer3_attention_weight_min": -47.202186584472656,
      "activations/layer4_attention_weight_max": 131.92616271972656,
      "activations/layer4_attention_weight_min": -115.01470947265625,
      "activations/layer5_attention_weight_max": 146.20420837402344,
      "activations/layer5_attention_weight_min": -127.41868591308594,
      "activations/layer6_attention_weight_max": 99.10464477539062,
      "activations/layer6_attention_weight_min": -90.5055923461914,
      "activations/layer7_attention_weight_max": 89.98723602294922,
      "activations/layer7_attention_weight_min": -80.02715301513672,
      "activations/layer8_attention_weight_max": 70.89603424072266,
      "activations/layer8_attention_weight_min": -62.16946792602539,
      "activations/layer9_attention_weight_max": 70.57027435302734,
      "activations/layer9_attention_weight_min": -68.95516967773438,
      "epoch": 0.33,
      "learning_rate": 0.00014940151515151513,
      "loss": 3.4179,
      "step": 5600
    },
    {
      "activations/layer0_attention_weight_max": 15.185458183288574,
      "activations/layer0_attention_weight_min": -10.225658416748047,
      "activations/layer10_attention_weight_max": 45.972408294677734,
      "activations/layer10_attention_weight_min": -36.342323303222656,
      "activations/layer11_attention_weight_max": 46.17281723022461,
      "activations/layer11_attention_weight_min": -34.18861770629883,
      "activations/layer12_attention_weight_max": 31.32404327392578,
      "activations/layer12_attention_weight_min": -21.674631118774414,
      "activations/layer13_attention_weight_max": 27.6357421875,
      "activations/layer13_attention_weight_min": -21.50986671447754,
      "activations/layer14_attention_weight_max": 34.056968688964844,
      "activations/layer14_attention_weight_min": -20.84064292907715,
      "activations/layer15_attention_weight_max": 26.578838348388672,
      "activations/layer15_attention_weight_min": -17.29155158996582,
      "activations/layer16_attention_weight_max": 25.507970809936523,
      "activations/layer16_attention_weight_min": -17.795207977294922,
      "activations/layer17_attention_weight_max": 29.07809829711914,
      "activations/layer17_attention_weight_min": -12.644682884216309,
      "activations/layer18_attention_weight_max": 21.032018661499023,
      "activations/layer18_attention_weight_min": -12.853195190429688,
      "activations/layer19_attention_weight_max": 25.557880401611328,
      "activations/layer19_attention_weight_min": -13.049275398254395,
      "activations/layer1_attention_weight_max": 16.095792770385742,
      "activations/layer1_attention_weight_min": -11.945265769958496,
      "activations/layer20_attention_weight_max": 23.522504806518555,
      "activations/layer20_attention_weight_min": -10.723793983459473,
      "activations/layer21_attention_weight_max": 23.048688888549805,
      "activations/layer21_attention_weight_min": -10.370048522949219,
      "activations/layer22_attention_weight_max": 21.76288604736328,
      "activations/layer22_attention_weight_min": -12.57883358001709,
      "activations/layer23_attention_weight_max": 16.34914779663086,
      "activations/layer23_attention_weight_min": -12.565866470336914,
      "activations/layer2_attention_weight_max": 29.350271224975586,
      "activations/layer2_attention_weight_min": -21.687559127807617,
      "activations/layer3_attention_weight_max": 61.53603744506836,
      "activations/layer3_attention_weight_min": -47.55335235595703,
      "activations/layer4_attention_weight_max": 131.93850708007812,
      "activations/layer4_attention_weight_min": -114.29174041748047,
      "activations/layer5_attention_weight_max": 149.67831420898438,
      "activations/layer5_attention_weight_min": -129.5496063232422,
      "activations/layer6_attention_weight_max": 100.05936431884766,
      "activations/layer6_attention_weight_min": -91.88289642333984,
      "activations/layer7_attention_weight_max": 91.79329681396484,
      "activations/layer7_attention_weight_min": -79.13685607910156,
      "activations/layer8_attention_weight_max": 72.04200744628906,
      "activations/layer8_attention_weight_min": -60.7761116027832,
      "activations/layer9_attention_weight_max": 76.87516021728516,
      "activations/layer9_attention_weight_min": -67.47234344482422,
      "epoch": 0.33,
      "learning_rate": 0.00014938257575757574,
      "loss": 3.4204,
      "step": 5650
    },
    {
      "activations/layer0_attention_weight_max": 13.667047500610352,
      "activations/layer0_attention_weight_min": -10.038274765014648,
      "activations/layer10_attention_weight_max": 45.5,
      "activations/layer10_attention_weight_min": -34.6561164855957,
      "activations/layer11_attention_weight_max": 43.66554260253906,
      "activations/layer11_attention_weight_min": -32.5125732421875,
      "activations/layer12_attention_weight_max": 34.47612762451172,
      "activations/layer12_attention_weight_min": -23.484872817993164,
      "activations/layer13_attention_weight_max": 27.412004470825195,
      "activations/layer13_attention_weight_min": -27.099445343017578,
      "activations/layer14_attention_weight_max": 29.678775787353516,
      "activations/layer14_attention_weight_min": -20.712064743041992,
      "activations/layer15_attention_weight_max": 29.179203033447266,
      "activations/layer15_attention_weight_min": -18.296939849853516,
      "activations/layer16_attention_weight_max": 23.998868942260742,
      "activations/layer16_attention_weight_min": -16.349681854248047,
      "activations/layer17_attention_weight_max": 28.366329193115234,
      "activations/layer17_attention_weight_min": -15.666851997375488,
      "activations/layer18_attention_weight_max": 22.889636993408203,
      "activations/layer18_attention_weight_min": -13.519989013671875,
      "activations/layer19_attention_weight_max": 22.46040916442871,
      "activations/layer19_attention_weight_min": -14.511488914489746,
      "activations/layer1_attention_weight_max": 14.034070014953613,
      "activations/layer1_attention_weight_min": -10.651848793029785,
      "activations/layer20_attention_weight_max": 23.736766815185547,
      "activations/layer20_attention_weight_min": -12.673312187194824,
      "activations/layer21_attention_weight_max": 22.945322036743164,
      "activations/layer21_attention_weight_min": -12.163800239562988,
      "activations/layer22_attention_weight_max": 24.95541763305664,
      "activations/layer22_attention_weight_min": -12.88720703125,
      "activations/layer23_attention_weight_max": 19.156784057617188,
      "activations/layer23_attention_weight_min": -13.434999465942383,
      "activations/layer2_attention_weight_max": 29.638439178466797,
      "activations/layer2_attention_weight_min": -22.057743072509766,
      "activations/layer3_attention_weight_max": 59.09480667114258,
      "activations/layer3_attention_weight_min": -48.73902893066406,
      "activations/layer4_attention_weight_max": 135.097900390625,
      "activations/layer4_attention_weight_min": -114.32960510253906,
      "activations/layer5_attention_weight_max": 150.3304901123047,
      "activations/layer5_attention_weight_min": -128.84121704101562,
      "activations/layer6_attention_weight_max": 95.78471374511719,
      "activations/layer6_attention_weight_min": -88.09254455566406,
      "activations/layer7_attention_weight_max": 87.63360595703125,
      "activations/layer7_attention_weight_min": -76.46430206298828,
      "activations/layer8_attention_weight_max": 80.23164367675781,
      "activations/layer8_attention_weight_min": -65.60840606689453,
      "activations/layer9_attention_weight_max": 69.67217254638672,
      "activations/layer9_attention_weight_min": -64.57952117919922,
      "epoch": 0.33,
      "learning_rate": 0.00014936363636363636,
      "loss": 3.4214,
      "step": 5700
    },
    {
      "activations/layer0_attention_weight_max": 14.025129318237305,
      "activations/layer0_attention_weight_min": -9.921222686767578,
      "activations/layer10_attention_weight_max": 43.957672119140625,
      "activations/layer10_attention_weight_min": -33.644073486328125,
      "activations/layer11_attention_weight_max": 50.31513595581055,
      "activations/layer11_attention_weight_min": -35.492713928222656,
      "activations/layer12_attention_weight_max": 26.611230850219727,
      "activations/layer12_attention_weight_min": -21.19137191772461,
      "activations/layer13_attention_weight_max": 27.653148651123047,
      "activations/layer13_attention_weight_min": -22.07408332824707,
      "activations/layer14_attention_weight_max": 31.46949577331543,
      "activations/layer14_attention_weight_min": -19.603456497192383,
      "activations/layer15_attention_weight_max": 33.42464065551758,
      "activations/layer15_attention_weight_min": -15.572599411010742,
      "activations/layer16_attention_weight_max": 27.26498794555664,
      "activations/layer16_attention_weight_min": -18.03409194946289,
      "activations/layer17_attention_weight_max": 22.53748893737793,
      "activations/layer17_attention_weight_min": -18.650432586669922,
      "activations/layer18_attention_weight_max": 20.971593856811523,
      "activations/layer18_attention_weight_min": -13.53491497039795,
      "activations/layer19_attention_weight_max": 21.219266891479492,
      "activations/layer19_attention_weight_min": -17.996374130249023,
      "activations/layer1_attention_weight_max": 15.903585433959961,
      "activations/layer1_attention_weight_min": -11.154608726501465,
      "activations/layer20_attention_weight_max": 25.934431076049805,
      "activations/layer20_attention_weight_min": -16.074054718017578,
      "activations/layer21_attention_weight_max": 20.647565841674805,
      "activations/layer21_attention_weight_min": -14.69527530670166,
      "activations/layer22_attention_weight_max": 20.798980712890625,
      "activations/layer22_attention_weight_min": -14.447997093200684,
      "activations/layer23_attention_weight_max": 15.352968215942383,
      "activations/layer23_attention_weight_min": -13.536337852478027,
      "activations/layer2_attention_weight_max": 28.841123580932617,
      "activations/layer2_attention_weight_min": -21.897029876708984,
      "activations/layer3_attention_weight_max": 59.489933013916016,
      "activations/layer3_attention_weight_min": -49.59663391113281,
      "activations/layer4_attention_weight_max": 128.59732055664062,
      "activations/layer4_attention_weight_min": -115.36673736572266,
      "activations/layer5_attention_weight_max": 149.0721893310547,
      "activations/layer5_attention_weight_min": -128.7466278076172,
      "activations/layer6_attention_weight_max": 97.18473052978516,
      "activations/layer6_attention_weight_min": -86.32340240478516,
      "activations/layer7_attention_weight_max": 90.36186218261719,
      "activations/layer7_attention_weight_min": -75.23900604248047,
      "activations/layer8_attention_weight_max": 69.91104125976562,
      "activations/layer8_attention_weight_min": -61.15968704223633,
      "activations/layer9_attention_weight_max": 66.65361022949219,
      "activations/layer9_attention_weight_min": -57.11301803588867,
      "epoch": 0.33,
      "learning_rate": 0.00014934469696969695,
      "loss": 3.3879,
      "step": 5750
    },
    {
      "activations/layer0_attention_weight_max": 14.523506164550781,
      "activations/layer0_attention_weight_min": -9.525409698486328,
      "activations/layer10_attention_weight_max": 45.044063568115234,
      "activations/layer10_attention_weight_min": -35.424415588378906,
      "activations/layer11_attention_weight_max": 41.707862854003906,
      "activations/layer11_attention_weight_min": -30.991024017333984,
      "activations/layer12_attention_weight_max": 27.567935943603516,
      "activations/layer12_attention_weight_min": -22.233680725097656,
      "activations/layer13_attention_weight_max": 31.734439849853516,
      "activations/layer13_attention_weight_min": -20.813047409057617,
      "activations/layer14_attention_weight_max": 31.36785125732422,
      "activations/layer14_attention_weight_min": -20.1621150970459,
      "activations/layer15_attention_weight_max": 29.174480438232422,
      "activations/layer15_attention_weight_min": -17.92483139038086,
      "activations/layer16_attention_weight_max": 19.72234344482422,
      "activations/layer16_attention_weight_min": -17.15764045715332,
      "activations/layer17_attention_weight_max": 19.53109359741211,
      "activations/layer17_attention_weight_min": -13.941335678100586,
      "activations/layer18_attention_weight_max": 20.426376342773438,
      "activations/layer18_attention_weight_min": -12.695347785949707,
      "activations/layer19_attention_weight_max": 20.708463668823242,
      "activations/layer19_attention_weight_min": -14.733850479125977,
      "activations/layer1_attention_weight_max": 13.8444242477417,
      "activations/layer1_attention_weight_min": -11.687313079833984,
      "activations/layer20_attention_weight_max": 18.90513801574707,
      "activations/layer20_attention_weight_min": -13.824734687805176,
      "activations/layer21_attention_weight_max": 19.101795196533203,
      "activations/layer21_attention_weight_min": -14.335738182067871,
      "activations/layer22_attention_weight_max": 21.010530471801758,
      "activations/layer22_attention_weight_min": -14.29189395904541,
      "activations/layer23_attention_weight_max": 16.204023361206055,
      "activations/layer23_attention_weight_min": -11.821271896362305,
      "activations/layer2_attention_weight_max": 29.297988891601562,
      "activations/layer2_attention_weight_min": -20.160171508789062,
      "activations/layer3_attention_weight_max": 59.4239501953125,
      "activations/layer3_attention_weight_min": -46.23085403442383,
      "activations/layer4_attention_weight_max": 133.67935180664062,
      "activations/layer4_attention_weight_min": -113.05696868896484,
      "activations/layer5_attention_weight_max": 155.62600708007812,
      "activations/layer5_attention_weight_min": -125.56683349609375,
      "activations/layer6_attention_weight_max": 102.53325653076172,
      "activations/layer6_attention_weight_min": -84.45867919921875,
      "activations/layer7_attention_weight_max": 81.60345458984375,
      "activations/layer7_attention_weight_min": -74.53888702392578,
      "activations/layer8_attention_weight_max": 76.59866333007812,
      "activations/layer8_attention_weight_min": -58.85055923461914,
      "activations/layer9_attention_weight_max": 78.92789459228516,
      "activations/layer9_attention_weight_min": -67.57001495361328,
      "epoch": 0.34,
      "learning_rate": 0.00014932575757575756,
      "loss": 3.4061,
      "step": 5800
    },
    {
      "activations/layer0_attention_weight_max": 13.83062744140625,
      "activations/layer0_attention_weight_min": -10.216652870178223,
      "activations/layer10_attention_weight_max": 43.649322509765625,
      "activations/layer10_attention_weight_min": -36.527565002441406,
      "activations/layer11_attention_weight_max": 49.50474166870117,
      "activations/layer11_attention_weight_min": -33.86089324951172,
      "activations/layer12_attention_weight_max": 32.18958282470703,
      "activations/layer12_attention_weight_min": -18.846744537353516,
      "activations/layer13_attention_weight_max": 29.751134872436523,
      "activations/layer13_attention_weight_min": -21.44095802307129,
      "activations/layer14_attention_weight_max": 32.519683837890625,
      "activations/layer14_attention_weight_min": -24.752185821533203,
      "activations/layer15_attention_weight_max": 24.273252487182617,
      "activations/layer15_attention_weight_min": -17.959808349609375,
      "activations/layer16_attention_weight_max": 23.20099639892578,
      "activations/layer16_attention_weight_min": -16.457490921020508,
      "activations/layer17_attention_weight_max": 22.825855255126953,
      "activations/layer17_attention_weight_min": -14.812491416931152,
      "activations/layer18_attention_weight_max": 19.19818687438965,
      "activations/layer18_attention_weight_min": -11.650830268859863,
      "activations/layer19_attention_weight_max": 22.77107048034668,
      "activations/layer19_attention_weight_min": -13.551918029785156,
      "activations/layer1_attention_weight_max": 16.321571350097656,
      "activations/layer1_attention_weight_min": -10.665060043334961,
      "activations/layer20_attention_weight_max": 25.658523559570312,
      "activations/layer20_attention_weight_min": -12.832608222961426,
      "activations/layer21_attention_weight_max": 23.10493278503418,
      "activations/layer21_attention_weight_min": -12.085850715637207,
      "activations/layer22_attention_weight_max": 21.800792694091797,
      "activations/layer22_attention_weight_min": -15.040769577026367,
      "activations/layer23_attention_weight_max": 19.037437438964844,
      "activations/layer23_attention_weight_min": -12.003049850463867,
      "activations/layer2_attention_weight_max": 26.683568954467773,
      "activations/layer2_attention_weight_min": -21.488101959228516,
      "activations/layer3_attention_weight_max": 56.785888671875,
      "activations/layer3_attention_weight_min": -46.88849639892578,
      "activations/layer4_attention_weight_max": 128.9324951171875,
      "activations/layer4_attention_weight_min": -116.3045425415039,
      "activations/layer5_attention_weight_max": 140.7375946044922,
      "activations/layer5_attention_weight_min": -129.34104919433594,
      "activations/layer6_attention_weight_max": 91.0269546508789,
      "activations/layer6_attention_weight_min": -86.9247055053711,
      "activations/layer7_attention_weight_max": 82.14723205566406,
      "activations/layer7_attention_weight_min": -74.87657928466797,
      "activations/layer8_attention_weight_max": 74.46334075927734,
      "activations/layer8_attention_weight_min": -64.7831802368164,
      "activations/layer9_attention_weight_max": 67.81768798828125,
      "activations/layer9_attention_weight_min": -64.4013900756836,
      "epoch": 0.34,
      "learning_rate": 0.00014930681818181818,
      "loss": 3.4025,
      "step": 5850
    },
    {
      "activations/layer0_attention_weight_max": 13.893342971801758,
      "activations/layer0_attention_weight_min": -9.289653778076172,
      "activations/layer10_attention_weight_max": 43.18595504760742,
      "activations/layer10_attention_weight_min": -34.33714294433594,
      "activations/layer11_attention_weight_max": 41.040733337402344,
      "activations/layer11_attention_weight_min": -32.44565200805664,
      "activations/layer12_attention_weight_max": 22.41543960571289,
      "activations/layer12_attention_weight_min": -24.32832145690918,
      "activations/layer13_attention_weight_max": 23.261924743652344,
      "activations/layer13_attention_weight_min": -23.510778427124023,
      "activations/layer14_attention_weight_max": 27.919607162475586,
      "activations/layer14_attention_weight_min": -19.04157829284668,
      "activations/layer15_attention_weight_max": 22.43683433532715,
      "activations/layer15_attention_weight_min": -16.211204528808594,
      "activations/layer16_attention_weight_max": 21.96392059326172,
      "activations/layer16_attention_weight_min": -16.58074188232422,
      "activations/layer17_attention_weight_max": 24.53680992126465,
      "activations/layer17_attention_weight_min": -14.887106895446777,
      "activations/layer18_attention_weight_max": 21.79227066040039,
      "activations/layer18_attention_weight_min": -11.828866958618164,
      "activations/layer19_attention_weight_max": 24.496671676635742,
      "activations/layer19_attention_weight_min": -12.935806274414062,
      "activations/layer1_attention_weight_max": 15.0901517868042,
      "activations/layer1_attention_weight_min": -11.983538627624512,
      "activations/layer20_attention_weight_max": 25.31938934326172,
      "activations/layer20_attention_weight_min": -12.33814525604248,
      "activations/layer21_attention_weight_max": 20.392610549926758,
      "activations/layer21_attention_weight_min": -10.527758598327637,
      "activations/layer22_attention_weight_max": 20.575014114379883,
      "activations/layer22_attention_weight_min": -13.60775375366211,
      "activations/layer23_attention_weight_max": 16.889863967895508,
      "activations/layer23_attention_weight_min": -11.205981254577637,
      "activations/layer2_attention_weight_max": 27.900554656982422,
      "activations/layer2_attention_weight_min": -21.839679718017578,
      "activations/layer3_attention_weight_max": 56.09520721435547,
      "activations/layer3_attention_weight_min": -48.582767486572266,
      "activations/layer4_attention_weight_max": 125.8372802734375,
      "activations/layer4_attention_weight_min": -116.91887664794922,
      "activations/layer5_attention_weight_max": 148.75201416015625,
      "activations/layer5_attention_weight_min": -133.86849975585938,
      "activations/layer6_attention_weight_max": 96.61284637451172,
      "activations/layer6_attention_weight_min": -88.30876922607422,
      "activations/layer7_attention_weight_max": 87.46356201171875,
      "activations/layer7_attention_weight_min": -76.35881042480469,
      "activations/layer8_attention_weight_max": 70.7083969116211,
      "activations/layer8_attention_weight_min": -62.6455078125,
      "activations/layer9_attention_weight_max": 59.21550369262695,
      "activations/layer9_attention_weight_min": -60.76020050048828,
      "epoch": 0.34,
      "learning_rate": 0.00014928787878787877,
      "loss": 3.3951,
      "step": 5900
    },
    {
      "activations/layer0_attention_weight_max": 15.047168731689453,
      "activations/layer0_attention_weight_min": -10.507328987121582,
      "activations/layer10_attention_weight_max": 46.976158142089844,
      "activations/layer10_attention_weight_min": -37.767547607421875,
      "activations/layer11_attention_weight_max": 52.06597900390625,
      "activations/layer11_attention_weight_min": -34.15439987182617,
      "activations/layer12_attention_weight_max": 22.130081176757812,
      "activations/layer12_attention_weight_min": -24.21220588684082,
      "activations/layer13_attention_weight_max": 26.794233322143555,
      "activations/layer13_attention_weight_min": -22.780033111572266,
      "activations/layer14_attention_weight_max": 32.7734489440918,
      "activations/layer14_attention_weight_min": -22.323495864868164,
      "activations/layer15_attention_weight_max": 23.549741744995117,
      "activations/layer15_attention_weight_min": -17.294448852539062,
      "activations/layer16_attention_weight_max": 24.734378814697266,
      "activations/layer16_attention_weight_min": -17.050783157348633,
      "activations/layer17_attention_weight_max": 22.505313873291016,
      "activations/layer17_attention_weight_min": -14.939422607421875,
      "activations/layer18_attention_weight_max": 21.66771125793457,
      "activations/layer18_attention_weight_min": -12.209159851074219,
      "activations/layer19_attention_weight_max": 24.788806915283203,
      "activations/layer19_attention_weight_min": -18.466428756713867,
      "activations/layer1_attention_weight_max": 14.604656219482422,
      "activations/layer1_attention_weight_min": -10.671195983886719,
      "activations/layer20_attention_weight_max": 24.129549026489258,
      "activations/layer20_attention_weight_min": -16.036090850830078,
      "activations/layer21_attention_weight_max": 21.672767639160156,
      "activations/layer21_attention_weight_min": -13.759740829467773,
      "activations/layer22_attention_weight_max": 22.579492568969727,
      "activations/layer22_attention_weight_min": -14.806120872497559,
      "activations/layer23_attention_weight_max": 18.848230361938477,
      "activations/layer23_attention_weight_min": -11.918951034545898,
      "activations/layer2_attention_weight_max": 28.965240478515625,
      "activations/layer2_attention_weight_min": -20.62813949584961,
      "activations/layer3_attention_weight_max": 60.04108810424805,
      "activations/layer3_attention_weight_min": -49.226680755615234,
      "activations/layer4_attention_weight_max": 131.5878143310547,
      "activations/layer4_attention_weight_min": -115.35051727294922,
      "activations/layer5_attention_weight_max": 152.8459014892578,
      "activations/layer5_attention_weight_min": -125.01089477539062,
      "activations/layer6_attention_weight_max": 97.23805236816406,
      "activations/layer6_attention_weight_min": -88.09890747070312,
      "activations/layer7_attention_weight_max": 88.53839874267578,
      "activations/layer7_attention_weight_min": -77.271728515625,
      "activations/layer8_attention_weight_max": 75.0748291015625,
      "activations/layer8_attention_weight_min": -62.92997360229492,
      "activations/layer9_attention_weight_max": 72.85152435302734,
      "activations/layer9_attention_weight_min": -63.33871078491211,
      "epoch": 0.35,
      "learning_rate": 0.00014926893939393938,
      "loss": 3.4039,
      "step": 5950
    },
    {
      "activations/layer0_attention_weight_max": 13.216788291931152,
      "activations/layer0_attention_weight_min": -10.53559684753418,
      "activations/layer10_attention_weight_max": 43.420204162597656,
      "activations/layer10_attention_weight_min": -35.1196174621582,
      "activations/layer11_attention_weight_max": 43.71446228027344,
      "activations/layer11_attention_weight_min": -36.631935119628906,
      "activations/layer12_attention_weight_max": 24.73567008972168,
      "activations/layer12_attention_weight_min": -23.324729919433594,
      "activations/layer13_attention_weight_max": 27.63498878479004,
      "activations/layer13_attention_weight_min": -22.954265594482422,
      "activations/layer14_attention_weight_max": 28.52397346496582,
      "activations/layer14_attention_weight_min": -22.263275146484375,
      "activations/layer15_attention_weight_max": 24.681241989135742,
      "activations/layer15_attention_weight_min": -16.96976661682129,
      "activations/layer16_attention_weight_max": 23.53997230529785,
      "activations/layer16_attention_weight_min": -17.52564239501953,
      "activations/layer17_attention_weight_max": 24.742778778076172,
      "activations/layer17_attention_weight_min": -14.810864448547363,
      "activations/layer18_attention_weight_max": 19.169116973876953,
      "activations/layer18_attention_weight_min": -14.460402488708496,
      "activations/layer19_attention_weight_max": 19.326387405395508,
      "activations/layer19_attention_weight_min": -15.2384614944458,
      "activations/layer1_attention_weight_max": 13.787652969360352,
      "activations/layer1_attention_weight_min": -10.565507888793945,
      "activations/layer20_attention_weight_max": 17.34693717956543,
      "activations/layer20_attention_weight_min": -15.421688079833984,
      "activations/layer21_attention_weight_max": 16.403053283691406,
      "activations/layer21_attention_weight_min": -13.67821216583252,
      "activations/layer22_attention_weight_max": 18.825965881347656,
      "activations/layer22_attention_weight_min": -15.032368659973145,
      "activations/layer23_attention_weight_max": 15.031157493591309,
      "activations/layer23_attention_weight_min": -11.391119003295898,
      "activations/layer2_attention_weight_max": 27.481998443603516,
      "activations/layer2_attention_weight_min": -21.737333297729492,
      "activations/layer3_attention_weight_max": 54.95100021362305,
      "activations/layer3_attention_weight_min": -46.873390197753906,
      "activations/layer4_attention_weight_max": 126.60613250732422,
      "activations/layer4_attention_weight_min": -115.34575653076172,
      "activations/layer5_attention_weight_max": 139.2002716064453,
      "activations/layer5_attention_weight_min": -126.29148864746094,
      "activations/layer6_attention_weight_max": 92.69895935058594,
      "activations/layer6_attention_weight_min": -84.9798812866211,
      "activations/layer7_attention_weight_max": 79.34263610839844,
      "activations/layer7_attention_weight_min": -75.69214630126953,
      "activations/layer8_attention_weight_max": 67.19680786132812,
      "activations/layer8_attention_weight_min": -62.48710250854492,
      "activations/layer9_attention_weight_max": 61.82878494262695,
      "activations/layer9_attention_weight_min": -58.10017776489258,
      "epoch": 0.35,
      "learning_rate": 0.00014925,
      "loss": 3.3992,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_loss": 3.34375,
      "eval_runtime": 8.2765,
      "eval_samples_per_second": 518.817,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_openwebtext_loss": 3.34375,
      "eval_openwebtext_ppl": 28.325147094540903,
      "eval_openwebtext_runtime": 8.2765,
      "eval_openwebtext_samples_per_second": 518.817,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_wikitext_loss": 3.677734375,
      "eval_wikitext_ppl": 39.556671887793456,
      "eval_wikitext_runtime": 1.6925,
      "eval_wikitext_samples_per_second": 269.431,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_lambada_loss": 4.14453125,
      "eval_lambada_ppl": 63.08804244970428,
      "eval_lambada_runtime": 9.2955,
      "eval_lambada_samples_per_second": 523.8,
      "step": 6000
    },
    {
      "activations/layer0_attention_weight_max": 13.088345527648926,
      "activations/layer0_attention_weight_min": -10.36465835571289,
      "activations/layer10_attention_weight_max": 38.354896545410156,
      "activations/layer10_attention_weight_min": -34.921852111816406,
      "activations/layer11_attention_weight_max": 60.774009704589844,
      "activations/layer11_attention_weight_min": -37.4326171875,
      "activations/layer12_attention_weight_max": 24.10772132873535,
      "activations/layer12_attention_weight_min": -23.10780143737793,
      "activations/layer13_attention_weight_max": 27.58942222595215,
      "activations/layer13_attention_weight_min": -23.557931900024414,
      "activations/layer14_attention_weight_max": 29.017236709594727,
      "activations/layer14_attention_weight_min": -21.33130645751953,
      "activations/layer15_attention_weight_max": 34.21318054199219,
      "activations/layer15_attention_weight_min": -20.295021057128906,
      "activations/layer16_attention_weight_max": 23.560791015625,
      "activations/layer16_attention_weight_min": -15.44725513458252,
      "activations/layer17_attention_weight_max": 22.398332595825195,
      "activations/layer17_attention_weight_min": -17.09511375427246,
      "activations/layer18_attention_weight_max": 23.555130004882812,
      "activations/layer18_attention_weight_min": -12.745160102844238,
      "activations/layer19_attention_weight_max": 22.18311882019043,
      "activations/layer19_attention_weight_min": -12.816665649414062,
      "activations/layer1_attention_weight_max": 14.849502563476562,
      "activations/layer1_attention_weight_min": -10.311575889587402,
      "activations/layer20_attention_weight_max": 19.391918182373047,
      "activations/layer20_attention_weight_min": -11.897079467773438,
      "activations/layer21_attention_weight_max": 18.7863712310791,
      "activations/layer21_attention_weight_min": -11.069401741027832,
      "activations/layer22_attention_weight_max": 20.383024215698242,
      "activations/layer22_attention_weight_min": -13.153532981872559,
      "activations/layer23_attention_weight_max": 16.17888641357422,
      "activations/layer23_attention_weight_min": -12.675989151000977,
      "activations/layer2_attention_weight_max": 28.849348068237305,
      "activations/layer2_attention_weight_min": -21.280208587646484,
      "activations/layer3_attention_weight_max": 56.77141189575195,
      "activations/layer3_attention_weight_min": -46.8853874206543,
      "activations/layer4_attention_weight_max": 123.29088592529297,
      "activations/layer4_attention_weight_min": -114.11446380615234,
      "activations/layer5_attention_weight_max": 141.16000366210938,
      "activations/layer5_attention_weight_min": -125.51199340820312,
      "activations/layer6_attention_weight_max": 87.94364929199219,
      "activations/layer6_attention_weight_min": -85.12584686279297,
      "activations/layer7_attention_weight_max": 77.34524536132812,
      "activations/layer7_attention_weight_min": -73.8654556274414,
      "activations/layer8_attention_weight_max": 66.8968734741211,
      "activations/layer8_attention_weight_min": -59.65886306762695,
      "activations/layer9_attention_weight_max": 65.5624008178711,
      "activations/layer9_attention_weight_min": -60.032379150390625,
      "epoch": 0.35,
      "learning_rate": 0.0001492314393939394,
      "loss": 3.3718,
      "step": 6050
    },
    {
      "activations/layer0_attention_weight_max": 14.478595733642578,
      "activations/layer0_attention_weight_min": -10.559683799743652,
      "activations/layer10_attention_weight_max": 45.67897415161133,
      "activations/layer10_attention_weight_min": -35.200321197509766,
      "activations/layer11_attention_weight_max": 53.96377182006836,
      "activations/layer11_attention_weight_min": -41.69953155517578,
      "activations/layer12_attention_weight_max": 24.171321868896484,
      "activations/layer12_attention_weight_min": -21.452672958374023,
      "activations/layer13_attention_weight_max": 27.996078491210938,
      "activations/layer13_attention_weight_min": -23.17991065979004,
      "activations/layer14_attention_weight_max": 33.58942413330078,
      "activations/layer14_attention_weight_min": -22.805776596069336,
      "activations/layer15_attention_weight_max": 27.78582763671875,
      "activations/layer15_attention_weight_min": -20.850032806396484,
      "activations/layer16_attention_weight_max": 22.372194290161133,
      "activations/layer16_attention_weight_min": -17.426530838012695,
      "activations/layer17_attention_weight_max": 20.38212013244629,
      "activations/layer17_attention_weight_min": -13.966510772705078,
      "activations/layer18_attention_weight_max": 20.13457679748535,
      "activations/layer18_attention_weight_min": -11.444326400756836,
      "activations/layer19_attention_weight_max": 21.302852630615234,
      "activations/layer19_attention_weight_min": -12.304850578308105,
      "activations/layer1_attention_weight_max": 14.416305541992188,
      "activations/layer1_attention_weight_min": -10.073116302490234,
      "activations/layer20_attention_weight_max": 23.208189010620117,
      "activations/layer20_attention_weight_min": -11.521422386169434,
      "activations/layer21_attention_weight_max": 20.516952514648438,
      "activations/layer21_attention_weight_min": -11.605766296386719,
      "activations/layer22_attention_weight_max": 19.267637252807617,
      "activations/layer22_attention_weight_min": -13.96077823638916,
      "activations/layer23_attention_weight_max": 15.778624534606934,
      "activations/layer23_attention_weight_min": -11.965087890625,
      "activations/layer2_attention_weight_max": 28.648880004882812,
      "activations/layer2_attention_weight_min": -21.90502166748047,
      "activations/layer3_attention_weight_max": 59.225399017333984,
      "activations/layer3_attention_weight_min": -48.78895950317383,
      "activations/layer4_attention_weight_max": 130.6425323486328,
      "activations/layer4_attention_weight_min": -117.40885162353516,
      "activations/layer5_attention_weight_max": 153.90579223632812,
      "activations/layer5_attention_weight_min": -128.1716766357422,
      "activations/layer6_attention_weight_max": 99.8521957397461,
      "activations/layer6_attention_weight_min": -85.49711608886719,
      "activations/layer7_attention_weight_max": 97.39315795898438,
      "activations/layer7_attention_weight_min": -75.97626495361328,
      "activations/layer8_attention_weight_max": 77.2149658203125,
      "activations/layer8_attention_weight_min": -63.66402053833008,
      "activations/layer9_attention_weight_max": 78.2176742553711,
      "activations/layer9_attention_weight_min": -61.6878776550293,
      "epoch": 0.35,
      "learning_rate": 0.0001492125,
      "loss": 3.3731,
      "step": 6100
    },
    {
      "activations/layer0_attention_weight_max": 14.470038414001465,
      "activations/layer0_attention_weight_min": -10.18868637084961,
      "activations/layer10_attention_weight_max": 43.13249588012695,
      "activations/layer10_attention_weight_min": -36.3663330078125,
      "activations/layer11_attention_weight_max": 45.38709259033203,
      "activations/layer11_attention_weight_min": -33.22275161743164,
      "activations/layer12_attention_weight_max": 31.723636627197266,
      "activations/layer12_attention_weight_min": -20.81031036376953,
      "activations/layer13_attention_weight_max": 28.05660629272461,
      "activations/layer13_attention_weight_min": -22.03087043762207,
      "activations/layer14_attention_weight_max": 30.776981353759766,
      "activations/layer14_attention_weight_min": -22.86610984802246,
      "activations/layer15_attention_weight_max": 21.06372833251953,
      "activations/layer15_attention_weight_min": -16.885114669799805,
      "activations/layer16_attention_weight_max": 20.83486557006836,
      "activations/layer16_attention_weight_min": -17.567779541015625,
      "activations/layer17_attention_weight_max": 22.233905792236328,
      "activations/layer17_attention_weight_min": -15.60148811340332,
      "activations/layer18_attention_weight_max": 17.15706443786621,
      "activations/layer18_attention_weight_min": -11.600775718688965,
      "activations/layer19_attention_weight_max": 23.701908111572266,
      "activations/layer19_attention_weight_min": -13.864359855651855,
      "activations/layer1_attention_weight_max": 16.04863739013672,
      "activations/layer1_attention_weight_min": -10.999960899353027,
      "activations/layer20_attention_weight_max": 22.73405647277832,
      "activations/layer20_attention_weight_min": -13.959131240844727,
      "activations/layer21_attention_weight_max": 19.616201400756836,
      "activations/layer21_attention_weight_min": -11.379338264465332,
      "activations/layer22_attention_weight_max": 19.12641143798828,
      "activations/layer22_attention_weight_min": -12.678483963012695,
      "activations/layer23_attention_weight_max": 16.54341697692871,
      "activations/layer23_attention_weight_min": -13.105195999145508,
      "activations/layer2_attention_weight_max": 28.767765045166016,
      "activations/layer2_attention_weight_min": -22.646713256835938,
      "activations/layer3_attention_weight_max": 57.69427490234375,
      "activations/layer3_attention_weight_min": -50.74751281738281,
      "activations/layer4_attention_weight_max": 126.14266204833984,
      "activations/layer4_attention_weight_min": -114.09929656982422,
      "activations/layer5_attention_weight_max": 142.8604736328125,
      "activations/layer5_attention_weight_min": -131.4709930419922,
      "activations/layer6_attention_weight_max": 88.17508697509766,
      "activations/layer6_attention_weight_min": -81.81614685058594,
      "activations/layer7_attention_weight_max": 84.15892028808594,
      "activations/layer7_attention_weight_min": -72.98600006103516,
      "activations/layer8_attention_weight_max": 71.44393920898438,
      "activations/layer8_attention_weight_min": -60.84815979003906,
      "activations/layer9_attention_weight_max": 66.5516586303711,
      "activations/layer9_attention_weight_min": -63.552947998046875,
      "epoch": 0.36,
      "learning_rate": 0.0001491935606060606,
      "loss": 3.401,
      "step": 6150
    },
    {
      "activations/layer0_attention_weight_max": 14.104339599609375,
      "activations/layer0_attention_weight_min": -10.122851371765137,
      "activations/layer10_attention_weight_max": 47.3719596862793,
      "activations/layer10_attention_weight_min": -38.197486877441406,
      "activations/layer11_attention_weight_max": 51.446006774902344,
      "activations/layer11_attention_weight_min": -36.01664733886719,
      "activations/layer12_attention_weight_max": 25.234806060791016,
      "activations/layer12_attention_weight_min": -22.353153228759766,
      "activations/layer13_attention_weight_max": 30.102054595947266,
      "activations/layer13_attention_weight_min": -23.48903465270996,
      "activations/layer14_attention_weight_max": 37.79731369018555,
      "activations/layer14_attention_weight_min": -22.541255950927734,
      "activations/layer15_attention_weight_max": 27.59329605102539,
      "activations/layer15_attention_weight_min": -19.706531524658203,
      "activations/layer16_attention_weight_max": 22.44182014465332,
      "activations/layer16_attention_weight_min": -18.109113693237305,
      "activations/layer17_attention_weight_max": 21.659170150756836,
      "activations/layer17_attention_weight_min": -19.40691375732422,
      "activations/layer18_attention_weight_max": 20.392919540405273,
      "activations/layer18_attention_weight_min": -12.769036293029785,
      "activations/layer19_attention_weight_max": 22.684146881103516,
      "activations/layer19_attention_weight_min": -17.95100212097168,
      "activations/layer1_attention_weight_max": 14.660021781921387,
      "activations/layer1_attention_weight_min": -10.53392505645752,
      "activations/layer20_attention_weight_max": 23.593151092529297,
      "activations/layer20_attention_weight_min": -13.459099769592285,
      "activations/layer21_attention_weight_max": 21.187469482421875,
      "activations/layer21_attention_weight_min": -13.64368724822998,
      "activations/layer22_attention_weight_max": 21.619726181030273,
      "activations/layer22_attention_weight_min": -14.728939056396484,
      "activations/layer23_attention_weight_max": 16.420015335083008,
      "activations/layer23_attention_weight_min": -12.369918823242188,
      "activations/layer2_attention_weight_max": 29.421863555908203,
      "activations/layer2_attention_weight_min": -21.646984100341797,
      "activations/layer3_attention_weight_max": 57.070770263671875,
      "activations/layer3_attention_weight_min": -49.35625076293945,
      "activations/layer4_attention_weight_max": 128.5050506591797,
      "activations/layer4_attention_weight_min": -111.16778564453125,
      "activations/layer5_attention_weight_max": 148.28109741210938,
      "activations/layer5_attention_weight_min": -125.4241943359375,
      "activations/layer6_attention_weight_max": 97.67430114746094,
      "activations/layer6_attention_weight_min": -81.611083984375,
      "activations/layer7_attention_weight_max": 90.53601837158203,
      "activations/layer7_attention_weight_min": -76.06358337402344,
      "activations/layer8_attention_weight_max": 74.0468521118164,
      "activations/layer8_attention_weight_min": -60.22602844238281,
      "activations/layer9_attention_weight_max": 71.9517593383789,
      "activations/layer9_attention_weight_min": -67.52888488769531,
      "epoch": 0.36,
      "learning_rate": 0.0001491746212121212,
      "loss": 3.3897,
      "step": 6200
    },
    {
      "activations/layer0_attention_weight_max": 14.753493309020996,
      "activations/layer0_attention_weight_min": -11.187966346740723,
      "activations/layer10_attention_weight_max": 47.23654556274414,
      "activations/layer10_attention_weight_min": -38.2188606262207,
      "activations/layer11_attention_weight_max": 43.7773551940918,
      "activations/layer11_attention_weight_min": -36.58476257324219,
      "activations/layer12_attention_weight_max": 27.555545806884766,
      "activations/layer12_attention_weight_min": -20.47174835205078,
      "activations/layer13_attention_weight_max": 27.844064712524414,
      "activations/layer13_attention_weight_min": -22.590219497680664,
      "activations/layer14_attention_weight_max": 34.057212829589844,
      "activations/layer14_attention_weight_min": -23.103302001953125,
      "activations/layer15_attention_weight_max": 26.30471420288086,
      "activations/layer15_attention_weight_min": -17.327625274658203,
      "activations/layer16_attention_weight_max": 22.45125389099121,
      "activations/layer16_attention_weight_min": -20.050100326538086,
      "activations/layer17_attention_weight_max": 21.3568058013916,
      "activations/layer17_attention_weight_min": -17.376188278198242,
      "activations/layer18_attention_weight_max": 19.892427444458008,
      "activations/layer18_attention_weight_min": -12.167811393737793,
      "activations/layer19_attention_weight_max": 20.385602951049805,
      "activations/layer19_attention_weight_min": -13.2191162109375,
      "activations/layer1_attention_weight_max": 14.6242036819458,
      "activations/layer1_attention_weight_min": -11.018898963928223,
      "activations/layer20_attention_weight_max": 21.483272552490234,
      "activations/layer20_attention_weight_min": -10.892223358154297,
      "activations/layer21_attention_weight_max": 18.611553192138672,
      "activations/layer21_attention_weight_min": -10.342020034790039,
      "activations/layer22_attention_weight_max": 24.577037811279297,
      "activations/layer22_attention_weight_min": -15.639633178710938,
      "activations/layer23_attention_weight_max": 18.513378143310547,
      "activations/layer23_attention_weight_min": -12.82240104675293,
      "activations/layer2_attention_weight_max": 26.77242660522461,
      "activations/layer2_attention_weight_min": -21.443410873413086,
      "activations/layer3_attention_weight_max": 55.12456130981445,
      "activations/layer3_attention_weight_min": -46.19009780883789,
      "activations/layer4_attention_weight_max": 123.3101577758789,
      "activations/layer4_attention_weight_min": -109.53787994384766,
      "activations/layer5_attention_weight_max": 141.981689453125,
      "activations/layer5_attention_weight_min": -120.39448547363281,
      "activations/layer6_attention_weight_max": 91.87631225585938,
      "activations/layer6_attention_weight_min": -85.03713989257812,
      "activations/layer7_attention_weight_max": 85.38851928710938,
      "activations/layer7_attention_weight_min": -73.00105285644531,
      "activations/layer8_attention_weight_max": 82.03201293945312,
      "activations/layer8_attention_weight_min": -67.82075500488281,
      "activations/layer9_attention_weight_max": 67.5881576538086,
      "activations/layer9_attention_weight_min": -60.262874603271484,
      "epoch": 0.36,
      "learning_rate": 0.00014915568181818182,
      "loss": 3.3922,
      "step": 6250
    },
    {
      "activations/layer0_attention_weight_max": 14.646763801574707,
      "activations/layer0_attention_weight_min": -11.043766021728516,
      "activations/layer10_attention_weight_max": 49.277374267578125,
      "activations/layer10_attention_weight_min": -36.905208587646484,
      "activations/layer11_attention_weight_max": 47.379119873046875,
      "activations/layer11_attention_weight_min": -37.786720275878906,
      "activations/layer12_attention_weight_max": 28.6023006439209,
      "activations/layer12_attention_weight_min": -22.03615951538086,
      "activations/layer13_attention_weight_max": 29.652040481567383,
      "activations/layer13_attention_weight_min": -22.81683921813965,
      "activations/layer14_attention_weight_max": 36.91551208496094,
      "activations/layer14_attention_weight_min": -21.819442749023438,
      "activations/layer15_attention_weight_max": 26.230884552001953,
      "activations/layer15_attention_weight_min": -18.119041442871094,
      "activations/layer16_attention_weight_max": 26.920509338378906,
      "activations/layer16_attention_weight_min": -16.940820693969727,
      "activations/layer17_attention_weight_max": 29.70137596130371,
      "activations/layer17_attention_weight_min": -14.111141204833984,
      "activations/layer18_attention_weight_max": 25.5355281829834,
      "activations/layer18_attention_weight_min": -11.76187515258789,
      "activations/layer19_attention_weight_max": 22.316951751708984,
      "activations/layer19_attention_weight_min": -13.488986015319824,
      "activations/layer1_attention_weight_max": 16.35944938659668,
      "activations/layer1_attention_weight_min": -10.766069412231445,
      "activations/layer20_attention_weight_max": 24.008134841918945,
      "activations/layer20_attention_weight_min": -14.959857940673828,
      "activations/layer21_attention_weight_max": 21.570528030395508,
      "activations/layer21_attention_weight_min": -12.131712913513184,
      "activations/layer22_attention_weight_max": 24.93933868408203,
      "activations/layer22_attention_weight_min": -14.924834251403809,
      "activations/layer23_attention_weight_max": 19.19881248474121,
      "activations/layer23_attention_weight_min": -13.884065628051758,
      "activations/layer2_attention_weight_max": 27.141666412353516,
      "activations/layer2_attention_weight_min": -21.621288299560547,
      "activations/layer3_attention_weight_max": 57.29197311401367,
      "activations/layer3_attention_weight_min": -48.727149963378906,
      "activations/layer4_attention_weight_max": 126.91468811035156,
      "activations/layer4_attention_weight_min": -112.07257080078125,
      "activations/layer5_attention_weight_max": 139.3731689453125,
      "activations/layer5_attention_weight_min": -123.44805908203125,
      "activations/layer6_attention_weight_max": 91.0541763305664,
      "activations/layer6_attention_weight_min": -84.5440444946289,
      "activations/layer7_attention_weight_max": 88.43590545654297,
      "activations/layer7_attention_weight_min": -72.87406921386719,
      "activations/layer8_attention_weight_max": 78.81526184082031,
      "activations/layer8_attention_weight_min": -62.46903610229492,
      "activations/layer9_attention_weight_max": 66.3385238647461,
      "activations/layer9_attention_weight_min": -62.00702667236328,
      "epoch": 0.37,
      "learning_rate": 0.0001491367424242424,
      "loss": 3.3803,
      "step": 6300
    },
    {
      "activations/layer0_attention_weight_max": 14.237586975097656,
      "activations/layer0_attention_weight_min": -10.735395431518555,
      "activations/layer10_attention_weight_max": 47.588348388671875,
      "activations/layer10_attention_weight_min": -37.690345764160156,
      "activations/layer11_attention_weight_max": 45.21593475341797,
      "activations/layer11_attention_weight_min": -36.861995697021484,
      "activations/layer12_attention_weight_max": 34.78096389770508,
      "activations/layer12_attention_weight_min": -20.942419052124023,
      "activations/layer13_attention_weight_max": 30.74850082397461,
      "activations/layer13_attention_weight_min": -20.7895450592041,
      "activations/layer14_attention_weight_max": 34.3832893371582,
      "activations/layer14_attention_weight_min": -21.31538963317871,
      "activations/layer15_attention_weight_max": 28.813570022583008,
      "activations/layer15_attention_weight_min": -18.00760269165039,
      "activations/layer16_attention_weight_max": 26.326648712158203,
      "activations/layer16_attention_weight_min": -14.139729499816895,
      "activations/layer17_attention_weight_max": 24.799970626831055,
      "activations/layer17_attention_weight_min": -13.883235931396484,
      "activations/layer18_attention_weight_max": 22.450546264648438,
      "activations/layer18_attention_weight_min": -10.36555290222168,
      "activations/layer19_attention_weight_max": 22.36899185180664,
      "activations/layer19_attention_weight_min": -11.904157638549805,
      "activations/layer1_attention_weight_max": 15.708114624023438,
      "activations/layer1_attention_weight_min": -11.399356842041016,
      "activations/layer20_attention_weight_max": 20.48948097229004,
      "activations/layer20_attention_weight_min": -11.92379093170166,
      "activations/layer21_attention_weight_max": 18.94863510131836,
      "activations/layer21_attention_weight_min": -9.864322662353516,
      "activations/layer22_attention_weight_max": 28.418773651123047,
      "activations/layer22_attention_weight_min": -13.36720085144043,
      "activations/layer23_attention_weight_max": 19.4641056060791,
      "activations/layer23_attention_weight_min": -10.663707733154297,
      "activations/layer2_attention_weight_max": 28.45223617553711,
      "activations/layer2_attention_weight_min": -21.20827865600586,
      "activations/layer3_attention_weight_max": 57.357669830322266,
      "activations/layer3_attention_weight_min": -47.42829513549805,
      "activations/layer4_attention_weight_max": 122.51114654541016,
      "activations/layer4_attention_weight_min": -108.994384765625,
      "activations/layer5_attention_weight_max": 143.46551513671875,
      "activations/layer5_attention_weight_min": -119.96431732177734,
      "activations/layer6_attention_weight_max": 94.10023498535156,
      "activations/layer6_attention_weight_min": -83.05435180664062,
      "activations/layer7_attention_weight_max": 91.8733901977539,
      "activations/layer7_attention_weight_min": -81.01327514648438,
      "activations/layer8_attention_weight_max": 76.9343490600586,
      "activations/layer8_attention_weight_min": -60.36721420288086,
      "activations/layer9_attention_weight_max": 67.06996154785156,
      "activations/layer9_attention_weight_min": -64.42965698242188,
      "epoch": 0.37,
      "learning_rate": 0.00014911780303030302,
      "loss": 3.3724,
      "step": 6350
    },
    {
      "activations/layer0_attention_weight_max": 13.692294120788574,
      "activations/layer0_attention_weight_min": -10.495922088623047,
      "activations/layer10_attention_weight_max": 47.710113525390625,
      "activations/layer10_attention_weight_min": -39.60039138793945,
      "activations/layer11_attention_weight_max": 45.203636169433594,
      "activations/layer11_attention_weight_min": -33.009193420410156,
      "activations/layer12_attention_weight_max": 27.093177795410156,
      "activations/layer12_attention_weight_min": -20.955909729003906,
      "activations/layer13_attention_weight_max": 28.892208099365234,
      "activations/layer13_attention_weight_min": -25.033206939697266,
      "activations/layer14_attention_weight_max": 34.26148223876953,
      "activations/layer14_attention_weight_min": -21.18003273010254,
      "activations/layer15_attention_weight_max": 35.51647186279297,
      "activations/layer15_attention_weight_min": -19.755598068237305,
      "activations/layer16_attention_weight_max": 23.22968864440918,
      "activations/layer16_attention_weight_min": -17.55379867553711,
      "activations/layer17_attention_weight_max": 20.519710540771484,
      "activations/layer17_attention_weight_min": -16.67108726501465,
      "activations/layer18_attention_weight_max": 22.129262924194336,
      "activations/layer18_attention_weight_min": -12.949064254760742,
      "activations/layer19_attention_weight_max": 20.231733322143555,
      "activations/layer19_attention_weight_min": -13.958330154418945,
      "activations/layer1_attention_weight_max": 15.303596496582031,
      "activations/layer1_attention_weight_min": -11.41242504119873,
      "activations/layer20_attention_weight_max": 22.533187866210938,
      "activations/layer20_attention_weight_min": -13.160099983215332,
      "activations/layer21_attention_weight_max": 17.599781036376953,
      "activations/layer21_attention_weight_min": -10.092172622680664,
      "activations/layer22_attention_weight_max": 21.314607620239258,
      "activations/layer22_attention_weight_min": -14.234984397888184,
      "activations/layer23_attention_weight_max": 17.007953643798828,
      "activations/layer23_attention_weight_min": -10.791426658630371,
      "activations/layer2_attention_weight_max": 27.505033493041992,
      "activations/layer2_attention_weight_min": -22.01697540283203,
      "activations/layer3_attention_weight_max": 55.780494689941406,
      "activations/layer3_attention_weight_min": -47.56542205810547,
      "activations/layer4_attention_weight_max": 121.11286926269531,
      "activations/layer4_attention_weight_min": -114.01921844482422,
      "activations/layer5_attention_weight_max": 139.07562255859375,
      "activations/layer5_attention_weight_min": -125.74590301513672,
      "activations/layer6_attention_weight_max": 86.64021301269531,
      "activations/layer6_attention_weight_min": -84.69117736816406,
      "activations/layer7_attention_weight_max": 80.987060546875,
      "activations/layer7_attention_weight_min": -70.76386260986328,
      "activations/layer8_attention_weight_max": 70.50164031982422,
      "activations/layer8_attention_weight_min": -61.0838623046875,
      "activations/layer9_attention_weight_max": 66.5685043334961,
      "activations/layer9_attention_weight_min": -61.03745651245117,
      "epoch": 0.37,
      "learning_rate": 0.00014909886363636364,
      "loss": 3.3807,
      "step": 6400
    },
    {
      "activations/layer0_attention_weight_max": 14.511165618896484,
      "activations/layer0_attention_weight_min": -10.49526596069336,
      "activations/layer10_attention_weight_max": 44.70753860473633,
      "activations/layer10_attention_weight_min": -37.2064094543457,
      "activations/layer11_attention_weight_max": 40.88914489746094,
      "activations/layer11_attention_weight_min": -35.67932891845703,
      "activations/layer12_attention_weight_max": 28.123226165771484,
      "activations/layer12_attention_weight_min": -25.50653648376465,
      "activations/layer13_attention_weight_max": 26.631370544433594,
      "activations/layer13_attention_weight_min": -26.01645278930664,
      "activations/layer14_attention_weight_max": 39.649234771728516,
      "activations/layer14_attention_weight_min": -26.566844940185547,
      "activations/layer15_attention_weight_max": 28.742164611816406,
      "activations/layer15_attention_weight_min": -21.464168548583984,
      "activations/layer16_attention_weight_max": 27.470476150512695,
      "activations/layer16_attention_weight_min": -16.198484420776367,
      "activations/layer17_attention_weight_max": 26.171049118041992,
      "activations/layer17_attention_weight_min": -14.814619064331055,
      "activations/layer18_attention_weight_max": 21.733253479003906,
      "activations/layer18_attention_weight_min": -12.051204681396484,
      "activations/layer19_attention_weight_max": 24.965024948120117,
      "activations/layer19_attention_weight_min": -13.62415599822998,
      "activations/layer1_attention_weight_max": 14.382197380065918,
      "activations/layer1_attention_weight_min": -10.935892105102539,
      "activations/layer20_attention_weight_max": 20.891490936279297,
      "activations/layer20_attention_weight_min": -12.399923324584961,
      "activations/layer21_attention_weight_max": 21.611286163330078,
      "activations/layer21_attention_weight_min": -11.916686058044434,
      "activations/layer22_attention_weight_max": 23.644433975219727,
      "activations/layer22_attention_weight_min": -12.617234230041504,
      "activations/layer23_attention_weight_max": 18.162044525146484,
      "activations/layer23_attention_weight_min": -13.719339370727539,
      "activations/layer2_attention_weight_max": 27.087848663330078,
      "activations/layer2_attention_weight_min": -21.58344841003418,
      "activations/layer3_attention_weight_max": 55.172447204589844,
      "activations/layer3_attention_weight_min": -48.82859420776367,
      "activations/layer4_attention_weight_max": 121.34317779541016,
      "activations/layer4_attention_weight_min": -116.9256362915039,
      "activations/layer5_attention_weight_max": 137.73892211914062,
      "activations/layer5_attention_weight_min": -124.95103454589844,
      "activations/layer6_attention_weight_max": 88.94924926757812,
      "activations/layer6_attention_weight_min": -82.68244934082031,
      "activations/layer7_attention_weight_max": 84.98119354248047,
      "activations/layer7_attention_weight_min": -73.5228271484375,
      "activations/layer8_attention_weight_max": 70.8073959350586,
      "activations/layer8_attention_weight_min": -65.00794982910156,
      "activations/layer9_attention_weight_max": 70.69087982177734,
      "activations/layer9_attention_weight_min": -59.1414794921875,
      "epoch": 0.37,
      "learning_rate": 0.00014907992424242423,
      "loss": 3.3921,
      "step": 6450
    },
    {
      "activations/layer0_attention_weight_max": 14.59870433807373,
      "activations/layer0_attention_weight_min": -10.298748016357422,
      "activations/layer10_attention_weight_max": 46.22584533691406,
      "activations/layer10_attention_weight_min": -38.27323913574219,
      "activations/layer11_attention_weight_max": 51.52940368652344,
      "activations/layer11_attention_weight_min": -36.601070404052734,
      "activations/layer12_attention_weight_max": 30.589988708496094,
      "activations/layer12_attention_weight_min": -23.6193904876709,
      "activations/layer13_attention_weight_max": 32.66065979003906,
      "activations/layer13_attention_weight_min": -22.364696502685547,
      "activations/layer14_attention_weight_max": 41.51234817504883,
      "activations/layer14_attention_weight_min": -23.536972045898438,
      "activations/layer15_attention_weight_max": 28.43326759338379,
      "activations/layer15_attention_weight_min": -21.375608444213867,
      "activations/layer16_attention_weight_max": 22.510160446166992,
      "activations/layer16_attention_weight_min": -17.01970100402832,
      "activations/layer17_attention_weight_max": 25.305444717407227,
      "activations/layer17_attention_weight_min": -17.32061195373535,
      "activations/layer18_attention_weight_max": 20.850875854492188,
      "activations/layer18_attention_weight_min": -11.717293739318848,
      "activations/layer19_attention_weight_max": 20.234891891479492,
      "activations/layer19_attention_weight_min": -14.84500789642334,
      "activations/layer1_attention_weight_max": 15.429259300231934,
      "activations/layer1_attention_weight_min": -11.036605834960938,
      "activations/layer20_attention_weight_max": 22.870044708251953,
      "activations/layer20_attention_weight_min": -11.339735984802246,
      "activations/layer21_attention_weight_max": 21.506444931030273,
      "activations/layer21_attention_weight_min": -9.492286682128906,
      "activations/layer22_attention_weight_max": 20.056306838989258,
      "activations/layer22_attention_weight_min": -13.258438110351562,
      "activations/layer23_attention_weight_max": 16.78070831298828,
      "activations/layer23_attention_weight_min": -10.929876327514648,
      "activations/layer2_attention_weight_max": 29.242826461791992,
      "activations/layer2_attention_weight_min": -21.692007064819336,
      "activations/layer3_attention_weight_max": 59.33940124511719,
      "activations/layer3_attention_weight_min": -49.195865631103516,
      "activations/layer4_attention_weight_max": 129.42904663085938,
      "activations/layer4_attention_weight_min": -112.14815521240234,
      "activations/layer5_attention_weight_max": 148.1878662109375,
      "activations/layer5_attention_weight_min": -127.74227142333984,
      "activations/layer6_attention_weight_max": 91.58145904541016,
      "activations/layer6_attention_weight_min": -81.70610046386719,
      "activations/layer7_attention_weight_max": 83.61012268066406,
      "activations/layer7_attention_weight_min": -74.44686889648438,
      "activations/layer8_attention_weight_max": 76.93956756591797,
      "activations/layer8_attention_weight_min": -62.24945831298828,
      "activations/layer9_attention_weight_max": 72.80213165283203,
      "activations/layer9_attention_weight_min": -63.25193405151367,
      "epoch": 0.38,
      "learning_rate": 0.00014906098484848484,
      "loss": 3.3786,
      "step": 6500
    },
    {
      "activations/layer0_attention_weight_max": 14.869426727294922,
      "activations/layer0_attention_weight_min": -10.309954643249512,
      "activations/layer10_attention_weight_max": 48.825347900390625,
      "activations/layer10_attention_weight_min": -38.13546371459961,
      "activations/layer11_attention_weight_max": 47.09955978393555,
      "activations/layer11_attention_weight_min": -35.31660461425781,
      "activations/layer12_attention_weight_max": 38.53912353515625,
      "activations/layer12_attention_weight_min": -27.55726432800293,
      "activations/layer13_attention_weight_max": 31.003963470458984,
      "activations/layer13_attention_weight_min": -29.687992095947266,
      "activations/layer14_attention_weight_max": 32.59366989135742,
      "activations/layer14_attention_weight_min": -24.130006790161133,
      "activations/layer15_attention_weight_max": 30.840089797973633,
      "activations/layer15_attention_weight_min": -21.6129207611084,
      "activations/layer16_attention_weight_max": 24.3868408203125,
      "activations/layer16_attention_weight_min": -18.508337020874023,
      "activations/layer17_attention_weight_max": 23.612245559692383,
      "activations/layer17_attention_weight_min": -15.619489669799805,
      "activations/layer18_attention_weight_max": 19.602611541748047,
      "activations/layer18_attention_weight_min": -12.159098625183105,
      "activations/layer19_attention_weight_max": 19.806297302246094,
      "activations/layer19_attention_weight_min": -14.741633415222168,
      "activations/layer1_attention_weight_max": 14.188746452331543,
      "activations/layer1_attention_weight_min": -11.059709548950195,
      "activations/layer20_attention_weight_max": 20.384885787963867,
      "activations/layer20_attention_weight_min": -13.341999053955078,
      "activations/layer21_attention_weight_max": 17.224937438964844,
      "activations/layer21_attention_weight_min": -10.993548393249512,
      "activations/layer22_attention_weight_max": 22.614309310913086,
      "activations/layer22_attention_weight_min": -14.45882797241211,
      "activations/layer23_attention_weight_max": 17.797645568847656,
      "activations/layer23_attention_weight_min": -13.230587005615234,
      "activations/layer2_attention_weight_max": 28.655187606811523,
      "activations/layer2_attention_weight_min": -21.823974609375,
      "activations/layer3_attention_weight_max": 58.8428955078125,
      "activations/layer3_attention_weight_min": -50.17332458496094,
      "activations/layer4_attention_weight_max": 127.52191162109375,
      "activations/layer4_attention_weight_min": -111.77855682373047,
      "activations/layer5_attention_weight_max": 145.2104949951172,
      "activations/layer5_attention_weight_min": -127.88618469238281,
      "activations/layer6_attention_weight_max": 90.0653305053711,
      "activations/layer6_attention_weight_min": -84.55978393554688,
      "activations/layer7_attention_weight_max": 82.6325912475586,
      "activations/layer7_attention_weight_min": -70.47821044921875,
      "activations/layer8_attention_weight_max": 72.17042541503906,
      "activations/layer8_attention_weight_min": -59.98685073852539,
      "activations/layer9_attention_weight_max": 69.0053939819336,
      "activations/layer9_attention_weight_min": -60.09904861450195,
      "epoch": 0.38,
      "learning_rate": 0.00014904204545454543,
      "loss": 3.3826,
      "step": 6550
    },
    {
      "activations/layer0_attention_weight_max": 14.640202522277832,
      "activations/layer0_attention_weight_min": -10.499884605407715,
      "activations/layer10_attention_weight_max": 49.70301055908203,
      "activations/layer10_attention_weight_min": -38.79092788696289,
      "activations/layer11_attention_weight_max": 53.421485900878906,
      "activations/layer11_attention_weight_min": -35.955806732177734,
      "activations/layer12_attention_weight_max": 28.26548957824707,
      "activations/layer12_attention_weight_min": -24.66881561279297,
      "activations/layer13_attention_weight_max": 30.457454681396484,
      "activations/layer13_attention_weight_min": -21.70574378967285,
      "activations/layer14_attention_weight_max": 35.58116149902344,
      "activations/layer14_attention_weight_min": -25.40589141845703,
      "activations/layer15_attention_weight_max": 26.495330810546875,
      "activations/layer15_attention_weight_min": -20.95047378540039,
      "activations/layer16_attention_weight_max": 18.75485610961914,
      "activations/layer16_attention_weight_min": -17.37693214416504,
      "activations/layer17_attention_weight_max": 17.033472061157227,
      "activations/layer17_attention_weight_min": -15.865090370178223,
      "activations/layer18_attention_weight_max": 16.01227378845215,
      "activations/layer18_attention_weight_min": -12.325850486755371,
      "activations/layer19_attention_weight_max": 19.825096130371094,
      "activations/layer19_attention_weight_min": -13.365750312805176,
      "activations/layer1_attention_weight_max": 15.772309303283691,
      "activations/layer1_attention_weight_min": -12.156340599060059,
      "activations/layer20_attention_weight_max": 19.326251983642578,
      "activations/layer20_attention_weight_min": -13.47913932800293,
      "activations/layer21_attention_weight_max": 18.803516387939453,
      "activations/layer21_attention_weight_min": -11.331687927246094,
      "activations/layer22_attention_weight_max": 22.867286682128906,
      "activations/layer22_attention_weight_min": -13.340188026428223,
      "activations/layer23_attention_weight_max": 17.313764572143555,
      "activations/layer23_attention_weight_min": -12.454882621765137,
      "activations/layer2_attention_weight_max": 30.772342681884766,
      "activations/layer2_attention_weight_min": -22.220417022705078,
      "activations/layer3_attention_weight_max": 65.18592071533203,
      "activations/layer3_attention_weight_min": -51.395843505859375,
      "activations/layer4_attention_weight_max": 133.80479431152344,
      "activations/layer4_attention_weight_min": -115.02599334716797,
      "activations/layer5_attention_weight_max": 160.63905334472656,
      "activations/layer5_attention_weight_min": -127.896240234375,
      "activations/layer6_attention_weight_max": 99.40506744384766,
      "activations/layer6_attention_weight_min": -84.8585205078125,
      "activations/layer7_attention_weight_max": 88.78958129882812,
      "activations/layer7_attention_weight_min": -71.75634765625,
      "activations/layer8_attention_weight_max": 76.01567840576172,
      "activations/layer8_attention_weight_min": -62.12133026123047,
      "activations/layer9_attention_weight_max": 87.86567687988281,
      "activations/layer9_attention_weight_min": -71.12503051757812,
      "epoch": 0.38,
      "learning_rate": 0.00014902310606060604,
      "loss": 3.3766,
      "step": 6600
    },
    {
      "activations/layer0_attention_weight_max": 14.928244590759277,
      "activations/layer0_attention_weight_min": -10.263596534729004,
      "activations/layer10_attention_weight_max": 46.3993034362793,
      "activations/layer10_attention_weight_min": -36.52558135986328,
      "activations/layer11_attention_weight_max": 52.753074645996094,
      "activations/layer11_attention_weight_min": -36.387142181396484,
      "activations/layer12_attention_weight_max": 30.53293228149414,
      "activations/layer12_attention_weight_min": -27.288990020751953,
      "activations/layer13_attention_weight_max": 30.03621482849121,
      "activations/layer13_attention_weight_min": -26.51451301574707,
      "activations/layer14_attention_weight_max": 30.971494674682617,
      "activations/layer14_attention_weight_min": -21.217243194580078,
      "activations/layer15_attention_weight_max": 26.740076065063477,
      "activations/layer15_attention_weight_min": -23.22121238708496,
      "activations/layer16_attention_weight_max": 25.484495162963867,
      "activations/layer16_attention_weight_min": -16.421003341674805,
      "activations/layer17_attention_weight_max": 27.548019409179688,
      "activations/layer17_attention_weight_min": -14.022627830505371,
      "activations/layer18_attention_weight_max": 22.16439437866211,
      "activations/layer18_attention_weight_min": -10.279691696166992,
      "activations/layer19_attention_weight_max": 21.907684326171875,
      "activations/layer19_attention_weight_min": -11.432600975036621,
      "activations/layer1_attention_weight_max": 14.980539321899414,
      "activations/layer1_attention_weight_min": -12.825188636779785,
      "activations/layer20_attention_weight_max": 21.0540714263916,
      "activations/layer20_attention_weight_min": -11.099345207214355,
      "activations/layer21_attention_weight_max": 17.73307228088379,
      "activations/layer21_attention_weight_min": -9.671460151672363,
      "activations/layer22_attention_weight_max": 23.893905639648438,
      "activations/layer22_attention_weight_min": -12.854775428771973,
      "activations/layer23_attention_weight_max": 17.621368408203125,
      "activations/layer23_attention_weight_min": -11.389955520629883,
      "activations/layer2_attention_weight_max": 31.454227447509766,
      "activations/layer2_attention_weight_min": -24.105274200439453,
      "activations/layer3_attention_weight_max": 61.793701171875,
      "activations/layer3_attention_weight_min": -50.241825103759766,
      "activations/layer4_attention_weight_max": 129.6055908203125,
      "activations/layer4_attention_weight_min": -118.88701629638672,
      "activations/layer5_attention_weight_max": 145.15072631835938,
      "activations/layer5_attention_weight_min": -124.21356201171875,
      "activations/layer6_attention_weight_max": 92.65422058105469,
      "activations/layer6_attention_weight_min": -81.22521209716797,
      "activations/layer7_attention_weight_max": 81.93728637695312,
      "activations/layer7_attention_weight_min": -72.52718353271484,
      "activations/layer8_attention_weight_max": 73.26763153076172,
      "activations/layer8_attention_weight_min": -61.26721954345703,
      "activations/layer9_attention_weight_max": 70.05977630615234,
      "activations/layer9_attention_weight_min": -55.83845901489258,
      "epoch": 0.39,
      "learning_rate": 0.00014900416666666666,
      "loss": 3.3738,
      "step": 6650
    },
    {
      "activations/layer0_attention_weight_max": 14.823826789855957,
      "activations/layer0_attention_weight_min": -10.347541809082031,
      "activations/layer10_attention_weight_max": 47.62567901611328,
      "activations/layer10_attention_weight_min": -42.60618209838867,
      "activations/layer11_attention_weight_max": 41.790443420410156,
      "activations/layer11_attention_weight_min": -32.57768249511719,
      "activations/layer12_attention_weight_max": 28.520931243896484,
      "activations/layer12_attention_weight_min": -21.051265716552734,
      "activations/layer13_attention_weight_max": 27.901100158691406,
      "activations/layer13_attention_weight_min": -23.79277992248535,
      "activations/layer14_attention_weight_max": 34.926239013671875,
      "activations/layer14_attention_weight_min": -27.390607833862305,
      "activations/layer15_attention_weight_max": 29.03510856628418,
      "activations/layer15_attention_weight_min": -20.460676193237305,
      "activations/layer16_attention_weight_max": 24.69953727722168,
      "activations/layer16_attention_weight_min": -20.53030014038086,
      "activations/layer17_attention_weight_max": 34.9598274230957,
      "activations/layer17_attention_weight_min": -15.93133544921875,
      "activations/layer18_attention_weight_max": 23.176801681518555,
      "activations/layer18_attention_weight_min": -13.196300506591797,
      "activations/layer19_attention_weight_max": 25.031213760375977,
      "activations/layer19_attention_weight_min": -16.352144241333008,
      "activations/layer1_attention_weight_max": 15.411944389343262,
      "activations/layer1_attention_weight_min": -11.082892417907715,
      "activations/layer20_attention_weight_max": 23.36874771118164,
      "activations/layer20_attention_weight_min": -15.780078887939453,
      "activations/layer21_attention_weight_max": 21.504968643188477,
      "activations/layer21_attention_weight_min": -13.905693054199219,
      "activations/layer22_attention_weight_max": 26.479373931884766,
      "activations/layer22_attention_weight_min": -16.45008659362793,
      "activations/layer23_attention_weight_max": 20.27255630493164,
      "activations/layer23_attention_weight_min": -14.229938507080078,
      "activations/layer2_attention_weight_max": 27.4202880859375,
      "activations/layer2_attention_weight_min": -22.380626678466797,
      "activations/layer3_attention_weight_max": 56.604679107666016,
      "activations/layer3_attention_weight_min": -49.07317352294922,
      "activations/layer4_attention_weight_max": 120.87151336669922,
      "activations/layer4_attention_weight_min": -109.0958480834961,
      "activations/layer5_attention_weight_max": 136.1285858154297,
      "activations/layer5_attention_weight_min": -120.28175354003906,
      "activations/layer6_attention_weight_max": 87.08000183105469,
      "activations/layer6_attention_weight_min": -81.18344116210938,
      "activations/layer7_attention_weight_max": 80.05610656738281,
      "activations/layer7_attention_weight_min": -71.42192077636719,
      "activations/layer8_attention_weight_max": 69.3131332397461,
      "activations/layer8_attention_weight_min": -62.87164306640625,
      "activations/layer9_attention_weight_max": 65.7811508178711,
      "activations/layer9_attention_weight_min": -65.97509002685547,
      "epoch": 0.39,
      "learning_rate": 0.00014898522727272725,
      "loss": 3.3531,
      "step": 6700
    },
    {
      "activations/layer0_attention_weight_max": 14.171435356140137,
      "activations/layer0_attention_weight_min": -10.273594856262207,
      "activations/layer10_attention_weight_max": 46.738731384277344,
      "activations/layer10_attention_weight_min": -36.87915802001953,
      "activations/layer11_attention_weight_max": 46.52529525756836,
      "activations/layer11_attention_weight_min": -42.38274383544922,
      "activations/layer12_attention_weight_max": 24.652423858642578,
      "activations/layer12_attention_weight_min": -28.740821838378906,
      "activations/layer13_attention_weight_max": 30.630046844482422,
      "activations/layer13_attention_weight_min": -29.05817222595215,
      "activations/layer14_attention_weight_max": 28.462158203125,
      "activations/layer14_attention_weight_min": -23.15980339050293,
      "activations/layer15_attention_weight_max": 23.30228614807129,
      "activations/layer15_attention_weight_min": -18.032865524291992,
      "activations/layer16_attention_weight_max": 19.858509063720703,
      "activations/layer16_attention_weight_min": -18.966325759887695,
      "activations/layer17_attention_weight_max": 19.599794387817383,
      "activations/layer17_attention_weight_min": -19.012069702148438,
      "activations/layer18_attention_weight_max": 15.178675651550293,
      "activations/layer18_attention_weight_min": -12.865907669067383,
      "activations/layer19_attention_weight_max": 17.5535831451416,
      "activations/layer19_attention_weight_min": -18.01223373413086,
      "activations/layer1_attention_weight_max": 14.377779006958008,
      "activations/layer1_attention_weight_min": -11.152152061462402,
      "activations/layer20_attention_weight_max": 19.33489418029785,
      "activations/layer20_attention_weight_min": -16.203874588012695,
      "activations/layer21_attention_weight_max": 15.08657455444336,
      "activations/layer21_attention_weight_min": -15.401084899902344,
      "activations/layer22_attention_weight_max": 19.453441619873047,
      "activations/layer22_attention_weight_min": -17.04891014099121,
      "activations/layer23_attention_weight_max": 15.830401420593262,
      "activations/layer23_attention_weight_min": -15.551589012145996,
      "activations/layer2_attention_weight_max": 28.846052169799805,
      "activations/layer2_attention_weight_min": -22.565078735351562,
      "activations/layer3_attention_weight_max": 57.78772735595703,
      "activations/layer3_attention_weight_min": -48.71434020996094,
      "activations/layer4_attention_weight_max": 127.90486907958984,
      "activations/layer4_attention_weight_min": -117.16561126708984,
      "activations/layer5_attention_weight_max": 144.616943359375,
      "activations/layer5_attention_weight_min": -128.42062377929688,
      "activations/layer6_attention_weight_max": 93.0140609741211,
      "activations/layer6_attention_weight_min": -81.65330505371094,
      "activations/layer7_attention_weight_max": 84.96176147460938,
      "activations/layer7_attention_weight_min": -71.69078063964844,
      "activations/layer8_attention_weight_max": 72.95121002197266,
      "activations/layer8_attention_weight_min": -58.07510757446289,
      "activations/layer9_attention_weight_max": 77.88311004638672,
      "activations/layer9_attention_weight_min": -62.01282501220703,
      "epoch": 0.39,
      "learning_rate": 0.00014896628787878786,
      "loss": 3.3547,
      "step": 6750
    },
    {
      "activations/layer0_attention_weight_max": 14.878186225891113,
      "activations/layer0_attention_weight_min": -10.819804191589355,
      "activations/layer10_attention_weight_max": 46.702911376953125,
      "activations/layer10_attention_weight_min": -38.293846130371094,
      "activations/layer11_attention_weight_max": 49.93650436401367,
      "activations/layer11_attention_weight_min": -38.583778381347656,
      "activations/layer12_attention_weight_max": 27.66596794128418,
      "activations/layer12_attention_weight_min": -21.148677825927734,
      "activations/layer13_attention_weight_max": 31.293819427490234,
      "activations/layer13_attention_weight_min": -25.346641540527344,
      "activations/layer14_attention_weight_max": 33.01214599609375,
      "activations/layer14_attention_weight_min": -29.309112548828125,
      "activations/layer15_attention_weight_max": 28.84380340576172,
      "activations/layer15_attention_weight_min": -22.133115768432617,
      "activations/layer16_attention_weight_max": 23.736919403076172,
      "activations/layer16_attention_weight_min": -17.460355758666992,
      "activations/layer17_attention_weight_max": 24.90352439880371,
      "activations/layer17_attention_weight_min": -14.818324089050293,
      "activations/layer18_attention_weight_max": 17.474403381347656,
      "activations/layer18_attention_weight_min": -12.157947540283203,
      "activations/layer19_attention_weight_max": 25.27275848388672,
      "activations/layer19_attention_weight_min": -18.221359252929688,
      "activations/layer1_attention_weight_max": 16.63106346130371,
      "activations/layer1_attention_weight_min": -10.826776504516602,
      "activations/layer20_attention_weight_max": 25.298654556274414,
      "activations/layer20_attention_weight_min": -13.846210479736328,
      "activations/layer21_attention_weight_max": 21.249853134155273,
      "activations/layer21_attention_weight_min": -13.449578285217285,
      "activations/layer22_attention_weight_max": 22.90336799621582,
      "activations/layer22_attention_weight_min": -16.35185432434082,
      "activations/layer23_attention_weight_max": 16.0404052734375,
      "activations/layer23_attention_weight_min": -13.194210052490234,
      "activations/layer2_attention_weight_max": 27.856121063232422,
      "activations/layer2_attention_weight_min": -21.459056854248047,
      "activations/layer3_attention_weight_max": 53.69832229614258,
      "activations/layer3_attention_weight_min": -45.67490768432617,
      "activations/layer4_attention_weight_max": 117.476318359375,
      "activations/layer4_attention_weight_min": -106.86677551269531,
      "activations/layer5_attention_weight_max": 142.57379150390625,
      "activations/layer5_attention_weight_min": -123.31820678710938,
      "activations/layer6_attention_weight_max": 91.69644927978516,
      "activations/layer6_attention_weight_min": -81.6167984008789,
      "activations/layer7_attention_weight_max": 87.91975402832031,
      "activations/layer7_attention_weight_min": -73.04721069335938,
      "activations/layer8_attention_weight_max": 75.33634185791016,
      "activations/layer8_attention_weight_min": -64.24565124511719,
      "activations/layer9_attention_weight_max": 71.62166595458984,
      "activations/layer9_attention_weight_min": -66.34960174560547,
      "epoch": 0.4,
      "learning_rate": 0.00014894734848484848,
      "loss": 3.344,
      "step": 6800
    },
    {
      "activations/layer0_attention_weight_max": 14.445775032043457,
      "activations/layer0_attention_weight_min": -10.95162582397461,
      "activations/layer10_attention_weight_max": 49.79005813598633,
      "activations/layer10_attention_weight_min": -41.59718704223633,
      "activations/layer11_attention_weight_max": 56.22607421875,
      "activations/layer11_attention_weight_min": -40.34182357788086,
      "activations/layer12_attention_weight_max": 28.33017921447754,
      "activations/layer12_attention_weight_min": -23.250320434570312,
      "activations/layer13_attention_weight_max": 32.55533981323242,
      "activations/layer13_attention_weight_min": -27.296438217163086,
      "activations/layer14_attention_weight_max": 38.34120178222656,
      "activations/layer14_attention_weight_min": -26.435932159423828,
      "activations/layer15_attention_weight_max": 29.88751220703125,
      "activations/layer15_attention_weight_min": -19.857370376586914,
      "activations/layer16_attention_weight_max": 22.228530883789062,
      "activations/layer16_attention_weight_min": -16.983186721801758,
      "activations/layer17_attention_weight_max": 22.891010284423828,
      "activations/layer17_attention_weight_min": -16.185842514038086,
      "activations/layer18_attention_weight_max": 24.38698959350586,
      "activations/layer18_attention_weight_min": -12.874263763427734,
      "activations/layer19_attention_weight_max": 24.234783172607422,
      "activations/layer19_attention_weight_min": -14.284891128540039,
      "activations/layer1_attention_weight_max": 15.726929664611816,
      "activations/layer1_attention_weight_min": -10.729212760925293,
      "activations/layer20_attention_weight_max": 25.51603889465332,
      "activations/layer20_attention_weight_min": -12.192621231079102,
      "activations/layer21_attention_weight_max": 22.565185546875,
      "activations/layer21_attention_weight_min": -11.557968139648438,
      "activations/layer22_attention_weight_max": 25.865869522094727,
      "activations/layer22_attention_weight_min": -13.513219833374023,
      "activations/layer23_attention_weight_max": 19.936824798583984,
      "activations/layer23_attention_weight_min": -10.680931091308594,
      "activations/layer2_attention_weight_max": 28.547523498535156,
      "activations/layer2_attention_weight_min": -23.09144401550293,
      "activations/layer3_attention_weight_max": 56.83447265625,
      "activations/layer3_attention_weight_min": -46.427223205566406,
      "activations/layer4_attention_weight_max": 117.89708709716797,
      "activations/layer4_attention_weight_min": -107.74623107910156,
      "activations/layer5_attention_weight_max": 137.71310424804688,
      "activations/layer5_attention_weight_min": -118.44816589355469,
      "activations/layer6_attention_weight_max": 84.98255920410156,
      "activations/layer6_attention_weight_min": -75.87223052978516,
      "activations/layer7_attention_weight_max": 81.63573455810547,
      "activations/layer7_attention_weight_min": -70.95431518554688,
      "activations/layer8_attention_weight_max": 67.69743347167969,
      "activations/layer8_attention_weight_min": -59.84158706665039,
      "activations/layer9_attention_weight_max": 66.85655975341797,
      "activations/layer9_attention_weight_min": -62.23811721801758,
      "epoch": 0.4,
      "learning_rate": 0.00014892840909090906,
      "loss": 3.3467,
      "step": 6850
    },
    {
      "activations/layer0_attention_weight_max": 16.0716552734375,
      "activations/layer0_attention_weight_min": -11.294685363769531,
      "activations/layer10_attention_weight_max": 48.927276611328125,
      "activations/layer10_attention_weight_min": -38.38724136352539,
      "activations/layer11_attention_weight_max": 55.49906921386719,
      "activations/layer11_attention_weight_min": -36.847808837890625,
      "activations/layer12_attention_weight_max": 30.121051788330078,
      "activations/layer12_attention_weight_min": -22.997495651245117,
      "activations/layer13_attention_weight_max": 34.507354736328125,
      "activations/layer13_attention_weight_min": -27.504568099975586,
      "activations/layer14_attention_weight_max": 45.587528228759766,
      "activations/layer14_attention_weight_min": -28.90138053894043,
      "activations/layer15_attention_weight_max": 39.32026672363281,
      "activations/layer15_attention_weight_min": -24.185518264770508,
      "activations/layer16_attention_weight_max": 28.0605411529541,
      "activations/layer16_attention_weight_min": -16.757259368896484,
      "activations/layer17_attention_weight_max": 28.191116333007812,
      "activations/layer17_attention_weight_min": -14.133116722106934,
      "activations/layer18_attention_weight_max": 20.411592483520508,
      "activations/layer18_attention_weight_min": -12.500688552856445,
      "activations/layer19_attention_weight_max": 24.28339958190918,
      "activations/layer19_attention_weight_min": -14.276762962341309,
      "activations/layer1_attention_weight_max": 15.689340591430664,
      "activations/layer1_attention_weight_min": -10.886054039001465,
      "activations/layer20_attention_weight_max": 28.614673614501953,
      "activations/layer20_attention_weight_min": -12.721474647521973,
      "activations/layer21_attention_weight_max": 21.311683654785156,
      "activations/layer21_attention_weight_min": -12.671675682067871,
      "activations/layer22_attention_weight_max": 26.00469970703125,
      "activations/layer22_attention_weight_min": -13.177144050598145,
      "activations/layer23_attention_weight_max": 19.867328643798828,
      "activations/layer23_attention_weight_min": -11.674402236938477,
      "activations/layer2_attention_weight_max": 30.418798446655273,
      "activations/layer2_attention_weight_min": -22.951704025268555,
      "activations/layer3_attention_weight_max": 58.45708084106445,
      "activations/layer3_attention_weight_min": -49.889854431152344,
      "activations/layer4_attention_weight_max": 124.79882049560547,
      "activations/layer4_attention_weight_min": -106.8149642944336,
      "activations/layer5_attention_weight_max": 147.11180114746094,
      "activations/layer5_attention_weight_min": -119.53624725341797,
      "activations/layer6_attention_weight_max": 97.72206115722656,
      "activations/layer6_attention_weight_min": -76.0074691772461,
      "activations/layer7_attention_weight_max": 86.02198791503906,
      "activations/layer7_attention_weight_min": -73.64478302001953,
      "activations/layer8_attention_weight_max": 80.47411346435547,
      "activations/layer8_attention_weight_min": -61.72687530517578,
      "activations/layer9_attention_weight_max": 74.0447769165039,
      "activations/layer9_attention_weight_min": -62.18855667114258,
      "epoch": 0.4,
      "learning_rate": 0.00014890946969696968,
      "loss": 3.3555,
      "step": 6900
    },
    {
      "activations/layer0_attention_weight_max": 15.821617126464844,
      "activations/layer0_attention_weight_min": -10.942383766174316,
      "activations/layer10_attention_weight_max": 47.450714111328125,
      "activations/layer10_attention_weight_min": -37.560359954833984,
      "activations/layer11_attention_weight_max": 48.26991271972656,
      "activations/layer11_attention_weight_min": -40.13350296020508,
      "activations/layer12_attention_weight_max": 28.872587203979492,
      "activations/layer12_attention_weight_min": -20.967018127441406,
      "activations/layer13_attention_weight_max": 33.15473556518555,
      "activations/layer13_attention_weight_min": -25.221073150634766,
      "activations/layer14_attention_weight_max": 36.24701690673828,
      "activations/layer14_attention_weight_min": -28.5264949798584,
      "activations/layer15_attention_weight_max": 34.82981872558594,
      "activations/layer15_attention_weight_min": -17.951932907104492,
      "activations/layer16_attention_weight_max": 28.97983169555664,
      "activations/layer16_attention_weight_min": -18.443191528320312,
      "activations/layer17_attention_weight_max": 25.878833770751953,
      "activations/layer17_attention_weight_min": -15.054936408996582,
      "activations/layer18_attention_weight_max": 23.2520694732666,
      "activations/layer18_attention_weight_min": -12.432881355285645,
      "activations/layer19_attention_weight_max": 26.408954620361328,
      "activations/layer19_attention_weight_min": -16.179197311401367,
      "activations/layer1_attention_weight_max": 14.795844078063965,
      "activations/layer1_attention_weight_min": -11.899467468261719,
      "activations/layer20_attention_weight_max": 25.77446746826172,
      "activations/layer20_attention_weight_min": -12.984724998474121,
      "activations/layer21_attention_weight_max": 28.687515258789062,
      "activations/layer21_attention_weight_min": -13.341407775878906,
      "activations/layer22_attention_weight_max": 28.703765869140625,
      "activations/layer22_attention_weight_min": -14.42370319366455,
      "activations/layer23_attention_weight_max": 22.594703674316406,
      "activations/layer23_attention_weight_min": -13.168190002441406,
      "activations/layer2_attention_weight_max": 28.12532615661621,
      "activations/layer2_attention_weight_min": -21.96227264404297,
      "activations/layer3_attention_weight_max": 57.05855178833008,
      "activations/layer3_attention_weight_min": -47.87981033325195,
      "activations/layer4_attention_weight_max": 121.65860748291016,
      "activations/layer4_attention_weight_min": -109.46916961669922,
      "activations/layer5_attention_weight_max": 139.14564514160156,
      "activations/layer5_attention_weight_min": -120.5725326538086,
      "activations/layer6_attention_weight_max": 85.88816833496094,
      "activations/layer6_attention_weight_min": -77.4097671508789,
      "activations/layer7_attention_weight_max": 80.55632019042969,
      "activations/layer7_attention_weight_min": -72.34408569335938,
      "activations/layer8_attention_weight_max": 73.4626235961914,
      "activations/layer8_attention_weight_min": -62.61515426635742,
      "activations/layer9_attention_weight_max": 70.58167266845703,
      "activations/layer9_attention_weight_min": -59.191680908203125,
      "epoch": 0.4,
      "learning_rate": 0.0001488905303030303,
      "loss": 3.3423,
      "step": 6950
    },
    {
      "activations/layer0_attention_weight_max": 17.770793914794922,
      "activations/layer0_attention_weight_min": -10.259249687194824,
      "activations/layer10_attention_weight_max": 46.27825927734375,
      "activations/layer10_attention_weight_min": -37.26924514770508,
      "activations/layer11_attention_weight_max": 47.90259552001953,
      "activations/layer11_attention_weight_min": -38.106754302978516,
      "activations/layer12_attention_weight_max": 32.73943328857422,
      "activations/layer12_attention_weight_min": -26.462284088134766,
      "activations/layer13_attention_weight_max": 35.37263107299805,
      "activations/layer13_attention_weight_min": -27.63692855834961,
      "activations/layer14_attention_weight_max": 35.98171615600586,
      "activations/layer14_attention_weight_min": -25.150623321533203,
      "activations/layer15_attention_weight_max": 40.725074768066406,
      "activations/layer15_attention_weight_min": -25.46319007873535,
      "activations/layer16_attention_weight_max": 27.502702713012695,
      "activations/layer16_attention_weight_min": -18.925704956054688,
      "activations/layer17_attention_weight_max": 22.61321449279785,
      "activations/layer17_attention_weight_min": -17.07207679748535,
      "activations/layer18_attention_weight_max": 22.505016326904297,
      "activations/layer18_attention_weight_min": -11.85490894317627,
      "activations/layer19_attention_weight_max": 22.15614128112793,
      "activations/layer19_attention_weight_min": -13.231051445007324,
      "activations/layer1_attention_weight_max": 14.43159294128418,
      "activations/layer1_attention_weight_min": -11.95378589630127,
      "activations/layer20_attention_weight_max": 20.310447692871094,
      "activations/layer20_attention_weight_min": -11.332486152648926,
      "activations/layer21_attention_weight_max": 18.202739715576172,
      "activations/layer21_attention_weight_min": -11.254632949829102,
      "activations/layer22_attention_weight_max": 23.389116287231445,
      "activations/layer22_attention_weight_min": -13.352145195007324,
      "activations/layer23_attention_weight_max": 18.61905860900879,
      "activations/layer23_attention_weight_min": -13.084417343139648,
      "activations/layer2_attention_weight_max": 26.938106536865234,
      "activations/layer2_attention_weight_min": -21.876279830932617,
      "activations/layer3_attention_weight_max": 53.73185348510742,
      "activations/layer3_attention_weight_min": -48.3426628112793,
      "activations/layer4_attention_weight_max": 114.28911590576172,
      "activations/layer4_attention_weight_min": -110.85514068603516,
      "activations/layer5_attention_weight_max": 139.1486053466797,
      "activations/layer5_attention_weight_min": -120.3525390625,
      "activations/layer6_attention_weight_max": 85.74939727783203,
      "activations/layer6_attention_weight_min": -77.78945922851562,
      "activations/layer7_attention_weight_max": 78.5541763305664,
      "activations/layer7_attention_weight_min": -71.62166595458984,
      "activations/layer8_attention_weight_max": 69.23634338378906,
      "activations/layer8_attention_weight_min": -57.34906005859375,
      "activations/layer9_attention_weight_max": 78.90587615966797,
      "activations/layer9_attention_weight_min": -61.04279327392578,
      "epoch": 0.41,
      "learning_rate": 0.00014887159090909088,
      "loss": 3.3368,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_loss": 3.28515625,
      "eval_runtime": 8.2274,
      "eval_samples_per_second": 521.917,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_openwebtext_loss": 3.28515625,
      "eval_openwebtext_ppl": 26.713157919740368,
      "eval_openwebtext_runtime": 8.2274,
      "eval_openwebtext_samples_per_second": 521.917,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_wikitext_loss": 3.599609375,
      "eval_wikitext_ppl": 36.58394105020859,
      "eval_wikitext_runtime": 1.7109,
      "eval_wikitext_samples_per_second": 266.52,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_lambada_loss": 4.0,
      "eval_lambada_ppl": 54.598150033144236,
      "eval_lambada_runtime": 9.3078,
      "eval_lambada_samples_per_second": 523.11,
      "step": 7000
    },
    {
      "activations/layer0_attention_weight_max": 24.569847106933594,
      "activations/layer0_attention_weight_min": -17.55438995361328,
      "activations/layer10_attention_weight_max": 46.01118087768555,
      "activations/layer10_attention_weight_min": -39.9393310546875,
      "activations/layer11_attention_weight_max": 46.1292839050293,
      "activations/layer11_attention_weight_min": -39.407249450683594,
      "activations/layer12_attention_weight_max": 25.2066593170166,
      "activations/layer12_attention_weight_min": -24.131057739257812,
      "activations/layer13_attention_weight_max": 30.122882843017578,
      "activations/layer13_attention_weight_min": -28.189266204833984,
      "activations/layer14_attention_weight_max": 36.0665283203125,
      "activations/layer14_attention_weight_min": -26.432422637939453,
      "activations/layer15_attention_weight_max": 33.96763229370117,
      "activations/layer15_attention_weight_min": -19.921274185180664,
      "activations/layer16_attention_weight_max": 24.81553840637207,
      "activations/layer16_attention_weight_min": -18.20781898498535,
      "activations/layer17_attention_weight_max": 19.797685623168945,
      "activations/layer17_attention_weight_min": -15.260578155517578,
      "activations/layer18_attention_weight_max": 17.631677627563477,
      "activations/layer18_attention_weight_min": -12.069060325622559,
      "activations/layer19_attention_weight_max": 21.614233016967773,
      "activations/layer19_attention_weight_min": -14.223889350891113,
      "activations/layer1_attention_weight_max": 19.236366271972656,
      "activations/layer1_attention_weight_min": -15.608458518981934,
      "activations/layer20_attention_weight_max": 25.92934226989746,
      "activations/layer20_attention_weight_min": -15.202288627624512,
      "activations/layer21_attention_weight_max": 19.36501693725586,
      "activations/layer21_attention_weight_min": -11.721282958984375,
      "activations/layer22_attention_weight_max": 20.352249145507812,
      "activations/layer22_attention_weight_min": -13.99797534942627,
      "activations/layer23_attention_weight_max": 17.25193214416504,
      "activations/layer23_attention_weight_min": -15.050276756286621,
      "activations/layer2_attention_weight_max": 22.576519012451172,
      "activations/layer2_attention_weight_min": -20.918094635009766,
      "activations/layer3_attention_weight_max": 37.126853942871094,
      "activations/layer3_attention_weight_min": -47.292640686035156,
      "activations/layer4_attention_weight_max": 92.20278930664062,
      "activations/layer4_attention_weight_min": -109.73695373535156,
      "activations/layer5_attention_weight_max": 125.81748962402344,
      "activations/layer5_attention_weight_min": -130.6796112060547,
      "activations/layer6_attention_weight_max": 80.57603454589844,
      "activations/layer6_attention_weight_min": -82.11132049560547,
      "activations/layer7_attention_weight_max": 75.31913757324219,
      "activations/layer7_attention_weight_min": -71.92461395263672,
      "activations/layer8_attention_weight_max": 68.40623474121094,
      "activations/layer8_attention_weight_min": -64.4417495727539,
      "activations/layer9_attention_weight_max": 67.6310043334961,
      "activations/layer9_attention_weight_min": -60.82419967651367,
      "epoch": 0.41,
      "learning_rate": 0.00014885378787878785,
      "loss": 3.7252,
      "step": 7050
    },
    {
      "activations/layer0_attention_weight_max": 21.797016143798828,
      "activations/layer0_attention_weight_min": -14.234085083007812,
      "activations/layer10_attention_weight_max": 45.9105224609375,
      "activations/layer10_attention_weight_min": -37.219581604003906,
      "activations/layer11_attention_weight_max": 47.63724899291992,
      "activations/layer11_attention_weight_min": -35.0941162109375,
      "activations/layer12_attention_weight_max": 34.109779357910156,
      "activations/layer12_attention_weight_min": -23.144424438476562,
      "activations/layer13_attention_weight_max": 37.87786865234375,
      "activations/layer13_attention_weight_min": -28.169214248657227,
      "activations/layer14_attention_weight_max": 37.65455627441406,
      "activations/layer14_attention_weight_min": -23.179676055908203,
      "activations/layer15_attention_weight_max": 26.213529586791992,
      "activations/layer15_attention_weight_min": -19.424528121948242,
      "activations/layer16_attention_weight_max": 26.175371170043945,
      "activations/layer16_attention_weight_min": -16.849220275878906,
      "activations/layer17_attention_weight_max": 28.690881729125977,
      "activations/layer17_attention_weight_min": -15.072021484375,
      "activations/layer18_attention_weight_max": 21.884885787963867,
      "activations/layer18_attention_weight_min": -11.20666790008545,
      "activations/layer19_attention_weight_max": 25.493566513061523,
      "activations/layer19_attention_weight_min": -13.163606643676758,
      "activations/layer1_attention_weight_max": 15.075898170471191,
      "activations/layer1_attention_weight_min": -10.366802215576172,
      "activations/layer20_attention_weight_max": 24.794221878051758,
      "activations/layer20_attention_weight_min": -10.948007583618164,
      "activations/layer21_attention_weight_max": 22.7528076171875,
      "activations/layer21_attention_weight_min": -10.101629257202148,
      "activations/layer22_attention_weight_max": 24.49630355834961,
      "activations/layer22_attention_weight_min": -12.092009544372559,
      "activations/layer23_attention_weight_max": 19.428129196166992,
      "activations/layer23_attention_weight_min": -11.96249771118164,
      "activations/layer2_attention_weight_max": 24.766576766967773,
      "activations/layer2_attention_weight_min": -17.115087509155273,
      "activations/layer3_attention_weight_max": 42.0656852722168,
      "activations/layer3_attention_weight_min": -38.94327926635742,
      "activations/layer4_attention_weight_max": 102.01494598388672,
      "activations/layer4_attention_weight_min": -96.69493865966797,
      "activations/layer5_attention_weight_max": 126.1703109741211,
      "activations/layer5_attention_weight_min": -114.02059173583984,
      "activations/layer6_attention_weight_max": 79.21427154541016,
      "activations/layer6_attention_weight_min": -77.00637817382812,
      "activations/layer7_attention_weight_max": 78.92709350585938,
      "activations/layer7_attention_weight_min": -71.3357925415039,
      "activations/layer8_attention_weight_max": 66.52076721191406,
      "activations/layer8_attention_weight_min": -55.46780776977539,
      "activations/layer9_attention_weight_max": 66.21623992919922,
      "activations/layer9_attention_weight_min": -61.99686813354492,
      "epoch": 0.41,
      "learning_rate": 0.00014883484848484847,
      "loss": 3.4087,
      "step": 7100
    },
    {
      "activations/layer0_attention_weight_max": 22.137590408325195,
      "activations/layer0_attention_weight_min": -13.397114753723145,
      "activations/layer10_attention_weight_max": 50.9193115234375,
      "activations/layer10_attention_weight_min": -39.49314498901367,
      "activations/layer11_attention_weight_max": 47.38850021362305,
      "activations/layer11_attention_weight_min": -34.31037521362305,
      "activations/layer12_attention_weight_max": 28.08226203918457,
      "activations/layer12_attention_weight_min": -20.818506240844727,
      "activations/layer13_attention_weight_max": 33.97843933105469,
      "activations/layer13_attention_weight_min": -28.572856903076172,
      "activations/layer14_attention_weight_max": 42.659549713134766,
      "activations/layer14_attention_weight_min": -26.971782684326172,
      "activations/layer15_attention_weight_max": 33.48724365234375,
      "activations/layer15_attention_weight_min": -21.490312576293945,
      "activations/layer16_attention_weight_max": 28.52556800842285,
      "activations/layer16_attention_weight_min": -19.20150375366211,
      "activations/layer17_attention_weight_max": 23.656179428100586,
      "activations/layer17_attention_weight_min": -15.716343879699707,
      "activations/layer18_attention_weight_max": 24.99052619934082,
      "activations/layer18_attention_weight_min": -11.478553771972656,
      "activations/layer19_attention_weight_max": 27.740814208984375,
      "activations/layer19_attention_weight_min": -15.281086921691895,
      "activations/layer1_attention_weight_max": 15.39314079284668,
      "activations/layer1_attention_weight_min": -11.087835311889648,
      "activations/layer20_attention_weight_max": 22.578227996826172,
      "activations/layer20_attention_weight_min": -11.38880443572998,
      "activations/layer21_attention_weight_max": 24.082143783569336,
      "activations/layer21_attention_weight_min": -11.307815551757812,
      "activations/layer22_attention_weight_max": 27.37022590637207,
      "activations/layer22_attention_weight_min": -15.690263748168945,
      "activations/layer23_attention_weight_max": 21.31198501586914,
      "activations/layer23_attention_weight_min": -14.661050796508789,
      "activations/layer2_attention_weight_max": 27.22754669189453,
      "activations/layer2_attention_weight_min": -19.9534969329834,
      "activations/layer3_attention_weight_max": 47.12770462036133,
      "activations/layer3_attention_weight_min": -42.87837219238281,
      "activations/layer4_attention_weight_max": 107.71988677978516,
      "activations/layer4_attention_weight_min": -100.54000854492188,
      "activations/layer5_attention_weight_max": 137.3209228515625,
      "activations/layer5_attention_weight_min": -121.95088195800781,
      "activations/layer6_attention_weight_max": 82.14580535888672,
      "activations/layer6_attention_weight_min": -80.21246337890625,
      "activations/layer7_attention_weight_max": 81.4985580444336,
      "activations/layer7_attention_weight_min": -71.2879867553711,
      "activations/layer8_attention_weight_max": 76.00852966308594,
      "activations/layer8_attention_weight_min": -63.60797882080078,
      "activations/layer9_attention_weight_max": 68.00716400146484,
      "activations/layer9_attention_weight_min": -62.0341682434082,
      "epoch": 0.42,
      "learning_rate": 0.00014881590909090908,
      "loss": 3.3555,
      "step": 7150
    },
    {
      "activations/layer0_attention_weight_max": 21.56712532043457,
      "activations/layer0_attention_weight_min": -11.932737350463867,
      "activations/layer10_attention_weight_max": 48.30168151855469,
      "activations/layer10_attention_weight_min": -40.00027084350586,
      "activations/layer11_attention_weight_max": 47.742305755615234,
      "activations/layer11_attention_weight_min": -38.877952575683594,
      "activations/layer12_attention_weight_max": 28.440977096557617,
      "activations/layer12_attention_weight_min": -21.055652618408203,
      "activations/layer13_attention_weight_max": 30.206966400146484,
      "activations/layer13_attention_weight_min": -28.552349090576172,
      "activations/layer14_attention_weight_max": 38.589630126953125,
      "activations/layer14_attention_weight_min": -26.384199142456055,
      "activations/layer15_attention_weight_max": 31.943145751953125,
      "activations/layer15_attention_weight_min": -23.248750686645508,
      "activations/layer16_attention_weight_max": 25.161468505859375,
      "activations/layer16_attention_weight_min": -17.573450088500977,
      "activations/layer17_attention_weight_max": 25.436412811279297,
      "activations/layer17_attention_weight_min": -15.59303092956543,
      "activations/layer18_attention_weight_max": 20.442771911621094,
      "activations/layer18_attention_weight_min": -11.192790031433105,
      "activations/layer19_attention_weight_max": 20.291563034057617,
      "activations/layer19_attention_weight_min": -12.807127952575684,
      "activations/layer1_attention_weight_max": 14.967633247375488,
      "activations/layer1_attention_weight_min": -10.832723617553711,
      "activations/layer20_attention_weight_max": 18.66299819946289,
      "activations/layer20_attention_weight_min": -11.469167709350586,
      "activations/layer21_attention_weight_max": 17.722320556640625,
      "activations/layer21_attention_weight_min": -10.69687271118164,
      "activations/layer22_attention_weight_max": 24.225603103637695,
      "activations/layer22_attention_weight_min": -14.816694259643555,
      "activations/layer23_attention_weight_max": 17.199464797973633,
      "activations/layer23_attention_weight_min": -14.322321891784668,
      "activations/layer2_attention_weight_max": 26.498130798339844,
      "activations/layer2_attention_weight_min": -21.166088104248047,
      "activations/layer3_attention_weight_max": 48.46817398071289,
      "activations/layer3_attention_weight_min": -42.02385711669922,
      "activations/layer4_attention_weight_max": 111.2117691040039,
      "activations/layer4_attention_weight_min": -104.5006103515625,
      "activations/layer5_attention_weight_max": 135.46502685546875,
      "activations/layer5_attention_weight_min": -117.784912109375,
      "activations/layer6_attention_weight_max": 84.21389770507812,
      "activations/layer6_attention_weight_min": -77.70596313476562,
      "activations/layer7_attention_weight_max": 81.81310272216797,
      "activations/layer7_attention_weight_min": -70.42871856689453,
      "activations/layer8_attention_weight_max": 76.58567810058594,
      "activations/layer8_attention_weight_min": -63.89857864379883,
      "activations/layer9_attention_weight_max": 74.39778900146484,
      "activations/layer9_attention_weight_min": -62.639739990234375,
      "epoch": 0.42,
      "learning_rate": 0.00014879696969696967,
      "loss": 3.3442,
      "step": 7200
    },
    {
      "activations/layer0_attention_weight_max": 20.45309066772461,
      "activations/layer0_attention_weight_min": -10.747446060180664,
      "activations/layer10_attention_weight_max": 47.69709396362305,
      "activations/layer10_attention_weight_min": -40.15523910522461,
      "activations/layer11_attention_weight_max": 53.35139465332031,
      "activations/layer11_attention_weight_min": -34.330135345458984,
      "activations/layer12_attention_weight_max": 65.04386901855469,
      "activations/layer12_attention_weight_min": -36.13017654418945,
      "activations/layer13_attention_weight_max": 56.077728271484375,
      "activations/layer13_attention_weight_min": -38.10127258300781,
      "activations/layer14_attention_weight_max": 33.572540283203125,
      "activations/layer14_attention_weight_min": -25.61635971069336,
      "activations/layer15_attention_weight_max": 26.171833038330078,
      "activations/layer15_attention_weight_min": -20.314363479614258,
      "activations/layer16_attention_weight_max": 24.595081329345703,
      "activations/layer16_attention_weight_min": -19.05760383605957,
      "activations/layer17_attention_weight_max": 27.515361785888672,
      "activations/layer17_attention_weight_min": -15.997560501098633,
      "activations/layer18_attention_weight_max": 19.04877281188965,
      "activations/layer18_attention_weight_min": -13.015381813049316,
      "activations/layer19_attention_weight_max": 19.11001205444336,
      "activations/layer19_attention_weight_min": -14.858383178710938,
      "activations/layer1_attention_weight_max": 15.09482192993164,
      "activations/layer1_attention_weight_min": -10.739227294921875,
      "activations/layer20_attention_weight_max": 19.450340270996094,
      "activations/layer20_attention_weight_min": -12.490889549255371,
      "activations/layer21_attention_weight_max": 18.319686889648438,
      "activations/layer21_attention_weight_min": -12.301597595214844,
      "activations/layer22_attention_weight_max": 26.68561553955078,
      "activations/layer22_attention_weight_min": -15.215534210205078,
      "activations/layer23_attention_weight_max": 21.09381103515625,
      "activations/layer23_attention_weight_min": -14.28962230682373,
      "activations/layer2_attention_weight_max": 26.143718719482422,
      "activations/layer2_attention_weight_min": -22.121673583984375,
      "activations/layer3_attention_weight_max": 51.48341369628906,
      "activations/layer3_attention_weight_min": -43.57883071899414,
      "activations/layer4_attention_weight_max": 113.69207000732422,
      "activations/layer4_attention_weight_min": -100.67021942138672,
      "activations/layer5_attention_weight_max": 142.28463745117188,
      "activations/layer5_attention_weight_min": -114.40914154052734,
      "activations/layer6_attention_weight_max": 86.90143585205078,
      "activations/layer6_attention_weight_min": -74.30184173583984,
      "activations/layer7_attention_weight_max": 75.16730499267578,
      "activations/layer7_attention_weight_min": -67.45195007324219,
      "activations/layer8_attention_weight_max": 69.93962860107422,
      "activations/layer8_attention_weight_min": -59.07889938354492,
      "activations/layer9_attention_weight_max": 69.06352996826172,
      "activations/layer9_attention_weight_min": -63.85120391845703,
      "epoch": 0.42,
      "learning_rate": 0.00014877803030303029,
      "loss": 3.3448,
      "step": 7250
    },
    {
      "activations/layer0_attention_weight_max": 20.420297622680664,
      "activations/layer0_attention_weight_min": -10.221412658691406,
      "activations/layer10_attention_weight_max": 45.988563537597656,
      "activations/layer10_attention_weight_min": -40.19328308105469,
      "activations/layer11_attention_weight_max": 48.17023849487305,
      "activations/layer11_attention_weight_min": -36.3907470703125,
      "activations/layer12_attention_weight_max": 39.81471633911133,
      "activations/layer12_attention_weight_min": -25.040430068969727,
      "activations/layer13_attention_weight_max": 32.912166595458984,
      "activations/layer13_attention_weight_min": -29.231477737426758,
      "activations/layer14_attention_weight_max": 40.374691009521484,
      "activations/layer14_attention_weight_min": -27.02840232849121,
      "activations/layer15_attention_weight_max": 33.56804275512695,
      "activations/layer15_attention_weight_min": -24.03666877746582,
      "activations/layer16_attention_weight_max": 30.824024200439453,
      "activations/layer16_attention_weight_min": -22.16942596435547,
      "activations/layer17_attention_weight_max": 36.3542366027832,
      "activations/layer17_attention_weight_min": -17.26243019104004,
      "activations/layer18_attention_weight_max": 22.590667724609375,
      "activations/layer18_attention_weight_min": -14.072507858276367,
      "activations/layer19_attention_weight_max": 24.435583114624023,
      "activations/layer19_attention_weight_min": -17.664648056030273,
      "activations/layer1_attention_weight_max": 17.072336196899414,
      "activations/layer1_attention_weight_min": -11.067806243896484,
      "activations/layer20_attention_weight_max": 23.79833984375,
      "activations/layer20_attention_weight_min": -15.547443389892578,
      "activations/layer21_attention_weight_max": 21.971044540405273,
      "activations/layer21_attention_weight_min": -12.504130363464355,
      "activations/layer22_attention_weight_max": 27.507017135620117,
      "activations/layer22_attention_weight_min": -15.313883781433105,
      "activations/layer23_attention_weight_max": 21.61931610107422,
      "activations/layer23_attention_weight_min": -13.978362083435059,
      "activations/layer2_attention_weight_max": 27.9267635345459,
      "activations/layer2_attention_weight_min": -20.794443130493164,
      "activations/layer3_attention_weight_max": 51.758419036865234,
      "activations/layer3_attention_weight_min": -45.6273078918457,
      "activations/layer4_attention_weight_max": 113.67183685302734,
      "activations/layer4_attention_weight_min": -104.94461822509766,
      "activations/layer5_attention_weight_max": 133.13465881347656,
      "activations/layer5_attention_weight_min": -122.36754608154297,
      "activations/layer6_attention_weight_max": 83.2236099243164,
      "activations/layer6_attention_weight_min": -79.6128158569336,
      "activations/layer7_attention_weight_max": 82.89968872070312,
      "activations/layer7_attention_weight_min": -74.82551574707031,
      "activations/layer8_attention_weight_max": 73.78103637695312,
      "activations/layer8_attention_weight_min": -63.91548156738281,
      "activations/layer9_attention_weight_max": 65.12201690673828,
      "activations/layer9_attention_weight_min": -55.7199821472168,
      "epoch": 0.42,
      "learning_rate": 0.0001487590909090909,
      "loss": 3.34,
      "step": 7300
    },
    {
      "activations/layer0_attention_weight_max": 20.014616012573242,
      "activations/layer0_attention_weight_min": -9.876566886901855,
      "activations/layer10_attention_weight_max": 45.266029357910156,
      "activations/layer10_attention_weight_min": -38.10651397705078,
      "activations/layer11_attention_weight_max": 49.28452682495117,
      "activations/layer11_attention_weight_min": -39.74443054199219,
      "activations/layer12_attention_weight_max": 33.4144172668457,
      "activations/layer12_attention_weight_min": -23.76593017578125,
      "activations/layer13_attention_weight_max": 36.19766616821289,
      "activations/layer13_attention_weight_min": -26.793123245239258,
      "activations/layer14_attention_weight_max": 37.801849365234375,
      "activations/layer14_attention_weight_min": -24.729293823242188,
      "activations/layer15_attention_weight_max": 31.759946823120117,
      "activations/layer15_attention_weight_min": -21.31914520263672,
      "activations/layer16_attention_weight_max": 28.099077224731445,
      "activations/layer16_attention_weight_min": -16.589662551879883,
      "activations/layer17_attention_weight_max": 28.262577056884766,
      "activations/layer17_attention_weight_min": -16.00921058654785,
      "activations/layer18_attention_weight_max": 19.71674346923828,
      "activations/layer18_attention_weight_min": -11.046722412109375,
      "activations/layer19_attention_weight_max": 23.40423583984375,
      "activations/layer19_attention_weight_min": -13.758018493652344,
      "activations/layer1_attention_weight_max": 14.81799030303955,
      "activations/layer1_attention_weight_min": -11.342514038085938,
      "activations/layer20_attention_weight_max": 22.313657760620117,
      "activations/layer20_attention_weight_min": -12.362112998962402,
      "activations/layer21_attention_weight_max": 19.291213989257812,
      "activations/layer21_attention_weight_min": -10.561236381530762,
      "activations/layer22_attention_weight_max": 27.70570182800293,
      "activations/layer22_attention_weight_min": -13.583630561828613,
      "activations/layer23_attention_weight_max": 21.002269744873047,
      "activations/layer23_attention_weight_min": -12.866347312927246,
      "activations/layer2_attention_weight_max": 27.96826171875,
      "activations/layer2_attention_weight_min": -21.125791549682617,
      "activations/layer3_attention_weight_max": 49.70785140991211,
      "activations/layer3_attention_weight_min": -45.571163177490234,
      "activations/layer4_attention_weight_max": 111.520751953125,
      "activations/layer4_attention_weight_min": -107.7171401977539,
      "activations/layer5_attention_weight_max": 136.7815399169922,
      "activations/layer5_attention_weight_min": -119.3094711303711,
      "activations/layer6_attention_weight_max": 83.89199829101562,
      "activations/layer6_attention_weight_min": -74.39623260498047,
      "activations/layer7_attention_weight_max": 83.44425964355469,
      "activations/layer7_attention_weight_min": -68.60004425048828,
      "activations/layer8_attention_weight_max": 75.04888916015625,
      "activations/layer8_attention_weight_min": -61.92795944213867,
      "activations/layer9_attention_weight_max": 70.97499084472656,
      "activations/layer9_attention_weight_min": -63.765228271484375,
      "epoch": 0.43,
      "learning_rate": 0.0001487401515151515,
      "loss": 3.3408,
      "step": 7350
    },
    {
      "activations/layer0_attention_weight_max": 19.95274543762207,
      "activations/layer0_attention_weight_min": -10.101757049560547,
      "activations/layer10_attention_weight_max": 45.5854606628418,
      "activations/layer10_attention_weight_min": -40.232147216796875,
      "activations/layer11_attention_weight_max": 47.76787567138672,
      "activations/layer11_attention_weight_min": -38.399635314941406,
      "activations/layer12_attention_weight_max": 32.17365264892578,
      "activations/layer12_attention_weight_min": -25.914751052856445,
      "activations/layer13_attention_weight_max": 37.0771598815918,
      "activations/layer13_attention_weight_min": -28.434814453125,
      "activations/layer14_attention_weight_max": 39.394290924072266,
      "activations/layer14_attention_weight_min": -25.2557373046875,
      "activations/layer15_attention_weight_max": 32.240108489990234,
      "activations/layer15_attention_weight_min": -21.796672821044922,
      "activations/layer16_attention_weight_max": 31.137950897216797,
      "activations/layer16_attention_weight_min": -18.607452392578125,
      "activations/layer17_attention_weight_max": 26.221790313720703,
      "activations/layer17_attention_weight_min": -14.5252685546875,
      "activations/layer18_attention_weight_max": 21.65900230407715,
      "activations/layer18_attention_weight_min": -11.49379825592041,
      "activations/layer19_attention_weight_max": 23.906490325927734,
      "activations/layer19_attention_weight_min": -13.034205436706543,
      "activations/layer1_attention_weight_max": 13.6730318069458,
      "activations/layer1_attention_weight_min": -11.68980884552002,
      "activations/layer20_attention_weight_max": 21.274221420288086,
      "activations/layer20_attention_weight_min": -12.694767951965332,
      "activations/layer21_attention_weight_max": 20.587434768676758,
      "activations/layer21_attention_weight_min": -10.535429954528809,
      "activations/layer22_attention_weight_max": 27.25078582763672,
      "activations/layer22_attention_weight_min": -13.461181640625,
      "activations/layer23_attention_weight_max": 19.628711700439453,
      "activations/layer23_attention_weight_min": -12.52143669128418,
      "activations/layer2_attention_weight_max": 28.386587142944336,
      "activations/layer2_attention_weight_min": -20.917556762695312,
      "activations/layer3_attention_weight_max": 51.30015563964844,
      "activations/layer3_attention_weight_min": -45.14337158203125,
      "activations/layer4_attention_weight_max": 112.8731460571289,
      "activations/layer4_attention_weight_min": -103.6230239868164,
      "activations/layer5_attention_weight_max": 134.68174743652344,
      "activations/layer5_attention_weight_min": -121.1345443725586,
      "activations/layer6_attention_weight_max": 83.64348602294922,
      "activations/layer6_attention_weight_min": -75.79417419433594,
      "activations/layer7_attention_weight_max": 73.76424407958984,
      "activations/layer7_attention_weight_min": -66.34191131591797,
      "activations/layer8_attention_weight_max": 70.33264923095703,
      "activations/layer8_attention_weight_min": -60.33849334716797,
      "activations/layer9_attention_weight_max": 69.86774444580078,
      "activations/layer9_attention_weight_min": -67.98931121826172,
      "epoch": 0.43,
      "learning_rate": 0.0001487212121212121,
      "loss": 3.3371,
      "step": 7400
    },
    {
      "activations/layer0_attention_weight_max": 19.894287109375,
      "activations/layer0_attention_weight_min": -10.523189544677734,
      "activations/layer10_attention_weight_max": 47.56666946411133,
      "activations/layer10_attention_weight_min": -41.58648681640625,
      "activations/layer11_attention_weight_max": 42.191184997558594,
      "activations/layer11_attention_weight_min": -36.36166763305664,
      "activations/layer12_attention_weight_max": 32.922874450683594,
      "activations/layer12_attention_weight_min": -24.612201690673828,
      "activations/layer13_attention_weight_max": 35.073768615722656,
      "activations/layer13_attention_weight_min": -28.246734619140625,
      "activations/layer14_attention_weight_max": 33.909114837646484,
      "activations/layer14_attention_weight_min": -25.842744827270508,
      "activations/layer15_attention_weight_max": 27.984018325805664,
      "activations/layer15_attention_weight_min": -20.53896713256836,
      "activations/layer16_attention_weight_max": 26.02656364440918,
      "activations/layer16_attention_weight_min": -20.377910614013672,
      "activations/layer17_attention_weight_max": 25.801280975341797,
      "activations/layer17_attention_weight_min": -16.44713592529297,
      "activations/layer18_attention_weight_max": 21.826383590698242,
      "activations/layer18_attention_weight_min": -11.96987247467041,
      "activations/layer19_attention_weight_max": 21.34599494934082,
      "activations/layer19_attention_weight_min": -15.591346740722656,
      "activations/layer1_attention_weight_max": 15.110159873962402,
      "activations/layer1_attention_weight_min": -10.436132431030273,
      "activations/layer20_attention_weight_max": 19.903221130371094,
      "activations/layer20_attention_weight_min": -10.493106842041016,
      "activations/layer21_attention_weight_max": 17.960145950317383,
      "activations/layer21_attention_weight_min": -9.956683158874512,
      "activations/layer22_attention_weight_max": 26.302108764648438,
      "activations/layer22_attention_weight_min": -14.052818298339844,
      "activations/layer23_attention_weight_max": 20.376651763916016,
      "activations/layer23_attention_weight_min": -13.617431640625,
      "activations/layer2_attention_weight_max": 26.70919418334961,
      "activations/layer2_attention_weight_min": -20.18527603149414,
      "activations/layer3_attention_weight_max": 48.76902389526367,
      "activations/layer3_attention_weight_min": -44.133113861083984,
      "activations/layer4_attention_weight_max": 106.8115463256836,
      "activations/layer4_attention_weight_min": -101.7996597290039,
      "activations/layer5_attention_weight_max": 130.9234619140625,
      "activations/layer5_attention_weight_min": -115.87196350097656,
      "activations/layer6_attention_weight_max": 78.43011474609375,
      "activations/layer6_attention_weight_min": -76.46463012695312,
      "activations/layer7_attention_weight_max": 74.94252014160156,
      "activations/layer7_attention_weight_min": -70.29550170898438,
      "activations/layer8_attention_weight_max": 69.56047058105469,
      "activations/layer8_attention_weight_min": -61.23520278930664,
      "activations/layer9_attention_weight_max": 72.23428344726562,
      "activations/layer9_attention_weight_min": -63.094722747802734,
      "epoch": 0.43,
      "learning_rate": 0.00014870227272727272,
      "loss": 3.3146,
      "step": 7450
    },
    {
      "activations/layer0_attention_weight_max": 18.795669555664062,
      "activations/layer0_attention_weight_min": -10.170560836791992,
      "activations/layer10_attention_weight_max": 50.986934661865234,
      "activations/layer10_attention_weight_min": -41.69255447387695,
      "activations/layer11_attention_weight_max": 52.169124603271484,
      "activations/layer11_attention_weight_min": -36.28645324707031,
      "activations/layer12_attention_weight_max": 33.481239318847656,
      "activations/layer12_attention_weight_min": -21.995389938354492,
      "activations/layer13_attention_weight_max": 37.26070022583008,
      "activations/layer13_attention_weight_min": -26.677453994750977,
      "activations/layer14_attention_weight_max": 43.28767776489258,
      "activations/layer14_attention_weight_min": -27.19906997680664,
      "activations/layer15_attention_weight_max": 30.754993438720703,
      "activations/layer15_attention_weight_min": -20.6997127532959,
      "activations/layer16_attention_weight_max": 28.313983917236328,
      "activations/layer16_attention_weight_min": -18.62421417236328,
      "activations/layer17_attention_weight_max": 23.088621139526367,
      "activations/layer17_attention_weight_min": -17.787403106689453,
      "activations/layer18_attention_weight_max": 22.536069869995117,
      "activations/layer18_attention_weight_min": -11.792901039123535,
      "activations/layer19_attention_weight_max": 22.350866317749023,
      "activations/layer19_attention_weight_min": -14.047009468078613,
      "activations/layer1_attention_weight_max": 13.502799987792969,
      "activations/layer1_attention_weight_min": -11.113067626953125,
      "activations/layer20_attention_weight_max": 20.95011329650879,
      "activations/layer20_attention_weight_min": -11.62374210357666,
      "activations/layer21_attention_weight_max": 19.733766555786133,
      "activations/layer21_attention_weight_min": -10.902728080749512,
      "activations/layer22_attention_weight_max": 27.448612213134766,
      "activations/layer22_attention_weight_min": -14.631814002990723,
      "activations/layer23_attention_weight_max": 21.59262466430664,
      "activations/layer23_attention_weight_min": -12.010591506958008,
      "activations/layer2_attention_weight_max": 28.967527389526367,
      "activations/layer2_attention_weight_min": -21.017004013061523,
      "activations/layer3_attention_weight_max": 53.196815490722656,
      "activations/layer3_attention_weight_min": -44.85255813598633,
      "activations/layer4_attention_weight_max": 112.86934661865234,
      "activations/layer4_attention_weight_min": -106.3818588256836,
      "activations/layer5_attention_weight_max": 135.30413818359375,
      "activations/layer5_attention_weight_min": -116.29637145996094,
      "activations/layer6_attention_weight_max": 85.12100982666016,
      "activations/layer6_attention_weight_min": -71.86830139160156,
      "activations/layer7_attention_weight_max": 80.57412719726562,
      "activations/layer7_attention_weight_min": -67.99046325683594,
      "activations/layer8_attention_weight_max": 73.44840240478516,
      "activations/layer8_attention_weight_min": -59.32032012939453,
      "activations/layer9_attention_weight_max": 68.02326202392578,
      "activations/layer9_attention_weight_min": -60.6923713684082,
      "epoch": 0.44,
      "learning_rate": 0.00014868333333333333,
      "loss": 3.3441,
      "step": 7500
    },
    {
      "activations/layer0_attention_weight_max": 19.164722442626953,
      "activations/layer0_attention_weight_min": -10.663191795349121,
      "activations/layer10_attention_weight_max": 56.79271697998047,
      "activations/layer10_attention_weight_min": -46.63570022583008,
      "activations/layer11_attention_weight_max": 51.11687469482422,
      "activations/layer11_attention_weight_min": -42.30693054199219,
      "activations/layer12_attention_weight_max": 27.744651794433594,
      "activations/layer12_attention_weight_min": -19.165048599243164,
      "activations/layer13_attention_weight_max": 33.45184326171875,
      "activations/layer13_attention_weight_min": -28.998979568481445,
      "activations/layer14_attention_weight_max": 38.181583404541016,
      "activations/layer14_attention_weight_min": -30.027334213256836,
      "activations/layer15_attention_weight_max": 34.628631591796875,
      "activations/layer15_attention_weight_min": -20.734251022338867,
      "activations/layer16_attention_weight_max": 25.947229385375977,
      "activations/layer16_attention_weight_min": -17.654569625854492,
      "activations/layer17_attention_weight_max": 26.407058715820312,
      "activations/layer17_attention_weight_min": -16.904644012451172,
      "activations/layer18_attention_weight_max": 19.98389434814453,
      "activations/layer18_attention_weight_min": -12.751245498657227,
      "activations/layer19_attention_weight_max": 20.69744873046875,
      "activations/layer19_attention_weight_min": -15.4181547164917,
      "activations/layer1_attention_weight_max": 14.380629539489746,
      "activations/layer1_attention_weight_min": -10.740633010864258,
      "activations/layer20_attention_weight_max": 20.288087844848633,
      "activations/layer20_attention_weight_min": -13.08802318572998,
      "activations/layer21_attention_weight_max": 19.167688369750977,
      "activations/layer21_attention_weight_min": -10.74121379852295,
      "activations/layer22_attention_weight_max": 24.93218231201172,
      "activations/layer22_attention_weight_min": -14.69489574432373,
      "activations/layer23_attention_weight_max": 17.75100326538086,
      "activations/layer23_attention_weight_min": -14.57519817352295,
      "activations/layer2_attention_weight_max": 28.1435546875,
      "activations/layer2_attention_weight_min": -22.458335876464844,
      "activations/layer3_attention_weight_max": 53.63983154296875,
      "activations/layer3_attention_weight_min": -45.65902328491211,
      "activations/layer4_attention_weight_max": 116.30965423583984,
      "activations/layer4_attention_weight_min": -102.5528335571289,
      "activations/layer5_attention_weight_max": 141.2104034423828,
      "activations/layer5_attention_weight_min": -117.82766723632812,
      "activations/layer6_attention_weight_max": 88.29525756835938,
      "activations/layer6_attention_weight_min": -77.54235076904297,
      "activations/layer7_attention_weight_max": 84.18164825439453,
      "activations/layer7_attention_weight_min": -73.03006744384766,
      "activations/layer8_attention_weight_max": 79.27320098876953,
      "activations/layer8_attention_weight_min": -65.99016571044922,
      "activations/layer9_attention_weight_max": 81.59627532958984,
      "activations/layer9_attention_weight_min": -70.62996673583984,
      "epoch": 0.44,
      "learning_rate": 0.00014866439393939392,
      "loss": 3.3313,
      "step": 7550
    },
    {
      "activations/layer0_attention_weight_max": 18.329484939575195,
      "activations/layer0_attention_weight_min": -10.648942947387695,
      "activations/layer10_attention_weight_max": 46.49045181274414,
      "activations/layer10_attention_weight_min": -39.273536682128906,
      "activations/layer11_attention_weight_max": 49.128448486328125,
      "activations/layer11_attention_weight_min": -39.57803726196289,
      "activations/layer12_attention_weight_max": 29.51395606994629,
      "activations/layer12_attention_weight_min": -21.022520065307617,
      "activations/layer13_attention_weight_max": 30.116125106811523,
      "activations/layer13_attention_weight_min": -28.67998695373535,
      "activations/layer14_attention_weight_max": 37.046695709228516,
      "activations/layer14_attention_weight_min": -30.600229263305664,
      "activations/layer15_attention_weight_max": 26.162385940551758,
      "activations/layer15_attention_weight_min": -21.62603187561035,
      "activations/layer16_attention_weight_max": 26.46372413635254,
      "activations/layer16_attention_weight_min": -19.532066345214844,
      "activations/layer17_attention_weight_max": 29.151235580444336,
      "activations/layer17_attention_weight_min": -18.259626388549805,
      "activations/layer18_attention_weight_max": 21.09843635559082,
      "activations/layer18_attention_weight_min": -13.32590103149414,
      "activations/layer19_attention_weight_max": 21.040611267089844,
      "activations/layer19_attention_weight_min": -15.229580879211426,
      "activations/layer1_attention_weight_max": 14.118566513061523,
      "activations/layer1_attention_weight_min": -11.18160343170166,
      "activations/layer20_attention_weight_max": 19.973661422729492,
      "activations/layer20_attention_weight_min": -11.3422269821167,
      "activations/layer21_attention_weight_max": 19.38663101196289,
      "activations/layer21_attention_weight_min": -11.646286010742188,
      "activations/layer22_attention_weight_max": 29.70096206665039,
      "activations/layer22_attention_weight_min": -14.734858512878418,
      "activations/layer23_attention_weight_max": 21.93813133239746,
      "activations/layer23_attention_weight_min": -13.030454635620117,
      "activations/layer2_attention_weight_max": 27.88746452331543,
      "activations/layer2_attention_weight_min": -22.94612693786621,
      "activations/layer3_attention_weight_max": 51.39104461669922,
      "activations/layer3_attention_weight_min": -46.69336700439453,
      "activations/layer4_attention_weight_max": 111.6502456665039,
      "activations/layer4_attention_weight_min": -105.83150482177734,
      "activations/layer5_attention_weight_max": 131.29507446289062,
      "activations/layer5_attention_weight_min": -118.79544067382812,
      "activations/layer6_attention_weight_max": 78.50180053710938,
      "activations/layer6_attention_weight_min": -75.17973327636719,
      "activations/layer7_attention_weight_max": 85.56224060058594,
      "activations/layer7_attention_weight_min": -74.31443786621094,
      "activations/layer8_attention_weight_max": 70.74628448486328,
      "activations/layer8_attention_weight_min": -60.15433883666992,
      "activations/layer9_attention_weight_max": 75.4871597290039,
      "activations/layer9_attention_weight_min": -66.66453552246094,
      "epoch": 0.44,
      "learning_rate": 0.00014864545454545454,
      "loss": 3.3307,
      "step": 7600
    },
    {
      "activations/layer0_attention_weight_max": 18.238462448120117,
      "activations/layer0_attention_weight_min": -11.015775680541992,
      "activations/layer10_attention_weight_max": 45.3960075378418,
      "activations/layer10_attention_weight_min": -43.89152145385742,
      "activations/layer11_attention_weight_max": 52.51371765136719,
      "activations/layer11_attention_weight_min": -44.92680358886719,
      "activations/layer12_attention_weight_max": 25.06533432006836,
      "activations/layer12_attention_weight_min": -24.151853561401367,
      "activations/layer13_attention_weight_max": 36.405181884765625,
      "activations/layer13_attention_weight_min": -32.38133239746094,
      "activations/layer14_attention_weight_max": 36.27407455444336,
      "activations/layer14_attention_weight_min": -30.63009262084961,
      "activations/layer15_attention_weight_max": 34.82554244995117,
      "activations/layer15_attention_weight_min": -26.752426147460938,
      "activations/layer16_attention_weight_max": 23.071443557739258,
      "activations/layer16_attention_weight_min": -17.71826934814453,
      "activations/layer17_attention_weight_max": 21.99469757080078,
      "activations/layer17_attention_weight_min": -16.179391860961914,
      "activations/layer18_attention_weight_max": 21.3659610748291,
      "activations/layer18_attention_weight_min": -12.2274751663208,
      "activations/layer19_attention_weight_max": 21.919811248779297,
      "activations/layer19_attention_weight_min": -15.153526306152344,
      "activations/layer1_attention_weight_max": 14.327619552612305,
      "activations/layer1_attention_weight_min": -10.762693405151367,
      "activations/layer20_attention_weight_max": 19.727811813354492,
      "activations/layer20_attention_weight_min": -11.931142807006836,
      "activations/layer21_attention_weight_max": 21.94805335998535,
      "activations/layer21_attention_weight_min": -11.074753761291504,
      "activations/layer22_attention_weight_max": 25.97402572631836,
      "activations/layer22_attention_weight_min": -13.376898765563965,
      "activations/layer23_attention_weight_max": 20.370582580566406,
      "activations/layer23_attention_weight_min": -14.246084213256836,
      "activations/layer2_attention_weight_max": 28.584081649780273,
      "activations/layer2_attention_weight_min": -22.809886932373047,
      "activations/layer3_attention_weight_max": 52.632057189941406,
      "activations/layer3_attention_weight_min": -47.37704086303711,
      "activations/layer4_attention_weight_max": 113.28466033935547,
      "activations/layer4_attention_weight_min": -106.5830307006836,
      "activations/layer5_attention_weight_max": 133.0447540283203,
      "activations/layer5_attention_weight_min": -117.93376159667969,
      "activations/layer6_attention_weight_max": 79.63370513916016,
      "activations/layer6_attention_weight_min": -76.40687561035156,
      "activations/layer7_attention_weight_max": 73.31700134277344,
      "activations/layer7_attention_weight_min": -69.26006317138672,
      "activations/layer8_attention_weight_max": 71.43711853027344,
      "activations/layer8_attention_weight_min": -60.25858688354492,
      "activations/layer9_attention_weight_max": 69.52726745605469,
      "activations/layer9_attention_weight_min": -65.79314422607422,
      "epoch": 0.44,
      "learning_rate": 0.00014862651515151515,
      "loss": 3.323,
      "step": 7650
    },
    {
      "activations/layer0_attention_weight_max": 18.036672592163086,
      "activations/layer0_attention_weight_min": -10.770758628845215,
      "activations/layer10_attention_weight_max": 48.682098388671875,
      "activations/layer10_attention_weight_min": -39.48671340942383,
      "activations/layer11_attention_weight_max": 47.317989349365234,
      "activations/layer11_attention_weight_min": -38.53015899658203,
      "activations/layer12_attention_weight_max": 27.165523529052734,
      "activations/layer12_attention_weight_min": -24.07044792175293,
      "activations/layer13_attention_weight_max": 36.38661575317383,
      "activations/layer13_attention_weight_min": -27.97127914428711,
      "activations/layer14_attention_weight_max": 38.14226531982422,
      "activations/layer14_attention_weight_min": -28.60127830505371,
      "activations/layer15_attention_weight_max": 28.156452178955078,
      "activations/layer15_attention_weight_min": -23.251873016357422,
      "activations/layer16_attention_weight_max": 24.884246826171875,
      "activations/layer16_attention_weight_min": -21.137651443481445,
      "activations/layer17_attention_weight_max": 25.921117782592773,
      "activations/layer17_attention_weight_min": -18.634790420532227,
      "activations/layer18_attention_weight_max": 20.914506912231445,
      "activations/layer18_attention_weight_min": -13.89884090423584,
      "activations/layer19_attention_weight_max": 23.27447509765625,
      "activations/layer19_attention_weight_min": -19.991422653198242,
      "activations/layer1_attention_weight_max": 14.511022567749023,
      "activations/layer1_attention_weight_min": -11.115872383117676,
      "activations/layer20_attention_weight_max": 22.122100830078125,
      "activations/layer20_attention_weight_min": -13.524812698364258,
      "activations/layer21_attention_weight_max": 24.145572662353516,
      "activations/layer21_attention_weight_min": -13.816191673278809,
      "activations/layer22_attention_weight_max": 27.562824249267578,
      "activations/layer22_attention_weight_min": -18.416589736938477,
      "activations/layer23_attention_weight_max": 19.939979553222656,
      "activations/layer23_attention_weight_min": -16.739992141723633,
      "activations/layer2_attention_weight_max": 29.75067138671875,
      "activations/layer2_attention_weight_min": -23.388282775878906,
      "activations/layer3_attention_weight_max": 57.66431427001953,
      "activations/layer3_attention_weight_min": -47.3216438293457,
      "activations/layer4_attention_weight_max": 119.37798309326172,
      "activations/layer4_attention_weight_min": -104.46414947509766,
      "activations/layer5_attention_weight_max": 138.19810485839844,
      "activations/layer5_attention_weight_min": -118.31570434570312,
      "activations/layer6_attention_weight_max": 84.18465423583984,
      "activations/layer6_attention_weight_min": -73.67408752441406,
      "activations/layer7_attention_weight_max": 83.78522491455078,
      "activations/layer7_attention_weight_min": -67.3022689819336,
      "activations/layer8_attention_weight_max": 71.83161163330078,
      "activations/layer8_attention_weight_min": -61.53615951538086,
      "activations/layer9_attention_weight_max": 70.37786102294922,
      "activations/layer9_attention_weight_min": -58.2147102355957,
      "epoch": 0.45,
      "learning_rate": 0.00014860757575757574,
      "loss": 3.3116,
      "step": 7700
    },
    {
      "activations/layer0_attention_weight_max": 17.806354522705078,
      "activations/layer0_attention_weight_min": -10.664787292480469,
      "activations/layer10_attention_weight_max": 47.49104690551758,
      "activations/layer10_attention_weight_min": -40.94108200073242,
      "activations/layer11_attention_weight_max": 56.03424835205078,
      "activations/layer11_attention_weight_min": -42.84942626953125,
      "activations/layer12_attention_weight_max": 33.18366241455078,
      "activations/layer12_attention_weight_min": -22.14562225341797,
      "activations/layer13_attention_weight_max": 38.4710807800293,
      "activations/layer13_attention_weight_min": -28.346839904785156,
      "activations/layer14_attention_weight_max": 48.49369430541992,
      "activations/layer14_attention_weight_min": -31.5985107421875,
      "activations/layer15_attention_weight_max": 34.03608703613281,
      "activations/layer15_attention_weight_min": -20.00352668762207,
      "activations/layer16_attention_weight_max": 27.688671112060547,
      "activations/layer16_attention_weight_min": -19.998920440673828,
      "activations/layer17_attention_weight_max": 28.838497161865234,
      "activations/layer17_attention_weight_min": -16.391036987304688,
      "activations/layer18_attention_weight_max": 23.692596435546875,
      "activations/layer18_attention_weight_min": -12.136810302734375,
      "activations/layer19_attention_weight_max": 23.984981536865234,
      "activations/layer19_attention_weight_min": -12.26684856414795,
      "activations/layer1_attention_weight_max": 13.939997673034668,
      "activations/layer1_attention_weight_min": -10.862593650817871,
      "activations/layer20_attention_weight_max": 22.020824432373047,
      "activations/layer20_attention_weight_min": -10.479246139526367,
      "activations/layer21_attention_weight_max": 21.497940063476562,
      "activations/layer21_attention_weight_min": -9.629033088684082,
      "activations/layer22_attention_weight_max": 36.618629455566406,
      "activations/layer22_attention_weight_min": -14.999429702758789,
      "activations/layer23_attention_weight_max": 24.835355758666992,
      "activations/layer23_attention_weight_min": -11.933540344238281,
      "activations/layer2_attention_weight_max": 29.287445068359375,
      "activations/layer2_attention_weight_min": -22.90963363647461,
      "activations/layer3_attention_weight_max": 54.423057556152344,
      "activations/layer3_attention_weight_min": -50.316219329833984,
      "activations/layer4_attention_weight_max": 119.30265045166016,
      "activations/layer4_attention_weight_min": -108.64302062988281,
      "activations/layer5_attention_weight_max": 134.55780029296875,
      "activations/layer5_attention_weight_min": -120.22767639160156,
      "activations/layer6_attention_weight_max": 87.19939422607422,
      "activations/layer6_attention_weight_min": -75.57606506347656,
      "activations/layer7_attention_weight_max": 79.21827697753906,
      "activations/layer7_attention_weight_min": -67.56871795654297,
      "activations/layer8_attention_weight_max": 66.62384033203125,
      "activations/layer8_attention_weight_min": -57.30879211425781,
      "activations/layer9_attention_weight_max": 72.16769409179688,
      "activations/layer9_attention_weight_min": -61.3246955871582,
      "epoch": 0.45,
      "learning_rate": 0.00014858863636363635,
      "loss": 3.2854,
      "step": 7750
    },
    {
      "activations/layer0_attention_weight_max": 18.04587745666504,
      "activations/layer0_attention_weight_min": -10.396954536437988,
      "activations/layer10_attention_weight_max": 49.79570007324219,
      "activations/layer10_attention_weight_min": -43.559776306152344,
      "activations/layer11_attention_weight_max": 48.10106658935547,
      "activations/layer11_attention_weight_min": -36.72492218017578,
      "activations/layer12_attention_weight_max": 32.09246063232422,
      "activations/layer12_attention_weight_min": -24.482397079467773,
      "activations/layer13_attention_weight_max": 35.559715270996094,
      "activations/layer13_attention_weight_min": -27.7232666015625,
      "activations/layer14_attention_weight_max": 42.919822692871094,
      "activations/layer14_attention_weight_min": -28.808528900146484,
      "activations/layer15_attention_weight_max": 25.87476348876953,
      "activations/layer15_attention_weight_min": -21.510372161865234,
      "activations/layer16_attention_weight_max": 22.38664436340332,
      "activations/layer16_attention_weight_min": -15.259580612182617,
      "activations/layer17_attention_weight_max": 29.597408294677734,
      "activations/layer17_attention_weight_min": -14.622384071350098,
      "activations/layer18_attention_weight_max": 21.508399963378906,
      "activations/layer18_attention_weight_min": -12.38965129852295,
      "activations/layer19_attention_weight_max": 19.8691463470459,
      "activations/layer19_attention_weight_min": -13.345117568969727,
      "activations/layer1_attention_weight_max": 13.403462409973145,
      "activations/layer1_attention_weight_min": -10.472458839416504,
      "activations/layer20_attention_weight_max": 19.04914093017578,
      "activations/layer20_attention_weight_min": -12.383349418640137,
      "activations/layer21_attention_weight_max": 19.834421157836914,
      "activations/layer21_attention_weight_min": -11.05349063873291,
      "activations/layer22_attention_weight_max": 21.42236328125,
      "activations/layer22_attention_weight_min": -14.002135276794434,
      "activations/layer23_attention_weight_max": 15.567572593688965,
      "activations/layer23_attention_weight_min": -13.565902709960938,
      "activations/layer2_attention_weight_max": 28.056564331054688,
      "activations/layer2_attention_weight_min": -23.43564224243164,
      "activations/layer3_attention_weight_max": 54.033843994140625,
      "activations/layer3_attention_weight_min": -49.43125915527344,
      "activations/layer4_attention_weight_max": 117.84464263916016,
      "activations/layer4_attention_weight_min": -107.43730926513672,
      "activations/layer5_attention_weight_max": 137.03872680664062,
      "activations/layer5_attention_weight_min": -119.64775848388672,
      "activations/layer6_attention_weight_max": 82.0958251953125,
      "activations/layer6_attention_weight_min": -76.99285888671875,
      "activations/layer7_attention_weight_max": 77.0962142944336,
      "activations/layer7_attention_weight_min": -67.98688507080078,
      "activations/layer8_attention_weight_max": 65.86931610107422,
      "activations/layer8_attention_weight_min": -61.12232971191406,
      "activations/layer9_attention_weight_max": 69.94232940673828,
      "activations/layer9_attention_weight_min": -56.50104522705078,
      "epoch": 0.45,
      "learning_rate": 0.00014856969696969697,
      "loss": 3.3015,
      "step": 7800
    },
    {
      "activations/layer0_attention_weight_max": 18.241455078125,
      "activations/layer0_attention_weight_min": -10.594977378845215,
      "activations/layer10_attention_weight_max": 48.303855895996094,
      "activations/layer10_attention_weight_min": -39.49738693237305,
      "activations/layer11_attention_weight_max": 47.096763610839844,
      "activations/layer11_attention_weight_min": -39.34376907348633,
      "activations/layer12_attention_weight_max": 34.75457000732422,
      "activations/layer12_attention_weight_min": -25.42852210998535,
      "activations/layer13_attention_weight_max": 39.57911682128906,
      "activations/layer13_attention_weight_min": -31.69301414489746,
      "activations/layer14_attention_weight_max": 38.68015670776367,
      "activations/layer14_attention_weight_min": -30.620662689208984,
      "activations/layer15_attention_weight_max": 35.88248825073242,
      "activations/layer15_attention_weight_min": -24.561874389648438,
      "activations/layer16_attention_weight_max": 29.984365463256836,
      "activations/layer16_attention_weight_min": -19.29155158996582,
      "activations/layer17_attention_weight_max": 25.414995193481445,
      "activations/layer17_attention_weight_min": -14.732969284057617,
      "activations/layer18_attention_weight_max": 20.765668869018555,
      "activations/layer18_attention_weight_min": -10.452127456665039,
      "activations/layer19_attention_weight_max": 24.754871368408203,
      "activations/layer19_attention_weight_min": -13.218855857849121,
      "activations/layer1_attention_weight_max": 13.392534255981445,
      "activations/layer1_attention_weight_min": -12.278932571411133,
      "activations/layer20_attention_weight_max": 20.240345001220703,
      "activations/layer20_attention_weight_min": -13.195394515991211,
      "activations/layer21_attention_weight_max": 19.053726196289062,
      "activations/layer21_attention_weight_min": -11.362454414367676,
      "activations/layer22_attention_weight_max": 27.810083389282227,
      "activations/layer22_attention_weight_min": -12.696671485900879,
      "activations/layer23_attention_weight_max": 18.769954681396484,
      "activations/layer23_attention_weight_min": -11.826029777526855,
      "activations/layer2_attention_weight_max": 29.413768768310547,
      "activations/layer2_attention_weight_min": -23.201345443725586,
      "activations/layer3_attention_weight_max": 53.65522384643555,
      "activations/layer3_attention_weight_min": -49.03512191772461,
      "activations/layer4_attention_weight_max": 118.9449234008789,
      "activations/layer4_attention_weight_min": -104.787109375,
      "activations/layer5_attention_weight_max": 135.40792846679688,
      "activations/layer5_attention_weight_min": -118.66057586669922,
      "activations/layer6_attention_weight_max": 82.3836669921875,
      "activations/layer6_attention_weight_min": -73.9310302734375,
      "activations/layer7_attention_weight_max": 75.69729614257812,
      "activations/layer7_attention_weight_min": -65.95137023925781,
      "activations/layer8_attention_weight_max": 72.60777282714844,
      "activations/layer8_attention_weight_min": -61.44003677368164,
      "activations/layer9_attention_weight_max": 68.56550598144531,
      "activations/layer9_attention_weight_min": -68.54065704345703,
      "epoch": 0.46,
      "learning_rate": 0.00014855075757575756,
      "loss": 3.3028,
      "step": 7850
    },
    {
      "activations/layer0_attention_weight_max": 18.448453903198242,
      "activations/layer0_attention_weight_min": -10.258255004882812,
      "activations/layer10_attention_weight_max": 47.545440673828125,
      "activations/layer10_attention_weight_min": -38.195594787597656,
      "activations/layer11_attention_weight_max": 48.674217224121094,
      "activations/layer11_attention_weight_min": -36.232261657714844,
      "activations/layer12_attention_weight_max": 30.962757110595703,
      "activations/layer12_attention_weight_min": -23.79969024658203,
      "activations/layer13_attention_weight_max": 32.77875518798828,
      "activations/layer13_attention_weight_min": -30.30852508544922,
      "activations/layer14_attention_weight_max": 36.77329635620117,
      "activations/layer14_attention_weight_min": -29.143720626831055,
      "activations/layer15_attention_weight_max": 25.176015853881836,
      "activations/layer15_attention_weight_min": -20.479127883911133,
      "activations/layer16_attention_weight_max": 20.436805725097656,
      "activations/layer16_attention_weight_min": -16.39870834350586,
      "activations/layer17_attention_weight_max": 23.44902992248535,
      "activations/layer17_attention_weight_min": -15.786409378051758,
      "activations/layer18_attention_weight_max": 19.073421478271484,
      "activations/layer18_attention_weight_min": -10.853896141052246,
      "activations/layer19_attention_weight_max": 19.011083602905273,
      "activations/layer19_attention_weight_min": -13.494847297668457,
      "activations/layer1_attention_weight_max": 14.125535011291504,
      "activations/layer1_attention_weight_min": -11.634466171264648,
      "activations/layer20_attention_weight_max": 16.90700340270996,
      "activations/layer20_attention_weight_min": -10.604315757751465,
      "activations/layer21_attention_weight_max": 15.59359359741211,
      "activations/layer21_attention_weight_min": -10.4840726852417,
      "activations/layer22_attention_weight_max": 27.432451248168945,
      "activations/layer22_attention_weight_min": -12.517098426818848,
      "activations/layer23_attention_weight_max": 18.776264190673828,
      "activations/layer23_attention_weight_min": -11.559926986694336,
      "activations/layer2_attention_weight_max": 29.637571334838867,
      "activations/layer2_attention_weight_min": -22.929153442382812,
      "activations/layer3_attention_weight_max": 55.40475845336914,
      "activations/layer3_attention_weight_min": -49.204490661621094,
      "activations/layer4_attention_weight_max": 115.9959945678711,
      "activations/layer4_attention_weight_min": -107.17207336425781,
      "activations/layer5_attention_weight_max": 136.68780517578125,
      "activations/layer5_attention_weight_min": -118.88710021972656,
      "activations/layer6_attention_weight_max": 86.40254211425781,
      "activations/layer6_attention_weight_min": -72.032470703125,
      "activations/layer7_attention_weight_max": 77.3353042602539,
      "activations/layer7_attention_weight_min": -66.33643341064453,
      "activations/layer8_attention_weight_max": 73.2704849243164,
      "activations/layer8_attention_weight_min": -62.07357406616211,
      "activations/layer9_attention_weight_max": 67.14689636230469,
      "activations/layer9_attention_weight_min": -60.004512786865234,
      "epoch": 0.46,
      "learning_rate": 0.00014853181818181817,
      "loss": 3.2821,
      "step": 7900
    },
    {
      "activations/layer0_attention_weight_max": 17.653179168701172,
      "activations/layer0_attention_weight_min": -10.646350860595703,
      "activations/layer10_attention_weight_max": 47.816612243652344,
      "activations/layer10_attention_weight_min": -40.99412536621094,
      "activations/layer11_attention_weight_max": 45.20757293701172,
      "activations/layer11_attention_weight_min": -39.23648452758789,
      "activations/layer12_attention_weight_max": 29.95989227294922,
      "activations/layer12_attention_weight_min": -25.634138107299805,
      "activations/layer13_attention_weight_max": 31.980628967285156,
      "activations/layer13_attention_weight_min": -31.05162239074707,
      "activations/layer14_attention_weight_max": 40.71900177001953,
      "activations/layer14_attention_weight_min": -31.559354782104492,
      "activations/layer15_attention_weight_max": 28.59014320373535,
      "activations/layer15_attention_weight_min": -23.221952438354492,
      "activations/layer16_attention_weight_max": 27.93861198425293,
      "activations/layer16_attention_weight_min": -24.585596084594727,
      "activations/layer17_attention_weight_max": 31.679012298583984,
      "activations/layer17_attention_weight_min": -19.13531494140625,
      "activations/layer18_attention_weight_max": 20.739351272583008,
      "activations/layer18_attention_weight_min": -13.203712463378906,
      "activations/layer19_attention_weight_max": 27.3107852935791,
      "activations/layer19_attention_weight_min": -13.702540397644043,
      "activations/layer1_attention_weight_max": 14.928832054138184,
      "activations/layer1_attention_weight_min": -11.304636001586914,
      "activations/layer20_attention_weight_max": 20.582685470581055,
      "activations/layer20_attention_weight_min": -11.276083946228027,
      "activations/layer21_attention_weight_max": 21.009733200073242,
      "activations/layer21_attention_weight_min": -10.738879203796387,
      "activations/layer22_attention_weight_max": 28.12423324584961,
      "activations/layer22_attention_weight_min": -14.92676830291748,
      "activations/layer23_attention_weight_max": 19.516674041748047,
      "activations/layer23_attention_weight_min": -11.732924461364746,
      "activations/layer2_attention_weight_max": 27.596988677978516,
      "activations/layer2_attention_weight_min": -22.36530113220215,
      "activations/layer3_attention_weight_max": 54.73160934448242,
      "activations/layer3_attention_weight_min": -50.529563903808594,
      "activations/layer4_attention_weight_max": 118.0310287475586,
      "activations/layer4_attention_weight_min": -109.2413330078125,
      "activations/layer5_attention_weight_max": 136.41795349121094,
      "activations/layer5_attention_weight_min": -123.44686889648438,
      "activations/layer6_attention_weight_max": 82.40351867675781,
      "activations/layer6_attention_weight_min": -74.31224060058594,
      "activations/layer7_attention_weight_max": 76.58425903320312,
      "activations/layer7_attention_weight_min": -67.02169036865234,
      "activations/layer8_attention_weight_max": 72.93517303466797,
      "activations/layer8_attention_weight_min": -65.20377349853516,
      "activations/layer9_attention_weight_max": 65.98316192626953,
      "activations/layer9_attention_weight_min": -63.61124801635742,
      "epoch": 0.46,
      "learning_rate": 0.0001485128787878788,
      "loss": 3.2745,
      "step": 7950
    },
    {
      "activations/layer0_attention_weight_max": 18.271690368652344,
      "activations/layer0_attention_weight_min": -11.20278549194336,
      "activations/layer10_attention_weight_max": 51.3472785949707,
      "activations/layer10_attention_weight_min": -42.29838943481445,
      "activations/layer11_attention_weight_max": 47.11750030517578,
      "activations/layer11_attention_weight_min": -36.42485046386719,
      "activations/layer12_attention_weight_max": 29.260622024536133,
      "activations/layer12_attention_weight_min": -22.383447647094727,
      "activations/layer13_attention_weight_max": 36.89142990112305,
      "activations/layer13_attention_weight_min": -33.90890884399414,
      "activations/layer14_attention_weight_max": 45.63957214355469,
      "activations/layer14_attention_weight_min": -32.60856246948242,
      "activations/layer15_attention_weight_max": 28.548067092895508,
      "activations/layer15_attention_weight_min": -21.86320686340332,
      "activations/layer16_attention_weight_max": 25.789705276489258,
      "activations/layer16_attention_weight_min": -20.631086349487305,
      "activations/layer17_attention_weight_max": 30.07423210144043,
      "activations/layer17_attention_weight_min": -16.537628173828125,
      "activations/layer18_attention_weight_max": 21.85380744934082,
      "activations/layer18_attention_weight_min": -11.555342674255371,
      "activations/layer19_attention_weight_max": 24.46132469177246,
      "activations/layer19_attention_weight_min": -14.917540550231934,
      "activations/layer1_attention_weight_max": 13.193944931030273,
      "activations/layer1_attention_weight_min": -10.872289657592773,
      "activations/layer20_attention_weight_max": 21.647480010986328,
      "activations/layer20_attention_weight_min": -11.997358322143555,
      "activations/layer21_attention_weight_max": 20.01397132873535,
      "activations/layer21_attention_weight_min": -11.29669189453125,
      "activations/layer22_attention_weight_max": 27.55838394165039,
      "activations/layer22_attention_weight_min": -15.50094985961914,
      "activations/layer23_attention_weight_max": 19.384193420410156,
      "activations/layer23_attention_weight_min": -14.455682754516602,
      "activations/layer2_attention_weight_max": 28.861133575439453,
      "activations/layer2_attention_weight_min": -23.3525390625,
      "activations/layer3_attention_weight_max": 53.60916519165039,
      "activations/layer3_attention_weight_min": -48.64454650878906,
      "activations/layer4_attention_weight_max": 117.64286041259766,
      "activations/layer4_attention_weight_min": -104.8873062133789,
      "activations/layer5_attention_weight_max": 136.07662963867188,
      "activations/layer5_attention_weight_min": -117.29869079589844,
      "activations/layer6_attention_weight_max": 82.52732849121094,
      "activations/layer6_attention_weight_min": -73.8023452758789,
      "activations/layer7_attention_weight_max": 82.24131774902344,
      "activations/layer7_attention_weight_min": -67.75171661376953,
      "activations/layer8_attention_weight_max": 78.29024505615234,
      "activations/layer8_attention_weight_min": -62.7728157043457,
      "activations/layer9_attention_weight_max": 67.58123016357422,
      "activations/layer9_attention_weight_min": -58.68758773803711,
      "epoch": 0.46,
      "learning_rate": 0.00014849393939393938,
      "loss": 3.2777,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_loss": 3.224609375,
      "eval_runtime": 8.4881,
      "eval_samples_per_second": 505.886,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_openwebtext_loss": 3.224609375,
      "eval_openwebtext_ppl": 25.1437504597293,
      "eval_openwebtext_runtime": 8.4881,
      "eval_openwebtext_samples_per_second": 505.886,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_wikitext_loss": 3.53125,
      "eval_wikitext_ppl": 34.16664924473404,
      "eval_wikitext_runtime": 1.9482,
      "eval_wikitext_samples_per_second": 234.066,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_lambada_loss": 3.970703125,
      "eval_lambada_ppl": 53.02179868836547,
      "eval_lambada_runtime": 9.5243,
      "eval_lambada_samples_per_second": 511.221,
      "step": 8000
    },
    {
      "activations/layer0_attention_weight_max": 17.514162063598633,
      "activations/layer0_attention_weight_min": -11.306244850158691,
      "activations/layer10_attention_weight_max": 50.1380729675293,
      "activations/layer10_attention_weight_min": -39.797935485839844,
      "activations/layer11_attention_weight_max": 46.115318298339844,
      "activations/layer11_attention_weight_min": -39.444820404052734,
      "activations/layer12_attention_weight_max": 31.275880813598633,
      "activations/layer12_attention_weight_min": -24.326093673706055,
      "activations/layer13_attention_weight_max": 35.301551818847656,
      "activations/layer13_attention_weight_min": -29.4908504486084,
      "activations/layer14_attention_weight_max": 43.051536560058594,
      "activations/layer14_attention_weight_min": -30.7446346282959,
      "activations/layer15_attention_weight_max": 29.952909469604492,
      "activations/layer15_attention_weight_min": -20.66267204284668,
      "activations/layer16_attention_weight_max": 26.0927677154541,
      "activations/layer16_attention_weight_min": -17.835439682006836,
      "activations/layer17_attention_weight_max": 23.06886100769043,
      "activations/layer17_attention_weight_min": -16.4232120513916,
      "activations/layer18_attention_weight_max": 17.34871482849121,
      "activations/layer18_attention_weight_min": -10.871199607849121,
      "activations/layer19_attention_weight_max": 22.16329002380371,
      "activations/layer19_attention_weight_min": -12.861292839050293,
      "activations/layer1_attention_weight_max": 15.405068397521973,
      "activations/layer1_attention_weight_min": -10.654914855957031,
      "activations/layer20_attention_weight_max": 20.796567916870117,
      "activations/layer20_attention_weight_min": -11.841707229614258,
      "activations/layer21_attention_weight_max": 20.827939987182617,
      "activations/layer21_attention_weight_min": -10.768961906433105,
      "activations/layer22_attention_weight_max": 29.211091995239258,
      "activations/layer22_attention_weight_min": -13.979575157165527,
      "activations/layer23_attention_weight_max": 19.750333786010742,
      "activations/layer23_attention_weight_min": -12.983156204223633,
      "activations/layer2_attention_weight_max": 28.798397064208984,
      "activations/layer2_attention_weight_min": -22.511159896850586,
      "activations/layer3_attention_weight_max": 55.21575164794922,
      "activations/layer3_attention_weight_min": -49.32320785522461,
      "activations/layer4_attention_weight_max": 116.92706298828125,
      "activations/layer4_attention_weight_min": -108.19050598144531,
      "activations/layer5_attention_weight_max": 135.5465545654297,
      "activations/layer5_attention_weight_min": -119.86436462402344,
      "activations/layer6_attention_weight_max": 82.28057098388672,
      "activations/layer6_attention_weight_min": -73.6426773071289,
      "activations/layer7_attention_weight_max": 81.07526397705078,
      "activations/layer7_attention_weight_min": -66.59574127197266,
      "activations/layer8_attention_weight_max": 71.99152374267578,
      "activations/layer8_attention_weight_min": -61.76108932495117,
      "activations/layer9_attention_weight_max": 63.69462966918945,
      "activations/layer9_attention_weight_min": -59.02462387084961,
      "epoch": 0.47,
      "learning_rate": 0.000148475,
      "loss": 3.2661,
      "step": 8050
    },
    {
      "activations/layer0_attention_weight_max": 17.54691505432129,
      "activations/layer0_attention_weight_min": -10.62401008605957,
      "activations/layer10_attention_weight_max": 49.79337692260742,
      "activations/layer10_attention_weight_min": -43.96285629272461,
      "activations/layer11_attention_weight_max": 52.954219818115234,
      "activations/layer11_attention_weight_min": -42.49435806274414,
      "activations/layer12_attention_weight_max": 27.782089233398438,
      "activations/layer12_attention_weight_min": -24.007320404052734,
      "activations/layer13_attention_weight_max": 33.82783126831055,
      "activations/layer13_attention_weight_min": -29.866256713867188,
      "activations/layer14_attention_weight_max": 40.92732238769531,
      "activations/layer14_attention_weight_min": -31.077720642089844,
      "activations/layer15_attention_weight_max": 28.08437728881836,
      "activations/layer15_attention_weight_min": -22.636146545410156,
      "activations/layer16_attention_weight_max": 24.411449432373047,
      "activations/layer16_attention_weight_min": -21.663681030273438,
      "activations/layer17_attention_weight_max": 26.387348175048828,
      "activations/layer17_attention_weight_min": -16.873762130737305,
      "activations/layer18_attention_weight_max": 19.892263412475586,
      "activations/layer18_attention_weight_min": -10.03056526184082,
      "activations/layer19_attention_weight_max": 19.170316696166992,
      "activations/layer19_attention_weight_min": -15.093652725219727,
      "activations/layer1_attention_weight_max": 13.396316528320312,
      "activations/layer1_attention_weight_min": -10.016536712646484,
      "activations/layer20_attention_weight_max": 22.362892150878906,
      "activations/layer20_attention_weight_min": -10.693029403686523,
      "activations/layer21_attention_weight_max": 18.339069366455078,
      "activations/layer21_attention_weight_min": -9.311182975769043,
      "activations/layer22_attention_weight_max": 30.414669036865234,
      "activations/layer22_attention_weight_min": -13.802129745483398,
      "activations/layer23_attention_weight_max": 20.575992584228516,
      "activations/layer23_attention_weight_min": -12.075953483581543,
      "activations/layer2_attention_weight_max": 29.51573944091797,
      "activations/layer2_attention_weight_min": -22.724563598632812,
      "activations/layer3_attention_weight_max": 53.73194122314453,
      "activations/layer3_attention_weight_min": -49.642852783203125,
      "activations/layer4_attention_weight_max": 115.54017639160156,
      "activations/layer4_attention_weight_min": -108.7250747680664,
      "activations/layer5_attention_weight_max": 129.43710327148438,
      "activations/layer5_attention_weight_min": -118.55137634277344,
      "activations/layer6_attention_weight_max": 82.70802307128906,
      "activations/layer6_attention_weight_min": -76.23689270019531,
      "activations/layer7_attention_weight_max": 75.72344970703125,
      "activations/layer7_attention_weight_min": -68.56075286865234,
      "activations/layer8_attention_weight_max": 67.01318359375,
      "activations/layer8_attention_weight_min": -65.6155776977539,
      "activations/layer9_attention_weight_max": 70.27297973632812,
      "activations/layer9_attention_weight_min": -61.560646057128906,
      "epoch": 0.47,
      "learning_rate": 0.0001484560606060606,
      "loss": 3.2857,
      "step": 8100
    },
    {
      "activations/layer0_attention_weight_max": 17.580718994140625,
      "activations/layer0_attention_weight_min": -10.987415313720703,
      "activations/layer10_attention_weight_max": 44.79807662963867,
      "activations/layer10_attention_weight_min": -40.188716888427734,
      "activations/layer11_attention_weight_max": 59.36890411376953,
      "activations/layer11_attention_weight_min": -43.3043212890625,
      "activations/layer12_attention_weight_max": 37.79473876953125,
      "activations/layer12_attention_weight_min": -28.475650787353516,
      "activations/layer13_attention_weight_max": 35.860897064208984,
      "activations/layer13_attention_weight_min": -31.628948211669922,
      "activations/layer14_attention_weight_max": 43.127342224121094,
      "activations/layer14_attention_weight_min": -34.89644241333008,
      "activations/layer15_attention_weight_max": 34.03871154785156,
      "activations/layer15_attention_weight_min": -24.574748992919922,
      "activations/layer16_attention_weight_max": 28.39789581298828,
      "activations/layer16_attention_weight_min": -21.815404891967773,
      "activations/layer17_attention_weight_max": 27.163700103759766,
      "activations/layer17_attention_weight_min": -17.323331832885742,
      "activations/layer18_attention_weight_max": 24.701065063476562,
      "activations/layer18_attention_weight_min": -13.366997718811035,
      "activations/layer19_attention_weight_max": 22.87118148803711,
      "activations/layer19_attention_weight_min": -18.194456100463867,
      "activations/layer1_attention_weight_max": 13.50737190246582,
      "activations/layer1_attention_weight_min": -10.941560745239258,
      "activations/layer20_attention_weight_max": 20.10468292236328,
      "activations/layer20_attention_weight_min": -13.156609535217285,
      "activations/layer21_attention_weight_max": 17.73068618774414,
      "activations/layer21_attention_weight_min": -12.799107551574707,
      "activations/layer22_attention_weight_max": 30.091218948364258,
      "activations/layer22_attention_weight_min": -15.484928131103516,
      "activations/layer23_attention_weight_max": 21.16202163696289,
      "activations/layer23_attention_weight_min": -14.214156150817871,
      "activations/layer2_attention_weight_max": 29.04458236694336,
      "activations/layer2_attention_weight_min": -22.72304344177246,
      "activations/layer3_attention_weight_max": 55.967830657958984,
      "activations/layer3_attention_weight_min": -49.17462158203125,
      "activations/layer4_attention_weight_max": 119.5158462524414,
      "activations/layer4_attention_weight_min": -109.92352294921875,
      "activations/layer5_attention_weight_max": 136.28614807128906,
      "activations/layer5_attention_weight_min": -116.31607055664062,
      "activations/layer6_attention_weight_max": 81.4549789428711,
      "activations/layer6_attention_weight_min": -73.17630767822266,
      "activations/layer7_attention_weight_max": 78.64686584472656,
      "activations/layer7_attention_weight_min": -72.46774291992188,
      "activations/layer8_attention_weight_max": 65.26921081542969,
      "activations/layer8_attention_weight_min": -58.79736328125,
      "activations/layer9_attention_weight_max": 63.42875289916992,
      "activations/layer9_attention_weight_min": -56.80914306640625,
      "epoch": 0.47,
      "learning_rate": 0.0001484371212121212,
      "loss": 3.2714,
      "step": 8150
    },
    {
      "activations/layer0_attention_weight_max": 17.12831687927246,
      "activations/layer0_attention_weight_min": -10.737096786499023,
      "activations/layer10_attention_weight_max": 49.72791290283203,
      "activations/layer10_attention_weight_min": -41.30936813354492,
      "activations/layer11_attention_weight_max": 53.950679779052734,
      "activations/layer11_attention_weight_min": -39.410396575927734,
      "activations/layer12_attention_weight_max": 25.956331253051758,
      "activations/layer12_attention_weight_min": -20.436681747436523,
      "activations/layer13_attention_weight_max": 34.63999938964844,
      "activations/layer13_attention_weight_min": -29.92355728149414,
      "activations/layer14_attention_weight_max": 47.00580978393555,
      "activations/layer14_attention_weight_min": -31.083066940307617,
      "activations/layer15_attention_weight_max": 31.7750186920166,
      "activations/layer15_attention_weight_min": -27.1752986907959,
      "activations/layer16_attention_weight_max": 26.3829402923584,
      "activations/layer16_attention_weight_min": -21.03209686279297,
      "activations/layer17_attention_weight_max": 34.63364028930664,
      "activations/layer17_attention_weight_min": -16.70115852355957,
      "activations/layer18_attention_weight_max": 22.299457550048828,
      "activations/layer18_attention_weight_min": -11.8429594039917,
      "activations/layer19_attention_weight_max": 26.231903076171875,
      "activations/layer19_attention_weight_min": -14.214447021484375,
      "activations/layer1_attention_weight_max": 13.931685447692871,
      "activations/layer1_attention_weight_min": -11.129490852355957,
      "activations/layer20_attention_weight_max": 25.594255447387695,
      "activations/layer20_attention_weight_min": -12.195982933044434,
      "activations/layer21_attention_weight_max": 22.9950008392334,
      "activations/layer21_attention_weight_min": -10.997961044311523,
      "activations/layer22_attention_weight_max": 27.108179092407227,
      "activations/layer22_attention_weight_min": -14.74895191192627,
      "activations/layer23_attention_weight_max": 19.9893741607666,
      "activations/layer23_attention_weight_min": -13.52863597869873,
      "activations/layer2_attention_weight_max": 27.622648239135742,
      "activations/layer2_attention_weight_min": -21.821670532226562,
      "activations/layer3_attention_weight_max": 54.68275451660156,
      "activations/layer3_attention_weight_min": -49.53586196899414,
      "activations/layer4_attention_weight_max": 117.4455337524414,
      "activations/layer4_attention_weight_min": -110.1126480102539,
      "activations/layer5_attention_weight_max": 136.74591064453125,
      "activations/layer5_attention_weight_min": -119.88862609863281,
      "activations/layer6_attention_weight_max": 81.57987976074219,
      "activations/layer6_attention_weight_min": -74.0477294921875,
      "activations/layer7_attention_weight_max": 78.07791900634766,
      "activations/layer7_attention_weight_min": -65.44286346435547,
      "activations/layer8_attention_weight_max": 66.67013549804688,
      "activations/layer8_attention_weight_min": -60.12758255004883,
      "activations/layer9_attention_weight_max": 67.54617309570312,
      "activations/layer9_attention_weight_min": -61.409332275390625,
      "epoch": 0.48,
      "learning_rate": 0.0001484181818181818,
      "loss": 3.2803,
      "step": 8200
    },
    {
      "activations/layer0_attention_weight_max": 16.813495635986328,
      "activations/layer0_attention_weight_min": -10.703130722045898,
      "activations/layer10_attention_weight_max": 46.09010696411133,
      "activations/layer10_attention_weight_min": -42.23445510864258,
      "activations/layer11_attention_weight_max": 49.406005859375,
      "activations/layer11_attention_weight_min": -37.11518096923828,
      "activations/layer12_attention_weight_max": 30.867778778076172,
      "activations/layer12_attention_weight_min": -24.37038803100586,
      "activations/layer13_attention_weight_max": 35.273780822753906,
      "activations/layer13_attention_weight_min": -34.07025909423828,
      "activations/layer14_attention_weight_max": 39.934268951416016,
      "activations/layer14_attention_weight_min": -30.845767974853516,
      "activations/layer15_attention_weight_max": 29.491470336914062,
      "activations/layer15_attention_weight_min": -31.654619216918945,
      "activations/layer16_attention_weight_max": 29.460956573486328,
      "activations/layer16_attention_weight_min": -21.239084243774414,
      "activations/layer17_attention_weight_max": 29.15170669555664,
      "activations/layer17_attention_weight_min": -16.933006286621094,
      "activations/layer18_attention_weight_max": 19.42531967163086,
      "activations/layer18_attention_weight_min": -13.29914665222168,
      "activations/layer19_attention_weight_max": 24.472469329833984,
      "activations/layer19_attention_weight_min": -16.93880844116211,
      "activations/layer1_attention_weight_max": 14.037830352783203,
      "activations/layer1_attention_weight_min": -11.63029956817627,
      "activations/layer20_attention_weight_max": 22.278648376464844,
      "activations/layer20_attention_weight_min": -14.248931884765625,
      "activations/layer21_attention_weight_max": 19.406503677368164,
      "activations/layer21_attention_weight_min": -13.999154090881348,
      "activations/layer22_attention_weight_max": 31.78844451904297,
      "activations/layer22_attention_weight_min": -16.540843963623047,
      "activations/layer23_attention_weight_max": 21.09499740600586,
      "activations/layer23_attention_weight_min": -16.443525314331055,
      "activations/layer2_attention_weight_max": 27.11339569091797,
      "activations/layer2_attention_weight_min": -23.38225555419922,
      "activations/layer3_attention_weight_max": 53.54399871826172,
      "activations/layer3_attention_weight_min": -49.16401290893555,
      "activations/layer4_attention_weight_max": 120.57118225097656,
      "activations/layer4_attention_weight_min": -107.35567474365234,
      "activations/layer5_attention_weight_max": 139.60604858398438,
      "activations/layer5_attention_weight_min": -120.17587280273438,
      "activations/layer6_attention_weight_max": 82.92801666259766,
      "activations/layer6_attention_weight_min": -76.41426849365234,
      "activations/layer7_attention_weight_max": 72.48172760009766,
      "activations/layer7_attention_weight_min": -68.7782974243164,
      "activations/layer8_attention_weight_max": 67.1756362915039,
      "activations/layer8_attention_weight_min": -60.68288803100586,
      "activations/layer9_attention_weight_max": 72.3954086303711,
      "activations/layer9_attention_weight_min": -61.60931396484375,
      "epoch": 0.48,
      "learning_rate": 0.0001483992424242424,
      "loss": 3.2815,
      "step": 8250
    },
    {
      "activations/layer0_attention_weight_max": 17.604429244995117,
      "activations/layer0_attention_weight_min": -10.686202049255371,
      "activations/layer10_attention_weight_max": 49.75358581542969,
      "activations/layer10_attention_weight_min": -43.61662673950195,
      "activations/layer11_attention_weight_max": 52.0118522644043,
      "activations/layer11_attention_weight_min": -39.089111328125,
      "activations/layer12_attention_weight_max": 42.46674346923828,
      "activations/layer12_attention_weight_min": -28.603792190551758,
      "activations/layer13_attention_weight_max": 35.8377571105957,
      "activations/layer13_attention_weight_min": -30.28548240661621,
      "activations/layer14_attention_weight_max": 49.73561477661133,
      "activations/layer14_attention_weight_min": -35.46954345703125,
      "activations/layer15_attention_weight_max": 30.559112548828125,
      "activations/layer15_attention_weight_min": -25.292264938354492,
      "activations/layer16_attention_weight_max": 23.70870590209961,
      "activations/layer16_attention_weight_min": -22.81268882751465,
      "activations/layer17_attention_weight_max": 25.65557289123535,
      "activations/layer17_attention_weight_min": -17.821779251098633,
      "activations/layer18_attention_weight_max": 18.83553695678711,
      "activations/layer18_attention_weight_min": -15.28885555267334,
      "activations/layer19_attention_weight_max": 22.434310913085938,
      "activations/layer19_attention_weight_min": -20.617206573486328,
      "activations/layer1_attention_weight_max": 13.776878356933594,
      "activations/layer1_attention_weight_min": -10.944589614868164,
      "activations/layer20_attention_weight_max": 21.149066925048828,
      "activations/layer20_attention_weight_min": -17.479888916015625,
      "activations/layer21_attention_weight_max": 16.921091079711914,
      "activations/layer21_attention_weight_min": -13.688642501831055,
      "activations/layer22_attention_weight_max": 25.788593292236328,
      "activations/layer22_attention_weight_min": -16.529319763183594,
      "activations/layer23_attention_weight_max": 16.566150665283203,
      "activations/layer23_attention_weight_min": -14.705988883972168,
      "activations/layer2_attention_weight_max": 29.03423309326172,
      "activations/layer2_attention_weight_min": -22.80306053161621,
      "activations/layer3_attention_weight_max": 55.054351806640625,
      "activations/layer3_attention_weight_min": -49.57102584838867,
      "activations/layer4_attention_weight_max": 120.02201080322266,
      "activations/layer4_attention_weight_min": -106.7024154663086,
      "activations/layer5_attention_weight_max": 139.49929809570312,
      "activations/layer5_attention_weight_min": -116.92797088623047,
      "activations/layer6_attention_weight_max": 83.77392578125,
      "activations/layer6_attention_weight_min": -74.29414367675781,
      "activations/layer7_attention_weight_max": 75.26143646240234,
      "activations/layer7_attention_weight_min": -65.22003936767578,
      "activations/layer8_attention_weight_max": 68.28372955322266,
      "activations/layer8_attention_weight_min": -61.17567825317383,
      "activations/layer9_attention_weight_max": 67.17192840576172,
      "activations/layer9_attention_weight_min": -67.25019073486328,
      "epoch": 0.48,
      "learning_rate": 0.000148380303030303,
      "loss": 3.2748,
      "step": 8300
    },
    {
      "activations/layer0_attention_weight_max": 16.447853088378906,
      "activations/layer0_attention_weight_min": -10.762451171875,
      "activations/layer10_attention_weight_max": 55.97838592529297,
      "activations/layer10_attention_weight_min": -44.41633605957031,
      "activations/layer11_attention_weight_max": 47.570045471191406,
      "activations/layer11_attention_weight_min": -41.99311065673828,
      "activations/layer12_attention_weight_max": 39.173519134521484,
      "activations/layer12_attention_weight_min": -24.688282012939453,
      "activations/layer13_attention_weight_max": 38.37821960449219,
      "activations/layer13_attention_weight_min": -29.50243377685547,
      "activations/layer14_attention_weight_max": 47.061134338378906,
      "activations/layer14_attention_weight_min": -35.00551223754883,
      "activations/layer15_attention_weight_max": 31.783456802368164,
      "activations/layer15_attention_weight_min": -23.74996566772461,
      "activations/layer16_attention_weight_max": 28.162437438964844,
      "activations/layer16_attention_weight_min": -20.00276756286621,
      "activations/layer17_attention_weight_max": 29.014564514160156,
      "activations/layer17_attention_weight_min": -13.754061698913574,
      "activations/layer18_attention_weight_max": 22.06165313720703,
      "activations/layer18_attention_weight_min": -13.331753730773926,
      "activations/layer19_attention_weight_max": 22.752920150756836,
      "activations/layer19_attention_weight_min": -16.5357723236084,
      "activations/layer1_attention_weight_max": 15.194229125976562,
      "activations/layer1_attention_weight_min": -11.287351608276367,
      "activations/layer20_attention_weight_max": 20.039587020874023,
      "activations/layer20_attention_weight_min": -14.745207786560059,
      "activations/layer21_attention_weight_max": 24.344924926757812,
      "activations/layer21_attention_weight_min": -11.024150848388672,
      "activations/layer22_attention_weight_max": 31.029327392578125,
      "activations/layer22_attention_weight_min": -13.981155395507812,
      "activations/layer23_attention_weight_max": 21.128135681152344,
      "activations/layer23_attention_weight_min": -12.964457511901855,
      "activations/layer2_attention_weight_max": 30.512128829956055,
      "activations/layer2_attention_weight_min": -23.93960189819336,
      "activations/layer3_attention_weight_max": 57.433597564697266,
      "activations/layer3_attention_weight_min": -51.29273986816406,
      "activations/layer4_attention_weight_max": 119.6553726196289,
      "activations/layer4_attention_weight_min": -112.76534271240234,
      "activations/layer5_attention_weight_max": 140.82601928710938,
      "activations/layer5_attention_weight_min": -122.72393798828125,
      "activations/layer6_attention_weight_max": 86.66024017333984,
      "activations/layer6_attention_weight_min": -73.4864273071289,
      "activations/layer7_attention_weight_max": 89.1318588256836,
      "activations/layer7_attention_weight_min": -71.72176361083984,
      "activations/layer8_attention_weight_max": 75.48542022705078,
      "activations/layer8_attention_weight_min": -61.51377487182617,
      "activations/layer9_attention_weight_max": 77.05506896972656,
      "activations/layer9_attention_weight_min": -62.66537094116211,
      "epoch": 0.49,
      "learning_rate": 0.00014836136363636363,
      "loss": 3.2801,
      "step": 8350
    },
    {
      "activations/layer0_attention_weight_max": 17.03489112854004,
      "activations/layer0_attention_weight_min": -10.707868576049805,
      "activations/layer10_attention_weight_max": 49.74256134033203,
      "activations/layer10_attention_weight_min": -42.72438430786133,
      "activations/layer11_attention_weight_max": 49.84257125854492,
      "activations/layer11_attention_weight_min": -41.032005310058594,
      "activations/layer12_attention_weight_max": 45.77760314941406,
      "activations/layer12_attention_weight_min": -29.780275344848633,
      "activations/layer13_attention_weight_max": 42.309932708740234,
      "activations/layer13_attention_weight_min": -32.051414489746094,
      "activations/layer14_attention_weight_max": 43.53068923950195,
      "activations/layer14_attention_weight_min": -31.41570472717285,
      "activations/layer15_attention_weight_max": 38.53509521484375,
      "activations/layer15_attention_weight_min": -25.108442306518555,
      "activations/layer16_attention_weight_max": 26.148283004760742,
      "activations/layer16_attention_weight_min": -24.137346267700195,
      "activations/layer17_attention_weight_max": 20.693899154663086,
      "activations/layer17_attention_weight_min": -15.46821403503418,
      "activations/layer18_attention_weight_max": 16.875797271728516,
      "activations/layer18_attention_weight_min": -13.832076072692871,
      "activations/layer19_attention_weight_max": 20.05934715270996,
      "activations/layer19_attention_weight_min": -18.838254928588867,
      "activations/layer1_attention_weight_max": 14.392048835754395,
      "activations/layer1_attention_weight_min": -11.406360626220703,
      "activations/layer20_attention_weight_max": 18.95638084411621,
      "activations/layer20_attention_weight_min": -18.191362380981445,
      "activations/layer21_attention_weight_max": 20.77628517150879,
      "activations/layer21_attention_weight_min": -11.661190032958984,
      "activations/layer22_attention_weight_max": 28.279401779174805,
      "activations/layer22_attention_weight_min": -14.970471382141113,
      "activations/layer23_attention_weight_max": 20.507457733154297,
      "activations/layer23_attention_weight_min": -13.228883743286133,
      "activations/layer2_attention_weight_max": 29.221729278564453,
      "activations/layer2_attention_weight_min": -23.1168212890625,
      "activations/layer3_attention_weight_max": 56.283260345458984,
      "activations/layer3_attention_weight_min": -52.0452995300293,
      "activations/layer4_attention_weight_max": 119.44575500488281,
      "activations/layer4_attention_weight_min": -105.51018524169922,
      "activations/layer5_attention_weight_max": 137.09506225585938,
      "activations/layer5_attention_weight_min": -119.09295654296875,
      "activations/layer6_attention_weight_max": 83.39906311035156,
      "activations/layer6_attention_weight_min": -75.27445220947266,
      "activations/layer7_attention_weight_max": 74.05119323730469,
      "activations/layer7_attention_weight_min": -66.1037368774414,
      "activations/layer8_attention_weight_max": 70.04104614257812,
      "activations/layer8_attention_weight_min": -60.6652717590332,
      "activations/layer9_attention_weight_max": 73.68842315673828,
      "activations/layer9_attention_weight_min": -60.660396575927734,
      "epoch": 0.49,
      "learning_rate": 0.00014834242424242421,
      "loss": 3.2892,
      "step": 8400
    },
    {
      "activations/layer0_attention_weight_max": 16.02627944946289,
      "activations/layer0_attention_weight_min": -10.630157470703125,
      "activations/layer10_attention_weight_max": 48.6956672668457,
      "activations/layer10_attention_weight_min": -40.2897834777832,
      "activations/layer11_attention_weight_max": 52.15052795410156,
      "activations/layer11_attention_weight_min": -44.74500274658203,
      "activations/layer12_attention_weight_max": 28.62125015258789,
      "activations/layer12_attention_weight_min": -23.127120971679688,
      "activations/layer13_attention_weight_max": 39.39173889160156,
      "activations/layer13_attention_weight_min": -31.470550537109375,
      "activations/layer14_attention_weight_max": 48.853187561035156,
      "activations/layer14_attention_weight_min": -34.99616241455078,
      "activations/layer15_attention_weight_max": 30.585420608520508,
      "activations/layer15_attention_weight_min": -19.972736358642578,
      "activations/layer16_attention_weight_max": 28.66606330871582,
      "activations/layer16_attention_weight_min": -24.656774520874023,
      "activations/layer17_attention_weight_max": 30.094482421875,
      "activations/layer17_attention_weight_min": -19.522628784179688,
      "activations/layer18_attention_weight_max": 19.518569946289062,
      "activations/layer18_attention_weight_min": -13.385578155517578,
      "activations/layer19_attention_weight_max": 25.6016788482666,
      "activations/layer19_attention_weight_min": -22.270570755004883,
      "activations/layer1_attention_weight_max": 13.674055099487305,
      "activations/layer1_attention_weight_min": -10.264908790588379,
      "activations/layer20_attention_weight_max": 19.072826385498047,
      "activations/layer20_attention_weight_min": -16.04177474975586,
      "activations/layer21_attention_weight_max": 19.673585891723633,
      "activations/layer21_attention_weight_min": -14.328182220458984,
      "activations/layer22_attention_weight_max": 23.708194732666016,
      "activations/layer22_attention_weight_min": -16.717912673950195,
      "activations/layer23_attention_weight_max": 17.546749114990234,
      "activations/layer23_attention_weight_min": -14.911933898925781,
      "activations/layer2_attention_weight_max": 27.96479034423828,
      "activations/layer2_attention_weight_min": -22.55342674255371,
      "activations/layer3_attention_weight_max": 56.26099395751953,
      "activations/layer3_attention_weight_min": -50.62749481201172,
      "activations/layer4_attention_weight_max": 119.0724868774414,
      "activations/layer4_attention_weight_min": -111.27166748046875,
      "activations/layer5_attention_weight_max": 139.8026885986328,
      "activations/layer5_attention_weight_min": -121.59159851074219,
      "activations/layer6_attention_weight_max": 81.7314682006836,
      "activations/layer6_attention_weight_min": -79.5433578491211,
      "activations/layer7_attention_weight_max": 75.23918914794922,
      "activations/layer7_attention_weight_min": -66.42443084716797,
      "activations/layer8_attention_weight_max": 68.56077575683594,
      "activations/layer8_attention_weight_min": -57.46470260620117,
      "activations/layer9_attention_weight_max": 67.03719329833984,
      "activations/layer9_attention_weight_min": -61.98696517944336,
      "epoch": 0.49,
      "learning_rate": 0.00014832348484848483,
      "loss": 3.26,
      "step": 8450
    },
    {
      "activations/layer0_attention_weight_max": 16.304487228393555,
      "activations/layer0_attention_weight_min": -11.032390594482422,
      "activations/layer10_attention_weight_max": 51.824951171875,
      "activations/layer10_attention_weight_min": -41.9593620300293,
      "activations/layer11_attention_weight_max": 50.51411437988281,
      "activations/layer11_attention_weight_min": -39.197471618652344,
      "activations/layer12_attention_weight_max": 49.31952667236328,
      "activations/layer12_attention_weight_min": -24.81955909729004,
      "activations/layer13_attention_weight_max": 52.19878387451172,
      "activations/layer13_attention_weight_min": -32.323951721191406,
      "activations/layer14_attention_weight_max": 37.96897888183594,
      "activations/layer14_attention_weight_min": -34.41913604736328,
      "activations/layer15_attention_weight_max": 41.42875671386719,
      "activations/layer15_attention_weight_min": -33.03647994995117,
      "activations/layer16_attention_weight_max": 21.81230354309082,
      "activations/layer16_attention_weight_min": -22.37911605834961,
      "activations/layer17_attention_weight_max": 25.960039138793945,
      "activations/layer17_attention_weight_min": -20.8561954498291,
      "activations/layer18_attention_weight_max": 17.176496505737305,
      "activations/layer18_attention_weight_min": -12.245603561401367,
      "activations/layer19_attention_weight_max": 19.51645851135254,
      "activations/layer19_attention_weight_min": -16.06703758239746,
      "activations/layer1_attention_weight_max": 12.858267784118652,
      "activations/layer1_attention_weight_min": -10.735260963439941,
      "activations/layer20_attention_weight_max": 17.27829360961914,
      "activations/layer20_attention_weight_min": -12.532242774963379,
      "activations/layer21_attention_weight_max": 20.114070892333984,
      "activations/layer21_attention_weight_min": -11.400604248046875,
      "activations/layer22_attention_weight_max": 23.836029052734375,
      "activations/layer22_attention_weight_min": -15.11158561706543,
      "activations/layer23_attention_weight_max": 17.20294189453125,
      "activations/layer23_attention_weight_min": -16.199682235717773,
      "activations/layer2_attention_weight_max": 29.312137603759766,
      "activations/layer2_attention_weight_min": -23.58571434020996,
      "activations/layer3_attention_weight_max": 55.2122917175293,
      "activations/layer3_attention_weight_min": -53.71327209472656,
      "activations/layer4_attention_weight_max": 117.76961517333984,
      "activations/layer4_attention_weight_min": -107.6401138305664,
      "activations/layer5_attention_weight_max": 141.76771545410156,
      "activations/layer5_attention_weight_min": -120.44469451904297,
      "activations/layer6_attention_weight_max": 84.3241958618164,
      "activations/layer6_attention_weight_min": -74.37237548828125,
      "activations/layer7_attention_weight_max": 83.57852172851562,
      "activations/layer7_attention_weight_min": -66.20098114013672,
      "activations/layer8_attention_weight_max": 75.82594299316406,
      "activations/layer8_attention_weight_min": -61.529541015625,
      "activations/layer9_attention_weight_max": 85.98136901855469,
      "activations/layer9_attention_weight_min": -65.30949401855469,
      "epoch": 0.49,
      "learning_rate": 0.00014830454545454544,
      "loss": 3.2641,
      "step": 8500
    },
    {
      "activations/layer0_attention_weight_max": 16.558942794799805,
      "activations/layer0_attention_weight_min": -10.531380653381348,
      "activations/layer10_attention_weight_max": 45.24055099487305,
      "activations/layer10_attention_weight_min": -41.758636474609375,
      "activations/layer11_attention_weight_max": 44.956817626953125,
      "activations/layer11_attention_weight_min": -40.279884338378906,
      "activations/layer12_attention_weight_max": 30.665441513061523,
      "activations/layer12_attention_weight_min": -23.720401763916016,
      "activations/layer13_attention_weight_max": 34.2952766418457,
      "activations/layer13_attention_weight_min": -32.47244644165039,
      "activations/layer14_attention_weight_max": 43.4740104675293,
      "activations/layer14_attention_weight_min": -31.24371910095215,
      "activations/layer15_attention_weight_max": 35.824989318847656,
      "activations/layer15_attention_weight_min": -23.584732055664062,
      "activations/layer16_attention_weight_max": 28.849321365356445,
      "activations/layer16_attention_weight_min": -22.23454475402832,
      "activations/layer17_attention_weight_max": 28.390588760375977,
      "activations/layer17_attention_weight_min": -16.972660064697266,
      "activations/layer18_attention_weight_max": 19.94911003112793,
      "activations/layer18_attention_weight_min": -11.227975845336914,
      "activations/layer19_attention_weight_max": 25.551544189453125,
      "activations/layer19_attention_weight_min": -15.970416069030762,
      "activations/layer1_attention_weight_max": 14.265399932861328,
      "activations/layer1_attention_weight_min": -10.813618659973145,
      "activations/layer20_attention_weight_max": 21.507482528686523,
      "activations/layer20_attention_weight_min": -11.619466781616211,
      "activations/layer21_attention_weight_max": 21.364320755004883,
      "activations/layer21_attention_weight_min": -11.685369491577148,
      "activations/layer22_attention_weight_max": 30.556814193725586,
      "activations/layer22_attention_weight_min": -17.232501983642578,
      "activations/layer23_attention_weight_max": 20.861536026000977,
      "activations/layer23_attention_weight_min": -16.426912307739258,
      "activations/layer2_attention_weight_max": 27.090925216674805,
      "activations/layer2_attention_weight_min": -23.95030975341797,
      "activations/layer3_attention_weight_max": 55.98630905151367,
      "activations/layer3_attention_weight_min": -53.66584014892578,
      "activations/layer4_attention_weight_max": 115.68839263916016,
      "activations/layer4_attention_weight_min": -111.30914306640625,
      "activations/layer5_attention_weight_max": 130.2552490234375,
      "activations/layer5_attention_weight_min": -119.94831848144531,
      "activations/layer6_attention_weight_max": 77.07159423828125,
      "activations/layer6_attention_weight_min": -73.32293701171875,
      "activations/layer7_attention_weight_max": 71.58038330078125,
      "activations/layer7_attention_weight_min": -65.80583953857422,
      "activations/layer8_attention_weight_max": 63.066162109375,
      "activations/layer8_attention_weight_min": -59.58611297607422,
      "activations/layer9_attention_weight_max": 61.653602600097656,
      "activations/layer9_attention_weight_min": -58.80927276611328,
      "epoch": 0.5,
      "learning_rate": 0.00014828560606060603,
      "loss": 3.2716,
      "step": 8550
    },
    {
      "activations/layer0_attention_weight_max": 16.70029067993164,
      "activations/layer0_attention_weight_min": -10.762971878051758,
      "activations/layer10_attention_weight_max": 52.15164566040039,
      "activations/layer10_attention_weight_min": -46.90985870361328,
      "activations/layer11_attention_weight_max": 44.829246520996094,
      "activations/layer11_attention_weight_min": -39.271270751953125,
      "activations/layer12_attention_weight_max": 28.481325149536133,
      "activations/layer12_attention_weight_min": -27.968177795410156,
      "activations/layer13_attention_weight_max": 40.09733581542969,
      "activations/layer13_attention_weight_min": -32.8721923828125,
      "activations/layer14_attention_weight_max": 50.61968994140625,
      "activations/layer14_attention_weight_min": -34.118900299072266,
      "activations/layer15_attention_weight_max": 46.52141571044922,
      "activations/layer15_attention_weight_min": -25.878835678100586,
      "activations/layer16_attention_weight_max": 33.19416809082031,
      "activations/layer16_attention_weight_min": -22.32876968383789,
      "activations/layer17_attention_weight_max": 40.92049789428711,
      "activations/layer17_attention_weight_min": -16.81942367553711,
      "activations/layer18_attention_weight_max": 21.960519790649414,
      "activations/layer18_attention_weight_min": -11.835355758666992,
      "activations/layer19_attention_weight_max": 33.342437744140625,
      "activations/layer19_attention_weight_min": -15.68437671661377,
      "activations/layer1_attention_weight_max": 14.964162826538086,
      "activations/layer1_attention_weight_min": -11.090919494628906,
      "activations/layer20_attention_weight_max": 25.182188034057617,
      "activations/layer20_attention_weight_min": -11.858526229858398,
      "activations/layer21_attention_weight_max": 24.87926483154297,
      "activations/layer21_attention_weight_min": -10.711852073669434,
      "activations/layer22_attention_weight_max": 34.14556884765625,
      "activations/layer22_attention_weight_min": -14.473862648010254,
      "activations/layer23_attention_weight_max": 26.63632583618164,
      "activations/layer23_attention_weight_min": -14.305506706237793,
      "activations/layer2_attention_weight_max": 28.895689010620117,
      "activations/layer2_attention_weight_min": -21.796043395996094,
      "activations/layer3_attention_weight_max": 54.20165252685547,
      "activations/layer3_attention_weight_min": -47.52607345581055,
      "activations/layer4_attention_weight_max": 116.71761322021484,
      "activations/layer4_attention_weight_min": -104.18807220458984,
      "activations/layer5_attention_weight_max": 133.6988525390625,
      "activations/layer5_attention_weight_min": -118.8155517578125,
      "activations/layer6_attention_weight_max": 79.79098510742188,
      "activations/layer6_attention_weight_min": -70.21665954589844,
      "activations/layer7_attention_weight_max": 74.41114807128906,
      "activations/layer7_attention_weight_min": -66.79121398925781,
      "activations/layer8_attention_weight_max": 70.92459869384766,
      "activations/layer8_attention_weight_min": -61.62614059448242,
      "activations/layer9_attention_weight_max": 76.802978515625,
      "activations/layer9_attention_weight_min": -61.94959259033203,
      "epoch": 0.5,
      "learning_rate": 0.00014826666666666665,
      "loss": 3.2712,
      "step": 8600
    },
    {
      "activations/layer0_attention_weight_max": 16.28502082824707,
      "activations/layer0_attention_weight_min": -11.231277465820312,
      "activations/layer10_attention_weight_max": 46.37388610839844,
      "activations/layer10_attention_weight_min": -39.4625244140625,
      "activations/layer11_attention_weight_max": 42.749752044677734,
      "activations/layer11_attention_weight_min": -37.78504180908203,
      "activations/layer12_attention_weight_max": 27.839113235473633,
      "activations/layer12_attention_weight_min": -23.524518966674805,
      "activations/layer13_attention_weight_max": 34.355342864990234,
      "activations/layer13_attention_weight_min": -31.624309539794922,
      "activations/layer14_attention_weight_max": 42.77933883666992,
      "activations/layer14_attention_weight_min": -34.64656448364258,
      "activations/layer15_attention_weight_max": 28.74896240234375,
      "activations/layer15_attention_weight_min": -25.205705642700195,
      "activations/layer16_attention_weight_max": 24.25362205505371,
      "activations/layer16_attention_weight_min": -21.657316207885742,
      "activations/layer17_attention_weight_max": 20.063188552856445,
      "activations/layer17_attention_weight_min": -15.622658729553223,
      "activations/layer18_attention_weight_max": 16.015945434570312,
      "activations/layer18_attention_weight_min": -12.322833061218262,
      "activations/layer19_attention_weight_max": 17.1439266204834,
      "activations/layer19_attention_weight_min": -16.167476654052734,
      "activations/layer1_attention_weight_max": 14.609870910644531,
      "activations/layer1_attention_weight_min": -11.031063079833984,
      "activations/layer20_attention_weight_max": 18.763185501098633,
      "activations/layer20_attention_weight_min": -12.417489051818848,
      "activations/layer21_attention_weight_max": 16.569677352905273,
      "activations/layer21_attention_weight_min": -10.66823673248291,
      "activations/layer22_attention_weight_max": 28.173120498657227,
      "activations/layer22_attention_weight_min": -16.528610229492188,
      "activations/layer23_attention_weight_max": 19.103618621826172,
      "activations/layer23_attention_weight_min": -13.934802055358887,
      "activations/layer2_attention_weight_max": 27.78182601928711,
      "activations/layer2_attention_weight_min": -23.627248764038086,
      "activations/layer3_attention_weight_max": 57.638912200927734,
      "activations/layer3_attention_weight_min": -54.10917282104492,
      "activations/layer4_attention_weight_max": 118.5062484741211,
      "activations/layer4_attention_weight_min": -112.02400970458984,
      "activations/layer5_attention_weight_max": 138.41595458984375,
      "activations/layer5_attention_weight_min": -119.10903930664062,
      "activations/layer6_attention_weight_max": 82.45106506347656,
      "activations/layer6_attention_weight_min": -73.13555145263672,
      "activations/layer7_attention_weight_max": 75.55694580078125,
      "activations/layer7_attention_weight_min": -66.19010925292969,
      "activations/layer8_attention_weight_max": 69.18927001953125,
      "activations/layer8_attention_weight_min": -58.7239875793457,
      "activations/layer9_attention_weight_max": 66.12814331054688,
      "activations/layer9_attention_weight_min": -63.17130661010742,
      "epoch": 0.5,
      "learning_rate": 0.00014824772727272726,
      "loss": 3.2696,
      "step": 8650
    },
    {
      "activations/layer0_attention_weight_max": 15.949873924255371,
      "activations/layer0_attention_weight_min": -10.577627182006836,
      "activations/layer10_attention_weight_max": 51.26941680908203,
      "activations/layer10_attention_weight_min": -41.122493743896484,
      "activations/layer11_attention_weight_max": 47.01451873779297,
      "activations/layer11_attention_weight_min": -37.80522537231445,
      "activations/layer12_attention_weight_max": 28.68022346496582,
      "activations/layer12_attention_weight_min": -23.69025993347168,
      "activations/layer13_attention_weight_max": 40.729793548583984,
      "activations/layer13_attention_weight_min": -31.64456558227539,
      "activations/layer14_attention_weight_max": 49.147335052490234,
      "activations/layer14_attention_weight_min": -35.16027069091797,
      "activations/layer15_attention_weight_max": 33.61715316772461,
      "activations/layer15_attention_weight_min": -21.091787338256836,
      "activations/layer16_attention_weight_max": 32.77479934692383,
      "activations/layer16_attention_weight_min": -20.971410751342773,
      "activations/layer17_attention_weight_max": 38.12974166870117,
      "activations/layer17_attention_weight_min": -15.255589485168457,
      "activations/layer18_attention_weight_max": 21.0419979095459,
      "activations/layer18_attention_weight_min": -11.937737464904785,
      "activations/layer19_attention_weight_max": 31.84639549255371,
      "activations/layer19_attention_weight_min": -16.497772216796875,
      "activations/layer1_attention_weight_max": 14.514345169067383,
      "activations/layer1_attention_weight_min": -11.900812149047852,
      "activations/layer20_attention_weight_max": 23.22614288330078,
      "activations/layer20_attention_weight_min": -10.614778518676758,
      "activations/layer21_attention_weight_max": 23.47159767150879,
      "activations/layer21_attention_weight_min": -11.192188262939453,
      "activations/layer22_attention_weight_max": 28.86776351928711,
      "activations/layer22_attention_weight_min": -15.056058883666992,
      "activations/layer23_attention_weight_max": 19.87459945678711,
      "activations/layer23_attention_weight_min": -14.198736190795898,
      "activations/layer2_attention_weight_max": 29.92153549194336,
      "activations/layer2_attention_weight_min": -25.20374870300293,
      "activations/layer3_attention_weight_max": 57.6034049987793,
      "activations/layer3_attention_weight_min": -52.6939697265625,
      "activations/layer4_attention_weight_max": 119.0076904296875,
      "activations/layer4_attention_weight_min": -106.76361083984375,
      "activations/layer5_attention_weight_max": 134.7293243408203,
      "activations/layer5_attention_weight_min": -116.3908920288086,
      "activations/layer6_attention_weight_max": 79.46026611328125,
      "activations/layer6_attention_weight_min": -71.91146850585938,
      "activations/layer7_attention_weight_max": 75.88360595703125,
      "activations/layer7_attention_weight_min": -66.22674560546875,
      "activations/layer8_attention_weight_max": 68.83126068115234,
      "activations/layer8_attention_weight_min": -59.30712127685547,
      "activations/layer9_attention_weight_max": 68.57106018066406,
      "activations/layer9_attention_weight_min": -61.310638427734375,
      "epoch": 0.51,
      "learning_rate": 0.00014822878787878785,
      "loss": 3.2638,
      "step": 8700
    },
    {
      "activations/layer0_attention_weight_max": 15.273426055908203,
      "activations/layer0_attention_weight_min": -11.417383193969727,
      "activations/layer10_attention_weight_max": 52.99106216430664,
      "activations/layer10_attention_weight_min": -44.983856201171875,
      "activations/layer11_attention_weight_max": 53.82931137084961,
      "activations/layer11_attention_weight_min": -39.32170867919922,
      "activations/layer12_attention_weight_max": 26.548362731933594,
      "activations/layer12_attention_weight_min": -22.068681716918945,
      "activations/layer13_attention_weight_max": 36.47606658935547,
      "activations/layer13_attention_weight_min": -29.73820686340332,
      "activations/layer14_attention_weight_max": 51.37139892578125,
      "activations/layer14_attention_weight_min": -35.41621780395508,
      "activations/layer15_attention_weight_max": 29.42948341369629,
      "activations/layer15_attention_weight_min": -23.065645217895508,
      "activations/layer16_attention_weight_max": 30.26987648010254,
      "activations/layer16_attention_weight_min": -23.149988174438477,
      "activations/layer17_attention_weight_max": 34.127323150634766,
      "activations/layer17_attention_weight_min": -18.624677658081055,
      "activations/layer18_attention_weight_max": 22.478565216064453,
      "activations/layer18_attention_weight_min": -11.580355644226074,
      "activations/layer19_attention_weight_max": 31.165353775024414,
      "activations/layer19_attention_weight_min": -16.966686248779297,
      "activations/layer1_attention_weight_max": 15.800665855407715,
      "activations/layer1_attention_weight_min": -11.136507034301758,
      "activations/layer20_attention_weight_max": 21.40674591064453,
      "activations/layer20_attention_weight_min": -12.80053424835205,
      "activations/layer21_attention_weight_max": 22.4305477142334,
      "activations/layer21_attention_weight_min": -12.112500190734863,
      "activations/layer22_attention_weight_max": 30.106382369995117,
      "activations/layer22_attention_weight_min": -16.808218002319336,
      "activations/layer23_attention_weight_max": 20.799373626708984,
      "activations/layer23_attention_weight_min": -13.72688102722168,
      "activations/layer2_attention_weight_max": 30.033069610595703,
      "activations/layer2_attention_weight_min": -23.834461212158203,
      "activations/layer3_attention_weight_max": 55.73426818847656,
      "activations/layer3_attention_weight_min": -50.936683654785156,
      "activations/layer4_attention_weight_max": 121.00504302978516,
      "activations/layer4_attention_weight_min": -110.3750228881836,
      "activations/layer5_attention_weight_max": 134.86599731445312,
      "activations/layer5_attention_weight_min": -118.56804656982422,
      "activations/layer6_attention_weight_max": 85.32271575927734,
      "activations/layer6_attention_weight_min": -75.37847137451172,
      "activations/layer7_attention_weight_max": 81.35641479492188,
      "activations/layer7_attention_weight_min": -67.02302551269531,
      "activations/layer8_attention_weight_max": 73.29588317871094,
      "activations/layer8_attention_weight_min": -60.66617202758789,
      "activations/layer9_attention_weight_max": 81.54122161865234,
      "activations/layer9_attention_weight_min": -65.36318969726562,
      "epoch": 0.51,
      "learning_rate": 0.00014820984848484847,
      "loss": 3.2783,
      "step": 8750
    },
    {
      "activations/layer0_attention_weight_max": 15.834680557250977,
      "activations/layer0_attention_weight_min": -10.769185066223145,
      "activations/layer10_attention_weight_max": 46.407127380371094,
      "activations/layer10_attention_weight_min": -39.391021728515625,
      "activations/layer11_attention_weight_max": 46.95138168334961,
      "activations/layer11_attention_weight_min": -36.424591064453125,
      "activations/layer12_attention_weight_max": 26.1118106842041,
      "activations/layer12_attention_weight_min": -20.739988327026367,
      "activations/layer13_attention_weight_max": 35.1121826171875,
      "activations/layer13_attention_weight_min": -32.09184265136719,
      "activations/layer14_attention_weight_max": 45.12843704223633,
      "activations/layer14_attention_weight_min": -30.14044189453125,
      "activations/layer15_attention_weight_max": 41.18113708496094,
      "activations/layer15_attention_weight_min": -29.444446563720703,
      "activations/layer16_attention_weight_max": 29.847110748291016,
      "activations/layer16_attention_weight_min": -21.069034576416016,
      "activations/layer17_attention_weight_max": 28.716272354125977,
      "activations/layer17_attention_weight_min": -15.481404304504395,
      "activations/layer18_attention_weight_max": 18.585790634155273,
      "activations/layer18_attention_weight_min": -11.685702323913574,
      "activations/layer19_attention_weight_max": 26.256183624267578,
      "activations/layer19_attention_weight_min": -14.995645523071289,
      "activations/layer1_attention_weight_max": 13.446592330932617,
      "activations/layer1_attention_weight_min": -10.608325958251953,
      "activations/layer20_attention_weight_max": 19.112871170043945,
      "activations/layer20_attention_weight_min": -12.035978317260742,
      "activations/layer21_attention_weight_max": 18.096132278442383,
      "activations/layer21_attention_weight_min": -13.317480087280273,
      "activations/layer22_attention_weight_max": 29.071578979492188,
      "activations/layer22_attention_weight_min": -13.467240333557129,
      "activations/layer23_attention_weight_max": 20.813119888305664,
      "activations/layer23_attention_weight_min": -12.510442733764648,
      "activations/layer2_attention_weight_max": 27.342205047607422,
      "activations/layer2_attention_weight_min": -23.0544490814209,
      "activations/layer3_attention_weight_max": 54.593963623046875,
      "activations/layer3_attention_weight_min": -49.8073844909668,
      "activations/layer4_attention_weight_max": 115.30915832519531,
      "activations/layer4_attention_weight_min": -107.3565673828125,
      "activations/layer5_attention_weight_max": 128.29727172851562,
      "activations/layer5_attention_weight_min": -117.01882934570312,
      "activations/layer6_attention_weight_max": 81.4837417602539,
      "activations/layer6_attention_weight_min": -71.40261840820312,
      "activations/layer7_attention_weight_max": 72.27757263183594,
      "activations/layer7_attention_weight_min": -64.47032928466797,
      "activations/layer8_attention_weight_max": 70.4214859008789,
      "activations/layer8_attention_weight_min": -57.65053176879883,
      "activations/layer9_attention_weight_max": 71.5210952758789,
      "activations/layer9_attention_weight_min": -64.98339080810547,
      "epoch": 0.51,
      "learning_rate": 0.00014819090909090908,
      "loss": 3.2762,
      "step": 8800
    },
    {
      "activations/layer0_attention_weight_max": 16.040998458862305,
      "activations/layer0_attention_weight_min": -10.35373306274414,
      "activations/layer10_attention_weight_max": 46.70783615112305,
      "activations/layer10_attention_weight_min": -47.99945068359375,
      "activations/layer11_attention_weight_max": 41.25225067138672,
      "activations/layer11_attention_weight_min": -40.34248733520508,
      "activations/layer12_attention_weight_max": 26.221229553222656,
      "activations/layer12_attention_weight_min": -21.9409122467041,
      "activations/layer13_attention_weight_max": 36.744537353515625,
      "activations/layer13_attention_weight_min": -33.873207092285156,
      "activations/layer14_attention_weight_max": 41.68508529663086,
      "activations/layer14_attention_weight_min": -36.19234085083008,
      "activations/layer15_attention_weight_max": 32.50356674194336,
      "activations/layer15_attention_weight_min": -25.330263137817383,
      "activations/layer16_attention_weight_max": 25.238954544067383,
      "activations/layer16_attention_weight_min": -21.248395919799805,
      "activations/layer17_attention_weight_max": 31.984771728515625,
      "activations/layer17_attention_weight_min": -22.078359603881836,
      "activations/layer18_attention_weight_max": 19.749916076660156,
      "activations/layer18_attention_weight_min": -10.958041191101074,
      "activations/layer19_attention_weight_max": 20.990938186645508,
      "activations/layer19_attention_weight_min": -16.602020263671875,
      "activations/layer1_attention_weight_max": 13.80593490600586,
      "activations/layer1_attention_weight_min": -11.005427360534668,
      "activations/layer20_attention_weight_max": 18.757009506225586,
      "activations/layer20_attention_weight_min": -11.272579193115234,
      "activations/layer21_attention_weight_max": 18.14356231689453,
      "activations/layer21_attention_weight_min": -10.615015983581543,
      "activations/layer22_attention_weight_max": 25.29046630859375,
      "activations/layer22_attention_weight_min": -14.150681495666504,
      "activations/layer23_attention_weight_max": 17.365819931030273,
      "activations/layer23_attention_weight_min": -10.946741104125977,
      "activations/layer2_attention_weight_max": 27.85854721069336,
      "activations/layer2_attention_weight_min": -22.788131713867188,
      "activations/layer3_attention_weight_max": 55.92231369018555,
      "activations/layer3_attention_weight_min": -49.601261138916016,
      "activations/layer4_attention_weight_max": 114.56758117675781,
      "activations/layer4_attention_weight_min": -106.9601058959961,
      "activations/layer5_attention_weight_max": 129.07138061523438,
      "activations/layer5_attention_weight_min": -116.85881805419922,
      "activations/layer6_attention_weight_max": 80.06493377685547,
      "activations/layer6_attention_weight_min": -70.98744201660156,
      "activations/layer7_attention_weight_max": 69.83431243896484,
      "activations/layer7_attention_weight_min": -67.46551513671875,
      "activations/layer8_attention_weight_max": 64.46781158447266,
      "activations/layer8_attention_weight_min": -59.6614990234375,
      "activations/layer9_attention_weight_max": 63.71445846557617,
      "activations/layer9_attention_weight_min": -61.90873336791992,
      "epoch": 0.51,
      "learning_rate": 0.0001481719696969697,
      "loss": 3.2805,
      "step": 8850
    },
    {
      "activations/layer0_attention_weight_max": 15.399344444274902,
      "activations/layer0_attention_weight_min": -10.657940864562988,
      "activations/layer10_attention_weight_max": 45.102561950683594,
      "activations/layer10_attention_weight_min": -40.744140625,
      "activations/layer11_attention_weight_max": 48.70092010498047,
      "activations/layer11_attention_weight_min": -40.67896270751953,
      "activations/layer12_attention_weight_max": 32.05475997924805,
      "activations/layer12_attention_weight_min": -29.200626373291016,
      "activations/layer13_attention_weight_max": 35.30440902709961,
      "activations/layer13_attention_weight_min": -31.466228485107422,
      "activations/layer14_attention_weight_max": 42.408817291259766,
      "activations/layer14_attention_weight_min": -32.808563232421875,
      "activations/layer15_attention_weight_max": 38.92387008666992,
      "activations/layer15_attention_weight_min": -27.836074829101562,
      "activations/layer16_attention_weight_max": 32.92909622192383,
      "activations/layer16_attention_weight_min": -21.898122787475586,
      "activations/layer17_attention_weight_max": 26.368410110473633,
      "activations/layer17_attention_weight_min": -15.443921089172363,
      "activations/layer18_attention_weight_max": 20.355436325073242,
      "activations/layer18_attention_weight_min": -12.11699390411377,
      "activations/layer19_attention_weight_max": 26.47524070739746,
      "activations/layer19_attention_weight_min": -14.4849853515625,
      "activations/layer1_attention_weight_max": 13.880196571350098,
      "activations/layer1_attention_weight_min": -10.812116622924805,
      "activations/layer20_attention_weight_max": 20.867137908935547,
      "activations/layer20_attention_weight_min": -11.013453483581543,
      "activations/layer21_attention_weight_max": 20.20285987854004,
      "activations/layer21_attention_weight_min": -10.257527351379395,
      "activations/layer22_attention_weight_max": 32.24333190917969,
      "activations/layer22_attention_weight_min": -15.895015716552734,
      "activations/layer23_attention_weight_max": 26.501272201538086,
      "activations/layer23_attention_weight_min": -13.629313468933105,
      "activations/layer2_attention_weight_max": 30.39996337890625,
      "activations/layer2_attention_weight_min": -23.480976104736328,
      "activations/layer3_attention_weight_max": 56.732364654541016,
      "activations/layer3_attention_weight_min": -51.008018493652344,
      "activations/layer4_attention_weight_max": 114.37017822265625,
      "activations/layer4_attention_weight_min": -110.81551361083984,
      "activations/layer5_attention_weight_max": 136.33702087402344,
      "activations/layer5_attention_weight_min": -117.64678192138672,
      "activations/layer6_attention_weight_max": 81.76579284667969,
      "activations/layer6_attention_weight_min": -73.86641693115234,
      "activations/layer7_attention_weight_max": 73.77902221679688,
      "activations/layer7_attention_weight_min": -65.00532531738281,
      "activations/layer8_attention_weight_max": 72.87639617919922,
      "activations/layer8_attention_weight_min": -58.54199981689453,
      "activations/layer9_attention_weight_max": 61.9923210144043,
      "activations/layer9_attention_weight_min": -60.929237365722656,
      "epoch": 0.52,
      "learning_rate": 0.00014815303030303028,
      "loss": 3.2749,
      "step": 8900
    },
    {
      "activations/layer0_attention_weight_max": 16.120967864990234,
      "activations/layer0_attention_weight_min": -10.746665954589844,
      "activations/layer10_attention_weight_max": 52.64881134033203,
      "activations/layer10_attention_weight_min": -47.002586364746094,
      "activations/layer11_attention_weight_max": 51.218345642089844,
      "activations/layer11_attention_weight_min": -41.424259185791016,
      "activations/layer12_attention_weight_max": 35.080692291259766,
      "activations/layer12_attention_weight_min": -22.979215621948242,
      "activations/layer13_attention_weight_max": 36.308372497558594,
      "activations/layer13_attention_weight_min": -30.845561981201172,
      "activations/layer14_attention_weight_max": 51.94854736328125,
      "activations/layer14_attention_weight_min": -34.380638122558594,
      "activations/layer15_attention_weight_max": 35.22642135620117,
      "activations/layer15_attention_weight_min": -23.831600189208984,
      "activations/layer16_attention_weight_max": 35.70134353637695,
      "activations/layer16_attention_weight_min": -20.795743942260742,
      "activations/layer17_attention_weight_max": 26.340478897094727,
      "activations/layer17_attention_weight_min": -17.47530174255371,
      "activations/layer18_attention_weight_max": 21.220996856689453,
      "activations/layer18_attention_weight_min": -12.009417533874512,
      "activations/layer19_attention_weight_max": 26.902708053588867,
      "activations/layer19_attention_weight_min": -17.801149368286133,
      "activations/layer1_attention_weight_max": 13.96764850616455,
      "activations/layer1_attention_weight_min": -10.941638946533203,
      "activations/layer20_attention_weight_max": 24.514066696166992,
      "activations/layer20_attention_weight_min": -12.547587394714355,
      "activations/layer21_attention_weight_max": 21.99698829650879,
      "activations/layer21_attention_weight_min": -11.619158744812012,
      "activations/layer22_attention_weight_max": 30.500648498535156,
      "activations/layer22_attention_weight_min": -14.811226844787598,
      "activations/layer23_attention_weight_max": 21.397964477539062,
      "activations/layer23_attention_weight_min": -13.643595695495605,
      "activations/layer2_attention_weight_max": 28.340652465820312,
      "activations/layer2_attention_weight_min": -24.09088706970215,
      "activations/layer3_attention_weight_max": 54.92435073852539,
      "activations/layer3_attention_weight_min": -53.76895523071289,
      "activations/layer4_attention_weight_max": 116.4527816772461,
      "activations/layer4_attention_weight_min": -108.47594451904297,
      "activations/layer5_attention_weight_max": 129.93731689453125,
      "activations/layer5_attention_weight_min": -119.78781127929688,
      "activations/layer6_attention_weight_max": 80.48353576660156,
      "activations/layer6_attention_weight_min": -75.23643493652344,
      "activations/layer7_attention_weight_max": 71.01707458496094,
      "activations/layer7_attention_weight_min": -66.73148345947266,
      "activations/layer8_attention_weight_max": 67.4031753540039,
      "activations/layer8_attention_weight_min": -58.81221008300781,
      "activations/layer9_attention_weight_max": 70.00406646728516,
      "activations/layer9_attention_weight_min": -59.50997543334961,
      "epoch": 0.52,
      "learning_rate": 0.0001481340909090909,
      "loss": 3.2713,
      "step": 8950
    },
    {
      "activations/layer0_attention_weight_max": 15.398273468017578,
      "activations/layer0_attention_weight_min": -10.544510841369629,
      "activations/layer10_attention_weight_max": 54.61659622192383,
      "activations/layer10_attention_weight_min": -45.085906982421875,
      "activations/layer11_attention_weight_max": 55.16002655029297,
      "activations/layer11_attention_weight_min": -41.15271759033203,
      "activations/layer12_attention_weight_max": 62.811790466308594,
      "activations/layer12_attention_weight_min": -33.6390495300293,
      "activations/layer13_attention_weight_max": 49.166175842285156,
      "activations/layer13_attention_weight_min": -32.63011932373047,
      "activations/layer14_attention_weight_max": 53.23065948486328,
      "activations/layer14_attention_weight_min": -33.483333587646484,
      "activations/layer15_attention_weight_max": 38.48420715332031,
      "activations/layer15_attention_weight_min": -23.774805068969727,
      "activations/layer16_attention_weight_max": 29.08123016357422,
      "activations/layer16_attention_weight_min": -18.293107986450195,
      "activations/layer17_attention_weight_max": 27.399356842041016,
      "activations/layer17_attention_weight_min": -15.265872955322266,
      "activations/layer18_attention_weight_max": 19.054412841796875,
      "activations/layer18_attention_weight_min": -10.620918273925781,
      "activations/layer19_attention_weight_max": 21.70262908935547,
      "activations/layer19_attention_weight_min": -14.230956077575684,
      "activations/layer1_attention_weight_max": 13.17697525024414,
      "activations/layer1_attention_weight_min": -12.514726638793945,
      "activations/layer20_attention_weight_max": 19.686382293701172,
      "activations/layer20_attention_weight_min": -10.848489761352539,
      "activations/layer21_attention_weight_max": 18.43429946899414,
      "activations/layer21_attention_weight_min": -9.829764366149902,
      "activations/layer22_attention_weight_max": 24.786333084106445,
      "activations/layer22_attention_weight_min": -14.707399368286133,
      "activations/layer23_attention_weight_max": 17.030231475830078,
      "activations/layer23_attention_weight_min": -13.889863967895508,
      "activations/layer2_attention_weight_max": 30.269811630249023,
      "activations/layer2_attention_weight_min": -23.516021728515625,
      "activations/layer3_attention_weight_max": 57.126155853271484,
      "activations/layer3_attention_weight_min": -49.95266342163086,
      "activations/layer4_attention_weight_max": 116.36212158203125,
      "activations/layer4_attention_weight_min": -109.53480529785156,
      "activations/layer5_attention_weight_max": 146.40139770507812,
      "activations/layer5_attention_weight_min": -118.23701477050781,
      "activations/layer6_attention_weight_max": 88.17863464355469,
      "activations/layer6_attention_weight_min": -71.68505096435547,
      "activations/layer7_attention_weight_max": 78.22917938232422,
      "activations/layer7_attention_weight_min": -63.59485626220703,
      "activations/layer8_attention_weight_max": 70.37106323242188,
      "activations/layer8_attention_weight_min": -58.04022979736328,
      "activations/layer9_attention_weight_max": 99.54833984375,
      "activations/layer9_attention_weight_min": -72.86092376708984,
      "epoch": 0.52,
      "learning_rate": 0.00014811515151515151,
      "loss": 3.2672,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_loss": 3.21484375,
      "eval_runtime": 8.4893,
      "eval_samples_per_second": 505.811,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_openwebtext_loss": 3.21484375,
      "eval_openwebtext_ppl": 24.89940107578365,
      "eval_openwebtext_runtime": 8.4893,
      "eval_openwebtext_samples_per_second": 505.811,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_wikitext_loss": 3.517578125,
      "eval_wikitext_ppl": 33.702705802144536,
      "eval_wikitext_runtime": 1.9293,
      "eval_wikitext_samples_per_second": 236.356,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_lambada_loss": 3.923828125,
      "eval_lambada_ppl": 50.5937537501147,
      "eval_lambada_runtime": 9.5337,
      "eval_lambada_samples_per_second": 510.715,
      "step": 9000
    },
    {
      "activations/layer0_attention_weight_max": 15.13420581817627,
      "activations/layer0_attention_weight_min": -11.000359535217285,
      "activations/layer10_attention_weight_max": 54.799190521240234,
      "activations/layer10_attention_weight_min": -42.79356002807617,
      "activations/layer11_attention_weight_max": 49.19225311279297,
      "activations/layer11_attention_weight_min": -41.54228973388672,
      "activations/layer12_attention_weight_max": 28.893678665161133,
      "activations/layer12_attention_weight_min": -22.93352699279785,
      "activations/layer13_attention_weight_max": 44.96514129638672,
      "activations/layer13_attention_weight_min": -35.429691314697266,
      "activations/layer14_attention_weight_max": 51.33074188232422,
      "activations/layer14_attention_weight_min": -41.272254943847656,
      "activations/layer15_attention_weight_max": 35.882591247558594,
      "activations/layer15_attention_weight_min": -27.84370231628418,
      "activations/layer16_attention_weight_max": 30.544422149658203,
      "activations/layer16_attention_weight_min": -22.077068328857422,
      "activations/layer17_attention_weight_max": 24.03452491760254,
      "activations/layer17_attention_weight_min": -14.9045991897583,
      "activations/layer18_attention_weight_max": 23.413366317749023,
      "activations/layer18_attention_weight_min": -14.046436309814453,
      "activations/layer19_attention_weight_max": 26.872129440307617,
      "activations/layer19_attention_weight_min": -18.16250991821289,
      "activations/layer1_attention_weight_max": 13.790825843811035,
      "activations/layer1_attention_weight_min": -10.57026481628418,
      "activations/layer20_attention_weight_max": 24.783029556274414,
      "activations/layer20_attention_weight_min": -11.426656723022461,
      "activations/layer21_attention_weight_max": 25.67035484313965,
      "activations/layer21_attention_weight_min": -11.00246810913086,
      "activations/layer22_attention_weight_max": 27.648454666137695,
      "activations/layer22_attention_weight_min": -15.44328498840332,
      "activations/layer23_attention_weight_max": 20.952383041381836,
      "activations/layer23_attention_weight_min": -14.521343231201172,
      "activations/layer2_attention_weight_max": 29.7789363861084,
      "activations/layer2_attention_weight_min": -24.145885467529297,
      "activations/layer3_attention_weight_max": 54.52550506591797,
      "activations/layer3_attention_weight_min": -51.90180969238281,
      "activations/layer4_attention_weight_max": 115.4386978149414,
      "activations/layer4_attention_weight_min": -110.0196762084961,
      "activations/layer5_attention_weight_max": 142.06838989257812,
      "activations/layer5_attention_weight_min": -120.95988464355469,
      "activations/layer6_attention_weight_max": 87.44480895996094,
      "activations/layer6_attention_weight_min": -74.77849578857422,
      "activations/layer7_attention_weight_max": 81.36795043945312,
      "activations/layer7_attention_weight_min": -65.47006225585938,
      "activations/layer8_attention_weight_max": 74.98783111572266,
      "activations/layer8_attention_weight_min": -63.76758575439453,
      "activations/layer9_attention_weight_max": 74.83967590332031,
      "activations/layer9_attention_weight_min": -62.55412673950195,
      "epoch": 0.53,
      "learning_rate": 0.00014809621212121213,
      "loss": 3.2707,
      "step": 9050
    },
    {
      "activations/layer0_attention_weight_max": 14.28892707824707,
      "activations/layer0_attention_weight_min": -10.768868446350098,
      "activations/layer10_attention_weight_max": 50.905967712402344,
      "activations/layer10_attention_weight_min": -42.3785285949707,
      "activations/layer11_attention_weight_max": 52.36932373046875,
      "activations/layer11_attention_weight_min": -37.18138885498047,
      "activations/layer12_attention_weight_max": 29.889690399169922,
      "activations/layer12_attention_weight_min": -26.74793815612793,
      "activations/layer13_attention_weight_max": 39.56835174560547,
      "activations/layer13_attention_weight_min": -30.623788833618164,
      "activations/layer14_attention_weight_max": 51.32422637939453,
      "activations/layer14_attention_weight_min": -36.04240036010742,
      "activations/layer15_attention_weight_max": 35.13622283935547,
      "activations/layer15_attention_weight_min": -27.78765296936035,
      "activations/layer16_attention_weight_max": 32.56883239746094,
      "activations/layer16_attention_weight_min": -28.764392852783203,
      "activations/layer17_attention_weight_max": 27.172378540039062,
      "activations/layer17_attention_weight_min": -16.593599319458008,
      "activations/layer18_attention_weight_max": 19.204492568969727,
      "activations/layer18_attention_weight_min": -12.17100715637207,
      "activations/layer19_attention_weight_max": 23.995954513549805,
      "activations/layer19_attention_weight_min": -16.455625534057617,
      "activations/layer1_attention_weight_max": 13.713802337646484,
      "activations/layer1_attention_weight_min": -11.283631324768066,
      "activations/layer20_attention_weight_max": 20.02762222290039,
      "activations/layer20_attention_weight_min": -12.962976455688477,
      "activations/layer21_attention_weight_max": 18.7866268157959,
      "activations/layer21_attention_weight_min": -10.250251770019531,
      "activations/layer22_attention_weight_max": 29.01680564880371,
      "activations/layer22_attention_weight_min": -14.944646835327148,
      "activations/layer23_attention_weight_max": 19.736921310424805,
      "activations/layer23_attention_weight_min": -12.21699333190918,
      "activations/layer2_attention_weight_max": 29.23415756225586,
      "activations/layer2_attention_weight_min": -24.098289489746094,
      "activations/layer3_attention_weight_max": 55.925724029541016,
      "activations/layer3_attention_weight_min": -53.452239990234375,
      "activations/layer4_attention_weight_max": 125.01020812988281,
      "activations/layer4_attention_weight_min": -109.84993743896484,
      "activations/layer5_attention_weight_max": 146.75184631347656,
      "activations/layer5_attention_weight_min": -120.08736419677734,
      "activations/layer6_attention_weight_max": 86.08104705810547,
      "activations/layer6_attention_weight_min": -73.6375732421875,
      "activations/layer7_attention_weight_max": 76.62635040283203,
      "activations/layer7_attention_weight_min": -64.21515655517578,
      "activations/layer8_attention_weight_max": 70.11618041992188,
      "activations/layer8_attention_weight_min": -62.30447006225586,
      "activations/layer9_attention_weight_max": 77.88497161865234,
      "activations/layer9_attention_weight_min": -62.8934326171875,
      "epoch": 0.53,
      "learning_rate": 0.00014807727272727272,
      "loss": 3.2751,
      "step": 9100
    },
    {
      "activations/layer0_attention_weight_max": 15.478062629699707,
      "activations/layer0_attention_weight_min": -10.487238883972168,
      "activations/layer10_attention_weight_max": 47.011077880859375,
      "activations/layer10_attention_weight_min": -39.93172836303711,
      "activations/layer11_attention_weight_max": 47.145660400390625,
      "activations/layer11_attention_weight_min": -36.480377197265625,
      "activations/layer12_attention_weight_max": 26.405176162719727,
      "activations/layer12_attention_weight_min": -28.80183219909668,
      "activations/layer13_attention_weight_max": 32.7822380065918,
      "activations/layer13_attention_weight_min": -33.576019287109375,
      "activations/layer14_attention_weight_max": 45.37791442871094,
      "activations/layer14_attention_weight_min": -38.33441925048828,
      "activations/layer15_attention_weight_max": 28.32777976989746,
      "activations/layer15_attention_weight_min": -23.046676635742188,
      "activations/layer16_attention_weight_max": 27.09337043762207,
      "activations/layer16_attention_weight_min": -23.5406494140625,
      "activations/layer17_attention_weight_max": 27.652528762817383,
      "activations/layer17_attention_weight_min": -18.014829635620117,
      "activations/layer18_attention_weight_max": 20.173398971557617,
      "activations/layer18_attention_weight_min": -12.911639213562012,
      "activations/layer19_attention_weight_max": 24.78472900390625,
      "activations/layer19_attention_weight_min": -19.008527755737305,
      "activations/layer1_attention_weight_max": 13.967994689941406,
      "activations/layer1_attention_weight_min": -10.896184921264648,
      "activations/layer20_attention_weight_max": 18.451871871948242,
      "activations/layer20_attention_weight_min": -10.763422012329102,
      "activations/layer21_attention_weight_max": 19.027116775512695,
      "activations/layer21_attention_weight_min": -12.120762825012207,
      "activations/layer22_attention_weight_max": 29.580787658691406,
      "activations/layer22_attention_weight_min": -15.64922046661377,
      "activations/layer23_attention_weight_max": 20.076580047607422,
      "activations/layer23_attention_weight_min": -13.56891918182373,
      "activations/layer2_attention_weight_max": 27.43299102783203,
      "activations/layer2_attention_weight_min": -24.23558807373047,
      "activations/layer3_attention_weight_max": 52.52785873413086,
      "activations/layer3_attention_weight_min": -51.44955062866211,
      "activations/layer4_attention_weight_max": 113.64376068115234,
      "activations/layer4_attention_weight_min": -108.93287658691406,
      "activations/layer5_attention_weight_max": 126.58715057373047,
      "activations/layer5_attention_weight_min": -119.98887634277344,
      "activations/layer6_attention_weight_max": 73.92598724365234,
      "activations/layer6_attention_weight_min": -75.19747924804688,
      "activations/layer7_attention_weight_max": 71.8968505859375,
      "activations/layer7_attention_weight_min": -68.72207641601562,
      "activations/layer8_attention_weight_max": 70.21038818359375,
      "activations/layer8_attention_weight_min": -62.01803207397461,
      "activations/layer9_attention_weight_max": 63.161354064941406,
      "activations/layer9_attention_weight_min": -58.791603088378906,
      "epoch": 0.53,
      "learning_rate": 0.00014805833333333333,
      "loss": 3.2752,
      "step": 9150
    },
    {
      "activations/layer0_attention_weight_max": 15.880072593688965,
      "activations/layer0_attention_weight_min": -10.938802719116211,
      "activations/layer10_attention_weight_max": 48.51308059692383,
      "activations/layer10_attention_weight_min": -42.07273864746094,
      "activations/layer11_attention_weight_max": 44.18114471435547,
      "activations/layer11_attention_weight_min": -35.44755554199219,
      "activations/layer12_attention_weight_max": 28.156923294067383,
      "activations/layer12_attention_weight_min": -32.771881103515625,
      "activations/layer13_attention_weight_max": 38.93794631958008,
      "activations/layer13_attention_weight_min": -37.417381286621094,
      "activations/layer14_attention_weight_max": 42.61204147338867,
      "activations/layer14_attention_weight_min": -34.0716552734375,
      "activations/layer15_attention_weight_max": 37.17548751831055,
      "activations/layer15_attention_weight_min": -28.002666473388672,
      "activations/layer16_attention_weight_max": 27.194849014282227,
      "activations/layer16_attention_weight_min": -22.879453659057617,
      "activations/layer17_attention_weight_max": 20.812789916992188,
      "activations/layer17_attention_weight_min": -17.766984939575195,
      "activations/layer18_attention_weight_max": 20.90997886657715,
      "activations/layer18_attention_weight_min": -11.258654594421387,
      "activations/layer19_attention_weight_max": 23.153348922729492,
      "activations/layer19_attention_weight_min": -17.720579147338867,
      "activations/layer1_attention_weight_max": 13.667929649353027,
      "activations/layer1_attention_weight_min": -11.233277320861816,
      "activations/layer20_attention_weight_max": 19.355846405029297,
      "activations/layer20_attention_weight_min": -13.806801795959473,
      "activations/layer21_attention_weight_max": 23.32350730895996,
      "activations/layer21_attention_weight_min": -11.343974113464355,
      "activations/layer22_attention_weight_max": 25.979963302612305,
      "activations/layer22_attention_weight_min": -14.728976249694824,
      "activations/layer23_attention_weight_max": 19.354297637939453,
      "activations/layer23_attention_weight_min": -14.195780754089355,
      "activations/layer2_attention_weight_max": 28.442968368530273,
      "activations/layer2_attention_weight_min": -24.546016693115234,
      "activations/layer3_attention_weight_max": 55.38450622558594,
      "activations/layer3_attention_weight_min": -51.42308044433594,
      "activations/layer4_attention_weight_max": 116.61064910888672,
      "activations/layer4_attention_weight_min": -108.8312759399414,
      "activations/layer5_attention_weight_max": 131.0203857421875,
      "activations/layer5_attention_weight_min": -120.20622253417969,
      "activations/layer6_attention_weight_max": 75.82999420166016,
      "activations/layer6_attention_weight_min": -74.8060302734375,
      "activations/layer7_attention_weight_max": 74.14885711669922,
      "activations/layer7_attention_weight_min": -66.33488464355469,
      "activations/layer8_attention_weight_max": 67.86487579345703,
      "activations/layer8_attention_weight_min": -57.54595184326172,
      "activations/layer9_attention_weight_max": 67.96883392333984,
      "activations/layer9_attention_weight_min": -59.24658203125,
      "epoch": 0.53,
      "learning_rate": 0.00014803939393939392,
      "loss": 3.2664,
      "step": 9200
    },
    {
      "activations/layer0_attention_weight_max": 15.012575149536133,
      "activations/layer0_attention_weight_min": -11.589248657226562,
      "activations/layer10_attention_weight_max": 52.38266372680664,
      "activations/layer10_attention_weight_min": -42.45368576049805,
      "activations/layer11_attention_weight_max": 45.82781982421875,
      "activations/layer11_attention_weight_min": -37.10736083984375,
      "activations/layer12_attention_weight_max": 37.330265045166016,
      "activations/layer12_attention_weight_min": -27.17987823486328,
      "activations/layer13_attention_weight_max": 42.36184310913086,
      "activations/layer13_attention_weight_min": -31.054903030395508,
      "activations/layer14_attention_weight_max": 49.159996032714844,
      "activations/layer14_attention_weight_min": -38.920623779296875,
      "activations/layer15_attention_weight_max": 40.53572082519531,
      "activations/layer15_attention_weight_min": -27.381004333496094,
      "activations/layer16_attention_weight_max": 29.29600715637207,
      "activations/layer16_attention_weight_min": -24.438940048217773,
      "activations/layer17_attention_weight_max": 29.302087783813477,
      "activations/layer17_attention_weight_min": -15.250605583190918,
      "activations/layer18_attention_weight_max": 20.160232543945312,
      "activations/layer18_attention_weight_min": -11.421814918518066,
      "activations/layer19_attention_weight_max": 23.7382869720459,
      "activations/layer19_attention_weight_min": -16.120023727416992,
      "activations/layer1_attention_weight_max": 15.22421646118164,
      "activations/layer1_attention_weight_min": -11.941996574401855,
      "activations/layer20_attention_weight_max": 21.116832733154297,
      "activations/layer20_attention_weight_min": -12.531193733215332,
      "activations/layer21_attention_weight_max": 18.968727111816406,
      "activations/layer21_attention_weight_min": -11.997758865356445,
      "activations/layer22_attention_weight_max": 23.42514419555664,
      "activations/layer22_attention_weight_min": -15.850192070007324,
      "activations/layer23_attention_weight_max": 15.519535064697266,
      "activations/layer23_attention_weight_min": -12.535737991333008,
      "activations/layer2_attention_weight_max": 30.741741180419922,
      "activations/layer2_attention_weight_min": -24.932540893554688,
      "activations/layer3_attention_weight_max": 57.09147262573242,
      "activations/layer3_attention_weight_min": -52.27859115600586,
      "activations/layer4_attention_weight_max": 114.4048843383789,
      "activations/layer4_attention_weight_min": -105.42391204833984,
      "activations/layer5_attention_weight_max": 135.20501708984375,
      "activations/layer5_attention_weight_min": -114.61663055419922,
      "activations/layer6_attention_weight_max": 81.59835052490234,
      "activations/layer6_attention_weight_min": -69.56686401367188,
      "activations/layer7_attention_weight_max": 74.30561065673828,
      "activations/layer7_attention_weight_min": -62.493408203125,
      "activations/layer8_attention_weight_max": 69.89276885986328,
      "activations/layer8_attention_weight_min": -57.51203155517578,
      "activations/layer9_attention_weight_max": 80.08385467529297,
      "activations/layer9_attention_weight_min": -63.6820182800293,
      "epoch": 0.54,
      "learning_rate": 0.00014802045454545453,
      "loss": 3.2634,
      "step": 9250
    },
    {
      "activations/layer0_attention_weight_max": 15.050298690795898,
      "activations/layer0_attention_weight_min": -10.900097846984863,
      "activations/layer10_attention_weight_max": 48.16270446777344,
      "activations/layer10_attention_weight_min": -41.98648452758789,
      "activations/layer11_attention_weight_max": 50.66356658935547,
      "activations/layer11_attention_weight_min": -38.86475372314453,
      "activations/layer12_attention_weight_max": 36.06968307495117,
      "activations/layer12_attention_weight_min": -25.545215606689453,
      "activations/layer13_attention_weight_max": 34.98662567138672,
      "activations/layer13_attention_weight_min": -38.43829345703125,
      "activations/layer14_attention_weight_max": 43.60123825073242,
      "activations/layer14_attention_weight_min": -39.80360794067383,
      "activations/layer15_attention_weight_max": 36.81121063232422,
      "activations/layer15_attention_weight_min": -27.17561149597168,
      "activations/layer16_attention_weight_max": 29.0842227935791,
      "activations/layer16_attention_weight_min": -25.539018630981445,
      "activations/layer17_attention_weight_max": 25.957521438598633,
      "activations/layer17_attention_weight_min": -17.188432693481445,
      "activations/layer18_attention_weight_max": 19.3223934173584,
      "activations/layer18_attention_weight_min": -14.555255889892578,
      "activations/layer19_attention_weight_max": 23.02910804748535,
      "activations/layer19_attention_weight_min": -17.784021377563477,
      "activations/layer1_attention_weight_max": 12.876893043518066,
      "activations/layer1_attention_weight_min": -10.77488899230957,
      "activations/layer20_attention_weight_max": 17.888097763061523,
      "activations/layer20_attention_weight_min": -12.888558387756348,
      "activations/layer21_attention_weight_max": 18.564834594726562,
      "activations/layer21_attention_weight_min": -11.672259330749512,
      "activations/layer22_attention_weight_max": 30.809900283813477,
      "activations/layer22_attention_weight_min": -17.865692138671875,
      "activations/layer23_attention_weight_max": 20.484317779541016,
      "activations/layer23_attention_weight_min": -15.163740158081055,
      "activations/layer2_attention_weight_max": 30.056114196777344,
      "activations/layer2_attention_weight_min": -24.638641357421875,
      "activations/layer3_attention_weight_max": 53.770748138427734,
      "activations/layer3_attention_weight_min": -54.300132751464844,
      "activations/layer4_attention_weight_max": 113.4786376953125,
      "activations/layer4_attention_weight_min": -110.28759002685547,
      "activations/layer5_attention_weight_max": 140.97341918945312,
      "activations/layer5_attention_weight_min": -121.67952728271484,
      "activations/layer6_attention_weight_max": 83.90978240966797,
      "activations/layer6_attention_weight_min": -73.87930297851562,
      "activations/layer7_attention_weight_max": 78.63169860839844,
      "activations/layer7_attention_weight_min": -66.47713470458984,
      "activations/layer8_attention_weight_max": 71.21764373779297,
      "activations/layer8_attention_weight_min": -56.64772415161133,
      "activations/layer9_attention_weight_max": 101.32939910888672,
      "activations/layer9_attention_weight_min": -68.3659896850586,
      "epoch": 0.54,
      "learning_rate": 0.00014800151515151515,
      "loss": 3.268,
      "step": 9300
    },
    {
      "activations/layer0_attention_weight_max": 15.326212882995605,
      "activations/layer0_attention_weight_min": -10.607585906982422,
      "activations/layer10_attention_weight_max": 48.484310150146484,
      "activations/layer10_attention_weight_min": -42.5996208190918,
      "activations/layer11_attention_weight_max": 42.750667572021484,
      "activations/layer11_attention_weight_min": -36.06675338745117,
      "activations/layer12_attention_weight_max": 33.86113739013672,
      "activations/layer12_attention_weight_min": -25.204050064086914,
      "activations/layer13_attention_weight_max": 38.62845993041992,
      "activations/layer13_attention_weight_min": -37.32752227783203,
      "activations/layer14_attention_weight_max": 48.876304626464844,
      "activations/layer14_attention_weight_min": -39.31478500366211,
      "activations/layer15_attention_weight_max": 30.768342971801758,
      "activations/layer15_attention_weight_min": -28.144229888916016,
      "activations/layer16_attention_weight_max": 31.30782127380371,
      "activations/layer16_attention_weight_min": -24.52829933166504,
      "activations/layer17_attention_weight_max": 25.469432830810547,
      "activations/layer17_attention_weight_min": -18.77216148376465,
      "activations/layer18_attention_weight_max": 19.085811614990234,
      "activations/layer18_attention_weight_min": -12.390620231628418,
      "activations/layer19_attention_weight_max": 24.352155685424805,
      "activations/layer19_attention_weight_min": -22.781675338745117,
      "activations/layer1_attention_weight_max": 13.55744743347168,
      "activations/layer1_attention_weight_min": -10.630229949951172,
      "activations/layer20_attention_weight_max": 19.394590377807617,
      "activations/layer20_attention_weight_min": -13.47512149810791,
      "activations/layer21_attention_weight_max": 17.138242721557617,
      "activations/layer21_attention_weight_min": -13.144807815551758,
      "activations/layer22_attention_weight_max": 31.768667221069336,
      "activations/layer22_attention_weight_min": -17.06947898864746,
      "activations/layer23_attention_weight_max": 19.79146957397461,
      "activations/layer23_attention_weight_min": -13.947660446166992,
      "activations/layer2_attention_weight_max": 28.744428634643555,
      "activations/layer2_attention_weight_min": -24.376632690429688,
      "activations/layer3_attention_weight_max": 53.51654052734375,
      "activations/layer3_attention_weight_min": -50.66283416748047,
      "activations/layer4_attention_weight_max": 110.03850555419922,
      "activations/layer4_attention_weight_min": -105.72314453125,
      "activations/layer5_attention_weight_max": 125.28680419921875,
      "activations/layer5_attention_weight_min": -115.29371643066406,
      "activations/layer6_attention_weight_max": 77.5429916381836,
      "activations/layer6_attention_weight_min": -73.31558227539062,
      "activations/layer7_attention_weight_max": 72.53375244140625,
      "activations/layer7_attention_weight_min": -64.22772216796875,
      "activations/layer8_attention_weight_max": 67.78379821777344,
      "activations/layer8_attention_weight_min": -57.95561599731445,
      "activations/layer9_attention_weight_max": 61.410675048828125,
      "activations/layer9_attention_weight_min": -54.79024887084961,
      "epoch": 0.54,
      "learning_rate": 0.00014798257575757574,
      "loss": 3.2407,
      "step": 9350
    },
    {
      "activations/layer0_attention_weight_max": 14.924161911010742,
      "activations/layer0_attention_weight_min": -11.309123039245605,
      "activations/layer10_attention_weight_max": 53.89714813232422,
      "activations/layer10_attention_weight_min": -41.5302848815918,
      "activations/layer11_attention_weight_max": 48.018402099609375,
      "activations/layer11_attention_weight_min": -35.52909851074219,
      "activations/layer12_attention_weight_max": 28.420814514160156,
      "activations/layer12_attention_weight_min": -26.584962844848633,
      "activations/layer13_attention_weight_max": 39.56515121459961,
      "activations/layer13_attention_weight_min": -33.50326919555664,
      "activations/layer14_attention_weight_max": 57.63789367675781,
      "activations/layer14_attention_weight_min": -37.82563781738281,
      "activations/layer15_attention_weight_max": 36.9109992980957,
      "activations/layer15_attention_weight_min": -30.41645050048828,
      "activations/layer16_attention_weight_max": 28.640750885009766,
      "activations/layer16_attention_weight_min": -24.32734489440918,
      "activations/layer17_attention_weight_max": 21.301586151123047,
      "activations/layer17_attention_weight_min": -16.350194931030273,
      "activations/layer18_attention_weight_max": 20.244735717773438,
      "activations/layer18_attention_weight_min": -11.80352783203125,
      "activations/layer19_attention_weight_max": 23.743824005126953,
      "activations/layer19_attention_weight_min": -17.971710205078125,
      "activations/layer1_attention_weight_max": 13.502669334411621,
      "activations/layer1_attention_weight_min": -11.186689376831055,
      "activations/layer20_attention_weight_max": 20.88738250732422,
      "activations/layer20_attention_weight_min": -13.146164894104004,
      "activations/layer21_attention_weight_max": 18.80466079711914,
      "activations/layer21_attention_weight_min": -12.461217880249023,
      "activations/layer22_attention_weight_max": 27.081377029418945,
      "activations/layer22_attention_weight_min": -15.776230812072754,
      "activations/layer23_attention_weight_max": 18.200153350830078,
      "activations/layer23_attention_weight_min": -13.850399017333984,
      "activations/layer2_attention_weight_max": 28.094226837158203,
      "activations/layer2_attention_weight_min": -23.848730087280273,
      "activations/layer3_attention_weight_max": 55.165550231933594,
      "activations/layer3_attention_weight_min": -50.99414825439453,
      "activations/layer4_attention_weight_max": 118.2833480834961,
      "activations/layer4_attention_weight_min": -109.24181365966797,
      "activations/layer5_attention_weight_max": 139.00453186035156,
      "activations/layer5_attention_weight_min": -120.19417572021484,
      "activations/layer6_attention_weight_max": 82.11862182617188,
      "activations/layer6_attention_weight_min": -70.75436401367188,
      "activations/layer7_attention_weight_max": 72.87355041503906,
      "activations/layer7_attention_weight_min": -64.56585693359375,
      "activations/layer8_attention_weight_max": 68.05253601074219,
      "activations/layer8_attention_weight_min": -59.42197799682617,
      "activations/layer9_attention_weight_max": 76.1103744506836,
      "activations/layer9_attention_weight_min": -63.729305267333984,
      "epoch": 0.55,
      "learning_rate": 0.00014796363636363635,
      "loss": 3.2831,
      "step": 9400
    },
    {
      "activations/layer0_attention_weight_max": 16.028499603271484,
      "activations/layer0_attention_weight_min": -10.74047565460205,
      "activations/layer10_attention_weight_max": 52.11371994018555,
      "activations/layer10_attention_weight_min": -45.42247009277344,
      "activations/layer11_attention_weight_max": 47.280635833740234,
      "activations/layer11_attention_weight_min": -36.87393569946289,
      "activations/layer12_attention_weight_max": 36.095497131347656,
      "activations/layer12_attention_weight_min": -31.49673080444336,
      "activations/layer13_attention_weight_max": 39.05210876464844,
      "activations/layer13_attention_weight_min": -35.05426788330078,
      "activations/layer14_attention_weight_max": 50.50973129272461,
      "activations/layer14_attention_weight_min": -38.28117370605469,
      "activations/layer15_attention_weight_max": 40.529685974121094,
      "activations/layer15_attention_weight_min": -25.81899070739746,
      "activations/layer16_attention_weight_max": 33.87411880493164,
      "activations/layer16_attention_weight_min": -23.45807456970215,
      "activations/layer17_attention_weight_max": 28.60944366455078,
      "activations/layer17_attention_weight_min": -17.534330368041992,
      "activations/layer18_attention_weight_max": 20.036008834838867,
      "activations/layer18_attention_weight_min": -11.367445945739746,
      "activations/layer19_attention_weight_max": 32.78116989135742,
      "activations/layer19_attention_weight_min": -18.794200897216797,
      "activations/layer1_attention_weight_max": 14.631888389587402,
      "activations/layer1_attention_weight_min": -11.563860893249512,
      "activations/layer20_attention_weight_max": 23.18090057373047,
      "activations/layer20_attention_weight_min": -11.100852966308594,
      "activations/layer21_attention_weight_max": 19.948617935180664,
      "activations/layer21_attention_weight_min": -13.665144920349121,
      "activations/layer22_attention_weight_max": 28.8724422454834,
      "activations/layer22_attention_weight_min": -15.566691398620605,
      "activations/layer23_attention_weight_max": 18.690235137939453,
      "activations/layer23_attention_weight_min": -12.849456787109375,
      "activations/layer2_attention_weight_max": 29.42902946472168,
      "activations/layer2_attention_weight_min": -23.08240509033203,
      "activations/layer3_attention_weight_max": 53.78694152832031,
      "activations/layer3_attention_weight_min": -50.322242736816406,
      "activations/layer4_attention_weight_max": 112.1747055053711,
      "activations/layer4_attention_weight_min": -107.76008605957031,
      "activations/layer5_attention_weight_max": 138.1938934326172,
      "activations/layer5_attention_weight_min": -116.71751403808594,
      "activations/layer6_attention_weight_max": 83.55549621582031,
      "activations/layer6_attention_weight_min": -68.7950439453125,
      "activations/layer7_attention_weight_max": 79.6765365600586,
      "activations/layer7_attention_weight_min": -65.41404724121094,
      "activations/layer8_attention_weight_max": 74.08940887451172,
      "activations/layer8_attention_weight_min": -62.0457763671875,
      "activations/layer9_attention_weight_max": 74.0953140258789,
      "activations/layer9_attention_weight_min": -65.9432601928711,
      "epoch": 0.55,
      "learning_rate": 0.00014794469696969697,
      "loss": 3.2654,
      "step": 9450
    },
    {
      "activations/layer0_attention_weight_max": 15.130511283874512,
      "activations/layer0_attention_weight_min": -11.627240180969238,
      "activations/layer10_attention_weight_max": 51.421409606933594,
      "activations/layer10_attention_weight_min": -42.91411209106445,
      "activations/layer11_attention_weight_max": 53.385398864746094,
      "activations/layer11_attention_weight_min": -41.519126892089844,
      "activations/layer12_attention_weight_max": 27.755023956298828,
      "activations/layer12_attention_weight_min": -24.451005935668945,
      "activations/layer13_attention_weight_max": 38.83561706542969,
      "activations/layer13_attention_weight_min": -32.46501922607422,
      "activations/layer14_attention_weight_max": 55.75523376464844,
      "activations/layer14_attention_weight_min": -39.67673110961914,
      "activations/layer15_attention_weight_max": 35.721336364746094,
      "activations/layer15_attention_weight_min": -26.13345718383789,
      "activations/layer16_attention_weight_max": 33.50941467285156,
      "activations/layer16_attention_weight_min": -23.016002655029297,
      "activations/layer17_attention_weight_max": 33.561729431152344,
      "activations/layer17_attention_weight_min": -17.26192283630371,
      "activations/layer18_attention_weight_max": 22.229387283325195,
      "activations/layer18_attention_weight_min": -13.44893741607666,
      "activations/layer19_attention_weight_max": 32.16946029663086,
      "activations/layer19_attention_weight_min": -16.96894645690918,
      "activations/layer1_attention_weight_max": 13.604331970214844,
      "activations/layer1_attention_weight_min": -12.048015594482422,
      "activations/layer20_attention_weight_max": 27.0044002532959,
      "activations/layer20_attention_weight_min": -13.972033500671387,
      "activations/layer21_attention_weight_max": 24.35930061340332,
      "activations/layer21_attention_weight_min": -11.327903747558594,
      "activations/layer22_attention_weight_max": 34.37139892578125,
      "activations/layer22_attention_weight_min": -13.591291427612305,
      "activations/layer23_attention_weight_max": 24.880483627319336,
      "activations/layer23_attention_weight_min": -12.1859769821167,
      "activations/layer2_attention_weight_max": 29.127166748046875,
      "activations/layer2_attention_weight_min": -24.128002166748047,
      "activations/layer3_attention_weight_max": 55.823726654052734,
      "activations/layer3_attention_weight_min": -52.32426071166992,
      "activations/layer4_attention_weight_max": 115.17647552490234,
      "activations/layer4_attention_weight_min": -105.93062591552734,
      "activations/layer5_attention_weight_max": 133.61814880371094,
      "activations/layer5_attention_weight_min": -117.02864074707031,
      "activations/layer6_attention_weight_max": 78.7401123046875,
      "activations/layer6_attention_weight_min": -69.8711166381836,
      "activations/layer7_attention_weight_max": 73.81029510498047,
      "activations/layer7_attention_weight_min": -65.74740600585938,
      "activations/layer8_attention_weight_max": 73.5381851196289,
      "activations/layer8_attention_weight_min": -61.40196990966797,
      "activations/layer9_attention_weight_max": 69.88910675048828,
      "activations/layer9_attention_weight_min": -59.2064323425293,
      "epoch": 0.55,
      "learning_rate": 0.00014792575757575756,
      "loss": 3.2617,
      "step": 9500
    },
    {
      "activations/layer0_attention_weight_max": 16.120229721069336,
      "activations/layer0_attention_weight_min": -11.216458320617676,
      "activations/layer10_attention_weight_max": 49.61532974243164,
      "activations/layer10_attention_weight_min": -43.66822052001953,
      "activations/layer11_attention_weight_max": 45.46315002441406,
      "activations/layer11_attention_weight_min": -44.0499267578125,
      "activations/layer12_attention_weight_max": 33.154239654541016,
      "activations/layer12_attention_weight_min": -29.367204666137695,
      "activations/layer13_attention_weight_max": 37.85380554199219,
      "activations/layer13_attention_weight_min": -34.237037658691406,
      "activations/layer14_attention_weight_max": 46.86555099487305,
      "activations/layer14_attention_weight_min": -38.92565155029297,
      "activations/layer15_attention_weight_max": 30.885948181152344,
      "activations/layer15_attention_weight_min": -26.9443302154541,
      "activations/layer16_attention_weight_max": 29.928165435791016,
      "activations/layer16_attention_weight_min": -30.357410430908203,
      "activations/layer17_attention_weight_max": 30.21158218383789,
      "activations/layer17_attention_weight_min": -21.010953903198242,
      "activations/layer18_attention_weight_max": 19.281639099121094,
      "activations/layer18_attention_weight_min": -12.686413764953613,
      "activations/layer19_attention_weight_max": 24.21849822998047,
      "activations/layer19_attention_weight_min": -18.472858428955078,
      "activations/layer1_attention_weight_max": 13.594583511352539,
      "activations/layer1_attention_weight_min": -11.978684425354004,
      "activations/layer20_attention_weight_max": 22.315988540649414,
      "activations/layer20_attention_weight_min": -12.567282676696777,
      "activations/layer21_attention_weight_max": 19.303485870361328,
      "activations/layer21_attention_weight_min": -12.130293846130371,
      "activations/layer22_attention_weight_max": 27.22989273071289,
      "activations/layer22_attention_weight_min": -16.162981033325195,
      "activations/layer23_attention_weight_max": 18.545269012451172,
      "activations/layer23_attention_weight_min": -14.271953582763672,
      "activations/layer2_attention_weight_max": 27.065261840820312,
      "activations/layer2_attention_weight_min": -22.494932174682617,
      "activations/layer3_attention_weight_max": 52.41978454589844,
      "activations/layer3_attention_weight_min": -48.562843322753906,
      "activations/layer4_attention_weight_max": 110.798583984375,
      "activations/layer4_attention_weight_min": -101.43346405029297,
      "activations/layer5_attention_weight_max": 127.50325775146484,
      "activations/layer5_attention_weight_min": -111.4596176147461,
      "activations/layer6_attention_weight_max": 75.03130340576172,
      "activations/layer6_attention_weight_min": -68.83264923095703,
      "activations/layer7_attention_weight_max": 73.7694320678711,
      "activations/layer7_attention_weight_min": -64.42542266845703,
      "activations/layer8_attention_weight_max": 69.66936492919922,
      "activations/layer8_attention_weight_min": -58.30615234375,
      "activations/layer9_attention_weight_max": 61.754695892333984,
      "activations/layer9_attention_weight_min": -60.36495590209961,
      "epoch": 0.55,
      "learning_rate": 0.00014790681818181817,
      "loss": 3.2494,
      "step": 9550
    },
    {
      "activations/layer0_attention_weight_max": 12.882329940795898,
      "activations/layer0_attention_weight_min": -11.161264419555664,
      "activations/layer10_attention_weight_max": 59.73815155029297,
      "activations/layer10_attention_weight_min": -47.24192428588867,
      "activations/layer11_attention_weight_max": 52.767547607421875,
      "activations/layer11_attention_weight_min": -37.48023223876953,
      "activations/layer12_attention_weight_max": 37.55657958984375,
      "activations/layer12_attention_weight_min": -25.497167587280273,
      "activations/layer13_attention_weight_max": 54.150264739990234,
      "activations/layer13_attention_weight_min": -35.87401580810547,
      "activations/layer14_attention_weight_max": 60.651912689208984,
      "activations/layer14_attention_weight_min": -35.38911056518555,
      "activations/layer15_attention_weight_max": 38.45241165161133,
      "activations/layer15_attention_weight_min": -24.974016189575195,
      "activations/layer16_attention_weight_max": 36.0370979309082,
      "activations/layer16_attention_weight_min": -21.278018951416016,
      "activations/layer17_attention_weight_max": 23.994531631469727,
      "activations/layer17_attention_weight_min": -15.245710372924805,
      "activations/layer18_attention_weight_max": 20.668901443481445,
      "activations/layer18_attention_weight_min": -11.831175804138184,
      "activations/layer19_attention_weight_max": 30.79146385192871,
      "activations/layer19_attention_weight_min": -17.066526412963867,
      "activations/layer1_attention_weight_max": 13.793389320373535,
      "activations/layer1_attention_weight_min": -12.652692794799805,
      "activations/layer20_attention_weight_max": 21.61872673034668,
      "activations/layer20_attention_weight_min": -11.984591484069824,
      "activations/layer21_attention_weight_max": 19.911758422851562,
      "activations/layer21_attention_weight_min": -12.57227611541748,
      "activations/layer22_attention_weight_max": 34.473575592041016,
      "activations/layer22_attention_weight_min": -15.875006675720215,
      "activations/layer23_attention_weight_max": 26.936595916748047,
      "activations/layer23_attention_weight_min": -15.111711502075195,
      "activations/layer2_attention_weight_max": 28.110538482666016,
      "activations/layer2_attention_weight_min": -23.857666015625,
      "activations/layer3_attention_weight_max": 54.216278076171875,
      "activations/layer3_attention_weight_min": -53.03941345214844,
      "activations/layer4_attention_weight_max": 114.18917083740234,
      "activations/layer4_attention_weight_min": -109.84062957763672,
      "activations/layer5_attention_weight_max": 131.0823211669922,
      "activations/layer5_attention_weight_min": -120.81549072265625,
      "activations/layer6_attention_weight_max": 78.88685607910156,
      "activations/layer6_attention_weight_min": -68.25025939941406,
      "activations/layer7_attention_weight_max": 78.25775909423828,
      "activations/layer7_attention_weight_min": -68.71720886230469,
      "activations/layer8_attention_weight_max": 69.62024688720703,
      "activations/layer8_attention_weight_min": -59.73125457763672,
      "activations/layer9_attention_weight_max": 77.22933197021484,
      "activations/layer9_attention_weight_min": -65.29253387451172,
      "epoch": 0.56,
      "learning_rate": 0.00014788787878787879,
      "loss": 3.2741,
      "step": 9600
    },
    {
      "activations/layer0_attention_weight_max": 16.672454833984375,
      "activations/layer0_attention_weight_min": -10.73098087310791,
      "activations/layer10_attention_weight_max": 50.21637725830078,
      "activations/layer10_attention_weight_min": -46.99485778808594,
      "activations/layer11_attention_weight_max": 46.28045654296875,
      "activations/layer11_attention_weight_min": -39.38918685913086,
      "activations/layer12_attention_weight_max": 45.79505920410156,
      "activations/layer12_attention_weight_min": -33.461265563964844,
      "activations/layer13_attention_weight_max": 36.5950813293457,
      "activations/layer13_attention_weight_min": -34.4245719909668,
      "activations/layer14_attention_weight_max": 47.67763137817383,
      "activations/layer14_attention_weight_min": -40.7995491027832,
      "activations/layer15_attention_weight_max": 31.238122940063477,
      "activations/layer15_attention_weight_min": -24.15970230102539,
      "activations/layer16_attention_weight_max": 26.109094619750977,
      "activations/layer16_attention_weight_min": -24.733488082885742,
      "activations/layer17_attention_weight_max": 25.471179962158203,
      "activations/layer17_attention_weight_min": -18.00091552734375,
      "activations/layer18_attention_weight_max": 18.008529663085938,
      "activations/layer18_attention_weight_min": -12.911527633666992,
      "activations/layer19_attention_weight_max": 22.529052734375,
      "activations/layer19_attention_weight_min": -20.863046646118164,
      "activations/layer1_attention_weight_max": 13.498499870300293,
      "activations/layer1_attention_weight_min": -11.236533164978027,
      "activations/layer20_attention_weight_max": 20.71026039123535,
      "activations/layer20_attention_weight_min": -14.612512588500977,
      "activations/layer21_attention_weight_max": 17.069828033447266,
      "activations/layer21_attention_weight_min": -10.961187362670898,
      "activations/layer22_attention_weight_max": 27.080320358276367,
      "activations/layer22_attention_weight_min": -15.02004337310791,
      "activations/layer23_attention_weight_max": 18.472679138183594,
      "activations/layer23_attention_weight_min": -13.710193634033203,
      "activations/layer2_attention_weight_max": 28.573856353759766,
      "activations/layer2_attention_weight_min": -24.42803955078125,
      "activations/layer3_attention_weight_max": 54.87416458129883,
      "activations/layer3_attention_weight_min": -51.73774337768555,
      "activations/layer4_attention_weight_max": 110.43218231201172,
      "activations/layer4_attention_weight_min": -102.194580078125,
      "activations/layer5_attention_weight_max": 128.57200622558594,
      "activations/layer5_attention_weight_min": -119.16450500488281,
      "activations/layer6_attention_weight_max": 76.10438537597656,
      "activations/layer6_attention_weight_min": -69.2141342163086,
      "activations/layer7_attention_weight_max": 72.46365356445312,
      "activations/layer7_attention_weight_min": -68.00376892089844,
      "activations/layer8_attention_weight_max": 64.50405883789062,
      "activations/layer8_attention_weight_min": -59.92000961303711,
      "activations/layer9_attention_weight_max": 75.84595489501953,
      "activations/layer9_attention_weight_min": -65.20287322998047,
      "epoch": 0.56,
      "learning_rate": 0.00014786893939393937,
      "loss": 3.2563,
      "step": 9650
    },
    {
      "activations/layer0_attention_weight_max": 14.456923484802246,
      "activations/layer0_attention_weight_min": -10.684895515441895,
      "activations/layer10_attention_weight_max": 51.2512321472168,
      "activations/layer10_attention_weight_min": -41.9698600769043,
      "activations/layer11_attention_weight_max": 53.24909591674805,
      "activations/layer11_attention_weight_min": -34.95326232910156,
      "activations/layer12_attention_weight_max": 91.85774230957031,
      "activations/layer12_attention_weight_min": -42.680938720703125,
      "activations/layer13_attention_weight_max": 79.22010803222656,
      "activations/layer13_attention_weight_min": -40.17503356933594,
      "activations/layer14_attention_weight_max": 51.77027130126953,
      "activations/layer14_attention_weight_min": -39.282806396484375,
      "activations/layer15_attention_weight_max": 30.55933380126953,
      "activations/layer15_attention_weight_min": -23.869808197021484,
      "activations/layer16_attention_weight_max": 29.11783218383789,
      "activations/layer16_attention_weight_min": -22.042640686035156,
      "activations/layer17_attention_weight_max": 19.35919952392578,
      "activations/layer17_attention_weight_min": -16.503671646118164,
      "activations/layer18_attention_weight_max": 16.498470306396484,
      "activations/layer18_attention_weight_min": -9.980101585388184,
      "activations/layer19_attention_weight_max": 23.334928512573242,
      "activations/layer19_attention_weight_min": -16.586030960083008,
      "activations/layer1_attention_weight_max": 14.23269271850586,
      "activations/layer1_attention_weight_min": -10.413985252380371,
      "activations/layer20_attention_weight_max": 16.447128295898438,
      "activations/layer20_attention_weight_min": -11.567172050476074,
      "activations/layer21_attention_weight_max": 15.44273567199707,
      "activations/layer21_attention_weight_min": -10.130134582519531,
      "activations/layer22_attention_weight_max": 25.143247604370117,
      "activations/layer22_attention_weight_min": -12.890671730041504,
      "activations/layer23_attention_weight_max": 16.848678588867188,
      "activations/layer23_attention_weight_min": -12.884177207946777,
      "activations/layer2_attention_weight_max": 28.90593719482422,
      "activations/layer2_attention_weight_min": -24.092802047729492,
      "activations/layer3_attention_weight_max": 55.001461029052734,
      "activations/layer3_attention_weight_min": -52.63750457763672,
      "activations/layer4_attention_weight_max": 114.90058898925781,
      "activations/layer4_attention_weight_min": -107.10497283935547,
      "activations/layer5_attention_weight_max": 134.16848754882812,
      "activations/layer5_attention_weight_min": -120.8065185546875,
      "activations/layer6_attention_weight_max": 77.7481918334961,
      "activations/layer6_attention_weight_min": -66.43707275390625,
      "activations/layer7_attention_weight_max": 71.44146728515625,
      "activations/layer7_attention_weight_min": -62.48948669433594,
      "activations/layer8_attention_weight_max": 73.9844741821289,
      "activations/layer8_attention_weight_min": -61.96485900878906,
      "activations/layer9_attention_weight_max": 95.67904663085938,
      "activations/layer9_attention_weight_min": -64.58802032470703,
      "epoch": 0.56,
      "learning_rate": 0.00014785,
      "loss": 3.2499,
      "step": 9700
    },
    {
      "activations/layer0_attention_weight_max": 15.64724063873291,
      "activations/layer0_attention_weight_min": -10.551056861877441,
      "activations/layer10_attention_weight_max": 50.646484375,
      "activations/layer10_attention_weight_min": -44.933841705322266,
      "activations/layer11_attention_weight_max": 49.11149215698242,
      "activations/layer11_attention_weight_min": -41.60265350341797,
      "activations/layer12_attention_weight_max": 29.655195236206055,
      "activations/layer12_attention_weight_min": -24.323747634887695,
      "activations/layer13_attention_weight_max": 42.705101013183594,
      "activations/layer13_attention_weight_min": -35.44892120361328,
      "activations/layer14_attention_weight_max": 53.12824249267578,
      "activations/layer14_attention_weight_min": -41.71495819091797,
      "activations/layer15_attention_weight_max": 37.49934005737305,
      "activations/layer15_attention_weight_min": -29.172523498535156,
      "activations/layer16_attention_weight_max": 32.87026596069336,
      "activations/layer16_attention_weight_min": -22.03525161743164,
      "activations/layer17_attention_weight_max": 30.431222915649414,
      "activations/layer17_attention_weight_min": -18.64874267578125,
      "activations/layer18_attention_weight_max": 20.40789031982422,
      "activations/layer18_attention_weight_min": -11.714734077453613,
      "activations/layer19_attention_weight_max": 30.259366989135742,
      "activations/layer19_attention_weight_min": -20.00922966003418,
      "activations/layer1_attention_weight_max": 14.376558303833008,
      "activations/layer1_attention_weight_min": -11.132545471191406,
      "activations/layer20_attention_weight_max": 25.77911949157715,
      "activations/layer20_attention_weight_min": -12.073894500732422,
      "activations/layer21_attention_weight_max": 21.27664566040039,
      "activations/layer21_attention_weight_min": -10.797825813293457,
      "activations/layer22_attention_weight_max": 27.718652725219727,
      "activations/layer22_attention_weight_min": -17.01878547668457,
      "activations/layer23_attention_weight_max": 19.285402297973633,
      "activations/layer23_attention_weight_min": -14.449684143066406,
      "activations/layer2_attention_weight_max": 28.816204071044922,
      "activations/layer2_attention_weight_min": -23.4979248046875,
      "activations/layer3_attention_weight_max": 58.15031433105469,
      "activations/layer3_attention_weight_min": -53.80910110473633,
      "activations/layer4_attention_weight_max": 113.92351531982422,
      "activations/layer4_attention_weight_min": -105.48421478271484,
      "activations/layer5_attention_weight_max": 133.46463012695312,
      "activations/layer5_attention_weight_min": -116.74129486083984,
      "activations/layer6_attention_weight_max": 79.1063232421875,
      "activations/layer6_attention_weight_min": -69.7278060913086,
      "activations/layer7_attention_weight_max": 76.89262390136719,
      "activations/layer7_attention_weight_min": -67.43875885009766,
      "activations/layer8_attention_weight_max": 68.57218170166016,
      "activations/layer8_attention_weight_min": -57.52197265625,
      "activations/layer9_attention_weight_max": 70.92975616455078,
      "activations/layer9_attention_weight_min": -64.61408233642578,
      "epoch": 0.57,
      "learning_rate": 0.00014783106060606058,
      "loss": 3.2382,
      "step": 9750
    },
    {
      "activations/layer0_attention_weight_max": 15.359374046325684,
      "activations/layer0_attention_weight_min": -10.66417121887207,
      "activations/layer10_attention_weight_max": 48.104942321777344,
      "activations/layer10_attention_weight_min": -44.03181076049805,
      "activations/layer11_attention_weight_max": 45.62367248535156,
      "activations/layer11_attention_weight_min": -37.03654479980469,
      "activations/layer12_attention_weight_max": 33.21009063720703,
      "activations/layer12_attention_weight_min": -25.028594970703125,
      "activations/layer13_attention_weight_max": 37.95284652709961,
      "activations/layer13_attention_weight_min": -33.14319610595703,
      "activations/layer14_attention_weight_max": 49.2886848449707,
      "activations/layer14_attention_weight_min": -39.97666931152344,
      "activations/layer15_attention_weight_max": 32.44095993041992,
      "activations/layer15_attention_weight_min": -25.33770751953125,
      "activations/layer16_attention_weight_max": 28.668184280395508,
      "activations/layer16_attention_weight_min": -23.923294067382812,
      "activations/layer17_attention_weight_max": 23.626249313354492,
      "activations/layer17_attention_weight_min": -17.33921241760254,
      "activations/layer18_attention_weight_max": 18.234655380249023,
      "activations/layer18_attention_weight_min": -12.068923950195312,
      "activations/layer19_attention_weight_max": 24.599580764770508,
      "activations/layer19_attention_weight_min": -17.900617599487305,
      "activations/layer1_attention_weight_max": 13.459151268005371,
      "activations/layer1_attention_weight_min": -11.939749717712402,
      "activations/layer20_attention_weight_max": 18.803813934326172,
      "activations/layer20_attention_weight_min": -10.111641883850098,
      "activations/layer21_attention_weight_max": 17.78481101989746,
      "activations/layer21_attention_weight_min": -9.759819030761719,
      "activations/layer22_attention_weight_max": 26.134763717651367,
      "activations/layer22_attention_weight_min": -17.231626510620117,
      "activations/layer23_attention_weight_max": 18.94387435913086,
      "activations/layer23_attention_weight_min": -13.574660301208496,
      "activations/layer2_attention_weight_max": 28.226730346679688,
      "activations/layer2_attention_weight_min": -24.71466636657715,
      "activations/layer3_attention_weight_max": 56.15402603149414,
      "activations/layer3_attention_weight_min": -52.85028839111328,
      "activations/layer4_attention_weight_max": 109.94178771972656,
      "activations/layer4_attention_weight_min": -107.599609375,
      "activations/layer5_attention_weight_max": 127.23542785644531,
      "activations/layer5_attention_weight_min": -116.77485656738281,
      "activations/layer6_attention_weight_max": 74.81660461425781,
      "activations/layer6_attention_weight_min": -67.31703186035156,
      "activations/layer7_attention_weight_max": 71.48573303222656,
      "activations/layer7_attention_weight_min": -63.12199401855469,
      "activations/layer8_attention_weight_max": 75.2308349609375,
      "activations/layer8_attention_weight_min": -62.86848449707031,
      "activations/layer9_attention_weight_max": 65.6685791015625,
      "activations/layer9_attention_weight_min": -59.1290283203125,
      "epoch": 0.57,
      "learning_rate": 0.0001478121212121212,
      "loss": 3.2575,
      "step": 9800
    },
    {
      "activations/layer0_attention_weight_max": 16.983989715576172,
      "activations/layer0_attention_weight_min": -11.332999229431152,
      "activations/layer10_attention_weight_max": 54.33658218383789,
      "activations/layer10_attention_weight_min": -44.2409553527832,
      "activations/layer11_attention_weight_max": 58.625789642333984,
      "activations/layer11_attention_weight_min": -41.45121383666992,
      "activations/layer12_attention_weight_max": 43.15557861328125,
      "activations/layer12_attention_weight_min": -39.399532318115234,
      "activations/layer13_attention_weight_max": 44.93280792236328,
      "activations/layer13_attention_weight_min": -41.38226318359375,
      "activations/layer14_attention_weight_max": 56.01919174194336,
      "activations/layer14_attention_weight_min": -41.167423248291016,
      "activations/layer15_attention_weight_max": 42.088809967041016,
      "activations/layer15_attention_weight_min": -34.01197814941406,
      "activations/layer16_attention_weight_max": 29.071430206298828,
      "activations/layer16_attention_weight_min": -27.447280883789062,
      "activations/layer17_attention_weight_max": 27.084983825683594,
      "activations/layer17_attention_weight_min": -20.119169235229492,
      "activations/layer18_attention_weight_max": 19.694787979125977,
      "activations/layer18_attention_weight_min": -15.978975296020508,
      "activations/layer19_attention_weight_max": 25.20347785949707,
      "activations/layer19_attention_weight_min": -24.334016799926758,
      "activations/layer1_attention_weight_max": 14.687875747680664,
      "activations/layer1_attention_weight_min": -10.862960815429688,
      "activations/layer20_attention_weight_max": 22.257158279418945,
      "activations/layer20_attention_weight_min": -19.912939071655273,
      "activations/layer21_attention_weight_max": 22.438894271850586,
      "activations/layer21_attention_weight_min": -16.212018966674805,
      "activations/layer22_attention_weight_max": 32.908592224121094,
      "activations/layer22_attention_weight_min": -16.16553497314453,
      "activations/layer23_attention_weight_max": 22.694541931152344,
      "activations/layer23_attention_weight_min": -16.94473648071289,
      "activations/layer2_attention_weight_max": 28.146604537963867,
      "activations/layer2_attention_weight_min": -23.264461517333984,
      "activations/layer3_attention_weight_max": 53.91339874267578,
      "activations/layer3_attention_weight_min": -52.797645568847656,
      "activations/layer4_attention_weight_max": 112.81624603271484,
      "activations/layer4_attention_weight_min": -111.15563201904297,
      "activations/layer5_attention_weight_max": 134.84324645996094,
      "activations/layer5_attention_weight_min": -116.24008178710938,
      "activations/layer6_attention_weight_max": 76.44825744628906,
      "activations/layer6_attention_weight_min": -69.49986267089844,
      "activations/layer7_attention_weight_max": 75.87715911865234,
      "activations/layer7_attention_weight_min": -68.89550018310547,
      "activations/layer8_attention_weight_max": 68.41300964355469,
      "activations/layer8_attention_weight_min": -60.60924530029297,
      "activations/layer9_attention_weight_max": 76.0621109008789,
      "activations/layer9_attention_weight_min": -70.19930267333984,
      "epoch": 0.57,
      "learning_rate": 0.0001477931818181818,
      "loss": 3.2575,
      "step": 9850
    },
    {
      "activations/layer0_attention_weight_max": 15.493971824645996,
      "activations/layer0_attention_weight_min": -10.832772254943848,
      "activations/layer10_attention_weight_max": 50.41556930541992,
      "activations/layer10_attention_weight_min": -45.55543518066406,
      "activations/layer11_attention_weight_max": 46.17983627319336,
      "activations/layer11_attention_weight_min": -36.822509765625,
      "activations/layer12_attention_weight_max": 33.20634841918945,
      "activations/layer12_attention_weight_min": -30.143367767333984,
      "activations/layer13_attention_weight_max": 40.95247268676758,
      "activations/layer13_attention_weight_min": -33.9758186340332,
      "activations/layer14_attention_weight_max": 53.911808013916016,
      "activations/layer14_attention_weight_min": -40.402828216552734,
      "activations/layer15_attention_weight_max": 34.93327331542969,
      "activations/layer15_attention_weight_min": -24.35492515563965,
      "activations/layer16_attention_weight_max": 41.12928009033203,
      "activations/layer16_attention_weight_min": -26.027427673339844,
      "activations/layer17_attention_weight_max": 33.732120513916016,
      "activations/layer17_attention_weight_min": -16.9376220703125,
      "activations/layer18_attention_weight_max": 27.673959732055664,
      "activations/layer18_attention_weight_min": -12.574853897094727,
      "activations/layer19_attention_weight_max": 36.93008041381836,
      "activations/layer19_attention_weight_min": -20.698486328125,
      "activations/layer1_attention_weight_max": 14.816136360168457,
      "activations/layer1_attention_weight_min": -12.181697845458984,
      "activations/layer20_attention_weight_max": 25.526676177978516,
      "activations/layer20_attention_weight_min": -14.805691719055176,
      "activations/layer21_attention_weight_max": 23.729991912841797,
      "activations/layer21_attention_weight_min": -12.780946731567383,
      "activations/layer22_attention_weight_max": 33.47480392456055,
      "activations/layer22_attention_weight_min": -17.427814483642578,
      "activations/layer23_attention_weight_max": 25.853374481201172,
      "activations/layer23_attention_weight_min": -15.928454399108887,
      "activations/layer2_attention_weight_max": 29.281909942626953,
      "activations/layer2_attention_weight_min": -24.786251068115234,
      "activations/layer3_attention_weight_max": 55.2863655090332,
      "activations/layer3_attention_weight_min": -51.59512710571289,
      "activations/layer4_attention_weight_max": 113.29267883300781,
      "activations/layer4_attention_weight_min": -104.59654998779297,
      "activations/layer5_attention_weight_max": 128.59906005859375,
      "activations/layer5_attention_weight_min": -113.76644134521484,
      "activations/layer6_attention_weight_max": 78.31920623779297,
      "activations/layer6_attention_weight_min": -70.08123779296875,
      "activations/layer7_attention_weight_max": 71.54386138916016,
      "activations/layer7_attention_weight_min": -64.30433654785156,
      "activations/layer8_attention_weight_max": 71.32510375976562,
      "activations/layer8_attention_weight_min": -60.3392219543457,
      "activations/layer9_attention_weight_max": 66.80113983154297,
      "activations/layer9_attention_weight_min": -61.308349609375,
      "epoch": 0.58,
      "learning_rate": 0.0001477742424242424,
      "loss": 3.2498,
      "step": 9900
    },
    {
      "activations/layer0_attention_weight_max": 14.12928581237793,
      "activations/layer0_attention_weight_min": -10.5126953125,
      "activations/layer10_attention_weight_max": 50.093505859375,
      "activations/layer10_attention_weight_min": -40.39274215698242,
      "activations/layer11_attention_weight_max": 49.4045524597168,
      "activations/layer11_attention_weight_min": -35.8720703125,
      "activations/layer12_attention_weight_max": 37.2714729309082,
      "activations/layer12_attention_weight_min": -28.374223709106445,
      "activations/layer13_attention_weight_max": 41.178977966308594,
      "activations/layer13_attention_weight_min": -36.12379837036133,
      "activations/layer14_attention_weight_max": 59.14876937866211,
      "activations/layer14_attention_weight_min": -41.017757415771484,
      "activations/layer15_attention_weight_max": 35.7285270690918,
      "activations/layer15_attention_weight_min": -28.148244857788086,
      "activations/layer16_attention_weight_max": 34.97893524169922,
      "activations/layer16_attention_weight_min": -24.703493118286133,
      "activations/layer17_attention_weight_max": 22.4761962890625,
      "activations/layer17_attention_weight_min": -15.835232734680176,
      "activations/layer18_attention_weight_max": 20.68497085571289,
      "activations/layer18_attention_weight_min": -11.154863357543945,
      "activations/layer19_attention_weight_max": 26.6600341796875,
      "activations/layer19_attention_weight_min": -19.53475570678711,
      "activations/layer1_attention_weight_max": 13.4839506149292,
      "activations/layer1_attention_weight_min": -12.083332061767578,
      "activations/layer20_attention_weight_max": 20.971738815307617,
      "activations/layer20_attention_weight_min": -11.288573265075684,
      "activations/layer21_attention_weight_max": 17.750959396362305,
      "activations/layer21_attention_weight_min": -11.782038688659668,
      "activations/layer22_attention_weight_max": 29.80126953125,
      "activations/layer22_attention_weight_min": -16.147424697875977,
      "activations/layer23_attention_weight_max": 19.52871322631836,
      "activations/layer23_attention_weight_min": -16.861225128173828,
      "activations/layer2_attention_weight_max": 29.708637237548828,
      "activations/layer2_attention_weight_min": -24.06806182861328,
      "activations/layer3_attention_weight_max": 55.48971176147461,
      "activations/layer3_attention_weight_min": -51.804378509521484,
      "activations/layer4_attention_weight_max": 112.16910552978516,
      "activations/layer4_attention_weight_min": -109.0366439819336,
      "activations/layer5_attention_weight_max": 133.48440551757812,
      "activations/layer5_attention_weight_min": -117.24862670898438,
      "activations/layer6_attention_weight_max": 77.13106536865234,
      "activations/layer6_attention_weight_min": -70.31927490234375,
      "activations/layer7_attention_weight_max": 75.7398910522461,
      "activations/layer7_attention_weight_min": -62.658447265625,
      "activations/layer8_attention_weight_max": 68.73466491699219,
      "activations/layer8_attention_weight_min": -54.11954879760742,
      "activations/layer9_attention_weight_max": 78.62211608886719,
      "activations/layer9_attention_weight_min": -61.413326263427734,
      "epoch": 0.58,
      "learning_rate": 0.000147755303030303,
      "loss": 3.2533,
      "step": 9950
    },
    {
      "activations/layer0_attention_weight_max": 15.319804191589355,
      "activations/layer0_attention_weight_min": -10.36098861694336,
      "activations/layer10_attention_weight_max": 48.21659469604492,
      "activations/layer10_attention_weight_min": -41.6724967956543,
      "activations/layer11_attention_weight_max": 48.185028076171875,
      "activations/layer11_attention_weight_min": -39.23397445678711,
      "activations/layer12_attention_weight_max": 47.21379852294922,
      "activations/layer12_attention_weight_min": -31.135820388793945,
      "activations/layer13_attention_weight_max": 46.588382720947266,
      "activations/layer13_attention_weight_min": -33.5938835144043,
      "activations/layer14_attention_weight_max": 60.35242462158203,
      "activations/layer14_attention_weight_min": -40.37967300415039,
      "activations/layer15_attention_weight_max": 36.925865173339844,
      "activations/layer15_attention_weight_min": -25.086570739746094,
      "activations/layer16_attention_weight_max": 32.01506042480469,
      "activations/layer16_attention_weight_min": -23.012516021728516,
      "activations/layer17_attention_weight_max": 21.491313934326172,
      "activations/layer17_attention_weight_min": -17.444955825805664,
      "activations/layer18_attention_weight_max": 21.514387130737305,
      "activations/layer18_attention_weight_min": -12.889325141906738,
      "activations/layer19_attention_weight_max": 24.026473999023438,
      "activations/layer19_attention_weight_min": -17.84918212890625,
      "activations/layer1_attention_weight_max": 13.313658714294434,
      "activations/layer1_attention_weight_min": -11.869683265686035,
      "activations/layer20_attention_weight_max": 19.934762954711914,
      "activations/layer20_attention_weight_min": -13.377859115600586,
      "activations/layer21_attention_weight_max": 23.484344482421875,
      "activations/layer21_attention_weight_min": -12.038093566894531,
      "activations/layer22_attention_weight_max": 30.553993225097656,
      "activations/layer22_attention_weight_min": -16.693885803222656,
      "activations/layer23_attention_weight_max": 21.372793197631836,
      "activations/layer23_attention_weight_min": -15.250194549560547,
      "activations/layer2_attention_weight_max": 28.333702087402344,
      "activations/layer2_attention_weight_min": -24.113431930541992,
      "activations/layer3_attention_weight_max": 57.10529327392578,
      "activations/layer3_attention_weight_min": -53.507225036621094,
      "activations/layer4_attention_weight_max": 112.32585906982422,
      "activations/layer4_attention_weight_min": -104.09532928466797,
      "activations/layer5_attention_weight_max": 127.8160629272461,
      "activations/layer5_attention_weight_min": -112.70704650878906,
      "activations/layer6_attention_weight_max": 73.71766662597656,
      "activations/layer6_attention_weight_min": -65.94186401367188,
      "activations/layer7_attention_weight_max": 71.86524963378906,
      "activations/layer7_attention_weight_min": -64.65076446533203,
      "activations/layer8_attention_weight_max": 65.83631134033203,
      "activations/layer8_attention_weight_min": -60.12820053100586,
      "activations/layer9_attention_weight_max": 74.18428802490234,
      "activations/layer9_attention_weight_min": -66.29007720947266,
      "epoch": 0.58,
      "learning_rate": 0.00014773636363636362,
      "loss": 3.2639,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_loss": 3.1875,
      "eval_runtime": 8.506,
      "eval_samples_per_second": 504.818,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_openwebtext_loss": 3.1875,
      "eval_openwebtext_ppl": 24.22778221261098,
      "eval_openwebtext_runtime": 8.506,
      "eval_openwebtext_samples_per_second": 504.818,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_wikitext_loss": 3.482421875,
      "eval_wikitext_ppl": 32.53843073806492,
      "eval_wikitext_runtime": 1.9387,
      "eval_wikitext_samples_per_second": 235.204,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_lambada_loss": 3.87890625,
      "eval_lambada_ppl": 48.37128004023317,
      "eval_lambada_runtime": 9.5298,
      "eval_lambada_samples_per_second": 510.925,
      "step": 10000
    },
    {
      "activations/layer0_attention_weight_max": 14.95295238494873,
      "activations/layer0_attention_weight_min": -10.671829223632812,
      "activations/layer10_attention_weight_max": 54.391021728515625,
      "activations/layer10_attention_weight_min": -41.784122467041016,
      "activations/layer11_attention_weight_max": 53.93376159667969,
      "activations/layer11_attention_weight_min": -37.703636169433594,
      "activations/layer12_attention_weight_max": 40.973854064941406,
      "activations/layer12_attention_weight_min": -27.921049118041992,
      "activations/layer13_attention_weight_max": 39.60503005981445,
      "activations/layer13_attention_weight_min": -33.239891052246094,
      "activations/layer14_attention_weight_max": 53.87479782104492,
      "activations/layer14_attention_weight_min": -39.24070739746094,
      "activations/layer15_attention_weight_max": 42.199180603027344,
      "activations/layer15_attention_weight_min": -28.588987350463867,
      "activations/layer16_attention_weight_max": 32.822200775146484,
      "activations/layer16_attention_weight_min": -26.517425537109375,
      "activations/layer17_attention_weight_max": 22.327356338500977,
      "activations/layer17_attention_weight_min": -15.014843940734863,
      "activations/layer18_attention_weight_max": 17.970035552978516,
      "activations/layer18_attention_weight_min": -11.184785842895508,
      "activations/layer19_attention_weight_max": 28.356225967407227,
      "activations/layer19_attention_weight_min": -19.89818000793457,
      "activations/layer1_attention_weight_max": 14.435389518737793,
      "activations/layer1_attention_weight_min": -11.376837730407715,
      "activations/layer20_attention_weight_max": 19.513904571533203,
      "activations/layer20_attention_weight_min": -10.634782791137695,
      "activations/layer21_attention_weight_max": 18.408061981201172,
      "activations/layer21_attention_weight_min": -12.325955390930176,
      "activations/layer22_attention_weight_max": 30.42831802368164,
      "activations/layer22_attention_weight_min": -17.27867889404297,
      "activations/layer23_attention_weight_max": 21.715290069580078,
      "activations/layer23_attention_weight_min": -14.237547874450684,
      "activations/layer2_attention_weight_max": 29.64035415649414,
      "activations/layer2_attention_weight_min": -25.87818717956543,
      "activations/layer3_attention_weight_max": 55.87268829345703,
      "activations/layer3_attention_weight_min": -51.674537658691406,
      "activations/layer4_attention_weight_max": 115.39214324951172,
      "activations/layer4_attention_weight_min": -108.3656005859375,
      "activations/layer5_attention_weight_max": 127.60924530029297,
      "activations/layer5_attention_weight_min": -114.76283264160156,
      "activations/layer6_attention_weight_max": 74.41619110107422,
      "activations/layer6_attention_weight_min": -67.1927719116211,
      "activations/layer7_attention_weight_max": 69.3158950805664,
      "activations/layer7_attention_weight_min": -62.9103889465332,
      "activations/layer8_attention_weight_max": 66.39629364013672,
      "activations/layer8_attention_weight_min": -58.01190185546875,
      "activations/layer9_attention_weight_max": 79.49160766601562,
      "activations/layer9_attention_weight_min": -68.1618881225586,
      "epoch": 0.58,
      "learning_rate": 0.00014771780303030302,
      "loss": 3.2483,
      "step": 10050
    },
    {
      "activations/layer0_attention_weight_max": 16.590106964111328,
      "activations/layer0_attention_weight_min": -10.568191528320312,
      "activations/layer10_attention_weight_max": 51.73105239868164,
      "activations/layer10_attention_weight_min": -44.967803955078125,
      "activations/layer11_attention_weight_max": 51.108726501464844,
      "activations/layer11_attention_weight_min": -42.62882995605469,
      "activations/layer12_attention_weight_max": 32.17249298095703,
      "activations/layer12_attention_weight_min": -28.87763023376465,
      "activations/layer13_attention_weight_max": 43.2393913269043,
      "activations/layer13_attention_weight_min": -32.681610107421875,
      "activations/layer14_attention_weight_max": 68.13082122802734,
      "activations/layer14_attention_weight_min": -43.33388137817383,
      "activations/layer15_attention_weight_max": 38.952232360839844,
      "activations/layer15_attention_weight_min": -28.23528289794922,
      "activations/layer16_attention_weight_max": 34.63448715209961,
      "activations/layer16_attention_weight_min": -26.344873428344727,
      "activations/layer17_attention_weight_max": 32.39142990112305,
      "activations/layer17_attention_weight_min": -16.730989456176758,
      "activations/layer18_attention_weight_max": 19.742774963378906,
      "activations/layer18_attention_weight_min": -13.972945213317871,
      "activations/layer19_attention_weight_max": 26.911911010742188,
      "activations/layer19_attention_weight_min": -19.935264587402344,
      "activations/layer1_attention_weight_max": 14.173368453979492,
      "activations/layer1_attention_weight_min": -11.321548461914062,
      "activations/layer20_attention_weight_max": 22.68705177307129,
      "activations/layer20_attention_weight_min": -10.86038589477539,
      "activations/layer21_attention_weight_max": 19.023225784301758,
      "activations/layer21_attention_weight_min": -10.549180030822754,
      "activations/layer22_attention_weight_max": 33.603206634521484,
      "activations/layer22_attention_weight_min": -13.429924011230469,
      "activations/layer23_attention_weight_max": 23.74567222595215,
      "activations/layer23_attention_weight_min": -12.945128440856934,
      "activations/layer2_attention_weight_max": 27.753623962402344,
      "activations/layer2_attention_weight_min": -23.674184799194336,
      "activations/layer3_attention_weight_max": 54.84634780883789,
      "activations/layer3_attention_weight_min": -51.44926834106445,
      "activations/layer4_attention_weight_max": 111.34022521972656,
      "activations/layer4_attention_weight_min": -105.76130676269531,
      "activations/layer5_attention_weight_max": 128.94039916992188,
      "activations/layer5_attention_weight_min": -111.2657470703125,
      "activations/layer6_attention_weight_max": 77.4947280883789,
      "activations/layer6_attention_weight_min": -71.987548828125,
      "activations/layer7_attention_weight_max": 74.322998046875,
      "activations/layer7_attention_weight_min": -64.53461456298828,
      "activations/layer8_attention_weight_max": 68.3620834350586,
      "activations/layer8_attention_weight_min": -54.889644622802734,
      "activations/layer9_attention_weight_max": 69.82943725585938,
      "activations/layer9_attention_weight_min": -59.476844787597656,
      "epoch": 0.59,
      "learning_rate": 0.00014769886363636363,
      "loss": 3.227,
      "step": 10100
    },
    {
      "activations/layer0_attention_weight_max": 16.263633728027344,
      "activations/layer0_attention_weight_min": -10.891717910766602,
      "activations/layer10_attention_weight_max": 46.50944900512695,
      "activations/layer10_attention_weight_min": -46.00583267211914,
      "activations/layer11_attention_weight_max": 47.79536437988281,
      "activations/layer11_attention_weight_min": -42.433956146240234,
      "activations/layer12_attention_weight_max": 62.180076599121094,
      "activations/layer12_attention_weight_min": -36.701419830322266,
      "activations/layer13_attention_weight_max": 57.50579833984375,
      "activations/layer13_attention_weight_min": -35.837646484375,
      "activations/layer14_attention_weight_max": 53.30211639404297,
      "activations/layer14_attention_weight_min": -47.59629821777344,
      "activations/layer15_attention_weight_max": 36.93595886230469,
      "activations/layer15_attention_weight_min": -30.22469711303711,
      "activations/layer16_attention_weight_max": 37.196563720703125,
      "activations/layer16_attention_weight_min": -22.894283294677734,
      "activations/layer17_attention_weight_max": 35.38901138305664,
      "activations/layer17_attention_weight_min": -16.43242645263672,
      "activations/layer18_attention_weight_max": 22.48263168334961,
      "activations/layer18_attention_weight_min": -11.134950637817383,
      "activations/layer19_attention_weight_max": 33.874664306640625,
      "activations/layer19_attention_weight_min": -21.19541358947754,
      "activations/layer1_attention_weight_max": 15.108490943908691,
      "activations/layer1_attention_weight_min": -11.193883895874023,
      "activations/layer20_attention_weight_max": 24.20479965209961,
      "activations/layer20_attention_weight_min": -12.397713661193848,
      "activations/layer21_attention_weight_max": 19.79688262939453,
      "activations/layer21_attention_weight_min": -13.266822814941406,
      "activations/layer22_attention_weight_max": 34.470481872558594,
      "activations/layer22_attention_weight_min": -15.385578155517578,
      "activations/layer23_attention_weight_max": 26.68173599243164,
      "activations/layer23_attention_weight_min": -13.971977233886719,
      "activations/layer2_attention_weight_max": 27.811019897460938,
      "activations/layer2_attention_weight_min": -23.413166046142578,
      "activations/layer3_attention_weight_max": 54.12990188598633,
      "activations/layer3_attention_weight_min": -50.8053092956543,
      "activations/layer4_attention_weight_max": 110.52392578125,
      "activations/layer4_attention_weight_min": -103.40618896484375,
      "activations/layer5_attention_weight_max": 130.4127960205078,
      "activations/layer5_attention_weight_min": -110.65886688232422,
      "activations/layer6_attention_weight_max": 76.842041015625,
      "activations/layer6_attention_weight_min": -68.47195434570312,
      "activations/layer7_attention_weight_max": 71.13447570800781,
      "activations/layer7_attention_weight_min": -62.454742431640625,
      "activations/layer8_attention_weight_max": 63.75822830200195,
      "activations/layer8_attention_weight_min": -54.7025146484375,
      "activations/layer9_attention_weight_max": 84.15660858154297,
      "activations/layer9_attention_weight_min": -67.68109130859375,
      "epoch": 0.59,
      "learning_rate": 0.00014767992424242422,
      "loss": 3.2322,
      "step": 10150
    },
    {
      "activations/layer0_attention_weight_max": 16.09835433959961,
      "activations/layer0_attention_weight_min": -10.654668807983398,
      "activations/layer10_attention_weight_max": 59.253929138183594,
      "activations/layer10_attention_weight_min": -43.6585578918457,
      "activations/layer11_attention_weight_max": 64.87633514404297,
      "activations/layer11_attention_weight_min": -41.62717819213867,
      "activations/layer12_attention_weight_max": 32.026729583740234,
      "activations/layer12_attention_weight_min": -27.285890579223633,
      "activations/layer13_attention_weight_max": 40.118988037109375,
      "activations/layer13_attention_weight_min": -33.99629211425781,
      "activations/layer14_attention_weight_max": 53.867347717285156,
      "activations/layer14_attention_weight_min": -41.792179107666016,
      "activations/layer15_attention_weight_max": 38.08855056762695,
      "activations/layer15_attention_weight_min": -32.166072845458984,
      "activations/layer16_attention_weight_max": 33.30739974975586,
      "activations/layer16_attention_weight_min": -25.325162887573242,
      "activations/layer17_attention_weight_max": 24.714731216430664,
      "activations/layer17_attention_weight_min": -17.09246826171875,
      "activations/layer18_attention_weight_max": 18.065200805664062,
      "activations/layer18_attention_weight_min": -11.981977462768555,
      "activations/layer19_attention_weight_max": 30.364572525024414,
      "activations/layer19_attention_weight_min": -19.570058822631836,
      "activations/layer1_attention_weight_max": 14.60616683959961,
      "activations/layer1_attention_weight_min": -10.463693618774414,
      "activations/layer20_attention_weight_max": 20.262418746948242,
      "activations/layer20_attention_weight_min": -13.003582000732422,
      "activations/layer21_attention_weight_max": 18.23517417907715,
      "activations/layer21_attention_weight_min": -11.713180541992188,
      "activations/layer22_attention_weight_max": 30.22651481628418,
      "activations/layer22_attention_weight_min": -17.21004867553711,
      "activations/layer23_attention_weight_max": 20.533050537109375,
      "activations/layer23_attention_weight_min": -14.587554931640625,
      "activations/layer2_attention_weight_max": 31.744108200073242,
      "activations/layer2_attention_weight_min": -25.17280387878418,
      "activations/layer3_attention_weight_max": 59.13673782348633,
      "activations/layer3_attention_weight_min": -53.74272918701172,
      "activations/layer4_attention_weight_max": 123.43328094482422,
      "activations/layer4_attention_weight_min": -104.85179901123047,
      "activations/layer5_attention_weight_max": 156.23243713378906,
      "activations/layer5_attention_weight_min": -119.10499572753906,
      "activations/layer6_attention_weight_max": 88.53325653076172,
      "activations/layer6_attention_weight_min": -66.55072021484375,
      "activations/layer7_attention_weight_max": 77.6436538696289,
      "activations/layer7_attention_weight_min": -63.66523361206055,
      "activations/layer8_attention_weight_max": 77.23067474365234,
      "activations/layer8_attention_weight_min": -59.58513641357422,
      "activations/layer9_attention_weight_max": 91.59658813476562,
      "activations/layer9_attention_weight_min": -78.06002044677734,
      "epoch": 0.59,
      "learning_rate": 0.00014766098484848483,
      "loss": 3.2414,
      "step": 10200
    },
    {
      "activations/layer0_attention_weight_max": 16.260560989379883,
      "activations/layer0_attention_weight_min": -10.7701997756958,
      "activations/layer10_attention_weight_max": 50.5311279296875,
      "activations/layer10_attention_weight_min": -42.280853271484375,
      "activations/layer11_attention_weight_max": 50.572044372558594,
      "activations/layer11_attention_weight_min": -44.74256134033203,
      "activations/layer12_attention_weight_max": 40.23460006713867,
      "activations/layer12_attention_weight_min": -33.541622161865234,
      "activations/layer13_attention_weight_max": 37.9556770324707,
      "activations/layer13_attention_weight_min": -32.51089096069336,
      "activations/layer14_attention_weight_max": 56.212852478027344,
      "activations/layer14_attention_weight_min": -41.6580696105957,
      "activations/layer15_attention_weight_max": 39.75479507446289,
      "activations/layer15_attention_weight_min": -30.278701782226562,
      "activations/layer16_attention_weight_max": 37.14353561401367,
      "activations/layer16_attention_weight_min": -23.4586181640625,
      "activations/layer17_attention_weight_max": 33.68632888793945,
      "activations/layer17_attention_weight_min": -19.10914421081543,
      "activations/layer18_attention_weight_max": 22.000926971435547,
      "activations/layer18_attention_weight_min": -12.739978790283203,
      "activations/layer19_attention_weight_max": 29.8342342376709,
      "activations/layer19_attention_weight_min": -18.715970993041992,
      "activations/layer1_attention_weight_max": 14.560215950012207,
      "activations/layer1_attention_weight_min": -11.601978302001953,
      "activations/layer20_attention_weight_max": 20.470548629760742,
      "activations/layer20_attention_weight_min": -15.769481658935547,
      "activations/layer21_attention_weight_max": 18.761627197265625,
      "activations/layer21_attention_weight_min": -11.220231056213379,
      "activations/layer22_attention_weight_max": 33.01462936401367,
      "activations/layer22_attention_weight_min": -17.948040008544922,
      "activations/layer23_attention_weight_max": 21.02311897277832,
      "activations/layer23_attention_weight_min": -13.313556671142578,
      "activations/layer2_attention_weight_max": 29.271976470947266,
      "activations/layer2_attention_weight_min": -24.820159912109375,
      "activations/layer3_attention_weight_max": 55.492225646972656,
      "activations/layer3_attention_weight_min": -53.14511489868164,
      "activations/layer4_attention_weight_max": 109.4902572631836,
      "activations/layer4_attention_weight_min": -105.9836196899414,
      "activations/layer5_attention_weight_max": 127.2528305053711,
      "activations/layer5_attention_weight_min": -115.6307373046875,
      "activations/layer6_attention_weight_max": 72.55126953125,
      "activations/layer6_attention_weight_min": -70.00032806396484,
      "activations/layer7_attention_weight_max": 67.56768035888672,
      "activations/layer7_attention_weight_min": -62.691932678222656,
      "activations/layer8_attention_weight_max": 66.2139663696289,
      "activations/layer8_attention_weight_min": -60.94176483154297,
      "activations/layer9_attention_weight_max": 69.3786392211914,
      "activations/layer9_attention_weight_min": -63.15602493286133,
      "epoch": 0.6,
      "learning_rate": 0.00014764204545454545,
      "loss": 3.2382,
      "step": 10250
    },
    {
      "activations/layer0_attention_weight_max": 15.487317085266113,
      "activations/layer0_attention_weight_min": -11.042375564575195,
      "activations/layer10_attention_weight_max": 50.71783447265625,
      "activations/layer10_attention_weight_min": -42.01552200317383,
      "activations/layer11_attention_weight_max": 50.17116928100586,
      "activations/layer11_attention_weight_min": -35.84516906738281,
      "activations/layer12_attention_weight_max": 33.769325256347656,
      "activations/layer12_attention_weight_min": -28.484107971191406,
      "activations/layer13_attention_weight_max": 48.921348571777344,
      "activations/layer13_attention_weight_min": -33.77465057373047,
      "activations/layer14_attention_weight_max": 66.28805541992188,
      "activations/layer14_attention_weight_min": -39.94601821899414,
      "activations/layer15_attention_weight_max": 43.21892547607422,
      "activations/layer15_attention_weight_min": -26.754884719848633,
      "activations/layer16_attention_weight_max": 36.429874420166016,
      "activations/layer16_attention_weight_min": -24.12970733642578,
      "activations/layer17_attention_weight_max": 24.38957977294922,
      "activations/layer17_attention_weight_min": -18.15068244934082,
      "activations/layer18_attention_weight_max": 20.594388961791992,
      "activations/layer18_attention_weight_min": -11.327238082885742,
      "activations/layer19_attention_weight_max": 33.945133209228516,
      "activations/layer19_attention_weight_min": -21.041305541992188,
      "activations/layer1_attention_weight_max": 14.644508361816406,
      "activations/layer1_attention_weight_min": -11.509010314941406,
      "activations/layer20_attention_weight_max": 22.401723861694336,
      "activations/layer20_attention_weight_min": -11.610450744628906,
      "activations/layer21_attention_weight_max": 21.87139320373535,
      "activations/layer21_attention_weight_min": -12.019353866577148,
      "activations/layer22_attention_weight_max": 31.15991973876953,
      "activations/layer22_attention_weight_min": -14.082603454589844,
      "activations/layer23_attention_weight_max": 20.003292083740234,
      "activations/layer23_attention_weight_min": -12.200156211853027,
      "activations/layer2_attention_weight_max": 30.066551208496094,
      "activations/layer2_attention_weight_min": -24.87910270690918,
      "activations/layer3_attention_weight_max": 56.24969482421875,
      "activations/layer3_attention_weight_min": -53.03502655029297,
      "activations/layer4_attention_weight_max": 111.24401092529297,
      "activations/layer4_attention_weight_min": -107.63622283935547,
      "activations/layer5_attention_weight_max": 130.84954833984375,
      "activations/layer5_attention_weight_min": -120.21739959716797,
      "activations/layer6_attention_weight_max": 78.9677505493164,
      "activations/layer6_attention_weight_min": -69.08958435058594,
      "activations/layer7_attention_weight_max": 78.36255645751953,
      "activations/layer7_attention_weight_min": -64.53793334960938,
      "activations/layer8_attention_weight_max": 73.7929458618164,
      "activations/layer8_attention_weight_min": -62.21361541748047,
      "activations/layer9_attention_weight_max": 68.93956756591797,
      "activations/layer9_attention_weight_min": -60.50691604614258,
      "epoch": 0.6,
      "learning_rate": 0.00014762310606060604,
      "loss": 3.2197,
      "step": 10300
    },
    {
      "activations/layer0_attention_weight_max": 15.716413497924805,
      "activations/layer0_attention_weight_min": -11.005903244018555,
      "activations/layer10_attention_weight_max": 46.10436248779297,
      "activations/layer10_attention_weight_min": -41.32474899291992,
      "activations/layer11_attention_weight_max": 42.88038635253906,
      "activations/layer11_attention_weight_min": -37.43049621582031,
      "activations/layer12_attention_weight_max": 30.16617202758789,
      "activations/layer12_attention_weight_min": -23.524768829345703,
      "activations/layer13_attention_weight_max": 36.13425827026367,
      "activations/layer13_attention_weight_min": -30.95444107055664,
      "activations/layer14_attention_weight_max": 54.758304595947266,
      "activations/layer14_attention_weight_min": -41.36054611206055,
      "activations/layer15_attention_weight_max": 39.09432601928711,
      "activations/layer15_attention_weight_min": -34.53738021850586,
      "activations/layer16_attention_weight_max": 33.507503509521484,
      "activations/layer16_attention_weight_min": -26.4947566986084,
      "activations/layer17_attention_weight_max": 30.72566032409668,
      "activations/layer17_attention_weight_min": -16.35162925720215,
      "activations/layer18_attention_weight_max": 19.691360473632812,
      "activations/layer18_attention_weight_min": -11.93847942352295,
      "activations/layer19_attention_weight_max": 32.7434196472168,
      "activations/layer19_attention_weight_min": -19.40693473815918,
      "activations/layer1_attention_weight_max": 13.632335662841797,
      "activations/layer1_attention_weight_min": -10.67707633972168,
      "activations/layer20_attention_weight_max": 20.339372634887695,
      "activations/layer20_attention_weight_min": -12.09841537475586,
      "activations/layer21_attention_weight_max": 19.716238021850586,
      "activations/layer21_attention_weight_min": -12.834249496459961,
      "activations/layer22_attention_weight_max": 31.125038146972656,
      "activations/layer22_attention_weight_min": -16.476600646972656,
      "activations/layer23_attention_weight_max": 21.45825958251953,
      "activations/layer23_attention_weight_min": -12.902511596679688,
      "activations/layer2_attention_weight_max": 27.32740020751953,
      "activations/layer2_attention_weight_min": -23.240882873535156,
      "activations/layer3_attention_weight_max": 54.54075622558594,
      "activations/layer3_attention_weight_min": -53.32373809814453,
      "activations/layer4_attention_weight_max": 112.05553436279297,
      "activations/layer4_attention_weight_min": -108.35945129394531,
      "activations/layer5_attention_weight_max": 126.81533813476562,
      "activations/layer5_attention_weight_min": -114.47917938232422,
      "activations/layer6_attention_weight_max": 72.50569915771484,
      "activations/layer6_attention_weight_min": -65.27082824707031,
      "activations/layer7_attention_weight_max": 68.8520278930664,
      "activations/layer7_attention_weight_min": -61.17859649658203,
      "activations/layer8_attention_weight_max": 64.03822326660156,
      "activations/layer8_attention_weight_min": -54.070281982421875,
      "activations/layer9_attention_weight_max": 65.68639373779297,
      "activations/layer9_attention_weight_min": -61.01557540893555,
      "epoch": 0.6,
      "learning_rate": 0.00014760416666666665,
      "loss": 3.2406,
      "step": 10350
    },
    {
      "activations/layer0_attention_weight_max": 14.12877368927002,
      "activations/layer0_attention_weight_min": -10.38298511505127,
      "activations/layer10_attention_weight_max": 51.22227096557617,
      "activations/layer10_attention_weight_min": -44.944889068603516,
      "activations/layer11_attention_weight_max": 48.27832794189453,
      "activations/layer11_attention_weight_min": -42.60079574584961,
      "activations/layer12_attention_weight_max": 32.01332092285156,
      "activations/layer12_attention_weight_min": -24.236127853393555,
      "activations/layer13_attention_weight_max": 40.742088317871094,
      "activations/layer13_attention_weight_min": -34.40436553955078,
      "activations/layer14_attention_weight_max": 66.79493713378906,
      "activations/layer14_attention_weight_min": -42.616783142089844,
      "activations/layer15_attention_weight_max": 46.47066879272461,
      "activations/layer15_attention_weight_min": -26.812114715576172,
      "activations/layer16_attention_weight_max": 44.46787643432617,
      "activations/layer16_attention_weight_min": -24.203125,
      "activations/layer17_attention_weight_max": 39.8887825012207,
      "activations/layer17_attention_weight_min": -18.441022872924805,
      "activations/layer18_attention_weight_max": 19.771390914916992,
      "activations/layer18_attention_weight_min": -12.150367736816406,
      "activations/layer19_attention_weight_max": 40.07365036010742,
      "activations/layer19_attention_weight_min": -19.25520133972168,
      "activations/layer1_attention_weight_max": 13.980632781982422,
      "activations/layer1_attention_weight_min": -11.649630546569824,
      "activations/layer20_attention_weight_max": 26.68621826171875,
      "activations/layer20_attention_weight_min": -12.819512367248535,
      "activations/layer21_attention_weight_max": 21.695709228515625,
      "activations/layer21_attention_weight_min": -13.482250213623047,
      "activations/layer22_attention_weight_max": 42.50819396972656,
      "activations/layer22_attention_weight_min": -16.958669662475586,
      "activations/layer23_attention_weight_max": 29.36263656616211,
      "activations/layer23_attention_weight_min": -14.542137145996094,
      "activations/layer2_attention_weight_max": 27.84560775756836,
      "activations/layer2_attention_weight_min": -24.21477508544922,
      "activations/layer3_attention_weight_max": 55.34934616088867,
      "activations/layer3_attention_weight_min": -54.04157257080078,
      "activations/layer4_attention_weight_max": 112.46781158447266,
      "activations/layer4_attention_weight_min": -104.4273681640625,
      "activations/layer5_attention_weight_max": 131.64764404296875,
      "activations/layer5_attention_weight_min": -117.26109313964844,
      "activations/layer6_attention_weight_max": 74.6127700805664,
      "activations/layer6_attention_weight_min": -66.6248550415039,
      "activations/layer7_attention_weight_max": 75.76191711425781,
      "activations/layer7_attention_weight_min": -62.223968505859375,
      "activations/layer8_attention_weight_max": 69.25786590576172,
      "activations/layer8_attention_weight_min": -56.027557373046875,
      "activations/layer9_attention_weight_max": 68.47962951660156,
      "activations/layer9_attention_weight_min": -56.546630859375,
      "epoch": 0.6,
      "learning_rate": 0.00014758522727272727,
      "loss": 3.2569,
      "step": 10400
    },
    {
      "activations/layer0_attention_weight_max": 15.757390022277832,
      "activations/layer0_attention_weight_min": -10.936450958251953,
      "activations/layer10_attention_weight_max": 51.76529312133789,
      "activations/layer10_attention_weight_min": -43.216575622558594,
      "activations/layer11_attention_weight_max": 54.85691452026367,
      "activations/layer11_attention_weight_min": -41.66675567626953,
      "activations/layer12_attention_weight_max": 35.54928970336914,
      "activations/layer12_attention_weight_min": -29.30730438232422,
      "activations/layer13_attention_weight_max": 41.18110656738281,
      "activations/layer13_attention_weight_min": -32.775421142578125,
      "activations/layer14_attention_weight_max": 58.723182678222656,
      "activations/layer14_attention_weight_min": -46.867984771728516,
      "activations/layer15_attention_weight_max": 50.28806686401367,
      "activations/layer15_attention_weight_min": -31.64417839050293,
      "activations/layer16_attention_weight_max": 44.42131805419922,
      "activations/layer16_attention_weight_min": -30.85138702392578,
      "activations/layer17_attention_weight_max": 41.9801025390625,
      "activations/layer17_attention_weight_min": -22.260583877563477,
      "activations/layer18_attention_weight_max": 22.709821701049805,
      "activations/layer18_attention_weight_min": -12.769847869873047,
      "activations/layer19_attention_weight_max": 39.98183822631836,
      "activations/layer19_attention_weight_min": -21.57887077331543,
      "activations/layer1_attention_weight_max": 14.673602104187012,
      "activations/layer1_attention_weight_min": -11.633333206176758,
      "activations/layer20_attention_weight_max": 24.175491333007812,
      "activations/layer20_attention_weight_min": -13.66919231414795,
      "activations/layer21_attention_weight_max": 21.422649383544922,
      "activations/layer21_attention_weight_min": -11.457001686096191,
      "activations/layer22_attention_weight_max": 33.07710647583008,
      "activations/layer22_attention_weight_min": -19.375255584716797,
      "activations/layer23_attention_weight_max": 24.538183212280273,
      "activations/layer23_attention_weight_min": -14.271669387817383,
      "activations/layer2_attention_weight_max": 28.736736297607422,
      "activations/layer2_attention_weight_min": -24.451614379882812,
      "activations/layer3_attention_weight_max": 56.23146438598633,
      "activations/layer3_attention_weight_min": -52.70191192626953,
      "activations/layer4_attention_weight_max": 109.72733306884766,
      "activations/layer4_attention_weight_min": -104.7069091796875,
      "activations/layer5_attention_weight_max": 126.8252182006836,
      "activations/layer5_attention_weight_min": -116.47280883789062,
      "activations/layer6_attention_weight_max": 74.57418060302734,
      "activations/layer6_attention_weight_min": -67.259765625,
      "activations/layer7_attention_weight_max": 68.84750366210938,
      "activations/layer7_attention_weight_min": -64.97406768798828,
      "activations/layer8_attention_weight_max": 64.28823852539062,
      "activations/layer8_attention_weight_min": -58.5117073059082,
      "activations/layer9_attention_weight_max": 69.66631317138672,
      "activations/layer9_attention_weight_min": -66.06368255615234,
      "epoch": 0.61,
      "learning_rate": 0.00014756628787878788,
      "loss": 3.2159,
      "step": 10450
    },
    {
      "activations/layer0_attention_weight_max": 15.582253456115723,
      "activations/layer0_attention_weight_min": -10.223069190979004,
      "activations/layer10_attention_weight_max": 52.41475296020508,
      "activations/layer10_attention_weight_min": -45.71332931518555,
      "activations/layer11_attention_weight_max": 49.61719512939453,
      "activations/layer11_attention_weight_min": -40.6197509765625,
      "activations/layer12_attention_weight_max": 29.769426345825195,
      "activations/layer12_attention_weight_min": -28.573915481567383,
      "activations/layer13_attention_weight_max": 37.586883544921875,
      "activations/layer13_attention_weight_min": -33.31044387817383,
      "activations/layer14_attention_weight_max": 64.04328155517578,
      "activations/layer14_attention_weight_min": -47.36763000488281,
      "activations/layer15_attention_weight_max": 38.56614303588867,
      "activations/layer15_attention_weight_min": -30.019359588623047,
      "activations/layer16_attention_weight_max": 38.558082580566406,
      "activations/layer16_attention_weight_min": -25.353668212890625,
      "activations/layer17_attention_weight_max": 30.132633209228516,
      "activations/layer17_attention_weight_min": -16.172958374023438,
      "activations/layer18_attention_weight_max": 22.07667350769043,
      "activations/layer18_attention_weight_min": -11.662444114685059,
      "activations/layer19_attention_weight_max": 33.54353713989258,
      "activations/layer19_attention_weight_min": -20.201026916503906,
      "activations/layer1_attention_weight_max": 14.350908279418945,
      "activations/layer1_attention_weight_min": -11.6394624710083,
      "activations/layer20_attention_weight_max": 21.047462463378906,
      "activations/layer20_attention_weight_min": -12.173053741455078,
      "activations/layer21_attention_weight_max": 17.49212074279785,
      "activations/layer21_attention_weight_min": -11.045251846313477,
      "activations/layer22_attention_weight_max": 32.54429626464844,
      "activations/layer22_attention_weight_min": -17.016727447509766,
      "activations/layer23_attention_weight_max": 22.63570785522461,
      "activations/layer23_attention_weight_min": -14.004337310791016,
      "activations/layer2_attention_weight_max": 25.692447662353516,
      "activations/layer2_attention_weight_min": -23.89438247680664,
      "activations/layer3_attention_weight_max": 55.262123107910156,
      "activations/layer3_attention_weight_min": -54.179195404052734,
      "activations/layer4_attention_weight_max": 113.97637939453125,
      "activations/layer4_attention_weight_min": -110.10157775878906,
      "activations/layer5_attention_weight_max": 129.39796447753906,
      "activations/layer5_attention_weight_min": -118.38115692138672,
      "activations/layer6_attention_weight_max": 71.53020477294922,
      "activations/layer6_attention_weight_min": -66.65473937988281,
      "activations/layer7_attention_weight_max": 66.34760284423828,
      "activations/layer7_attention_weight_min": -64.92086791992188,
      "activations/layer8_attention_weight_max": 63.95656967163086,
      "activations/layer8_attention_weight_min": -61.00681686401367,
      "activations/layer9_attention_weight_max": 65.52144622802734,
      "activations/layer9_attention_weight_min": -58.33345413208008,
      "epoch": 0.61,
      "learning_rate": 0.00014754734848484847,
      "loss": 3.2187,
      "step": 10500
    },
    {
      "activations/layer0_attention_weight_max": 15.629340171813965,
      "activations/layer0_attention_weight_min": -11.423320770263672,
      "activations/layer10_attention_weight_max": 55.688507080078125,
      "activations/layer10_attention_weight_min": -50.614925384521484,
      "activations/layer11_attention_weight_max": 53.87413787841797,
      "activations/layer11_attention_weight_min": -41.03391647338867,
      "activations/layer12_attention_weight_max": 28.70412826538086,
      "activations/layer12_attention_weight_min": -23.8901309967041,
      "activations/layer13_attention_weight_max": 40.249000549316406,
      "activations/layer13_attention_weight_min": -37.39006423950195,
      "activations/layer14_attention_weight_max": 61.32256317138672,
      "activations/layer14_attention_weight_min": -46.2776985168457,
      "activations/layer15_attention_weight_max": 40.796329498291016,
      "activations/layer15_attention_weight_min": -31.850542068481445,
      "activations/layer16_attention_weight_max": 33.4429817199707,
      "activations/layer16_attention_weight_min": -25.263240814208984,
      "activations/layer17_attention_weight_max": 28.56092071533203,
      "activations/layer17_attention_weight_min": -18.41280174255371,
      "activations/layer18_attention_weight_max": 21.504308700561523,
      "activations/layer18_attention_weight_min": -12.017732620239258,
      "activations/layer19_attention_weight_max": 33.621036529541016,
      "activations/layer19_attention_weight_min": -22.627227783203125,
      "activations/layer1_attention_weight_max": 13.960755348205566,
      "activations/layer1_attention_weight_min": -11.69516658782959,
      "activations/layer20_attention_weight_max": 22.96463966369629,
      "activations/layer20_attention_weight_min": -12.789586067199707,
      "activations/layer21_attention_weight_max": 21.01633644104004,
      "activations/layer21_attention_weight_min": -12.381460189819336,
      "activations/layer22_attention_weight_max": 28.956390380859375,
      "activations/layer22_attention_weight_min": -16.629581451416016,
      "activations/layer23_attention_weight_max": 19.952354431152344,
      "activations/layer23_attention_weight_min": -13.210829734802246,
      "activations/layer2_attention_weight_max": 26.90985107421875,
      "activations/layer2_attention_weight_min": -23.43812370300293,
      "activations/layer3_attention_weight_max": 54.177635192871094,
      "activations/layer3_attention_weight_min": -53.92308807373047,
      "activations/layer4_attention_weight_max": 107.69757080078125,
      "activations/layer4_attention_weight_min": -103.29546356201172,
      "activations/layer5_attention_weight_max": 127.87617492675781,
      "activations/layer5_attention_weight_min": -111.6968002319336,
      "activations/layer6_attention_weight_max": 75.03305053710938,
      "activations/layer6_attention_weight_min": -65.03132629394531,
      "activations/layer7_attention_weight_max": 74.40895080566406,
      "activations/layer7_attention_weight_min": -62.109683990478516,
      "activations/layer8_attention_weight_max": 67.85260009765625,
      "activations/layer8_attention_weight_min": -54.88492965698242,
      "activations/layer9_attention_weight_max": 72.62596130371094,
      "activations/layer9_attention_weight_min": -63.541316986083984,
      "epoch": 0.61,
      "learning_rate": 0.00014752840909090908,
      "loss": 3.237,
      "step": 10550
    },
    {
      "activations/layer0_attention_weight_max": 15.30947494506836,
      "activations/layer0_attention_weight_min": -10.565163612365723,
      "activations/layer10_attention_weight_max": 50.6505012512207,
      "activations/layer10_attention_weight_min": -40.14432907104492,
      "activations/layer11_attention_weight_max": 61.061729431152344,
      "activations/layer11_attention_weight_min": -36.77930450439453,
      "activations/layer12_attention_weight_max": 46.776947021484375,
      "activations/layer12_attention_weight_min": -25.97370147705078,
      "activations/layer13_attention_weight_max": 45.56050491333008,
      "activations/layer13_attention_weight_min": -35.22996139526367,
      "activations/layer14_attention_weight_max": 55.52902603149414,
      "activations/layer14_attention_weight_min": -38.722415924072266,
      "activations/layer15_attention_weight_max": 48.90907287597656,
      "activations/layer15_attention_weight_min": -31.830890655517578,
      "activations/layer16_attention_weight_max": 35.557777404785156,
      "activations/layer16_attention_weight_min": -26.12968635559082,
      "activations/layer17_attention_weight_max": 22.95965003967285,
      "activations/layer17_attention_weight_min": -15.908344268798828,
      "activations/layer18_attention_weight_max": 18.924856185913086,
      "activations/layer18_attention_weight_min": -11.374581336975098,
      "activations/layer19_attention_weight_max": 26.278186798095703,
      "activations/layer19_attention_weight_min": -20.17482566833496,
      "activations/layer1_attention_weight_max": 14.384820938110352,
      "activations/layer1_attention_weight_min": -11.642865180969238,
      "activations/layer20_attention_weight_max": 17.907611846923828,
      "activations/layer20_attention_weight_min": -10.480742454528809,
      "activations/layer21_attention_weight_max": 19.661890029907227,
      "activations/layer21_attention_weight_min": -10.524067878723145,
      "activations/layer22_attention_weight_max": 27.685699462890625,
      "activations/layer22_attention_weight_min": -13.468050003051758,
      "activations/layer23_attention_weight_max": 18.229612350463867,
      "activations/layer23_attention_weight_min": -12.291662216186523,
      "activations/layer2_attention_weight_max": 29.41242027282715,
      "activations/layer2_attention_weight_min": -24.010498046875,
      "activations/layer3_attention_weight_max": 60.009986877441406,
      "activations/layer3_attention_weight_min": -52.91854476928711,
      "activations/layer4_attention_weight_max": 120.23319244384766,
      "activations/layer4_attention_weight_min": -104.8053207397461,
      "activations/layer5_attention_weight_max": 149.80026245117188,
      "activations/layer5_attention_weight_min": -115.46986389160156,
      "activations/layer6_attention_weight_max": 81.47673034667969,
      "activations/layer6_attention_weight_min": -67.646240234375,
      "activations/layer7_attention_weight_max": 70.59733581542969,
      "activations/layer7_attention_weight_min": -60.46305465698242,
      "activations/layer8_attention_weight_max": 79.81706237792969,
      "activations/layer8_attention_weight_min": -61.24454879760742,
      "activations/layer9_attention_weight_max": 91.5560531616211,
      "activations/layer9_attention_weight_min": -62.9471549987793,
      "epoch": 0.62,
      "learning_rate": 0.0001475094696969697,
      "loss": 3.2205,
      "step": 10600
    },
    {
      "activations/layer0_attention_weight_max": 15.4803466796875,
      "activations/layer0_attention_weight_min": -11.143799781799316,
      "activations/layer10_attention_weight_max": 52.737884521484375,
      "activations/layer10_attention_weight_min": -43.135616302490234,
      "activations/layer11_attention_weight_max": 50.86277770996094,
      "activations/layer11_attention_weight_min": -43.20038604736328,
      "activations/layer12_attention_weight_max": 29.927356719970703,
      "activations/layer12_attention_weight_min": -24.327848434448242,
      "activations/layer13_attention_weight_max": 38.316192626953125,
      "activations/layer13_attention_weight_min": -33.64508056640625,
      "activations/layer14_attention_weight_max": 61.394779205322266,
      "activations/layer14_attention_weight_min": -43.54338836669922,
      "activations/layer15_attention_weight_max": 39.01193618774414,
      "activations/layer15_attention_weight_min": -31.037866592407227,
      "activations/layer16_attention_weight_max": 36.75634002685547,
      "activations/layer16_attention_weight_min": -26.31373405456543,
      "activations/layer17_attention_weight_max": 32.31559753417969,
      "activations/layer17_attention_weight_min": -15.63011646270752,
      "activations/layer18_attention_weight_max": 17.3438720703125,
      "activations/layer18_attention_weight_min": -10.895000457763672,
      "activations/layer19_attention_weight_max": 30.01214027404785,
      "activations/layer19_attention_weight_min": -19.115127563476562,
      "activations/layer1_attention_weight_max": 14.4043607711792,
      "activations/layer1_attention_weight_min": -12.116775512695312,
      "activations/layer20_attention_weight_max": 19.63686180114746,
      "activations/layer20_attention_weight_min": -11.198052406311035,
      "activations/layer21_attention_weight_max": 19.971290588378906,
      "activations/layer21_attention_weight_min": -12.588093757629395,
      "activations/layer22_attention_weight_max": 28.495351791381836,
      "activations/layer22_attention_weight_min": -19.543161392211914,
      "activations/layer23_attention_weight_max": 18.084922790527344,
      "activations/layer23_attention_weight_min": -14.950920104980469,
      "activations/layer2_attention_weight_max": 28.942533493041992,
      "activations/layer2_attention_weight_min": -24.661598205566406,
      "activations/layer3_attention_weight_max": 57.13707733154297,
      "activations/layer3_attention_weight_min": -55.12063980102539,
      "activations/layer4_attention_weight_max": 116.70293426513672,
      "activations/layer4_attention_weight_min": -107.74476623535156,
      "activations/layer5_attention_weight_max": 137.11874389648438,
      "activations/layer5_attention_weight_min": -116.05848693847656,
      "activations/layer6_attention_weight_max": 77.17645263671875,
      "activations/layer6_attention_weight_min": -68.28131866455078,
      "activations/layer7_attention_weight_max": 71.8621826171875,
      "activations/layer7_attention_weight_min": -63.22610092163086,
      "activations/layer8_attention_weight_max": 70.81849670410156,
      "activations/layer8_attention_weight_min": -60.92954635620117,
      "activations/layer9_attention_weight_max": 69.2699203491211,
      "activations/layer9_attention_weight_min": -64.67281341552734,
      "epoch": 0.62,
      "learning_rate": 0.0001474905303030303,
      "loss": 3.2212,
      "step": 10650
    },
    {
      "activations/layer0_attention_weight_max": 15.305561065673828,
      "activations/layer0_attention_weight_min": -10.22108268737793,
      "activations/layer10_attention_weight_max": 44.218788146972656,
      "activations/layer10_attention_weight_min": -42.66055679321289,
      "activations/layer11_attention_weight_max": 43.01899719238281,
      "activations/layer11_attention_weight_min": -39.5335693359375,
      "activations/layer12_attention_weight_max": 28.186141967773438,
      "activations/layer12_attention_weight_min": -22.649322509765625,
      "activations/layer13_attention_weight_max": 37.823143005371094,
      "activations/layer13_attention_weight_min": -33.00788497924805,
      "activations/layer14_attention_weight_max": 61.81746292114258,
      "activations/layer14_attention_weight_min": -43.96537399291992,
      "activations/layer15_attention_weight_max": 38.76743698120117,
      "activations/layer15_attention_weight_min": -26.85101890563965,
      "activations/layer16_attention_weight_max": 33.72930145263672,
      "activations/layer16_attention_weight_min": -25.919761657714844,
      "activations/layer17_attention_weight_max": 27.483203887939453,
      "activations/layer17_attention_weight_min": -16.149656295776367,
      "activations/layer18_attention_weight_max": 22.961063385009766,
      "activations/layer18_attention_weight_min": -11.522907257080078,
      "activations/layer19_attention_weight_max": 35.18491744995117,
      "activations/layer19_attention_weight_min": -18.38980484008789,
      "activations/layer1_attention_weight_max": 14.252126693725586,
      "activations/layer1_attention_weight_min": -12.57579231262207,
      "activations/layer20_attention_weight_max": 22.0322208404541,
      "activations/layer20_attention_weight_min": -12.142050743103027,
      "activations/layer21_attention_weight_max": 19.262529373168945,
      "activations/layer21_attention_weight_min": -12.985252380371094,
      "activations/layer22_attention_weight_max": 34.227256774902344,
      "activations/layer22_attention_weight_min": -16.369850158691406,
      "activations/layer23_attention_weight_max": 23.640968322753906,
      "activations/layer23_attention_weight_min": -13.739765167236328,
      "activations/layer2_attention_weight_max": 26.829544067382812,
      "activations/layer2_attention_weight_min": -24.179847717285156,
      "activations/layer3_attention_weight_max": 51.319847106933594,
      "activations/layer3_attention_weight_min": -52.90931701660156,
      "activations/layer4_attention_weight_max": 107.3634262084961,
      "activations/layer4_attention_weight_min": -106.29100799560547,
      "activations/layer5_attention_weight_max": 120.0625228881836,
      "activations/layer5_attention_weight_min": -112.3792724609375,
      "activations/layer6_attention_weight_max": 71.57500457763672,
      "activations/layer6_attention_weight_min": -65.9112548828125,
      "activations/layer7_attention_weight_max": 64.633056640625,
      "activations/layer7_attention_weight_min": -62.51205825805664,
      "activations/layer8_attention_weight_max": 61.53623580932617,
      "activations/layer8_attention_weight_min": -56.628379821777344,
      "activations/layer9_attention_weight_max": 66.54537963867188,
      "activations/layer9_attention_weight_min": -61.931365966796875,
      "epoch": 0.62,
      "learning_rate": 0.0001474715909090909,
      "loss": 3.2148,
      "step": 10700
    },
    {
      "activations/layer0_attention_weight_max": 15.803483009338379,
      "activations/layer0_attention_weight_min": -11.062650680541992,
      "activations/layer10_attention_weight_max": 49.74222183227539,
      "activations/layer10_attention_weight_min": -48.7426872253418,
      "activations/layer11_attention_weight_max": 41.15638732910156,
      "activations/layer11_attention_weight_min": -38.31501388549805,
      "activations/layer12_attention_weight_max": 28.871919631958008,
      "activations/layer12_attention_weight_min": -24.72787094116211,
      "activations/layer13_attention_weight_max": 39.66306686401367,
      "activations/layer13_attention_weight_min": -32.05109786987305,
      "activations/layer14_attention_weight_max": 62.118892669677734,
      "activations/layer14_attention_weight_min": -45.19292068481445,
      "activations/layer15_attention_weight_max": 47.36744689941406,
      "activations/layer15_attention_weight_min": -31.301807403564453,
      "activations/layer16_attention_weight_max": 38.12575912475586,
      "activations/layer16_attention_weight_min": -28.363624572753906,
      "activations/layer17_attention_weight_max": 24.683603286743164,
      "activations/layer17_attention_weight_min": -16.466278076171875,
      "activations/layer18_attention_weight_max": 17.207918167114258,
      "activations/layer18_attention_weight_min": -13.08167839050293,
      "activations/layer19_attention_weight_max": 31.82246971130371,
      "activations/layer19_attention_weight_min": -21.62126922607422,
      "activations/layer1_attention_weight_max": 13.460997581481934,
      "activations/layer1_attention_weight_min": -10.393268585205078,
      "activations/layer20_attention_weight_max": 19.91810417175293,
      "activations/layer20_attention_weight_min": -12.19875717163086,
      "activations/layer21_attention_weight_max": 19.274240493774414,
      "activations/layer21_attention_weight_min": -11.66583251953125,
      "activations/layer22_attention_weight_max": 35.13105010986328,
      "activations/layer22_attention_weight_min": -16.762104034423828,
      "activations/layer23_attention_weight_max": 25.059728622436523,
      "activations/layer23_attention_weight_min": -12.589944839477539,
      "activations/layer2_attention_weight_max": 27.664464950561523,
      "activations/layer2_attention_weight_min": -24.796911239624023,
      "activations/layer3_attention_weight_max": 54.87344741821289,
      "activations/layer3_attention_weight_min": -56.67583084106445,
      "activations/layer4_attention_weight_max": 113.8067626953125,
      "activations/layer4_attention_weight_min": -106.41625213623047,
      "activations/layer5_attention_weight_max": 125.14198303222656,
      "activations/layer5_attention_weight_min": -113.6064453125,
      "activations/layer6_attention_weight_max": 69.77500915527344,
      "activations/layer6_attention_weight_min": -69.12393188476562,
      "activations/layer7_attention_weight_max": 68.67552185058594,
      "activations/layer7_attention_weight_min": -66.94803619384766,
      "activations/layer8_attention_weight_max": 68.9818115234375,
      "activations/layer8_attention_weight_min": -56.58014678955078,
      "activations/layer9_attention_weight_max": 65.70629119873047,
      "activations/layer9_attention_weight_min": -58.29254150390625,
      "epoch": 0.62,
      "learning_rate": 0.00014745265151515152,
      "loss": 3.2334,
      "step": 10750
    },
    {
      "activations/layer0_attention_weight_max": 15.07091999053955,
      "activations/layer0_attention_weight_min": -10.430500030517578,
      "activations/layer10_attention_weight_max": 57.2696647644043,
      "activations/layer10_attention_weight_min": -51.1175422668457,
      "activations/layer11_attention_weight_max": 49.19947052001953,
      "activations/layer11_attention_weight_min": -39.14783477783203,
      "activations/layer12_attention_weight_max": 31.110822677612305,
      "activations/layer12_attention_weight_min": -25.010040283203125,
      "activations/layer13_attention_weight_max": 43.91709899902344,
      "activations/layer13_attention_weight_min": -32.3309440612793,
      "activations/layer14_attention_weight_max": 66.33859252929688,
      "activations/layer14_attention_weight_min": -48.831275939941406,
      "activations/layer15_attention_weight_max": 38.672271728515625,
      "activations/layer15_attention_weight_min": -32.95076370239258,
      "activations/layer16_attention_weight_max": 36.189170837402344,
      "activations/layer16_attention_weight_min": -25.66952133178711,
      "activations/layer17_attention_weight_max": 24.402616500854492,
      "activations/layer17_attention_weight_min": -19.3085880279541,
      "activations/layer18_attention_weight_max": 19.197378158569336,
      "activations/layer18_attention_weight_min": -13.808506965637207,
      "activations/layer19_attention_weight_max": 33.614322662353516,
      "activations/layer19_attention_weight_min": -24.391704559326172,
      "activations/layer1_attention_weight_max": 13.94129467010498,
      "activations/layer1_attention_weight_min": -11.322732925415039,
      "activations/layer20_attention_weight_max": 21.453330993652344,
      "activations/layer20_attention_weight_min": -16.379905700683594,
      "activations/layer21_attention_weight_max": 20.160297393798828,
      "activations/layer21_attention_weight_min": -13.21071720123291,
      "activations/layer22_attention_weight_max": 30.962989807128906,
      "activations/layer22_attention_weight_min": -19.35907745361328,
      "activations/layer23_attention_weight_max": 21.246387481689453,
      "activations/layer23_attention_weight_min": -17.323835372924805,
      "activations/layer2_attention_weight_max": 31.38723373413086,
      "activations/layer2_attention_weight_min": -25.580825805664062,
      "activations/layer3_attention_weight_max": 58.93313980102539,
      "activations/layer3_attention_weight_min": -61.74789810180664,
      "activations/layer4_attention_weight_max": 114.96650695800781,
      "activations/layer4_attention_weight_min": -113.1171646118164,
      "activations/layer5_attention_weight_max": 133.11663818359375,
      "activations/layer5_attention_weight_min": -114.28169250488281,
      "activations/layer6_attention_weight_max": 77.45013427734375,
      "activations/layer6_attention_weight_min": -65.43856048583984,
      "activations/layer7_attention_weight_max": 69.05138397216797,
      "activations/layer7_attention_weight_min": -67.11680603027344,
      "activations/layer8_attention_weight_max": 70.30133056640625,
      "activations/layer8_attention_weight_min": -54.48744583129883,
      "activations/layer9_attention_weight_max": 67.5206069946289,
      "activations/layer9_attention_weight_min": -60.87250518798828,
      "epoch": 0.63,
      "learning_rate": 0.0001474337121212121,
      "loss": 3.2184,
      "step": 10800
    },
    {
      "activations/layer0_attention_weight_max": 16.709972381591797,
      "activations/layer0_attention_weight_min": -10.253296852111816,
      "activations/layer10_attention_weight_max": 53.434200286865234,
      "activations/layer10_attention_weight_min": -42.76200866699219,
      "activations/layer11_attention_weight_max": 47.76398468017578,
      "activations/layer11_attention_weight_min": -36.601806640625,
      "activations/layer12_attention_weight_max": 32.455772399902344,
      "activations/layer12_attention_weight_min": -23.30171012878418,
      "activations/layer13_attention_weight_max": 42.19818115234375,
      "activations/layer13_attention_weight_min": -35.83844757080078,
      "activations/layer14_attention_weight_max": 67.376953125,
      "activations/layer14_attention_weight_min": -46.93098449707031,
      "activations/layer15_attention_weight_max": 45.68513870239258,
      "activations/layer15_attention_weight_min": -31.5009765625,
      "activations/layer16_attention_weight_max": 40.79273986816406,
      "activations/layer16_attention_weight_min": -25.63250732421875,
      "activations/layer17_attention_weight_max": 36.940731048583984,
      "activations/layer17_attention_weight_min": -21.721603393554688,
      "activations/layer18_attention_weight_max": 25.01251792907715,
      "activations/layer18_attention_weight_min": -13.614506721496582,
      "activations/layer19_attention_weight_max": 36.185218811035156,
      "activations/layer19_attention_weight_min": -20.54366683959961,
      "activations/layer1_attention_weight_max": 13.9091215133667,
      "activations/layer1_attention_weight_min": -11.341526985168457,
      "activations/layer20_attention_weight_max": 26.100217819213867,
      "activations/layer20_attention_weight_min": -13.368776321411133,
      "activations/layer21_attention_weight_max": 21.76382827758789,
      "activations/layer21_attention_weight_min": -14.81984806060791,
      "activations/layer22_attention_weight_max": 34.42560577392578,
      "activations/layer22_attention_weight_min": -18.12017250061035,
      "activations/layer23_attention_weight_max": 23.557897567749023,
      "activations/layer23_attention_weight_min": -13.918651580810547,
      "activations/layer2_attention_weight_max": 30.2203369140625,
      "activations/layer2_attention_weight_min": -24.119426727294922,
      "activations/layer3_attention_weight_max": 57.22532272338867,
      "activations/layer3_attention_weight_min": -56.15052795410156,
      "activations/layer4_attention_weight_max": 115.28966522216797,
      "activations/layer4_attention_weight_min": -113.1970443725586,
      "activations/layer5_attention_weight_max": 140.16793823242188,
      "activations/layer5_attention_weight_min": -124.45185852050781,
      "activations/layer6_attention_weight_max": 81.00505828857422,
      "activations/layer6_attention_weight_min": -67.38815307617188,
      "activations/layer7_attention_weight_max": 72.9445571899414,
      "activations/layer7_attention_weight_min": -62.68934631347656,
      "activations/layer8_attention_weight_max": 67.07788848876953,
      "activations/layer8_attention_weight_min": -55.7048225402832,
      "activations/layer9_attention_weight_max": 68.07390594482422,
      "activations/layer9_attention_weight_min": -57.668025970458984,
      "epoch": 0.63,
      "learning_rate": 0.00014741477272727272,
      "loss": 3.2242,
      "step": 10850
    },
    {
      "activations/layer0_attention_weight_max": 13.65477180480957,
      "activations/layer0_attention_weight_min": -10.848095893859863,
      "activations/layer10_attention_weight_max": 51.96321487426758,
      "activations/layer10_attention_weight_min": -44.18473434448242,
      "activations/layer11_attention_weight_max": 49.2371826171875,
      "activations/layer11_attention_weight_min": -40.11647415161133,
      "activations/layer12_attention_weight_max": 35.14045333862305,
      "activations/layer12_attention_weight_min": -28.585220336914062,
      "activations/layer13_attention_weight_max": 38.697811126708984,
      "activations/layer13_attention_weight_min": -34.995849609375,
      "activations/layer14_attention_weight_max": 67.2811279296875,
      "activations/layer14_attention_weight_min": -49.323448181152344,
      "activations/layer15_attention_weight_max": 43.23199462890625,
      "activations/layer15_attention_weight_min": -34.58022689819336,
      "activations/layer16_attention_weight_max": 41.63486862182617,
      "activations/layer16_attention_weight_min": -30.806299209594727,
      "activations/layer17_attention_weight_max": 33.5219612121582,
      "activations/layer17_attention_weight_min": -17.050867080688477,
      "activations/layer18_attention_weight_max": 23.045507431030273,
      "activations/layer18_attention_weight_min": -13.076415061950684,
      "activations/layer19_attention_weight_max": 37.1712532043457,
      "activations/layer19_attention_weight_min": -22.758230209350586,
      "activations/layer1_attention_weight_max": 13.718713760375977,
      "activations/layer1_attention_weight_min": -12.068708419799805,
      "activations/layer20_attention_weight_max": 23.44721794128418,
      "activations/layer20_attention_weight_min": -13.452489852905273,
      "activations/layer21_attention_weight_max": 20.517196655273438,
      "activations/layer21_attention_weight_min": -12.230535507202148,
      "activations/layer22_attention_weight_max": 35.15413284301758,
      "activations/layer22_attention_weight_min": -19.334999084472656,
      "activations/layer23_attention_weight_max": 26.408283233642578,
      "activations/layer23_attention_weight_min": -14.65992259979248,
      "activations/layer2_attention_weight_max": 27.837783813476562,
      "activations/layer2_attention_weight_min": -25.006805419921875,
      "activations/layer3_attention_weight_max": 57.42277526855469,
      "activations/layer3_attention_weight_min": -57.05188751220703,
      "activations/layer4_attention_weight_max": 106.77032470703125,
      "activations/layer4_attention_weight_min": -108.06340789794922,
      "activations/layer5_attention_weight_max": 125.59274291992188,
      "activations/layer5_attention_weight_min": -114.09930419921875,
      "activations/layer6_attention_weight_max": 72.26470184326172,
      "activations/layer6_attention_weight_min": -66.18992614746094,
      "activations/layer7_attention_weight_max": 67.8857650756836,
      "activations/layer7_attention_weight_min": -61.421138763427734,
      "activations/layer8_attention_weight_max": 63.06886291503906,
      "activations/layer8_attention_weight_min": -54.86415481567383,
      "activations/layer9_attention_weight_max": 66.20404052734375,
      "activations/layer9_attention_weight_min": -62.67832565307617,
      "epoch": 0.63,
      "learning_rate": 0.0001473958333333333,
      "loss": 3.2404,
      "step": 10900
    },
    {
      "activations/layer0_attention_weight_max": 15.852726936340332,
      "activations/layer0_attention_weight_min": -10.615885734558105,
      "activations/layer10_attention_weight_max": 52.64441680908203,
      "activations/layer10_attention_weight_min": -42.67423629760742,
      "activations/layer11_attention_weight_max": 44.466529846191406,
      "activations/layer11_attention_weight_min": -40.43527603149414,
      "activations/layer12_attention_weight_max": 33.40576171875,
      "activations/layer12_attention_weight_min": -26.709365844726562,
      "activations/layer13_attention_weight_max": 39.99633026123047,
      "activations/layer13_attention_weight_min": -32.637474060058594,
      "activations/layer14_attention_weight_max": 63.63995361328125,
      "activations/layer14_attention_weight_min": -46.6414794921875,
      "activations/layer15_attention_weight_max": 36.12410354614258,
      "activations/layer15_attention_weight_min": -27.477643966674805,
      "activations/layer16_attention_weight_max": 36.700252532958984,
      "activations/layer16_attention_weight_min": -29.814828872680664,
      "activations/layer17_attention_weight_max": 25.414512634277344,
      "activations/layer17_attention_weight_min": -15.077813148498535,
      "activations/layer18_attention_weight_max": 19.37353515625,
      "activations/layer18_attention_weight_min": -10.613297462463379,
      "activations/layer19_attention_weight_max": 30.05681800842285,
      "activations/layer19_attention_weight_min": -22.034832000732422,
      "activations/layer1_attention_weight_max": 14.58707046508789,
      "activations/layer1_attention_weight_min": -11.360404968261719,
      "activations/layer20_attention_weight_max": 22.68897819519043,
      "activations/layer20_attention_weight_min": -14.114635467529297,
      "activations/layer21_attention_weight_max": 24.018089294433594,
      "activations/layer21_attention_weight_min": -14.04737377166748,
      "activations/layer22_attention_weight_max": 37.019615173339844,
      "activations/layer22_attention_weight_min": -19.052169799804688,
      "activations/layer23_attention_weight_max": 28.02334976196289,
      "activations/layer23_attention_weight_min": -14.383722305297852,
      "activations/layer2_attention_weight_max": 26.77324676513672,
      "activations/layer2_attention_weight_min": -25.19170379638672,
      "activations/layer3_attention_weight_max": 53.231163024902344,
      "activations/layer3_attention_weight_min": -52.74809265136719,
      "activations/layer4_attention_weight_max": 105.82293701171875,
      "activations/layer4_attention_weight_min": -103.31428527832031,
      "activations/layer5_attention_weight_max": 122.52572631835938,
      "activations/layer5_attention_weight_min": -112.78585052490234,
      "activations/layer6_attention_weight_max": 71.53255462646484,
      "activations/layer6_attention_weight_min": -65.79497528076172,
      "activations/layer7_attention_weight_max": 68.90673828125,
      "activations/layer7_attention_weight_min": -62.25088119506836,
      "activations/layer8_attention_weight_max": 60.82894515991211,
      "activations/layer8_attention_weight_min": -55.90031814575195,
      "activations/layer9_attention_weight_max": 63.55946731567383,
      "activations/layer9_attention_weight_min": -61.67451095581055,
      "epoch": 0.64,
      "learning_rate": 0.00014737689393939392,
      "loss": 3.2175,
      "step": 10950
    },
    {
      "activations/layer0_attention_weight_max": 15.382967948913574,
      "activations/layer0_attention_weight_min": -11.522963523864746,
      "activations/layer10_attention_weight_max": 50.63993835449219,
      "activations/layer10_attention_weight_min": -44.796939849853516,
      "activations/layer11_attention_weight_max": 49.96908187866211,
      "activations/layer11_attention_weight_min": -37.00537872314453,
      "activations/layer12_attention_weight_max": 35.56865310668945,
      "activations/layer12_attention_weight_min": -28.85429573059082,
      "activations/layer13_attention_weight_max": 42.885005950927734,
      "activations/layer13_attention_weight_min": -35.96331787109375,
      "activations/layer14_attention_weight_max": 66.40169525146484,
      "activations/layer14_attention_weight_min": -46.300689697265625,
      "activations/layer15_attention_weight_max": 46.4116096496582,
      "activations/layer15_attention_weight_min": -29.840373992919922,
      "activations/layer16_attention_weight_max": 45.62522888183594,
      "activations/layer16_attention_weight_min": -35.022274017333984,
      "activations/layer17_attention_weight_max": 31.47714614868164,
      "activations/layer17_attention_weight_min": -18.10720443725586,
      "activations/layer18_attention_weight_max": 23.313671112060547,
      "activations/layer18_attention_weight_min": -11.475441932678223,
      "activations/layer19_attention_weight_max": 40.102561950683594,
      "activations/layer19_attention_weight_min": -25.750898361206055,
      "activations/layer1_attention_weight_max": 14.138519287109375,
      "activations/layer1_attention_weight_min": -11.973541259765625,
      "activations/layer20_attention_weight_max": 22.706445693969727,
      "activations/layer20_attention_weight_min": -11.043424606323242,
      "activations/layer21_attention_weight_max": 20.262332916259766,
      "activations/layer21_attention_weight_min": -12.68115520477295,
      "activations/layer22_attention_weight_max": 36.398155212402344,
      "activations/layer22_attention_weight_min": -16.852027893066406,
      "activations/layer23_attention_weight_max": 23.690494537353516,
      "activations/layer23_attention_weight_min": -11.788843154907227,
      "activations/layer2_attention_weight_max": 27.013145446777344,
      "activations/layer2_attention_weight_min": -23.380908966064453,
      "activations/layer3_attention_weight_max": 53.388145446777344,
      "activations/layer3_attention_weight_min": -53.25227737426758,
      "activations/layer4_attention_weight_max": 108.46331787109375,
      "activations/layer4_attention_weight_min": -104.94685363769531,
      "activations/layer5_attention_weight_max": 124.63241577148438,
      "activations/layer5_attention_weight_min": -116.82481384277344,
      "activations/layer6_attention_weight_max": 73.3104019165039,
      "activations/layer6_attention_weight_min": -68.26344299316406,
      "activations/layer7_attention_weight_max": 69.16740417480469,
      "activations/layer7_attention_weight_min": -63.63017654418945,
      "activations/layer8_attention_weight_max": 65.3335189819336,
      "activations/layer8_attention_weight_min": -59.18447494506836,
      "activations/layer9_attention_weight_max": 75.47978210449219,
      "activations/layer9_attention_weight_min": -69.22722625732422,
      "epoch": 0.64,
      "learning_rate": 0.00014735795454545454,
      "loss": 3.225,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_loss": 3.1640625,
      "eval_runtime": 8.5005,
      "eval_samples_per_second": 505.146,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_openwebtext_loss": 3.1640625,
      "eval_openwebtext_ppl": 23.666546242140512,
      "eval_openwebtext_runtime": 8.5005,
      "eval_openwebtext_samples_per_second": 505.146,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_wikitext_loss": 3.453125,
      "eval_wikitext_ppl": 31.598985005566618,
      "eval_wikitext_runtime": 1.9496,
      "eval_wikitext_samples_per_second": 233.889,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_lambada_loss": 3.68359375,
      "eval_lambada_ppl": 39.78912962559818,
      "eval_lambada_runtime": 9.5393,
      "eval_lambada_samples_per_second": 510.417,
      "step": 11000
    },
    {
      "activations/layer0_attention_weight_max": 17.212434768676758,
      "activations/layer0_attention_weight_min": -10.500569343566895,
      "activations/layer10_attention_weight_max": 44.66025161743164,
      "activations/layer10_attention_weight_min": -41.60835647583008,
      "activations/layer11_attention_weight_max": 46.9154052734375,
      "activations/layer11_attention_weight_min": -40.325767517089844,
      "activations/layer12_attention_weight_max": 28.621538162231445,
      "activations/layer12_attention_weight_min": -26.92230796813965,
      "activations/layer13_attention_weight_max": 32.4626579284668,
      "activations/layer13_attention_weight_min": -31.153221130371094,
      "activations/layer14_attention_weight_max": 51.94581604003906,
      "activations/layer14_attention_weight_min": -52.963863372802734,
      "activations/layer15_attention_weight_max": 35.30268478393555,
      "activations/layer15_attention_weight_min": -31.400915145874023,
      "activations/layer16_attention_weight_max": 28.318098068237305,
      "activations/layer16_attention_weight_min": -27.837574005126953,
      "activations/layer17_attention_weight_max": 32.626808166503906,
      "activations/layer17_attention_weight_min": -18.399368286132812,
      "activations/layer18_attention_weight_max": 19.688499450683594,
      "activations/layer18_attention_weight_min": -11.611230850219727,
      "activations/layer19_attention_weight_max": 30.29960060119629,
      "activations/layer19_attention_weight_min": -21.282329559326172,
      "activations/layer1_attention_weight_max": 13.804915428161621,
      "activations/layer1_attention_weight_min": -10.948081970214844,
      "activations/layer20_attention_weight_max": 18.948219299316406,
      "activations/layer20_attention_weight_min": -11.678422927856445,
      "activations/layer21_attention_weight_max": 18.525390625,
      "activations/layer21_attention_weight_min": -11.761858940124512,
      "activations/layer22_attention_weight_max": 34.604549407958984,
      "activations/layer22_attention_weight_min": -16.72172737121582,
      "activations/layer23_attention_weight_max": 21.977279663085938,
      "activations/layer23_attention_weight_min": -14.295888900756836,
      "activations/layer2_attention_weight_max": 26.156742095947266,
      "activations/layer2_attention_weight_min": -25.012290954589844,
      "activations/layer3_attention_weight_max": 55.92213821411133,
      "activations/layer3_attention_weight_min": -55.25636672973633,
      "activations/layer4_attention_weight_max": 109.0774154663086,
      "activations/layer4_attention_weight_min": -103.75731658935547,
      "activations/layer5_attention_weight_max": 121.97323608398438,
      "activations/layer5_attention_weight_min": -113.62515258789062,
      "activations/layer6_attention_weight_max": 69.54622650146484,
      "activations/layer6_attention_weight_min": -66.29690551757812,
      "activations/layer7_attention_weight_max": 73.79876708984375,
      "activations/layer7_attention_weight_min": -65.4596939086914,
      "activations/layer8_attention_weight_max": 61.78804397583008,
      "activations/layer8_attention_weight_min": -59.6961669921875,
      "activations/layer9_attention_weight_max": 67.33368682861328,
      "activations/layer9_attention_weight_min": -59.69660568237305,
      "epoch": 0.64,
      "learning_rate": 0.00014733939393939393,
      "loss": 3.2216,
      "step": 11050
    },
    {
      "activations/layer0_attention_weight_max": 15.024161338806152,
      "activations/layer0_attention_weight_min": -10.77596664428711,
      "activations/layer10_attention_weight_max": 51.21369171142578,
      "activations/layer10_attention_weight_min": -45.453887939453125,
      "activations/layer11_attention_weight_max": 60.13654327392578,
      "activations/layer11_attention_weight_min": -43.89231491088867,
      "activations/layer12_attention_weight_max": 78.38018035888672,
      "activations/layer12_attention_weight_min": -35.506683349609375,
      "activations/layer13_attention_weight_max": 65.63512420654297,
      "activations/layer13_attention_weight_min": -36.155311584472656,
      "activations/layer14_attention_weight_max": 57.46392059326172,
      "activations/layer14_attention_weight_min": -45.70109176635742,
      "activations/layer15_attention_weight_max": 46.13471221923828,
      "activations/layer15_attention_weight_min": -32.96415328979492,
      "activations/layer16_attention_weight_max": 32.16570281982422,
      "activations/layer16_attention_weight_min": -27.594179153442383,
      "activations/layer17_attention_weight_max": 27.506023406982422,
      "activations/layer17_attention_weight_min": -17.05956268310547,
      "activations/layer18_attention_weight_max": 19.253482818603516,
      "activations/layer18_attention_weight_min": -11.751055717468262,
      "activations/layer19_attention_weight_max": 30.183185577392578,
      "activations/layer19_attention_weight_min": -22.33049964904785,
      "activations/layer1_attention_weight_max": 14.333850860595703,
      "activations/layer1_attention_weight_min": -11.317483901977539,
      "activations/layer20_attention_weight_max": 20.48661994934082,
      "activations/layer20_attention_weight_min": -12.207995414733887,
      "activations/layer21_attention_weight_max": 20.071861267089844,
      "activations/layer21_attention_weight_min": -14.978401184082031,
      "activations/layer22_attention_weight_max": 32.40739440917969,
      "activations/layer22_attention_weight_min": -16.274738311767578,
      "activations/layer23_attention_weight_max": 20.27783203125,
      "activations/layer23_attention_weight_min": -14.119982719421387,
      "activations/layer2_attention_weight_max": 29.67089080810547,
      "activations/layer2_attention_weight_min": -23.89358901977539,
      "activations/layer3_attention_weight_max": 59.680152893066406,
      "activations/layer3_attention_weight_min": -57.74839401245117,
      "activations/layer4_attention_weight_max": 121.3164291381836,
      "activations/layer4_attention_weight_min": -107.39383697509766,
      "activations/layer5_attention_weight_max": 147.0379638671875,
      "activations/layer5_attention_weight_min": -121.08409881591797,
      "activations/layer6_attention_weight_max": 85.50261688232422,
      "activations/layer6_attention_weight_min": -68.35204315185547,
      "activations/layer7_attention_weight_max": 73.87873077392578,
      "activations/layer7_attention_weight_min": -69.88442993164062,
      "activations/layer8_attention_weight_max": 77.89588165283203,
      "activations/layer8_attention_weight_min": -58.89556121826172,
      "activations/layer9_attention_weight_max": 91.36114501953125,
      "activations/layer9_attention_weight_min": -71.16648864746094,
      "epoch": 0.64,
      "learning_rate": 0.00014732045454545452,
      "loss": 3.2227,
      "step": 11100
    },
    {
      "activations/layer0_attention_weight_max": 16.05117416381836,
      "activations/layer0_attention_weight_min": -10.67842960357666,
      "activations/layer10_attention_weight_max": 52.12411880493164,
      "activations/layer10_attention_weight_min": -45.87080001831055,
      "activations/layer11_attention_weight_max": 53.11341094970703,
      "activations/layer11_attention_weight_min": -42.45191955566406,
      "activations/layer12_attention_weight_max": 27.041770935058594,
      "activations/layer12_attention_weight_min": -26.06136703491211,
      "activations/layer13_attention_weight_max": 35.35316467285156,
      "activations/layer13_attention_weight_min": -36.75849533081055,
      "activations/layer14_attention_weight_max": 58.70296859741211,
      "activations/layer14_attention_weight_min": -51.405250549316406,
      "activations/layer15_attention_weight_max": 45.67328643798828,
      "activations/layer15_attention_weight_min": -35.90757751464844,
      "activations/layer16_attention_weight_max": 40.65682601928711,
      "activations/layer16_attention_weight_min": -30.502111434936523,
      "activations/layer17_attention_weight_max": 34.246910095214844,
      "activations/layer17_attention_weight_min": -16.297039031982422,
      "activations/layer18_attention_weight_max": 21.330257415771484,
      "activations/layer18_attention_weight_min": -11.83401107788086,
      "activations/layer19_attention_weight_max": 38.49453353881836,
      "activations/layer19_attention_weight_min": -22.664270401000977,
      "activations/layer1_attention_weight_max": 13.934921264648438,
      "activations/layer1_attention_weight_min": -11.329030990600586,
      "activations/layer20_attention_weight_max": 24.79728889465332,
      "activations/layer20_attention_weight_min": -14.420952796936035,
      "activations/layer21_attention_weight_max": 18.9476318359375,
      "activations/layer21_attention_weight_min": -14.08398723602295,
      "activations/layer22_attention_weight_max": 37.255619049072266,
      "activations/layer22_attention_weight_min": -18.858985900878906,
      "activations/layer23_attention_weight_max": 26.356128692626953,
      "activations/layer23_attention_weight_min": -15.272504806518555,
      "activations/layer2_attention_weight_max": 27.808258056640625,
      "activations/layer2_attention_weight_min": -23.55647850036621,
      "activations/layer3_attention_weight_max": 54.838623046875,
      "activations/layer3_attention_weight_min": -56.05801010131836,
      "activations/layer4_attention_weight_max": 106.6619873046875,
      "activations/layer4_attention_weight_min": -109.12088775634766,
      "activations/layer5_attention_weight_max": 127.502197265625,
      "activations/layer5_attention_weight_min": -117.74458312988281,
      "activations/layer6_attention_weight_max": 77.98616790771484,
      "activations/layer6_attention_weight_min": -69.15465545654297,
      "activations/layer7_attention_weight_max": 71.43708038330078,
      "activations/layer7_attention_weight_min": -63.8841552734375,
      "activations/layer8_attention_weight_max": 67.86180877685547,
      "activations/layer8_attention_weight_min": -57.03630828857422,
      "activations/layer9_attention_weight_max": 68.63020324707031,
      "activations/layer9_attention_weight_min": -59.734588623046875,
      "epoch": 0.65,
      "learning_rate": 0.00014730151515151513,
      "loss": 3.2121,
      "step": 11150
    },
    {
      "activations/layer0_attention_weight_max": 15.341120719909668,
      "activations/layer0_attention_weight_min": -11.0364351272583,
      "activations/layer10_attention_weight_max": 51.98646926879883,
      "activations/layer10_attention_weight_min": -46.28237533569336,
      "activations/layer11_attention_weight_max": 55.149574279785156,
      "activations/layer11_attention_weight_min": -37.03215026855469,
      "activations/layer12_attention_weight_max": 34.24600601196289,
      "activations/layer12_attention_weight_min": -30.474077224731445,
      "activations/layer13_attention_weight_max": 44.65101623535156,
      "activations/layer13_attention_weight_min": -33.12041473388672,
      "activations/layer14_attention_weight_max": 65.96122741699219,
      "activations/layer14_attention_weight_min": -43.61624526977539,
      "activations/layer15_attention_weight_max": 49.59712219238281,
      "activations/layer15_attention_weight_min": -35.58009338378906,
      "activations/layer16_attention_weight_max": 39.618927001953125,
      "activations/layer16_attention_weight_min": -29.942739486694336,
      "activations/layer17_attention_weight_max": 33.37176513671875,
      "activations/layer17_attention_weight_min": -23.400928497314453,
      "activations/layer18_attention_weight_max": 21.542598724365234,
      "activations/layer18_attention_weight_min": -15.378710746765137,
      "activations/layer19_attention_weight_max": 33.752410888671875,
      "activations/layer19_attention_weight_min": -23.54102897644043,
      "activations/layer1_attention_weight_max": 14.042571067810059,
      "activations/layer1_attention_weight_min": -12.669486999511719,
      "activations/layer20_attention_weight_max": 20.857269287109375,
      "activations/layer20_attention_weight_min": -15.171485900878906,
      "activations/layer21_attention_weight_max": 22.580488204956055,
      "activations/layer21_attention_weight_min": -14.368633270263672,
      "activations/layer22_attention_weight_max": 32.216915130615234,
      "activations/layer22_attention_weight_min": -19.892147064208984,
      "activations/layer23_attention_weight_max": 24.00104522705078,
      "activations/layer23_attention_weight_min": -17.790088653564453,
      "activations/layer2_attention_weight_max": 29.58894920349121,
      "activations/layer2_attention_weight_min": -24.945554733276367,
      "activations/layer3_attention_weight_max": 57.34843826293945,
      "activations/layer3_attention_weight_min": -55.70033645629883,
      "activations/layer4_attention_weight_max": 114.22120666503906,
      "activations/layer4_attention_weight_min": -107.0970230102539,
      "activations/layer5_attention_weight_max": 136.97596740722656,
      "activations/layer5_attention_weight_min": -117.3126449584961,
      "activations/layer6_attention_weight_max": 72.111572265625,
      "activations/layer6_attention_weight_min": -66.54336547851562,
      "activations/layer7_attention_weight_max": 70.79417419433594,
      "activations/layer7_attention_weight_min": -67.47821044921875,
      "activations/layer8_attention_weight_max": 69.14019775390625,
      "activations/layer8_attention_weight_min": -59.95320129394531,
      "activations/layer9_attention_weight_max": 74.04814147949219,
      "activations/layer9_attention_weight_min": -72.24091339111328,
      "epoch": 0.65,
      "learning_rate": 0.00014728257575757575,
      "loss": 3.1988,
      "step": 11200
    },
    {
      "activations/layer0_attention_weight_max": 15.071004867553711,
      "activations/layer0_attention_weight_min": -10.769190788269043,
      "activations/layer10_attention_weight_max": 55.26527786254883,
      "activations/layer10_attention_weight_min": -46.430274963378906,
      "activations/layer11_attention_weight_max": 50.67700958251953,
      "activations/layer11_attention_weight_min": -40.774898529052734,
      "activations/layer12_attention_weight_max": 42.33175277709961,
      "activations/layer12_attention_weight_min": -26.98008155822754,
      "activations/layer13_attention_weight_max": 41.478851318359375,
      "activations/layer13_attention_weight_min": -33.07861328125,
      "activations/layer14_attention_weight_max": 70.16666412353516,
      "activations/layer14_attention_weight_min": -45.24491500854492,
      "activations/layer15_attention_weight_max": 41.429298400878906,
      "activations/layer15_attention_weight_min": -31.1147403717041,
      "activations/layer16_attention_weight_max": 38.0240592956543,
      "activations/layer16_attention_weight_min": -28.72602081298828,
      "activations/layer17_attention_weight_max": 26.13180923461914,
      "activations/layer17_attention_weight_min": -15.973027229309082,
      "activations/layer18_attention_weight_max": 15.38208293914795,
      "activations/layer18_attention_weight_min": -12.42941951751709,
      "activations/layer19_attention_weight_max": 35.5949592590332,
      "activations/layer19_attention_weight_min": -22.23084831237793,
      "activations/layer1_attention_weight_max": 14.998994827270508,
      "activations/layer1_attention_weight_min": -11.762580871582031,
      "activations/layer20_attention_weight_max": 23.704267501831055,
      "activations/layer20_attention_weight_min": -11.45627498626709,
      "activations/layer21_attention_weight_max": 19.990381240844727,
      "activations/layer21_attention_weight_min": -13.2841215133667,
      "activations/layer22_attention_weight_max": 30.085556030273438,
      "activations/layer22_attention_weight_min": -15.432223320007324,
      "activations/layer23_attention_weight_max": 20.105541229248047,
      "activations/layer23_attention_weight_min": -13.302131652832031,
      "activations/layer2_attention_weight_max": 27.607175827026367,
      "activations/layer2_attention_weight_min": -23.393329620361328,
      "activations/layer3_attention_weight_max": 56.50811004638672,
      "activations/layer3_attention_weight_min": -53.43568420410156,
      "activations/layer4_attention_weight_max": 111.8912124633789,
      "activations/layer4_attention_weight_min": -100.88680267333984,
      "activations/layer5_attention_weight_max": 124.53673553466797,
      "activations/layer5_attention_weight_min": -110.62179565429688,
      "activations/layer6_attention_weight_max": 75.59329986572266,
      "activations/layer6_attention_weight_min": -65.69912719726562,
      "activations/layer7_attention_weight_max": 71.82025909423828,
      "activations/layer7_attention_weight_min": -65.917236328125,
      "activations/layer8_attention_weight_max": 67.31840515136719,
      "activations/layer8_attention_weight_min": -58.15032958984375,
      "activations/layer9_attention_weight_max": 79.61383819580078,
      "activations/layer9_attention_weight_min": -65.87284088134766,
      "epoch": 0.65,
      "learning_rate": 0.00014726363636363634,
      "loss": 3.2321,
      "step": 11250
    },
    {
      "activations/layer0_attention_weight_max": 16.085601806640625,
      "activations/layer0_attention_weight_min": -10.296952247619629,
      "activations/layer10_attention_weight_max": 49.829708099365234,
      "activations/layer10_attention_weight_min": -44.38935852050781,
      "activations/layer11_attention_weight_max": 44.243892669677734,
      "activations/layer11_attention_weight_min": -40.07593536376953,
      "activations/layer12_attention_weight_max": 31.654315948486328,
      "activations/layer12_attention_weight_min": -26.53972053527832,
      "activations/layer13_attention_weight_max": 34.722145080566406,
      "activations/layer13_attention_weight_min": -30.75801658630371,
      "activations/layer14_attention_weight_max": 51.8917236328125,
      "activations/layer14_attention_weight_min": -46.3604621887207,
      "activations/layer15_attention_weight_max": 36.44965362548828,
      "activations/layer15_attention_weight_min": -30.73243522644043,
      "activations/layer16_attention_weight_max": 33.19156265258789,
      "activations/layer16_attention_weight_min": -31.80524444580078,
      "activations/layer17_attention_weight_max": 27.458309173583984,
      "activations/layer17_attention_weight_min": -18.153606414794922,
      "activations/layer18_attention_weight_max": 21.367006301879883,
      "activations/layer18_attention_weight_min": -12.691905975341797,
      "activations/layer19_attention_weight_max": 33.28254318237305,
      "activations/layer19_attention_weight_min": -25.407184600830078,
      "activations/layer1_attention_weight_max": 14.564791679382324,
      "activations/layer1_attention_weight_min": -11.510905265808105,
      "activations/layer20_attention_weight_max": 19.794967651367188,
      "activations/layer20_attention_weight_min": -16.260156631469727,
      "activations/layer21_attention_weight_max": 19.801902770996094,
      "activations/layer21_attention_weight_min": -12.6672945022583,
      "activations/layer22_attention_weight_max": 27.90791893005371,
      "activations/layer22_attention_weight_min": -19.151500701904297,
      "activations/layer23_attention_weight_max": 20.12774658203125,
      "activations/layer23_attention_weight_min": -15.698488235473633,
      "activations/layer2_attention_weight_max": 28.204069137573242,
      "activations/layer2_attention_weight_min": -24.380239486694336,
      "activations/layer3_attention_weight_max": 51.98699951171875,
      "activations/layer3_attention_weight_min": -52.529109954833984,
      "activations/layer4_attention_weight_max": 104.8444595336914,
      "activations/layer4_attention_weight_min": -105.1417465209961,
      "activations/layer5_attention_weight_max": 120.35113525390625,
      "activations/layer5_attention_weight_min": -109.55609130859375,
      "activations/layer6_attention_weight_max": 65.74703979492188,
      "activations/layer6_attention_weight_min": -62.54109191894531,
      "activations/layer7_attention_weight_max": 71.61653137207031,
      "activations/layer7_attention_weight_min": -60.11608123779297,
      "activations/layer8_attention_weight_max": 62.0136604309082,
      "activations/layer8_attention_weight_min": -57.13081741333008,
      "activations/layer9_attention_weight_max": 70.05347442626953,
      "activations/layer9_attention_weight_min": -61.31671142578125,
      "epoch": 0.66,
      "learning_rate": 0.00014724469696969695,
      "loss": 3.2314,
      "step": 11300
    },
    {
      "activations/layer0_attention_weight_max": 15.209884643554688,
      "activations/layer0_attention_weight_min": -10.335171699523926,
      "activations/layer10_attention_weight_max": 53.12996292114258,
      "activations/layer10_attention_weight_min": -42.84516906738281,
      "activations/layer11_attention_weight_max": 61.294342041015625,
      "activations/layer11_attention_weight_min": -44.51494598388672,
      "activations/layer12_attention_weight_max": 38.28508758544922,
      "activations/layer12_attention_weight_min": -30.423933029174805,
      "activations/layer13_attention_weight_max": 44.24950408935547,
      "activations/layer13_attention_weight_min": -32.66668701171875,
      "activations/layer14_attention_weight_max": 67.13875579833984,
      "activations/layer14_attention_weight_min": -43.89548873901367,
      "activations/layer15_attention_weight_max": 48.71564483642578,
      "activations/layer15_attention_weight_min": -31.23476219177246,
      "activations/layer16_attention_weight_max": 36.01311111450195,
      "activations/layer16_attention_weight_min": -28.50496482849121,
      "activations/layer17_attention_weight_max": 31.584707260131836,
      "activations/layer17_attention_weight_min": -16.889009475708008,
      "activations/layer18_attention_weight_max": 21.29866600036621,
      "activations/layer18_attention_weight_min": -12.083293914794922,
      "activations/layer19_attention_weight_max": 35.68024444580078,
      "activations/layer19_attention_weight_min": -25.97113609313965,
      "activations/layer1_attention_weight_max": 14.266125679016113,
      "activations/layer1_attention_weight_min": -12.151724815368652,
      "activations/layer20_attention_weight_max": 26.786540985107422,
      "activations/layer20_attention_weight_min": -13.7509126663208,
      "activations/layer21_attention_weight_max": 22.24221420288086,
      "activations/layer21_attention_weight_min": -12.419029235839844,
      "activations/layer22_attention_weight_max": 34.29793930053711,
      "activations/layer22_attention_weight_min": -19.15312385559082,
      "activations/layer23_attention_weight_max": 22.181921005249023,
      "activations/layer23_attention_weight_min": -12.52295207977295,
      "activations/layer2_attention_weight_max": 29.985532760620117,
      "activations/layer2_attention_weight_min": -26.105365753173828,
      "activations/layer3_attention_weight_max": 54.67940139770508,
      "activations/layer3_attention_weight_min": -54.33831024169922,
      "activations/layer4_attention_weight_max": 109.9582748413086,
      "activations/layer4_attention_weight_min": -104.50267791748047,
      "activations/layer5_attention_weight_max": 129.5330810546875,
      "activations/layer5_attention_weight_min": -111.61260986328125,
      "activations/layer6_attention_weight_max": 76.6722183227539,
      "activations/layer6_attention_weight_min": -65.87240600585938,
      "activations/layer7_attention_weight_max": 70.46814727783203,
      "activations/layer7_attention_weight_min": -67.9540023803711,
      "activations/layer8_attention_weight_max": 64.0159683227539,
      "activations/layer8_attention_weight_min": -55.337520599365234,
      "activations/layer9_attention_weight_max": 69.60124206542969,
      "activations/layer9_attention_weight_min": -67.1661605834961,
      "epoch": 0.66,
      "learning_rate": 0.00014722575757575757,
      "loss": 3.2082,
      "step": 11350
    },
    {
      "activations/layer0_attention_weight_max": 14.598901748657227,
      "activations/layer0_attention_weight_min": -10.752755165100098,
      "activations/layer10_attention_weight_max": 52.079124450683594,
      "activations/layer10_attention_weight_min": -42.575958251953125,
      "activations/layer11_attention_weight_max": 52.08639144897461,
      "activations/layer11_attention_weight_min": -37.77190017700195,
      "activations/layer12_attention_weight_max": 42.03882598876953,
      "activations/layer12_attention_weight_min": -33.61966323852539,
      "activations/layer13_attention_weight_max": 51.308223724365234,
      "activations/layer13_attention_weight_min": -35.05270004272461,
      "activations/layer14_attention_weight_max": 73.77043914794922,
      "activations/layer14_attention_weight_min": -48.59878921508789,
      "activations/layer15_attention_weight_max": 55.577186584472656,
      "activations/layer15_attention_weight_min": -34.07416534423828,
      "activations/layer16_attention_weight_max": 45.81824493408203,
      "activations/layer16_attention_weight_min": -28.251516342163086,
      "activations/layer17_attention_weight_max": 27.579748153686523,
      "activations/layer17_attention_weight_min": -17.808250427246094,
      "activations/layer18_attention_weight_max": 21.367992401123047,
      "activations/layer18_attention_weight_min": -13.442692756652832,
      "activations/layer19_attention_weight_max": 45.864524841308594,
      "activations/layer19_attention_weight_min": -22.377225875854492,
      "activations/layer1_attention_weight_max": 14.488543510437012,
      "activations/layer1_attention_weight_min": -12.118253707885742,
      "activations/layer20_attention_weight_max": 30.42580795288086,
      "activations/layer20_attention_weight_min": -14.706887245178223,
      "activations/layer21_attention_weight_max": 22.02001953125,
      "activations/layer21_attention_weight_min": -12.860865592956543,
      "activations/layer22_attention_weight_max": 32.900245666503906,
      "activations/layer22_attention_weight_min": -16.020368576049805,
      "activations/layer23_attention_weight_max": 22.293203353881836,
      "activations/layer23_attention_weight_min": -14.1807279586792,
      "activations/layer2_attention_weight_max": 29.150028228759766,
      "activations/layer2_attention_weight_min": -25.39044952392578,
      "activations/layer3_attention_weight_max": 56.14055633544922,
      "activations/layer3_attention_weight_min": -53.9376335144043,
      "activations/layer4_attention_weight_max": 111.16957092285156,
      "activations/layer4_attention_weight_min": -105.09175872802734,
      "activations/layer5_attention_weight_max": 136.325927734375,
      "activations/layer5_attention_weight_min": -114.18395233154297,
      "activations/layer6_attention_weight_max": 80.60977172851562,
      "activations/layer6_attention_weight_min": -67.10770416259766,
      "activations/layer7_attention_weight_max": 70.61672973632812,
      "activations/layer7_attention_weight_min": -60.41432571411133,
      "activations/layer8_attention_weight_max": 65.49821472167969,
      "activations/layer8_attention_weight_min": -53.72951889038086,
      "activations/layer9_attention_weight_max": 79.38471221923828,
      "activations/layer9_attention_weight_min": -66.65201568603516,
      "epoch": 0.66,
      "learning_rate": 0.00014720681818181815,
      "loss": 3.2201,
      "step": 11400
    },
    {
      "activations/layer0_attention_weight_max": 15.324013710021973,
      "activations/layer0_attention_weight_min": -10.528216361999512,
      "activations/layer10_attention_weight_max": 51.09367370605469,
      "activations/layer10_attention_weight_min": -42.45589065551758,
      "activations/layer11_attention_weight_max": 49.837799072265625,
      "activations/layer11_attention_weight_min": -42.4782829284668,
      "activations/layer12_attention_weight_max": 29.12525749206543,
      "activations/layer12_attention_weight_min": -26.676395416259766,
      "activations/layer13_attention_weight_max": 45.11484146118164,
      "activations/layer13_attention_weight_min": -36.60999298095703,
      "activations/layer14_attention_weight_max": 66.58719635009766,
      "activations/layer14_attention_weight_min": -49.1059455871582,
      "activations/layer15_attention_weight_max": 44.990909576416016,
      "activations/layer15_attention_weight_min": -30.706174850463867,
      "activations/layer16_attention_weight_max": 45.24217224121094,
      "activations/layer16_attention_weight_min": -30.16064453125,
      "activations/layer17_attention_weight_max": 40.87458419799805,
      "activations/layer17_attention_weight_min": -19.908607482910156,
      "activations/layer18_attention_weight_max": 22.0979061126709,
      "activations/layer18_attention_weight_min": -15.788827896118164,
      "activations/layer19_attention_weight_max": 45.02938461303711,
      "activations/layer19_attention_weight_min": -21.99821662902832,
      "activations/layer1_attention_weight_max": 14.84494400024414,
      "activations/layer1_attention_weight_min": -13.196736335754395,
      "activations/layer20_attention_weight_max": 25.074705123901367,
      "activations/layer20_attention_weight_min": -16.621530532836914,
      "activations/layer21_attention_weight_max": 26.20807456970215,
      "activations/layer21_attention_weight_min": -13.914461135864258,
      "activations/layer22_attention_weight_max": 32.51768112182617,
      "activations/layer22_attention_weight_min": -18.027345657348633,
      "activations/layer23_attention_weight_max": 23.534931182861328,
      "activations/layer23_attention_weight_min": -15.004535675048828,
      "activations/layer2_attention_weight_max": 27.98815155029297,
      "activations/layer2_attention_weight_min": -25.536006927490234,
      "activations/layer3_attention_weight_max": 54.666404724121094,
      "activations/layer3_attention_weight_min": -55.617801666259766,
      "activations/layer4_attention_weight_max": 107.1654281616211,
      "activations/layer4_attention_weight_min": -105.10096740722656,
      "activations/layer5_attention_weight_max": 123.09197998046875,
      "activations/layer5_attention_weight_min": -113.46501922607422,
      "activations/layer6_attention_weight_max": 69.29882049560547,
      "activations/layer6_attention_weight_min": -64.25433349609375,
      "activations/layer7_attention_weight_max": 64.31086730957031,
      "activations/layer7_attention_weight_min": -61.87948989868164,
      "activations/layer8_attention_weight_max": 62.5643196105957,
      "activations/layer8_attention_weight_min": -57.27658462524414,
      "activations/layer9_attention_weight_max": 68.81426239013672,
      "activations/layer9_attention_weight_min": -65.07559204101562,
      "epoch": 0.67,
      "learning_rate": 0.00014718787878787877,
      "loss": 3.1986,
      "step": 11450
    },
    {
      "activations/layer0_attention_weight_max": 14.58858871459961,
      "activations/layer0_attention_weight_min": -10.706209182739258,
      "activations/layer10_attention_weight_max": 50.09086608886719,
      "activations/layer10_attention_weight_min": -44.63472366333008,
      "activations/layer11_attention_weight_max": 48.60820007324219,
      "activations/layer11_attention_weight_min": -41.08806228637695,
      "activations/layer12_attention_weight_max": 32.03205108642578,
      "activations/layer12_attention_weight_min": -23.23514175415039,
      "activations/layer13_attention_weight_max": 36.19588088989258,
      "activations/layer13_attention_weight_min": -32.95945739746094,
      "activations/layer14_attention_weight_max": 62.89545822143555,
      "activations/layer14_attention_weight_min": -45.73887252807617,
      "activations/layer15_attention_weight_max": 48.510860443115234,
      "activations/layer15_attention_weight_min": -33.57157897949219,
      "activations/layer16_attention_weight_max": 38.69582748413086,
      "activations/layer16_attention_weight_min": -28.419870376586914,
      "activations/layer17_attention_weight_max": 26.727331161499023,
      "activations/layer17_attention_weight_min": -19.17637062072754,
      "activations/layer18_attention_weight_max": 23.087005615234375,
      "activations/layer18_attention_weight_min": -12.351583480834961,
      "activations/layer19_attention_weight_max": 36.42131042480469,
      "activations/layer19_attention_weight_min": -23.854480743408203,
      "activations/layer1_attention_weight_max": 15.495512962341309,
      "activations/layer1_attention_weight_min": -11.895575523376465,
      "activations/layer20_attention_weight_max": 20.274112701416016,
      "activations/layer20_attention_weight_min": -12.402512550354004,
      "activations/layer21_attention_weight_max": 19.14639663696289,
      "activations/layer21_attention_weight_min": -13.204318046569824,
      "activations/layer22_attention_weight_max": 31.02886199951172,
      "activations/layer22_attention_weight_min": -17.540977478027344,
      "activations/layer23_attention_weight_max": 23.077415466308594,
      "activations/layer23_attention_weight_min": -16.02056121826172,
      "activations/layer2_attention_weight_max": 28.655012130737305,
      "activations/layer2_attention_weight_min": -25.444358825683594,
      "activations/layer3_attention_weight_max": 55.80870056152344,
      "activations/layer3_attention_weight_min": -55.659881591796875,
      "activations/layer4_attention_weight_max": 108.58434295654297,
      "activations/layer4_attention_weight_min": -103.54853820800781,
      "activations/layer5_attention_weight_max": 122.98802947998047,
      "activations/layer5_attention_weight_min": -113.40617370605469,
      "activations/layer6_attention_weight_max": 73.1981201171875,
      "activations/layer6_attention_weight_min": -63.94622039794922,
      "activations/layer7_attention_weight_max": 72.00740051269531,
      "activations/layer7_attention_weight_min": -63.0208625793457,
      "activations/layer8_attention_weight_max": 61.15635299682617,
      "activations/layer8_attention_weight_min": -56.312625885009766,
      "activations/layer9_attention_weight_max": 65.2197265625,
      "activations/layer9_attention_weight_min": -58.767494201660156,
      "epoch": 0.67,
      "learning_rate": 0.00014716893939393938,
      "loss": 3.2055,
      "step": 11500
    },
    {
      "activations/layer0_attention_weight_max": 15.408124923706055,
      "activations/layer0_attention_weight_min": -10.964750289916992,
      "activations/layer10_attention_weight_max": 53.87918472290039,
      "activations/layer10_attention_weight_min": -48.246482849121094,
      "activations/layer11_attention_weight_max": 49.48095703125,
      "activations/layer11_attention_weight_min": -38.52537155151367,
      "activations/layer12_attention_weight_max": 64.6851577758789,
      "activations/layer12_attention_weight_min": -38.72603988647461,
      "activations/layer13_attention_weight_max": 53.673954010009766,
      "activations/layer13_attention_weight_min": -33.743350982666016,
      "activations/layer14_attention_weight_max": 61.857749938964844,
      "activations/layer14_attention_weight_min": -46.25822067260742,
      "activations/layer15_attention_weight_max": 39.853675842285156,
      "activations/layer15_attention_weight_min": -30.48224639892578,
      "activations/layer16_attention_weight_max": 40.61630630493164,
      "activations/layer16_attention_weight_min": -31.97496795654297,
      "activations/layer17_attention_weight_max": 31.164791107177734,
      "activations/layer17_attention_weight_min": -16.85847282409668,
      "activations/layer18_attention_weight_max": 19.537307739257812,
      "activations/layer18_attention_weight_min": -11.087732315063477,
      "activations/layer19_attention_weight_max": 38.170475006103516,
      "activations/layer19_attention_weight_min": -24.371444702148438,
      "activations/layer1_attention_weight_max": 16.141069412231445,
      "activations/layer1_attention_weight_min": -11.824161529541016,
      "activations/layer20_attention_weight_max": 22.15147590637207,
      "activations/layer20_attention_weight_min": -12.658114433288574,
      "activations/layer21_attention_weight_max": 19.1552791595459,
      "activations/layer21_attention_weight_min": -11.5475492477417,
      "activations/layer22_attention_weight_max": 32.31182098388672,
      "activations/layer22_attention_weight_min": -16.41750144958496,
      "activations/layer23_attention_weight_max": 21.86016082763672,
      "activations/layer23_attention_weight_min": -13.32349681854248,
      "activations/layer2_attention_weight_max": 30.247608184814453,
      "activations/layer2_attention_weight_min": -23.762311935424805,
      "activations/layer3_attention_weight_max": 54.76179885864258,
      "activations/layer3_attention_weight_min": -53.03330993652344,
      "activations/layer4_attention_weight_max": 106.894287109375,
      "activations/layer4_attention_weight_min": -100.61174011230469,
      "activations/layer5_attention_weight_max": 134.33993530273438,
      "activations/layer5_attention_weight_min": -111.37532806396484,
      "activations/layer6_attention_weight_max": 74.19974517822266,
      "activations/layer6_attention_weight_min": -62.931922912597656,
      "activations/layer7_attention_weight_max": 76.55164337158203,
      "activations/layer7_attention_weight_min": -75.4197998046875,
      "activations/layer8_attention_weight_max": 70.36860656738281,
      "activations/layer8_attention_weight_min": -57.38030242919922,
      "activations/layer9_attention_weight_max": 94.95479583740234,
      "activations/layer9_attention_weight_min": -78.96792602539062,
      "epoch": 0.67,
      "learning_rate": 0.00014714999999999997,
      "loss": 3.2106,
      "step": 11550
    },
    {
      "activations/layer0_attention_weight_max": 16.390275955200195,
      "activations/layer0_attention_weight_min": -11.007893562316895,
      "activations/layer10_attention_weight_max": 49.9249382019043,
      "activations/layer10_attention_weight_min": -45.22334671020508,
      "activations/layer11_attention_weight_max": 43.98049545288086,
      "activations/layer11_attention_weight_min": -38.897438049316406,
      "activations/layer12_attention_weight_max": 34.25874328613281,
      "activations/layer12_attention_weight_min": -26.632993698120117,
      "activations/layer13_attention_weight_max": 39.608497619628906,
      "activations/layer13_attention_weight_min": -33.523887634277344,
      "activations/layer14_attention_weight_max": 73.32546997070312,
      "activations/layer14_attention_weight_min": -50.677101135253906,
      "activations/layer15_attention_weight_max": 50.19627380371094,
      "activations/layer15_attention_weight_min": -33.951202392578125,
      "activations/layer16_attention_weight_max": 38.92271041870117,
      "activations/layer16_attention_weight_min": -28.28911018371582,
      "activations/layer17_attention_weight_max": 37.410362243652344,
      "activations/layer17_attention_weight_min": -18.63448143005371,
      "activations/layer18_attention_weight_max": 25.384521484375,
      "activations/layer18_attention_weight_min": -12.960618019104004,
      "activations/layer19_attention_weight_max": 35.88630294799805,
      "activations/layer19_attention_weight_min": -23.241924285888672,
      "activations/layer1_attention_weight_max": 13.556550025939941,
      "activations/layer1_attention_weight_min": -13.273856163024902,
      "activations/layer20_attention_weight_max": 25.475341796875,
      "activations/layer20_attention_weight_min": -14.583845138549805,
      "activations/layer21_attention_weight_max": 22.44576072692871,
      "activations/layer21_attention_weight_min": -13.721427917480469,
      "activations/layer22_attention_weight_max": 35.799957275390625,
      "activations/layer22_attention_weight_min": -17.71488380432129,
      "activations/layer23_attention_weight_max": 23.746381759643555,
      "activations/layer23_attention_weight_min": -13.63672161102295,
      "activations/layer2_attention_weight_max": 28.440975189208984,
      "activations/layer2_attention_weight_min": -24.84347152709961,
      "activations/layer3_attention_weight_max": 54.806739807128906,
      "activations/layer3_attention_weight_min": -54.839111328125,
      "activations/layer4_attention_weight_max": 107.14801025390625,
      "activations/layer4_attention_weight_min": -103.92191314697266,
      "activations/layer5_attention_weight_max": 128.1388397216797,
      "activations/layer5_attention_weight_min": -109.96827697753906,
      "activations/layer6_attention_weight_max": 76.5396957397461,
      "activations/layer6_attention_weight_min": -62.09543991088867,
      "activations/layer7_attention_weight_max": 64.45194244384766,
      "activations/layer7_attention_weight_min": -62.33229446411133,
      "activations/layer8_attention_weight_max": 64.54347229003906,
      "activations/layer8_attention_weight_min": -54.64619064331055,
      "activations/layer9_attention_weight_max": 67.66163635253906,
      "activations/layer9_attention_weight_min": -60.151161193847656,
      "epoch": 0.67,
      "learning_rate": 0.0001471310606060606,
      "loss": 3.1909,
      "step": 11600
    },
    {
      "activations/layer0_attention_weight_max": 15.8793306350708,
      "activations/layer0_attention_weight_min": -11.063232421875,
      "activations/layer10_attention_weight_max": 51.887542724609375,
      "activations/layer10_attention_weight_min": -44.64857482910156,
      "activations/layer11_attention_weight_max": 45.22205352783203,
      "activations/layer11_attention_weight_min": -42.38751983642578,
      "activations/layer12_attention_weight_max": 34.09654998779297,
      "activations/layer12_attention_weight_min": -27.653085708618164,
      "activations/layer13_attention_weight_max": 39.649986267089844,
      "activations/layer13_attention_weight_min": -34.948524475097656,
      "activations/layer14_attention_weight_max": 59.679988861083984,
      "activations/layer14_attention_weight_min": -45.7841682434082,
      "activations/layer15_attention_weight_max": 45.57255554199219,
      "activations/layer15_attention_weight_min": -33.812808990478516,
      "activations/layer16_attention_weight_max": 36.72392654418945,
      "activations/layer16_attention_weight_min": -31.652189254760742,
      "activations/layer17_attention_weight_max": 23.829355239868164,
      "activations/layer17_attention_weight_min": -17.242412567138672,
      "activations/layer18_attention_weight_max": 21.200305938720703,
      "activations/layer18_attention_weight_min": -10.946131706237793,
      "activations/layer19_attention_weight_max": 35.999202728271484,
      "activations/layer19_attention_weight_min": -25.390640258789062,
      "activations/layer1_attention_weight_max": 14.375826835632324,
      "activations/layer1_attention_weight_min": -11.910269737243652,
      "activations/layer20_attention_weight_max": 20.936281204223633,
      "activations/layer20_attention_weight_min": -12.5807466506958,
      "activations/layer21_attention_weight_max": 18.66326904296875,
      "activations/layer21_attention_weight_min": -12.661492347717285,
      "activations/layer22_attention_weight_max": 29.07776641845703,
      "activations/layer22_attention_weight_min": -16.536787033081055,
      "activations/layer23_attention_weight_max": 19.785207748413086,
      "activations/layer23_attention_weight_min": -15.800585746765137,
      "activations/layer2_attention_weight_max": 26.408910751342773,
      "activations/layer2_attention_weight_min": -22.7327938079834,
      "activations/layer3_attention_weight_max": 55.3641471862793,
      "activations/layer3_attention_weight_min": -54.951480865478516,
      "activations/layer4_attention_weight_max": 104.45758056640625,
      "activations/layer4_attention_weight_min": -101.76067352294922,
      "activations/layer5_attention_weight_max": 125.79389953613281,
      "activations/layer5_attention_weight_min": -113.81182098388672,
      "activations/layer6_attention_weight_max": 72.79071807861328,
      "activations/layer6_attention_weight_min": -68.87289428710938,
      "activations/layer7_attention_weight_max": 64.74079895019531,
      "activations/layer7_attention_weight_min": -59.90703582763672,
      "activations/layer8_attention_weight_max": 63.55733108520508,
      "activations/layer8_attention_weight_min": -56.2464485168457,
      "activations/layer9_attention_weight_max": 66.5232162475586,
      "activations/layer9_attention_weight_min": -58.80930709838867,
      "epoch": 0.68,
      "learning_rate": 0.0001471121212121212,
      "loss": 3.2084,
      "step": 11650
    },
    {
      "activations/layer0_attention_weight_max": 14.58848762512207,
      "activations/layer0_attention_weight_min": -11.24937915802002,
      "activations/layer10_attention_weight_max": 52.419456481933594,
      "activations/layer10_attention_weight_min": -42.40217971801758,
      "activations/layer11_attention_weight_max": 52.17083740234375,
      "activations/layer11_attention_weight_min": -41.0069580078125,
      "activations/layer12_attention_weight_max": 30.889888763427734,
      "activations/layer12_attention_weight_min": -24.8200740814209,
      "activations/layer13_attention_weight_max": 38.40444564819336,
      "activations/layer13_attention_weight_min": -32.453617095947266,
      "activations/layer14_attention_weight_max": 60.97598648071289,
      "activations/layer14_attention_weight_min": -50.744144439697266,
      "activations/layer15_attention_weight_max": 41.83208465576172,
      "activations/layer15_attention_weight_min": -30.62757110595703,
      "activations/layer16_attention_weight_max": 37.434444427490234,
      "activations/layer16_attention_weight_min": -27.085893630981445,
      "activations/layer17_attention_weight_max": 29.228755950927734,
      "activations/layer17_attention_weight_min": -18.378904342651367,
      "activations/layer18_attention_weight_max": 20.42007064819336,
      "activations/layer18_attention_weight_min": -11.686661720275879,
      "activations/layer19_attention_weight_max": 43.37445831298828,
      "activations/layer19_attention_weight_min": -23.780004501342773,
      "activations/layer1_attention_weight_max": 15.285096168518066,
      "activations/layer1_attention_weight_min": -12.18140697479248,
      "activations/layer20_attention_weight_max": 25.969011306762695,
      "activations/layer20_attention_weight_min": -12.706884384155273,
      "activations/layer21_attention_weight_max": 19.365848541259766,
      "activations/layer21_attention_weight_min": -11.95110034942627,
      "activations/layer22_attention_weight_max": 32.91394805908203,
      "activations/layer22_attention_weight_min": -18.02958869934082,
      "activations/layer23_attention_weight_max": 20.12293815612793,
      "activations/layer23_attention_weight_min": -13.814959526062012,
      "activations/layer2_attention_weight_max": 28.00836753845215,
      "activations/layer2_attention_weight_min": -25.29094696044922,
      "activations/layer3_attention_weight_max": 54.809234619140625,
      "activations/layer3_attention_weight_min": -56.52902603149414,
      "activations/layer4_attention_weight_max": 106.3442153930664,
      "activations/layer4_attention_weight_min": -101.93042755126953,
      "activations/layer5_attention_weight_max": 123.53553771972656,
      "activations/layer5_attention_weight_min": -109.83587646484375,
      "activations/layer6_attention_weight_max": 72.93710327148438,
      "activations/layer6_attention_weight_min": -65.1617660522461,
      "activations/layer7_attention_weight_max": 70.10702514648438,
      "activations/layer7_attention_weight_min": -61.56783676147461,
      "activations/layer8_attention_weight_max": 61.63941192626953,
      "activations/layer8_attention_weight_min": -54.455810546875,
      "activations/layer9_attention_weight_max": 80.48926544189453,
      "activations/layer9_attention_weight_min": -62.62335968017578,
      "epoch": 0.68,
      "learning_rate": 0.00014709318181818182,
      "loss": 3.2045,
      "step": 11700
    },
    {
      "activations/layer0_attention_weight_max": 15.6669282913208,
      "activations/layer0_attention_weight_min": -11.087587356567383,
      "activations/layer10_attention_weight_max": 50.17148208618164,
      "activations/layer10_attention_weight_min": -43.89081954956055,
      "activations/layer11_attention_weight_max": 47.00569152832031,
      "activations/layer11_attention_weight_min": -37.26837158203125,
      "activations/layer12_attention_weight_max": 35.101593017578125,
      "activations/layer12_attention_weight_min": -29.247507095336914,
      "activations/layer13_attention_weight_max": 38.66106414794922,
      "activations/layer13_attention_weight_min": -36.341064453125,
      "activations/layer14_attention_weight_max": 61.01381301879883,
      "activations/layer14_attention_weight_min": -43.91355895996094,
      "activations/layer15_attention_weight_max": 48.999027252197266,
      "activations/layer15_attention_weight_min": -33.694488525390625,
      "activations/layer16_attention_weight_max": 36.24569320678711,
      "activations/layer16_attention_weight_min": -30.729812622070312,
      "activations/layer17_attention_weight_max": 27.787057876586914,
      "activations/layer17_attention_weight_min": -18.503915786743164,
      "activations/layer18_attention_weight_max": 19.95916748046875,
      "activations/layer18_attention_weight_min": -12.118040084838867,
      "activations/layer19_attention_weight_max": 30.910181045532227,
      "activations/layer19_attention_weight_min": -23.223804473876953,
      "activations/layer1_attention_weight_max": 14.398579597473145,
      "activations/layer1_attention_weight_min": -11.32630729675293,
      "activations/layer20_attention_weight_max": 19.53603172302246,
      "activations/layer20_attention_weight_min": -12.202186584472656,
      "activations/layer21_attention_weight_max": 22.296823501586914,
      "activations/layer21_attention_weight_min": -12.961929321289062,
      "activations/layer22_attention_weight_max": 29.121065139770508,
      "activations/layer22_attention_weight_min": -19.234073638916016,
      "activations/layer23_attention_weight_max": 21.442092895507812,
      "activations/layer23_attention_weight_min": -15.758966445922852,
      "activations/layer2_attention_weight_max": 28.258846282958984,
      "activations/layer2_attention_weight_min": -24.268362045288086,
      "activations/layer3_attention_weight_max": 55.98116683959961,
      "activations/layer3_attention_weight_min": -55.03934860229492,
      "activations/layer4_attention_weight_max": 106.36669921875,
      "activations/layer4_attention_weight_min": -106.8317642211914,
      "activations/layer5_attention_weight_max": 123.31341552734375,
      "activations/layer5_attention_weight_min": -113.95769500732422,
      "activations/layer6_attention_weight_max": 71.85562896728516,
      "activations/layer6_attention_weight_min": -62.6602783203125,
      "activations/layer7_attention_weight_max": 65.59028625488281,
      "activations/layer7_attention_weight_min": -62.068904876708984,
      "activations/layer8_attention_weight_max": 62.96249008178711,
      "activations/layer8_attention_weight_min": -57.1546516418457,
      "activations/layer9_attention_weight_max": 67.20320129394531,
      "activations/layer9_attention_weight_min": -58.31827926635742,
      "epoch": 0.68,
      "learning_rate": 0.0001470742424242424,
      "loss": 3.2035,
      "step": 11750
    },
    {
      "activations/layer0_attention_weight_max": 15.552497863769531,
      "activations/layer0_attention_weight_min": -10.235587120056152,
      "activations/layer10_attention_weight_max": 52.77008056640625,
      "activations/layer10_attention_weight_min": -40.685943603515625,
      "activations/layer11_attention_weight_max": 52.86943054199219,
      "activations/layer11_attention_weight_min": -38.262916564941406,
      "activations/layer12_attention_weight_max": 36.36673355102539,
      "activations/layer12_attention_weight_min": -32.153282165527344,
      "activations/layer13_attention_weight_max": 40.720603942871094,
      "activations/layer13_attention_weight_min": -31.778583526611328,
      "activations/layer14_attention_weight_max": 67.10269927978516,
      "activations/layer14_attention_weight_min": -46.82419204711914,
      "activations/layer15_attention_weight_max": 45.41778564453125,
      "activations/layer15_attention_weight_min": -32.750057220458984,
      "activations/layer16_attention_weight_max": 41.87722396850586,
      "activations/layer16_attention_weight_min": -25.822444915771484,
      "activations/layer17_attention_weight_max": 31.416656494140625,
      "activations/layer17_attention_weight_min": -16.642709732055664,
      "activations/layer18_attention_weight_max": 22.893722534179688,
      "activations/layer18_attention_weight_min": -12.526888847351074,
      "activations/layer19_attention_weight_max": 39.37908935546875,
      "activations/layer19_attention_weight_min": -23.604578018188477,
      "activations/layer1_attention_weight_max": 14.804737091064453,
      "activations/layer1_attention_weight_min": -12.32382869720459,
      "activations/layer20_attention_weight_max": 28.481365203857422,
      "activations/layer20_attention_weight_min": -13.856115341186523,
      "activations/layer21_attention_weight_max": 24.112089157104492,
      "activations/layer21_attention_weight_min": -13.534782409667969,
      "activations/layer22_attention_weight_max": 34.51797866821289,
      "activations/layer22_attention_weight_min": -16.567533493041992,
      "activations/layer23_attention_weight_max": 24.081480026245117,
      "activations/layer23_attention_weight_min": -14.466428756713867,
      "activations/layer2_attention_weight_max": 28.37419891357422,
      "activations/layer2_attention_weight_min": -25.77790069580078,
      "activations/layer3_attention_weight_max": 54.193260192871094,
      "activations/layer3_attention_weight_min": -55.68763732910156,
      "activations/layer4_attention_weight_max": 108.1662368774414,
      "activations/layer4_attention_weight_min": -108.51183319091797,
      "activations/layer5_attention_weight_max": 130.26364135742188,
      "activations/layer5_attention_weight_min": -117.91661071777344,
      "activations/layer6_attention_weight_max": 75.25431060791016,
      "activations/layer6_attention_weight_min": -64.22008514404297,
      "activations/layer7_attention_weight_max": 66.05302429199219,
      "activations/layer7_attention_weight_min": -64.55486297607422,
      "activations/layer8_attention_weight_max": 65.1866683959961,
      "activations/layer8_attention_weight_min": -56.63853073120117,
      "activations/layer9_attention_weight_max": 72.59786224365234,
      "activations/layer9_attention_weight_min": -78.06661224365234,
      "epoch": 0.69,
      "learning_rate": 0.00014705530303030302,
      "loss": 3.2042,
      "step": 11800
    },
    {
      "activations/layer0_attention_weight_max": 14.260466575622559,
      "activations/layer0_attention_weight_min": -11.193132400512695,
      "activations/layer10_attention_weight_max": 53.348751068115234,
      "activations/layer10_attention_weight_min": -47.507904052734375,
      "activations/layer11_attention_weight_max": 45.50531768798828,
      "activations/layer11_attention_weight_min": -41.453575134277344,
      "activations/layer12_attention_weight_max": 31.29595184326172,
      "activations/layer12_attention_weight_min": -25.487485885620117,
      "activations/layer13_attention_weight_max": 41.573760986328125,
      "activations/layer13_attention_weight_min": -31.081016540527344,
      "activations/layer14_attention_weight_max": 63.94925308227539,
      "activations/layer14_attention_weight_min": -52.18246078491211,
      "activations/layer15_attention_weight_max": 46.240230560302734,
      "activations/layer15_attention_weight_min": -38.86604690551758,
      "activations/layer16_attention_weight_max": 43.0163688659668,
      "activations/layer16_attention_weight_min": -31.874874114990234,
      "activations/layer17_attention_weight_max": 29.290557861328125,
      "activations/layer17_attention_weight_min": -18.425947189331055,
      "activations/layer18_attention_weight_max": 21.209205627441406,
      "activations/layer18_attention_weight_min": -12.91933822631836,
      "activations/layer19_attention_weight_max": 42.338802337646484,
      "activations/layer19_attention_weight_min": -27.948486328125,
      "activations/layer1_attention_weight_max": 14.338732719421387,
      "activations/layer1_attention_weight_min": -10.688493728637695,
      "activations/layer20_attention_weight_max": 24.751619338989258,
      "activations/layer20_attention_weight_min": -13.73066234588623,
      "activations/layer21_attention_weight_max": 23.72831153869629,
      "activations/layer21_attention_weight_min": -13.694344520568848,
      "activations/layer22_attention_weight_max": 37.5383186340332,
      "activations/layer22_attention_weight_min": -16.687589645385742,
      "activations/layer23_attention_weight_max": 23.8831787109375,
      "activations/layer23_attention_weight_min": -15.022025108337402,
      "activations/layer2_attention_weight_max": 28.10434341430664,
      "activations/layer2_attention_weight_min": -23.822059631347656,
      "activations/layer3_attention_weight_max": 58.1646842956543,
      "activations/layer3_attention_weight_min": -55.1744499206543,
      "activations/layer4_attention_weight_max": 113.11690521240234,
      "activations/layer4_attention_weight_min": -105.97856903076172,
      "activations/layer5_attention_weight_max": 126.67056274414062,
      "activations/layer5_attention_weight_min": -112.59536743164062,
      "activations/layer6_attention_weight_max": 71.66088104248047,
      "activations/layer6_attention_weight_min": -66.08110809326172,
      "activations/layer7_attention_weight_max": 64.83660888671875,
      "activations/layer7_attention_weight_min": -72.12110900878906,
      "activations/layer8_attention_weight_max": 63.51869201660156,
      "activations/layer8_attention_weight_min": -60.6263427734375,
      "activations/layer9_attention_weight_max": 64.37580108642578,
      "activations/layer9_attention_weight_min": -62.24799346923828,
      "epoch": 0.69,
      "learning_rate": 0.00014703636363636363,
      "loss": 3.1936,
      "step": 11850
    },
    {
      "activations/layer0_attention_weight_max": 15.807863235473633,
      "activations/layer0_attention_weight_min": -11.280638694763184,
      "activations/layer10_attention_weight_max": 52.04844665527344,
      "activations/layer10_attention_weight_min": -42.117454528808594,
      "activations/layer11_attention_weight_max": 52.104637145996094,
      "activations/layer11_attention_weight_min": -36.042972564697266,
      "activations/layer12_attention_weight_max": 28.96233367919922,
      "activations/layer12_attention_weight_min": -27.9165096282959,
      "activations/layer13_attention_weight_max": 38.415061950683594,
      "activations/layer13_attention_weight_min": -33.72690963745117,
      "activations/layer14_attention_weight_max": 61.6273193359375,
      "activations/layer14_attention_weight_min": -45.07496643066406,
      "activations/layer15_attention_weight_max": 39.20657730102539,
      "activations/layer15_attention_weight_min": -31.058069229125977,
      "activations/layer16_attention_weight_max": 36.99180603027344,
      "activations/layer16_attention_weight_min": -31.306045532226562,
      "activations/layer17_attention_weight_max": 28.179100036621094,
      "activations/layer17_attention_weight_min": -17.02709197998047,
      "activations/layer18_attention_weight_max": 19.371347427368164,
      "activations/layer18_attention_weight_min": -12.662919998168945,
      "activations/layer19_attention_weight_max": 39.403167724609375,
      "activations/layer19_attention_weight_min": -27.0864315032959,
      "activations/layer1_attention_weight_max": 14.487045288085938,
      "activations/layer1_attention_weight_min": -11.90929126739502,
      "activations/layer20_attention_weight_max": 24.294586181640625,
      "activations/layer20_attention_weight_min": -15.51473331451416,
      "activations/layer21_attention_weight_max": 17.834632873535156,
      "activations/layer21_attention_weight_min": -11.903849601745605,
      "activations/layer22_attention_weight_max": 33.293270111083984,
      "activations/layer22_attention_weight_min": -16.671005249023438,
      "activations/layer23_attention_weight_max": 23.877920150756836,
      "activations/layer23_attention_weight_min": -12.958850860595703,
      "activations/layer2_attention_weight_max": 28.351713180541992,
      "activations/layer2_attention_weight_min": -24.957332611083984,
      "activations/layer3_attention_weight_max": 58.608455657958984,
      "activations/layer3_attention_weight_min": -56.39918899536133,
      "activations/layer4_attention_weight_max": 108.76220703125,
      "activations/layer4_attention_weight_min": -109.05452728271484,
      "activations/layer5_attention_weight_max": 132.35220336914062,
      "activations/layer5_attention_weight_min": -115.19036102294922,
      "activations/layer6_attention_weight_max": 71.87023162841797,
      "activations/layer6_attention_weight_min": -63.047889709472656,
      "activations/layer7_attention_weight_max": 66.0323486328125,
      "activations/layer7_attention_weight_min": -62.726688385009766,
      "activations/layer8_attention_weight_max": 62.17618942260742,
      "activations/layer8_attention_weight_min": -52.564788818359375,
      "activations/layer9_attention_weight_max": 84.15494537353516,
      "activations/layer9_attention_weight_min": -81.41629791259766,
      "epoch": 0.69,
      "learning_rate": 0.00014701742424242425,
      "loss": 3.2068,
      "step": 11900
    },
    {
      "activations/layer0_attention_weight_max": 16.85970687866211,
      "activations/layer0_attention_weight_min": -11.159017562866211,
      "activations/layer10_attention_weight_max": 46.02328109741211,
      "activations/layer10_attention_weight_min": -42.478057861328125,
      "activations/layer11_attention_weight_max": 49.18716049194336,
      "activations/layer11_attention_weight_min": -38.05254364013672,
      "activations/layer12_attention_weight_max": 30.220714569091797,
      "activations/layer12_attention_weight_min": -25.895755767822266,
      "activations/layer13_attention_weight_max": 34.892879486083984,
      "activations/layer13_attention_weight_min": -34.71909713745117,
      "activations/layer14_attention_weight_max": 57.835052490234375,
      "activations/layer14_attention_weight_min": -54.75971603393555,
      "activations/layer15_attention_weight_max": 39.95950698852539,
      "activations/layer15_attention_weight_min": -32.760013580322266,
      "activations/layer16_attention_weight_max": 34.02969741821289,
      "activations/layer16_attention_weight_min": -28.229753494262695,
      "activations/layer17_attention_weight_max": 28.139162063598633,
      "activations/layer17_attention_weight_min": -17.98859405517578,
      "activations/layer18_attention_weight_max": 19.071075439453125,
      "activations/layer18_attention_weight_min": -13.141228675842285,
      "activations/layer19_attention_weight_max": 27.694168090820312,
      "activations/layer19_attention_weight_min": -25.69805908203125,
      "activations/layer1_attention_weight_max": 15.030245780944824,
      "activations/layer1_attention_weight_min": -12.498714447021484,
      "activations/layer20_attention_weight_max": 22.500089645385742,
      "activations/layer20_attention_weight_min": -14.24361801147461,
      "activations/layer21_attention_weight_max": 18.651630401611328,
      "activations/layer21_attention_weight_min": -13.434575080871582,
      "activations/layer22_attention_weight_max": 35.83247375488281,
      "activations/layer22_attention_weight_min": -19.194440841674805,
      "activations/layer23_attention_weight_max": 25.794233322143555,
      "activations/layer23_attention_weight_min": -13.442965507507324,
      "activations/layer2_attention_weight_max": 29.350879669189453,
      "activations/layer2_attention_weight_min": -25.03082275390625,
      "activations/layer3_attention_weight_max": 57.51567459106445,
      "activations/layer3_attention_weight_min": -55.39931869506836,
      "activations/layer4_attention_weight_max": 112.1829605102539,
      "activations/layer4_attention_weight_min": -108.8478775024414,
      "activations/layer5_attention_weight_max": 131.2432861328125,
      "activations/layer5_attention_weight_min": -114.67276000976562,
      "activations/layer6_attention_weight_max": 73.62300872802734,
      "activations/layer6_attention_weight_min": -70.0821762084961,
      "activations/layer7_attention_weight_max": 67.98120880126953,
      "activations/layer7_attention_weight_min": -68.9998550415039,
      "activations/layer8_attention_weight_max": 61.808349609375,
      "activations/layer8_attention_weight_min": -54.224029541015625,
      "activations/layer9_attention_weight_max": 70.96473693847656,
      "activations/layer9_attention_weight_min": -70.6684341430664,
      "epoch": 0.69,
      "learning_rate": 0.00014699848484848484,
      "loss": 3.1846,
      "step": 11950
    },
    {
      "activations/layer0_attention_weight_max": 15.461079597473145,
      "activations/layer0_attention_weight_min": -10.767298698425293,
      "activations/layer10_attention_weight_max": 51.71997833251953,
      "activations/layer10_attention_weight_min": -45.973915100097656,
      "activations/layer11_attention_weight_max": 41.02457046508789,
      "activations/layer11_attention_weight_min": -36.98470687866211,
      "activations/layer12_attention_weight_max": 37.27450180053711,
      "activations/layer12_attention_weight_min": -30.28310775756836,
      "activations/layer13_attention_weight_max": 44.75331497192383,
      "activations/layer13_attention_weight_min": -39.13862609863281,
      "activations/layer14_attention_weight_max": 66.39256286621094,
      "activations/layer14_attention_weight_min": -49.137107849121094,
      "activations/layer15_attention_weight_max": 42.4969596862793,
      "activations/layer15_attention_weight_min": -32.430877685546875,
      "activations/layer16_attention_weight_max": 43.26706314086914,
      "activations/layer16_attention_weight_min": -33.97843551635742,
      "activations/layer17_attention_weight_max": 27.858774185180664,
      "activations/layer17_attention_weight_min": -17.204368591308594,
      "activations/layer18_attention_weight_max": 20.42898178100586,
      "activations/layer18_attention_weight_min": -11.42732048034668,
      "activations/layer19_attention_weight_max": 39.60765838623047,
      "activations/layer19_attention_weight_min": -25.453628540039062,
      "activations/layer1_attention_weight_max": 14.921969413757324,
      "activations/layer1_attention_weight_min": -11.279656410217285,
      "activations/layer20_attention_weight_max": 25.5462646484375,
      "activations/layer20_attention_weight_min": -12.299691200256348,
      "activations/layer21_attention_weight_max": 20.262924194335938,
      "activations/layer21_attention_weight_min": -13.969223022460938,
      "activations/layer22_attention_weight_max": 35.086265563964844,
      "activations/layer22_attention_weight_min": -18.32355308532715,
      "activations/layer23_attention_weight_max": 22.431880950927734,
      "activations/layer23_attention_weight_min": -15.087085723876953,
      "activations/layer2_attention_weight_max": 26.991315841674805,
      "activations/layer2_attention_weight_min": -26.03363037109375,
      "activations/layer3_attention_weight_max": 56.37238311767578,
      "activations/layer3_attention_weight_min": -58.51580810546875,
      "activations/layer4_attention_weight_max": 109.01419830322266,
      "activations/layer4_attention_weight_min": -109.41832733154297,
      "activations/layer5_attention_weight_max": 122.9737548828125,
      "activations/layer5_attention_weight_min": -120.21243286132812,
      "activations/layer6_attention_weight_max": 69.54491424560547,
      "activations/layer6_attention_weight_min": -65.98593139648438,
      "activations/layer7_attention_weight_max": 63.596092224121094,
      "activations/layer7_attention_weight_min": -68.32825469970703,
      "activations/layer8_attention_weight_max": 60.41025924682617,
      "activations/layer8_attention_weight_min": -59.02558898925781,
      "activations/layer9_attention_weight_max": 69.26412200927734,
      "activations/layer9_attention_weight_min": -60.15336990356445,
      "epoch": 0.7,
      "learning_rate": 0.00014697954545454545,
      "loss": 3.1892,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_loss": 3.140625,
      "eval_runtime": 8.0978,
      "eval_samples_per_second": 530.27,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_openwebtext_loss": 3.140625,
      "eval_openwebtext_ppl": 23.118311288923124,
      "eval_openwebtext_runtime": 8.0978,
      "eval_openwebtext_samples_per_second": 530.27,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_wikitext_loss": 3.42578125,
      "eval_wikitext_ppl": 30.74665629604239,
      "eval_wikitext_runtime": 1.5329,
      "eval_wikitext_samples_per_second": 297.469,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_lambada_loss": 3.578125,
      "eval_lambada_ppl": 35.806340977539655,
      "eval_lambada_runtime": 9.0541,
      "eval_lambada_samples_per_second": 537.765,
      "step": 12000
    },
    {
      "activations/layer0_attention_weight_max": 16.019948959350586,
      "activations/layer0_attention_weight_min": -10.575861930847168,
      "activations/layer10_attention_weight_max": 53.17142868041992,
      "activations/layer10_attention_weight_min": -48.163265228271484,
      "activations/layer11_attention_weight_max": 45.902244567871094,
      "activations/layer11_attention_weight_min": -39.282440185546875,
      "activations/layer12_attention_weight_max": 32.12725830078125,
      "activations/layer12_attention_weight_min": -28.55445098876953,
      "activations/layer13_attention_weight_max": 37.602760314941406,
      "activations/layer13_attention_weight_min": -34.341644287109375,
      "activations/layer14_attention_weight_max": 59.94345474243164,
      "activations/layer14_attention_weight_min": -54.42618942260742,
      "activations/layer15_attention_weight_max": 38.464866638183594,
      "activations/layer15_attention_weight_min": -36.126861572265625,
      "activations/layer16_attention_weight_max": 48.814430236816406,
      "activations/layer16_attention_weight_min": -33.75680923461914,
      "activations/layer17_attention_weight_max": 32.59538650512695,
      "activations/layer17_attention_weight_min": -19.760581970214844,
      "activations/layer18_attention_weight_max": 26.710182189941406,
      "activations/layer18_attention_weight_min": -14.308344841003418,
      "activations/layer19_attention_weight_max": 41.80058288574219,
      "activations/layer19_attention_weight_min": -28.489660263061523,
      "activations/layer1_attention_weight_max": 14.745853424072266,
      "activations/layer1_attention_weight_min": -11.534436225891113,
      "activations/layer20_attention_weight_max": 22.90434455871582,
      "activations/layer20_attention_weight_min": -15.060147285461426,
      "activations/layer21_attention_weight_max": 18.552335739135742,
      "activations/layer21_attention_weight_min": -12.279766082763672,
      "activations/layer22_attention_weight_max": 31.62158966064453,
      "activations/layer22_attention_weight_min": -17.992399215698242,
      "activations/layer23_attention_weight_max": 23.93719482421875,
      "activations/layer23_attention_weight_min": -13.758566856384277,
      "activations/layer2_attention_weight_max": 28.108755111694336,
      "activations/layer2_attention_weight_min": -25.36465072631836,
      "activations/layer3_attention_weight_max": 54.77924346923828,
      "activations/layer3_attention_weight_min": -57.097190856933594,
      "activations/layer4_attention_weight_max": 111.035888671875,
      "activations/layer4_attention_weight_min": -108.9916000366211,
      "activations/layer5_attention_weight_max": 124.96893310546875,
      "activations/layer5_attention_weight_min": -112.41511535644531,
      "activations/layer6_attention_weight_max": 70.75386810302734,
      "activations/layer6_attention_weight_min": -63.56630325317383,
      "activations/layer7_attention_weight_max": 62.489349365234375,
      "activations/layer7_attention_weight_min": -61.63981246948242,
      "activations/layer8_attention_weight_max": 61.03151321411133,
      "activations/layer8_attention_weight_min": -56.9106330871582,
      "activations/layer9_attention_weight_max": 63.53898239135742,
      "activations/layer9_attention_weight_min": -61.77997970581055,
      "epoch": 0.7,
      "learning_rate": 0.00014696098484848484,
      "loss": 3.2033,
      "step": 12050
    },
    {
      "activations/layer0_attention_weight_max": 16.283000946044922,
      "activations/layer0_attention_weight_min": -11.304611206054688,
      "activations/layer10_attention_weight_max": 52.14482116699219,
      "activations/layer10_attention_weight_min": -46.06417465209961,
      "activations/layer11_attention_weight_max": 54.03336715698242,
      "activations/layer11_attention_weight_min": -41.54804611206055,
      "activations/layer12_attention_weight_max": 34.07170867919922,
      "activations/layer12_attention_weight_min": -27.971637725830078,
      "activations/layer13_attention_weight_max": 41.618751525878906,
      "activations/layer13_attention_weight_min": -35.791404724121094,
      "activations/layer14_attention_weight_max": 62.970123291015625,
      "activations/layer14_attention_weight_min": -53.035614013671875,
      "activations/layer15_attention_weight_max": 43.951351165771484,
      "activations/layer15_attention_weight_min": -33.87699890136719,
      "activations/layer16_attention_weight_max": 39.2490348815918,
      "activations/layer16_attention_weight_min": -32.12697982788086,
      "activations/layer17_attention_weight_max": 25.982803344726562,
      "activations/layer17_attention_weight_min": -19.332679748535156,
      "activations/layer18_attention_weight_max": 17.28219223022461,
      "activations/layer18_attention_weight_min": -12.369281768798828,
      "activations/layer19_attention_weight_max": 31.997190475463867,
      "activations/layer19_attention_weight_min": -25.714584350585938,
      "activations/layer1_attention_weight_max": 14.254158973693848,
      "activations/layer1_attention_weight_min": -11.133332252502441,
      "activations/layer20_attention_weight_max": 22.281774520874023,
      "activations/layer20_attention_weight_min": -13.865303993225098,
      "activations/layer21_attention_weight_max": 18.733196258544922,
      "activations/layer21_attention_weight_min": -13.370758056640625,
      "activations/layer22_attention_weight_max": 34.32806396484375,
      "activations/layer22_attention_weight_min": -17.55084800720215,
      "activations/layer23_attention_weight_max": 22.7050838470459,
      "activations/layer23_attention_weight_min": -13.535371780395508,
      "activations/layer2_attention_weight_max": 26.81841278076172,
      "activations/layer2_attention_weight_min": -23.962812423706055,
      "activations/layer3_attention_weight_max": 53.08747482299805,
      "activations/layer3_attention_weight_min": -53.11908721923828,
      "activations/layer4_attention_weight_max": 103.70650482177734,
      "activations/layer4_attention_weight_min": -101.03124237060547,
      "activations/layer5_attention_weight_max": 122.49837493896484,
      "activations/layer5_attention_weight_min": -108.1058578491211,
      "activations/layer6_attention_weight_max": 73.84771728515625,
      "activations/layer6_attention_weight_min": -65.17081451416016,
      "activations/layer7_attention_weight_max": 66.0959701538086,
      "activations/layer7_attention_weight_min": -62.83245849609375,
      "activations/layer8_attention_weight_max": 69.02082824707031,
      "activations/layer8_attention_weight_min": -58.90231704711914,
      "activations/layer9_attention_weight_max": 70.1732406616211,
      "activations/layer9_attention_weight_min": -62.84381866455078,
      "epoch": 0.7,
      "learning_rate": 0.00014694204545454546,
      "loss": 3.215,
      "step": 12100
    },
    {
      "activations/layer0_attention_weight_max": 15.647836685180664,
      "activations/layer0_attention_weight_min": -11.004638671875,
      "activations/layer10_attention_weight_max": 51.63187026977539,
      "activations/layer10_attention_weight_min": -45.13387680053711,
      "activations/layer11_attention_weight_max": 43.66061019897461,
      "activations/layer11_attention_weight_min": -42.863765716552734,
      "activations/layer12_attention_weight_max": 34.29390335083008,
      "activations/layer12_attention_weight_min": -26.266029357910156,
      "activations/layer13_attention_weight_max": 38.362213134765625,
      "activations/layer13_attention_weight_min": -36.114173889160156,
      "activations/layer14_attention_weight_max": 63.543827056884766,
      "activations/layer14_attention_weight_min": -49.315460205078125,
      "activations/layer15_attention_weight_max": 39.540809631347656,
      "activations/layer15_attention_weight_min": -32.14823532104492,
      "activations/layer16_attention_weight_max": 40.9765739440918,
      "activations/layer16_attention_weight_min": -32.7034912109375,
      "activations/layer17_attention_weight_max": 30.5414981842041,
      "activations/layer17_attention_weight_min": -18.068193435668945,
      "activations/layer18_attention_weight_max": 21.052412033081055,
      "activations/layer18_attention_weight_min": -13.128599166870117,
      "activations/layer19_attention_weight_max": 36.1301383972168,
      "activations/layer19_attention_weight_min": -23.391956329345703,
      "activations/layer1_attention_weight_max": 14.827498435974121,
      "activations/layer1_attention_weight_min": -11.3185396194458,
      "activations/layer20_attention_weight_max": 23.90314292907715,
      "activations/layer20_attention_weight_min": -14.296869277954102,
      "activations/layer21_attention_weight_max": 19.24441909790039,
      "activations/layer21_attention_weight_min": -13.776585578918457,
      "activations/layer22_attention_weight_max": 31.934127807617188,
      "activations/layer22_attention_weight_min": -18.039268493652344,
      "activations/layer23_attention_weight_max": 22.747718811035156,
      "activations/layer23_attention_weight_min": -16.613262176513672,
      "activations/layer2_attention_weight_max": 28.018871307373047,
      "activations/layer2_attention_weight_min": -25.342071533203125,
      "activations/layer3_attention_weight_max": 55.806427001953125,
      "activations/layer3_attention_weight_min": -55.523170471191406,
      "activations/layer4_attention_weight_max": 103.61701965332031,
      "activations/layer4_attention_weight_min": -102.8843765258789,
      "activations/layer5_attention_weight_max": 120.63826751708984,
      "activations/layer5_attention_weight_min": -110.44396209716797,
      "activations/layer6_attention_weight_max": 70.66635131835938,
      "activations/layer6_attention_weight_min": -62.81791687011719,
      "activations/layer7_attention_weight_max": 66.87065124511719,
      "activations/layer7_attention_weight_min": -61.97068405151367,
      "activations/layer8_attention_weight_max": 65.96548461914062,
      "activations/layer8_attention_weight_min": -58.57747268676758,
      "activations/layer9_attention_weight_max": 70.49748992919922,
      "activations/layer9_attention_weight_min": -60.558143615722656,
      "epoch": 0.71,
      "learning_rate": 0.00014692310606060605,
      "loss": 3.1912,
      "step": 12150
    },
    {
      "activations/layer0_attention_weight_max": 16.092647552490234,
      "activations/layer0_attention_weight_min": -11.138275146484375,
      "activations/layer10_attention_weight_max": 49.10422897338867,
      "activations/layer10_attention_weight_min": -43.98887252807617,
      "activations/layer11_attention_weight_max": 45.71531677246094,
      "activations/layer11_attention_weight_min": -40.711952209472656,
      "activations/layer12_attention_weight_max": 33.50026321411133,
      "activations/layer12_attention_weight_min": -28.274641036987305,
      "activations/layer13_attention_weight_max": 38.58362579345703,
      "activations/layer13_attention_weight_min": -38.083946228027344,
      "activations/layer14_attention_weight_max": 58.60599136352539,
      "activations/layer14_attention_weight_min": -47.44601058959961,
      "activations/layer15_attention_weight_max": 42.44943618774414,
      "activations/layer15_attention_weight_min": -33.15090560913086,
      "activations/layer16_attention_weight_max": 42.53058624267578,
      "activations/layer16_attention_weight_min": -34.06669235229492,
      "activations/layer17_attention_weight_max": 27.509607315063477,
      "activations/layer17_attention_weight_min": -17.88770294189453,
      "activations/layer18_attention_weight_max": 17.970382690429688,
      "activations/layer18_attention_weight_min": -11.047065734863281,
      "activations/layer19_attention_weight_max": 36.403343200683594,
      "activations/layer19_attention_weight_min": -26.38661003112793,
      "activations/layer1_attention_weight_max": 14.74467945098877,
      "activations/layer1_attention_weight_min": -11.014996528625488,
      "activations/layer20_attention_weight_max": 22.216894149780273,
      "activations/layer20_attention_weight_min": -13.832512855529785,
      "activations/layer21_attention_weight_max": 18.985082626342773,
      "activations/layer21_attention_weight_min": -15.521702766418457,
      "activations/layer22_attention_weight_max": 35.13577651977539,
      "activations/layer22_attention_weight_min": -17.69188117980957,
      "activations/layer23_attention_weight_max": 22.294034957885742,
      "activations/layer23_attention_weight_min": -13.492271423339844,
      "activations/layer2_attention_weight_max": 26.576282501220703,
      "activations/layer2_attention_weight_min": -24.602672576904297,
      "activations/layer3_attention_weight_max": 54.058406829833984,
      "activations/layer3_attention_weight_min": -54.45269012451172,
      "activations/layer4_attention_weight_max": 105.63590240478516,
      "activations/layer4_attention_weight_min": -102.34856414794922,
      "activations/layer5_attention_weight_max": 124.42182922363281,
      "activations/layer5_attention_weight_min": -110.9029541015625,
      "activations/layer6_attention_weight_max": 70.74566650390625,
      "activations/layer6_attention_weight_min": -64.75269317626953,
      "activations/layer7_attention_weight_max": 63.704795837402344,
      "activations/layer7_attention_weight_min": -65.73007202148438,
      "activations/layer8_attention_weight_max": 61.36329650878906,
      "activations/layer8_attention_weight_min": -55.00425338745117,
      "activations/layer9_attention_weight_max": 75.85588836669922,
      "activations/layer9_attention_weight_min": -67.31309509277344,
      "epoch": 0.71,
      "learning_rate": 0.00014690416666666666,
      "loss": 3.1831,
      "step": 12200
    },
    {
      "activations/layer0_attention_weight_max": 15.700215339660645,
      "activations/layer0_attention_weight_min": -10.725624084472656,
      "activations/layer10_attention_weight_max": 58.58332443237305,
      "activations/layer10_attention_weight_min": -47.805233001708984,
      "activations/layer11_attention_weight_max": 52.420894622802734,
      "activations/layer11_attention_weight_min": -40.44572830200195,
      "activations/layer12_attention_weight_max": 36.50722885131836,
      "activations/layer12_attention_weight_min": -28.400205612182617,
      "activations/layer13_attention_weight_max": 45.485530853271484,
      "activations/layer13_attention_weight_min": -34.54586410522461,
      "activations/layer14_attention_weight_max": 71.35166931152344,
      "activations/layer14_attention_weight_min": -49.59043884277344,
      "activations/layer15_attention_weight_max": 49.49280548095703,
      "activations/layer15_attention_weight_min": -37.373653411865234,
      "activations/layer16_attention_weight_max": 41.14656448364258,
      "activations/layer16_attention_weight_min": -31.29893684387207,
      "activations/layer17_attention_weight_max": 32.770606994628906,
      "activations/layer17_attention_weight_min": -17.82217788696289,
      "activations/layer18_attention_weight_max": 22.206939697265625,
      "activations/layer18_attention_weight_min": -12.469600677490234,
      "activations/layer19_attention_weight_max": 38.41777801513672,
      "activations/layer19_attention_weight_min": -26.354536056518555,
      "activations/layer1_attention_weight_max": 14.141204833984375,
      "activations/layer1_attention_weight_min": -11.817145347595215,
      "activations/layer20_attention_weight_max": 25.2860050201416,
      "activations/layer20_attention_weight_min": -14.573723793029785,
      "activations/layer21_attention_weight_max": 18.704195022583008,
      "activations/layer21_attention_weight_min": -15.242512702941895,
      "activations/layer22_attention_weight_max": 31.61745262145996,
      "activations/layer22_attention_weight_min": -15.932015419006348,
      "activations/layer23_attention_weight_max": 23.190418243408203,
      "activations/layer23_attention_weight_min": -15.167158126831055,
      "activations/layer2_attention_weight_max": 27.934247970581055,
      "activations/layer2_attention_weight_min": -25.57769775390625,
      "activations/layer3_attention_weight_max": 55.884586334228516,
      "activations/layer3_attention_weight_min": -56.84263229370117,
      "activations/layer4_attention_weight_max": 106.82386779785156,
      "activations/layer4_attention_weight_min": -110.48396301269531,
      "activations/layer5_attention_weight_max": 124.4952392578125,
      "activations/layer5_attention_weight_min": -120.0599594116211,
      "activations/layer6_attention_weight_max": 70.05067443847656,
      "activations/layer6_attention_weight_min": -63.93144989013672,
      "activations/layer7_attention_weight_max": 71.08599853515625,
      "activations/layer7_attention_weight_min": -62.420467376708984,
      "activations/layer8_attention_weight_max": 67.25375366210938,
      "activations/layer8_attention_weight_min": -58.80034637451172,
      "activations/layer9_attention_weight_max": 69.51595306396484,
      "activations/layer9_attention_weight_min": -63.391639709472656,
      "epoch": 0.71,
      "learning_rate": 0.00014688522727272725,
      "loss": 3.1709,
      "step": 12250
    },
    {
      "activations/layer0_attention_weight_max": 15.743743896484375,
      "activations/layer0_attention_weight_min": -10.543957710266113,
      "activations/layer10_attention_weight_max": 52.0228157043457,
      "activations/layer10_attention_weight_min": -45.06373977661133,
      "activations/layer11_attention_weight_max": 41.9261474609375,
      "activations/layer11_attention_weight_min": -39.072723388671875,
      "activations/layer12_attention_weight_max": 28.39969253540039,
      "activations/layer12_attention_weight_min": -24.65662956237793,
      "activations/layer13_attention_weight_max": 43.569725036621094,
      "activations/layer13_attention_weight_min": -35.222496032714844,
      "activations/layer14_attention_weight_max": 58.88666534423828,
      "activations/layer14_attention_weight_min": -50.40695571899414,
      "activations/layer15_attention_weight_max": 43.26752471923828,
      "activations/layer15_attention_weight_min": -33.45503234863281,
      "activations/layer16_attention_weight_max": 40.453922271728516,
      "activations/layer16_attention_weight_min": -30.957645416259766,
      "activations/layer17_attention_weight_max": 25.35072135925293,
      "activations/layer17_attention_weight_min": -19.135229110717773,
      "activations/layer18_attention_weight_max": 21.76835823059082,
      "activations/layer18_attention_weight_min": -12.749067306518555,
      "activations/layer19_attention_weight_max": 33.658782958984375,
      "activations/layer19_attention_weight_min": -23.417753219604492,
      "activations/layer1_attention_weight_max": 14.656038284301758,
      "activations/layer1_attention_weight_min": -12.266654014587402,
      "activations/layer20_attention_weight_max": 24.043006896972656,
      "activations/layer20_attention_weight_min": -13.926246643066406,
      "activations/layer21_attention_weight_max": 22.37567710876465,
      "activations/layer21_attention_weight_min": -12.933724403381348,
      "activations/layer22_attention_weight_max": 34.1611328125,
      "activations/layer22_attention_weight_min": -19.924358367919922,
      "activations/layer23_attention_weight_max": 22.298587799072266,
      "activations/layer23_attention_weight_min": -15.062727928161621,
      "activations/layer2_attention_weight_max": 26.77838897705078,
      "activations/layer2_attention_weight_min": -24.301855087280273,
      "activations/layer3_attention_weight_max": 55.6660270690918,
      "activations/layer3_attention_weight_min": -56.37751770019531,
      "activations/layer4_attention_weight_max": 110.26570129394531,
      "activations/layer4_attention_weight_min": -102.65313720703125,
      "activations/layer5_attention_weight_max": 129.2489013671875,
      "activations/layer5_attention_weight_min": -115.47527313232422,
      "activations/layer6_attention_weight_max": 69.43294525146484,
      "activations/layer6_attention_weight_min": -61.64029312133789,
      "activations/layer7_attention_weight_max": 65.26366424560547,
      "activations/layer7_attention_weight_min": -66.4183120727539,
      "activations/layer8_attention_weight_max": 61.71723937988281,
      "activations/layer8_attention_weight_min": -54.08153533935547,
      "activations/layer9_attention_weight_max": 67.86177825927734,
      "activations/layer9_attention_weight_min": -65.30328369140625,
      "epoch": 0.71,
      "learning_rate": 0.00014686628787878786,
      "loss": 3.1958,
      "step": 12300
    },
    {
      "activations/layer0_attention_weight_max": 13.987161636352539,
      "activations/layer0_attention_weight_min": -10.602785110473633,
      "activations/layer10_attention_weight_max": 56.87498474121094,
      "activations/layer10_attention_weight_min": -45.94413375854492,
      "activations/layer11_attention_weight_max": 56.01481628417969,
      "activations/layer11_attention_weight_min": -46.34794616699219,
      "activations/layer12_attention_weight_max": 36.11221694946289,
      "activations/layer12_attention_weight_min": -26.88555335998535,
      "activations/layer13_attention_weight_max": 53.138790130615234,
      "activations/layer13_attention_weight_min": -35.088714599609375,
      "activations/layer14_attention_weight_max": 73.4659194946289,
      "activations/layer14_attention_weight_min": -50.347084045410156,
      "activations/layer15_attention_weight_max": 52.48249816894531,
      "activations/layer15_attention_weight_min": -34.84483337402344,
      "activations/layer16_attention_weight_max": 50.00577926635742,
      "activations/layer16_attention_weight_min": -29.090456008911133,
      "activations/layer17_attention_weight_max": 40.00175094604492,
      "activations/layer17_attention_weight_min": -20.946577072143555,
      "activations/layer18_attention_weight_max": 24.20023536682129,
      "activations/layer18_attention_weight_min": -12.316571235656738,
      "activations/layer19_attention_weight_max": 49.089900970458984,
      "activations/layer19_attention_weight_min": -27.788818359375,
      "activations/layer1_attention_weight_max": 15.411191940307617,
      "activations/layer1_attention_weight_min": -11.950970649719238,
      "activations/layer20_attention_weight_max": 26.791765213012695,
      "activations/layer20_attention_weight_min": -14.827342987060547,
      "activations/layer21_attention_weight_max": 20.38680076599121,
      "activations/layer21_attention_weight_min": -13.928807258605957,
      "activations/layer22_attention_weight_max": 37.48344802856445,
      "activations/layer22_attention_weight_min": -20.400671005249023,
      "activations/layer23_attention_weight_max": 26.50191307067871,
      "activations/layer23_attention_weight_min": -14.662360191345215,
      "activations/layer2_attention_weight_max": 29.204450607299805,
      "activations/layer2_attention_weight_min": -26.087390899658203,
      "activations/layer3_attention_weight_max": 56.13376998901367,
      "activations/layer3_attention_weight_min": -55.19594192504883,
      "activations/layer4_attention_weight_max": 111.009033203125,
      "activations/layer4_attention_weight_min": -105.75028228759766,
      "activations/layer5_attention_weight_max": 129.25717163085938,
      "activations/layer5_attention_weight_min": -114.18919372558594,
      "activations/layer6_attention_weight_max": 70.62227630615234,
      "activations/layer6_attention_weight_min": -64.78596496582031,
      "activations/layer7_attention_weight_max": 73.85233306884766,
      "activations/layer7_attention_weight_min": -62.71866989135742,
      "activations/layer8_attention_weight_max": 75.4468994140625,
      "activations/layer8_attention_weight_min": -58.01528549194336,
      "activations/layer9_attention_weight_max": 66.58354949951172,
      "activations/layer9_attention_weight_min": -57.48847579956055,
      "epoch": 0.72,
      "learning_rate": 0.00014684734848484848,
      "loss": 3.1924,
      "step": 12350
    },
    {
      "activations/layer0_attention_weight_max": 14.049217224121094,
      "activations/layer0_attention_weight_min": -11.192649841308594,
      "activations/layer10_attention_weight_max": 59.16255569458008,
      "activations/layer10_attention_weight_min": -45.80043029785156,
      "activations/layer11_attention_weight_max": 57.33982849121094,
      "activations/layer11_attention_weight_min": -41.652870178222656,
      "activations/layer12_attention_weight_max": 42.38846206665039,
      "activations/layer12_attention_weight_min": -28.71497344970703,
      "activations/layer13_attention_weight_max": 56.97503662109375,
      "activations/layer13_attention_weight_min": -34.54899215698242,
      "activations/layer14_attention_weight_max": 60.3182258605957,
      "activations/layer14_attention_weight_min": -48.098052978515625,
      "activations/layer15_attention_weight_max": 62.03793716430664,
      "activations/layer15_attention_weight_min": -31.154489517211914,
      "activations/layer16_attention_weight_max": 56.51491165161133,
      "activations/layer16_attention_weight_min": -30.21930694580078,
      "activations/layer17_attention_weight_max": 26.388734817504883,
      "activations/layer17_attention_weight_min": -15.137773513793945,
      "activations/layer18_attention_weight_max": 22.931180953979492,
      "activations/layer18_attention_weight_min": -12.487430572509766,
      "activations/layer19_attention_weight_max": 43.86283493041992,
      "activations/layer19_attention_weight_min": -25.15776252746582,
      "activations/layer1_attention_weight_max": 15.276590347290039,
      "activations/layer1_attention_weight_min": -10.995413780212402,
      "activations/layer20_attention_weight_max": 25.707439422607422,
      "activations/layer20_attention_weight_min": -13.046319007873535,
      "activations/layer21_attention_weight_max": 18.816404342651367,
      "activations/layer21_attention_weight_min": -13.528524398803711,
      "activations/layer22_attention_weight_max": 37.823604583740234,
      "activations/layer22_attention_weight_min": -16.103343963623047,
      "activations/layer23_attention_weight_max": 24.438186645507812,
      "activations/layer23_attention_weight_min": -13.127138137817383,
      "activations/layer2_attention_weight_max": 28.84607696533203,
      "activations/layer2_attention_weight_min": -24.00631332397461,
      "activations/layer3_attention_weight_max": 59.09111022949219,
      "activations/layer3_attention_weight_min": -54.82984924316406,
      "activations/layer4_attention_weight_max": 117.94258880615234,
      "activations/layer4_attention_weight_min": -104.62593078613281,
      "activations/layer5_attention_weight_max": 141.9766845703125,
      "activations/layer5_attention_weight_min": -115.09126281738281,
      "activations/layer6_attention_weight_max": 83.1613540649414,
      "activations/layer6_attention_weight_min": -63.45820999145508,
      "activations/layer7_attention_weight_max": 71.12776947021484,
      "activations/layer7_attention_weight_min": -70.99182891845703,
      "activations/layer8_attention_weight_max": 71.11658477783203,
      "activations/layer8_attention_weight_min": -54.81581497192383,
      "activations/layer9_attention_weight_max": 80.40679168701172,
      "activations/layer9_attention_weight_min": -74.52410888671875,
      "epoch": 0.72,
      "learning_rate": 0.00014682840909090907,
      "loss": 3.1982,
      "step": 12400
    },
    {
      "activations/layer0_attention_weight_max": 15.763396263122559,
      "activations/layer0_attention_weight_min": -11.373909950256348,
      "activations/layer10_attention_weight_max": 54.78456497192383,
      "activations/layer10_attention_weight_min": -47.12870788574219,
      "activations/layer11_attention_weight_max": 47.444358825683594,
      "activations/layer11_attention_weight_min": -38.62921905517578,
      "activations/layer12_attention_weight_max": 33.000736236572266,
      "activations/layer12_attention_weight_min": -27.75486946105957,
      "activations/layer13_attention_weight_max": 43.865501403808594,
      "activations/layer13_attention_weight_min": -34.6705322265625,
      "activations/layer14_attention_weight_max": 60.42045211791992,
      "activations/layer14_attention_weight_min": -50.1556510925293,
      "activations/layer15_attention_weight_max": 43.57026672363281,
      "activations/layer15_attention_weight_min": -34.99428176879883,
      "activations/layer16_attention_weight_max": 39.96208572387695,
      "activations/layer16_attention_weight_min": -34.78724670410156,
      "activations/layer17_attention_weight_max": 26.437522888183594,
      "activations/layer17_attention_weight_min": -20.811243057250977,
      "activations/layer18_attention_weight_max": 18.22825813293457,
      "activations/layer18_attention_weight_min": -12.792366027832031,
      "activations/layer19_attention_weight_max": 37.19649887084961,
      "activations/layer19_attention_weight_min": -25.644968032836914,
      "activations/layer1_attention_weight_max": 14.803993225097656,
      "activations/layer1_attention_weight_min": -11.201727867126465,
      "activations/layer20_attention_weight_max": 22.147014617919922,
      "activations/layer20_attention_weight_min": -14.055685997009277,
      "activations/layer21_attention_weight_max": 20.45445442199707,
      "activations/layer21_attention_weight_min": -14.487374305725098,
      "activations/layer22_attention_weight_max": 28.92524528503418,
      "activations/layer22_attention_weight_min": -16.711517333984375,
      "activations/layer23_attention_weight_max": 19.218219757080078,
      "activations/layer23_attention_weight_min": -14.424895286560059,
      "activations/layer2_attention_weight_max": 28.073043823242188,
      "activations/layer2_attention_weight_min": -25.07558250427246,
      "activations/layer3_attention_weight_max": 55.571205139160156,
      "activations/layer3_attention_weight_min": -58.32434844970703,
      "activations/layer4_attention_weight_max": 107.17437744140625,
      "activations/layer4_attention_weight_min": -107.51705169677734,
      "activations/layer5_attention_weight_max": 125.9361572265625,
      "activations/layer5_attention_weight_min": -115.03204345703125,
      "activations/layer6_attention_weight_max": 70.85237884521484,
      "activations/layer6_attention_weight_min": -65.0526123046875,
      "activations/layer7_attention_weight_max": 69.44461059570312,
      "activations/layer7_attention_weight_min": -64.0836181640625,
      "activations/layer8_attention_weight_max": 63.41008377075195,
      "activations/layer8_attention_weight_min": -60.91016387939453,
      "activations/layer9_attention_weight_max": 68.69979095458984,
      "activations/layer9_attention_weight_min": -68.28954315185547,
      "epoch": 0.72,
      "learning_rate": 0.00014680946969696968,
      "loss": 3.1815,
      "step": 12450
    },
    {
      "activations/layer0_attention_weight_max": 15.479596138000488,
      "activations/layer0_attention_weight_min": -10.799261093139648,
      "activations/layer10_attention_weight_max": 46.21415710449219,
      "activations/layer10_attention_weight_min": -44.0861930847168,
      "activations/layer11_attention_weight_max": 43.063865661621094,
      "activations/layer11_attention_weight_min": -39.10576629638672,
      "activations/layer12_attention_weight_max": 42.628204345703125,
      "activations/layer12_attention_weight_min": -37.221893310546875,
      "activations/layer13_attention_weight_max": 39.423927307128906,
      "activations/layer13_attention_weight_min": -37.68210983276367,
      "activations/layer14_attention_weight_max": 57.811038970947266,
      "activations/layer14_attention_weight_min": -51.983394622802734,
      "activations/layer15_attention_weight_max": 46.2850341796875,
      "activations/layer15_attention_weight_min": -36.74336242675781,
      "activations/layer16_attention_weight_max": 38.520938873291016,
      "activations/layer16_attention_weight_min": -35.3411750793457,
      "activations/layer17_attention_weight_max": 25.419687271118164,
      "activations/layer17_attention_weight_min": -18.399965286254883,
      "activations/layer18_attention_weight_max": 18.155773162841797,
      "activations/layer18_attention_weight_min": -12.233109474182129,
      "activations/layer19_attention_weight_max": 37.65730667114258,
      "activations/layer19_attention_weight_min": -26.419607162475586,
      "activations/layer1_attention_weight_max": 14.149765968322754,
      "activations/layer1_attention_weight_min": -11.503835678100586,
      "activations/layer20_attention_weight_max": 27.073028564453125,
      "activations/layer20_attention_weight_min": -16.142316818237305,
      "activations/layer21_attention_weight_max": 19.140911102294922,
      "activations/layer21_attention_weight_min": -14.0892972946167,
      "activations/layer22_attention_weight_max": 31.839143753051758,
      "activations/layer22_attention_weight_min": -19.157548904418945,
      "activations/layer23_attention_weight_max": 20.81383514404297,
      "activations/layer23_attention_weight_min": -14.117908477783203,
      "activations/layer2_attention_weight_max": 27.340248107910156,
      "activations/layer2_attention_weight_min": -25.685192108154297,
      "activations/layer3_attention_weight_max": 55.483314514160156,
      "activations/layer3_attention_weight_min": -55.74687576293945,
      "activations/layer4_attention_weight_max": 104.31074523925781,
      "activations/layer4_attention_weight_min": -103.6484146118164,
      "activations/layer5_attention_weight_max": 118.44764709472656,
      "activations/layer5_attention_weight_min": -111.59806823730469,
      "activations/layer6_attention_weight_max": 65.64229583740234,
      "activations/layer6_attention_weight_min": -60.7775764465332,
      "activations/layer7_attention_weight_max": 62.50002670288086,
      "activations/layer7_attention_weight_min": -67.1378402709961,
      "activations/layer8_attention_weight_max": 56.852867126464844,
      "activations/layer8_attention_weight_min": -51.929412841796875,
      "activations/layer9_attention_weight_max": 72.57168579101562,
      "activations/layer9_attention_weight_min": -69.70372772216797,
      "epoch": 0.73,
      "learning_rate": 0.0001467905303030303,
      "loss": 3.1716,
      "step": 12500
    },
    {
      "activations/layer0_attention_weight_max": 14.356471061706543,
      "activations/layer0_attention_weight_min": -10.84997272491455,
      "activations/layer10_attention_weight_max": 51.34678649902344,
      "activations/layer10_attention_weight_min": -43.81039810180664,
      "activations/layer11_attention_weight_max": 47.197723388671875,
      "activations/layer11_attention_weight_min": -40.021202087402344,
      "activations/layer12_attention_weight_max": 65.1727066040039,
      "activations/layer12_attention_weight_min": -47.177181243896484,
      "activations/layer13_attention_weight_max": 58.984336853027344,
      "activations/layer13_attention_weight_min": -45.58295440673828,
      "activations/layer14_attention_weight_max": 64.97193145751953,
      "activations/layer14_attention_weight_min": -46.404029846191406,
      "activations/layer15_attention_weight_max": 49.781375885009766,
      "activations/layer15_attention_weight_min": -35.061790466308594,
      "activations/layer16_attention_weight_max": 48.28440475463867,
      "activations/layer16_attention_weight_min": -34.67528533935547,
      "activations/layer17_attention_weight_max": 31.969818115234375,
      "activations/layer17_attention_weight_min": -17.700010299682617,
      "activations/layer18_attention_weight_max": 22.86559295654297,
      "activations/layer18_attention_weight_min": -12.005106925964355,
      "activations/layer19_attention_weight_max": 49.23324966430664,
      "activations/layer19_attention_weight_min": -26.242284774780273,
      "activations/layer1_attention_weight_max": 15.076797485351562,
      "activations/layer1_attention_weight_min": -12.292187690734863,
      "activations/layer20_attention_weight_max": 28.868602752685547,
      "activations/layer20_attention_weight_min": -15.120097160339355,
      "activations/layer21_attention_weight_max": 17.92371940612793,
      "activations/layer21_attention_weight_min": -16.019954681396484,
      "activations/layer22_attention_weight_max": 28.56017303466797,
      "activations/layer22_attention_weight_min": -18.29703140258789,
      "activations/layer23_attention_weight_max": 20.9400577545166,
      "activations/layer23_attention_weight_min": -13.019648551940918,
      "activations/layer2_attention_weight_max": 29.01154899597168,
      "activations/layer2_attention_weight_min": -26.8363094329834,
      "activations/layer3_attention_weight_max": 58.19820022583008,
      "activations/layer3_attention_weight_min": -57.79789352416992,
      "activations/layer4_attention_weight_max": 110.28606414794922,
      "activations/layer4_attention_weight_min": -105.41607666015625,
      "activations/layer5_attention_weight_max": 126.35867309570312,
      "activations/layer5_attention_weight_min": -113.41558074951172,
      "activations/layer6_attention_weight_max": 70.97772979736328,
      "activations/layer6_attention_weight_min": -63.168033599853516,
      "activations/layer7_attention_weight_max": 64.80436706542969,
      "activations/layer7_attention_weight_min": -61.71256637573242,
      "activations/layer8_attention_weight_max": 67.24510955810547,
      "activations/layer8_attention_weight_min": -58.11863327026367,
      "activations/layer9_attention_weight_max": 73.35409545898438,
      "activations/layer9_attention_weight_min": -66.4807357788086,
      "epoch": 0.73,
      "learning_rate": 0.00014677159090909089,
      "loss": 3.1861,
      "step": 12550
    },
    {
      "activations/layer0_attention_weight_max": 15.409324645996094,
      "activations/layer0_attention_weight_min": -11.509546279907227,
      "activations/layer10_attention_weight_max": 48.65721130371094,
      "activations/layer10_attention_weight_min": -46.993377685546875,
      "activations/layer11_attention_weight_max": 46.71703338623047,
      "activations/layer11_attention_weight_min": -38.03385925292969,
      "activations/layer12_attention_weight_max": 35.256126403808594,
      "activations/layer12_attention_weight_min": -25.30784797668457,
      "activations/layer13_attention_weight_max": 39.593170166015625,
      "activations/layer13_attention_weight_min": -35.32557678222656,
      "activations/layer14_attention_weight_max": 65.2705078125,
      "activations/layer14_attention_weight_min": -48.61964797973633,
      "activations/layer15_attention_weight_max": 43.911415100097656,
      "activations/layer15_attention_weight_min": -31.153846740722656,
      "activations/layer16_attention_weight_max": 43.71261978149414,
      "activations/layer16_attention_weight_min": -30.985244750976562,
      "activations/layer17_attention_weight_max": 33.5499153137207,
      "activations/layer17_attention_weight_min": -18.296344757080078,
      "activations/layer18_attention_weight_max": 23.394222259521484,
      "activations/layer18_attention_weight_min": -13.590498924255371,
      "activations/layer19_attention_weight_max": 49.187740325927734,
      "activations/layer19_attention_weight_min": -26.560129165649414,
      "activations/layer1_attention_weight_max": 15.117203712463379,
      "activations/layer1_attention_weight_min": -11.100605964660645,
      "activations/layer20_attention_weight_max": 29.624221801757812,
      "activations/layer20_attention_weight_min": -16.099605560302734,
      "activations/layer21_attention_weight_max": 18.434499740600586,
      "activations/layer21_attention_weight_min": -15.145556449890137,
      "activations/layer22_attention_weight_max": 30.22067642211914,
      "activations/layer22_attention_weight_min": -17.708425521850586,
      "activations/layer23_attention_weight_max": 20.319103240966797,
      "activations/layer23_attention_weight_min": -13.97861385345459,
      "activations/layer2_attention_weight_max": 28.095752716064453,
      "activations/layer2_attention_weight_min": -25.931509017944336,
      "activations/layer3_attention_weight_max": 58.13750457763672,
      "activations/layer3_attention_weight_min": -58.064205169677734,
      "activations/layer4_attention_weight_max": 112.88166809082031,
      "activations/layer4_attention_weight_min": -106.06456756591797,
      "activations/layer5_attention_weight_max": 131.08644104003906,
      "activations/layer5_attention_weight_min": -112.13459777832031,
      "activations/layer6_attention_weight_max": 74.45626068115234,
      "activations/layer6_attention_weight_min": -62.500038146972656,
      "activations/layer7_attention_weight_max": 66.8631591796875,
      "activations/layer7_attention_weight_min": -63.219146728515625,
      "activations/layer8_attention_weight_max": 66.22859191894531,
      "activations/layer8_attention_weight_min": -56.39759826660156,
      "activations/layer9_attention_weight_max": 76.620849609375,
      "activations/layer9_attention_weight_min": -63.02704620361328,
      "epoch": 0.73,
      "learning_rate": 0.0001467526515151515,
      "loss": 3.1877,
      "step": 12600
    },
    {
      "activations/layer0_attention_weight_max": 14.66096019744873,
      "activations/layer0_attention_weight_min": -10.705745697021484,
      "activations/layer10_attention_weight_max": 60.98100280761719,
      "activations/layer10_attention_weight_min": -45.40804672241211,
      "activations/layer11_attention_weight_max": 45.80624008178711,
      "activations/layer11_attention_weight_min": -39.09431076049805,
      "activations/layer12_attention_weight_max": 57.29006576538086,
      "activations/layer12_attention_weight_min": -38.11255645751953,
      "activations/layer13_attention_weight_max": 47.14609146118164,
      "activations/layer13_attention_weight_min": -38.026206970214844,
      "activations/layer14_attention_weight_max": 72.29615783691406,
      "activations/layer14_attention_weight_min": -52.16389083862305,
      "activations/layer15_attention_weight_max": 57.314857482910156,
      "activations/layer15_attention_weight_min": -37.32365417480469,
      "activations/layer16_attention_weight_max": 48.339778900146484,
      "activations/layer16_attention_weight_min": -31.31363868713379,
      "activations/layer17_attention_weight_max": 23.89442253112793,
      "activations/layer17_attention_weight_min": -17.411333084106445,
      "activations/layer18_attention_weight_max": 18.74355125427246,
      "activations/layer18_attention_weight_min": -14.181795120239258,
      "activations/layer19_attention_weight_max": 45.787471771240234,
      "activations/layer19_attention_weight_min": -25.926837921142578,
      "activations/layer1_attention_weight_max": 15.482306480407715,
      "activations/layer1_attention_weight_min": -10.798378944396973,
      "activations/layer20_attention_weight_max": 26.77874183654785,
      "activations/layer20_attention_weight_min": -14.946640014648438,
      "activations/layer21_attention_weight_max": 21.611347198486328,
      "activations/layer21_attention_weight_min": -13.894573211669922,
      "activations/layer22_attention_weight_max": 27.906532287597656,
      "activations/layer22_attention_weight_min": -15.852676391601562,
      "activations/layer23_attention_weight_max": 18.943374633789062,
      "activations/layer23_attention_weight_min": -13.828714370727539,
      "activations/layer2_attention_weight_max": 30.126972198486328,
      "activations/layer2_attention_weight_min": -24.441822052001953,
      "activations/layer3_attention_weight_max": 58.30374526977539,
      "activations/layer3_attention_weight_min": -53.94247055053711,
      "activations/layer4_attention_weight_max": 110.37396240234375,
      "activations/layer4_attention_weight_min": -104.09581756591797,
      "activations/layer5_attention_weight_max": 136.80142211914062,
      "activations/layer5_attention_weight_min": -109.38555908203125,
      "activations/layer6_attention_weight_max": 78.12767028808594,
      "activations/layer6_attention_weight_min": -60.316932678222656,
      "activations/layer7_attention_weight_max": 72.54137420654297,
      "activations/layer7_attention_weight_min": -64.57903289794922,
      "activations/layer8_attention_weight_max": 68.30699920654297,
      "activations/layer8_attention_weight_min": -54.934200286865234,
      "activations/layer9_attention_weight_max": 105.78076171875,
      "activations/layer9_attention_weight_min": -78.23928833007812,
      "epoch": 0.74,
      "learning_rate": 0.00014673371212121212,
      "loss": 3.1897,
      "step": 12650
    },
    {
      "activations/layer0_attention_weight_max": 14.889139175415039,
      "activations/layer0_attention_weight_min": -11.00741958618164,
      "activations/layer10_attention_weight_max": 53.08605194091797,
      "activations/layer10_attention_weight_min": -44.008663177490234,
      "activations/layer11_attention_weight_max": 43.5037841796875,
      "activations/layer11_attention_weight_min": -36.43429183959961,
      "activations/layer12_attention_weight_max": 34.92515182495117,
      "activations/layer12_attention_weight_min": -28.609031677246094,
      "activations/layer13_attention_weight_max": 46.16593933105469,
      "activations/layer13_attention_weight_min": -34.18843078613281,
      "activations/layer14_attention_weight_max": 68.95936584472656,
      "activations/layer14_attention_weight_min": -48.92837142944336,
      "activations/layer15_attention_weight_max": 48.29193878173828,
      "activations/layer15_attention_weight_min": -33.176483154296875,
      "activations/layer16_attention_weight_max": 45.22514724731445,
      "activations/layer16_attention_weight_min": -30.292573928833008,
      "activations/layer17_attention_weight_max": 31.15895652770996,
      "activations/layer17_attention_weight_min": -19.448575973510742,
      "activations/layer18_attention_weight_max": 23.459814071655273,
      "activations/layer18_attention_weight_min": -13.947223663330078,
      "activations/layer19_attention_weight_max": 45.93909454345703,
      "activations/layer19_attention_weight_min": -25.391691207885742,
      "activations/layer1_attention_weight_max": 14.83665943145752,
      "activations/layer1_attention_weight_min": -11.730860710144043,
      "activations/layer20_attention_weight_max": 27.268043518066406,
      "activations/layer20_attention_weight_min": -15.556623458862305,
      "activations/layer21_attention_weight_max": 19.029706954956055,
      "activations/layer21_attention_weight_min": -12.922615051269531,
      "activations/layer22_attention_weight_max": 42.101341247558594,
      "activations/layer22_attention_weight_min": -17.369958877563477,
      "activations/layer23_attention_weight_max": 27.58075523376465,
      "activations/layer23_attention_weight_min": -13.962279319763184,
      "activations/layer2_attention_weight_max": 28.283039093017578,
      "activations/layer2_attention_weight_min": -25.39093589782715,
      "activations/layer3_attention_weight_max": 59.39656066894531,
      "activations/layer3_attention_weight_min": -60.831321716308594,
      "activations/layer4_attention_weight_max": 111.84820556640625,
      "activations/layer4_attention_weight_min": -107.58428192138672,
      "activations/layer5_attention_weight_max": 126.77678680419922,
      "activations/layer5_attention_weight_min": -115.11715698242188,
      "activations/layer6_attention_weight_max": 74.30792236328125,
      "activations/layer6_attention_weight_min": -62.75587463378906,
      "activations/layer7_attention_weight_max": 68.42781066894531,
      "activations/layer7_attention_weight_min": -63.76250076293945,
      "activations/layer8_attention_weight_max": 71.82801055908203,
      "activations/layer8_attention_weight_min": -58.3459358215332,
      "activations/layer9_attention_weight_max": 77.08736419677734,
      "activations/layer9_attention_weight_min": -68.96651458740234,
      "epoch": 0.74,
      "learning_rate": 0.0001467147727272727,
      "loss": 3.1797,
      "step": 12700
    },
    {
      "activations/layer0_attention_weight_max": 15.877471923828125,
      "activations/layer0_attention_weight_min": -11.103218078613281,
      "activations/layer10_attention_weight_max": 54.72105026245117,
      "activations/layer10_attention_weight_min": -52.2623291015625,
      "activations/layer11_attention_weight_max": 43.389434814453125,
      "activations/layer11_attention_weight_min": -36.43418884277344,
      "activations/layer12_attention_weight_max": 47.77229690551758,
      "activations/layer12_attention_weight_min": -34.176918029785156,
      "activations/layer13_attention_weight_max": 41.535091400146484,
      "activations/layer13_attention_weight_min": -34.58024597167969,
      "activations/layer14_attention_weight_max": 67.4634017944336,
      "activations/layer14_attention_weight_min": -51.77716827392578,
      "activations/layer15_attention_weight_max": 43.90713882446289,
      "activations/layer15_attention_weight_min": -35.89398956298828,
      "activations/layer16_attention_weight_max": 43.27560043334961,
      "activations/layer16_attention_weight_min": -34.01665115356445,
      "activations/layer17_attention_weight_max": 29.205204010009766,
      "activations/layer17_attention_weight_min": -21.536500930786133,
      "activations/layer18_attention_weight_max": 18.28778648376465,
      "activations/layer18_attention_weight_min": -13.47952938079834,
      "activations/layer19_attention_weight_max": 36.34199523925781,
      "activations/layer19_attention_weight_min": -24.81387710571289,
      "activations/layer1_attention_weight_max": 14.962437629699707,
      "activations/layer1_attention_weight_min": -10.810626983642578,
      "activations/layer20_attention_weight_max": 24.558752059936523,
      "activations/layer20_attention_weight_min": -18.391281127929688,
      "activations/layer21_attention_weight_max": 19.27029037475586,
      "activations/layer21_attention_weight_min": -16.01320457458496,
      "activations/layer22_attention_weight_max": 34.36556625366211,
      "activations/layer22_attention_weight_min": -23.87002182006836,
      "activations/layer23_attention_weight_max": 22.398738861083984,
      "activations/layer23_attention_weight_min": -18.49542999267578,
      "activations/layer2_attention_weight_max": 27.208463668823242,
      "activations/layer2_attention_weight_min": -25.513198852539062,
      "activations/layer3_attention_weight_max": 56.02367401123047,
      "activations/layer3_attention_weight_min": -55.8580322265625,
      "activations/layer4_attention_weight_max": 104.9026870727539,
      "activations/layer4_attention_weight_min": -102.79774475097656,
      "activations/layer5_attention_weight_max": 124.04793548583984,
      "activations/layer5_attention_weight_min": -111.46694946289062,
      "activations/layer6_attention_weight_max": 73.0907211303711,
      "activations/layer6_attention_weight_min": -66.6880874633789,
      "activations/layer7_attention_weight_max": 64.28181457519531,
      "activations/layer7_attention_weight_min": -67.54122924804688,
      "activations/layer8_attention_weight_max": 65.20540618896484,
      "activations/layer8_attention_weight_min": -60.46898651123047,
      "activations/layer9_attention_weight_max": 75.99422454833984,
      "activations/layer9_attention_weight_min": -68.23710632324219,
      "epoch": 0.74,
      "learning_rate": 0.00014669583333333332,
      "loss": 3.1673,
      "step": 12750
    },
    {
      "activations/layer0_attention_weight_max": 15.096508026123047,
      "activations/layer0_attention_weight_min": -10.695724487304688,
      "activations/layer10_attention_weight_max": 51.83818054199219,
      "activations/layer10_attention_weight_min": -48.5933952331543,
      "activations/layer11_attention_weight_max": 49.56346130371094,
      "activations/layer11_attention_weight_min": -44.56759262084961,
      "activations/layer12_attention_weight_max": 32.94747543334961,
      "activations/layer12_attention_weight_min": -28.640653610229492,
      "activations/layer13_attention_weight_max": 38.911094665527344,
      "activations/layer13_attention_weight_min": -34.02464294433594,
      "activations/layer14_attention_weight_max": 65.40996551513672,
      "activations/layer14_attention_weight_min": -53.39706802368164,
      "activations/layer15_attention_weight_max": 52.0444450378418,
      "activations/layer15_attention_weight_min": -37.32299041748047,
      "activations/layer16_attention_weight_max": 41.56428527832031,
      "activations/layer16_attention_weight_min": -33.20657730102539,
      "activations/layer17_attention_weight_max": 26.90268898010254,
      "activations/layer17_attention_weight_min": -19.947412490844727,
      "activations/layer18_attention_weight_max": 21.17418670654297,
      "activations/layer18_attention_weight_min": -11.978134155273438,
      "activations/layer19_attention_weight_max": 40.24748229980469,
      "activations/layer19_attention_weight_min": -36.136966705322266,
      "activations/layer1_attention_weight_max": 14.224119186401367,
      "activations/layer1_attention_weight_min": -11.406990051269531,
      "activations/layer20_attention_weight_max": 27.190340042114258,
      "activations/layer20_attention_weight_min": -14.726739883422852,
      "activations/layer21_attention_weight_max": 19.111217498779297,
      "activations/layer21_attention_weight_min": -14.328120231628418,
      "activations/layer22_attention_weight_max": 32.548274993896484,
      "activations/layer22_attention_weight_min": -16.518836975097656,
      "activations/layer23_attention_weight_max": 20.400951385498047,
      "activations/layer23_attention_weight_min": -14.942800521850586,
      "activations/layer2_attention_weight_max": 27.67458152770996,
      "activations/layer2_attention_weight_min": -25.222606658935547,
      "activations/layer3_attention_weight_max": 52.965999603271484,
      "activations/layer3_attention_weight_min": -55.18806838989258,
      "activations/layer4_attention_weight_max": 104.01380920410156,
      "activations/layer4_attention_weight_min": -105.80206298828125,
      "activations/layer5_attention_weight_max": 125.37651824951172,
      "activations/layer5_attention_weight_min": -116.01390075683594,
      "activations/layer6_attention_weight_max": 65.30879974365234,
      "activations/layer6_attention_weight_min": -61.25537872314453,
      "activations/layer7_attention_weight_max": 61.81620407104492,
      "activations/layer7_attention_weight_min": -66.70069885253906,
      "activations/layer8_attention_weight_max": 63.70279312133789,
      "activations/layer8_attention_weight_min": -56.78377151489258,
      "activations/layer9_attention_weight_max": 79.50741577148438,
      "activations/layer9_attention_weight_min": -67.43316650390625,
      "epoch": 0.74,
      "learning_rate": 0.0001466768939393939,
      "loss": 3.1721,
      "step": 12800
    },
    {
      "activations/layer0_attention_weight_max": 15.122084617614746,
      "activations/layer0_attention_weight_min": -11.740078926086426,
      "activations/layer10_attention_weight_max": 51.786163330078125,
      "activations/layer10_attention_weight_min": -47.67144775390625,
      "activations/layer11_attention_weight_max": 43.206764221191406,
      "activations/layer11_attention_weight_min": -37.21377182006836,
      "activations/layer12_attention_weight_max": 32.97608947753906,
      "activations/layer12_attention_weight_min": -29.165016174316406,
      "activations/layer13_attention_weight_max": 40.76298904418945,
      "activations/layer13_attention_weight_min": -34.91901397705078,
      "activations/layer14_attention_weight_max": 62.10892105102539,
      "activations/layer14_attention_weight_min": -56.61106491088867,
      "activations/layer15_attention_weight_max": 40.42180252075195,
      "activations/layer15_attention_weight_min": -38.77156448364258,
      "activations/layer16_attention_weight_max": 47.29710388183594,
      "activations/layer16_attention_weight_min": -36.68632125854492,
      "activations/layer17_attention_weight_max": 28.951704025268555,
      "activations/layer17_attention_weight_min": -21.170385360717773,
      "activations/layer18_attention_weight_max": 21.33890151977539,
      "activations/layer18_attention_weight_min": -14.743868827819824,
      "activations/layer19_attention_weight_max": 41.266685485839844,
      "activations/layer19_attention_weight_min": -26.677616119384766,
      "activations/layer1_attention_weight_max": 15.888668060302734,
      "activations/layer1_attention_weight_min": -11.43416976928711,
      "activations/layer20_attention_weight_max": 24.783781051635742,
      "activations/layer20_attention_weight_min": -16.27275848388672,
      "activations/layer21_attention_weight_max": 19.227928161621094,
      "activations/layer21_attention_weight_min": -13.336156845092773,
      "activations/layer22_attention_weight_max": 33.698970794677734,
      "activations/layer22_attention_weight_min": -17.867204666137695,
      "activations/layer23_attention_weight_max": 24.260631561279297,
      "activations/layer23_attention_weight_min": -14.775991439819336,
      "activations/layer2_attention_weight_max": 27.660484313964844,
      "activations/layer2_attention_weight_min": -24.059528350830078,
      "activations/layer3_attention_weight_max": 58.31935501098633,
      "activations/layer3_attention_weight_min": -56.228050231933594,
      "activations/layer4_attention_weight_max": 110.93316650390625,
      "activations/layer4_attention_weight_min": -102.18258666992188,
      "activations/layer5_attention_weight_max": 123.27361297607422,
      "activations/layer5_attention_weight_min": -109.84954833984375,
      "activations/layer6_attention_weight_max": 71.11297607421875,
      "activations/layer6_attention_weight_min": -61.14129638671875,
      "activations/layer7_attention_weight_max": 65.55464935302734,
      "activations/layer7_attention_weight_min": -61.098548889160156,
      "activations/layer8_attention_weight_max": 62.05624008178711,
      "activations/layer8_attention_weight_min": -53.03624725341797,
      "activations/layer9_attention_weight_max": 64.8681869506836,
      "activations/layer9_attention_weight_min": -60.32849884033203,
      "epoch": 0.75,
      "learning_rate": 0.00014665795454545452,
      "loss": 3.168,
      "step": 12850
    },
    {
      "activations/layer0_attention_weight_max": 15.426179885864258,
      "activations/layer0_attention_weight_min": -10.70346450805664,
      "activations/layer10_attention_weight_max": 50.3702507019043,
      "activations/layer10_attention_weight_min": -43.686248779296875,
      "activations/layer11_attention_weight_max": 49.732994079589844,
      "activations/layer11_attention_weight_min": -38.371036529541016,
      "activations/layer12_attention_weight_max": 39.76906967163086,
      "activations/layer12_attention_weight_min": -33.98399353027344,
      "activations/layer13_attention_weight_max": 39.25110626220703,
      "activations/layer13_attention_weight_min": -37.72811508178711,
      "activations/layer14_attention_weight_max": 66.5024642944336,
      "activations/layer14_attention_weight_min": -50.464576721191406,
      "activations/layer15_attention_weight_max": 47.37120056152344,
      "activations/layer15_attention_weight_min": -35.327171325683594,
      "activations/layer16_attention_weight_max": 43.633243560791016,
      "activations/layer16_attention_weight_min": -37.36347198486328,
      "activations/layer17_attention_weight_max": 32.15566635131836,
      "activations/layer17_attention_weight_min": -20.96786880493164,
      "activations/layer18_attention_weight_max": 19.39923667907715,
      "activations/layer18_attention_weight_min": -13.533074378967285,
      "activations/layer19_attention_weight_max": 40.23945236206055,
      "activations/layer19_attention_weight_min": -27.29302978515625,
      "activations/layer1_attention_weight_max": 15.253059387207031,
      "activations/layer1_attention_weight_min": -11.564602851867676,
      "activations/layer20_attention_weight_max": 23.66619873046875,
      "activations/layer20_attention_weight_min": -14.531638145446777,
      "activations/layer21_attention_weight_max": 17.499786376953125,
      "activations/layer21_attention_weight_min": -12.636042594909668,
      "activations/layer22_attention_weight_max": 37.63874053955078,
      "activations/layer22_attention_weight_min": -22.005399703979492,
      "activations/layer23_attention_weight_max": 22.55231285095215,
      "activations/layer23_attention_weight_min": -14.176332473754883,
      "activations/layer2_attention_weight_max": 28.75762939453125,
      "activations/layer2_attention_weight_min": -24.613014221191406,
      "activations/layer3_attention_weight_max": 55.61717224121094,
      "activations/layer3_attention_weight_min": -54.45286560058594,
      "activations/layer4_attention_weight_max": 105.8746109008789,
      "activations/layer4_attention_weight_min": -101.30419921875,
      "activations/layer5_attention_weight_max": 123.29186248779297,
      "activations/layer5_attention_weight_min": -111.96884155273438,
      "activations/layer6_attention_weight_max": 68.4571762084961,
      "activations/layer6_attention_weight_min": -64.29762268066406,
      "activations/layer7_attention_weight_max": 65.4500961303711,
      "activations/layer7_attention_weight_min": -70.43690490722656,
      "activations/layer8_attention_weight_max": 65.19576263427734,
      "activations/layer8_attention_weight_min": -56.681156158447266,
      "activations/layer9_attention_weight_max": 70.64458465576172,
      "activations/layer9_attention_weight_min": -64.86309814453125,
      "epoch": 0.75,
      "learning_rate": 0.00014663901515151514,
      "loss": 3.1546,
      "step": 12900
    },
    {
      "activations/layer0_attention_weight_max": 15.248085021972656,
      "activations/layer0_attention_weight_min": -11.572760581970215,
      "activations/layer10_attention_weight_max": 52.208702087402344,
      "activations/layer10_attention_weight_min": -45.246864318847656,
      "activations/layer11_attention_weight_max": 44.62493133544922,
      "activations/layer11_attention_weight_min": -37.25926971435547,
      "activations/layer12_attention_weight_max": 28.374780654907227,
      "activations/layer12_attention_weight_min": -26.083274841308594,
      "activations/layer13_attention_weight_max": 39.92937469482422,
      "activations/layer13_attention_weight_min": -33.764156341552734,
      "activations/layer14_attention_weight_max": 62.96863555908203,
      "activations/layer14_attention_weight_min": -48.58577346801758,
      "activations/layer15_attention_weight_max": 58.13414764404297,
      "activations/layer15_attention_weight_min": -38.4141731262207,
      "activations/layer16_attention_weight_max": 45.3182373046875,
      "activations/layer16_attention_weight_min": -35.28632354736328,
      "activations/layer17_attention_weight_max": 44.1202507019043,
      "activations/layer17_attention_weight_min": -19.12135124206543,
      "activations/layer18_attention_weight_max": 20.268917083740234,
      "activations/layer18_attention_weight_min": -12.48083782196045,
      "activations/layer19_attention_weight_max": 39.09804916381836,
      "activations/layer19_attention_weight_min": -28.38516616821289,
      "activations/layer1_attention_weight_max": 14.601317405700684,
      "activations/layer1_attention_weight_min": -11.140117645263672,
      "activations/layer20_attention_weight_max": 27.51862144470215,
      "activations/layer20_attention_weight_min": -14.784075736999512,
      "activations/layer21_attention_weight_max": 19.776535034179688,
      "activations/layer21_attention_weight_min": -14.049139022827148,
      "activations/layer22_attention_weight_max": 36.701866149902344,
      "activations/layer22_attention_weight_min": -23.313793182373047,
      "activations/layer23_attention_weight_max": 23.056692123413086,
      "activations/layer23_attention_weight_min": -17.036724090576172,
      "activations/layer2_attention_weight_max": 28.119503021240234,
      "activations/layer2_attention_weight_min": -24.616313934326172,
      "activations/layer3_attention_weight_max": 56.583621978759766,
      "activations/layer3_attention_weight_min": -58.381282806396484,
      "activations/layer4_attention_weight_max": 110.00416564941406,
      "activations/layer4_attention_weight_min": -106.84209442138672,
      "activations/layer5_attention_weight_max": 126.25191497802734,
      "activations/layer5_attention_weight_min": -115.99403381347656,
      "activations/layer6_attention_weight_max": 71.7595443725586,
      "activations/layer6_attention_weight_min": -62.658409118652344,
      "activations/layer7_attention_weight_max": 65.25373077392578,
      "activations/layer7_attention_weight_min": -64.00162506103516,
      "activations/layer8_attention_weight_max": 65.43091583251953,
      "activations/layer8_attention_weight_min": -54.24314498901367,
      "activations/layer9_attention_weight_max": 66.44222259521484,
      "activations/layer9_attention_weight_min": -60.276092529296875,
      "epoch": 0.75,
      "learning_rate": 0.00014662007575757575,
      "loss": 3.1813,
      "step": 12950
    },
    {
      "activations/layer0_attention_weight_max": 14.766436576843262,
      "activations/layer0_attention_weight_min": -11.835564613342285,
      "activations/layer10_attention_weight_max": 54.466670989990234,
      "activations/layer10_attention_weight_min": -43.700408935546875,
      "activations/layer11_attention_weight_max": 52.77171325683594,
      "activations/layer11_attention_weight_min": -39.748046875,
      "activations/layer12_attention_weight_max": 58.958038330078125,
      "activations/layer12_attention_weight_min": -30.646692276000977,
      "activations/layer13_attention_weight_max": 47.73133850097656,
      "activations/layer13_attention_weight_min": -34.544559478759766,
      "activations/layer14_attention_weight_max": 68.56082153320312,
      "activations/layer14_attention_weight_min": -51.03422164916992,
      "activations/layer15_attention_weight_max": 48.899314880371094,
      "activations/layer15_attention_weight_min": -34.44529724121094,
      "activations/layer16_attention_weight_max": 52.59762954711914,
      "activations/layer16_attention_weight_min": -32.20002746582031,
      "activations/layer17_attention_weight_max": 28.31759262084961,
      "activations/layer17_attention_weight_min": -17.320528030395508,
      "activations/layer18_attention_weight_max": 21.599517822265625,
      "activations/layer18_attention_weight_min": -12.810125350952148,
      "activations/layer19_attention_weight_max": 38.699310302734375,
      "activations/layer19_attention_weight_min": -27.32325553894043,
      "activations/layer1_attention_weight_max": 15.441983222961426,
      "activations/layer1_attention_weight_min": -11.497084617614746,
      "activations/layer20_attention_weight_max": 27.220584869384766,
      "activations/layer20_attention_weight_min": -15.896878242492676,
      "activations/layer21_attention_weight_max": 21.0755558013916,
      "activations/layer21_attention_weight_min": -16.0258731842041,
      "activations/layer22_attention_weight_max": 33.4672737121582,
      "activations/layer22_attention_weight_min": -16.39120864868164,
      "activations/layer23_attention_weight_max": 23.72430992126465,
      "activations/layer23_attention_weight_min": -13.859123229980469,
      "activations/layer2_attention_weight_max": 27.77417755126953,
      "activations/layer2_attention_weight_min": -24.127790451049805,
      "activations/layer3_attention_weight_max": 54.736392974853516,
      "activations/layer3_attention_weight_min": -53.52733612060547,
      "activations/layer4_attention_weight_max": 106.08131408691406,
      "activations/layer4_attention_weight_min": -101.38346099853516,
      "activations/layer5_attention_weight_max": 125.20320892333984,
      "activations/layer5_attention_weight_min": -113.11311340332031,
      "activations/layer6_attention_weight_max": 71.15911865234375,
      "activations/layer6_attention_weight_min": -60.15620422363281,
      "activations/layer7_attention_weight_max": 66.90840911865234,
      "activations/layer7_attention_weight_min": -62.88893508911133,
      "activations/layer8_attention_weight_max": 67.21355438232422,
      "activations/layer8_attention_weight_min": -58.5176887512207,
      "activations/layer9_attention_weight_max": 73.50896453857422,
      "activations/layer9_attention_weight_min": -61.500152587890625,
      "epoch": 0.76,
      "learning_rate": 0.00014660113636363634,
      "loss": 3.1903,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_loss": 3.1171875,
      "eval_runtime": 8.4888,
      "eval_samples_per_second": 505.844,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_openwebtext_loss": 3.1171875,
      "eval_openwebtext_ppl": 22.582776184718522,
      "eval_openwebtext_runtime": 8.4888,
      "eval_openwebtext_samples_per_second": 505.844,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_wikitext_loss": 3.40234375,
      "eval_wikitext_ppl": 30.034410770075546,
      "eval_wikitext_runtime": 1.9582,
      "eval_wikitext_samples_per_second": 232.864,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_lambada_loss": 3.677734375,
      "eval_lambada_ppl": 39.556671887793456,
      "eval_lambada_runtime": 9.577,
      "eval_lambada_samples_per_second": 508.406,
      "step": 13000
    },
    {
      "activations/layer0_attention_weight_max": 15.57494831085205,
      "activations/layer0_attention_weight_min": -11.041525840759277,
      "activations/layer10_attention_weight_max": 50.21454620361328,
      "activations/layer10_attention_weight_min": -46.92526626586914,
      "activations/layer11_attention_weight_max": 45.17107391357422,
      "activations/layer11_attention_weight_min": -40.34950637817383,
      "activations/layer12_attention_weight_max": 32.028175354003906,
      "activations/layer12_attention_weight_min": -29.309831619262695,
      "activations/layer13_attention_weight_max": 37.31104278564453,
      "activations/layer13_attention_weight_min": -36.6234245300293,
      "activations/layer14_attention_weight_max": 63.58932876586914,
      "activations/layer14_attention_weight_min": -52.735198974609375,
      "activations/layer15_attention_weight_max": 52.558021545410156,
      "activations/layer15_attention_weight_min": -37.73514175415039,
      "activations/layer16_attention_weight_max": 45.43423080444336,
      "activations/layer16_attention_weight_min": -31.710115432739258,
      "activations/layer17_attention_weight_max": 35.995113372802734,
      "activations/layer17_attention_weight_min": -19.037580490112305,
      "activations/layer18_attention_weight_max": 26.391128540039062,
      "activations/layer18_attention_weight_min": -11.851982116699219,
      "activations/layer19_attention_weight_max": 39.9422721862793,
      "activations/layer19_attention_weight_min": -25.465896606445312,
      "activations/layer1_attention_weight_max": 15.34239673614502,
      "activations/layer1_attention_weight_min": -11.348135948181152,
      "activations/layer20_attention_weight_max": 25.507572174072266,
      "activations/layer20_attention_weight_min": -15.975122451782227,
      "activations/layer21_attention_weight_max": 21.586097717285156,
      "activations/layer21_attention_weight_min": -12.561394691467285,
      "activations/layer22_attention_weight_max": 40.61000442504883,
      "activations/layer22_attention_weight_min": -15.915478706359863,
      "activations/layer23_attention_weight_max": 29.171829223632812,
      "activations/layer23_attention_weight_min": -13.263954162597656,
      "activations/layer2_attention_weight_max": 28.126455307006836,
      "activations/layer2_attention_weight_min": -24.968963623046875,
      "activations/layer3_attention_weight_max": 58.0609016418457,
      "activations/layer3_attention_weight_min": -57.64439392089844,
      "activations/layer4_attention_weight_max": 109.42640686035156,
      "activations/layer4_attention_weight_min": -106.57728576660156,
      "activations/layer5_attention_weight_max": 127.8636703491211,
      "activations/layer5_attention_weight_min": -116.3337631225586,
      "activations/layer6_attention_weight_max": 72.09162902832031,
      "activations/layer6_attention_weight_min": -61.3231086730957,
      "activations/layer7_attention_weight_max": 68.39881896972656,
      "activations/layer7_attention_weight_min": -66.5058364868164,
      "activations/layer8_attention_weight_max": 63.84579086303711,
      "activations/layer8_attention_weight_min": -55.9757194519043,
      "activations/layer9_attention_weight_max": 66.56392669677734,
      "activations/layer9_attention_weight_min": -67.98298645019531,
      "epoch": 0.76,
      "learning_rate": 0.00014658257575757573,
      "loss": 3.1741,
      "step": 13050
    },
    {
      "activations/layer0_attention_weight_max": 15.91378402709961,
      "activations/layer0_attention_weight_min": -11.162917137145996,
      "activations/layer10_attention_weight_max": 54.88658142089844,
      "activations/layer10_attention_weight_min": -46.238677978515625,
      "activations/layer11_attention_weight_max": 49.881378173828125,
      "activations/layer11_attention_weight_min": -38.17985916137695,
      "activations/layer12_attention_weight_max": 34.2217903137207,
      "activations/layer12_attention_weight_min": -27.45090675354004,
      "activations/layer13_attention_weight_max": 54.13373565673828,
      "activations/layer13_attention_weight_min": -35.8289909362793,
      "activations/layer14_attention_weight_max": 75.0201644897461,
      "activations/layer14_attention_weight_min": -49.82703399658203,
      "activations/layer15_attention_weight_max": 52.73164749145508,
      "activations/layer15_attention_weight_min": -34.41261672973633,
      "activations/layer16_attention_weight_max": 47.99238204956055,
      "activations/layer16_attention_weight_min": -37.72374725341797,
      "activations/layer17_attention_weight_max": 41.5011100769043,
      "activations/layer17_attention_weight_min": -21.28853416442871,
      "activations/layer18_attention_weight_max": 25.227373123168945,
      "activations/layer18_attention_weight_min": -12.339431762695312,
      "activations/layer19_attention_weight_max": 48.2297248840332,
      "activations/layer19_attention_weight_min": -28.821613311767578,
      "activations/layer1_attention_weight_max": 14.91834545135498,
      "activations/layer1_attention_weight_min": -11.647759437561035,
      "activations/layer20_attention_weight_max": 30.993574142456055,
      "activations/layer20_attention_weight_min": -16.260204315185547,
      "activations/layer21_attention_weight_max": 22.230649948120117,
      "activations/layer21_attention_weight_min": -16.03851318359375,
      "activations/layer22_attention_weight_max": 38.25136947631836,
      "activations/layer22_attention_weight_min": -24.04094696044922,
      "activations/layer23_attention_weight_max": 24.17024803161621,
      "activations/layer23_attention_weight_min": -15.471728324890137,
      "activations/layer2_attention_weight_max": 28.98592185974121,
      "activations/layer2_attention_weight_min": -25.7421875,
      "activations/layer3_attention_weight_max": 57.69720458984375,
      "activations/layer3_attention_weight_min": -56.49441146850586,
      "activations/layer4_attention_weight_max": 109.41117095947266,
      "activations/layer4_attention_weight_min": -102.73311614990234,
      "activations/layer5_attention_weight_max": 127.9426498413086,
      "activations/layer5_attention_weight_min": -111.70443725585938,
      "activations/layer6_attention_weight_max": 67.36942291259766,
      "activations/layer6_attention_weight_min": -65.36933135986328,
      "activations/layer7_attention_weight_max": 63.68265151977539,
      "activations/layer7_attention_weight_min": -61.66347885131836,
      "activations/layer8_attention_weight_max": 65.58485412597656,
      "activations/layer8_attention_weight_min": -55.00161361694336,
      "activations/layer9_attention_weight_max": 71.33931732177734,
      "activations/layer9_attention_weight_min": -63.14413070678711,
      "epoch": 0.76,
      "learning_rate": 0.00014656363636363635,
      "loss": 3.186,
      "step": 13100
    },
    {
      "activations/layer0_attention_weight_max": 13.989656448364258,
      "activations/layer0_attention_weight_min": -11.119426727294922,
      "activations/layer10_attention_weight_max": 49.66547393798828,
      "activations/layer10_attention_weight_min": -47.45139694213867,
      "activations/layer11_attention_weight_max": 46.279808044433594,
      "activations/layer11_attention_weight_min": -39.58982849121094,
      "activations/layer12_attention_weight_max": 39.24481201171875,
      "activations/layer12_attention_weight_min": -29.836322784423828,
      "activations/layer13_attention_weight_max": 40.38532257080078,
      "activations/layer13_attention_weight_min": -36.97200393676758,
      "activations/layer14_attention_weight_max": 62.64452362060547,
      "activations/layer14_attention_weight_min": -48.84852600097656,
      "activations/layer15_attention_weight_max": 47.399112701416016,
      "activations/layer15_attention_weight_min": -34.120140075683594,
      "activations/layer16_attention_weight_max": 44.24497985839844,
      "activations/layer16_attention_weight_min": -34.76737594604492,
      "activations/layer17_attention_weight_max": 29.660600662231445,
      "activations/layer17_attention_weight_min": -17.50663948059082,
      "activations/layer18_attention_weight_max": 25.13851547241211,
      "activations/layer18_attention_weight_min": -12.166817665100098,
      "activations/layer19_attention_weight_max": 43.34315872192383,
      "activations/layer19_attention_weight_min": -26.776166915893555,
      "activations/layer1_attention_weight_max": 14.600281715393066,
      "activations/layer1_attention_weight_min": -11.184001922607422,
      "activations/layer20_attention_weight_max": 26.419248580932617,
      "activations/layer20_attention_weight_min": -18.70677947998047,
      "activations/layer21_attention_weight_max": 18.530981063842773,
      "activations/layer21_attention_weight_min": -14.547564506530762,
      "activations/layer22_attention_weight_max": 31.363142013549805,
      "activations/layer22_attention_weight_min": -22.10376739501953,
      "activations/layer23_attention_weight_max": 21.834491729736328,
      "activations/layer23_attention_weight_min": -14.10890007019043,
      "activations/layer2_attention_weight_max": 27.657756805419922,
      "activations/layer2_attention_weight_min": -25.23334503173828,
      "activations/layer3_attention_weight_max": 58.325252532958984,
      "activations/layer3_attention_weight_min": -57.62208557128906,
      "activations/layer4_attention_weight_max": 112.10494232177734,
      "activations/layer4_attention_weight_min": -107.31005096435547,
      "activations/layer5_attention_weight_max": 125.37078857421875,
      "activations/layer5_attention_weight_min": -114.30917358398438,
      "activations/layer6_attention_weight_max": 68.09243774414062,
      "activations/layer6_attention_weight_min": -63.030303955078125,
      "activations/layer7_attention_weight_max": 65.9681167602539,
      "activations/layer7_attention_weight_min": -67.16943359375,
      "activations/layer8_attention_weight_max": 63.75066375732422,
      "activations/layer8_attention_weight_min": -54.39865493774414,
      "activations/layer9_attention_weight_max": 77.99029541015625,
      "activations/layer9_attention_weight_min": -65.4648666381836,
      "epoch": 0.76,
      "learning_rate": 0.00014654469696969696,
      "loss": 3.18,
      "step": 13150
    },
    {
      "activations/layer0_attention_weight_max": 14.04937744140625,
      "activations/layer0_attention_weight_min": -10.957547187805176,
      "activations/layer10_attention_weight_max": 53.24135208129883,
      "activations/layer10_attention_weight_min": -47.96957778930664,
      "activations/layer11_attention_weight_max": 45.031036376953125,
      "activations/layer11_attention_weight_min": -40.21868896484375,
      "activations/layer12_attention_weight_max": 30.941696166992188,
      "activations/layer12_attention_weight_min": -26.430458068847656,
      "activations/layer13_attention_weight_max": 45.342227935791016,
      "activations/layer13_attention_weight_min": -35.245941162109375,
      "activations/layer14_attention_weight_max": 59.955875396728516,
      "activations/layer14_attention_weight_min": -52.42814636230469,
      "activations/layer15_attention_weight_max": 47.21579360961914,
      "activations/layer15_attention_weight_min": -40.249549865722656,
      "activations/layer16_attention_weight_max": 40.654972076416016,
      "activations/layer16_attention_weight_min": -31.81290626525879,
      "activations/layer17_attention_weight_max": 26.85831642150879,
      "activations/layer17_attention_weight_min": -19.34947395324707,
      "activations/layer18_attention_weight_max": 19.653947830200195,
      "activations/layer18_attention_weight_min": -14.143697738647461,
      "activations/layer19_attention_weight_max": 38.98844528198242,
      "activations/layer19_attention_weight_min": -31.38747215270996,
      "activations/layer1_attention_weight_max": 15.883058547973633,
      "activations/layer1_attention_weight_min": -12.389484405517578,
      "activations/layer20_attention_weight_max": 25.72371482849121,
      "activations/layer20_attention_weight_min": -17.435606002807617,
      "activations/layer21_attention_weight_max": 20.838163375854492,
      "activations/layer21_attention_weight_min": -14.432735443115234,
      "activations/layer22_attention_weight_max": 32.87032699584961,
      "activations/layer22_attention_weight_min": -16.522396087646484,
      "activations/layer23_attention_weight_max": 22.498157501220703,
      "activations/layer23_attention_weight_min": -12.510244369506836,
      "activations/layer2_attention_weight_max": 27.369009017944336,
      "activations/layer2_attention_weight_min": -25.207599639892578,
      "activations/layer3_attention_weight_max": 55.45978927612305,
      "activations/layer3_attention_weight_min": -54.35719680786133,
      "activations/layer4_attention_weight_max": 106.07063293457031,
      "activations/layer4_attention_weight_min": -103.5171127319336,
      "activations/layer5_attention_weight_max": 121.26903533935547,
      "activations/layer5_attention_weight_min": -112.76775360107422,
      "activations/layer6_attention_weight_max": 69.67838287353516,
      "activations/layer6_attention_weight_min": -61.04328155517578,
      "activations/layer7_attention_weight_max": 65.01104736328125,
      "activations/layer7_attention_weight_min": -63.470577239990234,
      "activations/layer8_attention_weight_max": 58.96443176269531,
      "activations/layer8_attention_weight_min": -56.30842971801758,
      "activations/layer9_attention_weight_max": 66.16048431396484,
      "activations/layer9_attention_weight_min": -69.31612396240234,
      "epoch": 0.77,
      "learning_rate": 0.00014652575757575758,
      "loss": 3.1647,
      "step": 13200
    },
    {
      "activations/layer0_attention_weight_max": 14.125494003295898,
      "activations/layer0_attention_weight_min": -11.560543060302734,
      "activations/layer10_attention_weight_max": 51.36051559448242,
      "activations/layer10_attention_weight_min": -48.454010009765625,
      "activations/layer11_attention_weight_max": 40.81846618652344,
      "activations/layer11_attention_weight_min": -39.405704498291016,
      "activations/layer12_attention_weight_max": 34.14521789550781,
      "activations/layer12_attention_weight_min": -31.332063674926758,
      "activations/layer13_attention_weight_max": 38.375205993652344,
      "activations/layer13_attention_weight_min": -34.924354553222656,
      "activations/layer14_attention_weight_max": 62.799468994140625,
      "activations/layer14_attention_weight_min": -48.475799560546875,
      "activations/layer15_attention_weight_max": 43.55928039550781,
      "activations/layer15_attention_weight_min": -32.22998809814453,
      "activations/layer16_attention_weight_max": 42.613258361816406,
      "activations/layer16_attention_weight_min": -34.124786376953125,
      "activations/layer17_attention_weight_max": 26.856225967407227,
      "activations/layer17_attention_weight_min": -19.383886337280273,
      "activations/layer18_attention_weight_max": 22.813621520996094,
      "activations/layer18_attention_weight_min": -12.852404594421387,
      "activations/layer19_attention_weight_max": 42.22843933105469,
      "activations/layer19_attention_weight_min": -28.106983184814453,
      "activations/layer1_attention_weight_max": 15.424013137817383,
      "activations/layer1_attention_weight_min": -13.234892845153809,
      "activations/layer20_attention_weight_max": 27.238067626953125,
      "activations/layer20_attention_weight_min": -17.55496597290039,
      "activations/layer21_attention_weight_max": 20.16004180908203,
      "activations/layer21_attention_weight_min": -14.24951171875,
      "activations/layer22_attention_weight_max": 30.94219970703125,
      "activations/layer22_attention_weight_min": -18.900096893310547,
      "activations/layer23_attention_weight_max": 20.61560821533203,
      "activations/layer23_attention_weight_min": -15.062124252319336,
      "activations/layer2_attention_weight_max": 27.252256393432617,
      "activations/layer2_attention_weight_min": -25.113706588745117,
      "activations/layer3_attention_weight_max": 56.30155563354492,
      "activations/layer3_attention_weight_min": -55.073509216308594,
      "activations/layer4_attention_weight_max": 101.69409942626953,
      "activations/layer4_attention_weight_min": -100.6082534790039,
      "activations/layer5_attention_weight_max": 116.02763366699219,
      "activations/layer5_attention_weight_min": -107.0050048828125,
      "activations/layer6_attention_weight_max": 65.30241394042969,
      "activations/layer6_attention_weight_min": -61.37042236328125,
      "activations/layer7_attention_weight_max": 61.675636291503906,
      "activations/layer7_attention_weight_min": -60.94429016113281,
      "activations/layer8_attention_weight_max": 59.47544479370117,
      "activations/layer8_attention_weight_min": -54.2468376159668,
      "activations/layer9_attention_weight_max": 65.89291381835938,
      "activations/layer9_attention_weight_min": -66.27520751953125,
      "epoch": 0.77,
      "learning_rate": 0.00014650681818181816,
      "loss": 3.1782,
      "step": 13250
    },
    {
      "activations/layer0_attention_weight_max": 16.252254486083984,
      "activations/layer0_attention_weight_min": -11.539472579956055,
      "activations/layer10_attention_weight_max": 48.36096954345703,
      "activations/layer10_attention_weight_min": -44.73154830932617,
      "activations/layer11_attention_weight_max": 49.38051986694336,
      "activations/layer11_attention_weight_min": -40.054832458496094,
      "activations/layer12_attention_weight_max": 41.82487106323242,
      "activations/layer12_attention_weight_min": -34.06550979614258,
      "activations/layer13_attention_weight_max": 45.42441940307617,
      "activations/layer13_attention_weight_min": -43.98827362060547,
      "activations/layer14_attention_weight_max": 61.52260208129883,
      "activations/layer14_attention_weight_min": -49.525428771972656,
      "activations/layer15_attention_weight_max": 50.232852935791016,
      "activations/layer15_attention_weight_min": -37.04643249511719,
      "activations/layer16_attention_weight_max": 46.37913513183594,
      "activations/layer16_attention_weight_min": -36.171077728271484,
      "activations/layer17_attention_weight_max": 30.786102294921875,
      "activations/layer17_attention_weight_min": -18.634546279907227,
      "activations/layer18_attention_weight_max": 22.63043975830078,
      "activations/layer18_attention_weight_min": -13.10734748840332,
      "activations/layer19_attention_weight_max": 41.185604095458984,
      "activations/layer19_attention_weight_min": -30.833911895751953,
      "activations/layer1_attention_weight_max": 16.052953720092773,
      "activations/layer1_attention_weight_min": -11.64273452758789,
      "activations/layer20_attention_weight_max": 25.750486373901367,
      "activations/layer20_attention_weight_min": -15.394954681396484,
      "activations/layer21_attention_weight_max": 20.62904930114746,
      "activations/layer21_attention_weight_min": -14.750798225402832,
      "activations/layer22_attention_weight_max": 32.01865005493164,
      "activations/layer22_attention_weight_min": -18.861648559570312,
      "activations/layer23_attention_weight_max": 21.908140182495117,
      "activations/layer23_attention_weight_min": -15.171024322509766,
      "activations/layer2_attention_weight_max": 29.65091323852539,
      "activations/layer2_attention_weight_min": -25.011470794677734,
      "activations/layer3_attention_weight_max": 56.578372955322266,
      "activations/layer3_attention_weight_min": -58.67481231689453,
      "activations/layer4_attention_weight_max": 110.16143798828125,
      "activations/layer4_attention_weight_min": -103.7638931274414,
      "activations/layer5_attention_weight_max": 125.2772216796875,
      "activations/layer5_attention_weight_min": -109.90019989013672,
      "activations/layer6_attention_weight_max": 67.67996978759766,
      "activations/layer6_attention_weight_min": -61.08973693847656,
      "activations/layer7_attention_weight_max": 63.17675018310547,
      "activations/layer7_attention_weight_min": -63.3092155456543,
      "activations/layer8_attention_weight_max": 64.10462951660156,
      "activations/layer8_attention_weight_min": -59.12216567993164,
      "activations/layer9_attention_weight_max": 74.96637725830078,
      "activations/layer9_attention_weight_min": -73.47894287109375,
      "epoch": 0.77,
      "learning_rate": 0.00014648787878787878,
      "loss": 3.1649,
      "step": 13300
    },
    {
      "activations/layer0_attention_weight_max": 13.038305282592773,
      "activations/layer0_attention_weight_min": -11.137943267822266,
      "activations/layer10_attention_weight_max": 56.85255813598633,
      "activations/layer10_attention_weight_min": -47.77335739135742,
      "activations/layer11_attention_weight_max": 48.48026657104492,
      "activations/layer11_attention_weight_min": -39.1607666015625,
      "activations/layer12_attention_weight_max": 33.83481979370117,
      "activations/layer12_attention_weight_min": -28.026330947875977,
      "activations/layer13_attention_weight_max": 43.224002838134766,
      "activations/layer13_attention_weight_min": -33.63495635986328,
      "activations/layer14_attention_weight_max": 68.63056945800781,
      "activations/layer14_attention_weight_min": -49.85233688354492,
      "activations/layer15_attention_weight_max": 49.70286178588867,
      "activations/layer15_attention_weight_min": -36.395137786865234,
      "activations/layer16_attention_weight_max": 47.32681655883789,
      "activations/layer16_attention_weight_min": -39.817352294921875,
      "activations/layer17_attention_weight_max": 32.46662139892578,
      "activations/layer17_attention_weight_min": -20.95451545715332,
      "activations/layer18_attention_weight_max": 20.94791603088379,
      "activations/layer18_attention_weight_min": -13.468544006347656,
      "activations/layer19_attention_weight_max": 43.53076934814453,
      "activations/layer19_attention_weight_min": -29.87188148498535,
      "activations/layer1_attention_weight_max": 15.269271850585938,
      "activations/layer1_attention_weight_min": -12.062398910522461,
      "activations/layer20_attention_weight_max": 27.739200592041016,
      "activations/layer20_attention_weight_min": -17.53782844543457,
      "activations/layer21_attention_weight_max": 19.46183204650879,
      "activations/layer21_attention_weight_min": -14.342550277709961,
      "activations/layer22_attention_weight_max": 36.28598403930664,
      "activations/layer22_attention_weight_min": -21.49605369567871,
      "activations/layer23_attention_weight_max": 24.904945373535156,
      "activations/layer23_attention_weight_min": -13.193556785583496,
      "activations/layer2_attention_weight_max": 28.3236083984375,
      "activations/layer2_attention_weight_min": -25.309309005737305,
      "activations/layer3_attention_weight_max": 56.641929626464844,
      "activations/layer3_attention_weight_min": -57.447147369384766,
      "activations/layer4_attention_weight_max": 108.07820129394531,
      "activations/layer4_attention_weight_min": -108.61396789550781,
      "activations/layer5_attention_weight_max": 128.23089599609375,
      "activations/layer5_attention_weight_min": -115.8155517578125,
      "activations/layer6_attention_weight_max": 67.90975952148438,
      "activations/layer6_attention_weight_min": -62.46329879760742,
      "activations/layer7_attention_weight_max": 67.24021911621094,
      "activations/layer7_attention_weight_min": -66.6331787109375,
      "activations/layer8_attention_weight_max": 64.21720886230469,
      "activations/layer8_attention_weight_min": -56.254539489746094,
      "activations/layer9_attention_weight_max": 68.21556854248047,
      "activations/layer9_attention_weight_min": -63.156368255615234,
      "epoch": 0.78,
      "learning_rate": 0.0001464689393939394,
      "loss": 3.1766,
      "step": 13350
    },
    {
      "activations/layer0_attention_weight_max": 14.551106452941895,
      "activations/layer0_attention_weight_min": -10.935883522033691,
      "activations/layer10_attention_weight_max": 49.358795166015625,
      "activations/layer10_attention_weight_min": -43.689579010009766,
      "activations/layer11_attention_weight_max": 44.849822998046875,
      "activations/layer11_attention_weight_min": -38.99372100830078,
      "activations/layer12_attention_weight_max": 32.899749755859375,
      "activations/layer12_attention_weight_min": -27.44535255432129,
      "activations/layer13_attention_weight_max": 37.17533493041992,
      "activations/layer13_attention_weight_min": -35.31945037841797,
      "activations/layer14_attention_weight_max": 63.07332229614258,
      "activations/layer14_attention_weight_min": -51.369197845458984,
      "activations/layer15_attention_weight_max": 46.29391860961914,
      "activations/layer15_attention_weight_min": -38.39763259887695,
      "activations/layer16_attention_weight_max": 45.831390380859375,
      "activations/layer16_attention_weight_min": -35.8975715637207,
      "activations/layer17_attention_weight_max": 33.8438606262207,
      "activations/layer17_attention_weight_min": -23.2631893157959,
      "activations/layer18_attention_weight_max": 19.721654891967773,
      "activations/layer18_attention_weight_min": -12.585180282592773,
      "activations/layer19_attention_weight_max": 40.59812545776367,
      "activations/layer19_attention_weight_min": -28.71895408630371,
      "activations/layer1_attention_weight_max": 15.508749961853027,
      "activations/layer1_attention_weight_min": -11.422748565673828,
      "activations/layer20_attention_weight_max": 24.491804122924805,
      "activations/layer20_attention_weight_min": -17.53649139404297,
      "activations/layer21_attention_weight_max": 21.369632720947266,
      "activations/layer21_attention_weight_min": -13.743719100952148,
      "activations/layer22_attention_weight_max": 34.590091705322266,
      "activations/layer22_attention_weight_min": -22.3010311126709,
      "activations/layer23_attention_weight_max": 23.441627502441406,
      "activations/layer23_attention_weight_min": -15.960531234741211,
      "activations/layer2_attention_weight_max": 26.97755241394043,
      "activations/layer2_attention_weight_min": -25.581100463867188,
      "activations/layer3_attention_weight_max": 52.38795471191406,
      "activations/layer3_attention_weight_min": -55.63365173339844,
      "activations/layer4_attention_weight_max": 99.33648681640625,
      "activations/layer4_attention_weight_min": -102.15181732177734,
      "activations/layer5_attention_weight_max": 124.11320495605469,
      "activations/layer5_attention_weight_min": -108.82805633544922,
      "activations/layer6_attention_weight_max": 72.25192260742188,
      "activations/layer6_attention_weight_min": -59.12566375732422,
      "activations/layer7_attention_weight_max": 67.74954223632812,
      "activations/layer7_attention_weight_min": -62.76497268676758,
      "activations/layer8_attention_weight_max": 63.63671112060547,
      "activations/layer8_attention_weight_min": -50.919219970703125,
      "activations/layer9_attention_weight_max": 67.63750457763672,
      "activations/layer9_attention_weight_min": -61.98688888549805,
      "epoch": 0.78,
      "learning_rate": 0.00014644999999999998,
      "loss": 3.1736,
      "step": 13400
    },
    {
      "activations/layer0_attention_weight_max": 16.293193817138672,
      "activations/layer0_attention_weight_min": -10.533700942993164,
      "activations/layer10_attention_weight_max": 51.03692626953125,
      "activations/layer10_attention_weight_min": -44.06505584716797,
      "activations/layer11_attention_weight_max": 55.457862854003906,
      "activations/layer11_attention_weight_min": -46.63182067871094,
      "activations/layer12_attention_weight_max": 37.50193405151367,
      "activations/layer12_attention_weight_min": -30.678770065307617,
      "activations/layer13_attention_weight_max": 48.161293029785156,
      "activations/layer13_attention_weight_min": -37.514095306396484,
      "activations/layer14_attention_weight_max": 81.23231506347656,
      "activations/layer14_attention_weight_min": -56.63777160644531,
      "activations/layer15_attention_weight_max": 49.10350036621094,
      "activations/layer15_attention_weight_min": -37.219120025634766,
      "activations/layer16_attention_weight_max": 45.97842788696289,
      "activations/layer16_attention_weight_min": -38.16950225830078,
      "activations/layer17_attention_weight_max": 28.902870178222656,
      "activations/layer17_attention_weight_min": -18.331491470336914,
      "activations/layer18_attention_weight_max": 20.174718856811523,
      "activations/layer18_attention_weight_min": -12.296792030334473,
      "activations/layer19_attention_weight_max": 46.86688232421875,
      "activations/layer19_attention_weight_min": -28.916730880737305,
      "activations/layer1_attention_weight_max": 16.06910514831543,
      "activations/layer1_attention_weight_min": -12.001776695251465,
      "activations/layer20_attention_weight_max": 26.647144317626953,
      "activations/layer20_attention_weight_min": -17.501781463623047,
      "activations/layer21_attention_weight_max": 20.95343589782715,
      "activations/layer21_attention_weight_min": -13.24953842163086,
      "activations/layer22_attention_weight_max": 33.455196380615234,
      "activations/layer22_attention_weight_min": -20.28339385986328,
      "activations/layer23_attention_weight_max": 24.999011993408203,
      "activations/layer23_attention_weight_min": -15.518440246582031,
      "activations/layer2_attention_weight_max": 28.047405242919922,
      "activations/layer2_attention_weight_min": -25.670703887939453,
      "activations/layer3_attention_weight_max": 57.62342834472656,
      "activations/layer3_attention_weight_min": -57.774742126464844,
      "activations/layer4_attention_weight_max": 111.62348937988281,
      "activations/layer4_attention_weight_min": -106.63426971435547,
      "activations/layer5_attention_weight_max": 125.27611541748047,
      "activations/layer5_attention_weight_min": -114.99340057373047,
      "activations/layer6_attention_weight_max": 72.85397338867188,
      "activations/layer6_attention_weight_min": -61.937652587890625,
      "activations/layer7_attention_weight_max": 70.89861297607422,
      "activations/layer7_attention_weight_min": -69.80326080322266,
      "activations/layer8_attention_weight_max": 64.19590759277344,
      "activations/layer8_attention_weight_min": -56.178958892822266,
      "activations/layer9_attention_weight_max": 77.76815795898438,
      "activations/layer9_attention_weight_min": -66.8094253540039,
      "epoch": 0.78,
      "learning_rate": 0.0001464310606060606,
      "loss": 3.1745,
      "step": 13450
    },
    {
      "activations/layer0_attention_weight_max": 15.970416069030762,
      "activations/layer0_attention_weight_min": -10.840433120727539,
      "activations/layer10_attention_weight_max": 49.247161865234375,
      "activations/layer10_attention_weight_min": -41.69427490234375,
      "activations/layer11_attention_weight_max": 52.92798614501953,
      "activations/layer11_attention_weight_min": -35.505706787109375,
      "activations/layer12_attention_weight_max": 43.11762237548828,
      "activations/layer12_attention_weight_min": -35.785804748535156,
      "activations/layer13_attention_weight_max": 45.65080261230469,
      "activations/layer13_attention_weight_min": -34.44169235229492,
      "activations/layer14_attention_weight_max": 66.6398696899414,
      "activations/layer14_attention_weight_min": -52.52348327636719,
      "activations/layer15_attention_weight_max": 47.80425262451172,
      "activations/layer15_attention_weight_min": -34.60512161254883,
      "activations/layer16_attention_weight_max": 43.42922592163086,
      "activations/layer16_attention_weight_min": -35.576019287109375,
      "activations/layer17_attention_weight_max": 29.75318717956543,
      "activations/layer17_attention_weight_min": -20.8530216217041,
      "activations/layer18_attention_weight_max": 22.1484317779541,
      "activations/layer18_attention_weight_min": -12.808666229248047,
      "activations/layer19_attention_weight_max": 36.03493118286133,
      "activations/layer19_attention_weight_min": -25.929763793945312,
      "activations/layer1_attention_weight_max": 14.906977653503418,
      "activations/layer1_attention_weight_min": -11.190226554870605,
      "activations/layer20_attention_weight_max": 25.989717483520508,
      "activations/layer20_attention_weight_min": -15.591236114501953,
      "activations/layer21_attention_weight_max": 17.905500411987305,
      "activations/layer21_attention_weight_min": -13.168129920959473,
      "activations/layer22_attention_weight_max": 32.876426696777344,
      "activations/layer22_attention_weight_min": -17.479530334472656,
      "activations/layer23_attention_weight_max": 21.75485610961914,
      "activations/layer23_attention_weight_min": -13.87674617767334,
      "activations/layer2_attention_weight_max": 26.18678855895996,
      "activations/layer2_attention_weight_min": -24.161121368408203,
      "activations/layer3_attention_weight_max": 56.20722961425781,
      "activations/layer3_attention_weight_min": -57.287906646728516,
      "activations/layer4_attention_weight_max": 110.5060806274414,
      "activations/layer4_attention_weight_min": -102.91024017333984,
      "activations/layer5_attention_weight_max": 130.874267578125,
      "activations/layer5_attention_weight_min": -113.7685775756836,
      "activations/layer6_attention_weight_max": 74.68013000488281,
      "activations/layer6_attention_weight_min": -63.0064582824707,
      "activations/layer7_attention_weight_max": 73.18402862548828,
      "activations/layer7_attention_weight_min": -68.06990051269531,
      "activations/layer8_attention_weight_max": 61.32427978515625,
      "activations/layer8_attention_weight_min": -53.68517303466797,
      "activations/layer9_attention_weight_max": 83.89373016357422,
      "activations/layer9_attention_weight_min": -70.76904296875,
      "epoch": 0.78,
      "learning_rate": 0.0001464121212121212,
      "loss": 3.1508,
      "step": 13500
    },
    {
      "activations/layer0_attention_weight_max": 15.266022682189941,
      "activations/layer0_attention_weight_min": -10.454964637756348,
      "activations/layer10_attention_weight_max": 47.78996658325195,
      "activations/layer10_attention_weight_min": -47.453426361083984,
      "activations/layer11_attention_weight_max": 45.479888916015625,
      "activations/layer11_attention_weight_min": -40.268131256103516,
      "activations/layer12_attention_weight_max": 74.37776947021484,
      "activations/layer12_attention_weight_min": -45.634033203125,
      "activations/layer13_attention_weight_max": 66.478515625,
      "activations/layer13_attention_weight_min": -46.57465744018555,
      "activations/layer14_attention_weight_max": 61.581485748291016,
      "activations/layer14_attention_weight_min": -50.017303466796875,
      "activations/layer15_attention_weight_max": 50.98454284667969,
      "activations/layer15_attention_weight_min": -38.60172653198242,
      "activations/layer16_attention_weight_max": 41.7973518371582,
      "activations/layer16_attention_weight_min": -34.68055725097656,
      "activations/layer17_attention_weight_max": 32.13849639892578,
      "activations/layer17_attention_weight_min": -20.99205207824707,
      "activations/layer18_attention_weight_max": 20.2102108001709,
      "activations/layer18_attention_weight_min": -12.920695304870605,
      "activations/layer19_attention_weight_max": 41.36543655395508,
      "activations/layer19_attention_weight_min": -29.527374267578125,
      "activations/layer1_attention_weight_max": 15.287216186523438,
      "activations/layer1_attention_weight_min": -10.987966537475586,
      "activations/layer20_attention_weight_max": 28.2158145904541,
      "activations/layer20_attention_weight_min": -18.75727081298828,
      "activations/layer21_attention_weight_max": 18.286413192749023,
      "activations/layer21_attention_weight_min": -14.299334526062012,
      "activations/layer22_attention_weight_max": 37.4095458984375,
      "activations/layer22_attention_weight_min": -17.1768798828125,
      "activations/layer23_attention_weight_max": 23.023696899414062,
      "activations/layer23_attention_weight_min": -13.373222351074219,
      "activations/layer2_attention_weight_max": 25.642234802246094,
      "activations/layer2_attention_weight_min": -23.92525291442871,
      "activations/layer3_attention_weight_max": 53.82490921020508,
      "activations/layer3_attention_weight_min": -54.81889724731445,
      "activations/layer4_attention_weight_max": 105.09703826904297,
      "activations/layer4_attention_weight_min": -101.77751922607422,
      "activations/layer5_attention_weight_max": 122.07289123535156,
      "activations/layer5_attention_weight_min": -111.22325897216797,
      "activations/layer6_attention_weight_max": 65.68057250976562,
      "activations/layer6_attention_weight_min": -59.930824279785156,
      "activations/layer7_attention_weight_max": 64.68669128417969,
      "activations/layer7_attention_weight_min": -67.04662322998047,
      "activations/layer8_attention_weight_max": 61.35415267944336,
      "activations/layer8_attention_weight_min": -53.71526336669922,
      "activations/layer9_attention_weight_max": 73.40486907958984,
      "activations/layer9_attention_weight_min": -68.5421371459961,
      "epoch": 0.79,
      "learning_rate": 0.0001463931818181818,
      "loss": 3.1547,
      "step": 13550
    },
    {
      "activations/layer0_attention_weight_max": 14.653380393981934,
      "activations/layer0_attention_weight_min": -11.939654350280762,
      "activations/layer10_attention_weight_max": 50.77940368652344,
      "activations/layer10_attention_weight_min": -46.09175491333008,
      "activations/layer11_attention_weight_max": 46.626380920410156,
      "activations/layer11_attention_weight_min": -36.90135192871094,
      "activations/layer12_attention_weight_max": 38.377113342285156,
      "activations/layer12_attention_weight_min": -31.510631561279297,
      "activations/layer13_attention_weight_max": 43.1679573059082,
      "activations/layer13_attention_weight_min": -33.60407257080078,
      "activations/layer14_attention_weight_max": 68.31230926513672,
      "activations/layer14_attention_weight_min": -53.471168518066406,
      "activations/layer15_attention_weight_max": 47.65175247192383,
      "activations/layer15_attention_weight_min": -35.16179656982422,
      "activations/layer16_attention_weight_max": 48.4276008605957,
      "activations/layer16_attention_weight_min": -35.257240295410156,
      "activations/layer17_attention_weight_max": 34.780609130859375,
      "activations/layer17_attention_weight_min": -21.323562622070312,
      "activations/layer18_attention_weight_max": 20.155431747436523,
      "activations/layer18_attention_weight_min": -15.299410820007324,
      "activations/layer19_attention_weight_max": 46.37577819824219,
      "activations/layer19_attention_weight_min": -30.36151123046875,
      "activations/layer1_attention_weight_max": 15.353760719299316,
      "activations/layer1_attention_weight_min": -11.20187759399414,
      "activations/layer20_attention_weight_max": 30.62484359741211,
      "activations/layer20_attention_weight_min": -19.355058670043945,
      "activations/layer21_attention_weight_max": 17.999143600463867,
      "activations/layer21_attention_weight_min": -13.165745735168457,
      "activations/layer22_attention_weight_max": 35.19831085205078,
      "activations/layer22_attention_weight_min": -18.45370864868164,
      "activations/layer23_attention_weight_max": 22.50472640991211,
      "activations/layer23_attention_weight_min": -12.405112266540527,
      "activations/layer2_attention_weight_max": 27.471927642822266,
      "activations/layer2_attention_weight_min": -27.35399627685547,
      "activations/layer3_attention_weight_max": 56.370052337646484,
      "activations/layer3_attention_weight_min": -55.859432220458984,
      "activations/layer4_attention_weight_max": 110.01580810546875,
      "activations/layer4_attention_weight_min": -102.07034301757812,
      "activations/layer5_attention_weight_max": 124.48812103271484,
      "activations/layer5_attention_weight_min": -110.79597473144531,
      "activations/layer6_attention_weight_max": 70.4428482055664,
      "activations/layer6_attention_weight_min": -63.59441375732422,
      "activations/layer7_attention_weight_max": 66.77356719970703,
      "activations/layer7_attention_weight_min": -63.291072845458984,
      "activations/layer8_attention_weight_max": 63.82042694091797,
      "activations/layer8_attention_weight_min": -55.10453796386719,
      "activations/layer9_attention_weight_max": 68.89990234375,
      "activations/layer9_attention_weight_min": -63.72150421142578,
      "epoch": 0.79,
      "learning_rate": 0.00014637424242424241,
      "loss": 3.1624,
      "step": 13600
    },
    {
      "activations/layer0_attention_weight_max": 13.915969848632812,
      "activations/layer0_attention_weight_min": -11.454781532287598,
      "activations/layer10_attention_weight_max": 62.4316520690918,
      "activations/layer10_attention_weight_min": -48.55362319946289,
      "activations/layer11_attention_weight_max": 49.242408752441406,
      "activations/layer11_attention_weight_min": -36.49465560913086,
      "activations/layer12_attention_weight_max": 64.88321685791016,
      "activations/layer12_attention_weight_min": -39.42726516723633,
      "activations/layer13_attention_weight_max": 48.509254455566406,
      "activations/layer13_attention_weight_min": -33.40410232543945,
      "activations/layer14_attention_weight_max": 79.37135314941406,
      "activations/layer14_attention_weight_min": -51.97511291503906,
      "activations/layer15_attention_weight_max": 57.940269470214844,
      "activations/layer15_attention_weight_min": -36.97896194458008,
      "activations/layer16_attention_weight_max": 53.28604507446289,
      "activations/layer16_attention_weight_min": -30.509294509887695,
      "activations/layer17_attention_weight_max": 32.75660705566406,
      "activations/layer17_attention_weight_min": -19.696142196655273,
      "activations/layer18_attention_weight_max": 20.94701385498047,
      "activations/layer18_attention_weight_min": -12.042757987976074,
      "activations/layer19_attention_weight_max": 47.825565338134766,
      "activations/layer19_attention_weight_min": -25.433185577392578,
      "activations/layer1_attention_weight_max": 16.834793090820312,
      "activations/layer1_attention_weight_min": -11.18584156036377,
      "activations/layer20_attention_weight_max": 32.19157791137695,
      "activations/layer20_attention_weight_min": -14.810646057128906,
      "activations/layer21_attention_weight_max": 20.771852493286133,
      "activations/layer21_attention_weight_min": -15.440560340881348,
      "activations/layer22_attention_weight_max": 33.9207649230957,
      "activations/layer22_attention_weight_min": -15.890094757080078,
      "activations/layer23_attention_weight_max": 21.81805992126465,
      "activations/layer23_attention_weight_min": -13.060988426208496,
      "activations/layer2_attention_weight_max": 29.564945220947266,
      "activations/layer2_attention_weight_min": -25.784238815307617,
      "activations/layer3_attention_weight_max": 59.0197639465332,
      "activations/layer3_attention_weight_min": -57.86201477050781,
      "activations/layer4_attention_weight_max": 115.15471649169922,
      "activations/layer4_attention_weight_min": -105.16986083984375,
      "activations/layer5_attention_weight_max": 137.11029052734375,
      "activations/layer5_attention_weight_min": -115.61260223388672,
      "activations/layer6_attention_weight_max": 76.81375122070312,
      "activations/layer6_attention_weight_min": -63.525489807128906,
      "activations/layer7_attention_weight_max": 69.80183410644531,
      "activations/layer7_attention_weight_min": -64.38538360595703,
      "activations/layer8_attention_weight_max": 68.90560913085938,
      "activations/layer8_attention_weight_min": -56.9755744934082,
      "activations/layer9_attention_weight_max": 89.5435562133789,
      "activations/layer9_attention_weight_min": -73.19822692871094,
      "epoch": 0.79,
      "learning_rate": 0.00014635530303030303,
      "loss": 3.1604,
      "step": 13650
    },
    {
      "activations/layer0_attention_weight_max": 14.337027549743652,
      "activations/layer0_attention_weight_min": -11.928301811218262,
      "activations/layer10_attention_weight_max": 52.02155303955078,
      "activations/layer10_attention_weight_min": -54.058982849121094,
      "activations/layer11_attention_weight_max": 52.293487548828125,
      "activations/layer11_attention_weight_min": -40.32781219482422,
      "activations/layer12_attention_weight_max": 46.36680603027344,
      "activations/layer12_attention_weight_min": -29.53054428100586,
      "activations/layer13_attention_weight_max": 43.21953582763672,
      "activations/layer13_attention_weight_min": -36.643951416015625,
      "activations/layer14_attention_weight_max": 70.0429458618164,
      "activations/layer14_attention_weight_min": -60.73102951049805,
      "activations/layer15_attention_weight_max": 55.702144622802734,
      "activations/layer15_attention_weight_min": -37.691505432128906,
      "activations/layer16_attention_weight_max": 51.95224380493164,
      "activations/layer16_attention_weight_min": -35.71255874633789,
      "activations/layer17_attention_weight_max": 33.13385009765625,
      "activations/layer17_attention_weight_min": -19.327756881713867,
      "activations/layer18_attention_weight_max": 22.93731117248535,
      "activations/layer18_attention_weight_min": -13.65523624420166,
      "activations/layer19_attention_weight_max": 41.8977165222168,
      "activations/layer19_attention_weight_min": -28.63968849182129,
      "activations/layer1_attention_weight_max": 15.739625930786133,
      "activations/layer1_attention_weight_min": -12.617110252380371,
      "activations/layer20_attention_weight_max": 25.931861877441406,
      "activations/layer20_attention_weight_min": -18.62194061279297,
      "activations/layer21_attention_weight_max": 19.073396682739258,
      "activations/layer21_attention_weight_min": -13.129255294799805,
      "activations/layer22_attention_weight_max": 30.408105850219727,
      "activations/layer22_attention_weight_min": -21.5921630859375,
      "activations/layer23_attention_weight_max": 20.263242721557617,
      "activations/layer23_attention_weight_min": -13.721426010131836,
      "activations/layer2_attention_weight_max": 26.887413024902344,
      "activations/layer2_attention_weight_min": -24.937152862548828,
      "activations/layer3_attention_weight_max": 58.29973602294922,
      "activations/layer3_attention_weight_min": -54.345096588134766,
      "activations/layer4_attention_weight_max": 112.912353515625,
      "activations/layer4_attention_weight_min": -103.3237075805664,
      "activations/layer5_attention_weight_max": 137.7499237060547,
      "activations/layer5_attention_weight_min": -118.18669128417969,
      "activations/layer6_attention_weight_max": 73.83983612060547,
      "activations/layer6_attention_weight_min": -59.84025192260742,
      "activations/layer7_attention_weight_max": 69.18203735351562,
      "activations/layer7_attention_weight_min": -61.00453186035156,
      "activations/layer8_attention_weight_max": 60.20404052734375,
      "activations/layer8_attention_weight_min": -51.13847732543945,
      "activations/layer9_attention_weight_max": 88.30352020263672,
      "activations/layer9_attention_weight_min": -68.0199203491211,
      "epoch": 0.8,
      "learning_rate": 0.00014633636363636362,
      "loss": 3.151,
      "step": 13700
    },
    {
      "activations/layer0_attention_weight_max": 15.074177742004395,
      "activations/layer0_attention_weight_min": -11.481104850769043,
      "activations/layer10_attention_weight_max": 48.8138542175293,
      "activations/layer10_attention_weight_min": -42.343135833740234,
      "activations/layer11_attention_weight_max": 45.67829513549805,
      "activations/layer11_attention_weight_min": -41.60210418701172,
      "activations/layer12_attention_weight_max": 30.385051727294922,
      "activations/layer12_attention_weight_min": -25.58307647705078,
      "activations/layer13_attention_weight_max": 43.91617965698242,
      "activations/layer13_attention_weight_min": -33.46583557128906,
      "activations/layer14_attention_weight_max": 75.80538940429688,
      "activations/layer14_attention_weight_min": -54.82857894897461,
      "activations/layer15_attention_weight_max": 54.969295501708984,
      "activations/layer15_attention_weight_min": -39.71192932128906,
      "activations/layer16_attention_weight_max": 51.1642951965332,
      "activations/layer16_attention_weight_min": -35.36240005493164,
      "activations/layer17_attention_weight_max": 34.249900817871094,
      "activations/layer17_attention_weight_min": -19.52821159362793,
      "activations/layer18_attention_weight_max": 20.066024780273438,
      "activations/layer18_attention_weight_min": -12.6030912399292,
      "activations/layer19_attention_weight_max": 38.992794036865234,
      "activations/layer19_attention_weight_min": -26.348447799682617,
      "activations/layer1_attention_weight_max": 16.533287048339844,
      "activations/layer1_attention_weight_min": -11.108712196350098,
      "activations/layer20_attention_weight_max": 30.90129852294922,
      "activations/layer20_attention_weight_min": -17.6655330657959,
      "activations/layer21_attention_weight_max": 19.48849868774414,
      "activations/layer21_attention_weight_min": -13.686750411987305,
      "activations/layer22_attention_weight_max": 38.390769958496094,
      "activations/layer22_attention_weight_min": -20.945470809936523,
      "activations/layer23_attention_weight_max": 24.831480026245117,
      "activations/layer23_attention_weight_min": -14.739181518554688,
      "activations/layer2_attention_weight_max": 26.066648483276367,
      "activations/layer2_attention_weight_min": -24.18934440612793,
      "activations/layer3_attention_weight_max": 54.40201950073242,
      "activations/layer3_attention_weight_min": -57.16522216796875,
      "activations/layer4_attention_weight_max": 104.79918670654297,
      "activations/layer4_attention_weight_min": -104.66875457763672,
      "activations/layer5_attention_weight_max": 116.72611999511719,
      "activations/layer5_attention_weight_min": -109.1715087890625,
      "activations/layer6_attention_weight_max": 64.47480773925781,
      "activations/layer6_attention_weight_min": -63.21364212036133,
      "activations/layer7_attention_weight_max": 65.51648712158203,
      "activations/layer7_attention_weight_min": -59.63166809082031,
      "activations/layer8_attention_weight_max": 62.376708984375,
      "activations/layer8_attention_weight_min": -53.52214431762695,
      "activations/layer9_attention_weight_max": 66.1624984741211,
      "activations/layer9_attention_weight_min": -59.53390121459961,
      "epoch": 0.8,
      "learning_rate": 0.00014631742424242423,
      "loss": 3.1637,
      "step": 13750
    },
    {
      "activations/layer0_attention_weight_max": 14.567505836486816,
      "activations/layer0_attention_weight_min": -11.137948036193848,
      "activations/layer10_attention_weight_max": 59.337791442871094,
      "activations/layer10_attention_weight_min": -49.81684112548828,
      "activations/layer11_attention_weight_max": 46.32268524169922,
      "activations/layer11_attention_weight_min": -37.94010925292969,
      "activations/layer12_attention_weight_max": 37.19148635864258,
      "activations/layer12_attention_weight_min": -27.408767700195312,
      "activations/layer13_attention_weight_max": 48.127708435058594,
      "activations/layer13_attention_weight_min": -35.16868591308594,
      "activations/layer14_attention_weight_max": 76.85028076171875,
      "activations/layer14_attention_weight_min": -50.179439544677734,
      "activations/layer15_attention_weight_max": 58.031558990478516,
      "activations/layer15_attention_weight_min": -37.40966033935547,
      "activations/layer16_attention_weight_max": 56.62382888793945,
      "activations/layer16_attention_weight_min": -38.43413543701172,
      "activations/layer17_attention_weight_max": 29.21209144592285,
      "activations/layer17_attention_weight_min": -20.752702713012695,
      "activations/layer18_attention_weight_max": 18.55055809020996,
      "activations/layer18_attention_weight_min": -12.430414199829102,
      "activations/layer19_attention_weight_max": 49.51906967163086,
      "activations/layer19_attention_weight_min": -28.098440170288086,
      "activations/layer1_attention_weight_max": 14.738059043884277,
      "activations/layer1_attention_weight_min": -12.23962116241455,
      "activations/layer20_attention_weight_max": 29.524608612060547,
      "activations/layer20_attention_weight_min": -20.518756866455078,
      "activations/layer21_attention_weight_max": 20.8704833984375,
      "activations/layer21_attention_weight_min": -14.67263126373291,
      "activations/layer22_attention_weight_max": 30.15890121459961,
      "activations/layer22_attention_weight_min": -18.65509796142578,
      "activations/layer23_attention_weight_max": 20.952007293701172,
      "activations/layer23_attention_weight_min": -13.757704734802246,
      "activations/layer2_attention_weight_max": 30.117618560791016,
      "activations/layer2_attention_weight_min": -26.844680786132812,
      "activations/layer3_attention_weight_max": 58.577396392822266,
      "activations/layer3_attention_weight_min": -57.375511169433594,
      "activations/layer4_attention_weight_max": 109.51177978515625,
      "activations/layer4_attention_weight_min": -104.72600555419922,
      "activations/layer5_attention_weight_max": 129.19137573242188,
      "activations/layer5_attention_weight_min": -114.72460174560547,
      "activations/layer6_attention_weight_max": 69.5162353515625,
      "activations/layer6_attention_weight_min": -62.014286041259766,
      "activations/layer7_attention_weight_max": 65.588623046875,
      "activations/layer7_attention_weight_min": -62.74302673339844,
      "activations/layer8_attention_weight_max": 65.80900573730469,
      "activations/layer8_attention_weight_min": -56.85793685913086,
      "activations/layer9_attention_weight_max": 72.0354232788086,
      "activations/layer9_attention_weight_min": -64.91465759277344,
      "epoch": 0.8,
      "learning_rate": 0.00014629848484848485,
      "loss": 3.1702,
      "step": 13800
    },
    {
      "activations/layer0_attention_weight_max": 14.708039283752441,
      "activations/layer0_attention_weight_min": -10.975786209106445,
      "activations/layer10_attention_weight_max": 54.21159362792969,
      "activations/layer10_attention_weight_min": -45.75437927246094,
      "activations/layer11_attention_weight_max": 47.15641403198242,
      "activations/layer11_attention_weight_min": -38.697513580322266,
      "activations/layer12_attention_weight_max": 34.30585861206055,
      "activations/layer12_attention_weight_min": -27.77733039855957,
      "activations/layer13_attention_weight_max": 47.97394561767578,
      "activations/layer13_attention_weight_min": -36.50578308105469,
      "activations/layer14_attention_weight_max": 67.62226867675781,
      "activations/layer14_attention_weight_min": -50.18939208984375,
      "activations/layer15_attention_weight_max": 48.16352844238281,
      "activations/layer15_attention_weight_min": -35.8651123046875,
      "activations/layer16_attention_weight_max": 45.5363883972168,
      "activations/layer16_attention_weight_min": -35.8005256652832,
      "activations/layer17_attention_weight_max": 29.190176010131836,
      "activations/layer17_attention_weight_min": -20.015594482421875,
      "activations/layer18_attention_weight_max": 19.70983123779297,
      "activations/layer18_attention_weight_min": -12.727150917053223,
      "activations/layer19_attention_weight_max": 46.548805236816406,
      "activations/layer19_attention_weight_min": -33.38677978515625,
      "activations/layer1_attention_weight_max": 16.572853088378906,
      "activations/layer1_attention_weight_min": -12.01079273223877,
      "activations/layer20_attention_weight_max": 32.43814468383789,
      "activations/layer20_attention_weight_min": -20.447622299194336,
      "activations/layer21_attention_weight_max": 20.795185089111328,
      "activations/layer21_attention_weight_min": -14.309648513793945,
      "activations/layer22_attention_weight_max": 34.06449890136719,
      "activations/layer22_attention_weight_min": -19.416767120361328,
      "activations/layer23_attention_weight_max": 21.90749168395996,
      "activations/layer23_attention_weight_min": -13.810264587402344,
      "activations/layer2_attention_weight_max": 28.049129486083984,
      "activations/layer2_attention_weight_min": -27.505462646484375,
      "activations/layer3_attention_weight_max": 56.45276641845703,
      "activations/layer3_attention_weight_min": -56.94960021972656,
      "activations/layer4_attention_weight_max": 108.1218032836914,
      "activations/layer4_attention_weight_min": -101.82299041748047,
      "activations/layer5_attention_weight_max": 120.0997543334961,
      "activations/layer5_attention_weight_min": -107.92989349365234,
      "activations/layer6_attention_weight_max": 63.862037658691406,
      "activations/layer6_attention_weight_min": -59.55614471435547,
      "activations/layer7_attention_weight_max": 65.20237731933594,
      "activations/layer7_attention_weight_min": -65.6474609375,
      "activations/layer8_attention_weight_max": 60.44503402709961,
      "activations/layer8_attention_weight_min": -57.31159973144531,
      "activations/layer9_attention_weight_max": 71.64049530029297,
      "activations/layer9_attention_weight_min": -63.059783935546875,
      "epoch": 0.8,
      "learning_rate": 0.00014627954545454544,
      "loss": 3.1666,
      "step": 13850
    },
    {
      "activations/layer0_attention_weight_max": 15.002021789550781,
      "activations/layer0_attention_weight_min": -10.47449016571045,
      "activations/layer10_attention_weight_max": 52.9984245300293,
      "activations/layer10_attention_weight_min": -51.765960693359375,
      "activations/layer11_attention_weight_max": 46.07926940917969,
      "activations/layer11_attention_weight_min": -38.885276794433594,
      "activations/layer12_attention_weight_max": 37.20197296142578,
      "activations/layer12_attention_weight_min": -31.49604034423828,
      "activations/layer13_attention_weight_max": 41.62602996826172,
      "activations/layer13_attention_weight_min": -34.73776626586914,
      "activations/layer14_attention_weight_max": 73.15161895751953,
      "activations/layer14_attention_weight_min": -58.244964599609375,
      "activations/layer15_attention_weight_max": 48.153926849365234,
      "activations/layer15_attention_weight_min": -37.563148498535156,
      "activations/layer16_attention_weight_max": 47.11430740356445,
      "activations/layer16_attention_weight_min": -37.90465545654297,
      "activations/layer17_attention_weight_max": 29.722042083740234,
      "activations/layer17_attention_weight_min": -20.062217712402344,
      "activations/layer18_attention_weight_max": 16.691776275634766,
      "activations/layer18_attention_weight_min": -11.73847484588623,
      "activations/layer19_attention_weight_max": 42.421749114990234,
      "activations/layer19_attention_weight_min": -29.231922149658203,
      "activations/layer1_attention_weight_max": 15.051236152648926,
      "activations/layer1_attention_weight_min": -10.807394027709961,
      "activations/layer20_attention_weight_max": 28.03325843811035,
      "activations/layer20_attention_weight_min": -17.990896224975586,
      "activations/layer21_attention_weight_max": 18.284494400024414,
      "activations/layer21_attention_weight_min": -15.459367752075195,
      "activations/layer22_attention_weight_max": 32.1818733215332,
      "activations/layer22_attention_weight_min": -20.169635772705078,
      "activations/layer23_attention_weight_max": 21.889957427978516,
      "activations/layer23_attention_weight_min": -14.859694480895996,
      "activations/layer2_attention_weight_max": 27.14046287536621,
      "activations/layer2_attention_weight_min": -23.99738883972168,
      "activations/layer3_attention_weight_max": 55.203243255615234,
      "activations/layer3_attention_weight_min": -54.60380935668945,
      "activations/layer4_attention_weight_max": 107.50172424316406,
      "activations/layer4_attention_weight_min": -103.49962615966797,
      "activations/layer5_attention_weight_max": 123.31932067871094,
      "activations/layer5_attention_weight_min": -111.47929382324219,
      "activations/layer6_attention_weight_max": 66.54104614257812,
      "activations/layer6_attention_weight_min": -64.30805969238281,
      "activations/layer7_attention_weight_max": 63.94355010986328,
      "activations/layer7_attention_weight_min": -62.46018600463867,
      "activations/layer8_attention_weight_max": 62.3123664855957,
      "activations/layer8_attention_weight_min": -53.707332611083984,
      "activations/layer9_attention_weight_max": 73.24183654785156,
      "activations/layer9_attention_weight_min": -59.66449737548828,
      "epoch": 0.81,
      "learning_rate": 0.00014626060606060605,
      "loss": 3.1771,
      "step": 13900
    },
    {
      "activations/layer0_attention_weight_max": 15.462061882019043,
      "activations/layer0_attention_weight_min": -10.950141906738281,
      "activations/layer10_attention_weight_max": 56.68145751953125,
      "activations/layer10_attention_weight_min": -44.87117385864258,
      "activations/layer11_attention_weight_max": 47.548423767089844,
      "activations/layer11_attention_weight_min": -42.2642707824707,
      "activations/layer12_attention_weight_max": 36.16282272338867,
      "activations/layer12_attention_weight_min": -31.390430450439453,
      "activations/layer13_attention_weight_max": 45.22879409790039,
      "activations/layer13_attention_weight_min": -35.296661376953125,
      "activations/layer14_attention_weight_max": 71.36198425292969,
      "activations/layer14_attention_weight_min": -47.89337158203125,
      "activations/layer15_attention_weight_max": 50.324092864990234,
      "activations/layer15_attention_weight_min": -34.6923828125,
      "activations/layer16_attention_weight_max": 51.91444396972656,
      "activations/layer16_attention_weight_min": -34.76821517944336,
      "activations/layer17_attention_weight_max": 30.517005920410156,
      "activations/layer17_attention_weight_min": -18.636253356933594,
      "activations/layer18_attention_weight_max": 19.57857322692871,
      "activations/layer18_attention_weight_min": -12.141697883605957,
      "activations/layer19_attention_weight_max": 50.23457717895508,
      "activations/layer19_attention_weight_min": -29.504501342773438,
      "activations/layer1_attention_weight_max": 15.21302604675293,
      "activations/layer1_attention_weight_min": -11.855350494384766,
      "activations/layer20_attention_weight_max": 34.35888671875,
      "activations/layer20_attention_weight_min": -17.279403686523438,
      "activations/layer21_attention_weight_max": 20.451210021972656,
      "activations/layer21_attention_weight_min": -13.467289924621582,
      "activations/layer22_attention_weight_max": 34.61744689941406,
      "activations/layer22_attention_weight_min": -22.714557647705078,
      "activations/layer23_attention_weight_max": 22.404720306396484,
      "activations/layer23_attention_weight_min": -14.687666893005371,
      "activations/layer2_attention_weight_max": 27.78584098815918,
      "activations/layer2_attention_weight_min": -24.574386596679688,
      "activations/layer3_attention_weight_max": 55.54615020751953,
      "activations/layer3_attention_weight_min": -55.5879020690918,
      "activations/layer4_attention_weight_max": 108.1823959350586,
      "activations/layer4_attention_weight_min": -102.58678436279297,
      "activations/layer5_attention_weight_max": 124.27769470214844,
      "activations/layer5_attention_weight_min": -110.04061126708984,
      "activations/layer6_attention_weight_max": 67.63726806640625,
      "activations/layer6_attention_weight_min": -62.18251419067383,
      "activations/layer7_attention_weight_max": 67.28076171875,
      "activations/layer7_attention_weight_min": -65.61211395263672,
      "activations/layer8_attention_weight_max": 59.29742431640625,
      "activations/layer8_attention_weight_min": -51.30791091918945,
      "activations/layer9_attention_weight_max": 70.29442596435547,
      "activations/layer9_attention_weight_min": -58.4788932800293,
      "epoch": 0.81,
      "learning_rate": 0.00014624166666666664,
      "loss": 3.1616,
      "step": 13950
    },
    {
      "activations/layer0_attention_weight_max": 14.860249519348145,
      "activations/layer0_attention_weight_min": -11.51253604888916,
      "activations/layer10_attention_weight_max": 58.42565155029297,
      "activations/layer10_attention_weight_min": -47.6275634765625,
      "activations/layer11_attention_weight_max": 46.33826446533203,
      "activations/layer11_attention_weight_min": -36.85033416748047,
      "activations/layer12_attention_weight_max": 33.5873908996582,
      "activations/layer12_attention_weight_min": -29.850143432617188,
      "activations/layer13_attention_weight_max": 41.60313034057617,
      "activations/layer13_attention_weight_min": -32.90450668334961,
      "activations/layer14_attention_weight_max": 64.5567855834961,
      "activations/layer14_attention_weight_min": -50.686153411865234,
      "activations/layer15_attention_weight_max": 50.94564437866211,
      "activations/layer15_attention_weight_min": -36.964622497558594,
      "activations/layer16_attention_weight_max": 50.82215118408203,
      "activations/layer16_attention_weight_min": -36.2425422668457,
      "activations/layer17_attention_weight_max": 34.22905349731445,
      "activations/layer17_attention_weight_min": -20.37567138671875,
      "activations/layer18_attention_weight_max": 23.751684188842773,
      "activations/layer18_attention_weight_min": -12.869612693786621,
      "activations/layer19_attention_weight_max": 45.04344940185547,
      "activations/layer19_attention_weight_min": -32.294979095458984,
      "activations/layer1_attention_weight_max": 15.102691650390625,
      "activations/layer1_attention_weight_min": -12.793810844421387,
      "activations/layer20_attention_weight_max": 28.71126937866211,
      "activations/layer20_attention_weight_min": -17.885770797729492,
      "activations/layer21_attention_weight_max": 20.132633209228516,
      "activations/layer21_attention_weight_min": -14.397371292114258,
      "activations/layer22_attention_weight_max": 37.19395065307617,
      "activations/layer22_attention_weight_min": -22.39228630065918,
      "activations/layer23_attention_weight_max": 23.906118392944336,
      "activations/layer23_attention_weight_min": -14.754562377929688,
      "activations/layer2_attention_weight_max": 28.7670955657959,
      "activations/layer2_attention_weight_min": -26.59524917602539,
      "activations/layer3_attention_weight_max": 56.707706451416016,
      "activations/layer3_attention_weight_min": -54.4667854309082,
      "activations/layer4_attention_weight_max": 110.25170135498047,
      "activations/layer4_attention_weight_min": -99.82905578613281,
      "activations/layer5_attention_weight_max": 123.10247802734375,
      "activations/layer5_attention_weight_min": -107.57927703857422,
      "activations/layer6_attention_weight_max": 67.2066879272461,
      "activations/layer6_attention_weight_min": -62.6627311706543,
      "activations/layer7_attention_weight_max": 65.84850311279297,
      "activations/layer7_attention_weight_min": -61.61049270629883,
      "activations/layer8_attention_weight_max": 60.39190673828125,
      "activations/layer8_attention_weight_min": -50.59218215942383,
      "activations/layer9_attention_weight_max": 64.5693588256836,
      "activations/layer9_attention_weight_min": -60.57542037963867,
      "epoch": 0.81,
      "learning_rate": 0.00014622272727272725,
      "loss": 3.1658,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_loss": 3.09765625,
      "eval_runtime": 8.1061,
      "eval_samples_per_second": 529.725,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_openwebtext_loss": 3.09765625,
      "eval_openwebtext_ppl": 22.145985754016134,
      "eval_openwebtext_runtime": 8.1061,
      "eval_openwebtext_samples_per_second": 529.725,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_wikitext_loss": 3.384765625,
      "eval_wikitext_ppl": 29.511075247281195,
      "eval_wikitext_runtime": 1.5028,
      "eval_wikitext_samples_per_second": 303.433,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_lambada_loss": 3.646484375,
      "eval_lambada_ppl": 38.33964103892552,
      "eval_lambada_runtime": 9.5903,
      "eval_lambada_samples_per_second": 507.699,
      "step": 14000
    },
    {
      "activations/layer0_attention_weight_max": 15.275430679321289,
      "activations/layer0_attention_weight_min": -11.077747344970703,
      "activations/layer10_attention_weight_max": 56.450584411621094,
      "activations/layer10_attention_weight_min": -46.86915969848633,
      "activations/layer11_attention_weight_max": 44.534934997558594,
      "activations/layer11_attention_weight_min": -37.4153938293457,
      "activations/layer12_attention_weight_max": 34.176902770996094,
      "activations/layer12_attention_weight_min": -32.40800476074219,
      "activations/layer13_attention_weight_max": 43.28752517700195,
      "activations/layer13_attention_weight_min": -37.597747802734375,
      "activations/layer14_attention_weight_max": 70.215576171875,
      "activations/layer14_attention_weight_min": -53.949161529541016,
      "activations/layer15_attention_weight_max": 53.001853942871094,
      "activations/layer15_attention_weight_min": -39.61591339111328,
      "activations/layer16_attention_weight_max": 55.530052185058594,
      "activations/layer16_attention_weight_min": -38.850032806396484,
      "activations/layer17_attention_weight_max": 33.90333938598633,
      "activations/layer17_attention_weight_min": -21.6557559967041,
      "activations/layer18_attention_weight_max": 23.997488021850586,
      "activations/layer18_attention_weight_min": -12.554502487182617,
      "activations/layer19_attention_weight_max": 43.89455795288086,
      "activations/layer19_attention_weight_min": -28.117313385009766,
      "activations/layer1_attention_weight_max": 15.7589693069458,
      "activations/layer1_attention_weight_min": -11.257031440734863,
      "activations/layer20_attention_weight_max": 30.74226951599121,
      "activations/layer20_attention_weight_min": -17.488666534423828,
      "activations/layer21_attention_weight_max": 20.052261352539062,
      "activations/layer21_attention_weight_min": -14.416505813598633,
      "activations/layer22_attention_weight_max": 36.4971809387207,
      "activations/layer22_attention_weight_min": -21.00765037536621,
      "activations/layer23_attention_weight_max": 24.73287582397461,
      "activations/layer23_attention_weight_min": -14.276873588562012,
      "activations/layer2_attention_weight_max": 27.549171447753906,
      "activations/layer2_attention_weight_min": -26.313138961791992,
      "activations/layer3_attention_weight_max": 58.08905792236328,
      "activations/layer3_attention_weight_min": -58.18077850341797,
      "activations/layer4_attention_weight_max": 112.86854553222656,
      "activations/layer4_attention_weight_min": -105.7569580078125,
      "activations/layer5_attention_weight_max": 131.79283142089844,
      "activations/layer5_attention_weight_min": -110.94976043701172,
      "activations/layer6_attention_weight_max": 68.4609146118164,
      "activations/layer6_attention_weight_min": -65.60893249511719,
      "activations/layer7_attention_weight_max": 67.09526824951172,
      "activations/layer7_attention_weight_min": -62.836273193359375,
      "activations/layer8_attention_weight_max": 61.07793045043945,
      "activations/layer8_attention_weight_min": -52.48915481567383,
      "activations/layer9_attention_weight_max": 72.42821502685547,
      "activations/layer9_attention_weight_min": -57.722999572753906,
      "epoch": 0.82,
      "learning_rate": 0.00014620416666666664,
      "loss": 3.1555,
      "step": 14050
    },
    {
      "activations/layer0_attention_weight_max": 15.067484855651855,
      "activations/layer0_attention_weight_min": -10.383923530578613,
      "activations/layer10_attention_weight_max": 55.917747497558594,
      "activations/layer10_attention_weight_min": -46.085506439208984,
      "activations/layer11_attention_weight_max": 59.624961853027344,
      "activations/layer11_attention_weight_min": -43.478511810302734,
      "activations/layer12_attention_weight_max": 84.4786148071289,
      "activations/layer12_attention_weight_min": -45.723724365234375,
      "activations/layer13_attention_weight_max": 83.22759246826172,
      "activations/layer13_attention_weight_min": -44.51765060424805,
      "activations/layer14_attention_weight_max": 88.22976684570312,
      "activations/layer14_attention_weight_min": -49.945037841796875,
      "activations/layer15_attention_weight_max": 52.40788269042969,
      "activations/layer15_attention_weight_min": -33.95924377441406,
      "activations/layer16_attention_weight_max": 55.69573211669922,
      "activations/layer16_attention_weight_min": -37.09578323364258,
      "activations/layer17_attention_weight_max": 31.362037658691406,
      "activations/layer17_attention_weight_min": -20.084203720092773,
      "activations/layer18_attention_weight_max": 20.121829986572266,
      "activations/layer18_attention_weight_min": -13.223356246948242,
      "activations/layer19_attention_weight_max": 51.64895248413086,
      "activations/layer19_attention_weight_min": -24.76411247253418,
      "activations/layer1_attention_weight_max": 14.76646614074707,
      "activations/layer1_attention_weight_min": -11.504729270935059,
      "activations/layer20_attention_weight_max": 32.70649337768555,
      "activations/layer20_attention_weight_min": -15.46972942352295,
      "activations/layer21_attention_weight_max": 22.77256202697754,
      "activations/layer21_attention_weight_min": -12.84533977508545,
      "activations/layer22_attention_weight_max": 33.392494201660156,
      "activations/layer22_attention_weight_min": -17.500961303710938,
      "activations/layer23_attention_weight_max": 22.20163345336914,
      "activations/layer23_attention_weight_min": -14.136993408203125,
      "activations/layer2_attention_weight_max": 29.26160430908203,
      "activations/layer2_attention_weight_min": -26.127212524414062,
      "activations/layer3_attention_weight_max": 58.49162292480469,
      "activations/layer3_attention_weight_min": -59.49843978881836,
      "activations/layer4_attention_weight_max": 111.95709228515625,
      "activations/layer4_attention_weight_min": -105.4468002319336,
      "activations/layer5_attention_weight_max": 133.902099609375,
      "activations/layer5_attention_weight_min": -114.8680648803711,
      "activations/layer6_attention_weight_max": 72.73953247070312,
      "activations/layer6_attention_weight_min": -61.02969741821289,
      "activations/layer7_attention_weight_max": 79.18978881835938,
      "activations/layer7_attention_weight_min": -67.58119201660156,
      "activations/layer8_attention_weight_max": 80.2856674194336,
      "activations/layer8_attention_weight_min": -63.91167449951172,
      "activations/layer9_attention_weight_max": 106.7782211303711,
      "activations/layer9_attention_weight_min": -80.94647216796875,
      "epoch": 0.82,
      "learning_rate": 0.00014618522727272726,
      "loss": 3.1609,
      "step": 14100
    },
    {
      "activations/layer0_attention_weight_max": 15.48357105255127,
      "activations/layer0_attention_weight_min": -11.228094100952148,
      "activations/layer10_attention_weight_max": 52.465328216552734,
      "activations/layer10_attention_weight_min": -48.503868103027344,
      "activations/layer11_attention_weight_max": 41.32746124267578,
      "activations/layer11_attention_weight_min": -38.83095932006836,
      "activations/layer12_attention_weight_max": 54.85551834106445,
      "activations/layer12_attention_weight_min": -34.980567932128906,
      "activations/layer13_attention_weight_max": 53.51802444458008,
      "activations/layer13_attention_weight_min": -36.98225784301758,
      "activations/layer14_attention_weight_max": 65.94017028808594,
      "activations/layer14_attention_weight_min": -54.95014190673828,
      "activations/layer15_attention_weight_max": 55.296051025390625,
      "activations/layer15_attention_weight_min": -40.79552459716797,
      "activations/layer16_attention_weight_max": 45.951751708984375,
      "activations/layer16_attention_weight_min": -41.59117889404297,
      "activations/layer17_attention_weight_max": 27.586793899536133,
      "activations/layer17_attention_weight_min": -22.74296760559082,
      "activations/layer18_attention_weight_max": 19.96625518798828,
      "activations/layer18_attention_weight_min": -12.61245059967041,
      "activations/layer19_attention_weight_max": 39.64278793334961,
      "activations/layer19_attention_weight_min": -29.528371810913086,
      "activations/layer1_attention_weight_max": 15.446548461914062,
      "activations/layer1_attention_weight_min": -11.630231857299805,
      "activations/layer20_attention_weight_max": 28.318025588989258,
      "activations/layer20_attention_weight_min": -18.269622802734375,
      "activations/layer21_attention_weight_max": 18.44525718688965,
      "activations/layer21_attention_weight_min": -15.768081665039062,
      "activations/layer22_attention_weight_max": 30.654850006103516,
      "activations/layer22_attention_weight_min": -18.083520889282227,
      "activations/layer23_attention_weight_max": 21.393239974975586,
      "activations/layer23_attention_weight_min": -13.090190887451172,
      "activations/layer2_attention_weight_max": 28.1292724609375,
      "activations/layer2_attention_weight_min": -25.040000915527344,
      "activations/layer3_attention_weight_max": 56.14213562011719,
      "activations/layer3_attention_weight_min": -56.149658203125,
      "activations/layer4_attention_weight_max": 109.0550308227539,
      "activations/layer4_attention_weight_min": -103.25933074951172,
      "activations/layer5_attention_weight_max": 116.72423553466797,
      "activations/layer5_attention_weight_min": -109.8240966796875,
      "activations/layer6_attention_weight_max": 66.4554672241211,
      "activations/layer6_attention_weight_min": -61.062416076660156,
      "activations/layer7_attention_weight_max": 62.468135833740234,
      "activations/layer7_attention_weight_min": -67.7763671875,
      "activations/layer8_attention_weight_max": 57.16874313354492,
      "activations/layer8_attention_weight_min": -53.1536750793457,
      "activations/layer9_attention_weight_max": 75.8506088256836,
      "activations/layer9_attention_weight_min": -63.40230941772461,
      "epoch": 0.82,
      "learning_rate": 0.00014616628787878785,
      "loss": 3.1328,
      "step": 14150
    },
    {
      "activations/layer0_attention_weight_max": 15.879932403564453,
      "activations/layer0_attention_weight_min": -10.951898574829102,
      "activations/layer10_attention_weight_max": 55.764984130859375,
      "activations/layer10_attention_weight_min": -45.63698196411133,
      "activations/layer11_attention_weight_max": 46.810546875,
      "activations/layer11_attention_weight_min": -36.16066360473633,
      "activations/layer12_attention_weight_max": 68.78504943847656,
      "activations/layer12_attention_weight_min": -40.19782638549805,
      "activations/layer13_attention_weight_max": 55.748870849609375,
      "activations/layer13_attention_weight_min": -36.17339324951172,
      "activations/layer14_attention_weight_max": 72.67758178710938,
      "activations/layer14_attention_weight_min": -50.6053466796875,
      "activations/layer15_attention_weight_max": 50.49012756347656,
      "activations/layer15_attention_weight_min": -34.52052688598633,
      "activations/layer16_attention_weight_max": 39.63614273071289,
      "activations/layer16_attention_weight_min": -34.465457916259766,
      "activations/layer17_attention_weight_max": 33.83518600463867,
      "activations/layer17_attention_weight_min": -20.117286682128906,
      "activations/layer18_attention_weight_max": 17.375747680664062,
      "activations/layer18_attention_weight_min": -13.486374855041504,
      "activations/layer19_attention_weight_max": 40.59516525268555,
      "activations/layer19_attention_weight_min": -28.773786544799805,
      "activations/layer1_attention_weight_max": 16.076133728027344,
      "activations/layer1_attention_weight_min": -12.30937671661377,
      "activations/layer20_attention_weight_max": 24.941570281982422,
      "activations/layer20_attention_weight_min": -17.273786544799805,
      "activations/layer21_attention_weight_max": 17.801681518554688,
      "activations/layer21_attention_weight_min": -14.215770721435547,
      "activations/layer22_attention_weight_max": 36.97966384887695,
      "activations/layer22_attention_weight_min": -20.294116973876953,
      "activations/layer23_attention_weight_max": 25.106245040893555,
      "activations/layer23_attention_weight_min": -16.049245834350586,
      "activations/layer2_attention_weight_max": 27.817861557006836,
      "activations/layer2_attention_weight_min": -26.316999435424805,
      "activations/layer3_attention_weight_max": 57.446353912353516,
      "activations/layer3_attention_weight_min": -58.66134262084961,
      "activations/layer4_attention_weight_max": 109.12801361083984,
      "activations/layer4_attention_weight_min": -101.13817596435547,
      "activations/layer5_attention_weight_max": 124.79369354248047,
      "activations/layer5_attention_weight_min": -114.4087142944336,
      "activations/layer6_attention_weight_max": 72.2109603881836,
      "activations/layer6_attention_weight_min": -62.37681579589844,
      "activations/layer7_attention_weight_max": 76.69734954833984,
      "activations/layer7_attention_weight_min": -68.06812286376953,
      "activations/layer8_attention_weight_max": 63.62332534790039,
      "activations/layer8_attention_weight_min": -55.58595275878906,
      "activations/layer9_attention_weight_max": 91.31614685058594,
      "activations/layer9_attention_weight_min": -71.31663513183594,
      "epoch": 0.83,
      "learning_rate": 0.00014614734848484846,
      "loss": 3.1474,
      "step": 14200
    },
    {
      "activations/layer0_attention_weight_max": 16.267276763916016,
      "activations/layer0_attention_weight_min": -10.81932544708252,
      "activations/layer10_attention_weight_max": 53.60028076171875,
      "activations/layer10_attention_weight_min": -47.20183181762695,
      "activations/layer11_attention_weight_max": 44.16608428955078,
      "activations/layer11_attention_weight_min": -40.07048034667969,
      "activations/layer12_attention_weight_max": 31.85903549194336,
      "activations/layer12_attention_weight_min": -26.30699920654297,
      "activations/layer13_attention_weight_max": 39.32865524291992,
      "activations/layer13_attention_weight_min": -32.34280776977539,
      "activations/layer14_attention_weight_max": 65.94636535644531,
      "activations/layer14_attention_weight_min": -51.10115432739258,
      "activations/layer15_attention_weight_max": 49.988277435302734,
      "activations/layer15_attention_weight_min": -37.615177154541016,
      "activations/layer16_attention_weight_max": 58.75436782836914,
      "activations/layer16_attention_weight_min": -39.14434051513672,
      "activations/layer17_attention_weight_max": 33.522579193115234,
      "activations/layer17_attention_weight_min": -21.389272689819336,
      "activations/layer18_attention_weight_max": 21.538738250732422,
      "activations/layer18_attention_weight_min": -13.117171287536621,
      "activations/layer19_attention_weight_max": 55.60857009887695,
      "activations/layer19_attention_weight_min": -29.979801177978516,
      "activations/layer1_attention_weight_max": 15.36030101776123,
      "activations/layer1_attention_weight_min": -12.321922302246094,
      "activations/layer20_attention_weight_max": 32.84544372558594,
      "activations/layer20_attention_weight_min": -20.22869873046875,
      "activations/layer21_attention_weight_max": 19.990015029907227,
      "activations/layer21_attention_weight_min": -14.39799690246582,
      "activations/layer22_attention_weight_max": 37.241722106933594,
      "activations/layer22_attention_weight_min": -21.362932205200195,
      "activations/layer23_attention_weight_max": 24.277509689331055,
      "activations/layer23_attention_weight_min": -14.678213119506836,
      "activations/layer2_attention_weight_max": 28.69379997253418,
      "activations/layer2_attention_weight_min": -26.763771057128906,
      "activations/layer3_attention_weight_max": 55.47304916381836,
      "activations/layer3_attention_weight_min": -56.52847671508789,
      "activations/layer4_attention_weight_max": 103.1053695678711,
      "activations/layer4_attention_weight_min": -103.61927795410156,
      "activations/layer5_attention_weight_max": 119.35536193847656,
      "activations/layer5_attention_weight_min": -110.27796936035156,
      "activations/layer6_attention_weight_max": 70.62568664550781,
      "activations/layer6_attention_weight_min": -63.35438537597656,
      "activations/layer7_attention_weight_max": 64.14275360107422,
      "activations/layer7_attention_weight_min": -59.8161506652832,
      "activations/layer8_attention_weight_max": 60.69475555419922,
      "activations/layer8_attention_weight_min": -57.57480239868164,
      "activations/layer9_attention_weight_max": 70.75962829589844,
      "activations/layer9_attention_weight_min": -57.585548400878906,
      "epoch": 0.83,
      "learning_rate": 0.00014612840909090908,
      "loss": 3.1532,
      "step": 14250
    },
    {
      "activations/layer0_attention_weight_max": 15.417715072631836,
      "activations/layer0_attention_weight_min": -10.542956352233887,
      "activations/layer10_attention_weight_max": 52.97312545776367,
      "activations/layer10_attention_weight_min": -44.88932418823242,
      "activations/layer11_attention_weight_max": 47.1910514831543,
      "activations/layer11_attention_weight_min": -39.57844161987305,
      "activations/layer12_attention_weight_max": 45.080291748046875,
      "activations/layer12_attention_weight_min": -45.09134292602539,
      "activations/layer13_attention_weight_max": 43.53034591674805,
      "activations/layer13_attention_weight_min": -36.89824676513672,
      "activations/layer14_attention_weight_max": 69.37159729003906,
      "activations/layer14_attention_weight_min": -53.21769332885742,
      "activations/layer15_attention_weight_max": 52.519290924072266,
      "activations/layer15_attention_weight_min": -41.28599548339844,
      "activations/layer16_attention_weight_max": 54.401268005371094,
      "activations/layer16_attention_weight_min": -35.03630065917969,
      "activations/layer17_attention_weight_max": 35.27555847167969,
      "activations/layer17_attention_weight_min": -19.994815826416016,
      "activations/layer18_attention_weight_max": 21.695425033569336,
      "activations/layer18_attention_weight_min": -13.358275413513184,
      "activations/layer19_attention_weight_max": 44.519466400146484,
      "activations/layer19_attention_weight_min": -29.526020050048828,
      "activations/layer1_attention_weight_max": 15.456500053405762,
      "activations/layer1_attention_weight_min": -11.52529239654541,
      "activations/layer20_attention_weight_max": 31.834278106689453,
      "activations/layer20_attention_weight_min": -19.50653839111328,
      "activations/layer21_attention_weight_max": 22.52362632751465,
      "activations/layer21_attention_weight_min": -13.183648109436035,
      "activations/layer22_attention_weight_max": 38.801544189453125,
      "activations/layer22_attention_weight_min": -20.135086059570312,
      "activations/layer23_attention_weight_max": 25.566694259643555,
      "activations/layer23_attention_weight_min": -14.15174674987793,
      "activations/layer2_attention_weight_max": 30.15645980834961,
      "activations/layer2_attention_weight_min": -25.570796966552734,
      "activations/layer3_attention_weight_max": 59.3028564453125,
      "activations/layer3_attention_weight_min": -57.93307113647461,
      "activations/layer4_attention_weight_max": 111.02738189697266,
      "activations/layer4_attention_weight_min": -106.75691223144531,
      "activations/layer5_attention_weight_max": 131.84115600585938,
      "activations/layer5_attention_weight_min": -112.71025848388672,
      "activations/layer6_attention_weight_max": 72.46337127685547,
      "activations/layer6_attention_weight_min": -64.17384338378906,
      "activations/layer7_attention_weight_max": 72.01134490966797,
      "activations/layer7_attention_weight_min": -68.02747344970703,
      "activations/layer8_attention_weight_max": 67.614013671875,
      "activations/layer8_attention_weight_min": -56.19694519042969,
      "activations/layer9_attention_weight_max": 71.45003509521484,
      "activations/layer9_attention_weight_min": -61.911964416503906,
      "epoch": 0.83,
      "learning_rate": 0.00014610946969696967,
      "loss": 3.1633,
      "step": 14300
    },
    {
      "activations/layer0_attention_weight_max": 15.838127136230469,
      "activations/layer0_attention_weight_min": -10.476627349853516,
      "activations/layer10_attention_weight_max": 53.25468063354492,
      "activations/layer10_attention_weight_min": -44.359039306640625,
      "activations/layer11_attention_weight_max": 47.62443161010742,
      "activations/layer11_attention_weight_min": -38.24873352050781,
      "activations/layer12_attention_weight_max": 40.83314895629883,
      "activations/layer12_attention_weight_min": -29.820682525634766,
      "activations/layer13_attention_weight_max": 40.211830139160156,
      "activations/layer13_attention_weight_min": -37.48283767700195,
      "activations/layer14_attention_weight_max": 63.754150390625,
      "activations/layer14_attention_weight_min": -54.66358947753906,
      "activations/layer15_attention_weight_max": 60.84245300292969,
      "activations/layer15_attention_weight_min": -42.32722091674805,
      "activations/layer16_attention_weight_max": 47.604854583740234,
      "activations/layer16_attention_weight_min": -34.31369400024414,
      "activations/layer17_attention_weight_max": 34.11787033081055,
      "activations/layer17_attention_weight_min": -20.40894889831543,
      "activations/layer18_attention_weight_max": 20.98590850830078,
      "activations/layer18_attention_weight_min": -13.876230239868164,
      "activations/layer19_attention_weight_max": 47.14917755126953,
      "activations/layer19_attention_weight_min": -30.567251205444336,
      "activations/layer1_attention_weight_max": 15.646843910217285,
      "activations/layer1_attention_weight_min": -11.998801231384277,
      "activations/layer20_attention_weight_max": 27.229900360107422,
      "activations/layer20_attention_weight_min": -18.0364990234375,
      "activations/layer21_attention_weight_max": 18.72660255432129,
      "activations/layer21_attention_weight_min": -16.57949447631836,
      "activations/layer22_attention_weight_max": 29.879817962646484,
      "activations/layer22_attention_weight_min": -21.113313674926758,
      "activations/layer23_attention_weight_max": 23.276119232177734,
      "activations/layer23_attention_weight_min": -14.45391845703125,
      "activations/layer2_attention_weight_max": 26.23261070251465,
      "activations/layer2_attention_weight_min": -24.106794357299805,
      "activations/layer3_attention_weight_max": 55.34282302856445,
      "activations/layer3_attention_weight_min": -54.733829498291016,
      "activations/layer4_attention_weight_max": 105.3254165649414,
      "activations/layer4_attention_weight_min": -101.79027557373047,
      "activations/layer5_attention_weight_max": 119.25772094726562,
      "activations/layer5_attention_weight_min": -107.53914642333984,
      "activations/layer6_attention_weight_max": 66.4574203491211,
      "activations/layer6_attention_weight_min": -59.3390998840332,
      "activations/layer7_attention_weight_max": 68.00907897949219,
      "activations/layer7_attention_weight_min": -60.7386474609375,
      "activations/layer8_attention_weight_max": 58.42306137084961,
      "activations/layer8_attention_weight_min": -52.8694953918457,
      "activations/layer9_attention_weight_max": 94.37747192382812,
      "activations/layer9_attention_weight_min": -72.6978988647461,
      "epoch": 0.83,
      "learning_rate": 0.00014609053030303028,
      "loss": 3.1456,
      "step": 14350
    },
    {
      "activations/layer0_attention_weight_max": 15.261741638183594,
      "activations/layer0_attention_weight_min": -10.894936561584473,
      "activations/layer10_attention_weight_max": 57.82359313964844,
      "activations/layer10_attention_weight_min": -46.378170013427734,
      "activations/layer11_attention_weight_max": 47.64252853393555,
      "activations/layer11_attention_weight_min": -37.988929748535156,
      "activations/layer12_attention_weight_max": 77.99600219726562,
      "activations/layer12_attention_weight_min": -39.885658264160156,
      "activations/layer13_attention_weight_max": 75.6436538696289,
      "activations/layer13_attention_weight_min": -41.467979431152344,
      "activations/layer14_attention_weight_max": 86.64794921875,
      "activations/layer14_attention_weight_min": -50.43347930908203,
      "activations/layer15_attention_weight_max": 61.382930755615234,
      "activations/layer15_attention_weight_min": -33.993186950683594,
      "activations/layer16_attention_weight_max": 56.29591751098633,
      "activations/layer16_attention_weight_min": -33.553924560546875,
      "activations/layer17_attention_weight_max": 39.17082214355469,
      "activations/layer17_attention_weight_min": -21.585840225219727,
      "activations/layer18_attention_weight_max": 21.347238540649414,
      "activations/layer18_attention_weight_min": -14.197431564331055,
      "activations/layer19_attention_weight_max": 49.47933578491211,
      "activations/layer19_attention_weight_min": -31.007675170898438,
      "activations/layer1_attention_weight_max": 15.31948184967041,
      "activations/layer1_attention_weight_min": -12.204731941223145,
      "activations/layer20_attention_weight_max": 33.99529266357422,
      "activations/layer20_attention_weight_min": -18.952457427978516,
      "activations/layer21_attention_weight_max": 18.47097396850586,
      "activations/layer21_attention_weight_min": -11.994856834411621,
      "activations/layer22_attention_weight_max": 40.015228271484375,
      "activations/layer22_attention_weight_min": -20.352712631225586,
      "activations/layer23_attention_weight_max": 26.960914611816406,
      "activations/layer23_attention_weight_min": -17.55826187133789,
      "activations/layer2_attention_weight_max": 29.123891830444336,
      "activations/layer2_attention_weight_min": -26.07715606689453,
      "activations/layer3_attention_weight_max": 56.8360595703125,
      "activations/layer3_attention_weight_min": -56.46175003051758,
      "activations/layer4_attention_weight_max": 110.09980773925781,
      "activations/layer4_attention_weight_min": -107.37098693847656,
      "activations/layer5_attention_weight_max": 125.33675384521484,
      "activations/layer5_attention_weight_min": -109.51676940917969,
      "activations/layer6_attention_weight_max": 72.89625549316406,
      "activations/layer6_attention_weight_min": -64.22285461425781,
      "activations/layer7_attention_weight_max": 68.59562683105469,
      "activations/layer7_attention_weight_min": -59.283836364746094,
      "activations/layer8_attention_weight_max": 60.78721237182617,
      "activations/layer8_attention_weight_min": -52.28607177734375,
      "activations/layer9_attention_weight_max": 69.7099380493164,
      "activations/layer9_attention_weight_min": -63.85087203979492,
      "epoch": 0.84,
      "learning_rate": 0.0001460715909090909,
      "loss": 3.1355,
      "step": 14400
    },
    {
      "activations/layer0_attention_weight_max": 14.609841346740723,
      "activations/layer0_attention_weight_min": -10.930652618408203,
      "activations/layer10_attention_weight_max": 52.050498962402344,
      "activations/layer10_attention_weight_min": -46.133663177490234,
      "activations/layer11_attention_weight_max": 45.48517990112305,
      "activations/layer11_attention_weight_min": -39.411659240722656,
      "activations/layer12_attention_weight_max": 44.202144622802734,
      "activations/layer12_attention_weight_min": -35.733394622802734,
      "activations/layer13_attention_weight_max": 45.51177215576172,
      "activations/layer13_attention_weight_min": -36.6144905090332,
      "activations/layer14_attention_weight_max": 71.9368667602539,
      "activations/layer14_attention_weight_min": -57.44664764404297,
      "activations/layer15_attention_weight_max": 55.11798858642578,
      "activations/layer15_attention_weight_min": -39.6700553894043,
      "activations/layer16_attention_weight_max": 46.445579528808594,
      "activations/layer16_attention_weight_min": -33.99098587036133,
      "activations/layer17_attention_weight_max": 31.361406326293945,
      "activations/layer17_attention_weight_min": -22.99925994873047,
      "activations/layer18_attention_weight_max": 20.575223922729492,
      "activations/layer18_attention_weight_min": -12.943016052246094,
      "activations/layer19_attention_weight_max": 42.39345932006836,
      "activations/layer19_attention_weight_min": -26.76515769958496,
      "activations/layer1_attention_weight_max": 14.993215560913086,
      "activations/layer1_attention_weight_min": -13.039932250976562,
      "activations/layer20_attention_weight_max": 31.048738479614258,
      "activations/layer20_attention_weight_min": -18.196422576904297,
      "activations/layer21_attention_weight_max": 19.459800720214844,
      "activations/layer21_attention_weight_min": -14.983485221862793,
      "activations/layer22_attention_weight_max": 35.692832946777344,
      "activations/layer22_attention_weight_min": -20.04671859741211,
      "activations/layer23_attention_weight_max": 23.19215965270996,
      "activations/layer23_attention_weight_min": -13.699667930603027,
      "activations/layer2_attention_weight_max": 28.382308959960938,
      "activations/layer2_attention_weight_min": -26.843029022216797,
      "activations/layer3_attention_weight_max": 57.30313491821289,
      "activations/layer3_attention_weight_min": -57.244197845458984,
      "activations/layer4_attention_weight_max": 112.28022766113281,
      "activations/layer4_attention_weight_min": -105.910888671875,
      "activations/layer5_attention_weight_max": 122.36034393310547,
      "activations/layer5_attention_weight_min": -107.44148254394531,
      "activations/layer6_attention_weight_max": 67.74945831298828,
      "activations/layer6_attention_weight_min": -59.54461669921875,
      "activations/layer7_attention_weight_max": 63.9461669921875,
      "activations/layer7_attention_weight_min": -67.93936157226562,
      "activations/layer8_attention_weight_max": 65.65117645263672,
      "activations/layer8_attention_weight_min": -54.45732116699219,
      "activations/layer9_attention_weight_max": 67.77730560302734,
      "activations/layer9_attention_weight_min": -62.26760482788086,
      "epoch": 0.84,
      "learning_rate": 0.0001460526515151515,
      "loss": 3.144,
      "step": 14450
    },
    {
      "activations/layer0_attention_weight_max": 13.532527923583984,
      "activations/layer0_attention_weight_min": -11.118401527404785,
      "activations/layer10_attention_weight_max": 54.84920120239258,
      "activations/layer10_attention_weight_min": -46.618797302246094,
      "activations/layer11_attention_weight_max": 45.237327575683594,
      "activations/layer11_attention_weight_min": -43.62037658691406,
      "activations/layer12_attention_weight_max": 38.24995803833008,
      "activations/layer12_attention_weight_min": -27.222997665405273,
      "activations/layer13_attention_weight_max": 45.48042297363281,
      "activations/layer13_attention_weight_min": -34.91448211669922,
      "activations/layer14_attention_weight_max": 71.6501235961914,
      "activations/layer14_attention_weight_min": -49.95744705200195,
      "activations/layer15_attention_weight_max": 52.84736633300781,
      "activations/layer15_attention_weight_min": -40.24751663208008,
      "activations/layer16_attention_weight_max": 51.88153076171875,
      "activations/layer16_attention_weight_min": -36.93442153930664,
      "activations/layer17_attention_weight_max": 40.58310317993164,
      "activations/layer17_attention_weight_min": -20.650362014770508,
      "activations/layer18_attention_weight_max": 18.22219467163086,
      "activations/layer18_attention_weight_min": -13.059892654418945,
      "activations/layer19_attention_weight_max": 48.74014663696289,
      "activations/layer19_attention_weight_min": -31.01056480407715,
      "activations/layer1_attention_weight_max": 15.520166397094727,
      "activations/layer1_attention_weight_min": -11.199455261230469,
      "activations/layer20_attention_weight_max": 32.27125930786133,
      "activations/layer20_attention_weight_min": -18.8551025390625,
      "activations/layer21_attention_weight_max": 22.47450828552246,
      "activations/layer21_attention_weight_min": -14.586321830749512,
      "activations/layer22_attention_weight_max": 35.14335632324219,
      "activations/layer22_attention_weight_min": -21.514312744140625,
      "activations/layer23_attention_weight_max": 25.34395408630371,
      "activations/layer23_attention_weight_min": -14.782206535339355,
      "activations/layer2_attention_weight_max": 27.88937759399414,
      "activations/layer2_attention_weight_min": -24.263914108276367,
      "activations/layer3_attention_weight_max": 58.28527069091797,
      "activations/layer3_attention_weight_min": -59.153987884521484,
      "activations/layer4_attention_weight_max": 109.19502258300781,
      "activations/layer4_attention_weight_min": -105.38844299316406,
      "activations/layer5_attention_weight_max": 127.40179443359375,
      "activations/layer5_attention_weight_min": -112.19914245605469,
      "activations/layer6_attention_weight_max": 70.20459747314453,
      "activations/layer6_attention_weight_min": -60.737876892089844,
      "activations/layer7_attention_weight_max": 67.95355987548828,
      "activations/layer7_attention_weight_min": -63.933937072753906,
      "activations/layer8_attention_weight_max": 63.4393310546875,
      "activations/layer8_attention_weight_min": -56.85748291015625,
      "activations/layer9_attention_weight_max": 71.65853118896484,
      "activations/layer9_attention_weight_min": -63.778785705566406,
      "epoch": 0.84,
      "learning_rate": 0.0001460340909090909,
      "loss": 3.1609,
      "step": 14500
    },
    {
      "activations/layer0_attention_weight_max": 14.246153831481934,
      "activations/layer0_attention_weight_min": -10.99353313446045,
      "activations/layer10_attention_weight_max": 58.49018096923828,
      "activations/layer10_attention_weight_min": -45.73982238769531,
      "activations/layer11_attention_weight_max": 51.94580078125,
      "activations/layer11_attention_weight_min": -39.51782989501953,
      "activations/layer12_attention_weight_max": 38.670257568359375,
      "activations/layer12_attention_weight_min": -29.704954147338867,
      "activations/layer13_attention_weight_max": 41.06772994995117,
      "activations/layer13_attention_weight_min": -32.15850830078125,
      "activations/layer14_attention_weight_max": 69.99136352539062,
      "activations/layer14_attention_weight_min": -47.83121871948242,
      "activations/layer15_attention_weight_max": 46.13116455078125,
      "activations/layer15_attention_weight_min": -37.950096130371094,
      "activations/layer16_attention_weight_max": 51.76132583618164,
      "activations/layer16_attention_weight_min": -35.45639419555664,
      "activations/layer17_attention_weight_max": 30.48232650756836,
      "activations/layer17_attention_weight_min": -21.75593376159668,
      "activations/layer18_attention_weight_max": 16.826929092407227,
      "activations/layer18_attention_weight_min": -12.786614418029785,
      "activations/layer19_attention_weight_max": 43.13941955566406,
      "activations/layer19_attention_weight_min": -31.783950805664062,
      "activations/layer1_attention_weight_max": 15.854559898376465,
      "activations/layer1_attention_weight_min": -11.659472465515137,
      "activations/layer20_attention_weight_max": 29.685638427734375,
      "activations/layer20_attention_weight_min": -20.635480880737305,
      "activations/layer21_attention_weight_max": 17.35977554321289,
      "activations/layer21_attention_weight_min": -15.336094856262207,
      "activations/layer22_attention_weight_max": 31.259336471557617,
      "activations/layer22_attention_weight_min": -18.410268783569336,
      "activations/layer23_attention_weight_max": 19.83391571044922,
      "activations/layer23_attention_weight_min": -13.839807510375977,
      "activations/layer2_attention_weight_max": 26.094371795654297,
      "activations/layer2_attention_weight_min": -25.265806198120117,
      "activations/layer3_attention_weight_max": 59.321754455566406,
      "activations/layer3_attention_weight_min": -62.373443603515625,
      "activations/layer4_attention_weight_max": 113.41251373291016,
      "activations/layer4_attention_weight_min": -104.55144500732422,
      "activations/layer5_attention_weight_max": 124.68087768554688,
      "activations/layer5_attention_weight_min": -112.5422592163086,
      "activations/layer6_attention_weight_max": 66.57919311523438,
      "activations/layer6_attention_weight_min": -61.14567947387695,
      "activations/layer7_attention_weight_max": 63.4504280090332,
      "activations/layer7_attention_weight_min": -61.35993957519531,
      "activations/layer8_attention_weight_max": 62.05615997314453,
      "activations/layer8_attention_weight_min": -56.002506256103516,
      "activations/layer9_attention_weight_max": 82.78583526611328,
      "activations/layer9_attention_weight_min": -69.56732177734375,
      "epoch": 0.85,
      "learning_rate": 0.0001460151515151515,
      "loss": 3.1339,
      "step": 14550
    },
    {
      "activations/layer0_attention_weight_max": 15.143024444580078,
      "activations/layer0_attention_weight_min": -11.354201316833496,
      "activations/layer10_attention_weight_max": 49.0176887512207,
      "activations/layer10_attention_weight_min": -42.82273864746094,
      "activations/layer11_attention_weight_max": 44.53803253173828,
      "activations/layer11_attention_weight_min": -36.32124328613281,
      "activations/layer12_attention_weight_max": 33.24318313598633,
      "activations/layer12_attention_weight_min": -30.03809928894043,
      "activations/layer13_attention_weight_max": 48.04914093017578,
      "activations/layer13_attention_weight_min": -34.35960388183594,
      "activations/layer14_attention_weight_max": 70.7042236328125,
      "activations/layer14_attention_weight_min": -47.98213577270508,
      "activations/layer15_attention_weight_max": 59.16769027709961,
      "activations/layer15_attention_weight_min": -38.544044494628906,
      "activations/layer16_attention_weight_max": 50.9289436340332,
      "activations/layer16_attention_weight_min": -33.7815055847168,
      "activations/layer17_attention_weight_max": 36.788700103759766,
      "activations/layer17_attention_weight_min": -22.94194984436035,
      "activations/layer18_attention_weight_max": 20.39183235168457,
      "activations/layer18_attention_weight_min": -14.959257125854492,
      "activations/layer19_attention_weight_max": 39.911659240722656,
      "activations/layer19_attention_weight_min": -27.056055068969727,
      "activations/layer1_attention_weight_max": 15.743410110473633,
      "activations/layer1_attention_weight_min": -11.909974098205566,
      "activations/layer20_attention_weight_max": 33.91083526611328,
      "activations/layer20_attention_weight_min": -19.173282623291016,
      "activations/layer21_attention_weight_max": 19.032426834106445,
      "activations/layer21_attention_weight_min": -14.13636302947998,
      "activations/layer22_attention_weight_max": 32.56440734863281,
      "activations/layer22_attention_weight_min": -21.46326446533203,
      "activations/layer23_attention_weight_max": 19.75311851501465,
      "activations/layer23_attention_weight_min": -15.134349822998047,
      "activations/layer2_attention_weight_max": 28.296085357666016,
      "activations/layer2_attention_weight_min": -26.669822692871094,
      "activations/layer3_attention_weight_max": 57.548187255859375,
      "activations/layer3_attention_weight_min": -59.898292541503906,
      "activations/layer4_attention_weight_max": 107.27391815185547,
      "activations/layer4_attention_weight_min": -106.07527923583984,
      "activations/layer5_attention_weight_max": 118.87208557128906,
      "activations/layer5_attention_weight_min": -110.61740112304688,
      "activations/layer6_attention_weight_max": 64.00486755371094,
      "activations/layer6_attention_weight_min": -59.81412887573242,
      "activations/layer7_attention_weight_max": 62.1051139831543,
      "activations/layer7_attention_weight_min": -61.49031448364258,
      "activations/layer8_attention_weight_max": 58.53135299682617,
      "activations/layer8_attention_weight_min": -53.76103210449219,
      "activations/layer9_attention_weight_max": 68.43513488769531,
      "activations/layer9_attention_weight_min": -59.865020751953125,
      "epoch": 0.85,
      "learning_rate": 0.0001459962121212121,
      "loss": 3.1511,
      "step": 14600
    },
    {
      "activations/layer0_attention_weight_max": 16.533397674560547,
      "activations/layer0_attention_weight_min": -11.307408332824707,
      "activations/layer10_attention_weight_max": 54.10382843017578,
      "activations/layer10_attention_weight_min": -49.28042984008789,
      "activations/layer11_attention_weight_max": 43.49659729003906,
      "activations/layer11_attention_weight_min": -43.648956298828125,
      "activations/layer12_attention_weight_max": 36.72262191772461,
      "activations/layer12_attention_weight_min": -30.634845733642578,
      "activations/layer13_attention_weight_max": 42.47040557861328,
      "activations/layer13_attention_weight_min": -36.204307556152344,
      "activations/layer14_attention_weight_max": 75.53380584716797,
      "activations/layer14_attention_weight_min": -58.41292953491211,
      "activations/layer15_attention_weight_max": 53.16987991333008,
      "activations/layer15_attention_weight_min": -42.37149429321289,
      "activations/layer16_attention_weight_max": 53.097957611083984,
      "activations/layer16_attention_weight_min": -34.81391143798828,
      "activations/layer17_attention_weight_max": 34.552791595458984,
      "activations/layer17_attention_weight_min": -23.293195724487305,
      "activations/layer18_attention_weight_max": 23.399593353271484,
      "activations/layer18_attention_weight_min": -13.8776273727417,
      "activations/layer19_attention_weight_max": 42.85909652709961,
      "activations/layer19_attention_weight_min": -33.8531608581543,
      "activations/layer1_attention_weight_max": 16.538328170776367,
      "activations/layer1_attention_weight_min": -11.680603981018066,
      "activations/layer20_attention_weight_max": 33.59844207763672,
      "activations/layer20_attention_weight_min": -20.765369415283203,
      "activations/layer21_attention_weight_max": 20.35053825378418,
      "activations/layer21_attention_weight_min": -12.86218547821045,
      "activations/layer22_attention_weight_max": 33.5858039855957,
      "activations/layer22_attention_weight_min": -19.474891662597656,
      "activations/layer23_attention_weight_max": 20.372516632080078,
      "activations/layer23_attention_weight_min": -11.737441062927246,
      "activations/layer2_attention_weight_max": 27.07207489013672,
      "activations/layer2_attention_weight_min": -25.765155792236328,
      "activations/layer3_attention_weight_max": 55.74654769897461,
      "activations/layer3_attention_weight_min": -55.76294708251953,
      "activations/layer4_attention_weight_max": 107.24493408203125,
      "activations/layer4_attention_weight_min": -99.11659240722656,
      "activations/layer5_attention_weight_max": 121.10906219482422,
      "activations/layer5_attention_weight_min": -109.48323059082031,
      "activations/layer6_attention_weight_max": 67.50143432617188,
      "activations/layer6_attention_weight_min": -61.842742919921875,
      "activations/layer7_attention_weight_max": 66.67988586425781,
      "activations/layer7_attention_weight_min": -63.42685317993164,
      "activations/layer8_attention_weight_max": 64.24749755859375,
      "activations/layer8_attention_weight_min": -59.26759338378906,
      "activations/layer9_attention_weight_max": 73.0640869140625,
      "activations/layer9_attention_weight_min": -63.860260009765625,
      "epoch": 0.85,
      "learning_rate": 0.00014597727272727272,
      "loss": 3.1448,
      "step": 14650
    },
    {
      "activations/layer0_attention_weight_max": 15.346107482910156,
      "activations/layer0_attention_weight_min": -11.761109352111816,
      "activations/layer10_attention_weight_max": 51.457122802734375,
      "activations/layer10_attention_weight_min": -46.12166213989258,
      "activations/layer11_attention_weight_max": 51.48890686035156,
      "activations/layer11_attention_weight_min": -39.66424560546875,
      "activations/layer12_attention_weight_max": 36.34086990356445,
      "activations/layer12_attention_weight_min": -26.207704544067383,
      "activations/layer13_attention_weight_max": 41.72514724731445,
      "activations/layer13_attention_weight_min": -33.95121765136719,
      "activations/layer14_attention_weight_max": 68.31975555419922,
      "activations/layer14_attention_weight_min": -50.93525314331055,
      "activations/layer15_attention_weight_max": 53.709228515625,
      "activations/layer15_attention_weight_min": -35.9984130859375,
      "activations/layer16_attention_weight_max": 55.90291976928711,
      "activations/layer16_attention_weight_min": -36.42764663696289,
      "activations/layer17_attention_weight_max": 37.66813278198242,
      "activations/layer17_attention_weight_min": -23.580087661743164,
      "activations/layer18_attention_weight_max": 22.66949462890625,
      "activations/layer18_attention_weight_min": -12.500370979309082,
      "activations/layer19_attention_weight_max": 49.89399337768555,
      "activations/layer19_attention_weight_min": -28.561649322509766,
      "activations/layer1_attention_weight_max": 17.683767318725586,
      "activations/layer1_attention_weight_min": -11.695401191711426,
      "activations/layer20_attention_weight_max": 35.53396987915039,
      "activations/layer20_attention_weight_min": -18.902015686035156,
      "activations/layer21_attention_weight_max": 19.70561981201172,
      "activations/layer21_attention_weight_min": -12.908485412597656,
      "activations/layer22_attention_weight_max": 38.686607360839844,
      "activations/layer22_attention_weight_min": -18.069791793823242,
      "activations/layer23_attention_weight_max": 24.73273468017578,
      "activations/layer23_attention_weight_min": -14.31047248840332,
      "activations/layer2_attention_weight_max": 28.005016326904297,
      "activations/layer2_attention_weight_min": -25.495479583740234,
      "activations/layer3_attention_weight_max": 57.00674057006836,
      "activations/layer3_attention_weight_min": -60.125518798828125,
      "activations/layer4_attention_weight_max": 108.57628631591797,
      "activations/layer4_attention_weight_min": -107.24718475341797,
      "activations/layer5_attention_weight_max": 123.20226287841797,
      "activations/layer5_attention_weight_min": -113.71757507324219,
      "activations/layer6_attention_weight_max": 70.57476043701172,
      "activations/layer6_attention_weight_min": -60.46229553222656,
      "activations/layer7_attention_weight_max": 70.5993881225586,
      "activations/layer7_attention_weight_min": -66.1125259399414,
      "activations/layer8_attention_weight_max": 70.61289978027344,
      "activations/layer8_attention_weight_min": -57.43061065673828,
      "activations/layer9_attention_weight_max": 77.61345672607422,
      "activations/layer9_attention_weight_min": -70.0487289428711,
      "epoch": 0.85,
      "learning_rate": 0.00014595833333333333,
      "loss": 3.1331,
      "step": 14700
    },
    {
      "activations/layer0_attention_weight_max": 12.935306549072266,
      "activations/layer0_attention_weight_min": -11.037091255187988,
      "activations/layer10_attention_weight_max": 52.16553497314453,
      "activations/layer10_attention_weight_min": -46.56901931762695,
      "activations/layer11_attention_weight_max": 43.84940719604492,
      "activations/layer11_attention_weight_min": -39.933502197265625,
      "activations/layer12_attention_weight_max": 45.4627571105957,
      "activations/layer12_attention_weight_min": -43.326263427734375,
      "activations/layer13_attention_weight_max": 45.2076530456543,
      "activations/layer13_attention_weight_min": -42.74769973754883,
      "activations/layer14_attention_weight_max": 68.53864288330078,
      "activations/layer14_attention_weight_min": -53.14791488647461,
      "activations/layer15_attention_weight_max": 53.78255844116211,
      "activations/layer15_attention_weight_min": -41.08858871459961,
      "activations/layer16_attention_weight_max": 52.805362701416016,
      "activations/layer16_attention_weight_min": -37.740867614746094,
      "activations/layer17_attention_weight_max": 36.739620208740234,
      "activations/layer17_attention_weight_min": -20.223651885986328,
      "activations/layer18_attention_weight_max": 21.84128189086914,
      "activations/layer18_attention_weight_min": -13.731924057006836,
      "activations/layer19_attention_weight_max": 48.54801940917969,
      "activations/layer19_attention_weight_min": -26.243759155273438,
      "activations/layer1_attention_weight_max": 14.891712188720703,
      "activations/layer1_attention_weight_min": -12.389232635498047,
      "activations/layer20_attention_weight_max": 35.6422233581543,
      "activations/layer20_attention_weight_min": -20.8660831451416,
      "activations/layer21_attention_weight_max": 19.593124389648438,
      "activations/layer21_attention_weight_min": -13.202522277832031,
      "activations/layer22_attention_weight_max": 45.177894592285156,
      "activations/layer22_attention_weight_min": -18.973020553588867,
      "activations/layer23_attention_weight_max": 28.112979888916016,
      "activations/layer23_attention_weight_min": -15.584894180297852,
      "activations/layer2_attention_weight_max": 28.138593673706055,
      "activations/layer2_attention_weight_min": -25.434873580932617,
      "activations/layer3_attention_weight_max": 56.428096771240234,
      "activations/layer3_attention_weight_min": -58.62628936767578,
      "activations/layer4_attention_weight_max": 107.37623596191406,
      "activations/layer4_attention_weight_min": -104.3002700805664,
      "activations/layer5_attention_weight_max": 128.1943359375,
      "activations/layer5_attention_weight_min": -111.65570068359375,
      "activations/layer6_attention_weight_max": 68.35478210449219,
      "activations/layer6_attention_weight_min": -60.78215789794922,
      "activations/layer7_attention_weight_max": 63.3575325012207,
      "activations/layer7_attention_weight_min": -63.70754623413086,
      "activations/layer8_attention_weight_max": 63.16897964477539,
      "activations/layer8_attention_weight_min": -51.04875183105469,
      "activations/layer9_attention_weight_max": 69.70861053466797,
      "activations/layer9_attention_weight_min": -60.52632522583008,
      "epoch": 0.86,
      "learning_rate": 0.00014593939393939392,
      "loss": 3.1496,
      "step": 14750
    },
    {
      "activations/layer0_attention_weight_max": 13.486327171325684,
      "activations/layer0_attention_weight_min": -11.811835289001465,
      "activations/layer10_attention_weight_max": 59.61262130737305,
      "activations/layer10_attention_weight_min": -49.20432662963867,
      "activations/layer11_attention_weight_max": 48.40513610839844,
      "activations/layer11_attention_weight_min": -36.436180114746094,
      "activations/layer12_attention_weight_max": 39.5899772644043,
      "activations/layer12_attention_weight_min": -34.857086181640625,
      "activations/layer13_attention_weight_max": 48.039466857910156,
      "activations/layer13_attention_weight_min": -33.82029342651367,
      "activations/layer14_attention_weight_max": 72.91718292236328,
      "activations/layer14_attention_weight_min": -54.32130813598633,
      "activations/layer15_attention_weight_max": 51.6231575012207,
      "activations/layer15_attention_weight_min": -37.193965911865234,
      "activations/layer16_attention_weight_max": 58.205474853515625,
      "activations/layer16_attention_weight_min": -37.88426208496094,
      "activations/layer17_attention_weight_max": 33.0306510925293,
      "activations/layer17_attention_weight_min": -21.769617080688477,
      "activations/layer18_attention_weight_max": 22.780471801757812,
      "activations/layer18_attention_weight_min": -14.430699348449707,
      "activations/layer19_attention_weight_max": 60.421016693115234,
      "activations/layer19_attention_weight_min": -30.585718154907227,
      "activations/layer1_attention_weight_max": 15.720074653625488,
      "activations/layer1_attention_weight_min": -12.044963836669922,
      "activations/layer20_attention_weight_max": 36.935787200927734,
      "activations/layer20_attention_weight_min": -22.008739471435547,
      "activations/layer21_attention_weight_max": 22.285001754760742,
      "activations/layer21_attention_weight_min": -12.72545051574707,
      "activations/layer22_attention_weight_max": 43.858856201171875,
      "activations/layer22_attention_weight_min": -20.310571670532227,
      "activations/layer23_attention_weight_max": 29.10242462158203,
      "activations/layer23_attention_weight_min": -15.149352073669434,
      "activations/layer2_attention_weight_max": 28.27227020263672,
      "activations/layer2_attention_weight_min": -25.697729110717773,
      "activations/layer3_attention_weight_max": 57.13432693481445,
      "activations/layer3_attention_weight_min": -60.40679168701172,
      "activations/layer4_attention_weight_max": 109.05655670166016,
      "activations/layer4_attention_weight_min": -106.36400604248047,
      "activations/layer5_attention_weight_max": 130.9791259765625,
      "activations/layer5_attention_weight_min": -115.63365936279297,
      "activations/layer6_attention_weight_max": 72.90441131591797,
      "activations/layer6_attention_weight_min": -61.86083984375,
      "activations/layer7_attention_weight_max": 71.98706817626953,
      "activations/layer7_attention_weight_min": -63.98905944824219,
      "activations/layer8_attention_weight_max": 69.41496276855469,
      "activations/layer8_attention_weight_min": -54.7552604675293,
      "activations/layer9_attention_weight_max": 77.13756561279297,
      "activations/layer9_attention_weight_min": -62.50676345825195,
      "epoch": 0.86,
      "learning_rate": 0.00014592045454545454,
      "loss": 3.1418,
      "step": 14800
    },
    {
      "activations/layer0_attention_weight_max": 15.414676666259766,
      "activations/layer0_attention_weight_min": -11.44366455078125,
      "activations/layer10_attention_weight_max": 56.749046325683594,
      "activations/layer10_attention_weight_min": -49.09396743774414,
      "activations/layer11_attention_weight_max": 43.650062561035156,
      "activations/layer11_attention_weight_min": -39.53947448730469,
      "activations/layer12_attention_weight_max": 36.11600875854492,
      "activations/layer12_attention_weight_min": -29.25934410095215,
      "activations/layer13_attention_weight_max": 48.09577941894531,
      "activations/layer13_attention_weight_min": -33.713890075683594,
      "activations/layer14_attention_weight_max": 77.97262573242188,
      "activations/layer14_attention_weight_min": -52.162410736083984,
      "activations/layer15_attention_weight_max": 55.03255844116211,
      "activations/layer15_attention_weight_min": -38.42628479003906,
      "activations/layer16_attention_weight_max": 48.851524353027344,
      "activations/layer16_attention_weight_min": -33.54316711425781,
      "activations/layer17_attention_weight_max": 35.3662223815918,
      "activations/layer17_attention_weight_min": -22.726839065551758,
      "activations/layer18_attention_weight_max": 21.241716384887695,
      "activations/layer18_attention_weight_min": -12.972455978393555,
      "activations/layer19_attention_weight_max": 50.82142639160156,
      "activations/layer19_attention_weight_min": -29.35249137878418,
      "activations/layer1_attention_weight_max": 15.148043632507324,
      "activations/layer1_attention_weight_min": -11.841019630432129,
      "activations/layer20_attention_weight_max": 36.49515151977539,
      "activations/layer20_attention_weight_min": -20.369565963745117,
      "activations/layer21_attention_weight_max": 19.053096771240234,
      "activations/layer21_attention_weight_min": -13.662623405456543,
      "activations/layer22_attention_weight_max": 39.62441635131836,
      "activations/layer22_attention_weight_min": -19.111406326293945,
      "activations/layer23_attention_weight_max": 25.566335678100586,
      "activations/layer23_attention_weight_min": -14.74170207977295,
      "activations/layer2_attention_weight_max": 27.67003631591797,
      "activations/layer2_attention_weight_min": -26.200965881347656,
      "activations/layer3_attention_weight_max": 58.76813507080078,
      "activations/layer3_attention_weight_min": -57.57735061645508,
      "activations/layer4_attention_weight_max": 108.79920959472656,
      "activations/layer4_attention_weight_min": -102.4651107788086,
      "activations/layer5_attention_weight_max": 121.39874267578125,
      "activations/layer5_attention_weight_min": -107.37508392333984,
      "activations/layer6_attention_weight_max": 67.64344024658203,
      "activations/layer6_attention_weight_min": -60.029052734375,
      "activations/layer7_attention_weight_max": 74.31135559082031,
      "activations/layer7_attention_weight_min": -64.23582458496094,
      "activations/layer8_attention_weight_max": 59.11288070678711,
      "activations/layer8_attention_weight_min": -50.290794372558594,
      "activations/layer9_attention_weight_max": 83.57459259033203,
      "activations/layer9_attention_weight_min": -57.34144973754883,
      "epoch": 0.86,
      "learning_rate": 0.00014590151515151515,
      "loss": 3.1394,
      "step": 14850
    },
    {
      "activations/layer0_attention_weight_max": 15.338611602783203,
      "activations/layer0_attention_weight_min": -10.854302406311035,
      "activations/layer10_attention_weight_max": 61.27134323120117,
      "activations/layer10_attention_weight_min": -48.962913513183594,
      "activations/layer11_attention_weight_max": 48.886043548583984,
      "activations/layer11_attention_weight_min": -39.66014862060547,
      "activations/layer12_attention_weight_max": 41.78490447998047,
      "activations/layer12_attention_weight_min": -27.9257755279541,
      "activations/layer13_attention_weight_max": 44.751285552978516,
      "activations/layer13_attention_weight_min": -32.202693939208984,
      "activations/layer14_attention_weight_max": 73.99012756347656,
      "activations/layer14_attention_weight_min": -50.661746978759766,
      "activations/layer15_attention_weight_max": 52.031673431396484,
      "activations/layer15_attention_weight_min": -37.4251708984375,
      "activations/layer16_attention_weight_max": 54.5732307434082,
      "activations/layer16_attention_weight_min": -38.51486587524414,
      "activations/layer17_attention_weight_max": 31.221168518066406,
      "activations/layer17_attention_weight_min": -20.864910125732422,
      "activations/layer18_attention_weight_max": 20.25685691833496,
      "activations/layer18_attention_weight_min": -13.148188591003418,
      "activations/layer19_attention_weight_max": 47.479366302490234,
      "activations/layer19_attention_weight_min": -30.939971923828125,
      "activations/layer1_attention_weight_max": 15.941367149353027,
      "activations/layer1_attention_weight_min": -11.929498672485352,
      "activations/layer20_attention_weight_max": 36.93003845214844,
      "activations/layer20_attention_weight_min": -21.283206939697266,
      "activations/layer21_attention_weight_max": 21.184326171875,
      "activations/layer21_attention_weight_min": -16.47087287902832,
      "activations/layer22_attention_weight_max": 33.333213806152344,
      "activations/layer22_attention_weight_min": -17.40169334411621,
      "activations/layer23_attention_weight_max": 20.758056640625,
      "activations/layer23_attention_weight_min": -14.221595764160156,
      "activations/layer2_attention_weight_max": 27.677791595458984,
      "activations/layer2_attention_weight_min": -25.197010040283203,
      "activations/layer3_attention_weight_max": 55.68500518798828,
      "activations/layer3_attention_weight_min": -57.12580871582031,
      "activations/layer4_attention_weight_max": 108.55623626708984,
      "activations/layer4_attention_weight_min": -102.66535949707031,
      "activations/layer5_attention_weight_max": 126.93266296386719,
      "activations/layer5_attention_weight_min": -108.96226501464844,
      "activations/layer6_attention_weight_max": 71.51677703857422,
      "activations/layer6_attention_weight_min": -61.725242614746094,
      "activations/layer7_attention_weight_max": 65.74457550048828,
      "activations/layer7_attention_weight_min": -61.50288772583008,
      "activations/layer8_attention_weight_max": 69.64348602294922,
      "activations/layer8_attention_weight_min": -55.73478698730469,
      "activations/layer9_attention_weight_max": 75.12567901611328,
      "activations/layer9_attention_weight_min": -62.68034744262695,
      "epoch": 0.87,
      "learning_rate": 0.00014588257575757574,
      "loss": 3.1354,
      "step": 14900
    },
    {
      "activations/layer0_attention_weight_max": 13.967582702636719,
      "activations/layer0_attention_weight_min": -11.354227066040039,
      "activations/layer10_attention_weight_max": 52.49106216430664,
      "activations/layer10_attention_weight_min": -46.48744583129883,
      "activations/layer11_attention_weight_max": 48.05009460449219,
      "activations/layer11_attention_weight_min": -42.5413932800293,
      "activations/layer12_attention_weight_max": 37.644325256347656,
      "activations/layer12_attention_weight_min": -29.818370819091797,
      "activations/layer13_attention_weight_max": 40.0464973449707,
      "activations/layer13_attention_weight_min": -36.218746185302734,
      "activations/layer14_attention_weight_max": 65.3013916015625,
      "activations/layer14_attention_weight_min": -51.801998138427734,
      "activations/layer15_attention_weight_max": 46.34346008300781,
      "activations/layer15_attention_weight_min": -35.92061233520508,
      "activations/layer16_attention_weight_max": 46.7352294921875,
      "activations/layer16_attention_weight_min": -35.924217224121094,
      "activations/layer17_attention_weight_max": 33.56901931762695,
      "activations/layer17_attention_weight_min": -21.133333206176758,
      "activations/layer18_attention_weight_max": 19.41683578491211,
      "activations/layer18_attention_weight_min": -12.807186126708984,
      "activations/layer19_attention_weight_max": 46.42430114746094,
      "activations/layer19_attention_weight_min": -26.953107833862305,
      "activations/layer1_attention_weight_max": 15.777371406555176,
      "activations/layer1_attention_weight_min": -11.875086784362793,
      "activations/layer20_attention_weight_max": 30.956968307495117,
      "activations/layer20_attention_weight_min": -19.031131744384766,
      "activations/layer21_attention_weight_max": 17.26002311706543,
      "activations/layer21_attention_weight_min": -15.459693908691406,
      "activations/layer22_attention_weight_max": 38.415977478027344,
      "activations/layer22_attention_weight_min": -21.006135940551758,
      "activations/layer23_attention_weight_max": 26.43524932861328,
      "activations/layer23_attention_weight_min": -15.420555114746094,
      "activations/layer2_attention_weight_max": 30.428150177001953,
      "activations/layer2_attention_weight_min": -26.568984985351562,
      "activations/layer3_attention_weight_max": 57.49566650390625,
      "activations/layer3_attention_weight_min": -57.43901062011719,
      "activations/layer4_attention_weight_max": 111.84734344482422,
      "activations/layer4_attention_weight_min": -100.27149200439453,
      "activations/layer5_attention_weight_max": 129.6061553955078,
      "activations/layer5_attention_weight_min": -110.49856567382812,
      "activations/layer6_attention_weight_max": 69.71714782714844,
      "activations/layer6_attention_weight_min": -61.78662872314453,
      "activations/layer7_attention_weight_max": 65.4898910522461,
      "activations/layer7_attention_weight_min": -62.334312438964844,
      "activations/layer8_attention_weight_max": 62.77542495727539,
      "activations/layer8_attention_weight_min": -52.8593635559082,
      "activations/layer9_attention_weight_max": 74.33304595947266,
      "activations/layer9_attention_weight_min": -59.73472213745117,
      "epoch": 0.87,
      "learning_rate": 0.00014586363636363636,
      "loss": 3.1315,
      "step": 14950
    },
    {
      "activations/layer0_attention_weight_max": 15.25407886505127,
      "activations/layer0_attention_weight_min": -10.950154304504395,
      "activations/layer10_attention_weight_max": 51.80244827270508,
      "activations/layer10_attention_weight_min": -46.48272705078125,
      "activations/layer11_attention_weight_max": 44.06658172607422,
      "activations/layer11_attention_weight_min": -38.91640853881836,
      "activations/layer12_attention_weight_max": 37.31336975097656,
      "activations/layer12_attention_weight_min": -27.4478702545166,
      "activations/layer13_attention_weight_max": 48.26058578491211,
      "activations/layer13_attention_weight_min": -31.61713409423828,
      "activations/layer14_attention_weight_max": 65.20668029785156,
      "activations/layer14_attention_weight_min": -51.08103942871094,
      "activations/layer15_attention_weight_max": 54.44200897216797,
      "activations/layer15_attention_weight_min": -41.07196044921875,
      "activations/layer16_attention_weight_max": 48.1682014465332,
      "activations/layer16_attention_weight_min": -34.601173400878906,
      "activations/layer17_attention_weight_max": 32.83132553100586,
      "activations/layer17_attention_weight_min": -23.068710327148438,
      "activations/layer18_attention_weight_max": 20.353410720825195,
      "activations/layer18_attention_weight_min": -13.135526657104492,
      "activations/layer19_attention_weight_max": 42.03629684448242,
      "activations/layer19_attention_weight_min": -30.633930206298828,
      "activations/layer1_attention_weight_max": 16.260242462158203,
      "activations/layer1_attention_weight_min": -10.992159843444824,
      "activations/layer20_attention_weight_max": 32.05942916870117,
      "activations/layer20_attention_weight_min": -19.90353012084961,
      "activations/layer21_attention_weight_max": 18.12552833557129,
      "activations/layer21_attention_weight_min": -13.829597473144531,
      "activations/layer22_attention_weight_max": 36.82630920410156,
      "activations/layer22_attention_weight_min": -20.57228660583496,
      "activations/layer23_attention_weight_max": 21.280982971191406,
      "activations/layer23_attention_weight_min": -15.947065353393555,
      "activations/layer2_attention_weight_max": 26.705236434936523,
      "activations/layer2_attention_weight_min": -24.82270622253418,
      "activations/layer3_attention_weight_max": 55.56175231933594,
      "activations/layer3_attention_weight_min": -57.43553161621094,
      "activations/layer4_attention_weight_max": 103.5323715209961,
      "activations/layer4_attention_weight_min": -102.11328125,
      "activations/layer5_attention_weight_max": 114.42330932617188,
      "activations/layer5_attention_weight_min": -107.28836059570312,
      "activations/layer6_attention_weight_max": 66.87930297851562,
      "activations/layer6_attention_weight_min": -59.87693405151367,
      "activations/layer7_attention_weight_max": 63.609622955322266,
      "activations/layer7_attention_weight_min": -66.52599334716797,
      "activations/layer8_attention_weight_max": 60.55747604370117,
      "activations/layer8_attention_weight_min": -55.424102783203125,
      "activations/layer9_attention_weight_max": 70.55269622802734,
      "activations/layer9_attention_weight_min": -59.93862533569336,
      "epoch": 0.87,
      "learning_rate": 0.00014584469696969697,
      "loss": 3.1436,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_loss": 3.080078125,
      "eval_runtime": 8.4722,
      "eval_samples_per_second": 506.835,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_openwebtext_loss": 3.080078125,
      "eval_openwebtext_ppl": 21.760102337787384,
      "eval_openwebtext_runtime": 8.4722,
      "eval_openwebtext_samples_per_second": 506.835,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_wikitext_loss": 3.36328125,
      "eval_wikitext_ppl": 28.883810562338418,
      "eval_wikitext_runtime": 1.9702,
      "eval_wikitext_samples_per_second": 231.448,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_lambada_loss": 3.607421875,
      "eval_lambada_ppl": 36.87087245606033,
      "eval_lambada_runtime": 9.5243,
      "eval_lambada_samples_per_second": 511.221,
      "step": 15000
    },
    {
      "activations/layer0_attention_weight_max": 15.19986343383789,
      "activations/layer0_attention_weight_min": -11.375760078430176,
      "activations/layer10_attention_weight_max": 62.81645965576172,
      "activations/layer10_attention_weight_min": -51.23420333862305,
      "activations/layer11_attention_weight_max": 48.51109313964844,
      "activations/layer11_attention_weight_min": -41.28852081298828,
      "activations/layer12_attention_weight_max": 54.9119987487793,
      "activations/layer12_attention_weight_min": -34.313011169433594,
      "activations/layer13_attention_weight_max": 44.87803649902344,
      "activations/layer13_attention_weight_min": -34.776100158691406,
      "activations/layer14_attention_weight_max": 73.16753387451172,
      "activations/layer14_attention_weight_min": -53.85533905029297,
      "activations/layer15_attention_weight_max": 54.7335205078125,
      "activations/layer15_attention_weight_min": -42.093994140625,
      "activations/layer16_attention_weight_max": 52.8872184753418,
      "activations/layer16_attention_weight_min": -38.241580963134766,
      "activations/layer17_attention_weight_max": 36.21411895751953,
      "activations/layer17_attention_weight_min": -22.392431259155273,
      "activations/layer18_attention_weight_max": 19.058425903320312,
      "activations/layer18_attention_weight_min": -15.652857780456543,
      "activations/layer19_attention_weight_max": 46.091949462890625,
      "activations/layer19_attention_weight_min": -30.300189971923828,
      "activations/layer1_attention_weight_max": 15.769108772277832,
      "activations/layer1_attention_weight_min": -12.26715087890625,
      "activations/layer20_attention_weight_max": 38.3416862487793,
      "activations/layer20_attention_weight_min": -21.70438003540039,
      "activations/layer21_attention_weight_max": 19.26519203186035,
      "activations/layer21_attention_weight_min": -13.517924308776855,
      "activations/layer22_attention_weight_max": 41.40509033203125,
      "activations/layer22_attention_weight_min": -20.898374557495117,
      "activations/layer23_attention_weight_max": 28.49604034423828,
      "activations/layer23_attention_weight_min": -14.517194747924805,
      "activations/layer2_attention_weight_max": 27.019336700439453,
      "activations/layer2_attention_weight_min": -26.655553817749023,
      "activations/layer3_attention_weight_max": 55.43597412109375,
      "activations/layer3_attention_weight_min": -57.57295227050781,
      "activations/layer4_attention_weight_max": 108.39873504638672,
      "activations/layer4_attention_weight_min": -103.75968170166016,
      "activations/layer5_attention_weight_max": 125.64702606201172,
      "activations/layer5_attention_weight_min": -111.70790100097656,
      "activations/layer6_attention_weight_max": 67.84521484375,
      "activations/layer6_attention_weight_min": -59.6967887878418,
      "activations/layer7_attention_weight_max": 70.89021301269531,
      "activations/layer7_attention_weight_min": -63.19328689575195,
      "activations/layer8_attention_weight_max": 61.75352096557617,
      "activations/layer8_attention_weight_min": -54.46080017089844,
      "activations/layer9_attention_weight_max": 81.05098724365234,
      "activations/layer9_attention_weight_min": -67.56426239013672,
      "epoch": 0.87,
      "learning_rate": 0.00014582575757575756,
      "loss": 3.1455,
      "step": 15050
    },
    {
      "activations/layer0_attention_weight_max": 16.378183364868164,
      "activations/layer0_attention_weight_min": -11.534257888793945,
      "activations/layer10_attention_weight_max": 52.501583099365234,
      "activations/layer10_attention_weight_min": -48.620967864990234,
      "activations/layer11_attention_weight_max": 42.75025939941406,
      "activations/layer11_attention_weight_min": -43.602622985839844,
      "activations/layer12_attention_weight_max": 35.0390739440918,
      "activations/layer12_attention_weight_min": -32.73494338989258,
      "activations/layer13_attention_weight_max": 40.3768424987793,
      "activations/layer13_attention_weight_min": -32.95934295654297,
      "activations/layer14_attention_weight_max": 79.23861694335938,
      "activations/layer14_attention_weight_min": -59.000572204589844,
      "activations/layer15_attention_weight_max": 60.57340621948242,
      "activations/layer15_attention_weight_min": -43.43157958984375,
      "activations/layer16_attention_weight_max": 53.8280143737793,
      "activations/layer16_attention_weight_min": -36.27840042114258,
      "activations/layer17_attention_weight_max": 35.80160140991211,
      "activations/layer17_attention_weight_min": -21.19883155822754,
      "activations/layer18_attention_weight_max": 24.669078826904297,
      "activations/layer18_attention_weight_min": -14.375326156616211,
      "activations/layer19_attention_weight_max": 49.55996322631836,
      "activations/layer19_attention_weight_min": -29.263992309570312,
      "activations/layer1_attention_weight_max": 16.183490753173828,
      "activations/layer1_attention_weight_min": -12.189506530761719,
      "activations/layer20_attention_weight_max": 36.58134078979492,
      "activations/layer20_attention_weight_min": -21.521141052246094,
      "activations/layer21_attention_weight_max": 21.646806716918945,
      "activations/layer21_attention_weight_min": -16.086030960083008,
      "activations/layer22_attention_weight_max": 38.698219299316406,
      "activations/layer22_attention_weight_min": -21.691852569580078,
      "activations/layer23_attention_weight_max": 24.0163631439209,
      "activations/layer23_attention_weight_min": -14.271788597106934,
      "activations/layer2_attention_weight_max": 26.954853057861328,
      "activations/layer2_attention_weight_min": -26.440166473388672,
      "activations/layer3_attention_weight_max": 58.85458755493164,
      "activations/layer3_attention_weight_min": -62.562538146972656,
      "activations/layer4_attention_weight_max": 110.22379302978516,
      "activations/layer4_attention_weight_min": -105.06892395019531,
      "activations/layer5_attention_weight_max": 119.7046127319336,
      "activations/layer5_attention_weight_min": -113.24685668945312,
      "activations/layer6_attention_weight_max": 65.56059265136719,
      "activations/layer6_attention_weight_min": -60.70075607299805,
      "activations/layer7_attention_weight_max": 63.59019088745117,
      "activations/layer7_attention_weight_min": -64.85444641113281,
      "activations/layer8_attention_weight_max": 59.29954147338867,
      "activations/layer8_attention_weight_min": -52.24474334716797,
      "activations/layer9_attention_weight_max": 66.56847381591797,
      "activations/layer9_attention_weight_min": -62.040584564208984,
      "epoch": 0.88,
      "learning_rate": 0.00014580681818181817,
      "loss": 3.1392,
      "step": 15100
    },
    {
      "activations/layer0_attention_weight_max": 14.780950546264648,
      "activations/layer0_attention_weight_min": -11.224527359008789,
      "activations/layer10_attention_weight_max": 55.66703796386719,
      "activations/layer10_attention_weight_min": -48.396663665771484,
      "activations/layer11_attention_weight_max": 44.89530944824219,
      "activations/layer11_attention_weight_min": -39.12894058227539,
      "activations/layer12_attention_weight_max": 40.43608474731445,
      "activations/layer12_attention_weight_min": -34.40543746948242,
      "activations/layer13_attention_weight_max": 43.25624084472656,
      "activations/layer13_attention_weight_min": -37.5054817199707,
      "activations/layer14_attention_weight_max": 69.00169372558594,
      "activations/layer14_attention_weight_min": -52.231536865234375,
      "activations/layer15_attention_weight_max": 50.03298568725586,
      "activations/layer15_attention_weight_min": -37.84713363647461,
      "activations/layer16_attention_weight_max": 50.27964782714844,
      "activations/layer16_attention_weight_min": -39.84884262084961,
      "activations/layer17_attention_weight_max": 38.411468505859375,
      "activations/layer17_attention_weight_min": -23.074682235717773,
      "activations/layer18_attention_weight_max": 21.57933235168457,
      "activations/layer18_attention_weight_min": -13.016121864318848,
      "activations/layer19_attention_weight_max": 45.7230224609375,
      "activations/layer19_attention_weight_min": -29.557941436767578,
      "activations/layer1_attention_weight_max": 15.442488670349121,
      "activations/layer1_attention_weight_min": -12.81753158569336,
      "activations/layer20_attention_weight_max": 33.677818298339844,
      "activations/layer20_attention_weight_min": -23.706836700439453,
      "activations/layer21_attention_weight_max": 18.6204891204834,
      "activations/layer21_attention_weight_min": -14.573058128356934,
      "activations/layer22_attention_weight_max": 36.48450469970703,
      "activations/layer22_attention_weight_min": -20.858444213867188,
      "activations/layer23_attention_weight_max": 23.22207260131836,
      "activations/layer23_attention_weight_min": -13.712646484375,
      "activations/layer2_attention_weight_max": 26.15432357788086,
      "activations/layer2_attention_weight_min": -25.531047821044922,
      "activations/layer3_attention_weight_max": 57.95020294189453,
      "activations/layer3_attention_weight_min": -58.6856575012207,
      "activations/layer4_attention_weight_max": 105.3599624633789,
      "activations/layer4_attention_weight_min": -103.47286224365234,
      "activations/layer5_attention_weight_max": 119.1553955078125,
      "activations/layer5_attention_weight_min": -108.2548828125,
      "activations/layer6_attention_weight_max": 66.60188293457031,
      "activations/layer6_attention_weight_min": -62.194541931152344,
      "activations/layer7_attention_weight_max": 65.3309326171875,
      "activations/layer7_attention_weight_min": -64.44121551513672,
      "activations/layer8_attention_weight_max": 57.60817337036133,
      "activations/layer8_attention_weight_min": -52.00359344482422,
      "activations/layer9_attention_weight_max": 69.1506576538086,
      "activations/layer9_attention_weight_min": -63.371253967285156,
      "epoch": 0.88,
      "learning_rate": 0.0001457878787878788,
      "loss": 3.1266,
      "step": 15150
    },
    {
      "activations/layer0_attention_weight_max": 13.331801414489746,
      "activations/layer0_attention_weight_min": -11.02060317993164,
      "activations/layer10_attention_weight_max": 51.33650588989258,
      "activations/layer10_attention_weight_min": -42.8741340637207,
      "activations/layer11_attention_weight_max": 45.952171325683594,
      "activations/layer11_attention_weight_min": -37.13385772705078,
      "activations/layer12_attention_weight_max": 30.561908721923828,
      "activations/layer12_attention_weight_min": -26.603052139282227,
      "activations/layer13_attention_weight_max": 47.79100036621094,
      "activations/layer13_attention_weight_min": -35.47783279418945,
      "activations/layer14_attention_weight_max": 79.48641204833984,
      "activations/layer14_attention_weight_min": -52.35219955444336,
      "activations/layer15_attention_weight_max": 61.20476531982422,
      "activations/layer15_attention_weight_min": -41.48508834838867,
      "activations/layer16_attention_weight_max": 50.039894104003906,
      "activations/layer16_attention_weight_min": -35.57555389404297,
      "activations/layer17_attention_weight_max": 32.30012512207031,
      "activations/layer17_attention_weight_min": -20.619394302368164,
      "activations/layer18_attention_weight_max": 22.036134719848633,
      "activations/layer18_attention_weight_min": -12.928598403930664,
      "activations/layer19_attention_weight_max": 43.1157341003418,
      "activations/layer19_attention_weight_min": -32.0413932800293,
      "activations/layer1_attention_weight_max": 16.609582901000977,
      "activations/layer1_attention_weight_min": -11.914056777954102,
      "activations/layer20_attention_weight_max": 30.219970703125,
      "activations/layer20_attention_weight_min": -18.467918395996094,
      "activations/layer21_attention_weight_max": 20.372434616088867,
      "activations/layer21_attention_weight_min": -14.014849662780762,
      "activations/layer22_attention_weight_max": 34.493247985839844,
      "activations/layer22_attention_weight_min": -18.02786636352539,
      "activations/layer23_attention_weight_max": 22.24816131591797,
      "activations/layer23_attention_weight_min": -13.657707214355469,
      "activations/layer2_attention_weight_max": 27.59380340576172,
      "activations/layer2_attention_weight_min": -24.945594787597656,
      "activations/layer3_attention_weight_max": 60.464874267578125,
      "activations/layer3_attention_weight_min": -59.9154052734375,
      "activations/layer4_attention_weight_max": 114.0174331665039,
      "activations/layer4_attention_weight_min": -105.703125,
      "activations/layer5_attention_weight_max": 132.79037475585938,
      "activations/layer5_attention_weight_min": -116.24580383300781,
      "activations/layer6_attention_weight_max": 69.70323181152344,
      "activations/layer6_attention_weight_min": -58.95358657836914,
      "activations/layer7_attention_weight_max": 64.86054992675781,
      "activations/layer7_attention_weight_min": -59.793731689453125,
      "activations/layer8_attention_weight_max": 63.19850158691406,
      "activations/layer8_attention_weight_min": -56.139122009277344,
      "activations/layer9_attention_weight_max": 80.38692474365234,
      "activations/layer9_attention_weight_min": -69.91426849365234,
      "epoch": 0.88,
      "learning_rate": 0.00014576893939393938,
      "loss": 3.1436,
      "step": 15200
    },
    {
      "activations/layer0_attention_weight_max": 14.20981502532959,
      "activations/layer0_attention_weight_min": -11.038686752319336,
      "activations/layer10_attention_weight_max": 59.9853515625,
      "activations/layer10_attention_weight_min": -48.26022720336914,
      "activations/layer11_attention_weight_max": 47.925132751464844,
      "activations/layer11_attention_weight_min": -41.86494445800781,
      "activations/layer12_attention_weight_max": 68.89264678955078,
      "activations/layer12_attention_weight_min": -34.73883056640625,
      "activations/layer13_attention_weight_max": 52.492435455322266,
      "activations/layer13_attention_weight_min": -35.156837463378906,
      "activations/layer14_attention_weight_max": 82.46175384521484,
      "activations/layer14_attention_weight_min": -56.528900146484375,
      "activations/layer15_attention_weight_max": 57.722721099853516,
      "activations/layer15_attention_weight_min": -38.20979309082031,
      "activations/layer16_attention_weight_max": 56.50507736206055,
      "activations/layer16_attention_weight_min": -36.7693977355957,
      "activations/layer17_attention_weight_max": 33.55192947387695,
      "activations/layer17_attention_weight_min": -21.922727584838867,
      "activations/layer18_attention_weight_max": 19.685562133789062,
      "activations/layer18_attention_weight_min": -12.544431686401367,
      "activations/layer19_attention_weight_max": 48.178707122802734,
      "activations/layer19_attention_weight_min": -31.225759506225586,
      "activations/layer1_attention_weight_max": 15.772430419921875,
      "activations/layer1_attention_weight_min": -11.930318832397461,
      "activations/layer20_attention_weight_max": 33.97381591796875,
      "activations/layer20_attention_weight_min": -20.407987594604492,
      "activations/layer21_attention_weight_max": 18.968902587890625,
      "activations/layer21_attention_weight_min": -13.65993595123291,
      "activations/layer22_attention_weight_max": 33.52349853515625,
      "activations/layer22_attention_weight_min": -18.78435516357422,
      "activations/layer23_attention_weight_max": 20.736326217651367,
      "activations/layer23_attention_weight_min": -13.278227806091309,
      "activations/layer2_attention_weight_max": 26.791913986206055,
      "activations/layer2_attention_weight_min": -25.423755645751953,
      "activations/layer3_attention_weight_max": 56.49923324584961,
      "activations/layer3_attention_weight_min": -62.09048843383789,
      "activations/layer4_attention_weight_max": 106.15589904785156,
      "activations/layer4_attention_weight_min": -106.97700500488281,
      "activations/layer5_attention_weight_max": 121.99751281738281,
      "activations/layer5_attention_weight_min": -108.20339965820312,
      "activations/layer6_attention_weight_max": 65.49333190917969,
      "activations/layer6_attention_weight_min": -60.976112365722656,
      "activations/layer7_attention_weight_max": 64.56371307373047,
      "activations/layer7_attention_weight_min": -58.669891357421875,
      "activations/layer8_attention_weight_max": 68.79117584228516,
      "activations/layer8_attention_weight_min": -54.7646598815918,
      "activations/layer9_attention_weight_max": 78.70040130615234,
      "activations/layer9_attention_weight_min": -64.77486419677734,
      "epoch": 0.89,
      "learning_rate": 0.00014575,
      "loss": 3.134,
      "step": 15250
    },
    {
      "activations/layer0_attention_weight_max": 14.015936851501465,
      "activations/layer0_attention_weight_min": -11.677634239196777,
      "activations/layer10_attention_weight_max": 58.581111907958984,
      "activations/layer10_attention_weight_min": -51.11369705200195,
      "activations/layer11_attention_weight_max": 50.037071228027344,
      "activations/layer11_attention_weight_min": -39.9600944519043,
      "activations/layer12_attention_weight_max": 48.07455825805664,
      "activations/layer12_attention_weight_min": -38.33014678955078,
      "activations/layer13_attention_weight_max": 47.889671325683594,
      "activations/layer13_attention_weight_min": -38.741878509521484,
      "activations/layer14_attention_weight_max": 77.20143127441406,
      "activations/layer14_attention_weight_min": -59.094276428222656,
      "activations/layer15_attention_weight_max": 56.53770065307617,
      "activations/layer15_attention_weight_min": -41.22799301147461,
      "activations/layer16_attention_weight_max": 51.56169509887695,
      "activations/layer16_attention_weight_min": -36.224056243896484,
      "activations/layer17_attention_weight_max": 36.15205764770508,
      "activations/layer17_attention_weight_min": -23.986225128173828,
      "activations/layer18_attention_weight_max": 24.396263122558594,
      "activations/layer18_attention_weight_min": -13.204955101013184,
      "activations/layer19_attention_weight_max": 48.64717102050781,
      "activations/layer19_attention_weight_min": -30.143234252929688,
      "activations/layer1_attention_weight_max": 16.181243896484375,
      "activations/layer1_attention_weight_min": -12.41693115234375,
      "activations/layer20_attention_weight_max": 38.506221771240234,
      "activations/layer20_attention_weight_min": -21.120304107666016,
      "activations/layer21_attention_weight_max": 20.781967163085938,
      "activations/layer21_attention_weight_min": -14.429826736450195,
      "activations/layer22_attention_weight_max": 39.4205207824707,
      "activations/layer22_attention_weight_min": -25.067781448364258,
      "activations/layer23_attention_weight_max": 26.22968292236328,
      "activations/layer23_attention_weight_min": -14.35960578918457,
      "activations/layer2_attention_weight_max": 27.680286407470703,
      "activations/layer2_attention_weight_min": -26.102718353271484,
      "activations/layer3_attention_weight_max": 55.05365753173828,
      "activations/layer3_attention_weight_min": -61.26305389404297,
      "activations/layer4_attention_weight_max": 104.36205291748047,
      "activations/layer4_attention_weight_min": -104.29698944091797,
      "activations/layer5_attention_weight_max": 119.57585144042969,
      "activations/layer5_attention_weight_min": -107.7540283203125,
      "activations/layer6_attention_weight_max": 63.303253173828125,
      "activations/layer6_attention_weight_min": -61.94013977050781,
      "activations/layer7_attention_weight_max": 60.265018463134766,
      "activations/layer7_attention_weight_min": -63.37050247192383,
      "activations/layer8_attention_weight_max": 59.13870620727539,
      "activations/layer8_attention_weight_min": -52.07551956176758,
      "activations/layer9_attention_weight_max": 67.55369567871094,
      "activations/layer9_attention_weight_min": -61.66976547241211,
      "epoch": 0.89,
      "learning_rate": 0.0001457310606060606,
      "loss": 3.1245,
      "step": 15300
    },
    {
      "activations/layer0_attention_weight_max": 14.686588287353516,
      "activations/layer0_attention_weight_min": -11.285045623779297,
      "activations/layer10_attention_weight_max": 52.58179473876953,
      "activations/layer10_attention_weight_min": -54.01890182495117,
      "activations/layer11_attention_weight_max": 45.86384582519531,
      "activations/layer11_attention_weight_min": -38.22936248779297,
      "activations/layer12_attention_weight_max": 40.43526077270508,
      "activations/layer12_attention_weight_min": -30.094459533691406,
      "activations/layer13_attention_weight_max": 43.81041717529297,
      "activations/layer13_attention_weight_min": -33.189720153808594,
      "activations/layer14_attention_weight_max": 79.26978302001953,
      "activations/layer14_attention_weight_min": -57.08308410644531,
      "activations/layer15_attention_weight_max": 53.96349334716797,
      "activations/layer15_attention_weight_min": -38.834877014160156,
      "activations/layer16_attention_weight_max": 50.2013053894043,
      "activations/layer16_attention_weight_min": -36.156192779541016,
      "activations/layer17_attention_weight_max": 35.72832107543945,
      "activations/layer17_attention_weight_min": -21.124671936035156,
      "activations/layer18_attention_weight_max": 21.462289810180664,
      "activations/layer18_attention_weight_min": -13.597038269042969,
      "activations/layer19_attention_weight_max": 43.131385803222656,
      "activations/layer19_attention_weight_min": -27.126480102539062,
      "activations/layer1_attention_weight_max": 16.05301284790039,
      "activations/layer1_attention_weight_min": -11.683369636535645,
      "activations/layer20_attention_weight_max": 34.70927047729492,
      "activations/layer20_attention_weight_min": -19.592191696166992,
      "activations/layer21_attention_weight_max": 22.358428955078125,
      "activations/layer21_attention_weight_min": -13.950718879699707,
      "activations/layer22_attention_weight_max": 33.51193618774414,
      "activations/layer22_attention_weight_min": -19.08609962463379,
      "activations/layer23_attention_weight_max": 20.160181045532227,
      "activations/layer23_attention_weight_min": -14.524574279785156,
      "activations/layer2_attention_weight_max": 26.706565856933594,
      "activations/layer2_attention_weight_min": -25.91560935974121,
      "activations/layer3_attention_weight_max": 55.928802490234375,
      "activations/layer3_attention_weight_min": -58.565547943115234,
      "activations/layer4_attention_weight_max": 108.34394836425781,
      "activations/layer4_attention_weight_min": -103.91130828857422,
      "activations/layer5_attention_weight_max": 118.94835662841797,
      "activations/layer5_attention_weight_min": -109.69966888427734,
      "activations/layer6_attention_weight_max": 63.56818771362305,
      "activations/layer6_attention_weight_min": -62.4954833984375,
      "activations/layer7_attention_weight_max": 65.1778793334961,
      "activations/layer7_attention_weight_min": -63.94911575317383,
      "activations/layer8_attention_weight_max": 56.79838180541992,
      "activations/layer8_attention_weight_min": -52.24934768676758,
      "activations/layer9_attention_weight_max": 70.2190933227539,
      "activations/layer9_attention_weight_min": -63.383724212646484,
      "epoch": 0.89,
      "learning_rate": 0.0001457121212121212,
      "loss": 3.1128,
      "step": 15350
    },
    {
      "activations/layer0_attention_weight_max": 15.393548011779785,
      "activations/layer0_attention_weight_min": -11.51609992980957,
      "activations/layer10_attention_weight_max": 56.51585388183594,
      "activations/layer10_attention_weight_min": -49.3260612487793,
      "activations/layer11_attention_weight_max": 43.58293151855469,
      "activations/layer11_attention_weight_min": -37.46144104003906,
      "activations/layer12_attention_weight_max": 55.648555755615234,
      "activations/layer12_attention_weight_min": -35.24098205566406,
      "activations/layer13_attention_weight_max": 48.110538482666016,
      "activations/layer13_attention_weight_min": -40.54685592651367,
      "activations/layer14_attention_weight_max": 82.89901733398438,
      "activations/layer14_attention_weight_min": -56.691001892089844,
      "activations/layer15_attention_weight_max": 48.272953033447266,
      "activations/layer15_attention_weight_min": -39.37469482421875,
      "activations/layer16_attention_weight_max": 47.86630630493164,
      "activations/layer16_attention_weight_min": -38.43716812133789,
      "activations/layer17_attention_weight_max": 29.957143783569336,
      "activations/layer17_attention_weight_min": -21.84099578857422,
      "activations/layer18_attention_weight_max": 22.366403579711914,
      "activations/layer18_attention_weight_min": -13.594964027404785,
      "activations/layer19_attention_weight_max": 44.24956130981445,
      "activations/layer19_attention_weight_min": -30.7543888092041,
      "activations/layer1_attention_weight_max": 15.863086700439453,
      "activations/layer1_attention_weight_min": -11.814688682556152,
      "activations/layer20_attention_weight_max": 32.42444610595703,
      "activations/layer20_attention_weight_min": -23.27685546875,
      "activations/layer21_attention_weight_max": 19.522859573364258,
      "activations/layer21_attention_weight_min": -16.007293701171875,
      "activations/layer22_attention_weight_max": 33.41565704345703,
      "activations/layer22_attention_weight_min": -18.96346092224121,
      "activations/layer23_attention_weight_max": 21.47037124633789,
      "activations/layer23_attention_weight_min": -13.110790252685547,
      "activations/layer2_attention_weight_max": 26.155200958251953,
      "activations/layer2_attention_weight_min": -25.131277084350586,
      "activations/layer3_attention_weight_max": 57.0350456237793,
      "activations/layer3_attention_weight_min": -60.32326889038086,
      "activations/layer4_attention_weight_max": 108.02416229248047,
      "activations/layer4_attention_weight_min": -106.42138671875,
      "activations/layer5_attention_weight_max": 117.36878967285156,
      "activations/layer5_attention_weight_min": -110.22105407714844,
      "activations/layer6_attention_weight_max": 62.89589309692383,
      "activations/layer6_attention_weight_min": -59.687992095947266,
      "activations/layer7_attention_weight_max": 63.78581237792969,
      "activations/layer7_attention_weight_min": -60.563724517822266,
      "activations/layer8_attention_weight_max": 55.35528564453125,
      "activations/layer8_attention_weight_min": -52.18281936645508,
      "activations/layer9_attention_weight_max": 67.95753479003906,
      "activations/layer9_attention_weight_min": -66.5085220336914,
      "epoch": 0.89,
      "learning_rate": 0.0001456931818181818,
      "loss": 3.123,
      "step": 15400
    },
    {
      "activations/layer0_attention_weight_max": 15.726139068603516,
      "activations/layer0_attention_weight_min": -11.863814353942871,
      "activations/layer10_attention_weight_max": 53.60847091674805,
      "activations/layer10_attention_weight_min": -50.68861770629883,
      "activations/layer11_attention_weight_max": 46.835323333740234,
      "activations/layer11_attention_weight_min": -35.25129699707031,
      "activations/layer12_attention_weight_max": 32.4273681640625,
      "activations/layer12_attention_weight_min": -25.38987159729004,
      "activations/layer13_attention_weight_max": 41.73585891723633,
      "activations/layer13_attention_weight_min": -35.85685729980469,
      "activations/layer14_attention_weight_max": 67.57661437988281,
      "activations/layer14_attention_weight_min": -59.10365295410156,
      "activations/layer15_attention_weight_max": 58.06193542480469,
      "activations/layer15_attention_weight_min": -41.17424392700195,
      "activations/layer16_attention_weight_max": 61.34981918334961,
      "activations/layer16_attention_weight_min": -38.03104782104492,
      "activations/layer17_attention_weight_max": 38.50435256958008,
      "activations/layer17_attention_weight_min": -25.075525283813477,
      "activations/layer18_attention_weight_max": 24.72062873840332,
      "activations/layer18_attention_weight_min": -13.566058158874512,
      "activations/layer19_attention_weight_max": 59.094573974609375,
      "activations/layer19_attention_weight_min": -32.95766830444336,
      "activations/layer1_attention_weight_max": 15.60344409942627,
      "activations/layer1_attention_weight_min": -12.371668815612793,
      "activations/layer20_attention_weight_max": 40.99155807495117,
      "activations/layer20_attention_weight_min": -21.526487350463867,
      "activations/layer21_attention_weight_max": 21.362672805786133,
      "activations/layer21_attention_weight_min": -14.686532020568848,
      "activations/layer22_attention_weight_max": 40.77726364135742,
      "activations/layer22_attention_weight_min": -19.684898376464844,
      "activations/layer23_attention_weight_max": 26.65821647644043,
      "activations/layer23_attention_weight_min": -13.882722854614258,
      "activations/layer2_attention_weight_max": 27.75389862060547,
      "activations/layer2_attention_weight_min": -27.143287658691406,
      "activations/layer3_attention_weight_max": 59.223480224609375,
      "activations/layer3_attention_weight_min": -60.69446563720703,
      "activations/layer4_attention_weight_max": 115.34407043457031,
      "activations/layer4_attention_weight_min": -107.73149871826172,
      "activations/layer5_attention_weight_max": 126.9480209350586,
      "activations/layer5_attention_weight_min": -112.8609848022461,
      "activations/layer6_attention_weight_max": 70.04853820800781,
      "activations/layer6_attention_weight_min": -61.95161056518555,
      "activations/layer7_attention_weight_max": 61.701507568359375,
      "activations/layer7_attention_weight_min": -64.78423309326172,
      "activations/layer8_attention_weight_max": 59.36141586303711,
      "activations/layer8_attention_weight_min": -53.808204650878906,
      "activations/layer9_attention_weight_max": 73.98668670654297,
      "activations/layer9_attention_weight_min": -64.33183288574219,
      "epoch": 0.9,
      "learning_rate": 0.0001456742424242424,
      "loss": 3.1327,
      "step": 15450
    },
    {
      "activations/layer0_attention_weight_max": 13.986294746398926,
      "activations/layer0_attention_weight_min": -11.45665454864502,
      "activations/layer10_attention_weight_max": 62.00952911376953,
      "activations/layer10_attention_weight_min": -47.2757453918457,
      "activations/layer11_attention_weight_max": 44.530914306640625,
      "activations/layer11_attention_weight_min": -38.15497970581055,
      "activations/layer12_attention_weight_max": 35.97882843017578,
      "activations/layer12_attention_weight_min": -29.405494689941406,
      "activations/layer13_attention_weight_max": 55.9603157043457,
      "activations/layer13_attention_weight_min": -35.02077102661133,
      "activations/layer14_attention_weight_max": 77.33038330078125,
      "activations/layer14_attention_weight_min": -53.88862991333008,
      "activations/layer15_attention_weight_max": 59.110408782958984,
      "activations/layer15_attention_weight_min": -37.44913101196289,
      "activations/layer16_attention_weight_max": 48.35060501098633,
      "activations/layer16_attention_weight_min": -39.15373611450195,
      "activations/layer17_attention_weight_max": 33.33451843261719,
      "activations/layer17_attention_weight_min": -21.98404884338379,
      "activations/layer18_attention_weight_max": 22.885848999023438,
      "activations/layer18_attention_weight_min": -13.952113151550293,
      "activations/layer19_attention_weight_max": 48.63146209716797,
      "activations/layer19_attention_weight_min": -29.532419204711914,
      "activations/layer1_attention_weight_max": 15.39583683013916,
      "activations/layer1_attention_weight_min": -12.050176620483398,
      "activations/layer20_attention_weight_max": 37.579925537109375,
      "activations/layer20_attention_weight_min": -23.588367462158203,
      "activations/layer21_attention_weight_max": 20.292354583740234,
      "activations/layer21_attention_weight_min": -16.505558013916016,
      "activations/layer22_attention_weight_max": 39.57462692260742,
      "activations/layer22_attention_weight_min": -20.82349395751953,
      "activations/layer23_attention_weight_max": 26.0615291595459,
      "activations/layer23_attention_weight_min": -17.213165283203125,
      "activations/layer2_attention_weight_max": 27.351301193237305,
      "activations/layer2_attention_weight_min": -24.47780990600586,
      "activations/layer3_attention_weight_max": 56.43681335449219,
      "activations/layer3_attention_weight_min": -59.75117111206055,
      "activations/layer4_attention_weight_max": 107.64208984375,
      "activations/layer4_attention_weight_min": -106.14373779296875,
      "activations/layer5_attention_weight_max": 128.645263671875,
      "activations/layer5_attention_weight_min": -108.72503662109375,
      "activations/layer6_attention_weight_max": 69.80812072753906,
      "activations/layer6_attention_weight_min": -61.19170379638672,
      "activations/layer7_attention_weight_max": 70.8652572631836,
      "activations/layer7_attention_weight_min": -65.50730895996094,
      "activations/layer8_attention_weight_max": 66.26002502441406,
      "activations/layer8_attention_weight_min": -55.41376876831055,
      "activations/layer9_attention_weight_max": 78.45809173583984,
      "activations/layer9_attention_weight_min": -64.04061126708984,
      "epoch": 0.9,
      "learning_rate": 0.000145655303030303,
      "loss": 3.1389,
      "step": 15500
    },
    {
      "activations/layer0_attention_weight_max": 15.771381378173828,
      "activations/layer0_attention_weight_min": -11.441854476928711,
      "activations/layer10_attention_weight_max": 45.0243034362793,
      "activations/layer10_attention_weight_min": -43.779335021972656,
      "activations/layer11_attention_weight_max": 40.94428634643555,
      "activations/layer11_attention_weight_min": -39.97284698486328,
      "activations/layer12_attention_weight_max": 36.2603759765625,
      "activations/layer12_attention_weight_min": -31.491270065307617,
      "activations/layer13_attention_weight_max": 39.63300704956055,
      "activations/layer13_attention_weight_min": -35.18452453613281,
      "activations/layer14_attention_weight_max": 67.51968383789062,
      "activations/layer14_attention_weight_min": -56.238136291503906,
      "activations/layer15_attention_weight_max": 49.34028244018555,
      "activations/layer15_attention_weight_min": -42.372581481933594,
      "activations/layer16_attention_weight_max": 50.30434799194336,
      "activations/layer16_attention_weight_min": -37.955753326416016,
      "activations/layer17_attention_weight_max": 35.361167907714844,
      "activations/layer17_attention_weight_min": -21.054407119750977,
      "activations/layer18_attention_weight_max": 20.296167373657227,
      "activations/layer18_attention_weight_min": -13.154370307922363,
      "activations/layer19_attention_weight_max": 47.902244567871094,
      "activations/layer19_attention_weight_min": -30.351749420166016,
      "activations/layer1_attention_weight_max": 15.291958808898926,
      "activations/layer1_attention_weight_min": -11.650858879089355,
      "activations/layer20_attention_weight_max": 37.073448181152344,
      "activations/layer20_attention_weight_min": -22.742143630981445,
      "activations/layer21_attention_weight_max": 20.43573570251465,
      "activations/layer21_attention_weight_min": -15.807194709777832,
      "activations/layer22_attention_weight_max": 35.183677673339844,
      "activations/layer22_attention_weight_min": -23.15007209777832,
      "activations/layer23_attention_weight_max": 21.060863494873047,
      "activations/layer23_attention_weight_min": -14.198809623718262,
      "activations/layer2_attention_weight_max": 28.050086975097656,
      "activations/layer2_attention_weight_min": -24.28040885925293,
      "activations/layer3_attention_weight_max": 55.91666030883789,
      "activations/layer3_attention_weight_min": -54.90681838989258,
      "activations/layer4_attention_weight_max": 108.49787902832031,
      "activations/layer4_attention_weight_min": -102.2268295288086,
      "activations/layer5_attention_weight_max": 114.44320678710938,
      "activations/layer5_attention_weight_min": -106.63922119140625,
      "activations/layer6_attention_weight_max": 61.64762496948242,
      "activations/layer6_attention_weight_min": -60.26002883911133,
      "activations/layer7_attention_weight_max": 66.27845001220703,
      "activations/layer7_attention_weight_min": -63.898773193359375,
      "activations/layer8_attention_weight_max": 58.32834243774414,
      "activations/layer8_attention_weight_min": -54.871517181396484,
      "activations/layer9_attention_weight_max": 63.25360107421875,
      "activations/layer9_attention_weight_min": -68.38916015625,
      "epoch": 0.9,
      "learning_rate": 0.00014563636363636363,
      "loss": 3.1443,
      "step": 15550
    },
    {
      "activations/layer0_attention_weight_max": 15.539931297302246,
      "activations/layer0_attention_weight_min": -11.343969345092773,
      "activations/layer10_attention_weight_max": 55.38031768798828,
      "activations/layer10_attention_weight_min": -51.50046920776367,
      "activations/layer11_attention_weight_max": 55.95326232910156,
      "activations/layer11_attention_weight_min": -40.909332275390625,
      "activations/layer12_attention_weight_max": 35.281620025634766,
      "activations/layer12_attention_weight_min": -28.933448791503906,
      "activations/layer13_attention_weight_max": 42.96363830566406,
      "activations/layer13_attention_weight_min": -34.41639709472656,
      "activations/layer14_attention_weight_max": 80.8924789428711,
      "activations/layer14_attention_weight_min": -62.212440490722656,
      "activations/layer15_attention_weight_max": 50.11797332763672,
      "activations/layer15_attention_weight_min": -41.297061920166016,
      "activations/layer16_attention_weight_max": 55.29379653930664,
      "activations/layer16_attention_weight_min": -39.33091354370117,
      "activations/layer17_attention_weight_max": 33.330894470214844,
      "activations/layer17_attention_weight_min": -24.268983840942383,
      "activations/layer18_attention_weight_max": 22.18329429626465,
      "activations/layer18_attention_weight_min": -14.304862976074219,
      "activations/layer19_attention_weight_max": 45.774513244628906,
      "activations/layer19_attention_weight_min": -31.489276885986328,
      "activations/layer1_attention_weight_max": 15.356064796447754,
      "activations/layer1_attention_weight_min": -12.15749740600586,
      "activations/layer20_attention_weight_max": 34.32541275024414,
      "activations/layer20_attention_weight_min": -21.589622497558594,
      "activations/layer21_attention_weight_max": 19.784671783447266,
      "activations/layer21_attention_weight_min": -15.073958396911621,
      "activations/layer22_attention_weight_max": 42.77859878540039,
      "activations/layer22_attention_weight_min": -21.00079345703125,
      "activations/layer23_attention_weight_max": 24.356962203979492,
      "activations/layer23_attention_weight_min": -14.597564697265625,
      "activations/layer2_attention_weight_max": 29.37946128845215,
      "activations/layer2_attention_weight_min": -26.003829956054688,
      "activations/layer3_attention_weight_max": 60.99921417236328,
      "activations/layer3_attention_weight_min": -60.64223098754883,
      "activations/layer4_attention_weight_max": 112.91261291503906,
      "activations/layer4_attention_weight_min": -107.94636535644531,
      "activations/layer5_attention_weight_max": 126.56617736816406,
      "activations/layer5_attention_weight_min": -113.45599365234375,
      "activations/layer6_attention_weight_max": 68.99530029296875,
      "activations/layer6_attention_weight_min": -61.04486846923828,
      "activations/layer7_attention_weight_max": 62.2119026184082,
      "activations/layer7_attention_weight_min": -64.23770141601562,
      "activations/layer8_attention_weight_max": 69.0672607421875,
      "activations/layer8_attention_weight_min": -57.82907485961914,
      "activations/layer9_attention_weight_max": 87.85504150390625,
      "activations/layer9_attention_weight_min": -66.63735961914062,
      "epoch": 0.91,
      "learning_rate": 0.00014561742424242422,
      "loss": 3.1155,
      "step": 15600
    },
    {
      "activations/layer0_attention_weight_max": 15.257026672363281,
      "activations/layer0_attention_weight_min": -11.254239082336426,
      "activations/layer10_attention_weight_max": 52.37222671508789,
      "activations/layer10_attention_weight_min": -46.200164794921875,
      "activations/layer11_attention_weight_max": 50.09305953979492,
      "activations/layer11_attention_weight_min": -38.409141540527344,
      "activations/layer12_attention_weight_max": 43.31433868408203,
      "activations/layer12_attention_weight_min": -38.78239059448242,
      "activations/layer13_attention_weight_max": 49.88463592529297,
      "activations/layer13_attention_weight_min": -43.93737030029297,
      "activations/layer14_attention_weight_max": 58.94935989379883,
      "activations/layer14_attention_weight_min": -48.68538284301758,
      "activations/layer15_attention_weight_max": 50.545291900634766,
      "activations/layer15_attention_weight_min": -39.18156433105469,
      "activations/layer16_attention_weight_max": 52.95890426635742,
      "activations/layer16_attention_weight_min": -40.6862907409668,
      "activations/layer17_attention_weight_max": 33.69499588012695,
      "activations/layer17_attention_weight_min": -23.31646728515625,
      "activations/layer18_attention_weight_max": 20.97355079650879,
      "activations/layer18_attention_weight_min": -13.687905311584473,
      "activations/layer19_attention_weight_max": 51.5986328125,
      "activations/layer19_attention_weight_min": -32.496402740478516,
      "activations/layer1_attention_weight_max": 15.289443969726562,
      "activations/layer1_attention_weight_min": -11.83018970489502,
      "activations/layer20_attention_weight_max": 34.367244720458984,
      "activations/layer20_attention_weight_min": -21.336305618286133,
      "activations/layer21_attention_weight_max": 20.090946197509766,
      "activations/layer21_attention_weight_min": -12.684353828430176,
      "activations/layer22_attention_weight_max": 36.40495300292969,
      "activations/layer22_attention_weight_min": -20.135637283325195,
      "activations/layer23_attention_weight_max": 24.105154037475586,
      "activations/layer23_attention_weight_min": -13.30303955078125,
      "activations/layer2_attention_weight_max": 27.80799674987793,
      "activations/layer2_attention_weight_min": -26.2988338470459,
      "activations/layer3_attention_weight_max": 60.05533981323242,
      "activations/layer3_attention_weight_min": -61.16608810424805,
      "activations/layer4_attention_weight_max": 110.62774658203125,
      "activations/layer4_attention_weight_min": -104.98553466796875,
      "activations/layer5_attention_weight_max": 120.04945373535156,
      "activations/layer5_attention_weight_min": -109.06282043457031,
      "activations/layer6_attention_weight_max": 70.47173309326172,
      "activations/layer6_attention_weight_min": -61.192352294921875,
      "activations/layer7_attention_weight_max": 61.00659942626953,
      "activations/layer7_attention_weight_min": -66.54936981201172,
      "activations/layer8_attention_weight_max": 61.15024948120117,
      "activations/layer8_attention_weight_min": -54.602027893066406,
      "activations/layer9_attention_weight_max": 76.5564193725586,
      "activations/layer9_attention_weight_min": -66.40499114990234,
      "epoch": 0.91,
      "learning_rate": 0.00014559848484848483,
      "loss": 3.118,
      "step": 15650
    },
    {
      "activations/layer0_attention_weight_max": 15.516979217529297,
      "activations/layer0_attention_weight_min": -11.017427444458008,
      "activations/layer10_attention_weight_max": 59.577449798583984,
      "activations/layer10_attention_weight_min": -47.47386169433594,
      "activations/layer11_attention_weight_max": 49.0869140625,
      "activations/layer11_attention_weight_min": -40.75604248046875,
      "activations/layer12_attention_weight_max": 40.95873260498047,
      "activations/layer12_attention_weight_min": -31.46383285522461,
      "activations/layer13_attention_weight_max": 47.018680572509766,
      "activations/layer13_attention_weight_min": -34.42029571533203,
      "activations/layer14_attention_weight_max": 92.33784484863281,
      "activations/layer14_attention_weight_min": -57.689876556396484,
      "activations/layer15_attention_weight_max": 68.92228698730469,
      "activations/layer15_attention_weight_min": -44.204463958740234,
      "activations/layer16_attention_weight_max": 64.26133728027344,
      "activations/layer16_attention_weight_min": -41.48184585571289,
      "activations/layer17_attention_weight_max": 38.92744445800781,
      "activations/layer17_attention_weight_min": -23.132139205932617,
      "activations/layer18_attention_weight_max": 23.030881881713867,
      "activations/layer18_attention_weight_min": -11.896347999572754,
      "activations/layer19_attention_weight_max": 55.93342208862305,
      "activations/layer19_attention_weight_min": -31.21786880493164,
      "activations/layer1_attention_weight_max": 15.92382526397705,
      "activations/layer1_attention_weight_min": -11.13033676147461,
      "activations/layer20_attention_weight_max": 41.47307586669922,
      "activations/layer20_attention_weight_min": -21.3328857421875,
      "activations/layer21_attention_weight_max": 20.593908309936523,
      "activations/layer21_attention_weight_min": -13.45418643951416,
      "activations/layer22_attention_weight_max": 39.85824203491211,
      "activations/layer22_attention_weight_min": -22.38535499572754,
      "activations/layer23_attention_weight_max": 22.090238571166992,
      "activations/layer23_attention_weight_min": -18.291467666625977,
      "activations/layer2_attention_weight_max": 27.778919219970703,
      "activations/layer2_attention_weight_min": -25.237211227416992,
      "activations/layer3_attention_weight_max": 56.623779296875,
      "activations/layer3_attention_weight_min": -58.30023956298828,
      "activations/layer4_attention_weight_max": 108.27986907958984,
      "activations/layer4_attention_weight_min": -100.7740707397461,
      "activations/layer5_attention_weight_max": 120.59860229492188,
      "activations/layer5_attention_weight_min": -108.0694580078125,
      "activations/layer6_attention_weight_max": 66.31224060058594,
      "activations/layer6_attention_weight_min": -59.70501708984375,
      "activations/layer7_attention_weight_max": 62.87333297729492,
      "activations/layer7_attention_weight_min": -63.05419921875,
      "activations/layer8_attention_weight_max": 61.96935272216797,
      "activations/layer8_attention_weight_min": -57.22494125366211,
      "activations/layer9_attention_weight_max": 68.1561508178711,
      "activations/layer9_attention_weight_min": -56.62968826293945,
      "epoch": 0.91,
      "learning_rate": 0.00014557954545454545,
      "loss": 3.1122,
      "step": 15700
    },
    {
      "activations/layer0_attention_weight_max": 14.869667053222656,
      "activations/layer0_attention_weight_min": -10.833499908447266,
      "activations/layer10_attention_weight_max": 52.95941162109375,
      "activations/layer10_attention_weight_min": -47.38490676879883,
      "activations/layer11_attention_weight_max": 43.6786003112793,
      "activations/layer11_attention_weight_min": -36.585052490234375,
      "activations/layer12_attention_weight_max": 34.160850524902344,
      "activations/layer12_attention_weight_min": -27.346097946166992,
      "activations/layer13_attention_weight_max": 52.16135025024414,
      "activations/layer13_attention_weight_min": -35.80317306518555,
      "activations/layer14_attention_weight_max": 77.20214080810547,
      "activations/layer14_attention_weight_min": -52.96897888183594,
      "activations/layer15_attention_weight_max": 53.9084587097168,
      "activations/layer15_attention_weight_min": -37.180419921875,
      "activations/layer16_attention_weight_max": 51.195404052734375,
      "activations/layer16_attention_weight_min": -38.03445816040039,
      "activations/layer17_attention_weight_max": 33.843116760253906,
      "activations/layer17_attention_weight_min": -22.3933048248291,
      "activations/layer18_attention_weight_max": 24.63742446899414,
      "activations/layer18_attention_weight_min": -13.105363845825195,
      "activations/layer19_attention_weight_max": 50.309844970703125,
      "activations/layer19_attention_weight_min": -30.338455200195312,
      "activations/layer1_attention_weight_max": 16.337539672851562,
      "activations/layer1_attention_weight_min": -12.805423736572266,
      "activations/layer20_attention_weight_max": 41.67713165283203,
      "activations/layer20_attention_weight_min": -22.18193817138672,
      "activations/layer21_attention_weight_max": 20.413000106811523,
      "activations/layer21_attention_weight_min": -12.639657020568848,
      "activations/layer22_attention_weight_max": 41.57749938964844,
      "activations/layer22_attention_weight_min": -22.598918914794922,
      "activations/layer23_attention_weight_max": 25.47835922241211,
      "activations/layer23_attention_weight_min": -14.457996368408203,
      "activations/layer2_attention_weight_max": 28.827823638916016,
      "activations/layer2_attention_weight_min": -26.591032028198242,
      "activations/layer3_attention_weight_max": 59.07649230957031,
      "activations/layer3_attention_weight_min": -60.05128860473633,
      "activations/layer4_attention_weight_max": 112.35343170166016,
      "activations/layer4_attention_weight_min": -105.06475067138672,
      "activations/layer5_attention_weight_max": 123.15681457519531,
      "activations/layer5_attention_weight_min": -110.98239135742188,
      "activations/layer6_attention_weight_max": 69.59459686279297,
      "activations/layer6_attention_weight_min": -59.123207092285156,
      "activations/layer7_attention_weight_max": 64.93975067138672,
      "activations/layer7_attention_weight_min": -63.61562728881836,
      "activations/layer8_attention_weight_max": 58.98335266113281,
      "activations/layer8_attention_weight_min": -49.945125579833984,
      "activations/layer9_attention_weight_max": 71.47570037841797,
      "activations/layer9_attention_weight_min": -62.79824447631836,
      "epoch": 0.92,
      "learning_rate": 0.00014556060606060603,
      "loss": 3.123,
      "step": 15750
    },
    {
      "activations/layer0_attention_weight_max": 14.584094047546387,
      "activations/layer0_attention_weight_min": -10.851062774658203,
      "activations/layer10_attention_weight_max": 48.81809997558594,
      "activations/layer10_attention_weight_min": -45.5578727722168,
      "activations/layer11_attention_weight_max": 42.321075439453125,
      "activations/layer11_attention_weight_min": -38.88725280761719,
      "activations/layer12_attention_weight_max": 42.1855354309082,
      "activations/layer12_attention_weight_min": -31.697118759155273,
      "activations/layer13_attention_weight_max": 45.38105010986328,
      "activations/layer13_attention_weight_min": -36.3895378112793,
      "activations/layer14_attention_weight_max": 64.2265853881836,
      "activations/layer14_attention_weight_min": -56.15317916870117,
      "activations/layer15_attention_weight_max": 52.96693801879883,
      "activations/layer15_attention_weight_min": -36.89118194580078,
      "activations/layer16_attention_weight_max": 55.12115478515625,
      "activations/layer16_attention_weight_min": -37.02676010131836,
      "activations/layer17_attention_weight_max": 30.130754470825195,
      "activations/layer17_attention_weight_min": -22.58823585510254,
      "activations/layer18_attention_weight_max": 20.82518768310547,
      "activations/layer18_attention_weight_min": -13.214693069458008,
      "activations/layer19_attention_weight_max": 44.08311080932617,
      "activations/layer19_attention_weight_min": -28.153165817260742,
      "activations/layer1_attention_weight_max": 15.627313613891602,
      "activations/layer1_attention_weight_min": -11.573575019836426,
      "activations/layer20_attention_weight_max": 38.739871978759766,
      "activations/layer20_attention_weight_min": -20.46550941467285,
      "activations/layer21_attention_weight_max": 21.874834060668945,
      "activations/layer21_attention_weight_min": -13.220586776733398,
      "activations/layer22_attention_weight_max": 41.12580108642578,
      "activations/layer22_attention_weight_min": -20.435546875,
      "activations/layer23_attention_weight_max": 25.780593872070312,
      "activations/layer23_attention_weight_min": -14.088951110839844,
      "activations/layer2_attention_weight_max": 30.093481063842773,
      "activations/layer2_attention_weight_min": -26.36074447631836,
      "activations/layer3_attention_weight_max": 59.08464050292969,
      "activations/layer3_attention_weight_min": -58.93415451049805,
      "activations/layer4_attention_weight_max": 109.89837646484375,
      "activations/layer4_attention_weight_min": -105.27581024169922,
      "activations/layer5_attention_weight_max": 123.69038391113281,
      "activations/layer5_attention_weight_min": -115.9189682006836,
      "activations/layer6_attention_weight_max": 70.53604125976562,
      "activations/layer6_attention_weight_min": -61.716033935546875,
      "activations/layer7_attention_weight_max": 63.28954315185547,
      "activations/layer7_attention_weight_min": -60.987335205078125,
      "activations/layer8_attention_weight_max": 57.20853805541992,
      "activations/layer8_attention_weight_min": -49.922237396240234,
      "activations/layer9_attention_weight_max": 64.52933502197266,
      "activations/layer9_attention_weight_min": -63.5495491027832,
      "epoch": 0.92,
      "learning_rate": 0.00014554166666666665,
      "loss": 3.1256,
      "step": 15800
    },
    {
      "activations/layer0_attention_weight_max": 15.604541778564453,
      "activations/layer0_attention_weight_min": -10.972393989562988,
      "activations/layer10_attention_weight_max": 48.9067497253418,
      "activations/layer10_attention_weight_min": -43.280887603759766,
      "activations/layer11_attention_weight_max": 48.193458557128906,
      "activations/layer11_attention_weight_min": -37.90113830566406,
      "activations/layer12_attention_weight_max": 41.29094314575195,
      "activations/layer12_attention_weight_min": -32.3609619140625,
      "activations/layer13_attention_weight_max": 44.78266525268555,
      "activations/layer13_attention_weight_min": -35.19462966918945,
      "activations/layer14_attention_weight_max": 66.37089538574219,
      "activations/layer14_attention_weight_min": -51.73374938964844,
      "activations/layer15_attention_weight_max": 50.35363006591797,
      "activations/layer15_attention_weight_min": -36.190269470214844,
      "activations/layer16_attention_weight_max": 56.113399505615234,
      "activations/layer16_attention_weight_min": -45.05780029296875,
      "activations/layer17_attention_weight_max": 37.74708557128906,
      "activations/layer17_attention_weight_min": -23.088144302368164,
      "activations/layer18_attention_weight_max": 23.093263626098633,
      "activations/layer18_attention_weight_min": -13.692607879638672,
      "activations/layer19_attention_weight_max": 48.538795471191406,
      "activations/layer19_attention_weight_min": -30.87835693359375,
      "activations/layer1_attention_weight_max": 15.130809783935547,
      "activations/layer1_attention_weight_min": -11.412068367004395,
      "activations/layer20_attention_weight_max": 37.86418533325195,
      "activations/layer20_attention_weight_min": -21.87470817565918,
      "activations/layer21_attention_weight_max": 19.057668685913086,
      "activations/layer21_attention_weight_min": -13.242451667785645,
      "activations/layer22_attention_weight_max": 38.834537506103516,
      "activations/layer22_attention_weight_min": -24.38250732421875,
      "activations/layer23_attention_weight_max": 24.74114990234375,
      "activations/layer23_attention_weight_min": -13.602060317993164,
      "activations/layer2_attention_weight_max": 26.9178524017334,
      "activations/layer2_attention_weight_min": -24.913707733154297,
      "activations/layer3_attention_weight_max": 58.03488540649414,
      "activations/layer3_attention_weight_min": -60.810569763183594,
      "activations/layer4_attention_weight_max": 103.81021881103516,
      "activations/layer4_attention_weight_min": -103.0766372680664,
      "activations/layer5_attention_weight_max": 115.67477416992188,
      "activations/layer5_attention_weight_min": -108.33699798583984,
      "activations/layer6_attention_weight_max": 62.772804260253906,
      "activations/layer6_attention_weight_min": -57.806007385253906,
      "activations/layer7_attention_weight_max": 66.36993408203125,
      "activations/layer7_attention_weight_min": -64.78526306152344,
      "activations/layer8_attention_weight_max": 56.56678771972656,
      "activations/layer8_attention_weight_min": -50.94898223876953,
      "activations/layer9_attention_weight_max": 74.08373260498047,
      "activations/layer9_attention_weight_min": -61.81672286987305,
      "epoch": 0.92,
      "learning_rate": 0.00014552272727272726,
      "loss": 3.1333,
      "step": 15850
    },
    {
      "activations/layer0_attention_weight_max": 15.179162979125977,
      "activations/layer0_attention_weight_min": -11.026947021484375,
      "activations/layer10_attention_weight_max": 51.975486755371094,
      "activations/layer10_attention_weight_min": -46.8825798034668,
      "activations/layer11_attention_weight_max": 43.4891471862793,
      "activations/layer11_attention_weight_min": -36.140438079833984,
      "activations/layer12_attention_weight_max": 33.091339111328125,
      "activations/layer12_attention_weight_min": -26.0671329498291,
      "activations/layer13_attention_weight_max": 41.071205139160156,
      "activations/layer13_attention_weight_min": -32.970619201660156,
      "activations/layer14_attention_weight_max": 66.03700256347656,
      "activations/layer14_attention_weight_min": -53.80245590209961,
      "activations/layer15_attention_weight_max": 52.79790115356445,
      "activations/layer15_attention_weight_min": -37.951393127441406,
      "activations/layer16_attention_weight_max": 56.018280029296875,
      "activations/layer16_attention_weight_min": -37.43434143066406,
      "activations/layer17_attention_weight_max": 37.64118194580078,
      "activations/layer17_attention_weight_min": -24.039674758911133,
      "activations/layer18_attention_weight_max": 25.395219802856445,
      "activations/layer18_attention_weight_min": -13.353243827819824,
      "activations/layer19_attention_weight_max": 53.59212112426758,
      "activations/layer19_attention_weight_min": -29.017379760742188,
      "activations/layer1_attention_weight_max": 15.641982078552246,
      "activations/layer1_attention_weight_min": -12.070893287658691,
      "activations/layer20_attention_weight_max": 40.095184326171875,
      "activations/layer20_attention_weight_min": -22.554004669189453,
      "activations/layer21_attention_weight_max": 22.3059024810791,
      "activations/layer21_attention_weight_min": -14.071210861206055,
      "activations/layer22_attention_weight_max": 37.46067428588867,
      "activations/layer22_attention_weight_min": -18.53713035583496,
      "activations/layer23_attention_weight_max": 24.102787017822266,
      "activations/layer23_attention_weight_min": -12.27644157409668,
      "activations/layer2_attention_weight_max": 27.160924911499023,
      "activations/layer2_attention_weight_min": -23.716266632080078,
      "activations/layer3_attention_weight_max": 56.78744125366211,
      "activations/layer3_attention_weight_min": -59.42955780029297,
      "activations/layer4_attention_weight_max": 106.10255432128906,
      "activations/layer4_attention_weight_min": -107.36577606201172,
      "activations/layer5_attention_weight_max": 123.87931823730469,
      "activations/layer5_attention_weight_min": -112.73143768310547,
      "activations/layer6_attention_weight_max": 65.06462097167969,
      "activations/layer6_attention_weight_min": -57.95970153808594,
      "activations/layer7_attention_weight_max": 63.53403854370117,
      "activations/layer7_attention_weight_min": -65.32095336914062,
      "activations/layer8_attention_weight_max": 62.93119430541992,
      "activations/layer8_attention_weight_min": -57.63102340698242,
      "activations/layer9_attention_weight_max": 75.80445098876953,
      "activations/layer9_attention_weight_min": -60.3575439453125,
      "epoch": 0.92,
      "learning_rate": 0.00014550378787878785,
      "loss": 3.1315,
      "step": 15900
    },
    {
      "activations/layer0_attention_weight_max": 14.152710914611816,
      "activations/layer0_attention_weight_min": -11.08769416809082,
      "activations/layer10_attention_weight_max": 52.35736846923828,
      "activations/layer10_attention_weight_min": -47.66582107543945,
      "activations/layer11_attention_weight_max": 47.229915618896484,
      "activations/layer11_attention_weight_min": -36.477378845214844,
      "activations/layer12_attention_weight_max": 35.92425537109375,
      "activations/layer12_attention_weight_min": -28.513076782226562,
      "activations/layer13_attention_weight_max": 43.86758041381836,
      "activations/layer13_attention_weight_min": -36.27521896362305,
      "activations/layer14_attention_weight_max": 74.66168975830078,
      "activations/layer14_attention_weight_min": -60.058773040771484,
      "activations/layer15_attention_weight_max": 64.40240478515625,
      "activations/layer15_attention_weight_min": -44.790584564208984,
      "activations/layer16_attention_weight_max": 59.88626480102539,
      "activations/layer16_attention_weight_min": -41.76907730102539,
      "activations/layer17_attention_weight_max": 40.355350494384766,
      "activations/layer17_attention_weight_min": -24.35190200805664,
      "activations/layer18_attention_weight_max": 23.12601089477539,
      "activations/layer18_attention_weight_min": -13.862483024597168,
      "activations/layer19_attention_weight_max": 50.63175582885742,
      "activations/layer19_attention_weight_min": -29.285120010375977,
      "activations/layer1_attention_weight_max": 15.78463077545166,
      "activations/layer1_attention_weight_min": -11.832890510559082,
      "activations/layer20_attention_weight_max": 39.06843566894531,
      "activations/layer20_attention_weight_min": -22.858417510986328,
      "activations/layer21_attention_weight_max": 22.11826515197754,
      "activations/layer21_attention_weight_min": -14.66579532623291,
      "activations/layer22_attention_weight_max": 37.54133987426758,
      "activations/layer22_attention_weight_min": -22.103057861328125,
      "activations/layer23_attention_weight_max": 25.941482543945312,
      "activations/layer23_attention_weight_min": -15.412557601928711,
      "activations/layer2_attention_weight_max": 28.789962768554688,
      "activations/layer2_attention_weight_min": -25.695945739746094,
      "activations/layer3_attention_weight_max": 58.706016540527344,
      "activations/layer3_attention_weight_min": -63.19610595703125,
      "activations/layer4_attention_weight_max": 110.4272232055664,
      "activations/layer4_attention_weight_min": -111.9873275756836,
      "activations/layer5_attention_weight_max": 122.57931518554688,
      "activations/layer5_attention_weight_min": -116.9549789428711,
      "activations/layer6_attention_weight_max": 70.57423400878906,
      "activations/layer6_attention_weight_min": -63.30498123168945,
      "activations/layer7_attention_weight_max": 62.86426544189453,
      "activations/layer7_attention_weight_min": -67.48285675048828,
      "activations/layer8_attention_weight_max": 68.43475341796875,
      "activations/layer8_attention_weight_min": -61.36412811279297,
      "activations/layer9_attention_weight_max": 73.5059585571289,
      "activations/layer9_attention_weight_min": -67.00638580322266,
      "epoch": 0.93,
      "learning_rate": 0.00014548484848484847,
      "loss": 3.1291,
      "step": 15950
    },
    {
      "activations/layer0_attention_weight_max": 14.031293869018555,
      "activations/layer0_attention_weight_min": -11.574947357177734,
      "activations/layer10_attention_weight_max": 55.71215057373047,
      "activations/layer10_attention_weight_min": -50.85718536376953,
      "activations/layer11_attention_weight_max": 44.05491256713867,
      "activations/layer11_attention_weight_min": -40.17422103881836,
      "activations/layer12_attention_weight_max": 39.60089111328125,
      "activations/layer12_attention_weight_min": -32.098453521728516,
      "activations/layer13_attention_weight_max": 46.939762115478516,
      "activations/layer13_attention_weight_min": -37.79860305786133,
      "activations/layer14_attention_weight_max": 72.41033935546875,
      "activations/layer14_attention_weight_min": -60.20894241333008,
      "activations/layer15_attention_weight_max": 57.54341125488281,
      "activations/layer15_attention_weight_min": -45.11293411254883,
      "activations/layer16_attention_weight_max": 52.05048370361328,
      "activations/layer16_attention_weight_min": -38.91014099121094,
      "activations/layer17_attention_weight_max": 33.493263244628906,
      "activations/layer17_attention_weight_min": -23.924436569213867,
      "activations/layer18_attention_weight_max": 20.700597763061523,
      "activations/layer18_attention_weight_min": -14.412303924560547,
      "activations/layer19_attention_weight_max": 57.62652587890625,
      "activations/layer19_attention_weight_min": -33.94095230102539,
      "activations/layer1_attention_weight_max": 15.689641952514648,
      "activations/layer1_attention_weight_min": -11.56955623626709,
      "activations/layer20_attention_weight_max": 40.07578659057617,
      "activations/layer20_attention_weight_min": -22.532651901245117,
      "activations/layer21_attention_weight_max": 23.379880905151367,
      "activations/layer21_attention_weight_min": -14.456414222717285,
      "activations/layer22_attention_weight_max": 37.635250091552734,
      "activations/layer22_attention_weight_min": -21.46776008605957,
      "activations/layer23_attention_weight_max": 22.570632934570312,
      "activations/layer23_attention_weight_min": -16.53820037841797,
      "activations/layer2_attention_weight_max": 27.492048263549805,
      "activations/layer2_attention_weight_min": -24.603290557861328,
      "activations/layer3_attention_weight_max": 56.011531829833984,
      "activations/layer3_attention_weight_min": -58.475494384765625,
      "activations/layer4_attention_weight_max": 108.8769302368164,
      "activations/layer4_attention_weight_min": -105.76405334472656,
      "activations/layer5_attention_weight_max": 123.21597290039062,
      "activations/layer5_attention_weight_min": -105.7528076171875,
      "activations/layer6_attention_weight_max": 65.90914916992188,
      "activations/layer6_attention_weight_min": -60.80946350097656,
      "activations/layer7_attention_weight_max": 62.14816665649414,
      "activations/layer7_attention_weight_min": -62.580570220947266,
      "activations/layer8_attention_weight_max": 62.61858367919922,
      "activations/layer8_attention_weight_min": -55.6806640625,
      "activations/layer9_attention_weight_max": 69.67130279541016,
      "activations/layer9_attention_weight_min": -60.9041862487793,
      "epoch": 0.93,
      "learning_rate": 0.00014546590909090908,
      "loss": 3.1305,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_loss": 3.064453125,
      "eval_runtime": 8.5151,
      "eval_samples_per_second": 504.278,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_openwebtext_loss": 3.064453125,
      "eval_openwebtext_ppl": 21.42274322042474,
      "eval_openwebtext_runtime": 8.5151,
      "eval_openwebtext_samples_per_second": 504.278,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_wikitext_loss": 3.34765625,
      "eval_wikitext_ppl": 28.436008585761353,
      "eval_wikitext_runtime": 1.9653,
      "eval_wikitext_samples_per_second": 232.024,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_lambada_loss": 3.578125,
      "eval_lambada_ppl": 35.806340977539655,
      "eval_lambada_runtime": 9.5634,
      "eval_lambada_samples_per_second": 509.131,
      "step": 16000
    },
    {
      "activations/layer0_attention_weight_max": 15.851264953613281,
      "activations/layer0_attention_weight_min": -11.902227401733398,
      "activations/layer10_attention_weight_max": 53.73361587524414,
      "activations/layer10_attention_weight_min": -46.22552490234375,
      "activations/layer11_attention_weight_max": 45.06777572631836,
      "activations/layer11_attention_weight_min": -39.213748931884766,
      "activations/layer12_attention_weight_max": 32.37769317626953,
      "activations/layer12_attention_weight_min": -26.017431259155273,
      "activations/layer13_attention_weight_max": 43.74657440185547,
      "activations/layer13_attention_weight_min": -35.82066345214844,
      "activations/layer14_attention_weight_max": 67.69554901123047,
      "activations/layer14_attention_weight_min": -54.58198928833008,
      "activations/layer15_attention_weight_max": 51.590919494628906,
      "activations/layer15_attention_weight_min": -38.07065963745117,
      "activations/layer16_attention_weight_max": 50.99641799926758,
      "activations/layer16_attention_weight_min": -36.63132095336914,
      "activations/layer17_attention_weight_max": 36.02092361450195,
      "activations/layer17_attention_weight_min": -26.39929962158203,
      "activations/layer18_attention_weight_max": 22.18008041381836,
      "activations/layer18_attention_weight_min": -13.344186782836914,
      "activations/layer19_attention_weight_max": 46.226802825927734,
      "activations/layer19_attention_weight_min": -31.852750778198242,
      "activations/layer1_attention_weight_max": 15.850702285766602,
      "activations/layer1_attention_weight_min": -11.431350708007812,
      "activations/layer20_attention_weight_max": 36.9195671081543,
      "activations/layer20_attention_weight_min": -26.012744903564453,
      "activations/layer21_attention_weight_max": 18.411157608032227,
      "activations/layer21_attention_weight_min": -12.791322708129883,
      "activations/layer22_attention_weight_max": 31.91339111328125,
      "activations/layer22_attention_weight_min": -21.339441299438477,
      "activations/layer23_attention_weight_max": 20.705923080444336,
      "activations/layer23_attention_weight_min": -14.2952299118042,
      "activations/layer2_attention_weight_max": 29.04994010925293,
      "activations/layer2_attention_weight_min": -26.997467041015625,
      "activations/layer3_attention_weight_max": 61.61322021484375,
      "activations/layer3_attention_weight_min": -63.857215881347656,
      "activations/layer4_attention_weight_max": 115.45694732666016,
      "activations/layer4_attention_weight_min": -108.43607330322266,
      "activations/layer5_attention_weight_max": 123.92070007324219,
      "activations/layer5_attention_weight_min": -112.73033905029297,
      "activations/layer6_attention_weight_max": 67.68672943115234,
      "activations/layer6_attention_weight_min": -59.69762420654297,
      "activations/layer7_attention_weight_max": 63.68825149536133,
      "activations/layer7_attention_weight_min": -63.356300354003906,
      "activations/layer8_attention_weight_max": 57.02118682861328,
      "activations/layer8_attention_weight_min": -50.04256820678711,
      "activations/layer9_attention_weight_max": 68.66276550292969,
      "activations/layer9_attention_weight_min": -65.33614349365234,
      "epoch": 0.93,
      "learning_rate": 0.0001454469696969697,
      "loss": 3.1188,
      "step": 16050
    },
    {
      "activations/layer0_attention_weight_max": 13.591739654541016,
      "activations/layer0_attention_weight_min": -11.067238807678223,
      "activations/layer10_attention_weight_max": 63.588863372802734,
      "activations/layer10_attention_weight_min": -53.61954879760742,
      "activations/layer11_attention_weight_max": 41.12876892089844,
      "activations/layer11_attention_weight_min": -36.91547393798828,
      "activations/layer12_attention_weight_max": 52.01848602294922,
      "activations/layer12_attention_weight_min": -40.53596496582031,
      "activations/layer13_attention_weight_max": 52.47380065917969,
      "activations/layer13_attention_weight_min": -39.76655960083008,
      "activations/layer14_attention_weight_max": 84.84440612792969,
      "activations/layer14_attention_weight_min": -61.86782455444336,
      "activations/layer15_attention_weight_max": 60.41108703613281,
      "activations/layer15_attention_weight_min": -44.739601135253906,
      "activations/layer16_attention_weight_max": 50.06315612792969,
      "activations/layer16_attention_weight_min": -35.7211799621582,
      "activations/layer17_attention_weight_max": 37.8571662902832,
      "activations/layer17_attention_weight_min": -21.563373565673828,
      "activations/layer18_attention_weight_max": 21.649307250976562,
      "activations/layer18_attention_weight_min": -14.764967918395996,
      "activations/layer19_attention_weight_max": 46.41563415527344,
      "activations/layer19_attention_weight_min": -29.423871994018555,
      "activations/layer1_attention_weight_max": 16.232810974121094,
      "activations/layer1_attention_weight_min": -11.750157356262207,
      "activations/layer20_attention_weight_max": 37.97201156616211,
      "activations/layer20_attention_weight_min": -23.71290397644043,
      "activations/layer21_attention_weight_max": 22.61651039123535,
      "activations/layer21_attention_weight_min": -12.62596321105957,
      "activations/layer22_attention_weight_max": 35.9288215637207,
      "activations/layer22_attention_weight_min": -19.668622970581055,
      "activations/layer23_attention_weight_max": 23.247177124023438,
      "activations/layer23_attention_weight_min": -14.86917495727539,
      "activations/layer2_attention_weight_max": 27.45057487487793,
      "activations/layer2_attention_weight_min": -24.753986358642578,
      "activations/layer3_attention_weight_max": 59.099205017089844,
      "activations/layer3_attention_weight_min": -59.1527099609375,
      "activations/layer4_attention_weight_max": 111.66792297363281,
      "activations/layer4_attention_weight_min": -105.67498779296875,
      "activations/layer5_attention_weight_max": 129.197998046875,
      "activations/layer5_attention_weight_min": -117.75790405273438,
      "activations/layer6_attention_weight_max": 67.38545227050781,
      "activations/layer6_attention_weight_min": -64.4966049194336,
      "activations/layer7_attention_weight_max": 62.795310974121094,
      "activations/layer7_attention_weight_min": -70.45729064941406,
      "activations/layer8_attention_weight_max": 66.45458984375,
      "activations/layer8_attention_weight_min": -59.4239387512207,
      "activations/layer9_attention_weight_max": 72.93746185302734,
      "activations/layer9_attention_weight_min": -65.78716278076172,
      "epoch": 0.94,
      "learning_rate": 0.00014542803030303029,
      "loss": 3.1199,
      "step": 16100
    },
    {
      "activations/layer0_attention_weight_max": 15.189659118652344,
      "activations/layer0_attention_weight_min": -11.218356132507324,
      "activations/layer10_attention_weight_max": 62.36960983276367,
      "activations/layer10_attention_weight_min": -53.42186737060547,
      "activations/layer11_attention_weight_max": 49.11076354980469,
      "activations/layer11_attention_weight_min": -38.93196105957031,
      "activations/layer12_attention_weight_max": 84.05970764160156,
      "activations/layer12_attention_weight_min": -41.874725341796875,
      "activations/layer13_attention_weight_max": 66.64363861083984,
      "activations/layer13_attention_weight_min": -39.629371643066406,
      "activations/layer14_attention_weight_max": 78.09809112548828,
      "activations/layer14_attention_weight_min": -54.30955505371094,
      "activations/layer15_attention_weight_max": 50.25638961791992,
      "activations/layer15_attention_weight_min": -34.822105407714844,
      "activations/layer16_attention_weight_max": 58.10010528564453,
      "activations/layer16_attention_weight_min": -38.48885726928711,
      "activations/layer17_attention_weight_max": 36.91233444213867,
      "activations/layer17_attention_weight_min": -23.320459365844727,
      "activations/layer18_attention_weight_max": 24.91868782043457,
      "activations/layer18_attention_weight_min": -13.594094276428223,
      "activations/layer19_attention_weight_max": 50.43288803100586,
      "activations/layer19_attention_weight_min": -31.299911499023438,
      "activations/layer1_attention_weight_max": 17.27712059020996,
      "activations/layer1_attention_weight_min": -11.394420623779297,
      "activations/layer20_attention_weight_max": 38.55080032348633,
      "activations/layer20_attention_weight_min": -21.970502853393555,
      "activations/layer21_attention_weight_max": 20.678491592407227,
      "activations/layer21_attention_weight_min": -13.505403518676758,
      "activations/layer22_attention_weight_max": 40.95930480957031,
      "activations/layer22_attention_weight_min": -22.69865608215332,
      "activations/layer23_attention_weight_max": 26.546977996826172,
      "activations/layer23_attention_weight_min": -13.666572570800781,
      "activations/layer2_attention_weight_max": 26.87801170349121,
      "activations/layer2_attention_weight_min": -27.109289169311523,
      "activations/layer3_attention_weight_max": 58.676048278808594,
      "activations/layer3_attention_weight_min": -60.58213424682617,
      "activations/layer4_attention_weight_max": 109.80266571044922,
      "activations/layer4_attention_weight_min": -105.11820220947266,
      "activations/layer5_attention_weight_max": 134.94845581054688,
      "activations/layer5_attention_weight_min": -113.01564025878906,
      "activations/layer6_attention_weight_max": 71.70330047607422,
      "activations/layer6_attention_weight_min": -58.363555908203125,
      "activations/layer7_attention_weight_max": 76.23275756835938,
      "activations/layer7_attention_weight_min": -66.23004150390625,
      "activations/layer8_attention_weight_max": 69.38583374023438,
      "activations/layer8_attention_weight_min": -52.731258392333984,
      "activations/layer9_attention_weight_max": 106.5558853149414,
      "activations/layer9_attention_weight_min": -75.35751342773438,
      "epoch": 0.94,
      "learning_rate": 0.0001454090909090909,
      "loss": 3.1185,
      "step": 16150
    },
    {
      "activations/layer0_attention_weight_max": 15.781573295593262,
      "activations/layer0_attention_weight_min": -11.069986343383789,
      "activations/layer10_attention_weight_max": 55.60117721557617,
      "activations/layer10_attention_weight_min": -49.21928024291992,
      "activations/layer11_attention_weight_max": 47.72199249267578,
      "activations/layer11_attention_weight_min": -37.578590393066406,
      "activations/layer12_attention_weight_max": 38.68625259399414,
      "activations/layer12_attention_weight_min": -27.68369483947754,
      "activations/layer13_attention_weight_max": 45.804283142089844,
      "activations/layer13_attention_weight_min": -38.30028533935547,
      "activations/layer14_attention_weight_max": 68.7842788696289,
      "activations/layer14_attention_weight_min": -50.184574127197266,
      "activations/layer15_attention_weight_max": 52.280296325683594,
      "activations/layer15_attention_weight_min": -35.95143127441406,
      "activations/layer16_attention_weight_max": 63.33915328979492,
      "activations/layer16_attention_weight_min": -41.869224548339844,
      "activations/layer17_attention_weight_max": 39.516929626464844,
      "activations/layer17_attention_weight_min": -22.049510955810547,
      "activations/layer18_attention_weight_max": 21.779211044311523,
      "activations/layer18_attention_weight_min": -14.289811134338379,
      "activations/layer19_attention_weight_max": 50.70772933959961,
      "activations/layer19_attention_weight_min": -29.764881134033203,
      "activations/layer1_attention_weight_max": 16.69844627380371,
      "activations/layer1_attention_weight_min": -12.5682954788208,
      "activations/layer20_attention_weight_max": 38.04698181152344,
      "activations/layer20_attention_weight_min": -22.748380661010742,
      "activations/layer21_attention_weight_max": 19.255443572998047,
      "activations/layer21_attention_weight_min": -14.03754997253418,
      "activations/layer22_attention_weight_max": 37.9825439453125,
      "activations/layer22_attention_weight_min": -23.05586814880371,
      "activations/layer23_attention_weight_max": 24.636619567871094,
      "activations/layer23_attention_weight_min": -14.662942886352539,
      "activations/layer2_attention_weight_max": 28.793292999267578,
      "activations/layer2_attention_weight_min": -25.030073165893555,
      "activations/layer3_attention_weight_max": 56.09156036376953,
      "activations/layer3_attention_weight_min": -55.105796813964844,
      "activations/layer4_attention_weight_max": 109.12395477294922,
      "activations/layer4_attention_weight_min": -103.35318756103516,
      "activations/layer5_attention_weight_max": 125.75712585449219,
      "activations/layer5_attention_weight_min": -110.67951965332031,
      "activations/layer6_attention_weight_max": 64.42583465576172,
      "activations/layer6_attention_weight_min": -61.26124954223633,
      "activations/layer7_attention_weight_max": 61.41743087768555,
      "activations/layer7_attention_weight_min": -66.13645935058594,
      "activations/layer8_attention_weight_max": 56.33323287963867,
      "activations/layer8_attention_weight_min": -49.560874938964844,
      "activations/layer9_attention_weight_max": 67.689453125,
      "activations/layer9_attention_weight_min": -61.23848342895508,
      "epoch": 0.94,
      "learning_rate": 0.00014539015151515152,
      "loss": 3.1252,
      "step": 16200
    },
    {
      "activations/layer0_attention_weight_max": 14.771007537841797,
      "activations/layer0_attention_weight_min": -11.511557579040527,
      "activations/layer10_attention_weight_max": 53.57513427734375,
      "activations/layer10_attention_weight_min": -47.09663772583008,
      "activations/layer11_attention_weight_max": 46.54620361328125,
      "activations/layer11_attention_weight_min": -39.922828674316406,
      "activations/layer12_attention_weight_max": 34.88221740722656,
      "activations/layer12_attention_weight_min": -27.91275978088379,
      "activations/layer13_attention_weight_max": 53.51035690307617,
      "activations/layer13_attention_weight_min": -35.87588119506836,
      "activations/layer14_attention_weight_max": 70.08392333984375,
      "activations/layer14_attention_weight_min": -50.721351623535156,
      "activations/layer15_attention_weight_max": 52.659393310546875,
      "activations/layer15_attention_weight_min": -37.558746337890625,
      "activations/layer16_attention_weight_max": 52.847923278808594,
      "activations/layer16_attention_weight_min": -36.563941955566406,
      "activations/layer17_attention_weight_max": 38.488792419433594,
      "activations/layer17_attention_weight_min": -24.566503524780273,
      "activations/layer18_attention_weight_max": 22.741613388061523,
      "activations/layer18_attention_weight_min": -14.249734878540039,
      "activations/layer19_attention_weight_max": 46.7763671875,
      "activations/layer19_attention_weight_min": -31.381216049194336,
      "activations/layer1_attention_weight_max": 16.028833389282227,
      "activations/layer1_attention_weight_min": -11.537325859069824,
      "activations/layer20_attention_weight_max": 41.359596252441406,
      "activations/layer20_attention_weight_min": -22.86211585998535,
      "activations/layer21_attention_weight_max": 20.781091690063477,
      "activations/layer21_attention_weight_min": -12.272364616394043,
      "activations/layer22_attention_weight_max": 40.21253204345703,
      "activations/layer22_attention_weight_min": -18.981164932250977,
      "activations/layer23_attention_weight_max": 26.695449829101562,
      "activations/layer23_attention_weight_min": -14.525317192077637,
      "activations/layer2_attention_weight_max": 27.429723739624023,
      "activations/layer2_attention_weight_min": -26.39021873474121,
      "activations/layer3_attention_weight_max": 59.99766159057617,
      "activations/layer3_attention_weight_min": -60.29801940917969,
      "activations/layer4_attention_weight_max": 110.72661590576172,
      "activations/layer4_attention_weight_min": -105.39546203613281,
      "activations/layer5_attention_weight_max": 120.06552124023438,
      "activations/layer5_attention_weight_min": -114.81443786621094,
      "activations/layer6_attention_weight_max": 64.29190826416016,
      "activations/layer6_attention_weight_min": -60.222110748291016,
      "activations/layer7_attention_weight_max": 63.81643295288086,
      "activations/layer7_attention_weight_min": -66.38153076171875,
      "activations/layer8_attention_weight_max": 56.97945022583008,
      "activations/layer8_attention_weight_min": -51.25065231323242,
      "activations/layer9_attention_weight_max": 69.1689453125,
      "activations/layer9_attention_weight_min": -61.59716796875,
      "epoch": 0.94,
      "learning_rate": 0.0001453712121212121,
      "loss": 3.1082,
      "step": 16250
    },
    {
      "activations/layer0_attention_weight_max": 13.882534980773926,
      "activations/layer0_attention_weight_min": -11.492303848266602,
      "activations/layer10_attention_weight_max": 52.98979949951172,
      "activations/layer10_attention_weight_min": -48.54668045043945,
      "activations/layer11_attention_weight_max": 43.29383850097656,
      "activations/layer11_attention_weight_min": -40.33081817626953,
      "activations/layer12_attention_weight_max": 32.75775146484375,
      "activations/layer12_attention_weight_min": -28.123701095581055,
      "activations/layer13_attention_weight_max": 48.84384536743164,
      "activations/layer13_attention_weight_min": -36.231651306152344,
      "activations/layer14_attention_weight_max": 74.30049896240234,
      "activations/layer14_attention_weight_min": -56.996734619140625,
      "activations/layer15_attention_weight_max": 49.045101165771484,
      "activations/layer15_attention_weight_min": -39.82783508300781,
      "activations/layer16_attention_weight_max": 52.16437530517578,
      "activations/layer16_attention_weight_min": -41.008201599121094,
      "activations/layer17_attention_weight_max": 32.639801025390625,
      "activations/layer17_attention_weight_min": -23.18214225769043,
      "activations/layer18_attention_weight_max": 19.668182373046875,
      "activations/layer18_attention_weight_min": -13.742420196533203,
      "activations/layer19_attention_weight_max": 48.27616500854492,
      "activations/layer19_attention_weight_min": -30.987247467041016,
      "activations/layer1_attention_weight_max": 15.509300231933594,
      "activations/layer1_attention_weight_min": -11.858686447143555,
      "activations/layer20_attention_weight_max": 36.60469055175781,
      "activations/layer20_attention_weight_min": -21.427021026611328,
      "activations/layer21_attention_weight_max": 19.343664169311523,
      "activations/layer21_attention_weight_min": -13.76574993133545,
      "activations/layer22_attention_weight_max": 35.16207504272461,
      "activations/layer22_attention_weight_min": -21.529064178466797,
      "activations/layer23_attention_weight_max": 23.71706199645996,
      "activations/layer23_attention_weight_min": -16.207759857177734,
      "activations/layer2_attention_weight_max": 28.222999572753906,
      "activations/layer2_attention_weight_min": -25.65704917907715,
      "activations/layer3_attention_weight_max": 59.01499557495117,
      "activations/layer3_attention_weight_min": -60.19824981689453,
      "activations/layer4_attention_weight_max": 109.96736907958984,
      "activations/layer4_attention_weight_min": -107.53797912597656,
      "activations/layer5_attention_weight_max": 120.17797088623047,
      "activations/layer5_attention_weight_min": -117.61343383789062,
      "activations/layer6_attention_weight_max": 68.09376525878906,
      "activations/layer6_attention_weight_min": -64.620361328125,
      "activations/layer7_attention_weight_max": 63.80867385864258,
      "activations/layer7_attention_weight_min": -67.76130676269531,
      "activations/layer8_attention_weight_max": 58.89666748046875,
      "activations/layer8_attention_weight_min": -54.9986572265625,
      "activations/layer9_attention_weight_max": 75.94377899169922,
      "activations/layer9_attention_weight_min": -58.47711944580078,
      "epoch": 0.95,
      "learning_rate": 0.00014535227272727272,
      "loss": 3.1198,
      "step": 16300
    },
    {
      "activations/layer0_attention_weight_max": 14.468873023986816,
      "activations/layer0_attention_weight_min": -11.275792121887207,
      "activations/layer10_attention_weight_max": 52.32781982421875,
      "activations/layer10_attention_weight_min": -46.49116897583008,
      "activations/layer11_attention_weight_max": 45.133636474609375,
      "activations/layer11_attention_weight_min": -36.57117462158203,
      "activations/layer12_attention_weight_max": 46.008758544921875,
      "activations/layer12_attention_weight_min": -36.89137649536133,
      "activations/layer13_attention_weight_max": 50.902381896972656,
      "activations/layer13_attention_weight_min": -40.09675598144531,
      "activations/layer14_attention_weight_max": 71.52931213378906,
      "activations/layer14_attention_weight_min": -54.81382751464844,
      "activations/layer15_attention_weight_max": 53.09021759033203,
      "activations/layer15_attention_weight_min": -38.122886657714844,
      "activations/layer16_attention_weight_max": 51.7274055480957,
      "activations/layer16_attention_weight_min": -36.90351486206055,
      "activations/layer17_attention_weight_max": 32.88142013549805,
      "activations/layer17_attention_weight_min": -23.440011978149414,
      "activations/layer18_attention_weight_max": 24.34299087524414,
      "activations/layer18_attention_weight_min": -13.647817611694336,
      "activations/layer19_attention_weight_max": 51.49245071411133,
      "activations/layer19_attention_weight_min": -27.90829849243164,
      "activations/layer1_attention_weight_max": 16.154876708984375,
      "activations/layer1_attention_weight_min": -11.97851848602295,
      "activations/layer20_attention_weight_max": 42.231876373291016,
      "activations/layer20_attention_weight_min": -23.178478240966797,
      "activations/layer21_attention_weight_max": 22.162189483642578,
      "activations/layer21_attention_weight_min": -13.506185531616211,
      "activations/layer22_attention_weight_max": 39.05648422241211,
      "activations/layer22_attention_weight_min": -22.667634963989258,
      "activations/layer23_attention_weight_max": 25.16285514831543,
      "activations/layer23_attention_weight_min": -13.877692222595215,
      "activations/layer2_attention_weight_max": 28.925689697265625,
      "activations/layer2_attention_weight_min": -26.43819808959961,
      "activations/layer3_attention_weight_max": 64.66490173339844,
      "activations/layer3_attention_weight_min": -64.5794677734375,
      "activations/layer4_attention_weight_max": 119.07604217529297,
      "activations/layer4_attention_weight_min": -107.3990249633789,
      "activations/layer5_attention_weight_max": 136.01466369628906,
      "activations/layer5_attention_weight_min": -116.1363754272461,
      "activations/layer6_attention_weight_max": 69.99657440185547,
      "activations/layer6_attention_weight_min": -58.22822952270508,
      "activations/layer7_attention_weight_max": 67.27476501464844,
      "activations/layer7_attention_weight_min": -62.315608978271484,
      "activations/layer8_attention_weight_max": 59.02420425415039,
      "activations/layer8_attention_weight_min": -50.02024841308594,
      "activations/layer9_attention_weight_max": 70.38275146484375,
      "activations/layer9_attention_weight_min": -67.86695098876953,
      "epoch": 0.95,
      "learning_rate": 0.00014533333333333333,
      "loss": 3.1208,
      "step": 16350
    },
    {
      "activations/layer0_attention_weight_max": 14.953709602355957,
      "activations/layer0_attention_weight_min": -11.282661437988281,
      "activations/layer10_attention_weight_max": 47.381805419921875,
      "activations/layer10_attention_weight_min": -45.57976531982422,
      "activations/layer11_attention_weight_max": 41.60120391845703,
      "activations/layer11_attention_weight_min": -40.246803283691406,
      "activations/layer12_attention_weight_max": 36.89772415161133,
      "activations/layer12_attention_weight_min": -31.151111602783203,
      "activations/layer13_attention_weight_max": 47.47610092163086,
      "activations/layer13_attention_weight_min": -37.254791259765625,
      "activations/layer14_attention_weight_max": 67.39736938476562,
      "activations/layer14_attention_weight_min": -56.452301025390625,
      "activations/layer15_attention_weight_max": 45.046600341796875,
      "activations/layer15_attention_weight_min": -39.94868850708008,
      "activations/layer16_attention_weight_max": 43.23550796508789,
      "activations/layer16_attention_weight_min": -37.244911193847656,
      "activations/layer17_attention_weight_max": 31.899911880493164,
      "activations/layer17_attention_weight_min": -24.291704177856445,
      "activations/layer18_attention_weight_max": 20.120464324951172,
      "activations/layer18_attention_weight_min": -14.062835693359375,
      "activations/layer19_attention_weight_max": 42.61039733886719,
      "activations/layer19_attention_weight_min": -29.479211807250977,
      "activations/layer1_attention_weight_max": 16.47124481201172,
      "activations/layer1_attention_weight_min": -11.619312286376953,
      "activations/layer20_attention_weight_max": 36.8438606262207,
      "activations/layer20_attention_weight_min": -23.273906707763672,
      "activations/layer21_attention_weight_max": 17.023523330688477,
      "activations/layer21_attention_weight_min": -15.596627235412598,
      "activations/layer22_attention_weight_max": 31.6357479095459,
      "activations/layer22_attention_weight_min": -20.99956703186035,
      "activations/layer23_attention_weight_max": 21.634855270385742,
      "activations/layer23_attention_weight_min": -15.137628555297852,
      "activations/layer2_attention_weight_max": 27.571563720703125,
      "activations/layer2_attention_weight_min": -26.22745132446289,
      "activations/layer3_attention_weight_max": 58.51902389526367,
      "activations/layer3_attention_weight_min": -60.92078399658203,
      "activations/layer4_attention_weight_max": 111.1689682006836,
      "activations/layer4_attention_weight_min": -103.73431396484375,
      "activations/layer5_attention_weight_max": 127.32781982421875,
      "activations/layer5_attention_weight_min": -108.45741271972656,
      "activations/layer6_attention_weight_max": 66.86482238769531,
      "activations/layer6_attention_weight_min": -59.170799255371094,
      "activations/layer7_attention_weight_max": 64.20579528808594,
      "activations/layer7_attention_weight_min": -63.37189865112305,
      "activations/layer8_attention_weight_max": 57.87643814086914,
      "activations/layer8_attention_weight_min": -50.79888916015625,
      "activations/layer9_attention_weight_max": 72.06117248535156,
      "activations/layer9_attention_weight_min": -66.46212005615234,
      "epoch": 0.95,
      "learning_rate": 0.00014531439393939392,
      "loss": 3.1315,
      "step": 16400
    },
    {
      "activations/layer0_attention_weight_max": 15.533371925354004,
      "activations/layer0_attention_weight_min": -11.536168098449707,
      "activations/layer10_attention_weight_max": 54.34868621826172,
      "activations/layer10_attention_weight_min": -47.93292236328125,
      "activations/layer11_attention_weight_max": 52.21569061279297,
      "activations/layer11_attention_weight_min": -38.05327606201172,
      "activations/layer12_attention_weight_max": 43.24125289916992,
      "activations/layer12_attention_weight_min": -31.569990158081055,
      "activations/layer13_attention_weight_max": 48.342891693115234,
      "activations/layer13_attention_weight_min": -39.78819274902344,
      "activations/layer14_attention_weight_max": 72.626708984375,
      "activations/layer14_attention_weight_min": -51.48664474487305,
      "activations/layer15_attention_weight_max": 58.141273498535156,
      "activations/layer15_attention_weight_min": -36.620914459228516,
      "activations/layer16_attention_weight_max": 64.49781036376953,
      "activations/layer16_attention_weight_min": -42.94046401977539,
      "activations/layer17_attention_weight_max": 37.319374084472656,
      "activations/layer17_attention_weight_min": -23.054094314575195,
      "activations/layer18_attention_weight_max": 24.99520492553711,
      "activations/layer18_attention_weight_min": -17.108253479003906,
      "activations/layer19_attention_weight_max": 50.836055755615234,
      "activations/layer19_attention_weight_min": -32.34605026245117,
      "activations/layer1_attention_weight_max": 16.22119140625,
      "activations/layer1_attention_weight_min": -12.32892894744873,
      "activations/layer20_attention_weight_max": 40.89110565185547,
      "activations/layer20_attention_weight_min": -23.01382064819336,
      "activations/layer21_attention_weight_max": 22.359407424926758,
      "activations/layer21_attention_weight_min": -14.826579093933105,
      "activations/layer22_attention_weight_max": 43.56575012207031,
      "activations/layer22_attention_weight_min": -19.912700653076172,
      "activations/layer23_attention_weight_max": 29.8059139251709,
      "activations/layer23_attention_weight_min": -16.992643356323242,
      "activations/layer2_attention_weight_max": 28.170995712280273,
      "activations/layer2_attention_weight_min": -25.941617965698242,
      "activations/layer3_attention_weight_max": 59.482418060302734,
      "activations/layer3_attention_weight_min": -59.291744232177734,
      "activations/layer4_attention_weight_max": 110.99837493896484,
      "activations/layer4_attention_weight_min": -100.7358169555664,
      "activations/layer5_attention_weight_max": 127.42333221435547,
      "activations/layer5_attention_weight_min": -109.65998840332031,
      "activations/layer6_attention_weight_max": 65.71957397460938,
      "activations/layer6_attention_weight_min": -59.17940139770508,
      "activations/layer7_attention_weight_max": 71.07804107666016,
      "activations/layer7_attention_weight_min": -62.65419387817383,
      "activations/layer8_attention_weight_max": 61.81721878051758,
      "activations/layer8_attention_weight_min": -58.61897659301758,
      "activations/layer9_attention_weight_max": 80.77266693115234,
      "activations/layer9_attention_weight_min": -62.13045883178711,
      "epoch": 0.96,
      "learning_rate": 0.00014529545454545454,
      "loss": 3.1202,
      "step": 16450
    },
    {
      "activations/layer0_attention_weight_max": 15.342218399047852,
      "activations/layer0_attention_weight_min": -11.501015663146973,
      "activations/layer10_attention_weight_max": 69.2698974609375,
      "activations/layer10_attention_weight_min": -49.45932388305664,
      "activations/layer11_attention_weight_max": 48.96262741088867,
      "activations/layer11_attention_weight_min": -37.33927917480469,
      "activations/layer12_attention_weight_max": 63.65870666503906,
      "activations/layer12_attention_weight_min": -34.87842559814453,
      "activations/layer13_attention_weight_max": 64.4004135131836,
      "activations/layer13_attention_weight_min": -40.39636993408203,
      "activations/layer14_attention_weight_max": 72.69744873046875,
      "activations/layer14_attention_weight_min": -53.27858352661133,
      "activations/layer15_attention_weight_max": 57.40887451171875,
      "activations/layer15_attention_weight_min": -37.886505126953125,
      "activations/layer16_attention_weight_max": 53.30557632446289,
      "activations/layer16_attention_weight_min": -39.127017974853516,
      "activations/layer17_attention_weight_max": 35.55281448364258,
      "activations/layer17_attention_weight_min": -20.83902931213379,
      "activations/layer18_attention_weight_max": 24.13144874572754,
      "activations/layer18_attention_weight_min": -12.965594291687012,
      "activations/layer19_attention_weight_max": 44.50224685668945,
      "activations/layer19_attention_weight_min": -29.36687660217285,
      "activations/layer1_attention_weight_max": 17.1756649017334,
      "activations/layer1_attention_weight_min": -12.046210289001465,
      "activations/layer20_attention_weight_max": 39.88907241821289,
      "activations/layer20_attention_weight_min": -21.05320167541504,
      "activations/layer21_attention_weight_max": 22.910120010375977,
      "activations/layer21_attention_weight_min": -13.25446605682373,
      "activations/layer22_attention_weight_max": 44.47774124145508,
      "activations/layer22_attention_weight_min": -22.70793914794922,
      "activations/layer23_attention_weight_max": 25.49197006225586,
      "activations/layer23_attention_weight_min": -15.474647521972656,
      "activations/layer2_attention_weight_max": 29.905447006225586,
      "activations/layer2_attention_weight_min": -26.759136199951172,
      "activations/layer3_attention_weight_max": 60.950138092041016,
      "activations/layer3_attention_weight_min": -58.76775360107422,
      "activations/layer4_attention_weight_max": 116.91776275634766,
      "activations/layer4_attention_weight_min": -104.3286361694336,
      "activations/layer5_attention_weight_max": 139.8626708984375,
      "activations/layer5_attention_weight_min": -115.88945770263672,
      "activations/layer6_attention_weight_max": 74.58088684082031,
      "activations/layer6_attention_weight_min": -60.9217529296875,
      "activations/layer7_attention_weight_max": 68.53897094726562,
      "activations/layer7_attention_weight_min": -63.08732604980469,
      "activations/layer8_attention_weight_max": 71.77007293701172,
      "activations/layer8_attention_weight_min": -53.10237503051758,
      "activations/layer9_attention_weight_max": 95.59429931640625,
      "activations/layer9_attention_weight_min": -70.4764633178711,
      "epoch": 0.96,
      "learning_rate": 0.00014527689393939393,
      "loss": 3.1053,
      "step": 16500
    },
    {
      "activations/layer0_attention_weight_max": 16.650827407836914,
      "activations/layer0_attention_weight_min": -11.417475700378418,
      "activations/layer10_attention_weight_max": 70.4937973022461,
      "activations/layer10_attention_weight_min": -54.73846435546875,
      "activations/layer11_attention_weight_max": 49.92528533935547,
      "activations/layer11_attention_weight_min": -39.42045593261719,
      "activations/layer12_attention_weight_max": 58.85731887817383,
      "activations/layer12_attention_weight_min": -33.611968994140625,
      "activations/layer13_attention_weight_max": 79.54360961914062,
      "activations/layer13_attention_weight_min": -42.15531921386719,
      "activations/layer14_attention_weight_max": 99.57760620117188,
      "activations/layer14_attention_weight_min": -54.42073059082031,
      "activations/layer15_attention_weight_max": 69.00814819335938,
      "activations/layer15_attention_weight_min": -38.148284912109375,
      "activations/layer16_attention_weight_max": 65.91791534423828,
      "activations/layer16_attention_weight_min": -38.92852020263672,
      "activations/layer17_attention_weight_max": 41.04113006591797,
      "activations/layer17_attention_weight_min": -23.0654354095459,
      "activations/layer18_attention_weight_max": 25.802072525024414,
      "activations/layer18_attention_weight_min": -15.560103416442871,
      "activations/layer19_attention_weight_max": 53.2436637878418,
      "activations/layer19_attention_weight_min": -30.9897403717041,
      "activations/layer1_attention_weight_max": 16.592424392700195,
      "activations/layer1_attention_weight_min": -12.166786193847656,
      "activations/layer20_attention_weight_max": 44.30461120605469,
      "activations/layer20_attention_weight_min": -24.551286697387695,
      "activations/layer21_attention_weight_max": 20.555917739868164,
      "activations/layer21_attention_weight_min": -14.38110637664795,
      "activations/layer22_attention_weight_max": 42.8145637512207,
      "activations/layer22_attention_weight_min": -23.87497901916504,
      "activations/layer23_attention_weight_max": 25.3110408782959,
      "activations/layer23_attention_weight_min": -17.21091079711914,
      "activations/layer2_attention_weight_max": 28.41390609741211,
      "activations/layer2_attention_weight_min": -26.755695343017578,
      "activations/layer3_attention_weight_max": 59.64866256713867,
      "activations/layer3_attention_weight_min": -62.64421844482422,
      "activations/layer4_attention_weight_max": 110.23624420166016,
      "activations/layer4_attention_weight_min": -104.86710357666016,
      "activations/layer5_attention_weight_max": 134.0640106201172,
      "activations/layer5_attention_weight_min": -114.41014099121094,
      "activations/layer6_attention_weight_max": 73.98197174072266,
      "activations/layer6_attention_weight_min": -59.505313873291016,
      "activations/layer7_attention_weight_max": 68.8111343383789,
      "activations/layer7_attention_weight_min": -66.86016082763672,
      "activations/layer8_attention_weight_max": 60.79530334472656,
      "activations/layer8_attention_weight_min": -52.59685134887695,
      "activations/layer9_attention_weight_max": 86.93737030029297,
      "activations/layer9_attention_weight_min": -64.58309173583984,
      "epoch": 0.96,
      "learning_rate": 0.00014525795454545454,
      "loss": 3.1124,
      "step": 16550
    },
    {
      "activations/layer0_attention_weight_max": 13.58953857421875,
      "activations/layer0_attention_weight_min": -11.684534072875977,
      "activations/layer10_attention_weight_max": 53.439964294433594,
      "activations/layer10_attention_weight_min": -47.668697357177734,
      "activations/layer11_attention_weight_max": 43.37494659423828,
      "activations/layer11_attention_weight_min": -37.21235275268555,
      "activations/layer12_attention_weight_max": 32.69194793701172,
      "activations/layer12_attention_weight_min": -26.313377380371094,
      "activations/layer13_attention_weight_max": 48.66951370239258,
      "activations/layer13_attention_weight_min": -38.867774963378906,
      "activations/layer14_attention_weight_max": 72.50967407226562,
      "activations/layer14_attention_weight_min": -55.98574447631836,
      "activations/layer15_attention_weight_max": 52.86458969116211,
      "activations/layer15_attention_weight_min": -38.16145706176758,
      "activations/layer16_attention_weight_max": 56.813934326171875,
      "activations/layer16_attention_weight_min": -41.09226608276367,
      "activations/layer17_attention_weight_max": 34.58469009399414,
      "activations/layer17_attention_weight_min": -23.231441497802734,
      "activations/layer18_attention_weight_max": 22.357799530029297,
      "activations/layer18_attention_weight_min": -12.39631175994873,
      "activations/layer19_attention_weight_max": 47.72917175292969,
      "activations/layer19_attention_weight_min": -29.314722061157227,
      "activations/layer1_attention_weight_max": 17.43552589416504,
      "activations/layer1_attention_weight_min": -12.218132972717285,
      "activations/layer20_attention_weight_max": 38.54487991333008,
      "activations/layer20_attention_weight_min": -24.27168083190918,
      "activations/layer21_attention_weight_max": 21.44622802734375,
      "activations/layer21_attention_weight_min": -13.950129508972168,
      "activations/layer22_attention_weight_max": 39.51901626586914,
      "activations/layer22_attention_weight_min": -19.555194854736328,
      "activations/layer23_attention_weight_max": 24.75531768798828,
      "activations/layer23_attention_weight_min": -14.323443412780762,
      "activations/layer2_attention_weight_max": 27.820547103881836,
      "activations/layer2_attention_weight_min": -25.558334350585938,
      "activations/layer3_attention_weight_max": 60.21267318725586,
      "activations/layer3_attention_weight_min": -61.468685150146484,
      "activations/layer4_attention_weight_max": 111.4971923828125,
      "activations/layer4_attention_weight_min": -106.7092056274414,
      "activations/layer5_attention_weight_max": 127.85112762451172,
      "activations/layer5_attention_weight_min": -112.75656127929688,
      "activations/layer6_attention_weight_max": 67.21497344970703,
      "activations/layer6_attention_weight_min": -61.10845947265625,
      "activations/layer7_attention_weight_max": 61.94624710083008,
      "activations/layer7_attention_weight_min": -63.3355712890625,
      "activations/layer8_attention_weight_max": 62.61714172363281,
      "activations/layer8_attention_weight_min": -54.55314636230469,
      "activations/layer9_attention_weight_max": 69.20761108398438,
      "activations/layer9_attention_weight_min": -70.7452392578125,
      "epoch": 0.96,
      "learning_rate": 0.00014523901515151513,
      "loss": 3.1311,
      "step": 16600
    },
    {
      "activations/layer0_attention_weight_max": 15.516288757324219,
      "activations/layer0_attention_weight_min": -11.105889320373535,
      "activations/layer10_attention_weight_max": 64.94326782226562,
      "activations/layer10_attention_weight_min": -61.26396942138672,
      "activations/layer11_attention_weight_max": 45.47168731689453,
      "activations/layer11_attention_weight_min": -38.126922607421875,
      "activations/layer12_attention_weight_max": 44.096839904785156,
      "activations/layer12_attention_weight_min": -29.88850212097168,
      "activations/layer13_attention_weight_max": 62.66039276123047,
      "activations/layer13_attention_weight_min": -45.98043441772461,
      "activations/layer14_attention_weight_max": 80.18452453613281,
      "activations/layer14_attention_weight_min": -55.26141357421875,
      "activations/layer15_attention_weight_max": 69.17817687988281,
      "activations/layer15_attention_weight_min": -46.16459274291992,
      "activations/layer16_attention_weight_max": 57.218040466308594,
      "activations/layer16_attention_weight_min": -42.229976654052734,
      "activations/layer17_attention_weight_max": 37.178504943847656,
      "activations/layer17_attention_weight_min": -24.8280029296875,
      "activations/layer18_attention_weight_max": 23.359403610229492,
      "activations/layer18_attention_weight_min": -14.790911674499512,
      "activations/layer19_attention_weight_max": 51.80941390991211,
      "activations/layer19_attention_weight_min": -30.265466690063477,
      "activations/layer1_attention_weight_max": 16.14647674560547,
      "activations/layer1_attention_weight_min": -11.718709945678711,
      "activations/layer20_attention_weight_max": 49.74095153808594,
      "activations/layer20_attention_weight_min": -23.735136032104492,
      "activations/layer21_attention_weight_max": 24.713125228881836,
      "activations/layer21_attention_weight_min": -15.005325317382812,
      "activations/layer22_attention_weight_max": 40.96173095703125,
      "activations/layer22_attention_weight_min": -21.523136138916016,
      "activations/layer23_attention_weight_max": 25.694961547851562,
      "activations/layer23_attention_weight_min": -14.144392967224121,
      "activations/layer2_attention_weight_max": 29.780546188354492,
      "activations/layer2_attention_weight_min": -25.637165069580078,
      "activations/layer3_attention_weight_max": 59.932796478271484,
      "activations/layer3_attention_weight_min": -61.22891616821289,
      "activations/layer4_attention_weight_max": 111.84071350097656,
      "activations/layer4_attention_weight_min": -103.54329681396484,
      "activations/layer5_attention_weight_max": 123.49002075195312,
      "activations/layer5_attention_weight_min": -108.12669372558594,
      "activations/layer6_attention_weight_max": 64.59711456298828,
      "activations/layer6_attention_weight_min": -56.62392044067383,
      "activations/layer7_attention_weight_max": 63.771907806396484,
      "activations/layer7_attention_weight_min": -59.71385955810547,
      "activations/layer8_attention_weight_max": 63.8883056640625,
      "activations/layer8_attention_weight_min": -53.667327880859375,
      "activations/layer9_attention_weight_max": 68.76919555664062,
      "activations/layer9_attention_weight_min": -60.81486892700195,
      "epoch": 0.97,
      "learning_rate": 0.00014522007575757575,
      "loss": 3.1382,
      "step": 16650
    },
    {
      "activations/layer0_attention_weight_max": 14.1167573928833,
      "activations/layer0_attention_weight_min": -11.387589454650879,
      "activations/layer10_attention_weight_max": 53.26076126098633,
      "activations/layer10_attention_weight_min": -45.04754638671875,
      "activations/layer11_attention_weight_max": 47.00431823730469,
      "activations/layer11_attention_weight_min": -42.557010650634766,
      "activations/layer12_attention_weight_max": 34.5213737487793,
      "activations/layer12_attention_weight_min": -26.513792037963867,
      "activations/layer13_attention_weight_max": 43.870765686035156,
      "activations/layer13_attention_weight_min": -37.71772766113281,
      "activations/layer14_attention_weight_max": 70.0133056640625,
      "activations/layer14_attention_weight_min": -52.11471176147461,
      "activations/layer15_attention_weight_max": 53.06787872314453,
      "activations/layer15_attention_weight_min": -43.24496078491211,
      "activations/layer16_attention_weight_max": 54.46756362915039,
      "activations/layer16_attention_weight_min": -38.41843032836914,
      "activations/layer17_attention_weight_max": 35.60198974609375,
      "activations/layer17_attention_weight_min": -23.049409866333008,
      "activations/layer18_attention_weight_max": 24.264005661010742,
      "activations/layer18_attention_weight_min": -13.600008010864258,
      "activations/layer19_attention_weight_max": 51.49723434448242,
      "activations/layer19_attention_weight_min": -30.670913696289062,
      "activations/layer1_attention_weight_max": 15.587239265441895,
      "activations/layer1_attention_weight_min": -11.932842254638672,
      "activations/layer20_attention_weight_max": 38.15005111694336,
      "activations/layer20_attention_weight_min": -22.151575088500977,
      "activations/layer21_attention_weight_max": 20.154142379760742,
      "activations/layer21_attention_weight_min": -14.647944450378418,
      "activations/layer22_attention_weight_max": 38.952232360839844,
      "activations/layer22_attention_weight_min": -20.529590606689453,
      "activations/layer23_attention_weight_max": 24.28866195678711,
      "activations/layer23_attention_weight_min": -14.63576889038086,
      "activations/layer2_attention_weight_max": 27.816280364990234,
      "activations/layer2_attention_weight_min": -26.750272750854492,
      "activations/layer3_attention_weight_max": 60.33084487915039,
      "activations/layer3_attention_weight_min": -63.17727279663086,
      "activations/layer4_attention_weight_max": 117.52751922607422,
      "activations/layer4_attention_weight_min": -106.70833587646484,
      "activations/layer5_attention_weight_max": 135.12289428710938,
      "activations/layer5_attention_weight_min": -115.00731658935547,
      "activations/layer6_attention_weight_max": 65.60626983642578,
      "activations/layer6_attention_weight_min": -62.65427780151367,
      "activations/layer7_attention_weight_max": 65.21306610107422,
      "activations/layer7_attention_weight_min": -62.44838333129883,
      "activations/layer8_attention_weight_max": 58.26382827758789,
      "activations/layer8_attention_weight_min": -51.415260314941406,
      "activations/layer9_attention_weight_max": 80.00382995605469,
      "activations/layer9_attention_weight_min": -69.938232421875,
      "epoch": 0.97,
      "learning_rate": 0.00014520113636363636,
      "loss": 3.1193,
      "step": 16700
    },
    {
      "activations/layer0_attention_weight_max": 15.720867156982422,
      "activations/layer0_attention_weight_min": -11.351618766784668,
      "activations/layer10_attention_weight_max": 56.29023361206055,
      "activations/layer10_attention_weight_min": -46.100040435791016,
      "activations/layer11_attention_weight_max": 43.875946044921875,
      "activations/layer11_attention_weight_min": -39.90214920043945,
      "activations/layer12_attention_weight_max": 32.08514404296875,
      "activations/layer12_attention_weight_min": -31.778976440429688,
      "activations/layer13_attention_weight_max": 43.072391510009766,
      "activations/layer13_attention_weight_min": -36.27682113647461,
      "activations/layer14_attention_weight_max": 66.92646789550781,
      "activations/layer14_attention_weight_min": -56.3674430847168,
      "activations/layer15_attention_weight_max": 51.93804931640625,
      "activations/layer15_attention_weight_min": -37.59891128540039,
      "activations/layer16_attention_weight_max": 50.54987335205078,
      "activations/layer16_attention_weight_min": -43.24613952636719,
      "activations/layer17_attention_weight_max": 34.642967224121094,
      "activations/layer17_attention_weight_min": -24.319324493408203,
      "activations/layer18_attention_weight_max": 19.52799415588379,
      "activations/layer18_attention_weight_min": -13.494091033935547,
      "activations/layer19_attention_weight_max": 47.32979965209961,
      "activations/layer19_attention_weight_min": -31.821287155151367,
      "activations/layer1_attention_weight_max": 16.298128128051758,
      "activations/layer1_attention_weight_min": -11.481131553649902,
      "activations/layer20_attention_weight_max": 36.21738815307617,
      "activations/layer20_attention_weight_min": -24.910749435424805,
      "activations/layer21_attention_weight_max": 19.80815887451172,
      "activations/layer21_attention_weight_min": -13.75926399230957,
      "activations/layer22_attention_weight_max": 38.78028869628906,
      "activations/layer22_attention_weight_min": -22.975828170776367,
      "activations/layer23_attention_weight_max": 23.114791870117188,
      "activations/layer23_attention_weight_min": -16.853214263916016,
      "activations/layer2_attention_weight_max": 25.3992919921875,
      "activations/layer2_attention_weight_min": -25.241559982299805,
      "activations/layer3_attention_weight_max": 56.356746673583984,
      "activations/layer3_attention_weight_min": -60.541812896728516,
      "activations/layer4_attention_weight_max": 104.6547622680664,
      "activations/layer4_attention_weight_min": -103.45352935791016,
      "activations/layer5_attention_weight_max": 121.54753112792969,
      "activations/layer5_attention_weight_min": -105.05059051513672,
      "activations/layer6_attention_weight_max": 69.56800842285156,
      "activations/layer6_attention_weight_min": -58.23900604248047,
      "activations/layer7_attention_weight_max": 63.69464111328125,
      "activations/layer7_attention_weight_min": -63.25726318359375,
      "activations/layer8_attention_weight_max": 64.84452056884766,
      "activations/layer8_attention_weight_min": -55.63682556152344,
      "activations/layer9_attention_weight_max": 90.14945220947266,
      "activations/layer9_attention_weight_min": -73.39315032958984,
      "epoch": 0.97,
      "learning_rate": 0.00014518219696969695,
      "loss": 3.108,
      "step": 16750
    },
    {
      "activations/layer0_attention_weight_max": 14.675894737243652,
      "activations/layer0_attention_weight_min": -11.062530517578125,
      "activations/layer10_attention_weight_max": 52.12962341308594,
      "activations/layer10_attention_weight_min": -48.094932556152344,
      "activations/layer11_attention_weight_max": 41.65139389038086,
      "activations/layer11_attention_weight_min": -40.553009033203125,
      "activations/layer12_attention_weight_max": 31.379535675048828,
      "activations/layer12_attention_weight_min": -26.810489654541016,
      "activations/layer13_attention_weight_max": 47.57079315185547,
      "activations/layer13_attention_weight_min": -39.03746795654297,
      "activations/layer14_attention_weight_max": 63.746185302734375,
      "activations/layer14_attention_weight_min": -53.52305603027344,
      "activations/layer15_attention_weight_max": 42.605926513671875,
      "activations/layer15_attention_weight_min": -37.346214294433594,
      "activations/layer16_attention_weight_max": 45.782501220703125,
      "activations/layer16_attention_weight_min": -36.589378356933594,
      "activations/layer17_attention_weight_max": 36.05059051513672,
      "activations/layer17_attention_weight_min": -22.82402992248535,
      "activations/layer18_attention_weight_max": 21.844350814819336,
      "activations/layer18_attention_weight_min": -13.250181198120117,
      "activations/layer19_attention_weight_max": 48.100868225097656,
      "activations/layer19_attention_weight_min": -31.109251022338867,
      "activations/layer1_attention_weight_max": 16.585264205932617,
      "activations/layer1_attention_weight_min": -11.589200019836426,
      "activations/layer20_attention_weight_max": 37.95955276489258,
      "activations/layer20_attention_weight_min": -24.80536460876465,
      "activations/layer21_attention_weight_max": 19.175710678100586,
      "activations/layer21_attention_weight_min": -14.53424072265625,
      "activations/layer22_attention_weight_max": 37.73615646362305,
      "activations/layer22_attention_weight_min": -25.25737190246582,
      "activations/layer23_attention_weight_max": 22.985523223876953,
      "activations/layer23_attention_weight_min": -18.86125946044922,
      "activations/layer2_attention_weight_max": 26.796520233154297,
      "activations/layer2_attention_weight_min": -25.19552230834961,
      "activations/layer3_attention_weight_max": 57.23681640625,
      "activations/layer3_attention_weight_min": -58.4487190246582,
      "activations/layer4_attention_weight_max": 106.29314422607422,
      "activations/layer4_attention_weight_min": -103.2238540649414,
      "activations/layer5_attention_weight_max": 121.47274017333984,
      "activations/layer5_attention_weight_min": -110.31129455566406,
      "activations/layer6_attention_weight_max": 65.48101806640625,
      "activations/layer6_attention_weight_min": -60.10612869262695,
      "activations/layer7_attention_weight_max": 61.50216293334961,
      "activations/layer7_attention_weight_min": -61.19366455078125,
      "activations/layer8_attention_weight_max": 58.34994125366211,
      "activations/layer8_attention_weight_min": -51.16023254394531,
      "activations/layer9_attention_weight_max": 66.97425842285156,
      "activations/layer9_attention_weight_min": -65.8630142211914,
      "epoch": 0.98,
      "learning_rate": 0.00014516325757575756,
      "loss": 3.1058,
      "step": 16800
    },
    {
      "activations/layer0_attention_weight_max": 15.140570640563965,
      "activations/layer0_attention_weight_min": -11.570505142211914,
      "activations/layer10_attention_weight_max": 61.935203552246094,
      "activations/layer10_attention_weight_min": -53.595848083496094,
      "activations/layer11_attention_weight_max": 43.963470458984375,
      "activations/layer11_attention_weight_min": -37.265594482421875,
      "activations/layer12_attention_weight_max": 60.66899871826172,
      "activations/layer12_attention_weight_min": -35.64738845825195,
      "activations/layer13_attention_weight_max": 47.93794631958008,
      "activations/layer13_attention_weight_min": -33.517250061035156,
      "activations/layer14_attention_weight_max": 64.52615356445312,
      "activations/layer14_attention_weight_min": -54.85171127319336,
      "activations/layer15_attention_weight_max": 51.870418548583984,
      "activations/layer15_attention_weight_min": -36.329063415527344,
      "activations/layer16_attention_weight_max": 48.2328987121582,
      "activations/layer16_attention_weight_min": -34.951900482177734,
      "activations/layer17_attention_weight_max": 32.04738235473633,
      "activations/layer17_attention_weight_min": -25.92971420288086,
      "activations/layer18_attention_weight_max": 18.857284545898438,
      "activations/layer18_attention_weight_min": -15.155160903930664,
      "activations/layer19_attention_weight_max": 43.50995635986328,
      "activations/layer19_attention_weight_min": -30.981494903564453,
      "activations/layer1_attention_weight_max": 16.1951847076416,
      "activations/layer1_attention_weight_min": -11.626460075378418,
      "activations/layer20_attention_weight_max": 35.243019104003906,
      "activations/layer20_attention_weight_min": -23.31570816040039,
      "activations/layer21_attention_weight_max": 18.983325958251953,
      "activations/layer21_attention_weight_min": -13.087054252624512,
      "activations/layer22_attention_weight_max": 38.05582046508789,
      "activations/layer22_attention_weight_min": -22.30976676940918,
      "activations/layer23_attention_weight_max": 22.084562301635742,
      "activations/layer23_attention_weight_min": -13.558113098144531,
      "activations/layer2_attention_weight_max": 27.49786949157715,
      "activations/layer2_attention_weight_min": -26.93347930908203,
      "activations/layer3_attention_weight_max": 59.33871841430664,
      "activations/layer3_attention_weight_min": -59.23472595214844,
      "activations/layer4_attention_weight_max": 111.3521499633789,
      "activations/layer4_attention_weight_min": -103.4569320678711,
      "activations/layer5_attention_weight_max": 124.6545639038086,
      "activations/layer5_attention_weight_min": -108.41229248046875,
      "activations/layer6_attention_weight_max": 70.19496154785156,
      "activations/layer6_attention_weight_min": -60.21156311035156,
      "activations/layer7_attention_weight_max": 75.6162338256836,
      "activations/layer7_attention_weight_min": -67.802734375,
      "activations/layer8_attention_weight_max": 65.32276916503906,
      "activations/layer8_attention_weight_min": -58.29235076904297,
      "activations/layer9_attention_weight_max": 92.92589569091797,
      "activations/layer9_attention_weight_min": -76.01354217529297,
      "epoch": 0.98,
      "learning_rate": 0.00014514431818181818,
      "loss": 3.1081,
      "step": 16850
    },
    {
      "activations/layer0_attention_weight_max": 15.104711532592773,
      "activations/layer0_attention_weight_min": -10.758689880371094,
      "activations/layer10_attention_weight_max": 70.42359924316406,
      "activations/layer10_attention_weight_min": -56.109130859375,
      "activations/layer11_attention_weight_max": 48.41416931152344,
      "activations/layer11_attention_weight_min": -36.624271392822266,
      "activations/layer12_attention_weight_max": 38.60715866088867,
      "activations/layer12_attention_weight_min": -27.454856872558594,
      "activations/layer13_attention_weight_max": 53.836544036865234,
      "activations/layer13_attention_weight_min": -34.48554611206055,
      "activations/layer14_attention_weight_max": 92.05509948730469,
      "activations/layer14_attention_weight_min": -53.00724411010742,
      "activations/layer15_attention_weight_max": 68.69544982910156,
      "activations/layer15_attention_weight_min": -38.79914855957031,
      "activations/layer16_attention_weight_max": 58.84341812133789,
      "activations/layer16_attention_weight_min": -35.8388671875,
      "activations/layer17_attention_weight_max": 45.468326568603516,
      "activations/layer17_attention_weight_min": -24.018232345581055,
      "activations/layer18_attention_weight_max": 27.241655349731445,
      "activations/layer18_attention_weight_min": -13.908968925476074,
      "activations/layer19_attention_weight_max": 68.85802459716797,
      "activations/layer19_attention_weight_min": -30.5250301361084,
      "activations/layer1_attention_weight_max": 15.782913208007812,
      "activations/layer1_attention_weight_min": -11.901732444763184,
      "activations/layer20_attention_weight_max": 45.90373611450195,
      "activations/layer20_attention_weight_min": -24.73871421813965,
      "activations/layer21_attention_weight_max": 25.80000877380371,
      "activations/layer21_attention_weight_min": -13.526479721069336,
      "activations/layer22_attention_weight_max": 42.17393112182617,
      "activations/layer22_attention_weight_min": -22.25513458251953,
      "activations/layer23_attention_weight_max": 23.753803253173828,
      "activations/layer23_attention_weight_min": -15.133498191833496,
      "activations/layer2_attention_weight_max": 29.33371353149414,
      "activations/layer2_attention_weight_min": -26.819150924682617,
      "activations/layer3_attention_weight_max": 60.750457763671875,
      "activations/layer3_attention_weight_min": -61.66435623168945,
      "activations/layer4_attention_weight_max": 114.2714614868164,
      "activations/layer4_attention_weight_min": -107.3802490234375,
      "activations/layer5_attention_weight_max": 132.60989379882812,
      "activations/layer5_attention_weight_min": -111.0097885131836,
      "activations/layer6_attention_weight_max": 72.60502624511719,
      "activations/layer6_attention_weight_min": -62.89007568359375,
      "activations/layer7_attention_weight_max": 66.7152099609375,
      "activations/layer7_attention_weight_min": -63.42853546142578,
      "activations/layer8_attention_weight_max": 68.93424224853516,
      "activations/layer8_attention_weight_min": -51.51383972167969,
      "activations/layer9_attention_weight_max": 84.88550567626953,
      "activations/layer9_attention_weight_min": -71.19207000732422,
      "epoch": 0.98,
      "learning_rate": 0.00014512537878787877,
      "loss": 3.1113,
      "step": 16900
    },
    {
      "activations/layer0_attention_weight_max": 13.124780654907227,
      "activations/layer0_attention_weight_min": -11.057724952697754,
      "activations/layer10_attention_weight_max": 50.58953857421875,
      "activations/layer10_attention_weight_min": -43.71571731567383,
      "activations/layer11_attention_weight_max": 44.46239471435547,
      "activations/layer11_attention_weight_min": -43.61770248413086,
      "activations/layer12_attention_weight_max": 31.985166549682617,
      "activations/layer12_attention_weight_min": -27.16543197631836,
      "activations/layer13_attention_weight_max": 43.323299407958984,
      "activations/layer13_attention_weight_min": -35.57522201538086,
      "activations/layer14_attention_weight_max": 69.96267700195312,
      "activations/layer14_attention_weight_min": -55.032623291015625,
      "activations/layer15_attention_weight_max": 51.26667404174805,
      "activations/layer15_attention_weight_min": -37.61397933959961,
      "activations/layer16_attention_weight_max": 52.06430435180664,
      "activations/layer16_attention_weight_min": -42.13266372680664,
      "activations/layer17_attention_weight_max": 34.7059440612793,
      "activations/layer17_attention_weight_min": -25.9244441986084,
      "activations/layer18_attention_weight_max": 24.51908302307129,
      "activations/layer18_attention_weight_min": -14.743210792541504,
      "activations/layer19_attention_weight_max": 47.45241928100586,
      "activations/layer19_attention_weight_min": -30.62067222595215,
      "activations/layer1_attention_weight_max": 16.71405029296875,
      "activations/layer1_attention_weight_min": -11.06103515625,
      "activations/layer20_attention_weight_max": 42.416282653808594,
      "activations/layer20_attention_weight_min": -27.46232795715332,
      "activations/layer21_attention_weight_max": 26.78460693359375,
      "activations/layer21_attention_weight_min": -14.84163761138916,
      "activations/layer22_attention_weight_max": 36.47389602661133,
      "activations/layer22_attention_weight_min": -22.26334571838379,
      "activations/layer23_attention_weight_max": 23.502872467041016,
      "activations/layer23_attention_weight_min": -13.69693374633789,
      "activations/layer2_attention_weight_max": 27.283679962158203,
      "activations/layer2_attention_weight_min": -25.747264862060547,
      "activations/layer3_attention_weight_max": 57.193634033203125,
      "activations/layer3_attention_weight_min": -60.626739501953125,
      "activations/layer4_attention_weight_max": 114.62525939941406,
      "activations/layer4_attention_weight_min": -105.71510314941406,
      "activations/layer5_attention_weight_max": 127.49895477294922,
      "activations/layer5_attention_weight_min": -112.73223114013672,
      "activations/layer6_attention_weight_max": 63.45951843261719,
      "activations/layer6_attention_weight_min": -59.245277404785156,
      "activations/layer7_attention_weight_max": 63.74612045288086,
      "activations/layer7_attention_weight_min": -62.69753646850586,
      "activations/layer8_attention_weight_max": 55.7516975402832,
      "activations/layer8_attention_weight_min": -49.68449783325195,
      "activations/layer9_attention_weight_max": 72.14338684082031,
      "activations/layer9_attention_weight_min": -65.1837387084961,
      "epoch": 0.98,
      "learning_rate": 0.00014510643939393938,
      "loss": 3.1045,
      "step": 16950
    },
    {
      "activations/layer0_attention_weight_max": 15.08272647857666,
      "activations/layer0_attention_weight_min": -10.962535858154297,
      "activations/layer10_attention_weight_max": 58.51530075073242,
      "activations/layer10_attention_weight_min": -46.20493698120117,
      "activations/layer11_attention_weight_max": 52.94300079345703,
      "activations/layer11_attention_weight_min": -37.185752868652344,
      "activations/layer12_attention_weight_max": 34.56476974487305,
      "activations/layer12_attention_weight_min": -27.397741317749023,
      "activations/layer13_attention_weight_max": 43.96971130371094,
      "activations/layer13_attention_weight_min": -32.42975616455078,
      "activations/layer14_attention_weight_max": 71.83746337890625,
      "activations/layer14_attention_weight_min": -52.242000579833984,
      "activations/layer15_attention_weight_max": 52.514286041259766,
      "activations/layer15_attention_weight_min": -37.31033706665039,
      "activations/layer16_attention_weight_max": 46.12544250488281,
      "activations/layer16_attention_weight_min": -36.36398696899414,
      "activations/layer17_attention_weight_max": 39.92626190185547,
      "activations/layer17_attention_weight_min": -24.988128662109375,
      "activations/layer18_attention_weight_max": 24.2902774810791,
      "activations/layer18_attention_weight_min": -13.511223793029785,
      "activations/layer19_attention_weight_max": 45.595375061035156,
      "activations/layer19_attention_weight_min": -35.34830856323242,
      "activations/layer1_attention_weight_max": 15.556774139404297,
      "activations/layer1_attention_weight_min": -12.931318283081055,
      "activations/layer20_attention_weight_max": 41.72053527832031,
      "activations/layer20_attention_weight_min": -24.608232498168945,
      "activations/layer21_attention_weight_max": 22.907739639282227,
      "activations/layer21_attention_weight_min": -14.767255783081055,
      "activations/layer22_attention_weight_max": 40.5237922668457,
      "activations/layer22_attention_weight_min": -20.68318748474121,
      "activations/layer23_attention_weight_max": 25.624210357666016,
      "activations/layer23_attention_weight_min": -13.8525390625,
      "activations/layer2_attention_weight_max": 28.88318634033203,
      "activations/layer2_attention_weight_min": -26.89767074584961,
      "activations/layer3_attention_weight_max": 64.41204071044922,
      "activations/layer3_attention_weight_min": -68.2010269165039,
      "activations/layer4_attention_weight_max": 112.96825408935547,
      "activations/layer4_attention_weight_min": -112.88597869873047,
      "activations/layer5_attention_weight_max": 129.38766479492188,
      "activations/layer5_attention_weight_min": -118.03997039794922,
      "activations/layer6_attention_weight_max": 71.9775390625,
      "activations/layer6_attention_weight_min": -62.64423751831055,
      "activations/layer7_attention_weight_max": 65.68614196777344,
      "activations/layer7_attention_weight_min": -67.47956085205078,
      "activations/layer8_attention_weight_max": 66.27124786376953,
      "activations/layer8_attention_weight_min": -48.414154052734375,
      "activations/layer9_attention_weight_max": 79.29949951171875,
      "activations/layer9_attention_weight_min": -60.9546012878418,
      "epoch": 0.99,
      "learning_rate": 0.0001450875,
      "loss": 3.1076,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_loss": 3.052734375,
      "eval_runtime": 8.4957,
      "eval_samples_per_second": 505.432,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_openwebtext_loss": 3.052734375,
      "eval_openwebtext_ppl": 21.173160702117308,
      "eval_openwebtext_runtime": 8.4957,
      "eval_openwebtext_samples_per_second": 505.432,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_wikitext_loss": 3.3359375,
      "eval_wikitext_ppl": 28.1047190510635,
      "eval_wikitext_runtime": 2.0144,
      "eval_wikitext_samples_per_second": 226.366,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_lambada_loss": 3.431640625,
      "eval_lambada_ppl": 30.927341318986464,
      "eval_lambada_runtime": 9.5252,
      "eval_lambada_samples_per_second": 511.168,
      "step": 17000
    },
    {
      "activations/layer0_attention_weight_max": 15.817585945129395,
      "activations/layer0_attention_weight_min": -11.319321632385254,
      "activations/layer10_attention_weight_max": 51.235076904296875,
      "activations/layer10_attention_weight_min": -49.035701751708984,
      "activations/layer11_attention_weight_max": 42.29875946044922,
      "activations/layer11_attention_weight_min": -40.79356002807617,
      "activations/layer12_attention_weight_max": 30.819271087646484,
      "activations/layer12_attention_weight_min": -24.94403076171875,
      "activations/layer13_attention_weight_max": 42.64306640625,
      "activations/layer13_attention_weight_min": -35.79520034790039,
      "activations/layer14_attention_weight_max": 73.9991683959961,
      "activations/layer14_attention_weight_min": -55.60548400878906,
      "activations/layer15_attention_weight_max": 51.53324508666992,
      "activations/layer15_attention_weight_min": -38.586307525634766,
      "activations/layer16_attention_weight_max": 60.271427154541016,
      "activations/layer16_attention_weight_min": -43.657005310058594,
      "activations/layer17_attention_weight_max": 40.920413970947266,
      "activations/layer17_attention_weight_min": -26.886978149414062,
      "activations/layer18_attention_weight_max": 23.216978073120117,
      "activations/layer18_attention_weight_min": -13.093897819519043,
      "activations/layer19_attention_weight_max": 49.027122497558594,
      "activations/layer19_attention_weight_min": -31.328901290893555,
      "activations/layer1_attention_weight_max": 17.102581024169922,
      "activations/layer1_attention_weight_min": -11.473525047302246,
      "activations/layer20_attention_weight_max": 41.72870635986328,
      "activations/layer20_attention_weight_min": -25.24156951904297,
      "activations/layer21_attention_weight_max": 21.1912784576416,
      "activations/layer21_attention_weight_min": -13.679522514343262,
      "activations/layer22_attention_weight_max": 36.23078155517578,
      "activations/layer22_attention_weight_min": -21.368005752563477,
      "activations/layer23_attention_weight_max": 24.964237213134766,
      "activations/layer23_attention_weight_min": -14.302980422973633,
      "activations/layer2_attention_weight_max": 26.86178207397461,
      "activations/layer2_attention_weight_min": -27.226150512695312,
      "activations/layer3_attention_weight_max": 57.05860137939453,
      "activations/layer3_attention_weight_min": -59.579689025878906,
      "activations/layer4_attention_weight_max": 110.21559143066406,
      "activations/layer4_attention_weight_min": -103.91963958740234,
      "activations/layer5_attention_weight_max": 119.12236022949219,
      "activations/layer5_attention_weight_min": -108.91435241699219,
      "activations/layer6_attention_weight_max": 64.87167358398438,
      "activations/layer6_attention_weight_min": -60.21923065185547,
      "activations/layer7_attention_weight_max": 63.467689514160156,
      "activations/layer7_attention_weight_min": -62.778621673583984,
      "activations/layer8_attention_weight_max": 54.92973709106445,
      "activations/layer8_attention_weight_min": -50.65066146850586,
      "activations/layer9_attention_weight_max": 66.59517669677734,
      "activations/layer9_attention_weight_min": -64.87184143066406,
      "epoch": 0.99,
      "learning_rate": 0.00014506856060606058,
      "loss": 3.1275,
      "step": 17050
    },
    {
      "activations/layer0_attention_weight_max": 14.405909538269043,
      "activations/layer0_attention_weight_min": -11.295455932617188,
      "activations/layer10_attention_weight_max": 57.823299407958984,
      "activations/layer10_attention_weight_min": -49.946529388427734,
      "activations/layer11_attention_weight_max": 48.7078971862793,
      "activations/layer11_attention_weight_min": -39.02555465698242,
      "activations/layer12_attention_weight_max": 33.126373291015625,
      "activations/layer12_attention_weight_min": -26.556434631347656,
      "activations/layer13_attention_weight_max": 44.6873664855957,
      "activations/layer13_attention_weight_min": -35.01280212402344,
      "activations/layer14_attention_weight_max": 82.94707489013672,
      "activations/layer14_attention_weight_min": -61.90669631958008,
      "activations/layer15_attention_weight_max": 55.58322525024414,
      "activations/layer15_attention_weight_min": -38.174442291259766,
      "activations/layer16_attention_weight_max": 53.64479064941406,
      "activations/layer16_attention_weight_min": -40.37662124633789,
      "activations/layer17_attention_weight_max": 39.514930725097656,
      "activations/layer17_attention_weight_min": -23.43338966369629,
      "activations/layer18_attention_weight_max": 21.889402389526367,
      "activations/layer18_attention_weight_min": -14.440224647521973,
      "activations/layer19_attention_weight_max": 59.040283203125,
      "activations/layer19_attention_weight_min": -30.27557945251465,
      "activations/layer1_attention_weight_max": 16.954448699951172,
      "activations/layer1_attention_weight_min": -11.616840362548828,
      "activations/layer20_attention_weight_max": 49.848899841308594,
      "activations/layer20_attention_weight_min": -25.52935791015625,
      "activations/layer21_attention_weight_max": 19.88357925415039,
      "activations/layer21_attention_weight_min": -13.088982582092285,
      "activations/layer22_attention_weight_max": 38.069393157958984,
      "activations/layer22_attention_weight_min": -22.560897827148438,
      "activations/layer23_attention_weight_max": 22.820600509643555,
      "activations/layer23_attention_weight_min": -13.366716384887695,
      "activations/layer2_attention_weight_max": 27.33967399597168,
      "activations/layer2_attention_weight_min": -24.538131713867188,
      "activations/layer3_attention_weight_max": 58.611873626708984,
      "activations/layer3_attention_weight_min": -58.53291320800781,
      "activations/layer4_attention_weight_max": 108.0616683959961,
      "activations/layer4_attention_weight_min": -100.61453247070312,
      "activations/layer5_attention_weight_max": 121.12944030761719,
      "activations/layer5_attention_weight_min": -105.47062683105469,
      "activations/layer6_attention_weight_max": 68.24974822998047,
      "activations/layer6_attention_weight_min": -55.657413482666016,
      "activations/layer7_attention_weight_max": 65.50537872314453,
      "activations/layer7_attention_weight_min": -62.87413024902344,
      "activations/layer8_attention_weight_max": 58.44197082519531,
      "activations/layer8_attention_weight_min": -48.85484313964844,
      "activations/layer9_attention_weight_max": 90.1444320678711,
      "activations/layer9_attention_weight_min": -75.74505615234375,
      "epoch": 0.99,
      "learning_rate": 0.0001450496212121212,
      "loss": 3.0994,
      "step": 17100
    },
    {
      "activations/layer0_attention_weight_max": 15.166220664978027,
      "activations/layer0_attention_weight_min": -11.044210433959961,
      "activations/layer10_attention_weight_max": 57.83452606201172,
      "activations/layer10_attention_weight_min": -50.4899787902832,
      "activations/layer11_attention_weight_max": 50.36448669433594,
      "activations/layer11_attention_weight_min": -39.2650146484375,
      "activations/layer12_attention_weight_max": 33.64096450805664,
      "activations/layer12_attention_weight_min": -25.900047302246094,
      "activations/layer13_attention_weight_max": 44.61335754394531,
      "activations/layer13_attention_weight_min": -33.8658332824707,
      "activations/layer14_attention_weight_max": 69.98265838623047,
      "activations/layer14_attention_weight_min": -56.49894332885742,
      "activations/layer15_attention_weight_max": 50.98737335205078,
      "activations/layer15_attention_weight_min": -38.409610748291016,
      "activations/layer16_attention_weight_max": 51.45113754272461,
      "activations/layer16_attention_weight_min": -36.85784912109375,
      "activations/layer17_attention_weight_max": 37.26999282836914,
      "activations/layer17_attention_weight_min": -26.51308250427246,
      "activations/layer18_attention_weight_max": 20.78689193725586,
      "activations/layer18_attention_weight_min": -17.411611557006836,
      "activations/layer19_attention_weight_max": 55.3333625793457,
      "activations/layer19_attention_weight_min": -31.694921493530273,
      "activations/layer1_attention_weight_max": 16.51542091369629,
      "activations/layer1_attention_weight_min": -11.223389625549316,
      "activations/layer20_attention_weight_max": 41.283546447753906,
      "activations/layer20_attention_weight_min": -23.819988250732422,
      "activations/layer21_attention_weight_max": 18.9322566986084,
      "activations/layer21_attention_weight_min": -16.36309814453125,
      "activations/layer22_attention_weight_max": 38.75714874267578,
      "activations/layer22_attention_weight_min": -20.434803009033203,
      "activations/layer23_attention_weight_max": 22.992401123046875,
      "activations/layer23_attention_weight_min": -13.187800407409668,
      "activations/layer2_attention_weight_max": 27.603885650634766,
      "activations/layer2_attention_weight_min": -27.37493896484375,
      "activations/layer3_attention_weight_max": 57.79684829711914,
      "activations/layer3_attention_weight_min": -60.163394927978516,
      "activations/layer4_attention_weight_max": 106.28417205810547,
      "activations/layer4_attention_weight_min": -106.16243743896484,
      "activations/layer5_attention_weight_max": 119.40156555175781,
      "activations/layer5_attention_weight_min": -112.7745361328125,
      "activations/layer6_attention_weight_max": 64.13762664794922,
      "activations/layer6_attention_weight_min": -58.881202697753906,
      "activations/layer7_attention_weight_max": 62.73998260498047,
      "activations/layer7_attention_weight_min": -68.86205291748047,
      "activations/layer8_attention_weight_max": 55.62557601928711,
      "activations/layer8_attention_weight_min": -49.15623474121094,
      "activations/layer9_attention_weight_max": 72.89014434814453,
      "activations/layer9_attention_weight_min": -62.3165397644043,
      "epoch": 1.0,
      "learning_rate": 0.0001450306818181818,
      "loss": 3.1097,
      "step": 17150
    },
    {
      "activations/layer0_attention_weight_max": 15.27285385131836,
      "activations/layer0_attention_weight_min": -11.305810928344727,
      "activations/layer10_attention_weight_max": 61.09836196899414,
      "activations/layer10_attention_weight_min": -49.53953170776367,
      "activations/layer11_attention_weight_max": 45.962738037109375,
      "activations/layer11_attention_weight_min": -38.147884368896484,
      "activations/layer12_attention_weight_max": 42.37356948852539,
      "activations/layer12_attention_weight_min": -33.528533935546875,
      "activations/layer13_attention_weight_max": 48.84003448486328,
      "activations/layer13_attention_weight_min": -37.39985656738281,
      "activations/layer14_attention_weight_max": 75.54886627197266,
      "activations/layer14_attention_weight_min": -51.62397384643555,
      "activations/layer15_attention_weight_max": 59.37671661376953,
      "activations/layer15_attention_weight_min": -40.862728118896484,
      "activations/layer16_attention_weight_max": 59.446285247802734,
      "activations/layer16_attention_weight_min": -36.72836685180664,
      "activations/layer17_attention_weight_max": 41.068267822265625,
      "activations/layer17_attention_weight_min": -24.817705154418945,
      "activations/layer18_attention_weight_max": 24.944061279296875,
      "activations/layer18_attention_weight_min": -14.638045310974121,
      "activations/layer19_attention_weight_max": 54.731971740722656,
      "activations/layer19_attention_weight_min": -30.7027587890625,
      "activations/layer1_attention_weight_max": 16.698589324951172,
      "activations/layer1_attention_weight_min": -12.120853424072266,
      "activations/layer20_attention_weight_max": 45.05306625366211,
      "activations/layer20_attention_weight_min": -24.715391159057617,
      "activations/layer21_attention_weight_max": 21.86966896057129,
      "activations/layer21_attention_weight_min": -14.507478713989258,
      "activations/layer22_attention_weight_max": 44.01726150512695,
      "activations/layer22_attention_weight_min": -23.186643600463867,
      "activations/layer23_attention_weight_max": 23.99346160888672,
      "activations/layer23_attention_weight_min": -15.015242576599121,
      "activations/layer2_attention_weight_max": 27.583633422851562,
      "activations/layer2_attention_weight_min": -28.183700561523438,
      "activations/layer3_attention_weight_max": 60.43202209472656,
      "activations/layer3_attention_weight_min": -62.59341049194336,
      "activations/layer4_attention_weight_max": 109.09774017333984,
      "activations/layer4_attention_weight_min": -106.39482879638672,
      "activations/layer5_attention_weight_max": 124.32085418701172,
      "activations/layer5_attention_weight_min": -111.4674072265625,
      "activations/layer6_attention_weight_max": 65.50702667236328,
      "activations/layer6_attention_weight_min": -59.808441162109375,
      "activations/layer7_attention_weight_max": 67.44964599609375,
      "activations/layer7_attention_weight_min": -66.6957778930664,
      "activations/layer8_attention_weight_max": 61.44256591796875,
      "activations/layer8_attention_weight_min": -52.02787399291992,
      "activations/layer9_attention_weight_max": 69.70791625976562,
      "activations/layer9_attention_weight_min": -61.01485824584961,
      "epoch": 1.0,
      "learning_rate": 0.0001450117424242424,
      "loss": 3.1113,
      "step": 17200
    },
    {
      "activations/layer0_attention_weight_max": 15.103315353393555,
      "activations/layer0_attention_weight_min": -11.237224578857422,
      "activations/layer10_attention_weight_max": 57.04063034057617,
      "activations/layer10_attention_weight_min": -50.73553466796875,
      "activations/layer11_attention_weight_max": 46.99374008178711,
      "activations/layer11_attention_weight_min": -38.60874938964844,
      "activations/layer12_attention_weight_max": 39.75407791137695,
      "activations/layer12_attention_weight_min": -31.533470153808594,
      "activations/layer13_attention_weight_max": 42.5859260559082,
      "activations/layer13_attention_weight_min": -35.56014633178711,
      "activations/layer14_attention_weight_max": 72.90483093261719,
      "activations/layer14_attention_weight_min": -58.0396728515625,
      "activations/layer15_attention_weight_max": 55.09995651245117,
      "activations/layer15_attention_weight_min": -39.50012969970703,
      "activations/layer16_attention_weight_max": 51.289024353027344,
      "activations/layer16_attention_weight_min": -37.617977142333984,
      "activations/layer17_attention_weight_max": 44.25313949584961,
      "activations/layer17_attention_weight_min": -27.96860122680664,
      "activations/layer18_attention_weight_max": 21.793481826782227,
      "activations/layer18_attention_weight_min": -16.219654083251953,
      "activations/layer19_attention_weight_max": 46.71226501464844,
      "activations/layer19_attention_weight_min": -31.92648696899414,
      "activations/layer1_attention_weight_max": 16.45952606201172,
      "activations/layer1_attention_weight_min": -12.70669174194336,
      "activations/layer20_attention_weight_max": 38.32007598876953,
      "activations/layer20_attention_weight_min": -26.959150314331055,
      "activations/layer21_attention_weight_max": 21.10321617126465,
      "activations/layer21_attention_weight_min": -13.461433410644531,
      "activations/layer22_attention_weight_max": 38.0687141418457,
      "activations/layer22_attention_weight_min": -22.51750946044922,
      "activations/layer23_attention_weight_max": 22.247459411621094,
      "activations/layer23_attention_weight_min": -13.692047119140625,
      "activations/layer2_attention_weight_max": 28.111297607421875,
      "activations/layer2_attention_weight_min": -26.661481857299805,
      "activations/layer3_attention_weight_max": 58.570655822753906,
      "activations/layer3_attention_weight_min": -60.705867767333984,
      "activations/layer4_attention_weight_max": 110.5020751953125,
      "activations/layer4_attention_weight_min": -106.41864013671875,
      "activations/layer5_attention_weight_max": 119.00591278076172,
      "activations/layer5_attention_weight_min": -109.10487365722656,
      "activations/layer6_attention_weight_max": 65.3478012084961,
      "activations/layer6_attention_weight_min": -60.875308990478516,
      "activations/layer7_attention_weight_max": 63.530277252197266,
      "activations/layer7_attention_weight_min": -61.48029708862305,
      "activations/layer8_attention_weight_max": 58.25279998779297,
      "activations/layer8_attention_weight_min": -53.373844146728516,
      "activations/layer9_attention_weight_max": 71.71373748779297,
      "activations/layer9_attention_weight_min": -64.74282836914062,
      "epoch": 1.0,
      "learning_rate": 0.00014499280303030302,
      "loss": 3.1517,
      "step": 17250
    },
    {
      "activations/layer0_attention_weight_max": 15.541396141052246,
      "activations/layer0_attention_weight_min": -11.568534851074219,
      "activations/layer10_attention_weight_max": 67.84115600585938,
      "activations/layer10_attention_weight_min": -56.83250427246094,
      "activations/layer11_attention_weight_max": 51.67378234863281,
      "activations/layer11_attention_weight_min": -36.24600601196289,
      "activations/layer12_attention_weight_max": 43.97979736328125,
      "activations/layer12_attention_weight_min": -28.91258430480957,
      "activations/layer13_attention_weight_max": 57.48197937011719,
      "activations/layer13_attention_weight_min": -39.22334671020508,
      "activations/layer14_attention_weight_max": 86.43558502197266,
      "activations/layer14_attention_weight_min": -60.32240295410156,
      "activations/layer15_attention_weight_max": 56.5167236328125,
      "activations/layer15_attention_weight_min": -42.330684661865234,
      "activations/layer16_attention_weight_max": 56.77235794067383,
      "activations/layer16_attention_weight_min": -39.887718200683594,
      "activations/layer17_attention_weight_max": 36.743011474609375,
      "activations/layer17_attention_weight_min": -24.722076416015625,
      "activations/layer18_attention_weight_max": 21.83010482788086,
      "activations/layer18_attention_weight_min": -13.210736274719238,
      "activations/layer19_attention_weight_max": 51.29668045043945,
      "activations/layer19_attention_weight_min": -30.867664337158203,
      "activations/layer1_attention_weight_max": 16.17002296447754,
      "activations/layer1_attention_weight_min": -11.371024131774902,
      "activations/layer20_attention_weight_max": 46.177120208740234,
      "activations/layer20_attention_weight_min": -25.487709045410156,
      "activations/layer21_attention_weight_max": 20.88688850402832,
      "activations/layer21_attention_weight_min": -13.591407775878906,
      "activations/layer22_attention_weight_max": 39.5302619934082,
      "activations/layer22_attention_weight_min": -21.69791030883789,
      "activations/layer23_attention_weight_max": 24.516210556030273,
      "activations/layer23_attention_weight_min": -12.908759117126465,
      "activations/layer2_attention_weight_max": 26.878538131713867,
      "activations/layer2_attention_weight_min": -25.71853256225586,
      "activations/layer3_attention_weight_max": 62.74068832397461,
      "activations/layer3_attention_weight_min": -63.561126708984375,
      "activations/layer4_attention_weight_max": 114.64215087890625,
      "activations/layer4_attention_weight_min": -111.019775390625,
      "activations/layer5_attention_weight_max": 131.98777770996094,
      "activations/layer5_attention_weight_min": -113.71044921875,
      "activations/layer6_attention_weight_max": 73.92705535888672,
      "activations/layer6_attention_weight_min": -62.5256462097168,
      "activations/layer7_attention_weight_max": 65.54886627197266,
      "activations/layer7_attention_weight_min": -63.20953369140625,
      "activations/layer8_attention_weight_max": 67.71736907958984,
      "activations/layer8_attention_weight_min": -54.441200256347656,
      "activations/layer9_attention_weight_max": 84.05384826660156,
      "activations/layer9_attention_weight_min": -66.59278869628906,
      "epoch": 1.01,
      "learning_rate": 0.0001449738636363636,
      "loss": 3.0888,
      "step": 17300
    },
    {
      "activations/layer0_attention_weight_max": 15.569971084594727,
      "activations/layer0_attention_weight_min": -20.26213836669922,
      "activations/layer10_attention_weight_max": 56.38352584838867,
      "activations/layer10_attention_weight_min": -47.663970947265625,
      "activations/layer11_attention_weight_max": 42.877845764160156,
      "activations/layer11_attention_weight_min": -35.83640670776367,
      "activations/layer12_attention_weight_max": 31.59298324584961,
      "activations/layer12_attention_weight_min": -26.768526077270508,
      "activations/layer13_attention_weight_max": 40.98020935058594,
      "activations/layer13_attention_weight_min": -36.16608810424805,
      "activations/layer14_attention_weight_max": 69.46290588378906,
      "activations/layer14_attention_weight_min": -59.20886993408203,
      "activations/layer15_attention_weight_max": 54.08645248413086,
      "activations/layer15_attention_weight_min": -40.66476058959961,
      "activations/layer16_attention_weight_max": 45.25617980957031,
      "activations/layer16_attention_weight_min": -42.22772216796875,
      "activations/layer17_attention_weight_max": 35.16444396972656,
      "activations/layer17_attention_weight_min": -27.638578414916992,
      "activations/layer18_attention_weight_max": 19.129247665405273,
      "activations/layer18_attention_weight_min": -13.874408721923828,
      "activations/layer19_attention_weight_max": 42.089447021484375,
      "activations/layer19_attention_weight_min": -30.789968490600586,
      "activations/layer1_attention_weight_max": 15.569156646728516,
      "activations/layer1_attention_weight_min": -16.684309005737305,
      "activations/layer20_attention_weight_max": 35.507781982421875,
      "activations/layer20_attention_weight_min": -22.475374221801758,
      "activations/layer21_attention_weight_max": 18.613975524902344,
      "activations/layer21_attention_weight_min": -14.31859302520752,
      "activations/layer22_attention_weight_max": 40.91260528564453,
      "activations/layer22_attention_weight_min": -19.876529693603516,
      "activations/layer23_attention_weight_max": 21.069252014160156,
      "activations/layer23_attention_weight_min": -13.576208114624023,
      "activations/layer2_attention_weight_max": 24.53020477294922,
      "activations/layer2_attention_weight_min": -22.574073791503906,
      "activations/layer3_attention_weight_max": 42.06801986694336,
      "activations/layer3_attention_weight_min": -45.8389892578125,
      "activations/layer4_attention_weight_max": 88.65898895263672,
      "activations/layer4_attention_weight_min": -93.96378326416016,
      "activations/layer5_attention_weight_max": 107.822509765625,
      "activations/layer5_attention_weight_min": -103.10518646240234,
      "activations/layer6_attention_weight_max": 64.83271789550781,
      "activations/layer6_attention_weight_min": -58.11636734008789,
      "activations/layer7_attention_weight_max": 61.05308151245117,
      "activations/layer7_attention_weight_min": -62.7294807434082,
      "activations/layer8_attention_weight_max": 60.62010955810547,
      "activations/layer8_attention_weight_min": -59.26433181762695,
      "activations/layer9_attention_weight_max": 78.4822769165039,
      "activations/layer9_attention_weight_min": -60.653968811035156,
      "epoch": 1.01,
      "learning_rate": 0.00014495606060606057,
      "loss": 3.2744,
      "step": 17350
    },
    {
      "activations/layer0_attention_weight_max": 15.092493057250977,
      "activations/layer0_attention_weight_min": -17.142528533935547,
      "activations/layer10_attention_weight_max": 50.19488525390625,
      "activations/layer10_attention_weight_min": -49.73271179199219,
      "activations/layer11_attention_weight_max": 41.62620544433594,
      "activations/layer11_attention_weight_min": -41.85844421386719,
      "activations/layer12_attention_weight_max": 31.772911071777344,
      "activations/layer12_attention_weight_min": -25.950632095336914,
      "activations/layer13_attention_weight_max": 42.315460205078125,
      "activations/layer13_attention_weight_min": -37.55216598510742,
      "activations/layer14_attention_weight_max": 66.42778015136719,
      "activations/layer14_attention_weight_min": -54.7354850769043,
      "activations/layer15_attention_weight_max": 46.838130950927734,
      "activations/layer15_attention_weight_min": -35.901756286621094,
      "activations/layer16_attention_weight_max": 52.03621292114258,
      "activations/layer16_attention_weight_min": -37.95242691040039,
      "activations/layer17_attention_weight_max": 32.62791061401367,
      "activations/layer17_attention_weight_min": -22.17194366455078,
      "activations/layer18_attention_weight_max": 24.802751541137695,
      "activations/layer18_attention_weight_min": -13.976430892944336,
      "activations/layer19_attention_weight_max": 44.696327209472656,
      "activations/layer19_attention_weight_min": -30.396533966064453,
      "activations/layer1_attention_weight_max": 16.463979721069336,
      "activations/layer1_attention_weight_min": -15.334158897399902,
      "activations/layer20_attention_weight_max": 36.40212631225586,
      "activations/layer20_attention_weight_min": -26.4766788482666,
      "activations/layer21_attention_weight_max": 21.34251594543457,
      "activations/layer21_attention_weight_min": -15.826738357543945,
      "activations/layer22_attention_weight_max": 39.555267333984375,
      "activations/layer22_attention_weight_min": -23.260589599609375,
      "activations/layer23_attention_weight_max": 24.412649154663086,
      "activations/layer23_attention_weight_min": -17.618106842041016,
      "activations/layer2_attention_weight_max": 23.411527633666992,
      "activations/layer2_attention_weight_min": -21.90454864501953,
      "activations/layer3_attention_weight_max": 47.47732925415039,
      "activations/layer3_attention_weight_min": -51.70497131347656,
      "activations/layer4_attention_weight_max": 93.93964385986328,
      "activations/layer4_attention_weight_min": -93.0683822631836,
      "activations/layer5_attention_weight_max": 107.3779525756836,
      "activations/layer5_attention_weight_min": -103.57405090332031,
      "activations/layer6_attention_weight_max": 57.921939849853516,
      "activations/layer6_attention_weight_min": -54.84006118774414,
      "activations/layer7_attention_weight_max": 56.31158447265625,
      "activations/layer7_attention_weight_min": -58.64516067504883,
      "activations/layer8_attention_weight_max": 60.06380844116211,
      "activations/layer8_attention_weight_min": -54.24620819091797,
      "activations/layer9_attention_weight_max": 61.46417236328125,
      "activations/layer9_attention_weight_min": -57.970367431640625,
      "epoch": 1.01,
      "learning_rate": 0.0001449371212121212,
      "loss": 3.1175,
      "step": 17400
    },
    {
      "activations/layer0_attention_weight_max": 13.677433967590332,
      "activations/layer0_attention_weight_min": -16.870235443115234,
      "activations/layer10_attention_weight_max": 67.3719711303711,
      "activations/layer10_attention_weight_min": -53.84777069091797,
      "activations/layer11_attention_weight_max": 47.81362533569336,
      "activations/layer11_attention_weight_min": -44.07979965209961,
      "activations/layer12_attention_weight_max": 97.39093017578125,
      "activations/layer12_attention_weight_min": -51.5595817565918,
      "activations/layer13_attention_weight_max": 65.25770568847656,
      "activations/layer13_attention_weight_min": -40.66585922241211,
      "activations/layer14_attention_weight_max": 80.79843139648438,
      "activations/layer14_attention_weight_min": -56.753421783447266,
      "activations/layer15_attention_weight_max": 55.69788360595703,
      "activations/layer15_attention_weight_min": -38.61869812011719,
      "activations/layer16_attention_weight_max": 58.29974365234375,
      "activations/layer16_attention_weight_min": -43.151493072509766,
      "activations/layer17_attention_weight_max": 36.3878173828125,
      "activations/layer17_attention_weight_min": -24.91699981689453,
      "activations/layer18_attention_weight_max": 21.639415740966797,
      "activations/layer18_attention_weight_min": -14.063962936401367,
      "activations/layer19_attention_weight_max": 51.367671966552734,
      "activations/layer19_attention_weight_min": -31.093488693237305,
      "activations/layer1_attention_weight_max": 16.184267044067383,
      "activations/layer1_attention_weight_min": -15.341050148010254,
      "activations/layer20_attention_weight_max": 38.09744644165039,
      "activations/layer20_attention_weight_min": -23.365022659301758,
      "activations/layer21_attention_weight_max": 19.823413848876953,
      "activations/layer21_attention_weight_min": -15.342470169067383,
      "activations/layer22_attention_weight_max": 35.78130340576172,
      "activations/layer22_attention_weight_min": -20.21739387512207,
      "activations/layer23_attention_weight_max": 26.300811767578125,
      "activations/layer23_attention_weight_min": -13.876527786254883,
      "activations/layer2_attention_weight_max": 24.75191307067871,
      "activations/layer2_attention_weight_min": -22.96548080444336,
      "activations/layer3_attention_weight_max": 51.151092529296875,
      "activations/layer3_attention_weight_min": -52.016143798828125,
      "activations/layer4_attention_weight_max": 101.28968811035156,
      "activations/layer4_attention_weight_min": -98.75128936767578,
      "activations/layer5_attention_weight_max": 119.28004455566406,
      "activations/layer5_attention_weight_min": -104.68907165527344,
      "activations/layer6_attention_weight_max": 68.33663940429688,
      "activations/layer6_attention_weight_min": -57.922245025634766,
      "activations/layer7_attention_weight_max": 69.62202453613281,
      "activations/layer7_attention_weight_min": -66.050048828125,
      "activations/layer8_attention_weight_max": 64.46936798095703,
      "activations/layer8_attention_weight_min": -56.283958435058594,
      "activations/layer9_attention_weight_max": 90.33379364013672,
      "activations/layer9_attention_weight_min": -77.44576263427734,
      "epoch": 1.01,
      "learning_rate": 0.0001449181818181818,
      "loss": 3.1018,
      "step": 17450
    },
    {
      "activations/layer0_attention_weight_max": 15.050686836242676,
      "activations/layer0_attention_weight_min": -16.15032196044922,
      "activations/layer10_attention_weight_max": 62.7053337097168,
      "activations/layer10_attention_weight_min": -48.38847351074219,
      "activations/layer11_attention_weight_max": 46.350276947021484,
      "activations/layer11_attention_weight_min": -35.86236572265625,
      "activations/layer12_attention_weight_max": 34.347267150878906,
      "activations/layer12_attention_weight_min": -28.267332077026367,
      "activations/layer13_attention_weight_max": 46.63569259643555,
      "activations/layer13_attention_weight_min": -38.69667053222656,
      "activations/layer14_attention_weight_max": 80.9750747680664,
      "activations/layer14_attention_weight_min": -50.96694564819336,
      "activations/layer15_attention_weight_max": 60.64643478393555,
      "activations/layer15_attention_weight_min": -37.2597770690918,
      "activations/layer16_attention_weight_max": 63.27339553833008,
      "activations/layer16_attention_weight_min": -37.452266693115234,
      "activations/layer17_attention_weight_max": 47.25288391113281,
      "activations/layer17_attention_weight_min": -26.60039520263672,
      "activations/layer18_attention_weight_max": 24.078901290893555,
      "activations/layer18_attention_weight_min": -14.717087745666504,
      "activations/layer19_attention_weight_max": 57.786170959472656,
      "activations/layer19_attention_weight_min": -29.9366397857666,
      "activations/layer1_attention_weight_max": 14.92275619506836,
      "activations/layer1_attention_weight_min": -14.128167152404785,
      "activations/layer20_attention_weight_max": 47.56869888305664,
      "activations/layer20_attention_weight_min": -21.628910064697266,
      "activations/layer21_attention_weight_max": 21.107772827148438,
      "activations/layer21_attention_weight_min": -13.01503849029541,
      "activations/layer22_attention_weight_max": 42.79681396484375,
      "activations/layer22_attention_weight_min": -20.50480842590332,
      "activations/layer23_attention_weight_max": 23.251911163330078,
      "activations/layer23_attention_weight_min": -13.120894432067871,
      "activations/layer2_attention_weight_max": 24.529586791992188,
      "activations/layer2_attention_weight_min": -23.960529327392578,
      "activations/layer3_attention_weight_max": 51.75429153442383,
      "activations/layer3_attention_weight_min": -52.55308532714844,
      "activations/layer4_attention_weight_max": 103.56300354003906,
      "activations/layer4_attention_weight_min": -97.06316375732422,
      "activations/layer5_attention_weight_max": 119.91753387451172,
      "activations/layer5_attention_weight_min": -107.08861541748047,
      "activations/layer6_attention_weight_max": 65.00508880615234,
      "activations/layer6_attention_weight_min": -54.744224548339844,
      "activations/layer7_attention_weight_max": 62.587642669677734,
      "activations/layer7_attention_weight_min": -64.78500366210938,
      "activations/layer8_attention_weight_max": 59.79338836669922,
      "activations/layer8_attention_weight_min": -50.93460464477539,
      "activations/layer9_attention_weight_max": 74.12474822998047,
      "activations/layer9_attention_weight_min": -59.29555892944336,
      "epoch": 1.02,
      "learning_rate": 0.00014489924242424242,
      "loss": 3.0909,
      "step": 17500
    },
    {
      "activations/layer0_attention_weight_max": 15.54399299621582,
      "activations/layer0_attention_weight_min": -15.3295259475708,
      "activations/layer10_attention_weight_max": 60.75868606567383,
      "activations/layer10_attention_weight_min": -49.040679931640625,
      "activations/layer11_attention_weight_max": 47.76212692260742,
      "activations/layer11_attention_weight_min": -37.00930404663086,
      "activations/layer12_attention_weight_max": 36.30712890625,
      "activations/layer12_attention_weight_min": -28.297481536865234,
      "activations/layer13_attention_weight_max": 48.53345489501953,
      "activations/layer13_attention_weight_min": -35.087181091308594,
      "activations/layer14_attention_weight_max": 82.0185317993164,
      "activations/layer14_attention_weight_min": -56.608253479003906,
      "activations/layer15_attention_weight_max": 55.09260559082031,
      "activations/layer15_attention_weight_min": -37.09786605834961,
      "activations/layer16_attention_weight_max": 63.54639434814453,
      "activations/layer16_attention_weight_min": -39.7426872253418,
      "activations/layer17_attention_weight_max": 42.77994155883789,
      "activations/layer17_attention_weight_min": -27.0991268157959,
      "activations/layer18_attention_weight_max": 25.493614196777344,
      "activations/layer18_attention_weight_min": -14.660947799682617,
      "activations/layer19_attention_weight_max": 55.61912155151367,
      "activations/layer19_attention_weight_min": -32.30552291870117,
      "activations/layer1_attention_weight_max": 15.97930908203125,
      "activations/layer1_attention_weight_min": -14.49625301361084,
      "activations/layer20_attention_weight_max": 43.29804611206055,
      "activations/layer20_attention_weight_min": -28.08547019958496,
      "activations/layer21_attention_weight_max": 27.661346435546875,
      "activations/layer21_attention_weight_min": -15.221681594848633,
      "activations/layer22_attention_weight_max": 42.34928512573242,
      "activations/layer22_attention_weight_min": -23.363035202026367,
      "activations/layer23_attention_weight_max": 25.690231323242188,
      "activations/layer23_attention_weight_min": -14.831138610839844,
      "activations/layer2_attention_weight_max": 25.33397674560547,
      "activations/layer2_attention_weight_min": -25.15634536743164,
      "activations/layer3_attention_weight_max": 53.85926818847656,
      "activations/layer3_attention_weight_min": -60.12830352783203,
      "activations/layer4_attention_weight_max": 105.5380630493164,
      "activations/layer4_attention_weight_min": -106.00788879394531,
      "activations/layer5_attention_weight_max": 119.32921600341797,
      "activations/layer5_attention_weight_min": -110.58477020263672,
      "activations/layer6_attention_weight_max": 63.63772964477539,
      "activations/layer6_attention_weight_min": -58.50373458862305,
      "activations/layer7_attention_weight_max": 61.38828659057617,
      "activations/layer7_attention_weight_min": -60.45228576660156,
      "activations/layer8_attention_weight_max": 59.78008270263672,
      "activations/layer8_attention_weight_min": -53.17137908935547,
      "activations/layer9_attention_weight_max": 82.52930450439453,
      "activations/layer9_attention_weight_min": -67.7659683227539,
      "epoch": 1.02,
      "learning_rate": 0.000144880303030303,
      "loss": 3.0966,
      "step": 17550
    },
    {
      "activations/layer0_attention_weight_max": 16.00956153869629,
      "activations/layer0_attention_weight_min": -14.951933860778809,
      "activations/layer10_attention_weight_max": 62.310508728027344,
      "activations/layer10_attention_weight_min": -51.61948013305664,
      "activations/layer11_attention_weight_max": 43.691261291503906,
      "activations/layer11_attention_weight_min": -39.0329475402832,
      "activations/layer12_attention_weight_max": 36.30853271484375,
      "activations/layer12_attention_weight_min": -29.001802444458008,
      "activations/layer13_attention_weight_max": 41.44489669799805,
      "activations/layer13_attention_weight_min": -36.20598602294922,
      "activations/layer14_attention_weight_max": 72.9449691772461,
      "activations/layer14_attention_weight_min": -53.00705337524414,
      "activations/layer15_attention_weight_max": 57.7651481628418,
      "activations/layer15_attention_weight_min": -42.387489318847656,
      "activations/layer16_attention_weight_max": 54.66977310180664,
      "activations/layer16_attention_weight_min": -37.6977653503418,
      "activations/layer17_attention_weight_max": 40.03660202026367,
      "activations/layer17_attention_weight_min": -26.579395294189453,
      "activations/layer18_attention_weight_max": 24.00031852722168,
      "activations/layer18_attention_weight_min": -14.078457832336426,
      "activations/layer19_attention_weight_max": 52.335853576660156,
      "activations/layer19_attention_weight_min": -32.246395111083984,
      "activations/layer1_attention_weight_max": 15.81810474395752,
      "activations/layer1_attention_weight_min": -13.933340072631836,
      "activations/layer20_attention_weight_max": 42.29246139526367,
      "activations/layer20_attention_weight_min": -28.13519287109375,
      "activations/layer21_attention_weight_max": 22.44968605041504,
      "activations/layer21_attention_weight_min": -15.006195068359375,
      "activations/layer22_attention_weight_max": 39.18577194213867,
      "activations/layer22_attention_weight_min": -22.891826629638672,
      "activations/layer23_attention_weight_max": 24.532180786132812,
      "activations/layer23_attention_weight_min": -15.52878475189209,
      "activations/layer2_attention_weight_max": 24.9088134765625,
      "activations/layer2_attention_weight_min": -26.524917602539062,
      "activations/layer3_attention_weight_max": 56.25529861450195,
      "activations/layer3_attention_weight_min": -59.311283111572266,
      "activations/layer4_attention_weight_max": 105.3430404663086,
      "activations/layer4_attention_weight_min": -103.53621673583984,
      "activations/layer5_attention_weight_max": 121.2064208984375,
      "activations/layer5_attention_weight_min": -113.24345397949219,
      "activations/layer6_attention_weight_max": 65.06864929199219,
      "activations/layer6_attention_weight_min": -58.231781005859375,
      "activations/layer7_attention_weight_max": 60.9234504699707,
      "activations/layer7_attention_weight_min": -63.478965759277344,
      "activations/layer8_attention_weight_max": 59.99712371826172,
      "activations/layer8_attention_weight_min": -54.0174674987793,
      "activations/layer9_attention_weight_max": 70.55899810791016,
      "activations/layer9_attention_weight_min": -64.14237976074219,
      "epoch": 1.02,
      "learning_rate": 0.00014486136363636362,
      "loss": 3.1021,
      "step": 17600
    },
    {
      "activations/layer0_attention_weight_max": 16.0333309173584,
      "activations/layer0_attention_weight_min": -15.107830047607422,
      "activations/layer10_attention_weight_max": 50.89619827270508,
      "activations/layer10_attention_weight_min": -46.064144134521484,
      "activations/layer11_attention_weight_max": 42.95527648925781,
      "activations/layer11_attention_weight_min": -39.171146392822266,
      "activations/layer12_attention_weight_max": 44.25823211669922,
      "activations/layer12_attention_weight_min": -35.668697357177734,
      "activations/layer13_attention_weight_max": 51.21592712402344,
      "activations/layer13_attention_weight_min": -34.64585876464844,
      "activations/layer14_attention_weight_max": 71.96639251708984,
      "activations/layer14_attention_weight_min": -53.73478317260742,
      "activations/layer15_attention_weight_max": 49.310699462890625,
      "activations/layer15_attention_weight_min": -35.94943618774414,
      "activations/layer16_attention_weight_max": 50.36088562011719,
      "activations/layer16_attention_weight_min": -37.29544448852539,
      "activations/layer17_attention_weight_max": 33.691253662109375,
      "activations/layer17_attention_weight_min": -22.695711135864258,
      "activations/layer18_attention_weight_max": 27.06059455871582,
      "activations/layer18_attention_weight_min": -13.558208465576172,
      "activations/layer19_attention_weight_max": 44.30398178100586,
      "activations/layer19_attention_weight_min": -35.46875,
      "activations/layer1_attention_weight_max": 16.35492706298828,
      "activations/layer1_attention_weight_min": -12.7479248046875,
      "activations/layer20_attention_weight_max": 44.66271209716797,
      "activations/layer20_attention_weight_min": -24.073179244995117,
      "activations/layer21_attention_weight_max": 25.270889282226562,
      "activations/layer21_attention_weight_min": -15.377772331237793,
      "activations/layer22_attention_weight_max": 37.2740478515625,
      "activations/layer22_attention_weight_min": -22.062387466430664,
      "activations/layer23_attention_weight_max": 21.550634384155273,
      "activations/layer23_attention_weight_min": -14.240835189819336,
      "activations/layer2_attention_weight_max": 24.34653091430664,
      "activations/layer2_attention_weight_min": -24.740238189697266,
      "activations/layer3_attention_weight_max": 55.06883239746094,
      "activations/layer3_attention_weight_min": -57.137916564941406,
      "activations/layer4_attention_weight_max": 104.31098937988281,
      "activations/layer4_attention_weight_min": -100.94664001464844,
      "activations/layer5_attention_weight_max": 120.81838989257812,
      "activations/layer5_attention_weight_min": -110.44795227050781,
      "activations/layer6_attention_weight_max": 64.57449340820312,
      "activations/layer6_attention_weight_min": -57.966617584228516,
      "activations/layer7_attention_weight_max": 63.8599739074707,
      "activations/layer7_attention_weight_min": -61.12557601928711,
      "activations/layer8_attention_weight_max": 55.00080871582031,
      "activations/layer8_attention_weight_min": -47.91200256347656,
      "activations/layer9_attention_weight_max": 93.49724578857422,
      "activations/layer9_attention_weight_min": -73.30221557617188,
      "epoch": 1.03,
      "learning_rate": 0.00014484242424242424,
      "loss": 3.0808,
      "step": 17650
    },
    {
      "activations/layer0_attention_weight_max": 14.126378059387207,
      "activations/layer0_attention_weight_min": -14.882392883300781,
      "activations/layer10_attention_weight_max": 54.437774658203125,
      "activations/layer10_attention_weight_min": -45.6336669921875,
      "activations/layer11_attention_weight_max": 44.97547912597656,
      "activations/layer11_attention_weight_min": -39.2955322265625,
      "activations/layer12_attention_weight_max": 47.37580108642578,
      "activations/layer12_attention_weight_min": -39.5433349609375,
      "activations/layer13_attention_weight_max": 55.14305877685547,
      "activations/layer13_attention_weight_min": -42.61061477661133,
      "activations/layer14_attention_weight_max": 72.91130828857422,
      "activations/layer14_attention_weight_min": -51.68348693847656,
      "activations/layer15_attention_weight_max": 55.78572463989258,
      "activations/layer15_attention_weight_min": -38.699378967285156,
      "activations/layer16_attention_weight_max": 53.029605865478516,
      "activations/layer16_attention_weight_min": -38.89971923828125,
      "activations/layer17_attention_weight_max": 37.407535552978516,
      "activations/layer17_attention_weight_min": -25.739885330200195,
      "activations/layer18_attention_weight_max": 24.341548919677734,
      "activations/layer18_attention_weight_min": -14.678037643432617,
      "activations/layer19_attention_weight_max": 45.91999435424805,
      "activations/layer19_attention_weight_min": -32.95746994018555,
      "activations/layer1_attention_weight_max": 15.959009170532227,
      "activations/layer1_attention_weight_min": -13.103593826293945,
      "activations/layer20_attention_weight_max": 42.023895263671875,
      "activations/layer20_attention_weight_min": -27.220901489257812,
      "activations/layer21_attention_weight_max": 23.49985122680664,
      "activations/layer21_attention_weight_min": -14.768752098083496,
      "activations/layer22_attention_weight_max": 44.26082992553711,
      "activations/layer22_attention_weight_min": -22.51458168029785,
      "activations/layer23_attention_weight_max": 27.54909324645996,
      "activations/layer23_attention_weight_min": -15.139318466186523,
      "activations/layer2_attention_weight_max": 27.5467586517334,
      "activations/layer2_attention_weight_min": -25.63085174560547,
      "activations/layer3_attention_weight_max": 53.06694030761719,
      "activations/layer3_attention_weight_min": -55.63631057739258,
      "activations/layer4_attention_weight_max": 99.80986785888672,
      "activations/layer4_attention_weight_min": -99.94190979003906,
      "activations/layer5_attention_weight_max": 113.31246948242188,
      "activations/layer5_attention_weight_min": -105.12480926513672,
      "activations/layer6_attention_weight_max": 62.365509033203125,
      "activations/layer6_attention_weight_min": -58.83416748046875,
      "activations/layer7_attention_weight_max": 56.56447982788086,
      "activations/layer7_attention_weight_min": -57.222835540771484,
      "activations/layer8_attention_weight_max": 56.00823974609375,
      "activations/layer8_attention_weight_min": -54.86171340942383,
      "activations/layer9_attention_weight_max": 68.59185028076172,
      "activations/layer9_attention_weight_min": -62.9581413269043,
      "epoch": 1.03,
      "learning_rate": 0.00014482348484848483,
      "loss": 3.0758,
      "step": 17700
    },
    {
      "activations/layer0_attention_weight_max": 16.80782127380371,
      "activations/layer0_attention_weight_min": -15.251697540283203,
      "activations/layer10_attention_weight_max": 56.13069534301758,
      "activations/layer10_attention_weight_min": -51.69282150268555,
      "activations/layer11_attention_weight_max": 45.58301544189453,
      "activations/layer11_attention_weight_min": -42.88475799560547,
      "activations/layer12_attention_weight_max": 34.39139175415039,
      "activations/layer12_attention_weight_min": -28.408191680908203,
      "activations/layer13_attention_weight_max": 54.708763122558594,
      "activations/layer13_attention_weight_min": -38.53844451904297,
      "activations/layer14_attention_weight_max": 75.74005889892578,
      "activations/layer14_attention_weight_min": -56.078338623046875,
      "activations/layer15_attention_weight_max": 52.66372299194336,
      "activations/layer15_attention_weight_min": -41.18231964111328,
      "activations/layer16_attention_weight_max": 56.15713119506836,
      "activations/layer16_attention_weight_min": -41.74638748168945,
      "activations/layer17_attention_weight_max": 41.153263092041016,
      "activations/layer17_attention_weight_min": -25.563716888427734,
      "activations/layer18_attention_weight_max": 22.750099182128906,
      "activations/layer18_attention_weight_min": -14.590547561645508,
      "activations/layer19_attention_weight_max": 50.46006393432617,
      "activations/layer19_attention_weight_min": -36.54202651977539,
      "activations/layer1_attention_weight_max": 15.34788990020752,
      "activations/layer1_attention_weight_min": -13.523872375488281,
      "activations/layer20_attention_weight_max": 41.359004974365234,
      "activations/layer20_attention_weight_min": -24.543479919433594,
      "activations/layer21_attention_weight_max": 22.30710792541504,
      "activations/layer21_attention_weight_min": -14.207586288452148,
      "activations/layer22_attention_weight_max": 41.11402130126953,
      "activations/layer22_attention_weight_min": -26.120542526245117,
      "activations/layer23_attention_weight_max": 26.066715240478516,
      "activations/layer23_attention_weight_min": -12.934898376464844,
      "activations/layer2_attention_weight_max": 25.307422637939453,
      "activations/layer2_attention_weight_min": -26.160137176513672,
      "activations/layer3_attention_weight_max": 55.07083511352539,
      "activations/layer3_attention_weight_min": -57.724082946777344,
      "activations/layer4_attention_weight_max": 108.37457275390625,
      "activations/layer4_attention_weight_min": -100.01752471923828,
      "activations/layer5_attention_weight_max": 126.0196762084961,
      "activations/layer5_attention_weight_min": -107.53927612304688,
      "activations/layer6_attention_weight_max": 66.68663024902344,
      "activations/layer6_attention_weight_min": -59.17160415649414,
      "activations/layer7_attention_weight_max": 63.291595458984375,
      "activations/layer7_attention_weight_min": -59.26128005981445,
      "activations/layer8_attention_weight_max": 60.88717269897461,
      "activations/layer8_attention_weight_min": -53.07426071166992,
      "activations/layer9_attention_weight_max": 76.42093658447266,
      "activations/layer9_attention_weight_min": -60.81071853637695,
      "epoch": 1.03,
      "learning_rate": 0.00014480454545454544,
      "loss": 3.0758,
      "step": 17750
    },
    {
      "activations/layer0_attention_weight_max": 15.838528633117676,
      "activations/layer0_attention_weight_min": -13.375189781188965,
      "activations/layer10_attention_weight_max": 54.79787063598633,
      "activations/layer10_attention_weight_min": -45.09767532348633,
      "activations/layer11_attention_weight_max": 44.07801055908203,
      "activations/layer11_attention_weight_min": -36.373695373535156,
      "activations/layer12_attention_weight_max": 50.864559173583984,
      "activations/layer12_attention_weight_min": -37.88279342651367,
      "activations/layer13_attention_weight_max": 50.614627838134766,
      "activations/layer13_attention_weight_min": -41.1337890625,
      "activations/layer14_attention_weight_max": 75.05850219726562,
      "activations/layer14_attention_weight_min": -56.66570281982422,
      "activations/layer15_attention_weight_max": 59.94833755493164,
      "activations/layer15_attention_weight_min": -40.28007888793945,
      "activations/layer16_attention_weight_max": 56.46536636352539,
      "activations/layer16_attention_weight_min": -39.44152069091797,
      "activations/layer17_attention_weight_max": 37.88551330566406,
      "activations/layer17_attention_weight_min": -24.13329315185547,
      "activations/layer18_attention_weight_max": 22.502832412719727,
      "activations/layer18_attention_weight_min": -14.5558443069458,
      "activations/layer19_attention_weight_max": 49.53274154663086,
      "activations/layer19_attention_weight_min": -31.937789916992188,
      "activations/layer1_attention_weight_max": 15.78121280670166,
      "activations/layer1_attention_weight_min": -13.078996658325195,
      "activations/layer20_attention_weight_max": 38.190887451171875,
      "activations/layer20_attention_weight_min": -23.186729431152344,
      "activations/layer21_attention_weight_max": 22.929224014282227,
      "activations/layer21_attention_weight_min": -14.913009643554688,
      "activations/layer22_attention_weight_max": 34.999488830566406,
      "activations/layer22_attention_weight_min": -22.219484329223633,
      "activations/layer23_attention_weight_max": 22.560949325561523,
      "activations/layer23_attention_weight_min": -14.410552978515625,
      "activations/layer2_attention_weight_max": 25.763866424560547,
      "activations/layer2_attention_weight_min": -24.540563583374023,
      "activations/layer3_attention_weight_max": 57.123016357421875,
      "activations/layer3_attention_weight_min": -60.04233932495117,
      "activations/layer4_attention_weight_max": 110.38722229003906,
      "activations/layer4_attention_weight_min": -105.13066864013672,
      "activations/layer5_attention_weight_max": 119.60984802246094,
      "activations/layer5_attention_weight_min": -107.36344909667969,
      "activations/layer6_attention_weight_max": 63.345672607421875,
      "activations/layer6_attention_weight_min": -57.04197311401367,
      "activations/layer7_attention_weight_max": 60.37824630737305,
      "activations/layer7_attention_weight_min": -60.369754791259766,
      "activations/layer8_attention_weight_max": 63.10750198364258,
      "activations/layer8_attention_weight_min": -59.49238586425781,
      "activations/layer9_attention_weight_max": 72.11708068847656,
      "activations/layer9_attention_weight_min": -63.104408264160156,
      "epoch": 1.03,
      "learning_rate": 0.00014478560606060606,
      "loss": 3.0942,
      "step": 17800
    },
    {
      "activations/layer0_attention_weight_max": 16.084922790527344,
      "activations/layer0_attention_weight_min": -14.148091316223145,
      "activations/layer10_attention_weight_max": 56.09496307373047,
      "activations/layer10_attention_weight_min": -51.104576110839844,
      "activations/layer11_attention_weight_max": 42.320186614990234,
      "activations/layer11_attention_weight_min": -36.38801574707031,
      "activations/layer12_attention_weight_max": 39.2759895324707,
      "activations/layer12_attention_weight_min": -30.620283126831055,
      "activations/layer13_attention_weight_max": 50.783267974853516,
      "activations/layer13_attention_weight_min": -40.380157470703125,
      "activations/layer14_attention_weight_max": 86.80113220214844,
      "activations/layer14_attention_weight_min": -59.619693756103516,
      "activations/layer15_attention_weight_max": 53.749595642089844,
      "activations/layer15_attention_weight_min": -43.70235824584961,
      "activations/layer16_attention_weight_max": 55.480979919433594,
      "activations/layer16_attention_weight_min": -39.54039764404297,
      "activations/layer17_attention_weight_max": 36.221519470214844,
      "activations/layer17_attention_weight_min": -28.68970489501953,
      "activations/layer18_attention_weight_max": 21.640661239624023,
      "activations/layer18_attention_weight_min": -15.521770477294922,
      "activations/layer19_attention_weight_max": 48.564022064208984,
      "activations/layer19_attention_weight_min": -34.386348724365234,
      "activations/layer1_attention_weight_max": 15.749980926513672,
      "activations/layer1_attention_weight_min": -11.98553466796875,
      "activations/layer20_attention_weight_max": 39.889976501464844,
      "activations/layer20_attention_weight_min": -29.827552795410156,
      "activations/layer21_attention_weight_max": 22.441205978393555,
      "activations/layer21_attention_weight_min": -15.296257972717285,
      "activations/layer22_attention_weight_max": 41.7542610168457,
      "activations/layer22_attention_weight_min": -23.79541015625,
      "activations/layer23_attention_weight_max": 25.11471939086914,
      "activations/layer23_attention_weight_min": -16.821056365966797,
      "activations/layer2_attention_weight_max": 26.49854278564453,
      "activations/layer2_attention_weight_min": -25.044994354248047,
      "activations/layer3_attention_weight_max": 56.23889923095703,
      "activations/layer3_attention_weight_min": -56.90653610229492,
      "activations/layer4_attention_weight_max": 104.23465728759766,
      "activations/layer4_attention_weight_min": -103.675048828125,
      "activations/layer5_attention_weight_max": 120.07444763183594,
      "activations/layer5_attention_weight_min": -104.19413757324219,
      "activations/layer6_attention_weight_max": 63.54035949707031,
      "activations/layer6_attention_weight_min": -59.79667282104492,
      "activations/layer7_attention_weight_max": 60.721439361572266,
      "activations/layer7_attention_weight_min": -63.264854431152344,
      "activations/layer8_attention_weight_max": 58.88840103149414,
      "activations/layer8_attention_weight_min": -49.416770935058594,
      "activations/layer9_attention_weight_max": 74.82867431640625,
      "activations/layer9_attention_weight_min": -61.05054473876953,
      "epoch": 1.04,
      "learning_rate": 0.00014476666666666667,
      "loss": 3.083,
      "step": 17850
    },
    {
      "activations/layer0_attention_weight_max": 16.799537658691406,
      "activations/layer0_attention_weight_min": -13.693343162536621,
      "activations/layer10_attention_weight_max": 58.03754425048828,
      "activations/layer10_attention_weight_min": -49.81235122680664,
      "activations/layer11_attention_weight_max": 43.49457550048828,
      "activations/layer11_attention_weight_min": -35.834007263183594,
      "activations/layer12_attention_weight_max": 41.03236770629883,
      "activations/layer12_attention_weight_min": -32.68214797973633,
      "activations/layer13_attention_weight_max": 46.90174865722656,
      "activations/layer13_attention_weight_min": -36.05690002441406,
      "activations/layer14_attention_weight_max": 79.14520263671875,
      "activations/layer14_attention_weight_min": -53.558536529541016,
      "activations/layer15_attention_weight_max": 56.846412658691406,
      "activations/layer15_attention_weight_min": -37.279293060302734,
      "activations/layer16_attention_weight_max": 55.77326965332031,
      "activations/layer16_attention_weight_min": -38.942718505859375,
      "activations/layer17_attention_weight_max": 41.45840835571289,
      "activations/layer17_attention_weight_min": -26.66890525817871,
      "activations/layer18_attention_weight_max": 20.984010696411133,
      "activations/layer18_attention_weight_min": -14.676955223083496,
      "activations/layer19_attention_weight_max": 47.737300872802734,
      "activations/layer19_attention_weight_min": -28.529300689697266,
      "activations/layer1_attention_weight_max": 15.526260375976562,
      "activations/layer1_attention_weight_min": -12.589591979980469,
      "activations/layer20_attention_weight_max": 46.72603225708008,
      "activations/layer20_attention_weight_min": -22.501869201660156,
      "activations/layer21_attention_weight_max": 22.046157836914062,
      "activations/layer21_attention_weight_min": -14.561366081237793,
      "activations/layer22_attention_weight_max": 40.36844253540039,
      "activations/layer22_attention_weight_min": -23.031064987182617,
      "activations/layer23_attention_weight_max": 24.401960372924805,
      "activations/layer23_attention_weight_min": -14.48154354095459,
      "activations/layer2_attention_weight_max": 26.69826889038086,
      "activations/layer2_attention_weight_min": -25.088787078857422,
      "activations/layer3_attention_weight_max": 57.40784454345703,
      "activations/layer3_attention_weight_min": -57.92505645751953,
      "activations/layer4_attention_weight_max": 105.27434539794922,
      "activations/layer4_attention_weight_min": -99.67025756835938,
      "activations/layer5_attention_weight_max": 119.7738265991211,
      "activations/layer5_attention_weight_min": -104.9102783203125,
      "activations/layer6_attention_weight_max": 63.51301956176758,
      "activations/layer6_attention_weight_min": -60.33259201049805,
      "activations/layer7_attention_weight_max": 61.29824447631836,
      "activations/layer7_attention_weight_min": -64.18575286865234,
      "activations/layer8_attention_weight_max": 64.01344299316406,
      "activations/layer8_attention_weight_min": -49.13279342651367,
      "activations/layer9_attention_weight_max": 78.67719268798828,
      "activations/layer9_attention_weight_min": -62.180877685546875,
      "epoch": 1.04,
      "learning_rate": 0.00014474772727272726,
      "loss": 3.1015,
      "step": 17900
    },
    {
      "activations/layer0_attention_weight_max": 16.342348098754883,
      "activations/layer0_attention_weight_min": -13.866814613342285,
      "activations/layer10_attention_weight_max": 54.749267578125,
      "activations/layer10_attention_weight_min": -43.94576644897461,
      "activations/layer11_attention_weight_max": 51.607627868652344,
      "activations/layer11_attention_weight_min": -39.327232360839844,
      "activations/layer12_attention_weight_max": 53.42946243286133,
      "activations/layer12_attention_weight_min": -37.09033966064453,
      "activations/layer13_attention_weight_max": 66.81937408447266,
      "activations/layer13_attention_weight_min": -47.07783889770508,
      "activations/layer14_attention_weight_max": 84.54269409179688,
      "activations/layer14_attention_weight_min": -56.946815490722656,
      "activations/layer15_attention_weight_max": 66.09054565429688,
      "activations/layer15_attention_weight_min": -41.76171875,
      "activations/layer16_attention_weight_max": 53.34712219238281,
      "activations/layer16_attention_weight_min": -41.238067626953125,
      "activations/layer17_attention_weight_max": 35.21664810180664,
      "activations/layer17_attention_weight_min": -25.75704574584961,
      "activations/layer18_attention_weight_max": 22.705020904541016,
      "activations/layer18_attention_weight_min": -13.970256805419922,
      "activations/layer19_attention_weight_max": 44.46327209472656,
      "activations/layer19_attention_weight_min": -31.517663955688477,
      "activations/layer1_attention_weight_max": 15.577096939086914,
      "activations/layer1_attention_weight_min": -13.384210586547852,
      "activations/layer20_attention_weight_max": 37.277374267578125,
      "activations/layer20_attention_weight_min": -24.496496200561523,
      "activations/layer21_attention_weight_max": 19.6453914642334,
      "activations/layer21_attention_weight_min": -15.856379508972168,
      "activations/layer22_attention_weight_max": 36.42287063598633,
      "activations/layer22_attention_weight_min": -21.330419540405273,
      "activations/layer23_attention_weight_max": 25.139080047607422,
      "activations/layer23_attention_weight_min": -15.923332214355469,
      "activations/layer2_attention_weight_max": 26.10047149658203,
      "activations/layer2_attention_weight_min": -25.522964477539062,
      "activations/layer3_attention_weight_max": 56.10877227783203,
      "activations/layer3_attention_weight_min": -57.603153228759766,
      "activations/layer4_attention_weight_max": 106.06803131103516,
      "activations/layer4_attention_weight_min": -102.71334075927734,
      "activations/layer5_attention_weight_max": 126.02384948730469,
      "activations/layer5_attention_weight_min": -108.69935607910156,
      "activations/layer6_attention_weight_max": 66.35050964355469,
      "activations/layer6_attention_weight_min": -58.8585205078125,
      "activations/layer7_attention_weight_max": 62.71503829956055,
      "activations/layer7_attention_weight_min": -66.41078186035156,
      "activations/layer8_attention_weight_max": 66.67174530029297,
      "activations/layer8_attention_weight_min": -58.14868927001953,
      "activations/layer9_attention_weight_max": 83.8945541381836,
      "activations/layer9_attention_weight_min": -67.91795349121094,
      "epoch": 1.04,
      "learning_rate": 0.00014472878787878787,
      "loss": 3.0727,
      "step": 17950
    },
    {
      "activations/layer0_attention_weight_max": 16.12104606628418,
      "activations/layer0_attention_weight_min": -13.590928077697754,
      "activations/layer10_attention_weight_max": 49.80509567260742,
      "activations/layer10_attention_weight_min": -50.207794189453125,
      "activations/layer11_attention_weight_max": 43.283447265625,
      "activations/layer11_attention_weight_min": -40.746337890625,
      "activations/layer12_attention_weight_max": 41.47306442260742,
      "activations/layer12_attention_weight_min": -29.855424880981445,
      "activations/layer13_attention_weight_max": 44.152000427246094,
      "activations/layer13_attention_weight_min": -37.35519027709961,
      "activations/layer14_attention_weight_max": 67.91446685791016,
      "activations/layer14_attention_weight_min": -56.83869171142578,
      "activations/layer15_attention_weight_max": 48.683837890625,
      "activations/layer15_attention_weight_min": -38.128273010253906,
      "activations/layer16_attention_weight_max": 57.52413558959961,
      "activations/layer16_attention_weight_min": -40.859947204589844,
      "activations/layer17_attention_weight_max": 38.80956268310547,
      "activations/layer17_attention_weight_min": -25.925418853759766,
      "activations/layer18_attention_weight_max": 24.276432037353516,
      "activations/layer18_attention_weight_min": -17.248111724853516,
      "activations/layer19_attention_weight_max": 49.9791374206543,
      "activations/layer19_attention_weight_min": -33.85803985595703,
      "activations/layer1_attention_weight_max": 16.037458419799805,
      "activations/layer1_attention_weight_min": -11.778573036193848,
      "activations/layer20_attention_weight_max": 44.56487274169922,
      "activations/layer20_attention_weight_min": -26.32177734375,
      "activations/layer21_attention_weight_max": 21.09825325012207,
      "activations/layer21_attention_weight_min": -13.456130027770996,
      "activations/layer22_attention_weight_max": 39.9787483215332,
      "activations/layer22_attention_weight_min": -22.190771102905273,
      "activations/layer23_attention_weight_max": 23.543363571166992,
      "activations/layer23_attention_weight_min": -13.951820373535156,
      "activations/layer2_attention_weight_max": 25.71699333190918,
      "activations/layer2_attention_weight_min": -27.18878936767578,
      "activations/layer3_attention_weight_max": 56.86829376220703,
      "activations/layer3_attention_weight_min": -62.09856033325195,
      "activations/layer4_attention_weight_max": 104.05498504638672,
      "activations/layer4_attention_weight_min": -102.67559814453125,
      "activations/layer5_attention_weight_max": 113.29591369628906,
      "activations/layer5_attention_weight_min": -105.05885314941406,
      "activations/layer6_attention_weight_max": 59.566532135009766,
      "activations/layer6_attention_weight_min": -58.28899002075195,
      "activations/layer7_attention_weight_max": 61.07463073730469,
      "activations/layer7_attention_weight_min": -59.59599685668945,
      "activations/layer8_attention_weight_max": 52.56072998046875,
      "activations/layer8_attention_weight_min": -48.12327575683594,
      "activations/layer9_attention_weight_max": 62.95498275756836,
      "activations/layer9_attention_weight_min": -64.006103515625,
      "epoch": 1.05,
      "learning_rate": 0.0001447098484848485,
      "loss": 3.0781,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_loss": 3.0390625,
      "eval_runtime": 8.5405,
      "eval_samples_per_second": 502.781,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_openwebtext_loss": 3.0390625,
      "eval_openwebtext_ppl": 20.8856537535641,
      "eval_openwebtext_runtime": 8.5405,
      "eval_openwebtext_samples_per_second": 502.781,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_wikitext_loss": 3.306640625,
      "eval_wikitext_ppl": 27.293282919182133,
      "eval_wikitext_runtime": 1.9735,
      "eval_wikitext_samples_per_second": 231.062,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_lambada_loss": 3.595703125,
      "eval_lambada_ppl": 36.44131378072682,
      "eval_lambada_runtime": 9.5635,
      "eval_lambada_samples_per_second": 509.121,
      "step": 18000
    },
    {
      "activations/layer0_attention_weight_max": 16.069311141967773,
      "activations/layer0_attention_weight_min": -13.159175872802734,
      "activations/layer10_attention_weight_max": 57.28199005126953,
      "activations/layer10_attention_weight_min": -48.8817138671875,
      "activations/layer11_attention_weight_max": 46.53727340698242,
      "activations/layer11_attention_weight_min": -42.666412353515625,
      "activations/layer12_attention_weight_max": 60.261444091796875,
      "activations/layer12_attention_weight_min": -35.478755950927734,
      "activations/layer13_attention_weight_max": 42.73063278198242,
      "activations/layer13_attention_weight_min": -33.879676818847656,
      "activations/layer14_attention_weight_max": 68.07889556884766,
      "activations/layer14_attention_weight_min": -51.347599029541016,
      "activations/layer15_attention_weight_max": 54.20694351196289,
      "activations/layer15_attention_weight_min": -42.13550567626953,
      "activations/layer16_attention_weight_max": 46.78057098388672,
      "activations/layer16_attention_weight_min": -34.91303634643555,
      "activations/layer17_attention_weight_max": 32.3896484375,
      "activations/layer17_attention_weight_min": -24.326583862304688,
      "activations/layer18_attention_weight_max": 22.38136863708496,
      "activations/layer18_attention_weight_min": -14.350520133972168,
      "activations/layer19_attention_weight_max": 46.47087860107422,
      "activations/layer19_attention_weight_min": -31.082632064819336,
      "activations/layer1_attention_weight_max": 14.785845756530762,
      "activations/layer1_attention_weight_min": -11.711751937866211,
      "activations/layer20_attention_weight_max": 40.915531158447266,
      "activations/layer20_attention_weight_min": -25.11758041381836,
      "activations/layer21_attention_weight_max": 22.5751953125,
      "activations/layer21_attention_weight_min": -14.447991371154785,
      "activations/layer22_attention_weight_max": 38.3808708190918,
      "activations/layer22_attention_weight_min": -23.41871452331543,
      "activations/layer23_attention_weight_max": 22.35840606689453,
      "activations/layer23_attention_weight_min": -14.45554256439209,
      "activations/layer2_attention_weight_max": 25.4141845703125,
      "activations/layer2_attention_weight_min": -25.533123016357422,
      "activations/layer3_attention_weight_max": 56.53163528442383,
      "activations/layer3_attention_weight_min": -61.86479568481445,
      "activations/layer4_attention_weight_max": 108.39383697509766,
      "activations/layer4_attention_weight_min": -105.36268615722656,
      "activations/layer5_attention_weight_max": 120.77198791503906,
      "activations/layer5_attention_weight_min": -109.09355163574219,
      "activations/layer6_attention_weight_max": 61.96358108520508,
      "activations/layer6_attention_weight_min": -57.4520263671875,
      "activations/layer7_attention_weight_max": 65.25080871582031,
      "activations/layer7_attention_weight_min": -66.07637023925781,
      "activations/layer8_attention_weight_max": 58.72782516479492,
      "activations/layer8_attention_weight_min": -49.07135009765625,
      "activations/layer9_attention_weight_max": 72.76769256591797,
      "activations/layer9_attention_weight_min": -62.20207595825195,
      "epoch": 1.05,
      "learning_rate": 0.00014469090909090908,
      "loss": 3.0809,
      "step": 18050
    },
    {
      "activations/layer0_attention_weight_max": 14.191095352172852,
      "activations/layer0_attention_weight_min": -14.075934410095215,
      "activations/layer10_attention_weight_max": 59.65009689331055,
      "activations/layer10_attention_weight_min": -50.23554229736328,
      "activations/layer11_attention_weight_max": 45.584407806396484,
      "activations/layer11_attention_weight_min": -39.73204040527344,
      "activations/layer12_attention_weight_max": 34.6658821105957,
      "activations/layer12_attention_weight_min": -28.656599044799805,
      "activations/layer13_attention_weight_max": 47.745182037353516,
      "activations/layer13_attention_weight_min": -37.851661682128906,
      "activations/layer14_attention_weight_max": 79.20621490478516,
      "activations/layer14_attention_weight_min": -61.429908752441406,
      "activations/layer15_attention_weight_max": 54.5907096862793,
      "activations/layer15_attention_weight_min": -39.81031036376953,
      "activations/layer16_attention_weight_max": 61.69221878051758,
      "activations/layer16_attention_weight_min": -42.116912841796875,
      "activations/layer17_attention_weight_max": 35.71194076538086,
      "activations/layer17_attention_weight_min": -24.30152130126953,
      "activations/layer18_attention_weight_max": 21.47123146057129,
      "activations/layer18_attention_weight_min": -13.993880271911621,
      "activations/layer19_attention_weight_max": 45.965145111083984,
      "activations/layer19_attention_weight_min": -32.2242546081543,
      "activations/layer1_attention_weight_max": 15.63288688659668,
      "activations/layer1_attention_weight_min": -11.780753135681152,
      "activations/layer20_attention_weight_max": 38.68551254272461,
      "activations/layer20_attention_weight_min": -28.47990608215332,
      "activations/layer21_attention_weight_max": 22.985401153564453,
      "activations/layer21_attention_weight_min": -16.824817657470703,
      "activations/layer22_attention_weight_max": 36.824310302734375,
      "activations/layer22_attention_weight_min": -21.803552627563477,
      "activations/layer23_attention_weight_max": 21.842578887939453,
      "activations/layer23_attention_weight_min": -14.37055778503418,
      "activations/layer2_attention_weight_max": 26.599393844604492,
      "activations/layer2_attention_weight_min": -25.94121551513672,
      "activations/layer3_attention_weight_max": 57.80575180053711,
      "activations/layer3_attention_weight_min": -58.407310485839844,
      "activations/layer4_attention_weight_max": 105.7972412109375,
      "activations/layer4_attention_weight_min": -102.6163330078125,
      "activations/layer5_attention_weight_max": 113.95600891113281,
      "activations/layer5_attention_weight_min": -102.50057220458984,
      "activations/layer6_attention_weight_max": 63.504661560058594,
      "activations/layer6_attention_weight_min": -56.654693603515625,
      "activations/layer7_attention_weight_max": 60.47345733642578,
      "activations/layer7_attention_weight_min": -56.09617233276367,
      "activations/layer8_attention_weight_max": 57.55944061279297,
      "activations/layer8_attention_weight_min": -51.23665237426758,
      "activations/layer9_attention_weight_max": 71.53181457519531,
      "activations/layer9_attention_weight_min": -62.14051055908203,
      "epoch": 1.05,
      "learning_rate": 0.0001446719696969697,
      "loss": 3.1003,
      "step": 18100
    },
    {
      "activations/layer0_attention_weight_max": 16.52513885498047,
      "activations/layer0_attention_weight_min": -13.574199676513672,
      "activations/layer10_attention_weight_max": 52.06009292602539,
      "activations/layer10_attention_weight_min": -43.99896240234375,
      "activations/layer11_attention_weight_max": 44.534576416015625,
      "activations/layer11_attention_weight_min": -35.1418571472168,
      "activations/layer12_attention_weight_max": 40.37875747680664,
      "activations/layer12_attention_weight_min": -31.956384658813477,
      "activations/layer13_attention_weight_max": 49.58633041381836,
      "activations/layer13_attention_weight_min": -34.13261413574219,
      "activations/layer14_attention_weight_max": 65.62210845947266,
      "activations/layer14_attention_weight_min": -50.48881912231445,
      "activations/layer15_attention_weight_max": 49.245948791503906,
      "activations/layer15_attention_weight_min": -36.74904251098633,
      "activations/layer16_attention_weight_max": 51.923458099365234,
      "activations/layer16_attention_weight_min": -38.47309875488281,
      "activations/layer17_attention_weight_max": 36.570350646972656,
      "activations/layer17_attention_weight_min": -27.027055740356445,
      "activations/layer18_attention_weight_max": 26.427051544189453,
      "activations/layer18_attention_weight_min": -15.830984115600586,
      "activations/layer19_attention_weight_max": 54.01740264892578,
      "activations/layer19_attention_weight_min": -28.38956642150879,
      "activations/layer1_attention_weight_max": 16.226314544677734,
      "activations/layer1_attention_weight_min": -12.661288261413574,
      "activations/layer20_attention_weight_max": 37.69218063354492,
      "activations/layer20_attention_weight_min": -24.694833755493164,
      "activations/layer21_attention_weight_max": 21.57129669189453,
      "activations/layer21_attention_weight_min": -15.609307289123535,
      "activations/layer22_attention_weight_max": 37.421051025390625,
      "activations/layer22_attention_weight_min": -21.740732192993164,
      "activations/layer23_attention_weight_max": 23.763837814331055,
      "activations/layer23_attention_weight_min": -16.413185119628906,
      "activations/layer2_attention_weight_max": 26.609716415405273,
      "activations/layer2_attention_weight_min": -27.12836456298828,
      "activations/layer3_attention_weight_max": 58.22119140625,
      "activations/layer3_attention_weight_min": -59.34845733642578,
      "activations/layer4_attention_weight_max": 104.81461334228516,
      "activations/layer4_attention_weight_min": -102.7030258178711,
      "activations/layer5_attention_weight_max": 114.95018005371094,
      "activations/layer5_attention_weight_min": -106.79013061523438,
      "activations/layer6_attention_weight_max": 59.75404357910156,
      "activations/layer6_attention_weight_min": -55.234397888183594,
      "activations/layer7_attention_weight_max": 59.934844970703125,
      "activations/layer7_attention_weight_min": -58.98731994628906,
      "activations/layer8_attention_weight_max": 53.361026763916016,
      "activations/layer8_attention_weight_min": -47.271976470947266,
      "activations/layer9_attention_weight_max": 70.02349090576172,
      "activations/layer9_attention_weight_min": -63.823219299316406,
      "epoch": 1.05,
      "learning_rate": 0.0001446530303030303,
      "loss": 3.0884,
      "step": 18150
    },
    {
      "activations/layer0_attention_weight_max": 16.471694946289062,
      "activations/layer0_attention_weight_min": -14.127968788146973,
      "activations/layer10_attention_weight_max": 52.06943893432617,
      "activations/layer10_attention_weight_min": -46.60767364501953,
      "activations/layer11_attention_weight_max": 44.8330078125,
      "activations/layer11_attention_weight_min": -38.9547119140625,
      "activations/layer12_attention_weight_max": 38.91836166381836,
      "activations/layer12_attention_weight_min": -27.55817985534668,
      "activations/layer13_attention_weight_max": 50.94546890258789,
      "activations/layer13_attention_weight_min": -36.05036163330078,
      "activations/layer14_attention_weight_max": 75.49710083007812,
      "activations/layer14_attention_weight_min": -54.15058898925781,
      "activations/layer15_attention_weight_max": 61.05787658691406,
      "activations/layer15_attention_weight_min": -41.51126480102539,
      "activations/layer16_attention_weight_max": 67.4173355102539,
      "activations/layer16_attention_weight_min": -44.5257568359375,
      "activations/layer17_attention_weight_max": 41.48271179199219,
      "activations/layer17_attention_weight_min": -27.333778381347656,
      "activations/layer18_attention_weight_max": 25.037817001342773,
      "activations/layer18_attention_weight_min": -14.404508590698242,
      "activations/layer19_attention_weight_max": 61.97322463989258,
      "activations/layer19_attention_weight_min": -35.37255096435547,
      "activations/layer1_attention_weight_max": 15.982950210571289,
      "activations/layer1_attention_weight_min": -12.803170204162598,
      "activations/layer20_attention_weight_max": 45.37940979003906,
      "activations/layer20_attention_weight_min": -28.795936584472656,
      "activations/layer21_attention_weight_max": 25.351768493652344,
      "activations/layer21_attention_weight_min": -16.414926528930664,
      "activations/layer22_attention_weight_max": 42.7767333984375,
      "activations/layer22_attention_weight_min": -22.401615142822266,
      "activations/layer23_attention_weight_max": 27.05777359008789,
      "activations/layer23_attention_weight_min": -13.798437118530273,
      "activations/layer2_attention_weight_max": 28.139469146728516,
      "activations/layer2_attention_weight_min": -27.236295700073242,
      "activations/layer3_attention_weight_max": 58.371673583984375,
      "activations/layer3_attention_weight_min": -62.799678802490234,
      "activations/layer4_attention_weight_max": 108.77886199951172,
      "activations/layer4_attention_weight_min": -107.7659912109375,
      "activations/layer5_attention_weight_max": 122.20418548583984,
      "activations/layer5_attention_weight_min": -108.16656494140625,
      "activations/layer6_attention_weight_max": 66.55712890625,
      "activations/layer6_attention_weight_min": -57.59905242919922,
      "activations/layer7_attention_weight_max": 58.46902847290039,
      "activations/layer7_attention_weight_min": -56.56422424316406,
      "activations/layer8_attention_weight_max": 60.77655029296875,
      "activations/layer8_attention_weight_min": -48.66044998168945,
      "activations/layer9_attention_weight_max": 74.66313934326172,
      "activations/layer9_attention_weight_min": -58.972957611083984,
      "epoch": 1.06,
      "learning_rate": 0.0001446340909090909,
      "loss": 3.0817,
      "step": 18200
    },
    {
      "activations/layer0_attention_weight_max": 14.851737976074219,
      "activations/layer0_attention_weight_min": -13.17936897277832,
      "activations/layer10_attention_weight_max": 58.03124237060547,
      "activations/layer10_attention_weight_min": -47.51804733276367,
      "activations/layer11_attention_weight_max": 46.798927307128906,
      "activations/layer11_attention_weight_min": -42.863502502441406,
      "activations/layer12_attention_weight_max": 38.74762725830078,
      "activations/layer12_attention_weight_min": -28.47110366821289,
      "activations/layer13_attention_weight_max": 54.71654510498047,
      "activations/layer13_attention_weight_min": -41.10271453857422,
      "activations/layer14_attention_weight_max": 73.89160919189453,
      "activations/layer14_attention_weight_min": -55.76267623901367,
      "activations/layer15_attention_weight_max": 52.43950271606445,
      "activations/layer15_attention_weight_min": -43.75954818725586,
      "activations/layer16_attention_weight_max": 51.49538040161133,
      "activations/layer16_attention_weight_min": -37.37553787231445,
      "activations/layer17_attention_weight_max": 36.749969482421875,
      "activations/layer17_attention_weight_min": -24.25060272216797,
      "activations/layer18_attention_weight_max": 22.678916931152344,
      "activations/layer18_attention_weight_min": -14.043135643005371,
      "activations/layer19_attention_weight_max": 57.4063606262207,
      "activations/layer19_attention_weight_min": -35.00328826904297,
      "activations/layer1_attention_weight_max": 15.795248985290527,
      "activations/layer1_attention_weight_min": -11.935970306396484,
      "activations/layer20_attention_weight_max": 40.46833801269531,
      "activations/layer20_attention_weight_min": -28.109107971191406,
      "activations/layer21_attention_weight_max": 22.57393455505371,
      "activations/layer21_attention_weight_min": -16.87482261657715,
      "activations/layer22_attention_weight_max": 38.68653106689453,
      "activations/layer22_attention_weight_min": -24.242788314819336,
      "activations/layer23_attention_weight_max": 25.05614471435547,
      "activations/layer23_attention_weight_min": -15.420185089111328,
      "activations/layer2_attention_weight_max": 25.440465927124023,
      "activations/layer2_attention_weight_min": -25.385547637939453,
      "activations/layer3_attention_weight_max": 55.96711730957031,
      "activations/layer3_attention_weight_min": -57.811546325683594,
      "activations/layer4_attention_weight_max": 107.08514404296875,
      "activations/layer4_attention_weight_min": -104.90388488769531,
      "activations/layer5_attention_weight_max": 122.63526916503906,
      "activations/layer5_attention_weight_min": -105.2877197265625,
      "activations/layer6_attention_weight_max": 68.32429504394531,
      "activations/layer6_attention_weight_min": -55.96791458129883,
      "activations/layer7_attention_weight_max": 61.72908020019531,
      "activations/layer7_attention_weight_min": -59.754398345947266,
      "activations/layer8_attention_weight_max": 59.2115364074707,
      "activations/layer8_attention_weight_min": -50.56376647949219,
      "activations/layer9_attention_weight_max": 69.26466369628906,
      "activations/layer9_attention_weight_min": -68.85503387451172,
      "epoch": 1.06,
      "learning_rate": 0.0001446151515151515,
      "loss": 3.0931,
      "step": 18250
    },
    {
      "activations/layer0_attention_weight_max": 13.344717025756836,
      "activations/layer0_attention_weight_min": -11.918512344360352,
      "activations/layer10_attention_weight_max": 54.088741302490234,
      "activations/layer10_attention_weight_min": -45.90174865722656,
      "activations/layer11_attention_weight_max": 48.2449951171875,
      "activations/layer11_attention_weight_min": -41.399845123291016,
      "activations/layer12_attention_weight_max": 39.21937942504883,
      "activations/layer12_attention_weight_min": -30.675474166870117,
      "activations/layer13_attention_weight_max": 57.301116943359375,
      "activations/layer13_attention_weight_min": -36.892417907714844,
      "activations/layer14_attention_weight_max": 77.19062805175781,
      "activations/layer14_attention_weight_min": -53.09933853149414,
      "activations/layer15_attention_weight_max": 55.82673263549805,
      "activations/layer15_attention_weight_min": -37.04623031616211,
      "activations/layer16_attention_weight_max": 67.06303405761719,
      "activations/layer16_attention_weight_min": -42.42364501953125,
      "activations/layer17_attention_weight_max": 38.08928298950195,
      "activations/layer17_attention_weight_min": -26.36054801940918,
      "activations/layer18_attention_weight_max": 22.227909088134766,
      "activations/layer18_attention_weight_min": -15.258190155029297,
      "activations/layer19_attention_weight_max": 60.515716552734375,
      "activations/layer19_attention_weight_min": -33.78956985473633,
      "activations/layer1_attention_weight_max": 18.24393653869629,
      "activations/layer1_attention_weight_min": -12.449888229370117,
      "activations/layer20_attention_weight_max": 49.517696380615234,
      "activations/layer20_attention_weight_min": -26.593137741088867,
      "activations/layer21_attention_weight_max": 22.9372501373291,
      "activations/layer21_attention_weight_min": -13.572689056396484,
      "activations/layer22_attention_weight_max": 44.51533126831055,
      "activations/layer22_attention_weight_min": -26.72667121887207,
      "activations/layer23_attention_weight_max": 25.526044845581055,
      "activations/layer23_attention_weight_min": -15.44235610961914,
      "activations/layer2_attention_weight_max": 26.56696891784668,
      "activations/layer2_attention_weight_min": -25.713285446166992,
      "activations/layer3_attention_weight_max": 56.5599250793457,
      "activations/layer3_attention_weight_min": -58.143516540527344,
      "activations/layer4_attention_weight_max": 107.09620666503906,
      "activations/layer4_attention_weight_min": -105.1663589477539,
      "activations/layer5_attention_weight_max": 117.65956115722656,
      "activations/layer5_attention_weight_min": -105.90593719482422,
      "activations/layer6_attention_weight_max": 63.76409149169922,
      "activations/layer6_attention_weight_min": -59.892154693603516,
      "activations/layer7_attention_weight_max": 69.91764068603516,
      "activations/layer7_attention_weight_min": -60.49876022338867,
      "activations/layer8_attention_weight_max": 57.969482421875,
      "activations/layer8_attention_weight_min": -51.163002014160156,
      "activations/layer9_attention_weight_max": 80.77210235595703,
      "activations/layer9_attention_weight_min": -72.15096282958984,
      "epoch": 1.06,
      "learning_rate": 0.0001445962121212121,
      "loss": 3.0702,
      "step": 18300
    },
    {
      "activations/layer0_attention_weight_max": 13.7389554977417,
      "activations/layer0_attention_weight_min": -13.614580154418945,
      "activations/layer10_attention_weight_max": 54.96965026855469,
      "activations/layer10_attention_weight_min": -46.09160614013672,
      "activations/layer11_attention_weight_max": 44.7907829284668,
      "activations/layer11_attention_weight_min": -36.484336853027344,
      "activations/layer12_attention_weight_max": 39.95108413696289,
      "activations/layer12_attention_weight_min": -27.86517906188965,
      "activations/layer13_attention_weight_max": 50.84490203857422,
      "activations/layer13_attention_weight_min": -35.068790435791016,
      "activations/layer14_attention_weight_max": 82.14629364013672,
      "activations/layer14_attention_weight_min": -56.757747650146484,
      "activations/layer15_attention_weight_max": 59.74944305419922,
      "activations/layer15_attention_weight_min": -40.694114685058594,
      "activations/layer16_attention_weight_max": 57.03476333618164,
      "activations/layer16_attention_weight_min": -42.09950256347656,
      "activations/layer17_attention_weight_max": 41.89707946777344,
      "activations/layer17_attention_weight_min": -26.102819442749023,
      "activations/layer18_attention_weight_max": 23.967496871948242,
      "activations/layer18_attention_weight_min": -16.752239227294922,
      "activations/layer19_attention_weight_max": 53.2696647644043,
      "activations/layer19_attention_weight_min": -30.66388511657715,
      "activations/layer1_attention_weight_max": 16.597042083740234,
      "activations/layer1_attention_weight_min": -11.813685417175293,
      "activations/layer20_attention_weight_max": 39.96292495727539,
      "activations/layer20_attention_weight_min": -24.95952606201172,
      "activations/layer21_attention_weight_max": 24.986644744873047,
      "activations/layer21_attention_weight_min": -16.031414031982422,
      "activations/layer22_attention_weight_max": 39.83150100708008,
      "activations/layer22_attention_weight_min": -24.54409408569336,
      "activations/layer23_attention_weight_max": 24.112102508544922,
      "activations/layer23_attention_weight_min": -15.29532241821289,
      "activations/layer2_attention_weight_max": 29.19227409362793,
      "activations/layer2_attention_weight_min": -30.23592758178711,
      "activations/layer3_attention_weight_max": 60.33842468261719,
      "activations/layer3_attention_weight_min": -63.41755294799805,
      "activations/layer4_attention_weight_max": 110.81037902832031,
      "activations/layer4_attention_weight_min": -110.05204010009766,
      "activations/layer5_attention_weight_max": 126.61738586425781,
      "activations/layer5_attention_weight_min": -118.1006088256836,
      "activations/layer6_attention_weight_max": 67.13545989990234,
      "activations/layer6_attention_weight_min": -60.61589050292969,
      "activations/layer7_attention_weight_max": 65.93229675292969,
      "activations/layer7_attention_weight_min": -59.767112731933594,
      "activations/layer8_attention_weight_max": 61.04106521606445,
      "activations/layer8_attention_weight_min": -53.45623779296875,
      "activations/layer9_attention_weight_max": 85.01348114013672,
      "activations/layer9_attention_weight_min": -67.43584442138672,
      "epoch": 1.07,
      "learning_rate": 0.0001445772727272727,
      "loss": 3.0806,
      "step": 18350
    },
    {
      "activations/layer0_attention_weight_max": 14.228586196899414,
      "activations/layer0_attention_weight_min": -11.593673706054688,
      "activations/layer10_attention_weight_max": 55.99277877807617,
      "activations/layer10_attention_weight_min": -46.1228141784668,
      "activations/layer11_attention_weight_max": 43.95904541015625,
      "activations/layer11_attention_weight_min": -38.463226318359375,
      "activations/layer12_attention_weight_max": 51.867591857910156,
      "activations/layer12_attention_weight_min": -33.4874382019043,
      "activations/layer13_attention_weight_max": 63.48550796508789,
      "activations/layer13_attention_weight_min": -38.97303771972656,
      "activations/layer14_attention_weight_max": 96.15924072265625,
      "activations/layer14_attention_weight_min": -58.72663879394531,
      "activations/layer15_attention_weight_max": 56.88783645629883,
      "activations/layer15_attention_weight_min": -41.03909683227539,
      "activations/layer16_attention_weight_max": 53.0181770324707,
      "activations/layer16_attention_weight_min": -37.92446517944336,
      "activations/layer17_attention_weight_max": 41.54555130004883,
      "activations/layer17_attention_weight_min": -24.57936668395996,
      "activations/layer18_attention_weight_max": 26.721445083618164,
      "activations/layer18_attention_weight_min": -16.283323287963867,
      "activations/layer19_attention_weight_max": 52.07441329956055,
      "activations/layer19_attention_weight_min": -32.57304382324219,
      "activations/layer1_attention_weight_max": 15.444382667541504,
      "activations/layer1_attention_weight_min": -12.897475242614746,
      "activations/layer20_attention_weight_max": 45.93102264404297,
      "activations/layer20_attention_weight_min": -24.482221603393555,
      "activations/layer21_attention_weight_max": 23.807132720947266,
      "activations/layer21_attention_weight_min": -14.4773588180542,
      "activations/layer22_attention_weight_max": 36.91191482543945,
      "activations/layer22_attention_weight_min": -22.016538619995117,
      "activations/layer23_attention_weight_max": 23.502063751220703,
      "activations/layer23_attention_weight_min": -13.711844444274902,
      "activations/layer2_attention_weight_max": 26.86080551147461,
      "activations/layer2_attention_weight_min": -26.046649932861328,
      "activations/layer3_attention_weight_max": 62.04340744018555,
      "activations/layer3_attention_weight_min": -63.681644439697266,
      "activations/layer4_attention_weight_max": 112.44837951660156,
      "activations/layer4_attention_weight_min": -107.62471771240234,
      "activations/layer5_attention_weight_max": 123.033935546875,
      "activations/layer5_attention_weight_min": -110.27894592285156,
      "activations/layer6_attention_weight_max": 70.43417358398438,
      "activations/layer6_attention_weight_min": -58.99534606933594,
      "activations/layer7_attention_weight_max": 65.51402282714844,
      "activations/layer7_attention_weight_min": -57.6063117980957,
      "activations/layer8_attention_weight_max": 58.74370193481445,
      "activations/layer8_attention_weight_min": -50.54121398925781,
      "activations/layer9_attention_weight_max": 69.1423568725586,
      "activations/layer9_attention_weight_min": -54.93156814575195,
      "epoch": 1.07,
      "learning_rate": 0.00014455833333333333,
      "loss": 3.0969,
      "step": 18400
    },
    {
      "activations/layer0_attention_weight_max": 15.994149208068848,
      "activations/layer0_attention_weight_min": -12.003744125366211,
      "activations/layer10_attention_weight_max": 59.43033218383789,
      "activations/layer10_attention_weight_min": -46.65054702758789,
      "activations/layer11_attention_weight_max": 45.44635772705078,
      "activations/layer11_attention_weight_min": -35.62080764770508,
      "activations/layer12_attention_weight_max": 40.47053909301758,
      "activations/layer12_attention_weight_min": -27.402877807617188,
      "activations/layer13_attention_weight_max": 51.61211395263672,
      "activations/layer13_attention_weight_min": -40.826385498046875,
      "activations/layer14_attention_weight_max": 86.6434555053711,
      "activations/layer14_attention_weight_min": -65.21990966796875,
      "activations/layer15_attention_weight_max": 61.45804977416992,
      "activations/layer15_attention_weight_min": -43.9034423828125,
      "activations/layer16_attention_weight_max": 51.01372528076172,
      "activations/layer16_attention_weight_min": -40.21676254272461,
      "activations/layer17_attention_weight_max": 36.35950469970703,
      "activations/layer17_attention_weight_min": -25.706687927246094,
      "activations/layer18_attention_weight_max": 21.38312339782715,
      "activations/layer18_attention_weight_min": -14.96523380279541,
      "activations/layer19_attention_weight_max": 43.71342468261719,
      "activations/layer19_attention_weight_min": -32.653804779052734,
      "activations/layer1_attention_weight_max": 15.680046081542969,
      "activations/layer1_attention_weight_min": -11.279255867004395,
      "activations/layer20_attention_weight_max": 39.411155700683594,
      "activations/layer20_attention_weight_min": -23.537046432495117,
      "activations/layer21_attention_weight_max": 24.79447364807129,
      "activations/layer21_attention_weight_min": -17.099624633789062,
      "activations/layer22_attention_weight_max": 35.70191955566406,
      "activations/layer22_attention_weight_min": -22.344928741455078,
      "activations/layer23_attention_weight_max": 22.392597198486328,
      "activations/layer23_attention_weight_min": -15.577699661254883,
      "activations/layer2_attention_weight_max": 26.498294830322266,
      "activations/layer2_attention_weight_min": -26.15650749206543,
      "activations/layer3_attention_weight_max": 56.17264175415039,
      "activations/layer3_attention_weight_min": -58.0055046081543,
      "activations/layer4_attention_weight_max": 105.88789367675781,
      "activations/layer4_attention_weight_min": -102.7622299194336,
      "activations/layer5_attention_weight_max": 117.21902465820312,
      "activations/layer5_attention_weight_min": -106.142822265625,
      "activations/layer6_attention_weight_max": 62.255165100097656,
      "activations/layer6_attention_weight_min": -56.28477096557617,
      "activations/layer7_attention_weight_max": 60.79963684082031,
      "activations/layer7_attention_weight_min": -57.68948745727539,
      "activations/layer8_attention_weight_max": 58.47796630859375,
      "activations/layer8_attention_weight_min": -49.448917388916016,
      "activations/layer9_attention_weight_max": 67.74800872802734,
      "activations/layer9_attention_weight_min": -59.05147933959961,
      "epoch": 1.07,
      "learning_rate": 0.00014453939393939392,
      "loss": 3.082,
      "step": 18450
    },
    {
      "activations/layer0_attention_weight_max": 15.810995101928711,
      "activations/layer0_attention_weight_min": -12.29663372039795,
      "activations/layer10_attention_weight_max": 69.94403076171875,
      "activations/layer10_attention_weight_min": -53.25791931152344,
      "activations/layer11_attention_weight_max": 51.819026947021484,
      "activations/layer11_attention_weight_min": -37.97771453857422,
      "activations/layer12_attention_weight_max": 38.09613037109375,
      "activations/layer12_attention_weight_min": -27.67148780822754,
      "activations/layer13_attention_weight_max": 57.07027053833008,
      "activations/layer13_attention_weight_min": -37.58395004272461,
      "activations/layer14_attention_weight_max": 77.13878631591797,
      "activations/layer14_attention_weight_min": -54.46730422973633,
      "activations/layer15_attention_weight_max": 54.9342155456543,
      "activations/layer15_attention_weight_min": -42.66978073120117,
      "activations/layer16_attention_weight_max": 53.02920913696289,
      "activations/layer16_attention_weight_min": -41.7130126953125,
      "activations/layer17_attention_weight_max": 39.8652229309082,
      "activations/layer17_attention_weight_min": -28.881982803344727,
      "activations/layer18_attention_weight_max": 22.23856544494629,
      "activations/layer18_attention_weight_min": -15.482034683227539,
      "activations/layer19_attention_weight_max": 57.882354736328125,
      "activations/layer19_attention_weight_min": -32.26959228515625,
      "activations/layer1_attention_weight_max": 16.143190383911133,
      "activations/layer1_attention_weight_min": -12.652337074279785,
      "activations/layer20_attention_weight_max": 43.865623474121094,
      "activations/layer20_attention_weight_min": -24.870098114013672,
      "activations/layer21_attention_weight_max": 23.649473190307617,
      "activations/layer21_attention_weight_min": -15.046156883239746,
      "activations/layer22_attention_weight_max": 48.129661560058594,
      "activations/layer22_attention_weight_min": -25.742328643798828,
      "activations/layer23_attention_weight_max": 26.324520111083984,
      "activations/layer23_attention_weight_min": -16.704376220703125,
      "activations/layer2_attention_weight_max": 28.30776596069336,
      "activations/layer2_attention_weight_min": -28.588605880737305,
      "activations/layer3_attention_weight_max": 60.4400749206543,
      "activations/layer3_attention_weight_min": -63.965667724609375,
      "activations/layer4_attention_weight_max": 110.0870132446289,
      "activations/layer4_attention_weight_min": -106.9129409790039,
      "activations/layer5_attention_weight_max": 125.03469848632812,
      "activations/layer5_attention_weight_min": -108.80083465576172,
      "activations/layer6_attention_weight_max": 66.96963500976562,
      "activations/layer6_attention_weight_min": -58.791893005371094,
      "activations/layer7_attention_weight_max": 62.21467208862305,
      "activations/layer7_attention_weight_min": -61.208457946777344,
      "activations/layer8_attention_weight_max": 60.35763168334961,
      "activations/layer8_attention_weight_min": -50.205841064453125,
      "activations/layer9_attention_weight_max": 71.81949615478516,
      "activations/layer9_attention_weight_min": -64.10273742675781,
      "epoch": 1.07,
      "learning_rate": 0.00014452045454545453,
      "loss": 3.0775,
      "step": 18500
    },
    {
      "activations/layer0_attention_weight_max": 15.007905006408691,
      "activations/layer0_attention_weight_min": -13.217623710632324,
      "activations/layer10_attention_weight_max": 54.95543670654297,
      "activations/layer10_attention_weight_min": -49.23502731323242,
      "activations/layer11_attention_weight_max": 50.19022750854492,
      "activations/layer11_attention_weight_min": -40.95157241821289,
      "activations/layer12_attention_weight_max": 40.30369186401367,
      "activations/layer12_attention_weight_min": -30.064477920532227,
      "activations/layer13_attention_weight_max": 45.95295715332031,
      "activations/layer13_attention_weight_min": -39.10783004760742,
      "activations/layer14_attention_weight_max": 74.84539794921875,
      "activations/layer14_attention_weight_min": -59.51069641113281,
      "activations/layer15_attention_weight_max": 60.844573974609375,
      "activations/layer15_attention_weight_min": -42.63483428955078,
      "activations/layer16_attention_weight_max": 55.33671951293945,
      "activations/layer16_attention_weight_min": -40.38219451904297,
      "activations/layer17_attention_weight_max": 38.582000732421875,
      "activations/layer17_attention_weight_min": -27.841093063354492,
      "activations/layer18_attention_weight_max": 23.973819732666016,
      "activations/layer18_attention_weight_min": -16.448318481445312,
      "activations/layer19_attention_weight_max": 60.68503952026367,
      "activations/layer19_attention_weight_min": -32.845558166503906,
      "activations/layer1_attention_weight_max": 16.543594360351562,
      "activations/layer1_attention_weight_min": -12.554868698120117,
      "activations/layer20_attention_weight_max": 44.82391357421875,
      "activations/layer20_attention_weight_min": -26.967472076416016,
      "activations/layer21_attention_weight_max": 21.155738830566406,
      "activations/layer21_attention_weight_min": -14.520655632019043,
      "activations/layer22_attention_weight_max": 39.5926399230957,
      "activations/layer22_attention_weight_min": -23.448301315307617,
      "activations/layer23_attention_weight_max": 23.545988082885742,
      "activations/layer23_attention_weight_min": -15.015303611755371,
      "activations/layer2_attention_weight_max": 27.12306022644043,
      "activations/layer2_attention_weight_min": -27.11302947998047,
      "activations/layer3_attention_weight_max": 57.01997756958008,
      "activations/layer3_attention_weight_min": -58.38814163208008,
      "activations/layer4_attention_weight_max": 107.1572265625,
      "activations/layer4_attention_weight_min": -104.86720275878906,
      "activations/layer5_attention_weight_max": 117.12260437011719,
      "activations/layer5_attention_weight_min": -105.62606048583984,
      "activations/layer6_attention_weight_max": 63.13327407836914,
      "activations/layer6_attention_weight_min": -57.49760055541992,
      "activations/layer7_attention_weight_max": 59.95719909667969,
      "activations/layer7_attention_weight_min": -60.30331802368164,
      "activations/layer8_attention_weight_max": 57.38987350463867,
      "activations/layer8_attention_weight_min": -51.71363830566406,
      "activations/layer9_attention_weight_max": 80.19451141357422,
      "activations/layer9_attention_weight_min": -63.864288330078125,
      "epoch": 1.08,
      "learning_rate": 0.00014450151515151515,
      "loss": 3.0624,
      "step": 18550
    },
    {
      "activations/layer0_attention_weight_max": 16.405832290649414,
      "activations/layer0_attention_weight_min": -11.380636215209961,
      "activations/layer10_attention_weight_max": 55.392547607421875,
      "activations/layer10_attention_weight_min": -48.54130554199219,
      "activations/layer11_attention_weight_max": 41.787803649902344,
      "activations/layer11_attention_weight_min": -39.06262969970703,
      "activations/layer12_attention_weight_max": 35.63096237182617,
      "activations/layer12_attention_weight_min": -27.50475311279297,
      "activations/layer13_attention_weight_max": 51.48627853393555,
      "activations/layer13_attention_weight_min": -35.66730499267578,
      "activations/layer14_attention_weight_max": 75.48692321777344,
      "activations/layer14_attention_weight_min": -56.88999557495117,
      "activations/layer15_attention_weight_max": 56.482627868652344,
      "activations/layer15_attention_weight_min": -38.5694694519043,
      "activations/layer16_attention_weight_max": 59.32130813598633,
      "activations/layer16_attention_weight_min": -40.28363037109375,
      "activations/layer17_attention_weight_max": 38.439239501953125,
      "activations/layer17_attention_weight_min": -24.330978393554688,
      "activations/layer18_attention_weight_max": 22.333703994750977,
      "activations/layer18_attention_weight_min": -14.819314956665039,
      "activations/layer19_attention_weight_max": 55.97445297241211,
      "activations/layer19_attention_weight_min": -30.307172775268555,
      "activations/layer1_attention_weight_max": 16.49355125427246,
      "activations/layer1_attention_weight_min": -12.51692008972168,
      "activations/layer20_attention_weight_max": 42.609493255615234,
      "activations/layer20_attention_weight_min": -23.607654571533203,
      "activations/layer21_attention_weight_max": 20.50310707092285,
      "activations/layer21_attention_weight_min": -14.733779907226562,
      "activations/layer22_attention_weight_max": 38.07365036010742,
      "activations/layer22_attention_weight_min": -23.683530807495117,
      "activations/layer23_attention_weight_max": 25.118459701538086,
      "activations/layer23_attention_weight_min": -13.950282096862793,
      "activations/layer2_attention_weight_max": 27.312572479248047,
      "activations/layer2_attention_weight_min": -26.240718841552734,
      "activations/layer3_attention_weight_max": 58.867244720458984,
      "activations/layer3_attention_weight_min": -59.453041076660156,
      "activations/layer4_attention_weight_max": 107.54164123535156,
      "activations/layer4_attention_weight_min": -100.94371795654297,
      "activations/layer5_attention_weight_max": 117.73454284667969,
      "activations/layer5_attention_weight_min": -108.00619506835938,
      "activations/layer6_attention_weight_max": 60.91398239135742,
      "activations/layer6_attention_weight_min": -55.66461944580078,
      "activations/layer7_attention_weight_max": 60.67573928833008,
      "activations/layer7_attention_weight_min": -58.91789245605469,
      "activations/layer8_attention_weight_max": 59.7735481262207,
      "activations/layer8_attention_weight_min": -51.353416442871094,
      "activations/layer9_attention_weight_max": 73.63953399658203,
      "activations/layer9_attention_weight_min": -62.0689697265625,
      "epoch": 1.08,
      "learning_rate": 0.00014448257575757573,
      "loss": 3.0804,
      "step": 18600
    },
    {
      "activations/layer0_attention_weight_max": 15.7681245803833,
      "activations/layer0_attention_weight_min": -11.158019065856934,
      "activations/layer10_attention_weight_max": 55.719017028808594,
      "activations/layer10_attention_weight_min": -49.019771575927734,
      "activations/layer11_attention_weight_max": 52.17420959472656,
      "activations/layer11_attention_weight_min": -40.43592834472656,
      "activations/layer12_attention_weight_max": 47.19165802001953,
      "activations/layer12_attention_weight_min": -37.749855041503906,
      "activations/layer13_attention_weight_max": 52.33671569824219,
      "activations/layer13_attention_weight_min": -38.31047439575195,
      "activations/layer14_attention_weight_max": 78.76717376708984,
      "activations/layer14_attention_weight_min": -60.18903732299805,
      "activations/layer15_attention_weight_max": 58.44486999511719,
      "activations/layer15_attention_weight_min": -41.34150314331055,
      "activations/layer16_attention_weight_max": 55.68140411376953,
      "activations/layer16_attention_weight_min": -38.058780670166016,
      "activations/layer17_attention_weight_max": 34.425933837890625,
      "activations/layer17_attention_weight_min": -27.744237899780273,
      "activations/layer18_attention_weight_max": 25.422216415405273,
      "activations/layer18_attention_weight_min": -13.806520462036133,
      "activations/layer19_attention_weight_max": 48.43531036376953,
      "activations/layer19_attention_weight_min": -35.50388717651367,
      "activations/layer1_attention_weight_max": 16.386816024780273,
      "activations/layer1_attention_weight_min": -11.796805381774902,
      "activations/layer20_attention_weight_max": 35.31199264526367,
      "activations/layer20_attention_weight_min": -26.519725799560547,
      "activations/layer21_attention_weight_max": 21.2999210357666,
      "activations/layer21_attention_weight_min": -13.86308765411377,
      "activations/layer22_attention_weight_max": 33.115684509277344,
      "activations/layer22_attention_weight_min": -22.019535064697266,
      "activations/layer23_attention_weight_max": 21.846172332763672,
      "activations/layer23_attention_weight_min": -13.738661766052246,
      "activations/layer2_attention_weight_max": 26.386289596557617,
      "activations/layer2_attention_weight_min": -26.879383087158203,
      "activations/layer3_attention_weight_max": 60.83095932006836,
      "activations/layer3_attention_weight_min": -61.98225021362305,
      "activations/layer4_attention_weight_max": 113.62981414794922,
      "activations/layer4_attention_weight_min": -104.59490966796875,
      "activations/layer5_attention_weight_max": 129.050048828125,
      "activations/layer5_attention_weight_min": -111.25926208496094,
      "activations/layer6_attention_weight_max": 70.6357650756836,
      "activations/layer6_attention_weight_min": -56.78112030029297,
      "activations/layer7_attention_weight_max": 67.72327423095703,
      "activations/layer7_attention_weight_min": -65.59393310546875,
      "activations/layer8_attention_weight_max": 65.15157318115234,
      "activations/layer8_attention_weight_min": -57.76603317260742,
      "activations/layer9_attention_weight_max": 82.53475189208984,
      "activations/layer9_attention_weight_min": -73.3163070678711,
      "epoch": 1.08,
      "learning_rate": 0.00014446363636363635,
      "loss": 3.0847,
      "step": 18650
    },
    {
      "activations/layer0_attention_weight_max": 14.779345512390137,
      "activations/layer0_attention_weight_min": -10.703203201293945,
      "activations/layer10_attention_weight_max": 55.86506271362305,
      "activations/layer10_attention_weight_min": -49.81861114501953,
      "activations/layer11_attention_weight_max": 44.79250717163086,
      "activations/layer11_attention_weight_min": -39.572723388671875,
      "activations/layer12_attention_weight_max": 53.39836120605469,
      "activations/layer12_attention_weight_min": -39.37446975708008,
      "activations/layer13_attention_weight_max": 68.79813385009766,
      "activations/layer13_attention_weight_min": -52.432743072509766,
      "activations/layer14_attention_weight_max": 76.52928161621094,
      "activations/layer14_attention_weight_min": -55.927730560302734,
      "activations/layer15_attention_weight_max": 52.57719039916992,
      "activations/layer15_attention_weight_min": -35.75748062133789,
      "activations/layer16_attention_weight_max": 49.71648406982422,
      "activations/layer16_attention_weight_min": -38.773616790771484,
      "activations/layer17_attention_weight_max": 38.59013366699219,
      "activations/layer17_attention_weight_min": -24.97092628479004,
      "activations/layer18_attention_weight_max": 26.14274024963379,
      "activations/layer18_attention_weight_min": -14.335500717163086,
      "activations/layer19_attention_weight_max": 50.57765579223633,
      "activations/layer19_attention_weight_min": -29.519071578979492,
      "activations/layer1_attention_weight_max": 16.013687133789062,
      "activations/layer1_attention_weight_min": -12.518119812011719,
      "activations/layer20_attention_weight_max": 48.48564147949219,
      "activations/layer20_attention_weight_min": -23.634082794189453,
      "activations/layer21_attention_weight_max": 23.354990005493164,
      "activations/layer21_attention_weight_min": -13.002494812011719,
      "activations/layer22_attention_weight_max": 38.093868255615234,
      "activations/layer22_attention_weight_min": -23.29038429260254,
      "activations/layer23_attention_weight_max": 24.095481872558594,
      "activations/layer23_attention_weight_min": -16.303112030029297,
      "activations/layer2_attention_weight_max": 28.461246490478516,
      "activations/layer2_attention_weight_min": -30.205307006835938,
      "activations/layer3_attention_weight_max": 60.59614181518555,
      "activations/layer3_attention_weight_min": -61.00981140136719,
      "activations/layer4_attention_weight_max": 110.11980438232422,
      "activations/layer4_attention_weight_min": -104.06947326660156,
      "activations/layer5_attention_weight_max": 121.49666595458984,
      "activations/layer5_attention_weight_min": -106.40093994140625,
      "activations/layer6_attention_weight_max": 64.16996002197266,
      "activations/layer6_attention_weight_min": -58.24491882324219,
      "activations/layer7_attention_weight_max": 66.61248779296875,
      "activations/layer7_attention_weight_min": -58.00493240356445,
      "activations/layer8_attention_weight_max": 58.86636734008789,
      "activations/layer8_attention_weight_min": -54.82151794433594,
      "activations/layer9_attention_weight_max": 81.69210052490234,
      "activations/layer9_attention_weight_min": -71.6598129272461,
      "epoch": 1.09,
      "learning_rate": 0.00014444469696969694,
      "loss": 3.0781,
      "step": 18700
    },
    {
      "activations/layer0_attention_weight_max": 15.714591026306152,
      "activations/layer0_attention_weight_min": -12.355006217956543,
      "activations/layer10_attention_weight_max": 59.06293487548828,
      "activations/layer10_attention_weight_min": -48.22722244262695,
      "activations/layer11_attention_weight_max": 58.95096969604492,
      "activations/layer11_attention_weight_min": -38.643096923828125,
      "activations/layer12_attention_weight_max": 51.6012077331543,
      "activations/layer12_attention_weight_min": -30.93765640258789,
      "activations/layer13_attention_weight_max": 72.4769287109375,
      "activations/layer13_attention_weight_min": -41.129371643066406,
      "activations/layer14_attention_weight_max": 97.2247314453125,
      "activations/layer14_attention_weight_min": -58.8127555847168,
      "activations/layer15_attention_weight_max": 67.00435638427734,
      "activations/layer15_attention_weight_min": -38.1710090637207,
      "activations/layer16_attention_weight_max": 48.92469787597656,
      "activations/layer16_attention_weight_min": -44.97291946411133,
      "activations/layer17_attention_weight_max": 38.5316162109375,
      "activations/layer17_attention_weight_min": -26.82175636291504,
      "activations/layer18_attention_weight_max": 24.60535430908203,
      "activations/layer18_attention_weight_min": -14.993779182434082,
      "activations/layer19_attention_weight_max": 43.67680740356445,
      "activations/layer19_attention_weight_min": -30.818161010742188,
      "activations/layer1_attention_weight_max": 16.453800201416016,
      "activations/layer1_attention_weight_min": -12.013556480407715,
      "activations/layer20_attention_weight_max": 37.96168899536133,
      "activations/layer20_attention_weight_min": -24.960371017456055,
      "activations/layer21_attention_weight_max": 20.86429786682129,
      "activations/layer21_attention_weight_min": -14.763401985168457,
      "activations/layer22_attention_weight_max": 37.48520278930664,
      "activations/layer22_attention_weight_min": -19.889060974121094,
      "activations/layer23_attention_weight_max": 23.562942504882812,
      "activations/layer23_attention_weight_min": -14.207357406616211,
      "activations/layer2_attention_weight_max": 27.205930709838867,
      "activations/layer2_attention_weight_min": -26.988155364990234,
      "activations/layer3_attention_weight_max": 64.19933319091797,
      "activations/layer3_attention_weight_min": -65.47453308105469,
      "activations/layer4_attention_weight_max": 118.1722640991211,
      "activations/layer4_attention_weight_min": -110.54219055175781,
      "activations/layer5_attention_weight_max": 135.61807250976562,
      "activations/layer5_attention_weight_min": -118.8575668334961,
      "activations/layer6_attention_weight_max": 72.00896453857422,
      "activations/layer6_attention_weight_min": -61.11176300048828,
      "activations/layer7_attention_weight_max": 70.71370697021484,
      "activations/layer7_attention_weight_min": -60.57954406738281,
      "activations/layer8_attention_weight_max": 70.86670684814453,
      "activations/layer8_attention_weight_min": -59.09648895263672,
      "activations/layer9_attention_weight_max": 90.02052307128906,
      "activations/layer9_attention_weight_min": -75.3072280883789,
      "epoch": 1.09,
      "learning_rate": 0.00014442575757575755,
      "loss": 3.086,
      "step": 18750
    },
    {
      "activations/layer0_attention_weight_max": 15.546350479125977,
      "activations/layer0_attention_weight_min": -12.306084632873535,
      "activations/layer10_attention_weight_max": 52.439056396484375,
      "activations/layer10_attention_weight_min": -44.862579345703125,
      "activations/layer11_attention_weight_max": 52.67677307128906,
      "activations/layer11_attention_weight_min": -39.209686279296875,
      "activations/layer12_attention_weight_max": 38.69990921020508,
      "activations/layer12_attention_weight_min": -30.484636306762695,
      "activations/layer13_attention_weight_max": 42.5479850769043,
      "activations/layer13_attention_weight_min": -35.02647399902344,
      "activations/layer14_attention_weight_max": 69.28565979003906,
      "activations/layer14_attention_weight_min": -59.63454818725586,
      "activations/layer15_attention_weight_max": 47.88569259643555,
      "activations/layer15_attention_weight_min": -38.67520523071289,
      "activations/layer16_attention_weight_max": 50.83108139038086,
      "activations/layer16_attention_weight_min": -39.172767639160156,
      "activations/layer17_attention_weight_max": 39.15940475463867,
      "activations/layer17_attention_weight_min": -28.26089096069336,
      "activations/layer18_attention_weight_max": 22.484464645385742,
      "activations/layer18_attention_weight_min": -18.655635833740234,
      "activations/layer19_attention_weight_max": 51.63275909423828,
      "activations/layer19_attention_weight_min": -32.59684371948242,
      "activations/layer1_attention_weight_max": 15.630014419555664,
      "activations/layer1_attention_weight_min": -11.66482162475586,
      "activations/layer20_attention_weight_max": 39.26964569091797,
      "activations/layer20_attention_weight_min": -24.56832504272461,
      "activations/layer21_attention_weight_max": 25.1665096282959,
      "activations/layer21_attention_weight_min": -14.294952392578125,
      "activations/layer22_attention_weight_max": 41.579681396484375,
      "activations/layer22_attention_weight_min": -22.939016342163086,
      "activations/layer23_attention_weight_max": 25.42800521850586,
      "activations/layer23_attention_weight_min": -17.244239807128906,
      "activations/layer2_attention_weight_max": 27.39508819580078,
      "activations/layer2_attention_weight_min": -26.600509643554688,
      "activations/layer3_attention_weight_max": 59.1025390625,
      "activations/layer3_attention_weight_min": -60.91127395629883,
      "activations/layer4_attention_weight_max": 109.5636215209961,
      "activations/layer4_attention_weight_min": -105.70307922363281,
      "activations/layer5_attention_weight_max": 121.55331420898438,
      "activations/layer5_attention_weight_min": -109.98278045654297,
      "activations/layer6_attention_weight_max": 61.77587890625,
      "activations/layer6_attention_weight_min": -57.698699951171875,
      "activations/layer7_attention_weight_max": 57.512535095214844,
      "activations/layer7_attention_weight_min": -69.61174774169922,
      "activations/layer8_attention_weight_max": 54.532188415527344,
      "activations/layer8_attention_weight_min": -53.1429557800293,
      "activations/layer9_attention_weight_max": 69.3803939819336,
      "activations/layer9_attention_weight_min": -62.343353271484375,
      "epoch": 1.09,
      "learning_rate": 0.00014440681818181817,
      "loss": 3.0757,
      "step": 18800
    },
    {
      "activations/layer0_attention_weight_max": 16.548477172851562,
      "activations/layer0_attention_weight_min": -13.059144973754883,
      "activations/layer10_attention_weight_max": 52.94987487792969,
      "activations/layer10_attention_weight_min": -50.467342376708984,
      "activations/layer11_attention_weight_max": 42.5971794128418,
      "activations/layer11_attention_weight_min": -36.90316390991211,
      "activations/layer12_attention_weight_max": 34.16468048095703,
      "activations/layer12_attention_weight_min": -27.35862159729004,
      "activations/layer13_attention_weight_max": 48.98668670654297,
      "activations/layer13_attention_weight_min": -37.07380294799805,
      "activations/layer14_attention_weight_max": 71.31564331054688,
      "activations/layer14_attention_weight_min": -51.82868194580078,
      "activations/layer15_attention_weight_max": 53.26348876953125,
      "activations/layer15_attention_weight_min": -37.405704498291016,
      "activations/layer16_attention_weight_max": 57.33665084838867,
      "activations/layer16_attention_weight_min": -37.3985710144043,
      "activations/layer17_attention_weight_max": 37.21452713012695,
      "activations/layer17_attention_weight_min": -26.545618057250977,
      "activations/layer18_attention_weight_max": 21.667932510375977,
      "activations/layer18_attention_weight_min": -13.175092697143555,
      "activations/layer19_attention_weight_max": 53.892372131347656,
      "activations/layer19_attention_weight_min": -30.67072105407715,
      "activations/layer1_attention_weight_max": 15.391551971435547,
      "activations/layer1_attention_weight_min": -11.396260261535645,
      "activations/layer20_attention_weight_max": 44.15281677246094,
      "activations/layer20_attention_weight_min": -25.057628631591797,
      "activations/layer21_attention_weight_max": 22.866281509399414,
      "activations/layer21_attention_weight_min": -14.191963195800781,
      "activations/layer22_attention_weight_max": 37.412410736083984,
      "activations/layer22_attention_weight_min": -23.59357261657715,
      "activations/layer23_attention_weight_max": 25.917327880859375,
      "activations/layer23_attention_weight_min": -12.213984489440918,
      "activations/layer2_attention_weight_max": 28.62067985534668,
      "activations/layer2_attention_weight_min": -27.662336349487305,
      "activations/layer3_attention_weight_max": 60.66389465332031,
      "activations/layer3_attention_weight_min": -63.388614654541016,
      "activations/layer4_attention_weight_max": 111.83489227294922,
      "activations/layer4_attention_weight_min": -106.90067291259766,
      "activations/layer5_attention_weight_max": 120.7388916015625,
      "activations/layer5_attention_weight_min": -107.54487609863281,
      "activations/layer6_attention_weight_max": 62.7944221496582,
      "activations/layer6_attention_weight_min": -55.68980026245117,
      "activations/layer7_attention_weight_max": 58.972816467285156,
      "activations/layer7_attention_weight_min": -59.94732666015625,
      "activations/layer8_attention_weight_max": 59.50299835205078,
      "activations/layer8_attention_weight_min": -53.74019241333008,
      "activations/layer9_attention_weight_max": 69.52239227294922,
      "activations/layer9_attention_weight_min": -63.12286376953125,
      "epoch": 1.1,
      "learning_rate": 0.00014438787878787878,
      "loss": 3.0684,
      "step": 18850
    },
    {
      "activations/layer0_attention_weight_max": 15.348827362060547,
      "activations/layer0_attention_weight_min": -11.62551498413086,
      "activations/layer10_attention_weight_max": 51.3956413269043,
      "activations/layer10_attention_weight_min": -49.19743728637695,
      "activations/layer11_attention_weight_max": 44.156761169433594,
      "activations/layer11_attention_weight_min": -36.33772277832031,
      "activations/layer12_attention_weight_max": 30.312416076660156,
      "activations/layer12_attention_weight_min": -25.975343704223633,
      "activations/layer13_attention_weight_max": 46.629459381103516,
      "activations/layer13_attention_weight_min": -39.10713577270508,
      "activations/layer14_attention_weight_max": 72.78946685791016,
      "activations/layer14_attention_weight_min": -54.76662826538086,
      "activations/layer15_attention_weight_max": 58.550228118896484,
      "activations/layer15_attention_weight_min": -40.08010482788086,
      "activations/layer16_attention_weight_max": 49.40776062011719,
      "activations/layer16_attention_weight_min": -36.809505462646484,
      "activations/layer17_attention_weight_max": 40.04292678833008,
      "activations/layer17_attention_weight_min": -27.053712844848633,
      "activations/layer18_attention_weight_max": 21.998201370239258,
      "activations/layer18_attention_weight_min": -15.932316780090332,
      "activations/layer19_attention_weight_max": 45.712650299072266,
      "activations/layer19_attention_weight_min": -30.446924209594727,
      "activations/layer1_attention_weight_max": 16.108430862426758,
      "activations/layer1_attention_weight_min": -12.883843421936035,
      "activations/layer20_attention_weight_max": 44.94122314453125,
      "activations/layer20_attention_weight_min": -24.19985008239746,
      "activations/layer21_attention_weight_max": 24.326080322265625,
      "activations/layer21_attention_weight_min": -15.142372131347656,
      "activations/layer22_attention_weight_max": 36.30559539794922,
      "activations/layer22_attention_weight_min": -22.921337127685547,
      "activations/layer23_attention_weight_max": 24.248716354370117,
      "activations/layer23_attention_weight_min": -14.273307800292969,
      "activations/layer2_attention_weight_max": 27.66720199584961,
      "activations/layer2_attention_weight_min": -26.071767807006836,
      "activations/layer3_attention_weight_max": 61.58853530883789,
      "activations/layer3_attention_weight_min": -63.544830322265625,
      "activations/layer4_attention_weight_max": 111.39009857177734,
      "activations/layer4_attention_weight_min": -106.18627166748047,
      "activations/layer5_attention_weight_max": 115.52217864990234,
      "activations/layer5_attention_weight_min": -108.00434875488281,
      "activations/layer6_attention_weight_max": 67.02670288085938,
      "activations/layer6_attention_weight_min": -54.23837661743164,
      "activations/layer7_attention_weight_max": 58.71087646484375,
      "activations/layer7_attention_weight_min": -60.68552780151367,
      "activations/layer8_attention_weight_max": 52.614620208740234,
      "activations/layer8_attention_weight_min": -48.8338737487793,
      "activations/layer9_attention_weight_max": 74.227783203125,
      "activations/layer9_attention_weight_min": -62.549903869628906,
      "epoch": 1.1,
      "learning_rate": 0.00014436893939393937,
      "loss": 3.078,
      "step": 18900
    },
    {
      "activations/layer0_attention_weight_max": 15.428918838500977,
      "activations/layer0_attention_weight_min": -10.3977632522583,
      "activations/layer10_attention_weight_max": 54.36314392089844,
      "activations/layer10_attention_weight_min": -52.481990814208984,
      "activations/layer11_attention_weight_max": 37.782508850097656,
      "activations/layer11_attention_weight_min": -34.8719482421875,
      "activations/layer12_attention_weight_max": 38.849544525146484,
      "activations/layer12_attention_weight_min": -27.9658260345459,
      "activations/layer13_attention_weight_max": 49.08487319946289,
      "activations/layer13_attention_weight_min": -40.46488571166992,
      "activations/layer14_attention_weight_max": 69.55332946777344,
      "activations/layer14_attention_weight_min": -57.690284729003906,
      "activations/layer15_attention_weight_max": 48.69374465942383,
      "activations/layer15_attention_weight_min": -36.785221099853516,
      "activations/layer16_attention_weight_max": 45.27049255371094,
      "activations/layer16_attention_weight_min": -36.24934387207031,
      "activations/layer17_attention_weight_max": 38.1308479309082,
      "activations/layer17_attention_weight_min": -25.788135528564453,
      "activations/layer18_attention_weight_max": 21.29806900024414,
      "activations/layer18_attention_weight_min": -14.57497501373291,
      "activations/layer19_attention_weight_max": 48.938140869140625,
      "activations/layer19_attention_weight_min": -32.35140609741211,
      "activations/layer1_attention_weight_max": 16.438871383666992,
      "activations/layer1_attention_weight_min": -10.962434768676758,
      "activations/layer20_attention_weight_max": 40.71796417236328,
      "activations/layer20_attention_weight_min": -25.3083553314209,
      "activations/layer21_attention_weight_max": 26.181482315063477,
      "activations/layer21_attention_weight_min": -14.636916160583496,
      "activations/layer22_attention_weight_max": 42.1683464050293,
      "activations/layer22_attention_weight_min": -25.304609298706055,
      "activations/layer23_attention_weight_max": 24.369659423828125,
      "activations/layer23_attention_weight_min": -14.922908782958984,
      "activations/layer2_attention_weight_max": 26.288618087768555,
      "activations/layer2_attention_weight_min": -27.11981773376465,
      "activations/layer3_attention_weight_max": 59.40752410888672,
      "activations/layer3_attention_weight_min": -66.5788345336914,
      "activations/layer4_attention_weight_max": 106.6442642211914,
      "activations/layer4_attention_weight_min": -108.53181457519531,
      "activations/layer5_attention_weight_max": 115.10459899902344,
      "activations/layer5_attention_weight_min": -106.14883422851562,
      "activations/layer6_attention_weight_max": 59.083900451660156,
      "activations/layer6_attention_weight_min": -55.82390594482422,
      "activations/layer7_attention_weight_max": 58.4974479675293,
      "activations/layer7_attention_weight_min": -59.729061126708984,
      "activations/layer8_attention_weight_max": 51.514404296875,
      "activations/layer8_attention_weight_min": -48.30805206298828,
      "activations/layer9_attention_weight_max": 63.248268127441406,
      "activations/layer9_attention_weight_min": -56.940975189208984,
      "epoch": 1.1,
      "learning_rate": 0.00014434999999999999,
      "loss": 3.0762,
      "step": 18950
    },
    {
      "activations/layer0_attention_weight_max": 14.615882873535156,
      "activations/layer0_attention_weight_min": -10.619096755981445,
      "activations/layer10_attention_weight_max": 57.95130157470703,
      "activations/layer10_attention_weight_min": -50.97153091430664,
      "activations/layer11_attention_weight_max": 42.340065002441406,
      "activations/layer11_attention_weight_min": -39.48980712890625,
      "activations/layer12_attention_weight_max": 37.227088928222656,
      "activations/layer12_attention_weight_min": -29.034198760986328,
      "activations/layer13_attention_weight_max": 48.460453033447266,
      "activations/layer13_attention_weight_min": -37.32246017456055,
      "activations/layer14_attention_weight_max": 76.89053344726562,
      "activations/layer14_attention_weight_min": -60.006378173828125,
      "activations/layer15_attention_weight_max": 54.396785736083984,
      "activations/layer15_attention_weight_min": -38.168575286865234,
      "activations/layer16_attention_weight_max": 56.52534103393555,
      "activations/layer16_attention_weight_min": -39.79719161987305,
      "activations/layer17_attention_weight_max": 42.84187316894531,
      "activations/layer17_attention_weight_min": -25.37681770324707,
      "activations/layer18_attention_weight_max": 26.237533569335938,
      "activations/layer18_attention_weight_min": -15.3711519241333,
      "activations/layer19_attention_weight_max": 62.298126220703125,
      "activations/layer19_attention_weight_min": -33.50493621826172,
      "activations/layer1_attention_weight_max": 15.703615188598633,
      "activations/layer1_attention_weight_min": -12.382925987243652,
      "activations/layer20_attention_weight_max": 45.7201042175293,
      "activations/layer20_attention_weight_min": -25.43239974975586,
      "activations/layer21_attention_weight_max": 23.223636627197266,
      "activations/layer21_attention_weight_min": -13.761772155761719,
      "activations/layer22_attention_weight_max": 48.061279296875,
      "activations/layer22_attention_weight_min": -23.362884521484375,
      "activations/layer23_attention_weight_max": 25.619924545288086,
      "activations/layer23_attention_weight_min": -15.843851089477539,
      "activations/layer2_attention_weight_max": 27.5437068939209,
      "activations/layer2_attention_weight_min": -27.246654510498047,
      "activations/layer3_attention_weight_max": 60.70658493041992,
      "activations/layer3_attention_weight_min": -64.63751220703125,
      "activations/layer4_attention_weight_max": 111.46614074707031,
      "activations/layer4_attention_weight_min": -108.63216400146484,
      "activations/layer5_attention_weight_max": 120.86162567138672,
      "activations/layer5_attention_weight_min": -109.82058715820312,
      "activations/layer6_attention_weight_max": 65.28205871582031,
      "activations/layer6_attention_weight_min": -58.65653991699219,
      "activations/layer7_attention_weight_max": 62.003387451171875,
      "activations/layer7_attention_weight_min": -60.01187515258789,
      "activations/layer8_attention_weight_max": 56.097625732421875,
      "activations/layer8_attention_weight_min": -54.460365295410156,
      "activations/layer9_attention_weight_max": 78.05619049072266,
      "activations/layer9_attention_weight_min": -69.29927825927734,
      "epoch": 1.1,
      "learning_rate": 0.0001443310606060606,
      "loss": 3.0724,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_loss": 3.02734375,
      "eval_runtime": 8.5491,
      "eval_samples_per_second": 502.274,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_openwebtext_loss": 3.02734375,
      "eval_openwebtext_ppl": 20.64232851707703,
      "eval_openwebtext_runtime": 8.5491,
      "eval_openwebtext_samples_per_second": 502.274,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_wikitext_loss": 3.298828125,
      "eval_wikitext_ppl": 27.080884906424416,
      "eval_wikitext_runtime": 1.9862,
      "eval_wikitext_samples_per_second": 229.586,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_lambada_loss": 3.390625,
      "eval_lambada_ppl": 29.684499285876434,
      "eval_lambada_runtime": 9.5401,
      "eval_lambada_samples_per_second": 510.37,
      "step": 19000
    },
    {
      "activations/layer0_attention_weight_max": 15.44437313079834,
      "activations/layer0_attention_weight_min": -11.903376579284668,
      "activations/layer10_attention_weight_max": 52.88956832885742,
      "activations/layer10_attention_weight_min": -56.191986083984375,
      "activations/layer11_attention_weight_max": 40.80237579345703,
      "activations/layer11_attention_weight_min": -39.25247573852539,
      "activations/layer12_attention_weight_max": 33.3807258605957,
      "activations/layer12_attention_weight_min": -28.143774032592773,
      "activations/layer13_attention_weight_max": 44.65964126586914,
      "activations/layer13_attention_weight_min": -35.859371185302734,
      "activations/layer14_attention_weight_max": 64.12261199951172,
      "activations/layer14_attention_weight_min": -56.67910385131836,
      "activations/layer15_attention_weight_max": 52.97351837158203,
      "activations/layer15_attention_weight_min": -40.38016891479492,
      "activations/layer16_attention_weight_max": 53.14379119873047,
      "activations/layer16_attention_weight_min": -40.918663024902344,
      "activations/layer17_attention_weight_max": 37.61200714111328,
      "activations/layer17_attention_weight_min": -26.088281631469727,
      "activations/layer18_attention_weight_max": 26.031551361083984,
      "activations/layer18_attention_weight_min": -15.902871131896973,
      "activations/layer19_attention_weight_max": 60.01630401611328,
      "activations/layer19_attention_weight_min": -34.041290283203125,
      "activations/layer1_attention_weight_max": 16.441057205200195,
      "activations/layer1_attention_weight_min": -12.32837963104248,
      "activations/layer20_attention_weight_max": 43.6450080871582,
      "activations/layer20_attention_weight_min": -29.683244705200195,
      "activations/layer21_attention_weight_max": 24.654760360717773,
      "activations/layer21_attention_weight_min": -15.434102058410645,
      "activations/layer22_attention_weight_max": 46.22013473510742,
      "activations/layer22_attention_weight_min": -25.473766326904297,
      "activations/layer23_attention_weight_max": 26.24443817138672,
      "activations/layer23_attention_weight_min": -16.925752639770508,
      "activations/layer2_attention_weight_max": 27.256622314453125,
      "activations/layer2_attention_weight_min": -28.020061492919922,
      "activations/layer3_attention_weight_max": 59.525390625,
      "activations/layer3_attention_weight_min": -61.22703552246094,
      "activations/layer4_attention_weight_max": 112.18660736083984,
      "activations/layer4_attention_weight_min": -104.96353912353516,
      "activations/layer5_attention_weight_max": 119.8591537475586,
      "activations/layer5_attention_weight_min": -108.40324401855469,
      "activations/layer6_attention_weight_max": 62.419647216796875,
      "activations/layer6_attention_weight_min": -58.46750259399414,
      "activations/layer7_attention_weight_max": 62.87921905517578,
      "activations/layer7_attention_weight_min": -63.01095962524414,
      "activations/layer8_attention_weight_max": 55.30923080444336,
      "activations/layer8_attention_weight_min": -52.26677703857422,
      "activations/layer9_attention_weight_max": 74.98072814941406,
      "activations/layer9_attention_weight_min": -56.91901779174805,
      "epoch": 1.11,
      "learning_rate": 0.0001443121212121212,
      "loss": 3.0766,
      "step": 19050
    },
    {
      "activations/layer0_attention_weight_max": 14.050564765930176,
      "activations/layer0_attention_weight_min": -12.254411697387695,
      "activations/layer10_attention_weight_max": 66.70974731445312,
      "activations/layer10_attention_weight_min": -51.13957595825195,
      "activations/layer11_attention_weight_max": 52.07052230834961,
      "activations/layer11_attention_weight_min": -40.275733947753906,
      "activations/layer12_attention_weight_max": 31.991113662719727,
      "activations/layer12_attention_weight_min": -29.878694534301758,
      "activations/layer13_attention_weight_max": 42.06635284423828,
      "activations/layer13_attention_weight_min": -35.829551696777344,
      "activations/layer14_attention_weight_max": 72.51947784423828,
      "activations/layer14_attention_weight_min": -55.151912689208984,
      "activations/layer15_attention_weight_max": 53.53280258178711,
      "activations/layer15_attention_weight_min": -38.084896087646484,
      "activations/layer16_attention_weight_max": 52.4197998046875,
      "activations/layer16_attention_weight_min": -40.59308624267578,
      "activations/layer17_attention_weight_max": 36.263179779052734,
      "activations/layer17_attention_weight_min": -25.480318069458008,
      "activations/layer18_attention_weight_max": 22.33417510986328,
      "activations/layer18_attention_weight_min": -15.58920955657959,
      "activations/layer19_attention_weight_max": 52.64167022705078,
      "activations/layer19_attention_weight_min": -34.29757308959961,
      "activations/layer1_attention_weight_max": 16.3513126373291,
      "activations/layer1_attention_weight_min": -12.321587562561035,
      "activations/layer20_attention_weight_max": 42.430320739746094,
      "activations/layer20_attention_weight_min": -26.149694442749023,
      "activations/layer21_attention_weight_max": 21.113344192504883,
      "activations/layer21_attention_weight_min": -13.389845848083496,
      "activations/layer22_attention_weight_max": 38.24212646484375,
      "activations/layer22_attention_weight_min": -22.84908103942871,
      "activations/layer23_attention_weight_max": 22.801254272460938,
      "activations/layer23_attention_weight_min": -16.664400100708008,
      "activations/layer2_attention_weight_max": 28.284399032592773,
      "activations/layer2_attention_weight_min": -27.867061614990234,
      "activations/layer3_attention_weight_max": 61.15867614746094,
      "activations/layer3_attention_weight_min": -65.16941833496094,
      "activations/layer4_attention_weight_max": 108.49199676513672,
      "activations/layer4_attention_weight_min": -108.3763656616211,
      "activations/layer5_attention_weight_max": 119.93470764160156,
      "activations/layer5_attention_weight_min": -110.930908203125,
      "activations/layer6_attention_weight_max": 62.286746978759766,
      "activations/layer6_attention_weight_min": -57.90492248535156,
      "activations/layer7_attention_weight_max": 60.254669189453125,
      "activations/layer7_attention_weight_min": -58.380680084228516,
      "activations/layer8_attention_weight_max": 64.67234802246094,
      "activations/layer8_attention_weight_min": -53.94014358520508,
      "activations/layer9_attention_weight_max": 86.06000518798828,
      "activations/layer9_attention_weight_min": -70.0744857788086,
      "epoch": 1.11,
      "learning_rate": 0.0001442931818181818,
      "loss": 3.0717,
      "step": 19100
    },
    {
      "activations/layer0_attention_weight_max": 15.14012622833252,
      "activations/layer0_attention_weight_min": -11.106306076049805,
      "activations/layer10_attention_weight_max": 70.83601379394531,
      "activations/layer10_attention_weight_min": -53.91029739379883,
      "activations/layer11_attention_weight_max": 50.78797149658203,
      "activations/layer11_attention_weight_min": -39.67649841308594,
      "activations/layer12_attention_weight_max": 37.46599197387695,
      "activations/layer12_attention_weight_min": -27.196062088012695,
      "activations/layer13_attention_weight_max": 53.709266662597656,
      "activations/layer13_attention_weight_min": -36.589778900146484,
      "activations/layer14_attention_weight_max": 84.46173858642578,
      "activations/layer14_attention_weight_min": -55.636383056640625,
      "activations/layer15_attention_weight_max": 62.10585021972656,
      "activations/layer15_attention_weight_min": -38.620330810546875,
      "activations/layer16_attention_weight_max": 61.73068618774414,
      "activations/layer16_attention_weight_min": -41.55559158325195,
      "activations/layer17_attention_weight_max": 44.410072326660156,
      "activations/layer17_attention_weight_min": -27.068614959716797,
      "activations/layer18_attention_weight_max": 23.9700870513916,
      "activations/layer18_attention_weight_min": -16.210878372192383,
      "activations/layer19_attention_weight_max": 57.86640167236328,
      "activations/layer19_attention_weight_min": -33.36838150024414,
      "activations/layer1_attention_weight_max": 17.946447372436523,
      "activations/layer1_attention_weight_min": -12.868654251098633,
      "activations/layer20_attention_weight_max": 46.91593933105469,
      "activations/layer20_attention_weight_min": -24.399452209472656,
      "activations/layer21_attention_weight_max": 21.05600929260254,
      "activations/layer21_attention_weight_min": -14.80898666381836,
      "activations/layer22_attention_weight_max": 38.7578010559082,
      "activations/layer22_attention_weight_min": -22.51122283935547,
      "activations/layer23_attention_weight_max": 25.132164001464844,
      "activations/layer23_attention_weight_min": -13.826822280883789,
      "activations/layer2_attention_weight_max": 28.569915771484375,
      "activations/layer2_attention_weight_min": -28.128952026367188,
      "activations/layer3_attention_weight_max": 64.3510971069336,
      "activations/layer3_attention_weight_min": -65.099365234375,
      "activations/layer4_attention_weight_max": 121.08318328857422,
      "activations/layer4_attention_weight_min": -111.83153533935547,
      "activations/layer5_attention_weight_max": 130.427978515625,
      "activations/layer5_attention_weight_min": -114.2950439453125,
      "activations/layer6_attention_weight_max": 73.0411605834961,
      "activations/layer6_attention_weight_min": -63.23874282836914,
      "activations/layer7_attention_weight_max": 66.47994995117188,
      "activations/layer7_attention_weight_min": -68.05687713623047,
      "activations/layer8_attention_weight_max": 68.86062622070312,
      "activations/layer8_attention_weight_min": -56.17656326293945,
      "activations/layer9_attention_weight_max": 94.56588745117188,
      "activations/layer9_attention_weight_min": -73.80564880371094,
      "epoch": 1.11,
      "learning_rate": 0.00014427424242424242,
      "loss": 3.0831,
      "step": 19150
    },
    {
      "activations/layer0_attention_weight_max": 15.05642318725586,
      "activations/layer0_attention_weight_min": -11.281704902648926,
      "activations/layer10_attention_weight_max": 52.50988006591797,
      "activations/layer10_attention_weight_min": -45.7895622253418,
      "activations/layer11_attention_weight_max": 42.04433822631836,
      "activations/layer11_attention_weight_min": -37.04960632324219,
      "activations/layer12_attention_weight_max": 38.51180648803711,
      "activations/layer12_attention_weight_min": -29.85805892944336,
      "activations/layer13_attention_weight_max": 48.2665901184082,
      "activations/layer13_attention_weight_min": -33.89175033569336,
      "activations/layer14_attention_weight_max": 79.02527618408203,
      "activations/layer14_attention_weight_min": -56.78942108154297,
      "activations/layer15_attention_weight_max": 55.946533203125,
      "activations/layer15_attention_weight_min": -40.75448989868164,
      "activations/layer16_attention_weight_max": 59.60322570800781,
      "activations/layer16_attention_weight_min": -41.55435562133789,
      "activations/layer17_attention_weight_max": 37.95769119262695,
      "activations/layer17_attention_weight_min": -25.347280502319336,
      "activations/layer18_attention_weight_max": 24.34345054626465,
      "activations/layer18_attention_weight_min": -14.927565574645996,
      "activations/layer19_attention_weight_max": 46.0677604675293,
      "activations/layer19_attention_weight_min": -35.776241302490234,
      "activations/layer1_attention_weight_max": 15.603353500366211,
      "activations/layer1_attention_weight_min": -12.460140228271484,
      "activations/layer20_attention_weight_max": 38.37067413330078,
      "activations/layer20_attention_weight_min": -27.10414695739746,
      "activations/layer21_attention_weight_max": 25.6942195892334,
      "activations/layer21_attention_weight_min": -15.269490242004395,
      "activations/layer22_attention_weight_max": 38.57083511352539,
      "activations/layer22_attention_weight_min": -23.88206672668457,
      "activations/layer23_attention_weight_max": 25.747215270996094,
      "activations/layer23_attention_weight_min": -14.847625732421875,
      "activations/layer2_attention_weight_max": 27.643592834472656,
      "activations/layer2_attention_weight_min": -27.3646297454834,
      "activations/layer3_attention_weight_max": 59.00933837890625,
      "activations/layer3_attention_weight_min": -60.283851623535156,
      "activations/layer4_attention_weight_max": 107.7120132446289,
      "activations/layer4_attention_weight_min": -100.79463195800781,
      "activations/layer5_attention_weight_max": 112.74130249023438,
      "activations/layer5_attention_weight_min": -103.50180053710938,
      "activations/layer6_attention_weight_max": 62.392005920410156,
      "activations/layer6_attention_weight_min": -61.359195709228516,
      "activations/layer7_attention_weight_max": 62.47414779663086,
      "activations/layer7_attention_weight_min": -58.59719467163086,
      "activations/layer8_attention_weight_max": 56.95537185668945,
      "activations/layer8_attention_weight_min": -52.922386169433594,
      "activations/layer9_attention_weight_max": 68.19003295898438,
      "activations/layer9_attention_weight_min": -58.077510833740234,
      "epoch": 1.12,
      "learning_rate": 0.00014425530303030303,
      "loss": 3.0763,
      "step": 19200
    },
    {
      "activations/layer0_attention_weight_max": 15.204978942871094,
      "activations/layer0_attention_weight_min": -11.350302696228027,
      "activations/layer10_attention_weight_max": 57.75094985961914,
      "activations/layer10_attention_weight_min": -49.962066650390625,
      "activations/layer11_attention_weight_max": 45.706905364990234,
      "activations/layer11_attention_weight_min": -36.08542251586914,
      "activations/layer12_attention_weight_max": 32.37007522583008,
      "activations/layer12_attention_weight_min": -28.041156768798828,
      "activations/layer13_attention_weight_max": 43.77988815307617,
      "activations/layer13_attention_weight_min": -35.238929748535156,
      "activations/layer14_attention_weight_max": 68.23827362060547,
      "activations/layer14_attention_weight_min": -52.692649841308594,
      "activations/layer15_attention_weight_max": 59.76722717285156,
      "activations/layer15_attention_weight_min": -40.22257614135742,
      "activations/layer16_attention_weight_max": 51.98282241821289,
      "activations/layer16_attention_weight_min": -40.38557815551758,
      "activations/layer17_attention_weight_max": 43.9282112121582,
      "activations/layer17_attention_weight_min": -27.428531646728516,
      "activations/layer18_attention_weight_max": 22.76495933532715,
      "activations/layer18_attention_weight_min": -15.418317794799805,
      "activations/layer19_attention_weight_max": 49.04572677612305,
      "activations/layer19_attention_weight_min": -31.638620376586914,
      "activations/layer1_attention_weight_max": 17.46039581298828,
      "activations/layer1_attention_weight_min": -12.237060546875,
      "activations/layer20_attention_weight_max": 39.40895080566406,
      "activations/layer20_attention_weight_min": -27.444969177246094,
      "activations/layer21_attention_weight_max": 23.2919979095459,
      "activations/layer21_attention_weight_min": -16.71964454650879,
      "activations/layer22_attention_weight_max": 41.68738555908203,
      "activations/layer22_attention_weight_min": -22.82501983642578,
      "activations/layer23_attention_weight_max": 22.816864013671875,
      "activations/layer23_attention_weight_min": -13.793195724487305,
      "activations/layer2_attention_weight_max": 28.076343536376953,
      "activations/layer2_attention_weight_min": -28.6137752532959,
      "activations/layer3_attention_weight_max": 58.71244430541992,
      "activations/layer3_attention_weight_min": -62.232200622558594,
      "activations/layer4_attention_weight_max": 106.11922454833984,
      "activations/layer4_attention_weight_min": -106.54376220703125,
      "activations/layer5_attention_weight_max": 115.26300048828125,
      "activations/layer5_attention_weight_min": -107.0394287109375,
      "activations/layer6_attention_weight_max": 62.87616729736328,
      "activations/layer6_attention_weight_min": -54.66822814941406,
      "activations/layer7_attention_weight_max": 57.773380279541016,
      "activations/layer7_attention_weight_min": -59.17317581176758,
      "activations/layer8_attention_weight_max": 59.31125259399414,
      "activations/layer8_attention_weight_min": -49.31416702270508,
      "activations/layer9_attention_weight_max": 81.70630645751953,
      "activations/layer9_attention_weight_min": -63.9747200012207,
      "epoch": 1.12,
      "learning_rate": 0.00014423636363636362,
      "loss": 3.0618,
      "step": 19250
    },
    {
      "activations/layer0_attention_weight_max": 15.696051597595215,
      "activations/layer0_attention_weight_min": -12.2108736038208,
      "activations/layer10_attention_weight_max": 56.24064254760742,
      "activations/layer10_attention_weight_min": -47.29338073730469,
      "activations/layer11_attention_weight_max": 42.10856246948242,
      "activations/layer11_attention_weight_min": -35.54973602294922,
      "activations/layer12_attention_weight_max": 33.43451690673828,
      "activations/layer12_attention_weight_min": -28.19094467163086,
      "activations/layer13_attention_weight_max": 43.58499526977539,
      "activations/layer13_attention_weight_min": -34.10544204711914,
      "activations/layer14_attention_weight_max": 79.38809204101562,
      "activations/layer14_attention_weight_min": -59.191322326660156,
      "activations/layer15_attention_weight_max": 49.28584671020508,
      "activations/layer15_attention_weight_min": -41.74934387207031,
      "activations/layer16_attention_weight_max": 68.11804962158203,
      "activations/layer16_attention_weight_min": -42.0213737487793,
      "activations/layer17_attention_weight_max": 39.26070785522461,
      "activations/layer17_attention_weight_min": -24.537395477294922,
      "activations/layer18_attention_weight_max": 26.54146385192871,
      "activations/layer18_attention_weight_min": -15.246610641479492,
      "activations/layer19_attention_weight_max": 59.87006759643555,
      "activations/layer19_attention_weight_min": -34.4443473815918,
      "activations/layer1_attention_weight_max": 15.539401054382324,
      "activations/layer1_attention_weight_min": -12.215415000915527,
      "activations/layer20_attention_weight_max": 44.973060607910156,
      "activations/layer20_attention_weight_min": -26.63185691833496,
      "activations/layer21_attention_weight_max": 21.74694061279297,
      "activations/layer21_attention_weight_min": -14.837186813354492,
      "activations/layer22_attention_weight_max": 36.08155059814453,
      "activations/layer22_attention_weight_min": -21.7170352935791,
      "activations/layer23_attention_weight_max": 23.142616271972656,
      "activations/layer23_attention_weight_min": -14.188730239868164,
      "activations/layer2_attention_weight_max": 27.397443771362305,
      "activations/layer2_attention_weight_min": -27.355140686035156,
      "activations/layer3_attention_weight_max": 59.555973052978516,
      "activations/layer3_attention_weight_min": -61.76065444946289,
      "activations/layer4_attention_weight_max": 106.72747802734375,
      "activations/layer4_attention_weight_min": -103.33592987060547,
      "activations/layer5_attention_weight_max": 116.152587890625,
      "activations/layer5_attention_weight_min": -100.53842163085938,
      "activations/layer6_attention_weight_max": 65.3137435913086,
      "activations/layer6_attention_weight_min": -56.779075622558594,
      "activations/layer7_attention_weight_max": 61.8745002746582,
      "activations/layer7_attention_weight_min": -60.26121520996094,
      "activations/layer8_attention_weight_max": 58.22184371948242,
      "activations/layer8_attention_weight_min": -46.7327766418457,
      "activations/layer9_attention_weight_max": 74.49556732177734,
      "activations/layer9_attention_weight_min": -62.42476272583008,
      "epoch": 1.12,
      "learning_rate": 0.00014421742424242424,
      "loss": 3.0814,
      "step": 19300
    },
    {
      "activations/layer0_attention_weight_max": 15.512467384338379,
      "activations/layer0_attention_weight_min": -11.45164966583252,
      "activations/layer10_attention_weight_max": 57.085105895996094,
      "activations/layer10_attention_weight_min": -51.087799072265625,
      "activations/layer11_attention_weight_max": 52.7415885925293,
      "activations/layer11_attention_weight_min": -39.62025451660156,
      "activations/layer12_attention_weight_max": 73.4908218383789,
      "activations/layer12_attention_weight_min": -43.93075180053711,
      "activations/layer13_attention_weight_max": 71.1489486694336,
      "activations/layer13_attention_weight_min": -50.485172271728516,
      "activations/layer14_attention_weight_max": 71.59066772460938,
      "activations/layer14_attention_weight_min": -53.29430389404297,
      "activations/layer15_attention_weight_max": 55.53147506713867,
      "activations/layer15_attention_weight_min": -39.02885818481445,
      "activations/layer16_attention_weight_max": 57.2782096862793,
      "activations/layer16_attention_weight_min": -36.1456184387207,
      "activations/layer17_attention_weight_max": 35.39690017700195,
      "activations/layer17_attention_weight_min": -26.25771713256836,
      "activations/layer18_attention_weight_max": 22.767223358154297,
      "activations/layer18_attention_weight_min": -14.631145477294922,
      "activations/layer19_attention_weight_max": 64.3891830444336,
      "activations/layer19_attention_weight_min": -31.27030372619629,
      "activations/layer1_attention_weight_max": 16.66592025756836,
      "activations/layer1_attention_weight_min": -11.388490676879883,
      "activations/layer20_attention_weight_max": 42.329498291015625,
      "activations/layer20_attention_weight_min": -22.625900268554688,
      "activations/layer21_attention_weight_max": 22.49300765991211,
      "activations/layer21_attention_weight_min": -15.785728454589844,
      "activations/layer22_attention_weight_max": 47.99751663208008,
      "activations/layer22_attention_weight_min": -21.65806770324707,
      "activations/layer23_attention_weight_max": 25.132156372070312,
      "activations/layer23_attention_weight_min": -15.73292064666748,
      "activations/layer2_attention_weight_max": 29.1746883392334,
      "activations/layer2_attention_weight_min": -28.391559600830078,
      "activations/layer3_attention_weight_max": 65.58786010742188,
      "activations/layer3_attention_weight_min": -66.1256103515625,
      "activations/layer4_attention_weight_max": 119.86048889160156,
      "activations/layer4_attention_weight_min": -107.26390838623047,
      "activations/layer5_attention_weight_max": 136.64361572265625,
      "activations/layer5_attention_weight_min": -113.70662689208984,
      "activations/layer6_attention_weight_max": 71.94345092773438,
      "activations/layer6_attention_weight_min": -59.72496032714844,
      "activations/layer7_attention_weight_max": 71.15647888183594,
      "activations/layer7_attention_weight_min": -61.81186294555664,
      "activations/layer8_attention_weight_max": 69.91139221191406,
      "activations/layer8_attention_weight_min": -51.719207763671875,
      "activations/layer9_attention_weight_max": 95.6264877319336,
      "activations/layer9_attention_weight_min": -72.88884735107422,
      "epoch": 1.12,
      "learning_rate": 0.00014419848484848485,
      "loss": 3.0721,
      "step": 19350
    },
    {
      "activations/layer0_attention_weight_max": 15.230794906616211,
      "activations/layer0_attention_weight_min": -11.097180366516113,
      "activations/layer10_attention_weight_max": 56.72848892211914,
      "activations/layer10_attention_weight_min": -47.58628845214844,
      "activations/layer11_attention_weight_max": 48.81953430175781,
      "activations/layer11_attention_weight_min": -40.411888122558594,
      "activations/layer12_attention_weight_max": 54.43693923950195,
      "activations/layer12_attention_weight_min": -28.64359474182129,
      "activations/layer13_attention_weight_max": 45.81913375854492,
      "activations/layer13_attention_weight_min": -33.405540466308594,
      "activations/layer14_attention_weight_max": 76.49120330810547,
      "activations/layer14_attention_weight_min": -56.224639892578125,
      "activations/layer15_attention_weight_max": 56.26905059814453,
      "activations/layer15_attention_weight_min": -39.88996505737305,
      "activations/layer16_attention_weight_max": 55.520668029785156,
      "activations/layer16_attention_weight_min": -42.51466369628906,
      "activations/layer17_attention_weight_max": 41.36829376220703,
      "activations/layer17_attention_weight_min": -25.829605102539062,
      "activations/layer18_attention_weight_max": 22.483285903930664,
      "activations/layer18_attention_weight_min": -15.324319839477539,
      "activations/layer19_attention_weight_max": 52.91524887084961,
      "activations/layer19_attention_weight_min": -36.60500717163086,
      "activations/layer1_attention_weight_max": 15.994672775268555,
      "activations/layer1_attention_weight_min": -11.69622802734375,
      "activations/layer20_attention_weight_max": 42.21027374267578,
      "activations/layer20_attention_weight_min": -27.850400924682617,
      "activations/layer21_attention_weight_max": 22.684972763061523,
      "activations/layer21_attention_weight_min": -14.08187484741211,
      "activations/layer22_attention_weight_max": 38.13541030883789,
      "activations/layer22_attention_weight_min": -23.34296417236328,
      "activations/layer23_attention_weight_max": 21.92682647705078,
      "activations/layer23_attention_weight_min": -14.582540512084961,
      "activations/layer2_attention_weight_max": 26.158727645874023,
      "activations/layer2_attention_weight_min": -27.29556655883789,
      "activations/layer3_attention_weight_max": 59.203250885009766,
      "activations/layer3_attention_weight_min": -61.871639251708984,
      "activations/layer4_attention_weight_max": 105.9815902709961,
      "activations/layer4_attention_weight_min": -102.10420989990234,
      "activations/layer5_attention_weight_max": 113.22616577148438,
      "activations/layer5_attention_weight_min": -105.0703353881836,
      "activations/layer6_attention_weight_max": 66.82737731933594,
      "activations/layer6_attention_weight_min": -57.434391021728516,
      "activations/layer7_attention_weight_max": 58.54331970214844,
      "activations/layer7_attention_weight_min": -61.44451141357422,
      "activations/layer8_attention_weight_max": 53.0508918762207,
      "activations/layer8_attention_weight_min": -48.02557373046875,
      "activations/layer9_attention_weight_max": 74.07161712646484,
      "activations/layer9_attention_weight_min": -64.98125457763672,
      "epoch": 1.13,
      "learning_rate": 0.00014417954545454544,
      "loss": 3.0784,
      "step": 19400
    },
    {
      "activations/layer0_attention_weight_max": 15.967606544494629,
      "activations/layer0_attention_weight_min": -11.204697608947754,
      "activations/layer10_attention_weight_max": 54.57831954956055,
      "activations/layer10_attention_weight_min": -46.47166442871094,
      "activations/layer11_attention_weight_max": 44.073360443115234,
      "activations/layer11_attention_weight_min": -39.25440979003906,
      "activations/layer12_attention_weight_max": 78.00233459472656,
      "activations/layer12_attention_weight_min": -56.79300308227539,
      "activations/layer13_attention_weight_max": 66.16938781738281,
      "activations/layer13_attention_weight_min": -51.46266555786133,
      "activations/layer14_attention_weight_max": 78.44090270996094,
      "activations/layer14_attention_weight_min": -55.404747009277344,
      "activations/layer15_attention_weight_max": 56.433475494384766,
      "activations/layer15_attention_weight_min": -39.42764663696289,
      "activations/layer16_attention_weight_max": 49.327388763427734,
      "activations/layer16_attention_weight_min": -35.275753021240234,
      "activations/layer17_attention_weight_max": 39.18046951293945,
      "activations/layer17_attention_weight_min": -25.591569900512695,
      "activations/layer18_attention_weight_max": 22.49089241027832,
      "activations/layer18_attention_weight_min": -13.987171173095703,
      "activations/layer19_attention_weight_max": 45.615150451660156,
      "activations/layer19_attention_weight_min": -29.218847274780273,
      "activations/layer1_attention_weight_max": 16.080482482910156,
      "activations/layer1_attention_weight_min": -12.631016731262207,
      "activations/layer20_attention_weight_max": 38.06290054321289,
      "activations/layer20_attention_weight_min": -22.98646354675293,
      "activations/layer21_attention_weight_max": 24.902263641357422,
      "activations/layer21_attention_weight_min": -15.053157806396484,
      "activations/layer22_attention_weight_max": 41.915611267089844,
      "activations/layer22_attention_weight_min": -23.407817840576172,
      "activations/layer23_attention_weight_max": 23.644771575927734,
      "activations/layer23_attention_weight_min": -15.07184886932373,
      "activations/layer2_attention_weight_max": 27.967323303222656,
      "activations/layer2_attention_weight_min": -28.100744247436523,
      "activations/layer3_attention_weight_max": 66.57830047607422,
      "activations/layer3_attention_weight_min": -65.58882141113281,
      "activations/layer4_attention_weight_max": 118.23186492919922,
      "activations/layer4_attention_weight_min": -113.46685791015625,
      "activations/layer5_attention_weight_max": 125.63995361328125,
      "activations/layer5_attention_weight_min": -108.11045837402344,
      "activations/layer6_attention_weight_max": 69.82357788085938,
      "activations/layer6_attention_weight_min": -58.32130813598633,
      "activations/layer7_attention_weight_max": 61.85580825805664,
      "activations/layer7_attention_weight_min": -64.4199447631836,
      "activations/layer8_attention_weight_max": 60.57722854614258,
      "activations/layer8_attention_weight_min": -49.20710372924805,
      "activations/layer9_attention_weight_max": 73.29102325439453,
      "activations/layer9_attention_weight_min": -61.421966552734375,
      "epoch": 1.13,
      "learning_rate": 0.00014416060606060605,
      "loss": 3.074,
      "step": 19450
    },
    {
      "activations/layer0_attention_weight_max": 15.775300025939941,
      "activations/layer0_attention_weight_min": -11.6714448928833,
      "activations/layer10_attention_weight_max": 53.318023681640625,
      "activations/layer10_attention_weight_min": -46.93192672729492,
      "activations/layer11_attention_weight_max": 47.20713424682617,
      "activations/layer11_attention_weight_min": -40.34923553466797,
      "activations/layer12_attention_weight_max": 33.03875732421875,
      "activations/layer12_attention_weight_min": -31.083772659301758,
      "activations/layer13_attention_weight_max": 44.45094680786133,
      "activations/layer13_attention_weight_min": -34.452735900878906,
      "activations/layer14_attention_weight_max": 68.9510498046875,
      "activations/layer14_attention_weight_min": -55.61614227294922,
      "activations/layer15_attention_weight_max": 55.51244354248047,
      "activations/layer15_attention_weight_min": -40.87166976928711,
      "activations/layer16_attention_weight_max": 63.86473846435547,
      "activations/layer16_attention_weight_min": -44.34531021118164,
      "activations/layer17_attention_weight_max": 40.8446159362793,
      "activations/layer17_attention_weight_min": -25.911544799804688,
      "activations/layer18_attention_weight_max": 23.541791915893555,
      "activations/layer18_attention_weight_min": -15.968324661254883,
      "activations/layer19_attention_weight_max": 47.721702575683594,
      "activations/layer19_attention_weight_min": -32.260250091552734,
      "activations/layer1_attention_weight_max": 16.87394905090332,
      "activations/layer1_attention_weight_min": -12.112207412719727,
      "activations/layer20_attention_weight_max": 44.03376770019531,
      "activations/layer20_attention_weight_min": -26.104576110839844,
      "activations/layer21_attention_weight_max": 22.9416561126709,
      "activations/layer21_attention_weight_min": -15.149080276489258,
      "activations/layer22_attention_weight_max": 42.5679817199707,
      "activations/layer22_attention_weight_min": -23.38144874572754,
      "activations/layer23_attention_weight_max": 26.257251739501953,
      "activations/layer23_attention_weight_min": -16.13932991027832,
      "activations/layer2_attention_weight_max": 27.96406364440918,
      "activations/layer2_attention_weight_min": -29.687030792236328,
      "activations/layer3_attention_weight_max": 61.80432891845703,
      "activations/layer3_attention_weight_min": -63.13280487060547,
      "activations/layer4_attention_weight_max": 110.85456848144531,
      "activations/layer4_attention_weight_min": -107.95189666748047,
      "activations/layer5_attention_weight_max": 118.43917846679688,
      "activations/layer5_attention_weight_min": -107.93453216552734,
      "activations/layer6_attention_weight_max": 63.40105056762695,
      "activations/layer6_attention_weight_min": -57.08821487426758,
      "activations/layer7_attention_weight_max": 60.419334411621094,
      "activations/layer7_attention_weight_min": -60.2095947265625,
      "activations/layer8_attention_weight_max": 55.29289627075195,
      "activations/layer8_attention_weight_min": -48.377410888671875,
      "activations/layer9_attention_weight_max": 81.3941421508789,
      "activations/layer9_attention_weight_min": -75.06001281738281,
      "epoch": 1.13,
      "learning_rate": 0.00014414166666666667,
      "loss": 3.0734,
      "step": 19500
    },
    {
      "activations/layer0_attention_weight_max": 15.800618171691895,
      "activations/layer0_attention_weight_min": -11.361391067504883,
      "activations/layer10_attention_weight_max": 52.622894287109375,
      "activations/layer10_attention_weight_min": -49.76362991333008,
      "activations/layer11_attention_weight_max": 39.617164611816406,
      "activations/layer11_attention_weight_min": -35.402259826660156,
      "activations/layer12_attention_weight_max": 32.340248107910156,
      "activations/layer12_attention_weight_min": -28.520065307617188,
      "activations/layer13_attention_weight_max": 42.14518356323242,
      "activations/layer13_attention_weight_min": -35.3585205078125,
      "activations/layer14_attention_weight_max": 68.6912612915039,
      "activations/layer14_attention_weight_min": -51.74802017211914,
      "activations/layer15_attention_weight_max": 52.88340377807617,
      "activations/layer15_attention_weight_min": -37.40554428100586,
      "activations/layer16_attention_weight_max": 50.220176696777344,
      "activations/layer16_attention_weight_min": -40.34187316894531,
      "activations/layer17_attention_weight_max": 37.66246032714844,
      "activations/layer17_attention_weight_min": -25.401168823242188,
      "activations/layer18_attention_weight_max": 22.22748565673828,
      "activations/layer18_attention_weight_min": -14.926769256591797,
      "activations/layer19_attention_weight_max": 56.179237365722656,
      "activations/layer19_attention_weight_min": -32.28371810913086,
      "activations/layer1_attention_weight_max": 16.07315444946289,
      "activations/layer1_attention_weight_min": -11.621109008789062,
      "activations/layer20_attention_weight_max": 43.12427520751953,
      "activations/layer20_attention_weight_min": -27.548250198364258,
      "activations/layer21_attention_weight_max": 23.5908203125,
      "activations/layer21_attention_weight_min": -15.298176765441895,
      "activations/layer22_attention_weight_max": 43.25368118286133,
      "activations/layer22_attention_weight_min": -24.024112701416016,
      "activations/layer23_attention_weight_max": 26.619178771972656,
      "activations/layer23_attention_weight_min": -15.525359153747559,
      "activations/layer2_attention_weight_max": 26.831649780273438,
      "activations/layer2_attention_weight_min": -26.293546676635742,
      "activations/layer3_attention_weight_max": 62.881324768066406,
      "activations/layer3_attention_weight_min": -62.978973388671875,
      "activations/layer4_attention_weight_max": 109.3174057006836,
      "activations/layer4_attention_weight_min": -103.3991928100586,
      "activations/layer5_attention_weight_max": 114.95437622070312,
      "activations/layer5_attention_weight_min": -103.58708953857422,
      "activations/layer6_attention_weight_max": 62.49268341064453,
      "activations/layer6_attention_weight_min": -56.51264572143555,
      "activations/layer7_attention_weight_max": 57.55125427246094,
      "activations/layer7_attention_weight_min": -59.78240966796875,
      "activations/layer8_attention_weight_max": 56.55653762817383,
      "activations/layer8_attention_weight_min": -46.06141662597656,
      "activations/layer9_attention_weight_max": 69.01846313476562,
      "activations/layer9_attention_weight_min": -60.22868728637695,
      "epoch": 1.14,
      "learning_rate": 0.00014412272727272726,
      "loss": 3.0615,
      "step": 19550
    },
    {
      "activations/layer0_attention_weight_max": 15.944663047790527,
      "activations/layer0_attention_weight_min": -11.131625175476074,
      "activations/layer10_attention_weight_max": 56.37339782714844,
      "activations/layer10_attention_weight_min": -50.934295654296875,
      "activations/layer11_attention_weight_max": 55.217018127441406,
      "activations/layer11_attention_weight_min": -44.94379425048828,
      "activations/layer12_attention_weight_max": 49.79275131225586,
      "activations/layer12_attention_weight_min": -40.425838470458984,
      "activations/layer13_attention_weight_max": 62.4027099609375,
      "activations/layer13_attention_weight_min": -49.01115036010742,
      "activations/layer14_attention_weight_max": 71.42396545410156,
      "activations/layer14_attention_weight_min": -60.599117279052734,
      "activations/layer15_attention_weight_max": 50.10844421386719,
      "activations/layer15_attention_weight_min": -43.19937515258789,
      "activations/layer16_attention_weight_max": 54.74740982055664,
      "activations/layer16_attention_weight_min": -37.15033721923828,
      "activations/layer17_attention_weight_max": 37.06468963623047,
      "activations/layer17_attention_weight_min": -25.948122024536133,
      "activations/layer18_attention_weight_max": 24.107948303222656,
      "activations/layer18_attention_weight_min": -14.496201515197754,
      "activations/layer19_attention_weight_max": 48.63521957397461,
      "activations/layer19_attention_weight_min": -35.1552848815918,
      "activations/layer1_attention_weight_max": 17.998167037963867,
      "activations/layer1_attention_weight_min": -11.302752494812012,
      "activations/layer20_attention_weight_max": 41.86972427368164,
      "activations/layer20_attention_weight_min": -26.595304489135742,
      "activations/layer21_attention_weight_max": 22.5804500579834,
      "activations/layer21_attention_weight_min": -14.77241325378418,
      "activations/layer22_attention_weight_max": 44.0699348449707,
      "activations/layer22_attention_weight_min": -23.178417205810547,
      "activations/layer23_attention_weight_max": 26.7181396484375,
      "activations/layer23_attention_weight_min": -13.86075496673584,
      "activations/layer2_attention_weight_max": 26.717540740966797,
      "activations/layer2_attention_weight_min": -25.940536499023438,
      "activations/layer3_attention_weight_max": 60.9790153503418,
      "activations/layer3_attention_weight_min": -65.4063949584961,
      "activations/layer4_attention_weight_max": 112.3647232055664,
      "activations/layer4_attention_weight_min": -107.96514892578125,
      "activations/layer5_attention_weight_max": 116.65048217773438,
      "activations/layer5_attention_weight_min": -104.026611328125,
      "activations/layer6_attention_weight_max": 65.79780578613281,
      "activations/layer6_attention_weight_min": -57.99760437011719,
      "activations/layer7_attention_weight_max": 60.93346405029297,
      "activations/layer7_attention_weight_min": -61.70029830932617,
      "activations/layer8_attention_weight_max": 53.30656433105469,
      "activations/layer8_attention_weight_min": -50.95458984375,
      "activations/layer9_attention_weight_max": 73.33846282958984,
      "activations/layer9_attention_weight_min": -69.06486511230469,
      "epoch": 1.14,
      "learning_rate": 0.00014410378787878787,
      "loss": 3.0801,
      "step": 19600
    },
    {
      "activations/layer0_attention_weight_max": 16.18001937866211,
      "activations/layer0_attention_weight_min": -11.6869478225708,
      "activations/layer10_attention_weight_max": 68.41694641113281,
      "activations/layer10_attention_weight_min": -57.60900115966797,
      "activations/layer11_attention_weight_max": 55.52926254272461,
      "activations/layer11_attention_weight_min": -38.25121307373047,
      "activations/layer12_attention_weight_max": 60.611202239990234,
      "activations/layer12_attention_weight_min": -33.48755645751953,
      "activations/layer13_attention_weight_max": 80.79984283447266,
      "activations/layer13_attention_weight_min": -37.69661331176758,
      "activations/layer14_attention_weight_max": 119.78071594238281,
      "activations/layer14_attention_weight_min": -56.26283645629883,
      "activations/layer15_attention_weight_max": 61.52684020996094,
      "activations/layer15_attention_weight_min": -39.055057525634766,
      "activations/layer16_attention_weight_max": 58.59632110595703,
      "activations/layer16_attention_weight_min": -38.60819625854492,
      "activations/layer17_attention_weight_max": 32.58152770996094,
      "activations/layer17_attention_weight_min": -23.989059448242188,
      "activations/layer18_attention_weight_max": 30.796165466308594,
      "activations/layer18_attention_weight_min": -16.248994827270508,
      "activations/layer19_attention_weight_max": 52.4946403503418,
      "activations/layer19_attention_weight_min": -33.15799331665039,
      "activations/layer1_attention_weight_max": 16.34922218322754,
      "activations/layer1_attention_weight_min": -12.177390098571777,
      "activations/layer20_attention_weight_max": 37.647666931152344,
      "activations/layer20_attention_weight_min": -23.415557861328125,
      "activations/layer21_attention_weight_max": 22.4830265045166,
      "activations/layer21_attention_weight_min": -20.316022872924805,
      "activations/layer22_attention_weight_max": 40.22853088378906,
      "activations/layer22_attention_weight_min": -21.501415252685547,
      "activations/layer23_attention_weight_max": 26.00766372680664,
      "activations/layer23_attention_weight_min": -13.312423706054688,
      "activations/layer2_attention_weight_max": 26.96901512145996,
      "activations/layer2_attention_weight_min": -27.40578269958496,
      "activations/layer3_attention_weight_max": 70.74209594726562,
      "activations/layer3_attention_weight_min": -68.94673156738281,
      "activations/layer4_attention_weight_max": 132.7958984375,
      "activations/layer4_attention_weight_min": -110.86585998535156,
      "activations/layer5_attention_weight_max": 144.96041870117188,
      "activations/layer5_attention_weight_min": -117.23899841308594,
      "activations/layer6_attention_weight_max": 84.20692443847656,
      "activations/layer6_attention_weight_min": -61.17290115356445,
      "activations/layer7_attention_weight_max": 72.15760040283203,
      "activations/layer7_attention_weight_min": -63.64445495605469,
      "activations/layer8_attention_weight_max": 75.51341247558594,
      "activations/layer8_attention_weight_min": -60.36380386352539,
      "activations/layer9_attention_weight_max": 105.51236724853516,
      "activations/layer9_attention_weight_min": -69.9073486328125,
      "epoch": 1.14,
      "learning_rate": 0.00014408484848484846,
      "loss": 3.0822,
      "step": 19650
    },
    {
      "activations/layer0_attention_weight_max": 16.598291397094727,
      "activations/layer0_attention_weight_min": -11.523544311523438,
      "activations/layer10_attention_weight_max": 51.835662841796875,
      "activations/layer10_attention_weight_min": -47.832672119140625,
      "activations/layer11_attention_weight_max": 42.22357177734375,
      "activations/layer11_attention_weight_min": -36.20027160644531,
      "activations/layer12_attention_weight_max": 43.509132385253906,
      "activations/layer12_attention_weight_min": -34.33332443237305,
      "activations/layer13_attention_weight_max": 50.21965026855469,
      "activations/layer13_attention_weight_min": -38.90040588378906,
      "activations/layer14_attention_weight_max": 66.03728485107422,
      "activations/layer14_attention_weight_min": -50.99123764038086,
      "activations/layer15_attention_weight_max": 54.258544921875,
      "activations/layer15_attention_weight_min": -39.98858642578125,
      "activations/layer16_attention_weight_max": 56.58863067626953,
      "activations/layer16_attention_weight_min": -36.94257354736328,
      "activations/layer17_attention_weight_max": 37.75611114501953,
      "activations/layer17_attention_weight_min": -26.31669807434082,
      "activations/layer18_attention_weight_max": 24.650129318237305,
      "activations/layer18_attention_weight_min": -16.79282569885254,
      "activations/layer19_attention_weight_max": 49.74409484863281,
      "activations/layer19_attention_weight_min": -31.045530319213867,
      "activations/layer1_attention_weight_max": 17.567153930664062,
      "activations/layer1_attention_weight_min": -12.17374038696289,
      "activations/layer20_attention_weight_max": 39.966400146484375,
      "activations/layer20_attention_weight_min": -25.100505828857422,
      "activations/layer21_attention_weight_max": 21.865816116333008,
      "activations/layer21_attention_weight_min": -14.739843368530273,
      "activations/layer22_attention_weight_max": 39.68143844604492,
      "activations/layer22_attention_weight_min": -23.76954460144043,
      "activations/layer23_attention_weight_max": 25.015033721923828,
      "activations/layer23_attention_weight_min": -15.876090049743652,
      "activations/layer2_attention_weight_max": 26.98293113708496,
      "activations/layer2_attention_weight_min": -27.564308166503906,
      "activations/layer3_attention_weight_max": 58.73302459716797,
      "activations/layer3_attention_weight_min": -61.43855667114258,
      "activations/layer4_attention_weight_max": 104.03087615966797,
      "activations/layer4_attention_weight_min": -105.05785369873047,
      "activations/layer5_attention_weight_max": 113.53495788574219,
      "activations/layer5_attention_weight_min": -106.00756072998047,
      "activations/layer6_attention_weight_max": 62.877235412597656,
      "activations/layer6_attention_weight_min": -55.77045440673828,
      "activations/layer7_attention_weight_max": 58.01298522949219,
      "activations/layer7_attention_weight_min": -56.468299865722656,
      "activations/layer8_attention_weight_max": 55.33033752441406,
      "activations/layer8_attention_weight_min": -49.98666000366211,
      "activations/layer9_attention_weight_max": 62.305355072021484,
      "activations/layer9_attention_weight_min": -63.64832305908203,
      "epoch": 1.14,
      "learning_rate": 0.00014406590909090908,
      "loss": 3.0847,
      "step": 19700
    },
    {
      "activations/layer0_attention_weight_max": 16.533493041992188,
      "activations/layer0_attention_weight_min": -10.869356155395508,
      "activations/layer10_attention_weight_max": 49.84619903564453,
      "activations/layer10_attention_weight_min": -41.86800765991211,
      "activations/layer11_attention_weight_max": 41.45405578613281,
      "activations/layer11_attention_weight_min": -38.982269287109375,
      "activations/layer12_attention_weight_max": 43.76404571533203,
      "activations/layer12_attention_weight_min": -35.954837799072266,
      "activations/layer13_attention_weight_max": 44.91718292236328,
      "activations/layer13_attention_weight_min": -37.43571472167969,
      "activations/layer14_attention_weight_max": 73.85446166992188,
      "activations/layer14_attention_weight_min": -64.2694091796875,
      "activations/layer15_attention_weight_max": 56.86968994140625,
      "activations/layer15_attention_weight_min": -50.3083381652832,
      "activations/layer16_attention_weight_max": 50.40332794189453,
      "activations/layer16_attention_weight_min": -43.020240783691406,
      "activations/layer17_attention_weight_max": 36.34652328491211,
      "activations/layer17_attention_weight_min": -29.8112735748291,
      "activations/layer18_attention_weight_max": 20.119918823242188,
      "activations/layer18_attention_weight_min": -15.249749183654785,
      "activations/layer19_attention_weight_max": 47.192569732666016,
      "activations/layer19_attention_weight_min": -34.74772644042969,
      "activations/layer1_attention_weight_max": 16.091060638427734,
      "activations/layer1_attention_weight_min": -11.275763511657715,
      "activations/layer20_attention_weight_max": 39.58787536621094,
      "activations/layer20_attention_weight_min": -26.37872314453125,
      "activations/layer21_attention_weight_max": 21.234277725219727,
      "activations/layer21_attention_weight_min": -15.359107971191406,
      "activations/layer22_attention_weight_max": 36.74388122558594,
      "activations/layer22_attention_weight_min": -23.918792724609375,
      "activations/layer23_attention_weight_max": 22.48420524597168,
      "activations/layer23_attention_weight_min": -15.09210205078125,
      "activations/layer2_attention_weight_max": 26.832075119018555,
      "activations/layer2_attention_weight_min": -25.680875778198242,
      "activations/layer3_attention_weight_max": 57.98142623901367,
      "activations/layer3_attention_weight_min": -60.801902770996094,
      "activations/layer4_attention_weight_max": 108.1913833618164,
      "activations/layer4_attention_weight_min": -104.33312225341797,
      "activations/layer5_attention_weight_max": 117.37251281738281,
      "activations/layer5_attention_weight_min": -102.3558349609375,
      "activations/layer6_attention_weight_max": 63.3271484375,
      "activations/layer6_attention_weight_min": -54.37389373779297,
      "activations/layer7_attention_weight_max": 56.65525436401367,
      "activations/layer7_attention_weight_min": -58.77294921875,
      "activations/layer8_attention_weight_max": 55.65324401855469,
      "activations/layer8_attention_weight_min": -51.719146728515625,
      "activations/layer9_attention_weight_max": 78.69255828857422,
      "activations/layer9_attention_weight_min": -63.534088134765625,
      "epoch": 1.15,
      "learning_rate": 0.0001440469696969697,
      "loss": 3.0588,
      "step": 19750
    },
    {
      "activations/layer0_attention_weight_max": 15.9888334274292,
      "activations/layer0_attention_weight_min": -11.718586921691895,
      "activations/layer10_attention_weight_max": 51.154998779296875,
      "activations/layer10_attention_weight_min": -45.687503814697266,
      "activations/layer11_attention_weight_max": 39.726463317871094,
      "activations/layer11_attention_weight_min": -37.700035095214844,
      "activations/layer12_attention_weight_max": 53.1567497253418,
      "activations/layer12_attention_weight_min": -28.157747268676758,
      "activations/layer13_attention_weight_max": 50.367610931396484,
      "activations/layer13_attention_weight_min": -34.06511688232422,
      "activations/layer14_attention_weight_max": 67.6705322265625,
      "activations/layer14_attention_weight_min": -50.105979919433594,
      "activations/layer15_attention_weight_max": 47.557273864746094,
      "activations/layer15_attention_weight_min": -36.04075241088867,
      "activations/layer16_attention_weight_max": 53.861610412597656,
      "activations/layer16_attention_weight_min": -36.471622467041016,
      "activations/layer17_attention_weight_max": 43.515525817871094,
      "activations/layer17_attention_weight_min": -25.327783584594727,
      "activations/layer18_attention_weight_max": 29.750810623168945,
      "activations/layer18_attention_weight_min": -16.014301300048828,
      "activations/layer19_attention_weight_max": 45.41228103637695,
      "activations/layer19_attention_weight_min": -30.7246036529541,
      "activations/layer1_attention_weight_max": 17.641551971435547,
      "activations/layer1_attention_weight_min": -11.955839157104492,
      "activations/layer20_attention_weight_max": 41.43497085571289,
      "activations/layer20_attention_weight_min": -24.276519775390625,
      "activations/layer21_attention_weight_max": 21.22678565979004,
      "activations/layer21_attention_weight_min": -14.920957565307617,
      "activations/layer22_attention_weight_max": 41.07501983642578,
      "activations/layer22_attention_weight_min": -22.055950164794922,
      "activations/layer23_attention_weight_max": 23.534894943237305,
      "activations/layer23_attention_weight_min": -14.945796966552734,
      "activations/layer2_attention_weight_max": 26.84237289428711,
      "activations/layer2_attention_weight_min": -26.142009735107422,
      "activations/layer3_attention_weight_max": 59.96660614013672,
      "activations/layer3_attention_weight_min": -61.77902603149414,
      "activations/layer4_attention_weight_max": 103.4734878540039,
      "activations/layer4_attention_weight_min": -103.6009750366211,
      "activations/layer5_attention_weight_max": 107.91482543945312,
      "activations/layer5_attention_weight_min": -102.39022064208984,
      "activations/layer6_attention_weight_max": 58.74064636230469,
      "activations/layer6_attention_weight_min": -55.73224639892578,
      "activations/layer7_attention_weight_max": 57.11289596557617,
      "activations/layer7_attention_weight_min": -61.00504684448242,
      "activations/layer8_attention_weight_max": 51.693321228027344,
      "activations/layer8_attention_weight_min": -47.927215576171875,
      "activations/layer9_attention_weight_max": 65.80915069580078,
      "activations/layer9_attention_weight_min": -57.753047943115234,
      "epoch": 1.15,
      "learning_rate": 0.00014402803030303028,
      "loss": 3.0687,
      "step": 19800
    },
    {
      "activations/layer0_attention_weight_max": 16.676557540893555,
      "activations/layer0_attention_weight_min": -11.419336318969727,
      "activations/layer10_attention_weight_max": 54.9968376159668,
      "activations/layer10_attention_weight_min": -48.773223876953125,
      "activations/layer11_attention_weight_max": 43.64487838745117,
      "activations/layer11_attention_weight_min": -35.134334564208984,
      "activations/layer12_attention_weight_max": 34.14052963256836,
      "activations/layer12_attention_weight_min": -32.809181213378906,
      "activations/layer13_attention_weight_max": 45.52342987060547,
      "activations/layer13_attention_weight_min": -34.79414367675781,
      "activations/layer14_attention_weight_max": 81.02278137207031,
      "activations/layer14_attention_weight_min": -62.765037536621094,
      "activations/layer15_attention_weight_max": 61.07776641845703,
      "activations/layer15_attention_weight_min": -44.69477462768555,
      "activations/layer16_attention_weight_max": 56.274986267089844,
      "activations/layer16_attention_weight_min": -39.92355728149414,
      "activations/layer17_attention_weight_max": 39.29713821411133,
      "activations/layer17_attention_weight_min": -30.08660316467285,
      "activations/layer18_attention_weight_max": 21.988405227661133,
      "activations/layer18_attention_weight_min": -14.88895320892334,
      "activations/layer19_attention_weight_max": 52.02223587036133,
      "activations/layer19_attention_weight_min": -34.74244689941406,
      "activations/layer1_attention_weight_max": 16.976104736328125,
      "activations/layer1_attention_weight_min": -13.14963150024414,
      "activations/layer20_attention_weight_max": 45.92247009277344,
      "activations/layer20_attention_weight_min": -26.323347091674805,
      "activations/layer21_attention_weight_max": 24.195444107055664,
      "activations/layer21_attention_weight_min": -14.643684387207031,
      "activations/layer22_attention_weight_max": 39.39767074584961,
      "activations/layer22_attention_weight_min": -23.90169906616211,
      "activations/layer23_attention_weight_max": 24.184768676757812,
      "activations/layer23_attention_weight_min": -15.862138748168945,
      "activations/layer2_attention_weight_max": 28.058942794799805,
      "activations/layer2_attention_weight_min": -28.293142318725586,
      "activations/layer3_attention_weight_max": 62.297210693359375,
      "activations/layer3_attention_weight_min": -63.44904708862305,
      "activations/layer4_attention_weight_max": 111.51549530029297,
      "activations/layer4_attention_weight_min": -108.8268814086914,
      "activations/layer5_attention_weight_max": 120.39683532714844,
      "activations/layer5_attention_weight_min": -104.3582763671875,
      "activations/layer6_attention_weight_max": 64.8753890991211,
      "activations/layer6_attention_weight_min": -55.64396667480469,
      "activations/layer7_attention_weight_max": 61.85873031616211,
      "activations/layer7_attention_weight_min": -59.54792404174805,
      "activations/layer8_attention_weight_max": 53.313236236572266,
      "activations/layer8_attention_weight_min": -48.79692459106445,
      "activations/layer9_attention_weight_max": 72.71470642089844,
      "activations/layer9_attention_weight_min": -62.16238021850586,
      "epoch": 1.15,
      "learning_rate": 0.0001440090909090909,
      "loss": 3.072,
      "step": 19850
    },
    {
      "activations/layer0_attention_weight_max": 15.718830108642578,
      "activations/layer0_attention_weight_min": -11.620622634887695,
      "activations/layer10_attention_weight_max": 56.44942092895508,
      "activations/layer10_attention_weight_min": -53.03630828857422,
      "activations/layer11_attention_weight_max": 43.23857116699219,
      "activations/layer11_attention_weight_min": -38.77159881591797,
      "activations/layer12_attention_weight_max": 32.297035217285156,
      "activations/layer12_attention_weight_min": -25.313596725463867,
      "activations/layer13_attention_weight_max": 42.891197204589844,
      "activations/layer13_attention_weight_min": -36.354610443115234,
      "activations/layer14_attention_weight_max": 70.21192932128906,
      "activations/layer14_attention_weight_min": -56.02436065673828,
      "activations/layer15_attention_weight_max": 64.31060791015625,
      "activations/layer15_attention_weight_min": -46.96225357055664,
      "activations/layer16_attention_weight_max": 54.16187286376953,
      "activations/layer16_attention_weight_min": -38.46296310424805,
      "activations/layer17_attention_weight_max": 40.31128692626953,
      "activations/layer17_attention_weight_min": -29.21271514892578,
      "activations/layer18_attention_weight_max": 22.71454429626465,
      "activations/layer18_attention_weight_min": -15.5285062789917,
      "activations/layer19_attention_weight_max": 46.77669143676758,
      "activations/layer19_attention_weight_min": -32.96869659423828,
      "activations/layer1_attention_weight_max": 17.147369384765625,
      "activations/layer1_attention_weight_min": -11.637624740600586,
      "activations/layer20_attention_weight_max": 39.149391174316406,
      "activations/layer20_attention_weight_min": -27.107454299926758,
      "activations/layer21_attention_weight_max": 21.6568546295166,
      "activations/layer21_attention_weight_min": -16.261459350585938,
      "activations/layer22_attention_weight_max": 39.863136291503906,
      "activations/layer22_attention_weight_min": -23.58534812927246,
      "activations/layer23_attention_weight_max": 24.03927230834961,
      "activations/layer23_attention_weight_min": -15.013274192810059,
      "activations/layer2_attention_weight_max": 26.949844360351562,
      "activations/layer2_attention_weight_min": -26.663043975830078,
      "activations/layer3_attention_weight_max": 60.89455032348633,
      "activations/layer3_attention_weight_min": -63.52519226074219,
      "activations/layer4_attention_weight_max": 113.23175048828125,
      "activations/layer4_attention_weight_min": -114.5734634399414,
      "activations/layer5_attention_weight_max": 118.53097534179688,
      "activations/layer5_attention_weight_min": -107.4854736328125,
      "activations/layer6_attention_weight_max": 64.30558013916016,
      "activations/layer6_attention_weight_min": -59.10346984863281,
      "activations/layer7_attention_weight_max": 61.63408279418945,
      "activations/layer7_attention_weight_min": -59.54303741455078,
      "activations/layer8_attention_weight_max": 60.60947799682617,
      "activations/layer8_attention_weight_min": -49.48015594482422,
      "activations/layer9_attention_weight_max": 72.15727996826172,
      "activations/layer9_attention_weight_min": -62.369869232177734,
      "epoch": 1.16,
      "learning_rate": 0.0001439901515151515,
      "loss": 3.0528,
      "step": 19900
    },
    {
      "activations/layer0_attention_weight_max": 15.556427955627441,
      "activations/layer0_attention_weight_min": -11.717226028442383,
      "activations/layer10_attention_weight_max": 57.21454620361328,
      "activations/layer10_attention_weight_min": -49.89677810668945,
      "activations/layer11_attention_weight_max": 39.703826904296875,
      "activations/layer11_attention_weight_min": -37.3853874206543,
      "activations/layer12_attention_weight_max": 30.151264190673828,
      "activations/layer12_attention_weight_min": -29.585590362548828,
      "activations/layer13_attention_weight_max": 48.44579315185547,
      "activations/layer13_attention_weight_min": -36.0289192199707,
      "activations/layer14_attention_weight_max": 78.1207046508789,
      "activations/layer14_attention_weight_min": -56.972373962402344,
      "activations/layer15_attention_weight_max": 59.07024383544922,
      "activations/layer15_attention_weight_min": -43.41070556640625,
      "activations/layer16_attention_weight_max": 58.75354766845703,
      "activations/layer16_attention_weight_min": -38.38356018066406,
      "activations/layer17_attention_weight_max": 37.68100357055664,
      "activations/layer17_attention_weight_min": -26.911840438842773,
      "activations/layer18_attention_weight_max": 23.025047302246094,
      "activations/layer18_attention_weight_min": -14.497312545776367,
      "activations/layer19_attention_weight_max": 47.728599548339844,
      "activations/layer19_attention_weight_min": -33.418270111083984,
      "activations/layer1_attention_weight_max": 15.624224662780762,
      "activations/layer1_attention_weight_min": -12.445701599121094,
      "activations/layer20_attention_weight_max": 40.226585388183594,
      "activations/layer20_attention_weight_min": -25.591373443603516,
      "activations/layer21_attention_weight_max": 20.809284210205078,
      "activations/layer21_attention_weight_min": -14.814436912536621,
      "activations/layer22_attention_weight_max": 39.74958038330078,
      "activations/layer22_attention_weight_min": -24.830591201782227,
      "activations/layer23_attention_weight_max": 24.216711044311523,
      "activations/layer23_attention_weight_min": -15.459539413452148,
      "activations/layer2_attention_weight_max": 25.983409881591797,
      "activations/layer2_attention_weight_min": -26.547895431518555,
      "activations/layer3_attention_weight_max": 60.978275299072266,
      "activations/layer3_attention_weight_min": -66.98236846923828,
      "activations/layer4_attention_weight_max": 119.45423889160156,
      "activations/layer4_attention_weight_min": -110.7720947265625,
      "activations/layer5_attention_weight_max": 123.22212219238281,
      "activations/layer5_attention_weight_min": -108.3787841796875,
      "activations/layer6_attention_weight_max": 67.10111999511719,
      "activations/layer6_attention_weight_min": -58.15650177001953,
      "activations/layer7_attention_weight_max": 61.606788635253906,
      "activations/layer7_attention_weight_min": -58.29842758178711,
      "activations/layer8_attention_weight_max": 57.28839874267578,
      "activations/layer8_attention_weight_min": -50.43888854980469,
      "activations/layer9_attention_weight_max": 82.39433288574219,
      "activations/layer9_attention_weight_min": -66.43077850341797,
      "epoch": 1.16,
      "learning_rate": 0.0001439712121212121,
      "loss": 3.0603,
      "step": 19950
    },
    {
      "activations/layer0_attention_weight_max": 15.413003921508789,
      "activations/layer0_attention_weight_min": -11.851825714111328,
      "activations/layer10_attention_weight_max": 54.089263916015625,
      "activations/layer10_attention_weight_min": -46.880489349365234,
      "activations/layer11_attention_weight_max": 41.95378875732422,
      "activations/layer11_attention_weight_min": -35.64101028442383,
      "activations/layer12_attention_weight_max": 41.89167404174805,
      "activations/layer12_attention_weight_min": -30.906932830810547,
      "activations/layer13_attention_weight_max": 49.120643615722656,
      "activations/layer13_attention_weight_min": -36.99312210083008,
      "activations/layer14_attention_weight_max": 74.90376281738281,
      "activations/layer14_attention_weight_min": -52.542747497558594,
      "activations/layer15_attention_weight_max": 56.48689651489258,
      "activations/layer15_attention_weight_min": -39.344696044921875,
      "activations/layer16_attention_weight_max": 56.465824127197266,
      "activations/layer16_attention_weight_min": -38.338741302490234,
      "activations/layer17_attention_weight_max": 41.698734283447266,
      "activations/layer17_attention_weight_min": -26.882307052612305,
      "activations/layer18_attention_weight_max": 22.724321365356445,
      "activations/layer18_attention_weight_min": -15.46321964263916,
      "activations/layer19_attention_weight_max": 52.79386520385742,
      "activations/layer19_attention_weight_min": -30.543054580688477,
      "activations/layer1_attention_weight_max": 16.514535903930664,
      "activations/layer1_attention_weight_min": -10.869755744934082,
      "activations/layer20_attention_weight_max": 49.16790008544922,
      "activations/layer20_attention_weight_min": -24.664030075073242,
      "activations/layer21_attention_weight_max": 23.882389068603516,
      "activations/layer21_attention_weight_min": -14.67369270324707,
      "activations/layer22_attention_weight_max": 43.0438346862793,
      "activations/layer22_attention_weight_min": -23.12652587890625,
      "activations/layer23_attention_weight_max": 26.346603393554688,
      "activations/layer23_attention_weight_min": -13.589033126831055,
      "activations/layer2_attention_weight_max": 26.468448638916016,
      "activations/layer2_attention_weight_min": -27.258594512939453,
      "activations/layer3_attention_weight_max": 59.170162200927734,
      "activations/layer3_attention_weight_min": -59.309932708740234,
      "activations/layer4_attention_weight_max": 105.06239318847656,
      "activations/layer4_attention_weight_min": -99.32563781738281,
      "activations/layer5_attention_weight_max": 112.79188537597656,
      "activations/layer5_attention_weight_min": -101.13832092285156,
      "activations/layer6_attention_weight_max": 59.560699462890625,
      "activations/layer6_attention_weight_min": -56.464561462402344,
      "activations/layer7_attention_weight_max": 62.2458610534668,
      "activations/layer7_attention_weight_min": -60.69640350341797,
      "activations/layer8_attention_weight_max": 53.25587844848633,
      "activations/layer8_attention_weight_min": -51.96724319458008,
      "activations/layer9_attention_weight_max": 73.08875274658203,
      "activations/layer9_attention_weight_min": -57.17255783081055,
      "epoch": 1.16,
      "learning_rate": 0.0001439522727272727,
      "loss": 3.0661,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_loss": 3.017578125,
      "eval_runtime": 8.5143,
      "eval_samples_per_second": 504.325,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_openwebtext_loss": 3.017578125,
      "eval_openwebtext_ppl": 20.44172438427552,
      "eval_openwebtext_runtime": 8.5143,
      "eval_openwebtext_samples_per_second": 504.325,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_wikitext_loss": 3.291015625,
      "eval_wikitext_ppl": 26.87013978811538,
      "eval_wikitext_runtime": 1.9986,
      "eval_wikitext_samples_per_second": 228.164,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_lambada_loss": 3.37109375,
      "eval_lambada_ppl": 29.110349096271467,
      "eval_lambada_runtime": 9.5907,
      "eval_lambada_samples_per_second": 507.68,
      "step": 20000
    },
    {
      "activations/layer0_attention_weight_max": 15.500005722045898,
      "activations/layer0_attention_weight_min": -11.709160804748535,
      "activations/layer10_attention_weight_max": 59.73685073852539,
      "activations/layer10_attention_weight_min": -48.39812088012695,
      "activations/layer11_attention_weight_max": 56.11359786987305,
      "activations/layer11_attention_weight_min": -37.18259811401367,
      "activations/layer12_attention_weight_max": 32.91474533081055,
      "activations/layer12_attention_weight_min": -27.77623748779297,
      "activations/layer13_attention_weight_max": 40.50083541870117,
      "activations/layer13_attention_weight_min": -35.8864860534668,
      "activations/layer14_attention_weight_max": 64.36579132080078,
      "activations/layer14_attention_weight_min": -52.67021179199219,
      "activations/layer15_attention_weight_max": 49.59101104736328,
      "activations/layer15_attention_weight_min": -42.539798736572266,
      "activations/layer16_attention_weight_max": 55.66857147216797,
      "activations/layer16_attention_weight_min": -41.467655181884766,
      "activations/layer17_attention_weight_max": 42.82126998901367,
      "activations/layer17_attention_weight_min": -28.155780792236328,
      "activations/layer18_attention_weight_max": 25.880128860473633,
      "activations/layer18_attention_weight_min": -14.316908836364746,
      "activations/layer19_attention_weight_max": 53.11333084106445,
      "activations/layer19_attention_weight_min": -31.76535987854004,
      "activations/layer1_attention_weight_max": 17.641551971435547,
      "activations/layer1_attention_weight_min": -12.034991264343262,
      "activations/layer20_attention_weight_max": 44.0178337097168,
      "activations/layer20_attention_weight_min": -24.815444946289062,
      "activations/layer21_attention_weight_max": 26.067630767822266,
      "activations/layer21_attention_weight_min": -14.579383850097656,
      "activations/layer22_attention_weight_max": 42.18819046020508,
      "activations/layer22_attention_weight_min": -20.68943977355957,
      "activations/layer23_attention_weight_max": 24.198810577392578,
      "activations/layer23_attention_weight_min": -13.397143363952637,
      "activations/layer2_attention_weight_max": 26.834991455078125,
      "activations/layer2_attention_weight_min": -27.508888244628906,
      "activations/layer3_attention_weight_max": 59.352989196777344,
      "activations/layer3_attention_weight_min": -62.6927375793457,
      "activations/layer4_attention_weight_max": 113.2330322265625,
      "activations/layer4_attention_weight_min": -103.1642837524414,
      "activations/layer5_attention_weight_max": 129.84144592285156,
      "activations/layer5_attention_weight_min": -104.34840393066406,
      "activations/layer6_attention_weight_max": 71.28355407714844,
      "activations/layer6_attention_weight_min": -57.38300323486328,
      "activations/layer7_attention_weight_max": 70.66802215576172,
      "activations/layer7_attention_weight_min": -64.76921844482422,
      "activations/layer8_attention_weight_max": 68.50546264648438,
      "activations/layer8_attention_weight_min": -51.293880462646484,
      "activations/layer9_attention_weight_max": 87.1219253540039,
      "activations/layer9_attention_weight_min": -67.65603637695312,
      "epoch": 1.17,
      "learning_rate": 0.00014393333333333333,
      "loss": 3.0572,
      "step": 20050
    },
    {
      "activations/layer0_attention_weight_max": 16.423606872558594,
      "activations/layer0_attention_weight_min": -11.719833374023438,
      "activations/layer10_attention_weight_max": 61.67893600463867,
      "activations/layer10_attention_weight_min": -50.40811538696289,
      "activations/layer11_attention_weight_max": 45.490478515625,
      "activations/layer11_attention_weight_min": -40.532081604003906,
      "activations/layer12_attention_weight_max": 35.16259765625,
      "activations/layer12_attention_weight_min": -29.81639289855957,
      "activations/layer13_attention_weight_max": 51.686241149902344,
      "activations/layer13_attention_weight_min": -40.04323196411133,
      "activations/layer14_attention_weight_max": 89.4200439453125,
      "activations/layer14_attention_weight_min": -57.46052169799805,
      "activations/layer15_attention_weight_max": 63.30552673339844,
      "activations/layer15_attention_weight_min": -40.37310028076172,
      "activations/layer16_attention_weight_max": 63.35311508178711,
      "activations/layer16_attention_weight_min": -38.90901184082031,
      "activations/layer17_attention_weight_max": 41.40671920776367,
      "activations/layer17_attention_weight_min": -25.638269424438477,
      "activations/layer18_attention_weight_max": 25.379718780517578,
      "activations/layer18_attention_weight_min": -15.722817420959473,
      "activations/layer19_attention_weight_max": 58.11484909057617,
      "activations/layer19_attention_weight_min": -29.968276977539062,
      "activations/layer1_attention_weight_max": 16.27860450744629,
      "activations/layer1_attention_weight_min": -11.511045455932617,
      "activations/layer20_attention_weight_max": 47.426509857177734,
      "activations/layer20_attention_weight_min": -22.86742401123047,
      "activations/layer21_attention_weight_max": 23.33848762512207,
      "activations/layer21_attention_weight_min": -15.477508544921875,
      "activations/layer22_attention_weight_max": 42.50472640991211,
      "activations/layer22_attention_weight_min": -23.32353973388672,
      "activations/layer23_attention_weight_max": 23.289609909057617,
      "activations/layer23_attention_weight_min": -14.416717529296875,
      "activations/layer2_attention_weight_max": 26.48151397705078,
      "activations/layer2_attention_weight_min": -26.814319610595703,
      "activations/layer3_attention_weight_max": 59.69718933105469,
      "activations/layer3_attention_weight_min": -63.1950798034668,
      "activations/layer4_attention_weight_max": 104.70719146728516,
      "activations/layer4_attention_weight_min": -103.69972229003906,
      "activations/layer5_attention_weight_max": 115.87700653076172,
      "activations/layer5_attention_weight_min": -110.16934204101562,
      "activations/layer6_attention_weight_max": 67.11956787109375,
      "activations/layer6_attention_weight_min": -57.35969161987305,
      "activations/layer7_attention_weight_max": 57.99549102783203,
      "activations/layer7_attention_weight_min": -60.949310302734375,
      "activations/layer8_attention_weight_max": 58.87320327758789,
      "activations/layer8_attention_weight_min": -51.269264221191406,
      "activations/layer9_attention_weight_max": 74.94588470458984,
      "activations/layer9_attention_weight_min": -65.5987777709961,
      "epoch": 1.17,
      "learning_rate": 0.00014391439393939391,
      "loss": 3.0569,
      "step": 20100
    },
    {
      "activations/layer0_attention_weight_max": 15.39604663848877,
      "activations/layer0_attention_weight_min": -11.7855224609375,
      "activations/layer10_attention_weight_max": 54.99092483520508,
      "activations/layer10_attention_weight_min": -46.8383674621582,
      "activations/layer11_attention_weight_max": 47.84778594970703,
      "activations/layer11_attention_weight_min": -37.44129180908203,
      "activations/layer12_attention_weight_max": 36.91572189331055,
      "activations/layer12_attention_weight_min": -29.287439346313477,
      "activations/layer13_attention_weight_max": 49.699378967285156,
      "activations/layer13_attention_weight_min": -36.98757553100586,
      "activations/layer14_attention_weight_max": 74.6080551147461,
      "activations/layer14_attention_weight_min": -57.361663818359375,
      "activations/layer15_attention_weight_max": 56.9655876159668,
      "activations/layer15_attention_weight_min": -39.94473648071289,
      "activations/layer16_attention_weight_max": 47.574371337890625,
      "activations/layer16_attention_weight_min": -44.72385025024414,
      "activations/layer17_attention_weight_max": 48.33050537109375,
      "activations/layer17_attention_weight_min": -27.748336791992188,
      "activations/layer18_attention_weight_max": 22.746049880981445,
      "activations/layer18_attention_weight_min": -16.312793731689453,
      "activations/layer19_attention_weight_max": 49.70585250854492,
      "activations/layer19_attention_weight_min": -32.07395553588867,
      "activations/layer1_attention_weight_max": 17.356975555419922,
      "activations/layer1_attention_weight_min": -12.153253555297852,
      "activations/layer20_attention_weight_max": 45.9422492980957,
      "activations/layer20_attention_weight_min": -28.390308380126953,
      "activations/layer21_attention_weight_max": 25.014799118041992,
      "activations/layer21_attention_weight_min": -16.924732208251953,
      "activations/layer22_attention_weight_max": 37.22500991821289,
      "activations/layer22_attention_weight_min": -21.601346969604492,
      "activations/layer23_attention_weight_max": 23.646591186523438,
      "activations/layer23_attention_weight_min": -17.681228637695312,
      "activations/layer2_attention_weight_max": 27.46357536315918,
      "activations/layer2_attention_weight_min": -27.682109832763672,
      "activations/layer3_attention_weight_max": 60.905792236328125,
      "activations/layer3_attention_weight_min": -62.6754264831543,
      "activations/layer4_attention_weight_max": 110.32830810546875,
      "activations/layer4_attention_weight_min": -107.67952728271484,
      "activations/layer5_attention_weight_max": 120.1844482421875,
      "activations/layer5_attention_weight_min": -105.55839538574219,
      "activations/layer6_attention_weight_max": 64.86715698242188,
      "activations/layer6_attention_weight_min": -57.81879425048828,
      "activations/layer7_attention_weight_max": 66.712890625,
      "activations/layer7_attention_weight_min": -65.14836883544922,
      "activations/layer8_attention_weight_max": 58.28286361694336,
      "activations/layer8_attention_weight_min": -51.18661880493164,
      "activations/layer9_attention_weight_max": 73.51498413085938,
      "activations/layer9_attention_weight_min": -64.9421157836914,
      "epoch": 1.17,
      "learning_rate": 0.00014389545454545453,
      "loss": 3.0767,
      "step": 20150
    },
    {
      "activations/layer0_attention_weight_max": 13.756668090820312,
      "activations/layer0_attention_weight_min": -11.557273864746094,
      "activations/layer10_attention_weight_max": 57.247310638427734,
      "activations/layer10_attention_weight_min": -48.97650909423828,
      "activations/layer11_attention_weight_max": 42.28987121582031,
      "activations/layer11_attention_weight_min": -35.72507095336914,
      "activations/layer12_attention_weight_max": 33.535491943359375,
      "activations/layer12_attention_weight_min": -26.079513549804688,
      "activations/layer13_attention_weight_max": 43.79447937011719,
      "activations/layer13_attention_weight_min": -38.99906921386719,
      "activations/layer14_attention_weight_max": 76.69184112548828,
      "activations/layer14_attention_weight_min": -56.2758674621582,
      "activations/layer15_attention_weight_max": 49.33036422729492,
      "activations/layer15_attention_weight_min": -39.50362777709961,
      "activations/layer16_attention_weight_max": 50.751712799072266,
      "activations/layer16_attention_weight_min": -41.307552337646484,
      "activations/layer17_attention_weight_max": 40.45503616333008,
      "activations/layer17_attention_weight_min": -26.77316665649414,
      "activations/layer18_attention_weight_max": 24.30171012878418,
      "activations/layer18_attention_weight_min": -14.347672462463379,
      "activations/layer19_attention_weight_max": 47.339786529541016,
      "activations/layer19_attention_weight_min": -34.44830322265625,
      "activations/layer1_attention_weight_max": 17.47759246826172,
      "activations/layer1_attention_weight_min": -11.338447570800781,
      "activations/layer20_attention_weight_max": 44.64979553222656,
      "activations/layer20_attention_weight_min": -25.701492309570312,
      "activations/layer21_attention_weight_max": 22.529666900634766,
      "activations/layer21_attention_weight_min": -18.661754608154297,
      "activations/layer22_attention_weight_max": 45.758975982666016,
      "activations/layer22_attention_weight_min": -25.47264862060547,
      "activations/layer23_attention_weight_max": 24.713956832885742,
      "activations/layer23_attention_weight_min": -14.682161331176758,
      "activations/layer2_attention_weight_max": 26.616485595703125,
      "activations/layer2_attention_weight_min": -26.15546989440918,
      "activations/layer3_attention_weight_max": 60.7872200012207,
      "activations/layer3_attention_weight_min": -62.99338912963867,
      "activations/layer4_attention_weight_max": 110.2105712890625,
      "activations/layer4_attention_weight_min": -105.65450286865234,
      "activations/layer5_attention_weight_max": 117.66618347167969,
      "activations/layer5_attention_weight_min": -105.015869140625,
      "activations/layer6_attention_weight_max": 67.0368423461914,
      "activations/layer6_attention_weight_min": -58.082275390625,
      "activations/layer7_attention_weight_max": 61.79172897338867,
      "activations/layer7_attention_weight_min": -61.22844314575195,
      "activations/layer8_attention_weight_max": 54.19269943237305,
      "activations/layer8_attention_weight_min": -50.02406692504883,
      "activations/layer9_attention_weight_max": 69.33027648925781,
      "activations/layer9_attention_weight_min": -59.64469528198242,
      "epoch": 1.17,
      "learning_rate": 0.00014387651515151514,
      "loss": 3.062,
      "step": 20200
    },
    {
      "activations/layer0_attention_weight_max": 15.965425491333008,
      "activations/layer0_attention_weight_min": -11.429107666015625,
      "activations/layer10_attention_weight_max": 52.06625747680664,
      "activations/layer10_attention_weight_min": -47.03692626953125,
      "activations/layer11_attention_weight_max": 41.49018859863281,
      "activations/layer11_attention_weight_min": -36.2772331237793,
      "activations/layer12_attention_weight_max": 32.33972930908203,
      "activations/layer12_attention_weight_min": -28.20522689819336,
      "activations/layer13_attention_weight_max": 42.05499267578125,
      "activations/layer13_attention_weight_min": -36.3087272644043,
      "activations/layer14_attention_weight_max": 70.11559295654297,
      "activations/layer14_attention_weight_min": -57.03377151489258,
      "activations/layer15_attention_weight_max": 61.11035919189453,
      "activations/layer15_attention_weight_min": -47.2910270690918,
      "activations/layer16_attention_weight_max": 59.176910400390625,
      "activations/layer16_attention_weight_min": -40.257347106933594,
      "activations/layer17_attention_weight_max": 41.34883499145508,
      "activations/layer17_attention_weight_min": -25.863040924072266,
      "activations/layer18_attention_weight_max": 28.419147491455078,
      "activations/layer18_attention_weight_min": -16.171337127685547,
      "activations/layer19_attention_weight_max": 56.14801788330078,
      "activations/layer19_attention_weight_min": -33.57448196411133,
      "activations/layer1_attention_weight_max": 17.806188583374023,
      "activations/layer1_attention_weight_min": -11.832834243774414,
      "activations/layer20_attention_weight_max": 45.06840515136719,
      "activations/layer20_attention_weight_min": -25.807449340820312,
      "activations/layer21_attention_weight_max": 24.508834838867188,
      "activations/layer21_attention_weight_min": -16.924230575561523,
      "activations/layer22_attention_weight_max": 45.37894058227539,
      "activations/layer22_attention_weight_min": -23.36935043334961,
      "activations/layer23_attention_weight_max": 25.467727661132812,
      "activations/layer23_attention_weight_min": -15.134645462036133,
      "activations/layer2_attention_weight_max": 26.089397430419922,
      "activations/layer2_attention_weight_min": -26.145885467529297,
      "activations/layer3_attention_weight_max": 61.71213912963867,
      "activations/layer3_attention_weight_min": -63.813106536865234,
      "activations/layer4_attention_weight_max": 115.1120834350586,
      "activations/layer4_attention_weight_min": -114.22491455078125,
      "activations/layer5_attention_weight_max": 121.515380859375,
      "activations/layer5_attention_weight_min": -110.80571746826172,
      "activations/layer6_attention_weight_max": 64.12532806396484,
      "activations/layer6_attention_weight_min": -58.0633430480957,
      "activations/layer7_attention_weight_max": 61.6907958984375,
      "activations/layer7_attention_weight_min": -62.4189338684082,
      "activations/layer8_attention_weight_max": 54.88276672363281,
      "activations/layer8_attention_weight_min": -49.359840393066406,
      "activations/layer9_attention_weight_max": 72.10639953613281,
      "activations/layer9_attention_weight_min": -62.009456634521484,
      "epoch": 1.18,
      "learning_rate": 0.00014385757575757573,
      "loss": 3.0778,
      "step": 20250
    },
    {
      "activations/layer0_attention_weight_max": 16.544748306274414,
      "activations/layer0_attention_weight_min": -11.799192428588867,
      "activations/layer10_attention_weight_max": 53.46243667602539,
      "activations/layer10_attention_weight_min": -47.59910202026367,
      "activations/layer11_attention_weight_max": 46.32706832885742,
      "activations/layer11_attention_weight_min": -38.49466323852539,
      "activations/layer12_attention_weight_max": 41.133216857910156,
      "activations/layer12_attention_weight_min": -30.15554428100586,
      "activations/layer13_attention_weight_max": 53.63689422607422,
      "activations/layer13_attention_weight_min": -34.90320587158203,
      "activations/layer14_attention_weight_max": 79.09828186035156,
      "activations/layer14_attention_weight_min": -53.29996871948242,
      "activations/layer15_attention_weight_max": 55.59952926635742,
      "activations/layer15_attention_weight_min": -39.710933685302734,
      "activations/layer16_attention_weight_max": 55.672271728515625,
      "activations/layer16_attention_weight_min": -41.39574432373047,
      "activations/layer17_attention_weight_max": 45.6451416015625,
      "activations/layer17_attention_weight_min": -24.74519920349121,
      "activations/layer18_attention_weight_max": 25.47443199157715,
      "activations/layer18_attention_weight_min": -16.905776977539062,
      "activations/layer19_attention_weight_max": 52.670772552490234,
      "activations/layer19_attention_weight_min": -28.77996826171875,
      "activations/layer1_attention_weight_max": 16.27132797241211,
      "activations/layer1_attention_weight_min": -12.124558448791504,
      "activations/layer20_attention_weight_max": 41.133270263671875,
      "activations/layer20_attention_weight_min": -25.905439376831055,
      "activations/layer21_attention_weight_max": 28.8115291595459,
      "activations/layer21_attention_weight_min": -15.169713973999023,
      "activations/layer22_attention_weight_max": 39.82328414916992,
      "activations/layer22_attention_weight_min": -19.62297248840332,
      "activations/layer23_attention_weight_max": 24.678001403808594,
      "activations/layer23_attention_weight_min": -13.492732048034668,
      "activations/layer2_attention_weight_max": 28.01723861694336,
      "activations/layer2_attention_weight_min": -28.514760971069336,
      "activations/layer3_attention_weight_max": 59.651954650878906,
      "activations/layer3_attention_weight_min": -62.729557037353516,
      "activations/layer4_attention_weight_max": 110.87816619873047,
      "activations/layer4_attention_weight_min": -105.50788879394531,
      "activations/layer5_attention_weight_max": 117.79380798339844,
      "activations/layer5_attention_weight_min": -104.93425750732422,
      "activations/layer6_attention_weight_max": 63.975929260253906,
      "activations/layer6_attention_weight_min": -58.524112701416016,
      "activations/layer7_attention_weight_max": 60.70317459106445,
      "activations/layer7_attention_weight_min": -61.162513732910156,
      "activations/layer8_attention_weight_max": 58.93402099609375,
      "activations/layer8_attention_weight_min": -51.113285064697266,
      "activations/layer9_attention_weight_max": 79.02361297607422,
      "activations/layer9_attention_weight_min": -59.194313049316406,
      "epoch": 1.18,
      "learning_rate": 0.00014383863636363635,
      "loss": 3.0467,
      "step": 20300
    },
    {
      "activations/layer0_attention_weight_max": 16.417625427246094,
      "activations/layer0_attention_weight_min": -12.2401123046875,
      "activations/layer10_attention_weight_max": 56.33807373046875,
      "activations/layer10_attention_weight_min": -49.54216384887695,
      "activations/layer11_attention_weight_max": 46.03458786010742,
      "activations/layer11_attention_weight_min": -39.83320236206055,
      "activations/layer12_attention_weight_max": 48.524593353271484,
      "activations/layer12_attention_weight_min": -40.62089157104492,
      "activations/layer13_attention_weight_max": 51.45517349243164,
      "activations/layer13_attention_weight_min": -42.92143630981445,
      "activations/layer14_attention_weight_max": 79.18570709228516,
      "activations/layer14_attention_weight_min": -59.236019134521484,
      "activations/layer15_attention_weight_max": 59.100059509277344,
      "activations/layer15_attention_weight_min": -39.0756950378418,
      "activations/layer16_attention_weight_max": 52.52394104003906,
      "activations/layer16_attention_weight_min": -40.31900405883789,
      "activations/layer17_attention_weight_max": 39.858741760253906,
      "activations/layer17_attention_weight_min": -24.18364906311035,
      "activations/layer18_attention_weight_max": 25.93022346496582,
      "activations/layer18_attention_weight_min": -14.820462226867676,
      "activations/layer19_attention_weight_max": 51.53173828125,
      "activations/layer19_attention_weight_min": -31.241544723510742,
      "activations/layer1_attention_weight_max": 15.703410148620605,
      "activations/layer1_attention_weight_min": -12.358939170837402,
      "activations/layer20_attention_weight_max": 41.4047966003418,
      "activations/layer20_attention_weight_min": -23.998716354370117,
      "activations/layer21_attention_weight_max": 22.121231079101562,
      "activations/layer21_attention_weight_min": -15.925307273864746,
      "activations/layer22_attention_weight_max": 38.85266876220703,
      "activations/layer22_attention_weight_min": -20.582483291625977,
      "activations/layer23_attention_weight_max": 24.119190216064453,
      "activations/layer23_attention_weight_min": -14.338035583496094,
      "activations/layer2_attention_weight_max": 26.50176239013672,
      "activations/layer2_attention_weight_min": -26.00684928894043,
      "activations/layer3_attention_weight_max": 61.42148208618164,
      "activations/layer3_attention_weight_min": -61.8415412902832,
      "activations/layer4_attention_weight_max": 113.59520721435547,
      "activations/layer4_attention_weight_min": -106.5696029663086,
      "activations/layer5_attention_weight_max": 120.4809799194336,
      "activations/layer5_attention_weight_min": -109.37841796875,
      "activations/layer6_attention_weight_max": 65.51136779785156,
      "activations/layer6_attention_weight_min": -59.52667236328125,
      "activations/layer7_attention_weight_max": 61.619258880615234,
      "activations/layer7_attention_weight_min": -67.35987854003906,
      "activations/layer8_attention_weight_max": 68.9822998046875,
      "activations/layer8_attention_weight_min": -59.83575439453125,
      "activations/layer9_attention_weight_max": 79.41764831542969,
      "activations/layer9_attention_weight_min": -70.83486938476562,
      "epoch": 1.18,
      "learning_rate": 0.00014381969696969696,
      "loss": 3.058,
      "step": 20350
    },
    {
      "activations/layer0_attention_weight_max": 15.609289169311523,
      "activations/layer0_attention_weight_min": -11.289201736450195,
      "activations/layer10_attention_weight_max": 51.27165603637695,
      "activations/layer10_attention_weight_min": -50.11935806274414,
      "activations/layer11_attention_weight_max": 41.134605407714844,
      "activations/layer11_attention_weight_min": -39.66606903076172,
      "activations/layer12_attention_weight_max": 44.623077392578125,
      "activations/layer12_attention_weight_min": -35.512516021728516,
      "activations/layer13_attention_weight_max": 44.000404357910156,
      "activations/layer13_attention_weight_min": -34.65066909790039,
      "activations/layer14_attention_weight_max": 75.0846939086914,
      "activations/layer14_attention_weight_min": -56.13387680053711,
      "activations/layer15_attention_weight_max": 53.77133560180664,
      "activations/layer15_attention_weight_min": -40.96293258666992,
      "activations/layer16_attention_weight_max": 57.240970611572266,
      "activations/layer16_attention_weight_min": -43.499488830566406,
      "activations/layer17_attention_weight_max": 41.258399963378906,
      "activations/layer17_attention_weight_min": -28.48764991760254,
      "activations/layer18_attention_weight_max": 24.763792037963867,
      "activations/layer18_attention_weight_min": -15.273125648498535,
      "activations/layer19_attention_weight_max": 48.01518630981445,
      "activations/layer19_attention_weight_min": -32.87459945678711,
      "activations/layer1_attention_weight_max": 15.997941970825195,
      "activations/layer1_attention_weight_min": -11.463876724243164,
      "activations/layer20_attention_weight_max": 42.82404327392578,
      "activations/layer20_attention_weight_min": -26.963476181030273,
      "activations/layer21_attention_weight_max": 23.2922306060791,
      "activations/layer21_attention_weight_min": -15.21328067779541,
      "activations/layer22_attention_weight_max": 41.89226150512695,
      "activations/layer22_attention_weight_min": -20.243568420410156,
      "activations/layer23_attention_weight_max": 24.607074737548828,
      "activations/layer23_attention_weight_min": -16.037410736083984,
      "activations/layer2_attention_weight_max": 27.377304077148438,
      "activations/layer2_attention_weight_min": -27.459030151367188,
      "activations/layer3_attention_weight_max": 60.79182052612305,
      "activations/layer3_attention_weight_min": -64.53301239013672,
      "activations/layer4_attention_weight_max": 108.72456359863281,
      "activations/layer4_attention_weight_min": -106.94929504394531,
      "activations/layer5_attention_weight_max": 117.44647216796875,
      "activations/layer5_attention_weight_min": -109.16948699951172,
      "activations/layer6_attention_weight_max": 63.91911697387695,
      "activations/layer6_attention_weight_min": -55.876312255859375,
      "activations/layer7_attention_weight_max": 60.20147705078125,
      "activations/layer7_attention_weight_min": -62.76205062866211,
      "activations/layer8_attention_weight_max": 51.00086975097656,
      "activations/layer8_attention_weight_min": -48.76720428466797,
      "activations/layer9_attention_weight_max": 67.71007537841797,
      "activations/layer9_attention_weight_min": -63.75046920776367,
      "epoch": 1.19,
      "learning_rate": 0.00014380075757575755,
      "loss": 3.0516,
      "step": 20400
    },
    {
      "activations/layer0_attention_weight_max": 16.427417755126953,
      "activations/layer0_attention_weight_min": -10.84823226928711,
      "activations/layer10_attention_weight_max": 66.15284729003906,
      "activations/layer10_attention_weight_min": -52.2196159362793,
      "activations/layer11_attention_weight_max": 50.99578857421875,
      "activations/layer11_attention_weight_min": -34.2678337097168,
      "activations/layer12_attention_weight_max": 47.880226135253906,
      "activations/layer12_attention_weight_min": -27.790658950805664,
      "activations/layer13_attention_weight_max": 58.116668701171875,
      "activations/layer13_attention_weight_min": -39.10194396972656,
      "activations/layer14_attention_weight_max": 77.8593521118164,
      "activations/layer14_attention_weight_min": -59.760154724121094,
      "activations/layer15_attention_weight_max": 56.21792984008789,
      "activations/layer15_attention_weight_min": -36.996116638183594,
      "activations/layer16_attention_weight_max": 56.684844970703125,
      "activations/layer16_attention_weight_min": -40.627685546875,
      "activations/layer17_attention_weight_max": 42.860042572021484,
      "activations/layer17_attention_weight_min": -28.57514762878418,
      "activations/layer18_attention_weight_max": 22.085723876953125,
      "activations/layer18_attention_weight_min": -16.812286376953125,
      "activations/layer19_attention_weight_max": 48.51678466796875,
      "activations/layer19_attention_weight_min": -32.235897064208984,
      "activations/layer1_attention_weight_max": 16.62362289428711,
      "activations/layer1_attention_weight_min": -11.958837509155273,
      "activations/layer20_attention_weight_max": 41.30978012084961,
      "activations/layer20_attention_weight_min": -25.05434226989746,
      "activations/layer21_attention_weight_max": 24.769989013671875,
      "activations/layer21_attention_weight_min": -13.697383880615234,
      "activations/layer22_attention_weight_max": 47.40339660644531,
      "activations/layer22_attention_weight_min": -20.943872451782227,
      "activations/layer23_attention_weight_max": 26.275070190429688,
      "activations/layer23_attention_weight_min": -14.427801132202148,
      "activations/layer2_attention_weight_max": 27.0881404876709,
      "activations/layer2_attention_weight_min": -27.372222900390625,
      "activations/layer3_attention_weight_max": 63.36113357543945,
      "activations/layer3_attention_weight_min": -64.82324981689453,
      "activations/layer4_attention_weight_max": 115.64703369140625,
      "activations/layer4_attention_weight_min": -109.13060760498047,
      "activations/layer5_attention_weight_max": 123.89336395263672,
      "activations/layer5_attention_weight_min": -112.27275085449219,
      "activations/layer6_attention_weight_max": 66.30963134765625,
      "activations/layer6_attention_weight_min": -59.185707092285156,
      "activations/layer7_attention_weight_max": 62.02709197998047,
      "activations/layer7_attention_weight_min": -62.990478515625,
      "activations/layer8_attention_weight_max": 62.72522735595703,
      "activations/layer8_attention_weight_min": -53.20314407348633,
      "activations/layer9_attention_weight_max": 82.29039764404297,
      "activations/layer9_attention_weight_min": -66.15739440917969,
      "epoch": 1.19,
      "learning_rate": 0.00014378181818181817,
      "loss": 3.0597,
      "step": 20450
    },
    {
      "activations/layer0_attention_weight_max": 16.255691528320312,
      "activations/layer0_attention_weight_min": -10.886402130126953,
      "activations/layer10_attention_weight_max": 54.163448333740234,
      "activations/layer10_attention_weight_min": -50.287437438964844,
      "activations/layer11_attention_weight_max": 40.45329666137695,
      "activations/layer11_attention_weight_min": -36.31397247314453,
      "activations/layer12_attention_weight_max": 36.54426193237305,
      "activations/layer12_attention_weight_min": -25.91362953186035,
      "activations/layer13_attention_weight_max": 45.245967864990234,
      "activations/layer13_attention_weight_min": -36.36185836791992,
      "activations/layer14_attention_weight_max": 76.22598266601562,
      "activations/layer14_attention_weight_min": -57.21018600463867,
      "activations/layer15_attention_weight_max": 55.686180114746094,
      "activations/layer15_attention_weight_min": -44.917869567871094,
      "activations/layer16_attention_weight_max": 54.472434997558594,
      "activations/layer16_attention_weight_min": -37.177608489990234,
      "activations/layer17_attention_weight_max": 38.62015151977539,
      "activations/layer17_attention_weight_min": -27.132211685180664,
      "activations/layer18_attention_weight_max": 23.06026840209961,
      "activations/layer18_attention_weight_min": -14.839275360107422,
      "activations/layer19_attention_weight_max": 54.39158248901367,
      "activations/layer19_attention_weight_min": -30.616540908813477,
      "activations/layer1_attention_weight_max": 16.385101318359375,
      "activations/layer1_attention_weight_min": -11.176092147827148,
      "activations/layer20_attention_weight_max": 50.27980422973633,
      "activations/layer20_attention_weight_min": -27.2310848236084,
      "activations/layer21_attention_weight_max": 25.26397132873535,
      "activations/layer21_attention_weight_min": -15.94175910949707,
      "activations/layer22_attention_weight_max": 41.4674072265625,
      "activations/layer22_attention_weight_min": -29.374897003173828,
      "activations/layer23_attention_weight_max": 22.612165451049805,
      "activations/layer23_attention_weight_min": -14.345037460327148,
      "activations/layer2_attention_weight_max": 25.907264709472656,
      "activations/layer2_attention_weight_min": -25.51974105834961,
      "activations/layer3_attention_weight_max": 59.935577392578125,
      "activations/layer3_attention_weight_min": -62.52395248413086,
      "activations/layer4_attention_weight_max": 114.1977310180664,
      "activations/layer4_attention_weight_min": -103.47645568847656,
      "activations/layer5_attention_weight_max": 119.65858459472656,
      "activations/layer5_attention_weight_min": -104.12413024902344,
      "activations/layer6_attention_weight_max": 63.678009033203125,
      "activations/layer6_attention_weight_min": -57.46238708496094,
      "activations/layer7_attention_weight_max": 61.05552291870117,
      "activations/layer7_attention_weight_min": -65.34774017333984,
      "activations/layer8_attention_weight_max": 54.4412727355957,
      "activations/layer8_attention_weight_min": -48.07648849487305,
      "activations/layer9_attention_weight_max": 69.47358703613281,
      "activations/layer9_attention_weight_min": -66.68745422363281,
      "epoch": 1.19,
      "learning_rate": 0.00014376287878787878,
      "loss": 3.0478,
      "step": 20500
    },
    {
      "activations/layer0_attention_weight_max": 15.970877647399902,
      "activations/layer0_attention_weight_min": -11.388290405273438,
      "activations/layer10_attention_weight_max": 50.7368278503418,
      "activations/layer10_attention_weight_min": -46.6634635925293,
      "activations/layer11_attention_weight_max": 42.8889274597168,
      "activations/layer11_attention_weight_min": -38.04970169067383,
      "activations/layer12_attention_weight_max": 39.656471252441406,
      "activations/layer12_attention_weight_min": -28.375015258789062,
      "activations/layer13_attention_weight_max": 55.15903091430664,
      "activations/layer13_attention_weight_min": -37.0396842956543,
      "activations/layer14_attention_weight_max": 71.45380401611328,
      "activations/layer14_attention_weight_min": -58.06819152832031,
      "activations/layer15_attention_weight_max": 53.059234619140625,
      "activations/layer15_attention_weight_min": -37.9201545715332,
      "activations/layer16_attention_weight_max": 53.201717376708984,
      "activations/layer16_attention_weight_min": -37.90972137451172,
      "activations/layer17_attention_weight_max": 42.63199996948242,
      "activations/layer17_attention_weight_min": -30.57940673828125,
      "activations/layer18_attention_weight_max": 22.48472785949707,
      "activations/layer18_attention_weight_min": -15.030789375305176,
      "activations/layer19_attention_weight_max": 50.96775817871094,
      "activations/layer19_attention_weight_min": -31.289701461791992,
      "activations/layer1_attention_weight_max": 16.44270896911621,
      "activations/layer1_attention_weight_min": -12.835474967956543,
      "activations/layer20_attention_weight_max": 46.668914794921875,
      "activations/layer20_attention_weight_min": -25.775278091430664,
      "activations/layer21_attention_weight_max": 26.22435188293457,
      "activations/layer21_attention_weight_min": -17.252504348754883,
      "activations/layer22_attention_weight_max": 43.58329391479492,
      "activations/layer22_attention_weight_min": -24.788978576660156,
      "activations/layer23_attention_weight_max": 23.901721954345703,
      "activations/layer23_attention_weight_min": -15.0412015914917,
      "activations/layer2_attention_weight_max": 26.291759490966797,
      "activations/layer2_attention_weight_min": -28.194738388061523,
      "activations/layer3_attention_weight_max": 60.557228088378906,
      "activations/layer3_attention_weight_min": -66.09408569335938,
      "activations/layer4_attention_weight_max": 114.04467010498047,
      "activations/layer4_attention_weight_min": -108.650390625,
      "activations/layer5_attention_weight_max": 121.33465576171875,
      "activations/layer5_attention_weight_min": -108.32518768310547,
      "activations/layer6_attention_weight_max": 62.32280349731445,
      "activations/layer6_attention_weight_min": -58.75028991699219,
      "activations/layer7_attention_weight_max": 61.75774002075195,
      "activations/layer7_attention_weight_min": -59.475704193115234,
      "activations/layer8_attention_weight_max": 53.403377532958984,
      "activations/layer8_attention_weight_min": -50.00518798828125,
      "activations/layer9_attention_weight_max": 82.0472412109375,
      "activations/layer9_attention_weight_min": -68.28836822509766,
      "epoch": 1.19,
      "learning_rate": 0.0001437439393939394,
      "loss": 3.0535,
      "step": 20550
    },
    {
      "activations/layer0_attention_weight_max": 16.430761337280273,
      "activations/layer0_attention_weight_min": -11.073732376098633,
      "activations/layer10_attention_weight_max": 57.01495361328125,
      "activations/layer10_attention_weight_min": -48.96357345581055,
      "activations/layer11_attention_weight_max": 41.10918426513672,
      "activations/layer11_attention_weight_min": -36.6219482421875,
      "activations/layer12_attention_weight_max": 43.76506042480469,
      "activations/layer12_attention_weight_min": -30.787721633911133,
      "activations/layer13_attention_weight_max": 58.7526741027832,
      "activations/layer13_attention_weight_min": -40.08192825317383,
      "activations/layer14_attention_weight_max": 73.51179504394531,
      "activations/layer14_attention_weight_min": -58.20996856689453,
      "activations/layer15_attention_weight_max": 53.50692367553711,
      "activations/layer15_attention_weight_min": -40.74128341674805,
      "activations/layer16_attention_weight_max": 54.7196159362793,
      "activations/layer16_attention_weight_min": -38.50971221923828,
      "activations/layer17_attention_weight_max": 35.67669677734375,
      "activations/layer17_attention_weight_min": -28.17616081237793,
      "activations/layer18_attention_weight_max": 23.450286865234375,
      "activations/layer18_attention_weight_min": -16.984556198120117,
      "activations/layer19_attention_weight_max": 42.529720306396484,
      "activations/layer19_attention_weight_min": -34.11735153198242,
      "activations/layer1_attention_weight_max": 17.355215072631836,
      "activations/layer1_attention_weight_min": -11.652812957763672,
      "activations/layer20_attention_weight_max": 39.054874420166016,
      "activations/layer20_attention_weight_min": -27.411779403686523,
      "activations/layer21_attention_weight_max": 20.21013641357422,
      "activations/layer21_attention_weight_min": -14.435117721557617,
      "activations/layer22_attention_weight_max": 41.12958526611328,
      "activations/layer22_attention_weight_min": -25.98270606994629,
      "activations/layer23_attention_weight_max": 25.227569580078125,
      "activations/layer23_attention_weight_min": -16.9905948638916,
      "activations/layer2_attention_weight_max": 29.69029998779297,
      "activations/layer2_attention_weight_min": -28.475502014160156,
      "activations/layer3_attention_weight_max": 62.033470153808594,
      "activations/layer3_attention_weight_min": -64.80221557617188,
      "activations/layer4_attention_weight_max": 111.2631607055664,
      "activations/layer4_attention_weight_min": -106.29493713378906,
      "activations/layer5_attention_weight_max": 121.65615844726562,
      "activations/layer5_attention_weight_min": -105.13880920410156,
      "activations/layer6_attention_weight_max": 65.92548370361328,
      "activations/layer6_attention_weight_min": -54.701290130615234,
      "activations/layer7_attention_weight_max": 67.95511627197266,
      "activations/layer7_attention_weight_min": -60.503875732421875,
      "activations/layer8_attention_weight_max": 56.8426628112793,
      "activations/layer8_attention_weight_min": -48.760379791259766,
      "activations/layer9_attention_weight_max": 73.9854736328125,
      "activations/layer9_attention_weight_min": -65.37874603271484,
      "epoch": 1.2,
      "learning_rate": 0.00014372499999999998,
      "loss": 3.0541,
      "step": 20600
    },
    {
      "activations/layer0_attention_weight_max": 16.326717376708984,
      "activations/layer0_attention_weight_min": -11.44901180267334,
      "activations/layer10_attention_weight_max": 50.37470626831055,
      "activations/layer10_attention_weight_min": -50.944305419921875,
      "activations/layer11_attention_weight_max": 39.27238082885742,
      "activations/layer11_attention_weight_min": -35.66240692138672,
      "activations/layer12_attention_weight_max": 38.13921356201172,
      "activations/layer12_attention_weight_min": -32.477447509765625,
      "activations/layer13_attention_weight_max": 43.97999572753906,
      "activations/layer13_attention_weight_min": -33.51993942260742,
      "activations/layer14_attention_weight_max": 68.08113861083984,
      "activations/layer14_attention_weight_min": -56.135799407958984,
      "activations/layer15_attention_weight_max": 56.031883239746094,
      "activations/layer15_attention_weight_min": -41.00392532348633,
      "activations/layer16_attention_weight_max": 52.42559051513672,
      "activations/layer16_attention_weight_min": -39.63611602783203,
      "activations/layer17_attention_weight_max": 39.26744079589844,
      "activations/layer17_attention_weight_min": -31.862491607666016,
      "activations/layer18_attention_weight_max": 24.57265853881836,
      "activations/layer18_attention_weight_min": -14.865242958068848,
      "activations/layer19_attention_weight_max": 53.77558135986328,
      "activations/layer19_attention_weight_min": -35.58042526245117,
      "activations/layer1_attention_weight_max": 17.68329429626465,
      "activations/layer1_attention_weight_min": -11.724844932556152,
      "activations/layer20_attention_weight_max": 39.999656677246094,
      "activations/layer20_attention_weight_min": -34.19858169555664,
      "activations/layer21_attention_weight_max": 24.938486099243164,
      "activations/layer21_attention_weight_min": -14.963560104370117,
      "activations/layer22_attention_weight_max": 36.70407485961914,
      "activations/layer22_attention_weight_min": -23.080781936645508,
      "activations/layer23_attention_weight_max": 25.14260482788086,
      "activations/layer23_attention_weight_min": -15.951338768005371,
      "activations/layer2_attention_weight_max": 27.166614532470703,
      "activations/layer2_attention_weight_min": -27.5255184173584,
      "activations/layer3_attention_weight_max": 61.355892181396484,
      "activations/layer3_attention_weight_min": -64.48526000976562,
      "activations/layer4_attention_weight_max": 109.4894027709961,
      "activations/layer4_attention_weight_min": -107.98892974853516,
      "activations/layer5_attention_weight_max": 116.72637176513672,
      "activations/layer5_attention_weight_min": -109.28166961669922,
      "activations/layer6_attention_weight_max": 67.84349822998047,
      "activations/layer6_attention_weight_min": -60.09779357910156,
      "activations/layer7_attention_weight_max": 58.07497787475586,
      "activations/layer7_attention_weight_min": -58.01298904418945,
      "activations/layer8_attention_weight_max": 55.59669494628906,
      "activations/layer8_attention_weight_min": -52.33600616455078,
      "activations/layer9_attention_weight_max": 68.44817352294922,
      "activations/layer9_attention_weight_min": -63.33402633666992,
      "epoch": 1.2,
      "learning_rate": 0.0001437060606060606,
      "loss": 3.0578,
      "step": 20650
    },
    {
      "activations/layer0_attention_weight_max": 16.518413543701172,
      "activations/layer0_attention_weight_min": -11.349874496459961,
      "activations/layer10_attention_weight_max": 60.03683853149414,
      "activations/layer10_attention_weight_min": -49.986717224121094,
      "activations/layer11_attention_weight_max": 49.0166015625,
      "activations/layer11_attention_weight_min": -35.60417938232422,
      "activations/layer12_attention_weight_max": 53.33417510986328,
      "activations/layer12_attention_weight_min": -33.675628662109375,
      "activations/layer13_attention_weight_max": 60.12602233886719,
      "activations/layer13_attention_weight_min": -41.81471633911133,
      "activations/layer14_attention_weight_max": 83.82673645019531,
      "activations/layer14_attention_weight_min": -58.80445098876953,
      "activations/layer15_attention_weight_max": 53.721221923828125,
      "activations/layer15_attention_weight_min": -41.888057708740234,
      "activations/layer16_attention_weight_max": 52.51047134399414,
      "activations/layer16_attention_weight_min": -38.878578186035156,
      "activations/layer17_attention_weight_max": 31.772701263427734,
      "activations/layer17_attention_weight_min": -27.015836715698242,
      "activations/layer18_attention_weight_max": 23.572118759155273,
      "activations/layer18_attention_weight_min": -15.552750587463379,
      "activations/layer19_attention_weight_max": 46.0860710144043,
      "activations/layer19_attention_weight_min": -33.94066619873047,
      "activations/layer1_attention_weight_max": 16.6387882232666,
      "activations/layer1_attention_weight_min": -12.82530689239502,
      "activations/layer20_attention_weight_max": 34.841819763183594,
      "activations/layer20_attention_weight_min": -24.983659744262695,
      "activations/layer21_attention_weight_max": 20.435659408569336,
      "activations/layer21_attention_weight_min": -14.949065208435059,
      "activations/layer22_attention_weight_max": 35.582942962646484,
      "activations/layer22_attention_weight_min": -23.3635196685791,
      "activations/layer23_attention_weight_max": 22.771352767944336,
      "activations/layer23_attention_weight_min": -14.955348014831543,
      "activations/layer2_attention_weight_max": 27.928367614746094,
      "activations/layer2_attention_weight_min": -26.574811935424805,
      "activations/layer3_attention_weight_max": 64.39140319824219,
      "activations/layer3_attention_weight_min": -66.64325714111328,
      "activations/layer4_attention_weight_max": 117.67900848388672,
      "activations/layer4_attention_weight_min": -113.99959564208984,
      "activations/layer5_attention_weight_max": 127.04092407226562,
      "activations/layer5_attention_weight_min": -109.3642578125,
      "activations/layer6_attention_weight_max": 68.12669372558594,
      "activations/layer6_attention_weight_min": -58.75093460083008,
      "activations/layer7_attention_weight_max": 65.77241516113281,
      "activations/layer7_attention_weight_min": -59.16810989379883,
      "activations/layer8_attention_weight_max": 65.70677947998047,
      "activations/layer8_attention_weight_min": -50.96342849731445,
      "activations/layer9_attention_weight_max": 85.96092224121094,
      "activations/layer9_attention_weight_min": -69.15067291259766,
      "epoch": 1.2,
      "learning_rate": 0.0001436871212121212,
      "loss": 3.0519,
      "step": 20700
    },
    {
      "activations/layer0_attention_weight_max": 16.536273956298828,
      "activations/layer0_attention_weight_min": -12.41132640838623,
      "activations/layer10_attention_weight_max": 54.204734802246094,
      "activations/layer10_attention_weight_min": -48.5625114440918,
      "activations/layer11_attention_weight_max": 40.98868179321289,
      "activations/layer11_attention_weight_min": -38.48603820800781,
      "activations/layer12_attention_weight_max": 36.645503997802734,
      "activations/layer12_attention_weight_min": -31.093753814697266,
      "activations/layer13_attention_weight_max": 49.44325256347656,
      "activations/layer13_attention_weight_min": -40.03779220581055,
      "activations/layer14_attention_weight_max": 67.70848846435547,
      "activations/layer14_attention_weight_min": -60.161399841308594,
      "activations/layer15_attention_weight_max": 51.62672805786133,
      "activations/layer15_attention_weight_min": -39.935543060302734,
      "activations/layer16_attention_weight_max": 54.25653076171875,
      "activations/layer16_attention_weight_min": -41.642066955566406,
      "activations/layer17_attention_weight_max": 43.50484085083008,
      "activations/layer17_attention_weight_min": -29.38245964050293,
      "activations/layer18_attention_weight_max": 22.720903396606445,
      "activations/layer18_attention_weight_min": -14.470364570617676,
      "activations/layer19_attention_weight_max": 52.36280059814453,
      "activations/layer19_attention_weight_min": -35.427738189697266,
      "activations/layer1_attention_weight_max": 17.198745727539062,
      "activations/layer1_attention_weight_min": -11.776399612426758,
      "activations/layer20_attention_weight_max": 43.862464904785156,
      "activations/layer20_attention_weight_min": -26.641273498535156,
      "activations/layer21_attention_weight_max": 25.075590133666992,
      "activations/layer21_attention_weight_min": -14.406654357910156,
      "activations/layer22_attention_weight_max": 41.534732818603516,
      "activations/layer22_attention_weight_min": -24.07074737548828,
      "activations/layer23_attention_weight_max": 26.633888244628906,
      "activations/layer23_attention_weight_min": -17.433048248291016,
      "activations/layer2_attention_weight_max": 26.235403060913086,
      "activations/layer2_attention_weight_min": -24.4163818359375,
      "activations/layer3_attention_weight_max": 62.371315002441406,
      "activations/layer3_attention_weight_min": -64.77143096923828,
      "activations/layer4_attention_weight_max": 112.74494934082031,
      "activations/layer4_attention_weight_min": -110.09779357910156,
      "activations/layer5_attention_weight_max": 115.02716064453125,
      "activations/layer5_attention_weight_min": -102.37492370605469,
      "activations/layer6_attention_weight_max": 64.25704956054688,
      "activations/layer6_attention_weight_min": -54.63414764404297,
      "activations/layer7_attention_weight_max": 60.451133728027344,
      "activations/layer7_attention_weight_min": -59.96261978149414,
      "activations/layer8_attention_weight_max": 52.262149810791016,
      "activations/layer8_attention_weight_min": -47.319156646728516,
      "activations/layer9_attention_weight_max": 65.1491928100586,
      "activations/layer9_attention_weight_min": -58.10708236694336,
      "epoch": 1.21,
      "learning_rate": 0.0001436681818181818,
      "loss": 3.0649,
      "step": 20750
    },
    {
      "activations/layer0_attention_weight_max": 16.48872947692871,
      "activations/layer0_attention_weight_min": -11.897595405578613,
      "activations/layer10_attention_weight_max": 53.36888122558594,
      "activations/layer10_attention_weight_min": -49.19340896606445,
      "activations/layer11_attention_weight_max": 42.090232849121094,
      "activations/layer11_attention_weight_min": -37.25178146362305,
      "activations/layer12_attention_weight_max": 48.55270767211914,
      "activations/layer12_attention_weight_min": -41.82732391357422,
      "activations/layer13_attention_weight_max": 48.83073806762695,
      "activations/layer13_attention_weight_min": -39.5224723815918,
      "activations/layer14_attention_weight_max": 73.2236099243164,
      "activations/layer14_attention_weight_min": -52.266178131103516,
      "activations/layer15_attention_weight_max": 52.036617279052734,
      "activations/layer15_attention_weight_min": -37.4661979675293,
      "activations/layer16_attention_weight_max": 57.841243743896484,
      "activations/layer16_attention_weight_min": -36.783077239990234,
      "activations/layer17_attention_weight_max": 46.27955627441406,
      "activations/layer17_attention_weight_min": -26.421207427978516,
      "activations/layer18_attention_weight_max": 29.462419509887695,
      "activations/layer18_attention_weight_min": -15.232501029968262,
      "activations/layer19_attention_weight_max": 54.26770782470703,
      "activations/layer19_attention_weight_min": -31.761886596679688,
      "activations/layer1_attention_weight_max": 16.40277671813965,
      "activations/layer1_attention_weight_min": -11.49903678894043,
      "activations/layer20_attention_weight_max": 46.873138427734375,
      "activations/layer20_attention_weight_min": -23.717885971069336,
      "activations/layer21_attention_weight_max": 25.211593627929688,
      "activations/layer21_attention_weight_min": -13.786251068115234,
      "activations/layer22_attention_weight_max": 44.866600036621094,
      "activations/layer22_attention_weight_min": -22.361522674560547,
      "activations/layer23_attention_weight_max": 27.516284942626953,
      "activations/layer23_attention_weight_min": -15.719093322753906,
      "activations/layer2_attention_weight_max": 26.64875602722168,
      "activations/layer2_attention_weight_min": -26.602359771728516,
      "activations/layer3_attention_weight_max": 60.35780715942383,
      "activations/layer3_attention_weight_min": -64.19495391845703,
      "activations/layer4_attention_weight_max": 108.8393783569336,
      "activations/layer4_attention_weight_min": -112.961181640625,
      "activations/layer5_attention_weight_max": 114.39985656738281,
      "activations/layer5_attention_weight_min": -108.49830627441406,
      "activations/layer6_attention_weight_max": 62.05046463012695,
      "activations/layer6_attention_weight_min": -59.30182647705078,
      "activations/layer7_attention_weight_max": 59.567832946777344,
      "activations/layer7_attention_weight_min": -61.094852447509766,
      "activations/layer8_attention_weight_max": 56.383872985839844,
      "activations/layer8_attention_weight_min": -50.58671569824219,
      "activations/layer9_attention_weight_max": 73.29906463623047,
      "activations/layer9_attention_weight_min": -66.19820404052734,
      "epoch": 1.21,
      "learning_rate": 0.00014364924242424242,
      "loss": 3.0491,
      "step": 20800
    },
    {
      "activations/layer0_attention_weight_max": 13.378242492675781,
      "activations/layer0_attention_weight_min": -11.79614543914795,
      "activations/layer10_attention_weight_max": 58.932308197021484,
      "activations/layer10_attention_weight_min": -47.56660079956055,
      "activations/layer11_attention_weight_max": 42.94964599609375,
      "activations/layer11_attention_weight_min": -34.0017204284668,
      "activations/layer12_attention_weight_max": 67.90374755859375,
      "activations/layer12_attention_weight_min": -40.01455307006836,
      "activations/layer13_attention_weight_max": 64.50045013427734,
      "activations/layer13_attention_weight_min": -49.839054107666016,
      "activations/layer14_attention_weight_max": 74.82484436035156,
      "activations/layer14_attention_weight_min": -55.29895782470703,
      "activations/layer15_attention_weight_max": 54.12095260620117,
      "activations/layer15_attention_weight_min": -39.308353424072266,
      "activations/layer16_attention_weight_max": 56.31679153442383,
      "activations/layer16_attention_weight_min": -40.17950439453125,
      "activations/layer17_attention_weight_max": 45.37038803100586,
      "activations/layer17_attention_weight_min": -28.029367446899414,
      "activations/layer18_attention_weight_max": 25.117624282836914,
      "activations/layer18_attention_weight_min": -16.862926483154297,
      "activations/layer19_attention_weight_max": 54.89971923828125,
      "activations/layer19_attention_weight_min": -32.03557205200195,
      "activations/layer1_attention_weight_max": 16.838224411010742,
      "activations/layer1_attention_weight_min": -11.740544319152832,
      "activations/layer20_attention_weight_max": 44.762508392333984,
      "activations/layer20_attention_weight_min": -22.90283966064453,
      "activations/layer21_attention_weight_max": 24.972667694091797,
      "activations/layer21_attention_weight_min": -14.137904167175293,
      "activations/layer22_attention_weight_max": 48.67062759399414,
      "activations/layer22_attention_weight_min": -23.60966682434082,
      "activations/layer23_attention_weight_max": 26.18802261352539,
      "activations/layer23_attention_weight_min": -15.232694625854492,
      "activations/layer2_attention_weight_max": 28.215042114257812,
      "activations/layer2_attention_weight_min": -27.306278228759766,
      "activations/layer3_attention_weight_max": 63.09645080566406,
      "activations/layer3_attention_weight_min": -66.82078552246094,
      "activations/layer4_attention_weight_max": 114.11711883544922,
      "activations/layer4_attention_weight_min": -107.76634979248047,
      "activations/layer5_attention_weight_max": 120.29853820800781,
      "activations/layer5_attention_weight_min": -108.37450408935547,
      "activations/layer6_attention_weight_max": 64.81364440917969,
      "activations/layer6_attention_weight_min": -55.76076126098633,
      "activations/layer7_attention_weight_max": 63.6643180847168,
      "activations/layer7_attention_weight_min": -69.391845703125,
      "activations/layer8_attention_weight_max": 55.999061584472656,
      "activations/layer8_attention_weight_min": -49.37522506713867,
      "activations/layer9_attention_weight_max": 70.54656982421875,
      "activations/layer9_attention_weight_min": -60.87009811401367,
      "epoch": 1.21,
      "learning_rate": 0.00014363030303030303,
      "loss": 3.0724,
      "step": 20850
    },
    {
      "activations/layer0_attention_weight_max": 16.172273635864258,
      "activations/layer0_attention_weight_min": -11.818449020385742,
      "activations/layer10_attention_weight_max": 57.270076751708984,
      "activations/layer10_attention_weight_min": -51.02736282348633,
      "activations/layer11_attention_weight_max": 51.40143966674805,
      "activations/layer11_attention_weight_min": -44.81865692138672,
      "activations/layer12_attention_weight_max": 42.78360366821289,
      "activations/layer12_attention_weight_min": -26.269065856933594,
      "activations/layer13_attention_weight_max": 63.34831619262695,
      "activations/layer13_attention_weight_min": -38.90407180786133,
      "activations/layer14_attention_weight_max": 90.60214233398438,
      "activations/layer14_attention_weight_min": -59.118446350097656,
      "activations/layer15_attention_weight_max": 56.678550720214844,
      "activations/layer15_attention_weight_min": -37.19597244262695,
      "activations/layer16_attention_weight_max": 70.72943115234375,
      "activations/layer16_attention_weight_min": -48.5920524597168,
      "activations/layer17_attention_weight_max": 44.463924407958984,
      "activations/layer17_attention_weight_min": -26.7355899810791,
      "activations/layer18_attention_weight_max": 30.060874938964844,
      "activations/layer18_attention_weight_min": -16.9694766998291,
      "activations/layer19_attention_weight_max": 61.45559310913086,
      "activations/layer19_attention_weight_min": -32.98301315307617,
      "activations/layer1_attention_weight_max": 16.614837646484375,
      "activations/layer1_attention_weight_min": -12.112417221069336,
      "activations/layer20_attention_weight_max": 56.920654296875,
      "activations/layer20_attention_weight_min": -26.34054183959961,
      "activations/layer21_attention_weight_max": 30.476871490478516,
      "activations/layer21_attention_weight_min": -18.59269905090332,
      "activations/layer22_attention_weight_max": 43.47401428222656,
      "activations/layer22_attention_weight_min": -26.009965896606445,
      "activations/layer23_attention_weight_max": 25.033546447753906,
      "activations/layer23_attention_weight_min": -15.226218223571777,
      "activations/layer2_attention_weight_max": 27.22859001159668,
      "activations/layer2_attention_weight_min": -27.236217498779297,
      "activations/layer3_attention_weight_max": 65.37039947509766,
      "activations/layer3_attention_weight_min": -67.4312515258789,
      "activations/layer4_attention_weight_max": 115.86573791503906,
      "activations/layer4_attention_weight_min": -111.44813537597656,
      "activations/layer5_attention_weight_max": 124.71065521240234,
      "activations/layer5_attention_weight_min": -107.31674194335938,
      "activations/layer6_attention_weight_max": 64.28731536865234,
      "activations/layer6_attention_weight_min": -56.67970275878906,
      "activations/layer7_attention_weight_max": 65.74473571777344,
      "activations/layer7_attention_weight_min": -63.95940017700195,
      "activations/layer8_attention_weight_max": 54.775115966796875,
      "activations/layer8_attention_weight_min": -46.973628997802734,
      "activations/layer9_attention_weight_max": 70.13491821289062,
      "activations/layer9_attention_weight_min": -68.23021697998047,
      "epoch": 1.21,
      "learning_rate": 0.00014361136363636362,
      "loss": 3.073,
      "step": 20900
    },
    {
      "activations/layer0_attention_weight_max": 16.228818893432617,
      "activations/layer0_attention_weight_min": -11.588136672973633,
      "activations/layer10_attention_weight_max": 60.68960952758789,
      "activations/layer10_attention_weight_min": -53.790645599365234,
      "activations/layer11_attention_weight_max": 49.553382873535156,
      "activations/layer11_attention_weight_min": -41.82155227661133,
      "activations/layer12_attention_weight_max": 39.77249526977539,
      "activations/layer12_attention_weight_min": -33.63117218017578,
      "activations/layer13_attention_weight_max": 45.34565734863281,
      "activations/layer13_attention_weight_min": -38.48223876953125,
      "activations/layer14_attention_weight_max": 66.60124969482422,
      "activations/layer14_attention_weight_min": -59.2784538269043,
      "activations/layer15_attention_weight_max": 52.1403694152832,
      "activations/layer15_attention_weight_min": -37.66359329223633,
      "activations/layer16_attention_weight_max": 54.59632873535156,
      "activations/layer16_attention_weight_min": -41.30198287963867,
      "activations/layer17_attention_weight_max": 43.22574996948242,
      "activations/layer17_attention_weight_min": -28.055709838867188,
      "activations/layer18_attention_weight_max": 23.71297836303711,
      "activations/layer18_attention_weight_min": -15.650439262390137,
      "activations/layer19_attention_weight_max": 42.465152740478516,
      "activations/layer19_attention_weight_min": -33.4322624206543,
      "activations/layer1_attention_weight_max": 17.347803115844727,
      "activations/layer1_attention_weight_min": -11.87354850769043,
      "activations/layer20_attention_weight_max": 37.92082977294922,
      "activations/layer20_attention_weight_min": -26.555736541748047,
      "activations/layer21_attention_weight_max": 26.81439208984375,
      "activations/layer21_attention_weight_min": -17.296525955200195,
      "activations/layer22_attention_weight_max": 40.535423278808594,
      "activations/layer22_attention_weight_min": -22.029857635498047,
      "activations/layer23_attention_weight_max": 24.357664108276367,
      "activations/layer23_attention_weight_min": -14.115970611572266,
      "activations/layer2_attention_weight_max": 28.21417236328125,
      "activations/layer2_attention_weight_min": -26.859731674194336,
      "activations/layer3_attention_weight_max": 64.1101303100586,
      "activations/layer3_attention_weight_min": -68.6380386352539,
      "activations/layer4_attention_weight_max": 116.82293701171875,
      "activations/layer4_attention_weight_min": -118.33673095703125,
      "activations/layer5_attention_weight_max": 120.66853332519531,
      "activations/layer5_attention_weight_min": -116.34447479248047,
      "activations/layer6_attention_weight_max": 63.582847595214844,
      "activations/layer6_attention_weight_min": -58.746368408203125,
      "activations/layer7_attention_weight_max": 63.153968811035156,
      "activations/layer7_attention_weight_min": -61.481475830078125,
      "activations/layer8_attention_weight_max": 61.9355354309082,
      "activations/layer8_attention_weight_min": -54.487335205078125,
      "activations/layer9_attention_weight_max": 85.7474136352539,
      "activations/layer9_attention_weight_min": -67.1875228881836,
      "epoch": 1.22,
      "learning_rate": 0.00014359242424242423,
      "loss": 3.0485,
      "step": 20950
    },
    {
      "activations/layer0_attention_weight_max": 16.370819091796875,
      "activations/layer0_attention_weight_min": -12.008983612060547,
      "activations/layer10_attention_weight_max": 61.10936737060547,
      "activations/layer10_attention_weight_min": -53.93427276611328,
      "activations/layer11_attention_weight_max": 48.88861846923828,
      "activations/layer11_attention_weight_min": -36.86872100830078,
      "activations/layer12_attention_weight_max": 93.17263793945312,
      "activations/layer12_attention_weight_min": -45.931373596191406,
      "activations/layer13_attention_weight_max": 73.50154113769531,
      "activations/layer13_attention_weight_min": -45.30356216430664,
      "activations/layer14_attention_weight_max": 87.15443420410156,
      "activations/layer14_attention_weight_min": -63.389644622802734,
      "activations/layer15_attention_weight_max": 58.46329116821289,
      "activations/layer15_attention_weight_min": -43.28352737426758,
      "activations/layer16_attention_weight_max": 53.97304153442383,
      "activations/layer16_attention_weight_min": -40.65155792236328,
      "activations/layer17_attention_weight_max": 43.74386215209961,
      "activations/layer17_attention_weight_min": -26.473466873168945,
      "activations/layer18_attention_weight_max": 28.073753356933594,
      "activations/layer18_attention_weight_min": -17.673709869384766,
      "activations/layer19_attention_weight_max": 51.098575592041016,
      "activations/layer19_attention_weight_min": -34.22977066040039,
      "activations/layer1_attention_weight_max": 16.198373794555664,
      "activations/layer1_attention_weight_min": -12.688650131225586,
      "activations/layer20_attention_weight_max": 44.33555603027344,
      "activations/layer20_attention_weight_min": -26.956687927246094,
      "activations/layer21_attention_weight_max": 23.40675926208496,
      "activations/layer21_attention_weight_min": -15.327498435974121,
      "activations/layer22_attention_weight_max": 44.023868560791016,
      "activations/layer22_attention_weight_min": -23.13347053527832,
      "activations/layer23_attention_weight_max": 25.40704917907715,
      "activations/layer23_attention_weight_min": -13.771366119384766,
      "activations/layer2_attention_weight_max": 27.66236114501953,
      "activations/layer2_attention_weight_min": -27.698837280273438,
      "activations/layer3_attention_weight_max": 63.5435905456543,
      "activations/layer3_attention_weight_min": -67.04671478271484,
      "activations/layer4_attention_weight_max": 112.8823013305664,
      "activations/layer4_attention_weight_min": -110.01570129394531,
      "activations/layer5_attention_weight_max": 123.22106170654297,
      "activations/layer5_attention_weight_min": -108.62190246582031,
      "activations/layer6_attention_weight_max": 67.635986328125,
      "activations/layer6_attention_weight_min": -61.15580749511719,
      "activations/layer7_attention_weight_max": 72.3230972290039,
      "activations/layer7_attention_weight_min": -63.112037658691406,
      "activations/layer8_attention_weight_max": 61.40461730957031,
      "activations/layer8_attention_weight_min": -55.500282287597656,
      "activations/layer9_attention_weight_max": 78.13372039794922,
      "activations/layer9_attention_weight_min": -66.27015686035156,
      "epoch": 1.22,
      "learning_rate": 0.00014357348484848485,
      "loss": 3.0657,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_loss": 3.005859375,
      "eval_runtime": 8.4994,
      "eval_samples_per_second": 505.212,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_openwebtext_loss": 3.005859375,
      "eval_openwebtext_ppl": 20.203571081597193,
      "eval_openwebtext_runtime": 8.4994,
      "eval_openwebtext_samples_per_second": 505.212,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_wikitext_loss": 3.283203125,
      "eval_wikitext_ppl": 26.66103470132838,
      "eval_wikitext_runtime": 2.0209,
      "eval_wikitext_samples_per_second": 225.642,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_lambada_loss": 3.39453125,
      "eval_lambada_ppl": 29.800681131145613,
      "eval_lambada_runtime": 9.551,
      "eval_lambada_samples_per_second": 509.792,
      "step": 21000
    },
    {
      "activations/layer0_attention_weight_max": 16.363971710205078,
      "activations/layer0_attention_weight_min": -11.925219535827637,
      "activations/layer10_attention_weight_max": 62.551788330078125,
      "activations/layer10_attention_weight_min": -46.43913650512695,
      "activations/layer11_attention_weight_max": 48.16284942626953,
      "activations/layer11_attention_weight_min": -35.82025146484375,
      "activations/layer12_attention_weight_max": 91.21406555175781,
      "activations/layer12_attention_weight_min": -47.11426544189453,
      "activations/layer13_attention_weight_max": 100.1347427368164,
      "activations/layer13_attention_weight_min": -56.97971725463867,
      "activations/layer14_attention_weight_max": 92.45217895507812,
      "activations/layer14_attention_weight_min": -60.15545654296875,
      "activations/layer15_attention_weight_max": 56.430973052978516,
      "activations/layer15_attention_weight_min": -37.64243698120117,
      "activations/layer16_attention_weight_max": 58.302066802978516,
      "activations/layer16_attention_weight_min": -39.2867431640625,
      "activations/layer17_attention_weight_max": 42.38090515136719,
      "activations/layer17_attention_weight_min": -27.69090461730957,
      "activations/layer18_attention_weight_max": 27.488525390625,
      "activations/layer18_attention_weight_min": -18.2086238861084,
      "activations/layer19_attention_weight_max": 60.014259338378906,
      "activations/layer19_attention_weight_min": -36.73685073852539,
      "activations/layer1_attention_weight_max": 17.44857406616211,
      "activations/layer1_attention_weight_min": -11.30268669128418,
      "activations/layer20_attention_weight_max": 53.11555099487305,
      "activations/layer20_attention_weight_min": -27.005979537963867,
      "activations/layer21_attention_weight_max": 21.803396224975586,
      "activations/layer21_attention_weight_min": -16.08182144165039,
      "activations/layer22_attention_weight_max": 54.66640853881836,
      "activations/layer22_attention_weight_min": -26.03093910217285,
      "activations/layer23_attention_weight_max": 30.668066024780273,
      "activations/layer23_attention_weight_min": -16.39739990234375,
      "activations/layer2_attention_weight_max": 27.26764678955078,
      "activations/layer2_attention_weight_min": -26.150150299072266,
      "activations/layer3_attention_weight_max": 66.14484405517578,
      "activations/layer3_attention_weight_min": -63.43401336669922,
      "activations/layer4_attention_weight_max": 120.86197662353516,
      "activations/layer4_attention_weight_min": -109.15056610107422,
      "activations/layer5_attention_weight_max": 132.099365234375,
      "activations/layer5_attention_weight_min": -113.10958099365234,
      "activations/layer6_attention_weight_max": 75.66032409667969,
      "activations/layer6_attention_weight_min": -58.23808288574219,
      "activations/layer7_attention_weight_max": 76.04590606689453,
      "activations/layer7_attention_weight_min": -69.83454895019531,
      "activations/layer8_attention_weight_max": 71.2039566040039,
      "activations/layer8_attention_weight_min": -57.65728759765625,
      "activations/layer9_attention_weight_max": 91.87886810302734,
      "activations/layer9_attention_weight_min": -77.09431457519531,
      "epoch": 1.22,
      "learning_rate": 0.00014355454545454544,
      "loss": 3.0722,
      "step": 21050
    },
    {
      "activations/layer0_attention_weight_max": 16.728185653686523,
      "activations/layer0_attention_weight_min": -12.453259468078613,
      "activations/layer10_attention_weight_max": 59.32298278808594,
      "activations/layer10_attention_weight_min": -50.278533935546875,
      "activations/layer11_attention_weight_max": 41.83510971069336,
      "activations/layer11_attention_weight_min": -36.3841552734375,
      "activations/layer12_attention_weight_max": 40.50871658325195,
      "activations/layer12_attention_weight_min": -26.82176399230957,
      "activations/layer13_attention_weight_max": 66.23646545410156,
      "activations/layer13_attention_weight_min": -43.3310661315918,
      "activations/layer14_attention_weight_max": 99.70428466796875,
      "activations/layer14_attention_weight_min": -69.83877563476562,
      "activations/layer15_attention_weight_max": 56.17909240722656,
      "activations/layer15_attention_weight_min": -40.36583709716797,
      "activations/layer16_attention_weight_max": 56.04884719848633,
      "activations/layer16_attention_weight_min": -40.76247024536133,
      "activations/layer17_attention_weight_max": 41.26256561279297,
      "activations/layer17_attention_weight_min": -26.166677474975586,
      "activations/layer18_attention_weight_max": 23.563568115234375,
      "activations/layer18_attention_weight_min": -16.11315155029297,
      "activations/layer19_attention_weight_max": 55.19794464111328,
      "activations/layer19_attention_weight_min": -30.98945426940918,
      "activations/layer1_attention_weight_max": 16.610408782958984,
      "activations/layer1_attention_weight_min": -11.402911186218262,
      "activations/layer20_attention_weight_max": 46.04058837890625,
      "activations/layer20_attention_weight_min": -25.432605743408203,
      "activations/layer21_attention_weight_max": 23.612197875976562,
      "activations/layer21_attention_weight_min": -14.210439682006836,
      "activations/layer22_attention_weight_max": 41.062965393066406,
      "activations/layer22_attention_weight_min": -22.413433074951172,
      "activations/layer23_attention_weight_max": 24.700111389160156,
      "activations/layer23_attention_weight_min": -14.96769905090332,
      "activations/layer2_attention_weight_max": 26.25965690612793,
      "activations/layer2_attention_weight_min": -26.425697326660156,
      "activations/layer3_attention_weight_max": 61.043949127197266,
      "activations/layer3_attention_weight_min": -61.8421630859375,
      "activations/layer4_attention_weight_max": 110.82755279541016,
      "activations/layer4_attention_weight_min": -103.44713592529297,
      "activations/layer5_attention_weight_max": 119.78425598144531,
      "activations/layer5_attention_weight_min": -102.56381225585938,
      "activations/layer6_attention_weight_max": 63.356163024902344,
      "activations/layer6_attention_weight_min": -53.87593460083008,
      "activations/layer7_attention_weight_max": 60.73746871948242,
      "activations/layer7_attention_weight_min": -63.72676086425781,
      "activations/layer8_attention_weight_max": 53.96467971801758,
      "activations/layer8_attention_weight_min": -49.01289367675781,
      "activations/layer9_attention_weight_max": 75.4216537475586,
      "activations/layer9_attention_weight_min": -66.95431518554688,
      "epoch": 1.23,
      "learning_rate": 0.00014353560606060605,
      "loss": 3.0492,
      "step": 21100
    },
    {
      "activations/layer0_attention_weight_max": 15.859028816223145,
      "activations/layer0_attention_weight_min": -11.968160629272461,
      "activations/layer10_attention_weight_max": 57.999542236328125,
      "activations/layer10_attention_weight_min": -49.5243034362793,
      "activations/layer11_attention_weight_max": 41.0002555847168,
      "activations/layer11_attention_weight_min": -39.064876556396484,
      "activations/layer12_attention_weight_max": 51.67945098876953,
      "activations/layer12_attention_weight_min": -37.518741607666016,
      "activations/layer13_attention_weight_max": 46.59739685058594,
      "activations/layer13_attention_weight_min": -37.0224723815918,
      "activations/layer14_attention_weight_max": 81.58992004394531,
      "activations/layer14_attention_weight_min": -59.11989974975586,
      "activations/layer15_attention_weight_max": 54.51323318481445,
      "activations/layer15_attention_weight_min": -39.567604064941406,
      "activations/layer16_attention_weight_max": 57.7119255065918,
      "activations/layer16_attention_weight_min": -43.16752624511719,
      "activations/layer17_attention_weight_max": 38.732505798339844,
      "activations/layer17_attention_weight_min": -28.510040283203125,
      "activations/layer18_attention_weight_max": 25.08689308166504,
      "activations/layer18_attention_weight_min": -15.445657730102539,
      "activations/layer19_attention_weight_max": 56.260589599609375,
      "activations/layer19_attention_weight_min": -33.20401382446289,
      "activations/layer1_attention_weight_max": 17.51446533203125,
      "activations/layer1_attention_weight_min": -11.47750186920166,
      "activations/layer20_attention_weight_max": 41.156551361083984,
      "activations/layer20_attention_weight_min": -26.634275436401367,
      "activations/layer21_attention_weight_max": 24.272167205810547,
      "activations/layer21_attention_weight_min": -15.84009075164795,
      "activations/layer22_attention_weight_max": 40.15581130981445,
      "activations/layer22_attention_weight_min": -29.422138214111328,
      "activations/layer23_attention_weight_max": 21.9459285736084,
      "activations/layer23_attention_weight_min": -15.468252182006836,
      "activations/layer2_attention_weight_max": 26.908935546875,
      "activations/layer2_attention_weight_min": -26.93233299255371,
      "activations/layer3_attention_weight_max": 59.880496978759766,
      "activations/layer3_attention_weight_min": -64.26658630371094,
      "activations/layer4_attention_weight_max": 110.5955581665039,
      "activations/layer4_attention_weight_min": -105.2157211303711,
      "activations/layer5_attention_weight_max": 116.49629211425781,
      "activations/layer5_attention_weight_min": -107.05050659179688,
      "activations/layer6_attention_weight_max": 64.70149230957031,
      "activations/layer6_attention_weight_min": -57.57694625854492,
      "activations/layer7_attention_weight_max": 58.9019889831543,
      "activations/layer7_attention_weight_min": -60.63230895996094,
      "activations/layer8_attention_weight_max": 54.36759567260742,
      "activations/layer8_attention_weight_min": -47.1484489440918,
      "activations/layer9_attention_weight_max": 78.9411849975586,
      "activations/layer9_attention_weight_min": -59.98419189453125,
      "epoch": 1.23,
      "learning_rate": 0.00014351666666666664,
      "loss": 3.0427,
      "step": 21150
    },
    {
      "activations/layer0_attention_weight_max": 16.122516632080078,
      "activations/layer0_attention_weight_min": -11.770532608032227,
      "activations/layer10_attention_weight_max": 60.16014862060547,
      "activations/layer10_attention_weight_min": -48.8571662902832,
      "activations/layer11_attention_weight_max": 41.56209182739258,
      "activations/layer11_attention_weight_min": -38.774288177490234,
      "activations/layer12_attention_weight_max": 45.38026428222656,
      "activations/layer12_attention_weight_min": -34.96326446533203,
      "activations/layer13_attention_weight_max": 57.127845764160156,
      "activations/layer13_attention_weight_min": -42.204498291015625,
      "activations/layer14_attention_weight_max": 76.69947052001953,
      "activations/layer14_attention_weight_min": -53.45051956176758,
      "activations/layer15_attention_weight_max": 58.5848503112793,
      "activations/layer15_attention_weight_min": -39.88325119018555,
      "activations/layer16_attention_weight_max": 57.96247863769531,
      "activations/layer16_attention_weight_min": -39.43370819091797,
      "activations/layer17_attention_weight_max": 38.30660629272461,
      "activations/layer17_attention_weight_min": -28.750741958618164,
      "activations/layer18_attention_weight_max": 24.981351852416992,
      "activations/layer18_attention_weight_min": -16.81899642944336,
      "activations/layer19_attention_weight_max": 57.32866287231445,
      "activations/layer19_attention_weight_min": -33.342529296875,
      "activations/layer1_attention_weight_max": 16.533599853515625,
      "activations/layer1_attention_weight_min": -12.379718780517578,
      "activations/layer20_attention_weight_max": 48.93110275268555,
      "activations/layer20_attention_weight_min": -26.522985458374023,
      "activations/layer21_attention_weight_max": 24.31637954711914,
      "activations/layer21_attention_weight_min": -14.573471069335938,
      "activations/layer22_attention_weight_max": 38.99052810668945,
      "activations/layer22_attention_weight_min": -26.665681838989258,
      "activations/layer23_attention_weight_max": 22.839580535888672,
      "activations/layer23_attention_weight_min": -14.191850662231445,
      "activations/layer2_attention_weight_max": 26.982084274291992,
      "activations/layer2_attention_weight_min": -25.727018356323242,
      "activations/layer3_attention_weight_max": 60.975379943847656,
      "activations/layer3_attention_weight_min": -63.096309661865234,
      "activations/layer4_attention_weight_max": 113.38481903076172,
      "activations/layer4_attention_weight_min": -107.26903533935547,
      "activations/layer5_attention_weight_max": 116.85753631591797,
      "activations/layer5_attention_weight_min": -110.08302307128906,
      "activations/layer6_attention_weight_max": 62.568511962890625,
      "activations/layer6_attention_weight_min": -58.65214920043945,
      "activations/layer7_attention_weight_max": 59.46297836303711,
      "activations/layer7_attention_weight_min": -60.300743103027344,
      "activations/layer8_attention_weight_max": 53.105770111083984,
      "activations/layer8_attention_weight_min": -53.112266540527344,
      "activations/layer9_attention_weight_max": 85.37908172607422,
      "activations/layer9_attention_weight_min": -70.05097961425781,
      "epoch": 1.23,
      "learning_rate": 0.00014349772727272726,
      "loss": 3.0758,
      "step": 21200
    },
    {
      "activations/layer0_attention_weight_max": 15.995786666870117,
      "activations/layer0_attention_weight_min": -11.353435516357422,
      "activations/layer10_attention_weight_max": 55.52061462402344,
      "activations/layer10_attention_weight_min": -48.310386657714844,
      "activations/layer11_attention_weight_max": 43.804500579833984,
      "activations/layer11_attention_weight_min": -40.68765640258789,
      "activations/layer12_attention_weight_max": 34.126121520996094,
      "activations/layer12_attention_weight_min": -26.9874210357666,
      "activations/layer13_attention_weight_max": 46.47916793823242,
      "activations/layer13_attention_weight_min": -40.75200271606445,
      "activations/layer14_attention_weight_max": 85.63664245605469,
      "activations/layer14_attention_weight_min": -57.280372619628906,
      "activations/layer15_attention_weight_max": 56.98466491699219,
      "activations/layer15_attention_weight_min": -41.28111267089844,
      "activations/layer16_attention_weight_max": 61.29117202758789,
      "activations/layer16_attention_weight_min": -38.041038513183594,
      "activations/layer17_attention_weight_max": 47.533695220947266,
      "activations/layer17_attention_weight_min": -29.07352638244629,
      "activations/layer18_attention_weight_max": 27.446876525878906,
      "activations/layer18_attention_weight_min": -15.216459274291992,
      "activations/layer19_attention_weight_max": 54.7927131652832,
      "activations/layer19_attention_weight_min": -31.320531845092773,
      "activations/layer1_attention_weight_max": 17.27267837524414,
      "activations/layer1_attention_weight_min": -11.785231590270996,
      "activations/layer20_attention_weight_max": 40.907737731933594,
      "activations/layer20_attention_weight_min": -23.705780029296875,
      "activations/layer21_attention_weight_max": 24.462278366088867,
      "activations/layer21_attention_weight_min": -14.53337287902832,
      "activations/layer22_attention_weight_max": 43.80299377441406,
      "activations/layer22_attention_weight_min": -23.57532501220703,
      "activations/layer23_attention_weight_max": 27.805675506591797,
      "activations/layer23_attention_weight_min": -14.957720756530762,
      "activations/layer2_attention_weight_max": 28.456832885742188,
      "activations/layer2_attention_weight_min": -27.951919555664062,
      "activations/layer3_attention_weight_max": 61.26633834838867,
      "activations/layer3_attention_weight_min": -63.454856872558594,
      "activations/layer4_attention_weight_max": 114.1493911743164,
      "activations/layer4_attention_weight_min": -108.23656463623047,
      "activations/layer5_attention_weight_max": 117.59994506835938,
      "activations/layer5_attention_weight_min": -102.76545715332031,
      "activations/layer6_attention_weight_max": 62.59455108642578,
      "activations/layer6_attention_weight_min": -57.48640060424805,
      "activations/layer7_attention_weight_max": 60.069576263427734,
      "activations/layer7_attention_weight_min": -61.05803298950195,
      "activations/layer8_attention_weight_max": 53.00923538208008,
      "activations/layer8_attention_weight_min": -49.91071701049805,
      "activations/layer9_attention_weight_max": 76.44930267333984,
      "activations/layer9_attention_weight_min": -62.055904388427734,
      "epoch": 1.23,
      "learning_rate": 0.00014347878787878787,
      "loss": 3.0773,
      "step": 21250
    },
    {
      "activations/layer0_attention_weight_max": 15.450569152832031,
      "activations/layer0_attention_weight_min": -11.814984321594238,
      "activations/layer10_attention_weight_max": 55.96500015258789,
      "activations/layer10_attention_weight_min": -51.60860061645508,
      "activations/layer11_attention_weight_max": 47.08673858642578,
      "activations/layer11_attention_weight_min": -36.19606018066406,
      "activations/layer12_attention_weight_max": 66.43565368652344,
      "activations/layer12_attention_weight_min": -38.552215576171875,
      "activations/layer13_attention_weight_max": 45.44050598144531,
      "activations/layer13_attention_weight_min": -34.13435363769531,
      "activations/layer14_attention_weight_max": 69.7459487915039,
      "activations/layer14_attention_weight_min": -56.85116958618164,
      "activations/layer15_attention_weight_max": 61.62508010864258,
      "activations/layer15_attention_weight_min": -43.01594161987305,
      "activations/layer16_attention_weight_max": 51.507110595703125,
      "activations/layer16_attention_weight_min": -39.642005920410156,
      "activations/layer17_attention_weight_max": 39.900657653808594,
      "activations/layer17_attention_weight_min": -26.49468231201172,
      "activations/layer18_attention_weight_max": 23.816362380981445,
      "activations/layer18_attention_weight_min": -14.622084617614746,
      "activations/layer19_attention_weight_max": 47.90809631347656,
      "activations/layer19_attention_weight_min": -32.54129409790039,
      "activations/layer1_attention_weight_max": 16.81378746032715,
      "activations/layer1_attention_weight_min": -12.06600570678711,
      "activations/layer20_attention_weight_max": 41.45454788208008,
      "activations/layer20_attention_weight_min": -25.481603622436523,
      "activations/layer21_attention_weight_max": 26.977317810058594,
      "activations/layer21_attention_weight_min": -15.796919822692871,
      "activations/layer22_attention_weight_max": 40.95673751831055,
      "activations/layer22_attention_weight_min": -24.904767990112305,
      "activations/layer23_attention_weight_max": 23.046001434326172,
      "activations/layer23_attention_weight_min": -15.163097381591797,
      "activations/layer2_attention_weight_max": 27.258392333984375,
      "activations/layer2_attention_weight_min": -26.70474624633789,
      "activations/layer3_attention_weight_max": 61.866878509521484,
      "activations/layer3_attention_weight_min": -62.91191864013672,
      "activations/layer4_attention_weight_max": 113.5784912109375,
      "activations/layer4_attention_weight_min": -106.5264663696289,
      "activations/layer5_attention_weight_max": 117.99102783203125,
      "activations/layer5_attention_weight_min": -106.5792236328125,
      "activations/layer6_attention_weight_max": 62.52301025390625,
      "activations/layer6_attention_weight_min": -56.19792938232422,
      "activations/layer7_attention_weight_max": 70.01443481445312,
      "activations/layer7_attention_weight_min": -70.70726013183594,
      "activations/layer8_attention_weight_max": 58.59637451171875,
      "activations/layer8_attention_weight_min": -50.49675369262695,
      "activations/layer9_attention_weight_max": 86.50816345214844,
      "activations/layer9_attention_weight_min": -78.43721771240234,
      "epoch": 1.24,
      "learning_rate": 0.00014345984848484846,
      "loss": 3.0572,
      "step": 21300
    },
    {
      "activations/layer0_attention_weight_max": 15.655510902404785,
      "activations/layer0_attention_weight_min": -11.84611701965332,
      "activations/layer10_attention_weight_max": 61.63821792602539,
      "activations/layer10_attention_weight_min": -52.317134857177734,
      "activations/layer11_attention_weight_max": 43.922096252441406,
      "activations/layer11_attention_weight_min": -38.44193649291992,
      "activations/layer12_attention_weight_max": 45.782588958740234,
      "activations/layer12_attention_weight_min": -30.879682540893555,
      "activations/layer13_attention_weight_max": 49.42036819458008,
      "activations/layer13_attention_weight_min": -35.4145622253418,
      "activations/layer14_attention_weight_max": 77.00504302978516,
      "activations/layer14_attention_weight_min": -54.67697525024414,
      "activations/layer15_attention_weight_max": 55.206207275390625,
      "activations/layer15_attention_weight_min": -41.42766571044922,
      "activations/layer16_attention_weight_max": 53.64809799194336,
      "activations/layer16_attention_weight_min": -38.219791412353516,
      "activations/layer17_attention_weight_max": 43.370269775390625,
      "activations/layer17_attention_weight_min": -27.438491821289062,
      "activations/layer18_attention_weight_max": 24.165279388427734,
      "activations/layer18_attention_weight_min": -16.441497802734375,
      "activations/layer19_attention_weight_max": 48.44123458862305,
      "activations/layer19_attention_weight_min": -31.0722599029541,
      "activations/layer1_attention_weight_max": 17.187997817993164,
      "activations/layer1_attention_weight_min": -12.108223915100098,
      "activations/layer20_attention_weight_max": 37.10768127441406,
      "activations/layer20_attention_weight_min": -26.45891761779785,
      "activations/layer21_attention_weight_max": 25.501493453979492,
      "activations/layer21_attention_weight_min": -16.699975967407227,
      "activations/layer22_attention_weight_max": 40.77702331542969,
      "activations/layer22_attention_weight_min": -23.430484771728516,
      "activations/layer23_attention_weight_max": 26.098899841308594,
      "activations/layer23_attention_weight_min": -15.278597831726074,
      "activations/layer2_attention_weight_max": 28.094730377197266,
      "activations/layer2_attention_weight_min": -27.79767608642578,
      "activations/layer3_attention_weight_max": 65.50789642333984,
      "activations/layer3_attention_weight_min": -65.79436492919922,
      "activations/layer4_attention_weight_max": 117.22171783447266,
      "activations/layer4_attention_weight_min": -109.75823211669922,
      "activations/layer5_attention_weight_max": 122.03179168701172,
      "activations/layer5_attention_weight_min": -109.9547348022461,
      "activations/layer6_attention_weight_max": 66.99447631835938,
      "activations/layer6_attention_weight_min": -58.1129264831543,
      "activations/layer7_attention_weight_max": 62.20878601074219,
      "activations/layer7_attention_weight_min": -64.59950256347656,
      "activations/layer8_attention_weight_max": 59.11695098876953,
      "activations/layer8_attention_weight_min": -54.19265365600586,
      "activations/layer9_attention_weight_max": 75.61474609375,
      "activations/layer9_attention_weight_min": -70.92064666748047,
      "epoch": 1.24,
      "learning_rate": 0.00014344128787878785,
      "loss": 3.0525,
      "step": 21350
    },
    {
      "activations/layer0_attention_weight_max": 15.806855201721191,
      "activations/layer0_attention_weight_min": -12.353882789611816,
      "activations/layer10_attention_weight_max": 58.79431915283203,
      "activations/layer10_attention_weight_min": -48.38960647583008,
      "activations/layer11_attention_weight_max": 44.30973434448242,
      "activations/layer11_attention_weight_min": -36.907188415527344,
      "activations/layer12_attention_weight_max": 34.242095947265625,
      "activations/layer12_attention_weight_min": -30.084251403808594,
      "activations/layer13_attention_weight_max": 55.8433952331543,
      "activations/layer13_attention_weight_min": -38.830623626708984,
      "activations/layer14_attention_weight_max": 83.31004333496094,
      "activations/layer14_attention_weight_min": -58.9155158996582,
      "activations/layer15_attention_weight_max": 61.01585388183594,
      "activations/layer15_attention_weight_min": -39.45765686035156,
      "activations/layer16_attention_weight_max": 58.0671501159668,
      "activations/layer16_attention_weight_min": -43.13296890258789,
      "activations/layer17_attention_weight_max": 49.04783630371094,
      "activations/layer17_attention_weight_min": -28.57805824279785,
      "activations/layer18_attention_weight_max": 25.8731632232666,
      "activations/layer18_attention_weight_min": -15.678755760192871,
      "activations/layer19_attention_weight_max": 54.27311325073242,
      "activations/layer19_attention_weight_min": -31.674726486206055,
      "activations/layer1_attention_weight_max": 16.962112426757812,
      "activations/layer1_attention_weight_min": -11.206806182861328,
      "activations/layer20_attention_weight_max": 43.826412200927734,
      "activations/layer20_attention_weight_min": -26.3341121673584,
      "activations/layer21_attention_weight_max": 27.285470962524414,
      "activations/layer21_attention_weight_min": -16.53755760192871,
      "activations/layer22_attention_weight_max": 42.20810317993164,
      "activations/layer22_attention_weight_min": -29.0040283203125,
      "activations/layer23_attention_weight_max": 24.200824737548828,
      "activations/layer23_attention_weight_min": -15.156518936157227,
      "activations/layer2_attention_weight_max": 28.03897476196289,
      "activations/layer2_attention_weight_min": -27.997974395751953,
      "activations/layer3_attention_weight_max": 60.29283142089844,
      "activations/layer3_attention_weight_min": -63.27296829223633,
      "activations/layer4_attention_weight_max": 118.07417297363281,
      "activations/layer4_attention_weight_min": -108.46717834472656,
      "activations/layer5_attention_weight_max": 126.6959228515625,
      "activations/layer5_attention_weight_min": -104.90116882324219,
      "activations/layer6_attention_weight_max": 66.03556060791016,
      "activations/layer6_attention_weight_min": -55.43065643310547,
      "activations/layer7_attention_weight_max": 61.552513122558594,
      "activations/layer7_attention_weight_min": -60.35920715332031,
      "activations/layer8_attention_weight_max": 57.47456741333008,
      "activations/layer8_attention_weight_min": -48.46097183227539,
      "activations/layer9_attention_weight_max": 76.36283111572266,
      "activations/layer9_attention_weight_min": -58.58667755126953,
      "epoch": 1.24,
      "learning_rate": 0.00014342234848484846,
      "loss": 3.0435,
      "step": 21400
    },
    {
      "activations/layer0_attention_weight_max": 16.02912139892578,
      "activations/layer0_attention_weight_min": -11.737792015075684,
      "activations/layer10_attention_weight_max": 64.10882568359375,
      "activations/layer10_attention_weight_min": -50.79523849487305,
      "activations/layer11_attention_weight_max": 59.12004852294922,
      "activations/layer11_attention_weight_min": -40.751441955566406,
      "activations/layer12_attention_weight_max": 35.60298538208008,
      "activations/layer12_attention_weight_min": -27.348957061767578,
      "activations/layer13_attention_weight_max": 51.91326141357422,
      "activations/layer13_attention_weight_min": -36.14716339111328,
      "activations/layer14_attention_weight_max": 75.93025970458984,
      "activations/layer14_attention_weight_min": -50.2817268371582,
      "activations/layer15_attention_weight_max": 54.11741638183594,
      "activations/layer15_attention_weight_min": -37.32988739013672,
      "activations/layer16_attention_weight_max": 58.61520767211914,
      "activations/layer16_attention_weight_min": -36.8240966796875,
      "activations/layer17_attention_weight_max": 47.44816970825195,
      "activations/layer17_attention_weight_min": -29.22273826599121,
      "activations/layer18_attention_weight_max": 24.96090316772461,
      "activations/layer18_attention_weight_min": -15.406139373779297,
      "activations/layer19_attention_weight_max": 60.183101654052734,
      "activations/layer19_attention_weight_min": -35.6928825378418,
      "activations/layer1_attention_weight_max": 17.61105728149414,
      "activations/layer1_attention_weight_min": -11.49271011352539,
      "activations/layer20_attention_weight_max": 44.26768493652344,
      "activations/layer20_attention_weight_min": -28.366504669189453,
      "activations/layer21_attention_weight_max": 21.8958740234375,
      "activations/layer21_attention_weight_min": -13.816957473754883,
      "activations/layer22_attention_weight_max": 50.18913650512695,
      "activations/layer22_attention_weight_min": -23.49602699279785,
      "activations/layer23_attention_weight_max": 27.1012020111084,
      "activations/layer23_attention_weight_min": -14.80555248260498,
      "activations/layer2_attention_weight_max": 27.48129653930664,
      "activations/layer2_attention_weight_min": -26.564620971679688,
      "activations/layer3_attention_weight_max": 62.049591064453125,
      "activations/layer3_attention_weight_min": -64.56958770751953,
      "activations/layer4_attention_weight_max": 112.10228729248047,
      "activations/layer4_attention_weight_min": -106.63895416259766,
      "activations/layer5_attention_weight_max": 124.86611938476562,
      "activations/layer5_attention_weight_min": -107.26543426513672,
      "activations/layer6_attention_weight_max": 65.17776489257812,
      "activations/layer6_attention_weight_min": -55.593509674072266,
      "activations/layer7_attention_weight_max": 70.24945831298828,
      "activations/layer7_attention_weight_min": -62.16630172729492,
      "activations/layer8_attention_weight_max": 61.91147232055664,
      "activations/layer8_attention_weight_min": -49.95891189575195,
      "activations/layer9_attention_weight_max": 95.39928436279297,
      "activations/layer9_attention_weight_min": -76.15725708007812,
      "epoch": 1.25,
      "learning_rate": 0.00014340340909090908,
      "loss": 3.0422,
      "step": 21450
    },
    {
      "activations/layer0_attention_weight_max": 16.44487762451172,
      "activations/layer0_attention_weight_min": -11.507211685180664,
      "activations/layer10_attention_weight_max": 60.15055465698242,
      "activations/layer10_attention_weight_min": -48.84407424926758,
      "activations/layer11_attention_weight_max": 44.014747619628906,
      "activations/layer11_attention_weight_min": -40.67442321777344,
      "activations/layer12_attention_weight_max": 38.010215759277344,
      "activations/layer12_attention_weight_min": -28.876916885375977,
      "activations/layer13_attention_weight_max": 48.862483978271484,
      "activations/layer13_attention_weight_min": -34.44144821166992,
      "activations/layer14_attention_weight_max": 75.88582611083984,
      "activations/layer14_attention_weight_min": -57.150787353515625,
      "activations/layer15_attention_weight_max": 50.282196044921875,
      "activations/layer15_attention_weight_min": -38.33704376220703,
      "activations/layer16_attention_weight_max": 52.91950607299805,
      "activations/layer16_attention_weight_min": -42.28171920776367,
      "activations/layer17_attention_weight_max": 39.84044647216797,
      "activations/layer17_attention_weight_min": -28.718469619750977,
      "activations/layer18_attention_weight_max": 21.45993423461914,
      "activations/layer18_attention_weight_min": -14.887256622314453,
      "activations/layer19_attention_weight_max": 53.8198356628418,
      "activations/layer19_attention_weight_min": -35.789100646972656,
      "activations/layer1_attention_weight_max": 17.650327682495117,
      "activations/layer1_attention_weight_min": -11.344771385192871,
      "activations/layer20_attention_weight_max": 43.3629150390625,
      "activations/layer20_attention_weight_min": -28.114458084106445,
      "activations/layer21_attention_weight_max": 22.16156005859375,
      "activations/layer21_attention_weight_min": -13.829450607299805,
      "activations/layer22_attention_weight_max": 42.817718505859375,
      "activations/layer22_attention_weight_min": -26.18828773498535,
      "activations/layer23_attention_weight_max": 23.296405792236328,
      "activations/layer23_attention_weight_min": -15.631629943847656,
      "activations/layer2_attention_weight_max": 27.438331604003906,
      "activations/layer2_attention_weight_min": -27.59250259399414,
      "activations/layer3_attention_weight_max": 63.338321685791016,
      "activations/layer3_attention_weight_min": -64.99264526367188,
      "activations/layer4_attention_weight_max": 119.22090148925781,
      "activations/layer4_attention_weight_min": -109.6108169555664,
      "activations/layer5_attention_weight_max": 124.61262512207031,
      "activations/layer5_attention_weight_min": -103.78364562988281,
      "activations/layer6_attention_weight_max": 67.12771606445312,
      "activations/layer6_attention_weight_min": -55.196929931640625,
      "activations/layer7_attention_weight_max": 64.17452239990234,
      "activations/layer7_attention_weight_min": -60.10176086425781,
      "activations/layer8_attention_weight_max": 61.98180389404297,
      "activations/layer8_attention_weight_min": -46.97881317138672,
      "activations/layer9_attention_weight_max": 75.37619018554688,
      "activations/layer9_attention_weight_min": -66.20594024658203,
      "epoch": 1.25,
      "learning_rate": 0.00014338446969696967,
      "loss": 3.0499,
      "step": 21500
    },
    {
      "activations/layer0_attention_weight_max": 16.725339889526367,
      "activations/layer0_attention_weight_min": -11.571128845214844,
      "activations/layer10_attention_weight_max": 50.41981887817383,
      "activations/layer10_attention_weight_min": -50.59068298339844,
      "activations/layer11_attention_weight_max": 45.60395812988281,
      "activations/layer11_attention_weight_min": -37.16875457763672,
      "activations/layer12_attention_weight_max": 30.365232467651367,
      "activations/layer12_attention_weight_min": -26.954246520996094,
      "activations/layer13_attention_weight_max": 48.857852935791016,
      "activations/layer13_attention_weight_min": -35.66426086425781,
      "activations/layer14_attention_weight_max": 84.92776489257812,
      "activations/layer14_attention_weight_min": -55.28893280029297,
      "activations/layer15_attention_weight_max": 59.44398880004883,
      "activations/layer15_attention_weight_min": -37.78718185424805,
      "activations/layer16_attention_weight_max": 59.66276931762695,
      "activations/layer16_attention_weight_min": -41.641204833984375,
      "activations/layer17_attention_weight_max": 46.4820442199707,
      "activations/layer17_attention_weight_min": -26.244741439819336,
      "activations/layer18_attention_weight_max": 23.422027587890625,
      "activations/layer18_attention_weight_min": -14.728048324584961,
      "activations/layer19_attention_weight_max": 60.89070510864258,
      "activations/layer19_attention_weight_min": -33.98280334472656,
      "activations/layer1_attention_weight_max": 16.86978530883789,
      "activations/layer1_attention_weight_min": -10.953252792358398,
      "activations/layer20_attention_weight_max": 45.62276077270508,
      "activations/layer20_attention_weight_min": -25.08207893371582,
      "activations/layer21_attention_weight_max": 22.998666763305664,
      "activations/layer21_attention_weight_min": -15.549370765686035,
      "activations/layer22_attention_weight_max": 43.14749526977539,
      "activations/layer22_attention_weight_min": -24.396808624267578,
      "activations/layer23_attention_weight_max": 25.504650115966797,
      "activations/layer23_attention_weight_min": -13.925813674926758,
      "activations/layer2_attention_weight_max": 26.903038024902344,
      "activations/layer2_attention_weight_min": -26.976303100585938,
      "activations/layer3_attention_weight_max": 64.66519927978516,
      "activations/layer3_attention_weight_min": -68.44010925292969,
      "activations/layer4_attention_weight_max": 117.0639419555664,
      "activations/layer4_attention_weight_min": -111.70207977294922,
      "activations/layer5_attention_weight_max": 120.97410583496094,
      "activations/layer5_attention_weight_min": -113.65935516357422,
      "activations/layer6_attention_weight_max": 62.62751007080078,
      "activations/layer6_attention_weight_min": -58.479312896728516,
      "activations/layer7_attention_weight_max": 60.26824951171875,
      "activations/layer7_attention_weight_min": -67.38236999511719,
      "activations/layer8_attention_weight_max": 53.231773376464844,
      "activations/layer8_attention_weight_min": -49.24695587158203,
      "activations/layer9_attention_weight_max": 68.64198303222656,
      "activations/layer9_attention_weight_min": -67.06560516357422,
      "epoch": 1.25,
      "learning_rate": 0.00014336553030303028,
      "loss": 3.05,
      "step": 21550
    },
    {
      "activations/layer0_attention_weight_max": 15.333809852600098,
      "activations/layer0_attention_weight_min": -11.792049407958984,
      "activations/layer10_attention_weight_max": 61.1925163269043,
      "activations/layer10_attention_weight_min": -52.97609329223633,
      "activations/layer11_attention_weight_max": 45.79966735839844,
      "activations/layer11_attention_weight_min": -40.257843017578125,
      "activations/layer12_attention_weight_max": 40.60297775268555,
      "activations/layer12_attention_weight_min": -28.37284278869629,
      "activations/layer13_attention_weight_max": 46.254493713378906,
      "activations/layer13_attention_weight_min": -38.75602340698242,
      "activations/layer14_attention_weight_max": 76.6057357788086,
      "activations/layer14_attention_weight_min": -63.78510284423828,
      "activations/layer15_attention_weight_max": 54.54537582397461,
      "activations/layer15_attention_weight_min": -50.759315490722656,
      "activations/layer16_attention_weight_max": 52.741661071777344,
      "activations/layer16_attention_weight_min": -42.548362731933594,
      "activations/layer17_attention_weight_max": 41.72270965576172,
      "activations/layer17_attention_weight_min": -35.62351608276367,
      "activations/layer18_attention_weight_max": 22.11804962158203,
      "activations/layer18_attention_weight_min": -15.669147491455078,
      "activations/layer19_attention_weight_max": 46.9149169921875,
      "activations/layer19_attention_weight_min": -31.634958267211914,
      "activations/layer1_attention_weight_max": 16.566211700439453,
      "activations/layer1_attention_weight_min": -11.295339584350586,
      "activations/layer20_attention_weight_max": 41.73410415649414,
      "activations/layer20_attention_weight_min": -27.88218879699707,
      "activations/layer21_attention_weight_max": 21.66619873046875,
      "activations/layer21_attention_weight_min": -17.827795028686523,
      "activations/layer22_attention_weight_max": 39.833370208740234,
      "activations/layer22_attention_weight_min": -25.751489639282227,
      "activations/layer23_attention_weight_max": 25.91075897216797,
      "activations/layer23_attention_weight_min": -15.346701622009277,
      "activations/layer2_attention_weight_max": 27.85961151123047,
      "activations/layer2_attention_weight_min": -25.837444305419922,
      "activations/layer3_attention_weight_max": 63.30293273925781,
      "activations/layer3_attention_weight_min": -67.01227569580078,
      "activations/layer4_attention_weight_max": 111.0640640258789,
      "activations/layer4_attention_weight_min": -112.62959289550781,
      "activations/layer5_attention_weight_max": 113.5436019897461,
      "activations/layer5_attention_weight_min": -107.6400375366211,
      "activations/layer6_attention_weight_max": 61.934207916259766,
      "activations/layer6_attention_weight_min": -56.26551818847656,
      "activations/layer7_attention_weight_max": 58.83976745605469,
      "activations/layer7_attention_weight_min": -61.761940002441406,
      "activations/layer8_attention_weight_max": 59.83999252319336,
      "activations/layer8_attention_weight_min": -53.2816047668457,
      "activations/layer9_attention_weight_max": 70.75642395019531,
      "activations/layer9_attention_weight_min": -66.2334976196289,
      "epoch": 1.26,
      "learning_rate": 0.0001433465909090909,
      "loss": 3.0552,
      "step": 21600
    },
    {
      "activations/layer0_attention_weight_max": 15.600516319274902,
      "activations/layer0_attention_weight_min": -12.195711135864258,
      "activations/layer10_attention_weight_max": 55.39740753173828,
      "activations/layer10_attention_weight_min": -48.364402770996094,
      "activations/layer11_attention_weight_max": 54.382667541503906,
      "activations/layer11_attention_weight_min": -42.849693298339844,
      "activations/layer12_attention_weight_max": 38.43812942504883,
      "activations/layer12_attention_weight_min": -26.56614875793457,
      "activations/layer13_attention_weight_max": 55.28271484375,
      "activations/layer13_attention_weight_min": -40.93565368652344,
      "activations/layer14_attention_weight_max": 79.60075378417969,
      "activations/layer14_attention_weight_min": -57.0067138671875,
      "activations/layer15_attention_weight_max": 58.1645622253418,
      "activations/layer15_attention_weight_min": -43.92821502685547,
      "activations/layer16_attention_weight_max": 62.88102340698242,
      "activations/layer16_attention_weight_min": -44.0390510559082,
      "activations/layer17_attention_weight_max": 45.22554397583008,
      "activations/layer17_attention_weight_min": -26.52790641784668,
      "activations/layer18_attention_weight_max": 25.576906204223633,
      "activations/layer18_attention_weight_min": -15.097094535827637,
      "activations/layer19_attention_weight_max": 60.87272262573242,
      "activations/layer19_attention_weight_min": -33.44489669799805,
      "activations/layer1_attention_weight_max": 17.50885772705078,
      "activations/layer1_attention_weight_min": -11.515925407409668,
      "activations/layer20_attention_weight_max": 43.60897445678711,
      "activations/layer20_attention_weight_min": -25.949596405029297,
      "activations/layer21_attention_weight_max": 31.60287094116211,
      "activations/layer21_attention_weight_min": -15.031825065612793,
      "activations/layer22_attention_weight_max": 44.253875732421875,
      "activations/layer22_attention_weight_min": -23.405548095703125,
      "activations/layer23_attention_weight_max": 24.849830627441406,
      "activations/layer23_attention_weight_min": -14.238078117370605,
      "activations/layer2_attention_weight_max": 27.905284881591797,
      "activations/layer2_attention_weight_min": -26.970508575439453,
      "activations/layer3_attention_weight_max": 62.773441314697266,
      "activations/layer3_attention_weight_min": -65.73466491699219,
      "activations/layer4_attention_weight_max": 114.32238006591797,
      "activations/layer4_attention_weight_min": -111.52130126953125,
      "activations/layer5_attention_weight_max": 122.20162963867188,
      "activations/layer5_attention_weight_min": -110.2456283569336,
      "activations/layer6_attention_weight_max": 64.42398071289062,
      "activations/layer6_attention_weight_min": -56.72562026977539,
      "activations/layer7_attention_weight_max": 66.51307678222656,
      "activations/layer7_attention_weight_min": -62.277976989746094,
      "activations/layer8_attention_weight_max": 65.5145034790039,
      "activations/layer8_attention_weight_min": -55.00080108642578,
      "activations/layer9_attention_weight_max": 93.29744720458984,
      "activations/layer9_attention_weight_min": -71.84246826171875,
      "epoch": 1.26,
      "learning_rate": 0.00014332765151515149,
      "loss": 3.0531,
      "step": 21650
    },
    {
      "activations/layer0_attention_weight_max": 15.562332153320312,
      "activations/layer0_attention_weight_min": -11.930601119995117,
      "activations/layer10_attention_weight_max": 53.56380081176758,
      "activations/layer10_attention_weight_min": -49.14649200439453,
      "activations/layer11_attention_weight_max": 49.740867614746094,
      "activations/layer11_attention_weight_min": -41.57759094238281,
      "activations/layer12_attention_weight_max": 29.56464385986328,
      "activations/layer12_attention_weight_min": -29.745952606201172,
      "activations/layer13_attention_weight_max": 42.71645736694336,
      "activations/layer13_attention_weight_min": -36.26020431518555,
      "activations/layer14_attention_weight_max": 66.87163543701172,
      "activations/layer14_attention_weight_min": -64.69615936279297,
      "activations/layer15_attention_weight_max": 50.40960693359375,
      "activations/layer15_attention_weight_min": -43.55223846435547,
      "activations/layer16_attention_weight_max": 50.86489486694336,
      "activations/layer16_attention_weight_min": -47.489253997802734,
      "activations/layer17_attention_weight_max": 36.904747009277344,
      "activations/layer17_attention_weight_min": -28.57815170288086,
      "activations/layer18_attention_weight_max": 20.13498878479004,
      "activations/layer18_attention_weight_min": -15.30612850189209,
      "activations/layer19_attention_weight_max": 44.19735336303711,
      "activations/layer19_attention_weight_min": -33.13460922241211,
      "activations/layer1_attention_weight_max": 16.23006248474121,
      "activations/layer1_attention_weight_min": -10.963709831237793,
      "activations/layer20_attention_weight_max": 36.8204345703125,
      "activations/layer20_attention_weight_min": -25.78420066833496,
      "activations/layer21_attention_weight_max": 22.761329650878906,
      "activations/layer21_attention_weight_min": -17.745452880859375,
      "activations/layer22_attention_weight_max": 34.577430725097656,
      "activations/layer22_attention_weight_min": -23.26450538635254,
      "activations/layer23_attention_weight_max": 23.24820327758789,
      "activations/layer23_attention_weight_min": -15.689825057983398,
      "activations/layer2_attention_weight_max": 26.61874771118164,
      "activations/layer2_attention_weight_min": -25.77735137939453,
      "activations/layer3_attention_weight_max": 59.51466751098633,
      "activations/layer3_attention_weight_min": -62.11250305175781,
      "activations/layer4_attention_weight_max": 112.76069641113281,
      "activations/layer4_attention_weight_min": -104.3474349975586,
      "activations/layer5_attention_weight_max": 116.23979187011719,
      "activations/layer5_attention_weight_min": -109.66239929199219,
      "activations/layer6_attention_weight_max": 63.48487854003906,
      "activations/layer6_attention_weight_min": -60.1134147644043,
      "activations/layer7_attention_weight_max": 72.62638854980469,
      "activations/layer7_attention_weight_min": -69.76270294189453,
      "activations/layer8_attention_weight_max": 62.03087615966797,
      "activations/layer8_attention_weight_min": -61.6000862121582,
      "activations/layer9_attention_weight_max": 89.190673828125,
      "activations/layer9_attention_weight_min": -79.32864379882812,
      "epoch": 1.26,
      "learning_rate": 0.0001433087121212121,
      "loss": 3.0476,
      "step": 21700
    },
    {
      "activations/layer0_attention_weight_max": 15.9135103225708,
      "activations/layer0_attention_weight_min": -12.135162353515625,
      "activations/layer10_attention_weight_max": 54.00865173339844,
      "activations/layer10_attention_weight_min": -47.3563232421875,
      "activations/layer11_attention_weight_max": 45.63056945800781,
      "activations/layer11_attention_weight_min": -38.41078186035156,
      "activations/layer12_attention_weight_max": 36.888362884521484,
      "activations/layer12_attention_weight_min": -31.780628204345703,
      "activations/layer13_attention_weight_max": 47.78838348388672,
      "activations/layer13_attention_weight_min": -38.345088958740234,
      "activations/layer14_attention_weight_max": 79.93396759033203,
      "activations/layer14_attention_weight_min": -62.87037658691406,
      "activations/layer15_attention_weight_max": 53.34640884399414,
      "activations/layer15_attention_weight_min": -37.642704010009766,
      "activations/layer16_attention_weight_max": 61.92097854614258,
      "activations/layer16_attention_weight_min": -45.62234115600586,
      "activations/layer17_attention_weight_max": 39.45241165161133,
      "activations/layer17_attention_weight_min": -25.131149291992188,
      "activations/layer18_attention_weight_max": 24.34627342224121,
      "activations/layer18_attention_weight_min": -16.235183715820312,
      "activations/layer19_attention_weight_max": 63.46535110473633,
      "activations/layer19_attention_weight_min": -36.372894287109375,
      "activations/layer1_attention_weight_max": 17.733854293823242,
      "activations/layer1_attention_weight_min": -12.12929630279541,
      "activations/layer20_attention_weight_max": 41.31734085083008,
      "activations/layer20_attention_weight_min": -25.751548767089844,
      "activations/layer21_attention_weight_max": 25.265941619873047,
      "activations/layer21_attention_weight_min": -14.810014724731445,
      "activations/layer22_attention_weight_max": 42.641971588134766,
      "activations/layer22_attention_weight_min": -25.55401039123535,
      "activations/layer23_attention_weight_max": 24.653409957885742,
      "activations/layer23_attention_weight_min": -13.629737854003906,
      "activations/layer2_attention_weight_max": 26.943632125854492,
      "activations/layer2_attention_weight_min": -26.458959579467773,
      "activations/layer3_attention_weight_max": 60.652137756347656,
      "activations/layer3_attention_weight_min": -64.32115173339844,
      "activations/layer4_attention_weight_max": 111.25331115722656,
      "activations/layer4_attention_weight_min": -109.0933837890625,
      "activations/layer5_attention_weight_max": 117.0050048828125,
      "activations/layer5_attention_weight_min": -105.03727722167969,
      "activations/layer6_attention_weight_max": 63.925537109375,
      "activations/layer6_attention_weight_min": -56.24631881713867,
      "activations/layer7_attention_weight_max": 59.889469146728516,
      "activations/layer7_attention_weight_min": -64.22469329833984,
      "activations/layer8_attention_weight_max": 56.91048812866211,
      "activations/layer8_attention_weight_min": -53.2303352355957,
      "activations/layer9_attention_weight_max": 71.21412658691406,
      "activations/layer9_attention_weight_min": -67.51699829101562,
      "epoch": 1.26,
      "learning_rate": 0.00014328977272727272,
      "loss": 3.0408,
      "step": 21750
    },
    {
      "activations/layer0_attention_weight_max": 16.357744216918945,
      "activations/layer0_attention_weight_min": -12.029580116271973,
      "activations/layer10_attention_weight_max": 52.43211364746094,
      "activations/layer10_attention_weight_min": -48.69081497192383,
      "activations/layer11_attention_weight_max": 41.51994705200195,
      "activations/layer11_attention_weight_min": -35.70707702636719,
      "activations/layer12_attention_weight_max": 42.28590774536133,
      "activations/layer12_attention_weight_min": -32.706939697265625,
      "activations/layer13_attention_weight_max": 46.11756134033203,
      "activations/layer13_attention_weight_min": -40.11442947387695,
      "activations/layer14_attention_weight_max": 78.68470001220703,
      "activations/layer14_attention_weight_min": -63.018070220947266,
      "activations/layer15_attention_weight_max": 62.82870101928711,
      "activations/layer15_attention_weight_min": -46.665706634521484,
      "activations/layer16_attention_weight_max": 57.43620300292969,
      "activations/layer16_attention_weight_min": -39.45001983642578,
      "activations/layer17_attention_weight_max": 53.955116271972656,
      "activations/layer17_attention_weight_min": -27.579448699951172,
      "activations/layer18_attention_weight_max": 26.966875076293945,
      "activations/layer18_attention_weight_min": -15.042097091674805,
      "activations/layer19_attention_weight_max": 57.56050491333008,
      "activations/layer19_attention_weight_min": -33.27727127075195,
      "activations/layer1_attention_weight_max": 16.458337783813477,
      "activations/layer1_attention_weight_min": -12.67106819152832,
      "activations/layer20_attention_weight_max": 46.54203414916992,
      "activations/layer20_attention_weight_min": -31.251224517822266,
      "activations/layer21_attention_weight_max": 26.872936248779297,
      "activations/layer21_attention_weight_min": -17.73558807373047,
      "activations/layer22_attention_weight_max": 44.2688102722168,
      "activations/layer22_attention_weight_min": -26.79734230041504,
      "activations/layer23_attention_weight_max": 25.098228454589844,
      "activations/layer23_attention_weight_min": -15.758246421813965,
      "activations/layer2_attention_weight_max": 29.164485931396484,
      "activations/layer2_attention_weight_min": -27.871402740478516,
      "activations/layer3_attention_weight_max": 61.58970260620117,
      "activations/layer3_attention_weight_min": -62.749977111816406,
      "activations/layer4_attention_weight_max": 110.07254791259766,
      "activations/layer4_attention_weight_min": -106.69580078125,
      "activations/layer5_attention_weight_max": 115.5602035522461,
      "activations/layer5_attention_weight_min": -104.20005798339844,
      "activations/layer6_attention_weight_max": 60.27665328979492,
      "activations/layer6_attention_weight_min": -55.87976837158203,
      "activations/layer7_attention_weight_max": 60.72344207763672,
      "activations/layer7_attention_weight_min": -68.37316131591797,
      "activations/layer8_attention_weight_max": 51.80585861206055,
      "activations/layer8_attention_weight_min": -47.163063049316406,
      "activations/layer9_attention_weight_max": 65.13823699951172,
      "activations/layer9_attention_weight_min": -57.73554611206055,
      "epoch": 1.27,
      "learning_rate": 0.00014327083333333333,
      "loss": 3.0511,
      "step": 21800
    },
    {
      "activations/layer0_attention_weight_max": 16.005887985229492,
      "activations/layer0_attention_weight_min": -11.299275398254395,
      "activations/layer10_attention_weight_max": 61.21002960205078,
      "activations/layer10_attention_weight_min": -48.39274597167969,
      "activations/layer11_attention_weight_max": 52.55699920654297,
      "activations/layer11_attention_weight_min": -41.106258392333984,
      "activations/layer12_attention_weight_max": 38.82825469970703,
      "activations/layer12_attention_weight_min": -30.145933151245117,
      "activations/layer13_attention_weight_max": 58.51480484008789,
      "activations/layer13_attention_weight_min": -45.21025466918945,
      "activations/layer14_attention_weight_max": 95.49948120117188,
      "activations/layer14_attention_weight_min": -57.851593017578125,
      "activations/layer15_attention_weight_max": 63.5254020690918,
      "activations/layer15_attention_weight_min": -40.26694107055664,
      "activations/layer16_attention_weight_max": 60.25436019897461,
      "activations/layer16_attention_weight_min": -38.97775650024414,
      "activations/layer17_attention_weight_max": 51.67305374145508,
      "activations/layer17_attention_weight_min": -29.77701187133789,
      "activations/layer18_attention_weight_max": 26.69830894470215,
      "activations/layer18_attention_weight_min": -15.59348201751709,
      "activations/layer19_attention_weight_max": 63.4810791015625,
      "activations/layer19_attention_weight_min": -31.110166549682617,
      "activations/layer1_attention_weight_max": 17.56360626220703,
      "activations/layer1_attention_weight_min": -11.716028213500977,
      "activations/layer20_attention_weight_max": 48.645137786865234,
      "activations/layer20_attention_weight_min": -25.561511993408203,
      "activations/layer21_attention_weight_max": 31.317941665649414,
      "activations/layer21_attention_weight_min": -13.914535522460938,
      "activations/layer22_attention_weight_max": 48.08705139160156,
      "activations/layer22_attention_weight_min": -24.93208122253418,
      "activations/layer23_attention_weight_max": 27.270282745361328,
      "activations/layer23_attention_weight_min": -16.873287200927734,
      "activations/layer2_attention_weight_max": 26.904151916503906,
      "activations/layer2_attention_weight_min": -25.846702575683594,
      "activations/layer3_attention_weight_max": 59.332454681396484,
      "activations/layer3_attention_weight_min": -62.91498565673828,
      "activations/layer4_attention_weight_max": 108.22838592529297,
      "activations/layer4_attention_weight_min": -102.77886199951172,
      "activations/layer5_attention_weight_max": 112.48445129394531,
      "activations/layer5_attention_weight_min": -103.554443359375,
      "activations/layer6_attention_weight_max": 65.4187240600586,
      "activations/layer6_attention_weight_min": -56.97661209106445,
      "activations/layer7_attention_weight_max": 61.76067352294922,
      "activations/layer7_attention_weight_min": -63.203914642333984,
      "activations/layer8_attention_weight_max": 54.438560485839844,
      "activations/layer8_attention_weight_min": -47.50068664550781,
      "activations/layer9_attention_weight_max": 76.2220687866211,
      "activations/layer9_attention_weight_min": -59.45355987548828,
      "epoch": 1.27,
      "learning_rate": 0.00014325189393939392,
      "loss": 3.0474,
      "step": 21850
    },
    {
      "activations/layer0_attention_weight_max": 16.829437255859375,
      "activations/layer0_attention_weight_min": -12.007055282592773,
      "activations/layer10_attention_weight_max": 56.884483337402344,
      "activations/layer10_attention_weight_min": -49.06687545776367,
      "activations/layer11_attention_weight_max": 44.00874328613281,
      "activations/layer11_attention_weight_min": -35.64691925048828,
      "activations/layer12_attention_weight_max": 33.229774475097656,
      "activations/layer12_attention_weight_min": -24.696380615234375,
      "activations/layer13_attention_weight_max": 44.11522674560547,
      "activations/layer13_attention_weight_min": -38.18650436401367,
      "activations/layer14_attention_weight_max": 77.92851257324219,
      "activations/layer14_attention_weight_min": -56.82501983642578,
      "activations/layer15_attention_weight_max": 67.9686050415039,
      "activations/layer15_attention_weight_min": -40.600624084472656,
      "activations/layer16_attention_weight_max": 61.359352111816406,
      "activations/layer16_attention_weight_min": -44.4596061706543,
      "activations/layer17_attention_weight_max": 43.97444534301758,
      "activations/layer17_attention_weight_min": -27.375070571899414,
      "activations/layer18_attention_weight_max": 24.630931854248047,
      "activations/layer18_attention_weight_min": -15.016396522521973,
      "activations/layer19_attention_weight_max": 55.195472717285156,
      "activations/layer19_attention_weight_min": -35.166011810302734,
      "activations/layer1_attention_weight_max": 17.650415420532227,
      "activations/layer1_attention_weight_min": -11.467869758605957,
      "activations/layer20_attention_weight_max": 41.58383560180664,
      "activations/layer20_attention_weight_min": -24.11767578125,
      "activations/layer21_attention_weight_max": 23.08282470703125,
      "activations/layer21_attention_weight_min": -15.161270141601562,
      "activations/layer22_attention_weight_max": 46.98625564575195,
      "activations/layer22_attention_weight_min": -23.875282287597656,
      "activations/layer23_attention_weight_max": 28.017789840698242,
      "activations/layer23_attention_weight_min": -14.08364486694336,
      "activations/layer2_attention_weight_max": 27.7674560546875,
      "activations/layer2_attention_weight_min": -27.45242691040039,
      "activations/layer3_attention_weight_max": 64.17350769042969,
      "activations/layer3_attention_weight_min": -66.48377227783203,
      "activations/layer4_attention_weight_max": 118.57128143310547,
      "activations/layer4_attention_weight_min": -109.452880859375,
      "activations/layer5_attention_weight_max": 127.56513977050781,
      "activations/layer5_attention_weight_min": -107.7829818725586,
      "activations/layer6_attention_weight_max": 69.4167709350586,
      "activations/layer6_attention_weight_min": -57.3599853515625,
      "activations/layer7_attention_weight_max": 66.63442993164062,
      "activations/layer7_attention_weight_min": -63.1855354309082,
      "activations/layer8_attention_weight_max": 60.21023178100586,
      "activations/layer8_attention_weight_min": -50.818241119384766,
      "activations/layer9_attention_weight_max": 80.197998046875,
      "activations/layer9_attention_weight_min": -65.15601348876953,
      "epoch": 1.27,
      "learning_rate": 0.00014323295454545453,
      "loss": 3.0512,
      "step": 21900
    },
    {
      "activations/layer0_attention_weight_max": 14.866847038269043,
      "activations/layer0_attention_weight_min": -11.623828887939453,
      "activations/layer10_attention_weight_max": 51.24872589111328,
      "activations/layer10_attention_weight_min": -46.23478698730469,
      "activations/layer11_attention_weight_max": 37.38495635986328,
      "activations/layer11_attention_weight_min": -34.638614654541016,
      "activations/layer12_attention_weight_max": 41.22918701171875,
      "activations/layer12_attention_weight_min": -37.07912826538086,
      "activations/layer13_attention_weight_max": 45.83882522583008,
      "activations/layer13_attention_weight_min": -41.3137321472168,
      "activations/layer14_attention_weight_max": 69.68603515625,
      "activations/layer14_attention_weight_min": -50.61903762817383,
      "activations/layer15_attention_weight_max": 55.54209518432617,
      "activations/layer15_attention_weight_min": -35.55665588378906,
      "activations/layer16_attention_weight_max": 58.65322494506836,
      "activations/layer16_attention_weight_min": -38.189430236816406,
      "activations/layer17_attention_weight_max": 47.058746337890625,
      "activations/layer17_attention_weight_min": -29.00128936767578,
      "activations/layer18_attention_weight_max": 26.32598114013672,
      "activations/layer18_attention_weight_min": -16.809749603271484,
      "activations/layer19_attention_weight_max": 52.509700775146484,
      "activations/layer19_attention_weight_min": -32.317100524902344,
      "activations/layer1_attention_weight_max": 17.671981811523438,
      "activations/layer1_attention_weight_min": -12.516108512878418,
      "activations/layer20_attention_weight_max": 41.48291778564453,
      "activations/layer20_attention_weight_min": -25.057931900024414,
      "activations/layer21_attention_weight_max": 27.766754150390625,
      "activations/layer21_attention_weight_min": -15.805031776428223,
      "activations/layer22_attention_weight_max": 44.24908447265625,
      "activations/layer22_attention_weight_min": -25.10253143310547,
      "activations/layer23_attention_weight_max": 28.896286010742188,
      "activations/layer23_attention_weight_min": -16.02863311767578,
      "activations/layer2_attention_weight_max": 27.280149459838867,
      "activations/layer2_attention_weight_min": -26.088346481323242,
      "activations/layer3_attention_weight_max": 62.07709884643555,
      "activations/layer3_attention_weight_min": -68.8833999633789,
      "activations/layer4_attention_weight_max": 111.75587463378906,
      "activations/layer4_attention_weight_min": -106.16261291503906,
      "activations/layer5_attention_weight_max": 116.00472259521484,
      "activations/layer5_attention_weight_min": -107.2566146850586,
      "activations/layer6_attention_weight_max": 62.67274475097656,
      "activations/layer6_attention_weight_min": -57.170562744140625,
      "activations/layer7_attention_weight_max": 58.51324462890625,
      "activations/layer7_attention_weight_min": -62.138607025146484,
      "activations/layer8_attention_weight_max": 60.21419906616211,
      "activations/layer8_attention_weight_min": -54.54122543334961,
      "activations/layer9_attention_weight_max": 68.83440399169922,
      "activations/layer9_attention_weight_min": -63.54246139526367,
      "epoch": 1.28,
      "learning_rate": 0.00014321401515151515,
      "loss": 3.0675,
      "step": 21950
    },
    {
      "activations/layer0_attention_weight_max": 16.352670669555664,
      "activations/layer0_attention_weight_min": -11.581266403198242,
      "activations/layer10_attention_weight_max": 59.12440872192383,
      "activations/layer10_attention_weight_min": -48.736751556396484,
      "activations/layer11_attention_weight_max": 45.5770263671875,
      "activations/layer11_attention_weight_min": -35.45432662963867,
      "activations/layer12_attention_weight_max": 35.48563766479492,
      "activations/layer12_attention_weight_min": -27.71552276611328,
      "activations/layer13_attention_weight_max": 48.55310821533203,
      "activations/layer13_attention_weight_min": -35.83816146850586,
      "activations/layer14_attention_weight_max": 71.56099700927734,
      "activations/layer14_attention_weight_min": -56.18552780151367,
      "activations/layer15_attention_weight_max": 58.778106689453125,
      "activations/layer15_attention_weight_min": -39.91682815551758,
      "activations/layer16_attention_weight_max": 61.82319259643555,
      "activations/layer16_attention_weight_min": -40.481597900390625,
      "activations/layer17_attention_weight_max": 43.1934700012207,
      "activations/layer17_attention_weight_min": -28.0768985748291,
      "activations/layer18_attention_weight_max": 24.99073600769043,
      "activations/layer18_attention_weight_min": -16.45943832397461,
      "activations/layer19_attention_weight_max": 62.0410041809082,
      "activations/layer19_attention_weight_min": -32.32365036010742,
      "activations/layer1_attention_weight_max": 16.693893432617188,
      "activations/layer1_attention_weight_min": -11.461841583251953,
      "activations/layer20_attention_weight_max": 45.96188735961914,
      "activations/layer20_attention_weight_min": -27.21712875366211,
      "activations/layer21_attention_weight_max": 30.74007797241211,
      "activations/layer21_attention_weight_min": -14.769988059997559,
      "activations/layer22_attention_weight_max": 41.125244140625,
      "activations/layer22_attention_weight_min": -26.19283103942871,
      "activations/layer23_attention_weight_max": 24.331356048583984,
      "activations/layer23_attention_weight_min": -16.397842407226562,
      "activations/layer2_attention_weight_max": 26.721446990966797,
      "activations/layer2_attention_weight_min": -25.350866317749023,
      "activations/layer3_attention_weight_max": 63.831153869628906,
      "activations/layer3_attention_weight_min": -63.07286834716797,
      "activations/layer4_attention_weight_max": 121.34600067138672,
      "activations/layer4_attention_weight_min": -107.4482192993164,
      "activations/layer5_attention_weight_max": 124.26187133789062,
      "activations/layer5_attention_weight_min": -107.016357421875,
      "activations/layer6_attention_weight_max": 67.06362915039062,
      "activations/layer6_attention_weight_min": -56.23482131958008,
      "activations/layer7_attention_weight_max": 70.58037567138672,
      "activations/layer7_attention_weight_min": -62.12448501586914,
      "activations/layer8_attention_weight_max": 58.55232620239258,
      "activations/layer8_attention_weight_min": -49.62271499633789,
      "activations/layer9_attention_weight_max": 76.50719451904297,
      "activations/layer9_attention_weight_min": -60.83427810668945,
      "epoch": 1.28,
      "learning_rate": 0.00014319507575757576,
      "loss": 3.0761,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_loss": 2.99609375,
      "eval_runtime": 8.5147,
      "eval_samples_per_second": 504.305,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_openwebtext_loss": 2.99609375,
      "eval_openwebtext_ppl": 20.007230835729846,
      "eval_openwebtext_runtime": 8.5147,
      "eval_openwebtext_samples_per_second": 504.305,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_wikitext_loss": 3.2734375,
      "eval_wikitext_ppl": 26.401940203271806,
      "eval_wikitext_runtime": 1.9922,
      "eval_wikitext_samples_per_second": 228.893,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_lambada_loss": 3.4375,
      "eval_lambada_ppl": 31.10908815096766,
      "eval_lambada_runtime": 9.5293,
      "eval_lambada_samples_per_second": 510.953,
      "step": 22000
    },
    {
      "activations/layer0_attention_weight_max": 15.205528259277344,
      "activations/layer0_attention_weight_min": -11.359655380249023,
      "activations/layer10_attention_weight_max": 54.29418182373047,
      "activations/layer10_attention_weight_min": -45.17544937133789,
      "activations/layer11_attention_weight_max": 44.101295471191406,
      "activations/layer11_attention_weight_min": -36.15837478637695,
      "activations/layer12_attention_weight_max": 37.5111198425293,
      "activations/layer12_attention_weight_min": -29.737014770507812,
      "activations/layer13_attention_weight_max": 52.318214416503906,
      "activations/layer13_attention_weight_min": -39.66304397583008,
      "activations/layer14_attention_weight_max": 82.97523498535156,
      "activations/layer14_attention_weight_min": -63.41189956665039,
      "activations/layer15_attention_weight_max": 63.4912109375,
      "activations/layer15_attention_weight_min": -41.50468826293945,
      "activations/layer16_attention_weight_max": 62.5093994140625,
      "activations/layer16_attention_weight_min": -42.76881408691406,
      "activations/layer17_attention_weight_max": 44.3714485168457,
      "activations/layer17_attention_weight_min": -29.038381576538086,
      "activations/layer18_attention_weight_max": 28.889902114868164,
      "activations/layer18_attention_weight_min": -16.382068634033203,
      "activations/layer19_attention_weight_max": 54.5085334777832,
      "activations/layer19_attention_weight_min": -33.637489318847656,
      "activations/layer1_attention_weight_max": 17.899450302124023,
      "activations/layer1_attention_weight_min": -12.006338119506836,
      "activations/layer20_attention_weight_max": 46.59659194946289,
      "activations/layer20_attention_weight_min": -24.799650192260742,
      "activations/layer21_attention_weight_max": 26.43782615661621,
      "activations/layer21_attention_weight_min": -18.928115844726562,
      "activations/layer22_attention_weight_max": 47.0225830078125,
      "activations/layer22_attention_weight_min": -23.83732032775879,
      "activations/layer23_attention_weight_max": 25.250425338745117,
      "activations/layer23_attention_weight_min": -13.456098556518555,
      "activations/layer2_attention_weight_max": 27.294340133666992,
      "activations/layer2_attention_weight_min": -26.1986141204834,
      "activations/layer3_attention_weight_max": 63.11709976196289,
      "activations/layer3_attention_weight_min": -64.61262512207031,
      "activations/layer4_attention_weight_max": 113.34976959228516,
      "activations/layer4_attention_weight_min": -108.5413589477539,
      "activations/layer5_attention_weight_max": 119.20138549804688,
      "activations/layer5_attention_weight_min": -105.81953430175781,
      "activations/layer6_attention_weight_max": 65.21051788330078,
      "activations/layer6_attention_weight_min": -55.9973030090332,
      "activations/layer7_attention_weight_max": 61.50215530395508,
      "activations/layer7_attention_weight_min": -68.2724838256836,
      "activations/layer8_attention_weight_max": 57.38084030151367,
      "activations/layer8_attention_weight_min": -51.41413497924805,
      "activations/layer9_attention_weight_max": 79.13069915771484,
      "activations/layer9_attention_weight_min": -71.368896484375,
      "epoch": 1.28,
      "learning_rate": 0.00014317613636363635,
      "loss": 3.0552,
      "step": 22050
    },
    {
      "activations/layer0_attention_weight_max": 15.193934440612793,
      "activations/layer0_attention_weight_min": -11.430046081542969,
      "activations/layer10_attention_weight_max": 54.618499755859375,
      "activations/layer10_attention_weight_min": -46.17191696166992,
      "activations/layer11_attention_weight_max": 42.430419921875,
      "activations/layer11_attention_weight_min": -37.59547424316406,
      "activations/layer12_attention_weight_max": 32.16093826293945,
      "activations/layer12_attention_weight_min": -27.17835807800293,
      "activations/layer13_attention_weight_max": 55.36427307128906,
      "activations/layer13_attention_weight_min": -35.49298858642578,
      "activations/layer14_attention_weight_max": 78.80290985107422,
      "activations/layer14_attention_weight_min": -54.536842346191406,
      "activations/layer15_attention_weight_max": 56.54634475708008,
      "activations/layer15_attention_weight_min": -37.317073822021484,
      "activations/layer16_attention_weight_max": 56.50381088256836,
      "activations/layer16_attention_weight_min": -37.911521911621094,
      "activations/layer17_attention_weight_max": 43.14059829711914,
      "activations/layer17_attention_weight_min": -26.153444290161133,
      "activations/layer18_attention_weight_max": 23.98324966430664,
      "activations/layer18_attention_weight_min": -14.698800086975098,
      "activations/layer19_attention_weight_max": 51.32044219970703,
      "activations/layer19_attention_weight_min": -30.556055068969727,
      "activations/layer1_attention_weight_max": 17.372791290283203,
      "activations/layer1_attention_weight_min": -12.030172348022461,
      "activations/layer20_attention_weight_max": 42.86092758178711,
      "activations/layer20_attention_weight_min": -25.090181350708008,
      "activations/layer21_attention_weight_max": 31.28691864013672,
      "activations/layer21_attention_weight_min": -15.434575080871582,
      "activations/layer22_attention_weight_max": 39.54798126220703,
      "activations/layer22_attention_weight_min": -21.61712074279785,
      "activations/layer23_attention_weight_max": 24.497060775756836,
      "activations/layer23_attention_weight_min": -16.22463607788086,
      "activations/layer2_attention_weight_max": 28.09492301940918,
      "activations/layer2_attention_weight_min": -26.752845764160156,
      "activations/layer3_attention_weight_max": 62.12873840332031,
      "activations/layer3_attention_weight_min": -64.67927551269531,
      "activations/layer4_attention_weight_max": 115.00414276123047,
      "activations/layer4_attention_weight_min": -104.8646240234375,
      "activations/layer5_attention_weight_max": 119.40206146240234,
      "activations/layer5_attention_weight_min": -103.836669921875,
      "activations/layer6_attention_weight_max": 64.59519958496094,
      "activations/layer6_attention_weight_min": -56.11577224731445,
      "activations/layer7_attention_weight_max": 60.445037841796875,
      "activations/layer7_attention_weight_min": -61.90592956542969,
      "activations/layer8_attention_weight_max": 53.21324920654297,
      "activations/layer8_attention_weight_min": -51.56153869628906,
      "activations/layer9_attention_weight_max": 73.11568450927734,
      "activations/layer9_attention_weight_min": -58.71671676635742,
      "epoch": 1.28,
      "learning_rate": 0.00014315719696969697,
      "loss": 3.0514,
      "step": 22100
    },
    {
      "activations/layer0_attention_weight_max": 16.336536407470703,
      "activations/layer0_attention_weight_min": -11.134227752685547,
      "activations/layer10_attention_weight_max": 55.42840576171875,
      "activations/layer10_attention_weight_min": -46.57642364501953,
      "activations/layer11_attention_weight_max": 40.33894348144531,
      "activations/layer11_attention_weight_min": -35.42178726196289,
      "activations/layer12_attention_weight_max": 48.53692626953125,
      "activations/layer12_attention_weight_min": -35.12811279296875,
      "activations/layer13_attention_weight_max": 55.4428825378418,
      "activations/layer13_attention_weight_min": -44.92367935180664,
      "activations/layer14_attention_weight_max": 70.1380844116211,
      "activations/layer14_attention_weight_min": -52.919254302978516,
      "activations/layer15_attention_weight_max": 51.648780822753906,
      "activations/layer15_attention_weight_min": -41.70602798461914,
      "activations/layer16_attention_weight_max": 64.64140319824219,
      "activations/layer16_attention_weight_min": -42.130859375,
      "activations/layer17_attention_weight_max": 39.773101806640625,
      "activations/layer17_attention_weight_min": -29.51719856262207,
      "activations/layer18_attention_weight_max": 22.801353454589844,
      "activations/layer18_attention_weight_min": -14.956422805786133,
      "activations/layer19_attention_weight_max": 55.75991439819336,
      "activations/layer19_attention_weight_min": -33.43235397338867,
      "activations/layer1_attention_weight_max": 16.709636688232422,
      "activations/layer1_attention_weight_min": -10.879422187805176,
      "activations/layer20_attention_weight_max": 41.84996032714844,
      "activations/layer20_attention_weight_min": -25.466753005981445,
      "activations/layer21_attention_weight_max": 28.74271011352539,
      "activations/layer21_attention_weight_min": -16.513099670410156,
      "activations/layer22_attention_weight_max": 39.22377014160156,
      "activations/layer22_attention_weight_min": -22.863571166992188,
      "activations/layer23_attention_weight_max": 22.488460540771484,
      "activations/layer23_attention_weight_min": -15.542086601257324,
      "activations/layer2_attention_weight_max": 27.20384979248047,
      "activations/layer2_attention_weight_min": -25.29475212097168,
      "activations/layer3_attention_weight_max": 59.25548553466797,
      "activations/layer3_attention_weight_min": -60.30390167236328,
      "activations/layer4_attention_weight_max": 107.41249084472656,
      "activations/layer4_attention_weight_min": -102.2466049194336,
      "activations/layer5_attention_weight_max": 110.51023864746094,
      "activations/layer5_attention_weight_min": -102.32923889160156,
      "activations/layer6_attention_weight_max": 58.656700134277344,
      "activations/layer6_attention_weight_min": -57.03217315673828,
      "activations/layer7_attention_weight_max": 58.954830169677734,
      "activations/layer7_attention_weight_min": -65.90550231933594,
      "activations/layer8_attention_weight_max": 53.07197952270508,
      "activations/layer8_attention_weight_min": -51.67009353637695,
      "activations/layer9_attention_weight_max": 66.72518157958984,
      "activations/layer9_attention_weight_min": -54.67078399658203,
      "epoch": 1.29,
      "learning_rate": 0.00014313825757575758,
      "loss": 3.0533,
      "step": 22150
    },
    {
      "activations/layer0_attention_weight_max": 15.528215408325195,
      "activations/layer0_attention_weight_min": -11.800801277160645,
      "activations/layer10_attention_weight_max": 59.33094024658203,
      "activations/layer10_attention_weight_min": -52.61138916015625,
      "activations/layer11_attention_weight_max": 41.26921081542969,
      "activations/layer11_attention_weight_min": -38.397953033447266,
      "activations/layer12_attention_weight_max": 73.47364807128906,
      "activations/layer12_attention_weight_min": -41.6500358581543,
      "activations/layer13_attention_weight_max": 71.6202392578125,
      "activations/layer13_attention_weight_min": -45.29142761230469,
      "activations/layer14_attention_weight_max": 80.4470443725586,
      "activations/layer14_attention_weight_min": -56.72792434692383,
      "activations/layer15_attention_weight_max": 62.22289276123047,
      "activations/layer15_attention_weight_min": -44.16813659667969,
      "activations/layer16_attention_weight_max": 64.30257415771484,
      "activations/layer16_attention_weight_min": -44.92180252075195,
      "activations/layer17_attention_weight_max": 48.911949157714844,
      "activations/layer17_attention_weight_min": -28.833953857421875,
      "activations/layer18_attention_weight_max": 25.342063903808594,
      "activations/layer18_attention_weight_min": -17.195472717285156,
      "activations/layer19_attention_weight_max": 57.99172592163086,
      "activations/layer19_attention_weight_min": -34.56157684326172,
      "activations/layer1_attention_weight_max": 16.564281463623047,
      "activations/layer1_attention_weight_min": -12.001859664916992,
      "activations/layer20_attention_weight_max": 50.998443603515625,
      "activations/layer20_attention_weight_min": -27.427410125732422,
      "activations/layer21_attention_weight_max": 29.448862075805664,
      "activations/layer21_attention_weight_min": -16.564861297607422,
      "activations/layer22_attention_weight_max": 45.04725646972656,
      "activations/layer22_attention_weight_min": -25.837890625,
      "activations/layer23_attention_weight_max": 24.764869689941406,
      "activations/layer23_attention_weight_min": -15.182218551635742,
      "activations/layer2_attention_weight_max": 26.274473190307617,
      "activations/layer2_attention_weight_min": -25.603317260742188,
      "activations/layer3_attention_weight_max": 64.4734878540039,
      "activations/layer3_attention_weight_min": -65.78839874267578,
      "activations/layer4_attention_weight_max": 114.3246078491211,
      "activations/layer4_attention_weight_min": -106.5635757446289,
      "activations/layer5_attention_weight_max": 119.54150390625,
      "activations/layer5_attention_weight_min": -105.97691345214844,
      "activations/layer6_attention_weight_max": 62.847801208496094,
      "activations/layer6_attention_weight_min": -53.50043487548828,
      "activations/layer7_attention_weight_max": 60.25636672973633,
      "activations/layer7_attention_weight_min": -60.1823616027832,
      "activations/layer8_attention_weight_max": 48.157958984375,
      "activations/layer8_attention_weight_min": -47.04845428466797,
      "activations/layer9_attention_weight_max": 72.88748168945312,
      "activations/layer9_attention_weight_min": -66.05422973632812,
      "epoch": 1.29,
      "learning_rate": 0.00014311931818181817,
      "loss": 3.0339,
      "step": 22200
    },
    {
      "activations/layer0_attention_weight_max": 16.958372116088867,
      "activations/layer0_attention_weight_min": -12.448236465454102,
      "activations/layer10_attention_weight_max": 56.87773895263672,
      "activations/layer10_attention_weight_min": -49.01621627807617,
      "activations/layer11_attention_weight_max": 45.161338806152344,
      "activations/layer11_attention_weight_min": -34.59681701660156,
      "activations/layer12_attention_weight_max": 43.400123596191406,
      "activations/layer12_attention_weight_min": -35.88910675048828,
      "activations/layer13_attention_weight_max": 51.3578987121582,
      "activations/layer13_attention_weight_min": -40.131072998046875,
      "activations/layer14_attention_weight_max": 67.77176666259766,
      "activations/layer14_attention_weight_min": -58.16632080078125,
      "activations/layer15_attention_weight_max": 48.860755920410156,
      "activations/layer15_attention_weight_min": -40.037696838378906,
      "activations/layer16_attention_weight_max": 55.70560073852539,
      "activations/layer16_attention_weight_min": -40.335662841796875,
      "activations/layer17_attention_weight_max": 41.61540222167969,
      "activations/layer17_attention_weight_min": -29.69952392578125,
      "activations/layer18_attention_weight_max": 27.093421936035156,
      "activations/layer18_attention_weight_min": -16.502670288085938,
      "activations/layer19_attention_weight_max": 54.44728469848633,
      "activations/layer19_attention_weight_min": -33.70095443725586,
      "activations/layer1_attention_weight_max": 16.80597686767578,
      "activations/layer1_attention_weight_min": -12.02528190612793,
      "activations/layer20_attention_weight_max": 41.1470947265625,
      "activations/layer20_attention_weight_min": -26.43252182006836,
      "activations/layer21_attention_weight_max": 26.124792098999023,
      "activations/layer21_attention_weight_min": -16.264156341552734,
      "activations/layer22_attention_weight_max": 43.369384765625,
      "activations/layer22_attention_weight_min": -23.54399299621582,
      "activations/layer23_attention_weight_max": 22.564674377441406,
      "activations/layer23_attention_weight_min": -15.394891738891602,
      "activations/layer2_attention_weight_max": 27.93068504333496,
      "activations/layer2_attention_weight_min": -27.301692962646484,
      "activations/layer3_attention_weight_max": 62.04323196411133,
      "activations/layer3_attention_weight_min": -65.02934265136719,
      "activations/layer4_attention_weight_max": 109.0292739868164,
      "activations/layer4_attention_weight_min": -106.99639129638672,
      "activations/layer5_attention_weight_max": 113.4046859741211,
      "activations/layer5_attention_weight_min": -101.21957397460938,
      "activations/layer6_attention_weight_max": 60.10734176635742,
      "activations/layer6_attention_weight_min": -56.2623405456543,
      "activations/layer7_attention_weight_max": 62.7801628112793,
      "activations/layer7_attention_weight_min": -59.62955856323242,
      "activations/layer8_attention_weight_max": 55.15028762817383,
      "activations/layer8_attention_weight_min": -48.70905303955078,
      "activations/layer9_attention_weight_max": 70.35000610351562,
      "activations/layer9_attention_weight_min": -62.02412033081055,
      "epoch": 1.29,
      "learning_rate": 0.00014310037878787878,
      "loss": 3.0355,
      "step": 22250
    },
    {
      "activations/layer0_attention_weight_max": 16.316640853881836,
      "activations/layer0_attention_weight_min": -11.928168296813965,
      "activations/layer10_attention_weight_max": 56.175045013427734,
      "activations/layer10_attention_weight_min": -46.802215576171875,
      "activations/layer11_attention_weight_max": 42.988502502441406,
      "activations/layer11_attention_weight_min": -36.54877471923828,
      "activations/layer12_attention_weight_max": 42.296566009521484,
      "activations/layer12_attention_weight_min": -35.18583297729492,
      "activations/layer13_attention_weight_max": 49.785335540771484,
      "activations/layer13_attention_weight_min": -38.342185974121094,
      "activations/layer14_attention_weight_max": 67.70296478271484,
      "activations/layer14_attention_weight_min": -55.73137664794922,
      "activations/layer15_attention_weight_max": 51.45623016357422,
      "activations/layer15_attention_weight_min": -39.419349670410156,
      "activations/layer16_attention_weight_max": 54.23917007446289,
      "activations/layer16_attention_weight_min": -38.86871337890625,
      "activations/layer17_attention_weight_max": 41.34256362915039,
      "activations/layer17_attention_weight_min": -28.046977996826172,
      "activations/layer18_attention_weight_max": 23.42831802368164,
      "activations/layer18_attention_weight_min": -16.03231430053711,
      "activations/layer19_attention_weight_max": 49.3173713684082,
      "activations/layer19_attention_weight_min": -32.83710861206055,
      "activations/layer1_attention_weight_max": 17.44364356994629,
      "activations/layer1_attention_weight_min": -12.627555847167969,
      "activations/layer20_attention_weight_max": 39.756317138671875,
      "activations/layer20_attention_weight_min": -25.621252059936523,
      "activations/layer21_attention_weight_max": 26.861459732055664,
      "activations/layer21_attention_weight_min": -18.367599487304688,
      "activations/layer22_attention_weight_max": 42.424625396728516,
      "activations/layer22_attention_weight_min": -25.7955322265625,
      "activations/layer23_attention_weight_max": 21.765989303588867,
      "activations/layer23_attention_weight_min": -14.217754364013672,
      "activations/layer2_attention_weight_max": 27.140172958374023,
      "activations/layer2_attention_weight_min": -27.389923095703125,
      "activations/layer3_attention_weight_max": 63.41945266723633,
      "activations/layer3_attention_weight_min": -66.40744018554688,
      "activations/layer4_attention_weight_max": 118.21759033203125,
      "activations/layer4_attention_weight_min": -110.34183502197266,
      "activations/layer5_attention_weight_max": 120.98583984375,
      "activations/layer5_attention_weight_min": -103.94132995605469,
      "activations/layer6_attention_weight_max": 63.68238067626953,
      "activations/layer6_attention_weight_min": -55.725975036621094,
      "activations/layer7_attention_weight_max": 59.857147216796875,
      "activations/layer7_attention_weight_min": -58.09052658081055,
      "activations/layer8_attention_weight_max": 49.606834411621094,
      "activations/layer8_attention_weight_min": -49.98932647705078,
      "activations/layer9_attention_weight_max": 73.30224609375,
      "activations/layer9_attention_weight_min": -55.990386962890625,
      "epoch": 1.3,
      "learning_rate": 0.00014308143939393937,
      "loss": 3.0548,
      "step": 22300
    },
    {
      "activations/layer0_attention_weight_max": 16.48015022277832,
      "activations/layer0_attention_weight_min": -11.877723693847656,
      "activations/layer10_attention_weight_max": 51.31702423095703,
      "activations/layer10_attention_weight_min": -49.94679641723633,
      "activations/layer11_attention_weight_max": 43.29051208496094,
      "activations/layer11_attention_weight_min": -38.77427291870117,
      "activations/layer12_attention_weight_max": 29.624906539916992,
      "activations/layer12_attention_weight_min": -24.77676773071289,
      "activations/layer13_attention_weight_max": 47.98772048950195,
      "activations/layer13_attention_weight_min": -40.78235626220703,
      "activations/layer14_attention_weight_max": 81.74687194824219,
      "activations/layer14_attention_weight_min": -64.17974090576172,
      "activations/layer15_attention_weight_max": 54.05508804321289,
      "activations/layer15_attention_weight_min": -40.795650482177734,
      "activations/layer16_attention_weight_max": 51.43629837036133,
      "activations/layer16_attention_weight_min": -39.08829879760742,
      "activations/layer17_attention_weight_max": 42.44902038574219,
      "activations/layer17_attention_weight_min": -28.98931312561035,
      "activations/layer18_attention_weight_max": 25.392480850219727,
      "activations/layer18_attention_weight_min": -14.462394714355469,
      "activations/layer19_attention_weight_max": 50.58379364013672,
      "activations/layer19_attention_weight_min": -32.97600555419922,
      "activations/layer1_attention_weight_max": 16.91374969482422,
      "activations/layer1_attention_weight_min": -11.873697280883789,
      "activations/layer20_attention_weight_max": 39.66712951660156,
      "activations/layer20_attention_weight_min": -27.082279205322266,
      "activations/layer21_attention_weight_max": 23.038965225219727,
      "activations/layer21_attention_weight_min": -15.190325736999512,
      "activations/layer22_attention_weight_max": 39.19438552856445,
      "activations/layer22_attention_weight_min": -24.527679443359375,
      "activations/layer23_attention_weight_max": 21.57167625427246,
      "activations/layer23_attention_weight_min": -13.776015281677246,
      "activations/layer2_attention_weight_max": 25.619976043701172,
      "activations/layer2_attention_weight_min": -25.662212371826172,
      "activations/layer3_attention_weight_max": 62.37831115722656,
      "activations/layer3_attention_weight_min": -65.71051788330078,
      "activations/layer4_attention_weight_max": 114.10485076904297,
      "activations/layer4_attention_weight_min": -106.7623519897461,
      "activations/layer5_attention_weight_max": 125.23580169677734,
      "activations/layer5_attention_weight_min": -105.726806640625,
      "activations/layer6_attention_weight_max": 63.95466232299805,
      "activations/layer6_attention_weight_min": -57.26091766357422,
      "activations/layer7_attention_weight_max": 58.568504333496094,
      "activations/layer7_attention_weight_min": -58.789791107177734,
      "activations/layer8_attention_weight_max": 52.38859939575195,
      "activations/layer8_attention_weight_min": -45.77974319458008,
      "activations/layer9_attention_weight_max": 69.11479949951172,
      "activations/layer9_attention_weight_min": -56.94154739379883,
      "epoch": 1.3,
      "learning_rate": 0.0001430628787878788,
      "loss": 3.0483,
      "step": 22350
    },
    {
      "activations/layer0_attention_weight_max": 15.981407165527344,
      "activations/layer0_attention_weight_min": -11.73643684387207,
      "activations/layer10_attention_weight_max": 67.01312255859375,
      "activations/layer10_attention_weight_min": -53.61931610107422,
      "activations/layer11_attention_weight_max": 58.381011962890625,
      "activations/layer11_attention_weight_min": -43.506954193115234,
      "activations/layer12_attention_weight_max": 60.046913146972656,
      "activations/layer12_attention_weight_min": -35.06272888183594,
      "activations/layer13_attention_weight_max": 61.63102340698242,
      "activations/layer13_attention_weight_min": -42.561676025390625,
      "activations/layer14_attention_weight_max": 78.58950805664062,
      "activations/layer14_attention_weight_min": -55.4766845703125,
      "activations/layer15_attention_weight_max": 51.30765914916992,
      "activations/layer15_attention_weight_min": -37.81045913696289,
      "activations/layer16_attention_weight_max": 61.95515823364258,
      "activations/layer16_attention_weight_min": -41.40274429321289,
      "activations/layer17_attention_weight_max": 46.776100158691406,
      "activations/layer17_attention_weight_min": -30.17436408996582,
      "activations/layer18_attention_weight_max": 25.2092227935791,
      "activations/layer18_attention_weight_min": -14.021099090576172,
      "activations/layer19_attention_weight_max": 56.700164794921875,
      "activations/layer19_attention_weight_min": -32.391597747802734,
      "activations/layer1_attention_weight_max": 17.34413719177246,
      "activations/layer1_attention_weight_min": -11.41938591003418,
      "activations/layer20_attention_weight_max": 42.12449264526367,
      "activations/layer20_attention_weight_min": -23.97684669494629,
      "activations/layer21_attention_weight_max": 28.4611873626709,
      "activations/layer21_attention_weight_min": -16.654308319091797,
      "activations/layer22_attention_weight_max": 41.578521728515625,
      "activations/layer22_attention_weight_min": -24.15056610107422,
      "activations/layer23_attention_weight_max": 23.19988250732422,
      "activations/layer23_attention_weight_min": -13.757518768310547,
      "activations/layer2_attention_weight_max": 27.621341705322266,
      "activations/layer2_attention_weight_min": -26.510364532470703,
      "activations/layer3_attention_weight_max": 61.65875244140625,
      "activations/layer3_attention_weight_min": -63.642024993896484,
      "activations/layer4_attention_weight_max": 121.83043670654297,
      "activations/layer4_attention_weight_min": -108.41583251953125,
      "activations/layer5_attention_weight_max": 132.1834716796875,
      "activations/layer5_attention_weight_min": -107.6262435913086,
      "activations/layer6_attention_weight_max": 75.63916015625,
      "activations/layer6_attention_weight_min": -55.83736038208008,
      "activations/layer7_attention_weight_max": 75.58943939208984,
      "activations/layer7_attention_weight_min": -66.70040893554688,
      "activations/layer8_attention_weight_max": 68.5984878540039,
      "activations/layer8_attention_weight_min": -50.80634689331055,
      "activations/layer9_attention_weight_max": 105.3071060180664,
      "activations/layer9_attention_weight_min": -90.97692108154297,
      "epoch": 1.3,
      "learning_rate": 0.00014304393939393938,
      "loss": 3.044,
      "step": 22400
    },
    {
      "activations/layer0_attention_weight_max": 14.72635269165039,
      "activations/layer0_attention_weight_min": -12.162726402282715,
      "activations/layer10_attention_weight_max": 52.337738037109375,
      "activations/layer10_attention_weight_min": -46.045509338378906,
      "activations/layer11_attention_weight_max": 46.160057067871094,
      "activations/layer11_attention_weight_min": -38.06170654296875,
      "activations/layer12_attention_weight_max": 72.46953582763672,
      "activations/layer12_attention_weight_min": -34.62403106689453,
      "activations/layer13_attention_weight_max": 43.88603591918945,
      "activations/layer13_attention_weight_min": -36.896934509277344,
      "activations/layer14_attention_weight_max": 73.02574157714844,
      "activations/layer14_attention_weight_min": -62.58949661254883,
      "activations/layer15_attention_weight_max": 56.456356048583984,
      "activations/layer15_attention_weight_min": -42.90264892578125,
      "activations/layer16_attention_weight_max": 51.95612716674805,
      "activations/layer16_attention_weight_min": -43.5665397644043,
      "activations/layer17_attention_weight_max": 41.970333099365234,
      "activations/layer17_attention_weight_min": -30.470075607299805,
      "activations/layer18_attention_weight_max": 24.625627517700195,
      "activations/layer18_attention_weight_min": -19.48753547668457,
      "activations/layer19_attention_weight_max": 52.740840911865234,
      "activations/layer19_attention_weight_min": -29.669370651245117,
      "activations/layer1_attention_weight_max": 16.674779891967773,
      "activations/layer1_attention_weight_min": -11.354828834533691,
      "activations/layer20_attention_weight_max": 40.97245788574219,
      "activations/layer20_attention_weight_min": -25.488040924072266,
      "activations/layer21_attention_weight_max": 28.348247528076172,
      "activations/layer21_attention_weight_min": -17.835006713867188,
      "activations/layer22_attention_weight_max": 45.79306411743164,
      "activations/layer22_attention_weight_min": -26.242538452148438,
      "activations/layer23_attention_weight_max": 23.96809959411621,
      "activations/layer23_attention_weight_min": -15.179697036743164,
      "activations/layer2_attention_weight_max": 27.33718490600586,
      "activations/layer2_attention_weight_min": -26.503860473632812,
      "activations/layer3_attention_weight_max": 61.380882263183594,
      "activations/layer3_attention_weight_min": -63.45156478881836,
      "activations/layer4_attention_weight_max": 112.50929260253906,
      "activations/layer4_attention_weight_min": -103.38111877441406,
      "activations/layer5_attention_weight_max": 116.46585083007812,
      "activations/layer5_attention_weight_min": -100.71217346191406,
      "activations/layer6_attention_weight_max": 65.80353546142578,
      "activations/layer6_attention_weight_min": -55.23544692993164,
      "activations/layer7_attention_weight_max": 61.27677536010742,
      "activations/layer7_attention_weight_min": -60.698944091796875,
      "activations/layer8_attention_weight_max": 56.79645919799805,
      "activations/layer8_attention_weight_min": -45.69259262084961,
      "activations/layer9_attention_weight_max": 73.99881744384766,
      "activations/layer9_attention_weight_min": -60.74850082397461,
      "epoch": 1.3,
      "learning_rate": 0.000143025,
      "loss": 3.0424,
      "step": 22450
    },
    {
      "activations/layer0_attention_weight_max": 15.777617454528809,
      "activations/layer0_attention_weight_min": -12.309513092041016,
      "activations/layer10_attention_weight_max": 57.24427032470703,
      "activations/layer10_attention_weight_min": -50.56144714355469,
      "activations/layer11_attention_weight_max": 43.07357406616211,
      "activations/layer11_attention_weight_min": -38.71660614013672,
      "activations/layer12_attention_weight_max": 68.0195083618164,
      "activations/layer12_attention_weight_min": -35.0362663269043,
      "activations/layer13_attention_weight_max": 51.77277374267578,
      "activations/layer13_attention_weight_min": -37.09700393676758,
      "activations/layer14_attention_weight_max": 87.83907318115234,
      "activations/layer14_attention_weight_min": -57.00001525878906,
      "activations/layer15_attention_weight_max": 54.582481384277344,
      "activations/layer15_attention_weight_min": -40.02495574951172,
      "activations/layer16_attention_weight_max": 54.64856719970703,
      "activations/layer16_attention_weight_min": -37.286197662353516,
      "activations/layer17_attention_weight_max": 40.398223876953125,
      "activations/layer17_attention_weight_min": -27.611286163330078,
      "activations/layer18_attention_weight_max": 26.12201499938965,
      "activations/layer18_attention_weight_min": -16.138565063476562,
      "activations/layer19_attention_weight_max": 57.024879455566406,
      "activations/layer19_attention_weight_min": -33.12873840332031,
      "activations/layer1_attention_weight_max": 16.787803649902344,
      "activations/layer1_attention_weight_min": -11.885618209838867,
      "activations/layer20_attention_weight_max": 43.82044219970703,
      "activations/layer20_attention_weight_min": -25.546497344970703,
      "activations/layer21_attention_weight_max": 22.898147583007812,
      "activations/layer21_attention_weight_min": -14.85331916809082,
      "activations/layer22_attention_weight_max": 40.44464111328125,
      "activations/layer22_attention_weight_min": -22.335979461669922,
      "activations/layer23_attention_weight_max": 24.544652938842773,
      "activations/layer23_attention_weight_min": -15.03106689453125,
      "activations/layer2_attention_weight_max": 28.367900848388672,
      "activations/layer2_attention_weight_min": -26.244184494018555,
      "activations/layer3_attention_weight_max": 63.532203674316406,
      "activations/layer3_attention_weight_min": -63.34782409667969,
      "activations/layer4_attention_weight_max": 112.20311737060547,
      "activations/layer4_attention_weight_min": -108.5678482055664,
      "activations/layer5_attention_weight_max": 116.23934936523438,
      "activations/layer5_attention_weight_min": -108.23880004882812,
      "activations/layer6_attention_weight_max": 60.916507720947266,
      "activations/layer6_attention_weight_min": -57.01474380493164,
      "activations/layer7_attention_weight_max": 67.35469818115234,
      "activations/layer7_attention_weight_min": -61.209312438964844,
      "activations/layer8_attention_weight_max": 59.30105972290039,
      "activations/layer8_attention_weight_min": -52.016624450683594,
      "activations/layer9_attention_weight_max": 86.1587905883789,
      "activations/layer9_attention_weight_min": -68.95537567138672,
      "epoch": 1.31,
      "learning_rate": 0.0001430060606060606,
      "loss": 3.0531,
      "step": 22500
    },
    {
      "activations/layer0_attention_weight_max": 16.26817512512207,
      "activations/layer0_attention_weight_min": -12.051080703735352,
      "activations/layer10_attention_weight_max": 55.02615737915039,
      "activations/layer10_attention_weight_min": -49.936546325683594,
      "activations/layer11_attention_weight_max": 50.61437225341797,
      "activations/layer11_attention_weight_min": -38.16493225097656,
      "activations/layer12_attention_weight_max": 63.31610870361328,
      "activations/layer12_attention_weight_min": -36.59016799926758,
      "activations/layer13_attention_weight_max": 50.17332458496094,
      "activations/layer13_attention_weight_min": -37.217403411865234,
      "activations/layer14_attention_weight_max": 74.49186706542969,
      "activations/layer14_attention_weight_min": -53.876705169677734,
      "activations/layer15_attention_weight_max": 54.88077163696289,
      "activations/layer15_attention_weight_min": -41.26539993286133,
      "activations/layer16_attention_weight_max": 50.51083755493164,
      "activations/layer16_attention_weight_min": -39.529178619384766,
      "activations/layer17_attention_weight_max": 44.556678771972656,
      "activations/layer17_attention_weight_min": -28.282983779907227,
      "activations/layer18_attention_weight_max": 27.847408294677734,
      "activations/layer18_attention_weight_min": -15.502612113952637,
      "activations/layer19_attention_weight_max": 47.57628631591797,
      "activations/layer19_attention_weight_min": -29.85898780822754,
      "activations/layer1_attention_weight_max": 17.151390075683594,
      "activations/layer1_attention_weight_min": -11.317075729370117,
      "activations/layer20_attention_weight_max": 40.84272384643555,
      "activations/layer20_attention_weight_min": -24.072866439819336,
      "activations/layer21_attention_weight_max": 25.650650024414062,
      "activations/layer21_attention_weight_min": -16.23384666442871,
      "activations/layer22_attention_weight_max": 41.32816696166992,
      "activations/layer22_attention_weight_min": -22.525543212890625,
      "activations/layer23_attention_weight_max": 26.56868553161621,
      "activations/layer23_attention_weight_min": -17.144874572753906,
      "activations/layer2_attention_weight_max": 27.59164810180664,
      "activations/layer2_attention_weight_min": -26.4097957611084,
      "activations/layer3_attention_weight_max": 59.640689849853516,
      "activations/layer3_attention_weight_min": -61.195308685302734,
      "activations/layer4_attention_weight_max": 108.99346923828125,
      "activations/layer4_attention_weight_min": -105.89888000488281,
      "activations/layer5_attention_weight_max": 114.43077087402344,
      "activations/layer5_attention_weight_min": -105.4142074584961,
      "activations/layer6_attention_weight_max": 61.235130310058594,
      "activations/layer6_attention_weight_min": -57.908538818359375,
      "activations/layer7_attention_weight_max": 63.593833923339844,
      "activations/layer7_attention_weight_min": -60.42029571533203,
      "activations/layer8_attention_weight_max": 55.645294189453125,
      "activations/layer8_attention_weight_min": -44.926490783691406,
      "activations/layer9_attention_weight_max": 72.61936950683594,
      "activations/layer9_attention_weight_min": -63.2547492980957,
      "epoch": 1.31,
      "learning_rate": 0.0001429871212121212,
      "loss": 3.0548,
      "step": 22550
    },
    {
      "activations/layer0_attention_weight_max": 15.841778755187988,
      "activations/layer0_attention_weight_min": -12.15296745300293,
      "activations/layer10_attention_weight_max": 57.388023376464844,
      "activations/layer10_attention_weight_min": -47.84754180908203,
      "activations/layer11_attention_weight_max": 44.24741744995117,
      "activations/layer11_attention_weight_min": -42.721221923828125,
      "activations/layer12_attention_weight_max": 38.78779602050781,
      "activations/layer12_attention_weight_min": -32.01622772216797,
      "activations/layer13_attention_weight_max": 49.98793029785156,
      "activations/layer13_attention_weight_min": -38.15653991699219,
      "activations/layer14_attention_weight_max": 71.72393035888672,
      "activations/layer14_attention_weight_min": -56.145565032958984,
      "activations/layer15_attention_weight_max": 51.03336715698242,
      "activations/layer15_attention_weight_min": -37.07158279418945,
      "activations/layer16_attention_weight_max": 54.67734909057617,
      "activations/layer16_attention_weight_min": -38.985164642333984,
      "activations/layer17_attention_weight_max": 44.77409362792969,
      "activations/layer17_attention_weight_min": -28.999135971069336,
      "activations/layer18_attention_weight_max": 23.136783599853516,
      "activations/layer18_attention_weight_min": -17.3756160736084,
      "activations/layer19_attention_weight_max": 52.06647872924805,
      "activations/layer19_attention_weight_min": -34.2177848815918,
      "activations/layer1_attention_weight_max": 16.76741600036621,
      "activations/layer1_attention_weight_min": -11.673249244689941,
      "activations/layer20_attention_weight_max": 42.960750579833984,
      "activations/layer20_attention_weight_min": -28.728452682495117,
      "activations/layer21_attention_weight_max": 26.794919967651367,
      "activations/layer21_attention_weight_min": -17.08091163635254,
      "activations/layer22_attention_weight_max": 40.271358489990234,
      "activations/layer22_attention_weight_min": -24.999374389648438,
      "activations/layer23_attention_weight_max": 25.578048706054688,
      "activations/layer23_attention_weight_min": -17.59650421142578,
      "activations/layer2_attention_weight_max": 27.523733139038086,
      "activations/layer2_attention_weight_min": -26.80037498474121,
      "activations/layer3_attention_weight_max": 63.06449890136719,
      "activations/layer3_attention_weight_min": -65.16071319580078,
      "activations/layer4_attention_weight_max": 114.23807525634766,
      "activations/layer4_attention_weight_min": -105.07718658447266,
      "activations/layer5_attention_weight_max": 119.14556884765625,
      "activations/layer5_attention_weight_min": -102.945068359375,
      "activations/layer6_attention_weight_max": 65.211669921875,
      "activations/layer6_attention_weight_min": -54.17884826660156,
      "activations/layer7_attention_weight_max": 62.44883346557617,
      "activations/layer7_attention_weight_min": -62.1113166809082,
      "activations/layer8_attention_weight_max": 52.482460021972656,
      "activations/layer8_attention_weight_min": -44.6446418762207,
      "activations/layer9_attention_weight_max": 77.56437683105469,
      "activations/layer9_attention_weight_min": -66.88395690917969,
      "epoch": 1.31,
      "learning_rate": 0.0001429681818181818,
      "loss": 3.0562,
      "step": 22600
    },
    {
      "activations/layer0_attention_weight_max": 14.792433738708496,
      "activations/layer0_attention_weight_min": -12.402578353881836,
      "activations/layer10_attention_weight_max": 60.04572677612305,
      "activations/layer10_attention_weight_min": -48.429046630859375,
      "activations/layer11_attention_weight_max": 57.4382438659668,
      "activations/layer11_attention_weight_min": -43.02273178100586,
      "activations/layer12_attention_weight_max": 52.91199493408203,
      "activations/layer12_attention_weight_min": -36.05213165283203,
      "activations/layer13_attention_weight_max": 53.787315368652344,
      "activations/layer13_attention_weight_min": -37.97003173828125,
      "activations/layer14_attention_weight_max": 73.11612701416016,
      "activations/layer14_attention_weight_min": -53.858097076416016,
      "activations/layer15_attention_weight_max": 54.90465545654297,
      "activations/layer15_attention_weight_min": -37.12958908081055,
      "activations/layer16_attention_weight_max": 51.24674606323242,
      "activations/layer16_attention_weight_min": -44.295928955078125,
      "activations/layer17_attention_weight_max": 43.46091079711914,
      "activations/layer17_attention_weight_min": -33.6159553527832,
      "activations/layer18_attention_weight_max": 28.666582107543945,
      "activations/layer18_attention_weight_min": -16.524259567260742,
      "activations/layer19_attention_weight_max": 45.59490203857422,
      "activations/layer19_attention_weight_min": -32.58255386352539,
      "activations/layer1_attention_weight_max": 17.08953094482422,
      "activations/layer1_attention_weight_min": -12.502580642700195,
      "activations/layer20_attention_weight_max": 35.229469299316406,
      "activations/layer20_attention_weight_min": -27.13579750061035,
      "activations/layer21_attention_weight_max": 21.898916244506836,
      "activations/layer21_attention_weight_min": -15.41632080078125,
      "activations/layer22_attention_weight_max": 39.83403778076172,
      "activations/layer22_attention_weight_min": -22.603017807006836,
      "activations/layer23_attention_weight_max": 22.650976181030273,
      "activations/layer23_attention_weight_min": -15.20273494720459,
      "activations/layer2_attention_weight_max": 26.622516632080078,
      "activations/layer2_attention_weight_min": -26.806377410888672,
      "activations/layer3_attention_weight_max": 62.29573059082031,
      "activations/layer3_attention_weight_min": -63.79197311401367,
      "activations/layer4_attention_weight_max": 112.77674865722656,
      "activations/layer4_attention_weight_min": -112.22843170166016,
      "activations/layer5_attention_weight_max": 121.59690856933594,
      "activations/layer5_attention_weight_min": -111.79854583740234,
      "activations/layer6_attention_weight_max": 66.7936019897461,
      "activations/layer6_attention_weight_min": -56.79567337036133,
      "activations/layer7_attention_weight_max": 70.03423309326172,
      "activations/layer7_attention_weight_min": -74.8891830444336,
      "activations/layer8_attention_weight_max": 65.99466705322266,
      "activations/layer8_attention_weight_min": -56.65580368041992,
      "activations/layer9_attention_weight_max": 102.40789031982422,
      "activations/layer9_attention_weight_min": -77.06273651123047,
      "epoch": 1.32,
      "learning_rate": 0.0001429492424242424,
      "loss": 3.0451,
      "step": 22650
    },
    {
      "activations/layer0_attention_weight_max": 16.39576530456543,
      "activations/layer0_attention_weight_min": -12.85241413116455,
      "activations/layer10_attention_weight_max": 55.44384002685547,
      "activations/layer10_attention_weight_min": -50.01456832885742,
      "activations/layer11_attention_weight_max": 43.98264694213867,
      "activations/layer11_attention_weight_min": -40.19966506958008,
      "activations/layer12_attention_weight_max": 47.71355056762695,
      "activations/layer12_attention_weight_min": -41.30625534057617,
      "activations/layer13_attention_weight_max": 48.49799346923828,
      "activations/layer13_attention_weight_min": -40.89313507080078,
      "activations/layer14_attention_weight_max": 71.05064392089844,
      "activations/layer14_attention_weight_min": -56.4018440246582,
      "activations/layer15_attention_weight_max": 50.39857482910156,
      "activations/layer15_attention_weight_min": -39.18028259277344,
      "activations/layer16_attention_weight_max": 56.643985748291016,
      "activations/layer16_attention_weight_min": -40.3929557800293,
      "activations/layer17_attention_weight_max": 40.60809326171875,
      "activations/layer17_attention_weight_min": -27.293649673461914,
      "activations/layer18_attention_weight_max": 26.543909072875977,
      "activations/layer18_attention_weight_min": -16.56216812133789,
      "activations/layer19_attention_weight_max": 56.612152099609375,
      "activations/layer19_attention_weight_min": -33.0462760925293,
      "activations/layer1_attention_weight_max": 17.540246963500977,
      "activations/layer1_attention_weight_min": -12.479768753051758,
      "activations/layer20_attention_weight_max": 39.31093978881836,
      "activations/layer20_attention_weight_min": -30.25463104248047,
      "activations/layer21_attention_weight_max": 26.821975708007812,
      "activations/layer21_attention_weight_min": -17.376598358154297,
      "activations/layer22_attention_weight_max": 45.669578552246094,
      "activations/layer22_attention_weight_min": -23.227581024169922,
      "activations/layer23_attention_weight_max": 25.500205993652344,
      "activations/layer23_attention_weight_min": -15.364110946655273,
      "activations/layer2_attention_weight_max": 26.163375854492188,
      "activations/layer2_attention_weight_min": -26.532344818115234,
      "activations/layer3_attention_weight_max": 61.76754379272461,
      "activations/layer3_attention_weight_min": -62.8404541015625,
      "activations/layer4_attention_weight_max": 114.13665771484375,
      "activations/layer4_attention_weight_min": -103.9644775390625,
      "activations/layer5_attention_weight_max": 117.23573303222656,
      "activations/layer5_attention_weight_min": -104.2152099609375,
      "activations/layer6_attention_weight_max": 62.36854934692383,
      "activations/layer6_attention_weight_min": -56.50360870361328,
      "activations/layer7_attention_weight_max": 66.48716735839844,
      "activations/layer7_attention_weight_min": -60.80851745605469,
      "activations/layer8_attention_weight_max": 57.41109848022461,
      "activations/layer8_attention_weight_min": -48.843353271484375,
      "activations/layer9_attention_weight_max": 75.09307098388672,
      "activations/layer9_attention_weight_min": -61.19552230834961,
      "epoch": 1.32,
      "learning_rate": 0.00014293030303030301,
      "loss": 3.0544,
      "step": 22700
    },
    {
      "activations/layer0_attention_weight_max": 13.38242244720459,
      "activations/layer0_attention_weight_min": -12.366399765014648,
      "activations/layer10_attention_weight_max": 52.28593063354492,
      "activations/layer10_attention_weight_min": -48.47282791137695,
      "activations/layer11_attention_weight_max": 45.98708724975586,
      "activations/layer11_attention_weight_min": -35.58131408691406,
      "activations/layer12_attention_weight_max": 38.1934814453125,
      "activations/layer12_attention_weight_min": -28.29474639892578,
      "activations/layer13_attention_weight_max": 53.44939041137695,
      "activations/layer13_attention_weight_min": -42.77479934692383,
      "activations/layer14_attention_weight_max": 82.20137786865234,
      "activations/layer14_attention_weight_min": -64.63276672363281,
      "activations/layer15_attention_weight_max": 56.508941650390625,
      "activations/layer15_attention_weight_min": -41.84575653076172,
      "activations/layer16_attention_weight_max": 66.2360610961914,
      "activations/layer16_attention_weight_min": -45.6525993347168,
      "activations/layer17_attention_weight_max": 46.45599365234375,
      "activations/layer17_attention_weight_min": -28.190235137939453,
      "activations/layer18_attention_weight_max": 26.768489837646484,
      "activations/layer18_attention_weight_min": -16.104543685913086,
      "activations/layer19_attention_weight_max": 56.620296478271484,
      "activations/layer19_attention_weight_min": -35.540706634521484,
      "activations/layer1_attention_weight_max": 16.641834259033203,
      "activations/layer1_attention_weight_min": -11.4054536819458,
      "activations/layer20_attention_weight_max": 50.32746887207031,
      "activations/layer20_attention_weight_min": -28.537572860717773,
      "activations/layer21_attention_weight_max": 29.366594314575195,
      "activations/layer21_attention_weight_min": -20.068666458129883,
      "activations/layer22_attention_weight_max": 45.952049255371094,
      "activations/layer22_attention_weight_min": -23.410099029541016,
      "activations/layer23_attention_weight_max": 25.607398986816406,
      "activations/layer23_attention_weight_min": -14.407126426696777,
      "activations/layer2_attention_weight_max": 28.079421997070312,
      "activations/layer2_attention_weight_min": -26.77998161315918,
      "activations/layer3_attention_weight_max": 63.771812438964844,
      "activations/layer3_attention_weight_min": -65.25141143798828,
      "activations/layer4_attention_weight_max": 117.20906829833984,
      "activations/layer4_attention_weight_min": -109.37474822998047,
      "activations/layer5_attention_weight_max": 118.16029357910156,
      "activations/layer5_attention_weight_min": -102.2376937866211,
      "activations/layer6_attention_weight_max": 62.744415283203125,
      "activations/layer6_attention_weight_min": -55.56300354003906,
      "activations/layer7_attention_weight_max": 64.38066101074219,
      "activations/layer7_attention_weight_min": -61.17341995239258,
      "activations/layer8_attention_weight_max": 54.89105987548828,
      "activations/layer8_attention_weight_min": -46.805023193359375,
      "activations/layer9_attention_weight_max": 67.96334838867188,
      "activations/layer9_attention_weight_min": -56.031776428222656,
      "epoch": 1.32,
      "learning_rate": 0.00014291136363636363,
      "loss": 3.049,
      "step": 22750
    },
    {
      "activations/layer0_attention_weight_max": 17.28738021850586,
      "activations/layer0_attention_weight_min": -12.775163650512695,
      "activations/layer10_attention_weight_max": 60.33390808105469,
      "activations/layer10_attention_weight_min": -51.91566848754883,
      "activations/layer11_attention_weight_max": 40.802738189697266,
      "activations/layer11_attention_weight_min": -35.623191833496094,
      "activations/layer12_attention_weight_max": 37.95722579956055,
      "activations/layer12_attention_weight_min": -31.436626434326172,
      "activations/layer13_attention_weight_max": 47.51188278198242,
      "activations/layer13_attention_weight_min": -39.21575164794922,
      "activations/layer14_attention_weight_max": 72.7152099609375,
      "activations/layer14_attention_weight_min": -67.20262145996094,
      "activations/layer15_attention_weight_max": 56.944541931152344,
      "activations/layer15_attention_weight_min": -38.361507415771484,
      "activations/layer16_attention_weight_max": 54.941307067871094,
      "activations/layer16_attention_weight_min": -41.404354095458984,
      "activations/layer17_attention_weight_max": 42.46239471435547,
      "activations/layer17_attention_weight_min": -27.236536026000977,
      "activations/layer18_attention_weight_max": 23.05023765563965,
      "activations/layer18_attention_weight_min": -14.855365753173828,
      "activations/layer19_attention_weight_max": 55.38105392456055,
      "activations/layer19_attention_weight_min": -34.82479476928711,
      "activations/layer1_attention_weight_max": 16.71901512145996,
      "activations/layer1_attention_weight_min": -12.350947380065918,
      "activations/layer20_attention_weight_max": 42.19163513183594,
      "activations/layer20_attention_weight_min": -26.367319107055664,
      "activations/layer21_attention_weight_max": 22.29781150817871,
      "activations/layer21_attention_weight_min": -15.522605895996094,
      "activations/layer22_attention_weight_max": 38.83625793457031,
      "activations/layer22_attention_weight_min": -25.23880386352539,
      "activations/layer23_attention_weight_max": 22.824050903320312,
      "activations/layer23_attention_weight_min": -15.971504211425781,
      "activations/layer2_attention_weight_max": 27.701557159423828,
      "activations/layer2_attention_weight_min": -27.309194564819336,
      "activations/layer3_attention_weight_max": 65.02722930908203,
      "activations/layer3_attention_weight_min": -68.06732940673828,
      "activations/layer4_attention_weight_max": 118.25566864013672,
      "activations/layer4_attention_weight_min": -110.48750305175781,
      "activations/layer5_attention_weight_max": 123.7354965209961,
      "activations/layer5_attention_weight_min": -108.73793029785156,
      "activations/layer6_attention_weight_max": 63.862369537353516,
      "activations/layer6_attention_weight_min": -56.35112380981445,
      "activations/layer7_attention_weight_max": 65.9545669555664,
      "activations/layer7_attention_weight_min": -63.46868133544922,
      "activations/layer8_attention_weight_max": 57.57772445678711,
      "activations/layer8_attention_weight_min": -49.378719329833984,
      "activations/layer9_attention_weight_max": 76.22348022460938,
      "activations/layer9_attention_weight_min": -62.15829086303711,
      "epoch": 1.32,
      "learning_rate": 0.00014289242424242422,
      "loss": 3.055,
      "step": 22800
    },
    {
      "activations/layer0_attention_weight_max": 16.480764389038086,
      "activations/layer0_attention_weight_min": -12.657593727111816,
      "activations/layer10_attention_weight_max": 59.24687194824219,
      "activations/layer10_attention_weight_min": -47.156402587890625,
      "activations/layer11_attention_weight_max": 45.80015563964844,
      "activations/layer11_attention_weight_min": -38.81735610961914,
      "activations/layer12_attention_weight_max": 44.08013153076172,
      "activations/layer12_attention_weight_min": -28.324594497680664,
      "activations/layer13_attention_weight_max": 51.96366882324219,
      "activations/layer13_attention_weight_min": -34.085140228271484,
      "activations/layer14_attention_weight_max": 82.78453826904297,
      "activations/layer14_attention_weight_min": -54.43396759033203,
      "activations/layer15_attention_weight_max": 64.33248138427734,
      "activations/layer15_attention_weight_min": -37.240516662597656,
      "activations/layer16_attention_weight_max": 57.39513397216797,
      "activations/layer16_attention_weight_min": -40.76353454589844,
      "activations/layer17_attention_weight_max": 47.772220611572266,
      "activations/layer17_attention_weight_min": -28.28746223449707,
      "activations/layer18_attention_weight_max": 25.594816207885742,
      "activations/layer18_attention_weight_min": -15.936513900756836,
      "activations/layer19_attention_weight_max": 54.87146759033203,
      "activations/layer19_attention_weight_min": -32.724281311035156,
      "activations/layer1_attention_weight_max": 17.483434677124023,
      "activations/layer1_attention_weight_min": -12.198755264282227,
      "activations/layer20_attention_weight_max": 42.98652267456055,
      "activations/layer20_attention_weight_min": -27.562166213989258,
      "activations/layer21_attention_weight_max": 25.537572860717773,
      "activations/layer21_attention_weight_min": -15.997149467468262,
      "activations/layer22_attention_weight_max": 42.36691665649414,
      "activations/layer22_attention_weight_min": -23.150951385498047,
      "activations/layer23_attention_weight_max": 26.982131958007812,
      "activations/layer23_attention_weight_min": -16.963136672973633,
      "activations/layer2_attention_weight_max": 26.441884994506836,
      "activations/layer2_attention_weight_min": -26.691375732421875,
      "activations/layer3_attention_weight_max": 65.89306640625,
      "activations/layer3_attention_weight_min": -64.48554992675781,
      "activations/layer4_attention_weight_max": 119.2530288696289,
      "activations/layer4_attention_weight_min": -111.51712799072266,
      "activations/layer5_attention_weight_max": 124.60271453857422,
      "activations/layer5_attention_weight_min": -106.01773071289062,
      "activations/layer6_attention_weight_max": 65.54768371582031,
      "activations/layer6_attention_weight_min": -55.321693420410156,
      "activations/layer7_attention_weight_max": 63.44240951538086,
      "activations/layer7_attention_weight_min": -61.45054626464844,
      "activations/layer8_attention_weight_max": 56.13953399658203,
      "activations/layer8_attention_weight_min": -47.698123931884766,
      "activations/layer9_attention_weight_max": 79.66423034667969,
      "activations/layer9_attention_weight_min": -63.8694953918457,
      "epoch": 1.33,
      "learning_rate": 0.00014287348484848483,
      "loss": 3.0395,
      "step": 22850
    },
    {
      "activations/layer0_attention_weight_max": 17.122711181640625,
      "activations/layer0_attention_weight_min": -12.154220581054688,
      "activations/layer10_attention_weight_max": 49.14958190917969,
      "activations/layer10_attention_weight_min": -45.251949310302734,
      "activations/layer11_attention_weight_max": 38.57130813598633,
      "activations/layer11_attention_weight_min": -35.952857971191406,
      "activations/layer12_attention_weight_max": 41.25592041015625,
      "activations/layer12_attention_weight_min": -27.974241256713867,
      "activations/layer13_attention_weight_max": 48.63161849975586,
      "activations/layer13_attention_weight_min": -40.916259765625,
      "activations/layer14_attention_weight_max": 76.02594757080078,
      "activations/layer14_attention_weight_min": -54.3482780456543,
      "activations/layer15_attention_weight_max": 51.624149322509766,
      "activations/layer15_attention_weight_min": -40.32145690917969,
      "activations/layer16_attention_weight_max": 52.49967575073242,
      "activations/layer16_attention_weight_min": -39.88577651977539,
      "activations/layer17_attention_weight_max": 42.813419342041016,
      "activations/layer17_attention_weight_min": -26.02593994140625,
      "activations/layer18_attention_weight_max": 23.789745330810547,
      "activations/layer18_attention_weight_min": -16.947555541992188,
      "activations/layer19_attention_weight_max": 53.342891693115234,
      "activations/layer19_attention_weight_min": -33.472686767578125,
      "activations/layer1_attention_weight_max": 17.180130004882812,
      "activations/layer1_attention_weight_min": -11.562122344970703,
      "activations/layer20_attention_weight_max": 38.95615005493164,
      "activations/layer20_attention_weight_min": -25.50421142578125,
      "activations/layer21_attention_weight_max": 22.807083129882812,
      "activations/layer21_attention_weight_min": -15.373031616210938,
      "activations/layer22_attention_weight_max": 44.02653884887695,
      "activations/layer22_attention_weight_min": -24.9913330078125,
      "activations/layer23_attention_weight_max": 22.053253173828125,
      "activations/layer23_attention_weight_min": -14.213041305541992,
      "activations/layer2_attention_weight_max": 26.434234619140625,
      "activations/layer2_attention_weight_min": -27.597049713134766,
      "activations/layer3_attention_weight_max": 63.12375259399414,
      "activations/layer3_attention_weight_min": -65.57996368408203,
      "activations/layer4_attention_weight_max": 115.03187561035156,
      "activations/layer4_attention_weight_min": -108.9897689819336,
      "activations/layer5_attention_weight_max": 114.13591003417969,
      "activations/layer5_attention_weight_min": -107.15057373046875,
      "activations/layer6_attention_weight_max": 62.71345138549805,
      "activations/layer6_attention_weight_min": -57.04275894165039,
      "activations/layer7_attention_weight_max": 59.78341293334961,
      "activations/layer7_attention_weight_min": -62.892730712890625,
      "activations/layer8_attention_weight_max": 50.36003112792969,
      "activations/layer8_attention_weight_min": -45.0712776184082,
      "activations/layer9_attention_weight_max": 68.66960906982422,
      "activations/layer9_attention_weight_min": -56.97495651245117,
      "epoch": 1.33,
      "learning_rate": 0.00014285454545454545,
      "loss": 3.0492,
      "step": 22900
    },
    {
      "activations/layer0_attention_weight_max": 14.497515678405762,
      "activations/layer0_attention_weight_min": -12.460976600646973,
      "activations/layer10_attention_weight_max": 55.580772399902344,
      "activations/layer10_attention_weight_min": -46.00374984741211,
      "activations/layer11_attention_weight_max": 40.51427459716797,
      "activations/layer11_attention_weight_min": -36.57632064819336,
      "activations/layer12_attention_weight_max": 35.138458251953125,
      "activations/layer12_attention_weight_min": -28.709684371948242,
      "activations/layer13_attention_weight_max": 54.197792053222656,
      "activations/layer13_attention_weight_min": -37.6091194152832,
      "activations/layer14_attention_weight_max": 76.91822052001953,
      "activations/layer14_attention_weight_min": -56.34180450439453,
      "activations/layer15_attention_weight_max": 53.486629486083984,
      "activations/layer15_attention_weight_min": -38.58613204956055,
      "activations/layer16_attention_weight_max": 60.30345153808594,
      "activations/layer16_attention_weight_min": -35.88178634643555,
      "activations/layer17_attention_weight_max": 47.11894989013672,
      "activations/layer17_attention_weight_min": -30.764265060424805,
      "activations/layer18_attention_weight_max": 22.91176414489746,
      "activations/layer18_attention_weight_min": -14.595338821411133,
      "activations/layer19_attention_weight_max": 60.01060104370117,
      "activations/layer19_attention_weight_min": -29.446735382080078,
      "activations/layer1_attention_weight_max": 17.338045120239258,
      "activations/layer1_attention_weight_min": -12.190409660339355,
      "activations/layer20_attention_weight_max": 57.04426193237305,
      "activations/layer20_attention_weight_min": -24.271427154541016,
      "activations/layer21_attention_weight_max": 27.323490142822266,
      "activations/layer21_attention_weight_min": -18.600496292114258,
      "activations/layer22_attention_weight_max": 46.993038177490234,
      "activations/layer22_attention_weight_min": -26.125715255737305,
      "activations/layer23_attention_weight_max": 31.192882537841797,
      "activations/layer23_attention_weight_min": -14.219573020935059,
      "activations/layer2_attention_weight_max": 29.379108428955078,
      "activations/layer2_attention_weight_min": -27.190343856811523,
      "activations/layer3_attention_weight_max": 62.43395233154297,
      "activations/layer3_attention_weight_min": -64.19136047363281,
      "activations/layer4_attention_weight_max": 115.12496185302734,
      "activations/layer4_attention_weight_min": -102.17822265625,
      "activations/layer5_attention_weight_max": 119.86809539794922,
      "activations/layer5_attention_weight_min": -98.65052795410156,
      "activations/layer6_attention_weight_max": 64.07875061035156,
      "activations/layer6_attention_weight_min": -54.429786682128906,
      "activations/layer7_attention_weight_max": 62.2984619140625,
      "activations/layer7_attention_weight_min": -62.164283752441406,
      "activations/layer8_attention_weight_max": 59.01316452026367,
      "activations/layer8_attention_weight_min": -49.38153839111328,
      "activations/layer9_attention_weight_max": 72.50679779052734,
      "activations/layer9_attention_weight_min": -61.0857048034668,
      "epoch": 1.33,
      "learning_rate": 0.00014283560606060604,
      "loss": 3.0195,
      "step": 22950
    },
    {
      "activations/layer0_attention_weight_max": 15.829386711120605,
      "activations/layer0_attention_weight_min": -12.006722450256348,
      "activations/layer10_attention_weight_max": 58.28252410888672,
      "activations/layer10_attention_weight_min": -47.70664596557617,
      "activations/layer11_attention_weight_max": 43.967308044433594,
      "activations/layer11_attention_weight_min": -36.96891784667969,
      "activations/layer12_attention_weight_max": 41.45503234863281,
      "activations/layer12_attention_weight_min": -28.399084091186523,
      "activations/layer13_attention_weight_max": 53.165687561035156,
      "activations/layer13_attention_weight_min": -36.63142776489258,
      "activations/layer14_attention_weight_max": 83.37947082519531,
      "activations/layer14_attention_weight_min": -52.525390625,
      "activations/layer15_attention_weight_max": 55.639076232910156,
      "activations/layer15_attention_weight_min": -37.68883514404297,
      "activations/layer16_attention_weight_max": 56.36980438232422,
      "activations/layer16_attention_weight_min": -41.47046661376953,
      "activations/layer17_attention_weight_max": 49.487083435058594,
      "activations/layer17_attention_weight_min": -30.383554458618164,
      "activations/layer18_attention_weight_max": 26.001962661743164,
      "activations/layer18_attention_weight_min": -14.362255096435547,
      "activations/layer19_attention_weight_max": 54.56471633911133,
      "activations/layer19_attention_weight_min": -31.811681747436523,
      "activations/layer1_attention_weight_max": 17.242855072021484,
      "activations/layer1_attention_weight_min": -11.370635986328125,
      "activations/layer20_attention_weight_max": 42.26632308959961,
      "activations/layer20_attention_weight_min": -26.43597412109375,
      "activations/layer21_attention_weight_max": 26.11349868774414,
      "activations/layer21_attention_weight_min": -17.49935531616211,
      "activations/layer22_attention_weight_max": 41.52042007446289,
      "activations/layer22_attention_weight_min": -22.35796356201172,
      "activations/layer23_attention_weight_max": 24.485891342163086,
      "activations/layer23_attention_weight_min": -14.889217376708984,
      "activations/layer2_attention_weight_max": 28.424293518066406,
      "activations/layer2_attention_weight_min": -26.63793182373047,
      "activations/layer3_attention_weight_max": 62.55146789550781,
      "activations/layer3_attention_weight_min": -65.52686309814453,
      "activations/layer4_attention_weight_max": 113.24592590332031,
      "activations/layer4_attention_weight_min": -107.83870697021484,
      "activations/layer5_attention_weight_max": 116.3275375366211,
      "activations/layer5_attention_weight_min": -104.12435150146484,
      "activations/layer6_attention_weight_max": 64.099365234375,
      "activations/layer6_attention_weight_min": -55.30127716064453,
      "activations/layer7_attention_weight_max": 61.388919830322266,
      "activations/layer7_attention_weight_min": -60.19837951660156,
      "activations/layer8_attention_weight_max": 59.24730682373047,
      "activations/layer8_attention_weight_min": -55.1929931640625,
      "activations/layer9_attention_weight_max": 68.93819427490234,
      "activations/layer9_attention_weight_min": -61.388484954833984,
      "epoch": 1.34,
      "learning_rate": 0.00014281666666666665,
      "loss": 3.0383,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_loss": 2.986328125,
      "eval_runtime": 8.5044,
      "eval_samples_per_second": 504.916,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_openwebtext_loss": 2.986328125,
      "eval_openwebtext_ppl": 19.812798643245316,
      "eval_openwebtext_runtime": 8.5044,
      "eval_openwebtext_samples_per_second": 504.916,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_wikitext_loss": 3.259765625,
      "eval_wikitext_ppl": 26.043432497666576,
      "eval_wikitext_runtime": 1.9852,
      "eval_wikitext_samples_per_second": 229.698,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_lambada_loss": 3.375,
      "eval_lambada_ppl": 29.22428378123494,
      "eval_lambada_runtime": 9.5606,
      "eval_lambada_samples_per_second": 509.277,
      "step": 23000
    },
    {
      "activations/layer0_attention_weight_max": 16.12911033630371,
      "activations/layer0_attention_weight_min": -12.268882751464844,
      "activations/layer10_attention_weight_max": 47.58945083618164,
      "activations/layer10_attention_weight_min": -44.29389572143555,
      "activations/layer11_attention_weight_max": 40.142677307128906,
      "activations/layer11_attention_weight_min": -35.08328628540039,
      "activations/layer12_attention_weight_max": 33.19775390625,
      "activations/layer12_attention_weight_min": -26.90685272216797,
      "activations/layer13_attention_weight_max": 42.43662643432617,
      "activations/layer13_attention_weight_min": -34.76115036010742,
      "activations/layer14_attention_weight_max": 74.3891830444336,
      "activations/layer14_attention_weight_min": -61.23582458496094,
      "activations/layer15_attention_weight_max": 56.45384979248047,
      "activations/layer15_attention_weight_min": -42.314903259277344,
      "activations/layer16_attention_weight_max": 57.217769622802734,
      "activations/layer16_attention_weight_min": -36.67192459106445,
      "activations/layer17_attention_weight_max": 44.733951568603516,
      "activations/layer17_attention_weight_min": -27.847497940063477,
      "activations/layer18_attention_weight_max": 25.84596824645996,
      "activations/layer18_attention_weight_min": -17.360870361328125,
      "activations/layer19_attention_weight_max": 53.55792999267578,
      "activations/layer19_attention_weight_min": -30.73305320739746,
      "activations/layer1_attention_weight_max": 18.07267951965332,
      "activations/layer1_attention_weight_min": -12.745835304260254,
      "activations/layer20_attention_weight_max": 50.992549896240234,
      "activations/layer20_attention_weight_min": -28.701004028320312,
      "activations/layer21_attention_weight_max": 31.117509841918945,
      "activations/layer21_attention_weight_min": -18.05746078491211,
      "activations/layer22_attention_weight_max": 43.570960998535156,
      "activations/layer22_attention_weight_min": -24.10862159729004,
      "activations/layer23_attention_weight_max": 26.70973014831543,
      "activations/layer23_attention_weight_min": -14.654808044433594,
      "activations/layer2_attention_weight_max": 26.814619064331055,
      "activations/layer2_attention_weight_min": -26.726377487182617,
      "activations/layer3_attention_weight_max": 61.62202453613281,
      "activations/layer3_attention_weight_min": -65.49215698242188,
      "activations/layer4_attention_weight_max": 108.36112213134766,
      "activations/layer4_attention_weight_min": -106.24193572998047,
      "activations/layer5_attention_weight_max": 112.16647338867188,
      "activations/layer5_attention_weight_min": -100.9533920288086,
      "activations/layer6_attention_weight_max": 59.33354568481445,
      "activations/layer6_attention_weight_min": -53.313838958740234,
      "activations/layer7_attention_weight_max": 57.70384216308594,
      "activations/layer7_attention_weight_min": -55.56154251098633,
      "activations/layer8_attention_weight_max": 52.7427864074707,
      "activations/layer8_attention_weight_min": -47.19658279418945,
      "activations/layer9_attention_weight_max": 64.80853271484375,
      "activations/layer9_attention_weight_min": -59.27146530151367,
      "epoch": 1.34,
      "learning_rate": 0.00014279772727272727,
      "loss": 3.0475,
      "step": 23050
    },
    {
      "activations/layer0_attention_weight_max": 14.261115074157715,
      "activations/layer0_attention_weight_min": -12.32992172241211,
      "activations/layer10_attention_weight_max": 62.43314743041992,
      "activations/layer10_attention_weight_min": -51.129783630371094,
      "activations/layer11_attention_weight_max": 47.045013427734375,
      "activations/layer11_attention_weight_min": -39.15582275390625,
      "activations/layer12_attention_weight_max": 39.89519119262695,
      "activations/layer12_attention_weight_min": -27.509201049804688,
      "activations/layer13_attention_weight_max": 52.74687576293945,
      "activations/layer13_attention_weight_min": -42.14604949951172,
      "activations/layer14_attention_weight_max": 87.01011657714844,
      "activations/layer14_attention_weight_min": -56.802978515625,
      "activations/layer15_attention_weight_max": 57.835201263427734,
      "activations/layer15_attention_weight_min": -40.63019561767578,
      "activations/layer16_attention_weight_max": 56.78014373779297,
      "activations/layer16_attention_weight_min": -41.30044937133789,
      "activations/layer17_attention_weight_max": 41.54844665527344,
      "activations/layer17_attention_weight_min": -27.55113410949707,
      "activations/layer18_attention_weight_max": 24.087438583374023,
      "activations/layer18_attention_weight_min": -16.61784553527832,
      "activations/layer19_attention_weight_max": 57.428558349609375,
      "activations/layer19_attention_weight_min": -32.89992904663086,
      "activations/layer1_attention_weight_max": 17.760128021240234,
      "activations/layer1_attention_weight_min": -11.700547218322754,
      "activations/layer20_attention_weight_max": 44.83937454223633,
      "activations/layer20_attention_weight_min": -28.101173400878906,
      "activations/layer21_attention_weight_max": 25.166086196899414,
      "activations/layer21_attention_weight_min": -16.38107681274414,
      "activations/layer22_attention_weight_max": 51.12907028198242,
      "activations/layer22_attention_weight_min": -24.755186080932617,
      "activations/layer23_attention_weight_max": 27.749488830566406,
      "activations/layer23_attention_weight_min": -15.666963577270508,
      "activations/layer2_attention_weight_max": 27.180667877197266,
      "activations/layer2_attention_weight_min": -28.305450439453125,
      "activations/layer3_attention_weight_max": 63.03118133544922,
      "activations/layer3_attention_weight_min": -66.15090942382812,
      "activations/layer4_attention_weight_max": 117.20037841796875,
      "activations/layer4_attention_weight_min": -108.59783172607422,
      "activations/layer5_attention_weight_max": 120.70933532714844,
      "activations/layer5_attention_weight_min": -108.20797729492188,
      "activations/layer6_attention_weight_max": 64.88826751708984,
      "activations/layer6_attention_weight_min": -57.68899917602539,
      "activations/layer7_attention_weight_max": 71.85930633544922,
      "activations/layer7_attention_weight_min": -63.29448699951172,
      "activations/layer8_attention_weight_max": 57.27467346191406,
      "activations/layer8_attention_weight_min": -46.24660110473633,
      "activations/layer9_attention_weight_max": 75.2606201171875,
      "activations/layer9_attention_weight_min": -63.657466888427734,
      "epoch": 1.34,
      "learning_rate": 0.00014277878787878785,
      "loss": 3.0533,
      "step": 23100
    },
    {
      "activations/layer0_attention_weight_max": 15.059686660766602,
      "activations/layer0_attention_weight_min": -11.869012832641602,
      "activations/layer10_attention_weight_max": 57.979331970214844,
      "activations/layer10_attention_weight_min": -51.71866989135742,
      "activations/layer11_attention_weight_max": 42.20561981201172,
      "activations/layer11_attention_weight_min": -40.49565887451172,
      "activations/layer12_attention_weight_max": 37.15911102294922,
      "activations/layer12_attention_weight_min": -27.672672271728516,
      "activations/layer13_attention_weight_max": 53.45283889770508,
      "activations/layer13_attention_weight_min": -38.69232940673828,
      "activations/layer14_attention_weight_max": 78.43550109863281,
      "activations/layer14_attention_weight_min": -62.318946838378906,
      "activations/layer15_attention_weight_max": 59.61543655395508,
      "activations/layer15_attention_weight_min": -38.01932144165039,
      "activations/layer16_attention_weight_max": 55.905250549316406,
      "activations/layer16_attention_weight_min": -40.35050582885742,
      "activations/layer17_attention_weight_max": 37.8455696105957,
      "activations/layer17_attention_weight_min": -29.099191665649414,
      "activations/layer18_attention_weight_max": 23.937376022338867,
      "activations/layer18_attention_weight_min": -16.237796783447266,
      "activations/layer19_attention_weight_max": 53.14026641845703,
      "activations/layer19_attention_weight_min": -35.32593536376953,
      "activations/layer1_attention_weight_max": 17.493627548217773,
      "activations/layer1_attention_weight_min": -11.543802261352539,
      "activations/layer20_attention_weight_max": 40.96442794799805,
      "activations/layer20_attention_weight_min": -26.194007873535156,
      "activations/layer21_attention_weight_max": 23.760385513305664,
      "activations/layer21_attention_weight_min": -16.098129272460938,
      "activations/layer22_attention_weight_max": 37.41187286376953,
      "activations/layer22_attention_weight_min": -22.92938232421875,
      "activations/layer23_attention_weight_max": 25.745290756225586,
      "activations/layer23_attention_weight_min": -16.3015193939209,
      "activations/layer2_attention_weight_max": 27.232648849487305,
      "activations/layer2_attention_weight_min": -26.971355438232422,
      "activations/layer3_attention_weight_max": 60.68844223022461,
      "activations/layer3_attention_weight_min": -65.96126556396484,
      "activations/layer4_attention_weight_max": 113.58614349365234,
      "activations/layer4_attention_weight_min": -111.6845474243164,
      "activations/layer5_attention_weight_max": 112.58497619628906,
      "activations/layer5_attention_weight_min": -105.19186401367188,
      "activations/layer6_attention_weight_max": 61.3557014465332,
      "activations/layer6_attention_weight_min": -57.956886291503906,
      "activations/layer7_attention_weight_max": 61.113677978515625,
      "activations/layer7_attention_weight_min": -62.61933898925781,
      "activations/layer8_attention_weight_max": 50.983802795410156,
      "activations/layer8_attention_weight_min": -45.02049255371094,
      "activations/layer9_attention_weight_max": 68.18275451660156,
      "activations/layer9_attention_weight_min": -61.76040267944336,
      "epoch": 1.35,
      "learning_rate": 0.00014275984848484847,
      "loss": 3.0573,
      "step": 23150
    },
    {
      "activations/layer0_attention_weight_max": 16.199466705322266,
      "activations/layer0_attention_weight_min": -12.7483549118042,
      "activations/layer10_attention_weight_max": 60.373992919921875,
      "activations/layer10_attention_weight_min": -47.69111251831055,
      "activations/layer11_attention_weight_max": 49.22272491455078,
      "activations/layer11_attention_weight_min": -37.377410888671875,
      "activations/layer12_attention_weight_max": 57.01903533935547,
      "activations/layer12_attention_weight_min": -31.57539176940918,
      "activations/layer13_attention_weight_max": 45.56606674194336,
      "activations/layer13_attention_weight_min": -34.676841735839844,
      "activations/layer14_attention_weight_max": 72.85564422607422,
      "activations/layer14_attention_weight_min": -59.47355270385742,
      "activations/layer15_attention_weight_max": 52.33648681640625,
      "activations/layer15_attention_weight_min": -37.26723861694336,
      "activations/layer16_attention_weight_max": 52.47940444946289,
      "activations/layer16_attention_weight_min": -38.08761978149414,
      "activations/layer17_attention_weight_max": 38.56508255004883,
      "activations/layer17_attention_weight_min": -24.99928092956543,
      "activations/layer18_attention_weight_max": 24.83414077758789,
      "activations/layer18_attention_weight_min": -16.543960571289062,
      "activations/layer19_attention_weight_max": 48.42118453979492,
      "activations/layer19_attention_weight_min": -32.337093353271484,
      "activations/layer1_attention_weight_max": 17.391538619995117,
      "activations/layer1_attention_weight_min": -11.40397834777832,
      "activations/layer20_attention_weight_max": 39.5301399230957,
      "activations/layer20_attention_weight_min": -26.052490234375,
      "activations/layer21_attention_weight_max": 24.41115379333496,
      "activations/layer21_attention_weight_min": -17.203418731689453,
      "activations/layer22_attention_weight_max": 37.801082611083984,
      "activations/layer22_attention_weight_min": -23.22046661376953,
      "activations/layer23_attention_weight_max": 24.065719604492188,
      "activations/layer23_attention_weight_min": -14.437762260437012,
      "activations/layer2_attention_weight_max": 29.020002365112305,
      "activations/layer2_attention_weight_min": -28.04991912841797,
      "activations/layer3_attention_weight_max": 62.96318054199219,
      "activations/layer3_attention_weight_min": -66.64688110351562,
      "activations/layer4_attention_weight_max": 122.715087890625,
      "activations/layer4_attention_weight_min": -107.0683364868164,
      "activations/layer5_attention_weight_max": 130.35916137695312,
      "activations/layer5_attention_weight_min": -104.71526336669922,
      "activations/layer6_attention_weight_max": 75.50585174560547,
      "activations/layer6_attention_weight_min": -57.49592590332031,
      "activations/layer7_attention_weight_max": 72.0948715209961,
      "activations/layer7_attention_weight_min": -63.327239990234375,
      "activations/layer8_attention_weight_max": 64.24919128417969,
      "activations/layer8_attention_weight_min": -53.429141998291016,
      "activations/layer9_attention_weight_max": 108.68409729003906,
      "activations/layer9_attention_weight_min": -77.37643432617188,
      "epoch": 1.35,
      "learning_rate": 0.00014274090909090908,
      "loss": 3.0444,
      "step": 23200
    },
    {
      "activations/layer0_attention_weight_max": 16.077049255371094,
      "activations/layer0_attention_weight_min": -13.113656044006348,
      "activations/layer10_attention_weight_max": 53.2020263671875,
      "activations/layer10_attention_weight_min": -47.70954513549805,
      "activations/layer11_attention_weight_max": 47.686065673828125,
      "activations/layer11_attention_weight_min": -37.84034729003906,
      "activations/layer12_attention_weight_max": 34.322811126708984,
      "activations/layer12_attention_weight_min": -29.736482620239258,
      "activations/layer13_attention_weight_max": 46.196006774902344,
      "activations/layer13_attention_weight_min": -34.307491302490234,
      "activations/layer14_attention_weight_max": 71.07624053955078,
      "activations/layer14_attention_weight_min": -54.68363952636719,
      "activations/layer15_attention_weight_max": 48.33922576904297,
      "activations/layer15_attention_weight_min": -37.691925048828125,
      "activations/layer16_attention_weight_max": 50.893714904785156,
      "activations/layer16_attention_weight_min": -40.462425231933594,
      "activations/layer17_attention_weight_max": 43.35995864868164,
      "activations/layer17_attention_weight_min": -30.051843643188477,
      "activations/layer18_attention_weight_max": 25.584346771240234,
      "activations/layer18_attention_weight_min": -15.955881118774414,
      "activations/layer19_attention_weight_max": 54.01510238647461,
      "activations/layer19_attention_weight_min": -32.677772521972656,
      "activations/layer1_attention_weight_max": 17.21115493774414,
      "activations/layer1_attention_weight_min": -13.023411750793457,
      "activations/layer20_attention_weight_max": 50.83694839477539,
      "activations/layer20_attention_weight_min": -27.693880081176758,
      "activations/layer21_attention_weight_max": 26.697946548461914,
      "activations/layer21_attention_weight_min": -17.219289779663086,
      "activations/layer22_attention_weight_max": 45.17313003540039,
      "activations/layer22_attention_weight_min": -23.82132911682129,
      "activations/layer23_attention_weight_max": 25.85773468017578,
      "activations/layer23_attention_weight_min": -14.782798767089844,
      "activations/layer2_attention_weight_max": 27.29718780517578,
      "activations/layer2_attention_weight_min": -27.982330322265625,
      "activations/layer3_attention_weight_max": 67.77462005615234,
      "activations/layer3_attention_weight_min": -73.85829162597656,
      "activations/layer4_attention_weight_max": 121.34920501708984,
      "activations/layer4_attention_weight_min": -121.34517669677734,
      "activations/layer5_attention_weight_max": 125.45980834960938,
      "activations/layer5_attention_weight_min": -113.42720031738281,
      "activations/layer6_attention_weight_max": 65.1805648803711,
      "activations/layer6_attention_weight_min": -62.77618408203125,
      "activations/layer7_attention_weight_max": 66.25521850585938,
      "activations/layer7_attention_weight_min": -65.38304901123047,
      "activations/layer8_attention_weight_max": 55.43910217285156,
      "activations/layer8_attention_weight_min": -51.32355880737305,
      "activations/layer9_attention_weight_max": 71.86491394042969,
      "activations/layer9_attention_weight_min": -64.52236938476562,
      "epoch": 1.35,
      "learning_rate": 0.00014272196969696967,
      "loss": 3.0475,
      "step": 23250
    },
    {
      "activations/layer0_attention_weight_max": 15.891949653625488,
      "activations/layer0_attention_weight_min": -12.302108764648438,
      "activations/layer10_attention_weight_max": 59.689048767089844,
      "activations/layer10_attention_weight_min": -49.12876510620117,
      "activations/layer11_attention_weight_max": 55.58612060546875,
      "activations/layer11_attention_weight_min": -49.7895622253418,
      "activations/layer12_attention_weight_max": 37.4851188659668,
      "activations/layer12_attention_weight_min": -28.039913177490234,
      "activations/layer13_attention_weight_max": 58.99866485595703,
      "activations/layer13_attention_weight_min": -44.51995086669922,
      "activations/layer14_attention_weight_max": 97.06119537353516,
      "activations/layer14_attention_weight_min": -74.3558120727539,
      "activations/layer15_attention_weight_max": 63.61786651611328,
      "activations/layer15_attention_weight_min": -43.17731475830078,
      "activations/layer16_attention_weight_max": 65.95173645019531,
      "activations/layer16_attention_weight_min": -40.16178894042969,
      "activations/layer17_attention_weight_max": 44.48136520385742,
      "activations/layer17_attention_weight_min": -31.16988754272461,
      "activations/layer18_attention_weight_max": 26.082120895385742,
      "activations/layer18_attention_weight_min": -16.008806228637695,
      "activations/layer19_attention_weight_max": 66.0237808227539,
      "activations/layer19_attention_weight_min": -32.53163528442383,
      "activations/layer1_attention_weight_max": 16.561559677124023,
      "activations/layer1_attention_weight_min": -10.94470500946045,
      "activations/layer20_attention_weight_max": 49.62027359008789,
      "activations/layer20_attention_weight_min": -26.259292602539062,
      "activations/layer21_attention_weight_max": 25.60016441345215,
      "activations/layer21_attention_weight_min": -15.570408821105957,
      "activations/layer22_attention_weight_max": 46.718299865722656,
      "activations/layer22_attention_weight_min": -25.027246475219727,
      "activations/layer23_attention_weight_max": 28.7297420501709,
      "activations/layer23_attention_weight_min": -16.307741165161133,
      "activations/layer2_attention_weight_max": 27.322372436523438,
      "activations/layer2_attention_weight_min": -27.160659790039062,
      "activations/layer3_attention_weight_max": 66.13107299804688,
      "activations/layer3_attention_weight_min": -67.80357360839844,
      "activations/layer4_attention_weight_max": 120.15559387207031,
      "activations/layer4_attention_weight_min": -110.04317474365234,
      "activations/layer5_attention_weight_max": 123.87387084960938,
      "activations/layer5_attention_weight_min": -105.82290649414062,
      "activations/layer6_attention_weight_max": 65.79569244384766,
      "activations/layer6_attention_weight_min": -54.54242706298828,
      "activations/layer7_attention_weight_max": 64.22962188720703,
      "activations/layer7_attention_weight_min": -60.623844146728516,
      "activations/layer8_attention_weight_max": 64.99388885498047,
      "activations/layer8_attention_weight_min": -55.07539367675781,
      "activations/layer9_attention_weight_max": 78.78130340576172,
      "activations/layer9_attention_weight_min": -77.53045654296875,
      "epoch": 1.35,
      "learning_rate": 0.0001427030303030303,
      "loss": 3.0407,
      "step": 23300
    },
    {
      "activations/layer0_attention_weight_max": 16.296802520751953,
      "activations/layer0_attention_weight_min": -11.806289672851562,
      "activations/layer10_attention_weight_max": 60.32925796508789,
      "activations/layer10_attention_weight_min": -49.097896575927734,
      "activations/layer11_attention_weight_max": 42.6847038269043,
      "activations/layer11_attention_weight_min": -37.42833709716797,
      "activations/layer12_attention_weight_max": 35.877716064453125,
      "activations/layer12_attention_weight_min": -25.333728790283203,
      "activations/layer13_attention_weight_max": 50.901390075683594,
      "activations/layer13_attention_weight_min": -34.478363037109375,
      "activations/layer14_attention_weight_max": 78.07085418701172,
      "activations/layer14_attention_weight_min": -54.627220153808594,
      "activations/layer15_attention_weight_max": 52.44407272338867,
      "activations/layer15_attention_weight_min": -37.87257766723633,
      "activations/layer16_attention_weight_max": 57.43281173706055,
      "activations/layer16_attention_weight_min": -38.53146743774414,
      "activations/layer17_attention_weight_max": 46.32673263549805,
      "activations/layer17_attention_weight_min": -27.64029884338379,
      "activations/layer18_attention_weight_max": 26.309070587158203,
      "activations/layer18_attention_weight_min": -14.82185173034668,
      "activations/layer19_attention_weight_max": 56.44970703125,
      "activations/layer19_attention_weight_min": -35.480960845947266,
      "activations/layer1_attention_weight_max": 18.596384048461914,
      "activations/layer1_attention_weight_min": -12.700079917907715,
      "activations/layer20_attention_weight_max": 42.975746154785156,
      "activations/layer20_attention_weight_min": -26.464670181274414,
      "activations/layer21_attention_weight_max": 24.991811752319336,
      "activations/layer21_attention_weight_min": -15.211052894592285,
      "activations/layer22_attention_weight_max": 42.76106643676758,
      "activations/layer22_attention_weight_min": -24.448823928833008,
      "activations/layer23_attention_weight_max": 26.219465255737305,
      "activations/layer23_attention_weight_min": -14.893698692321777,
      "activations/layer2_attention_weight_max": 28.349544525146484,
      "activations/layer2_attention_weight_min": -27.975757598876953,
      "activations/layer3_attention_weight_max": 63.73004150390625,
      "activations/layer3_attention_weight_min": -66.64876556396484,
      "activations/layer4_attention_weight_max": 115.61030578613281,
      "activations/layer4_attention_weight_min": -109.49476623535156,
      "activations/layer5_attention_weight_max": 119.19636535644531,
      "activations/layer5_attention_weight_min": -107.48290252685547,
      "activations/layer6_attention_weight_max": 67.62751770019531,
      "activations/layer6_attention_weight_min": -59.625640869140625,
      "activations/layer7_attention_weight_max": 59.77750015258789,
      "activations/layer7_attention_weight_min": -60.778778076171875,
      "activations/layer8_attention_weight_max": 54.45180130004883,
      "activations/layer8_attention_weight_min": -49.67072296142578,
      "activations/layer9_attention_weight_max": 73.64620208740234,
      "activations/layer9_attention_weight_min": -61.387062072753906,
      "epoch": 1.36,
      "learning_rate": 0.00014268446969696968,
      "loss": 3.0285,
      "step": 23350
    },
    {
      "activations/layer0_attention_weight_max": 16.607318878173828,
      "activations/layer0_attention_weight_min": -11.811092376708984,
      "activations/layer10_attention_weight_max": 56.39377212524414,
      "activations/layer10_attention_weight_min": -55.05805587768555,
      "activations/layer11_attention_weight_max": 41.99322509765625,
      "activations/layer11_attention_weight_min": -38.081546783447266,
      "activations/layer12_attention_weight_max": 35.23023223876953,
      "activations/layer12_attention_weight_min": -32.29785919189453,
      "activations/layer13_attention_weight_max": 51.365684509277344,
      "activations/layer13_attention_weight_min": -36.78782272338867,
      "activations/layer14_attention_weight_max": 69.04509735107422,
      "activations/layer14_attention_weight_min": -57.8133659362793,
      "activations/layer15_attention_weight_max": 52.39469528198242,
      "activations/layer15_attention_weight_min": -41.96379470825195,
      "activations/layer16_attention_weight_max": 53.332237243652344,
      "activations/layer16_attention_weight_min": -38.89833068847656,
      "activations/layer17_attention_weight_max": 40.81477737426758,
      "activations/layer17_attention_weight_min": -26.540369033813477,
      "activations/layer18_attention_weight_max": 22.98837661743164,
      "activations/layer18_attention_weight_min": -16.979000091552734,
      "activations/layer19_attention_weight_max": 53.05210494995117,
      "activations/layer19_attention_weight_min": -34.82590866088867,
      "activations/layer1_attention_weight_max": 16.617435455322266,
      "activations/layer1_attention_weight_min": -11.755413055419922,
      "activations/layer20_attention_weight_max": 42.07034683227539,
      "activations/layer20_attention_weight_min": -28.02764320373535,
      "activations/layer21_attention_weight_max": 24.65028953552246,
      "activations/layer21_attention_weight_min": -17.78203582763672,
      "activations/layer22_attention_weight_max": 42.59528350830078,
      "activations/layer22_attention_weight_min": -25.56632423400879,
      "activations/layer23_attention_weight_max": 26.258087158203125,
      "activations/layer23_attention_weight_min": -15.546762466430664,
      "activations/layer2_attention_weight_max": 31.6800537109375,
      "activations/layer2_attention_weight_min": -28.082054138183594,
      "activations/layer3_attention_weight_max": 65.20885467529297,
      "activations/layer3_attention_weight_min": -67.28096008300781,
      "activations/layer4_attention_weight_max": 120.00316619873047,
      "activations/layer4_attention_weight_min": -111.53645324707031,
      "activations/layer5_attention_weight_max": 120.52748107910156,
      "activations/layer5_attention_weight_min": -101.70674896240234,
      "activations/layer6_attention_weight_max": 65.96231842041016,
      "activations/layer6_attention_weight_min": -55.38007736206055,
      "activations/layer7_attention_weight_max": 64.2098159790039,
      "activations/layer7_attention_weight_min": -70.20203399658203,
      "activations/layer8_attention_weight_max": 52.58554458618164,
      "activations/layer8_attention_weight_min": -47.74766159057617,
      "activations/layer9_attention_weight_max": 75.890869140625,
      "activations/layer9_attention_weight_min": -67.99913787841797,
      "epoch": 1.36,
      "learning_rate": 0.0001426655303030303,
      "loss": 3.0362,
      "step": 23400
    },
    {
      "activations/layer0_attention_weight_max": 16.61994743347168,
      "activations/layer0_attention_weight_min": -11.465699195861816,
      "activations/layer10_attention_weight_max": 66.8566665649414,
      "activations/layer10_attention_weight_min": -56.091400146484375,
      "activations/layer11_attention_weight_max": 67.83011627197266,
      "activations/layer11_attention_weight_min": -45.257301330566406,
      "activations/layer12_attention_weight_max": 36.55684280395508,
      "activations/layer12_attention_weight_min": -27.908756256103516,
      "activations/layer13_attention_weight_max": 56.42388916015625,
      "activations/layer13_attention_weight_min": -37.034385681152344,
      "activations/layer14_attention_weight_max": 81.71522521972656,
      "activations/layer14_attention_weight_min": -56.8957405090332,
      "activations/layer15_attention_weight_max": 65.48625946044922,
      "activations/layer15_attention_weight_min": -39.34996032714844,
      "activations/layer16_attention_weight_max": 56.44577407836914,
      "activations/layer16_attention_weight_min": -39.294307708740234,
      "activations/layer17_attention_weight_max": 43.35952377319336,
      "activations/layer17_attention_weight_min": -28.591230392456055,
      "activations/layer18_attention_weight_max": 28.94981575012207,
      "activations/layer18_attention_weight_min": -15.518061637878418,
      "activations/layer19_attention_weight_max": 53.628658294677734,
      "activations/layer19_attention_weight_min": -32.5445671081543,
      "activations/layer1_attention_weight_max": 16.97595977783203,
      "activations/layer1_attention_weight_min": -10.789280891418457,
      "activations/layer20_attention_weight_max": 46.806907653808594,
      "activations/layer20_attention_weight_min": -25.79132652282715,
      "activations/layer21_attention_weight_max": 25.23477554321289,
      "activations/layer21_attention_weight_min": -15.536467552185059,
      "activations/layer22_attention_weight_max": 43.99006652832031,
      "activations/layer22_attention_weight_min": -27.375715255737305,
      "activations/layer23_attention_weight_max": 25.97900390625,
      "activations/layer23_attention_weight_min": -14.10660457611084,
      "activations/layer2_attention_weight_max": 26.92720603942871,
      "activations/layer2_attention_weight_min": -26.656322479248047,
      "activations/layer3_attention_weight_max": 65.28244018554688,
      "activations/layer3_attention_weight_min": -65.83894348144531,
      "activations/layer4_attention_weight_max": 118.17040252685547,
      "activations/layer4_attention_weight_min": -110.13578033447266,
      "activations/layer5_attention_weight_max": 128.75155639648438,
      "activations/layer5_attention_weight_min": -106.12123107910156,
      "activations/layer6_attention_weight_max": 72.16771697998047,
      "activations/layer6_attention_weight_min": -55.76435470581055,
      "activations/layer7_attention_weight_max": 69.2685775756836,
      "activations/layer7_attention_weight_min": -60.660099029541016,
      "activations/layer8_attention_weight_max": 67.99949645996094,
      "activations/layer8_attention_weight_min": -51.24817657470703,
      "activations/layer9_attention_weight_max": 100.66429901123047,
      "activations/layer9_attention_weight_min": -74.70140838623047,
      "epoch": 1.36,
      "learning_rate": 0.0001426465909090909,
      "loss": 3.0379,
      "step": 23450
    },
    {
      "activations/layer0_attention_weight_max": 17.102935791015625,
      "activations/layer0_attention_weight_min": -12.260388374328613,
      "activations/layer10_attention_weight_max": 49.77225112915039,
      "activations/layer10_attention_weight_min": -43.94757080078125,
      "activations/layer11_attention_weight_max": 42.40574645996094,
      "activations/layer11_attention_weight_min": -34.53998947143555,
      "activations/layer12_attention_weight_max": 34.30595779418945,
      "activations/layer12_attention_weight_min": -27.56835174560547,
      "activations/layer13_attention_weight_max": 50.0173454284668,
      "activations/layer13_attention_weight_min": -37.836585998535156,
      "activations/layer14_attention_weight_max": 71.91584777832031,
      "activations/layer14_attention_weight_min": -55.951698303222656,
      "activations/layer15_attention_weight_max": 59.02534866333008,
      "activations/layer15_attention_weight_min": -42.80017852783203,
      "activations/layer16_attention_weight_max": 52.75838088989258,
      "activations/layer16_attention_weight_min": -39.16165542602539,
      "activations/layer17_attention_weight_max": 39.26573944091797,
      "activations/layer17_attention_weight_min": -28.985084533691406,
      "activations/layer18_attention_weight_max": 24.67890739440918,
      "activations/layer18_attention_weight_min": -15.527498245239258,
      "activations/layer19_attention_weight_max": 57.28989791870117,
      "activations/layer19_attention_weight_min": -34.13309860229492,
      "activations/layer1_attention_weight_max": 17.63462257385254,
      "activations/layer1_attention_weight_min": -12.009398460388184,
      "activations/layer20_attention_weight_max": 41.521480560302734,
      "activations/layer20_attention_weight_min": -27.399864196777344,
      "activations/layer21_attention_weight_max": 26.868406295776367,
      "activations/layer21_attention_weight_min": -15.949316024780273,
      "activations/layer22_attention_weight_max": 42.98802185058594,
      "activations/layer22_attention_weight_min": -25.399452209472656,
      "activations/layer23_attention_weight_max": 26.041540145874023,
      "activations/layer23_attention_weight_min": -15.197761535644531,
      "activations/layer2_attention_weight_max": 27.405967712402344,
      "activations/layer2_attention_weight_min": -29.168455123901367,
      "activations/layer3_attention_weight_max": 61.514381408691406,
      "activations/layer3_attention_weight_min": -64.11405944824219,
      "activations/layer4_attention_weight_max": 111.40472412109375,
      "activations/layer4_attention_weight_min": -111.0997314453125,
      "activations/layer5_attention_weight_max": 111.52964782714844,
      "activations/layer5_attention_weight_min": -102.30660247802734,
      "activations/layer6_attention_weight_max": 57.206565856933594,
      "activations/layer6_attention_weight_min": -52.8358268737793,
      "activations/layer7_attention_weight_max": 58.46232604980469,
      "activations/layer7_attention_weight_min": -58.59577941894531,
      "activations/layer8_attention_weight_max": 47.539180755615234,
      "activations/layer8_attention_weight_min": -44.980194091796875,
      "activations/layer9_attention_weight_max": 73.87274932861328,
      "activations/layer9_attention_weight_min": -59.35237503051758,
      "epoch": 1.37,
      "learning_rate": 0.0001426276515151515,
      "loss": 3.0256,
      "step": 23500
    },
    {
      "activations/layer0_attention_weight_max": 15.508912086486816,
      "activations/layer0_attention_weight_min": -12.013781547546387,
      "activations/layer10_attention_weight_max": 50.02277374267578,
      "activations/layer10_attention_weight_min": -42.789241790771484,
      "activations/layer11_attention_weight_max": 41.52538299560547,
      "activations/layer11_attention_weight_min": -34.58414840698242,
      "activations/layer12_attention_weight_max": 33.5277214050293,
      "activations/layer12_attention_weight_min": -26.714624404907227,
      "activations/layer13_attention_weight_max": 52.78587341308594,
      "activations/layer13_attention_weight_min": -36.96952438354492,
      "activations/layer14_attention_weight_max": 83.80083465576172,
      "activations/layer14_attention_weight_min": -60.84225082397461,
      "activations/layer15_attention_weight_max": 55.44464111328125,
      "activations/layer15_attention_weight_min": -39.43550491333008,
      "activations/layer16_attention_weight_max": 55.0576057434082,
      "activations/layer16_attention_weight_min": -38.680355072021484,
      "activations/layer17_attention_weight_max": 48.37276840209961,
      "activations/layer17_attention_weight_min": -33.62825012207031,
      "activations/layer18_attention_weight_max": 26.583606719970703,
      "activations/layer18_attention_weight_min": -15.643068313598633,
      "activations/layer19_attention_weight_max": 51.5927734375,
      "activations/layer19_attention_weight_min": -32.931541442871094,
      "activations/layer1_attention_weight_max": 17.2503719329834,
      "activations/layer1_attention_weight_min": -11.373603820800781,
      "activations/layer20_attention_weight_max": 43.908653259277344,
      "activations/layer20_attention_weight_min": -24.732547760009766,
      "activations/layer21_attention_weight_max": 23.292461395263672,
      "activations/layer21_attention_weight_min": -14.899806022644043,
      "activations/layer22_attention_weight_max": 43.68021774291992,
      "activations/layer22_attention_weight_min": -25.41712188720703,
      "activations/layer23_attention_weight_max": 27.252620697021484,
      "activations/layer23_attention_weight_min": -18.256731033325195,
      "activations/layer2_attention_weight_max": 26.730642318725586,
      "activations/layer2_attention_weight_min": -26.83489990234375,
      "activations/layer3_attention_weight_max": 63.59275436401367,
      "activations/layer3_attention_weight_min": -65.79408264160156,
      "activations/layer4_attention_weight_max": 114.0179672241211,
      "activations/layer4_attention_weight_min": -108.7382583618164,
      "activations/layer5_attention_weight_max": 119.91136932373047,
      "activations/layer5_attention_weight_min": -103.58590698242188,
      "activations/layer6_attention_weight_max": 65.35906219482422,
      "activations/layer6_attention_weight_min": -54.358619689941406,
      "activations/layer7_attention_weight_max": 62.09761047363281,
      "activations/layer7_attention_weight_min": -61.235443115234375,
      "activations/layer8_attention_weight_max": 52.388885498046875,
      "activations/layer8_attention_weight_min": -46.84959411621094,
      "activations/layer9_attention_weight_max": 80.1692123413086,
      "activations/layer9_attention_weight_min": -63.838348388671875,
      "epoch": 1.37,
      "learning_rate": 0.0001426087121212121,
      "loss": 3.0233,
      "step": 23550
    },
    {
      "activations/layer0_attention_weight_max": 16.025005340576172,
      "activations/layer0_attention_weight_min": -11.923296928405762,
      "activations/layer10_attention_weight_max": 49.177513122558594,
      "activations/layer10_attention_weight_min": -46.98280715942383,
      "activations/layer11_attention_weight_max": 45.385494232177734,
      "activations/layer11_attention_weight_min": -37.56050491333008,
      "activations/layer12_attention_weight_max": 73.4132080078125,
      "activations/layer12_attention_weight_min": -37.45596694946289,
      "activations/layer13_attention_weight_max": 47.90592575073242,
      "activations/layer13_attention_weight_min": -39.148040771484375,
      "activations/layer14_attention_weight_max": 74.98131561279297,
      "activations/layer14_attention_weight_min": -55.31650924682617,
      "activations/layer15_attention_weight_max": 56.575286865234375,
      "activations/layer15_attention_weight_min": -38.869022369384766,
      "activations/layer16_attention_weight_max": 62.20927047729492,
      "activations/layer16_attention_weight_min": -40.091556549072266,
      "activations/layer17_attention_weight_max": 42.004127502441406,
      "activations/layer17_attention_weight_min": -30.392642974853516,
      "activations/layer18_attention_weight_max": 23.734344482421875,
      "activations/layer18_attention_weight_min": -15.384809494018555,
      "activations/layer19_attention_weight_max": 57.59932327270508,
      "activations/layer19_attention_weight_min": -32.331504821777344,
      "activations/layer1_attention_weight_max": 17.596376419067383,
      "activations/layer1_attention_weight_min": -12.539795875549316,
      "activations/layer20_attention_weight_max": 46.104740142822266,
      "activations/layer20_attention_weight_min": -25.361900329589844,
      "activations/layer21_attention_weight_max": 25.592391967773438,
      "activations/layer21_attention_weight_min": -16.32574462890625,
      "activations/layer22_attention_weight_max": 40.177757263183594,
      "activations/layer22_attention_weight_min": -24.479124069213867,
      "activations/layer23_attention_weight_max": 22.616161346435547,
      "activations/layer23_attention_weight_min": -15.474002838134766,
      "activations/layer2_attention_weight_max": 28.26453399658203,
      "activations/layer2_attention_weight_min": -28.761009216308594,
      "activations/layer3_attention_weight_max": 65.43310546875,
      "activations/layer3_attention_weight_min": -66.83069610595703,
      "activations/layer4_attention_weight_max": 117.80651092529297,
      "activations/layer4_attention_weight_min": -111.9688720703125,
      "activations/layer5_attention_weight_max": 119.19987487792969,
      "activations/layer5_attention_weight_min": -106.05693817138672,
      "activations/layer6_attention_weight_max": 64.96199798583984,
      "activations/layer6_attention_weight_min": -57.23847198486328,
      "activations/layer7_attention_weight_max": 65.72496795654297,
      "activations/layer7_attention_weight_min": -66.46207427978516,
      "activations/layer8_attention_weight_max": 52.3157958984375,
      "activations/layer8_attention_weight_min": -48.75484848022461,
      "activations/layer9_attention_weight_max": 72.56890106201172,
      "activations/layer9_attention_weight_min": -63.99945068359375,
      "epoch": 1.37,
      "learning_rate": 0.00014258977272727273,
      "loss": 3.0375,
      "step": 23600
    },
    {
      "activations/layer0_attention_weight_max": 15.672174453735352,
      "activations/layer0_attention_weight_min": -11.973475456237793,
      "activations/layer10_attention_weight_max": 59.61908721923828,
      "activations/layer10_attention_weight_min": -46.180084228515625,
      "activations/layer11_attention_weight_max": 50.53120040893555,
      "activations/layer11_attention_weight_min": -38.66083526611328,
      "activations/layer12_attention_weight_max": 44.17938232421875,
      "activations/layer12_attention_weight_min": -28.915088653564453,
      "activations/layer13_attention_weight_max": 52.231956481933594,
      "activations/layer13_attention_weight_min": -38.18114471435547,
      "activations/layer14_attention_weight_max": 83.17083740234375,
      "activations/layer14_attention_weight_min": -57.22766876220703,
      "activations/layer15_attention_weight_max": 63.035152435302734,
      "activations/layer15_attention_weight_min": -42.00332260131836,
      "activations/layer16_attention_weight_max": 56.66592788696289,
      "activations/layer16_attention_weight_min": -39.22513198852539,
      "activations/layer17_attention_weight_max": 40.4655647277832,
      "activations/layer17_attention_weight_min": -28.96934700012207,
      "activations/layer18_attention_weight_max": 25.835235595703125,
      "activations/layer18_attention_weight_min": -16.24894905090332,
      "activations/layer19_attention_weight_max": 48.71036911010742,
      "activations/layer19_attention_weight_min": -35.196903228759766,
      "activations/layer1_attention_weight_max": 17.0158748626709,
      "activations/layer1_attention_weight_min": -10.689729690551758,
      "activations/layer20_attention_weight_max": 37.22105407714844,
      "activations/layer20_attention_weight_min": -25.098289489746094,
      "activations/layer21_attention_weight_max": 29.25656509399414,
      "activations/layer21_attention_weight_min": -18.239124298095703,
      "activations/layer22_attention_weight_max": 37.845726013183594,
      "activations/layer22_attention_weight_min": -25.38580322265625,
      "activations/layer23_attention_weight_max": 21.724681854248047,
      "activations/layer23_attention_weight_min": -17.737987518310547,
      "activations/layer2_attention_weight_max": 28.020580291748047,
      "activations/layer2_attention_weight_min": -27.95679473876953,
      "activations/layer3_attention_weight_max": 62.58244323730469,
      "activations/layer3_attention_weight_min": -66.36841583251953,
      "activations/layer4_attention_weight_max": 114.46073150634766,
      "activations/layer4_attention_weight_min": -109.42253875732422,
      "activations/layer5_attention_weight_max": 116.59980773925781,
      "activations/layer5_attention_weight_min": -102.98619079589844,
      "activations/layer6_attention_weight_max": 67.6090316772461,
      "activations/layer6_attention_weight_min": -56.72709274291992,
      "activations/layer7_attention_weight_max": 69.86233520507812,
      "activations/layer7_attention_weight_min": -61.628150939941406,
      "activations/layer8_attention_weight_max": 57.8536262512207,
      "activations/layer8_attention_weight_min": -47.27731704711914,
      "activations/layer9_attention_weight_max": 97.46363830566406,
      "activations/layer9_attention_weight_min": -71.03349304199219,
      "epoch": 1.37,
      "learning_rate": 0.00014257083333333334,
      "loss": 3.0554,
      "step": 23650
    },
    {
      "activations/layer0_attention_weight_max": 16.278148651123047,
      "activations/layer0_attention_weight_min": -12.535977363586426,
      "activations/layer10_attention_weight_max": 59.484275817871094,
      "activations/layer10_attention_weight_min": -48.35219192504883,
      "activations/layer11_attention_weight_max": 43.87312316894531,
      "activations/layer11_attention_weight_min": -36.58831787109375,
      "activations/layer12_attention_weight_max": 31.4216251373291,
      "activations/layer12_attention_weight_min": -28.37984848022461,
      "activations/layer13_attention_weight_max": 48.37492370605469,
      "activations/layer13_attention_weight_min": -36.889644622802734,
      "activations/layer14_attention_weight_max": 83.29974365234375,
      "activations/layer14_attention_weight_min": -56.33815383911133,
      "activations/layer15_attention_weight_max": 62.62785339355469,
      "activations/layer15_attention_weight_min": -39.03241729736328,
      "activations/layer16_attention_weight_max": 64.3545913696289,
      "activations/layer16_attention_weight_min": -40.65773391723633,
      "activations/layer17_attention_weight_max": 53.83528137207031,
      "activations/layer17_attention_weight_min": -28.275203704833984,
      "activations/layer18_attention_weight_max": 27.624284744262695,
      "activations/layer18_attention_weight_min": -14.619257926940918,
      "activations/layer19_attention_weight_max": 65.42632293701172,
      "activations/layer19_attention_weight_min": -33.649940490722656,
      "activations/layer1_attention_weight_max": 17.36797523498535,
      "activations/layer1_attention_weight_min": -12.180941581726074,
      "activations/layer20_attention_weight_max": 47.077598571777344,
      "activations/layer20_attention_weight_min": -24.809282302856445,
      "activations/layer21_attention_weight_max": 30.73944091796875,
      "activations/layer21_attention_weight_min": -17.52985191345215,
      "activations/layer22_attention_weight_max": 47.62567138671875,
      "activations/layer22_attention_weight_min": -26.431245803833008,
      "activations/layer23_attention_weight_max": 27.252689361572266,
      "activations/layer23_attention_weight_min": -14.477909088134766,
      "activations/layer2_attention_weight_max": 30.250646591186523,
      "activations/layer2_attention_weight_min": -29.901283264160156,
      "activations/layer3_attention_weight_max": 68.29302978515625,
      "activations/layer3_attention_weight_min": -67.70970916748047,
      "activations/layer4_attention_weight_max": 121.03231048583984,
      "activations/layer4_attention_weight_min": -112.09553527832031,
      "activations/layer5_attention_weight_max": 127.76776123046875,
      "activations/layer5_attention_weight_min": -107.76744079589844,
      "activations/layer6_attention_weight_max": 66.04544067382812,
      "activations/layer6_attention_weight_min": -56.4953727722168,
      "activations/layer7_attention_weight_max": 63.839900970458984,
      "activations/layer7_attention_weight_min": -61.4686279296875,
      "activations/layer8_attention_weight_max": 54.67417526245117,
      "activations/layer8_attention_weight_min": -48.567405700683594,
      "activations/layer9_attention_weight_max": 83.19342041015625,
      "activations/layer9_attention_weight_min": -64.43114471435547,
      "epoch": 1.38,
      "learning_rate": 0.00014255189393939393,
      "loss": 3.0203,
      "step": 23700
    },
    {
      "activations/layer0_attention_weight_max": 15.432327270507812,
      "activations/layer0_attention_weight_min": -12.974785804748535,
      "activations/layer10_attention_weight_max": 48.98817825317383,
      "activations/layer10_attention_weight_min": -44.94649124145508,
      "activations/layer11_attention_weight_max": 39.72868347167969,
      "activations/layer11_attention_weight_min": -36.47904968261719,
      "activations/layer12_attention_weight_max": 31.01345443725586,
      "activations/layer12_attention_weight_min": -29.11393928527832,
      "activations/layer13_attention_weight_max": 46.63343811035156,
      "activations/layer13_attention_weight_min": -36.426326751708984,
      "activations/layer14_attention_weight_max": 75.24919128417969,
      "activations/layer14_attention_weight_min": -54.402896881103516,
      "activations/layer15_attention_weight_max": 52.25797653198242,
      "activations/layer15_attention_weight_min": -39.780967712402344,
      "activations/layer16_attention_weight_max": 61.60074996948242,
      "activations/layer16_attention_weight_min": -44.45731735229492,
      "activations/layer17_attention_weight_max": 43.4191780090332,
      "activations/layer17_attention_weight_min": -30.89681053161621,
      "activations/layer18_attention_weight_max": 24.317861557006836,
      "activations/layer18_attention_weight_min": -15.453707695007324,
      "activations/layer19_attention_weight_max": 55.633296966552734,
      "activations/layer19_attention_weight_min": -34.09685516357422,
      "activations/layer1_attention_weight_max": 17.324970245361328,
      "activations/layer1_attention_weight_min": -12.494818687438965,
      "activations/layer20_attention_weight_max": 38.69222640991211,
      "activations/layer20_attention_weight_min": -29.3438663482666,
      "activations/layer21_attention_weight_max": 24.483684539794922,
      "activations/layer21_attention_weight_min": -18.604631423950195,
      "activations/layer22_attention_weight_max": 42.96834182739258,
      "activations/layer22_attention_weight_min": -23.299985885620117,
      "activations/layer23_attention_weight_max": 24.838539123535156,
      "activations/layer23_attention_weight_min": -13.28577995300293,
      "activations/layer2_attention_weight_max": 29.323009490966797,
      "activations/layer2_attention_weight_min": -27.040237426757812,
      "activations/layer3_attention_weight_max": 67.42589569091797,
      "activations/layer3_attention_weight_min": -67.82207489013672,
      "activations/layer4_attention_weight_max": 122.0124740600586,
      "activations/layer4_attention_weight_min": -113.34008026123047,
      "activations/layer5_attention_weight_max": 117.94313049316406,
      "activations/layer5_attention_weight_min": -101.05642700195312,
      "activations/layer6_attention_weight_max": 64.11720275878906,
      "activations/layer6_attention_weight_min": -55.610294342041016,
      "activations/layer7_attention_weight_max": 65.14399719238281,
      "activations/layer7_attention_weight_min": -66.67803955078125,
      "activations/layer8_attention_weight_max": 53.82948684692383,
      "activations/layer8_attention_weight_min": -47.69538879394531,
      "activations/layer9_attention_weight_max": 69.5182113647461,
      "activations/layer9_attention_weight_min": -59.2133903503418,
      "epoch": 1.38,
      "learning_rate": 0.00014253295454545454,
      "loss": 3.0455,
      "step": 23750
    },
    {
      "activations/layer0_attention_weight_max": 16.523550033569336,
      "activations/layer0_attention_weight_min": -12.304734230041504,
      "activations/layer10_attention_weight_max": 56.12701416015625,
      "activations/layer10_attention_weight_min": -51.68259811401367,
      "activations/layer11_attention_weight_max": 42.725059509277344,
      "activations/layer11_attention_weight_min": -36.379661560058594,
      "activations/layer12_attention_weight_max": 29.67239761352539,
      "activations/layer12_attention_weight_min": -25.370643615722656,
      "activations/layer13_attention_weight_max": 41.717140197753906,
      "activations/layer13_attention_weight_min": -35.97365951538086,
      "activations/layer14_attention_weight_max": 68.73881530761719,
      "activations/layer14_attention_weight_min": -55.74778747558594,
      "activations/layer15_attention_weight_max": 54.93937683105469,
      "activations/layer15_attention_weight_min": -42.38676452636719,
      "activations/layer16_attention_weight_max": 57.130428314208984,
      "activations/layer16_attention_weight_min": -41.55986022949219,
      "activations/layer17_attention_weight_max": 39.31559753417969,
      "activations/layer17_attention_weight_min": -26.90311622619629,
      "activations/layer18_attention_weight_max": 21.393327713012695,
      "activations/layer18_attention_weight_min": -17.643386840820312,
      "activations/layer19_attention_weight_max": 56.566871643066406,
      "activations/layer19_attention_weight_min": -28.521747589111328,
      "activations/layer1_attention_weight_max": 17.245389938354492,
      "activations/layer1_attention_weight_min": -11.874942779541016,
      "activations/layer20_attention_weight_max": 42.139896392822266,
      "activations/layer20_attention_weight_min": -25.26729393005371,
      "activations/layer21_attention_weight_max": 25.75956153869629,
      "activations/layer21_attention_weight_min": -18.60672378540039,
      "activations/layer22_attention_weight_max": 41.30379867553711,
      "activations/layer22_attention_weight_min": -25.811553955078125,
      "activations/layer23_attention_weight_max": 24.035259246826172,
      "activations/layer23_attention_weight_min": -14.741859436035156,
      "activations/layer2_attention_weight_max": 25.651655197143555,
      "activations/layer2_attention_weight_min": -26.24832534790039,
      "activations/layer3_attention_weight_max": 63.88174819946289,
      "activations/layer3_attention_weight_min": -66.79209899902344,
      "activations/layer4_attention_weight_max": 117.69486236572266,
      "activations/layer4_attention_weight_min": -113.91802978515625,
      "activations/layer5_attention_weight_max": 116.30712127685547,
      "activations/layer5_attention_weight_min": -102.63301849365234,
      "activations/layer6_attention_weight_max": 62.228729248046875,
      "activations/layer6_attention_weight_min": -54.473384857177734,
      "activations/layer7_attention_weight_max": 60.22318649291992,
      "activations/layer7_attention_weight_min": -62.27797317504883,
      "activations/layer8_attention_weight_max": 49.0582160949707,
      "activations/layer8_attention_weight_min": -45.18397903442383,
      "activations/layer9_attention_weight_max": 70.58214569091797,
      "activations/layer9_attention_weight_min": -69.37078094482422,
      "epoch": 1.38,
      "learning_rate": 0.00014251401515151513,
      "loss": 3.0281,
      "step": 23800
    },
    {
      "activations/layer0_attention_weight_max": 16.02468490600586,
      "activations/layer0_attention_weight_min": -12.387653350830078,
      "activations/layer10_attention_weight_max": 51.38187789916992,
      "activations/layer10_attention_weight_min": -47.97172164916992,
      "activations/layer11_attention_weight_max": 39.707489013671875,
      "activations/layer11_attention_weight_min": -38.970367431640625,
      "activations/layer12_attention_weight_max": 31.496932983398438,
      "activations/layer12_attention_weight_min": -25.45917510986328,
      "activations/layer13_attention_weight_max": 47.004432678222656,
      "activations/layer13_attention_weight_min": -37.42658615112305,
      "activations/layer14_attention_weight_max": 72.33120727539062,
      "activations/layer14_attention_weight_min": -60.27070236206055,
      "activations/layer15_attention_weight_max": 53.78908920288086,
      "activations/layer15_attention_weight_min": -39.49269485473633,
      "activations/layer16_attention_weight_max": 52.0338020324707,
      "activations/layer16_attention_weight_min": -41.322105407714844,
      "activations/layer17_attention_weight_max": 45.909610748291016,
      "activations/layer17_attention_weight_min": -32.23332595825195,
      "activations/layer18_attention_weight_max": 25.123685836791992,
      "activations/layer18_attention_weight_min": -15.686539649963379,
      "activations/layer19_attention_weight_max": 53.11835479736328,
      "activations/layer19_attention_weight_min": -33.94866943359375,
      "activations/layer1_attention_weight_max": 16.882200241088867,
      "activations/layer1_attention_weight_min": -11.100526809692383,
      "activations/layer20_attention_weight_max": 41.866817474365234,
      "activations/layer20_attention_weight_min": -29.701948165893555,
      "activations/layer21_attention_weight_max": 24.672758102416992,
      "activations/layer21_attention_weight_min": -16.934040069580078,
      "activations/layer22_attention_weight_max": 43.386051177978516,
      "activations/layer22_attention_weight_min": -24.299097061157227,
      "activations/layer23_attention_weight_max": 24.482023239135742,
      "activations/layer23_attention_weight_min": -14.576166152954102,
      "activations/layer2_attention_weight_max": 27.28400421142578,
      "activations/layer2_attention_weight_min": -28.179359436035156,
      "activations/layer3_attention_weight_max": 66.4155502319336,
      "activations/layer3_attention_weight_min": -66.59487915039062,
      "activations/layer4_attention_weight_max": 116.4329605102539,
      "activations/layer4_attention_weight_min": -112.47334289550781,
      "activations/layer5_attention_weight_max": 115.37786102294922,
      "activations/layer5_attention_weight_min": -102.69499969482422,
      "activations/layer6_attention_weight_max": 60.4428596496582,
      "activations/layer6_attention_weight_min": -58.77446365356445,
      "activations/layer7_attention_weight_max": 60.79015350341797,
      "activations/layer7_attention_weight_min": -60.34941101074219,
      "activations/layer8_attention_weight_max": 51.82596206665039,
      "activations/layer8_attention_weight_min": -48.46787643432617,
      "activations/layer9_attention_weight_max": 71.48897552490234,
      "activations/layer9_attention_weight_min": -61.8746452331543,
      "epoch": 1.39,
      "learning_rate": 0.00014249507575757575,
      "loss": 3.0206,
      "step": 23850
    },
    {
      "activations/layer0_attention_weight_max": 15.865818977355957,
      "activations/layer0_attention_weight_min": -12.261284828186035,
      "activations/layer10_attention_weight_max": 54.118377685546875,
      "activations/layer10_attention_weight_min": -47.089473724365234,
      "activations/layer11_attention_weight_max": 48.70518493652344,
      "activations/layer11_attention_weight_min": -37.4592170715332,
      "activations/layer12_attention_weight_max": 32.52546691894531,
      "activations/layer12_attention_weight_min": -26.89344596862793,
      "activations/layer13_attention_weight_max": 51.211830139160156,
      "activations/layer13_attention_weight_min": -37.487342834472656,
      "activations/layer14_attention_weight_max": 92.0788345336914,
      "activations/layer14_attention_weight_min": -66.31130981445312,
      "activations/layer15_attention_weight_max": 57.64316177368164,
      "activations/layer15_attention_weight_min": -41.56318664550781,
      "activations/layer16_attention_weight_max": 66.71651458740234,
      "activations/layer16_attention_weight_min": -40.23482894897461,
      "activations/layer17_attention_weight_max": 49.374755859375,
      "activations/layer17_attention_weight_min": -32.149742126464844,
      "activations/layer18_attention_weight_max": 26.173660278320312,
      "activations/layer18_attention_weight_min": -15.841591835021973,
      "activations/layer19_attention_weight_max": 60.10077667236328,
      "activations/layer19_attention_weight_min": -34.184932708740234,
      "activations/layer1_attention_weight_max": 17.991352081298828,
      "activations/layer1_attention_weight_min": -11.213419914245605,
      "activations/layer20_attention_weight_max": 43.640499114990234,
      "activations/layer20_attention_weight_min": -26.08315658569336,
      "activations/layer21_attention_weight_max": 24.91727066040039,
      "activations/layer21_attention_weight_min": -16.090185165405273,
      "activations/layer22_attention_weight_max": 42.36793518066406,
      "activations/layer22_attention_weight_min": -25.47948455810547,
      "activations/layer23_attention_weight_max": 26.817237854003906,
      "activations/layer23_attention_weight_min": -14.461284637451172,
      "activations/layer2_attention_weight_max": 30.269941329956055,
      "activations/layer2_attention_weight_min": -28.260183334350586,
      "activations/layer3_attention_weight_max": 63.9956169128418,
      "activations/layer3_attention_weight_min": -67.06963348388672,
      "activations/layer4_attention_weight_max": 116.36681365966797,
      "activations/layer4_attention_weight_min": -107.44390869140625,
      "activations/layer5_attention_weight_max": 125.38948059082031,
      "activations/layer5_attention_weight_min": -103.9120101928711,
      "activations/layer6_attention_weight_max": 67.4010009765625,
      "activations/layer6_attention_weight_min": -52.65324401855469,
      "activations/layer7_attention_weight_max": 61.98066711425781,
      "activations/layer7_attention_weight_min": -61.615596771240234,
      "activations/layer8_attention_weight_max": 56.23934555053711,
      "activations/layer8_attention_weight_min": -47.82624816894531,
      "activations/layer9_attention_weight_max": 78.2329330444336,
      "activations/layer9_attention_weight_min": -65.21305847167969,
      "epoch": 1.39,
      "learning_rate": 0.00014247613636363636,
      "loss": 3.045,
      "step": 23900
    },
    {
      "activations/layer0_attention_weight_max": 16.23389434814453,
      "activations/layer0_attention_weight_min": -12.040513038635254,
      "activations/layer10_attention_weight_max": 55.67247009277344,
      "activations/layer10_attention_weight_min": -45.099449157714844,
      "activations/layer11_attention_weight_max": 44.22624206542969,
      "activations/layer11_attention_weight_min": -37.56221389770508,
      "activations/layer12_attention_weight_max": 39.59737014770508,
      "activations/layer12_attention_weight_min": -26.52992057800293,
      "activations/layer13_attention_weight_max": 60.80096435546875,
      "activations/layer13_attention_weight_min": -39.74518966674805,
      "activations/layer14_attention_weight_max": 91.82440948486328,
      "activations/layer14_attention_weight_min": -58.88082504272461,
      "activations/layer15_attention_weight_max": 61.95864486694336,
      "activations/layer15_attention_weight_min": -39.42220687866211,
      "activations/layer16_attention_weight_max": 60.15457534790039,
      "activations/layer16_attention_weight_min": -39.56703186035156,
      "activations/layer17_attention_weight_max": 51.5832405090332,
      "activations/layer17_attention_weight_min": -34.58007049560547,
      "activations/layer18_attention_weight_max": 31.846702575683594,
      "activations/layer18_attention_weight_min": -15.273089408874512,
      "activations/layer19_attention_weight_max": 64.6198959350586,
      "activations/layer19_attention_weight_min": -34.63148880004883,
      "activations/layer1_attention_weight_max": 17.04510498046875,
      "activations/layer1_attention_weight_min": -11.661685943603516,
      "activations/layer20_attention_weight_max": 52.9748649597168,
      "activations/layer20_attention_weight_min": -28.716501235961914,
      "activations/layer21_attention_weight_max": 31.04513168334961,
      "activations/layer21_attention_weight_min": -16.96653938293457,
      "activations/layer22_attention_weight_max": 57.09429931640625,
      "activations/layer22_attention_weight_min": -21.16470718383789,
      "activations/layer23_attention_weight_max": 29.0325984954834,
      "activations/layer23_attention_weight_min": -14.569643020629883,
      "activations/layer2_attention_weight_max": 29.926979064941406,
      "activations/layer2_attention_weight_min": -29.961231231689453,
      "activations/layer3_attention_weight_max": 65.67796325683594,
      "activations/layer3_attention_weight_min": -68.96631622314453,
      "activations/layer4_attention_weight_max": 123.20729064941406,
      "activations/layer4_attention_weight_min": -110.00428771972656,
      "activations/layer5_attention_weight_max": 129.01486206054688,
      "activations/layer5_attention_weight_min": -102.89631652832031,
      "activations/layer6_attention_weight_max": 70.61092376708984,
      "activations/layer6_attention_weight_min": -54.13275909423828,
      "activations/layer7_attention_weight_max": 70.69925689697266,
      "activations/layer7_attention_weight_min": -57.882530212402344,
      "activations/layer8_attention_weight_max": 52.74702072143555,
      "activations/layer8_attention_weight_min": -46.091976165771484,
      "activations/layer9_attention_weight_max": 67.97130584716797,
      "activations/layer9_attention_weight_min": -56.0592155456543,
      "epoch": 1.39,
      "learning_rate": 0.00014245719696969695,
      "loss": 3.023,
      "step": 23950
    },
    {
      "activations/layer0_attention_weight_max": 15.938516616821289,
      "activations/layer0_attention_weight_min": -12.256986618041992,
      "activations/layer10_attention_weight_max": 58.04874801635742,
      "activations/layer10_attention_weight_min": -45.407352447509766,
      "activations/layer11_attention_weight_max": 40.21826171875,
      "activations/layer11_attention_weight_min": -35.15074920654297,
      "activations/layer12_attention_weight_max": 40.92198944091797,
      "activations/layer12_attention_weight_min": -28.52155113220215,
      "activations/layer13_attention_weight_max": 50.18944549560547,
      "activations/layer13_attention_weight_min": -36.7606201171875,
      "activations/layer14_attention_weight_max": 75.8909683227539,
      "activations/layer14_attention_weight_min": -55.179595947265625,
      "activations/layer15_attention_weight_max": 65.90147399902344,
      "activations/layer15_attention_weight_min": -46.02915573120117,
      "activations/layer16_attention_weight_max": 53.23878860473633,
      "activations/layer16_attention_weight_min": -41.47538757324219,
      "activations/layer17_attention_weight_max": 51.42473602294922,
      "activations/layer17_attention_weight_min": -29.085168838500977,
      "activations/layer18_attention_weight_max": 25.316932678222656,
      "activations/layer18_attention_weight_min": -16.008407592773438,
      "activations/layer19_attention_weight_max": 54.93174362182617,
      "activations/layer19_attention_weight_min": -33.90450668334961,
      "activations/layer1_attention_weight_max": 17.501937866210938,
      "activations/layer1_attention_weight_min": -10.479253768920898,
      "activations/layer20_attention_weight_max": 44.08250427246094,
      "activations/layer20_attention_weight_min": -27.82445526123047,
      "activations/layer21_attention_weight_max": 26.255447387695312,
      "activations/layer21_attention_weight_min": -16.321237564086914,
      "activations/layer22_attention_weight_max": 44.90744400024414,
      "activations/layer22_attention_weight_min": -26.921175003051758,
      "activations/layer23_attention_weight_max": 25.016559600830078,
      "activations/layer23_attention_weight_min": -14.38819408416748,
      "activations/layer2_attention_weight_max": 28.856769561767578,
      "activations/layer2_attention_weight_min": -25.69620132446289,
      "activations/layer3_attention_weight_max": 62.525550842285156,
      "activations/layer3_attention_weight_min": -63.19774627685547,
      "activations/layer4_attention_weight_max": 119.26710510253906,
      "activations/layer4_attention_weight_min": -108.44563293457031,
      "activations/layer5_attention_weight_max": 124.56800842285156,
      "activations/layer5_attention_weight_min": -102.71122741699219,
      "activations/layer6_attention_weight_max": 65.76915740966797,
      "activations/layer6_attention_weight_min": -52.02448272705078,
      "activations/layer7_attention_weight_max": 65.07432556152344,
      "activations/layer7_attention_weight_min": -63.050018310546875,
      "activations/layer8_attention_weight_max": 58.68663787841797,
      "activations/layer8_attention_weight_min": -43.1832275390625,
      "activations/layer9_attention_weight_max": 78.31209564208984,
      "activations/layer9_attention_weight_min": -64.03487396240234,
      "epoch": 1.39,
      "learning_rate": 0.00014243825757575756,
      "loss": 3.0335,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_loss": 2.98046875,
      "eval_runtime": 8.5426,
      "eval_samples_per_second": 502.655,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_openwebtext_loss": 2.98046875,
      "eval_openwebtext_ppl": 19.697047472120772,
      "eval_openwebtext_runtime": 8.5426,
      "eval_openwebtext_samples_per_second": 502.655,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_wikitext_loss": 3.2578125,
      "eval_wikitext_ppl": 25.992616060150944,
      "eval_wikitext_runtime": 2.0236,
      "eval_wikitext_samples_per_second": 225.343,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_lambada_loss": 3.33203125,
      "eval_lambada_ppl": 27.995149135337257,
      "eval_lambada_runtime": 9.6132,
      "eval_lambada_samples_per_second": 506.491,
      "step": 24000
    },
    {
      "activations/layer0_attention_weight_max": 13.862447738647461,
      "activations/layer0_attention_weight_min": -12.120022773742676,
      "activations/layer10_attention_weight_max": 53.080352783203125,
      "activations/layer10_attention_weight_min": -48.42737579345703,
      "activations/layer11_attention_weight_max": 39.52873229980469,
      "activations/layer11_attention_weight_min": -38.37550354003906,
      "activations/layer12_attention_weight_max": 48.877864837646484,
      "activations/layer12_attention_weight_min": -39.61616134643555,
      "activations/layer13_attention_weight_max": 50.188655853271484,
      "activations/layer13_attention_weight_min": -39.52873992919922,
      "activations/layer14_attention_weight_max": 80.56739807128906,
      "activations/layer14_attention_weight_min": -61.586280822753906,
      "activations/layer15_attention_weight_max": 57.882015228271484,
      "activations/layer15_attention_weight_min": -41.8234977722168,
      "activations/layer16_attention_weight_max": 60.04718017578125,
      "activations/layer16_attention_weight_min": -39.99428939819336,
      "activations/layer17_attention_weight_max": 42.81434631347656,
      "activations/layer17_attention_weight_min": -27.229145050048828,
      "activations/layer18_attention_weight_max": 24.896427154541016,
      "activations/layer18_attention_weight_min": -17.713382720947266,
      "activations/layer19_attention_weight_max": 55.9376335144043,
      "activations/layer19_attention_weight_min": -34.28246307373047,
      "activations/layer1_attention_weight_max": 16.72702980041504,
      "activations/layer1_attention_weight_min": -12.701687812805176,
      "activations/layer20_attention_weight_max": 41.01337814331055,
      "activations/layer20_attention_weight_min": -23.651853561401367,
      "activations/layer21_attention_weight_max": 27.7142391204834,
      "activations/layer21_attention_weight_min": -17.273866653442383,
      "activations/layer22_attention_weight_max": 44.25568389892578,
      "activations/layer22_attention_weight_min": -25.292144775390625,
      "activations/layer23_attention_weight_max": 26.566864013671875,
      "activations/layer23_attention_weight_min": -16.61618995666504,
      "activations/layer2_attention_weight_max": 28.495769500732422,
      "activations/layer2_attention_weight_min": -27.971546173095703,
      "activations/layer3_attention_weight_max": 63.36943817138672,
      "activations/layer3_attention_weight_min": -67.52621459960938,
      "activations/layer4_attention_weight_max": 111.75923156738281,
      "activations/layer4_attention_weight_min": -108.20344543457031,
      "activations/layer5_attention_weight_max": 115.85932922363281,
      "activations/layer5_attention_weight_min": -101.54202270507812,
      "activations/layer6_attention_weight_max": 62.97211837768555,
      "activations/layer6_attention_weight_min": -54.11238479614258,
      "activations/layer7_attention_weight_max": 65.4095458984375,
      "activations/layer7_attention_weight_min": -61.82286834716797,
      "activations/layer8_attention_weight_max": 51.29106521606445,
      "activations/layer8_attention_weight_min": -48.6946907043457,
      "activations/layer9_attention_weight_max": 70.36699676513672,
      "activations/layer9_attention_weight_min": -61.75103759765625,
      "epoch": 1.4,
      "learning_rate": 0.00014241931818181818,
      "loss": 3.0377,
      "step": 24050
    },
    {
      "activations/layer0_attention_weight_max": 16.130699157714844,
      "activations/layer0_attention_weight_min": -11.834550857543945,
      "activations/layer10_attention_weight_max": 61.11433029174805,
      "activations/layer10_attention_weight_min": -48.81519317626953,
      "activations/layer11_attention_weight_max": 41.26539611816406,
      "activations/layer11_attention_weight_min": -37.22528076171875,
      "activations/layer12_attention_weight_max": 53.291683197021484,
      "activations/layer12_attention_weight_min": -39.65279006958008,
      "activations/layer13_attention_weight_max": 53.8991584777832,
      "activations/layer13_attention_weight_min": -38.901668548583984,
      "activations/layer14_attention_weight_max": 81.38786315917969,
      "activations/layer14_attention_weight_min": -58.346275329589844,
      "activations/layer15_attention_weight_max": 66.96996307373047,
      "activations/layer15_attention_weight_min": -45.403663635253906,
      "activations/layer16_attention_weight_max": 57.35932922363281,
      "activations/layer16_attention_weight_min": -37.024024963378906,
      "activations/layer17_attention_weight_max": 50.687774658203125,
      "activations/layer17_attention_weight_min": -34.78510665893555,
      "activations/layer18_attention_weight_max": 25.68996238708496,
      "activations/layer18_attention_weight_min": -15.602781295776367,
      "activations/layer19_attention_weight_max": 54.704833984375,
      "activations/layer19_attention_weight_min": -31.422494888305664,
      "activations/layer1_attention_weight_max": 18.385726928710938,
      "activations/layer1_attention_weight_min": -11.729397773742676,
      "activations/layer20_attention_weight_max": 46.08650588989258,
      "activations/layer20_attention_weight_min": -25.4227352142334,
      "activations/layer21_attention_weight_max": 27.804367065429688,
      "activations/layer21_attention_weight_min": -16.236852645874023,
      "activations/layer22_attention_weight_max": 42.66033935546875,
      "activations/layer22_attention_weight_min": -24.006887435913086,
      "activations/layer23_attention_weight_max": 25.343961715698242,
      "activations/layer23_attention_weight_min": -14.696173667907715,
      "activations/layer2_attention_weight_max": 28.293537139892578,
      "activations/layer2_attention_weight_min": -27.356613159179688,
      "activations/layer3_attention_weight_max": 65.45948028564453,
      "activations/layer3_attention_weight_min": -68.38066864013672,
      "activations/layer4_attention_weight_max": 120.25296783447266,
      "activations/layer4_attention_weight_min": -111.16387176513672,
      "activations/layer5_attention_weight_max": 120.99736785888672,
      "activations/layer5_attention_weight_min": -103.73649597167969,
      "activations/layer6_attention_weight_max": 66.39383697509766,
      "activations/layer6_attention_weight_min": -54.423927307128906,
      "activations/layer7_attention_weight_max": 68.45691680908203,
      "activations/layer7_attention_weight_min": -60.1462287902832,
      "activations/layer8_attention_weight_max": 57.03981399536133,
      "activations/layer8_attention_weight_min": -49.13557815551758,
      "activations/layer9_attention_weight_max": 79.96964263916016,
      "activations/layer9_attention_weight_min": -74.1690902709961,
      "epoch": 1.4,
      "learning_rate": 0.00014240075757575757,
      "loss": 3.0326,
      "step": 24100
    },
    {
      "activations/layer0_attention_weight_max": 16.001371383666992,
      "activations/layer0_attention_weight_min": -11.592705726623535,
      "activations/layer10_attention_weight_max": 63.019126892089844,
      "activations/layer10_attention_weight_min": -48.916622161865234,
      "activations/layer11_attention_weight_max": 42.458351135253906,
      "activations/layer11_attention_weight_min": -36.10816955566406,
      "activations/layer12_attention_weight_max": 56.44942855834961,
      "activations/layer12_attention_weight_min": -43.8935432434082,
      "activations/layer13_attention_weight_max": 50.55921936035156,
      "activations/layer13_attention_weight_min": -37.427650451660156,
      "activations/layer14_attention_weight_max": 77.22390747070312,
      "activations/layer14_attention_weight_min": -63.80519104003906,
      "activations/layer15_attention_weight_max": 59.09923553466797,
      "activations/layer15_attention_weight_min": -48.44202423095703,
      "activations/layer16_attention_weight_max": 59.38249588012695,
      "activations/layer16_attention_weight_min": -39.666744232177734,
      "activations/layer17_attention_weight_max": 44.987632751464844,
      "activations/layer17_attention_weight_min": -29.470001220703125,
      "activations/layer18_attention_weight_max": 26.277538299560547,
      "activations/layer18_attention_weight_min": -15.936944007873535,
      "activations/layer19_attention_weight_max": 56.8259391784668,
      "activations/layer19_attention_weight_min": -32.87180709838867,
      "activations/layer1_attention_weight_max": 17.818517684936523,
      "activations/layer1_attention_weight_min": -11.25910758972168,
      "activations/layer20_attention_weight_max": 42.965755462646484,
      "activations/layer20_attention_weight_min": -27.626487731933594,
      "activations/layer21_attention_weight_max": 25.280179977416992,
      "activations/layer21_attention_weight_min": -15.501591682434082,
      "activations/layer22_attention_weight_max": 48.6849479675293,
      "activations/layer22_attention_weight_min": -23.911663055419922,
      "activations/layer23_attention_weight_max": 26.464214324951172,
      "activations/layer23_attention_weight_min": -15.719168663024902,
      "activations/layer2_attention_weight_max": 27.23202133178711,
      "activations/layer2_attention_weight_min": -28.32860565185547,
      "activations/layer3_attention_weight_max": 68.33183288574219,
      "activations/layer3_attention_weight_min": -71.71620178222656,
      "activations/layer4_attention_weight_max": 122.60161590576172,
      "activations/layer4_attention_weight_min": -113.91937255859375,
      "activations/layer5_attention_weight_max": 127.10505676269531,
      "activations/layer5_attention_weight_min": -111.68682098388672,
      "activations/layer6_attention_weight_max": 69.18217468261719,
      "activations/layer6_attention_weight_min": -59.612674713134766,
      "activations/layer7_attention_weight_max": 63.732791900634766,
      "activations/layer7_attention_weight_min": -60.56735610961914,
      "activations/layer8_attention_weight_max": 63.86849594116211,
      "activations/layer8_attention_weight_min": -52.26677703857422,
      "activations/layer9_attention_weight_max": 81.35233306884766,
      "activations/layer9_attention_weight_min": -62.535884857177734,
      "epoch": 1.4,
      "learning_rate": 0.00014238181818181816,
      "loss": 3.0309,
      "step": 24150
    },
    {
      "activations/layer0_attention_weight_max": 15.289615631103516,
      "activations/layer0_attention_weight_min": -12.021440505981445,
      "activations/layer10_attention_weight_max": 59.71509552001953,
      "activations/layer10_attention_weight_min": -52.172447204589844,
      "activations/layer11_attention_weight_max": 43.5391845703125,
      "activations/layer11_attention_weight_min": -39.81269073486328,
      "activations/layer12_attention_weight_max": 35.45421600341797,
      "activations/layer12_attention_weight_min": -28.879947662353516,
      "activations/layer13_attention_weight_max": 44.11217498779297,
      "activations/layer13_attention_weight_min": -36.9570426940918,
      "activations/layer14_attention_weight_max": 77.92565155029297,
      "activations/layer14_attention_weight_min": -62.174591064453125,
      "activations/layer15_attention_weight_max": 52.692440032958984,
      "activations/layer15_attention_weight_min": -41.198524475097656,
      "activations/layer16_attention_weight_max": 53.48853302001953,
      "activations/layer16_attention_weight_min": -39.81328582763672,
      "activations/layer17_attention_weight_max": 40.36673355102539,
      "activations/layer17_attention_weight_min": -31.88105010986328,
      "activations/layer18_attention_weight_max": 23.831573486328125,
      "activations/layer18_attention_weight_min": -15.896251678466797,
      "activations/layer19_attention_weight_max": 63.8018684387207,
      "activations/layer19_attention_weight_min": -35.31904220581055,
      "activations/layer1_attention_weight_max": 18.37036895751953,
      "activations/layer1_attention_weight_min": -11.036566734313965,
      "activations/layer20_attention_weight_max": 47.34815979003906,
      "activations/layer20_attention_weight_min": -26.5537166595459,
      "activations/layer21_attention_weight_max": 24.54039764404297,
      "activations/layer21_attention_weight_min": -17.50873374938965,
      "activations/layer22_attention_weight_max": 42.01932907104492,
      "activations/layer22_attention_weight_min": -23.12499237060547,
      "activations/layer23_attention_weight_max": 25.248779296875,
      "activations/layer23_attention_weight_min": -16.25881004333496,
      "activations/layer2_attention_weight_max": 27.414630889892578,
      "activations/layer2_attention_weight_min": -26.989004135131836,
      "activations/layer3_attention_weight_max": 65.44245910644531,
      "activations/layer3_attention_weight_min": -66.37145233154297,
      "activations/layer4_attention_weight_max": 116.27796936035156,
      "activations/layer4_attention_weight_min": -109.96968841552734,
      "activations/layer5_attention_weight_max": 120.18426513671875,
      "activations/layer5_attention_weight_min": -107.95909881591797,
      "activations/layer6_attention_weight_max": 64.27062225341797,
      "activations/layer6_attention_weight_min": -59.07081985473633,
      "activations/layer7_attention_weight_max": 60.80723190307617,
      "activations/layer7_attention_weight_min": -63.829917907714844,
      "activations/layer8_attention_weight_max": 55.250003814697266,
      "activations/layer8_attention_weight_min": -51.67991256713867,
      "activations/layer9_attention_weight_max": 69.20751190185547,
      "activations/layer9_attention_weight_min": -59.9892692565918,
      "epoch": 1.41,
      "learning_rate": 0.00014236287878787877,
      "loss": 3.0422,
      "step": 24200
    },
    {
      "activations/layer0_attention_weight_max": 15.788387298583984,
      "activations/layer0_attention_weight_min": -11.844040870666504,
      "activations/layer10_attention_weight_max": 55.11509704589844,
      "activations/layer10_attention_weight_min": -49.29478454589844,
      "activations/layer11_attention_weight_max": 39.848838806152344,
      "activations/layer11_attention_weight_min": -38.470977783203125,
      "activations/layer12_attention_weight_max": 37.56367111206055,
      "activations/layer12_attention_weight_min": -30.02888298034668,
      "activations/layer13_attention_weight_max": 44.874507904052734,
      "activations/layer13_attention_weight_min": -38.30914306640625,
      "activations/layer14_attention_weight_max": 75.55949401855469,
      "activations/layer14_attention_weight_min": -57.61067581176758,
      "activations/layer15_attention_weight_max": 57.35304641723633,
      "activations/layer15_attention_weight_min": -39.869407653808594,
      "activations/layer16_attention_weight_max": 55.71249008178711,
      "activations/layer16_attention_weight_min": -40.10621643066406,
      "activations/layer17_attention_weight_max": 41.602394104003906,
      "activations/layer17_attention_weight_min": -29.75982093811035,
      "activations/layer18_attention_weight_max": 26.189226150512695,
      "activations/layer18_attention_weight_min": -16.10238265991211,
      "activations/layer19_attention_weight_max": 56.43087387084961,
      "activations/layer19_attention_weight_min": -33.2269172668457,
      "activations/layer1_attention_weight_max": 17.960948944091797,
      "activations/layer1_attention_weight_min": -12.049736976623535,
      "activations/layer20_attention_weight_max": 41.85398483276367,
      "activations/layer20_attention_weight_min": -26.250638961791992,
      "activations/layer21_attention_weight_max": 29.698339462280273,
      "activations/layer21_attention_weight_min": -17.35959815979004,
      "activations/layer22_attention_weight_max": 42.96435546875,
      "activations/layer22_attention_weight_min": -26.3511962890625,
      "activations/layer23_attention_weight_max": 26.147064208984375,
      "activations/layer23_attention_weight_min": -15.89456844329834,
      "activations/layer2_attention_weight_max": 26.17634391784668,
      "activations/layer2_attention_weight_min": -26.70258903503418,
      "activations/layer3_attention_weight_max": 61.90379333496094,
      "activations/layer3_attention_weight_min": -64.941162109375,
      "activations/layer4_attention_weight_max": 109.88448333740234,
      "activations/layer4_attention_weight_min": -104.50028228759766,
      "activations/layer5_attention_weight_max": 110.1187744140625,
      "activations/layer5_attention_weight_min": -99.60558319091797,
      "activations/layer6_attention_weight_max": 61.679710388183594,
      "activations/layer6_attention_weight_min": -52.62867736816406,
      "activations/layer7_attention_weight_max": 64.75092315673828,
      "activations/layer7_attention_weight_min": -67.55465698242188,
      "activations/layer8_attention_weight_max": 48.45804214477539,
      "activations/layer8_attention_weight_min": -45.9762077331543,
      "activations/layer9_attention_weight_max": 70.89586639404297,
      "activations/layer9_attention_weight_min": -63.5954475402832,
      "epoch": 1.41,
      "learning_rate": 0.0001423439393939394,
      "loss": 3.0207,
      "step": 24250
    },
    {
      "activations/layer0_attention_weight_max": 16.140869140625,
      "activations/layer0_attention_weight_min": -11.673869132995605,
      "activations/layer10_attention_weight_max": 58.1129264831543,
      "activations/layer10_attention_weight_min": -52.00189208984375,
      "activations/layer11_attention_weight_max": 42.57414627075195,
      "activations/layer11_attention_weight_min": -35.38526916503906,
      "activations/layer12_attention_weight_max": 46.622589111328125,
      "activations/layer12_attention_weight_min": -31.118242263793945,
      "activations/layer13_attention_weight_max": 42.5670280456543,
      "activations/layer13_attention_weight_min": -33.767818450927734,
      "activations/layer14_attention_weight_max": 73.67555236816406,
      "activations/layer14_attention_weight_min": -58.561737060546875,
      "activations/layer15_attention_weight_max": 54.83905792236328,
      "activations/layer15_attention_weight_min": -39.95637893676758,
      "activations/layer16_attention_weight_max": 55.18659973144531,
      "activations/layer16_attention_weight_min": -38.67136764526367,
      "activations/layer17_attention_weight_max": 45.51762008666992,
      "activations/layer17_attention_weight_min": -29.347084045410156,
      "activations/layer18_attention_weight_max": 23.552637100219727,
      "activations/layer18_attention_weight_min": -16.481307983398438,
      "activations/layer19_attention_weight_max": 50.20944595336914,
      "activations/layer19_attention_weight_min": -32.48603057861328,
      "activations/layer1_attention_weight_max": 16.969120025634766,
      "activations/layer1_attention_weight_min": -11.733756065368652,
      "activations/layer20_attention_weight_max": 39.2669563293457,
      "activations/layer20_attention_weight_min": -25.946853637695312,
      "activations/layer21_attention_weight_max": 25.65181541442871,
      "activations/layer21_attention_weight_min": -17.130176544189453,
      "activations/layer22_attention_weight_max": 41.30802917480469,
      "activations/layer22_attention_weight_min": -23.5142879486084,
      "activations/layer23_attention_weight_max": 24.373884201049805,
      "activations/layer23_attention_weight_min": -14.640176773071289,
      "activations/layer2_attention_weight_max": 28.231496810913086,
      "activations/layer2_attention_weight_min": -26.355758666992188,
      "activations/layer3_attention_weight_max": 66.41614532470703,
      "activations/layer3_attention_weight_min": -65.4758529663086,
      "activations/layer4_attention_weight_max": 123.72412109375,
      "activations/layer4_attention_weight_min": -106.6646728515625,
      "activations/layer5_attention_weight_max": 128.5636749267578,
      "activations/layer5_attention_weight_min": -105.3271484375,
      "activations/layer6_attention_weight_max": 70.83374786376953,
      "activations/layer6_attention_weight_min": -55.97779083251953,
      "activations/layer7_attention_weight_max": 67.1435317993164,
      "activations/layer7_attention_weight_min": -61.29780578613281,
      "activations/layer8_attention_weight_max": 58.20183563232422,
      "activations/layer8_attention_weight_min": -43.0311164855957,
      "activations/layer9_attention_weight_max": 72.62818908691406,
      "activations/layer9_attention_weight_min": -60.882568359375,
      "epoch": 1.41,
      "learning_rate": 0.00014232499999999998,
      "loss": 3.0277,
      "step": 24300
    },
    {
      "activations/layer0_attention_weight_max": 15.013065338134766,
      "activations/layer0_attention_weight_min": -11.395941734313965,
      "activations/layer10_attention_weight_max": 62.291587829589844,
      "activations/layer10_attention_weight_min": -46.30951690673828,
      "activations/layer11_attention_weight_max": 50.87449645996094,
      "activations/layer11_attention_weight_min": -37.80824279785156,
      "activations/layer12_attention_weight_max": 95.06173706054688,
      "activations/layer12_attention_weight_min": -46.00325393676758,
      "activations/layer13_attention_weight_max": 103.02842712402344,
      "activations/layer13_attention_weight_min": -53.54253387451172,
      "activations/layer14_attention_weight_max": 95.32731628417969,
      "activations/layer14_attention_weight_min": -55.294124603271484,
      "activations/layer15_attention_weight_max": 70.03578186035156,
      "activations/layer15_attention_weight_min": -39.273372650146484,
      "activations/layer16_attention_weight_max": 65.0003890991211,
      "activations/layer16_attention_weight_min": -36.789485931396484,
      "activations/layer17_attention_weight_max": 47.49513244628906,
      "activations/layer17_attention_weight_min": -28.308027267456055,
      "activations/layer18_attention_weight_max": 30.0240478515625,
      "activations/layer18_attention_weight_min": -15.284859657287598,
      "activations/layer19_attention_weight_max": 52.42481231689453,
      "activations/layer19_attention_weight_min": -28.990747451782227,
      "activations/layer1_attention_weight_max": 18.08711051940918,
      "activations/layer1_attention_weight_min": -13.06419849395752,
      "activations/layer20_attention_weight_max": 40.490638732910156,
      "activations/layer20_attention_weight_min": -24.118520736694336,
      "activations/layer21_attention_weight_max": 21.95146942138672,
      "activations/layer21_attention_weight_min": -17.022735595703125,
      "activations/layer22_attention_weight_max": 40.36056900024414,
      "activations/layer22_attention_weight_min": -21.325658798217773,
      "activations/layer23_attention_weight_max": 22.49648666381836,
      "activations/layer23_attention_weight_min": -13.922866821289062,
      "activations/layer2_attention_weight_max": 30.57570457458496,
      "activations/layer2_attention_weight_min": -27.991741180419922,
      "activations/layer3_attention_weight_max": 71.93709564208984,
      "activations/layer3_attention_weight_min": -70.1070785522461,
      "activations/layer4_attention_weight_max": 134.3765411376953,
      "activations/layer4_attention_weight_min": -114.88459777832031,
      "activations/layer5_attention_weight_max": 139.47840881347656,
      "activations/layer5_attention_weight_min": -117.01250457763672,
      "activations/layer6_attention_weight_max": 76.61979675292969,
      "activations/layer6_attention_weight_min": -60.22795104980469,
      "activations/layer7_attention_weight_max": 67.27198028564453,
      "activations/layer7_attention_weight_min": -71.33712005615234,
      "activations/layer8_attention_weight_max": 68.86576080322266,
      "activations/layer8_attention_weight_min": -54.24580764770508,
      "activations/layer9_attention_weight_max": 106.58080291748047,
      "activations/layer9_attention_weight_min": -75.5953140258789,
      "epoch": 1.41,
      "learning_rate": 0.0001423060606060606,
      "loss": 3.0198,
      "step": 24350
    },
    {
      "activations/layer0_attention_weight_max": 16.160219192504883,
      "activations/layer0_attention_weight_min": -12.041580200195312,
      "activations/layer10_attention_weight_max": 60.1732177734375,
      "activations/layer10_attention_weight_min": -51.40694046020508,
      "activations/layer11_attention_weight_max": 44.785423278808594,
      "activations/layer11_attention_weight_min": -39.08650207519531,
      "activations/layer12_attention_weight_max": 35.372398376464844,
      "activations/layer12_attention_weight_min": -32.4008674621582,
      "activations/layer13_attention_weight_max": 48.81272888183594,
      "activations/layer13_attention_weight_min": -38.5177001953125,
      "activations/layer14_attention_weight_max": 73.32880401611328,
      "activations/layer14_attention_weight_min": -62.187679290771484,
      "activations/layer15_attention_weight_max": 54.611114501953125,
      "activations/layer15_attention_weight_min": -38.59048080444336,
      "activations/layer16_attention_weight_max": 55.18918991088867,
      "activations/layer16_attention_weight_min": -37.79176330566406,
      "activations/layer17_attention_weight_max": 44.93711471557617,
      "activations/layer17_attention_weight_min": -30.400421142578125,
      "activations/layer18_attention_weight_max": 24.527311325073242,
      "activations/layer18_attention_weight_min": -16.002126693725586,
      "activations/layer19_attention_weight_max": 69.65966033935547,
      "activations/layer19_attention_weight_min": -37.23847198486328,
      "activations/layer1_attention_weight_max": 17.637310028076172,
      "activations/layer1_attention_weight_min": -11.679563522338867,
      "activations/layer20_attention_weight_max": 46.47332763671875,
      "activations/layer20_attention_weight_min": -25.717124938964844,
      "activations/layer21_attention_weight_max": 28.357662200927734,
      "activations/layer21_attention_weight_min": -15.55241870880127,
      "activations/layer22_attention_weight_max": 40.77439498901367,
      "activations/layer22_attention_weight_min": -25.7367000579834,
      "activations/layer23_attention_weight_max": 27.4038143157959,
      "activations/layer23_attention_weight_min": -15.817682266235352,
      "activations/layer2_attention_weight_max": 28.470680236816406,
      "activations/layer2_attention_weight_min": -27.39871597290039,
      "activations/layer3_attention_weight_max": 68.20216369628906,
      "activations/layer3_attention_weight_min": -70.3528823852539,
      "activations/layer4_attention_weight_max": 120.39562225341797,
      "activations/layer4_attention_weight_min": -110.09832000732422,
      "activations/layer5_attention_weight_max": 119.86690521240234,
      "activations/layer5_attention_weight_min": -103.33717346191406,
      "activations/layer6_attention_weight_max": 64.66268157958984,
      "activations/layer6_attention_weight_min": -54.1242790222168,
      "activations/layer7_attention_weight_max": 67.17717742919922,
      "activations/layer7_attention_weight_min": -59.80582046508789,
      "activations/layer8_attention_weight_max": 53.03420639038086,
      "activations/layer8_attention_weight_min": -45.6077995300293,
      "activations/layer9_attention_weight_max": 67.03707885742188,
      "activations/layer9_attention_weight_min": -59.70548629760742,
      "epoch": 1.42,
      "learning_rate": 0.0001422871212121212,
      "loss": 3.0296,
      "step": 24400
    },
    {
      "activations/layer0_attention_weight_max": 15.118896484375,
      "activations/layer0_attention_weight_min": -11.524227142333984,
      "activations/layer10_attention_weight_max": 50.1992301940918,
      "activations/layer10_attention_weight_min": -45.9142951965332,
      "activations/layer11_attention_weight_max": 43.15504455566406,
      "activations/layer11_attention_weight_min": -39.02129364013672,
      "activations/layer12_attention_weight_max": 35.266231536865234,
      "activations/layer12_attention_weight_min": -27.553180694580078,
      "activations/layer13_attention_weight_max": 42.5975227355957,
      "activations/layer13_attention_weight_min": -37.02640914916992,
      "activations/layer14_attention_weight_max": 70.89237213134766,
      "activations/layer14_attention_weight_min": -56.519832611083984,
      "activations/layer15_attention_weight_max": 53.5181884765625,
      "activations/layer15_attention_weight_min": -39.56074142456055,
      "activations/layer16_attention_weight_max": 50.018218994140625,
      "activations/layer16_attention_weight_min": -38.70417022705078,
      "activations/layer17_attention_weight_max": 48.3311767578125,
      "activations/layer17_attention_weight_min": -27.39340591430664,
      "activations/layer18_attention_weight_max": 25.36957550048828,
      "activations/layer18_attention_weight_min": -14.6912202835083,
      "activations/layer19_attention_weight_max": 51.8371467590332,
      "activations/layer19_attention_weight_min": -31.776498794555664,
      "activations/layer1_attention_weight_max": 17.760540008544922,
      "activations/layer1_attention_weight_min": -11.22343921661377,
      "activations/layer20_attention_weight_max": 41.34138107299805,
      "activations/layer20_attention_weight_min": -27.8492431640625,
      "activations/layer21_attention_weight_max": 28.156827926635742,
      "activations/layer21_attention_weight_min": -18.190887451171875,
      "activations/layer22_attention_weight_max": 45.5068473815918,
      "activations/layer22_attention_weight_min": -27.17085075378418,
      "activations/layer23_attention_weight_max": 24.925498962402344,
      "activations/layer23_attention_weight_min": -15.02399730682373,
      "activations/layer2_attention_weight_max": 27.152629852294922,
      "activations/layer2_attention_weight_min": -25.793941497802734,
      "activations/layer3_attention_weight_max": 64.46713256835938,
      "activations/layer3_attention_weight_min": -65.52214813232422,
      "activations/layer4_attention_weight_max": 118.75897216796875,
      "activations/layer4_attention_weight_min": -106.2520751953125,
      "activations/layer5_attention_weight_max": 116.91392517089844,
      "activations/layer5_attention_weight_min": -102.63542938232422,
      "activations/layer6_attention_weight_max": 62.731101989746094,
      "activations/layer6_attention_weight_min": -56.10983657836914,
      "activations/layer7_attention_weight_max": 64.37654113769531,
      "activations/layer7_attention_weight_min": -61.53477096557617,
      "activations/layer8_attention_weight_max": 54.92269515991211,
      "activations/layer8_attention_weight_min": -46.446807861328125,
      "activations/layer9_attention_weight_max": 66.13404846191406,
      "activations/layer9_attention_weight_min": -58.255611419677734,
      "epoch": 1.42,
      "learning_rate": 0.0001422681818181818,
      "loss": 3.0377,
      "step": 24450
    },
    {
      "activations/layer0_attention_weight_max": 15.772480964660645,
      "activations/layer0_attention_weight_min": -11.572615623474121,
      "activations/layer10_attention_weight_max": 56.17388916015625,
      "activations/layer10_attention_weight_min": -49.89630126953125,
      "activations/layer11_attention_weight_max": 47.685569763183594,
      "activations/layer11_attention_weight_min": -35.80455017089844,
      "activations/layer12_attention_weight_max": 42.566463470458984,
      "activations/layer12_attention_weight_min": -28.04941749572754,
      "activations/layer13_attention_weight_max": 53.47959518432617,
      "activations/layer13_attention_weight_min": -38.49474334716797,
      "activations/layer14_attention_weight_max": 86.44448852539062,
      "activations/layer14_attention_weight_min": -65.5317611694336,
      "activations/layer15_attention_weight_max": 60.956539154052734,
      "activations/layer15_attention_weight_min": -41.03126525878906,
      "activations/layer16_attention_weight_max": 52.88501739501953,
      "activations/layer16_attention_weight_min": -40.26484298706055,
      "activations/layer17_attention_weight_max": 47.54220199584961,
      "activations/layer17_attention_weight_min": -32.08064651489258,
      "activations/layer18_attention_weight_max": 24.888893127441406,
      "activations/layer18_attention_weight_min": -15.995603561401367,
      "activations/layer19_attention_weight_max": 54.998992919921875,
      "activations/layer19_attention_weight_min": -32.97670364379883,
      "activations/layer1_attention_weight_max": 17.81777572631836,
      "activations/layer1_attention_weight_min": -10.88410758972168,
      "activations/layer20_attention_weight_max": 43.23149490356445,
      "activations/layer20_attention_weight_min": -26.49895668029785,
      "activations/layer21_attention_weight_max": 26.09478759765625,
      "activations/layer21_attention_weight_min": -15.52163028717041,
      "activations/layer22_attention_weight_max": 46.37672424316406,
      "activations/layer22_attention_weight_min": -25.638425827026367,
      "activations/layer23_attention_weight_max": 28.048099517822266,
      "activations/layer23_attention_weight_min": -14.266759872436523,
      "activations/layer2_attention_weight_max": 28.682544708251953,
      "activations/layer2_attention_weight_min": -26.8444881439209,
      "activations/layer3_attention_weight_max": 67.7569580078125,
      "activations/layer3_attention_weight_min": -66.35578155517578,
      "activations/layer4_attention_weight_max": 117.36893463134766,
      "activations/layer4_attention_weight_min": -113.54449462890625,
      "activations/layer5_attention_weight_max": 117.47412109375,
      "activations/layer5_attention_weight_min": -104.49968719482422,
      "activations/layer6_attention_weight_max": 65.35464477539062,
      "activations/layer6_attention_weight_min": -54.69977569580078,
      "activations/layer7_attention_weight_max": 62.8228874206543,
      "activations/layer7_attention_weight_min": -61.502288818359375,
      "activations/layer8_attention_weight_max": 54.78580856323242,
      "activations/layer8_attention_weight_min": -46.47385787963867,
      "activations/layer9_attention_weight_max": 69.72286224365234,
      "activations/layer9_attention_weight_min": -58.591590881347656,
      "epoch": 1.42,
      "learning_rate": 0.0001422492424242424,
      "loss": 3.0446,
      "step": 24500
    },
    {
      "activations/layer0_attention_weight_max": 15.849703788757324,
      "activations/layer0_attention_weight_min": -11.758546829223633,
      "activations/layer10_attention_weight_max": 57.82060241699219,
      "activations/layer10_attention_weight_min": -49.46051788330078,
      "activations/layer11_attention_weight_max": 41.90849685668945,
      "activations/layer11_attention_weight_min": -37.25032043457031,
      "activations/layer12_attention_weight_max": 35.32804489135742,
      "activations/layer12_attention_weight_min": -24.980960845947266,
      "activations/layer13_attention_weight_max": 45.45725631713867,
      "activations/layer13_attention_weight_min": -36.51689910888672,
      "activations/layer14_attention_weight_max": 72.77635192871094,
      "activations/layer14_attention_weight_min": -59.8597297668457,
      "activations/layer15_attention_weight_max": 55.67136001586914,
      "activations/layer15_attention_weight_min": -41.638858795166016,
      "activations/layer16_attention_weight_max": 51.858795166015625,
      "activations/layer16_attention_weight_min": -38.17155456542969,
      "activations/layer17_attention_weight_max": 45.40532302856445,
      "activations/layer17_attention_weight_min": -29.20443344116211,
      "activations/layer18_attention_weight_max": 27.505733489990234,
      "activations/layer18_attention_weight_min": -15.166958808898926,
      "activations/layer19_attention_weight_max": 54.21921920776367,
      "activations/layer19_attention_weight_min": -32.40018844604492,
      "activations/layer1_attention_weight_max": 16.448875427246094,
      "activations/layer1_attention_weight_min": -10.889822006225586,
      "activations/layer20_attention_weight_max": 42.87400436401367,
      "activations/layer20_attention_weight_min": -25.475778579711914,
      "activations/layer21_attention_weight_max": 33.42327880859375,
      "activations/layer21_attention_weight_min": -16.668094635009766,
      "activations/layer22_attention_weight_max": 46.11606216430664,
      "activations/layer22_attention_weight_min": -24.699914932250977,
      "activations/layer23_attention_weight_max": 25.533119201660156,
      "activations/layer23_attention_weight_min": -14.827325820922852,
      "activations/layer2_attention_weight_max": 29.4354305267334,
      "activations/layer2_attention_weight_min": -28.906179428100586,
      "activations/layer3_attention_weight_max": 67.49388122558594,
      "activations/layer3_attention_weight_min": -70.56956481933594,
      "activations/layer4_attention_weight_max": 123.11573791503906,
      "activations/layer4_attention_weight_min": -112.11995697021484,
      "activations/layer5_attention_weight_max": 123.47357177734375,
      "activations/layer5_attention_weight_min": -106.22373962402344,
      "activations/layer6_attention_weight_max": 68.70388793945312,
      "activations/layer6_attention_weight_min": -54.73581314086914,
      "activations/layer7_attention_weight_max": 65.70692443847656,
      "activations/layer7_attention_weight_min": -63.56526184082031,
      "activations/layer8_attention_weight_max": 53.09922409057617,
      "activations/layer8_attention_weight_min": -46.27642822265625,
      "activations/layer9_attention_weight_max": 77.01153564453125,
      "activations/layer9_attention_weight_min": -60.4243278503418,
      "epoch": 1.43,
      "learning_rate": 0.00014223030303030302,
      "loss": 3.0252,
      "step": 24550
    },
    {
      "activations/layer0_attention_weight_max": 15.777109146118164,
      "activations/layer0_attention_weight_min": -11.726038932800293,
      "activations/layer10_attention_weight_max": 54.756099700927734,
      "activations/layer10_attention_weight_min": -45.713253021240234,
      "activations/layer11_attention_weight_max": 45.48737716674805,
      "activations/layer11_attention_weight_min": -34.66167068481445,
      "activations/layer12_attention_weight_max": 42.51813507080078,
      "activations/layer12_attention_weight_min": -31.455310821533203,
      "activations/layer13_attention_weight_max": 50.137290954589844,
      "activations/layer13_attention_weight_min": -42.72663879394531,
      "activations/layer14_attention_weight_max": 71.610595703125,
      "activations/layer14_attention_weight_min": -53.81381607055664,
      "activations/layer15_attention_weight_max": 55.0738525390625,
      "activations/layer15_attention_weight_min": -39.27701187133789,
      "activations/layer16_attention_weight_max": 53.28168487548828,
      "activations/layer16_attention_weight_min": -39.10517120361328,
      "activations/layer17_attention_weight_max": 46.057376861572266,
      "activations/layer17_attention_weight_min": -28.037086486816406,
      "activations/layer18_attention_weight_max": 23.706491470336914,
      "activations/layer18_attention_weight_min": -14.758681297302246,
      "activations/layer19_attention_weight_max": 54.97676467895508,
      "activations/layer19_attention_weight_min": -30.72580909729004,
      "activations/layer1_attention_weight_max": 17.666723251342773,
      "activations/layer1_attention_weight_min": -11.714766502380371,
      "activations/layer20_attention_weight_max": 37.3949089050293,
      "activations/layer20_attention_weight_min": -24.185386657714844,
      "activations/layer21_attention_weight_max": 27.181249618530273,
      "activations/layer21_attention_weight_min": -14.948741912841797,
      "activations/layer22_attention_weight_max": 40.11807632446289,
      "activations/layer22_attention_weight_min": -24.037151336669922,
      "activations/layer23_attention_weight_max": 21.55017852783203,
      "activations/layer23_attention_weight_min": -15.162687301635742,
      "activations/layer2_attention_weight_max": 29.072256088256836,
      "activations/layer2_attention_weight_min": -27.93173599243164,
      "activations/layer3_attention_weight_max": 64.86019897460938,
      "activations/layer3_attention_weight_min": -69.33516693115234,
      "activations/layer4_attention_weight_max": 112.08857727050781,
      "activations/layer4_attention_weight_min": -109.74517822265625,
      "activations/layer5_attention_weight_max": 114.36622619628906,
      "activations/layer5_attention_weight_min": -102.45451354980469,
      "activations/layer6_attention_weight_max": 61.50715255737305,
      "activations/layer6_attention_weight_min": -53.624977111816406,
      "activations/layer7_attention_weight_max": 66.5533218383789,
      "activations/layer7_attention_weight_min": -64.79845428466797,
      "activations/layer8_attention_weight_max": 56.31676483154297,
      "activations/layer8_attention_weight_min": -49.904685974121094,
      "activations/layer9_attention_weight_max": 76.70106506347656,
      "activations/layer9_attention_weight_min": -65.49110412597656,
      "epoch": 1.43,
      "learning_rate": 0.0001422113636363636,
      "loss": 3.0312,
      "step": 24600
    },
    {
      "activations/layer0_attention_weight_max": 15.795572280883789,
      "activations/layer0_attention_weight_min": -12.047813415527344,
      "activations/layer10_attention_weight_max": 62.85491180419922,
      "activations/layer10_attention_weight_min": -47.898067474365234,
      "activations/layer11_attention_weight_max": 46.87528991699219,
      "activations/layer11_attention_weight_min": -38.02668762207031,
      "activations/layer12_attention_weight_max": 45.32375717163086,
      "activations/layer12_attention_weight_min": -32.43941116333008,
      "activations/layer13_attention_weight_max": 58.1124267578125,
      "activations/layer13_attention_weight_min": -38.034889221191406,
      "activations/layer14_attention_weight_max": 79.5752944946289,
      "activations/layer14_attention_weight_min": -57.296485900878906,
      "activations/layer15_attention_weight_max": 58.115684509277344,
      "activations/layer15_attention_weight_min": -40.42802429199219,
      "activations/layer16_attention_weight_max": 56.90671920776367,
      "activations/layer16_attention_weight_min": -37.11914825439453,
      "activations/layer17_attention_weight_max": 47.61865997314453,
      "activations/layer17_attention_weight_min": -30.063337326049805,
      "activations/layer18_attention_weight_max": 25.80443572998047,
      "activations/layer18_attention_weight_min": -16.538585662841797,
      "activations/layer19_attention_weight_max": 56.238372802734375,
      "activations/layer19_attention_weight_min": -33.46253204345703,
      "activations/layer1_attention_weight_max": 16.81630516052246,
      "activations/layer1_attention_weight_min": -11.627744674682617,
      "activations/layer20_attention_weight_max": 43.119808197021484,
      "activations/layer20_attention_weight_min": -26.921213150024414,
      "activations/layer21_attention_weight_max": 26.35954475402832,
      "activations/layer21_attention_weight_min": -16.810813903808594,
      "activations/layer22_attention_weight_max": 43.33789825439453,
      "activations/layer22_attention_weight_min": -24.230010986328125,
      "activations/layer23_attention_weight_max": 25.833410263061523,
      "activations/layer23_attention_weight_min": -16.03225326538086,
      "activations/layer2_attention_weight_max": 27.66548728942871,
      "activations/layer2_attention_weight_min": -26.53731918334961,
      "activations/layer3_attention_weight_max": 65.32865905761719,
      "activations/layer3_attention_weight_min": -66.50704956054688,
      "activations/layer4_attention_weight_max": 113.6435317993164,
      "activations/layer4_attention_weight_min": -106.85792541503906,
      "activations/layer5_attention_weight_max": 118.79228973388672,
      "activations/layer5_attention_weight_min": -100.56146240234375,
      "activations/layer6_attention_weight_max": 62.66033172607422,
      "activations/layer6_attention_weight_min": -54.442543029785156,
      "activations/layer7_attention_weight_max": 60.99432373046875,
      "activations/layer7_attention_weight_min": -67.20582580566406,
      "activations/layer8_attention_weight_max": 55.66073989868164,
      "activations/layer8_attention_weight_min": -47.49543380737305,
      "activations/layer9_attention_weight_max": 74.46028900146484,
      "activations/layer9_attention_weight_min": -67.10895538330078,
      "epoch": 1.43,
      "learning_rate": 0.00014219242424242423,
      "loss": 3.0315,
      "step": 24650
    },
    {
      "activations/layer0_attention_weight_max": 16.522357940673828,
      "activations/layer0_attention_weight_min": -12.145185470581055,
      "activations/layer10_attention_weight_max": 55.83417892456055,
      "activations/layer10_attention_weight_min": -43.76356506347656,
      "activations/layer11_attention_weight_max": 38.74732971191406,
      "activations/layer11_attention_weight_min": -35.89984893798828,
      "activations/layer12_attention_weight_max": 36.089508056640625,
      "activations/layer12_attention_weight_min": -28.569168090820312,
      "activations/layer13_attention_weight_max": 60.1675910949707,
      "activations/layer13_attention_weight_min": -42.238887786865234,
      "activations/layer14_attention_weight_max": 69.96599578857422,
      "activations/layer14_attention_weight_min": -51.2944450378418,
      "activations/layer15_attention_weight_max": 58.715633392333984,
      "activations/layer15_attention_weight_min": -43.39870834350586,
      "activations/layer16_attention_weight_max": 48.375,
      "activations/layer16_attention_weight_min": -36.179473876953125,
      "activations/layer17_attention_weight_max": 42.3293571472168,
      "activations/layer17_attention_weight_min": -30.189701080322266,
      "activations/layer18_attention_weight_max": 23.081357955932617,
      "activations/layer18_attention_weight_min": -18.00018882751465,
      "activations/layer19_attention_weight_max": 49.15483856201172,
      "activations/layer19_attention_weight_min": -31.437971115112305,
      "activations/layer1_attention_weight_max": 19.080793380737305,
      "activations/layer1_attention_weight_min": -11.469016075134277,
      "activations/layer20_attention_weight_max": 40.68735122680664,
      "activations/layer20_attention_weight_min": -26.21809959411621,
      "activations/layer21_attention_weight_max": 28.92275619506836,
      "activations/layer21_attention_weight_min": -17.139385223388672,
      "activations/layer22_attention_weight_max": 40.73086929321289,
      "activations/layer22_attention_weight_min": -24.205469131469727,
      "activations/layer23_attention_weight_max": 24.24298667907715,
      "activations/layer23_attention_weight_min": -15.233198165893555,
      "activations/layer2_attention_weight_max": 27.193748474121094,
      "activations/layer2_attention_weight_min": -26.647680282592773,
      "activations/layer3_attention_weight_max": 65.8896255493164,
      "activations/layer3_attention_weight_min": -64.9168701171875,
      "activations/layer4_attention_weight_max": 115.32462310791016,
      "activations/layer4_attention_weight_min": -107.92061614990234,
      "activations/layer5_attention_weight_max": 114.16508483886719,
      "activations/layer5_attention_weight_min": -97.49040985107422,
      "activations/layer6_attention_weight_max": 63.39031219482422,
      "activations/layer6_attention_weight_min": -53.74695587158203,
      "activations/layer7_attention_weight_max": 63.73566436767578,
      "activations/layer7_attention_weight_min": -61.65042495727539,
      "activations/layer8_attention_weight_max": 52.03384017944336,
      "activations/layer8_attention_weight_min": -41.94819641113281,
      "activations/layer9_attention_weight_max": 62.924842834472656,
      "activations/layer9_attention_weight_min": -56.5606575012207,
      "epoch": 1.44,
      "learning_rate": 0.00014217348484848484,
      "loss": 3.0277,
      "step": 24700
    },
    {
      "activations/layer0_attention_weight_max": 15.49279499053955,
      "activations/layer0_attention_weight_min": -12.25644588470459,
      "activations/layer10_attention_weight_max": 54.936431884765625,
      "activations/layer10_attention_weight_min": -48.13907241821289,
      "activations/layer11_attention_weight_max": 55.98649597167969,
      "activations/layer11_attention_weight_min": -40.90966796875,
      "activations/layer12_attention_weight_max": 63.603004455566406,
      "activations/layer12_attention_weight_min": -32.56462097167969,
      "activations/layer13_attention_weight_max": 67.39248657226562,
      "activations/layer13_attention_weight_min": -40.758544921875,
      "activations/layer14_attention_weight_max": 85.09619903564453,
      "activations/layer14_attention_weight_min": -64.9013442993164,
      "activations/layer15_attention_weight_max": 54.375732421875,
      "activations/layer15_attention_weight_min": -41.72231674194336,
      "activations/layer16_attention_weight_max": 50.959739685058594,
      "activations/layer16_attention_weight_min": -37.43081283569336,
      "activations/layer17_attention_weight_max": 44.8967170715332,
      "activations/layer17_attention_weight_min": -29.183753967285156,
      "activations/layer18_attention_weight_max": 23.642518997192383,
      "activations/layer18_attention_weight_min": -16.695837020874023,
      "activations/layer19_attention_weight_max": 50.59078598022461,
      "activations/layer19_attention_weight_min": -31.071094512939453,
      "activations/layer1_attention_weight_max": 18.287078857421875,
      "activations/layer1_attention_weight_min": -11.986435890197754,
      "activations/layer20_attention_weight_max": 39.973289489746094,
      "activations/layer20_attention_weight_min": -24.602861404418945,
      "activations/layer21_attention_weight_max": 28.47029685974121,
      "activations/layer21_attention_weight_min": -16.85106658935547,
      "activations/layer22_attention_weight_max": 39.05095291137695,
      "activations/layer22_attention_weight_min": -26.30266761779785,
      "activations/layer23_attention_weight_max": 24.734600067138672,
      "activations/layer23_attention_weight_min": -14.416991233825684,
      "activations/layer2_attention_weight_max": 26.50863265991211,
      "activations/layer2_attention_weight_min": -25.52181625366211,
      "activations/layer3_attention_weight_max": 59.843994140625,
      "activations/layer3_attention_weight_min": -63.32400131225586,
      "activations/layer4_attention_weight_max": 110.51039123535156,
      "activations/layer4_attention_weight_min": -103.94740295410156,
      "activations/layer5_attention_weight_max": 114.70983123779297,
      "activations/layer5_attention_weight_min": -99.17716979980469,
      "activations/layer6_attention_weight_max": 65.41191101074219,
      "activations/layer6_attention_weight_min": -55.277076721191406,
      "activations/layer7_attention_weight_max": 78.00739288330078,
      "activations/layer7_attention_weight_min": -67.58672332763672,
      "activations/layer8_attention_weight_max": 59.457275390625,
      "activations/layer8_attention_weight_min": -49.78559875488281,
      "activations/layer9_attention_weight_max": 101.0918197631836,
      "activations/layer9_attention_weight_min": -79.39099884033203,
      "epoch": 1.44,
      "learning_rate": 0.00014215454545454543,
      "loss": 3.0353,
      "step": 24750
    },
    {
      "activations/layer0_attention_weight_max": 15.252645492553711,
      "activations/layer0_attention_weight_min": -11.8798246383667,
      "activations/layer10_attention_weight_max": 61.67656707763672,
      "activations/layer10_attention_weight_min": -46.28228759765625,
      "activations/layer11_attention_weight_max": 48.08300018310547,
      "activations/layer11_attention_weight_min": -34.011497497558594,
      "activations/layer12_attention_weight_max": 42.68592834472656,
      "activations/layer12_attention_weight_min": -28.605884552001953,
      "activations/layer13_attention_weight_max": 52.2806510925293,
      "activations/layer13_attention_weight_min": -38.288509368896484,
      "activations/layer14_attention_weight_max": 95.54286193847656,
      "activations/layer14_attention_weight_min": -63.3240966796875,
      "activations/layer15_attention_weight_max": 70.1086654663086,
      "activations/layer15_attention_weight_min": -40.34200668334961,
      "activations/layer16_attention_weight_max": 59.67500305175781,
      "activations/layer16_attention_weight_min": -40.52487564086914,
      "activations/layer17_attention_weight_max": 51.63566207885742,
      "activations/layer17_attention_weight_min": -30.29017448425293,
      "activations/layer18_attention_weight_max": 28.9279727935791,
      "activations/layer18_attention_weight_min": -17.0992488861084,
      "activations/layer19_attention_weight_max": 62.23580551147461,
      "activations/layer19_attention_weight_min": -36.57902526855469,
      "activations/layer1_attention_weight_max": 16.979597091674805,
      "activations/layer1_attention_weight_min": -11.703800201416016,
      "activations/layer20_attention_weight_max": 50.001033782958984,
      "activations/layer20_attention_weight_min": -27.955387115478516,
      "activations/layer21_attention_weight_max": 29.493383407592773,
      "activations/layer21_attention_weight_min": -17.381853103637695,
      "activations/layer22_attention_weight_max": 41.53921890258789,
      "activations/layer22_attention_weight_min": -23.51776885986328,
      "activations/layer23_attention_weight_max": 26.631420135498047,
      "activations/layer23_attention_weight_min": -15.705883026123047,
      "activations/layer2_attention_weight_max": 29.710655212402344,
      "activations/layer2_attention_weight_min": -26.982620239257812,
      "activations/layer3_attention_weight_max": 64.81816101074219,
      "activations/layer3_attention_weight_min": -66.22742462158203,
      "activations/layer4_attention_weight_max": 121.963623046875,
      "activations/layer4_attention_weight_min": -110.83455657958984,
      "activations/layer5_attention_weight_max": 122.71017456054688,
      "activations/layer5_attention_weight_min": -103.45318603515625,
      "activations/layer6_attention_weight_max": 68.05719757080078,
      "activations/layer6_attention_weight_min": -55.51850891113281,
      "activations/layer7_attention_weight_max": 62.965579986572266,
      "activations/layer7_attention_weight_min": -66.0709228515625,
      "activations/layer8_attention_weight_max": 56.40559768676758,
      "activations/layer8_attention_weight_min": -45.332576751708984,
      "activations/layer9_attention_weight_max": 79.46973419189453,
      "activations/layer9_attention_weight_min": -66.73592376708984,
      "epoch": 1.44,
      "learning_rate": 0.00014213560606060605,
      "loss": 3.0208,
      "step": 24800
    },
    {
      "activations/layer0_attention_weight_max": 15.813299179077148,
      "activations/layer0_attention_weight_min": -11.444098472595215,
      "activations/layer10_attention_weight_max": 56.648834228515625,
      "activations/layer10_attention_weight_min": -46.490936279296875,
      "activations/layer11_attention_weight_max": 42.05656051635742,
      "activations/layer11_attention_weight_min": -37.53299331665039,
      "activations/layer12_attention_weight_max": 53.663822174072266,
      "activations/layer12_attention_weight_min": -31.15630340576172,
      "activations/layer13_attention_weight_max": 55.22977066040039,
      "activations/layer13_attention_weight_min": -38.031002044677734,
      "activations/layer14_attention_weight_max": 83.26265716552734,
      "activations/layer14_attention_weight_min": -56.15120315551758,
      "activations/layer15_attention_weight_max": 55.97872543334961,
      "activations/layer15_attention_weight_min": -38.7552490234375,
      "activations/layer16_attention_weight_max": 56.27115249633789,
      "activations/layer16_attention_weight_min": -40.285552978515625,
      "activations/layer17_attention_weight_max": 45.73109817504883,
      "activations/layer17_attention_weight_min": -29.218557357788086,
      "activations/layer18_attention_weight_max": 25.945878982543945,
      "activations/layer18_attention_weight_min": -15.159104347229004,
      "activations/layer19_attention_weight_max": 62.66804885864258,
      "activations/layer19_attention_weight_min": -33.389278411865234,
      "activations/layer1_attention_weight_max": 17.37576675415039,
      "activations/layer1_attention_weight_min": -11.284886360168457,
      "activations/layer20_attention_weight_max": 44.46300506591797,
      "activations/layer20_attention_weight_min": -25.9816951751709,
      "activations/layer21_attention_weight_max": 26.80936622619629,
      "activations/layer21_attention_weight_min": -18.243879318237305,
      "activations/layer22_attention_weight_max": 45.84689712524414,
      "activations/layer22_attention_weight_min": -25.16065788269043,
      "activations/layer23_attention_weight_max": 26.936843872070312,
      "activations/layer23_attention_weight_min": -15.562525749206543,
      "activations/layer2_attention_weight_max": 26.79110336303711,
      "activations/layer2_attention_weight_min": -26.244396209716797,
      "activations/layer3_attention_weight_max": 62.68015670776367,
      "activations/layer3_attention_weight_min": -66.8277816772461,
      "activations/layer4_attention_weight_max": 113.39418029785156,
      "activations/layer4_attention_weight_min": -109.12519836425781,
      "activations/layer5_attention_weight_max": 109.47344207763672,
      "activations/layer5_attention_weight_min": -100.82222747802734,
      "activations/layer6_attention_weight_max": 62.14168167114258,
      "activations/layer6_attention_weight_min": -54.30432891845703,
      "activations/layer7_attention_weight_max": 66.07432556152344,
      "activations/layer7_attention_weight_min": -61.12732696533203,
      "activations/layer8_attention_weight_max": 54.07289505004883,
      "activations/layer8_attention_weight_min": -46.62526321411133,
      "activations/layer9_attention_weight_max": 72.2531509399414,
      "activations/layer9_attention_weight_min": -63.605812072753906,
      "epoch": 1.44,
      "learning_rate": 0.00014211666666666666,
      "loss": 3.0143,
      "step": 24850
    },
    {
      "activations/layer0_attention_weight_max": 16.196531295776367,
      "activations/layer0_attention_weight_min": -11.826142311096191,
      "activations/layer10_attention_weight_max": 53.50803756713867,
      "activations/layer10_attention_weight_min": -45.7336311340332,
      "activations/layer11_attention_weight_max": 42.807586669921875,
      "activations/layer11_attention_weight_min": -40.182979583740234,
      "activations/layer12_attention_weight_max": 33.74040603637695,
      "activations/layer12_attention_weight_min": -28.72390365600586,
      "activations/layer13_attention_weight_max": 52.69237518310547,
      "activations/layer13_attention_weight_min": -40.3039436340332,
      "activations/layer14_attention_weight_max": 70.2210693359375,
      "activations/layer14_attention_weight_min": -59.501834869384766,
      "activations/layer15_attention_weight_max": 49.752525329589844,
      "activations/layer15_attention_weight_min": -38.83664321899414,
      "activations/layer16_attention_weight_max": 50.320186614990234,
      "activations/layer16_attention_weight_min": -36.184757232666016,
      "activations/layer17_attention_weight_max": 40.82390213012695,
      "activations/layer17_attention_weight_min": -30.27001953125,
      "activations/layer18_attention_weight_max": 29.17640495300293,
      "activations/layer18_attention_weight_min": -18.025096893310547,
      "activations/layer19_attention_weight_max": 49.90884017944336,
      "activations/layer19_attention_weight_min": -32.16811752319336,
      "activations/layer1_attention_weight_max": 17.609392166137695,
      "activations/layer1_attention_weight_min": -11.754353523254395,
      "activations/layer20_attention_weight_max": 41.21179962158203,
      "activations/layer20_attention_weight_min": -24.55031967163086,
      "activations/layer21_attention_weight_max": 28.045591354370117,
      "activations/layer21_attention_weight_min": -18.622364044189453,
      "activations/layer22_attention_weight_max": 43.117515563964844,
      "activations/layer22_attention_weight_min": -25.44597053527832,
      "activations/layer23_attention_weight_max": 26.84090232849121,
      "activations/layer23_attention_weight_min": -16.771390914916992,
      "activations/layer2_attention_weight_max": 28.573383331298828,
      "activations/layer2_attention_weight_min": -27.741588592529297,
      "activations/layer3_attention_weight_max": 64.02800750732422,
      "activations/layer3_attention_weight_min": -67.14490509033203,
      "activations/layer4_attention_weight_max": 115.3661880493164,
      "activations/layer4_attention_weight_min": -114.33805084228516,
      "activations/layer5_attention_weight_max": 115.51148986816406,
      "activations/layer5_attention_weight_min": -104.68299865722656,
      "activations/layer6_attention_weight_max": 61.417911529541016,
      "activations/layer6_attention_weight_min": -57.149505615234375,
      "activations/layer7_attention_weight_max": 66.6608657836914,
      "activations/layer7_attention_weight_min": -60.9880256652832,
      "activations/layer8_attention_weight_max": 51.57451248168945,
      "activations/layer8_attention_weight_min": -45.34467315673828,
      "activations/layer9_attention_weight_max": 74.29995727539062,
      "activations/layer9_attention_weight_min": -67.42355346679688,
      "epoch": 1.45,
      "learning_rate": 0.00014209772727272728,
      "loss": 3.0504,
      "step": 24900
    },
    {
      "activations/layer0_attention_weight_max": 16.42733383178711,
      "activations/layer0_attention_weight_min": -12.540398597717285,
      "activations/layer10_attention_weight_max": 57.024200439453125,
      "activations/layer10_attention_weight_min": -50.08855056762695,
      "activations/layer11_attention_weight_max": 47.57366943359375,
      "activations/layer11_attention_weight_min": -36.302330017089844,
      "activations/layer12_attention_weight_max": 39.71026611328125,
      "activations/layer12_attention_weight_min": -28.36932945251465,
      "activations/layer13_attention_weight_max": 57.714805603027344,
      "activations/layer13_attention_weight_min": -43.163509368896484,
      "activations/layer14_attention_weight_max": 75.96147918701172,
      "activations/layer14_attention_weight_min": -59.249847412109375,
      "activations/layer15_attention_weight_max": 55.235294342041016,
      "activations/layer15_attention_weight_min": -39.89678192138672,
      "activations/layer16_attention_weight_max": 52.239749908447266,
      "activations/layer16_attention_weight_min": -37.05235290527344,
      "activations/layer17_attention_weight_max": 44.029911041259766,
      "activations/layer17_attention_weight_min": -28.811431884765625,
      "activations/layer18_attention_weight_max": 27.798494338989258,
      "activations/layer18_attention_weight_min": -15.784562110900879,
      "activations/layer19_attention_weight_max": 54.154388427734375,
      "activations/layer19_attention_weight_min": -33.084964752197266,
      "activations/layer1_attention_weight_max": 16.934171676635742,
      "activations/layer1_attention_weight_min": -11.577795028686523,
      "activations/layer20_attention_weight_max": 40.06059265136719,
      "activations/layer20_attention_weight_min": -27.488067626953125,
      "activations/layer21_attention_weight_max": 25.802837371826172,
      "activations/layer21_attention_weight_min": -15.935503005981445,
      "activations/layer22_attention_weight_max": 44.6445198059082,
      "activations/layer22_attention_weight_min": -25.21608543395996,
      "activations/layer23_attention_weight_max": 24.335790634155273,
      "activations/layer23_attention_weight_min": -15.570391654968262,
      "activations/layer2_attention_weight_max": 28.062713623046875,
      "activations/layer2_attention_weight_min": -26.75951385498047,
      "activations/layer3_attention_weight_max": 63.49068069458008,
      "activations/layer3_attention_weight_min": -66.84915924072266,
      "activations/layer4_attention_weight_max": 118.54589080810547,
      "activations/layer4_attention_weight_min": -114.204345703125,
      "activations/layer5_attention_weight_max": 124.37237548828125,
      "activations/layer5_attention_weight_min": -106.74163055419922,
      "activations/layer6_attention_weight_max": 67.58306121826172,
      "activations/layer6_attention_weight_min": -55.80091094970703,
      "activations/layer7_attention_weight_max": 62.67416000366211,
      "activations/layer7_attention_weight_min": -62.9296875,
      "activations/layer8_attention_weight_max": 56.0074462890625,
      "activations/layer8_attention_weight_min": -48.78266525268555,
      "activations/layer9_attention_weight_max": 82.82794189453125,
      "activations/layer9_attention_weight_min": -61.6455192565918,
      "epoch": 1.45,
      "learning_rate": 0.00014207878787878786,
      "loss": 3.0172,
      "step": 24950
    },
    {
      "activations/layer0_attention_weight_max": 16.152963638305664,
      "activations/layer0_attention_weight_min": -12.458585739135742,
      "activations/layer10_attention_weight_max": 51.721458435058594,
      "activations/layer10_attention_weight_min": -48.101158142089844,
      "activations/layer11_attention_weight_max": 40.840492248535156,
      "activations/layer11_attention_weight_min": -34.58379364013672,
      "activations/layer12_attention_weight_max": 36.64604568481445,
      "activations/layer12_attention_weight_min": -27.31928825378418,
      "activations/layer13_attention_weight_max": 47.04790115356445,
      "activations/layer13_attention_weight_min": -37.98735809326172,
      "activations/layer14_attention_weight_max": 73.16039276123047,
      "activations/layer14_attention_weight_min": -63.42665100097656,
      "activations/layer15_attention_weight_max": 50.883934020996094,
      "activations/layer15_attention_weight_min": -44.372623443603516,
      "activations/layer16_attention_weight_max": 47.491939544677734,
      "activations/layer16_attention_weight_min": -37.60078430175781,
      "activations/layer17_attention_weight_max": 47.38637161254883,
      "activations/layer17_attention_weight_min": -31.57864761352539,
      "activations/layer18_attention_weight_max": 27.523273468017578,
      "activations/layer18_attention_weight_min": -19.33249282836914,
      "activations/layer19_attention_weight_max": 46.157047271728516,
      "activations/layer19_attention_weight_min": -33.08839797973633,
      "activations/layer1_attention_weight_max": 17.224464416503906,
      "activations/layer1_attention_weight_min": -11.710607528686523,
      "activations/layer20_attention_weight_max": 42.432064056396484,
      "activations/layer20_attention_weight_min": -26.197179794311523,
      "activations/layer21_attention_weight_max": 26.000566482543945,
      "activations/layer21_attention_weight_min": -16.5849609375,
      "activations/layer22_attention_weight_max": 41.063629150390625,
      "activations/layer22_attention_weight_min": -23.729143142700195,
      "activations/layer23_attention_weight_max": 24.58983612060547,
      "activations/layer23_attention_weight_min": -13.712943077087402,
      "activations/layer2_attention_weight_max": 28.903608322143555,
      "activations/layer2_attention_weight_min": -27.078243255615234,
      "activations/layer3_attention_weight_max": 63.421348571777344,
      "activations/layer3_attention_weight_min": -68.47410583496094,
      "activations/layer4_attention_weight_max": 113.99629211425781,
      "activations/layer4_attention_weight_min": -107.14991760253906,
      "activations/layer5_attention_weight_max": 113.9553451538086,
      "activations/layer5_attention_weight_min": -101.07728576660156,
      "activations/layer6_attention_weight_max": 63.007511138916016,
      "activations/layer6_attention_weight_min": -52.0956916809082,
      "activations/layer7_attention_weight_max": 60.17845916748047,
      "activations/layer7_attention_weight_min": -66.40901184082031,
      "activations/layer8_attention_weight_max": 50.844871520996094,
      "activations/layer8_attention_weight_min": -44.77389907836914,
      "activations/layer9_attention_weight_max": 82.1674575805664,
      "activations/layer9_attention_weight_min": -59.091182708740234,
      "epoch": 1.45,
      "learning_rate": 0.00014205984848484848,
      "loss": 3.0266,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_loss": 2.970703125,
      "eval_runtime": 8.5199,
      "eval_samples_per_second": 503.995,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_openwebtext_loss": 2.970703125,
      "eval_openwebtext_ppl": 19.505629671380603,
      "eval_openwebtext_runtime": 8.5199,
      "eval_openwebtext_samples_per_second": 503.995,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_wikitext_loss": 3.248046875,
      "eval_wikitext_ppl": 25.740017318702005,
      "eval_wikitext_runtime": 1.9579,
      "eval_wikitext_samples_per_second": 232.905,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_lambada_loss": 3.3125,
      "eval_lambada_ppl": 27.4536739354601,
      "eval_lambada_runtime": 9.517,
      "eval_lambada_samples_per_second": 511.61,
      "step": 25000
    },
    {
      "activations/layer0_attention_weight_max": 16.07829475402832,
      "activations/layer0_attention_weight_min": -12.201181411743164,
      "activations/layer10_attention_weight_max": 47.397918701171875,
      "activations/layer10_attention_weight_min": -43.78706741333008,
      "activations/layer11_attention_weight_max": 41.328765869140625,
      "activations/layer11_attention_weight_min": -37.76911544799805,
      "activations/layer12_attention_weight_max": 45.809425354003906,
      "activations/layer12_attention_weight_min": -28.111188888549805,
      "activations/layer13_attention_weight_max": 50.42710494995117,
      "activations/layer13_attention_weight_min": -40.146541595458984,
      "activations/layer14_attention_weight_max": 69.37906646728516,
      "activations/layer14_attention_weight_min": -56.6002082824707,
      "activations/layer15_attention_weight_max": 49.802703857421875,
      "activations/layer15_attention_weight_min": -36.812286376953125,
      "activations/layer16_attention_weight_max": 50.73828887939453,
      "activations/layer16_attention_weight_min": -38.6026725769043,
      "activations/layer17_attention_weight_max": 46.05269241333008,
      "activations/layer17_attention_weight_min": -30.293603897094727,
      "activations/layer18_attention_weight_max": 24.668546676635742,
      "activations/layer18_attention_weight_min": -18.31812286376953,
      "activations/layer19_attention_weight_max": 44.99888229370117,
      "activations/layer19_attention_weight_min": -30.403335571289062,
      "activations/layer1_attention_weight_max": 17.296106338500977,
      "activations/layer1_attention_weight_min": -10.785571098327637,
      "activations/layer20_attention_weight_max": 39.47653579711914,
      "activations/layer20_attention_weight_min": -27.13258934020996,
      "activations/layer21_attention_weight_max": 24.44281578063965,
      "activations/layer21_attention_weight_min": -19.513986587524414,
      "activations/layer22_attention_weight_max": 37.17452621459961,
      "activations/layer22_attention_weight_min": -23.081886291503906,
      "activations/layer23_attention_weight_max": 22.042449951171875,
      "activations/layer23_attention_weight_min": -15.806166648864746,
      "activations/layer2_attention_weight_max": 26.53917694091797,
      "activations/layer2_attention_weight_min": -25.471527099609375,
      "activations/layer3_attention_weight_max": 64.06013488769531,
      "activations/layer3_attention_weight_min": -65.23391723632812,
      "activations/layer4_attention_weight_max": 115.7356948852539,
      "activations/layer4_attention_weight_min": -103.88516998291016,
      "activations/layer5_attention_weight_max": 113.5404052734375,
      "activations/layer5_attention_weight_min": -96.42503356933594,
      "activations/layer6_attention_weight_max": 59.25773620605469,
      "activations/layer6_attention_weight_min": -52.57502365112305,
      "activations/layer7_attention_weight_max": 66.95854187011719,
      "activations/layer7_attention_weight_min": -60.17255401611328,
      "activations/layer8_attention_weight_max": 46.307403564453125,
      "activations/layer8_attention_weight_min": -42.873931884765625,
      "activations/layer9_attention_weight_max": 69.16732025146484,
      "activations/layer9_attention_weight_min": -58.27415084838867,
      "epoch": 1.46,
      "learning_rate": 0.0001420409090909091,
      "loss": 3.0196,
      "step": 25050
    },
    {
      "activations/layer0_attention_weight_max": 16.15831184387207,
      "activations/layer0_attention_weight_min": -12.388314247131348,
      "activations/layer10_attention_weight_max": 62.698699951171875,
      "activations/layer10_attention_weight_min": -47.999603271484375,
      "activations/layer11_attention_weight_max": 44.61540222167969,
      "activations/layer11_attention_weight_min": -39.49089050292969,
      "activations/layer12_attention_weight_max": 37.513675689697266,
      "activations/layer12_attention_weight_min": -28.556800842285156,
      "activations/layer13_attention_weight_max": 50.826412200927734,
      "activations/layer13_attention_weight_min": -38.743316650390625,
      "activations/layer14_attention_weight_max": 84.43719482421875,
      "activations/layer14_attention_weight_min": -61.853363037109375,
      "activations/layer15_attention_weight_max": 57.56466293334961,
      "activations/layer15_attention_weight_min": -42.19380569458008,
      "activations/layer16_attention_weight_max": 61.9730224609375,
      "activations/layer16_attention_weight_min": -34.483497619628906,
      "activations/layer17_attention_weight_max": 49.5262451171875,
      "activations/layer17_attention_weight_min": -28.81966781616211,
      "activations/layer18_attention_weight_max": 27.074748992919922,
      "activations/layer18_attention_weight_min": -17.17902374267578,
      "activations/layer19_attention_weight_max": 63.09145736694336,
      "activations/layer19_attention_weight_min": -32.718780517578125,
      "activations/layer1_attention_weight_max": 18.66010856628418,
      "activations/layer1_attention_weight_min": -11.289402961730957,
      "activations/layer20_attention_weight_max": 47.9442253112793,
      "activations/layer20_attention_weight_min": -25.891263961791992,
      "activations/layer21_attention_weight_max": 30.60614013671875,
      "activations/layer21_attention_weight_min": -14.909965515136719,
      "activations/layer22_attention_weight_max": 50.780975341796875,
      "activations/layer22_attention_weight_min": -23.07461166381836,
      "activations/layer23_attention_weight_max": 34.8589973449707,
      "activations/layer23_attention_weight_min": -14.522082328796387,
      "activations/layer2_attention_weight_max": 28.07155418395996,
      "activations/layer2_attention_weight_min": -27.600727081298828,
      "activations/layer3_attention_weight_max": 68.65593719482422,
      "activations/layer3_attention_weight_min": -69.29048919677734,
      "activations/layer4_attention_weight_max": 123.28260803222656,
      "activations/layer4_attention_weight_min": -110.1147232055664,
      "activations/layer5_attention_weight_max": 126.86949157714844,
      "activations/layer5_attention_weight_min": -107.49163055419922,
      "activations/layer6_attention_weight_max": 66.49344635009766,
      "activations/layer6_attention_weight_min": -56.17692184448242,
      "activations/layer7_attention_weight_max": 67.86981201171875,
      "activations/layer7_attention_weight_min": -62.55002212524414,
      "activations/layer8_attention_weight_max": 56.33777618408203,
      "activations/layer8_attention_weight_min": -47.240264892578125,
      "activations/layer9_attention_weight_max": 81.52632904052734,
      "activations/layer9_attention_weight_min": -65.03529357910156,
      "epoch": 1.46,
      "learning_rate": 0.00014202196969696968,
      "loss": 3.0272,
      "step": 25100
    },
    {
      "activations/layer0_attention_weight_max": 14.890644073486328,
      "activations/layer0_attention_weight_min": -11.707978248596191,
      "activations/layer10_attention_weight_max": 55.21971893310547,
      "activations/layer10_attention_weight_min": -45.174720764160156,
      "activations/layer11_attention_weight_max": 49.925498962402344,
      "activations/layer11_attention_weight_min": -38.90940856933594,
      "activations/layer12_attention_weight_max": 40.28310012817383,
      "activations/layer12_attention_weight_min": -29.54290008544922,
      "activations/layer13_attention_weight_max": 51.825435638427734,
      "activations/layer13_attention_weight_min": -36.34512710571289,
      "activations/layer14_attention_weight_max": 84.50767517089844,
      "activations/layer14_attention_weight_min": -66.21278381347656,
      "activations/layer15_attention_weight_max": 62.437076568603516,
      "activations/layer15_attention_weight_min": -46.074134826660156,
      "activations/layer16_attention_weight_max": 58.99363327026367,
      "activations/layer16_attention_weight_min": -39.788299560546875,
      "activations/layer17_attention_weight_max": 47.20932388305664,
      "activations/layer17_attention_weight_min": -29.39346694946289,
      "activations/layer18_attention_weight_max": 27.557476043701172,
      "activations/layer18_attention_weight_min": -17.791423797607422,
      "activations/layer19_attention_weight_max": 53.976844787597656,
      "activations/layer19_attention_weight_min": -40.07282638549805,
      "activations/layer1_attention_weight_max": 17.51421546936035,
      "activations/layer1_attention_weight_min": -10.885403633117676,
      "activations/layer20_attention_weight_max": 44.05485534667969,
      "activations/layer20_attention_weight_min": -29.690959930419922,
      "activations/layer21_attention_weight_max": 26.386552810668945,
      "activations/layer21_attention_weight_min": -19.694610595703125,
      "activations/layer22_attention_weight_max": 43.72358322143555,
      "activations/layer22_attention_weight_min": -25.45296859741211,
      "activations/layer23_attention_weight_max": 24.30605125427246,
      "activations/layer23_attention_weight_min": -15.043604850769043,
      "activations/layer2_attention_weight_max": 27.903736114501953,
      "activations/layer2_attention_weight_min": -27.856901168823242,
      "activations/layer3_attention_weight_max": 64.23147583007812,
      "activations/layer3_attention_weight_min": -69.55780792236328,
      "activations/layer4_attention_weight_max": 119.26847076416016,
      "activations/layer4_attention_weight_min": -108.56410217285156,
      "activations/layer5_attention_weight_max": 121.35708618164062,
      "activations/layer5_attention_weight_min": -107.09637451171875,
      "activations/layer6_attention_weight_max": 64.6904296875,
      "activations/layer6_attention_weight_min": -54.87736892700195,
      "activations/layer7_attention_weight_max": 68.76799774169922,
      "activations/layer7_attention_weight_min": -59.63502883911133,
      "activations/layer8_attention_weight_max": 55.56438446044922,
      "activations/layer8_attention_weight_min": -52.29235076904297,
      "activations/layer9_attention_weight_max": 82.23258972167969,
      "activations/layer9_attention_weight_min": -71.61941528320312,
      "epoch": 1.46,
      "learning_rate": 0.0001420030303030303,
      "loss": 3.0308,
      "step": 25150
    },
    {
      "activations/layer0_attention_weight_max": 15.845446586608887,
      "activations/layer0_attention_weight_min": -11.996493339538574,
      "activations/layer10_attention_weight_max": 50.93320083618164,
      "activations/layer10_attention_weight_min": -48.54277038574219,
      "activations/layer11_attention_weight_max": 46.331016540527344,
      "activations/layer11_attention_weight_min": -40.58971405029297,
      "activations/layer12_attention_weight_max": 46.010406494140625,
      "activations/layer12_attention_weight_min": -34.20328903198242,
      "activations/layer13_attention_weight_max": 71.4150619506836,
      "activations/layer13_attention_weight_min": -52.22439956665039,
      "activations/layer14_attention_weight_max": 82.71514892578125,
      "activations/layer14_attention_weight_min": -58.748695373535156,
      "activations/layer15_attention_weight_max": 61.96836471557617,
      "activations/layer15_attention_weight_min": -39.12038803100586,
      "activations/layer16_attention_weight_max": 57.86122512817383,
      "activations/layer16_attention_weight_min": -43.684871673583984,
      "activations/layer17_attention_weight_max": 43.08666229248047,
      "activations/layer17_attention_weight_min": -29.17304801940918,
      "activations/layer18_attention_weight_max": 30.87004280090332,
      "activations/layer18_attention_weight_min": -16.45661735534668,
      "activations/layer19_attention_weight_max": 70.06978607177734,
      "activations/layer19_attention_weight_min": -32.88243865966797,
      "activations/layer1_attention_weight_max": 18.10183334350586,
      "activations/layer1_attention_weight_min": -12.78098201751709,
      "activations/layer20_attention_weight_max": 55.99192810058594,
      "activations/layer20_attention_weight_min": -25.742830276489258,
      "activations/layer21_attention_weight_max": 26.905899047851562,
      "activations/layer21_attention_weight_min": -17.06067657470703,
      "activations/layer22_attention_weight_max": 51.89841842651367,
      "activations/layer22_attention_weight_min": -24.336650848388672,
      "activations/layer23_attention_weight_max": 29.15884017944336,
      "activations/layer23_attention_weight_min": -14.956905364990234,
      "activations/layer2_attention_weight_max": 27.5092830657959,
      "activations/layer2_attention_weight_min": -27.819021224975586,
      "activations/layer3_attention_weight_max": 69.00851440429688,
      "activations/layer3_attention_weight_min": -67.85879516601562,
      "activations/layer4_attention_weight_max": 121.61328125,
      "activations/layer4_attention_weight_min": -110.68951416015625,
      "activations/layer5_attention_weight_max": 121.74723815917969,
      "activations/layer5_attention_weight_min": -101.44438934326172,
      "activations/layer6_attention_weight_max": 65.0123291015625,
      "activations/layer6_attention_weight_min": -54.73435592651367,
      "activations/layer7_attention_weight_max": 65.13705444335938,
      "activations/layer7_attention_weight_min": -63.0756721496582,
      "activations/layer8_attention_weight_max": 57.05940628051758,
      "activations/layer8_attention_weight_min": -46.22336959838867,
      "activations/layer9_attention_weight_max": 74.5572509765625,
      "activations/layer9_attention_weight_min": -67.22502136230469,
      "epoch": 1.46,
      "learning_rate": 0.0001419840909090909,
      "loss": 3.0214,
      "step": 25200
    },
    {
      "activations/layer0_attention_weight_max": 15.795048713684082,
      "activations/layer0_attention_weight_min": -11.821392059326172,
      "activations/layer10_attention_weight_max": 53.61371994018555,
      "activations/layer10_attention_weight_min": -45.604156494140625,
      "activations/layer11_attention_weight_max": 43.48350524902344,
      "activations/layer11_attention_weight_min": -37.677467346191406,
      "activations/layer12_attention_weight_max": 37.20389938354492,
      "activations/layer12_attention_weight_min": -30.98523712158203,
      "activations/layer13_attention_weight_max": 55.24962615966797,
      "activations/layer13_attention_weight_min": -38.35146713256836,
      "activations/layer14_attention_weight_max": 74.53280639648438,
      "activations/layer14_attention_weight_min": -58.45136260986328,
      "activations/layer15_attention_weight_max": 57.407474517822266,
      "activations/layer15_attention_weight_min": -39.8183708190918,
      "activations/layer16_attention_weight_max": 55.66102981567383,
      "activations/layer16_attention_weight_min": -42.321746826171875,
      "activations/layer17_attention_weight_max": 46.41838836669922,
      "activations/layer17_attention_weight_min": -29.734479904174805,
      "activations/layer18_attention_weight_max": 25.24820327758789,
      "activations/layer18_attention_weight_min": -15.247708320617676,
      "activations/layer19_attention_weight_max": 55.240413665771484,
      "activations/layer19_attention_weight_min": -32.17161178588867,
      "activations/layer1_attention_weight_max": 17.209854125976562,
      "activations/layer1_attention_weight_min": -11.369804382324219,
      "activations/layer20_attention_weight_max": 40.998844146728516,
      "activations/layer20_attention_weight_min": -25.381248474121094,
      "activations/layer21_attention_weight_max": 27.244524002075195,
      "activations/layer21_attention_weight_min": -16.697525024414062,
      "activations/layer22_attention_weight_max": 47.89841079711914,
      "activations/layer22_attention_weight_min": -25.590255737304688,
      "activations/layer23_attention_weight_max": 22.695232391357422,
      "activations/layer23_attention_weight_min": -15.011146545410156,
      "activations/layer2_attention_weight_max": 27.370258331298828,
      "activations/layer2_attention_weight_min": -26.738492965698242,
      "activations/layer3_attention_weight_max": 64.06224060058594,
      "activations/layer3_attention_weight_min": -65.11769104003906,
      "activations/layer4_attention_weight_max": 117.80281066894531,
      "activations/layer4_attention_weight_min": -108.72063446044922,
      "activations/layer5_attention_weight_max": 126.37568664550781,
      "activations/layer5_attention_weight_min": -107.26063537597656,
      "activations/layer6_attention_weight_max": 65.29644775390625,
      "activations/layer6_attention_weight_min": -57.578861236572266,
      "activations/layer7_attention_weight_max": 72.00794982910156,
      "activations/layer7_attention_weight_min": -64.12085723876953,
      "activations/layer8_attention_weight_max": 61.03378677368164,
      "activations/layer8_attention_weight_min": -52.11436080932617,
      "activations/layer9_attention_weight_max": 73.764404296875,
      "activations/layer9_attention_weight_min": -66.4122085571289,
      "epoch": 1.47,
      "learning_rate": 0.0001419651515151515,
      "loss": 2.9994,
      "step": 25250
    },
    {
      "activations/layer0_attention_weight_max": 16.137727737426758,
      "activations/layer0_attention_weight_min": -12.183317184448242,
      "activations/layer10_attention_weight_max": 56.8525505065918,
      "activations/layer10_attention_weight_min": -48.0357551574707,
      "activations/layer11_attention_weight_max": 39.647613525390625,
      "activations/layer11_attention_weight_min": -38.967247009277344,
      "activations/layer12_attention_weight_max": 39.08148956298828,
      "activations/layer12_attention_weight_min": -31.08551788330078,
      "activations/layer13_attention_weight_max": 50.16501235961914,
      "activations/layer13_attention_weight_min": -42.1846923828125,
      "activations/layer14_attention_weight_max": 73.9198226928711,
      "activations/layer14_attention_weight_min": -59.255516052246094,
      "activations/layer15_attention_weight_max": 48.954750061035156,
      "activations/layer15_attention_weight_min": -39.73829650878906,
      "activations/layer16_attention_weight_max": 50.995567321777344,
      "activations/layer16_attention_weight_min": -36.18829345703125,
      "activations/layer17_attention_weight_max": 36.17229080200195,
      "activations/layer17_attention_weight_min": -28.867835998535156,
      "activations/layer18_attention_weight_max": 23.4703426361084,
      "activations/layer18_attention_weight_min": -16.842241287231445,
      "activations/layer19_attention_weight_max": 49.77677917480469,
      "activations/layer19_attention_weight_min": -33.500362396240234,
      "activations/layer1_attention_weight_max": 16.338468551635742,
      "activations/layer1_attention_weight_min": -11.753228187561035,
      "activations/layer20_attention_weight_max": 43.85852813720703,
      "activations/layer20_attention_weight_min": -25.83649253845215,
      "activations/layer21_attention_weight_max": 30.383941650390625,
      "activations/layer21_attention_weight_min": -19.114730834960938,
      "activations/layer22_attention_weight_max": 39.22344970703125,
      "activations/layer22_attention_weight_min": -25.597511291503906,
      "activations/layer23_attention_weight_max": 24.253429412841797,
      "activations/layer23_attention_weight_min": -15.17258071899414,
      "activations/layer2_attention_weight_max": 28.59039878845215,
      "activations/layer2_attention_weight_min": -28.384258270263672,
      "activations/layer3_attention_weight_max": 65.6188735961914,
      "activations/layer3_attention_weight_min": -69.8108139038086,
      "activations/layer4_attention_weight_max": 116.53752899169922,
      "activations/layer4_attention_weight_min": -109.56196594238281,
      "activations/layer5_attention_weight_max": 115.9363021850586,
      "activations/layer5_attention_weight_min": -104.44182586669922,
      "activations/layer6_attention_weight_max": 64.03087615966797,
      "activations/layer6_attention_weight_min": -55.92319107055664,
      "activations/layer7_attention_weight_max": 67.28189086914062,
      "activations/layer7_attention_weight_min": -62.861122131347656,
      "activations/layer8_attention_weight_max": 54.91144561767578,
      "activations/layer8_attention_weight_min": -49.1983642578125,
      "activations/layer9_attention_weight_max": 76.57726287841797,
      "activations/layer9_attention_weight_min": -62.20647048950195,
      "epoch": 1.47,
      "learning_rate": 0.00014194621212121211,
      "loss": 3.0268,
      "step": 25300
    },
    {
      "activations/layer0_attention_weight_max": 15.760391235351562,
      "activations/layer0_attention_weight_min": -11.95790958404541,
      "activations/layer10_attention_weight_max": 52.70747756958008,
      "activations/layer10_attention_weight_min": -47.10141372680664,
      "activations/layer11_attention_weight_max": 38.812278747558594,
      "activations/layer11_attention_weight_min": -34.962181091308594,
      "activations/layer12_attention_weight_max": 51.60088348388672,
      "activations/layer12_attention_weight_min": -47.32258224487305,
      "activations/layer13_attention_weight_max": 55.041629791259766,
      "activations/layer13_attention_weight_min": -53.13862228393555,
      "activations/layer14_attention_weight_max": 92.0323486328125,
      "activations/layer14_attention_weight_min": -65.0552978515625,
      "activations/layer15_attention_weight_max": 58.97651672363281,
      "activations/layer15_attention_weight_min": -43.127811431884766,
      "activations/layer16_attention_weight_max": 52.052085876464844,
      "activations/layer16_attention_weight_min": -37.90869903564453,
      "activations/layer17_attention_weight_max": 48.47189712524414,
      "activations/layer17_attention_weight_min": -29.707759857177734,
      "activations/layer18_attention_weight_max": 27.417333602905273,
      "activations/layer18_attention_weight_min": -16.34317970275879,
      "activations/layer19_attention_weight_max": 52.66975021362305,
      "activations/layer19_attention_weight_min": -32.21238708496094,
      "activations/layer1_attention_weight_max": 16.928150177001953,
      "activations/layer1_attention_weight_min": -11.142809867858887,
      "activations/layer20_attention_weight_max": 39.67980194091797,
      "activations/layer20_attention_weight_min": -26.486276626586914,
      "activations/layer21_attention_weight_max": 23.08199691772461,
      "activations/layer21_attention_weight_min": -17.01883316040039,
      "activations/layer22_attention_weight_max": 42.45744705200195,
      "activations/layer22_attention_weight_min": -25.405616760253906,
      "activations/layer23_attention_weight_max": 23.65266990661621,
      "activations/layer23_attention_weight_min": -14.860345840454102,
      "activations/layer2_attention_weight_max": 27.843093872070312,
      "activations/layer2_attention_weight_min": -26.175174713134766,
      "activations/layer3_attention_weight_max": 64.44532012939453,
      "activations/layer3_attention_weight_min": -68.48181915283203,
      "activations/layer4_attention_weight_max": 112.1346435546875,
      "activations/layer4_attention_weight_min": -112.669677734375,
      "activations/layer5_attention_weight_max": 112.97257995605469,
      "activations/layer5_attention_weight_min": -100.23054504394531,
      "activations/layer6_attention_weight_max": 61.87413024902344,
      "activations/layer6_attention_weight_min": -57.60115051269531,
      "activations/layer7_attention_weight_max": 65.56078338623047,
      "activations/layer7_attention_weight_min": -67.69624328613281,
      "activations/layer8_attention_weight_max": 49.185001373291016,
      "activations/layer8_attention_weight_min": -45.3901481628418,
      "activations/layer9_attention_weight_max": 73.90806579589844,
      "activations/layer9_attention_weight_min": -69.76670837402344,
      "epoch": 1.47,
      "learning_rate": 0.00014192727272727273,
      "loss": 2.9995,
      "step": 25350
    },
    {
      "activations/layer0_attention_weight_max": 14.745387077331543,
      "activations/layer0_attention_weight_min": -11.857028007507324,
      "activations/layer10_attention_weight_max": 52.584007263183594,
      "activations/layer10_attention_weight_min": -47.12931823730469,
      "activations/layer11_attention_weight_max": 41.37501907348633,
      "activations/layer11_attention_weight_min": -39.98802947998047,
      "activations/layer12_attention_weight_max": 47.55742263793945,
      "activations/layer12_attention_weight_min": -34.99872970581055,
      "activations/layer13_attention_weight_max": 53.988792419433594,
      "activations/layer13_attention_weight_min": -44.80540084838867,
      "activations/layer14_attention_weight_max": 71.06632995605469,
      "activations/layer14_attention_weight_min": -58.95180130004883,
      "activations/layer15_attention_weight_max": 65.8315658569336,
      "activations/layer15_attention_weight_min": -40.021339416503906,
      "activations/layer16_attention_weight_max": 58.17359161376953,
      "activations/layer16_attention_weight_min": -39.04110336303711,
      "activations/layer17_attention_weight_max": 43.55228805541992,
      "activations/layer17_attention_weight_min": -30.626956939697266,
      "activations/layer18_attention_weight_max": 29.861797332763672,
      "activations/layer18_attention_weight_min": -19.252464294433594,
      "activations/layer19_attention_weight_max": 58.125709533691406,
      "activations/layer19_attention_weight_min": -34.89515686035156,
      "activations/layer1_attention_weight_max": 17.068103790283203,
      "activations/layer1_attention_weight_min": -12.833183288574219,
      "activations/layer20_attention_weight_max": 44.20098114013672,
      "activations/layer20_attention_weight_min": -32.41560745239258,
      "activations/layer21_attention_weight_max": 31.219350814819336,
      "activations/layer21_attention_weight_min": -23.836929321289062,
      "activations/layer22_attention_weight_max": 43.405906677246094,
      "activations/layer22_attention_weight_min": -26.429094314575195,
      "activations/layer23_attention_weight_max": 22.891727447509766,
      "activations/layer23_attention_weight_min": -16.60518455505371,
      "activations/layer2_attention_weight_max": 26.288959503173828,
      "activations/layer2_attention_weight_min": -25.618932723999023,
      "activations/layer3_attention_weight_max": 63.57390594482422,
      "activations/layer3_attention_weight_min": -68.31871795654297,
      "activations/layer4_attention_weight_max": 113.77168273925781,
      "activations/layer4_attention_weight_min": -108.5735092163086,
      "activations/layer5_attention_weight_max": 107.69377136230469,
      "activations/layer5_attention_weight_min": -101.25599670410156,
      "activations/layer6_attention_weight_max": 57.379150390625,
      "activations/layer6_attention_weight_min": -54.18828201293945,
      "activations/layer7_attention_weight_max": 60.326141357421875,
      "activations/layer7_attention_weight_min": -61.79093933105469,
      "activations/layer8_attention_weight_max": 50.615814208984375,
      "activations/layer8_attention_weight_min": -45.446533203125,
      "activations/layer9_attention_weight_max": 68.71009063720703,
      "activations/layer9_attention_weight_min": -65.48420715332031,
      "epoch": 1.48,
      "learning_rate": 0.00014190833333333332,
      "loss": 3.0261,
      "step": 25400
    },
    {
      "activations/layer0_attention_weight_max": 15.805953025817871,
      "activations/layer0_attention_weight_min": -12.029767036437988,
      "activations/layer10_attention_weight_max": 54.67844009399414,
      "activations/layer10_attention_weight_min": -45.82632064819336,
      "activations/layer11_attention_weight_max": 46.80790328979492,
      "activations/layer11_attention_weight_min": -36.91691589355469,
      "activations/layer12_attention_weight_max": 36.496307373046875,
      "activations/layer12_attention_weight_min": -27.963438034057617,
      "activations/layer13_attention_weight_max": 52.45944595336914,
      "activations/layer13_attention_weight_min": -38.41224670410156,
      "activations/layer14_attention_weight_max": 77.32112884521484,
      "activations/layer14_attention_weight_min": -55.32379150390625,
      "activations/layer15_attention_weight_max": 57.949459075927734,
      "activations/layer15_attention_weight_min": -38.406768798828125,
      "activations/layer16_attention_weight_max": 61.17195510864258,
      "activations/layer16_attention_weight_min": -36.722381591796875,
      "activations/layer17_attention_weight_max": 44.51191329956055,
      "activations/layer17_attention_weight_min": -29.75453758239746,
      "activations/layer18_attention_weight_max": 25.183645248413086,
      "activations/layer18_attention_weight_min": -14.796412467956543,
      "activations/layer19_attention_weight_max": 53.39882278442383,
      "activations/layer19_attention_weight_min": -30.46095848083496,
      "activations/layer1_attention_weight_max": 18.57108497619629,
      "activations/layer1_attention_weight_min": -11.760622024536133,
      "activations/layer20_attention_weight_max": 41.20012283325195,
      "activations/layer20_attention_weight_min": -26.37117576599121,
      "activations/layer21_attention_weight_max": 26.648338317871094,
      "activations/layer21_attention_weight_min": -17.651229858398438,
      "activations/layer22_attention_weight_max": 41.59615707397461,
      "activations/layer22_attention_weight_min": -22.623268127441406,
      "activations/layer23_attention_weight_max": 23.458602905273438,
      "activations/layer23_attention_weight_min": -17.023902893066406,
      "activations/layer2_attention_weight_max": 27.92790985107422,
      "activations/layer2_attention_weight_min": -26.137287139892578,
      "activations/layer3_attention_weight_max": 63.10411071777344,
      "activations/layer3_attention_weight_min": -66.39702606201172,
      "activations/layer4_attention_weight_max": 122.64861297607422,
      "activations/layer4_attention_weight_min": -112.3807601928711,
      "activations/layer5_attention_weight_max": 128.07533264160156,
      "activations/layer5_attention_weight_min": -112.04606628417969,
      "activations/layer6_attention_weight_max": 68.6923828125,
      "activations/layer6_attention_weight_min": -56.8522834777832,
      "activations/layer7_attention_weight_max": 67.79759979248047,
      "activations/layer7_attention_weight_min": -62.1614990234375,
      "activations/layer8_attention_weight_max": 53.37931823730469,
      "activations/layer8_attention_weight_min": -43.432594299316406,
      "activations/layer9_attention_weight_max": 78.09191131591797,
      "activations/layer9_attention_weight_min": -70.56932830810547,
      "epoch": 1.48,
      "learning_rate": 0.00014188939393939393,
      "loss": 3.0238,
      "step": 25450
    },
    {
      "activations/layer0_attention_weight_max": 16.00716209411621,
      "activations/layer0_attention_weight_min": -12.338854789733887,
      "activations/layer10_attention_weight_max": 63.6789665222168,
      "activations/layer10_attention_weight_min": -49.26092529296875,
      "activations/layer11_attention_weight_max": 43.91210174560547,
      "activations/layer11_attention_weight_min": -38.950618743896484,
      "activations/layer12_attention_weight_max": 35.64344787597656,
      "activations/layer12_attention_weight_min": -26.494159698486328,
      "activations/layer13_attention_weight_max": 51.065006256103516,
      "activations/layer13_attention_weight_min": -35.69147872924805,
      "activations/layer14_attention_weight_max": 88.80992126464844,
      "activations/layer14_attention_weight_min": -56.93898010253906,
      "activations/layer15_attention_weight_max": 55.804481506347656,
      "activations/layer15_attention_weight_min": -42.17344284057617,
      "activations/layer16_attention_weight_max": 52.475318908691406,
      "activations/layer16_attention_weight_min": -42.17281723022461,
      "activations/layer17_attention_weight_max": 45.62805938720703,
      "activations/layer17_attention_weight_min": -29.744354248046875,
      "activations/layer18_attention_weight_max": 26.423986434936523,
      "activations/layer18_attention_weight_min": -15.73089599609375,
      "activations/layer19_attention_weight_max": 58.87382888793945,
      "activations/layer19_attention_weight_min": -32.18833541870117,
      "activations/layer1_attention_weight_max": 17.164878845214844,
      "activations/layer1_attention_weight_min": -11.306755065917969,
      "activations/layer20_attention_weight_max": 37.78656768798828,
      "activations/layer20_attention_weight_min": -24.956748962402344,
      "activations/layer21_attention_weight_max": 22.96068572998047,
      "activations/layer21_attention_weight_min": -18.096534729003906,
      "activations/layer22_attention_weight_max": 43.51850128173828,
      "activations/layer22_attention_weight_min": -24.575361251831055,
      "activations/layer23_attention_weight_max": 25.538429260253906,
      "activations/layer23_attention_weight_min": -14.373161315917969,
      "activations/layer2_attention_weight_max": 26.906356811523438,
      "activations/layer2_attention_weight_min": -27.641971588134766,
      "activations/layer3_attention_weight_max": 68.8556137084961,
      "activations/layer3_attention_weight_min": -71.51197052001953,
      "activations/layer4_attention_weight_max": 117.50279998779297,
      "activations/layer4_attention_weight_min": -112.56488037109375,
      "activations/layer5_attention_weight_max": 114.91741943359375,
      "activations/layer5_attention_weight_min": -99.89930725097656,
      "activations/layer6_attention_weight_max": 61.58185577392578,
      "activations/layer6_attention_weight_min": -54.75580978393555,
      "activations/layer7_attention_weight_max": 63.79599380493164,
      "activations/layer7_attention_weight_min": -62.67987823486328,
      "activations/layer8_attention_weight_max": 54.069454193115234,
      "activations/layer8_attention_weight_min": -45.67890930175781,
      "activations/layer9_attention_weight_max": 74.61389923095703,
      "activations/layer9_attention_weight_min": -69.23199462890625,
      "epoch": 1.48,
      "learning_rate": 0.00014187045454545452,
      "loss": 3.0382,
      "step": 25500
    },
    {
      "activations/layer0_attention_weight_max": 16.525432586669922,
      "activations/layer0_attention_weight_min": -13.68140697479248,
      "activations/layer10_attention_weight_max": 65.6236572265625,
      "activations/layer10_attention_weight_min": -45.52370071411133,
      "activations/layer11_attention_weight_max": 50.407470703125,
      "activations/layer11_attention_weight_min": -37.953102111816406,
      "activations/layer12_attention_weight_max": 52.82597351074219,
      "activations/layer12_attention_weight_min": -33.55219650268555,
      "activations/layer13_attention_weight_max": 60.82856750488281,
      "activations/layer13_attention_weight_min": -39.50526428222656,
      "activations/layer14_attention_weight_max": 91.67363739013672,
      "activations/layer14_attention_weight_min": -56.8440055847168,
      "activations/layer15_attention_weight_max": 71.51354217529297,
      "activations/layer15_attention_weight_min": -40.86725616455078,
      "activations/layer16_attention_weight_max": 58.90618133544922,
      "activations/layer16_attention_weight_min": -39.13911056518555,
      "activations/layer17_attention_weight_max": 53.62294387817383,
      "activations/layer17_attention_weight_min": -29.56962013244629,
      "activations/layer18_attention_weight_max": 27.1453857421875,
      "activations/layer18_attention_weight_min": -16.307449340820312,
      "activations/layer19_attention_weight_max": 49.072113037109375,
      "activations/layer19_attention_weight_min": -31.92356300354004,
      "activations/layer1_attention_weight_max": 22.089948654174805,
      "activations/layer1_attention_weight_min": -15.157812118530273,
      "activations/layer20_attention_weight_max": 38.399253845214844,
      "activations/layer20_attention_weight_min": -23.217248916625977,
      "activations/layer21_attention_weight_max": 23.011920928955078,
      "activations/layer21_attention_weight_min": -17.333892822265625,
      "activations/layer22_attention_weight_max": 41.83623123168945,
      "activations/layer22_attention_weight_min": -23.050216674804688,
      "activations/layer23_attention_weight_max": 24.34815216064453,
      "activations/layer23_attention_weight_min": -14.145270347595215,
      "activations/layer2_attention_weight_max": 33.413970947265625,
      "activations/layer2_attention_weight_min": -27.38148307800293,
      "activations/layer3_attention_weight_max": 62.256797790527344,
      "activations/layer3_attention_weight_min": -62.52492141723633,
      "activations/layer4_attention_weight_max": 122.6384048461914,
      "activations/layer4_attention_weight_min": -114.6549301147461,
      "activations/layer5_attention_weight_max": 139.58206176757812,
      "activations/layer5_attention_weight_min": -124.4820785522461,
      "activations/layer6_attention_weight_max": 75.8170394897461,
      "activations/layer6_attention_weight_min": -66.07978057861328,
      "activations/layer7_attention_weight_max": 67.37596893310547,
      "activations/layer7_attention_weight_min": -74.9495849609375,
      "activations/layer8_attention_weight_max": 60.1568489074707,
      "activations/layer8_attention_weight_min": -50.25616455078125,
      "activations/layer9_attention_weight_max": 92.24300384521484,
      "activations/layer9_attention_weight_min": -68.0672836303711,
      "epoch": 1.48,
      "learning_rate": 0.00014185189393939394,
      "loss": 3.0552,
      "step": 25550
    },
    {
      "activations/layer0_attention_weight_max": 14.392865180969238,
      "activations/layer0_attention_weight_min": -11.270115852355957,
      "activations/layer10_attention_weight_max": 56.38593673706055,
      "activations/layer10_attention_weight_min": -48.03221130371094,
      "activations/layer11_attention_weight_max": 39.74099349975586,
      "activations/layer11_attention_weight_min": -36.34546661376953,
      "activations/layer12_attention_weight_max": 33.47641372680664,
      "activations/layer12_attention_weight_min": -26.52271842956543,
      "activations/layer13_attention_weight_max": 48.052974700927734,
      "activations/layer13_attention_weight_min": -35.95020294189453,
      "activations/layer14_attention_weight_max": 94.8643798828125,
      "activations/layer14_attention_weight_min": -62.656455993652344,
      "activations/layer15_attention_weight_max": 63.22803497314453,
      "activations/layer15_attention_weight_min": -39.79533004760742,
      "activations/layer16_attention_weight_max": 55.90654373168945,
      "activations/layer16_attention_weight_min": -40.25557327270508,
      "activations/layer17_attention_weight_max": 47.672401428222656,
      "activations/layer17_attention_weight_min": -29.37854766845703,
      "activations/layer18_attention_weight_max": 27.165264129638672,
      "activations/layer18_attention_weight_min": -16.50570297241211,
      "activations/layer19_attention_weight_max": 62.30458450317383,
      "activations/layer19_attention_weight_min": -33.44938659667969,
      "activations/layer1_attention_weight_max": 16.279685974121094,
      "activations/layer1_attention_weight_min": -11.426962852478027,
      "activations/layer20_attention_weight_max": 44.57551193237305,
      "activations/layer20_attention_weight_min": -23.504796981811523,
      "activations/layer21_attention_weight_max": 30.254806518554688,
      "activations/layer21_attention_weight_min": -18.099157333374023,
      "activations/layer22_attention_weight_max": 43.33170700073242,
      "activations/layer22_attention_weight_min": -24.672571182250977,
      "activations/layer23_attention_weight_max": 24.66223907470703,
      "activations/layer23_attention_weight_min": -14.272727012634277,
      "activations/layer2_attention_weight_max": 25.171735763549805,
      "activations/layer2_attention_weight_min": -21.285554885864258,
      "activations/layer3_attention_weight_max": 53.571929931640625,
      "activations/layer3_attention_weight_min": -57.700191497802734,
      "activations/layer4_attention_weight_max": 102.55006408691406,
      "activations/layer4_attention_weight_min": -97.96078491210938,
      "activations/layer5_attention_weight_max": 101.71389770507812,
      "activations/layer5_attention_weight_min": -97.00376892089844,
      "activations/layer6_attention_weight_max": 55.68206024169922,
      "activations/layer6_attention_weight_min": -52.04435729980469,
      "activations/layer7_attention_weight_max": 61.38362503051758,
      "activations/layer7_attention_weight_min": -60.924644470214844,
      "activations/layer8_attention_weight_max": 51.672245025634766,
      "activations/layer8_attention_weight_min": -48.563236236572266,
      "activations/layer9_attention_weight_max": 74.15526580810547,
      "activations/layer9_attention_weight_min": -56.99946212768555,
      "epoch": 1.49,
      "learning_rate": 0.00014183333333333333,
      "loss": 3.0839,
      "step": 25600
    },
    {
      "activations/layer0_attention_weight_max": 15.139276504516602,
      "activations/layer0_attention_weight_min": -10.864042282104492,
      "activations/layer10_attention_weight_max": 49.665218353271484,
      "activations/layer10_attention_weight_min": -46.8294677734375,
      "activations/layer11_attention_weight_max": 38.311309814453125,
      "activations/layer11_attention_weight_min": -41.375919342041016,
      "activations/layer12_attention_weight_max": 32.67461395263672,
      "activations/layer12_attention_weight_min": -25.975818634033203,
      "activations/layer13_attention_weight_max": 45.225399017333984,
      "activations/layer13_attention_weight_min": -38.52743148803711,
      "activations/layer14_attention_weight_max": 66.45073699951172,
      "activations/layer14_attention_weight_min": -60.44917678833008,
      "activations/layer15_attention_weight_max": 50.093467712402344,
      "activations/layer15_attention_weight_min": -38.69792175292969,
      "activations/layer16_attention_weight_max": 49.3481330871582,
      "activations/layer16_attention_weight_min": -38.36880111694336,
      "activations/layer17_attention_weight_max": 38.814422607421875,
      "activations/layer17_attention_weight_min": -26.393720626831055,
      "activations/layer18_attention_weight_max": 26.087404251098633,
      "activations/layer18_attention_weight_min": -15.688302040100098,
      "activations/layer19_attention_weight_max": 62.007598876953125,
      "activations/layer19_attention_weight_min": -34.652381896972656,
      "activations/layer1_attention_weight_max": 16.52296257019043,
      "activations/layer1_attention_weight_min": -11.09429931640625,
      "activations/layer20_attention_weight_max": 35.6270866394043,
      "activations/layer20_attention_weight_min": -24.29376220703125,
      "activations/layer21_attention_weight_max": 27.51558494567871,
      "activations/layer21_attention_weight_min": -15.528923034667969,
      "activations/layer22_attention_weight_max": 40.41444778442383,
      "activations/layer22_attention_weight_min": -24.069580078125,
      "activations/layer23_attention_weight_max": 23.34906768798828,
      "activations/layer23_attention_weight_min": -18.725845336914062,
      "activations/layer2_attention_weight_max": 25.031131744384766,
      "activations/layer2_attention_weight_min": -21.993732452392578,
      "activations/layer3_attention_weight_max": 59.96189498901367,
      "activations/layer3_attention_weight_min": -62.731956481933594,
      "activations/layer4_attention_weight_max": 103.89570617675781,
      "activations/layer4_attention_weight_min": -106.67877197265625,
      "activations/layer5_attention_weight_max": 108.16253662109375,
      "activations/layer5_attention_weight_min": -102.41293334960938,
      "activations/layer6_attention_weight_max": 57.82797622680664,
      "activations/layer6_attention_weight_min": -53.9188117980957,
      "activations/layer7_attention_weight_max": 62.035057067871094,
      "activations/layer7_attention_weight_min": -60.48753356933594,
      "activations/layer8_attention_weight_max": 51.623558044433594,
      "activations/layer8_attention_weight_min": -47.63103103637695,
      "activations/layer9_attention_weight_max": 71.14698791503906,
      "activations/layer9_attention_weight_min": -64.16381072998047,
      "epoch": 1.49,
      "learning_rate": 0.00014181439393939392,
      "loss": 3.0347,
      "step": 25650
    },
    {
      "activations/layer0_attention_weight_max": 15.878375053405762,
      "activations/layer0_attention_weight_min": -10.229990005493164,
      "activations/layer10_attention_weight_max": 53.001033782958984,
      "activations/layer10_attention_weight_min": -48.51985549926758,
      "activations/layer11_attention_weight_max": 39.895687103271484,
      "activations/layer11_attention_weight_min": -36.22989273071289,
      "activations/layer12_attention_weight_max": 35.79376220703125,
      "activations/layer12_attention_weight_min": -29.499897003173828,
      "activations/layer13_attention_weight_max": 49.40009307861328,
      "activations/layer13_attention_weight_min": -38.57210922241211,
      "activations/layer14_attention_weight_max": 74.58924865722656,
      "activations/layer14_attention_weight_min": -56.35036849975586,
      "activations/layer15_attention_weight_max": 53.291259765625,
      "activations/layer15_attention_weight_min": -39.422977447509766,
      "activations/layer16_attention_weight_max": 50.59046173095703,
      "activations/layer16_attention_weight_min": -37.24580001831055,
      "activations/layer17_attention_weight_max": 40.115936279296875,
      "activations/layer17_attention_weight_min": -29.26544189453125,
      "activations/layer18_attention_weight_max": 25.03487777709961,
      "activations/layer18_attention_weight_min": -17.219696044921875,
      "activations/layer19_attention_weight_max": 48.382442474365234,
      "activations/layer19_attention_weight_min": -31.081735610961914,
      "activations/layer1_attention_weight_max": 16.12225341796875,
      "activations/layer1_attention_weight_min": -11.598893165588379,
      "activations/layer20_attention_weight_max": 44.27790451049805,
      "activations/layer20_attention_weight_min": -27.045303344726562,
      "activations/layer21_attention_weight_max": 27.124706268310547,
      "activations/layer21_attention_weight_min": -17.36238670349121,
      "activations/layer22_attention_weight_max": 43.16096496582031,
      "activations/layer22_attention_weight_min": -25.4964656829834,
      "activations/layer23_attention_weight_max": 24.108776092529297,
      "activations/layer23_attention_weight_min": -15.063974380493164,
      "activations/layer2_attention_weight_max": 26.269031524658203,
      "activations/layer2_attention_weight_min": -23.691869735717773,
      "activations/layer3_attention_weight_max": 57.47856140136719,
      "activations/layer3_attention_weight_min": -65.14598846435547,
      "activations/layer4_attention_weight_max": 104.47466278076172,
      "activations/layer4_attention_weight_min": -107.1125259399414,
      "activations/layer5_attention_weight_max": 105.96330261230469,
      "activations/layer5_attention_weight_min": -102.18965148925781,
      "activations/layer6_attention_weight_max": 54.57324981689453,
      "activations/layer6_attention_weight_min": -53.04290008544922,
      "activations/layer7_attention_weight_max": 64.8942642211914,
      "activations/layer7_attention_weight_min": -60.3446159362793,
      "activations/layer8_attention_weight_max": 46.8252067565918,
      "activations/layer8_attention_weight_min": -45.0958137512207,
      "activations/layer9_attention_weight_max": 68.82947540283203,
      "activations/layer9_attention_weight_min": -55.791378021240234,
      "epoch": 1.49,
      "learning_rate": 0.00014179545454545453,
      "loss": 3.0294,
      "step": 25700
    },
    {
      "activations/layer0_attention_weight_max": 15.758463859558105,
      "activations/layer0_attention_weight_min": -9.582869529724121,
      "activations/layer10_attention_weight_max": 49.65986251831055,
      "activations/layer10_attention_weight_min": -42.92598342895508,
      "activations/layer11_attention_weight_max": 42.89564514160156,
      "activations/layer11_attention_weight_min": -35.00859832763672,
      "activations/layer12_attention_weight_max": 30.290695190429688,
      "activations/layer12_attention_weight_min": -26.139562606811523,
      "activations/layer13_attention_weight_max": 50.90351486206055,
      "activations/layer13_attention_weight_min": -40.444087982177734,
      "activations/layer14_attention_weight_max": 75.65340423583984,
      "activations/layer14_attention_weight_min": -55.98640823364258,
      "activations/layer15_attention_weight_max": 54.17386245727539,
      "activations/layer15_attention_weight_min": -40.64942169189453,
      "activations/layer16_attention_weight_max": 54.72605895996094,
      "activations/layer16_attention_weight_min": -40.623695373535156,
      "activations/layer17_attention_weight_max": 39.93361282348633,
      "activations/layer17_attention_weight_min": -28.8389835357666,
      "activations/layer18_attention_weight_max": 27.212181091308594,
      "activations/layer18_attention_weight_min": -16.398483276367188,
      "activations/layer19_attention_weight_max": 53.767066955566406,
      "activations/layer19_attention_weight_min": -30.691404342651367,
      "activations/layer1_attention_weight_max": 17.49370002746582,
      "activations/layer1_attention_weight_min": -11.196513175964355,
      "activations/layer20_attention_weight_max": 42.88368606567383,
      "activations/layer20_attention_weight_min": -26.760009765625,
      "activations/layer21_attention_weight_max": 27.388423919677734,
      "activations/layer21_attention_weight_min": -16.293865203857422,
      "activations/layer22_attention_weight_max": 45.384971618652344,
      "activations/layer22_attention_weight_min": -23.48188018798828,
      "activations/layer23_attention_weight_max": 25.167356491088867,
      "activations/layer23_attention_weight_min": -14.873183250427246,
      "activations/layer2_attention_weight_max": 25.204742431640625,
      "activations/layer2_attention_weight_min": -23.545509338378906,
      "activations/layer3_attention_weight_max": 57.01415252685547,
      "activations/layer3_attention_weight_min": -59.36207580566406,
      "activations/layer4_attention_weight_max": 108.23725891113281,
      "activations/layer4_attention_weight_min": -101.57469940185547,
      "activations/layer5_attention_weight_max": 110.12969970703125,
      "activations/layer5_attention_weight_min": -93.87696838378906,
      "activations/layer6_attention_weight_max": 60.3297233581543,
      "activations/layer6_attention_weight_min": -56.43518829345703,
      "activations/layer7_attention_weight_max": 63.8366813659668,
      "activations/layer7_attention_weight_min": -62.98612594604492,
      "activations/layer8_attention_weight_max": 49.16524887084961,
      "activations/layer8_attention_weight_min": -46.500118255615234,
      "activations/layer9_attention_weight_max": 85.33037567138672,
      "activations/layer9_attention_weight_min": -68.2098388671875,
      "epoch": 1.5,
      "learning_rate": 0.00014177651515151515,
      "loss": 3.0317,
      "step": 25750
    },
    {
      "activations/layer0_attention_weight_max": 16.111621856689453,
      "activations/layer0_attention_weight_min": -9.65791130065918,
      "activations/layer10_attention_weight_max": 58.92329025268555,
      "activations/layer10_attention_weight_min": -50.64556884765625,
      "activations/layer11_attention_weight_max": 44.341941833496094,
      "activations/layer11_attention_weight_min": -34.66512680053711,
      "activations/layer12_attention_weight_max": 45.03733825683594,
      "activations/layer12_attention_weight_min": -30.901996612548828,
      "activations/layer13_attention_weight_max": 72.2048568725586,
      "activations/layer13_attention_weight_min": -51.84028244018555,
      "activations/layer14_attention_weight_max": 77.96353912353516,
      "activations/layer14_attention_weight_min": -65.34188842773438,
      "activations/layer15_attention_weight_max": 59.48969268798828,
      "activations/layer15_attention_weight_min": -41.3592414855957,
      "activations/layer16_attention_weight_max": 72.46370697021484,
      "activations/layer16_attention_weight_min": -44.24763488769531,
      "activations/layer17_attention_weight_max": 46.79206466674805,
      "activations/layer17_attention_weight_min": -29.32510757446289,
      "activations/layer18_attention_weight_max": 29.872859954833984,
      "activations/layer18_attention_weight_min": -18.763355255126953,
      "activations/layer19_attention_weight_max": 63.83225631713867,
      "activations/layer19_attention_weight_min": -32.983238220214844,
      "activations/layer1_attention_weight_max": 16.500083923339844,
      "activations/layer1_attention_weight_min": -11.659207344055176,
      "activations/layer20_attention_weight_max": 44.767147064208984,
      "activations/layer20_attention_weight_min": -27.98440170288086,
      "activations/layer21_attention_weight_max": 28.37421226501465,
      "activations/layer21_attention_weight_min": -17.372804641723633,
      "activations/layer22_attention_weight_max": 47.73112106323242,
      "activations/layer22_attention_weight_min": -22.733314514160156,
      "activations/layer23_attention_weight_max": 27.241857528686523,
      "activations/layer23_attention_weight_min": -15.632865905761719,
      "activations/layer2_attention_weight_max": 29.014923095703125,
      "activations/layer2_attention_weight_min": -24.127851486206055,
      "activations/layer3_attention_weight_max": 64.35179901123047,
      "activations/layer3_attention_weight_min": -63.381500244140625,
      "activations/layer4_attention_weight_max": 115.82513427734375,
      "activations/layer4_attention_weight_min": -105.62801361083984,
      "activations/layer5_attention_weight_max": 119.57357025146484,
      "activations/layer5_attention_weight_min": -98.50249481201172,
      "activations/layer6_attention_weight_max": 62.82495880126953,
      "activations/layer6_attention_weight_min": -53.72518539428711,
      "activations/layer7_attention_weight_max": 64.63831329345703,
      "activations/layer7_attention_weight_min": -58.222110748291016,
      "activations/layer8_attention_weight_max": 56.22362518310547,
      "activations/layer8_attention_weight_min": -45.564796447753906,
      "activations/layer9_attention_weight_max": 71.63373565673828,
      "activations/layer9_attention_weight_min": -57.10413360595703,
      "epoch": 1.5,
      "learning_rate": 0.00014175757575757574,
      "loss": 3.0076,
      "step": 25800
    },
    {
      "activations/layer0_attention_weight_max": 14.536279678344727,
      "activations/layer0_attention_weight_min": -9.894501686096191,
      "activations/layer10_attention_weight_max": 54.395713806152344,
      "activations/layer10_attention_weight_min": -44.46355056762695,
      "activations/layer11_attention_weight_max": 44.06299591064453,
      "activations/layer11_attention_weight_min": -36.97309494018555,
      "activations/layer12_attention_weight_max": 38.95581817626953,
      "activations/layer12_attention_weight_min": -25.796457290649414,
      "activations/layer13_attention_weight_max": 53.607383728027344,
      "activations/layer13_attention_weight_min": -37.9397087097168,
      "activations/layer14_attention_weight_max": 83.73393249511719,
      "activations/layer14_attention_weight_min": -61.46234130859375,
      "activations/layer15_attention_weight_max": 55.693511962890625,
      "activations/layer15_attention_weight_min": -42.114036560058594,
      "activations/layer16_attention_weight_max": 62.89449691772461,
      "activations/layer16_attention_weight_min": -39.95911407470703,
      "activations/layer17_attention_weight_max": 51.121437072753906,
      "activations/layer17_attention_weight_min": -32.45937728881836,
      "activations/layer18_attention_weight_max": 27.066818237304688,
      "activations/layer18_attention_weight_min": -17.142135620117188,
      "activations/layer19_attention_weight_max": 67.94720458984375,
      "activations/layer19_attention_weight_min": -37.01249313354492,
      "activations/layer1_attention_weight_max": 16.29142951965332,
      "activations/layer1_attention_weight_min": -11.583215713500977,
      "activations/layer20_attention_weight_max": 42.19255447387695,
      "activations/layer20_attention_weight_min": -26.510475158691406,
      "activations/layer21_attention_weight_max": 27.390737533569336,
      "activations/layer21_attention_weight_min": -16.411474227905273,
      "activations/layer22_attention_weight_max": 46.689083099365234,
      "activations/layer22_attention_weight_min": -24.92945671081543,
      "activations/layer23_attention_weight_max": 26.07037353515625,
      "activations/layer23_attention_weight_min": -14.97645378112793,
      "activations/layer2_attention_weight_max": 25.941608428955078,
      "activations/layer2_attention_weight_min": -24.659530639648438,
      "activations/layer3_attention_weight_max": 61.57870101928711,
      "activations/layer3_attention_weight_min": -65.1532974243164,
      "activations/layer4_attention_weight_max": 110.04487609863281,
      "activations/layer4_attention_weight_min": -108.88966369628906,
      "activations/layer5_attention_weight_max": 111.62400817871094,
      "activations/layer5_attention_weight_min": -99.67916870117188,
      "activations/layer6_attention_weight_max": 61.631324768066406,
      "activations/layer6_attention_weight_min": -52.797935485839844,
      "activations/layer7_attention_weight_max": 74.314208984375,
      "activations/layer7_attention_weight_min": -64.02540588378906,
      "activations/layer8_attention_weight_max": 56.6164665222168,
      "activations/layer8_attention_weight_min": -46.29546356201172,
      "activations/layer9_attention_weight_max": 76.59252166748047,
      "activations/layer9_attention_weight_min": -61.2808952331543,
      "epoch": 1.5,
      "learning_rate": 0.00014173863636363635,
      "loss": 3.03,
      "step": 25850
    },
    {
      "activations/layer0_attention_weight_max": 15.718731880187988,
      "activations/layer0_attention_weight_min": -9.866547584533691,
      "activations/layer10_attention_weight_max": 50.1211051940918,
      "activations/layer10_attention_weight_min": -42.96556854248047,
      "activations/layer11_attention_weight_max": 38.831504821777344,
      "activations/layer11_attention_weight_min": -36.640846252441406,
      "activations/layer12_attention_weight_max": 31.335403442382812,
      "activations/layer12_attention_weight_min": -26.304166793823242,
      "activations/layer13_attention_weight_max": 47.45813751220703,
      "activations/layer13_attention_weight_min": -37.47453308105469,
      "activations/layer14_attention_weight_max": 72.76708984375,
      "activations/layer14_attention_weight_min": -53.395816802978516,
      "activations/layer15_attention_weight_max": 53.71729278564453,
      "activations/layer15_attention_weight_min": -38.398597717285156,
      "activations/layer16_attention_weight_max": 52.466636657714844,
      "activations/layer16_attention_weight_min": -40.07960891723633,
      "activations/layer17_attention_weight_max": 40.213897705078125,
      "activations/layer17_attention_weight_min": -26.907344818115234,
      "activations/layer18_attention_weight_max": 26.732791900634766,
      "activations/layer18_attention_weight_min": -14.60060977935791,
      "activations/layer19_attention_weight_max": 54.87849044799805,
      "activations/layer19_attention_weight_min": -34.7403450012207,
      "activations/layer1_attention_weight_max": 17.417638778686523,
      "activations/layer1_attention_weight_min": -11.614995956420898,
      "activations/layer20_attention_weight_max": 41.47171401977539,
      "activations/layer20_attention_weight_min": -26.054174423217773,
      "activations/layer21_attention_weight_max": 35.343265533447266,
      "activations/layer21_attention_weight_min": -19.37578773498535,
      "activations/layer22_attention_weight_max": 43.447566986083984,
      "activations/layer22_attention_weight_min": -26.69203758239746,
      "activations/layer23_attention_weight_max": 29.215543746948242,
      "activations/layer23_attention_weight_min": -16.805347442626953,
      "activations/layer2_attention_weight_max": 26.4320068359375,
      "activations/layer2_attention_weight_min": -25.384227752685547,
      "activations/layer3_attention_weight_max": 62.8380012512207,
      "activations/layer3_attention_weight_min": -69.20796966552734,
      "activations/layer4_attention_weight_max": 116.79534912109375,
      "activations/layer4_attention_weight_min": -112.5121078491211,
      "activations/layer5_attention_weight_max": 116.32084655761719,
      "activations/layer5_attention_weight_min": -103.05265808105469,
      "activations/layer6_attention_weight_max": 62.91880416870117,
      "activations/layer6_attention_weight_min": -56.103607177734375,
      "activations/layer7_attention_weight_max": 60.9930305480957,
      "activations/layer7_attention_weight_min": -59.21733856201172,
      "activations/layer8_attention_weight_max": 53.34212112426758,
      "activations/layer8_attention_weight_min": -49.207698822021484,
      "activations/layer9_attention_weight_max": 64.5525131225586,
      "activations/layer9_attention_weight_min": -57.26388168334961,
      "epoch": 1.5,
      "learning_rate": 0.00014171969696969694,
      "loss": 3.0134,
      "step": 25900
    },
    {
      "activations/layer0_attention_weight_max": 15.96937084197998,
      "activations/layer0_attention_weight_min": -9.710461616516113,
      "activations/layer10_attention_weight_max": 57.83251190185547,
      "activations/layer10_attention_weight_min": -43.45232009887695,
      "activations/layer11_attention_weight_max": 43.791168212890625,
      "activations/layer11_attention_weight_min": -34.762847900390625,
      "activations/layer12_attention_weight_max": 75.7069091796875,
      "activations/layer12_attention_weight_min": -38.82033920288086,
      "activations/layer13_attention_weight_max": 55.81217956542969,
      "activations/layer13_attention_weight_min": -40.108802795410156,
      "activations/layer14_attention_weight_max": 72.53175354003906,
      "activations/layer14_attention_weight_min": -57.229331970214844,
      "activations/layer15_attention_weight_max": 54.11789321899414,
      "activations/layer15_attention_weight_min": -38.540470123291016,
      "activations/layer16_attention_weight_max": 54.44843673706055,
      "activations/layer16_attention_weight_min": -36.70197296142578,
      "activations/layer17_attention_weight_max": 38.579872131347656,
      "activations/layer17_attention_weight_min": -27.1903076171875,
      "activations/layer18_attention_weight_max": 24.69060516357422,
      "activations/layer18_attention_weight_min": -13.987957000732422,
      "activations/layer19_attention_weight_max": 54.597869873046875,
      "activations/layer19_attention_weight_min": -32.69415283203125,
      "activations/layer1_attention_weight_max": 16.469602584838867,
      "activations/layer1_attention_weight_min": -12.510370254516602,
      "activations/layer20_attention_weight_max": 44.422630310058594,
      "activations/layer20_attention_weight_min": -23.446483612060547,
      "activations/layer21_attention_weight_max": 28.633281707763672,
      "activations/layer21_attention_weight_min": -18.804973602294922,
      "activations/layer22_attention_weight_max": 47.829532623291016,
      "activations/layer22_attention_weight_min": -22.2966365814209,
      "activations/layer23_attention_weight_max": 24.69326400756836,
      "activations/layer23_attention_weight_min": -14.31201457977295,
      "activations/layer2_attention_weight_max": 25.877552032470703,
      "activations/layer2_attention_weight_min": -24.526268005371094,
      "activations/layer3_attention_weight_max": 59.74421310424805,
      "activations/layer3_attention_weight_min": -64.73404693603516,
      "activations/layer4_attention_weight_max": 113.54422760009766,
      "activations/layer4_attention_weight_min": -107.21878814697266,
      "activations/layer5_attention_weight_max": 117.26939392089844,
      "activations/layer5_attention_weight_min": -100.33556365966797,
      "activations/layer6_attention_weight_max": 66.38333892822266,
      "activations/layer6_attention_weight_min": -56.26609802246094,
      "activations/layer7_attention_weight_max": 60.38180160522461,
      "activations/layer7_attention_weight_min": -58.67154312133789,
      "activations/layer8_attention_weight_max": 62.86000061035156,
      "activations/layer8_attention_weight_min": -51.24089050292969,
      "activations/layer9_attention_weight_max": 83.33320617675781,
      "activations/layer9_attention_weight_min": -65.14439392089844,
      "epoch": 1.51,
      "learning_rate": 0.00014170075757575755,
      "loss": 3.0145,
      "step": 25950
    },
    {
      "activations/layer0_attention_weight_max": 16.085859298706055,
      "activations/layer0_attention_weight_min": -10.125383377075195,
      "activations/layer10_attention_weight_max": 61.53632736206055,
      "activations/layer10_attention_weight_min": -48.90940475463867,
      "activations/layer11_attention_weight_max": 42.74420166015625,
      "activations/layer11_attention_weight_min": -37.109779357910156,
      "activations/layer12_attention_weight_max": 45.70378112792969,
      "activations/layer12_attention_weight_min": -34.68416213989258,
      "activations/layer13_attention_weight_max": 49.21674346923828,
      "activations/layer13_attention_weight_min": -37.724632263183594,
      "activations/layer14_attention_weight_max": 81.14354705810547,
      "activations/layer14_attention_weight_min": -60.085697174072266,
      "activations/layer15_attention_weight_max": 61.89564895629883,
      "activations/layer15_attention_weight_min": -40.35094451904297,
      "activations/layer16_attention_weight_max": 51.7762565612793,
      "activations/layer16_attention_weight_min": -39.76533508300781,
      "activations/layer17_attention_weight_max": 40.702632904052734,
      "activations/layer17_attention_weight_min": -27.67936134338379,
      "activations/layer18_attention_weight_max": 24.68903160095215,
      "activations/layer18_attention_weight_min": -19.82847785949707,
      "activations/layer19_attention_weight_max": 47.94551467895508,
      "activations/layer19_attention_weight_min": -33.127193450927734,
      "activations/layer1_attention_weight_max": 17.349830627441406,
      "activations/layer1_attention_weight_min": -11.535017967224121,
      "activations/layer20_attention_weight_max": 36.70170974731445,
      "activations/layer20_attention_weight_min": -25.002944946289062,
      "activations/layer21_attention_weight_max": 27.489843368530273,
      "activations/layer21_attention_weight_min": -19.32996368408203,
      "activations/layer22_attention_weight_max": 42.43605041503906,
      "activations/layer22_attention_weight_min": -25.00566864013672,
      "activations/layer23_attention_weight_max": 25.8062801361084,
      "activations/layer23_attention_weight_min": -15.302879333496094,
      "activations/layer2_attention_weight_max": 27.481395721435547,
      "activations/layer2_attention_weight_min": -25.179527282714844,
      "activations/layer3_attention_weight_max": 62.879398345947266,
      "activations/layer3_attention_weight_min": -65.57233428955078,
      "activations/layer4_attention_weight_max": 118.6825180053711,
      "activations/layer4_attention_weight_min": -108.84767150878906,
      "activations/layer5_attention_weight_max": 115.1173095703125,
      "activations/layer5_attention_weight_min": -103.26930236816406,
      "activations/layer6_attention_weight_max": 62.127967834472656,
      "activations/layer6_attention_weight_min": -53.067283630371094,
      "activations/layer7_attention_weight_max": 62.98255157470703,
      "activations/layer7_attention_weight_min": -62.3477668762207,
      "activations/layer8_attention_weight_max": 53.8334846496582,
      "activations/layer8_attention_weight_min": -45.40703201293945,
      "activations/layer9_attention_weight_max": 79.04376220703125,
      "activations/layer9_attention_weight_min": -63.61572265625,
      "epoch": 1.51,
      "learning_rate": 0.00014168181818181817,
      "loss": 3.0165,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_loss": 2.962890625,
      "eval_runtime": 8.4611,
      "eval_samples_per_second": 507.501,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_openwebtext_loss": 2.962890625,
      "eval_openwebtext_ppl": 19.353835657005096,
      "eval_openwebtext_runtime": 8.4611,
      "eval_openwebtext_samples_per_second": 507.501,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_wikitext_loss": 3.2265625,
      "eval_wikitext_ppl": 25.192907336481866,
      "eval_wikitext_runtime": 1.9643,
      "eval_wikitext_samples_per_second": 232.143,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_lambada_loss": 3.193359375,
      "eval_lambada_ppl": 24.370158584949497,
      "eval_lambada_runtime": 9.5691,
      "eval_lambada_samples_per_second": 508.823,
      "step": 26000
    },
    {
      "activations/layer0_attention_weight_max": 16.04745864868164,
      "activations/layer0_attention_weight_min": -10.570018768310547,
      "activations/layer10_attention_weight_max": 55.91963577270508,
      "activations/layer10_attention_weight_min": -49.81215286254883,
      "activations/layer11_attention_weight_max": 41.97486877441406,
      "activations/layer11_attention_weight_min": -37.844566345214844,
      "activations/layer12_attention_weight_max": 39.4240608215332,
      "activations/layer12_attention_weight_min": -30.26167106628418,
      "activations/layer13_attention_weight_max": 51.05122375488281,
      "activations/layer13_attention_weight_min": -37.552162170410156,
      "activations/layer14_attention_weight_max": 75.56942749023438,
      "activations/layer14_attention_weight_min": -56.67378616333008,
      "activations/layer15_attention_weight_max": 50.90115737915039,
      "activations/layer15_attention_weight_min": -36.87171936035156,
      "activations/layer16_attention_weight_max": 47.38233184814453,
      "activations/layer16_attention_weight_min": -36.78002166748047,
      "activations/layer17_attention_weight_max": 41.217830657958984,
      "activations/layer17_attention_weight_min": -29.31833267211914,
      "activations/layer18_attention_weight_max": 24.626968383789062,
      "activations/layer18_attention_weight_min": -15.584376335144043,
      "activations/layer19_attention_weight_max": 49.32986831665039,
      "activations/layer19_attention_weight_min": -32.18372344970703,
      "activations/layer1_attention_weight_max": 18.794343948364258,
      "activations/layer1_attention_weight_min": -11.604747772216797,
      "activations/layer20_attention_weight_max": 42.811702728271484,
      "activations/layer20_attention_weight_min": -25.181455612182617,
      "activations/layer21_attention_weight_max": 26.490476608276367,
      "activations/layer21_attention_weight_min": -16.86821746826172,
      "activations/layer22_attention_weight_max": 44.90632629394531,
      "activations/layer22_attention_weight_min": -26.097198486328125,
      "activations/layer23_attention_weight_max": 24.263586044311523,
      "activations/layer23_attention_weight_min": -14.912343978881836,
      "activations/layer2_attention_weight_max": 26.988567352294922,
      "activations/layer2_attention_weight_min": -25.25426483154297,
      "activations/layer3_attention_weight_max": 62.455299377441406,
      "activations/layer3_attention_weight_min": -65.8012924194336,
      "activations/layer4_attention_weight_max": 116.3558120727539,
      "activations/layer4_attention_weight_min": -115.09413146972656,
      "activations/layer5_attention_weight_max": 115.57103729248047,
      "activations/layer5_attention_weight_min": -101.08589172363281,
      "activations/layer6_attention_weight_max": 58.885955810546875,
      "activations/layer6_attention_weight_min": -52.80915069580078,
      "activations/layer7_attention_weight_max": 60.558441162109375,
      "activations/layer7_attention_weight_min": -59.835777282714844,
      "activations/layer8_attention_weight_max": 48.61201095581055,
      "activations/layer8_attention_weight_min": -45.8871955871582,
      "activations/layer9_attention_weight_max": 74.82891845703125,
      "activations/layer9_attention_weight_min": -57.19746780395508,
      "epoch": 1.51,
      "learning_rate": 0.00014166287878787876,
      "loss": 3.0211,
      "step": 26050
    },
    {
      "activations/layer0_attention_weight_max": 16.053953170776367,
      "activations/layer0_attention_weight_min": -10.647465705871582,
      "activations/layer10_attention_weight_max": 51.335201263427734,
      "activations/layer10_attention_weight_min": -49.6429328918457,
      "activations/layer11_attention_weight_max": 47.47468566894531,
      "activations/layer11_attention_weight_min": -49.53806686401367,
      "activations/layer12_attention_weight_max": 42.762813568115234,
      "activations/layer12_attention_weight_min": -31.479066848754883,
      "activations/layer13_attention_weight_max": 46.40806198120117,
      "activations/layer13_attention_weight_min": -36.451656341552734,
      "activations/layer14_attention_weight_max": 74.92549133300781,
      "activations/layer14_attention_weight_min": -58.07126998901367,
      "activations/layer15_attention_weight_max": 50.016151428222656,
      "activations/layer15_attention_weight_min": -39.8005256652832,
      "activations/layer16_attention_weight_max": 51.34149932861328,
      "activations/layer16_attention_weight_min": -39.681549072265625,
      "activations/layer17_attention_weight_max": 45.69416427612305,
      "activations/layer17_attention_weight_min": -29.561784744262695,
      "activations/layer18_attention_weight_max": 30.373708724975586,
      "activations/layer18_attention_weight_min": -16.483787536621094,
      "activations/layer19_attention_weight_max": 51.1801872253418,
      "activations/layer19_attention_weight_min": -31.80982780456543,
      "activations/layer1_attention_weight_max": 17.049392700195312,
      "activations/layer1_attention_weight_min": -11.714286804199219,
      "activations/layer20_attention_weight_max": 41.41149139404297,
      "activations/layer20_attention_weight_min": -25.18280029296875,
      "activations/layer21_attention_weight_max": 27.97307014465332,
      "activations/layer21_attention_weight_min": -18.11760902404785,
      "activations/layer22_attention_weight_max": 41.92790222167969,
      "activations/layer22_attention_weight_min": -27.91265106201172,
      "activations/layer23_attention_weight_max": 22.926372528076172,
      "activations/layer23_attention_weight_min": -16.21764373779297,
      "activations/layer2_attention_weight_max": 25.888294219970703,
      "activations/layer2_attention_weight_min": -24.610824584960938,
      "activations/layer3_attention_weight_max": 60.986366271972656,
      "activations/layer3_attention_weight_min": -66.70939636230469,
      "activations/layer4_attention_weight_max": 115.6440658569336,
      "activations/layer4_attention_weight_min": -110.0864486694336,
      "activations/layer5_attention_weight_max": 117.09963989257812,
      "activations/layer5_attention_weight_min": -104.35282897949219,
      "activations/layer6_attention_weight_max": 62.47920227050781,
      "activations/layer6_attention_weight_min": -55.45560073852539,
      "activations/layer7_attention_weight_max": 61.529781341552734,
      "activations/layer7_attention_weight_min": -61.53617858886719,
      "activations/layer8_attention_weight_max": 54.17350387573242,
      "activations/layer8_attention_weight_min": -51.29729461669922,
      "activations/layer9_attention_weight_max": 80.0085220336914,
      "activations/layer9_attention_weight_min": -72.06536102294922,
      "epoch": 1.52,
      "learning_rate": 0.00014164393939393937,
      "loss": 3.0211,
      "step": 26100
    },
    {
      "activations/layer0_attention_weight_max": 16.04432487487793,
      "activations/layer0_attention_weight_min": -10.41703987121582,
      "activations/layer10_attention_weight_max": 53.80634689331055,
      "activations/layer10_attention_weight_min": -42.792964935302734,
      "activations/layer11_attention_weight_max": 41.36149978637695,
      "activations/layer11_attention_weight_min": -36.5285530090332,
      "activations/layer12_attention_weight_max": 43.33089828491211,
      "activations/layer12_attention_weight_min": -32.881927490234375,
      "activations/layer13_attention_weight_max": 52.122581481933594,
      "activations/layer13_attention_weight_min": -34.80031967163086,
      "activations/layer14_attention_weight_max": 78.44136810302734,
      "activations/layer14_attention_weight_min": -56.652679443359375,
      "activations/layer15_attention_weight_max": 52.2679557800293,
      "activations/layer15_attention_weight_min": -36.247684478759766,
      "activations/layer16_attention_weight_max": 56.20509719848633,
      "activations/layer16_attention_weight_min": -38.63119888305664,
      "activations/layer17_attention_weight_max": 48.07602310180664,
      "activations/layer17_attention_weight_min": -29.209346771240234,
      "activations/layer18_attention_weight_max": 28.132083892822266,
      "activations/layer18_attention_weight_min": -16.650592803955078,
      "activations/layer19_attention_weight_max": 54.61859893798828,
      "activations/layer19_attention_weight_min": -31.998693466186523,
      "activations/layer1_attention_weight_max": 17.138622283935547,
      "activations/layer1_attention_weight_min": -11.540492057800293,
      "activations/layer20_attention_weight_max": 44.9890251159668,
      "activations/layer20_attention_weight_min": -25.060832977294922,
      "activations/layer21_attention_weight_max": 29.33336639404297,
      "activations/layer21_attention_weight_min": -17.5250301361084,
      "activations/layer22_attention_weight_max": 43.21156311035156,
      "activations/layer22_attention_weight_min": -25.21015739440918,
      "activations/layer23_attention_weight_max": 23.413009643554688,
      "activations/layer23_attention_weight_min": -17.20977020263672,
      "activations/layer2_attention_weight_max": 28.057655334472656,
      "activations/layer2_attention_weight_min": -26.951444625854492,
      "activations/layer3_attention_weight_max": 64.90226745605469,
      "activations/layer3_attention_weight_min": -67.71635437011719,
      "activations/layer4_attention_weight_max": 117.0967025756836,
      "activations/layer4_attention_weight_min": -115.42845916748047,
      "activations/layer5_attention_weight_max": 117.8963851928711,
      "activations/layer5_attention_weight_min": -104.1031723022461,
      "activations/layer6_attention_weight_max": 65.77218627929688,
      "activations/layer6_attention_weight_min": -56.221439361572266,
      "activations/layer7_attention_weight_max": 65.58837127685547,
      "activations/layer7_attention_weight_min": -59.603267669677734,
      "activations/layer8_attention_weight_max": 53.13419723510742,
      "activations/layer8_attention_weight_min": -46.57522964477539,
      "activations/layer9_attention_weight_max": 80.56391143798828,
      "activations/layer9_attention_weight_min": -58.951271057128906,
      "epoch": 1.52,
      "learning_rate": 0.000141625,
      "loss": 3.0102,
      "step": 26150
    },
    {
      "activations/layer0_attention_weight_max": 16.925857543945312,
      "activations/layer0_attention_weight_min": -10.718890190124512,
      "activations/layer10_attention_weight_max": 55.12901306152344,
      "activations/layer10_attention_weight_min": -47.559959411621094,
      "activations/layer11_attention_weight_max": 48.055747985839844,
      "activations/layer11_attention_weight_min": -39.878231048583984,
      "activations/layer12_attention_weight_max": 59.13750457763672,
      "activations/layer12_attention_weight_min": -40.322242736816406,
      "activations/layer13_attention_weight_max": 59.265480041503906,
      "activations/layer13_attention_weight_min": -42.09534454345703,
      "activations/layer14_attention_weight_max": 74.15309143066406,
      "activations/layer14_attention_weight_min": -55.456539154052734,
      "activations/layer15_attention_weight_max": 52.56248092651367,
      "activations/layer15_attention_weight_min": -37.44137191772461,
      "activations/layer16_attention_weight_max": 52.82343292236328,
      "activations/layer16_attention_weight_min": -38.4932861328125,
      "activations/layer17_attention_weight_max": 46.03733825683594,
      "activations/layer17_attention_weight_min": -30.580881118774414,
      "activations/layer18_attention_weight_max": 24.722612380981445,
      "activations/layer18_attention_weight_min": -15.824069023132324,
      "activations/layer19_attention_weight_max": 52.80628204345703,
      "activations/layer19_attention_weight_min": -32.31387710571289,
      "activations/layer1_attention_weight_max": 15.866156578063965,
      "activations/layer1_attention_weight_min": -10.818185806274414,
      "activations/layer20_attention_weight_max": 47.54706573486328,
      "activations/layer20_attention_weight_min": -29.049848556518555,
      "activations/layer21_attention_weight_max": 30.774166107177734,
      "activations/layer21_attention_weight_min": -18.578189849853516,
      "activations/layer22_attention_weight_max": 40.64164352416992,
      "activations/layer22_attention_weight_min": -28.112533569335938,
      "activations/layer23_attention_weight_max": 22.18820571899414,
      "activations/layer23_attention_weight_min": -16.649478912353516,
      "activations/layer2_attention_weight_max": 26.68976593017578,
      "activations/layer2_attention_weight_min": -26.074691772460938,
      "activations/layer3_attention_weight_max": 62.1962890625,
      "activations/layer3_attention_weight_min": -66.25836181640625,
      "activations/layer4_attention_weight_max": 113.84162902832031,
      "activations/layer4_attention_weight_min": -105.9548568725586,
      "activations/layer5_attention_weight_max": 110.78135681152344,
      "activations/layer5_attention_weight_min": -103.37428283691406,
      "activations/layer6_attention_weight_max": 59.518272399902344,
      "activations/layer6_attention_weight_min": -56.108314514160156,
      "activations/layer7_attention_weight_max": 64.457763671875,
      "activations/layer7_attention_weight_min": -64.18267059326172,
      "activations/layer8_attention_weight_max": 55.03316116333008,
      "activations/layer8_attention_weight_min": -43.73245620727539,
      "activations/layer9_attention_weight_max": 78.19564819335938,
      "activations/layer9_attention_weight_min": -63.439491271972656,
      "epoch": 1.52,
      "learning_rate": 0.0001416060606060606,
      "loss": 3.0126,
      "step": 26200
    },
    {
      "activations/layer0_attention_weight_max": 15.209044456481934,
      "activations/layer0_attention_weight_min": -10.781585693359375,
      "activations/layer10_attention_weight_max": 56.15236282348633,
      "activations/layer10_attention_weight_min": -49.490562438964844,
      "activations/layer11_attention_weight_max": 50.72101593017578,
      "activations/layer11_attention_weight_min": -41.74213790893555,
      "activations/layer12_attention_weight_max": 38.2409782409668,
      "activations/layer12_attention_weight_min": -26.598773956298828,
      "activations/layer13_attention_weight_max": 53.80297088623047,
      "activations/layer13_attention_weight_min": -37.94170379638672,
      "activations/layer14_attention_weight_max": 73.12589263916016,
      "activations/layer14_attention_weight_min": -54.67679977416992,
      "activations/layer15_attention_weight_max": 58.915504455566406,
      "activations/layer15_attention_weight_min": -38.099891662597656,
      "activations/layer16_attention_weight_max": 56.20488739013672,
      "activations/layer16_attention_weight_min": -35.495662689208984,
      "activations/layer17_attention_weight_max": 41.688533782958984,
      "activations/layer17_attention_weight_min": -24.65474510192871,
      "activations/layer18_attention_weight_max": 26.16072654724121,
      "activations/layer18_attention_weight_min": -16.205724716186523,
      "activations/layer19_attention_weight_max": 50.09334945678711,
      "activations/layer19_attention_weight_min": -31.107858657836914,
      "activations/layer1_attention_weight_max": 16.538089752197266,
      "activations/layer1_attention_weight_min": -11.126022338867188,
      "activations/layer20_attention_weight_max": 43.48352813720703,
      "activations/layer20_attention_weight_min": -25.192447662353516,
      "activations/layer21_attention_weight_max": 30.455913543701172,
      "activations/layer21_attention_weight_min": -18.946327209472656,
      "activations/layer22_attention_weight_max": 37.84449005126953,
      "activations/layer22_attention_weight_min": -21.363428115844727,
      "activations/layer23_attention_weight_max": 25.402875900268555,
      "activations/layer23_attention_weight_min": -14.077213287353516,
      "activations/layer2_attention_weight_max": 27.96398162841797,
      "activations/layer2_attention_weight_min": -27.316068649291992,
      "activations/layer3_attention_weight_max": 66.19293212890625,
      "activations/layer3_attention_weight_min": -66.25885772705078,
      "activations/layer4_attention_weight_max": 122.5655746459961,
      "activations/layer4_attention_weight_min": -108.5513687133789,
      "activations/layer5_attention_weight_max": 120.73845672607422,
      "activations/layer5_attention_weight_min": -103.799560546875,
      "activations/layer6_attention_weight_max": 63.82771682739258,
      "activations/layer6_attention_weight_min": -54.09251022338867,
      "activations/layer7_attention_weight_max": 71.86427307128906,
      "activations/layer7_attention_weight_min": -63.250244140625,
      "activations/layer8_attention_weight_max": 55.309593200683594,
      "activations/layer8_attention_weight_min": -48.01232147216797,
      "activations/layer9_attention_weight_max": 85.87924194335938,
      "activations/layer9_attention_weight_min": -69.5151596069336,
      "epoch": 1.53,
      "learning_rate": 0.0001415871212121212,
      "loss": 3.0113,
      "step": 26250
    },
    {
      "activations/layer0_attention_weight_max": 16.88976287841797,
      "activations/layer0_attention_weight_min": -10.562296867370605,
      "activations/layer10_attention_weight_max": 65.53968811035156,
      "activations/layer10_attention_weight_min": -49.22751235961914,
      "activations/layer11_attention_weight_max": 56.39506149291992,
      "activations/layer11_attention_weight_min": -39.835243225097656,
      "activations/layer12_attention_weight_max": 88.26139068603516,
      "activations/layer12_attention_weight_min": -47.35443115234375,
      "activations/layer13_attention_weight_max": 96.97052764892578,
      "activations/layer13_attention_weight_min": -55.98023223876953,
      "activations/layer14_attention_weight_max": 84.56150817871094,
      "activations/layer14_attention_weight_min": -62.66250228881836,
      "activations/layer15_attention_weight_max": 56.359825134277344,
      "activations/layer15_attention_weight_min": -40.210147857666016,
      "activations/layer16_attention_weight_max": 55.38349533081055,
      "activations/layer16_attention_weight_min": -37.934661865234375,
      "activations/layer17_attention_weight_max": 45.429725646972656,
      "activations/layer17_attention_weight_min": -28.948604583740234,
      "activations/layer18_attention_weight_max": 26.342422485351562,
      "activations/layer18_attention_weight_min": -15.906527519226074,
      "activations/layer19_attention_weight_max": 52.13637924194336,
      "activations/layer19_attention_weight_min": -31.68195152282715,
      "activations/layer1_attention_weight_max": 17.699810028076172,
      "activations/layer1_attention_weight_min": -10.555429458618164,
      "activations/layer20_attention_weight_max": 42.62626647949219,
      "activations/layer20_attention_weight_min": -23.635391235351562,
      "activations/layer21_attention_weight_max": 27.928842544555664,
      "activations/layer21_attention_weight_min": -14.43608283996582,
      "activations/layer22_attention_weight_max": 40.869930267333984,
      "activations/layer22_attention_weight_min": -23.27487564086914,
      "activations/layer23_attention_weight_max": 24.771530151367188,
      "activations/layer23_attention_weight_min": -14.610453605651855,
      "activations/layer2_attention_weight_max": 26.52515983581543,
      "activations/layer2_attention_weight_min": -26.407604217529297,
      "activations/layer3_attention_weight_max": 66.83150482177734,
      "activations/layer3_attention_weight_min": -66.69944763183594,
      "activations/layer4_attention_weight_max": 122.3617935180664,
      "activations/layer4_attention_weight_min": -108.68731689453125,
      "activations/layer5_attention_weight_max": 124.24075317382812,
      "activations/layer5_attention_weight_min": -104.82298278808594,
      "activations/layer6_attention_weight_max": 68.3597640991211,
      "activations/layer6_attention_weight_min": -54.587928771972656,
      "activations/layer7_attention_weight_max": 70.82836151123047,
      "activations/layer7_attention_weight_min": -63.36081314086914,
      "activations/layer8_attention_weight_max": 67.70545196533203,
      "activations/layer8_attention_weight_min": -54.45903778076172,
      "activations/layer9_attention_weight_max": 85.59351348876953,
      "activations/layer9_attention_weight_min": -69.2714614868164,
      "epoch": 1.53,
      "learning_rate": 0.0001415681818181818,
      "loss": 3.0191,
      "step": 26300
    },
    {
      "activations/layer0_attention_weight_max": 16.825326919555664,
      "activations/layer0_attention_weight_min": -10.605908393859863,
      "activations/layer10_attention_weight_max": 52.14947509765625,
      "activations/layer10_attention_weight_min": -43.966941833496094,
      "activations/layer11_attention_weight_max": 42.017662048339844,
      "activations/layer11_attention_weight_min": -37.386444091796875,
      "activations/layer12_attention_weight_max": 42.4778938293457,
      "activations/layer12_attention_weight_min": -29.97576141357422,
      "activations/layer13_attention_weight_max": 53.60813903808594,
      "activations/layer13_attention_weight_min": -38.414363861083984,
      "activations/layer14_attention_weight_max": 79.9657974243164,
      "activations/layer14_attention_weight_min": -60.61236572265625,
      "activations/layer15_attention_weight_max": 60.637168884277344,
      "activations/layer15_attention_weight_min": -44.31167984008789,
      "activations/layer16_attention_weight_max": 56.01854705810547,
      "activations/layer16_attention_weight_min": -37.639957427978516,
      "activations/layer17_attention_weight_max": 51.6811408996582,
      "activations/layer17_attention_weight_min": -31.919809341430664,
      "activations/layer18_attention_weight_max": 31.009521484375,
      "activations/layer18_attention_weight_min": -17.046472549438477,
      "activations/layer19_attention_weight_max": 66.85186767578125,
      "activations/layer19_attention_weight_min": -35.39356994628906,
      "activations/layer1_attention_weight_max": 17.325271606445312,
      "activations/layer1_attention_weight_min": -11.735580444335938,
      "activations/layer20_attention_weight_max": 49.649227142333984,
      "activations/layer20_attention_weight_min": -26.822324752807617,
      "activations/layer21_attention_weight_max": 27.08171272277832,
      "activations/layer21_attention_weight_min": -18.9739933013916,
      "activations/layer22_attention_weight_max": 48.7463264465332,
      "activations/layer22_attention_weight_min": -24.114566802978516,
      "activations/layer23_attention_weight_max": 27.73084831237793,
      "activations/layer23_attention_weight_min": -14.379013061523438,
      "activations/layer2_attention_weight_max": 27.21693992614746,
      "activations/layer2_attention_weight_min": -26.58554458618164,
      "activations/layer3_attention_weight_max": 63.508453369140625,
      "activations/layer3_attention_weight_min": -68.0787124633789,
      "activations/layer4_attention_weight_max": 117.30570220947266,
      "activations/layer4_attention_weight_min": -114.72240447998047,
      "activations/layer5_attention_weight_max": 118.32608032226562,
      "activations/layer5_attention_weight_min": -106.87271118164062,
      "activations/layer6_attention_weight_max": 64.19856262207031,
      "activations/layer6_attention_weight_min": -56.56950378417969,
      "activations/layer7_attention_weight_max": 64.4137954711914,
      "activations/layer7_attention_weight_min": -63.777164459228516,
      "activations/layer8_attention_weight_max": 48.900489807128906,
      "activations/layer8_attention_weight_min": -44.46085739135742,
      "activations/layer9_attention_weight_max": 70.79267120361328,
      "activations/layer9_attention_weight_min": -57.15339279174805,
      "epoch": 1.53,
      "learning_rate": 0.00014154924242424242,
      "loss": 3.013,
      "step": 26350
    },
    {
      "activations/layer0_attention_weight_max": 16.946985244750977,
      "activations/layer0_attention_weight_min": -10.962810516357422,
      "activations/layer10_attention_weight_max": 53.95198440551758,
      "activations/layer10_attention_weight_min": -42.55668258666992,
      "activations/layer11_attention_weight_max": 40.29527282714844,
      "activations/layer11_attention_weight_min": -35.07709503173828,
      "activations/layer12_attention_weight_max": 40.39876174926758,
      "activations/layer12_attention_weight_min": -29.592899322509766,
      "activations/layer13_attention_weight_max": 47.51676559448242,
      "activations/layer13_attention_weight_min": -36.68246078491211,
      "activations/layer14_attention_weight_max": 67.68802642822266,
      "activations/layer14_attention_weight_min": -52.850685119628906,
      "activations/layer15_attention_weight_max": 48.83127212524414,
      "activations/layer15_attention_weight_min": -36.780311584472656,
      "activations/layer16_attention_weight_max": 57.80598449707031,
      "activations/layer16_attention_weight_min": -39.31996536254883,
      "activations/layer17_attention_weight_max": 41.524478912353516,
      "activations/layer17_attention_weight_min": -27.752107620239258,
      "activations/layer18_attention_weight_max": 23.483129501342773,
      "activations/layer18_attention_weight_min": -15.995736122131348,
      "activations/layer19_attention_weight_max": 48.78325271606445,
      "activations/layer19_attention_weight_min": -31.547082901000977,
      "activations/layer1_attention_weight_max": 19.459293365478516,
      "activations/layer1_attention_weight_min": -12.242342948913574,
      "activations/layer20_attention_weight_max": 40.53290939331055,
      "activations/layer20_attention_weight_min": -26.20083236694336,
      "activations/layer21_attention_weight_max": 28.06071662902832,
      "activations/layer21_attention_weight_min": -19.743871688842773,
      "activations/layer22_attention_weight_max": 42.916160583496094,
      "activations/layer22_attention_weight_min": -24.32438850402832,
      "activations/layer23_attention_weight_max": 25.519508361816406,
      "activations/layer23_attention_weight_min": -18.611488342285156,
      "activations/layer2_attention_weight_max": 28.64363670349121,
      "activations/layer2_attention_weight_min": -27.24574851989746,
      "activations/layer3_attention_weight_max": 66.0807876586914,
      "activations/layer3_attention_weight_min": -69.05680084228516,
      "activations/layer4_attention_weight_max": 118.25963592529297,
      "activations/layer4_attention_weight_min": -112.99077606201172,
      "activations/layer5_attention_weight_max": 114.60240173339844,
      "activations/layer5_attention_weight_min": -102.46287536621094,
      "activations/layer6_attention_weight_max": 63.56953430175781,
      "activations/layer6_attention_weight_min": -52.723697662353516,
      "activations/layer7_attention_weight_max": 60.71339416503906,
      "activations/layer7_attention_weight_min": -60.46308898925781,
      "activations/layer8_attention_weight_max": 52.35146713256836,
      "activations/layer8_attention_weight_min": -44.85601043701172,
      "activations/layer9_attention_weight_max": 68.15135955810547,
      "activations/layer9_attention_weight_min": -60.780635833740234,
      "epoch": 1.53,
      "learning_rate": 0.00014153030303030303,
      "loss": 3.0199,
      "step": 26400
    },
    {
      "activations/layer0_attention_weight_max": 17.25459861755371,
      "activations/layer0_attention_weight_min": -11.146453857421875,
      "activations/layer10_attention_weight_max": 58.590049743652344,
      "activations/layer10_attention_weight_min": -44.73337173461914,
      "activations/layer11_attention_weight_max": 53.838436126708984,
      "activations/layer11_attention_weight_min": -37.5354118347168,
      "activations/layer12_attention_weight_max": 70.0279312133789,
      "activations/layer12_attention_weight_min": -28.142065048217773,
      "activations/layer13_attention_weight_max": 70.13934326171875,
      "activations/layer13_attention_weight_min": -39.421051025390625,
      "activations/layer14_attention_weight_max": 100.9905776977539,
      "activations/layer14_attention_weight_min": -60.093544006347656,
      "activations/layer15_attention_weight_max": 78.62168884277344,
      "activations/layer15_attention_weight_min": -40.52555465698242,
      "activations/layer16_attention_weight_max": 63.11164474487305,
      "activations/layer16_attention_weight_min": -36.76433181762695,
      "activations/layer17_attention_weight_max": 54.88351058959961,
      "activations/layer17_attention_weight_min": -28.64690589904785,
      "activations/layer18_attention_weight_max": 26.57447624206543,
      "activations/layer18_attention_weight_min": -15.426374435424805,
      "activations/layer19_attention_weight_max": 58.41697311401367,
      "activations/layer19_attention_weight_min": -32.1834831237793,
      "activations/layer1_attention_weight_max": 16.865320205688477,
      "activations/layer1_attention_weight_min": -12.274115562438965,
      "activations/layer20_attention_weight_max": 39.808467864990234,
      "activations/layer20_attention_weight_min": -27.44521141052246,
      "activations/layer21_attention_weight_max": 36.05902099609375,
      "activations/layer21_attention_weight_min": -18.02652931213379,
      "activations/layer22_attention_weight_max": 45.29131317138672,
      "activations/layer22_attention_weight_min": -24.458520889282227,
      "activations/layer23_attention_weight_max": 24.44011688232422,
      "activations/layer23_attention_weight_min": -15.683374404907227,
      "activations/layer2_attention_weight_max": 28.625900268554688,
      "activations/layer2_attention_weight_min": -27.994312286376953,
      "activations/layer3_attention_weight_max": 66.70329284667969,
      "activations/layer3_attention_weight_min": -67.99016571044922,
      "activations/layer4_attention_weight_max": 137.36851501464844,
      "activations/layer4_attention_weight_min": -112.5506362915039,
      "activations/layer5_attention_weight_max": 147.10145568847656,
      "activations/layer5_attention_weight_min": -107.04379272460938,
      "activations/layer6_attention_weight_max": 78.89481353759766,
      "activations/layer6_attention_weight_min": -56.75511932373047,
      "activations/layer7_attention_weight_max": 71.84315490722656,
      "activations/layer7_attention_weight_min": -59.955196380615234,
      "activations/layer8_attention_weight_max": 61.14927291870117,
      "activations/layer8_attention_weight_min": -46.03550338745117,
      "activations/layer9_attention_weight_max": 87.21793365478516,
      "activations/layer9_attention_weight_min": -60.787879943847656,
      "epoch": 1.54,
      "learning_rate": 0.00014151136363636362,
      "loss": 3.0135,
      "step": 26450
    },
    {
      "activations/layer0_attention_weight_max": 17.25916290283203,
      "activations/layer0_attention_weight_min": -11.403888702392578,
      "activations/layer10_attention_weight_max": 58.16110610961914,
      "activations/layer10_attention_weight_min": -46.96360778808594,
      "activations/layer11_attention_weight_max": 55.50794982910156,
      "activations/layer11_attention_weight_min": -41.76955795288086,
      "activations/layer12_attention_weight_max": 45.07096862792969,
      "activations/layer12_attention_weight_min": -33.17329406738281,
      "activations/layer13_attention_weight_max": 52.72419738769531,
      "activations/layer13_attention_weight_min": -41.16963577270508,
      "activations/layer14_attention_weight_max": 70.51664733886719,
      "activations/layer14_attention_weight_min": -55.95052719116211,
      "activations/layer15_attention_weight_max": 51.15034484863281,
      "activations/layer15_attention_weight_min": -40.94438171386719,
      "activations/layer16_attention_weight_max": 54.658878326416016,
      "activations/layer16_attention_weight_min": -38.82566452026367,
      "activations/layer17_attention_weight_max": 45.605716705322266,
      "activations/layer17_attention_weight_min": -28.914316177368164,
      "activations/layer18_attention_weight_max": 27.695655822753906,
      "activations/layer18_attention_weight_min": -18.34627914428711,
      "activations/layer19_attention_weight_max": 52.38666915893555,
      "activations/layer19_attention_weight_min": -32.428226470947266,
      "activations/layer1_attention_weight_max": 17.18345069885254,
      "activations/layer1_attention_weight_min": -11.891265869140625,
      "activations/layer20_attention_weight_max": 45.83633804321289,
      "activations/layer20_attention_weight_min": -25.244792938232422,
      "activations/layer21_attention_weight_max": 29.27219009399414,
      "activations/layer21_attention_weight_min": -16.99195098876953,
      "activations/layer22_attention_weight_max": 43.64552307128906,
      "activations/layer22_attention_weight_min": -28.464590072631836,
      "activations/layer23_attention_weight_max": 28.264862060546875,
      "activations/layer23_attention_weight_min": -16.133079528808594,
      "activations/layer2_attention_weight_max": 29.892728805541992,
      "activations/layer2_attention_weight_min": -29.097946166992188,
      "activations/layer3_attention_weight_max": 70.82585906982422,
      "activations/layer3_attention_weight_min": -72.27922821044922,
      "activations/layer4_attention_weight_max": 127.59691619873047,
      "activations/layer4_attention_weight_min": -121.3913803100586,
      "activations/layer5_attention_weight_max": 129.06356811523438,
      "activations/layer5_attention_weight_min": -116.79497528076172,
      "activations/layer6_attention_weight_max": 66.06069946289062,
      "activations/layer6_attention_weight_min": -59.77758026123047,
      "activations/layer7_attention_weight_max": 66.22984313964844,
      "activations/layer7_attention_weight_min": -62.86476516723633,
      "activations/layer8_attention_weight_max": 64.69124603271484,
      "activations/layer8_attention_weight_min": -50.58522033691406,
      "activations/layer9_attention_weight_max": 89.9482650756836,
      "activations/layer9_attention_weight_min": -74.17418670654297,
      "epoch": 1.54,
      "learning_rate": 0.00014149242424242424,
      "loss": 3.0095,
      "step": 26500
    },
    {
      "activations/layer0_attention_weight_max": 13.425670623779297,
      "activations/layer0_attention_weight_min": -11.24057388305664,
      "activations/layer10_attention_weight_max": 60.07184982299805,
      "activations/layer10_attention_weight_min": -47.432830810546875,
      "activations/layer11_attention_weight_max": 48.16011047363281,
      "activations/layer11_attention_weight_min": -36.217281341552734,
      "activations/layer12_attention_weight_max": 42.44860076904297,
      "activations/layer12_attention_weight_min": -28.781497955322266,
      "activations/layer13_attention_weight_max": 56.50086975097656,
      "activations/layer13_attention_weight_min": -37.166961669921875,
      "activations/layer14_attention_weight_max": 90.96156311035156,
      "activations/layer14_attention_weight_min": -67.238525390625,
      "activations/layer15_attention_weight_max": 60.2842903137207,
      "activations/layer15_attention_weight_min": -41.336952209472656,
      "activations/layer16_attention_weight_max": 70.11821746826172,
      "activations/layer16_attention_weight_min": -42.1058235168457,
      "activations/layer17_attention_weight_max": 46.125160217285156,
      "activations/layer17_attention_weight_min": -30.17524528503418,
      "activations/layer18_attention_weight_max": 31.92750358581543,
      "activations/layer18_attention_weight_min": -17.365615844726562,
      "activations/layer19_attention_weight_max": 72.75106811523438,
      "activations/layer19_attention_weight_min": -32.4219970703125,
      "activations/layer1_attention_weight_max": 17.006698608398438,
      "activations/layer1_attention_weight_min": -11.1619234085083,
      "activations/layer20_attention_weight_max": 56.856510162353516,
      "activations/layer20_attention_weight_min": -26.080827713012695,
      "activations/layer21_attention_weight_max": 29.09357261657715,
      "activations/layer21_attention_weight_min": -18.0264835357666,
      "activations/layer22_attention_weight_max": 50.14654541015625,
      "activations/layer22_attention_weight_min": -24.88531494140625,
      "activations/layer23_attention_weight_max": 26.974714279174805,
      "activations/layer23_attention_weight_min": -14.519610404968262,
      "activations/layer2_attention_weight_max": 26.577617645263672,
      "activations/layer2_attention_weight_min": -28.129226684570312,
      "activations/layer3_attention_weight_max": 66.12091827392578,
      "activations/layer3_attention_weight_min": -67.35713958740234,
      "activations/layer4_attention_weight_max": 119.1531753540039,
      "activations/layer4_attention_weight_min": -113.5029296875,
      "activations/layer5_attention_weight_max": 119.48503875732422,
      "activations/layer5_attention_weight_min": -109.49639129638672,
      "activations/layer6_attention_weight_max": 65.42747497558594,
      "activations/layer6_attention_weight_min": -56.740455627441406,
      "activations/layer7_attention_weight_max": 64.40616607666016,
      "activations/layer7_attention_weight_min": -60.71636199951172,
      "activations/layer8_attention_weight_max": 56.46139907836914,
      "activations/layer8_attention_weight_min": -45.69501495361328,
      "activations/layer9_attention_weight_max": 66.52782440185547,
      "activations/layer9_attention_weight_min": -59.18644332885742,
      "epoch": 1.54,
      "learning_rate": 0.00014147348484848485,
      "loss": 3.0166,
      "step": 26550
    },
    {
      "activations/layer0_attention_weight_max": 16.605560302734375,
      "activations/layer0_attention_weight_min": -11.146134376525879,
      "activations/layer10_attention_weight_max": 57.26181411743164,
      "activations/layer10_attention_weight_min": -47.900203704833984,
      "activations/layer11_attention_weight_max": 47.2127685546875,
      "activations/layer11_attention_weight_min": -36.5987663269043,
      "activations/layer12_attention_weight_max": 45.89633560180664,
      "activations/layer12_attention_weight_min": -34.7829475402832,
      "activations/layer13_attention_weight_max": 48.44263458251953,
      "activations/layer13_attention_weight_min": -36.554351806640625,
      "activations/layer14_attention_weight_max": 85.16477966308594,
      "activations/layer14_attention_weight_min": -65.72575378417969,
      "activations/layer15_attention_weight_max": 59.30002975463867,
      "activations/layer15_attention_weight_min": -41.08903503417969,
      "activations/layer16_attention_weight_max": 59.85270690917969,
      "activations/layer16_attention_weight_min": -40.20214080810547,
      "activations/layer17_attention_weight_max": 52.191349029541016,
      "activations/layer17_attention_weight_min": -31.564767837524414,
      "activations/layer18_attention_weight_max": 27.071992874145508,
      "activations/layer18_attention_weight_min": -17.146705627441406,
      "activations/layer19_attention_weight_max": 55.362464904785156,
      "activations/layer19_attention_weight_min": -39.02094650268555,
      "activations/layer1_attention_weight_max": 18.65015983581543,
      "activations/layer1_attention_weight_min": -10.737836837768555,
      "activations/layer20_attention_weight_max": 51.17868423461914,
      "activations/layer20_attention_weight_min": -25.5755558013916,
      "activations/layer21_attention_weight_max": 29.616947174072266,
      "activations/layer21_attention_weight_min": -19.73693084716797,
      "activations/layer22_attention_weight_max": 43.88655471801758,
      "activations/layer22_attention_weight_min": -24.381799697875977,
      "activations/layer23_attention_weight_max": 25.596118927001953,
      "activations/layer23_attention_weight_min": -14.655847549438477,
      "activations/layer2_attention_weight_max": 28.416881561279297,
      "activations/layer2_attention_weight_min": -28.738773345947266,
      "activations/layer3_attention_weight_max": 69.2518310546875,
      "activations/layer3_attention_weight_min": -73.07398223876953,
      "activations/layer4_attention_weight_max": 122.13237762451172,
      "activations/layer4_attention_weight_min": -114.46435546875,
      "activations/layer5_attention_weight_max": 117.99488830566406,
      "activations/layer5_attention_weight_min": -105.11630249023438,
      "activations/layer6_attention_weight_max": 64.31153106689453,
      "activations/layer6_attention_weight_min": -55.329124450683594,
      "activations/layer7_attention_weight_max": 64.43467712402344,
      "activations/layer7_attention_weight_min": -62.2637825012207,
      "activations/layer8_attention_weight_max": 52.44319534301758,
      "activations/layer8_attention_weight_min": -49.90413284301758,
      "activations/layer9_attention_weight_max": 79.96297454833984,
      "activations/layer9_attention_weight_min": -64.63219451904297,
      "epoch": 1.55,
      "learning_rate": 0.00014145454545454544,
      "loss": 3.0116,
      "step": 26600
    },
    {
      "activations/layer0_attention_weight_max": 16.287864685058594,
      "activations/layer0_attention_weight_min": -10.878669738769531,
      "activations/layer10_attention_weight_max": 57.638729095458984,
      "activations/layer10_attention_weight_min": -49.29203796386719,
      "activations/layer11_attention_weight_max": 42.05758285522461,
      "activations/layer11_attention_weight_min": -37.83771514892578,
      "activations/layer12_attention_weight_max": 38.678565979003906,
      "activations/layer12_attention_weight_min": -29.46079444885254,
      "activations/layer13_attention_weight_max": 53.17943572998047,
      "activations/layer13_attention_weight_min": -42.241859436035156,
      "activations/layer14_attention_weight_max": 84.36970520019531,
      "activations/layer14_attention_weight_min": -67.2980728149414,
      "activations/layer15_attention_weight_max": 57.02546310424805,
      "activations/layer15_attention_weight_min": -40.85257339477539,
      "activations/layer16_attention_weight_max": 56.07025146484375,
      "activations/layer16_attention_weight_min": -42.198062896728516,
      "activations/layer17_attention_weight_max": 43.30622863769531,
      "activations/layer17_attention_weight_min": -27.44341468811035,
      "activations/layer18_attention_weight_max": 28.397159576416016,
      "activations/layer18_attention_weight_min": -17.475257873535156,
      "activations/layer19_attention_weight_max": 58.60425567626953,
      "activations/layer19_attention_weight_min": -32.984619140625,
      "activations/layer1_attention_weight_max": 16.941911697387695,
      "activations/layer1_attention_weight_min": -11.061466217041016,
      "activations/layer20_attention_weight_max": 43.03300476074219,
      "activations/layer20_attention_weight_min": -27.82965087890625,
      "activations/layer21_attention_weight_max": 26.779556274414062,
      "activations/layer21_attention_weight_min": -18.369028091430664,
      "activations/layer22_attention_weight_max": 47.437904357910156,
      "activations/layer22_attention_weight_min": -26.68027687072754,
      "activations/layer23_attention_weight_max": 26.49371337890625,
      "activations/layer23_attention_weight_min": -17.064943313598633,
      "activations/layer2_attention_weight_max": 27.13412094116211,
      "activations/layer2_attention_weight_min": -26.653114318847656,
      "activations/layer3_attention_weight_max": 65.95246887207031,
      "activations/layer3_attention_weight_min": -67.104248046875,
      "activations/layer4_attention_weight_max": 121.14814758300781,
      "activations/layer4_attention_weight_min": -114.97847747802734,
      "activations/layer5_attention_weight_max": 113.76582336425781,
      "activations/layer5_attention_weight_min": -105.0770263671875,
      "activations/layer6_attention_weight_max": 61.815826416015625,
      "activations/layer6_attention_weight_min": -53.739830017089844,
      "activations/layer7_attention_weight_max": 62.40338897705078,
      "activations/layer7_attention_weight_min": -62.704071044921875,
      "activations/layer8_attention_weight_max": 50.61914825439453,
      "activations/layer8_attention_weight_min": -45.5762939453125,
      "activations/layer9_attention_weight_max": 80.85647583007812,
      "activations/layer9_attention_weight_min": -63.92218017578125,
      "epoch": 1.55,
      "learning_rate": 0.00014143560606060606,
      "loss": 3.0282,
      "step": 26650
    },
    {
      "activations/layer0_attention_weight_max": 14.840622901916504,
      "activations/layer0_attention_weight_min": -11.02776050567627,
      "activations/layer10_attention_weight_max": 62.62748718261719,
      "activations/layer10_attention_weight_min": -47.29201126098633,
      "activations/layer11_attention_weight_max": 52.857906341552734,
      "activations/layer11_attention_weight_min": -37.925331115722656,
      "activations/layer12_attention_weight_max": 67.1662368774414,
      "activations/layer12_attention_weight_min": -40.284156799316406,
      "activations/layer13_attention_weight_max": 78.35001373291016,
      "activations/layer13_attention_weight_min": -50.37934112548828,
      "activations/layer14_attention_weight_max": 81.8757095336914,
      "activations/layer14_attention_weight_min": -65.1677017211914,
      "activations/layer15_attention_weight_max": 53.56378936767578,
      "activations/layer15_attention_weight_min": -39.05351638793945,
      "activations/layer16_attention_weight_max": 56.16789245605469,
      "activations/layer16_attention_weight_min": -42.69203567504883,
      "activations/layer17_attention_weight_max": 40.04277420043945,
      "activations/layer17_attention_weight_min": -31.3121337890625,
      "activations/layer18_attention_weight_max": 25.693538665771484,
      "activations/layer18_attention_weight_min": -15.994241714477539,
      "activations/layer19_attention_weight_max": 57.73499298095703,
      "activations/layer19_attention_weight_min": -35.20439910888672,
      "activations/layer1_attention_weight_max": 16.289003372192383,
      "activations/layer1_attention_weight_min": -10.719365119934082,
      "activations/layer20_attention_weight_max": 42.350154876708984,
      "activations/layer20_attention_weight_min": -24.208024978637695,
      "activations/layer21_attention_weight_max": 27.840110778808594,
      "activations/layer21_attention_weight_min": -16.92446517944336,
      "activations/layer22_attention_weight_max": 43.13203811645508,
      "activations/layer22_attention_weight_min": -25.539928436279297,
      "activations/layer23_attention_weight_max": 24.470853805541992,
      "activations/layer23_attention_weight_min": -15.748857498168945,
      "activations/layer2_attention_weight_max": 29.633398056030273,
      "activations/layer2_attention_weight_min": -26.55359649658203,
      "activations/layer3_attention_weight_max": 66.7087631225586,
      "activations/layer3_attention_weight_min": -63.83058547973633,
      "activations/layer4_attention_weight_max": 123.0374984741211,
      "activations/layer4_attention_weight_min": -104.9051742553711,
      "activations/layer5_attention_weight_max": 118.67254638671875,
      "activations/layer5_attention_weight_min": -97.49514770507812,
      "activations/layer6_attention_weight_max": 65.54594421386719,
      "activations/layer6_attention_weight_min": -50.94145584106445,
      "activations/layer7_attention_weight_max": 67.28398132324219,
      "activations/layer7_attention_weight_min": -61.58740234375,
      "activations/layer8_attention_weight_max": 57.13923645019531,
      "activations/layer8_attention_weight_min": -46.03615188598633,
      "activations/layer9_attention_weight_max": 89.17574310302734,
      "activations/layer9_attention_weight_min": -65.23761749267578,
      "epoch": 1.55,
      "learning_rate": 0.00014141666666666667,
      "loss": 3.0221,
      "step": 26700
    },
    {
      "activations/layer0_attention_weight_max": 16.69818115234375,
      "activations/layer0_attention_weight_min": -11.277302742004395,
      "activations/layer10_attention_weight_max": 54.99032974243164,
      "activations/layer10_attention_weight_min": -47.29555130004883,
      "activations/layer11_attention_weight_max": 44.60762023925781,
      "activations/layer11_attention_weight_min": -36.87400817871094,
      "activations/layer12_attention_weight_max": 53.496952056884766,
      "activations/layer12_attention_weight_min": -36.54851531982422,
      "activations/layer13_attention_weight_max": 63.468711853027344,
      "activations/layer13_attention_weight_min": -53.383541107177734,
      "activations/layer14_attention_weight_max": 75.09259033203125,
      "activations/layer14_attention_weight_min": -59.34537887573242,
      "activations/layer15_attention_weight_max": 61.39793014526367,
      "activations/layer15_attention_weight_min": -43.16912078857422,
      "activations/layer16_attention_weight_max": 59.31521987915039,
      "activations/layer16_attention_weight_min": -42.43880081176758,
      "activations/layer17_attention_weight_max": 45.88657760620117,
      "activations/layer17_attention_weight_min": -30.124393463134766,
      "activations/layer18_attention_weight_max": 28.065134048461914,
      "activations/layer18_attention_weight_min": -18.333200454711914,
      "activations/layer19_attention_weight_max": 55.630340576171875,
      "activations/layer19_attention_weight_min": -35.463375091552734,
      "activations/layer1_attention_weight_max": 17.492277145385742,
      "activations/layer1_attention_weight_min": -10.954686164855957,
      "activations/layer20_attention_weight_max": 40.35576248168945,
      "activations/layer20_attention_weight_min": -25.55548095703125,
      "activations/layer21_attention_weight_max": 28.291685104370117,
      "activations/layer21_attention_weight_min": -17.559906005859375,
      "activations/layer22_attention_weight_max": 42.55131912231445,
      "activations/layer22_attention_weight_min": -23.49680519104004,
      "activations/layer23_attention_weight_max": 25.76315689086914,
      "activations/layer23_attention_weight_min": -15.441431999206543,
      "activations/layer2_attention_weight_max": 27.618511199951172,
      "activations/layer2_attention_weight_min": -28.35449981689453,
      "activations/layer3_attention_weight_max": 64.87277221679688,
      "activations/layer3_attention_weight_min": -70.39248657226562,
      "activations/layer4_attention_weight_max": 119.4221420288086,
      "activations/layer4_attention_weight_min": -122.11663055419922,
      "activations/layer5_attention_weight_max": 112.55748748779297,
      "activations/layer5_attention_weight_min": -104.14320373535156,
      "activations/layer6_attention_weight_max": 59.78459167480469,
      "activations/layer6_attention_weight_min": -57.54590606689453,
      "activations/layer7_attention_weight_max": 62.2021598815918,
      "activations/layer7_attention_weight_min": -64.44361114501953,
      "activations/layer8_attention_weight_max": 48.949302673339844,
      "activations/layer8_attention_weight_min": -44.26584243774414,
      "activations/layer9_attention_weight_max": 64.9728775024414,
      "activations/layer9_attention_weight_min": -61.03541946411133,
      "epoch": 1.55,
      "learning_rate": 0.00014139772727272726,
      "loss": 3.0075,
      "step": 26750
    },
    {
      "activations/layer0_attention_weight_max": 16.84855079650879,
      "activations/layer0_attention_weight_min": -11.381613731384277,
      "activations/layer10_attention_weight_max": 53.297969818115234,
      "activations/layer10_attention_weight_min": -46.98773956298828,
      "activations/layer11_attention_weight_max": 52.338924407958984,
      "activations/layer11_attention_weight_min": -41.42308807373047,
      "activations/layer12_attention_weight_max": 31.330368041992188,
      "activations/layer12_attention_weight_min": -28.279359817504883,
      "activations/layer13_attention_weight_max": 48.3171272277832,
      "activations/layer13_attention_weight_min": -37.939781188964844,
      "activations/layer14_attention_weight_max": 77.07414245605469,
      "activations/layer14_attention_weight_min": -55.28956604003906,
      "activations/layer15_attention_weight_max": 53.81955337524414,
      "activations/layer15_attention_weight_min": -38.08216094970703,
      "activations/layer16_attention_weight_max": 60.759674072265625,
      "activations/layer16_attention_weight_min": -41.697574615478516,
      "activations/layer17_attention_weight_max": 42.83560562133789,
      "activations/layer17_attention_weight_min": -27.032779693603516,
      "activations/layer18_attention_weight_max": 23.85037612915039,
      "activations/layer18_attention_weight_min": -16.18760871887207,
      "activations/layer19_attention_weight_max": 54.115264892578125,
      "activations/layer19_attention_weight_min": -34.93994903564453,
      "activations/layer1_attention_weight_max": 16.84378433227539,
      "activations/layer1_attention_weight_min": -14.02295207977295,
      "activations/layer20_attention_weight_max": 40.51560592651367,
      "activations/layer20_attention_weight_min": -24.81169319152832,
      "activations/layer21_attention_weight_max": 28.32913589477539,
      "activations/layer21_attention_weight_min": -15.26917552947998,
      "activations/layer22_attention_weight_max": 39.40544128417969,
      "activations/layer22_attention_weight_min": -24.04714012145996,
      "activations/layer23_attention_weight_max": 24.395652770996094,
      "activations/layer23_attention_weight_min": -16.706523895263672,
      "activations/layer2_attention_weight_max": 27.657318115234375,
      "activations/layer2_attention_weight_min": -27.2631893157959,
      "activations/layer3_attention_weight_max": 66.71932983398438,
      "activations/layer3_attention_weight_min": -68.95990753173828,
      "activations/layer4_attention_weight_max": 116.02508544921875,
      "activations/layer4_attention_weight_min": -111.9752426147461,
      "activations/layer5_attention_weight_max": 110.36380004882812,
      "activations/layer5_attention_weight_min": -100.70125579833984,
      "activations/layer6_attention_weight_max": 61.25614929199219,
      "activations/layer6_attention_weight_min": -58.68129348754883,
      "activations/layer7_attention_weight_max": 61.80281448364258,
      "activations/layer7_attention_weight_min": -61.74296188354492,
      "activations/layer8_attention_weight_max": 52.314483642578125,
      "activations/layer8_attention_weight_min": -46.06216812133789,
      "activations/layer9_attention_weight_max": 82.97958374023438,
      "activations/layer9_attention_weight_min": -69.27818298339844,
      "epoch": 1.56,
      "learning_rate": 0.00014137878787878787,
      "loss": 3.0166,
      "step": 26800
    },
    {
      "activations/layer0_attention_weight_max": 16.41714096069336,
      "activations/layer0_attention_weight_min": -11.05048942565918,
      "activations/layer10_attention_weight_max": 53.42082214355469,
      "activations/layer10_attention_weight_min": -46.68174362182617,
      "activations/layer11_attention_weight_max": 42.07429122924805,
      "activations/layer11_attention_weight_min": -38.898067474365234,
      "activations/layer12_attention_weight_max": 35.29418182373047,
      "activations/layer12_attention_weight_min": -26.971818923950195,
      "activations/layer13_attention_weight_max": 51.15171813964844,
      "activations/layer13_attention_weight_min": -36.76777267456055,
      "activations/layer14_attention_weight_max": 82.27193450927734,
      "activations/layer14_attention_weight_min": -60.66157913208008,
      "activations/layer15_attention_weight_max": 59.73135757446289,
      "activations/layer15_attention_weight_min": -39.739810943603516,
      "activations/layer16_attention_weight_max": 56.02812957763672,
      "activations/layer16_attention_weight_min": -38.88393020629883,
      "activations/layer17_attention_weight_max": 55.14033508300781,
      "activations/layer17_attention_weight_min": -30.127328872680664,
      "activations/layer18_attention_weight_max": 32.23427963256836,
      "activations/layer18_attention_weight_min": -16.549755096435547,
      "activations/layer19_attention_weight_max": 63.0389518737793,
      "activations/layer19_attention_weight_min": -31.861042022705078,
      "activations/layer1_attention_weight_max": 17.065332412719727,
      "activations/layer1_attention_weight_min": -10.81212043762207,
      "activations/layer20_attention_weight_max": 47.03597640991211,
      "activations/layer20_attention_weight_min": -24.60049057006836,
      "activations/layer21_attention_weight_max": 38.66943359375,
      "activations/layer21_attention_weight_min": -17.059940338134766,
      "activations/layer22_attention_weight_max": 50.30127716064453,
      "activations/layer22_attention_weight_min": -25.710155487060547,
      "activations/layer23_attention_weight_max": 30.88189697265625,
      "activations/layer23_attention_weight_min": -16.235960006713867,
      "activations/layer2_attention_weight_max": 27.310558319091797,
      "activations/layer2_attention_weight_min": -27.238492965698242,
      "activations/layer3_attention_weight_max": 65.58638763427734,
      "activations/layer3_attention_weight_min": -67.54590606689453,
      "activations/layer4_attention_weight_max": 125.78887176513672,
      "activations/layer4_attention_weight_min": -114.60347747802734,
      "activations/layer5_attention_weight_max": 119.20700073242188,
      "activations/layer5_attention_weight_min": -101.45974731445312,
      "activations/layer6_attention_weight_max": 61.73896026611328,
      "activations/layer6_attention_weight_min": -53.36534881591797,
      "activations/layer7_attention_weight_max": 68.0810317993164,
      "activations/layer7_attention_weight_min": -62.562320709228516,
      "activations/layer8_attention_weight_max": 49.061458587646484,
      "activations/layer8_attention_weight_min": -44.0091438293457,
      "activations/layer9_attention_weight_max": 65.35745239257812,
      "activations/layer9_attention_weight_min": -57.565731048583984,
      "epoch": 1.56,
      "learning_rate": 0.00014135984848484846,
      "loss": 3.0238,
      "step": 26850
    },
    {
      "activations/layer0_attention_weight_max": 16.391117095947266,
      "activations/layer0_attention_weight_min": -11.82485580444336,
      "activations/layer10_attention_weight_max": 69.9638442993164,
      "activations/layer10_attention_weight_min": -57.62809753417969,
      "activations/layer11_attention_weight_max": 53.02873992919922,
      "activations/layer11_attention_weight_min": -36.07688903808594,
      "activations/layer12_attention_weight_max": 42.71919631958008,
      "activations/layer12_attention_weight_min": -28.51860237121582,
      "activations/layer13_attention_weight_max": 59.491668701171875,
      "activations/layer13_attention_weight_min": -40.2318229675293,
      "activations/layer14_attention_weight_max": 92.69149017333984,
      "activations/layer14_attention_weight_min": -63.12791442871094,
      "activations/layer15_attention_weight_max": 57.50618362426758,
      "activations/layer15_attention_weight_min": -37.416683197021484,
      "activations/layer16_attention_weight_max": 62.91773223876953,
      "activations/layer16_attention_weight_min": -39.29265594482422,
      "activations/layer17_attention_weight_max": 51.05740737915039,
      "activations/layer17_attention_weight_min": -29.431325912475586,
      "activations/layer18_attention_weight_max": 26.889644622802734,
      "activations/layer18_attention_weight_min": -16.189355850219727,
      "activations/layer19_attention_weight_max": 57.83545684814453,
      "activations/layer19_attention_weight_min": -35.14590835571289,
      "activations/layer1_attention_weight_max": 18.44161033630371,
      "activations/layer1_attention_weight_min": -11.236282348632812,
      "activations/layer20_attention_weight_max": 44.83525848388672,
      "activations/layer20_attention_weight_min": -25.53871726989746,
      "activations/layer21_attention_weight_max": 31.311620712280273,
      "activations/layer21_attention_weight_min": -17.25240135192871,
      "activations/layer22_attention_weight_max": 51.547332763671875,
      "activations/layer22_attention_weight_min": -26.059724807739258,
      "activations/layer23_attention_weight_max": 28.420331954956055,
      "activations/layer23_attention_weight_min": -14.377836227416992,
      "activations/layer2_attention_weight_max": 27.862838745117188,
      "activations/layer2_attention_weight_min": -26.882291793823242,
      "activations/layer3_attention_weight_max": 65.72454833984375,
      "activations/layer3_attention_weight_min": -67.73250579833984,
      "activations/layer4_attention_weight_max": 116.3895263671875,
      "activations/layer4_attention_weight_min": -108.68265533447266,
      "activations/layer5_attention_weight_max": 123.5093994140625,
      "activations/layer5_attention_weight_min": -104.6903305053711,
      "activations/layer6_attention_weight_max": 66.32754516601562,
      "activations/layer6_attention_weight_min": -54.540626525878906,
      "activations/layer7_attention_weight_max": 75.61494445800781,
      "activations/layer7_attention_weight_min": -61.015892028808594,
      "activations/layer8_attention_weight_max": 62.26276397705078,
      "activations/layer8_attention_weight_min": -48.83256912231445,
      "activations/layer9_attention_weight_max": 106.17037200927734,
      "activations/layer9_attention_weight_min": -81.80410766601562,
      "epoch": 1.56,
      "learning_rate": 0.00014134090909090908,
      "loss": 3.0178,
      "step": 26900
    },
    {
      "activations/layer0_attention_weight_max": 16.775712966918945,
      "activations/layer0_attention_weight_min": -11.110981941223145,
      "activations/layer10_attention_weight_max": 46.15057373046875,
      "activations/layer10_attention_weight_min": -43.78739929199219,
      "activations/layer11_attention_weight_max": 40.50004577636719,
      "activations/layer11_attention_weight_min": -34.91326904296875,
      "activations/layer12_attention_weight_max": 45.42913818359375,
      "activations/layer12_attention_weight_min": -33.27925491333008,
      "activations/layer13_attention_weight_max": 53.95224380493164,
      "activations/layer13_attention_weight_min": -41.539390563964844,
      "activations/layer14_attention_weight_max": 84.00665283203125,
      "activations/layer14_attention_weight_min": -60.42488479614258,
      "activations/layer15_attention_weight_max": 57.16499328613281,
      "activations/layer15_attention_weight_min": -38.45428466796875,
      "activations/layer16_attention_weight_max": 63.819580078125,
      "activations/layer16_attention_weight_min": -44.78349304199219,
      "activations/layer17_attention_weight_max": 47.76958084106445,
      "activations/layer17_attention_weight_min": -29.009973526000977,
      "activations/layer18_attention_weight_max": 28.481698989868164,
      "activations/layer18_attention_weight_min": -17.005130767822266,
      "activations/layer19_attention_weight_max": 73.50334930419922,
      "activations/layer19_attention_weight_min": -36.82875442504883,
      "activations/layer1_attention_weight_max": 17.226802825927734,
      "activations/layer1_attention_weight_min": -11.30134105682373,
      "activations/layer20_attention_weight_max": 51.91947937011719,
      "activations/layer20_attention_weight_min": -24.397382736206055,
      "activations/layer21_attention_weight_max": 28.95556640625,
      "activations/layer21_attention_weight_min": -17.177927017211914,
      "activations/layer22_attention_weight_max": 51.87208557128906,
      "activations/layer22_attention_weight_min": -23.86488151550293,
      "activations/layer23_attention_weight_max": 29.248207092285156,
      "activations/layer23_attention_weight_min": -15.47589111328125,
      "activations/layer2_attention_weight_max": 28.44137191772461,
      "activations/layer2_attention_weight_min": -26.967159271240234,
      "activations/layer3_attention_weight_max": 64.1089096069336,
      "activations/layer3_attention_weight_min": -65.96222686767578,
      "activations/layer4_attention_weight_max": 112.1631851196289,
      "activations/layer4_attention_weight_min": -105.22161102294922,
      "activations/layer5_attention_weight_max": 110.30165100097656,
      "activations/layer5_attention_weight_min": -97.9540023803711,
      "activations/layer6_attention_weight_max": 58.989158630371094,
      "activations/layer6_attention_weight_min": -53.490455627441406,
      "activations/layer7_attention_weight_max": 62.37260055541992,
      "activations/layer7_attention_weight_min": -58.18094253540039,
      "activations/layer8_attention_weight_max": 50.05127716064453,
      "activations/layer8_attention_weight_min": -44.15176010131836,
      "activations/layer9_attention_weight_max": 71.72653198242188,
      "activations/layer9_attention_weight_min": -70.6249008178711,
      "epoch": 1.57,
      "learning_rate": 0.0001413219696969697,
      "loss": 3.0054,
      "step": 26950
    },
    {
      "activations/layer0_attention_weight_max": 17.118391036987305,
      "activations/layer0_attention_weight_min": -11.13979434967041,
      "activations/layer10_attention_weight_max": 55.84689712524414,
      "activations/layer10_attention_weight_min": -44.7776985168457,
      "activations/layer11_attention_weight_max": 43.76519775390625,
      "activations/layer11_attention_weight_min": -37.3944206237793,
      "activations/layer12_attention_weight_max": 31.10365104675293,
      "activations/layer12_attention_weight_min": -27.939069747924805,
      "activations/layer13_attention_weight_max": 55.96622848510742,
      "activations/layer13_attention_weight_min": -40.2996940612793,
      "activations/layer14_attention_weight_max": 74.85144805908203,
      "activations/layer14_attention_weight_min": -57.663421630859375,
      "activations/layer15_attention_weight_max": 54.80929946899414,
      "activations/layer15_attention_weight_min": -42.55270767211914,
      "activations/layer16_attention_weight_max": 58.3270149230957,
      "activations/layer16_attention_weight_min": -43.40108871459961,
      "activations/layer17_attention_weight_max": 52.41969299316406,
      "activations/layer17_attention_weight_min": -30.636667251586914,
      "activations/layer18_attention_weight_max": 30.585163116455078,
      "activations/layer18_attention_weight_min": -15.88549518585205,
      "activations/layer19_attention_weight_max": 61.44331741333008,
      "activations/layer19_attention_weight_min": -39.268917083740234,
      "activations/layer1_attention_weight_max": 17.52579689025879,
      "activations/layer1_attention_weight_min": -11.697600364685059,
      "activations/layer20_attention_weight_max": 42.904624938964844,
      "activations/layer20_attention_weight_min": -25.880603790283203,
      "activations/layer21_attention_weight_max": 30.71642303466797,
      "activations/layer21_attention_weight_min": -17.592052459716797,
      "activations/layer22_attention_weight_max": 40.26416778564453,
      "activations/layer22_attention_weight_min": -26.77492332458496,
      "activations/layer23_attention_weight_max": 26.085851669311523,
      "activations/layer23_attention_weight_min": -15.600372314453125,
      "activations/layer2_attention_weight_max": 28.49659538269043,
      "activations/layer2_attention_weight_min": -28.865314483642578,
      "activations/layer3_attention_weight_max": 68.67697143554688,
      "activations/layer3_attention_weight_min": -70.45195007324219,
      "activations/layer4_attention_weight_max": 126.74163818359375,
      "activations/layer4_attention_weight_min": -115.31414794921875,
      "activations/layer5_attention_weight_max": 125.64573669433594,
      "activations/layer5_attention_weight_min": -107.03331756591797,
      "activations/layer6_attention_weight_max": 64.37621307373047,
      "activations/layer6_attention_weight_min": -55.2862548828125,
      "activations/layer7_attention_weight_max": 61.64368438720703,
      "activations/layer7_attention_weight_min": -59.35807418823242,
      "activations/layer8_attention_weight_max": 51.31501770019531,
      "activations/layer8_attention_weight_min": -46.501197814941406,
      "activations/layer9_attention_weight_max": 71.87480926513672,
      "activations/layer9_attention_weight_min": -61.874855041503906,
      "epoch": 1.57,
      "learning_rate": 0.00014130303030303028,
      "loss": 2.9937,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_loss": 2.95703125,
      "eval_runtime": 8.5288,
      "eval_samples_per_second": 503.468,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_openwebtext_loss": 2.95703125,
      "eval_openwebtext_ppl": 19.24076585887165,
      "eval_openwebtext_runtime": 8.5288,
      "eval_openwebtext_samples_per_second": 503.468,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_wikitext_loss": 3.2265625,
      "eval_wikitext_ppl": 25.192907336481866,
      "eval_wikitext_runtime": 1.9687,
      "eval_wikitext_samples_per_second": 231.629,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_lambada_loss": 3.337890625,
      "eval_lambada_ppl": 28.15966472087403,
      "eval_lambada_runtime": 9.5407,
      "eval_lambada_samples_per_second": 510.337,
      "step": 27000
    },
    {
      "activations/layer0_attention_weight_max": 17.10133934020996,
      "activations/layer0_attention_weight_min": -11.318547248840332,
      "activations/layer10_attention_weight_max": 47.04444122314453,
      "activations/layer10_attention_weight_min": -45.10685729980469,
      "activations/layer11_attention_weight_max": 40.97536849975586,
      "activations/layer11_attention_weight_min": -36.292518615722656,
      "activations/layer12_attention_weight_max": 43.53765869140625,
      "activations/layer12_attention_weight_min": -36.43060302734375,
      "activations/layer13_attention_weight_max": 48.64712905883789,
      "activations/layer13_attention_weight_min": -43.0189323425293,
      "activations/layer14_attention_weight_max": 72.8713607788086,
      "activations/layer14_attention_weight_min": -62.03437805175781,
      "activations/layer15_attention_weight_max": 54.4979362487793,
      "activations/layer15_attention_weight_min": -41.67291259765625,
      "activations/layer16_attention_weight_max": 52.19697570800781,
      "activations/layer16_attention_weight_min": -38.12221908569336,
      "activations/layer17_attention_weight_max": 49.623626708984375,
      "activations/layer17_attention_weight_min": -35.92243576049805,
      "activations/layer18_attention_weight_max": 26.77071189880371,
      "activations/layer18_attention_weight_min": -18.541038513183594,
      "activations/layer19_attention_weight_max": 52.933509826660156,
      "activations/layer19_attention_weight_min": -31.613983154296875,
      "activations/layer1_attention_weight_max": 17.34581184387207,
      "activations/layer1_attention_weight_min": -11.048067092895508,
      "activations/layer20_attention_weight_max": 43.71527862548828,
      "activations/layer20_attention_weight_min": -23.669897079467773,
      "activations/layer21_attention_weight_max": 28.609935760498047,
      "activations/layer21_attention_weight_min": -19.55803108215332,
      "activations/layer22_attention_weight_max": 46.09576416015625,
      "activations/layer22_attention_weight_min": -25.568819046020508,
      "activations/layer23_attention_weight_max": 24.820425033569336,
      "activations/layer23_attention_weight_min": -15.344283103942871,
      "activations/layer2_attention_weight_max": 26.820676803588867,
      "activations/layer2_attention_weight_min": -26.352893829345703,
      "activations/layer3_attention_weight_max": 65.46575927734375,
      "activations/layer3_attention_weight_min": -67.72371673583984,
      "activations/layer4_attention_weight_max": 114.69908905029297,
      "activations/layer4_attention_weight_min": -111.23221588134766,
      "activations/layer5_attention_weight_max": 110.36211395263672,
      "activations/layer5_attention_weight_min": -100.53756713867188,
      "activations/layer6_attention_weight_max": 59.290313720703125,
      "activations/layer6_attention_weight_min": -56.21067428588867,
      "activations/layer7_attention_weight_max": 61.089698791503906,
      "activations/layer7_attention_weight_min": -60.97285461425781,
      "activations/layer8_attention_weight_max": 48.866294860839844,
      "activations/layer8_attention_weight_min": -43.7880744934082,
      "activations/layer9_attention_weight_max": 64.8712387084961,
      "activations/layer9_attention_weight_min": -55.520076751708984,
      "epoch": 1.57,
      "learning_rate": 0.0001412840909090909,
      "loss": 3.0012,
      "step": 27050
    },
    {
      "activations/layer0_attention_weight_max": 15.174907684326172,
      "activations/layer0_attention_weight_min": -11.77764892578125,
      "activations/layer10_attention_weight_max": 65.31526184082031,
      "activations/layer10_attention_weight_min": -53.33727264404297,
      "activations/layer11_attention_weight_max": 55.62885665893555,
      "activations/layer11_attention_weight_min": -39.495750427246094,
      "activations/layer12_attention_weight_max": 72.2362289428711,
      "activations/layer12_attention_weight_min": -40.113739013671875,
      "activations/layer13_attention_weight_max": 93.36612701416016,
      "activations/layer13_attention_weight_min": -60.8787841796875,
      "activations/layer14_attention_weight_max": 82.64817810058594,
      "activations/layer14_attention_weight_min": -61.43547439575195,
      "activations/layer15_attention_weight_max": 67.66241455078125,
      "activations/layer15_attention_weight_min": -44.94068908691406,
      "activations/layer16_attention_weight_max": 50.98993682861328,
      "activations/layer16_attention_weight_min": -40.78567886352539,
      "activations/layer17_attention_weight_max": 42.949501037597656,
      "activations/layer17_attention_weight_min": -32.392967224121094,
      "activations/layer18_attention_weight_max": 28.601970672607422,
      "activations/layer18_attention_weight_min": -20.68881607055664,
      "activations/layer19_attention_weight_max": 51.24555206298828,
      "activations/layer19_attention_weight_min": -36.42638397216797,
      "activations/layer1_attention_weight_max": 18.11400604248047,
      "activations/layer1_attention_weight_min": -12.130425453186035,
      "activations/layer20_attention_weight_max": 37.76289749145508,
      "activations/layer20_attention_weight_min": -26.002714157104492,
      "activations/layer21_attention_weight_max": 26.853734970092773,
      "activations/layer21_attention_weight_min": -16.408512115478516,
      "activations/layer22_attention_weight_max": 38.76985549926758,
      "activations/layer22_attention_weight_min": -26.245420455932617,
      "activations/layer23_attention_weight_max": 22.81465721130371,
      "activations/layer23_attention_weight_min": -18.256927490234375,
      "activations/layer2_attention_weight_max": 28.58379364013672,
      "activations/layer2_attention_weight_min": -27.584720611572266,
      "activations/layer3_attention_weight_max": 69.49850463867188,
      "activations/layer3_attention_weight_min": -71.57047271728516,
      "activations/layer4_attention_weight_max": 121.96166229248047,
      "activations/layer4_attention_weight_min": -113.7918930053711,
      "activations/layer5_attention_weight_max": 136.59149169921875,
      "activations/layer5_attention_weight_min": -114.10691833496094,
      "activations/layer6_attention_weight_max": 72.82524108886719,
      "activations/layer6_attention_weight_min": -55.906219482421875,
      "activations/layer7_attention_weight_max": 82.97425842285156,
      "activations/layer7_attention_weight_min": -68.76068115234375,
      "activations/layer8_attention_weight_max": 70.2215347290039,
      "activations/layer8_attention_weight_min": -54.18636703491211,
      "activations/layer9_attention_weight_max": 104.1657485961914,
      "activations/layer9_attention_weight_min": -85.60196685791016,
      "epoch": 1.57,
      "learning_rate": 0.0001412651515151515,
      "loss": 2.998,
      "step": 27100
    },
    {
      "activations/layer0_attention_weight_max": 17.06073760986328,
      "activations/layer0_attention_weight_min": -11.263082504272461,
      "activations/layer10_attention_weight_max": 51.1851692199707,
      "activations/layer10_attention_weight_min": -44.08091354370117,
      "activations/layer11_attention_weight_max": 41.05548858642578,
      "activations/layer11_attention_weight_min": -36.958736419677734,
      "activations/layer12_attention_weight_max": 69.2352294921875,
      "activations/layer12_attention_weight_min": -50.41644287109375,
      "activations/layer13_attention_weight_max": 70.19808959960938,
      "activations/layer13_attention_weight_min": -60.05353546142578,
      "activations/layer14_attention_weight_max": 73.88797760009766,
      "activations/layer14_attention_weight_min": -57.96849822998047,
      "activations/layer15_attention_weight_max": 52.60535430908203,
      "activations/layer15_attention_weight_min": -40.2524299621582,
      "activations/layer16_attention_weight_max": 51.690040588378906,
      "activations/layer16_attention_weight_min": -40.019065856933594,
      "activations/layer17_attention_weight_max": 42.783363342285156,
      "activations/layer17_attention_weight_min": -29.959035873413086,
      "activations/layer18_attention_weight_max": 26.638200759887695,
      "activations/layer18_attention_weight_min": -18.06206703186035,
      "activations/layer19_attention_weight_max": 49.64118957519531,
      "activations/layer19_attention_weight_min": -31.244413375854492,
      "activations/layer1_attention_weight_max": 17.25919532775879,
      "activations/layer1_attention_weight_min": -11.67281723022461,
      "activations/layer20_attention_weight_max": 44.3546257019043,
      "activations/layer20_attention_weight_min": -31.02031135559082,
      "activations/layer21_attention_weight_max": 27.97380256652832,
      "activations/layer21_attention_weight_min": -18.144960403442383,
      "activations/layer22_attention_weight_max": 41.30294418334961,
      "activations/layer22_attention_weight_min": -26.54583740234375,
      "activations/layer23_attention_weight_max": 22.522499084472656,
      "activations/layer23_attention_weight_min": -18.033288955688477,
      "activations/layer2_attention_weight_max": 28.804479598999023,
      "activations/layer2_attention_weight_min": -27.696441650390625,
      "activations/layer3_attention_weight_max": 65.96524810791016,
      "activations/layer3_attention_weight_min": -68.92842102050781,
      "activations/layer4_attention_weight_max": 118.87979888916016,
      "activations/layer4_attention_weight_min": -109.231201171875,
      "activations/layer5_attention_weight_max": 114.97041320800781,
      "activations/layer5_attention_weight_min": -97.83638763427734,
      "activations/layer6_attention_weight_max": 60.3719482421875,
      "activations/layer6_attention_weight_min": -51.944950103759766,
      "activations/layer7_attention_weight_max": 62.177860260009766,
      "activations/layer7_attention_weight_min": -61.48503875732422,
      "activations/layer8_attention_weight_max": 50.38359451293945,
      "activations/layer8_attention_weight_min": -44.88454055786133,
      "activations/layer9_attention_weight_max": 72.49832153320312,
      "activations/layer9_attention_weight_min": -62.22509765625,
      "epoch": 1.58,
      "learning_rate": 0.0001412462121212121,
      "loss": 3.0088,
      "step": 27150
    },
    {
      "activations/layer0_attention_weight_max": 16.403745651245117,
      "activations/layer0_attention_weight_min": -11.4877347946167,
      "activations/layer10_attention_weight_max": 56.03671646118164,
      "activations/layer10_attention_weight_min": -47.76426315307617,
      "activations/layer11_attention_weight_max": 39.928138732910156,
      "activations/layer11_attention_weight_min": -36.820526123046875,
      "activations/layer12_attention_weight_max": 39.84446334838867,
      "activations/layer12_attention_weight_min": -29.065465927124023,
      "activations/layer13_attention_weight_max": 53.7778205871582,
      "activations/layer13_attention_weight_min": -38.99477005004883,
      "activations/layer14_attention_weight_max": 71.58956146240234,
      "activations/layer14_attention_weight_min": -53.901336669921875,
      "activations/layer15_attention_weight_max": 59.196861267089844,
      "activations/layer15_attention_weight_min": -42.074337005615234,
      "activations/layer16_attention_weight_max": 61.6787109375,
      "activations/layer16_attention_weight_min": -41.12050247192383,
      "activations/layer17_attention_weight_max": 42.391605377197266,
      "activations/layer17_attention_weight_min": -26.67702293395996,
      "activations/layer18_attention_weight_max": 27.64179801940918,
      "activations/layer18_attention_weight_min": -16.19771957397461,
      "activations/layer19_attention_weight_max": 48.82656478881836,
      "activations/layer19_attention_weight_min": -31.974821090698242,
      "activations/layer1_attention_weight_max": 17.820297241210938,
      "activations/layer1_attention_weight_min": -10.907304763793945,
      "activations/layer20_attention_weight_max": 41.55369567871094,
      "activations/layer20_attention_weight_min": -24.99919891357422,
      "activations/layer21_attention_weight_max": 28.328502655029297,
      "activations/layer21_attention_weight_min": -18.087635040283203,
      "activations/layer22_attention_weight_max": 43.14600372314453,
      "activations/layer22_attention_weight_min": -24.330577850341797,
      "activations/layer23_attention_weight_max": 24.613563537597656,
      "activations/layer23_attention_weight_min": -18.889141082763672,
      "activations/layer2_attention_weight_max": 27.399681091308594,
      "activations/layer2_attention_weight_min": -27.142532348632812,
      "activations/layer3_attention_weight_max": 66.05397033691406,
      "activations/layer3_attention_weight_min": -68.7954330444336,
      "activations/layer4_attention_weight_max": 116.487548828125,
      "activations/layer4_attention_weight_min": -116.58731842041016,
      "activations/layer5_attention_weight_max": 111.4217758178711,
      "activations/layer5_attention_weight_min": -105.68059539794922,
      "activations/layer6_attention_weight_max": 60.021446228027344,
      "activations/layer6_attention_weight_min": -54.604488372802734,
      "activations/layer7_attention_weight_max": 61.361907958984375,
      "activations/layer7_attention_weight_min": -63.956207275390625,
      "activations/layer8_attention_weight_max": 50.25886917114258,
      "activations/layer8_attention_weight_min": -45.910240173339844,
      "activations/layer9_attention_weight_max": 62.85871124267578,
      "activations/layer9_attention_weight_min": -59.189453125,
      "epoch": 1.58,
      "learning_rate": 0.0001412272727272727,
      "loss": 3.0069,
      "step": 27200
    },
    {
      "activations/layer0_attention_weight_max": 16.892227172851562,
      "activations/layer0_attention_weight_min": -11.052573204040527,
      "activations/layer10_attention_weight_max": 56.47737503051758,
      "activations/layer10_attention_weight_min": -45.17776107788086,
      "activations/layer11_attention_weight_max": 45.2176628112793,
      "activations/layer11_attention_weight_min": -35.782958984375,
      "activations/layer12_attention_weight_max": 49.719703674316406,
      "activations/layer12_attention_weight_min": -31.192834854125977,
      "activations/layer13_attention_weight_max": 52.983848571777344,
      "activations/layer13_attention_weight_min": -38.674774169921875,
      "activations/layer14_attention_weight_max": 82.30513000488281,
      "activations/layer14_attention_weight_min": -57.20343780517578,
      "activations/layer15_attention_weight_max": 60.48863220214844,
      "activations/layer15_attention_weight_min": -37.77665328979492,
      "activations/layer16_attention_weight_max": 57.23915100097656,
      "activations/layer16_attention_weight_min": -37.28976058959961,
      "activations/layer17_attention_weight_max": 42.1406135559082,
      "activations/layer17_attention_weight_min": -27.862232208251953,
      "activations/layer18_attention_weight_max": 24.55166244506836,
      "activations/layer18_attention_weight_min": -16.895465850830078,
      "activations/layer19_attention_weight_max": 57.58738327026367,
      "activations/layer19_attention_weight_min": -34.37867736816406,
      "activations/layer1_attention_weight_max": 16.919921875,
      "activations/layer1_attention_weight_min": -11.882599830627441,
      "activations/layer20_attention_weight_max": 39.221492767333984,
      "activations/layer20_attention_weight_min": -24.791547775268555,
      "activations/layer21_attention_weight_max": 33.72103500366211,
      "activations/layer21_attention_weight_min": -18.20176887512207,
      "activations/layer22_attention_weight_max": 43.281776428222656,
      "activations/layer22_attention_weight_min": -25.64840316772461,
      "activations/layer23_attention_weight_max": 24.403059005737305,
      "activations/layer23_attention_weight_min": -15.742316246032715,
      "activations/layer2_attention_weight_max": 28.094707489013672,
      "activations/layer2_attention_weight_min": -26.54395294189453,
      "activations/layer3_attention_weight_max": 64.63780212402344,
      "activations/layer3_attention_weight_min": -67.31104278564453,
      "activations/layer4_attention_weight_max": 118.48783111572266,
      "activations/layer4_attention_weight_min": -107.5454330444336,
      "activations/layer5_attention_weight_max": 114.93526458740234,
      "activations/layer5_attention_weight_min": -98.427978515625,
      "activations/layer6_attention_weight_max": 60.42408752441406,
      "activations/layer6_attention_weight_min": -53.45930862426758,
      "activations/layer7_attention_weight_max": 65.01541900634766,
      "activations/layer7_attention_weight_min": -60.999820709228516,
      "activations/layer8_attention_weight_max": 50.64342498779297,
      "activations/layer8_attention_weight_min": -44.245452880859375,
      "activations/layer9_attention_weight_max": 68.88909149169922,
      "activations/layer9_attention_weight_min": -64.37544250488281,
      "epoch": 1.58,
      "learning_rate": 0.00014120833333333333,
      "loss": 3.0057,
      "step": 27250
    },
    {
      "activations/layer0_attention_weight_max": 16.897197723388672,
      "activations/layer0_attention_weight_min": -11.445298194885254,
      "activations/layer10_attention_weight_max": 54.18357467651367,
      "activations/layer10_attention_weight_min": -45.50484085083008,
      "activations/layer11_attention_weight_max": 40.00507354736328,
      "activations/layer11_attention_weight_min": -36.36790466308594,
      "activations/layer12_attention_weight_max": 33.79551315307617,
      "activations/layer12_attention_weight_min": -28.297584533691406,
      "activations/layer13_attention_weight_max": 52.33311462402344,
      "activations/layer13_attention_weight_min": -44.98648452758789,
      "activations/layer14_attention_weight_max": 78.76602172851562,
      "activations/layer14_attention_weight_min": -62.459197998046875,
      "activations/layer15_attention_weight_max": 58.15420913696289,
      "activations/layer15_attention_weight_min": -42.295989990234375,
      "activations/layer16_attention_weight_max": 47.76860427856445,
      "activations/layer16_attention_weight_min": -37.10196304321289,
      "activations/layer17_attention_weight_max": 47.88478469848633,
      "activations/layer17_attention_weight_min": -30.605079650878906,
      "activations/layer18_attention_weight_max": 25.36283302307129,
      "activations/layer18_attention_weight_min": -18.234477996826172,
      "activations/layer19_attention_weight_max": 60.85658645629883,
      "activations/layer19_attention_weight_min": -38.838462829589844,
      "activations/layer1_attention_weight_max": 17.526561737060547,
      "activations/layer1_attention_weight_min": -12.297212600708008,
      "activations/layer20_attention_weight_max": 36.8801383972168,
      "activations/layer20_attention_weight_min": -26.73601722717285,
      "activations/layer21_attention_weight_max": 32.03656768798828,
      "activations/layer21_attention_weight_min": -20.891077041625977,
      "activations/layer22_attention_weight_max": 47.3265495300293,
      "activations/layer22_attention_weight_min": -26.921695709228516,
      "activations/layer23_attention_weight_max": 28.473960876464844,
      "activations/layer23_attention_weight_min": -16.293087005615234,
      "activations/layer2_attention_weight_max": 28.029502868652344,
      "activations/layer2_attention_weight_min": -26.970975875854492,
      "activations/layer3_attention_weight_max": 67.60594940185547,
      "activations/layer3_attention_weight_min": -68.06287384033203,
      "activations/layer4_attention_weight_max": 118.47411346435547,
      "activations/layer4_attention_weight_min": -109.64385223388672,
      "activations/layer5_attention_weight_max": 115.25767517089844,
      "activations/layer5_attention_weight_min": -101.22860717773438,
      "activations/layer6_attention_weight_max": 58.52744674682617,
      "activations/layer6_attention_weight_min": -54.622779846191406,
      "activations/layer7_attention_weight_max": 66.05445098876953,
      "activations/layer7_attention_weight_min": -61.40655517578125,
      "activations/layer8_attention_weight_max": 54.38591384887695,
      "activations/layer8_attention_weight_min": -50.612998962402344,
      "activations/layer9_attention_weight_max": 64.4884262084961,
      "activations/layer9_attention_weight_min": -55.02778244018555,
      "epoch": 1.59,
      "learning_rate": 0.00014118939393939392,
      "loss": 3.0009,
      "step": 27300
    },
    {
      "activations/layer0_attention_weight_max": 16.95900535583496,
      "activations/layer0_attention_weight_min": -11.41225814819336,
      "activations/layer10_attention_weight_max": 53.72333908081055,
      "activations/layer10_attention_weight_min": -45.13771438598633,
      "activations/layer11_attention_weight_max": 43.135292053222656,
      "activations/layer11_attention_weight_min": -36.708892822265625,
      "activations/layer12_attention_weight_max": 66.51936340332031,
      "activations/layer12_attention_weight_min": -42.29301071166992,
      "activations/layer13_attention_weight_max": 72.2416000366211,
      "activations/layer13_attention_weight_min": -51.576534271240234,
      "activations/layer14_attention_weight_max": 81.91991424560547,
      "activations/layer14_attention_weight_min": -58.54567337036133,
      "activations/layer15_attention_weight_max": 59.34380340576172,
      "activations/layer15_attention_weight_min": -42.85896301269531,
      "activations/layer16_attention_weight_max": 57.173492431640625,
      "activations/layer16_attention_weight_min": -37.95462417602539,
      "activations/layer17_attention_weight_max": 43.57563400268555,
      "activations/layer17_attention_weight_min": -28.177579879760742,
      "activations/layer18_attention_weight_max": 24.995946884155273,
      "activations/layer18_attention_weight_min": -17.718172073364258,
      "activations/layer19_attention_weight_max": 57.429901123046875,
      "activations/layer19_attention_weight_min": -34.60159683227539,
      "activations/layer1_attention_weight_max": 17.2325439453125,
      "activations/layer1_attention_weight_min": -12.105973243713379,
      "activations/layer20_attention_weight_max": 47.29951095581055,
      "activations/layer20_attention_weight_min": -25.398460388183594,
      "activations/layer21_attention_weight_max": 38.624935150146484,
      "activations/layer21_attention_weight_min": -22.579626083374023,
      "activations/layer22_attention_weight_max": 46.382286071777344,
      "activations/layer22_attention_weight_min": -23.57932472229004,
      "activations/layer23_attention_weight_max": 24.81147003173828,
      "activations/layer23_attention_weight_min": -14.6668701171875,
      "activations/layer2_attention_weight_max": 27.313047409057617,
      "activations/layer2_attention_weight_min": -27.491561889648438,
      "activations/layer3_attention_weight_max": 66.023193359375,
      "activations/layer3_attention_weight_min": -67.82778930664062,
      "activations/layer4_attention_weight_max": 118.45182037353516,
      "activations/layer4_attention_weight_min": -112.50740051269531,
      "activations/layer5_attention_weight_max": 113.88259887695312,
      "activations/layer5_attention_weight_min": -100.60006713867188,
      "activations/layer6_attention_weight_max": 61.33725357055664,
      "activations/layer6_attention_weight_min": -53.97633361816406,
      "activations/layer7_attention_weight_max": 60.31269836425781,
      "activations/layer7_attention_weight_min": -61.43244171142578,
      "activations/layer8_attention_weight_max": 52.0385627746582,
      "activations/layer8_attention_weight_min": -43.50155258178711,
      "activations/layer9_attention_weight_max": 66.86298370361328,
      "activations/layer9_attention_weight_min": -63.1903190612793,
      "epoch": 1.59,
      "learning_rate": 0.00014117045454545453,
      "loss": 3.0203,
      "step": 27350
    },
    {
      "activations/layer0_attention_weight_max": 16.609783172607422,
      "activations/layer0_attention_weight_min": -11.88550090789795,
      "activations/layer10_attention_weight_max": 52.902854919433594,
      "activations/layer10_attention_weight_min": -44.509700775146484,
      "activations/layer11_attention_weight_max": 40.341941833496094,
      "activations/layer11_attention_weight_min": -38.7429084777832,
      "activations/layer12_attention_weight_max": 29.21099090576172,
      "activations/layer12_attention_weight_min": -29.167442321777344,
      "activations/layer13_attention_weight_max": 46.43989181518555,
      "activations/layer13_attention_weight_min": -33.76221466064453,
      "activations/layer14_attention_weight_max": 72.18377685546875,
      "activations/layer14_attention_weight_min": -60.74073028564453,
      "activations/layer15_attention_weight_max": 49.49135971069336,
      "activations/layer15_attention_weight_min": -37.66416931152344,
      "activations/layer16_attention_weight_max": 52.42764663696289,
      "activations/layer16_attention_weight_min": -34.59286880493164,
      "activations/layer17_attention_weight_max": 38.64372634887695,
      "activations/layer17_attention_weight_min": -28.327085494995117,
      "activations/layer18_attention_weight_max": 24.66577911376953,
      "activations/layer18_attention_weight_min": -16.537677764892578,
      "activations/layer19_attention_weight_max": 51.50641632080078,
      "activations/layer19_attention_weight_min": -32.90842819213867,
      "activations/layer1_attention_weight_max": 17.157573699951172,
      "activations/layer1_attention_weight_min": -11.25905990600586,
      "activations/layer20_attention_weight_max": 45.36392593383789,
      "activations/layer20_attention_weight_min": -25.599308013916016,
      "activations/layer21_attention_weight_max": 24.04925537109375,
      "activations/layer21_attention_weight_min": -19.403583526611328,
      "activations/layer22_attention_weight_max": 41.36935043334961,
      "activations/layer22_attention_weight_min": -25.1180477142334,
      "activations/layer23_attention_weight_max": 22.263572692871094,
      "activations/layer23_attention_weight_min": -16.626747131347656,
      "activations/layer2_attention_weight_max": 27.078107833862305,
      "activations/layer2_attention_weight_min": -26.237821578979492,
      "activations/layer3_attention_weight_max": 64.26951599121094,
      "activations/layer3_attention_weight_min": -68.05846405029297,
      "activations/layer4_attention_weight_max": 112.7940673828125,
      "activations/layer4_attention_weight_min": -111.26049041748047,
      "activations/layer5_attention_weight_max": 110.37643432617188,
      "activations/layer5_attention_weight_min": -102.40190124511719,
      "activations/layer6_attention_weight_max": 63.538761138916016,
      "activations/layer6_attention_weight_min": -53.11878967285156,
      "activations/layer7_attention_weight_max": 55.84117889404297,
      "activations/layer7_attention_weight_min": -59.83686447143555,
      "activations/layer8_attention_weight_max": 49.41130447387695,
      "activations/layer8_attention_weight_min": -41.37282943725586,
      "activations/layer9_attention_weight_max": 70.49993896484375,
      "activations/layer9_attention_weight_min": -65.62879943847656,
      "epoch": 1.59,
      "learning_rate": 0.00014115151515151512,
      "loss": 2.9981,
      "step": 27400
    },
    {
      "activations/layer0_attention_weight_max": 16.435251235961914,
      "activations/layer0_attention_weight_min": -11.716618537902832,
      "activations/layer10_attention_weight_max": 59.720054626464844,
      "activations/layer10_attention_weight_min": -46.2415657043457,
      "activations/layer11_attention_weight_max": 59.26960754394531,
      "activations/layer11_attention_weight_min": -40.06414031982422,
      "activations/layer12_attention_weight_max": 61.997528076171875,
      "activations/layer12_attention_weight_min": -37.1077766418457,
      "activations/layer13_attention_weight_max": 44.37905502319336,
      "activations/layer13_attention_weight_min": -41.80427169799805,
      "activations/layer14_attention_weight_max": 68.51856994628906,
      "activations/layer14_attention_weight_min": -59.841773986816406,
      "activations/layer15_attention_weight_max": 47.659725189208984,
      "activations/layer15_attention_weight_min": -39.74959182739258,
      "activations/layer16_attention_weight_max": 44.91952896118164,
      "activations/layer16_attention_weight_min": -36.36066818237305,
      "activations/layer17_attention_weight_max": 40.42085266113281,
      "activations/layer17_attention_weight_min": -28.8676700592041,
      "activations/layer18_attention_weight_max": 23.332487106323242,
      "activations/layer18_attention_weight_min": -19.34993553161621,
      "activations/layer19_attention_weight_max": 41.811031341552734,
      "activations/layer19_attention_weight_min": -30.018606185913086,
      "activations/layer1_attention_weight_max": 18.989744186401367,
      "activations/layer1_attention_weight_min": -10.544084548950195,
      "activations/layer20_attention_weight_max": 33.29264831542969,
      "activations/layer20_attention_weight_min": -24.377647399902344,
      "activations/layer21_attention_weight_max": 28.691280364990234,
      "activations/layer21_attention_weight_min": -16.307952880859375,
      "activations/layer22_attention_weight_max": 39.1650276184082,
      "activations/layer22_attention_weight_min": -25.42743492126465,
      "activations/layer23_attention_weight_max": 20.05385971069336,
      "activations/layer23_attention_weight_min": -14.063085556030273,
      "activations/layer2_attention_weight_max": 28.85536003112793,
      "activations/layer2_attention_weight_min": -28.150775909423828,
      "activations/layer3_attention_weight_max": 67.26409912109375,
      "activations/layer3_attention_weight_min": -71.8180923461914,
      "activations/layer4_attention_weight_max": 119.51042938232422,
      "activations/layer4_attention_weight_min": -113.98583984375,
      "activations/layer5_attention_weight_max": 116.16789245605469,
      "activations/layer5_attention_weight_min": -107.97002410888672,
      "activations/layer6_attention_weight_max": 64.28755187988281,
      "activations/layer6_attention_weight_min": -56.83039093017578,
      "activations/layer7_attention_weight_max": 74.62958526611328,
      "activations/layer7_attention_weight_min": -71.41822052001953,
      "activations/layer8_attention_weight_max": 68.07462310791016,
      "activations/layer8_attention_weight_min": -55.06663131713867,
      "activations/layer9_attention_weight_max": 104.96147918701172,
      "activations/layer9_attention_weight_min": -83.3932113647461,
      "epoch": 1.59,
      "learning_rate": 0.00014113257575757573,
      "loss": 2.987,
      "step": 27450
    },
    {
      "activations/layer0_attention_weight_max": 16.34885025024414,
      "activations/layer0_attention_weight_min": -11.219780921936035,
      "activations/layer10_attention_weight_max": 52.31702423095703,
      "activations/layer10_attention_weight_min": -49.56510925292969,
      "activations/layer11_attention_weight_max": 42.894615173339844,
      "activations/layer11_attention_weight_min": -38.83909606933594,
      "activations/layer12_attention_weight_max": 53.04677963256836,
      "activations/layer12_attention_weight_min": -40.64547348022461,
      "activations/layer13_attention_weight_max": 47.04670333862305,
      "activations/layer13_attention_weight_min": -40.776607513427734,
      "activations/layer14_attention_weight_max": 68.34123229980469,
      "activations/layer14_attention_weight_min": -58.773902893066406,
      "activations/layer15_attention_weight_max": 50.939945220947266,
      "activations/layer15_attention_weight_min": -38.81188201904297,
      "activations/layer16_attention_weight_max": 51.902774810791016,
      "activations/layer16_attention_weight_min": -39.36453628540039,
      "activations/layer17_attention_weight_max": 45.78236389160156,
      "activations/layer17_attention_weight_min": -30.420772552490234,
      "activations/layer18_attention_weight_max": 24.061952590942383,
      "activations/layer18_attention_weight_min": -16.287174224853516,
      "activations/layer19_attention_weight_max": 52.976680755615234,
      "activations/layer19_attention_weight_min": -34.53910446166992,
      "activations/layer1_attention_weight_max": 17.414142608642578,
      "activations/layer1_attention_weight_min": -11.970690727233887,
      "activations/layer20_attention_weight_max": 41.63600158691406,
      "activations/layer20_attention_weight_min": -28.458024978637695,
      "activations/layer21_attention_weight_max": 25.31960105895996,
      "activations/layer21_attention_weight_min": -17.51993179321289,
      "activations/layer22_attention_weight_max": 43.387840270996094,
      "activations/layer22_attention_weight_min": -26.03831672668457,
      "activations/layer23_attention_weight_max": 26.172344207763672,
      "activations/layer23_attention_weight_min": -16.221050262451172,
      "activations/layer2_attention_weight_max": 27.712486267089844,
      "activations/layer2_attention_weight_min": -26.03731918334961,
      "activations/layer3_attention_weight_max": 68.35480499267578,
      "activations/layer3_attention_weight_min": -69.70679473876953,
      "activations/layer4_attention_weight_max": 121.29475402832031,
      "activations/layer4_attention_weight_min": -112.1797866821289,
      "activations/layer5_attention_weight_max": 117.07460021972656,
      "activations/layer5_attention_weight_min": -105.88631439208984,
      "activations/layer6_attention_weight_max": 62.75285720825195,
      "activations/layer6_attention_weight_min": -54.42323303222656,
      "activations/layer7_attention_weight_max": 63.640811920166016,
      "activations/layer7_attention_weight_min": -62.27957534790039,
      "activations/layer8_attention_weight_max": 59.8197021484375,
      "activations/layer8_attention_weight_min": -51.29488754272461,
      "activations/layer9_attention_weight_max": 75.61529541015625,
      "activations/layer9_attention_weight_min": -72.05261993408203,
      "epoch": 1.6,
      "learning_rate": 0.00014111363636363635,
      "loss": 3.004,
      "step": 27500
    },
    {
      "activations/layer0_attention_weight_max": 16.46575164794922,
      "activations/layer0_attention_weight_min": -11.416794776916504,
      "activations/layer10_attention_weight_max": 51.22760009765625,
      "activations/layer10_attention_weight_min": -44.09867477416992,
      "activations/layer11_attention_weight_max": 42.78846740722656,
      "activations/layer11_attention_weight_min": -35.768314361572266,
      "activations/layer12_attention_weight_max": 35.222991943359375,
      "activations/layer12_attention_weight_min": -26.615182876586914,
      "activations/layer13_attention_weight_max": 53.64686584472656,
      "activations/layer13_attention_weight_min": -44.438743591308594,
      "activations/layer14_attention_weight_max": 85.99909973144531,
      "activations/layer14_attention_weight_min": -63.702857971191406,
      "activations/layer15_attention_weight_max": 56.79583740234375,
      "activations/layer15_attention_weight_min": -39.08562469482422,
      "activations/layer16_attention_weight_max": 64.68135070800781,
      "activations/layer16_attention_weight_min": -41.7485237121582,
      "activations/layer17_attention_weight_max": 50.41058349609375,
      "activations/layer17_attention_weight_min": -26.56481170654297,
      "activations/layer18_attention_weight_max": 29.765592575073242,
      "activations/layer18_attention_weight_min": -19.029680252075195,
      "activations/layer19_attention_weight_max": 66.69722747802734,
      "activations/layer19_attention_weight_min": -34.33076095581055,
      "activations/layer1_attention_weight_max": 17.357086181640625,
      "activations/layer1_attention_weight_min": -11.02057933807373,
      "activations/layer20_attention_weight_max": 47.21446228027344,
      "activations/layer20_attention_weight_min": -26.919963836669922,
      "activations/layer21_attention_weight_max": 30.66957664489746,
      "activations/layer21_attention_weight_min": -18.353656768798828,
      "activations/layer22_attention_weight_max": 53.03188705444336,
      "activations/layer22_attention_weight_min": -26.130090713500977,
      "activations/layer23_attention_weight_max": 28.53822898864746,
      "activations/layer23_attention_weight_min": -16.34419059753418,
      "activations/layer2_attention_weight_max": 29.361412048339844,
      "activations/layer2_attention_weight_min": -27.285343170166016,
      "activations/layer3_attention_weight_max": 69.57624053955078,
      "activations/layer3_attention_weight_min": -70.83854675292969,
      "activations/layer4_attention_weight_max": 126.54729461669922,
      "activations/layer4_attention_weight_min": -115.22696685791016,
      "activations/layer5_attention_weight_max": 117.30867004394531,
      "activations/layer5_attention_weight_min": -103.56591033935547,
      "activations/layer6_attention_weight_max": 63.6273307800293,
      "activations/layer6_attention_weight_min": -55.79158401489258,
      "activations/layer7_attention_weight_max": 63.40725326538086,
      "activations/layer7_attention_weight_min": -68.82535552978516,
      "activations/layer8_attention_weight_max": 50.03193664550781,
      "activations/layer8_attention_weight_min": -42.36162185668945,
      "activations/layer9_attention_weight_max": 66.44041442871094,
      "activations/layer9_attention_weight_min": -60.2250862121582,
      "epoch": 1.6,
      "learning_rate": 0.00014109469696969696,
      "loss": 3.0111,
      "step": 27550
    },
    {
      "activations/layer0_attention_weight_max": 16.816740036010742,
      "activations/layer0_attention_weight_min": -11.2536039352417,
      "activations/layer10_attention_weight_max": 51.00276565551758,
      "activations/layer10_attention_weight_min": -46.427120208740234,
      "activations/layer11_attention_weight_max": 42.422847747802734,
      "activations/layer11_attention_weight_min": -34.80534362792969,
      "activations/layer12_attention_weight_max": 34.5518913269043,
      "activations/layer12_attention_weight_min": -28.059146881103516,
      "activations/layer13_attention_weight_max": 53.11460494995117,
      "activations/layer13_attention_weight_min": -42.49309158325195,
      "activations/layer14_attention_weight_max": 85.9252700805664,
      "activations/layer14_attention_weight_min": -58.98057174682617,
      "activations/layer15_attention_weight_max": 59.78276824951172,
      "activations/layer15_attention_weight_min": -37.35160827636719,
      "activations/layer16_attention_weight_max": 71.09069061279297,
      "activations/layer16_attention_weight_min": -41.566810607910156,
      "activations/layer17_attention_weight_max": 49.86055374145508,
      "activations/layer17_attention_weight_min": -31.459144592285156,
      "activations/layer18_attention_weight_max": 27.867753982543945,
      "activations/layer18_attention_weight_min": -15.741964340209961,
      "activations/layer19_attention_weight_max": 69.18065643310547,
      "activations/layer19_attention_weight_min": -33.36393737792969,
      "activations/layer1_attention_weight_max": 17.901302337646484,
      "activations/layer1_attention_weight_min": -11.122435569763184,
      "activations/layer20_attention_weight_max": 51.396602630615234,
      "activations/layer20_attention_weight_min": -26.047014236450195,
      "activations/layer21_attention_weight_max": 36.27981185913086,
      "activations/layer21_attention_weight_min": -17.10573959350586,
      "activations/layer22_attention_weight_max": 53.11004638671875,
      "activations/layer22_attention_weight_min": -23.535438537597656,
      "activations/layer23_attention_weight_max": 27.64397430419922,
      "activations/layer23_attention_weight_min": -15.86353588104248,
      "activations/layer2_attention_weight_max": 26.61907958984375,
      "activations/layer2_attention_weight_min": -26.694347381591797,
      "activations/layer3_attention_weight_max": 68.14119720458984,
      "activations/layer3_attention_weight_min": -68.51338195800781,
      "activations/layer4_attention_weight_max": 119.32588195800781,
      "activations/layer4_attention_weight_min": -114.21453094482422,
      "activations/layer5_attention_weight_max": 113.25807189941406,
      "activations/layer5_attention_weight_min": -103.77743530273438,
      "activations/layer6_attention_weight_max": 60.47338104248047,
      "activations/layer6_attention_weight_min": -55.81564712524414,
      "activations/layer7_attention_weight_max": 64.1337661743164,
      "activations/layer7_attention_weight_min": -61.098262786865234,
      "activations/layer8_attention_weight_max": 51.67206573486328,
      "activations/layer8_attention_weight_min": -44.443519592285156,
      "activations/layer9_attention_weight_max": 74.16390991210938,
      "activations/layer9_attention_weight_min": -63.71680450439453,
      "epoch": 1.6,
      "learning_rate": 0.00014107575757575755,
      "loss": 3.0082,
      "step": 27600
    },
    {
      "activations/layer0_attention_weight_max": 15.637431144714355,
      "activations/layer0_attention_weight_min": -11.362337112426758,
      "activations/layer10_attention_weight_max": 60.08192825317383,
      "activations/layer10_attention_weight_min": -52.26328659057617,
      "activations/layer11_attention_weight_max": 38.44038009643555,
      "activations/layer11_attention_weight_min": -39.544063568115234,
      "activations/layer12_attention_weight_max": 42.04100799560547,
      "activations/layer12_attention_weight_min": -38.20315170288086,
      "activations/layer13_attention_weight_max": 54.735530853271484,
      "activations/layer13_attention_weight_min": -46.489173889160156,
      "activations/layer14_attention_weight_max": 79.54290771484375,
      "activations/layer14_attention_weight_min": -65.07028198242188,
      "activations/layer15_attention_weight_max": 56.02176284790039,
      "activations/layer15_attention_weight_min": -39.677940368652344,
      "activations/layer16_attention_weight_max": 53.5084342956543,
      "activations/layer16_attention_weight_min": -39.292476654052734,
      "activations/layer17_attention_weight_max": 45.43467712402344,
      "activations/layer17_attention_weight_min": -28.191808700561523,
      "activations/layer18_attention_weight_max": 23.454038619995117,
      "activations/layer18_attention_weight_min": -21.533138275146484,
      "activations/layer19_attention_weight_max": 52.5556755065918,
      "activations/layer19_attention_weight_min": -36.97187805175781,
      "activations/layer1_attention_weight_max": 17.556730270385742,
      "activations/layer1_attention_weight_min": -12.408618927001953,
      "activations/layer20_attention_weight_max": 39.13618469238281,
      "activations/layer20_attention_weight_min": -26.680923461914062,
      "activations/layer21_attention_weight_max": 29.518997192382812,
      "activations/layer21_attention_weight_min": -17.272457122802734,
      "activations/layer22_attention_weight_max": 39.793460845947266,
      "activations/layer22_attention_weight_min": -26.59747886657715,
      "activations/layer23_attention_weight_max": 26.05677032470703,
      "activations/layer23_attention_weight_min": -16.263134002685547,
      "activations/layer2_attention_weight_max": 26.700626373291016,
      "activations/layer2_attention_weight_min": -28.20442008972168,
      "activations/layer3_attention_weight_max": 66.89714813232422,
      "activations/layer3_attention_weight_min": -70.3111343383789,
      "activations/layer4_attention_weight_max": 127.30401611328125,
      "activations/layer4_attention_weight_min": -114.98172760009766,
      "activations/layer5_attention_weight_max": 113.99090576171875,
      "activations/layer5_attention_weight_min": -103.3524398803711,
      "activations/layer6_attention_weight_max": 61.071258544921875,
      "activations/layer6_attention_weight_min": -55.01816940307617,
      "activations/layer7_attention_weight_max": 60.29600143432617,
      "activations/layer7_attention_weight_min": -59.82159423828125,
      "activations/layer8_attention_weight_max": 50.810447692871094,
      "activations/layer8_attention_weight_min": -45.60847854614258,
      "activations/layer9_attention_weight_max": 76.9496078491211,
      "activations/layer9_attention_weight_min": -58.19570541381836,
      "epoch": 1.61,
      "learning_rate": 0.00014105681818181817,
      "loss": 3.0159,
      "step": 27650
    },
    {
      "activations/layer0_attention_weight_max": 16.954343795776367,
      "activations/layer0_attention_weight_min": -11.250633239746094,
      "activations/layer10_attention_weight_max": 64.1994400024414,
      "activations/layer10_attention_weight_min": -48.9898567199707,
      "activations/layer11_attention_weight_max": 61.74708557128906,
      "activations/layer11_attention_weight_min": -42.05155944824219,
      "activations/layer12_attention_weight_max": 49.917327880859375,
      "activations/layer12_attention_weight_min": -35.850250244140625,
      "activations/layer13_attention_weight_max": 58.26347351074219,
      "activations/layer13_attention_weight_min": -47.55472183227539,
      "activations/layer14_attention_weight_max": 72.83523559570312,
      "activations/layer14_attention_weight_min": -58.452354431152344,
      "activations/layer15_attention_weight_max": 53.72185134887695,
      "activations/layer15_attention_weight_min": -43.80539321899414,
      "activations/layer16_attention_weight_max": 56.1424446105957,
      "activations/layer16_attention_weight_min": -40.84796905517578,
      "activations/layer17_attention_weight_max": 48.72413635253906,
      "activations/layer17_attention_weight_min": -27.976486206054688,
      "activations/layer18_attention_weight_max": 24.682096481323242,
      "activations/layer18_attention_weight_min": -16.115385055541992,
      "activations/layer19_attention_weight_max": 53.413543701171875,
      "activations/layer19_attention_weight_min": -39.83030319213867,
      "activations/layer1_attention_weight_max": 17.090490341186523,
      "activations/layer1_attention_weight_min": -11.792959213256836,
      "activations/layer20_attention_weight_max": 39.262603759765625,
      "activations/layer20_attention_weight_min": -26.555513381958008,
      "activations/layer21_attention_weight_max": 27.538930892944336,
      "activations/layer21_attention_weight_min": -17.87320327758789,
      "activations/layer22_attention_weight_max": 48.24695587158203,
      "activations/layer22_attention_weight_min": -24.533275604248047,
      "activations/layer23_attention_weight_max": 25.181819915771484,
      "activations/layer23_attention_weight_min": -15.799783706665039,
      "activations/layer2_attention_weight_max": 28.397863388061523,
      "activations/layer2_attention_weight_min": -27.86639404296875,
      "activations/layer3_attention_weight_max": 71.22216796875,
      "activations/layer3_attention_weight_min": -73.5073471069336,
      "activations/layer4_attention_weight_max": 125.40242767333984,
      "activations/layer4_attention_weight_min": -114.27709197998047,
      "activations/layer5_attention_weight_max": 125.60314178466797,
      "activations/layer5_attention_weight_min": -105.68950653076172,
      "activations/layer6_attention_weight_max": 67.85403442382812,
      "activations/layer6_attention_weight_min": -55.205284118652344,
      "activations/layer7_attention_weight_max": 77.58236694335938,
      "activations/layer7_attention_weight_min": -67.52516174316406,
      "activations/layer8_attention_weight_max": 63.79301071166992,
      "activations/layer8_attention_weight_min": -48.367706298828125,
      "activations/layer9_attention_weight_max": 100.50640869140625,
      "activations/layer9_attention_weight_min": -77.4247055053711,
      "epoch": 1.61,
      "learning_rate": 0.00014103787878787878,
      "loss": 3.0099,
      "step": 27700
    },
    {
      "activations/layer0_attention_weight_max": 14.533064842224121,
      "activations/layer0_attention_weight_min": -11.139251708984375,
      "activations/layer10_attention_weight_max": 52.290771484375,
      "activations/layer10_attention_weight_min": -44.205650329589844,
      "activations/layer11_attention_weight_max": 39.55370330810547,
      "activations/layer11_attention_weight_min": -37.418704986572266,
      "activations/layer12_attention_weight_max": 38.33811569213867,
      "activations/layer12_attention_weight_min": -30.31574821472168,
      "activations/layer13_attention_weight_max": 48.3669319152832,
      "activations/layer13_attention_weight_min": -38.31585693359375,
      "activations/layer14_attention_weight_max": 76.52584838867188,
      "activations/layer14_attention_weight_min": -57.667152404785156,
      "activations/layer15_attention_weight_max": 50.21965408325195,
      "activations/layer15_attention_weight_min": -43.63007354736328,
      "activations/layer16_attention_weight_max": 48.23506546020508,
      "activations/layer16_attention_weight_min": -37.71441650390625,
      "activations/layer17_attention_weight_max": 44.25408935546875,
      "activations/layer17_attention_weight_min": -28.952796936035156,
      "activations/layer18_attention_weight_max": 25.883548736572266,
      "activations/layer18_attention_weight_min": -18.153837203979492,
      "activations/layer19_attention_weight_max": 54.07014846801758,
      "activations/layer19_attention_weight_min": -34.39117431640625,
      "activations/layer1_attention_weight_max": 17.21333122253418,
      "activations/layer1_attention_weight_min": -11.429911613464355,
      "activations/layer20_attention_weight_max": 38.32585906982422,
      "activations/layer20_attention_weight_min": -25.48119354248047,
      "activations/layer21_attention_weight_max": 26.32952880859375,
      "activations/layer21_attention_weight_min": -20.81465721130371,
      "activations/layer22_attention_weight_max": 40.53708267211914,
      "activations/layer22_attention_weight_min": -30.116065979003906,
      "activations/layer23_attention_weight_max": 28.6236629486084,
      "activations/layer23_attention_weight_min": -16.52284049987793,
      "activations/layer2_attention_weight_max": 28.065853118896484,
      "activations/layer2_attention_weight_min": -28.204620361328125,
      "activations/layer3_attention_weight_max": 67.54520416259766,
      "activations/layer3_attention_weight_min": -74.27588653564453,
      "activations/layer4_attention_weight_max": 126.41793060302734,
      "activations/layer4_attention_weight_min": -116.6354751586914,
      "activations/layer5_attention_weight_max": 119.4327621459961,
      "activations/layer5_attention_weight_min": -101.91700744628906,
      "activations/layer6_attention_weight_max": 65.55823516845703,
      "activations/layer6_attention_weight_min": -56.528648376464844,
      "activations/layer7_attention_weight_max": 63.381629943847656,
      "activations/layer7_attention_weight_min": -59.73625946044922,
      "activations/layer8_attention_weight_max": 52.84113311767578,
      "activations/layer8_attention_weight_min": -44.649009704589844,
      "activations/layer9_attention_weight_max": 66.50228118896484,
      "activations/layer9_attention_weight_min": -70.86971282958984,
      "epoch": 1.61,
      "learning_rate": 0.0001410189393939394,
      "loss": 3.0038,
      "step": 27750
    },
    {
      "activations/layer0_attention_weight_max": 16.352556228637695,
      "activations/layer0_attention_weight_min": -11.073586463928223,
      "activations/layer10_attention_weight_max": 54.77974319458008,
      "activations/layer10_attention_weight_min": -47.26411056518555,
      "activations/layer11_attention_weight_max": 38.9242057800293,
      "activations/layer11_attention_weight_min": -37.36577606201172,
      "activations/layer12_attention_weight_max": 31.660438537597656,
      "activations/layer12_attention_weight_min": -27.979700088500977,
      "activations/layer13_attention_weight_max": 47.244590759277344,
      "activations/layer13_attention_weight_min": -36.60297393798828,
      "activations/layer14_attention_weight_max": 69.19464111328125,
      "activations/layer14_attention_weight_min": -55.82516860961914,
      "activations/layer15_attention_weight_max": 53.56389617919922,
      "activations/layer15_attention_weight_min": -41.38121032714844,
      "activations/layer16_attention_weight_max": 47.72469711303711,
      "activations/layer16_attention_weight_min": -39.05899429321289,
      "activations/layer17_attention_weight_max": 41.315643310546875,
      "activations/layer17_attention_weight_min": -28.197221755981445,
      "activations/layer18_attention_weight_max": 25.94252586364746,
      "activations/layer18_attention_weight_min": -17.358802795410156,
      "activations/layer19_attention_weight_max": 52.61662673950195,
      "activations/layer19_attention_weight_min": -33.133174896240234,
      "activations/layer1_attention_weight_max": 17.62879753112793,
      "activations/layer1_attention_weight_min": -11.011556625366211,
      "activations/layer20_attention_weight_max": 38.06337356567383,
      "activations/layer20_attention_weight_min": -26.073596954345703,
      "activations/layer21_attention_weight_max": 28.470504760742188,
      "activations/layer21_attention_weight_min": -17.345239639282227,
      "activations/layer22_attention_weight_max": 44.711978912353516,
      "activations/layer22_attention_weight_min": -26.191205978393555,
      "activations/layer23_attention_weight_max": 27.933124542236328,
      "activations/layer23_attention_weight_min": -15.999320983886719,
      "activations/layer2_attention_weight_max": 27.555335998535156,
      "activations/layer2_attention_weight_min": -26.923988342285156,
      "activations/layer3_attention_weight_max": 66.61603546142578,
      "activations/layer3_attention_weight_min": -67.90785217285156,
      "activations/layer4_attention_weight_max": 115.58882904052734,
      "activations/layer4_attention_weight_min": -115.23299407958984,
      "activations/layer5_attention_weight_max": 111.21826171875,
      "activations/layer5_attention_weight_min": -103.13038635253906,
      "activations/layer6_attention_weight_max": 59.82700729370117,
      "activations/layer6_attention_weight_min": -55.20420837402344,
      "activations/layer7_attention_weight_max": 56.82647705078125,
      "activations/layer7_attention_weight_min": -57.30111312866211,
      "activations/layer8_attention_weight_max": 49.89533996582031,
      "activations/layer8_attention_weight_min": -43.851375579833984,
      "activations/layer9_attention_weight_max": 68.10768127441406,
      "activations/layer9_attention_weight_min": -57.97248458862305,
      "epoch": 1.62,
      "learning_rate": 0.00014099999999999998,
      "loss": 3.016,
      "step": 27800
    },
    {
      "activations/layer0_attention_weight_max": 15.733015060424805,
      "activations/layer0_attention_weight_min": -11.197731018066406,
      "activations/layer10_attention_weight_max": 49.65757751464844,
      "activations/layer10_attention_weight_min": -43.312442779541016,
      "activations/layer11_attention_weight_max": 42.14083480834961,
      "activations/layer11_attention_weight_min": -34.78517150878906,
      "activations/layer12_attention_weight_max": 36.60640335083008,
      "activations/layer12_attention_weight_min": -29.267606735229492,
      "activations/layer13_attention_weight_max": 50.203147888183594,
      "activations/layer13_attention_weight_min": -37.515769958496094,
      "activations/layer14_attention_weight_max": 73.55221557617188,
      "activations/layer14_attention_weight_min": -68.62303161621094,
      "activations/layer15_attention_weight_max": 53.55389404296875,
      "activations/layer15_attention_weight_min": -42.08970642089844,
      "activations/layer16_attention_weight_max": 54.30440139770508,
      "activations/layer16_attention_weight_min": -39.931724548339844,
      "activations/layer17_attention_weight_max": 44.1883430480957,
      "activations/layer17_attention_weight_min": -25.666927337646484,
      "activations/layer18_attention_weight_max": 26.139108657836914,
      "activations/layer18_attention_weight_min": -15.861478805541992,
      "activations/layer19_attention_weight_max": 50.2324104309082,
      "activations/layer19_attention_weight_min": -33.91801071166992,
      "activations/layer1_attention_weight_max": 16.719894409179688,
      "activations/layer1_attention_weight_min": -11.68224811553955,
      "activations/layer20_attention_weight_max": 39.91283416748047,
      "activations/layer20_attention_weight_min": -25.733396530151367,
      "activations/layer21_attention_weight_max": 26.207290649414062,
      "activations/layer21_attention_weight_min": -17.706573486328125,
      "activations/layer22_attention_weight_max": 43.89394760131836,
      "activations/layer22_attention_weight_min": -25.191335678100586,
      "activations/layer23_attention_weight_max": 24.48236083984375,
      "activations/layer23_attention_weight_min": -16.050621032714844,
      "activations/layer2_attention_weight_max": 28.722349166870117,
      "activations/layer2_attention_weight_min": -27.594213485717773,
      "activations/layer3_attention_weight_max": 65.56218719482422,
      "activations/layer3_attention_weight_min": -69.16401672363281,
      "activations/layer4_attention_weight_max": 115.8476333618164,
      "activations/layer4_attention_weight_min": -112.74771881103516,
      "activations/layer5_attention_weight_max": 115.05523681640625,
      "activations/layer5_attention_weight_min": -101.68795013427734,
      "activations/layer6_attention_weight_max": 61.743473052978516,
      "activations/layer6_attention_weight_min": -52.51424026489258,
      "activations/layer7_attention_weight_max": 63.058677673339844,
      "activations/layer7_attention_weight_min": -58.9233512878418,
      "activations/layer8_attention_weight_max": 58.24099349975586,
      "activations/layer8_attention_weight_min": -46.045997619628906,
      "activations/layer9_attention_weight_max": 72.71929931640625,
      "activations/layer9_attention_weight_min": -64.1059341430664,
      "epoch": 1.62,
      "learning_rate": 0.0001409810606060606,
      "loss": 3.001,
      "step": 27850
    },
    {
      "activations/layer0_attention_weight_max": 17.069683074951172,
      "activations/layer0_attention_weight_min": -11.149351119995117,
      "activations/layer10_attention_weight_max": 58.00831985473633,
      "activations/layer10_attention_weight_min": -46.547088623046875,
      "activations/layer11_attention_weight_max": 48.3567008972168,
      "activations/layer11_attention_weight_min": -39.45652770996094,
      "activations/layer12_attention_weight_max": 53.86955642700195,
      "activations/layer12_attention_weight_min": -31.832290649414062,
      "activations/layer13_attention_weight_max": 68.64909362792969,
      "activations/layer13_attention_weight_min": -40.350013732910156,
      "activations/layer14_attention_weight_max": 109.57404327392578,
      "activations/layer14_attention_weight_min": -73.59233093261719,
      "activations/layer15_attention_weight_max": 58.90452575683594,
      "activations/layer15_attention_weight_min": -45.12358474731445,
      "activations/layer16_attention_weight_max": 56.5472297668457,
      "activations/layer16_attention_weight_min": -38.40837478637695,
      "activations/layer17_attention_weight_max": 45.79354476928711,
      "activations/layer17_attention_weight_min": -32.24068069458008,
      "activations/layer18_attention_weight_max": 31.892107009887695,
      "activations/layer18_attention_weight_min": -16.63975715637207,
      "activations/layer19_attention_weight_max": 66.1518325805664,
      "activations/layer19_attention_weight_min": -31.95096778869629,
      "activations/layer1_attention_weight_max": 17.380815505981445,
      "activations/layer1_attention_weight_min": -11.320436477661133,
      "activations/layer20_attention_weight_max": 62.82727813720703,
      "activations/layer20_attention_weight_min": -28.03826904296875,
      "activations/layer21_attention_weight_max": 36.60748291015625,
      "activations/layer21_attention_weight_min": -19.27887725830078,
      "activations/layer22_attention_weight_max": 51.843910217285156,
      "activations/layer22_attention_weight_min": -25.755226135253906,
      "activations/layer23_attention_weight_max": 25.43878746032715,
      "activations/layer23_attention_weight_min": -14.34559440612793,
      "activations/layer2_attention_weight_max": 30.519025802612305,
      "activations/layer2_attention_weight_min": -30.562917709350586,
      "activations/layer3_attention_weight_max": 67.53546905517578,
      "activations/layer3_attention_weight_min": -69.5765609741211,
      "activations/layer4_attention_weight_max": 125.96124267578125,
      "activations/layer4_attention_weight_min": -111.90616607666016,
      "activations/layer5_attention_weight_max": 125.35576629638672,
      "activations/layer5_attention_weight_min": -104.52355194091797,
      "activations/layer6_attention_weight_max": 73.86083221435547,
      "activations/layer6_attention_weight_min": -57.30978775024414,
      "activations/layer7_attention_weight_max": 78.39037322998047,
      "activations/layer7_attention_weight_min": -62.26180648803711,
      "activations/layer8_attention_weight_max": 54.525962829589844,
      "activations/layer8_attention_weight_min": -50.20246505737305,
      "activations/layer9_attention_weight_max": 92.0570297241211,
      "activations/layer9_attention_weight_min": -69.31661224365234,
      "epoch": 1.62,
      "learning_rate": 0.00014096212121212122,
      "loss": 2.9858,
      "step": 27900
    },
    {
      "activations/layer0_attention_weight_max": 16.365718841552734,
      "activations/layer0_attention_weight_min": -11.302119255065918,
      "activations/layer10_attention_weight_max": 57.91361999511719,
      "activations/layer10_attention_weight_min": -48.2075080871582,
      "activations/layer11_attention_weight_max": 39.54204559326172,
      "activations/layer11_attention_weight_min": -37.87242889404297,
      "activations/layer12_attention_weight_max": 31.87551498413086,
      "activations/layer12_attention_weight_min": -28.92872428894043,
      "activations/layer13_attention_weight_max": 49.26447296142578,
      "activations/layer13_attention_weight_min": -38.06877517700195,
      "activations/layer14_attention_weight_max": 71.02174377441406,
      "activations/layer14_attention_weight_min": -62.00135040283203,
      "activations/layer15_attention_weight_max": 59.9502067565918,
      "activations/layer15_attention_weight_min": -47.37808609008789,
      "activations/layer16_attention_weight_max": 50.58744430541992,
      "activations/layer16_attention_weight_min": -41.22698211669922,
      "activations/layer17_attention_weight_max": 41.17106628417969,
      "activations/layer17_attention_weight_min": -32.09981918334961,
      "activations/layer18_attention_weight_max": 26.520994186401367,
      "activations/layer18_attention_weight_min": -16.76445198059082,
      "activations/layer19_attention_weight_max": 53.362972259521484,
      "activations/layer19_attention_weight_min": -35.53550338745117,
      "activations/layer1_attention_weight_max": 18.181072235107422,
      "activations/layer1_attention_weight_min": -11.642340660095215,
      "activations/layer20_attention_weight_max": 39.75361633300781,
      "activations/layer20_attention_weight_min": -27.705284118652344,
      "activations/layer21_attention_weight_max": 33.542823791503906,
      "activations/layer21_attention_weight_min": -22.442874908447266,
      "activations/layer22_attention_weight_max": 45.82888412475586,
      "activations/layer22_attention_weight_min": -27.112865447998047,
      "activations/layer23_attention_weight_max": 26.789653778076172,
      "activations/layer23_attention_weight_min": -16.499868392944336,
      "activations/layer2_attention_weight_max": 27.694639205932617,
      "activations/layer2_attention_weight_min": -26.87740707397461,
      "activations/layer3_attention_weight_max": 66.00164031982422,
      "activations/layer3_attention_weight_min": -67.03234100341797,
      "activations/layer4_attention_weight_max": 116.4032211303711,
      "activations/layer4_attention_weight_min": -112.77862548828125,
      "activations/layer5_attention_weight_max": 112.021484375,
      "activations/layer5_attention_weight_min": -103.61798095703125,
      "activations/layer6_attention_weight_max": 60.91178894042969,
      "activations/layer6_attention_weight_min": -57.17333221435547,
      "activations/layer7_attention_weight_max": 60.24262237548828,
      "activations/layer7_attention_weight_min": -64.4027099609375,
      "activations/layer8_attention_weight_max": 49.93214797973633,
      "activations/layer8_attention_weight_min": -50.02136993408203,
      "activations/layer9_attention_weight_max": 69.00932312011719,
      "activations/layer9_attention_weight_min": -62.5506477355957,
      "epoch": 1.62,
      "learning_rate": 0.0001409431818181818,
      "loss": 2.985,
      "step": 27950
    },
    {
      "activations/layer0_attention_weight_max": 16.837427139282227,
      "activations/layer0_attention_weight_min": -11.510102272033691,
      "activations/layer10_attention_weight_max": 54.574214935302734,
      "activations/layer10_attention_weight_min": -47.30363845825195,
      "activations/layer11_attention_weight_max": 44.16376876831055,
      "activations/layer11_attention_weight_min": -36.93023681640625,
      "activations/layer12_attention_weight_max": 54.89946365356445,
      "activations/layer12_attention_weight_min": -36.2388916015625,
      "activations/layer13_attention_weight_max": 56.65888214111328,
      "activations/layer13_attention_weight_min": -43.3061637878418,
      "activations/layer14_attention_weight_max": 85.38330841064453,
      "activations/layer14_attention_weight_min": -58.67877960205078,
      "activations/layer15_attention_weight_max": 59.80488204956055,
      "activations/layer15_attention_weight_min": -38.38761901855469,
      "activations/layer16_attention_weight_max": 57.94310760498047,
      "activations/layer16_attention_weight_min": -36.40143585205078,
      "activations/layer17_attention_weight_max": 45.210670471191406,
      "activations/layer17_attention_weight_min": -31.224721908569336,
      "activations/layer18_attention_weight_max": 29.0672607421875,
      "activations/layer18_attention_weight_min": -16.0485897064209,
      "activations/layer19_attention_weight_max": 54.92251205444336,
      "activations/layer19_attention_weight_min": -32.835060119628906,
      "activations/layer1_attention_weight_max": 17.331283569335938,
      "activations/layer1_attention_weight_min": -11.011731147766113,
      "activations/layer20_attention_weight_max": 41.15762710571289,
      "activations/layer20_attention_weight_min": -25.257022857666016,
      "activations/layer21_attention_weight_max": 29.894821166992188,
      "activations/layer21_attention_weight_min": -17.56182289123535,
      "activations/layer22_attention_weight_max": 42.83531951904297,
      "activations/layer22_attention_weight_min": -26.589738845825195,
      "activations/layer23_attention_weight_max": 25.908130645751953,
      "activations/layer23_attention_weight_min": -19.573244094848633,
      "activations/layer2_attention_weight_max": 27.431913375854492,
      "activations/layer2_attention_weight_min": -28.216726303100586,
      "activations/layer3_attention_weight_max": 64.60945892333984,
      "activations/layer3_attention_weight_min": -69.57941436767578,
      "activations/layer4_attention_weight_max": 118.973876953125,
      "activations/layer4_attention_weight_min": -111.87870025634766,
      "activations/layer5_attention_weight_max": 110.40718078613281,
      "activations/layer5_attention_weight_min": -96.67294311523438,
      "activations/layer6_attention_weight_max": 61.33200454711914,
      "activations/layer6_attention_weight_min": -52.52621841430664,
      "activations/layer7_attention_weight_max": 64.44347381591797,
      "activations/layer7_attention_weight_min": -59.22166442871094,
      "activations/layer8_attention_weight_max": 50.24854278564453,
      "activations/layer8_attention_weight_min": -44.005680084228516,
      "activations/layer9_attention_weight_max": 68.35734558105469,
      "activations/layer9_attention_weight_min": -58.655242919921875,
      "epoch": 1.63,
      "learning_rate": 0.00014092424242424242,
      "loss": 2.9933,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_loss": 2.94921875,
      "eval_runtime": 8.5138,
      "eval_samples_per_second": 504.356,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_openwebtext_loss": 2.94921875,
      "eval_openwebtext_ppl": 19.091033031037714,
      "eval_openwebtext_runtime": 8.5138,
      "eval_openwebtext_samples_per_second": 504.356,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_wikitext_loss": 3.224609375,
      "eval_wikitext_ppl": 25.1437504597293,
      "eval_wikitext_runtime": 1.9421,
      "eval_wikitext_samples_per_second": 234.799,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_lambada_loss": 3.22265625,
      "eval_lambada_ppl": 25.094689498803355,
      "eval_lambada_runtime": 9.5471,
      "eval_lambada_samples_per_second": 509.998,
      "step": 28000
    },
    {
      "activations/layer0_attention_weight_max": 16.064773559570312,
      "activations/layer0_attention_weight_min": -10.969486236572266,
      "activations/layer10_attention_weight_max": 57.59208297729492,
      "activations/layer10_attention_weight_min": -51.5024299621582,
      "activations/layer11_attention_weight_max": 41.822349548339844,
      "activations/layer11_attention_weight_min": -37.06768798828125,
      "activations/layer12_attention_weight_max": 41.03614044189453,
      "activations/layer12_attention_weight_min": -30.694320678710938,
      "activations/layer13_attention_weight_max": 50.50800323486328,
      "activations/layer13_attention_weight_min": -40.36833572387695,
      "activations/layer14_attention_weight_max": 83.17979431152344,
      "activations/layer14_attention_weight_min": -67.68172454833984,
      "activations/layer15_attention_weight_max": 54.5190315246582,
      "activations/layer15_attention_weight_min": -39.60728073120117,
      "activations/layer16_attention_weight_max": 56.4316520690918,
      "activations/layer16_attention_weight_min": -38.304908752441406,
      "activations/layer17_attention_weight_max": 52.49298858642578,
      "activations/layer17_attention_weight_min": -30.208953857421875,
      "activations/layer18_attention_weight_max": 26.09908103942871,
      "activations/layer18_attention_weight_min": -15.560685157775879,
      "activations/layer19_attention_weight_max": 60.919654846191406,
      "activations/layer19_attention_weight_min": -33.2808952331543,
      "activations/layer1_attention_weight_max": 17.073490142822266,
      "activations/layer1_attention_weight_min": -11.0518159866333,
      "activations/layer20_attention_weight_max": 43.23889923095703,
      "activations/layer20_attention_weight_min": -25.83123779296875,
      "activations/layer21_attention_weight_max": 27.34209442138672,
      "activations/layer21_attention_weight_min": -18.33024787902832,
      "activations/layer22_attention_weight_max": 48.29905700683594,
      "activations/layer22_attention_weight_min": -25.298547744750977,
      "activations/layer23_attention_weight_max": 24.445621490478516,
      "activations/layer23_attention_weight_min": -15.382368087768555,
      "activations/layer2_attention_weight_max": 28.122394561767578,
      "activations/layer2_attention_weight_min": -27.262619018554688,
      "activations/layer3_attention_weight_max": 68.63553619384766,
      "activations/layer3_attention_weight_min": -69.1323471069336,
      "activations/layer4_attention_weight_max": 126.7040786743164,
      "activations/layer4_attention_weight_min": -117.56290435791016,
      "activations/layer5_attention_weight_max": 120.76742553710938,
      "activations/layer5_attention_weight_min": -102.22958374023438,
      "activations/layer6_attention_weight_max": 63.30956268310547,
      "activations/layer6_attention_weight_min": -53.49130630493164,
      "activations/layer7_attention_weight_max": 60.56694412231445,
      "activations/layer7_attention_weight_min": -59.808284759521484,
      "activations/layer8_attention_weight_max": 52.09990310668945,
      "activations/layer8_attention_weight_min": -45.742435455322266,
      "activations/layer9_attention_weight_max": 69.05345916748047,
      "activations/layer9_attention_weight_min": -65.94392395019531,
      "epoch": 1.63,
      "learning_rate": 0.00014090530303030303,
      "loss": 3.0015,
      "step": 28050
    },
    {
      "activations/layer0_attention_weight_max": 16.344863891601562,
      "activations/layer0_attention_weight_min": -11.713489532470703,
      "activations/layer10_attention_weight_max": 49.35639190673828,
      "activations/layer10_attention_weight_min": -45.27924728393555,
      "activations/layer11_attention_weight_max": 39.86968994140625,
      "activations/layer11_attention_weight_min": -35.3067626953125,
      "activations/layer12_attention_weight_max": 33.60630416870117,
      "activations/layer12_attention_weight_min": -31.697586059570312,
      "activations/layer13_attention_weight_max": 53.55461883544922,
      "activations/layer13_attention_weight_min": -40.72593307495117,
      "activations/layer14_attention_weight_max": 75.31564331054688,
      "activations/layer14_attention_weight_min": -59.83692169189453,
      "activations/layer15_attention_weight_max": 48.75831985473633,
      "activations/layer15_attention_weight_min": -39.476139068603516,
      "activations/layer16_attention_weight_max": 52.75989532470703,
      "activations/layer16_attention_weight_min": -39.188255310058594,
      "activations/layer17_attention_weight_max": 45.17863845825195,
      "activations/layer17_attention_weight_min": -29.318885803222656,
      "activations/layer18_attention_weight_max": 24.497623443603516,
      "activations/layer18_attention_weight_min": -16.681570053100586,
      "activations/layer19_attention_weight_max": 51.086910247802734,
      "activations/layer19_attention_weight_min": -34.991512298583984,
      "activations/layer1_attention_weight_max": 16.54547119140625,
      "activations/layer1_attention_weight_min": -11.110506057739258,
      "activations/layer20_attention_weight_max": 36.532371520996094,
      "activations/layer20_attention_weight_min": -26.053544998168945,
      "activations/layer21_attention_weight_max": 33.21870803833008,
      "activations/layer21_attention_weight_min": -17.460596084594727,
      "activations/layer22_attention_weight_max": 44.27553939819336,
      "activations/layer22_attention_weight_min": -23.61197280883789,
      "activations/layer23_attention_weight_max": 26.679000854492188,
      "activations/layer23_attention_weight_min": -15.547934532165527,
      "activations/layer2_attention_weight_max": 28.485424041748047,
      "activations/layer2_attention_weight_min": -25.873512268066406,
      "activations/layer3_attention_weight_max": 67.16414642333984,
      "activations/layer3_attention_weight_min": -70.75752258300781,
      "activations/layer4_attention_weight_max": 130.28074645996094,
      "activations/layer4_attention_weight_min": -114.2415542602539,
      "activations/layer5_attention_weight_max": 125.510009765625,
      "activations/layer5_attention_weight_min": -102.42083740234375,
      "activations/layer6_attention_weight_max": 68.23412322998047,
      "activations/layer6_attention_weight_min": -55.42586135864258,
      "activations/layer7_attention_weight_max": 58.91582489013672,
      "activations/layer7_attention_weight_min": -59.51858139038086,
      "activations/layer8_attention_weight_max": 53.13697052001953,
      "activations/layer8_attention_weight_min": -48.969425201416016,
      "activations/layer9_attention_weight_max": 75.0329360961914,
      "activations/layer9_attention_weight_min": -59.40279769897461,
      "epoch": 1.63,
      "learning_rate": 0.00014088636363636362,
      "loss": 3.0004,
      "step": 28100
    },
    {
      "activations/layer0_attention_weight_max": 16.791959762573242,
      "activations/layer0_attention_weight_min": -11.21567153930664,
      "activations/layer10_attention_weight_max": 54.23318862915039,
      "activations/layer10_attention_weight_min": -40.24687576293945,
      "activations/layer11_attention_weight_max": 43.3076171875,
      "activations/layer11_attention_weight_min": -34.99518585205078,
      "activations/layer12_attention_weight_max": 47.09907531738281,
      "activations/layer12_attention_weight_min": -28.27151870727539,
      "activations/layer13_attention_weight_max": 62.98204803466797,
      "activations/layer13_attention_weight_min": -41.65642547607422,
      "activations/layer14_attention_weight_max": 81.90131378173828,
      "activations/layer14_attention_weight_min": -52.842506408691406,
      "activations/layer15_attention_weight_max": 64.53844451904297,
      "activations/layer15_attention_weight_min": -38.72542953491211,
      "activations/layer16_attention_weight_max": 60.41121292114258,
      "activations/layer16_attention_weight_min": -41.18578338623047,
      "activations/layer17_attention_weight_max": 50.50269317626953,
      "activations/layer17_attention_weight_min": -30.220638275146484,
      "activations/layer18_attention_weight_max": 35.3076286315918,
      "activations/layer18_attention_weight_min": -17.47659683227539,
      "activations/layer19_attention_weight_max": 65.84974670410156,
      "activations/layer19_attention_weight_min": -32.87824630737305,
      "activations/layer1_attention_weight_max": 18.13088035583496,
      "activations/layer1_attention_weight_min": -11.063807487487793,
      "activations/layer20_attention_weight_max": 53.704288482666016,
      "activations/layer20_attention_weight_min": -24.709217071533203,
      "activations/layer21_attention_weight_max": 30.52048683166504,
      "activations/layer21_attention_weight_min": -18.350488662719727,
      "activations/layer22_attention_weight_max": 53.45835494995117,
      "activations/layer22_attention_weight_min": -25.49082374572754,
      "activations/layer23_attention_weight_max": 32.008480072021484,
      "activations/layer23_attention_weight_min": -19.047565460205078,
      "activations/layer2_attention_weight_max": 27.56917381286621,
      "activations/layer2_attention_weight_min": -25.863571166992188,
      "activations/layer3_attention_weight_max": 65.51851654052734,
      "activations/layer3_attention_weight_min": -66.41424560546875,
      "activations/layer4_attention_weight_max": 121.69274139404297,
      "activations/layer4_attention_weight_min": -111.61956024169922,
      "activations/layer5_attention_weight_max": 115.47314453125,
      "activations/layer5_attention_weight_min": -94.9889144897461,
      "activations/layer6_attention_weight_max": 62.72209930419922,
      "activations/layer6_attention_weight_min": -53.179649353027344,
      "activations/layer7_attention_weight_max": 63.005027770996094,
      "activations/layer7_attention_weight_min": -61.76165008544922,
      "activations/layer8_attention_weight_max": 52.10971450805664,
      "activations/layer8_attention_weight_min": -48.67815399169922,
      "activations/layer9_attention_weight_max": 69.23970794677734,
      "activations/layer9_attention_weight_min": -59.4562873840332,
      "epoch": 1.64,
      "learning_rate": 0.00014086742424242424,
      "loss": 3.0116,
      "step": 28150
    },
    {
      "activations/layer0_attention_weight_max": 15.311018943786621,
      "activations/layer0_attention_weight_min": -11.746685981750488,
      "activations/layer10_attention_weight_max": 57.15620040893555,
      "activations/layer10_attention_weight_min": -51.70276641845703,
      "activations/layer11_attention_weight_max": 45.988075256347656,
      "activations/layer11_attention_weight_min": -44.005523681640625,
      "activations/layer12_attention_weight_max": 44.51875305175781,
      "activations/layer12_attention_weight_min": -30.654993057250977,
      "activations/layer13_attention_weight_max": 45.63430404663086,
      "activations/layer13_attention_weight_min": -38.49721908569336,
      "activations/layer14_attention_weight_max": 78.08883666992188,
      "activations/layer14_attention_weight_min": -59.31029510498047,
      "activations/layer15_attention_weight_max": 64.6867446899414,
      "activations/layer15_attention_weight_min": -44.417320251464844,
      "activations/layer16_attention_weight_max": 49.940025329589844,
      "activations/layer16_attention_weight_min": -35.84952926635742,
      "activations/layer17_attention_weight_max": 43.23311233520508,
      "activations/layer17_attention_weight_min": -30.627126693725586,
      "activations/layer18_attention_weight_max": 25.358732223510742,
      "activations/layer18_attention_weight_min": -15.533846855163574,
      "activations/layer19_attention_weight_max": 47.42467498779297,
      "activations/layer19_attention_weight_min": -31.952835083007812,
      "activations/layer1_attention_weight_max": 17.348909378051758,
      "activations/layer1_attention_weight_min": -12.329601287841797,
      "activations/layer20_attention_weight_max": 35.42350387573242,
      "activations/layer20_attention_weight_min": -23.70171356201172,
      "activations/layer21_attention_weight_max": 23.346725463867188,
      "activations/layer21_attention_weight_min": -17.636150360107422,
      "activations/layer22_attention_weight_max": 37.02479553222656,
      "activations/layer22_attention_weight_min": -26.644643783569336,
      "activations/layer23_attention_weight_max": 24.302371978759766,
      "activations/layer23_attention_weight_min": -15.17371654510498,
      "activations/layer2_attention_weight_max": 28.269134521484375,
      "activations/layer2_attention_weight_min": -27.060935974121094,
      "activations/layer3_attention_weight_max": 67.06233215332031,
      "activations/layer3_attention_weight_min": -68.06803131103516,
      "activations/layer4_attention_weight_max": 119.90653228759766,
      "activations/layer4_attention_weight_min": -115.62322998046875,
      "activations/layer5_attention_weight_max": 123.60206604003906,
      "activations/layer5_attention_weight_min": -104.07405090332031,
      "activations/layer6_attention_weight_max": 69.10716247558594,
      "activations/layer6_attention_weight_min": -56.46485900878906,
      "activations/layer7_attention_weight_max": 63.63105392456055,
      "activations/layer7_attention_weight_min": -63.334442138671875,
      "activations/layer8_attention_weight_max": 58.21756362915039,
      "activations/layer8_attention_weight_min": -57.0279426574707,
      "activations/layer9_attention_weight_max": 79.66545104980469,
      "activations/layer9_attention_weight_min": -68.68553924560547,
      "epoch": 1.64,
      "learning_rate": 0.00014084848484848485,
      "loss": 3.0049,
      "step": 28200
    },
    {
      "activations/layer0_attention_weight_max": 16.835847854614258,
      "activations/layer0_attention_weight_min": -11.363116264343262,
      "activations/layer10_attention_weight_max": 48.36037063598633,
      "activations/layer10_attention_weight_min": -42.37738037109375,
      "activations/layer11_attention_weight_max": 39.25288391113281,
      "activations/layer11_attention_weight_min": -34.78295135498047,
      "activations/layer12_attention_weight_max": 55.863128662109375,
      "activations/layer12_attention_weight_min": -42.58683395385742,
      "activations/layer13_attention_weight_max": 51.08302688598633,
      "activations/layer13_attention_weight_min": -40.894317626953125,
      "activations/layer14_attention_weight_max": 70.10601806640625,
      "activations/layer14_attention_weight_min": -54.40938186645508,
      "activations/layer15_attention_weight_max": 48.5953254699707,
      "activations/layer15_attention_weight_min": -35.91365432739258,
      "activations/layer16_attention_weight_max": 47.99313735961914,
      "activations/layer16_attention_weight_min": -40.82066345214844,
      "activations/layer17_attention_weight_max": 41.349422454833984,
      "activations/layer17_attention_weight_min": -27.859752655029297,
      "activations/layer18_attention_weight_max": 24.53292465209961,
      "activations/layer18_attention_weight_min": -16.733585357666016,
      "activations/layer19_attention_weight_max": 44.54283142089844,
      "activations/layer19_attention_weight_min": -33.60036087036133,
      "activations/layer1_attention_weight_max": 16.603092193603516,
      "activations/layer1_attention_weight_min": -12.265748023986816,
      "activations/layer20_attention_weight_max": 34.978702545166016,
      "activations/layer20_attention_weight_min": -27.576963424682617,
      "activations/layer21_attention_weight_max": 32.363136291503906,
      "activations/layer21_attention_weight_min": -23.10021209716797,
      "activations/layer22_attention_weight_max": 39.778167724609375,
      "activations/layer22_attention_weight_min": -27.26264762878418,
      "activations/layer23_attention_weight_max": 23.926008224487305,
      "activations/layer23_attention_weight_min": -15.539061546325684,
      "activations/layer2_attention_weight_max": 27.613142013549805,
      "activations/layer2_attention_weight_min": -25.833066940307617,
      "activations/layer3_attention_weight_max": 66.26249694824219,
      "activations/layer3_attention_weight_min": -66.65937805175781,
      "activations/layer4_attention_weight_max": 123.146484375,
      "activations/layer4_attention_weight_min": -111.91853332519531,
      "activations/layer5_attention_weight_max": 121.32286071777344,
      "activations/layer5_attention_weight_min": -97.99284362792969,
      "activations/layer6_attention_weight_max": 65.8063735961914,
      "activations/layer6_attention_weight_min": -52.82346725463867,
      "activations/layer7_attention_weight_max": 62.15999221801758,
      "activations/layer7_attention_weight_min": -60.00384521484375,
      "activations/layer8_attention_weight_max": 51.73533630371094,
      "activations/layer8_attention_weight_min": -44.664276123046875,
      "activations/layer9_attention_weight_max": 71.26484680175781,
      "activations/layer9_attention_weight_min": -58.979095458984375,
      "epoch": 1.64,
      "learning_rate": 0.00014082954545454544,
      "loss": 3.0083,
      "step": 28250
    },
    {
      "activations/layer0_attention_weight_max": 16.085290908813477,
      "activations/layer0_attention_weight_min": -11.115300178527832,
      "activations/layer10_attention_weight_max": 51.69550323486328,
      "activations/layer10_attention_weight_min": -44.84048080444336,
      "activations/layer11_attention_weight_max": 42.445674896240234,
      "activations/layer11_attention_weight_min": -33.817466735839844,
      "activations/layer12_attention_weight_max": 33.29265213012695,
      "activations/layer12_attention_weight_min": -30.00818634033203,
      "activations/layer13_attention_weight_max": 47.33478927612305,
      "activations/layer13_attention_weight_min": -40.28348922729492,
      "activations/layer14_attention_weight_max": 67.16807556152344,
      "activations/layer14_attention_weight_min": -56.22652816772461,
      "activations/layer15_attention_weight_max": 47.829044342041016,
      "activations/layer15_attention_weight_min": -41.07740783691406,
      "activations/layer16_attention_weight_max": 52.04924392700195,
      "activations/layer16_attention_weight_min": -40.66072082519531,
      "activations/layer17_attention_weight_max": 44.403438568115234,
      "activations/layer17_attention_weight_min": -30.68988800048828,
      "activations/layer18_attention_weight_max": 27.57479476928711,
      "activations/layer18_attention_weight_min": -16.750572204589844,
      "activations/layer19_attention_weight_max": 50.129638671875,
      "activations/layer19_attention_weight_min": -33.18789291381836,
      "activations/layer1_attention_weight_max": 16.839656829833984,
      "activations/layer1_attention_weight_min": -11.620556831359863,
      "activations/layer20_attention_weight_max": 36.339195251464844,
      "activations/layer20_attention_weight_min": -27.273937225341797,
      "activations/layer21_attention_weight_max": 22.605125427246094,
      "activations/layer21_attention_weight_min": -16.389894485473633,
      "activations/layer22_attention_weight_max": 39.43717575073242,
      "activations/layer22_attention_weight_min": -22.661502838134766,
      "activations/layer23_attention_weight_max": 24.102523803710938,
      "activations/layer23_attention_weight_min": -15.108919143676758,
      "activations/layer2_attention_weight_max": 27.20722007751465,
      "activations/layer2_attention_weight_min": -27.499082565307617,
      "activations/layer3_attention_weight_max": 69.5490951538086,
      "activations/layer3_attention_weight_min": -72.8906021118164,
      "activations/layer4_attention_weight_max": 122.67835998535156,
      "activations/layer4_attention_weight_min": -111.53631591796875,
      "activations/layer5_attention_weight_max": 113.13185119628906,
      "activations/layer5_attention_weight_min": -101.920166015625,
      "activations/layer6_attention_weight_max": 63.82782745361328,
      "activations/layer6_attention_weight_min": -52.64592361450195,
      "activations/layer7_attention_weight_max": 61.1775016784668,
      "activations/layer7_attention_weight_min": -61.681053161621094,
      "activations/layer8_attention_weight_max": 55.36757278442383,
      "activations/layer8_attention_weight_min": -44.428489685058594,
      "activations/layer9_attention_weight_max": 80.67211151123047,
      "activations/layer9_attention_weight_min": -63.08502960205078,
      "epoch": 1.64,
      "learning_rate": 0.00014081060606060605,
      "loss": 3.0007,
      "step": 28300
    },
    {
      "activations/layer0_attention_weight_max": 16.775453567504883,
      "activations/layer0_attention_weight_min": -11.196674346923828,
      "activations/layer10_attention_weight_max": 53.81047821044922,
      "activations/layer10_attention_weight_min": -49.5801887512207,
      "activations/layer11_attention_weight_max": 41.854434967041016,
      "activations/layer11_attention_weight_min": -36.68864440917969,
      "activations/layer12_attention_weight_max": 37.72724151611328,
      "activations/layer12_attention_weight_min": -26.412263870239258,
      "activations/layer13_attention_weight_max": 54.41958999633789,
      "activations/layer13_attention_weight_min": -37.725154876708984,
      "activations/layer14_attention_weight_max": 81.23455047607422,
      "activations/layer14_attention_weight_min": -59.343997955322266,
      "activations/layer15_attention_weight_max": 51.4567985534668,
      "activations/layer15_attention_weight_min": -37.05035400390625,
      "activations/layer16_attention_weight_max": 57.17416763305664,
      "activations/layer16_attention_weight_min": -37.379783630371094,
      "activations/layer17_attention_weight_max": 46.45991897583008,
      "activations/layer17_attention_weight_min": -29.29638671875,
      "activations/layer18_attention_weight_max": 24.32352638244629,
      "activations/layer18_attention_weight_min": -16.627464294433594,
      "activations/layer19_attention_weight_max": 52.34128952026367,
      "activations/layer19_attention_weight_min": -33.540042877197266,
      "activations/layer1_attention_weight_max": 16.629728317260742,
      "activations/layer1_attention_weight_min": -11.951948165893555,
      "activations/layer20_attention_weight_max": 37.92929458618164,
      "activations/layer20_attention_weight_min": -24.78451156616211,
      "activations/layer21_attention_weight_max": 32.244808197021484,
      "activations/layer21_attention_weight_min": -17.527748107910156,
      "activations/layer22_attention_weight_max": 42.67627716064453,
      "activations/layer22_attention_weight_min": -27.25529670715332,
      "activations/layer23_attention_weight_max": 25.44675064086914,
      "activations/layer23_attention_weight_min": -15.118255615234375,
      "activations/layer2_attention_weight_max": 29.53388786315918,
      "activations/layer2_attention_weight_min": -27.234554290771484,
      "activations/layer3_attention_weight_max": 69.29180145263672,
      "activations/layer3_attention_weight_min": -70.51673126220703,
      "activations/layer4_attention_weight_max": 122.5064468383789,
      "activations/layer4_attention_weight_min": -115.26521301269531,
      "activations/layer5_attention_weight_max": 121.48784637451172,
      "activations/layer5_attention_weight_min": -107.5655746459961,
      "activations/layer6_attention_weight_max": 66.05465698242188,
      "activations/layer6_attention_weight_min": -59.79141616821289,
      "activations/layer7_attention_weight_max": 61.83949279785156,
      "activations/layer7_attention_weight_min": -63.06918716430664,
      "activations/layer8_attention_weight_max": 53.132102966308594,
      "activations/layer8_attention_weight_min": -44.4645881652832,
      "activations/layer9_attention_weight_max": 73.68790435791016,
      "activations/layer9_attention_weight_min": -65.86309814453125,
      "epoch": 1.65,
      "learning_rate": 0.00014079166666666664,
      "loss": 3.0025,
      "step": 28350
    },
    {
      "activations/layer0_attention_weight_max": 15.809709548950195,
      "activations/layer0_attention_weight_min": -11.026721954345703,
      "activations/layer10_attention_weight_max": 54.25632858276367,
      "activations/layer10_attention_weight_min": -45.0395393371582,
      "activations/layer11_attention_weight_max": 40.497894287109375,
      "activations/layer11_attention_weight_min": -34.26128387451172,
      "activations/layer12_attention_weight_max": 66.8757553100586,
      "activations/layer12_attention_weight_min": -48.41066360473633,
      "activations/layer13_attention_weight_max": 63.72224807739258,
      "activations/layer13_attention_weight_min": -46.14569854736328,
      "activations/layer14_attention_weight_max": 80.93672943115234,
      "activations/layer14_attention_weight_min": -57.643775939941406,
      "activations/layer15_attention_weight_max": 62.80840301513672,
      "activations/layer15_attention_weight_min": -41.77766418457031,
      "activations/layer16_attention_weight_max": 58.32514572143555,
      "activations/layer16_attention_weight_min": -38.951507568359375,
      "activations/layer17_attention_weight_max": 53.62477493286133,
      "activations/layer17_attention_weight_min": -31.797950744628906,
      "activations/layer18_attention_weight_max": 29.999460220336914,
      "activations/layer18_attention_weight_min": -18.028799057006836,
      "activations/layer19_attention_weight_max": 59.93359375,
      "activations/layer19_attention_weight_min": -38.69093322753906,
      "activations/layer1_attention_weight_max": 17.784095764160156,
      "activations/layer1_attention_weight_min": -12.514571189880371,
      "activations/layer20_attention_weight_max": 46.57188415527344,
      "activations/layer20_attention_weight_min": -24.576499938964844,
      "activations/layer21_attention_weight_max": 33.47853469848633,
      "activations/layer21_attention_weight_min": -18.489517211914062,
      "activations/layer22_attention_weight_max": 45.5028190612793,
      "activations/layer22_attention_weight_min": -25.69777488708496,
      "activations/layer23_attention_weight_max": 29.058856964111328,
      "activations/layer23_attention_weight_min": -18.227947235107422,
      "activations/layer2_attention_weight_max": 30.290769577026367,
      "activations/layer2_attention_weight_min": -29.362747192382812,
      "activations/layer3_attention_weight_max": 69.2251968383789,
      "activations/layer3_attention_weight_min": -69.2364730834961,
      "activations/layer4_attention_weight_max": 123.36856842041016,
      "activations/layer4_attention_weight_min": -109.35145568847656,
      "activations/layer5_attention_weight_max": 123.97673797607422,
      "activations/layer5_attention_weight_min": -103.60311889648438,
      "activations/layer6_attention_weight_max": 64.37844848632812,
      "activations/layer6_attention_weight_min": -52.44736862182617,
      "activations/layer7_attention_weight_max": 66.33428955078125,
      "activations/layer7_attention_weight_min": -60.04335021972656,
      "activations/layer8_attention_weight_max": 55.22273635864258,
      "activations/layer8_attention_weight_min": -46.53434753417969,
      "activations/layer9_attention_weight_max": 82.08021545410156,
      "activations/layer9_attention_weight_min": -65.69477844238281,
      "epoch": 1.65,
      "learning_rate": 0.00014077272727272726,
      "loss": 3.0005,
      "step": 28400
    },
    {
      "activations/layer0_attention_weight_max": 16.650358200073242,
      "activations/layer0_attention_weight_min": -11.548299789428711,
      "activations/layer10_attention_weight_max": 58.15210723876953,
      "activations/layer10_attention_weight_min": -51.50154495239258,
      "activations/layer11_attention_weight_max": 42.549835205078125,
      "activations/layer11_attention_weight_min": -36.312530517578125,
      "activations/layer12_attention_weight_max": 37.709041595458984,
      "activations/layer12_attention_weight_min": -31.767620086669922,
      "activations/layer13_attention_weight_max": 42.91252517700195,
      "activations/layer13_attention_weight_min": -37.54402160644531,
      "activations/layer14_attention_weight_max": 72.92929077148438,
      "activations/layer14_attention_weight_min": -63.49312210083008,
      "activations/layer15_attention_weight_max": 51.67635726928711,
      "activations/layer15_attention_weight_min": -38.11601638793945,
      "activations/layer16_attention_weight_max": 44.71540832519531,
      "activations/layer16_attention_weight_min": -35.31427764892578,
      "activations/layer17_attention_weight_max": 37.26832962036133,
      "activations/layer17_attention_weight_min": -28.401308059692383,
      "activations/layer18_attention_weight_max": 26.44108772277832,
      "activations/layer18_attention_weight_min": -18.40375328063965,
      "activations/layer19_attention_weight_max": 47.89777374267578,
      "activations/layer19_attention_weight_min": -31.45127296447754,
      "activations/layer1_attention_weight_max": 17.188213348388672,
      "activations/layer1_attention_weight_min": -13.264925956726074,
      "activations/layer20_attention_weight_max": 37.23887252807617,
      "activations/layer20_attention_weight_min": -26.587501525878906,
      "activations/layer21_attention_weight_max": 30.131309509277344,
      "activations/layer21_attention_weight_min": -18.881702423095703,
      "activations/layer22_attention_weight_max": 40.82196044921875,
      "activations/layer22_attention_weight_min": -26.83224105834961,
      "activations/layer23_attention_weight_max": 22.558467864990234,
      "activations/layer23_attention_weight_min": -13.650541305541992,
      "activations/layer2_attention_weight_max": 28.637413024902344,
      "activations/layer2_attention_weight_min": -28.453359603881836,
      "activations/layer3_attention_weight_max": 70.52086639404297,
      "activations/layer3_attention_weight_min": -69.68144226074219,
      "activations/layer4_attention_weight_max": 122.85066986083984,
      "activations/layer4_attention_weight_min": -113.56328582763672,
      "activations/layer5_attention_weight_max": 115.84706115722656,
      "activations/layer5_attention_weight_min": -104.44230651855469,
      "activations/layer6_attention_weight_max": 60.167869567871094,
      "activations/layer6_attention_weight_min": -54.6751594543457,
      "activations/layer7_attention_weight_max": 59.685020446777344,
      "activations/layer7_attention_weight_min": -60.71590805053711,
      "activations/layer8_attention_weight_max": 48.83991241455078,
      "activations/layer8_attention_weight_min": -47.11442565917969,
      "activations/layer9_attention_weight_max": 66.95807647705078,
      "activations/layer9_attention_weight_min": -69.68050384521484,
      "epoch": 1.65,
      "learning_rate": 0.00014075378787878787,
      "loss": 3.0154,
      "step": 28450
    },
    {
      "activations/layer0_attention_weight_max": 16.747676849365234,
      "activations/layer0_attention_weight_min": -11.517663955688477,
      "activations/layer10_attention_weight_max": 50.073604583740234,
      "activations/layer10_attention_weight_min": -42.03544616699219,
      "activations/layer11_attention_weight_max": 47.66849899291992,
      "activations/layer11_attention_weight_min": -39.386844635009766,
      "activations/layer12_attention_weight_max": 44.4163818359375,
      "activations/layer12_attention_weight_min": -27.69475555419922,
      "activations/layer13_attention_weight_max": 69.78145599365234,
      "activations/layer13_attention_weight_min": -47.35271072387695,
      "activations/layer14_attention_weight_max": 91.843994140625,
      "activations/layer14_attention_weight_min": -66.5857925415039,
      "activations/layer15_attention_weight_max": 63.61500549316406,
      "activations/layer15_attention_weight_min": -39.052799224853516,
      "activations/layer16_attention_weight_max": 57.064510345458984,
      "activations/layer16_attention_weight_min": -42.675743103027344,
      "activations/layer17_attention_weight_max": 52.2686653137207,
      "activations/layer17_attention_weight_min": -28.13553810119629,
      "activations/layer18_attention_weight_max": 33.439208984375,
      "activations/layer18_attention_weight_min": -21.734325408935547,
      "activations/layer19_attention_weight_max": 56.618751525878906,
      "activations/layer19_attention_weight_min": -35.027278900146484,
      "activations/layer1_attention_weight_max": 17.596731185913086,
      "activations/layer1_attention_weight_min": -11.64825439453125,
      "activations/layer20_attention_weight_max": 47.474910736083984,
      "activations/layer20_attention_weight_min": -28.213943481445312,
      "activations/layer21_attention_weight_max": 26.226699829101562,
      "activations/layer21_attention_weight_min": -16.493391036987305,
      "activations/layer22_attention_weight_max": 49.37311553955078,
      "activations/layer22_attention_weight_min": -26.60064697265625,
      "activations/layer23_attention_weight_max": 25.184492111206055,
      "activations/layer23_attention_weight_min": -16.009933471679688,
      "activations/layer2_attention_weight_max": 28.643735885620117,
      "activations/layer2_attention_weight_min": -27.235084533691406,
      "activations/layer3_attention_weight_max": 66.14134216308594,
      "activations/layer3_attention_weight_min": -69.58354187011719,
      "activations/layer4_attention_weight_max": 121.4793701171875,
      "activations/layer4_attention_weight_min": -117.20763397216797,
      "activations/layer5_attention_weight_max": 111.29777526855469,
      "activations/layer5_attention_weight_min": -100.81394958496094,
      "activations/layer6_attention_weight_max": 60.32274627685547,
      "activations/layer6_attention_weight_min": -53.538883209228516,
      "activations/layer7_attention_weight_max": 61.933597564697266,
      "activations/layer7_attention_weight_min": -62.35984420776367,
      "activations/layer8_attention_weight_max": 52.64312744140625,
      "activations/layer8_attention_weight_min": -42.63200759887695,
      "activations/layer9_attention_weight_max": 71.23433685302734,
      "activations/layer9_attention_weight_min": -59.01504135131836,
      "epoch": 1.66,
      "learning_rate": 0.00014073484848484846,
      "loss": 3.0061,
      "step": 28500
    },
    {
      "activations/layer0_attention_weight_max": 16.441442489624023,
      "activations/layer0_attention_weight_min": -11.234636306762695,
      "activations/layer10_attention_weight_max": 51.91796875,
      "activations/layer10_attention_weight_min": -44.51917266845703,
      "activations/layer11_attention_weight_max": 42.563934326171875,
      "activations/layer11_attention_weight_min": -35.6114501953125,
      "activations/layer12_attention_weight_max": 31.03911590576172,
      "activations/layer12_attention_weight_min": -28.96761703491211,
      "activations/layer13_attention_weight_max": 47.706905364990234,
      "activations/layer13_attention_weight_min": -38.30199432373047,
      "activations/layer14_attention_weight_max": 84.60616302490234,
      "activations/layer14_attention_weight_min": -61.8007698059082,
      "activations/layer15_attention_weight_max": 61.487022399902344,
      "activations/layer15_attention_weight_min": -41.81402587890625,
      "activations/layer16_attention_weight_max": 56.323184967041016,
      "activations/layer16_attention_weight_min": -42.00628662109375,
      "activations/layer17_attention_weight_max": 48.6407585144043,
      "activations/layer17_attention_weight_min": -30.01265525817871,
      "activations/layer18_attention_weight_max": 29.4409122467041,
      "activations/layer18_attention_weight_min": -16.567983627319336,
      "activations/layer19_attention_weight_max": 61.692359924316406,
      "activations/layer19_attention_weight_min": -32.37641525268555,
      "activations/layer1_attention_weight_max": 17.19264030456543,
      "activations/layer1_attention_weight_min": -11.814520835876465,
      "activations/layer20_attention_weight_max": 46.189571380615234,
      "activations/layer20_attention_weight_min": -27.774635314941406,
      "activations/layer21_attention_weight_max": 37.3977165222168,
      "activations/layer21_attention_weight_min": -18.75041961669922,
      "activations/layer22_attention_weight_max": 46.010807037353516,
      "activations/layer22_attention_weight_min": -25.105241775512695,
      "activations/layer23_attention_weight_max": 26.78798484802246,
      "activations/layer23_attention_weight_min": -15.231074333190918,
      "activations/layer2_attention_weight_max": 31.029638290405273,
      "activations/layer2_attention_weight_min": -28.2358455657959,
      "activations/layer3_attention_weight_max": 67.4840087890625,
      "activations/layer3_attention_weight_min": -71.74585723876953,
      "activations/layer4_attention_weight_max": 124.89617156982422,
      "activations/layer4_attention_weight_min": -110.53517150878906,
      "activations/layer5_attention_weight_max": 127.81391906738281,
      "activations/layer5_attention_weight_min": -100.3783187866211,
      "activations/layer6_attention_weight_max": 69.44204711914062,
      "activations/layer6_attention_weight_min": -54.81205368041992,
      "activations/layer7_attention_weight_max": 66.12080383300781,
      "activations/layer7_attention_weight_min": -59.167728424072266,
      "activations/layer8_attention_weight_max": 54.87315368652344,
      "activations/layer8_attention_weight_min": -45.63777542114258,
      "activations/layer9_attention_weight_max": 72.05803680419922,
      "activations/layer9_attention_weight_min": -58.10377883911133,
      "epoch": 1.66,
      "learning_rate": 0.00014071590909090908,
      "loss": 3.0021,
      "step": 28550
    },
    {
      "activations/layer0_attention_weight_max": 15.476240158081055,
      "activations/layer0_attention_weight_min": -11.409021377563477,
      "activations/layer10_attention_weight_max": 57.80924606323242,
      "activations/layer10_attention_weight_min": -47.29800796508789,
      "activations/layer11_attention_weight_max": 39.10718536376953,
      "activations/layer11_attention_weight_min": -35.69342041015625,
      "activations/layer12_attention_weight_max": 31.96235466003418,
      "activations/layer12_attention_weight_min": -27.504796981811523,
      "activations/layer13_attention_weight_max": 48.44425582885742,
      "activations/layer13_attention_weight_min": -35.49142837524414,
      "activations/layer14_attention_weight_max": 82.12957000732422,
      "activations/layer14_attention_weight_min": -65.65320587158203,
      "activations/layer15_attention_weight_max": 54.97187042236328,
      "activations/layer15_attention_weight_min": -40.71424865722656,
      "activations/layer16_attention_weight_max": 51.89673614501953,
      "activations/layer16_attention_weight_min": -37.06656265258789,
      "activations/layer17_attention_weight_max": 43.04790115356445,
      "activations/layer17_attention_weight_min": -27.46045684814453,
      "activations/layer18_attention_weight_max": 24.869972229003906,
      "activations/layer18_attention_weight_min": -16.456317901611328,
      "activations/layer19_attention_weight_max": 51.5806770324707,
      "activations/layer19_attention_weight_min": -32.64408493041992,
      "activations/layer1_attention_weight_max": 17.058225631713867,
      "activations/layer1_attention_weight_min": -12.220666885375977,
      "activations/layer20_attention_weight_max": 37.94436264038086,
      "activations/layer20_attention_weight_min": -27.575210571289062,
      "activations/layer21_attention_weight_max": 35.66616439819336,
      "activations/layer21_attention_weight_min": -19.19443130493164,
      "activations/layer22_attention_weight_max": 41.639625549316406,
      "activations/layer22_attention_weight_min": -25.605276107788086,
      "activations/layer23_attention_weight_max": 22.244091033935547,
      "activations/layer23_attention_weight_min": -17.118446350097656,
      "activations/layer2_attention_weight_max": 29.07002067565918,
      "activations/layer2_attention_weight_min": -29.07880401611328,
      "activations/layer3_attention_weight_max": 68.82720184326172,
      "activations/layer3_attention_weight_min": -71.53365325927734,
      "activations/layer4_attention_weight_max": 125.8880844116211,
      "activations/layer4_attention_weight_min": -116.49919891357422,
      "activations/layer5_attention_weight_max": 117.8272933959961,
      "activations/layer5_attention_weight_min": -101.27178192138672,
      "activations/layer6_attention_weight_max": 62.51063919067383,
      "activations/layer6_attention_weight_min": -55.33479309082031,
      "activations/layer7_attention_weight_max": 65.142333984375,
      "activations/layer7_attention_weight_min": -64.40618133544922,
      "activations/layer8_attention_weight_max": 50.86164093017578,
      "activations/layer8_attention_weight_min": -46.3519401550293,
      "activations/layer9_attention_weight_max": 66.82038116455078,
      "activations/layer9_attention_weight_min": -55.54362106323242,
      "epoch": 1.66,
      "learning_rate": 0.00014069734848484847,
      "loss": 3.0088,
      "step": 28600
    },
    {
      "activations/layer0_attention_weight_max": 16.922365188598633,
      "activations/layer0_attention_weight_min": -11.415567398071289,
      "activations/layer10_attention_weight_max": 48.391719818115234,
      "activations/layer10_attention_weight_min": -43.434486389160156,
      "activations/layer11_attention_weight_max": 37.974151611328125,
      "activations/layer11_attention_weight_min": -33.08701705932617,
      "activations/layer12_attention_weight_max": 33.03614807128906,
      "activations/layer12_attention_weight_min": -30.22474479675293,
      "activations/layer13_attention_weight_max": 53.330692291259766,
      "activations/layer13_attention_weight_min": -41.435237884521484,
      "activations/layer14_attention_weight_max": 71.67635345458984,
      "activations/layer14_attention_weight_min": -56.05201721191406,
      "activations/layer15_attention_weight_max": 50.673377990722656,
      "activations/layer15_attention_weight_min": -38.21666717529297,
      "activations/layer16_attention_weight_max": 55.315223693847656,
      "activations/layer16_attention_weight_min": -42.286590576171875,
      "activations/layer17_attention_weight_max": 47.31555938720703,
      "activations/layer17_attention_weight_min": -29.43288803100586,
      "activations/layer18_attention_weight_max": 25.60367202758789,
      "activations/layer18_attention_weight_min": -16.218582153320312,
      "activations/layer19_attention_weight_max": 51.31512451171875,
      "activations/layer19_attention_weight_min": -35.40818786621094,
      "activations/layer1_attention_weight_max": 17.74250030517578,
      "activations/layer1_attention_weight_min": -11.315939903259277,
      "activations/layer20_attention_weight_max": 39.290367126464844,
      "activations/layer20_attention_weight_min": -24.666946411132812,
      "activations/layer21_attention_weight_max": 38.500274658203125,
      "activations/layer21_attention_weight_min": -19.665191650390625,
      "activations/layer22_attention_weight_max": 41.17573165893555,
      "activations/layer22_attention_weight_min": -22.028560638427734,
      "activations/layer23_attention_weight_max": 24.119586944580078,
      "activations/layer23_attention_weight_min": -15.646747589111328,
      "activations/layer2_attention_weight_max": 26.987394332885742,
      "activations/layer2_attention_weight_min": -27.676496505737305,
      "activations/layer3_attention_weight_max": 67.54317474365234,
      "activations/layer3_attention_weight_min": -73.13426971435547,
      "activations/layer4_attention_weight_max": 122.15216064453125,
      "activations/layer4_attention_weight_min": -116.56001281738281,
      "activations/layer5_attention_weight_max": 112.58143615722656,
      "activations/layer5_attention_weight_min": -104.50775146484375,
      "activations/layer6_attention_weight_max": 62.71885681152344,
      "activations/layer6_attention_weight_min": -55.419639587402344,
      "activations/layer7_attention_weight_max": 62.650203704833984,
      "activations/layer7_attention_weight_min": -61.74223327636719,
      "activations/layer8_attention_weight_max": 47.17745590209961,
      "activations/layer8_attention_weight_min": -40.17521667480469,
      "activations/layer9_attention_weight_max": 66.63206481933594,
      "activations/layer9_attention_weight_min": -58.05601119995117,
      "epoch": 1.66,
      "learning_rate": 0.00014067840909090908,
      "loss": 2.9948,
      "step": 28650
    },
    {
      "activations/layer0_attention_weight_max": 16.51897430419922,
      "activations/layer0_attention_weight_min": -11.618517875671387,
      "activations/layer10_attention_weight_max": 61.607200622558594,
      "activations/layer10_attention_weight_min": -48.60468292236328,
      "activations/layer11_attention_weight_max": 50.033897399902344,
      "activations/layer11_attention_weight_min": -42.751976013183594,
      "activations/layer12_attention_weight_max": 46.52839660644531,
      "activations/layer12_attention_weight_min": -30.2252140045166,
      "activations/layer13_attention_weight_max": 51.04072952270508,
      "activations/layer13_attention_weight_min": -38.031227111816406,
      "activations/layer14_attention_weight_max": 86.00067138671875,
      "activations/layer14_attention_weight_min": -62.982147216796875,
      "activations/layer15_attention_weight_max": 58.534820556640625,
      "activations/layer15_attention_weight_min": -40.28855514526367,
      "activations/layer16_attention_weight_max": 53.91757583618164,
      "activations/layer16_attention_weight_min": -36.304203033447266,
      "activations/layer17_attention_weight_max": 48.10402297973633,
      "activations/layer17_attention_weight_min": -32.68513870239258,
      "activations/layer18_attention_weight_max": 25.395994186401367,
      "activations/layer18_attention_weight_min": -18.505390167236328,
      "activations/layer19_attention_weight_max": 54.33882522583008,
      "activations/layer19_attention_weight_min": -32.008487701416016,
      "activations/layer1_attention_weight_max": 18.176679611206055,
      "activations/layer1_attention_weight_min": -11.872161865234375,
      "activations/layer20_attention_weight_max": 41.58530044555664,
      "activations/layer20_attention_weight_min": -24.148160934448242,
      "activations/layer21_attention_weight_max": 25.029905319213867,
      "activations/layer21_attention_weight_min": -18.176395416259766,
      "activations/layer22_attention_weight_max": 45.78879928588867,
      "activations/layer22_attention_weight_min": -24.72614097595215,
      "activations/layer23_attention_weight_max": 25.59252166748047,
      "activations/layer23_attention_weight_min": -18.371334075927734,
      "activations/layer2_attention_weight_max": 28.754741668701172,
      "activations/layer2_attention_weight_min": -27.719228744506836,
      "activations/layer3_attention_weight_max": 67.01708221435547,
      "activations/layer3_attention_weight_min": -69.82362365722656,
      "activations/layer4_attention_weight_max": 121.49761962890625,
      "activations/layer4_attention_weight_min": -112.43917083740234,
      "activations/layer5_attention_weight_max": 120.28826904296875,
      "activations/layer5_attention_weight_min": -101.66561889648438,
      "activations/layer6_attention_weight_max": 66.35103607177734,
      "activations/layer6_attention_weight_min": -54.66542053222656,
      "activations/layer7_attention_weight_max": 69.75564575195312,
      "activations/layer7_attention_weight_min": -68.18937683105469,
      "activations/layer8_attention_weight_max": 57.33629608154297,
      "activations/layer8_attention_weight_min": -47.907859802246094,
      "activations/layer9_attention_weight_max": 87.5911636352539,
      "activations/layer9_attention_weight_min": -62.12713623046875,
      "epoch": 1.67,
      "learning_rate": 0.00014065946969696967,
      "loss": 2.9962,
      "step": 28700
    },
    {
      "activations/layer0_attention_weight_max": 16.53495979309082,
      "activations/layer0_attention_weight_min": -11.835367202758789,
      "activations/layer10_attention_weight_max": 54.31011962890625,
      "activations/layer10_attention_weight_min": -48.2839241027832,
      "activations/layer11_attention_weight_max": 39.52252960205078,
      "activations/layer11_attention_weight_min": -36.46894836425781,
      "activations/layer12_attention_weight_max": 38.7584342956543,
      "activations/layer12_attention_weight_min": -31.580610275268555,
      "activations/layer13_attention_weight_max": 49.13379669189453,
      "activations/layer13_attention_weight_min": -42.181488037109375,
      "activations/layer14_attention_weight_max": 78.89486694335938,
      "activations/layer14_attention_weight_min": -61.76448059082031,
      "activations/layer15_attention_weight_max": 52.31297302246094,
      "activations/layer15_attention_weight_min": -38.763980865478516,
      "activations/layer16_attention_weight_max": 47.835391998291016,
      "activations/layer16_attention_weight_min": -39.258235931396484,
      "activations/layer17_attention_weight_max": 39.40099334716797,
      "activations/layer17_attention_weight_min": -31.09171485900879,
      "activations/layer18_attention_weight_max": 26.87765121459961,
      "activations/layer18_attention_weight_min": -17.481182098388672,
      "activations/layer19_attention_weight_max": 51.41792678833008,
      "activations/layer19_attention_weight_min": -33.195133209228516,
      "activations/layer1_attention_weight_max": 16.618457794189453,
      "activations/layer1_attention_weight_min": -13.066226959228516,
      "activations/layer20_attention_weight_max": 39.47734451293945,
      "activations/layer20_attention_weight_min": -25.67304039001465,
      "activations/layer21_attention_weight_max": 26.161766052246094,
      "activations/layer21_attention_weight_min": -19.609283447265625,
      "activations/layer22_attention_weight_max": 43.17856216430664,
      "activations/layer22_attention_weight_min": -23.620765686035156,
      "activations/layer23_attention_weight_max": 25.921600341796875,
      "activations/layer23_attention_weight_min": -15.763080596923828,
      "activations/layer2_attention_weight_max": 29.648284912109375,
      "activations/layer2_attention_weight_min": -27.924121856689453,
      "activations/layer3_attention_weight_max": 67.34440612792969,
      "activations/layer3_attention_weight_min": -69.3503646850586,
      "activations/layer4_attention_weight_max": 121.806396484375,
      "activations/layer4_attention_weight_min": -112.0483169555664,
      "activations/layer5_attention_weight_max": 116.19560241699219,
      "activations/layer5_attention_weight_min": -99.8681640625,
      "activations/layer6_attention_weight_max": 61.33549499511719,
      "activations/layer6_attention_weight_min": -53.61557388305664,
      "activations/layer7_attention_weight_max": 60.31976318359375,
      "activations/layer7_attention_weight_min": -66.00060272216797,
      "activations/layer8_attention_weight_max": 47.646026611328125,
      "activations/layer8_attention_weight_min": -42.43767166137695,
      "activations/layer9_attention_weight_max": 67.8701171875,
      "activations/layer9_attention_weight_min": -59.584869384765625,
      "epoch": 1.67,
      "learning_rate": 0.00014064053030303028,
      "loss": 3.0057,
      "step": 28750
    },
    {
      "activations/layer0_attention_weight_max": 17.179691314697266,
      "activations/layer0_attention_weight_min": -11.420191764831543,
      "activations/layer10_attention_weight_max": 53.87199020385742,
      "activations/layer10_attention_weight_min": -43.131378173828125,
      "activations/layer11_attention_weight_max": 43.30538558959961,
      "activations/layer11_attention_weight_min": -34.375648498535156,
      "activations/layer12_attention_weight_max": 68.04720306396484,
      "activations/layer12_attention_weight_min": -34.88104248046875,
      "activations/layer13_attention_weight_max": 60.139713287353516,
      "activations/layer13_attention_weight_min": -38.20705795288086,
      "activations/layer14_attention_weight_max": 92.57498931884766,
      "activations/layer14_attention_weight_min": -59.03501510620117,
      "activations/layer15_attention_weight_max": 59.08921813964844,
      "activations/layer15_attention_weight_min": -36.006656646728516,
      "activations/layer16_attention_weight_max": 68.970703125,
      "activations/layer16_attention_weight_min": -42.11923599243164,
      "activations/layer17_attention_weight_max": 54.183998107910156,
      "activations/layer17_attention_weight_min": -31.831377029418945,
      "activations/layer18_attention_weight_max": 25.597061157226562,
      "activations/layer18_attention_weight_min": -17.26925277709961,
      "activations/layer19_attention_weight_max": 55.933502197265625,
      "activations/layer19_attention_weight_min": -34.28733444213867,
      "activations/layer1_attention_weight_max": 17.22049903869629,
      "activations/layer1_attention_weight_min": -11.435141563415527,
      "activations/layer20_attention_weight_max": 42.26042175292969,
      "activations/layer20_attention_weight_min": -27.596744537353516,
      "activations/layer21_attention_weight_max": 33.125457763671875,
      "activations/layer21_attention_weight_min": -20.3173828125,
      "activations/layer22_attention_weight_max": 49.2867546081543,
      "activations/layer22_attention_weight_min": -26.18202781677246,
      "activations/layer23_attention_weight_max": 24.62708854675293,
      "activations/layer23_attention_weight_min": -16.606197357177734,
      "activations/layer2_attention_weight_max": 28.24053192138672,
      "activations/layer2_attention_weight_min": -27.0887508392334,
      "activations/layer3_attention_weight_max": 67.77771759033203,
      "activations/layer3_attention_weight_min": -67.37520599365234,
      "activations/layer4_attention_weight_max": 119.03777313232422,
      "activations/layer4_attention_weight_min": -111.5918197631836,
      "activations/layer5_attention_weight_max": 117.09266662597656,
      "activations/layer5_attention_weight_min": -96.47396087646484,
      "activations/layer6_attention_weight_max": 64.16590118408203,
      "activations/layer6_attention_weight_min": -51.30880355834961,
      "activations/layer7_attention_weight_max": 63.705814361572266,
      "activations/layer7_attention_weight_min": -60.426177978515625,
      "activations/layer8_attention_weight_max": 51.032920837402344,
      "activations/layer8_attention_weight_min": -41.44233322143555,
      "activations/layer9_attention_weight_max": 68.01606750488281,
      "activations/layer9_attention_weight_min": -56.73147201538086,
      "epoch": 1.67,
      "learning_rate": 0.0001406215909090909,
      "loss": 2.9989,
      "step": 28800
    },
    {
      "activations/layer0_attention_weight_max": 16.72458839416504,
      "activations/layer0_attention_weight_min": -11.520429611206055,
      "activations/layer10_attention_weight_max": 51.527976989746094,
      "activations/layer10_attention_weight_min": -43.48762512207031,
      "activations/layer11_attention_weight_max": 40.41299057006836,
      "activations/layer11_attention_weight_min": -33.71550750732422,
      "activations/layer12_attention_weight_max": 36.79873275756836,
      "activations/layer12_attention_weight_min": -25.349807739257812,
      "activations/layer13_attention_weight_max": 55.00565719604492,
      "activations/layer13_attention_weight_min": -42.549766540527344,
      "activations/layer14_attention_weight_max": 84.92485046386719,
      "activations/layer14_attention_weight_min": -64.39832305908203,
      "activations/layer15_attention_weight_max": 58.171974182128906,
      "activations/layer15_attention_weight_min": -43.387516021728516,
      "activations/layer16_attention_weight_max": 56.70906066894531,
      "activations/layer16_attention_weight_min": -39.6024284362793,
      "activations/layer17_attention_weight_max": 46.320838928222656,
      "activations/layer17_attention_weight_min": -29.57752799987793,
      "activations/layer18_attention_weight_max": 27.661489486694336,
      "activations/layer18_attention_weight_min": -17.415868759155273,
      "activations/layer19_attention_weight_max": 60.3022346496582,
      "activations/layer19_attention_weight_min": -35.16077423095703,
      "activations/layer1_attention_weight_max": 16.838151931762695,
      "activations/layer1_attention_weight_min": -11.087745666503906,
      "activations/layer20_attention_weight_max": 36.64692306518555,
      "activations/layer20_attention_weight_min": -24.899219512939453,
      "activations/layer21_attention_weight_max": 30.80967903137207,
      "activations/layer21_attention_weight_min": -19.465618133544922,
      "activations/layer22_attention_weight_max": 41.81195068359375,
      "activations/layer22_attention_weight_min": -25.68144416809082,
      "activations/layer23_attention_weight_max": 23.394926071166992,
      "activations/layer23_attention_weight_min": -16.290569305419922,
      "activations/layer2_attention_weight_max": 26.40627098083496,
      "activations/layer2_attention_weight_min": -27.00104522705078,
      "activations/layer3_attention_weight_max": 65.50254821777344,
      "activations/layer3_attention_weight_min": -72.30464172363281,
      "activations/layer4_attention_weight_max": 118.223876953125,
      "activations/layer4_attention_weight_min": -115.89318084716797,
      "activations/layer5_attention_weight_max": 113.36131286621094,
      "activations/layer5_attention_weight_min": -101.29683685302734,
      "activations/layer6_attention_weight_max": 64.18472290039062,
      "activations/layer6_attention_weight_min": -53.62775421142578,
      "activations/layer7_attention_weight_max": 63.63641357421875,
      "activations/layer7_attention_weight_min": -59.39312744140625,
      "activations/layer8_attention_weight_max": 53.80380630493164,
      "activations/layer8_attention_weight_min": -41.32026672363281,
      "activations/layer9_attention_weight_max": 70.49830627441406,
      "activations/layer9_attention_weight_min": -55.219234466552734,
      "epoch": 1.68,
      "learning_rate": 0.0001406026515151515,
      "loss": 2.992,
      "step": 28850
    },
    {
      "activations/layer0_attention_weight_max": 16.85870933532715,
      "activations/layer0_attention_weight_min": -11.734441757202148,
      "activations/layer10_attention_weight_max": 50.38273620605469,
      "activations/layer10_attention_weight_min": -41.449588775634766,
      "activations/layer11_attention_weight_max": 38.6192626953125,
      "activations/layer11_attention_weight_min": -37.9767951965332,
      "activations/layer12_attention_weight_max": 32.93655014038086,
      "activations/layer12_attention_weight_min": -26.6085147857666,
      "activations/layer13_attention_weight_max": 46.130409240722656,
      "activations/layer13_attention_weight_min": -36.629817962646484,
      "activations/layer14_attention_weight_max": 74.22431945800781,
      "activations/layer14_attention_weight_min": -63.29005813598633,
      "activations/layer15_attention_weight_max": 55.45625686645508,
      "activations/layer15_attention_weight_min": -40.15353775024414,
      "activations/layer16_attention_weight_max": 49.22064208984375,
      "activations/layer16_attention_weight_min": -36.561279296875,
      "activations/layer17_attention_weight_max": 45.718963623046875,
      "activations/layer17_attention_weight_min": -32.650489807128906,
      "activations/layer18_attention_weight_max": 25.45296287536621,
      "activations/layer18_attention_weight_min": -17.804027557373047,
      "activations/layer19_attention_weight_max": 57.41716384887695,
      "activations/layer19_attention_weight_min": -36.7049674987793,
      "activations/layer1_attention_weight_max": 17.167461395263672,
      "activations/layer1_attention_weight_min": -11.205284118652344,
      "activations/layer20_attention_weight_max": 52.444026947021484,
      "activations/layer20_attention_weight_min": -26.777904510498047,
      "activations/layer21_attention_weight_max": 28.09745979309082,
      "activations/layer21_attention_weight_min": -17.602210998535156,
      "activations/layer22_attention_weight_max": 43.89965057373047,
      "activations/layer22_attention_weight_min": -26.03238296508789,
      "activations/layer23_attention_weight_max": 23.922225952148438,
      "activations/layer23_attention_weight_min": -15.549779891967773,
      "activations/layer2_attention_weight_max": 28.30649185180664,
      "activations/layer2_attention_weight_min": -28.780811309814453,
      "activations/layer3_attention_weight_max": 67.37205505371094,
      "activations/layer3_attention_weight_min": -73.53767395019531,
      "activations/layer4_attention_weight_max": 123.5394287109375,
      "activations/layer4_attention_weight_min": -114.08087158203125,
      "activations/layer5_attention_weight_max": 119.38722229003906,
      "activations/layer5_attention_weight_min": -103.8899154663086,
      "activations/layer6_attention_weight_max": 65.7630615234375,
      "activations/layer6_attention_weight_min": -53.303688049316406,
      "activations/layer7_attention_weight_max": 60.272823333740234,
      "activations/layer7_attention_weight_min": -60.491798400878906,
      "activations/layer8_attention_weight_max": 48.45606231689453,
      "activations/layer8_attention_weight_min": -41.741607666015625,
      "activations/layer9_attention_weight_max": 67.37605285644531,
      "activations/layer9_attention_weight_min": -54.8816032409668,
      "epoch": 1.68,
      "learning_rate": 0.0001405837121212121,
      "loss": 3.0104,
      "step": 28900
    },
    {
      "activations/layer0_attention_weight_max": 17.047752380371094,
      "activations/layer0_attention_weight_min": -11.64934253692627,
      "activations/layer10_attention_weight_max": 50.57879638671875,
      "activations/layer10_attention_weight_min": -46.60008239746094,
      "activations/layer11_attention_weight_max": 40.576904296875,
      "activations/layer11_attention_weight_min": -36.88507080078125,
      "activations/layer12_attention_weight_max": 32.37531661987305,
      "activations/layer12_attention_weight_min": -25.93193244934082,
      "activations/layer13_attention_weight_max": 49.297916412353516,
      "activations/layer13_attention_weight_min": -39.99359893798828,
      "activations/layer14_attention_weight_max": 68.88329315185547,
      "activations/layer14_attention_weight_min": -53.1867790222168,
      "activations/layer15_attention_weight_max": 51.760475158691406,
      "activations/layer15_attention_weight_min": -40.9440803527832,
      "activations/layer16_attention_weight_max": 52.816741943359375,
      "activations/layer16_attention_weight_min": -37.22245788574219,
      "activations/layer17_attention_weight_max": 46.284690856933594,
      "activations/layer17_attention_weight_min": -31.209712982177734,
      "activations/layer18_attention_weight_max": 25.963600158691406,
      "activations/layer18_attention_weight_min": -16.177133560180664,
      "activations/layer19_attention_weight_max": 52.671791076660156,
      "activations/layer19_attention_weight_min": -32.05678939819336,
      "activations/layer1_attention_weight_max": 17.23613739013672,
      "activations/layer1_attention_weight_min": -10.461752891540527,
      "activations/layer20_attention_weight_max": 33.981929779052734,
      "activations/layer20_attention_weight_min": -23.932771682739258,
      "activations/layer21_attention_weight_max": 34.3487434387207,
      "activations/layer21_attention_weight_min": -21.39618492126465,
      "activations/layer22_attention_weight_max": 42.91536331176758,
      "activations/layer22_attention_weight_min": -26.480384826660156,
      "activations/layer23_attention_weight_max": 28.55022430419922,
      "activations/layer23_attention_weight_min": -16.50971221923828,
      "activations/layer2_attention_weight_max": 27.463027954101562,
      "activations/layer2_attention_weight_min": -27.54030990600586,
      "activations/layer3_attention_weight_max": 68.316162109375,
      "activations/layer3_attention_weight_min": -70.62022399902344,
      "activations/layer4_attention_weight_max": 120.97513580322266,
      "activations/layer4_attention_weight_min": -114.25483703613281,
      "activations/layer5_attention_weight_max": 113.26910400390625,
      "activations/layer5_attention_weight_min": -104.5066146850586,
      "activations/layer6_attention_weight_max": 60.21820831298828,
      "activations/layer6_attention_weight_min": -54.52002716064453,
      "activations/layer7_attention_weight_max": 69.19397735595703,
      "activations/layer7_attention_weight_min": -63.903133392333984,
      "activations/layer8_attention_weight_max": 46.84770965576172,
      "activations/layer8_attention_weight_min": -44.46464157104492,
      "activations/layer9_attention_weight_max": 63.22975540161133,
      "activations/layer9_attention_weight_min": -58.851898193359375,
      "epoch": 1.68,
      "learning_rate": 0.00014056477272727272,
      "loss": 3.0043,
      "step": 28950
    },
    {
      "activations/layer0_attention_weight_max": 16.362152099609375,
      "activations/layer0_attention_weight_min": -11.54734992980957,
      "activations/layer10_attention_weight_max": 56.243099212646484,
      "activations/layer10_attention_weight_min": -44.29480743408203,
      "activations/layer11_attention_weight_max": 42.558860778808594,
      "activations/layer11_attention_weight_min": -33.60113525390625,
      "activations/layer12_attention_weight_max": 39.509063720703125,
      "activations/layer12_attention_weight_min": -27.78604507446289,
      "activations/layer13_attention_weight_max": 55.46115493774414,
      "activations/layer13_attention_weight_min": -38.085872650146484,
      "activations/layer14_attention_weight_max": 91.8674545288086,
      "activations/layer14_attention_weight_min": -67.40216827392578,
      "activations/layer15_attention_weight_max": 57.266239166259766,
      "activations/layer15_attention_weight_min": -37.32920837402344,
      "activations/layer16_attention_weight_max": 60.680694580078125,
      "activations/layer16_attention_weight_min": -37.59085464477539,
      "activations/layer17_attention_weight_max": 48.53879165649414,
      "activations/layer17_attention_weight_min": -28.360370635986328,
      "activations/layer18_attention_weight_max": 26.0506649017334,
      "activations/layer18_attention_weight_min": -20.64748764038086,
      "activations/layer19_attention_weight_max": 54.655250549316406,
      "activations/layer19_attention_weight_min": -31.6317195892334,
      "activations/layer1_attention_weight_max": 18.16988182067871,
      "activations/layer1_attention_weight_min": -12.073996543884277,
      "activations/layer20_attention_weight_max": 40.83278274536133,
      "activations/layer20_attention_weight_min": -30.469573974609375,
      "activations/layer21_attention_weight_max": 34.929786682128906,
      "activations/layer21_attention_weight_min": -19.839527130126953,
      "activations/layer22_attention_weight_max": 46.521812438964844,
      "activations/layer22_attention_weight_min": -27.687702178955078,
      "activations/layer23_attention_weight_max": 26.524547576904297,
      "activations/layer23_attention_weight_min": -16.869121551513672,
      "activations/layer2_attention_weight_max": 29.34760284423828,
      "activations/layer2_attention_weight_min": -28.180517196655273,
      "activations/layer3_attention_weight_max": 68.75704193115234,
      "activations/layer3_attention_weight_min": -69.76763916015625,
      "activations/layer4_attention_weight_max": 124.83277893066406,
      "activations/layer4_attention_weight_min": -111.39088439941406,
      "activations/layer5_attention_weight_max": 113.83978271484375,
      "activations/layer5_attention_weight_min": -101.2655029296875,
      "activations/layer6_attention_weight_max": 64.87034606933594,
      "activations/layer6_attention_weight_min": -54.70448303222656,
      "activations/layer7_attention_weight_max": 68.0585708618164,
      "activations/layer7_attention_weight_min": -61.80874252319336,
      "activations/layer8_attention_weight_max": 56.0816650390625,
      "activations/layer8_attention_weight_min": -43.43111038208008,
      "activations/layer9_attention_weight_max": 80.4042739868164,
      "activations/layer9_attention_weight_min": -56.5037727355957,
      "epoch": 1.69,
      "learning_rate": 0.00014054583333333333,
      "loss": 3.0155,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_loss": 2.943359375,
      "eval_runtime": 8.5587,
      "eval_samples_per_second": 501.711,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_openwebtext_loss": 2.943359375,
      "eval_openwebtext_ppl": 18.979498589532987,
      "eval_openwebtext_runtime": 8.5587,
      "eval_openwebtext_samples_per_second": 501.711,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_wikitext_loss": 3.216796875,
      "eval_wikitext_ppl": 24.948080241282657,
      "eval_wikitext_runtime": 1.9996,
      "eval_wikitext_samples_per_second": 228.051,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_lambada_loss": 3.208984375,
      "eval_lambada_ppl": 24.753932740556596,
      "eval_lambada_runtime": 9.5816,
      "eval_lambada_samples_per_second": 508.16,
      "step": 29000
    },
    {
      "activations/layer0_attention_weight_max": 15.937324523925781,
      "activations/layer0_attention_weight_min": -11.762048721313477,
      "activations/layer10_attention_weight_max": 60.1007194519043,
      "activations/layer10_attention_weight_min": -48.4210319519043,
      "activations/layer11_attention_weight_max": 40.48885726928711,
      "activations/layer11_attention_weight_min": -37.60535430908203,
      "activations/layer12_attention_weight_max": 49.73958206176758,
      "activations/layer12_attention_weight_min": -29.140913009643555,
      "activations/layer13_attention_weight_max": 56.26607131958008,
      "activations/layer13_attention_weight_min": -38.08013916015625,
      "activations/layer14_attention_weight_max": 81.39788818359375,
      "activations/layer14_attention_weight_min": -58.673702239990234,
      "activations/layer15_attention_weight_max": 56.748138427734375,
      "activations/layer15_attention_weight_min": -36.715633392333984,
      "activations/layer16_attention_weight_max": 68.9665756225586,
      "activations/layer16_attention_weight_min": -42.91920471191406,
      "activations/layer17_attention_weight_max": 45.32735824584961,
      "activations/layer17_attention_weight_min": -28.157859802246094,
      "activations/layer18_attention_weight_max": 28.301776885986328,
      "activations/layer18_attention_weight_min": -16.567996978759766,
      "activations/layer19_attention_weight_max": 70.25634765625,
      "activations/layer19_attention_weight_min": -34.0711784362793,
      "activations/layer1_attention_weight_max": 17.31487274169922,
      "activations/layer1_attention_weight_min": -11.451168060302734,
      "activations/layer20_attention_weight_max": 41.61204528808594,
      "activations/layer20_attention_weight_min": -22.76659393310547,
      "activations/layer21_attention_weight_max": 27.464447021484375,
      "activations/layer21_attention_weight_min": -16.4515323638916,
      "activations/layer22_attention_weight_max": 45.84580612182617,
      "activations/layer22_attention_weight_min": -26.35863494873047,
      "activations/layer23_attention_weight_max": 27.51303482055664,
      "activations/layer23_attention_weight_min": -14.653751373291016,
      "activations/layer2_attention_weight_max": 27.754737854003906,
      "activations/layer2_attention_weight_min": -26.437366485595703,
      "activations/layer3_attention_weight_max": 68.06695556640625,
      "activations/layer3_attention_weight_min": -66.7861099243164,
      "activations/layer4_attention_weight_max": 122.01213836669922,
      "activations/layer4_attention_weight_min": -109.25804901123047,
      "activations/layer5_attention_weight_max": 115.03997802734375,
      "activations/layer5_attention_weight_min": -104.13594818115234,
      "activations/layer6_attention_weight_max": 63.92306900024414,
      "activations/layer6_attention_weight_min": -55.2859992980957,
      "activations/layer7_attention_weight_max": 66.2552261352539,
      "activations/layer7_attention_weight_min": -59.363040924072266,
      "activations/layer8_attention_weight_max": 50.99264144897461,
      "activations/layer8_attention_weight_min": -42.93521499633789,
      "activations/layer9_attention_weight_max": 69.31461334228516,
      "activations/layer9_attention_weight_min": -59.931732177734375,
      "epoch": 1.69,
      "learning_rate": 0.00014052689393939392,
      "loss": 3.004,
      "step": 29050
    },
    {
      "activations/layer0_attention_weight_max": 14.831944465637207,
      "activations/layer0_attention_weight_min": -11.454309463500977,
      "activations/layer10_attention_weight_max": 48.48118591308594,
      "activations/layer10_attention_weight_min": -45.69693374633789,
      "activations/layer11_attention_weight_max": 41.77007293701172,
      "activations/layer11_attention_weight_min": -37.86018371582031,
      "activations/layer12_attention_weight_max": 34.29913330078125,
      "activations/layer12_attention_weight_min": -27.792146682739258,
      "activations/layer13_attention_weight_max": 51.16386032104492,
      "activations/layer13_attention_weight_min": -40.6759033203125,
      "activations/layer14_attention_weight_max": 73.93561553955078,
      "activations/layer14_attention_weight_min": -61.45677185058594,
      "activations/layer15_attention_weight_max": 51.817543029785156,
      "activations/layer15_attention_weight_min": -40.8157844543457,
      "activations/layer16_attention_weight_max": 47.75959014892578,
      "activations/layer16_attention_weight_min": -35.20443344116211,
      "activations/layer17_attention_weight_max": 44.30951690673828,
      "activations/layer17_attention_weight_min": -31.02515983581543,
      "activations/layer18_attention_weight_max": 26.365901947021484,
      "activations/layer18_attention_weight_min": -20.363414764404297,
      "activations/layer19_attention_weight_max": 49.92134094238281,
      "activations/layer19_attention_weight_min": -31.483808517456055,
      "activations/layer1_attention_weight_max": 17.484861373901367,
      "activations/layer1_attention_weight_min": -11.918745994567871,
      "activations/layer20_attention_weight_max": 36.15579605102539,
      "activations/layer20_attention_weight_min": -23.990093231201172,
      "activations/layer21_attention_weight_max": 28.939037322998047,
      "activations/layer21_attention_weight_min": -18.516183853149414,
      "activations/layer22_attention_weight_max": 42.515689849853516,
      "activations/layer22_attention_weight_min": -26.98350715637207,
      "activations/layer23_attention_weight_max": 22.647626876831055,
      "activations/layer23_attention_weight_min": -18.122882843017578,
      "activations/layer2_attention_weight_max": 27.297584533691406,
      "activations/layer2_attention_weight_min": -27.50484848022461,
      "activations/layer3_attention_weight_max": 69.49114990234375,
      "activations/layer3_attention_weight_min": -71.14317321777344,
      "activations/layer4_attention_weight_max": 119.50520324707031,
      "activations/layer4_attention_weight_min": -116.07038879394531,
      "activations/layer5_attention_weight_max": 115.73243713378906,
      "activations/layer5_attention_weight_min": -103.79556274414062,
      "activations/layer6_attention_weight_max": 61.60996627807617,
      "activations/layer6_attention_weight_min": -54.21187210083008,
      "activations/layer7_attention_weight_max": 64.27770233154297,
      "activations/layer7_attention_weight_min": -69.7696304321289,
      "activations/layer8_attention_weight_max": 52.571285247802734,
      "activations/layer8_attention_weight_min": -52.425140380859375,
      "activations/layer9_attention_weight_max": 75.19733428955078,
      "activations/layer9_attention_weight_min": -66.09249114990234,
      "epoch": 1.69,
      "learning_rate": 0.00014050795454545454,
      "loss": 2.9994,
      "step": 29100
    },
    {
      "activations/layer0_attention_weight_max": 17.023202896118164,
      "activations/layer0_attention_weight_min": -11.58658218383789,
      "activations/layer10_attention_weight_max": 54.52846145629883,
      "activations/layer10_attention_weight_min": -46.67059326171875,
      "activations/layer11_attention_weight_max": 41.10544967651367,
      "activations/layer11_attention_weight_min": -35.780914306640625,
      "activations/layer12_attention_weight_max": 37.158992767333984,
      "activations/layer12_attention_weight_min": -28.35003089904785,
      "activations/layer13_attention_weight_max": 58.44821548461914,
      "activations/layer13_attention_weight_min": -48.96861267089844,
      "activations/layer14_attention_weight_max": 71.90702819824219,
      "activations/layer14_attention_weight_min": -58.25178909301758,
      "activations/layer15_attention_weight_max": 53.30826187133789,
      "activations/layer15_attention_weight_min": -39.64641571044922,
      "activations/layer16_attention_weight_max": 57.72545623779297,
      "activations/layer16_attention_weight_min": -37.563716888427734,
      "activations/layer17_attention_weight_max": 45.45908737182617,
      "activations/layer17_attention_weight_min": -30.24121856689453,
      "activations/layer18_attention_weight_max": 26.810144424438477,
      "activations/layer18_attention_weight_min": -18.754470825195312,
      "activations/layer19_attention_weight_max": 56.23591995239258,
      "activations/layer19_attention_weight_min": -33.906490325927734,
      "activations/layer1_attention_weight_max": 17.48007583618164,
      "activations/layer1_attention_weight_min": -11.343816757202148,
      "activations/layer20_attention_weight_max": 41.70738983154297,
      "activations/layer20_attention_weight_min": -24.445104598999023,
      "activations/layer21_attention_weight_max": 33.707183837890625,
      "activations/layer21_attention_weight_min": -18.608165740966797,
      "activations/layer22_attention_weight_max": 49.20610809326172,
      "activations/layer22_attention_weight_min": -26.18940544128418,
      "activations/layer23_attention_weight_max": 26.923274993896484,
      "activations/layer23_attention_weight_min": -15.741068840026855,
      "activations/layer2_attention_weight_max": 29.314579010009766,
      "activations/layer2_attention_weight_min": -27.268787384033203,
      "activations/layer3_attention_weight_max": 66.37518310546875,
      "activations/layer3_attention_weight_min": -68.21215057373047,
      "activations/layer4_attention_weight_max": 127.22309875488281,
      "activations/layer4_attention_weight_min": -112.47722625732422,
      "activations/layer5_attention_weight_max": 120.6535873413086,
      "activations/layer5_attention_weight_min": -99.33226013183594,
      "activations/layer6_attention_weight_max": 69.97237396240234,
      "activations/layer6_attention_weight_min": -54.678096771240234,
      "activations/layer7_attention_weight_max": 63.21206283569336,
      "activations/layer7_attention_weight_min": -62.615474700927734,
      "activations/layer8_attention_weight_max": 49.48871612548828,
      "activations/layer8_attention_weight_min": -43.10116958618164,
      "activations/layer9_attention_weight_max": 71.28970336914062,
      "activations/layer9_attention_weight_min": -55.89186477661133,
      "epoch": 1.69,
      "learning_rate": 0.00014048901515151515,
      "loss": 2.9907,
      "step": 29150
    },
    {
      "activations/layer0_attention_weight_max": 16.668424606323242,
      "activations/layer0_attention_weight_min": -11.70276927947998,
      "activations/layer10_attention_weight_max": 51.419883728027344,
      "activations/layer10_attention_weight_min": -46.08284378051758,
      "activations/layer11_attention_weight_max": 46.90003204345703,
      "activations/layer11_attention_weight_min": -34.91361618041992,
      "activations/layer12_attention_weight_max": 74.06148529052734,
      "activations/layer12_attention_weight_min": -39.64101028442383,
      "activations/layer13_attention_weight_max": 64.55906677246094,
      "activations/layer13_attention_weight_min": -42.93122863769531,
      "activations/layer14_attention_weight_max": 89.7669448852539,
      "activations/layer14_attention_weight_min": -69.33695983886719,
      "activations/layer15_attention_weight_max": 57.83491516113281,
      "activations/layer15_attention_weight_min": -38.65229034423828,
      "activations/layer16_attention_weight_max": 52.29595947265625,
      "activations/layer16_attention_weight_min": -41.48808670043945,
      "activations/layer17_attention_weight_max": 44.27138137817383,
      "activations/layer17_attention_weight_min": -29.009450912475586,
      "activations/layer18_attention_weight_max": 24.553699493408203,
      "activations/layer18_attention_weight_min": -15.838912010192871,
      "activations/layer19_attention_weight_max": 45.81277847290039,
      "activations/layer19_attention_weight_min": -31.9365291595459,
      "activations/layer1_attention_weight_max": 17.749736785888672,
      "activations/layer1_attention_weight_min": -10.982990264892578,
      "activations/layer20_attention_weight_max": 43.240230560302734,
      "activations/layer20_attention_weight_min": -24.796972274780273,
      "activations/layer21_attention_weight_max": 28.621641159057617,
      "activations/layer21_attention_weight_min": -19.732681274414062,
      "activations/layer22_attention_weight_max": 41.942813873291016,
      "activations/layer22_attention_weight_min": -24.814207077026367,
      "activations/layer23_attention_weight_max": 24.728729248046875,
      "activations/layer23_attention_weight_min": -15.477949142456055,
      "activations/layer2_attention_weight_max": 26.313121795654297,
      "activations/layer2_attention_weight_min": -27.890567779541016,
      "activations/layer3_attention_weight_max": 66.23626708984375,
      "activations/layer3_attention_weight_min": -69.07756805419922,
      "activations/layer4_attention_weight_max": 123.52416229248047,
      "activations/layer4_attention_weight_min": -116.41849517822266,
      "activations/layer5_attention_weight_max": 117.44454956054688,
      "activations/layer5_attention_weight_min": -99.98190307617188,
      "activations/layer6_attention_weight_max": 63.555259704589844,
      "activations/layer6_attention_weight_min": -52.66852569580078,
      "activations/layer7_attention_weight_max": 65.80098724365234,
      "activations/layer7_attention_weight_min": -62.69791793823242,
      "activations/layer8_attention_weight_max": 48.814979553222656,
      "activations/layer8_attention_weight_min": -40.97465896606445,
      "activations/layer9_attention_weight_max": 80.41970825195312,
      "activations/layer9_attention_weight_min": -58.696510314941406,
      "epoch": 1.7,
      "learning_rate": 0.00014047007575757574,
      "loss": 3.0177,
      "step": 29200
    },
    {
      "activations/layer0_attention_weight_max": 17.22014617919922,
      "activations/layer0_attention_weight_min": -11.5056734085083,
      "activations/layer10_attention_weight_max": 55.743377685546875,
      "activations/layer10_attention_weight_min": -51.02335739135742,
      "activations/layer11_attention_weight_max": 40.84903335571289,
      "activations/layer11_attention_weight_min": -34.80818176269531,
      "activations/layer12_attention_weight_max": 42.3189582824707,
      "activations/layer12_attention_weight_min": -27.643863677978516,
      "activations/layer13_attention_weight_max": 55.356483459472656,
      "activations/layer13_attention_weight_min": -41.759971618652344,
      "activations/layer14_attention_weight_max": 81.97090148925781,
      "activations/layer14_attention_weight_min": -56.839542388916016,
      "activations/layer15_attention_weight_max": 55.40662384033203,
      "activations/layer15_attention_weight_min": -38.267120361328125,
      "activations/layer16_attention_weight_max": 59.426780700683594,
      "activations/layer16_attention_weight_min": -43.0803108215332,
      "activations/layer17_attention_weight_max": 42.838623046875,
      "activations/layer17_attention_weight_min": -31.98002815246582,
      "activations/layer18_attention_weight_max": 31.153362274169922,
      "activations/layer18_attention_weight_min": -17.768455505371094,
      "activations/layer19_attention_weight_max": 56.2054443359375,
      "activations/layer19_attention_weight_min": -31.628820419311523,
      "activations/layer1_attention_weight_max": 18.55786895751953,
      "activations/layer1_attention_weight_min": -11.165261268615723,
      "activations/layer20_attention_weight_max": 44.01077651977539,
      "activations/layer20_attention_weight_min": -28.132949829101562,
      "activations/layer21_attention_weight_max": 36.600791931152344,
      "activations/layer21_attention_weight_min": -20.383480072021484,
      "activations/layer22_attention_weight_max": 48.07038116455078,
      "activations/layer22_attention_weight_min": -29.453125,
      "activations/layer23_attention_weight_max": 27.28204345703125,
      "activations/layer23_attention_weight_min": -15.865405082702637,
      "activations/layer2_attention_weight_max": 27.335895538330078,
      "activations/layer2_attention_weight_min": -27.256322860717773,
      "activations/layer3_attention_weight_max": 67.02195739746094,
      "activations/layer3_attention_weight_min": -67.60292053222656,
      "activations/layer4_attention_weight_max": 129.68406677246094,
      "activations/layer4_attention_weight_min": -114.79808807373047,
      "activations/layer5_attention_weight_max": 118.5020751953125,
      "activations/layer5_attention_weight_min": -105.16234588623047,
      "activations/layer6_attention_weight_max": 65.54898834228516,
      "activations/layer6_attention_weight_min": -56.52190017700195,
      "activations/layer7_attention_weight_max": 64.21542358398438,
      "activations/layer7_attention_weight_min": -60.79492950439453,
      "activations/layer8_attention_weight_max": 49.475914001464844,
      "activations/layer8_attention_weight_min": -48.56739807128906,
      "activations/layer9_attention_weight_max": 73.4880142211914,
      "activations/layer9_attention_weight_min": -58.546913146972656,
      "epoch": 1.7,
      "learning_rate": 0.00014045113636363635,
      "loss": 3.0013,
      "step": 29250
    },
    {
      "activations/layer0_attention_weight_max": 16.582073211669922,
      "activations/layer0_attention_weight_min": -11.947285652160645,
      "activations/layer10_attention_weight_max": 49.0640754699707,
      "activations/layer10_attention_weight_min": -44.21561813354492,
      "activations/layer11_attention_weight_max": 38.441131591796875,
      "activations/layer11_attention_weight_min": -32.3497314453125,
      "activations/layer12_attention_weight_max": 37.5131721496582,
      "activations/layer12_attention_weight_min": -26.66771125793457,
      "activations/layer13_attention_weight_max": 47.59248733520508,
      "activations/layer13_attention_weight_min": -36.80213165283203,
      "activations/layer14_attention_weight_max": 70.718994140625,
      "activations/layer14_attention_weight_min": -52.330711364746094,
      "activations/layer15_attention_weight_max": 52.555538177490234,
      "activations/layer15_attention_weight_min": -36.711185455322266,
      "activations/layer16_attention_weight_max": 49.58407211303711,
      "activations/layer16_attention_weight_min": -36.538970947265625,
      "activations/layer17_attention_weight_max": 48.44461441040039,
      "activations/layer17_attention_weight_min": -28.371917724609375,
      "activations/layer18_attention_weight_max": 28.496061325073242,
      "activations/layer18_attention_weight_min": -17.90300750732422,
      "activations/layer19_attention_weight_max": 59.60332107543945,
      "activations/layer19_attention_weight_min": -31.7944278717041,
      "activations/layer1_attention_weight_max": 17.661617279052734,
      "activations/layer1_attention_weight_min": -11.707670211791992,
      "activations/layer20_attention_weight_max": 44.934043884277344,
      "activations/layer20_attention_weight_min": -24.542362213134766,
      "activations/layer21_attention_weight_max": 29.867576599121094,
      "activations/layer21_attention_weight_min": -20.00465202331543,
      "activations/layer22_attention_weight_max": 47.52541732788086,
      "activations/layer22_attention_weight_min": -25.8444881439209,
      "activations/layer23_attention_weight_max": 24.782209396362305,
      "activations/layer23_attention_weight_min": -15.496292114257812,
      "activations/layer2_attention_weight_max": 27.45522689819336,
      "activations/layer2_attention_weight_min": -27.253589630126953,
      "activations/layer3_attention_weight_max": 65.544189453125,
      "activations/layer3_attention_weight_min": -65.6880874633789,
      "activations/layer4_attention_weight_max": 117.6815185546875,
      "activations/layer4_attention_weight_min": -113.2242431640625,
      "activations/layer5_attention_weight_max": 110.23735046386719,
      "activations/layer5_attention_weight_min": -96.59722137451172,
      "activations/layer6_attention_weight_max": 59.08096694946289,
      "activations/layer6_attention_weight_min": -51.66111373901367,
      "activations/layer7_attention_weight_max": 59.87269592285156,
      "activations/layer7_attention_weight_min": -58.749427795410156,
      "activations/layer8_attention_weight_max": 50.31517028808594,
      "activations/layer8_attention_weight_min": -43.46402359008789,
      "activations/layer9_attention_weight_max": 63.4139289855957,
      "activations/layer9_attention_weight_min": -58.50497055053711,
      "epoch": 1.7,
      "learning_rate": 0.00014043219696969697,
      "loss": 2.9997,
      "step": 29300
    },
    {
      "activations/layer0_attention_weight_max": 16.724056243896484,
      "activations/layer0_attention_weight_min": -11.806589126586914,
      "activations/layer10_attention_weight_max": 48.658443450927734,
      "activations/layer10_attention_weight_min": -45.761993408203125,
      "activations/layer11_attention_weight_max": 41.08021545410156,
      "activations/layer11_attention_weight_min": -38.51847457885742,
      "activations/layer12_attention_weight_max": 26.949430465698242,
      "activations/layer12_attention_weight_min": -29.343931198120117,
      "activations/layer13_attention_weight_max": 43.42863464355469,
      "activations/layer13_attention_weight_min": -36.495445251464844,
      "activations/layer14_attention_weight_max": 73.8374252319336,
      "activations/layer14_attention_weight_min": -61.52699661254883,
      "activations/layer15_attention_weight_max": 53.957191467285156,
      "activations/layer15_attention_weight_min": -40.25817108154297,
      "activations/layer16_attention_weight_max": 49.87408447265625,
      "activations/layer16_attention_weight_min": -40.645477294921875,
      "activations/layer17_attention_weight_max": 39.7866325378418,
      "activations/layer17_attention_weight_min": -28.722463607788086,
      "activations/layer18_attention_weight_max": 25.475534439086914,
      "activations/layer18_attention_weight_min": -17.618648529052734,
      "activations/layer19_attention_weight_max": 51.12871170043945,
      "activations/layer19_attention_weight_min": -34.923099517822266,
      "activations/layer1_attention_weight_max": 17.37599754333496,
      "activations/layer1_attention_weight_min": -11.057004928588867,
      "activations/layer20_attention_weight_max": 37.24544143676758,
      "activations/layer20_attention_weight_min": -26.930219650268555,
      "activations/layer21_attention_weight_max": 27.324569702148438,
      "activations/layer21_attention_weight_min": -19.679920196533203,
      "activations/layer22_attention_weight_max": 41.70172119140625,
      "activations/layer22_attention_weight_min": -25.329635620117188,
      "activations/layer23_attention_weight_max": 23.35785484313965,
      "activations/layer23_attention_weight_min": -15.116216659545898,
      "activations/layer2_attention_weight_max": 27.715763092041016,
      "activations/layer2_attention_weight_min": -26.893415451049805,
      "activations/layer3_attention_weight_max": 66.74720001220703,
      "activations/layer3_attention_weight_min": -69.11731719970703,
      "activations/layer4_attention_weight_max": 121.5369644165039,
      "activations/layer4_attention_weight_min": -120.37552642822266,
      "activations/layer5_attention_weight_max": 118.50883483886719,
      "activations/layer5_attention_weight_min": -101.3723373413086,
      "activations/layer6_attention_weight_max": 63.421783447265625,
      "activations/layer6_attention_weight_min": -54.408363342285156,
      "activations/layer7_attention_weight_max": 66.69670867919922,
      "activations/layer7_attention_weight_min": -60.341617584228516,
      "activations/layer8_attention_weight_max": 46.627532958984375,
      "activations/layer8_attention_weight_min": -44.682952880859375,
      "activations/layer9_attention_weight_max": 73.28136444091797,
      "activations/layer9_attention_weight_min": -65.35972595214844,
      "epoch": 1.71,
      "learning_rate": 0.00014041325757575758,
      "loss": 3.0096,
      "step": 29350
    },
    {
      "activations/layer0_attention_weight_max": 16.41250228881836,
      "activations/layer0_attention_weight_min": -12.098281860351562,
      "activations/layer10_attention_weight_max": 60.27016067504883,
      "activations/layer10_attention_weight_min": -46.27946090698242,
      "activations/layer11_attention_weight_max": 43.19972229003906,
      "activations/layer11_attention_weight_min": -35.31669998168945,
      "activations/layer12_attention_weight_max": 38.09535217285156,
      "activations/layer12_attention_weight_min": -28.106203079223633,
      "activations/layer13_attention_weight_max": 51.07925033569336,
      "activations/layer13_attention_weight_min": -41.77564239501953,
      "activations/layer14_attention_weight_max": 74.49510955810547,
      "activations/layer14_attention_weight_min": -55.947975158691406,
      "activations/layer15_attention_weight_max": 64.25457763671875,
      "activations/layer15_attention_weight_min": -44.272464752197266,
      "activations/layer16_attention_weight_max": 47.91646957397461,
      "activations/layer16_attention_weight_min": -33.858829498291016,
      "activations/layer17_attention_weight_max": 47.95423126220703,
      "activations/layer17_attention_weight_min": -29.91767120361328,
      "activations/layer18_attention_weight_max": 28.473411560058594,
      "activations/layer18_attention_weight_min": -20.393333435058594,
      "activations/layer19_attention_weight_max": 54.084407806396484,
      "activations/layer19_attention_weight_min": -34.079200744628906,
      "activations/layer1_attention_weight_max": 17.78365135192871,
      "activations/layer1_attention_weight_min": -11.35416030883789,
      "activations/layer20_attention_weight_max": 38.00272750854492,
      "activations/layer20_attention_weight_min": -24.05773162841797,
      "activations/layer21_attention_weight_max": 28.690868377685547,
      "activations/layer21_attention_weight_min": -16.684940338134766,
      "activations/layer22_attention_weight_max": 43.88751983642578,
      "activations/layer22_attention_weight_min": -24.78944206237793,
      "activations/layer23_attention_weight_max": 25.159101486206055,
      "activations/layer23_attention_weight_min": -16.164466857910156,
      "activations/layer2_attention_weight_max": 29.076337814331055,
      "activations/layer2_attention_weight_min": -28.248722076416016,
      "activations/layer3_attention_weight_max": 71.5246810913086,
      "activations/layer3_attention_weight_min": -70.3880615234375,
      "activations/layer4_attention_weight_max": 125.8113784790039,
      "activations/layer4_attention_weight_min": -110.34215545654297,
      "activations/layer5_attention_weight_max": 124.86510467529297,
      "activations/layer5_attention_weight_min": -99.86519622802734,
      "activations/layer6_attention_weight_max": 66.64921569824219,
      "activations/layer6_attention_weight_min": -54.29997253417969,
      "activations/layer7_attention_weight_max": 66.70536041259766,
      "activations/layer7_attention_weight_min": -58.9284553527832,
      "activations/layer8_attention_weight_max": 54.36048889160156,
      "activations/layer8_attention_weight_min": -41.271514892578125,
      "activations/layer9_attention_weight_max": 69.91072845458984,
      "activations/layer9_attention_weight_min": -59.01721954345703,
      "epoch": 1.71,
      "learning_rate": 0.00014039431818181817,
      "loss": 2.9911,
      "step": 29400
    },
    {
      "activations/layer0_attention_weight_max": 16.335092544555664,
      "activations/layer0_attention_weight_min": -11.63525676727295,
      "activations/layer10_attention_weight_max": 49.53068542480469,
      "activations/layer10_attention_weight_min": -46.36888122558594,
      "activations/layer11_attention_weight_max": 40.09234619140625,
      "activations/layer11_attention_weight_min": -35.016578674316406,
      "activations/layer12_attention_weight_max": 49.239227294921875,
      "activations/layer12_attention_weight_min": -41.83403396606445,
      "activations/layer13_attention_weight_max": 45.71163558959961,
      "activations/layer13_attention_weight_min": -38.03218078613281,
      "activations/layer14_attention_weight_max": 73.86563873291016,
      "activations/layer14_attention_weight_min": -56.83786392211914,
      "activations/layer15_attention_weight_max": 54.82227325439453,
      "activations/layer15_attention_weight_min": -39.53767776489258,
      "activations/layer16_attention_weight_max": 55.45790481567383,
      "activations/layer16_attention_weight_min": -44.102699279785156,
      "activations/layer17_attention_weight_max": 46.6127815246582,
      "activations/layer17_attention_weight_min": -30.912363052368164,
      "activations/layer18_attention_weight_max": 29.5260009765625,
      "activations/layer18_attention_weight_min": -16.591533660888672,
      "activations/layer19_attention_weight_max": 55.27817916870117,
      "activations/layer19_attention_weight_min": -32.65974807739258,
      "activations/layer1_attention_weight_max": 17.508983612060547,
      "activations/layer1_attention_weight_min": -10.979630470275879,
      "activations/layer20_attention_weight_max": 43.59369659423828,
      "activations/layer20_attention_weight_min": -27.01304817199707,
      "activations/layer21_attention_weight_max": 30.74209213256836,
      "activations/layer21_attention_weight_min": -19.85560417175293,
      "activations/layer22_attention_weight_max": 46.1551628112793,
      "activations/layer22_attention_weight_min": -26.558015823364258,
      "activations/layer23_attention_weight_max": 26.672016143798828,
      "activations/layer23_attention_weight_min": -17.014450073242188,
      "activations/layer2_attention_weight_max": 28.598737716674805,
      "activations/layer2_attention_weight_min": -27.988643646240234,
      "activations/layer3_attention_weight_max": 66.54776763916016,
      "activations/layer3_attention_weight_min": -71.61365509033203,
      "activations/layer4_attention_weight_max": 123.77215576171875,
      "activations/layer4_attention_weight_min": -113.59041595458984,
      "activations/layer5_attention_weight_max": 121.71099853515625,
      "activations/layer5_attention_weight_min": -101.96543884277344,
      "activations/layer6_attention_weight_max": 63.22501754760742,
      "activations/layer6_attention_weight_min": -55.78113555908203,
      "activations/layer7_attention_weight_max": 62.49281692504883,
      "activations/layer7_attention_weight_min": -64.77783203125,
      "activations/layer8_attention_weight_max": 50.70714569091797,
      "activations/layer8_attention_weight_min": -45.36161422729492,
      "activations/layer9_attention_weight_max": 67.85753631591797,
      "activations/layer9_attention_weight_min": -59.70779037475586,
      "epoch": 1.71,
      "learning_rate": 0.00014037537878787879,
      "loss": 2.9833,
      "step": 29450
    },
    {
      "activations/layer0_attention_weight_max": 17.458663940429688,
      "activations/layer0_attention_weight_min": -11.289548873901367,
      "activations/layer10_attention_weight_max": 58.474082946777344,
      "activations/layer10_attention_weight_min": -47.8476676940918,
      "activations/layer11_attention_weight_max": 44.44062423706055,
      "activations/layer11_attention_weight_min": -36.103885650634766,
      "activations/layer12_attention_weight_max": 36.74525833129883,
      "activations/layer12_attention_weight_min": -27.32451629638672,
      "activations/layer13_attention_weight_max": 49.558570861816406,
      "activations/layer13_attention_weight_min": -38.48450469970703,
      "activations/layer14_attention_weight_max": 73.55217742919922,
      "activations/layer14_attention_weight_min": -56.457733154296875,
      "activations/layer15_attention_weight_max": 57.39084243774414,
      "activations/layer15_attention_weight_min": -42.090911865234375,
      "activations/layer16_attention_weight_max": 55.90266418457031,
      "activations/layer16_attention_weight_min": -36.131629943847656,
      "activations/layer17_attention_weight_max": 47.9144287109375,
      "activations/layer17_attention_weight_min": -33.65456008911133,
      "activations/layer18_attention_weight_max": 27.76998519897461,
      "activations/layer18_attention_weight_min": -17.132787704467773,
      "activations/layer19_attention_weight_max": 61.575557708740234,
      "activations/layer19_attention_weight_min": -33.50517654418945,
      "activations/layer1_attention_weight_max": 17.29363250732422,
      "activations/layer1_attention_weight_min": -10.944517135620117,
      "activations/layer20_attention_weight_max": 46.49028778076172,
      "activations/layer20_attention_weight_min": -26.762222290039062,
      "activations/layer21_attention_weight_max": 45.05287551879883,
      "activations/layer21_attention_weight_min": -23.26149559020996,
      "activations/layer22_attention_weight_max": 49.91521072387695,
      "activations/layer22_attention_weight_min": -27.05349349975586,
      "activations/layer23_attention_weight_max": 27.295825958251953,
      "activations/layer23_attention_weight_min": -17.309741973876953,
      "activations/layer2_attention_weight_max": 29.379552841186523,
      "activations/layer2_attention_weight_min": -27.52008628845215,
      "activations/layer3_attention_weight_max": 69.03521728515625,
      "activations/layer3_attention_weight_min": -70.42140197753906,
      "activations/layer4_attention_weight_max": 121.06208801269531,
      "activations/layer4_attention_weight_min": -114.49626922607422,
      "activations/layer5_attention_weight_max": 116.30052185058594,
      "activations/layer5_attention_weight_min": -98.20454406738281,
      "activations/layer6_attention_weight_max": 64.05656433105469,
      "activations/layer6_attention_weight_min": -55.18810272216797,
      "activations/layer7_attention_weight_max": 65.11164093017578,
      "activations/layer7_attention_weight_min": -60.536529541015625,
      "activations/layer8_attention_weight_max": 53.53905487060547,
      "activations/layer8_attention_weight_min": -42.19480514526367,
      "activations/layer9_attention_weight_max": 65.64263916015625,
      "activations/layer9_attention_weight_min": -59.601314544677734,
      "epoch": 1.71,
      "learning_rate": 0.00014035643939393937,
      "loss": 2.9909,
      "step": 29500
    },
    {
      "activations/layer0_attention_weight_max": 14.06070327758789,
      "activations/layer0_attention_weight_min": -11.911978721618652,
      "activations/layer10_attention_weight_max": 51.298187255859375,
      "activations/layer10_attention_weight_min": -43.62533950805664,
      "activations/layer11_attention_weight_max": 37.56047821044922,
      "activations/layer11_attention_weight_min": -35.90924835205078,
      "activations/layer12_attention_weight_max": 68.19865417480469,
      "activations/layer12_attention_weight_min": -35.153724670410156,
      "activations/layer13_attention_weight_max": 59.24162292480469,
      "activations/layer13_attention_weight_min": -37.70337677001953,
      "activations/layer14_attention_weight_max": 95.16862487792969,
      "activations/layer14_attention_weight_min": -65.66219329833984,
      "activations/layer15_attention_weight_max": 54.497188568115234,
      "activations/layer15_attention_weight_min": -41.65155029296875,
      "activations/layer16_attention_weight_max": 52.87260055541992,
      "activations/layer16_attention_weight_min": -35.925071716308594,
      "activations/layer17_attention_weight_max": 48.82393264770508,
      "activations/layer17_attention_weight_min": -31.192317962646484,
      "activations/layer18_attention_weight_max": 33.98469543457031,
      "activations/layer18_attention_weight_min": -15.989457130432129,
      "activations/layer19_attention_weight_max": 58.65576171875,
      "activations/layer19_attention_weight_min": -32.48014450073242,
      "activations/layer1_attention_weight_max": 16.78983497619629,
      "activations/layer1_attention_weight_min": -11.489212036132812,
      "activations/layer20_attention_weight_max": 43.199859619140625,
      "activations/layer20_attention_weight_min": -25.538185119628906,
      "activations/layer21_attention_weight_max": 31.52776336669922,
      "activations/layer21_attention_weight_min": -19.940139770507812,
      "activations/layer22_attention_weight_max": 44.465049743652344,
      "activations/layer22_attention_weight_min": -25.87100601196289,
      "activations/layer23_attention_weight_max": 24.99622917175293,
      "activations/layer23_attention_weight_min": -16.297073364257812,
      "activations/layer2_attention_weight_max": 29.012901306152344,
      "activations/layer2_attention_weight_min": -28.224639892578125,
      "activations/layer3_attention_weight_max": 67.30272674560547,
      "activations/layer3_attention_weight_min": -70.44564056396484,
      "activations/layer4_attention_weight_max": 119.93501281738281,
      "activations/layer4_attention_weight_min": -116.44474029541016,
      "activations/layer5_attention_weight_max": 112.11355590820312,
      "activations/layer5_attention_weight_min": -100.15480041503906,
      "activations/layer6_attention_weight_max": 63.839515686035156,
      "activations/layer6_attention_weight_min": -55.997314453125,
      "activations/layer7_attention_weight_max": 70.27882385253906,
      "activations/layer7_attention_weight_min": -60.03757095336914,
      "activations/layer8_attention_weight_max": 54.699520111083984,
      "activations/layer8_attention_weight_min": -46.942840576171875,
      "activations/layer9_attention_weight_max": 71.15966796875,
      "activations/layer9_attention_weight_min": -62.330684661865234,
      "epoch": 1.72,
      "learning_rate": 0.0001403375,
      "loss": 2.9877,
      "step": 29550
    },
    {
      "activations/layer0_attention_weight_max": 16.21958351135254,
      "activations/layer0_attention_weight_min": -11.623830795288086,
      "activations/layer10_attention_weight_max": 54.54393768310547,
      "activations/layer10_attention_weight_min": -48.630855560302734,
      "activations/layer11_attention_weight_max": 38.14262008666992,
      "activations/layer11_attention_weight_min": -35.873565673828125,
      "activations/layer12_attention_weight_max": 29.922754287719727,
      "activations/layer12_attention_weight_min": -24.444543838500977,
      "activations/layer13_attention_weight_max": 60.376346588134766,
      "activations/layer13_attention_weight_min": -44.054683685302734,
      "activations/layer14_attention_weight_max": 74.32792663574219,
      "activations/layer14_attention_weight_min": -62.889949798583984,
      "activations/layer15_attention_weight_max": 51.081050872802734,
      "activations/layer15_attention_weight_min": -39.918067932128906,
      "activations/layer16_attention_weight_max": 51.116065979003906,
      "activations/layer16_attention_weight_min": -42.206199645996094,
      "activations/layer17_attention_weight_max": 45.646080017089844,
      "activations/layer17_attention_weight_min": -34.50163650512695,
      "activations/layer18_attention_weight_max": 22.87235450744629,
      "activations/layer18_attention_weight_min": -15.884941101074219,
      "activations/layer19_attention_weight_max": 45.86870574951172,
      "activations/layer19_attention_weight_min": -32.31974411010742,
      "activations/layer1_attention_weight_max": 17.13984489440918,
      "activations/layer1_attention_weight_min": -11.935517311096191,
      "activations/layer20_attention_weight_max": 38.28384780883789,
      "activations/layer20_attention_weight_min": -25.15557098388672,
      "activations/layer21_attention_weight_max": 31.50008201599121,
      "activations/layer21_attention_weight_min": -18.255258560180664,
      "activations/layer22_attention_weight_max": 42.51415252685547,
      "activations/layer22_attention_weight_min": -26.286794662475586,
      "activations/layer23_attention_weight_max": 25.129179000854492,
      "activations/layer23_attention_weight_min": -14.722623825073242,
      "activations/layer2_attention_weight_max": 28.000499725341797,
      "activations/layer2_attention_weight_min": -27.833282470703125,
      "activations/layer3_attention_weight_max": 68.21460723876953,
      "activations/layer3_attention_weight_min": -70.58448028564453,
      "activations/layer4_attention_weight_max": 118.62701416015625,
      "activations/layer4_attention_weight_min": -112.64928436279297,
      "activations/layer5_attention_weight_max": 111.360595703125,
      "activations/layer5_attention_weight_min": -101.08653259277344,
      "activations/layer6_attention_weight_max": 61.61241912841797,
      "activations/layer6_attention_weight_min": -55.056121826171875,
      "activations/layer7_attention_weight_max": 62.345577239990234,
      "activations/layer7_attention_weight_min": -63.35505294799805,
      "activations/layer8_attention_weight_max": 50.89558410644531,
      "activations/layer8_attention_weight_min": -44.51637649536133,
      "activations/layer9_attention_weight_max": 68.69218444824219,
      "activations/layer9_attention_weight_min": -55.42487716674805,
      "epoch": 1.72,
      "learning_rate": 0.00014031893939393938,
      "loss": 2.9972,
      "step": 29600
    },
    {
      "activations/layer0_attention_weight_max": 17.367141723632812,
      "activations/layer0_attention_weight_min": -11.6671142578125,
      "activations/layer10_attention_weight_max": 59.12324905395508,
      "activations/layer10_attention_weight_min": -45.2151985168457,
      "activations/layer11_attention_weight_max": 41.298187255859375,
      "activations/layer11_attention_weight_min": -34.79541778564453,
      "activations/layer12_attention_weight_max": 49.5391731262207,
      "activations/layer12_attention_weight_min": -37.0771598815918,
      "activations/layer13_attention_weight_max": 50.29869079589844,
      "activations/layer13_attention_weight_min": -37.20886993408203,
      "activations/layer14_attention_weight_max": 81.24571990966797,
      "activations/layer14_attention_weight_min": -61.95984649658203,
      "activations/layer15_attention_weight_max": 55.769588470458984,
      "activations/layer15_attention_weight_min": -37.020301818847656,
      "activations/layer16_attention_weight_max": 52.45920181274414,
      "activations/layer16_attention_weight_min": -39.36358642578125,
      "activations/layer17_attention_weight_max": 50.23716354370117,
      "activations/layer17_attention_weight_min": -28.51387596130371,
      "activations/layer18_attention_weight_max": 29.883825302124023,
      "activations/layer18_attention_weight_min": -17.171390533447266,
      "activations/layer19_attention_weight_max": 54.32575225830078,
      "activations/layer19_attention_weight_min": -33.7407341003418,
      "activations/layer1_attention_weight_max": 17.038759231567383,
      "activations/layer1_attention_weight_min": -10.978721618652344,
      "activations/layer20_attention_weight_max": 42.397586822509766,
      "activations/layer20_attention_weight_min": -24.712539672851562,
      "activations/layer21_attention_weight_max": 29.465377807617188,
      "activations/layer21_attention_weight_min": -19.80042839050293,
      "activations/layer22_attention_weight_max": 42.46741485595703,
      "activations/layer22_attention_weight_min": -24.40875244140625,
      "activations/layer23_attention_weight_max": 27.07015609741211,
      "activations/layer23_attention_weight_min": -15.323798179626465,
      "activations/layer2_attention_weight_max": 28.415029525756836,
      "activations/layer2_attention_weight_min": -26.30883026123047,
      "activations/layer3_attention_weight_max": 66.33032989501953,
      "activations/layer3_attention_weight_min": -69.1272964477539,
      "activations/layer4_attention_weight_max": 126.53717041015625,
      "activations/layer4_attention_weight_min": -113.33160400390625,
      "activations/layer5_attention_weight_max": 123.50688934326172,
      "activations/layer5_attention_weight_min": -99.99400329589844,
      "activations/layer6_attention_weight_max": 62.76007080078125,
      "activations/layer6_attention_weight_min": -56.3009033203125,
      "activations/layer7_attention_weight_max": 64.09966278076172,
      "activations/layer7_attention_weight_min": -59.75370407104492,
      "activations/layer8_attention_weight_max": 51.79863357543945,
      "activations/layer8_attention_weight_min": -44.03616714477539,
      "activations/layer9_attention_weight_max": 73.98484802246094,
      "activations/layer9_attention_weight_min": -55.670875549316406,
      "epoch": 1.72,
      "learning_rate": 0.0001403,
      "loss": 2.9907,
      "step": 29650
    },
    {
      "activations/layer0_attention_weight_max": 17.305339813232422,
      "activations/layer0_attention_weight_min": -11.60299301147461,
      "activations/layer10_attention_weight_max": 56.287261962890625,
      "activations/layer10_attention_weight_min": -47.5706672668457,
      "activations/layer11_attention_weight_max": 49.460750579833984,
      "activations/layer11_attention_weight_min": -43.177215576171875,
      "activations/layer12_attention_weight_max": 36.843318939208984,
      "activations/layer12_attention_weight_min": -27.882509231567383,
      "activations/layer13_attention_weight_max": 52.15889358520508,
      "activations/layer13_attention_weight_min": -40.589088439941406,
      "activations/layer14_attention_weight_max": 79.43816375732422,
      "activations/layer14_attention_weight_min": -63.60660171508789,
      "activations/layer15_attention_weight_max": 59.0267448425293,
      "activations/layer15_attention_weight_min": -40.9656982421875,
      "activations/layer16_attention_weight_max": 55.4118766784668,
      "activations/layer16_attention_weight_min": -37.14672088623047,
      "activations/layer17_attention_weight_max": 45.30231475830078,
      "activations/layer17_attention_weight_min": -30.74079132080078,
      "activations/layer18_attention_weight_max": 27.81539535522461,
      "activations/layer18_attention_weight_min": -17.06072998046875,
      "activations/layer19_attention_weight_max": 54.79744338989258,
      "activations/layer19_attention_weight_min": -33.125064849853516,
      "activations/layer1_attention_weight_max": 17.119630813598633,
      "activations/layer1_attention_weight_min": -11.085824012756348,
      "activations/layer20_attention_weight_max": 37.7032356262207,
      "activations/layer20_attention_weight_min": -25.804655075073242,
      "activations/layer21_attention_weight_max": 30.701189041137695,
      "activations/layer21_attention_weight_min": -18.741331100463867,
      "activations/layer22_attention_weight_max": 46.10600662231445,
      "activations/layer22_attention_weight_min": -25.91962432861328,
      "activations/layer23_attention_weight_max": 24.566879272460938,
      "activations/layer23_attention_weight_min": -15.51016902923584,
      "activations/layer2_attention_weight_max": 28.774755477905273,
      "activations/layer2_attention_weight_min": -28.754003524780273,
      "activations/layer3_attention_weight_max": 67.07862091064453,
      "activations/layer3_attention_weight_min": -70.5510025024414,
      "activations/layer4_attention_weight_max": 119.8465805053711,
      "activations/layer4_attention_weight_min": -113.7989730834961,
      "activations/layer5_attention_weight_max": 116.88737487792969,
      "activations/layer5_attention_weight_min": -101.86141204833984,
      "activations/layer6_attention_weight_max": 63.413597106933594,
      "activations/layer6_attention_weight_min": -55.76594924926758,
      "activations/layer7_attention_weight_max": 68.04708099365234,
      "activations/layer7_attention_weight_min": -61.384559631347656,
      "activations/layer8_attention_weight_max": 51.0418815612793,
      "activations/layer8_attention_weight_min": -43.78522491455078,
      "activations/layer9_attention_weight_max": 70.66011810302734,
      "activations/layer9_attention_weight_min": -53.571441650390625,
      "epoch": 1.73,
      "learning_rate": 0.00014028106060606058,
      "loss": 3.0016,
      "step": 29700
    },
    {
      "activations/layer0_attention_weight_max": 15.682141304016113,
      "activations/layer0_attention_weight_min": -11.58194637298584,
      "activations/layer10_attention_weight_max": 60.06686782836914,
      "activations/layer10_attention_weight_min": -47.51871109008789,
      "activations/layer11_attention_weight_max": 49.8807487487793,
      "activations/layer11_attention_weight_min": -40.246124267578125,
      "activations/layer12_attention_weight_max": 54.012794494628906,
      "activations/layer12_attention_weight_min": -34.41104507446289,
      "activations/layer13_attention_weight_max": 66.89336395263672,
      "activations/layer13_attention_weight_min": -45.45299530029297,
      "activations/layer14_attention_weight_max": 94.16512298583984,
      "activations/layer14_attention_weight_min": -62.846221923828125,
      "activations/layer15_attention_weight_max": 66.79570770263672,
      "activations/layer15_attention_weight_min": -41.639732360839844,
      "activations/layer16_attention_weight_max": 69.08026885986328,
      "activations/layer16_attention_weight_min": -36.87119674682617,
      "activations/layer17_attention_weight_max": 53.13206481933594,
      "activations/layer17_attention_weight_min": -30.787099838256836,
      "activations/layer18_attention_weight_max": 35.071380615234375,
      "activations/layer18_attention_weight_min": -21.3288516998291,
      "activations/layer19_attention_weight_max": 79.36505889892578,
      "activations/layer19_attention_weight_min": -35.31119918823242,
      "activations/layer1_attention_weight_max": 17.69428253173828,
      "activations/layer1_attention_weight_min": -10.687893867492676,
      "activations/layer20_attention_weight_max": 60.858760833740234,
      "activations/layer20_attention_weight_min": -25.82052230834961,
      "activations/layer21_attention_weight_max": 39.09223175048828,
      "activations/layer21_attention_weight_min": -22.290767669677734,
      "activations/layer22_attention_weight_max": 55.58880615234375,
      "activations/layer22_attention_weight_min": -29.21352767944336,
      "activations/layer23_attention_weight_max": 31.467193603515625,
      "activations/layer23_attention_weight_min": -18.532316207885742,
      "activations/layer2_attention_weight_max": 28.458919525146484,
      "activations/layer2_attention_weight_min": -29.269880294799805,
      "activations/layer3_attention_weight_max": 69.38520812988281,
      "activations/layer3_attention_weight_min": -73.51131439208984,
      "activations/layer4_attention_weight_max": 124.32612609863281,
      "activations/layer4_attention_weight_min": -119.8397445678711,
      "activations/layer5_attention_weight_max": 118.17520141601562,
      "activations/layer5_attention_weight_min": -104.79330444335938,
      "activations/layer6_attention_weight_max": 63.44943618774414,
      "activations/layer6_attention_weight_min": -57.12959671020508,
      "activations/layer7_attention_weight_max": 63.46195983886719,
      "activations/layer7_attention_weight_min": -62.0661735534668,
      "activations/layer8_attention_weight_max": 55.61326599121094,
      "activations/layer8_attention_weight_min": -46.76547622680664,
      "activations/layer9_attention_weight_max": 72.33848571777344,
      "activations/layer9_attention_weight_min": -59.246437072753906,
      "epoch": 1.73,
      "learning_rate": 0.0001402621212121212,
      "loss": 2.9855,
      "step": 29750
    },
    {
      "activations/layer0_attention_weight_max": 17.064151763916016,
      "activations/layer0_attention_weight_min": -11.439798355102539,
      "activations/layer10_attention_weight_max": 48.12820816040039,
      "activations/layer10_attention_weight_min": -41.72078323364258,
      "activations/layer11_attention_weight_max": 43.38714599609375,
      "activations/layer11_attention_weight_min": -35.84610366821289,
      "activations/layer12_attention_weight_max": 38.272403717041016,
      "activations/layer12_attention_weight_min": -30.97053337097168,
      "activations/layer13_attention_weight_max": 47.865901947021484,
      "activations/layer13_attention_weight_min": -42.36332321166992,
      "activations/layer14_attention_weight_max": 75.44956970214844,
      "activations/layer14_attention_weight_min": -59.511722564697266,
      "activations/layer15_attention_weight_max": 53.110347747802734,
      "activations/layer15_attention_weight_min": -40.96021270751953,
      "activations/layer16_attention_weight_max": 53.364925384521484,
      "activations/layer16_attention_weight_min": -47.305667877197266,
      "activations/layer17_attention_weight_max": 40.772735595703125,
      "activations/layer17_attention_weight_min": -28.87946891784668,
      "activations/layer18_attention_weight_max": 24.659259796142578,
      "activations/layer18_attention_weight_min": -16.163414001464844,
      "activations/layer19_attention_weight_max": 55.68721389770508,
      "activations/layer19_attention_weight_min": -38.833892822265625,
      "activations/layer1_attention_weight_max": 19.171735763549805,
      "activations/layer1_attention_weight_min": -11.543328285217285,
      "activations/layer20_attention_weight_max": 46.3065185546875,
      "activations/layer20_attention_weight_min": -27.441421508789062,
      "activations/layer21_attention_weight_max": 27.994462966918945,
      "activations/layer21_attention_weight_min": -19.2546329498291,
      "activations/layer22_attention_weight_max": 51.20708084106445,
      "activations/layer22_attention_weight_min": -26.912738800048828,
      "activations/layer23_attention_weight_max": 27.64496612548828,
      "activations/layer23_attention_weight_min": -18.3297119140625,
      "activations/layer2_attention_weight_max": 28.5631160736084,
      "activations/layer2_attention_weight_min": -29.07342529296875,
      "activations/layer3_attention_weight_max": 66.74595642089844,
      "activations/layer3_attention_weight_min": -69.91725158691406,
      "activations/layer4_attention_weight_max": 123.41887664794922,
      "activations/layer4_attention_weight_min": -113.48650360107422,
      "activations/layer5_attention_weight_max": 113.45653533935547,
      "activations/layer5_attention_weight_min": -97.76832580566406,
      "activations/layer6_attention_weight_max": 63.66957473754883,
      "activations/layer6_attention_weight_min": -53.275794982910156,
      "activations/layer7_attention_weight_max": 70.8450698852539,
      "activations/layer7_attention_weight_min": -65.35003662109375,
      "activations/layer8_attention_weight_max": 49.25579071044922,
      "activations/layer8_attention_weight_min": -43.19091033935547,
      "activations/layer9_attention_weight_max": 79.9745864868164,
      "activations/layer9_attention_weight_min": -60.17808151245117,
      "epoch": 1.73,
      "learning_rate": 0.0001402431818181818,
      "loss": 2.9927,
      "step": 29800
    },
    {
      "activations/layer0_attention_weight_max": 15.514321327209473,
      "activations/layer0_attention_weight_min": -11.31173038482666,
      "activations/layer10_attention_weight_max": 57.82309341430664,
      "activations/layer10_attention_weight_min": -47.351741790771484,
      "activations/layer11_attention_weight_max": 40.86743927001953,
      "activations/layer11_attention_weight_min": -35.96238708496094,
      "activations/layer12_attention_weight_max": 39.25641632080078,
      "activations/layer12_attention_weight_min": -26.426362991333008,
      "activations/layer13_attention_weight_max": 60.735816955566406,
      "activations/layer13_attention_weight_min": -42.01093292236328,
      "activations/layer14_attention_weight_max": 82.35977935791016,
      "activations/layer14_attention_weight_min": -58.958866119384766,
      "activations/layer15_attention_weight_max": 60.02310562133789,
      "activations/layer15_attention_weight_min": -39.46608352661133,
      "activations/layer16_attention_weight_max": 53.102298736572266,
      "activations/layer16_attention_weight_min": -37.845428466796875,
      "activations/layer17_attention_weight_max": 46.112770080566406,
      "activations/layer17_attention_weight_min": -29.994394302368164,
      "activations/layer18_attention_weight_max": 24.6242733001709,
      "activations/layer18_attention_weight_min": -18.115270614624023,
      "activations/layer19_attention_weight_max": 51.31951904296875,
      "activations/layer19_attention_weight_min": -33.96342849731445,
      "activations/layer1_attention_weight_max": 16.47856330871582,
      "activations/layer1_attention_weight_min": -11.705265045166016,
      "activations/layer20_attention_weight_max": 37.431121826171875,
      "activations/layer20_attention_weight_min": -27.15435028076172,
      "activations/layer21_attention_weight_max": 31.206432342529297,
      "activations/layer21_attention_weight_min": -17.82063102722168,
      "activations/layer22_attention_weight_max": 49.87374496459961,
      "activations/layer22_attention_weight_min": -25.515012741088867,
      "activations/layer23_attention_weight_max": 27.662418365478516,
      "activations/layer23_attention_weight_min": -14.915397644042969,
      "activations/layer2_attention_weight_max": 28.799856185913086,
      "activations/layer2_attention_weight_min": -28.159862518310547,
      "activations/layer3_attention_weight_max": 69.64253997802734,
      "activations/layer3_attention_weight_min": -70.5787582397461,
      "activations/layer4_attention_weight_max": 125.67510223388672,
      "activations/layer4_attention_weight_min": -118.85700988769531,
      "activations/layer5_attention_weight_max": 117.40367889404297,
      "activations/layer5_attention_weight_min": -103.69686889648438,
      "activations/layer6_attention_weight_max": 62.49991226196289,
      "activations/layer6_attention_weight_min": -56.69073486328125,
      "activations/layer7_attention_weight_max": 63.947444915771484,
      "activations/layer7_attention_weight_min": -67.91376495361328,
      "activations/layer8_attention_weight_max": 54.01771545410156,
      "activations/layer8_attention_weight_min": -45.18825149536133,
      "activations/layer9_attention_weight_max": 73.5035171508789,
      "activations/layer9_attention_weight_min": -65.05400848388672,
      "epoch": 1.73,
      "learning_rate": 0.0001402242424242424,
      "loss": 2.984,
      "step": 29850
    },
    {
      "activations/layer0_attention_weight_max": 14.812893867492676,
      "activations/layer0_attention_weight_min": -11.389960289001465,
      "activations/layer10_attention_weight_max": 48.7577018737793,
      "activations/layer10_attention_weight_min": -44.56369400024414,
      "activations/layer11_attention_weight_max": 41.59349822998047,
      "activations/layer11_attention_weight_min": -34.7072868347168,
      "activations/layer12_attention_weight_max": 34.98135757446289,
      "activations/layer12_attention_weight_min": -26.253049850463867,
      "activations/layer13_attention_weight_max": 51.16879653930664,
      "activations/layer13_attention_weight_min": -38.04633712768555,
      "activations/layer14_attention_weight_max": 84.58333587646484,
      "activations/layer14_attention_weight_min": -61.001304626464844,
      "activations/layer15_attention_weight_max": 57.756465911865234,
      "activations/layer15_attention_weight_min": -44.57691955566406,
      "activations/layer16_attention_weight_max": 55.28463363647461,
      "activations/layer16_attention_weight_min": -41.23362731933594,
      "activations/layer17_attention_weight_max": 47.689205169677734,
      "activations/layer17_attention_weight_min": -28.973535537719727,
      "activations/layer18_attention_weight_max": 29.49110221862793,
      "activations/layer18_attention_weight_min": -16.952571868896484,
      "activations/layer19_attention_weight_max": 61.34196853637695,
      "activations/layer19_attention_weight_min": -32.540184020996094,
      "activations/layer1_attention_weight_max": 16.999202728271484,
      "activations/layer1_attention_weight_min": -10.9669771194458,
      "activations/layer20_attention_weight_max": 39.98783874511719,
      "activations/layer20_attention_weight_min": -24.951215744018555,
      "activations/layer21_attention_weight_max": 34.325748443603516,
      "activations/layer21_attention_weight_min": -19.698776245117188,
      "activations/layer22_attention_weight_max": 47.51963424682617,
      "activations/layer22_attention_weight_min": -25.75998878479004,
      "activations/layer23_attention_weight_max": 26.62347984313965,
      "activations/layer23_attention_weight_min": -15.971660614013672,
      "activations/layer2_attention_weight_max": 27.934951782226562,
      "activations/layer2_attention_weight_min": -28.488216400146484,
      "activations/layer3_attention_weight_max": 68.53036499023438,
      "activations/layer3_attention_weight_min": -71.76385498046875,
      "activations/layer4_attention_weight_max": 124.6148681640625,
      "activations/layer4_attention_weight_min": -118.89117431640625,
      "activations/layer5_attention_weight_max": 112.89669036865234,
      "activations/layer5_attention_weight_min": -99.70454406738281,
      "activations/layer6_attention_weight_max": 62.60523223876953,
      "activations/layer6_attention_weight_min": -54.5860710144043,
      "activations/layer7_attention_weight_max": 62.134857177734375,
      "activations/layer7_attention_weight_min": -67.85552215576172,
      "activations/layer8_attention_weight_max": 49.16950607299805,
      "activations/layer8_attention_weight_min": -42.24861526489258,
      "activations/layer9_attention_weight_max": 67.86439514160156,
      "activations/layer9_attention_weight_min": -55.06067657470703,
      "epoch": 1.74,
      "learning_rate": 0.00014020530303030302,
      "loss": 2.9813,
      "step": 29900
    },
    {
      "activations/layer0_attention_weight_max": 16.58527374267578,
      "activations/layer0_attention_weight_min": -11.90438175201416,
      "activations/layer10_attention_weight_max": 52.78564453125,
      "activations/layer10_attention_weight_min": -44.28771209716797,
      "activations/layer11_attention_weight_max": 48.265830993652344,
      "activations/layer11_attention_weight_min": -34.379547119140625,
      "activations/layer12_attention_weight_max": 56.4950065612793,
      "activations/layer12_attention_weight_min": -36.725101470947266,
      "activations/layer13_attention_weight_max": 59.13233184814453,
      "activations/layer13_attention_weight_min": -39.59154510498047,
      "activations/layer14_attention_weight_max": 87.52409362792969,
      "activations/layer14_attention_weight_min": -61.536537170410156,
      "activations/layer15_attention_weight_max": 58.470741271972656,
      "activations/layer15_attention_weight_min": -36.687713623046875,
      "activations/layer16_attention_weight_max": 55.09238052368164,
      "activations/layer16_attention_weight_min": -36.80202865600586,
      "activations/layer17_attention_weight_max": 50.02907180786133,
      "activations/layer17_attention_weight_min": -26.41864013671875,
      "activations/layer18_attention_weight_max": 26.231792449951172,
      "activations/layer18_attention_weight_min": -18.26959800720215,
      "activations/layer19_attention_weight_max": 56.331661224365234,
      "activations/layer19_attention_weight_min": -30.856046676635742,
      "activations/layer1_attention_weight_max": 17.610557556152344,
      "activations/layer1_attention_weight_min": -11.968668937683105,
      "activations/layer20_attention_weight_max": 42.252803802490234,
      "activations/layer20_attention_weight_min": -23.767112731933594,
      "activations/layer21_attention_weight_max": 30.979698181152344,
      "activations/layer21_attention_weight_min": -17.501068115234375,
      "activations/layer22_attention_weight_max": 45.52114486694336,
      "activations/layer22_attention_weight_min": -23.98065185546875,
      "activations/layer23_attention_weight_max": 24.774765014648438,
      "activations/layer23_attention_weight_min": -17.755125045776367,
      "activations/layer2_attention_weight_max": 29.088516235351562,
      "activations/layer2_attention_weight_min": -28.872249603271484,
      "activations/layer3_attention_weight_max": 70.96919250488281,
      "activations/layer3_attention_weight_min": -71.6705322265625,
      "activations/layer4_attention_weight_max": 128.12025451660156,
      "activations/layer4_attention_weight_min": -118.58226776123047,
      "activations/layer5_attention_weight_max": 115.08553314208984,
      "activations/layer5_attention_weight_min": -105.44756317138672,
      "activations/layer6_attention_weight_max": 65.03596496582031,
      "activations/layer6_attention_weight_min": -54.87578201293945,
      "activations/layer7_attention_weight_max": 72.62252807617188,
      "activations/layer7_attention_weight_min": -61.12373352050781,
      "activations/layer8_attention_weight_max": 54.006752014160156,
      "activations/layer8_attention_weight_min": -45.46417999267578,
      "activations/layer9_attention_weight_max": 88.94644927978516,
      "activations/layer9_attention_weight_min": -69.3360366821289,
      "epoch": 1.74,
      "learning_rate": 0.00014018636363636363,
      "loss": 2.9973,
      "step": 29950
    },
    {
      "activations/layer0_attention_weight_max": 16.573774337768555,
      "activations/layer0_attention_weight_min": -11.857447624206543,
      "activations/layer10_attention_weight_max": 52.376461029052734,
      "activations/layer10_attention_weight_min": -50.57378005981445,
      "activations/layer11_attention_weight_max": 40.79981231689453,
      "activations/layer11_attention_weight_min": -35.746612548828125,
      "activations/layer12_attention_weight_max": 49.92201614379883,
      "activations/layer12_attention_weight_min": -29.389694213867188,
      "activations/layer13_attention_weight_max": 52.21628952026367,
      "activations/layer13_attention_weight_min": -38.425289154052734,
      "activations/layer14_attention_weight_max": 76.59960174560547,
      "activations/layer14_attention_weight_min": -55.506683349609375,
      "activations/layer15_attention_weight_max": 54.673606872558594,
      "activations/layer15_attention_weight_min": -41.75946807861328,
      "activations/layer16_attention_weight_max": 58.2336540222168,
      "activations/layer16_attention_weight_min": -38.69234085083008,
      "activations/layer17_attention_weight_max": 49.17847442626953,
      "activations/layer17_attention_weight_min": -30.89990997314453,
      "activations/layer18_attention_weight_max": 25.976266860961914,
      "activations/layer18_attention_weight_min": -17.77999496459961,
      "activations/layer19_attention_weight_max": 54.361732482910156,
      "activations/layer19_attention_weight_min": -31.919157028198242,
      "activations/layer1_attention_weight_max": 16.998685836791992,
      "activations/layer1_attention_weight_min": -12.60513973236084,
      "activations/layer20_attention_weight_max": 41.394832611083984,
      "activations/layer20_attention_weight_min": -27.536903381347656,
      "activations/layer21_attention_weight_max": 33.27526092529297,
      "activations/layer21_attention_weight_min": -19.663164138793945,
      "activations/layer22_attention_weight_max": 44.8551025390625,
      "activations/layer22_attention_weight_min": -29.11277198791504,
      "activations/layer23_attention_weight_max": 24.671253204345703,
      "activations/layer23_attention_weight_min": -14.862689971923828,
      "activations/layer2_attention_weight_max": 29.52545928955078,
      "activations/layer2_attention_weight_min": -26.73884391784668,
      "activations/layer3_attention_weight_max": 68.51799011230469,
      "activations/layer3_attention_weight_min": -69.1343002319336,
      "activations/layer4_attention_weight_max": 133.25054931640625,
      "activations/layer4_attention_weight_min": -114.5523452758789,
      "activations/layer5_attention_weight_max": 133.2744903564453,
      "activations/layer5_attention_weight_min": -103.55653381347656,
      "activations/layer6_attention_weight_max": 69.15636444091797,
      "activations/layer6_attention_weight_min": -54.921207427978516,
      "activations/layer7_attention_weight_max": 67.40589141845703,
      "activations/layer7_attention_weight_min": -63.765480041503906,
      "activations/layer8_attention_weight_max": 54.06233596801758,
      "activations/layer8_attention_weight_min": -42.994205474853516,
      "activations/layer9_attention_weight_max": 66.33809661865234,
      "activations/layer9_attention_weight_min": -56.62359619140625,
      "epoch": 1.74,
      "learning_rate": 0.00014016742424242422,
      "loss": 2.9767,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_loss": 2.9375,
      "eval_runtime": 8.5261,
      "eval_samples_per_second": 503.63,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_openwebtext_loss": 2.9375,
      "eval_openwebtext_ppl": 18.868615759264884,
      "eval_openwebtext_runtime": 8.5261,
      "eval_openwebtext_samples_per_second": 503.63,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_wikitext_loss": 3.21484375,
      "eval_wikitext_ppl": 24.89940107578365,
      "eval_wikitext_runtime": 2.0059,
      "eval_wikitext_samples_per_second": 227.335,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_lambada_loss": 3.263671875,
      "eval_lambada_ppl": 26.14536361045317,
      "eval_lambada_runtime": 9.5792,
      "eval_lambada_samples_per_second": 508.286,
      "step": 30000
    },
    {
      "activations/layer0_attention_weight_max": 15.232293128967285,
      "activations/layer0_attention_weight_min": -11.62181568145752,
      "activations/layer10_attention_weight_max": 52.67829513549805,
      "activations/layer10_attention_weight_min": -41.9034423828125,
      "activations/layer11_attention_weight_max": 45.66682434082031,
      "activations/layer11_attention_weight_min": -38.864891052246094,
      "activations/layer12_attention_weight_max": 55.35136413574219,
      "activations/layer12_attention_weight_min": -34.92287826538086,
      "activations/layer13_attention_weight_max": 53.91741180419922,
      "activations/layer13_attention_weight_min": -36.84004211425781,
      "activations/layer14_attention_weight_max": 71.03939056396484,
      "activations/layer14_attention_weight_min": -55.915157318115234,
      "activations/layer15_attention_weight_max": 50.815673828125,
      "activations/layer15_attention_weight_min": -38.37993240356445,
      "activations/layer16_attention_weight_max": 48.24546432495117,
      "activations/layer16_attention_weight_min": -35.440120697021484,
      "activations/layer17_attention_weight_max": 43.609046936035156,
      "activations/layer17_attention_weight_min": -27.291505813598633,
      "activations/layer18_attention_weight_max": 28.046602249145508,
      "activations/layer18_attention_weight_min": -16.4649600982666,
      "activations/layer19_attention_weight_max": 51.018558502197266,
      "activations/layer19_attention_weight_min": -31.464370727539062,
      "activations/layer1_attention_weight_max": 17.5804443359375,
      "activations/layer1_attention_weight_min": -12.258234024047852,
      "activations/layer20_attention_weight_max": 41.13240432739258,
      "activations/layer20_attention_weight_min": -24.023765563964844,
      "activations/layer21_attention_weight_max": 28.910465240478516,
      "activations/layer21_attention_weight_min": -18.9092960357666,
      "activations/layer22_attention_weight_max": 40.90632629394531,
      "activations/layer22_attention_weight_min": -25.526596069335938,
      "activations/layer23_attention_weight_max": 25.154972076416016,
      "activations/layer23_attention_weight_min": -15.682180404663086,
      "activations/layer2_attention_weight_max": 27.824188232421875,
      "activations/layer2_attention_weight_min": -28.263181686401367,
      "activations/layer3_attention_weight_max": 70.25224304199219,
      "activations/layer3_attention_weight_min": -69.83506774902344,
      "activations/layer4_attention_weight_max": 126.09964752197266,
      "activations/layer4_attention_weight_min": -118.87943267822266,
      "activations/layer5_attention_weight_max": 119.41459655761719,
      "activations/layer5_attention_weight_min": -105.09846496582031,
      "activations/layer6_attention_weight_max": 63.095970153808594,
      "activations/layer6_attention_weight_min": -55.502708435058594,
      "activations/layer7_attention_weight_max": 65.96483612060547,
      "activations/layer7_attention_weight_min": -66.83706665039062,
      "activations/layer8_attention_weight_max": 55.98129653930664,
      "activations/layer8_attention_weight_min": -48.79650115966797,
      "activations/layer9_attention_weight_max": 89.21809387207031,
      "activations/layer9_attention_weight_min": -68.888427734375,
      "epoch": 1.75,
      "learning_rate": 0.00014014848484848483,
      "loss": 2.999,
      "step": 30050
    },
    {
      "activations/layer0_attention_weight_max": 15.744987487792969,
      "activations/layer0_attention_weight_min": -12.203153610229492,
      "activations/layer10_attention_weight_max": 53.45957565307617,
      "activations/layer10_attention_weight_min": -45.420867919921875,
      "activations/layer11_attention_weight_max": 42.04537582397461,
      "activations/layer11_attention_weight_min": -35.87244415283203,
      "activations/layer12_attention_weight_max": 42.135284423828125,
      "activations/layer12_attention_weight_min": -30.723100662231445,
      "activations/layer13_attention_weight_max": 60.23283767700195,
      "activations/layer13_attention_weight_min": -46.1871223449707,
      "activations/layer14_attention_weight_max": 87.81780242919922,
      "activations/layer14_attention_weight_min": -72.79866027832031,
      "activations/layer15_attention_weight_max": 54.08379364013672,
      "activations/layer15_attention_weight_min": -40.8918571472168,
      "activations/layer16_attention_weight_max": 58.30599594116211,
      "activations/layer16_attention_weight_min": -45.865108489990234,
      "activations/layer17_attention_weight_max": 43.337406158447266,
      "activations/layer17_attention_weight_min": -31.30357551574707,
      "activations/layer18_attention_weight_max": 22.317230224609375,
      "activations/layer18_attention_weight_min": -17.482070922851562,
      "activations/layer19_attention_weight_max": 54.37506866455078,
      "activations/layer19_attention_weight_min": -38.72264862060547,
      "activations/layer1_attention_weight_max": 17.35270118713379,
      "activations/layer1_attention_weight_min": -11.1417875289917,
      "activations/layer20_attention_weight_max": 43.106224060058594,
      "activations/layer20_attention_weight_min": -27.124847412109375,
      "activations/layer21_attention_weight_max": 26.388290405273438,
      "activations/layer21_attention_weight_min": -17.0413761138916,
      "activations/layer22_attention_weight_max": 40.34006118774414,
      "activations/layer22_attention_weight_min": -28.772581100463867,
      "activations/layer23_attention_weight_max": 23.376895904541016,
      "activations/layer23_attention_weight_min": -16.027603149414062,
      "activations/layer2_attention_weight_max": 29.930633544921875,
      "activations/layer2_attention_weight_min": -29.09491729736328,
      "activations/layer3_attention_weight_max": 70.1076431274414,
      "activations/layer3_attention_weight_min": -69.81107330322266,
      "activations/layer4_attention_weight_max": 123.44496154785156,
      "activations/layer4_attention_weight_min": -113.46553039550781,
      "activations/layer5_attention_weight_max": 118.15752410888672,
      "activations/layer5_attention_weight_min": -102.28927612304688,
      "activations/layer6_attention_weight_max": 62.08406448364258,
      "activations/layer6_attention_weight_min": -53.03264236450195,
      "activations/layer7_attention_weight_max": 69.10784912109375,
      "activations/layer7_attention_weight_min": -61.0616340637207,
      "activations/layer8_attention_weight_max": 51.61349105834961,
      "activations/layer8_attention_weight_min": -41.86360549926758,
      "activations/layer9_attention_weight_max": 68.63298797607422,
      "activations/layer9_attention_weight_min": -57.4183349609375,
      "epoch": 1.75,
      "learning_rate": 0.00014012954545454545,
      "loss": 2.9958,
      "step": 30100
    },
    {
      "activations/layer0_attention_weight_max": 16.332050323486328,
      "activations/layer0_attention_weight_min": -11.927170753479004,
      "activations/layer10_attention_weight_max": 50.26554870605469,
      "activations/layer10_attention_weight_min": -45.09827423095703,
      "activations/layer11_attention_weight_max": 39.52952575683594,
      "activations/layer11_attention_weight_min": -32.2054328918457,
      "activations/layer12_attention_weight_max": 30.409875869750977,
      "activations/layer12_attention_weight_min": -29.227073669433594,
      "activations/layer13_attention_weight_max": 50.15913391113281,
      "activations/layer13_attention_weight_min": -39.5023307800293,
      "activations/layer14_attention_weight_max": 64.06243896484375,
      "activations/layer14_attention_weight_min": -55.37262725830078,
      "activations/layer15_attention_weight_max": 51.29022216796875,
      "activations/layer15_attention_weight_min": -36.122291564941406,
      "activations/layer16_attention_weight_max": 47.07483673095703,
      "activations/layer16_attention_weight_min": -40.29300308227539,
      "activations/layer17_attention_weight_max": 41.44114303588867,
      "activations/layer17_attention_weight_min": -30.700056076049805,
      "activations/layer18_attention_weight_max": 29.706722259521484,
      "activations/layer18_attention_weight_min": -20.72120475769043,
      "activations/layer19_attention_weight_max": 50.839778900146484,
      "activations/layer19_attention_weight_min": -35.951839447021484,
      "activations/layer1_attention_weight_max": 18.7800350189209,
      "activations/layer1_attention_weight_min": -10.787552833557129,
      "activations/layer20_attention_weight_max": 36.0877799987793,
      "activations/layer20_attention_weight_min": -29.13554573059082,
      "activations/layer21_attention_weight_max": 28.596519470214844,
      "activations/layer21_attention_weight_min": -20.732824325561523,
      "activations/layer22_attention_weight_max": 41.463623046875,
      "activations/layer22_attention_weight_min": -30.313154220581055,
      "activations/layer23_attention_weight_max": 25.251869201660156,
      "activations/layer23_attention_weight_min": -17.52916717529297,
      "activations/layer2_attention_weight_max": 27.834257125854492,
      "activations/layer2_attention_weight_min": -28.468833923339844,
      "activations/layer3_attention_weight_max": 68.4409408569336,
      "activations/layer3_attention_weight_min": -71.30309295654297,
      "activations/layer4_attention_weight_max": 122.4488525390625,
      "activations/layer4_attention_weight_min": -115.47340393066406,
      "activations/layer5_attention_weight_max": 114.43893432617188,
      "activations/layer5_attention_weight_min": -98.79534149169922,
      "activations/layer6_attention_weight_max": 60.73154830932617,
      "activations/layer6_attention_weight_min": -53.850852966308594,
      "activations/layer7_attention_weight_max": 63.61697769165039,
      "activations/layer7_attention_weight_min": -61.06059265136719,
      "activations/layer8_attention_weight_max": 48.960052490234375,
      "activations/layer8_attention_weight_min": -40.05965042114258,
      "activations/layer9_attention_weight_max": 64.09908294677734,
      "activations/layer9_attention_weight_min": -57.65821075439453,
      "epoch": 1.75,
      "learning_rate": 0.00014011060606060604,
      "loss": 3.0122,
      "step": 30150
    },
    {
      "activations/layer0_attention_weight_max": 15.96357536315918,
      "activations/layer0_attention_weight_min": -11.689088821411133,
      "activations/layer10_attention_weight_max": 54.93255615234375,
      "activations/layer10_attention_weight_min": -48.4447135925293,
      "activations/layer11_attention_weight_max": 54.670127868652344,
      "activations/layer11_attention_weight_min": -35.52095031738281,
      "activations/layer12_attention_weight_max": 43.69261932373047,
      "activations/layer12_attention_weight_min": -25.388242721557617,
      "activations/layer13_attention_weight_max": 64.69070434570312,
      "activations/layer13_attention_weight_min": -40.226356506347656,
      "activations/layer14_attention_weight_max": 98.94635772705078,
      "activations/layer14_attention_weight_min": -63.04096984863281,
      "activations/layer15_attention_weight_max": 52.550811767578125,
      "activations/layer15_attention_weight_min": -38.075992584228516,
      "activations/layer16_attention_weight_max": 62.60995864868164,
      "activations/layer16_attention_weight_min": -36.54813766479492,
      "activations/layer17_attention_weight_max": 40.880882263183594,
      "activations/layer17_attention_weight_min": -28.098892211914062,
      "activations/layer18_attention_weight_max": 31.92123031616211,
      "activations/layer18_attention_weight_min": -18.224348068237305,
      "activations/layer19_attention_weight_max": 55.265621185302734,
      "activations/layer19_attention_weight_min": -34.520111083984375,
      "activations/layer1_attention_weight_max": 16.955310821533203,
      "activations/layer1_attention_weight_min": -11.668063163757324,
      "activations/layer20_attention_weight_max": 46.62514877319336,
      "activations/layer20_attention_weight_min": -24.384489059448242,
      "activations/layer21_attention_weight_max": 30.419490814208984,
      "activations/layer21_attention_weight_min": -18.017316818237305,
      "activations/layer22_attention_weight_max": 45.35763931274414,
      "activations/layer22_attention_weight_min": -25.356613159179688,
      "activations/layer23_attention_weight_max": 25.794872283935547,
      "activations/layer23_attention_weight_min": -16.024715423583984,
      "activations/layer2_attention_weight_max": 27.87373924255371,
      "activations/layer2_attention_weight_min": -28.17430877685547,
      "activations/layer3_attention_weight_max": 71.57032012939453,
      "activations/layer3_attention_weight_min": -72.82209777832031,
      "activations/layer4_attention_weight_max": 127.27861785888672,
      "activations/layer4_attention_weight_min": -120.69620513916016,
      "activations/layer5_attention_weight_max": 123.95567321777344,
      "activations/layer5_attention_weight_min": -111.90746307373047,
      "activations/layer6_attention_weight_max": 65.8528060913086,
      "activations/layer6_attention_weight_min": -59.55514907836914,
      "activations/layer7_attention_weight_max": 76.33308410644531,
      "activations/layer7_attention_weight_min": -73.81717681884766,
      "activations/layer8_attention_weight_max": 67.30171966552734,
      "activations/layer8_attention_weight_min": -49.13716125488281,
      "activations/layer9_attention_weight_max": 93.21769714355469,
      "activations/layer9_attention_weight_min": -73.48413848876953,
      "epoch": 1.75,
      "learning_rate": 0.00014009166666666665,
      "loss": 2.9672,
      "step": 30200
    },
    {
      "activations/layer0_attention_weight_max": 17.066410064697266,
      "activations/layer0_attention_weight_min": -11.81100082397461,
      "activations/layer10_attention_weight_max": 48.76542282104492,
      "activations/layer10_attention_weight_min": -43.741764068603516,
      "activations/layer11_attention_weight_max": 39.57427215576172,
      "activations/layer11_attention_weight_min": -36.29713439941406,
      "activations/layer12_attention_weight_max": 37.86026382446289,
      "activations/layer12_attention_weight_min": -27.548547744750977,
      "activations/layer13_attention_weight_max": 50.73777770996094,
      "activations/layer13_attention_weight_min": -36.74448776245117,
      "activations/layer14_attention_weight_max": 83.98367309570312,
      "activations/layer14_attention_weight_min": -63.37584686279297,
      "activations/layer15_attention_weight_max": 58.28761672973633,
      "activations/layer15_attention_weight_min": -41.6595573425293,
      "activations/layer16_attention_weight_max": 56.49607467651367,
      "activations/layer16_attention_weight_min": -38.864501953125,
      "activations/layer17_attention_weight_max": 51.682682037353516,
      "activations/layer17_attention_weight_min": -29.429744720458984,
      "activations/layer18_attention_weight_max": 30.72907829284668,
      "activations/layer18_attention_weight_min": -17.43654441833496,
      "activations/layer19_attention_weight_max": 57.98352813720703,
      "activations/layer19_attention_weight_min": -32.048885345458984,
      "activations/layer1_attention_weight_max": 18.728818893432617,
      "activations/layer1_attention_weight_min": -11.8120756149292,
      "activations/layer20_attention_weight_max": 43.468780517578125,
      "activations/layer20_attention_weight_min": -24.596092224121094,
      "activations/layer21_attention_weight_max": 41.141544342041016,
      "activations/layer21_attention_weight_min": -19.983198165893555,
      "activations/layer22_attention_weight_max": 51.359195709228516,
      "activations/layer22_attention_weight_min": -26.69104766845703,
      "activations/layer23_attention_weight_max": 28.95870018005371,
      "activations/layer23_attention_weight_min": -15.804115295410156,
      "activations/layer2_attention_weight_max": 28.269636154174805,
      "activations/layer2_attention_weight_min": -27.47596549987793,
      "activations/layer3_attention_weight_max": 67.25292205810547,
      "activations/layer3_attention_weight_min": -71.73865509033203,
      "activations/layer4_attention_weight_max": 128.89231872558594,
      "activations/layer4_attention_weight_min": -114.2645263671875,
      "activations/layer5_attention_weight_max": 127.62123107910156,
      "activations/layer5_attention_weight_min": -104.52108001708984,
      "activations/layer6_attention_weight_max": 63.441612243652344,
      "activations/layer6_attention_weight_min": -52.75749588012695,
      "activations/layer7_attention_weight_max": 64.43090057373047,
      "activations/layer7_attention_weight_min": -63.66361618041992,
      "activations/layer8_attention_weight_max": 49.24861526489258,
      "activations/layer8_attention_weight_min": -41.355098724365234,
      "activations/layer9_attention_weight_max": 72.52510070800781,
      "activations/layer9_attention_weight_min": -63.787532806396484,
      "epoch": 1.76,
      "learning_rate": 0.00014007272727272724,
      "loss": 2.9919,
      "step": 30250
    },
    {
      "activations/layer0_attention_weight_max": 16.36647605895996,
      "activations/layer0_attention_weight_min": -10.906201362609863,
      "activations/layer10_attention_weight_max": 63.99964141845703,
      "activations/layer10_attention_weight_min": -54.27880859375,
      "activations/layer11_attention_weight_max": 50.01768112182617,
      "activations/layer11_attention_weight_min": -45.04435348510742,
      "activations/layer12_attention_weight_max": 33.33856964111328,
      "activations/layer12_attention_weight_min": -30.549114227294922,
      "activations/layer13_attention_weight_max": 57.84242630004883,
      "activations/layer13_attention_weight_min": -39.65149688720703,
      "activations/layer14_attention_weight_max": 79.08306884765625,
      "activations/layer14_attention_weight_min": -59.3830680847168,
      "activations/layer15_attention_weight_max": 61.03511428833008,
      "activations/layer15_attention_weight_min": -38.50457763671875,
      "activations/layer16_attention_weight_max": 49.994686126708984,
      "activations/layer16_attention_weight_min": -38.35436248779297,
      "activations/layer17_attention_weight_max": 44.48630142211914,
      "activations/layer17_attention_weight_min": -29.302730560302734,
      "activations/layer18_attention_weight_max": 23.75786590576172,
      "activations/layer18_attention_weight_min": -17.229671478271484,
      "activations/layer19_attention_weight_max": 48.324520111083984,
      "activations/layer19_attention_weight_min": -34.848690032958984,
      "activations/layer1_attention_weight_max": 18.835302352905273,
      "activations/layer1_attention_weight_min": -10.926236152648926,
      "activations/layer20_attention_weight_max": 36.406497955322266,
      "activations/layer20_attention_weight_min": -24.725757598876953,
      "activations/layer21_attention_weight_max": 28.65781021118164,
      "activations/layer21_attention_weight_min": -20.316701889038086,
      "activations/layer22_attention_weight_max": 40.104705810546875,
      "activations/layer22_attention_weight_min": -24.339279174804688,
      "activations/layer23_attention_weight_max": 25.384199142456055,
      "activations/layer23_attention_weight_min": -14.330774307250977,
      "activations/layer2_attention_weight_max": 28.01732635498047,
      "activations/layer2_attention_weight_min": -26.51934814453125,
      "activations/layer3_attention_weight_max": 69.86283111572266,
      "activations/layer3_attention_weight_min": -69.59107208251953,
      "activations/layer4_attention_weight_max": 129.4425048828125,
      "activations/layer4_attention_weight_min": -117.8410873413086,
      "activations/layer5_attention_weight_max": 122.36897277832031,
      "activations/layer5_attention_weight_min": -108.12895202636719,
      "activations/layer6_attention_weight_max": 68.08037567138672,
      "activations/layer6_attention_weight_min": -56.37389373779297,
      "activations/layer7_attention_weight_max": 68.11019897460938,
      "activations/layer7_attention_weight_min": -63.66172409057617,
      "activations/layer8_attention_weight_max": 55.50664520263672,
      "activations/layer8_attention_weight_min": -49.73237609863281,
      "activations/layer9_attention_weight_max": 78.47089385986328,
      "activations/layer9_attention_weight_min": -63.29071807861328,
      "epoch": 1.76,
      "learning_rate": 0.00014005378787878786,
      "loss": 2.9955,
      "step": 30300
    },
    {
      "activations/layer0_attention_weight_max": 15.635125160217285,
      "activations/layer0_attention_weight_min": -11.108838081359863,
      "activations/layer10_attention_weight_max": 53.01346206665039,
      "activations/layer10_attention_weight_min": -47.027523040771484,
      "activations/layer11_attention_weight_max": 41.98334503173828,
      "activations/layer11_attention_weight_min": -37.22217559814453,
      "activations/layer12_attention_weight_max": 56.45915222167969,
      "activations/layer12_attention_weight_min": -29.558656692504883,
      "activations/layer13_attention_weight_max": 44.585514068603516,
      "activations/layer13_attention_weight_min": -35.53542709350586,
      "activations/layer14_attention_weight_max": 67.3162612915039,
      "activations/layer14_attention_weight_min": -58.42808151245117,
      "activations/layer15_attention_weight_max": 47.06884002685547,
      "activations/layer15_attention_weight_min": -39.839168548583984,
      "activations/layer16_attention_weight_max": 51.608238220214844,
      "activations/layer16_attention_weight_min": -37.34571838378906,
      "activations/layer17_attention_weight_max": 39.83377456665039,
      "activations/layer17_attention_weight_min": -33.975433349609375,
      "activations/layer18_attention_weight_max": 24.782421112060547,
      "activations/layer18_attention_weight_min": -17.791139602661133,
      "activations/layer19_attention_weight_max": 52.111202239990234,
      "activations/layer19_attention_weight_min": -33.80099105834961,
      "activations/layer1_attention_weight_max": 17.44375228881836,
      "activations/layer1_attention_weight_min": -11.146695137023926,
      "activations/layer20_attention_weight_max": 33.739498138427734,
      "activations/layer20_attention_weight_min": -27.14675521850586,
      "activations/layer21_attention_weight_max": 27.325834274291992,
      "activations/layer21_attention_weight_min": -19.886568069458008,
      "activations/layer22_attention_weight_max": 38.43132400512695,
      "activations/layer22_attention_weight_min": -25.91480827331543,
      "activations/layer23_attention_weight_max": 24.12080192565918,
      "activations/layer23_attention_weight_min": -15.206977844238281,
      "activations/layer2_attention_weight_max": 27.606887817382812,
      "activations/layer2_attention_weight_min": -27.13643455505371,
      "activations/layer3_attention_weight_max": 68.59957885742188,
      "activations/layer3_attention_weight_min": -72.59868621826172,
      "activations/layer4_attention_weight_max": 122.72879791259766,
      "activations/layer4_attention_weight_min": -113.99144744873047,
      "activations/layer5_attention_weight_max": 116.93536376953125,
      "activations/layer5_attention_weight_min": -97.55955505371094,
      "activations/layer6_attention_weight_max": 61.08296585083008,
      "activations/layer6_attention_weight_min": -53.674842834472656,
      "activations/layer7_attention_weight_max": 65.39482879638672,
      "activations/layer7_attention_weight_min": -61.43354415893555,
      "activations/layer8_attention_weight_max": 49.75191116333008,
      "activations/layer8_attention_weight_min": -44.40464782714844,
      "activations/layer9_attention_weight_max": 69.61245727539062,
      "activations/layer9_attention_weight_min": -58.88961410522461,
      "epoch": 1.76,
      "learning_rate": 0.00014003484848484847,
      "loss": 2.9855,
      "step": 30350
    },
    {
      "activations/layer0_attention_weight_max": 16.8131160736084,
      "activations/layer0_attention_weight_min": -11.294864654541016,
      "activations/layer10_attention_weight_max": 57.64970397949219,
      "activations/layer10_attention_weight_min": -47.965087890625,
      "activations/layer11_attention_weight_max": 45.79331970214844,
      "activations/layer11_attention_weight_min": -38.47265625,
      "activations/layer12_attention_weight_max": 39.96736145019531,
      "activations/layer12_attention_weight_min": -27.979867935180664,
      "activations/layer13_attention_weight_max": 46.293190002441406,
      "activations/layer13_attention_weight_min": -35.06240463256836,
      "activations/layer14_attention_weight_max": 73.40534210205078,
      "activations/layer14_attention_weight_min": -56.1512565612793,
      "activations/layer15_attention_weight_max": 57.94935607910156,
      "activations/layer15_attention_weight_min": -41.07353210449219,
      "activations/layer16_attention_weight_max": 52.73674774169922,
      "activations/layer16_attention_weight_min": -39.358306884765625,
      "activations/layer17_attention_weight_max": 49.81888961791992,
      "activations/layer17_attention_weight_min": -35.168296813964844,
      "activations/layer18_attention_weight_max": 27.244577407836914,
      "activations/layer18_attention_weight_min": -17.829818725585938,
      "activations/layer19_attention_weight_max": 53.314205169677734,
      "activations/layer19_attention_weight_min": -33.41025161743164,
      "activations/layer1_attention_weight_max": 17.982044219970703,
      "activations/layer1_attention_weight_min": -11.061331748962402,
      "activations/layer20_attention_weight_max": 40.674072265625,
      "activations/layer20_attention_weight_min": -25.2213077545166,
      "activations/layer21_attention_weight_max": 29.22442626953125,
      "activations/layer21_attention_weight_min": -19.475217819213867,
      "activations/layer22_attention_weight_max": 45.41827392578125,
      "activations/layer22_attention_weight_min": -26.079126358032227,
      "activations/layer23_attention_weight_max": 26.799009323120117,
      "activations/layer23_attention_weight_min": -15.84269905090332,
      "activations/layer2_attention_weight_max": 28.011280059814453,
      "activations/layer2_attention_weight_min": -27.74660873413086,
      "activations/layer3_attention_weight_max": 67.62358093261719,
      "activations/layer3_attention_weight_min": -69.5075912475586,
      "activations/layer4_attention_weight_max": 124.22513580322266,
      "activations/layer4_attention_weight_min": -112.30218505859375,
      "activations/layer5_attention_weight_max": 114.55984497070312,
      "activations/layer5_attention_weight_min": -99.63838958740234,
      "activations/layer6_attention_weight_max": 60.61067199707031,
      "activations/layer6_attention_weight_min": -53.077754974365234,
      "activations/layer7_attention_weight_max": 61.766788482666016,
      "activations/layer7_attention_weight_min": -63.9359130859375,
      "activations/layer8_attention_weight_max": 46.43742370605469,
      "activations/layer8_attention_weight_min": -42.9617805480957,
      "activations/layer9_attention_weight_max": 75.01290893554688,
      "activations/layer9_attention_weight_min": -68.82642364501953,
      "epoch": 1.77,
      "learning_rate": 0.00014001590909090909,
      "loss": 2.9887,
      "step": 30400
    },
    {
      "activations/layer0_attention_weight_max": 16.572086334228516,
      "activations/layer0_attention_weight_min": -11.563634872436523,
      "activations/layer10_attention_weight_max": 60.885833740234375,
      "activations/layer10_attention_weight_min": -48.22847366333008,
      "activations/layer11_attention_weight_max": 44.47496795654297,
      "activations/layer11_attention_weight_min": -39.38270568847656,
      "activations/layer12_attention_weight_max": 48.9406852722168,
      "activations/layer12_attention_weight_min": -36.625572204589844,
      "activations/layer13_attention_weight_max": 54.727088928222656,
      "activations/layer13_attention_weight_min": -39.337032318115234,
      "activations/layer14_attention_weight_max": 77.93569946289062,
      "activations/layer14_attention_weight_min": -60.36123275756836,
      "activations/layer15_attention_weight_max": 54.504676818847656,
      "activations/layer15_attention_weight_min": -39.50825119018555,
      "activations/layer16_attention_weight_max": 54.57674026489258,
      "activations/layer16_attention_weight_min": -38.89033508300781,
      "activations/layer17_attention_weight_max": 48.827735900878906,
      "activations/layer17_attention_weight_min": -29.835914611816406,
      "activations/layer18_attention_weight_max": 28.56560707092285,
      "activations/layer18_attention_weight_min": -16.130212783813477,
      "activations/layer19_attention_weight_max": 52.656982421875,
      "activations/layer19_attention_weight_min": -35.53812789916992,
      "activations/layer1_attention_weight_max": 16.962011337280273,
      "activations/layer1_attention_weight_min": -11.044819831848145,
      "activations/layer20_attention_weight_max": 40.91675567626953,
      "activations/layer20_attention_weight_min": -26.14858055114746,
      "activations/layer21_attention_weight_max": 34.28148651123047,
      "activations/layer21_attention_weight_min": -20.263416290283203,
      "activations/layer22_attention_weight_max": 46.72266387939453,
      "activations/layer22_attention_weight_min": -30.624048233032227,
      "activations/layer23_attention_weight_max": 23.619136810302734,
      "activations/layer23_attention_weight_min": -16.7903995513916,
      "activations/layer2_attention_weight_max": 28.19856834411621,
      "activations/layer2_attention_weight_min": -27.515653610229492,
      "activations/layer3_attention_weight_max": 69.7018814086914,
      "activations/layer3_attention_weight_min": -72.38153076171875,
      "activations/layer4_attention_weight_max": 125.25128173828125,
      "activations/layer4_attention_weight_min": -118.26063537597656,
      "activations/layer5_attention_weight_max": 119.09142303466797,
      "activations/layer5_attention_weight_min": -106.2212142944336,
      "activations/layer6_attention_weight_max": 64.02288818359375,
      "activations/layer6_attention_weight_min": -56.83723831176758,
      "activations/layer7_attention_weight_max": 64.08604431152344,
      "activations/layer7_attention_weight_min": -59.548606872558594,
      "activations/layer8_attention_weight_max": 55.83232498168945,
      "activations/layer8_attention_weight_min": -45.881431579589844,
      "activations/layer9_attention_weight_max": 76.29206848144531,
      "activations/layer9_attention_weight_min": -63.273719787597656,
      "epoch": 1.77,
      "learning_rate": 0.00013999696969696967,
      "loss": 2.9845,
      "step": 30450
    },
    {
      "activations/layer0_attention_weight_max": 17.189071655273438,
      "activations/layer0_attention_weight_min": -11.089478492736816,
      "activations/layer10_attention_weight_max": 57.40208053588867,
      "activations/layer10_attention_weight_min": -45.5848503112793,
      "activations/layer11_attention_weight_max": 41.967830657958984,
      "activations/layer11_attention_weight_min": -33.18342590332031,
      "activations/layer12_attention_weight_max": 36.69879913330078,
      "activations/layer12_attention_weight_min": -29.958295822143555,
      "activations/layer13_attention_weight_max": 61.209861755371094,
      "activations/layer13_attention_weight_min": -37.40373229980469,
      "activations/layer14_attention_weight_max": 97.56876373291016,
      "activations/layer14_attention_weight_min": -68.001220703125,
      "activations/layer15_attention_weight_max": 62.24848937988281,
      "activations/layer15_attention_weight_min": -39.28093719482422,
      "activations/layer16_attention_weight_max": 56.90435028076172,
      "activations/layer16_attention_weight_min": -36.95258712768555,
      "activations/layer17_attention_weight_max": 52.3590202331543,
      "activations/layer17_attention_weight_min": -32.4486198425293,
      "activations/layer18_attention_weight_max": 29.96978759765625,
      "activations/layer18_attention_weight_min": -18.465497970581055,
      "activations/layer19_attention_weight_max": 63.293701171875,
      "activations/layer19_attention_weight_min": -33.208473205566406,
      "activations/layer1_attention_weight_max": 16.98611831665039,
      "activations/layer1_attention_weight_min": -11.683441162109375,
      "activations/layer20_attention_weight_max": 50.82841110229492,
      "activations/layer20_attention_weight_min": -24.9334716796875,
      "activations/layer21_attention_weight_max": 32.56993103027344,
      "activations/layer21_attention_weight_min": -16.993215560913086,
      "activations/layer22_attention_weight_max": 51.62485122680664,
      "activations/layer22_attention_weight_min": -25.7260684967041,
      "activations/layer23_attention_weight_max": 30.195110321044922,
      "activations/layer23_attention_weight_min": -16.26280403137207,
      "activations/layer2_attention_weight_max": 28.53053855895996,
      "activations/layer2_attention_weight_min": -26.912555694580078,
      "activations/layer3_attention_weight_max": 69.00186157226562,
      "activations/layer3_attention_weight_min": -68.39087677001953,
      "activations/layer4_attention_weight_max": 125.48455047607422,
      "activations/layer4_attention_weight_min": -114.97354888916016,
      "activations/layer5_attention_weight_max": 116.38868713378906,
      "activations/layer5_attention_weight_min": -105.56786346435547,
      "activations/layer6_attention_weight_max": 62.47100830078125,
      "activations/layer6_attention_weight_min": -55.600311279296875,
      "activations/layer7_attention_weight_max": 63.22415542602539,
      "activations/layer7_attention_weight_min": -62.224281311035156,
      "activations/layer8_attention_weight_max": 49.280174255371094,
      "activations/layer8_attention_weight_min": -44.001930236816406,
      "activations/layer9_attention_weight_max": 72.33663177490234,
      "activations/layer9_attention_weight_min": -60.34223556518555,
      "epoch": 1.77,
      "learning_rate": 0.0001399780303030303,
      "loss": 2.9805,
      "step": 30500
    },
    {
      "activations/layer0_attention_weight_max": 17.032207489013672,
      "activations/layer0_attention_weight_min": -11.402379035949707,
      "activations/layer10_attention_weight_max": 50.2225227355957,
      "activations/layer10_attention_weight_min": -44.42338943481445,
      "activations/layer11_attention_weight_max": 38.09079360961914,
      "activations/layer11_attention_weight_min": -34.52696990966797,
      "activations/layer12_attention_weight_max": 40.978485107421875,
      "activations/layer12_attention_weight_min": -29.534269332885742,
      "activations/layer13_attention_weight_max": 46.26470184326172,
      "activations/layer13_attention_weight_min": -37.510047912597656,
      "activations/layer14_attention_weight_max": 75.6119613647461,
      "activations/layer14_attention_weight_min": -55.79993438720703,
      "activations/layer15_attention_weight_max": 55.54779052734375,
      "activations/layer15_attention_weight_min": -40.886322021484375,
      "activations/layer16_attention_weight_max": 54.8686637878418,
      "activations/layer16_attention_weight_min": -40.12697982788086,
      "activations/layer17_attention_weight_max": 43.806053161621094,
      "activations/layer17_attention_weight_min": -31.688053131103516,
      "activations/layer18_attention_weight_max": 25.63967514038086,
      "activations/layer18_attention_weight_min": -18.97835922241211,
      "activations/layer19_attention_weight_max": 50.75252914428711,
      "activations/layer19_attention_weight_min": -30.881101608276367,
      "activations/layer1_attention_weight_max": 18.384923934936523,
      "activations/layer1_attention_weight_min": -11.09231185913086,
      "activations/layer20_attention_weight_max": 36.69330596923828,
      "activations/layer20_attention_weight_min": -25.435489654541016,
      "activations/layer21_attention_weight_max": 30.774351119995117,
      "activations/layer21_attention_weight_min": -16.598430633544922,
      "activations/layer22_attention_weight_max": 45.98807907104492,
      "activations/layer22_attention_weight_min": -23.40091323852539,
      "activations/layer23_attention_weight_max": 24.267065048217773,
      "activations/layer23_attention_weight_min": -14.929864883422852,
      "activations/layer2_attention_weight_max": 30.375465393066406,
      "activations/layer2_attention_weight_min": -28.85389518737793,
      "activations/layer3_attention_weight_max": 71.21087646484375,
      "activations/layer3_attention_weight_min": -73.68008422851562,
      "activations/layer4_attention_weight_max": 128.8672332763672,
      "activations/layer4_attention_weight_min": -115.90547943115234,
      "activations/layer5_attention_weight_max": 119.64208221435547,
      "activations/layer5_attention_weight_min": -97.3763427734375,
      "activations/layer6_attention_weight_max": 63.47101593017578,
      "activations/layer6_attention_weight_min": -52.03276062011719,
      "activations/layer7_attention_weight_max": 68.69235229492188,
      "activations/layer7_attention_weight_min": -60.788482666015625,
      "activations/layer8_attention_weight_max": 56.286495208740234,
      "activations/layer8_attention_weight_min": -46.69459915161133,
      "activations/layer9_attention_weight_max": 75.10615539550781,
      "activations/layer9_attention_weight_min": -56.8527717590332,
      "epoch": 1.78,
      "learning_rate": 0.0001399590909090909,
      "loss": 2.9817,
      "step": 30550
    },
    {
      "activations/layer0_attention_weight_max": 14.426192283630371,
      "activations/layer0_attention_weight_min": -11.927605628967285,
      "activations/layer10_attention_weight_max": 51.0832633972168,
      "activations/layer10_attention_weight_min": -47.91004180908203,
      "activations/layer11_attention_weight_max": 50.8218994140625,
      "activations/layer11_attention_weight_min": -42.55099868774414,
      "activations/layer12_attention_weight_max": 67.96785736083984,
      "activations/layer12_attention_weight_min": -33.290557861328125,
      "activations/layer13_attention_weight_max": 57.09107208251953,
      "activations/layer13_attention_weight_min": -44.36874771118164,
      "activations/layer14_attention_weight_max": 82.62452697753906,
      "activations/layer14_attention_weight_min": -69.456787109375,
      "activations/layer15_attention_weight_max": 53.43069076538086,
      "activations/layer15_attention_weight_min": -37.587100982666016,
      "activations/layer16_attention_weight_max": 54.12076950073242,
      "activations/layer16_attention_weight_min": -34.968170166015625,
      "activations/layer17_attention_weight_max": 44.86946487426758,
      "activations/layer17_attention_weight_min": -32.214698791503906,
      "activations/layer18_attention_weight_max": 24.646852493286133,
      "activations/layer18_attention_weight_min": -17.076095581054688,
      "activations/layer19_attention_weight_max": 55.30360794067383,
      "activations/layer19_attention_weight_min": -31.852319717407227,
      "activations/layer1_attention_weight_max": 17.096153259277344,
      "activations/layer1_attention_weight_min": -13.231766700744629,
      "activations/layer20_attention_weight_max": 34.57926940917969,
      "activations/layer20_attention_weight_min": -25.230873107910156,
      "activations/layer21_attention_weight_max": 26.69866180419922,
      "activations/layer21_attention_weight_min": -17.49016571044922,
      "activations/layer22_attention_weight_max": 43.75296401977539,
      "activations/layer22_attention_weight_min": -26.081865310668945,
      "activations/layer23_attention_weight_max": 26.633136749267578,
      "activations/layer23_attention_weight_min": -16.06661033630371,
      "activations/layer2_attention_weight_max": 27.178775787353516,
      "activations/layer2_attention_weight_min": -27.4453182220459,
      "activations/layer3_attention_weight_max": 67.41670227050781,
      "activations/layer3_attention_weight_min": -71.06661224365234,
      "activations/layer4_attention_weight_max": 124.71095275878906,
      "activations/layer4_attention_weight_min": -116.66326141357422,
      "activations/layer5_attention_weight_max": 111.19378662109375,
      "activations/layer5_attention_weight_min": -96.94890594482422,
      "activations/layer6_attention_weight_max": 60.7861328125,
      "activations/layer6_attention_weight_min": -53.27592849731445,
      "activations/layer7_attention_weight_max": 72.46786499023438,
      "activations/layer7_attention_weight_min": -64.1106185913086,
      "activations/layer8_attention_weight_max": 47.44731521606445,
      "activations/layer8_attention_weight_min": -46.43401336669922,
      "activations/layer9_attention_weight_max": 86.12584686279297,
      "activations/layer9_attention_weight_min": -67.5688705444336,
      "epoch": 1.78,
      "learning_rate": 0.0001399405303030303,
      "loss": 2.9838,
      "step": 30600
    },
    {
      "activations/layer0_attention_weight_max": 16.84531021118164,
      "activations/layer0_attention_weight_min": -11.891852378845215,
      "activations/layer10_attention_weight_max": 55.83277130126953,
      "activations/layer10_attention_weight_min": -46.284637451171875,
      "activations/layer11_attention_weight_max": 41.779937744140625,
      "activations/layer11_attention_weight_min": -39.109405517578125,
      "activations/layer12_attention_weight_max": 40.21732711791992,
      "activations/layer12_attention_weight_min": -27.286962509155273,
      "activations/layer13_attention_weight_max": 50.88731384277344,
      "activations/layer13_attention_weight_min": -38.02830123901367,
      "activations/layer14_attention_weight_max": 81.85057067871094,
      "activations/layer14_attention_weight_min": -61.869110107421875,
      "activations/layer15_attention_weight_max": 54.57179260253906,
      "activations/layer15_attention_weight_min": -40.634822845458984,
      "activations/layer16_attention_weight_max": 53.73556137084961,
      "activations/layer16_attention_weight_min": -38.93011474609375,
      "activations/layer17_attention_weight_max": 45.243099212646484,
      "activations/layer17_attention_weight_min": -30.587623596191406,
      "activations/layer18_attention_weight_max": 26.87611198425293,
      "activations/layer18_attention_weight_min": -18.094682693481445,
      "activations/layer19_attention_weight_max": 56.2342643737793,
      "activations/layer19_attention_weight_min": -36.065181732177734,
      "activations/layer1_attention_weight_max": 16.60963249206543,
      "activations/layer1_attention_weight_min": -11.515729904174805,
      "activations/layer20_attention_weight_max": 43.027469635009766,
      "activations/layer20_attention_weight_min": -24.213790893554688,
      "activations/layer21_attention_weight_max": 31.339170455932617,
      "activations/layer21_attention_weight_min": -20.122865676879883,
      "activations/layer22_attention_weight_max": 44.215599060058594,
      "activations/layer22_attention_weight_min": -25.822357177734375,
      "activations/layer23_attention_weight_max": 27.63132095336914,
      "activations/layer23_attention_weight_min": -15.385340690612793,
      "activations/layer2_attention_weight_max": 28.438323974609375,
      "activations/layer2_attention_weight_min": -28.975807189941406,
      "activations/layer3_attention_weight_max": 68.94447326660156,
      "activations/layer3_attention_weight_min": -71.26377868652344,
      "activations/layer4_attention_weight_max": 123.72761535644531,
      "activations/layer4_attention_weight_min": -112.0704574584961,
      "activations/layer5_attention_weight_max": 111.61135864257812,
      "activations/layer5_attention_weight_min": -104.62983703613281,
      "activations/layer6_attention_weight_max": 60.13420104980469,
      "activations/layer6_attention_weight_min": -53.53871154785156,
      "activations/layer7_attention_weight_max": 68.44923400878906,
      "activations/layer7_attention_weight_min": -63.390968322753906,
      "activations/layer8_attention_weight_max": 51.095115661621094,
      "activations/layer8_attention_weight_min": -40.12102508544922,
      "activations/layer9_attention_weight_max": 70.13663482666016,
      "activations/layer9_attention_weight_min": -58.716285705566406,
      "epoch": 1.78,
      "learning_rate": 0.0001399215909090909,
      "loss": 2.9789,
      "step": 30650
    },
    {
      "activations/layer0_attention_weight_max": 16.428064346313477,
      "activations/layer0_attention_weight_min": -11.700286865234375,
      "activations/layer10_attention_weight_max": 55.3908805847168,
      "activations/layer10_attention_weight_min": -46.38059616088867,
      "activations/layer11_attention_weight_max": 45.49547576904297,
      "activations/layer11_attention_weight_min": -35.179725646972656,
      "activations/layer12_attention_weight_max": 33.86873245239258,
      "activations/layer12_attention_weight_min": -26.434335708618164,
      "activations/layer13_attention_weight_max": 55.96818542480469,
      "activations/layer13_attention_weight_min": -37.316829681396484,
      "activations/layer14_attention_weight_max": 85.84196472167969,
      "activations/layer14_attention_weight_min": -60.26161193847656,
      "activations/layer15_attention_weight_max": 55.36292266845703,
      "activations/layer15_attention_weight_min": -41.48487854003906,
      "activations/layer16_attention_weight_max": 64.0230484008789,
      "activations/layer16_attention_weight_min": -38.001773834228516,
      "activations/layer17_attention_weight_max": 49.43980407714844,
      "activations/layer17_attention_weight_min": -30.166311264038086,
      "activations/layer18_attention_weight_max": 33.644535064697266,
      "activations/layer18_attention_weight_min": -18.058490753173828,
      "activations/layer19_attention_weight_max": 59.96696853637695,
      "activations/layer19_attention_weight_min": -34.138309478759766,
      "activations/layer1_attention_weight_max": 17.65515899658203,
      "activations/layer1_attention_weight_min": -11.214435577392578,
      "activations/layer20_attention_weight_max": 42.96177291870117,
      "activations/layer20_attention_weight_min": -25.69835662841797,
      "activations/layer21_attention_weight_max": 29.809871673583984,
      "activations/layer21_attention_weight_min": -20.317930221557617,
      "activations/layer22_attention_weight_max": 48.907588958740234,
      "activations/layer22_attention_weight_min": -26.657691955566406,
      "activations/layer23_attention_weight_max": 28.51948356628418,
      "activations/layer23_attention_weight_min": -15.250778198242188,
      "activations/layer2_attention_weight_max": 28.85409164428711,
      "activations/layer2_attention_weight_min": -26.91522216796875,
      "activations/layer3_attention_weight_max": 69.51445770263672,
      "activations/layer3_attention_weight_min": -70.73831939697266,
      "activations/layer4_attention_weight_max": 128.6391143798828,
      "activations/layer4_attention_weight_min": -121.5618896484375,
      "activations/layer5_attention_weight_max": 122.11155700683594,
      "activations/layer5_attention_weight_min": -101.98717498779297,
      "activations/layer6_attention_weight_max": 66.65736389160156,
      "activations/layer6_attention_weight_min": -55.02531051635742,
      "activations/layer7_attention_weight_max": 68.0826644897461,
      "activations/layer7_attention_weight_min": -61.92818069458008,
      "activations/layer8_attention_weight_max": 46.78835678100586,
      "activations/layer8_attention_weight_min": -41.98050308227539,
      "activations/layer9_attention_weight_max": 61.82386016845703,
      "activations/layer9_attention_weight_min": -51.634010314941406,
      "epoch": 1.78,
      "learning_rate": 0.0001399026515151515,
      "loss": 2.9836,
      "step": 30700
    },
    {
      "activations/layer0_attention_weight_max": 16.38137435913086,
      "activations/layer0_attention_weight_min": -12.094300270080566,
      "activations/layer10_attention_weight_max": 57.16400909423828,
      "activations/layer10_attention_weight_min": -47.99857711791992,
      "activations/layer11_attention_weight_max": 39.95155334472656,
      "activations/layer11_attention_weight_min": -35.48866271972656,
      "activations/layer12_attention_weight_max": 32.63649368286133,
      "activations/layer12_attention_weight_min": -26.330184936523438,
      "activations/layer13_attention_weight_max": 48.4938850402832,
      "activations/layer13_attention_weight_min": -36.239559173583984,
      "activations/layer14_attention_weight_max": 76.43495178222656,
      "activations/layer14_attention_weight_min": -58.235252380371094,
      "activations/layer15_attention_weight_max": 53.30064010620117,
      "activations/layer15_attention_weight_min": -37.095462799072266,
      "activations/layer16_attention_weight_max": 50.9364128112793,
      "activations/layer16_attention_weight_min": -37.40299987792969,
      "activations/layer17_attention_weight_max": 47.8913459777832,
      "activations/layer17_attention_weight_min": -28.66681671142578,
      "activations/layer18_attention_weight_max": 26.6929988861084,
      "activations/layer18_attention_weight_min": -16.56490135192871,
      "activations/layer19_attention_weight_max": 52.479034423828125,
      "activations/layer19_attention_weight_min": -34.8000602722168,
      "activations/layer1_attention_weight_max": 17.72138023376465,
      "activations/layer1_attention_weight_min": -10.733729362487793,
      "activations/layer20_attention_weight_max": 40.28079605102539,
      "activations/layer20_attention_weight_min": -24.19501304626465,
      "activations/layer21_attention_weight_max": 27.57486915588379,
      "activations/layer21_attention_weight_min": -18.113239288330078,
      "activations/layer22_attention_weight_max": 46.18508529663086,
      "activations/layer22_attention_weight_min": -28.352924346923828,
      "activations/layer23_attention_weight_max": 24.960826873779297,
      "activations/layer23_attention_weight_min": -16.255767822265625,
      "activations/layer2_attention_weight_max": 29.90376853942871,
      "activations/layer2_attention_weight_min": -29.09607696533203,
      "activations/layer3_attention_weight_max": 68.92015838623047,
      "activations/layer3_attention_weight_min": -71.74988555908203,
      "activations/layer4_attention_weight_max": 131.00437927246094,
      "activations/layer4_attention_weight_min": -116.51949310302734,
      "activations/layer5_attention_weight_max": 117.06523895263672,
      "activations/layer5_attention_weight_min": -103.03094482421875,
      "activations/layer6_attention_weight_max": 63.823490142822266,
      "activations/layer6_attention_weight_min": -56.14609146118164,
      "activations/layer7_attention_weight_max": 67.19258880615234,
      "activations/layer7_attention_weight_min": -59.83218765258789,
      "activations/layer8_attention_weight_max": 51.0965461730957,
      "activations/layer8_attention_weight_min": -42.0317268371582,
      "activations/layer9_attention_weight_max": 69.7467269897461,
      "activations/layer9_attention_weight_min": -58.406856536865234,
      "epoch": 1.79,
      "learning_rate": 0.0001398837121212121,
      "loss": 2.9919,
      "step": 30750
    },
    {
      "activations/layer0_attention_weight_max": 16.8543758392334,
      "activations/layer0_attention_weight_min": -11.63120174407959,
      "activations/layer10_attention_weight_max": 61.64674758911133,
      "activations/layer10_attention_weight_min": -43.28516387939453,
      "activations/layer11_attention_weight_max": 53.37139892578125,
      "activations/layer11_attention_weight_min": -45.63848114013672,
      "activations/layer12_attention_weight_max": 52.17082977294922,
      "activations/layer12_attention_weight_min": -28.754045486450195,
      "activations/layer13_attention_weight_max": 65.28795623779297,
      "activations/layer13_attention_weight_min": -40.7202262878418,
      "activations/layer14_attention_weight_max": 93.50994873046875,
      "activations/layer14_attention_weight_min": -53.562347412109375,
      "activations/layer15_attention_weight_max": 67.43675994873047,
      "activations/layer15_attention_weight_min": -40.49262619018555,
      "activations/layer16_attention_weight_max": 57.646427154541016,
      "activations/layer16_attention_weight_min": -39.363075256347656,
      "activations/layer17_attention_weight_max": 50.96272277832031,
      "activations/layer17_attention_weight_min": -30.29124641418457,
      "activations/layer18_attention_weight_max": 30.365413665771484,
      "activations/layer18_attention_weight_min": -18.842321395874023,
      "activations/layer19_attention_weight_max": 55.7039794921875,
      "activations/layer19_attention_weight_min": -34.89999771118164,
      "activations/layer1_attention_weight_max": 16.769521713256836,
      "activations/layer1_attention_weight_min": -11.071715354919434,
      "activations/layer20_attention_weight_max": 41.20448303222656,
      "activations/layer20_attention_weight_min": -25.296934127807617,
      "activations/layer21_attention_weight_max": 37.269996643066406,
      "activations/layer21_attention_weight_min": -18.536344528198242,
      "activations/layer22_attention_weight_max": 51.05536651611328,
      "activations/layer22_attention_weight_min": -27.336681365966797,
      "activations/layer23_attention_weight_max": 26.00031089782715,
      "activations/layer23_attention_weight_min": -16.76830291748047,
      "activations/layer2_attention_weight_max": 27.932048797607422,
      "activations/layer2_attention_weight_min": -27.842510223388672,
      "activations/layer3_attention_weight_max": 66.4686279296875,
      "activations/layer3_attention_weight_min": -70.71438598632812,
      "activations/layer4_attention_weight_max": 131.05210876464844,
      "activations/layer4_attention_weight_min": -118.33686828613281,
      "activations/layer5_attention_weight_max": 119.89946746826172,
      "activations/layer5_attention_weight_min": -99.1837158203125,
      "activations/layer6_attention_weight_max": 69.7528076171875,
      "activations/layer6_attention_weight_min": -53.80354690551758,
      "activations/layer7_attention_weight_max": 64.04059600830078,
      "activations/layer7_attention_weight_min": -61.314910888671875,
      "activations/layer8_attention_weight_max": 56.37440490722656,
      "activations/layer8_attention_weight_min": -41.074913024902344,
      "activations/layer9_attention_weight_max": 79.37390899658203,
      "activations/layer9_attention_weight_min": -61.053443908691406,
      "epoch": 1.79,
      "learning_rate": 0.00013986477272727273,
      "loss": 2.9813,
      "step": 30800
    },
    {
      "activations/layer0_attention_weight_max": 16.745512008666992,
      "activations/layer0_attention_weight_min": -11.469635009765625,
      "activations/layer10_attention_weight_max": 49.615928649902344,
      "activations/layer10_attention_weight_min": -43.507381439208984,
      "activations/layer11_attention_weight_max": 38.840492248535156,
      "activations/layer11_attention_weight_min": -41.502838134765625,
      "activations/layer12_attention_weight_max": 44.707427978515625,
      "activations/layer12_attention_weight_min": -35.63801193237305,
      "activations/layer13_attention_weight_max": 62.997642517089844,
      "activations/layer13_attention_weight_min": -49.31930160522461,
      "activations/layer14_attention_weight_max": 78.89605712890625,
      "activations/layer14_attention_weight_min": -62.705657958984375,
      "activations/layer15_attention_weight_max": 58.249576568603516,
      "activations/layer15_attention_weight_min": -37.059322357177734,
      "activations/layer16_attention_weight_max": 54.599090576171875,
      "activations/layer16_attention_weight_min": -35.37398910522461,
      "activations/layer17_attention_weight_max": 46.34978485107422,
      "activations/layer17_attention_weight_min": -31.57295036315918,
      "activations/layer18_attention_weight_max": 26.949020385742188,
      "activations/layer18_attention_weight_min": -16.782615661621094,
      "activations/layer19_attention_weight_max": 62.32664108276367,
      "activations/layer19_attention_weight_min": -33.382015228271484,
      "activations/layer1_attention_weight_max": 16.879274368286133,
      "activations/layer1_attention_weight_min": -11.554239273071289,
      "activations/layer20_attention_weight_max": 42.1116828918457,
      "activations/layer20_attention_weight_min": -23.23239517211914,
      "activations/layer21_attention_weight_max": 34.0861930847168,
      "activations/layer21_attention_weight_min": -17.742130279541016,
      "activations/layer22_attention_weight_max": 47.19707489013672,
      "activations/layer22_attention_weight_min": -26.256927490234375,
      "activations/layer23_attention_weight_max": 30.416915893554688,
      "activations/layer23_attention_weight_min": -15.616744995117188,
      "activations/layer2_attention_weight_max": 27.336271286010742,
      "activations/layer2_attention_weight_min": -26.456531524658203,
      "activations/layer3_attention_weight_max": 65.70575714111328,
      "activations/layer3_attention_weight_min": -68.12714385986328,
      "activations/layer4_attention_weight_max": 117.34967041015625,
      "activations/layer4_attention_weight_min": -111.04805755615234,
      "activations/layer5_attention_weight_max": 109.89796447753906,
      "activations/layer5_attention_weight_min": -97.1605453491211,
      "activations/layer6_attention_weight_max": 58.79434585571289,
      "activations/layer6_attention_weight_min": -52.55931091308594,
      "activations/layer7_attention_weight_max": 65.02874755859375,
      "activations/layer7_attention_weight_min": -65.88346862792969,
      "activations/layer8_attention_weight_max": 47.73499298095703,
      "activations/layer8_attention_weight_min": -44.94198226928711,
      "activations/layer9_attention_weight_max": 72.22891235351562,
      "activations/layer9_attention_weight_min": -60.35797119140625,
      "epoch": 1.79,
      "learning_rate": 0.00013984583333333334,
      "loss": 2.9821,
      "step": 30850
    },
    {
      "activations/layer0_attention_weight_max": 14.69148063659668,
      "activations/layer0_attention_weight_min": -12.052191734313965,
      "activations/layer10_attention_weight_max": 52.80699920654297,
      "activations/layer10_attention_weight_min": -49.0951042175293,
      "activations/layer11_attention_weight_max": 39.12187957763672,
      "activations/layer11_attention_weight_min": -34.365318298339844,
      "activations/layer12_attention_weight_max": 63.41511535644531,
      "activations/layer12_attention_weight_min": -34.82506561279297,
      "activations/layer13_attention_weight_max": 63.226104736328125,
      "activations/layer13_attention_weight_min": -39.73369598388672,
      "activations/layer14_attention_weight_max": 81.8907241821289,
      "activations/layer14_attention_weight_min": -62.09787368774414,
      "activations/layer15_attention_weight_max": 51.626007080078125,
      "activations/layer15_attention_weight_min": -40.03322219848633,
      "activations/layer16_attention_weight_max": 48.518245697021484,
      "activations/layer16_attention_weight_min": -39.058074951171875,
      "activations/layer17_attention_weight_max": 42.70134735107422,
      "activations/layer17_attention_weight_min": -28.68079948425293,
      "activations/layer18_attention_weight_max": 33.10277557373047,
      "activations/layer18_attention_weight_min": -18.190263748168945,
      "activations/layer19_attention_weight_max": 57.158660888671875,
      "activations/layer19_attention_weight_min": -33.41563415527344,
      "activations/layer1_attention_weight_max": 16.532453536987305,
      "activations/layer1_attention_weight_min": -12.273957252502441,
      "activations/layer20_attention_weight_max": 40.41476821899414,
      "activations/layer20_attention_weight_min": -26.227088928222656,
      "activations/layer21_attention_weight_max": 29.895261764526367,
      "activations/layer21_attention_weight_min": -20.45885467529297,
      "activations/layer22_attention_weight_max": 47.25568389892578,
      "activations/layer22_attention_weight_min": -24.72349739074707,
      "activations/layer23_attention_weight_max": 27.246557235717773,
      "activations/layer23_attention_weight_min": -17.585901260375977,
      "activations/layer2_attention_weight_max": 30.103918075561523,
      "activations/layer2_attention_weight_min": -28.52272605895996,
      "activations/layer3_attention_weight_max": 68.38492584228516,
      "activations/layer3_attention_weight_min": -70.54150390625,
      "activations/layer4_attention_weight_max": 124.8153305053711,
      "activations/layer4_attention_weight_min": -119.60514068603516,
      "activations/layer5_attention_weight_max": 118.6906967163086,
      "activations/layer5_attention_weight_min": -98.71292114257812,
      "activations/layer6_attention_weight_max": 63.56235885620117,
      "activations/layer6_attention_weight_min": -52.9679069519043,
      "activations/layer7_attention_weight_max": 66.63664245605469,
      "activations/layer7_attention_weight_min": -61.361167907714844,
      "activations/layer8_attention_weight_max": 49.60200881958008,
      "activations/layer8_attention_weight_min": -44.304046630859375,
      "activations/layer9_attention_weight_max": 70.83687591552734,
      "activations/layer9_attention_weight_min": -57.613563537597656,
      "epoch": 1.8,
      "learning_rate": 0.00013982689393939393,
      "loss": 2.9917,
      "step": 30900
    },
    {
      "activations/layer0_attention_weight_max": 16.71900749206543,
      "activations/layer0_attention_weight_min": -11.879422187805176,
      "activations/layer10_attention_weight_max": 63.95295333862305,
      "activations/layer10_attention_weight_min": -51.24028396606445,
      "activations/layer11_attention_weight_max": 43.639503479003906,
      "activations/layer11_attention_weight_min": -35.26313018798828,
      "activations/layer12_attention_weight_max": 34.39830017089844,
      "activations/layer12_attention_weight_min": -28.919605255126953,
      "activations/layer13_attention_weight_max": 50.05232620239258,
      "activations/layer13_attention_weight_min": -39.548927307128906,
      "activations/layer14_attention_weight_max": 81.21308898925781,
      "activations/layer14_attention_weight_min": -61.28282928466797,
      "activations/layer15_attention_weight_max": 47.76621627807617,
      "activations/layer15_attention_weight_min": -40.5273551940918,
      "activations/layer16_attention_weight_max": 46.483760833740234,
      "activations/layer16_attention_weight_min": -37.87260055541992,
      "activations/layer17_attention_weight_max": 46.0583610534668,
      "activations/layer17_attention_weight_min": -31.30219841003418,
      "activations/layer18_attention_weight_max": 27.1583251953125,
      "activations/layer18_attention_weight_min": -19.938222885131836,
      "activations/layer19_attention_weight_max": 61.17302322387695,
      "activations/layer19_attention_weight_min": -37.650299072265625,
      "activations/layer1_attention_weight_max": 17.608121871948242,
      "activations/layer1_attention_weight_min": -11.848797798156738,
      "activations/layer20_attention_weight_max": 42.60782241821289,
      "activations/layer20_attention_weight_min": -26.233936309814453,
      "activations/layer21_attention_weight_max": 25.146448135375977,
      "activations/layer21_attention_weight_min": -17.863872528076172,
      "activations/layer22_attention_weight_max": 40.87802505493164,
      "activations/layer22_attention_weight_min": -27.87796974182129,
      "activations/layer23_attention_weight_max": 27.836454391479492,
      "activations/layer23_attention_weight_min": -16.576135635375977,
      "activations/layer2_attention_weight_max": 26.4884090423584,
      "activations/layer2_attention_weight_min": -26.865583419799805,
      "activations/layer3_attention_weight_max": 66.07484436035156,
      "activations/layer3_attention_weight_min": -69.60139465332031,
      "activations/layer4_attention_weight_max": 126.11624908447266,
      "activations/layer4_attention_weight_min": -122.91036224365234,
      "activations/layer5_attention_weight_max": 119.8641586303711,
      "activations/layer5_attention_weight_min": -104.84285736083984,
      "activations/layer6_attention_weight_max": 67.80592346191406,
      "activations/layer6_attention_weight_min": -56.39414596557617,
      "activations/layer7_attention_weight_max": 72.20304107666016,
      "activations/layer7_attention_weight_min": -59.74237060546875,
      "activations/layer8_attention_weight_max": 54.92369842529297,
      "activations/layer8_attention_weight_min": -44.44524383544922,
      "activations/layer9_attention_weight_max": 74.24456024169922,
      "activations/layer9_attention_weight_min": -54.2180290222168,
      "epoch": 1.8,
      "learning_rate": 0.00013980795454545455,
      "loss": 2.9757,
      "step": 30950
    },
    {
      "activations/layer0_attention_weight_max": 16.596969604492188,
      "activations/layer0_attention_weight_min": -12.248162269592285,
      "activations/layer10_attention_weight_max": 49.409423828125,
      "activations/layer10_attention_weight_min": -44.54484176635742,
      "activations/layer11_attention_weight_max": 38.63985824584961,
      "activations/layer11_attention_weight_min": -36.80419921875,
      "activations/layer12_attention_weight_max": 32.134857177734375,
      "activations/layer12_attention_weight_min": -26.658252716064453,
      "activations/layer13_attention_weight_max": 47.16084289550781,
      "activations/layer13_attention_weight_min": -36.66248321533203,
      "activations/layer14_attention_weight_max": 80.19544982910156,
      "activations/layer14_attention_weight_min": -60.82044219970703,
      "activations/layer15_attention_weight_max": 53.57915496826172,
      "activations/layer15_attention_weight_min": -39.61775588989258,
      "activations/layer16_attention_weight_max": 48.56007766723633,
      "activations/layer16_attention_weight_min": -36.84871292114258,
      "activations/layer17_attention_weight_max": 44.06155776977539,
      "activations/layer17_attention_weight_min": -30.02156639099121,
      "activations/layer18_attention_weight_max": 25.752229690551758,
      "activations/layer18_attention_weight_min": -17.80921173095703,
      "activations/layer19_attention_weight_max": 48.605525970458984,
      "activations/layer19_attention_weight_min": -34.218143463134766,
      "activations/layer1_attention_weight_max": 19.33927345275879,
      "activations/layer1_attention_weight_min": -11.393858909606934,
      "activations/layer20_attention_weight_max": 34.914913177490234,
      "activations/layer20_attention_weight_min": -27.118667602539062,
      "activations/layer21_attention_weight_max": 29.429357528686523,
      "activations/layer21_attention_weight_min": -20.385290145874023,
      "activations/layer22_attention_weight_max": 38.26283645629883,
      "activations/layer22_attention_weight_min": -23.97747039794922,
      "activations/layer23_attention_weight_max": 24.789472579956055,
      "activations/layer23_attention_weight_min": -15.14197063446045,
      "activations/layer2_attention_weight_max": 26.61031723022461,
      "activations/layer2_attention_weight_min": -27.566734313964844,
      "activations/layer3_attention_weight_max": 65.38504791259766,
      "activations/layer3_attention_weight_min": -69.6282958984375,
      "activations/layer4_attention_weight_max": 123.96839904785156,
      "activations/layer4_attention_weight_min": -111.89568328857422,
      "activations/layer5_attention_weight_max": 112.24943542480469,
      "activations/layer5_attention_weight_min": -98.74620819091797,
      "activations/layer6_attention_weight_max": 56.72232437133789,
      "activations/layer6_attention_weight_min": -52.378971099853516,
      "activations/layer7_attention_weight_max": 61.95328903198242,
      "activations/layer7_attention_weight_min": -62.470848083496094,
      "activations/layer8_attention_weight_max": 44.97057342529297,
      "activations/layer8_attention_weight_min": -41.30413055419922,
      "activations/layer9_attention_weight_max": 67.02517700195312,
      "activations/layer9_attention_weight_min": -54.52761459350586,
      "epoch": 1.8,
      "learning_rate": 0.00013978901515151513,
      "loss": 2.9887,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_loss": 2.931640625,
      "eval_runtime": 8.4816,
      "eval_samples_per_second": 506.274,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_openwebtext_loss": 2.931640625,
      "eval_openwebtext_ppl": 18.758380733362646,
      "eval_openwebtext_runtime": 8.4816,
      "eval_openwebtext_samples_per_second": 506.274,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_wikitext_loss": 3.205078125,
      "eval_wikitext_ppl": 24.657426302639607,
      "eval_wikitext_runtime": 1.9731,
      "eval_wikitext_samples_per_second": 231.109,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_lambada_loss": 3.169921875,
      "eval_lambada_ppl": 23.80562446936611,
      "eval_lambada_runtime": 9.5605,
      "eval_lambada_samples_per_second": 509.281,
      "step": 31000
    },
    {
      "activations/layer0_attention_weight_max": 16.62816619873047,
      "activations/layer0_attention_weight_min": -11.601123809814453,
      "activations/layer10_attention_weight_max": 52.86338424682617,
      "activations/layer10_attention_weight_min": -45.80227279663086,
      "activations/layer11_attention_weight_max": 44.30445861816406,
      "activations/layer11_attention_weight_min": -39.56163024902344,
      "activations/layer12_attention_weight_max": 45.67771911621094,
      "activations/layer12_attention_weight_min": -27.88152503967285,
      "activations/layer13_attention_weight_max": 54.63880157470703,
      "activations/layer13_attention_weight_min": -40.19346237182617,
      "activations/layer14_attention_weight_max": 72.83002471923828,
      "activations/layer14_attention_weight_min": -58.30497360229492,
      "activations/layer15_attention_weight_max": 50.7135009765625,
      "activations/layer15_attention_weight_min": -36.938636779785156,
      "activations/layer16_attention_weight_max": 53.45304870605469,
      "activations/layer16_attention_weight_min": -39.20786666870117,
      "activations/layer17_attention_weight_max": 49.45252227783203,
      "activations/layer17_attention_weight_min": -30.573293685913086,
      "activations/layer18_attention_weight_max": 23.98586654663086,
      "activations/layer18_attention_weight_min": -16.10067367553711,
      "activations/layer19_attention_weight_max": 55.3902473449707,
      "activations/layer19_attention_weight_min": -31.235700607299805,
      "activations/layer1_attention_weight_max": 17.461395263671875,
      "activations/layer1_attention_weight_min": -11.640300750732422,
      "activations/layer20_attention_weight_max": 38.509674072265625,
      "activations/layer20_attention_weight_min": -24.882001876831055,
      "activations/layer21_attention_weight_max": 28.673227310180664,
      "activations/layer21_attention_weight_min": -18.772186279296875,
      "activations/layer22_attention_weight_max": 42.85325622558594,
      "activations/layer22_attention_weight_min": -30.372482299804688,
      "activations/layer23_attention_weight_max": 25.98567008972168,
      "activations/layer23_attention_weight_min": -16.08981704711914,
      "activations/layer2_attention_weight_max": 30.0833797454834,
      "activations/layer2_attention_weight_min": -29.80709457397461,
      "activations/layer3_attention_weight_max": 67.29061889648438,
      "activations/layer3_attention_weight_min": -71.31095886230469,
      "activations/layer4_attention_weight_max": 128.3575897216797,
      "activations/layer4_attention_weight_min": -119.6563491821289,
      "activations/layer5_attention_weight_max": 116.3973388671875,
      "activations/layer5_attention_weight_min": -103.58985900878906,
      "activations/layer6_attention_weight_max": 63.56378173828125,
      "activations/layer6_attention_weight_min": -59.52360534667969,
      "activations/layer7_attention_weight_max": 67.11250305175781,
      "activations/layer7_attention_weight_min": -60.229209899902344,
      "activations/layer8_attention_weight_max": 51.47419738769531,
      "activations/layer8_attention_weight_min": -47.58385467529297,
      "activations/layer9_attention_weight_max": 62.10334396362305,
      "activations/layer9_attention_weight_min": -54.39906692504883,
      "epoch": 1.8,
      "learning_rate": 0.00013977007575757575,
      "loss": 2.9818,
      "step": 31050
    },
    {
      "activations/layer0_attention_weight_max": 17.257600784301758,
      "activations/layer0_attention_weight_min": -10.758055686950684,
      "activations/layer10_attention_weight_max": 51.38288497924805,
      "activations/layer10_attention_weight_min": -46.080841064453125,
      "activations/layer11_attention_weight_max": 41.51157760620117,
      "activations/layer11_attention_weight_min": -35.03296661376953,
      "activations/layer12_attention_weight_max": 37.37284469604492,
      "activations/layer12_attention_weight_min": -28.933368682861328,
      "activations/layer13_attention_weight_max": 48.945194244384766,
      "activations/layer13_attention_weight_min": -40.020103454589844,
      "activations/layer14_attention_weight_max": 70.36724090576172,
      "activations/layer14_attention_weight_min": -55.88745880126953,
      "activations/layer15_attention_weight_max": 48.10543441772461,
      "activations/layer15_attention_weight_min": -39.86812210083008,
      "activations/layer16_attention_weight_max": 45.51449966430664,
      "activations/layer16_attention_weight_min": -35.311851501464844,
      "activations/layer17_attention_weight_max": 41.065608978271484,
      "activations/layer17_attention_weight_min": -31.552656173706055,
      "activations/layer18_attention_weight_max": 23.39667320251465,
      "activations/layer18_attention_weight_min": -18.014183044433594,
      "activations/layer19_attention_weight_max": 47.145450592041016,
      "activations/layer19_attention_weight_min": -28.957834243774414,
      "activations/layer1_attention_weight_max": 17.00718879699707,
      "activations/layer1_attention_weight_min": -12.681929588317871,
      "activations/layer20_attention_weight_max": 37.71244812011719,
      "activations/layer20_attention_weight_min": -22.755199432373047,
      "activations/layer21_attention_weight_max": 28.78434181213379,
      "activations/layer21_attention_weight_min": -18.633007049560547,
      "activations/layer22_attention_weight_max": 40.078636169433594,
      "activations/layer22_attention_weight_min": -25.875228881835938,
      "activations/layer23_attention_weight_max": 23.073974609375,
      "activations/layer23_attention_weight_min": -15.312301635742188,
      "activations/layer2_attention_weight_max": 28.43407440185547,
      "activations/layer2_attention_weight_min": -27.745285034179688,
      "activations/layer3_attention_weight_max": 67.66571807861328,
      "activations/layer3_attention_weight_min": -69.1332015991211,
      "activations/layer4_attention_weight_max": 124.91786193847656,
      "activations/layer4_attention_weight_min": -115.16373443603516,
      "activations/layer5_attention_weight_max": 110.27345275878906,
      "activations/layer5_attention_weight_min": -105.71966552734375,
      "activations/layer6_attention_weight_max": 59.56482696533203,
      "activations/layer6_attention_weight_min": -54.53614807128906,
      "activations/layer7_attention_weight_max": 63.98323440551758,
      "activations/layer7_attention_weight_min": -60.78002166748047,
      "activations/layer8_attention_weight_max": 43.84593200683594,
      "activations/layer8_attention_weight_min": -38.871944427490234,
      "activations/layer9_attention_weight_max": 60.31831741333008,
      "activations/layer9_attention_weight_min": -58.277374267578125,
      "epoch": 1.81,
      "learning_rate": 0.00013975113636363636,
      "loss": 2.9965,
      "step": 31100
    },
    {
      "activations/layer0_attention_weight_max": 17.14188575744629,
      "activations/layer0_attention_weight_min": -11.40324878692627,
      "activations/layer10_attention_weight_max": 55.479278564453125,
      "activations/layer10_attention_weight_min": -54.55234909057617,
      "activations/layer11_attention_weight_max": 40.411338806152344,
      "activations/layer11_attention_weight_min": -35.350807189941406,
      "activations/layer12_attention_weight_max": 62.166343688964844,
      "activations/layer12_attention_weight_min": -40.919830322265625,
      "activations/layer13_attention_weight_max": 56.29267501831055,
      "activations/layer13_attention_weight_min": -40.71284866333008,
      "activations/layer14_attention_weight_max": 92.01286315917969,
      "activations/layer14_attention_weight_min": -61.181495666503906,
      "activations/layer15_attention_weight_max": 55.69169616699219,
      "activations/layer15_attention_weight_min": -38.63309097290039,
      "activations/layer16_attention_weight_max": 56.51943588256836,
      "activations/layer16_attention_weight_min": -38.01768112182617,
      "activations/layer17_attention_weight_max": 56.48518753051758,
      "activations/layer17_attention_weight_min": -30.09876251220703,
      "activations/layer18_attention_weight_max": 34.26109313964844,
      "activations/layer18_attention_weight_min": -18.371994018554688,
      "activations/layer19_attention_weight_max": 54.1487922668457,
      "activations/layer19_attention_weight_min": -32.25419616699219,
      "activations/layer1_attention_weight_max": 17.923555374145508,
      "activations/layer1_attention_weight_min": -11.262653350830078,
      "activations/layer20_attention_weight_max": 37.87569808959961,
      "activations/layer20_attention_weight_min": -24.925397872924805,
      "activations/layer21_attention_weight_max": 26.927940368652344,
      "activations/layer21_attention_weight_min": -17.530519485473633,
      "activations/layer22_attention_weight_max": 49.26679229736328,
      "activations/layer22_attention_weight_min": -24.85579490661621,
      "activations/layer23_attention_weight_max": 27.377445220947266,
      "activations/layer23_attention_weight_min": -15.234320640563965,
      "activations/layer2_attention_weight_max": 26.510272979736328,
      "activations/layer2_attention_weight_min": -26.698331832885742,
      "activations/layer3_attention_weight_max": 67.68316650390625,
      "activations/layer3_attention_weight_min": -69.82191467285156,
      "activations/layer4_attention_weight_max": 125.09870910644531,
      "activations/layer4_attention_weight_min": -114.8552017211914,
      "activations/layer5_attention_weight_max": 112.2991943359375,
      "activations/layer5_attention_weight_min": -101.39395141601562,
      "activations/layer6_attention_weight_max": 60.31884002685547,
      "activations/layer6_attention_weight_min": -55.9406852722168,
      "activations/layer7_attention_weight_max": 62.49742126464844,
      "activations/layer7_attention_weight_min": -62.07823944091797,
      "activations/layer8_attention_weight_max": 44.52350616455078,
      "activations/layer8_attention_weight_min": -40.77177047729492,
      "activations/layer9_attention_weight_max": 72.06932067871094,
      "activations/layer9_attention_weight_min": -62.72488021850586,
      "epoch": 1.81,
      "learning_rate": 0.00013973219696969695,
      "loss": 3.0001,
      "step": 31150
    },
    {
      "activations/layer0_attention_weight_max": 16.937427520751953,
      "activations/layer0_attention_weight_min": -11.623973846435547,
      "activations/layer10_attention_weight_max": 52.64142608642578,
      "activations/layer10_attention_weight_min": -45.51451873779297,
      "activations/layer11_attention_weight_max": 52.97230529785156,
      "activations/layer11_attention_weight_min": -41.72652053833008,
      "activations/layer12_attention_weight_max": 36.06385040283203,
      "activations/layer12_attention_weight_min": -31.75973892211914,
      "activations/layer13_attention_weight_max": 55.66157531738281,
      "activations/layer13_attention_weight_min": -42.86616134643555,
      "activations/layer14_attention_weight_max": 76.35181427001953,
      "activations/layer14_attention_weight_min": -58.48178482055664,
      "activations/layer15_attention_weight_max": 50.831016540527344,
      "activations/layer15_attention_weight_min": -37.11737060546875,
      "activations/layer16_attention_weight_max": 49.04069519042969,
      "activations/layer16_attention_weight_min": -39.139408111572266,
      "activations/layer17_attention_weight_max": 49.614654541015625,
      "activations/layer17_attention_weight_min": -29.913591384887695,
      "activations/layer18_attention_weight_max": 26.038000106811523,
      "activations/layer18_attention_weight_min": -18.30434226989746,
      "activations/layer19_attention_weight_max": 53.92439651489258,
      "activations/layer19_attention_weight_min": -34.57939910888672,
      "activations/layer1_attention_weight_max": 17.612531661987305,
      "activations/layer1_attention_weight_min": -12.252704620361328,
      "activations/layer20_attention_weight_max": 43.56990432739258,
      "activations/layer20_attention_weight_min": -27.0422306060791,
      "activations/layer21_attention_weight_max": 33.76570510864258,
      "activations/layer21_attention_weight_min": -22.713836669921875,
      "activations/layer22_attention_weight_max": 50.46757125854492,
      "activations/layer22_attention_weight_min": -28.635427474975586,
      "activations/layer23_attention_weight_max": 26.155162811279297,
      "activations/layer23_attention_weight_min": -14.863992691040039,
      "activations/layer2_attention_weight_max": 27.70627212524414,
      "activations/layer2_attention_weight_min": -26.645275115966797,
      "activations/layer3_attention_weight_max": 68.86621856689453,
      "activations/layer3_attention_weight_min": -72.52989196777344,
      "activations/layer4_attention_weight_max": 119.859619140625,
      "activations/layer4_attention_weight_min": -118.16930389404297,
      "activations/layer5_attention_weight_max": 110.1318588256836,
      "activations/layer5_attention_weight_min": -100.8411865234375,
      "activations/layer6_attention_weight_max": 60.00929260253906,
      "activations/layer6_attention_weight_min": -53.82450866699219,
      "activations/layer7_attention_weight_max": 73.44732666015625,
      "activations/layer7_attention_weight_min": -66.24263000488281,
      "activations/layer8_attention_weight_max": 52.124088287353516,
      "activations/layer8_attention_weight_min": -42.29878616333008,
      "activations/layer9_attention_weight_max": 88.2170639038086,
      "activations/layer9_attention_weight_min": -72.6694564819336,
      "epoch": 1.81,
      "learning_rate": 0.00013971325757575757,
      "loss": 2.9949,
      "step": 31200
    },
    {
      "activations/layer0_attention_weight_max": 16.779787063598633,
      "activations/layer0_attention_weight_min": -11.77706527709961,
      "activations/layer10_attention_weight_max": 55.31732940673828,
      "activations/layer10_attention_weight_min": -45.93525314331055,
      "activations/layer11_attention_weight_max": 41.82836151123047,
      "activations/layer11_attention_weight_min": -36.90750503540039,
      "activations/layer12_attention_weight_max": 45.77668380737305,
      "activations/layer12_attention_weight_min": -32.249473571777344,
      "activations/layer13_attention_weight_max": 50.37200164794922,
      "activations/layer13_attention_weight_min": -41.089599609375,
      "activations/layer14_attention_weight_max": 82.57140350341797,
      "activations/layer14_attention_weight_min": -61.22371292114258,
      "activations/layer15_attention_weight_max": 62.57814025878906,
      "activations/layer15_attention_weight_min": -44.11478042602539,
      "activations/layer16_attention_weight_max": 52.17573547363281,
      "activations/layer16_attention_weight_min": -35.622459411621094,
      "activations/layer17_attention_weight_max": 55.00386428833008,
      "activations/layer17_attention_weight_min": -32.50324249267578,
      "activations/layer18_attention_weight_max": 28.37307357788086,
      "activations/layer18_attention_weight_min": -17.353214263916016,
      "activations/layer19_attention_weight_max": 50.909454345703125,
      "activations/layer19_attention_weight_min": -31.76211929321289,
      "activations/layer1_attention_weight_max": 17.711702346801758,
      "activations/layer1_attention_weight_min": -10.724504470825195,
      "activations/layer20_attention_weight_max": 38.79852294921875,
      "activations/layer20_attention_weight_min": -26.614274978637695,
      "activations/layer21_attention_weight_max": 29.82133674621582,
      "activations/layer21_attention_weight_min": -21.00608253479004,
      "activations/layer22_attention_weight_max": 50.270992279052734,
      "activations/layer22_attention_weight_min": -28.704227447509766,
      "activations/layer23_attention_weight_max": 25.402801513671875,
      "activations/layer23_attention_weight_min": -16.03752899169922,
      "activations/layer2_attention_weight_max": 27.664899826049805,
      "activations/layer2_attention_weight_min": -25.979801177978516,
      "activations/layer3_attention_weight_max": 66.96347045898438,
      "activations/layer3_attention_weight_min": -69.45661163330078,
      "activations/layer4_attention_weight_max": 125.67414855957031,
      "activations/layer4_attention_weight_min": -113.4146728515625,
      "activations/layer5_attention_weight_max": 119.74775695800781,
      "activations/layer5_attention_weight_min": -100.19839477539062,
      "activations/layer6_attention_weight_max": 63.856937408447266,
      "activations/layer6_attention_weight_min": -53.531917572021484,
      "activations/layer7_attention_weight_max": 63.651493072509766,
      "activations/layer7_attention_weight_min": -64.48475646972656,
      "activations/layer8_attention_weight_max": 49.323692321777344,
      "activations/layer8_attention_weight_min": -42.06528091430664,
      "activations/layer9_attention_weight_max": 71.11781311035156,
      "activations/layer9_attention_weight_min": -57.98283767700195,
      "epoch": 1.82,
      "learning_rate": 0.00013969431818181818,
      "loss": 2.9923,
      "step": 31250
    },
    {
      "activations/layer0_attention_weight_max": 17.242807388305664,
      "activations/layer0_attention_weight_min": -11.235848426818848,
      "activations/layer10_attention_weight_max": 60.906673431396484,
      "activations/layer10_attention_weight_min": -52.137474060058594,
      "activations/layer11_attention_weight_max": 42.60655212402344,
      "activations/layer11_attention_weight_min": -33.66825866699219,
      "activations/layer12_attention_weight_max": 39.9860725402832,
      "activations/layer12_attention_weight_min": -30.262971878051758,
      "activations/layer13_attention_weight_max": 52.280582427978516,
      "activations/layer13_attention_weight_min": -40.01435470581055,
      "activations/layer14_attention_weight_max": 88.13383483886719,
      "activations/layer14_attention_weight_min": -61.599666595458984,
      "activations/layer15_attention_weight_max": 55.0512809753418,
      "activations/layer15_attention_weight_min": -41.173648834228516,
      "activations/layer16_attention_weight_max": 53.33686828613281,
      "activations/layer16_attention_weight_min": -35.23542785644531,
      "activations/layer17_attention_weight_max": 49.1895637512207,
      "activations/layer17_attention_weight_min": -29.579721450805664,
      "activations/layer18_attention_weight_max": 26.831417083740234,
      "activations/layer18_attention_weight_min": -17.066829681396484,
      "activations/layer19_attention_weight_max": 49.61220169067383,
      "activations/layer19_attention_weight_min": -32.34681701660156,
      "activations/layer1_attention_weight_max": 17.552167892456055,
      "activations/layer1_attention_weight_min": -12.483536720275879,
      "activations/layer20_attention_weight_max": 41.504920959472656,
      "activations/layer20_attention_weight_min": -24.83635902404785,
      "activations/layer21_attention_weight_max": 31.17296600341797,
      "activations/layer21_attention_weight_min": -20.450836181640625,
      "activations/layer22_attention_weight_max": 46.47522735595703,
      "activations/layer22_attention_weight_min": -27.691938400268555,
      "activations/layer23_attention_weight_max": 25.601383209228516,
      "activations/layer23_attention_weight_min": -17.159727096557617,
      "activations/layer2_attention_weight_max": 27.929149627685547,
      "activations/layer2_attention_weight_min": -27.019561767578125,
      "activations/layer3_attention_weight_max": 71.53089904785156,
      "activations/layer3_attention_weight_min": -71.06932067871094,
      "activations/layer4_attention_weight_max": 129.4138641357422,
      "activations/layer4_attention_weight_min": -116.34651184082031,
      "activations/layer5_attention_weight_max": 122.02284240722656,
      "activations/layer5_attention_weight_min": -99.82357788085938,
      "activations/layer6_attention_weight_max": 72.17227172851562,
      "activations/layer6_attention_weight_min": -55.260215759277344,
      "activations/layer7_attention_weight_max": 70.50384521484375,
      "activations/layer7_attention_weight_min": -61.14177703857422,
      "activations/layer8_attention_weight_max": 57.24338912963867,
      "activations/layer8_attention_weight_min": -48.35111999511719,
      "activations/layer9_attention_weight_max": 84.93904876708984,
      "activations/layer9_attention_weight_min": -71.4529800415039,
      "epoch": 1.82,
      "learning_rate": 0.00013967537878787877,
      "loss": 2.9968,
      "step": 31300
    },
    {
      "activations/layer0_attention_weight_max": 16.631553649902344,
      "activations/layer0_attention_weight_min": -11.357816696166992,
      "activations/layer10_attention_weight_max": 51.0653190612793,
      "activations/layer10_attention_weight_min": -41.36533737182617,
      "activations/layer11_attention_weight_max": 42.49531173706055,
      "activations/layer11_attention_weight_min": -34.298362731933594,
      "activations/layer12_attention_weight_max": 35.392913818359375,
      "activations/layer12_attention_weight_min": -30.12812042236328,
      "activations/layer13_attention_weight_max": 49.068504333496094,
      "activations/layer13_attention_weight_min": -36.36933517456055,
      "activations/layer14_attention_weight_max": 80.5130844116211,
      "activations/layer14_attention_weight_min": -60.80110549926758,
      "activations/layer15_attention_weight_max": 57.750709533691406,
      "activations/layer15_attention_weight_min": -39.491573333740234,
      "activations/layer16_attention_weight_max": 50.95218276977539,
      "activations/layer16_attention_weight_min": -37.980873107910156,
      "activations/layer17_attention_weight_max": 44.44240188598633,
      "activations/layer17_attention_weight_min": -29.530649185180664,
      "activations/layer18_attention_weight_max": 25.3817195892334,
      "activations/layer18_attention_weight_min": -16.613325119018555,
      "activations/layer19_attention_weight_max": 48.58687973022461,
      "activations/layer19_attention_weight_min": -32.2904167175293,
      "activations/layer1_attention_weight_max": 18.13361930847168,
      "activations/layer1_attention_weight_min": -10.46186637878418,
      "activations/layer20_attention_weight_max": 38.387489318847656,
      "activations/layer20_attention_weight_min": -23.826705932617188,
      "activations/layer21_attention_weight_max": 34.36189270019531,
      "activations/layer21_attention_weight_min": -20.715194702148438,
      "activations/layer22_attention_weight_max": 44.05815505981445,
      "activations/layer22_attention_weight_min": -25.55520248413086,
      "activations/layer23_attention_weight_max": 25.811613082885742,
      "activations/layer23_attention_weight_min": -17.272167205810547,
      "activations/layer2_attention_weight_max": 29.22400665283203,
      "activations/layer2_attention_weight_min": -28.021808624267578,
      "activations/layer3_attention_weight_max": 74.805908203125,
      "activations/layer3_attention_weight_min": -82.17957305908203,
      "activations/layer4_attention_weight_max": 126.2802963256836,
      "activations/layer4_attention_weight_min": -122.90425109863281,
      "activations/layer5_attention_weight_max": 119.3521728515625,
      "activations/layer5_attention_weight_min": -108.2594223022461,
      "activations/layer6_attention_weight_max": 61.99409484863281,
      "activations/layer6_attention_weight_min": -54.30109786987305,
      "activations/layer7_attention_weight_max": 62.55661392211914,
      "activations/layer7_attention_weight_min": -67.3236312866211,
      "activations/layer8_attention_weight_max": 47.47702407836914,
      "activations/layer8_attention_weight_min": -43.21952819824219,
      "activations/layer9_attention_weight_max": 69.15901184082031,
      "activations/layer9_attention_weight_min": -57.72251510620117,
      "epoch": 1.82,
      "learning_rate": 0.00013965643939393938,
      "loss": 2.9958,
      "step": 31350
    },
    {
      "activations/layer0_attention_weight_max": 14.097541809082031,
      "activations/layer0_attention_weight_min": -11.556215286254883,
      "activations/layer10_attention_weight_max": 53.99757766723633,
      "activations/layer10_attention_weight_min": -48.402610778808594,
      "activations/layer11_attention_weight_max": 42.06554412841797,
      "activations/layer11_attention_weight_min": -35.4483757019043,
      "activations/layer12_attention_weight_max": 36.03650665283203,
      "activations/layer12_attention_weight_min": -35.23267364501953,
      "activations/layer13_attention_weight_max": 51.80979919433594,
      "activations/layer13_attention_weight_min": -37.38486862182617,
      "activations/layer14_attention_weight_max": 75.77318572998047,
      "activations/layer14_attention_weight_min": -58.473716735839844,
      "activations/layer15_attention_weight_max": 57.621360778808594,
      "activations/layer15_attention_weight_min": -41.8086051940918,
      "activations/layer16_attention_weight_max": 54.844940185546875,
      "activations/layer16_attention_weight_min": -38.34217071533203,
      "activations/layer17_attention_weight_max": 46.10326385498047,
      "activations/layer17_attention_weight_min": -30.086450576782227,
      "activations/layer18_attention_weight_max": 28.52179527282715,
      "activations/layer18_attention_weight_min": -16.958120346069336,
      "activations/layer19_attention_weight_max": 54.69025421142578,
      "activations/layer19_attention_weight_min": -32.9229621887207,
      "activations/layer1_attention_weight_max": 18.442636489868164,
      "activations/layer1_attention_weight_min": -10.505800247192383,
      "activations/layer20_attention_weight_max": 36.9813117980957,
      "activations/layer20_attention_weight_min": -25.196809768676758,
      "activations/layer21_attention_weight_max": 29.567882537841797,
      "activations/layer21_attention_weight_min": -17.953048706054688,
      "activations/layer22_attention_weight_max": 42.44431686401367,
      "activations/layer22_attention_weight_min": -25.56652069091797,
      "activations/layer23_attention_weight_max": 24.882999420166016,
      "activations/layer23_attention_weight_min": -14.48081111907959,
      "activations/layer2_attention_weight_max": 30.73630142211914,
      "activations/layer2_attention_weight_min": -30.192106246948242,
      "activations/layer3_attention_weight_max": 70.55026245117188,
      "activations/layer3_attention_weight_min": -71.9198989868164,
      "activations/layer4_attention_weight_max": 124.1011962890625,
      "activations/layer4_attention_weight_min": -112.28108978271484,
      "activations/layer5_attention_weight_max": 114.05406951904297,
      "activations/layer5_attention_weight_min": -97.2199935913086,
      "activations/layer6_attention_weight_max": 64.8734359741211,
      "activations/layer6_attention_weight_min": -52.65437316894531,
      "activations/layer7_attention_weight_max": 69.45162200927734,
      "activations/layer7_attention_weight_min": -62.92974090576172,
      "activations/layer8_attention_weight_max": 48.684226989746094,
      "activations/layer8_attention_weight_min": -39.72296142578125,
      "activations/layer9_attention_weight_max": 72.09999084472656,
      "activations/layer9_attention_weight_min": -59.8691291809082,
      "epoch": 1.82,
      "learning_rate": 0.00013963749999999997,
      "loss": 2.9827,
      "step": 31400
    },
    {
      "activations/layer0_attention_weight_max": 16.945884704589844,
      "activations/layer0_attention_weight_min": -11.661382675170898,
      "activations/layer10_attention_weight_max": 52.100120544433594,
      "activations/layer10_attention_weight_min": -41.81040954589844,
      "activations/layer11_attention_weight_max": 42.4237174987793,
      "activations/layer11_attention_weight_min": -34.721012115478516,
      "activations/layer12_attention_weight_max": 32.93017578125,
      "activations/layer12_attention_weight_min": -25.86969566345215,
      "activations/layer13_attention_weight_max": 50.173038482666016,
      "activations/layer13_attention_weight_min": -37.39809036254883,
      "activations/layer14_attention_weight_max": 79.45064544677734,
      "activations/layer14_attention_weight_min": -56.330020904541016,
      "activations/layer15_attention_weight_max": 56.75874328613281,
      "activations/layer15_attention_weight_min": -38.41431427001953,
      "activations/layer16_attention_weight_max": 54.288116455078125,
      "activations/layer16_attention_weight_min": -36.56300735473633,
      "activations/layer17_attention_weight_max": 50.0541877746582,
      "activations/layer17_attention_weight_min": -30.85521125793457,
      "activations/layer18_attention_weight_max": 24.72968101501465,
      "activations/layer18_attention_weight_min": -20.47669792175293,
      "activations/layer19_attention_weight_max": 58.00049591064453,
      "activations/layer19_attention_weight_min": -33.61732864379883,
      "activations/layer1_attention_weight_max": 17.703937530517578,
      "activations/layer1_attention_weight_min": -11.641389846801758,
      "activations/layer20_attention_weight_max": 39.16777038574219,
      "activations/layer20_attention_weight_min": -28.863109588623047,
      "activations/layer21_attention_weight_max": 27.906509399414062,
      "activations/layer21_attention_weight_min": -17.519302368164062,
      "activations/layer22_attention_weight_max": 40.30677032470703,
      "activations/layer22_attention_weight_min": -26.286497116088867,
      "activations/layer23_attention_weight_max": 26.599544525146484,
      "activations/layer23_attention_weight_min": -16.932514190673828,
      "activations/layer2_attention_weight_max": 27.431903839111328,
      "activations/layer2_attention_weight_min": -27.390531539916992,
      "activations/layer3_attention_weight_max": 66.39244842529297,
      "activations/layer3_attention_weight_min": -70.31082916259766,
      "activations/layer4_attention_weight_max": 120.51944732666016,
      "activations/layer4_attention_weight_min": -118.4793701171875,
      "activations/layer5_attention_weight_max": 109.18755340576172,
      "activations/layer5_attention_weight_min": -98.9631576538086,
      "activations/layer6_attention_weight_max": 59.4990119934082,
      "activations/layer6_attention_weight_min": -53.42050552368164,
      "activations/layer7_attention_weight_max": 66.37691497802734,
      "activations/layer7_attention_weight_min": -61.15016555786133,
      "activations/layer8_attention_weight_max": 51.0704231262207,
      "activations/layer8_attention_weight_min": -42.99300003051758,
      "activations/layer9_attention_weight_max": 59.31608200073242,
      "activations/layer9_attention_weight_min": -51.876216888427734,
      "epoch": 1.83,
      "learning_rate": 0.0001396185606060606,
      "loss": 2.9879,
      "step": 31450
    },
    {
      "activations/layer0_attention_weight_max": 16.00562286376953,
      "activations/layer0_attention_weight_min": -11.966216087341309,
      "activations/layer10_attention_weight_max": 55.14048767089844,
      "activations/layer10_attention_weight_min": -44.71979904174805,
      "activations/layer11_attention_weight_max": 41.106353759765625,
      "activations/layer11_attention_weight_min": -34.708099365234375,
      "activations/layer12_attention_weight_max": 38.46208953857422,
      "activations/layer12_attention_weight_min": -29.777128219604492,
      "activations/layer13_attention_weight_max": 49.98737716674805,
      "activations/layer13_attention_weight_min": -41.67626953125,
      "activations/layer14_attention_weight_max": 76.56586456298828,
      "activations/layer14_attention_weight_min": -58.69711685180664,
      "activations/layer15_attention_weight_max": 54.76435852050781,
      "activations/layer15_attention_weight_min": -38.94114685058594,
      "activations/layer16_attention_weight_max": 49.93413543701172,
      "activations/layer16_attention_weight_min": -36.362056732177734,
      "activations/layer17_attention_weight_max": 44.86994934082031,
      "activations/layer17_attention_weight_min": -31.073652267456055,
      "activations/layer18_attention_weight_max": 26.084348678588867,
      "activations/layer18_attention_weight_min": -16.587215423583984,
      "activations/layer19_attention_weight_max": 53.59528732299805,
      "activations/layer19_attention_weight_min": -34.28853988647461,
      "activations/layer1_attention_weight_max": 17.267436981201172,
      "activations/layer1_attention_weight_min": -12.450727462768555,
      "activations/layer20_attention_weight_max": 36.23702621459961,
      "activations/layer20_attention_weight_min": -23.430477142333984,
      "activations/layer21_attention_weight_max": 29.818273544311523,
      "activations/layer21_attention_weight_min": -18.00924301147461,
      "activations/layer22_attention_weight_max": 42.22675323486328,
      "activations/layer22_attention_weight_min": -24.998334884643555,
      "activations/layer23_attention_weight_max": 25.178632736206055,
      "activations/layer23_attention_weight_min": -15.784716606140137,
      "activations/layer2_attention_weight_max": 27.146421432495117,
      "activations/layer2_attention_weight_min": -27.451276779174805,
      "activations/layer3_attention_weight_max": 69.07111358642578,
      "activations/layer3_attention_weight_min": -75.39300537109375,
      "activations/layer4_attention_weight_max": 129.0264434814453,
      "activations/layer4_attention_weight_min": -116.718017578125,
      "activations/layer5_attention_weight_max": 118.41842651367188,
      "activations/layer5_attention_weight_min": -101.04744720458984,
      "activations/layer6_attention_weight_max": 65.98788452148438,
      "activations/layer6_attention_weight_min": -57.8618278503418,
      "activations/layer7_attention_weight_max": 68.91744995117188,
      "activations/layer7_attention_weight_min": -63.284034729003906,
      "activations/layer8_attention_weight_max": 49.3512077331543,
      "activations/layer8_attention_weight_min": -41.328243255615234,
      "activations/layer9_attention_weight_max": 74.09154510498047,
      "activations/layer9_attention_weight_min": -55.4068489074707,
      "epoch": 1.83,
      "learning_rate": 0.0001395996212121212,
      "loss": 2.9986,
      "step": 31500
    },
    {
      "activations/layer0_attention_weight_max": 15.486245155334473,
      "activations/layer0_attention_weight_min": -11.93750286102295,
      "activations/layer10_attention_weight_max": 51.52729415893555,
      "activations/layer10_attention_weight_min": -46.4713020324707,
      "activations/layer11_attention_weight_max": 46.37474060058594,
      "activations/layer11_attention_weight_min": -37.729400634765625,
      "activations/layer12_attention_weight_max": 37.179664611816406,
      "activations/layer12_attention_weight_min": -25.13711929321289,
      "activations/layer13_attention_weight_max": 65.90077209472656,
      "activations/layer13_attention_weight_min": -41.892574310302734,
      "activations/layer14_attention_weight_max": 76.85728454589844,
      "activations/layer14_attention_weight_min": -61.31488800048828,
      "activations/layer15_attention_weight_max": 61.59743881225586,
      "activations/layer15_attention_weight_min": -37.29517364501953,
      "activations/layer16_attention_weight_max": 53.70981216430664,
      "activations/layer16_attention_weight_min": -42.28098678588867,
      "activations/layer17_attention_weight_max": 47.20771408081055,
      "activations/layer17_attention_weight_min": -27.648998260498047,
      "activations/layer18_attention_weight_max": 32.540382385253906,
      "activations/layer18_attention_weight_min": -19.308935165405273,
      "activations/layer19_attention_weight_max": 59.807464599609375,
      "activations/layer19_attention_weight_min": -34.553855895996094,
      "activations/layer1_attention_weight_max": 16.72879409790039,
      "activations/layer1_attention_weight_min": -11.631223678588867,
      "activations/layer20_attention_weight_max": 40.81329345703125,
      "activations/layer20_attention_weight_min": -24.131790161132812,
      "activations/layer21_attention_weight_max": 29.342632293701172,
      "activations/layer21_attention_weight_min": -18.353809356689453,
      "activations/layer22_attention_weight_max": 43.65692901611328,
      "activations/layer22_attention_weight_min": -26.837005615234375,
      "activations/layer23_attention_weight_max": 24.23137664794922,
      "activations/layer23_attention_weight_min": -16.772403717041016,
      "activations/layer2_attention_weight_max": 29.821800231933594,
      "activations/layer2_attention_weight_min": -29.37935447692871,
      "activations/layer3_attention_weight_max": 69.44721984863281,
      "activations/layer3_attention_weight_min": -71.71724700927734,
      "activations/layer4_attention_weight_max": 127.6452407836914,
      "activations/layer4_attention_weight_min": -116.1565170288086,
      "activations/layer5_attention_weight_max": 115.74053192138672,
      "activations/layer5_attention_weight_min": -95.68234252929688,
      "activations/layer6_attention_weight_max": 63.46214294433594,
      "activations/layer6_attention_weight_min": -55.96984100341797,
      "activations/layer7_attention_weight_max": 70.67577362060547,
      "activations/layer7_attention_weight_min": -63.3831901550293,
      "activations/layer8_attention_weight_max": 55.170955657958984,
      "activations/layer8_attention_weight_min": -45.58095169067383,
      "activations/layer9_attention_weight_max": 84.73786926269531,
      "activations/layer9_attention_weight_min": -60.71075439453125,
      "epoch": 1.83,
      "learning_rate": 0.0001395806818181818,
      "loss": 2.9654,
      "step": 31550
    },
    {
      "activations/layer0_attention_weight_max": 16.690269470214844,
      "activations/layer0_attention_weight_min": -12.49375057220459,
      "activations/layer10_attention_weight_max": 53.2096061706543,
      "activations/layer10_attention_weight_min": -41.61969757080078,
      "activations/layer11_attention_weight_max": 41.1406364440918,
      "activations/layer11_attention_weight_min": -35.189208984375,
      "activations/layer12_attention_weight_max": 44.97952651977539,
      "activations/layer12_attention_weight_min": -34.63208770751953,
      "activations/layer13_attention_weight_max": 54.84737777709961,
      "activations/layer13_attention_weight_min": -37.20304870605469,
      "activations/layer14_attention_weight_max": 78.07227325439453,
      "activations/layer14_attention_weight_min": -54.835445404052734,
      "activations/layer15_attention_weight_max": 56.489349365234375,
      "activations/layer15_attention_weight_min": -38.66982650756836,
      "activations/layer16_attention_weight_max": 52.56079864501953,
      "activations/layer16_attention_weight_min": -37.94001770019531,
      "activations/layer17_attention_weight_max": 50.325111389160156,
      "activations/layer17_attention_weight_min": -32.335391998291016,
      "activations/layer18_attention_weight_max": 27.063983917236328,
      "activations/layer18_attention_weight_min": -19.058189392089844,
      "activations/layer19_attention_weight_max": 52.666900634765625,
      "activations/layer19_attention_weight_min": -33.72423553466797,
      "activations/layer1_attention_weight_max": 18.3084774017334,
      "activations/layer1_attention_weight_min": -10.93466567993164,
      "activations/layer20_attention_weight_max": 41.45690155029297,
      "activations/layer20_attention_weight_min": -25.6811580657959,
      "activations/layer21_attention_weight_max": 34.639652252197266,
      "activations/layer21_attention_weight_min": -19.17978286743164,
      "activations/layer22_attention_weight_max": 48.01458740234375,
      "activations/layer22_attention_weight_min": -27.25881576538086,
      "activations/layer23_attention_weight_max": 24.99762535095215,
      "activations/layer23_attention_weight_min": -15.782958984375,
      "activations/layer2_attention_weight_max": 28.031911849975586,
      "activations/layer2_attention_weight_min": -26.536815643310547,
      "activations/layer3_attention_weight_max": 69.0478286743164,
      "activations/layer3_attention_weight_min": -70.95208740234375,
      "activations/layer4_attention_weight_max": 124.7959213256836,
      "activations/layer4_attention_weight_min": -114.89697265625,
      "activations/layer5_attention_weight_max": 117.59249114990234,
      "activations/layer5_attention_weight_min": -97.98658752441406,
      "activations/layer6_attention_weight_max": 69.6402587890625,
      "activations/layer6_attention_weight_min": -53.9254264831543,
      "activations/layer7_attention_weight_max": 64.71390533447266,
      "activations/layer7_attention_weight_min": -64.82813262939453,
      "activations/layer8_attention_weight_max": 50.066612243652344,
      "activations/layer8_attention_weight_min": -41.41415023803711,
      "activations/layer9_attention_weight_max": 70.17823028564453,
      "activations/layer9_attention_weight_min": -63.8560676574707,
      "epoch": 1.84,
      "learning_rate": 0.00013956212121212118,
      "loss": 3.0091,
      "step": 31600
    },
    {
      "activations/layer0_attention_weight_max": 17.307735443115234,
      "activations/layer0_attention_weight_min": -12.089632034301758,
      "activations/layer10_attention_weight_max": 60.15238952636719,
      "activations/layer10_attention_weight_min": -48.7175407409668,
      "activations/layer11_attention_weight_max": 45.39427947998047,
      "activations/layer11_attention_weight_min": -39.4910888671875,
      "activations/layer12_attention_weight_max": 42.06049346923828,
      "activations/layer12_attention_weight_min": -31.505603790283203,
      "activations/layer13_attention_weight_max": 54.91814041137695,
      "activations/layer13_attention_weight_min": -38.87751388549805,
      "activations/layer14_attention_weight_max": 87.4883804321289,
      "activations/layer14_attention_weight_min": -59.915618896484375,
      "activations/layer15_attention_weight_max": 64.64215850830078,
      "activations/layer15_attention_weight_min": -40.7768440246582,
      "activations/layer16_attention_weight_max": 52.70289611816406,
      "activations/layer16_attention_weight_min": -38.13390350341797,
      "activations/layer17_attention_weight_max": 53.22298049926758,
      "activations/layer17_attention_weight_min": -34.39726638793945,
      "activations/layer18_attention_weight_max": 31.357614517211914,
      "activations/layer18_attention_weight_min": -16.17522621154785,
      "activations/layer19_attention_weight_max": 56.4051628112793,
      "activations/layer19_attention_weight_min": -34.34181594848633,
      "activations/layer1_attention_weight_max": 18.307727813720703,
      "activations/layer1_attention_weight_min": -11.268129348754883,
      "activations/layer20_attention_weight_max": 42.061241149902344,
      "activations/layer20_attention_weight_min": -26.210193634033203,
      "activations/layer21_attention_weight_max": 34.29557800292969,
      "activations/layer21_attention_weight_min": -20.020545959472656,
      "activations/layer22_attention_weight_max": 45.872802734375,
      "activations/layer22_attention_weight_min": -24.93907928466797,
      "activations/layer23_attention_weight_max": 26.323638916015625,
      "activations/layer23_attention_weight_min": -16.807819366455078,
      "activations/layer2_attention_weight_max": 30.093303680419922,
      "activations/layer2_attention_weight_min": -30.146589279174805,
      "activations/layer3_attention_weight_max": 72.25654602050781,
      "activations/layer3_attention_weight_min": -76.10588073730469,
      "activations/layer4_attention_weight_max": 129.36773681640625,
      "activations/layer4_attention_weight_min": -123.85279846191406,
      "activations/layer5_attention_weight_max": 121.36279296875,
      "activations/layer5_attention_weight_min": -100.69268798828125,
      "activations/layer6_attention_weight_max": 65.69058227539062,
      "activations/layer6_attention_weight_min": -55.085304260253906,
      "activations/layer7_attention_weight_max": 66.50062561035156,
      "activations/layer7_attention_weight_min": -63.777923583984375,
      "activations/layer8_attention_weight_max": 55.29276657104492,
      "activations/layer8_attention_weight_min": -43.21688461303711,
      "activations/layer9_attention_weight_max": 73.15384674072266,
      "activations/layer9_attention_weight_min": -59.11190414428711,
      "epoch": 1.84,
      "learning_rate": 0.0001395431818181818,
      "loss": 2.9922,
      "step": 31650
    },
    {
      "activations/layer0_attention_weight_max": 15.621495246887207,
      "activations/layer0_attention_weight_min": -11.827381134033203,
      "activations/layer10_attention_weight_max": 53.94406509399414,
      "activations/layer10_attention_weight_min": -42.339691162109375,
      "activations/layer11_attention_weight_max": 39.85090255737305,
      "activations/layer11_attention_weight_min": -34.77201461791992,
      "activations/layer12_attention_weight_max": 32.376312255859375,
      "activations/layer12_attention_weight_min": -29.624343872070312,
      "activations/layer13_attention_weight_max": 46.2540168762207,
      "activations/layer13_attention_weight_min": -45.691978454589844,
      "activations/layer14_attention_weight_max": 72.89729309082031,
      "activations/layer14_attention_weight_min": -61.55168914794922,
      "activations/layer15_attention_weight_max": 49.76177215576172,
      "activations/layer15_attention_weight_min": -37.25212097167969,
      "activations/layer16_attention_weight_max": 54.1832160949707,
      "activations/layer16_attention_weight_min": -37.76506042480469,
      "activations/layer17_attention_weight_max": 43.59684371948242,
      "activations/layer17_attention_weight_min": -29.307031631469727,
      "activations/layer18_attention_weight_max": 25.79866600036621,
      "activations/layer18_attention_weight_min": -18.735441207885742,
      "activations/layer19_attention_weight_max": 60.74858474731445,
      "activations/layer19_attention_weight_min": -34.08189010620117,
      "activations/layer1_attention_weight_max": 17.05207061767578,
      "activations/layer1_attention_weight_min": -12.763256072998047,
      "activations/layer20_attention_weight_max": 36.374473571777344,
      "activations/layer20_attention_weight_min": -25.3319091796875,
      "activations/layer21_attention_weight_max": 33.93562316894531,
      "activations/layer21_attention_weight_min": -18.912029266357422,
      "activations/layer22_attention_weight_max": 42.43714904785156,
      "activations/layer22_attention_weight_min": -24.99460792541504,
      "activations/layer23_attention_weight_max": 24.360530853271484,
      "activations/layer23_attention_weight_min": -16.023117065429688,
      "activations/layer2_attention_weight_max": 28.093963623046875,
      "activations/layer2_attention_weight_min": -29.16432762145996,
      "activations/layer3_attention_weight_max": 70.40489959716797,
      "activations/layer3_attention_weight_min": -75.32616424560547,
      "activations/layer4_attention_weight_max": 129.00148010253906,
      "activations/layer4_attention_weight_min": -115.06805419921875,
      "activations/layer5_attention_weight_max": 119.13921356201172,
      "activations/layer5_attention_weight_min": -101.97177124023438,
      "activations/layer6_attention_weight_max": 65.81581115722656,
      "activations/layer6_attention_weight_min": -55.0401496887207,
      "activations/layer7_attention_weight_max": 65.7261734008789,
      "activations/layer7_attention_weight_min": -63.40151596069336,
      "activations/layer8_attention_weight_max": 48.23598098754883,
      "activations/layer8_attention_weight_min": -41.4095573425293,
      "activations/layer9_attention_weight_max": 64.82035064697266,
      "activations/layer9_attention_weight_min": -53.3082275390625,
      "epoch": 1.84,
      "learning_rate": 0.0001395242424242424,
      "loss": 2.9887,
      "step": 31700
    },
    {
      "activations/layer0_attention_weight_max": 17.556690216064453,
      "activations/layer0_attention_weight_min": -11.608648300170898,
      "activations/layer10_attention_weight_max": 49.30711364746094,
      "activations/layer10_attention_weight_min": -40.50374221801758,
      "activations/layer11_attention_weight_max": 36.84835433959961,
      "activations/layer11_attention_weight_min": -33.775718688964844,
      "activations/layer12_attention_weight_max": 33.38728713989258,
      "activations/layer12_attention_weight_min": -24.646371841430664,
      "activations/layer13_attention_weight_max": 45.18019485473633,
      "activations/layer13_attention_weight_min": -38.60809326171875,
      "activations/layer14_attention_weight_max": 67.37963104248047,
      "activations/layer14_attention_weight_min": -57.618690490722656,
      "activations/layer15_attention_weight_max": 52.89913558959961,
      "activations/layer15_attention_weight_min": -38.051300048828125,
      "activations/layer16_attention_weight_max": 50.440853118896484,
      "activations/layer16_attention_weight_min": -37.47427749633789,
      "activations/layer17_attention_weight_max": 44.0606575012207,
      "activations/layer17_attention_weight_min": -28.56067657470703,
      "activations/layer18_attention_weight_max": 24.701534271240234,
      "activations/layer18_attention_weight_min": -16.08690643310547,
      "activations/layer19_attention_weight_max": 52.145179748535156,
      "activations/layer19_attention_weight_min": -32.22611618041992,
      "activations/layer1_attention_weight_max": 18.43675994873047,
      "activations/layer1_attention_weight_min": -11.262872695922852,
      "activations/layer20_attention_weight_max": 39.78093719482422,
      "activations/layer20_attention_weight_min": -25.984973907470703,
      "activations/layer21_attention_weight_max": 33.43488311767578,
      "activations/layer21_attention_weight_min": -20.687631607055664,
      "activations/layer22_attention_weight_max": 44.62165832519531,
      "activations/layer22_attention_weight_min": -26.74982452392578,
      "activations/layer23_attention_weight_max": 25.240253448486328,
      "activations/layer23_attention_weight_min": -16.654155731201172,
      "activations/layer2_attention_weight_max": 27.429006576538086,
      "activations/layer2_attention_weight_min": -27.598426818847656,
      "activations/layer3_attention_weight_max": 68.61087799072266,
      "activations/layer3_attention_weight_min": -69.93472290039062,
      "activations/layer4_attention_weight_max": 118.18267822265625,
      "activations/layer4_attention_weight_min": -112.0451889038086,
      "activations/layer5_attention_weight_max": 112.40512084960938,
      "activations/layer5_attention_weight_min": -96.43772888183594,
      "activations/layer6_attention_weight_max": 59.4483528137207,
      "activations/layer6_attention_weight_min": -56.2099609375,
      "activations/layer7_attention_weight_max": 60.28971481323242,
      "activations/layer7_attention_weight_min": -58.06482696533203,
      "activations/layer8_attention_weight_max": 46.53013229370117,
      "activations/layer8_attention_weight_min": -40.487300872802734,
      "activations/layer9_attention_weight_max": 57.108314514160156,
      "activations/layer9_attention_weight_min": -49.198638916015625,
      "epoch": 1.84,
      "learning_rate": 0.000139505303030303,
      "loss": 2.9808,
      "step": 31750
    },
    {
      "activations/layer0_attention_weight_max": 16.43423843383789,
      "activations/layer0_attention_weight_min": -11.33059310913086,
      "activations/layer10_attention_weight_max": 51.54611587524414,
      "activations/layer10_attention_weight_min": -44.85246658325195,
      "activations/layer11_attention_weight_max": 36.080806732177734,
      "activations/layer11_attention_weight_min": -34.617408752441406,
      "activations/layer12_attention_weight_max": 33.260398864746094,
      "activations/layer12_attention_weight_min": -30.86199188232422,
      "activations/layer13_attention_weight_max": 51.06059265136719,
      "activations/layer13_attention_weight_min": -43.90260696411133,
      "activations/layer14_attention_weight_max": 83.14359283447266,
      "activations/layer14_attention_weight_min": -64.5840835571289,
      "activations/layer15_attention_weight_max": 58.04622268676758,
      "activations/layer15_attention_weight_min": -41.523338317871094,
      "activations/layer16_attention_weight_max": 50.89749526977539,
      "activations/layer16_attention_weight_min": -33.7865104675293,
      "activations/layer17_attention_weight_max": 44.10023880004883,
      "activations/layer17_attention_weight_min": -30.069679260253906,
      "activations/layer18_attention_weight_max": 27.331552505493164,
      "activations/layer18_attention_weight_min": -17.33116912841797,
      "activations/layer19_attention_weight_max": 52.500518798828125,
      "activations/layer19_attention_weight_min": -30.143152236938477,
      "activations/layer1_attention_weight_max": 18.71977424621582,
      "activations/layer1_attention_weight_min": -10.869857788085938,
      "activations/layer20_attention_weight_max": 38.399959564208984,
      "activations/layer20_attention_weight_min": -24.37644386291504,
      "activations/layer21_attention_weight_max": 32.53249740600586,
      "activations/layer21_attention_weight_min": -19.658145904541016,
      "activations/layer22_attention_weight_max": 44.74787902832031,
      "activations/layer22_attention_weight_min": -24.078001022338867,
      "activations/layer23_attention_weight_max": 25.508140563964844,
      "activations/layer23_attention_weight_min": -16.521068572998047,
      "activations/layer2_attention_weight_max": 27.928199768066406,
      "activations/layer2_attention_weight_min": -27.442562103271484,
      "activations/layer3_attention_weight_max": 69.37232971191406,
      "activations/layer3_attention_weight_min": -68.12467193603516,
      "activations/layer4_attention_weight_max": 121.61710357666016,
      "activations/layer4_attention_weight_min": -109.50935363769531,
      "activations/layer5_attention_weight_max": 113.45443725585938,
      "activations/layer5_attention_weight_min": -93.49153137207031,
      "activations/layer6_attention_weight_max": 60.354774475097656,
      "activations/layer6_attention_weight_min": -53.22468185424805,
      "activations/layer7_attention_weight_max": 63.946746826171875,
      "activations/layer7_attention_weight_min": -59.37857437133789,
      "activations/layer8_attention_weight_max": 45.493717193603516,
      "activations/layer8_attention_weight_min": -38.84146499633789,
      "activations/layer9_attention_weight_max": 71.98456573486328,
      "activations/layer9_attention_weight_min": -53.603004455566406,
      "epoch": 1.85,
      "learning_rate": 0.00013948636363636361,
      "loss": 2.9895,
      "step": 31800
    },
    {
      "activations/layer0_attention_weight_max": 17.197490692138672,
      "activations/layer0_attention_weight_min": -11.931601524353027,
      "activations/layer10_attention_weight_max": 55.476165771484375,
      "activations/layer10_attention_weight_min": -43.33395004272461,
      "activations/layer11_attention_weight_max": 38.07489776611328,
      "activations/layer11_attention_weight_min": -32.58476257324219,
      "activations/layer12_attention_weight_max": 37.80294418334961,
      "activations/layer12_attention_weight_min": -26.915420532226562,
      "activations/layer13_attention_weight_max": 49.86133575439453,
      "activations/layer13_attention_weight_min": -35.42301559448242,
      "activations/layer14_attention_weight_max": 77.2063217163086,
      "activations/layer14_attention_weight_min": -56.699344635009766,
      "activations/layer15_attention_weight_max": 53.996944427490234,
      "activations/layer15_attention_weight_min": -36.48743438720703,
      "activations/layer16_attention_weight_max": 49.72168731689453,
      "activations/layer16_attention_weight_min": -36.96745681762695,
      "activations/layer17_attention_weight_max": 48.33068084716797,
      "activations/layer17_attention_weight_min": -30.88888168334961,
      "activations/layer18_attention_weight_max": 27.84187126159668,
      "activations/layer18_attention_weight_min": -18.129865646362305,
      "activations/layer19_attention_weight_max": 51.905879974365234,
      "activations/layer19_attention_weight_min": -32.53841781616211,
      "activations/layer1_attention_weight_max": 17.841434478759766,
      "activations/layer1_attention_weight_min": -11.102621078491211,
      "activations/layer20_attention_weight_max": 43.998504638671875,
      "activations/layer20_attention_weight_min": -26.071521759033203,
      "activations/layer21_attention_weight_max": 28.969120025634766,
      "activations/layer21_attention_weight_min": -19.18900489807129,
      "activations/layer22_attention_weight_max": 45.838932037353516,
      "activations/layer22_attention_weight_min": -28.525054931640625,
      "activations/layer23_attention_weight_max": 23.4134578704834,
      "activations/layer23_attention_weight_min": -16.071029663085938,
      "activations/layer2_attention_weight_max": 28.93977928161621,
      "activations/layer2_attention_weight_min": -28.05365562438965,
      "activations/layer3_attention_weight_max": 69.45513916015625,
      "activations/layer3_attention_weight_min": -70.73526763916016,
      "activations/layer4_attention_weight_max": 126.83731842041016,
      "activations/layer4_attention_weight_min": -115.01429748535156,
      "activations/layer5_attention_weight_max": 122.249755859375,
      "activations/layer5_attention_weight_min": -97.5879135131836,
      "activations/layer6_attention_weight_max": 65.23246765136719,
      "activations/layer6_attention_weight_min": -52.15208053588867,
      "activations/layer7_attention_weight_max": 67.11869812011719,
      "activations/layer7_attention_weight_min": -62.00514602661133,
      "activations/layer8_attention_weight_max": 47.874210357666016,
      "activations/layer8_attention_weight_min": -40.86954879760742,
      "activations/layer9_attention_weight_max": 66.48944854736328,
      "activations/layer9_attention_weight_min": -52.566864013671875,
      "epoch": 1.85,
      "learning_rate": 0.00013946742424242423,
      "loss": 2.9925,
      "step": 31850
    },
    {
      "activations/layer0_attention_weight_max": 17.11050796508789,
      "activations/layer0_attention_weight_min": -12.544798851013184,
      "activations/layer10_attention_weight_max": 48.5855598449707,
      "activations/layer10_attention_weight_min": -42.94813537597656,
      "activations/layer11_attention_weight_max": 38.54909896850586,
      "activations/layer11_attention_weight_min": -33.23395538330078,
      "activations/layer12_attention_weight_max": 36.27535629272461,
      "activations/layer12_attention_weight_min": -32.23838424682617,
      "activations/layer13_attention_weight_max": 49.044490814208984,
      "activations/layer13_attention_weight_min": -37.3073616027832,
      "activations/layer14_attention_weight_max": 69.36329650878906,
      "activations/layer14_attention_weight_min": -51.84321594238281,
      "activations/layer15_attention_weight_max": 48.70354461669922,
      "activations/layer15_attention_weight_min": -35.37733840942383,
      "activations/layer16_attention_weight_max": 47.98851776123047,
      "activations/layer16_attention_weight_min": -37.43703079223633,
      "activations/layer17_attention_weight_max": 44.18477249145508,
      "activations/layer17_attention_weight_min": -32.52986526489258,
      "activations/layer18_attention_weight_max": 31.177669525146484,
      "activations/layer18_attention_weight_min": -16.867887496948242,
      "activations/layer19_attention_weight_max": 53.06502151489258,
      "activations/layer19_attention_weight_min": -32.88758850097656,
      "activations/layer1_attention_weight_max": 16.988527297973633,
      "activations/layer1_attention_weight_min": -11.967001914978027,
      "activations/layer20_attention_weight_max": 39.61721420288086,
      "activations/layer20_attention_weight_min": -27.914133071899414,
      "activations/layer21_attention_weight_max": 32.37060546875,
      "activations/layer21_attention_weight_min": -19.481151580810547,
      "activations/layer22_attention_weight_max": 46.43415069580078,
      "activations/layer22_attention_weight_min": -28.096799850463867,
      "activations/layer23_attention_weight_max": 28.18019676208496,
      "activations/layer23_attention_weight_min": -15.840964317321777,
      "activations/layer2_attention_weight_max": 27.867210388183594,
      "activations/layer2_attention_weight_min": -28.557104110717773,
      "activations/layer3_attention_weight_max": 69.41206359863281,
      "activations/layer3_attention_weight_min": -74.91460418701172,
      "activations/layer4_attention_weight_max": 125.906005859375,
      "activations/layer4_attention_weight_min": -116.15059661865234,
      "activations/layer5_attention_weight_max": 112.9913330078125,
      "activations/layer5_attention_weight_min": -101.4725570678711,
      "activations/layer6_attention_weight_max": 60.211124420166016,
      "activations/layer6_attention_weight_min": -54.32588195800781,
      "activations/layer7_attention_weight_max": 61.328460693359375,
      "activations/layer7_attention_weight_min": -60.41102600097656,
      "activations/layer8_attention_weight_max": 48.25226593017578,
      "activations/layer8_attention_weight_min": -45.635414123535156,
      "activations/layer9_attention_weight_max": 73.3511962890625,
      "activations/layer9_attention_weight_min": -55.99126052856445,
      "epoch": 1.85,
      "learning_rate": 0.00013944848484848484,
      "loss": 2.9941,
      "step": 31900
    },
    {
      "activations/layer0_attention_weight_max": 16.690492630004883,
      "activations/layer0_attention_weight_min": -12.297178268432617,
      "activations/layer10_attention_weight_max": 57.38296890258789,
      "activations/layer10_attention_weight_min": -43.58568572998047,
      "activations/layer11_attention_weight_max": 44.73679733276367,
      "activations/layer11_attention_weight_min": -37.55082702636719,
      "activations/layer12_attention_weight_max": 54.83971405029297,
      "activations/layer12_attention_weight_min": -40.34368896484375,
      "activations/layer13_attention_weight_max": 60.09457778930664,
      "activations/layer13_attention_weight_min": -41.037384033203125,
      "activations/layer14_attention_weight_max": 84.26397705078125,
      "activations/layer14_attention_weight_min": -61.873817443847656,
      "activations/layer15_attention_weight_max": 60.58878707885742,
      "activations/layer15_attention_weight_min": -39.63070297241211,
      "activations/layer16_attention_weight_max": 63.116641998291016,
      "activations/layer16_attention_weight_min": -37.9368896484375,
      "activations/layer17_attention_weight_max": 42.79501724243164,
      "activations/layer17_attention_weight_min": -27.765888214111328,
      "activations/layer18_attention_weight_max": 25.77202033996582,
      "activations/layer18_attention_weight_min": -18.461702346801758,
      "activations/layer19_attention_weight_max": 60.88648223876953,
      "activations/layer19_attention_weight_min": -29.501245498657227,
      "activations/layer1_attention_weight_max": 18.85462188720703,
      "activations/layer1_attention_weight_min": -10.374698638916016,
      "activations/layer20_attention_weight_max": 55.53155517578125,
      "activations/layer20_attention_weight_min": -24.668062210083008,
      "activations/layer21_attention_weight_max": 36.60089874267578,
      "activations/layer21_attention_weight_min": -19.090347290039062,
      "activations/layer22_attention_weight_max": 44.804161071777344,
      "activations/layer22_attention_weight_min": -23.936187744140625,
      "activations/layer23_attention_weight_max": 26.859291076660156,
      "activations/layer23_attention_weight_min": -15.21931266784668,
      "activations/layer2_attention_weight_max": 27.783981323242188,
      "activations/layer2_attention_weight_min": -25.94150161743164,
      "activations/layer3_attention_weight_max": 68.6394271850586,
      "activations/layer3_attention_weight_min": -69.66016387939453,
      "activations/layer4_attention_weight_max": 125.34590911865234,
      "activations/layer4_attention_weight_min": -112.3896484375,
      "activations/layer5_attention_weight_max": 110.24557495117188,
      "activations/layer5_attention_weight_min": -92.08412170410156,
      "activations/layer6_attention_weight_max": 60.98810958862305,
      "activations/layer6_attention_weight_min": -52.48691940307617,
      "activations/layer7_attention_weight_max": 69.3896255493164,
      "activations/layer7_attention_weight_min": -64.64534759521484,
      "activations/layer8_attention_weight_max": 51.60895538330078,
      "activations/layer8_attention_weight_min": -45.41265869140625,
      "activations/layer9_attention_weight_max": 73.10674285888672,
      "activations/layer9_attention_weight_min": -62.53639602661133,
      "epoch": 1.86,
      "learning_rate": 0.00013942954545454543,
      "loss": 2.9872,
      "step": 31950
    },
    {
      "activations/layer0_attention_weight_max": 16.573720932006836,
      "activations/layer0_attention_weight_min": -11.869732856750488,
      "activations/layer10_attention_weight_max": 49.715057373046875,
      "activations/layer10_attention_weight_min": -44.5112190246582,
      "activations/layer11_attention_weight_max": 39.23143768310547,
      "activations/layer11_attention_weight_min": -33.85491943359375,
      "activations/layer12_attention_weight_max": 36.21178436279297,
      "activations/layer12_attention_weight_min": -28.206674575805664,
      "activations/layer13_attention_weight_max": 54.48318099975586,
      "activations/layer13_attention_weight_min": -39.60097885131836,
      "activations/layer14_attention_weight_max": 79.66020202636719,
      "activations/layer14_attention_weight_min": -59.3673095703125,
      "activations/layer15_attention_weight_max": 54.05470657348633,
      "activations/layer15_attention_weight_min": -37.08576202392578,
      "activations/layer16_attention_weight_max": 55.050045013427734,
      "activations/layer16_attention_weight_min": -41.2958984375,
      "activations/layer17_attention_weight_max": 46.30220413208008,
      "activations/layer17_attention_weight_min": -30.03025245666504,
      "activations/layer18_attention_weight_max": 26.260351181030273,
      "activations/layer18_attention_weight_min": -17.887252807617188,
      "activations/layer19_attention_weight_max": 52.71162033081055,
      "activations/layer19_attention_weight_min": -33.57155990600586,
      "activations/layer1_attention_weight_max": 17.295997619628906,
      "activations/layer1_attention_weight_min": -11.516822814941406,
      "activations/layer20_attention_weight_max": 37.786067962646484,
      "activations/layer20_attention_weight_min": -25.858469009399414,
      "activations/layer21_attention_weight_max": 34.43134689331055,
      "activations/layer21_attention_weight_min": -17.174890518188477,
      "activations/layer22_attention_weight_max": 48.372894287109375,
      "activations/layer22_attention_weight_min": -25.40664291381836,
      "activations/layer23_attention_weight_max": 29.4248046875,
      "activations/layer23_attention_weight_min": -19.928003311157227,
      "activations/layer2_attention_weight_max": 27.272659301757812,
      "activations/layer2_attention_weight_min": -27.370929718017578,
      "activations/layer3_attention_weight_max": 69.23827362060547,
      "activations/layer3_attention_weight_min": -70.6683349609375,
      "activations/layer4_attention_weight_max": 124.34602355957031,
      "activations/layer4_attention_weight_min": -115.68330383300781,
      "activations/layer5_attention_weight_max": 110.84310913085938,
      "activations/layer5_attention_weight_min": -97.90875244140625,
      "activations/layer6_attention_weight_max": 60.21668243408203,
      "activations/layer6_attention_weight_min": -52.47468566894531,
      "activations/layer7_attention_weight_max": 69.03370666503906,
      "activations/layer7_attention_weight_min": -63.82560348510742,
      "activations/layer8_attention_weight_max": 47.87956619262695,
      "activations/layer8_attention_weight_min": -44.63133239746094,
      "activations/layer9_attention_weight_max": 71.63352966308594,
      "activations/layer9_attention_weight_min": -58.7800178527832,
      "epoch": 1.86,
      "learning_rate": 0.00013941060606060605,
      "loss": 2.9912,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_loss": 2.927734375,
      "eval_runtime": 8.4917,
      "eval_samples_per_second": 505.669,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_openwebtext_loss": 2.927734375,
      "eval_openwebtext_ppl": 18.6852487375444,
      "eval_openwebtext_runtime": 8.4917,
      "eval_openwebtext_samples_per_second": 505.669,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_wikitext_loss": 3.1953125,
      "eval_wikitext_ppl": 24.417803063601138,
      "eval_wikitext_runtime": 1.9621,
      "eval_wikitext_samples_per_second": 232.407,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_lambada_loss": 3.203125,
      "eval_lambada_ppl": 24.609314266596865,
      "eval_lambada_runtime": 9.561,
      "eval_lambada_samples_per_second": 509.257,
      "step": 32000
    },
    {
      "activations/layer0_attention_weight_max": 17.30756378173828,
      "activations/layer0_attention_weight_min": -12.340723991394043,
      "activations/layer10_attention_weight_max": 52.07393264770508,
      "activations/layer10_attention_weight_min": -46.43265151977539,
      "activations/layer11_attention_weight_max": 43.00836181640625,
      "activations/layer11_attention_weight_min": -36.075416564941406,
      "activations/layer12_attention_weight_max": 64.71279907226562,
      "activations/layer12_attention_weight_min": -31.73615074157715,
      "activations/layer13_attention_weight_max": 54.229427337646484,
      "activations/layer13_attention_weight_min": -41.34185791015625,
      "activations/layer14_attention_weight_max": 81.5699691772461,
      "activations/layer14_attention_weight_min": -60.25703048706055,
      "activations/layer15_attention_weight_max": 50.99485778808594,
      "activations/layer15_attention_weight_min": -38.4522590637207,
      "activations/layer16_attention_weight_max": 54.90847396850586,
      "activations/layer16_attention_weight_min": -40.494808197021484,
      "activations/layer17_attention_weight_max": 41.389122009277344,
      "activations/layer17_attention_weight_min": -30.330034255981445,
      "activations/layer18_attention_weight_max": 25.129880905151367,
      "activations/layer18_attention_weight_min": -17.912639617919922,
      "activations/layer19_attention_weight_max": 54.33369064331055,
      "activations/layer19_attention_weight_min": -31.859968185424805,
      "activations/layer1_attention_weight_max": 17.597135543823242,
      "activations/layer1_attention_weight_min": -11.360281944274902,
      "activations/layer20_attention_weight_max": 37.584144592285156,
      "activations/layer20_attention_weight_min": -25.33885383605957,
      "activations/layer21_attention_weight_max": 31.142154693603516,
      "activations/layer21_attention_weight_min": -19.569530487060547,
      "activations/layer22_attention_weight_max": 44.43849182128906,
      "activations/layer22_attention_weight_min": -28.402925491333008,
      "activations/layer23_attention_weight_max": 24.805456161499023,
      "activations/layer23_attention_weight_min": -15.313682556152344,
      "activations/layer2_attention_weight_max": 27.91646957397461,
      "activations/layer2_attention_weight_min": -27.084280014038086,
      "activations/layer3_attention_weight_max": 66.37422943115234,
      "activations/layer3_attention_weight_min": -70.33039855957031,
      "activations/layer4_attention_weight_max": 120.71172332763672,
      "activations/layer4_attention_weight_min": -111.64946746826172,
      "activations/layer5_attention_weight_max": 111.71246337890625,
      "activations/layer5_attention_weight_min": -94.45870971679688,
      "activations/layer6_attention_weight_max": 68.70789337158203,
      "activations/layer6_attention_weight_min": -55.2098503112793,
      "activations/layer7_attention_weight_max": 63.44406509399414,
      "activations/layer7_attention_weight_min": -62.914249420166016,
      "activations/layer8_attention_weight_max": 49.75433349609375,
      "activations/layer8_attention_weight_min": -42.86294937133789,
      "activations/layer9_attention_weight_max": 74.01163482666016,
      "activations/layer9_attention_weight_min": -52.03874588012695,
      "epoch": 1.86,
      "learning_rate": 0.00013939166666666666,
      "loss": 2.9749,
      "step": 32050
    },
    {
      "activations/layer0_attention_weight_max": 14.271556854248047,
      "activations/layer0_attention_weight_min": -11.949716567993164,
      "activations/layer10_attention_weight_max": 62.627437591552734,
      "activations/layer10_attention_weight_min": -53.187679290771484,
      "activations/layer11_attention_weight_max": 45.477638244628906,
      "activations/layer11_attention_weight_min": -40.277618408203125,
      "activations/layer12_attention_weight_max": 65.86331939697266,
      "activations/layer12_attention_weight_min": -43.438148498535156,
      "activations/layer13_attention_weight_max": 58.85099411010742,
      "activations/layer13_attention_weight_min": -37.9404296875,
      "activations/layer14_attention_weight_max": 89.62440490722656,
      "activations/layer14_attention_weight_min": -63.88933181762695,
      "activations/layer15_attention_weight_max": 61.951881408691406,
      "activations/layer15_attention_weight_min": -39.74784851074219,
      "activations/layer16_attention_weight_max": 57.269866943359375,
      "activations/layer16_attention_weight_min": -35.257633209228516,
      "activations/layer17_attention_weight_max": 52.33671951293945,
      "activations/layer17_attention_weight_min": -31.26387596130371,
      "activations/layer18_attention_weight_max": 31.136432647705078,
      "activations/layer18_attention_weight_min": -17.94146728515625,
      "activations/layer19_attention_weight_max": 56.86874008178711,
      "activations/layer19_attention_weight_min": -32.07015609741211,
      "activations/layer1_attention_weight_max": 17.677444458007812,
      "activations/layer1_attention_weight_min": -11.315394401550293,
      "activations/layer20_attention_weight_max": 42.706363677978516,
      "activations/layer20_attention_weight_min": -24.810800552368164,
      "activations/layer21_attention_weight_max": 31.048336029052734,
      "activations/layer21_attention_weight_min": -19.35527992248535,
      "activations/layer22_attention_weight_max": 43.99851608276367,
      "activations/layer22_attention_weight_min": -23.31873893737793,
      "activations/layer23_attention_weight_max": 24.864810943603516,
      "activations/layer23_attention_weight_min": -15.14737319946289,
      "activations/layer2_attention_weight_max": 28.16522216796875,
      "activations/layer2_attention_weight_min": -28.059219360351562,
      "activations/layer3_attention_weight_max": 71.03234100341797,
      "activations/layer3_attention_weight_min": -74.87114715576172,
      "activations/layer4_attention_weight_max": 135.552978515625,
      "activations/layer4_attention_weight_min": -126.47834777832031,
      "activations/layer5_attention_weight_max": 126.33912658691406,
      "activations/layer5_attention_weight_min": -105.6931381225586,
      "activations/layer6_attention_weight_max": 66.90460205078125,
      "activations/layer6_attention_weight_min": -54.76600646972656,
      "activations/layer7_attention_weight_max": 68.9386215209961,
      "activations/layer7_attention_weight_min": -66.19893646240234,
      "activations/layer8_attention_weight_max": 56.220584869384766,
      "activations/layer8_attention_weight_min": -45.30095291137695,
      "activations/layer9_attention_weight_max": 80.06431579589844,
      "activations/layer9_attention_weight_min": -66.50591278076172,
      "epoch": 1.87,
      "learning_rate": 0.00013937272727272725,
      "loss": 2.9671,
      "step": 32100
    },
    {
      "activations/layer0_attention_weight_max": 16.448152542114258,
      "activations/layer0_attention_weight_min": -12.369139671325684,
      "activations/layer10_attention_weight_max": 58.685203552246094,
      "activations/layer10_attention_weight_min": -46.38471603393555,
      "activations/layer11_attention_weight_max": 52.68565368652344,
      "activations/layer11_attention_weight_min": -38.218772888183594,
      "activations/layer12_attention_weight_max": 42.65432357788086,
      "activations/layer12_attention_weight_min": -30.321027755737305,
      "activations/layer13_attention_weight_max": 52.06139373779297,
      "activations/layer13_attention_weight_min": -35.67179489135742,
      "activations/layer14_attention_weight_max": 91.48734283447266,
      "activations/layer14_attention_weight_min": -62.09337615966797,
      "activations/layer15_attention_weight_max": 60.76408767700195,
      "activations/layer15_attention_weight_min": -36.050537109375,
      "activations/layer16_attention_weight_max": 66.04607391357422,
      "activations/layer16_attention_weight_min": -38.07566452026367,
      "activations/layer17_attention_weight_max": 51.120357513427734,
      "activations/layer17_attention_weight_min": -28.647363662719727,
      "activations/layer18_attention_weight_max": 30.206668853759766,
      "activations/layer18_attention_weight_min": -16.926109313964844,
      "activations/layer19_attention_weight_max": 57.128536224365234,
      "activations/layer19_attention_weight_min": -31.468738555908203,
      "activations/layer1_attention_weight_max": 19.248611450195312,
      "activations/layer1_attention_weight_min": -11.650439262390137,
      "activations/layer20_attention_weight_max": 45.956485748291016,
      "activations/layer20_attention_weight_min": -22.867475509643555,
      "activations/layer21_attention_weight_max": 29.645658493041992,
      "activations/layer21_attention_weight_min": -19.301673889160156,
      "activations/layer22_attention_weight_max": 41.80791091918945,
      "activations/layer22_attention_weight_min": -24.35527229309082,
      "activations/layer23_attention_weight_max": 21.891159057617188,
      "activations/layer23_attention_weight_min": -15.073126792907715,
      "activations/layer2_attention_weight_max": 31.409427642822266,
      "activations/layer2_attention_weight_min": -28.966140747070312,
      "activations/layer3_attention_weight_max": 77.05654907226562,
      "activations/layer3_attention_weight_min": -77.3554916381836,
      "activations/layer4_attention_weight_max": 137.1191864013672,
      "activations/layer4_attention_weight_min": -121.38228607177734,
      "activations/layer5_attention_weight_max": 127.3862533569336,
      "activations/layer5_attention_weight_min": -108.29232025146484,
      "activations/layer6_attention_weight_max": 71.4975814819336,
      "activations/layer6_attention_weight_min": -56.19841766357422,
      "activations/layer7_attention_weight_max": 75.60348510742188,
      "activations/layer7_attention_weight_min": -65.75769805908203,
      "activations/layer8_attention_weight_max": 64.8990478515625,
      "activations/layer8_attention_weight_min": -48.828338623046875,
      "activations/layer9_attention_weight_max": 92.3567886352539,
      "activations/layer9_attention_weight_min": -68.37490844726562,
      "epoch": 1.87,
      "learning_rate": 0.00013935378787878787,
      "loss": 2.9879,
      "step": 32150
    },
    {
      "activations/layer0_attention_weight_max": 16.3674259185791,
      "activations/layer0_attention_weight_min": -12.085746765136719,
      "activations/layer10_attention_weight_max": 57.46091079711914,
      "activations/layer10_attention_weight_min": -42.32975769042969,
      "activations/layer11_attention_weight_max": 44.405635833740234,
      "activations/layer11_attention_weight_min": -35.19999694824219,
      "activations/layer12_attention_weight_max": 94.05050659179688,
      "activations/layer12_attention_weight_min": -47.70622253417969,
      "activations/layer13_attention_weight_max": 76.68083190917969,
      "activations/layer13_attention_weight_min": -47.24057388305664,
      "activations/layer14_attention_weight_max": 93.30239868164062,
      "activations/layer14_attention_weight_min": -57.84160232543945,
      "activations/layer15_attention_weight_max": 74.43280792236328,
      "activations/layer15_attention_weight_min": -43.10148620605469,
      "activations/layer16_attention_weight_max": 62.36927032470703,
      "activations/layer16_attention_weight_min": -38.366695404052734,
      "activations/layer17_attention_weight_max": 49.46711730957031,
      "activations/layer17_attention_weight_min": -30.861263275146484,
      "activations/layer18_attention_weight_max": 28.08904457092285,
      "activations/layer18_attention_weight_min": -17.214595794677734,
      "activations/layer19_attention_weight_max": 57.32667922973633,
      "activations/layer19_attention_weight_min": -33.18212127685547,
      "activations/layer1_attention_weight_max": 17.578468322753906,
      "activations/layer1_attention_weight_min": -11.24694538116455,
      "activations/layer20_attention_weight_max": 42.20719528198242,
      "activations/layer20_attention_weight_min": -23.79869842529297,
      "activations/layer21_attention_weight_max": 39.37515640258789,
      "activations/layer21_attention_weight_min": -21.255874633789062,
      "activations/layer22_attention_weight_max": 47.960655212402344,
      "activations/layer22_attention_weight_min": -26.436819076538086,
      "activations/layer23_attention_weight_max": 28.570295333862305,
      "activations/layer23_attention_weight_min": -17.14537811279297,
      "activations/layer2_attention_weight_max": 27.88648796081543,
      "activations/layer2_attention_weight_min": -28.231571197509766,
      "activations/layer3_attention_weight_max": 68.87669372558594,
      "activations/layer3_attention_weight_min": -70.60659790039062,
      "activations/layer4_attention_weight_max": 122.6100845336914,
      "activations/layer4_attention_weight_min": -113.84163665771484,
      "activations/layer5_attention_weight_max": 117.99727630615234,
      "activations/layer5_attention_weight_min": -98.45414733886719,
      "activations/layer6_attention_weight_max": 63.676483154296875,
      "activations/layer6_attention_weight_min": -54.662208557128906,
      "activations/layer7_attention_weight_max": 65.41337585449219,
      "activations/layer7_attention_weight_min": -61.52665710449219,
      "activations/layer8_attention_weight_max": 48.725311279296875,
      "activations/layer8_attention_weight_min": -39.3680419921875,
      "activations/layer9_attention_weight_max": 65.26903533935547,
      "activations/layer9_attention_weight_min": -62.94649124145508,
      "epoch": 1.87,
      "learning_rate": 0.00013933484848484848,
      "loss": 2.9793,
      "step": 32200
    },
    {
      "activations/layer0_attention_weight_max": 16.364259719848633,
      "activations/layer0_attention_weight_min": -11.491227149963379,
      "activations/layer10_attention_weight_max": 51.870452880859375,
      "activations/layer10_attention_weight_min": -46.26586151123047,
      "activations/layer11_attention_weight_max": 39.71735763549805,
      "activations/layer11_attention_weight_min": -37.07425308227539,
      "activations/layer12_attention_weight_max": 35.78629684448242,
      "activations/layer12_attention_weight_min": -30.33360481262207,
      "activations/layer13_attention_weight_max": 53.02303695678711,
      "activations/layer13_attention_weight_min": -38.158782958984375,
      "activations/layer14_attention_weight_max": 78.19915771484375,
      "activations/layer14_attention_weight_min": -57.68248748779297,
      "activations/layer15_attention_weight_max": 60.42977523803711,
      "activations/layer15_attention_weight_min": -40.97560501098633,
      "activations/layer16_attention_weight_max": 58.17673873901367,
      "activations/layer16_attention_weight_min": -38.638240814208984,
      "activations/layer17_attention_weight_max": 47.54173278808594,
      "activations/layer17_attention_weight_min": -29.40981674194336,
      "activations/layer18_attention_weight_max": 29.41975212097168,
      "activations/layer18_attention_weight_min": -18.7833309173584,
      "activations/layer19_attention_weight_max": 58.211483001708984,
      "activations/layer19_attention_weight_min": -32.999610900878906,
      "activations/layer1_attention_weight_max": 16.80133056640625,
      "activations/layer1_attention_weight_min": -11.909988403320312,
      "activations/layer20_attention_weight_max": 46.303009033203125,
      "activations/layer20_attention_weight_min": -24.084646224975586,
      "activations/layer21_attention_weight_max": 30.21934700012207,
      "activations/layer21_attention_weight_min": -17.651765823364258,
      "activations/layer22_attention_weight_max": 51.38890075683594,
      "activations/layer22_attention_weight_min": -27.45052146911621,
      "activations/layer23_attention_weight_max": 31.591976165771484,
      "activations/layer23_attention_weight_min": -17.888227462768555,
      "activations/layer2_attention_weight_max": 27.122257232666016,
      "activations/layer2_attention_weight_min": -26.925086975097656,
      "activations/layer3_attention_weight_max": 69.69400024414062,
      "activations/layer3_attention_weight_min": -75.6019515991211,
      "activations/layer4_attention_weight_max": 127.3715591430664,
      "activations/layer4_attention_weight_min": -120.42665100097656,
      "activations/layer5_attention_weight_max": 119.06746673583984,
      "activations/layer5_attention_weight_min": -103.76909637451172,
      "activations/layer6_attention_weight_max": 63.93846130371094,
      "activations/layer6_attention_weight_min": -52.29918670654297,
      "activations/layer7_attention_weight_max": 65.28279876708984,
      "activations/layer7_attention_weight_min": -61.38747787475586,
      "activations/layer8_attention_weight_max": 53.427127838134766,
      "activations/layer8_attention_weight_min": -42.35453796386719,
      "activations/layer9_attention_weight_max": 73.6256332397461,
      "activations/layer9_attention_weight_min": -56.60028076171875,
      "epoch": 1.87,
      "learning_rate": 0.0001393159090909091,
      "loss": 2.9875,
      "step": 32250
    },
    {
      "activations/layer0_attention_weight_max": 16.053117752075195,
      "activations/layer0_attention_weight_min": -12.326557159423828,
      "activations/layer10_attention_weight_max": 50.0602912902832,
      "activations/layer10_attention_weight_min": -44.14344024658203,
      "activations/layer11_attention_weight_max": 40.134918212890625,
      "activations/layer11_attention_weight_min": -36.710357666015625,
      "activations/layer12_attention_weight_max": 33.036006927490234,
      "activations/layer12_attention_weight_min": -27.733604431152344,
      "activations/layer13_attention_weight_max": 51.32966613769531,
      "activations/layer13_attention_weight_min": -36.59149169921875,
      "activations/layer14_attention_weight_max": 88.55107879638672,
      "activations/layer14_attention_weight_min": -62.435359954833984,
      "activations/layer15_attention_weight_max": 59.14427185058594,
      "activations/layer15_attention_weight_min": -38.36867904663086,
      "activations/layer16_attention_weight_max": 60.66618728637695,
      "activations/layer16_attention_weight_min": -39.50596618652344,
      "activations/layer17_attention_weight_max": 47.26506042480469,
      "activations/layer17_attention_weight_min": -28.58073616027832,
      "activations/layer18_attention_weight_max": 33.5595817565918,
      "activations/layer18_attention_weight_min": -18.736425399780273,
      "activations/layer19_attention_weight_max": 63.4549674987793,
      "activations/layer19_attention_weight_min": -34.85072708129883,
      "activations/layer1_attention_weight_max": 17.3809814453125,
      "activations/layer1_attention_weight_min": -11.218274116516113,
      "activations/layer20_attention_weight_max": 43.64360427856445,
      "activations/layer20_attention_weight_min": -26.17583465576172,
      "activations/layer21_attention_weight_max": 39.1827392578125,
      "activations/layer21_attention_weight_min": -18.21816062927246,
      "activations/layer22_attention_weight_max": 45.85824966430664,
      "activations/layer22_attention_weight_min": -26.85712432861328,
      "activations/layer23_attention_weight_max": 27.256114959716797,
      "activations/layer23_attention_weight_min": -15.209888458251953,
      "activations/layer2_attention_weight_max": 28.918241500854492,
      "activations/layer2_attention_weight_min": -28.216381072998047,
      "activations/layer3_attention_weight_max": 68.26427459716797,
      "activations/layer3_attention_weight_min": -71.57478332519531,
      "activations/layer4_attention_weight_max": 126.07940673828125,
      "activations/layer4_attention_weight_min": -110.0065689086914,
      "activations/layer5_attention_weight_max": 116.82447814941406,
      "activations/layer5_attention_weight_min": -98.09103393554688,
      "activations/layer6_attention_weight_max": 62.605308532714844,
      "activations/layer6_attention_weight_min": -51.671592712402344,
      "activations/layer7_attention_weight_max": 67.3951187133789,
      "activations/layer7_attention_weight_min": -63.215850830078125,
      "activations/layer8_attention_weight_max": 51.24565505981445,
      "activations/layer8_attention_weight_min": -44.203941345214844,
      "activations/layer9_attention_weight_max": 81.15801239013672,
      "activations/layer9_attention_weight_min": -63.43767166137695,
      "epoch": 1.88,
      "learning_rate": 0.00013929696969696968,
      "loss": 2.9861,
      "step": 32300
    },
    {
      "activations/layer0_attention_weight_max": 15.690875053405762,
      "activations/layer0_attention_weight_min": -12.163372039794922,
      "activations/layer10_attention_weight_max": 50.77302169799805,
      "activations/layer10_attention_weight_min": -45.62500762939453,
      "activations/layer11_attention_weight_max": 37.862831115722656,
      "activations/layer11_attention_weight_min": -35.499473571777344,
      "activations/layer12_attention_weight_max": 39.07814407348633,
      "activations/layer12_attention_weight_min": -29.898815155029297,
      "activations/layer13_attention_weight_max": 51.54764175415039,
      "activations/layer13_attention_weight_min": -38.5699348449707,
      "activations/layer14_attention_weight_max": 81.25243377685547,
      "activations/layer14_attention_weight_min": -63.37111282348633,
      "activations/layer15_attention_weight_max": 54.89057540893555,
      "activations/layer15_attention_weight_min": -40.18305587768555,
      "activations/layer16_attention_weight_max": 60.43592071533203,
      "activations/layer16_attention_weight_min": -37.302513122558594,
      "activations/layer17_attention_weight_max": 44.97273635864258,
      "activations/layer17_attention_weight_min": -28.513736724853516,
      "activations/layer18_attention_weight_max": 27.136146545410156,
      "activations/layer18_attention_weight_min": -16.6701717376709,
      "activations/layer19_attention_weight_max": 62.5870475769043,
      "activations/layer19_attention_weight_min": -34.69868087768555,
      "activations/layer1_attention_weight_max": 17.694473266601562,
      "activations/layer1_attention_weight_min": -10.997283935546875,
      "activations/layer20_attention_weight_max": 42.1438102722168,
      "activations/layer20_attention_weight_min": -25.327861785888672,
      "activations/layer21_attention_weight_max": 29.346654891967773,
      "activations/layer21_attention_weight_min": -21.795074462890625,
      "activations/layer22_attention_weight_max": 44.90620422363281,
      "activations/layer22_attention_weight_min": -25.767961502075195,
      "activations/layer23_attention_weight_max": 26.98011016845703,
      "activations/layer23_attention_weight_min": -18.435707092285156,
      "activations/layer2_attention_weight_max": 28.166316986083984,
      "activations/layer2_attention_weight_min": -27.622854232788086,
      "activations/layer3_attention_weight_max": 68.20045471191406,
      "activations/layer3_attention_weight_min": -72.31275939941406,
      "activations/layer4_attention_weight_max": 122.96929168701172,
      "activations/layer4_attention_weight_min": -114.91178131103516,
      "activations/layer5_attention_weight_max": 107.82795715332031,
      "activations/layer5_attention_weight_min": -99.2548828125,
      "activations/layer6_attention_weight_max": 60.84117126464844,
      "activations/layer6_attention_weight_min": -53.82056427001953,
      "activations/layer7_attention_weight_max": 64.19587707519531,
      "activations/layer7_attention_weight_min": -66.30021667480469,
      "activations/layer8_attention_weight_max": 50.60404586791992,
      "activations/layer8_attention_weight_min": -43.485103607177734,
      "activations/layer9_attention_weight_max": 75.43009185791016,
      "activations/layer9_attention_weight_min": -56.722625732421875,
      "epoch": 1.88,
      "learning_rate": 0.0001392780303030303,
      "loss": 2.9741,
      "step": 32350
    },
    {
      "activations/layer0_attention_weight_max": 15.539569854736328,
      "activations/layer0_attention_weight_min": -12.15126895904541,
      "activations/layer10_attention_weight_max": 49.610565185546875,
      "activations/layer10_attention_weight_min": -43.912559509277344,
      "activations/layer11_attention_weight_max": 38.904571533203125,
      "activations/layer11_attention_weight_min": -33.66386413574219,
      "activations/layer12_attention_weight_max": 35.30437088012695,
      "activations/layer12_attention_weight_min": -27.473020553588867,
      "activations/layer13_attention_weight_max": 52.26622772216797,
      "activations/layer13_attention_weight_min": -40.80930709838867,
      "activations/layer14_attention_weight_max": 81.49434661865234,
      "activations/layer14_attention_weight_min": -65.14800262451172,
      "activations/layer15_attention_weight_max": 57.13178253173828,
      "activations/layer15_attention_weight_min": -39.777320861816406,
      "activations/layer16_attention_weight_max": 58.39722442626953,
      "activations/layer16_attention_weight_min": -36.81949234008789,
      "activations/layer17_attention_weight_max": 49.48292922973633,
      "activations/layer17_attention_weight_min": -31.24927520751953,
      "activations/layer18_attention_weight_max": 26.455413818359375,
      "activations/layer18_attention_weight_min": -16.321548461914062,
      "activations/layer19_attention_weight_max": 51.958736419677734,
      "activations/layer19_attention_weight_min": -31.008197784423828,
      "activations/layer1_attention_weight_max": 18.645248413085938,
      "activations/layer1_attention_weight_min": -11.067277908325195,
      "activations/layer20_attention_weight_max": 40.40829849243164,
      "activations/layer20_attention_weight_min": -26.115636825561523,
      "activations/layer21_attention_weight_max": 26.49806785583496,
      "activations/layer21_attention_weight_min": -20.530601501464844,
      "activations/layer22_attention_weight_max": 45.836788177490234,
      "activations/layer22_attention_weight_min": -32.18681716918945,
      "activations/layer23_attention_weight_max": 26.02251434326172,
      "activations/layer23_attention_weight_min": -16.021289825439453,
      "activations/layer2_attention_weight_max": 27.57879638671875,
      "activations/layer2_attention_weight_min": -27.72241973876953,
      "activations/layer3_attention_weight_max": 69.72746276855469,
      "activations/layer3_attention_weight_min": -69.26311492919922,
      "activations/layer4_attention_weight_max": 125.2747802734375,
      "activations/layer4_attention_weight_min": -115.2598648071289,
      "activations/layer5_attention_weight_max": 112.92245483398438,
      "activations/layer5_attention_weight_min": -96.20220184326172,
      "activations/layer6_attention_weight_max": 65.65222930908203,
      "activations/layer6_attention_weight_min": -50.70823287963867,
      "activations/layer7_attention_weight_max": 65.61526489257812,
      "activations/layer7_attention_weight_min": -61.83514404296875,
      "activations/layer8_attention_weight_max": 44.34297180175781,
      "activations/layer8_attention_weight_min": -40.046714782714844,
      "activations/layer9_attention_weight_max": 62.49397659301758,
      "activations/layer9_attention_weight_min": -52.991607666015625,
      "epoch": 1.88,
      "learning_rate": 0.0001392590909090909,
      "loss": 2.98,
      "step": 32400
    },
    {
      "activations/layer0_attention_weight_max": 17.210378646850586,
      "activations/layer0_attention_weight_min": -11.645200729370117,
      "activations/layer10_attention_weight_max": 48.56198501586914,
      "activations/layer10_attention_weight_min": -47.05851364135742,
      "activations/layer11_attention_weight_max": 40.608192443847656,
      "activations/layer11_attention_weight_min": -38.35796356201172,
      "activations/layer12_attention_weight_max": 35.22113037109375,
      "activations/layer12_attention_weight_min": -27.104127883911133,
      "activations/layer13_attention_weight_max": 55.05814743041992,
      "activations/layer13_attention_weight_min": -37.775543212890625,
      "activations/layer14_attention_weight_max": 89.29634094238281,
      "activations/layer14_attention_weight_min": -66.68932342529297,
      "activations/layer15_attention_weight_max": 63.099552154541016,
      "activations/layer15_attention_weight_min": -40.28809356689453,
      "activations/layer16_attention_weight_max": 51.840091705322266,
      "activations/layer16_attention_weight_min": -36.95063781738281,
      "activations/layer17_attention_weight_max": 48.86333084106445,
      "activations/layer17_attention_weight_min": -30.162879943847656,
      "activations/layer18_attention_weight_max": 30.499187469482422,
      "activations/layer18_attention_weight_min": -18.41913604736328,
      "activations/layer19_attention_weight_max": 57.494972229003906,
      "activations/layer19_attention_weight_min": -32.68540954589844,
      "activations/layer1_attention_weight_max": 17.374128341674805,
      "activations/layer1_attention_weight_min": -11.10036563873291,
      "activations/layer20_attention_weight_max": 42.92195129394531,
      "activations/layer20_attention_weight_min": -25.545181274414062,
      "activations/layer21_attention_weight_max": 39.55635452270508,
      "activations/layer21_attention_weight_min": -20.0288028717041,
      "activations/layer22_attention_weight_max": 49.267940521240234,
      "activations/layer22_attention_weight_min": -25.207313537597656,
      "activations/layer23_attention_weight_max": 25.19305419921875,
      "activations/layer23_attention_weight_min": -14.693385124206543,
      "activations/layer2_attention_weight_max": 28.179241180419922,
      "activations/layer2_attention_weight_min": -27.047346115112305,
      "activations/layer3_attention_weight_max": 68.64752197265625,
      "activations/layer3_attention_weight_min": -72.1136245727539,
      "activations/layer4_attention_weight_max": 122.89137268066406,
      "activations/layer4_attention_weight_min": -112.1038818359375,
      "activations/layer5_attention_weight_max": 111.1225814819336,
      "activations/layer5_attention_weight_min": -96.02301025390625,
      "activations/layer6_attention_weight_max": 58.80537033081055,
      "activations/layer6_attention_weight_min": -52.959007263183594,
      "activations/layer7_attention_weight_max": 65.1897964477539,
      "activations/layer7_attention_weight_min": -64.07553100585938,
      "activations/layer8_attention_weight_max": 47.22407913208008,
      "activations/layer8_attention_weight_min": -45.070350646972656,
      "activations/layer9_attention_weight_max": 63.69385528564453,
      "activations/layer9_attention_weight_min": -55.48700714111328,
      "epoch": 1.89,
      "learning_rate": 0.0001392401515151515,
      "loss": 2.9842,
      "step": 32450
    },
    {
      "activations/layer0_attention_weight_max": 16.47307777404785,
      "activations/layer0_attention_weight_min": -12.224868774414062,
      "activations/layer10_attention_weight_max": 53.04880905151367,
      "activations/layer10_attention_weight_min": -43.97712707519531,
      "activations/layer11_attention_weight_max": 39.486358642578125,
      "activations/layer11_attention_weight_min": -34.780723571777344,
      "activations/layer12_attention_weight_max": 33.04117965698242,
      "activations/layer12_attention_weight_min": -27.464574813842773,
      "activations/layer13_attention_weight_max": 48.70409393310547,
      "activations/layer13_attention_weight_min": -38.98572540283203,
      "activations/layer14_attention_weight_max": 78.54643249511719,
      "activations/layer14_attention_weight_min": -57.77182388305664,
      "activations/layer15_attention_weight_max": 51.80134963989258,
      "activations/layer15_attention_weight_min": -42.26571273803711,
      "activations/layer16_attention_weight_max": 54.914371490478516,
      "activations/layer16_attention_weight_min": -37.01441192626953,
      "activations/layer17_attention_weight_max": 43.9886589050293,
      "activations/layer17_attention_weight_min": -29.153778076171875,
      "activations/layer18_attention_weight_max": 26.207544326782227,
      "activations/layer18_attention_weight_min": -17.31240463256836,
      "activations/layer19_attention_weight_max": 61.8721809387207,
      "activations/layer19_attention_weight_min": -32.38135528564453,
      "activations/layer1_attention_weight_max": 17.761980056762695,
      "activations/layer1_attention_weight_min": -11.556745529174805,
      "activations/layer20_attention_weight_max": 40.758785247802734,
      "activations/layer20_attention_weight_min": -27.254186630249023,
      "activations/layer21_attention_weight_max": 29.13547706604004,
      "activations/layer21_attention_weight_min": -19.66815757751465,
      "activations/layer22_attention_weight_max": 53.02619934082031,
      "activations/layer22_attention_weight_min": -25.885576248168945,
      "activations/layer23_attention_weight_max": 27.98975944519043,
      "activations/layer23_attention_weight_min": -15.552062034606934,
      "activations/layer2_attention_weight_max": 28.57907485961914,
      "activations/layer2_attention_weight_min": -29.533954620361328,
      "activations/layer3_attention_weight_max": 69.12299346923828,
      "activations/layer3_attention_weight_min": -72.6446762084961,
      "activations/layer4_attention_weight_max": 128.26101684570312,
      "activations/layer4_attention_weight_min": -116.4438247680664,
      "activations/layer5_attention_weight_max": 119.14556884765625,
      "activations/layer5_attention_weight_min": -102.4287338256836,
      "activations/layer6_attention_weight_max": 64.32992553710938,
      "activations/layer6_attention_weight_min": -56.14426803588867,
      "activations/layer7_attention_weight_max": 67.26444244384766,
      "activations/layer7_attention_weight_min": -61.96739196777344,
      "activations/layer8_attention_weight_max": 46.852989196777344,
      "activations/layer8_attention_weight_min": -39.56623458862305,
      "activations/layer9_attention_weight_max": 73.59905242919922,
      "activations/layer9_attention_weight_min": -56.26519775390625,
      "epoch": 1.89,
      "learning_rate": 0.00013922121212121212,
      "loss": 2.9741,
      "step": 32500
    },
    {
      "activations/layer0_attention_weight_max": 16.46512222290039,
      "activations/layer0_attention_weight_min": -11.705788612365723,
      "activations/layer10_attention_weight_max": 50.36945343017578,
      "activations/layer10_attention_weight_min": -43.41108322143555,
      "activations/layer11_attention_weight_max": 38.85711669921875,
      "activations/layer11_attention_weight_min": -31.5087890625,
      "activations/layer12_attention_weight_max": 43.38124084472656,
      "activations/layer12_attention_weight_min": -31.34958267211914,
      "activations/layer13_attention_weight_max": 47.85078048706055,
      "activations/layer13_attention_weight_min": -34.433319091796875,
      "activations/layer14_attention_weight_max": 76.2602310180664,
      "activations/layer14_attention_weight_min": -61.70740509033203,
      "activations/layer15_attention_weight_max": 55.78540802001953,
      "activations/layer15_attention_weight_min": -40.356021881103516,
      "activations/layer16_attention_weight_max": 61.73605728149414,
      "activations/layer16_attention_weight_min": -38.937557220458984,
      "activations/layer17_attention_weight_max": 55.08873748779297,
      "activations/layer17_attention_weight_min": -30.383182525634766,
      "activations/layer18_attention_weight_max": 27.66217041015625,
      "activations/layer18_attention_weight_min": -17.131200790405273,
      "activations/layer19_attention_weight_max": 59.7942008972168,
      "activations/layer19_attention_weight_min": -32.18537521362305,
      "activations/layer1_attention_weight_max": 18.224536895751953,
      "activations/layer1_attention_weight_min": -11.65246868133545,
      "activations/layer20_attention_weight_max": 46.58354568481445,
      "activations/layer20_attention_weight_min": -25.069429397583008,
      "activations/layer21_attention_weight_max": 33.39137649536133,
      "activations/layer21_attention_weight_min": -19.598861694335938,
      "activations/layer22_attention_weight_max": 46.26984405517578,
      "activations/layer22_attention_weight_min": -28.747211456298828,
      "activations/layer23_attention_weight_max": 23.359668731689453,
      "activations/layer23_attention_weight_min": -16.782371520996094,
      "activations/layer2_attention_weight_max": 28.838359832763672,
      "activations/layer2_attention_weight_min": -28.689926147460938,
      "activations/layer3_attention_weight_max": 72.51213836669922,
      "activations/layer3_attention_weight_min": -74.25856018066406,
      "activations/layer4_attention_weight_max": 129.6343231201172,
      "activations/layer4_attention_weight_min": -117.0486831665039,
      "activations/layer5_attention_weight_max": 114.53966522216797,
      "activations/layer5_attention_weight_min": -101.36178588867188,
      "activations/layer6_attention_weight_max": 62.55421447753906,
      "activations/layer6_attention_weight_min": -54.58212661743164,
      "activations/layer7_attention_weight_max": 69.37664031982422,
      "activations/layer7_attention_weight_min": -64.00501251220703,
      "activations/layer8_attention_weight_max": 52.24858093261719,
      "activations/layer8_attention_weight_min": -42.51455307006836,
      "activations/layer9_attention_weight_max": 64.45757293701172,
      "activations/layer9_attention_weight_min": -56.5057487487793,
      "epoch": 1.89,
      "learning_rate": 0.00013920227272727273,
      "loss": 2.9861,
      "step": 32550
    },
    {
      "activations/layer0_attention_weight_max": 15.052031517028809,
      "activations/layer0_attention_weight_min": -12.246564865112305,
      "activations/layer10_attention_weight_max": 51.42988204956055,
      "activations/layer10_attention_weight_min": -45.60789489746094,
      "activations/layer11_attention_weight_max": 41.31604766845703,
      "activations/layer11_attention_weight_min": -33.95793151855469,
      "activations/layer12_attention_weight_max": 55.22551345825195,
      "activations/layer12_attention_weight_min": -31.464170455932617,
      "activations/layer13_attention_weight_max": 50.28590774536133,
      "activations/layer13_attention_weight_min": -35.37042236328125,
      "activations/layer14_attention_weight_max": 78.20711517333984,
      "activations/layer14_attention_weight_min": -52.70232391357422,
      "activations/layer15_attention_weight_max": 55.44736099243164,
      "activations/layer15_attention_weight_min": -36.72492218017578,
      "activations/layer16_attention_weight_max": 53.638648986816406,
      "activations/layer16_attention_weight_min": -39.24174118041992,
      "activations/layer17_attention_weight_max": 48.4183464050293,
      "activations/layer17_attention_weight_min": -32.29214859008789,
      "activations/layer18_attention_weight_max": 31.711078643798828,
      "activations/layer18_attention_weight_min": -18.36649513244629,
      "activations/layer19_attention_weight_max": 57.48512649536133,
      "activations/layer19_attention_weight_min": -28.948362350463867,
      "activations/layer1_attention_weight_max": 18.11752700805664,
      "activations/layer1_attention_weight_min": -12.247171401977539,
      "activations/layer20_attention_weight_max": 40.19437026977539,
      "activations/layer20_attention_weight_min": -22.815258026123047,
      "activations/layer21_attention_weight_max": 34.264129638671875,
      "activations/layer21_attention_weight_min": -17.76778793334961,
      "activations/layer22_attention_weight_max": 50.13836669921875,
      "activations/layer22_attention_weight_min": -27.804576873779297,
      "activations/layer23_attention_weight_max": 32.063270568847656,
      "activations/layer23_attention_weight_min": -16.985973358154297,
      "activations/layer2_attention_weight_max": 27.722152709960938,
      "activations/layer2_attention_weight_min": -27.421871185302734,
      "activations/layer3_attention_weight_max": 69.03128051757812,
      "activations/layer3_attention_weight_min": -70.05757904052734,
      "activations/layer4_attention_weight_max": 129.31202697753906,
      "activations/layer4_attention_weight_min": -114.25020599365234,
      "activations/layer5_attention_weight_max": 116.12300872802734,
      "activations/layer5_attention_weight_min": -100.58139038085938,
      "activations/layer6_attention_weight_max": 66.14176940917969,
      "activations/layer6_attention_weight_min": -52.63489532470703,
      "activations/layer7_attention_weight_max": 69.2674560546875,
      "activations/layer7_attention_weight_min": -62.86713409423828,
      "activations/layer8_attention_weight_max": 46.5909538269043,
      "activations/layer8_attention_weight_min": -39.118953704833984,
      "activations/layer9_attention_weight_max": 67.7541732788086,
      "activations/layer9_attention_weight_min": -54.728633880615234,
      "epoch": 1.89,
      "learning_rate": 0.00013918371212121212,
      "loss": 2.9831,
      "step": 32600
    },
    {
      "activations/layer0_attention_weight_max": 14.904541015625,
      "activations/layer0_attention_weight_min": -12.258004188537598,
      "activations/layer10_attention_weight_max": 67.40081024169922,
      "activations/layer10_attention_weight_min": -46.806758880615234,
      "activations/layer11_attention_weight_max": 53.654354095458984,
      "activations/layer11_attention_weight_min": -38.30902099609375,
      "activations/layer12_attention_weight_max": 44.62739181518555,
      "activations/layer12_attention_weight_min": -32.721248626708984,
      "activations/layer13_attention_weight_max": 53.30397033691406,
      "activations/layer13_attention_weight_min": -38.49058532714844,
      "activations/layer14_attention_weight_max": 92.49932861328125,
      "activations/layer14_attention_weight_min": -65.0938491821289,
      "activations/layer15_attention_weight_max": 63.570091247558594,
      "activations/layer15_attention_weight_min": -43.76066207885742,
      "activations/layer16_attention_weight_max": 57.38793182373047,
      "activations/layer16_attention_weight_min": -39.81468963623047,
      "activations/layer17_attention_weight_max": 51.52499771118164,
      "activations/layer17_attention_weight_min": -29.946332931518555,
      "activations/layer18_attention_weight_max": 34.50653076171875,
      "activations/layer18_attention_weight_min": -19.466806411743164,
      "activations/layer19_attention_weight_max": 55.2409553527832,
      "activations/layer19_attention_weight_min": -33.56171798706055,
      "activations/layer1_attention_weight_max": 17.502199172973633,
      "activations/layer1_attention_weight_min": -12.502409934997559,
      "activations/layer20_attention_weight_max": 41.67584991455078,
      "activations/layer20_attention_weight_min": -27.47075080871582,
      "activations/layer21_attention_weight_max": 28.56388282775879,
      "activations/layer21_attention_weight_min": -16.385826110839844,
      "activations/layer22_attention_weight_max": 55.32862854003906,
      "activations/layer22_attention_weight_min": -25.57658576965332,
      "activations/layer23_attention_weight_max": 29.387784957885742,
      "activations/layer23_attention_weight_min": -16.231822967529297,
      "activations/layer2_attention_weight_max": 30.470630645751953,
      "activations/layer2_attention_weight_min": -29.001005172729492,
      "activations/layer3_attention_weight_max": 76.45358276367188,
      "activations/layer3_attention_weight_min": -76.8492202758789,
      "activations/layer4_attention_weight_max": 139.86618041992188,
      "activations/layer4_attention_weight_min": -121.20328521728516,
      "activations/layer5_attention_weight_max": 129.23529052734375,
      "activations/layer5_attention_weight_min": -100.11286926269531,
      "activations/layer6_attention_weight_max": 69.33984375,
      "activations/layer6_attention_weight_min": -55.59757995605469,
      "activations/layer7_attention_weight_max": 77.18177795410156,
      "activations/layer7_attention_weight_min": -67.70816040039062,
      "activations/layer8_attention_weight_max": 61.32657241821289,
      "activations/layer8_attention_weight_min": -46.00852584838867,
      "activations/layer9_attention_weight_max": 83.23433685302734,
      "activations/layer9_attention_weight_min": -60.94852828979492,
      "epoch": 1.9,
      "learning_rate": 0.0001391647727272727,
      "loss": 2.9666,
      "step": 32650
    },
    {
      "activations/layer0_attention_weight_max": 15.886306762695312,
      "activations/layer0_attention_weight_min": -11.97587776184082,
      "activations/layer10_attention_weight_max": 60.59650802612305,
      "activations/layer10_attention_weight_min": -49.3812255859375,
      "activations/layer11_attention_weight_max": 50.448883056640625,
      "activations/layer11_attention_weight_min": -36.181922912597656,
      "activations/layer12_attention_weight_max": 40.840606689453125,
      "activations/layer12_attention_weight_min": -29.14369010925293,
      "activations/layer13_attention_weight_max": 65.78416442871094,
      "activations/layer13_attention_weight_min": -46.58836364746094,
      "activations/layer14_attention_weight_max": 84.16194915771484,
      "activations/layer14_attention_weight_min": -71.38152313232422,
      "activations/layer15_attention_weight_max": 58.784423828125,
      "activations/layer15_attention_weight_min": -38.98084259033203,
      "activations/layer16_attention_weight_max": 60.730377197265625,
      "activations/layer16_attention_weight_min": -38.37305450439453,
      "activations/layer17_attention_weight_max": 53.31242370605469,
      "activations/layer17_attention_weight_min": -33.31085205078125,
      "activations/layer18_attention_weight_max": 29.968368530273438,
      "activations/layer18_attention_weight_min": -18.319820404052734,
      "activations/layer19_attention_weight_max": 61.5021858215332,
      "activations/layer19_attention_weight_min": -34.009944915771484,
      "activations/layer1_attention_weight_max": 17.041439056396484,
      "activations/layer1_attention_weight_min": -11.50000286102295,
      "activations/layer20_attention_weight_max": 44.59549331665039,
      "activations/layer20_attention_weight_min": -23.906131744384766,
      "activations/layer21_attention_weight_max": 34.81777572631836,
      "activations/layer21_attention_weight_min": -18.814579010009766,
      "activations/layer22_attention_weight_max": 56.55664825439453,
      "activations/layer22_attention_weight_min": -23.59787940979004,
      "activations/layer23_attention_weight_max": 31.701095581054688,
      "activations/layer23_attention_weight_min": -18.06821060180664,
      "activations/layer2_attention_weight_max": 28.235692977905273,
      "activations/layer2_attention_weight_min": -28.0015926361084,
      "activations/layer3_attention_weight_max": 69.84374237060547,
      "activations/layer3_attention_weight_min": -71.27021789550781,
      "activations/layer4_attention_weight_max": 129.54522705078125,
      "activations/layer4_attention_weight_min": -121.71862030029297,
      "activations/layer5_attention_weight_max": 120.49510192871094,
      "activations/layer5_attention_weight_min": -107.0669174194336,
      "activations/layer6_attention_weight_max": 66.32534790039062,
      "activations/layer6_attention_weight_min": -55.78032302856445,
      "activations/layer7_attention_weight_max": 70.90815734863281,
      "activations/layer7_attention_weight_min": -62.88164520263672,
      "activations/layer8_attention_weight_max": 60.68325424194336,
      "activations/layer8_attention_weight_min": -47.72779846191406,
      "activations/layer9_attention_weight_max": 74.66098022460938,
      "activations/layer9_attention_weight_min": -52.92473220825195,
      "epoch": 1.9,
      "learning_rate": 0.00013914583333333333,
      "loss": 2.993,
      "step": 32700
    },
    {
      "activations/layer0_attention_weight_max": 17.0033016204834,
      "activations/layer0_attention_weight_min": -12.339993476867676,
      "activations/layer10_attention_weight_max": 50.22209548950195,
      "activations/layer10_attention_weight_min": -41.70554733276367,
      "activations/layer11_attention_weight_max": 45.016502380371094,
      "activations/layer11_attention_weight_min": -36.5731201171875,
      "activations/layer12_attention_weight_max": 32.81704330444336,
      "activations/layer12_attention_weight_min": -28.06918716430664,
      "activations/layer13_attention_weight_max": 49.02610778808594,
      "activations/layer13_attention_weight_min": -39.4317512512207,
      "activations/layer14_attention_weight_max": 79.1479721069336,
      "activations/layer14_attention_weight_min": -59.777130126953125,
      "activations/layer15_attention_weight_max": 56.43600082397461,
      "activations/layer15_attention_weight_min": -43.08124923706055,
      "activations/layer16_attention_weight_max": 53.5185432434082,
      "activations/layer16_attention_weight_min": -35.18649673461914,
      "activations/layer17_attention_weight_max": 47.2651481628418,
      "activations/layer17_attention_weight_min": -28.65398597717285,
      "activations/layer18_attention_weight_max": 32.158782958984375,
      "activations/layer18_attention_weight_min": -21.93699836730957,
      "activations/layer19_attention_weight_max": 48.508975982666016,
      "activations/layer19_attention_weight_min": -29.964710235595703,
      "activations/layer1_attention_weight_max": 17.414051055908203,
      "activations/layer1_attention_weight_min": -11.453725814819336,
      "activations/layer20_attention_weight_max": 41.0191764831543,
      "activations/layer20_attention_weight_min": -26.219444274902344,
      "activations/layer21_attention_weight_max": 34.923011779785156,
      "activations/layer21_attention_weight_min": -22.74704360961914,
      "activations/layer22_attention_weight_max": 46.978065490722656,
      "activations/layer22_attention_weight_min": -25.713600158691406,
      "activations/layer23_attention_weight_max": 25.562503814697266,
      "activations/layer23_attention_weight_min": -15.693540573120117,
      "activations/layer2_attention_weight_max": 28.805301666259766,
      "activations/layer2_attention_weight_min": -28.688953399658203,
      "activations/layer3_attention_weight_max": 71.85514831542969,
      "activations/layer3_attention_weight_min": -74.89946746826172,
      "activations/layer4_attention_weight_max": 124.36722564697266,
      "activations/layer4_attention_weight_min": -115.8472671508789,
      "activations/layer5_attention_weight_max": 114.31845092773438,
      "activations/layer5_attention_weight_min": -99.08116912841797,
      "activations/layer6_attention_weight_max": 63.367103576660156,
      "activations/layer6_attention_weight_min": -50.609535217285156,
      "activations/layer7_attention_weight_max": 69.95159912109375,
      "activations/layer7_attention_weight_min": -63.60044479370117,
      "activations/layer8_attention_weight_max": 46.59556198120117,
      "activations/layer8_attention_weight_min": -39.94697952270508,
      "activations/layer9_attention_weight_max": 73.63446807861328,
      "activations/layer9_attention_weight_min": -65.55883026123047,
      "epoch": 1.9,
      "learning_rate": 0.00013912689393939394,
      "loss": 2.9732,
      "step": 32750
    },
    {
      "activations/layer0_attention_weight_max": 16.7648983001709,
      "activations/layer0_attention_weight_min": -11.559603691101074,
      "activations/layer10_attention_weight_max": 52.30369567871094,
      "activations/layer10_attention_weight_min": -43.4410400390625,
      "activations/layer11_attention_weight_max": 36.633323669433594,
      "activations/layer11_attention_weight_min": -35.683197021484375,
      "activations/layer12_attention_weight_max": 50.00390625,
      "activations/layer12_attention_weight_min": -29.701183319091797,
      "activations/layer13_attention_weight_max": 52.6238899230957,
      "activations/layer13_attention_weight_min": -37.39521408081055,
      "activations/layer14_attention_weight_max": 75.91161346435547,
      "activations/layer14_attention_weight_min": -59.54872131347656,
      "activations/layer15_attention_weight_max": 47.88448715209961,
      "activations/layer15_attention_weight_min": -38.680049896240234,
      "activations/layer16_attention_weight_max": 49.21542739868164,
      "activations/layer16_attention_weight_min": -38.2903938293457,
      "activations/layer17_attention_weight_max": 44.74610900878906,
      "activations/layer17_attention_weight_min": -29.74131202697754,
      "activations/layer18_attention_weight_max": 27.310367584228516,
      "activations/layer18_attention_weight_min": -17.78803062438965,
      "activations/layer19_attention_weight_max": 49.322235107421875,
      "activations/layer19_attention_weight_min": -31.684186935424805,
      "activations/layer1_attention_weight_max": 16.60520362854004,
      "activations/layer1_attention_weight_min": -11.670585632324219,
      "activations/layer20_attention_weight_max": 43.11717987060547,
      "activations/layer20_attention_weight_min": -25.8322811126709,
      "activations/layer21_attention_weight_max": 34.52702713012695,
      "activations/layer21_attention_weight_min": -21.23208999633789,
      "activations/layer22_attention_weight_max": 46.77655792236328,
      "activations/layer22_attention_weight_min": -26.452804565429688,
      "activations/layer23_attention_weight_max": 26.05047607421875,
      "activations/layer23_attention_weight_min": -14.875238418579102,
      "activations/layer2_attention_weight_max": 27.882062911987305,
      "activations/layer2_attention_weight_min": -27.1845760345459,
      "activations/layer3_attention_weight_max": 71.16484832763672,
      "activations/layer3_attention_weight_min": -72.5248794555664,
      "activations/layer4_attention_weight_max": 121.36385345458984,
      "activations/layer4_attention_weight_min": -116.94129943847656,
      "activations/layer5_attention_weight_max": 106.74363708496094,
      "activations/layer5_attention_weight_min": -95.87681579589844,
      "activations/layer6_attention_weight_max": 57.08515548706055,
      "activations/layer6_attention_weight_min": -52.58220291137695,
      "activations/layer7_attention_weight_max": 65.63020324707031,
      "activations/layer7_attention_weight_min": -60.105018615722656,
      "activations/layer8_attention_weight_max": 47.449623107910156,
      "activations/layer8_attention_weight_min": -39.78197479248047,
      "activations/layer9_attention_weight_max": 75.77513885498047,
      "activations/layer9_attention_weight_min": -55.78774642944336,
      "epoch": 1.91,
      "learning_rate": 0.00013910795454545453,
      "loss": 2.9766,
      "step": 32800
    },
    {
      "activations/layer0_attention_weight_max": 16.6575927734375,
      "activations/layer0_attention_weight_min": -11.741530418395996,
      "activations/layer10_attention_weight_max": 47.03871536254883,
      "activations/layer10_attention_weight_min": -48.54417037963867,
      "activations/layer11_attention_weight_max": 38.08671569824219,
      "activations/layer11_attention_weight_min": -35.27117156982422,
      "activations/layer12_attention_weight_max": 42.30876922607422,
      "activations/layer12_attention_weight_min": -33.00790023803711,
      "activations/layer13_attention_weight_max": 42.560462951660156,
      "activations/layer13_attention_weight_min": -40.01976013183594,
      "activations/layer14_attention_weight_max": 69.84418487548828,
      "activations/layer14_attention_weight_min": -61.306854248046875,
      "activations/layer15_attention_weight_max": 49.393409729003906,
      "activations/layer15_attention_weight_min": -42.205196380615234,
      "activations/layer16_attention_weight_max": 44.511070251464844,
      "activations/layer16_attention_weight_min": -37.315677642822266,
      "activations/layer17_attention_weight_max": 39.566749572753906,
      "activations/layer17_attention_weight_min": -31.59489631652832,
      "activations/layer18_attention_weight_max": 25.273052215576172,
      "activations/layer18_attention_weight_min": -20.296117782592773,
      "activations/layer19_attention_weight_max": 49.793418884277344,
      "activations/layer19_attention_weight_min": -35.64433288574219,
      "activations/layer1_attention_weight_max": 18.60986328125,
      "activations/layer1_attention_weight_min": -11.33519458770752,
      "activations/layer20_attention_weight_max": 34.2603645324707,
      "activations/layer20_attention_weight_min": -28.812999725341797,
      "activations/layer21_attention_weight_max": 28.720285415649414,
      "activations/layer21_attention_weight_min": -21.13788414001465,
      "activations/layer22_attention_weight_max": 40.76742935180664,
      "activations/layer22_attention_weight_min": -27.46754264831543,
      "activations/layer23_attention_weight_max": 22.785079956054688,
      "activations/layer23_attention_weight_min": -16.12240982055664,
      "activations/layer2_attention_weight_max": 29.510398864746094,
      "activations/layer2_attention_weight_min": -27.347755432128906,
      "activations/layer3_attention_weight_max": 69.39927673339844,
      "activations/layer3_attention_weight_min": -73.43866729736328,
      "activations/layer4_attention_weight_max": 116.40644836425781,
      "activations/layer4_attention_weight_min": -115.30340576171875,
      "activations/layer5_attention_weight_max": 108.81722259521484,
      "activations/layer5_attention_weight_min": -101.19164276123047,
      "activations/layer6_attention_weight_max": 62.79191970825195,
      "activations/layer6_attention_weight_min": -55.62691116333008,
      "activations/layer7_attention_weight_max": 66.1544189453125,
      "activations/layer7_attention_weight_min": -62.3133430480957,
      "activations/layer8_attention_weight_max": 45.2683219909668,
      "activations/layer8_attention_weight_min": -40.462337493896484,
      "activations/layer9_attention_weight_max": 70.4063491821289,
      "activations/layer9_attention_weight_min": -61.18545150756836,
      "epoch": 1.91,
      "learning_rate": 0.00013908901515151514,
      "loss": 2.9972,
      "step": 32850
    },
    {
      "activations/layer0_attention_weight_max": 16.18291473388672,
      "activations/layer0_attention_weight_min": -11.512187957763672,
      "activations/layer10_attention_weight_max": 50.734214782714844,
      "activations/layer10_attention_weight_min": -42.558502197265625,
      "activations/layer11_attention_weight_max": 39.886138916015625,
      "activations/layer11_attention_weight_min": -34.58845901489258,
      "activations/layer12_attention_weight_max": 40.87343215942383,
      "activations/layer12_attention_weight_min": -36.0063591003418,
      "activations/layer13_attention_weight_max": 50.00343322753906,
      "activations/layer13_attention_weight_min": -38.279876708984375,
      "activations/layer14_attention_weight_max": 74.5253677368164,
      "activations/layer14_attention_weight_min": -61.542049407958984,
      "activations/layer15_attention_weight_max": 54.805511474609375,
      "activations/layer15_attention_weight_min": -38.211944580078125,
      "activations/layer16_attention_weight_max": 49.06769561767578,
      "activations/layer16_attention_weight_min": -38.19889831542969,
      "activations/layer17_attention_weight_max": 47.88652420043945,
      "activations/layer17_attention_weight_min": -31.06050682067871,
      "activations/layer18_attention_weight_max": 26.681503295898438,
      "activations/layer18_attention_weight_min": -19.452848434448242,
      "activations/layer19_attention_weight_max": 51.61109161376953,
      "activations/layer19_attention_weight_min": -31.438220977783203,
      "activations/layer1_attention_weight_max": 17.045700073242188,
      "activations/layer1_attention_weight_min": -11.370201110839844,
      "activations/layer20_attention_weight_max": 38.761714935302734,
      "activations/layer20_attention_weight_min": -23.065099716186523,
      "activations/layer21_attention_weight_max": 29.685619354248047,
      "activations/layer21_attention_weight_min": -20.01841163635254,
      "activations/layer22_attention_weight_max": 43.7872200012207,
      "activations/layer22_attention_weight_min": -25.56369972229004,
      "activations/layer23_attention_weight_max": 23.105077743530273,
      "activations/layer23_attention_weight_min": -15.661043167114258,
      "activations/layer2_attention_weight_max": 27.11801528930664,
      "activations/layer2_attention_weight_min": -26.246826171875,
      "activations/layer3_attention_weight_max": 66.3080062866211,
      "activations/layer3_attention_weight_min": -71.18365478515625,
      "activations/layer4_attention_weight_max": 115.94979095458984,
      "activations/layer4_attention_weight_min": -117.86775970458984,
      "activations/layer5_attention_weight_max": 105.65947723388672,
      "activations/layer5_attention_weight_min": -103.65895080566406,
      "activations/layer6_attention_weight_max": 59.01729202270508,
      "activations/layer6_attention_weight_min": -58.170536041259766,
      "activations/layer7_attention_weight_max": 60.24496841430664,
      "activations/layer7_attention_weight_min": -60.855201721191406,
      "activations/layer8_attention_weight_max": 43.74314880371094,
      "activations/layer8_attention_weight_min": -39.258888244628906,
      "activations/layer9_attention_weight_max": 65.47088623046875,
      "activations/layer9_attention_weight_min": -61.545989990234375,
      "epoch": 1.91,
      "learning_rate": 0.00013907007575757573,
      "loss": 2.9854,
      "step": 32900
    },
    {
      "activations/layer0_attention_weight_max": 14.46782398223877,
      "activations/layer0_attention_weight_min": -11.597670555114746,
      "activations/layer10_attention_weight_max": 53.94684982299805,
      "activations/layer10_attention_weight_min": -45.66998291015625,
      "activations/layer11_attention_weight_max": 41.11497497558594,
      "activations/layer11_attention_weight_min": -32.21479797363281,
      "activations/layer12_attention_weight_max": 103.38524627685547,
      "activations/layer12_attention_weight_min": -51.51553726196289,
      "activations/layer13_attention_weight_max": 73.5010757446289,
      "activations/layer13_attention_weight_min": -47.62767791748047,
      "activations/layer14_attention_weight_max": 81.03475189208984,
      "activations/layer14_attention_weight_min": -62.26262664794922,
      "activations/layer15_attention_weight_max": 49.74143981933594,
      "activations/layer15_attention_weight_min": -39.0095329284668,
      "activations/layer16_attention_weight_max": 48.257320404052734,
      "activations/layer16_attention_weight_min": -36.09235763549805,
      "activations/layer17_attention_weight_max": 47.488441467285156,
      "activations/layer17_attention_weight_min": -32.50350570678711,
      "activations/layer18_attention_weight_max": 26.794269561767578,
      "activations/layer18_attention_weight_min": -18.82754135131836,
      "activations/layer19_attention_weight_max": 46.9796028137207,
      "activations/layer19_attention_weight_min": -32.88570022583008,
      "activations/layer1_attention_weight_max": 17.49357795715332,
      "activations/layer1_attention_weight_min": -12.438787460327148,
      "activations/layer20_attention_weight_max": 37.026424407958984,
      "activations/layer20_attention_weight_min": -30.35192108154297,
      "activations/layer21_attention_weight_max": 31.941864013671875,
      "activations/layer21_attention_weight_min": -21.115798950195312,
      "activations/layer22_attention_weight_max": 53.22406005859375,
      "activations/layer22_attention_weight_min": -27.18187141418457,
      "activations/layer23_attention_weight_max": 28.44368553161621,
      "activations/layer23_attention_weight_min": -18.372833251953125,
      "activations/layer2_attention_weight_max": 30.19851303100586,
      "activations/layer2_attention_weight_min": -28.464284896850586,
      "activations/layer3_attention_weight_max": 68.85823822021484,
      "activations/layer3_attention_weight_min": -73.86173248291016,
      "activations/layer4_attention_weight_max": 121.6185531616211,
      "activations/layer4_attention_weight_min": -113.24652099609375,
      "activations/layer5_attention_weight_max": 113.50508880615234,
      "activations/layer5_attention_weight_min": -98.17832946777344,
      "activations/layer6_attention_weight_max": 62.82768630981445,
      "activations/layer6_attention_weight_min": -52.628658294677734,
      "activations/layer7_attention_weight_max": 63.7971305847168,
      "activations/layer7_attention_weight_min": -64.17920684814453,
      "activations/layer8_attention_weight_max": 49.83983612060547,
      "activations/layer8_attention_weight_min": -40.98390579223633,
      "activations/layer9_attention_weight_max": 74.17925262451172,
      "activations/layer9_attention_weight_min": -61.36468505859375,
      "epoch": 1.91,
      "learning_rate": 0.00013905113636363635,
      "loss": 2.992,
      "step": 32950
    },
    {
      "activations/layer0_attention_weight_max": 16.575733184814453,
      "activations/layer0_attention_weight_min": -11.975037574768066,
      "activations/layer10_attention_weight_max": 52.50215530395508,
      "activations/layer10_attention_weight_min": -49.39392852783203,
      "activations/layer11_attention_weight_max": 40.18086242675781,
      "activations/layer11_attention_weight_min": -33.414825439453125,
      "activations/layer12_attention_weight_max": 38.45329666137695,
      "activations/layer12_attention_weight_min": -29.336490631103516,
      "activations/layer13_attention_weight_max": 49.996177673339844,
      "activations/layer13_attention_weight_min": -41.4072265625,
      "activations/layer14_attention_weight_max": 71.71563720703125,
      "activations/layer14_attention_weight_min": -58.114341735839844,
      "activations/layer15_attention_weight_max": 52.823646545410156,
      "activations/layer15_attention_weight_min": -39.22275924682617,
      "activations/layer16_attention_weight_max": 54.53849411010742,
      "activations/layer16_attention_weight_min": -36.89120864868164,
      "activations/layer17_attention_weight_max": 44.45004653930664,
      "activations/layer17_attention_weight_min": -31.053829193115234,
      "activations/layer18_attention_weight_max": 25.967878341674805,
      "activations/layer18_attention_weight_min": -16.25501823425293,
      "activations/layer19_attention_weight_max": 51.747779846191406,
      "activations/layer19_attention_weight_min": -32.220577239990234,
      "activations/layer1_attention_weight_max": 19.37847900390625,
      "activations/layer1_attention_weight_min": -10.558666229248047,
      "activations/layer20_attention_weight_max": 41.454383850097656,
      "activations/layer20_attention_weight_min": -28.421592712402344,
      "activations/layer21_attention_weight_max": 31.508819580078125,
      "activations/layer21_attention_weight_min": -20.80345344543457,
      "activations/layer22_attention_weight_max": 40.9356689453125,
      "activations/layer22_attention_weight_min": -24.95615005493164,
      "activations/layer23_attention_weight_max": 26.80269432067871,
      "activations/layer23_attention_weight_min": -16.797611236572266,
      "activations/layer2_attention_weight_max": 27.863309860229492,
      "activations/layer2_attention_weight_min": -27.354137420654297,
      "activations/layer3_attention_weight_max": 68.09686279296875,
      "activations/layer3_attention_weight_min": -71.35896301269531,
      "activations/layer4_attention_weight_max": 127.10936737060547,
      "activations/layer4_attention_weight_min": -117.53337860107422,
      "activations/layer5_attention_weight_max": 114.65950775146484,
      "activations/layer5_attention_weight_min": -102.97856903076172,
      "activations/layer6_attention_weight_max": 64.8957748413086,
      "activations/layer6_attention_weight_min": -54.21113967895508,
      "activations/layer7_attention_weight_max": 68.04424285888672,
      "activations/layer7_attention_weight_min": -59.57461166381836,
      "activations/layer8_attention_weight_max": 48.05387496948242,
      "activations/layer8_attention_weight_min": -38.770206451416016,
      "activations/layer9_attention_weight_max": 72.4580078125,
      "activations/layer9_attention_weight_min": -53.495849609375,
      "epoch": 1.92,
      "learning_rate": 0.00013903219696969696,
      "loss": 2.9983,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_loss": 2.921875,
      "eval_runtime": 8.4859,
      "eval_samples_per_second": 506.016,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_openwebtext_loss": 2.921875,
      "eval_openwebtext_ppl": 18.576084986220376,
      "eval_openwebtext_runtime": 8.4859,
      "eval_openwebtext_samples_per_second": 506.016,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_wikitext_loss": 3.193359375,
      "eval_wikitext_ppl": 24.370158584949497,
      "eval_wikitext_runtime": 2.0019,
      "eval_wikitext_samples_per_second": 227.781,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_lambada_loss": 3.09765625,
      "eval_lambada_ppl": 22.145985754016134,
      "eval_lambada_runtime": 9.593,
      "eval_lambada_samples_per_second": 507.556,
      "step": 33000
    },
    {
      "activations/layer0_attention_weight_max": 16.41950798034668,
      "activations/layer0_attention_weight_min": -11.347375869750977,
      "activations/layer10_attention_weight_max": 52.693115234375,
      "activations/layer10_attention_weight_min": -43.03694534301758,
      "activations/layer11_attention_weight_max": 42.58235549926758,
      "activations/layer11_attention_weight_min": -34.130104064941406,
      "activations/layer12_attention_weight_max": 37.85357666015625,
      "activations/layer12_attention_weight_min": -29.541528701782227,
      "activations/layer13_attention_weight_max": 49.55618667602539,
      "activations/layer13_attention_weight_min": -39.43461990356445,
      "activations/layer14_attention_weight_max": 82.35002136230469,
      "activations/layer14_attention_weight_min": -59.596858978271484,
      "activations/layer15_attention_weight_max": 64.70723724365234,
      "activations/layer15_attention_weight_min": -43.16361999511719,
      "activations/layer16_attention_weight_max": 55.4720573425293,
      "activations/layer16_attention_weight_min": -38.726524353027344,
      "activations/layer17_attention_weight_max": 43.52391815185547,
      "activations/layer17_attention_weight_min": -29.138980865478516,
      "activations/layer18_attention_weight_max": 30.728229522705078,
      "activations/layer18_attention_weight_min": -19.270767211914062,
      "activations/layer19_attention_weight_max": 56.98957443237305,
      "activations/layer19_attention_weight_min": -41.04403305053711,
      "activations/layer1_attention_weight_max": 17.870407104492188,
      "activations/layer1_attention_weight_min": -11.48869800567627,
      "activations/layer20_attention_weight_max": 38.566917419433594,
      "activations/layer20_attention_weight_min": -25.687477111816406,
      "activations/layer21_attention_weight_max": 34.22426223754883,
      "activations/layer21_attention_weight_min": -20.617599487304688,
      "activations/layer22_attention_weight_max": 42.42673873901367,
      "activations/layer22_attention_weight_min": -28.34164047241211,
      "activations/layer23_attention_weight_max": 26.2991943359375,
      "activations/layer23_attention_weight_min": -15.793119430541992,
      "activations/layer2_attention_weight_max": 27.888748168945312,
      "activations/layer2_attention_weight_min": -26.553386688232422,
      "activations/layer3_attention_weight_max": 68.22505950927734,
      "activations/layer3_attention_weight_min": -70.06871795654297,
      "activations/layer4_attention_weight_max": 123.56058502197266,
      "activations/layer4_attention_weight_min": -110.86614227294922,
      "activations/layer5_attention_weight_max": 113.75484466552734,
      "activations/layer5_attention_weight_min": -96.97554016113281,
      "activations/layer6_attention_weight_max": 63.9195556640625,
      "activations/layer6_attention_weight_min": -53.252708435058594,
      "activations/layer7_attention_weight_max": 66.64752960205078,
      "activations/layer7_attention_weight_min": -66.92668151855469,
      "activations/layer8_attention_weight_max": 44.66849899291992,
      "activations/layer8_attention_weight_min": -41.38656997680664,
      "activations/layer9_attention_weight_max": 74.01226043701172,
      "activations/layer9_attention_weight_min": -58.64387893676758,
      "epoch": 1.92,
      "learning_rate": 0.00013901325757575755,
      "loss": 2.9783,
      "step": 33050
    },
    {
      "activations/layer0_attention_weight_max": 16.692691802978516,
      "activations/layer0_attention_weight_min": -11.988044738769531,
      "activations/layer10_attention_weight_max": 57.424434661865234,
      "activations/layer10_attention_weight_min": -46.669368743896484,
      "activations/layer11_attention_weight_max": 46.92431640625,
      "activations/layer11_attention_weight_min": -36.74147415161133,
      "activations/layer12_attention_weight_max": 54.023902893066406,
      "activations/layer12_attention_weight_min": -32.76422882080078,
      "activations/layer13_attention_weight_max": 48.791053771972656,
      "activations/layer13_attention_weight_min": -40.902076721191406,
      "activations/layer14_attention_weight_max": 82.32743835449219,
      "activations/layer14_attention_weight_min": -59.6013069152832,
      "activations/layer15_attention_weight_max": 55.868675231933594,
      "activations/layer15_attention_weight_min": -43.19456481933594,
      "activations/layer16_attention_weight_max": 55.648597717285156,
      "activations/layer16_attention_weight_min": -38.14469909667969,
      "activations/layer17_attention_weight_max": 50.68808364868164,
      "activations/layer17_attention_weight_min": -30.424604415893555,
      "activations/layer18_attention_weight_max": 26.87726402282715,
      "activations/layer18_attention_weight_min": -19.501102447509766,
      "activations/layer19_attention_weight_max": 54.31938552856445,
      "activations/layer19_attention_weight_min": -31.56440544128418,
      "activations/layer1_attention_weight_max": 18.794700622558594,
      "activations/layer1_attention_weight_min": -12.18467903137207,
      "activations/layer20_attention_weight_max": 33.54800033569336,
      "activations/layer20_attention_weight_min": -23.15159797668457,
      "activations/layer21_attention_weight_max": 31.922542572021484,
      "activations/layer21_attention_weight_min": -19.32786750793457,
      "activations/layer22_attention_weight_max": 42.70063400268555,
      "activations/layer22_attention_weight_min": -26.864295959472656,
      "activations/layer23_attention_weight_max": 29.99964714050293,
      "activations/layer23_attention_weight_min": -16.643234252929688,
      "activations/layer2_attention_weight_max": 29.531673431396484,
      "activations/layer2_attention_weight_min": -29.56369400024414,
      "activations/layer3_attention_weight_max": 77.22174835205078,
      "activations/layer3_attention_weight_min": -77.17594146728516,
      "activations/layer4_attention_weight_max": 139.43447875976562,
      "activations/layer4_attention_weight_min": -118.9292984008789,
      "activations/layer5_attention_weight_max": 129.3079071044922,
      "activations/layer5_attention_weight_min": -106.1433334350586,
      "activations/layer6_attention_weight_max": 69.64250946044922,
      "activations/layer6_attention_weight_min": -57.47795104980469,
      "activations/layer7_attention_weight_max": 77.61842346191406,
      "activations/layer7_attention_weight_min": -67.95677185058594,
      "activations/layer8_attention_weight_max": 57.77769088745117,
      "activations/layer8_attention_weight_min": -47.11159133911133,
      "activations/layer9_attention_weight_max": 81.69207000732422,
      "activations/layer9_attention_weight_min": -70.12457275390625,
      "epoch": 1.92,
      "learning_rate": 0.00013899431818181816,
      "loss": 2.9707,
      "step": 33100
    },
    {
      "activations/layer0_attention_weight_max": 14.736185073852539,
      "activations/layer0_attention_weight_min": -11.76796817779541,
      "activations/layer10_attention_weight_max": 50.887962341308594,
      "activations/layer10_attention_weight_min": -42.69514846801758,
      "activations/layer11_attention_weight_max": 43.237953186035156,
      "activations/layer11_attention_weight_min": -35.587886810302734,
      "activations/layer12_attention_weight_max": 34.11676025390625,
      "activations/layer12_attention_weight_min": -27.024553298950195,
      "activations/layer13_attention_weight_max": 48.241024017333984,
      "activations/layer13_attention_weight_min": -37.333290100097656,
      "activations/layer14_attention_weight_max": 93.81922912597656,
      "activations/layer14_attention_weight_min": -78.62591552734375,
      "activations/layer15_attention_weight_max": 58.24213409423828,
      "activations/layer15_attention_weight_min": -43.11526107788086,
      "activations/layer16_attention_weight_max": 51.4692497253418,
      "activations/layer16_attention_weight_min": -40.1185188293457,
      "activations/layer17_attention_weight_max": 48.16566467285156,
      "activations/layer17_attention_weight_min": -31.078060150146484,
      "activations/layer18_attention_weight_max": 25.484399795532227,
      "activations/layer18_attention_weight_min": -16.87946891784668,
      "activations/layer19_attention_weight_max": 64.35492706298828,
      "activations/layer19_attention_weight_min": -36.70005798339844,
      "activations/layer1_attention_weight_max": 17.302385330200195,
      "activations/layer1_attention_weight_min": -11.427003860473633,
      "activations/layer20_attention_weight_max": 43.05888366699219,
      "activations/layer20_attention_weight_min": -27.3438663482666,
      "activations/layer21_attention_weight_max": 32.22733688354492,
      "activations/layer21_attention_weight_min": -18.79817008972168,
      "activations/layer22_attention_weight_max": 46.330726623535156,
      "activations/layer22_attention_weight_min": -25.96377182006836,
      "activations/layer23_attention_weight_max": 28.70229148864746,
      "activations/layer23_attention_weight_min": -17.930566787719727,
      "activations/layer2_attention_weight_max": 26.462657928466797,
      "activations/layer2_attention_weight_min": -25.39900016784668,
      "activations/layer3_attention_weight_max": 67.67194366455078,
      "activations/layer3_attention_weight_min": -71.78306579589844,
      "activations/layer4_attention_weight_max": 124.8673095703125,
      "activations/layer4_attention_weight_min": -120.5027084350586,
      "activations/layer5_attention_weight_max": 113.59243774414062,
      "activations/layer5_attention_weight_min": -96.6361083984375,
      "activations/layer6_attention_weight_max": 62.803253173828125,
      "activations/layer6_attention_weight_min": -52.50007247924805,
      "activations/layer7_attention_weight_max": 67.50300598144531,
      "activations/layer7_attention_weight_min": -63.403621673583984,
      "activations/layer8_attention_weight_max": 47.31984329223633,
      "activations/layer8_attention_weight_min": -40.70569610595703,
      "activations/layer9_attention_weight_max": 64.30651092529297,
      "activations/layer9_attention_weight_min": -54.1722412109375,
      "epoch": 1.93,
      "learning_rate": 0.00013897537878787878,
      "loss": 2.9798,
      "step": 33150
    },
    {
      "activations/layer0_attention_weight_max": 16.71746063232422,
      "activations/layer0_attention_weight_min": -11.514031410217285,
      "activations/layer10_attention_weight_max": 50.899925231933594,
      "activations/layer10_attention_weight_min": -40.940486907958984,
      "activations/layer11_attention_weight_max": 38.552635192871094,
      "activations/layer11_attention_weight_min": -36.52437210083008,
      "activations/layer12_attention_weight_max": 60.54100799560547,
      "activations/layer12_attention_weight_min": -33.43889236450195,
      "activations/layer13_attention_weight_max": 48.854164123535156,
      "activations/layer13_attention_weight_min": -38.277626037597656,
      "activations/layer14_attention_weight_max": 75.99665832519531,
      "activations/layer14_attention_weight_min": -59.12100601196289,
      "activations/layer15_attention_weight_max": 54.68163299560547,
      "activations/layer15_attention_weight_min": -42.873374938964844,
      "activations/layer16_attention_weight_max": 51.89380645751953,
      "activations/layer16_attention_weight_min": -39.020469665527344,
      "activations/layer17_attention_weight_max": 42.78704071044922,
      "activations/layer17_attention_weight_min": -31.80995750427246,
      "activations/layer18_attention_weight_max": 25.462831497192383,
      "activations/layer18_attention_weight_min": -17.288061141967773,
      "activations/layer19_attention_weight_max": 47.99924850463867,
      "activations/layer19_attention_weight_min": -30.850927352905273,
      "activations/layer1_attention_weight_max": 17.502042770385742,
      "activations/layer1_attention_weight_min": -11.703646659851074,
      "activations/layer20_attention_weight_max": 36.617774963378906,
      "activations/layer20_attention_weight_min": -25.0567626953125,
      "activations/layer21_attention_weight_max": 34.19141387939453,
      "activations/layer21_attention_weight_min": -20.28219985961914,
      "activations/layer22_attention_weight_max": 44.36964797973633,
      "activations/layer22_attention_weight_min": -24.01119041442871,
      "activations/layer23_attention_weight_max": 25.060440063476562,
      "activations/layer23_attention_weight_min": -16.629901885986328,
      "activations/layer2_attention_weight_max": 28.297637939453125,
      "activations/layer2_attention_weight_min": -28.23318862915039,
      "activations/layer3_attention_weight_max": 70.02774047851562,
      "activations/layer3_attention_weight_min": -75.02582550048828,
      "activations/layer4_attention_weight_max": 123.07405090332031,
      "activations/layer4_attention_weight_min": -117.93033599853516,
      "activations/layer5_attention_weight_max": 112.33110046386719,
      "activations/layer5_attention_weight_min": -99.93496704101562,
      "activations/layer6_attention_weight_max": 60.21015548706055,
      "activations/layer6_attention_weight_min": -53.28960418701172,
      "activations/layer7_attention_weight_max": 62.57707595825195,
      "activations/layer7_attention_weight_min": -61.43291473388672,
      "activations/layer8_attention_weight_max": 46.58894729614258,
      "activations/layer8_attention_weight_min": -39.52886199951172,
      "activations/layer9_attention_weight_max": 66.61573791503906,
      "activations/layer9_attention_weight_min": -58.84930419921875,
      "epoch": 1.93,
      "learning_rate": 0.00013895643939393937,
      "loss": 2.9804,
      "step": 33200
    },
    {
      "activations/layer0_attention_weight_max": 16.997941970825195,
      "activations/layer0_attention_weight_min": -11.325067520141602,
      "activations/layer10_attention_weight_max": 51.20535659790039,
      "activations/layer10_attention_weight_min": -45.0694694519043,
      "activations/layer11_attention_weight_max": 41.974674224853516,
      "activations/layer11_attention_weight_min": -39.0161247253418,
      "activations/layer12_attention_weight_max": 42.77531814575195,
      "activations/layer12_attention_weight_min": -33.70951461791992,
      "activations/layer13_attention_weight_max": 52.67335891723633,
      "activations/layer13_attention_weight_min": -41.24152755737305,
      "activations/layer14_attention_weight_max": 73.62895202636719,
      "activations/layer14_attention_weight_min": -54.31660461425781,
      "activations/layer15_attention_weight_max": 49.362239837646484,
      "activations/layer15_attention_weight_min": -36.1034049987793,
      "activations/layer16_attention_weight_max": 64.95187377929688,
      "activations/layer16_attention_weight_min": -43.18960189819336,
      "activations/layer17_attention_weight_max": 48.4025764465332,
      "activations/layer17_attention_weight_min": -29.08430290222168,
      "activations/layer18_attention_weight_max": 26.262401580810547,
      "activations/layer18_attention_weight_min": -17.793739318847656,
      "activations/layer19_attention_weight_max": 56.01339340209961,
      "activations/layer19_attention_weight_min": -35.43442916870117,
      "activations/layer1_attention_weight_max": 18.16288948059082,
      "activations/layer1_attention_weight_min": -11.26429271697998,
      "activations/layer20_attention_weight_max": 44.42086410522461,
      "activations/layer20_attention_weight_min": -24.673709869384766,
      "activations/layer21_attention_weight_max": 32.323219299316406,
      "activations/layer21_attention_weight_min": -19.44955825805664,
      "activations/layer22_attention_weight_max": 48.873046875,
      "activations/layer22_attention_weight_min": -25.8256893157959,
      "activations/layer23_attention_weight_max": 25.969533920288086,
      "activations/layer23_attention_weight_min": -14.79349422454834,
      "activations/layer2_attention_weight_max": 28.49676513671875,
      "activations/layer2_attention_weight_min": -27.73488426208496,
      "activations/layer3_attention_weight_max": 69.13526153564453,
      "activations/layer3_attention_weight_min": -70.04498291015625,
      "activations/layer4_attention_weight_max": 125.44500732421875,
      "activations/layer4_attention_weight_min": -112.75630187988281,
      "activations/layer5_attention_weight_max": 113.23374938964844,
      "activations/layer5_attention_weight_min": -98.70469665527344,
      "activations/layer6_attention_weight_max": 63.25422668457031,
      "activations/layer6_attention_weight_min": -53.856746673583984,
      "activations/layer7_attention_weight_max": 83.05513763427734,
      "activations/layer7_attention_weight_min": -62.17466735839844,
      "activations/layer8_attention_weight_max": 56.53224182128906,
      "activations/layer8_attention_weight_min": -48.5559196472168,
      "activations/layer9_attention_weight_max": 84.25364685058594,
      "activations/layer9_attention_weight_min": -59.89031982421875,
      "epoch": 1.93,
      "learning_rate": 0.00013893749999999998,
      "loss": 2.98,
      "step": 33250
    },
    {
      "activations/layer0_attention_weight_max": 16.82230567932129,
      "activations/layer0_attention_weight_min": -11.458956718444824,
      "activations/layer10_attention_weight_max": 52.42749786376953,
      "activations/layer10_attention_weight_min": -45.38019561767578,
      "activations/layer11_attention_weight_max": 38.303035736083984,
      "activations/layer11_attention_weight_min": -35.71995162963867,
      "activations/layer12_attention_weight_max": 36.76063537597656,
      "activations/layer12_attention_weight_min": -30.207969665527344,
      "activations/layer13_attention_weight_max": 47.73421859741211,
      "activations/layer13_attention_weight_min": -37.14347839355469,
      "activations/layer14_attention_weight_max": 76.07843017578125,
      "activations/layer14_attention_weight_min": -64.24313354492188,
      "activations/layer15_attention_weight_max": 50.79156494140625,
      "activations/layer15_attention_weight_min": -39.410770416259766,
      "activations/layer16_attention_weight_max": 44.77702713012695,
      "activations/layer16_attention_weight_min": -34.627750396728516,
      "activations/layer17_attention_weight_max": 43.11374282836914,
      "activations/layer17_attention_weight_min": -31.774518966674805,
      "activations/layer18_attention_weight_max": 24.92556381225586,
      "activations/layer18_attention_weight_min": -17.597400665283203,
      "activations/layer19_attention_weight_max": 50.687416076660156,
      "activations/layer19_attention_weight_min": -29.62464714050293,
      "activations/layer1_attention_weight_max": 16.73301887512207,
      "activations/layer1_attention_weight_min": -11.608271598815918,
      "activations/layer20_attention_weight_max": 39.10247039794922,
      "activations/layer20_attention_weight_min": -24.746929168701172,
      "activations/layer21_attention_weight_max": 27.608749389648438,
      "activations/layer21_attention_weight_min": -19.330537796020508,
      "activations/layer22_attention_weight_max": 45.22237014770508,
      "activations/layer22_attention_weight_min": -26.976337432861328,
      "activations/layer23_attention_weight_max": 24.265724182128906,
      "activations/layer23_attention_weight_min": -16.847198486328125,
      "activations/layer2_attention_weight_max": 26.859060287475586,
      "activations/layer2_attention_weight_min": -28.26906967163086,
      "activations/layer3_attention_weight_max": 66.87227630615234,
      "activations/layer3_attention_weight_min": -72.02861022949219,
      "activations/layer4_attention_weight_max": 122.6490249633789,
      "activations/layer4_attention_weight_min": -116.02193450927734,
      "activations/layer5_attention_weight_max": 108.81051635742188,
      "activations/layer5_attention_weight_min": -100.84858703613281,
      "activations/layer6_attention_weight_max": 61.830787658691406,
      "activations/layer6_attention_weight_min": -56.28769302368164,
      "activations/layer7_attention_weight_max": 65.16676330566406,
      "activations/layer7_attention_weight_min": -59.87813186645508,
      "activations/layer8_attention_weight_max": 47.42708206176758,
      "activations/layer8_attention_weight_min": -40.36555099487305,
      "activations/layer9_attention_weight_max": 62.27418899536133,
      "activations/layer9_attention_weight_min": -55.3692512512207,
      "epoch": 1.93,
      "learning_rate": 0.0001389185606060606,
      "loss": 2.9773,
      "step": 33300
    },
    {
      "activations/layer0_attention_weight_max": 14.95051383972168,
      "activations/layer0_attention_weight_min": -11.783098220825195,
      "activations/layer10_attention_weight_max": 59.95302200317383,
      "activations/layer10_attention_weight_min": -49.57947540283203,
      "activations/layer11_attention_weight_max": 41.678409576416016,
      "activations/layer11_attention_weight_min": -33.41903305053711,
      "activations/layer12_attention_weight_max": 54.768280029296875,
      "activations/layer12_attention_weight_min": -28.99150276184082,
      "activations/layer13_attention_weight_max": 52.64338684082031,
      "activations/layer13_attention_weight_min": -36.82898712158203,
      "activations/layer14_attention_weight_max": 78.32308197021484,
      "activations/layer14_attention_weight_min": -56.57021713256836,
      "activations/layer15_attention_weight_max": 54.750057220458984,
      "activations/layer15_attention_weight_min": -37.61711883544922,
      "activations/layer16_attention_weight_max": 50.29166030883789,
      "activations/layer16_attention_weight_min": -35.19966506958008,
      "activations/layer17_attention_weight_max": 46.80241775512695,
      "activations/layer17_attention_weight_min": -30.200693130493164,
      "activations/layer18_attention_weight_max": 26.03799057006836,
      "activations/layer18_attention_weight_min": -16.586645126342773,
      "activations/layer19_attention_weight_max": 54.915740966796875,
      "activations/layer19_attention_weight_min": -31.962392807006836,
      "activations/layer1_attention_weight_max": 18.37742805480957,
      "activations/layer1_attention_weight_min": -11.56274700164795,
      "activations/layer20_attention_weight_max": 42.88882064819336,
      "activations/layer20_attention_weight_min": -23.024818420410156,
      "activations/layer21_attention_weight_max": 31.116952896118164,
      "activations/layer21_attention_weight_min": -18.080841064453125,
      "activations/layer22_attention_weight_max": 47.91114044189453,
      "activations/layer22_attention_weight_min": -23.547780990600586,
      "activations/layer23_attention_weight_max": 26.737226486206055,
      "activations/layer23_attention_weight_min": -14.87431526184082,
      "activations/layer2_attention_weight_max": 28.161727905273438,
      "activations/layer2_attention_weight_min": -27.960865020751953,
      "activations/layer3_attention_weight_max": 67.22036743164062,
      "activations/layer3_attention_weight_min": -69.47978973388672,
      "activations/layer4_attention_weight_max": 126.43395233154297,
      "activations/layer4_attention_weight_min": -113.16124725341797,
      "activations/layer5_attention_weight_max": 121.41639709472656,
      "activations/layer5_attention_weight_min": -101.93011474609375,
      "activations/layer6_attention_weight_max": 65.1931381225586,
      "activations/layer6_attention_weight_min": -57.90768814086914,
      "activations/layer7_attention_weight_max": 64.80970001220703,
      "activations/layer7_attention_weight_min": -62.84275436401367,
      "activations/layer8_attention_weight_max": 50.53455352783203,
      "activations/layer8_attention_weight_min": -41.923274993896484,
      "activations/layer9_attention_weight_max": 68.00008392333984,
      "activations/layer9_attention_weight_min": -52.63186264038086,
      "epoch": 1.94,
      "learning_rate": 0.00013889962121212119,
      "loss": 2.9779,
      "step": 33350
    },
    {
      "activations/layer0_attention_weight_max": 16.08259391784668,
      "activations/layer0_attention_weight_min": -11.580328941345215,
      "activations/layer10_attention_weight_max": 62.25998306274414,
      "activations/layer10_attention_weight_min": -44.84018325805664,
      "activations/layer11_attention_weight_max": 50.10722351074219,
      "activations/layer11_attention_weight_min": -35.31819534301758,
      "activations/layer12_attention_weight_max": 52.71134948730469,
      "activations/layer12_attention_weight_min": -30.5753231048584,
      "activations/layer13_attention_weight_max": 81.31083679199219,
      "activations/layer13_attention_weight_min": -53.1978874206543,
      "activations/layer14_attention_weight_max": 92.89802551269531,
      "activations/layer14_attention_weight_min": -59.959861755371094,
      "activations/layer15_attention_weight_max": 61.04294967651367,
      "activations/layer15_attention_weight_min": -39.44601821899414,
      "activations/layer16_attention_weight_max": 59.91349411010742,
      "activations/layer16_attention_weight_min": -38.339935302734375,
      "activations/layer17_attention_weight_max": 50.876468658447266,
      "activations/layer17_attention_weight_min": -28.65004539489746,
      "activations/layer18_attention_weight_max": 44.93830490112305,
      "activations/layer18_attention_weight_min": -20.490211486816406,
      "activations/layer19_attention_weight_max": 64.80176544189453,
      "activations/layer19_attention_weight_min": -31.328643798828125,
      "activations/layer1_attention_weight_max": 16.89162826538086,
      "activations/layer1_attention_weight_min": -10.77912425994873,
      "activations/layer20_attention_weight_max": 54.542381286621094,
      "activations/layer20_attention_weight_min": -23.575029373168945,
      "activations/layer21_attention_weight_max": 41.59587478637695,
      "activations/layer21_attention_weight_min": -20.87132453918457,
      "activations/layer22_attention_weight_max": 62.29486083984375,
      "activations/layer22_attention_weight_min": -24.18001937866211,
      "activations/layer23_attention_weight_max": 32.939697265625,
      "activations/layer23_attention_weight_min": -16.47154998779297,
      "activations/layer2_attention_weight_max": 27.084577560424805,
      "activations/layer2_attention_weight_min": -26.233135223388672,
      "activations/layer3_attention_weight_max": 70.31407928466797,
      "activations/layer3_attention_weight_min": -73.3138427734375,
      "activations/layer4_attention_weight_max": 127.38497161865234,
      "activations/layer4_attention_weight_min": -117.80709075927734,
      "activations/layer5_attention_weight_max": 114.6925048828125,
      "activations/layer5_attention_weight_min": -103.13032531738281,
      "activations/layer6_attention_weight_max": 63.05094909667969,
      "activations/layer6_attention_weight_min": -52.49672317504883,
      "activations/layer7_attention_weight_max": 69.47046661376953,
      "activations/layer7_attention_weight_min": -63.45404052734375,
      "activations/layer8_attention_weight_max": 52.73716735839844,
      "activations/layer8_attention_weight_min": -44.57327651977539,
      "activations/layer9_attention_weight_max": 78.15459442138672,
      "activations/layer9_attention_weight_min": -64.32320404052734,
      "epoch": 1.94,
      "learning_rate": 0.0001388806818181818,
      "loss": 2.9672,
      "step": 33400
    },
    {
      "activations/layer0_attention_weight_max": 15.57989501953125,
      "activations/layer0_attention_weight_min": -11.322431564331055,
      "activations/layer10_attention_weight_max": 57.2491569519043,
      "activations/layer10_attention_weight_min": -43.33711624145508,
      "activations/layer11_attention_weight_max": 44.87672424316406,
      "activations/layer11_attention_weight_min": -33.27273941040039,
      "activations/layer12_attention_weight_max": 55.286624908447266,
      "activations/layer12_attention_weight_min": -41.75078201293945,
      "activations/layer13_attention_weight_max": 54.450782775878906,
      "activations/layer13_attention_weight_min": -40.35353088378906,
      "activations/layer14_attention_weight_max": 90.31883239746094,
      "activations/layer14_attention_weight_min": -64.1152572631836,
      "activations/layer15_attention_weight_max": 59.21908187866211,
      "activations/layer15_attention_weight_min": -36.37279510498047,
      "activations/layer16_attention_weight_max": 65.12739562988281,
      "activations/layer16_attention_weight_min": -36.276084899902344,
      "activations/layer17_attention_weight_max": 47.0301628112793,
      "activations/layer17_attention_weight_min": -28.643911361694336,
      "activations/layer18_attention_weight_max": 32.5949592590332,
      "activations/layer18_attention_weight_min": -20.90167808532715,
      "activations/layer19_attention_weight_max": 68.08578491210938,
      "activations/layer19_attention_weight_min": -33.888694763183594,
      "activations/layer1_attention_weight_max": 17.582950592041016,
      "activations/layer1_attention_weight_min": -11.0701265335083,
      "activations/layer20_attention_weight_max": 44.18230438232422,
      "activations/layer20_attention_weight_min": -24.621004104614258,
      "activations/layer21_attention_weight_max": 32.471946716308594,
      "activations/layer21_attention_weight_min": -17.660762786865234,
      "activations/layer22_attention_weight_max": 50.571868896484375,
      "activations/layer22_attention_weight_min": -29.269243240356445,
      "activations/layer23_attention_weight_max": 29.324344635009766,
      "activations/layer23_attention_weight_min": -17.23424530029297,
      "activations/layer2_attention_weight_max": 29.10814666748047,
      "activations/layer2_attention_weight_min": -27.233327865600586,
      "activations/layer3_attention_weight_max": 74.77155303955078,
      "activations/layer3_attention_weight_min": -71.70504760742188,
      "activations/layer4_attention_weight_max": 133.0649871826172,
      "activations/layer4_attention_weight_min": -122.04328918457031,
      "activations/layer5_attention_weight_max": 123.56319427490234,
      "activations/layer5_attention_weight_min": -99.69880676269531,
      "activations/layer6_attention_weight_max": 66.6091537475586,
      "activations/layer6_attention_weight_min": -53.53078842163086,
      "activations/layer7_attention_weight_max": 74.11478424072266,
      "activations/layer7_attention_weight_min": -64.77825927734375,
      "activations/layer8_attention_weight_max": 57.14371109008789,
      "activations/layer8_attention_weight_min": -41.406532287597656,
      "activations/layer9_attention_weight_max": 69.75225067138672,
      "activations/layer9_attention_weight_min": -60.600372314453125,
      "epoch": 1.94,
      "learning_rate": 0.00013886174242424242,
      "loss": 2.9556,
      "step": 33450
    },
    {
      "activations/layer0_attention_weight_max": 16.236255645751953,
      "activations/layer0_attention_weight_min": -11.959794998168945,
      "activations/layer10_attention_weight_max": 49.97587585449219,
      "activations/layer10_attention_weight_min": -47.358421325683594,
      "activations/layer11_attention_weight_max": 40.156341552734375,
      "activations/layer11_attention_weight_min": -34.00382995605469,
      "activations/layer12_attention_weight_max": 36.658748626708984,
      "activations/layer12_attention_weight_min": -30.6169376373291,
      "activations/layer13_attention_weight_max": 47.461769104003906,
      "activations/layer13_attention_weight_min": -37.16720199584961,
      "activations/layer14_attention_weight_max": 74.2097396850586,
      "activations/layer14_attention_weight_min": -58.77065658569336,
      "activations/layer15_attention_weight_max": 50.665592193603516,
      "activations/layer15_attention_weight_min": -39.077003479003906,
      "activations/layer16_attention_weight_max": 55.05484390258789,
      "activations/layer16_attention_weight_min": -36.04104995727539,
      "activations/layer17_attention_weight_max": 52.40653991699219,
      "activations/layer17_attention_weight_min": -30.103633880615234,
      "activations/layer18_attention_weight_max": 28.079345703125,
      "activations/layer18_attention_weight_min": -18.010526657104492,
      "activations/layer19_attention_weight_max": 51.895870208740234,
      "activations/layer19_attention_weight_min": -34.24764633178711,
      "activations/layer1_attention_weight_max": 18.14790916442871,
      "activations/layer1_attention_weight_min": -10.411075592041016,
      "activations/layer20_attention_weight_max": 41.22028350830078,
      "activations/layer20_attention_weight_min": -26.926774978637695,
      "activations/layer21_attention_weight_max": 32.59374237060547,
      "activations/layer21_attention_weight_min": -17.83452606201172,
      "activations/layer22_attention_weight_max": 44.274688720703125,
      "activations/layer22_attention_weight_min": -25.226499557495117,
      "activations/layer23_attention_weight_max": 24.62579917907715,
      "activations/layer23_attention_weight_min": -15.357359886169434,
      "activations/layer2_attention_weight_max": 27.47595977783203,
      "activations/layer2_attention_weight_min": -27.09671401977539,
      "activations/layer3_attention_weight_max": 68.99179077148438,
      "activations/layer3_attention_weight_min": -74.62206268310547,
      "activations/layer4_attention_weight_max": 118.1146240234375,
      "activations/layer4_attention_weight_min": -115.62303924560547,
      "activations/layer5_attention_weight_max": 106.88241577148438,
      "activations/layer5_attention_weight_min": -94.86178588867188,
      "activations/layer6_attention_weight_max": 57.64454650878906,
      "activations/layer6_attention_weight_min": -52.87366485595703,
      "activations/layer7_attention_weight_max": 64.48363494873047,
      "activations/layer7_attention_weight_min": -62.24653625488281,
      "activations/layer8_attention_weight_max": 49.412925720214844,
      "activations/layer8_attention_weight_min": -40.36769485473633,
      "activations/layer9_attention_weight_max": 61.863094329833984,
      "activations/layer9_attention_weight_min": -56.116798400878906,
      "epoch": 1.95,
      "learning_rate": 0.00013884280303030303,
      "loss": 2.9837,
      "step": 33500
    },
    {
      "activations/layer0_attention_weight_max": 16.249805450439453,
      "activations/layer0_attention_weight_min": -11.652960777282715,
      "activations/layer10_attention_weight_max": 53.35175704956055,
      "activations/layer10_attention_weight_min": -45.07685470581055,
      "activations/layer11_attention_weight_max": 46.92554473876953,
      "activations/layer11_attention_weight_min": -35.51776123046875,
      "activations/layer12_attention_weight_max": 68.36961364746094,
      "activations/layer12_attention_weight_min": -39.80582046508789,
      "activations/layer13_attention_weight_max": 50.613136291503906,
      "activations/layer13_attention_weight_min": -38.36696243286133,
      "activations/layer14_attention_weight_max": 73.80341339111328,
      "activations/layer14_attention_weight_min": -61.74782180786133,
      "activations/layer15_attention_weight_max": 46.8799934387207,
      "activations/layer15_attention_weight_min": -35.52150344848633,
      "activations/layer16_attention_weight_max": 56.719024658203125,
      "activations/layer16_attention_weight_min": -36.16130828857422,
      "activations/layer17_attention_weight_max": 44.73557662963867,
      "activations/layer17_attention_weight_min": -29.761396408081055,
      "activations/layer18_attention_weight_max": 26.733732223510742,
      "activations/layer18_attention_weight_min": -17.29061508178711,
      "activations/layer19_attention_weight_max": 49.908470153808594,
      "activations/layer19_attention_weight_min": -31.505603790283203,
      "activations/layer1_attention_weight_max": 18.0152587890625,
      "activations/layer1_attention_weight_min": -11.362813949584961,
      "activations/layer20_attention_weight_max": 36.35696792602539,
      "activations/layer20_attention_weight_min": -25.70473289489746,
      "activations/layer21_attention_weight_max": 31.77081871032715,
      "activations/layer21_attention_weight_min": -19.241655349731445,
      "activations/layer22_attention_weight_max": 46.0079460144043,
      "activations/layer22_attention_weight_min": -24.67209815979004,
      "activations/layer23_attention_weight_max": 25.504833221435547,
      "activations/layer23_attention_weight_min": -17.160232543945312,
      "activations/layer2_attention_weight_max": 28.54543113708496,
      "activations/layer2_attention_weight_min": -28.610328674316406,
      "activations/layer3_attention_weight_max": 71.3472671508789,
      "activations/layer3_attention_weight_min": -74.26424407958984,
      "activations/layer4_attention_weight_max": 127.69013977050781,
      "activations/layer4_attention_weight_min": -117.00106048583984,
      "activations/layer5_attention_weight_max": 113.20246887207031,
      "activations/layer5_attention_weight_min": -97.02828216552734,
      "activations/layer6_attention_weight_max": 62.32599639892578,
      "activations/layer6_attention_weight_min": -54.5885124206543,
      "activations/layer7_attention_weight_max": 66.2849349975586,
      "activations/layer7_attention_weight_min": -61.74029541015625,
      "activations/layer8_attention_weight_max": 50.97517013549805,
      "activations/layer8_attention_weight_min": -42.14148712158203,
      "activations/layer9_attention_weight_max": 73.1475601196289,
      "activations/layer9_attention_weight_min": -54.821533203125,
      "epoch": 1.95,
      "learning_rate": 0.00013882386363636362,
      "loss": 2.9674,
      "step": 33550
    },
    {
      "activations/layer0_attention_weight_max": 16.424795150756836,
      "activations/layer0_attention_weight_min": -11.21950626373291,
      "activations/layer10_attention_weight_max": 53.76154708862305,
      "activations/layer10_attention_weight_min": -42.63603591918945,
      "activations/layer11_attention_weight_max": 41.21318054199219,
      "activations/layer11_attention_weight_min": -39.18321990966797,
      "activations/layer12_attention_weight_max": 39.90174102783203,
      "activations/layer12_attention_weight_min": -28.09800910949707,
      "activations/layer13_attention_weight_max": 45.38724899291992,
      "activations/layer13_attention_weight_min": -34.450462341308594,
      "activations/layer14_attention_weight_max": 72.25603485107422,
      "activations/layer14_attention_weight_min": -53.14241409301758,
      "activations/layer15_attention_weight_max": 53.30636978149414,
      "activations/layer15_attention_weight_min": -38.46529769897461,
      "activations/layer16_attention_weight_max": 48.5399169921875,
      "activations/layer16_attention_weight_min": -39.39152145385742,
      "activations/layer17_attention_weight_max": 43.82900619506836,
      "activations/layer17_attention_weight_min": -30.40235710144043,
      "activations/layer18_attention_weight_max": 23.176700592041016,
      "activations/layer18_attention_weight_min": -17.43331527709961,
      "activations/layer19_attention_weight_max": 56.59491729736328,
      "activations/layer19_attention_weight_min": -33.837982177734375,
      "activations/layer1_attention_weight_max": 18.080141067504883,
      "activations/layer1_attention_weight_min": -11.362822532653809,
      "activations/layer20_attention_weight_max": 35.78696060180664,
      "activations/layer20_attention_weight_min": -25.534582138061523,
      "activations/layer21_attention_weight_max": 29.77145004272461,
      "activations/layer21_attention_weight_min": -17.673662185668945,
      "activations/layer22_attention_weight_max": 39.601348876953125,
      "activations/layer22_attention_weight_min": -28.021833419799805,
      "activations/layer23_attention_weight_max": 22.10788345336914,
      "activations/layer23_attention_weight_min": -15.863937377929688,
      "activations/layer2_attention_weight_max": 28.282020568847656,
      "activations/layer2_attention_weight_min": -27.98208236694336,
      "activations/layer3_attention_weight_max": 67.49797058105469,
      "activations/layer3_attention_weight_min": -74.5567626953125,
      "activations/layer4_attention_weight_max": 128.23194885253906,
      "activations/layer4_attention_weight_min": -118.43924713134766,
      "activations/layer5_attention_weight_max": 112.81956481933594,
      "activations/layer5_attention_weight_min": -96.08526611328125,
      "activations/layer6_attention_weight_max": 60.036563873291016,
      "activations/layer6_attention_weight_min": -50.88984298706055,
      "activations/layer7_attention_weight_max": 65.04498291015625,
      "activations/layer7_attention_weight_min": -59.372135162353516,
      "activations/layer8_attention_weight_max": 49.463523864746094,
      "activations/layer8_attention_weight_min": -40.806095123291016,
      "activations/layer9_attention_weight_max": 59.63220977783203,
      "activations/layer9_attention_weight_min": -59.0600471496582,
      "epoch": 1.95,
      "learning_rate": 0.000138805303030303,
      "loss": 2.9553,
      "step": 33600
    },
    {
      "activations/layer0_attention_weight_max": 16.542631149291992,
      "activations/layer0_attention_weight_min": -11.67371940612793,
      "activations/layer10_attention_weight_max": 48.75723648071289,
      "activations/layer10_attention_weight_min": -44.326202392578125,
      "activations/layer11_attention_weight_max": 36.08113479614258,
      "activations/layer11_attention_weight_min": -31.660085678100586,
      "activations/layer12_attention_weight_max": 37.806610107421875,
      "activations/layer12_attention_weight_min": -25.336917877197266,
      "activations/layer13_attention_weight_max": 48.15050506591797,
      "activations/layer13_attention_weight_min": -38.14306640625,
      "activations/layer14_attention_weight_max": 83.2247543334961,
      "activations/layer14_attention_weight_min": -62.621543884277344,
      "activations/layer15_attention_weight_max": 49.98276901245117,
      "activations/layer15_attention_weight_min": -35.42609405517578,
      "activations/layer16_attention_weight_max": 48.04454040527344,
      "activations/layer16_attention_weight_min": -35.19445037841797,
      "activations/layer17_attention_weight_max": 46.39650344848633,
      "activations/layer17_attention_weight_min": -28.87843894958496,
      "activations/layer18_attention_weight_max": 28.733713150024414,
      "activations/layer18_attention_weight_min": -17.1822452545166,
      "activations/layer19_attention_weight_max": 46.80247497558594,
      "activations/layer19_attention_weight_min": -34.03651428222656,
      "activations/layer1_attention_weight_max": 17.536922454833984,
      "activations/layer1_attention_weight_min": -11.085532188415527,
      "activations/layer20_attention_weight_max": 33.571468353271484,
      "activations/layer20_attention_weight_min": -22.797714233398438,
      "activations/layer21_attention_weight_max": 33.1406135559082,
      "activations/layer21_attention_weight_min": -21.0411319732666,
      "activations/layer22_attention_weight_max": 44.12625503540039,
      "activations/layer22_attention_weight_min": -27.8015193939209,
      "activations/layer23_attention_weight_max": 23.99616050720215,
      "activations/layer23_attention_weight_min": -16.78829002380371,
      "activations/layer2_attention_weight_max": 27.56178092956543,
      "activations/layer2_attention_weight_min": -26.64410972595215,
      "activations/layer3_attention_weight_max": 66.51988220214844,
      "activations/layer3_attention_weight_min": -70.03793334960938,
      "activations/layer4_attention_weight_max": 123.47835540771484,
      "activations/layer4_attention_weight_min": -112.79957580566406,
      "activations/layer5_attention_weight_max": 113.59934997558594,
      "activations/layer5_attention_weight_min": -96.92529296875,
      "activations/layer6_attention_weight_max": 59.345279693603516,
      "activations/layer6_attention_weight_min": -52.195491790771484,
      "activations/layer7_attention_weight_max": 64.56755828857422,
      "activations/layer7_attention_weight_min": -59.958377838134766,
      "activations/layer8_attention_weight_max": 46.93840789794922,
      "activations/layer8_attention_weight_min": -38.473575592041016,
      "activations/layer9_attention_weight_max": 68.42642211914062,
      "activations/layer9_attention_weight_min": -54.480804443359375,
      "epoch": 1.96,
      "learning_rate": 0.00013878636363636362,
      "loss": 2.9656,
      "step": 33650
    },
    {
      "activations/layer0_attention_weight_max": 17.14529800415039,
      "activations/layer0_attention_weight_min": -11.449239730834961,
      "activations/layer10_attention_weight_max": 47.34878921508789,
      "activations/layer10_attention_weight_min": -42.37593078613281,
      "activations/layer11_attention_weight_max": 38.46492004394531,
      "activations/layer11_attention_weight_min": -33.53495407104492,
      "activations/layer12_attention_weight_max": 38.32270812988281,
      "activations/layer12_attention_weight_min": -34.41438674926758,
      "activations/layer13_attention_weight_max": 46.8540153503418,
      "activations/layer13_attention_weight_min": -40.47090148925781,
      "activations/layer14_attention_weight_max": 73.16425323486328,
      "activations/layer14_attention_weight_min": -60.33488845825195,
      "activations/layer15_attention_weight_max": 46.74372863769531,
      "activations/layer15_attention_weight_min": -34.43814468383789,
      "activations/layer16_attention_weight_max": 53.017730712890625,
      "activations/layer16_attention_weight_min": -38.057701110839844,
      "activations/layer17_attention_weight_max": 43.423492431640625,
      "activations/layer17_attention_weight_min": -26.638200759887695,
      "activations/layer18_attention_weight_max": 26.487552642822266,
      "activations/layer18_attention_weight_min": -20.079626083374023,
      "activations/layer19_attention_weight_max": 50.91912841796875,
      "activations/layer19_attention_weight_min": -31.682714462280273,
      "activations/layer1_attention_weight_max": 18.195491790771484,
      "activations/layer1_attention_weight_min": -12.008706092834473,
      "activations/layer20_attention_weight_max": 43.60305404663086,
      "activations/layer20_attention_weight_min": -24.641277313232422,
      "activations/layer21_attention_weight_max": 41.58842849731445,
      "activations/layer21_attention_weight_min": -24.214696884155273,
      "activations/layer22_attention_weight_max": 46.380615234375,
      "activations/layer22_attention_weight_min": -30.241546630859375,
      "activations/layer23_attention_weight_max": 28.262426376342773,
      "activations/layer23_attention_weight_min": -15.456859588623047,
      "activations/layer2_attention_weight_max": 29.22940444946289,
      "activations/layer2_attention_weight_min": -27.259201049804688,
      "activations/layer3_attention_weight_max": 70.74445343017578,
      "activations/layer3_attention_weight_min": -72.03059387207031,
      "activations/layer4_attention_weight_max": 125.63545227050781,
      "activations/layer4_attention_weight_min": -117.97406005859375,
      "activations/layer5_attention_weight_max": 115.15509033203125,
      "activations/layer5_attention_weight_min": -99.53158569335938,
      "activations/layer6_attention_weight_max": 63.090415954589844,
      "activations/layer6_attention_weight_min": -52.521934509277344,
      "activations/layer7_attention_weight_max": 65.47689056396484,
      "activations/layer7_attention_weight_min": -61.592002868652344,
      "activations/layer8_attention_weight_max": 44.775211334228516,
      "activations/layer8_attention_weight_min": -41.17405319213867,
      "activations/layer9_attention_weight_max": 69.9820556640625,
      "activations/layer9_attention_weight_min": -61.74858474731445,
      "epoch": 1.96,
      "learning_rate": 0.00013876742424242424,
      "loss": 2.9815,
      "step": 33700
    },
    {
      "activations/layer0_attention_weight_max": 16.389650344848633,
      "activations/layer0_attention_weight_min": -11.421072959899902,
      "activations/layer10_attention_weight_max": 52.500484466552734,
      "activations/layer10_attention_weight_min": -45.349998474121094,
      "activations/layer11_attention_weight_max": 45.73924255371094,
      "activations/layer11_attention_weight_min": -33.78611755371094,
      "activations/layer12_attention_weight_max": 60.06731033325195,
      "activations/layer12_attention_weight_min": -30.641250610351562,
      "activations/layer13_attention_weight_max": 50.06486511230469,
      "activations/layer13_attention_weight_min": -38.998619079589844,
      "activations/layer14_attention_weight_max": 69.08098602294922,
      "activations/layer14_attention_weight_min": -60.591251373291016,
      "activations/layer15_attention_weight_max": 50.2804069519043,
      "activations/layer15_attention_weight_min": -37.999149322509766,
      "activations/layer16_attention_weight_max": 53.1069221496582,
      "activations/layer16_attention_weight_min": -36.96875,
      "activations/layer17_attention_weight_max": 44.36298751831055,
      "activations/layer17_attention_weight_min": -28.601911544799805,
      "activations/layer18_attention_weight_max": 26.581764221191406,
      "activations/layer18_attention_weight_min": -17.786170959472656,
      "activations/layer19_attention_weight_max": 45.27998352050781,
      "activations/layer19_attention_weight_min": -33.11960220336914,
      "activations/layer1_attention_weight_max": 19.076902389526367,
      "activations/layer1_attention_weight_min": -11.329785346984863,
      "activations/layer20_attention_weight_max": 36.29724884033203,
      "activations/layer20_attention_weight_min": -23.90410614013672,
      "activations/layer21_attention_weight_max": 27.434646606445312,
      "activations/layer21_attention_weight_min": -19.742046356201172,
      "activations/layer22_attention_weight_max": 36.6455078125,
      "activations/layer22_attention_weight_min": -21.535295486450195,
      "activations/layer23_attention_weight_max": 22.570587158203125,
      "activations/layer23_attention_weight_min": -15.327803611755371,
      "activations/layer2_attention_weight_max": 28.60938835144043,
      "activations/layer2_attention_weight_min": -27.66339683532715,
      "activations/layer3_attention_weight_max": 67.76422119140625,
      "activations/layer3_attention_weight_min": -71.3106460571289,
      "activations/layer4_attention_weight_max": 125.92842102050781,
      "activations/layer4_attention_weight_min": -113.8662109375,
      "activations/layer5_attention_weight_max": 111.38775634765625,
      "activations/layer5_attention_weight_min": -94.6971664428711,
      "activations/layer6_attention_weight_max": 60.497013092041016,
      "activations/layer6_attention_weight_min": -51.99077224731445,
      "activations/layer7_attention_weight_max": 66.57978820800781,
      "activations/layer7_attention_weight_min": -62.30904006958008,
      "activations/layer8_attention_weight_max": 47.72282028198242,
      "activations/layer8_attention_weight_min": -40.00795364379883,
      "activations/layer9_attention_weight_max": 78.66869354248047,
      "activations/layer9_attention_weight_min": -61.07367706298828,
      "epoch": 1.96,
      "learning_rate": 0.00013874848484848485,
      "loss": 2.9821,
      "step": 33750
    },
    {
      "activations/layer0_attention_weight_max": 16.25804901123047,
      "activations/layer0_attention_weight_min": -11.336577415466309,
      "activations/layer10_attention_weight_max": 72.11261749267578,
      "activations/layer10_attention_weight_min": -48.240169525146484,
      "activations/layer11_attention_weight_max": 62.63806915283203,
      "activations/layer11_attention_weight_min": -36.77704620361328,
      "activations/layer12_attention_weight_max": 110.9596939086914,
      "activations/layer12_attention_weight_min": -45.82368469238281,
      "activations/layer13_attention_weight_max": 87.22521209716797,
      "activations/layer13_attention_weight_min": -44.55739212036133,
      "activations/layer14_attention_weight_max": 77.78244018554688,
      "activations/layer14_attention_weight_min": -55.50541305541992,
      "activations/layer15_attention_weight_max": 54.85333251953125,
      "activations/layer15_attention_weight_min": -36.82201385498047,
      "activations/layer16_attention_weight_max": 48.701202392578125,
      "activations/layer16_attention_weight_min": -34.410491943359375,
      "activations/layer17_attention_weight_max": 45.82666015625,
      "activations/layer17_attention_weight_min": -31.217329025268555,
      "activations/layer18_attention_weight_max": 26.743444442749023,
      "activations/layer18_attention_weight_min": -17.000619888305664,
      "activations/layer19_attention_weight_max": 54.2529182434082,
      "activations/layer19_attention_weight_min": -30.16011619567871,
      "activations/layer1_attention_weight_max": 18.021930694580078,
      "activations/layer1_attention_weight_min": -10.642178535461426,
      "activations/layer20_attention_weight_max": 36.48954772949219,
      "activations/layer20_attention_weight_min": -24.449045181274414,
      "activations/layer21_attention_weight_max": 25.024131774902344,
      "activations/layer21_attention_weight_min": -18.275482177734375,
      "activations/layer22_attention_weight_max": 44.30696487426758,
      "activations/layer22_attention_weight_min": -26.5693302154541,
      "activations/layer23_attention_weight_max": 24.0278377532959,
      "activations/layer23_attention_weight_min": -14.592791557312012,
      "activations/layer2_attention_weight_max": 27.899234771728516,
      "activations/layer2_attention_weight_min": -28.73675537109375,
      "activations/layer3_attention_weight_max": 70.39017486572266,
      "activations/layer3_attention_weight_min": -72.8062744140625,
      "activations/layer4_attention_weight_max": 130.7312469482422,
      "activations/layer4_attention_weight_min": -120.0124282836914,
      "activations/layer5_attention_weight_max": 126.73589324951172,
      "activations/layer5_attention_weight_min": -104.77943420410156,
      "activations/layer6_attention_weight_max": 71.75125885009766,
      "activations/layer6_attention_weight_min": -55.772159576416016,
      "activations/layer7_attention_weight_max": 81.36051177978516,
      "activations/layer7_attention_weight_min": -66.56865692138672,
      "activations/layer8_attention_weight_max": 66.68408966064453,
      "activations/layer8_attention_weight_min": -46.14752960205078,
      "activations/layer9_attention_weight_max": 109.70527648925781,
      "activations/layer9_attention_weight_min": -66.28156280517578,
      "epoch": 1.96,
      "learning_rate": 0.00013872954545454544,
      "loss": 2.9582,
      "step": 33800
    },
    {
      "activations/layer0_attention_weight_max": 16.207637786865234,
      "activations/layer0_attention_weight_min": -11.717937469482422,
      "activations/layer10_attention_weight_max": 58.44251251220703,
      "activations/layer10_attention_weight_min": -47.21958923339844,
      "activations/layer11_attention_weight_max": 41.35403060913086,
      "activations/layer11_attention_weight_min": -35.71760559082031,
      "activations/layer12_attention_weight_max": 30.977537155151367,
      "activations/layer12_attention_weight_min": -27.242868423461914,
      "activations/layer13_attention_weight_max": 47.77753448486328,
      "activations/layer13_attention_weight_min": -37.19784164428711,
      "activations/layer14_attention_weight_max": 78.66468811035156,
      "activations/layer14_attention_weight_min": -59.928680419921875,
      "activations/layer15_attention_weight_max": 59.35165786743164,
      "activations/layer15_attention_weight_min": -39.232635498046875,
      "activations/layer16_attention_weight_max": 50.35806655883789,
      "activations/layer16_attention_weight_min": -40.10723114013672,
      "activations/layer17_attention_weight_max": 47.31886672973633,
      "activations/layer17_attention_weight_min": -28.354055404663086,
      "activations/layer18_attention_weight_max": 25.534826278686523,
      "activations/layer18_attention_weight_min": -18.06443214416504,
      "activations/layer19_attention_weight_max": 56.65969467163086,
      "activations/layer19_attention_weight_min": -31.09943962097168,
      "activations/layer1_attention_weight_max": 17.108339309692383,
      "activations/layer1_attention_weight_min": -12.386792182922363,
      "activations/layer20_attention_weight_max": 33.87273025512695,
      "activations/layer20_attention_weight_min": -23.356033325195312,
      "activations/layer21_attention_weight_max": 27.060863494873047,
      "activations/layer21_attention_weight_min": -21.569557189941406,
      "activations/layer22_attention_weight_max": 42.15171432495117,
      "activations/layer22_attention_weight_min": -24.180755615234375,
      "activations/layer23_attention_weight_max": 24.14215850830078,
      "activations/layer23_attention_weight_min": -16.20977783203125,
      "activations/layer2_attention_weight_max": 30.641706466674805,
      "activations/layer2_attention_weight_min": -28.947782516479492,
      "activations/layer3_attention_weight_max": 74.3954086303711,
      "activations/layer3_attention_weight_min": -76.64509582519531,
      "activations/layer4_attention_weight_max": 133.90878295898438,
      "activations/layer4_attention_weight_min": -123.85570526123047,
      "activations/layer5_attention_weight_max": 120.20345306396484,
      "activations/layer5_attention_weight_min": -102.33778381347656,
      "activations/layer6_attention_weight_max": 64.7361068725586,
      "activations/layer6_attention_weight_min": -53.962162017822266,
      "activations/layer7_attention_weight_max": 65.50739288330078,
      "activations/layer7_attention_weight_min": -65.2249526977539,
      "activations/layer8_attention_weight_max": 45.56836700439453,
      "activations/layer8_attention_weight_min": -42.81935501098633,
      "activations/layer9_attention_weight_max": 68.59896087646484,
      "activations/layer9_attention_weight_min": -51.87300491333008,
      "epoch": 1.97,
      "learning_rate": 0.00013871060606060606,
      "loss": 2.9784,
      "step": 33850
    },
    {
      "activations/layer0_attention_weight_max": 16.218584060668945,
      "activations/layer0_attention_weight_min": -11.823051452636719,
      "activations/layer10_attention_weight_max": 49.45960235595703,
      "activations/layer10_attention_weight_min": -40.536285400390625,
      "activations/layer11_attention_weight_max": 35.652503967285156,
      "activations/layer11_attention_weight_min": -34.19045639038086,
      "activations/layer12_attention_weight_max": 35.33583450317383,
      "activations/layer12_attention_weight_min": -28.55341911315918,
      "activations/layer13_attention_weight_max": 45.75223922729492,
      "activations/layer13_attention_weight_min": -38.98562240600586,
      "activations/layer14_attention_weight_max": 66.2718276977539,
      "activations/layer14_attention_weight_min": -54.80080795288086,
      "activations/layer15_attention_weight_max": 48.78547668457031,
      "activations/layer15_attention_weight_min": -37.392879486083984,
      "activations/layer16_attention_weight_max": 44.96430206298828,
      "activations/layer16_attention_weight_min": -36.593017578125,
      "activations/layer17_attention_weight_max": 43.61797332763672,
      "activations/layer17_attention_weight_min": -29.327510833740234,
      "activations/layer18_attention_weight_max": 24.15227699279785,
      "activations/layer18_attention_weight_min": -16.808609008789062,
      "activations/layer19_attention_weight_max": 51.876434326171875,
      "activations/layer19_attention_weight_min": -31.87497901916504,
      "activations/layer1_attention_weight_max": 17.483993530273438,
      "activations/layer1_attention_weight_min": -12.145499229431152,
      "activations/layer20_attention_weight_max": 34.32083511352539,
      "activations/layer20_attention_weight_min": -23.246692657470703,
      "activations/layer21_attention_weight_max": 30.804088592529297,
      "activations/layer21_attention_weight_min": -20.430192947387695,
      "activations/layer22_attention_weight_max": 44.52308654785156,
      "activations/layer22_attention_weight_min": -24.803112030029297,
      "activations/layer23_attention_weight_max": 23.892906188964844,
      "activations/layer23_attention_weight_min": -15.47385025024414,
      "activations/layer2_attention_weight_max": 28.386281967163086,
      "activations/layer2_attention_weight_min": -27.099279403686523,
      "activations/layer3_attention_weight_max": 69.81413269042969,
      "activations/layer3_attention_weight_min": -72.51415252685547,
      "activations/layer4_attention_weight_max": 120.72755432128906,
      "activations/layer4_attention_weight_min": -116.49333953857422,
      "activations/layer5_attention_weight_max": 109.30498504638672,
      "activations/layer5_attention_weight_min": -98.50749969482422,
      "activations/layer6_attention_weight_max": 60.53544616699219,
      "activations/layer6_attention_weight_min": -54.45027542114258,
      "activations/layer7_attention_weight_max": 63.41869354248047,
      "activations/layer7_attention_weight_min": -63.90917205810547,
      "activations/layer8_attention_weight_max": 46.91671371459961,
      "activations/layer8_attention_weight_min": -42.05278015136719,
      "activations/layer9_attention_weight_max": 68.32874298095703,
      "activations/layer9_attention_weight_min": -56.2547607421875,
      "epoch": 1.97,
      "learning_rate": 0.00013869204545454545,
      "loss": 2.9831,
      "step": 33900
    },
    {
      "activations/layer0_attention_weight_max": 15.3123197555542,
      "activations/layer0_attention_weight_min": -12.01297664642334,
      "activations/layer10_attention_weight_max": 46.16905212402344,
      "activations/layer10_attention_weight_min": -37.94050598144531,
      "activations/layer11_attention_weight_max": 37.95345687866211,
      "activations/layer11_attention_weight_min": -33.760528564453125,
      "activations/layer12_attention_weight_max": 30.61159896850586,
      "activations/layer12_attention_weight_min": -33.17473602294922,
      "activations/layer13_attention_weight_max": 48.8120002746582,
      "activations/layer13_attention_weight_min": -36.419830322265625,
      "activations/layer14_attention_weight_max": 73.6255111694336,
      "activations/layer14_attention_weight_min": -60.95320129394531,
      "activations/layer15_attention_weight_max": 54.72426223754883,
      "activations/layer15_attention_weight_min": -40.421058654785156,
      "activations/layer16_attention_weight_max": 53.09848403930664,
      "activations/layer16_attention_weight_min": -36.59824752807617,
      "activations/layer17_attention_weight_max": 52.04404830932617,
      "activations/layer17_attention_weight_min": -35.833282470703125,
      "activations/layer18_attention_weight_max": 25.765562057495117,
      "activations/layer18_attention_weight_min": -19.003402709960938,
      "activations/layer19_attention_weight_max": 52.60211944580078,
      "activations/layer19_attention_weight_min": -33.449527740478516,
      "activations/layer1_attention_weight_max": 17.887638092041016,
      "activations/layer1_attention_weight_min": -11.976037979125977,
      "activations/layer20_attention_weight_max": 40.177001953125,
      "activations/layer20_attention_weight_min": -25.2467041015625,
      "activations/layer21_attention_weight_max": 38.08550262451172,
      "activations/layer21_attention_weight_min": -23.607852935791016,
      "activations/layer22_attention_weight_max": 48.82173538208008,
      "activations/layer22_attention_weight_min": -27.346208572387695,
      "activations/layer23_attention_weight_max": 27.61555290222168,
      "activations/layer23_attention_weight_min": -16.764911651611328,
      "activations/layer2_attention_weight_max": 29.251203536987305,
      "activations/layer2_attention_weight_min": -28.462081909179688,
      "activations/layer3_attention_weight_max": 69.59822082519531,
      "activations/layer3_attention_weight_min": -70.99661254882812,
      "activations/layer4_attention_weight_max": 124.30950927734375,
      "activations/layer4_attention_weight_min": -114.48341369628906,
      "activations/layer5_attention_weight_max": 115.03623962402344,
      "activations/layer5_attention_weight_min": -98.46459197998047,
      "activations/layer6_attention_weight_max": 60.66298294067383,
      "activations/layer6_attention_weight_min": -50.84873962402344,
      "activations/layer7_attention_weight_max": 66.44452667236328,
      "activations/layer7_attention_weight_min": -62.64403533935547,
      "activations/layer8_attention_weight_max": 44.493587493896484,
      "activations/layer8_attention_weight_min": -39.7626838684082,
      "activations/layer9_attention_weight_max": 62.14349365234375,
      "activations/layer9_attention_weight_min": -50.76199722290039,
      "epoch": 1.97,
      "learning_rate": 0.00013867310606060606,
      "loss": 2.973,
      "step": 33950
    },
    {
      "activations/layer0_attention_weight_max": 15.170071601867676,
      "activations/layer0_attention_weight_min": -11.718360900878906,
      "activations/layer10_attention_weight_max": 58.196739196777344,
      "activations/layer10_attention_weight_min": -47.951229095458984,
      "activations/layer11_attention_weight_max": 52.36611557006836,
      "activations/layer11_attention_weight_min": -37.33161926269531,
      "activations/layer12_attention_weight_max": 60.65415954589844,
      "activations/layer12_attention_weight_min": -34.998451232910156,
      "activations/layer13_attention_weight_max": 77.12113952636719,
      "activations/layer13_attention_weight_min": -55.63071060180664,
      "activations/layer14_attention_weight_max": 85.1502685546875,
      "activations/layer14_attention_weight_min": -61.78407669067383,
      "activations/layer15_attention_weight_max": 75.30066680908203,
      "activations/layer15_attention_weight_min": -42.33881759643555,
      "activations/layer16_attention_weight_max": 52.96542739868164,
      "activations/layer16_attention_weight_min": -35.70603942871094,
      "activations/layer17_attention_weight_max": 42.51640701293945,
      "activations/layer17_attention_weight_min": -29.115447998046875,
      "activations/layer18_attention_weight_max": 37.09849166870117,
      "activations/layer18_attention_weight_min": -20.49026870727539,
      "activations/layer19_attention_weight_max": 44.500545501708984,
      "activations/layer19_attention_weight_min": -28.869775772094727,
      "activations/layer1_attention_weight_max": 18.613113403320312,
      "activations/layer1_attention_weight_min": -11.79046630859375,
      "activations/layer20_attention_weight_max": 38.039306640625,
      "activations/layer20_attention_weight_min": -22.549331665039062,
      "activations/layer21_attention_weight_max": 35.13675308227539,
      "activations/layer21_attention_weight_min": -21.428653717041016,
      "activations/layer22_attention_weight_max": 40.536678314208984,
      "activations/layer22_attention_weight_min": -25.161659240722656,
      "activations/layer23_attention_weight_max": 25.89632797241211,
      "activations/layer23_attention_weight_min": -15.949458122253418,
      "activations/layer2_attention_weight_max": 28.334239959716797,
      "activations/layer2_attention_weight_min": -28.702529907226562,
      "activations/layer3_attention_weight_max": 73.47386169433594,
      "activations/layer3_attention_weight_min": -76.70484161376953,
      "activations/layer4_attention_weight_max": 141.91421508789062,
      "activations/layer4_attention_weight_min": -119.52336883544922,
      "activations/layer5_attention_weight_max": 126.80186462402344,
      "activations/layer5_attention_weight_min": -100.48604583740234,
      "activations/layer6_attention_weight_max": 67.72747039794922,
      "activations/layer6_attention_weight_min": -55.930076599121094,
      "activations/layer7_attention_weight_max": 72.47856140136719,
      "activations/layer7_attention_weight_min": -70.96231842041016,
      "activations/layer8_attention_weight_max": 57.28783416748047,
      "activations/layer8_attention_weight_min": -48.076019287109375,
      "activations/layer9_attention_weight_max": 79.17774963378906,
      "activations/layer9_attention_weight_min": -64.1773452758789,
      "epoch": 1.98,
      "learning_rate": 0.00013865416666666665,
      "loss": 2.962,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_loss": 2.916015625,
      "eval_runtime": 8.5212,
      "eval_samples_per_second": 503.919,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_openwebtext_loss": 2.916015625,
      "eval_openwebtext_ppl": 18.467558995986426,
      "eval_openwebtext_runtime": 8.5212,
      "eval_openwebtext_samples_per_second": 503.919,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_wikitext_loss": 3.185546875,
      "eval_wikitext_ppl": 24.18050850623376,
      "eval_wikitext_runtime": 2.0169,
      "eval_wikitext_samples_per_second": 226.09,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_lambada_loss": 3.064453125,
      "eval_lambada_ppl": 21.42274322042474,
      "eval_lambada_runtime": 9.5892,
      "eval_lambada_samples_per_second": 507.759,
      "step": 34000
    },
    {
      "activations/layer0_attention_weight_max": 16.86859893798828,
      "activations/layer0_attention_weight_min": -11.79470443725586,
      "activations/layer10_attention_weight_max": 53.83880615234375,
      "activations/layer10_attention_weight_min": -41.25068283081055,
      "activations/layer11_attention_weight_max": 44.92091369628906,
      "activations/layer11_attention_weight_min": -39.29016876220703,
      "activations/layer12_attention_weight_max": 35.08209991455078,
      "activations/layer12_attention_weight_min": -32.7096061706543,
      "activations/layer13_attention_weight_max": 49.21431350708008,
      "activations/layer13_attention_weight_min": -36.38185119628906,
      "activations/layer14_attention_weight_max": 76.86177062988281,
      "activations/layer14_attention_weight_min": -54.12916946411133,
      "activations/layer15_attention_weight_max": 58.389427185058594,
      "activations/layer15_attention_weight_min": -37.62602615356445,
      "activations/layer16_attention_weight_max": 58.595218658447266,
      "activations/layer16_attention_weight_min": -38.84541702270508,
      "activations/layer17_attention_weight_max": 44.177207946777344,
      "activations/layer17_attention_weight_min": -30.12791633605957,
      "activations/layer18_attention_weight_max": 26.773366928100586,
      "activations/layer18_attention_weight_min": -16.470102310180664,
      "activations/layer19_attention_weight_max": 58.6646842956543,
      "activations/layer19_attention_weight_min": -31.3856201171875,
      "activations/layer1_attention_weight_max": 17.387126922607422,
      "activations/layer1_attention_weight_min": -11.601161003112793,
      "activations/layer20_attention_weight_max": 41.46757125854492,
      "activations/layer20_attention_weight_min": -26.72798728942871,
      "activations/layer21_attention_weight_max": 34.8602294921875,
      "activations/layer21_attention_weight_min": -19.732379913330078,
      "activations/layer22_attention_weight_max": 46.18196487426758,
      "activations/layer22_attention_weight_min": -27.64169692993164,
      "activations/layer23_attention_weight_max": 24.052425384521484,
      "activations/layer23_attention_weight_min": -14.588584899902344,
      "activations/layer2_attention_weight_max": 28.224143981933594,
      "activations/layer2_attention_weight_min": -29.418691635131836,
      "activations/layer3_attention_weight_max": 73.72354125976562,
      "activations/layer3_attention_weight_min": -73.26611328125,
      "activations/layer4_attention_weight_max": 134.6111297607422,
      "activations/layer4_attention_weight_min": -119.02046203613281,
      "activations/layer5_attention_weight_max": 119.08007049560547,
      "activations/layer5_attention_weight_min": -99.59927368164062,
      "activations/layer6_attention_weight_max": 70.04705810546875,
      "activations/layer6_attention_weight_min": -55.15616226196289,
      "activations/layer7_attention_weight_max": 73.04488372802734,
      "activations/layer7_attention_weight_min": -65.94598388671875,
      "activations/layer8_attention_weight_max": 50.651771545410156,
      "activations/layer8_attention_weight_min": -42.21654510498047,
      "activations/layer9_attention_weight_max": 63.192298889160156,
      "activations/layer9_attention_weight_min": -61.7838020324707,
      "epoch": 1.98,
      "learning_rate": 0.00013863522727272727,
      "loss": 2.9901,
      "step": 34050
    },
    {
      "activations/layer0_attention_weight_max": 16.827194213867188,
      "activations/layer0_attention_weight_min": -11.69970989227295,
      "activations/layer10_attention_weight_max": 53.26869583129883,
      "activations/layer10_attention_weight_min": -45.4620475769043,
      "activations/layer11_attention_weight_max": 43.82514953613281,
      "activations/layer11_attention_weight_min": -37.51317596435547,
      "activations/layer12_attention_weight_max": 31.187049865722656,
      "activations/layer12_attention_weight_min": -25.400100708007812,
      "activations/layer13_attention_weight_max": 53.41081237792969,
      "activations/layer13_attention_weight_min": -40.444496154785156,
      "activations/layer14_attention_weight_max": 77.18677520751953,
      "activations/layer14_attention_weight_min": -56.56100845336914,
      "activations/layer15_attention_weight_max": 57.48643112182617,
      "activations/layer15_attention_weight_min": -36.66217041015625,
      "activations/layer16_attention_weight_max": 59.942138671875,
      "activations/layer16_attention_weight_min": -38.80962371826172,
      "activations/layer17_attention_weight_max": 44.34479904174805,
      "activations/layer17_attention_weight_min": -28.30617904663086,
      "activations/layer18_attention_weight_max": 26.661256790161133,
      "activations/layer18_attention_weight_min": -17.180923461914062,
      "activations/layer19_attention_weight_max": 64.29741668701172,
      "activations/layer19_attention_weight_min": -30.432373046875,
      "activations/layer1_attention_weight_max": 17.17154312133789,
      "activations/layer1_attention_weight_min": -11.261835098266602,
      "activations/layer20_attention_weight_max": 46.5976676940918,
      "activations/layer20_attention_weight_min": -24.58187484741211,
      "activations/layer21_attention_weight_max": 34.624298095703125,
      "activations/layer21_attention_weight_min": -20.152175903320312,
      "activations/layer22_attention_weight_max": 51.72809600830078,
      "activations/layer22_attention_weight_min": -26.14675521850586,
      "activations/layer23_attention_weight_max": 28.68267822265625,
      "activations/layer23_attention_weight_min": -17.002744674682617,
      "activations/layer2_attention_weight_max": 29.286197662353516,
      "activations/layer2_attention_weight_min": -27.75890350341797,
      "activations/layer3_attention_weight_max": 70.38333129882812,
      "activations/layer3_attention_weight_min": -72.56999969482422,
      "activations/layer4_attention_weight_max": 132.8046875,
      "activations/layer4_attention_weight_min": -122.02423095703125,
      "activations/layer5_attention_weight_max": 119.40168762207031,
      "activations/layer5_attention_weight_min": -102.72994232177734,
      "activations/layer6_attention_weight_max": 67.13448333740234,
      "activations/layer6_attention_weight_min": -54.77677536010742,
      "activations/layer7_attention_weight_max": 75.8138198852539,
      "activations/layer7_attention_weight_min": -65.1681900024414,
      "activations/layer8_attention_weight_max": 51.304630279541016,
      "activations/layer8_attention_weight_min": -41.542057037353516,
      "activations/layer9_attention_weight_max": 77.62992858886719,
      "activations/layer9_attention_weight_min": -53.19022750854492,
      "epoch": 1.98,
      "learning_rate": 0.00013861628787878788,
      "loss": 2.9801,
      "step": 34100
    },
    {
      "activations/layer0_attention_weight_max": 15.258368492126465,
      "activations/layer0_attention_weight_min": -11.521203994750977,
      "activations/layer10_attention_weight_max": 60.86349868774414,
      "activations/layer10_attention_weight_min": -53.29608154296875,
      "activations/layer11_attention_weight_max": 46.25176239013672,
      "activations/layer11_attention_weight_min": -38.048587799072266,
      "activations/layer12_attention_weight_max": 32.10881805419922,
      "activations/layer12_attention_weight_min": -28.305500030517578,
      "activations/layer13_attention_weight_max": 60.46327590942383,
      "activations/layer13_attention_weight_min": -48.631080627441406,
      "activations/layer14_attention_weight_max": 76.36156463623047,
      "activations/layer14_attention_weight_min": -55.993812561035156,
      "activations/layer15_attention_weight_max": 52.797698974609375,
      "activations/layer15_attention_weight_min": -38.03114700317383,
      "activations/layer16_attention_weight_max": 55.98166275024414,
      "activations/layer16_attention_weight_min": -41.6597785949707,
      "activations/layer17_attention_weight_max": 46.515907287597656,
      "activations/layer17_attention_weight_min": -28.117280960083008,
      "activations/layer18_attention_weight_max": 25.974609375,
      "activations/layer18_attention_weight_min": -16.105255126953125,
      "activations/layer19_attention_weight_max": 52.64284133911133,
      "activations/layer19_attention_weight_min": -32.452789306640625,
      "activations/layer1_attention_weight_max": 18.41378402709961,
      "activations/layer1_attention_weight_min": -10.47531509399414,
      "activations/layer20_attention_weight_max": 38.16019821166992,
      "activations/layer20_attention_weight_min": -22.698760986328125,
      "activations/layer21_attention_weight_max": 27.855167388916016,
      "activations/layer21_attention_weight_min": -17.63161277770996,
      "activations/layer22_attention_weight_max": 43.938987731933594,
      "activations/layer22_attention_weight_min": -22.575883865356445,
      "activations/layer23_attention_weight_max": 23.327190399169922,
      "activations/layer23_attention_weight_min": -15.88645076751709,
      "activations/layer2_attention_weight_max": 26.465660095214844,
      "activations/layer2_attention_weight_min": -26.87609100341797,
      "activations/layer3_attention_weight_max": 68.26839447021484,
      "activations/layer3_attention_weight_min": -71.84977722167969,
      "activations/layer4_attention_weight_max": 122.89763641357422,
      "activations/layer4_attention_weight_min": -118.215087890625,
      "activations/layer5_attention_weight_max": 107.76445770263672,
      "activations/layer5_attention_weight_min": -98.53050231933594,
      "activations/layer6_attention_weight_max": 57.66808319091797,
      "activations/layer6_attention_weight_min": -55.07366180419922,
      "activations/layer7_attention_weight_max": 69.02000427246094,
      "activations/layer7_attention_weight_min": -64.05583953857422,
      "activations/layer8_attention_weight_max": 42.39237976074219,
      "activations/layer8_attention_weight_min": -38.3076171875,
      "activations/layer9_attention_weight_max": 69.21231079101562,
      "activations/layer9_attention_weight_min": -61.904659271240234,
      "epoch": 1.98,
      "learning_rate": 0.00013859734848484847,
      "loss": 2.9685,
      "step": 34150
    },
    {
      "activations/layer0_attention_weight_max": 16.377588272094727,
      "activations/layer0_attention_weight_min": -11.766812324523926,
      "activations/layer10_attention_weight_max": 50.560813903808594,
      "activations/layer10_attention_weight_min": -41.84678649902344,
      "activations/layer11_attention_weight_max": 38.37681579589844,
      "activations/layer11_attention_weight_min": -33.96440124511719,
      "activations/layer12_attention_weight_max": 41.71097183227539,
      "activations/layer12_attention_weight_min": -28.094013214111328,
      "activations/layer13_attention_weight_max": 52.283939361572266,
      "activations/layer13_attention_weight_min": -36.90533447265625,
      "activations/layer14_attention_weight_max": 76.73023223876953,
      "activations/layer14_attention_weight_min": -55.032196044921875,
      "activations/layer15_attention_weight_max": 55.576194763183594,
      "activations/layer15_attention_weight_min": -36.00215148925781,
      "activations/layer16_attention_weight_max": 53.89146041870117,
      "activations/layer16_attention_weight_min": -38.92770004272461,
      "activations/layer17_attention_weight_max": 52.41288757324219,
      "activations/layer17_attention_weight_min": -28.231416702270508,
      "activations/layer18_attention_weight_max": 29.1008243560791,
      "activations/layer18_attention_weight_min": -17.95351791381836,
      "activations/layer19_attention_weight_max": 48.17705535888672,
      "activations/layer19_attention_weight_min": -34.71821212768555,
      "activations/layer1_attention_weight_max": 18.470182418823242,
      "activations/layer1_attention_weight_min": -11.109688758850098,
      "activations/layer20_attention_weight_max": 35.80652618408203,
      "activations/layer20_attention_weight_min": -23.945480346679688,
      "activations/layer21_attention_weight_max": 37.151824951171875,
      "activations/layer21_attention_weight_min": -22.286161422729492,
      "activations/layer22_attention_weight_max": 44.293052673339844,
      "activations/layer22_attention_weight_min": -28.454017639160156,
      "activations/layer23_attention_weight_max": 25.48768424987793,
      "activations/layer23_attention_weight_min": -17.372241973876953,
      "activations/layer2_attention_weight_max": 27.70611572265625,
      "activations/layer2_attention_weight_min": -26.372114181518555,
      "activations/layer3_attention_weight_max": 68.04644775390625,
      "activations/layer3_attention_weight_min": -73.06755065917969,
      "activations/layer4_attention_weight_max": 122.86370086669922,
      "activations/layer4_attention_weight_min": -121.1781997680664,
      "activations/layer5_attention_weight_max": 110.84173583984375,
      "activations/layer5_attention_weight_min": -98.48583984375,
      "activations/layer6_attention_weight_max": 60.3198127746582,
      "activations/layer6_attention_weight_min": -55.16343688964844,
      "activations/layer7_attention_weight_max": 68.71627807617188,
      "activations/layer7_attention_weight_min": -64.38462829589844,
      "activations/layer8_attention_weight_max": 46.88117980957031,
      "activations/layer8_attention_weight_min": -39.52836990356445,
      "activations/layer9_attention_weight_max": 73.9674072265625,
      "activations/layer9_attention_weight_min": -54.67097091674805,
      "epoch": 1.99,
      "learning_rate": 0.00013857840909090908,
      "loss": 2.9661,
      "step": 34200
    },
    {
      "activations/layer0_attention_weight_max": 17.63733673095703,
      "activations/layer0_attention_weight_min": -11.656404495239258,
      "activations/layer10_attention_weight_max": 53.18771743774414,
      "activations/layer10_attention_weight_min": -41.62693786621094,
      "activations/layer11_attention_weight_max": 42.13731384277344,
      "activations/layer11_attention_weight_min": -34.8564453125,
      "activations/layer12_attention_weight_max": 35.088741302490234,
      "activations/layer12_attention_weight_min": -40.5697021484375,
      "activations/layer13_attention_weight_max": 46.55192184448242,
      "activations/layer13_attention_weight_min": -37.77728271484375,
      "activations/layer14_attention_weight_max": 71.44512176513672,
      "activations/layer14_attention_weight_min": -54.08515167236328,
      "activations/layer15_attention_weight_max": 47.4495964050293,
      "activations/layer15_attention_weight_min": -35.20632553100586,
      "activations/layer16_attention_weight_max": 47.709835052490234,
      "activations/layer16_attention_weight_min": -35.71530532836914,
      "activations/layer17_attention_weight_max": 43.07414627075195,
      "activations/layer17_attention_weight_min": -25.89365577697754,
      "activations/layer18_attention_weight_max": 24.5340633392334,
      "activations/layer18_attention_weight_min": -16.977157592773438,
      "activations/layer19_attention_weight_max": 50.29457473754883,
      "activations/layer19_attention_weight_min": -33.99259567260742,
      "activations/layer1_attention_weight_max": 18.3085994720459,
      "activations/layer1_attention_weight_min": -12.326171875,
      "activations/layer20_attention_weight_max": 35.72762680053711,
      "activations/layer20_attention_weight_min": -24.756025314331055,
      "activations/layer21_attention_weight_max": 30.8023681640625,
      "activations/layer21_attention_weight_min": -21.203283309936523,
      "activations/layer22_attention_weight_max": 44.23573303222656,
      "activations/layer22_attention_weight_min": -26.451725006103516,
      "activations/layer23_attention_weight_max": 25.883159637451172,
      "activations/layer23_attention_weight_min": -17.426551818847656,
      "activations/layer2_attention_weight_max": 28.252243041992188,
      "activations/layer2_attention_weight_min": -28.2995662689209,
      "activations/layer3_attention_weight_max": 72.58540344238281,
      "activations/layer3_attention_weight_min": -73.95270538330078,
      "activations/layer4_attention_weight_max": 131.84555053710938,
      "activations/layer4_attention_weight_min": -115.47081756591797,
      "activations/layer5_attention_weight_max": 126.41490936279297,
      "activations/layer5_attention_weight_min": -101.58980560302734,
      "activations/layer6_attention_weight_max": 67.22081756591797,
      "activations/layer6_attention_weight_min": -54.36655044555664,
      "activations/layer7_attention_weight_max": 72.24761199951172,
      "activations/layer7_attention_weight_min": -69.6585922241211,
      "activations/layer8_attention_weight_max": 52.56568908691406,
      "activations/layer8_attention_weight_min": -39.58944320678711,
      "activations/layer9_attention_weight_max": 79.46333312988281,
      "activations/layer9_attention_weight_min": -60.020263671875,
      "epoch": 1.99,
      "learning_rate": 0.00013855946969696967,
      "loss": 2.9766,
      "step": 34250
    },
    {
      "activations/layer0_attention_weight_max": 16.047142028808594,
      "activations/layer0_attention_weight_min": -12.189269065856934,
      "activations/layer10_attention_weight_max": 63.30917739868164,
      "activations/layer10_attention_weight_min": -44.896881103515625,
      "activations/layer11_attention_weight_max": 47.37544250488281,
      "activations/layer11_attention_weight_min": -36.75954818725586,
      "activations/layer12_attention_weight_max": 81.38579559326172,
      "activations/layer12_attention_weight_min": -40.97317123413086,
      "activations/layer13_attention_weight_max": 83.5730972290039,
      "activations/layer13_attention_weight_min": -49.578346252441406,
      "activations/layer14_attention_weight_max": 106.22431945800781,
      "activations/layer14_attention_weight_min": -66.55340576171875,
      "activations/layer15_attention_weight_max": 69.60395812988281,
      "activations/layer15_attention_weight_min": -40.417118072509766,
      "activations/layer16_attention_weight_max": 65.14459228515625,
      "activations/layer16_attention_weight_min": -38.15074920654297,
      "activations/layer17_attention_weight_max": 50.85646438598633,
      "activations/layer17_attention_weight_min": -27.95122528076172,
      "activations/layer18_attention_weight_max": 33.09103775024414,
      "activations/layer18_attention_weight_min": -18.362825393676758,
      "activations/layer19_attention_weight_max": 78.19430541992188,
      "activations/layer19_attention_weight_min": -35.9377326965332,
      "activations/layer1_attention_weight_max": 18.483308792114258,
      "activations/layer1_attention_weight_min": -11.226487159729004,
      "activations/layer20_attention_weight_max": 53.98616027832031,
      "activations/layer20_attention_weight_min": -24.041902542114258,
      "activations/layer21_attention_weight_max": 37.53968048095703,
      "activations/layer21_attention_weight_min": -17.84612464904785,
      "activations/layer22_attention_weight_max": 65.93701934814453,
      "activations/layer22_attention_weight_min": -27.923452377319336,
      "activations/layer23_attention_weight_max": 30.944599151611328,
      "activations/layer23_attention_weight_min": -13.863761901855469,
      "activations/layer2_attention_weight_max": 30.069255828857422,
      "activations/layer2_attention_weight_min": -28.476318359375,
      "activations/layer3_attention_weight_max": 74.38751983642578,
      "activations/layer3_attention_weight_min": -72.68233489990234,
      "activations/layer4_attention_weight_max": 144.69229125976562,
      "activations/layer4_attention_weight_min": -121.36148834228516,
      "activations/layer5_attention_weight_max": 132.9849853515625,
      "activations/layer5_attention_weight_min": -98.58706665039062,
      "activations/layer6_attention_weight_max": 76.69628143310547,
      "activations/layer6_attention_weight_min": -56.81590270996094,
      "activations/layer7_attention_weight_max": 78.33429718017578,
      "activations/layer7_attention_weight_min": -68.77664947509766,
      "activations/layer8_attention_weight_max": 61.45828628540039,
      "activations/layer8_attention_weight_min": -43.494911193847656,
      "activations/layer9_attention_weight_max": 79.09912872314453,
      "activations/layer9_attention_weight_min": -59.14412307739258,
      "epoch": 1.99,
      "learning_rate": 0.0001385405303030303,
      "loss": 2.969,
      "step": 34300
    },
    {
      "activations/layer0_attention_weight_max": 16.094701766967773,
      "activations/layer0_attention_weight_min": -11.278063774108887,
      "activations/layer10_attention_weight_max": 60.45405960083008,
      "activations/layer10_attention_weight_min": -46.639495849609375,
      "activations/layer11_attention_weight_max": 52.53398132324219,
      "activations/layer11_attention_weight_min": -40.47278594970703,
      "activations/layer12_attention_weight_max": 67.61631774902344,
      "activations/layer12_attention_weight_min": -37.59212112426758,
      "activations/layer13_attention_weight_max": 85.18733978271484,
      "activations/layer13_attention_weight_min": -48.73229217529297,
      "activations/layer14_attention_weight_max": 103.08212280273438,
      "activations/layer14_attention_weight_min": -62.0901985168457,
      "activations/layer15_attention_weight_max": 65.52796936035156,
      "activations/layer15_attention_weight_min": -35.66779708862305,
      "activations/layer16_attention_weight_max": 57.329803466796875,
      "activations/layer16_attention_weight_min": -38.059898376464844,
      "activations/layer17_attention_weight_max": 51.0352783203125,
      "activations/layer17_attention_weight_min": -30.97015380859375,
      "activations/layer18_attention_weight_max": 28.596553802490234,
      "activations/layer18_attention_weight_min": -16.61639404296875,
      "activations/layer19_attention_weight_max": 56.616058349609375,
      "activations/layer19_attention_weight_min": -33.6739501953125,
      "activations/layer1_attention_weight_max": 17.971050262451172,
      "activations/layer1_attention_weight_min": -11.03357982635498,
      "activations/layer20_attention_weight_max": 38.51523208618164,
      "activations/layer20_attention_weight_min": -24.022422790527344,
      "activations/layer21_attention_weight_max": 30.91219139099121,
      "activations/layer21_attention_weight_min": -19.91999626159668,
      "activations/layer22_attention_weight_max": 44.970237731933594,
      "activations/layer22_attention_weight_min": -25.675249099731445,
      "activations/layer23_attention_weight_max": 25.59613609313965,
      "activations/layer23_attention_weight_min": -14.22958755493164,
      "activations/layer2_attention_weight_max": 31.706016540527344,
      "activations/layer2_attention_weight_min": -29.01178550720215,
      "activations/layer3_attention_weight_max": 79.76404571533203,
      "activations/layer3_attention_weight_min": -80.86416625976562,
      "activations/layer4_attention_weight_max": 155.7931365966797,
      "activations/layer4_attention_weight_min": -129.3038330078125,
      "activations/layer5_attention_weight_max": 134.5751190185547,
      "activations/layer5_attention_weight_min": -107.98909759521484,
      "activations/layer6_attention_weight_max": 75.95891571044922,
      "activations/layer6_attention_weight_min": -55.88831329345703,
      "activations/layer7_attention_weight_max": 82.29002380371094,
      "activations/layer7_attention_weight_min": -66.52938842773438,
      "activations/layer8_attention_weight_max": 63.2198371887207,
      "activations/layer8_attention_weight_min": -46.646568298339844,
      "activations/layer9_attention_weight_max": 96.75147247314453,
      "activations/layer9_attention_weight_min": -66.3500747680664,
      "epoch": 2.0,
      "learning_rate": 0.0001385215909090909,
      "loss": 2.9723,
      "step": 34350
    },
    {
      "activations/layer0_attention_weight_max": 16.344615936279297,
      "activations/layer0_attention_weight_min": -11.425436019897461,
      "activations/layer10_attention_weight_max": 54.74626922607422,
      "activations/layer10_attention_weight_min": -45.10980987548828,
      "activations/layer11_attention_weight_max": 49.025421142578125,
      "activations/layer11_attention_weight_min": -34.758235931396484,
      "activations/layer12_attention_weight_max": 39.39187240600586,
      "activations/layer12_attention_weight_min": -27.663084030151367,
      "activations/layer13_attention_weight_max": 51.18827819824219,
      "activations/layer13_attention_weight_min": -36.68962860107422,
      "activations/layer14_attention_weight_max": 79.60760498046875,
      "activations/layer14_attention_weight_min": -57.206939697265625,
      "activations/layer15_attention_weight_max": 58.05875015258789,
      "activations/layer15_attention_weight_min": -41.92091751098633,
      "activations/layer16_attention_weight_max": 56.79069519042969,
      "activations/layer16_attention_weight_min": -40.65835952758789,
      "activations/layer17_attention_weight_max": 54.8967170715332,
      "activations/layer17_attention_weight_min": -28.56863784790039,
      "activations/layer18_attention_weight_max": 33.94841003417969,
      "activations/layer18_attention_weight_min": -18.276832580566406,
      "activations/layer19_attention_weight_max": 49.73360824584961,
      "activations/layer19_attention_weight_min": -33.85337448120117,
      "activations/layer1_attention_weight_max": 17.689897537231445,
      "activations/layer1_attention_weight_min": -11.390427589416504,
      "activations/layer20_attention_weight_max": 39.47058868408203,
      "activations/layer20_attention_weight_min": -27.446598052978516,
      "activations/layer21_attention_weight_max": 30.594173431396484,
      "activations/layer21_attention_weight_min": -18.72344970703125,
      "activations/layer22_attention_weight_max": 44.34904098510742,
      "activations/layer22_attention_weight_min": -27.39482879638672,
      "activations/layer23_attention_weight_max": 25.97420883178711,
      "activations/layer23_attention_weight_min": -16.0338077545166,
      "activations/layer2_attention_weight_max": 28.108097076416016,
      "activations/layer2_attention_weight_min": -27.558982849121094,
      "activations/layer3_attention_weight_max": 71.35494995117188,
      "activations/layer3_attention_weight_min": -73.98323059082031,
      "activations/layer4_attention_weight_max": 128.05870056152344,
      "activations/layer4_attention_weight_min": -114.1763916015625,
      "activations/layer5_attention_weight_max": 117.29798889160156,
      "activations/layer5_attention_weight_min": -98.24310302734375,
      "activations/layer6_attention_weight_max": 66.49568176269531,
      "activations/layer6_attention_weight_min": -54.23867416381836,
      "activations/layer7_attention_weight_max": 67.90058898925781,
      "activations/layer7_attention_weight_min": -61.886138916015625,
      "activations/layer8_attention_weight_max": 55.25990676879883,
      "activations/layer8_attention_weight_min": -45.676239013671875,
      "activations/layer9_attention_weight_max": 78.42325592041016,
      "activations/layer9_attention_weight_min": -61.61005401611328,
      "epoch": 2.0,
      "learning_rate": 0.0001385026515151515,
      "loss": 2.956,
      "step": 34400
    },
    {
      "activations/layer0_attention_weight_max": 16.54916763305664,
      "activations/layer0_attention_weight_min": -12.424263000488281,
      "activations/layer10_attention_weight_max": 54.17121124267578,
      "activations/layer10_attention_weight_min": -41.76787185668945,
      "activations/layer11_attention_weight_max": 42.062217712402344,
      "activations/layer11_attention_weight_min": -32.180885314941406,
      "activations/layer12_attention_weight_max": 47.65445327758789,
      "activations/layer12_attention_weight_min": -38.50423049926758,
      "activations/layer13_attention_weight_max": 62.31525421142578,
      "activations/layer13_attention_weight_min": -42.16399002075195,
      "activations/layer14_attention_weight_max": 89.97504425048828,
      "activations/layer14_attention_weight_min": -61.423858642578125,
      "activations/layer15_attention_weight_max": 61.739601135253906,
      "activations/layer15_attention_weight_min": -37.67667007446289,
      "activations/layer16_attention_weight_max": 52.04410171508789,
      "activations/layer16_attention_weight_min": -35.187110900878906,
      "activations/layer17_attention_weight_max": 49.78153991699219,
      "activations/layer17_attention_weight_min": -28.858285903930664,
      "activations/layer18_attention_weight_max": 31.739816665649414,
      "activations/layer18_attention_weight_min": -21.575450897216797,
      "activations/layer19_attention_weight_max": 52.4664306640625,
      "activations/layer19_attention_weight_min": -31.544809341430664,
      "activations/layer1_attention_weight_max": 18.44955062866211,
      "activations/layer1_attention_weight_min": -11.596169471740723,
      "activations/layer20_attention_weight_max": 43.77802658081055,
      "activations/layer20_attention_weight_min": -25.4382381439209,
      "activations/layer21_attention_weight_max": 34.420814514160156,
      "activations/layer21_attention_weight_min": -19.099605560302734,
      "activations/layer22_attention_weight_max": 48.45936584472656,
      "activations/layer22_attention_weight_min": -26.136077880859375,
      "activations/layer23_attention_weight_max": 24.499786376953125,
      "activations/layer23_attention_weight_min": -15.98108959197998,
      "activations/layer2_attention_weight_max": 28.09217071533203,
      "activations/layer2_attention_weight_min": -28.774978637695312,
      "activations/layer3_attention_weight_max": 73.3633041381836,
      "activations/layer3_attention_weight_min": -74.72819519042969,
      "activations/layer4_attention_weight_max": 125.22457122802734,
      "activations/layer4_attention_weight_min": -114.8171157836914,
      "activations/layer5_attention_weight_max": 110.38641357421875,
      "activations/layer5_attention_weight_min": -100.47958374023438,
      "activations/layer6_attention_weight_max": 62.082950592041016,
      "activations/layer6_attention_weight_min": -55.63520431518555,
      "activations/layer7_attention_weight_max": 70.1037368774414,
      "activations/layer7_attention_weight_min": -65.51873016357422,
      "activations/layer8_attention_weight_max": 45.563133239746094,
      "activations/layer8_attention_weight_min": -42.175045013427734,
      "activations/layer9_attention_weight_max": 67.50556945800781,
      "activations/layer9_attention_weight_min": -52.544715881347656,
      "epoch": 2.0,
      "learning_rate": 0.00013848333333333333,
      "loss": 3.0142,
      "step": 34450
    },
    {
      "activations/layer0_attention_weight_max": 16.837757110595703,
      "activations/layer0_attention_weight_min": -11.425166130065918,
      "activations/layer10_attention_weight_max": 47.66703796386719,
      "activations/layer10_attention_weight_min": -44.606834411621094,
      "activations/layer11_attention_weight_max": 35.95745849609375,
      "activations/layer11_attention_weight_min": -35.78644561767578,
      "activations/layer12_attention_weight_max": 30.88450813293457,
      "activations/layer12_attention_weight_min": -25.39118194580078,
      "activations/layer13_attention_weight_max": 44.97828674316406,
      "activations/layer13_attention_weight_min": -39.31673049926758,
      "activations/layer14_attention_weight_max": 62.352996826171875,
      "activations/layer14_attention_weight_min": -55.38479995727539,
      "activations/layer15_attention_weight_max": 46.93741989135742,
      "activations/layer15_attention_weight_min": -36.59823226928711,
      "activations/layer16_attention_weight_max": 47.90123748779297,
      "activations/layer16_attention_weight_min": -38.176788330078125,
      "activations/layer17_attention_weight_max": 42.604488372802734,
      "activations/layer17_attention_weight_min": -30.454193115234375,
      "activations/layer18_attention_weight_max": 29.091556549072266,
      "activations/layer18_attention_weight_min": -18.711212158203125,
      "activations/layer19_attention_weight_max": 46.81532669067383,
      "activations/layer19_attention_weight_min": -33.50011444091797,
      "activations/layer1_attention_weight_max": 18.200149536132812,
      "activations/layer1_attention_weight_min": -10.220226287841797,
      "activations/layer20_attention_weight_max": 34.58035659790039,
      "activations/layer20_attention_weight_min": -26.12383270263672,
      "activations/layer21_attention_weight_max": 34.20365524291992,
      "activations/layer21_attention_weight_min": -23.623611450195312,
      "activations/layer22_attention_weight_max": 42.10926818847656,
      "activations/layer22_attention_weight_min": -24.067733764648438,
      "activations/layer23_attention_weight_max": 23.146512985229492,
      "activations/layer23_attention_weight_min": -16.270660400390625,
      "activations/layer2_attention_weight_max": 26.758312225341797,
      "activations/layer2_attention_weight_min": -26.76211929321289,
      "activations/layer3_attention_weight_max": 68.9083480834961,
      "activations/layer3_attention_weight_min": -72.24156951904297,
      "activations/layer4_attention_weight_max": 119.67569732666016,
      "activations/layer4_attention_weight_min": -119.12041473388672,
      "activations/layer5_attention_weight_max": 108.12364959716797,
      "activations/layer5_attention_weight_min": -100.85195922851562,
      "activations/layer6_attention_weight_max": 57.140472412109375,
      "activations/layer6_attention_weight_min": -56.27103805541992,
      "activations/layer7_attention_weight_max": 65.80508422851562,
      "activations/layer7_attention_weight_min": -66.73882293701172,
      "activations/layer8_attention_weight_max": 41.5911979675293,
      "activations/layer8_attention_weight_min": -38.7984733581543,
      "activations/layer9_attention_weight_max": 65.91211700439453,
      "activations/layer9_attention_weight_min": -53.23430252075195,
      "epoch": 2.0,
      "learning_rate": 0.00013846439393939392,
      "loss": 2.9566,
      "step": 34500
    },
    {
      "activations/layer0_attention_weight_max": 16.291519165039062,
      "activations/layer0_attention_weight_min": -11.315749168395996,
      "activations/layer10_attention_weight_max": 50.706417083740234,
      "activations/layer10_attention_weight_min": -42.03205490112305,
      "activations/layer11_attention_weight_max": 36.9960823059082,
      "activations/layer11_attention_weight_min": -34.65322494506836,
      "activations/layer12_attention_weight_max": 30.45279312133789,
      "activations/layer12_attention_weight_min": -25.46621322631836,
      "activations/layer13_attention_weight_max": 52.02322769165039,
      "activations/layer13_attention_weight_min": -39.00950622558594,
      "activations/layer14_attention_weight_max": 78.7249984741211,
      "activations/layer14_attention_weight_min": -60.9598274230957,
      "activations/layer15_attention_weight_max": 48.233585357666016,
      "activations/layer15_attention_weight_min": -36.2265510559082,
      "activations/layer16_attention_weight_max": 50.72187042236328,
      "activations/layer16_attention_weight_min": -36.17483901977539,
      "activations/layer17_attention_weight_max": 48.201866149902344,
      "activations/layer17_attention_weight_min": -28.61174201965332,
      "activations/layer18_attention_weight_max": 27.856740951538086,
      "activations/layer18_attention_weight_min": -19.07479476928711,
      "activations/layer19_attention_weight_max": 45.48414993286133,
      "activations/layer19_attention_weight_min": -31.746505737304688,
      "activations/layer1_attention_weight_max": 18.685543060302734,
      "activations/layer1_attention_weight_min": -11.787129402160645,
      "activations/layer20_attention_weight_max": 38.31242370605469,
      "activations/layer20_attention_weight_min": -26.8166446685791,
      "activations/layer21_attention_weight_max": 29.183958053588867,
      "activations/layer21_attention_weight_min": -18.975196838378906,
      "activations/layer22_attention_weight_max": 49.63026428222656,
      "activations/layer22_attention_weight_min": -26.43498420715332,
      "activations/layer23_attention_weight_max": 24.550312042236328,
      "activations/layer23_attention_weight_min": -19.597454071044922,
      "activations/layer2_attention_weight_max": 28.0127010345459,
      "activations/layer2_attention_weight_min": -27.88373374938965,
      "activations/layer3_attention_weight_max": 68.71929931640625,
      "activations/layer3_attention_weight_min": -70.70838928222656,
      "activations/layer4_attention_weight_max": 125.04454803466797,
      "activations/layer4_attention_weight_min": -124.090087890625,
      "activations/layer5_attention_weight_max": 111.57305908203125,
      "activations/layer5_attention_weight_min": -98.93035125732422,
      "activations/layer6_attention_weight_max": 62.24327087402344,
      "activations/layer6_attention_weight_min": -53.66689682006836,
      "activations/layer7_attention_weight_max": 67.26559448242188,
      "activations/layer7_attention_weight_min": -65.72054290771484,
      "activations/layer8_attention_weight_max": 44.381473541259766,
      "activations/layer8_attention_weight_min": -37.56678771972656,
      "activations/layer9_attention_weight_max": 65.88207244873047,
      "activations/layer9_attention_weight_min": -50.10599136352539,
      "epoch": 2.01,
      "learning_rate": 0.00013844545454545453,
      "loss": 2.955,
      "step": 34550
    },
    {
      "activations/layer0_attention_weight_max": 16.00697135925293,
      "activations/layer0_attention_weight_min": -11.140047073364258,
      "activations/layer10_attention_weight_max": 46.68565368652344,
      "activations/layer10_attention_weight_min": -43.39101791381836,
      "activations/layer11_attention_weight_max": 37.303428649902344,
      "activations/layer11_attention_weight_min": -32.808448791503906,
      "activations/layer12_attention_weight_max": 33.43471145629883,
      "activations/layer12_attention_weight_min": -27.641170501708984,
      "activations/layer13_attention_weight_max": 48.97637176513672,
      "activations/layer13_attention_weight_min": -36.333003997802734,
      "activations/layer14_attention_weight_max": 75.46424865722656,
      "activations/layer14_attention_weight_min": -56.95942687988281,
      "activations/layer15_attention_weight_max": 52.28904342651367,
      "activations/layer15_attention_weight_min": -36.76430130004883,
      "activations/layer16_attention_weight_max": 46.14372634887695,
      "activations/layer16_attention_weight_min": -37.00192642211914,
      "activations/layer17_attention_weight_max": 44.91146469116211,
      "activations/layer17_attention_weight_min": -29.54281997680664,
      "activations/layer18_attention_weight_max": 29.452754974365234,
      "activations/layer18_attention_weight_min": -17.069805145263672,
      "activations/layer19_attention_weight_max": 48.68312072753906,
      "activations/layer19_attention_weight_min": -31.994504928588867,
      "activations/layer1_attention_weight_max": 18.547122955322266,
      "activations/layer1_attention_weight_min": -13.298542976379395,
      "activations/layer20_attention_weight_max": 38.02366638183594,
      "activations/layer20_attention_weight_min": -26.106176376342773,
      "activations/layer21_attention_weight_max": 31.023555755615234,
      "activations/layer21_attention_weight_min": -21.49221420288086,
      "activations/layer22_attention_weight_max": 43.46472930908203,
      "activations/layer22_attention_weight_min": -27.02395248413086,
      "activations/layer23_attention_weight_max": 26.060314178466797,
      "activations/layer23_attention_weight_min": -17.682342529296875,
      "activations/layer2_attention_weight_max": 31.104068756103516,
      "activations/layer2_attention_weight_min": -30.81835174560547,
      "activations/layer3_attention_weight_max": 74.2071533203125,
      "activations/layer3_attention_weight_min": -79.29755401611328,
      "activations/layer4_attention_weight_max": 126.35485076904297,
      "activations/layer4_attention_weight_min": -127.1773910522461,
      "activations/layer5_attention_weight_max": 114.90225219726562,
      "activations/layer5_attention_weight_min": -108.64305114746094,
      "activations/layer6_attention_weight_max": 60.82475280761719,
      "activations/layer6_attention_weight_min": -56.45689392089844,
      "activations/layer7_attention_weight_max": 65.11347198486328,
      "activations/layer7_attention_weight_min": -61.99513626098633,
      "activations/layer8_attention_weight_max": 46.76995086669922,
      "activations/layer8_attention_weight_min": -41.48618698120117,
      "activations/layer9_attention_weight_max": 63.68498611450195,
      "activations/layer9_attention_weight_min": -54.50593948364258,
      "epoch": 2.01,
      "learning_rate": 0.00013842651515151512,
      "loss": 2.9508,
      "step": 34600
    },
    {
      "activations/layer0_attention_weight_max": 16.52458953857422,
      "activations/layer0_attention_weight_min": -11.140986442565918,
      "activations/layer10_attention_weight_max": 54.45041275024414,
      "activations/layer10_attention_weight_min": -50.93887710571289,
      "activations/layer11_attention_weight_max": 37.239837646484375,
      "activations/layer11_attention_weight_min": -34.78020477294922,
      "activations/layer12_attention_weight_max": 35.0200309753418,
      "activations/layer12_attention_weight_min": -31.63921356201172,
      "activations/layer13_attention_weight_max": 45.599117279052734,
      "activations/layer13_attention_weight_min": -36.61750411987305,
      "activations/layer14_attention_weight_max": 68.57540893554688,
      "activations/layer14_attention_weight_min": -57.704158782958984,
      "activations/layer15_attention_weight_max": 48.33549118041992,
      "activations/layer15_attention_weight_min": -40.51470947265625,
      "activations/layer16_attention_weight_max": 49.08176803588867,
      "activations/layer16_attention_weight_min": -35.13387680053711,
      "activations/layer17_attention_weight_max": 44.70554733276367,
      "activations/layer17_attention_weight_min": -32.5453987121582,
      "activations/layer18_attention_weight_max": 30.50932502746582,
      "activations/layer18_attention_weight_min": -19.159690856933594,
      "activations/layer19_attention_weight_max": 55.21954345703125,
      "activations/layer19_attention_weight_min": -32.05004119873047,
      "activations/layer1_attention_weight_max": 18.38471221923828,
      "activations/layer1_attention_weight_min": -10.792513847351074,
      "activations/layer20_attention_weight_max": 38.927974700927734,
      "activations/layer20_attention_weight_min": -25.248044967651367,
      "activations/layer21_attention_weight_max": 29.669790267944336,
      "activations/layer21_attention_weight_min": -20.750205993652344,
      "activations/layer22_attention_weight_max": 43.94987487792969,
      "activations/layer22_attention_weight_min": -27.916259765625,
      "activations/layer23_attention_weight_max": 24.778289794921875,
      "activations/layer23_attention_weight_min": -16.952131271362305,
      "activations/layer2_attention_weight_max": 28.55396270751953,
      "activations/layer2_attention_weight_min": -28.049419403076172,
      "activations/layer3_attention_weight_max": 69.245849609375,
      "activations/layer3_attention_weight_min": -71.34446716308594,
      "activations/layer4_attention_weight_max": 125.38880920410156,
      "activations/layer4_attention_weight_min": -113.70344543457031,
      "activations/layer5_attention_weight_max": 111.72239685058594,
      "activations/layer5_attention_weight_min": -93.18749237060547,
      "activations/layer6_attention_weight_max": 62.548072814941406,
      "activations/layer6_attention_weight_min": -52.68960952758789,
      "activations/layer7_attention_weight_max": 61.88616943359375,
      "activations/layer7_attention_weight_min": -61.59278869628906,
      "activations/layer8_attention_weight_max": 44.576290130615234,
      "activations/layer8_attention_weight_min": -36.562564849853516,
      "activations/layer9_attention_weight_max": 65.4134750366211,
      "activations/layer9_attention_weight_min": -52.74424362182617,
      "epoch": 2.01,
      "learning_rate": 0.00013840757575757574,
      "loss": 2.9587,
      "step": 34650
    },
    {
      "activations/layer0_attention_weight_max": 16.151126861572266,
      "activations/layer0_attention_weight_min": -11.251741409301758,
      "activations/layer10_attention_weight_max": 53.28776550292969,
      "activations/layer10_attention_weight_min": -44.367279052734375,
      "activations/layer11_attention_weight_max": 40.08155822753906,
      "activations/layer11_attention_weight_min": -34.279014587402344,
      "activations/layer12_attention_weight_max": 32.98772430419922,
      "activations/layer12_attention_weight_min": -29.09884262084961,
      "activations/layer13_attention_weight_max": 52.42563247680664,
      "activations/layer13_attention_weight_min": -37.87294387817383,
      "activations/layer14_attention_weight_max": 86.09537506103516,
      "activations/layer14_attention_weight_min": -60.818023681640625,
      "activations/layer15_attention_weight_max": 60.099002838134766,
      "activations/layer15_attention_weight_min": -39.11703109741211,
      "activations/layer16_attention_weight_max": 61.30796813964844,
      "activations/layer16_attention_weight_min": -36.34640884399414,
      "activations/layer17_attention_weight_max": 48.82514572143555,
      "activations/layer17_attention_weight_min": -28.824487686157227,
      "activations/layer18_attention_weight_max": 27.89387321472168,
      "activations/layer18_attention_weight_min": -18.33561134338379,
      "activations/layer19_attention_weight_max": 60.74843215942383,
      "activations/layer19_attention_weight_min": -34.41960525512695,
      "activations/layer1_attention_weight_max": 17.273452758789062,
      "activations/layer1_attention_weight_min": -11.265039443969727,
      "activations/layer20_attention_weight_max": 42.5222053527832,
      "activations/layer20_attention_weight_min": -26.142343521118164,
      "activations/layer21_attention_weight_max": 37.62810516357422,
      "activations/layer21_attention_weight_min": -18.148094177246094,
      "activations/layer22_attention_weight_max": 46.9388542175293,
      "activations/layer22_attention_weight_min": -26.43917465209961,
      "activations/layer23_attention_weight_max": 28.566328048706055,
      "activations/layer23_attention_weight_min": -15.773351669311523,
      "activations/layer2_attention_weight_max": 31.022315979003906,
      "activations/layer2_attention_weight_min": -29.31361961364746,
      "activations/layer3_attention_weight_max": 73.83734130859375,
      "activations/layer3_attention_weight_min": -73.76296997070312,
      "activations/layer4_attention_weight_max": 128.24246215820312,
      "activations/layer4_attention_weight_min": -119.06890106201172,
      "activations/layer5_attention_weight_max": 118.36161804199219,
      "activations/layer5_attention_weight_min": -100.51911926269531,
      "activations/layer6_attention_weight_max": 64.35408020019531,
      "activations/layer6_attention_weight_min": -54.401954650878906,
      "activations/layer7_attention_weight_max": 66.77164459228516,
      "activations/layer7_attention_weight_min": -66.6003646850586,
      "activations/layer8_attention_weight_max": 44.7352294921875,
      "activations/layer8_attention_weight_min": -37.26194763183594,
      "activations/layer9_attention_weight_max": 67.96460723876953,
      "activations/layer9_attention_weight_min": -53.19134521484375,
      "epoch": 2.02,
      "learning_rate": 0.00013838863636363635,
      "loss": 2.971,
      "step": 34700
    },
    {
      "activations/layer0_attention_weight_max": 16.23090934753418,
      "activations/layer0_attention_weight_min": -11.018906593322754,
      "activations/layer10_attention_weight_max": 68.64411163330078,
      "activations/layer10_attention_weight_min": -45.42039489746094,
      "activations/layer11_attention_weight_max": 59.85999298095703,
      "activations/layer11_attention_weight_min": -33.91867446899414,
      "activations/layer12_attention_weight_max": 58.3994026184082,
      "activations/layer12_attention_weight_min": -30.078754425048828,
      "activations/layer13_attention_weight_max": 91.76449584960938,
      "activations/layer13_attention_weight_min": -50.01145553588867,
      "activations/layer14_attention_weight_max": 103.90580749511719,
      "activations/layer14_attention_weight_min": -61.037776947021484,
      "activations/layer15_attention_weight_max": 68.83450317382812,
      "activations/layer15_attention_weight_min": -39.08568572998047,
      "activations/layer16_attention_weight_max": 62.37751388549805,
      "activations/layer16_attention_weight_min": -35.25424575805664,
      "activations/layer17_attention_weight_max": 47.623146057128906,
      "activations/layer17_attention_weight_min": -28.92991065979004,
      "activations/layer18_attention_weight_max": 39.35077667236328,
      "activations/layer18_attention_weight_min": -17.617191314697266,
      "activations/layer19_attention_weight_max": 68.02115631103516,
      "activations/layer19_attention_weight_min": -30.347436904907227,
      "activations/layer1_attention_weight_max": 16.992691040039062,
      "activations/layer1_attention_weight_min": -11.639877319335938,
      "activations/layer20_attention_weight_max": 46.888065338134766,
      "activations/layer20_attention_weight_min": -25.438175201416016,
      "activations/layer21_attention_weight_max": 29.302947998046875,
      "activations/layer21_attention_weight_min": -19.55682945251465,
      "activations/layer22_attention_weight_max": 46.8983039855957,
      "activations/layer22_attention_weight_min": -25.83554458618164,
      "activations/layer23_attention_weight_max": 27.41573143005371,
      "activations/layer23_attention_weight_min": -15.774149894714355,
      "activations/layer2_attention_weight_max": 27.78204345703125,
      "activations/layer2_attention_weight_min": -26.976306915283203,
      "activations/layer3_attention_weight_max": 72.95034790039062,
      "activations/layer3_attention_weight_min": -74.36173248291016,
      "activations/layer4_attention_weight_max": 136.10728454589844,
      "activations/layer4_attention_weight_min": -121.23272705078125,
      "activations/layer5_attention_weight_max": 128.9066619873047,
      "activations/layer5_attention_weight_min": -104.72967529296875,
      "activations/layer6_attention_weight_max": 72.33949279785156,
      "activations/layer6_attention_weight_min": -56.26963424682617,
      "activations/layer7_attention_weight_max": 81.21278381347656,
      "activations/layer7_attention_weight_min": -64.18666076660156,
      "activations/layer8_attention_weight_max": 59.74512481689453,
      "activations/layer8_attention_weight_min": -45.22209548950195,
      "activations/layer9_attention_weight_max": 93.77432250976562,
      "activations/layer9_attention_weight_min": -67.36436462402344,
      "epoch": 2.02,
      "learning_rate": 0.00013836969696969697,
      "loss": 2.9619,
      "step": 34750
    },
    {
      "activations/layer0_attention_weight_max": 16.635211944580078,
      "activations/layer0_attention_weight_min": -11.708878517150879,
      "activations/layer10_attention_weight_max": 50.791473388671875,
      "activations/layer10_attention_weight_min": -42.14494323730469,
      "activations/layer11_attention_weight_max": 38.103797912597656,
      "activations/layer11_attention_weight_min": -36.151920318603516,
      "activations/layer12_attention_weight_max": 35.83415985107422,
      "activations/layer12_attention_weight_min": -27.777524948120117,
      "activations/layer13_attention_weight_max": 67.95645141601562,
      "activations/layer13_attention_weight_min": -45.082252502441406,
      "activations/layer14_attention_weight_max": 85.75274658203125,
      "activations/layer14_attention_weight_min": -58.14888000488281,
      "activations/layer15_attention_weight_max": 50.5040168762207,
      "activations/layer15_attention_weight_min": -36.46971893310547,
      "activations/layer16_attention_weight_max": 58.03303146362305,
      "activations/layer16_attention_weight_min": -35.40144729614258,
      "activations/layer17_attention_weight_max": 47.39253616333008,
      "activations/layer17_attention_weight_min": -30.169458389282227,
      "activations/layer18_attention_weight_max": 26.43314552307129,
      "activations/layer18_attention_weight_min": -19.07857894897461,
      "activations/layer19_attention_weight_max": 49.798828125,
      "activations/layer19_attention_weight_min": -31.089994430541992,
      "activations/layer1_attention_weight_max": 17.803482055664062,
      "activations/layer1_attention_weight_min": -12.125658988952637,
      "activations/layer20_attention_weight_max": 36.0521240234375,
      "activations/layer20_attention_weight_min": -26.789112091064453,
      "activations/layer21_attention_weight_max": 32.665382385253906,
      "activations/layer21_attention_weight_min": -20.183443069458008,
      "activations/layer22_attention_weight_max": 41.405948638916016,
      "activations/layer22_attention_weight_min": -26.44226837158203,
      "activations/layer23_attention_weight_max": 23.270551681518555,
      "activations/layer23_attention_weight_min": -14.590254783630371,
      "activations/layer2_attention_weight_max": 30.154571533203125,
      "activations/layer2_attention_weight_min": -29.65423583984375,
      "activations/layer3_attention_weight_max": 69.93895721435547,
      "activations/layer3_attention_weight_min": -72.1364517211914,
      "activations/layer4_attention_weight_max": 121.6378402709961,
      "activations/layer4_attention_weight_min": -117.11122131347656,
      "activations/layer5_attention_weight_max": 108.13320922851562,
      "activations/layer5_attention_weight_min": -98.47906494140625,
      "activations/layer6_attention_weight_max": 59.88151931762695,
      "activations/layer6_attention_weight_min": -52.00950622558594,
      "activations/layer7_attention_weight_max": 67.85014343261719,
      "activations/layer7_attention_weight_min": -64.27922821044922,
      "activations/layer8_attention_weight_max": 44.97019958496094,
      "activations/layer8_attention_weight_min": -39.130455017089844,
      "activations/layer9_attention_weight_max": 60.4427375793457,
      "activations/layer9_attention_weight_min": -51.01101303100586,
      "epoch": 2.02,
      "learning_rate": 0.00013835075757575755,
      "loss": 2.9625,
      "step": 34800
    },
    {
      "activations/layer0_attention_weight_max": 16.368207931518555,
      "activations/layer0_attention_weight_min": -11.638633728027344,
      "activations/layer10_attention_weight_max": 56.05743408203125,
      "activations/layer10_attention_weight_min": -45.07991409301758,
      "activations/layer11_attention_weight_max": 36.966949462890625,
      "activations/layer11_attention_weight_min": -35.064945220947266,
      "activations/layer12_attention_weight_max": 39.403133392333984,
      "activations/layer12_attention_weight_min": -34.074684143066406,
      "activations/layer13_attention_weight_max": 59.30244445800781,
      "activations/layer13_attention_weight_min": -42.88511657714844,
      "activations/layer14_attention_weight_max": 86.32759857177734,
      "activations/layer14_attention_weight_min": -69.0208740234375,
      "activations/layer15_attention_weight_max": 56.86687088012695,
      "activations/layer15_attention_weight_min": -35.41252899169922,
      "activations/layer16_attention_weight_max": 52.67715835571289,
      "activations/layer16_attention_weight_min": -36.091800689697266,
      "activations/layer17_attention_weight_max": 47.850341796875,
      "activations/layer17_attention_weight_min": -28.055007934570312,
      "activations/layer18_attention_weight_max": 27.232091903686523,
      "activations/layer18_attention_weight_min": -18.967342376708984,
      "activations/layer19_attention_weight_max": 48.47409439086914,
      "activations/layer19_attention_weight_min": -30.74777603149414,
      "activations/layer1_attention_weight_max": 17.90496253967285,
      "activations/layer1_attention_weight_min": -11.106733322143555,
      "activations/layer20_attention_weight_max": 39.267784118652344,
      "activations/layer20_attention_weight_min": -24.136919021606445,
      "activations/layer21_attention_weight_max": 31.899168014526367,
      "activations/layer21_attention_weight_min": -22.27288818359375,
      "activations/layer22_attention_weight_max": 46.03623962402344,
      "activations/layer22_attention_weight_min": -25.252290725708008,
      "activations/layer23_attention_weight_max": 26.87387466430664,
      "activations/layer23_attention_weight_min": -15.967920303344727,
      "activations/layer2_attention_weight_max": 28.822216033935547,
      "activations/layer2_attention_weight_min": -27.637441635131836,
      "activations/layer3_attention_weight_max": 71.70387268066406,
      "activations/layer3_attention_weight_min": -74.40840911865234,
      "activations/layer4_attention_weight_max": 125.96388244628906,
      "activations/layer4_attention_weight_min": -125.81145477294922,
      "activations/layer5_attention_weight_max": 112.10859680175781,
      "activations/layer5_attention_weight_min": -105.81739807128906,
      "activations/layer6_attention_weight_max": 60.8587646484375,
      "activations/layer6_attention_weight_min": -54.66013717651367,
      "activations/layer7_attention_weight_max": 66.15421295166016,
      "activations/layer7_attention_weight_min": -62.053138732910156,
      "activations/layer8_attention_weight_max": 47.922874450683594,
      "activations/layer8_attention_weight_min": -43.616493225097656,
      "activations/layer9_attention_weight_max": 59.49477767944336,
      "activations/layer9_attention_weight_min": -53.922149658203125,
      "epoch": 2.02,
      "learning_rate": 0.00013833181818181817,
      "loss": 2.9529,
      "step": 34850
    },
    {
      "activations/layer0_attention_weight_max": 16.188318252563477,
      "activations/layer0_attention_weight_min": -11.81643295288086,
      "activations/layer10_attention_weight_max": 50.97862243652344,
      "activations/layer10_attention_weight_min": -42.307891845703125,
      "activations/layer11_attention_weight_max": 40.03508758544922,
      "activations/layer11_attention_weight_min": -34.187904357910156,
      "activations/layer12_attention_weight_max": 43.65732192993164,
      "activations/layer12_attention_weight_min": -33.48045349121094,
      "activations/layer13_attention_weight_max": 59.08195877075195,
      "activations/layer13_attention_weight_min": -51.32195281982422,
      "activations/layer14_attention_weight_max": 86.74148559570312,
      "activations/layer14_attention_weight_min": -61.91242980957031,
      "activations/layer15_attention_weight_max": 55.08784866333008,
      "activations/layer15_attention_weight_min": -37.64202880859375,
      "activations/layer16_attention_weight_max": 53.17497634887695,
      "activations/layer16_attention_weight_min": -39.44925308227539,
      "activations/layer17_attention_weight_max": 45.61985778808594,
      "activations/layer17_attention_weight_min": -29.002384185791016,
      "activations/layer18_attention_weight_max": 28.73969078063965,
      "activations/layer18_attention_weight_min": -19.5660457611084,
      "activations/layer19_attention_weight_max": 54.70003890991211,
      "activations/layer19_attention_weight_min": -31.994001388549805,
      "activations/layer1_attention_weight_max": 17.105497360229492,
      "activations/layer1_attention_weight_min": -10.513631820678711,
      "activations/layer20_attention_weight_max": 45.12186813354492,
      "activations/layer20_attention_weight_min": -26.33258056640625,
      "activations/layer21_attention_weight_max": 32.67575454711914,
      "activations/layer21_attention_weight_min": -18.996692657470703,
      "activations/layer22_attention_weight_max": 44.64161682128906,
      "activations/layer22_attention_weight_min": -26.067964553833008,
      "activations/layer23_attention_weight_max": 27.55499267578125,
      "activations/layer23_attention_weight_min": -15.918341636657715,
      "activations/layer2_attention_weight_max": 27.185543060302734,
      "activations/layer2_attention_weight_min": -27.599632263183594,
      "activations/layer3_attention_weight_max": 70.32134246826172,
      "activations/layer3_attention_weight_min": -72.95134735107422,
      "activations/layer4_attention_weight_max": 130.1966552734375,
      "activations/layer4_attention_weight_min": -118.6810302734375,
      "activations/layer5_attention_weight_max": 116.08685302734375,
      "activations/layer5_attention_weight_min": -98.73391723632812,
      "activations/layer6_attention_weight_max": 62.894981384277344,
      "activations/layer6_attention_weight_min": -54.97443771362305,
      "activations/layer7_attention_weight_max": 68.68968963623047,
      "activations/layer7_attention_weight_min": -65.12340545654297,
      "activations/layer8_attention_weight_max": 46.84477233886719,
      "activations/layer8_attention_weight_min": -38.630615234375,
      "activations/layer9_attention_weight_max": 61.74076461791992,
      "activations/layer9_attention_weight_min": -52.0958366394043,
      "epoch": 2.03,
      "learning_rate": 0.00013831287878787878,
      "loss": 2.9736,
      "step": 34900
    },
    {
      "activations/layer0_attention_weight_max": 16.201631546020508,
      "activations/layer0_attention_weight_min": -11.604257583618164,
      "activations/layer10_attention_weight_max": 48.125938415527344,
      "activations/layer10_attention_weight_min": -42.65009689331055,
      "activations/layer11_attention_weight_max": 35.92572021484375,
      "activations/layer11_attention_weight_min": -32.528499603271484,
      "activations/layer12_attention_weight_max": 39.748653411865234,
      "activations/layer12_attention_weight_min": -30.00330352783203,
      "activations/layer13_attention_weight_max": 51.14806365966797,
      "activations/layer13_attention_weight_min": -35.198829650878906,
      "activations/layer14_attention_weight_max": 72.9773178100586,
      "activations/layer14_attention_weight_min": -53.80976486206055,
      "activations/layer15_attention_weight_max": 51.60988998413086,
      "activations/layer15_attention_weight_min": -37.718685150146484,
      "activations/layer16_attention_weight_max": 52.33533477783203,
      "activations/layer16_attention_weight_min": -36.17216873168945,
      "activations/layer17_attention_weight_max": 45.056556701660156,
      "activations/layer17_attention_weight_min": -27.525257110595703,
      "activations/layer18_attention_weight_max": 26.24993133544922,
      "activations/layer18_attention_weight_min": -17.54758644104004,
      "activations/layer19_attention_weight_max": 53.455970764160156,
      "activations/layer19_attention_weight_min": -31.174503326416016,
      "activations/layer1_attention_weight_max": 17.751020431518555,
      "activations/layer1_attention_weight_min": -12.40354061126709,
      "activations/layer20_attention_weight_max": 40.28915023803711,
      "activations/layer20_attention_weight_min": -25.7504940032959,
      "activations/layer21_attention_weight_max": 31.875850677490234,
      "activations/layer21_attention_weight_min": -17.810760498046875,
      "activations/layer22_attention_weight_max": 47.34627151489258,
      "activations/layer22_attention_weight_min": -28.03395652770996,
      "activations/layer23_attention_weight_max": 28.389888763427734,
      "activations/layer23_attention_weight_min": -17.551889419555664,
      "activations/layer2_attention_weight_max": 27.851896286010742,
      "activations/layer2_attention_weight_min": -27.67136001586914,
      "activations/layer3_attention_weight_max": 70.04822540283203,
      "activations/layer3_attention_weight_min": -72.53672790527344,
      "activations/layer4_attention_weight_max": 118.9021987915039,
      "activations/layer4_attention_weight_min": -112.96831512451172,
      "activations/layer5_attention_weight_max": 113.26956176757812,
      "activations/layer5_attention_weight_min": -97.59848022460938,
      "activations/layer6_attention_weight_max": 64.57350158691406,
      "activations/layer6_attention_weight_min": -50.76219940185547,
      "activations/layer7_attention_weight_max": 65.83326721191406,
      "activations/layer7_attention_weight_min": -61.50819778442383,
      "activations/layer8_attention_weight_max": 44.17542266845703,
      "activations/layer8_attention_weight_min": -37.204978942871094,
      "activations/layer9_attention_weight_max": 70.85040283203125,
      "activations/layer9_attention_weight_min": -50.01535415649414,
      "epoch": 2.03,
      "learning_rate": 0.00013829393939393937,
      "loss": 2.9652,
      "step": 34950
    },
    {
      "activations/layer0_attention_weight_max": 16.85110092163086,
      "activations/layer0_attention_weight_min": -11.300318717956543,
      "activations/layer10_attention_weight_max": 56.40826416015625,
      "activations/layer10_attention_weight_min": -49.08549880981445,
      "activations/layer11_attention_weight_max": 38.19586944580078,
      "activations/layer11_attention_weight_min": -32.978759765625,
      "activations/layer12_attention_weight_max": 54.74778747558594,
      "activations/layer12_attention_weight_min": -47.9283447265625,
      "activations/layer13_attention_weight_max": 45.896751403808594,
      "activations/layer13_attention_weight_min": -42.68489456176758,
      "activations/layer14_attention_weight_max": 79.14201354980469,
      "activations/layer14_attention_weight_min": -64.03504943847656,
      "activations/layer15_attention_weight_max": 46.16737747192383,
      "activations/layer15_attention_weight_min": -38.561744689941406,
      "activations/layer16_attention_weight_max": 47.30741882324219,
      "activations/layer16_attention_weight_min": -37.07904815673828,
      "activations/layer17_attention_weight_max": 45.12489700317383,
      "activations/layer17_attention_weight_min": -31.797006607055664,
      "activations/layer18_attention_weight_max": 28.010456085205078,
      "activations/layer18_attention_weight_min": -19.92559242248535,
      "activations/layer19_attention_weight_max": 52.859893798828125,
      "activations/layer19_attention_weight_min": -35.934017181396484,
      "activations/layer1_attention_weight_max": 18.90201759338379,
      "activations/layer1_attention_weight_min": -10.492526054382324,
      "activations/layer20_attention_weight_max": 36.863468170166016,
      "activations/layer20_attention_weight_min": -27.154279708862305,
      "activations/layer21_attention_weight_max": 32.06186294555664,
      "activations/layer21_attention_weight_min": -19.233118057250977,
      "activations/layer22_attention_weight_max": 45.80286407470703,
      "activations/layer22_attention_weight_min": -22.85733985900879,
      "activations/layer23_attention_weight_max": 25.35260772705078,
      "activations/layer23_attention_weight_min": -15.94815731048584,
      "activations/layer2_attention_weight_max": 27.735210418701172,
      "activations/layer2_attention_weight_min": -27.938140869140625,
      "activations/layer3_attention_weight_max": 70.41963958740234,
      "activations/layer3_attention_weight_min": -73.38917541503906,
      "activations/layer4_attention_weight_max": 126.36980438232422,
      "activations/layer4_attention_weight_min": -121.73735809326172,
      "activations/layer5_attention_weight_max": 113.08187103271484,
      "activations/layer5_attention_weight_min": -99.95430755615234,
      "activations/layer6_attention_weight_max": 63.024497985839844,
      "activations/layer6_attention_weight_min": -52.961368560791016,
      "activations/layer7_attention_weight_max": 67.78003692626953,
      "activations/layer7_attention_weight_min": -61.90965270996094,
      "activations/layer8_attention_weight_max": 49.927223205566406,
      "activations/layer8_attention_weight_min": -38.84031677246094,
      "activations/layer9_attention_weight_max": 68.3568344116211,
      "activations/layer9_attention_weight_min": -60.2380485534668,
      "epoch": 2.03,
      "learning_rate": 0.000138275,
      "loss": 2.9642,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_loss": 2.912109375,
      "eval_runtime": 8.526,
      "eval_samples_per_second": 503.638,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_openwebtext_loss": 2.912109375,
      "eval_openwebtext_ppl": 18.39556080667227,
      "eval_openwebtext_runtime": 8.526,
      "eval_openwebtext_samples_per_second": 503.638,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_wikitext_loss": 3.1875,
      "eval_wikitext_ppl": 24.22778221261098,
      "eval_wikitext_runtime": 2.0225,
      "eval_wikitext_samples_per_second": 225.462,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_lambada_loss": 3.12890625,
      "eval_lambada_ppl": 22.84897480427519,
      "eval_lambada_runtime": 9.5904,
      "eval_lambada_samples_per_second": 507.695,
      "step": 35000
    },
    {
      "activations/layer0_attention_weight_max": 16.579612731933594,
      "activations/layer0_attention_weight_min": -11.68398666381836,
      "activations/layer10_attention_weight_max": 56.727447509765625,
      "activations/layer10_attention_weight_min": -46.828975677490234,
      "activations/layer11_attention_weight_max": 42.01593017578125,
      "activations/layer11_attention_weight_min": -37.879329681396484,
      "activations/layer12_attention_weight_max": 38.788734436035156,
      "activations/layer12_attention_weight_min": -29.678607940673828,
      "activations/layer13_attention_weight_max": 58.3792839050293,
      "activations/layer13_attention_weight_min": -41.485618591308594,
      "activations/layer14_attention_weight_max": 85.86798095703125,
      "activations/layer14_attention_weight_min": -60.43118667602539,
      "activations/layer15_attention_weight_max": 59.75800704956055,
      "activations/layer15_attention_weight_min": -39.423343658447266,
      "activations/layer16_attention_weight_max": 58.09120178222656,
      "activations/layer16_attention_weight_min": -39.112220764160156,
      "activations/layer17_attention_weight_max": 56.542945861816406,
      "activations/layer17_attention_weight_min": -30.45046615600586,
      "activations/layer18_attention_weight_max": 30.737455368041992,
      "activations/layer18_attention_weight_min": -17.740665435791016,
      "activations/layer19_attention_weight_max": 57.4638671875,
      "activations/layer19_attention_weight_min": -34.435707092285156,
      "activations/layer1_attention_weight_max": 17.596534729003906,
      "activations/layer1_attention_weight_min": -10.770994186401367,
      "activations/layer20_attention_weight_max": 40.15861129760742,
      "activations/layer20_attention_weight_min": -23.407455444335938,
      "activations/layer21_attention_weight_max": 37.635597229003906,
      "activations/layer21_attention_weight_min": -23.68296241760254,
      "activations/layer22_attention_weight_max": 49.93794250488281,
      "activations/layer22_attention_weight_min": -27.50321388244629,
      "activations/layer23_attention_weight_max": 28.44526481628418,
      "activations/layer23_attention_weight_min": -18.091341018676758,
      "activations/layer2_attention_weight_max": 28.785594940185547,
      "activations/layer2_attention_weight_min": -28.43360710144043,
      "activations/layer3_attention_weight_max": 70.1535415649414,
      "activations/layer3_attention_weight_min": -73.45225524902344,
      "activations/layer4_attention_weight_max": 130.559814453125,
      "activations/layer4_attention_weight_min": -118.28053283691406,
      "activations/layer5_attention_weight_max": 117.97753143310547,
      "activations/layer5_attention_weight_min": -99.27496337890625,
      "activations/layer6_attention_weight_max": 62.526554107666016,
      "activations/layer6_attention_weight_min": -53.748966217041016,
      "activations/layer7_attention_weight_max": 65.23760986328125,
      "activations/layer7_attention_weight_min": -61.4157829284668,
      "activations/layer8_attention_weight_max": 48.73051834106445,
      "activations/layer8_attention_weight_min": -37.43501663208008,
      "activations/layer9_attention_weight_max": 65.49458312988281,
      "activations/layer9_attention_weight_min": -55.99318313598633,
      "epoch": 2.04,
      "learning_rate": 0.0001382560606060606,
      "loss": 2.9293,
      "step": 35050
    },
    {
      "activations/layer0_attention_weight_max": 16.12497329711914,
      "activations/layer0_attention_weight_min": -11.431084632873535,
      "activations/layer10_attention_weight_max": 59.425567626953125,
      "activations/layer10_attention_weight_min": -44.523353576660156,
      "activations/layer11_attention_weight_max": 42.66259765625,
      "activations/layer11_attention_weight_min": -37.200096130371094,
      "activations/layer12_attention_weight_max": 51.35615158081055,
      "activations/layer12_attention_weight_min": -34.651554107666016,
      "activations/layer13_attention_weight_max": 53.030250549316406,
      "activations/layer13_attention_weight_min": -39.67723083496094,
      "activations/layer14_attention_weight_max": 89.4570083618164,
      "activations/layer14_attention_weight_min": -59.990753173828125,
      "activations/layer15_attention_weight_max": 63.662601470947266,
      "activations/layer15_attention_weight_min": -40.51694869995117,
      "activations/layer16_attention_weight_max": 51.63731002807617,
      "activations/layer16_attention_weight_min": -39.36425018310547,
      "activations/layer17_attention_weight_max": 50.9632453918457,
      "activations/layer17_attention_weight_min": -31.35433006286621,
      "activations/layer18_attention_weight_max": 27.34058952331543,
      "activations/layer18_attention_weight_min": -16.662853240966797,
      "activations/layer19_attention_weight_max": 52.78871536254883,
      "activations/layer19_attention_weight_min": -32.89601135253906,
      "activations/layer1_attention_weight_max": 17.371200561523438,
      "activations/layer1_attention_weight_min": -11.436212539672852,
      "activations/layer20_attention_weight_max": 38.01165771484375,
      "activations/layer20_attention_weight_min": -25.367443084716797,
      "activations/layer21_attention_weight_max": 33.54095458984375,
      "activations/layer21_attention_weight_min": -20.394351959228516,
      "activations/layer22_attention_weight_max": 44.86429977416992,
      "activations/layer22_attention_weight_min": -24.939830780029297,
      "activations/layer23_attention_weight_max": 25.26222801208496,
      "activations/layer23_attention_weight_min": -18.826568603515625,
      "activations/layer2_attention_weight_max": 28.993553161621094,
      "activations/layer2_attention_weight_min": -28.24147605895996,
      "activations/layer3_attention_weight_max": 71.95447540283203,
      "activations/layer3_attention_weight_min": -75.8813705444336,
      "activations/layer4_attention_weight_max": 129.6067352294922,
      "activations/layer4_attention_weight_min": -125.04219818115234,
      "activations/layer5_attention_weight_max": 116.91183471679688,
      "activations/layer5_attention_weight_min": -101.80880737304688,
      "activations/layer6_attention_weight_max": 63.685997009277344,
      "activations/layer6_attention_weight_min": -53.45637512207031,
      "activations/layer7_attention_weight_max": 73.35821533203125,
      "activations/layer7_attention_weight_min": -64.62010955810547,
      "activations/layer8_attention_weight_max": 50.303627014160156,
      "activations/layer8_attention_weight_min": -39.40824508666992,
      "activations/layer9_attention_weight_max": 69.05655670166016,
      "activations/layer9_attention_weight_min": -56.06843185424805,
      "epoch": 2.04,
      "learning_rate": 0.00013823712121212122,
      "loss": 2.9298,
      "step": 35100
    },
    {
      "activations/layer0_attention_weight_max": 16.518404006958008,
      "activations/layer0_attention_weight_min": -11.336942672729492,
      "activations/layer10_attention_weight_max": 44.39667510986328,
      "activations/layer10_attention_weight_min": -37.92551040649414,
      "activations/layer11_attention_weight_max": 37.90108871459961,
      "activations/layer11_attention_weight_min": -31.56820297241211,
      "activations/layer12_attention_weight_max": 32.30182647705078,
      "activations/layer12_attention_weight_min": -27.987794876098633,
      "activations/layer13_attention_weight_max": 45.42387771606445,
      "activations/layer13_attention_weight_min": -37.87977600097656,
      "activations/layer14_attention_weight_max": 65.2960433959961,
      "activations/layer14_attention_weight_min": -54.881404876708984,
      "activations/layer15_attention_weight_max": 50.141788482666016,
      "activations/layer15_attention_weight_min": -44.537906646728516,
      "activations/layer16_attention_weight_max": 47.92005920410156,
      "activations/layer16_attention_weight_min": -35.00331115722656,
      "activations/layer17_attention_weight_max": 42.47579574584961,
      "activations/layer17_attention_weight_min": -29.214656829833984,
      "activations/layer18_attention_weight_max": 23.988248825073242,
      "activations/layer18_attention_weight_min": -20.89113998413086,
      "activations/layer19_attention_weight_max": 49.75982666015625,
      "activations/layer19_attention_weight_min": -33.35930252075195,
      "activations/layer1_attention_weight_max": 19.644372940063477,
      "activations/layer1_attention_weight_min": -11.452401161193848,
      "activations/layer20_attention_weight_max": 33.42455291748047,
      "activations/layer20_attention_weight_min": -23.921401977539062,
      "activations/layer21_attention_weight_max": 30.60135269165039,
      "activations/layer21_attention_weight_min": -18.03474998474121,
      "activations/layer22_attention_weight_max": 42.60857391357422,
      "activations/layer22_attention_weight_min": -27.08858299255371,
      "activations/layer23_attention_weight_max": 25.912221908569336,
      "activations/layer23_attention_weight_min": -15.182631492614746,
      "activations/layer2_attention_weight_max": 29.745101928710938,
      "activations/layer2_attention_weight_min": -30.263023376464844,
      "activations/layer3_attention_weight_max": 73.297607421875,
      "activations/layer3_attention_weight_min": -72.49915313720703,
      "activations/layer4_attention_weight_max": 128.7642059326172,
      "activations/layer4_attention_weight_min": -115.56477355957031,
      "activations/layer5_attention_weight_max": 118.83802795410156,
      "activations/layer5_attention_weight_min": -97.03019714355469,
      "activations/layer6_attention_weight_max": 62.4134635925293,
      "activations/layer6_attention_weight_min": -51.51136779785156,
      "activations/layer7_attention_weight_max": 64.93177795410156,
      "activations/layer7_attention_weight_min": -66.43834686279297,
      "activations/layer8_attention_weight_max": 46.90135955810547,
      "activations/layer8_attention_weight_min": -40.313838958740234,
      "activations/layer9_attention_weight_max": 63.227027893066406,
      "activations/layer9_attention_weight_min": -56.57841873168945,
      "epoch": 2.04,
      "learning_rate": 0.0001382181818181818,
      "loss": 2.9294,
      "step": 35150
    },
    {
      "activations/layer0_attention_weight_max": 16.688013076782227,
      "activations/layer0_attention_weight_min": -11.26664924621582,
      "activations/layer10_attention_weight_max": 58.37895202636719,
      "activations/layer10_attention_weight_min": -45.71315002441406,
      "activations/layer11_attention_weight_max": 40.506649017333984,
      "activations/layer11_attention_weight_min": -32.51052474975586,
      "activations/layer12_attention_weight_max": 36.981414794921875,
      "activations/layer12_attention_weight_min": -28.112220764160156,
      "activations/layer13_attention_weight_max": 61.658973693847656,
      "activations/layer13_attention_weight_min": -40.83651351928711,
      "activations/layer14_attention_weight_max": 87.00688934326172,
      "activations/layer14_attention_weight_min": -70.01175689697266,
      "activations/layer15_attention_weight_max": 49.268028259277344,
      "activations/layer15_attention_weight_min": -36.15098571777344,
      "activations/layer16_attention_weight_max": 48.449832916259766,
      "activations/layer16_attention_weight_min": -36.13993453979492,
      "activations/layer17_attention_weight_max": 44.8953742980957,
      "activations/layer17_attention_weight_min": -28.9123477935791,
      "activations/layer18_attention_weight_max": 30.463504791259766,
      "activations/layer18_attention_weight_min": -20.745384216308594,
      "activations/layer19_attention_weight_max": 57.694366455078125,
      "activations/layer19_attention_weight_min": -31.696744918823242,
      "activations/layer1_attention_weight_max": 17.076152801513672,
      "activations/layer1_attention_weight_min": -10.999433517456055,
      "activations/layer20_attention_weight_max": 39.873252868652344,
      "activations/layer20_attention_weight_min": -23.238018035888672,
      "activations/layer21_attention_weight_max": 33.03931427001953,
      "activations/layer21_attention_weight_min": -16.440189361572266,
      "activations/layer22_attention_weight_max": 53.37521743774414,
      "activations/layer22_attention_weight_min": -26.74282455444336,
      "activations/layer23_attention_weight_max": 26.56629180908203,
      "activations/layer23_attention_weight_min": -16.126609802246094,
      "activations/layer2_attention_weight_max": 28.685604095458984,
      "activations/layer2_attention_weight_min": -27.760404586791992,
      "activations/layer3_attention_weight_max": 72.8562240600586,
      "activations/layer3_attention_weight_min": -73.3944091796875,
      "activations/layer4_attention_weight_max": 132.7501678466797,
      "activations/layer4_attention_weight_min": -122.27821350097656,
      "activations/layer5_attention_weight_max": 122.31379699707031,
      "activations/layer5_attention_weight_min": -101.62661743164062,
      "activations/layer6_attention_weight_max": 65.45278930664062,
      "activations/layer6_attention_weight_min": -56.460350036621094,
      "activations/layer7_attention_weight_max": 72.97032165527344,
      "activations/layer7_attention_weight_min": -64.99518585205078,
      "activations/layer8_attention_weight_max": 50.06215286254883,
      "activations/layer8_attention_weight_min": -39.82462692260742,
      "activations/layer9_attention_weight_max": 77.76990509033203,
      "activations/layer9_attention_weight_min": -53.575355529785156,
      "epoch": 2.05,
      "learning_rate": 0.00013819924242424242,
      "loss": 2.9161,
      "step": 35200
    },
    {
      "activations/layer0_attention_weight_max": 16.492294311523438,
      "activations/layer0_attention_weight_min": -11.614995956420898,
      "activations/layer10_attention_weight_max": 51.006744384765625,
      "activations/layer10_attention_weight_min": -40.63286209106445,
      "activations/layer11_attention_weight_max": 43.988521575927734,
      "activations/layer11_attention_weight_min": -35.82378387451172,
      "activations/layer12_attention_weight_max": 31.672462463378906,
      "activations/layer12_attention_weight_min": -30.178180694580078,
      "activations/layer13_attention_weight_max": 47.88357925415039,
      "activations/layer13_attention_weight_min": -38.35175323486328,
      "activations/layer14_attention_weight_max": 85.40835571289062,
      "activations/layer14_attention_weight_min": -66.19390106201172,
      "activations/layer15_attention_weight_max": 50.856754302978516,
      "activations/layer15_attention_weight_min": -40.089420318603516,
      "activations/layer16_attention_weight_max": 48.34220886230469,
      "activations/layer16_attention_weight_min": -38.896209716796875,
      "activations/layer17_attention_weight_max": 44.98830795288086,
      "activations/layer17_attention_weight_min": -28.409250259399414,
      "activations/layer18_attention_weight_max": 27.2581729888916,
      "activations/layer18_attention_weight_min": -17.641372680664062,
      "activations/layer19_attention_weight_max": 49.04481887817383,
      "activations/layer19_attention_weight_min": -32.19819641113281,
      "activations/layer1_attention_weight_max": 16.800567626953125,
      "activations/layer1_attention_weight_min": -10.48774528503418,
      "activations/layer20_attention_weight_max": 38.33135986328125,
      "activations/layer20_attention_weight_min": -29.429208755493164,
      "activations/layer21_attention_weight_max": 33.139774322509766,
      "activations/layer21_attention_weight_min": -17.741870880126953,
      "activations/layer22_attention_weight_max": 40.262718200683594,
      "activations/layer22_attention_weight_min": -23.686006546020508,
      "activations/layer23_attention_weight_max": 23.17707633972168,
      "activations/layer23_attention_weight_min": -15.644149780273438,
      "activations/layer2_attention_weight_max": 29.17494010925293,
      "activations/layer2_attention_weight_min": -28.23847770690918,
      "activations/layer3_attention_weight_max": 72.73178100585938,
      "activations/layer3_attention_weight_min": -74.00530242919922,
      "activations/layer4_attention_weight_max": 131.73899841308594,
      "activations/layer4_attention_weight_min": -123.75421142578125,
      "activations/layer5_attention_weight_max": 115.03801727294922,
      "activations/layer5_attention_weight_min": -101.52667236328125,
      "activations/layer6_attention_weight_max": 61.69566345214844,
      "activations/layer6_attention_weight_min": -53.9416618347168,
      "activations/layer7_attention_weight_max": 69.77940368652344,
      "activations/layer7_attention_weight_min": -63.323753356933594,
      "activations/layer8_attention_weight_max": 46.49317932128906,
      "activations/layer8_attention_weight_min": -39.21760559082031,
      "activations/layer9_attention_weight_max": 72.54393768310547,
      "activations/layer9_attention_weight_min": -57.284156799316406,
      "epoch": 2.05,
      "learning_rate": 0.00013818030303030303,
      "loss": 2.9379,
      "step": 35250
    },
    {
      "activations/layer0_attention_weight_max": 15.878793716430664,
      "activations/layer0_attention_weight_min": -10.996737480163574,
      "activations/layer10_attention_weight_max": 56.93762969970703,
      "activations/layer10_attention_weight_min": -44.62382507324219,
      "activations/layer11_attention_weight_max": 48.485565185546875,
      "activations/layer11_attention_weight_min": -42.032894134521484,
      "activations/layer12_attention_weight_max": 45.06074905395508,
      "activations/layer12_attention_weight_min": -31.67343521118164,
      "activations/layer13_attention_weight_max": 60.49015426635742,
      "activations/layer13_attention_weight_min": -42.792137145996094,
      "activations/layer14_attention_weight_max": 82.79694366455078,
      "activations/layer14_attention_weight_min": -55.291873931884766,
      "activations/layer15_attention_weight_max": 56.68770980834961,
      "activations/layer15_attention_weight_min": -37.8637580871582,
      "activations/layer16_attention_weight_max": 55.42594528198242,
      "activations/layer16_attention_weight_min": -37.51799774169922,
      "activations/layer17_attention_weight_max": 46.93087387084961,
      "activations/layer17_attention_weight_min": -28.614892959594727,
      "activations/layer18_attention_weight_max": 26.779682159423828,
      "activations/layer18_attention_weight_min": -16.58649444580078,
      "activations/layer19_attention_weight_max": 61.30910873413086,
      "activations/layer19_attention_weight_min": -33.99039840698242,
      "activations/layer1_attention_weight_max": 17.449342727661133,
      "activations/layer1_attention_weight_min": -11.349740982055664,
      "activations/layer20_attention_weight_max": 42.83207321166992,
      "activations/layer20_attention_weight_min": -23.949174880981445,
      "activations/layer21_attention_weight_max": 36.03313064575195,
      "activations/layer21_attention_weight_min": -18.974042892456055,
      "activations/layer22_attention_weight_max": 51.70560836791992,
      "activations/layer22_attention_weight_min": -23.89024543762207,
      "activations/layer23_attention_weight_max": 28.461071014404297,
      "activations/layer23_attention_weight_min": -15.454668045043945,
      "activations/layer2_attention_weight_max": 28.544940948486328,
      "activations/layer2_attention_weight_min": -27.660627365112305,
      "activations/layer3_attention_weight_max": 76.11151123046875,
      "activations/layer3_attention_weight_min": -77.42780303955078,
      "activations/layer4_attention_weight_max": 132.4830322265625,
      "activations/layer4_attention_weight_min": -127.82002258300781,
      "activations/layer5_attention_weight_max": 117.79491424560547,
      "activations/layer5_attention_weight_min": -106.04379272460938,
      "activations/layer6_attention_weight_max": 64.8329086303711,
      "activations/layer6_attention_weight_min": -56.11512756347656,
      "activations/layer7_attention_weight_max": 71.2612533569336,
      "activations/layer7_attention_weight_min": -67.14453887939453,
      "activations/layer8_attention_weight_max": 56.45706558227539,
      "activations/layer8_attention_weight_min": -44.43846893310547,
      "activations/layer9_attention_weight_max": 71.9000244140625,
      "activations/layer9_attention_weight_min": -59.109710693359375,
      "epoch": 2.05,
      "learning_rate": 0.00013816136363636362,
      "loss": 2.9213,
      "step": 35300
    },
    {
      "activations/layer0_attention_weight_max": 16.472057342529297,
      "activations/layer0_attention_weight_min": -11.534127235412598,
      "activations/layer10_attention_weight_max": 46.45033645629883,
      "activations/layer10_attention_weight_min": -38.49025344848633,
      "activations/layer11_attention_weight_max": 37.07698440551758,
      "activations/layer11_attention_weight_min": -32.64567565917969,
      "activations/layer12_attention_weight_max": 31.067195892333984,
      "activations/layer12_attention_weight_min": -25.520103454589844,
      "activations/layer13_attention_weight_max": 50.464534759521484,
      "activations/layer13_attention_weight_min": -35.94056701660156,
      "activations/layer14_attention_weight_max": 82.02464294433594,
      "activations/layer14_attention_weight_min": -59.68619155883789,
      "activations/layer15_attention_weight_max": 53.28780746459961,
      "activations/layer15_attention_weight_min": -38.93286895751953,
      "activations/layer16_attention_weight_max": 55.13214111328125,
      "activations/layer16_attention_weight_min": -37.235076904296875,
      "activations/layer17_attention_weight_max": 46.91196060180664,
      "activations/layer17_attention_weight_min": -28.165103912353516,
      "activations/layer18_attention_weight_max": 26.208114624023438,
      "activations/layer18_attention_weight_min": -19.11187171936035,
      "activations/layer19_attention_weight_max": 58.57514190673828,
      "activations/layer19_attention_weight_min": -35.714805603027344,
      "activations/layer1_attention_weight_max": 16.50396156311035,
      "activations/layer1_attention_weight_min": -11.671703338623047,
      "activations/layer20_attention_weight_max": 38.330814361572266,
      "activations/layer20_attention_weight_min": -24.109825134277344,
      "activations/layer21_attention_weight_max": 34.473121643066406,
      "activations/layer21_attention_weight_min": -20.513208389282227,
      "activations/layer22_attention_weight_max": 46.77699279785156,
      "activations/layer22_attention_weight_min": -26.426986694335938,
      "activations/layer23_attention_weight_max": 25.489171981811523,
      "activations/layer23_attention_weight_min": -15.791515350341797,
      "activations/layer2_attention_weight_max": 29.071638107299805,
      "activations/layer2_attention_weight_min": -28.18753433227539,
      "activations/layer3_attention_weight_max": 71.51160430908203,
      "activations/layer3_attention_weight_min": -75.90998077392578,
      "activations/layer4_attention_weight_max": 126.44872283935547,
      "activations/layer4_attention_weight_min": -120.75813293457031,
      "activations/layer5_attention_weight_max": 110.19061279296875,
      "activations/layer5_attention_weight_min": -98.12120819091797,
      "activations/layer6_attention_weight_max": 60.00288391113281,
      "activations/layer6_attention_weight_min": -54.99100875854492,
      "activations/layer7_attention_weight_max": 64.51091766357422,
      "activations/layer7_attention_weight_min": -66.94115447998047,
      "activations/layer8_attention_weight_max": 45.06248092651367,
      "activations/layer8_attention_weight_min": -38.61330032348633,
      "activations/layer9_attention_weight_max": 60.46282958984375,
      "activations/layer9_attention_weight_min": -56.226234436035156,
      "epoch": 2.05,
      "learning_rate": 0.00013814242424242424,
      "loss": 2.9281,
      "step": 35350
    },
    {
      "activations/layer0_attention_weight_max": 15.922127723693848,
      "activations/layer0_attention_weight_min": -11.091591835021973,
      "activations/layer10_attention_weight_max": 53.81272888183594,
      "activations/layer10_attention_weight_min": -43.31459426879883,
      "activations/layer11_attention_weight_max": 42.020896911621094,
      "activations/layer11_attention_weight_min": -34.99364471435547,
      "activations/layer12_attention_weight_max": 33.50678253173828,
      "activations/layer12_attention_weight_min": -28.75831413269043,
      "activations/layer13_attention_weight_max": 46.93825149536133,
      "activations/layer13_attention_weight_min": -38.61036682128906,
      "activations/layer14_attention_weight_max": 77.98673248291016,
      "activations/layer14_attention_weight_min": -56.30638885498047,
      "activations/layer15_attention_weight_max": 55.768890380859375,
      "activations/layer15_attention_weight_min": -38.29057693481445,
      "activations/layer16_attention_weight_max": 59.854766845703125,
      "activations/layer16_attention_weight_min": -37.766395568847656,
      "activations/layer17_attention_weight_max": 45.59861373901367,
      "activations/layer17_attention_weight_min": -30.11134910583496,
      "activations/layer18_attention_weight_max": 27.16534996032715,
      "activations/layer18_attention_weight_min": -18.846115112304688,
      "activations/layer19_attention_weight_max": 63.17585372924805,
      "activations/layer19_attention_weight_min": -32.55427169799805,
      "activations/layer1_attention_weight_max": 18.10545539855957,
      "activations/layer1_attention_weight_min": -11.512282371520996,
      "activations/layer20_attention_weight_max": 39.240638732910156,
      "activations/layer20_attention_weight_min": -22.345523834228516,
      "activations/layer21_attention_weight_max": 35.52973175048828,
      "activations/layer21_attention_weight_min": -21.974349975585938,
      "activations/layer22_attention_weight_max": 46.654579162597656,
      "activations/layer22_attention_weight_min": -28.79877471923828,
      "activations/layer23_attention_weight_max": 26.05805206298828,
      "activations/layer23_attention_weight_min": -16.6634578704834,
      "activations/layer2_attention_weight_max": 29.062549591064453,
      "activations/layer2_attention_weight_min": -29.557979583740234,
      "activations/layer3_attention_weight_max": 73.41349029541016,
      "activations/layer3_attention_weight_min": -73.03886413574219,
      "activations/layer4_attention_weight_max": 126.25801849365234,
      "activations/layer4_attention_weight_min": -116.70406341552734,
      "activations/layer5_attention_weight_max": 120.8480453491211,
      "activations/layer5_attention_weight_min": -98.89030456542969,
      "activations/layer6_attention_weight_max": 69.84239196777344,
      "activations/layer6_attention_weight_min": -51.73759841918945,
      "activations/layer7_attention_weight_max": 78.78857421875,
      "activations/layer7_attention_weight_min": -64.84452056884766,
      "activations/layer8_attention_weight_max": 47.357234954833984,
      "activations/layer8_attention_weight_min": -39.786468505859375,
      "activations/layer9_attention_weight_max": 68.69329833984375,
      "activations/layer9_attention_weight_min": -51.14639663696289,
      "epoch": 2.06,
      "learning_rate": 0.00013812348484848485,
      "loss": 2.9249,
      "step": 35400
    },
    {
      "activations/layer0_attention_weight_max": 16.466798782348633,
      "activations/layer0_attention_weight_min": -11.098329544067383,
      "activations/layer10_attention_weight_max": 48.93609619140625,
      "activations/layer10_attention_weight_min": -39.41875457763672,
      "activations/layer11_attention_weight_max": 39.98387908935547,
      "activations/layer11_attention_weight_min": -32.342247009277344,
      "activations/layer12_attention_weight_max": 29.621753692626953,
      "activations/layer12_attention_weight_min": -25.568851470947266,
      "activations/layer13_attention_weight_max": 49.21821212768555,
      "activations/layer13_attention_weight_min": -36.51809310913086,
      "activations/layer14_attention_weight_max": 67.89657592773438,
      "activations/layer14_attention_weight_min": -55.09276580810547,
      "activations/layer15_attention_weight_max": 50.452396392822266,
      "activations/layer15_attention_weight_min": -38.71565628051758,
      "activations/layer16_attention_weight_max": 49.40567398071289,
      "activations/layer16_attention_weight_min": -38.93218994140625,
      "activations/layer17_attention_weight_max": 43.32280349731445,
      "activations/layer17_attention_weight_min": -29.89419937133789,
      "activations/layer18_attention_weight_max": 27.983383178710938,
      "activations/layer18_attention_weight_min": -21.144264221191406,
      "activations/layer19_attention_weight_max": 53.5201301574707,
      "activations/layer19_attention_weight_min": -34.93352127075195,
      "activations/layer1_attention_weight_max": 16.865901947021484,
      "activations/layer1_attention_weight_min": -11.393377304077148,
      "activations/layer20_attention_weight_max": 37.7014045715332,
      "activations/layer20_attention_weight_min": -24.444547653198242,
      "activations/layer21_attention_weight_max": 29.26120948791504,
      "activations/layer21_attention_weight_min": -20.77604866027832,
      "activations/layer22_attention_weight_max": 44.654117584228516,
      "activations/layer22_attention_weight_min": -28.684860229492188,
      "activations/layer23_attention_weight_max": 25.936439514160156,
      "activations/layer23_attention_weight_min": -16.220386505126953,
      "activations/layer2_attention_weight_max": 28.60055923461914,
      "activations/layer2_attention_weight_min": -28.22993278503418,
      "activations/layer3_attention_weight_max": 70.43358612060547,
      "activations/layer3_attention_weight_min": -72.04676055908203,
      "activations/layer4_attention_weight_max": 127.02754211425781,
      "activations/layer4_attention_weight_min": -116.31792449951172,
      "activations/layer5_attention_weight_max": 110.50239562988281,
      "activations/layer5_attention_weight_min": -95.28401184082031,
      "activations/layer6_attention_weight_max": 58.92078399658203,
      "activations/layer6_attention_weight_min": -53.51006317138672,
      "activations/layer7_attention_weight_max": 64.63982391357422,
      "activations/layer7_attention_weight_min": -59.758705139160156,
      "activations/layer8_attention_weight_max": 45.99384689331055,
      "activations/layer8_attention_weight_min": -41.054508209228516,
      "activations/layer9_attention_weight_max": 57.062435150146484,
      "activations/layer9_attention_weight_min": -50.03334426879883,
      "epoch": 2.06,
      "learning_rate": 0.00013810454545454544,
      "loss": 2.921,
      "step": 35450
    },
    {
      "activations/layer0_attention_weight_max": 15.60058879852295,
      "activations/layer0_attention_weight_min": -10.966361999511719,
      "activations/layer10_attention_weight_max": 52.888668060302734,
      "activations/layer10_attention_weight_min": -42.01261520385742,
      "activations/layer11_attention_weight_max": 41.93389129638672,
      "activations/layer11_attention_weight_min": -33.84523010253906,
      "activations/layer12_attention_weight_max": 84.30333709716797,
      "activations/layer12_attention_weight_min": -41.01068115234375,
      "activations/layer13_attention_weight_max": 79.4364242553711,
      "activations/layer13_attention_weight_min": -41.15684127807617,
      "activations/layer14_attention_weight_max": 122.76577758789062,
      "activations/layer14_attention_weight_min": -71.48564147949219,
      "activations/layer15_attention_weight_max": 60.22512435913086,
      "activations/layer15_attention_weight_min": -37.82939529418945,
      "activations/layer16_attention_weight_max": 66.79036712646484,
      "activations/layer16_attention_weight_min": -35.940345764160156,
      "activations/layer17_attention_weight_max": 61.55234909057617,
      "activations/layer17_attention_weight_min": -28.417200088500977,
      "activations/layer18_attention_weight_max": 39.4401969909668,
      "activations/layer18_attention_weight_min": -18.81294059753418,
      "activations/layer19_attention_weight_max": 69.60307312011719,
      "activations/layer19_attention_weight_min": -33.72325897216797,
      "activations/layer1_attention_weight_max": 17.490224838256836,
      "activations/layer1_attention_weight_min": -10.593789100646973,
      "activations/layer20_attention_weight_max": 50.1292724609375,
      "activations/layer20_attention_weight_min": -23.0708065032959,
      "activations/layer21_attention_weight_max": 43.603912353515625,
      "activations/layer21_attention_weight_min": -21.75272560119629,
      "activations/layer22_attention_weight_max": 59.91187286376953,
      "activations/layer22_attention_weight_min": -25.60358428955078,
      "activations/layer23_attention_weight_max": 35.153419494628906,
      "activations/layer23_attention_weight_min": -16.072208404541016,
      "activations/layer2_attention_weight_max": 28.698081970214844,
      "activations/layer2_attention_weight_min": -28.334754943847656,
      "activations/layer3_attention_weight_max": 71.63796997070312,
      "activations/layer3_attention_weight_min": -77.25008392333984,
      "activations/layer4_attention_weight_max": 134.3154754638672,
      "activations/layer4_attention_weight_min": -119.9806900024414,
      "activations/layer5_attention_weight_max": 120.51234436035156,
      "activations/layer5_attention_weight_min": -100.366455078125,
      "activations/layer6_attention_weight_max": 64.84498596191406,
      "activations/layer6_attention_weight_min": -54.30017852783203,
      "activations/layer7_attention_weight_max": 68.64707946777344,
      "activations/layer7_attention_weight_min": -64.1671371459961,
      "activations/layer8_attention_weight_max": 48.57503890991211,
      "activations/layer8_attention_weight_min": -39.802093505859375,
      "activations/layer9_attention_weight_max": 69.35733032226562,
      "activations/layer9_attention_weight_min": -62.27410888671875,
      "epoch": 2.06,
      "learning_rate": 0.00013808560606060606,
      "loss": 2.9296,
      "step": 35500
    },
    {
      "activations/layer0_attention_weight_max": 16.461807250976562,
      "activations/layer0_attention_weight_min": -11.21514892578125,
      "activations/layer10_attention_weight_max": 51.335201263427734,
      "activations/layer10_attention_weight_min": -44.319480895996094,
      "activations/layer11_attention_weight_max": 42.952110290527344,
      "activations/layer11_attention_weight_min": -36.62724304199219,
      "activations/layer12_attention_weight_max": 41.46034622192383,
      "activations/layer12_attention_weight_min": -27.513595581054688,
      "activations/layer13_attention_weight_max": 66.79256439208984,
      "activations/layer13_attention_weight_min": -48.79841613769531,
      "activations/layer14_attention_weight_max": 85.04045867919922,
      "activations/layer14_attention_weight_min": -62.93132019042969,
      "activations/layer15_attention_weight_max": 55.40738296508789,
      "activations/layer15_attention_weight_min": -36.088504791259766,
      "activations/layer16_attention_weight_max": 54.234737396240234,
      "activations/layer16_attention_weight_min": -40.65000915527344,
      "activations/layer17_attention_weight_max": 52.54950714111328,
      "activations/layer17_attention_weight_min": -30.350236892700195,
      "activations/layer18_attention_weight_max": 33.848175048828125,
      "activations/layer18_attention_weight_min": -19.847936630249023,
      "activations/layer19_attention_weight_max": 60.84221267700195,
      "activations/layer19_attention_weight_min": -34.27738571166992,
      "activations/layer1_attention_weight_max": 17.572368621826172,
      "activations/layer1_attention_weight_min": -11.220376968383789,
      "activations/layer20_attention_weight_max": 42.43582534790039,
      "activations/layer20_attention_weight_min": -22.744298934936523,
      "activations/layer21_attention_weight_max": 38.54901885986328,
      "activations/layer21_attention_weight_min": -19.12824821472168,
      "activations/layer22_attention_weight_max": 52.03542709350586,
      "activations/layer22_attention_weight_min": -27.563264846801758,
      "activations/layer23_attention_weight_max": 30.17937469482422,
      "activations/layer23_attention_weight_min": -15.386951446533203,
      "activations/layer2_attention_weight_max": 29.219404220581055,
      "activations/layer2_attention_weight_min": -27.963224411010742,
      "activations/layer3_attention_weight_max": 75.28462982177734,
      "activations/layer3_attention_weight_min": -74.37018585205078,
      "activations/layer4_attention_weight_max": 131.87522888183594,
      "activations/layer4_attention_weight_min": -119.3865737915039,
      "activations/layer5_attention_weight_max": 119.89019012451172,
      "activations/layer5_attention_weight_min": -105.93395233154297,
      "activations/layer6_attention_weight_max": 68.6240463256836,
      "activations/layer6_attention_weight_min": -53.169429779052734,
      "activations/layer7_attention_weight_max": 70.1308822631836,
      "activations/layer7_attention_weight_min": -68.34393310546875,
      "activations/layer8_attention_weight_max": 49.07566833496094,
      "activations/layer8_attention_weight_min": -38.25644302368164,
      "activations/layer9_attention_weight_max": 86.29480743408203,
      "activations/layer9_attention_weight_min": -67.13872528076172,
      "epoch": 2.07,
      "learning_rate": 0.00013806666666666664,
      "loss": 2.9304,
      "step": 35550
    },
    {
      "activations/layer0_attention_weight_max": 16.01223373413086,
      "activations/layer0_attention_weight_min": -11.065739631652832,
      "activations/layer10_attention_weight_max": 49.80577087402344,
      "activations/layer10_attention_weight_min": -40.31541061401367,
      "activations/layer11_attention_weight_max": 40.0816650390625,
      "activations/layer11_attention_weight_min": -32.872589111328125,
      "activations/layer12_attention_weight_max": 39.494903564453125,
      "activations/layer12_attention_weight_min": -29.13009262084961,
      "activations/layer13_attention_weight_max": 61.11616897583008,
      "activations/layer13_attention_weight_min": -46.155738830566406,
      "activations/layer14_attention_weight_max": 71.02188873291016,
      "activations/layer14_attention_weight_min": -55.417240142822266,
      "activations/layer15_attention_weight_max": 51.452293395996094,
      "activations/layer15_attention_weight_min": -34.57158279418945,
      "activations/layer16_attention_weight_max": 50.79315185546875,
      "activations/layer16_attention_weight_min": -35.7431755065918,
      "activations/layer17_attention_weight_max": 48.60689163208008,
      "activations/layer17_attention_weight_min": -28.290922164916992,
      "activations/layer18_attention_weight_max": 30.113605499267578,
      "activations/layer18_attention_weight_min": -17.82058334350586,
      "activations/layer19_attention_weight_max": 53.36947250366211,
      "activations/layer19_attention_weight_min": -32.48344039916992,
      "activations/layer1_attention_weight_max": 17.404470443725586,
      "activations/layer1_attention_weight_min": -11.216257095336914,
      "activations/layer20_attention_weight_max": 37.231544494628906,
      "activations/layer20_attention_weight_min": -27.0246639251709,
      "activations/layer21_attention_weight_max": 35.586143493652344,
      "activations/layer21_attention_weight_min": -19.73917579650879,
      "activations/layer22_attention_weight_max": 43.27764892578125,
      "activations/layer22_attention_weight_min": -24.59693145751953,
      "activations/layer23_attention_weight_max": 25.69305992126465,
      "activations/layer23_attention_weight_min": -16.829240798950195,
      "activations/layer2_attention_weight_max": 28.680892944335938,
      "activations/layer2_attention_weight_min": -28.638656616210938,
      "activations/layer3_attention_weight_max": 70.89739227294922,
      "activations/layer3_attention_weight_min": -73.20472717285156,
      "activations/layer4_attention_weight_max": 122.04898834228516,
      "activations/layer4_attention_weight_min": -115.74666595458984,
      "activations/layer5_attention_weight_max": 106.3469467163086,
      "activations/layer5_attention_weight_min": -98.34209442138672,
      "activations/layer6_attention_weight_max": 59.19856262207031,
      "activations/layer6_attention_weight_min": -50.766597747802734,
      "activations/layer7_attention_weight_max": 64.90272521972656,
      "activations/layer7_attention_weight_min": -63.040199279785156,
      "activations/layer8_attention_weight_max": 45.17530822753906,
      "activations/layer8_attention_weight_min": -39.80286407470703,
      "activations/layer9_attention_weight_max": 63.135189056396484,
      "activations/layer9_attention_weight_min": -52.19929885864258,
      "epoch": 2.07,
      "learning_rate": 0.00013804772727272726,
      "loss": 2.9239,
      "step": 35600
    },
    {
      "activations/layer0_attention_weight_max": 16.555622100830078,
      "activations/layer0_attention_weight_min": -11.391568183898926,
      "activations/layer10_attention_weight_max": 53.353973388671875,
      "activations/layer10_attention_weight_min": -43.464073181152344,
      "activations/layer11_attention_weight_max": 39.93370056152344,
      "activations/layer11_attention_weight_min": -37.07469177246094,
      "activations/layer12_attention_weight_max": 40.28835678100586,
      "activations/layer12_attention_weight_min": -29.550891876220703,
      "activations/layer13_attention_weight_max": 51.2401237487793,
      "activations/layer13_attention_weight_min": -40.32586669921875,
      "activations/layer14_attention_weight_max": 82.14556884765625,
      "activations/layer14_attention_weight_min": -69.64029693603516,
      "activations/layer15_attention_weight_max": 50.79476547241211,
      "activations/layer15_attention_weight_min": -39.61069107055664,
      "activations/layer16_attention_weight_max": 50.61762237548828,
      "activations/layer16_attention_weight_min": -39.814701080322266,
      "activations/layer17_attention_weight_max": 45.15303039550781,
      "activations/layer17_attention_weight_min": -29.466508865356445,
      "activations/layer18_attention_weight_max": 27.26307487487793,
      "activations/layer18_attention_weight_min": -18.60798454284668,
      "activations/layer19_attention_weight_max": 49.03531265258789,
      "activations/layer19_attention_weight_min": -33.29751205444336,
      "activations/layer1_attention_weight_max": 16.593442916870117,
      "activations/layer1_attention_weight_min": -11.068618774414062,
      "activations/layer20_attention_weight_max": 35.47861862182617,
      "activations/layer20_attention_weight_min": -26.20842933654785,
      "activations/layer21_attention_weight_max": 33.8785285949707,
      "activations/layer21_attention_weight_min": -21.35133934020996,
      "activations/layer22_attention_weight_max": 46.52762222290039,
      "activations/layer22_attention_weight_min": -29.262231826782227,
      "activations/layer23_attention_weight_max": 26.104896545410156,
      "activations/layer23_attention_weight_min": -15.606473922729492,
      "activations/layer2_attention_weight_max": 28.67244529724121,
      "activations/layer2_attention_weight_min": -28.146469116210938,
      "activations/layer3_attention_weight_max": 72.28117370605469,
      "activations/layer3_attention_weight_min": -76.40380859375,
      "activations/layer4_attention_weight_max": 128.54373168945312,
      "activations/layer4_attention_weight_min": -123.89330291748047,
      "activations/layer5_attention_weight_max": 108.1900863647461,
      "activations/layer5_attention_weight_min": -99.18133544921875,
      "activations/layer6_attention_weight_max": 59.15264892578125,
      "activations/layer6_attention_weight_min": -54.282615661621094,
      "activations/layer7_attention_weight_max": 72.26951599121094,
      "activations/layer7_attention_weight_min": -62.166709899902344,
      "activations/layer8_attention_weight_max": 42.827674865722656,
      "activations/layer8_attention_weight_min": -40.23297119140625,
      "activations/layer9_attention_weight_max": 63.802284240722656,
      "activations/layer9_attention_weight_min": -48.461578369140625,
      "epoch": 2.07,
      "learning_rate": 0.00013802878787878787,
      "loss": 2.9307,
      "step": 35650
    },
    {
      "activations/layer0_attention_weight_max": 15.413753509521484,
      "activations/layer0_attention_weight_min": -11.328131675720215,
      "activations/layer10_attention_weight_max": 55.33323287963867,
      "activations/layer10_attention_weight_min": -43.3853645324707,
      "activations/layer11_attention_weight_max": 38.14875793457031,
      "activations/layer11_attention_weight_min": -35.521575927734375,
      "activations/layer12_attention_weight_max": 37.381587982177734,
      "activations/layer12_attention_weight_min": -28.649744033813477,
      "activations/layer13_attention_weight_max": 54.537879943847656,
      "activations/layer13_attention_weight_min": -37.12384796142578,
      "activations/layer14_attention_weight_max": 84.13947296142578,
      "activations/layer14_attention_weight_min": -60.820472717285156,
      "activations/layer15_attention_weight_max": 51.44475555419922,
      "activations/layer15_attention_weight_min": -37.82523727416992,
      "activations/layer16_attention_weight_max": 52.0727653503418,
      "activations/layer16_attention_weight_min": -35.206974029541016,
      "activations/layer17_attention_weight_max": 47.5517463684082,
      "activations/layer17_attention_weight_min": -29.889404296875,
      "activations/layer18_attention_weight_max": 28.454191207885742,
      "activations/layer18_attention_weight_min": -18.726160049438477,
      "activations/layer19_attention_weight_max": 57.05205154418945,
      "activations/layer19_attention_weight_min": -31.9932861328125,
      "activations/layer1_attention_weight_max": 19.063085556030273,
      "activations/layer1_attention_weight_min": -11.177586555480957,
      "activations/layer20_attention_weight_max": 39.538230895996094,
      "activations/layer20_attention_weight_min": -25.38100814819336,
      "activations/layer21_attention_weight_max": 34.06962966918945,
      "activations/layer21_attention_weight_min": -20.92814826965332,
      "activations/layer22_attention_weight_max": 49.25699234008789,
      "activations/layer22_attention_weight_min": -24.733642578125,
      "activations/layer23_attention_weight_max": 30.446109771728516,
      "activations/layer23_attention_weight_min": -15.4218168258667,
      "activations/layer2_attention_weight_max": 28.978195190429688,
      "activations/layer2_attention_weight_min": -27.597837448120117,
      "activations/layer3_attention_weight_max": 73.6181640625,
      "activations/layer3_attention_weight_min": -76.65179443359375,
      "activations/layer4_attention_weight_max": 135.0791473388672,
      "activations/layer4_attention_weight_min": -120.2728042602539,
      "activations/layer5_attention_weight_max": 118.48529052734375,
      "activations/layer5_attention_weight_min": -103.03858947753906,
      "activations/layer6_attention_weight_max": 64.41708374023438,
      "activations/layer6_attention_weight_min": -54.0195198059082,
      "activations/layer7_attention_weight_max": 70.56906127929688,
      "activations/layer7_attention_weight_min": -67.15420532226562,
      "activations/layer8_attention_weight_max": 49.03008270263672,
      "activations/layer8_attention_weight_min": -40.4110221862793,
      "activations/layer9_attention_weight_max": 68.59123992919922,
      "activations/layer9_attention_weight_min": -53.4870719909668,
      "epoch": 2.07,
      "learning_rate": 0.00013800984848484846,
      "loss": 2.9402,
      "step": 35700
    },
    {
      "activations/layer0_attention_weight_max": 16.069232940673828,
      "activations/layer0_attention_weight_min": -11.082059860229492,
      "activations/layer10_attention_weight_max": 54.55373001098633,
      "activations/layer10_attention_weight_min": -40.54269790649414,
      "activations/layer11_attention_weight_max": 57.29724884033203,
      "activations/layer11_attention_weight_min": -42.56305694580078,
      "activations/layer12_attention_weight_max": 48.97096633911133,
      "activations/layer12_attention_weight_min": -29.93665313720703,
      "activations/layer13_attention_weight_max": 68.70504760742188,
      "activations/layer13_attention_weight_min": -44.33308792114258,
      "activations/layer14_attention_weight_max": 87.82563018798828,
      "activations/layer14_attention_weight_min": -62.84709930419922,
      "activations/layer15_attention_weight_max": 56.18609619140625,
      "activations/layer15_attention_weight_min": -35.191368103027344,
      "activations/layer16_attention_weight_max": 54.53063201904297,
      "activations/layer16_attention_weight_min": -37.69784927368164,
      "activations/layer17_attention_weight_max": 52.13984298706055,
      "activations/layer17_attention_weight_min": -28.81013298034668,
      "activations/layer18_attention_weight_max": 28.665470123291016,
      "activations/layer18_attention_weight_min": -21.27437400817871,
      "activations/layer19_attention_weight_max": 53.99635696411133,
      "activations/layer19_attention_weight_min": -31.24036979675293,
      "activations/layer1_attention_weight_max": 18.12148666381836,
      "activations/layer1_attention_weight_min": -10.786670684814453,
      "activations/layer20_attention_weight_max": 42.623897552490234,
      "activations/layer20_attention_weight_min": -23.888355255126953,
      "activations/layer21_attention_weight_max": 34.626129150390625,
      "activations/layer21_attention_weight_min": -19.607711791992188,
      "activations/layer22_attention_weight_max": 46.03721618652344,
      "activations/layer22_attention_weight_min": -25.761474609375,
      "activations/layer23_attention_weight_max": 24.6376953125,
      "activations/layer23_attention_weight_min": -16.165115356445312,
      "activations/layer2_attention_weight_max": 29.628597259521484,
      "activations/layer2_attention_weight_min": -30.292905807495117,
      "activations/layer3_attention_weight_max": 72.98285675048828,
      "activations/layer3_attention_weight_min": -75.48223876953125,
      "activations/layer4_attention_weight_max": 135.4212646484375,
      "activations/layer4_attention_weight_min": -129.55941772460938,
      "activations/layer5_attention_weight_max": 129.06927490234375,
      "activations/layer5_attention_weight_min": -109.3440170288086,
      "activations/layer6_attention_weight_max": 70.89120483398438,
      "activations/layer6_attention_weight_min": -57.64708709716797,
      "activations/layer7_attention_weight_max": 67.42609405517578,
      "activations/layer7_attention_weight_min": -66.42230224609375,
      "activations/layer8_attention_weight_max": 53.833099365234375,
      "activations/layer8_attention_weight_min": -43.922916412353516,
      "activations/layer9_attention_weight_max": 88.38225555419922,
      "activations/layer9_attention_weight_min": -66.99535369873047,
      "epoch": 2.08,
      "learning_rate": 0.00013799090909090908,
      "loss": 2.9355,
      "step": 35750
    },
    {
      "activations/layer0_attention_weight_max": 15.640941619873047,
      "activations/layer0_attention_weight_min": -11.08532428741455,
      "activations/layer10_attention_weight_max": 50.1417121887207,
      "activations/layer10_attention_weight_min": -40.730770111083984,
      "activations/layer11_attention_weight_max": 44.226531982421875,
      "activations/layer11_attention_weight_min": -36.50312805175781,
      "activations/layer12_attention_weight_max": 45.92941665649414,
      "activations/layer12_attention_weight_min": -32.5609130859375,
      "activations/layer13_attention_weight_max": 48.27482223510742,
      "activations/layer13_attention_weight_min": -39.94194030761719,
      "activations/layer14_attention_weight_max": 65.85366821289062,
      "activations/layer14_attention_weight_min": -55.07933807373047,
      "activations/layer15_attention_weight_max": 49.44877243041992,
      "activations/layer15_attention_weight_min": -40.06821823120117,
      "activations/layer16_attention_weight_max": 45.18046951293945,
      "activations/layer16_attention_weight_min": -35.123939514160156,
      "activations/layer17_attention_weight_max": 42.9222297668457,
      "activations/layer17_attention_weight_min": -31.265480041503906,
      "activations/layer18_attention_weight_max": 27.24053955078125,
      "activations/layer18_attention_weight_min": -18.58050537109375,
      "activations/layer19_attention_weight_max": 44.50631332397461,
      "activations/layer19_attention_weight_min": -32.904136657714844,
      "activations/layer1_attention_weight_max": 18.46401023864746,
      "activations/layer1_attention_weight_min": -10.965333938598633,
      "activations/layer20_attention_weight_max": 34.89407730102539,
      "activations/layer20_attention_weight_min": -26.66424560546875,
      "activations/layer21_attention_weight_max": 30.384014129638672,
      "activations/layer21_attention_weight_min": -18.787242889404297,
      "activations/layer22_attention_weight_max": 45.642005920410156,
      "activations/layer22_attention_weight_min": -30.079641342163086,
      "activations/layer23_attention_weight_max": 25.58719253540039,
      "activations/layer23_attention_weight_min": -17.87957763671875,
      "activations/layer2_attention_weight_max": 28.738178253173828,
      "activations/layer2_attention_weight_min": -27.605283737182617,
      "activations/layer3_attention_weight_max": 72.9759292602539,
      "activations/layer3_attention_weight_min": -74.4805679321289,
      "activations/layer4_attention_weight_max": 133.27024841308594,
      "activations/layer4_attention_weight_min": -120.6346206665039,
      "activations/layer5_attention_weight_max": 119.76750183105469,
      "activations/layer5_attention_weight_min": -102.46437072753906,
      "activations/layer6_attention_weight_max": 66.91722869873047,
      "activations/layer6_attention_weight_min": -55.38600158691406,
      "activations/layer7_attention_weight_max": 79.35340881347656,
      "activations/layer7_attention_weight_min": -64.57259368896484,
      "activations/layer8_attention_weight_max": 56.40037536621094,
      "activations/layer8_attention_weight_min": -41.96690368652344,
      "activations/layer9_attention_weight_max": 92.46379089355469,
      "activations/layer9_attention_weight_min": -70.4507064819336,
      "epoch": 2.08,
      "learning_rate": 0.0001379719696969697,
      "loss": 2.9202,
      "step": 35800
    },
    {
      "activations/layer0_attention_weight_max": 16.921354293823242,
      "activations/layer0_attention_weight_min": -11.138886451721191,
      "activations/layer10_attention_weight_max": 50.991539001464844,
      "activations/layer10_attention_weight_min": -43.88722610473633,
      "activations/layer11_attention_weight_max": 39.35443878173828,
      "activations/layer11_attention_weight_min": -34.253204345703125,
      "activations/layer12_attention_weight_max": 39.77328109741211,
      "activations/layer12_attention_weight_min": -27.93035316467285,
      "activations/layer13_attention_weight_max": 57.1090087890625,
      "activations/layer13_attention_weight_min": -40.2393913269043,
      "activations/layer14_attention_weight_max": 82.48841094970703,
      "activations/layer14_attention_weight_min": -62.335079193115234,
      "activations/layer15_attention_weight_max": 50.52111053466797,
      "activations/layer15_attention_weight_min": -38.634910583496094,
      "activations/layer16_attention_weight_max": 53.10082244873047,
      "activations/layer16_attention_weight_min": -35.6590461730957,
      "activations/layer17_attention_weight_max": 56.68857192993164,
      "activations/layer17_attention_weight_min": -30.427547454833984,
      "activations/layer18_attention_weight_max": 30.134925842285156,
      "activations/layer18_attention_weight_min": -19.560443878173828,
      "activations/layer19_attention_weight_max": 51.38808059692383,
      "activations/layer19_attention_weight_min": -31.15717887878418,
      "activations/layer1_attention_weight_max": 17.77986717224121,
      "activations/layer1_attention_weight_min": -10.911128044128418,
      "activations/layer20_attention_weight_max": 38.688411712646484,
      "activations/layer20_attention_weight_min": -26.822986602783203,
      "activations/layer21_attention_weight_max": 28.818191528320312,
      "activations/layer21_attention_weight_min": -21.079591751098633,
      "activations/layer22_attention_weight_max": 43.547645568847656,
      "activations/layer22_attention_weight_min": -25.22562599182129,
      "activations/layer23_attention_weight_max": 21.871421813964844,
      "activations/layer23_attention_weight_min": -15.029243469238281,
      "activations/layer2_attention_weight_max": 29.237857818603516,
      "activations/layer2_attention_weight_min": -28.766630172729492,
      "activations/layer3_attention_weight_max": 73.97901916503906,
      "activations/layer3_attention_weight_min": -74.62190246582031,
      "activations/layer4_attention_weight_max": 131.40821838378906,
      "activations/layer4_attention_weight_min": -125.4767074584961,
      "activations/layer5_attention_weight_max": 120.32797241210938,
      "activations/layer5_attention_weight_min": -105.13926696777344,
      "activations/layer6_attention_weight_max": 61.84511184692383,
      "activations/layer6_attention_weight_min": -54.56120681762695,
      "activations/layer7_attention_weight_max": 70.07757568359375,
      "activations/layer7_attention_weight_min": -63.335716247558594,
      "activations/layer8_attention_weight_max": 46.510738372802734,
      "activations/layer8_attention_weight_min": -39.1070442199707,
      "activations/layer9_attention_weight_max": 71.1817626953125,
      "activations/layer9_attention_weight_min": -51.28120803833008,
      "epoch": 2.08,
      "learning_rate": 0.00013795303030303028,
      "loss": 2.9393,
      "step": 35850
    },
    {
      "activations/layer0_attention_weight_max": 16.416189193725586,
      "activations/layer0_attention_weight_min": -11.779573440551758,
      "activations/layer10_attention_weight_max": 49.79010009765625,
      "activations/layer10_attention_weight_min": -42.73512649536133,
      "activations/layer11_attention_weight_max": 36.463348388671875,
      "activations/layer11_attention_weight_min": -33.812252044677734,
      "activations/layer12_attention_weight_max": 43.59414291381836,
      "activations/layer12_attention_weight_min": -30.11968421936035,
      "activations/layer13_attention_weight_max": 52.152076721191406,
      "activations/layer13_attention_weight_min": -39.88618469238281,
      "activations/layer14_attention_weight_max": 78.51751708984375,
      "activations/layer14_attention_weight_min": -63.10482406616211,
      "activations/layer15_attention_weight_max": 48.35094451904297,
      "activations/layer15_attention_weight_min": -37.87297058105469,
      "activations/layer16_attention_weight_max": 56.5580940246582,
      "activations/layer16_attention_weight_min": -38.51560974121094,
      "activations/layer17_attention_weight_max": 44.1327018737793,
      "activations/layer17_attention_weight_min": -29.10919952392578,
      "activations/layer18_attention_weight_max": 27.9075927734375,
      "activations/layer18_attention_weight_min": -18.455957412719727,
      "activations/layer19_attention_weight_max": 59.34135055541992,
      "activations/layer19_attention_weight_min": -32.48664474487305,
      "activations/layer1_attention_weight_max": 16.743982315063477,
      "activations/layer1_attention_weight_min": -11.020977020263672,
      "activations/layer20_attention_weight_max": 44.24211883544922,
      "activations/layer20_attention_weight_min": -23.458538055419922,
      "activations/layer21_attention_weight_max": 41.34086227416992,
      "activations/layer21_attention_weight_min": -20.94675636291504,
      "activations/layer22_attention_weight_max": 45.20037841796875,
      "activations/layer22_attention_weight_min": -29.451820373535156,
      "activations/layer23_attention_weight_max": 26.62423324584961,
      "activations/layer23_attention_weight_min": -17.564697265625,
      "activations/layer2_attention_weight_max": 29.339462280273438,
      "activations/layer2_attention_weight_min": -27.81183624267578,
      "activations/layer3_attention_weight_max": 71.91057586669922,
      "activations/layer3_attention_weight_min": -73.59796905517578,
      "activations/layer4_attention_weight_max": 131.50572204589844,
      "activations/layer4_attention_weight_min": -116.2314453125,
      "activations/layer5_attention_weight_max": 113.3861083984375,
      "activations/layer5_attention_weight_min": -100.5938949584961,
      "activations/layer6_attention_weight_max": 61.434547424316406,
      "activations/layer6_attention_weight_min": -55.66801834106445,
      "activations/layer7_attention_weight_max": 65.79900360107422,
      "activations/layer7_attention_weight_min": -62.6185302734375,
      "activations/layer8_attention_weight_max": 46.31772232055664,
      "activations/layer8_attention_weight_min": -37.99174499511719,
      "activations/layer9_attention_weight_max": 58.76941680908203,
      "activations/layer9_attention_weight_min": -53.648223876953125,
      "epoch": 2.09,
      "learning_rate": 0.00013793446969696967,
      "loss": 2.9343,
      "step": 35900
    },
    {
      "activations/layer0_attention_weight_max": 16.412614822387695,
      "activations/layer0_attention_weight_min": -11.288348197937012,
      "activations/layer10_attention_weight_max": 55.50708770751953,
      "activations/layer10_attention_weight_min": -45.795589447021484,
      "activations/layer11_attention_weight_max": 39.11679458618164,
      "activations/layer11_attention_weight_min": -32.137474060058594,
      "activations/layer12_attention_weight_max": 39.29926300048828,
      "activations/layer12_attention_weight_min": -27.96404457092285,
      "activations/layer13_attention_weight_max": 45.29002380371094,
      "activations/layer13_attention_weight_min": -36.73900604248047,
      "activations/layer14_attention_weight_max": 68.69483947753906,
      "activations/layer14_attention_weight_min": -50.43918991088867,
      "activations/layer15_attention_weight_max": 50.64963150024414,
      "activations/layer15_attention_weight_min": -38.66962432861328,
      "activations/layer16_attention_weight_max": 50.01767349243164,
      "activations/layer16_attention_weight_min": -34.528297424316406,
      "activations/layer17_attention_weight_max": 45.15162658691406,
      "activations/layer17_attention_weight_min": -28.552223205566406,
      "activations/layer18_attention_weight_max": 29.39663314819336,
      "activations/layer18_attention_weight_min": -19.55637550354004,
      "activations/layer19_attention_weight_max": 53.14225387573242,
      "activations/layer19_attention_weight_min": -32.73240280151367,
      "activations/layer1_attention_weight_max": 17.191692352294922,
      "activations/layer1_attention_weight_min": -11.70615005493164,
      "activations/layer20_attention_weight_max": 34.61981964111328,
      "activations/layer20_attention_weight_min": -23.91179084777832,
      "activations/layer21_attention_weight_max": 30.050537109375,
      "activations/layer21_attention_weight_min": -19.80862045288086,
      "activations/layer22_attention_weight_max": 45.92692947387695,
      "activations/layer22_attention_weight_min": -27.459333419799805,
      "activations/layer23_attention_weight_max": 26.32904815673828,
      "activations/layer23_attention_weight_min": -17.612445831298828,
      "activations/layer2_attention_weight_max": 28.220182418823242,
      "activations/layer2_attention_weight_min": -28.78491973876953,
      "activations/layer3_attention_weight_max": 68.15900421142578,
      "activations/layer3_attention_weight_min": -72.32633209228516,
      "activations/layer4_attention_weight_max": 121.57135009765625,
      "activations/layer4_attention_weight_min": -112.86441802978516,
      "activations/layer5_attention_weight_max": 109.24203491210938,
      "activations/layer5_attention_weight_min": -96.30000305175781,
      "activations/layer6_attention_weight_max": 57.96083068847656,
      "activations/layer6_attention_weight_min": -51.59709167480469,
      "activations/layer7_attention_weight_max": 70.20365905761719,
      "activations/layer7_attention_weight_min": -61.27410888671875,
      "activations/layer8_attention_weight_max": 42.22578811645508,
      "activations/layer8_attention_weight_min": -38.09773635864258,
      "activations/layer9_attention_weight_max": 60.47254180908203,
      "activations/layer9_attention_weight_min": -50.30641174316406,
      "epoch": 2.09,
      "learning_rate": 0.00013791553030303029,
      "loss": 2.9363,
      "step": 35950
    },
    {
      "activations/layer0_attention_weight_max": 17.15144157409668,
      "activations/layer0_attention_weight_min": -10.799001693725586,
      "activations/layer10_attention_weight_max": 55.30603790283203,
      "activations/layer10_attention_weight_min": -45.933067321777344,
      "activations/layer11_attention_weight_max": 41.98981857299805,
      "activations/layer11_attention_weight_min": -35.70529556274414,
      "activations/layer12_attention_weight_max": 37.64984130859375,
      "activations/layer12_attention_weight_min": -30.784317016601562,
      "activations/layer13_attention_weight_max": 55.47764587402344,
      "activations/layer13_attention_weight_min": -41.93986892700195,
      "activations/layer14_attention_weight_max": 81.79003143310547,
      "activations/layer14_attention_weight_min": -60.2801628112793,
      "activations/layer15_attention_weight_max": 52.05609130859375,
      "activations/layer15_attention_weight_min": -35.61640548706055,
      "activations/layer16_attention_weight_max": 50.77397155761719,
      "activations/layer16_attention_weight_min": -37.97400665283203,
      "activations/layer17_attention_weight_max": 48.6980094909668,
      "activations/layer17_attention_weight_min": -30.366455078125,
      "activations/layer18_attention_weight_max": 29.92556381225586,
      "activations/layer18_attention_weight_min": -19.682422637939453,
      "activations/layer19_attention_weight_max": 59.8544921875,
      "activations/layer19_attention_weight_min": -31.99491310119629,
      "activations/layer1_attention_weight_max": 18.213855743408203,
      "activations/layer1_attention_weight_min": -10.140085220336914,
      "activations/layer20_attention_weight_max": 46.43251037597656,
      "activations/layer20_attention_weight_min": -23.019704818725586,
      "activations/layer21_attention_weight_max": 38.255859375,
      "activations/layer21_attention_weight_min": -19.903776168823242,
      "activations/layer22_attention_weight_max": 52.930946350097656,
      "activations/layer22_attention_weight_min": -27.382707595825195,
      "activations/layer23_attention_weight_max": 25.979778289794922,
      "activations/layer23_attention_weight_min": -15.514838218688965,
      "activations/layer2_attention_weight_max": 28.467559814453125,
      "activations/layer2_attention_weight_min": -29.00349235534668,
      "activations/layer3_attention_weight_max": 72.25775909423828,
      "activations/layer3_attention_weight_min": -73.17695617675781,
      "activations/layer4_attention_weight_max": 138.5248260498047,
      "activations/layer4_attention_weight_min": -126.49330139160156,
      "activations/layer5_attention_weight_max": 125.67766571044922,
      "activations/layer5_attention_weight_min": -103.50240325927734,
      "activations/layer6_attention_weight_max": 71.80216979980469,
      "activations/layer6_attention_weight_min": -56.00879669189453,
      "activations/layer7_attention_weight_max": 70.88362121582031,
      "activations/layer7_attention_weight_min": -65.82705688476562,
      "activations/layer8_attention_weight_max": 54.5933837890625,
      "activations/layer8_attention_weight_min": -47.069759368896484,
      "activations/layer9_attention_weight_max": 77.55392456054688,
      "activations/layer9_attention_weight_min": -58.59202194213867,
      "epoch": 2.09,
      "learning_rate": 0.00013789696969696968,
      "loss": 2.9474,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_loss": 2.8984375,
      "eval_runtime": 8.5565,
      "eval_samples_per_second": 501.842,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_openwebtext_loss": 2.8984375,
      "eval_openwebtext_ppl": 18.145770440989054,
      "eval_openwebtext_runtime": 8.5565,
      "eval_openwebtext_samples_per_second": 501.842,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_wikitext_loss": 3.171875,
      "eval_wikitext_ppl": 23.852165264858517,
      "eval_wikitext_runtime": 2.0226,
      "eval_wikitext_samples_per_second": 225.45,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_lambada_loss": 3.0625,
      "eval_lambada_ppl": 21.380942759123343,
      "eval_lambada_runtime": 9.6803,
      "eval_lambada_samples_per_second": 502.981,
      "step": 36000
    },
    {
      "activations/layer0_attention_weight_max": 16.549575805664062,
      "activations/layer0_attention_weight_min": -11.357848167419434,
      "activations/layer10_attention_weight_max": 52.95646286010742,
      "activations/layer10_attention_weight_min": -42.890106201171875,
      "activations/layer11_attention_weight_max": 41.766780853271484,
      "activations/layer11_attention_weight_min": -33.86420440673828,
      "activations/layer12_attention_weight_max": 35.10586929321289,
      "activations/layer12_attention_weight_min": -26.725814819335938,
      "activations/layer13_attention_weight_max": 51.77182388305664,
      "activations/layer13_attention_weight_min": -41.36690139770508,
      "activations/layer14_attention_weight_max": 73.65351104736328,
      "activations/layer14_attention_weight_min": -59.039554595947266,
      "activations/layer15_attention_weight_max": 49.45442581176758,
      "activations/layer15_attention_weight_min": -37.93110656738281,
      "activations/layer16_attention_weight_max": 47.628753662109375,
      "activations/layer16_attention_weight_min": -37.573524475097656,
      "activations/layer17_attention_weight_max": 53.19556427001953,
      "activations/layer17_attention_weight_min": -33.51384353637695,
      "activations/layer18_attention_weight_max": 28.32850456237793,
      "activations/layer18_attention_weight_min": -19.2656307220459,
      "activations/layer19_attention_weight_max": 55.052181243896484,
      "activations/layer19_attention_weight_min": -37.25004577636719,
      "activations/layer1_attention_weight_max": 17.545141220092773,
      "activations/layer1_attention_weight_min": -11.009490966796875,
      "activations/layer20_attention_weight_max": 35.18130874633789,
      "activations/layer20_attention_weight_min": -30.078250885009766,
      "activations/layer21_attention_weight_max": 25.661624908447266,
      "activations/layer21_attention_weight_min": -18.825477600097656,
      "activations/layer22_attention_weight_max": 40.367103576660156,
      "activations/layer22_attention_weight_min": -27.89518165588379,
      "activations/layer23_attention_weight_max": 25.69403648376465,
      "activations/layer23_attention_weight_min": -15.290353775024414,
      "activations/layer2_attention_weight_max": 29.407941818237305,
      "activations/layer2_attention_weight_min": -29.698108673095703,
      "activations/layer3_attention_weight_max": 69.78410339355469,
      "activations/layer3_attention_weight_min": -73.8860855102539,
      "activations/layer4_attention_weight_max": 125.76798248291016,
      "activations/layer4_attention_weight_min": -117.02428436279297,
      "activations/layer5_attention_weight_max": 110.64614868164062,
      "activations/layer5_attention_weight_min": -97.0025634765625,
      "activations/layer6_attention_weight_max": 59.709136962890625,
      "activations/layer6_attention_weight_min": -49.913002014160156,
      "activations/layer7_attention_weight_max": 66.6561050415039,
      "activations/layer7_attention_weight_min": -69.82949829101562,
      "activations/layer8_attention_weight_max": 43.76049041748047,
      "activations/layer8_attention_weight_min": -37.75613784790039,
      "activations/layer9_attention_weight_max": 64.43700408935547,
      "activations/layer9_attention_weight_min": -51.028236389160156,
      "epoch": 2.09,
      "learning_rate": 0.0001378780303030303,
      "loss": 2.9592,
      "step": 36050
    },
    {
      "activations/layer0_attention_weight_max": 16.53759765625,
      "activations/layer0_attention_weight_min": -11.199851036071777,
      "activations/layer10_attention_weight_max": 50.107112884521484,
      "activations/layer10_attention_weight_min": -40.35689163208008,
      "activations/layer11_attention_weight_max": 41.87358856201172,
      "activations/layer11_attention_weight_min": -34.299964904785156,
      "activations/layer12_attention_weight_max": 53.64717483520508,
      "activations/layer12_attention_weight_min": -35.54789733886719,
      "activations/layer13_attention_weight_max": 52.10694122314453,
      "activations/layer13_attention_weight_min": -37.823814392089844,
      "activations/layer14_attention_weight_max": 76.0549545288086,
      "activations/layer14_attention_weight_min": -55.56260681152344,
      "activations/layer15_attention_weight_max": 53.96921920776367,
      "activations/layer15_attention_weight_min": -36.9250602722168,
      "activations/layer16_attention_weight_max": 48.14569854736328,
      "activations/layer16_attention_weight_min": -32.01203918457031,
      "activations/layer17_attention_weight_max": 49.3477897644043,
      "activations/layer17_attention_weight_min": -30.396753311157227,
      "activations/layer18_attention_weight_max": 28.978792190551758,
      "activations/layer18_attention_weight_min": -17.652225494384766,
      "activations/layer19_attention_weight_max": 51.951412200927734,
      "activations/layer19_attention_weight_min": -29.959766387939453,
      "activations/layer1_attention_weight_max": 17.717735290527344,
      "activations/layer1_attention_weight_min": -12.574700355529785,
      "activations/layer20_attention_weight_max": 41.90068054199219,
      "activations/layer20_attention_weight_min": -24.58305549621582,
      "activations/layer21_attention_weight_max": 35.08366394042969,
      "activations/layer21_attention_weight_min": -19.55398178100586,
      "activations/layer22_attention_weight_max": 49.67349624633789,
      "activations/layer22_attention_weight_min": -26.64115333557129,
      "activations/layer23_attention_weight_max": 28.506546020507812,
      "activations/layer23_attention_weight_min": -17.19745635986328,
      "activations/layer2_attention_weight_max": 30.1179256439209,
      "activations/layer2_attention_weight_min": -30.008779525756836,
      "activations/layer3_attention_weight_max": 74.39794158935547,
      "activations/layer3_attention_weight_min": -76.39605712890625,
      "activations/layer4_attention_weight_max": 132.74169921875,
      "activations/layer4_attention_weight_min": -122.91484832763672,
      "activations/layer5_attention_weight_max": 117.40753173828125,
      "activations/layer5_attention_weight_min": -103.56721496582031,
      "activations/layer6_attention_weight_max": 62.73601150512695,
      "activations/layer6_attention_weight_min": -52.742652893066406,
      "activations/layer7_attention_weight_max": 66.81183624267578,
      "activations/layer7_attention_weight_min": -61.4171142578125,
      "activations/layer8_attention_weight_max": 54.08417892456055,
      "activations/layer8_attention_weight_min": -41.74834060668945,
      "activations/layer9_attention_weight_max": 78.06439971923828,
      "activations/layer9_attention_weight_min": -64.53133392333984,
      "epoch": 2.1,
      "learning_rate": 0.00013785909090909088,
      "loss": 2.9483,
      "step": 36100
    },
    {
      "activations/layer0_attention_weight_max": 17.12791633605957,
      "activations/layer0_attention_weight_min": -11.592042922973633,
      "activations/layer10_attention_weight_max": 59.2995719909668,
      "activations/layer10_attention_weight_min": -46.129661560058594,
      "activations/layer11_attention_weight_max": 44.929500579833984,
      "activations/layer11_attention_weight_min": -35.52458953857422,
      "activations/layer12_attention_weight_max": 42.08582305908203,
      "activations/layer12_attention_weight_min": -27.423948287963867,
      "activations/layer13_attention_weight_max": 52.81958770751953,
      "activations/layer13_attention_weight_min": -41.17128372192383,
      "activations/layer14_attention_weight_max": 71.70301818847656,
      "activations/layer14_attention_weight_min": -55.03034973144531,
      "activations/layer15_attention_weight_max": 49.13097381591797,
      "activations/layer15_attention_weight_min": -37.12044906616211,
      "activations/layer16_attention_weight_max": 43.22038269042969,
      "activations/layer16_attention_weight_min": -33.834293365478516,
      "activations/layer17_attention_weight_max": 45.52812576293945,
      "activations/layer17_attention_weight_min": -28.742469787597656,
      "activations/layer18_attention_weight_max": 23.511459350585938,
      "activations/layer18_attention_weight_min": -18.006526947021484,
      "activations/layer19_attention_weight_max": 47.02830123901367,
      "activations/layer19_attention_weight_min": -31.625904083251953,
      "activations/layer1_attention_weight_max": 18.121288299560547,
      "activations/layer1_attention_weight_min": -11.433991432189941,
      "activations/layer20_attention_weight_max": 37.07403564453125,
      "activations/layer20_attention_weight_min": -22.056974411010742,
      "activations/layer21_attention_weight_max": 29.036771774291992,
      "activations/layer21_attention_weight_min": -19.445785522460938,
      "activations/layer22_attention_weight_max": 41.53123474121094,
      "activations/layer22_attention_weight_min": -25.118249893188477,
      "activations/layer23_attention_weight_max": 23.799896240234375,
      "activations/layer23_attention_weight_min": -17.512104034423828,
      "activations/layer2_attention_weight_max": 28.519367218017578,
      "activations/layer2_attention_weight_min": -28.162389755249023,
      "activations/layer3_attention_weight_max": 70.97745513916016,
      "activations/layer3_attention_weight_min": -72.47843933105469,
      "activations/layer4_attention_weight_max": 131.22793579101562,
      "activations/layer4_attention_weight_min": -116.5338363647461,
      "activations/layer5_attention_weight_max": 120.27172088623047,
      "activations/layer5_attention_weight_min": -101.2964096069336,
      "activations/layer6_attention_weight_max": 62.37192916870117,
      "activations/layer6_attention_weight_min": -55.95989990234375,
      "activations/layer7_attention_weight_max": 66.72567749023438,
      "activations/layer7_attention_weight_min": -63.3235969543457,
      "activations/layer8_attention_weight_max": 44.63390350341797,
      "activations/layer8_attention_weight_min": -38.921085357666016,
      "activations/layer9_attention_weight_max": 68.57331085205078,
      "activations/layer9_attention_weight_min": -57.67047882080078,
      "epoch": 2.1,
      "learning_rate": 0.0001378401515151515,
      "loss": 2.9448,
      "step": 36150
    },
    {
      "activations/layer0_attention_weight_max": 14.084720611572266,
      "activations/layer0_attention_weight_min": -11.715434074401855,
      "activations/layer10_attention_weight_max": 55.71885299682617,
      "activations/layer10_attention_weight_min": -44.411922454833984,
      "activations/layer11_attention_weight_max": 41.37446594238281,
      "activations/layer11_attention_weight_min": -35.528526306152344,
      "activations/layer12_attention_weight_max": 36.76603317260742,
      "activations/layer12_attention_weight_min": -28.2763614654541,
      "activations/layer13_attention_weight_max": 49.515689849853516,
      "activations/layer13_attention_weight_min": -37.00584030151367,
      "activations/layer14_attention_weight_max": 93.14164733886719,
      "activations/layer14_attention_weight_min": -63.351966857910156,
      "activations/layer15_attention_weight_max": 52.56078338623047,
      "activations/layer15_attention_weight_min": -40.567962646484375,
      "activations/layer16_attention_weight_max": 50.79827880859375,
      "activations/layer16_attention_weight_min": -32.55205535888672,
      "activations/layer17_attention_weight_max": 46.54033279418945,
      "activations/layer17_attention_weight_min": -28.295167922973633,
      "activations/layer18_attention_weight_max": 31.64763069152832,
      "activations/layer18_attention_weight_min": -17.88697624206543,
      "activations/layer19_attention_weight_max": 59.78178024291992,
      "activations/layer19_attention_weight_min": -32.500640869140625,
      "activations/layer1_attention_weight_max": 18.510250091552734,
      "activations/layer1_attention_weight_min": -10.841434478759766,
      "activations/layer20_attention_weight_max": 42.958641052246094,
      "activations/layer20_attention_weight_min": -22.67670249938965,
      "activations/layer21_attention_weight_max": 31.38255500793457,
      "activations/layer21_attention_weight_min": -20.670188903808594,
      "activations/layer22_attention_weight_max": 48.344940185546875,
      "activations/layer22_attention_weight_min": -26.82979965209961,
      "activations/layer23_attention_weight_max": 25.963613510131836,
      "activations/layer23_attention_weight_min": -19.0699520111084,
      "activations/layer2_attention_weight_max": 28.726390838623047,
      "activations/layer2_attention_weight_min": -27.122032165527344,
      "activations/layer3_attention_weight_max": 73.29043579101562,
      "activations/layer3_attention_weight_min": -73.41907501220703,
      "activations/layer4_attention_weight_max": 134.53057861328125,
      "activations/layer4_attention_weight_min": -117.92069244384766,
      "activations/layer5_attention_weight_max": 119.60581970214844,
      "activations/layer5_attention_weight_min": -98.87679290771484,
      "activations/layer6_attention_weight_max": 63.48065948486328,
      "activations/layer6_attention_weight_min": -52.70409393310547,
      "activations/layer7_attention_weight_max": 74.5748062133789,
      "activations/layer7_attention_weight_min": -72.15338897705078,
      "activations/layer8_attention_weight_max": 47.535308837890625,
      "activations/layer8_attention_weight_min": -41.62332534790039,
      "activations/layer9_attention_weight_max": 72.00704193115234,
      "activations/layer9_attention_weight_min": -59.564788818359375,
      "epoch": 2.1,
      "learning_rate": 0.0001378212121212121,
      "loss": 2.9642,
      "step": 36200
    },
    {
      "activations/layer0_attention_weight_max": 16.641935348510742,
      "activations/layer0_attention_weight_min": -11.367044448852539,
      "activations/layer10_attention_weight_max": 53.27263641357422,
      "activations/layer10_attention_weight_min": -40.29351806640625,
      "activations/layer11_attention_weight_max": 42.747581481933594,
      "activations/layer11_attention_weight_min": -36.02900695800781,
      "activations/layer12_attention_weight_max": 80.70034790039062,
      "activations/layer12_attention_weight_min": -41.42190170288086,
      "activations/layer13_attention_weight_max": 59.83589553833008,
      "activations/layer13_attention_weight_min": -41.99036407470703,
      "activations/layer14_attention_weight_max": 93.27023315429688,
      "activations/layer14_attention_weight_min": -59.138694763183594,
      "activations/layer15_attention_weight_max": 56.95881652832031,
      "activations/layer15_attention_weight_min": -41.150089263916016,
      "activations/layer16_attention_weight_max": 56.418312072753906,
      "activations/layer16_attention_weight_min": -34.83722686767578,
      "activations/layer17_attention_weight_max": 52.89341354370117,
      "activations/layer17_attention_weight_min": -31.38393783569336,
      "activations/layer18_attention_weight_max": 33.2845573425293,
      "activations/layer18_attention_weight_min": -18.453052520751953,
      "activations/layer19_attention_weight_max": 60.6532096862793,
      "activations/layer19_attention_weight_min": -30.295194625854492,
      "activations/layer1_attention_weight_max": 18.378515243530273,
      "activations/layer1_attention_weight_min": -11.7177095413208,
      "activations/layer20_attention_weight_max": 44.299739837646484,
      "activations/layer20_attention_weight_min": -25.16357421875,
      "activations/layer21_attention_weight_max": 35.4182014465332,
      "activations/layer21_attention_weight_min": -22.579309463500977,
      "activations/layer22_attention_weight_max": 51.740867614746094,
      "activations/layer22_attention_weight_min": -26.38082504272461,
      "activations/layer23_attention_weight_max": 28.66823387145996,
      "activations/layer23_attention_weight_min": -15.133296966552734,
      "activations/layer2_attention_weight_max": 28.766393661499023,
      "activations/layer2_attention_weight_min": -27.446552276611328,
      "activations/layer3_attention_weight_max": 73.69601440429688,
      "activations/layer3_attention_weight_min": -76.62261199951172,
      "activations/layer4_attention_weight_max": 137.18064880371094,
      "activations/layer4_attention_weight_min": -122.8162612915039,
      "activations/layer5_attention_weight_max": 124.1917953491211,
      "activations/layer5_attention_weight_min": -99.66900634765625,
      "activations/layer6_attention_weight_max": 66.34688568115234,
      "activations/layer6_attention_weight_min": -53.10184097290039,
      "activations/layer7_attention_weight_max": 74.00030517578125,
      "activations/layer7_attention_weight_min": -64.8819808959961,
      "activations/layer8_attention_weight_max": 48.34055709838867,
      "activations/layer8_attention_weight_min": -39.419918060302734,
      "activations/layer9_attention_weight_max": 68.7065658569336,
      "activations/layer9_attention_weight_min": -54.19828414916992,
      "epoch": 2.11,
      "learning_rate": 0.00013780227272727272,
      "loss": 2.9442,
      "step": 36250
    },
    {
      "activations/layer0_attention_weight_max": 16.841140747070312,
      "activations/layer0_attention_weight_min": -11.437644004821777,
      "activations/layer10_attention_weight_max": 57.86398696899414,
      "activations/layer10_attention_weight_min": -41.33987045288086,
      "activations/layer11_attention_weight_max": 50.66082000732422,
      "activations/layer11_attention_weight_min": -37.94795227050781,
      "activations/layer12_attention_weight_max": 65.21427154541016,
      "activations/layer12_attention_weight_min": -37.15614700317383,
      "activations/layer13_attention_weight_max": 53.5423698425293,
      "activations/layer13_attention_weight_min": -37.835182189941406,
      "activations/layer14_attention_weight_max": 95.23224639892578,
      "activations/layer14_attention_weight_min": -72.79365539550781,
      "activations/layer15_attention_weight_max": 62.401790618896484,
      "activations/layer15_attention_weight_min": -37.40839385986328,
      "activations/layer16_attention_weight_max": 56.396305084228516,
      "activations/layer16_attention_weight_min": -37.930686950683594,
      "activations/layer17_attention_weight_max": 46.56322479248047,
      "activations/layer17_attention_weight_min": -30.77543830871582,
      "activations/layer18_attention_weight_max": 30.13436508178711,
      "activations/layer18_attention_weight_min": -18.214040756225586,
      "activations/layer19_attention_weight_max": 62.95461654663086,
      "activations/layer19_attention_weight_min": -30.887226104736328,
      "activations/layer1_attention_weight_max": 17.683494567871094,
      "activations/layer1_attention_weight_min": -10.926862716674805,
      "activations/layer20_attention_weight_max": 40.1347770690918,
      "activations/layer20_attention_weight_min": -23.574132919311523,
      "activations/layer21_attention_weight_max": 31.734649658203125,
      "activations/layer21_attention_weight_min": -18.5247802734375,
      "activations/layer22_attention_weight_max": 52.119598388671875,
      "activations/layer22_attention_weight_min": -27.104280471801758,
      "activations/layer23_attention_weight_max": 29.006465911865234,
      "activations/layer23_attention_weight_min": -16.773033142089844,
      "activations/layer2_attention_weight_max": 29.655834197998047,
      "activations/layer2_attention_weight_min": -28.23414421081543,
      "activations/layer3_attention_weight_max": 72.48043823242188,
      "activations/layer3_attention_weight_min": -77.98270416259766,
      "activations/layer4_attention_weight_max": 136.08628845214844,
      "activations/layer4_attention_weight_min": -120.2670669555664,
      "activations/layer5_attention_weight_max": 120.05254364013672,
      "activations/layer5_attention_weight_min": -96.42037200927734,
      "activations/layer6_attention_weight_max": 68.20628356933594,
      "activations/layer6_attention_weight_min": -53.77783966064453,
      "activations/layer7_attention_weight_max": 82.59578704833984,
      "activations/layer7_attention_weight_min": -63.485225677490234,
      "activations/layer8_attention_weight_max": 53.33759307861328,
      "activations/layer8_attention_weight_min": -41.8438835144043,
      "activations/layer9_attention_weight_max": 76.82879638671875,
      "activations/layer9_attention_weight_min": -50.51420974731445,
      "epoch": 2.11,
      "learning_rate": 0.0001377833333333333,
      "loss": 2.9448,
      "step": 36300
    },
    {
      "activations/layer0_attention_weight_max": 15.54227352142334,
      "activations/layer0_attention_weight_min": -11.471248626708984,
      "activations/layer10_attention_weight_max": 58.63600540161133,
      "activations/layer10_attention_weight_min": -46.81718826293945,
      "activations/layer11_attention_weight_max": 43.53774642944336,
      "activations/layer11_attention_weight_min": -36.00553512573242,
      "activations/layer12_attention_weight_max": 40.13958740234375,
      "activations/layer12_attention_weight_min": -24.8341007232666,
      "activations/layer13_attention_weight_max": 55.41950225830078,
      "activations/layer13_attention_weight_min": -38.8427848815918,
      "activations/layer14_attention_weight_max": 74.75121307373047,
      "activations/layer14_attention_weight_min": -57.58539962768555,
      "activations/layer15_attention_weight_max": 56.36567687988281,
      "activations/layer15_attention_weight_min": -40.36330795288086,
      "activations/layer16_attention_weight_max": 51.6208381652832,
      "activations/layer16_attention_weight_min": -33.68763732910156,
      "activations/layer17_attention_weight_max": 55.04672622680664,
      "activations/layer17_attention_weight_min": -30.96026039123535,
      "activations/layer18_attention_weight_max": 32.14970397949219,
      "activations/layer18_attention_weight_min": -19.23066520690918,
      "activations/layer19_attention_weight_max": 59.0721435546875,
      "activations/layer19_attention_weight_min": -33.53257369995117,
      "activations/layer1_attention_weight_max": 17.078397750854492,
      "activations/layer1_attention_weight_min": -11.578280448913574,
      "activations/layer20_attention_weight_max": 43.37017059326172,
      "activations/layer20_attention_weight_min": -27.21514892578125,
      "activations/layer21_attention_weight_max": 31.34779930114746,
      "activations/layer21_attention_weight_min": -22.86236000061035,
      "activations/layer22_attention_weight_max": 50.190223693847656,
      "activations/layer22_attention_weight_min": -27.746112823486328,
      "activations/layer23_attention_weight_max": 26.738515853881836,
      "activations/layer23_attention_weight_min": -16.317304611206055,
      "activations/layer2_attention_weight_max": 29.635061264038086,
      "activations/layer2_attention_weight_min": -30.142202377319336,
      "activations/layer3_attention_weight_max": 72.74130249023438,
      "activations/layer3_attention_weight_min": -75.74750518798828,
      "activations/layer4_attention_weight_max": 133.34153747558594,
      "activations/layer4_attention_weight_min": -122.36773681640625,
      "activations/layer5_attention_weight_max": 121.8731918334961,
      "activations/layer5_attention_weight_min": -99.00465393066406,
      "activations/layer6_attention_weight_max": 67.43611145019531,
      "activations/layer6_attention_weight_min": -52.09560012817383,
      "activations/layer7_attention_weight_max": 67.84662628173828,
      "activations/layer7_attention_weight_min": -62.03413009643555,
      "activations/layer8_attention_weight_max": 48.105613708496094,
      "activations/layer8_attention_weight_min": -38.804317474365234,
      "activations/layer9_attention_weight_max": 74.2526626586914,
      "activations/layer9_attention_weight_min": -59.627803802490234,
      "epoch": 2.11,
      "learning_rate": 0.00013776439393939393,
      "loss": 2.9294,
      "step": 36350
    },
    {
      "activations/layer0_attention_weight_max": 15.311384201049805,
      "activations/layer0_attention_weight_min": -11.324350357055664,
      "activations/layer10_attention_weight_max": 49.12974548339844,
      "activations/layer10_attention_weight_min": -41.42680740356445,
      "activations/layer11_attention_weight_max": 38.281761169433594,
      "activations/layer11_attention_weight_min": -35.15449905395508,
      "activations/layer12_attention_weight_max": 39.34526443481445,
      "activations/layer12_attention_weight_min": -27.19931983947754,
      "activations/layer13_attention_weight_max": 51.46525955200195,
      "activations/layer13_attention_weight_min": -35.209896087646484,
      "activations/layer14_attention_weight_max": 72.67098236083984,
      "activations/layer14_attention_weight_min": -55.08612823486328,
      "activations/layer15_attention_weight_max": 50.46419143676758,
      "activations/layer15_attention_weight_min": -36.26007080078125,
      "activations/layer16_attention_weight_max": 44.703189849853516,
      "activations/layer16_attention_weight_min": -33.90378189086914,
      "activations/layer17_attention_weight_max": 44.24876403808594,
      "activations/layer17_attention_weight_min": -27.860681533813477,
      "activations/layer18_attention_weight_max": 27.676809310913086,
      "activations/layer18_attention_weight_min": -18.052059173583984,
      "activations/layer19_attention_weight_max": 44.88920211791992,
      "activations/layer19_attention_weight_min": -29.053564071655273,
      "activations/layer1_attention_weight_max": 17.169090270996094,
      "activations/layer1_attention_weight_min": -10.892261505126953,
      "activations/layer20_attention_weight_max": 33.93938064575195,
      "activations/layer20_attention_weight_min": -26.249555587768555,
      "activations/layer21_attention_weight_max": 32.149879455566406,
      "activations/layer21_attention_weight_min": -19.124916076660156,
      "activations/layer22_attention_weight_max": 40.063438415527344,
      "activations/layer22_attention_weight_min": -28.036937713623047,
      "activations/layer23_attention_weight_max": 24.093000411987305,
      "activations/layer23_attention_weight_min": -16.514328002929688,
      "activations/layer2_attention_weight_max": 30.060882568359375,
      "activations/layer2_attention_weight_min": -28.348617553710938,
      "activations/layer3_attention_weight_max": 70.30874633789062,
      "activations/layer3_attention_weight_min": -72.93650817871094,
      "activations/layer4_attention_weight_max": 125.11795806884766,
      "activations/layer4_attention_weight_min": -116.81426239013672,
      "activations/layer5_attention_weight_max": 109.5902099609375,
      "activations/layer5_attention_weight_min": -99.0184326171875,
      "activations/layer6_attention_weight_max": 61.4495849609375,
      "activations/layer6_attention_weight_min": -52.08451461791992,
      "activations/layer7_attention_weight_max": 64.92092895507812,
      "activations/layer7_attention_weight_min": -62.249210357666016,
      "activations/layer8_attention_weight_max": 45.00653076171875,
      "activations/layer8_attention_weight_min": -40.54460144042969,
      "activations/layer9_attention_weight_max": 61.2322998046875,
      "activations/layer9_attention_weight_min": -47.864959716796875,
      "epoch": 2.12,
      "learning_rate": 0.00013774545454545454,
      "loss": 2.9339,
      "step": 36400
    },
    {
      "activations/layer0_attention_weight_max": 16.090560913085938,
      "activations/layer0_attention_weight_min": -11.431954383850098,
      "activations/layer10_attention_weight_max": 52.29297637939453,
      "activations/layer10_attention_weight_min": -43.28506851196289,
      "activations/layer11_attention_weight_max": 44.56440353393555,
      "activations/layer11_attention_weight_min": -41.5001220703125,
      "activations/layer12_attention_weight_max": 43.38302993774414,
      "activations/layer12_attention_weight_min": -33.412391662597656,
      "activations/layer13_attention_weight_max": 46.19428253173828,
      "activations/layer13_attention_weight_min": -35.6409797668457,
      "activations/layer14_attention_weight_max": 71.15860748291016,
      "activations/layer14_attention_weight_min": -58.26750946044922,
      "activations/layer15_attention_weight_max": 48.95759582519531,
      "activations/layer15_attention_weight_min": -35.545494079589844,
      "activations/layer16_attention_weight_max": 46.37135314941406,
      "activations/layer16_attention_weight_min": -34.6019401550293,
      "activations/layer17_attention_weight_max": 46.01097869873047,
      "activations/layer17_attention_weight_min": -29.68049430847168,
      "activations/layer18_attention_weight_max": 29.76239585876465,
      "activations/layer18_attention_weight_min": -19.4145565032959,
      "activations/layer19_attention_weight_max": 51.0010986328125,
      "activations/layer19_attention_weight_min": -31.8703556060791,
      "activations/layer1_attention_weight_max": 16.858291625976562,
      "activations/layer1_attention_weight_min": -10.59708309173584,
      "activations/layer20_attention_weight_max": 39.774112701416016,
      "activations/layer20_attention_weight_min": -24.3715877532959,
      "activations/layer21_attention_weight_max": 33.53451919555664,
      "activations/layer21_attention_weight_min": -18.8005428314209,
      "activations/layer22_attention_weight_max": 48.543861389160156,
      "activations/layer22_attention_weight_min": -26.803173065185547,
      "activations/layer23_attention_weight_max": 25.15327262878418,
      "activations/layer23_attention_weight_min": -16.982887268066406,
      "activations/layer2_attention_weight_max": 27.946338653564453,
      "activations/layer2_attention_weight_min": -27.502307891845703,
      "activations/layer3_attention_weight_max": 73.62568664550781,
      "activations/layer3_attention_weight_min": -74.04673767089844,
      "activations/layer4_attention_weight_max": 129.4952392578125,
      "activations/layer4_attention_weight_min": -118.46590423583984,
      "activations/layer5_attention_weight_max": 114.2896728515625,
      "activations/layer5_attention_weight_min": -99.90888977050781,
      "activations/layer6_attention_weight_max": 63.89792251586914,
      "activations/layer6_attention_weight_min": -55.20659637451172,
      "activations/layer7_attention_weight_max": 71.21878051757812,
      "activations/layer7_attention_weight_min": -63.55524444580078,
      "activations/layer8_attention_weight_max": 45.27931213378906,
      "activations/layer8_attention_weight_min": -40.69428634643555,
      "activations/layer9_attention_weight_max": 67.60897064208984,
      "activations/layer9_attention_weight_min": -57.614356994628906,
      "epoch": 2.12,
      "learning_rate": 0.00013772651515151513,
      "loss": 2.9698,
      "step": 36450
    },
    {
      "activations/layer0_attention_weight_max": 16.27629852294922,
      "activations/layer0_attention_weight_min": -11.50102710723877,
      "activations/layer10_attention_weight_max": 53.8463249206543,
      "activations/layer10_attention_weight_min": -43.35258102416992,
      "activations/layer11_attention_weight_max": 58.49622344970703,
      "activations/layer11_attention_weight_min": -40.29666519165039,
      "activations/layer12_attention_weight_max": 65.32222747802734,
      "activations/layer12_attention_weight_min": -35.90534591674805,
      "activations/layer13_attention_weight_max": 52.58188247680664,
      "activations/layer13_attention_weight_min": -39.2514762878418,
      "activations/layer14_attention_weight_max": 74.1429214477539,
      "activations/layer14_attention_weight_min": -53.780818939208984,
      "activations/layer15_attention_weight_max": 54.616703033447266,
      "activations/layer15_attention_weight_min": -37.68901062011719,
      "activations/layer16_attention_weight_max": 51.25735855102539,
      "activations/layer16_attention_weight_min": -38.88640213012695,
      "activations/layer17_attention_weight_max": 47.470577239990234,
      "activations/layer17_attention_weight_min": -30.761369705200195,
      "activations/layer18_attention_weight_max": 26.769044876098633,
      "activations/layer18_attention_weight_min": -18.302658081054688,
      "activations/layer19_attention_weight_max": 53.29724884033203,
      "activations/layer19_attention_weight_min": -32.589515686035156,
      "activations/layer1_attention_weight_max": 17.292924880981445,
      "activations/layer1_attention_weight_min": -10.828926086425781,
      "activations/layer20_attention_weight_max": 40.95012283325195,
      "activations/layer20_attention_weight_min": -24.56427574157715,
      "activations/layer21_attention_weight_max": 29.15340805053711,
      "activations/layer21_attention_weight_min": -19.764955520629883,
      "activations/layer22_attention_weight_max": 44.94999313354492,
      "activations/layer22_attention_weight_min": -24.258867263793945,
      "activations/layer23_attention_weight_max": 25.4552001953125,
      "activations/layer23_attention_weight_min": -16.2220458984375,
      "activations/layer2_attention_weight_max": 28.2180233001709,
      "activations/layer2_attention_weight_min": -27.535762786865234,
      "activations/layer3_attention_weight_max": 71.8836669921875,
      "activations/layer3_attention_weight_min": -77.04142761230469,
      "activations/layer4_attention_weight_max": 125.45440673828125,
      "activations/layer4_attention_weight_min": -119.8615951538086,
      "activations/layer5_attention_weight_max": 112.36572265625,
      "activations/layer5_attention_weight_min": -101.87046813964844,
      "activations/layer6_attention_weight_max": 63.54289245605469,
      "activations/layer6_attention_weight_min": -54.1956672668457,
      "activations/layer7_attention_weight_max": 72.23477935791016,
      "activations/layer7_attention_weight_min": -70.40856170654297,
      "activations/layer8_attention_weight_max": 55.73190689086914,
      "activations/layer8_attention_weight_min": -44.14173889160156,
      "activations/layer9_attention_weight_max": 83.5320053100586,
      "activations/layer9_attention_weight_min": -61.72883224487305,
      "epoch": 2.12,
      "learning_rate": 0.00013770757575757575,
      "loss": 2.9449,
      "step": 36500
    },
    {
      "activations/layer0_attention_weight_max": 16.15938377380371,
      "activations/layer0_attention_weight_min": -11.479005813598633,
      "activations/layer10_attention_weight_max": 54.536956787109375,
      "activations/layer10_attention_weight_min": -41.356903076171875,
      "activations/layer11_attention_weight_max": 42.9522819519043,
      "activations/layer11_attention_weight_min": -33.7021369934082,
      "activations/layer12_attention_weight_max": 39.898040771484375,
      "activations/layer12_attention_weight_min": -30.21025848388672,
      "activations/layer13_attention_weight_max": 70.65304565429688,
      "activations/layer13_attention_weight_min": -53.833683013916016,
      "activations/layer14_attention_weight_max": 84.66874694824219,
      "activations/layer14_attention_weight_min": -64.09597778320312,
      "activations/layer15_attention_weight_max": 53.68276596069336,
      "activations/layer15_attention_weight_min": -37.52485275268555,
      "activations/layer16_attention_weight_max": 49.42987060546875,
      "activations/layer16_attention_weight_min": -34.28048324584961,
      "activations/layer17_attention_weight_max": 52.52756118774414,
      "activations/layer17_attention_weight_min": -31.151514053344727,
      "activations/layer18_attention_weight_max": 27.129459381103516,
      "activations/layer18_attention_weight_min": -18.951303482055664,
      "activations/layer19_attention_weight_max": 50.5755615234375,
      "activations/layer19_attention_weight_min": -31.955636978149414,
      "activations/layer1_attention_weight_max": 17.822898864746094,
      "activations/layer1_attention_weight_min": -10.968092918395996,
      "activations/layer20_attention_weight_max": 36.311004638671875,
      "activations/layer20_attention_weight_min": -25.7298583984375,
      "activations/layer21_attention_weight_max": 31.61504364013672,
      "activations/layer21_attention_weight_min": -21.376174926757812,
      "activations/layer22_attention_weight_max": 44.647422790527344,
      "activations/layer22_attention_weight_min": -28.310579299926758,
      "activations/layer23_attention_weight_max": 26.863622665405273,
      "activations/layer23_attention_weight_min": -17.509471893310547,
      "activations/layer2_attention_weight_max": 29.035306930541992,
      "activations/layer2_attention_weight_min": -28.61493682861328,
      "activations/layer3_attention_weight_max": 71.55824279785156,
      "activations/layer3_attention_weight_min": -73.34092712402344,
      "activations/layer4_attention_weight_max": 126.64812469482422,
      "activations/layer4_attention_weight_min": -119.70491027832031,
      "activations/layer5_attention_weight_max": 111.36947631835938,
      "activations/layer5_attention_weight_min": -97.83706665039062,
      "activations/layer6_attention_weight_max": 58.74136734008789,
      "activations/layer6_attention_weight_min": -55.732913970947266,
      "activations/layer7_attention_weight_max": 63.75795364379883,
      "activations/layer7_attention_weight_min": -66.58262634277344,
      "activations/layer8_attention_weight_max": 44.97362518310547,
      "activations/layer8_attention_weight_min": -38.51136016845703,
      "activations/layer9_attention_weight_max": 64.95274353027344,
      "activations/layer9_attention_weight_min": -52.25390625,
      "epoch": 2.12,
      "learning_rate": 0.00013768863636363636,
      "loss": 2.9554,
      "step": 36550
    },
    {
      "activations/layer0_attention_weight_max": 13.575352668762207,
      "activations/layer0_attention_weight_min": -11.480185508728027,
      "activations/layer10_attention_weight_max": 59.77283477783203,
      "activations/layer10_attention_weight_min": -43.17467498779297,
      "activations/layer11_attention_weight_max": 44.88601303100586,
      "activations/layer11_attention_weight_min": -32.76041030883789,
      "activations/layer12_attention_weight_max": 32.89647674560547,
      "activations/layer12_attention_weight_min": -26.74132537841797,
      "activations/layer13_attention_weight_max": 71.06587982177734,
      "activations/layer13_attention_weight_min": -37.92125701904297,
      "activations/layer14_attention_weight_max": 102.33965301513672,
      "activations/layer14_attention_weight_min": -55.52949142456055,
      "activations/layer15_attention_weight_max": 67.87728881835938,
      "activations/layer15_attention_weight_min": -35.870113372802734,
      "activations/layer16_attention_weight_max": 58.595890045166016,
      "activations/layer16_attention_weight_min": -34.65704345703125,
      "activations/layer17_attention_weight_max": 60.21138381958008,
      "activations/layer17_attention_weight_min": -28.53843879699707,
      "activations/layer18_attention_weight_max": 27.60693359375,
      "activations/layer18_attention_weight_min": -17.964323043823242,
      "activations/layer19_attention_weight_max": 54.99776077270508,
      "activations/layer19_attention_weight_min": -34.70353317260742,
      "activations/layer1_attention_weight_max": 18.050106048583984,
      "activations/layer1_attention_weight_min": -11.24291706085205,
      "activations/layer20_attention_weight_max": 45.64548873901367,
      "activations/layer20_attention_weight_min": -26.855358123779297,
      "activations/layer21_attention_weight_max": 35.679080963134766,
      "activations/layer21_attention_weight_min": -19.820571899414062,
      "activations/layer22_attention_weight_max": 53.371429443359375,
      "activations/layer22_attention_weight_min": -28.14130401611328,
      "activations/layer23_attention_weight_max": 26.34799575805664,
      "activations/layer23_attention_weight_min": -17.198148727416992,
      "activations/layer2_attention_weight_max": 28.78728485107422,
      "activations/layer2_attention_weight_min": -28.903104782104492,
      "activations/layer3_attention_weight_max": 75.45538330078125,
      "activations/layer3_attention_weight_min": -77.19205474853516,
      "activations/layer4_attention_weight_max": 132.6016082763672,
      "activations/layer4_attention_weight_min": -121.7551040649414,
      "activations/layer5_attention_weight_max": 122.0149154663086,
      "activations/layer5_attention_weight_min": -108.72453308105469,
      "activations/layer6_attention_weight_max": 68.7857666015625,
      "activations/layer6_attention_weight_min": -55.03682327270508,
      "activations/layer7_attention_weight_max": 77.4792709350586,
      "activations/layer7_attention_weight_min": -66.58549499511719,
      "activations/layer8_attention_weight_max": 54.03412628173828,
      "activations/layer8_attention_weight_min": -39.798797607421875,
      "activations/layer9_attention_weight_max": 63.22364044189453,
      "activations/layer9_attention_weight_min": -49.91413879394531,
      "epoch": 2.13,
      "learning_rate": 0.00013766969696969698,
      "loss": 2.9596,
      "step": 36600
    },
    {
      "activations/layer0_attention_weight_max": 16.84241485595703,
      "activations/layer0_attention_weight_min": -11.62701416015625,
      "activations/layer10_attention_weight_max": 46.13051986694336,
      "activations/layer10_attention_weight_min": -39.112003326416016,
      "activations/layer11_attention_weight_max": 35.60304260253906,
      "activations/layer11_attention_weight_min": -35.75286102294922,
      "activations/layer12_attention_weight_max": 36.62348937988281,
      "activations/layer12_attention_weight_min": -24.65239143371582,
      "activations/layer13_attention_weight_max": 45.657630920410156,
      "activations/layer13_attention_weight_min": -36.27064514160156,
      "activations/layer14_attention_weight_max": 69.39008331298828,
      "activations/layer14_attention_weight_min": -56.56043243408203,
      "activations/layer15_attention_weight_max": 51.41447067260742,
      "activations/layer15_attention_weight_min": -36.54418182373047,
      "activations/layer16_attention_weight_max": 48.90485763549805,
      "activations/layer16_attention_weight_min": -38.30316162109375,
      "activations/layer17_attention_weight_max": 42.046142578125,
      "activations/layer17_attention_weight_min": -29.428335189819336,
      "activations/layer18_attention_weight_max": 28.064960479736328,
      "activations/layer18_attention_weight_min": -18.004648208618164,
      "activations/layer19_attention_weight_max": 51.50807571411133,
      "activations/layer19_attention_weight_min": -32.4460563659668,
      "activations/layer1_attention_weight_max": 17.515832901000977,
      "activations/layer1_attention_weight_min": -10.702527046203613,
      "activations/layer20_attention_weight_max": 38.24155044555664,
      "activations/layer20_attention_weight_min": -23.506139755249023,
      "activations/layer21_attention_weight_max": 37.54805374145508,
      "activations/layer21_attention_weight_min": -23.53928565979004,
      "activations/layer22_attention_weight_max": 47.09333038330078,
      "activations/layer22_attention_weight_min": -26.491790771484375,
      "activations/layer23_attention_weight_max": 25.66161346435547,
      "activations/layer23_attention_weight_min": -13.45816421508789,
      "activations/layer2_attention_weight_max": 26.889114379882812,
      "activations/layer2_attention_weight_min": -26.777265548706055,
      "activations/layer3_attention_weight_max": 69.28451538085938,
      "activations/layer3_attention_weight_min": -73.85257720947266,
      "activations/layer4_attention_weight_max": 119.26210021972656,
      "activations/layer4_attention_weight_min": -118.3608169555664,
      "activations/layer5_attention_weight_max": 105.83037567138672,
      "activations/layer5_attention_weight_min": -94.65995788574219,
      "activations/layer6_attention_weight_max": 58.056644439697266,
      "activations/layer6_attention_weight_min": -51.51577377319336,
      "activations/layer7_attention_weight_max": 63.962303161621094,
      "activations/layer7_attention_weight_min": -64.65341186523438,
      "activations/layer8_attention_weight_max": 46.14783477783203,
      "activations/layer8_attention_weight_min": -37.52499771118164,
      "activations/layer9_attention_weight_max": 59.05987548828125,
      "activations/layer9_attention_weight_min": -51.066097259521484,
      "epoch": 2.13,
      "learning_rate": 0.00013765075757575756,
      "loss": 2.9469,
      "step": 36650
    },
    {
      "activations/layer0_attention_weight_max": 16.589298248291016,
      "activations/layer0_attention_weight_min": -11.568692207336426,
      "activations/layer10_attention_weight_max": 47.34024429321289,
      "activations/layer10_attention_weight_min": -41.630985260009766,
      "activations/layer11_attention_weight_max": 36.368003845214844,
      "activations/layer11_attention_weight_min": -35.54549026489258,
      "activations/layer12_attention_weight_max": 33.89552688598633,
      "activations/layer12_attention_weight_min": -29.076942443847656,
      "activations/layer13_attention_weight_max": 50.066734313964844,
      "activations/layer13_attention_weight_min": -38.89641571044922,
      "activations/layer14_attention_weight_max": 75.09222412109375,
      "activations/layer14_attention_weight_min": -60.603641510009766,
      "activations/layer15_attention_weight_max": 53.112247467041016,
      "activations/layer15_attention_weight_min": -36.77974319458008,
      "activations/layer16_attention_weight_max": 46.63182067871094,
      "activations/layer16_attention_weight_min": -33.425071716308594,
      "activations/layer17_attention_weight_max": 46.4121208190918,
      "activations/layer17_attention_weight_min": -32.74199295043945,
      "activations/layer18_attention_weight_max": 25.89174461364746,
      "activations/layer18_attention_weight_min": -18.310108184814453,
      "activations/layer19_attention_weight_max": 56.49202346801758,
      "activations/layer19_attention_weight_min": -34.462703704833984,
      "activations/layer1_attention_weight_max": 17.443601608276367,
      "activations/layer1_attention_weight_min": -11.355120658874512,
      "activations/layer20_attention_weight_max": 38.77933120727539,
      "activations/layer20_attention_weight_min": -26.138099670410156,
      "activations/layer21_attention_weight_max": 37.7333869934082,
      "activations/layer21_attention_weight_min": -21.38939666748047,
      "activations/layer22_attention_weight_max": 46.74089813232422,
      "activations/layer22_attention_weight_min": -28.478384017944336,
      "activations/layer23_attention_weight_max": 26.745384216308594,
      "activations/layer23_attention_weight_min": -18.14801025390625,
      "activations/layer2_attention_weight_max": 29.323184967041016,
      "activations/layer2_attention_weight_min": -28.9093017578125,
      "activations/layer3_attention_weight_max": 69.32340240478516,
      "activations/layer3_attention_weight_min": -73.57717895507812,
      "activations/layer4_attention_weight_max": 124.8777847290039,
      "activations/layer4_attention_weight_min": -112.90083312988281,
      "activations/layer5_attention_weight_max": 105.58268737792969,
      "activations/layer5_attention_weight_min": -98.84789276123047,
      "activations/layer6_attention_weight_max": 60.5878791809082,
      "activations/layer6_attention_weight_min": -54.27854537963867,
      "activations/layer7_attention_weight_max": 66.96475219726562,
      "activations/layer7_attention_weight_min": -62.240013122558594,
      "activations/layer8_attention_weight_max": 46.29267120361328,
      "activations/layer8_attention_weight_min": -42.259639739990234,
      "activations/layer9_attention_weight_max": 62.746673583984375,
      "activations/layer9_attention_weight_min": -58.13454818725586,
      "epoch": 2.13,
      "learning_rate": 0.00013763181818181818,
      "loss": 2.9449,
      "step": 36700
    },
    {
      "activations/layer0_attention_weight_max": 13.537504196166992,
      "activations/layer0_attention_weight_min": -11.40892219543457,
      "activations/layer10_attention_weight_max": 52.6339225769043,
      "activations/layer10_attention_weight_min": -42.94767379760742,
      "activations/layer11_attention_weight_max": 45.365013122558594,
      "activations/layer11_attention_weight_min": -39.981361389160156,
      "activations/layer12_attention_weight_max": 33.185813903808594,
      "activations/layer12_attention_weight_min": -27.348495483398438,
      "activations/layer13_attention_weight_max": 50.84553527832031,
      "activations/layer13_attention_weight_min": -40.04696273803711,
      "activations/layer14_attention_weight_max": 72.97840881347656,
      "activations/layer14_attention_weight_min": -56.607017517089844,
      "activations/layer15_attention_weight_max": 49.271018981933594,
      "activations/layer15_attention_weight_min": -39.23995590209961,
      "activations/layer16_attention_weight_max": 55.677520751953125,
      "activations/layer16_attention_weight_min": -40.54388427734375,
      "activations/layer17_attention_weight_max": 46.68025207519531,
      "activations/layer17_attention_weight_min": -28.976736068725586,
      "activations/layer18_attention_weight_max": 27.875652313232422,
      "activations/layer18_attention_weight_min": -18.860309600830078,
      "activations/layer19_attention_weight_max": 57.984046936035156,
      "activations/layer19_attention_weight_min": -32.442726135253906,
      "activations/layer1_attention_weight_max": 17.501211166381836,
      "activations/layer1_attention_weight_min": -10.3893461227417,
      "activations/layer20_attention_weight_max": 42.108909606933594,
      "activations/layer20_attention_weight_min": -23.7740535736084,
      "activations/layer21_attention_weight_max": 37.385467529296875,
      "activations/layer21_attention_weight_min": -21.298839569091797,
      "activations/layer22_attention_weight_max": 44.99033737182617,
      "activations/layer22_attention_weight_min": -24.75244903564453,
      "activations/layer23_attention_weight_max": 26.42336654663086,
      "activations/layer23_attention_weight_min": -16.783578872680664,
      "activations/layer2_attention_weight_max": 28.801435470581055,
      "activations/layer2_attention_weight_min": -29.852317810058594,
      "activations/layer3_attention_weight_max": 72.5423355102539,
      "activations/layer3_attention_weight_min": -76.04820251464844,
      "activations/layer4_attention_weight_max": 127.99836730957031,
      "activations/layer4_attention_weight_min": -125.27375793457031,
      "activations/layer5_attention_weight_max": 111.63848114013672,
      "activations/layer5_attention_weight_min": -101.61470031738281,
      "activations/layer6_attention_weight_max": 60.48050308227539,
      "activations/layer6_attention_weight_min": -53.45192337036133,
      "activations/layer7_attention_weight_max": 65.7710952758789,
      "activations/layer7_attention_weight_min": -62.71861267089844,
      "activations/layer8_attention_weight_max": 44.236854553222656,
      "activations/layer8_attention_weight_min": -38.67470169067383,
      "activations/layer9_attention_weight_max": 61.68636703491211,
      "activations/layer9_attention_weight_min": -49.14432144165039,
      "epoch": 2.14,
      "learning_rate": 0.0001376128787878788,
      "loss": 2.9363,
      "step": 36750
    },
    {
      "activations/layer0_attention_weight_max": 14.939165115356445,
      "activations/layer0_attention_weight_min": -11.657513618469238,
      "activations/layer10_attention_weight_max": 50.16598892211914,
      "activations/layer10_attention_weight_min": -41.24593734741211,
      "activations/layer11_attention_weight_max": 38.246734619140625,
      "activations/layer11_attention_weight_min": -36.1270637512207,
      "activations/layer12_attention_weight_max": 46.497798919677734,
      "activations/layer12_attention_weight_min": -30.260360717773438,
      "activations/layer13_attention_weight_max": 52.055992126464844,
      "activations/layer13_attention_weight_min": -38.90914535522461,
      "activations/layer14_attention_weight_max": 77.29154205322266,
      "activations/layer14_attention_weight_min": -58.557411193847656,
      "activations/layer15_attention_weight_max": 48.83028030395508,
      "activations/layer15_attention_weight_min": -36.49070358276367,
      "activations/layer16_attention_weight_max": 49.11417007446289,
      "activations/layer16_attention_weight_min": -33.535919189453125,
      "activations/layer17_attention_weight_max": 43.093509674072266,
      "activations/layer17_attention_weight_min": -29.57676124572754,
      "activations/layer18_attention_weight_max": 25.994165420532227,
      "activations/layer18_attention_weight_min": -18.028337478637695,
      "activations/layer19_attention_weight_max": 46.3578987121582,
      "activations/layer19_attention_weight_min": -31.181442260742188,
      "activations/layer1_attention_weight_max": 18.057331085205078,
      "activations/layer1_attention_weight_min": -10.368931770324707,
      "activations/layer20_attention_weight_max": 32.488929748535156,
      "activations/layer20_attention_weight_min": -22.813634872436523,
      "activations/layer21_attention_weight_max": 33.392784118652344,
      "activations/layer21_attention_weight_min": -19.740808486938477,
      "activations/layer22_attention_weight_max": 50.21623611450195,
      "activations/layer22_attention_weight_min": -24.2541561126709,
      "activations/layer23_attention_weight_max": 27.735097885131836,
      "activations/layer23_attention_weight_min": -16.968463897705078,
      "activations/layer2_attention_weight_max": 27.773496627807617,
      "activations/layer2_attention_weight_min": -26.79738998413086,
      "activations/layer3_attention_weight_max": 69.45386505126953,
      "activations/layer3_attention_weight_min": -74.1573715209961,
      "activations/layer4_attention_weight_max": 119.53844451904297,
      "activations/layer4_attention_weight_min": -115.87981414794922,
      "activations/layer5_attention_weight_max": 107.02362060546875,
      "activations/layer5_attention_weight_min": -97.30708312988281,
      "activations/layer6_attention_weight_max": 58.33817672729492,
      "activations/layer6_attention_weight_min": -52.16053009033203,
      "activations/layer7_attention_weight_max": 65.49989318847656,
      "activations/layer7_attention_weight_min": -64.96574401855469,
      "activations/layer8_attention_weight_max": 43.5084114074707,
      "activations/layer8_attention_weight_min": -41.07450866699219,
      "activations/layer9_attention_weight_max": 64.6774673461914,
      "activations/layer9_attention_weight_min": -66.30529022216797,
      "epoch": 2.14,
      "learning_rate": 0.00013759393939393938,
      "loss": 2.9473,
      "step": 36800
    },
    {
      "activations/layer0_attention_weight_max": 16.258737564086914,
      "activations/layer0_attention_weight_min": -11.702052116394043,
      "activations/layer10_attention_weight_max": 53.27006149291992,
      "activations/layer10_attention_weight_min": -41.553646087646484,
      "activations/layer11_attention_weight_max": 43.686275482177734,
      "activations/layer11_attention_weight_min": -36.35486602783203,
      "activations/layer12_attention_weight_max": 32.17837905883789,
      "activations/layer12_attention_weight_min": -28.793251037597656,
      "activations/layer13_attention_weight_max": 50.701515197753906,
      "activations/layer13_attention_weight_min": -35.266849517822266,
      "activations/layer14_attention_weight_max": 91.01944732666016,
      "activations/layer14_attention_weight_min": -61.48340606689453,
      "activations/layer15_attention_weight_max": 56.58048629760742,
      "activations/layer15_attention_weight_min": -40.40770721435547,
      "activations/layer16_attention_weight_max": 52.92356491088867,
      "activations/layer16_attention_weight_min": -38.28982925415039,
      "activations/layer17_attention_weight_max": 43.23546600341797,
      "activations/layer17_attention_weight_min": -29.92772102355957,
      "activations/layer18_attention_weight_max": 24.927812576293945,
      "activations/layer18_attention_weight_min": -18.317855834960938,
      "activations/layer19_attention_weight_max": 53.12830352783203,
      "activations/layer19_attention_weight_min": -36.35590362548828,
      "activations/layer1_attention_weight_max": 17.05335235595703,
      "activations/layer1_attention_weight_min": -10.286884307861328,
      "activations/layer20_attention_weight_max": 40.15643310546875,
      "activations/layer20_attention_weight_min": -23.9635066986084,
      "activations/layer21_attention_weight_max": 31.169963836669922,
      "activations/layer21_attention_weight_min": -20.475915908813477,
      "activations/layer22_attention_weight_max": 43.614437103271484,
      "activations/layer22_attention_weight_min": -27.521753311157227,
      "activations/layer23_attention_weight_max": 24.69159507751465,
      "activations/layer23_attention_weight_min": -15.033349990844727,
      "activations/layer2_attention_weight_max": 27.75824546813965,
      "activations/layer2_attention_weight_min": -27.603139877319336,
      "activations/layer3_attention_weight_max": 70.90940856933594,
      "activations/layer3_attention_weight_min": -72.76569366455078,
      "activations/layer4_attention_weight_max": 127.28541564941406,
      "activations/layer4_attention_weight_min": -116.14519500732422,
      "activations/layer5_attention_weight_max": 111.61314392089844,
      "activations/layer5_attention_weight_min": -96.50120544433594,
      "activations/layer6_attention_weight_max": 63.02991485595703,
      "activations/layer6_attention_weight_min": -53.57160949707031,
      "activations/layer7_attention_weight_max": 70.8057632446289,
      "activations/layer7_attention_weight_min": -63.59615707397461,
      "activations/layer8_attention_weight_max": 48.053749084472656,
      "activations/layer8_attention_weight_min": -38.596168518066406,
      "activations/layer9_attention_weight_max": 71.08197021484375,
      "activations/layer9_attention_weight_min": -52.59058380126953,
      "epoch": 2.14,
      "learning_rate": 0.000137575,
      "loss": 2.9491,
      "step": 36850
    },
    {
      "activations/layer0_attention_weight_max": 16.166793823242188,
      "activations/layer0_attention_weight_min": -11.59207534790039,
      "activations/layer10_attention_weight_max": 48.5482292175293,
      "activations/layer10_attention_weight_min": -39.470741271972656,
      "activations/layer11_attention_weight_max": 40.119544982910156,
      "activations/layer11_attention_weight_min": -32.79133987426758,
      "activations/layer12_attention_weight_max": 55.93960189819336,
      "activations/layer12_attention_weight_min": -36.29167938232422,
      "activations/layer13_attention_weight_max": 58.351078033447266,
      "activations/layer13_attention_weight_min": -41.545616149902344,
      "activations/layer14_attention_weight_max": 88.75379180908203,
      "activations/layer14_attention_weight_min": -65.42147064208984,
      "activations/layer15_attention_weight_max": 53.73172378540039,
      "activations/layer15_attention_weight_min": -36.636268615722656,
      "activations/layer16_attention_weight_max": 56.231571197509766,
      "activations/layer16_attention_weight_min": -41.0445442199707,
      "activations/layer17_attention_weight_max": 44.81489944458008,
      "activations/layer17_attention_weight_min": -27.861360549926758,
      "activations/layer18_attention_weight_max": 32.250885009765625,
      "activations/layer18_attention_weight_min": -20.626319885253906,
      "activations/layer19_attention_weight_max": 56.19840621948242,
      "activations/layer19_attention_weight_min": -35.763675689697266,
      "activations/layer1_attention_weight_max": 16.448850631713867,
      "activations/layer1_attention_weight_min": -10.735976219177246,
      "activations/layer20_attention_weight_max": 34.2772331237793,
      "activations/layer20_attention_weight_min": -22.99853515625,
      "activations/layer21_attention_weight_max": 33.52664566040039,
      "activations/layer21_attention_weight_min": -20.900527954101562,
      "activations/layer22_attention_weight_max": 46.88294219970703,
      "activations/layer22_attention_weight_min": -28.980066299438477,
      "activations/layer23_attention_weight_max": 25.741321563720703,
      "activations/layer23_attention_weight_min": -16.688161849975586,
      "activations/layer2_attention_weight_max": 28.20827865600586,
      "activations/layer2_attention_weight_min": -27.490196228027344,
      "activations/layer3_attention_weight_max": 69.9330825805664,
      "activations/layer3_attention_weight_min": -70.84001922607422,
      "activations/layer4_attention_weight_max": 124.33919525146484,
      "activations/layer4_attention_weight_min": -115.29447174072266,
      "activations/layer5_attention_weight_max": 114.02693939208984,
      "activations/layer5_attention_weight_min": -96.77604675292969,
      "activations/layer6_attention_weight_max": 59.971214294433594,
      "activations/layer6_attention_weight_min": -53.67054748535156,
      "activations/layer7_attention_weight_max": 71.5494384765625,
      "activations/layer7_attention_weight_min": -64.69751739501953,
      "activations/layer8_attention_weight_max": 43.3470344543457,
      "activations/layer8_attention_weight_min": -41.839080810546875,
      "activations/layer9_attention_weight_max": 80.05081176757812,
      "activations/layer9_attention_weight_min": -49.79657745361328,
      "epoch": 2.14,
      "learning_rate": 0.00013755606060606058,
      "loss": 2.9333,
      "step": 36900
    },
    {
      "activations/layer0_attention_weight_max": 15.831937789916992,
      "activations/layer0_attention_weight_min": -11.393195152282715,
      "activations/layer10_attention_weight_max": 53.493064880371094,
      "activations/layer10_attention_weight_min": -43.34855651855469,
      "activations/layer11_attention_weight_max": 43.815025329589844,
      "activations/layer11_attention_weight_min": -35.28656768798828,
      "activations/layer12_attention_weight_max": 35.72495651245117,
      "activations/layer12_attention_weight_min": -29.33540153503418,
      "activations/layer13_attention_weight_max": 58.20810317993164,
      "activations/layer13_attention_weight_min": -50.42203903198242,
      "activations/layer14_attention_weight_max": 74.2233657836914,
      "activations/layer14_attention_weight_min": -56.91737365722656,
      "activations/layer15_attention_weight_max": 51.63969039916992,
      "activations/layer15_attention_weight_min": -35.78207015991211,
      "activations/layer16_attention_weight_max": 55.089134216308594,
      "activations/layer16_attention_weight_min": -39.239566802978516,
      "activations/layer17_attention_weight_max": 43.89015197753906,
      "activations/layer17_attention_weight_min": -29.05403709411621,
      "activations/layer18_attention_weight_max": 27.917634963989258,
      "activations/layer18_attention_weight_min": -17.267690658569336,
      "activations/layer19_attention_weight_max": 57.48341751098633,
      "activations/layer19_attention_weight_min": -36.03182601928711,
      "activations/layer1_attention_weight_max": 19.169466018676758,
      "activations/layer1_attention_weight_min": -10.696809768676758,
      "activations/layer20_attention_weight_max": 36.789794921875,
      "activations/layer20_attention_weight_min": -23.633237838745117,
      "activations/layer21_attention_weight_max": 31.681211471557617,
      "activations/layer21_attention_weight_min": -19.154144287109375,
      "activations/layer22_attention_weight_max": 39.202274322509766,
      "activations/layer22_attention_weight_min": -23.759803771972656,
      "activations/layer23_attention_weight_max": 24.000797271728516,
      "activations/layer23_attention_weight_min": -16.907352447509766,
      "activations/layer2_attention_weight_max": 29.001379013061523,
      "activations/layer2_attention_weight_min": -27.75334930419922,
      "activations/layer3_attention_weight_max": 70.41477966308594,
      "activations/layer3_attention_weight_min": -76.71287536621094,
      "activations/layer4_attention_weight_max": 127.2760009765625,
      "activations/layer4_attention_weight_min": -125.1548080444336,
      "activations/layer5_attention_weight_max": 108.77079772949219,
      "activations/layer5_attention_weight_min": -99.38280487060547,
      "activations/layer6_attention_weight_max": 58.52496337890625,
      "activations/layer6_attention_weight_min": -52.58592987060547,
      "activations/layer7_attention_weight_max": 75.1805419921875,
      "activations/layer7_attention_weight_min": -64.18917846679688,
      "activations/layer8_attention_weight_max": 49.71237564086914,
      "activations/layer8_attention_weight_min": -42.00074768066406,
      "activations/layer9_attention_weight_max": 76.39105224609375,
      "activations/layer9_attention_weight_min": -65.3276596069336,
      "epoch": 2.15,
      "learning_rate": 0.0001375371212121212,
      "loss": 2.9433,
      "step": 36950
    },
    {
      "activations/layer0_attention_weight_max": 16.219406127929688,
      "activations/layer0_attention_weight_min": -11.456929206848145,
      "activations/layer10_attention_weight_max": 47.09257125854492,
      "activations/layer10_attention_weight_min": -40.49378967285156,
      "activations/layer11_attention_weight_max": 40.83916473388672,
      "activations/layer11_attention_weight_min": -32.49951171875,
      "activations/layer12_attention_weight_max": 44.39044952392578,
      "activations/layer12_attention_weight_min": -31.60759735107422,
      "activations/layer13_attention_weight_max": 54.00132751464844,
      "activations/layer13_attention_weight_min": -41.295387268066406,
      "activations/layer14_attention_weight_max": 78.52700805664062,
      "activations/layer14_attention_weight_min": -58.563926696777344,
      "activations/layer15_attention_weight_max": 49.48843002319336,
      "activations/layer15_attention_weight_min": -42.35236358642578,
      "activations/layer16_attention_weight_max": 45.35400390625,
      "activations/layer16_attention_weight_min": -38.29682159423828,
      "activations/layer17_attention_weight_max": 44.96049880981445,
      "activations/layer17_attention_weight_min": -28.346904754638672,
      "activations/layer18_attention_weight_max": 28.220027923583984,
      "activations/layer18_attention_weight_min": -20.3585205078125,
      "activations/layer19_attention_weight_max": 47.55653381347656,
      "activations/layer19_attention_weight_min": -30.485885620117188,
      "activations/layer1_attention_weight_max": 17.0261287689209,
      "activations/layer1_attention_weight_min": -12.140153884887695,
      "activations/layer20_attention_weight_max": 37.23161697387695,
      "activations/layer20_attention_weight_min": -24.545631408691406,
      "activations/layer21_attention_weight_max": 30.934982299804688,
      "activations/layer21_attention_weight_min": -19.293621063232422,
      "activations/layer22_attention_weight_max": 43.08103561401367,
      "activations/layer22_attention_weight_min": -25.661855697631836,
      "activations/layer23_attention_weight_max": 25.45932388305664,
      "activations/layer23_attention_weight_min": -16.68061065673828,
      "activations/layer2_attention_weight_max": 29.90652847290039,
      "activations/layer2_attention_weight_min": -27.16144371032715,
      "activations/layer3_attention_weight_max": 69.12094116210938,
      "activations/layer3_attention_weight_min": -70.75759887695312,
      "activations/layer4_attention_weight_max": 123.30240631103516,
      "activations/layer4_attention_weight_min": -119.7652816772461,
      "activations/layer5_attention_weight_max": 108.48390197753906,
      "activations/layer5_attention_weight_min": -98.71400451660156,
      "activations/layer6_attention_weight_max": 58.10700225830078,
      "activations/layer6_attention_weight_min": -52.672752380371094,
      "activations/layer7_attention_weight_max": 67.36933135986328,
      "activations/layer7_attention_weight_min": -64.20062255859375,
      "activations/layer8_attention_weight_max": 46.35346603393555,
      "activations/layer8_attention_weight_min": -41.20842742919922,
      "activations/layer9_attention_weight_max": 69.28968048095703,
      "activations/layer9_attention_weight_min": -49.752105712890625,
      "epoch": 2.15,
      "learning_rate": 0.00013751818181818181,
      "loss": 2.9509,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_loss": 2.90234375,
      "eval_runtime": 8.5066,
      "eval_samples_per_second": 504.784,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_openwebtext_loss": 2.90234375,
      "eval_openwebtext_ppl": 18.216790978454146,
      "eval_openwebtext_runtime": 8.5066,
      "eval_openwebtext_samples_per_second": 504.784,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_wikitext_loss": 3.169921875,
      "eval_wikitext_ppl": 23.80562446936611,
      "eval_wikitext_runtime": 1.9985,
      "eval_wikitext_samples_per_second": 228.175,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_lambada_loss": 3.078125,
      "eval_lambada_ppl": 21.717643615002626,
      "eval_lambada_runtime": 9.5636,
      "eval_lambada_samples_per_second": 509.119,
      "step": 37000
    },
    {
      "activations/layer0_attention_weight_max": 16.322784423828125,
      "activations/layer0_attention_weight_min": -11.155829429626465,
      "activations/layer10_attention_weight_max": 50.38709259033203,
      "activations/layer10_attention_weight_min": -39.49147415161133,
      "activations/layer11_attention_weight_max": 38.538883209228516,
      "activations/layer11_attention_weight_min": -35.4849853515625,
      "activations/layer12_attention_weight_max": 35.37360382080078,
      "activations/layer12_attention_weight_min": -28.567787170410156,
      "activations/layer13_attention_weight_max": 57.34577941894531,
      "activations/layer13_attention_weight_min": -40.43653106689453,
      "activations/layer14_attention_weight_max": 75.3218002319336,
      "activations/layer14_attention_weight_min": -56.120601654052734,
      "activations/layer15_attention_weight_max": 54.23195266723633,
      "activations/layer15_attention_weight_min": -36.66981887817383,
      "activations/layer16_attention_weight_max": 51.71490478515625,
      "activations/layer16_attention_weight_min": -33.36100769042969,
      "activations/layer17_attention_weight_max": 50.66792678833008,
      "activations/layer17_attention_weight_min": -29.88414192199707,
      "activations/layer18_attention_weight_max": 26.559486389160156,
      "activations/layer18_attention_weight_min": -17.357824325561523,
      "activations/layer19_attention_weight_max": 46.08745193481445,
      "activations/layer19_attention_weight_min": -32.42356491088867,
      "activations/layer1_attention_weight_max": 17.8413143157959,
      "activations/layer1_attention_weight_min": -10.800915718078613,
      "activations/layer20_attention_weight_max": 36.91785430908203,
      "activations/layer20_attention_weight_min": -24.615821838378906,
      "activations/layer21_attention_weight_max": 26.73680877685547,
      "activations/layer21_attention_weight_min": -19.005149841308594,
      "activations/layer22_attention_weight_max": 47.16191864013672,
      "activations/layer22_attention_weight_min": -25.33920669555664,
      "activations/layer23_attention_weight_max": 25.663970947265625,
      "activations/layer23_attention_weight_min": -16.40076446533203,
      "activations/layer2_attention_weight_max": 28.530517578125,
      "activations/layer2_attention_weight_min": -28.313024520874023,
      "activations/layer3_attention_weight_max": 69.74823760986328,
      "activations/layer3_attention_weight_min": -71.36235046386719,
      "activations/layer4_attention_weight_max": 129.7056121826172,
      "activations/layer4_attention_weight_min": -119.95977783203125,
      "activations/layer5_attention_weight_max": 117.00823211669922,
      "activations/layer5_attention_weight_min": -95.64883422851562,
      "activations/layer6_attention_weight_max": 62.77223205566406,
      "activations/layer6_attention_weight_min": -53.71228790283203,
      "activations/layer7_attention_weight_max": 77.4105224609375,
      "activations/layer7_attention_weight_min": -62.36622619628906,
      "activations/layer8_attention_weight_max": 48.34333801269531,
      "activations/layer8_attention_weight_min": -37.91160583496094,
      "activations/layer9_attention_weight_max": 66.05399322509766,
      "activations/layer9_attention_weight_min": -56.01766586303711,
      "epoch": 2.15,
      "learning_rate": 0.0001374992424242424,
      "loss": 2.9498,
      "step": 37050
    },
    {
      "activations/layer0_attention_weight_max": 17.010150909423828,
      "activations/layer0_attention_weight_min": -11.190569877624512,
      "activations/layer10_attention_weight_max": 46.515289306640625,
      "activations/layer10_attention_weight_min": -38.71282958984375,
      "activations/layer11_attention_weight_max": 35.44926452636719,
      "activations/layer11_attention_weight_min": -33.190513610839844,
      "activations/layer12_attention_weight_max": 31.12214469909668,
      "activations/layer12_attention_weight_min": -30.159440994262695,
      "activations/layer13_attention_weight_max": 48.4263801574707,
      "activations/layer13_attention_weight_min": -40.15301513671875,
      "activations/layer14_attention_weight_max": 78.87142181396484,
      "activations/layer14_attention_weight_min": -64.00608825683594,
      "activations/layer15_attention_weight_max": 51.457611083984375,
      "activations/layer15_attention_weight_min": -38.52304458618164,
      "activations/layer16_attention_weight_max": 45.74017333984375,
      "activations/layer16_attention_weight_min": -34.484901428222656,
      "activations/layer17_attention_weight_max": 45.986480712890625,
      "activations/layer17_attention_weight_min": -30.39197540283203,
      "activations/layer18_attention_weight_max": 26.353952407836914,
      "activations/layer18_attention_weight_min": -21.095050811767578,
      "activations/layer19_attention_weight_max": 47.93544006347656,
      "activations/layer19_attention_weight_min": -33.49486541748047,
      "activations/layer1_attention_weight_max": 18.907339096069336,
      "activations/layer1_attention_weight_min": -12.062987327575684,
      "activations/layer20_attention_weight_max": 37.739986419677734,
      "activations/layer20_attention_weight_min": -26.162586212158203,
      "activations/layer21_attention_weight_max": 31.348949432373047,
      "activations/layer21_attention_weight_min": -20.302255630493164,
      "activations/layer22_attention_weight_max": 43.15884780883789,
      "activations/layer22_attention_weight_min": -28.82721710205078,
      "activations/layer23_attention_weight_max": 25.755874633789062,
      "activations/layer23_attention_weight_min": -16.92855453491211,
      "activations/layer2_attention_weight_max": 28.688488006591797,
      "activations/layer2_attention_weight_min": -27.310169219970703,
      "activations/layer3_attention_weight_max": 71.69435119628906,
      "activations/layer3_attention_weight_min": -72.00848388671875,
      "activations/layer4_attention_weight_max": 126.3419418334961,
      "activations/layer4_attention_weight_min": -113.8850326538086,
      "activations/layer5_attention_weight_max": 109.88448333740234,
      "activations/layer5_attention_weight_min": -96.89388275146484,
      "activations/layer6_attention_weight_max": 58.67368698120117,
      "activations/layer6_attention_weight_min": -53.19013595581055,
      "activations/layer7_attention_weight_max": 62.64550018310547,
      "activations/layer7_attention_weight_min": -61.53975296020508,
      "activations/layer8_attention_weight_max": 44.9161376953125,
      "activations/layer8_attention_weight_min": -38.57529830932617,
      "activations/layer9_attention_weight_max": 60.239498138427734,
      "activations/layer9_attention_weight_min": -53.07136154174805,
      "epoch": 2.16,
      "learning_rate": 0.00013748030303030302,
      "loss": 2.9625,
      "step": 37100
    },
    {
      "activations/layer0_attention_weight_max": 16.701879501342773,
      "activations/layer0_attention_weight_min": -11.291523933410645,
      "activations/layer10_attention_weight_max": 49.79136657714844,
      "activations/layer10_attention_weight_min": -38.508296966552734,
      "activations/layer11_attention_weight_max": 54.22312545776367,
      "activations/layer11_attention_weight_min": -41.56388854980469,
      "activations/layer12_attention_weight_max": 34.21848678588867,
      "activations/layer12_attention_weight_min": -26.77202796936035,
      "activations/layer13_attention_weight_max": 54.20351791381836,
      "activations/layer13_attention_weight_min": -38.22866439819336,
      "activations/layer14_attention_weight_max": 70.72380065917969,
      "activations/layer14_attention_weight_min": -57.63142013549805,
      "activations/layer15_attention_weight_max": 45.11438751220703,
      "activations/layer15_attention_weight_min": -35.46296691894531,
      "activations/layer16_attention_weight_max": 49.43178176879883,
      "activations/layer16_attention_weight_min": -36.69212341308594,
      "activations/layer17_attention_weight_max": 39.05766296386719,
      "activations/layer17_attention_weight_min": -27.39470863342285,
      "activations/layer18_attention_weight_max": 26.68808364868164,
      "activations/layer18_attention_weight_min": -18.605880737304688,
      "activations/layer19_attention_weight_max": 49.4826545715332,
      "activations/layer19_attention_weight_min": -34.5418815612793,
      "activations/layer1_attention_weight_max": 17.59331512451172,
      "activations/layer1_attention_weight_min": -10.647053718566895,
      "activations/layer20_attention_weight_max": 33.31667709350586,
      "activations/layer20_attention_weight_min": -25.99309539794922,
      "activations/layer21_attention_weight_max": 32.115238189697266,
      "activations/layer21_attention_weight_min": -18.473482131958008,
      "activations/layer22_attention_weight_max": 38.962066650390625,
      "activations/layer22_attention_weight_min": -23.46435546875,
      "activations/layer23_attention_weight_max": 24.56317138671875,
      "activations/layer23_attention_weight_min": -15.488123893737793,
      "activations/layer2_attention_weight_max": 29.76362419128418,
      "activations/layer2_attention_weight_min": -28.565673828125,
      "activations/layer3_attention_weight_max": 72.38126373291016,
      "activations/layer3_attention_weight_min": -72.92707061767578,
      "activations/layer4_attention_weight_max": 129.94451904296875,
      "activations/layer4_attention_weight_min": -118.7463150024414,
      "activations/layer5_attention_weight_max": 114.79747772216797,
      "activations/layer5_attention_weight_min": -98.20246887207031,
      "activations/layer6_attention_weight_max": 62.44279861450195,
      "activations/layer6_attention_weight_min": -55.845829010009766,
      "activations/layer7_attention_weight_max": 78.50752258300781,
      "activations/layer7_attention_weight_min": -60.180809020996094,
      "activations/layer8_attention_weight_max": 52.393280029296875,
      "activations/layer8_attention_weight_min": -41.794673919677734,
      "activations/layer9_attention_weight_max": 79.91486358642578,
      "activations/layer9_attention_weight_min": -68.21499633789062,
      "epoch": 2.16,
      "learning_rate": 0.00013746136363636363,
      "loss": 2.9349,
      "step": 37150
    },
    {
      "activations/layer0_attention_weight_max": 16.79386329650879,
      "activations/layer0_attention_weight_min": -12.193079948425293,
      "activations/layer10_attention_weight_max": 49.7012939453125,
      "activations/layer10_attention_weight_min": -43.368099212646484,
      "activations/layer11_attention_weight_max": 39.074188232421875,
      "activations/layer11_attention_weight_min": -31.91497039794922,
      "activations/layer12_attention_weight_max": 41.06391143798828,
      "activations/layer12_attention_weight_min": -28.00747299194336,
      "activations/layer13_attention_weight_max": 49.582130432128906,
      "activations/layer13_attention_weight_min": -36.83866500854492,
      "activations/layer14_attention_weight_max": 81.45523071289062,
      "activations/layer14_attention_weight_min": -59.8726921081543,
      "activations/layer15_attention_weight_max": 53.59111785888672,
      "activations/layer15_attention_weight_min": -35.984310150146484,
      "activations/layer16_attention_weight_max": 49.929603576660156,
      "activations/layer16_attention_weight_min": -37.228851318359375,
      "activations/layer17_attention_weight_max": 50.14433670043945,
      "activations/layer17_attention_weight_min": -27.351036071777344,
      "activations/layer18_attention_weight_max": 28.91701889038086,
      "activations/layer18_attention_weight_min": -19.987712860107422,
      "activations/layer19_attention_weight_max": 53.74560546875,
      "activations/layer19_attention_weight_min": -34.0526237487793,
      "activations/layer1_attention_weight_max": 17.18489646911621,
      "activations/layer1_attention_weight_min": -11.506964683532715,
      "activations/layer20_attention_weight_max": 39.65641784667969,
      "activations/layer20_attention_weight_min": -24.928678512573242,
      "activations/layer21_attention_weight_max": 33.095245361328125,
      "activations/layer21_attention_weight_min": -21.84624671936035,
      "activations/layer22_attention_weight_max": 48.66529083251953,
      "activations/layer22_attention_weight_min": -26.986156463623047,
      "activations/layer23_attention_weight_max": 27.529804229736328,
      "activations/layer23_attention_weight_min": -17.375293731689453,
      "activations/layer2_attention_weight_max": 27.422916412353516,
      "activations/layer2_attention_weight_min": -27.69379425048828,
      "activations/layer3_attention_weight_max": 70.26799011230469,
      "activations/layer3_attention_weight_min": -76.00129699707031,
      "activations/layer4_attention_weight_max": 126.64631652832031,
      "activations/layer4_attention_weight_min": -123.85932922363281,
      "activations/layer5_attention_weight_max": 105.18495178222656,
      "activations/layer5_attention_weight_min": -95.89447021484375,
      "activations/layer6_attention_weight_max": 59.22361755371094,
      "activations/layer6_attention_weight_min": -52.90221405029297,
      "activations/layer7_attention_weight_max": 67.12873077392578,
      "activations/layer7_attention_weight_min": -65.69647216796875,
      "activations/layer8_attention_weight_max": 41.597652435302734,
      "activations/layer8_attention_weight_min": -38.84548568725586,
      "activations/layer9_attention_weight_max": 64.80033874511719,
      "activations/layer9_attention_weight_min": -49.97536087036133,
      "epoch": 2.16,
      "learning_rate": 0.00013744280303030302,
      "loss": 2.9547,
      "step": 37200
    },
    {
      "activations/layer0_attention_weight_max": 14.885692596435547,
      "activations/layer0_attention_weight_min": -11.637321472167969,
      "activations/layer10_attention_weight_max": 56.403770446777344,
      "activations/layer10_attention_weight_min": -42.36893081665039,
      "activations/layer11_attention_weight_max": 47.24115753173828,
      "activations/layer11_attention_weight_min": -37.52279281616211,
      "activations/layer12_attention_weight_max": 29.478883743286133,
      "activations/layer12_attention_weight_min": -28.981550216674805,
      "activations/layer13_attention_weight_max": 47.43936538696289,
      "activations/layer13_attention_weight_min": -35.51263427734375,
      "activations/layer14_attention_weight_max": 78.79066467285156,
      "activations/layer14_attention_weight_min": -62.51520538330078,
      "activations/layer15_attention_weight_max": 51.019962310791016,
      "activations/layer15_attention_weight_min": -36.528709411621094,
      "activations/layer16_attention_weight_max": 49.98933410644531,
      "activations/layer16_attention_weight_min": -39.94268798828125,
      "activations/layer17_attention_weight_max": 44.34035110473633,
      "activations/layer17_attention_weight_min": -30.70601463317871,
      "activations/layer18_attention_weight_max": 32.531646728515625,
      "activations/layer18_attention_weight_min": -19.673389434814453,
      "activations/layer19_attention_weight_max": 51.31622314453125,
      "activations/layer19_attention_weight_min": -32.4299430847168,
      "activations/layer1_attention_weight_max": 16.70794677734375,
      "activations/layer1_attention_weight_min": -11.550326347351074,
      "activations/layer20_attention_weight_max": 34.48162078857422,
      "activations/layer20_attention_weight_min": -24.639293670654297,
      "activations/layer21_attention_weight_max": 32.52632141113281,
      "activations/layer21_attention_weight_min": -18.7758731842041,
      "activations/layer22_attention_weight_max": 43.99851608276367,
      "activations/layer22_attention_weight_min": -25.978384017944336,
      "activations/layer23_attention_weight_max": 24.01757049560547,
      "activations/layer23_attention_weight_min": -14.507125854492188,
      "activations/layer2_attention_weight_max": 32.1030387878418,
      "activations/layer2_attention_weight_min": -30.669179916381836,
      "activations/layer3_attention_weight_max": 75.93988800048828,
      "activations/layer3_attention_weight_min": -77.11844635009766,
      "activations/layer4_attention_weight_max": 140.41539001464844,
      "activations/layer4_attention_weight_min": -119.9189682006836,
      "activations/layer5_attention_weight_max": 126.82643127441406,
      "activations/layer5_attention_weight_min": -97.80669403076172,
      "activations/layer6_attention_weight_max": 72.28923034667969,
      "activations/layer6_attention_weight_min": -52.22532653808594,
      "activations/layer7_attention_weight_max": 82.26022338867188,
      "activations/layer7_attention_weight_min": -63.666725158691406,
      "activations/layer8_attention_weight_max": 58.69025421142578,
      "activations/layer8_attention_weight_min": -40.590187072753906,
      "activations/layer9_attention_weight_max": 90.59872436523438,
      "activations/layer9_attention_weight_min": -64.8750228881836,
      "epoch": 2.16,
      "learning_rate": 0.0001374238636363636,
      "loss": 2.958,
      "step": 37250
    },
    {
      "activations/layer0_attention_weight_max": 17.014312744140625,
      "activations/layer0_attention_weight_min": -11.612360000610352,
      "activations/layer10_attention_weight_max": 55.11103439331055,
      "activations/layer10_attention_weight_min": -45.05796432495117,
      "activations/layer11_attention_weight_max": 47.06371307373047,
      "activations/layer11_attention_weight_min": -36.23963928222656,
      "activations/layer12_attention_weight_max": 32.76144790649414,
      "activations/layer12_attention_weight_min": -27.81454849243164,
      "activations/layer13_attention_weight_max": 57.83542251586914,
      "activations/layer13_attention_weight_min": -37.24907684326172,
      "activations/layer14_attention_weight_max": 79.81600952148438,
      "activations/layer14_attention_weight_min": -56.313270568847656,
      "activations/layer15_attention_weight_max": 60.08392333984375,
      "activations/layer15_attention_weight_min": -35.20389938354492,
      "activations/layer16_attention_weight_max": 51.15361404418945,
      "activations/layer16_attention_weight_min": -33.85917663574219,
      "activations/layer17_attention_weight_max": 51.17062759399414,
      "activations/layer17_attention_weight_min": -27.487112045288086,
      "activations/layer18_attention_weight_max": 28.045291900634766,
      "activations/layer18_attention_weight_min": -16.968830108642578,
      "activations/layer19_attention_weight_max": 65.28569030761719,
      "activations/layer19_attention_weight_min": -32.00721740722656,
      "activations/layer1_attention_weight_max": 16.977378845214844,
      "activations/layer1_attention_weight_min": -11.26069450378418,
      "activations/layer20_attention_weight_max": 43.616249084472656,
      "activations/layer20_attention_weight_min": -28.00048065185547,
      "activations/layer21_attention_weight_max": 35.46689987182617,
      "activations/layer21_attention_weight_min": -18.22413444519043,
      "activations/layer22_attention_weight_max": 48.74693298339844,
      "activations/layer22_attention_weight_min": -25.333499908447266,
      "activations/layer23_attention_weight_max": 28.561248779296875,
      "activations/layer23_attention_weight_min": -15.216560363769531,
      "activations/layer2_attention_weight_max": 30.15522003173828,
      "activations/layer2_attention_weight_min": -29.6923828125,
      "activations/layer3_attention_weight_max": 73.07791900634766,
      "activations/layer3_attention_weight_min": -77.75409698486328,
      "activations/layer4_attention_weight_max": 128.28811645507812,
      "activations/layer4_attention_weight_min": -123.23656463623047,
      "activations/layer5_attention_weight_max": 120.57149505615234,
      "activations/layer5_attention_weight_min": -98.62692260742188,
      "activations/layer6_attention_weight_max": 68.71367645263672,
      "activations/layer6_attention_weight_min": -54.79786682128906,
      "activations/layer7_attention_weight_max": 76.22342681884766,
      "activations/layer7_attention_weight_min": -66.84400177001953,
      "activations/layer8_attention_weight_max": 51.27094650268555,
      "activations/layer8_attention_weight_min": -45.0509033203125,
      "activations/layer9_attention_weight_max": 65.63992309570312,
      "activations/layer9_attention_weight_min": -50.381710052490234,
      "epoch": 2.17,
      "learning_rate": 0.00013740492424242423,
      "loss": 2.9536,
      "step": 37300
    },
    {
      "activations/layer0_attention_weight_max": 16.265649795532227,
      "activations/layer0_attention_weight_min": -11.325119018554688,
      "activations/layer10_attention_weight_max": 44.8026123046875,
      "activations/layer10_attention_weight_min": -38.882904052734375,
      "activations/layer11_attention_weight_max": 38.12956237792969,
      "activations/layer11_attention_weight_min": -32.909629821777344,
      "activations/layer12_attention_weight_max": 32.507110595703125,
      "activations/layer12_attention_weight_min": -26.833345413208008,
      "activations/layer13_attention_weight_max": 49.89448547363281,
      "activations/layer13_attention_weight_min": -41.6713981628418,
      "activations/layer14_attention_weight_max": 90.29830932617188,
      "activations/layer14_attention_weight_min": -70.26859283447266,
      "activations/layer15_attention_weight_max": 48.21519088745117,
      "activations/layer15_attention_weight_min": -36.00939178466797,
      "activations/layer16_attention_weight_max": 44.181758880615234,
      "activations/layer16_attention_weight_min": -33.622657775878906,
      "activations/layer17_attention_weight_max": 41.78471374511719,
      "activations/layer17_attention_weight_min": -29.717445373535156,
      "activations/layer18_attention_weight_max": 28.372051239013672,
      "activations/layer18_attention_weight_min": -18.937206268310547,
      "activations/layer19_attention_weight_max": 45.359989166259766,
      "activations/layer19_attention_weight_min": -30.549259185791016,
      "activations/layer1_attention_weight_max": 18.664257049560547,
      "activations/layer1_attention_weight_min": -10.712248802185059,
      "activations/layer20_attention_weight_max": 39.40109634399414,
      "activations/layer20_attention_weight_min": -24.3518009185791,
      "activations/layer21_attention_weight_max": 33.017478942871094,
      "activations/layer21_attention_weight_min": -21.934404373168945,
      "activations/layer22_attention_weight_max": 49.00583267211914,
      "activations/layer22_attention_weight_min": -25.76883316040039,
      "activations/layer23_attention_weight_max": 24.295207977294922,
      "activations/layer23_attention_weight_min": -16.687440872192383,
      "activations/layer2_attention_weight_max": 27.9751033782959,
      "activations/layer2_attention_weight_min": -28.794231414794922,
      "activations/layer3_attention_weight_max": 73.660400390625,
      "activations/layer3_attention_weight_min": -74.80530548095703,
      "activations/layer4_attention_weight_max": 121.65235900878906,
      "activations/layer4_attention_weight_min": -111.36238861083984,
      "activations/layer5_attention_weight_max": 110.44044494628906,
      "activations/layer5_attention_weight_min": -93.90187072753906,
      "activations/layer6_attention_weight_max": 61.16542053222656,
      "activations/layer6_attention_weight_min": -52.03756332397461,
      "activations/layer7_attention_weight_max": 68.07453155517578,
      "activations/layer7_attention_weight_min": -59.71942901611328,
      "activations/layer8_attention_weight_max": 48.003990173339844,
      "activations/layer8_attention_weight_min": -36.59341812133789,
      "activations/layer9_attention_weight_max": 59.24146270751953,
      "activations/layer9_attention_weight_min": -47.981346130371094,
      "epoch": 2.17,
      "learning_rate": 0.00013738598484848484,
      "loss": 2.9517,
      "step": 37350
    },
    {
      "activations/layer0_attention_weight_max": 16.477279663085938,
      "activations/layer0_attention_weight_min": -12.082606315612793,
      "activations/layer10_attention_weight_max": 56.77109909057617,
      "activations/layer10_attention_weight_min": -40.65424346923828,
      "activations/layer11_attention_weight_max": 48.80352020263672,
      "activations/layer11_attention_weight_min": -34.44273376464844,
      "activations/layer12_attention_weight_max": 36.7322998046875,
      "activations/layer12_attention_weight_min": -24.708940505981445,
      "activations/layer13_attention_weight_max": 54.62251663208008,
      "activations/layer13_attention_weight_min": -34.03763961791992,
      "activations/layer14_attention_weight_max": 87.2813720703125,
      "activations/layer14_attention_weight_min": -61.671791076660156,
      "activations/layer15_attention_weight_max": 57.04912567138672,
      "activations/layer15_attention_weight_min": -37.786617279052734,
      "activations/layer16_attention_weight_max": 58.08122253417969,
      "activations/layer16_attention_weight_min": -34.99085235595703,
      "activations/layer17_attention_weight_max": 47.48644256591797,
      "activations/layer17_attention_weight_min": -29.597633361816406,
      "activations/layer18_attention_weight_max": 27.320486068725586,
      "activations/layer18_attention_weight_min": -17.03652572631836,
      "activations/layer19_attention_weight_max": 54.89459228515625,
      "activations/layer19_attention_weight_min": -31.05861473083496,
      "activations/layer1_attention_weight_max": 18.05026626586914,
      "activations/layer1_attention_weight_min": -11.227762222290039,
      "activations/layer20_attention_weight_max": 43.2791862487793,
      "activations/layer20_attention_weight_min": -24.681371688842773,
      "activations/layer21_attention_weight_max": 29.53652000427246,
      "activations/layer21_attention_weight_min": -19.30621337890625,
      "activations/layer22_attention_weight_max": 49.499168395996094,
      "activations/layer22_attention_weight_min": -24.343814849853516,
      "activations/layer23_attention_weight_max": 24.075271606445312,
      "activations/layer23_attention_weight_min": -17.42536163330078,
      "activations/layer2_attention_weight_max": 28.597517013549805,
      "activations/layer2_attention_weight_min": -27.879169464111328,
      "activations/layer3_attention_weight_max": 74.3770751953125,
      "activations/layer3_attention_weight_min": -79.10338592529297,
      "activations/layer4_attention_weight_max": 131.51341247558594,
      "activations/layer4_attention_weight_min": -130.94142150878906,
      "activations/layer5_attention_weight_max": 120.00487518310547,
      "activations/layer5_attention_weight_min": -105.79855346679688,
      "activations/layer6_attention_weight_max": 68.25841522216797,
      "activations/layer6_attention_weight_min": -55.389469146728516,
      "activations/layer7_attention_weight_max": 88.03748321533203,
      "activations/layer7_attention_weight_min": -69.3293228149414,
      "activations/layer8_attention_weight_max": 55.45341873168945,
      "activations/layer8_attention_weight_min": -42.46199417114258,
      "activations/layer9_attention_weight_max": 70.03437805175781,
      "activations/layer9_attention_weight_min": -57.18082809448242,
      "epoch": 2.17,
      "learning_rate": 0.00013736704545454543,
      "loss": 2.9398,
      "step": 37400
    },
    {
      "activations/layer0_attention_weight_max": 16.897125244140625,
      "activations/layer0_attention_weight_min": -12.221831321716309,
      "activations/layer10_attention_weight_max": 46.947811126708984,
      "activations/layer10_attention_weight_min": -39.92994689941406,
      "activations/layer11_attention_weight_max": 41.540184020996094,
      "activations/layer11_attention_weight_min": -36.162940979003906,
      "activations/layer12_attention_weight_max": 71.51102447509766,
      "activations/layer12_attention_weight_min": -46.2252197265625,
      "activations/layer13_attention_weight_max": 74.67445373535156,
      "activations/layer13_attention_weight_min": -53.79945373535156,
      "activations/layer14_attention_weight_max": 103.16029357910156,
      "activations/layer14_attention_weight_min": -66.87339782714844,
      "activations/layer15_attention_weight_max": 52.361366271972656,
      "activations/layer15_attention_weight_min": -38.20146560668945,
      "activations/layer16_attention_weight_max": 49.15795135498047,
      "activations/layer16_attention_weight_min": -31.860261917114258,
      "activations/layer17_attention_weight_max": 50.994422912597656,
      "activations/layer17_attention_weight_min": -29.7999210357666,
      "activations/layer18_attention_weight_max": 28.49530029296875,
      "activations/layer18_attention_weight_min": -17.138320922851562,
      "activations/layer19_attention_weight_max": 53.804866790771484,
      "activations/layer19_attention_weight_min": -31.21529197692871,
      "activations/layer1_attention_weight_max": 16.996627807617188,
      "activations/layer1_attention_weight_min": -11.993429183959961,
      "activations/layer20_attention_weight_max": 38.819580078125,
      "activations/layer20_attention_weight_min": -22.989887237548828,
      "activations/layer21_attention_weight_max": 28.10899543762207,
      "activations/layer21_attention_weight_min": -16.841835021972656,
      "activations/layer22_attention_weight_max": 49.25873565673828,
      "activations/layer22_attention_weight_min": -25.8169002532959,
      "activations/layer23_attention_weight_max": 25.337574005126953,
      "activations/layer23_attention_weight_min": -15.469523429870605,
      "activations/layer2_attention_weight_max": 29.210052490234375,
      "activations/layer2_attention_weight_min": -29.135984420776367,
      "activations/layer3_attention_weight_max": 75.82904052734375,
      "activations/layer3_attention_weight_min": -77.98703002929688,
      "activations/layer4_attention_weight_max": 142.1437530517578,
      "activations/layer4_attention_weight_min": -125.31890869140625,
      "activations/layer5_attention_weight_max": 121.65969848632812,
      "activations/layer5_attention_weight_min": -101.33291625976562,
      "activations/layer6_attention_weight_max": 69.06470489501953,
      "activations/layer6_attention_weight_min": -54.01070022583008,
      "activations/layer7_attention_weight_max": 67.31023406982422,
      "activations/layer7_attention_weight_min": -64.91973876953125,
      "activations/layer8_attention_weight_max": 51.8460578918457,
      "activations/layer8_attention_weight_min": -41.83668899536133,
      "activations/layer9_attention_weight_max": 68.194091796875,
      "activations/layer9_attention_weight_min": -60.49768829345703,
      "epoch": 2.18,
      "learning_rate": 0.00013734810606060604,
      "loss": 2.9322,
      "step": 37450
    },
    {
      "activations/layer0_attention_weight_max": 16.73192596435547,
      "activations/layer0_attention_weight_min": -12.005396842956543,
      "activations/layer10_attention_weight_max": 54.19926452636719,
      "activations/layer10_attention_weight_min": -45.86936569213867,
      "activations/layer11_attention_weight_max": 55.92036437988281,
      "activations/layer11_attention_weight_min": -38.13096618652344,
      "activations/layer12_attention_weight_max": 33.552833557128906,
      "activations/layer12_attention_weight_min": -27.926204681396484,
      "activations/layer13_attention_weight_max": 55.384864807128906,
      "activations/layer13_attention_weight_min": -35.740814208984375,
      "activations/layer14_attention_weight_max": 75.80980682373047,
      "activations/layer14_attention_weight_min": -57.48370361328125,
      "activations/layer15_attention_weight_max": 56.540733337402344,
      "activations/layer15_attention_weight_min": -36.372310638427734,
      "activations/layer16_attention_weight_max": 53.14877700805664,
      "activations/layer16_attention_weight_min": -38.245643615722656,
      "activations/layer17_attention_weight_max": 49.027767181396484,
      "activations/layer17_attention_weight_min": -28.98007583618164,
      "activations/layer18_attention_weight_max": 32.4325065612793,
      "activations/layer18_attention_weight_min": -18.09259605407715,
      "activations/layer19_attention_weight_max": 59.439205169677734,
      "activations/layer19_attention_weight_min": -32.06314468383789,
      "activations/layer1_attention_weight_max": 17.585662841796875,
      "activations/layer1_attention_weight_min": -12.130563735961914,
      "activations/layer20_attention_weight_max": 40.20353698730469,
      "activations/layer20_attention_weight_min": -23.48775291442871,
      "activations/layer21_attention_weight_max": 40.75925827026367,
      "activations/layer21_attention_weight_min": -19.186782836914062,
      "activations/layer22_attention_weight_max": 46.42154312133789,
      "activations/layer22_attention_weight_min": -23.97795867919922,
      "activations/layer23_attention_weight_max": 28.069461822509766,
      "activations/layer23_attention_weight_min": -16.916797637939453,
      "activations/layer2_attention_weight_max": 29.411556243896484,
      "activations/layer2_attention_weight_min": -28.534175872802734,
      "activations/layer3_attention_weight_max": 73.17726135253906,
      "activations/layer3_attention_weight_min": -79.15853881835938,
      "activations/layer4_attention_weight_max": 133.9640350341797,
      "activations/layer4_attention_weight_min": -123.6290512084961,
      "activations/layer5_attention_weight_max": 122.74888610839844,
      "activations/layer5_attention_weight_min": -98.91313171386719,
      "activations/layer6_attention_weight_max": 72.2799072265625,
      "activations/layer6_attention_weight_min": -54.33995819091797,
      "activations/layer7_attention_weight_max": 78.94266510009766,
      "activations/layer7_attention_weight_min": -62.406246185302734,
      "activations/layer8_attention_weight_max": 57.38457489013672,
      "activations/layer8_attention_weight_min": -42.039485931396484,
      "activations/layer9_attention_weight_max": 89.8385238647461,
      "activations/layer9_attention_weight_min": -61.66912841796875,
      "epoch": 2.18,
      "learning_rate": 0.00013732916666666666,
      "loss": 2.9646,
      "step": 37500
    },
    {
      "activations/layer0_attention_weight_max": 16.0379638671875,
      "activations/layer0_attention_weight_min": -11.792283058166504,
      "activations/layer10_attention_weight_max": 60.14105224609375,
      "activations/layer10_attention_weight_min": -45.032989501953125,
      "activations/layer11_attention_weight_max": 58.65516662597656,
      "activations/layer11_attention_weight_min": -39.52211380004883,
      "activations/layer12_attention_weight_max": 68.4601821899414,
      "activations/layer12_attention_weight_min": -34.878536224365234,
      "activations/layer13_attention_weight_max": 86.99740600585938,
      "activations/layer13_attention_weight_min": -51.34324264526367,
      "activations/layer14_attention_weight_max": 104.97570037841797,
      "activations/layer14_attention_weight_min": -66.12287902832031,
      "activations/layer15_attention_weight_max": 74.6539306640625,
      "activations/layer15_attention_weight_min": -41.99730682373047,
      "activations/layer16_attention_weight_max": 53.91324996948242,
      "activations/layer16_attention_weight_min": -37.339847564697266,
      "activations/layer17_attention_weight_max": 51.493614196777344,
      "activations/layer17_attention_weight_min": -32.49662399291992,
      "activations/layer18_attention_weight_max": 46.031700134277344,
      "activations/layer18_attention_weight_min": -23.06732749938965,
      "activations/layer19_attention_weight_max": 49.95695114135742,
      "activations/layer19_attention_weight_min": -31.004140853881836,
      "activations/layer1_attention_weight_max": 19.35651969909668,
      "activations/layer1_attention_weight_min": -11.50265121459961,
      "activations/layer20_attention_weight_max": 40.33233642578125,
      "activations/layer20_attention_weight_min": -23.834264755249023,
      "activations/layer21_attention_weight_max": 34.27594757080078,
      "activations/layer21_attention_weight_min": -19.310606002807617,
      "activations/layer22_attention_weight_max": 47.34507751464844,
      "activations/layer22_attention_weight_min": -25.469860076904297,
      "activations/layer23_attention_weight_max": 25.590042114257812,
      "activations/layer23_attention_weight_min": -15.07243537902832,
      "activations/layer2_attention_weight_max": 28.607498168945312,
      "activations/layer2_attention_weight_min": -27.63855743408203,
      "activations/layer3_attention_weight_max": 69.43920135498047,
      "activations/layer3_attention_weight_min": -72.21514129638672,
      "activations/layer4_attention_weight_max": 125.1333999633789,
      "activations/layer4_attention_weight_min": -113.58123016357422,
      "activations/layer5_attention_weight_max": 114.85723876953125,
      "activations/layer5_attention_weight_min": -94.18423461914062,
      "activations/layer6_attention_weight_max": 66.40164184570312,
      "activations/layer6_attention_weight_min": -49.825992584228516,
      "activations/layer7_attention_weight_max": 82.51380920410156,
      "activations/layer7_attention_weight_min": -67.25946044921875,
      "activations/layer8_attention_weight_max": 58.3895149230957,
      "activations/layer8_attention_weight_min": -42.614173889160156,
      "activations/layer9_attention_weight_max": 96.72774505615234,
      "activations/layer9_attention_weight_min": -61.709991455078125,
      "epoch": 2.18,
      "learning_rate": 0.00013731022727272725,
      "loss": 2.9498,
      "step": 37550
    },
    {
      "activations/layer0_attention_weight_max": 16.54744529724121,
      "activations/layer0_attention_weight_min": -11.677659034729004,
      "activations/layer10_attention_weight_max": 43.5323371887207,
      "activations/layer10_attention_weight_min": -39.82579040527344,
      "activations/layer11_attention_weight_max": 46.303688049316406,
      "activations/layer11_attention_weight_min": -35.79778289794922,
      "activations/layer12_attention_weight_max": 50.74614334106445,
      "activations/layer12_attention_weight_min": -34.54126739501953,
      "activations/layer13_attention_weight_max": 48.23395538330078,
      "activations/layer13_attention_weight_min": -35.868743896484375,
      "activations/layer14_attention_weight_max": 64.7400131225586,
      "activations/layer14_attention_weight_min": -52.099002838134766,
      "activations/layer15_attention_weight_max": 44.11086654663086,
      "activations/layer15_attention_weight_min": -35.059261322021484,
      "activations/layer16_attention_weight_max": 41.556095123291016,
      "activations/layer16_attention_weight_min": -34.417449951171875,
      "activations/layer17_attention_weight_max": 38.97712326049805,
      "activations/layer17_attention_weight_min": -26.631601333618164,
      "activations/layer18_attention_weight_max": 24.733530044555664,
      "activations/layer18_attention_weight_min": -16.6463680267334,
      "activations/layer19_attention_weight_max": 49.801612854003906,
      "activations/layer19_attention_weight_min": -31.862558364868164,
      "activations/layer1_attention_weight_max": 17.25241470336914,
      "activations/layer1_attention_weight_min": -11.104488372802734,
      "activations/layer20_attention_weight_max": 36.98039245605469,
      "activations/layer20_attention_weight_min": -25.22222137451172,
      "activations/layer21_attention_weight_max": 31.175962448120117,
      "activations/layer21_attention_weight_min": -18.184093475341797,
      "activations/layer22_attention_weight_max": 46.03852844238281,
      "activations/layer22_attention_weight_min": -26.805580139160156,
      "activations/layer23_attention_weight_max": 27.02735710144043,
      "activations/layer23_attention_weight_min": -16.76551055908203,
      "activations/layer2_attention_weight_max": 27.864219665527344,
      "activations/layer2_attention_weight_min": -28.404380798339844,
      "activations/layer3_attention_weight_max": 69.95880889892578,
      "activations/layer3_attention_weight_min": -73.07260131835938,
      "activations/layer4_attention_weight_max": 130.55230712890625,
      "activations/layer4_attention_weight_min": -117.6191635131836,
      "activations/layer5_attention_weight_max": 115.27523803710938,
      "activations/layer5_attention_weight_min": -101.41873168945312,
      "activations/layer6_attention_weight_max": 60.80430603027344,
      "activations/layer6_attention_weight_min": -52.11090850830078,
      "activations/layer7_attention_weight_max": 66.53701782226562,
      "activations/layer7_attention_weight_min": -60.34196472167969,
      "activations/layer8_attention_weight_max": 51.01838302612305,
      "activations/layer8_attention_weight_min": -40.062705993652344,
      "activations/layer9_attention_weight_max": 65.45433807373047,
      "activations/layer9_attention_weight_min": -55.5385856628418,
      "epoch": 2.18,
      "learning_rate": 0.00013729128787878786,
      "loss": 2.9368,
      "step": 37600
    },
    {
      "activations/layer0_attention_weight_max": 16.998088836669922,
      "activations/layer0_attention_weight_min": -11.54823112487793,
      "activations/layer10_attention_weight_max": 44.854949951171875,
      "activations/layer10_attention_weight_min": -40.99889373779297,
      "activations/layer11_attention_weight_max": 38.48124694824219,
      "activations/layer11_attention_weight_min": -34.55115509033203,
      "activations/layer12_attention_weight_max": 31.537412643432617,
      "activations/layer12_attention_weight_min": -30.500703811645508,
      "activations/layer13_attention_weight_max": 52.26447296142578,
      "activations/layer13_attention_weight_min": -38.315277099609375,
      "activations/layer14_attention_weight_max": 86.94271087646484,
      "activations/layer14_attention_weight_min": -59.14640426635742,
      "activations/layer15_attention_weight_max": 57.26948165893555,
      "activations/layer15_attention_weight_min": -36.809654235839844,
      "activations/layer16_attention_weight_max": 56.45125198364258,
      "activations/layer16_attention_weight_min": -36.017127990722656,
      "activations/layer17_attention_weight_max": 53.066707611083984,
      "activations/layer17_attention_weight_min": -29.38174819946289,
      "activations/layer18_attention_weight_max": 33.37963104248047,
      "activations/layer18_attention_weight_min": -19.65098762512207,
      "activations/layer19_attention_weight_max": 59.22492218017578,
      "activations/layer19_attention_weight_min": -32.44071578979492,
      "activations/layer1_attention_weight_max": 16.900911331176758,
      "activations/layer1_attention_weight_min": -11.131771087646484,
      "activations/layer20_attention_weight_max": 41.637081146240234,
      "activations/layer20_attention_weight_min": -24.419599533081055,
      "activations/layer21_attention_weight_max": 33.813480377197266,
      "activations/layer21_attention_weight_min": -18.884845733642578,
      "activations/layer22_attention_weight_max": 48.73662567138672,
      "activations/layer22_attention_weight_min": -27.94839859008789,
      "activations/layer23_attention_weight_max": 28.127330780029297,
      "activations/layer23_attention_weight_min": -17.986753463745117,
      "activations/layer2_attention_weight_max": 26.796142578125,
      "activations/layer2_attention_weight_min": -26.934688568115234,
      "activations/layer3_attention_weight_max": 72.58778381347656,
      "activations/layer3_attention_weight_min": -73.63638305664062,
      "activations/layer4_attention_weight_max": 123.19310760498047,
      "activations/layer4_attention_weight_min": -118.85601806640625,
      "activations/layer5_attention_weight_max": 110.92234802246094,
      "activations/layer5_attention_weight_min": -93.98794555664062,
      "activations/layer6_attention_weight_max": 57.519168853759766,
      "activations/layer6_attention_weight_min": -51.996517181396484,
      "activations/layer7_attention_weight_max": 65.7630615234375,
      "activations/layer7_attention_weight_min": -60.42622375488281,
      "activations/layer8_attention_weight_max": 47.01118850708008,
      "activations/layer8_attention_weight_min": -39.90642166137695,
      "activations/layer9_attention_weight_max": 59.51369094848633,
      "activations/layer9_attention_weight_min": -52.48939895629883,
      "epoch": 2.19,
      "learning_rate": 0.00013727234848484848,
      "loss": 2.9595,
      "step": 37650
    },
    {
      "activations/layer0_attention_weight_max": 17.317874908447266,
      "activations/layer0_attention_weight_min": -11.199372291564941,
      "activations/layer10_attention_weight_max": 51.264366149902344,
      "activations/layer10_attention_weight_min": -41.46068572998047,
      "activations/layer11_attention_weight_max": 40.91490936279297,
      "activations/layer11_attention_weight_min": -37.80093765258789,
      "activations/layer12_attention_weight_max": 43.47942352294922,
      "activations/layer12_attention_weight_min": -29.712099075317383,
      "activations/layer13_attention_weight_max": 49.74018478393555,
      "activations/layer13_attention_weight_min": -37.96257400512695,
      "activations/layer14_attention_weight_max": 82.72357940673828,
      "activations/layer14_attention_weight_min": -66.57876586914062,
      "activations/layer15_attention_weight_max": 52.254146575927734,
      "activations/layer15_attention_weight_min": -38.51331329345703,
      "activations/layer16_attention_weight_max": 52.88905715942383,
      "activations/layer16_attention_weight_min": -36.05498123168945,
      "activations/layer17_attention_weight_max": 44.930809020996094,
      "activations/layer17_attention_weight_min": -31.045162200927734,
      "activations/layer18_attention_weight_max": 27.02226448059082,
      "activations/layer18_attention_weight_min": -18.475566864013672,
      "activations/layer19_attention_weight_max": 50.93122482299805,
      "activations/layer19_attention_weight_min": -31.44516372680664,
      "activations/layer1_attention_weight_max": 18.704442977905273,
      "activations/layer1_attention_weight_min": -9.906450271606445,
      "activations/layer20_attention_weight_max": 39.35236740112305,
      "activations/layer20_attention_weight_min": -25.645402908325195,
      "activations/layer21_attention_weight_max": 33.798316955566406,
      "activations/layer21_attention_weight_min": -19.296424865722656,
      "activations/layer22_attention_weight_max": 47.733001708984375,
      "activations/layer22_attention_weight_min": -24.8496036529541,
      "activations/layer23_attention_weight_max": 25.467336654663086,
      "activations/layer23_attention_weight_min": -17.30411148071289,
      "activations/layer2_attention_weight_max": 27.39504623413086,
      "activations/layer2_attention_weight_min": -27.589033126831055,
      "activations/layer3_attention_weight_max": 69.89599609375,
      "activations/layer3_attention_weight_min": -71.53768920898438,
      "activations/layer4_attention_weight_max": 128.7041473388672,
      "activations/layer4_attention_weight_min": -118.91876220703125,
      "activations/layer5_attention_weight_max": 110.69535827636719,
      "activations/layer5_attention_weight_min": -93.98960876464844,
      "activations/layer6_attention_weight_max": 60.64331817626953,
      "activations/layer6_attention_weight_min": -53.51102828979492,
      "activations/layer7_attention_weight_max": 66.32506561279297,
      "activations/layer7_attention_weight_min": -69.80268859863281,
      "activations/layer8_attention_weight_max": 49.43363952636719,
      "activations/layer8_attention_weight_min": -40.20927429199219,
      "activations/layer9_attention_weight_max": 71.99047088623047,
      "activations/layer9_attention_weight_min": -50.442134857177734,
      "epoch": 2.19,
      "learning_rate": 0.00013725340909090907,
      "loss": 2.9677,
      "step": 37700
    },
    {
      "activations/layer0_attention_weight_max": 15.145463943481445,
      "activations/layer0_attention_weight_min": -11.484146118164062,
      "activations/layer10_attention_weight_max": 45.18048095703125,
      "activations/layer10_attention_weight_min": -40.1915283203125,
      "activations/layer11_attention_weight_max": 36.941070556640625,
      "activations/layer11_attention_weight_min": -33.254150390625,
      "activations/layer12_attention_weight_max": 28.9377384185791,
      "activations/layer12_attention_weight_min": -25.88283920288086,
      "activations/layer13_attention_weight_max": 49.35116195678711,
      "activations/layer13_attention_weight_min": -35.520233154296875,
      "activations/layer14_attention_weight_max": 68.95648193359375,
      "activations/layer14_attention_weight_min": -55.37979507446289,
      "activations/layer15_attention_weight_max": 54.291465759277344,
      "activations/layer15_attention_weight_min": -39.6004638671875,
      "activations/layer16_attention_weight_max": 49.205345153808594,
      "activations/layer16_attention_weight_min": -40.79463195800781,
      "activations/layer17_attention_weight_max": 45.006431579589844,
      "activations/layer17_attention_weight_min": -30.594757080078125,
      "activations/layer18_attention_weight_max": 27.773332595825195,
      "activations/layer18_attention_weight_min": -19.37398910522461,
      "activations/layer19_attention_weight_max": 55.95056915283203,
      "activations/layer19_attention_weight_min": -33.78028106689453,
      "activations/layer1_attention_weight_max": 17.61212921142578,
      "activations/layer1_attention_weight_min": -10.816527366638184,
      "activations/layer20_attention_weight_max": 37.474159240722656,
      "activations/layer20_attention_weight_min": -24.126304626464844,
      "activations/layer21_attention_weight_max": 32.126075744628906,
      "activations/layer21_attention_weight_min": -20.657812118530273,
      "activations/layer22_attention_weight_max": 44.63782501220703,
      "activations/layer22_attention_weight_min": -24.918598175048828,
      "activations/layer23_attention_weight_max": 26.311870574951172,
      "activations/layer23_attention_weight_min": -16.928375244140625,
      "activations/layer2_attention_weight_max": 28.925275802612305,
      "activations/layer2_attention_weight_min": -29.577735900878906,
      "activations/layer3_attention_weight_max": 71.81761932373047,
      "activations/layer3_attention_weight_min": -73.680419921875,
      "activations/layer4_attention_weight_max": 121.9222183227539,
      "activations/layer4_attention_weight_min": -113.6840591430664,
      "activations/layer5_attention_weight_max": 110.83895874023438,
      "activations/layer5_attention_weight_min": -94.2459945678711,
      "activations/layer6_attention_weight_max": 59.79573440551758,
      "activations/layer6_attention_weight_min": -50.36449432373047,
      "activations/layer7_attention_weight_max": 68.75204467773438,
      "activations/layer7_attention_weight_min": -67.79988861083984,
      "activations/layer8_attention_weight_max": 42.944522857666016,
      "activations/layer8_attention_weight_min": -37.75664520263672,
      "activations/layer9_attention_weight_max": 59.691436767578125,
      "activations/layer9_attention_weight_min": -47.88348388671875,
      "epoch": 2.19,
      "learning_rate": 0.00013723446969696968,
      "loss": 2.9424,
      "step": 37750
    },
    {
      "activations/layer0_attention_weight_max": 16.764253616333008,
      "activations/layer0_attention_weight_min": -12.043395042419434,
      "activations/layer10_attention_weight_max": 50.34260940551758,
      "activations/layer10_attention_weight_min": -39.80884552001953,
      "activations/layer11_attention_weight_max": 42.157405853271484,
      "activations/layer11_attention_weight_min": -37.008758544921875,
      "activations/layer12_attention_weight_max": 47.132232666015625,
      "activations/layer12_attention_weight_min": -30.24197006225586,
      "activations/layer13_attention_weight_max": 58.610660552978516,
      "activations/layer13_attention_weight_min": -39.624595642089844,
      "activations/layer14_attention_weight_max": 84.96200561523438,
      "activations/layer14_attention_weight_min": -58.73447799682617,
      "activations/layer15_attention_weight_max": 52.299434661865234,
      "activations/layer15_attention_weight_min": -34.00029754638672,
      "activations/layer16_attention_weight_max": 53.1147346496582,
      "activations/layer16_attention_weight_min": -35.67416000366211,
      "activations/layer17_attention_weight_max": 40.37239456176758,
      "activations/layer17_attention_weight_min": -28.31437110900879,
      "activations/layer18_attention_weight_max": 26.20444679260254,
      "activations/layer18_attention_weight_min": -17.69173812866211,
      "activations/layer19_attention_weight_max": 54.2233772277832,
      "activations/layer19_attention_weight_min": -30.297277450561523,
      "activations/layer1_attention_weight_max": 18.64645767211914,
      "activations/layer1_attention_weight_min": -10.57326889038086,
      "activations/layer20_attention_weight_max": 43.818748474121094,
      "activations/layer20_attention_weight_min": -24.218978881835938,
      "activations/layer21_attention_weight_max": 31.345823287963867,
      "activations/layer21_attention_weight_min": -18.095169067382812,
      "activations/layer22_attention_weight_max": 44.021549224853516,
      "activations/layer22_attention_weight_min": -25.579404830932617,
      "activations/layer23_attention_weight_max": 28.452648162841797,
      "activations/layer23_attention_weight_min": -14.494768142700195,
      "activations/layer2_attention_weight_max": 28.907772064208984,
      "activations/layer2_attention_weight_min": -29.83260726928711,
      "activations/layer3_attention_weight_max": 73.63143157958984,
      "activations/layer3_attention_weight_min": -75.01744079589844,
      "activations/layer4_attention_weight_max": 124.75504302978516,
      "activations/layer4_attention_weight_min": -118.39418029785156,
      "activations/layer5_attention_weight_max": 115.26145935058594,
      "activations/layer5_attention_weight_min": -99.43853759765625,
      "activations/layer6_attention_weight_max": 63.71870040893555,
      "activations/layer6_attention_weight_min": -57.55360412597656,
      "activations/layer7_attention_weight_max": 68.24378204345703,
      "activations/layer7_attention_weight_min": -66.6214370727539,
      "activations/layer8_attention_weight_max": 47.56072235107422,
      "activations/layer8_attention_weight_min": -42.34296798706055,
      "activations/layer9_attention_weight_max": 67.46315002441406,
      "activations/layer9_attention_weight_min": -51.93182373046875,
      "epoch": 2.2,
      "learning_rate": 0.0001372155303030303,
      "loss": 2.9486,
      "step": 37800
    },
    {
      "activations/layer0_attention_weight_max": 16.856948852539062,
      "activations/layer0_attention_weight_min": -12.227679252624512,
      "activations/layer10_attention_weight_max": 61.19342041015625,
      "activations/layer10_attention_weight_min": -46.59861373901367,
      "activations/layer11_attention_weight_max": 43.08074951171875,
      "activations/layer11_attention_weight_min": -35.55009460449219,
      "activations/layer12_attention_weight_max": 35.54347229003906,
      "activations/layer12_attention_weight_min": -25.032489776611328,
      "activations/layer13_attention_weight_max": 58.85413360595703,
      "activations/layer13_attention_weight_min": -39.49858093261719,
      "activations/layer14_attention_weight_max": 90.21717071533203,
      "activations/layer14_attention_weight_min": -61.05086898803711,
      "activations/layer15_attention_weight_max": 50.46621322631836,
      "activations/layer15_attention_weight_min": -41.17247772216797,
      "activations/layer16_attention_weight_max": 46.136383056640625,
      "activations/layer16_attention_weight_min": -34.45256423950195,
      "activations/layer17_attention_weight_max": 47.74238586425781,
      "activations/layer17_attention_weight_min": -35.52147674560547,
      "activations/layer18_attention_weight_max": 29.805763244628906,
      "activations/layer18_attention_weight_min": -19.895187377929688,
      "activations/layer19_attention_weight_max": 55.40083694458008,
      "activations/layer19_attention_weight_min": -34.32661056518555,
      "activations/layer1_attention_weight_max": 17.93143653869629,
      "activations/layer1_attention_weight_min": -11.365372657775879,
      "activations/layer20_attention_weight_max": 42.956974029541016,
      "activations/layer20_attention_weight_min": -26.642009735107422,
      "activations/layer21_attention_weight_max": 31.80275535583496,
      "activations/layer21_attention_weight_min": -23.03228759765625,
      "activations/layer22_attention_weight_max": 50.11798858642578,
      "activations/layer22_attention_weight_min": -26.443405151367188,
      "activations/layer23_attention_weight_max": 28.511333465576172,
      "activations/layer23_attention_weight_min": -15.926982879638672,
      "activations/layer2_attention_weight_max": 28.97539520263672,
      "activations/layer2_attention_weight_min": -28.38121795654297,
      "activations/layer3_attention_weight_max": 71.54241943359375,
      "activations/layer3_attention_weight_min": -74.71419525146484,
      "activations/layer4_attention_weight_max": 128.40646362304688,
      "activations/layer4_attention_weight_min": -117.78376770019531,
      "activations/layer5_attention_weight_max": 112.89630126953125,
      "activations/layer5_attention_weight_min": -100.01011657714844,
      "activations/layer6_attention_weight_max": 62.07512283325195,
      "activations/layer6_attention_weight_min": -53.23209762573242,
      "activations/layer7_attention_weight_max": 65.39546966552734,
      "activations/layer7_attention_weight_min": -65.83277130126953,
      "activations/layer8_attention_weight_max": 45.58604049682617,
      "activations/layer8_attention_weight_min": -37.545867919921875,
      "activations/layer9_attention_weight_max": 65.90308380126953,
      "activations/layer9_attention_weight_min": -51.2537727355957,
      "epoch": 2.2,
      "learning_rate": 0.0001371965909090909,
      "loss": 2.9496,
      "step": 37850
    },
    {
      "activations/layer0_attention_weight_max": 14.82143783569336,
      "activations/layer0_attention_weight_min": -12.075407981872559,
      "activations/layer10_attention_weight_max": 48.637847900390625,
      "activations/layer10_attention_weight_min": -40.98318862915039,
      "activations/layer11_attention_weight_max": 40.02762222290039,
      "activations/layer11_attention_weight_min": -37.26873779296875,
      "activations/layer12_attention_weight_max": 36.21730041503906,
      "activations/layer12_attention_weight_min": -28.04337501525879,
      "activations/layer13_attention_weight_max": 56.256317138671875,
      "activations/layer13_attention_weight_min": -38.75996398925781,
      "activations/layer14_attention_weight_max": 81.22970581054688,
      "activations/layer14_attention_weight_min": -63.632694244384766,
      "activations/layer15_attention_weight_max": 53.810943603515625,
      "activations/layer15_attention_weight_min": -38.23447036743164,
      "activations/layer16_attention_weight_max": 52.107444763183594,
      "activations/layer16_attention_weight_min": -36.23440170288086,
      "activations/layer17_attention_weight_max": 48.143436431884766,
      "activations/layer17_attention_weight_min": -30.3709659576416,
      "activations/layer18_attention_weight_max": 30.260353088378906,
      "activations/layer18_attention_weight_min": -18.81148338317871,
      "activations/layer19_attention_weight_max": 48.61164093017578,
      "activations/layer19_attention_weight_min": -33.59236145019531,
      "activations/layer1_attention_weight_max": 17.74690055847168,
      "activations/layer1_attention_weight_min": -10.52694320678711,
      "activations/layer20_attention_weight_max": 36.706180572509766,
      "activations/layer20_attention_weight_min": -26.120267868041992,
      "activations/layer21_attention_weight_max": 34.27671813964844,
      "activations/layer21_attention_weight_min": -22.814733505249023,
      "activations/layer22_attention_weight_max": 45.89351272583008,
      "activations/layer22_attention_weight_min": -25.1264591217041,
      "activations/layer23_attention_weight_max": 26.320005416870117,
      "activations/layer23_attention_weight_min": -17.15936279296875,
      "activations/layer2_attention_weight_max": 30.7181396484375,
      "activations/layer2_attention_weight_min": -29.766874313354492,
      "activations/layer3_attention_weight_max": 77.00305938720703,
      "activations/layer3_attention_weight_min": -77.84574127197266,
      "activations/layer4_attention_weight_max": 129.8902587890625,
      "activations/layer4_attention_weight_min": -120.08238983154297,
      "activations/layer5_attention_weight_max": 114.105224609375,
      "activations/layer5_attention_weight_min": -102.49079895019531,
      "activations/layer6_attention_weight_max": 59.89080047607422,
      "activations/layer6_attention_weight_min": -53.60560989379883,
      "activations/layer7_attention_weight_max": 71.66539764404297,
      "activations/layer7_attention_weight_min": -66.3714370727539,
      "activations/layer8_attention_weight_max": 43.687068939208984,
      "activations/layer8_attention_weight_min": -39.57450866699219,
      "activations/layer9_attention_weight_max": 69.34898376464844,
      "activations/layer9_attention_weight_min": -60.64884567260742,
      "epoch": 2.2,
      "learning_rate": 0.0001371776515151515,
      "loss": 2.9438,
      "step": 37900
    },
    {
      "activations/layer0_attention_weight_max": 14.676604270935059,
      "activations/layer0_attention_weight_min": -11.980731964111328,
      "activations/layer10_attention_weight_max": 57.940914154052734,
      "activations/layer10_attention_weight_min": -42.6103630065918,
      "activations/layer11_attention_weight_max": 48.66045379638672,
      "activations/layer11_attention_weight_min": -38.23269271850586,
      "activations/layer12_attention_weight_max": 47.32831573486328,
      "activations/layer12_attention_weight_min": -33.71683883666992,
      "activations/layer13_attention_weight_max": 59.05565643310547,
      "activations/layer13_attention_weight_min": -34.702354431152344,
      "activations/layer14_attention_weight_max": 94.36695861816406,
      "activations/layer14_attention_weight_min": -59.74055862426758,
      "activations/layer15_attention_weight_max": 62.19392013549805,
      "activations/layer15_attention_weight_min": -38.32608413696289,
      "activations/layer16_attention_weight_max": 55.02534484863281,
      "activations/layer16_attention_weight_min": -37.921791076660156,
      "activations/layer17_attention_weight_max": 58.8264045715332,
      "activations/layer17_attention_weight_min": -30.82274055480957,
      "activations/layer18_attention_weight_max": 35.17336654663086,
      "activations/layer18_attention_weight_min": -18.126192092895508,
      "activations/layer19_attention_weight_max": 53.71381759643555,
      "activations/layer19_attention_weight_min": -28.897140502929688,
      "activations/layer1_attention_weight_max": 17.710939407348633,
      "activations/layer1_attention_weight_min": -11.35928726196289,
      "activations/layer20_attention_weight_max": 40.40372085571289,
      "activations/layer20_attention_weight_min": -22.151914596557617,
      "activations/layer21_attention_weight_max": 33.21526336669922,
      "activations/layer21_attention_weight_min": -19.584869384765625,
      "activations/layer22_attention_weight_max": 47.57047653198242,
      "activations/layer22_attention_weight_min": -25.80127716064453,
      "activations/layer23_attention_weight_max": 24.749595642089844,
      "activations/layer23_attention_weight_min": -17.23061752319336,
      "activations/layer2_attention_weight_max": 30.790042877197266,
      "activations/layer2_attention_weight_min": -29.96759605407715,
      "activations/layer3_attention_weight_max": 78.05381774902344,
      "activations/layer3_attention_weight_min": -78.00577545166016,
      "activations/layer4_attention_weight_max": 136.7696075439453,
      "activations/layer4_attention_weight_min": -126.45909881591797,
      "activations/layer5_attention_weight_max": 123.05560302734375,
      "activations/layer5_attention_weight_min": -102.55973815917969,
      "activations/layer6_attention_weight_max": 64.78600311279297,
      "activations/layer6_attention_weight_min": -54.31288528442383,
      "activations/layer7_attention_weight_max": 74.2300796508789,
      "activations/layer7_attention_weight_min": -67.4461898803711,
      "activations/layer8_attention_weight_max": 47.85215377807617,
      "activations/layer8_attention_weight_min": -40.42341995239258,
      "activations/layer9_attention_weight_max": 74.12833404541016,
      "activations/layer9_attention_weight_min": -52.08317947387695,
      "epoch": 2.21,
      "learning_rate": 0.00013715871212121211,
      "loss": 2.9586,
      "step": 37950
    },
    {
      "activations/layer0_attention_weight_max": 16.560100555419922,
      "activations/layer0_attention_weight_min": -12.089402198791504,
      "activations/layer10_attention_weight_max": 55.93565368652344,
      "activations/layer10_attention_weight_min": -42.60935974121094,
      "activations/layer11_attention_weight_max": 40.84390640258789,
      "activations/layer11_attention_weight_min": -32.3877067565918,
      "activations/layer12_attention_weight_max": 60.54749298095703,
      "activations/layer12_attention_weight_min": -39.609676361083984,
      "activations/layer13_attention_weight_max": 53.95199203491211,
      "activations/layer13_attention_weight_min": -46.25693893432617,
      "activations/layer14_attention_weight_max": 86.57640075683594,
      "activations/layer14_attention_weight_min": -53.880916595458984,
      "activations/layer15_attention_weight_max": 51.69585418701172,
      "activations/layer15_attention_weight_min": -37.10537338256836,
      "activations/layer16_attention_weight_max": 48.758872985839844,
      "activations/layer16_attention_weight_min": -34.84164047241211,
      "activations/layer17_attention_weight_max": 47.20055389404297,
      "activations/layer17_attention_weight_min": -31.073246002197266,
      "activations/layer18_attention_weight_max": 28.162004470825195,
      "activations/layer18_attention_weight_min": -19.435962677001953,
      "activations/layer19_attention_weight_max": 52.78815460205078,
      "activations/layer19_attention_weight_min": -30.1690731048584,
      "activations/layer1_attention_weight_max": 17.709999084472656,
      "activations/layer1_attention_weight_min": -11.056015014648438,
      "activations/layer20_attention_weight_max": 37.539756774902344,
      "activations/layer20_attention_weight_min": -23.73301887512207,
      "activations/layer21_attention_weight_max": 31.192150115966797,
      "activations/layer21_attention_weight_min": -20.345090866088867,
      "activations/layer22_attention_weight_max": 42.74683380126953,
      "activations/layer22_attention_weight_min": -27.85381317138672,
      "activations/layer23_attention_weight_max": 24.713279724121094,
      "activations/layer23_attention_weight_min": -17.13815689086914,
      "activations/layer2_attention_weight_max": 29.73444366455078,
      "activations/layer2_attention_weight_min": -29.237899780273438,
      "activations/layer3_attention_weight_max": 74.25765228271484,
      "activations/layer3_attention_weight_min": -75.9419174194336,
      "activations/layer4_attention_weight_max": 130.45297241210938,
      "activations/layer4_attention_weight_min": -119.46833801269531,
      "activations/layer5_attention_weight_max": 116.79620361328125,
      "activations/layer5_attention_weight_min": -96.03660583496094,
      "activations/layer6_attention_weight_max": 66.6292953491211,
      "activations/layer6_attention_weight_min": -52.52307891845703,
      "activations/layer7_attention_weight_max": 70.44925689697266,
      "activations/layer7_attention_weight_min": -65.66548156738281,
      "activations/layer8_attention_weight_max": 49.34120559692383,
      "activations/layer8_attention_weight_min": -39.71348190307617,
      "activations/layer9_attention_weight_max": 75.03065490722656,
      "activations/layer9_attention_weight_min": -66.76647186279297,
      "epoch": 2.21,
      "learning_rate": 0.00013713977272727273,
      "loss": 2.954,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_loss": 2.900390625,
      "eval_runtime": 8.4887,
      "eval_samples_per_second": 505.852,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_openwebtext_loss": 2.900390625,
      "eval_openwebtext_ppl": 18.181246031735814,
      "eval_openwebtext_runtime": 8.4887,
      "eval_openwebtext_samples_per_second": 505.852,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_wikitext_loss": 3.169921875,
      "eval_wikitext_ppl": 23.80562446936611,
      "eval_wikitext_runtime": 2.0236,
      "eval_wikitext_samples_per_second": 225.341,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_lambada_loss": 3.01171875,
      "eval_lambada_ppl": 20.322298876557472,
      "eval_lambada_runtime": 9.5834,
      "eval_lambada_samples_per_second": 508.065,
      "step": 38000
    },
    {
      "activations/layer0_attention_weight_max": 16.045679092407227,
      "activations/layer0_attention_weight_min": -11.416187286376953,
      "activations/layer10_attention_weight_max": 52.85835647583008,
      "activations/layer10_attention_weight_min": -42.76590347290039,
      "activations/layer11_attention_weight_max": 43.916046142578125,
      "activations/layer11_attention_weight_min": -34.10624694824219,
      "activations/layer12_attention_weight_max": 38.230159759521484,
      "activations/layer12_attention_weight_min": -28.77057456970215,
      "activations/layer13_attention_weight_max": 60.11104202270508,
      "activations/layer13_attention_weight_min": -40.27058029174805,
      "activations/layer14_attention_weight_max": 96.88488006591797,
      "activations/layer14_attention_weight_min": -63.07901382446289,
      "activations/layer15_attention_weight_max": 57.07456970214844,
      "activations/layer15_attention_weight_min": -37.072750091552734,
      "activations/layer16_attention_weight_max": 54.368682861328125,
      "activations/layer16_attention_weight_min": -36.5943489074707,
      "activations/layer17_attention_weight_max": 52.574737548828125,
      "activations/layer17_attention_weight_min": -34.802616119384766,
      "activations/layer18_attention_weight_max": 33.371978759765625,
      "activations/layer18_attention_weight_min": -17.793420791625977,
      "activations/layer19_attention_weight_max": 61.45849609375,
      "activations/layer19_attention_weight_min": -32.263221740722656,
      "activations/layer1_attention_weight_max": 17.884014129638672,
      "activations/layer1_attention_weight_min": -10.418619155883789,
      "activations/layer20_attention_weight_max": 38.3978385925293,
      "activations/layer20_attention_weight_min": -22.73603057861328,
      "activations/layer21_attention_weight_max": 36.62167739868164,
      "activations/layer21_attention_weight_min": -21.78885269165039,
      "activations/layer22_attention_weight_max": 49.65446853637695,
      "activations/layer22_attention_weight_min": -25.291780471801758,
      "activations/layer23_attention_weight_max": 28.175106048583984,
      "activations/layer23_attention_weight_min": -16.009214401245117,
      "activations/layer2_attention_weight_max": 27.911344528198242,
      "activations/layer2_attention_weight_min": -26.501243591308594,
      "activations/layer3_attention_weight_max": 66.69642639160156,
      "activations/layer3_attention_weight_min": -73.02648162841797,
      "activations/layer4_attention_weight_max": 126.02230072021484,
      "activations/layer4_attention_weight_min": -125.13516998291016,
      "activations/layer5_attention_weight_max": 110.37434387207031,
      "activations/layer5_attention_weight_min": -99.40386199951172,
      "activations/layer6_attention_weight_max": 61.916446685791016,
      "activations/layer6_attention_weight_min": -53.47179412841797,
      "activations/layer7_attention_weight_max": 73.97101593017578,
      "activations/layer7_attention_weight_min": -66.41500854492188,
      "activations/layer8_attention_weight_max": 49.024505615234375,
      "activations/layer8_attention_weight_min": -39.71324157714844,
      "activations/layer9_attention_weight_max": 70.47549438476562,
      "activations/layer9_attention_weight_min": -53.44271469116211,
      "epoch": 2.21,
      "learning_rate": 0.00013712083333333332,
      "loss": 2.9484,
      "step": 38050
    },
    {
      "activations/layer0_attention_weight_max": 16.43267250061035,
      "activations/layer0_attention_weight_min": -11.804461479187012,
      "activations/layer10_attention_weight_max": 54.90287780761719,
      "activations/layer10_attention_weight_min": -45.309444427490234,
      "activations/layer11_attention_weight_max": 41.588016510009766,
      "activations/layer11_attention_weight_min": -35.430843353271484,
      "activations/layer12_attention_weight_max": 69.57918548583984,
      "activations/layer12_attention_weight_min": -45.32563018798828,
      "activations/layer13_attention_weight_max": 82.4027099609375,
      "activations/layer13_attention_weight_min": -58.59360122680664,
      "activations/layer14_attention_weight_max": 73.56754302978516,
      "activations/layer14_attention_weight_min": -60.12728500366211,
      "activations/layer15_attention_weight_max": 53.949039459228516,
      "activations/layer15_attention_weight_min": -37.4398078918457,
      "activations/layer16_attention_weight_max": 53.24960708618164,
      "activations/layer16_attention_weight_min": -41.48513412475586,
      "activations/layer17_attention_weight_max": 43.67269515991211,
      "activations/layer17_attention_weight_min": -28.267803192138672,
      "activations/layer18_attention_weight_max": 27.647663116455078,
      "activations/layer18_attention_weight_min": -17.70629119873047,
      "activations/layer19_attention_weight_max": 53.3067512512207,
      "activations/layer19_attention_weight_min": -34.426815032958984,
      "activations/layer1_attention_weight_max": 17.705350875854492,
      "activations/layer1_attention_weight_min": -12.881939888000488,
      "activations/layer20_attention_weight_max": 37.51227569580078,
      "activations/layer20_attention_weight_min": -25.373388290405273,
      "activations/layer21_attention_weight_max": 32.66817855834961,
      "activations/layer21_attention_weight_min": -20.280132293701172,
      "activations/layer22_attention_weight_max": 45.02442169189453,
      "activations/layer22_attention_weight_min": -24.25011444091797,
      "activations/layer23_attention_weight_max": 26.99130630493164,
      "activations/layer23_attention_weight_min": -15.973830223083496,
      "activations/layer2_attention_weight_max": 29.156963348388672,
      "activations/layer2_attention_weight_min": -28.332685470581055,
      "activations/layer3_attention_weight_max": 68.1523666381836,
      "activations/layer3_attention_weight_min": -72.0204849243164,
      "activations/layer4_attention_weight_max": 127.6863784790039,
      "activations/layer4_attention_weight_min": -120.10855865478516,
      "activations/layer5_attention_weight_max": 112.04266357421875,
      "activations/layer5_attention_weight_min": -102.02267456054688,
      "activations/layer6_attention_weight_max": 63.2071647644043,
      "activations/layer6_attention_weight_min": -59.928131103515625,
      "activations/layer7_attention_weight_max": 69.8327407836914,
      "activations/layer7_attention_weight_min": -75.96502685546875,
      "activations/layer8_attention_weight_max": 46.56755447387695,
      "activations/layer8_attention_weight_min": -41.800960540771484,
      "activations/layer9_attention_weight_max": 87.46119689941406,
      "activations/layer9_attention_weight_min": -66.42589569091797,
      "epoch": 2.21,
      "learning_rate": 0.00013710189393939393,
      "loss": 2.9531,
      "step": 38100
    },
    {
      "activations/layer0_attention_weight_max": 15.610159873962402,
      "activations/layer0_attention_weight_min": -12.461052894592285,
      "activations/layer10_attention_weight_max": 47.48630905151367,
      "activations/layer10_attention_weight_min": -38.09759521484375,
      "activations/layer11_attention_weight_max": 46.74967956542969,
      "activations/layer11_attention_weight_min": -33.07112121582031,
      "activations/layer12_attention_weight_max": 39.79486083984375,
      "activations/layer12_attention_weight_min": -32.238311767578125,
      "activations/layer13_attention_weight_max": 41.18193817138672,
      "activations/layer13_attention_weight_min": -35.303585052490234,
      "activations/layer14_attention_weight_max": 83.06234741210938,
      "activations/layer14_attention_weight_min": -62.7144775390625,
      "activations/layer15_attention_weight_max": 47.88119888305664,
      "activations/layer15_attention_weight_min": -38.22449493408203,
      "activations/layer16_attention_weight_max": 49.86704635620117,
      "activations/layer16_attention_weight_min": -34.5531120300293,
      "activations/layer17_attention_weight_max": 43.75382614135742,
      "activations/layer17_attention_weight_min": -30.753530502319336,
      "activations/layer18_attention_weight_max": 25.660358428955078,
      "activations/layer18_attention_weight_min": -17.913301467895508,
      "activations/layer19_attention_weight_max": 46.275508880615234,
      "activations/layer19_attention_weight_min": -30.775558471679688,
      "activations/layer1_attention_weight_max": 17.928781509399414,
      "activations/layer1_attention_weight_min": -10.92117691040039,
      "activations/layer20_attention_weight_max": 35.567657470703125,
      "activations/layer20_attention_weight_min": -26.303037643432617,
      "activations/layer21_attention_weight_max": 31.218027114868164,
      "activations/layer21_attention_weight_min": -20.560829162597656,
      "activations/layer22_attention_weight_max": 40.439334869384766,
      "activations/layer22_attention_weight_min": -26.153181076049805,
      "activations/layer23_attention_weight_max": 25.015457153320312,
      "activations/layer23_attention_weight_min": -15.664130210876465,
      "activations/layer2_attention_weight_max": 28.936277389526367,
      "activations/layer2_attention_weight_min": -29.032955169677734,
      "activations/layer3_attention_weight_max": 70.61592864990234,
      "activations/layer3_attention_weight_min": -73.60157775878906,
      "activations/layer4_attention_weight_max": 128.73318481445312,
      "activations/layer4_attention_weight_min": -120.4767837524414,
      "activations/layer5_attention_weight_max": 107.63922119140625,
      "activations/layer5_attention_weight_min": -96.98600769042969,
      "activations/layer6_attention_weight_max": 60.95645523071289,
      "activations/layer6_attention_weight_min": -53.35783767700195,
      "activations/layer7_attention_weight_max": 76.10348510742188,
      "activations/layer7_attention_weight_min": -64.00206756591797,
      "activations/layer8_attention_weight_max": 50.010311126708984,
      "activations/layer8_attention_weight_min": -40.41811752319336,
      "activations/layer9_attention_weight_max": 69.71881866455078,
      "activations/layer9_attention_weight_min": -49.637359619140625,
      "epoch": 2.22,
      "learning_rate": 0.00013708295454545455,
      "loss": 2.947,
      "step": 38150
    },
    {
      "activations/layer0_attention_weight_max": 16.756122589111328,
      "activations/layer0_attention_weight_min": -12.11633586883545,
      "activations/layer10_attention_weight_max": 44.219085693359375,
      "activations/layer10_attention_weight_min": -37.46409225463867,
      "activations/layer11_attention_weight_max": 39.201438903808594,
      "activations/layer11_attention_weight_min": -33.985687255859375,
      "activations/layer12_attention_weight_max": 48.68124008178711,
      "activations/layer12_attention_weight_min": -28.100704193115234,
      "activations/layer13_attention_weight_max": 51.90782165527344,
      "activations/layer13_attention_weight_min": -38.29723358154297,
      "activations/layer14_attention_weight_max": 77.69912719726562,
      "activations/layer14_attention_weight_min": -55.91199493408203,
      "activations/layer15_attention_weight_max": 53.0887336730957,
      "activations/layer15_attention_weight_min": -35.70295333862305,
      "activations/layer16_attention_weight_max": 51.831607818603516,
      "activations/layer16_attention_weight_min": -38.7119255065918,
      "activations/layer17_attention_weight_max": 46.31464385986328,
      "activations/layer17_attention_weight_min": -28.357019424438477,
      "activations/layer18_attention_weight_max": 28.4443416595459,
      "activations/layer18_attention_weight_min": -19.020736694335938,
      "activations/layer19_attention_weight_max": 49.48907470703125,
      "activations/layer19_attention_weight_min": -34.441673278808594,
      "activations/layer1_attention_weight_max": 17.661968231201172,
      "activations/layer1_attention_weight_min": -11.028925895690918,
      "activations/layer20_attention_weight_max": 39.179134368896484,
      "activations/layer20_attention_weight_min": -25.834327697753906,
      "activations/layer21_attention_weight_max": 31.56545639038086,
      "activations/layer21_attention_weight_min": -18.708728790283203,
      "activations/layer22_attention_weight_max": 44.86111831665039,
      "activations/layer22_attention_weight_min": -25.57395362854004,
      "activations/layer23_attention_weight_max": 22.9848690032959,
      "activations/layer23_attention_weight_min": -15.540472984313965,
      "activations/layer2_attention_weight_max": 27.673551559448242,
      "activations/layer2_attention_weight_min": -27.6688232421875,
      "activations/layer3_attention_weight_max": 71.30036926269531,
      "activations/layer3_attention_weight_min": -71.52252197265625,
      "activations/layer4_attention_weight_max": 125.5436782836914,
      "activations/layer4_attention_weight_min": -121.47749328613281,
      "activations/layer5_attention_weight_max": 108.41346740722656,
      "activations/layer5_attention_weight_min": -97.18402099609375,
      "activations/layer6_attention_weight_max": 61.41192626953125,
      "activations/layer6_attention_weight_min": -52.47335433959961,
      "activations/layer7_attention_weight_max": 68.1584243774414,
      "activations/layer7_attention_weight_min": -68.24726104736328,
      "activations/layer8_attention_weight_max": 44.2302131652832,
      "activations/layer8_attention_weight_min": -37.94485855102539,
      "activations/layer9_attention_weight_max": 65.92781829833984,
      "activations/layer9_attention_weight_min": -48.808509826660156,
      "epoch": 2.22,
      "learning_rate": 0.00013706401515151513,
      "loss": 2.9548,
      "step": 38200
    },
    {
      "activations/layer0_attention_weight_max": 16.799528121948242,
      "activations/layer0_attention_weight_min": -12.228312492370605,
      "activations/layer10_attention_weight_max": 56.04119873046875,
      "activations/layer10_attention_weight_min": -41.88475036621094,
      "activations/layer11_attention_weight_max": 52.384666442871094,
      "activations/layer11_attention_weight_min": -39.35041427612305,
      "activations/layer12_attention_weight_max": 45.78801727294922,
      "activations/layer12_attention_weight_min": -32.98126983642578,
      "activations/layer13_attention_weight_max": 57.54530334472656,
      "activations/layer13_attention_weight_min": -39.579925537109375,
      "activations/layer14_attention_weight_max": 110.66767883300781,
      "activations/layer14_attention_weight_min": -68.23689270019531,
      "activations/layer15_attention_weight_max": 52.458492279052734,
      "activations/layer15_attention_weight_min": -34.442623138427734,
      "activations/layer16_attention_weight_max": 47.39664077758789,
      "activations/layer16_attention_weight_min": -35.10751724243164,
      "activations/layer17_attention_weight_max": 45.719886779785156,
      "activations/layer17_attention_weight_min": -27.798120498657227,
      "activations/layer18_attention_weight_max": 26.43789291381836,
      "activations/layer18_attention_weight_min": -18.65483856201172,
      "activations/layer19_attention_weight_max": 45.106201171875,
      "activations/layer19_attention_weight_min": -31.19060707092285,
      "activations/layer1_attention_weight_max": 16.846067428588867,
      "activations/layer1_attention_weight_min": -12.305679321289062,
      "activations/layer20_attention_weight_max": 35.580902099609375,
      "activations/layer20_attention_weight_min": -24.63552474975586,
      "activations/layer21_attention_weight_max": 31.524808883666992,
      "activations/layer21_attention_weight_min": -22.690641403198242,
      "activations/layer22_attention_weight_max": 42.82098388671875,
      "activations/layer22_attention_weight_min": -25.437049865722656,
      "activations/layer23_attention_weight_max": 23.84462547302246,
      "activations/layer23_attention_weight_min": -15.58320140838623,
      "activations/layer2_attention_weight_max": 30.040014266967773,
      "activations/layer2_attention_weight_min": -29.97806167602539,
      "activations/layer3_attention_weight_max": 75.28804016113281,
      "activations/layer3_attention_weight_min": -76.0674057006836,
      "activations/layer4_attention_weight_max": 130.65428161621094,
      "activations/layer4_attention_weight_min": -117.58666229248047,
      "activations/layer5_attention_weight_max": 118.5780029296875,
      "activations/layer5_attention_weight_min": -103.13728332519531,
      "activations/layer6_attention_weight_max": 62.32719421386719,
      "activations/layer6_attention_weight_min": -55.81927490234375,
      "activations/layer7_attention_weight_max": 67.98466491699219,
      "activations/layer7_attention_weight_min": -69.81938171386719,
      "activations/layer8_attention_weight_max": 53.347023010253906,
      "activations/layer8_attention_weight_min": -44.14714431762695,
      "activations/layer9_attention_weight_max": 79.70335388183594,
      "activations/layer9_attention_weight_min": -62.272029876708984,
      "epoch": 2.22,
      "learning_rate": 0.00013704507575757575,
      "loss": 2.9483,
      "step": 38250
    },
    {
      "activations/layer0_attention_weight_max": 17.079710006713867,
      "activations/layer0_attention_weight_min": -11.819218635559082,
      "activations/layer10_attention_weight_max": 55.05796813964844,
      "activations/layer10_attention_weight_min": -44.047393798828125,
      "activations/layer11_attention_weight_max": 38.647891998291016,
      "activations/layer11_attention_weight_min": -33.45613098144531,
      "activations/layer12_attention_weight_max": 43.00946044921875,
      "activations/layer12_attention_weight_min": -30.991853713989258,
      "activations/layer13_attention_weight_max": 52.35658645629883,
      "activations/layer13_attention_weight_min": -38.67356872558594,
      "activations/layer14_attention_weight_max": 87.44479370117188,
      "activations/layer14_attention_weight_min": -58.87155532836914,
      "activations/layer15_attention_weight_max": 53.931453704833984,
      "activations/layer15_attention_weight_min": -38.760677337646484,
      "activations/layer16_attention_weight_max": 50.90126037597656,
      "activations/layer16_attention_weight_min": -34.72055435180664,
      "activations/layer17_attention_weight_max": 45.72389602661133,
      "activations/layer17_attention_weight_min": -30.3634033203125,
      "activations/layer18_attention_weight_max": 29.04293441772461,
      "activations/layer18_attention_weight_min": -17.884254455566406,
      "activations/layer19_attention_weight_max": 53.501068115234375,
      "activations/layer19_attention_weight_min": -31.413461685180664,
      "activations/layer1_attention_weight_max": 18.07748794555664,
      "activations/layer1_attention_weight_min": -11.383234977722168,
      "activations/layer20_attention_weight_max": 40.87839889526367,
      "activations/layer20_attention_weight_min": -23.548471450805664,
      "activations/layer21_attention_weight_max": 32.63772201538086,
      "activations/layer21_attention_weight_min": -20.76995849609375,
      "activations/layer22_attention_weight_max": 46.889549255371094,
      "activations/layer22_attention_weight_min": -26.59490394592285,
      "activations/layer23_attention_weight_max": 23.828250885009766,
      "activations/layer23_attention_weight_min": -16.122272491455078,
      "activations/layer2_attention_weight_max": 29.634536743164062,
      "activations/layer2_attention_weight_min": -28.625341415405273,
      "activations/layer3_attention_weight_max": 73.76177215576172,
      "activations/layer3_attention_weight_min": -75.453857421875,
      "activations/layer4_attention_weight_max": 126.72962951660156,
      "activations/layer4_attention_weight_min": -117.7908706665039,
      "activations/layer5_attention_weight_max": 108.13548278808594,
      "activations/layer5_attention_weight_min": -95.11095428466797,
      "activations/layer6_attention_weight_max": 59.03336715698242,
      "activations/layer6_attention_weight_min": -54.25619125366211,
      "activations/layer7_attention_weight_max": 66.7109146118164,
      "activations/layer7_attention_weight_min": -62.34003829956055,
      "activations/layer8_attention_weight_max": 47.5660514831543,
      "activations/layer8_attention_weight_min": -39.72993850708008,
      "activations/layer9_attention_weight_max": 57.13653564453125,
      "activations/layer9_attention_weight_min": -46.671993255615234,
      "epoch": 2.23,
      "learning_rate": 0.00013702613636363636,
      "loss": 2.9489,
      "step": 38300
    },
    {
      "activations/layer0_attention_weight_max": 16.711318969726562,
      "activations/layer0_attention_weight_min": -12.318212509155273,
      "activations/layer10_attention_weight_max": 47.571075439453125,
      "activations/layer10_attention_weight_min": -37.754356384277344,
      "activations/layer11_attention_weight_max": 37.191532135009766,
      "activations/layer11_attention_weight_min": -32.03584671020508,
      "activations/layer12_attention_weight_max": 34.86989974975586,
      "activations/layer12_attention_weight_min": -25.98581314086914,
      "activations/layer13_attention_weight_max": 44.57970428466797,
      "activations/layer13_attention_weight_min": -35.14690399169922,
      "activations/layer14_attention_weight_max": 71.54498291015625,
      "activations/layer14_attention_weight_min": -53.998695373535156,
      "activations/layer15_attention_weight_max": 50.917911529541016,
      "activations/layer15_attention_weight_min": -37.22099304199219,
      "activations/layer16_attention_weight_max": 45.72831344604492,
      "activations/layer16_attention_weight_min": -36.85514450073242,
      "activations/layer17_attention_weight_max": 40.436973571777344,
      "activations/layer17_attention_weight_min": -27.125864028930664,
      "activations/layer18_attention_weight_max": 26.36526870727539,
      "activations/layer18_attention_weight_min": -19.940481185913086,
      "activations/layer19_attention_weight_max": 49.76227951049805,
      "activations/layer19_attention_weight_min": -34.912109375,
      "activations/layer1_attention_weight_max": 17.161602020263672,
      "activations/layer1_attention_weight_min": -10.870522499084473,
      "activations/layer20_attention_weight_max": 38.10645294189453,
      "activations/layer20_attention_weight_min": -24.470102310180664,
      "activations/layer21_attention_weight_max": 26.731853485107422,
      "activations/layer21_attention_weight_min": -20.4233455657959,
      "activations/layer22_attention_weight_max": 39.65110778808594,
      "activations/layer22_attention_weight_min": -27.644630432128906,
      "activations/layer23_attention_weight_max": 25.025312423706055,
      "activations/layer23_attention_weight_min": -18.220802307128906,
      "activations/layer2_attention_weight_max": 30.455589294433594,
      "activations/layer2_attention_weight_min": -30.426816940307617,
      "activations/layer3_attention_weight_max": 76.18107604980469,
      "activations/layer3_attention_weight_min": -76.47481536865234,
      "activations/layer4_attention_weight_max": 125.65362548828125,
      "activations/layer4_attention_weight_min": -121.70121002197266,
      "activations/layer5_attention_weight_max": 110.43244934082031,
      "activations/layer5_attention_weight_min": -99.64842224121094,
      "activations/layer6_attention_weight_max": 58.1390380859375,
      "activations/layer6_attention_weight_min": -52.40103530883789,
      "activations/layer7_attention_weight_max": 71.71988677978516,
      "activations/layer7_attention_weight_min": -67.73613739013672,
      "activations/layer8_attention_weight_max": 46.04331970214844,
      "activations/layer8_attention_weight_min": -37.505126953125,
      "activations/layer9_attention_weight_max": 71.51055908203125,
      "activations/layer9_attention_weight_min": -56.07119369506836,
      "epoch": 2.23,
      "learning_rate": 0.00013700719696969695,
      "loss": 2.9314,
      "step": 38350
    },
    {
      "activations/layer0_attention_weight_max": 16.092893600463867,
      "activations/layer0_attention_weight_min": -12.043885231018066,
      "activations/layer10_attention_weight_max": 48.11029052734375,
      "activations/layer10_attention_weight_min": -41.2216682434082,
      "activations/layer11_attention_weight_max": 39.987709045410156,
      "activations/layer11_attention_weight_min": -37.907501220703125,
      "activations/layer12_attention_weight_max": 37.4898796081543,
      "activations/layer12_attention_weight_min": -26.7303409576416,
      "activations/layer13_attention_weight_max": 47.144161224365234,
      "activations/layer13_attention_weight_min": -36.9108772277832,
      "activations/layer14_attention_weight_max": 76.15398406982422,
      "activations/layer14_attention_weight_min": -56.16706848144531,
      "activations/layer15_attention_weight_max": 55.958675384521484,
      "activations/layer15_attention_weight_min": -37.86625289916992,
      "activations/layer16_attention_weight_max": 52.56342315673828,
      "activations/layer16_attention_weight_min": -36.56410217285156,
      "activations/layer17_attention_weight_max": 45.37835693359375,
      "activations/layer17_attention_weight_min": -31.09724998474121,
      "activations/layer18_attention_weight_max": 28.046546936035156,
      "activations/layer18_attention_weight_min": -20.80071449279785,
      "activations/layer19_attention_weight_max": 53.69172286987305,
      "activations/layer19_attention_weight_min": -31.7227725982666,
      "activations/layer1_attention_weight_max": 19.17340850830078,
      "activations/layer1_attention_weight_min": -10.924281120300293,
      "activations/layer20_attention_weight_max": 34.503395080566406,
      "activations/layer20_attention_weight_min": -23.941240310668945,
      "activations/layer21_attention_weight_max": 38.617984771728516,
      "activations/layer21_attention_weight_min": -23.438501358032227,
      "activations/layer22_attention_weight_max": 46.893394470214844,
      "activations/layer22_attention_weight_min": -29.16096305847168,
      "activations/layer23_attention_weight_max": 24.09026336669922,
      "activations/layer23_attention_weight_min": -17.420154571533203,
      "activations/layer2_attention_weight_max": 29.127079010009766,
      "activations/layer2_attention_weight_min": -29.144271850585938,
      "activations/layer3_attention_weight_max": 71.8189468383789,
      "activations/layer3_attention_weight_min": -75.32154083251953,
      "activations/layer4_attention_weight_max": 125.9581298828125,
      "activations/layer4_attention_weight_min": -115.7896728515625,
      "activations/layer5_attention_weight_max": 109.71914672851562,
      "activations/layer5_attention_weight_min": -100.50222778320312,
      "activations/layer6_attention_weight_max": 58.897125244140625,
      "activations/layer6_attention_weight_min": -54.83940505981445,
      "activations/layer7_attention_weight_max": 64.14010620117188,
      "activations/layer7_attention_weight_min": -65.98029327392578,
      "activations/layer8_attention_weight_max": 46.27529525756836,
      "activations/layer8_attention_weight_min": -38.6810417175293,
      "activations/layer9_attention_weight_max": 62.810150146484375,
      "activations/layer9_attention_weight_min": -55.190853118896484,
      "epoch": 2.23,
      "learning_rate": 0.00013698825757575757,
      "loss": 2.9597,
      "step": 38400
    },
    {
      "activations/layer0_attention_weight_max": 16.76447296142578,
      "activations/layer0_attention_weight_min": -12.02271842956543,
      "activations/layer10_attention_weight_max": 49.922821044921875,
      "activations/layer10_attention_weight_min": -39.06598663330078,
      "activations/layer11_attention_weight_max": 37.74395751953125,
      "activations/layer11_attention_weight_min": -31.07784080505371,
      "activations/layer12_attention_weight_max": 31.064714431762695,
      "activations/layer12_attention_weight_min": -28.963150024414062,
      "activations/layer13_attention_weight_max": 48.28653335571289,
      "activations/layer13_attention_weight_min": -33.53159713745117,
      "activations/layer14_attention_weight_max": 76.6600112915039,
      "activations/layer14_attention_weight_min": -56.924312591552734,
      "activations/layer15_attention_weight_max": 49.14288330078125,
      "activations/layer15_attention_weight_min": -35.76274490356445,
      "activations/layer16_attention_weight_max": 44.363250732421875,
      "activations/layer16_attention_weight_min": -34.44319534301758,
      "activations/layer17_attention_weight_max": 46.141815185546875,
      "activations/layer17_attention_weight_min": -29.599573135375977,
      "activations/layer18_attention_weight_max": 29.549894332885742,
      "activations/layer18_attention_weight_min": -18.791847229003906,
      "activations/layer19_attention_weight_max": 45.36521530151367,
      "activations/layer19_attention_weight_min": -30.025476455688477,
      "activations/layer1_attention_weight_max": 19.03915023803711,
      "activations/layer1_attention_weight_min": -12.396364212036133,
      "activations/layer20_attention_weight_max": 35.039207458496094,
      "activations/layer20_attention_weight_min": -24.253948211669922,
      "activations/layer21_attention_weight_max": 30.058883666992188,
      "activations/layer21_attention_weight_min": -21.266782760620117,
      "activations/layer22_attention_weight_max": 45.43022918701172,
      "activations/layer22_attention_weight_min": -28.65593910217285,
      "activations/layer23_attention_weight_max": 26.242355346679688,
      "activations/layer23_attention_weight_min": -18.785921096801758,
      "activations/layer2_attention_weight_max": 30.343727111816406,
      "activations/layer2_attention_weight_min": -29.283645629882812,
      "activations/layer3_attention_weight_max": 77.57511901855469,
      "activations/layer3_attention_weight_min": -75.98513793945312,
      "activations/layer4_attention_weight_max": 131.2310791015625,
      "activations/layer4_attention_weight_min": -120.17012786865234,
      "activations/layer5_attention_weight_max": 113.84764099121094,
      "activations/layer5_attention_weight_min": -97.08340454101562,
      "activations/layer6_attention_weight_max": 61.46866989135742,
      "activations/layer6_attention_weight_min": -54.04585266113281,
      "activations/layer7_attention_weight_max": 67.7048110961914,
      "activations/layer7_attention_weight_min": -63.766944885253906,
      "activations/layer8_attention_weight_max": 44.21480941772461,
      "activations/layer8_attention_weight_min": -35.5484619140625,
      "activations/layer9_attention_weight_max": 61.59025955200195,
      "activations/layer9_attention_weight_min": -53.34369659423828,
      "epoch": 2.23,
      "learning_rate": 0.00013696931818181818,
      "loss": 2.9438,
      "step": 38450
    },
    {
      "activations/layer0_attention_weight_max": 15.834293365478516,
      "activations/layer0_attention_weight_min": -11.587150573730469,
      "activations/layer10_attention_weight_max": 49.20485305786133,
      "activations/layer10_attention_weight_min": -39.959957122802734,
      "activations/layer11_attention_weight_max": 43.51548385620117,
      "activations/layer11_attention_weight_min": -34.974891662597656,
      "activations/layer12_attention_weight_max": 44.456390380859375,
      "activations/layer12_attention_weight_min": -29.652034759521484,
      "activations/layer13_attention_weight_max": 58.7110710144043,
      "activations/layer13_attention_weight_min": -41.233699798583984,
      "activations/layer14_attention_weight_max": 81.77203369140625,
      "activations/layer14_attention_weight_min": -61.98921203613281,
      "activations/layer15_attention_weight_max": 52.68227767944336,
      "activations/layer15_attention_weight_min": -37.88367462158203,
      "activations/layer16_attention_weight_max": 51.55552673339844,
      "activations/layer16_attention_weight_min": -36.36784362792969,
      "activations/layer17_attention_weight_max": 52.394203186035156,
      "activations/layer17_attention_weight_min": -29.47834014892578,
      "activations/layer18_attention_weight_max": 32.289276123046875,
      "activations/layer18_attention_weight_min": -17.22382354736328,
      "activations/layer19_attention_weight_max": 57.696712493896484,
      "activations/layer19_attention_weight_min": -30.86948585510254,
      "activations/layer1_attention_weight_max": 18.449474334716797,
      "activations/layer1_attention_weight_min": -14.140851974487305,
      "activations/layer20_attention_weight_max": 45.71554946899414,
      "activations/layer20_attention_weight_min": -23.746122360229492,
      "activations/layer21_attention_weight_max": 34.0749397277832,
      "activations/layer21_attention_weight_min": -19.760095596313477,
      "activations/layer22_attention_weight_max": 49.08380126953125,
      "activations/layer22_attention_weight_min": -25.02580451965332,
      "activations/layer23_attention_weight_max": 25.554685592651367,
      "activations/layer23_attention_weight_min": -17.17529296875,
      "activations/layer2_attention_weight_max": 27.81476593017578,
      "activations/layer2_attention_weight_min": -26.749664306640625,
      "activations/layer3_attention_weight_max": 70.646728515625,
      "activations/layer3_attention_weight_min": -75.38652801513672,
      "activations/layer4_attention_weight_max": 129.66244506835938,
      "activations/layer4_attention_weight_min": -121.38225555419922,
      "activations/layer5_attention_weight_max": 114.90142822265625,
      "activations/layer5_attention_weight_min": -97.05686950683594,
      "activations/layer6_attention_weight_max": 59.65550994873047,
      "activations/layer6_attention_weight_min": -54.84253692626953,
      "activations/layer7_attention_weight_max": 67.99710083007812,
      "activations/layer7_attention_weight_min": -65.39431762695312,
      "activations/layer8_attention_weight_max": 50.110294342041016,
      "activations/layer8_attention_weight_min": -41.718994140625,
      "activations/layer9_attention_weight_max": 62.87168502807617,
      "activations/layer9_attention_weight_min": -55.46311569213867,
      "epoch": 2.24,
      "learning_rate": 0.00013695037878787877,
      "loss": 2.9417,
      "step": 38500
    },
    {
      "activations/layer0_attention_weight_max": 16.819013595581055,
      "activations/layer0_attention_weight_min": -12.326380729675293,
      "activations/layer10_attention_weight_max": 48.68352127075195,
      "activations/layer10_attention_weight_min": -39.91576385498047,
      "activations/layer11_attention_weight_max": 51.73106002807617,
      "activations/layer11_attention_weight_min": -36.32122802734375,
      "activations/layer12_attention_weight_max": 44.302711486816406,
      "activations/layer12_attention_weight_min": -29.546266555786133,
      "activations/layer13_attention_weight_max": 45.43934631347656,
      "activations/layer13_attention_weight_min": -33.98413848876953,
      "activations/layer14_attention_weight_max": 81.07389831542969,
      "activations/layer14_attention_weight_min": -64.28414916992188,
      "activations/layer15_attention_weight_max": 55.31179428100586,
      "activations/layer15_attention_weight_min": -39.84446716308594,
      "activations/layer16_attention_weight_max": 49.519989013671875,
      "activations/layer16_attention_weight_min": -35.908851623535156,
      "activations/layer17_attention_weight_max": 41.123233795166016,
      "activations/layer17_attention_weight_min": -27.917945861816406,
      "activations/layer18_attention_weight_max": 25.564958572387695,
      "activations/layer18_attention_weight_min": -17.620256423950195,
      "activations/layer19_attention_weight_max": 47.5951042175293,
      "activations/layer19_attention_weight_min": -29.67336082458496,
      "activations/layer1_attention_weight_max": 20.2774715423584,
      "activations/layer1_attention_weight_min": -11.373758316040039,
      "activations/layer20_attention_weight_max": 38.867759704589844,
      "activations/layer20_attention_weight_min": -24.239248275756836,
      "activations/layer21_attention_weight_max": 29.197263717651367,
      "activations/layer21_attention_weight_min": -21.37946128845215,
      "activations/layer22_attention_weight_max": 45.69580078125,
      "activations/layer22_attention_weight_min": -24.417865753173828,
      "activations/layer23_attention_weight_max": 27.39792251586914,
      "activations/layer23_attention_weight_min": -16.1329345703125,
      "activations/layer2_attention_weight_max": 27.402212142944336,
      "activations/layer2_attention_weight_min": -28.480192184448242,
      "activations/layer3_attention_weight_max": 69.51669311523438,
      "activations/layer3_attention_weight_min": -74.2551498413086,
      "activations/layer4_attention_weight_max": 120.52062225341797,
      "activations/layer4_attention_weight_min": -124.08036041259766,
      "activations/layer5_attention_weight_max": 103.48287963867188,
      "activations/layer5_attention_weight_min": -106.41793060302734,
      "activations/layer6_attention_weight_max": 57.24152755737305,
      "activations/layer6_attention_weight_min": -50.75096130371094,
      "activations/layer7_attention_weight_max": 76.07441711425781,
      "activations/layer7_attention_weight_min": -65.73977661132812,
      "activations/layer8_attention_weight_max": 48.46316146850586,
      "activations/layer8_attention_weight_min": -42.55567932128906,
      "activations/layer9_attention_weight_max": 93.97074127197266,
      "activations/layer9_attention_weight_min": -71.86226654052734,
      "epoch": 2.24,
      "learning_rate": 0.00013693143939393939,
      "loss": 2.9513,
      "step": 38550
    },
    {
      "activations/layer0_attention_weight_max": 15.812527656555176,
      "activations/layer0_attention_weight_min": -11.70145034790039,
      "activations/layer10_attention_weight_max": 56.369285583496094,
      "activations/layer10_attention_weight_min": -44.516204833984375,
      "activations/layer11_attention_weight_max": 41.267539978027344,
      "activations/layer11_attention_weight_min": -36.26282501220703,
      "activations/layer12_attention_weight_max": 48.871334075927734,
      "activations/layer12_attention_weight_min": -32.86771011352539,
      "activations/layer13_attention_weight_max": 47.49097442626953,
      "activations/layer13_attention_weight_min": -37.69319152832031,
      "activations/layer14_attention_weight_max": 74.65643310546875,
      "activations/layer14_attention_weight_min": -59.65007781982422,
      "activations/layer15_attention_weight_max": 51.336055755615234,
      "activations/layer15_attention_weight_min": -36.641998291015625,
      "activations/layer16_attention_weight_max": 52.90087890625,
      "activations/layer16_attention_weight_min": -35.02647399902344,
      "activations/layer17_attention_weight_max": 40.203895568847656,
      "activations/layer17_attention_weight_min": -28.54097557067871,
      "activations/layer18_attention_weight_max": 26.34090805053711,
      "activations/layer18_attention_weight_min": -19.403980255126953,
      "activations/layer19_attention_weight_max": 54.43828201293945,
      "activations/layer19_attention_weight_min": -31.801502227783203,
      "activations/layer1_attention_weight_max": 18.03059196472168,
      "activations/layer1_attention_weight_min": -10.357678413391113,
      "activations/layer20_attention_weight_max": 36.94312286376953,
      "activations/layer20_attention_weight_min": -24.64711570739746,
      "activations/layer21_attention_weight_max": 30.723827362060547,
      "activations/layer21_attention_weight_min": -19.531356811523438,
      "activations/layer22_attention_weight_max": 46.85667037963867,
      "activations/layer22_attention_weight_min": -27.74274444580078,
      "activations/layer23_attention_weight_max": 26.498517990112305,
      "activations/layer23_attention_weight_min": -15.529437065124512,
      "activations/layer2_attention_weight_max": 28.43933868408203,
      "activations/layer2_attention_weight_min": -28.499967575073242,
      "activations/layer3_attention_weight_max": 71.45113372802734,
      "activations/layer3_attention_weight_min": -73.61136627197266,
      "activations/layer4_attention_weight_max": 131.03416442871094,
      "activations/layer4_attention_weight_min": -125.09844970703125,
      "activations/layer5_attention_weight_max": 114.91166687011719,
      "activations/layer5_attention_weight_min": -94.07695007324219,
      "activations/layer6_attention_weight_max": 60.74980926513672,
      "activations/layer6_attention_weight_min": -51.427730560302734,
      "activations/layer7_attention_weight_max": 67.08361053466797,
      "activations/layer7_attention_weight_min": -66.94978332519531,
      "activations/layer8_attention_weight_max": 43.89793014526367,
      "activations/layer8_attention_weight_min": -38.58980178833008,
      "activations/layer9_attention_weight_max": 67.24696350097656,
      "activations/layer9_attention_weight_min": -61.067317962646484,
      "epoch": 2.24,
      "learning_rate": 0.00013691249999999997,
      "loss": 2.954,
      "step": 38600
    },
    {
      "activations/layer0_attention_weight_max": 16.498811721801758,
      "activations/layer0_attention_weight_min": -12.486114501953125,
      "activations/layer10_attention_weight_max": 40.79878616333008,
      "activations/layer10_attention_weight_min": -39.3303108215332,
      "activations/layer11_attention_weight_max": 33.63860321044922,
      "activations/layer11_attention_weight_min": -33.08595275878906,
      "activations/layer12_attention_weight_max": 35.02899932861328,
      "activations/layer12_attention_weight_min": -39.17716598510742,
      "activations/layer13_attention_weight_max": 48.19425964355469,
      "activations/layer13_attention_weight_min": -35.39238739013672,
      "activations/layer14_attention_weight_max": 66.44112396240234,
      "activations/layer14_attention_weight_min": -56.59682846069336,
      "activations/layer15_attention_weight_max": 45.040382385253906,
      "activations/layer15_attention_weight_min": -37.010154724121094,
      "activations/layer16_attention_weight_max": 48.418338775634766,
      "activations/layer16_attention_weight_min": -35.25339889526367,
      "activations/layer17_attention_weight_max": 41.42433547973633,
      "activations/layer17_attention_weight_min": -28.93474006652832,
      "activations/layer18_attention_weight_max": 27.730045318603516,
      "activations/layer18_attention_weight_min": -18.34993553161621,
      "activations/layer19_attention_weight_max": 53.34858322143555,
      "activations/layer19_attention_weight_min": -34.02280807495117,
      "activations/layer1_attention_weight_max": 18.157371520996094,
      "activations/layer1_attention_weight_min": -12.0465726852417,
      "activations/layer20_attention_weight_max": 37.71223068237305,
      "activations/layer20_attention_weight_min": -23.927080154418945,
      "activations/layer21_attention_weight_max": 31.501371383666992,
      "activations/layer21_attention_weight_min": -21.720794677734375,
      "activations/layer22_attention_weight_max": 48.30390930175781,
      "activations/layer22_attention_weight_min": -27.609699249267578,
      "activations/layer23_attention_weight_max": 24.800186157226562,
      "activations/layer23_attention_weight_min": -18.332847595214844,
      "activations/layer2_attention_weight_max": 28.150035858154297,
      "activations/layer2_attention_weight_min": -27.694629669189453,
      "activations/layer3_attention_weight_max": 70.53504943847656,
      "activations/layer3_attention_weight_min": -73.12236022949219,
      "activations/layer4_attention_weight_max": 121.47924041748047,
      "activations/layer4_attention_weight_min": -114.81829833984375,
      "activations/layer5_attention_weight_max": 113.1605453491211,
      "activations/layer5_attention_weight_min": -94.71639251708984,
      "activations/layer6_attention_weight_max": 61.90834426879883,
      "activations/layer6_attention_weight_min": -48.387630462646484,
      "activations/layer7_attention_weight_max": 78.26255798339844,
      "activations/layer7_attention_weight_min": -71.07128143310547,
      "activations/layer8_attention_weight_max": 43.972259521484375,
      "activations/layer8_attention_weight_min": -38.287654876708984,
      "activations/layer9_attention_weight_max": 68.04631805419922,
      "activations/layer9_attention_weight_min": -68.74539184570312,
      "epoch": 2.25,
      "learning_rate": 0.0001368935606060606,
      "loss": 2.9215,
      "step": 38650
    },
    {
      "activations/layer0_attention_weight_max": 16.413084030151367,
      "activations/layer0_attention_weight_min": -12.160126686096191,
      "activations/layer10_attention_weight_max": 50.67405319213867,
      "activations/layer10_attention_weight_min": -41.1020393371582,
      "activations/layer11_attention_weight_max": 42.268592834472656,
      "activations/layer11_attention_weight_min": -34.8077507019043,
      "activations/layer12_attention_weight_max": 38.75508499145508,
      "activations/layer12_attention_weight_min": -26.3681583404541,
      "activations/layer13_attention_weight_max": 55.82719039916992,
      "activations/layer13_attention_weight_min": -43.9930305480957,
      "activations/layer14_attention_weight_max": 85.39534759521484,
      "activations/layer14_attention_weight_min": -60.586788177490234,
      "activations/layer15_attention_weight_max": 53.35247802734375,
      "activations/layer15_attention_weight_min": -38.115718841552734,
      "activations/layer16_attention_weight_max": 49.6777458190918,
      "activations/layer16_attention_weight_min": -35.679256439208984,
      "activations/layer17_attention_weight_max": 44.65679168701172,
      "activations/layer17_attention_weight_min": -31.198923110961914,
      "activations/layer18_attention_weight_max": 28.588478088378906,
      "activations/layer18_attention_weight_min": -18.459577560424805,
      "activations/layer19_attention_weight_max": 50.31794357299805,
      "activations/layer19_attention_weight_min": -37.5871467590332,
      "activations/layer1_attention_weight_max": 17.632259368896484,
      "activations/layer1_attention_weight_min": -10.34199333190918,
      "activations/layer20_attention_weight_max": 43.77326965332031,
      "activations/layer20_attention_weight_min": -29.42327117919922,
      "activations/layer21_attention_weight_max": 36.139774322509766,
      "activations/layer21_attention_weight_min": -24.014036178588867,
      "activations/layer22_attention_weight_max": 44.2180290222168,
      "activations/layer22_attention_weight_min": -26.764102935791016,
      "activations/layer23_attention_weight_max": 26.157550811767578,
      "activations/layer23_attention_weight_min": -15.43100643157959,
      "activations/layer2_attention_weight_max": 28.684349060058594,
      "activations/layer2_attention_weight_min": -28.286367416381836,
      "activations/layer3_attention_weight_max": 71.88207244873047,
      "activations/layer3_attention_weight_min": -79.97817993164062,
      "activations/layer4_attention_weight_max": 127.47613525390625,
      "activations/layer4_attention_weight_min": -126.09209442138672,
      "activations/layer5_attention_weight_max": 108.02578735351562,
      "activations/layer5_attention_weight_min": -98.03201293945312,
      "activations/layer6_attention_weight_max": 59.86553192138672,
      "activations/layer6_attention_weight_min": -53.590755462646484,
      "activations/layer7_attention_weight_max": 68.21070861816406,
      "activations/layer7_attention_weight_min": -66.0982894897461,
      "activations/layer8_attention_weight_max": 47.153053283691406,
      "activations/layer8_attention_weight_min": -40.612857818603516,
      "activations/layer9_attention_weight_max": 70.06338500976562,
      "activations/layer9_attention_weight_min": -52.56013870239258,
      "epoch": 2.25,
      "learning_rate": 0.0001368746212121212,
      "loss": 2.955,
      "step": 38700
    },
    {
      "activations/layer0_attention_weight_max": 14.055862426757812,
      "activations/layer0_attention_weight_min": -11.5161771774292,
      "activations/layer10_attention_weight_max": 50.42029571533203,
      "activations/layer10_attention_weight_min": -41.29938888549805,
      "activations/layer11_attention_weight_max": 47.67747116088867,
      "activations/layer11_attention_weight_min": -36.78461456298828,
      "activations/layer12_attention_weight_max": 41.757171630859375,
      "activations/layer12_attention_weight_min": -30.430662155151367,
      "activations/layer13_attention_weight_max": 49.55402374267578,
      "activations/layer13_attention_weight_min": -36.93272018432617,
      "activations/layer14_attention_weight_max": 84.11305236816406,
      "activations/layer14_attention_weight_min": -67.9452133178711,
      "activations/layer15_attention_weight_max": 51.89188766479492,
      "activations/layer15_attention_weight_min": -40.366580963134766,
      "activations/layer16_attention_weight_max": 54.932430267333984,
      "activations/layer16_attention_weight_min": -35.123348236083984,
      "activations/layer17_attention_weight_max": 44.617828369140625,
      "activations/layer17_attention_weight_min": -27.59007453918457,
      "activations/layer18_attention_weight_max": 27.43020248413086,
      "activations/layer18_attention_weight_min": -17.38089370727539,
      "activations/layer19_attention_weight_max": 53.66810989379883,
      "activations/layer19_attention_weight_min": -30.394229888916016,
      "activations/layer1_attention_weight_max": 17.81351089477539,
      "activations/layer1_attention_weight_min": -9.966629028320312,
      "activations/layer20_attention_weight_max": 39.09144973754883,
      "activations/layer20_attention_weight_min": -22.980344772338867,
      "activations/layer21_attention_weight_max": 36.15789031982422,
      "activations/layer21_attention_weight_min": -20.1911563873291,
      "activations/layer22_attention_weight_max": 43.86932373046875,
      "activations/layer22_attention_weight_min": -26.32335090637207,
      "activations/layer23_attention_weight_max": 25.83699607849121,
      "activations/layer23_attention_weight_min": -16.03966522216797,
      "activations/layer2_attention_weight_max": 31.107154846191406,
      "activations/layer2_attention_weight_min": -30.109424591064453,
      "activations/layer3_attention_weight_max": 75.94208526611328,
      "activations/layer3_attention_weight_min": -78.2260513305664,
      "activations/layer4_attention_weight_max": 126.78607177734375,
      "activations/layer4_attention_weight_min": -118.34966278076172,
      "activations/layer5_attention_weight_max": 110.43594360351562,
      "activations/layer5_attention_weight_min": -98.76985168457031,
      "activations/layer6_attention_weight_max": 60.699668884277344,
      "activations/layer6_attention_weight_min": -53.87666702270508,
      "activations/layer7_attention_weight_max": 69.08348083496094,
      "activations/layer7_attention_weight_min": -69.92606353759766,
      "activations/layer8_attention_weight_max": 50.67494583129883,
      "activations/layer8_attention_weight_min": -42.453670501708984,
      "activations/layer9_attention_weight_max": 74.6194076538086,
      "activations/layer9_attention_weight_min": -73.37773895263672,
      "epoch": 2.25,
      "learning_rate": 0.0001368556818181818,
      "loss": 2.9337,
      "step": 38750
    },
    {
      "activations/layer0_attention_weight_max": 16.58419418334961,
      "activations/layer0_attention_weight_min": -11.929060935974121,
      "activations/layer10_attention_weight_max": 44.442832946777344,
      "activations/layer10_attention_weight_min": -41.269901275634766,
      "activations/layer11_attention_weight_max": 34.71967697143555,
      "activations/layer11_attention_weight_min": -32.61745834350586,
      "activations/layer12_attention_weight_max": 31.434680938720703,
      "activations/layer12_attention_weight_min": -26.854372024536133,
      "activations/layer13_attention_weight_max": 53.28695297241211,
      "activations/layer13_attention_weight_min": -47.35226821899414,
      "activations/layer14_attention_weight_max": 71.09085083007812,
      "activations/layer14_attention_weight_min": -56.187862396240234,
      "activations/layer15_attention_weight_max": 50.676841735839844,
      "activations/layer15_attention_weight_min": -35.44318771362305,
      "activations/layer16_attention_weight_max": 48.08591842651367,
      "activations/layer16_attention_weight_min": -37.46194839477539,
      "activations/layer17_attention_weight_max": 44.29667282104492,
      "activations/layer17_attention_weight_min": -27.589075088500977,
      "activations/layer18_attention_weight_max": 29.262874603271484,
      "activations/layer18_attention_weight_min": -19.075571060180664,
      "activations/layer19_attention_weight_max": 54.67116165161133,
      "activations/layer19_attention_weight_min": -30.202436447143555,
      "activations/layer1_attention_weight_max": 18.108932495117188,
      "activations/layer1_attention_weight_min": -12.686643600463867,
      "activations/layer20_attention_weight_max": 38.15760040283203,
      "activations/layer20_attention_weight_min": -22.78982162475586,
      "activations/layer21_attention_weight_max": 29.067123413085938,
      "activations/layer21_attention_weight_min": -18.645381927490234,
      "activations/layer22_attention_weight_max": 44.24105453491211,
      "activations/layer22_attention_weight_min": -26.749698638916016,
      "activations/layer23_attention_weight_max": 26.10647201538086,
      "activations/layer23_attention_weight_min": -15.520545959472656,
      "activations/layer2_attention_weight_max": 28.865440368652344,
      "activations/layer2_attention_weight_min": -29.949115753173828,
      "activations/layer3_attention_weight_max": 72.2396469116211,
      "activations/layer3_attention_weight_min": -76.42426300048828,
      "activations/layer4_attention_weight_max": 124.08635711669922,
      "activations/layer4_attention_weight_min": -119.7270278930664,
      "activations/layer5_attention_weight_max": 108.44173431396484,
      "activations/layer5_attention_weight_min": -95.20085144042969,
      "activations/layer6_attention_weight_max": 61.74912643432617,
      "activations/layer6_attention_weight_min": -50.49321365356445,
      "activations/layer7_attention_weight_max": 64.98507690429688,
      "activations/layer7_attention_weight_min": -66.80326080322266,
      "activations/layer8_attention_weight_max": 45.793949127197266,
      "activations/layer8_attention_weight_min": -36.70199203491211,
      "activations/layer9_attention_weight_max": 61.9188232421875,
      "activations/layer9_attention_weight_min": -49.35099792480469,
      "epoch": 2.25,
      "learning_rate": 0.0001368367424242424,
      "loss": 2.9398,
      "step": 38800
    },
    {
      "activations/layer0_attention_weight_max": 17.41457176208496,
      "activations/layer0_attention_weight_min": -11.897906303405762,
      "activations/layer10_attention_weight_max": 48.454830169677734,
      "activations/layer10_attention_weight_min": -40.77313232421875,
      "activations/layer11_attention_weight_max": 43.004364013671875,
      "activations/layer11_attention_weight_min": -34.8988151550293,
      "activations/layer12_attention_weight_max": 31.042333602905273,
      "activations/layer12_attention_weight_min": -25.40934181213379,
      "activations/layer13_attention_weight_max": 47.32197952270508,
      "activations/layer13_attention_weight_min": -36.48036193847656,
      "activations/layer14_attention_weight_max": 69.90780639648438,
      "activations/layer14_attention_weight_min": -52.822574615478516,
      "activations/layer15_attention_weight_max": 60.32564926147461,
      "activations/layer15_attention_weight_min": -38.834190368652344,
      "activations/layer16_attention_weight_max": 47.573936462402344,
      "activations/layer16_attention_weight_min": -37.50112533569336,
      "activations/layer17_attention_weight_max": 45.62514114379883,
      "activations/layer17_attention_weight_min": -32.22740936279297,
      "activations/layer18_attention_weight_max": 29.803329467773438,
      "activations/layer18_attention_weight_min": -18.635684967041016,
      "activations/layer19_attention_weight_max": 51.42635726928711,
      "activations/layer19_attention_weight_min": -32.17060470581055,
      "activations/layer1_attention_weight_max": 18.249958038330078,
      "activations/layer1_attention_weight_min": -11.089249610900879,
      "activations/layer20_attention_weight_max": 36.029850006103516,
      "activations/layer20_attention_weight_min": -25.43025779724121,
      "activations/layer21_attention_weight_max": 27.535526275634766,
      "activations/layer21_attention_weight_min": -19.547273635864258,
      "activations/layer22_attention_weight_max": 45.1838264465332,
      "activations/layer22_attention_weight_min": -27.873519897460938,
      "activations/layer23_attention_weight_max": 25.88357925415039,
      "activations/layer23_attention_weight_min": -17.184030532836914,
      "activations/layer2_attention_weight_max": 30.834043502807617,
      "activations/layer2_attention_weight_min": -29.33557891845703,
      "activations/layer3_attention_weight_max": 72.40821838378906,
      "activations/layer3_attention_weight_min": -76.2782211303711,
      "activations/layer4_attention_weight_max": 125.00599670410156,
      "activations/layer4_attention_weight_min": -118.67572021484375,
      "activations/layer5_attention_weight_max": 107.38973999023438,
      "activations/layer5_attention_weight_min": -95.18121337890625,
      "activations/layer6_attention_weight_max": 57.26905822753906,
      "activations/layer6_attention_weight_min": -52.825618743896484,
      "activations/layer7_attention_weight_max": 65.66378021240234,
      "activations/layer7_attention_weight_min": -64.4720458984375,
      "activations/layer8_attention_weight_max": 41.66615676879883,
      "activations/layer8_attention_weight_min": -37.27500915527344,
      "activations/layer9_attention_weight_max": 59.702491760253906,
      "activations/layer9_attention_weight_min": -48.15922927856445,
      "epoch": 2.26,
      "learning_rate": 0.00013681780303030302,
      "loss": 2.945,
      "step": 38850
    },
    {
      "activations/layer0_attention_weight_max": 16.62961196899414,
      "activations/layer0_attention_weight_min": -11.680094718933105,
      "activations/layer10_attention_weight_max": 58.2530632019043,
      "activations/layer10_attention_weight_min": -43.2298583984375,
      "activations/layer11_attention_weight_max": 37.62908935546875,
      "activations/layer11_attention_weight_min": -32.559791564941406,
      "activations/layer12_attention_weight_max": 32.60663604736328,
      "activations/layer12_attention_weight_min": -26.17325782775879,
      "activations/layer13_attention_weight_max": 53.60316848754883,
      "activations/layer13_attention_weight_min": -42.0092887878418,
      "activations/layer14_attention_weight_max": 74.76395416259766,
      "activations/layer14_attention_weight_min": -53.91905212402344,
      "activations/layer15_attention_weight_max": 45.82843017578125,
      "activations/layer15_attention_weight_min": -35.643104553222656,
      "activations/layer16_attention_weight_max": 44.471405029296875,
      "activations/layer16_attention_weight_min": -32.861053466796875,
      "activations/layer17_attention_weight_max": 42.381290435791016,
      "activations/layer17_attention_weight_min": -27.598068237304688,
      "activations/layer18_attention_weight_max": 24.473661422729492,
      "activations/layer18_attention_weight_min": -20.185834884643555,
      "activations/layer19_attention_weight_max": 47.504638671875,
      "activations/layer19_attention_weight_min": -31.707767486572266,
      "activations/layer1_attention_weight_max": 17.140443801879883,
      "activations/layer1_attention_weight_min": -11.004380226135254,
      "activations/layer20_attention_weight_max": 37.09160232543945,
      "activations/layer20_attention_weight_min": -24.505735397338867,
      "activations/layer21_attention_weight_max": 29.779422760009766,
      "activations/layer21_attention_weight_min": -20.728525161743164,
      "activations/layer22_attention_weight_max": 42.43891525268555,
      "activations/layer22_attention_weight_min": -28.001691818237305,
      "activations/layer23_attention_weight_max": 24.24789047241211,
      "activations/layer23_attention_weight_min": -16.587289810180664,
      "activations/layer2_attention_weight_max": 30.169286727905273,
      "activations/layer2_attention_weight_min": -29.147937774658203,
      "activations/layer3_attention_weight_max": 72.69645690917969,
      "activations/layer3_attention_weight_min": -74.20062255859375,
      "activations/layer4_attention_weight_max": 125.56910705566406,
      "activations/layer4_attention_weight_min": -115.45946502685547,
      "activations/layer5_attention_weight_max": 110.29647064208984,
      "activations/layer5_attention_weight_min": -94.86866760253906,
      "activations/layer6_attention_weight_max": 59.68898010253906,
      "activations/layer6_attention_weight_min": -53.997806549072266,
      "activations/layer7_attention_weight_max": 72.90013122558594,
      "activations/layer7_attention_weight_min": -64.86933135986328,
      "activations/layer8_attention_weight_max": 45.84058380126953,
      "activations/layer8_attention_weight_min": -39.54668045043945,
      "activations/layer9_attention_weight_max": 75.7741470336914,
      "activations/layer9_attention_weight_min": -47.22583770751953,
      "epoch": 2.26,
      "learning_rate": 0.0001367988636363636,
      "loss": 2.9336,
      "step": 38900
    },
    {
      "activations/layer0_attention_weight_max": 16.46710777282715,
      "activations/layer0_attention_weight_min": -12.228547096252441,
      "activations/layer10_attention_weight_max": 48.00529861450195,
      "activations/layer10_attention_weight_min": -43.38124465942383,
      "activations/layer11_attention_weight_max": 35.96998977661133,
      "activations/layer11_attention_weight_min": -32.610111236572266,
      "activations/layer12_attention_weight_max": 31.607601165771484,
      "activations/layer12_attention_weight_min": -25.32563591003418,
      "activations/layer13_attention_weight_max": 48.95502853393555,
      "activations/layer13_attention_weight_min": -38.01255798339844,
      "activations/layer14_attention_weight_max": 68.4636001586914,
      "activations/layer14_attention_weight_min": -56.7076416015625,
      "activations/layer15_attention_weight_max": 49.80307388305664,
      "activations/layer15_attention_weight_min": -38.481563568115234,
      "activations/layer16_attention_weight_max": 47.28378677368164,
      "activations/layer16_attention_weight_min": -32.76416015625,
      "activations/layer17_attention_weight_max": 43.21528244018555,
      "activations/layer17_attention_weight_min": -28.09587860107422,
      "activations/layer18_attention_weight_max": 26.799028396606445,
      "activations/layer18_attention_weight_min": -18.518571853637695,
      "activations/layer19_attention_weight_max": 54.55105209350586,
      "activations/layer19_attention_weight_min": -30.796112060546875,
      "activations/layer1_attention_weight_max": 17.456979751586914,
      "activations/layer1_attention_weight_min": -13.020240783691406,
      "activations/layer20_attention_weight_max": 35.602474212646484,
      "activations/layer20_attention_weight_min": -24.699438095092773,
      "activations/layer21_attention_weight_max": 33.3152961730957,
      "activations/layer21_attention_weight_min": -20.915740966796875,
      "activations/layer22_attention_weight_max": 43.31451416015625,
      "activations/layer22_attention_weight_min": -27.57564353942871,
      "activations/layer23_attention_weight_max": 25.546689987182617,
      "activations/layer23_attention_weight_min": -19.103317260742188,
      "activations/layer2_attention_weight_max": 30.314481735229492,
      "activations/layer2_attention_weight_min": -29.903141021728516,
      "activations/layer3_attention_weight_max": 72.10774993896484,
      "activations/layer3_attention_weight_min": -79.14442443847656,
      "activations/layer4_attention_weight_max": 122.48353576660156,
      "activations/layer4_attention_weight_min": -120.49159240722656,
      "activations/layer5_attention_weight_max": 109.88534545898438,
      "activations/layer5_attention_weight_min": -97.74916076660156,
      "activations/layer6_attention_weight_max": 58.794105529785156,
      "activations/layer6_attention_weight_min": -52.28304672241211,
      "activations/layer7_attention_weight_max": 66.23358154296875,
      "activations/layer7_attention_weight_min": -65.06059265136719,
      "activations/layer8_attention_weight_max": 43.914344787597656,
      "activations/layer8_attention_weight_min": -38.44182586669922,
      "activations/layer9_attention_weight_max": 58.64144515991211,
      "activations/layer9_attention_weight_min": -49.132808685302734,
      "epoch": 2.26,
      "learning_rate": 0.00013677992424242422,
      "loss": 2.9361,
      "step": 38950
    },
    {
      "activations/layer0_attention_weight_max": 17.146528244018555,
      "activations/layer0_attention_weight_min": -11.819643020629883,
      "activations/layer10_attention_weight_max": 47.590084075927734,
      "activations/layer10_attention_weight_min": -43.00483322143555,
      "activations/layer11_attention_weight_max": 31.67927360534668,
      "activations/layer11_attention_weight_min": -32.449310302734375,
      "activations/layer12_attention_weight_max": 33.487003326416016,
      "activations/layer12_attention_weight_min": -29.952966690063477,
      "activations/layer13_attention_weight_max": 47.74124526977539,
      "activations/layer13_attention_weight_min": -39.12672424316406,
      "activations/layer14_attention_weight_max": 81.37174224853516,
      "activations/layer14_attention_weight_min": -80.79804992675781,
      "activations/layer15_attention_weight_max": 46.25407409667969,
      "activations/layer15_attention_weight_min": -38.670249938964844,
      "activations/layer16_attention_weight_max": 47.15761184692383,
      "activations/layer16_attention_weight_min": -37.56868362426758,
      "activations/layer17_attention_weight_max": 41.434791564941406,
      "activations/layer17_attention_weight_min": -28.12006378173828,
      "activations/layer18_attention_weight_max": 25.9111385345459,
      "activations/layer18_attention_weight_min": -18.482662200927734,
      "activations/layer19_attention_weight_max": 49.693702697753906,
      "activations/layer19_attention_weight_min": -32.080326080322266,
      "activations/layer1_attention_weight_max": 18.300891876220703,
      "activations/layer1_attention_weight_min": -11.220511436462402,
      "activations/layer20_attention_weight_max": 33.666648864746094,
      "activations/layer20_attention_weight_min": -24.034130096435547,
      "activations/layer21_attention_weight_max": 26.35399627685547,
      "activations/layer21_attention_weight_min": -20.721399307250977,
      "activations/layer22_attention_weight_max": 39.583919525146484,
      "activations/layer22_attention_weight_min": -25.453176498413086,
      "activations/layer23_attention_weight_max": 25.59537124633789,
      "activations/layer23_attention_weight_min": -18.757352828979492,
      "activations/layer2_attention_weight_max": 31.190185546875,
      "activations/layer2_attention_weight_min": -31.08440399169922,
      "activations/layer3_attention_weight_max": 72.56877899169922,
      "activations/layer3_attention_weight_min": -76.46467590332031,
      "activations/layer4_attention_weight_max": 122.39391326904297,
      "activations/layer4_attention_weight_min": -123.22087860107422,
      "activations/layer5_attention_weight_max": 104.57366180419922,
      "activations/layer5_attention_weight_min": -97.55400085449219,
      "activations/layer6_attention_weight_max": 56.26698303222656,
      "activations/layer6_attention_weight_min": -54.197811126708984,
      "activations/layer7_attention_weight_max": 59.92851638793945,
      "activations/layer7_attention_weight_min": -64.50227355957031,
      "activations/layer8_attention_weight_max": 40.835044860839844,
      "activations/layer8_attention_weight_min": -39.02358627319336,
      "activations/layer9_attention_weight_max": 64.81444549560547,
      "activations/layer9_attention_weight_min": -53.18949508666992,
      "epoch": 2.27,
      "learning_rate": 0.00013676098484848484,
      "loss": 2.948,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_loss": 2.896484375,
      "eval_runtime": 8.4818,
      "eval_samples_per_second": 506.261,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_openwebtext_loss": 2.896484375,
      "eval_openwebtext_ppl": 18.110364070885183,
      "eval_openwebtext_runtime": 8.4818,
      "eval_openwebtext_samples_per_second": 506.261,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_wikitext_loss": 3.16796875,
      "eval_wikitext_ppl": 23.75917448515314,
      "eval_wikitext_runtime": 2.0211,
      "eval_wikitext_samples_per_second": 225.616,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_lambada_loss": 3.048828125,
      "eval_lambada_ppl": 21.090614371389922,
      "eval_lambada_runtime": 9.552,
      "eval_lambada_samples_per_second": 509.737,
      "step": 39000
    },
    {
      "activations/layer0_attention_weight_max": 16.95435333251953,
      "activations/layer0_attention_weight_min": -11.923171997070312,
      "activations/layer10_attention_weight_max": 53.25802993774414,
      "activations/layer10_attention_weight_min": -38.34172439575195,
      "activations/layer11_attention_weight_max": 42.23298645019531,
      "activations/layer11_attention_weight_min": -34.323463439941406,
      "activations/layer12_attention_weight_max": 36.577880859375,
      "activations/layer12_attention_weight_min": -27.280454635620117,
      "activations/layer13_attention_weight_max": 62.787017822265625,
      "activations/layer13_attention_weight_min": -41.073238372802734,
      "activations/layer14_attention_weight_max": 131.65208435058594,
      "activations/layer14_attention_weight_min": -80.61842346191406,
      "activations/layer15_attention_weight_max": 69.84638977050781,
      "activations/layer15_attention_weight_min": -37.92103576660156,
      "activations/layer16_attention_weight_max": 79.29800415039062,
      "activations/layer16_attention_weight_min": -41.53853988647461,
      "activations/layer17_attention_weight_max": 51.3078727722168,
      "activations/layer17_attention_weight_min": -27.79193115234375,
      "activations/layer18_attention_weight_max": 36.94408416748047,
      "activations/layer18_attention_weight_min": -19.174976348876953,
      "activations/layer19_attention_weight_max": 72.6521224975586,
      "activations/layer19_attention_weight_min": -31.834623336791992,
      "activations/layer1_attention_weight_max": 17.514745712280273,
      "activations/layer1_attention_weight_min": -11.209695816040039,
      "activations/layer20_attention_weight_max": 47.846900939941406,
      "activations/layer20_attention_weight_min": -25.384309768676758,
      "activations/layer21_attention_weight_max": 31.522592544555664,
      "activations/layer21_attention_weight_min": -21.365530014038086,
      "activations/layer22_attention_weight_max": 55.55916976928711,
      "activations/layer22_attention_weight_min": -25.581079483032227,
      "activations/layer23_attention_weight_max": 30.44342041015625,
      "activations/layer23_attention_weight_min": -17.266216278076172,
      "activations/layer2_attention_weight_max": 29.012130737304688,
      "activations/layer2_attention_weight_min": -29.444168090820312,
      "activations/layer3_attention_weight_max": 75.87489318847656,
      "activations/layer3_attention_weight_min": -79.77031707763672,
      "activations/layer4_attention_weight_max": 136.98272705078125,
      "activations/layer4_attention_weight_min": -125.9699935913086,
      "activations/layer5_attention_weight_max": 117.70098876953125,
      "activations/layer5_attention_weight_min": -98.66014099121094,
      "activations/layer6_attention_weight_max": 64.60042572021484,
      "activations/layer6_attention_weight_min": -55.3250732421875,
      "activations/layer7_attention_weight_max": 76.58373260498047,
      "activations/layer7_attention_weight_min": -72.7723159790039,
      "activations/layer8_attention_weight_max": 53.096656799316406,
      "activations/layer8_attention_weight_min": -40.58458709716797,
      "activations/layer9_attention_weight_max": 65.15941619873047,
      "activations/layer9_attention_weight_min": -49.343994140625,
      "epoch": 2.27,
      "learning_rate": 0.00013674204545454543,
      "loss": 2.9321,
      "step": 39050
    },
    {
      "activations/layer0_attention_weight_max": 17.25164222717285,
      "activations/layer0_attention_weight_min": -11.466172218322754,
      "activations/layer10_attention_weight_max": 42.844268798828125,
      "activations/layer10_attention_weight_min": -39.7012939453125,
      "activations/layer11_attention_weight_max": 39.22509765625,
      "activations/layer11_attention_weight_min": -34.16486358642578,
      "activations/layer12_attention_weight_max": 42.26967239379883,
      "activations/layer12_attention_weight_min": -26.20134162902832,
      "activations/layer13_attention_weight_max": 54.4351921081543,
      "activations/layer13_attention_weight_min": -37.272674560546875,
      "activations/layer14_attention_weight_max": 94.3822250366211,
      "activations/layer14_attention_weight_min": -67.10015869140625,
      "activations/layer15_attention_weight_max": 58.16067123413086,
      "activations/layer15_attention_weight_min": -38.84445571899414,
      "activations/layer16_attention_weight_max": 60.57822036743164,
      "activations/layer16_attention_weight_min": -35.46127700805664,
      "activations/layer17_attention_weight_max": 48.367977142333984,
      "activations/layer17_attention_weight_min": -29.96055793762207,
      "activations/layer18_attention_weight_max": 29.415380477905273,
      "activations/layer18_attention_weight_min": -18.933935165405273,
      "activations/layer19_attention_weight_max": 60.97756576538086,
      "activations/layer19_attention_weight_min": -30.509342193603516,
      "activations/layer1_attention_weight_max": 17.87733268737793,
      "activations/layer1_attention_weight_min": -11.75052547454834,
      "activations/layer20_attention_weight_max": 42.557456970214844,
      "activations/layer20_attention_weight_min": -22.934690475463867,
      "activations/layer21_attention_weight_max": 39.847930908203125,
      "activations/layer21_attention_weight_min": -18.927440643310547,
      "activations/layer22_attention_weight_max": 46.266353607177734,
      "activations/layer22_attention_weight_min": -23.652456283569336,
      "activations/layer23_attention_weight_max": 29.08198356628418,
      "activations/layer23_attention_weight_min": -15.315559387207031,
      "activations/layer2_attention_weight_max": 29.86749839782715,
      "activations/layer2_attention_weight_min": -27.820087432861328,
      "activations/layer3_attention_weight_max": 73.9859390258789,
      "activations/layer3_attention_weight_min": -77.67493438720703,
      "activations/layer4_attention_weight_max": 127.89007568359375,
      "activations/layer4_attention_weight_min": -118.84576416015625,
      "activations/layer5_attention_weight_max": 106.87734985351562,
      "activations/layer5_attention_weight_min": -95.17916870117188,
      "activations/layer6_attention_weight_max": 59.445255279541016,
      "activations/layer6_attention_weight_min": -51.48332595825195,
      "activations/layer7_attention_weight_max": 60.80262756347656,
      "activations/layer7_attention_weight_min": -64.54415893554688,
      "activations/layer8_attention_weight_max": 45.76137924194336,
      "activations/layer8_attention_weight_min": -36.538394927978516,
      "activations/layer9_attention_weight_max": 58.421504974365234,
      "activations/layer9_attention_weight_min": -52.06612014770508,
      "epoch": 2.27,
      "learning_rate": 0.00013672310606060604,
      "loss": 2.9446,
      "step": 39100
    },
    {
      "activations/layer0_attention_weight_max": 16.705421447753906,
      "activations/layer0_attention_weight_min": -11.410499572753906,
      "activations/layer10_attention_weight_max": 48.01289749145508,
      "activations/layer10_attention_weight_min": -38.7137336730957,
      "activations/layer11_attention_weight_max": 41.45209884643555,
      "activations/layer11_attention_weight_min": -35.23072814941406,
      "activations/layer12_attention_weight_max": 40.43265914916992,
      "activations/layer12_attention_weight_min": -32.55707550048828,
      "activations/layer13_attention_weight_max": 45.0391731262207,
      "activations/layer13_attention_weight_min": -35.26195526123047,
      "activations/layer14_attention_weight_max": 66.56510162353516,
      "activations/layer14_attention_weight_min": -53.87771987915039,
      "activations/layer15_attention_weight_max": 49.43855285644531,
      "activations/layer15_attention_weight_min": -36.31695556640625,
      "activations/layer16_attention_weight_max": 48.31297302246094,
      "activations/layer16_attention_weight_min": -36.43087387084961,
      "activations/layer17_attention_weight_max": 47.504600524902344,
      "activations/layer17_attention_weight_min": -30.154653549194336,
      "activations/layer18_attention_weight_max": 29.297462463378906,
      "activations/layer18_attention_weight_min": -17.4440975189209,
      "activations/layer19_attention_weight_max": 50.27820587158203,
      "activations/layer19_attention_weight_min": -34.18838882446289,
      "activations/layer1_attention_weight_max": 17.910593032836914,
      "activations/layer1_attention_weight_min": -10.261911392211914,
      "activations/layer20_attention_weight_max": 37.259761810302734,
      "activations/layer20_attention_weight_min": -24.5627498626709,
      "activations/layer21_attention_weight_max": 30.395183563232422,
      "activations/layer21_attention_weight_min": -19.036861419677734,
      "activations/layer22_attention_weight_max": 43.563812255859375,
      "activations/layer22_attention_weight_min": -25.391090393066406,
      "activations/layer23_attention_weight_max": 23.087064743041992,
      "activations/layer23_attention_weight_min": -15.893717765808105,
      "activations/layer2_attention_weight_max": 27.686859130859375,
      "activations/layer2_attention_weight_min": -27.834074020385742,
      "activations/layer3_attention_weight_max": 70.90443420410156,
      "activations/layer3_attention_weight_min": -73.70492553710938,
      "activations/layer4_attention_weight_max": 130.0459442138672,
      "activations/layer4_attention_weight_min": -117.19049072265625,
      "activations/layer5_attention_weight_max": 112.3653793334961,
      "activations/layer5_attention_weight_min": -93.98268127441406,
      "activations/layer6_attention_weight_max": 60.67155838012695,
      "activations/layer6_attention_weight_min": -51.95463562011719,
      "activations/layer7_attention_weight_max": 65.21570587158203,
      "activations/layer7_attention_weight_min": -66.50064086914062,
      "activations/layer8_attention_weight_max": 45.27143096923828,
      "activations/layer8_attention_weight_min": -38.9387321472168,
      "activations/layer9_attention_weight_max": 57.75568389892578,
      "activations/layer9_attention_weight_min": -50.75919723510742,
      "epoch": 2.27,
      "learning_rate": 0.00013670416666666666,
      "loss": 2.9303,
      "step": 39150
    },
    {
      "activations/layer0_attention_weight_max": 15.877401351928711,
      "activations/layer0_attention_weight_min": -12.23047924041748,
      "activations/layer10_attention_weight_max": 50.40748977661133,
      "activations/layer10_attention_weight_min": -39.68225860595703,
      "activations/layer11_attention_weight_max": 37.282066345214844,
      "activations/layer11_attention_weight_min": -32.126407623291016,
      "activations/layer12_attention_weight_max": 33.46518325805664,
      "activations/layer12_attention_weight_min": -27.50824546813965,
      "activations/layer13_attention_weight_max": 57.80680465698242,
      "activations/layer13_attention_weight_min": -38.68165588378906,
      "activations/layer14_attention_weight_max": 83.76349639892578,
      "activations/layer14_attention_weight_min": -57.13833999633789,
      "activations/layer15_attention_weight_max": 53.97471618652344,
      "activations/layer15_attention_weight_min": -35.08705139160156,
      "activations/layer16_attention_weight_max": 52.66619873046875,
      "activations/layer16_attention_weight_min": -34.15937423706055,
      "activations/layer17_attention_weight_max": 50.98537826538086,
      "activations/layer17_attention_weight_min": -29.434669494628906,
      "activations/layer18_attention_weight_max": 31.57058334350586,
      "activations/layer18_attention_weight_min": -17.70266342163086,
      "activations/layer19_attention_weight_max": 53.5949821472168,
      "activations/layer19_attention_weight_min": -30.964460372924805,
      "activations/layer1_attention_weight_max": 18.050294876098633,
      "activations/layer1_attention_weight_min": -10.850017547607422,
      "activations/layer20_attention_weight_max": 42.01352310180664,
      "activations/layer20_attention_weight_min": -23.885927200317383,
      "activations/layer21_attention_weight_max": 37.61955642700195,
      "activations/layer21_attention_weight_min": -21.953710556030273,
      "activations/layer22_attention_weight_max": 47.08860778808594,
      "activations/layer22_attention_weight_min": -26.131635665893555,
      "activations/layer23_attention_weight_max": 26.55573081970215,
      "activations/layer23_attention_weight_min": -15.848806381225586,
      "activations/layer2_attention_weight_max": 29.51641082763672,
      "activations/layer2_attention_weight_min": -29.464834213256836,
      "activations/layer3_attention_weight_max": 71.70597076416016,
      "activations/layer3_attention_weight_min": -78.33405303955078,
      "activations/layer4_attention_weight_max": 130.73826599121094,
      "activations/layer4_attention_weight_min": -121.21233367919922,
      "activations/layer5_attention_weight_max": 113.72057342529297,
      "activations/layer5_attention_weight_min": -98.23198699951172,
      "activations/layer6_attention_weight_max": 64.34552001953125,
      "activations/layer6_attention_weight_min": -55.34858703613281,
      "activations/layer7_attention_weight_max": 67.03189849853516,
      "activations/layer7_attention_weight_min": -67.39641571044922,
      "activations/layer8_attention_weight_max": 43.37971878051758,
      "activations/layer8_attention_weight_min": -40.28626251220703,
      "activations/layer9_attention_weight_max": 54.82455062866211,
      "activations/layer9_attention_weight_min": -44.948944091796875,
      "epoch": 2.28,
      "learning_rate": 0.00013668560606060605,
      "loss": 2.9416,
      "step": 39200
    },
    {
      "activations/layer0_attention_weight_max": 16.27564239501953,
      "activations/layer0_attention_weight_min": -11.060007095336914,
      "activations/layer10_attention_weight_max": 42.54680633544922,
      "activations/layer10_attention_weight_min": -39.1453742980957,
      "activations/layer11_attention_weight_max": 35.267189025878906,
      "activations/layer11_attention_weight_min": -36.44487762451172,
      "activations/layer12_attention_weight_max": 29.796049118041992,
      "activations/layer12_attention_weight_min": -27.02591896057129,
      "activations/layer13_attention_weight_max": 46.50762176513672,
      "activations/layer13_attention_weight_min": -34.75971603393555,
      "activations/layer14_attention_weight_max": 76.89566802978516,
      "activations/layer14_attention_weight_min": -57.470054626464844,
      "activations/layer15_attention_weight_max": 52.92802429199219,
      "activations/layer15_attention_weight_min": -38.12541580200195,
      "activations/layer16_attention_weight_max": 47.05601119995117,
      "activations/layer16_attention_weight_min": -38.054805755615234,
      "activations/layer17_attention_weight_max": 44.9022331237793,
      "activations/layer17_attention_weight_min": -29.977848052978516,
      "activations/layer18_attention_weight_max": 31.070899963378906,
      "activations/layer18_attention_weight_min": -19.904088973999023,
      "activations/layer19_attention_weight_max": 45.02325439453125,
      "activations/layer19_attention_weight_min": -30.591405868530273,
      "activations/layer1_attention_weight_max": 17.082494735717773,
      "activations/layer1_attention_weight_min": -10.891570091247559,
      "activations/layer20_attention_weight_max": 35.37339782714844,
      "activations/layer20_attention_weight_min": -24.418638229370117,
      "activations/layer21_attention_weight_max": 31.122812271118164,
      "activations/layer21_attention_weight_min": -22.772157669067383,
      "activations/layer22_attention_weight_max": 46.793617248535156,
      "activations/layer22_attention_weight_min": -28.04587745666504,
      "activations/layer23_attention_weight_max": 24.34740447998047,
      "activations/layer23_attention_weight_min": -17.037153244018555,
      "activations/layer2_attention_weight_max": 30.252086639404297,
      "activations/layer2_attention_weight_min": -28.880550384521484,
      "activations/layer3_attention_weight_max": 73.82977294921875,
      "activations/layer3_attention_weight_min": -75.52861785888672,
      "activations/layer4_attention_weight_max": 124.50882720947266,
      "activations/layer4_attention_weight_min": -125.48248291015625,
      "activations/layer5_attention_weight_max": 105.24529266357422,
      "activations/layer5_attention_weight_min": -99.61235809326172,
      "activations/layer6_attention_weight_max": 59.058326721191406,
      "activations/layer6_attention_weight_min": -53.342018127441406,
      "activations/layer7_attention_weight_max": 65.44037628173828,
      "activations/layer7_attention_weight_min": -62.18177032470703,
      "activations/layer8_attention_weight_max": 41.42790222167969,
      "activations/layer8_attention_weight_min": -36.82289505004883,
      "activations/layer9_attention_weight_max": 62.5473518371582,
      "activations/layer9_attention_weight_min": -47.39646530151367,
      "epoch": 2.28,
      "learning_rate": 0.00013666666666666666,
      "loss": 2.9484,
      "step": 39250
    },
    {
      "activations/layer0_attention_weight_max": 16.68535804748535,
      "activations/layer0_attention_weight_min": -11.342203140258789,
      "activations/layer10_attention_weight_max": 44.005577087402344,
      "activations/layer10_attention_weight_min": -41.17092514038086,
      "activations/layer11_attention_weight_max": 35.62095642089844,
      "activations/layer11_attention_weight_min": -31.703533172607422,
      "activations/layer12_attention_weight_max": 26.707685470581055,
      "activations/layer12_attention_weight_min": -27.733434677124023,
      "activations/layer13_attention_weight_max": 47.89810562133789,
      "activations/layer13_attention_weight_min": -35.40371322631836,
      "activations/layer14_attention_weight_max": 78.95527648925781,
      "activations/layer14_attention_weight_min": -67.38945770263672,
      "activations/layer15_attention_weight_max": 48.156883239746094,
      "activations/layer15_attention_weight_min": -35.512245178222656,
      "activations/layer16_attention_weight_max": 43.15127944946289,
      "activations/layer16_attention_weight_min": -35.0854377746582,
      "activations/layer17_attention_weight_max": 42.64793395996094,
      "activations/layer17_attention_weight_min": -30.224159240722656,
      "activations/layer18_attention_weight_max": 26.15872573852539,
      "activations/layer18_attention_weight_min": -19.4801082611084,
      "activations/layer19_attention_weight_max": 45.773040771484375,
      "activations/layer19_attention_weight_min": -33.990543365478516,
      "activations/layer1_attention_weight_max": 18.514427185058594,
      "activations/layer1_attention_weight_min": -12.867844581604004,
      "activations/layer20_attention_weight_max": 35.803043365478516,
      "activations/layer20_attention_weight_min": -25.34393882751465,
      "activations/layer21_attention_weight_max": 30.79262924194336,
      "activations/layer21_attention_weight_min": -20.131528854370117,
      "activations/layer22_attention_weight_max": 43.55978775024414,
      "activations/layer22_attention_weight_min": -27.62656021118164,
      "activations/layer23_attention_weight_max": 24.733360290527344,
      "activations/layer23_attention_weight_min": -17.59796714782715,
      "activations/layer2_attention_weight_max": 29.75613784790039,
      "activations/layer2_attention_weight_min": -28.514671325683594,
      "activations/layer3_attention_weight_max": 73.74266052246094,
      "activations/layer3_attention_weight_min": -76.52242279052734,
      "activations/layer4_attention_weight_max": 123.47430419921875,
      "activations/layer4_attention_weight_min": -116.96134185791016,
      "activations/layer5_attention_weight_max": 103.89814758300781,
      "activations/layer5_attention_weight_min": -92.362548828125,
      "activations/layer6_attention_weight_max": 59.00636291503906,
      "activations/layer6_attention_weight_min": -51.85704803466797,
      "activations/layer7_attention_weight_max": 63.66310501098633,
      "activations/layer7_attention_weight_min": -65.74623107910156,
      "activations/layer8_attention_weight_max": 43.62542724609375,
      "activations/layer8_attention_weight_min": -36.69302749633789,
      "activations/layer9_attention_weight_max": 63.11275100708008,
      "activations/layer9_attention_weight_min": -57.947425842285156,
      "epoch": 2.28,
      "learning_rate": 0.00013664772727272725,
      "loss": 2.9357,
      "step": 39300
    },
    {
      "activations/layer0_attention_weight_max": 14.939574241638184,
      "activations/layer0_attention_weight_min": -11.922469139099121,
      "activations/layer10_attention_weight_max": 43.79890060424805,
      "activations/layer10_attention_weight_min": -41.0841064453125,
      "activations/layer11_attention_weight_max": 36.76842498779297,
      "activations/layer11_attention_weight_min": -37.021671295166016,
      "activations/layer12_attention_weight_max": 34.030609130859375,
      "activations/layer12_attention_weight_min": -25.279930114746094,
      "activations/layer13_attention_weight_max": 45.77341842651367,
      "activations/layer13_attention_weight_min": -35.62077713012695,
      "activations/layer14_attention_weight_max": 70.333984375,
      "activations/layer14_attention_weight_min": -53.540687561035156,
      "activations/layer15_attention_weight_max": 53.80739974975586,
      "activations/layer15_attention_weight_min": -42.621524810791016,
      "activations/layer16_attention_weight_max": 44.804508209228516,
      "activations/layer16_attention_weight_min": -36.536460876464844,
      "activations/layer17_attention_weight_max": 42.21857833862305,
      "activations/layer17_attention_weight_min": -30.21994400024414,
      "activations/layer18_attention_weight_max": 28.84542465209961,
      "activations/layer18_attention_weight_min": -18.813997268676758,
      "activations/layer19_attention_weight_max": 46.88671112060547,
      "activations/layer19_attention_weight_min": -30.45318603515625,
      "activations/layer1_attention_weight_max": 17.267959594726562,
      "activations/layer1_attention_weight_min": -10.632036209106445,
      "activations/layer20_attention_weight_max": 39.548561096191406,
      "activations/layer20_attention_weight_min": -24.5587215423584,
      "activations/layer21_attention_weight_max": 39.83952713012695,
      "activations/layer21_attention_weight_min": -22.934263229370117,
      "activations/layer22_attention_weight_max": 46.6928596496582,
      "activations/layer22_attention_weight_min": -27.331201553344727,
      "activations/layer23_attention_weight_max": 26.831348419189453,
      "activations/layer23_attention_weight_min": -18.25197982788086,
      "activations/layer2_attention_weight_max": 28.230588912963867,
      "activations/layer2_attention_weight_min": -28.427391052246094,
      "activations/layer3_attention_weight_max": 72.17289733886719,
      "activations/layer3_attention_weight_min": -71.86064910888672,
      "activations/layer4_attention_weight_max": 128.79197692871094,
      "activations/layer4_attention_weight_min": -117.49971008300781,
      "activations/layer5_attention_weight_max": 106.29937744140625,
      "activations/layer5_attention_weight_min": -93.50035095214844,
      "activations/layer6_attention_weight_max": 59.2972412109375,
      "activations/layer6_attention_weight_min": -51.5561637878418,
      "activations/layer7_attention_weight_max": 66.55950164794922,
      "activations/layer7_attention_weight_min": -61.93076705932617,
      "activations/layer8_attention_weight_max": 39.46852493286133,
      "activations/layer8_attention_weight_min": -38.706974029541016,
      "activations/layer9_attention_weight_max": 54.48134231567383,
      "activations/layer9_attention_weight_min": -48.49715042114258,
      "epoch": 2.29,
      "learning_rate": 0.00013662878787878787,
      "loss": 2.9339,
      "step": 39350
    },
    {
      "activations/layer0_attention_weight_max": 15.871912956237793,
      "activations/layer0_attention_weight_min": -11.11331558227539,
      "activations/layer10_attention_weight_max": 51.21402359008789,
      "activations/layer10_attention_weight_min": -40.378292083740234,
      "activations/layer11_attention_weight_max": 40.853294372558594,
      "activations/layer11_attention_weight_min": -35.07049560546875,
      "activations/layer12_attention_weight_max": 47.51983642578125,
      "activations/layer12_attention_weight_min": -35.12617874145508,
      "activations/layer13_attention_weight_max": 60.31488037109375,
      "activations/layer13_attention_weight_min": -48.4343147277832,
      "activations/layer14_attention_weight_max": 62.100486755371094,
      "activations/layer14_attention_weight_min": -50.35823440551758,
      "activations/layer15_attention_weight_max": 45.98250961303711,
      "activations/layer15_attention_weight_min": -37.406715393066406,
      "activations/layer16_attention_weight_max": 44.833370208740234,
      "activations/layer16_attention_weight_min": -32.1262092590332,
      "activations/layer17_attention_weight_max": 45.75352478027344,
      "activations/layer17_attention_weight_min": -31.08060073852539,
      "activations/layer18_attention_weight_max": 33.52299499511719,
      "activations/layer18_attention_weight_min": -19.200374603271484,
      "activations/layer19_attention_weight_max": 50.244407653808594,
      "activations/layer19_attention_weight_min": -28.32599449157715,
      "activations/layer1_attention_weight_max": 17.25937843322754,
      "activations/layer1_attention_weight_min": -11.416707992553711,
      "activations/layer20_attention_weight_max": 36.56421661376953,
      "activations/layer20_attention_weight_min": -22.684541702270508,
      "activations/layer21_attention_weight_max": 30.102048873901367,
      "activations/layer21_attention_weight_min": -20.641632080078125,
      "activations/layer22_attention_weight_max": 44.01826477050781,
      "activations/layer22_attention_weight_min": -25.253007888793945,
      "activations/layer23_attention_weight_max": 23.00632095336914,
      "activations/layer23_attention_weight_min": -17.00857925415039,
      "activations/layer2_attention_weight_max": 28.130523681640625,
      "activations/layer2_attention_weight_min": -26.525609970092773,
      "activations/layer3_attention_weight_max": 71.33373260498047,
      "activations/layer3_attention_weight_min": -73.3792953491211,
      "activations/layer4_attention_weight_max": 122.54912567138672,
      "activations/layer4_attention_weight_min": -113.7602767944336,
      "activations/layer5_attention_weight_max": 105.73768615722656,
      "activations/layer5_attention_weight_min": -95.6949691772461,
      "activations/layer6_attention_weight_max": 59.57201385498047,
      "activations/layer6_attention_weight_min": -51.401065826416016,
      "activations/layer7_attention_weight_max": 71.07420349121094,
      "activations/layer7_attention_weight_min": -66.00164031982422,
      "activations/layer8_attention_weight_max": 43.58159255981445,
      "activations/layer8_attention_weight_min": -36.2208137512207,
      "activations/layer9_attention_weight_max": 64.12028503417969,
      "activations/layer9_attention_weight_min": -50.46888732910156,
      "epoch": 2.29,
      "learning_rate": 0.00013660984848484848,
      "loss": 2.9403,
      "step": 39400
    },
    {
      "activations/layer0_attention_weight_max": 15.07999038696289,
      "activations/layer0_attention_weight_min": -11.528210639953613,
      "activations/layer10_attention_weight_max": 52.654170989990234,
      "activations/layer10_attention_weight_min": -43.76985168457031,
      "activations/layer11_attention_weight_max": 50.53953170776367,
      "activations/layer11_attention_weight_min": -36.17570877075195,
      "activations/layer12_attention_weight_max": 65.68980407714844,
      "activations/layer12_attention_weight_min": -40.38998794555664,
      "activations/layer13_attention_weight_max": 68.62973022460938,
      "activations/layer13_attention_weight_min": -46.07459259033203,
      "activations/layer14_attention_weight_max": 90.69329833984375,
      "activations/layer14_attention_weight_min": -62.92750930786133,
      "activations/layer15_attention_weight_max": 57.07230758666992,
      "activations/layer15_attention_weight_min": -37.667457580566406,
      "activations/layer16_attention_weight_max": 53.44762420654297,
      "activations/layer16_attention_weight_min": -38.98406982421875,
      "activations/layer17_attention_weight_max": 49.32478713989258,
      "activations/layer17_attention_weight_min": -28.99009132385254,
      "activations/layer18_attention_weight_max": 31.850322723388672,
      "activations/layer18_attention_weight_min": -18.571577072143555,
      "activations/layer19_attention_weight_max": 52.9041862487793,
      "activations/layer19_attention_weight_min": -33.26687240600586,
      "activations/layer1_attention_weight_max": 17.048269271850586,
      "activations/layer1_attention_weight_min": -11.658330917358398,
      "activations/layer20_attention_weight_max": 37.31192398071289,
      "activations/layer20_attention_weight_min": -22.429414749145508,
      "activations/layer21_attention_weight_max": 26.249481201171875,
      "activations/layer21_attention_weight_min": -17.117584228515625,
      "activations/layer22_attention_weight_max": 46.39052963256836,
      "activations/layer22_attention_weight_min": -28.409101486206055,
      "activations/layer23_attention_weight_max": 28.01848602294922,
      "activations/layer23_attention_weight_min": -17.276491165161133,
      "activations/layer2_attention_weight_max": 30.04327964782715,
      "activations/layer2_attention_weight_min": -30.46190643310547,
      "activations/layer3_attention_weight_max": 82.80448913574219,
      "activations/layer3_attention_weight_min": -84.09557342529297,
      "activations/layer4_attention_weight_max": 147.75791931152344,
      "activations/layer4_attention_weight_min": -130.41416931152344,
      "activations/layer5_attention_weight_max": 129.0831298828125,
      "activations/layer5_attention_weight_min": -100.23561096191406,
      "activations/layer6_attention_weight_max": 75.84435272216797,
      "activations/layer6_attention_weight_min": -55.12260437011719,
      "activations/layer7_attention_weight_max": 83.40762329101562,
      "activations/layer7_attention_weight_min": -66.22791290283203,
      "activations/layer8_attention_weight_max": 57.50287628173828,
      "activations/layer8_attention_weight_min": -43.90586853027344,
      "activations/layer9_attention_weight_max": 84.86066436767578,
      "activations/layer9_attention_weight_min": -62.11838912963867,
      "epoch": 2.29,
      "learning_rate": 0.0001365909090909091,
      "loss": 2.9575,
      "step": 39450
    },
    {
      "activations/layer0_attention_weight_max": 16.34849739074707,
      "activations/layer0_attention_weight_min": -11.29914379119873,
      "activations/layer10_attention_weight_max": 48.63774108886719,
      "activations/layer10_attention_weight_min": -43.25425338745117,
      "activations/layer11_attention_weight_max": 38.30482864379883,
      "activations/layer11_attention_weight_min": -33.65549087524414,
      "activations/layer12_attention_weight_max": 39.233524322509766,
      "activations/layer12_attention_weight_min": -39.82124328613281,
      "activations/layer13_attention_weight_max": 48.48582077026367,
      "activations/layer13_attention_weight_min": -37.61894607543945,
      "activations/layer14_attention_weight_max": 89.67948150634766,
      "activations/layer14_attention_weight_min": -68.49261474609375,
      "activations/layer15_attention_weight_max": 50.9603157043457,
      "activations/layer15_attention_weight_min": -37.41874694824219,
      "activations/layer16_attention_weight_max": 49.03786849975586,
      "activations/layer16_attention_weight_min": -34.141754150390625,
      "activations/layer17_attention_weight_max": 51.31452178955078,
      "activations/layer17_attention_weight_min": -27.68364906311035,
      "activations/layer18_attention_weight_max": 29.231664657592773,
      "activations/layer18_attention_weight_min": -17.90865707397461,
      "activations/layer19_attention_weight_max": 50.54221725463867,
      "activations/layer19_attention_weight_min": -31.409265518188477,
      "activations/layer1_attention_weight_max": 18.451263427734375,
      "activations/layer1_attention_weight_min": -11.760906219482422,
      "activations/layer20_attention_weight_max": 39.70463562011719,
      "activations/layer20_attention_weight_min": -25.27260398864746,
      "activations/layer21_attention_weight_max": 33.2179069519043,
      "activations/layer21_attention_weight_min": -19.27277946472168,
      "activations/layer22_attention_weight_max": 53.56928253173828,
      "activations/layer22_attention_weight_min": -27.129941940307617,
      "activations/layer23_attention_weight_max": 25.658611297607422,
      "activations/layer23_attention_weight_min": -18.915271759033203,
      "activations/layer2_attention_weight_max": 28.226318359375,
      "activations/layer2_attention_weight_min": -27.734403610229492,
      "activations/layer3_attention_weight_max": 70.3995132446289,
      "activations/layer3_attention_weight_min": -72.88787841796875,
      "activations/layer4_attention_weight_max": 127.31086730957031,
      "activations/layer4_attention_weight_min": -117.44705963134766,
      "activations/layer5_attention_weight_max": 112.20244598388672,
      "activations/layer5_attention_weight_min": -96.18508911132812,
      "activations/layer6_attention_weight_max": 60.63261413574219,
      "activations/layer6_attention_weight_min": -53.04985809326172,
      "activations/layer7_attention_weight_max": 68.20835876464844,
      "activations/layer7_attention_weight_min": -63.908145904541016,
      "activations/layer8_attention_weight_max": 43.79426956176758,
      "activations/layer8_attention_weight_min": -37.305198669433594,
      "activations/layer9_attention_weight_max": 65.10851287841797,
      "activations/layer9_attention_weight_min": -56.47944259643555,
      "epoch": 2.3,
      "learning_rate": 0.0001365723484848485,
      "loss": 2.9418,
      "step": 39500
    },
    {
      "activations/layer0_attention_weight_max": 17.114543914794922,
      "activations/layer0_attention_weight_min": -11.411365509033203,
      "activations/layer10_attention_weight_max": 43.196041107177734,
      "activations/layer10_attention_weight_min": -38.8835563659668,
      "activations/layer11_attention_weight_max": 35.943756103515625,
      "activations/layer11_attention_weight_min": -32.91714859008789,
      "activations/layer12_attention_weight_max": 42.714229583740234,
      "activations/layer12_attention_weight_min": -29.540197372436523,
      "activations/layer13_attention_weight_max": 44.0109748840332,
      "activations/layer13_attention_weight_min": -36.247066497802734,
      "activations/layer14_attention_weight_max": 71.79306030273438,
      "activations/layer14_attention_weight_min": -57.9725456237793,
      "activations/layer15_attention_weight_max": 46.14545440673828,
      "activations/layer15_attention_weight_min": -35.76823425292969,
      "activations/layer16_attention_weight_max": 47.92552947998047,
      "activations/layer16_attention_weight_min": -35.10820770263672,
      "activations/layer17_attention_weight_max": 43.285888671875,
      "activations/layer17_attention_weight_min": -30.08766746520996,
      "activations/layer18_attention_weight_max": 25.467435836791992,
      "activations/layer18_attention_weight_min": -17.77606773376465,
      "activations/layer19_attention_weight_max": 45.36876678466797,
      "activations/layer19_attention_weight_min": -30.632720947265625,
      "activations/layer1_attention_weight_max": 17.95082664489746,
      "activations/layer1_attention_weight_min": -10.631759643554688,
      "activations/layer20_attention_weight_max": 33.609283447265625,
      "activations/layer20_attention_weight_min": -24.961912155151367,
      "activations/layer21_attention_weight_max": 33.1845588684082,
      "activations/layer21_attention_weight_min": -22.0046443939209,
      "activations/layer22_attention_weight_max": 40.794097900390625,
      "activations/layer22_attention_weight_min": -27.15123748779297,
      "activations/layer23_attention_weight_max": 23.011394500732422,
      "activations/layer23_attention_weight_min": -17.524324417114258,
      "activations/layer2_attention_weight_max": 29.434110641479492,
      "activations/layer2_attention_weight_min": -29.182079315185547,
      "activations/layer3_attention_weight_max": 72.25931549072266,
      "activations/layer3_attention_weight_min": -76.38729095458984,
      "activations/layer4_attention_weight_max": 127.31591796875,
      "activations/layer4_attention_weight_min": -118.3390884399414,
      "activations/layer5_attention_weight_max": 108.40165710449219,
      "activations/layer5_attention_weight_min": -93.22085571289062,
      "activations/layer6_attention_weight_max": 56.23162078857422,
      "activations/layer6_attention_weight_min": -50.78218078613281,
      "activations/layer7_attention_weight_max": 66.1268310546875,
      "activations/layer7_attention_weight_min": -63.63081741333008,
      "activations/layer8_attention_weight_max": 42.704383850097656,
      "activations/layer8_attention_weight_min": -39.5465087890625,
      "activations/layer9_attention_weight_max": 56.19222640991211,
      "activations/layer9_attention_weight_min": -46.54970169067383,
      "epoch": 2.3,
      "learning_rate": 0.00013655340909090908,
      "loss": 2.9453,
      "step": 39550
    },
    {
      "activations/layer0_attention_weight_max": 16.254735946655273,
      "activations/layer0_attention_weight_min": -11.519722938537598,
      "activations/layer10_attention_weight_max": 51.879119873046875,
      "activations/layer10_attention_weight_min": -41.473487854003906,
      "activations/layer11_attention_weight_max": 40.97810745239258,
      "activations/layer11_attention_weight_min": -33.495933532714844,
      "activations/layer12_attention_weight_max": 35.416038513183594,
      "activations/layer12_attention_weight_min": -30.279138565063477,
      "activations/layer13_attention_weight_max": 46.12014389038086,
      "activations/layer13_attention_weight_min": -36.057273864746094,
      "activations/layer14_attention_weight_max": 75.29874420166016,
      "activations/layer14_attention_weight_min": -56.39811325073242,
      "activations/layer15_attention_weight_max": 50.00237274169922,
      "activations/layer15_attention_weight_min": -38.12470626831055,
      "activations/layer16_attention_weight_max": 56.271751403808594,
      "activations/layer16_attention_weight_min": -36.12212371826172,
      "activations/layer17_attention_weight_max": 46.48922348022461,
      "activations/layer17_attention_weight_min": -29.54107666015625,
      "activations/layer18_attention_weight_max": 26.975849151611328,
      "activations/layer18_attention_weight_min": -19.194433212280273,
      "activations/layer19_attention_weight_max": 60.25074005126953,
      "activations/layer19_attention_weight_min": -32.434932708740234,
      "activations/layer1_attention_weight_max": 17.14047622680664,
      "activations/layer1_attention_weight_min": -11.68839168548584,
      "activations/layer20_attention_weight_max": 37.32965087890625,
      "activations/layer20_attention_weight_min": -24.521888732910156,
      "activations/layer21_attention_weight_max": 29.423599243164062,
      "activations/layer21_attention_weight_min": -18.544755935668945,
      "activations/layer22_attention_weight_max": 43.400672912597656,
      "activations/layer22_attention_weight_min": -27.993053436279297,
      "activations/layer23_attention_weight_max": 25.11017608642578,
      "activations/layer23_attention_weight_min": -19.172122955322266,
      "activations/layer2_attention_weight_max": 28.584381103515625,
      "activations/layer2_attention_weight_min": -28.516632080078125,
      "activations/layer3_attention_weight_max": 73.52229309082031,
      "activations/layer3_attention_weight_min": -74.28544616699219,
      "activations/layer4_attention_weight_max": 125.65217590332031,
      "activations/layer4_attention_weight_min": -115.32572174072266,
      "activations/layer5_attention_weight_max": 109.47250366210938,
      "activations/layer5_attention_weight_min": -94.84780883789062,
      "activations/layer6_attention_weight_max": 60.46687698364258,
      "activations/layer6_attention_weight_min": -52.42100143432617,
      "activations/layer7_attention_weight_max": 70.29863739013672,
      "activations/layer7_attention_weight_min": -67.09027099609375,
      "activations/layer8_attention_weight_max": 48.891517639160156,
      "activations/layer8_attention_weight_min": -39.49662780761719,
      "activations/layer9_attention_weight_max": 60.19340896606445,
      "activations/layer9_attention_weight_min": -47.14213943481445,
      "epoch": 2.3,
      "learning_rate": 0.0001365344696969697,
      "loss": 2.9529,
      "step": 39600
    },
    {
      "activations/layer0_attention_weight_max": 16.368755340576172,
      "activations/layer0_attention_weight_min": -12.332171440124512,
      "activations/layer10_attention_weight_max": 57.0323486328125,
      "activations/layer10_attention_weight_min": -42.742950439453125,
      "activations/layer11_attention_weight_max": 48.980464935302734,
      "activations/layer11_attention_weight_min": -37.457618713378906,
      "activations/layer12_attention_weight_max": 36.288551330566406,
      "activations/layer12_attention_weight_min": -30.587736129760742,
      "activations/layer13_attention_weight_max": 55.78183364868164,
      "activations/layer13_attention_weight_min": -43.19346237182617,
      "activations/layer14_attention_weight_max": 68.3598403930664,
      "activations/layer14_attention_weight_min": -56.697418212890625,
      "activations/layer15_attention_weight_max": 47.76995086669922,
      "activations/layer15_attention_weight_min": -35.48890686035156,
      "activations/layer16_attention_weight_max": 49.464115142822266,
      "activations/layer16_attention_weight_min": -36.11764144897461,
      "activations/layer17_attention_weight_max": 41.452430725097656,
      "activations/layer17_attention_weight_min": -28.39783477783203,
      "activations/layer18_attention_weight_max": 27.713090896606445,
      "activations/layer18_attention_weight_min": -17.31777572631836,
      "activations/layer19_attention_weight_max": 50.080169677734375,
      "activations/layer19_attention_weight_min": -33.17892074584961,
      "activations/layer1_attention_weight_max": 17.341825485229492,
      "activations/layer1_attention_weight_min": -11.603209495544434,
      "activations/layer20_attention_weight_max": 36.608421325683594,
      "activations/layer20_attention_weight_min": -24.44659423828125,
      "activations/layer21_attention_weight_max": 31.703548431396484,
      "activations/layer21_attention_weight_min": -20.472436904907227,
      "activations/layer22_attention_weight_max": 49.20894241333008,
      "activations/layer22_attention_weight_min": -30.925922393798828,
      "activations/layer23_attention_weight_max": 25.02640151977539,
      "activations/layer23_attention_weight_min": -15.947576522827148,
      "activations/layer2_attention_weight_max": 29.242582321166992,
      "activations/layer2_attention_weight_min": -28.45345687866211,
      "activations/layer3_attention_weight_max": 73.4168930053711,
      "activations/layer3_attention_weight_min": -77.3857421875,
      "activations/layer4_attention_weight_max": 134.090576171875,
      "activations/layer4_attention_weight_min": -124.31627655029297,
      "activations/layer5_attention_weight_max": 117.50257873535156,
      "activations/layer5_attention_weight_min": -104.3470458984375,
      "activations/layer6_attention_weight_max": 64.80048370361328,
      "activations/layer6_attention_weight_min": -55.328369140625,
      "activations/layer7_attention_weight_max": 72.62700653076172,
      "activations/layer7_attention_weight_min": -64.95381927490234,
      "activations/layer8_attention_weight_max": 52.904632568359375,
      "activations/layer8_attention_weight_min": -42.55801010131836,
      "activations/layer9_attention_weight_max": 76.01712799072266,
      "activations/layer9_attention_weight_min": -55.2899055480957,
      "epoch": 2.3,
      "learning_rate": 0.0001365155303030303,
      "loss": 2.9454,
      "step": 39650
    },
    {
      "activations/layer0_attention_weight_max": 16.773405075073242,
      "activations/layer0_attention_weight_min": -11.781071662902832,
      "activations/layer10_attention_weight_max": 48.547454833984375,
      "activations/layer10_attention_weight_min": -37.90032958984375,
      "activations/layer11_attention_weight_max": 43.24282455444336,
      "activations/layer11_attention_weight_min": -34.014015197753906,
      "activations/layer12_attention_weight_max": 39.614620208740234,
      "activations/layer12_attention_weight_min": -31.81615447998047,
      "activations/layer13_attention_weight_max": 51.927101135253906,
      "activations/layer13_attention_weight_min": -36.16256332397461,
      "activations/layer14_attention_weight_max": 82.50480651855469,
      "activations/layer14_attention_weight_min": -58.56947708129883,
      "activations/layer15_attention_weight_max": 62.16627502441406,
      "activations/layer15_attention_weight_min": -39.14551544189453,
      "activations/layer16_attention_weight_max": 53.74024200439453,
      "activations/layer16_attention_weight_min": -37.95441818237305,
      "activations/layer17_attention_weight_max": 43.306556701660156,
      "activations/layer17_attention_weight_min": -30.567909240722656,
      "activations/layer18_attention_weight_max": 26.947628021240234,
      "activations/layer18_attention_weight_min": -21.12166404724121,
      "activations/layer19_attention_weight_max": 59.64228439331055,
      "activations/layer19_attention_weight_min": -30.9307804107666,
      "activations/layer1_attention_weight_max": 17.435535430908203,
      "activations/layer1_attention_weight_min": -10.731691360473633,
      "activations/layer20_attention_weight_max": 47.07910919189453,
      "activations/layer20_attention_weight_min": -27.95039939880371,
      "activations/layer21_attention_weight_max": 33.75566101074219,
      "activations/layer21_attention_weight_min": -22.40542984008789,
      "activations/layer22_attention_weight_max": 48.29553985595703,
      "activations/layer22_attention_weight_min": -26.168855667114258,
      "activations/layer23_attention_weight_max": 25.187347412109375,
      "activations/layer23_attention_weight_min": -16.229637145996094,
      "activations/layer2_attention_weight_max": 29.973976135253906,
      "activations/layer2_attention_weight_min": -30.337066650390625,
      "activations/layer3_attention_weight_max": 73.90196990966797,
      "activations/layer3_attention_weight_min": -77.38201141357422,
      "activations/layer4_attention_weight_max": 142.3675994873047,
      "activations/layer4_attention_weight_min": -122.73835754394531,
      "activations/layer5_attention_weight_max": 121.37489318847656,
      "activations/layer5_attention_weight_min": -94.9414291381836,
      "activations/layer6_attention_weight_max": 65.3436508178711,
      "activations/layer6_attention_weight_min": -53.09117889404297,
      "activations/layer7_attention_weight_max": 79.93038177490234,
      "activations/layer7_attention_weight_min": -66.89126586914062,
      "activations/layer8_attention_weight_max": 49.02644729614258,
      "activations/layer8_attention_weight_min": -38.21257400512695,
      "activations/layer9_attention_weight_max": 65.66261291503906,
      "activations/layer9_attention_weight_min": -55.235992431640625,
      "epoch": 2.31,
      "learning_rate": 0.0001364965909090909,
      "loss": 2.9568,
      "step": 39700
    },
    {
      "activations/layer0_attention_weight_max": 15.817610740661621,
      "activations/layer0_attention_weight_min": -11.971834182739258,
      "activations/layer10_attention_weight_max": 53.9383544921875,
      "activations/layer10_attention_weight_min": -45.2100715637207,
      "activations/layer11_attention_weight_max": 39.477691650390625,
      "activations/layer11_attention_weight_min": -33.708282470703125,
      "activations/layer12_attention_weight_max": 48.48792266845703,
      "activations/layer12_attention_weight_min": -28.009315490722656,
      "activations/layer13_attention_weight_max": 49.26020812988281,
      "activations/layer13_attention_weight_min": -37.93535614013672,
      "activations/layer14_attention_weight_max": 81.56958770751953,
      "activations/layer14_attention_weight_min": -57.28649139404297,
      "activations/layer15_attention_weight_max": 55.2809944152832,
      "activations/layer15_attention_weight_min": -36.870540618896484,
      "activations/layer16_attention_weight_max": 51.77022171020508,
      "activations/layer16_attention_weight_min": -37.245365142822266,
      "activations/layer17_attention_weight_max": 52.42406463623047,
      "activations/layer17_attention_weight_min": -30.92129135131836,
      "activations/layer18_attention_weight_max": 31.999813079833984,
      "activations/layer18_attention_weight_min": -18.41262435913086,
      "activations/layer19_attention_weight_max": 50.55405044555664,
      "activations/layer19_attention_weight_min": -32.46635437011719,
      "activations/layer1_attention_weight_max": 18.40444564819336,
      "activations/layer1_attention_weight_min": -11.267413139343262,
      "activations/layer20_attention_weight_max": 39.56031036376953,
      "activations/layer20_attention_weight_min": -24.29926109313965,
      "activations/layer21_attention_weight_max": 36.898651123046875,
      "activations/layer21_attention_weight_min": -22.30544662475586,
      "activations/layer22_attention_weight_max": 49.03790283203125,
      "activations/layer22_attention_weight_min": -26.65591812133789,
      "activations/layer23_attention_weight_max": 27.586462020874023,
      "activations/layer23_attention_weight_min": -17.605356216430664,
      "activations/layer2_attention_weight_max": 28.52037811279297,
      "activations/layer2_attention_weight_min": -28.07878875732422,
      "activations/layer3_attention_weight_max": 74.48358154296875,
      "activations/layer3_attention_weight_min": -77.904052734375,
      "activations/layer4_attention_weight_max": 128.44961547851562,
      "activations/layer4_attention_weight_min": -126.67256927490234,
      "activations/layer5_attention_weight_max": 109.05474853515625,
      "activations/layer5_attention_weight_min": -95.79949951171875,
      "activations/layer6_attention_weight_max": 58.74019241333008,
      "activations/layer6_attention_weight_min": -52.26057815551758,
      "activations/layer7_attention_weight_max": 66.7125473022461,
      "activations/layer7_attention_weight_min": -64.00578308105469,
      "activations/layer8_attention_weight_max": 42.74355697631836,
      "activations/layer8_attention_weight_min": -36.271121978759766,
      "activations/layer9_attention_weight_max": 63.154388427734375,
      "activations/layer9_attention_weight_min": -47.3847541809082,
      "epoch": 2.31,
      "learning_rate": 0.0001364776515151515,
      "loss": 2.9599,
      "step": 39750
    },
    {
      "activations/layer0_attention_weight_max": 14.068001747131348,
      "activations/layer0_attention_weight_min": -11.816189765930176,
      "activations/layer10_attention_weight_max": 64.09661102294922,
      "activations/layer10_attention_weight_min": -43.34926223754883,
      "activations/layer11_attention_weight_max": 59.118408203125,
      "activations/layer11_attention_weight_min": -39.91168212890625,
      "activations/layer12_attention_weight_max": 77.1986083984375,
      "activations/layer12_attention_weight_min": -38.29899215698242,
      "activations/layer13_attention_weight_max": 76.9109115600586,
      "activations/layer13_attention_weight_min": -47.66773223876953,
      "activations/layer14_attention_weight_max": 78.11811065673828,
      "activations/layer14_attention_weight_min": -55.925453186035156,
      "activations/layer15_attention_weight_max": 51.90414810180664,
      "activations/layer15_attention_weight_min": -34.231475830078125,
      "activations/layer16_attention_weight_max": 45.101531982421875,
      "activations/layer16_attention_weight_min": -31.14111328125,
      "activations/layer17_attention_weight_max": 45.45230484008789,
      "activations/layer17_attention_weight_min": -28.810890197753906,
      "activations/layer18_attention_weight_max": 28.688583374023438,
      "activations/layer18_attention_weight_min": -17.949756622314453,
      "activations/layer19_attention_weight_max": 46.527015686035156,
      "activations/layer19_attention_weight_min": -29.802337646484375,
      "activations/layer1_attention_weight_max": 18.298280715942383,
      "activations/layer1_attention_weight_min": -12.032234191894531,
      "activations/layer20_attention_weight_max": 36.01765060424805,
      "activations/layer20_attention_weight_min": -23.156633377075195,
      "activations/layer21_attention_weight_max": 32.187957763671875,
      "activations/layer21_attention_weight_min": -19.8597354888916,
      "activations/layer22_attention_weight_max": 43.180259704589844,
      "activations/layer22_attention_weight_min": -28.7972354888916,
      "activations/layer23_attention_weight_max": 22.56130027770996,
      "activations/layer23_attention_weight_min": -16.547273635864258,
      "activations/layer2_attention_weight_max": 29.05685806274414,
      "activations/layer2_attention_weight_min": -28.448284149169922,
      "activations/layer3_attention_weight_max": 72.6325912475586,
      "activations/layer3_attention_weight_min": -76.34954071044922,
      "activations/layer4_attention_weight_max": 136.74948120117188,
      "activations/layer4_attention_weight_min": -124.00843811035156,
      "activations/layer5_attention_weight_max": 121.41720581054688,
      "activations/layer5_attention_weight_min": -101.5893783569336,
      "activations/layer6_attention_weight_max": 70.89442443847656,
      "activations/layer6_attention_weight_min": -57.60391616821289,
      "activations/layer7_attention_weight_max": 82.29638671875,
      "activations/layer7_attention_weight_min": -67.27375030517578,
      "activations/layer8_attention_weight_max": 63.15864944458008,
      "activations/layer8_attention_weight_min": -43.41420364379883,
      "activations/layer9_attention_weight_max": 94.0085678100586,
      "activations/layer9_attention_weight_min": -71.8178939819336,
      "epoch": 2.31,
      "learning_rate": 0.00013645871212121212,
      "loss": 2.95,
      "step": 39800
    },
    {
      "activations/layer0_attention_weight_max": 16.264022827148438,
      "activations/layer0_attention_weight_min": -11.25804328918457,
      "activations/layer10_attention_weight_max": 47.3677864074707,
      "activations/layer10_attention_weight_min": -41.68984603881836,
      "activations/layer11_attention_weight_max": 37.85155487060547,
      "activations/layer11_attention_weight_min": -31.11231803894043,
      "activations/layer12_attention_weight_max": 42.00092315673828,
      "activations/layer12_attention_weight_min": -28.15561294555664,
      "activations/layer13_attention_weight_max": 57.45631790161133,
      "activations/layer13_attention_weight_min": -40.56909942626953,
      "activations/layer14_attention_weight_max": 84.3261947631836,
      "activations/layer14_attention_weight_min": -59.8284912109375,
      "activations/layer15_attention_weight_max": 52.690467834472656,
      "activations/layer15_attention_weight_min": -35.78917694091797,
      "activations/layer16_attention_weight_max": 45.35113525390625,
      "activations/layer16_attention_weight_min": -34.93755340576172,
      "activations/layer17_attention_weight_max": 46.62065505981445,
      "activations/layer17_attention_weight_min": -27.023305892944336,
      "activations/layer18_attention_weight_max": 26.49395179748535,
      "activations/layer18_attention_weight_min": -17.943470001220703,
      "activations/layer19_attention_weight_max": 51.35243606567383,
      "activations/layer19_attention_weight_min": -30.043903350830078,
      "activations/layer1_attention_weight_max": 17.209566116333008,
      "activations/layer1_attention_weight_min": -10.946137428283691,
      "activations/layer20_attention_weight_max": 40.88711929321289,
      "activations/layer20_attention_weight_min": -24.224899291992188,
      "activations/layer21_attention_weight_max": 33.157020568847656,
      "activations/layer21_attention_weight_min": -21.301937103271484,
      "activations/layer22_attention_weight_max": 48.62818908691406,
      "activations/layer22_attention_weight_min": -27.105243682861328,
      "activations/layer23_attention_weight_max": 28.21772575378418,
      "activations/layer23_attention_weight_min": -15.992823600769043,
      "activations/layer2_attention_weight_max": 28.728534698486328,
      "activations/layer2_attention_weight_min": -28.078054428100586,
      "activations/layer3_attention_weight_max": 70.61734771728516,
      "activations/layer3_attention_weight_min": -73.22949981689453,
      "activations/layer4_attention_weight_max": 122.65130615234375,
      "activations/layer4_attention_weight_min": -120.32958221435547,
      "activations/layer5_attention_weight_max": 105.22652435302734,
      "activations/layer5_attention_weight_min": -96.12664794921875,
      "activations/layer6_attention_weight_max": 57.778324127197266,
      "activations/layer6_attention_weight_min": -55.3746452331543,
      "activations/layer7_attention_weight_max": 65.88388061523438,
      "activations/layer7_attention_weight_min": -64.5796890258789,
      "activations/layer8_attention_weight_max": 43.203651428222656,
      "activations/layer8_attention_weight_min": -37.07249450683594,
      "activations/layer9_attention_weight_max": 54.462066650390625,
      "activations/layer9_attention_weight_min": -53.3341178894043,
      "epoch": 2.32,
      "learning_rate": 0.0001364397727272727,
      "loss": 2.9456,
      "step": 39850
    },
    {
      "activations/layer0_attention_weight_max": 16.55516242980957,
      "activations/layer0_attention_weight_min": -11.749445915222168,
      "activations/layer10_attention_weight_max": 54.09345245361328,
      "activations/layer10_attention_weight_min": -43.65165328979492,
      "activations/layer11_attention_weight_max": 45.90009307861328,
      "activations/layer11_attention_weight_min": -37.76952362060547,
      "activations/layer12_attention_weight_max": 37.41510772705078,
      "activations/layer12_attention_weight_min": -29.127424240112305,
      "activations/layer13_attention_weight_max": 76.08309173583984,
      "activations/layer13_attention_weight_min": -64.31613159179688,
      "activations/layer14_attention_weight_max": 78.52528381347656,
      "activations/layer14_attention_weight_min": -57.03502655029297,
      "activations/layer15_attention_weight_max": 58.51298904418945,
      "activations/layer15_attention_weight_min": -49.672813415527344,
      "activations/layer16_attention_weight_max": 45.46540832519531,
      "activations/layer16_attention_weight_min": -34.5935173034668,
      "activations/layer17_attention_weight_max": 45.03687286376953,
      "activations/layer17_attention_weight_min": -30.75341796875,
      "activations/layer18_attention_weight_max": 26.983623504638672,
      "activations/layer18_attention_weight_min": -23.3535099029541,
      "activations/layer19_attention_weight_max": 46.8975715637207,
      "activations/layer19_attention_weight_min": -33.99485397338867,
      "activations/layer1_attention_weight_max": 17.990243911743164,
      "activations/layer1_attention_weight_min": -11.956456184387207,
      "activations/layer20_attention_weight_max": 35.27964782714844,
      "activations/layer20_attention_weight_min": -24.96294403076172,
      "activations/layer21_attention_weight_max": 32.3607063293457,
      "activations/layer21_attention_weight_min": -19.87067985534668,
      "activations/layer22_attention_weight_max": 45.63590621948242,
      "activations/layer22_attention_weight_min": -27.20183563232422,
      "activations/layer23_attention_weight_max": 23.544185638427734,
      "activations/layer23_attention_weight_min": -15.439586639404297,
      "activations/layer2_attention_weight_max": 30.07916831970215,
      "activations/layer2_attention_weight_min": -28.654579162597656,
      "activations/layer3_attention_weight_max": 73.42407989501953,
      "activations/layer3_attention_weight_min": -76.23748779296875,
      "activations/layer4_attention_weight_max": 123.8916244506836,
      "activations/layer4_attention_weight_min": -120.87568664550781,
      "activations/layer5_attention_weight_max": 109.80538940429688,
      "activations/layer5_attention_weight_min": -101.2801513671875,
      "activations/layer6_attention_weight_max": 60.375999450683594,
      "activations/layer6_attention_weight_min": -51.26369094848633,
      "activations/layer7_attention_weight_max": 66.6020736694336,
      "activations/layer7_attention_weight_min": -64.81275177001953,
      "activations/layer8_attention_weight_max": 46.283138275146484,
      "activations/layer8_attention_weight_min": -39.13211441040039,
      "activations/layer9_attention_weight_max": 65.67147064208984,
      "activations/layer9_attention_weight_min": -65.1654281616211,
      "epoch": 2.32,
      "learning_rate": 0.00013642083333333333,
      "loss": 2.9427,
      "step": 39900
    },
    {
      "activations/layer0_attention_weight_max": 16.68817901611328,
      "activations/layer0_attention_weight_min": -11.204200744628906,
      "activations/layer10_attention_weight_max": 54.86091995239258,
      "activations/layer10_attention_weight_min": -42.47554397583008,
      "activations/layer11_attention_weight_max": 40.450965881347656,
      "activations/layer11_attention_weight_min": -33.300086975097656,
      "activations/layer12_attention_weight_max": 35.31486129760742,
      "activations/layer12_attention_weight_min": -30.28032112121582,
      "activations/layer13_attention_weight_max": 48.62120056152344,
      "activations/layer13_attention_weight_min": -37.14014434814453,
      "activations/layer14_attention_weight_max": 79.86803436279297,
      "activations/layer14_attention_weight_min": -62.00434112548828,
      "activations/layer15_attention_weight_max": 52.312957763671875,
      "activations/layer15_attention_weight_min": -37.98120880126953,
      "activations/layer16_attention_weight_max": 50.70741653442383,
      "activations/layer16_attention_weight_min": -39.87971878051758,
      "activations/layer17_attention_weight_max": 44.304054260253906,
      "activations/layer17_attention_weight_min": -30.62458610534668,
      "activations/layer18_attention_weight_max": 28.483356475830078,
      "activations/layer18_attention_weight_min": -18.405879974365234,
      "activations/layer19_attention_weight_max": 51.136287689208984,
      "activations/layer19_attention_weight_min": -29.8381290435791,
      "activations/layer1_attention_weight_max": 17.650407791137695,
      "activations/layer1_attention_weight_min": -11.836565017700195,
      "activations/layer20_attention_weight_max": 37.17368698120117,
      "activations/layer20_attention_weight_min": -24.551250457763672,
      "activations/layer21_attention_weight_max": 36.38754653930664,
      "activations/layer21_attention_weight_min": -20.636789321899414,
      "activations/layer22_attention_weight_max": 47.21864318847656,
      "activations/layer22_attention_weight_min": -24.260028839111328,
      "activations/layer23_attention_weight_max": 26.516576766967773,
      "activations/layer23_attention_weight_min": -16.792396545410156,
      "activations/layer2_attention_weight_max": 28.424497604370117,
      "activations/layer2_attention_weight_min": -27.652896881103516,
      "activations/layer3_attention_weight_max": 71.18244171142578,
      "activations/layer3_attention_weight_min": -75.402099609375,
      "activations/layer4_attention_weight_max": 128.18910217285156,
      "activations/layer4_attention_weight_min": -117.34815979003906,
      "activations/layer5_attention_weight_max": 113.83134460449219,
      "activations/layer5_attention_weight_min": -92.45089721679688,
      "activations/layer6_attention_weight_max": 60.59295654296875,
      "activations/layer6_attention_weight_min": -51.88414764404297,
      "activations/layer7_attention_weight_max": 71.82720947265625,
      "activations/layer7_attention_weight_min": -66.7166748046875,
      "activations/layer8_attention_weight_max": 46.78062057495117,
      "activations/layer8_attention_weight_min": -36.85712814331055,
      "activations/layer9_attention_weight_max": 63.0853157043457,
      "activations/layer9_attention_weight_min": -54.33285140991211,
      "epoch": 2.32,
      "learning_rate": 0.00013640189393939391,
      "loss": 2.9426,
      "step": 39950
    },
    {
      "activations/layer0_attention_weight_max": 13.62813663482666,
      "activations/layer0_attention_weight_min": -11.96358585357666,
      "activations/layer10_attention_weight_max": 55.76123809814453,
      "activations/layer10_attention_weight_min": -41.94615173339844,
      "activations/layer11_attention_weight_max": 61.560115814208984,
      "activations/layer11_attention_weight_min": -38.010101318359375,
      "activations/layer12_attention_weight_max": 61.84597396850586,
      "activations/layer12_attention_weight_min": -31.060672760009766,
      "activations/layer13_attention_weight_max": 95.45787048339844,
      "activations/layer13_attention_weight_min": -57.70955276489258,
      "activations/layer14_attention_weight_max": 87.12959289550781,
      "activations/layer14_attention_weight_min": -63.05306625366211,
      "activations/layer15_attention_weight_max": 57.32486343383789,
      "activations/layer15_attention_weight_min": -35.898799896240234,
      "activations/layer16_attention_weight_max": 44.95723342895508,
      "activations/layer16_attention_weight_min": -34.4538459777832,
      "activations/layer17_attention_weight_max": 44.916988372802734,
      "activations/layer17_attention_weight_min": -28.8859806060791,
      "activations/layer18_attention_weight_max": 33.36051559448242,
      "activations/layer18_attention_weight_min": -20.16718101501465,
      "activations/layer19_attention_weight_max": 51.591121673583984,
      "activations/layer19_attention_weight_min": -31.033533096313477,
      "activations/layer1_attention_weight_max": 16.533580780029297,
      "activations/layer1_attention_weight_min": -12.881802558898926,
      "activations/layer20_attention_weight_max": 34.940147399902344,
      "activations/layer20_attention_weight_min": -24.53517723083496,
      "activations/layer21_attention_weight_max": 30.828195571899414,
      "activations/layer21_attention_weight_min": -21.051177978515625,
      "activations/layer22_attention_weight_max": 47.98337936401367,
      "activations/layer22_attention_weight_min": -25.335784912109375,
      "activations/layer23_attention_weight_max": 24.285354614257812,
      "activations/layer23_attention_weight_min": -18.707063674926758,
      "activations/layer2_attention_weight_max": 31.535850524902344,
      "activations/layer2_attention_weight_min": -26.99036407470703,
      "activations/layer3_attention_weight_max": 71.06266021728516,
      "activations/layer3_attention_weight_min": -66.68849182128906,
      "activations/layer4_attention_weight_max": 135.66314697265625,
      "activations/layer4_attention_weight_min": -102.71315002441406,
      "activations/layer5_attention_weight_max": 124.15177917480469,
      "activations/layer5_attention_weight_min": -90.90751647949219,
      "activations/layer6_attention_weight_max": 72.70740509033203,
      "activations/layer6_attention_weight_min": -54.98950958251953,
      "activations/layer7_attention_weight_max": 87.79515075683594,
      "activations/layer7_attention_weight_min": -64.814453125,
      "activations/layer8_attention_weight_max": 55.14890670776367,
      "activations/layer8_attention_weight_min": -38.05824279785156,
      "activations/layer9_attention_weight_max": 86.1476058959961,
      "activations/layer9_attention_weight_min": -63.354915618896484,
      "epoch": 2.32,
      "learning_rate": 0.00013638333333333333,
      "loss": 2.9867,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_loss": 2.927734375,
      "eval_runtime": 8.4916,
      "eval_samples_per_second": 505.674,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_openwebtext_loss": 2.927734375,
      "eval_openwebtext_ppl": 18.6852487375444,
      "eval_openwebtext_runtime": 8.4916,
      "eval_openwebtext_samples_per_second": 505.674,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_wikitext_loss": 3.177734375,
      "eval_wikitext_ppl": 23.992334296139543,
      "eval_wikitext_runtime": 1.9953,
      "eval_wikitext_samples_per_second": 228.532,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_lambada_loss": 3.15625,
      "eval_lambada_ppl": 23.482371718117374,
      "eval_lambada_runtime": 9.5677,
      "eval_lambada_samples_per_second": 508.898,
      "step": 40000
    },
    {
      "activations/layer0_attention_weight_max": 14.266951560974121,
      "activations/layer0_attention_weight_min": -12.389720916748047,
      "activations/layer10_attention_weight_max": 52.154571533203125,
      "activations/layer10_attention_weight_min": -40.01697540283203,
      "activations/layer11_attention_weight_max": 38.78017044067383,
      "activations/layer11_attention_weight_min": -33.162353515625,
      "activations/layer12_attention_weight_max": 83.2949447631836,
      "activations/layer12_attention_weight_min": -37.45348358154297,
      "activations/layer13_attention_weight_max": 63.219993591308594,
      "activations/layer13_attention_weight_min": -36.55925750732422,
      "activations/layer14_attention_weight_max": 84.93793487548828,
      "activations/layer14_attention_weight_min": -57.964881896972656,
      "activations/layer15_attention_weight_max": 54.105045318603516,
      "activations/layer15_attention_weight_min": -36.427677154541016,
      "activations/layer16_attention_weight_max": 55.48631286621094,
      "activations/layer16_attention_weight_min": -36.43306350708008,
      "activations/layer17_attention_weight_max": 44.11686325073242,
      "activations/layer17_attention_weight_min": -27.993722915649414,
      "activations/layer18_attention_weight_max": 32.87712860107422,
      "activations/layer18_attention_weight_min": -18.35592269897461,
      "activations/layer19_attention_weight_max": 52.268585205078125,
      "activations/layer19_attention_weight_min": -31.097192764282227,
      "activations/layer1_attention_weight_max": 16.619403839111328,
      "activations/layer1_attention_weight_min": -11.792009353637695,
      "activations/layer20_attention_weight_max": 40.51604080200195,
      "activations/layer20_attention_weight_min": -24.56414794921875,
      "activations/layer21_attention_weight_max": 35.680965423583984,
      "activations/layer21_attention_weight_min": -20.83515167236328,
      "activations/layer22_attention_weight_max": 45.01594924926758,
      "activations/layer22_attention_weight_min": -27.505144119262695,
      "activations/layer23_attention_weight_max": 25.799924850463867,
      "activations/layer23_attention_weight_min": -16.073837280273438,
      "activations/layer2_attention_weight_max": 25.160053253173828,
      "activations/layer2_attention_weight_min": -25.011701583862305,
      "activations/layer3_attention_weight_max": 62.20183563232422,
      "activations/layer3_attention_weight_min": -67.09757995605469,
      "activations/layer4_attention_weight_max": 112.95977020263672,
      "activations/layer4_attention_weight_min": -106.44721984863281,
      "activations/layer5_attention_weight_max": 107.47381591796875,
      "activations/layer5_attention_weight_min": -92.29705047607422,
      "activations/layer6_attention_weight_max": 58.53106689453125,
      "activations/layer6_attention_weight_min": -50.3675651550293,
      "activations/layer7_attention_weight_max": 66.1523208618164,
      "activations/layer7_attention_weight_min": -60.91927719116211,
      "activations/layer8_attention_weight_max": 45.3034553527832,
      "activations/layer8_attention_weight_min": -37.69280242919922,
      "activations/layer9_attention_weight_max": 56.354286193847656,
      "activations/layer9_attention_weight_min": -50.24128341674805,
      "epoch": 2.33,
      "learning_rate": 0.00013636439393939392,
      "loss": 2.9633,
      "step": 40050
    },
    {
      "activations/layer0_attention_weight_max": 13.770461082458496,
      "activations/layer0_attention_weight_min": -12.18243408203125,
      "activations/layer10_attention_weight_max": 48.21754837036133,
      "activations/layer10_attention_weight_min": -41.37005615234375,
      "activations/layer11_attention_weight_max": 35.933250427246094,
      "activations/layer11_attention_weight_min": -33.60260009765625,
      "activations/layer12_attention_weight_max": 30.761655807495117,
      "activations/layer12_attention_weight_min": -27.17344093322754,
      "activations/layer13_attention_weight_max": 47.47492599487305,
      "activations/layer13_attention_weight_min": -38.586585998535156,
      "activations/layer14_attention_weight_max": 71.51033782958984,
      "activations/layer14_attention_weight_min": -59.48762893676758,
      "activations/layer15_attention_weight_max": 52.48385238647461,
      "activations/layer15_attention_weight_min": -35.82981872558594,
      "activations/layer16_attention_weight_max": 49.61553192138672,
      "activations/layer16_attention_weight_min": -33.2982063293457,
      "activations/layer17_attention_weight_max": 43.996856689453125,
      "activations/layer17_attention_weight_min": -29.212549209594727,
      "activations/layer18_attention_weight_max": 32.430938720703125,
      "activations/layer18_attention_weight_min": -18.038236618041992,
      "activations/layer19_attention_weight_max": 47.3098030090332,
      "activations/layer19_attention_weight_min": -30.216699600219727,
      "activations/layer1_attention_weight_max": 17.826068878173828,
      "activations/layer1_attention_weight_min": -10.89834213256836,
      "activations/layer20_attention_weight_max": 34.83712387084961,
      "activations/layer20_attention_weight_min": -23.197101593017578,
      "activations/layer21_attention_weight_max": 31.94247817993164,
      "activations/layer21_attention_weight_min": -20.444011688232422,
      "activations/layer22_attention_weight_max": 42.60205078125,
      "activations/layer22_attention_weight_min": -26.631359100341797,
      "activations/layer23_attention_weight_max": 25.475255966186523,
      "activations/layer23_attention_weight_min": -15.519335746765137,
      "activations/layer2_attention_weight_max": 24.51654624938965,
      "activations/layer2_attention_weight_min": -23.452655792236328,
      "activations/layer3_attention_weight_max": 60.814571380615234,
      "activations/layer3_attention_weight_min": -65.55154418945312,
      "activations/layer4_attention_weight_max": 115.35958099365234,
      "activations/layer4_attention_weight_min": -111.8835678100586,
      "activations/layer5_attention_weight_max": 98.41893005371094,
      "activations/layer5_attention_weight_min": -90.7843246459961,
      "activations/layer6_attention_weight_max": 54.83457565307617,
      "activations/layer6_attention_weight_min": -48.00666046142578,
      "activations/layer7_attention_weight_max": 63.07041931152344,
      "activations/layer7_attention_weight_min": -62.02184295654297,
      "activations/layer8_attention_weight_max": 43.294700622558594,
      "activations/layer8_attention_weight_min": -38.05278396606445,
      "activations/layer9_attention_weight_max": 56.9400634765625,
      "activations/layer9_attention_weight_min": -46.28915023803711,
      "epoch": 2.33,
      "learning_rate": 0.00013634545454545454,
      "loss": 2.9466,
      "step": 40100
    },
    {
      "activations/layer0_attention_weight_max": 14.862083435058594,
      "activations/layer0_attention_weight_min": -12.089741706848145,
      "activations/layer10_attention_weight_max": 53.46720504760742,
      "activations/layer10_attention_weight_min": -45.68121337890625,
      "activations/layer11_attention_weight_max": 43.728919982910156,
      "activations/layer11_attention_weight_min": -37.4804801940918,
      "activations/layer12_attention_weight_max": 67.55184173583984,
      "activations/layer12_attention_weight_min": -37.17356491088867,
      "activations/layer13_attention_weight_max": 72.142822265625,
      "activations/layer13_attention_weight_min": -48.60765075683594,
      "activations/layer14_attention_weight_max": 103.20714569091797,
      "activations/layer14_attention_weight_min": -68.39945220947266,
      "activations/layer15_attention_weight_max": 57.1583366394043,
      "activations/layer15_attention_weight_min": -39.76423645019531,
      "activations/layer16_attention_weight_max": 44.262123107910156,
      "activations/layer16_attention_weight_min": -32.99281692504883,
      "activations/layer17_attention_weight_max": 48.4598388671875,
      "activations/layer17_attention_weight_min": -32.12017059326172,
      "activations/layer18_attention_weight_max": 34.07333755493164,
      "activations/layer18_attention_weight_min": -19.03290367126465,
      "activations/layer19_attention_weight_max": 53.10317611694336,
      "activations/layer19_attention_weight_min": -31.434906005859375,
      "activations/layer1_attention_weight_max": 16.264175415039062,
      "activations/layer1_attention_weight_min": -11.628033638000488,
      "activations/layer20_attention_weight_max": 36.88239288330078,
      "activations/layer20_attention_weight_min": -23.837078094482422,
      "activations/layer21_attention_weight_max": 31.869619369506836,
      "activations/layer21_attention_weight_min": -19.061071395874023,
      "activations/layer22_attention_weight_max": 56.119476318359375,
      "activations/layer22_attention_weight_min": -30.011640548706055,
      "activations/layer23_attention_weight_max": 25.055715560913086,
      "activations/layer23_attention_weight_min": -16.000106811523438,
      "activations/layer2_attention_weight_max": 27.18014907836914,
      "activations/layer2_attention_weight_min": -24.747833251953125,
      "activations/layer3_attention_weight_max": 68.00647735595703,
      "activations/layer3_attention_weight_min": -70.52821350097656,
      "activations/layer4_attention_weight_max": 123.76423645019531,
      "activations/layer4_attention_weight_min": -122.4427719116211,
      "activations/layer5_attention_weight_max": 109.87939453125,
      "activations/layer5_attention_weight_min": -98.70094299316406,
      "activations/layer6_attention_weight_max": 60.15935516357422,
      "activations/layer6_attention_weight_min": -53.15874481201172,
      "activations/layer7_attention_weight_max": 76.1484146118164,
      "activations/layer7_attention_weight_min": -69.02565002441406,
      "activations/layer8_attention_weight_max": 51.883544921875,
      "activations/layer8_attention_weight_min": -42.82960510253906,
      "activations/layer9_attention_weight_max": 69.85484313964844,
      "activations/layer9_attention_weight_min": -57.17385482788086,
      "epoch": 2.33,
      "learning_rate": 0.00013632651515151515,
      "loss": 2.9445,
      "step": 40150
    },
    {
      "activations/layer0_attention_weight_max": 14.104409217834473,
      "activations/layer0_attention_weight_min": -12.227142333984375,
      "activations/layer10_attention_weight_max": 47.9298095703125,
      "activations/layer10_attention_weight_min": -39.4514274597168,
      "activations/layer11_attention_weight_max": 35.9572639465332,
      "activations/layer11_attention_weight_min": -33.538360595703125,
      "activations/layer12_attention_weight_max": 31.42365837097168,
      "activations/layer12_attention_weight_min": -26.447914123535156,
      "activations/layer13_attention_weight_max": 50.637027740478516,
      "activations/layer13_attention_weight_min": -36.96985626220703,
      "activations/layer14_attention_weight_max": 72.78189086914062,
      "activations/layer14_attention_weight_min": -54.941261291503906,
      "activations/layer15_attention_weight_max": 51.39299011230469,
      "activations/layer15_attention_weight_min": -35.54090881347656,
      "activations/layer16_attention_weight_max": 46.88127517700195,
      "activations/layer16_attention_weight_min": -35.24662780761719,
      "activations/layer17_attention_weight_max": 49.57012939453125,
      "activations/layer17_attention_weight_min": -29.706539154052734,
      "activations/layer18_attention_weight_max": 27.43963623046875,
      "activations/layer18_attention_weight_min": -18.982404708862305,
      "activations/layer19_attention_weight_max": 49.62812423706055,
      "activations/layer19_attention_weight_min": -33.57301330566406,
      "activations/layer1_attention_weight_max": 17.106199264526367,
      "activations/layer1_attention_weight_min": -11.10763168334961,
      "activations/layer20_attention_weight_max": 38.85020446777344,
      "activations/layer20_attention_weight_min": -24.576215744018555,
      "activations/layer21_attention_weight_max": 33.95362854003906,
      "activations/layer21_attention_weight_min": -21.12335777282715,
      "activations/layer22_attention_weight_max": 42.2486686706543,
      "activations/layer22_attention_weight_min": -28.94406509399414,
      "activations/layer23_attention_weight_max": 24.607738494873047,
      "activations/layer23_attention_weight_min": -15.668708801269531,
      "activations/layer2_attention_weight_max": 27.697364807128906,
      "activations/layer2_attention_weight_min": -26.723554611206055,
      "activations/layer3_attention_weight_max": 66.04066467285156,
      "activations/layer3_attention_weight_min": -68.18984985351562,
      "activations/layer4_attention_weight_max": 129.1412353515625,
      "activations/layer4_attention_weight_min": -111.60717010498047,
      "activations/layer5_attention_weight_max": 109.2817611694336,
      "activations/layer5_attention_weight_min": -92.1927719116211,
      "activations/layer6_attention_weight_max": 59.65653991699219,
      "activations/layer6_attention_weight_min": -52.66341781616211,
      "activations/layer7_attention_weight_max": 67.8993911743164,
      "activations/layer7_attention_weight_min": -63.61756134033203,
      "activations/layer8_attention_weight_max": 45.09709548950195,
      "activations/layer8_attention_weight_min": -36.435543060302734,
      "activations/layer9_attention_weight_max": 59.44367599487305,
      "activations/layer9_attention_weight_min": -49.48999786376953,
      "epoch": 2.34,
      "learning_rate": 0.00013630757575757574,
      "loss": 2.953,
      "step": 40200
    },
    {
      "activations/layer0_attention_weight_max": 14.571706771850586,
      "activations/layer0_attention_weight_min": -12.215743064880371,
      "activations/layer10_attention_weight_max": 47.604461669921875,
      "activations/layer10_attention_weight_min": -38.16933059692383,
      "activations/layer11_attention_weight_max": 39.66251754760742,
      "activations/layer11_attention_weight_min": -34.290565490722656,
      "activations/layer12_attention_weight_max": 57.35512161254883,
      "activations/layer12_attention_weight_min": -34.45613098144531,
      "activations/layer13_attention_weight_max": 58.027835845947266,
      "activations/layer13_attention_weight_min": -44.837947845458984,
      "activations/layer14_attention_weight_max": 79.72391510009766,
      "activations/layer14_attention_weight_min": -55.764251708984375,
      "activations/layer15_attention_weight_max": 57.94376754760742,
      "activations/layer15_attention_weight_min": -35.32596206665039,
      "activations/layer16_attention_weight_max": 61.84473419189453,
      "activations/layer16_attention_weight_min": -39.14180374145508,
      "activations/layer17_attention_weight_max": 48.79542922973633,
      "activations/layer17_attention_weight_min": -30.095876693725586,
      "activations/layer18_attention_weight_max": 31.423864364624023,
      "activations/layer18_attention_weight_min": -19.883296966552734,
      "activations/layer19_attention_weight_max": 61.48828887939453,
      "activations/layer19_attention_weight_min": -33.73468780517578,
      "activations/layer1_attention_weight_max": 16.749622344970703,
      "activations/layer1_attention_weight_min": -12.078765869140625,
      "activations/layer20_attention_weight_max": 49.23697280883789,
      "activations/layer20_attention_weight_min": -23.91212272644043,
      "activations/layer21_attention_weight_max": 32.94401550292969,
      "activations/layer21_attention_weight_min": -20.814285278320312,
      "activations/layer22_attention_weight_max": 60.89873504638672,
      "activations/layer22_attention_weight_min": -25.709239959716797,
      "activations/layer23_attention_weight_max": 30.89780044555664,
      "activations/layer23_attention_weight_min": -17.08827018737793,
      "activations/layer2_attention_weight_max": 28.866374969482422,
      "activations/layer2_attention_weight_min": -26.461463928222656,
      "activations/layer3_attention_weight_max": 73.51331329345703,
      "activations/layer3_attention_weight_min": -73.38764953613281,
      "activations/layer4_attention_weight_max": 128.9575653076172,
      "activations/layer4_attention_weight_min": -125.73486328125,
      "activations/layer5_attention_weight_max": 117.88565063476562,
      "activations/layer5_attention_weight_min": -102.40718841552734,
      "activations/layer6_attention_weight_max": 63.1583366394043,
      "activations/layer6_attention_weight_min": -53.332523345947266,
      "activations/layer7_attention_weight_max": 70.02722930908203,
      "activations/layer7_attention_weight_min": -64.19791412353516,
      "activations/layer8_attention_weight_max": 45.22348403930664,
      "activations/layer8_attention_weight_min": -38.10710906982422,
      "activations/layer9_attention_weight_max": 60.91396713256836,
      "activations/layer9_attention_weight_min": -47.711727142333984,
      "epoch": 2.34,
      "learning_rate": 0.00013628863636363635,
      "loss": 2.9417,
      "step": 40250
    },
    {
      "activations/layer0_attention_weight_max": 14.778407096862793,
      "activations/layer0_attention_weight_min": -12.021562576293945,
      "activations/layer10_attention_weight_max": 50.56855392456055,
      "activations/layer10_attention_weight_min": -39.674903869628906,
      "activations/layer11_attention_weight_max": 50.47405242919922,
      "activations/layer11_attention_weight_min": -34.12218475341797,
      "activations/layer12_attention_weight_max": 62.98657989501953,
      "activations/layer12_attention_weight_min": -34.8610725402832,
      "activations/layer13_attention_weight_max": 86.35948181152344,
      "activations/layer13_attention_weight_min": -57.123085021972656,
      "activations/layer14_attention_weight_max": 83.42896270751953,
      "activations/layer14_attention_weight_min": -62.780033111572266,
      "activations/layer15_attention_weight_max": 64.56111907958984,
      "activations/layer15_attention_weight_min": -40.48775863647461,
      "activations/layer16_attention_weight_max": 55.06557083129883,
      "activations/layer16_attention_weight_min": -37.58306884765625,
      "activations/layer17_attention_weight_max": 50.26688766479492,
      "activations/layer17_attention_weight_min": -29.02475357055664,
      "activations/layer18_attention_weight_max": 27.574296951293945,
      "activations/layer18_attention_weight_min": -20.693872451782227,
      "activations/layer19_attention_weight_max": 52.1182861328125,
      "activations/layer19_attention_weight_min": -34.463523864746094,
      "activations/layer1_attention_weight_max": 18.159805297851562,
      "activations/layer1_attention_weight_min": -12.929387092590332,
      "activations/layer20_attention_weight_max": 35.23427963256836,
      "activations/layer20_attention_weight_min": -24.706701278686523,
      "activations/layer21_attention_weight_max": 30.685543060302734,
      "activations/layer21_attention_weight_min": -19.970869064331055,
      "activations/layer22_attention_weight_max": 45.353553771972656,
      "activations/layer22_attention_weight_min": -23.638757705688477,
      "activations/layer23_attention_weight_max": 27.169689178466797,
      "activations/layer23_attention_weight_min": -17.64752960205078,
      "activations/layer2_attention_weight_max": 28.543331146240234,
      "activations/layer2_attention_weight_min": -27.055587768554688,
      "activations/layer3_attention_weight_max": 70.54878997802734,
      "activations/layer3_attention_weight_min": -75.15528106689453,
      "activations/layer4_attention_weight_max": 127.60314178466797,
      "activations/layer4_attention_weight_min": -118.2042236328125,
      "activations/layer5_attention_weight_max": 111.89562225341797,
      "activations/layer5_attention_weight_min": -95.212158203125,
      "activations/layer6_attention_weight_max": 60.554039001464844,
      "activations/layer6_attention_weight_min": -52.637577056884766,
      "activations/layer7_attention_weight_max": 77.09300231933594,
      "activations/layer7_attention_weight_min": -74.03939819335938,
      "activations/layer8_attention_weight_max": 52.29603958129883,
      "activations/layer8_attention_weight_min": -40.65291976928711,
      "activations/layer9_attention_weight_max": 79.56684875488281,
      "activations/layer9_attention_weight_min": -65.04944610595703,
      "epoch": 2.34,
      "learning_rate": 0.00013626969696969694,
      "loss": 2.9432,
      "step": 40300
    },
    {
      "activations/layer0_attention_weight_max": 14.419816970825195,
      "activations/layer0_attention_weight_min": -11.777180671691895,
      "activations/layer10_attention_weight_max": 46.809322357177734,
      "activations/layer10_attention_weight_min": -39.622169494628906,
      "activations/layer11_attention_weight_max": 37.232784271240234,
      "activations/layer11_attention_weight_min": -32.88418960571289,
      "activations/layer12_attention_weight_max": 29.97393035888672,
      "activations/layer12_attention_weight_min": -32.149169921875,
      "activations/layer13_attention_weight_max": 50.884891510009766,
      "activations/layer13_attention_weight_min": -35.87865447998047,
      "activations/layer14_attention_weight_max": 85.39352416992188,
      "activations/layer14_attention_weight_min": -63.99372100830078,
      "activations/layer15_attention_weight_max": 52.74677276611328,
      "activations/layer15_attention_weight_min": -34.583309173583984,
      "activations/layer16_attention_weight_max": 48.36234664916992,
      "activations/layer16_attention_weight_min": -32.45103073120117,
      "activations/layer17_attention_weight_max": 46.549835205078125,
      "activations/layer17_attention_weight_min": -29.82803726196289,
      "activations/layer18_attention_weight_max": 29.656835556030273,
      "activations/layer18_attention_weight_min": -18.944828033447266,
      "activations/layer19_attention_weight_max": 48.948280334472656,
      "activations/layer19_attention_weight_min": -29.565673828125,
      "activations/layer1_attention_weight_max": 16.425350189208984,
      "activations/layer1_attention_weight_min": -11.32908821105957,
      "activations/layer20_attention_weight_max": 40.89215850830078,
      "activations/layer20_attention_weight_min": -25.069995880126953,
      "activations/layer21_attention_weight_max": 33.22187423706055,
      "activations/layer21_attention_weight_min": -19.217792510986328,
      "activations/layer22_attention_weight_max": 50.545040130615234,
      "activations/layer22_attention_weight_min": -25.369327545166016,
      "activations/layer23_attention_weight_max": 27.08527374267578,
      "activations/layer23_attention_weight_min": -17.239591598510742,
      "activations/layer2_attention_weight_max": 27.19977378845215,
      "activations/layer2_attention_weight_min": -26.845794677734375,
      "activations/layer3_attention_weight_max": 65.59634399414062,
      "activations/layer3_attention_weight_min": -71.81771850585938,
      "activations/layer4_attention_weight_max": 121.19078826904297,
      "activations/layer4_attention_weight_min": -112.68965911865234,
      "activations/layer5_attention_weight_max": 103.70195007324219,
      "activations/layer5_attention_weight_min": -92.75118255615234,
      "activations/layer6_attention_weight_max": 59.381927490234375,
      "activations/layer6_attention_weight_min": -51.15959167480469,
      "activations/layer7_attention_weight_max": 69.20314025878906,
      "activations/layer7_attention_weight_min": -62.40863800048828,
      "activations/layer8_attention_weight_max": 43.491539001464844,
      "activations/layer8_attention_weight_min": -36.59359359741211,
      "activations/layer9_attention_weight_max": 67.76444244384766,
      "activations/layer9_attention_weight_min": -52.5780029296875,
      "epoch": 2.34,
      "learning_rate": 0.00013625075757575756,
      "loss": 2.9357,
      "step": 40350
    },
    {
      "activations/layer0_attention_weight_max": 15.340085983276367,
      "activations/layer0_attention_weight_min": -12.045201301574707,
      "activations/layer10_attention_weight_max": 48.179649353027344,
      "activations/layer10_attention_weight_min": -40.76533889770508,
      "activations/layer11_attention_weight_max": 39.620872497558594,
      "activations/layer11_attention_weight_min": -36.71593475341797,
      "activations/layer12_attention_weight_max": 39.207271575927734,
      "activations/layer12_attention_weight_min": -31.812894821166992,
      "activations/layer13_attention_weight_max": 57.199119567871094,
      "activations/layer13_attention_weight_min": -49.989158630371094,
      "activations/layer14_attention_weight_max": 81.41031646728516,
      "activations/layer14_attention_weight_min": -59.0072021484375,
      "activations/layer15_attention_weight_max": 47.4012336730957,
      "activations/layer15_attention_weight_min": -35.335418701171875,
      "activations/layer16_attention_weight_max": 45.13276290893555,
      "activations/layer16_attention_weight_min": -34.33255386352539,
      "activations/layer17_attention_weight_max": 50.18803787231445,
      "activations/layer17_attention_weight_min": -30.669336318969727,
      "activations/layer18_attention_weight_max": 34.87935256958008,
      "activations/layer18_attention_weight_min": -20.455167770385742,
      "activations/layer19_attention_weight_max": 42.75408172607422,
      "activations/layer19_attention_weight_min": -29.232177734375,
      "activations/layer1_attention_weight_max": 16.915735244750977,
      "activations/layer1_attention_weight_min": -10.74835205078125,
      "activations/layer20_attention_weight_max": 31.312788009643555,
      "activations/layer20_attention_weight_min": -23.506179809570312,
      "activations/layer21_attention_weight_max": 28.640348434448242,
      "activations/layer21_attention_weight_min": -18.98957633972168,
      "activations/layer22_attention_weight_max": 43.64841842651367,
      "activations/layer22_attention_weight_min": -29.01270866394043,
      "activations/layer23_attention_weight_max": 24.733007431030273,
      "activations/layer23_attention_weight_min": -15.846630096435547,
      "activations/layer2_attention_weight_max": 27.9787654876709,
      "activations/layer2_attention_weight_min": -28.107824325561523,
      "activations/layer3_attention_weight_max": 70.7810287475586,
      "activations/layer3_attention_weight_min": -76.52564239501953,
      "activations/layer4_attention_weight_max": 123.61114501953125,
      "activations/layer4_attention_weight_min": -118.67015838623047,
      "activations/layer5_attention_weight_max": 109.75376892089844,
      "activations/layer5_attention_weight_min": -95.55315399169922,
      "activations/layer6_attention_weight_max": 58.38850784301758,
      "activations/layer6_attention_weight_min": -53.575931549072266,
      "activations/layer7_attention_weight_max": 70.52696228027344,
      "activations/layer7_attention_weight_min": -67.2099609375,
      "activations/layer8_attention_weight_max": 43.37917709350586,
      "activations/layer8_attention_weight_min": -37.93550491333008,
      "activations/layer9_attention_weight_max": 57.1629524230957,
      "activations/layer9_attention_weight_min": -52.833370208740234,
      "epoch": 2.35,
      "learning_rate": 0.00013623181818181817,
      "loss": 2.9348,
      "step": 40400
    },
    {
      "activations/layer0_attention_weight_max": 14.267133712768555,
      "activations/layer0_attention_weight_min": -11.642373085021973,
      "activations/layer10_attention_weight_max": 51.61552429199219,
      "activations/layer10_attention_weight_min": -40.27852249145508,
      "activations/layer11_attention_weight_max": 51.025299072265625,
      "activations/layer11_attention_weight_min": -35.44364929199219,
      "activations/layer12_attention_weight_max": 39.93737030029297,
      "activations/layer12_attention_weight_min": -31.932880401611328,
      "activations/layer13_attention_weight_max": 50.687965393066406,
      "activations/layer13_attention_weight_min": -38.397457122802734,
      "activations/layer14_attention_weight_max": 74.97119903564453,
      "activations/layer14_attention_weight_min": -54.523563385009766,
      "activations/layer15_attention_weight_max": 55.29910659790039,
      "activations/layer15_attention_weight_min": -37.68515396118164,
      "activations/layer16_attention_weight_max": 54.367549896240234,
      "activations/layer16_attention_weight_min": -33.7100944519043,
      "activations/layer17_attention_weight_max": 43.084266662597656,
      "activations/layer17_attention_weight_min": -29.342628479003906,
      "activations/layer18_attention_weight_max": 29.38688850402832,
      "activations/layer18_attention_weight_min": -19.004623413085938,
      "activations/layer19_attention_weight_max": 49.61385726928711,
      "activations/layer19_attention_weight_min": -32.4299201965332,
      "activations/layer1_attention_weight_max": 18.532608032226562,
      "activations/layer1_attention_weight_min": -10.753528594970703,
      "activations/layer20_attention_weight_max": 43.13010025024414,
      "activations/layer20_attention_weight_min": -25.424549102783203,
      "activations/layer21_attention_weight_max": 32.6031379699707,
      "activations/layer21_attention_weight_min": -23.226581573486328,
      "activations/layer22_attention_weight_max": 48.14329147338867,
      "activations/layer22_attention_weight_min": -27.40117073059082,
      "activations/layer23_attention_weight_max": 27.782880783081055,
      "activations/layer23_attention_weight_min": -16.57478904724121,
      "activations/layer2_attention_weight_max": 28.65274429321289,
      "activations/layer2_attention_weight_min": -27.646385192871094,
      "activations/layer3_attention_weight_max": 71.0060043334961,
      "activations/layer3_attention_weight_min": -74.83740997314453,
      "activations/layer4_attention_weight_max": 126.6334228515625,
      "activations/layer4_attention_weight_min": -117.89009857177734,
      "activations/layer5_attention_weight_max": 111.40382385253906,
      "activations/layer5_attention_weight_min": -96.03728485107422,
      "activations/layer6_attention_weight_max": 62.887489318847656,
      "activations/layer6_attention_weight_min": -53.815677642822266,
      "activations/layer7_attention_weight_max": 76.89317321777344,
      "activations/layer7_attention_weight_min": -69.51226806640625,
      "activations/layer8_attention_weight_max": 47.616966247558594,
      "activations/layer8_attention_weight_min": -39.576751708984375,
      "activations/layer9_attention_weight_max": 67.38793182373047,
      "activations/layer9_attention_weight_min": -54.99494552612305,
      "epoch": 2.35,
      "learning_rate": 0.00013621287878787876,
      "loss": 2.9366,
      "step": 40450
    },
    {
      "activations/layer0_attention_weight_max": 14.969430923461914,
      "activations/layer0_attention_weight_min": -11.66146183013916,
      "activations/layer10_attention_weight_max": 48.3565559387207,
      "activations/layer10_attention_weight_min": -38.862281799316406,
      "activations/layer11_attention_weight_max": 37.83171081542969,
      "activations/layer11_attention_weight_min": -32.56969451904297,
      "activations/layer12_attention_weight_max": 40.235023498535156,
      "activations/layer12_attention_weight_min": -27.084712982177734,
      "activations/layer13_attention_weight_max": 49.62349319458008,
      "activations/layer13_attention_weight_min": -37.02516555786133,
      "activations/layer14_attention_weight_max": 84.87921905517578,
      "activations/layer14_attention_weight_min": -60.06722640991211,
      "activations/layer15_attention_weight_max": 53.94160461425781,
      "activations/layer15_attention_weight_min": -36.050140380859375,
      "activations/layer16_attention_weight_max": 46.478614807128906,
      "activations/layer16_attention_weight_min": -35.083309173583984,
      "activations/layer17_attention_weight_max": 44.10329055786133,
      "activations/layer17_attention_weight_min": -28.7726993560791,
      "activations/layer18_attention_weight_max": 29.930850982666016,
      "activations/layer18_attention_weight_min": -20.10610008239746,
      "activations/layer19_attention_weight_max": 47.50201416015625,
      "activations/layer19_attention_weight_min": -30.10040855407715,
      "activations/layer1_attention_weight_max": 16.76268196105957,
      "activations/layer1_attention_weight_min": -12.74975872039795,
      "activations/layer20_attention_weight_max": 39.61796569824219,
      "activations/layer20_attention_weight_min": -26.27467918395996,
      "activations/layer21_attention_weight_max": 41.383277893066406,
      "activations/layer21_attention_weight_min": -21.3397159576416,
      "activations/layer22_attention_weight_max": 44.348514556884766,
      "activations/layer22_attention_weight_min": -27.0145320892334,
      "activations/layer23_attention_weight_max": 26.32288932800293,
      "activations/layer23_attention_weight_min": -16.086170196533203,
      "activations/layer2_attention_weight_max": 27.22418212890625,
      "activations/layer2_attention_weight_min": -26.404081344604492,
      "activations/layer3_attention_weight_max": 69.45375061035156,
      "activations/layer3_attention_weight_min": -70.75057983398438,
      "activations/layer4_attention_weight_max": 124.62611389160156,
      "activations/layer4_attention_weight_min": -114.34659576416016,
      "activations/layer5_attention_weight_max": 104.17874145507812,
      "activations/layer5_attention_weight_min": -93.336181640625,
      "activations/layer6_attention_weight_max": 57.32431411743164,
      "activations/layer6_attention_weight_min": -50.7636833190918,
      "activations/layer7_attention_weight_max": 65.12579345703125,
      "activations/layer7_attention_weight_min": -62.375919342041016,
      "activations/layer8_attention_weight_max": 44.01892852783203,
      "activations/layer8_attention_weight_min": -35.84467697143555,
      "activations/layer9_attention_weight_max": 55.80137252807617,
      "activations/layer9_attention_weight_min": -49.6159553527832,
      "epoch": 2.35,
      "learning_rate": 0.00013619393939393937,
      "loss": 2.9211,
      "step": 40500
    },
    {
      "activations/layer0_attention_weight_max": 14.78502082824707,
      "activations/layer0_attention_weight_min": -12.10478401184082,
      "activations/layer10_attention_weight_max": 58.483673095703125,
      "activations/layer10_attention_weight_min": -43.0945930480957,
      "activations/layer11_attention_weight_max": 58.32821273803711,
      "activations/layer11_attention_weight_min": -43.229820251464844,
      "activations/layer12_attention_weight_max": 72.95565032958984,
      "activations/layer12_attention_weight_min": -42.37641143798828,
      "activations/layer13_attention_weight_max": 72.99120330810547,
      "activations/layer13_attention_weight_min": -49.837181091308594,
      "activations/layer14_attention_weight_max": 76.57743835449219,
      "activations/layer14_attention_weight_min": -60.48428726196289,
      "activations/layer15_attention_weight_max": 55.96684265136719,
      "activations/layer15_attention_weight_min": -37.673030853271484,
      "activations/layer16_attention_weight_max": 49.84836196899414,
      "activations/layer16_attention_weight_min": -34.69063186645508,
      "activations/layer17_attention_weight_max": 47.08984375,
      "activations/layer17_attention_weight_min": -26.56787109375,
      "activations/layer18_attention_weight_max": 29.468074798583984,
      "activations/layer18_attention_weight_min": -20.081966400146484,
      "activations/layer19_attention_weight_max": 47.53409957885742,
      "activations/layer19_attention_weight_min": -29.989042282104492,
      "activations/layer1_attention_weight_max": 17.547882080078125,
      "activations/layer1_attention_weight_min": -11.196389198303223,
      "activations/layer20_attention_weight_max": 36.126277923583984,
      "activations/layer20_attention_weight_min": -24.269365310668945,
      "activations/layer21_attention_weight_max": 27.778316497802734,
      "activations/layer21_attention_weight_min": -20.98702049255371,
      "activations/layer22_attention_weight_max": 48.82830047607422,
      "activations/layer22_attention_weight_min": -29.67866325378418,
      "activations/layer23_attention_weight_max": 25.303068161010742,
      "activations/layer23_attention_weight_min": -15.259379386901855,
      "activations/layer2_attention_weight_max": 29.48762321472168,
      "activations/layer2_attention_weight_min": -29.311866760253906,
      "activations/layer3_attention_weight_max": 72.6889419555664,
      "activations/layer3_attention_weight_min": -75.11346435546875,
      "activations/layer4_attention_weight_max": 124.77912902832031,
      "activations/layer4_attention_weight_min": -114.48921203613281,
      "activations/layer5_attention_weight_max": 108.73684692382812,
      "activations/layer5_attention_weight_min": -93.96377563476562,
      "activations/layer6_attention_weight_max": 60.966819763183594,
      "activations/layer6_attention_weight_min": -53.07077407836914,
      "activations/layer7_attention_weight_max": 78.61534118652344,
      "activations/layer7_attention_weight_min": -64.24711608886719,
      "activations/layer8_attention_weight_max": 53.74225997924805,
      "activations/layer8_attention_weight_min": -38.15922927856445,
      "activations/layer9_attention_weight_max": 91.05086517333984,
      "activations/layer9_attention_weight_min": -61.933319091796875,
      "epoch": 2.36,
      "learning_rate": 0.000136175,
      "loss": 2.9268,
      "step": 40550
    },
    {
      "activations/layer0_attention_weight_max": 15.238401412963867,
      "activations/layer0_attention_weight_min": -11.52016544342041,
      "activations/layer10_attention_weight_max": 67.77386474609375,
      "activations/layer10_attention_weight_min": -46.026546478271484,
      "activations/layer11_attention_weight_max": 67.17646789550781,
      "activations/layer11_attention_weight_min": -43.01070785522461,
      "activations/layer12_attention_weight_max": 76.66255950927734,
      "activations/layer12_attention_weight_min": -34.70518112182617,
      "activations/layer13_attention_weight_max": 72.96715545654297,
      "activations/layer13_attention_weight_min": -45.5571174621582,
      "activations/layer14_attention_weight_max": 87.81693267822266,
      "activations/layer14_attention_weight_min": -58.02507400512695,
      "activations/layer15_attention_weight_max": 59.445308685302734,
      "activations/layer15_attention_weight_min": -34.59206008911133,
      "activations/layer16_attention_weight_max": 52.57421875,
      "activations/layer16_attention_weight_min": -33.71100997924805,
      "activations/layer17_attention_weight_max": 50.206600189208984,
      "activations/layer17_attention_weight_min": -28.607309341430664,
      "activations/layer18_attention_weight_max": 28.8421688079834,
      "activations/layer18_attention_weight_min": -18.60854148864746,
      "activations/layer19_attention_weight_max": 57.227657318115234,
      "activations/layer19_attention_weight_min": -31.326791763305664,
      "activations/layer1_attention_weight_max": 21.30988883972168,
      "activations/layer1_attention_weight_min": -11.151734352111816,
      "activations/layer20_attention_weight_max": 43.16456985473633,
      "activations/layer20_attention_weight_min": -26.16991424560547,
      "activations/layer21_attention_weight_max": 32.63934326171875,
      "activations/layer21_attention_weight_min": -22.3511905670166,
      "activations/layer22_attention_weight_max": 48.549766540527344,
      "activations/layer22_attention_weight_min": -25.108842849731445,
      "activations/layer23_attention_weight_max": 25.66647720336914,
      "activations/layer23_attention_weight_min": -14.212779998779297,
      "activations/layer2_attention_weight_max": 29.131345748901367,
      "activations/layer2_attention_weight_min": -28.416969299316406,
      "activations/layer3_attention_weight_max": 72.00210571289062,
      "activations/layer3_attention_weight_min": -76.68771362304688,
      "activations/layer4_attention_weight_max": 123.6806640625,
      "activations/layer4_attention_weight_min": -122.47721099853516,
      "activations/layer5_attention_weight_max": 112.12492370605469,
      "activations/layer5_attention_weight_min": -103.65948486328125,
      "activations/layer6_attention_weight_max": 64.03189086914062,
      "activations/layer6_attention_weight_min": -55.15899658203125,
      "activations/layer7_attention_weight_max": 82.48284149169922,
      "activations/layer7_attention_weight_min": -68.67910766601562,
      "activations/layer8_attention_weight_max": 54.867210388183594,
      "activations/layer8_attention_weight_min": -40.167232513427734,
      "activations/layer9_attention_weight_max": 101.89952850341797,
      "activations/layer9_attention_weight_min": -74.71636962890625,
      "epoch": 2.36,
      "learning_rate": 0.00013615606060606058,
      "loss": 2.9359,
      "step": 40600
    },
    {
      "activations/layer0_attention_weight_max": 15.49630069732666,
      "activations/layer0_attention_weight_min": -11.809428215026855,
      "activations/layer10_attention_weight_max": 42.8172607421875,
      "activations/layer10_attention_weight_min": -38.44635009765625,
      "activations/layer11_attention_weight_max": 36.493499755859375,
      "activations/layer11_attention_weight_min": -32.00508117675781,
      "activations/layer12_attention_weight_max": 67.07915496826172,
      "activations/layer12_attention_weight_min": -33.61888122558594,
      "activations/layer13_attention_weight_max": 55.26482391357422,
      "activations/layer13_attention_weight_min": -38.46245574951172,
      "activations/layer14_attention_weight_max": 80.39340209960938,
      "activations/layer14_attention_weight_min": -66.40648651123047,
      "activations/layer15_attention_weight_max": 48.68619155883789,
      "activations/layer15_attention_weight_min": -37.020263671875,
      "activations/layer16_attention_weight_max": 44.12584686279297,
      "activations/layer16_attention_weight_min": -33.81210708618164,
      "activations/layer17_attention_weight_max": 44.061737060546875,
      "activations/layer17_attention_weight_min": -31.755868911743164,
      "activations/layer18_attention_weight_max": 28.747581481933594,
      "activations/layer18_attention_weight_min": -20.335100173950195,
      "activations/layer19_attention_weight_max": 43.935699462890625,
      "activations/layer19_attention_weight_min": -30.979022979736328,
      "activations/layer1_attention_weight_max": 17.06624412536621,
      "activations/layer1_attention_weight_min": -11.240621566772461,
      "activations/layer20_attention_weight_max": 34.81871032714844,
      "activations/layer20_attention_weight_min": -24.733884811401367,
      "activations/layer21_attention_weight_max": 30.203824996948242,
      "activations/layer21_attention_weight_min": -21.199848175048828,
      "activations/layer22_attention_weight_max": 42.43704605102539,
      "activations/layer22_attention_weight_min": -25.828399658203125,
      "activations/layer23_attention_weight_max": 26.052284240722656,
      "activations/layer23_attention_weight_min": -15.898112297058105,
      "activations/layer2_attention_weight_max": 28.915403366088867,
      "activations/layer2_attention_weight_min": -27.275283813476562,
      "activations/layer3_attention_weight_max": 70.98796844482422,
      "activations/layer3_attention_weight_min": -74.9128646850586,
      "activations/layer4_attention_weight_max": 119.23839569091797,
      "activations/layer4_attention_weight_min": -115.67288970947266,
      "activations/layer5_attention_weight_max": 103.62093353271484,
      "activations/layer5_attention_weight_min": -93.72225952148438,
      "activations/layer6_attention_weight_max": 55.19488525390625,
      "activations/layer6_attention_weight_min": -52.0357780456543,
      "activations/layer7_attention_weight_max": 65.1010513305664,
      "activations/layer7_attention_weight_min": -65.2571029663086,
      "activations/layer8_attention_weight_max": 39.857017517089844,
      "activations/layer8_attention_weight_min": -35.13880920410156,
      "activations/layer9_attention_weight_max": 61.042457580566406,
      "activations/layer9_attention_weight_min": -49.61453628540039,
      "epoch": 2.36,
      "learning_rate": 0.0001361371212121212,
      "loss": 2.9404,
      "step": 40650
    },
    {
      "activations/layer0_attention_weight_max": 15.904703140258789,
      "activations/layer0_attention_weight_min": -11.673957824707031,
      "activations/layer10_attention_weight_max": 48.30381393432617,
      "activations/layer10_attention_weight_min": -42.56861877441406,
      "activations/layer11_attention_weight_max": 36.04667663574219,
      "activations/layer11_attention_weight_min": -33.90815353393555,
      "activations/layer12_attention_weight_max": 28.671411514282227,
      "activations/layer12_attention_weight_min": -32.43547821044922,
      "activations/layer13_attention_weight_max": 51.48368835449219,
      "activations/layer13_attention_weight_min": -37.65962219238281,
      "activations/layer14_attention_weight_max": 76.14417266845703,
      "activations/layer14_attention_weight_min": -53.940185546875,
      "activations/layer15_attention_weight_max": 51.362274169921875,
      "activations/layer15_attention_weight_min": -34.02379608154297,
      "activations/layer16_attention_weight_max": 49.19626235961914,
      "activations/layer16_attention_weight_min": -31.59971046447754,
      "activations/layer17_attention_weight_max": 44.995887756347656,
      "activations/layer17_attention_weight_min": -29.868974685668945,
      "activations/layer18_attention_weight_max": 30.311532974243164,
      "activations/layer18_attention_weight_min": -17.88749122619629,
      "activations/layer19_attention_weight_max": 53.56010055541992,
      "activations/layer19_attention_weight_min": -33.12895202636719,
      "activations/layer1_attention_weight_max": 16.658105850219727,
      "activations/layer1_attention_weight_min": -12.531061172485352,
      "activations/layer20_attention_weight_max": 43.122886657714844,
      "activations/layer20_attention_weight_min": -22.38414192199707,
      "activations/layer21_attention_weight_max": 41.52040481567383,
      "activations/layer21_attention_weight_min": -18.004362106323242,
      "activations/layer22_attention_weight_max": 55.425331115722656,
      "activations/layer22_attention_weight_min": -26.316078186035156,
      "activations/layer23_attention_weight_max": 27.22523307800293,
      "activations/layer23_attention_weight_min": -15.551046371459961,
      "activations/layer2_attention_weight_max": 27.797882080078125,
      "activations/layer2_attention_weight_min": -27.34088897705078,
      "activations/layer3_attention_weight_max": 70.27828216552734,
      "activations/layer3_attention_weight_min": -74.36571502685547,
      "activations/layer4_attention_weight_max": 124.76356506347656,
      "activations/layer4_attention_weight_min": -116.677001953125,
      "activations/layer5_attention_weight_max": 108.76043701171875,
      "activations/layer5_attention_weight_min": -98.149169921875,
      "activations/layer6_attention_weight_max": 56.87498092651367,
      "activations/layer6_attention_weight_min": -53.30007553100586,
      "activations/layer7_attention_weight_max": 67.08900451660156,
      "activations/layer7_attention_weight_min": -63.50667953491211,
      "activations/layer8_attention_weight_max": 41.611026763916016,
      "activations/layer8_attention_weight_min": -36.98610305786133,
      "activations/layer9_attention_weight_max": 61.69476318359375,
      "activations/layer9_attention_weight_min": -49.0422477722168,
      "epoch": 2.36,
      "learning_rate": 0.0001361181818181818,
      "loss": 2.9497,
      "step": 40700
    },
    {
      "activations/layer0_attention_weight_max": 15.357137680053711,
      "activations/layer0_attention_weight_min": -11.471652030944824,
      "activations/layer10_attention_weight_max": 49.699607849121094,
      "activations/layer10_attention_weight_min": -37.74703598022461,
      "activations/layer11_attention_weight_max": 38.318603515625,
      "activations/layer11_attention_weight_min": -33.373294830322266,
      "activations/layer12_attention_weight_max": 30.9851016998291,
      "activations/layer12_attention_weight_min": -25.044269561767578,
      "activations/layer13_attention_weight_max": 61.682498931884766,
      "activations/layer13_attention_weight_min": -48.80990219116211,
      "activations/layer14_attention_weight_max": 75.1447982788086,
      "activations/layer14_attention_weight_min": -56.37687301635742,
      "activations/layer15_attention_weight_max": 52.94648742675781,
      "activations/layer15_attention_weight_min": -36.27165603637695,
      "activations/layer16_attention_weight_max": 46.1809196472168,
      "activations/layer16_attention_weight_min": -36.900699615478516,
      "activations/layer17_attention_weight_max": 47.212772369384766,
      "activations/layer17_attention_weight_min": -27.761667251586914,
      "activations/layer18_attention_weight_max": 32.67844772338867,
      "activations/layer18_attention_weight_min": -18.301143646240234,
      "activations/layer19_attention_weight_max": 45.35102462768555,
      "activations/layer19_attention_weight_min": -32.05437088012695,
      "activations/layer1_attention_weight_max": 17.75446891784668,
      "activations/layer1_attention_weight_min": -11.8265380859375,
      "activations/layer20_attention_weight_max": 37.20570755004883,
      "activations/layer20_attention_weight_min": -24.250106811523438,
      "activations/layer21_attention_weight_max": 37.88837814331055,
      "activations/layer21_attention_weight_min": -18.808034896850586,
      "activations/layer22_attention_weight_max": 44.60520553588867,
      "activations/layer22_attention_weight_min": -24.42903709411621,
      "activations/layer23_attention_weight_max": 23.370107650756836,
      "activations/layer23_attention_weight_min": -17.51670265197754,
      "activations/layer2_attention_weight_max": 28.810422897338867,
      "activations/layer2_attention_weight_min": -27.45232391357422,
      "activations/layer3_attention_weight_max": 74.01866149902344,
      "activations/layer3_attention_weight_min": -74.37344360351562,
      "activations/layer4_attention_weight_max": 131.74404907226562,
      "activations/layer4_attention_weight_min": -120.13215637207031,
      "activations/layer5_attention_weight_max": 110.44325256347656,
      "activations/layer5_attention_weight_min": -95.1339340209961,
      "activations/layer6_attention_weight_max": 63.797584533691406,
      "activations/layer6_attention_weight_min": -56.43179702758789,
      "activations/layer7_attention_weight_max": 69.54598999023438,
      "activations/layer7_attention_weight_min": -64.22773742675781,
      "activations/layer8_attention_weight_max": 44.967010498046875,
      "activations/layer8_attention_weight_min": -36.4438362121582,
      "activations/layer9_attention_weight_max": 69.3792953491211,
      "activations/layer9_attention_weight_min": -50.27986145019531,
      "epoch": 2.37,
      "learning_rate": 0.00013609924242424242,
      "loss": 2.9468,
      "step": 40750
    },
    {
      "activations/layer0_attention_weight_max": 15.368637084960938,
      "activations/layer0_attention_weight_min": -11.778321266174316,
      "activations/layer10_attention_weight_max": 50.07338333129883,
      "activations/layer10_attention_weight_min": -36.143280029296875,
      "activations/layer11_attention_weight_max": 39.256954193115234,
      "activations/layer11_attention_weight_min": -31.338537216186523,
      "activations/layer12_attention_weight_max": 32.99006652832031,
      "activations/layer12_attention_weight_min": -24.145832061767578,
      "activations/layer13_attention_weight_max": 47.06755828857422,
      "activations/layer13_attention_weight_min": -35.88199234008789,
      "activations/layer14_attention_weight_max": 72.1821060180664,
      "activations/layer14_attention_weight_min": -55.72744369506836,
      "activations/layer15_attention_weight_max": 53.34998321533203,
      "activations/layer15_attention_weight_min": -37.88702392578125,
      "activations/layer16_attention_weight_max": 55.48075866699219,
      "activations/layer16_attention_weight_min": -33.97881317138672,
      "activations/layer17_attention_weight_max": 41.226844787597656,
      "activations/layer17_attention_weight_min": -31.16440773010254,
      "activations/layer18_attention_weight_max": 26.29825782775879,
      "activations/layer18_attention_weight_min": -19.68341636657715,
      "activations/layer19_attention_weight_max": 53.783992767333984,
      "activations/layer19_attention_weight_min": -31.3172664642334,
      "activations/layer1_attention_weight_max": 17.042211532592773,
      "activations/layer1_attention_weight_min": -11.624367713928223,
      "activations/layer20_attention_weight_max": 36.68356704711914,
      "activations/layer20_attention_weight_min": -24.72985076904297,
      "activations/layer21_attention_weight_max": 30.814359664916992,
      "activations/layer21_attention_weight_min": -22.648765563964844,
      "activations/layer22_attention_weight_max": 44.80088424682617,
      "activations/layer22_attention_weight_min": -26.891775131225586,
      "activations/layer23_attention_weight_max": 23.679950714111328,
      "activations/layer23_attention_weight_min": -15.156737327575684,
      "activations/layer2_attention_weight_max": 28.266666412353516,
      "activations/layer2_attention_weight_min": -29.00231170654297,
      "activations/layer3_attention_weight_max": 69.05647277832031,
      "activations/layer3_attention_weight_min": -76.1531982421875,
      "activations/layer4_attention_weight_max": 124.86608123779297,
      "activations/layer4_attention_weight_min": -114.96134185791016,
      "activations/layer5_attention_weight_max": 104.78917694091797,
      "activations/layer5_attention_weight_min": -91.3296127319336,
      "activations/layer6_attention_weight_max": 56.77680587768555,
      "activations/layer6_attention_weight_min": -48.9471321105957,
      "activations/layer7_attention_weight_max": 66.81044006347656,
      "activations/layer7_attention_weight_min": -62.34632873535156,
      "activations/layer8_attention_weight_max": 45.599998474121094,
      "activations/layer8_attention_weight_min": -36.19859313964844,
      "activations/layer9_attention_weight_max": 55.5727653503418,
      "activations/layer9_attention_weight_min": -47.113033294677734,
      "epoch": 2.37,
      "learning_rate": 0.000136080303030303,
      "loss": 2.9113,
      "step": 40800
    },
    {
      "activations/layer0_attention_weight_max": 16.156230926513672,
      "activations/layer0_attention_weight_min": -11.825329780578613,
      "activations/layer10_attention_weight_max": 50.628021240234375,
      "activations/layer10_attention_weight_min": -39.344329833984375,
      "activations/layer11_attention_weight_max": 43.92070007324219,
      "activations/layer11_attention_weight_min": -31.774242401123047,
      "activations/layer12_attention_weight_max": 64.49433898925781,
      "activations/layer12_attention_weight_min": -35.958858489990234,
      "activations/layer13_attention_weight_max": 86.32939147949219,
      "activations/layer13_attention_weight_min": -53.84679412841797,
      "activations/layer14_attention_weight_max": 140.59068298339844,
      "activations/layer14_attention_weight_min": -84.84037017822266,
      "activations/layer15_attention_weight_max": 60.72244644165039,
      "activations/layer15_attention_weight_min": -36.53597640991211,
      "activations/layer16_attention_weight_max": 55.281368255615234,
      "activations/layer16_attention_weight_min": -32.75751495361328,
      "activations/layer17_attention_weight_max": 49.242637634277344,
      "activations/layer17_attention_weight_min": -27.797622680664062,
      "activations/layer18_attention_weight_max": 32.3125114440918,
      "activations/layer18_attention_weight_min": -18.497787475585938,
      "activations/layer19_attention_weight_max": 67.79759979248047,
      "activations/layer19_attention_weight_min": -32.885231018066406,
      "activations/layer1_attention_weight_max": 18.086559295654297,
      "activations/layer1_attention_weight_min": -11.985383987426758,
      "activations/layer20_attention_weight_max": 49.59675216674805,
      "activations/layer20_attention_weight_min": -24.215896606445312,
      "activations/layer21_attention_weight_max": 28.810941696166992,
      "activations/layer21_attention_weight_min": -19.285663604736328,
      "activations/layer22_attention_weight_max": 54.840763092041016,
      "activations/layer22_attention_weight_min": -26.280651092529297,
      "activations/layer23_attention_weight_max": 32.702659606933594,
      "activations/layer23_attention_weight_min": -17.470979690551758,
      "activations/layer2_attention_weight_max": 30.184906005859375,
      "activations/layer2_attention_weight_min": -28.771896362304688,
      "activations/layer3_attention_weight_max": 73.69937896728516,
      "activations/layer3_attention_weight_min": -79.10023498535156,
      "activations/layer4_attention_weight_max": 137.90281677246094,
      "activations/layer4_attention_weight_min": -123.96893310546875,
      "activations/layer5_attention_weight_max": 128.76083374023438,
      "activations/layer5_attention_weight_min": -102.52601623535156,
      "activations/layer6_attention_weight_max": 68.31718444824219,
      "activations/layer6_attention_weight_min": -59.173736572265625,
      "activations/layer7_attention_weight_max": 75.77091217041016,
      "activations/layer7_attention_weight_min": -72.30432891845703,
      "activations/layer8_attention_weight_max": 51.479007720947266,
      "activations/layer8_attention_weight_min": -42.49965286254883,
      "activations/layer9_attention_weight_max": 79.3330307006836,
      "activations/layer9_attention_weight_min": -67.7461166381836,
      "epoch": 2.37,
      "learning_rate": 0.00013606136363636363,
      "loss": 2.9504,
      "step": 40850
    },
    {
      "activations/layer0_attention_weight_max": 16.439716339111328,
      "activations/layer0_attention_weight_min": -11.85767650604248,
      "activations/layer10_attention_weight_max": 49.37455368041992,
      "activations/layer10_attention_weight_min": -37.180458068847656,
      "activations/layer11_attention_weight_max": 37.519290924072266,
      "activations/layer11_attention_weight_min": -30.758291244506836,
      "activations/layer12_attention_weight_max": 41.49276351928711,
      "activations/layer12_attention_weight_min": -26.853670120239258,
      "activations/layer13_attention_weight_max": 45.54237365722656,
      "activations/layer13_attention_weight_min": -38.46564865112305,
      "activations/layer14_attention_weight_max": 72.32169342041016,
      "activations/layer14_attention_weight_min": -52.36355972290039,
      "activations/layer15_attention_weight_max": 51.984920501708984,
      "activations/layer15_attention_weight_min": -40.85482406616211,
      "activations/layer16_attention_weight_max": 41.40571212768555,
      "activations/layer16_attention_weight_min": -32.33892822265625,
      "activations/layer17_attention_weight_max": 43.220523834228516,
      "activations/layer17_attention_weight_min": -30.743207931518555,
      "activations/layer18_attention_weight_max": 29.893192291259766,
      "activations/layer18_attention_weight_min": -18.55259895324707,
      "activations/layer19_attention_weight_max": 47.83599853515625,
      "activations/layer19_attention_weight_min": -29.403818130493164,
      "activations/layer1_attention_weight_max": 16.987192153930664,
      "activations/layer1_attention_weight_min": -11.328438758850098,
      "activations/layer20_attention_weight_max": 38.68060302734375,
      "activations/layer20_attention_weight_min": -25.373750686645508,
      "activations/layer21_attention_weight_max": 27.091562271118164,
      "activations/layer21_attention_weight_min": -24.300765991210938,
      "activations/layer22_attention_weight_max": 44.33738327026367,
      "activations/layer22_attention_weight_min": -28.4943904876709,
      "activations/layer23_attention_weight_max": 24.119131088256836,
      "activations/layer23_attention_weight_min": -18.387805938720703,
      "activations/layer2_attention_weight_max": 29.6757869720459,
      "activations/layer2_attention_weight_min": -27.966672897338867,
      "activations/layer3_attention_weight_max": 70.88462829589844,
      "activations/layer3_attention_weight_min": -72.91838836669922,
      "activations/layer4_attention_weight_max": 125.74005126953125,
      "activations/layer4_attention_weight_min": -115.57280731201172,
      "activations/layer5_attention_weight_max": 107.65145874023438,
      "activations/layer5_attention_weight_min": -95.14265441894531,
      "activations/layer6_attention_weight_max": 59.22391891479492,
      "activations/layer6_attention_weight_min": -52.64264678955078,
      "activations/layer7_attention_weight_max": 64.74835205078125,
      "activations/layer7_attention_weight_min": -64.68313598632812,
      "activations/layer8_attention_weight_max": 45.679290771484375,
      "activations/layer8_attention_weight_min": -38.66303634643555,
      "activations/layer9_attention_weight_max": 60.838191986083984,
      "activations/layer9_attention_weight_min": -44.55378341674805,
      "epoch": 2.38,
      "learning_rate": 0.00013604242424242424,
      "loss": 2.9386,
      "step": 40900
    },
    {
      "activations/layer0_attention_weight_max": 15.86713695526123,
      "activations/layer0_attention_weight_min": -11.498885154724121,
      "activations/layer10_attention_weight_max": 48.55490493774414,
      "activations/layer10_attention_weight_min": -40.248016357421875,
      "activations/layer11_attention_weight_max": 41.699302673339844,
      "activations/layer11_attention_weight_min": -33.25045394897461,
      "activations/layer12_attention_weight_max": 68.2327880859375,
      "activations/layer12_attention_weight_min": -37.48434829711914,
      "activations/layer13_attention_weight_max": 56.00421905517578,
      "activations/layer13_attention_weight_min": -39.49800491333008,
      "activations/layer14_attention_weight_max": 68.04583740234375,
      "activations/layer14_attention_weight_min": -54.37370300292969,
      "activations/layer15_attention_weight_max": 53.306148529052734,
      "activations/layer15_attention_weight_min": -38.460880279541016,
      "activations/layer16_attention_weight_max": 45.49812316894531,
      "activations/layer16_attention_weight_min": -36.58547592163086,
      "activations/layer17_attention_weight_max": 39.052459716796875,
      "activations/layer17_attention_weight_min": -27.514728546142578,
      "activations/layer18_attention_weight_max": 28.07203483581543,
      "activations/layer18_attention_weight_min": -17.420686721801758,
      "activations/layer19_attention_weight_max": 47.73844528198242,
      "activations/layer19_attention_weight_min": -31.851972579956055,
      "activations/layer1_attention_weight_max": 18.255308151245117,
      "activations/layer1_attention_weight_min": -10.99641227722168,
      "activations/layer20_attention_weight_max": 39.119266510009766,
      "activations/layer20_attention_weight_min": -27.08205223083496,
      "activations/layer21_attention_weight_max": 32.58322525024414,
      "activations/layer21_attention_weight_min": -19.846118927001953,
      "activations/layer22_attention_weight_max": 42.73886489868164,
      "activations/layer22_attention_weight_min": -26.237878799438477,
      "activations/layer23_attention_weight_max": 25.60457992553711,
      "activations/layer23_attention_weight_min": -16.40249252319336,
      "activations/layer2_attention_weight_max": 27.703075408935547,
      "activations/layer2_attention_weight_min": -27.602497100830078,
      "activations/layer3_attention_weight_max": 70.02184295654297,
      "activations/layer3_attention_weight_min": -73.23088073730469,
      "activations/layer4_attention_weight_max": 126.4478530883789,
      "activations/layer4_attention_weight_min": -120.01356506347656,
      "activations/layer5_attention_weight_max": 106.30281066894531,
      "activations/layer5_attention_weight_min": -95.98079681396484,
      "activations/layer6_attention_weight_max": 56.54582977294922,
      "activations/layer6_attention_weight_min": -51.80153274536133,
      "activations/layer7_attention_weight_max": 64.9395523071289,
      "activations/layer7_attention_weight_min": -64.03797149658203,
      "activations/layer8_attention_weight_max": 44.749916076660156,
      "activations/layer8_attention_weight_min": -38.01414489746094,
      "activations/layer9_attention_weight_max": 59.757568359375,
      "activations/layer9_attention_weight_min": -49.415706634521484,
      "epoch": 2.38,
      "learning_rate": 0.00013602348484848486,
      "loss": 2.9481,
      "step": 40950
    },
    {
      "activations/layer0_attention_weight_max": 16.151573181152344,
      "activations/layer0_attention_weight_min": -11.27593994140625,
      "activations/layer10_attention_weight_max": 53.78333282470703,
      "activations/layer10_attention_weight_min": -40.32511901855469,
      "activations/layer11_attention_weight_max": 44.13117980957031,
      "activations/layer11_attention_weight_min": -32.480228424072266,
      "activations/layer12_attention_weight_max": 54.699642181396484,
      "activations/layer12_attention_weight_min": -35.48521041870117,
      "activations/layer13_attention_weight_max": 68.4735336303711,
      "activations/layer13_attention_weight_min": -41.032440185546875,
      "activations/layer14_attention_weight_max": 102.68994903564453,
      "activations/layer14_attention_weight_min": -60.696170806884766,
      "activations/layer15_attention_weight_max": 66.25691223144531,
      "activations/layer15_attention_weight_min": -36.17852020263672,
      "activations/layer16_attention_weight_max": 62.906105041503906,
      "activations/layer16_attention_weight_min": -36.57406997680664,
      "activations/layer17_attention_weight_max": 64.8465805053711,
      "activations/layer17_attention_weight_min": -28.89236068725586,
      "activations/layer18_attention_weight_max": 35.266536712646484,
      "activations/layer18_attention_weight_min": -21.62411880493164,
      "activations/layer19_attention_weight_max": 58.32619857788086,
      "activations/layer19_attention_weight_min": -33.42296600341797,
      "activations/layer1_attention_weight_max": 17.512054443359375,
      "activations/layer1_attention_weight_min": -11.663202285766602,
      "activations/layer20_attention_weight_max": 46.29379653930664,
      "activations/layer20_attention_weight_min": -22.98076629638672,
      "activations/layer21_attention_weight_max": 39.64302062988281,
      "activations/layer21_attention_weight_min": -21.99381446838379,
      "activations/layer22_attention_weight_max": 56.28388977050781,
      "activations/layer22_attention_weight_min": -30.020458221435547,
      "activations/layer23_attention_weight_max": 27.57367515563965,
      "activations/layer23_attention_weight_min": -20.55574607849121,
      "activations/layer2_attention_weight_max": 29.474836349487305,
      "activations/layer2_attention_weight_min": -28.999160766601562,
      "activations/layer3_attention_weight_max": 73.43720245361328,
      "activations/layer3_attention_weight_min": -76.70313262939453,
      "activations/layer4_attention_weight_max": 124.3710708618164,
      "activations/layer4_attention_weight_min": -117.58857727050781,
      "activations/layer5_attention_weight_max": 108.61363220214844,
      "activations/layer5_attention_weight_min": -95.60107421875,
      "activations/layer6_attention_weight_max": 63.44807434082031,
      "activations/layer6_attention_weight_min": -52.91706085205078,
      "activations/layer7_attention_weight_max": 73.1480484008789,
      "activations/layer7_attention_weight_min": -64.77818298339844,
      "activations/layer8_attention_weight_max": 46.94804763793945,
      "activations/layer8_attention_weight_min": -35.82088088989258,
      "activations/layer9_attention_weight_max": 61.31389236450195,
      "activations/layer9_attention_weight_min": -52.990840911865234,
      "epoch": 2.38,
      "learning_rate": 0.00013600454545454544,
      "loss": 2.9406,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_loss": 2.890625,
      "eval_runtime": 8.5189,
      "eval_samples_per_second": 504.057,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_openwebtext_loss": 2.890625,
      "eval_openwebtext_ppl": 18.00455893510183,
      "eval_openwebtext_runtime": 8.5189,
      "eval_openwebtext_samples_per_second": 504.057,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_wikitext_loss": 3.16015625,
      "eval_wikitext_ppl": 23.574279122424027,
      "eval_wikitext_runtime": 1.6904,
      "eval_wikitext_samples_per_second": 269.764,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_lambada_loss": 3.005859375,
      "eval_lambada_ppl": 20.203571081597193,
      "eval_lambada_runtime": 9.6133,
      "eval_lambada_samples_per_second": 506.484,
      "step": 41000
    },
    {
      "activations/layer0_attention_weight_max": 15.690385818481445,
      "activations/layer0_attention_weight_min": -11.384443283081055,
      "activations/layer10_attention_weight_max": 45.48255920410156,
      "activations/layer10_attention_weight_min": -38.243629455566406,
      "activations/layer11_attention_weight_max": 38.09013366699219,
      "activations/layer11_attention_weight_min": -33.94882583618164,
      "activations/layer12_attention_weight_max": 38.7401237487793,
      "activations/layer12_attention_weight_min": -29.51613998413086,
      "activations/layer13_attention_weight_max": 63.54103469848633,
      "activations/layer13_attention_weight_min": -47.68819046020508,
      "activations/layer14_attention_weight_max": 77.1632308959961,
      "activations/layer14_attention_weight_min": -58.33921432495117,
      "activations/layer15_attention_weight_max": 57.707298278808594,
      "activations/layer15_attention_weight_min": -39.47336959838867,
      "activations/layer16_attention_weight_max": 52.72720718383789,
      "activations/layer16_attention_weight_min": -35.51283264160156,
      "activations/layer17_attention_weight_max": 48.37300109863281,
      "activations/layer17_attention_weight_min": -30.61602783203125,
      "activations/layer18_attention_weight_max": 37.56330490112305,
      "activations/layer18_attention_weight_min": -23.79499626159668,
      "activations/layer19_attention_weight_max": 53.78927230834961,
      "activations/layer19_attention_weight_min": -29.834848403930664,
      "activations/layer1_attention_weight_max": 17.486093521118164,
      "activations/layer1_attention_weight_min": -11.263427734375,
      "activations/layer20_attention_weight_max": 39.08557891845703,
      "activations/layer20_attention_weight_min": -23.763750076293945,
      "activations/layer21_attention_weight_max": 29.35188865661621,
      "activations/layer21_attention_weight_min": -19.303409576416016,
      "activations/layer22_attention_weight_max": 49.40584945678711,
      "activations/layer22_attention_weight_min": -27.861326217651367,
      "activations/layer23_attention_weight_max": 27.297393798828125,
      "activations/layer23_attention_weight_min": -17.172866821289062,
      "activations/layer2_attention_weight_max": 30.083486557006836,
      "activations/layer2_attention_weight_min": -29.861011505126953,
      "activations/layer3_attention_weight_max": 76.06886291503906,
      "activations/layer3_attention_weight_min": -82.20115661621094,
      "activations/layer4_attention_weight_max": 128.1100616455078,
      "activations/layer4_attention_weight_min": -131.274169921875,
      "activations/layer5_attention_weight_max": 108.09857177734375,
      "activations/layer5_attention_weight_min": -101.33518981933594,
      "activations/layer6_attention_weight_max": 57.715152740478516,
      "activations/layer6_attention_weight_min": -55.453369140625,
      "activations/layer7_attention_weight_max": 66.54158020019531,
      "activations/layer7_attention_weight_min": -64.50392150878906,
      "activations/layer8_attention_weight_max": 44.78898239135742,
      "activations/layer8_attention_weight_min": -36.83342742919922,
      "activations/layer9_attention_weight_max": 60.80097198486328,
      "activations/layer9_attention_weight_min": -53.02254867553711,
      "epoch": 2.39,
      "learning_rate": 0.00013598560606060606,
      "loss": 2.9467,
      "step": 41050
    },
    {
      "activations/layer0_attention_weight_max": 16.210636138916016,
      "activations/layer0_attention_weight_min": -12.17214298248291,
      "activations/layer10_attention_weight_max": 42.4677619934082,
      "activations/layer10_attention_weight_min": -38.00534439086914,
      "activations/layer11_attention_weight_max": 34.60216522216797,
      "activations/layer11_attention_weight_min": -31.23072052001953,
      "activations/layer12_attention_weight_max": 34.55119705200195,
      "activations/layer12_attention_weight_min": -29.677698135375977,
      "activations/layer13_attention_weight_max": 49.07487487792969,
      "activations/layer13_attention_weight_min": -39.970916748046875,
      "activations/layer14_attention_weight_max": 62.358734130859375,
      "activations/layer14_attention_weight_min": -56.40151596069336,
      "activations/layer15_attention_weight_max": 45.96171569824219,
      "activations/layer15_attention_weight_min": -36.635398864746094,
      "activations/layer16_attention_weight_max": 43.74111557006836,
      "activations/layer16_attention_weight_min": -37.13029098510742,
      "activations/layer17_attention_weight_max": 38.958065032958984,
      "activations/layer17_attention_weight_min": -30.324487686157227,
      "activations/layer18_attention_weight_max": 25.395925521850586,
      "activations/layer18_attention_weight_min": -18.5660343170166,
      "activations/layer19_attention_weight_max": 40.314002990722656,
      "activations/layer19_attention_weight_min": -31.460195541381836,
      "activations/layer1_attention_weight_max": 17.120166778564453,
      "activations/layer1_attention_weight_min": -12.680501937866211,
      "activations/layer20_attention_weight_max": 31.911508560180664,
      "activations/layer20_attention_weight_min": -22.682966232299805,
      "activations/layer21_attention_weight_max": 35.31511306762695,
      "activations/layer21_attention_weight_min": -19.553194046020508,
      "activations/layer22_attention_weight_max": 37.61677551269531,
      "activations/layer22_attention_weight_min": -24.15517234802246,
      "activations/layer23_attention_weight_max": 24.390472412109375,
      "activations/layer23_attention_weight_min": -18.046958923339844,
      "activations/layer2_attention_weight_max": 29.89249038696289,
      "activations/layer2_attention_weight_min": -29.36418914794922,
      "activations/layer3_attention_weight_max": 75.70512390136719,
      "activations/layer3_attention_weight_min": -77.2861099243164,
      "activations/layer4_attention_weight_max": 125.7367935180664,
      "activations/layer4_attention_weight_min": -122.8726577758789,
      "activations/layer5_attention_weight_max": 107.89237976074219,
      "activations/layer5_attention_weight_min": -95.21449279785156,
      "activations/layer6_attention_weight_max": 56.81831741333008,
      "activations/layer6_attention_weight_min": -51.90468215942383,
      "activations/layer7_attention_weight_max": 68.23048400878906,
      "activations/layer7_attention_weight_min": -64.2501220703125,
      "activations/layer8_attention_weight_max": 40.866092681884766,
      "activations/layer8_attention_weight_min": -36.136192321777344,
      "activations/layer9_attention_weight_max": 59.1320915222168,
      "activations/layer9_attention_weight_min": -51.224403381347656,
      "epoch": 2.39,
      "learning_rate": 0.00013596666666666667,
      "loss": 2.9405,
      "step": 41100
    },
    {
      "activations/layer0_attention_weight_max": 16.002113342285156,
      "activations/layer0_attention_weight_min": -11.330957412719727,
      "activations/layer10_attention_weight_max": 52.428184509277344,
      "activations/layer10_attention_weight_min": -42.46858596801758,
      "activations/layer11_attention_weight_max": 55.58515548706055,
      "activations/layer11_attention_weight_min": -34.83049011230469,
      "activations/layer12_attention_weight_max": 36.32944869995117,
      "activations/layer12_attention_weight_min": -29.74217414855957,
      "activations/layer13_attention_weight_max": 53.123252868652344,
      "activations/layer13_attention_weight_min": -36.7239990234375,
      "activations/layer14_attention_weight_max": 78.43450927734375,
      "activations/layer14_attention_weight_min": -60.56405258178711,
      "activations/layer15_attention_weight_max": 55.819095611572266,
      "activations/layer15_attention_weight_min": -39.36410903930664,
      "activations/layer16_attention_weight_max": 49.380367279052734,
      "activations/layer16_attention_weight_min": -41.521141052246094,
      "activations/layer17_attention_weight_max": 41.38362503051758,
      "activations/layer17_attention_weight_min": -29.512136459350586,
      "activations/layer18_attention_weight_max": 28.765939712524414,
      "activations/layer18_attention_weight_min": -17.690568923950195,
      "activations/layer19_attention_weight_max": 47.70759963989258,
      "activations/layer19_attention_weight_min": -34.86556625366211,
      "activations/layer1_attention_weight_max": 17.081890106201172,
      "activations/layer1_attention_weight_min": -10.76365852355957,
      "activations/layer20_attention_weight_max": 35.049766540527344,
      "activations/layer20_attention_weight_min": -22.978609085083008,
      "activations/layer21_attention_weight_max": 33.42530822753906,
      "activations/layer21_attention_weight_min": -20.34256935119629,
      "activations/layer22_attention_weight_max": 44.04495620727539,
      "activations/layer22_attention_weight_min": -27.495115280151367,
      "activations/layer23_attention_weight_max": 25.296560287475586,
      "activations/layer23_attention_weight_min": -15.284414291381836,
      "activations/layer2_attention_weight_max": 28.645498275756836,
      "activations/layer2_attention_weight_min": -29.258726119995117,
      "activations/layer3_attention_weight_max": 75.16793823242188,
      "activations/layer3_attention_weight_min": -77.93862915039062,
      "activations/layer4_attention_weight_max": 127.02291107177734,
      "activations/layer4_attention_weight_min": -119.85823059082031,
      "activations/layer5_attention_weight_max": 113.43049621582031,
      "activations/layer5_attention_weight_min": -93.10311889648438,
      "activations/layer6_attention_weight_max": 63.12527847290039,
      "activations/layer6_attention_weight_min": -51.9285774230957,
      "activations/layer7_attention_weight_max": 75.24857330322266,
      "activations/layer7_attention_weight_min": -63.8072624206543,
      "activations/layer8_attention_weight_max": 50.642208099365234,
      "activations/layer8_attention_weight_min": -43.04778289794922,
      "activations/layer9_attention_weight_max": 81.8755111694336,
      "activations/layer9_attention_weight_min": -62.487823486328125,
      "epoch": 2.39,
      "learning_rate": 0.00013594772727272726,
      "loss": 2.9395,
      "step": 41150
    },
    {
      "activations/layer0_attention_weight_max": 16.078840255737305,
      "activations/layer0_attention_weight_min": -11.498567581176758,
      "activations/layer10_attention_weight_max": 43.75874710083008,
      "activations/layer10_attention_weight_min": -38.267822265625,
      "activations/layer11_attention_weight_max": 37.058563232421875,
      "activations/layer11_attention_weight_min": -32.081138610839844,
      "activations/layer12_attention_weight_max": 41.10206985473633,
      "activations/layer12_attention_weight_min": -35.168907165527344,
      "activations/layer13_attention_weight_max": 47.0771369934082,
      "activations/layer13_attention_weight_min": -39.36666488647461,
      "activations/layer14_attention_weight_max": 67.31038665771484,
      "activations/layer14_attention_weight_min": -53.98506546020508,
      "activations/layer15_attention_weight_max": 46.89685821533203,
      "activations/layer15_attention_weight_min": -34.337059020996094,
      "activations/layer16_attention_weight_max": 47.632232666015625,
      "activations/layer16_attention_weight_min": -36.36223220825195,
      "activations/layer17_attention_weight_max": 48.759891510009766,
      "activations/layer17_attention_weight_min": -30.36342430114746,
      "activations/layer18_attention_weight_max": 32.255126953125,
      "activations/layer18_attention_weight_min": -22.584321975708008,
      "activations/layer19_attention_weight_max": 52.811279296875,
      "activations/layer19_attention_weight_min": -31.88193130493164,
      "activations/layer1_attention_weight_max": 17.439058303833008,
      "activations/layer1_attention_weight_min": -11.017804145812988,
      "activations/layer20_attention_weight_max": 34.68280792236328,
      "activations/layer20_attention_weight_min": -23.346651077270508,
      "activations/layer21_attention_weight_max": 33.645748138427734,
      "activations/layer21_attention_weight_min": -20.7536563873291,
      "activations/layer22_attention_weight_max": 49.03766632080078,
      "activations/layer22_attention_weight_min": -27.005449295043945,
      "activations/layer23_attention_weight_max": 27.88439178466797,
      "activations/layer23_attention_weight_min": -17.348934173583984,
      "activations/layer2_attention_weight_max": 29.556076049804688,
      "activations/layer2_attention_weight_min": -29.483549118041992,
      "activations/layer3_attention_weight_max": 74.12751770019531,
      "activations/layer3_attention_weight_min": -80.2298355102539,
      "activations/layer4_attention_weight_max": 136.4456024169922,
      "activations/layer4_attention_weight_min": -127.17607879638672,
      "activations/layer5_attention_weight_max": 123.15328216552734,
      "activations/layer5_attention_weight_min": -103.91838073730469,
      "activations/layer6_attention_weight_max": 66.54255676269531,
      "activations/layer6_attention_weight_min": -57.73724365234375,
      "activations/layer7_attention_weight_max": 71.06898498535156,
      "activations/layer7_attention_weight_min": -71.26344299316406,
      "activations/layer8_attention_weight_max": 44.35466003417969,
      "activations/layer8_attention_weight_min": -38.006351470947266,
      "activations/layer9_attention_weight_max": 64.23039245605469,
      "activations/layer9_attention_weight_min": -52.3106689453125,
      "epoch": 2.39,
      "learning_rate": 0.00013592878787878788,
      "loss": 2.9384,
      "step": 41200
    },
    {
      "activations/layer0_attention_weight_max": 15.777974128723145,
      "activations/layer0_attention_weight_min": -11.276968002319336,
      "activations/layer10_attention_weight_max": 53.21708679199219,
      "activations/layer10_attention_weight_min": -39.68431091308594,
      "activations/layer11_attention_weight_max": 36.68687438964844,
      "activations/layer11_attention_weight_min": -35.909393310546875,
      "activations/layer12_attention_weight_max": 29.8115177154541,
      "activations/layer12_attention_weight_min": -27.18867301940918,
      "activations/layer13_attention_weight_max": 52.47286605834961,
      "activations/layer13_attention_weight_min": -39.50197982788086,
      "activations/layer14_attention_weight_max": 70.96583557128906,
      "activations/layer14_attention_weight_min": -55.92085647583008,
      "activations/layer15_attention_weight_max": 48.43598175048828,
      "activations/layer15_attention_weight_min": -34.333717346191406,
      "activations/layer16_attention_weight_max": 45.66727066040039,
      "activations/layer16_attention_weight_min": -34.284820556640625,
      "activations/layer17_attention_weight_max": 49.39085388183594,
      "activations/layer17_attention_weight_min": -27.55785369873047,
      "activations/layer18_attention_weight_max": 30.956083297729492,
      "activations/layer18_attention_weight_min": -18.208524703979492,
      "activations/layer19_attention_weight_max": 45.377349853515625,
      "activations/layer19_attention_weight_min": -30.578481674194336,
      "activations/layer1_attention_weight_max": 18.012592315673828,
      "activations/layer1_attention_weight_min": -11.313925743103027,
      "activations/layer20_attention_weight_max": 32.53053283691406,
      "activations/layer20_attention_weight_min": -23.231483459472656,
      "activations/layer21_attention_weight_max": 28.407455444335938,
      "activations/layer21_attention_weight_min": -20.26630973815918,
      "activations/layer22_attention_weight_max": 55.249507904052734,
      "activations/layer22_attention_weight_min": -28.879547119140625,
      "activations/layer23_attention_weight_max": 26.35248374938965,
      "activations/layer23_attention_weight_min": -15.997772216796875,
      "activations/layer2_attention_weight_max": 28.634674072265625,
      "activations/layer2_attention_weight_min": -29.063230514526367,
      "activations/layer3_attention_weight_max": 74.68672943115234,
      "activations/layer3_attention_weight_min": -78.56832885742188,
      "activations/layer4_attention_weight_max": 133.1910858154297,
      "activations/layer4_attention_weight_min": -125.46903228759766,
      "activations/layer5_attention_weight_max": 113.40238952636719,
      "activations/layer5_attention_weight_min": -98.00833129882812,
      "activations/layer6_attention_weight_max": 61.87531661987305,
      "activations/layer6_attention_weight_min": -52.30132293701172,
      "activations/layer7_attention_weight_max": 64.23383331298828,
      "activations/layer7_attention_weight_min": -61.56657028198242,
      "activations/layer8_attention_weight_max": 42.29364776611328,
      "activations/layer8_attention_weight_min": -39.67754364013672,
      "activations/layer9_attention_weight_max": 67.62952423095703,
      "activations/layer9_attention_weight_min": -52.0252799987793,
      "epoch": 2.4,
      "learning_rate": 0.00013590984848484846,
      "loss": 2.9517,
      "step": 41250
    },
    {
      "activations/layer0_attention_weight_max": 14.445472717285156,
      "activations/layer0_attention_weight_min": -11.449905395507812,
      "activations/layer10_attention_weight_max": 43.753292083740234,
      "activations/layer10_attention_weight_min": -35.865352630615234,
      "activations/layer11_attention_weight_max": 36.960060119628906,
      "activations/layer11_attention_weight_min": -32.432212829589844,
      "activations/layer12_attention_weight_max": 55.744319915771484,
      "activations/layer12_attention_weight_min": -30.759035110473633,
      "activations/layer13_attention_weight_max": 50.026145935058594,
      "activations/layer13_attention_weight_min": -37.18320846557617,
      "activations/layer14_attention_weight_max": 71.47126007080078,
      "activations/layer14_attention_weight_min": -55.03696823120117,
      "activations/layer15_attention_weight_max": 50.90972900390625,
      "activations/layer15_attention_weight_min": -35.429405212402344,
      "activations/layer16_attention_weight_max": 49.20209884643555,
      "activations/layer16_attention_weight_min": -38.033687591552734,
      "activations/layer17_attention_weight_max": 48.30295944213867,
      "activations/layer17_attention_weight_min": -28.062238693237305,
      "activations/layer18_attention_weight_max": 32.200233459472656,
      "activations/layer18_attention_weight_min": -19.723278045654297,
      "activations/layer19_attention_weight_max": 51.83181381225586,
      "activations/layer19_attention_weight_min": -32.42919158935547,
      "activations/layer1_attention_weight_max": 17.900617599487305,
      "activations/layer1_attention_weight_min": -11.988882064819336,
      "activations/layer20_attention_weight_max": 41.09275436401367,
      "activations/layer20_attention_weight_min": -24.63174057006836,
      "activations/layer21_attention_weight_max": 32.635379791259766,
      "activations/layer21_attention_weight_min": -20.16085433959961,
      "activations/layer22_attention_weight_max": 48.79157257080078,
      "activations/layer22_attention_weight_min": -26.67782211303711,
      "activations/layer23_attention_weight_max": 28.34424591064453,
      "activations/layer23_attention_weight_min": -19.6751766204834,
      "activations/layer2_attention_weight_max": 28.899168014526367,
      "activations/layer2_attention_weight_min": -28.99128532409668,
      "activations/layer3_attention_weight_max": 73.90426635742188,
      "activations/layer3_attention_weight_min": -77.67176818847656,
      "activations/layer4_attention_weight_max": 126.77123260498047,
      "activations/layer4_attention_weight_min": -116.9967041015625,
      "activations/layer5_attention_weight_max": 101.35626220703125,
      "activations/layer5_attention_weight_min": -99.23182678222656,
      "activations/layer6_attention_weight_max": 57.732154846191406,
      "activations/layer6_attention_weight_min": -51.01332092285156,
      "activations/layer7_attention_weight_max": 70.0263900756836,
      "activations/layer7_attention_weight_min": -65.77820587158203,
      "activations/layer8_attention_weight_max": 44.57806396484375,
      "activations/layer8_attention_weight_min": -36.401851654052734,
      "activations/layer9_attention_weight_max": 55.880043029785156,
      "activations/layer9_attention_weight_min": -46.583187103271484,
      "epoch": 2.4,
      "learning_rate": 0.00013589090909090908,
      "loss": 2.9459,
      "step": 41300
    },
    {
      "activations/layer0_attention_weight_max": 15.882877349853516,
      "activations/layer0_attention_weight_min": -11.773683547973633,
      "activations/layer10_attention_weight_max": 51.30800247192383,
      "activations/layer10_attention_weight_min": -45.47578430175781,
      "activations/layer11_attention_weight_max": 41.482093811035156,
      "activations/layer11_attention_weight_min": -35.798728942871094,
      "activations/layer12_attention_weight_max": 36.559627532958984,
      "activations/layer12_attention_weight_min": -28.985239028930664,
      "activations/layer13_attention_weight_max": 57.926151275634766,
      "activations/layer13_attention_weight_min": -42.76950454711914,
      "activations/layer14_attention_weight_max": 74.53275299072266,
      "activations/layer14_attention_weight_min": -61.689842224121094,
      "activations/layer15_attention_weight_max": 46.08759689331055,
      "activations/layer15_attention_weight_min": -34.74504089355469,
      "activations/layer16_attention_weight_max": 43.96794891357422,
      "activations/layer16_attention_weight_min": -33.48880386352539,
      "activations/layer17_attention_weight_max": 42.2010498046875,
      "activations/layer17_attention_weight_min": -28.76047134399414,
      "activations/layer18_attention_weight_max": 26.508440017700195,
      "activations/layer18_attention_weight_min": -19.488767623901367,
      "activations/layer19_attention_weight_max": 40.25401306152344,
      "activations/layer19_attention_weight_min": -33.0237922668457,
      "activations/layer1_attention_weight_max": 16.817615509033203,
      "activations/layer1_attention_weight_min": -12.853211402893066,
      "activations/layer20_attention_weight_max": 31.857402801513672,
      "activations/layer20_attention_weight_min": -25.150074005126953,
      "activations/layer21_attention_weight_max": 30.860549926757812,
      "activations/layer21_attention_weight_min": -21.141088485717773,
      "activations/layer22_attention_weight_max": 44.252864837646484,
      "activations/layer22_attention_weight_min": -26.81295394897461,
      "activations/layer23_attention_weight_max": 25.4763240814209,
      "activations/layer23_attention_weight_min": -16.779329299926758,
      "activations/layer2_attention_weight_max": 30.521167755126953,
      "activations/layer2_attention_weight_min": -28.49702262878418,
      "activations/layer3_attention_weight_max": 70.63214111328125,
      "activations/layer3_attention_weight_min": -78.45646667480469,
      "activations/layer4_attention_weight_max": 129.7563934326172,
      "activations/layer4_attention_weight_min": -120.47594451904297,
      "activations/layer5_attention_weight_max": 110.5426025390625,
      "activations/layer5_attention_weight_min": -95.74859619140625,
      "activations/layer6_attention_weight_max": 61.6534309387207,
      "activations/layer6_attention_weight_min": -51.27147674560547,
      "activations/layer7_attention_weight_max": 63.75898361206055,
      "activations/layer7_attention_weight_min": -63.820377349853516,
      "activations/layer8_attention_weight_max": 41.952911376953125,
      "activations/layer8_attention_weight_min": -39.34483337402344,
      "activations/layer9_attention_weight_max": 62.26763916015625,
      "activations/layer9_attention_weight_min": -59.85014724731445,
      "epoch": 2.4,
      "learning_rate": 0.0001358719696969697,
      "loss": 2.9371,
      "step": 41350
    },
    {
      "activations/layer0_attention_weight_max": 15.838622093200684,
      "activations/layer0_attention_weight_min": -11.555804252624512,
      "activations/layer10_attention_weight_max": 44.0474853515625,
      "activations/layer10_attention_weight_min": -36.648193359375,
      "activations/layer11_attention_weight_max": 36.844581604003906,
      "activations/layer11_attention_weight_min": -30.96932601928711,
      "activations/layer12_attention_weight_max": 46.21360778808594,
      "activations/layer12_attention_weight_min": -34.69486618041992,
      "activations/layer13_attention_weight_max": 47.84707260131836,
      "activations/layer13_attention_weight_min": -35.09902572631836,
      "activations/layer14_attention_weight_max": 74.5114974975586,
      "activations/layer14_attention_weight_min": -55.09727096557617,
      "activations/layer15_attention_weight_max": 50.40458297729492,
      "activations/layer15_attention_weight_min": -35.55449676513672,
      "activations/layer16_attention_weight_max": 54.81602478027344,
      "activations/layer16_attention_weight_min": -39.520469665527344,
      "activations/layer17_attention_weight_max": 40.88801956176758,
      "activations/layer17_attention_weight_min": -29.82455825805664,
      "activations/layer18_attention_weight_max": 28.735673904418945,
      "activations/layer18_attention_weight_min": -19.08767318725586,
      "activations/layer19_attention_weight_max": 51.59761428833008,
      "activations/layer19_attention_weight_min": -30.97735595703125,
      "activations/layer1_attention_weight_max": 17.57464599609375,
      "activations/layer1_attention_weight_min": -13.89836597442627,
      "activations/layer20_attention_weight_max": 36.62814712524414,
      "activations/layer20_attention_weight_min": -25.663028717041016,
      "activations/layer21_attention_weight_max": 34.7197151184082,
      "activations/layer21_attention_weight_min": -22.773435592651367,
      "activations/layer22_attention_weight_max": 48.11175537109375,
      "activations/layer22_attention_weight_min": -26.529499053955078,
      "activations/layer23_attention_weight_max": 27.644315719604492,
      "activations/layer23_attention_weight_min": -15.016029357910156,
      "activations/layer2_attention_weight_max": 28.36446762084961,
      "activations/layer2_attention_weight_min": -27.386089324951172,
      "activations/layer3_attention_weight_max": 74.0110092163086,
      "activations/layer3_attention_weight_min": -72.21907043457031,
      "activations/layer4_attention_weight_max": 131.38186645507812,
      "activations/layer4_attention_weight_min": -118.80110168457031,
      "activations/layer5_attention_weight_max": 111.79261779785156,
      "activations/layer5_attention_weight_min": -91.903564453125,
      "activations/layer6_attention_weight_max": 61.841094970703125,
      "activations/layer6_attention_weight_min": -51.2126579284668,
      "activations/layer7_attention_weight_max": 71.35089874267578,
      "activations/layer7_attention_weight_min": -61.048580169677734,
      "activations/layer8_attention_weight_max": 44.383724212646484,
      "activations/layer8_attention_weight_min": -35.53312301635742,
      "activations/layer9_attention_weight_max": 66.02386474609375,
      "activations/layer9_attention_weight_min": -55.812408447265625,
      "epoch": 2.41,
      "learning_rate": 0.00013585303030303028,
      "loss": 2.9461,
      "step": 41400
    },
    {
      "activations/layer0_attention_weight_max": 16.320785522460938,
      "activations/layer0_attention_weight_min": -11.901773452758789,
      "activations/layer10_attention_weight_max": 50.264923095703125,
      "activations/layer10_attention_weight_min": -38.669403076171875,
      "activations/layer11_attention_weight_max": 41.026092529296875,
      "activations/layer11_attention_weight_min": -34.96009826660156,
      "activations/layer12_attention_weight_max": 83.10792541503906,
      "activations/layer12_attention_weight_min": -40.318477630615234,
      "activations/layer13_attention_weight_max": 53.96402359008789,
      "activations/layer13_attention_weight_min": -38.80220031738281,
      "activations/layer14_attention_weight_max": 73.98099517822266,
      "activations/layer14_attention_weight_min": -61.603538513183594,
      "activations/layer15_attention_weight_max": 49.79072952270508,
      "activations/layer15_attention_weight_min": -37.157711029052734,
      "activations/layer16_attention_weight_max": 48.873626708984375,
      "activations/layer16_attention_weight_min": -35.109413146972656,
      "activations/layer17_attention_weight_max": 41.997501373291016,
      "activations/layer17_attention_weight_min": -31.319772720336914,
      "activations/layer18_attention_weight_max": 27.594722747802734,
      "activations/layer18_attention_weight_min": -16.944412231445312,
      "activations/layer19_attention_weight_max": 47.37008285522461,
      "activations/layer19_attention_weight_min": -33.84792709350586,
      "activations/layer1_attention_weight_max": 17.26230239868164,
      "activations/layer1_attention_weight_min": -11.158560752868652,
      "activations/layer20_attention_weight_max": 38.302833557128906,
      "activations/layer20_attention_weight_min": -26.037565231323242,
      "activations/layer21_attention_weight_max": 32.09181594848633,
      "activations/layer21_attention_weight_min": -22.348793029785156,
      "activations/layer22_attention_weight_max": 44.77476501464844,
      "activations/layer22_attention_weight_min": -29.01169776916504,
      "activations/layer23_attention_weight_max": 24.901256561279297,
      "activations/layer23_attention_weight_min": -15.155868530273438,
      "activations/layer2_attention_weight_max": 28.24153709411621,
      "activations/layer2_attention_weight_min": -27.591552734375,
      "activations/layer3_attention_weight_max": 73.29174041748047,
      "activations/layer3_attention_weight_min": -73.0462417602539,
      "activations/layer4_attention_weight_max": 131.6993408203125,
      "activations/layer4_attention_weight_min": -119.88362121582031,
      "activations/layer5_attention_weight_max": 111.1952133178711,
      "activations/layer5_attention_weight_min": -99.53939056396484,
      "activations/layer6_attention_weight_max": 59.172367095947266,
      "activations/layer6_attention_weight_min": -55.859474182128906,
      "activations/layer7_attention_weight_max": 69.91752624511719,
      "activations/layer7_attention_weight_min": -64.91277313232422,
      "activations/layer8_attention_weight_max": 43.69383239746094,
      "activations/layer8_attention_weight_min": -37.08292007446289,
      "activations/layer9_attention_weight_max": 57.61084747314453,
      "activations/layer9_attention_weight_min": -52.788002014160156,
      "epoch": 2.41,
      "learning_rate": 0.0001358340909090909,
      "loss": 2.9307,
      "step": 41450
    },
    {
      "activations/layer0_attention_weight_max": 15.400313377380371,
      "activations/layer0_attention_weight_min": -11.421148300170898,
      "activations/layer10_attention_weight_max": 48.3177604675293,
      "activations/layer10_attention_weight_min": -37.906063079833984,
      "activations/layer11_attention_weight_max": 40.65760040283203,
      "activations/layer11_attention_weight_min": -33.3291015625,
      "activations/layer12_attention_weight_max": 30.837753295898438,
      "activations/layer12_attention_weight_min": -29.817781448364258,
      "activations/layer13_attention_weight_max": 53.013729095458984,
      "activations/layer13_attention_weight_min": -35.72435760498047,
      "activations/layer14_attention_weight_max": 81.49333953857422,
      "activations/layer14_attention_weight_min": -55.448890686035156,
      "activations/layer15_attention_weight_max": 58.35692596435547,
      "activations/layer15_attention_weight_min": -43.53907775878906,
      "activations/layer16_attention_weight_max": 48.67531967163086,
      "activations/layer16_attention_weight_min": -36.38235092163086,
      "activations/layer17_attention_weight_max": 44.46958541870117,
      "activations/layer17_attention_weight_min": -27.9777774810791,
      "activations/layer18_attention_weight_max": 28.341299057006836,
      "activations/layer18_attention_weight_min": -17.779842376708984,
      "activations/layer19_attention_weight_max": 49.52745819091797,
      "activations/layer19_attention_weight_min": -31.10848045349121,
      "activations/layer1_attention_weight_max": 17.466001510620117,
      "activations/layer1_attention_weight_min": -10.306994438171387,
      "activations/layer20_attention_weight_max": 37.984649658203125,
      "activations/layer20_attention_weight_min": -23.883872985839844,
      "activations/layer21_attention_weight_max": 36.574684143066406,
      "activations/layer21_attention_weight_min": -20.120567321777344,
      "activations/layer22_attention_weight_max": 44.280574798583984,
      "activations/layer22_attention_weight_min": -23.54894256591797,
      "activations/layer23_attention_weight_max": 25.629058837890625,
      "activations/layer23_attention_weight_min": -15.615878105163574,
      "activations/layer2_attention_weight_max": 31.14328384399414,
      "activations/layer2_attention_weight_min": -28.915077209472656,
      "activations/layer3_attention_weight_max": 79.15179443359375,
      "activations/layer3_attention_weight_min": -82.97611999511719,
      "activations/layer4_attention_weight_max": 146.2036590576172,
      "activations/layer4_attention_weight_min": -134.3682098388672,
      "activations/layer5_attention_weight_max": 126.37303924560547,
      "activations/layer5_attention_weight_min": -104.48881530761719,
      "activations/layer6_attention_weight_max": 68.78166198730469,
      "activations/layer6_attention_weight_min": -56.96396255493164,
      "activations/layer7_attention_weight_max": 74.38203430175781,
      "activations/layer7_attention_weight_min": -67.99337005615234,
      "activations/layer8_attention_weight_max": 50.57011032104492,
      "activations/layer8_attention_weight_min": -38.036537170410156,
      "activations/layer9_attention_weight_max": 68.07811737060547,
      "activations/layer9_attention_weight_min": -53.02328109741211,
      "epoch": 2.41,
      "learning_rate": 0.0001358151515151515,
      "loss": 2.9487,
      "step": 41500
    },
    {
      "activations/layer0_attention_weight_max": 14.563918113708496,
      "activations/layer0_attention_weight_min": -11.498845100402832,
      "activations/layer10_attention_weight_max": 53.21849060058594,
      "activations/layer10_attention_weight_min": -39.70734405517578,
      "activations/layer11_attention_weight_max": 44.698787689208984,
      "activations/layer11_attention_weight_min": -37.660499572753906,
      "activations/layer12_attention_weight_max": 39.8167839050293,
      "activations/layer12_attention_weight_min": -29.17505645751953,
      "activations/layer13_attention_weight_max": 50.93819808959961,
      "activations/layer13_attention_weight_min": -35.37700653076172,
      "activations/layer14_attention_weight_max": 75.42816162109375,
      "activations/layer14_attention_weight_min": -59.108642578125,
      "activations/layer15_attention_weight_max": 52.154090881347656,
      "activations/layer15_attention_weight_min": -38.015655517578125,
      "activations/layer16_attention_weight_max": 51.85914611816406,
      "activations/layer16_attention_weight_min": -36.852272033691406,
      "activations/layer17_attention_weight_max": 47.62464141845703,
      "activations/layer17_attention_weight_min": -32.073116302490234,
      "activations/layer18_attention_weight_max": 28.1240177154541,
      "activations/layer18_attention_weight_min": -18.067649841308594,
      "activations/layer19_attention_weight_max": 52.151611328125,
      "activations/layer19_attention_weight_min": -34.906620025634766,
      "activations/layer1_attention_weight_max": 18.30613899230957,
      "activations/layer1_attention_weight_min": -11.40425968170166,
      "activations/layer20_attention_weight_max": 34.62015914916992,
      "activations/layer20_attention_weight_min": -26.663822174072266,
      "activations/layer21_attention_weight_max": 32.96898651123047,
      "activations/layer21_attention_weight_min": -19.48391342163086,
      "activations/layer22_attention_weight_max": 45.38289260864258,
      "activations/layer22_attention_weight_min": -24.838769912719727,
      "activations/layer23_attention_weight_max": 25.238597869873047,
      "activations/layer23_attention_weight_min": -15.381763458251953,
      "activations/layer2_attention_weight_max": 29.015714645385742,
      "activations/layer2_attention_weight_min": -28.725135803222656,
      "activations/layer3_attention_weight_max": 73.94189453125,
      "activations/layer3_attention_weight_min": -77.49530029296875,
      "activations/layer4_attention_weight_max": 131.0663299560547,
      "activations/layer4_attention_weight_min": -122.79491424560547,
      "activations/layer5_attention_weight_max": 116.79066467285156,
      "activations/layer5_attention_weight_min": -96.51815032958984,
      "activations/layer6_attention_weight_max": 64.7743911743164,
      "activations/layer6_attention_weight_min": -52.343482971191406,
      "activations/layer7_attention_weight_max": 70.94248962402344,
      "activations/layer7_attention_weight_min": -67.9477767944336,
      "activations/layer8_attention_weight_max": 47.3194694519043,
      "activations/layer8_attention_weight_min": -38.83155059814453,
      "activations/layer9_attention_weight_max": 69.4410629272461,
      "activations/layer9_attention_weight_min": -55.947566986083984,
      "epoch": 2.41,
      "learning_rate": 0.0001357962121212121,
      "loss": 2.9421,
      "step": 41550
    },
    {
      "activations/layer0_attention_weight_max": 16.477624893188477,
      "activations/layer0_attention_weight_min": -11.254748344421387,
      "activations/layer10_attention_weight_max": 45.99050521850586,
      "activations/layer10_attention_weight_min": -39.55195236206055,
      "activations/layer11_attention_weight_max": 39.25360870361328,
      "activations/layer11_attention_weight_min": -33.25450134277344,
      "activations/layer12_attention_weight_max": 31.609146118164062,
      "activations/layer12_attention_weight_min": -27.59691047668457,
      "activations/layer13_attention_weight_max": 43.8979606628418,
      "activations/layer13_attention_weight_min": -35.641536712646484,
      "activations/layer14_attention_weight_max": 73.65135192871094,
      "activations/layer14_attention_weight_min": -58.70846176147461,
      "activations/layer15_attention_weight_max": 49.20923614501953,
      "activations/layer15_attention_weight_min": -36.496910095214844,
      "activations/layer16_attention_weight_max": 47.759822845458984,
      "activations/layer16_attention_weight_min": -32.301212310791016,
      "activations/layer17_attention_weight_max": 46.48092269897461,
      "activations/layer17_attention_weight_min": -30.800466537475586,
      "activations/layer18_attention_weight_max": 29.6812686920166,
      "activations/layer18_attention_weight_min": -19.522138595581055,
      "activations/layer19_attention_weight_max": 47.53380584716797,
      "activations/layer19_attention_weight_min": -33.33675765991211,
      "activations/layer1_attention_weight_max": 17.94322395324707,
      "activations/layer1_attention_weight_min": -11.59594440460205,
      "activations/layer20_attention_weight_max": 36.14012145996094,
      "activations/layer20_attention_weight_min": -24.567777633666992,
      "activations/layer21_attention_weight_max": 34.29393005371094,
      "activations/layer21_attention_weight_min": -20.826143264770508,
      "activations/layer22_attention_weight_max": 47.71857452392578,
      "activations/layer22_attention_weight_min": -26.277233123779297,
      "activations/layer23_attention_weight_max": 25.469409942626953,
      "activations/layer23_attention_weight_min": -18.739782333374023,
      "activations/layer2_attention_weight_max": 30.359607696533203,
      "activations/layer2_attention_weight_min": -29.307907104492188,
      "activations/layer3_attention_weight_max": 72.65939331054688,
      "activations/layer3_attention_weight_min": -75.98063659667969,
      "activations/layer4_attention_weight_max": 124.8000717163086,
      "activations/layer4_attention_weight_min": -114.5953369140625,
      "activations/layer5_attention_weight_max": 106.70327758789062,
      "activations/layer5_attention_weight_min": -93.38583374023438,
      "activations/layer6_attention_weight_max": 56.527931213378906,
      "activations/layer6_attention_weight_min": -51.964778900146484,
      "activations/layer7_attention_weight_max": 65.4090347290039,
      "activations/layer7_attention_weight_min": -65.73136901855469,
      "activations/layer8_attention_weight_max": 39.322608947753906,
      "activations/layer8_attention_weight_min": -35.70719528198242,
      "activations/layer9_attention_weight_max": 54.17921829223633,
      "activations/layer9_attention_weight_min": -43.908939361572266,
      "epoch": 2.42,
      "learning_rate": 0.0001357776515151515,
      "loss": 2.9512,
      "step": 41600
    },
    {
      "activations/layer0_attention_weight_max": 15.205450057983398,
      "activations/layer0_attention_weight_min": -10.860525131225586,
      "activations/layer10_attention_weight_max": 45.1287727355957,
      "activations/layer10_attention_weight_min": -37.31927490234375,
      "activations/layer11_attention_weight_max": 38.53639602661133,
      "activations/layer11_attention_weight_min": -32.68439865112305,
      "activations/layer12_attention_weight_max": 30.749778747558594,
      "activations/layer12_attention_weight_min": -27.107126235961914,
      "activations/layer13_attention_weight_max": 52.23971939086914,
      "activations/layer13_attention_weight_min": -37.38784408569336,
      "activations/layer14_attention_weight_max": 82.15374755859375,
      "activations/layer14_attention_weight_min": -56.281436920166016,
      "activations/layer15_attention_weight_max": 48.82351303100586,
      "activations/layer15_attention_weight_min": -34.4665641784668,
      "activations/layer16_attention_weight_max": 50.326839447021484,
      "activations/layer16_attention_weight_min": -34.365638732910156,
      "activations/layer17_attention_weight_max": 41.80715560913086,
      "activations/layer17_attention_weight_min": -27.694416046142578,
      "activations/layer18_attention_weight_max": 29.90431022644043,
      "activations/layer18_attention_weight_min": -18.085365295410156,
      "activations/layer19_attention_weight_max": 54.65957260131836,
      "activations/layer19_attention_weight_min": -31.048070907592773,
      "activations/layer1_attention_weight_max": 16.720680236816406,
      "activations/layer1_attention_weight_min": -11.782461166381836,
      "activations/layer20_attention_weight_max": 36.080562591552734,
      "activations/layer20_attention_weight_min": -24.492557525634766,
      "activations/layer21_attention_weight_max": 33.62076950073242,
      "activations/layer21_attention_weight_min": -19.41624641418457,
      "activations/layer22_attention_weight_max": 42.748069763183594,
      "activations/layer22_attention_weight_min": -28.24911880493164,
      "activations/layer23_attention_weight_max": 28.757925033569336,
      "activations/layer23_attention_weight_min": -16.359600067138672,
      "activations/layer2_attention_weight_max": 29.041181564331055,
      "activations/layer2_attention_weight_min": -27.9859619140625,
      "activations/layer3_attention_weight_max": 73.95387268066406,
      "activations/layer3_attention_weight_min": -80.42678833007812,
      "activations/layer4_attention_weight_max": 130.98367309570312,
      "activations/layer4_attention_weight_min": -118.85344696044922,
      "activations/layer5_attention_weight_max": 110.46257781982422,
      "activations/layer5_attention_weight_min": -94.96282196044922,
      "activations/layer6_attention_weight_max": 59.29874801635742,
      "activations/layer6_attention_weight_min": -51.84407043457031,
      "activations/layer7_attention_weight_max": 67.52272033691406,
      "activations/layer7_attention_weight_min": -69.4814453125,
      "activations/layer8_attention_weight_max": 41.850215911865234,
      "activations/layer8_attention_weight_min": -36.657867431640625,
      "activations/layer9_attention_weight_max": 60.890655517578125,
      "activations/layer9_attention_weight_min": -48.259735107421875,
      "epoch": 2.42,
      "learning_rate": 0.0001357587121212121,
      "loss": 2.9455,
      "step": 41650
    },
    {
      "activations/layer0_attention_weight_max": 16.4484920501709,
      "activations/layer0_attention_weight_min": -11.131658554077148,
      "activations/layer10_attention_weight_max": 46.95145034790039,
      "activations/layer10_attention_weight_min": -42.628929138183594,
      "activations/layer11_attention_weight_max": 37.01636505126953,
      "activations/layer11_attention_weight_min": -32.30644226074219,
      "activations/layer12_attention_weight_max": 55.05263900756836,
      "activations/layer12_attention_weight_min": -35.58738708496094,
      "activations/layer13_attention_weight_max": 52.71723556518555,
      "activations/layer13_attention_weight_min": -39.56522750854492,
      "activations/layer14_attention_weight_max": 85.34422302246094,
      "activations/layer14_attention_weight_min": -56.94082260131836,
      "activations/layer15_attention_weight_max": 47.36299133300781,
      "activations/layer15_attention_weight_min": -33.76088333129883,
      "activations/layer16_attention_weight_max": 52.27557373046875,
      "activations/layer16_attention_weight_min": -37.578369140625,
      "activations/layer17_attention_weight_max": 45.68562316894531,
      "activations/layer17_attention_weight_min": -31.735769271850586,
      "activations/layer18_attention_weight_max": 26.607261657714844,
      "activations/layer18_attention_weight_min": -18.509628295898438,
      "activations/layer19_attention_weight_max": 51.7664909362793,
      "activations/layer19_attention_weight_min": -34.87556076049805,
      "activations/layer1_attention_weight_max": 17.275609970092773,
      "activations/layer1_attention_weight_min": -11.313812255859375,
      "activations/layer20_attention_weight_max": 34.998435974121094,
      "activations/layer20_attention_weight_min": -24.6473331451416,
      "activations/layer21_attention_weight_max": 30.170339584350586,
      "activations/layer21_attention_weight_min": -19.432113647460938,
      "activations/layer22_attention_weight_max": 47.19818878173828,
      "activations/layer22_attention_weight_min": -25.70000457763672,
      "activations/layer23_attention_weight_max": 25.125946044921875,
      "activations/layer23_attention_weight_min": -16.969545364379883,
      "activations/layer2_attention_weight_max": 29.268657684326172,
      "activations/layer2_attention_weight_min": -29.33155632019043,
      "activations/layer3_attention_weight_max": 71.46148681640625,
      "activations/layer3_attention_weight_min": -77.2752685546875,
      "activations/layer4_attention_weight_max": 128.72117614746094,
      "activations/layer4_attention_weight_min": -118.26029205322266,
      "activations/layer5_attention_weight_max": 107.80009460449219,
      "activations/layer5_attention_weight_min": -93.75392150878906,
      "activations/layer6_attention_weight_max": 57.50438690185547,
      "activations/layer6_attention_weight_min": -51.57823181152344,
      "activations/layer7_attention_weight_max": 68.54328155517578,
      "activations/layer7_attention_weight_min": -65.57601928710938,
      "activations/layer8_attention_weight_max": 42.352577209472656,
      "activations/layer8_attention_weight_min": -36.88203811645508,
      "activations/layer9_attention_weight_max": 57.819679260253906,
      "activations/layer9_attention_weight_min": -51.13829040527344,
      "epoch": 2.42,
      "learning_rate": 0.00013573977272727272,
      "loss": 2.9403,
      "step": 41700
    },
    {
      "activations/layer0_attention_weight_max": 16.439851760864258,
      "activations/layer0_attention_weight_min": -11.557052612304688,
      "activations/layer10_attention_weight_max": 45.50067138671875,
      "activations/layer10_attention_weight_min": -39.61115264892578,
      "activations/layer11_attention_weight_max": 38.69439697265625,
      "activations/layer11_attention_weight_min": -32.095943450927734,
      "activations/layer12_attention_weight_max": 53.030033111572266,
      "activations/layer12_attention_weight_min": -36.389747619628906,
      "activations/layer13_attention_weight_max": 58.663394927978516,
      "activations/layer13_attention_weight_min": -48.158329010009766,
      "activations/layer14_attention_weight_max": 78.7762222290039,
      "activations/layer14_attention_weight_min": -61.09093475341797,
      "activations/layer15_attention_weight_max": 53.84375762939453,
      "activations/layer15_attention_weight_min": -36.07250213623047,
      "activations/layer16_attention_weight_max": 49.17601776123047,
      "activations/layer16_attention_weight_min": -35.92340850830078,
      "activations/layer17_attention_weight_max": 42.2486572265625,
      "activations/layer17_attention_weight_min": -33.60189437866211,
      "activations/layer18_attention_weight_max": 30.213802337646484,
      "activations/layer18_attention_weight_min": -18.31896209716797,
      "activations/layer19_attention_weight_max": 52.69782257080078,
      "activations/layer19_attention_weight_min": -30.44109535217285,
      "activations/layer1_attention_weight_max": 18.505949020385742,
      "activations/layer1_attention_weight_min": -11.487369537353516,
      "activations/layer20_attention_weight_max": 36.54877853393555,
      "activations/layer20_attention_weight_min": -24.360515594482422,
      "activations/layer21_attention_weight_max": 37.2012825012207,
      "activations/layer21_attention_weight_min": -21.18379783630371,
      "activations/layer22_attention_weight_max": 43.86427688598633,
      "activations/layer22_attention_weight_min": -27.766220092773438,
      "activations/layer23_attention_weight_max": 27.775829315185547,
      "activations/layer23_attention_weight_min": -16.78833770751953,
      "activations/layer2_attention_weight_max": 32.3809700012207,
      "activations/layer2_attention_weight_min": -28.636268615722656,
      "activations/layer3_attention_weight_max": 75.78579711914062,
      "activations/layer3_attention_weight_min": -77.36466979980469,
      "activations/layer4_attention_weight_max": 131.31520080566406,
      "activations/layer4_attention_weight_min": -120.72081756591797,
      "activations/layer5_attention_weight_max": 116.37816619873047,
      "activations/layer5_attention_weight_min": -95.75057220458984,
      "activations/layer6_attention_weight_max": 63.0765380859375,
      "activations/layer6_attention_weight_min": -52.101356506347656,
      "activations/layer7_attention_weight_max": 72.29393768310547,
      "activations/layer7_attention_weight_min": -67.2638931274414,
      "activations/layer8_attention_weight_max": 45.82185745239258,
      "activations/layer8_attention_weight_min": -40.40915298461914,
      "activations/layer9_attention_weight_max": 64.17567443847656,
      "activations/layer9_attention_weight_min": -50.64946365356445,
      "epoch": 2.43,
      "learning_rate": 0.0001357208333333333,
      "loss": 2.9513,
      "step": 41750
    },
    {
      "activations/layer0_attention_weight_max": 16.49710464477539,
      "activations/layer0_attention_weight_min": -11.51610279083252,
      "activations/layer10_attention_weight_max": 50.01527786254883,
      "activations/layer10_attention_weight_min": -40.618202209472656,
      "activations/layer11_attention_weight_max": 44.14730453491211,
      "activations/layer11_attention_weight_min": -35.5316276550293,
      "activations/layer12_attention_weight_max": 33.306095123291016,
      "activations/layer12_attention_weight_min": -25.877750396728516,
      "activations/layer13_attention_weight_max": 52.45319747924805,
      "activations/layer13_attention_weight_min": -34.6978874206543,
      "activations/layer14_attention_weight_max": 90.63557434082031,
      "activations/layer14_attention_weight_min": -65.89823150634766,
      "activations/layer15_attention_weight_max": 53.08757781982422,
      "activations/layer15_attention_weight_min": -35.986412048339844,
      "activations/layer16_attention_weight_max": 46.476680755615234,
      "activations/layer16_attention_weight_min": -34.14500427246094,
      "activations/layer17_attention_weight_max": 44.85918045043945,
      "activations/layer17_attention_weight_min": -28.093122482299805,
      "activations/layer18_attention_weight_max": 27.389986038208008,
      "activations/layer18_attention_weight_min": -16.18271827697754,
      "activations/layer19_attention_weight_max": 51.409664154052734,
      "activations/layer19_attention_weight_min": -30.1253719329834,
      "activations/layer1_attention_weight_max": 16.597991943359375,
      "activations/layer1_attention_weight_min": -11.61473560333252,
      "activations/layer20_attention_weight_max": 35.589717864990234,
      "activations/layer20_attention_weight_min": -23.378385543823242,
      "activations/layer21_attention_weight_max": 36.52516174316406,
      "activations/layer21_attention_weight_min": -20.944095611572266,
      "activations/layer22_attention_weight_max": 52.287227630615234,
      "activations/layer22_attention_weight_min": -26.663652420043945,
      "activations/layer23_attention_weight_max": 28.017154693603516,
      "activations/layer23_attention_weight_min": -16.503324508666992,
      "activations/layer2_attention_weight_max": 30.078105926513672,
      "activations/layer2_attention_weight_min": -28.910621643066406,
      "activations/layer3_attention_weight_max": 76.4582290649414,
      "activations/layer3_attention_weight_min": -77.23241424560547,
      "activations/layer4_attention_weight_max": 132.7323455810547,
      "activations/layer4_attention_weight_min": -121.88531494140625,
      "activations/layer5_attention_weight_max": 116.03532409667969,
      "activations/layer5_attention_weight_min": -97.80079650878906,
      "activations/layer6_attention_weight_max": 63.10249328613281,
      "activations/layer6_attention_weight_min": -51.91592025756836,
      "activations/layer7_attention_weight_max": 67.3286361694336,
      "activations/layer7_attention_weight_min": -63.1659049987793,
      "activations/layer8_attention_weight_max": 45.22596740722656,
      "activations/layer8_attention_weight_min": -36.368934631347656,
      "activations/layer9_attention_weight_max": 55.762359619140625,
      "activations/layer9_attention_weight_min": -48.69803237915039,
      "epoch": 2.43,
      "learning_rate": 0.00013570189393939392,
      "loss": 2.9428,
      "step": 41800
    },
    {
      "activations/layer0_attention_weight_max": 16.49741554260254,
      "activations/layer0_attention_weight_min": -11.22533130645752,
      "activations/layer10_attention_weight_max": 51.3570556640625,
      "activations/layer10_attention_weight_min": -39.27784729003906,
      "activations/layer11_attention_weight_max": 38.135536193847656,
      "activations/layer11_attention_weight_min": -32.740291595458984,
      "activations/layer12_attention_weight_max": 31.70317840576172,
      "activations/layer12_attention_weight_min": -30.816654205322266,
      "activations/layer13_attention_weight_max": 47.24073791503906,
      "activations/layer13_attention_weight_min": -33.29985427856445,
      "activations/layer14_attention_weight_max": 83.38152313232422,
      "activations/layer14_attention_weight_min": -63.52273941040039,
      "activations/layer15_attention_weight_max": 51.24503707885742,
      "activations/layer15_attention_weight_min": -35.67506408691406,
      "activations/layer16_attention_weight_max": 45.44480895996094,
      "activations/layer16_attention_weight_min": -33.88459014892578,
      "activations/layer17_attention_weight_max": 42.39548110961914,
      "activations/layer17_attention_weight_min": -25.93454933166504,
      "activations/layer18_attention_weight_max": 30.424320220947266,
      "activations/layer18_attention_weight_min": -18.03298568725586,
      "activations/layer19_attention_weight_max": 44.21376419067383,
      "activations/layer19_attention_weight_min": -31.79618263244629,
      "activations/layer1_attention_weight_max": 17.055734634399414,
      "activations/layer1_attention_weight_min": -12.148040771484375,
      "activations/layer20_attention_weight_max": 35.926448822021484,
      "activations/layer20_attention_weight_min": -23.841636657714844,
      "activations/layer21_attention_weight_max": 30.41902732849121,
      "activations/layer21_attention_weight_min": -20.825429916381836,
      "activations/layer22_attention_weight_max": 47.198970794677734,
      "activations/layer22_attention_weight_min": -27.243793487548828,
      "activations/layer23_attention_weight_max": 23.730052947998047,
      "activations/layer23_attention_weight_min": -16.709877014160156,
      "activations/layer2_attention_weight_max": 28.472278594970703,
      "activations/layer2_attention_weight_min": -28.568103790283203,
      "activations/layer3_attention_weight_max": 70.61860656738281,
      "activations/layer3_attention_weight_min": -72.97198486328125,
      "activations/layer4_attention_weight_max": 120.54637908935547,
      "activations/layer4_attention_weight_min": -115.3643798828125,
      "activations/layer5_attention_weight_max": 106.49026489257812,
      "activations/layer5_attention_weight_min": -93.14696502685547,
      "activations/layer6_attention_weight_max": 59.07001495361328,
      "activations/layer6_attention_weight_min": -55.221954345703125,
      "activations/layer7_attention_weight_max": 67.32977294921875,
      "activations/layer7_attention_weight_min": -60.352134704589844,
      "activations/layer8_attention_weight_max": 41.78647994995117,
      "activations/layer8_attention_weight_min": -35.83003234863281,
      "activations/layer9_attention_weight_max": 55.746124267578125,
      "activations/layer9_attention_weight_min": -47.34319305419922,
      "epoch": 2.43,
      "learning_rate": 0.0001356829545454545,
      "loss": 2.9205,
      "step": 41850
    },
    {
      "activations/layer0_attention_weight_max": 16.142192840576172,
      "activations/layer0_attention_weight_min": -11.567654609680176,
      "activations/layer10_attention_weight_max": 43.75493240356445,
      "activations/layer10_attention_weight_min": -36.40555953979492,
      "activations/layer11_attention_weight_max": 37.19648361206055,
      "activations/layer11_attention_weight_min": -31.446378707885742,
      "activations/layer12_attention_weight_max": 35.172035217285156,
      "activations/layer12_attention_weight_min": -27.537580490112305,
      "activations/layer13_attention_weight_max": 57.47310256958008,
      "activations/layer13_attention_weight_min": -43.10837936401367,
      "activations/layer14_attention_weight_max": 76.7654800415039,
      "activations/layer14_attention_weight_min": -58.696170806884766,
      "activations/layer15_attention_weight_max": 48.53044891357422,
      "activations/layer15_attention_weight_min": -36.174827575683594,
      "activations/layer16_attention_weight_max": 47.275569915771484,
      "activations/layer16_attention_weight_min": -34.63993453979492,
      "activations/layer17_attention_weight_max": 54.11070251464844,
      "activations/layer17_attention_weight_min": -32.28679275512695,
      "activations/layer18_attention_weight_max": 29.126201629638672,
      "activations/layer18_attention_weight_min": -20.451147079467773,
      "activations/layer19_attention_weight_max": 49.13296127319336,
      "activations/layer19_attention_weight_min": -31.079788208007812,
      "activations/layer1_attention_weight_max": 17.636159896850586,
      "activations/layer1_attention_weight_min": -11.74856948852539,
      "activations/layer20_attention_weight_max": 37.89442443847656,
      "activations/layer20_attention_weight_min": -22.65013313293457,
      "activations/layer21_attention_weight_max": 35.76092529296875,
      "activations/layer21_attention_weight_min": -21.10178565979004,
      "activations/layer22_attention_weight_max": 47.848533630371094,
      "activations/layer22_attention_weight_min": -27.90875816345215,
      "activations/layer23_attention_weight_max": 26.583114624023438,
      "activations/layer23_attention_weight_min": -15.851866722106934,
      "activations/layer2_attention_weight_max": 30.124725341796875,
      "activations/layer2_attention_weight_min": -30.385040283203125,
      "activations/layer3_attention_weight_max": 73.91348266601562,
      "activations/layer3_attention_weight_min": -76.17984771728516,
      "activations/layer4_attention_weight_max": 137.4717254638672,
      "activations/layer4_attention_weight_min": -120.7691421508789,
      "activations/layer5_attention_weight_max": 110.7912826538086,
      "activations/layer5_attention_weight_min": -92.78160858154297,
      "activations/layer6_attention_weight_max": 60.05330276489258,
      "activations/layer6_attention_weight_min": -52.603981018066406,
      "activations/layer7_attention_weight_max": 68.72797393798828,
      "activations/layer7_attention_weight_min": -63.167816162109375,
      "activations/layer8_attention_weight_max": 45.10704803466797,
      "activations/layer8_attention_weight_min": -37.1011848449707,
      "activations/layer9_attention_weight_max": 56.665283203125,
      "activations/layer9_attention_weight_min": -47.6517333984375,
      "epoch": 2.43,
      "learning_rate": 0.00013566401515151513,
      "loss": 2.9178,
      "step": 41900
    },
    {
      "activations/layer0_attention_weight_max": 16.045230865478516,
      "activations/layer0_attention_weight_min": -11.693449974060059,
      "activations/layer10_attention_weight_max": 42.40351486206055,
      "activations/layer10_attention_weight_min": -33.0379638671875,
      "activations/layer11_attention_weight_max": 36.882774353027344,
      "activations/layer11_attention_weight_min": -32.4111213684082,
      "activations/layer12_attention_weight_max": 37.65925979614258,
      "activations/layer12_attention_weight_min": -34.37730026245117,
      "activations/layer13_attention_weight_max": 47.93412780761719,
      "activations/layer13_attention_weight_min": -36.78078079223633,
      "activations/layer14_attention_weight_max": 82.14054870605469,
      "activations/layer14_attention_weight_min": -61.07135009765625,
      "activations/layer15_attention_weight_max": 49.43645095825195,
      "activations/layer15_attention_weight_min": -33.363975524902344,
      "activations/layer16_attention_weight_max": 54.01461410522461,
      "activations/layer16_attention_weight_min": -33.65208435058594,
      "activations/layer17_attention_weight_max": 46.11030578613281,
      "activations/layer17_attention_weight_min": -28.6028995513916,
      "activations/layer18_attention_weight_max": 29.33376693725586,
      "activations/layer18_attention_weight_min": -18.308149337768555,
      "activations/layer19_attention_weight_max": 54.39494705200195,
      "activations/layer19_attention_weight_min": -33.395042419433594,
      "activations/layer1_attention_weight_max": 18.213109970092773,
      "activations/layer1_attention_weight_min": -11.395522117614746,
      "activations/layer20_attention_weight_max": 38.79961013793945,
      "activations/layer20_attention_weight_min": -22.24376678466797,
      "activations/layer21_attention_weight_max": 31.520402908325195,
      "activations/layer21_attention_weight_min": -19.529834747314453,
      "activations/layer22_attention_weight_max": 51.061370849609375,
      "activations/layer22_attention_weight_min": -24.553661346435547,
      "activations/layer23_attention_weight_max": 30.679967880249023,
      "activations/layer23_attention_weight_min": -15.69375228881836,
      "activations/layer2_attention_weight_max": 27.540725708007812,
      "activations/layer2_attention_weight_min": -27.408367156982422,
      "activations/layer3_attention_weight_max": 73.8761215209961,
      "activations/layer3_attention_weight_min": -75.91820526123047,
      "activations/layer4_attention_weight_max": 126.9737548828125,
      "activations/layer4_attention_weight_min": -123.18544006347656,
      "activations/layer5_attention_weight_max": 110.04490661621094,
      "activations/layer5_attention_weight_min": -95.9594955444336,
      "activations/layer6_attention_weight_max": 62.60150909423828,
      "activations/layer6_attention_weight_min": -54.487422943115234,
      "activations/layer7_attention_weight_max": 68.1087417602539,
      "activations/layer7_attention_weight_min": -64.49468231201172,
      "activations/layer8_attention_weight_max": 42.64945983886719,
      "activations/layer8_attention_weight_min": -37.23281478881836,
      "activations/layer9_attention_weight_max": 59.83974075317383,
      "activations/layer9_attention_weight_min": -47.19124984741211,
      "epoch": 2.44,
      "learning_rate": 0.00013564507575757574,
      "loss": 2.9242,
      "step": 41950
    },
    {
      "activations/layer0_attention_weight_max": 16.482662200927734,
      "activations/layer0_attention_weight_min": -11.619215965270996,
      "activations/layer10_attention_weight_max": 51.745574951171875,
      "activations/layer10_attention_weight_min": -41.73989486694336,
      "activations/layer11_attention_weight_max": 41.11766052246094,
      "activations/layer11_attention_weight_min": -37.433509826660156,
      "activations/layer12_attention_weight_max": 35.36155700683594,
      "activations/layer12_attention_weight_min": -28.458948135375977,
      "activations/layer13_attention_weight_max": 47.024635314941406,
      "activations/layer13_attention_weight_min": -36.931549072265625,
      "activations/layer14_attention_weight_max": 78.88654327392578,
      "activations/layer14_attention_weight_min": -62.37712097167969,
      "activations/layer15_attention_weight_max": 51.269649505615234,
      "activations/layer15_attention_weight_min": -35.84318542480469,
      "activations/layer16_attention_weight_max": 46.01665115356445,
      "activations/layer16_attention_weight_min": -34.38195037841797,
      "activations/layer17_attention_weight_max": 45.04795455932617,
      "activations/layer17_attention_weight_min": -28.016550064086914,
      "activations/layer18_attention_weight_max": 28.280921936035156,
      "activations/layer18_attention_weight_min": -18.49518394470215,
      "activations/layer19_attention_weight_max": 49.10892868041992,
      "activations/layer19_attention_weight_min": -31.22609519958496,
      "activations/layer1_attention_weight_max": 17.71729278564453,
      "activations/layer1_attention_weight_min": -11.698653221130371,
      "activations/layer20_attention_weight_max": 39.28407287597656,
      "activations/layer20_attention_weight_min": -23.922367095947266,
      "activations/layer21_attention_weight_max": 33.540924072265625,
      "activations/layer21_attention_weight_min": -18.66097068786621,
      "activations/layer22_attention_weight_max": 43.30058670043945,
      "activations/layer22_attention_weight_min": -26.751476287841797,
      "activations/layer23_attention_weight_max": 27.116897583007812,
      "activations/layer23_attention_weight_min": -17.25181770324707,
      "activations/layer2_attention_weight_max": 28.40782928466797,
      "activations/layer2_attention_weight_min": -30.4117488861084,
      "activations/layer3_attention_weight_max": 72.25203704833984,
      "activations/layer3_attention_weight_min": -76.77324676513672,
      "activations/layer4_attention_weight_max": 134.96273803710938,
      "activations/layer4_attention_weight_min": -120.5599365234375,
      "activations/layer5_attention_weight_max": 108.87706756591797,
      "activations/layer5_attention_weight_min": -93.52421569824219,
      "activations/layer6_attention_weight_max": 58.53531265258789,
      "activations/layer6_attention_weight_min": -54.57282257080078,
      "activations/layer7_attention_weight_max": 71.21184539794922,
      "activations/layer7_attention_weight_min": -66.58739471435547,
      "activations/layer8_attention_weight_max": 42.31989288330078,
      "activations/layer8_attention_weight_min": -37.87118911743164,
      "activations/layer9_attention_weight_max": 55.3361701965332,
      "activations/layer9_attention_weight_min": -53.31727981567383,
      "epoch": 2.44,
      "learning_rate": 0.00013562613636363636,
      "loss": 2.9191,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_loss": 2.88671875,
      "eval_runtime": 8.512,
      "eval_samples_per_second": 504.463,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_openwebtext_loss": 2.88671875,
      "eval_openwebtext_ppl": 17.934365811960372,
      "eval_openwebtext_runtime": 8.512,
      "eval_openwebtext_samples_per_second": 504.463,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_wikitext_loss": 3.154296875,
      "eval_wikitext_ppl": 23.4365524707797,
      "eval_wikitext_runtime": 2.0153,
      "eval_wikitext_samples_per_second": 226.266,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_lambada_loss": 2.95703125,
      "eval_lambada_ppl": 19.24076585887165,
      "eval_lambada_runtime": 9.5822,
      "eval_lambada_samples_per_second": 508.132,
      "step": 42000
    },
    {
      "activations/layer0_attention_weight_max": 16.503326416015625,
      "activations/layer0_attention_weight_min": -11.711647033691406,
      "activations/layer10_attention_weight_max": 46.84954833984375,
      "activations/layer10_attention_weight_min": -36.139076232910156,
      "activations/layer11_attention_weight_max": 39.061279296875,
      "activations/layer11_attention_weight_min": -29.876419067382812,
      "activations/layer12_attention_weight_max": 30.36635971069336,
      "activations/layer12_attention_weight_min": -27.74376678466797,
      "activations/layer13_attention_weight_max": 50.93042755126953,
      "activations/layer13_attention_weight_min": -35.15723419189453,
      "activations/layer14_attention_weight_max": 80.2313003540039,
      "activations/layer14_attention_weight_min": -59.199886322021484,
      "activations/layer15_attention_weight_max": 49.6631965637207,
      "activations/layer15_attention_weight_min": -35.93992233276367,
      "activations/layer16_attention_weight_max": 46.572086334228516,
      "activations/layer16_attention_weight_min": -34.927188873291016,
      "activations/layer17_attention_weight_max": 41.63222885131836,
      "activations/layer17_attention_weight_min": -26.828502655029297,
      "activations/layer18_attention_weight_max": 28.07813835144043,
      "activations/layer18_attention_weight_min": -18.44272804260254,
      "activations/layer19_attention_weight_max": 51.59580612182617,
      "activations/layer19_attention_weight_min": -31.91531753540039,
      "activations/layer1_attention_weight_max": 18.0865478515625,
      "activations/layer1_attention_weight_min": -12.987312316894531,
      "activations/layer20_attention_weight_max": 34.71623229980469,
      "activations/layer20_attention_weight_min": -23.50751495361328,
      "activations/layer21_attention_weight_max": 30.36907196044922,
      "activations/layer21_attention_weight_min": -21.508657455444336,
      "activations/layer22_attention_weight_max": 45.20109176635742,
      "activations/layer22_attention_weight_min": -26.07161521911621,
      "activations/layer23_attention_weight_max": 26.247364044189453,
      "activations/layer23_attention_weight_min": -16.895164489746094,
      "activations/layer2_attention_weight_max": 28.12116241455078,
      "activations/layer2_attention_weight_min": -28.888805389404297,
      "activations/layer3_attention_weight_max": 76.36431121826172,
      "activations/layer3_attention_weight_min": -77.66796875,
      "activations/layer4_attention_weight_max": 123.20787811279297,
      "activations/layer4_attention_weight_min": -120.42339324951172,
      "activations/layer5_attention_weight_max": 110.0910873413086,
      "activations/layer5_attention_weight_min": -100.90721130371094,
      "activations/layer6_attention_weight_max": 60.227298736572266,
      "activations/layer6_attention_weight_min": -51.37532043457031,
      "activations/layer7_attention_weight_max": 67.02650451660156,
      "activations/layer7_attention_weight_min": -65.00455474853516,
      "activations/layer8_attention_weight_max": 42.902225494384766,
      "activations/layer8_attention_weight_min": -38.642372131347656,
      "activations/layer9_attention_weight_max": 54.70853805541992,
      "activations/layer9_attention_weight_min": -43.653385162353516,
      "epoch": 2.44,
      "learning_rate": 0.00013560719696969695,
      "loss": 2.9294,
      "step": 42050
    },
    {
      "activations/layer0_attention_weight_max": 15.427604675292969,
      "activations/layer0_attention_weight_min": -11.671683311462402,
      "activations/layer10_attention_weight_max": 56.06157684326172,
      "activations/layer10_attention_weight_min": -40.438987731933594,
      "activations/layer11_attention_weight_max": 45.923133850097656,
      "activations/layer11_attention_weight_min": -32.63373565673828,
      "activations/layer12_attention_weight_max": 62.59210968017578,
      "activations/layer12_attention_weight_min": -28.286455154418945,
      "activations/layer13_attention_weight_max": 59.673431396484375,
      "activations/layer13_attention_weight_min": -35.010555267333984,
      "activations/layer14_attention_weight_max": 103.92554473876953,
      "activations/layer14_attention_weight_min": -62.83986282348633,
      "activations/layer15_attention_weight_max": 54.02217102050781,
      "activations/layer15_attention_weight_min": -35.608158111572266,
      "activations/layer16_attention_weight_max": 60.3367919921875,
      "activations/layer16_attention_weight_min": -37.75187301635742,
      "activations/layer17_attention_weight_max": 41.71662139892578,
      "activations/layer17_attention_weight_min": -27.645370483398438,
      "activations/layer18_attention_weight_max": 32.42024612426758,
      "activations/layer18_attention_weight_min": -17.89771270751953,
      "activations/layer19_attention_weight_max": 61.476600646972656,
      "activations/layer19_attention_weight_min": -32.71644973754883,
      "activations/layer1_attention_weight_max": 18.087570190429688,
      "activations/layer1_attention_weight_min": -11.903666496276855,
      "activations/layer20_attention_weight_max": 40.21510314941406,
      "activations/layer20_attention_weight_min": -25.886388778686523,
      "activations/layer21_attention_weight_max": 29.3277645111084,
      "activations/layer21_attention_weight_min": -21.347042083740234,
      "activations/layer22_attention_weight_max": 44.79885482788086,
      "activations/layer22_attention_weight_min": -26.29262924194336,
      "activations/layer23_attention_weight_max": 25.777706146240234,
      "activations/layer23_attention_weight_min": -15.780196189880371,
      "activations/layer2_attention_weight_max": 28.386350631713867,
      "activations/layer2_attention_weight_min": -27.549015045166016,
      "activations/layer3_attention_weight_max": 75.82909393310547,
      "activations/layer3_attention_weight_min": -78.62814331054688,
      "activations/layer4_attention_weight_max": 132.7977752685547,
      "activations/layer4_attention_weight_min": -121.90044403076172,
      "activations/layer5_attention_weight_max": 116.82479858398438,
      "activations/layer5_attention_weight_min": -97.15736389160156,
      "activations/layer6_attention_weight_max": 66.49882507324219,
      "activations/layer6_attention_weight_min": -53.80363845825195,
      "activations/layer7_attention_weight_max": 77.4482421875,
      "activations/layer7_attention_weight_min": -68.07440948486328,
      "activations/layer8_attention_weight_max": 50.84843444824219,
      "activations/layer8_attention_weight_min": -38.47846221923828,
      "activations/layer9_attention_weight_max": 69.4063720703125,
      "activations/layer9_attention_weight_min": -48.467864990234375,
      "epoch": 2.45,
      "learning_rate": 0.00013558825757575756,
      "loss": 2.936,
      "step": 42100
    },
    {
      "activations/layer0_attention_weight_max": 16.313989639282227,
      "activations/layer0_attention_weight_min": -11.895624160766602,
      "activations/layer10_attention_weight_max": 46.36569595336914,
      "activations/layer10_attention_weight_min": -37.683998107910156,
      "activations/layer11_attention_weight_max": 38.50173568725586,
      "activations/layer11_attention_weight_min": -31.963603973388672,
      "activations/layer12_attention_weight_max": 31.285417556762695,
      "activations/layer12_attention_weight_min": -28.51384162902832,
      "activations/layer13_attention_weight_max": 46.38344955444336,
      "activations/layer13_attention_weight_min": -35.5428581237793,
      "activations/layer14_attention_weight_max": 71.09262084960938,
      "activations/layer14_attention_weight_min": -49.081764221191406,
      "activations/layer15_attention_weight_max": 46.63288497924805,
      "activations/layer15_attention_weight_min": -32.21904373168945,
      "activations/layer16_attention_weight_max": 43.028846740722656,
      "activations/layer16_attention_weight_min": -34.53920364379883,
      "activations/layer17_attention_weight_max": 48.936344146728516,
      "activations/layer17_attention_weight_min": -28.81260871887207,
      "activations/layer18_attention_weight_max": 31.140464782714844,
      "activations/layer18_attention_weight_min": -20.767370223999023,
      "activations/layer19_attention_weight_max": 49.604122161865234,
      "activations/layer19_attention_weight_min": -30.60223388671875,
      "activations/layer1_attention_weight_max": 18.602144241333008,
      "activations/layer1_attention_weight_min": -10.991572380065918,
      "activations/layer20_attention_weight_max": 37.57138442993164,
      "activations/layer20_attention_weight_min": -23.4366455078125,
      "activations/layer21_attention_weight_max": 32.78764343261719,
      "activations/layer21_attention_weight_min": -18.9814395904541,
      "activations/layer22_attention_weight_max": 44.735591888427734,
      "activations/layer22_attention_weight_min": -28.236251831054688,
      "activations/layer23_attention_weight_max": 24.936359405517578,
      "activations/layer23_attention_weight_min": -16.88400650024414,
      "activations/layer2_attention_weight_max": 30.473804473876953,
      "activations/layer2_attention_weight_min": -30.105819702148438,
      "activations/layer3_attention_weight_max": 75.23249053955078,
      "activations/layer3_attention_weight_min": -77.18135833740234,
      "activations/layer4_attention_weight_max": 126.7249984741211,
      "activations/layer4_attention_weight_min": -124.21263885498047,
      "activations/layer5_attention_weight_max": 110.8321533203125,
      "activations/layer5_attention_weight_min": -103.37033081054688,
      "activations/layer6_attention_weight_max": 59.49137496948242,
      "activations/layer6_attention_weight_min": -55.36044692993164,
      "activations/layer7_attention_weight_max": 67.7448501586914,
      "activations/layer7_attention_weight_min": -65.22119903564453,
      "activations/layer8_attention_weight_max": 40.34466552734375,
      "activations/layer8_attention_weight_min": -37.3590087890625,
      "activations/layer9_attention_weight_max": 53.89121627807617,
      "activations/layer9_attention_weight_min": -41.64872741699219,
      "epoch": 2.45,
      "learning_rate": 0.00013556931818181818,
      "loss": 2.9486,
      "step": 42150
    },
    {
      "activations/layer0_attention_weight_max": 15.776047706604004,
      "activations/layer0_attention_weight_min": -11.389294624328613,
      "activations/layer10_attention_weight_max": 47.686767578125,
      "activations/layer10_attention_weight_min": -40.87717819213867,
      "activations/layer11_attention_weight_max": 36.34513854980469,
      "activations/layer11_attention_weight_min": -31.23562240600586,
      "activations/layer12_attention_weight_max": 50.3759880065918,
      "activations/layer12_attention_weight_min": -32.14384841918945,
      "activations/layer13_attention_weight_max": 43.42766571044922,
      "activations/layer13_attention_weight_min": -35.537254333496094,
      "activations/layer14_attention_weight_max": 65.65018463134766,
      "activations/layer14_attention_weight_min": -57.545005798339844,
      "activations/layer15_attention_weight_max": 46.52608871459961,
      "activations/layer15_attention_weight_min": -34.18976974487305,
      "activations/layer16_attention_weight_max": 44.23765563964844,
      "activations/layer16_attention_weight_min": -33.153282165527344,
      "activations/layer17_attention_weight_max": 44.343135833740234,
      "activations/layer17_attention_weight_min": -28.91985511779785,
      "activations/layer18_attention_weight_max": 34.77986145019531,
      "activations/layer18_attention_weight_min": -22.727205276489258,
      "activations/layer19_attention_weight_max": 45.96317672729492,
      "activations/layer19_attention_weight_min": -31.870946884155273,
      "activations/layer1_attention_weight_max": 17.329978942871094,
      "activations/layer1_attention_weight_min": -12.04513168334961,
      "activations/layer20_attention_weight_max": 36.81196594238281,
      "activations/layer20_attention_weight_min": -23.77338218688965,
      "activations/layer21_attention_weight_max": 31.665828704833984,
      "activations/layer21_attention_weight_min": -19.844070434570312,
      "activations/layer22_attention_weight_max": 45.401485443115234,
      "activations/layer22_attention_weight_min": -24.29966926574707,
      "activations/layer23_attention_weight_max": 25.52161407470703,
      "activations/layer23_attention_weight_min": -17.243148803710938,
      "activations/layer2_attention_weight_max": 28.824689865112305,
      "activations/layer2_attention_weight_min": -27.290569305419922,
      "activations/layer3_attention_weight_max": 72.08192443847656,
      "activations/layer3_attention_weight_min": -74.36746215820312,
      "activations/layer4_attention_weight_max": 128.6061553955078,
      "activations/layer4_attention_weight_min": -118.18450927734375,
      "activations/layer5_attention_weight_max": 104.37560272216797,
      "activations/layer5_attention_weight_min": -94.8999252319336,
      "activations/layer6_attention_weight_max": 59.87411880493164,
      "activations/layer6_attention_weight_min": -50.66025924682617,
      "activations/layer7_attention_weight_max": 65.83405303955078,
      "activations/layer7_attention_weight_min": -63.30699157714844,
      "activations/layer8_attention_weight_max": 41.619049072265625,
      "activations/layer8_attention_weight_min": -36.65193557739258,
      "activations/layer9_attention_weight_max": 58.65928268432617,
      "activations/layer9_attention_weight_min": -45.54021453857422,
      "epoch": 2.45,
      "learning_rate": 0.00013555037878787876,
      "loss": 2.9442,
      "step": 42200
    },
    {
      "activations/layer0_attention_weight_max": 15.692351341247559,
      "activations/layer0_attention_weight_min": -11.401876449584961,
      "activations/layer10_attention_weight_max": 49.30994415283203,
      "activations/layer10_attention_weight_min": -40.520843505859375,
      "activations/layer11_attention_weight_max": 44.068870544433594,
      "activations/layer11_attention_weight_min": -36.99527359008789,
      "activations/layer12_attention_weight_max": 39.079505920410156,
      "activations/layer12_attention_weight_min": -28.17452621459961,
      "activations/layer13_attention_weight_max": 46.64420700073242,
      "activations/layer13_attention_weight_min": -35.2994499206543,
      "activations/layer14_attention_weight_max": 72.18867492675781,
      "activations/layer14_attention_weight_min": -51.596435546875,
      "activations/layer15_attention_weight_max": 50.69047546386719,
      "activations/layer15_attention_weight_min": -37.13532638549805,
      "activations/layer16_attention_weight_max": 50.8476676940918,
      "activations/layer16_attention_weight_min": -34.8784065246582,
      "activations/layer17_attention_weight_max": 47.11989212036133,
      "activations/layer17_attention_weight_min": -29.736507415771484,
      "activations/layer18_attention_weight_max": 33.62705612182617,
      "activations/layer18_attention_weight_min": -18.927778244018555,
      "activations/layer19_attention_weight_max": 59.107154846191406,
      "activations/layer19_attention_weight_min": -33.696449279785156,
      "activations/layer1_attention_weight_max": 17.924341201782227,
      "activations/layer1_attention_weight_min": -12.417824745178223,
      "activations/layer20_attention_weight_max": 38.13837432861328,
      "activations/layer20_attention_weight_min": -26.621164321899414,
      "activations/layer21_attention_weight_max": 40.1015625,
      "activations/layer21_attention_weight_min": -21.579345703125,
      "activations/layer22_attention_weight_max": 52.26791763305664,
      "activations/layer22_attention_weight_min": -27.194580078125,
      "activations/layer23_attention_weight_max": 29.07229995727539,
      "activations/layer23_attention_weight_min": -16.36588478088379,
      "activations/layer2_attention_weight_max": 30.583118438720703,
      "activations/layer2_attention_weight_min": -29.096004486083984,
      "activations/layer3_attention_weight_max": 76.40509796142578,
      "activations/layer3_attention_weight_min": -76.26016998291016,
      "activations/layer4_attention_weight_max": 132.78836059570312,
      "activations/layer4_attention_weight_min": -122.8038101196289,
      "activations/layer5_attention_weight_max": 115.19911193847656,
      "activations/layer5_attention_weight_min": -99.57887268066406,
      "activations/layer6_attention_weight_max": 63.839141845703125,
      "activations/layer6_attention_weight_min": -53.451866149902344,
      "activations/layer7_attention_weight_max": 74.47295379638672,
      "activations/layer7_attention_weight_min": -70.21165466308594,
      "activations/layer8_attention_weight_max": 46.506141662597656,
      "activations/layer8_attention_weight_min": -39.702632904052734,
      "activations/layer9_attention_weight_max": 67.88257598876953,
      "activations/layer9_attention_weight_min": -50.23453903198242,
      "epoch": 2.45,
      "learning_rate": 0.00013553143939393938,
      "loss": 2.9291,
      "step": 42250
    },
    {
      "activations/layer0_attention_weight_max": 15.962366104125977,
      "activations/layer0_attention_weight_min": -11.929583549499512,
      "activations/layer10_attention_weight_max": 49.20167922973633,
      "activations/layer10_attention_weight_min": -43.04170608520508,
      "activations/layer11_attention_weight_max": 44.99992370605469,
      "activations/layer11_attention_weight_min": -39.40531921386719,
      "activations/layer12_attention_weight_max": 32.475547790527344,
      "activations/layer12_attention_weight_min": -25.521499633789062,
      "activations/layer13_attention_weight_max": 45.745750427246094,
      "activations/layer13_attention_weight_min": -34.55517578125,
      "activations/layer14_attention_weight_max": 68.19937133789062,
      "activations/layer14_attention_weight_min": -60.20823287963867,
      "activations/layer15_attention_weight_max": 47.70663833618164,
      "activations/layer15_attention_weight_min": -36.32418441772461,
      "activations/layer16_attention_weight_max": 48.566650390625,
      "activations/layer16_attention_weight_min": -34.56040954589844,
      "activations/layer17_attention_weight_max": 41.8111457824707,
      "activations/layer17_attention_weight_min": -27.613161087036133,
      "activations/layer18_attention_weight_max": 25.309528350830078,
      "activations/layer18_attention_weight_min": -18.997892379760742,
      "activations/layer19_attention_weight_max": 51.256996154785156,
      "activations/layer19_attention_weight_min": -30.232542037963867,
      "activations/layer1_attention_weight_max": 17.508373260498047,
      "activations/layer1_attention_weight_min": -11.081974029541016,
      "activations/layer20_attention_weight_max": 37.76090621948242,
      "activations/layer20_attention_weight_min": -24.069950103759766,
      "activations/layer21_attention_weight_max": 24.232452392578125,
      "activations/layer21_attention_weight_min": -18.609909057617188,
      "activations/layer22_attention_weight_max": 45.28064727783203,
      "activations/layer22_attention_weight_min": -25.901643753051758,
      "activations/layer23_attention_weight_max": 25.760356903076172,
      "activations/layer23_attention_weight_min": -17.380582809448242,
      "activations/layer2_attention_weight_max": 30.494258880615234,
      "activations/layer2_attention_weight_min": -28.846092224121094,
      "activations/layer3_attention_weight_max": 74.84428405761719,
      "activations/layer3_attention_weight_min": -74.3078384399414,
      "activations/layer4_attention_weight_max": 130.46339416503906,
      "activations/layer4_attention_weight_min": -113.63875579833984,
      "activations/layer5_attention_weight_max": 114.39430236816406,
      "activations/layer5_attention_weight_min": -93.2483139038086,
      "activations/layer6_attention_weight_max": 61.45795440673828,
      "activations/layer6_attention_weight_min": -51.18171691894531,
      "activations/layer7_attention_weight_max": 74.32009887695312,
      "activations/layer7_attention_weight_min": -62.66423416137695,
      "activations/layer8_attention_weight_max": 49.130496978759766,
      "activations/layer8_attention_weight_min": -38.94583511352539,
      "activations/layer9_attention_weight_max": 61.45821762084961,
      "activations/layer9_attention_weight_min": -53.171485900878906,
      "epoch": 2.46,
      "learning_rate": 0.0001355125,
      "loss": 2.9256,
      "step": 42300
    },
    {
      "activations/layer0_attention_weight_max": 16.436450958251953,
      "activations/layer0_attention_weight_min": -11.372023582458496,
      "activations/layer10_attention_weight_max": 50.15118408203125,
      "activations/layer10_attention_weight_min": -41.7806282043457,
      "activations/layer11_attention_weight_max": 40.76249694824219,
      "activations/layer11_attention_weight_min": -35.402713775634766,
      "activations/layer12_attention_weight_max": 32.73296356201172,
      "activations/layer12_attention_weight_min": -26.429529190063477,
      "activations/layer13_attention_weight_max": 54.7249641418457,
      "activations/layer13_attention_weight_min": -35.54462432861328,
      "activations/layer14_attention_weight_max": 100.04875183105469,
      "activations/layer14_attention_weight_min": -62.99182891845703,
      "activations/layer15_attention_weight_max": 50.19574737548828,
      "activations/layer15_attention_weight_min": -33.895355224609375,
      "activations/layer16_attention_weight_max": 49.30651092529297,
      "activations/layer16_attention_weight_min": -32.46961975097656,
      "activations/layer17_attention_weight_max": 45.64973831176758,
      "activations/layer17_attention_weight_min": -26.820205688476562,
      "activations/layer18_attention_weight_max": 30.773717880249023,
      "activations/layer18_attention_weight_min": -18.40703773498535,
      "activations/layer19_attention_weight_max": 46.43870162963867,
      "activations/layer19_attention_weight_min": -29.187509536743164,
      "activations/layer1_attention_weight_max": 16.5074405670166,
      "activations/layer1_attention_weight_min": -12.872191429138184,
      "activations/layer20_attention_weight_max": 39.11720275878906,
      "activations/layer20_attention_weight_min": -24.66927719116211,
      "activations/layer21_attention_weight_max": 37.775352478027344,
      "activations/layer21_attention_weight_min": -21.12383460998535,
      "activations/layer22_attention_weight_max": 47.77442169189453,
      "activations/layer22_attention_weight_min": -27.990102767944336,
      "activations/layer23_attention_weight_max": 24.77203369140625,
      "activations/layer23_attention_weight_min": -16.375492095947266,
      "activations/layer2_attention_weight_max": 29.945432662963867,
      "activations/layer2_attention_weight_min": -29.870819091796875,
      "activations/layer3_attention_weight_max": 73.84577178955078,
      "activations/layer3_attention_weight_min": -76.88721466064453,
      "activations/layer4_attention_weight_max": 130.98867797851562,
      "activations/layer4_attention_weight_min": -116.951904296875,
      "activations/layer5_attention_weight_max": 109.30644226074219,
      "activations/layer5_attention_weight_min": -93.68669891357422,
      "activations/layer6_attention_weight_max": 60.23764419555664,
      "activations/layer6_attention_weight_min": -54.286441802978516,
      "activations/layer7_attention_weight_max": 65.6319808959961,
      "activations/layer7_attention_weight_min": -66.84302520751953,
      "activations/layer8_attention_weight_max": 46.47978973388672,
      "activations/layer8_attention_weight_min": -35.74445343017578,
      "activations/layer9_attention_weight_max": 54.7498664855957,
      "activations/layer9_attention_weight_min": -43.915042877197266,
      "epoch": 2.46,
      "learning_rate": 0.0001354935606060606,
      "loss": 2.9386,
      "step": 42350
    },
    {
      "activations/layer0_attention_weight_max": 16.483680725097656,
      "activations/layer0_attention_weight_min": -11.292071342468262,
      "activations/layer10_attention_weight_max": 49.4493408203125,
      "activations/layer10_attention_weight_min": -43.165252685546875,
      "activations/layer11_attention_weight_max": 38.52885437011719,
      "activations/layer11_attention_weight_min": -34.447715759277344,
      "activations/layer12_attention_weight_max": 31.785083770751953,
      "activations/layer12_attention_weight_min": -26.980966567993164,
      "activations/layer13_attention_weight_max": 47.46757125854492,
      "activations/layer13_attention_weight_min": -37.63557815551758,
      "activations/layer14_attention_weight_max": 64.1738052368164,
      "activations/layer14_attention_weight_min": -51.398536682128906,
      "activations/layer15_attention_weight_max": 49.76251220703125,
      "activations/layer15_attention_weight_min": -40.80150604248047,
      "activations/layer16_attention_weight_max": 44.90956497192383,
      "activations/layer16_attention_weight_min": -32.72127914428711,
      "activations/layer17_attention_weight_max": 40.25117874145508,
      "activations/layer17_attention_weight_min": -32.38715362548828,
      "activations/layer18_attention_weight_max": 31.48835563659668,
      "activations/layer18_attention_weight_min": -19.461284637451172,
      "activations/layer19_attention_weight_max": 46.864681243896484,
      "activations/layer19_attention_weight_min": -31.70367431640625,
      "activations/layer1_attention_weight_max": 17.261919021606445,
      "activations/layer1_attention_weight_min": -11.4938383102417,
      "activations/layer20_attention_weight_max": 35.11602783203125,
      "activations/layer20_attention_weight_min": -23.492021560668945,
      "activations/layer21_attention_weight_max": 30.85593605041504,
      "activations/layer21_attention_weight_min": -20.586090087890625,
      "activations/layer22_attention_weight_max": 46.80938720703125,
      "activations/layer22_attention_weight_min": -25.844676971435547,
      "activations/layer23_attention_weight_max": 25.72418975830078,
      "activations/layer23_attention_weight_min": -18.177013397216797,
      "activations/layer2_attention_weight_max": 28.341079711914062,
      "activations/layer2_attention_weight_min": -26.660213470458984,
      "activations/layer3_attention_weight_max": 73.25114440917969,
      "activations/layer3_attention_weight_min": -76.1153335571289,
      "activations/layer4_attention_weight_max": 128.67221069335938,
      "activations/layer4_attention_weight_min": -127.5258560180664,
      "activations/layer5_attention_weight_max": 112.19204711914062,
      "activations/layer5_attention_weight_min": -104.30169677734375,
      "activations/layer6_attention_weight_max": 60.564788818359375,
      "activations/layer6_attention_weight_min": -58.4525032043457,
      "activations/layer7_attention_weight_max": 65.34406280517578,
      "activations/layer7_attention_weight_min": -68.23172760009766,
      "activations/layer8_attention_weight_max": 43.7569694519043,
      "activations/layer8_attention_weight_min": -40.49026107788086,
      "activations/layer9_attention_weight_max": 75.16787719726562,
      "activations/layer9_attention_weight_min": -49.6084098815918,
      "epoch": 2.46,
      "learning_rate": 0.0001354746212121212,
      "loss": 2.9151,
      "step": 42400
    },
    {
      "activations/layer0_attention_weight_max": 16.398405075073242,
      "activations/layer0_attention_weight_min": -11.12789535522461,
      "activations/layer10_attention_weight_max": 50.7672004699707,
      "activations/layer10_attention_weight_min": -37.28801345825195,
      "activations/layer11_attention_weight_max": 49.197933197021484,
      "activations/layer11_attention_weight_min": -33.756046295166016,
      "activations/layer12_attention_weight_max": 53.546512603759766,
      "activations/layer12_attention_weight_min": -31.282100677490234,
      "activations/layer13_attention_weight_max": 75.54627990722656,
      "activations/layer13_attention_weight_min": -45.400455474853516,
      "activations/layer14_attention_weight_max": 95.73616027832031,
      "activations/layer14_attention_weight_min": -57.061058044433594,
      "activations/layer15_attention_weight_max": 54.423099517822266,
      "activations/layer15_attention_weight_min": -32.13920974731445,
      "activations/layer16_attention_weight_max": 51.27238082885742,
      "activations/layer16_attention_weight_min": -33.308353424072266,
      "activations/layer17_attention_weight_max": 50.37049865722656,
      "activations/layer17_attention_weight_min": -26.589937210083008,
      "activations/layer18_attention_weight_max": 33.65183639526367,
      "activations/layer18_attention_weight_min": -17.838783264160156,
      "activations/layer19_attention_weight_max": 50.34798049926758,
      "activations/layer19_attention_weight_min": -31.231658935546875,
      "activations/layer1_attention_weight_max": 17.486419677734375,
      "activations/layer1_attention_weight_min": -11.32349681854248,
      "activations/layer20_attention_weight_max": 38.51987075805664,
      "activations/layer20_attention_weight_min": -23.137022018432617,
      "activations/layer21_attention_weight_max": 33.00052261352539,
      "activations/layer21_attention_weight_min": -21.618932723999023,
      "activations/layer22_attention_weight_max": 50.5172233581543,
      "activations/layer22_attention_weight_min": -25.594161987304688,
      "activations/layer23_attention_weight_max": 27.136245727539062,
      "activations/layer23_attention_weight_min": -17.580612182617188,
      "activations/layer2_attention_weight_max": 27.957380294799805,
      "activations/layer2_attention_weight_min": -24.979755401611328,
      "activations/layer3_attention_weight_max": 60.714317321777344,
      "activations/layer3_attention_weight_min": -67.56307220458984,
      "activations/layer4_attention_weight_max": 121.62589263916016,
      "activations/layer4_attention_weight_min": -129.0177459716797,
      "activations/layer5_attention_weight_max": 115.05342864990234,
      "activations/layer5_attention_weight_min": -116.56904602050781,
      "activations/layer6_attention_weight_max": 58.8618049621582,
      "activations/layer6_attention_weight_min": -59.62937927246094,
      "activations/layer7_attention_weight_max": 67.57879638671875,
      "activations/layer7_attention_weight_min": -73.80648803710938,
      "activations/layer8_attention_weight_max": 41.1871337890625,
      "activations/layer8_attention_weight_min": -36.78656768798828,
      "activations/layer9_attention_weight_max": 70.5687026977539,
      "activations/layer9_attention_weight_min": -54.69575119018555,
      "epoch": 2.47,
      "learning_rate": 0.0001354556818181818,
      "loss": 2.9642,
      "step": 42450
    },
    {
      "activations/layer0_attention_weight_max": 15.741613388061523,
      "activations/layer0_attention_weight_min": -15.557573318481445,
      "activations/layer10_attention_weight_max": 44.70344924926758,
      "activations/layer10_attention_weight_min": -40.59014892578125,
      "activations/layer11_attention_weight_max": 38.737728118896484,
      "activations/layer11_attention_weight_min": -34.02289581298828,
      "activations/layer12_attention_weight_max": 29.938644409179688,
      "activations/layer12_attention_weight_min": -29.28614044189453,
      "activations/layer13_attention_weight_max": 55.15641403198242,
      "activations/layer13_attention_weight_min": -37.96458053588867,
      "activations/layer14_attention_weight_max": 84.3281478881836,
      "activations/layer14_attention_weight_min": -62.02883529663086,
      "activations/layer15_attention_weight_max": 45.71759796142578,
      "activations/layer15_attention_weight_min": -33.201316833496094,
      "activations/layer16_attention_weight_max": 48.285888671875,
      "activations/layer16_attention_weight_min": -35.24010467529297,
      "activations/layer17_attention_weight_max": 42.02690505981445,
      "activations/layer17_attention_weight_min": -26.805208206176758,
      "activations/layer18_attention_weight_max": 30.432453155517578,
      "activations/layer18_attention_weight_min": -18.8050479888916,
      "activations/layer19_attention_weight_max": 45.397193908691406,
      "activations/layer19_attention_weight_min": -31.907678604125977,
      "activations/layer1_attention_weight_max": 17.56300163269043,
      "activations/layer1_attention_weight_min": -13.004714012145996,
      "activations/layer20_attention_weight_max": 34.56925964355469,
      "activations/layer20_attention_weight_min": -24.918861389160156,
      "activations/layer21_attention_weight_max": 33.1660041809082,
      "activations/layer21_attention_weight_min": -20.008190155029297,
      "activations/layer22_attention_weight_max": 44.605289459228516,
      "activations/layer22_attention_weight_min": -28.52373695373535,
      "activations/layer23_attention_weight_max": 26.531957626342773,
      "activations/layer23_attention_weight_min": -17.341829299926758,
      "activations/layer2_attention_weight_max": 28.878665924072266,
      "activations/layer2_attention_weight_min": -26.354736328125,
      "activations/layer3_attention_weight_max": 65.3571548461914,
      "activations/layer3_attention_weight_min": -67.90064239501953,
      "activations/layer4_attention_weight_max": 122.00626373291016,
      "activations/layer4_attention_weight_min": -108.2512435913086,
      "activations/layer5_attention_weight_max": 107.62826538085938,
      "activations/layer5_attention_weight_min": -88.63497161865234,
      "activations/layer6_attention_weight_max": 61.52458953857422,
      "activations/layer6_attention_weight_min": -49.766990661621094,
      "activations/layer7_attention_weight_max": 73.23954772949219,
      "activations/layer7_attention_weight_min": -67.17881774902344,
      "activations/layer8_attention_weight_max": 42.662567138671875,
      "activations/layer8_attention_weight_min": -38.453285217285156,
      "activations/layer9_attention_weight_max": 58.83944320678711,
      "activations/layer9_attention_weight_min": -45.910186767578125,
      "epoch": 2.47,
      "learning_rate": 0.00013543674242424243,
      "loss": 2.9993,
      "step": 42500
    },
    {
      "activations/layer0_attention_weight_max": 14.939787864685059,
      "activations/layer0_attention_weight_min": -13.952820777893066,
      "activations/layer10_attention_weight_max": 47.19584655761719,
      "activations/layer10_attention_weight_min": -37.5673828125,
      "activations/layer11_attention_weight_max": 35.96723937988281,
      "activations/layer11_attention_weight_min": -34.51506042480469,
      "activations/layer12_attention_weight_max": 35.82204055786133,
      "activations/layer12_attention_weight_min": -31.913572311401367,
      "activations/layer13_attention_weight_max": 47.525123596191406,
      "activations/layer13_attention_weight_min": -38.01300811767578,
      "activations/layer14_attention_weight_max": 66.29801940917969,
      "activations/layer14_attention_weight_min": -55.03559494018555,
      "activations/layer15_attention_weight_max": 44.84505081176758,
      "activations/layer15_attention_weight_min": -33.4022102355957,
      "activations/layer16_attention_weight_max": 44.57704162597656,
      "activations/layer16_attention_weight_min": -33.5172004699707,
      "activations/layer17_attention_weight_max": 47.57777786254883,
      "activations/layer17_attention_weight_min": -29.796735763549805,
      "activations/layer18_attention_weight_max": 25.052675247192383,
      "activations/layer18_attention_weight_min": -18.562294006347656,
      "activations/layer19_attention_weight_max": 48.68783187866211,
      "activations/layer19_attention_weight_min": -31.110319137573242,
      "activations/layer1_attention_weight_max": 17.83454132080078,
      "activations/layer1_attention_weight_min": -12.37272834777832,
      "activations/layer20_attention_weight_max": 34.26865005493164,
      "activations/layer20_attention_weight_min": -24.526832580566406,
      "activations/layer21_attention_weight_max": 28.260087966918945,
      "activations/layer21_attention_weight_min": -20.072174072265625,
      "activations/layer22_attention_weight_max": 42.69721603393555,
      "activations/layer22_attention_weight_min": -29.209627151489258,
      "activations/layer23_attention_weight_max": 23.49980926513672,
      "activations/layer23_attention_weight_min": -15.13980484008789,
      "activations/layer2_attention_weight_max": 27.90020751953125,
      "activations/layer2_attention_weight_min": -27.948410034179688,
      "activations/layer3_attention_weight_max": 71.01016235351562,
      "activations/layer3_attention_weight_min": -73.86964416503906,
      "activations/layer4_attention_weight_max": 122.40557098388672,
      "activations/layer4_attention_weight_min": -107.4802474975586,
      "activations/layer5_attention_weight_max": 106.77490234375,
      "activations/layer5_attention_weight_min": -88.03103637695312,
      "activations/layer6_attention_weight_max": 57.37890625,
      "activations/layer6_attention_weight_min": -51.12284851074219,
      "activations/layer7_attention_weight_max": 69.89763641357422,
      "activations/layer7_attention_weight_min": -62.02717208862305,
      "activations/layer8_attention_weight_max": 42.15985107421875,
      "activations/layer8_attention_weight_min": -35.99739456176758,
      "activations/layer9_attention_weight_max": 66.37525939941406,
      "activations/layer9_attention_weight_min": -55.64752960205078,
      "epoch": 2.47,
      "learning_rate": 0.00013541780303030301,
      "loss": 2.9411,
      "step": 42550
    },
    {
      "activations/layer0_attention_weight_max": 15.032679557800293,
      "activations/layer0_attention_weight_min": -13.227458000183105,
      "activations/layer10_attention_weight_max": 48.49896240234375,
      "activations/layer10_attention_weight_min": -39.17689514160156,
      "activations/layer11_attention_weight_max": 43.860939025878906,
      "activations/layer11_attention_weight_min": -35.25092315673828,
      "activations/layer12_attention_weight_max": 31.530776977539062,
      "activations/layer12_attention_weight_min": -27.111160278320312,
      "activations/layer13_attention_weight_max": 48.75295639038086,
      "activations/layer13_attention_weight_min": -38.42061996459961,
      "activations/layer14_attention_weight_max": 73.22057342529297,
      "activations/layer14_attention_weight_min": -59.10054397583008,
      "activations/layer15_attention_weight_max": 50.339454650878906,
      "activations/layer15_attention_weight_min": -35.248329162597656,
      "activations/layer16_attention_weight_max": 47.054141998291016,
      "activations/layer16_attention_weight_min": -33.92605972290039,
      "activations/layer17_attention_weight_max": 45.25252151489258,
      "activations/layer17_attention_weight_min": -26.840269088745117,
      "activations/layer18_attention_weight_max": 27.416057586669922,
      "activations/layer18_attention_weight_min": -17.349618911743164,
      "activations/layer19_attention_weight_max": 47.85675048828125,
      "activations/layer19_attention_weight_min": -30.2272891998291,
      "activations/layer1_attention_weight_max": 17.356094360351562,
      "activations/layer1_attention_weight_min": -12.028489112854004,
      "activations/layer20_attention_weight_max": 37.86503219604492,
      "activations/layer20_attention_weight_min": -24.54258918762207,
      "activations/layer21_attention_weight_max": 33.2262077331543,
      "activations/layer21_attention_weight_min": -20.577112197875977,
      "activations/layer22_attention_weight_max": 43.75923156738281,
      "activations/layer22_attention_weight_min": -26.81617546081543,
      "activations/layer23_attention_weight_max": 24.741191864013672,
      "activations/layer23_attention_weight_min": -16.35881233215332,
      "activations/layer2_attention_weight_max": 28.23462677001953,
      "activations/layer2_attention_weight_min": -28.189626693725586,
      "activations/layer3_attention_weight_max": 68.81429290771484,
      "activations/layer3_attention_weight_min": -72.43082427978516,
      "activations/layer4_attention_weight_max": 121.23949432373047,
      "activations/layer4_attention_weight_min": -112.33715057373047,
      "activations/layer5_attention_weight_max": 105.88357543945312,
      "activations/layer5_attention_weight_min": -91.03541564941406,
      "activations/layer6_attention_weight_max": 61.3487663269043,
      "activations/layer6_attention_weight_min": -48.483680725097656,
      "activations/layer7_attention_weight_max": 66.88172149658203,
      "activations/layer7_attention_weight_min": -63.14643096923828,
      "activations/layer8_attention_weight_max": 44.83275604248047,
      "activations/layer8_attention_weight_min": -40.266971588134766,
      "activations/layer9_attention_weight_max": 59.338783264160156,
      "activations/layer9_attention_weight_min": -54.96806716918945,
      "epoch": 2.48,
      "learning_rate": 0.00013539886363636363,
      "loss": 2.953,
      "step": 42600
    },
    {
      "activations/layer0_attention_weight_max": 15.60351276397705,
      "activations/layer0_attention_weight_min": -12.550912857055664,
      "activations/layer10_attention_weight_max": 44.92293930053711,
      "activations/layer10_attention_weight_min": -38.79586410522461,
      "activations/layer11_attention_weight_max": 42.34735870361328,
      "activations/layer11_attention_weight_min": -36.439605712890625,
      "activations/layer12_attention_weight_max": 34.35309982299805,
      "activations/layer12_attention_weight_min": -30.815311431884766,
      "activations/layer13_attention_weight_max": 48.82630920410156,
      "activations/layer13_attention_weight_min": -36.83966827392578,
      "activations/layer14_attention_weight_max": 88.2225112915039,
      "activations/layer14_attention_weight_min": -59.02814865112305,
      "activations/layer15_attention_weight_max": 49.976837158203125,
      "activations/layer15_attention_weight_min": -36.032039642333984,
      "activations/layer16_attention_weight_max": 54.131229400634766,
      "activations/layer16_attention_weight_min": -35.030399322509766,
      "activations/layer17_attention_weight_max": 47.44585418701172,
      "activations/layer17_attention_weight_min": -27.906776428222656,
      "activations/layer18_attention_weight_max": 28.642656326293945,
      "activations/layer18_attention_weight_min": -17.062301635742188,
      "activations/layer19_attention_weight_max": 52.15871047973633,
      "activations/layer19_attention_weight_min": -30.029443740844727,
      "activations/layer1_attention_weight_max": 17.96942901611328,
      "activations/layer1_attention_weight_min": -14.339470863342285,
      "activations/layer20_attention_weight_max": 34.96781921386719,
      "activations/layer20_attention_weight_min": -24.709199905395508,
      "activations/layer21_attention_weight_max": 34.43112564086914,
      "activations/layer21_attention_weight_min": -20.551193237304688,
      "activations/layer22_attention_weight_max": 47.84565734863281,
      "activations/layer22_attention_weight_min": -26.547910690307617,
      "activations/layer23_attention_weight_max": 25.553585052490234,
      "activations/layer23_attention_weight_min": -15.56619644165039,
      "activations/layer2_attention_weight_max": 29.418190002441406,
      "activations/layer2_attention_weight_min": -27.923818588256836,
      "activations/layer3_attention_weight_max": 72.57832336425781,
      "activations/layer3_attention_weight_min": -73.77960968017578,
      "activations/layer4_attention_weight_max": 130.7300567626953,
      "activations/layer4_attention_weight_min": -120.64949035644531,
      "activations/layer5_attention_weight_max": 107.55288696289062,
      "activations/layer5_attention_weight_min": -97.2632064819336,
      "activations/layer6_attention_weight_max": 59.365821838378906,
      "activations/layer6_attention_weight_min": -52.27231216430664,
      "activations/layer7_attention_weight_max": 69.76771545410156,
      "activations/layer7_attention_weight_min": -69.7363510131836,
      "activations/layer8_attention_weight_max": 42.998313903808594,
      "activations/layer8_attention_weight_min": -41.41836166381836,
      "activations/layer9_attention_weight_max": 73.63752746582031,
      "activations/layer9_attention_weight_min": -62.74755859375,
      "epoch": 2.48,
      "learning_rate": 0.00013537992424242424,
      "loss": 2.9335,
      "step": 42650
    },
    {
      "activations/layer0_attention_weight_max": 15.864052772521973,
      "activations/layer0_attention_weight_min": -12.433833122253418,
      "activations/layer10_attention_weight_max": 53.79431915283203,
      "activations/layer10_attention_weight_min": -39.67360305786133,
      "activations/layer11_attention_weight_max": 37.58565902709961,
      "activations/layer11_attention_weight_min": -34.44148254394531,
      "activations/layer12_attention_weight_max": 71.58805084228516,
      "activations/layer12_attention_weight_min": -32.7964973449707,
      "activations/layer13_attention_weight_max": 76.75096893310547,
      "activations/layer13_attention_weight_min": -42.102500915527344,
      "activations/layer14_attention_weight_max": 87.44808197021484,
      "activations/layer14_attention_weight_min": -57.524837493896484,
      "activations/layer15_attention_weight_max": 50.31709671020508,
      "activations/layer15_attention_weight_min": -33.450870513916016,
      "activations/layer16_attention_weight_max": 49.545223236083984,
      "activations/layer16_attention_weight_min": -34.054405212402344,
      "activations/layer17_attention_weight_max": 53.18402862548828,
      "activations/layer17_attention_weight_min": -29.342952728271484,
      "activations/layer18_attention_weight_max": 28.615055084228516,
      "activations/layer18_attention_weight_min": -19.167383193969727,
      "activations/layer19_attention_weight_max": 52.0609245300293,
      "activations/layer19_attention_weight_min": -29.1629695892334,
      "activations/layer1_attention_weight_max": 18.907487869262695,
      "activations/layer1_attention_weight_min": -12.583161354064941,
      "activations/layer20_attention_weight_max": 37.61924362182617,
      "activations/layer20_attention_weight_min": -22.41455841064453,
      "activations/layer21_attention_weight_max": 29.617467880249023,
      "activations/layer21_attention_weight_min": -19.035730361938477,
      "activations/layer22_attention_weight_max": 52.9208869934082,
      "activations/layer22_attention_weight_min": -24.813737869262695,
      "activations/layer23_attention_weight_max": 27.646896362304688,
      "activations/layer23_attention_weight_min": -15.387389183044434,
      "activations/layer2_attention_weight_max": 28.797536849975586,
      "activations/layer2_attention_weight_min": -29.225439071655273,
      "activations/layer3_attention_weight_max": 72.94231414794922,
      "activations/layer3_attention_weight_min": -75.47789764404297,
      "activations/layer4_attention_weight_max": 134.45396423339844,
      "activations/layer4_attention_weight_min": -118.861083984375,
      "activations/layer5_attention_weight_max": 113.28121948242188,
      "activations/layer5_attention_weight_min": -93.58985900878906,
      "activations/layer6_attention_weight_max": 61.3068962097168,
      "activations/layer6_attention_weight_min": -50.622005462646484,
      "activations/layer7_attention_weight_max": 77.46968078613281,
      "activations/layer7_attention_weight_min": -72.49473571777344,
      "activations/layer8_attention_weight_max": 50.41980743408203,
      "activations/layer8_attention_weight_min": -37.36160659790039,
      "activations/layer9_attention_weight_max": 73.2987289428711,
      "activations/layer9_attention_weight_min": -59.34836959838867,
      "epoch": 2.48,
      "learning_rate": 0.00013536098484848483,
      "loss": 2.9512,
      "step": 42700
    },
    {
      "activations/layer0_attention_weight_max": 15.69941234588623,
      "activations/layer0_attention_weight_min": -12.209465980529785,
      "activations/layer10_attention_weight_max": 47.08924102783203,
      "activations/layer10_attention_weight_min": -38.36412048339844,
      "activations/layer11_attention_weight_max": 42.01411819458008,
      "activations/layer11_attention_weight_min": -36.652198791503906,
      "activations/layer12_attention_weight_max": 45.75798416137695,
      "activations/layer12_attention_weight_min": -31.403953552246094,
      "activations/layer13_attention_weight_max": 52.882144927978516,
      "activations/layer13_attention_weight_min": -39.6303596496582,
      "activations/layer14_attention_weight_max": 68.10636901855469,
      "activations/layer14_attention_weight_min": -52.96270751953125,
      "activations/layer15_attention_weight_max": 46.0787353515625,
      "activations/layer15_attention_weight_min": -33.58256530761719,
      "activations/layer16_attention_weight_max": 49.06093978881836,
      "activations/layer16_attention_weight_min": -36.53194046020508,
      "activations/layer17_attention_weight_max": 41.118438720703125,
      "activations/layer17_attention_weight_min": -29.066524505615234,
      "activations/layer18_attention_weight_max": 32.57215118408203,
      "activations/layer18_attention_weight_min": -20.03328514099121,
      "activations/layer19_attention_weight_max": 52.8463249206543,
      "activations/layer19_attention_weight_min": -31.688207626342773,
      "activations/layer1_attention_weight_max": 18.407743453979492,
      "activations/layer1_attention_weight_min": -12.522266387939453,
      "activations/layer20_attention_weight_max": 39.663936614990234,
      "activations/layer20_attention_weight_min": -23.853303909301758,
      "activations/layer21_attention_weight_max": 30.43319320678711,
      "activations/layer21_attention_weight_min": -19.07585906982422,
      "activations/layer22_attention_weight_max": 49.37836456298828,
      "activations/layer22_attention_weight_min": -27.36501121520996,
      "activations/layer23_attention_weight_max": 29.518545150756836,
      "activations/layer23_attention_weight_min": -19.138120651245117,
      "activations/layer2_attention_weight_max": 28.98575210571289,
      "activations/layer2_attention_weight_min": -27.54372215270996,
      "activations/layer3_attention_weight_max": 70.1273193359375,
      "activations/layer3_attention_weight_min": -70.8058090209961,
      "activations/layer4_attention_weight_max": 128.88758850097656,
      "activations/layer4_attention_weight_min": -112.66641998291016,
      "activations/layer5_attention_weight_max": 116.96797180175781,
      "activations/layer5_attention_weight_min": -93.17913055419922,
      "activations/layer6_attention_weight_max": 66.67504119873047,
      "activations/layer6_attention_weight_min": -49.79352951049805,
      "activations/layer7_attention_weight_max": 69.70587158203125,
      "activations/layer7_attention_weight_min": -64.97119903564453,
      "activations/layer8_attention_weight_max": 47.74018859863281,
      "activations/layer8_attention_weight_min": -38.190528869628906,
      "activations/layer9_attention_weight_max": 64.82555389404297,
      "activations/layer9_attention_weight_min": -56.50739288330078,
      "epoch": 2.48,
      "learning_rate": 0.00013534204545454545,
      "loss": 2.9289,
      "step": 42750
    },
    {
      "activations/layer0_attention_weight_max": 15.649724960327148,
      "activations/layer0_attention_weight_min": -12.059319496154785,
      "activations/layer10_attention_weight_max": 50.13367462158203,
      "activations/layer10_attention_weight_min": -39.96992874145508,
      "activations/layer11_attention_weight_max": 37.8265495300293,
      "activations/layer11_attention_weight_min": -32.06390380859375,
      "activations/layer12_attention_weight_max": 32.91645812988281,
      "activations/layer12_attention_weight_min": -26.85923957824707,
      "activations/layer13_attention_weight_max": 49.89198303222656,
      "activations/layer13_attention_weight_min": -37.18807601928711,
      "activations/layer14_attention_weight_max": 75.48982238769531,
      "activations/layer14_attention_weight_min": -59.6087646484375,
      "activations/layer15_attention_weight_max": 56.73598861694336,
      "activations/layer15_attention_weight_min": -36.41819763183594,
      "activations/layer16_attention_weight_max": 52.37065887451172,
      "activations/layer16_attention_weight_min": -32.41987609863281,
      "activations/layer17_attention_weight_max": 44.55792236328125,
      "activations/layer17_attention_weight_min": -27.288843154907227,
      "activations/layer18_attention_weight_max": 29.028427124023438,
      "activations/layer18_attention_weight_min": -18.58416748046875,
      "activations/layer19_attention_weight_max": 52.02842330932617,
      "activations/layer19_attention_weight_min": -33.54029846191406,
      "activations/layer1_attention_weight_max": 16.786264419555664,
      "activations/layer1_attention_weight_min": -11.988229751586914,
      "activations/layer20_attention_weight_max": 37.457576751708984,
      "activations/layer20_attention_weight_min": -25.219520568847656,
      "activations/layer21_attention_weight_max": 29.4859619140625,
      "activations/layer21_attention_weight_min": -22.40973472595215,
      "activations/layer22_attention_weight_max": 49.161739349365234,
      "activations/layer22_attention_weight_min": -26.3676815032959,
      "activations/layer23_attention_weight_max": 30.441055297851562,
      "activations/layer23_attention_weight_min": -17.188026428222656,
      "activations/layer2_attention_weight_max": 28.36372947692871,
      "activations/layer2_attention_weight_min": -28.283050537109375,
      "activations/layer3_attention_weight_max": 71.20088958740234,
      "activations/layer3_attention_weight_min": -74.43130493164062,
      "activations/layer4_attention_weight_max": 128.0817413330078,
      "activations/layer4_attention_weight_min": -122.92400360107422,
      "activations/layer5_attention_weight_max": 111.74260711669922,
      "activations/layer5_attention_weight_min": -99.76594543457031,
      "activations/layer6_attention_weight_max": 59.53354263305664,
      "activations/layer6_attention_weight_min": -54.90378952026367,
      "activations/layer7_attention_weight_max": 69.1888656616211,
      "activations/layer7_attention_weight_min": -64.29866027832031,
      "activations/layer8_attention_weight_max": 43.6677360534668,
      "activations/layer8_attention_weight_min": -40.03375244140625,
      "activations/layer9_attention_weight_max": 62.26251983642578,
      "activations/layer9_attention_weight_min": -45.29466247558594,
      "epoch": 2.49,
      "learning_rate": 0.00013532310606060606,
      "loss": 2.9421,
      "step": 42800
    },
    {
      "activations/layer0_attention_weight_max": 16.089380264282227,
      "activations/layer0_attention_weight_min": -11.61853313446045,
      "activations/layer10_attention_weight_max": 57.85275650024414,
      "activations/layer10_attention_weight_min": -43.35511016845703,
      "activations/layer11_attention_weight_max": 43.46140670776367,
      "activations/layer11_attention_weight_min": -34.579036712646484,
      "activations/layer12_attention_weight_max": 39.788002014160156,
      "activations/layer12_attention_weight_min": -29.468935012817383,
      "activations/layer13_attention_weight_max": 59.20984649658203,
      "activations/layer13_attention_weight_min": -38.17686080932617,
      "activations/layer14_attention_weight_max": 102.38395690917969,
      "activations/layer14_attention_weight_min": -64.88616180419922,
      "activations/layer15_attention_weight_max": 54.3427619934082,
      "activations/layer15_attention_weight_min": -36.95624542236328,
      "activations/layer16_attention_weight_max": 57.81041717529297,
      "activations/layer16_attention_weight_min": -37.04765701293945,
      "activations/layer17_attention_weight_max": 48.842594146728516,
      "activations/layer17_attention_weight_min": -31.25995635986328,
      "activations/layer18_attention_weight_max": 36.20309829711914,
      "activations/layer18_attention_weight_min": -18.70738410949707,
      "activations/layer19_attention_weight_max": 52.09552001953125,
      "activations/layer19_attention_weight_min": -32.94956970214844,
      "activations/layer1_attention_weight_max": 17.712297439575195,
      "activations/layer1_attention_weight_min": -12.955537796020508,
      "activations/layer20_attention_weight_max": 41.75611114501953,
      "activations/layer20_attention_weight_min": -24.990636825561523,
      "activations/layer21_attention_weight_max": 35.415931701660156,
      "activations/layer21_attention_weight_min": -23.588098526000977,
      "activations/layer22_attention_weight_max": 58.02962875366211,
      "activations/layer22_attention_weight_min": -29.081989288330078,
      "activations/layer23_attention_weight_max": 30.94778060913086,
      "activations/layer23_attention_weight_min": -15.693220138549805,
      "activations/layer2_attention_weight_max": 31.325101852416992,
      "activations/layer2_attention_weight_min": -30.8216552734375,
      "activations/layer3_attention_weight_max": 75.47535705566406,
      "activations/layer3_attention_weight_min": -79.49113464355469,
      "activations/layer4_attention_weight_max": 131.48733520507812,
      "activations/layer4_attention_weight_min": -123.20735168457031,
      "activations/layer5_attention_weight_max": 114.12102508544922,
      "activations/layer5_attention_weight_min": -101.01982116699219,
      "activations/layer6_attention_weight_max": 61.653717041015625,
      "activations/layer6_attention_weight_min": -53.738033294677734,
      "activations/layer7_attention_weight_max": 73.76309967041016,
      "activations/layer7_attention_weight_min": -66.88825225830078,
      "activations/layer8_attention_weight_max": 46.87917709350586,
      "activations/layer8_attention_weight_min": -39.23137664794922,
      "activations/layer9_attention_weight_max": 71.48894500732422,
      "activations/layer9_attention_weight_min": -49.10506057739258,
      "epoch": 2.49,
      "learning_rate": 0.00013530416666666665,
      "loss": 2.934,
      "step": 42850
    },
    {
      "activations/layer0_attention_weight_max": 15.930861473083496,
      "activations/layer0_attention_weight_min": -11.442777633666992,
      "activations/layer10_attention_weight_max": 50.444515228271484,
      "activations/layer10_attention_weight_min": -39.26216125488281,
      "activations/layer11_attention_weight_max": 44.0986328125,
      "activations/layer11_attention_weight_min": -34.43149948120117,
      "activations/layer12_attention_weight_max": 30.01250648498535,
      "activations/layer12_attention_weight_min": -29.498306274414062,
      "activations/layer13_attention_weight_max": 48.04201889038086,
      "activations/layer13_attention_weight_min": -36.65571594238281,
      "activations/layer14_attention_weight_max": 86.0476303100586,
      "activations/layer14_attention_weight_min": -63.135887145996094,
      "activations/layer15_attention_weight_max": 48.253971099853516,
      "activations/layer15_attention_weight_min": -33.46876907348633,
      "activations/layer16_attention_weight_max": 48.704315185546875,
      "activations/layer16_attention_weight_min": -34.709285736083984,
      "activations/layer17_attention_weight_max": 47.17634582519531,
      "activations/layer17_attention_weight_min": -27.474130630493164,
      "activations/layer18_attention_weight_max": 29.661104202270508,
      "activations/layer18_attention_weight_min": -18.816696166992188,
      "activations/layer19_attention_weight_max": 50.75729751586914,
      "activations/layer19_attention_weight_min": -33.94296646118164,
      "activations/layer1_attention_weight_max": 17.147443771362305,
      "activations/layer1_attention_weight_min": -11.751111030578613,
      "activations/layer20_attention_weight_max": 40.17412567138672,
      "activations/layer20_attention_weight_min": -23.31656265258789,
      "activations/layer21_attention_weight_max": 31.162166595458984,
      "activations/layer21_attention_weight_min": -18.835004806518555,
      "activations/layer22_attention_weight_max": 46.49736404418945,
      "activations/layer22_attention_weight_min": -29.019683837890625,
      "activations/layer23_attention_weight_max": 26.83673667907715,
      "activations/layer23_attention_weight_min": -16.921855926513672,
      "activations/layer2_attention_weight_max": 28.99585723876953,
      "activations/layer2_attention_weight_min": -28.683780670166016,
      "activations/layer3_attention_weight_max": 72.9439697265625,
      "activations/layer3_attention_weight_min": -76.02281951904297,
      "activations/layer4_attention_weight_max": 132.186767578125,
      "activations/layer4_attention_weight_min": -122.0755386352539,
      "activations/layer5_attention_weight_max": 107.78695678710938,
      "activations/layer5_attention_weight_min": -94.13043212890625,
      "activations/layer6_attention_weight_max": 58.00617218017578,
      "activations/layer6_attention_weight_min": -52.39088821411133,
      "activations/layer7_attention_weight_max": 65.76404571533203,
      "activations/layer7_attention_weight_min": -62.677101135253906,
      "activations/layer8_attention_weight_max": 47.096031188964844,
      "activations/layer8_attention_weight_min": -35.78208541870117,
      "activations/layer9_attention_weight_max": 63.217830657958984,
      "activations/layer9_attention_weight_min": -49.55802536010742,
      "epoch": 2.49,
      "learning_rate": 0.00013528522727272727,
      "loss": 2.9374,
      "step": 42900
    },
    {
      "activations/layer0_attention_weight_max": 15.884578704833984,
      "activations/layer0_attention_weight_min": -11.572914123535156,
      "activations/layer10_attention_weight_max": 48.58707046508789,
      "activations/layer10_attention_weight_min": -37.8580322265625,
      "activations/layer11_attention_weight_max": 40.424888610839844,
      "activations/layer11_attention_weight_min": -33.61460494995117,
      "activations/layer12_attention_weight_max": 40.58194351196289,
      "activations/layer12_attention_weight_min": -29.234399795532227,
      "activations/layer13_attention_weight_max": 42.83664321899414,
      "activations/layer13_attention_weight_min": -34.86847686767578,
      "activations/layer14_attention_weight_max": 67.55191040039062,
      "activations/layer14_attention_weight_min": -53.3029899597168,
      "activations/layer15_attention_weight_max": 48.51201629638672,
      "activations/layer15_attention_weight_min": -34.442054748535156,
      "activations/layer16_attention_weight_max": 43.82597732543945,
      "activations/layer16_attention_weight_min": -33.76831817626953,
      "activations/layer17_attention_weight_max": 43.10751724243164,
      "activations/layer17_attention_weight_min": -28.98952865600586,
      "activations/layer18_attention_weight_max": 29.109426498413086,
      "activations/layer18_attention_weight_min": -19.184024810791016,
      "activations/layer19_attention_weight_max": 44.120506286621094,
      "activations/layer19_attention_weight_min": -31.6707763671875,
      "activations/layer1_attention_weight_max": 18.20316505432129,
      "activations/layer1_attention_weight_min": -12.034542083740234,
      "activations/layer20_attention_weight_max": 37.960506439208984,
      "activations/layer20_attention_weight_min": -23.123050689697266,
      "activations/layer21_attention_weight_max": 34.2998046875,
      "activations/layer21_attention_weight_min": -23.221765518188477,
      "activations/layer22_attention_weight_max": 51.58747100830078,
      "activations/layer22_attention_weight_min": -27.49671745300293,
      "activations/layer23_attention_weight_max": 25.171483993530273,
      "activations/layer23_attention_weight_min": -16.101680755615234,
      "activations/layer2_attention_weight_max": 29.852338790893555,
      "activations/layer2_attention_weight_min": -29.926982879638672,
      "activations/layer3_attention_weight_max": 73.89100646972656,
      "activations/layer3_attention_weight_min": -77.94966888427734,
      "activations/layer4_attention_weight_max": 133.52076721191406,
      "activations/layer4_attention_weight_min": -119.958984375,
      "activations/layer5_attention_weight_max": 118.30064392089844,
      "activations/layer5_attention_weight_min": -96.75611877441406,
      "activations/layer6_attention_weight_max": 62.37382888793945,
      "activations/layer6_attention_weight_min": -50.386146545410156,
      "activations/layer7_attention_weight_max": 67.05878448486328,
      "activations/layer7_attention_weight_min": -61.61292266845703,
      "activations/layer8_attention_weight_max": 45.20263671875,
      "activations/layer8_attention_weight_min": -34.27324295043945,
      "activations/layer9_attention_weight_max": 65.51854705810547,
      "activations/layer9_attention_weight_min": -49.99974822998047,
      "epoch": 2.5,
      "learning_rate": 0.00013526628787878785,
      "loss": 2.9373,
      "step": 42950
    },
    {
      "activations/layer0_attention_weight_max": 15.636922836303711,
      "activations/layer0_attention_weight_min": -11.108983993530273,
      "activations/layer10_attention_weight_max": 48.98637390136719,
      "activations/layer10_attention_weight_min": -40.311519622802734,
      "activations/layer11_attention_weight_max": 40.04216384887695,
      "activations/layer11_attention_weight_min": -34.10165786743164,
      "activations/layer12_attention_weight_max": 32.33283233642578,
      "activations/layer12_attention_weight_min": -25.90760040283203,
      "activations/layer13_attention_weight_max": 50.4088020324707,
      "activations/layer13_attention_weight_min": -41.67683410644531,
      "activations/layer14_attention_weight_max": 73.47003936767578,
      "activations/layer14_attention_weight_min": -57.990821838378906,
      "activations/layer15_attention_weight_max": 53.89876174926758,
      "activations/layer15_attention_weight_min": -36.32113265991211,
      "activations/layer16_attention_weight_max": 45.10447311401367,
      "activations/layer16_attention_weight_min": -33.486202239990234,
      "activations/layer17_attention_weight_max": 44.357418060302734,
      "activations/layer17_attention_weight_min": -28.90398597717285,
      "activations/layer18_attention_weight_max": 28.697458267211914,
      "activations/layer18_attention_weight_min": -18.554616928100586,
      "activations/layer19_attention_weight_max": 46.03401184082031,
      "activations/layer19_attention_weight_min": -31.122638702392578,
      "activations/layer1_attention_weight_max": 17.58823013305664,
      "activations/layer1_attention_weight_min": -12.10483169555664,
      "activations/layer20_attention_weight_max": 37.980064392089844,
      "activations/layer20_attention_weight_min": -22.928688049316406,
      "activations/layer21_attention_weight_max": 28.19567108154297,
      "activations/layer21_attention_weight_min": -21.1879940032959,
      "activations/layer22_attention_weight_max": 44.16287612915039,
      "activations/layer22_attention_weight_min": -26.3591365814209,
      "activations/layer23_attention_weight_max": 28.75446891784668,
      "activations/layer23_attention_weight_min": -16.49556541442871,
      "activations/layer2_attention_weight_max": 29.78523826599121,
      "activations/layer2_attention_weight_min": -28.970111846923828,
      "activations/layer3_attention_weight_max": 75.09122467041016,
      "activations/layer3_attention_weight_min": -77.38262176513672,
      "activations/layer4_attention_weight_max": 129.80592346191406,
      "activations/layer4_attention_weight_min": -122.76964569091797,
      "activations/layer5_attention_weight_max": 112.2894515991211,
      "activations/layer5_attention_weight_min": -95.90770721435547,
      "activations/layer6_attention_weight_max": 56.4953498840332,
      "activations/layer6_attention_weight_min": -52.211082458496094,
      "activations/layer7_attention_weight_max": 67.33869934082031,
      "activations/layer7_attention_weight_min": -65.69538879394531,
      "activations/layer8_attention_weight_max": 39.45930099487305,
      "activations/layer8_attention_weight_min": -36.37300491333008,
      "activations/layer9_attention_weight_max": 58.76045608520508,
      "activations/layer9_attention_weight_min": -50.46988296508789,
      "epoch": 2.5,
      "learning_rate": 0.00013524734848484847,
      "loss": 2.9394,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_loss": 2.884765625,
      "eval_runtime": 9.5701,
      "eval_samples_per_second": 448.689,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_openwebtext_loss": 2.884765625,
      "eval_openwebtext_ppl": 17.89937193856263,
      "eval_openwebtext_runtime": 9.5701,
      "eval_openwebtext_samples_per_second": 448.689,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_wikitext_loss": 3.126953125,
      "eval_wikitext_ppl": 22.80439145296227,
      "eval_wikitext_runtime": 2.0267,
      "eval_wikitext_samples_per_second": 224.997,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_lambada_loss": 2.93359375,
      "eval_lambada_ppl": 18.79505399780919,
      "eval_lambada_runtime": 9.6011,
      "eval_lambada_samples_per_second": 507.13,
      "step": 43000
    },
    {
      "activations/layer0_attention_weight_max": 16.360200881958008,
      "activations/layer0_attention_weight_min": -11.268019676208496,
      "activations/layer10_attention_weight_max": 50.26420593261719,
      "activations/layer10_attention_weight_min": -42.685150146484375,
      "activations/layer11_attention_weight_max": 47.40720748901367,
      "activations/layer11_attention_weight_min": -34.42694854736328,
      "activations/layer12_attention_weight_max": 49.45317459106445,
      "activations/layer12_attention_weight_min": -30.344417572021484,
      "activations/layer13_attention_weight_max": 53.83223342895508,
      "activations/layer13_attention_weight_min": -37.12006378173828,
      "activations/layer14_attention_weight_max": 86.5474624633789,
      "activations/layer14_attention_weight_min": -63.875816345214844,
      "activations/layer15_attention_weight_max": 47.94979476928711,
      "activations/layer15_attention_weight_min": -32.87481689453125,
      "activations/layer16_attention_weight_max": 48.36369323730469,
      "activations/layer16_attention_weight_min": -31.84777069091797,
      "activations/layer17_attention_weight_max": 49.146820068359375,
      "activations/layer17_attention_weight_min": -28.986690521240234,
      "activations/layer18_attention_weight_max": 33.20112228393555,
      "activations/layer18_attention_weight_min": -18.80559539794922,
      "activations/layer19_attention_weight_max": 59.7666015625,
      "activations/layer19_attention_weight_min": -29.329578399658203,
      "activations/layer1_attention_weight_max": 18.347614288330078,
      "activations/layer1_attention_weight_min": -11.553288459777832,
      "activations/layer20_attention_weight_max": 39.18248748779297,
      "activations/layer20_attention_weight_min": -22.597082138061523,
      "activations/layer21_attention_weight_max": 33.27124786376953,
      "activations/layer21_attention_weight_min": -19.178760528564453,
      "activations/layer22_attention_weight_max": 51.20008087158203,
      "activations/layer22_attention_weight_min": -24.135347366333008,
      "activations/layer23_attention_weight_max": 28.205223083496094,
      "activations/layer23_attention_weight_min": -15.069936752319336,
      "activations/layer2_attention_weight_max": 31.60344696044922,
      "activations/layer2_attention_weight_min": -31.849353790283203,
      "activations/layer3_attention_weight_max": 86.64236450195312,
      "activations/layer3_attention_weight_min": -86.99335479736328,
      "activations/layer4_attention_weight_max": 147.59349060058594,
      "activations/layer4_attention_weight_min": -134.3289794921875,
      "activations/layer5_attention_weight_max": 133.46034240722656,
      "activations/layer5_attention_weight_min": -114.0765380859375,
      "activations/layer6_attention_weight_max": 74.06190490722656,
      "activations/layer6_attention_weight_min": -59.24256896972656,
      "activations/layer7_attention_weight_max": 80.74925994873047,
      "activations/layer7_attention_weight_min": -74.08549499511719,
      "activations/layer8_attention_weight_max": 51.12543487548828,
      "activations/layer8_attention_weight_min": -39.0487060546875,
      "activations/layer9_attention_weight_max": 73.17384338378906,
      "activations/layer9_attention_weight_min": -67.04332733154297,
      "epoch": 2.5,
      "learning_rate": 0.00013522840909090908,
      "loss": 2.945,
      "step": 43050
    },
    {
      "activations/layer0_attention_weight_max": 16.104585647583008,
      "activations/layer0_attention_weight_min": -10.911381721496582,
      "activations/layer10_attention_weight_max": 42.30344772338867,
      "activations/layer10_attention_weight_min": -38.65625,
      "activations/layer11_attention_weight_max": 35.34629821777344,
      "activations/layer11_attention_weight_min": -31.18445587158203,
      "activations/layer12_attention_weight_max": 30.18218231201172,
      "activations/layer12_attention_weight_min": -28.115955352783203,
      "activations/layer13_attention_weight_max": 41.82661437988281,
      "activations/layer13_attention_weight_min": -34.41376876831055,
      "activations/layer14_attention_weight_max": 68.85513305664062,
      "activations/layer14_attention_weight_min": -55.660831451416016,
      "activations/layer15_attention_weight_max": 48.59763717651367,
      "activations/layer15_attention_weight_min": -36.899776458740234,
      "activations/layer16_attention_weight_max": 44.20580291748047,
      "activations/layer16_attention_weight_min": -32.51894760131836,
      "activations/layer17_attention_weight_max": 43.83172607421875,
      "activations/layer17_attention_weight_min": -29.46193504333496,
      "activations/layer18_attention_weight_max": 27.7479305267334,
      "activations/layer18_attention_weight_min": -23.259201049804688,
      "activations/layer19_attention_weight_max": 54.299312591552734,
      "activations/layer19_attention_weight_min": -32.04020309448242,
      "activations/layer1_attention_weight_max": 16.714603424072266,
      "activations/layer1_attention_weight_min": -12.085983276367188,
      "activations/layer20_attention_weight_max": 35.31487274169922,
      "activations/layer20_attention_weight_min": -27.150768280029297,
      "activations/layer21_attention_weight_max": 32.6861457824707,
      "activations/layer21_attention_weight_min": -20.491092681884766,
      "activations/layer22_attention_weight_max": 47.59760284423828,
      "activations/layer22_attention_weight_min": -29.358938217163086,
      "activations/layer23_attention_weight_max": 30.328481674194336,
      "activations/layer23_attention_weight_min": -16.856590270996094,
      "activations/layer2_attention_weight_max": 29.79745864868164,
      "activations/layer2_attention_weight_min": -29.821029663085938,
      "activations/layer3_attention_weight_max": 72.05590057373047,
      "activations/layer3_attention_weight_min": -75.41773986816406,
      "activations/layer4_attention_weight_max": 125.3050765991211,
      "activations/layer4_attention_weight_min": -117.07933807373047,
      "activations/layer5_attention_weight_max": 103.40486907958984,
      "activations/layer5_attention_weight_min": -90.41045379638672,
      "activations/layer6_attention_weight_max": 56.111907958984375,
      "activations/layer6_attention_weight_min": -51.58518981933594,
      "activations/layer7_attention_weight_max": 64.73004913330078,
      "activations/layer7_attention_weight_min": -61.03887176513672,
      "activations/layer8_attention_weight_max": 40.98496627807617,
      "activations/layer8_attention_weight_min": -34.051124572753906,
      "activations/layer9_attention_weight_max": 63.13209915161133,
      "activations/layer9_attention_weight_min": -50.61323928833008,
      "epoch": 2.5,
      "learning_rate": 0.00013520946969696967,
      "loss": 2.943,
      "step": 43100
    },
    {
      "activations/layer0_attention_weight_max": 14.598995208740234,
      "activations/layer0_attention_weight_min": -11.177298545837402,
      "activations/layer10_attention_weight_max": 59.42527770996094,
      "activations/layer10_attention_weight_min": -45.82036209106445,
      "activations/layer11_attention_weight_max": 40.7631721496582,
      "activations/layer11_attention_weight_min": -33.75096130371094,
      "activations/layer12_attention_weight_max": 50.614620208740234,
      "activations/layer12_attention_weight_min": -42.15715408325195,
      "activations/layer13_attention_weight_max": 58.30815887451172,
      "activations/layer13_attention_weight_min": -46.84128189086914,
      "activations/layer14_attention_weight_max": 81.12640380859375,
      "activations/layer14_attention_weight_min": -61.53832244873047,
      "activations/layer15_attention_weight_max": 53.55756378173828,
      "activations/layer15_attention_weight_min": -35.312652587890625,
      "activations/layer16_attention_weight_max": 48.30563735961914,
      "activations/layer16_attention_weight_min": -36.9082145690918,
      "activations/layer17_attention_weight_max": 49.342369079589844,
      "activations/layer17_attention_weight_min": -29.8238468170166,
      "activations/layer18_attention_weight_max": 31.070180892944336,
      "activations/layer18_attention_weight_min": -17.99197006225586,
      "activations/layer19_attention_weight_max": 51.44203567504883,
      "activations/layer19_attention_weight_min": -29.808496475219727,
      "activations/layer1_attention_weight_max": 17.73409080505371,
      "activations/layer1_attention_weight_min": -11.582986831665039,
      "activations/layer20_attention_weight_max": 36.343475341796875,
      "activations/layer20_attention_weight_min": -22.1263370513916,
      "activations/layer21_attention_weight_max": 35.825435638427734,
      "activations/layer21_attention_weight_min": -19.177621841430664,
      "activations/layer22_attention_weight_max": 47.38174819946289,
      "activations/layer22_attention_weight_min": -26.613401412963867,
      "activations/layer23_attention_weight_max": 23.80361557006836,
      "activations/layer23_attention_weight_min": -19.539813995361328,
      "activations/layer2_attention_weight_max": 30.06149673461914,
      "activations/layer2_attention_weight_min": -29.263288497924805,
      "activations/layer3_attention_weight_max": 76.09981536865234,
      "activations/layer3_attention_weight_min": -76.4429931640625,
      "activations/layer4_attention_weight_max": 134.6280975341797,
      "activations/layer4_attention_weight_min": -117.01045227050781,
      "activations/layer5_attention_weight_max": 112.98845672607422,
      "activations/layer5_attention_weight_min": -93.17701721191406,
      "activations/layer6_attention_weight_max": 60.385196685791016,
      "activations/layer6_attention_weight_min": -53.17858123779297,
      "activations/layer7_attention_weight_max": 70.88031005859375,
      "activations/layer7_attention_weight_min": -64.83027648925781,
      "activations/layer8_attention_weight_max": 47.92923355102539,
      "activations/layer8_attention_weight_min": -37.43385696411133,
      "activations/layer9_attention_weight_max": 70.68915557861328,
      "activations/layer9_attention_weight_min": -55.80461502075195,
      "epoch": 2.51,
      "learning_rate": 0.0001351905303030303,
      "loss": 2.9206,
      "step": 43150
    },
    {
      "activations/layer0_attention_weight_max": 16.324995040893555,
      "activations/layer0_attention_weight_min": -11.104978561401367,
      "activations/layer10_attention_weight_max": 49.54828643798828,
      "activations/layer10_attention_weight_min": -39.19778823852539,
      "activations/layer11_attention_weight_max": 38.28030776977539,
      "activations/layer11_attention_weight_min": -31.90682029724121,
      "activations/layer12_attention_weight_max": 47.967952728271484,
      "activations/layer12_attention_weight_min": -28.917564392089844,
      "activations/layer13_attention_weight_max": 50.08861541748047,
      "activations/layer13_attention_weight_min": -39.97195053100586,
      "activations/layer14_attention_weight_max": 75.53395080566406,
      "activations/layer14_attention_weight_min": -59.63819122314453,
      "activations/layer15_attention_weight_max": 49.41706466674805,
      "activations/layer15_attention_weight_min": -36.9625358581543,
      "activations/layer16_attention_weight_max": 48.51620101928711,
      "activations/layer16_attention_weight_min": -33.79689407348633,
      "activations/layer17_attention_weight_max": 45.0998420715332,
      "activations/layer17_attention_weight_min": -27.840795516967773,
      "activations/layer18_attention_weight_max": 29.8619441986084,
      "activations/layer18_attention_weight_min": -18.798776626586914,
      "activations/layer19_attention_weight_max": 45.74799728393555,
      "activations/layer19_attention_weight_min": -31.36789321899414,
      "activations/layer1_attention_weight_max": 17.457862854003906,
      "activations/layer1_attention_weight_min": -10.903177261352539,
      "activations/layer20_attention_weight_max": 37.592952728271484,
      "activations/layer20_attention_weight_min": -24.32534408569336,
      "activations/layer21_attention_weight_max": 30.124359130859375,
      "activations/layer21_attention_weight_min": -19.19400978088379,
      "activations/layer22_attention_weight_max": 44.0174560546875,
      "activations/layer22_attention_weight_min": -27.601335525512695,
      "activations/layer23_attention_weight_max": 24.65834617614746,
      "activations/layer23_attention_weight_min": -17.252727508544922,
      "activations/layer2_attention_weight_max": 28.984466552734375,
      "activations/layer2_attention_weight_min": -28.465726852416992,
      "activations/layer3_attention_weight_max": 76.117431640625,
      "activations/layer3_attention_weight_min": -77.3620834350586,
      "activations/layer4_attention_weight_max": 133.48085021972656,
      "activations/layer4_attention_weight_min": -124.35250091552734,
      "activations/layer5_attention_weight_max": 111.1505355834961,
      "activations/layer5_attention_weight_min": -102.71682739257812,
      "activations/layer6_attention_weight_max": 62.046592712402344,
      "activations/layer6_attention_weight_min": -52.903907775878906,
      "activations/layer7_attention_weight_max": 71.70805358886719,
      "activations/layer7_attention_weight_min": -66.2688217163086,
      "activations/layer8_attention_weight_max": 45.443603515625,
      "activations/layer8_attention_weight_min": -39.34832000732422,
      "activations/layer9_attention_weight_max": 60.54651641845703,
      "activations/layer9_attention_weight_min": -52.30286407470703,
      "epoch": 2.51,
      "learning_rate": 0.0001351715909090909,
      "loss": 2.9391,
      "step": 43200
    },
    {
      "activations/layer0_attention_weight_max": 16.406269073486328,
      "activations/layer0_attention_weight_min": -11.114372253417969,
      "activations/layer10_attention_weight_max": 44.582489013671875,
      "activations/layer10_attention_weight_min": -37.240943908691406,
      "activations/layer11_attention_weight_max": 35.317073822021484,
      "activations/layer11_attention_weight_min": -28.94683837890625,
      "activations/layer12_attention_weight_max": 49.57219314575195,
      "activations/layer12_attention_weight_min": -29.246191024780273,
      "activations/layer13_attention_weight_max": 59.20859146118164,
      "activations/layer13_attention_weight_min": -46.08194351196289,
      "activations/layer14_attention_weight_max": 75.52411651611328,
      "activations/layer14_attention_weight_min": -59.443965911865234,
      "activations/layer15_attention_weight_max": 52.39006042480469,
      "activations/layer15_attention_weight_min": -36.729549407958984,
      "activations/layer16_attention_weight_max": 53.18334197998047,
      "activations/layer16_attention_weight_min": -33.42710876464844,
      "activations/layer17_attention_weight_max": 44.99281311035156,
      "activations/layer17_attention_weight_min": -31.992048263549805,
      "activations/layer18_attention_weight_max": 28.293781280517578,
      "activations/layer18_attention_weight_min": -19.408309936523438,
      "activations/layer19_attention_weight_max": 52.141231536865234,
      "activations/layer19_attention_weight_min": -32.7460823059082,
      "activations/layer1_attention_weight_max": 17.748926162719727,
      "activations/layer1_attention_weight_min": -11.503750801086426,
      "activations/layer20_attention_weight_max": 36.77500915527344,
      "activations/layer20_attention_weight_min": -27.0960693359375,
      "activations/layer21_attention_weight_max": 38.7558479309082,
      "activations/layer21_attention_weight_min": -22.940044403076172,
      "activations/layer22_attention_weight_max": 45.6888313293457,
      "activations/layer22_attention_weight_min": -31.996707916259766,
      "activations/layer23_attention_weight_max": 28.293357849121094,
      "activations/layer23_attention_weight_min": -16.463672637939453,
      "activations/layer2_attention_weight_max": 30.542282104492188,
      "activations/layer2_attention_weight_min": -28.441551208496094,
      "activations/layer3_attention_weight_max": 75.00498962402344,
      "activations/layer3_attention_weight_min": -75.49249267578125,
      "activations/layer4_attention_weight_max": 130.099365234375,
      "activations/layer4_attention_weight_min": -118.22457122802734,
      "activations/layer5_attention_weight_max": 108.42443084716797,
      "activations/layer5_attention_weight_min": -92.77505493164062,
      "activations/layer6_attention_weight_max": 58.99656295776367,
      "activations/layer6_attention_weight_min": -53.20152282714844,
      "activations/layer7_attention_weight_max": 72.82181549072266,
      "activations/layer7_attention_weight_min": -64.30418395996094,
      "activations/layer8_attention_weight_max": 42.564430236816406,
      "activations/layer8_attention_weight_min": -35.1384391784668,
      "activations/layer9_attention_weight_max": 52.706878662109375,
      "activations/layer9_attention_weight_min": -45.420433044433594,
      "epoch": 2.51,
      "learning_rate": 0.0001351526515151515,
      "loss": 2.9352,
      "step": 43250
    },
    {
      "activations/layer0_attention_weight_max": 16.57309913635254,
      "activations/layer0_attention_weight_min": -10.981074333190918,
      "activations/layer10_attention_weight_max": 46.60211181640625,
      "activations/layer10_attention_weight_min": -37.73052978515625,
      "activations/layer11_attention_weight_max": 37.41184997558594,
      "activations/layer11_attention_weight_min": -34.09683609008789,
      "activations/layer12_attention_weight_max": 35.05386734008789,
      "activations/layer12_attention_weight_min": -28.474729537963867,
      "activations/layer13_attention_weight_max": 43.60462951660156,
      "activations/layer13_attention_weight_min": -34.672325134277344,
      "activations/layer14_attention_weight_max": 77.31790161132812,
      "activations/layer14_attention_weight_min": -52.19618606567383,
      "activations/layer15_attention_weight_max": 46.73799514770508,
      "activations/layer15_attention_weight_min": -33.17463302612305,
      "activations/layer16_attention_weight_max": 47.44102096557617,
      "activations/layer16_attention_weight_min": -33.2834358215332,
      "activations/layer17_attention_weight_max": 47.675758361816406,
      "activations/layer17_attention_weight_min": -26.909982681274414,
      "activations/layer18_attention_weight_max": 26.368684768676758,
      "activations/layer18_attention_weight_min": -17.12233543395996,
      "activations/layer19_attention_weight_max": 53.4800910949707,
      "activations/layer19_attention_weight_min": -30.498273849487305,
      "activations/layer1_attention_weight_max": 18.222490310668945,
      "activations/layer1_attention_weight_min": -12.432962417602539,
      "activations/layer20_attention_weight_max": 32.97126007080078,
      "activations/layer20_attention_weight_min": -23.556947708129883,
      "activations/layer21_attention_weight_max": 32.53894805908203,
      "activations/layer21_attention_weight_min": -18.212018966674805,
      "activations/layer22_attention_weight_max": 47.78279113769531,
      "activations/layer22_attention_weight_min": -25.974533081054688,
      "activations/layer23_attention_weight_max": 28.05331039428711,
      "activations/layer23_attention_weight_min": -18.608238220214844,
      "activations/layer2_attention_weight_max": 30.857410430908203,
      "activations/layer2_attention_weight_min": -30.253055572509766,
      "activations/layer3_attention_weight_max": 74.1906509399414,
      "activations/layer3_attention_weight_min": -76.53450775146484,
      "activations/layer4_attention_weight_max": 123.54996490478516,
      "activations/layer4_attention_weight_min": -117.638427734375,
      "activations/layer5_attention_weight_max": 108.2294921875,
      "activations/layer5_attention_weight_min": -95.84333801269531,
      "activations/layer6_attention_weight_max": 61.09700393676758,
      "activations/layer6_attention_weight_min": -52.89998245239258,
      "activations/layer7_attention_weight_max": 69.71755981445312,
      "activations/layer7_attention_weight_min": -64.61077117919922,
      "activations/layer8_attention_weight_max": 46.01897048950195,
      "activations/layer8_attention_weight_min": -35.98749923706055,
      "activations/layer9_attention_weight_max": 59.85795974731445,
      "activations/layer9_attention_weight_min": -50.481658935546875,
      "epoch": 2.52,
      "learning_rate": 0.0001351337121212121,
      "loss": 2.9388,
      "step": 43300
    },
    {
      "activations/layer0_attention_weight_max": 13.896434783935547,
      "activations/layer0_attention_weight_min": -11.004386901855469,
      "activations/layer10_attention_weight_max": 44.7391242980957,
      "activations/layer10_attention_weight_min": -36.809688568115234,
      "activations/layer11_attention_weight_max": 36.16342544555664,
      "activations/layer11_attention_weight_min": -31.478256225585938,
      "activations/layer12_attention_weight_max": 57.610225677490234,
      "activations/layer12_attention_weight_min": -34.81047439575195,
      "activations/layer13_attention_weight_max": 48.84836959838867,
      "activations/layer13_attention_weight_min": -37.86397171020508,
      "activations/layer14_attention_weight_max": 75.42162322998047,
      "activations/layer14_attention_weight_min": -53.79281997680664,
      "activations/layer15_attention_weight_max": 47.194007873535156,
      "activations/layer15_attention_weight_min": -37.639678955078125,
      "activations/layer16_attention_weight_max": 44.36819839477539,
      "activations/layer16_attention_weight_min": -35.61344909667969,
      "activations/layer17_attention_weight_max": 43.93341064453125,
      "activations/layer17_attention_weight_min": -31.464664459228516,
      "activations/layer18_attention_weight_max": 27.528902053833008,
      "activations/layer18_attention_weight_min": -17.55936050415039,
      "activations/layer19_attention_weight_max": 47.04442596435547,
      "activations/layer19_attention_weight_min": -33.00657653808594,
      "activations/layer1_attention_weight_max": 18.003324508666992,
      "activations/layer1_attention_weight_min": -13.539295196533203,
      "activations/layer20_attention_weight_max": 38.017234802246094,
      "activations/layer20_attention_weight_min": -28.6390380859375,
      "activations/layer21_attention_weight_max": 28.702646255493164,
      "activations/layer21_attention_weight_min": -20.787391662597656,
      "activations/layer22_attention_weight_max": 43.56304168701172,
      "activations/layer22_attention_weight_min": -24.918460845947266,
      "activations/layer23_attention_weight_max": 23.98854637145996,
      "activations/layer23_attention_weight_min": -17.867145538330078,
      "activations/layer2_attention_weight_max": 30.888904571533203,
      "activations/layer2_attention_weight_min": -31.718223571777344,
      "activations/layer3_attention_weight_max": 79.16370391845703,
      "activations/layer3_attention_weight_min": -77.29141998291016,
      "activations/layer4_attention_weight_max": 128.95831298828125,
      "activations/layer4_attention_weight_min": -127.46898651123047,
      "activations/layer5_attention_weight_max": 109.06963348388672,
      "activations/layer5_attention_weight_min": -97.4259033203125,
      "activations/layer6_attention_weight_max": 58.40626525878906,
      "activations/layer6_attention_weight_min": -53.174320220947266,
      "activations/layer7_attention_weight_max": 72.5313949584961,
      "activations/layer7_attention_weight_min": -63.911930084228516,
      "activations/layer8_attention_weight_max": 45.93361282348633,
      "activations/layer8_attention_weight_min": -37.376583099365234,
      "activations/layer9_attention_weight_max": 67.50505065917969,
      "activations/layer9_attention_weight_min": -55.92792892456055,
      "epoch": 2.52,
      "learning_rate": 0.00013511477272727272,
      "loss": 2.9445,
      "step": 43350
    },
    {
      "activations/layer0_attention_weight_max": 14.651918411254883,
      "activations/layer0_attention_weight_min": -10.89579963684082,
      "activations/layer10_attention_weight_max": 50.48158645629883,
      "activations/layer10_attention_weight_min": -40.436004638671875,
      "activations/layer11_attention_weight_max": 41.19916915893555,
      "activations/layer11_attention_weight_min": -34.227210998535156,
      "activations/layer12_attention_weight_max": 34.18364334106445,
      "activations/layer12_attention_weight_min": -30.74077606201172,
      "activations/layer13_attention_weight_max": 50.64784622192383,
      "activations/layer13_attention_weight_min": -35.35602951049805,
      "activations/layer14_attention_weight_max": 82.45850372314453,
      "activations/layer14_attention_weight_min": -67.23281860351562,
      "activations/layer15_attention_weight_max": 50.139766693115234,
      "activations/layer15_attention_weight_min": -37.151615142822266,
      "activations/layer16_attention_weight_max": 49.22972106933594,
      "activations/layer16_attention_weight_min": -38.20043182373047,
      "activations/layer17_attention_weight_max": 42.29874801635742,
      "activations/layer17_attention_weight_min": -28.71378517150879,
      "activations/layer18_attention_weight_max": 28.29713249206543,
      "activations/layer18_attention_weight_min": -19.448087692260742,
      "activations/layer19_attention_weight_max": 47.668113708496094,
      "activations/layer19_attention_weight_min": -32.84758758544922,
      "activations/layer1_attention_weight_max": 17.660070419311523,
      "activations/layer1_attention_weight_min": -12.097732543945312,
      "activations/layer20_attention_weight_max": 36.12043762207031,
      "activations/layer20_attention_weight_min": -25.1602783203125,
      "activations/layer21_attention_weight_max": 27.9067440032959,
      "activations/layer21_attention_weight_min": -18.923484802246094,
      "activations/layer22_attention_weight_max": 46.35417556762695,
      "activations/layer22_attention_weight_min": -28.92744255065918,
      "activations/layer23_attention_weight_max": 26.403667449951172,
      "activations/layer23_attention_weight_min": -20.710418701171875,
      "activations/layer2_attention_weight_max": 31.309947967529297,
      "activations/layer2_attention_weight_min": -30.598752975463867,
      "activations/layer3_attention_weight_max": 83.48494720458984,
      "activations/layer3_attention_weight_min": -81.37468719482422,
      "activations/layer4_attention_weight_max": 136.78953552246094,
      "activations/layer4_attention_weight_min": -123.13748931884766,
      "activations/layer5_attention_weight_max": 113.17318725585938,
      "activations/layer5_attention_weight_min": -98.30428314208984,
      "activations/layer6_attention_weight_max": 59.01602554321289,
      "activations/layer6_attention_weight_min": -50.54707336425781,
      "activations/layer7_attention_weight_max": 69.6507797241211,
      "activations/layer7_attention_weight_min": -65.04579162597656,
      "activations/layer8_attention_weight_max": 45.57850646972656,
      "activations/layer8_attention_weight_min": -38.21525573730469,
      "activations/layer9_attention_weight_max": 64.37362670898438,
      "activations/layer9_attention_weight_min": -58.7077751159668,
      "epoch": 2.52,
      "learning_rate": 0.0001350958333333333,
      "loss": 2.9451,
      "step": 43400
    },
    {
      "activations/layer0_attention_weight_max": 16.552282333374023,
      "activations/layer0_attention_weight_min": -11.450264930725098,
      "activations/layer10_attention_weight_max": 55.42500686645508,
      "activations/layer10_attention_weight_min": -45.97079849243164,
      "activations/layer11_attention_weight_max": 39.77147674560547,
      "activations/layer11_attention_weight_min": -33.906856536865234,
      "activations/layer12_attention_weight_max": 41.876461029052734,
      "activations/layer12_attention_weight_min": -32.01625442504883,
      "activations/layer13_attention_weight_max": 51.58472442626953,
      "activations/layer13_attention_weight_min": -38.02556228637695,
      "activations/layer14_attention_weight_max": 77.14720153808594,
      "activations/layer14_attention_weight_min": -56.82268524169922,
      "activations/layer15_attention_weight_max": 58.86034393310547,
      "activations/layer15_attention_weight_min": -39.33243179321289,
      "activations/layer16_attention_weight_max": 48.9759407043457,
      "activations/layer16_attention_weight_min": -36.47305679321289,
      "activations/layer17_attention_weight_max": 46.88353729248047,
      "activations/layer17_attention_weight_min": -27.9592227935791,
      "activations/layer18_attention_weight_max": 27.876867294311523,
      "activations/layer18_attention_weight_min": -17.609342575073242,
      "activations/layer19_attention_weight_max": 45.861324310302734,
      "activations/layer19_attention_weight_min": -31.377426147460938,
      "activations/layer1_attention_weight_max": 17.00933265686035,
      "activations/layer1_attention_weight_min": -12.566813468933105,
      "activations/layer20_attention_weight_max": 39.957420349121094,
      "activations/layer20_attention_weight_min": -23.191158294677734,
      "activations/layer21_attention_weight_max": 36.19902420043945,
      "activations/layer21_attention_weight_min": -19.53521156311035,
      "activations/layer22_attention_weight_max": 50.10502243041992,
      "activations/layer22_attention_weight_min": -29.79275894165039,
      "activations/layer23_attention_weight_max": 27.459186553955078,
      "activations/layer23_attention_weight_min": -17.99030876159668,
      "activations/layer2_attention_weight_max": 31.417530059814453,
      "activations/layer2_attention_weight_min": -31.2782039642334,
      "activations/layer3_attention_weight_max": 77.30538940429688,
      "activations/layer3_attention_weight_min": -79.65740203857422,
      "activations/layer4_attention_weight_max": 127.24454498291016,
      "activations/layer4_attention_weight_min": -121.30297088623047,
      "activations/layer5_attention_weight_max": 109.31845092773438,
      "activations/layer5_attention_weight_min": -96.46441650390625,
      "activations/layer6_attention_weight_max": 60.979637145996094,
      "activations/layer6_attention_weight_min": -53.48233413696289,
      "activations/layer7_attention_weight_max": 74.10944366455078,
      "activations/layer7_attention_weight_min": -66.18939971923828,
      "activations/layer8_attention_weight_max": 46.281856536865234,
      "activations/layer8_attention_weight_min": -40.246578216552734,
      "activations/layer9_attention_weight_max": 58.624542236328125,
      "activations/layer9_attention_weight_min": -48.485286712646484,
      "epoch": 2.52,
      "learning_rate": 0.00013507689393939392,
      "loss": 2.9329,
      "step": 43450
    },
    {
      "activations/layer0_attention_weight_max": 15.698321342468262,
      "activations/layer0_attention_weight_min": -10.840371131896973,
      "activations/layer10_attention_weight_max": 48.76652908325195,
      "activations/layer10_attention_weight_min": -40.017181396484375,
      "activations/layer11_attention_weight_max": 39.23381042480469,
      "activations/layer11_attention_weight_min": -31.950599670410156,
      "activations/layer12_attention_weight_max": 40.76621627807617,
      "activations/layer12_attention_weight_min": -30.54224967956543,
      "activations/layer13_attention_weight_max": 50.54871368408203,
      "activations/layer13_attention_weight_min": -37.89897537231445,
      "activations/layer14_attention_weight_max": 70.36580657958984,
      "activations/layer14_attention_weight_min": -49.57432174682617,
      "activations/layer15_attention_weight_max": 48.807037353515625,
      "activations/layer15_attention_weight_min": -33.949520111083984,
      "activations/layer16_attention_weight_max": 48.18943786621094,
      "activations/layer16_attention_weight_min": -31.59778594970703,
      "activations/layer17_attention_weight_max": 44.05867004394531,
      "activations/layer17_attention_weight_min": -26.936580657958984,
      "activations/layer18_attention_weight_max": 29.467924118041992,
      "activations/layer18_attention_weight_min": -19.134742736816406,
      "activations/layer19_attention_weight_max": 47.4395866394043,
      "activations/layer19_attention_weight_min": -29.890005111694336,
      "activations/layer1_attention_weight_max": 17.116579055786133,
      "activations/layer1_attention_weight_min": -12.313138961791992,
      "activations/layer20_attention_weight_max": 36.66896057128906,
      "activations/layer20_attention_weight_min": -21.811717987060547,
      "activations/layer21_attention_weight_max": 31.78859519958496,
      "activations/layer21_attention_weight_min": -21.196378707885742,
      "activations/layer22_attention_weight_max": 44.05491638183594,
      "activations/layer22_attention_weight_min": -29.87313461303711,
      "activations/layer23_attention_weight_max": 23.56721305847168,
      "activations/layer23_attention_weight_min": -13.692705154418945,
      "activations/layer2_attention_weight_max": 28.811939239501953,
      "activations/layer2_attention_weight_min": -29.154678344726562,
      "activations/layer3_attention_weight_max": 72.37403106689453,
      "activations/layer3_attention_weight_min": -77.60677337646484,
      "activations/layer4_attention_weight_max": 127.98565673828125,
      "activations/layer4_attention_weight_min": -118.77184295654297,
      "activations/layer5_attention_weight_max": 109.020751953125,
      "activations/layer5_attention_weight_min": -93.85704803466797,
      "activations/layer6_attention_weight_max": 60.480690002441406,
      "activations/layer6_attention_weight_min": -51.90178680419922,
      "activations/layer7_attention_weight_max": 67.67688751220703,
      "activations/layer7_attention_weight_min": -63.11831283569336,
      "activations/layer8_attention_weight_max": 42.56209182739258,
      "activations/layer8_attention_weight_min": -35.55006408691406,
      "activations/layer9_attention_weight_max": 60.758583068847656,
      "activations/layer9_attention_weight_min": -50.909000396728516,
      "epoch": 2.53,
      "learning_rate": 0.00013505795454545454,
      "loss": 2.9482,
      "step": 43500
    },
    {
      "activations/layer0_attention_weight_max": 15.922173500061035,
      "activations/layer0_attention_weight_min": -10.86068058013916,
      "activations/layer10_attention_weight_max": 48.2928466796875,
      "activations/layer10_attention_weight_min": -36.27145767211914,
      "activations/layer11_attention_weight_max": 40.80073547363281,
      "activations/layer11_attention_weight_min": -35.76214599609375,
      "activations/layer12_attention_weight_max": 35.9450569152832,
      "activations/layer12_attention_weight_min": -30.986129760742188,
      "activations/layer13_attention_weight_max": 55.77751541137695,
      "activations/layer13_attention_weight_min": -38.293575286865234,
      "activations/layer14_attention_weight_max": 70.49446105957031,
      "activations/layer14_attention_weight_min": -55.980751037597656,
      "activations/layer15_attention_weight_max": 47.52824020385742,
      "activations/layer15_attention_weight_min": -36.40913009643555,
      "activations/layer16_attention_weight_max": 47.74257278442383,
      "activations/layer16_attention_weight_min": -33.69453430175781,
      "activations/layer17_attention_weight_max": 45.014163970947266,
      "activations/layer17_attention_weight_min": -31.333642959594727,
      "activations/layer18_attention_weight_max": 29.015460968017578,
      "activations/layer18_attention_weight_min": -18.6629695892334,
      "activations/layer19_attention_weight_max": 51.34089279174805,
      "activations/layer19_attention_weight_min": -29.000707626342773,
      "activations/layer1_attention_weight_max": 17.302413940429688,
      "activations/layer1_attention_weight_min": -11.685248374938965,
      "activations/layer20_attention_weight_max": 34.399986267089844,
      "activations/layer20_attention_weight_min": -23.937978744506836,
      "activations/layer21_attention_weight_max": 37.17852783203125,
      "activations/layer21_attention_weight_min": -21.82638931274414,
      "activations/layer22_attention_weight_max": 52.12652587890625,
      "activations/layer22_attention_weight_min": -26.030351638793945,
      "activations/layer23_attention_weight_max": 27.745262145996094,
      "activations/layer23_attention_weight_min": -15.57547378540039,
      "activations/layer2_attention_weight_max": 30.212759017944336,
      "activations/layer2_attention_weight_min": -29.25606346130371,
      "activations/layer3_attention_weight_max": 75.71172332763672,
      "activations/layer3_attention_weight_min": -79.58872985839844,
      "activations/layer4_attention_weight_max": 130.59420776367188,
      "activations/layer4_attention_weight_min": -117.959716796875,
      "activations/layer5_attention_weight_max": 110.27864074707031,
      "activations/layer5_attention_weight_min": -90.02230834960938,
      "activations/layer6_attention_weight_max": 65.23290252685547,
      "activations/layer6_attention_weight_min": -52.80503845214844,
      "activations/layer7_attention_weight_max": 73.6126480102539,
      "activations/layer7_attention_weight_min": -64.45832824707031,
      "activations/layer8_attention_weight_max": 43.942535400390625,
      "activations/layer8_attention_weight_min": -37.7795524597168,
      "activations/layer9_attention_weight_max": 68.53819274902344,
      "activations/layer9_attention_weight_min": -48.571868896484375,
      "epoch": 2.53,
      "learning_rate": 0.00013503901515151515,
      "loss": 2.9461,
      "step": 43550
    },
    {
      "activations/layer0_attention_weight_max": 16.265460968017578,
      "activations/layer0_attention_weight_min": -11.067856788635254,
      "activations/layer10_attention_weight_max": 43.884769439697266,
      "activations/layer10_attention_weight_min": -39.024505615234375,
      "activations/layer11_attention_weight_max": 40.99535369873047,
      "activations/layer11_attention_weight_min": -36.841304779052734,
      "activations/layer12_attention_weight_max": 43.7793083190918,
      "activations/layer12_attention_weight_min": -31.579059600830078,
      "activations/layer13_attention_weight_max": 58.003238677978516,
      "activations/layer13_attention_weight_min": -47.45836639404297,
      "activations/layer14_attention_weight_max": 66.93174743652344,
      "activations/layer14_attention_weight_min": -55.629032135009766,
      "activations/layer15_attention_weight_max": 48.3804931640625,
      "activations/layer15_attention_weight_min": -34.99862289428711,
      "activations/layer16_attention_weight_max": 54.33589172363281,
      "activations/layer16_attention_weight_min": -34.36357879638672,
      "activations/layer17_attention_weight_max": 43.31678771972656,
      "activations/layer17_attention_weight_min": -29.889978408813477,
      "activations/layer18_attention_weight_max": 28.236364364624023,
      "activations/layer18_attention_weight_min": -17.59379768371582,
      "activations/layer19_attention_weight_max": 46.297698974609375,
      "activations/layer19_attention_weight_min": -28.11138916015625,
      "activations/layer1_attention_weight_max": 18.09012222290039,
      "activations/layer1_attention_weight_min": -13.111923217773438,
      "activations/layer20_attention_weight_max": 36.81772994995117,
      "activations/layer20_attention_weight_min": -23.699384689331055,
      "activations/layer21_attention_weight_max": 31.982755661010742,
      "activations/layer21_attention_weight_min": -21.53922462463379,
      "activations/layer22_attention_weight_max": 43.011070251464844,
      "activations/layer22_attention_weight_min": -25.63410186767578,
      "activations/layer23_attention_weight_max": 25.820892333984375,
      "activations/layer23_attention_weight_min": -14.85312557220459,
      "activations/layer2_attention_weight_max": 30.66402816772461,
      "activations/layer2_attention_weight_min": -31.59735107421875,
      "activations/layer3_attention_weight_max": 74.28116607666016,
      "activations/layer3_attention_weight_min": -79.30477905273438,
      "activations/layer4_attention_weight_max": 137.78819274902344,
      "activations/layer4_attention_weight_min": -125.20698547363281,
      "activations/layer5_attention_weight_max": 108.079833984375,
      "activations/layer5_attention_weight_min": -94.29302978515625,
      "activations/layer6_attention_weight_max": 61.72903823852539,
      "activations/layer6_attention_weight_min": -51.370765686035156,
      "activations/layer7_attention_weight_max": 72.06853485107422,
      "activations/layer7_attention_weight_min": -68.94660949707031,
      "activations/layer8_attention_weight_max": 47.835418701171875,
      "activations/layer8_attention_weight_min": -38.682350158691406,
      "activations/layer9_attention_weight_max": 68.1677474975586,
      "activations/layer9_attention_weight_min": -67.43270111083984,
      "epoch": 2.53,
      "learning_rate": 0.00013502007575757574,
      "loss": 2.9431,
      "step": 43600
    },
    {
      "activations/layer0_attention_weight_max": 15.223160743713379,
      "activations/layer0_attention_weight_min": -11.35474681854248,
      "activations/layer10_attention_weight_max": 49.09535598754883,
      "activations/layer10_attention_weight_min": -38.75061798095703,
      "activations/layer11_attention_weight_max": 41.45347595214844,
      "activations/layer11_attention_weight_min": -36.61453628540039,
      "activations/layer12_attention_weight_max": 41.28768539428711,
      "activations/layer12_attention_weight_min": -28.067712783813477,
      "activations/layer13_attention_weight_max": 50.6014289855957,
      "activations/layer13_attention_weight_min": -36.84638214111328,
      "activations/layer14_attention_weight_max": 74.51874542236328,
      "activations/layer14_attention_weight_min": -55.78531265258789,
      "activations/layer15_attention_weight_max": 49.822269439697266,
      "activations/layer15_attention_weight_min": -33.969932556152344,
      "activations/layer16_attention_weight_max": 45.7855110168457,
      "activations/layer16_attention_weight_min": -33.18368911743164,
      "activations/layer17_attention_weight_max": 43.92449951171875,
      "activations/layer17_attention_weight_min": -27.682008743286133,
      "activations/layer18_attention_weight_max": 29.262065887451172,
      "activations/layer18_attention_weight_min": -18.862010955810547,
      "activations/layer19_attention_weight_max": 44.84914779663086,
      "activations/layer19_attention_weight_min": -30.367170333862305,
      "activations/layer1_attention_weight_max": 17.290674209594727,
      "activations/layer1_attention_weight_min": -11.850292205810547,
      "activations/layer20_attention_weight_max": 36.21158218383789,
      "activations/layer20_attention_weight_min": -25.64845848083496,
      "activations/layer21_attention_weight_max": 33.21385192871094,
      "activations/layer21_attention_weight_min": -19.41805076599121,
      "activations/layer22_attention_weight_max": 47.57032775878906,
      "activations/layer22_attention_weight_min": -24.363401412963867,
      "activations/layer23_attention_weight_max": 26.185400009155273,
      "activations/layer23_attention_weight_min": -17.315303802490234,
      "activations/layer2_attention_weight_max": 29.806272506713867,
      "activations/layer2_attention_weight_min": -28.276081085205078,
      "activations/layer3_attention_weight_max": 72.09829711914062,
      "activations/layer3_attention_weight_min": -75.33990478515625,
      "activations/layer4_attention_weight_max": 128.1459503173828,
      "activations/layer4_attention_weight_min": -118.84647369384766,
      "activations/layer5_attention_weight_max": 103.38442993164062,
      "activations/layer5_attention_weight_min": -96.96593475341797,
      "activations/layer6_attention_weight_max": 57.81528854370117,
      "activations/layer6_attention_weight_min": -51.12609100341797,
      "activations/layer7_attention_weight_max": 70.68373107910156,
      "activations/layer7_attention_weight_min": -62.26177978515625,
      "activations/layer8_attention_weight_max": 41.49994659423828,
      "activations/layer8_attention_weight_min": -36.72150421142578,
      "activations/layer9_attention_weight_max": 60.88466262817383,
      "activations/layer9_attention_weight_min": -43.64301681518555,
      "epoch": 2.54,
      "learning_rate": 0.00013500113636363636,
      "loss": 2.9304,
      "step": 43650
    },
    {
      "activations/layer0_attention_weight_max": 15.940943717956543,
      "activations/layer0_attention_weight_min": -11.355384826660156,
      "activations/layer10_attention_weight_max": 53.87564468383789,
      "activations/layer10_attention_weight_min": -40.375762939453125,
      "activations/layer11_attention_weight_max": 38.250858306884766,
      "activations/layer11_attention_weight_min": -30.26443099975586,
      "activations/layer12_attention_weight_max": 39.19050598144531,
      "activations/layer12_attention_weight_min": -30.737470626831055,
      "activations/layer13_attention_weight_max": 49.19246292114258,
      "activations/layer13_attention_weight_min": -40.25249481201172,
      "activations/layer14_attention_weight_max": 72.96610260009766,
      "activations/layer14_attention_weight_min": -54.23134231567383,
      "activations/layer15_attention_weight_max": 55.306575775146484,
      "activations/layer15_attention_weight_min": -34.27861404418945,
      "activations/layer16_attention_weight_max": 48.625606536865234,
      "activations/layer16_attention_weight_min": -32.0362663269043,
      "activations/layer17_attention_weight_max": 48.7696533203125,
      "activations/layer17_attention_weight_min": -26.870018005371094,
      "activations/layer18_attention_weight_max": 28.91288948059082,
      "activations/layer18_attention_weight_min": -18.031330108642578,
      "activations/layer19_attention_weight_max": 52.21297073364258,
      "activations/layer19_attention_weight_min": -32.69181442260742,
      "activations/layer1_attention_weight_max": 19.05824089050293,
      "activations/layer1_attention_weight_min": -11.448114395141602,
      "activations/layer20_attention_weight_max": 36.29538345336914,
      "activations/layer20_attention_weight_min": -26.63994026184082,
      "activations/layer21_attention_weight_max": 33.812618255615234,
      "activations/layer21_attention_weight_min": -22.4936466217041,
      "activations/layer22_attention_weight_max": 54.91575622558594,
      "activations/layer22_attention_weight_min": -25.879119873046875,
      "activations/layer23_attention_weight_max": 27.690587997436523,
      "activations/layer23_attention_weight_min": -19.47347068786621,
      "activations/layer2_attention_weight_max": 30.840667724609375,
      "activations/layer2_attention_weight_min": -29.88387107849121,
      "activations/layer3_attention_weight_max": 79.24715423583984,
      "activations/layer3_attention_weight_min": -81.28643798828125,
      "activations/layer4_attention_weight_max": 133.25279235839844,
      "activations/layer4_attention_weight_min": -125.28802490234375,
      "activations/layer5_attention_weight_max": 109.15351867675781,
      "activations/layer5_attention_weight_min": -100.3412094116211,
      "activations/layer6_attention_weight_max": 58.63510513305664,
      "activations/layer6_attention_weight_min": -53.671104431152344,
      "activations/layer7_attention_weight_max": 69.54741668701172,
      "activations/layer7_attention_weight_min": -65.3287582397461,
      "activations/layer8_attention_weight_max": 47.3273811340332,
      "activations/layer8_attention_weight_min": -38.010826110839844,
      "activations/layer9_attention_weight_max": 58.09486770629883,
      "activations/layer9_attention_weight_min": -51.68070602416992,
      "epoch": 2.54,
      "learning_rate": 0.00013498219696969697,
      "loss": 2.9465,
      "step": 43700
    },
    {
      "activations/layer0_attention_weight_max": 15.737175941467285,
      "activations/layer0_attention_weight_min": -11.541316986083984,
      "activations/layer10_attention_weight_max": 48.02387619018555,
      "activations/layer10_attention_weight_min": -41.66648864746094,
      "activations/layer11_attention_weight_max": 39.2105712890625,
      "activations/layer11_attention_weight_min": -35.13749313354492,
      "activations/layer12_attention_weight_max": 34.058860778808594,
      "activations/layer12_attention_weight_min": -30.311906814575195,
      "activations/layer13_attention_weight_max": 51.343326568603516,
      "activations/layer13_attention_weight_min": -42.10430145263672,
      "activations/layer14_attention_weight_max": 66.71832275390625,
      "activations/layer14_attention_weight_min": -56.80419921875,
      "activations/layer15_attention_weight_max": 48.37999725341797,
      "activations/layer15_attention_weight_min": -36.276893615722656,
      "activations/layer16_attention_weight_max": 43.07066345214844,
      "activations/layer16_attention_weight_min": -32.03055191040039,
      "activations/layer17_attention_weight_max": 46.56215286254883,
      "activations/layer17_attention_weight_min": -28.31109619140625,
      "activations/layer18_attention_weight_max": 25.228302001953125,
      "activations/layer18_attention_weight_min": -20.034910202026367,
      "activations/layer19_attention_weight_max": 49.612396240234375,
      "activations/layer19_attention_weight_min": -30.57870864868164,
      "activations/layer1_attention_weight_max": 17.92816734313965,
      "activations/layer1_attention_weight_min": -12.027894020080566,
      "activations/layer20_attention_weight_max": 36.82489013671875,
      "activations/layer20_attention_weight_min": -24.772809982299805,
      "activations/layer21_attention_weight_max": 30.947601318359375,
      "activations/layer21_attention_weight_min": -21.902883529663086,
      "activations/layer22_attention_weight_max": 42.207088470458984,
      "activations/layer22_attention_weight_min": -31.571088790893555,
      "activations/layer23_attention_weight_max": 24.998523712158203,
      "activations/layer23_attention_weight_min": -15.371882438659668,
      "activations/layer2_attention_weight_max": 28.851560592651367,
      "activations/layer2_attention_weight_min": -28.626630783081055,
      "activations/layer3_attention_weight_max": 77.60011291503906,
      "activations/layer3_attention_weight_min": -80.55782318115234,
      "activations/layer4_attention_weight_max": 130.40589904785156,
      "activations/layer4_attention_weight_min": -126.1143798828125,
      "activations/layer5_attention_weight_max": 106.74837493896484,
      "activations/layer5_attention_weight_min": -100.03499603271484,
      "activations/layer6_attention_weight_max": 61.42026901245117,
      "activations/layer6_attention_weight_min": -55.3929328918457,
      "activations/layer7_attention_weight_max": 70.78020477294922,
      "activations/layer7_attention_weight_min": -66.31755065917969,
      "activations/layer8_attention_weight_max": 46.85316467285156,
      "activations/layer8_attention_weight_min": -38.52324295043945,
      "activations/layer9_attention_weight_max": 57.45723342895508,
      "activations/layer9_attention_weight_min": -51.9102897644043,
      "epoch": 2.54,
      "learning_rate": 0.00013496325757575756,
      "loss": 2.954,
      "step": 43750
    },
    {
      "activations/layer0_attention_weight_max": 14.483219146728516,
      "activations/layer0_attention_weight_min": -10.92385196685791,
      "activations/layer10_attention_weight_max": 45.57301712036133,
      "activations/layer10_attention_weight_min": -38.42064666748047,
      "activations/layer11_attention_weight_max": 38.89030456542969,
      "activations/layer11_attention_weight_min": -30.956769943237305,
      "activations/layer12_attention_weight_max": 36.35649490356445,
      "activations/layer12_attention_weight_min": -28.209903717041016,
      "activations/layer13_attention_weight_max": 42.91212463378906,
      "activations/layer13_attention_weight_min": -33.83236312866211,
      "activations/layer14_attention_weight_max": 76.22869873046875,
      "activations/layer14_attention_weight_min": -56.441139221191406,
      "activations/layer15_attention_weight_max": 47.877105712890625,
      "activations/layer15_attention_weight_min": -34.50390625,
      "activations/layer16_attention_weight_max": 45.623191833496094,
      "activations/layer16_attention_weight_min": -30.367761611938477,
      "activations/layer17_attention_weight_max": 45.18970489501953,
      "activations/layer17_attention_weight_min": -26.937959671020508,
      "activations/layer18_attention_weight_max": 27.850797653198242,
      "activations/layer18_attention_weight_min": -19.03061294555664,
      "activations/layer19_attention_weight_max": 47.71322250366211,
      "activations/layer19_attention_weight_min": -32.2625846862793,
      "activations/layer1_attention_weight_max": 17.57732391357422,
      "activations/layer1_attention_weight_min": -12.156915664672852,
      "activations/layer20_attention_weight_max": 38.08535385131836,
      "activations/layer20_attention_weight_min": -23.856184005737305,
      "activations/layer21_attention_weight_max": 33.95222091674805,
      "activations/layer21_attention_weight_min": -22.276321411132812,
      "activations/layer22_attention_weight_max": 46.022300720214844,
      "activations/layer22_attention_weight_min": -27.074724197387695,
      "activations/layer23_attention_weight_max": 24.592605590820312,
      "activations/layer23_attention_weight_min": -18.1381778717041,
      "activations/layer2_attention_weight_max": 29.2513484954834,
      "activations/layer2_attention_weight_min": -29.491981506347656,
      "activations/layer3_attention_weight_max": 74.92503356933594,
      "activations/layer3_attention_weight_min": -76.80757904052734,
      "activations/layer4_attention_weight_max": 126.61384582519531,
      "activations/layer4_attention_weight_min": -116.81279754638672,
      "activations/layer5_attention_weight_max": 105.05781555175781,
      "activations/layer5_attention_weight_min": -91.22903442382812,
      "activations/layer6_attention_weight_max": 58.11570358276367,
      "activations/layer6_attention_weight_min": -49.71316146850586,
      "activations/layer7_attention_weight_max": 69.5374755859375,
      "activations/layer7_attention_weight_min": -61.989219665527344,
      "activations/layer8_attention_weight_max": 41.26729202270508,
      "activations/layer8_attention_weight_min": -39.15767288208008,
      "activations/layer9_attention_weight_max": 55.12247848510742,
      "activations/layer9_attention_weight_min": -48.40147399902344,
      "epoch": 2.55,
      "learning_rate": 0.00013494431818181817,
      "loss": 2.9398,
      "step": 43800
    },
    {
      "activations/layer0_attention_weight_max": 15.482924461364746,
      "activations/layer0_attention_weight_min": -11.090022087097168,
      "activations/layer10_attention_weight_max": 47.8824348449707,
      "activations/layer10_attention_weight_min": -36.343204498291016,
      "activations/layer11_attention_weight_max": 38.38858413696289,
      "activations/layer11_attention_weight_min": -30.524951934814453,
      "activations/layer12_attention_weight_max": 35.63205337524414,
      "activations/layer12_attention_weight_min": -31.069482803344727,
      "activations/layer13_attention_weight_max": 53.98405838012695,
      "activations/layer13_attention_weight_min": -35.387088775634766,
      "activations/layer14_attention_weight_max": 76.7520523071289,
      "activations/layer14_attention_weight_min": -53.368106842041016,
      "activations/layer15_attention_weight_max": 53.10643768310547,
      "activations/layer15_attention_weight_min": -35.89506912231445,
      "activations/layer16_attention_weight_max": 48.33047866821289,
      "activations/layer16_attention_weight_min": -34.2767333984375,
      "activations/layer17_attention_weight_max": 46.330074310302734,
      "activations/layer17_attention_weight_min": -29.14702796936035,
      "activations/layer18_attention_weight_max": 32.58861541748047,
      "activations/layer18_attention_weight_min": -18.65907096862793,
      "activations/layer19_attention_weight_max": 49.66194152832031,
      "activations/layer19_attention_weight_min": -31.770944595336914,
      "activations/layer1_attention_weight_max": 17.024999618530273,
      "activations/layer1_attention_weight_min": -11.474579811096191,
      "activations/layer20_attention_weight_max": 36.477359771728516,
      "activations/layer20_attention_weight_min": -22.206993103027344,
      "activations/layer21_attention_weight_max": 32.598609924316406,
      "activations/layer21_attention_weight_min": -19.522714614868164,
      "activations/layer22_attention_weight_max": 48.8466911315918,
      "activations/layer22_attention_weight_min": -29.7684383392334,
      "activations/layer23_attention_weight_max": 29.40520477294922,
      "activations/layer23_attention_weight_min": -17.394390106201172,
      "activations/layer2_attention_weight_max": 29.330608367919922,
      "activations/layer2_attention_weight_min": -29.762836456298828,
      "activations/layer3_attention_weight_max": 74.494140625,
      "activations/layer3_attention_weight_min": -79.80585479736328,
      "activations/layer4_attention_weight_max": 134.10511779785156,
      "activations/layer4_attention_weight_min": -126.43291473388672,
      "activations/layer5_attention_weight_max": 108.71699523925781,
      "activations/layer5_attention_weight_min": -92.65866088867188,
      "activations/layer6_attention_weight_max": 58.46388244628906,
      "activations/layer6_attention_weight_min": -52.39066696166992,
      "activations/layer7_attention_weight_max": 66.62541198730469,
      "activations/layer7_attention_weight_min": -63.16434097290039,
      "activations/layer8_attention_weight_max": 45.86994934082031,
      "activations/layer8_attention_weight_min": -34.92974090576172,
      "activations/layer9_attention_weight_max": 52.38471221923828,
      "activations/layer9_attention_weight_min": -46.37353515625,
      "epoch": 2.55,
      "learning_rate": 0.0001349253787878788,
      "loss": 2.9285,
      "step": 43850
    },
    {
      "activations/layer0_attention_weight_max": 15.118939399719238,
      "activations/layer0_attention_weight_min": -10.615570068359375,
      "activations/layer10_attention_weight_max": 48.6539192199707,
      "activations/layer10_attention_weight_min": -37.815399169921875,
      "activations/layer11_attention_weight_max": 44.976680755615234,
      "activations/layer11_attention_weight_min": -35.976173400878906,
      "activations/layer12_attention_weight_max": 45.64585876464844,
      "activations/layer12_attention_weight_min": -30.539600372314453,
      "activations/layer13_attention_weight_max": 52.52823257446289,
      "activations/layer13_attention_weight_min": -37.754112243652344,
      "activations/layer14_attention_weight_max": 87.20317840576172,
      "activations/layer14_attention_weight_min": -63.8694953918457,
      "activations/layer15_attention_weight_max": 49.9827766418457,
      "activations/layer15_attention_weight_min": -30.82931900024414,
      "activations/layer16_attention_weight_max": 50.46143341064453,
      "activations/layer16_attention_weight_min": -33.13056945800781,
      "activations/layer17_attention_weight_max": 47.688629150390625,
      "activations/layer17_attention_weight_min": -26.70948600769043,
      "activations/layer18_attention_weight_max": 34.31305694580078,
      "activations/layer18_attention_weight_min": -18.971450805664062,
      "activations/layer19_attention_weight_max": 49.241390228271484,
      "activations/layer19_attention_weight_min": -31.47269630432129,
      "activations/layer1_attention_weight_max": 17.458986282348633,
      "activations/layer1_attention_weight_min": -12.195907592773438,
      "activations/layer20_attention_weight_max": 40.030364990234375,
      "activations/layer20_attention_weight_min": -24.32480239868164,
      "activations/layer21_attention_weight_max": 40.2365608215332,
      "activations/layer21_attention_weight_min": -20.673934936523438,
      "activations/layer22_attention_weight_max": 48.97747802734375,
      "activations/layer22_attention_weight_min": -25.9622745513916,
      "activations/layer23_attention_weight_max": 27.442501068115234,
      "activations/layer23_attention_weight_min": -18.532255172729492,
      "activations/layer2_attention_weight_max": 30.76474380493164,
      "activations/layer2_attention_weight_min": -29.637935638427734,
      "activations/layer3_attention_weight_max": 78.49561309814453,
      "activations/layer3_attention_weight_min": -80.03093719482422,
      "activations/layer4_attention_weight_max": 134.50218200683594,
      "activations/layer4_attention_weight_min": -123.04900360107422,
      "activations/layer5_attention_weight_max": 110.00462341308594,
      "activations/layer5_attention_weight_min": -95.88719940185547,
      "activations/layer6_attention_weight_max": 59.348392486572266,
      "activations/layer6_attention_weight_min": -51.74397277832031,
      "activations/layer7_attention_weight_max": 67.60247802734375,
      "activations/layer7_attention_weight_min": -70.4281997680664,
      "activations/layer8_attention_weight_max": 46.461578369140625,
      "activations/layer8_attention_weight_min": -37.17542266845703,
      "activations/layer9_attention_weight_max": 63.796722412109375,
      "activations/layer9_attention_weight_min": -55.98667526245117,
      "epoch": 2.55,
      "learning_rate": 0.00013490643939393938,
      "loss": 2.935,
      "step": 43900
    },
    {
      "activations/layer0_attention_weight_max": 16.00320053100586,
      "activations/layer0_attention_weight_min": -11.13298225402832,
      "activations/layer10_attention_weight_max": 47.312400817871094,
      "activations/layer10_attention_weight_min": -36.01858139038086,
      "activations/layer11_attention_weight_max": 43.26698303222656,
      "activations/layer11_attention_weight_min": -31.248334884643555,
      "activations/layer12_attention_weight_max": 71.33851623535156,
      "activations/layer12_attention_weight_min": -40.3342170715332,
      "activations/layer13_attention_weight_max": 73.39556884765625,
      "activations/layer13_attention_weight_min": -49.390079498291016,
      "activations/layer14_attention_weight_max": 85.55043029785156,
      "activations/layer14_attention_weight_min": -63.16781997680664,
      "activations/layer15_attention_weight_max": 58.89262390136719,
      "activations/layer15_attention_weight_min": -38.369842529296875,
      "activations/layer16_attention_weight_max": 48.205848693847656,
      "activations/layer16_attention_weight_min": -33.695430755615234,
      "activations/layer17_attention_weight_max": 40.56990432739258,
      "activations/layer17_attention_weight_min": -26.749706268310547,
      "activations/layer18_attention_weight_max": 29.367610931396484,
      "activations/layer18_attention_weight_min": -19.599050521850586,
      "activations/layer19_attention_weight_max": 47.33418655395508,
      "activations/layer19_attention_weight_min": -31.225263595581055,
      "activations/layer1_attention_weight_max": 18.42595672607422,
      "activations/layer1_attention_weight_min": -12.431999206542969,
      "activations/layer20_attention_weight_max": 35.75743103027344,
      "activations/layer20_attention_weight_min": -24.17865562438965,
      "activations/layer21_attention_weight_max": 30.374191284179688,
      "activations/layer21_attention_weight_min": -20.749155044555664,
      "activations/layer22_attention_weight_max": 48.564395904541016,
      "activations/layer22_attention_weight_min": -27.835325241088867,
      "activations/layer23_attention_weight_max": 27.111129760742188,
      "activations/layer23_attention_weight_min": -15.988306045532227,
      "activations/layer2_attention_weight_max": 29.401473999023438,
      "activations/layer2_attention_weight_min": -29.402931213378906,
      "activations/layer3_attention_weight_max": 80.35828399658203,
      "activations/layer3_attention_weight_min": -80.82227325439453,
      "activations/layer4_attention_weight_max": 135.60836791992188,
      "activations/layer4_attention_weight_min": -128.0574951171875,
      "activations/layer5_attention_weight_max": 113.64158630371094,
      "activations/layer5_attention_weight_min": -99.22265625,
      "activations/layer6_attention_weight_max": 62.456363677978516,
      "activations/layer6_attention_weight_min": -51.92693328857422,
      "activations/layer7_attention_weight_max": 72.50336456298828,
      "activations/layer7_attention_weight_min": -64.12210083007812,
      "activations/layer8_attention_weight_max": 47.81622314453125,
      "activations/layer8_attention_weight_min": -39.42241287231445,
      "activations/layer9_attention_weight_max": 68.79332733154297,
      "activations/layer9_attention_weight_min": -56.767391204833984,
      "epoch": 2.55,
      "learning_rate": 0.0001348875,
      "loss": 2.941,
      "step": 43950
    },
    {
      "activations/layer0_attention_weight_max": 15.739861488342285,
      "activations/layer0_attention_weight_min": -10.788585662841797,
      "activations/layer10_attention_weight_max": 54.672607421875,
      "activations/layer10_attention_weight_min": -40.949012756347656,
      "activations/layer11_attention_weight_max": 47.908668518066406,
      "activations/layer11_attention_weight_min": -34.262046813964844,
      "activations/layer12_attention_weight_max": 45.86920166015625,
      "activations/layer12_attention_weight_min": -30.599580764770508,
      "activations/layer13_attention_weight_max": 54.778419494628906,
      "activations/layer13_attention_weight_min": -42.150142669677734,
      "activations/layer14_attention_weight_max": 76.79192352294922,
      "activations/layer14_attention_weight_min": -57.17967987060547,
      "activations/layer15_attention_weight_max": 44.350929260253906,
      "activations/layer15_attention_weight_min": -31.913986206054688,
      "activations/layer16_attention_weight_max": 43.2700080871582,
      "activations/layer16_attention_weight_min": -34.802730560302734,
      "activations/layer17_attention_weight_max": 42.46651077270508,
      "activations/layer17_attention_weight_min": -28.828609466552734,
      "activations/layer18_attention_weight_max": 23.935075759887695,
      "activations/layer18_attention_weight_min": -22.33012580871582,
      "activations/layer19_attention_weight_max": 40.041595458984375,
      "activations/layer19_attention_weight_min": -29.343036651611328,
      "activations/layer1_attention_weight_max": 18.273054122924805,
      "activations/layer1_attention_weight_min": -11.690810203552246,
      "activations/layer20_attention_weight_max": 33.5545539855957,
      "activations/layer20_attention_weight_min": -22.34783172607422,
      "activations/layer21_attention_weight_max": 25.316471099853516,
      "activations/layer21_attention_weight_min": -18.818767547607422,
      "activations/layer22_attention_weight_max": 43.2219123840332,
      "activations/layer22_attention_weight_min": -25.80512809753418,
      "activations/layer23_attention_weight_max": 24.961040496826172,
      "activations/layer23_attention_weight_min": -15.975024223327637,
      "activations/layer2_attention_weight_max": 29.4571590423584,
      "activations/layer2_attention_weight_min": -30.00238609313965,
      "activations/layer3_attention_weight_max": 74.28953552246094,
      "activations/layer3_attention_weight_min": -77.48233795166016,
      "activations/layer4_attention_weight_max": 134.4820098876953,
      "activations/layer4_attention_weight_min": -123.90579986572266,
      "activations/layer5_attention_weight_max": 115.96233367919922,
      "activations/layer5_attention_weight_min": -98.0720443725586,
      "activations/layer6_attention_weight_max": 59.809181213378906,
      "activations/layer6_attention_weight_min": -52.47788619995117,
      "activations/layer7_attention_weight_max": 72.70552062988281,
      "activations/layer7_attention_weight_min": -70.80342864990234,
      "activations/layer8_attention_weight_max": 49.13701248168945,
      "activations/layer8_attention_weight_min": -38.57195281982422,
      "activations/layer9_attention_weight_max": 67.1209945678711,
      "activations/layer9_attention_weight_min": -53.3244743347168,
      "epoch": 2.56,
      "learning_rate": 0.0001348685606060606,
      "loss": 2.9376,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_loss": 2.880859375,
      "eval_runtime": 8.4781,
      "eval_samples_per_second": 506.483,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_openwebtext_loss": 2.880859375,
      "eval_openwebtext_ppl": 17.829588900657228,
      "eval_openwebtext_runtime": 8.4781,
      "eval_openwebtext_samples_per_second": 506.483,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_wikitext_loss": 3.1328125,
      "eval_wikitext_ppl": 22.938403163153815,
      "eval_wikitext_runtime": 2.0136,
      "eval_wikitext_samples_per_second": 226.455,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_lambada_loss": 2.90625,
      "eval_lambada_ppl": 18.28808948244362,
      "eval_lambada_runtime": 9.6005,
      "eval_lambada_samples_per_second": 507.161,
      "step": 44000
    },
    {
      "activations/layer0_attention_weight_max": 14.946794509887695,
      "activations/layer0_attention_weight_min": -11.028277397155762,
      "activations/layer10_attention_weight_max": 49.54617691040039,
      "activations/layer10_attention_weight_min": -37.5721321105957,
      "activations/layer11_attention_weight_max": 45.24752426147461,
      "activations/layer11_attention_weight_min": -33.50690460205078,
      "activations/layer12_attention_weight_max": 43.79254150390625,
      "activations/layer12_attention_weight_min": -32.055084228515625,
      "activations/layer13_attention_weight_max": 60.03166580200195,
      "activations/layer13_attention_weight_min": -38.47657775878906,
      "activations/layer14_attention_weight_max": 95.86967468261719,
      "activations/layer14_attention_weight_min": -58.99555206298828,
      "activations/layer15_attention_weight_max": 52.18991470336914,
      "activations/layer15_attention_weight_min": -37.698612213134766,
      "activations/layer16_attention_weight_max": 49.54403305053711,
      "activations/layer16_attention_weight_min": -34.64552688598633,
      "activations/layer17_attention_weight_max": 43.7220573425293,
      "activations/layer17_attention_weight_min": -27.54615020751953,
      "activations/layer18_attention_weight_max": 29.03463363647461,
      "activations/layer18_attention_weight_min": -18.894760131835938,
      "activations/layer19_attention_weight_max": 49.155086517333984,
      "activations/layer19_attention_weight_min": -31.469751358032227,
      "activations/layer1_attention_weight_max": 17.911298751831055,
      "activations/layer1_attention_weight_min": -11.958052635192871,
      "activations/layer20_attention_weight_max": 38.577205657958984,
      "activations/layer20_attention_weight_min": -24.13779640197754,
      "activations/layer21_attention_weight_max": 32.61662673950195,
      "activations/layer21_attention_weight_min": -20.817838668823242,
      "activations/layer22_attention_weight_max": 47.42787170410156,
      "activations/layer22_attention_weight_min": -28.124013900756836,
      "activations/layer23_attention_weight_max": 27.856334686279297,
      "activations/layer23_attention_weight_min": -17.1253604888916,
      "activations/layer2_attention_weight_max": 30.295560836791992,
      "activations/layer2_attention_weight_min": -28.736492156982422,
      "activations/layer3_attention_weight_max": 76.36445617675781,
      "activations/layer3_attention_weight_min": -78.609130859375,
      "activations/layer4_attention_weight_max": 132.46163940429688,
      "activations/layer4_attention_weight_min": -122.968505859375,
      "activations/layer5_attention_weight_max": 113.96614837646484,
      "activations/layer5_attention_weight_min": -97.63053894042969,
      "activations/layer6_attention_weight_max": 63.66220474243164,
      "activations/layer6_attention_weight_min": -57.742759704589844,
      "activations/layer7_attention_weight_max": 72.6382064819336,
      "activations/layer7_attention_weight_min": -68.64812469482422,
      "activations/layer8_attention_weight_max": 49.82743453979492,
      "activations/layer8_attention_weight_min": -41.376651763916016,
      "activations/layer9_attention_weight_max": 78.85635375976562,
      "activations/layer9_attention_weight_min": -58.03896713256836,
      "epoch": 2.56,
      "learning_rate": 0.0001348496212121212,
      "loss": 2.9273,
      "step": 44050
    },
    {
      "activations/layer0_attention_weight_max": 16.58970069885254,
      "activations/layer0_attention_weight_min": -11.192859649658203,
      "activations/layer10_attention_weight_max": 52.982872009277344,
      "activations/layer10_attention_weight_min": -42.26957702636719,
      "activations/layer11_attention_weight_max": 41.45988845825195,
      "activations/layer11_attention_weight_min": -37.26100158691406,
      "activations/layer12_attention_weight_max": 31.879919052124023,
      "activations/layer12_attention_weight_min": -27.91449546813965,
      "activations/layer13_attention_weight_max": 54.25772476196289,
      "activations/layer13_attention_weight_min": -38.26543045043945,
      "activations/layer14_attention_weight_max": 84.64167022705078,
      "activations/layer14_attention_weight_min": -58.35190200805664,
      "activations/layer15_attention_weight_max": 60.335960388183594,
      "activations/layer15_attention_weight_min": -39.72519302368164,
      "activations/layer16_attention_weight_max": 49.90826416015625,
      "activations/layer16_attention_weight_min": -33.348819732666016,
      "activations/layer17_attention_weight_max": 46.73880386352539,
      "activations/layer17_attention_weight_min": -28.206928253173828,
      "activations/layer18_attention_weight_max": 32.88407897949219,
      "activations/layer18_attention_weight_min": -19.344863891601562,
      "activations/layer19_attention_weight_max": 55.2678108215332,
      "activations/layer19_attention_weight_min": -32.658321380615234,
      "activations/layer1_attention_weight_max": 17.450056076049805,
      "activations/layer1_attention_weight_min": -11.430710792541504,
      "activations/layer20_attention_weight_max": 45.749820709228516,
      "activations/layer20_attention_weight_min": -23.88959312438965,
      "activations/layer21_attention_weight_max": 36.4799690246582,
      "activations/layer21_attention_weight_min": -22.84515953063965,
      "activations/layer22_attention_weight_max": 54.794944763183594,
      "activations/layer22_attention_weight_min": -27.74054527282715,
      "activations/layer23_attention_weight_max": 27.985980987548828,
      "activations/layer23_attention_weight_min": -17.53728675842285,
      "activations/layer2_attention_weight_max": 30.574302673339844,
      "activations/layer2_attention_weight_min": -29.228086471557617,
      "activations/layer3_attention_weight_max": 76.8756103515625,
      "activations/layer3_attention_weight_min": -78.41288757324219,
      "activations/layer4_attention_weight_max": 136.4589080810547,
      "activations/layer4_attention_weight_min": -125.16383361816406,
      "activations/layer5_attention_weight_max": 114.74678039550781,
      "activations/layer5_attention_weight_min": -100.01982879638672,
      "activations/layer6_attention_weight_max": 65.768798828125,
      "activations/layer6_attention_weight_min": -55.201507568359375,
      "activations/layer7_attention_weight_max": 70.577880859375,
      "activations/layer7_attention_weight_min": -66.77277374267578,
      "activations/layer8_attention_weight_max": 46.462913513183594,
      "activations/layer8_attention_weight_min": -38.55519104003906,
      "activations/layer9_attention_weight_max": 60.93145751953125,
      "activations/layer9_attention_weight_min": -44.65707015991211,
      "epoch": 2.56,
      "learning_rate": 0.0001348306818181818,
      "loss": 2.9207,
      "step": 44100
    },
    {
      "activations/layer0_attention_weight_max": 16.050174713134766,
      "activations/layer0_attention_weight_min": -11.122769355773926,
      "activations/layer10_attention_weight_max": 45.29768371582031,
      "activations/layer10_attention_weight_min": -37.1972541809082,
      "activations/layer11_attention_weight_max": 42.23414993286133,
      "activations/layer11_attention_weight_min": -36.43199157714844,
      "activations/layer12_attention_weight_max": 29.443361282348633,
      "activations/layer12_attention_weight_min": -24.902484893798828,
      "activations/layer13_attention_weight_max": 44.111244201660156,
      "activations/layer13_attention_weight_min": -34.55980682373047,
      "activations/layer14_attention_weight_max": 73.28470611572266,
      "activations/layer14_attention_weight_min": -53.37752151489258,
      "activations/layer15_attention_weight_max": 48.03285217285156,
      "activations/layer15_attention_weight_min": -32.31473922729492,
      "activations/layer16_attention_weight_max": 48.97667694091797,
      "activations/layer16_attention_weight_min": -33.14891052246094,
      "activations/layer17_attention_weight_max": 42.27132797241211,
      "activations/layer17_attention_weight_min": -28.07904052734375,
      "activations/layer18_attention_weight_max": 27.563020706176758,
      "activations/layer18_attention_weight_min": -17.807876586914062,
      "activations/layer19_attention_weight_max": 49.64283752441406,
      "activations/layer19_attention_weight_min": -31.68134880065918,
      "activations/layer1_attention_weight_max": 16.857135772705078,
      "activations/layer1_attention_weight_min": -12.391862869262695,
      "activations/layer20_attention_weight_max": 32.99909973144531,
      "activations/layer20_attention_weight_min": -23.167964935302734,
      "activations/layer21_attention_weight_max": 29.2463436126709,
      "activations/layer21_attention_weight_min": -20.565500259399414,
      "activations/layer22_attention_weight_max": 39.776248931884766,
      "activations/layer22_attention_weight_min": -28.96125030517578,
      "activations/layer23_attention_weight_max": 23.387874603271484,
      "activations/layer23_attention_weight_min": -19.208148956298828,
      "activations/layer2_attention_weight_max": 30.707786560058594,
      "activations/layer2_attention_weight_min": -30.109867095947266,
      "activations/layer3_attention_weight_max": 74.0757064819336,
      "activations/layer3_attention_weight_min": -78.52648162841797,
      "activations/layer4_attention_weight_max": 125.10992431640625,
      "activations/layer4_attention_weight_min": -122.04676818847656,
      "activations/layer5_attention_weight_max": 102.14620208740234,
      "activations/layer5_attention_weight_min": -94.3913803100586,
      "activations/layer6_attention_weight_max": 57.72163772583008,
      "activations/layer6_attention_weight_min": -53.54154586791992,
      "activations/layer7_attention_weight_max": 64.76087951660156,
      "activations/layer7_attention_weight_min": -63.6898078918457,
      "activations/layer8_attention_weight_max": 42.84211349487305,
      "activations/layer8_attention_weight_min": -36.85802459716797,
      "activations/layer9_attention_weight_max": 55.126338958740234,
      "activations/layer9_attention_weight_min": -53.48685836791992,
      "epoch": 2.57,
      "learning_rate": 0.00013481174242424243,
      "loss": 2.9435,
      "step": 44150
    },
    {
      "activations/layer0_attention_weight_max": 15.77753734588623,
      "activations/layer0_attention_weight_min": -10.987186431884766,
      "activations/layer10_attention_weight_max": 53.7467041015625,
      "activations/layer10_attention_weight_min": -41.052825927734375,
      "activations/layer11_attention_weight_max": 42.394874572753906,
      "activations/layer11_attention_weight_min": -34.033390045166016,
      "activations/layer12_attention_weight_max": 70.42328643798828,
      "activations/layer12_attention_weight_min": -35.550010681152344,
      "activations/layer13_attention_weight_max": 51.96731185913086,
      "activations/layer13_attention_weight_min": -35.840240478515625,
      "activations/layer14_attention_weight_max": 72.89688110351562,
      "activations/layer14_attention_weight_min": -53.52146911621094,
      "activations/layer15_attention_weight_max": 56.3336181640625,
      "activations/layer15_attention_weight_min": -37.31147384643555,
      "activations/layer16_attention_weight_max": 48.461788177490234,
      "activations/layer16_attention_weight_min": -33.94493103027344,
      "activations/layer17_attention_weight_max": 46.941532135009766,
      "activations/layer17_attention_weight_min": -30.56702995300293,
      "activations/layer18_attention_weight_max": 30.350637435913086,
      "activations/layer18_attention_weight_min": -18.466880798339844,
      "activations/layer19_attention_weight_max": 49.654151916503906,
      "activations/layer19_attention_weight_min": -32.368804931640625,
      "activations/layer1_attention_weight_max": 17.4917049407959,
      "activations/layer1_attention_weight_min": -11.160656929016113,
      "activations/layer20_attention_weight_max": 37.988346099853516,
      "activations/layer20_attention_weight_min": -23.8488712310791,
      "activations/layer21_attention_weight_max": 38.59465789794922,
      "activations/layer21_attention_weight_min": -22.645605087280273,
      "activations/layer22_attention_weight_max": 45.67951202392578,
      "activations/layer22_attention_weight_min": -29.534427642822266,
      "activations/layer23_attention_weight_max": 27.29169273376465,
      "activations/layer23_attention_weight_min": -18.133195877075195,
      "activations/layer2_attention_weight_max": 28.719070434570312,
      "activations/layer2_attention_weight_min": -28.61768913269043,
      "activations/layer3_attention_weight_max": 73.72793579101562,
      "activations/layer3_attention_weight_min": -76.36508178710938,
      "activations/layer4_attention_weight_max": 127.0494155883789,
      "activations/layer4_attention_weight_min": -120.62019348144531,
      "activations/layer5_attention_weight_max": 104.17991638183594,
      "activations/layer5_attention_weight_min": -90.00505065917969,
      "activations/layer6_attention_weight_max": 59.152061462402344,
      "activations/layer6_attention_weight_min": -51.80726623535156,
      "activations/layer7_attention_weight_max": 65.7565689086914,
      "activations/layer7_attention_weight_min": -63.405426025390625,
      "activations/layer8_attention_weight_max": 41.2952766418457,
      "activations/layer8_attention_weight_min": -35.75502395629883,
      "activations/layer9_attention_weight_max": 55.916378021240234,
      "activations/layer9_attention_weight_min": -46.59010314941406,
      "epoch": 2.57,
      "learning_rate": 0.000134792803030303,
      "loss": 2.9424,
      "step": 44200
    },
    {
      "activations/layer0_attention_weight_max": 16.22003746032715,
      "activations/layer0_attention_weight_min": -10.981095314025879,
      "activations/layer10_attention_weight_max": 45.76031494140625,
      "activations/layer10_attention_weight_min": -39.063499450683594,
      "activations/layer11_attention_weight_max": 48.45299530029297,
      "activations/layer11_attention_weight_min": -35.1328010559082,
      "activations/layer12_attention_weight_max": 40.30427932739258,
      "activations/layer12_attention_weight_min": -28.696298599243164,
      "activations/layer13_attention_weight_max": 53.5429573059082,
      "activations/layer13_attention_weight_min": -37.60890579223633,
      "activations/layer14_attention_weight_max": 78.86906433105469,
      "activations/layer14_attention_weight_min": -61.68743896484375,
      "activations/layer15_attention_weight_max": 49.931297302246094,
      "activations/layer15_attention_weight_min": -34.77457046508789,
      "activations/layer16_attention_weight_max": 44.81287384033203,
      "activations/layer16_attention_weight_min": -32.08683395385742,
      "activations/layer17_attention_weight_max": 41.79575729370117,
      "activations/layer17_attention_weight_min": -27.863279342651367,
      "activations/layer18_attention_weight_max": 28.378286361694336,
      "activations/layer18_attention_weight_min": -17.544124603271484,
      "activations/layer19_attention_weight_max": 48.744747161865234,
      "activations/layer19_attention_weight_min": -27.5143985748291,
      "activations/layer1_attention_weight_max": 16.91407585144043,
      "activations/layer1_attention_weight_min": -11.979668617248535,
      "activations/layer20_attention_weight_max": 39.55773162841797,
      "activations/layer20_attention_weight_min": -22.92120361328125,
      "activations/layer21_attention_weight_max": 30.71544075012207,
      "activations/layer21_attention_weight_min": -19.532848358154297,
      "activations/layer22_attention_weight_max": 43.78477096557617,
      "activations/layer22_attention_weight_min": -26.43825340270996,
      "activations/layer23_attention_weight_max": 25.77594757080078,
      "activations/layer23_attention_weight_min": -17.072811126708984,
      "activations/layer2_attention_weight_max": 29.804786682128906,
      "activations/layer2_attention_weight_min": -29.715763092041016,
      "activations/layer3_attention_weight_max": 75.62547302246094,
      "activations/layer3_attention_weight_min": -75.58206176757812,
      "activations/layer4_attention_weight_max": 130.48391723632812,
      "activations/layer4_attention_weight_min": -117.85235595703125,
      "activations/layer5_attention_weight_max": 111.6593017578125,
      "activations/layer5_attention_weight_min": -94.15866088867188,
      "activations/layer6_attention_weight_max": 59.75956344604492,
      "activations/layer6_attention_weight_min": -51.47239303588867,
      "activations/layer7_attention_weight_max": 79.9757080078125,
      "activations/layer7_attention_weight_min": -62.47058868408203,
      "activations/layer8_attention_weight_max": 45.43714904785156,
      "activations/layer8_attention_weight_min": -36.54653549194336,
      "activations/layer9_attention_weight_max": 75.16978454589844,
      "activations/layer9_attention_weight_min": -56.070709228515625,
      "epoch": 2.57,
      "learning_rate": 0.00013477386363636363,
      "loss": 2.9377,
      "step": 44250
    },
    {
      "activations/layer0_attention_weight_max": 14.290434837341309,
      "activations/layer0_attention_weight_min": -11.272157669067383,
      "activations/layer10_attention_weight_max": 45.33380889892578,
      "activations/layer10_attention_weight_min": -38.097103118896484,
      "activations/layer11_attention_weight_max": 36.767887115478516,
      "activations/layer11_attention_weight_min": -32.015113830566406,
      "activations/layer12_attention_weight_max": 38.573097229003906,
      "activations/layer12_attention_weight_min": -26.97515296936035,
      "activations/layer13_attention_weight_max": 54.95988082885742,
      "activations/layer13_attention_weight_min": -38.51812744140625,
      "activations/layer14_attention_weight_max": 74.64360046386719,
      "activations/layer14_attention_weight_min": -53.36586380004883,
      "activations/layer15_attention_weight_max": 51.30439758300781,
      "activations/layer15_attention_weight_min": -37.00712203979492,
      "activations/layer16_attention_weight_max": 48.4185905456543,
      "activations/layer16_attention_weight_min": -34.06494140625,
      "activations/layer17_attention_weight_max": 44.18084716796875,
      "activations/layer17_attention_weight_min": -27.4228458404541,
      "activations/layer18_attention_weight_max": 30.359262466430664,
      "activations/layer18_attention_weight_min": -19.27962875366211,
      "activations/layer19_attention_weight_max": 52.9344596862793,
      "activations/layer19_attention_weight_min": -32.529541015625,
      "activations/layer1_attention_weight_max": 17.811979293823242,
      "activations/layer1_attention_weight_min": -12.04887866973877,
      "activations/layer20_attention_weight_max": 40.68093490600586,
      "activations/layer20_attention_weight_min": -24.048383712768555,
      "activations/layer21_attention_weight_max": 31.418315887451172,
      "activations/layer21_attention_weight_min": -21.25322723388672,
      "activations/layer22_attention_weight_max": 51.6621208190918,
      "activations/layer22_attention_weight_min": -28.42375373840332,
      "activations/layer23_attention_weight_max": 30.469051361083984,
      "activations/layer23_attention_weight_min": -17.4271240234375,
      "activations/layer2_attention_weight_max": 30.414447784423828,
      "activations/layer2_attention_weight_min": -30.016881942749023,
      "activations/layer3_attention_weight_max": 81.86357879638672,
      "activations/layer3_attention_weight_min": -77.27346801757812,
      "activations/layer4_attention_weight_max": 139.2501983642578,
      "activations/layer4_attention_weight_min": -120.215087890625,
      "activations/layer5_attention_weight_max": 109.27684783935547,
      "activations/layer5_attention_weight_min": -91.13937377929688,
      "activations/layer6_attention_weight_max": 60.056732177734375,
      "activations/layer6_attention_weight_min": -49.314369201660156,
      "activations/layer7_attention_weight_max": 66.50916290283203,
      "activations/layer7_attention_weight_min": -64.17715454101562,
      "activations/layer8_attention_weight_max": 43.16261291503906,
      "activations/layer8_attention_weight_min": -34.51560974121094,
      "activations/layer9_attention_weight_max": 60.00471115112305,
      "activations/layer9_attention_weight_min": -44.6400146484375,
      "epoch": 2.57,
      "learning_rate": 0.00013475492424242422,
      "loss": 2.9417,
      "step": 44300
    },
    {
      "activations/layer0_attention_weight_max": 16.048795700073242,
      "activations/layer0_attention_weight_min": -10.934727668762207,
      "activations/layer10_attention_weight_max": 41.04539489746094,
      "activations/layer10_attention_weight_min": -35.452545166015625,
      "activations/layer11_attention_weight_max": 35.36153030395508,
      "activations/layer11_attention_weight_min": -34.09142303466797,
      "activations/layer12_attention_weight_max": 31.379505157470703,
      "activations/layer12_attention_weight_min": -28.97361183166504,
      "activations/layer13_attention_weight_max": 50.01314926147461,
      "activations/layer13_attention_weight_min": -39.09960174560547,
      "activations/layer14_attention_weight_max": 76.03917694091797,
      "activations/layer14_attention_weight_min": -53.47664260864258,
      "activations/layer15_attention_weight_max": 57.63465118408203,
      "activations/layer15_attention_weight_min": -39.83769226074219,
      "activations/layer16_attention_weight_max": 51.55070877075195,
      "activations/layer16_attention_weight_min": -33.83572006225586,
      "activations/layer17_attention_weight_max": 48.14405059814453,
      "activations/layer17_attention_weight_min": -30.33173942565918,
      "activations/layer18_attention_weight_max": 27.72332763671875,
      "activations/layer18_attention_weight_min": -17.877361297607422,
      "activations/layer19_attention_weight_max": 55.08393478393555,
      "activations/layer19_attention_weight_min": -34.0547981262207,
      "activations/layer1_attention_weight_max": 17.138763427734375,
      "activations/layer1_attention_weight_min": -11.443157196044922,
      "activations/layer20_attention_weight_max": 37.825538635253906,
      "activations/layer20_attention_weight_min": -25.85814094543457,
      "activations/layer21_attention_weight_max": 43.28340148925781,
      "activations/layer21_attention_weight_min": -22.590917587280273,
      "activations/layer22_attention_weight_max": 51.88779830932617,
      "activations/layer22_attention_weight_min": -28.367523193359375,
      "activations/layer23_attention_weight_max": 29.246768951416016,
      "activations/layer23_attention_weight_min": -17.53993797302246,
      "activations/layer2_attention_weight_max": 31.957351684570312,
      "activations/layer2_attention_weight_min": -30.327247619628906,
      "activations/layer3_attention_weight_max": 79.25318145751953,
      "activations/layer3_attention_weight_min": -82.01811218261719,
      "activations/layer4_attention_weight_max": 128.906982421875,
      "activations/layer4_attention_weight_min": -121.20519256591797,
      "activations/layer5_attention_weight_max": 105.2901611328125,
      "activations/layer5_attention_weight_min": -91.45977020263672,
      "activations/layer6_attention_weight_max": 56.88083267211914,
      "activations/layer6_attention_weight_min": -51.41680145263672,
      "activations/layer7_attention_weight_max": 69.83576202392578,
      "activations/layer7_attention_weight_min": -64.80746459960938,
      "activations/layer8_attention_weight_max": 41.7936897277832,
      "activations/layer8_attention_weight_min": -34.28764724731445,
      "activations/layer9_attention_weight_max": 59.48116683959961,
      "activations/layer9_attention_weight_min": -42.81156539916992,
      "epoch": 2.58,
      "learning_rate": 0.00013473598484848483,
      "loss": 2.9378,
      "step": 44350
    },
    {
      "activations/layer0_attention_weight_max": 17.16804313659668,
      "activations/layer0_attention_weight_min": -11.327037811279297,
      "activations/layer10_attention_weight_max": 49.86845779418945,
      "activations/layer10_attention_weight_min": -38.675201416015625,
      "activations/layer11_attention_weight_max": 57.228206634521484,
      "activations/layer11_attention_weight_min": -37.79868698120117,
      "activations/layer12_attention_weight_max": 34.36958694458008,
      "activations/layer12_attention_weight_min": -29.16098976135254,
      "activations/layer13_attention_weight_max": 55.71803665161133,
      "activations/layer13_attention_weight_min": -39.791072845458984,
      "activations/layer14_attention_weight_max": 83.75935363769531,
      "activations/layer14_attention_weight_min": -61.65341567993164,
      "activations/layer15_attention_weight_max": 54.781593322753906,
      "activations/layer15_attention_weight_min": -36.26949691772461,
      "activations/layer16_attention_weight_max": 50.119842529296875,
      "activations/layer16_attention_weight_min": -33.51898193359375,
      "activations/layer17_attention_weight_max": 44.128883361816406,
      "activations/layer17_attention_weight_min": -28.777252197265625,
      "activations/layer18_attention_weight_max": 30.085893630981445,
      "activations/layer18_attention_weight_min": -20.98428726196289,
      "activations/layer19_attention_weight_max": 54.35134506225586,
      "activations/layer19_attention_weight_min": -32.11239242553711,
      "activations/layer1_attention_weight_max": 17.60357093811035,
      "activations/layer1_attention_weight_min": -11.877107620239258,
      "activations/layer20_attention_weight_max": 44.363319396972656,
      "activations/layer20_attention_weight_min": -24.196271896362305,
      "activations/layer21_attention_weight_max": 31.181039810180664,
      "activations/layer21_attention_weight_min": -19.455589294433594,
      "activations/layer22_attention_weight_max": 58.30401611328125,
      "activations/layer22_attention_weight_min": -25.79327392578125,
      "activations/layer23_attention_weight_max": 30.665189743041992,
      "activations/layer23_attention_weight_min": -16.635608673095703,
      "activations/layer2_attention_weight_max": 29.603275299072266,
      "activations/layer2_attention_weight_min": -28.472745895385742,
      "activations/layer3_attention_weight_max": 79.84474182128906,
      "activations/layer3_attention_weight_min": -79.07593536376953,
      "activations/layer4_attention_weight_max": 135.85533142089844,
      "activations/layer4_attention_weight_min": -129.04498291015625,
      "activations/layer5_attention_weight_max": 114.99700927734375,
      "activations/layer5_attention_weight_min": -106.6386947631836,
      "activations/layer6_attention_weight_max": 63.6003303527832,
      "activations/layer6_attention_weight_min": -54.45218276977539,
      "activations/layer7_attention_weight_max": 77.86274719238281,
      "activations/layer7_attention_weight_min": -70.115966796875,
      "activations/layer8_attention_weight_max": 49.75413131713867,
      "activations/layer8_attention_weight_min": -42.67133712768555,
      "activations/layer9_attention_weight_max": 84.31178283691406,
      "activations/layer9_attention_weight_min": -63.058773040771484,
      "epoch": 2.58,
      "learning_rate": 0.00013471704545454545,
      "loss": 2.9437,
      "step": 44400
    },
    {
      "activations/layer0_attention_weight_max": 17.31424903869629,
      "activations/layer0_attention_weight_min": -11.182649612426758,
      "activations/layer10_attention_weight_max": 49.31480026245117,
      "activations/layer10_attention_weight_min": -38.874900817871094,
      "activations/layer11_attention_weight_max": 41.880950927734375,
      "activations/layer11_attention_weight_min": -35.62554168701172,
      "activations/layer12_attention_weight_max": 34.55466842651367,
      "activations/layer12_attention_weight_min": -27.443029403686523,
      "activations/layer13_attention_weight_max": 60.0482063293457,
      "activations/layer13_attention_weight_min": -39.14613342285156,
      "activations/layer14_attention_weight_max": 77.38751220703125,
      "activations/layer14_attention_weight_min": -55.51359558105469,
      "activations/layer15_attention_weight_max": 49.26422882080078,
      "activations/layer15_attention_weight_min": -36.92582321166992,
      "activations/layer16_attention_weight_max": 48.65236282348633,
      "activations/layer16_attention_weight_min": -31.86557960510254,
      "activations/layer17_attention_weight_max": 48.27540969848633,
      "activations/layer17_attention_weight_min": -28.50954246520996,
      "activations/layer18_attention_weight_max": 29.322404861450195,
      "activations/layer18_attention_weight_min": -19.90310287475586,
      "activations/layer19_attention_weight_max": 51.7005729675293,
      "activations/layer19_attention_weight_min": -30.148391723632812,
      "activations/layer1_attention_weight_max": 17.55596160888672,
      "activations/layer1_attention_weight_min": -13.17712116241455,
      "activations/layer20_attention_weight_max": 37.82741165161133,
      "activations/layer20_attention_weight_min": -24.69682502746582,
      "activations/layer21_attention_weight_max": 28.32220458984375,
      "activations/layer21_attention_weight_min": -18.414342880249023,
      "activations/layer22_attention_weight_max": 50.33134078979492,
      "activations/layer22_attention_weight_min": -26.86015510559082,
      "activations/layer23_attention_weight_max": 27.201465606689453,
      "activations/layer23_attention_weight_min": -17.231056213378906,
      "activations/layer2_attention_weight_max": 30.754995346069336,
      "activations/layer2_attention_weight_min": -30.04498863220215,
      "activations/layer3_attention_weight_max": 78.830078125,
      "activations/layer3_attention_weight_min": -82.85260009765625,
      "activations/layer4_attention_weight_max": 132.41514587402344,
      "activations/layer4_attention_weight_min": -124.36488342285156,
      "activations/layer5_attention_weight_max": 107.64547729492188,
      "activations/layer5_attention_weight_min": -97.75634002685547,
      "activations/layer6_attention_weight_max": 60.58013916015625,
      "activations/layer6_attention_weight_min": -56.32132339477539,
      "activations/layer7_attention_weight_max": 66.78538513183594,
      "activations/layer7_attention_weight_min": -65.63075256347656,
      "activations/layer8_attention_weight_max": 43.749244689941406,
      "activations/layer8_attention_weight_min": -37.17609786987305,
      "activations/layer9_attention_weight_max": 57.361968994140625,
      "activations/layer9_attention_weight_min": -43.8801383972168,
      "epoch": 2.58,
      "learning_rate": 0.00013469810606060603,
      "loss": 2.9462,
      "step": 44450
    },
    {
      "activations/layer0_attention_weight_max": 15.690852165222168,
      "activations/layer0_attention_weight_min": -10.988020896911621,
      "activations/layer10_attention_weight_max": 50.160614013671875,
      "activations/layer10_attention_weight_min": -38.44274139404297,
      "activations/layer11_attention_weight_max": 44.044334411621094,
      "activations/layer11_attention_weight_min": -33.914485931396484,
      "activations/layer12_attention_weight_max": 61.661354064941406,
      "activations/layer12_attention_weight_min": -34.9970703125,
      "activations/layer13_attention_weight_max": 72.37112426757812,
      "activations/layer13_attention_weight_min": -47.31336212158203,
      "activations/layer14_attention_weight_max": 83.29988861083984,
      "activations/layer14_attention_weight_min": -57.230201721191406,
      "activations/layer15_attention_weight_max": 52.09402847290039,
      "activations/layer15_attention_weight_min": -35.42926788330078,
      "activations/layer16_attention_weight_max": 46.727073669433594,
      "activations/layer16_attention_weight_min": -33.170997619628906,
      "activations/layer17_attention_weight_max": 44.22372055053711,
      "activations/layer17_attention_weight_min": -28.197635650634766,
      "activations/layer18_attention_weight_max": 30.594486236572266,
      "activations/layer18_attention_weight_min": -19.241308212280273,
      "activations/layer19_attention_weight_max": 47.74475860595703,
      "activations/layer19_attention_weight_min": -29.02872085571289,
      "activations/layer1_attention_weight_max": 17.490943908691406,
      "activations/layer1_attention_weight_min": -12.499935150146484,
      "activations/layer20_attention_weight_max": 38.505367279052734,
      "activations/layer20_attention_weight_min": -22.4359188079834,
      "activations/layer21_attention_weight_max": 32.58831024169922,
      "activations/layer21_attention_weight_min": -18.197284698486328,
      "activations/layer22_attention_weight_max": 55.83113479614258,
      "activations/layer22_attention_weight_min": -26.65204429626465,
      "activations/layer23_attention_weight_max": 25.482635498046875,
      "activations/layer23_attention_weight_min": -15.132299423217773,
      "activations/layer2_attention_weight_max": 30.228622436523438,
      "activations/layer2_attention_weight_min": -29.80914878845215,
      "activations/layer3_attention_weight_max": 75.77039337158203,
      "activations/layer3_attention_weight_min": -76.79878997802734,
      "activations/layer4_attention_weight_max": 130.7433319091797,
      "activations/layer4_attention_weight_min": -122.29689025878906,
      "activations/layer5_attention_weight_max": 109.42733764648438,
      "activations/layer5_attention_weight_min": -92.89108276367188,
      "activations/layer6_attention_weight_max": 60.3731803894043,
      "activations/layer6_attention_weight_min": -52.76753234863281,
      "activations/layer7_attention_weight_max": 73.15579223632812,
      "activations/layer7_attention_weight_min": -66.67063903808594,
      "activations/layer8_attention_weight_max": 45.137596130371094,
      "activations/layer8_attention_weight_min": -38.084373474121094,
      "activations/layer9_attention_weight_max": 65.46491241455078,
      "activations/layer9_attention_weight_min": -46.227012634277344,
      "epoch": 2.59,
      "learning_rate": 0.00013467916666666665,
      "loss": 2.9397,
      "step": 44500
    },
    {
      "activations/layer0_attention_weight_max": 17.00725746154785,
      "activations/layer0_attention_weight_min": -11.46396255493164,
      "activations/layer10_attention_weight_max": 46.261287689208984,
      "activations/layer10_attention_weight_min": -36.226802825927734,
      "activations/layer11_attention_weight_max": 41.9853401184082,
      "activations/layer11_attention_weight_min": -32.819332122802734,
      "activations/layer12_attention_weight_max": 51.90806579589844,
      "activations/layer12_attention_weight_min": -27.02586555480957,
      "activations/layer13_attention_weight_max": 82.84968566894531,
      "activations/layer13_attention_weight_min": -48.08784484863281,
      "activations/layer14_attention_weight_max": 78.8085708618164,
      "activations/layer14_attention_weight_min": -56.02772903442383,
      "activations/layer15_attention_weight_max": 53.10591125488281,
      "activations/layer15_attention_weight_min": -38.06383514404297,
      "activations/layer16_attention_weight_max": 49.613826751708984,
      "activations/layer16_attention_weight_min": -33.66099548339844,
      "activations/layer17_attention_weight_max": 46.73884201049805,
      "activations/layer17_attention_weight_min": -28.392892837524414,
      "activations/layer18_attention_weight_max": 34.153175354003906,
      "activations/layer18_attention_weight_min": -18.562725067138672,
      "activations/layer19_attention_weight_max": 53.76226806640625,
      "activations/layer19_attention_weight_min": -30.630126953125,
      "activations/layer1_attention_weight_max": 18.158329010009766,
      "activations/layer1_attention_weight_min": -11.228654861450195,
      "activations/layer20_attention_weight_max": 40.26530838012695,
      "activations/layer20_attention_weight_min": -22.154571533203125,
      "activations/layer21_attention_weight_max": 34.74818801879883,
      "activations/layer21_attention_weight_min": -19.2979793548584,
      "activations/layer22_attention_weight_max": 60.755760192871094,
      "activations/layer22_attention_weight_min": -28.46544075012207,
      "activations/layer23_attention_weight_max": 32.148624420166016,
      "activations/layer23_attention_weight_min": -17.310752868652344,
      "activations/layer2_attention_weight_max": 29.249725341796875,
      "activations/layer2_attention_weight_min": -30.48177719116211,
      "activations/layer3_attention_weight_max": 74.8875961303711,
      "activations/layer3_attention_weight_min": -80.87264251708984,
      "activations/layer4_attention_weight_max": 129.4777069091797,
      "activations/layer4_attention_weight_min": -122.62188720703125,
      "activations/layer5_attention_weight_max": 108.51225280761719,
      "activations/layer5_attention_weight_min": -97.32232666015625,
      "activations/layer6_attention_weight_max": 61.44406509399414,
      "activations/layer6_attention_weight_min": -57.1540641784668,
      "activations/layer7_attention_weight_max": 73.61780548095703,
      "activations/layer7_attention_weight_min": -67.831298828125,
      "activations/layer8_attention_weight_max": 47.22822952270508,
      "activations/layer8_attention_weight_min": -42.491127014160156,
      "activations/layer9_attention_weight_max": 78.13361358642578,
      "activations/layer9_attention_weight_min": -58.928558349609375,
      "epoch": 2.59,
      "learning_rate": 0.00013466022727272726,
      "loss": 2.9227,
      "step": 44550
    },
    {
      "activations/layer0_attention_weight_max": 17.434932708740234,
      "activations/layer0_attention_weight_min": -11.287915229797363,
      "activations/layer10_attention_weight_max": 53.33354949951172,
      "activations/layer10_attention_weight_min": -36.66643524169922,
      "activations/layer11_attention_weight_max": 41.939239501953125,
      "activations/layer11_attention_weight_min": -33.071842193603516,
      "activations/layer12_attention_weight_max": 46.52696228027344,
      "activations/layer12_attention_weight_min": -31.640047073364258,
      "activations/layer13_attention_weight_max": 68.0641860961914,
      "activations/layer13_attention_weight_min": -43.99106979370117,
      "activations/layer14_attention_weight_max": 90.7320785522461,
      "activations/layer14_attention_weight_min": -55.86958312988281,
      "activations/layer15_attention_weight_max": 54.342185974121094,
      "activations/layer15_attention_weight_min": -35.50945281982422,
      "activations/layer16_attention_weight_max": 51.610816955566406,
      "activations/layer16_attention_weight_min": -31.561182022094727,
      "activations/layer17_attention_weight_max": 50.461585998535156,
      "activations/layer17_attention_weight_min": -27.668331146240234,
      "activations/layer18_attention_weight_max": 39.57423782348633,
      "activations/layer18_attention_weight_min": -19.802196502685547,
      "activations/layer19_attention_weight_max": 58.68380355834961,
      "activations/layer19_attention_weight_min": -29.9931697845459,
      "activations/layer1_attention_weight_max": 18.28596305847168,
      "activations/layer1_attention_weight_min": -11.558396339416504,
      "activations/layer20_attention_weight_max": 45.08661651611328,
      "activations/layer20_attention_weight_min": -23.713468551635742,
      "activations/layer21_attention_weight_max": 38.6851692199707,
      "activations/layer21_attention_weight_min": -19.41288185119629,
      "activations/layer22_attention_weight_max": 55.697261810302734,
      "activations/layer22_attention_weight_min": -30.844703674316406,
      "activations/layer23_attention_weight_max": 28.299278259277344,
      "activations/layer23_attention_weight_min": -16.636262893676758,
      "activations/layer2_attention_weight_max": 31.214384078979492,
      "activations/layer2_attention_weight_min": -30.12596893310547,
      "activations/layer3_attention_weight_max": 77.75743103027344,
      "activations/layer3_attention_weight_min": -81.43280029296875,
      "activations/layer4_attention_weight_max": 140.11570739746094,
      "activations/layer4_attention_weight_min": -124.76600646972656,
      "activations/layer5_attention_weight_max": 116.73049926757812,
      "activations/layer5_attention_weight_min": -95.85054779052734,
      "activations/layer6_attention_weight_max": 63.2456169128418,
      "activations/layer6_attention_weight_min": -54.237430572509766,
      "activations/layer7_attention_weight_max": 70.66046142578125,
      "activations/layer7_attention_weight_min": -66.79869842529297,
      "activations/layer8_attention_weight_max": 46.888160705566406,
      "activations/layer8_attention_weight_min": -37.12885284423828,
      "activations/layer9_attention_weight_max": 60.519813537597656,
      "activations/layer9_attention_weight_min": -49.219940185546875,
      "epoch": 2.59,
      "learning_rate": 0.00013464166666666666,
      "loss": 2.9249,
      "step": 44600
    },
    {
      "activations/layer0_attention_weight_max": 15.996923446655273,
      "activations/layer0_attention_weight_min": -11.439287185668945,
      "activations/layer10_attention_weight_max": 53.52344512939453,
      "activations/layer10_attention_weight_min": -41.796661376953125,
      "activations/layer11_attention_weight_max": 50.15436935424805,
      "activations/layer11_attention_weight_min": -33.64552688598633,
      "activations/layer12_attention_weight_max": 60.99472427368164,
      "activations/layer12_attention_weight_min": -34.746376037597656,
      "activations/layer13_attention_weight_max": 63.73591995239258,
      "activations/layer13_attention_weight_min": -46.39833450317383,
      "activations/layer14_attention_weight_max": 76.63302612304688,
      "activations/layer14_attention_weight_min": -56.26091766357422,
      "activations/layer15_attention_weight_max": 58.10810089111328,
      "activations/layer15_attention_weight_min": -34.501068115234375,
      "activations/layer16_attention_weight_max": 45.559574127197266,
      "activations/layer16_attention_weight_min": -35.56614303588867,
      "activations/layer17_attention_weight_max": 44.926353454589844,
      "activations/layer17_attention_weight_min": -28.369815826416016,
      "activations/layer18_attention_weight_max": 29.00848388671875,
      "activations/layer18_attention_weight_min": -19.20585060119629,
      "activations/layer19_attention_weight_max": 49.48344039916992,
      "activations/layer19_attention_weight_min": -34.67343521118164,
      "activations/layer1_attention_weight_max": 17.868850708007812,
      "activations/layer1_attention_weight_min": -12.114458084106445,
      "activations/layer20_attention_weight_max": 33.65980529785156,
      "activations/layer20_attention_weight_min": -23.698497772216797,
      "activations/layer21_attention_weight_max": 33.105194091796875,
      "activations/layer21_attention_weight_min": -20.99889373779297,
      "activations/layer22_attention_weight_max": 46.78343200683594,
      "activations/layer22_attention_weight_min": -29.70443344116211,
      "activations/layer23_attention_weight_max": 24.05426025390625,
      "activations/layer23_attention_weight_min": -20.88335418701172,
      "activations/layer2_attention_weight_max": 32.27318572998047,
      "activations/layer2_attention_weight_min": -30.038633346557617,
      "activations/layer3_attention_weight_max": 82.41706085205078,
      "activations/layer3_attention_weight_min": -80.54963684082031,
      "activations/layer4_attention_weight_max": 145.4185791015625,
      "activations/layer4_attention_weight_min": -127.44954681396484,
      "activations/layer5_attention_weight_max": 118.62272644042969,
      "activations/layer5_attention_weight_min": -97.26765441894531,
      "activations/layer6_attention_weight_max": 65.6196517944336,
      "activations/layer6_attention_weight_min": -52.751338958740234,
      "activations/layer7_attention_weight_max": 74.73548126220703,
      "activations/layer7_attention_weight_min": -71.57764434814453,
      "activations/layer8_attention_weight_max": 53.66770553588867,
      "activations/layer8_attention_weight_min": -40.24394226074219,
      "activations/layer9_attention_weight_max": 80.70282745361328,
      "activations/layer9_attention_weight_min": -56.26808547973633,
      "epoch": 2.59,
      "learning_rate": 0.00013462272727272724,
      "loss": 2.9397,
      "step": 44650
    },
    {
      "activations/layer0_attention_weight_max": 15.301261901855469,
      "activations/layer0_attention_weight_min": -11.264577865600586,
      "activations/layer10_attention_weight_max": 43.194278717041016,
      "activations/layer10_attention_weight_min": -37.09737777709961,
      "activations/layer11_attention_weight_max": 34.704952239990234,
      "activations/layer11_attention_weight_min": -34.64464569091797,
      "activations/layer12_attention_weight_max": 38.67262649536133,
      "activations/layer12_attention_weight_min": -28.445701599121094,
      "activations/layer13_attention_weight_max": 50.61604309082031,
      "activations/layer13_attention_weight_min": -36.848045349121094,
      "activations/layer14_attention_weight_max": 66.6576156616211,
      "activations/layer14_attention_weight_min": -53.04004669189453,
      "activations/layer15_attention_weight_max": 52.95769119262695,
      "activations/layer15_attention_weight_min": -39.75260925292969,
      "activations/layer16_attention_weight_max": 47.280887603759766,
      "activations/layer16_attention_weight_min": -33.210601806640625,
      "activations/layer17_attention_weight_max": 44.26592254638672,
      "activations/layer17_attention_weight_min": -30.569332122802734,
      "activations/layer18_attention_weight_max": 27.781972885131836,
      "activations/layer18_attention_weight_min": -19.102584838867188,
      "activations/layer19_attention_weight_max": 47.9691162109375,
      "activations/layer19_attention_weight_min": -29.327878952026367,
      "activations/layer1_attention_weight_max": 17.833093643188477,
      "activations/layer1_attention_weight_min": -11.31999683380127,
      "activations/layer20_attention_weight_max": 34.49870300292969,
      "activations/layer20_attention_weight_min": -22.508359909057617,
      "activations/layer21_attention_weight_max": 32.59537124633789,
      "activations/layer21_attention_weight_min": -21.19919776916504,
      "activations/layer22_attention_weight_max": 50.23701858520508,
      "activations/layer22_attention_weight_min": -28.10002899169922,
      "activations/layer23_attention_weight_max": 24.35588836669922,
      "activations/layer23_attention_weight_min": -15.694392204284668,
      "activations/layer2_attention_weight_max": 30.137287139892578,
      "activations/layer2_attention_weight_min": -29.66823387145996,
      "activations/layer3_attention_weight_max": 75.11156463623047,
      "activations/layer3_attention_weight_min": -83.1691665649414,
      "activations/layer4_attention_weight_max": 124.3604507446289,
      "activations/layer4_attention_weight_min": -129.0501708984375,
      "activations/layer5_attention_weight_max": 99.35673522949219,
      "activations/layer5_attention_weight_min": -90.73774719238281,
      "activations/layer6_attention_weight_max": 54.35205841064453,
      "activations/layer6_attention_weight_min": -50.32139205932617,
      "activations/layer7_attention_weight_max": 66.04356384277344,
      "activations/layer7_attention_weight_min": -63.12588882446289,
      "activations/layer8_attention_weight_max": 39.726898193359375,
      "activations/layer8_attention_weight_min": -36.25326156616211,
      "activations/layer9_attention_weight_max": 50.02177047729492,
      "activations/layer9_attention_weight_min": -40.4467887878418,
      "epoch": 2.6,
      "learning_rate": 0.00013460378787878786,
      "loss": 2.9504,
      "step": 44700
    },
    {
      "activations/layer0_attention_weight_max": 16.402292251586914,
      "activations/layer0_attention_weight_min": -11.023693084716797,
      "activations/layer10_attention_weight_max": 48.27106857299805,
      "activations/layer10_attention_weight_min": -41.67387008666992,
      "activations/layer11_attention_weight_max": 48.35902404785156,
      "activations/layer11_attention_weight_min": -37.083168029785156,
      "activations/layer12_attention_weight_max": 34.497894287109375,
      "activations/layer12_attention_weight_min": -26.239761352539062,
      "activations/layer13_attention_weight_max": 70.7940902709961,
      "activations/layer13_attention_weight_min": -47.73332595825195,
      "activations/layer14_attention_weight_max": 82.45508575439453,
      "activations/layer14_attention_weight_min": -60.709354400634766,
      "activations/layer15_attention_weight_max": 59.07857894897461,
      "activations/layer15_attention_weight_min": -38.885955810546875,
      "activations/layer16_attention_weight_max": 45.211509704589844,
      "activations/layer16_attention_weight_min": -34.11698913574219,
      "activations/layer17_attention_weight_max": 42.30469512939453,
      "activations/layer17_attention_weight_min": -29.91087532043457,
      "activations/layer18_attention_weight_max": 33.552921295166016,
      "activations/layer18_attention_weight_min": -19.6141300201416,
      "activations/layer19_attention_weight_max": 47.65900802612305,
      "activations/layer19_attention_weight_min": -32.46865463256836,
      "activations/layer1_attention_weight_max": 17.59218978881836,
      "activations/layer1_attention_weight_min": -13.717316627502441,
      "activations/layer20_attention_weight_max": 36.12955093383789,
      "activations/layer20_attention_weight_min": -24.3792667388916,
      "activations/layer21_attention_weight_max": 33.36151885986328,
      "activations/layer21_attention_weight_min": -20.43144989013672,
      "activations/layer22_attention_weight_max": 45.4805793762207,
      "activations/layer22_attention_weight_min": -28.3995304107666,
      "activations/layer23_attention_weight_max": 26.60706901550293,
      "activations/layer23_attention_weight_min": -16.568439483642578,
      "activations/layer2_attention_weight_max": 31.672849655151367,
      "activations/layer2_attention_weight_min": -32.64549255371094,
      "activations/layer3_attention_weight_max": 83.24879455566406,
      "activations/layer3_attention_weight_min": -85.17405700683594,
      "activations/layer4_attention_weight_max": 133.96356201171875,
      "activations/layer4_attention_weight_min": -123.8936538696289,
      "activations/layer5_attention_weight_max": 110.96551513671875,
      "activations/layer5_attention_weight_min": -98.25909423828125,
      "activations/layer6_attention_weight_max": 59.998085021972656,
      "activations/layer6_attention_weight_min": -52.8037223815918,
      "activations/layer7_attention_weight_max": 71.62664794921875,
      "activations/layer7_attention_weight_min": -72.05216217041016,
      "activations/layer8_attention_weight_max": 48.82626724243164,
      "activations/layer8_attention_weight_min": -37.72310256958008,
      "activations/layer9_attention_weight_max": 73.6475830078125,
      "activations/layer9_attention_weight_min": -59.57563018798828,
      "epoch": 2.6,
      "learning_rate": 0.00013458484848484847,
      "loss": 2.9361,
      "step": 44750
    },
    {
      "activations/layer0_attention_weight_max": 16.828828811645508,
      "activations/layer0_attention_weight_min": -11.62843132019043,
      "activations/layer10_attention_weight_max": 43.4761848449707,
      "activations/layer10_attention_weight_min": -35.599090576171875,
      "activations/layer11_attention_weight_max": 40.52253341674805,
      "activations/layer11_attention_weight_min": -29.708669662475586,
      "activations/layer12_attention_weight_max": 31.244367599487305,
      "activations/layer12_attention_weight_min": -34.146942138671875,
      "activations/layer13_attention_weight_max": 46.836612701416016,
      "activations/layer13_attention_weight_min": -36.95103454589844,
      "activations/layer14_attention_weight_max": 68.88964080810547,
      "activations/layer14_attention_weight_min": -51.78185272216797,
      "activations/layer15_attention_weight_max": 49.4478874206543,
      "activations/layer15_attention_weight_min": -31.886295318603516,
      "activations/layer16_attention_weight_max": 47.93525314331055,
      "activations/layer16_attention_weight_min": -35.251976013183594,
      "activations/layer17_attention_weight_max": 41.31446838378906,
      "activations/layer17_attention_weight_min": -27.858774185180664,
      "activations/layer18_attention_weight_max": 27.178129196166992,
      "activations/layer18_attention_weight_min": -16.99518394470215,
      "activations/layer19_attention_weight_max": 44.86731719970703,
      "activations/layer19_attention_weight_min": -29.633527755737305,
      "activations/layer1_attention_weight_max": 17.091333389282227,
      "activations/layer1_attention_weight_min": -11.775124549865723,
      "activations/layer20_attention_weight_max": 34.30503845214844,
      "activations/layer20_attention_weight_min": -23.027965545654297,
      "activations/layer21_attention_weight_max": 27.6386775970459,
      "activations/layer21_attention_weight_min": -20.069869995117188,
      "activations/layer22_attention_weight_max": 42.08595275878906,
      "activations/layer22_attention_weight_min": -28.5600528717041,
      "activations/layer23_attention_weight_max": 23.77046012878418,
      "activations/layer23_attention_weight_min": -17.632444381713867,
      "activations/layer2_attention_weight_max": 30.354475021362305,
      "activations/layer2_attention_weight_min": -30.629974365234375,
      "activations/layer3_attention_weight_max": 78.8127670288086,
      "activations/layer3_attention_weight_min": -80.04045867919922,
      "activations/layer4_attention_weight_max": 133.28440856933594,
      "activations/layer4_attention_weight_min": -119.77489471435547,
      "activations/layer5_attention_weight_max": 110.23018646240234,
      "activations/layer5_attention_weight_min": -98.06010437011719,
      "activations/layer6_attention_weight_max": 61.12998580932617,
      "activations/layer6_attention_weight_min": -53.432411193847656,
      "activations/layer7_attention_weight_max": 69.15693664550781,
      "activations/layer7_attention_weight_min": -66.20877075195312,
      "activations/layer8_attention_weight_max": 44.26016616821289,
      "activations/layer8_attention_weight_min": -38.48655319213867,
      "activations/layer9_attention_weight_max": 62.550331115722656,
      "activations/layer9_attention_weight_min": -51.86854934692383,
      "epoch": 2.6,
      "learning_rate": 0.00013456590909090906,
      "loss": 2.9408,
      "step": 44800
    },
    {
      "activations/layer0_attention_weight_max": 17.00884437561035,
      "activations/layer0_attention_weight_min": -11.248710632324219,
      "activations/layer10_attention_weight_max": 47.56427764892578,
      "activations/layer10_attention_weight_min": -39.616512298583984,
      "activations/layer11_attention_weight_max": 36.990455627441406,
      "activations/layer11_attention_weight_min": -31.591236114501953,
      "activations/layer12_attention_weight_max": 33.68604278564453,
      "activations/layer12_attention_weight_min": -24.926185607910156,
      "activations/layer13_attention_weight_max": 51.41153335571289,
      "activations/layer13_attention_weight_min": -38.92983627319336,
      "activations/layer14_attention_weight_max": 67.5994644165039,
      "activations/layer14_attention_weight_min": -54.701393127441406,
      "activations/layer15_attention_weight_max": 50.14522933959961,
      "activations/layer15_attention_weight_min": -35.22921371459961,
      "activations/layer16_attention_weight_max": 43.83667755126953,
      "activations/layer16_attention_weight_min": -34.57313537597656,
      "activations/layer17_attention_weight_max": 45.11539840698242,
      "activations/layer17_attention_weight_min": -30.388444900512695,
      "activations/layer18_attention_weight_max": 26.11469078063965,
      "activations/layer18_attention_weight_min": -20.7647762298584,
      "activations/layer19_attention_weight_max": 48.06425094604492,
      "activations/layer19_attention_weight_min": -31.352689743041992,
      "activations/layer1_attention_weight_max": 17.690513610839844,
      "activations/layer1_attention_weight_min": -12.080491065979004,
      "activations/layer20_attention_weight_max": 33.73508834838867,
      "activations/layer20_attention_weight_min": -24.740800857543945,
      "activations/layer21_attention_weight_max": 34.08594512939453,
      "activations/layer21_attention_weight_min": -19.671977996826172,
      "activations/layer22_attention_weight_max": 45.546478271484375,
      "activations/layer22_attention_weight_min": -28.30244255065918,
      "activations/layer23_attention_weight_max": 23.955841064453125,
      "activations/layer23_attention_weight_min": -16.259729385375977,
      "activations/layer2_attention_weight_max": 31.594078063964844,
      "activations/layer2_attention_weight_min": -28.295177459716797,
      "activations/layer3_attention_weight_max": 73.65934753417969,
      "activations/layer3_attention_weight_min": -75.24053955078125,
      "activations/layer4_attention_weight_max": 127.0576171875,
      "activations/layer4_attention_weight_min": -123.27099609375,
      "activations/layer5_attention_weight_max": 103.59455871582031,
      "activations/layer5_attention_weight_min": -90.45528411865234,
      "activations/layer6_attention_weight_max": 58.1002082824707,
      "activations/layer6_attention_weight_min": -51.412174224853516,
      "activations/layer7_attention_weight_max": 68.47427368164062,
      "activations/layer7_attention_weight_min": -70.8189926147461,
      "activations/layer8_attention_weight_max": 46.331668853759766,
      "activations/layer8_attention_weight_min": -37.48857879638672,
      "activations/layer9_attention_weight_max": 57.95267868041992,
      "activations/layer9_attention_weight_min": -52.73460006713867,
      "epoch": 2.61,
      "learning_rate": 0.00013454696969696968,
      "loss": 2.9303,
      "step": 44850
    },
    {
      "activations/layer0_attention_weight_max": 16.85999298095703,
      "activations/layer0_attention_weight_min": -11.558684349060059,
      "activations/layer10_attention_weight_max": 44.593658447265625,
      "activations/layer10_attention_weight_min": -34.9622688293457,
      "activations/layer11_attention_weight_max": 44.39756774902344,
      "activations/layer11_attention_weight_min": -34.020721435546875,
      "activations/layer12_attention_weight_max": 49.84299087524414,
      "activations/layer12_attention_weight_min": -29.35350227355957,
      "activations/layer13_attention_weight_max": 66.994873046875,
      "activations/layer13_attention_weight_min": -50.020206451416016,
      "activations/layer14_attention_weight_max": 87.61122131347656,
      "activations/layer14_attention_weight_min": -59.065914154052734,
      "activations/layer15_attention_weight_max": 62.742191314697266,
      "activations/layer15_attention_weight_min": -36.054283142089844,
      "activations/layer16_attention_weight_max": 51.97538757324219,
      "activations/layer16_attention_weight_min": -39.2751350402832,
      "activations/layer17_attention_weight_max": 50.2105598449707,
      "activations/layer17_attention_weight_min": -28.39773941040039,
      "activations/layer18_attention_weight_max": 32.97841262817383,
      "activations/layer18_attention_weight_min": -18.914030075073242,
      "activations/layer19_attention_weight_max": 59.9859733581543,
      "activations/layer19_attention_weight_min": -34.90462875366211,
      "activations/layer1_attention_weight_max": 17.082006454467773,
      "activations/layer1_attention_weight_min": -10.88688850402832,
      "activations/layer20_attention_weight_max": 46.708274841308594,
      "activations/layer20_attention_weight_min": -24.27121353149414,
      "activations/layer21_attention_weight_max": 39.61715316772461,
      "activations/layer21_attention_weight_min": -20.205799102783203,
      "activations/layer22_attention_weight_max": 57.41148376464844,
      "activations/layer22_attention_weight_min": -28.120948791503906,
      "activations/layer23_attention_weight_max": 31.773298263549805,
      "activations/layer23_attention_weight_min": -19.188825607299805,
      "activations/layer2_attention_weight_max": 28.549163818359375,
      "activations/layer2_attention_weight_min": -27.72003936767578,
      "activations/layer3_attention_weight_max": 75.87003326416016,
      "activations/layer3_attention_weight_min": -77.35897827148438,
      "activations/layer4_attention_weight_max": 125.01583099365234,
      "activations/layer4_attention_weight_min": -119.56732177734375,
      "activations/layer5_attention_weight_max": 105.71974182128906,
      "activations/layer5_attention_weight_min": -93.65850067138672,
      "activations/layer6_attention_weight_max": 59.60606384277344,
      "activations/layer6_attention_weight_min": -51.8145637512207,
      "activations/layer7_attention_weight_max": 69.84386444091797,
      "activations/layer7_attention_weight_min": -64.71331024169922,
      "activations/layer8_attention_weight_max": 43.29999923706055,
      "activations/layer8_attention_weight_min": -38.754791259765625,
      "activations/layer9_attention_weight_max": 56.6930046081543,
      "activations/layer9_attention_weight_min": -42.07437515258789,
      "epoch": 2.61,
      "learning_rate": 0.0001345280303030303,
      "loss": 2.9224,
      "step": 44900
    },
    {
      "activations/layer0_attention_weight_max": 16.127073287963867,
      "activations/layer0_attention_weight_min": -11.567203521728516,
      "activations/layer10_attention_weight_max": 42.79392623901367,
      "activations/layer10_attention_weight_min": -36.64930725097656,
      "activations/layer11_attention_weight_max": 36.68785858154297,
      "activations/layer11_attention_weight_min": -33.0329704284668,
      "activations/layer12_attention_weight_max": 29.946504592895508,
      "activations/layer12_attention_weight_min": -24.70400619506836,
      "activations/layer13_attention_weight_max": 50.00547790527344,
      "activations/layer13_attention_weight_min": -39.955284118652344,
      "activations/layer14_attention_weight_max": 79.42859649658203,
      "activations/layer14_attention_weight_min": -55.688568115234375,
      "activations/layer15_attention_weight_max": 46.866512298583984,
      "activations/layer15_attention_weight_min": -33.529296875,
      "activations/layer16_attention_weight_max": 41.3217887878418,
      "activations/layer16_attention_weight_min": -31.382705688476562,
      "activations/layer17_attention_weight_max": 41.2574462890625,
      "activations/layer17_attention_weight_min": -28.227039337158203,
      "activations/layer18_attention_weight_max": 28.76411247253418,
      "activations/layer18_attention_weight_min": -19.639583587646484,
      "activations/layer19_attention_weight_max": 42.72678756713867,
      "activations/layer19_attention_weight_min": -29.835861206054688,
      "activations/layer1_attention_weight_max": 17.9941349029541,
      "activations/layer1_attention_weight_min": -12.333232879638672,
      "activations/layer20_attention_weight_max": 33.531002044677734,
      "activations/layer20_attention_weight_min": -25.08809471130371,
      "activations/layer21_attention_weight_max": 30.704818725585938,
      "activations/layer21_attention_weight_min": -21.067163467407227,
      "activations/layer22_attention_weight_max": 45.11374282836914,
      "activations/layer22_attention_weight_min": -28.34339714050293,
      "activations/layer23_attention_weight_max": 27.830276489257812,
      "activations/layer23_attention_weight_min": -18.813793182373047,
      "activations/layer2_attention_weight_max": 30.218589782714844,
      "activations/layer2_attention_weight_min": -29.9400634765625,
      "activations/layer3_attention_weight_max": 76.64082336425781,
      "activations/layer3_attention_weight_min": -83.51525115966797,
      "activations/layer4_attention_weight_max": 133.49427795410156,
      "activations/layer4_attention_weight_min": -119.32130432128906,
      "activations/layer5_attention_weight_max": 105.1690902709961,
      "activations/layer5_attention_weight_min": -95.12551879882812,
      "activations/layer6_attention_weight_max": 58.784751892089844,
      "activations/layer6_attention_weight_min": -50.777748107910156,
      "activations/layer7_attention_weight_max": 65.04573059082031,
      "activations/layer7_attention_weight_min": -62.12314224243164,
      "activations/layer8_attention_weight_max": 39.473209381103516,
      "activations/layer8_attention_weight_min": -37.681217193603516,
      "activations/layer9_attention_weight_max": 58.71904373168945,
      "activations/layer9_attention_weight_min": -45.462188720703125,
      "epoch": 2.61,
      "learning_rate": 0.0001345090909090909,
      "loss": 2.9272,
      "step": 44950
    },
    {
      "activations/layer0_attention_weight_max": 15.268495559692383,
      "activations/layer0_attention_weight_min": -11.35085678100586,
      "activations/layer10_attention_weight_max": 48.91827392578125,
      "activations/layer10_attention_weight_min": -42.782955169677734,
      "activations/layer11_attention_weight_max": 37.071449279785156,
      "activations/layer11_attention_weight_min": -31.85848617553711,
      "activations/layer12_attention_weight_max": 34.536582946777344,
      "activations/layer12_attention_weight_min": -26.07184600830078,
      "activations/layer13_attention_weight_max": 50.06222152709961,
      "activations/layer13_attention_weight_min": -37.05573654174805,
      "activations/layer14_attention_weight_max": 89.13513946533203,
      "activations/layer14_attention_weight_min": -72.61493682861328,
      "activations/layer15_attention_weight_max": 47.86524963378906,
      "activations/layer15_attention_weight_min": -36.53236389160156,
      "activations/layer16_attention_weight_max": 44.14377212524414,
      "activations/layer16_attention_weight_min": -35.41042709350586,
      "activations/layer17_attention_weight_max": 47.628170013427734,
      "activations/layer17_attention_weight_min": -30.128726959228516,
      "activations/layer18_attention_weight_max": 29.05776596069336,
      "activations/layer18_attention_weight_min": -20.04839515686035,
      "activations/layer19_attention_weight_max": 47.02346420288086,
      "activations/layer19_attention_weight_min": -34.21688461303711,
      "activations/layer1_attention_weight_max": 17.04835319519043,
      "activations/layer1_attention_weight_min": -12.376238822937012,
      "activations/layer20_attention_weight_max": 35.64474868774414,
      "activations/layer20_attention_weight_min": -24.425357818603516,
      "activations/layer21_attention_weight_max": 34.111961364746094,
      "activations/layer21_attention_weight_min": -19.306697845458984,
      "activations/layer22_attention_weight_max": 43.56049728393555,
      "activations/layer22_attention_weight_min": -26.83763885498047,
      "activations/layer23_attention_weight_max": 27.039424896240234,
      "activations/layer23_attention_weight_min": -17.513347625732422,
      "activations/layer2_attention_weight_max": 30.203123092651367,
      "activations/layer2_attention_weight_min": -29.567859649658203,
      "activations/layer3_attention_weight_max": 75.60354614257812,
      "activations/layer3_attention_weight_min": -78.50225830078125,
      "activations/layer4_attention_weight_max": 127.16912841796875,
      "activations/layer4_attention_weight_min": -118.51778411865234,
      "activations/layer5_attention_weight_max": 113.28762817382812,
      "activations/layer5_attention_weight_min": -94.00555419921875,
      "activations/layer6_attention_weight_max": 64.4303207397461,
      "activations/layer6_attention_weight_min": -52.97893524169922,
      "activations/layer7_attention_weight_max": 69.09221649169922,
      "activations/layer7_attention_weight_min": -65.39080810546875,
      "activations/layer8_attention_weight_max": 42.43629837036133,
      "activations/layer8_attention_weight_min": -37.99503707885742,
      "activations/layer9_attention_weight_max": 54.85972213745117,
      "activations/layer9_attention_weight_min": -47.686805725097656,
      "epoch": 2.61,
      "learning_rate": 0.0001344901515151515,
      "loss": 2.9504,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_loss": 2.87890625,
      "eval_runtime": 8.4831,
      "eval_samples_per_second": 506.181,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_openwebtext_loss": 2.87890625,
      "eval_openwebtext_ppl": 17.79479946994832,
      "eval_openwebtext_runtime": 8.4831,
      "eval_openwebtext_samples_per_second": 506.181,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_wikitext_loss": 3.138671875,
      "eval_wikitext_ppl": 23.073202403172917,
      "eval_wikitext_runtime": 1.9952,
      "eval_wikitext_samples_per_second": 228.544,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_lambada_loss": 3.048828125,
      "eval_lambada_ppl": 21.090614371389922,
      "eval_lambada_runtime": 9.5615,
      "eval_lambada_samples_per_second": 509.229,
      "step": 45000
    },
    {
      "activations/layer0_attention_weight_max": 15.019972801208496,
      "activations/layer0_attention_weight_min": -11.541215896606445,
      "activations/layer10_attention_weight_max": 48.78742599487305,
      "activations/layer10_attention_weight_min": -35.82173538208008,
      "activations/layer11_attention_weight_max": 42.2969856262207,
      "activations/layer11_attention_weight_min": -32.24613952636719,
      "activations/layer12_attention_weight_max": 37.93631362915039,
      "activations/layer12_attention_weight_min": -30.338850021362305,
      "activations/layer13_attention_weight_max": 67.10955047607422,
      "activations/layer13_attention_weight_min": -43.89451599121094,
      "activations/layer14_attention_weight_max": 89.92513275146484,
      "activations/layer14_attention_weight_min": -58.89811325073242,
      "activations/layer15_attention_weight_max": 52.43547821044922,
      "activations/layer15_attention_weight_min": -34.26825714111328,
      "activations/layer16_attention_weight_max": 52.22859573364258,
      "activations/layer16_attention_weight_min": -34.50986862182617,
      "activations/layer17_attention_weight_max": 48.028377532958984,
      "activations/layer17_attention_weight_min": -26.777881622314453,
      "activations/layer18_attention_weight_max": 38.090267181396484,
      "activations/layer18_attention_weight_min": -18.670324325561523,
      "activations/layer19_attention_weight_max": 57.568641662597656,
      "activations/layer19_attention_weight_min": -30.83237075805664,
      "activations/layer1_attention_weight_max": 17.674760818481445,
      "activations/layer1_attention_weight_min": -11.701218605041504,
      "activations/layer20_attention_weight_max": 38.27579116821289,
      "activations/layer20_attention_weight_min": -23.33026695251465,
      "activations/layer21_attention_weight_max": 37.74959945678711,
      "activations/layer21_attention_weight_min": -19.710107803344727,
      "activations/layer22_attention_weight_max": 52.83976364135742,
      "activations/layer22_attention_weight_min": -28.349607467651367,
      "activations/layer23_attention_weight_max": 29.699491500854492,
      "activations/layer23_attention_weight_min": -18.85222816467285,
      "activations/layer2_attention_weight_max": 29.203397750854492,
      "activations/layer2_attention_weight_min": -29.25124740600586,
      "activations/layer3_attention_weight_max": 74.76383209228516,
      "activations/layer3_attention_weight_min": -79.50235748291016,
      "activations/layer4_attention_weight_max": 134.07391357421875,
      "activations/layer4_attention_weight_min": -127.31932830810547,
      "activations/layer5_attention_weight_max": 107.21096801757812,
      "activations/layer5_attention_weight_min": -102.08431243896484,
      "activations/layer6_attention_weight_max": 62.30712127685547,
      "activations/layer6_attention_weight_min": -55.875144958496094,
      "activations/layer7_attention_weight_max": 81.96041107177734,
      "activations/layer7_attention_weight_min": -65.44271850585938,
      "activations/layer8_attention_weight_max": 46.83114242553711,
      "activations/layer8_attention_weight_min": -41.73957443237305,
      "activations/layer9_attention_weight_max": 66.20446014404297,
      "activations/layer9_attention_weight_min": -56.72168731689453,
      "epoch": 2.62,
      "learning_rate": 0.0001344712121212121,
      "loss": 2.9116,
      "step": 45050
    },
    {
      "activations/layer0_attention_weight_max": 17.129741668701172,
      "activations/layer0_attention_weight_min": -11.570225715637207,
      "activations/layer10_attention_weight_max": 43.32767868041992,
      "activations/layer10_attention_weight_min": -37.050350189208984,
      "activations/layer11_attention_weight_max": 33.25032424926758,
      "activations/layer11_attention_weight_min": -30.935928344726562,
      "activations/layer12_attention_weight_max": 34.20913314819336,
      "activations/layer12_attention_weight_min": -34.91291427612305,
      "activations/layer13_attention_weight_max": 50.72844314575195,
      "activations/layer13_attention_weight_min": -37.213165283203125,
      "activations/layer14_attention_weight_max": 70.65827941894531,
      "activations/layer14_attention_weight_min": -53.36898422241211,
      "activations/layer15_attention_weight_max": 45.8648681640625,
      "activations/layer15_attention_weight_min": -34.099327087402344,
      "activations/layer16_attention_weight_max": 45.3124885559082,
      "activations/layer16_attention_weight_min": -32.3934440612793,
      "activations/layer17_attention_weight_max": 42.62770462036133,
      "activations/layer17_attention_weight_min": -25.608909606933594,
      "activations/layer18_attention_weight_max": 28.329708099365234,
      "activations/layer18_attention_weight_min": -19.651145935058594,
      "activations/layer19_attention_weight_max": 48.046634674072266,
      "activations/layer19_attention_weight_min": -30.8396053314209,
      "activations/layer1_attention_weight_max": 17.568613052368164,
      "activations/layer1_attention_weight_min": -12.010845184326172,
      "activations/layer20_attention_weight_max": 37.92947006225586,
      "activations/layer20_attention_weight_min": -24.447580337524414,
      "activations/layer21_attention_weight_max": 35.241455078125,
      "activations/layer21_attention_weight_min": -19.578418731689453,
      "activations/layer22_attention_weight_max": 48.05754470825195,
      "activations/layer22_attention_weight_min": -24.924150466918945,
      "activations/layer23_attention_weight_max": 27.391006469726562,
      "activations/layer23_attention_weight_min": -16.603343963623047,
      "activations/layer2_attention_weight_max": 30.94319725036621,
      "activations/layer2_attention_weight_min": -30.592382431030273,
      "activations/layer3_attention_weight_max": 74.45738983154297,
      "activations/layer3_attention_weight_min": -79.62550354003906,
      "activations/layer4_attention_weight_max": 129.08238220214844,
      "activations/layer4_attention_weight_min": -129.60009765625,
      "activations/layer5_attention_weight_max": 104.86918640136719,
      "activations/layer5_attention_weight_min": -94.26708984375,
      "activations/layer6_attention_weight_max": 58.556068420410156,
      "activations/layer6_attention_weight_min": -53.31626892089844,
      "activations/layer7_attention_weight_max": 68.36124420166016,
      "activations/layer7_attention_weight_min": -65.16677856445312,
      "activations/layer8_attention_weight_max": 40.96529006958008,
      "activations/layer8_attention_weight_min": -38.89994812011719,
      "activations/layer9_attention_weight_max": 52.87500762939453,
      "activations/layer9_attention_weight_min": -43.67721176147461,
      "epoch": 2.62,
      "learning_rate": 0.00013445227272727272,
      "loss": 2.936,
      "step": 45100
    },
    {
      "activations/layer0_attention_weight_max": 17.41006088256836,
      "activations/layer0_attention_weight_min": -11.204282760620117,
      "activations/layer10_attention_weight_max": 50.165767669677734,
      "activations/layer10_attention_weight_min": -42.02821731567383,
      "activations/layer11_attention_weight_max": 40.184715270996094,
      "activations/layer11_attention_weight_min": -31.142370223999023,
      "activations/layer12_attention_weight_max": 38.04743576049805,
      "activations/layer12_attention_weight_min": -26.554834365844727,
      "activations/layer13_attention_weight_max": 52.51527404785156,
      "activations/layer13_attention_weight_min": -41.16847229003906,
      "activations/layer14_attention_weight_max": 70.92298889160156,
      "activations/layer14_attention_weight_min": -57.090049743652344,
      "activations/layer15_attention_weight_max": 46.23599624633789,
      "activations/layer15_attention_weight_min": -35.8055419921875,
      "activations/layer16_attention_weight_max": 50.831302642822266,
      "activations/layer16_attention_weight_min": -37.48263931274414,
      "activations/layer17_attention_weight_max": 45.56980895996094,
      "activations/layer17_attention_weight_min": -29.55021858215332,
      "activations/layer18_attention_weight_max": 31.54888343811035,
      "activations/layer18_attention_weight_min": -20.964841842651367,
      "activations/layer19_attention_weight_max": 47.26370620727539,
      "activations/layer19_attention_weight_min": -36.641632080078125,
      "activations/layer1_attention_weight_max": 16.917062759399414,
      "activations/layer1_attention_weight_min": -13.387601852416992,
      "activations/layer20_attention_weight_max": 34.69367218017578,
      "activations/layer20_attention_weight_min": -25.347793579101562,
      "activations/layer21_attention_weight_max": 32.2259407043457,
      "activations/layer21_attention_weight_min": -19.200122833251953,
      "activations/layer22_attention_weight_max": 45.409202575683594,
      "activations/layer22_attention_weight_min": -25.25693130493164,
      "activations/layer23_attention_weight_max": 28.637632369995117,
      "activations/layer23_attention_weight_min": -18.000526428222656,
      "activations/layer2_attention_weight_max": 30.71621322631836,
      "activations/layer2_attention_weight_min": -30.17193603515625,
      "activations/layer3_attention_weight_max": 73.38135528564453,
      "activations/layer3_attention_weight_min": -82.44131469726562,
      "activations/layer4_attention_weight_max": 121.970703125,
      "activations/layer4_attention_weight_min": -118.5909423828125,
      "activations/layer5_attention_weight_max": 105.1259765625,
      "activations/layer5_attention_weight_min": -92.00135803222656,
      "activations/layer6_attention_weight_max": 57.73627471923828,
      "activations/layer6_attention_weight_min": -53.42348098754883,
      "activations/layer7_attention_weight_max": 64.78186798095703,
      "activations/layer7_attention_weight_min": -63.03937911987305,
      "activations/layer8_attention_weight_max": 41.0970573425293,
      "activations/layer8_attention_weight_min": -40.68794250488281,
      "activations/layer9_attention_weight_max": 54.538551330566406,
      "activations/layer9_attention_weight_min": -46.0203971862793,
      "epoch": 2.62,
      "learning_rate": 0.00013443333333333334,
      "loss": 2.926,
      "step": 45150
    },
    {
      "activations/layer0_attention_weight_max": 14.972809791564941,
      "activations/layer0_attention_weight_min": -11.79847526550293,
      "activations/layer10_attention_weight_max": 48.95188903808594,
      "activations/layer10_attention_weight_min": -38.26778793334961,
      "activations/layer11_attention_weight_max": 37.573455810546875,
      "activations/layer11_attention_weight_min": -32.40433120727539,
      "activations/layer12_attention_weight_max": 52.014564514160156,
      "activations/layer12_attention_weight_min": -36.83746337890625,
      "activations/layer13_attention_weight_max": 52.804500579833984,
      "activations/layer13_attention_weight_min": -39.00151443481445,
      "activations/layer14_attention_weight_max": 81.34793090820312,
      "activations/layer14_attention_weight_min": -61.136234283447266,
      "activations/layer15_attention_weight_max": 47.93745803833008,
      "activations/layer15_attention_weight_min": -32.62255096435547,
      "activations/layer16_attention_weight_max": 45.65432357788086,
      "activations/layer16_attention_weight_min": -31.63714599609375,
      "activations/layer17_attention_weight_max": 43.37940216064453,
      "activations/layer17_attention_weight_min": -28.19287109375,
      "activations/layer18_attention_weight_max": 27.93120002746582,
      "activations/layer18_attention_weight_min": -18.817398071289062,
      "activations/layer19_attention_weight_max": 45.77881622314453,
      "activations/layer19_attention_weight_min": -30.900604248046875,
      "activations/layer1_attention_weight_max": 17.14751625061035,
      "activations/layer1_attention_weight_min": -12.810469627380371,
      "activations/layer20_attention_weight_max": 35.718727111816406,
      "activations/layer20_attention_weight_min": -27.697303771972656,
      "activations/layer21_attention_weight_max": 35.1591796875,
      "activations/layer21_attention_weight_min": -19.756818771362305,
      "activations/layer22_attention_weight_max": 45.4559211730957,
      "activations/layer22_attention_weight_min": -26.591352462768555,
      "activations/layer23_attention_weight_max": 28.603893280029297,
      "activations/layer23_attention_weight_min": -16.986591339111328,
      "activations/layer2_attention_weight_max": 29.446773529052734,
      "activations/layer2_attention_weight_min": -29.2525634765625,
      "activations/layer3_attention_weight_max": 75.61764526367188,
      "activations/layer3_attention_weight_min": -79.70100402832031,
      "activations/layer4_attention_weight_max": 132.41152954101562,
      "activations/layer4_attention_weight_min": -123.585205078125,
      "activations/layer5_attention_weight_max": 109.67153930664062,
      "activations/layer5_attention_weight_min": -92.03884887695312,
      "activations/layer6_attention_weight_max": 57.51881790161133,
      "activations/layer6_attention_weight_min": -50.57822036743164,
      "activations/layer7_attention_weight_max": 70.40217590332031,
      "activations/layer7_attention_weight_min": -65.08528900146484,
      "activations/layer8_attention_weight_max": 41.41240310668945,
      "activations/layer8_attention_weight_min": -39.487030029296875,
      "activations/layer9_attention_weight_max": 51.7360954284668,
      "activations/layer9_attention_weight_min": -45.36750411987305,
      "epoch": 2.63,
      "learning_rate": 0.00013441439393939393,
      "loss": 2.9382,
      "step": 45200
    },
    {
      "activations/layer0_attention_weight_max": 16.4486083984375,
      "activations/layer0_attention_weight_min": -11.341434478759766,
      "activations/layer10_attention_weight_max": 52.1650276184082,
      "activations/layer10_attention_weight_min": -41.731689453125,
      "activations/layer11_attention_weight_max": 51.23496627807617,
      "activations/layer11_attention_weight_min": -34.48539733886719,
      "activations/layer12_attention_weight_max": 39.07115936279297,
      "activations/layer12_attention_weight_min": -27.681982040405273,
      "activations/layer13_attention_weight_max": 58.45561599731445,
      "activations/layer13_attention_weight_min": -40.70753479003906,
      "activations/layer14_attention_weight_max": 104.14861297607422,
      "activations/layer14_attention_weight_min": -73.93339538574219,
      "activations/layer15_attention_weight_max": 58.302040100097656,
      "activations/layer15_attention_weight_min": -35.459781646728516,
      "activations/layer16_attention_weight_max": 48.487586975097656,
      "activations/layer16_attention_weight_min": -32.91844940185547,
      "activations/layer17_attention_weight_max": 44.295291900634766,
      "activations/layer17_attention_weight_min": -28.91659164428711,
      "activations/layer18_attention_weight_max": 36.259544372558594,
      "activations/layer18_attention_weight_min": -19.92173957824707,
      "activations/layer19_attention_weight_max": 57.26247024536133,
      "activations/layer19_attention_weight_min": -31.92195701599121,
      "activations/layer1_attention_weight_max": 18.2460880279541,
      "activations/layer1_attention_weight_min": -11.757811546325684,
      "activations/layer20_attention_weight_max": 39.83360290527344,
      "activations/layer20_attention_weight_min": -27.223642349243164,
      "activations/layer21_attention_weight_max": 37.17837905883789,
      "activations/layer21_attention_weight_min": -19.225879669189453,
      "activations/layer22_attention_weight_max": 51.16928482055664,
      "activations/layer22_attention_weight_min": -27.40195083618164,
      "activations/layer23_attention_weight_max": 30.399253845214844,
      "activations/layer23_attention_weight_min": -18.338022232055664,
      "activations/layer2_attention_weight_max": 30.245832443237305,
      "activations/layer2_attention_weight_min": -29.61726188659668,
      "activations/layer3_attention_weight_max": 81.01020812988281,
      "activations/layer3_attention_weight_min": -85.73027801513672,
      "activations/layer4_attention_weight_max": 137.44796752929688,
      "activations/layer4_attention_weight_min": -129.96194458007812,
      "activations/layer5_attention_weight_max": 113.52812957763672,
      "activations/layer5_attention_weight_min": -102.31611633300781,
      "activations/layer6_attention_weight_max": 66.5532455444336,
      "activations/layer6_attention_weight_min": -56.814476013183594,
      "activations/layer7_attention_weight_max": 80.54888916015625,
      "activations/layer7_attention_weight_min": -73.0824203491211,
      "activations/layer8_attention_weight_max": 54.05421829223633,
      "activations/layer8_attention_weight_min": -44.70429229736328,
      "activations/layer9_attention_weight_max": 66.95974731445312,
      "activations/layer9_attention_weight_min": -57.13007354736328,
      "epoch": 2.63,
      "learning_rate": 0.00013439545454545454,
      "loss": 2.9511,
      "step": 45250
    },
    {
      "activations/layer0_attention_weight_max": 17.100479125976562,
      "activations/layer0_attention_weight_min": -11.850895881652832,
      "activations/layer10_attention_weight_max": 51.37211608886719,
      "activations/layer10_attention_weight_min": -38.568115234375,
      "activations/layer11_attention_weight_max": 41.34532165527344,
      "activations/layer11_attention_weight_min": -32.92333984375,
      "activations/layer12_attention_weight_max": 33.05693054199219,
      "activations/layer12_attention_weight_min": -26.63697052001953,
      "activations/layer13_attention_weight_max": 53.388912200927734,
      "activations/layer13_attention_weight_min": -39.28734588623047,
      "activations/layer14_attention_weight_max": 73.81947326660156,
      "activations/layer14_attention_weight_min": -56.12800216674805,
      "activations/layer15_attention_weight_max": 51.91473388671875,
      "activations/layer15_attention_weight_min": -36.63949966430664,
      "activations/layer16_attention_weight_max": 52.31519317626953,
      "activations/layer16_attention_weight_min": -37.411678314208984,
      "activations/layer17_attention_weight_max": 44.228858947753906,
      "activations/layer17_attention_weight_min": -31.136308670043945,
      "activations/layer18_attention_weight_max": 31.61688804626465,
      "activations/layer18_attention_weight_min": -21.060562133789062,
      "activations/layer19_attention_weight_max": 49.970516204833984,
      "activations/layer19_attention_weight_min": -35.6684684753418,
      "activations/layer1_attention_weight_max": 17.223756790161133,
      "activations/layer1_attention_weight_min": -13.181892395019531,
      "activations/layer20_attention_weight_max": 38.949283599853516,
      "activations/layer20_attention_weight_min": -25.908470153808594,
      "activations/layer21_attention_weight_max": 40.47285842895508,
      "activations/layer21_attention_weight_min": -23.9205265045166,
      "activations/layer22_attention_weight_max": 45.06864547729492,
      "activations/layer22_attention_weight_min": -29.451374053955078,
      "activations/layer23_attention_weight_max": 26.736875534057617,
      "activations/layer23_attention_weight_min": -14.953981399536133,
      "activations/layer2_attention_weight_max": 29.258771896362305,
      "activations/layer2_attention_weight_min": -28.30958366394043,
      "activations/layer3_attention_weight_max": 77.40839385986328,
      "activations/layer3_attention_weight_min": -80.85989379882812,
      "activations/layer4_attention_weight_max": 139.7328338623047,
      "activations/layer4_attention_weight_min": -127.85260009765625,
      "activations/layer5_attention_weight_max": 116.08717346191406,
      "activations/layer5_attention_weight_min": -95.9282455444336,
      "activations/layer6_attention_weight_max": 63.09836196899414,
      "activations/layer6_attention_weight_min": -50.63657760620117,
      "activations/layer7_attention_weight_max": 65.90959167480469,
      "activations/layer7_attention_weight_min": -64.6181411743164,
      "activations/layer8_attention_weight_max": 45.28278732299805,
      "activations/layer8_attention_weight_min": -35.893943786621094,
      "activations/layer9_attention_weight_max": 54.63117599487305,
      "activations/layer9_attention_weight_min": -48.258060455322266,
      "epoch": 2.63,
      "learning_rate": 0.00013437651515151516,
      "loss": 2.9313,
      "step": 45300
    },
    {
      "activations/layer0_attention_weight_max": 13.686188697814941,
      "activations/layer0_attention_weight_min": -11.396380424499512,
      "activations/layer10_attention_weight_max": 40.72969436645508,
      "activations/layer10_attention_weight_min": -34.872684478759766,
      "activations/layer11_attention_weight_max": 42.38141632080078,
      "activations/layer11_attention_weight_min": -34.50153732299805,
      "activations/layer12_attention_weight_max": 51.62559509277344,
      "activations/layer12_attention_weight_min": -31.813196182250977,
      "activations/layer13_attention_weight_max": 49.216896057128906,
      "activations/layer13_attention_weight_min": -36.89341354370117,
      "activations/layer14_attention_weight_max": 95.7075424194336,
      "activations/layer14_attention_weight_min": -69.62577819824219,
      "activations/layer15_attention_weight_max": 47.883941650390625,
      "activations/layer15_attention_weight_min": -33.076148986816406,
      "activations/layer16_attention_weight_max": 53.08286666870117,
      "activations/layer16_attention_weight_min": -32.81818771362305,
      "activations/layer17_attention_weight_max": 40.553035736083984,
      "activations/layer17_attention_weight_min": -27.756568908691406,
      "activations/layer18_attention_weight_max": 33.30564498901367,
      "activations/layer18_attention_weight_min": -18.348983764648438,
      "activations/layer19_attention_weight_max": 51.41389083862305,
      "activations/layer19_attention_weight_min": -31.56210708618164,
      "activations/layer1_attention_weight_max": 17.231185913085938,
      "activations/layer1_attention_weight_min": -13.380924224853516,
      "activations/layer20_attention_weight_max": 36.64462661743164,
      "activations/layer20_attention_weight_min": -23.48391342163086,
      "activations/layer21_attention_weight_max": 31.13669776916504,
      "activations/layer21_attention_weight_min": -18.73845100402832,
      "activations/layer22_attention_weight_max": 48.27240753173828,
      "activations/layer22_attention_weight_min": -29.06014060974121,
      "activations/layer23_attention_weight_max": 29.740280151367188,
      "activations/layer23_attention_weight_min": -17.643718719482422,
      "activations/layer2_attention_weight_max": 31.87525749206543,
      "activations/layer2_attention_weight_min": -31.98183250427246,
      "activations/layer3_attention_weight_max": 82.86824035644531,
      "activations/layer3_attention_weight_min": -86.04285430908203,
      "activations/layer4_attention_weight_max": 140.06715393066406,
      "activations/layer4_attention_weight_min": -125.12316131591797,
      "activations/layer5_attention_weight_max": 109.52650451660156,
      "activations/layer5_attention_weight_min": -93.81539154052734,
      "activations/layer6_attention_weight_max": 62.17061233520508,
      "activations/layer6_attention_weight_min": -51.24473190307617,
      "activations/layer7_attention_weight_max": 68.44752502441406,
      "activations/layer7_attention_weight_min": -66.38359832763672,
      "activations/layer8_attention_weight_max": 42.8946533203125,
      "activations/layer8_attention_weight_min": -41.713138580322266,
      "activations/layer9_attention_weight_max": 62.18281936645508,
      "activations/layer9_attention_weight_min": -52.84370803833008,
      "epoch": 2.64,
      "learning_rate": 0.00013435757575757575,
      "loss": 2.926,
      "step": 45350
    },
    {
      "activations/layer0_attention_weight_max": 16.449655532836914,
      "activations/layer0_attention_weight_min": -11.424959182739258,
      "activations/layer10_attention_weight_max": 45.51225662231445,
      "activations/layer10_attention_weight_min": -36.65987014770508,
      "activations/layer11_attention_weight_max": 44.453529357910156,
      "activations/layer11_attention_weight_min": -35.102989196777344,
      "activations/layer12_attention_weight_max": 29.970848083496094,
      "activations/layer12_attention_weight_min": -27.848722457885742,
      "activations/layer13_attention_weight_max": 53.12227249145508,
      "activations/layer13_attention_weight_min": -36.61644744873047,
      "activations/layer14_attention_weight_max": 79.04698944091797,
      "activations/layer14_attention_weight_min": -55.43004608154297,
      "activations/layer15_attention_weight_max": 51.04629135131836,
      "activations/layer15_attention_weight_min": -36.49132537841797,
      "activations/layer16_attention_weight_max": 55.16169357299805,
      "activations/layer16_attention_weight_min": -34.85988998413086,
      "activations/layer17_attention_weight_max": 45.78066635131836,
      "activations/layer17_attention_weight_min": -26.795886993408203,
      "activations/layer18_attention_weight_max": 32.64939880371094,
      "activations/layer18_attention_weight_min": -18.295528411865234,
      "activations/layer19_attention_weight_max": 61.21290969848633,
      "activations/layer19_attention_weight_min": -31.90305519104004,
      "activations/layer1_attention_weight_max": 19.93927001953125,
      "activations/layer1_attention_weight_min": -12.589560508728027,
      "activations/layer20_attention_weight_max": 42.318077087402344,
      "activations/layer20_attention_weight_min": -24.971691131591797,
      "activations/layer21_attention_weight_max": 41.17151641845703,
      "activations/layer21_attention_weight_min": -21.613204956054688,
      "activations/layer22_attention_weight_max": 49.496726989746094,
      "activations/layer22_attention_weight_min": -27.222166061401367,
      "activations/layer23_attention_weight_max": 29.590774536132812,
      "activations/layer23_attention_weight_min": -15.470691680908203,
      "activations/layer2_attention_weight_max": 29.727964401245117,
      "activations/layer2_attention_weight_min": -29.035598754882812,
      "activations/layer3_attention_weight_max": 77.67715454101562,
      "activations/layer3_attention_weight_min": -83.39505767822266,
      "activations/layer4_attention_weight_max": 140.00784301757812,
      "activations/layer4_attention_weight_min": -130.22329711914062,
      "activations/layer5_attention_weight_max": 112.18355560302734,
      "activations/layer5_attention_weight_min": -98.082763671875,
      "activations/layer6_attention_weight_max": 59.74735641479492,
      "activations/layer6_attention_weight_min": -53.7454948425293,
      "activations/layer7_attention_weight_max": 69.15010833740234,
      "activations/layer7_attention_weight_min": -66.1440658569336,
      "activations/layer8_attention_weight_max": 44.42313003540039,
      "activations/layer8_attention_weight_min": -39.26506423950195,
      "activations/layer9_attention_weight_max": 62.86780548095703,
      "activations/layer9_attention_weight_min": -57.20692825317383,
      "epoch": 2.64,
      "learning_rate": 0.00013433863636363636,
      "loss": 2.9373,
      "step": 45400
    },
    {
      "activations/layer0_attention_weight_max": 15.836053848266602,
      "activations/layer0_attention_weight_min": -11.377754211425781,
      "activations/layer10_attention_weight_max": 44.961700439453125,
      "activations/layer10_attention_weight_min": -36.42985534667969,
      "activations/layer11_attention_weight_max": 38.97771453857422,
      "activations/layer11_attention_weight_min": -33.83717346191406,
      "activations/layer12_attention_weight_max": 37.47511672973633,
      "activations/layer12_attention_weight_min": -26.801118850708008,
      "activations/layer13_attention_weight_max": 55.82912063598633,
      "activations/layer13_attention_weight_min": -38.78814697265625,
      "activations/layer14_attention_weight_max": 87.58677673339844,
      "activations/layer14_attention_weight_min": -62.19733810424805,
      "activations/layer15_attention_weight_max": 52.21950149536133,
      "activations/layer15_attention_weight_min": -36.42229461669922,
      "activations/layer16_attention_weight_max": 45.5457649230957,
      "activations/layer16_attention_weight_min": -32.765438079833984,
      "activations/layer17_attention_weight_max": 49.094322204589844,
      "activations/layer17_attention_weight_min": -27.96354103088379,
      "activations/layer18_attention_weight_max": 31.11353874206543,
      "activations/layer18_attention_weight_min": -18.220474243164062,
      "activations/layer19_attention_weight_max": 48.98345184326172,
      "activations/layer19_attention_weight_min": -29.595447540283203,
      "activations/layer1_attention_weight_max": 17.748687744140625,
      "activations/layer1_attention_weight_min": -13.340489387512207,
      "activations/layer20_attention_weight_max": 40.22789764404297,
      "activations/layer20_attention_weight_min": -24.916105270385742,
      "activations/layer21_attention_weight_max": 36.612335205078125,
      "activations/layer21_attention_weight_min": -21.800119400024414,
      "activations/layer22_attention_weight_max": 53.20994186401367,
      "activations/layer22_attention_weight_min": -28.18076515197754,
      "activations/layer23_attention_weight_max": 26.53845977783203,
      "activations/layer23_attention_weight_min": -17.426393508911133,
      "activations/layer2_attention_weight_max": 29.581676483154297,
      "activations/layer2_attention_weight_min": -29.142898559570312,
      "activations/layer3_attention_weight_max": 76.13313293457031,
      "activations/layer3_attention_weight_min": -77.07676696777344,
      "activations/layer4_attention_weight_max": 130.4820556640625,
      "activations/layer4_attention_weight_min": -118.14261627197266,
      "activations/layer5_attention_weight_max": 109.80189514160156,
      "activations/layer5_attention_weight_min": -93.97956848144531,
      "activations/layer6_attention_weight_max": 57.92576217651367,
      "activations/layer6_attention_weight_min": -51.501712799072266,
      "activations/layer7_attention_weight_max": 68.2171630859375,
      "activations/layer7_attention_weight_min": -62.181724548339844,
      "activations/layer8_attention_weight_max": 41.689273834228516,
      "activations/layer8_attention_weight_min": -36.518733978271484,
      "activations/layer9_attention_weight_max": 64.28358459472656,
      "activations/layer9_attention_weight_min": -51.504730224609375,
      "epoch": 2.64,
      "learning_rate": 0.00013431969696969698,
      "loss": 2.9212,
      "step": 45450
    },
    {
      "activations/layer0_attention_weight_max": 16.813066482543945,
      "activations/layer0_attention_weight_min": -11.697273254394531,
      "activations/layer10_attention_weight_max": 42.65961837768555,
      "activations/layer10_attention_weight_min": -37.4743537902832,
      "activations/layer11_attention_weight_max": 34.94451141357422,
      "activations/layer11_attention_weight_min": -34.382469177246094,
      "activations/layer12_attention_weight_max": 31.264629364013672,
      "activations/layer12_attention_weight_min": -26.922290802001953,
      "activations/layer13_attention_weight_max": 60.41560745239258,
      "activations/layer13_attention_weight_min": -45.19929504394531,
      "activations/layer14_attention_weight_max": 75.64623260498047,
      "activations/layer14_attention_weight_min": -53.861881256103516,
      "activations/layer15_attention_weight_max": 49.17195510864258,
      "activations/layer15_attention_weight_min": -38.514015197753906,
      "activations/layer16_attention_weight_max": 50.90510559082031,
      "activations/layer16_attention_weight_min": -33.90065002441406,
      "activations/layer17_attention_weight_max": 43.81657028198242,
      "activations/layer17_attention_weight_min": -29.24761390686035,
      "activations/layer18_attention_weight_max": 31.835975646972656,
      "activations/layer18_attention_weight_min": -19.22699546813965,
      "activations/layer19_attention_weight_max": 47.498695373535156,
      "activations/layer19_attention_weight_min": -30.45339012145996,
      "activations/layer1_attention_weight_max": 17.546951293945312,
      "activations/layer1_attention_weight_min": -12.2830228805542,
      "activations/layer20_attention_weight_max": 35.788963317871094,
      "activations/layer20_attention_weight_min": -23.46466827392578,
      "activations/layer21_attention_weight_max": 38.45048141479492,
      "activations/layer21_attention_weight_min": -20.42145347595215,
      "activations/layer22_attention_weight_max": 48.00434112548828,
      "activations/layer22_attention_weight_min": -28.46916389465332,
      "activations/layer23_attention_weight_max": 28.12579345703125,
      "activations/layer23_attention_weight_min": -16.714664459228516,
      "activations/layer2_attention_weight_max": 30.725231170654297,
      "activations/layer2_attention_weight_min": -30.307004928588867,
      "activations/layer3_attention_weight_max": 77.7209243774414,
      "activations/layer3_attention_weight_min": -78.43907165527344,
      "activations/layer4_attention_weight_max": 126.7776107788086,
      "activations/layer4_attention_weight_min": -123.21480560302734,
      "activations/layer5_attention_weight_max": 102.70833587646484,
      "activations/layer5_attention_weight_min": -95.48777770996094,
      "activations/layer6_attention_weight_max": 56.99013137817383,
      "activations/layer6_attention_weight_min": -53.88080596923828,
      "activations/layer7_attention_weight_max": 65.1283187866211,
      "activations/layer7_attention_weight_min": -65.46379852294922,
      "activations/layer8_attention_weight_max": 41.50004196166992,
      "activations/layer8_attention_weight_min": -37.7248649597168,
      "activations/layer9_attention_weight_max": 52.537776947021484,
      "activations/layer9_attention_weight_min": -44.45827865600586,
      "epoch": 2.64,
      "learning_rate": 0.00013430075757575756,
      "loss": 2.9175,
      "step": 45500
    },
    {
      "activations/layer0_attention_weight_max": 17.315061569213867,
      "activations/layer0_attention_weight_min": -11.935443878173828,
      "activations/layer10_attention_weight_max": 45.15745544433594,
      "activations/layer10_attention_weight_min": -38.41766357421875,
      "activations/layer11_attention_weight_max": 38.672359466552734,
      "activations/layer11_attention_weight_min": -31.03985595703125,
      "activations/layer12_attention_weight_max": 30.437047958374023,
      "activations/layer12_attention_weight_min": -25.955829620361328,
      "activations/layer13_attention_weight_max": 45.47064971923828,
      "activations/layer13_attention_weight_min": -36.6716194152832,
      "activations/layer14_attention_weight_max": 72.74199676513672,
      "activations/layer14_attention_weight_min": -57.18994140625,
      "activations/layer15_attention_weight_max": 50.24135208129883,
      "activations/layer15_attention_weight_min": -33.440757751464844,
      "activations/layer16_attention_weight_max": 41.195377349853516,
      "activations/layer16_attention_weight_min": -33.590972900390625,
      "activations/layer17_attention_weight_max": 45.04982376098633,
      "activations/layer17_attention_weight_min": -29.85694122314453,
      "activations/layer18_attention_weight_max": 30.40908432006836,
      "activations/layer18_attention_weight_min": -19.467979431152344,
      "activations/layer19_attention_weight_max": 46.63523864746094,
      "activations/layer19_attention_weight_min": -31.74021339416504,
      "activations/layer1_attention_weight_max": 17.306976318359375,
      "activations/layer1_attention_weight_min": -12.959013938903809,
      "activations/layer20_attention_weight_max": 31.983625411987305,
      "activations/layer20_attention_weight_min": -25.515703201293945,
      "activations/layer21_attention_weight_max": 30.31201171875,
      "activations/layer21_attention_weight_min": -22.122549057006836,
      "activations/layer22_attention_weight_max": 46.8572883605957,
      "activations/layer22_attention_weight_min": -27.78563690185547,
      "activations/layer23_attention_weight_max": 25.536235809326172,
      "activations/layer23_attention_weight_min": -17.066001892089844,
      "activations/layer2_attention_weight_max": 30.866666793823242,
      "activations/layer2_attention_weight_min": -29.087806701660156,
      "activations/layer3_attention_weight_max": 79.37947082519531,
      "activations/layer3_attention_weight_min": -82.32672119140625,
      "activations/layer4_attention_weight_max": 131.45816040039062,
      "activations/layer4_attention_weight_min": -120.98351287841797,
      "activations/layer5_attention_weight_max": 106.2880859375,
      "activations/layer5_attention_weight_min": -91.91729736328125,
      "activations/layer6_attention_weight_max": 60.490753173828125,
      "activations/layer6_attention_weight_min": -54.995182037353516,
      "activations/layer7_attention_weight_max": 71.66284942626953,
      "activations/layer7_attention_weight_min": -66.02197265625,
      "activations/layer8_attention_weight_max": 43.970096588134766,
      "activations/layer8_attention_weight_min": -38.95025634765625,
      "activations/layer9_attention_weight_max": 54.54681396484375,
      "activations/layer9_attention_weight_min": -48.71247482299805,
      "epoch": 2.65,
      "learning_rate": 0.00013428181818181818,
      "loss": 2.9267,
      "step": 45550
    },
    {
      "activations/layer0_attention_weight_max": 16.972562789916992,
      "activations/layer0_attention_weight_min": -11.090020179748535,
      "activations/layer10_attention_weight_max": 43.47657775878906,
      "activations/layer10_attention_weight_min": -36.004920959472656,
      "activations/layer11_attention_weight_max": 40.019187927246094,
      "activations/layer11_attention_weight_min": -32.772464752197266,
      "activations/layer12_attention_weight_max": 35.79044723510742,
      "activations/layer12_attention_weight_min": -27.777463912963867,
      "activations/layer13_attention_weight_max": 60.52537536621094,
      "activations/layer13_attention_weight_min": -43.338233947753906,
      "activations/layer14_attention_weight_max": 76.60134887695312,
      "activations/layer14_attention_weight_min": -62.20285415649414,
      "activations/layer15_attention_weight_max": 53.1829948425293,
      "activations/layer15_attention_weight_min": -37.245792388916016,
      "activations/layer16_attention_weight_max": 47.62227249145508,
      "activations/layer16_attention_weight_min": -36.73441696166992,
      "activations/layer17_attention_weight_max": 39.51459503173828,
      "activations/layer17_attention_weight_min": -31.217350006103516,
      "activations/layer18_attention_weight_max": 28.63396644592285,
      "activations/layer18_attention_weight_min": -18.30838966369629,
      "activations/layer19_attention_weight_max": 51.37307357788086,
      "activations/layer19_attention_weight_min": -33.988460540771484,
      "activations/layer1_attention_weight_max": 18.450927734375,
      "activations/layer1_attention_weight_min": -12.392035484313965,
      "activations/layer20_attention_weight_max": 38.41300964355469,
      "activations/layer20_attention_weight_min": -22.38043212890625,
      "activations/layer21_attention_weight_max": 32.08757781982422,
      "activations/layer21_attention_weight_min": -18.99261474609375,
      "activations/layer22_attention_weight_max": 43.786766052246094,
      "activations/layer22_attention_weight_min": -26.73843765258789,
      "activations/layer23_attention_weight_max": 30.184772491455078,
      "activations/layer23_attention_weight_min": -16.279277801513672,
      "activations/layer2_attention_weight_max": 31.187681198120117,
      "activations/layer2_attention_weight_min": -30.204885482788086,
      "activations/layer3_attention_weight_max": 80.14370727539062,
      "activations/layer3_attention_weight_min": -82.95135498046875,
      "activations/layer4_attention_weight_max": 125.9091567993164,
      "activations/layer4_attention_weight_min": -121.4503402709961,
      "activations/layer5_attention_weight_max": 106.9360580444336,
      "activations/layer5_attention_weight_min": -91.18196105957031,
      "activations/layer6_attention_weight_max": 62.87165069580078,
      "activations/layer6_attention_weight_min": -51.3028450012207,
      "activations/layer7_attention_weight_max": 73.7500228881836,
      "activations/layer7_attention_weight_min": -62.017738342285156,
      "activations/layer8_attention_weight_max": 39.657291412353516,
      "activations/layer8_attention_weight_min": -38.320899963378906,
      "activations/layer9_attention_weight_max": 58.06545639038086,
      "activations/layer9_attention_weight_min": -49.122474670410156,
      "epoch": 2.65,
      "learning_rate": 0.00013426325757575757,
      "loss": 2.9316,
      "step": 45600
    },
    {
      "activations/layer0_attention_weight_max": 14.864667892456055,
      "activations/layer0_attention_weight_min": -10.943644523620605,
      "activations/layer10_attention_weight_max": 43.63673782348633,
      "activations/layer10_attention_weight_min": -39.04425048828125,
      "activations/layer11_attention_weight_max": 35.07485580444336,
      "activations/layer11_attention_weight_min": -31.909467697143555,
      "activations/layer12_attention_weight_max": 41.664642333984375,
      "activations/layer12_attention_weight_min": -26.174882888793945,
      "activations/layer13_attention_weight_max": 46.55426788330078,
      "activations/layer13_attention_weight_min": -37.067298889160156,
      "activations/layer14_attention_weight_max": 71.35464477539062,
      "activations/layer14_attention_weight_min": -56.88639450073242,
      "activations/layer15_attention_weight_max": 43.789913177490234,
      "activations/layer15_attention_weight_min": -33.56613540649414,
      "activations/layer16_attention_weight_max": 45.564048767089844,
      "activations/layer16_attention_weight_min": -34.97868347167969,
      "activations/layer17_attention_weight_max": 43.247657775878906,
      "activations/layer17_attention_weight_min": -26.6373233795166,
      "activations/layer18_attention_weight_max": 30.01192855834961,
      "activations/layer18_attention_weight_min": -18.457063674926758,
      "activations/layer19_attention_weight_max": 51.17489242553711,
      "activations/layer19_attention_weight_min": -31.361906051635742,
      "activations/layer1_attention_weight_max": 19.86476707458496,
      "activations/layer1_attention_weight_min": -12.713825225830078,
      "activations/layer20_attention_weight_max": 35.878753662109375,
      "activations/layer20_attention_weight_min": -23.331613540649414,
      "activations/layer21_attention_weight_max": 28.353927612304688,
      "activations/layer21_attention_weight_min": -17.5804386138916,
      "activations/layer22_attention_weight_max": 49.24520492553711,
      "activations/layer22_attention_weight_min": -27.413951873779297,
      "activations/layer23_attention_weight_max": 29.980924606323242,
      "activations/layer23_attention_weight_min": -17.541362762451172,
      "activations/layer2_attention_weight_max": 29.074737548828125,
      "activations/layer2_attention_weight_min": -28.73516082763672,
      "activations/layer3_attention_weight_max": 78.66849517822266,
      "activations/layer3_attention_weight_min": -77.7226333618164,
      "activations/layer4_attention_weight_max": 130.09275817871094,
      "activations/layer4_attention_weight_min": -125.7935791015625,
      "activations/layer5_attention_weight_max": 112.46016693115234,
      "activations/layer5_attention_weight_min": -99.86211395263672,
      "activations/layer6_attention_weight_max": 63.09089279174805,
      "activations/layer6_attention_weight_min": -52.231178283691406,
      "activations/layer7_attention_weight_max": 65.77057647705078,
      "activations/layer7_attention_weight_min": -68.08226013183594,
      "activations/layer8_attention_weight_max": 47.12593460083008,
      "activations/layer8_attention_weight_min": -36.834136962890625,
      "activations/layer9_attention_weight_max": 64.46758270263672,
      "activations/layer9_attention_weight_min": -51.78898239135742,
      "epoch": 2.65,
      "learning_rate": 0.00013424431818181818,
      "loss": 2.935,
      "step": 45650
    },
    {
      "activations/layer0_attention_weight_max": 16.741775512695312,
      "activations/layer0_attention_weight_min": -11.616936683654785,
      "activations/layer10_attention_weight_max": 50.87163162231445,
      "activations/layer10_attention_weight_min": -37.784812927246094,
      "activations/layer11_attention_weight_max": 39.30339813232422,
      "activations/layer11_attention_weight_min": -32.423553466796875,
      "activations/layer12_attention_weight_max": 47.88926696777344,
      "activations/layer12_attention_weight_min": -28.294185638427734,
      "activations/layer13_attention_weight_max": 74.35282135009766,
      "activations/layer13_attention_weight_min": -47.744083404541016,
      "activations/layer14_attention_weight_max": 81.76504516601562,
      "activations/layer14_attention_weight_min": -55.47505187988281,
      "activations/layer15_attention_weight_max": 51.343475341796875,
      "activations/layer15_attention_weight_min": -33.09832000732422,
      "activations/layer16_attention_weight_max": 46.02107620239258,
      "activations/layer16_attention_weight_min": -33.120574951171875,
      "activations/layer17_attention_weight_max": 47.170040130615234,
      "activations/layer17_attention_weight_min": -28.4678955078125,
      "activations/layer18_attention_weight_max": 31.358604431152344,
      "activations/layer18_attention_weight_min": -18.632116317749023,
      "activations/layer19_attention_weight_max": 50.46106719970703,
      "activations/layer19_attention_weight_min": -28.853254318237305,
      "activations/layer1_attention_weight_max": 17.37328338623047,
      "activations/layer1_attention_weight_min": -13.855974197387695,
      "activations/layer20_attention_weight_max": 36.645328521728516,
      "activations/layer20_attention_weight_min": -22.84663963317871,
      "activations/layer21_attention_weight_max": 34.600318908691406,
      "activations/layer21_attention_weight_min": -18.317861557006836,
      "activations/layer22_attention_weight_max": 44.761390686035156,
      "activations/layer22_attention_weight_min": -26.596446990966797,
      "activations/layer23_attention_weight_max": 25.144428253173828,
      "activations/layer23_attention_weight_min": -16.272212982177734,
      "activations/layer2_attention_weight_max": 30.392309188842773,
      "activations/layer2_attention_weight_min": -29.70021629333496,
      "activations/layer3_attention_weight_max": 75.66996002197266,
      "activations/layer3_attention_weight_min": -79.5367431640625,
      "activations/layer4_attention_weight_max": 127.85579681396484,
      "activations/layer4_attention_weight_min": -125.75142669677734,
      "activations/layer5_attention_weight_max": 101.817138671875,
      "activations/layer5_attention_weight_min": -93.36247253417969,
      "activations/layer6_attention_weight_max": 57.54713439941406,
      "activations/layer6_attention_weight_min": -52.02778244018555,
      "activations/layer7_attention_weight_max": 66.92335510253906,
      "activations/layer7_attention_weight_min": -63.22428512573242,
      "activations/layer8_attention_weight_max": 42.733360290527344,
      "activations/layer8_attention_weight_min": -35.71416091918945,
      "activations/layer9_attention_weight_max": 50.3592643737793,
      "activations/layer9_attention_weight_min": -44.45254135131836,
      "epoch": 2.66,
      "learning_rate": 0.00013422537878787877,
      "loss": 2.915,
      "step": 45700
    },
    {
      "activations/layer0_attention_weight_max": 16.746362686157227,
      "activations/layer0_attention_weight_min": -11.259747505187988,
      "activations/layer10_attention_weight_max": 45.37403869628906,
      "activations/layer10_attention_weight_min": -39.18616485595703,
      "activations/layer11_attention_weight_max": 38.577110290527344,
      "activations/layer11_attention_weight_min": -32.559715270996094,
      "activations/layer12_attention_weight_max": 66.24811553955078,
      "activations/layer12_attention_weight_min": -32.320899963378906,
      "activations/layer13_attention_weight_max": 72.13025665283203,
      "activations/layer13_attention_weight_min": -42.25322341918945,
      "activations/layer14_attention_weight_max": 69.88890838623047,
      "activations/layer14_attention_weight_min": -52.57107925415039,
      "activations/layer15_attention_weight_max": 51.449546813964844,
      "activations/layer15_attention_weight_min": -40.08659362792969,
      "activations/layer16_attention_weight_max": 46.76559066772461,
      "activations/layer16_attention_weight_min": -34.22384262084961,
      "activations/layer17_attention_weight_max": 48.81574249267578,
      "activations/layer17_attention_weight_min": -31.41331672668457,
      "activations/layer18_attention_weight_max": 30.22773551940918,
      "activations/layer18_attention_weight_min": -19.1884822845459,
      "activations/layer19_attention_weight_max": 50.466766357421875,
      "activations/layer19_attention_weight_min": -29.569761276245117,
      "activations/layer1_attention_weight_max": 17.659847259521484,
      "activations/layer1_attention_weight_min": -14.052659034729004,
      "activations/layer20_attention_weight_max": 34.02800750732422,
      "activations/layer20_attention_weight_min": -23.39906120300293,
      "activations/layer21_attention_weight_max": 36.96193313598633,
      "activations/layer21_attention_weight_min": -21.70598793029785,
      "activations/layer22_attention_weight_max": 48.15250778198242,
      "activations/layer22_attention_weight_min": -25.37114715576172,
      "activations/layer23_attention_weight_max": 25.766666412353516,
      "activations/layer23_attention_weight_min": -17.51725196838379,
      "activations/layer2_attention_weight_max": 29.325355529785156,
      "activations/layer2_attention_weight_min": -28.416440963745117,
      "activations/layer3_attention_weight_max": 75.63616180419922,
      "activations/layer3_attention_weight_min": -79.6217269897461,
      "activations/layer4_attention_weight_max": 132.3124237060547,
      "activations/layer4_attention_weight_min": -128.10911560058594,
      "activations/layer5_attention_weight_max": 106.83898162841797,
      "activations/layer5_attention_weight_min": -95.48155212402344,
      "activations/layer6_attention_weight_max": 61.66450881958008,
      "activations/layer6_attention_weight_min": -51.93317794799805,
      "activations/layer7_attention_weight_max": 73.40928649902344,
      "activations/layer7_attention_weight_min": -65.15121459960938,
      "activations/layer8_attention_weight_max": 40.107505798339844,
      "activations/layer8_attention_weight_min": -37.20262908935547,
      "activations/layer9_attention_weight_max": 56.668617248535156,
      "activations/layer9_attention_weight_min": -51.230384826660156,
      "epoch": 2.66,
      "learning_rate": 0.0001342064393939394,
      "loss": 2.9235,
      "step": 45750
    },
    {
      "activations/layer0_attention_weight_max": 15.456013679504395,
      "activations/layer0_attention_weight_min": -11.707842826843262,
      "activations/layer10_attention_weight_max": 57.15286636352539,
      "activations/layer10_attention_weight_min": -38.34597396850586,
      "activations/layer11_attention_weight_max": 41.610076904296875,
      "activations/layer11_attention_weight_min": -30.573543548583984,
      "activations/layer12_attention_weight_max": 58.05870056152344,
      "activations/layer12_attention_weight_min": -28.552417755126953,
      "activations/layer13_attention_weight_max": 73.36030578613281,
      "activations/layer13_attention_weight_min": -38.578155517578125,
      "activations/layer14_attention_weight_max": 100.58467102050781,
      "activations/layer14_attention_weight_min": -56.26901626586914,
      "activations/layer15_attention_weight_max": 66.0411605834961,
      "activations/layer15_attention_weight_min": -37.283592224121094,
      "activations/layer16_attention_weight_max": 66.10782623291016,
      "activations/layer16_attention_weight_min": -33.15888214111328,
      "activations/layer17_attention_weight_max": 63.43682861328125,
      "activations/layer17_attention_weight_min": -27.626319885253906,
      "activations/layer18_attention_weight_max": 33.83008575439453,
      "activations/layer18_attention_weight_min": -21.777956008911133,
      "activations/layer19_attention_weight_max": 75.7572250366211,
      "activations/layer19_attention_weight_min": -29.499685287475586,
      "activations/layer1_attention_weight_max": 18.635494232177734,
      "activations/layer1_attention_weight_min": -13.350250244140625,
      "activations/layer20_attention_weight_max": 48.364051818847656,
      "activations/layer20_attention_weight_min": -23.82862091064453,
      "activations/layer21_attention_weight_max": 31.889806747436523,
      "activations/layer21_attention_weight_min": -21.10223960876465,
      "activations/layer22_attention_weight_max": 57.32683181762695,
      "activations/layer22_attention_weight_min": -25.90273666381836,
      "activations/layer23_attention_weight_max": 29.558979034423828,
      "activations/layer23_attention_weight_min": -15.496774673461914,
      "activations/layer2_attention_weight_max": 29.141857147216797,
      "activations/layer2_attention_weight_min": -28.957433700561523,
      "activations/layer3_attention_weight_max": 77.27118682861328,
      "activations/layer3_attention_weight_min": -78.77456665039062,
      "activations/layer4_attention_weight_max": 133.2084197998047,
      "activations/layer4_attention_weight_min": -119.36714172363281,
      "activations/layer5_attention_weight_max": 110.95307922363281,
      "activations/layer5_attention_weight_min": -96.27787780761719,
      "activations/layer6_attention_weight_max": 61.42227554321289,
      "activations/layer6_attention_weight_min": -52.735008239746094,
      "activations/layer7_attention_weight_max": 73.14279174804688,
      "activations/layer7_attention_weight_min": -68.17460632324219,
      "activations/layer8_attention_weight_max": 46.84332275390625,
      "activations/layer8_attention_weight_min": -37.268760681152344,
      "activations/layer9_attention_weight_max": 72.26390838623047,
      "activations/layer9_attention_weight_min": -55.55176544189453,
      "epoch": 2.66,
      "learning_rate": 0.00013418749999999998,
      "loss": 2.9151,
      "step": 45800
    },
    {
      "activations/layer0_attention_weight_max": 16.953474044799805,
      "activations/layer0_attention_weight_min": -11.613871574401855,
      "activations/layer10_attention_weight_max": 45.0965461730957,
      "activations/layer10_attention_weight_min": -36.90330123901367,
      "activations/layer11_attention_weight_max": 39.78977966308594,
      "activations/layer11_attention_weight_min": -31.093198776245117,
      "activations/layer12_attention_weight_max": 55.03703689575195,
      "activations/layer12_attention_weight_min": -29.838829040527344,
      "activations/layer13_attention_weight_max": 48.95973587036133,
      "activations/layer13_attention_weight_min": -37.918190002441406,
      "activations/layer14_attention_weight_max": 70.54622650146484,
      "activations/layer14_attention_weight_min": -53.35614013671875,
      "activations/layer15_attention_weight_max": 46.00149917602539,
      "activations/layer15_attention_weight_min": -33.14763259887695,
      "activations/layer16_attention_weight_max": 51.05291748046875,
      "activations/layer16_attention_weight_min": -35.79925537109375,
      "activations/layer17_attention_weight_max": 40.83123016357422,
      "activations/layer17_attention_weight_min": -27.242595672607422,
      "activations/layer18_attention_weight_max": 31.99917221069336,
      "activations/layer18_attention_weight_min": -19.73017692565918,
      "activations/layer19_attention_weight_max": 45.22845458984375,
      "activations/layer19_attention_weight_min": -30.756635665893555,
      "activations/layer1_attention_weight_max": 18.526243209838867,
      "activations/layer1_attention_weight_min": -11.815735816955566,
      "activations/layer20_attention_weight_max": 35.01573181152344,
      "activations/layer20_attention_weight_min": -23.366640090942383,
      "activations/layer21_attention_weight_max": 34.38357162475586,
      "activations/layer21_attention_weight_min": -22.316011428833008,
      "activations/layer22_attention_weight_max": 47.15957260131836,
      "activations/layer22_attention_weight_min": -25.406932830810547,
      "activations/layer23_attention_weight_max": 27.520008087158203,
      "activations/layer23_attention_weight_min": -17.813579559326172,
      "activations/layer2_attention_weight_max": 29.43402099609375,
      "activations/layer2_attention_weight_min": -28.990394592285156,
      "activations/layer3_attention_weight_max": 76.97000122070312,
      "activations/layer3_attention_weight_min": -79.86714935302734,
      "activations/layer4_attention_weight_max": 123.97362518310547,
      "activations/layer4_attention_weight_min": -124.0426254272461,
      "activations/layer5_attention_weight_max": 105.197509765625,
      "activations/layer5_attention_weight_min": -92.81770324707031,
      "activations/layer6_attention_weight_max": 57.42405700683594,
      "activations/layer6_attention_weight_min": -51.317325592041016,
      "activations/layer7_attention_weight_max": 67.72447204589844,
      "activations/layer7_attention_weight_min": -63.82529067993164,
      "activations/layer8_attention_weight_max": 41.929256439208984,
      "activations/layer8_attention_weight_min": -37.0587043762207,
      "activations/layer9_attention_weight_max": 60.988502502441406,
      "activations/layer9_attention_weight_min": -45.72026824951172,
      "epoch": 2.66,
      "learning_rate": 0.0001341685606060606,
      "loss": 2.9267,
      "step": 45850
    },
    {
      "activations/layer0_attention_weight_max": 17.44801902770996,
      "activations/layer0_attention_weight_min": -11.684425354003906,
      "activations/layer10_attention_weight_max": 52.51422882080078,
      "activations/layer10_attention_weight_min": -39.12079620361328,
      "activations/layer11_attention_weight_max": 43.89760971069336,
      "activations/layer11_attention_weight_min": -32.18080139160156,
      "activations/layer12_attention_weight_max": 35.26422882080078,
      "activations/layer12_attention_weight_min": -27.317182540893555,
      "activations/layer13_attention_weight_max": 56.15216064453125,
      "activations/layer13_attention_weight_min": -41.32143020629883,
      "activations/layer14_attention_weight_max": 85.97586059570312,
      "activations/layer14_attention_weight_min": -56.69271469116211,
      "activations/layer15_attention_weight_max": 54.94764709472656,
      "activations/layer15_attention_weight_min": -34.237152099609375,
      "activations/layer16_attention_weight_max": 63.329647064208984,
      "activations/layer16_attention_weight_min": -36.12404251098633,
      "activations/layer17_attention_weight_max": 45.4799919128418,
      "activations/layer17_attention_weight_min": -27.24163246154785,
      "activations/layer18_attention_weight_max": 30.833078384399414,
      "activations/layer18_attention_weight_min": -19.00118064880371,
      "activations/layer19_attention_weight_max": 53.54486083984375,
      "activations/layer19_attention_weight_min": -32.18415069580078,
      "activations/layer1_attention_weight_max": 17.1704044342041,
      "activations/layer1_attention_weight_min": -14.04179573059082,
      "activations/layer20_attention_weight_max": 37.84049987792969,
      "activations/layer20_attention_weight_min": -23.281530380249023,
      "activations/layer21_attention_weight_max": 35.46965408325195,
      "activations/layer21_attention_weight_min": -21.547786712646484,
      "activations/layer22_attention_weight_max": 55.240478515625,
      "activations/layer22_attention_weight_min": -29.46599006652832,
      "activations/layer23_attention_weight_max": 29.84004020690918,
      "activations/layer23_attention_weight_min": -18.413949966430664,
      "activations/layer2_attention_weight_max": 29.813621520996094,
      "activations/layer2_attention_weight_min": -29.672954559326172,
      "activations/layer3_attention_weight_max": 78.27332305908203,
      "activations/layer3_attention_weight_min": -79.53683471679688,
      "activations/layer4_attention_weight_max": 132.9508819580078,
      "activations/layer4_attention_weight_min": -119.32332611083984,
      "activations/layer5_attention_weight_max": 114.45263671875,
      "activations/layer5_attention_weight_min": -96.56893920898438,
      "activations/layer6_attention_weight_max": 64.91358184814453,
      "activations/layer6_attention_weight_min": -57.41416931152344,
      "activations/layer7_attention_weight_max": 68.31204986572266,
      "activations/layer7_attention_weight_min": -67.01216888427734,
      "activations/layer8_attention_weight_max": 46.40202713012695,
      "activations/layer8_attention_weight_min": -37.031585693359375,
      "activations/layer9_attention_weight_max": 57.456336975097656,
      "activations/layer9_attention_weight_min": -51.15863037109375,
      "epoch": 2.67,
      "learning_rate": 0.0001341496212121212,
      "loss": 2.9418,
      "step": 45900
    },
    {
      "activations/layer0_attention_weight_max": 16.710407257080078,
      "activations/layer0_attention_weight_min": -12.012445449829102,
      "activations/layer10_attention_weight_max": 44.15556716918945,
      "activations/layer10_attention_weight_min": -38.04314041137695,
      "activations/layer11_attention_weight_max": 38.0131950378418,
      "activations/layer11_attention_weight_min": -32.24805450439453,
      "activations/layer12_attention_weight_max": 34.167449951171875,
      "activations/layer12_attention_weight_min": -27.46992301940918,
      "activations/layer13_attention_weight_max": 49.838809967041016,
      "activations/layer13_attention_weight_min": -33.04180145263672,
      "activations/layer14_attention_weight_max": 69.19877624511719,
      "activations/layer14_attention_weight_min": -50.92875671386719,
      "activations/layer15_attention_weight_max": 44.83016586303711,
      "activations/layer15_attention_weight_min": -35.34870529174805,
      "activations/layer16_attention_weight_max": 41.80490493774414,
      "activations/layer16_attention_weight_min": -34.37018585205078,
      "activations/layer17_attention_weight_max": 47.67763900756836,
      "activations/layer17_attention_weight_min": -30.27046012878418,
      "activations/layer18_attention_weight_max": 26.310020446777344,
      "activations/layer18_attention_weight_min": -17.718612670898438,
      "activations/layer19_attention_weight_max": 45.566184997558594,
      "activations/layer19_attention_weight_min": -31.497976303100586,
      "activations/layer1_attention_weight_max": 17.650436401367188,
      "activations/layer1_attention_weight_min": -14.343052864074707,
      "activations/layer20_attention_weight_max": 35.184383392333984,
      "activations/layer20_attention_weight_min": -23.62517547607422,
      "activations/layer21_attention_weight_max": 31.880929946899414,
      "activations/layer21_attention_weight_min": -20.344825744628906,
      "activations/layer22_attention_weight_max": 46.59657287597656,
      "activations/layer22_attention_weight_min": -25.763431549072266,
      "activations/layer23_attention_weight_max": 26.118961334228516,
      "activations/layer23_attention_weight_min": -16.689212799072266,
      "activations/layer2_attention_weight_max": 30.409854888916016,
      "activations/layer2_attention_weight_min": -31.130569458007812,
      "activations/layer3_attention_weight_max": 78.38505554199219,
      "activations/layer3_attention_weight_min": -80.18549346923828,
      "activations/layer4_attention_weight_max": 125.7021484375,
      "activations/layer4_attention_weight_min": -122.3080062866211,
      "activations/layer5_attention_weight_max": 104.84954071044922,
      "activations/layer5_attention_weight_min": -94.32038879394531,
      "activations/layer6_attention_weight_max": 57.585777282714844,
      "activations/layer6_attention_weight_min": -56.55390548706055,
      "activations/layer7_attention_weight_max": 69.51863098144531,
      "activations/layer7_attention_weight_min": -65.15558624267578,
      "activations/layer8_attention_weight_max": 40.96128463745117,
      "activations/layer8_attention_weight_min": -35.95833206176758,
      "activations/layer9_attention_weight_max": 51.37855911254883,
      "activations/layer9_attention_weight_min": -45.23443603515625,
      "epoch": 2.67,
      "learning_rate": 0.0001341306818181818,
      "loss": 2.9046,
      "step": 45950
    },
    {
      "activations/layer0_attention_weight_max": 15.92892074584961,
      "activations/layer0_attention_weight_min": -12.210936546325684,
      "activations/layer10_attention_weight_max": 45.124359130859375,
      "activations/layer10_attention_weight_min": -34.576297760009766,
      "activations/layer11_attention_weight_max": 40.79261779785156,
      "activations/layer11_attention_weight_min": -33.897438049316406,
      "activations/layer12_attention_weight_max": 47.52296829223633,
      "activations/layer12_attention_weight_min": -33.33616256713867,
      "activations/layer13_attention_weight_max": 52.23037338256836,
      "activations/layer13_attention_weight_min": -38.812744140625,
      "activations/layer14_attention_weight_max": 68.55899810791016,
      "activations/layer14_attention_weight_min": -54.193782806396484,
      "activations/layer15_attention_weight_max": 50.7964973449707,
      "activations/layer15_attention_weight_min": -39.821346282958984,
      "activations/layer16_attention_weight_max": 48.3958625793457,
      "activations/layer16_attention_weight_min": -38.828975677490234,
      "activations/layer17_attention_weight_max": 45.65993881225586,
      "activations/layer17_attention_weight_min": -30.85611915588379,
      "activations/layer18_attention_weight_max": 29.05140495300293,
      "activations/layer18_attention_weight_min": -21.82037353515625,
      "activations/layer19_attention_weight_max": 51.91657638549805,
      "activations/layer19_attention_weight_min": -32.8171272277832,
      "activations/layer1_attention_weight_max": 20.839862823486328,
      "activations/layer1_attention_weight_min": -12.160459518432617,
      "activations/layer20_attention_weight_max": 39.78751754760742,
      "activations/layer20_attention_weight_min": -25.110910415649414,
      "activations/layer21_attention_weight_max": 30.730772018432617,
      "activations/layer21_attention_weight_min": -22.73735237121582,
      "activations/layer22_attention_weight_max": 48.756736755371094,
      "activations/layer22_attention_weight_min": -27.61850357055664,
      "activations/layer23_attention_weight_max": 29.03875732421875,
      "activations/layer23_attention_weight_min": -20.057601928710938,
      "activations/layer2_attention_weight_max": 31.054672241210938,
      "activations/layer2_attention_weight_min": -29.585784912109375,
      "activations/layer3_attention_weight_max": 82.5337142944336,
      "activations/layer3_attention_weight_min": -83.97374725341797,
      "activations/layer4_attention_weight_max": 131.59202575683594,
      "activations/layer4_attention_weight_min": -124.1962890625,
      "activations/layer5_attention_weight_max": 110.09404754638672,
      "activations/layer5_attention_weight_min": -96.51351165771484,
      "activations/layer6_attention_weight_max": 64.33219146728516,
      "activations/layer6_attention_weight_min": -53.61164474487305,
      "activations/layer7_attention_weight_max": 68.89205169677734,
      "activations/layer7_attention_weight_min": -70.89776611328125,
      "activations/layer8_attention_weight_max": 44.234310150146484,
      "activations/layer8_attention_weight_min": -37.24796676635742,
      "activations/layer9_attention_weight_max": 63.196773529052734,
      "activations/layer9_attention_weight_min": -55.12317657470703,
      "epoch": 2.67,
      "learning_rate": 0.0001341117424242424,
      "loss": 2.9306,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_loss": 2.875,
      "eval_runtime": 8.5252,
      "eval_samples_per_second": 503.682,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_openwebtext_loss": 2.875,
      "eval_openwebtext_ppl": 17.725424121461643,
      "eval_openwebtext_runtime": 8.5252,
      "eval_openwebtext_samples_per_second": 503.682,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_wikitext_loss": 3.130859375,
      "eval_wikitext_ppl": 22.893645317537526,
      "eval_wikitext_runtime": 1.9983,
      "eval_wikitext_samples_per_second": 228.195,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_lambada_loss": 3.03515625,
      "eval_lambada_ppl": 20.8042283062039,
      "eval_lambada_runtime": 9.589,
      "eval_lambada_samples_per_second": 507.767,
      "step": 46000
    },
    {
      "activations/layer0_attention_weight_max": 16.79362678527832,
      "activations/layer0_attention_weight_min": -12.11473274230957,
      "activations/layer10_attention_weight_max": 45.00577163696289,
      "activations/layer10_attention_weight_min": -35.57239532470703,
      "activations/layer11_attention_weight_max": 36.596580505371094,
      "activations/layer11_attention_weight_min": -30.55022430419922,
      "activations/layer12_attention_weight_max": 49.90092468261719,
      "activations/layer12_attention_weight_min": -28.612916946411133,
      "activations/layer13_attention_weight_max": 58.34123611450195,
      "activations/layer13_attention_weight_min": -38.55462646484375,
      "activations/layer14_attention_weight_max": 81.99038696289062,
      "activations/layer14_attention_weight_min": -60.474708557128906,
      "activations/layer15_attention_weight_max": 54.7308464050293,
      "activations/layer15_attention_weight_min": -34.93252944946289,
      "activations/layer16_attention_weight_max": 49.491676330566406,
      "activations/layer16_attention_weight_min": -32.1915168762207,
      "activations/layer17_attention_weight_max": 44.99513626098633,
      "activations/layer17_attention_weight_min": -27.67172622680664,
      "activations/layer18_attention_weight_max": 34.80094909667969,
      "activations/layer18_attention_weight_min": -19.96779441833496,
      "activations/layer19_attention_weight_max": 49.17472457885742,
      "activations/layer19_attention_weight_min": -29.697275161743164,
      "activations/layer1_attention_weight_max": 17.95591926574707,
      "activations/layer1_attention_weight_min": -14.861640930175781,
      "activations/layer20_attention_weight_max": 39.2012939453125,
      "activations/layer20_attention_weight_min": -24.218402862548828,
      "activations/layer21_attention_weight_max": 32.741085052490234,
      "activations/layer21_attention_weight_min": -18.919893264770508,
      "activations/layer22_attention_weight_max": 49.13801956176758,
      "activations/layer22_attention_weight_min": -26.286211013793945,
      "activations/layer23_attention_weight_max": 30.280019760131836,
      "activations/layer23_attention_weight_min": -17.720537185668945,
      "activations/layer2_attention_weight_max": 30.2679443359375,
      "activations/layer2_attention_weight_min": -29.921737670898438,
      "activations/layer3_attention_weight_max": 75.56473541259766,
      "activations/layer3_attention_weight_min": -79.35901641845703,
      "activations/layer4_attention_weight_max": 127.4194107055664,
      "activations/layer4_attention_weight_min": -123.47600555419922,
      "activations/layer5_attention_weight_max": 105.85072326660156,
      "activations/layer5_attention_weight_min": -92.34530639648438,
      "activations/layer6_attention_weight_max": 59.0409049987793,
      "activations/layer6_attention_weight_min": -49.77867889404297,
      "activations/layer7_attention_weight_max": 70.37262725830078,
      "activations/layer7_attention_weight_min": -66.81092071533203,
      "activations/layer8_attention_weight_max": 43.54509735107422,
      "activations/layer8_attention_weight_min": -39.99628448486328,
      "activations/layer9_attention_weight_max": 53.86635208129883,
      "activations/layer9_attention_weight_min": -45.66586685180664,
      "epoch": 2.68,
      "learning_rate": 0.0001340931818181818,
      "loss": 2.9345,
      "step": 46050
    },
    {
      "activations/layer0_attention_weight_max": 17.224191665649414,
      "activations/layer0_attention_weight_min": -11.356226921081543,
      "activations/layer10_attention_weight_max": 49.22849655151367,
      "activations/layer10_attention_weight_min": -35.086517333984375,
      "activations/layer11_attention_weight_max": 53.46246337890625,
      "activations/layer11_attention_weight_min": -35.94574737548828,
      "activations/layer12_attention_weight_max": 33.88266372680664,
      "activations/layer12_attention_weight_min": -26.89610481262207,
      "activations/layer13_attention_weight_max": 49.77753448486328,
      "activations/layer13_attention_weight_min": -35.12017822265625,
      "activations/layer14_attention_weight_max": 72.49083709716797,
      "activations/layer14_attention_weight_min": -53.56685256958008,
      "activations/layer15_attention_weight_max": 48.644615173339844,
      "activations/layer15_attention_weight_min": -32.24229431152344,
      "activations/layer16_attention_weight_max": 58.239402770996094,
      "activations/layer16_attention_weight_min": -33.33172607421875,
      "activations/layer17_attention_weight_max": 48.27915954589844,
      "activations/layer17_attention_weight_min": -25.82395362854004,
      "activations/layer18_attention_weight_max": 28.484838485717773,
      "activations/layer18_attention_weight_min": -20.71889877319336,
      "activations/layer19_attention_weight_max": 49.690208435058594,
      "activations/layer19_attention_weight_min": -28.969924926757812,
      "activations/layer1_attention_weight_max": 17.60749053955078,
      "activations/layer1_attention_weight_min": -12.638961791992188,
      "activations/layer20_attention_weight_max": 41.38706970214844,
      "activations/layer20_attention_weight_min": -24.57363510131836,
      "activations/layer21_attention_weight_max": 35.46755599975586,
      "activations/layer21_attention_weight_min": -18.798477172851562,
      "activations/layer22_attention_weight_max": 50.20428466796875,
      "activations/layer22_attention_weight_min": -27.32538414001465,
      "activations/layer23_attention_weight_max": 26.381765365600586,
      "activations/layer23_attention_weight_min": -15.107919692993164,
      "activations/layer2_attention_weight_max": 30.89394187927246,
      "activations/layer2_attention_weight_min": -32.708168029785156,
      "activations/layer3_attention_weight_max": 79.12326049804688,
      "activations/layer3_attention_weight_min": -80.06887817382812,
      "activations/layer4_attention_weight_max": 138.2703857421875,
      "activations/layer4_attention_weight_min": -124.95430755615234,
      "activations/layer5_attention_weight_max": 116.06343078613281,
      "activations/layer5_attention_weight_min": -96.13619232177734,
      "activations/layer6_attention_weight_max": 62.970306396484375,
      "activations/layer6_attention_weight_min": -54.607112884521484,
      "activations/layer7_attention_weight_max": 70.44036102294922,
      "activations/layer7_attention_weight_min": -66.85079193115234,
      "activations/layer8_attention_weight_max": 49.441978454589844,
      "activations/layer8_attention_weight_min": -36.5606803894043,
      "activations/layer9_attention_weight_max": 64.91143798828125,
      "activations/layer9_attention_weight_min": -56.327449798583984,
      "epoch": 2.68,
      "learning_rate": 0.00013407424242424241,
      "loss": 2.9292,
      "step": 46100
    },
    {
      "activations/layer0_attention_weight_max": 17.055349349975586,
      "activations/layer0_attention_weight_min": -11.164624214172363,
      "activations/layer10_attention_weight_max": 42.27764129638672,
      "activations/layer10_attention_weight_min": -32.47475051879883,
      "activations/layer11_attention_weight_max": 33.29108428955078,
      "activations/layer11_attention_weight_min": -30.99403953552246,
      "activations/layer12_attention_weight_max": 28.06941032409668,
      "activations/layer12_attention_weight_min": -32.63640594482422,
      "activations/layer13_attention_weight_max": 45.679588317871094,
      "activations/layer13_attention_weight_min": -34.70582962036133,
      "activations/layer14_attention_weight_max": 64.5363540649414,
      "activations/layer14_attention_weight_min": -52.09694290161133,
      "activations/layer15_attention_weight_max": 46.264381408691406,
      "activations/layer15_attention_weight_min": -34.40427017211914,
      "activations/layer16_attention_weight_max": 43.50970458984375,
      "activations/layer16_attention_weight_min": -31.751949310302734,
      "activations/layer17_attention_weight_max": 39.42681884765625,
      "activations/layer17_attention_weight_min": -27.783878326416016,
      "activations/layer18_attention_weight_max": 32.518592834472656,
      "activations/layer18_attention_weight_min": -20.581796646118164,
      "activations/layer19_attention_weight_max": 43.705257415771484,
      "activations/layer19_attention_weight_min": -32.556007385253906,
      "activations/layer1_attention_weight_max": 18.063199996948242,
      "activations/layer1_attention_weight_min": -12.093537330627441,
      "activations/layer20_attention_weight_max": 34.72844696044922,
      "activations/layer20_attention_weight_min": -24.8546142578125,
      "activations/layer21_attention_weight_max": 36.57762908935547,
      "activations/layer21_attention_weight_min": -19.454687118530273,
      "activations/layer22_attention_weight_max": 40.404563903808594,
      "activations/layer22_attention_weight_min": -26.33527374267578,
      "activations/layer23_attention_weight_max": 24.987590789794922,
      "activations/layer23_attention_weight_min": -16.93323516845703,
      "activations/layer2_attention_weight_max": 30.918882369995117,
      "activations/layer2_attention_weight_min": -31.10540771484375,
      "activations/layer3_attention_weight_max": 77.17784118652344,
      "activations/layer3_attention_weight_min": -78.36705017089844,
      "activations/layer4_attention_weight_max": 128.20648193359375,
      "activations/layer4_attention_weight_min": -119.48296356201172,
      "activations/layer5_attention_weight_max": 109.39749145507812,
      "activations/layer5_attention_weight_min": -96.646728515625,
      "activations/layer6_attention_weight_max": 59.41734313964844,
      "activations/layer6_attention_weight_min": -50.891109466552734,
      "activations/layer7_attention_weight_max": 69.53641510009766,
      "activations/layer7_attention_weight_min": -68.53495788574219,
      "activations/layer8_attention_weight_max": 40.250797271728516,
      "activations/layer8_attention_weight_min": -34.796714782714844,
      "activations/layer9_attention_weight_max": 50.753814697265625,
      "activations/layer9_attention_weight_min": -45.31589889526367,
      "epoch": 2.68,
      "learning_rate": 0.000134055303030303,
      "loss": 2.9263,
      "step": 46150
    },
    {
      "activations/layer0_attention_weight_max": 16.821428298950195,
      "activations/layer0_attention_weight_min": -11.407054901123047,
      "activations/layer10_attention_weight_max": 44.06316375732422,
      "activations/layer10_attention_weight_min": -36.39990997314453,
      "activations/layer11_attention_weight_max": 44.281829833984375,
      "activations/layer11_attention_weight_min": -31.53946304321289,
      "activations/layer12_attention_weight_max": 77.18917083740234,
      "activations/layer12_attention_weight_min": -38.51719284057617,
      "activations/layer13_attention_weight_max": 55.776920318603516,
      "activations/layer13_attention_weight_min": -37.40971374511719,
      "activations/layer14_attention_weight_max": 82.0409164428711,
      "activations/layer14_attention_weight_min": -55.20097351074219,
      "activations/layer15_attention_weight_max": 50.437625885009766,
      "activations/layer15_attention_weight_min": -32.4406852722168,
      "activations/layer16_attention_weight_max": 47.93595886230469,
      "activations/layer16_attention_weight_min": -32.56263732910156,
      "activations/layer17_attention_weight_max": 49.0761604309082,
      "activations/layer17_attention_weight_min": -29.423242568969727,
      "activations/layer18_attention_weight_max": 30.76495933532715,
      "activations/layer18_attention_weight_min": -20.510643005371094,
      "activations/layer19_attention_weight_max": 53.360595703125,
      "activations/layer19_attention_weight_min": -29.08843421936035,
      "activations/layer1_attention_weight_max": 17.608888626098633,
      "activations/layer1_attention_weight_min": -15.157393455505371,
      "activations/layer20_attention_weight_max": 37.20283508300781,
      "activations/layer20_attention_weight_min": -22.204700469970703,
      "activations/layer21_attention_weight_max": 38.53108596801758,
      "activations/layer21_attention_weight_min": -21.04877471923828,
      "activations/layer22_attention_weight_max": 48.580589294433594,
      "activations/layer22_attention_weight_min": -27.72438621520996,
      "activations/layer23_attention_weight_max": 25.081045150756836,
      "activations/layer23_attention_weight_min": -15.056422233581543,
      "activations/layer2_attention_weight_max": 29.57124137878418,
      "activations/layer2_attention_weight_min": -28.763633728027344,
      "activations/layer3_attention_weight_max": 78.43339538574219,
      "activations/layer3_attention_weight_min": -77.21508026123047,
      "activations/layer4_attention_weight_max": 130.41490173339844,
      "activations/layer4_attention_weight_min": -119.2961196899414,
      "activations/layer5_attention_weight_max": 104.85740661621094,
      "activations/layer5_attention_weight_min": -89.45525360107422,
      "activations/layer6_attention_weight_max": 59.49738311767578,
      "activations/layer6_attention_weight_min": -49.81194305419922,
      "activations/layer7_attention_weight_max": 72.3902587890625,
      "activations/layer7_attention_weight_min": -65.09517669677734,
      "activations/layer8_attention_weight_max": 41.17941665649414,
      "activations/layer8_attention_weight_min": -38.19465637207031,
      "activations/layer9_attention_weight_max": 55.27024459838867,
      "activations/layer9_attention_weight_min": -42.15932083129883,
      "epoch": 2.68,
      "learning_rate": 0.0001340367424242424,
      "loss": 2.934,
      "step": 46200
    },
    {
      "activations/layer0_attention_weight_max": 17.63640594482422,
      "activations/layer0_attention_weight_min": -11.016488075256348,
      "activations/layer10_attention_weight_max": 47.272552490234375,
      "activations/layer10_attention_weight_min": -37.54916763305664,
      "activations/layer11_attention_weight_max": 41.84743881225586,
      "activations/layer11_attention_weight_min": -32.99403381347656,
      "activations/layer12_attention_weight_max": 47.86119079589844,
      "activations/layer12_attention_weight_min": -28.626798629760742,
      "activations/layer13_attention_weight_max": 65.05736541748047,
      "activations/layer13_attention_weight_min": -44.8386344909668,
      "activations/layer14_attention_weight_max": 96.5696792602539,
      "activations/layer14_attention_weight_min": -62.993709564208984,
      "activations/layer15_attention_weight_max": 56.33256912231445,
      "activations/layer15_attention_weight_min": -35.277889251708984,
      "activations/layer16_attention_weight_max": 53.1605339050293,
      "activations/layer16_attention_weight_min": -31.865888595581055,
      "activations/layer17_attention_weight_max": 50.20602798461914,
      "activations/layer17_attention_weight_min": -26.71554183959961,
      "activations/layer18_attention_weight_max": 32.833919525146484,
      "activations/layer18_attention_weight_min": -19.299497604370117,
      "activations/layer19_attention_weight_max": 47.750423431396484,
      "activations/layer19_attention_weight_min": -31.56044578552246,
      "activations/layer1_attention_weight_max": 17.286428451538086,
      "activations/layer1_attention_weight_min": -12.567610740661621,
      "activations/layer20_attention_weight_max": 38.536720275878906,
      "activations/layer20_attention_weight_min": -22.655906677246094,
      "activations/layer21_attention_weight_max": 30.733402252197266,
      "activations/layer21_attention_weight_min": -16.79461097717285,
      "activations/layer22_attention_weight_max": 49.67618942260742,
      "activations/layer22_attention_weight_min": -29.762529373168945,
      "activations/layer23_attention_weight_max": 27.756797790527344,
      "activations/layer23_attention_weight_min": -17.805801391601562,
      "activations/layer2_attention_weight_max": 31.610233306884766,
      "activations/layer2_attention_weight_min": -30.449256896972656,
      "activations/layer3_attention_weight_max": 80.75818634033203,
      "activations/layer3_attention_weight_min": -81.42249298095703,
      "activations/layer4_attention_weight_max": 133.85508728027344,
      "activations/layer4_attention_weight_min": -132.65846252441406,
      "activations/layer5_attention_weight_max": 114.54659271240234,
      "activations/layer5_attention_weight_min": -102.47552490234375,
      "activations/layer6_attention_weight_max": 62.681156158447266,
      "activations/layer6_attention_weight_min": -57.01609802246094,
      "activations/layer7_attention_weight_max": 71.63719177246094,
      "activations/layer7_attention_weight_min": -66.70733642578125,
      "activations/layer8_attention_weight_max": 47.32111740112305,
      "activations/layer8_attention_weight_min": -36.63723373413086,
      "activations/layer9_attention_weight_max": 56.893096923828125,
      "activations/layer9_attention_weight_min": -47.179508209228516,
      "epoch": 2.69,
      "learning_rate": 0.000134017803030303,
      "loss": 2.9137,
      "step": 46250
    },
    {
      "activations/layer0_attention_weight_max": 17.64879608154297,
      "activations/layer0_attention_weight_min": -11.187816619873047,
      "activations/layer10_attention_weight_max": 47.148128509521484,
      "activations/layer10_attention_weight_min": -40.86970520019531,
      "activations/layer11_attention_weight_max": 35.9643440246582,
      "activations/layer11_attention_weight_min": -31.706504821777344,
      "activations/layer12_attention_weight_max": 32.484779357910156,
      "activations/layer12_attention_weight_min": -27.57186508178711,
      "activations/layer13_attention_weight_max": 50.85932922363281,
      "activations/layer13_attention_weight_min": -35.023067474365234,
      "activations/layer14_attention_weight_max": 80.68760681152344,
      "activations/layer14_attention_weight_min": -58.97297286987305,
      "activations/layer15_attention_weight_max": 49.03367233276367,
      "activations/layer15_attention_weight_min": -30.735912322998047,
      "activations/layer16_attention_weight_max": 53.111392974853516,
      "activations/layer16_attention_weight_min": -30.23404884338379,
      "activations/layer17_attention_weight_max": 47.34156036376953,
      "activations/layer17_attention_weight_min": -30.014442443847656,
      "activations/layer18_attention_weight_max": 30.378299713134766,
      "activations/layer18_attention_weight_min": -19.185684204101562,
      "activations/layer19_attention_weight_max": 48.121490478515625,
      "activations/layer19_attention_weight_min": -27.465621948242188,
      "activations/layer1_attention_weight_max": 17.660770416259766,
      "activations/layer1_attention_weight_min": -11.425214767456055,
      "activations/layer20_attention_weight_max": 40.65422439575195,
      "activations/layer20_attention_weight_min": -22.791234970092773,
      "activations/layer21_attention_weight_max": 34.0671501159668,
      "activations/layer21_attention_weight_min": -18.963382720947266,
      "activations/layer22_attention_weight_max": 51.2070198059082,
      "activations/layer22_attention_weight_min": -26.866039276123047,
      "activations/layer23_attention_weight_max": 28.699329376220703,
      "activations/layer23_attention_weight_min": -17.38656997680664,
      "activations/layer2_attention_weight_max": 29.613548278808594,
      "activations/layer2_attention_weight_min": -30.603199005126953,
      "activations/layer3_attention_weight_max": 78.10711669921875,
      "activations/layer3_attention_weight_min": -80.6338882446289,
      "activations/layer4_attention_weight_max": 130.59312438964844,
      "activations/layer4_attention_weight_min": -128.94740295410156,
      "activations/layer5_attention_weight_max": 108.10528564453125,
      "activations/layer5_attention_weight_min": -97.7857666015625,
      "activations/layer6_attention_weight_max": 59.7126579284668,
      "activations/layer6_attention_weight_min": -54.16756057739258,
      "activations/layer7_attention_weight_max": 66.61820220947266,
      "activations/layer7_attention_weight_min": -64.20684814453125,
      "activations/layer8_attention_weight_max": 41.22182083129883,
      "activations/layer8_attention_weight_min": -37.16472244262695,
      "activations/layer9_attention_weight_max": 57.161376953125,
      "activations/layer9_attention_weight_min": -52.147762298583984,
      "epoch": 2.69,
      "learning_rate": 0.00013399886363636362,
      "loss": 2.9141,
      "step": 46300
    },
    {
      "activations/layer0_attention_weight_max": 17.184268951416016,
      "activations/layer0_attention_weight_min": -10.99779987335205,
      "activations/layer10_attention_weight_max": 43.87255859375,
      "activations/layer10_attention_weight_min": -38.0662841796875,
      "activations/layer11_attention_weight_max": 37.61119079589844,
      "activations/layer11_attention_weight_min": -33.60000228881836,
      "activations/layer12_attention_weight_max": 30.02581024169922,
      "activations/layer12_attention_weight_min": -25.919565200805664,
      "activations/layer13_attention_weight_max": 48.24467086791992,
      "activations/layer13_attention_weight_min": -34.422245025634766,
      "activations/layer14_attention_weight_max": 81.87202453613281,
      "activations/layer14_attention_weight_min": -61.532840728759766,
      "activations/layer15_attention_weight_max": 59.70558547973633,
      "activations/layer15_attention_weight_min": -39.71364974975586,
      "activations/layer16_attention_weight_max": 47.967437744140625,
      "activations/layer16_attention_weight_min": -34.8258056640625,
      "activations/layer17_attention_weight_max": 47.39798355102539,
      "activations/layer17_attention_weight_min": -26.98421287536621,
      "activations/layer18_attention_weight_max": 30.339998245239258,
      "activations/layer18_attention_weight_min": -17.739051818847656,
      "activations/layer19_attention_weight_max": 52.80419921875,
      "activations/layer19_attention_weight_min": -29.982831954956055,
      "activations/layer1_attention_weight_max": 17.110368728637695,
      "activations/layer1_attention_weight_min": -12.537581443786621,
      "activations/layer20_attention_weight_max": 33.792823791503906,
      "activations/layer20_attention_weight_min": -25.452669143676758,
      "activations/layer21_attention_weight_max": 32.47620391845703,
      "activations/layer21_attention_weight_min": -21.15464210510254,
      "activations/layer22_attention_weight_max": 50.565185546875,
      "activations/layer22_attention_weight_min": -26.977142333984375,
      "activations/layer23_attention_weight_max": 26.969358444213867,
      "activations/layer23_attention_weight_min": -18.774234771728516,
      "activations/layer2_attention_weight_max": 29.708904266357422,
      "activations/layer2_attention_weight_min": -29.07467269897461,
      "activations/layer3_attention_weight_max": 76.21549224853516,
      "activations/layer3_attention_weight_min": -75.98921966552734,
      "activations/layer4_attention_weight_max": 131.3379364013672,
      "activations/layer4_attention_weight_min": -119.46268463134766,
      "activations/layer5_attention_weight_max": 112.12776184082031,
      "activations/layer5_attention_weight_min": -97.9372787475586,
      "activations/layer6_attention_weight_max": 58.53840255737305,
      "activations/layer6_attention_weight_min": -52.735450744628906,
      "activations/layer7_attention_weight_max": 68.91752624511719,
      "activations/layer7_attention_weight_min": -67.94515991210938,
      "activations/layer8_attention_weight_max": 47.56675720214844,
      "activations/layer8_attention_weight_min": -38.78610610961914,
      "activations/layer9_attention_weight_max": 54.49622344970703,
      "activations/layer9_attention_weight_min": -41.03501510620117,
      "epoch": 2.69,
      "learning_rate": 0.00013397992424242424,
      "loss": 2.9331,
      "step": 46350
    },
    {
      "activations/layer0_attention_weight_max": 16.707843780517578,
      "activations/layer0_attention_weight_min": -11.405896186828613,
      "activations/layer10_attention_weight_max": 42.57586669921875,
      "activations/layer10_attention_weight_min": -36.46554946899414,
      "activations/layer11_attention_weight_max": 35.153404235839844,
      "activations/layer11_attention_weight_min": -31.707237243652344,
      "activations/layer12_attention_weight_max": 36.4674072265625,
      "activations/layer12_attention_weight_min": -27.82541275024414,
      "activations/layer13_attention_weight_max": 53.85437774658203,
      "activations/layer13_attention_weight_min": -37.91532897949219,
      "activations/layer14_attention_weight_max": 73.37641143798828,
      "activations/layer14_attention_weight_min": -58.526161193847656,
      "activations/layer15_attention_weight_max": 50.66605758666992,
      "activations/layer15_attention_weight_min": -35.01105499267578,
      "activations/layer16_attention_weight_max": 41.93673324584961,
      "activations/layer16_attention_weight_min": -31.243494033813477,
      "activations/layer17_attention_weight_max": 46.73503875732422,
      "activations/layer17_attention_weight_min": -26.548152923583984,
      "activations/layer18_attention_weight_max": 36.003536224365234,
      "activations/layer18_attention_weight_min": -20.068740844726562,
      "activations/layer19_attention_weight_max": 44.35813522338867,
      "activations/layer19_attention_weight_min": -29.91756248474121,
      "activations/layer1_attention_weight_max": 17.648836135864258,
      "activations/layer1_attention_weight_min": -13.302130699157715,
      "activations/layer20_attention_weight_max": 34.48386001586914,
      "activations/layer20_attention_weight_min": -22.22355079650879,
      "activations/layer21_attention_weight_max": 33.12486267089844,
      "activations/layer21_attention_weight_min": -20.21590805053711,
      "activations/layer22_attention_weight_max": 52.19879150390625,
      "activations/layer22_attention_weight_min": -28.952911376953125,
      "activations/layer23_attention_weight_max": 29.488361358642578,
      "activations/layer23_attention_weight_min": -20.474193572998047,
      "activations/layer2_attention_weight_max": 30.781970977783203,
      "activations/layer2_attention_weight_min": -31.472652435302734,
      "activations/layer3_attention_weight_max": 77.47158813476562,
      "activations/layer3_attention_weight_min": -80.71675872802734,
      "activations/layer4_attention_weight_max": 135.2832489013672,
      "activations/layer4_attention_weight_min": -126.21565246582031,
      "activations/layer5_attention_weight_max": 109.38197326660156,
      "activations/layer5_attention_weight_min": -96.84710693359375,
      "activations/layer6_attention_weight_max": 58.58978271484375,
      "activations/layer6_attention_weight_min": -52.87545394897461,
      "activations/layer7_attention_weight_max": 67.95406341552734,
      "activations/layer7_attention_weight_min": -66.08281707763672,
      "activations/layer8_attention_weight_max": 39.785518646240234,
      "activations/layer8_attention_weight_min": -36.55148696899414,
      "activations/layer9_attention_weight_max": 50.4542350769043,
      "activations/layer9_attention_weight_min": -41.30170822143555,
      "epoch": 2.7,
      "learning_rate": 0.00013396098484848483,
      "loss": 2.9176,
      "step": 46400
    },
    {
      "activations/layer0_attention_weight_max": 17.15645980834961,
      "activations/layer0_attention_weight_min": -11.625027656555176,
      "activations/layer10_attention_weight_max": 47.64418411254883,
      "activations/layer10_attention_weight_min": -37.52449417114258,
      "activations/layer11_attention_weight_max": 48.331878662109375,
      "activations/layer11_attention_weight_min": -37.55644989013672,
      "activations/layer12_attention_weight_max": 36.541015625,
      "activations/layer12_attention_weight_min": -30.768278121948242,
      "activations/layer13_attention_weight_max": 48.87049102783203,
      "activations/layer13_attention_weight_min": -42.316009521484375,
      "activations/layer14_attention_weight_max": 85.81930541992188,
      "activations/layer14_attention_weight_min": -62.55757141113281,
      "activations/layer15_attention_weight_max": 45.43286895751953,
      "activations/layer15_attention_weight_min": -32.817359924316406,
      "activations/layer16_attention_weight_max": 42.47429275512695,
      "activations/layer16_attention_weight_min": -34.31638717651367,
      "activations/layer17_attention_weight_max": 39.41158676147461,
      "activations/layer17_attention_weight_min": -28.825551986694336,
      "activations/layer18_attention_weight_max": 27.885860443115234,
      "activations/layer18_attention_weight_min": -19.68064308166504,
      "activations/layer19_attention_weight_max": 47.60275650024414,
      "activations/layer19_attention_weight_min": -29.693456649780273,
      "activations/layer1_attention_weight_max": 18.25591468811035,
      "activations/layer1_attention_weight_min": -12.162296295166016,
      "activations/layer20_attention_weight_max": 41.75960922241211,
      "activations/layer20_attention_weight_min": -25.880842208862305,
      "activations/layer21_attention_weight_max": 34.375762939453125,
      "activations/layer21_attention_weight_min": -20.811662673950195,
      "activations/layer22_attention_weight_max": 48.78326416015625,
      "activations/layer22_attention_weight_min": -29.629638671875,
      "activations/layer23_attention_weight_max": 26.454326629638672,
      "activations/layer23_attention_weight_min": -20.66866111755371,
      "activations/layer2_attention_weight_max": 30.77362060546875,
      "activations/layer2_attention_weight_min": -29.028623580932617,
      "activations/layer3_attention_weight_max": 76.7477798461914,
      "activations/layer3_attention_weight_min": -77.2406997680664,
      "activations/layer4_attention_weight_max": 134.2633056640625,
      "activations/layer4_attention_weight_min": -122.42207336425781,
      "activations/layer5_attention_weight_max": 114.73543548583984,
      "activations/layer5_attention_weight_min": -91.72027587890625,
      "activations/layer6_attention_weight_max": 66.88041687011719,
      "activations/layer6_attention_weight_min": -50.843257904052734,
      "activations/layer7_attention_weight_max": 81.0986557006836,
      "activations/layer7_attention_weight_min": -71.04319763183594,
      "activations/layer8_attention_weight_max": 49.90864944458008,
      "activations/layer8_attention_weight_min": -39.834678649902344,
      "activations/layer9_attention_weight_max": 63.21638107299805,
      "activations/layer9_attention_weight_min": -60.25883865356445,
      "epoch": 2.7,
      "learning_rate": 0.00013394204545454544,
      "loss": 2.9197,
      "step": 46450
    },
    {
      "activations/layer0_attention_weight_max": 16.76785659790039,
      "activations/layer0_attention_weight_min": -11.33153247833252,
      "activations/layer10_attention_weight_max": 52.668479919433594,
      "activations/layer10_attention_weight_min": -38.21065139770508,
      "activations/layer11_attention_weight_max": 40.62744140625,
      "activations/layer11_attention_weight_min": -32.80208206176758,
      "activations/layer12_attention_weight_max": 35.829986572265625,
      "activations/layer12_attention_weight_min": -31.37627601623535,
      "activations/layer13_attention_weight_max": 57.759620666503906,
      "activations/layer13_attention_weight_min": -37.55168533325195,
      "activations/layer14_attention_weight_max": 92.7204818725586,
      "activations/layer14_attention_weight_min": -59.98181915283203,
      "activations/layer15_attention_weight_max": 56.90061569213867,
      "activations/layer15_attention_weight_min": -34.42823028564453,
      "activations/layer16_attention_weight_max": 44.108428955078125,
      "activations/layer16_attention_weight_min": -33.19839859008789,
      "activations/layer17_attention_weight_max": 48.51526641845703,
      "activations/layer17_attention_weight_min": -28.45806312561035,
      "activations/layer18_attention_weight_max": 30.634559631347656,
      "activations/layer18_attention_weight_min": -19.38677978515625,
      "activations/layer19_attention_weight_max": 44.299251556396484,
      "activations/layer19_attention_weight_min": -29.492538452148438,
      "activations/layer1_attention_weight_max": 17.322734832763672,
      "activations/layer1_attention_weight_min": -10.773391723632812,
      "activations/layer20_attention_weight_max": 37.981910705566406,
      "activations/layer20_attention_weight_min": -23.704912185668945,
      "activations/layer21_attention_weight_max": 37.88994598388672,
      "activations/layer21_attention_weight_min": -20.79470443725586,
      "activations/layer22_attention_weight_max": 46.3027229309082,
      "activations/layer22_attention_weight_min": -25.902545928955078,
      "activations/layer23_attention_weight_max": 25.70705795288086,
      "activations/layer23_attention_weight_min": -16.46590805053711,
      "activations/layer2_attention_weight_max": 30.51520347595215,
      "activations/layer2_attention_weight_min": -30.523136138916016,
      "activations/layer3_attention_weight_max": 73.57998657226562,
      "activations/layer3_attention_weight_min": -79.53988647460938,
      "activations/layer4_attention_weight_max": 124.28636932373047,
      "activations/layer4_attention_weight_min": -125.29378509521484,
      "activations/layer5_attention_weight_max": 104.80058288574219,
      "activations/layer5_attention_weight_min": -96.32994079589844,
      "activations/layer6_attention_weight_max": 58.28173065185547,
      "activations/layer6_attention_weight_min": -56.06183624267578,
      "activations/layer7_attention_weight_max": 67.06915283203125,
      "activations/layer7_attention_weight_min": -67.78357696533203,
      "activations/layer8_attention_weight_max": 44.77750015258789,
      "activations/layer8_attention_weight_min": -37.039649963378906,
      "activations/layer9_attention_weight_max": 51.18879318237305,
      "activations/layer9_attention_weight_min": -43.057315826416016,
      "epoch": 2.7,
      "learning_rate": 0.00013392310606060606,
      "loss": 2.9139,
      "step": 46500
    },
    {
      "activations/layer0_attention_weight_max": 16.78179931640625,
      "activations/layer0_attention_weight_min": -11.901512145996094,
      "activations/layer10_attention_weight_max": 42.3323860168457,
      "activations/layer10_attention_weight_min": -36.844032287597656,
      "activations/layer11_attention_weight_max": 35.84268569946289,
      "activations/layer11_attention_weight_min": -30.215818405151367,
      "activations/layer12_attention_weight_max": 54.745059967041016,
      "activations/layer12_attention_weight_min": -37.1591796875,
      "activations/layer13_attention_weight_max": 67.99362182617188,
      "activations/layer13_attention_weight_min": -53.94055938720703,
      "activations/layer14_attention_weight_max": 68.79412078857422,
      "activations/layer14_attention_weight_min": -59.389007568359375,
      "activations/layer15_attention_weight_max": 48.92569351196289,
      "activations/layer15_attention_weight_min": -34.46399688720703,
      "activations/layer16_attention_weight_max": 44.00485610961914,
      "activations/layer16_attention_weight_min": -33.44559097290039,
      "activations/layer17_attention_weight_max": 44.379425048828125,
      "activations/layer17_attention_weight_min": -29.17918586730957,
      "activations/layer18_attention_weight_max": 28.2287654876709,
      "activations/layer18_attention_weight_min": -20.466711044311523,
      "activations/layer19_attention_weight_max": 41.51138687133789,
      "activations/layer19_attention_weight_min": -29.02301025390625,
      "activations/layer1_attention_weight_max": 18.123104095458984,
      "activations/layer1_attention_weight_min": -12.781938552856445,
      "activations/layer20_attention_weight_max": 34.072811126708984,
      "activations/layer20_attention_weight_min": -22.949365615844727,
      "activations/layer21_attention_weight_max": 30.76180648803711,
      "activations/layer21_attention_weight_min": -17.900951385498047,
      "activations/layer22_attention_weight_max": 46.76472854614258,
      "activations/layer22_attention_weight_min": -26.125152587890625,
      "activations/layer23_attention_weight_max": 22.880352020263672,
      "activations/layer23_attention_weight_min": -16.38688087463379,
      "activations/layer2_attention_weight_max": 27.362951278686523,
      "activations/layer2_attention_weight_min": -30.23134422302246,
      "activations/layer3_attention_weight_max": 72.76972961425781,
      "activations/layer3_attention_weight_min": -73.89940643310547,
      "activations/layer4_attention_weight_max": 126.14322662353516,
      "activations/layer4_attention_weight_min": -121.595458984375,
      "activations/layer5_attention_weight_max": 107.81844329833984,
      "activations/layer5_attention_weight_min": -94.89155578613281,
      "activations/layer6_attention_weight_max": 58.02997589111328,
      "activations/layer6_attention_weight_min": -52.61912536621094,
      "activations/layer7_attention_weight_max": 72.03630828857422,
      "activations/layer7_attention_weight_min": -69.81372833251953,
      "activations/layer8_attention_weight_max": 38.75714111328125,
      "activations/layer8_attention_weight_min": -37.46060562133789,
      "activations/layer9_attention_weight_max": 56.928382873535156,
      "activations/layer9_attention_weight_min": -54.625709533691406,
      "epoch": 2.7,
      "learning_rate": 0.00013390416666666664,
      "loss": 2.9343,
      "step": 46550
    },
    {
      "activations/layer0_attention_weight_max": 15.856733322143555,
      "activations/layer0_attention_weight_min": -11.419559478759766,
      "activations/layer10_attention_weight_max": 45.46272659301758,
      "activations/layer10_attention_weight_min": -33.97835159301758,
      "activations/layer11_attention_weight_max": 36.5842399597168,
      "activations/layer11_attention_weight_min": -29.615848541259766,
      "activations/layer12_attention_weight_max": 32.869964599609375,
      "activations/layer12_attention_weight_min": -26.24038314819336,
      "activations/layer13_attention_weight_max": 59.020790100097656,
      "activations/layer13_attention_weight_min": -38.18806838989258,
      "activations/layer14_attention_weight_max": 117.6128921508789,
      "activations/layer14_attention_weight_min": -69.61958312988281,
      "activations/layer15_attention_weight_max": 48.057491302490234,
      "activations/layer15_attention_weight_min": -31.55131721496582,
      "activations/layer16_attention_weight_max": 48.16534423828125,
      "activations/layer16_attention_weight_min": -34.604087829589844,
      "activations/layer17_attention_weight_max": 49.37070083618164,
      "activations/layer17_attention_weight_min": -28.308507919311523,
      "activations/layer18_attention_weight_max": 43.01250457763672,
      "activations/layer18_attention_weight_min": -19.82891082763672,
      "activations/layer19_attention_weight_max": 56.336280822753906,
      "activations/layer19_attention_weight_min": -30.468463897705078,
      "activations/layer1_attention_weight_max": 17.301668167114258,
      "activations/layer1_attention_weight_min": -12.38631820678711,
      "activations/layer20_attention_weight_max": 44.73857498168945,
      "activations/layer20_attention_weight_min": -24.08754539489746,
      "activations/layer21_attention_weight_max": 41.693904876708984,
      "activations/layer21_attention_weight_min": -19.44100570678711,
      "activations/layer22_attention_weight_max": 55.05173110961914,
      "activations/layer22_attention_weight_min": -27.93462562561035,
      "activations/layer23_attention_weight_max": 29.191543579101562,
      "activations/layer23_attention_weight_min": -18.596527099609375,
      "activations/layer2_attention_weight_max": 29.090171813964844,
      "activations/layer2_attention_weight_min": -28.28998565673828,
      "activations/layer3_attention_weight_max": 75.46976470947266,
      "activations/layer3_attention_weight_min": -79.64326477050781,
      "activations/layer4_attention_weight_max": 130.73036193847656,
      "activations/layer4_attention_weight_min": -129.1260528564453,
      "activations/layer5_attention_weight_max": 105.83938598632812,
      "activations/layer5_attention_weight_min": -95.23692321777344,
      "activations/layer6_attention_weight_max": 61.81614303588867,
      "activations/layer6_attention_weight_min": -52.44145965576172,
      "activations/layer7_attention_weight_max": 74.38301849365234,
      "activations/layer7_attention_weight_min": -68.02354431152344,
      "activations/layer8_attention_weight_max": 41.81764602661133,
      "activations/layer8_attention_weight_min": -38.62812042236328,
      "activations/layer9_attention_weight_max": 53.39935302734375,
      "activations/layer9_attention_weight_min": -41.940067291259766,
      "epoch": 2.71,
      "learning_rate": 0.00013388522727272726,
      "loss": 2.9218,
      "step": 46600
    },
    {
      "activations/layer0_attention_weight_max": 16.171419143676758,
      "activations/layer0_attention_weight_min": -12.289204597473145,
      "activations/layer10_attention_weight_max": 45.47789001464844,
      "activations/layer10_attention_weight_min": -38.63458251953125,
      "activations/layer11_attention_weight_max": 39.67457580566406,
      "activations/layer11_attention_weight_min": -33.1851921081543,
      "activations/layer12_attention_weight_max": 29.883323669433594,
      "activations/layer12_attention_weight_min": -33.767425537109375,
      "activations/layer13_attention_weight_max": 57.30116653442383,
      "activations/layer13_attention_weight_min": -38.32472229003906,
      "activations/layer14_attention_weight_max": 91.25161743164062,
      "activations/layer14_attention_weight_min": -67.47032165527344,
      "activations/layer15_attention_weight_max": 53.415992736816406,
      "activations/layer15_attention_weight_min": -36.81101608276367,
      "activations/layer16_attention_weight_max": 53.61020278930664,
      "activations/layer16_attention_weight_min": -34.985565185546875,
      "activations/layer17_attention_weight_max": 50.38317108154297,
      "activations/layer17_attention_weight_min": -29.98545265197754,
      "activations/layer18_attention_weight_max": 34.143314361572266,
      "activations/layer18_attention_weight_min": -18.1400089263916,
      "activations/layer19_attention_weight_max": 48.28741455078125,
      "activations/layer19_attention_weight_min": -31.018171310424805,
      "activations/layer1_attention_weight_max": 17.62763023376465,
      "activations/layer1_attention_weight_min": -13.046692848205566,
      "activations/layer20_attention_weight_max": 37.90591049194336,
      "activations/layer20_attention_weight_min": -22.778331756591797,
      "activations/layer21_attention_weight_max": 34.698883056640625,
      "activations/layer21_attention_weight_min": -19.547103881835938,
      "activations/layer22_attention_weight_max": 50.47037124633789,
      "activations/layer22_attention_weight_min": -28.030656814575195,
      "activations/layer23_attention_weight_max": 28.44643783569336,
      "activations/layer23_attention_weight_min": -16.78377914428711,
      "activations/layer2_attention_weight_max": 30.216045379638672,
      "activations/layer2_attention_weight_min": -29.688467025756836,
      "activations/layer3_attention_weight_max": 75.59590148925781,
      "activations/layer3_attention_weight_min": -79.18214416503906,
      "activations/layer4_attention_weight_max": 129.26263427734375,
      "activations/layer4_attention_weight_min": -130.4479217529297,
      "activations/layer5_attention_weight_max": 105.84706115722656,
      "activations/layer5_attention_weight_min": -95.9675521850586,
      "activations/layer6_attention_weight_max": 58.34437942504883,
      "activations/layer6_attention_weight_min": -56.25856399536133,
      "activations/layer7_attention_weight_max": 66.38629150390625,
      "activations/layer7_attention_weight_min": -72.91468811035156,
      "activations/layer8_attention_weight_max": 43.71554183959961,
      "activations/layer8_attention_weight_min": -39.53609085083008,
      "activations/layer9_attention_weight_max": 52.28709030151367,
      "activations/layer9_attention_weight_min": -44.23128890991211,
      "epoch": 2.71,
      "learning_rate": 0.00013386628787878787,
      "loss": 2.9233,
      "step": 46650
    },
    {
      "activations/layer0_attention_weight_max": 16.389245986938477,
      "activations/layer0_attention_weight_min": -11.504053115844727,
      "activations/layer10_attention_weight_max": 47.2465934753418,
      "activations/layer10_attention_weight_min": -34.99330520629883,
      "activations/layer11_attention_weight_max": 39.16627502441406,
      "activations/layer11_attention_weight_min": -31.23419189453125,
      "activations/layer12_attention_weight_max": 33.910675048828125,
      "activations/layer12_attention_weight_min": -25.767723083496094,
      "activations/layer13_attention_weight_max": 55.364906311035156,
      "activations/layer13_attention_weight_min": -40.63441467285156,
      "activations/layer14_attention_weight_max": 89.7939682006836,
      "activations/layer14_attention_weight_min": -58.660465240478516,
      "activations/layer15_attention_weight_max": 49.142799377441406,
      "activations/layer15_attention_weight_min": -31.27100372314453,
      "activations/layer16_attention_weight_max": 50.30641555786133,
      "activations/layer16_attention_weight_min": -31.880887985229492,
      "activations/layer17_attention_weight_max": 50.89480972290039,
      "activations/layer17_attention_weight_min": -28.098604202270508,
      "activations/layer18_attention_weight_max": 30.655502319335938,
      "activations/layer18_attention_weight_min": -20.16847801208496,
      "activations/layer19_attention_weight_max": 49.73789596557617,
      "activations/layer19_attention_weight_min": -28.387853622436523,
      "activations/layer1_attention_weight_max": 17.678512573242188,
      "activations/layer1_attention_weight_min": -12.361845970153809,
      "activations/layer20_attention_weight_max": 38.99787521362305,
      "activations/layer20_attention_weight_min": -23.350610733032227,
      "activations/layer21_attention_weight_max": 36.12139129638672,
      "activations/layer21_attention_weight_min": -19.920839309692383,
      "activations/layer22_attention_weight_max": 51.3280029296875,
      "activations/layer22_attention_weight_min": -28.21418571472168,
      "activations/layer23_attention_weight_max": 27.40135955810547,
      "activations/layer23_attention_weight_min": -16.25963592529297,
      "activations/layer2_attention_weight_max": 28.184852600097656,
      "activations/layer2_attention_weight_min": -28.437145233154297,
      "activations/layer3_attention_weight_max": 74.59138488769531,
      "activations/layer3_attention_weight_min": -77.30671691894531,
      "activations/layer4_attention_weight_max": 135.28077697753906,
      "activations/layer4_attention_weight_min": -120.68538665771484,
      "activations/layer5_attention_weight_max": 108.60830688476562,
      "activations/layer5_attention_weight_min": -94.74465942382812,
      "activations/layer6_attention_weight_max": 61.82229995727539,
      "activations/layer6_attention_weight_min": -52.38920974731445,
      "activations/layer7_attention_weight_max": 71.67333984375,
      "activations/layer7_attention_weight_min": -67.66756439208984,
      "activations/layer8_attention_weight_max": 45.7514533996582,
      "activations/layer8_attention_weight_min": -36.52745819091797,
      "activations/layer9_attention_weight_max": 53.358673095703125,
      "activations/layer9_attention_weight_min": -42.833038330078125,
      "epoch": 2.71,
      "learning_rate": 0.0001338473484848485,
      "loss": 2.907,
      "step": 46700
    },
    {
      "activations/layer0_attention_weight_max": 17.693944931030273,
      "activations/layer0_attention_weight_min": -11.626630783081055,
      "activations/layer10_attention_weight_max": 44.19298553466797,
      "activations/layer10_attention_weight_min": -36.78607940673828,
      "activations/layer11_attention_weight_max": 45.902496337890625,
      "activations/layer11_attention_weight_min": -30.324806213378906,
      "activations/layer12_attention_weight_max": 65.30127716064453,
      "activations/layer12_attention_weight_min": -38.9488639831543,
      "activations/layer13_attention_weight_max": 54.63999938964844,
      "activations/layer13_attention_weight_min": -40.64393615722656,
      "activations/layer14_attention_weight_max": 70.84278869628906,
      "activations/layer14_attention_weight_min": -56.17140197753906,
      "activations/layer15_attention_weight_max": 50.425594329833984,
      "activations/layer15_attention_weight_min": -33.36997604370117,
      "activations/layer16_attention_weight_max": 41.07755661010742,
      "activations/layer16_attention_weight_min": -31.229000091552734,
      "activations/layer17_attention_weight_max": 47.13139343261719,
      "activations/layer17_attention_weight_min": -29.21265411376953,
      "activations/layer18_attention_weight_max": 32.75360870361328,
      "activations/layer18_attention_weight_min": -19.61871910095215,
      "activations/layer19_attention_weight_max": 38.3011589050293,
      "activations/layer19_attention_weight_min": -27.81723976135254,
      "activations/layer1_attention_weight_max": 17.087310791015625,
      "activations/layer1_attention_weight_min": -12.120728492736816,
      "activations/layer20_attention_weight_max": 30.959110260009766,
      "activations/layer20_attention_weight_min": -21.339170455932617,
      "activations/layer21_attention_weight_max": 33.240234375,
      "activations/layer21_attention_weight_min": -19.977943420410156,
      "activations/layer22_attention_weight_max": 45.603759765625,
      "activations/layer22_attention_weight_min": -24.27651023864746,
      "activations/layer23_attention_weight_max": 25.731657028198242,
      "activations/layer23_attention_weight_min": -14.387465476989746,
      "activations/layer2_attention_weight_max": 29.992116928100586,
      "activations/layer2_attention_weight_min": -29.16790008544922,
      "activations/layer3_attention_weight_max": 73.73096466064453,
      "activations/layer3_attention_weight_min": -79.53134155273438,
      "activations/layer4_attention_weight_max": 129.1556854248047,
      "activations/layer4_attention_weight_min": -122.24177551269531,
      "activations/layer5_attention_weight_max": 106.95423889160156,
      "activations/layer5_attention_weight_min": -94.65758514404297,
      "activations/layer6_attention_weight_max": 58.927032470703125,
      "activations/layer6_attention_weight_min": -52.400901794433594,
      "activations/layer7_attention_weight_max": 76.33474731445312,
      "activations/layer7_attention_weight_min": -63.70930480957031,
      "activations/layer8_attention_weight_max": 42.9922981262207,
      "activations/layer8_attention_weight_min": -35.11826705932617,
      "activations/layer9_attention_weight_max": 64.39002990722656,
      "activations/layer9_attention_weight_min": -55.738277435302734,
      "epoch": 2.72,
      "learning_rate": 0.00013382840909090908,
      "loss": 2.9244,
      "step": 46750
    },
    {
      "activations/layer0_attention_weight_max": 17.10308837890625,
      "activations/layer0_attention_weight_min": -11.6900634765625,
      "activations/layer10_attention_weight_max": 44.80134201049805,
      "activations/layer10_attention_weight_min": -36.049476623535156,
      "activations/layer11_attention_weight_max": 41.182281494140625,
      "activations/layer11_attention_weight_min": -33.88771057128906,
      "activations/layer12_attention_weight_max": 27.83283042907715,
      "activations/layer12_attention_weight_min": -35.380043029785156,
      "activations/layer13_attention_weight_max": 46.66231918334961,
      "activations/layer13_attention_weight_min": -32.652618408203125,
      "activations/layer14_attention_weight_max": 87.53556060791016,
      "activations/layer14_attention_weight_min": -65.06034851074219,
      "activations/layer15_attention_weight_max": 47.270816802978516,
      "activations/layer15_attention_weight_min": -32.43935775756836,
      "activations/layer16_attention_weight_max": 42.95755386352539,
      "activations/layer16_attention_weight_min": -31.231733322143555,
      "activations/layer17_attention_weight_max": 45.16749954223633,
      "activations/layer17_attention_weight_min": -29.329620361328125,
      "activations/layer18_attention_weight_max": 28.98753547668457,
      "activations/layer18_attention_weight_min": -18.305030822753906,
      "activations/layer19_attention_weight_max": 43.77939224243164,
      "activations/layer19_attention_weight_min": -28.915176391601562,
      "activations/layer1_attention_weight_max": 18.0350399017334,
      "activations/layer1_attention_weight_min": -11.881936073303223,
      "activations/layer20_attention_weight_max": 34.695674896240234,
      "activations/layer20_attention_weight_min": -23.81454086303711,
      "activations/layer21_attention_weight_max": 30.501842498779297,
      "activations/layer21_attention_weight_min": -21.953893661499023,
      "activations/layer22_attention_weight_max": 44.22412109375,
      "activations/layer22_attention_weight_min": -31.31138038635254,
      "activations/layer23_attention_weight_max": 29.13899803161621,
      "activations/layer23_attention_weight_min": -18.68305778503418,
      "activations/layer2_attention_weight_max": 29.282894134521484,
      "activations/layer2_attention_weight_min": -28.8629150390625,
      "activations/layer3_attention_weight_max": 76.80805969238281,
      "activations/layer3_attention_weight_min": -80.89193725585938,
      "activations/layer4_attention_weight_max": 128.3956756591797,
      "activations/layer4_attention_weight_min": -120.41539764404297,
      "activations/layer5_attention_weight_max": 103.18775939941406,
      "activations/layer5_attention_weight_min": -92.32655334472656,
      "activations/layer6_attention_weight_max": 60.22910690307617,
      "activations/layer6_attention_weight_min": -52.19182586669922,
      "activations/layer7_attention_weight_max": 74.15397644042969,
      "activations/layer7_attention_weight_min": -66.23853302001953,
      "activations/layer8_attention_weight_max": 47.136966705322266,
      "activations/layer8_attention_weight_min": -40.43035125732422,
      "activations/layer9_attention_weight_max": 62.19355392456055,
      "activations/layer9_attention_weight_min": -58.198829650878906,
      "epoch": 2.72,
      "learning_rate": 0.0001338094696969697,
      "loss": 2.9211,
      "step": 46800
    },
    {
      "activations/layer0_attention_weight_max": 17.362340927124023,
      "activations/layer0_attention_weight_min": -13.144453048706055,
      "activations/layer10_attention_weight_max": 49.58612823486328,
      "activations/layer10_attention_weight_min": -35.90708541870117,
      "activations/layer11_attention_weight_max": 38.048919677734375,
      "activations/layer11_attention_weight_min": -30.755992889404297,
      "activations/layer12_attention_weight_max": 27.891279220581055,
      "activations/layer12_attention_weight_min": -24.290964126586914,
      "activations/layer13_attention_weight_max": 54.85017013549805,
      "activations/layer13_attention_weight_min": -38.84360885620117,
      "activations/layer14_attention_weight_max": 73.1469497680664,
      "activations/layer14_attention_weight_min": -54.49283981323242,
      "activations/layer15_attention_weight_max": 59.861289978027344,
      "activations/layer15_attention_weight_min": -35.76176452636719,
      "activations/layer16_attention_weight_max": 45.550453186035156,
      "activations/layer16_attention_weight_min": -33.00200271606445,
      "activations/layer17_attention_weight_max": 47.123023986816406,
      "activations/layer17_attention_weight_min": -27.160612106323242,
      "activations/layer18_attention_weight_max": 30.132041931152344,
      "activations/layer18_attention_weight_min": -18.407955169677734,
      "activations/layer19_attention_weight_max": 45.20557403564453,
      "activations/layer19_attention_weight_min": -31.2651424407959,
      "activations/layer1_attention_weight_max": 17.801259994506836,
      "activations/layer1_attention_weight_min": -12.593940734863281,
      "activations/layer20_attention_weight_max": 36.0713005065918,
      "activations/layer20_attention_weight_min": -25.54530906677246,
      "activations/layer21_attention_weight_max": 32.083011627197266,
      "activations/layer21_attention_weight_min": -20.64992904663086,
      "activations/layer22_attention_weight_max": 49.59526443481445,
      "activations/layer22_attention_weight_min": -28.07921600341797,
      "activations/layer23_attention_weight_max": 27.724164962768555,
      "activations/layer23_attention_weight_min": -17.727890014648438,
      "activations/layer2_attention_weight_max": 31.05331039428711,
      "activations/layer2_attention_weight_min": -29.680252075195312,
      "activations/layer3_attention_weight_max": 76.58694458007812,
      "activations/layer3_attention_weight_min": -79.56999206542969,
      "activations/layer4_attention_weight_max": 136.47251892089844,
      "activations/layer4_attention_weight_min": -119.7123031616211,
      "activations/layer5_attention_weight_max": 111.06196594238281,
      "activations/layer5_attention_weight_min": -91.8843002319336,
      "activations/layer6_attention_weight_max": 58.82175064086914,
      "activations/layer6_attention_weight_min": -51.93667221069336,
      "activations/layer7_attention_weight_max": 64.54698944091797,
      "activations/layer7_attention_weight_min": -65.20877838134766,
      "activations/layer8_attention_weight_max": 43.9075813293457,
      "activations/layer8_attention_weight_min": -39.22293472290039,
      "activations/layer9_attention_weight_max": 53.320159912109375,
      "activations/layer9_attention_weight_min": -42.789283752441406,
      "epoch": 2.72,
      "learning_rate": 0.0001337905303030303,
      "loss": 2.9302,
      "step": 46850
    },
    {
      "activations/layer0_attention_weight_max": 17.20250129699707,
      "activations/layer0_attention_weight_min": -11.386881828308105,
      "activations/layer10_attention_weight_max": 46.0827751159668,
      "activations/layer10_attention_weight_min": -36.03619384765625,
      "activations/layer11_attention_weight_max": 42.86729431152344,
      "activations/layer11_attention_weight_min": -33.75613021850586,
      "activations/layer12_attention_weight_max": 43.73200988769531,
      "activations/layer12_attention_weight_min": -31.017963409423828,
      "activations/layer13_attention_weight_max": 46.63875198364258,
      "activations/layer13_attention_weight_min": -36.855918884277344,
      "activations/layer14_attention_weight_max": 68.99800872802734,
      "activations/layer14_attention_weight_min": -52.8846435546875,
      "activations/layer15_attention_weight_max": 48.03590393066406,
      "activations/layer15_attention_weight_min": -34.58756637573242,
      "activations/layer16_attention_weight_max": 42.951072692871094,
      "activations/layer16_attention_weight_min": -33.75408935546875,
      "activations/layer17_attention_weight_max": 40.92865753173828,
      "activations/layer17_attention_weight_min": -29.547950744628906,
      "activations/layer18_attention_weight_max": 27.29041862487793,
      "activations/layer18_attention_weight_min": -21.34575080871582,
      "activations/layer19_attention_weight_max": 45.41524887084961,
      "activations/layer19_attention_weight_min": -28.891088485717773,
      "activations/layer1_attention_weight_max": 17.329757690429688,
      "activations/layer1_attention_weight_min": -14.130306243896484,
      "activations/layer20_attention_weight_max": 33.873634338378906,
      "activations/layer20_attention_weight_min": -23.17384910583496,
      "activations/layer21_attention_weight_max": 35.15862274169922,
      "activations/layer21_attention_weight_min": -21.342178344726562,
      "activations/layer22_attention_weight_max": 43.34691619873047,
      "activations/layer22_attention_weight_min": -29.912683486938477,
      "activations/layer23_attention_weight_max": 25.759326934814453,
      "activations/layer23_attention_weight_min": -15.813211441040039,
      "activations/layer2_attention_weight_max": 30.728605270385742,
      "activations/layer2_attention_weight_min": -29.362098693847656,
      "activations/layer3_attention_weight_max": 77.99868774414062,
      "activations/layer3_attention_weight_min": -80.5101089477539,
      "activations/layer4_attention_weight_max": 128.7195281982422,
      "activations/layer4_attention_weight_min": -125.07695770263672,
      "activations/layer5_attention_weight_max": 103.02873229980469,
      "activations/layer5_attention_weight_min": -90.77162170410156,
      "activations/layer6_attention_weight_max": 55.43419647216797,
      "activations/layer6_attention_weight_min": -53.2707405090332,
      "activations/layer7_attention_weight_max": 69.90892028808594,
      "activations/layer7_attention_weight_min": -67.85697937011719,
      "activations/layer8_attention_weight_max": 47.46196365356445,
      "activations/layer8_attention_weight_min": -39.113834381103516,
      "activations/layer9_attention_weight_max": 59.84169387817383,
      "activations/layer9_attention_weight_min": -54.956092834472656,
      "epoch": 2.73,
      "learning_rate": 0.0001337715909090909,
      "loss": 2.9246,
      "step": 46900
    },
    {
      "activations/layer0_attention_weight_max": 17.59999656677246,
      "activations/layer0_attention_weight_min": -11.941617965698242,
      "activations/layer10_attention_weight_max": 51.314605712890625,
      "activations/layer10_attention_weight_min": -43.76531982421875,
      "activations/layer11_attention_weight_max": 33.95766830444336,
      "activations/layer11_attention_weight_min": -31.081134796142578,
      "activations/layer12_attention_weight_max": 28.883804321289062,
      "activations/layer12_attention_weight_min": -38.0682487487793,
      "activations/layer13_attention_weight_max": 58.538028717041016,
      "activations/layer13_attention_weight_min": -41.93656539916992,
      "activations/layer14_attention_weight_max": 66.55359649658203,
      "activations/layer14_attention_weight_min": -53.167964935302734,
      "activations/layer15_attention_weight_max": 44.831966400146484,
      "activations/layer15_attention_weight_min": -30.767536163330078,
      "activations/layer16_attention_weight_max": 44.77235794067383,
      "activations/layer16_attention_weight_min": -34.102115631103516,
      "activations/layer17_attention_weight_max": 44.143707275390625,
      "activations/layer17_attention_weight_min": -26.062646865844727,
      "activations/layer18_attention_weight_max": 29.913063049316406,
      "activations/layer18_attention_weight_min": -19.43455696105957,
      "activations/layer19_attention_weight_max": 43.92625045776367,
      "activations/layer19_attention_weight_min": -29.282684326171875,
      "activations/layer1_attention_weight_max": 17.231897354125977,
      "activations/layer1_attention_weight_min": -11.028298377990723,
      "activations/layer20_attention_weight_max": 30.59306526184082,
      "activations/layer20_attention_weight_min": -23.096147537231445,
      "activations/layer21_attention_weight_max": 31.938640594482422,
      "activations/layer21_attention_weight_min": -22.641515731811523,
      "activations/layer22_attention_weight_max": 44.07566452026367,
      "activations/layer22_attention_weight_min": -26.133975982666016,
      "activations/layer23_attention_weight_max": 24.011594772338867,
      "activations/layer23_attention_weight_min": -17.52781105041504,
      "activations/layer2_attention_weight_max": 25.86909294128418,
      "activations/layer2_attention_weight_min": -26.797269821166992,
      "activations/layer3_attention_weight_max": 71.9203872680664,
      "activations/layer3_attention_weight_min": -73.61457061767578,
      "activations/layer4_attention_weight_max": 119.78254699707031,
      "activations/layer4_attention_weight_min": -122.5758285522461,
      "activations/layer5_attention_weight_max": 101.84371948242188,
      "activations/layer5_attention_weight_min": -94.10227966308594,
      "activations/layer6_attention_weight_max": 57.15123748779297,
      "activations/layer6_attention_weight_min": -54.347740173339844,
      "activations/layer7_attention_weight_max": 71.5282211303711,
      "activations/layer7_attention_weight_min": -70.57428741455078,
      "activations/layer8_attention_weight_max": 39.161827087402344,
      "activations/layer8_attention_weight_min": -35.40509796142578,
      "activations/layer9_attention_weight_max": 63.734130859375,
      "activations/layer9_attention_weight_min": -57.76320266723633,
      "epoch": 2.73,
      "learning_rate": 0.0001337526515151515,
      "loss": 2.9331,
      "step": 46950
    },
    {
      "activations/layer0_attention_weight_max": 17.557771682739258,
      "activations/layer0_attention_weight_min": -11.983882904052734,
      "activations/layer10_attention_weight_max": 42.373138427734375,
      "activations/layer10_attention_weight_min": -35.19548034667969,
      "activations/layer11_attention_weight_max": 34.87931823730469,
      "activations/layer11_attention_weight_min": -31.33053970336914,
      "activations/layer12_attention_weight_max": 33.00951385498047,
      "activations/layer12_attention_weight_min": -27.487730026245117,
      "activations/layer13_attention_weight_max": 48.69859313964844,
      "activations/layer13_attention_weight_min": -34.06178283691406,
      "activations/layer14_attention_weight_max": 70.22225189208984,
      "activations/layer14_attention_weight_min": -52.63509750366211,
      "activations/layer15_attention_weight_max": 49.37174606323242,
      "activations/layer15_attention_weight_min": -33.58240509033203,
      "activations/layer16_attention_weight_max": 43.77578353881836,
      "activations/layer16_attention_weight_min": -31.497695922851562,
      "activations/layer17_attention_weight_max": 44.40082931518555,
      "activations/layer17_attention_weight_min": -28.284517288208008,
      "activations/layer18_attention_weight_max": 30.741849899291992,
      "activations/layer18_attention_weight_min": -23.26128578186035,
      "activations/layer19_attention_weight_max": 42.37264633178711,
      "activations/layer19_attention_weight_min": -28.393518447875977,
      "activations/layer1_attention_weight_max": 19.173845291137695,
      "activations/layer1_attention_weight_min": -14.14966869354248,
      "activations/layer20_attention_weight_max": 35.69581604003906,
      "activations/layer20_attention_weight_min": -23.91254234313965,
      "activations/layer21_attention_weight_max": 35.20746994018555,
      "activations/layer21_attention_weight_min": -24.039419174194336,
      "activations/layer22_attention_weight_max": 44.752113342285156,
      "activations/layer22_attention_weight_min": -30.15260124206543,
      "activations/layer23_attention_weight_max": 28.681312561035156,
      "activations/layer23_attention_weight_min": -20.57456398010254,
      "activations/layer2_attention_weight_max": 29.067134857177734,
      "activations/layer2_attention_weight_min": -29.333480834960938,
      "activations/layer3_attention_weight_max": 73.59087371826172,
      "activations/layer3_attention_weight_min": -79.23706817626953,
      "activations/layer4_attention_weight_max": 127.75628662109375,
      "activations/layer4_attention_weight_min": -124.49334716796875,
      "activations/layer5_attention_weight_max": 106.8327865600586,
      "activations/layer5_attention_weight_min": -93.9359130859375,
      "activations/layer6_attention_weight_max": 58.32205581665039,
      "activations/layer6_attention_weight_min": -51.9559440612793,
      "activations/layer7_attention_weight_max": 70.81986236572266,
      "activations/layer7_attention_weight_min": -66.51578521728516,
      "activations/layer8_attention_weight_max": 40.30421829223633,
      "activations/layer8_attention_weight_min": -38.23384475708008,
      "activations/layer9_attention_weight_max": 54.18857955932617,
      "activations/layer9_attention_weight_min": -41.78896713256836,
      "epoch": 2.73,
      "learning_rate": 0.00013373371212121213,
      "loss": 2.9408,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_loss": 2.87109375,
      "eval_runtime": 8.4998,
      "eval_samples_per_second": 505.187,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_openwebtext_loss": 2.87109375,
      "eval_openwebtext_ppl": 17.656319241826594,
      "eval_openwebtext_runtime": 8.4998,
      "eval_openwebtext_samples_per_second": 505.187,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_wikitext_loss": 3.125,
      "eval_wikitext_ppl": 22.75989509352673,
      "eval_wikitext_runtime": 2.0181,
      "eval_wikitext_samples_per_second": 225.952,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_lambada_loss": 3.0078125,
      "eval_lambada_ppl": 20.24306974171993,
      "eval_lambada_runtime": 9.5826,
      "eval_lambada_samples_per_second": 508.111,
      "step": 47000
    },
    {
      "activations/layer0_attention_weight_max": 17.336227416992188,
      "activations/layer0_attention_weight_min": -12.47597885131836,
      "activations/layer10_attention_weight_max": 49.13496017456055,
      "activations/layer10_attention_weight_min": -37.493690490722656,
      "activations/layer11_attention_weight_max": 40.60298156738281,
      "activations/layer11_attention_weight_min": -33.369140625,
      "activations/layer12_attention_weight_max": 62.05290603637695,
      "activations/layer12_attention_weight_min": -30.11211585998535,
      "activations/layer13_attention_weight_max": 53.43362808227539,
      "activations/layer13_attention_weight_min": -36.25556182861328,
      "activations/layer14_attention_weight_max": 68.35562133789062,
      "activations/layer14_attention_weight_min": -48.18288040161133,
      "activations/layer15_attention_weight_max": 50.62290954589844,
      "activations/layer15_attention_weight_min": -34.049293518066406,
      "activations/layer16_attention_weight_max": 45.00355529785156,
      "activations/layer16_attention_weight_min": -32.919742584228516,
      "activations/layer17_attention_weight_max": 45.783390045166016,
      "activations/layer17_attention_weight_min": -28.711002349853516,
      "activations/layer18_attention_weight_max": 26.494565963745117,
      "activations/layer18_attention_weight_min": -17.916255950927734,
      "activations/layer19_attention_weight_max": 44.15153503417969,
      "activations/layer19_attention_weight_min": -28.23044776916504,
      "activations/layer1_attention_weight_max": 17.71770477294922,
      "activations/layer1_attention_weight_min": -14.373285293579102,
      "activations/layer20_attention_weight_max": 34.64347839355469,
      "activations/layer20_attention_weight_min": -22.329099655151367,
      "activations/layer21_attention_weight_max": 36.8954963684082,
      "activations/layer21_attention_weight_min": -20.423612594604492,
      "activations/layer22_attention_weight_max": 44.8740119934082,
      "activations/layer22_attention_weight_min": -25.667850494384766,
      "activations/layer23_attention_weight_max": 27.857648849487305,
      "activations/layer23_attention_weight_min": -16.274824142456055,
      "activations/layer2_attention_weight_max": 27.73182487487793,
      "activations/layer2_attention_weight_min": -28.24281883239746,
      "activations/layer3_attention_weight_max": 72.96276092529297,
      "activations/layer3_attention_weight_min": -74.83340454101562,
      "activations/layer4_attention_weight_max": 120.3035659790039,
      "activations/layer4_attention_weight_min": -117.10538482666016,
      "activations/layer5_attention_weight_max": 100.96463012695312,
      "activations/layer5_attention_weight_min": -91.62902069091797,
      "activations/layer6_attention_weight_max": 55.45421600341797,
      "activations/layer6_attention_weight_min": -51.56459045410156,
      "activations/layer7_attention_weight_max": 68.09333801269531,
      "activations/layer7_attention_weight_min": -68.22821807861328,
      "activations/layer8_attention_weight_max": 38.05460739135742,
      "activations/layer8_attention_weight_min": -36.08140182495117,
      "activations/layer9_attention_weight_max": 57.37501907348633,
      "activations/layer9_attention_weight_min": -44.01496505737305,
      "epoch": 2.73,
      "learning_rate": 0.0001337147727272727,
      "loss": 2.9169,
      "step": 47050
    },
    {
      "activations/layer0_attention_weight_max": 17.690673828125,
      "activations/layer0_attention_weight_min": -11.136845588684082,
      "activations/layer10_attention_weight_max": 41.867462158203125,
      "activations/layer10_attention_weight_min": -33.74966049194336,
      "activations/layer11_attention_weight_max": 45.391258239746094,
      "activations/layer11_attention_weight_min": -37.692691802978516,
      "activations/layer12_attention_weight_max": 44.1865119934082,
      "activations/layer12_attention_weight_min": -32.971012115478516,
      "activations/layer13_attention_weight_max": 59.40190505981445,
      "activations/layer13_attention_weight_min": -43.22562026977539,
      "activations/layer14_attention_weight_max": 92.37566375732422,
      "activations/layer14_attention_weight_min": -66.49229431152344,
      "activations/layer15_attention_weight_max": 58.130672454833984,
      "activations/layer15_attention_weight_min": -37.320823669433594,
      "activations/layer16_attention_weight_max": 51.65547561645508,
      "activations/layer16_attention_weight_min": -35.84017562866211,
      "activations/layer17_attention_weight_max": 50.95742416381836,
      "activations/layer17_attention_weight_min": -28.379180908203125,
      "activations/layer18_attention_weight_max": 29.2450008392334,
      "activations/layer18_attention_weight_min": -19.908611297607422,
      "activations/layer19_attention_weight_max": 44.453521728515625,
      "activations/layer19_attention_weight_min": -29.83182716369629,
      "activations/layer1_attention_weight_max": 17.95009994506836,
      "activations/layer1_attention_weight_min": -13.182756423950195,
      "activations/layer20_attention_weight_max": 32.75803756713867,
      "activations/layer20_attention_weight_min": -22.886384963989258,
      "activations/layer21_attention_weight_max": 32.5313606262207,
      "activations/layer21_attention_weight_min": -19.73418617248535,
      "activations/layer22_attention_weight_max": 47.2468376159668,
      "activations/layer22_attention_weight_min": -27.22121238708496,
      "activations/layer23_attention_weight_max": 25.11372947692871,
      "activations/layer23_attention_weight_min": -16.103622436523438,
      "activations/layer2_attention_weight_max": 29.34838104248047,
      "activations/layer2_attention_weight_min": -27.947559356689453,
      "activations/layer3_attention_weight_max": 74.45030975341797,
      "activations/layer3_attention_weight_min": -76.53959655761719,
      "activations/layer4_attention_weight_max": 133.2606658935547,
      "activations/layer4_attention_weight_min": -118.8204345703125,
      "activations/layer5_attention_weight_max": 106.08354949951172,
      "activations/layer5_attention_weight_min": -87.51435852050781,
      "activations/layer6_attention_weight_max": 57.26566696166992,
      "activations/layer6_attention_weight_min": -49.91102981567383,
      "activations/layer7_attention_weight_max": 75.41815185546875,
      "activations/layer7_attention_weight_min": -75.79197692871094,
      "activations/layer8_attention_weight_max": 42.871253967285156,
      "activations/layer8_attention_weight_min": -38.5207633972168,
      "activations/layer9_attention_weight_max": 59.395198822021484,
      "activations/layer9_attention_weight_min": -56.63161849975586,
      "epoch": 2.74,
      "learning_rate": 0.00013369583333333333,
      "loss": 2.9202,
      "step": 47100
    },
    {
      "activations/layer0_attention_weight_max": 17.483837127685547,
      "activations/layer0_attention_weight_min": -11.787866592407227,
      "activations/layer10_attention_weight_max": 48.4005012512207,
      "activations/layer10_attention_weight_min": -36.519569396972656,
      "activations/layer11_attention_weight_max": 40.22213363647461,
      "activations/layer11_attention_weight_min": -33.094791412353516,
      "activations/layer12_attention_weight_max": 33.88608169555664,
      "activations/layer12_attention_weight_min": -26.802160263061523,
      "activations/layer13_attention_weight_max": 55.89707565307617,
      "activations/layer13_attention_weight_min": -36.59299850463867,
      "activations/layer14_attention_weight_max": 96.75213623046875,
      "activations/layer14_attention_weight_min": -59.21509552001953,
      "activations/layer15_attention_weight_max": 53.25684356689453,
      "activations/layer15_attention_weight_min": -30.858896255493164,
      "activations/layer16_attention_weight_max": 52.18806457519531,
      "activations/layer16_attention_weight_min": -31.42461395263672,
      "activations/layer17_attention_weight_max": 44.70841598510742,
      "activations/layer17_attention_weight_min": -26.604307174682617,
      "activations/layer18_attention_weight_max": 30.42953109741211,
      "activations/layer18_attention_weight_min": -18.327754974365234,
      "activations/layer19_attention_weight_max": 46.38961410522461,
      "activations/layer19_attention_weight_min": -30.759815216064453,
      "activations/layer1_attention_weight_max": 16.65628433227539,
      "activations/layer1_attention_weight_min": -15.0177583694458,
      "activations/layer20_attention_weight_max": 38.730224609375,
      "activations/layer20_attention_weight_min": -23.108501434326172,
      "activations/layer21_attention_weight_max": 29.546875,
      "activations/layer21_attention_weight_min": -18.7426815032959,
      "activations/layer22_attention_weight_max": 49.85103988647461,
      "activations/layer22_attention_weight_min": -27.940149307250977,
      "activations/layer23_attention_weight_max": 27.26814079284668,
      "activations/layer23_attention_weight_min": -19.419780731201172,
      "activations/layer2_attention_weight_max": 32.59103012084961,
      "activations/layer2_attention_weight_min": -31.695091247558594,
      "activations/layer3_attention_weight_max": 79.12857818603516,
      "activations/layer3_attention_weight_min": -83.81602478027344,
      "activations/layer4_attention_weight_max": 138.8450927734375,
      "activations/layer4_attention_weight_min": -134.83558654785156,
      "activations/layer5_attention_weight_max": 114.87963104248047,
      "activations/layer5_attention_weight_min": -103.64755249023438,
      "activations/layer6_attention_weight_max": 66.68083190917969,
      "activations/layer6_attention_weight_min": -59.6905517578125,
      "activations/layer7_attention_weight_max": 80.72172546386719,
      "activations/layer7_attention_weight_min": -70.70417022705078,
      "activations/layer8_attention_weight_max": 49.48124313354492,
      "activations/layer8_attention_weight_min": -38.64443588256836,
      "activations/layer9_attention_weight_max": 60.34522247314453,
      "activations/layer9_attention_weight_min": -48.38831329345703,
      "epoch": 2.74,
      "learning_rate": 0.00013367689393939392,
      "loss": 2.9412,
      "step": 47150
    },
    {
      "activations/layer0_attention_weight_max": 16.065420150756836,
      "activations/layer0_attention_weight_min": -11.319540977478027,
      "activations/layer10_attention_weight_max": 45.605979919433594,
      "activations/layer10_attention_weight_min": -35.40751266479492,
      "activations/layer11_attention_weight_max": 34.499961853027344,
      "activations/layer11_attention_weight_min": -28.956897735595703,
      "activations/layer12_attention_weight_max": 37.559906005859375,
      "activations/layer12_attention_weight_min": -29.087574005126953,
      "activations/layer13_attention_weight_max": 52.082271575927734,
      "activations/layer13_attention_weight_min": -35.993465423583984,
      "activations/layer14_attention_weight_max": 67.6940689086914,
      "activations/layer14_attention_weight_min": -50.31404113769531,
      "activations/layer15_attention_weight_max": 47.95986557006836,
      "activations/layer15_attention_weight_min": -33.88005447387695,
      "activations/layer16_attention_weight_max": 45.33377456665039,
      "activations/layer16_attention_weight_min": -30.394872665405273,
      "activations/layer17_attention_weight_max": 44.978759765625,
      "activations/layer17_attention_weight_min": -27.42325210571289,
      "activations/layer18_attention_weight_max": 30.598861694335938,
      "activations/layer18_attention_weight_min": -19.373123168945312,
      "activations/layer19_attention_weight_max": 47.028221130371094,
      "activations/layer19_attention_weight_min": -28.665409088134766,
      "activations/layer1_attention_weight_max": 17.651504516601562,
      "activations/layer1_attention_weight_min": -13.30191421508789,
      "activations/layer20_attention_weight_max": 35.49469757080078,
      "activations/layer20_attention_weight_min": -23.817075729370117,
      "activations/layer21_attention_weight_max": 40.121986389160156,
      "activations/layer21_attention_weight_min": -22.400489807128906,
      "activations/layer22_attention_weight_max": 46.868350982666016,
      "activations/layer22_attention_weight_min": -28.95319366455078,
      "activations/layer23_attention_weight_max": 28.913999557495117,
      "activations/layer23_attention_weight_min": -15.860316276550293,
      "activations/layer2_attention_weight_max": 29.473159790039062,
      "activations/layer2_attention_weight_min": -29.944992065429688,
      "activations/layer3_attention_weight_max": 75.88249969482422,
      "activations/layer3_attention_weight_min": -78.71866607666016,
      "activations/layer4_attention_weight_max": 132.36985778808594,
      "activations/layer4_attention_weight_min": -123.5342788696289,
      "activations/layer5_attention_weight_max": 108.85612487792969,
      "activations/layer5_attention_weight_min": -95.87400817871094,
      "activations/layer6_attention_weight_max": 61.28227996826172,
      "activations/layer6_attention_weight_min": -54.66945266723633,
      "activations/layer7_attention_weight_max": 68.88067626953125,
      "activations/layer7_attention_weight_min": -67.82372283935547,
      "activations/layer8_attention_weight_max": 42.3077278137207,
      "activations/layer8_attention_weight_min": -35.78144454956055,
      "activations/layer9_attention_weight_max": 51.20048904418945,
      "activations/layer9_attention_weight_min": -41.20858383178711,
      "epoch": 2.74,
      "learning_rate": 0.00013365795454545453,
      "loss": 2.9364,
      "step": 47200
    },
    {
      "activations/layer0_attention_weight_max": 17.61004638671875,
      "activations/layer0_attention_weight_min": -11.522550582885742,
      "activations/layer10_attention_weight_max": 44.916831970214844,
      "activations/layer10_attention_weight_min": -35.590484619140625,
      "activations/layer11_attention_weight_max": 39.150211334228516,
      "activations/layer11_attention_weight_min": -31.091793060302734,
      "activations/layer12_attention_weight_max": 32.10883331298828,
      "activations/layer12_attention_weight_min": -24.625812530517578,
      "activations/layer13_attention_weight_max": 50.48184585571289,
      "activations/layer13_attention_weight_min": -39.76460266113281,
      "activations/layer14_attention_weight_max": 80.4590835571289,
      "activations/layer14_attention_weight_min": -62.93618392944336,
      "activations/layer15_attention_weight_max": 49.95574951171875,
      "activations/layer15_attention_weight_min": -35.58264923095703,
      "activations/layer16_attention_weight_max": 44.50833511352539,
      "activations/layer16_attention_weight_min": -34.87387466430664,
      "activations/layer17_attention_weight_max": 41.96509552001953,
      "activations/layer17_attention_weight_min": -28.38736343383789,
      "activations/layer18_attention_weight_max": 26.1922664642334,
      "activations/layer18_attention_weight_min": -23.18769645690918,
      "activations/layer19_attention_weight_max": 46.95927429199219,
      "activations/layer19_attention_weight_min": -28.669876098632812,
      "activations/layer1_attention_weight_max": 18.36149024963379,
      "activations/layer1_attention_weight_min": -13.673345565795898,
      "activations/layer20_attention_weight_max": 34.95606994628906,
      "activations/layer20_attention_weight_min": -24.719093322753906,
      "activations/layer21_attention_weight_max": 35.114593505859375,
      "activations/layer21_attention_weight_min": -21.31687355041504,
      "activations/layer22_attention_weight_max": 48.05681610107422,
      "activations/layer22_attention_weight_min": -27.344982147216797,
      "activations/layer23_attention_weight_max": 27.806997299194336,
      "activations/layer23_attention_weight_min": -15.905573844909668,
      "activations/layer2_attention_weight_max": 30.48809051513672,
      "activations/layer2_attention_weight_min": -30.10055923461914,
      "activations/layer3_attention_weight_max": 76.43751525878906,
      "activations/layer3_attention_weight_min": -80.49295043945312,
      "activations/layer4_attention_weight_max": 128.6957244873047,
      "activations/layer4_attention_weight_min": -122.0433578491211,
      "activations/layer5_attention_weight_max": 104.25231170654297,
      "activations/layer5_attention_weight_min": -94.51815795898438,
      "activations/layer6_attention_weight_max": 61.60203552246094,
      "activations/layer6_attention_weight_min": -52.69612503051758,
      "activations/layer7_attention_weight_max": 69.79756927490234,
      "activations/layer7_attention_weight_min": -64.7112808227539,
      "activations/layer8_attention_weight_max": 43.49138641357422,
      "activations/layer8_attention_weight_min": -36.36325454711914,
      "activations/layer9_attention_weight_max": 53.863037109375,
      "activations/layer9_attention_weight_min": -42.80881881713867,
      "epoch": 2.75,
      "learning_rate": 0.00013363901515151515,
      "loss": 2.9086,
      "step": 47250
    },
    {
      "activations/layer0_attention_weight_max": 17.268163681030273,
      "activations/layer0_attention_weight_min": -11.745686531066895,
      "activations/layer10_attention_weight_max": 45.72910690307617,
      "activations/layer10_attention_weight_min": -38.17270278930664,
      "activations/layer11_attention_weight_max": 36.58256912231445,
      "activations/layer11_attention_weight_min": -34.65837478637695,
      "activations/layer12_attention_weight_max": 37.41848373413086,
      "activations/layer12_attention_weight_min": -27.671018600463867,
      "activations/layer13_attention_weight_max": 51.492618560791016,
      "activations/layer13_attention_weight_min": -34.14845275878906,
      "activations/layer14_attention_weight_max": 79.35325622558594,
      "activations/layer14_attention_weight_min": -53.82261657714844,
      "activations/layer15_attention_weight_max": 55.484432220458984,
      "activations/layer15_attention_weight_min": -38.82754898071289,
      "activations/layer16_attention_weight_max": 46.79828643798828,
      "activations/layer16_attention_weight_min": -33.292354583740234,
      "activations/layer17_attention_weight_max": 45.105010986328125,
      "activations/layer17_attention_weight_min": -27.205564498901367,
      "activations/layer18_attention_weight_max": 26.645000457763672,
      "activations/layer18_attention_weight_min": -17.22426414489746,
      "activations/layer19_attention_weight_max": 46.971038818359375,
      "activations/layer19_attention_weight_min": -28.442296981811523,
      "activations/layer1_attention_weight_max": 17.317352294921875,
      "activations/layer1_attention_weight_min": -12.274606704711914,
      "activations/layer20_attention_weight_max": 35.924034118652344,
      "activations/layer20_attention_weight_min": -22.327564239501953,
      "activations/layer21_attention_weight_max": 32.10113525390625,
      "activations/layer21_attention_weight_min": -18.984479904174805,
      "activations/layer22_attention_weight_max": 49.854286193847656,
      "activations/layer22_attention_weight_min": -24.580713272094727,
      "activations/layer23_attention_weight_max": 25.635509490966797,
      "activations/layer23_attention_weight_min": -15.555625915527344,
      "activations/layer2_attention_weight_max": 30.126758575439453,
      "activations/layer2_attention_weight_min": -29.437545776367188,
      "activations/layer3_attention_weight_max": 79.65275573730469,
      "activations/layer3_attention_weight_min": -82.22225189208984,
      "activations/layer4_attention_weight_max": 135.30845642089844,
      "activations/layer4_attention_weight_min": -123.76653289794922,
      "activations/layer5_attention_weight_max": 105.43020629882812,
      "activations/layer5_attention_weight_min": -89.65731048583984,
      "activations/layer6_attention_weight_max": 58.90195083618164,
      "activations/layer6_attention_weight_min": -51.7232551574707,
      "activations/layer7_attention_weight_max": 75.63719940185547,
      "activations/layer7_attention_weight_min": -63.71760177612305,
      "activations/layer8_attention_weight_max": 43.25152587890625,
      "activations/layer8_attention_weight_min": -37.08644485473633,
      "activations/layer9_attention_weight_max": 54.24787521362305,
      "activations/layer9_attention_weight_min": -47.93494415283203,
      "epoch": 2.75,
      "learning_rate": 0.00013362007575757573,
      "loss": 2.9215,
      "step": 47300
    },
    {
      "activations/layer0_attention_weight_max": 17.575328826904297,
      "activations/layer0_attention_weight_min": -12.829329490661621,
      "activations/layer10_attention_weight_max": 48.132049560546875,
      "activations/layer10_attention_weight_min": -37.060550689697266,
      "activations/layer11_attention_weight_max": 40.028934478759766,
      "activations/layer11_attention_weight_min": -31.149534225463867,
      "activations/layer12_attention_weight_max": 33.415565490722656,
      "activations/layer12_attention_weight_min": -28.424711227416992,
      "activations/layer13_attention_weight_max": 47.83380889892578,
      "activations/layer13_attention_weight_min": -36.65566635131836,
      "activations/layer14_attention_weight_max": 82.09443664550781,
      "activations/layer14_attention_weight_min": -55.33704376220703,
      "activations/layer15_attention_weight_max": 46.27884292602539,
      "activations/layer15_attention_weight_min": -32.060909271240234,
      "activations/layer16_attention_weight_max": 52.18864059448242,
      "activations/layer16_attention_weight_min": -32.78911590576172,
      "activations/layer17_attention_weight_max": 43.533287048339844,
      "activations/layer17_attention_weight_min": -28.73444366455078,
      "activations/layer18_attention_weight_max": 27.182044982910156,
      "activations/layer18_attention_weight_min": -18.643415451049805,
      "activations/layer19_attention_weight_max": 50.623836517333984,
      "activations/layer19_attention_weight_min": -31.2073917388916,
      "activations/layer1_attention_weight_max": 17.829294204711914,
      "activations/layer1_attention_weight_min": -12.599743843078613,
      "activations/layer20_attention_weight_max": 33.88197326660156,
      "activations/layer20_attention_weight_min": -23.525753021240234,
      "activations/layer21_attention_weight_max": 32.2556037902832,
      "activations/layer21_attention_weight_min": -20.702251434326172,
      "activations/layer22_attention_weight_max": 46.047157287597656,
      "activations/layer22_attention_weight_min": -28.22182273864746,
      "activations/layer23_attention_weight_max": 26.974760055541992,
      "activations/layer23_attention_weight_min": -17.653167724609375,
      "activations/layer2_attention_weight_max": 30.901683807373047,
      "activations/layer2_attention_weight_min": -29.34050750732422,
      "activations/layer3_attention_weight_max": 81.53009796142578,
      "activations/layer3_attention_weight_min": -83.13613891601562,
      "activations/layer4_attention_weight_max": 139.2839813232422,
      "activations/layer4_attention_weight_min": -129.694580078125,
      "activations/layer5_attention_weight_max": 110.75780487060547,
      "activations/layer5_attention_weight_min": -95.13278198242188,
      "activations/layer6_attention_weight_max": 64.87287139892578,
      "activations/layer6_attention_weight_min": -55.119903564453125,
      "activations/layer7_attention_weight_max": 73.75084686279297,
      "activations/layer7_attention_weight_min": -71.96805572509766,
      "activations/layer8_attention_weight_max": 43.86488342285156,
      "activations/layer8_attention_weight_min": -38.184532165527344,
      "activations/layer9_attention_weight_max": 61.6860466003418,
      "activations/layer9_attention_weight_min": -53.729854583740234,
      "epoch": 2.75,
      "learning_rate": 0.00013360113636363635,
      "loss": 2.9157,
      "step": 47350
    },
    {
      "activations/layer0_attention_weight_max": 13.994020462036133,
      "activations/layer0_attention_weight_min": -10.97785758972168,
      "activations/layer10_attention_weight_max": 46.6531867980957,
      "activations/layer10_attention_weight_min": -38.13899230957031,
      "activations/layer11_attention_weight_max": 37.87751388549805,
      "activations/layer11_attention_weight_min": -30.044958114624023,
      "activations/layer12_attention_weight_max": 32.60120391845703,
      "activations/layer12_attention_weight_min": -27.118106842041016,
      "activations/layer13_attention_weight_max": 54.54084014892578,
      "activations/layer13_attention_weight_min": -37.7406005859375,
      "activations/layer14_attention_weight_max": 72.22283172607422,
      "activations/layer14_attention_weight_min": -49.92973327636719,
      "activations/layer15_attention_weight_max": 49.58664321899414,
      "activations/layer15_attention_weight_min": -34.397926330566406,
      "activations/layer16_attention_weight_max": 47.789913177490234,
      "activations/layer16_attention_weight_min": -31.294145584106445,
      "activations/layer17_attention_weight_max": 42.358642578125,
      "activations/layer17_attention_weight_min": -26.57925796508789,
      "activations/layer18_attention_weight_max": 28.429475784301758,
      "activations/layer18_attention_weight_min": -20.111766815185547,
      "activations/layer19_attention_weight_max": 47.23148727416992,
      "activations/layer19_attention_weight_min": -29.330942153930664,
      "activations/layer1_attention_weight_max": 17.157419204711914,
      "activations/layer1_attention_weight_min": -12.71144962310791,
      "activations/layer20_attention_weight_max": 34.29034423828125,
      "activations/layer20_attention_weight_min": -24.7384033203125,
      "activations/layer21_attention_weight_max": 38.860374450683594,
      "activations/layer21_attention_weight_min": -22.09882354736328,
      "activations/layer22_attention_weight_max": 47.09029006958008,
      "activations/layer22_attention_weight_min": -28.67451286315918,
      "activations/layer23_attention_weight_max": 24.729373931884766,
      "activations/layer23_attention_weight_min": -16.721282958984375,
      "activations/layer2_attention_weight_max": 28.335037231445312,
      "activations/layer2_attention_weight_min": -28.358783721923828,
      "activations/layer3_attention_weight_max": 73.47821807861328,
      "activations/layer3_attention_weight_min": -79.466552734375,
      "activations/layer4_attention_weight_max": 123.4398422241211,
      "activations/layer4_attention_weight_min": -121.2995834350586,
      "activations/layer5_attention_weight_max": 101.00189208984375,
      "activations/layer5_attention_weight_min": -88.76771545410156,
      "activations/layer6_attention_weight_max": 58.812522888183594,
      "activations/layer6_attention_weight_min": -51.016300201416016,
      "activations/layer7_attention_weight_max": 66.56163024902344,
      "activations/layer7_attention_weight_min": -63.70568084716797,
      "activations/layer8_attention_weight_max": 40.650142669677734,
      "activations/layer8_attention_weight_min": -35.9191780090332,
      "activations/layer9_attention_weight_max": 50.38176345825195,
      "activations/layer9_attention_weight_min": -39.589290618896484,
      "epoch": 2.75,
      "learning_rate": 0.00013358219696969696,
      "loss": 2.9404,
      "step": 47400
    },
    {
      "activations/layer0_attention_weight_max": 17.215879440307617,
      "activations/layer0_attention_weight_min": -11.416050910949707,
      "activations/layer10_attention_weight_max": 47.8187370300293,
      "activations/layer10_attention_weight_min": -38.730690002441406,
      "activations/layer11_attention_weight_max": 50.318214416503906,
      "activations/layer11_attention_weight_min": -42.841522216796875,
      "activations/layer12_attention_weight_max": 29.392723083496094,
      "activations/layer12_attention_weight_min": -24.355838775634766,
      "activations/layer13_attention_weight_max": 51.03061294555664,
      "activations/layer13_attention_weight_min": -40.30569076538086,
      "activations/layer14_attention_weight_max": 79.51380920410156,
      "activations/layer14_attention_weight_min": -61.3791618347168,
      "activations/layer15_attention_weight_max": 48.87604904174805,
      "activations/layer15_attention_weight_min": -32.44554138183594,
      "activations/layer16_attention_weight_max": 43.29408645629883,
      "activations/layer16_attention_weight_min": -30.680631637573242,
      "activations/layer17_attention_weight_max": 44.06422424316406,
      "activations/layer17_attention_weight_min": -26.94607925415039,
      "activations/layer18_attention_weight_max": 27.297700881958008,
      "activations/layer18_attention_weight_min": -17.628374099731445,
      "activations/layer19_attention_weight_max": 44.70452117919922,
      "activations/layer19_attention_weight_min": -29.601224899291992,
      "activations/layer1_attention_weight_max": 18.087194442749023,
      "activations/layer1_attention_weight_min": -12.706039428710938,
      "activations/layer20_attention_weight_max": 32.93144989013672,
      "activations/layer20_attention_weight_min": -24.177989959716797,
      "activations/layer21_attention_weight_max": 29.683635711669922,
      "activations/layer21_attention_weight_min": -20.697851181030273,
      "activations/layer22_attention_weight_max": 48.823726654052734,
      "activations/layer22_attention_weight_min": -28.098358154296875,
      "activations/layer23_attention_weight_max": 27.315671920776367,
      "activations/layer23_attention_weight_min": -17.08898162841797,
      "activations/layer2_attention_weight_max": 31.164047241210938,
      "activations/layer2_attention_weight_min": -29.809558868408203,
      "activations/layer3_attention_weight_max": 81.45748901367188,
      "activations/layer3_attention_weight_min": -81.8284912109375,
      "activations/layer4_attention_weight_max": 134.62217712402344,
      "activations/layer4_attention_weight_min": -127.899169921875,
      "activations/layer5_attention_weight_max": 110.00277709960938,
      "activations/layer5_attention_weight_min": -98.72673034667969,
      "activations/layer6_attention_weight_max": 62.578800201416016,
      "activations/layer6_attention_weight_min": -54.66120529174805,
      "activations/layer7_attention_weight_max": 70.23882293701172,
      "activations/layer7_attention_weight_min": -69.84574890136719,
      "activations/layer8_attention_weight_max": 47.90679931640625,
      "activations/layer8_attention_weight_min": -39.15077590942383,
      "activations/layer9_attention_weight_max": 57.12063217163086,
      "activations/layer9_attention_weight_min": -54.6314811706543,
      "epoch": 2.76,
      "learning_rate": 0.00013356325757575755,
      "loss": 2.9266,
      "step": 47450
    },
    {
      "activations/layer0_attention_weight_max": 17.164175033569336,
      "activations/layer0_attention_weight_min": -11.244447708129883,
      "activations/layer10_attention_weight_max": 52.55281448364258,
      "activations/layer10_attention_weight_min": -43.715972900390625,
      "activations/layer11_attention_weight_max": 38.2991943359375,
      "activations/layer11_attention_weight_min": -30.496740341186523,
      "activations/layer12_attention_weight_max": 42.420562744140625,
      "activations/layer12_attention_weight_min": -26.090564727783203,
      "activations/layer13_attention_weight_max": 56.36924743652344,
      "activations/layer13_attention_weight_min": -39.1400146484375,
      "activations/layer14_attention_weight_max": 84.7032699584961,
      "activations/layer14_attention_weight_min": -61.1005744934082,
      "activations/layer15_attention_weight_max": 52.229427337646484,
      "activations/layer15_attention_weight_min": -35.78056716918945,
      "activations/layer16_attention_weight_max": 41.85355758666992,
      "activations/layer16_attention_weight_min": -31.088960647583008,
      "activations/layer17_attention_weight_max": 42.5408821105957,
      "activations/layer17_attention_weight_min": -26.04235076904297,
      "activations/layer18_attention_weight_max": 32.15636444091797,
      "activations/layer18_attention_weight_min": -20.309030532836914,
      "activations/layer19_attention_weight_max": 44.81962966918945,
      "activations/layer19_attention_weight_min": -30.048603057861328,
      "activations/layer1_attention_weight_max": 17.647729873657227,
      "activations/layer1_attention_weight_min": -11.86196231842041,
      "activations/layer20_attention_weight_max": 33.239662170410156,
      "activations/layer20_attention_weight_min": -22.960308074951172,
      "activations/layer21_attention_weight_max": 29.044612884521484,
      "activations/layer21_attention_weight_min": -19.839372634887695,
      "activations/layer22_attention_weight_max": 42.98435974121094,
      "activations/layer22_attention_weight_min": -25.730289459228516,
      "activations/layer23_attention_weight_max": 26.70370864868164,
      "activations/layer23_attention_weight_min": -16.66222381591797,
      "activations/layer2_attention_weight_max": 30.030624389648438,
      "activations/layer2_attention_weight_min": -31.30605125427246,
      "activations/layer3_attention_weight_max": 80.95783233642578,
      "activations/layer3_attention_weight_min": -83.95983123779297,
      "activations/layer4_attention_weight_max": 135.48390197753906,
      "activations/layer4_attention_weight_min": -130.119140625,
      "activations/layer5_attention_weight_max": 112.9854965209961,
      "activations/layer5_attention_weight_min": -102.66305541992188,
      "activations/layer6_attention_weight_max": 63.09428405761719,
      "activations/layer6_attention_weight_min": -59.70330047607422,
      "activations/layer7_attention_weight_max": 78.3795394897461,
      "activations/layer7_attention_weight_min": -71.12625122070312,
      "activations/layer8_attention_weight_max": 43.110008239746094,
      "activations/layer8_attention_weight_min": -39.22529220581055,
      "activations/layer9_attention_weight_max": 65.2870864868164,
      "activations/layer9_attention_weight_min": -49.33662414550781,
      "epoch": 2.76,
      "learning_rate": 0.00013354431818181817,
      "loss": 2.9181,
      "step": 47500
    },
    {
      "activations/layer0_attention_weight_max": 17.68524742126465,
      "activations/layer0_attention_weight_min": -11.301182746887207,
      "activations/layer10_attention_weight_max": 44.94565200805664,
      "activations/layer10_attention_weight_min": -36.96536636352539,
      "activations/layer11_attention_weight_max": 39.419090270996094,
      "activations/layer11_attention_weight_min": -33.154571533203125,
      "activations/layer12_attention_weight_max": 46.7415885925293,
      "activations/layer12_attention_weight_min": -27.779699325561523,
      "activations/layer13_attention_weight_max": 77.32072448730469,
      "activations/layer13_attention_weight_min": -49.68352127075195,
      "activations/layer14_attention_weight_max": 96.25260162353516,
      "activations/layer14_attention_weight_min": -61.38618850708008,
      "activations/layer15_attention_weight_max": 81.89293670654297,
      "activations/layer15_attention_weight_min": -44.26270294189453,
      "activations/layer16_attention_weight_max": 59.1336784362793,
      "activations/layer16_attention_weight_min": -32.57416534423828,
      "activations/layer17_attention_weight_max": 53.510948181152344,
      "activations/layer17_attention_weight_min": -27.562829971313477,
      "activations/layer18_attention_weight_max": 43.816925048828125,
      "activations/layer18_attention_weight_min": -25.211740493774414,
      "activations/layer19_attention_weight_max": 62.71809768676758,
      "activations/layer19_attention_weight_min": -31.57275390625,
      "activations/layer1_attention_weight_max": 17.577627182006836,
      "activations/layer1_attention_weight_min": -13.015588760375977,
      "activations/layer20_attention_weight_max": 44.610225677490234,
      "activations/layer20_attention_weight_min": -23.91441535949707,
      "activations/layer21_attention_weight_max": 34.38360595703125,
      "activations/layer21_attention_weight_min": -19.937326431274414,
      "activations/layer22_attention_weight_max": 66.41971588134766,
      "activations/layer22_attention_weight_min": -28.349626541137695,
      "activations/layer23_attention_weight_max": 37.455604553222656,
      "activations/layer23_attention_weight_min": -18.15863609313965,
      "activations/layer2_attention_weight_max": 30.870708465576172,
      "activations/layer2_attention_weight_min": -32.407814025878906,
      "activations/layer3_attention_weight_max": 78.42842102050781,
      "activations/layer3_attention_weight_min": -84.6568603515625,
      "activations/layer4_attention_weight_max": 129.27308654785156,
      "activations/layer4_attention_weight_min": -122.70706939697266,
      "activations/layer5_attention_weight_max": 106.1930160522461,
      "activations/layer5_attention_weight_min": -91.94268798828125,
      "activations/layer6_attention_weight_max": 59.832550048828125,
      "activations/layer6_attention_weight_min": -50.778751373291016,
      "activations/layer7_attention_weight_max": 72.15171813964844,
      "activations/layer7_attention_weight_min": -66.55596923828125,
      "activations/layer8_attention_weight_max": 43.08411407470703,
      "activations/layer8_attention_weight_min": -38.32509231567383,
      "activations/layer9_attention_weight_max": 59.8583869934082,
      "activations/layer9_attention_weight_min": -44.572689056396484,
      "epoch": 2.76,
      "learning_rate": 0.00013352537878787878,
      "loss": 2.9188,
      "step": 47550
    },
    {
      "activations/layer0_attention_weight_max": 17.510906219482422,
      "activations/layer0_attention_weight_min": -10.973938941955566,
      "activations/layer10_attention_weight_max": 42.44472122192383,
      "activations/layer10_attention_weight_min": -34.06600570678711,
      "activations/layer11_attention_weight_max": 40.00012969970703,
      "activations/layer11_attention_weight_min": -30.620586395263672,
      "activations/layer12_attention_weight_max": 29.571613311767578,
      "activations/layer12_attention_weight_min": -30.35556411743164,
      "activations/layer13_attention_weight_max": 51.53358840942383,
      "activations/layer13_attention_weight_min": -34.5887451171875,
      "activations/layer14_attention_weight_max": 77.18976593017578,
      "activations/layer14_attention_weight_min": -57.527366638183594,
      "activations/layer15_attention_weight_max": 59.5207405090332,
      "activations/layer15_attention_weight_min": -33.48701095581055,
      "activations/layer16_attention_weight_max": 50.51523208618164,
      "activations/layer16_attention_weight_min": -32.92131805419922,
      "activations/layer17_attention_weight_max": 53.496490478515625,
      "activations/layer17_attention_weight_min": -26.470285415649414,
      "activations/layer18_attention_weight_max": 34.137535095214844,
      "activations/layer18_attention_weight_min": -20.197778701782227,
      "activations/layer19_attention_weight_max": 54.02694320678711,
      "activations/layer19_attention_weight_min": -29.793127059936523,
      "activations/layer1_attention_weight_max": 19.4265193939209,
      "activations/layer1_attention_weight_min": -12.023917198181152,
      "activations/layer20_attention_weight_max": 40.19252395629883,
      "activations/layer20_attention_weight_min": -23.920825958251953,
      "activations/layer21_attention_weight_max": 40.69806671142578,
      "activations/layer21_attention_weight_min": -21.671228408813477,
      "activations/layer22_attention_weight_max": 54.353240966796875,
      "activations/layer22_attention_weight_min": -27.884601593017578,
      "activations/layer23_attention_weight_max": 28.870267868041992,
      "activations/layer23_attention_weight_min": -16.80984878540039,
      "activations/layer2_attention_weight_max": 29.559326171875,
      "activations/layer2_attention_weight_min": -29.543630599975586,
      "activations/layer3_attention_weight_max": 75.40719604492188,
      "activations/layer3_attention_weight_min": -78.58428955078125,
      "activations/layer4_attention_weight_max": 134.76031494140625,
      "activations/layer4_attention_weight_min": -123.65953826904297,
      "activations/layer5_attention_weight_max": 109.0686264038086,
      "activations/layer5_attention_weight_min": -96.83610534667969,
      "activations/layer6_attention_weight_max": 58.60725402832031,
      "activations/layer6_attention_weight_min": -51.9598274230957,
      "activations/layer7_attention_weight_max": 66.1534423828125,
      "activations/layer7_attention_weight_min": -66.4642105102539,
      "activations/layer8_attention_weight_max": 43.91677474975586,
      "activations/layer8_attention_weight_min": -35.28938293457031,
      "activations/layer9_attention_weight_max": 49.86065673828125,
      "activations/layer9_attention_weight_min": -43.45550537109375,
      "epoch": 2.77,
      "learning_rate": 0.00013350643939393937,
      "loss": 2.9143,
      "step": 47600
    },
    {
      "activations/layer0_attention_weight_max": 17.130216598510742,
      "activations/layer0_attention_weight_min": -10.944192886352539,
      "activations/layer10_attention_weight_max": 42.652706146240234,
      "activations/layer10_attention_weight_min": -33.950069427490234,
      "activations/layer11_attention_weight_max": 40.62871551513672,
      "activations/layer11_attention_weight_min": -30.749021530151367,
      "activations/layer12_attention_weight_max": 35.53986358642578,
      "activations/layer12_attention_weight_min": -28.719547271728516,
      "activations/layer13_attention_weight_max": 53.17876052856445,
      "activations/layer13_attention_weight_min": -36.33674621582031,
      "activations/layer14_attention_weight_max": 103.65155029296875,
      "activations/layer14_attention_weight_min": -73.50398254394531,
      "activations/layer15_attention_weight_max": 47.00392532348633,
      "activations/layer15_attention_weight_min": -32.752559661865234,
      "activations/layer16_attention_weight_max": 43.21597671508789,
      "activations/layer16_attention_weight_min": -34.09854507446289,
      "activations/layer17_attention_weight_max": 42.26011657714844,
      "activations/layer17_attention_weight_min": -28.96407127380371,
      "activations/layer18_attention_weight_max": 29.545024871826172,
      "activations/layer18_attention_weight_min": -19.534128189086914,
      "activations/layer19_attention_weight_max": 46.02523422241211,
      "activations/layer19_attention_weight_min": -30.61311149597168,
      "activations/layer1_attention_weight_max": 17.78791046142578,
      "activations/layer1_attention_weight_min": -14.726274490356445,
      "activations/layer20_attention_weight_max": 34.09558868408203,
      "activations/layer20_attention_weight_min": -23.254165649414062,
      "activations/layer21_attention_weight_max": 32.02986526489258,
      "activations/layer21_attention_weight_min": -21.74025535583496,
      "activations/layer22_attention_weight_max": 45.01953125,
      "activations/layer22_attention_weight_min": -26.71971321105957,
      "activations/layer23_attention_weight_max": 27.154874801635742,
      "activations/layer23_attention_weight_min": -22.3029842376709,
      "activations/layer2_attention_weight_max": 29.85942840576172,
      "activations/layer2_attention_weight_min": -29.589099884033203,
      "activations/layer3_attention_weight_max": 76.71395874023438,
      "activations/layer3_attention_weight_min": -79.90957641601562,
      "activations/layer4_attention_weight_max": 134.61260986328125,
      "activations/layer4_attention_weight_min": -126.99608612060547,
      "activations/layer5_attention_weight_max": 110.84807586669922,
      "activations/layer5_attention_weight_min": -94.95702362060547,
      "activations/layer6_attention_weight_max": 59.848487854003906,
      "activations/layer6_attention_weight_min": -53.99995040893555,
      "activations/layer7_attention_weight_max": 68.92489624023438,
      "activations/layer7_attention_weight_min": -66.45173645019531,
      "activations/layer8_attention_weight_max": 45.34138488769531,
      "activations/layer8_attention_weight_min": -35.53269577026367,
      "activations/layer9_attention_weight_max": 51.805885314941406,
      "activations/layer9_attention_weight_min": -44.286380767822266,
      "epoch": 2.77,
      "learning_rate": 0.00013348749999999999,
      "loss": 2.9168,
      "step": 47650
    },
    {
      "activations/layer0_attention_weight_max": 15.943352699279785,
      "activations/layer0_attention_weight_min": -11.085453987121582,
      "activations/layer10_attention_weight_max": 44.1549186706543,
      "activations/layer10_attention_weight_min": -34.61806106567383,
      "activations/layer11_attention_weight_max": 38.47808837890625,
      "activations/layer11_attention_weight_min": -28.78920555114746,
      "activations/layer12_attention_weight_max": 51.3877067565918,
      "activations/layer12_attention_weight_min": -29.992586135864258,
      "activations/layer13_attention_weight_max": 50.46025085449219,
      "activations/layer13_attention_weight_min": -36.01301193237305,
      "activations/layer14_attention_weight_max": 89.04232025146484,
      "activations/layer14_attention_weight_min": -57.155757904052734,
      "activations/layer15_attention_weight_max": 53.38513946533203,
      "activations/layer15_attention_weight_min": -35.47509002685547,
      "activations/layer16_attention_weight_max": 46.11267852783203,
      "activations/layer16_attention_weight_min": -34.36863708496094,
      "activations/layer17_attention_weight_max": 49.26752471923828,
      "activations/layer17_attention_weight_min": -28.83863067626953,
      "activations/layer18_attention_weight_max": 32.96255874633789,
      "activations/layer18_attention_weight_min": -20.889362335205078,
      "activations/layer19_attention_weight_max": 48.92798614501953,
      "activations/layer19_attention_weight_min": -29.668289184570312,
      "activations/layer1_attention_weight_max": 17.770734786987305,
      "activations/layer1_attention_weight_min": -13.356277465820312,
      "activations/layer20_attention_weight_max": 43.325897216796875,
      "activations/layer20_attention_weight_min": -23.901548385620117,
      "activations/layer21_attention_weight_max": 33.94623947143555,
      "activations/layer21_attention_weight_min": -18.253467559814453,
      "activations/layer22_attention_weight_max": 53.04250717163086,
      "activations/layer22_attention_weight_min": -27.56459617614746,
      "activations/layer23_attention_weight_max": 27.013500213623047,
      "activations/layer23_attention_weight_min": -17.647674560546875,
      "activations/layer2_attention_weight_max": 30.56769371032715,
      "activations/layer2_attention_weight_min": -31.12869644165039,
      "activations/layer3_attention_weight_max": 78.84335327148438,
      "activations/layer3_attention_weight_min": -82.32904815673828,
      "activations/layer4_attention_weight_max": 132.89549255371094,
      "activations/layer4_attention_weight_min": -126.82186126708984,
      "activations/layer5_attention_weight_max": 103.67034149169922,
      "activations/layer5_attention_weight_min": -90.57180786132812,
      "activations/layer6_attention_weight_max": 56.71330642700195,
      "activations/layer6_attention_weight_min": -52.09832000732422,
      "activations/layer7_attention_weight_max": 66.83985137939453,
      "activations/layer7_attention_weight_min": -65.86344909667969,
      "activations/layer8_attention_weight_max": 40.108985900878906,
      "activations/layer8_attention_weight_min": -37.27326202392578,
      "activations/layer9_attention_weight_max": 53.032745361328125,
      "activations/layer9_attention_weight_min": -38.68337631225586,
      "epoch": 2.77,
      "learning_rate": 0.0001334685606060606,
      "loss": 2.9214,
      "step": 47700
    },
    {
      "activations/layer0_attention_weight_max": 16.121009826660156,
      "activations/layer0_attention_weight_min": -11.558733940124512,
      "activations/layer10_attention_weight_max": 42.0876579284668,
      "activations/layer10_attention_weight_min": -36.597137451171875,
      "activations/layer11_attention_weight_max": 37.10553741455078,
      "activations/layer11_attention_weight_min": -34.28657531738281,
      "activations/layer12_attention_weight_max": 59.10671615600586,
      "activations/layer12_attention_weight_min": -40.151641845703125,
      "activations/layer13_attention_weight_max": 77.91802215576172,
      "activations/layer13_attention_weight_min": -54.627750396728516,
      "activations/layer14_attention_weight_max": 72.09413146972656,
      "activations/layer14_attention_weight_min": -53.10155487060547,
      "activations/layer15_attention_weight_max": 53.23818588256836,
      "activations/layer15_attention_weight_min": -33.90407943725586,
      "activations/layer16_attention_weight_max": 47.74403762817383,
      "activations/layer16_attention_weight_min": -31.62744140625,
      "activations/layer17_attention_weight_max": 46.407772064208984,
      "activations/layer17_attention_weight_min": -28.559871673583984,
      "activations/layer18_attention_weight_max": 34.7187614440918,
      "activations/layer18_attention_weight_min": -21.215869903564453,
      "activations/layer19_attention_weight_max": 48.02930450439453,
      "activations/layer19_attention_weight_min": -30.070850372314453,
      "activations/layer1_attention_weight_max": 19.12163543701172,
      "activations/layer1_attention_weight_min": -13.767134666442871,
      "activations/layer20_attention_weight_max": 38.8143310546875,
      "activations/layer20_attention_weight_min": -25.37383270263672,
      "activations/layer21_attention_weight_max": 35.4970588684082,
      "activations/layer21_attention_weight_min": -19.877201080322266,
      "activations/layer22_attention_weight_max": 53.37875747680664,
      "activations/layer22_attention_weight_min": -26.35012435913086,
      "activations/layer23_attention_weight_max": 30.94146728515625,
      "activations/layer23_attention_weight_min": -16.06093406677246,
      "activations/layer2_attention_weight_max": 30.016942977905273,
      "activations/layer2_attention_weight_min": -30.095069885253906,
      "activations/layer3_attention_weight_max": 78.39761352539062,
      "activations/layer3_attention_weight_min": -80.39728546142578,
      "activations/layer4_attention_weight_max": 126.9334945678711,
      "activations/layer4_attention_weight_min": -119.5262222290039,
      "activations/layer5_attention_weight_max": 111.87960815429688,
      "activations/layer5_attention_weight_min": -92.15829467773438,
      "activations/layer6_attention_weight_max": 62.81114196777344,
      "activations/layer6_attention_weight_min": -53.267337799072266,
      "activations/layer7_attention_weight_max": 68.73670196533203,
      "activations/layer7_attention_weight_min": -70.35481262207031,
      "activations/layer8_attention_weight_max": 40.699127197265625,
      "activations/layer8_attention_weight_min": -38.94035720825195,
      "activations/layer9_attention_weight_max": 53.65419387817383,
      "activations/layer9_attention_weight_min": -43.54574966430664,
      "epoch": 2.77,
      "learning_rate": 0.0001334496212121212,
      "loss": 2.9276,
      "step": 47750
    },
    {
      "activations/layer0_attention_weight_max": 17.290199279785156,
      "activations/layer0_attention_weight_min": -11.670928001403809,
      "activations/layer10_attention_weight_max": 45.16073226928711,
      "activations/layer10_attention_weight_min": -36.20053482055664,
      "activations/layer11_attention_weight_max": 40.01111602783203,
      "activations/layer11_attention_weight_min": -32.53734588623047,
      "activations/layer12_attention_weight_max": 36.245147705078125,
      "activations/layer12_attention_weight_min": -29.834529876708984,
      "activations/layer13_attention_weight_max": 53.745121002197266,
      "activations/layer13_attention_weight_min": -37.88804626464844,
      "activations/layer14_attention_weight_max": 69.45364379882812,
      "activations/layer14_attention_weight_min": -52.04254150390625,
      "activations/layer15_attention_weight_max": 49.440452575683594,
      "activations/layer15_attention_weight_min": -34.45094680786133,
      "activations/layer16_attention_weight_max": 49.67683410644531,
      "activations/layer16_attention_weight_min": -36.24443054199219,
      "activations/layer17_attention_weight_max": 47.495758056640625,
      "activations/layer17_attention_weight_min": -28.070362091064453,
      "activations/layer18_attention_weight_max": 27.69146156311035,
      "activations/layer18_attention_weight_min": -19.64409828186035,
      "activations/layer19_attention_weight_max": 50.77231979370117,
      "activations/layer19_attention_weight_min": -32.9612922668457,
      "activations/layer1_attention_weight_max": 17.170513153076172,
      "activations/layer1_attention_weight_min": -14.684228897094727,
      "activations/layer20_attention_weight_max": 35.01618957519531,
      "activations/layer20_attention_weight_min": -23.346818923950195,
      "activations/layer21_attention_weight_max": 33.619606018066406,
      "activations/layer21_attention_weight_min": -20.006258010864258,
      "activations/layer22_attention_weight_max": 49.50174331665039,
      "activations/layer22_attention_weight_min": -29.95301628112793,
      "activations/layer23_attention_weight_max": 27.958797454833984,
      "activations/layer23_attention_weight_min": -20.8829345703125,
      "activations/layer2_attention_weight_max": 30.256881713867188,
      "activations/layer2_attention_weight_min": -29.36191177368164,
      "activations/layer3_attention_weight_max": 75.6242904663086,
      "activations/layer3_attention_weight_min": -82.0765380859375,
      "activations/layer4_attention_weight_max": 131.48936462402344,
      "activations/layer4_attention_weight_min": -126.96974182128906,
      "activations/layer5_attention_weight_max": 104.9827880859375,
      "activations/layer5_attention_weight_min": -92.88388061523438,
      "activations/layer6_attention_weight_max": 59.891258239746094,
      "activations/layer6_attention_weight_min": -53.366703033447266,
      "activations/layer7_attention_weight_max": 72.81749725341797,
      "activations/layer7_attention_weight_min": -68.14611053466797,
      "activations/layer8_attention_weight_max": 41.6385498046875,
      "activations/layer8_attention_weight_min": -38.95193862915039,
      "activations/layer9_attention_weight_max": 56.71450424194336,
      "activations/layer9_attention_weight_min": -46.62562561035156,
      "epoch": 2.78,
      "learning_rate": 0.0001334306818181818,
      "loss": 2.9284,
      "step": 47800
    },
    {
      "activations/layer0_attention_weight_max": 16.63558006286621,
      "activations/layer0_attention_weight_min": -11.146496772766113,
      "activations/layer10_attention_weight_max": 42.489742279052734,
      "activations/layer10_attention_weight_min": -36.09429931640625,
      "activations/layer11_attention_weight_max": 38.626949310302734,
      "activations/layer11_attention_weight_min": -28.83095932006836,
      "activations/layer12_attention_weight_max": 31.043621063232422,
      "activations/layer12_attention_weight_min": -27.350122451782227,
      "activations/layer13_attention_weight_max": 45.68708038330078,
      "activations/layer13_attention_weight_min": -33.81443405151367,
      "activations/layer14_attention_weight_max": 71.98063659667969,
      "activations/layer14_attention_weight_min": -54.189144134521484,
      "activations/layer15_attention_weight_max": 47.791439056396484,
      "activations/layer15_attention_weight_min": -33.897151947021484,
      "activations/layer16_attention_weight_max": 51.5895881652832,
      "activations/layer16_attention_weight_min": -34.09961700439453,
      "activations/layer17_attention_weight_max": 46.48601150512695,
      "activations/layer17_attention_weight_min": -28.04944610595703,
      "activations/layer18_attention_weight_max": 29.14968490600586,
      "activations/layer18_attention_weight_min": -17.918981552124023,
      "activations/layer19_attention_weight_max": 47.58861541748047,
      "activations/layer19_attention_weight_min": -30.535287857055664,
      "activations/layer1_attention_weight_max": 17.17771339416504,
      "activations/layer1_attention_weight_min": -13.535877227783203,
      "activations/layer20_attention_weight_max": 42.68939971923828,
      "activations/layer20_attention_weight_min": -22.940471649169922,
      "activations/layer21_attention_weight_max": 38.440311431884766,
      "activations/layer21_attention_weight_min": -20.659820556640625,
      "activations/layer22_attention_weight_max": 48.59964370727539,
      "activations/layer22_attention_weight_min": -24.621532440185547,
      "activations/layer23_attention_weight_max": 26.848453521728516,
      "activations/layer23_attention_weight_min": -16.07227325439453,
      "activations/layer2_attention_weight_max": 29.363731384277344,
      "activations/layer2_attention_weight_min": -29.47100830078125,
      "activations/layer3_attention_weight_max": 77.92327117919922,
      "activations/layer3_attention_weight_min": -78.607421875,
      "activations/layer4_attention_weight_max": 131.51573181152344,
      "activations/layer4_attention_weight_min": -118.5962905883789,
      "activations/layer5_attention_weight_max": 110.03671264648438,
      "activations/layer5_attention_weight_min": -97.35494232177734,
      "activations/layer6_attention_weight_max": 59.8338737487793,
      "activations/layer6_attention_weight_min": -52.86219024658203,
      "activations/layer7_attention_weight_max": 71.3117446899414,
      "activations/layer7_attention_weight_min": -65.09175109863281,
      "activations/layer8_attention_weight_max": 40.603519439697266,
      "activations/layer8_attention_weight_min": -34.099361419677734,
      "activations/layer9_attention_weight_max": 58.29316329956055,
      "activations/layer9_attention_weight_min": -42.44355392456055,
      "epoch": 2.78,
      "learning_rate": 0.00013341174242424242,
      "loss": 2.9141,
      "step": 47850
    },
    {
      "activations/layer0_attention_weight_max": 17.213319778442383,
      "activations/layer0_attention_weight_min": -10.605690956115723,
      "activations/layer10_attention_weight_max": 48.24200439453125,
      "activations/layer10_attention_weight_min": -36.7165641784668,
      "activations/layer11_attention_weight_max": 40.463470458984375,
      "activations/layer11_attention_weight_min": -31.92339324951172,
      "activations/layer12_attention_weight_max": 36.69240188598633,
      "activations/layer12_attention_weight_min": -26.89214324951172,
      "activations/layer13_attention_weight_max": 57.71265411376953,
      "activations/layer13_attention_weight_min": -41.590248107910156,
      "activations/layer14_attention_weight_max": 87.32075500488281,
      "activations/layer14_attention_weight_min": -60.2696647644043,
      "activations/layer15_attention_weight_max": 49.38623809814453,
      "activations/layer15_attention_weight_min": -34.59994125366211,
      "activations/layer16_attention_weight_max": 46.2470588684082,
      "activations/layer16_attention_weight_min": -32.17544937133789,
      "activations/layer17_attention_weight_max": 48.91635513305664,
      "activations/layer17_attention_weight_min": -28.946815490722656,
      "activations/layer18_attention_weight_max": 40.67757034301758,
      "activations/layer18_attention_weight_min": -22.559335708618164,
      "activations/layer19_attention_weight_max": 48.1657829284668,
      "activations/layer19_attention_weight_min": -30.00050163269043,
      "activations/layer1_attention_weight_max": 17.086618423461914,
      "activations/layer1_attention_weight_min": -12.27025318145752,
      "activations/layer20_attention_weight_max": 40.885494232177734,
      "activations/layer20_attention_weight_min": -24.084598541259766,
      "activations/layer21_attention_weight_max": 38.65821075439453,
      "activations/layer21_attention_weight_min": -24.354761123657227,
      "activations/layer22_attention_weight_max": 48.207340240478516,
      "activations/layer22_attention_weight_min": -28.29558753967285,
      "activations/layer23_attention_weight_max": 25.609928131103516,
      "activations/layer23_attention_weight_min": -17.583847045898438,
      "activations/layer2_attention_weight_max": 30.659648895263672,
      "activations/layer2_attention_weight_min": -30.037921905517578,
      "activations/layer3_attention_weight_max": 78.32785034179688,
      "activations/layer3_attention_weight_min": -82.31554412841797,
      "activations/layer4_attention_weight_max": 136.4722137451172,
      "activations/layer4_attention_weight_min": -128.3274688720703,
      "activations/layer5_attention_weight_max": 115.42884826660156,
      "activations/layer5_attention_weight_min": -99.6817626953125,
      "activations/layer6_attention_weight_max": 64.25640869140625,
      "activations/layer6_attention_weight_min": -54.748844146728516,
      "activations/layer7_attention_weight_max": 72.60630798339844,
      "activations/layer7_attention_weight_min": -71.80110168457031,
      "activations/layer8_attention_weight_max": 46.551021575927734,
      "activations/layer8_attention_weight_min": -39.85314178466797,
      "activations/layer9_attention_weight_max": 58.39125442504883,
      "activations/layer9_attention_weight_min": -44.31538772583008,
      "epoch": 2.78,
      "learning_rate": 0.000133392803030303,
      "loss": 2.9086,
      "step": 47900
    },
    {
      "activations/layer0_attention_weight_max": 17.661985397338867,
      "activations/layer0_attention_weight_min": -10.961386680603027,
      "activations/layer10_attention_weight_max": 41.49835968017578,
      "activations/layer10_attention_weight_min": -33.840911865234375,
      "activations/layer11_attention_weight_max": 35.28107452392578,
      "activations/layer11_attention_weight_min": -31.255165100097656,
      "activations/layer12_attention_weight_max": 32.09257888793945,
      "activations/layer12_attention_weight_min": -28.098752975463867,
      "activations/layer13_attention_weight_max": 49.34929656982422,
      "activations/layer13_attention_weight_min": -37.10537338256836,
      "activations/layer14_attention_weight_max": 64.4052963256836,
      "activations/layer14_attention_weight_min": -54.0579948425293,
      "activations/layer15_attention_weight_max": 48.27206039428711,
      "activations/layer15_attention_weight_min": -35.458580017089844,
      "activations/layer16_attention_weight_max": 44.26474380493164,
      "activations/layer16_attention_weight_min": -32.257389068603516,
      "activations/layer17_attention_weight_max": 48.504756927490234,
      "activations/layer17_attention_weight_min": -30.02434730529785,
      "activations/layer18_attention_weight_max": 26.78316879272461,
      "activations/layer18_attention_weight_min": -19.296144485473633,
      "activations/layer19_attention_weight_max": 48.703182220458984,
      "activations/layer19_attention_weight_min": -32.67490768432617,
      "activations/layer1_attention_weight_max": 16.999677658081055,
      "activations/layer1_attention_weight_min": -13.935088157653809,
      "activations/layer20_attention_weight_max": 34.78050231933594,
      "activations/layer20_attention_weight_min": -26.03731346130371,
      "activations/layer21_attention_weight_max": 33.50557327270508,
      "activations/layer21_attention_weight_min": -24.900197982788086,
      "activations/layer22_attention_weight_max": 47.92475128173828,
      "activations/layer22_attention_weight_min": -24.914289474487305,
      "activations/layer23_attention_weight_max": 24.50728416442871,
      "activations/layer23_attention_weight_min": -17.836427688598633,
      "activations/layer2_attention_weight_max": 29.327449798583984,
      "activations/layer2_attention_weight_min": -30.568334579467773,
      "activations/layer3_attention_weight_max": 77.00761413574219,
      "activations/layer3_attention_weight_min": -82.71676635742188,
      "activations/layer4_attention_weight_max": 127.30645751953125,
      "activations/layer4_attention_weight_min": -119.79170989990234,
      "activations/layer5_attention_weight_max": 105.17234802246094,
      "activations/layer5_attention_weight_min": -90.33717346191406,
      "activations/layer6_attention_weight_max": 56.335609436035156,
      "activations/layer6_attention_weight_min": -52.69570541381836,
      "activations/layer7_attention_weight_max": 70.34337615966797,
      "activations/layer7_attention_weight_min": -65.9266128540039,
      "activations/layer8_attention_weight_max": 40.290374755859375,
      "activations/layer8_attention_weight_min": -35.5727653503418,
      "activations/layer9_attention_weight_max": 48.25768280029297,
      "activations/layer9_attention_weight_min": -38.1457405090332,
      "epoch": 2.79,
      "learning_rate": 0.00013337386363636362,
      "loss": 2.9447,
      "step": 47950
    },
    {
      "activations/layer0_attention_weight_max": 17.400650024414062,
      "activations/layer0_attention_weight_min": -11.902767181396484,
      "activations/layer10_attention_weight_max": 47.7068977355957,
      "activations/layer10_attention_weight_min": -34.126922607421875,
      "activations/layer11_attention_weight_max": 38.571983337402344,
      "activations/layer11_attention_weight_min": -30.513273239135742,
      "activations/layer12_attention_weight_max": 30.656078338623047,
      "activations/layer12_attention_weight_min": -29.72719955444336,
      "activations/layer13_attention_weight_max": 60.076725006103516,
      "activations/layer13_attention_weight_min": -39.185035705566406,
      "activations/layer14_attention_weight_max": 100.22731018066406,
      "activations/layer14_attention_weight_min": -63.07130432128906,
      "activations/layer15_attention_weight_max": 53.94495391845703,
      "activations/layer15_attention_weight_min": -32.704647064208984,
      "activations/layer16_attention_weight_max": 46.31575393676758,
      "activations/layer16_attention_weight_min": -31.15191650390625,
      "activations/layer17_attention_weight_max": 47.865028381347656,
      "activations/layer17_attention_weight_min": -29.26926040649414,
      "activations/layer18_attention_weight_max": 31.884977340698242,
      "activations/layer18_attention_weight_min": -21.734188079833984,
      "activations/layer19_attention_weight_max": 52.59824752807617,
      "activations/layer19_attention_weight_min": -31.006559371948242,
      "activations/layer1_attention_weight_max": 18.04586410522461,
      "activations/layer1_attention_weight_min": -14.810773849487305,
      "activations/layer20_attention_weight_max": 43.90666961669922,
      "activations/layer20_attention_weight_min": -23.835872650146484,
      "activations/layer21_attention_weight_max": 37.206756591796875,
      "activations/layer21_attention_weight_min": -23.436912536621094,
      "activations/layer22_attention_weight_max": 53.78460693359375,
      "activations/layer22_attention_weight_min": -28.815637588500977,
      "activations/layer23_attention_weight_max": 30.9943904876709,
      "activations/layer23_attention_weight_min": -18.377925872802734,
      "activations/layer2_attention_weight_max": 32.527488708496094,
      "activations/layer2_attention_weight_min": -30.025861740112305,
      "activations/layer3_attention_weight_max": 77.55717468261719,
      "activations/layer3_attention_weight_min": -81.93354034423828,
      "activations/layer4_attention_weight_max": 134.59507751464844,
      "activations/layer4_attention_weight_min": -128.90814208984375,
      "activations/layer5_attention_weight_max": 108.35218811035156,
      "activations/layer5_attention_weight_min": -100.2696762084961,
      "activations/layer6_attention_weight_max": 61.09463882446289,
      "activations/layer6_attention_weight_min": -55.07007598876953,
      "activations/layer7_attention_weight_max": 72.41401672363281,
      "activations/layer7_attention_weight_min": -66.03343200683594,
      "activations/layer8_attention_weight_max": 46.35807418823242,
      "activations/layer8_attention_weight_min": -36.76512908935547,
      "activations/layer9_attention_weight_max": 54.63779830932617,
      "activations/layer9_attention_weight_min": -39.35999298095703,
      "epoch": 2.79,
      "learning_rate": 0.00013335492424242424,
      "loss": 2.9228,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_loss": 2.869140625,
      "eval_runtime": 8.5205,
      "eval_samples_per_second": 503.959,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_openwebtext_loss": 2.869140625,
      "eval_openwebtext_ppl": 17.6218678981495,
      "eval_openwebtext_runtime": 8.5205,
      "eval_openwebtext_samples_per_second": 503.959,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_wikitext_loss": 3.13671875,
      "eval_wikitext_ppl": 23.028181534732802,
      "eval_wikitext_runtime": 2.0017,
      "eval_wikitext_samples_per_second": 227.81,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_lambada_loss": 2.978515625,
      "eval_lambada_ppl": 19.658614220966218,
      "eval_lambada_runtime": 9.6099,
      "eval_lambada_samples_per_second": 506.667,
      "step": 48000
    },
    {
      "activations/layer0_attention_weight_max": 17.022968292236328,
      "activations/layer0_attention_weight_min": -11.761265754699707,
      "activations/layer10_attention_weight_max": 50.81427764892578,
      "activations/layer10_attention_weight_min": -42.06203079223633,
      "activations/layer11_attention_weight_max": 43.86954879760742,
      "activations/layer11_attention_weight_min": -33.59307098388672,
      "activations/layer12_attention_weight_max": 29.187326431274414,
      "activations/layer12_attention_weight_min": -26.706483840942383,
      "activations/layer13_attention_weight_max": 46.692893981933594,
      "activations/layer13_attention_weight_min": -38.32123947143555,
      "activations/layer14_attention_weight_max": 74.23177337646484,
      "activations/layer14_attention_weight_min": -59.714195251464844,
      "activations/layer15_attention_weight_max": 51.12254333496094,
      "activations/layer15_attention_weight_min": -32.498470306396484,
      "activations/layer16_attention_weight_max": 43.91962814331055,
      "activations/layer16_attention_weight_min": -30.965713500976562,
      "activations/layer17_attention_weight_max": 40.73503112792969,
      "activations/layer17_attention_weight_min": -29.504066467285156,
      "activations/layer18_attention_weight_max": 28.11497688293457,
      "activations/layer18_attention_weight_min": -18.77016830444336,
      "activations/layer19_attention_weight_max": 45.289424896240234,
      "activations/layer19_attention_weight_min": -30.409299850463867,
      "activations/layer1_attention_weight_max": 17.74225425720215,
      "activations/layer1_attention_weight_min": -12.99978256225586,
      "activations/layer20_attention_weight_max": 34.82350540161133,
      "activations/layer20_attention_weight_min": -22.817302703857422,
      "activations/layer21_attention_weight_max": 31.003801345825195,
      "activations/layer21_attention_weight_min": -19.742055892944336,
      "activations/layer22_attention_weight_max": 47.705078125,
      "activations/layer22_attention_weight_min": -32.066802978515625,
      "activations/layer23_attention_weight_max": 27.423385620117188,
      "activations/layer23_attention_weight_min": -19.331031799316406,
      "activations/layer2_attention_weight_max": 29.421499252319336,
      "activations/layer2_attention_weight_min": -30.65632438659668,
      "activations/layer3_attention_weight_max": 78.44435119628906,
      "activations/layer3_attention_weight_min": -80.50402069091797,
      "activations/layer4_attention_weight_max": 129.08103942871094,
      "activations/layer4_attention_weight_min": -123.9712905883789,
      "activations/layer5_attention_weight_max": 106.302490234375,
      "activations/layer5_attention_weight_min": -95.65516662597656,
      "activations/layer6_attention_weight_max": 59.405521392822266,
      "activations/layer6_attention_weight_min": -50.84083557128906,
      "activations/layer7_attention_weight_max": 75.15986633300781,
      "activations/layer7_attention_weight_min": -72.20010375976562,
      "activations/layer8_attention_weight_max": 45.09111404418945,
      "activations/layer8_attention_weight_min": -38.63481140136719,
      "activations/layer9_attention_weight_max": 63.101234436035156,
      "activations/layer9_attention_weight_min": -52.2694091796875,
      "epoch": 2.79,
      "learning_rate": 0.00013333598484848485,
      "loss": 2.9112,
      "step": 48050
    },
    {
      "activations/layer0_attention_weight_max": 16.60399627685547,
      "activations/layer0_attention_weight_min": -11.172592163085938,
      "activations/layer10_attention_weight_max": 43.49882507324219,
      "activations/layer10_attention_weight_min": -35.5440673828125,
      "activations/layer11_attention_weight_max": 38.76064682006836,
      "activations/layer11_attention_weight_min": -31.69441795349121,
      "activations/layer12_attention_weight_max": 39.31304931640625,
      "activations/layer12_attention_weight_min": -28.368410110473633,
      "activations/layer13_attention_weight_max": 46.85665512084961,
      "activations/layer13_attention_weight_min": -36.2515754699707,
      "activations/layer14_attention_weight_max": 69.29362487792969,
      "activations/layer14_attention_weight_min": -47.649349212646484,
      "activations/layer15_attention_weight_max": 48.06892776489258,
      "activations/layer15_attention_weight_min": -33.73420333862305,
      "activations/layer16_attention_weight_max": 43.345829010009766,
      "activations/layer16_attention_weight_min": -32.79066467285156,
      "activations/layer17_attention_weight_max": 43.114131927490234,
      "activations/layer17_attention_weight_min": -27.52297019958496,
      "activations/layer18_attention_weight_max": 26.359041213989258,
      "activations/layer18_attention_weight_min": -19.733015060424805,
      "activations/layer19_attention_weight_max": 41.45231628417969,
      "activations/layer19_attention_weight_min": -31.159626007080078,
      "activations/layer1_attention_weight_max": 18.217161178588867,
      "activations/layer1_attention_weight_min": -14.09311294555664,
      "activations/layer20_attention_weight_max": 37.43240737915039,
      "activations/layer20_attention_weight_min": -23.790666580200195,
      "activations/layer21_attention_weight_max": 29.536026000976562,
      "activations/layer21_attention_weight_min": -20.383930206298828,
      "activations/layer22_attention_weight_max": 50.11258316040039,
      "activations/layer22_attention_weight_min": -26.74783706665039,
      "activations/layer23_attention_weight_max": 27.035018920898438,
      "activations/layer23_attention_weight_min": -17.007923126220703,
      "activations/layer2_attention_weight_max": 28.350326538085938,
      "activations/layer2_attention_weight_min": -28.31703758239746,
      "activations/layer3_attention_weight_max": 75.53770446777344,
      "activations/layer3_attention_weight_min": -77.54965209960938,
      "activations/layer4_attention_weight_max": 121.6231460571289,
      "activations/layer4_attention_weight_min": -120.1469497680664,
      "activations/layer5_attention_weight_max": 97.57476806640625,
      "activations/layer5_attention_weight_min": -90.61788177490234,
      "activations/layer6_attention_weight_max": 54.23678970336914,
      "activations/layer6_attention_weight_min": -49.63343811035156,
      "activations/layer7_attention_weight_max": 70.04409790039062,
      "activations/layer7_attention_weight_min": -67.3086166381836,
      "activations/layer8_attention_weight_max": 42.43112564086914,
      "activations/layer8_attention_weight_min": -35.047088623046875,
      "activations/layer9_attention_weight_max": 52.2935676574707,
      "activations/layer9_attention_weight_min": -43.24885177612305,
      "epoch": 2.79,
      "learning_rate": 0.00013331704545454544,
      "loss": 2.923,
      "step": 48100
    },
    {
      "activations/layer0_attention_weight_max": 17.13003158569336,
      "activations/layer0_attention_weight_min": -11.364869117736816,
      "activations/layer10_attention_weight_max": 43.76128005981445,
      "activations/layer10_attention_weight_min": -33.373985290527344,
      "activations/layer11_attention_weight_max": 35.22016906738281,
      "activations/layer11_attention_weight_min": -30.206893920898438,
      "activations/layer12_attention_weight_max": 39.82260513305664,
      "activations/layer12_attention_weight_min": -28.21111488342285,
      "activations/layer13_attention_weight_max": 48.75438690185547,
      "activations/layer13_attention_weight_min": -37.97431945800781,
      "activations/layer14_attention_weight_max": 84.21768188476562,
      "activations/layer14_attention_weight_min": -63.079673767089844,
      "activations/layer15_attention_weight_max": 48.43379592895508,
      "activations/layer15_attention_weight_min": -34.98138427734375,
      "activations/layer16_attention_weight_max": 45.27288055419922,
      "activations/layer16_attention_weight_min": -34.07141876220703,
      "activations/layer17_attention_weight_max": 43.71818923950195,
      "activations/layer17_attention_weight_min": -28.278987884521484,
      "activations/layer18_attention_weight_max": 31.813016891479492,
      "activations/layer18_attention_weight_min": -19.355703353881836,
      "activations/layer19_attention_weight_max": 47.1392936706543,
      "activations/layer19_attention_weight_min": -35.89468765258789,
      "activations/layer1_attention_weight_max": 16.943710327148438,
      "activations/layer1_attention_weight_min": -11.803221702575684,
      "activations/layer20_attention_weight_max": 38.12952423095703,
      "activations/layer20_attention_weight_min": -23.72587776184082,
      "activations/layer21_attention_weight_max": 31.47581672668457,
      "activations/layer21_attention_weight_min": -18.90203094482422,
      "activations/layer22_attention_weight_max": 47.13241958618164,
      "activations/layer22_attention_weight_min": -29.79116439819336,
      "activations/layer23_attention_weight_max": 27.307044982910156,
      "activations/layer23_attention_weight_min": -17.931867599487305,
      "activations/layer2_attention_weight_max": 29.48062515258789,
      "activations/layer2_attention_weight_min": -29.21993637084961,
      "activations/layer3_attention_weight_max": 76.323486328125,
      "activations/layer3_attention_weight_min": -75.4903335571289,
      "activations/layer4_attention_weight_max": 126.26371002197266,
      "activations/layer4_attention_weight_min": -121.83788299560547,
      "activations/layer5_attention_weight_max": 103.75169372558594,
      "activations/layer5_attention_weight_min": -91.43121337890625,
      "activations/layer6_attention_weight_max": 57.90701675415039,
      "activations/layer6_attention_weight_min": -53.81834030151367,
      "activations/layer7_attention_weight_max": 73.13562774658203,
      "activations/layer7_attention_weight_min": -69.5566635131836,
      "activations/layer8_attention_weight_max": 43.867591857910156,
      "activations/layer8_attention_weight_min": -37.5482292175293,
      "activations/layer9_attention_weight_max": 52.66477584838867,
      "activations/layer9_attention_weight_min": -46.516326904296875,
      "epoch": 2.8,
      "learning_rate": 0.00013329810606060605,
      "loss": 2.9191,
      "step": 48150
    },
    {
      "activations/layer0_attention_weight_max": 15.102855682373047,
      "activations/layer0_attention_weight_min": -11.294703483581543,
      "activations/layer10_attention_weight_max": 47.02142333984375,
      "activations/layer10_attention_weight_min": -33.25639724731445,
      "activations/layer11_attention_weight_max": 44.17139434814453,
      "activations/layer11_attention_weight_min": -32.76477813720703,
      "activations/layer12_attention_weight_max": 81.44896697998047,
      "activations/layer12_attention_weight_min": -32.87990188598633,
      "activations/layer13_attention_weight_max": 72.90521240234375,
      "activations/layer13_attention_weight_min": -39.85091018676758,
      "activations/layer14_attention_weight_max": 88.82535552978516,
      "activations/layer14_attention_weight_min": -53.35254669189453,
      "activations/layer15_attention_weight_max": 60.641204833984375,
      "activations/layer15_attention_weight_min": -32.49503707885742,
      "activations/layer16_attention_weight_max": 53.42790985107422,
      "activations/layer16_attention_weight_min": -34.31669998168945,
      "activations/layer17_attention_weight_max": 54.608070373535156,
      "activations/layer17_attention_weight_min": -29.657569885253906,
      "activations/layer18_attention_weight_max": 52.94382095336914,
      "activations/layer18_attention_weight_min": -22.55246353149414,
      "activations/layer19_attention_weight_max": 51.168758392333984,
      "activations/layer19_attention_weight_min": -30.1776180267334,
      "activations/layer1_attention_weight_max": 17.84272575378418,
      "activations/layer1_attention_weight_min": -13.908637046813965,
      "activations/layer20_attention_weight_max": 40.505977630615234,
      "activations/layer20_attention_weight_min": -25.028087615966797,
      "activations/layer21_attention_weight_max": 31.273008346557617,
      "activations/layer21_attention_weight_min": -22.32882308959961,
      "activations/layer22_attention_weight_max": 55.16811752319336,
      "activations/layer22_attention_weight_min": -31.50464630126953,
      "activations/layer23_attention_weight_max": 28.491621017456055,
      "activations/layer23_attention_weight_min": -18.478267669677734,
      "activations/layer2_attention_weight_max": 31.247394561767578,
      "activations/layer2_attention_weight_min": -30.176631927490234,
      "activations/layer3_attention_weight_max": 77.66069793701172,
      "activations/layer3_attention_weight_min": -81.21923065185547,
      "activations/layer4_attention_weight_max": 130.71820068359375,
      "activations/layer4_attention_weight_min": -128.6702880859375,
      "activations/layer5_attention_weight_max": 113.11909484863281,
      "activations/layer5_attention_weight_min": -102.68798828125,
      "activations/layer6_attention_weight_max": 60.97328186035156,
      "activations/layer6_attention_weight_min": -57.163028717041016,
      "activations/layer7_attention_weight_max": 72.78153228759766,
      "activations/layer7_attention_weight_min": -68.79142761230469,
      "activations/layer8_attention_weight_max": 41.82681655883789,
      "activations/layer8_attention_weight_min": -38.91547393798828,
      "activations/layer9_attention_weight_max": 53.21120071411133,
      "activations/layer9_attention_weight_min": -39.5489387512207,
      "epoch": 2.8,
      "learning_rate": 0.00013327916666666667,
      "loss": 2.9031,
      "step": 48200
    },
    {
      "activations/layer0_attention_weight_max": 16.524797439575195,
      "activations/layer0_attention_weight_min": -11.139309883117676,
      "activations/layer10_attention_weight_max": 44.706153869628906,
      "activations/layer10_attention_weight_min": -35.33647918701172,
      "activations/layer11_attention_weight_max": 40.99866485595703,
      "activations/layer11_attention_weight_min": -30.793678283691406,
      "activations/layer12_attention_weight_max": 34.57919692993164,
      "activations/layer12_attention_weight_min": -25.408340454101562,
      "activations/layer13_attention_weight_max": 51.36157989501953,
      "activations/layer13_attention_weight_min": -38.14908981323242,
      "activations/layer14_attention_weight_max": 77.24323272705078,
      "activations/layer14_attention_weight_min": -54.332191467285156,
      "activations/layer15_attention_weight_max": 53.63197326660156,
      "activations/layer15_attention_weight_min": -35.664710998535156,
      "activations/layer16_attention_weight_max": 47.7535285949707,
      "activations/layer16_attention_weight_min": -34.86937713623047,
      "activations/layer17_attention_weight_max": 45.49534225463867,
      "activations/layer17_attention_weight_min": -27.969655990600586,
      "activations/layer18_attention_weight_max": 31.688194274902344,
      "activations/layer18_attention_weight_min": -21.447187423706055,
      "activations/layer19_attention_weight_max": 53.389930725097656,
      "activations/layer19_attention_weight_min": -30.3802490234375,
      "activations/layer1_attention_weight_max": 17.820701599121094,
      "activations/layer1_attention_weight_min": -13.636256217956543,
      "activations/layer20_attention_weight_max": 37.56650161743164,
      "activations/layer20_attention_weight_min": -26.1868839263916,
      "activations/layer21_attention_weight_max": 38.305885314941406,
      "activations/layer21_attention_weight_min": -22.940710067749023,
      "activations/layer22_attention_weight_max": 49.92324447631836,
      "activations/layer22_attention_weight_min": -31.04174041748047,
      "activations/layer23_attention_weight_max": 29.47272491455078,
      "activations/layer23_attention_weight_min": -18.245885848999023,
      "activations/layer2_attention_weight_max": 29.039947509765625,
      "activations/layer2_attention_weight_min": -28.659589767456055,
      "activations/layer3_attention_weight_max": 79.97913360595703,
      "activations/layer3_attention_weight_min": -79.96585083007812,
      "activations/layer4_attention_weight_max": 129.14120483398438,
      "activations/layer4_attention_weight_min": -124.072021484375,
      "activations/layer5_attention_weight_max": 104.95137786865234,
      "activations/layer5_attention_weight_min": -93.81756591796875,
      "activations/layer6_attention_weight_max": 58.5002555847168,
      "activations/layer6_attention_weight_min": -55.0155029296875,
      "activations/layer7_attention_weight_max": 67.74320220947266,
      "activations/layer7_attention_weight_min": -66.4448471069336,
      "activations/layer8_attention_weight_max": 40.45863342285156,
      "activations/layer8_attention_weight_min": -36.25358963012695,
      "activations/layer9_attention_weight_max": 52.87885284423828,
      "activations/layer9_attention_weight_min": -42.233009338378906,
      "epoch": 2.8,
      "learning_rate": 0.00013326022727272726,
      "loss": 2.9192,
      "step": 48250
    },
    {
      "activations/layer0_attention_weight_max": 15.942157745361328,
      "activations/layer0_attention_weight_min": -11.3125,
      "activations/layer10_attention_weight_max": 57.55889129638672,
      "activations/layer10_attention_weight_min": -38.72968673706055,
      "activations/layer11_attention_weight_max": 44.94978332519531,
      "activations/layer11_attention_weight_min": -29.054288864135742,
      "activations/layer12_attention_weight_max": 75.3198471069336,
      "activations/layer12_attention_weight_min": -38.473018646240234,
      "activations/layer13_attention_weight_max": 59.40614700317383,
      "activations/layer13_attention_weight_min": -37.373497009277344,
      "activations/layer14_attention_weight_max": 80.58566284179688,
      "activations/layer14_attention_weight_min": -57.902732849121094,
      "activations/layer15_attention_weight_max": 55.250064849853516,
      "activations/layer15_attention_weight_min": -35.44108200073242,
      "activations/layer16_attention_weight_max": 54.47732925415039,
      "activations/layer16_attention_weight_min": -34.002906799316406,
      "activations/layer17_attention_weight_max": 43.7986946105957,
      "activations/layer17_attention_weight_min": -26.967939376831055,
      "activations/layer18_attention_weight_max": 28.127094268798828,
      "activations/layer18_attention_weight_min": -17.42676544189453,
      "activations/layer19_attention_weight_max": 55.42193603515625,
      "activations/layer19_attention_weight_min": -33.145469665527344,
      "activations/layer1_attention_weight_max": 17.50957679748535,
      "activations/layer1_attention_weight_min": -12.472494125366211,
      "activations/layer20_attention_weight_max": 36.434085845947266,
      "activations/layer20_attention_weight_min": -28.10113525390625,
      "activations/layer21_attention_weight_max": 29.708288192749023,
      "activations/layer21_attention_weight_min": -18.28927993774414,
      "activations/layer22_attention_weight_max": 50.15195083618164,
      "activations/layer22_attention_weight_min": -31.009416580200195,
      "activations/layer23_attention_weight_max": 27.66170310974121,
      "activations/layer23_attention_weight_min": -16.693035125732422,
      "activations/layer2_attention_weight_max": 29.411405563354492,
      "activations/layer2_attention_weight_min": -29.598228454589844,
      "activations/layer3_attention_weight_max": 77.27770233154297,
      "activations/layer3_attention_weight_min": -80.63764953613281,
      "activations/layer4_attention_weight_max": 132.84710693359375,
      "activations/layer4_attention_weight_min": -124.2925796508789,
      "activations/layer5_attention_weight_max": 109.11502075195312,
      "activations/layer5_attention_weight_min": -93.9593734741211,
      "activations/layer6_attention_weight_max": 60.27473068237305,
      "activations/layer6_attention_weight_min": -52.496910095214844,
      "activations/layer7_attention_weight_max": 73.0844497680664,
      "activations/layer7_attention_weight_min": -69.74699401855469,
      "activations/layer8_attention_weight_max": 43.5205192565918,
      "activations/layer8_attention_weight_min": -36.26132583618164,
      "activations/layer9_attention_weight_max": 72.33989715576172,
      "activations/layer9_attention_weight_min": -52.8149299621582,
      "epoch": 2.81,
      "learning_rate": 0.00013324128787878787,
      "loss": 2.9077,
      "step": 48300
    },
    {
      "activations/layer0_attention_weight_max": 17.467052459716797,
      "activations/layer0_attention_weight_min": -12.174991607666016,
      "activations/layer10_attention_weight_max": 41.8261833190918,
      "activations/layer10_attention_weight_min": -33.51251983642578,
      "activations/layer11_attention_weight_max": 33.16286087036133,
      "activations/layer11_attention_weight_min": -28.97181510925293,
      "activations/layer12_attention_weight_max": 40.98601531982422,
      "activations/layer12_attention_weight_min": -29.417024612426758,
      "activations/layer13_attention_weight_max": 44.21238327026367,
      "activations/layer13_attention_weight_min": -33.72015380859375,
      "activations/layer14_attention_weight_max": 76.56778717041016,
      "activations/layer14_attention_weight_min": -56.10000991821289,
      "activations/layer15_attention_weight_max": 42.293766021728516,
      "activations/layer15_attention_weight_min": -35.46955108642578,
      "activations/layer16_attention_weight_max": 43.05674743652344,
      "activations/layer16_attention_weight_min": -30.011808395385742,
      "activations/layer17_attention_weight_max": 43.62067413330078,
      "activations/layer17_attention_weight_min": -30.083473205566406,
      "activations/layer18_attention_weight_max": 27.76211166381836,
      "activations/layer18_attention_weight_min": -17.92011833190918,
      "activations/layer19_attention_weight_max": 47.46967697143555,
      "activations/layer19_attention_weight_min": -28.772626876831055,
      "activations/layer1_attention_weight_max": 17.376232147216797,
      "activations/layer1_attention_weight_min": -12.653189659118652,
      "activations/layer20_attention_weight_max": 28.93660545349121,
      "activations/layer20_attention_weight_min": -22.657976150512695,
      "activations/layer21_attention_weight_max": 30.09694480895996,
      "activations/layer21_attention_weight_min": -21.208993911743164,
      "activations/layer22_attention_weight_max": 39.621009826660156,
      "activations/layer22_attention_weight_min": -27.06757926940918,
      "activations/layer23_attention_weight_max": 24.11574363708496,
      "activations/layer23_attention_weight_min": -17.79523468017578,
      "activations/layer2_attention_weight_max": 29.201663970947266,
      "activations/layer2_attention_weight_min": -29.661481857299805,
      "activations/layer3_attention_weight_max": 75.97850036621094,
      "activations/layer3_attention_weight_min": -78.3394546508789,
      "activations/layer4_attention_weight_max": 125.86498260498047,
      "activations/layer4_attention_weight_min": -127.80006408691406,
      "activations/layer5_attention_weight_max": 102.5833969116211,
      "activations/layer5_attention_weight_min": -91.04847717285156,
      "activations/layer6_attention_weight_max": 58.309906005859375,
      "activations/layer6_attention_weight_min": -54.0502815246582,
      "activations/layer7_attention_weight_max": 67.07268524169922,
      "activations/layer7_attention_weight_min": -66.3719482421875,
      "activations/layer8_attention_weight_max": 41.832645416259766,
      "activations/layer8_attention_weight_min": -37.204166412353516,
      "activations/layer9_attention_weight_max": 46.89787292480469,
      "activations/layer9_attention_weight_min": -39.061317443847656,
      "epoch": 2.81,
      "learning_rate": 0.0001332223484848485,
      "loss": 2.9216,
      "step": 48350
    },
    {
      "activations/layer0_attention_weight_max": 17.54416275024414,
      "activations/layer0_attention_weight_min": -11.21335506439209,
      "activations/layer10_attention_weight_max": 47.04926300048828,
      "activations/layer10_attention_weight_min": -38.52595901489258,
      "activations/layer11_attention_weight_max": 36.84186935424805,
      "activations/layer11_attention_weight_min": -32.32626724243164,
      "activations/layer12_attention_weight_max": 36.76823806762695,
      "activations/layer12_attention_weight_min": -28.173070907592773,
      "activations/layer13_attention_weight_max": 47.1842155456543,
      "activations/layer13_attention_weight_min": -33.9402961730957,
      "activations/layer14_attention_weight_max": 69.5857925415039,
      "activations/layer14_attention_weight_min": -50.846954345703125,
      "activations/layer15_attention_weight_max": 44.05888366699219,
      "activations/layer15_attention_weight_min": -33.071136474609375,
      "activations/layer16_attention_weight_max": 45.327117919921875,
      "activations/layer16_attention_weight_min": -34.22735595703125,
      "activations/layer17_attention_weight_max": 47.07954788208008,
      "activations/layer17_attention_weight_min": -29.165279388427734,
      "activations/layer18_attention_weight_max": 31.000871658325195,
      "activations/layer18_attention_weight_min": -21.597944259643555,
      "activations/layer19_attention_weight_max": 45.22102355957031,
      "activations/layer19_attention_weight_min": -30.87237548828125,
      "activations/layer1_attention_weight_max": 18.61578369140625,
      "activations/layer1_attention_weight_min": -12.47384262084961,
      "activations/layer20_attention_weight_max": 36.17856979370117,
      "activations/layer20_attention_weight_min": -26.888370513916016,
      "activations/layer21_attention_weight_max": 35.21086502075195,
      "activations/layer21_attention_weight_min": -23.0471134185791,
      "activations/layer22_attention_weight_max": 50.82694625854492,
      "activations/layer22_attention_weight_min": -27.769380569458008,
      "activations/layer23_attention_weight_max": 28.705015182495117,
      "activations/layer23_attention_weight_min": -16.825437545776367,
      "activations/layer2_attention_weight_max": 29.159299850463867,
      "activations/layer2_attention_weight_min": -29.589414596557617,
      "activations/layer3_attention_weight_max": 76.97901153564453,
      "activations/layer3_attention_weight_min": -80.24625396728516,
      "activations/layer4_attention_weight_max": 125.9496078491211,
      "activations/layer4_attention_weight_min": -117.0186538696289,
      "activations/layer5_attention_weight_max": 100.91217041015625,
      "activations/layer5_attention_weight_min": -88.96752166748047,
      "activations/layer6_attention_weight_max": 58.064151763916016,
      "activations/layer6_attention_weight_min": -49.7130012512207,
      "activations/layer7_attention_weight_max": 72.8396224975586,
      "activations/layer7_attention_weight_min": -65.71041870117188,
      "activations/layer8_attention_weight_max": 42.75978469848633,
      "activations/layer8_attention_weight_min": -35.10809326171875,
      "activations/layer9_attention_weight_max": 54.20648956298828,
      "activations/layer9_attention_weight_min": -47.82735061645508,
      "epoch": 2.81,
      "learning_rate": 0.00013320340909090908,
      "loss": 2.9248,
      "step": 48400
    },
    {
      "activations/layer0_attention_weight_max": 17.810989379882812,
      "activations/layer0_attention_weight_min": -11.717135429382324,
      "activations/layer10_attention_weight_max": 44.48381805419922,
      "activations/layer10_attention_weight_min": -37.280914306640625,
      "activations/layer11_attention_weight_max": 39.62080383300781,
      "activations/layer11_attention_weight_min": -31.592754364013672,
      "activations/layer12_attention_weight_max": 35.4089241027832,
      "activations/layer12_attention_weight_min": -23.80061149597168,
      "activations/layer13_attention_weight_max": 56.26079177856445,
      "activations/layer13_attention_weight_min": -40.0399284362793,
      "activations/layer14_attention_weight_max": 83.36851501464844,
      "activations/layer14_attention_weight_min": -61.51984786987305,
      "activations/layer15_attention_weight_max": 50.8491325378418,
      "activations/layer15_attention_weight_min": -31.792831420898438,
      "activations/layer16_attention_weight_max": 48.056156158447266,
      "activations/layer16_attention_weight_min": -31.420364379882812,
      "activations/layer17_attention_weight_max": 48.08211898803711,
      "activations/layer17_attention_weight_min": -29.749746322631836,
      "activations/layer18_attention_weight_max": 32.721656799316406,
      "activations/layer18_attention_weight_min": -19.08701515197754,
      "activations/layer19_attention_weight_max": 47.09273147583008,
      "activations/layer19_attention_weight_min": -30.457300186157227,
      "activations/layer1_attention_weight_max": 17.472518920898438,
      "activations/layer1_attention_weight_min": -13.510071754455566,
      "activations/layer20_attention_weight_max": 36.105308532714844,
      "activations/layer20_attention_weight_min": -27.12956428527832,
      "activations/layer21_attention_weight_max": 30.83919906616211,
      "activations/layer21_attention_weight_min": -21.680185317993164,
      "activations/layer22_attention_weight_max": 63.25541305541992,
      "activations/layer22_attention_weight_min": -27.575145721435547,
      "activations/layer23_attention_weight_max": 27.79904556274414,
      "activations/layer23_attention_weight_min": -18.042604446411133,
      "activations/layer2_attention_weight_max": 30.979488372802734,
      "activations/layer2_attention_weight_min": -30.007726669311523,
      "activations/layer3_attention_weight_max": 78.53031921386719,
      "activations/layer3_attention_weight_min": -78.8719253540039,
      "activations/layer4_attention_weight_max": 126.58446502685547,
      "activations/layer4_attention_weight_min": -122.4618911743164,
      "activations/layer5_attention_weight_max": 106.86956787109375,
      "activations/layer5_attention_weight_min": -93.2693099975586,
      "activations/layer6_attention_weight_max": 61.8408317565918,
      "activations/layer6_attention_weight_min": -52.7499885559082,
      "activations/layer7_attention_weight_max": 71.8592758178711,
      "activations/layer7_attention_weight_min": -65.10588836669922,
      "activations/layer8_attention_weight_max": 42.51741409301758,
      "activations/layer8_attention_weight_min": -36.650108337402344,
      "activations/layer9_attention_weight_max": 55.829708099365234,
      "activations/layer9_attention_weight_min": -39.00656509399414,
      "epoch": 2.82,
      "learning_rate": 0.0001331844696969697,
      "loss": 2.9155,
      "step": 48450
    },
    {
      "activations/layer0_attention_weight_max": 17.432153701782227,
      "activations/layer0_attention_weight_min": -11.235333442687988,
      "activations/layer10_attention_weight_max": 49.62984085083008,
      "activations/layer10_attention_weight_min": -40.60805130004883,
      "activations/layer11_attention_weight_max": 39.548187255859375,
      "activations/layer11_attention_weight_min": -38.52616500854492,
      "activations/layer12_attention_weight_max": 50.77760696411133,
      "activations/layer12_attention_weight_min": -36.390907287597656,
      "activations/layer13_attention_weight_max": 48.63198471069336,
      "activations/layer13_attention_weight_min": -40.94205856323242,
      "activations/layer14_attention_weight_max": 67.64856719970703,
      "activations/layer14_attention_weight_min": -53.487247467041016,
      "activations/layer15_attention_weight_max": 49.208274841308594,
      "activations/layer15_attention_weight_min": -32.90467834472656,
      "activations/layer16_attention_weight_max": 44.05060958862305,
      "activations/layer16_attention_weight_min": -32.94877624511719,
      "activations/layer17_attention_weight_max": 49.0357551574707,
      "activations/layer17_attention_weight_min": -28.0146541595459,
      "activations/layer18_attention_weight_max": 32.05656051635742,
      "activations/layer18_attention_weight_min": -20.055255889892578,
      "activations/layer19_attention_weight_max": 47.162086486816406,
      "activations/layer19_attention_weight_min": -30.4679012298584,
      "activations/layer1_attention_weight_max": 18.295795440673828,
      "activations/layer1_attention_weight_min": -12.352763175964355,
      "activations/layer20_attention_weight_max": 39.27476501464844,
      "activations/layer20_attention_weight_min": -22.619434356689453,
      "activations/layer21_attention_weight_max": 37.195343017578125,
      "activations/layer21_attention_weight_min": -22.95853614807129,
      "activations/layer22_attention_weight_max": 52.39426040649414,
      "activations/layer22_attention_weight_min": -26.915267944335938,
      "activations/layer23_attention_weight_max": 27.056224822998047,
      "activations/layer23_attention_weight_min": -17.63106918334961,
      "activations/layer2_attention_weight_max": 30.88631248474121,
      "activations/layer2_attention_weight_min": -29.6766414642334,
      "activations/layer3_attention_weight_max": 79.7366714477539,
      "activations/layer3_attention_weight_min": -83.03919219970703,
      "activations/layer4_attention_weight_max": 132.8386688232422,
      "activations/layer4_attention_weight_min": -123.18094635009766,
      "activations/layer5_attention_weight_max": 102.12237548828125,
      "activations/layer5_attention_weight_min": -96.0639877319336,
      "activations/layer6_attention_weight_max": 58.293766021728516,
      "activations/layer6_attention_weight_min": -53.179893493652344,
      "activations/layer7_attention_weight_max": 78.3785171508789,
      "activations/layer7_attention_weight_min": -74.51592254638672,
      "activations/layer8_attention_weight_max": 41.566776275634766,
      "activations/layer8_attention_weight_min": -38.476932525634766,
      "activations/layer9_attention_weight_max": 54.9231071472168,
      "activations/layer9_attention_weight_min": -57.30751419067383,
      "epoch": 2.82,
      "learning_rate": 0.0001331655303030303,
      "loss": 2.9277,
      "step": 48500
    },
    {
      "activations/layer0_attention_weight_max": 17.32025146484375,
      "activations/layer0_attention_weight_min": -11.94478988647461,
      "activations/layer10_attention_weight_max": 48.03830337524414,
      "activations/layer10_attention_weight_min": -37.71261978149414,
      "activations/layer11_attention_weight_max": 34.78654479980469,
      "activations/layer11_attention_weight_min": -29.70624351501465,
      "activations/layer12_attention_weight_max": 39.28032302856445,
      "activations/layer12_attention_weight_min": -28.2958927154541,
      "activations/layer13_attention_weight_max": 44.64915084838867,
      "activations/layer13_attention_weight_min": -37.91461944580078,
      "activations/layer14_attention_weight_max": 65.27598571777344,
      "activations/layer14_attention_weight_min": -55.32975769042969,
      "activations/layer15_attention_weight_max": 45.67551803588867,
      "activations/layer15_attention_weight_min": -33.12799835205078,
      "activations/layer16_attention_weight_max": 46.67658233642578,
      "activations/layer16_attention_weight_min": -32.05500030517578,
      "activations/layer17_attention_weight_max": 38.8437614440918,
      "activations/layer17_attention_weight_min": -28.13077163696289,
      "activations/layer18_attention_weight_max": 28.32780647277832,
      "activations/layer18_attention_weight_min": -18.536048889160156,
      "activations/layer19_attention_weight_max": 41.30924987792969,
      "activations/layer19_attention_weight_min": -29.1221923828125,
      "activations/layer1_attention_weight_max": 17.855539321899414,
      "activations/layer1_attention_weight_min": -13.118109703063965,
      "activations/layer20_attention_weight_max": 33.43855667114258,
      "activations/layer20_attention_weight_min": -25.127117156982422,
      "activations/layer21_attention_weight_max": 32.715293884277344,
      "activations/layer21_attention_weight_min": -23.153085708618164,
      "activations/layer22_attention_weight_max": 46.72393798828125,
      "activations/layer22_attention_weight_min": -27.806461334228516,
      "activations/layer23_attention_weight_max": 29.469223022460938,
      "activations/layer23_attention_weight_min": -18.97919273376465,
      "activations/layer2_attention_weight_max": 30.463674545288086,
      "activations/layer2_attention_weight_min": -29.100746154785156,
      "activations/layer3_attention_weight_max": 78.35150909423828,
      "activations/layer3_attention_weight_min": -79.18326568603516,
      "activations/layer4_attention_weight_max": 131.99794006347656,
      "activations/layer4_attention_weight_min": -121.58404541015625,
      "activations/layer5_attention_weight_max": 105.87374114990234,
      "activations/layer5_attention_weight_min": -97.43220520019531,
      "activations/layer6_attention_weight_max": 56.82726287841797,
      "activations/layer6_attention_weight_min": -54.12100601196289,
      "activations/layer7_attention_weight_max": 71.86204528808594,
      "activations/layer7_attention_weight_min": -70.50533294677734,
      "activations/layer8_attention_weight_max": 40.61418533325195,
      "activations/layer8_attention_weight_min": -36.167293548583984,
      "activations/layer9_attention_weight_max": 53.07405471801758,
      "activations/layer9_attention_weight_min": -42.73151779174805,
      "epoch": 2.82,
      "learning_rate": 0.0001331465909090909,
      "loss": 2.9249,
      "step": 48550
    },
    {
      "activations/layer0_attention_weight_max": 17.347585678100586,
      "activations/layer0_attention_weight_min": -11.323628425598145,
      "activations/layer10_attention_weight_max": 43.22443771362305,
      "activations/layer10_attention_weight_min": -33.547035217285156,
      "activations/layer11_attention_weight_max": 42.920188903808594,
      "activations/layer11_attention_weight_min": -34.27500534057617,
      "activations/layer12_attention_weight_max": 42.451171875,
      "activations/layer12_attention_weight_min": -30.097558975219727,
      "activations/layer13_attention_weight_max": 50.19034957885742,
      "activations/layer13_attention_weight_min": -38.511531829833984,
      "activations/layer14_attention_weight_max": 73.86102294921875,
      "activations/layer14_attention_weight_min": -48.07535171508789,
      "activations/layer15_attention_weight_max": 43.8648567199707,
      "activations/layer15_attention_weight_min": -29.42408561706543,
      "activations/layer16_attention_weight_max": 45.76295471191406,
      "activations/layer16_attention_weight_min": -31.246782302856445,
      "activations/layer17_attention_weight_max": 41.6407470703125,
      "activations/layer17_attention_weight_min": -25.94899559020996,
      "activations/layer18_attention_weight_max": 26.909826278686523,
      "activations/layer18_attention_weight_min": -18.57016944885254,
      "activations/layer19_attention_weight_max": 41.669158935546875,
      "activations/layer19_attention_weight_min": -30.383926391601562,
      "activations/layer1_attention_weight_max": 17.394262313842773,
      "activations/layer1_attention_weight_min": -12.600180625915527,
      "activations/layer20_attention_weight_max": 31.345317840576172,
      "activations/layer20_attention_weight_min": -23.491039276123047,
      "activations/layer21_attention_weight_max": 26.30702781677246,
      "activations/layer21_attention_weight_min": -18.573970794677734,
      "activations/layer22_attention_weight_max": 43.94529342651367,
      "activations/layer22_attention_weight_min": -28.97842025756836,
      "activations/layer23_attention_weight_max": 26.29627227783203,
      "activations/layer23_attention_weight_min": -17.57535171508789,
      "activations/layer2_attention_weight_max": 29.31157684326172,
      "activations/layer2_attention_weight_min": -28.85641098022461,
      "activations/layer3_attention_weight_max": 79.49127960205078,
      "activations/layer3_attention_weight_min": -78.48503875732422,
      "activations/layer4_attention_weight_max": 148.03933715820312,
      "activations/layer4_attention_weight_min": -124.13817596435547,
      "activations/layer5_attention_weight_max": 119.17832946777344,
      "activations/layer5_attention_weight_min": -97.95366668701172,
      "activations/layer6_attention_weight_max": 65.27725219726562,
      "activations/layer6_attention_weight_min": -55.27991485595703,
      "activations/layer7_attention_weight_max": 79.5504379272461,
      "activations/layer7_attention_weight_min": -69.65565490722656,
      "activations/layer8_attention_weight_max": 48.5853157043457,
      "activations/layer8_attention_weight_min": -38.0413703918457,
      "activations/layer9_attention_weight_max": 65.82621002197266,
      "activations/layer9_attention_weight_min": -52.25407028198242,
      "epoch": 2.82,
      "learning_rate": 0.0001331276515151515,
      "loss": 2.9257,
      "step": 48600
    },
    {
      "activations/layer0_attention_weight_max": 16.856708526611328,
      "activations/layer0_attention_weight_min": -11.327025413513184,
      "activations/layer10_attention_weight_max": 45.422115325927734,
      "activations/layer10_attention_weight_min": -34.80788040161133,
      "activations/layer11_attention_weight_max": 39.398162841796875,
      "activations/layer11_attention_weight_min": -33.44594192504883,
      "activations/layer12_attention_weight_max": 29.83559799194336,
      "activations/layer12_attention_weight_min": -28.537809371948242,
      "activations/layer13_attention_weight_max": 52.413108825683594,
      "activations/layer13_attention_weight_min": -40.569393157958984,
      "activations/layer14_attention_weight_max": 78.80809783935547,
      "activations/layer14_attention_weight_min": -59.50563430786133,
      "activations/layer15_attention_weight_max": 47.46467590332031,
      "activations/layer15_attention_weight_min": -32.762481689453125,
      "activations/layer16_attention_weight_max": 52.98291015625,
      "activations/layer16_attention_weight_min": -33.58651351928711,
      "activations/layer17_attention_weight_max": 45.79994201660156,
      "activations/layer17_attention_weight_min": -30.8099365234375,
      "activations/layer18_attention_weight_max": 33.30302429199219,
      "activations/layer18_attention_weight_min": -21.355545043945312,
      "activations/layer19_attention_weight_max": 54.896602630615234,
      "activations/layer19_attention_weight_min": -31.832239151000977,
      "activations/layer1_attention_weight_max": 18.77609634399414,
      "activations/layer1_attention_weight_min": -12.393290519714355,
      "activations/layer20_attention_weight_max": 35.07615280151367,
      "activations/layer20_attention_weight_min": -22.795177459716797,
      "activations/layer21_attention_weight_max": 35.41481399536133,
      "activations/layer21_attention_weight_min": -20.85091209411621,
      "activations/layer22_attention_weight_max": 47.592689514160156,
      "activations/layer22_attention_weight_min": -26.12322998046875,
      "activations/layer23_attention_weight_max": 29.833045959472656,
      "activations/layer23_attention_weight_min": -19.011899948120117,
      "activations/layer2_attention_weight_max": 30.464574813842773,
      "activations/layer2_attention_weight_min": -30.610580444335938,
      "activations/layer3_attention_weight_max": 84.81268310546875,
      "activations/layer3_attention_weight_min": -83.99372100830078,
      "activations/layer4_attention_weight_max": 136.93988037109375,
      "activations/layer4_attention_weight_min": -127.2718734741211,
      "activations/layer5_attention_weight_max": 111.763671875,
      "activations/layer5_attention_weight_min": -95.47850799560547,
      "activations/layer6_attention_weight_max": 62.281883239746094,
      "activations/layer6_attention_weight_min": -55.98462677001953,
      "activations/layer7_attention_weight_max": 73.13525390625,
      "activations/layer7_attention_weight_min": -74.82101440429688,
      "activations/layer8_attention_weight_max": 46.15121841430664,
      "activations/layer8_attention_weight_min": -40.2449836730957,
      "activations/layer9_attention_weight_max": 55.381072998046875,
      "activations/layer9_attention_weight_min": -52.25175857543945,
      "epoch": 2.83,
      "learning_rate": 0.0001331087121212121,
      "loss": 2.9266,
      "step": 48650
    },
    {
      "activations/layer0_attention_weight_max": 17.220645904541016,
      "activations/layer0_attention_weight_min": -12.60570240020752,
      "activations/layer10_attention_weight_max": 43.40729904174805,
      "activations/layer10_attention_weight_min": -34.32168197631836,
      "activations/layer11_attention_weight_max": 43.2340202331543,
      "activations/layer11_attention_weight_min": -35.206207275390625,
      "activations/layer12_attention_weight_max": 43.46553039550781,
      "activations/layer12_attention_weight_min": -30.9356632232666,
      "activations/layer13_attention_weight_max": 59.473541259765625,
      "activations/layer13_attention_weight_min": -38.226722717285156,
      "activations/layer14_attention_weight_max": 91.5168685913086,
      "activations/layer14_attention_weight_min": -62.779510498046875,
      "activations/layer15_attention_weight_max": 52.0931282043457,
      "activations/layer15_attention_weight_min": -41.147544860839844,
      "activations/layer16_attention_weight_max": 42.2254524230957,
      "activations/layer16_attention_weight_min": -31.253385543823242,
      "activations/layer17_attention_weight_max": 47.32650375366211,
      "activations/layer17_attention_weight_min": -27.478282928466797,
      "activations/layer18_attention_weight_max": 34.51435470581055,
      "activations/layer18_attention_weight_min": -19.833433151245117,
      "activations/layer19_attention_weight_max": 45.20968246459961,
      "activations/layer19_attention_weight_min": -28.933691024780273,
      "activations/layer1_attention_weight_max": 18.057811737060547,
      "activations/layer1_attention_weight_min": -12.686272621154785,
      "activations/layer20_attention_weight_max": 38.544700622558594,
      "activations/layer20_attention_weight_min": -22.372066497802734,
      "activations/layer21_attention_weight_max": 32.34334182739258,
      "activations/layer21_attention_weight_min": -20.437711715698242,
      "activations/layer22_attention_weight_max": 51.68478012084961,
      "activations/layer22_attention_weight_min": -25.93537139892578,
      "activations/layer23_attention_weight_max": 28.141372680664062,
      "activations/layer23_attention_weight_min": -14.975269317626953,
      "activations/layer2_attention_weight_max": 30.533702850341797,
      "activations/layer2_attention_weight_min": -30.17682647705078,
      "activations/layer3_attention_weight_max": 80.7728042602539,
      "activations/layer3_attention_weight_min": -82.6251449584961,
      "activations/layer4_attention_weight_max": 131.64988708496094,
      "activations/layer4_attention_weight_min": -128.5298309326172,
      "activations/layer5_attention_weight_max": 105.03253173828125,
      "activations/layer5_attention_weight_min": -97.0487289428711,
      "activations/layer6_attention_weight_max": 61.02841567993164,
      "activations/layer6_attention_weight_min": -55.22892379760742,
      "activations/layer7_attention_weight_max": 77.88338470458984,
      "activations/layer7_attention_weight_min": -71.36808013916016,
      "activations/layer8_attention_weight_max": 41.537025451660156,
      "activations/layer8_attention_weight_min": -39.20766830444336,
      "activations/layer9_attention_weight_max": 75.8872299194336,
      "activations/layer9_attention_weight_min": -55.2430305480957,
      "epoch": 2.83,
      "learning_rate": 0.0001330897727272727,
      "loss": 2.8982,
      "step": 48700
    },
    {
      "activations/layer0_attention_weight_max": 17.036327362060547,
      "activations/layer0_attention_weight_min": -11.100384712219238,
      "activations/layer10_attention_weight_max": 44.03379440307617,
      "activations/layer10_attention_weight_min": -35.145042419433594,
      "activations/layer11_attention_weight_max": 39.609439849853516,
      "activations/layer11_attention_weight_min": -32.705078125,
      "activations/layer12_attention_weight_max": 29.558473587036133,
      "activations/layer12_attention_weight_min": -29.78984260559082,
      "activations/layer13_attention_weight_max": 47.353797912597656,
      "activations/layer13_attention_weight_min": -35.595664978027344,
      "activations/layer14_attention_weight_max": 86.737060546875,
      "activations/layer14_attention_weight_min": -63.396324157714844,
      "activations/layer15_attention_weight_max": 52.178306579589844,
      "activations/layer15_attention_weight_min": -31.296228408813477,
      "activations/layer16_attention_weight_max": 43.43246078491211,
      "activations/layer16_attention_weight_min": -33.69520950317383,
      "activations/layer17_attention_weight_max": 46.05915451049805,
      "activations/layer17_attention_weight_min": -29.909027099609375,
      "activations/layer18_attention_weight_max": 29.014911651611328,
      "activations/layer18_attention_weight_min": -18.658273696899414,
      "activations/layer19_attention_weight_max": 46.61492156982422,
      "activations/layer19_attention_weight_min": -32.46452331542969,
      "activations/layer1_attention_weight_max": 18.22615623474121,
      "activations/layer1_attention_weight_min": -12.051925659179688,
      "activations/layer20_attention_weight_max": 33.650386810302734,
      "activations/layer20_attention_weight_min": -24.294618606567383,
      "activations/layer21_attention_weight_max": 27.868038177490234,
      "activations/layer21_attention_weight_min": -19.685077667236328,
      "activations/layer22_attention_weight_max": 45.87384796142578,
      "activations/layer22_attention_weight_min": -24.850019454956055,
      "activations/layer23_attention_weight_max": 28.896564483642578,
      "activations/layer23_attention_weight_min": -16.494434356689453,
      "activations/layer2_attention_weight_max": 29.258827209472656,
      "activations/layer2_attention_weight_min": -27.67019271850586,
      "activations/layer3_attention_weight_max": 75.27931213378906,
      "activations/layer3_attention_weight_min": -76.95893096923828,
      "activations/layer4_attention_weight_max": 126.93578338623047,
      "activations/layer4_attention_weight_min": -119.1920394897461,
      "activations/layer5_attention_weight_max": 104.42549133300781,
      "activations/layer5_attention_weight_min": -94.03499603271484,
      "activations/layer6_attention_weight_max": 57.83210372924805,
      "activations/layer6_attention_weight_min": -51.682926177978516,
      "activations/layer7_attention_weight_max": 70.1968765258789,
      "activations/layer7_attention_weight_min": -67.73233032226562,
      "activations/layer8_attention_weight_max": 42.740631103515625,
      "activations/layer8_attention_weight_min": -36.97615432739258,
      "activations/layer9_attention_weight_max": 57.0369758605957,
      "activations/layer9_attention_weight_min": -43.75043487548828,
      "epoch": 2.83,
      "learning_rate": 0.00013307083333333333,
      "loss": 2.9074,
      "step": 48750
    },
    {
      "activations/layer0_attention_weight_max": 17.586509704589844,
      "activations/layer0_attention_weight_min": -11.710318565368652,
      "activations/layer10_attention_weight_max": 44.13615417480469,
      "activations/layer10_attention_weight_min": -34.766090393066406,
      "activations/layer11_attention_weight_max": 48.14069747924805,
      "activations/layer11_attention_weight_min": -35.75900650024414,
      "activations/layer12_attention_weight_max": 32.401580810546875,
      "activations/layer12_attention_weight_min": -26.10356903076172,
      "activations/layer13_attention_weight_max": 69.71575164794922,
      "activations/layer13_attention_weight_min": -41.35811996459961,
      "activations/layer14_attention_weight_max": 86.32827758789062,
      "activations/layer14_attention_weight_min": -52.739593505859375,
      "activations/layer15_attention_weight_max": 56.60651397705078,
      "activations/layer15_attention_weight_min": -36.35371780395508,
      "activations/layer16_attention_weight_max": 48.33177947998047,
      "activations/layer16_attention_weight_min": -33.82457733154297,
      "activations/layer17_attention_weight_max": 57.4825439453125,
      "activations/layer17_attention_weight_min": -32.19612121582031,
      "activations/layer18_attention_weight_max": 31.307104110717773,
      "activations/layer18_attention_weight_min": -19.050655364990234,
      "activations/layer19_attention_weight_max": 57.393524169921875,
      "activations/layer19_attention_weight_min": -31.801401138305664,
      "activations/layer1_attention_weight_max": 17.592613220214844,
      "activations/layer1_attention_weight_min": -12.37507438659668,
      "activations/layer20_attention_weight_max": 46.94266128540039,
      "activations/layer20_attention_weight_min": -24.616857528686523,
      "activations/layer21_attention_weight_max": 35.1761474609375,
      "activations/layer21_attention_weight_min": -22.0999698638916,
      "activations/layer22_attention_weight_max": 59.053749084472656,
      "activations/layer22_attention_weight_min": -28.23202896118164,
      "activations/layer23_attention_weight_max": 32.58854675292969,
      "activations/layer23_attention_weight_min": -16.843204498291016,
      "activations/layer2_attention_weight_max": 30.654691696166992,
      "activations/layer2_attention_weight_min": -30.023258209228516,
      "activations/layer3_attention_weight_max": 78.40547180175781,
      "activations/layer3_attention_weight_min": -80.86119079589844,
      "activations/layer4_attention_weight_max": 138.94224548339844,
      "activations/layer4_attention_weight_min": -128.7479248046875,
      "activations/layer5_attention_weight_max": 117.70894622802734,
      "activations/layer5_attention_weight_min": -96.69253540039062,
      "activations/layer6_attention_weight_max": 63.06148147583008,
      "activations/layer6_attention_weight_min": -53.85032653808594,
      "activations/layer7_attention_weight_max": 74.04835510253906,
      "activations/layer7_attention_weight_min": -69.2713394165039,
      "activations/layer8_attention_weight_max": 43.222652435302734,
      "activations/layer8_attention_weight_min": -36.51279067993164,
      "activations/layer9_attention_weight_max": 64.55184173583984,
      "activations/layer9_attention_weight_min": -52.329647064208984,
      "epoch": 2.84,
      "learning_rate": 0.00013305189393939391,
      "loss": 2.924,
      "step": 48800
    },
    {
      "activations/layer0_attention_weight_max": 17.148330688476562,
      "activations/layer0_attention_weight_min": -13.020308494567871,
      "activations/layer10_attention_weight_max": 39.46847915649414,
      "activations/layer10_attention_weight_min": -33.778900146484375,
      "activations/layer11_attention_weight_max": 32.87852478027344,
      "activations/layer11_attention_weight_min": -30.544883728027344,
      "activations/layer12_attention_weight_max": 44.388057708740234,
      "activations/layer12_attention_weight_min": -27.52815055847168,
      "activations/layer13_attention_weight_max": 51.04188919067383,
      "activations/layer13_attention_weight_min": -38.556854248046875,
      "activations/layer14_attention_weight_max": 77.22712707519531,
      "activations/layer14_attention_weight_min": -58.76572036743164,
      "activations/layer15_attention_weight_max": 43.033790588378906,
      "activations/layer15_attention_weight_min": -33.571922302246094,
      "activations/layer16_attention_weight_max": 43.65981674194336,
      "activations/layer16_attention_weight_min": -32.148040771484375,
      "activations/layer17_attention_weight_max": 42.19822692871094,
      "activations/layer17_attention_weight_min": -26.756237030029297,
      "activations/layer18_attention_weight_max": 31.118690490722656,
      "activations/layer18_attention_weight_min": -18.841474533081055,
      "activations/layer19_attention_weight_max": 43.767822265625,
      "activations/layer19_attention_weight_min": -29.93377113342285,
      "activations/layer1_attention_weight_max": 17.295055389404297,
      "activations/layer1_attention_weight_min": -11.658321380615234,
      "activations/layer20_attention_weight_max": 33.300262451171875,
      "activations/layer20_attention_weight_min": -22.722501754760742,
      "activations/layer21_attention_weight_max": 31.68539047241211,
      "activations/layer21_attention_weight_min": -24.8250789642334,
      "activations/layer22_attention_weight_max": 45.141666412353516,
      "activations/layer22_attention_weight_min": -29.048778533935547,
      "activations/layer23_attention_weight_max": 27.9019775390625,
      "activations/layer23_attention_weight_min": -17.350479125976562,
      "activations/layer2_attention_weight_max": 29.824155807495117,
      "activations/layer2_attention_weight_min": -30.275135040283203,
      "activations/layer3_attention_weight_max": 76.76195526123047,
      "activations/layer3_attention_weight_min": -84.16039276123047,
      "activations/layer4_attention_weight_max": 133.92286682128906,
      "activations/layer4_attention_weight_min": -134.97848510742188,
      "activations/layer5_attention_weight_max": 102.496337890625,
      "activations/layer5_attention_weight_min": -97.26351165771484,
      "activations/layer6_attention_weight_max": 56.70991897583008,
      "activations/layer6_attention_weight_min": -54.21950149536133,
      "activations/layer7_attention_weight_max": 66.9525146484375,
      "activations/layer7_attention_weight_min": -65.27123260498047,
      "activations/layer8_attention_weight_max": 40.513675689697266,
      "activations/layer8_attention_weight_min": -35.29188919067383,
      "activations/layer9_attention_weight_max": 57.10877227783203,
      "activations/layer9_attention_weight_min": -42.467830657958984,
      "epoch": 2.84,
      "learning_rate": 0.00013303295454545453,
      "loss": 2.905,
      "step": 48850
    },
    {
      "activations/layer0_attention_weight_max": 17.59341812133789,
      "activations/layer0_attention_weight_min": -11.583672523498535,
      "activations/layer10_attention_weight_max": 50.77363204956055,
      "activations/layer10_attention_weight_min": -36.867042541503906,
      "activations/layer11_attention_weight_max": 44.531463623046875,
      "activations/layer11_attention_weight_min": -32.00143814086914,
      "activations/layer12_attention_weight_max": 38.405452728271484,
      "activations/layer12_attention_weight_min": -30.45366096496582,
      "activations/layer13_attention_weight_max": 48.97085952758789,
      "activations/layer13_attention_weight_min": -35.69625473022461,
      "activations/layer14_attention_weight_max": 68.95394134521484,
      "activations/layer14_attention_weight_min": -52.32536315917969,
      "activations/layer15_attention_weight_max": 46.26816177368164,
      "activations/layer15_attention_weight_min": -31.206031799316406,
      "activations/layer16_attention_weight_max": 43.08261489868164,
      "activations/layer16_attention_weight_min": -31.27051544189453,
      "activations/layer17_attention_weight_max": 44.1973762512207,
      "activations/layer17_attention_weight_min": -27.658748626708984,
      "activations/layer18_attention_weight_max": 35.0766716003418,
      "activations/layer18_attention_weight_min": -19.446908950805664,
      "activations/layer19_attention_weight_max": 43.90267562866211,
      "activations/layer19_attention_weight_min": -27.706811904907227,
      "activations/layer1_attention_weight_max": 17.95131492614746,
      "activations/layer1_attention_weight_min": -12.095053672790527,
      "activations/layer20_attention_weight_max": 35.26506042480469,
      "activations/layer20_attention_weight_min": -22.628847122192383,
      "activations/layer21_attention_weight_max": 35.31425476074219,
      "activations/layer21_attention_weight_min": -20.99771499633789,
      "activations/layer22_attention_weight_max": 47.67622375488281,
      "activations/layer22_attention_weight_min": -30.987825393676758,
      "activations/layer23_attention_weight_max": 25.62441062927246,
      "activations/layer23_attention_weight_min": -16.879581451416016,
      "activations/layer2_attention_weight_max": 30.043476104736328,
      "activations/layer2_attention_weight_min": -29.737030029296875,
      "activations/layer3_attention_weight_max": 75.6439208984375,
      "activations/layer3_attention_weight_min": -77.50529479980469,
      "activations/layer4_attention_weight_max": 126.27375793457031,
      "activations/layer4_attention_weight_min": -123.7688980102539,
      "activations/layer5_attention_weight_max": 104.40636444091797,
      "activations/layer5_attention_weight_min": -93.066650390625,
      "activations/layer6_attention_weight_max": 58.331085205078125,
      "activations/layer6_attention_weight_min": -51.775875091552734,
      "activations/layer7_attention_weight_max": 72.37042999267578,
      "activations/layer7_attention_weight_min": -63.30455017089844,
      "activations/layer8_attention_weight_max": 40.749149322509766,
      "activations/layer8_attention_weight_min": -37.08998107910156,
      "activations/layer9_attention_weight_max": 54.32532501220703,
      "activations/layer9_attention_weight_min": -43.47539138793945,
      "epoch": 2.84,
      "learning_rate": 0.00013301401515151514,
      "loss": 2.9177,
      "step": 48900
    },
    {
      "activations/layer0_attention_weight_max": 17.708383560180664,
      "activations/layer0_attention_weight_min": -11.996298789978027,
      "activations/layer10_attention_weight_max": 46.12215042114258,
      "activations/layer10_attention_weight_min": -35.95003128051758,
      "activations/layer11_attention_weight_max": 34.79704284667969,
      "activations/layer11_attention_weight_min": -30.074459075927734,
      "activations/layer12_attention_weight_max": 45.9626350402832,
      "activations/layer12_attention_weight_min": -27.87491798400879,
      "activations/layer13_attention_weight_max": 59.93251419067383,
      "activations/layer13_attention_weight_min": -38.08643341064453,
      "activations/layer14_attention_weight_max": 72.53314208984375,
      "activations/layer14_attention_weight_min": -51.92385482788086,
      "activations/layer15_attention_weight_max": 51.7739143371582,
      "activations/layer15_attention_weight_min": -34.068511962890625,
      "activations/layer16_attention_weight_max": 44.58635330200195,
      "activations/layer16_attention_weight_min": -28.96166229248047,
      "activations/layer17_attention_weight_max": 42.62837600708008,
      "activations/layer17_attention_weight_min": -24.879201889038086,
      "activations/layer18_attention_weight_max": 30.517656326293945,
      "activations/layer18_attention_weight_min": -18.1317195892334,
      "activations/layer19_attention_weight_max": 50.99662780761719,
      "activations/layer19_attention_weight_min": -27.4794864654541,
      "activations/layer1_attention_weight_max": 16.69413948059082,
      "activations/layer1_attention_weight_min": -12.74160099029541,
      "activations/layer20_attention_weight_max": 34.66157913208008,
      "activations/layer20_attention_weight_min": -22.613811492919922,
      "activations/layer21_attention_weight_max": 37.15320587158203,
      "activations/layer21_attention_weight_min": -20.355295181274414,
      "activations/layer22_attention_weight_max": 50.21678161621094,
      "activations/layer22_attention_weight_min": -25.2011775970459,
      "activations/layer23_attention_weight_max": 27.856754302978516,
      "activations/layer23_attention_weight_min": -17.307456970214844,
      "activations/layer2_attention_weight_max": 29.28913116455078,
      "activations/layer2_attention_weight_min": -29.446170806884766,
      "activations/layer3_attention_weight_max": 79.11529541015625,
      "activations/layer3_attention_weight_min": -81.82267761230469,
      "activations/layer4_attention_weight_max": 128.1848602294922,
      "activations/layer4_attention_weight_min": -123.63720703125,
      "activations/layer5_attention_weight_max": 105.95006561279297,
      "activations/layer5_attention_weight_min": -90.31370544433594,
      "activations/layer6_attention_weight_max": 61.13252258300781,
      "activations/layer6_attention_weight_min": -52.70292282104492,
      "activations/layer7_attention_weight_max": 70.8720703125,
      "activations/layer7_attention_weight_min": -76.07586669921875,
      "activations/layer8_attention_weight_max": 41.03376770019531,
      "activations/layer8_attention_weight_min": -35.785308837890625,
      "activations/layer9_attention_weight_max": 48.95964431762695,
      "activations/layer9_attention_weight_min": -43.468074798583984,
      "epoch": 2.84,
      "learning_rate": 0.00013299545454545454,
      "loss": 2.9171,
      "step": 48950
    },
    {
      "activations/layer0_attention_weight_max": 17.122577667236328,
      "activations/layer0_attention_weight_min": -11.254947662353516,
      "activations/layer10_attention_weight_max": 45.343475341796875,
      "activations/layer10_attention_weight_min": -35.02861022949219,
      "activations/layer11_attention_weight_max": 40.894805908203125,
      "activations/layer11_attention_weight_min": -31.193403244018555,
      "activations/layer12_attention_weight_max": 32.18004608154297,
      "activations/layer12_attention_weight_min": -26.03542709350586,
      "activations/layer13_attention_weight_max": 48.587833404541016,
      "activations/layer13_attention_weight_min": -34.11264419555664,
      "activations/layer14_attention_weight_max": 74.4037857055664,
      "activations/layer14_attention_weight_min": -50.69439697265625,
      "activations/layer15_attention_weight_max": 52.220272064208984,
      "activations/layer15_attention_weight_min": -32.194000244140625,
      "activations/layer16_attention_weight_max": 41.235897064208984,
      "activations/layer16_attention_weight_min": -32.34375,
      "activations/layer17_attention_weight_max": 47.211116790771484,
      "activations/layer17_attention_weight_min": -27.297391891479492,
      "activations/layer18_attention_weight_max": 30.223251342773438,
      "activations/layer18_attention_weight_min": -19.935407638549805,
      "activations/layer19_attention_weight_max": 43.17206954956055,
      "activations/layer19_attention_weight_min": -27.177473068237305,
      "activations/layer1_attention_weight_max": 17.997547149658203,
      "activations/layer1_attention_weight_min": -12.661128044128418,
      "activations/layer20_attention_weight_max": 35.81066131591797,
      "activations/layer20_attention_weight_min": -23.747140884399414,
      "activations/layer21_attention_weight_max": 30.80674171447754,
      "activations/layer21_attention_weight_min": -22.13983726501465,
      "activations/layer22_attention_weight_max": 46.31782913208008,
      "activations/layer22_attention_weight_min": -29.818492889404297,
      "activations/layer23_attention_weight_max": 26.892452239990234,
      "activations/layer23_attention_weight_min": -16.306827545166016,
      "activations/layer2_attention_weight_max": 31.967313766479492,
      "activations/layer2_attention_weight_min": -30.55752944946289,
      "activations/layer3_attention_weight_max": 82.95415496826172,
      "activations/layer3_attention_weight_min": -82.91365814208984,
      "activations/layer4_attention_weight_max": 140.92735290527344,
      "activations/layer4_attention_weight_min": -137.27903747558594,
      "activations/layer5_attention_weight_max": 119.14949035644531,
      "activations/layer5_attention_weight_min": -106.99812316894531,
      "activations/layer6_attention_weight_max": 61.804237365722656,
      "activations/layer6_attention_weight_min": -55.27568054199219,
      "activations/layer7_attention_weight_max": 77.01175689697266,
      "activations/layer7_attention_weight_min": -68.70341491699219,
      "activations/layer8_attention_weight_max": 46.92226028442383,
      "activations/layer8_attention_weight_min": -38.69734573364258,
      "activations/layer9_attention_weight_max": 61.140071868896484,
      "activations/layer9_attention_weight_min": -42.555328369140625,
      "epoch": 2.85,
      "learning_rate": 0.00013297651515151512,
      "loss": 2.9245,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_loss": 2.8671875,
      "eval_runtime": 8.5446,
      "eval_samples_per_second": 502.54,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_openwebtext_loss": 2.8671875,
      "eval_openwebtext_ppl": 17.587483776585056,
      "eval_openwebtext_runtime": 8.5446,
      "eval_openwebtext_samples_per_second": 502.54,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_wikitext_loss": 3.130859375,
      "eval_wikitext_ppl": 22.893645317537526,
      "eval_wikitext_runtime": 2.0156,
      "eval_wikitext_samples_per_second": 226.234,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_lambada_loss": 3.0,
      "eval_lambada_ppl": 20.085536923187668,
      "eval_lambada_runtime": 9.6234,
      "eval_lambada_samples_per_second": 505.952,
      "step": 49000
    },
    {
      "activations/layer0_attention_weight_max": 15.858377456665039,
      "activations/layer0_attention_weight_min": -11.228140830993652,
      "activations/layer10_attention_weight_max": 43.12013244628906,
      "activations/layer10_attention_weight_min": -36.324405670166016,
      "activations/layer11_attention_weight_max": 43.239013671875,
      "activations/layer11_attention_weight_min": -30.91274642944336,
      "activations/layer12_attention_weight_max": 53.01097106933594,
      "activations/layer12_attention_weight_min": -32.904083251953125,
      "activations/layer13_attention_weight_max": 70.38394927978516,
      "activations/layer13_attention_weight_min": -43.19228744506836,
      "activations/layer14_attention_weight_max": 82.66590881347656,
      "activations/layer14_attention_weight_min": -54.411338806152344,
      "activations/layer15_attention_weight_max": 54.954833984375,
      "activations/layer15_attention_weight_min": -33.454017639160156,
      "activations/layer16_attention_weight_max": 53.7838249206543,
      "activations/layer16_attention_weight_min": -32.30389404296875,
      "activations/layer17_attention_weight_max": 47.73590087890625,
      "activations/layer17_attention_weight_min": -28.69868278503418,
      "activations/layer18_attention_weight_max": 34.735687255859375,
      "activations/layer18_attention_weight_min": -20.085756301879883,
      "activations/layer19_attention_weight_max": 54.778900146484375,
      "activations/layer19_attention_weight_min": -32.78216552734375,
      "activations/layer1_attention_weight_max": 17.216047286987305,
      "activations/layer1_attention_weight_min": -13.697834014892578,
      "activations/layer20_attention_weight_max": 38.81778335571289,
      "activations/layer20_attention_weight_min": -24.97560691833496,
      "activations/layer21_attention_weight_max": 41.81159591674805,
      "activations/layer21_attention_weight_min": -22.03833770751953,
      "activations/layer22_attention_weight_max": 56.74508285522461,
      "activations/layer22_attention_weight_min": -28.19332504272461,
      "activations/layer23_attention_weight_max": 33.194976806640625,
      "activations/layer23_attention_weight_min": -19.803081512451172,
      "activations/layer2_attention_weight_max": 31.54861068725586,
      "activations/layer2_attention_weight_min": -30.2855281829834,
      "activations/layer3_attention_weight_max": 79.90752410888672,
      "activations/layer3_attention_weight_min": -82.59366607666016,
      "activations/layer4_attention_weight_max": 134.84922790527344,
      "activations/layer4_attention_weight_min": -123.70755767822266,
      "activations/layer5_attention_weight_max": 108.04235076904297,
      "activations/layer5_attention_weight_min": -92.04802703857422,
      "activations/layer6_attention_weight_max": 61.38187789916992,
      "activations/layer6_attention_weight_min": -53.577781677246094,
      "activations/layer7_attention_weight_max": 77.52654266357422,
      "activations/layer7_attention_weight_min": -69.94137573242188,
      "activations/layer8_attention_weight_max": 45.03308868408203,
      "activations/layer8_attention_weight_min": -35.753387451171875,
      "activations/layer9_attention_weight_max": 53.688594818115234,
      "activations/layer9_attention_weight_min": -41.85668182373047,
      "epoch": 2.85,
      "learning_rate": 0.00013295757575757574,
      "loss": 2.9,
      "step": 49050
    },
    {
      "activations/layer0_attention_weight_max": 17.807157516479492,
      "activations/layer0_attention_weight_min": -11.358942985534668,
      "activations/layer10_attention_weight_max": 37.91466522216797,
      "activations/layer10_attention_weight_min": -32.77165603637695,
      "activations/layer11_attention_weight_max": 33.799278259277344,
      "activations/layer11_attention_weight_min": -27.7020263671875,
      "activations/layer12_attention_weight_max": 37.265724182128906,
      "activations/layer12_attention_weight_min": -28.83216667175293,
      "activations/layer13_attention_weight_max": 44.911380767822266,
      "activations/layer13_attention_weight_min": -35.114742279052734,
      "activations/layer14_attention_weight_max": 59.74494552612305,
      "activations/layer14_attention_weight_min": -45.690345764160156,
      "activations/layer15_attention_weight_max": 38.64834213256836,
      "activations/layer15_attention_weight_min": -30.707046508789062,
      "activations/layer16_attention_weight_max": 42.750083923339844,
      "activations/layer16_attention_weight_min": -29.66238784790039,
      "activations/layer17_attention_weight_max": 38.348846435546875,
      "activations/layer17_attention_weight_min": -26.96271324157715,
      "activations/layer18_attention_weight_max": 27.304542541503906,
      "activations/layer18_attention_weight_min": -18.268047332763672,
      "activations/layer19_attention_weight_max": 42.55490493774414,
      "activations/layer19_attention_weight_min": -28.825855255126953,
      "activations/layer1_attention_weight_max": 18.21998405456543,
      "activations/layer1_attention_weight_min": -12.809699058532715,
      "activations/layer20_attention_weight_max": 30.045555114746094,
      "activations/layer20_attention_weight_min": -22.831506729125977,
      "activations/layer21_attention_weight_max": 32.18079376220703,
      "activations/layer21_attention_weight_min": -21.30205535888672,
      "activations/layer22_attention_weight_max": 43.932762145996094,
      "activations/layer22_attention_weight_min": -29.15374755859375,
      "activations/layer23_attention_weight_max": 25.307411193847656,
      "activations/layer23_attention_weight_min": -17.448230743408203,
      "activations/layer2_attention_weight_max": 29.08279037475586,
      "activations/layer2_attention_weight_min": -27.73444938659668,
      "activations/layer3_attention_weight_max": 72.69225311279297,
      "activations/layer3_attention_weight_min": -77.05049133300781,
      "activations/layer4_attention_weight_max": 113.02922821044922,
      "activations/layer4_attention_weight_min": -112.7873306274414,
      "activations/layer5_attention_weight_max": 90.79622650146484,
      "activations/layer5_attention_weight_min": -94.64983367919922,
      "activations/layer6_attention_weight_max": 51.96199035644531,
      "activations/layer6_attention_weight_min": -51.77775955200195,
      "activations/layer7_attention_weight_max": 68.60314178466797,
      "activations/layer7_attention_weight_min": -64.71187591552734,
      "activations/layer8_attention_weight_max": 38.457908630371094,
      "activations/layer8_attention_weight_min": -36.16323471069336,
      "activations/layer9_attention_weight_max": 54.6351318359375,
      "activations/layer9_attention_weight_min": -43.81391525268555,
      "epoch": 2.85,
      "learning_rate": 0.00013293863636363635,
      "loss": 2.9142,
      "step": 49100
    },
    {
      "activations/layer0_attention_weight_max": 16.4596004486084,
      "activations/layer0_attention_weight_min": -11.266294479370117,
      "activations/layer10_attention_weight_max": 40.8880729675293,
      "activations/layer10_attention_weight_min": -33.259124755859375,
      "activations/layer11_attention_weight_max": 36.400917053222656,
      "activations/layer11_attention_weight_min": -29.124502182006836,
      "activations/layer12_attention_weight_max": 31.855100631713867,
      "activations/layer12_attention_weight_min": -26.63724708557129,
      "activations/layer13_attention_weight_max": 50.14689636230469,
      "activations/layer13_attention_weight_min": -36.71892547607422,
      "activations/layer14_attention_weight_max": 69.52547454833984,
      "activations/layer14_attention_weight_min": -49.13120651245117,
      "activations/layer15_attention_weight_max": 46.70525360107422,
      "activations/layer15_attention_weight_min": -30.800243377685547,
      "activations/layer16_attention_weight_max": 45.46574401855469,
      "activations/layer16_attention_weight_min": -33.58512496948242,
      "activations/layer17_attention_weight_max": 42.399147033691406,
      "activations/layer17_attention_weight_min": -28.422828674316406,
      "activations/layer18_attention_weight_max": 30.64555549621582,
      "activations/layer18_attention_weight_min": -21.189531326293945,
      "activations/layer19_attention_weight_max": 47.6225471496582,
      "activations/layer19_attention_weight_min": -29.6699275970459,
      "activations/layer1_attention_weight_max": 17.440650939941406,
      "activations/layer1_attention_weight_min": -14.864337921142578,
      "activations/layer20_attention_weight_max": 38.66392135620117,
      "activations/layer20_attention_weight_min": -22.9468936920166,
      "activations/layer21_attention_weight_max": 34.444488525390625,
      "activations/layer21_attention_weight_min": -19.92562484741211,
      "activations/layer22_attention_weight_max": 52.413612365722656,
      "activations/layer22_attention_weight_min": -28.60000228881836,
      "activations/layer23_attention_weight_max": 29.83487319946289,
      "activations/layer23_attention_weight_min": -18.601348876953125,
      "activations/layer2_attention_weight_max": 30.998191833496094,
      "activations/layer2_attention_weight_min": -29.863920211791992,
      "activations/layer3_attention_weight_max": 76.22838592529297,
      "activations/layer3_attention_weight_min": -80.6483383178711,
      "activations/layer4_attention_weight_max": 132.55445861816406,
      "activations/layer4_attention_weight_min": -129.39517211914062,
      "activations/layer5_attention_weight_max": 104.198974609375,
      "activations/layer5_attention_weight_min": -94.060302734375,
      "activations/layer6_attention_weight_max": 54.330631256103516,
      "activations/layer6_attention_weight_min": -53.6809196472168,
      "activations/layer7_attention_weight_max": 70.66754913330078,
      "activations/layer7_attention_weight_min": -67.70319366455078,
      "activations/layer8_attention_weight_max": 41.40544509887695,
      "activations/layer8_attention_weight_min": -36.216976165771484,
      "activations/layer9_attention_weight_max": 50.38774490356445,
      "activations/layer9_attention_weight_min": -50.10422134399414,
      "epoch": 2.86,
      "learning_rate": 0.00013291969696969694,
      "loss": 2.9172,
      "step": 49150
    },
    {
      "activations/layer0_attention_weight_max": 17.01373863220215,
      "activations/layer0_attention_weight_min": -11.036955833435059,
      "activations/layer10_attention_weight_max": 47.15536880493164,
      "activations/layer10_attention_weight_min": -34.40814971923828,
      "activations/layer11_attention_weight_max": 35.23501968383789,
      "activations/layer11_attention_weight_min": -28.31665802001953,
      "activations/layer12_attention_weight_max": 31.539730072021484,
      "activations/layer12_attention_weight_min": -30.632661819458008,
      "activations/layer13_attention_weight_max": 46.126155853271484,
      "activations/layer13_attention_weight_min": -37.24359893798828,
      "activations/layer14_attention_weight_max": 73.67716979980469,
      "activations/layer14_attention_weight_min": -57.2630729675293,
      "activations/layer15_attention_weight_max": 51.68626022338867,
      "activations/layer15_attention_weight_min": -34.99261474609375,
      "activations/layer16_attention_weight_max": 42.183650970458984,
      "activations/layer16_attention_weight_min": -30.127656936645508,
      "activations/layer17_attention_weight_max": 44.86329650878906,
      "activations/layer17_attention_weight_min": -28.99909782409668,
      "activations/layer18_attention_weight_max": 33.176761627197266,
      "activations/layer18_attention_weight_min": -18.651586532592773,
      "activations/layer19_attention_weight_max": 49.88072204589844,
      "activations/layer19_attention_weight_min": -31.430585861206055,
      "activations/layer1_attention_weight_max": 18.802196502685547,
      "activations/layer1_attention_weight_min": -13.476755142211914,
      "activations/layer20_attention_weight_max": 37.07670593261719,
      "activations/layer20_attention_weight_min": -21.92961311340332,
      "activations/layer21_attention_weight_max": 33.86553192138672,
      "activations/layer21_attention_weight_min": -21.620214462280273,
      "activations/layer22_attention_weight_max": 51.43673324584961,
      "activations/layer22_attention_weight_min": -31.100971221923828,
      "activations/layer23_attention_weight_max": 30.664724349975586,
      "activations/layer23_attention_weight_min": -17.22873306274414,
      "activations/layer2_attention_weight_max": 30.184234619140625,
      "activations/layer2_attention_weight_min": -29.329078674316406,
      "activations/layer3_attention_weight_max": 77.06996154785156,
      "activations/layer3_attention_weight_min": -79.09056091308594,
      "activations/layer4_attention_weight_max": 126.92569732666016,
      "activations/layer4_attention_weight_min": -119.6214599609375,
      "activations/layer5_attention_weight_max": 103.15789794921875,
      "activations/layer5_attention_weight_min": -91.39395141601562,
      "activations/layer6_attention_weight_max": 57.0332145690918,
      "activations/layer6_attention_weight_min": -50.27497863769531,
      "activations/layer7_attention_weight_max": 72.07450866699219,
      "activations/layer7_attention_weight_min": -64.83665466308594,
      "activations/layer8_attention_weight_max": 40.44112014770508,
      "activations/layer8_attention_weight_min": -34.525508880615234,
      "activations/layer9_attention_weight_max": 51.09579086303711,
      "activations/layer9_attention_weight_min": -38.35152816772461,
      "epoch": 2.86,
      "learning_rate": 0.00013290075757575756,
      "loss": 2.9356,
      "step": 49200
    },
    {
      "activations/layer0_attention_weight_max": 16.63510513305664,
      "activations/layer0_attention_weight_min": -11.314225196838379,
      "activations/layer10_attention_weight_max": 49.53430938720703,
      "activations/layer10_attention_weight_min": -35.149574279785156,
      "activations/layer11_attention_weight_max": 51.08253479003906,
      "activations/layer11_attention_weight_min": -36.058692932128906,
      "activations/layer12_attention_weight_max": 58.32040786743164,
      "activations/layer12_attention_weight_min": -37.088802337646484,
      "activations/layer13_attention_weight_max": 70.23464965820312,
      "activations/layer13_attention_weight_min": -45.78144073486328,
      "activations/layer14_attention_weight_max": 88.43766784667969,
      "activations/layer14_attention_weight_min": -61.077903747558594,
      "activations/layer15_attention_weight_max": 54.47300338745117,
      "activations/layer15_attention_weight_min": -33.899269104003906,
      "activations/layer16_attention_weight_max": 49.70417022705078,
      "activations/layer16_attention_weight_min": -37.61546325683594,
      "activations/layer17_attention_weight_max": 42.08003616333008,
      "activations/layer17_attention_weight_min": -25.646268844604492,
      "activations/layer18_attention_weight_max": 26.20968246459961,
      "activations/layer18_attention_weight_min": -20.042646408081055,
      "activations/layer19_attention_weight_max": 49.1220588684082,
      "activations/layer19_attention_weight_min": -34.49581527709961,
      "activations/layer1_attention_weight_max": 18.794836044311523,
      "activations/layer1_attention_weight_min": -11.342146873474121,
      "activations/layer20_attention_weight_max": 42.8309211730957,
      "activations/layer20_attention_weight_min": -24.594552993774414,
      "activations/layer21_attention_weight_max": 45.28815841674805,
      "activations/layer21_attention_weight_min": -22.982208251953125,
      "activations/layer22_attention_weight_max": 47.81929397583008,
      "activations/layer22_attention_weight_min": -26.40715217590332,
      "activations/layer23_attention_weight_max": 24.822675704956055,
      "activations/layer23_attention_weight_min": -17.70907211303711,
      "activations/layer2_attention_weight_max": 31.254148483276367,
      "activations/layer2_attention_weight_min": -29.216474533081055,
      "activations/layer3_attention_weight_max": 82.32058715820312,
      "activations/layer3_attention_weight_min": -83.7541732788086,
      "activations/layer4_attention_weight_max": 138.8108367919922,
      "activations/layer4_attention_weight_min": -122.40758514404297,
      "activations/layer5_attention_weight_max": 109.22002410888672,
      "activations/layer5_attention_weight_min": -92.75712585449219,
      "activations/layer6_attention_weight_max": 59.99136734008789,
      "activations/layer6_attention_weight_min": -50.82080841064453,
      "activations/layer7_attention_weight_max": 78.4410171508789,
      "activations/layer7_attention_weight_min": -67.16546630859375,
      "activations/layer8_attention_weight_max": 47.39306640625,
      "activations/layer8_attention_weight_min": -38.0028076171875,
      "activations/layer9_attention_weight_max": 71.33514404296875,
      "activations/layer9_attention_weight_min": -52.077056884765625,
      "epoch": 2.86,
      "learning_rate": 0.00013288181818181817,
      "loss": 2.916,
      "step": 49250
    },
    {
      "activations/layer0_attention_weight_max": 18.04523277282715,
      "activations/layer0_attention_weight_min": -12.375714302062988,
      "activations/layer10_attention_weight_max": 38.74629592895508,
      "activations/layer10_attention_weight_min": -33.20579147338867,
      "activations/layer11_attention_weight_max": 35.39702606201172,
      "activations/layer11_attention_weight_min": -33.38056182861328,
      "activations/layer12_attention_weight_max": 35.96683883666992,
      "activations/layer12_attention_weight_min": -26.887666702270508,
      "activations/layer13_attention_weight_max": 47.13382339477539,
      "activations/layer13_attention_weight_min": -35.86009979248047,
      "activations/layer14_attention_weight_max": 69.6313247680664,
      "activations/layer14_attention_weight_min": -53.99631118774414,
      "activations/layer15_attention_weight_max": 46.021156311035156,
      "activations/layer15_attention_weight_min": -34.280399322509766,
      "activations/layer16_attention_weight_max": 43.37063217163086,
      "activations/layer16_attention_weight_min": -32.11484146118164,
      "activations/layer17_attention_weight_max": 43.99884796142578,
      "activations/layer17_attention_weight_min": -29.670082092285156,
      "activations/layer18_attention_weight_max": 30.24525260925293,
      "activations/layer18_attention_weight_min": -22.395891189575195,
      "activations/layer19_attention_weight_max": 45.97416305541992,
      "activations/layer19_attention_weight_min": -34.13903045654297,
      "activations/layer1_attention_weight_max": 17.30146026611328,
      "activations/layer1_attention_weight_min": -13.233015060424805,
      "activations/layer20_attention_weight_max": 35.752628326416016,
      "activations/layer20_attention_weight_min": -22.94448471069336,
      "activations/layer21_attention_weight_max": 39.070648193359375,
      "activations/layer21_attention_weight_min": -22.040014266967773,
      "activations/layer22_attention_weight_max": 52.84392547607422,
      "activations/layer22_attention_weight_min": -27.048887252807617,
      "activations/layer23_attention_weight_max": 30.230985641479492,
      "activations/layer23_attention_weight_min": -17.242904663085938,
      "activations/layer2_attention_weight_max": 29.618446350097656,
      "activations/layer2_attention_weight_min": -27.999311447143555,
      "activations/layer3_attention_weight_max": 76.94690704345703,
      "activations/layer3_attention_weight_min": -78.76648712158203,
      "activations/layer4_attention_weight_max": 127.49757385253906,
      "activations/layer4_attention_weight_min": -121.70953369140625,
      "activations/layer5_attention_weight_max": 107.10881042480469,
      "activations/layer5_attention_weight_min": -91.02413177490234,
      "activations/layer6_attention_weight_max": 60.745582580566406,
      "activations/layer6_attention_weight_min": -53.0047721862793,
      "activations/layer7_attention_weight_max": 68.85271453857422,
      "activations/layer7_attention_weight_min": -69.00455474853516,
      "activations/layer8_attention_weight_max": 45.00335693359375,
      "activations/layer8_attention_weight_min": -37.068416595458984,
      "activations/layer9_attention_weight_max": 48.7091178894043,
      "activations/layer9_attention_weight_min": -41.18927764892578,
      "epoch": 2.86,
      "learning_rate": 0.0001328628787878788,
      "loss": 2.909,
      "step": 49300
    },
    {
      "activations/layer0_attention_weight_max": 15.837075233459473,
      "activations/layer0_attention_weight_min": -12.18132209777832,
      "activations/layer10_attention_weight_max": 43.03556442260742,
      "activations/layer10_attention_weight_min": -34.84510040283203,
      "activations/layer11_attention_weight_max": 37.53685760498047,
      "activations/layer11_attention_weight_min": -28.908523559570312,
      "activations/layer12_attention_weight_max": 39.262298583984375,
      "activations/layer12_attention_weight_min": -28.49081039428711,
      "activations/layer13_attention_weight_max": 46.81584548950195,
      "activations/layer13_attention_weight_min": -35.22943115234375,
      "activations/layer14_attention_weight_max": 69.26812744140625,
      "activations/layer14_attention_weight_min": -47.94688415527344,
      "activations/layer15_attention_weight_max": 49.90780258178711,
      "activations/layer15_attention_weight_min": -31.967220306396484,
      "activations/layer16_attention_weight_max": 50.800071716308594,
      "activations/layer16_attention_weight_min": -36.797611236572266,
      "activations/layer17_attention_weight_max": 43.72208786010742,
      "activations/layer17_attention_weight_min": -26.058032989501953,
      "activations/layer18_attention_weight_max": 31.199447631835938,
      "activations/layer18_attention_weight_min": -20.70305061340332,
      "activations/layer19_attention_weight_max": 43.05363082885742,
      "activations/layer19_attention_weight_min": -31.57131004333496,
      "activations/layer1_attention_weight_max": 16.81529426574707,
      "activations/layer1_attention_weight_min": -12.5272216796875,
      "activations/layer20_attention_weight_max": 34.21750259399414,
      "activations/layer20_attention_weight_min": -24.667234420776367,
      "activations/layer21_attention_weight_max": 31.860158920288086,
      "activations/layer21_attention_weight_min": -19.059864044189453,
      "activations/layer22_attention_weight_max": 43.23977279663086,
      "activations/layer22_attention_weight_min": -27.853818893432617,
      "activations/layer23_attention_weight_max": 26.50777244567871,
      "activations/layer23_attention_weight_min": -18.70215606689453,
      "activations/layer2_attention_weight_max": 29.106903076171875,
      "activations/layer2_attention_weight_min": -28.622785568237305,
      "activations/layer3_attention_weight_max": 76.94241333007812,
      "activations/layer3_attention_weight_min": -79.85950469970703,
      "activations/layer4_attention_weight_max": 126.77298736572266,
      "activations/layer4_attention_weight_min": -126.89861297607422,
      "activations/layer5_attention_weight_max": 101.49179077148438,
      "activations/layer5_attention_weight_min": -87.7075424194336,
      "activations/layer6_attention_weight_max": 56.918277740478516,
      "activations/layer6_attention_weight_min": -50.23851776123047,
      "activations/layer7_attention_weight_max": 67.56661987304688,
      "activations/layer7_attention_weight_min": -65.29630279541016,
      "activations/layer8_attention_weight_max": 40.73615646362305,
      "activations/layer8_attention_weight_min": -37.10310745239258,
      "activations/layer9_attention_weight_max": 48.21757125854492,
      "activations/layer9_attention_weight_min": -39.75971984863281,
      "epoch": 2.87,
      "learning_rate": 0.00013284393939393937,
      "loss": 2.8973,
      "step": 49350
    },
    {
      "activations/layer0_attention_weight_max": 18.55167579650879,
      "activations/layer0_attention_weight_min": -12.238556861877441,
      "activations/layer10_attention_weight_max": 42.3986930847168,
      "activations/layer10_attention_weight_min": -33.764183044433594,
      "activations/layer11_attention_weight_max": 40.10227966308594,
      "activations/layer11_attention_weight_min": -33.86737823486328,
      "activations/layer12_attention_weight_max": 56.243709564208984,
      "activations/layer12_attention_weight_min": -27.825706481933594,
      "activations/layer13_attention_weight_max": 53.929786682128906,
      "activations/layer13_attention_weight_min": -38.59413528442383,
      "activations/layer14_attention_weight_max": 74.84004211425781,
      "activations/layer14_attention_weight_min": -52.65739440917969,
      "activations/layer15_attention_weight_max": 48.85182571411133,
      "activations/layer15_attention_weight_min": -33.11616897583008,
      "activations/layer16_attention_weight_max": 60.677547454833984,
      "activations/layer16_attention_weight_min": -33.939117431640625,
      "activations/layer17_attention_weight_max": 50.15190505981445,
      "activations/layer17_attention_weight_min": -27.320552825927734,
      "activations/layer18_attention_weight_max": 37.92308044433594,
      "activations/layer18_attention_weight_min": -20.591665267944336,
      "activations/layer19_attention_weight_max": 58.84133529663086,
      "activations/layer19_attention_weight_min": -31.8836669921875,
      "activations/layer1_attention_weight_max": 19.047208786010742,
      "activations/layer1_attention_weight_min": -12.709846496582031,
      "activations/layer20_attention_weight_max": 44.33412551879883,
      "activations/layer20_attention_weight_min": -25.630735397338867,
      "activations/layer21_attention_weight_max": 41.11681365966797,
      "activations/layer21_attention_weight_min": -23.841323852539062,
      "activations/layer22_attention_weight_max": 54.79335021972656,
      "activations/layer22_attention_weight_min": -28.31937599182129,
      "activations/layer23_attention_weight_max": 28.581226348876953,
      "activations/layer23_attention_weight_min": -16.473159790039062,
      "activations/layer2_attention_weight_max": 30.025236129760742,
      "activations/layer2_attention_weight_min": -29.888561248779297,
      "activations/layer3_attention_weight_max": 81.56759643554688,
      "activations/layer3_attention_weight_min": -82.46640014648438,
      "activations/layer4_attention_weight_max": 135.20248413085938,
      "activations/layer4_attention_weight_min": -129.09124755859375,
      "activations/layer5_attention_weight_max": 105.72950744628906,
      "activations/layer5_attention_weight_min": -92.3069076538086,
      "activations/layer6_attention_weight_max": 58.9632682800293,
      "activations/layer6_attention_weight_min": -54.9669303894043,
      "activations/layer7_attention_weight_max": 72.9743881225586,
      "activations/layer7_attention_weight_min": -69.43408966064453,
      "activations/layer8_attention_weight_max": 40.66525650024414,
      "activations/layer8_attention_weight_min": -35.58253860473633,
      "activations/layer9_attention_weight_max": 54.210853576660156,
      "activations/layer9_attention_weight_min": -43.47940444946289,
      "epoch": 2.87,
      "learning_rate": 0.000132825,
      "loss": 2.9318,
      "step": 49400
    },
    {
      "activations/layer0_attention_weight_max": 17.146482467651367,
      "activations/layer0_attention_weight_min": -10.767167091369629,
      "activations/layer10_attention_weight_max": 41.38945388793945,
      "activations/layer10_attention_weight_min": -35.50175857543945,
      "activations/layer11_attention_weight_max": 39.158348083496094,
      "activations/layer11_attention_weight_min": -30.524580001831055,
      "activations/layer12_attention_weight_max": 32.27963638305664,
      "activations/layer12_attention_weight_min": -29.528467178344727,
      "activations/layer13_attention_weight_max": 54.35715103149414,
      "activations/layer13_attention_weight_min": -35.20810317993164,
      "activations/layer14_attention_weight_max": 84.31429290771484,
      "activations/layer14_attention_weight_min": -51.00263977050781,
      "activations/layer15_attention_weight_max": 51.00585174560547,
      "activations/layer15_attention_weight_min": -31.28853988647461,
      "activations/layer16_attention_weight_max": 49.21555709838867,
      "activations/layer16_attention_weight_min": -32.97520446777344,
      "activations/layer17_attention_weight_max": 48.3568000793457,
      "activations/layer17_attention_weight_min": -26.932283401489258,
      "activations/layer18_attention_weight_max": 38.251373291015625,
      "activations/layer18_attention_weight_min": -18.607345581054688,
      "activations/layer19_attention_weight_max": 56.1436653137207,
      "activations/layer19_attention_weight_min": -30.380859375,
      "activations/layer1_attention_weight_max": 17.12883949279785,
      "activations/layer1_attention_weight_min": -13.494430541992188,
      "activations/layer20_attention_weight_max": 38.06580352783203,
      "activations/layer20_attention_weight_min": -22.563936233520508,
      "activations/layer21_attention_weight_max": 34.55281448364258,
      "activations/layer21_attention_weight_min": -19.461200714111328,
      "activations/layer22_attention_weight_max": 51.8214111328125,
      "activations/layer22_attention_weight_min": -25.6344051361084,
      "activations/layer23_attention_weight_max": 28.018293380737305,
      "activations/layer23_attention_weight_min": -15.778488159179688,
      "activations/layer2_attention_weight_max": 31.57710075378418,
      "activations/layer2_attention_weight_min": -30.861282348632812,
      "activations/layer3_attention_weight_max": 85.15567016601562,
      "activations/layer3_attention_weight_min": -83.90731048583984,
      "activations/layer4_attention_weight_max": 133.6019744873047,
      "activations/layer4_attention_weight_min": -123.852294921875,
      "activations/layer5_attention_weight_max": 106.93095397949219,
      "activations/layer5_attention_weight_min": -94.04853820800781,
      "activations/layer6_attention_weight_max": 60.658897399902344,
      "activations/layer6_attention_weight_min": -53.18503189086914,
      "activations/layer7_attention_weight_max": 73.42098236083984,
      "activations/layer7_attention_weight_min": -67.29137420654297,
      "activations/layer8_attention_weight_max": 41.41533279418945,
      "activations/layer8_attention_weight_min": -37.74629211425781,
      "activations/layer9_attention_weight_max": 52.83635330200195,
      "activations/layer9_attention_weight_min": -46.884674072265625,
      "epoch": 2.87,
      "learning_rate": 0.0001328060606060606,
      "loss": 2.9164,
      "step": 49450
    },
    {
      "activations/layer0_attention_weight_max": 17.98855972290039,
      "activations/layer0_attention_weight_min": -11.893115043640137,
      "activations/layer10_attention_weight_max": 44.98029708862305,
      "activations/layer10_attention_weight_min": -37.07168197631836,
      "activations/layer11_attention_weight_max": 38.31683349609375,
      "activations/layer11_attention_weight_min": -32.26005554199219,
      "activations/layer12_attention_weight_max": 26.973026275634766,
      "activations/layer12_attention_weight_min": -27.704721450805664,
      "activations/layer13_attention_weight_max": 49.81242752075195,
      "activations/layer13_attention_weight_min": -39.32618713378906,
      "activations/layer14_attention_weight_max": 65.25263214111328,
      "activations/layer14_attention_weight_min": -52.079345703125,
      "activations/layer15_attention_weight_max": 47.34272003173828,
      "activations/layer15_attention_weight_min": -33.29557800292969,
      "activations/layer16_attention_weight_max": 41.7850456237793,
      "activations/layer16_attention_weight_min": -32.272377014160156,
      "activations/layer17_attention_weight_max": 43.07769775390625,
      "activations/layer17_attention_weight_min": -27.51846694946289,
      "activations/layer18_attention_weight_max": 28.341035842895508,
      "activations/layer18_attention_weight_min": -19.134057998657227,
      "activations/layer19_attention_weight_max": 45.30979537963867,
      "activations/layer19_attention_weight_min": -29.813852310180664,
      "activations/layer1_attention_weight_max": 17.32832908630371,
      "activations/layer1_attention_weight_min": -12.446972846984863,
      "activations/layer20_attention_weight_max": 34.34459686279297,
      "activations/layer20_attention_weight_min": -22.552391052246094,
      "activations/layer21_attention_weight_max": 26.63079833984375,
      "activations/layer21_attention_weight_min": -20.152353286743164,
      "activations/layer22_attention_weight_max": 50.79976272583008,
      "activations/layer22_attention_weight_min": -27.13804817199707,
      "activations/layer23_attention_weight_max": 27.109729766845703,
      "activations/layer23_attention_weight_min": -17.643075942993164,
      "activations/layer2_attention_weight_max": 31.905439376831055,
      "activations/layer2_attention_weight_min": -31.115665435791016,
      "activations/layer3_attention_weight_max": 82.33332061767578,
      "activations/layer3_attention_weight_min": -83.71900177001953,
      "activations/layer4_attention_weight_max": 132.6449432373047,
      "activations/layer4_attention_weight_min": -125.21439361572266,
      "activations/layer5_attention_weight_max": 104.0538558959961,
      "activations/layer5_attention_weight_min": -97.3602294921875,
      "activations/layer6_attention_weight_max": 60.00151062011719,
      "activations/layer6_attention_weight_min": -55.581607818603516,
      "activations/layer7_attention_weight_max": 72.31947326660156,
      "activations/layer7_attention_weight_min": -72.94088745117188,
      "activations/layer8_attention_weight_max": 45.48398971557617,
      "activations/layer8_attention_weight_min": -38.988525390625,
      "activations/layer9_attention_weight_max": 52.981292724609375,
      "activations/layer9_attention_weight_min": -39.2995719909668,
      "epoch": 2.88,
      "learning_rate": 0.0001327871212121212,
      "loss": 2.9121,
      "step": 49500
    },
    {
      "activations/layer0_attention_weight_max": 16.777385711669922,
      "activations/layer0_attention_weight_min": -10.998437881469727,
      "activations/layer10_attention_weight_max": 42.76921844482422,
      "activations/layer10_attention_weight_min": -35.08179473876953,
      "activations/layer11_attention_weight_max": 34.78058624267578,
      "activations/layer11_attention_weight_min": -29.380348205566406,
      "activations/layer12_attention_weight_max": 33.87630081176758,
      "activations/layer12_attention_weight_min": -29.295223236083984,
      "activations/layer13_attention_weight_max": 49.398414611816406,
      "activations/layer13_attention_weight_min": -36.48981475830078,
      "activations/layer14_attention_weight_max": 79.90160369873047,
      "activations/layer14_attention_weight_min": -57.77043533325195,
      "activations/layer15_attention_weight_max": 46.360191345214844,
      "activations/layer15_attention_weight_min": -34.11177062988281,
      "activations/layer16_attention_weight_max": 40.51030731201172,
      "activations/layer16_attention_weight_min": -31.203857421875,
      "activations/layer17_attention_weight_max": 38.14781951904297,
      "activations/layer17_attention_weight_min": -25.731348037719727,
      "activations/layer18_attention_weight_max": 29.209270477294922,
      "activations/layer18_attention_weight_min": -19.810422897338867,
      "activations/layer19_attention_weight_max": 47.32652282714844,
      "activations/layer19_attention_weight_min": -30.475778579711914,
      "activations/layer1_attention_weight_max": 17.75128173828125,
      "activations/layer1_attention_weight_min": -14.843706130981445,
      "activations/layer20_attention_weight_max": 34.14496612548828,
      "activations/layer20_attention_weight_min": -23.621374130249023,
      "activations/layer21_attention_weight_max": 31.904033660888672,
      "activations/layer21_attention_weight_min": -20.77408790588379,
      "activations/layer22_attention_weight_max": 46.844482421875,
      "activations/layer22_attention_weight_min": -27.641918182373047,
      "activations/layer23_attention_weight_max": 31.03152847290039,
      "activations/layer23_attention_weight_min": -17.800235748291016,
      "activations/layer2_attention_weight_max": 32.50415802001953,
      "activations/layer2_attention_weight_min": -30.534107208251953,
      "activations/layer3_attention_weight_max": 79.51280212402344,
      "activations/layer3_attention_weight_min": -83.42240905761719,
      "activations/layer4_attention_weight_max": 128.4512176513672,
      "activations/layer4_attention_weight_min": -124.65589904785156,
      "activations/layer5_attention_weight_max": 102.19593811035156,
      "activations/layer5_attention_weight_min": -91.88622283935547,
      "activations/layer6_attention_weight_max": 57.12948989868164,
      "activations/layer6_attention_weight_min": -52.24205780029297,
      "activations/layer7_attention_weight_max": 77.1390380859375,
      "activations/layer7_attention_weight_min": -66.4078369140625,
      "activations/layer8_attention_weight_max": 40.334808349609375,
      "activations/layer8_attention_weight_min": -37.39800262451172,
      "activations/layer9_attention_weight_max": 55.18278884887695,
      "activations/layer9_attention_weight_min": -40.78988265991211,
      "epoch": 2.88,
      "learning_rate": 0.0001327681818181818,
      "loss": 2.9094,
      "step": 49550
    },
    {
      "activations/layer0_attention_weight_max": 14.591081619262695,
      "activations/layer0_attention_weight_min": -11.10414981842041,
      "activations/layer10_attention_weight_max": 45.37588882446289,
      "activations/layer10_attention_weight_min": -34.867000579833984,
      "activations/layer11_attention_weight_max": 51.24197006225586,
      "activations/layer11_attention_weight_min": -37.0678596496582,
      "activations/layer12_attention_weight_max": 38.54410171508789,
      "activations/layer12_attention_weight_min": -28.868864059448242,
      "activations/layer13_attention_weight_max": 56.19860076904297,
      "activations/layer13_attention_weight_min": -43.634544372558594,
      "activations/layer14_attention_weight_max": 71.61847686767578,
      "activations/layer14_attention_weight_min": -52.946815490722656,
      "activations/layer15_attention_weight_max": 62.695213317871094,
      "activations/layer15_attention_weight_min": -33.87766647338867,
      "activations/layer16_attention_weight_max": 41.61627197265625,
      "activations/layer16_attention_weight_min": -31.64504051208496,
      "activations/layer17_attention_weight_max": 44.72151184082031,
      "activations/layer17_attention_weight_min": -27.170074462890625,
      "activations/layer18_attention_weight_max": 35.10831069946289,
      "activations/layer18_attention_weight_min": -20.054214477539062,
      "activations/layer19_attention_weight_max": 53.643795013427734,
      "activations/layer19_attention_weight_min": -31.68890953063965,
      "activations/layer1_attention_weight_max": 17.475418090820312,
      "activations/layer1_attention_weight_min": -12.287402153015137,
      "activations/layer20_attention_weight_max": 38.67266845703125,
      "activations/layer20_attention_weight_min": -22.73149871826172,
      "activations/layer21_attention_weight_max": 42.13499069213867,
      "activations/layer21_attention_weight_min": -24.562768936157227,
      "activations/layer22_attention_weight_max": 49.91255569458008,
      "activations/layer22_attention_weight_min": -26.555505752563477,
      "activations/layer23_attention_weight_max": 26.274715423583984,
      "activations/layer23_attention_weight_min": -15.154440879821777,
      "activations/layer2_attention_weight_max": 31.922718048095703,
      "activations/layer2_attention_weight_min": -30.252958297729492,
      "activations/layer3_attention_weight_max": 85.46293640136719,
      "activations/layer3_attention_weight_min": -83.2122573852539,
      "activations/layer4_attention_weight_max": 142.4676513671875,
      "activations/layer4_attention_weight_min": -130.92234802246094,
      "activations/layer5_attention_weight_max": 115.77310180664062,
      "activations/layer5_attention_weight_min": -100.02816009521484,
      "activations/layer6_attention_weight_max": 64.48075866699219,
      "activations/layer6_attention_weight_min": -54.4522819519043,
      "activations/layer7_attention_weight_max": 83.68769836425781,
      "activations/layer7_attention_weight_min": -68.47417449951172,
      "activations/layer8_attention_weight_max": 46.133087158203125,
      "activations/layer8_attention_weight_min": -38.176422119140625,
      "activations/layer9_attention_weight_max": 67.61071014404297,
      "activations/layer9_attention_weight_min": -50.49169158935547,
      "epoch": 2.88,
      "learning_rate": 0.00013274924242424242,
      "loss": 2.9146,
      "step": 49600
    },
    {
      "activations/layer0_attention_weight_max": 17.923715591430664,
      "activations/layer0_attention_weight_min": -11.26709270477295,
      "activations/layer10_attention_weight_max": 42.1072883605957,
      "activations/layer10_attention_weight_min": -33.08679962158203,
      "activations/layer11_attention_weight_max": 39.855369567871094,
      "activations/layer11_attention_weight_min": -30.440542221069336,
      "activations/layer12_attention_weight_max": 44.07106399536133,
      "activations/layer12_attention_weight_min": -31.921785354614258,
      "activations/layer13_attention_weight_max": 65.9606704711914,
      "activations/layer13_attention_weight_min": -52.76970291137695,
      "activations/layer14_attention_weight_max": 67.70700073242188,
      "activations/layer14_attention_weight_min": -50.54745101928711,
      "activations/layer15_attention_weight_max": 48.60646057128906,
      "activations/layer15_attention_weight_min": -39.8509635925293,
      "activations/layer16_attention_weight_max": 44.24317932128906,
      "activations/layer16_attention_weight_min": -30.53837013244629,
      "activations/layer17_attention_weight_max": 45.04115676879883,
      "activations/layer17_attention_weight_min": -25.657331466674805,
      "activations/layer18_attention_weight_max": 29.828304290771484,
      "activations/layer18_attention_weight_min": -19.83844757080078,
      "activations/layer19_attention_weight_max": 44.08134460449219,
      "activations/layer19_attention_weight_min": -31.830570220947266,
      "activations/layer1_attention_weight_max": 19.363346099853516,
      "activations/layer1_attention_weight_min": -13.673266410827637,
      "activations/layer20_attention_weight_max": 37.56100845336914,
      "activations/layer20_attention_weight_min": -25.454296112060547,
      "activations/layer21_attention_weight_max": 28.85909080505371,
      "activations/layer21_attention_weight_min": -18.863967895507812,
      "activations/layer22_attention_weight_max": 45.23033142089844,
      "activations/layer22_attention_weight_min": -27.3397274017334,
      "activations/layer23_attention_weight_max": 29.276763916015625,
      "activations/layer23_attention_weight_min": -18.53433609008789,
      "activations/layer2_attention_weight_max": 28.288631439208984,
      "activations/layer2_attention_weight_min": -28.937271118164062,
      "activations/layer3_attention_weight_max": 74.34358978271484,
      "activations/layer3_attention_weight_min": -77.89859008789062,
      "activations/layer4_attention_weight_max": 126.7353286743164,
      "activations/layer4_attention_weight_min": -119.5124740600586,
      "activations/layer5_attention_weight_max": 107.75808715820312,
      "activations/layer5_attention_weight_min": -88.06925964355469,
      "activations/layer6_attention_weight_max": 61.92209243774414,
      "activations/layer6_attention_weight_min": -52.8205680847168,
      "activations/layer7_attention_weight_max": 75.79788208007812,
      "activations/layer7_attention_weight_min": -66.85460662841797,
      "activations/layer8_attention_weight_max": 42.17068862915039,
      "activations/layer8_attention_weight_min": -36.288116455078125,
      "activations/layer9_attention_weight_max": 70.0315170288086,
      "activations/layer9_attention_weight_min": -53.95051193237305,
      "epoch": 2.88,
      "learning_rate": 0.00013273030303030304,
      "loss": 2.8985,
      "step": 49650
    },
    {
      "activations/layer0_attention_weight_max": 17.950817108154297,
      "activations/layer0_attention_weight_min": -11.05724048614502,
      "activations/layer10_attention_weight_max": 43.43122100830078,
      "activations/layer10_attention_weight_min": -38.3255500793457,
      "activations/layer11_attention_weight_max": 38.35966110229492,
      "activations/layer11_attention_weight_min": -32.958351135253906,
      "activations/layer12_attention_weight_max": 49.865966796875,
      "activations/layer12_attention_weight_min": -26.391536712646484,
      "activations/layer13_attention_weight_max": 55.76380157470703,
      "activations/layer13_attention_weight_min": -40.837303161621094,
      "activations/layer14_attention_weight_max": 71.84270477294922,
      "activations/layer14_attention_weight_min": -54.83359146118164,
      "activations/layer15_attention_weight_max": 48.90511703491211,
      "activations/layer15_attention_weight_min": -33.806331634521484,
      "activations/layer16_attention_weight_max": 46.1247673034668,
      "activations/layer16_attention_weight_min": -30.17763328552246,
      "activations/layer17_attention_weight_max": 45.07115173339844,
      "activations/layer17_attention_weight_min": -27.511789321899414,
      "activations/layer18_attention_weight_max": 31.383108139038086,
      "activations/layer18_attention_weight_min": -18.447057723999023,
      "activations/layer19_attention_weight_max": 51.052303314208984,
      "activations/layer19_attention_weight_min": -28.658660888671875,
      "activations/layer1_attention_weight_max": 19.113346099853516,
      "activations/layer1_attention_weight_min": -13.519086837768555,
      "activations/layer20_attention_weight_max": 38.165428161621094,
      "activations/layer20_attention_weight_min": -22.8522891998291,
      "activations/layer21_attention_weight_max": 35.11723709106445,
      "activations/layer21_attention_weight_min": -20.94795036315918,
      "activations/layer22_attention_weight_max": 51.4713134765625,
      "activations/layer22_attention_weight_min": -27.14670181274414,
      "activations/layer23_attention_weight_max": 26.803070068359375,
      "activations/layer23_attention_weight_min": -18.829349517822266,
      "activations/layer2_attention_weight_max": 30.651382446289062,
      "activations/layer2_attention_weight_min": -30.58847427368164,
      "activations/layer3_attention_weight_max": 79.69315338134766,
      "activations/layer3_attention_weight_min": -81.56742858886719,
      "activations/layer4_attention_weight_max": 134.7533416748047,
      "activations/layer4_attention_weight_min": -126.606689453125,
      "activations/layer5_attention_weight_max": 104.68404388427734,
      "activations/layer5_attention_weight_min": -95.18050384521484,
      "activations/layer6_attention_weight_max": 57.31406021118164,
      "activations/layer6_attention_weight_min": -53.583736419677734,
      "activations/layer7_attention_weight_max": 73.40404510498047,
      "activations/layer7_attention_weight_min": -67.62786102294922,
      "activations/layer8_attention_weight_max": 45.1893424987793,
      "activations/layer8_attention_weight_min": -37.0423583984375,
      "activations/layer9_attention_weight_max": 48.38863754272461,
      "activations/layer9_attention_weight_min": -41.789085388183594,
      "epoch": 2.89,
      "learning_rate": 0.00013271136363636363,
      "loss": 2.9163,
      "step": 49700
    },
    {
      "activations/layer0_attention_weight_max": 17.66053009033203,
      "activations/layer0_attention_weight_min": -11.47311019897461,
      "activations/layer10_attention_weight_max": 50.14765548706055,
      "activations/layer10_attention_weight_min": -37.424964904785156,
      "activations/layer11_attention_weight_max": 35.793914794921875,
      "activations/layer11_attention_weight_min": -30.028701782226562,
      "activations/layer12_attention_weight_max": 32.046478271484375,
      "activations/layer12_attention_weight_min": -26.614585876464844,
      "activations/layer13_attention_weight_max": 49.5258674621582,
      "activations/layer13_attention_weight_min": -33.70151138305664,
      "activations/layer14_attention_weight_max": 86.27334594726562,
      "activations/layer14_attention_weight_min": -56.46276092529297,
      "activations/layer15_attention_weight_max": 54.70729446411133,
      "activations/layer15_attention_weight_min": -35.39146041870117,
      "activations/layer16_attention_weight_max": 44.46131896972656,
      "activations/layer16_attention_weight_min": -32.58161544799805,
      "activations/layer17_attention_weight_max": 44.67829132080078,
      "activations/layer17_attention_weight_min": -26.866849899291992,
      "activations/layer18_attention_weight_max": 29.37000846862793,
      "activations/layer18_attention_weight_min": -17.98645782470703,
      "activations/layer19_attention_weight_max": 44.676753997802734,
      "activations/layer19_attention_weight_min": -32.57007598876953,
      "activations/layer1_attention_weight_max": 17.472219467163086,
      "activations/layer1_attention_weight_min": -13.452071189880371,
      "activations/layer20_attention_weight_max": 36.30009460449219,
      "activations/layer20_attention_weight_min": -23.33980369567871,
      "activations/layer21_attention_weight_max": 28.99898910522461,
      "activations/layer21_attention_weight_min": -19.465789794921875,
      "activations/layer22_attention_weight_max": 49.99134063720703,
      "activations/layer22_attention_weight_min": -27.308767318725586,
      "activations/layer23_attention_weight_max": 26.943134307861328,
      "activations/layer23_attention_weight_min": -17.735126495361328,
      "activations/layer2_attention_weight_max": 29.14845848083496,
      "activations/layer2_attention_weight_min": -29.16305160522461,
      "activations/layer3_attention_weight_max": 75.22321319580078,
      "activations/layer3_attention_weight_min": -80.97440338134766,
      "activations/layer4_attention_weight_max": 129.10772705078125,
      "activations/layer4_attention_weight_min": -117.0015640258789,
      "activations/layer5_attention_weight_max": 98.51616668701172,
      "activations/layer5_attention_weight_min": -89.37872314453125,
      "activations/layer6_attention_weight_max": 55.79322052001953,
      "activations/layer6_attention_weight_min": -55.12488555908203,
      "activations/layer7_attention_weight_max": 72.06780242919922,
      "activations/layer7_attention_weight_min": -70.73295593261719,
      "activations/layer8_attention_weight_max": 40.70642852783203,
      "activations/layer8_attention_weight_min": -36.614444732666016,
      "activations/layer9_attention_weight_max": 50.75421905517578,
      "activations/layer9_attention_weight_min": -45.11404800415039,
      "epoch": 2.89,
      "learning_rate": 0.00013269242424242424,
      "loss": 2.9154,
      "step": 49750
    },
    {
      "activations/layer0_attention_weight_max": 16.639497756958008,
      "activations/layer0_attention_weight_min": -10.927901268005371,
      "activations/layer10_attention_weight_max": 43.78818130493164,
      "activations/layer10_attention_weight_min": -35.266685485839844,
      "activations/layer11_attention_weight_max": 36.403297424316406,
      "activations/layer11_attention_weight_min": -29.847423553466797,
      "activations/layer12_attention_weight_max": 41.51873779296875,
      "activations/layer12_attention_weight_min": -31.517667770385742,
      "activations/layer13_attention_weight_max": 55.1608772277832,
      "activations/layer13_attention_weight_min": -37.56071853637695,
      "activations/layer14_attention_weight_max": 78.23890686035156,
      "activations/layer14_attention_weight_min": -59.619102478027344,
      "activations/layer15_attention_weight_max": 49.93577194213867,
      "activations/layer15_attention_weight_min": -34.17885208129883,
      "activations/layer16_attention_weight_max": 46.72478103637695,
      "activations/layer16_attention_weight_min": -33.014320373535156,
      "activations/layer17_attention_weight_max": 47.091041564941406,
      "activations/layer17_attention_weight_min": -29.400320053100586,
      "activations/layer18_attention_weight_max": 30.173063278198242,
      "activations/layer18_attention_weight_min": -19.56830406188965,
      "activations/layer19_attention_weight_max": 50.47024154663086,
      "activations/layer19_attention_weight_min": -28.85407829284668,
      "activations/layer1_attention_weight_max": 17.83331298828125,
      "activations/layer1_attention_weight_min": -13.221012115478516,
      "activations/layer20_attention_weight_max": 38.433292388916016,
      "activations/layer20_attention_weight_min": -26.14604377746582,
      "activations/layer21_attention_weight_max": 34.713890075683594,
      "activations/layer21_attention_weight_min": -20.363622665405273,
      "activations/layer22_attention_weight_max": 45.50508499145508,
      "activations/layer22_attention_weight_min": -26.955387115478516,
      "activations/layer23_attention_weight_max": 27.135786056518555,
      "activations/layer23_attention_weight_min": -17.513872146606445,
      "activations/layer2_attention_weight_max": 31.642127990722656,
      "activations/layer2_attention_weight_min": -31.97243881225586,
      "activations/layer3_attention_weight_max": 83.65624237060547,
      "activations/layer3_attention_weight_min": -81.53691864013672,
      "activations/layer4_attention_weight_max": 135.9340057373047,
      "activations/layer4_attention_weight_min": -128.048828125,
      "activations/layer5_attention_weight_max": 113.85304260253906,
      "activations/layer5_attention_weight_min": -97.7117919921875,
      "activations/layer6_attention_weight_max": 60.57579803466797,
      "activations/layer6_attention_weight_min": -52.19770431518555,
      "activations/layer7_attention_weight_max": 70.36811828613281,
      "activations/layer7_attention_weight_min": -71.45752716064453,
      "activations/layer8_attention_weight_max": 42.56407928466797,
      "activations/layer8_attention_weight_min": -39.238040924072266,
      "activations/layer9_attention_weight_max": 48.62394714355469,
      "activations/layer9_attention_weight_min": -41.26670455932617,
      "epoch": 2.89,
      "learning_rate": 0.00013267348484848483,
      "loss": 2.9215,
      "step": 49800
    },
    {
      "activations/layer0_attention_weight_max": 15.262569427490234,
      "activations/layer0_attention_weight_min": -11.866361618041992,
      "activations/layer10_attention_weight_max": 44.11935806274414,
      "activations/layer10_attention_weight_min": -34.77073669433594,
      "activations/layer11_attention_weight_max": 40.976131439208984,
      "activations/layer11_attention_weight_min": -31.11385154724121,
      "activations/layer12_attention_weight_max": 48.387611389160156,
      "activations/layer12_attention_weight_min": -31.097658157348633,
      "activations/layer13_attention_weight_max": 52.401187896728516,
      "activations/layer13_attention_weight_min": -40.08061218261719,
      "activations/layer14_attention_weight_max": 77.24334716796875,
      "activations/layer14_attention_weight_min": -56.94816970825195,
      "activations/layer15_attention_weight_max": 47.8950309753418,
      "activations/layer15_attention_weight_min": -32.75178909301758,
      "activations/layer16_attention_weight_max": 43.67195510864258,
      "activations/layer16_attention_weight_min": -32.278472900390625,
      "activations/layer17_attention_weight_max": 44.46611404418945,
      "activations/layer17_attention_weight_min": -26.332794189453125,
      "activations/layer18_attention_weight_max": 30.187894821166992,
      "activations/layer18_attention_weight_min": -18.804058074951172,
      "activations/layer19_attention_weight_max": 48.396949768066406,
      "activations/layer19_attention_weight_min": -30.610401153564453,
      "activations/layer1_attention_weight_max": 17.36569595336914,
      "activations/layer1_attention_weight_min": -13.016717910766602,
      "activations/layer20_attention_weight_max": 34.07808303833008,
      "activations/layer20_attention_weight_min": -24.849637985229492,
      "activations/layer21_attention_weight_max": 35.368160247802734,
      "activations/layer21_attention_weight_min": -20.099430084228516,
      "activations/layer22_attention_weight_max": 53.97918701171875,
      "activations/layer22_attention_weight_min": -26.332000732421875,
      "activations/layer23_attention_weight_max": 27.754024505615234,
      "activations/layer23_attention_weight_min": -18.422584533691406,
      "activations/layer2_attention_weight_max": 30.23162078857422,
      "activations/layer2_attention_weight_min": -29.61819076538086,
      "activations/layer3_attention_weight_max": 81.22107696533203,
      "activations/layer3_attention_weight_min": -83.49130249023438,
      "activations/layer4_attention_weight_max": 133.4773406982422,
      "activations/layer4_attention_weight_min": -127.53932189941406,
      "activations/layer5_attention_weight_max": 107.26364135742188,
      "activations/layer5_attention_weight_min": -92.52030944824219,
      "activations/layer6_attention_weight_max": 67.3812255859375,
      "activations/layer6_attention_weight_min": -52.64793395996094,
      "activations/layer7_attention_weight_max": 79.19409942626953,
      "activations/layer7_attention_weight_min": -65.4422836303711,
      "activations/layer8_attention_weight_max": 43.92741775512695,
      "activations/layer8_attention_weight_min": -36.589080810546875,
      "activations/layer9_attention_weight_max": 58.98613357543945,
      "activations/layer9_attention_weight_min": -47.734439849853516,
      "epoch": 2.9,
      "learning_rate": 0.00013265454545454544,
      "loss": 2.9211,
      "step": 49850
    },
    {
      "activations/layer0_attention_weight_max": 17.19476318359375,
      "activations/layer0_attention_weight_min": -11.170943260192871,
      "activations/layer10_attention_weight_max": 41.69863510131836,
      "activations/layer10_attention_weight_min": -32.76031494140625,
      "activations/layer11_attention_weight_max": 36.595542907714844,
      "activations/layer11_attention_weight_min": -28.56991958618164,
      "activations/layer12_attention_weight_max": 32.27424240112305,
      "activations/layer12_attention_weight_min": -27.308002471923828,
      "activations/layer13_attention_weight_max": 52.583900451660156,
      "activations/layer13_attention_weight_min": -39.769134521484375,
      "activations/layer14_attention_weight_max": 81.85401916503906,
      "activations/layer14_attention_weight_min": -53.595272064208984,
      "activations/layer15_attention_weight_max": 52.59410095214844,
      "activations/layer15_attention_weight_min": -32.79365158081055,
      "activations/layer16_attention_weight_max": 54.42795181274414,
      "activations/layer16_attention_weight_min": -35.183956146240234,
      "activations/layer17_attention_weight_max": 53.73181915283203,
      "activations/layer17_attention_weight_min": -28.065576553344727,
      "activations/layer18_attention_weight_max": 41.236515045166016,
      "activations/layer18_attention_weight_min": -21.035860061645508,
      "activations/layer19_attention_weight_max": 56.25641632080078,
      "activations/layer19_attention_weight_min": -33.381248474121094,
      "activations/layer1_attention_weight_max": 17.099430084228516,
      "activations/layer1_attention_weight_min": -13.61988353729248,
      "activations/layer20_attention_weight_max": 35.90409851074219,
      "activations/layer20_attention_weight_min": -22.600053787231445,
      "activations/layer21_attention_weight_max": 32.223663330078125,
      "activations/layer21_attention_weight_min": -20.181472778320312,
      "activations/layer22_attention_weight_max": 52.86093521118164,
      "activations/layer22_attention_weight_min": -26.56208610534668,
      "activations/layer23_attention_weight_max": 29.836036682128906,
      "activations/layer23_attention_weight_min": -17.682498931884766,
      "activations/layer2_attention_weight_max": 32.73908233642578,
      "activations/layer2_attention_weight_min": -32.77494430541992,
      "activations/layer3_attention_weight_max": 81.72447204589844,
      "activations/layer3_attention_weight_min": -82.1276626586914,
      "activations/layer4_attention_weight_max": 131.08880615234375,
      "activations/layer4_attention_weight_min": -122.16930389404297,
      "activations/layer5_attention_weight_max": 106.36858367919922,
      "activations/layer5_attention_weight_min": -90.80741882324219,
      "activations/layer6_attention_weight_max": 58.20025634765625,
      "activations/layer6_attention_weight_min": -53.63181686401367,
      "activations/layer7_attention_weight_max": 72.31903076171875,
      "activations/layer7_attention_weight_min": -65.86876678466797,
      "activations/layer8_attention_weight_max": 41.696781158447266,
      "activations/layer8_attention_weight_min": -36.274112701416016,
      "activations/layer9_attention_weight_max": 57.418548583984375,
      "activations/layer9_attention_weight_min": -43.915931701660156,
      "epoch": 2.9,
      "learning_rate": 0.00013263560606060606,
      "loss": 2.9096,
      "step": 49900
    },
    {
      "activations/layer0_attention_weight_max": 17.293563842773438,
      "activations/layer0_attention_weight_min": -11.990861892700195,
      "activations/layer10_attention_weight_max": 40.955474853515625,
      "activations/layer10_attention_weight_min": -35.028324127197266,
      "activations/layer11_attention_weight_max": 37.475040435791016,
      "activations/layer11_attention_weight_min": -30.313169479370117,
      "activations/layer12_attention_weight_max": 49.65934753417969,
      "activations/layer12_attention_weight_min": -30.676807403564453,
      "activations/layer13_attention_weight_max": 48.51137161254883,
      "activations/layer13_attention_weight_min": -35.67063522338867,
      "activations/layer14_attention_weight_max": 68.84749603271484,
      "activations/layer14_attention_weight_min": -54.32801055908203,
      "activations/layer15_attention_weight_max": 50.581661224365234,
      "activations/layer15_attention_weight_min": -33.25950241088867,
      "activations/layer16_attention_weight_max": 44.68312454223633,
      "activations/layer16_attention_weight_min": -30.336259841918945,
      "activations/layer17_attention_weight_max": 42.218570709228516,
      "activations/layer17_attention_weight_min": -27.53291893005371,
      "activations/layer18_attention_weight_max": 32.10710906982422,
      "activations/layer18_attention_weight_min": -18.99124526977539,
      "activations/layer19_attention_weight_max": 48.0687370300293,
      "activations/layer19_attention_weight_min": -32.495845794677734,
      "activations/layer1_attention_weight_max": 18.18950653076172,
      "activations/layer1_attention_weight_min": -13.540755271911621,
      "activations/layer20_attention_weight_max": 36.857975006103516,
      "activations/layer20_attention_weight_min": -24.01774787902832,
      "activations/layer21_attention_weight_max": 31.681119918823242,
      "activations/layer21_attention_weight_min": -20.19446563720703,
      "activations/layer22_attention_weight_max": 45.390464782714844,
      "activations/layer22_attention_weight_min": -27.48147201538086,
      "activations/layer23_attention_weight_max": 26.947071075439453,
      "activations/layer23_attention_weight_min": -18.121843338012695,
      "activations/layer2_attention_weight_max": 31.531959533691406,
      "activations/layer2_attention_weight_min": -30.98103141784668,
      "activations/layer3_attention_weight_max": 76.66259765625,
      "activations/layer3_attention_weight_min": -81.12602233886719,
      "activations/layer4_attention_weight_max": 129.45303344726562,
      "activations/layer4_attention_weight_min": -121.32142639160156,
      "activations/layer5_attention_weight_max": 103.853759765625,
      "activations/layer5_attention_weight_min": -92.24671936035156,
      "activations/layer6_attention_weight_max": 57.83589172363281,
      "activations/layer6_attention_weight_min": -53.42842102050781,
      "activations/layer7_attention_weight_max": 70.11351013183594,
      "activations/layer7_attention_weight_min": -69.47383880615234,
      "activations/layer8_attention_weight_max": 41.58073043823242,
      "activations/layer8_attention_weight_min": -34.931396484375,
      "activations/layer9_attention_weight_max": 52.32561111450195,
      "activations/layer9_attention_weight_min": -43.16628646850586,
      "epoch": 2.9,
      "learning_rate": 0.00013261666666666665,
      "loss": 2.9071,
      "step": 49950
    },
    {
      "activations/layer0_attention_weight_max": 17.90970230102539,
      "activations/layer0_attention_weight_min": -12.121119499206543,
      "activations/layer10_attention_weight_max": 47.20918655395508,
      "activations/layer10_attention_weight_min": -31.51833152770996,
      "activations/layer11_attention_weight_max": 48.33933639526367,
      "activations/layer11_attention_weight_min": -33.72098159790039,
      "activations/layer12_attention_weight_max": 65.10932922363281,
      "activations/layer12_attention_weight_min": -32.76939392089844,
      "activations/layer13_attention_weight_max": 83.46024322509766,
      "activations/layer13_attention_weight_min": -49.293663024902344,
      "activations/layer14_attention_weight_max": 85.06830596923828,
      "activations/layer14_attention_weight_min": -55.91573715209961,
      "activations/layer15_attention_weight_max": 67.34745788574219,
      "activations/layer15_attention_weight_min": -38.2025146484375,
      "activations/layer16_attention_weight_max": 43.131507873535156,
      "activations/layer16_attention_weight_min": -31.986433029174805,
      "activations/layer17_attention_weight_max": 44.6812629699707,
      "activations/layer17_attention_weight_min": -28.566871643066406,
      "activations/layer18_attention_weight_max": 33.699832916259766,
      "activations/layer18_attention_weight_min": -21.126392364501953,
      "activations/layer19_attention_weight_max": 52.9818229675293,
      "activations/layer19_attention_weight_min": -30.107919692993164,
      "activations/layer1_attention_weight_max": 18.871248245239258,
      "activations/layer1_attention_weight_min": -12.973307609558105,
      "activations/layer20_attention_weight_max": 42.978023529052734,
      "activations/layer20_attention_weight_min": -26.515478134155273,
      "activations/layer21_attention_weight_max": 40.0381965637207,
      "activations/layer21_attention_weight_min": -22.242155075073242,
      "activations/layer22_attention_weight_max": 50.87117385864258,
      "activations/layer22_attention_weight_min": -25.403303146362305,
      "activations/layer23_attention_weight_max": 26.381139755249023,
      "activations/layer23_attention_weight_min": -18.327415466308594,
      "activations/layer2_attention_weight_max": 32.61835479736328,
      "activations/layer2_attention_weight_min": -31.857778549194336,
      "activations/layer3_attention_weight_max": 85.84477233886719,
      "activations/layer3_attention_weight_min": -87.45574951171875,
      "activations/layer4_attention_weight_max": 139.4453887939453,
      "activations/layer4_attention_weight_min": -124.78431701660156,
      "activations/layer5_attention_weight_max": 121.76912689208984,
      "activations/layer5_attention_weight_min": -102.20323181152344,
      "activations/layer6_attention_weight_max": 68.0834732055664,
      "activations/layer6_attention_weight_min": -54.00222396850586,
      "activations/layer7_attention_weight_max": 88.4142837524414,
      "activations/layer7_attention_weight_min": -78.86327362060547,
      "activations/layer8_attention_weight_max": 51.5665397644043,
      "activations/layer8_attention_weight_min": -38.15202331542969,
      "activations/layer9_attention_weight_max": 74.0069580078125,
      "activations/layer9_attention_weight_min": -63.775447845458984,
      "epoch": 2.91,
      "learning_rate": 0.00013259772727272726,
      "loss": 2.9098,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_loss": 2.86328125,
      "eval_runtime": 8.5321,
      "eval_samples_per_second": 503.276,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_openwebtext_loss": 2.86328125,
      "eval_openwebtext_ppl": 17.51891667538985,
      "eval_openwebtext_runtime": 8.5321,
      "eval_openwebtext_samples_per_second": 503.276,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_wikitext_loss": 3.1171875,
      "eval_wikitext_ppl": 22.582776184718522,
      "eval_wikitext_runtime": 2.0013,
      "eval_wikitext_samples_per_second": 227.853,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_lambada_loss": 2.947265625,
      "eval_lambada_ppl": 19.053782246709744,
      "eval_lambada_runtime": 9.6021,
      "eval_lambada_samples_per_second": 507.076,
      "step": 50000
    },
    {
      "activations/layer0_attention_weight_max": 15.342806816101074,
      "activations/layer0_attention_weight_min": -11.53200626373291,
      "activations/layer10_attention_weight_max": 42.9250373840332,
      "activations/layer10_attention_weight_min": -35.542510986328125,
      "activations/layer11_attention_weight_max": 38.86156463623047,
      "activations/layer11_attention_weight_min": -32.421146392822266,
      "activations/layer12_attention_weight_max": 59.925987243652344,
      "activations/layer12_attention_weight_min": -39.07342529296875,
      "activations/layer13_attention_weight_max": 44.808677673339844,
      "activations/layer13_attention_weight_min": -32.5633430480957,
      "activations/layer14_attention_weight_max": 72.84832763671875,
      "activations/layer14_attention_weight_min": -61.87649154663086,
      "activations/layer15_attention_weight_max": 48.82334518432617,
      "activations/layer15_attention_weight_min": -36.92494201660156,
      "activations/layer16_attention_weight_max": 47.04798126220703,
      "activations/layer16_attention_weight_min": -33.62732696533203,
      "activations/layer17_attention_weight_max": 41.51369094848633,
      "activations/layer17_attention_weight_min": -27.795429229736328,
      "activations/layer18_attention_weight_max": 28.36208724975586,
      "activations/layer18_attention_weight_min": -19.00732421875,
      "activations/layer19_attention_weight_max": 48.74238586425781,
      "activations/layer19_attention_weight_min": -31.48463249206543,
      "activations/layer1_attention_weight_max": 17.553804397583008,
      "activations/layer1_attention_weight_min": -12.204301834106445,
      "activations/layer20_attention_weight_max": 39.80244445800781,
      "activations/layer20_attention_weight_min": -23.202428817749023,
      "activations/layer21_attention_weight_max": 35.8775520324707,
      "activations/layer21_attention_weight_min": -22.20942497253418,
      "activations/layer22_attention_weight_max": 54.35737609863281,
      "activations/layer22_attention_weight_min": -25.236984252929688,
      "activations/layer23_attention_weight_max": 27.545103073120117,
      "activations/layer23_attention_weight_min": -15.742838859558105,
      "activations/layer2_attention_weight_max": 29.68037223815918,
      "activations/layer2_attention_weight_min": -28.952909469604492,
      "activations/layer3_attention_weight_max": 78.89147186279297,
      "activations/layer3_attention_weight_min": -83.19338989257812,
      "activations/layer4_attention_weight_max": 129.90176391601562,
      "activations/layer4_attention_weight_min": -123.76049041748047,
      "activations/layer5_attention_weight_max": 105.23387145996094,
      "activations/layer5_attention_weight_min": -96.6993637084961,
      "activations/layer6_attention_weight_max": 59.85231399536133,
      "activations/layer6_attention_weight_min": -54.00778579711914,
      "activations/layer7_attention_weight_max": 74.86512756347656,
      "activations/layer7_attention_weight_min": -72.85985565185547,
      "activations/layer8_attention_weight_max": 44.290531158447266,
      "activations/layer8_attention_weight_min": -40.05628204345703,
      "activations/layer9_attention_weight_max": 61.50222396850586,
      "activations/layer9_attention_weight_min": -47.37665939331055,
      "epoch": 2.91,
      "learning_rate": 0.00013257878787878788,
      "loss": 2.9073,
      "step": 50050
    },
    {
      "activations/layer0_attention_weight_max": 17.336116790771484,
      "activations/layer0_attention_weight_min": -11.140677452087402,
      "activations/layer10_attention_weight_max": 48.0041389465332,
      "activations/layer10_attention_weight_min": -36.066932678222656,
      "activations/layer11_attention_weight_max": 37.479705810546875,
      "activations/layer11_attention_weight_min": -29.773038864135742,
      "activations/layer12_attention_weight_max": 32.40670394897461,
      "activations/layer12_attention_weight_min": -24.947834014892578,
      "activations/layer13_attention_weight_max": 50.899681091308594,
      "activations/layer13_attention_weight_min": -36.523441314697266,
      "activations/layer14_attention_weight_max": 64.14827728271484,
      "activations/layer14_attention_weight_min": -51.756839752197266,
      "activations/layer15_attention_weight_max": 47.965389251708984,
      "activations/layer15_attention_weight_min": -34.61653137207031,
      "activations/layer16_attention_weight_max": 44.14271545410156,
      "activations/layer16_attention_weight_min": -30.576290130615234,
      "activations/layer17_attention_weight_max": 44.16510009765625,
      "activations/layer17_attention_weight_min": -29.499359130859375,
      "activations/layer18_attention_weight_max": 27.591398239135742,
      "activations/layer18_attention_weight_min": -18.739715576171875,
      "activations/layer19_attention_weight_max": 49.12501525878906,
      "activations/layer19_attention_weight_min": -29.548831939697266,
      "activations/layer1_attention_weight_max": 18.48470115661621,
      "activations/layer1_attention_weight_min": -13.006050109863281,
      "activations/layer20_attention_weight_max": 37.24313735961914,
      "activations/layer20_attention_weight_min": -24.170183181762695,
      "activations/layer21_attention_weight_max": 34.732078552246094,
      "activations/layer21_attention_weight_min": -19.276226043701172,
      "activations/layer22_attention_weight_max": 48.307220458984375,
      "activations/layer22_attention_weight_min": -25.656633377075195,
      "activations/layer23_attention_weight_max": 27.751480102539062,
      "activations/layer23_attention_weight_min": -18.052593231201172,
      "activations/layer2_attention_weight_max": 28.466167449951172,
      "activations/layer2_attention_weight_min": -30.004470825195312,
      "activations/layer3_attention_weight_max": 76.94395446777344,
      "activations/layer3_attention_weight_min": -78.46284484863281,
      "activations/layer4_attention_weight_max": 134.12803649902344,
      "activations/layer4_attention_weight_min": -122.29996490478516,
      "activations/layer5_attention_weight_max": 107.52546691894531,
      "activations/layer5_attention_weight_min": -96.91502380371094,
      "activations/layer6_attention_weight_max": 64.92921447753906,
      "activations/layer6_attention_weight_min": -54.66040802001953,
      "activations/layer7_attention_weight_max": 70.97447204589844,
      "activations/layer7_attention_weight_min": -73.08344268798828,
      "activations/layer8_attention_weight_max": 42.027122497558594,
      "activations/layer8_attention_weight_min": -35.1243782043457,
      "activations/layer9_attention_weight_max": 49.60944747924805,
      "activations/layer9_attention_weight_min": -37.570438385009766,
      "epoch": 2.91,
      "learning_rate": 0.00013255984848484846,
      "loss": 2.9239,
      "step": 50100
    },
    {
      "activations/layer0_attention_weight_max": 17.101205825805664,
      "activations/layer0_attention_weight_min": -12.200830459594727,
      "activations/layer10_attention_weight_max": 52.69743347167969,
      "activations/layer10_attention_weight_min": -39.260719299316406,
      "activations/layer11_attention_weight_max": 52.311012268066406,
      "activations/layer11_attention_weight_min": -33.09579849243164,
      "activations/layer12_attention_weight_max": 87.10221099853516,
      "activations/layer12_attention_weight_min": -41.44169998168945,
      "activations/layer13_attention_weight_max": 80.59571838378906,
      "activations/layer13_attention_weight_min": -46.66434860229492,
      "activations/layer14_attention_weight_max": 87.36821746826172,
      "activations/layer14_attention_weight_min": -56.924156188964844,
      "activations/layer15_attention_weight_max": 62.26056671142578,
      "activations/layer15_attention_weight_min": -35.777679443359375,
      "activations/layer16_attention_weight_max": 61.415584564208984,
      "activations/layer16_attention_weight_min": -33.16849136352539,
      "activations/layer17_attention_weight_max": 46.93644332885742,
      "activations/layer17_attention_weight_min": -25.85603904724121,
      "activations/layer18_attention_weight_max": 31.631811141967773,
      "activations/layer18_attention_weight_min": -20.52745246887207,
      "activations/layer19_attention_weight_max": 56.38762283325195,
      "activations/layer19_attention_weight_min": -31.592788696289062,
      "activations/layer1_attention_weight_max": 16.891082763671875,
      "activations/layer1_attention_weight_min": -14.479576110839844,
      "activations/layer20_attention_weight_max": 37.212223052978516,
      "activations/layer20_attention_weight_min": -24.328184127807617,
      "activations/layer21_attention_weight_max": 38.192359924316406,
      "activations/layer21_attention_weight_min": -22.0617733001709,
      "activations/layer22_attention_weight_max": 47.70780563354492,
      "activations/layer22_attention_weight_min": -31.97904396057129,
      "activations/layer23_attention_weight_max": 26.8868465423584,
      "activations/layer23_attention_weight_min": -17.079002380371094,
      "activations/layer2_attention_weight_max": 31.058652877807617,
      "activations/layer2_attention_weight_min": -29.880704879760742,
      "activations/layer3_attention_weight_max": 82.92206573486328,
      "activations/layer3_attention_weight_min": -83.82267761230469,
      "activations/layer4_attention_weight_max": 150.27850341796875,
      "activations/layer4_attention_weight_min": -125.11360931396484,
      "activations/layer5_attention_weight_max": 115.09494018554688,
      "activations/layer5_attention_weight_min": -94.85497283935547,
      "activations/layer6_attention_weight_max": 66.60252380371094,
      "activations/layer6_attention_weight_min": -53.34056091308594,
      "activations/layer7_attention_weight_max": 85.85018157958984,
      "activations/layer7_attention_weight_min": -72.44803619384766,
      "activations/layer8_attention_weight_max": 52.269832611083984,
      "activations/layer8_attention_weight_min": -39.831993103027344,
      "activations/layer9_attention_weight_max": 70.54034423828125,
      "activations/layer9_attention_weight_min": -52.880714416503906,
      "epoch": 2.91,
      "learning_rate": 0.00013254090909090908,
      "loss": 2.9022,
      "step": 50150
    },
    {
      "activations/layer0_attention_weight_max": 17.51117515563965,
      "activations/layer0_attention_weight_min": -12.000391960144043,
      "activations/layer10_attention_weight_max": 40.72098159790039,
      "activations/layer10_attention_weight_min": -34.08665466308594,
      "activations/layer11_attention_weight_max": 35.41381072998047,
      "activations/layer11_attention_weight_min": -29.50597381591797,
      "activations/layer12_attention_weight_max": 27.57811164855957,
      "activations/layer12_attention_weight_min": -28.567899703979492,
      "activations/layer13_attention_weight_max": 43.03609085083008,
      "activations/layer13_attention_weight_min": -35.39301300048828,
      "activations/layer14_attention_weight_max": 65.7320556640625,
      "activations/layer14_attention_weight_min": -50.95384216308594,
      "activations/layer15_attention_weight_max": 45.130619049072266,
      "activations/layer15_attention_weight_min": -31.60370635986328,
      "activations/layer16_attention_weight_max": 43.67286682128906,
      "activations/layer16_attention_weight_min": -33.21418762207031,
      "activations/layer17_attention_weight_max": 39.56010437011719,
      "activations/layer17_attention_weight_min": -28.17481803894043,
      "activations/layer18_attention_weight_max": 30.548959732055664,
      "activations/layer18_attention_weight_min": -21.933143615722656,
      "activations/layer19_attention_weight_max": 39.40485763549805,
      "activations/layer19_attention_weight_min": -30.126495361328125,
      "activations/layer1_attention_weight_max": 17.928255081176758,
      "activations/layer1_attention_weight_min": -11.989481925964355,
      "activations/layer20_attention_weight_max": 32.23043441772461,
      "activations/layer20_attention_weight_min": -23.58222007751465,
      "activations/layer21_attention_weight_max": 33.20088195800781,
      "activations/layer21_attention_weight_min": -21.793630599975586,
      "activations/layer22_attention_weight_max": 46.27882385253906,
      "activations/layer22_attention_weight_min": -29.23728370666504,
      "activations/layer23_attention_weight_max": 26.56191062927246,
      "activations/layer23_attention_weight_min": -17.664722442626953,
      "activations/layer2_attention_weight_max": 29.41103172302246,
      "activations/layer2_attention_weight_min": -29.242033004760742,
      "activations/layer3_attention_weight_max": 77.25797271728516,
      "activations/layer3_attention_weight_min": -83.37450408935547,
      "activations/layer4_attention_weight_max": 125.01055145263672,
      "activations/layer4_attention_weight_min": -125.92791748046875,
      "activations/layer5_attention_weight_max": 100.70355224609375,
      "activations/layer5_attention_weight_min": -91.8691635131836,
      "activations/layer6_attention_weight_max": 60.85955810546875,
      "activations/layer6_attention_weight_min": -54.354270935058594,
      "activations/layer7_attention_weight_max": 70.56490325927734,
      "activations/layer7_attention_weight_min": -65.10372161865234,
      "activations/layer8_attention_weight_max": 40.67003631591797,
      "activations/layer8_attention_weight_min": -35.360557556152344,
      "activations/layer9_attention_weight_max": 49.261444091796875,
      "activations/layer9_attention_weight_min": -43.83713150024414,
      "epoch": 2.92,
      "learning_rate": 0.0001325219696969697,
      "loss": 2.9149,
      "step": 50200
    },
    {
      "activations/layer0_attention_weight_max": 17.47227668762207,
      "activations/layer0_attention_weight_min": -12.29432201385498,
      "activations/layer10_attention_weight_max": 43.1381721496582,
      "activations/layer10_attention_weight_min": -35.78496170043945,
      "activations/layer11_attention_weight_max": 35.10104751586914,
      "activations/layer11_attention_weight_min": -29.19427490234375,
      "activations/layer12_attention_weight_max": 42.87420654296875,
      "activations/layer12_attention_weight_min": -30.435989379882812,
      "activations/layer13_attention_weight_max": 48.49767303466797,
      "activations/layer13_attention_weight_min": -35.88166809082031,
      "activations/layer14_attention_weight_max": 71.93538665771484,
      "activations/layer14_attention_weight_min": -52.839210510253906,
      "activations/layer15_attention_weight_max": 47.632816314697266,
      "activations/layer15_attention_weight_min": -31.874073028564453,
      "activations/layer16_attention_weight_max": 43.861717224121094,
      "activations/layer16_attention_weight_min": -29.672365188598633,
      "activations/layer17_attention_weight_max": 44.206912994384766,
      "activations/layer17_attention_weight_min": -28.58972930908203,
      "activations/layer18_attention_weight_max": 35.279720306396484,
      "activations/layer18_attention_weight_min": -21.512062072753906,
      "activations/layer19_attention_weight_max": 44.85359191894531,
      "activations/layer19_attention_weight_min": -30.05409049987793,
      "activations/layer1_attention_weight_max": 16.994678497314453,
      "activations/layer1_attention_weight_min": -12.55582046508789,
      "activations/layer20_attention_weight_max": 33.98966979980469,
      "activations/layer20_attention_weight_min": -22.279077529907227,
      "activations/layer21_attention_weight_max": 34.21930694580078,
      "activations/layer21_attention_weight_min": -21.123411178588867,
      "activations/layer22_attention_weight_max": 50.80384826660156,
      "activations/layer22_attention_weight_min": -27.632797241210938,
      "activations/layer23_attention_weight_max": 27.02264976501465,
      "activations/layer23_attention_weight_min": -20.349132537841797,
      "activations/layer2_attention_weight_max": 29.418216705322266,
      "activations/layer2_attention_weight_min": -29.038787841796875,
      "activations/layer3_attention_weight_max": 78.37721252441406,
      "activations/layer3_attention_weight_min": -80.98991394042969,
      "activations/layer4_attention_weight_max": 132.9202117919922,
      "activations/layer4_attention_weight_min": -129.3836669921875,
      "activations/layer5_attention_weight_max": 108.54368591308594,
      "activations/layer5_attention_weight_min": -94.7845458984375,
      "activations/layer6_attention_weight_max": 57.61431884765625,
      "activations/layer6_attention_weight_min": -52.233985900878906,
      "activations/layer7_attention_weight_max": 68.49063873291016,
      "activations/layer7_attention_weight_min": -66.97518920898438,
      "activations/layer8_attention_weight_max": 40.70989227294922,
      "activations/layer8_attention_weight_min": -36.04113006591797,
      "activations/layer9_attention_weight_max": 58.7195930480957,
      "activations/layer9_attention_weight_min": -40.10609817504883,
      "epoch": 2.92,
      "learning_rate": 0.00013250303030303028,
      "loss": 2.9075,
      "step": 50250
    },
    {
      "activations/layer0_attention_weight_max": 17.4384765625,
      "activations/layer0_attention_weight_min": -11.535954475402832,
      "activations/layer10_attention_weight_max": 43.46907043457031,
      "activations/layer10_attention_weight_min": -35.7138557434082,
      "activations/layer11_attention_weight_max": 36.026798248291016,
      "activations/layer11_attention_weight_min": -30.246082305908203,
      "activations/layer12_attention_weight_max": 56.17848205566406,
      "activations/layer12_attention_weight_min": -34.63772964477539,
      "activations/layer13_attention_weight_max": 61.534873962402344,
      "activations/layer13_attention_weight_min": -49.55329132080078,
      "activations/layer14_attention_weight_max": 79.16947174072266,
      "activations/layer14_attention_weight_min": -64.2240982055664,
      "activations/layer15_attention_weight_max": 46.496768951416016,
      "activations/layer15_attention_weight_min": -31.839807510375977,
      "activations/layer16_attention_weight_max": 48.311458587646484,
      "activations/layer16_attention_weight_min": -32.666507720947266,
      "activations/layer17_attention_weight_max": 43.347537994384766,
      "activations/layer17_attention_weight_min": -27.643280029296875,
      "activations/layer18_attention_weight_max": 29.664413452148438,
      "activations/layer18_attention_weight_min": -18.39850425720215,
      "activations/layer19_attention_weight_max": 50.14392852783203,
      "activations/layer19_attention_weight_min": -29.824798583984375,
      "activations/layer1_attention_weight_max": 18.000194549560547,
      "activations/layer1_attention_weight_min": -12.899114608764648,
      "activations/layer20_attention_weight_max": 35.15765380859375,
      "activations/layer20_attention_weight_min": -23.22513198852539,
      "activations/layer21_attention_weight_max": 40.27312088012695,
      "activations/layer21_attention_weight_min": -23.37236976623535,
      "activations/layer22_attention_weight_max": 46.89860153198242,
      "activations/layer22_attention_weight_min": -27.136919021606445,
      "activations/layer23_attention_weight_max": 29.937679290771484,
      "activations/layer23_attention_weight_min": -20.049686431884766,
      "activations/layer2_attention_weight_max": 30.078842163085938,
      "activations/layer2_attention_weight_min": -28.846820831298828,
      "activations/layer3_attention_weight_max": 78.43836975097656,
      "activations/layer3_attention_weight_min": -81.25675964355469,
      "activations/layer4_attention_weight_max": 128.47718811035156,
      "activations/layer4_attention_weight_min": -131.49644470214844,
      "activations/layer5_attention_weight_max": 101.11592102050781,
      "activations/layer5_attention_weight_min": -99.0457992553711,
      "activations/layer6_attention_weight_max": 57.80256652832031,
      "activations/layer6_attention_weight_min": -58.33939743041992,
      "activations/layer7_attention_weight_max": 70.60931396484375,
      "activations/layer7_attention_weight_min": -68.48834991455078,
      "activations/layer8_attention_weight_max": 39.110965728759766,
      "activations/layer8_attention_weight_min": -36.44243240356445,
      "activations/layer9_attention_weight_max": 50.59159851074219,
      "activations/layer9_attention_weight_min": -47.943809509277344,
      "epoch": 2.92,
      "learning_rate": 0.0001324840909090909,
      "loss": 2.9287,
      "step": 50300
    },
    {
      "activations/layer0_attention_weight_max": 17.522396087646484,
      "activations/layer0_attention_weight_min": -11.45710563659668,
      "activations/layer10_attention_weight_max": 44.254146575927734,
      "activations/layer10_attention_weight_min": -34.82057571411133,
      "activations/layer11_attention_weight_max": 39.308753967285156,
      "activations/layer11_attention_weight_min": -30.857275009155273,
      "activations/layer12_attention_weight_max": 32.403411865234375,
      "activations/layer12_attention_weight_min": -27.966947555541992,
      "activations/layer13_attention_weight_max": 61.2283935546875,
      "activations/layer13_attention_weight_min": -41.01453399658203,
      "activations/layer14_attention_weight_max": 85.50179290771484,
      "activations/layer14_attention_weight_min": -59.41112518310547,
      "activations/layer15_attention_weight_max": 53.274574279785156,
      "activations/layer15_attention_weight_min": -31.776704788208008,
      "activations/layer16_attention_weight_max": 44.847511291503906,
      "activations/layer16_attention_weight_min": -31.70436668395996,
      "activations/layer17_attention_weight_max": 48.04558181762695,
      "activations/layer17_attention_weight_min": -27.6312255859375,
      "activations/layer18_attention_weight_max": 34.10319900512695,
      "activations/layer18_attention_weight_min": -18.15330696105957,
      "activations/layer19_attention_weight_max": 52.43622589111328,
      "activations/layer19_attention_weight_min": -33.67856216430664,
      "activations/layer1_attention_weight_max": 18.42423439025879,
      "activations/layer1_attention_weight_min": -12.780963897705078,
      "activations/layer20_attention_weight_max": 41.39368438720703,
      "activations/layer20_attention_weight_min": -23.026233673095703,
      "activations/layer21_attention_weight_max": 38.25331497192383,
      "activations/layer21_attention_weight_min": -21.055152893066406,
      "activations/layer22_attention_weight_max": 51.69687271118164,
      "activations/layer22_attention_weight_min": -29.16458511352539,
      "activations/layer23_attention_weight_max": 28.241817474365234,
      "activations/layer23_attention_weight_min": -17.000873565673828,
      "activations/layer2_attention_weight_max": 29.145893096923828,
      "activations/layer2_attention_weight_min": -27.89881706237793,
      "activations/layer3_attention_weight_max": 79.26620483398438,
      "activations/layer3_attention_weight_min": -82.18412780761719,
      "activations/layer4_attention_weight_max": 122.99381256103516,
      "activations/layer4_attention_weight_min": -123.39298248291016,
      "activations/layer5_attention_weight_max": 99.62548065185547,
      "activations/layer5_attention_weight_min": -90.79762268066406,
      "activations/layer6_attention_weight_max": 58.33604049682617,
      "activations/layer6_attention_weight_min": -53.847286224365234,
      "activations/layer7_attention_weight_max": 67.67780303955078,
      "activations/layer7_attention_weight_min": -71.66885375976562,
      "activations/layer8_attention_weight_max": 42.73023986816406,
      "activations/layer8_attention_weight_min": -38.8718147277832,
      "activations/layer9_attention_weight_max": 51.54619216918945,
      "activations/layer9_attention_weight_min": -37.154457092285156,
      "epoch": 2.93,
      "learning_rate": 0.00013246515151515149,
      "loss": 2.9247,
      "step": 50350
    },
    {
      "activations/layer0_attention_weight_max": 17.257184982299805,
      "activations/layer0_attention_weight_min": -11.436152458190918,
      "activations/layer10_attention_weight_max": 46.469749450683594,
      "activations/layer10_attention_weight_min": -33.24728012084961,
      "activations/layer11_attention_weight_max": 46.92682647705078,
      "activations/layer11_attention_weight_min": -32.502235412597656,
      "activations/layer12_attention_weight_max": 33.84125518798828,
      "activations/layer12_attention_weight_min": -29.108461380004883,
      "activations/layer13_attention_weight_max": 67.4212646484375,
      "activations/layer13_attention_weight_min": -43.103843688964844,
      "activations/layer14_attention_weight_max": 78.80345153808594,
      "activations/layer14_attention_weight_min": -56.12377166748047,
      "activations/layer15_attention_weight_max": 65.0982666015625,
      "activations/layer15_attention_weight_min": -37.5584831237793,
      "activations/layer16_attention_weight_max": 45.73972702026367,
      "activations/layer16_attention_weight_min": -31.745988845825195,
      "activations/layer17_attention_weight_max": 45.06984329223633,
      "activations/layer17_attention_weight_min": -24.868144989013672,
      "activations/layer18_attention_weight_max": 33.44451141357422,
      "activations/layer18_attention_weight_min": -20.90072250366211,
      "activations/layer19_attention_weight_max": 46.17756271362305,
      "activations/layer19_attention_weight_min": -30.33921241760254,
      "activations/layer1_attention_weight_max": 17.84221076965332,
      "activations/layer1_attention_weight_min": -14.371397972106934,
      "activations/layer20_attention_weight_max": 35.95273971557617,
      "activations/layer20_attention_weight_min": -23.169357299804688,
      "activations/layer21_attention_weight_max": 43.39656448364258,
      "activations/layer21_attention_weight_min": -21.27092933654785,
      "activations/layer22_attention_weight_max": 48.972816467285156,
      "activations/layer22_attention_weight_min": -27.994966506958008,
      "activations/layer23_attention_weight_max": 25.543447494506836,
      "activations/layer23_attention_weight_min": -17.38261604309082,
      "activations/layer2_attention_weight_max": 30.039833068847656,
      "activations/layer2_attention_weight_min": -28.891498565673828,
      "activations/layer3_attention_weight_max": 77.15604400634766,
      "activations/layer3_attention_weight_min": -79.8958740234375,
      "activations/layer4_attention_weight_max": 131.8759002685547,
      "activations/layer4_attention_weight_min": -128.58604431152344,
      "activations/layer5_attention_weight_max": 105.10098266601562,
      "activations/layer5_attention_weight_min": -91.26091003417969,
      "activations/layer6_attention_weight_max": 61.597450256347656,
      "activations/layer6_attention_weight_min": -50.24637985229492,
      "activations/layer7_attention_weight_max": 81.18749237060547,
      "activations/layer7_attention_weight_min": -65.44590759277344,
      "activations/layer8_attention_weight_max": 42.92392349243164,
      "activations/layer8_attention_weight_min": -34.768150329589844,
      "activations/layer9_attention_weight_max": 57.55762481689453,
      "activations/layer9_attention_weight_min": -39.61454391479492,
      "epoch": 2.93,
      "learning_rate": 0.0001324462121212121,
      "loss": 2.9153,
      "step": 50400
    },
    {
      "activations/layer0_attention_weight_max": 17.75586700439453,
      "activations/layer0_attention_weight_min": -11.367186546325684,
      "activations/layer10_attention_weight_max": 42.71104049682617,
      "activations/layer10_attention_weight_min": -37.24665451049805,
      "activations/layer11_attention_weight_max": 37.546173095703125,
      "activations/layer11_attention_weight_min": -29.09669303894043,
      "activations/layer12_attention_weight_max": 39.5079345703125,
      "activations/layer12_attention_weight_min": -32.207191467285156,
      "activations/layer13_attention_weight_max": 46.87820816040039,
      "activations/layer13_attention_weight_min": -33.80257797241211,
      "activations/layer14_attention_weight_max": 88.23857116699219,
      "activations/layer14_attention_weight_min": -59.1768913269043,
      "activations/layer15_attention_weight_max": 48.107173919677734,
      "activations/layer15_attention_weight_min": -33.381385803222656,
      "activations/layer16_attention_weight_max": 51.72250747680664,
      "activations/layer16_attention_weight_min": -32.51835250854492,
      "activations/layer17_attention_weight_max": 44.15676498413086,
      "activations/layer17_attention_weight_min": -28.018953323364258,
      "activations/layer18_attention_weight_max": 33.352169036865234,
      "activations/layer18_attention_weight_min": -20.523405075073242,
      "activations/layer19_attention_weight_max": 53.645240783691406,
      "activations/layer19_attention_weight_min": -32.8949089050293,
      "activations/layer1_attention_weight_max": 17.76443862915039,
      "activations/layer1_attention_weight_min": -13.24232006072998,
      "activations/layer20_attention_weight_max": 39.5428352355957,
      "activations/layer20_attention_weight_min": -24.463645935058594,
      "activations/layer21_attention_weight_max": 35.80567932128906,
      "activations/layer21_attention_weight_min": -22.669662475585938,
      "activations/layer22_attention_weight_max": 52.59294509887695,
      "activations/layer22_attention_weight_min": -33.518821716308594,
      "activations/layer23_attention_weight_max": 30.994482040405273,
      "activations/layer23_attention_weight_min": -19.25366973876953,
      "activations/layer2_attention_weight_max": 30.328567504882812,
      "activations/layer2_attention_weight_min": -29.611377716064453,
      "activations/layer3_attention_weight_max": 80.68167114257812,
      "activations/layer3_attention_weight_min": -81.70684814453125,
      "activations/layer4_attention_weight_max": 131.7091522216797,
      "activations/layer4_attention_weight_min": -129.93336486816406,
      "activations/layer5_attention_weight_max": 105.2364501953125,
      "activations/layer5_attention_weight_min": -92.86955261230469,
      "activations/layer6_attention_weight_max": 60.67274856567383,
      "activations/layer6_attention_weight_min": -51.25672912597656,
      "activations/layer7_attention_weight_max": 79.34318542480469,
      "activations/layer7_attention_weight_min": -67.6890869140625,
      "activations/layer8_attention_weight_max": 43.0850715637207,
      "activations/layer8_attention_weight_min": -34.72090530395508,
      "activations/layer9_attention_weight_max": 51.84614944458008,
      "activations/layer9_attention_weight_min": -42.1124267578125,
      "epoch": 2.93,
      "learning_rate": 0.00013242727272727272,
      "loss": 2.9212,
      "step": 50450
    },
    {
      "activations/layer0_attention_weight_max": 16.701616287231445,
      "activations/layer0_attention_weight_min": -11.135421752929688,
      "activations/layer10_attention_weight_max": 41.574649810791016,
      "activations/layer10_attention_weight_min": -32.269386291503906,
      "activations/layer11_attention_weight_max": 36.960723876953125,
      "activations/layer11_attention_weight_min": -34.87084197998047,
      "activations/layer12_attention_weight_max": 50.511409759521484,
      "activations/layer12_attention_weight_min": -32.52739715576172,
      "activations/layer13_attention_weight_max": 51.843448638916016,
      "activations/layer13_attention_weight_min": -40.3090934753418,
      "activations/layer14_attention_weight_max": 78.68761444091797,
      "activations/layer14_attention_weight_min": -50.55734634399414,
      "activations/layer15_attention_weight_max": 52.79173278808594,
      "activations/layer15_attention_weight_min": -33.04161834716797,
      "activations/layer16_attention_weight_max": 46.09312057495117,
      "activations/layer16_attention_weight_min": -29.085205078125,
      "activations/layer17_attention_weight_max": 42.58989334106445,
      "activations/layer17_attention_weight_min": -25.724876403808594,
      "activations/layer18_attention_weight_max": 29.126371383666992,
      "activations/layer18_attention_weight_min": -18.225439071655273,
      "activations/layer19_attention_weight_max": 45.35089111328125,
      "activations/layer19_attention_weight_min": -29.450027465820312,
      "activations/layer1_attention_weight_max": 17.574472427368164,
      "activations/layer1_attention_weight_min": -12.090648651123047,
      "activations/layer20_attention_weight_max": 34.25515365600586,
      "activations/layer20_attention_weight_min": -21.575111389160156,
      "activations/layer21_attention_weight_max": 37.09253692626953,
      "activations/layer21_attention_weight_min": -18.186004638671875,
      "activations/layer22_attention_weight_max": 47.91407775878906,
      "activations/layer22_attention_weight_min": -28.08411979675293,
      "activations/layer23_attention_weight_max": 26.932945251464844,
      "activations/layer23_attention_weight_min": -16.170202255249023,
      "activations/layer2_attention_weight_max": 29.600448608398438,
      "activations/layer2_attention_weight_min": -29.828746795654297,
      "activations/layer3_attention_weight_max": 82.37461853027344,
      "activations/layer3_attention_weight_min": -83.19010162353516,
      "activations/layer4_attention_weight_max": 130.90090942382812,
      "activations/layer4_attention_weight_min": -124.9328842163086,
      "activations/layer5_attention_weight_max": 107.74673461914062,
      "activations/layer5_attention_weight_min": -96.6284408569336,
      "activations/layer6_attention_weight_max": 58.3760986328125,
      "activations/layer6_attention_weight_min": -54.81980895996094,
      "activations/layer7_attention_weight_max": 71.21981048583984,
      "activations/layer7_attention_weight_min": -73.31769561767578,
      "activations/layer8_attention_weight_max": 42.3898811340332,
      "activations/layer8_attention_weight_min": -37.49850082397461,
      "activations/layer9_attention_weight_max": 45.96902084350586,
      "activations/layer9_attention_weight_min": -47.80282211303711,
      "epoch": 2.93,
      "learning_rate": 0.0001324083333333333,
      "loss": 2.9183,
      "step": 50500
    },
    {
      "activations/layer0_attention_weight_max": 16.793729782104492,
      "activations/layer0_attention_weight_min": -11.532691955566406,
      "activations/layer10_attention_weight_max": 42.242835998535156,
      "activations/layer10_attention_weight_min": -33.3629150390625,
      "activations/layer11_attention_weight_max": 35.480281829833984,
      "activations/layer11_attention_weight_min": -28.973968505859375,
      "activations/layer12_attention_weight_max": 31.73269271850586,
      "activations/layer12_attention_weight_min": -28.562442779541016,
      "activations/layer13_attention_weight_max": 49.66872787475586,
      "activations/layer13_attention_weight_min": -35.00640869140625,
      "activations/layer14_attention_weight_max": 77.11334228515625,
      "activations/layer14_attention_weight_min": -52.726924896240234,
      "activations/layer15_attention_weight_max": 52.6400032043457,
      "activations/layer15_attention_weight_min": -30.994178771972656,
      "activations/layer16_attention_weight_max": 49.41024398803711,
      "activations/layer16_attention_weight_min": -32.79581069946289,
      "activations/layer17_attention_weight_max": 46.682498931884766,
      "activations/layer17_attention_weight_min": -26.380563735961914,
      "activations/layer18_attention_weight_max": 39.41158676147461,
      "activations/layer18_attention_weight_min": -19.289051055908203,
      "activations/layer19_attention_weight_max": 48.76375961303711,
      "activations/layer19_attention_weight_min": -29.8197021484375,
      "activations/layer1_attention_weight_max": 17.641551971435547,
      "activations/layer1_attention_weight_min": -13.020759582519531,
      "activations/layer20_attention_weight_max": 38.112396240234375,
      "activations/layer20_attention_weight_min": -22.48839569091797,
      "activations/layer21_attention_weight_max": 38.6442985534668,
      "activations/layer21_attention_weight_min": -23.054229736328125,
      "activations/layer22_attention_weight_max": 51.03272247314453,
      "activations/layer22_attention_weight_min": -27.942787170410156,
      "activations/layer23_attention_weight_max": 26.480712890625,
      "activations/layer23_attention_weight_min": -19.017887115478516,
      "activations/layer2_attention_weight_max": 32.408241271972656,
      "activations/layer2_attention_weight_min": -31.252737045288086,
      "activations/layer3_attention_weight_max": 77.97309112548828,
      "activations/layer3_attention_weight_min": -82.12744903564453,
      "activations/layer4_attention_weight_max": 131.34481811523438,
      "activations/layer4_attention_weight_min": -126.556884765625,
      "activations/layer5_attention_weight_max": 103.13298034667969,
      "activations/layer5_attention_weight_min": -89.77105712890625,
      "activations/layer6_attention_weight_max": 57.595584869384766,
      "activations/layer6_attention_weight_min": -54.74406433105469,
      "activations/layer7_attention_weight_max": 75.84980773925781,
      "activations/layer7_attention_weight_min": -67.3797607421875,
      "activations/layer8_attention_weight_max": 39.75856399536133,
      "activations/layer8_attention_weight_min": -35.035640716552734,
      "activations/layer9_attention_weight_max": 47.26824188232422,
      "activations/layer9_attention_weight_min": -41.097225189208984,
      "epoch": 2.94,
      "learning_rate": 0.00013238939393939392,
      "loss": 2.9092,
      "step": 50550
    },
    {
      "activations/layer0_attention_weight_max": 17.617155075073242,
      "activations/layer0_attention_weight_min": -11.914566993713379,
      "activations/layer10_attention_weight_max": 42.012699127197266,
      "activations/layer10_attention_weight_min": -37.25096893310547,
      "activations/layer11_attention_weight_max": 35.178314208984375,
      "activations/layer11_attention_weight_min": -29.038227081298828,
      "activations/layer12_attention_weight_max": 34.39457702636719,
      "activations/layer12_attention_weight_min": -31.63222885131836,
      "activations/layer13_attention_weight_max": 49.8707160949707,
      "activations/layer13_attention_weight_min": -37.39775848388672,
      "activations/layer14_attention_weight_max": 71.35374450683594,
      "activations/layer14_attention_weight_min": -52.79105758666992,
      "activations/layer15_attention_weight_max": 48.05619812011719,
      "activations/layer15_attention_weight_min": -32.32575607299805,
      "activations/layer16_attention_weight_max": 43.61654281616211,
      "activations/layer16_attention_weight_min": -31.582067489624023,
      "activations/layer17_attention_weight_max": 47.47646713256836,
      "activations/layer17_attention_weight_min": -26.76251983642578,
      "activations/layer18_attention_weight_max": 31.25246810913086,
      "activations/layer18_attention_weight_min": -19.534284591674805,
      "activations/layer19_attention_weight_max": 49.25342559814453,
      "activations/layer19_attention_weight_min": -30.479190826416016,
      "activations/layer1_attention_weight_max": 17.176799774169922,
      "activations/layer1_attention_weight_min": -12.506925582885742,
      "activations/layer20_attention_weight_max": 35.651039123535156,
      "activations/layer20_attention_weight_min": -23.7293758392334,
      "activations/layer21_attention_weight_max": 34.913124084472656,
      "activations/layer21_attention_weight_min": -19.590442657470703,
      "activations/layer22_attention_weight_max": 54.57450485229492,
      "activations/layer22_attention_weight_min": -27.977294921875,
      "activations/layer23_attention_weight_max": 30.143062591552734,
      "activations/layer23_attention_weight_min": -17.322513580322266,
      "activations/layer2_attention_weight_max": 30.202037811279297,
      "activations/layer2_attention_weight_min": -29.061737060546875,
      "activations/layer3_attention_weight_max": 73.47533416748047,
      "activations/layer3_attention_weight_min": -76.23690032958984,
      "activations/layer4_attention_weight_max": 123.5186996459961,
      "activations/layer4_attention_weight_min": -115.17813873291016,
      "activations/layer5_attention_weight_max": 99.19195556640625,
      "activations/layer5_attention_weight_min": -86.52474975585938,
      "activations/layer6_attention_weight_max": 56.59396743774414,
      "activations/layer6_attention_weight_min": -50.683528900146484,
      "activations/layer7_attention_weight_max": 69.42526245117188,
      "activations/layer7_attention_weight_min": -68.90066528320312,
      "activations/layer8_attention_weight_max": 42.72175216674805,
      "activations/layer8_attention_weight_min": -38.28242874145508,
      "activations/layer9_attention_weight_max": 47.19472885131836,
      "activations/layer9_attention_weight_min": -41.012447357177734,
      "epoch": 2.94,
      "learning_rate": 0.00013237045454545453,
      "loss": 2.9236,
      "step": 50600
    },
    {
      "activations/layer0_attention_weight_max": 17.366872787475586,
      "activations/layer0_attention_weight_min": -10.821919441223145,
      "activations/layer10_attention_weight_max": 43.215118408203125,
      "activations/layer10_attention_weight_min": -35.00962448120117,
      "activations/layer11_attention_weight_max": 37.523712158203125,
      "activations/layer11_attention_weight_min": -32.019195556640625,
      "activations/layer12_attention_weight_max": 35.02479553222656,
      "activations/layer12_attention_weight_min": -26.39569664001465,
      "activations/layer13_attention_weight_max": 48.711605072021484,
      "activations/layer13_attention_weight_min": -36.2561149597168,
      "activations/layer14_attention_weight_max": 75.07695770263672,
      "activations/layer14_attention_weight_min": -54.57868957519531,
      "activations/layer15_attention_weight_max": 53.350101470947266,
      "activations/layer15_attention_weight_min": -32.69180679321289,
      "activations/layer16_attention_weight_max": 45.09389114379883,
      "activations/layer16_attention_weight_min": -29.576446533203125,
      "activations/layer17_attention_weight_max": 46.271759033203125,
      "activations/layer17_attention_weight_min": -26.017921447753906,
      "activations/layer18_attention_weight_max": 28.086862564086914,
      "activations/layer18_attention_weight_min": -19.077428817749023,
      "activations/layer19_attention_weight_max": 54.443965911865234,
      "activations/layer19_attention_weight_min": -28.229379653930664,
      "activations/layer1_attention_weight_max": 18.763282775878906,
      "activations/layer1_attention_weight_min": -12.068028450012207,
      "activations/layer20_attention_weight_max": 43.982566833496094,
      "activations/layer20_attention_weight_min": -22.218929290771484,
      "activations/layer21_attention_weight_max": 32.57493591308594,
      "activations/layer21_attention_weight_min": -19.763450622558594,
      "activations/layer22_attention_weight_max": 54.66566467285156,
      "activations/layer22_attention_weight_min": -27.3055419921875,
      "activations/layer23_attention_weight_max": 26.372669219970703,
      "activations/layer23_attention_weight_min": -17.037216186523438,
      "activations/layer2_attention_weight_max": 29.257423400878906,
      "activations/layer2_attention_weight_min": -29.65776824951172,
      "activations/layer3_attention_weight_max": 77.12802124023438,
      "activations/layer3_attention_weight_min": -82.32694244384766,
      "activations/layer4_attention_weight_max": 128.79217529296875,
      "activations/layer4_attention_weight_min": -124.97428131103516,
      "activations/layer5_attention_weight_max": 99.04034423828125,
      "activations/layer5_attention_weight_min": -92.10842895507812,
      "activations/layer6_attention_weight_max": 57.542762756347656,
      "activations/layer6_attention_weight_min": -53.48086166381836,
      "activations/layer7_attention_weight_max": 68.37113189697266,
      "activations/layer7_attention_weight_min": -72.4959487915039,
      "activations/layer8_attention_weight_max": 37.63517379760742,
      "activations/layer8_attention_weight_min": -35.77500534057617,
      "activations/layer9_attention_weight_max": 46.27888870239258,
      "activations/layer9_attention_weight_min": -37.950836181640625,
      "epoch": 2.94,
      "learning_rate": 0.00013235151515151515,
      "loss": 2.9209,
      "step": 50650
    },
    {
      "activations/layer0_attention_weight_max": 17.553722381591797,
      "activations/layer0_attention_weight_min": -10.60313606262207,
      "activations/layer10_attention_weight_max": 43.087493896484375,
      "activations/layer10_attention_weight_min": -32.326560974121094,
      "activations/layer11_attention_weight_max": 38.740413665771484,
      "activations/layer11_attention_weight_min": -31.211170196533203,
      "activations/layer12_attention_weight_max": 65.84012603759766,
      "activations/layer12_attention_weight_min": -30.698137283325195,
      "activations/layer13_attention_weight_max": 51.675174713134766,
      "activations/layer13_attention_weight_min": -37.00349807739258,
      "activations/layer14_attention_weight_max": 82.16275787353516,
      "activations/layer14_attention_weight_min": -55.90705490112305,
      "activations/layer15_attention_weight_max": 51.52656555175781,
      "activations/layer15_attention_weight_min": -33.01932907104492,
      "activations/layer16_attention_weight_max": 51.09136962890625,
      "activations/layer16_attention_weight_min": -31.995311737060547,
      "activations/layer17_attention_weight_max": 50.34712219238281,
      "activations/layer17_attention_weight_min": -27.84979820251465,
      "activations/layer18_attention_weight_max": 35.09072494506836,
      "activations/layer18_attention_weight_min": -20.8496150970459,
      "activations/layer19_attention_weight_max": 57.386871337890625,
      "activations/layer19_attention_weight_min": -31.25132179260254,
      "activations/layer1_attention_weight_max": 17.946786880493164,
      "activations/layer1_attention_weight_min": -13.306818962097168,
      "activations/layer20_attention_weight_max": 46.10057067871094,
      "activations/layer20_attention_weight_min": -22.817529678344727,
      "activations/layer21_attention_weight_max": 39.12504959106445,
      "activations/layer21_attention_weight_min": -21.941299438476562,
      "activations/layer22_attention_weight_max": 56.984127044677734,
      "activations/layer22_attention_weight_min": -29.42058563232422,
      "activations/layer23_attention_weight_max": 31.3853816986084,
      "activations/layer23_attention_weight_min": -17.183818817138672,
      "activations/layer2_attention_weight_max": 29.19438934326172,
      "activations/layer2_attention_weight_min": -28.891624450683594,
      "activations/layer3_attention_weight_max": 76.38018798828125,
      "activations/layer3_attention_weight_min": -81.1025390625,
      "activations/layer4_attention_weight_max": 134.9651336669922,
      "activations/layer4_attention_weight_min": -128.92369079589844,
      "activations/layer5_attention_weight_max": 106.35820770263672,
      "activations/layer5_attention_weight_min": -96.69509887695312,
      "activations/layer6_attention_weight_max": 59.0247917175293,
      "activations/layer6_attention_weight_min": -52.53068923950195,
      "activations/layer7_attention_weight_max": 77.46707153320312,
      "activations/layer7_attention_weight_min": -71.0550537109375,
      "activations/layer8_attention_weight_max": 42.54704666137695,
      "activations/layer8_attention_weight_min": -40.02241897583008,
      "activations/layer9_attention_weight_max": 57.511898040771484,
      "activations/layer9_attention_weight_min": -46.84140396118164,
      "epoch": 2.95,
      "learning_rate": 0.00013233257575757574,
      "loss": 2.904,
      "step": 50700
    },
    {
      "activations/layer0_attention_weight_max": 16.947229385375977,
      "activations/layer0_attention_weight_min": -11.486822128295898,
      "activations/layer10_attention_weight_max": 45.80601119995117,
      "activations/layer10_attention_weight_min": -38.09427261352539,
      "activations/layer11_attention_weight_max": 40.48341751098633,
      "activations/layer11_attention_weight_min": -31.304594039916992,
      "activations/layer12_attention_weight_max": 36.020572662353516,
      "activations/layer12_attention_weight_min": -27.33323097229004,
      "activations/layer13_attention_weight_max": 48.55521011352539,
      "activations/layer13_attention_weight_min": -35.21675109863281,
      "activations/layer14_attention_weight_max": 70.23188781738281,
      "activations/layer14_attention_weight_min": -51.59014892578125,
      "activations/layer15_attention_weight_max": 47.733619689941406,
      "activations/layer15_attention_weight_min": -30.116962432861328,
      "activations/layer16_attention_weight_max": 46.137874603271484,
      "activations/layer16_attention_weight_min": -30.72593116760254,
      "activations/layer17_attention_weight_max": 47.36598587036133,
      "activations/layer17_attention_weight_min": -28.336780548095703,
      "activations/layer18_attention_weight_max": 30.61664390563965,
      "activations/layer18_attention_weight_min": -17.890989303588867,
      "activations/layer19_attention_weight_max": 45.92002487182617,
      "activations/layer19_attention_weight_min": -30.041461944580078,
      "activations/layer1_attention_weight_max": 18.64146614074707,
      "activations/layer1_attention_weight_min": -12.096435546875,
      "activations/layer20_attention_weight_max": 35.227508544921875,
      "activations/layer20_attention_weight_min": -21.766382217407227,
      "activations/layer21_attention_weight_max": 37.945396423339844,
      "activations/layer21_attention_weight_min": -18.922500610351562,
      "activations/layer22_attention_weight_max": 49.54098892211914,
      "activations/layer22_attention_weight_min": -27.159454345703125,
      "activations/layer23_attention_weight_max": 30.868284225463867,
      "activations/layer23_attention_weight_min": -18.068710327148438,
      "activations/layer2_attention_weight_max": 28.82510757446289,
      "activations/layer2_attention_weight_min": -28.886676788330078,
      "activations/layer3_attention_weight_max": 74.38312530517578,
      "activations/layer3_attention_weight_min": -76.19849395751953,
      "activations/layer4_attention_weight_max": 132.8136749267578,
      "activations/layer4_attention_weight_min": -117.8904037475586,
      "activations/layer5_attention_weight_max": 108.31007385253906,
      "activations/layer5_attention_weight_min": -88.91648864746094,
      "activations/layer6_attention_weight_max": 60.448001861572266,
      "activations/layer6_attention_weight_min": -51.83149719238281,
      "activations/layer7_attention_weight_max": 68.88983917236328,
      "activations/layer7_attention_weight_min": -68.66883850097656,
      "activations/layer8_attention_weight_max": 40.407691955566406,
      "activations/layer8_attention_weight_min": -35.699745178222656,
      "activations/layer9_attention_weight_max": 44.83999252319336,
      "activations/layer9_attention_weight_min": -37.776092529296875,
      "epoch": 2.95,
      "learning_rate": 0.00013231363636363635,
      "loss": 2.9139,
      "step": 50750
    },
    {
      "activations/layer0_attention_weight_max": 17.1323184967041,
      "activations/layer0_attention_weight_min": -12.105588912963867,
      "activations/layer10_attention_weight_max": 47.30446243286133,
      "activations/layer10_attention_weight_min": -35.9916877746582,
      "activations/layer11_attention_weight_max": 35.69994354248047,
      "activations/layer11_attention_weight_min": -30.064725875854492,
      "activations/layer12_attention_weight_max": 43.76466369628906,
      "activations/layer12_attention_weight_min": -26.532590866088867,
      "activations/layer13_attention_weight_max": 57.21886444091797,
      "activations/layer13_attention_weight_min": -36.795982360839844,
      "activations/layer14_attention_weight_max": 78.9345932006836,
      "activations/layer14_attention_weight_min": -52.68385314941406,
      "activations/layer15_attention_weight_max": 49.71434783935547,
      "activations/layer15_attention_weight_min": -34.9903450012207,
      "activations/layer16_attention_weight_max": 50.978851318359375,
      "activations/layer16_attention_weight_min": -34.01105880737305,
      "activations/layer17_attention_weight_max": 48.5842170715332,
      "activations/layer17_attention_weight_min": -27.291580200195312,
      "activations/layer18_attention_weight_max": 32.20909118652344,
      "activations/layer18_attention_weight_min": -18.9912166595459,
      "activations/layer19_attention_weight_max": 52.12548828125,
      "activations/layer19_attention_weight_min": -30.79558753967285,
      "activations/layer1_attention_weight_max": 17.741756439208984,
      "activations/layer1_attention_weight_min": -12.859314918518066,
      "activations/layer20_attention_weight_max": 37.21570587158203,
      "activations/layer20_attention_weight_min": -24.800186157226562,
      "activations/layer21_attention_weight_max": 31.707942962646484,
      "activations/layer21_attention_weight_min": -21.730005264282227,
      "activations/layer22_attention_weight_max": 51.18120574951172,
      "activations/layer22_attention_weight_min": -28.339895248413086,
      "activations/layer23_attention_weight_max": 28.84304428100586,
      "activations/layer23_attention_weight_min": -18.94825553894043,
      "activations/layer2_attention_weight_max": 29.598203659057617,
      "activations/layer2_attention_weight_min": -28.845563888549805,
      "activations/layer3_attention_weight_max": 78.55984497070312,
      "activations/layer3_attention_weight_min": -80.09248352050781,
      "activations/layer4_attention_weight_max": 130.32786560058594,
      "activations/layer4_attention_weight_min": -125.5313491821289,
      "activations/layer5_attention_weight_max": 104.35469818115234,
      "activations/layer5_attention_weight_min": -93.34342956542969,
      "activations/layer6_attention_weight_max": 58.58438491821289,
      "activations/layer6_attention_weight_min": -52.24337387084961,
      "activations/layer7_attention_weight_max": 72.8323745727539,
      "activations/layer7_attention_weight_min": -67.32498931884766,
      "activations/layer8_attention_weight_max": 42.00553894042969,
      "activations/layer8_attention_weight_min": -36.5497932434082,
      "activations/layer9_attention_weight_max": 50.67176055908203,
      "activations/layer9_attention_weight_min": -45.30073547363281,
      "epoch": 2.95,
      "learning_rate": 0.00013229469696969697,
      "loss": 2.925,
      "step": 50800
    },
    {
      "activations/layer0_attention_weight_max": 16.90514373779297,
      "activations/layer0_attention_weight_min": -11.363739013671875,
      "activations/layer10_attention_weight_max": 38.7851676940918,
      "activations/layer10_attention_weight_min": -32.378387451171875,
      "activations/layer11_attention_weight_max": 33.74646759033203,
      "activations/layer11_attention_weight_min": -30.953617095947266,
      "activations/layer12_attention_weight_max": 29.71794319152832,
      "activations/layer12_attention_weight_min": -27.82920265197754,
      "activations/layer13_attention_weight_max": 55.958595275878906,
      "activations/layer13_attention_weight_min": -36.268348693847656,
      "activations/layer14_attention_weight_max": 83.89157104492188,
      "activations/layer14_attention_weight_min": -57.90546798706055,
      "activations/layer15_attention_weight_max": 48.24392318725586,
      "activations/layer15_attention_weight_min": -32.85837936401367,
      "activations/layer16_attention_weight_max": 45.10329818725586,
      "activations/layer16_attention_weight_min": -31.329694747924805,
      "activations/layer17_attention_weight_max": 43.70253372192383,
      "activations/layer17_attention_weight_min": -26.75616455078125,
      "activations/layer18_attention_weight_max": 33.129249572753906,
      "activations/layer18_attention_weight_min": -18.822107315063477,
      "activations/layer19_attention_weight_max": 52.95597457885742,
      "activations/layer19_attention_weight_min": -31.446826934814453,
      "activations/layer1_attention_weight_max": 18.41493034362793,
      "activations/layer1_attention_weight_min": -13.985074043273926,
      "activations/layer20_attention_weight_max": 37.0241813659668,
      "activations/layer20_attention_weight_min": -22.539600372314453,
      "activations/layer21_attention_weight_max": 36.32954788208008,
      "activations/layer21_attention_weight_min": -19.62900733947754,
      "activations/layer22_attention_weight_max": 51.93376541137695,
      "activations/layer22_attention_weight_min": -27.35267448425293,
      "activations/layer23_attention_weight_max": 27.930709838867188,
      "activations/layer23_attention_weight_min": -15.642473220825195,
      "activations/layer2_attention_weight_max": 31.729642868041992,
      "activations/layer2_attention_weight_min": -30.79854965209961,
      "activations/layer3_attention_weight_max": 80.22111511230469,
      "activations/layer3_attention_weight_min": -81.14871215820312,
      "activations/layer4_attention_weight_max": 136.1094207763672,
      "activations/layer4_attention_weight_min": -127.58989715576172,
      "activations/layer5_attention_weight_max": 109.62496185302734,
      "activations/layer5_attention_weight_min": -99.63265991210938,
      "activations/layer6_attention_weight_max": 63.840484619140625,
      "activations/layer6_attention_weight_min": -55.15083694458008,
      "activations/layer7_attention_weight_max": 70.27381134033203,
      "activations/layer7_attention_weight_min": -65.16934204101562,
      "activations/layer8_attention_weight_max": 39.305686950683594,
      "activations/layer8_attention_weight_min": -34.902828216552734,
      "activations/layer9_attention_weight_max": 55.130279541015625,
      "activations/layer9_attention_weight_min": -42.54876708984375,
      "epoch": 2.95,
      "learning_rate": 0.00013227575757575755,
      "loss": 2.8961,
      "step": 50850
    },
    {
      "activations/layer0_attention_weight_max": 16.586435317993164,
      "activations/layer0_attention_weight_min": -11.26305103302002,
      "activations/layer10_attention_weight_max": 50.643341064453125,
      "activations/layer10_attention_weight_min": -35.04288864135742,
      "activations/layer11_attention_weight_max": 42.85028839111328,
      "activations/layer11_attention_weight_min": -30.229576110839844,
      "activations/layer12_attention_weight_max": 46.33674240112305,
      "activations/layer12_attention_weight_min": -36.13154220581055,
      "activations/layer13_attention_weight_max": 59.020957946777344,
      "activations/layer13_attention_weight_min": -34.795780181884766,
      "activations/layer14_attention_weight_max": 95.11174011230469,
      "activations/layer14_attention_weight_min": -60.464942932128906,
      "activations/layer15_attention_weight_max": 57.140995025634766,
      "activations/layer15_attention_weight_min": -33.21201705932617,
      "activations/layer16_attention_weight_max": 50.37259292602539,
      "activations/layer16_attention_weight_min": -31.64787483215332,
      "activations/layer17_attention_weight_max": 38.82057571411133,
      "activations/layer17_attention_weight_min": -27.766746520996094,
      "activations/layer18_attention_weight_max": 33.6113395690918,
      "activations/layer18_attention_weight_min": -22.08871078491211,
      "activations/layer19_attention_weight_max": 49.56581115722656,
      "activations/layer19_attention_weight_min": -30.1468563079834,
      "activations/layer1_attention_weight_max": 17.409595489501953,
      "activations/layer1_attention_weight_min": -13.631464958190918,
      "activations/layer20_attention_weight_max": 34.83339309692383,
      "activations/layer20_attention_weight_min": -23.583654403686523,
      "activations/layer21_attention_weight_max": 32.38336944580078,
      "activations/layer21_attention_weight_min": -20.016433715820312,
      "activations/layer22_attention_weight_max": 48.20022964477539,
      "activations/layer22_attention_weight_min": -27.97456169128418,
      "activations/layer23_attention_weight_max": 29.2954158782959,
      "activations/layer23_attention_weight_min": -18.600156784057617,
      "activations/layer2_attention_weight_max": 34.90290832519531,
      "activations/layer2_attention_weight_min": -32.287742614746094,
      "activations/layer3_attention_weight_max": 93.32566833496094,
      "activations/layer3_attention_weight_min": -89.23880767822266,
      "activations/layer4_attention_weight_max": 157.4139862060547,
      "activations/layer4_attention_weight_min": -136.87159729003906,
      "activations/layer5_attention_weight_max": 121.60375213623047,
      "activations/layer5_attention_weight_min": -106.14412689208984,
      "activations/layer6_attention_weight_max": 73.71537780761719,
      "activations/layer6_attention_weight_min": -58.9826545715332,
      "activations/layer7_attention_weight_max": 86.05587768554688,
      "activations/layer7_attention_weight_min": -77.48017120361328,
      "activations/layer8_attention_weight_max": 52.413265228271484,
      "activations/layer8_attention_weight_min": -42.523712158203125,
      "activations/layer9_attention_weight_max": 60.917293548583984,
      "activations/layer9_attention_weight_min": -49.263309478759766,
      "epoch": 2.96,
      "learning_rate": 0.00013225681818181817,
      "loss": 2.9231,
      "step": 50900
    },
    {
      "activations/layer0_attention_weight_max": 17.46396255493164,
      "activations/layer0_attention_weight_min": -12.233161926269531,
      "activations/layer10_attention_weight_max": 42.33683395385742,
      "activations/layer10_attention_weight_min": -34.05278015136719,
      "activations/layer11_attention_weight_max": 38.036651611328125,
      "activations/layer11_attention_weight_min": -33.11494827270508,
      "activations/layer12_attention_weight_max": 32.51549530029297,
      "activations/layer12_attention_weight_min": -29.6140193939209,
      "activations/layer13_attention_weight_max": 44.853614807128906,
      "activations/layer13_attention_weight_min": -35.89472579956055,
      "activations/layer14_attention_weight_max": 71.50457763671875,
      "activations/layer14_attention_weight_min": -54.78623962402344,
      "activations/layer15_attention_weight_max": 45.83134460449219,
      "activations/layer15_attention_weight_min": -36.035091400146484,
      "activations/layer16_attention_weight_max": 48.74116516113281,
      "activations/layer16_attention_weight_min": -37.12092590332031,
      "activations/layer17_attention_weight_max": 44.777469635009766,
      "activations/layer17_attention_weight_min": -30.864545822143555,
      "activations/layer18_attention_weight_max": 29.453847885131836,
      "activations/layer18_attention_weight_min": -19.508161544799805,
      "activations/layer19_attention_weight_max": 46.87751388549805,
      "activations/layer19_attention_weight_min": -32.488983154296875,
      "activations/layer1_attention_weight_max": 18.67013931274414,
      "activations/layer1_attention_weight_min": -13.961390495300293,
      "activations/layer20_attention_weight_max": 31.247777938842773,
      "activations/layer20_attention_weight_min": -23.225156784057617,
      "activations/layer21_attention_weight_max": 33.76396179199219,
      "activations/layer21_attention_weight_min": -20.47747802734375,
      "activations/layer22_attention_weight_max": 46.92767333984375,
      "activations/layer22_attention_weight_min": -32.642730712890625,
      "activations/layer23_attention_weight_max": 25.101598739624023,
      "activations/layer23_attention_weight_min": -19.567481994628906,
      "activations/layer2_attention_weight_max": 29.791484832763672,
      "activations/layer2_attention_weight_min": -29.32026481628418,
      "activations/layer3_attention_weight_max": 83.03334045410156,
      "activations/layer3_attention_weight_min": -84.7015380859375,
      "activations/layer4_attention_weight_max": 130.33453369140625,
      "activations/layer4_attention_weight_min": -124.86626434326172,
      "activations/layer5_attention_weight_max": 105.7890625,
      "activations/layer5_attention_weight_min": -92.40670776367188,
      "activations/layer6_attention_weight_max": 60.90822219848633,
      "activations/layer6_attention_weight_min": -53.23847961425781,
      "activations/layer7_attention_weight_max": 79.77561950683594,
      "activations/layer7_attention_weight_min": -77.21308135986328,
      "activations/layer8_attention_weight_max": 43.50851058959961,
      "activations/layer8_attention_weight_min": -35.35591506958008,
      "activations/layer9_attention_weight_max": 51.19858932495117,
      "activations/layer9_attention_weight_min": -44.51995086669922,
      "epoch": 2.96,
      "learning_rate": 0.00013223825757575756,
      "loss": 2.9097,
      "step": 50950
    },
    {
      "activations/layer0_attention_weight_max": 17.998844146728516,
      "activations/layer0_attention_weight_min": -11.215405464172363,
      "activations/layer10_attention_weight_max": 40.9000358581543,
      "activations/layer10_attention_weight_min": -31.84745979309082,
      "activations/layer11_attention_weight_max": 37.99784851074219,
      "activations/layer11_attention_weight_min": -30.588605880737305,
      "activations/layer12_attention_weight_max": 28.88652801513672,
      "activations/layer12_attention_weight_min": -26.143577575683594,
      "activations/layer13_attention_weight_max": 50.14635467529297,
      "activations/layer13_attention_weight_min": -37.495216369628906,
      "activations/layer14_attention_weight_max": 64.24107360839844,
      "activations/layer14_attention_weight_min": -49.431602478027344,
      "activations/layer15_attention_weight_max": 48.41808319091797,
      "activations/layer15_attention_weight_min": -33.00999069213867,
      "activations/layer16_attention_weight_max": 39.82078170776367,
      "activations/layer16_attention_weight_min": -33.38040542602539,
      "activations/layer17_attention_weight_max": 44.30741500854492,
      "activations/layer17_attention_weight_min": -26.676530838012695,
      "activations/layer18_attention_weight_max": 31.353805541992188,
      "activations/layer18_attention_weight_min": -21.029510498046875,
      "activations/layer19_attention_weight_max": 49.46003341674805,
      "activations/layer19_attention_weight_min": -30.4020938873291,
      "activations/layer1_attention_weight_max": 17.340593338012695,
      "activations/layer1_attention_weight_min": -14.608366966247559,
      "activations/layer20_attention_weight_max": 34.8214225769043,
      "activations/layer20_attention_weight_min": -23.653032302856445,
      "activations/layer21_attention_weight_max": 36.786346435546875,
      "activations/layer21_attention_weight_min": -21.714786529541016,
      "activations/layer22_attention_weight_max": 42.798030853271484,
      "activations/layer22_attention_weight_min": -24.06277084350586,
      "activations/layer23_attention_weight_max": 26.40011978149414,
      "activations/layer23_attention_weight_min": -17.162094116210938,
      "activations/layer2_attention_weight_max": 31.564014434814453,
      "activations/layer2_attention_weight_min": -30.639272689819336,
      "activations/layer3_attention_weight_max": 75.74259185791016,
      "activations/layer3_attention_weight_min": -81.83543395996094,
      "activations/layer4_attention_weight_max": 125.80830383300781,
      "activations/layer4_attention_weight_min": -121.6767349243164,
      "activations/layer5_attention_weight_max": 102.58068084716797,
      "activations/layer5_attention_weight_min": -92.90971374511719,
      "activations/layer6_attention_weight_max": 56.19136047363281,
      "activations/layer6_attention_weight_min": -53.93439865112305,
      "activations/layer7_attention_weight_max": 70.79007720947266,
      "activations/layer7_attention_weight_min": -67.56201171875,
      "activations/layer8_attention_weight_max": 40.42384719848633,
      "activations/layer8_attention_weight_min": -37.4134521484375,
      "activations/layer9_attention_weight_max": 46.89420700073242,
      "activations/layer9_attention_weight_min": -35.448429107666016,
      "epoch": 2.96,
      "learning_rate": 0.00013221931818181818,
      "loss": 2.9177,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_loss": 2.861328125,
      "eval_runtime": 8.517,
      "eval_samples_per_second": 504.17,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_openwebtext_loss": 2.861328125,
      "eval_openwebtext_ppl": 17.484733434196265,
      "eval_openwebtext_runtime": 8.517,
      "eval_openwebtext_samples_per_second": 504.17,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_wikitext_loss": 3.119140625,
      "eval_wikitext_ppl": 22.626926270737744,
      "eval_wikitext_runtime": 1.689,
      "eval_wikitext_samples_per_second": 269.987,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_lambada_loss": 2.916015625,
      "eval_lambada_ppl": 18.467558995986426,
      "eval_lambada_runtime": 9.5748,
      "eval_lambada_samples_per_second": 508.52,
      "step": 51000
    },
    {
      "activations/layer0_attention_weight_max": 16.086776733398438,
      "activations/layer0_attention_weight_min": -12.619810104370117,
      "activations/layer10_attention_weight_max": 41.28158187866211,
      "activations/layer10_attention_weight_min": -35.49882507324219,
      "activations/layer11_attention_weight_max": 36.76504135131836,
      "activations/layer11_attention_weight_min": -30.3371639251709,
      "activations/layer12_attention_weight_max": 46.49132537841797,
      "activations/layer12_attention_weight_min": -32.408931732177734,
      "activations/layer13_attention_weight_max": 52.237144470214844,
      "activations/layer13_attention_weight_min": -38.267574310302734,
      "activations/layer14_attention_weight_max": 76.07601165771484,
      "activations/layer14_attention_weight_min": -52.371768951416016,
      "activations/layer15_attention_weight_max": 49.83880615234375,
      "activations/layer15_attention_weight_min": -32.77469253540039,
      "activations/layer16_attention_weight_max": 41.0428466796875,
      "activations/layer16_attention_weight_min": -31.428958892822266,
      "activations/layer17_attention_weight_max": 44.223270416259766,
      "activations/layer17_attention_weight_min": -26.79254150390625,
      "activations/layer18_attention_weight_max": 25.952430725097656,
      "activations/layer18_attention_weight_min": -20.344329833984375,
      "activations/layer19_attention_weight_max": 40.8182373046875,
      "activations/layer19_attention_weight_min": -27.452716827392578,
      "activations/layer1_attention_weight_max": 17.83977699279785,
      "activations/layer1_attention_weight_min": -12.740921020507812,
      "activations/layer20_attention_weight_max": 31.97382164001465,
      "activations/layer20_attention_weight_min": -22.527570724487305,
      "activations/layer21_attention_weight_max": 29.901479721069336,
      "activations/layer21_attention_weight_min": -21.255352020263672,
      "activations/layer22_attention_weight_max": 45.01609802246094,
      "activations/layer22_attention_weight_min": -27.179243087768555,
      "activations/layer23_attention_weight_max": 24.82778549194336,
      "activations/layer23_attention_weight_min": -17.484485626220703,
      "activations/layer2_attention_weight_max": 31.299407958984375,
      "activations/layer2_attention_weight_min": -30.812847137451172,
      "activations/layer3_attention_weight_max": 78.9916000366211,
      "activations/layer3_attention_weight_min": -80.36530303955078,
      "activations/layer4_attention_weight_max": 131.8070068359375,
      "activations/layer4_attention_weight_min": -124.41194152832031,
      "activations/layer5_attention_weight_max": 105.91175842285156,
      "activations/layer5_attention_weight_min": -91.61918640136719,
      "activations/layer6_attention_weight_max": 57.98308181762695,
      "activations/layer6_attention_weight_min": -54.60480880737305,
      "activations/layer7_attention_weight_max": 66.17017364501953,
      "activations/layer7_attention_weight_min": -67.87953186035156,
      "activations/layer8_attention_weight_max": 39.26750946044922,
      "activations/layer8_attention_weight_min": -36.44757843017578,
      "activations/layer9_attention_weight_max": 51.370906829833984,
      "activations/layer9_attention_weight_min": -43.64346694946289,
      "epoch": 2.97,
      "learning_rate": 0.0001322003787878788,
      "loss": 2.8946,
      "step": 51050
    },
    {
      "activations/layer0_attention_weight_max": 17.813819885253906,
      "activations/layer0_attention_weight_min": -11.19563102722168,
      "activations/layer10_attention_weight_max": 37.266265869140625,
      "activations/layer10_attention_weight_min": -31.056758880615234,
      "activations/layer11_attention_weight_max": 33.75035095214844,
      "activations/layer11_attention_weight_min": -28.446561813354492,
      "activations/layer12_attention_weight_max": 72.76750946044922,
      "activations/layer12_attention_weight_min": -33.226890563964844,
      "activations/layer13_attention_weight_max": 49.70067596435547,
      "activations/layer13_attention_weight_min": -33.42323303222656,
      "activations/layer14_attention_weight_max": 70.3400650024414,
      "activations/layer14_attention_weight_min": -53.27303695678711,
      "activations/layer15_attention_weight_max": 45.18483352661133,
      "activations/layer15_attention_weight_min": -31.486713409423828,
      "activations/layer16_attention_weight_max": 48.360595703125,
      "activations/layer16_attention_weight_min": -30.42582893371582,
      "activations/layer17_attention_weight_max": 42.59761428833008,
      "activations/layer17_attention_weight_min": -27.179370880126953,
      "activations/layer18_attention_weight_max": 29.67039680480957,
      "activations/layer18_attention_weight_min": -20.853755950927734,
      "activations/layer19_attention_weight_max": 56.040279388427734,
      "activations/layer19_attention_weight_min": -31.107526779174805,
      "activations/layer1_attention_weight_max": 17.76707649230957,
      "activations/layer1_attention_weight_min": -12.872394561767578,
      "activations/layer20_attention_weight_max": 40.60635757446289,
      "activations/layer20_attention_weight_min": -23.607608795166016,
      "activations/layer21_attention_weight_max": 35.51386260986328,
      "activations/layer21_attention_weight_min": -20.192569732666016,
      "activations/layer22_attention_weight_max": 47.90434646606445,
      "activations/layer22_attention_weight_min": -26.048168182373047,
      "activations/layer23_attention_weight_max": 24.26228904724121,
      "activations/layer23_attention_weight_min": -16.242712020874023,
      "activations/layer2_attention_weight_max": 30.831241607666016,
      "activations/layer2_attention_weight_min": -29.212766647338867,
      "activations/layer3_attention_weight_max": 75.40740203857422,
      "activations/layer3_attention_weight_min": -78.67538452148438,
      "activations/layer4_attention_weight_max": 127.96517181396484,
      "activations/layer4_attention_weight_min": -120.27140045166016,
      "activations/layer5_attention_weight_max": 103.36996459960938,
      "activations/layer5_attention_weight_min": -92.47804260253906,
      "activations/layer6_attention_weight_max": 56.13401412963867,
      "activations/layer6_attention_weight_min": -50.763919830322266,
      "activations/layer7_attention_weight_max": 72.71993255615234,
      "activations/layer7_attention_weight_min": -67.10883331298828,
      "activations/layer8_attention_weight_max": 38.768795013427734,
      "activations/layer8_attention_weight_min": -35.59754943847656,
      "activations/layer9_attention_weight_max": 45.2802848815918,
      "activations/layer9_attention_weight_min": -38.97554397583008,
      "epoch": 2.97,
      "learning_rate": 0.00013218143939393938,
      "loss": 2.918,
      "step": 51100
    },
    {
      "activations/layer0_attention_weight_max": 17.43417739868164,
      "activations/layer0_attention_weight_min": -11.128725051879883,
      "activations/layer10_attention_weight_max": 43.311614990234375,
      "activations/layer10_attention_weight_min": -33.86516189575195,
      "activations/layer11_attention_weight_max": 36.442405700683594,
      "activations/layer11_attention_weight_min": -31.831693649291992,
      "activations/layer12_attention_weight_max": 30.75227165222168,
      "activations/layer12_attention_weight_min": -28.55883026123047,
      "activations/layer13_attention_weight_max": 43.43898010253906,
      "activations/layer13_attention_weight_min": -34.58207321166992,
      "activations/layer14_attention_weight_max": 81.93925476074219,
      "activations/layer14_attention_weight_min": -61.19153594970703,
      "activations/layer15_attention_weight_max": 47.51833724975586,
      "activations/layer15_attention_weight_min": -34.5198974609375,
      "activations/layer16_attention_weight_max": 39.67900466918945,
      "activations/layer16_attention_weight_min": -31.877668380737305,
      "activations/layer17_attention_weight_max": 41.8744010925293,
      "activations/layer17_attention_weight_min": -28.18246078491211,
      "activations/layer18_attention_weight_max": 29.280658721923828,
      "activations/layer18_attention_weight_min": -18.595754623413086,
      "activations/layer19_attention_weight_max": 41.74032974243164,
      "activations/layer19_attention_weight_min": -32.508480072021484,
      "activations/layer1_attention_weight_max": 17.007516860961914,
      "activations/layer1_attention_weight_min": -12.864495277404785,
      "activations/layer20_attention_weight_max": 30.46944808959961,
      "activations/layer20_attention_weight_min": -24.456579208374023,
      "activations/layer21_attention_weight_max": 33.1005744934082,
      "activations/layer21_attention_weight_min": -20.998682022094727,
      "activations/layer22_attention_weight_max": 44.35150909423828,
      "activations/layer22_attention_weight_min": -26.627872467041016,
      "activations/layer23_attention_weight_max": 25.26190948486328,
      "activations/layer23_attention_weight_min": -19.012332916259766,
      "activations/layer2_attention_weight_max": 29.591686248779297,
      "activations/layer2_attention_weight_min": -29.331050872802734,
      "activations/layer3_attention_weight_max": 75.14122772216797,
      "activations/layer3_attention_weight_min": -77.67890167236328,
      "activations/layer4_attention_weight_max": 127.57772064208984,
      "activations/layer4_attention_weight_min": -122.8703842163086,
      "activations/layer5_attention_weight_max": 96.97380065917969,
      "activations/layer5_attention_weight_min": -92.21404266357422,
      "activations/layer6_attention_weight_max": 56.131839752197266,
      "activations/layer6_attention_weight_min": -52.83892059326172,
      "activations/layer7_attention_weight_max": 74.24646759033203,
      "activations/layer7_attention_weight_min": -70.39366149902344,
      "activations/layer8_attention_weight_max": 40.09398651123047,
      "activations/layer8_attention_weight_min": -37.0461311340332,
      "activations/layer9_attention_weight_max": 45.83641052246094,
      "activations/layer9_attention_weight_min": -43.10502243041992,
      "epoch": 2.97,
      "learning_rate": 0.0001321625,
      "loss": 2.9152,
      "step": 51150
    },
    {
      "activations/layer0_attention_weight_max": 17.16994857788086,
      "activations/layer0_attention_weight_min": -11.131075859069824,
      "activations/layer10_attention_weight_max": 46.4539909362793,
      "activations/layer10_attention_weight_min": -35.436668395996094,
      "activations/layer11_attention_weight_max": 48.63641357421875,
      "activations/layer11_attention_weight_min": -30.704395294189453,
      "activations/layer12_attention_weight_max": 41.96544647216797,
      "activations/layer12_attention_weight_min": -30.616897583007812,
      "activations/layer13_attention_weight_max": 68.92064666748047,
      "activations/layer13_attention_weight_min": -43.38650131225586,
      "activations/layer14_attention_weight_max": 76.22000885009766,
      "activations/layer14_attention_weight_min": -54.027095794677734,
      "activations/layer15_attention_weight_max": 54.352272033691406,
      "activations/layer15_attention_weight_min": -34.18343734741211,
      "activations/layer16_attention_weight_max": 52.86082458496094,
      "activations/layer16_attention_weight_min": -33.67961502075195,
      "activations/layer17_attention_weight_max": 51.000282287597656,
      "activations/layer17_attention_weight_min": -27.3763370513916,
      "activations/layer18_attention_weight_max": 35.460296630859375,
      "activations/layer18_attention_weight_min": -19.03007698059082,
      "activations/layer19_attention_weight_max": 52.570308685302734,
      "activations/layer19_attention_weight_min": -32.41554641723633,
      "activations/layer1_attention_weight_max": 18.5926456451416,
      "activations/layer1_attention_weight_min": -13.279540061950684,
      "activations/layer20_attention_weight_max": 39.99516296386719,
      "activations/layer20_attention_weight_min": -23.606849670410156,
      "activations/layer21_attention_weight_max": 41.39110565185547,
      "activations/layer21_attention_weight_min": -20.855539321899414,
      "activations/layer22_attention_weight_max": 55.609588623046875,
      "activations/layer22_attention_weight_min": -28.199338912963867,
      "activations/layer23_attention_weight_max": 31.774642944335938,
      "activations/layer23_attention_weight_min": -18.051828384399414,
      "activations/layer2_attention_weight_max": 30.54659652709961,
      "activations/layer2_attention_weight_min": -29.51983642578125,
      "activations/layer3_attention_weight_max": 86.31656646728516,
      "activations/layer3_attention_weight_min": -86.73507690429688,
      "activations/layer4_attention_weight_max": 143.30523681640625,
      "activations/layer4_attention_weight_min": -130.03204345703125,
      "activations/layer5_attention_weight_max": 111.55947875976562,
      "activations/layer5_attention_weight_min": -95.15287780761719,
      "activations/layer6_attention_weight_max": 65.3735122680664,
      "activations/layer6_attention_weight_min": -53.21037673950195,
      "activations/layer7_attention_weight_max": 80.05506896972656,
      "activations/layer7_attention_weight_min": -71.80155181884766,
      "activations/layer8_attention_weight_max": 47.02524948120117,
      "activations/layer8_attention_weight_min": -38.6668815612793,
      "activations/layer9_attention_weight_max": 66.57178497314453,
      "activations/layer9_attention_weight_min": -48.584346771240234,
      "epoch": 2.97,
      "learning_rate": 0.00013214393939393938,
      "loss": 2.9055,
      "step": 51200
    },
    {
      "activations/layer0_attention_weight_max": 16.6554012298584,
      "activations/layer0_attention_weight_min": -12.791757583618164,
      "activations/layer10_attention_weight_max": 46.54974365234375,
      "activations/layer10_attention_weight_min": -35.89211654663086,
      "activations/layer11_attention_weight_max": 45.037227630615234,
      "activations/layer11_attention_weight_min": -30.450115203857422,
      "activations/layer12_attention_weight_max": 41.849273681640625,
      "activations/layer12_attention_weight_min": -30.323579788208008,
      "activations/layer13_attention_weight_max": 50.2672004699707,
      "activations/layer13_attention_weight_min": -37.460838317871094,
      "activations/layer14_attention_weight_max": 69.88626861572266,
      "activations/layer14_attention_weight_min": -52.76616668701172,
      "activations/layer15_attention_weight_max": 48.72990798950195,
      "activations/layer15_attention_weight_min": -35.81574249267578,
      "activations/layer16_attention_weight_max": 45.404178619384766,
      "activations/layer16_attention_weight_min": -32.4339714050293,
      "activations/layer17_attention_weight_max": 46.49954605102539,
      "activations/layer17_attention_weight_min": -26.922454833984375,
      "activations/layer18_attention_weight_max": 30.59072494506836,
      "activations/layer18_attention_weight_min": -17.908132553100586,
      "activations/layer19_attention_weight_max": 47.1943473815918,
      "activations/layer19_attention_weight_min": -29.934906005859375,
      "activations/layer1_attention_weight_max": 17.19881248474121,
      "activations/layer1_attention_weight_min": -13.422530174255371,
      "activations/layer20_attention_weight_max": 37.48836898803711,
      "activations/layer20_attention_weight_min": -23.988780975341797,
      "activations/layer21_attention_weight_max": 36.54618835449219,
      "activations/layer21_attention_weight_min": -19.78850555419922,
      "activations/layer22_attention_weight_max": 48.93024444580078,
      "activations/layer22_attention_weight_min": -31.17583465576172,
      "activations/layer23_attention_weight_max": 25.94411849975586,
      "activations/layer23_attention_weight_min": -18.468433380126953,
      "activations/layer2_attention_weight_max": 29.437118530273438,
      "activations/layer2_attention_weight_min": -29.100500106811523,
      "activations/layer3_attention_weight_max": 78.44929504394531,
      "activations/layer3_attention_weight_min": -80.12918853759766,
      "activations/layer4_attention_weight_max": 137.81724548339844,
      "activations/layer4_attention_weight_min": -119.99523162841797,
      "activations/layer5_attention_weight_max": 114.0079345703125,
      "activations/layer5_attention_weight_min": -92.36702728271484,
      "activations/layer6_attention_weight_max": 62.99799728393555,
      "activations/layer6_attention_weight_min": -53.55752944946289,
      "activations/layer7_attention_weight_max": 73.84872436523438,
      "activations/layer7_attention_weight_min": -67.38871765136719,
      "activations/layer8_attention_weight_max": 43.03841018676758,
      "activations/layer8_attention_weight_min": -38.02922439575195,
      "activations/layer9_attention_weight_max": 65.56934356689453,
      "activations/layer9_attention_weight_min": -50.69129180908203,
      "epoch": 2.98,
      "learning_rate": 0.000132125,
      "loss": 2.912,
      "step": 51250
    },
    {
      "activations/layer0_attention_weight_max": 17.509241104125977,
      "activations/layer0_attention_weight_min": -12.062243461608887,
      "activations/layer10_attention_weight_max": 42.17680358886719,
      "activations/layer10_attention_weight_min": -33.04346466064453,
      "activations/layer11_attention_weight_max": 40.916961669921875,
      "activations/layer11_attention_weight_min": -29.3831844329834,
      "activations/layer12_attention_weight_max": 34.33644485473633,
      "activations/layer12_attention_weight_min": -27.936275482177734,
      "activations/layer13_attention_weight_max": 48.00400924682617,
      "activations/layer13_attention_weight_min": -34.834938049316406,
      "activations/layer14_attention_weight_max": 80.16040802001953,
      "activations/layer14_attention_weight_min": -60.22216796875,
      "activations/layer15_attention_weight_max": 45.050270080566406,
      "activations/layer15_attention_weight_min": -32.28045654296875,
      "activations/layer16_attention_weight_max": 43.636863708496094,
      "activations/layer16_attention_weight_min": -32.58384323120117,
      "activations/layer17_attention_weight_max": 47.645572662353516,
      "activations/layer17_attention_weight_min": -29.27191162109375,
      "activations/layer18_attention_weight_max": 34.236656188964844,
      "activations/layer18_attention_weight_min": -20.65545082092285,
      "activations/layer19_attention_weight_max": 43.98124313354492,
      "activations/layer19_attention_weight_min": -29.1165828704834,
      "activations/layer1_attention_weight_max": 16.921316146850586,
      "activations/layer1_attention_weight_min": -12.96142292022705,
      "activations/layer20_attention_weight_max": 36.7631721496582,
      "activations/layer20_attention_weight_min": -23.947874069213867,
      "activations/layer21_attention_weight_max": 33.65419006347656,
      "activations/layer21_attention_weight_min": -23.635061264038086,
      "activations/layer22_attention_weight_max": 48.9681510925293,
      "activations/layer22_attention_weight_min": -32.08375549316406,
      "activations/layer23_attention_weight_max": 27.589147567749023,
      "activations/layer23_attention_weight_min": -16.93987274169922,
      "activations/layer2_attention_weight_max": 30.43013572692871,
      "activations/layer2_attention_weight_min": -30.509475708007812,
      "activations/layer3_attention_weight_max": 79.3863754272461,
      "activations/layer3_attention_weight_min": -83.13020324707031,
      "activations/layer4_attention_weight_max": 131.23521423339844,
      "activations/layer4_attention_weight_min": -127.95423889160156,
      "activations/layer5_attention_weight_max": 104.2824935913086,
      "activations/layer5_attention_weight_min": -94.48455810546875,
      "activations/layer6_attention_weight_max": 60.04402160644531,
      "activations/layer6_attention_weight_min": -54.560855865478516,
      "activations/layer7_attention_weight_max": 69.72872924804688,
      "activations/layer7_attention_weight_min": -67.93199920654297,
      "activations/layer8_attention_weight_max": 39.746368408203125,
      "activations/layer8_attention_weight_min": -36.27756118774414,
      "activations/layer9_attention_weight_max": 51.553466796875,
      "activations/layer9_attention_weight_min": -36.118709564208984,
      "epoch": 2.98,
      "learning_rate": 0.0001321060606060606,
      "loss": 2.8984,
      "step": 51300
    },
    {
      "activations/layer0_attention_weight_max": 17.172130584716797,
      "activations/layer0_attention_weight_min": -12.108835220336914,
      "activations/layer10_attention_weight_max": 41.566829681396484,
      "activations/layer10_attention_weight_min": -34.78826141357422,
      "activations/layer11_attention_weight_max": 33.70806884765625,
      "activations/layer11_attention_weight_min": -30.92291831970215,
      "activations/layer12_attention_weight_max": 46.873085021972656,
      "activations/layer12_attention_weight_min": -29.244287490844727,
      "activations/layer13_attention_weight_max": 47.734657287597656,
      "activations/layer13_attention_weight_min": -37.477813720703125,
      "activations/layer14_attention_weight_max": 72.80894470214844,
      "activations/layer14_attention_weight_min": -52.577518463134766,
      "activations/layer15_attention_weight_max": 50.12885284423828,
      "activations/layer15_attention_weight_min": -34.450592041015625,
      "activations/layer16_attention_weight_max": 42.29713439941406,
      "activations/layer16_attention_weight_min": -30.277050018310547,
      "activations/layer17_attention_weight_max": 40.52841567993164,
      "activations/layer17_attention_weight_min": -26.86216163635254,
      "activations/layer18_attention_weight_max": 30.808752059936523,
      "activations/layer18_attention_weight_min": -18.230398178100586,
      "activations/layer19_attention_weight_max": 43.71902847290039,
      "activations/layer19_attention_weight_min": -33.001277923583984,
      "activations/layer1_attention_weight_max": 18.64218521118164,
      "activations/layer1_attention_weight_min": -13.912246704101562,
      "activations/layer20_attention_weight_max": 31.727781295776367,
      "activations/layer20_attention_weight_min": -26.470495223999023,
      "activations/layer21_attention_weight_max": 36.74850845336914,
      "activations/layer21_attention_weight_min": -20.86418342590332,
      "activations/layer22_attention_weight_max": 48.674354553222656,
      "activations/layer22_attention_weight_min": -30.11764907836914,
      "activations/layer23_attention_weight_max": 26.880020141601562,
      "activations/layer23_attention_weight_min": -19.1057186126709,
      "activations/layer2_attention_weight_max": 30.69696617126465,
      "activations/layer2_attention_weight_min": -29.639080047607422,
      "activations/layer3_attention_weight_max": 79.01476287841797,
      "activations/layer3_attention_weight_min": -85.3541030883789,
      "activations/layer4_attention_weight_max": 130.22447204589844,
      "activations/layer4_attention_weight_min": -126.92769622802734,
      "activations/layer5_attention_weight_max": 100.75566864013672,
      "activations/layer5_attention_weight_min": -91.97432708740234,
      "activations/layer6_attention_weight_max": 60.6104736328125,
      "activations/layer6_attention_weight_min": -53.298927307128906,
      "activations/layer7_attention_weight_max": 73.95177459716797,
      "activations/layer7_attention_weight_min": -74.15948486328125,
      "activations/layer8_attention_weight_max": 39.29659652709961,
      "activations/layer8_attention_weight_min": -36.02204895019531,
      "activations/layer9_attention_weight_max": 51.91402053833008,
      "activations/layer9_attention_weight_min": -50.556114196777344,
      "epoch": 2.98,
      "learning_rate": 0.0001320871212121212,
      "loss": 2.9152,
      "step": 51350
    },
    {
      "activations/layer0_attention_weight_max": 15.268980979919434,
      "activations/layer0_attention_weight_min": -12.055464744567871,
      "activations/layer10_attention_weight_max": 43.220672607421875,
      "activations/layer10_attention_weight_min": -35.52594757080078,
      "activations/layer11_attention_weight_max": 34.34415817260742,
      "activations/layer11_attention_weight_min": -28.11903953552246,
      "activations/layer12_attention_weight_max": 30.901166915893555,
      "activations/layer12_attention_weight_min": -28.607648849487305,
      "activations/layer13_attention_weight_max": 54.88112258911133,
      "activations/layer13_attention_weight_min": -37.70771026611328,
      "activations/layer14_attention_weight_max": 90.54684448242188,
      "activations/layer14_attention_weight_min": -65.40121459960938,
      "activations/layer15_attention_weight_max": 47.516624450683594,
      "activations/layer15_attention_weight_min": -29.8358097076416,
      "activations/layer16_attention_weight_max": 42.663238525390625,
      "activations/layer16_attention_weight_min": -31.718229293823242,
      "activations/layer17_attention_weight_max": 44.31801223754883,
      "activations/layer17_attention_weight_min": -27.119062423706055,
      "activations/layer18_attention_weight_max": 31.7673397064209,
      "activations/layer18_attention_weight_min": -19.38824462890625,
      "activations/layer19_attention_weight_max": 41.31948471069336,
      "activations/layer19_attention_weight_min": -28.0422306060791,
      "activations/layer1_attention_weight_max": 17.431838989257812,
      "activations/layer1_attention_weight_min": -12.236177444458008,
      "activations/layer20_attention_weight_max": 34.9835319519043,
      "activations/layer20_attention_weight_min": -23.147706985473633,
      "activations/layer21_attention_weight_max": 36.620399475097656,
      "activations/layer21_attention_weight_min": -21.228878021240234,
      "activations/layer22_attention_weight_max": 49.518646240234375,
      "activations/layer22_attention_weight_min": -27.470491409301758,
      "activations/layer23_attention_weight_max": 25.549091339111328,
      "activations/layer23_attention_weight_min": -18.360122680664062,
      "activations/layer2_attention_weight_max": 30.26783561706543,
      "activations/layer2_attention_weight_min": -29.3723201751709,
      "activations/layer3_attention_weight_max": 80.13877868652344,
      "activations/layer3_attention_weight_min": -84.80199432373047,
      "activations/layer4_attention_weight_max": 128.1065216064453,
      "activations/layer4_attention_weight_min": -126.09803771972656,
      "activations/layer5_attention_weight_max": 104.10476684570312,
      "activations/layer5_attention_weight_min": -95.22200775146484,
      "activations/layer6_attention_weight_max": 58.022621154785156,
      "activations/layer6_attention_weight_min": -56.762123107910156,
      "activations/layer7_attention_weight_max": 68.36125946044922,
      "activations/layer7_attention_weight_min": -70.9819564819336,
      "activations/layer8_attention_weight_max": 37.882423400878906,
      "activations/layer8_attention_weight_min": -37.4593620300293,
      "activations/layer9_attention_weight_max": 53.63185501098633,
      "activations/layer9_attention_weight_min": -38.27668380737305,
      "epoch": 2.99,
      "learning_rate": 0.00013206818181818182,
      "loss": 2.9133,
      "step": 51400
    },
    {
      "activations/layer0_attention_weight_max": 14.548605918884277,
      "activations/layer0_attention_weight_min": -11.38251781463623,
      "activations/layer10_attention_weight_max": 45.048553466796875,
      "activations/layer10_attention_weight_min": -35.566017150878906,
      "activations/layer11_attention_weight_max": 40.297019958496094,
      "activations/layer11_attention_weight_min": -31.69574737548828,
      "activations/layer12_attention_weight_max": 42.80445098876953,
      "activations/layer12_attention_weight_min": -29.60000228881836,
      "activations/layer13_attention_weight_max": 67.74632263183594,
      "activations/layer13_attention_weight_min": -43.22220993041992,
      "activations/layer14_attention_weight_max": 98.15921020507812,
      "activations/layer14_attention_weight_min": -61.78957748413086,
      "activations/layer15_attention_weight_max": 55.743675231933594,
      "activations/layer15_attention_weight_min": -40.0365104675293,
      "activations/layer16_attention_weight_max": 57.11699676513672,
      "activations/layer16_attention_weight_min": -29.340702056884766,
      "activations/layer17_attention_weight_max": 47.859153747558594,
      "activations/layer17_attention_weight_min": -28.80025863647461,
      "activations/layer18_attention_weight_max": 34.52351379394531,
      "activations/layer18_attention_weight_min": -19.02517318725586,
      "activations/layer19_attention_weight_max": 60.790199279785156,
      "activations/layer19_attention_weight_min": -31.635507583618164,
      "activations/layer1_attention_weight_max": 17.343944549560547,
      "activations/layer1_attention_weight_min": -14.446672439575195,
      "activations/layer20_attention_weight_max": 43.885650634765625,
      "activations/layer20_attention_weight_min": -26.957988739013672,
      "activations/layer21_attention_weight_max": 45.68862533569336,
      "activations/layer21_attention_weight_min": -22.393945693969727,
      "activations/layer22_attention_weight_max": 61.4937744140625,
      "activations/layer22_attention_weight_min": -26.420610427856445,
      "activations/layer23_attention_weight_max": 37.37870788574219,
      "activations/layer23_attention_weight_min": -16.04953384399414,
      "activations/layer2_attention_weight_max": 29.769447326660156,
      "activations/layer2_attention_weight_min": -28.97390365600586,
      "activations/layer3_attention_weight_max": 78.27195739746094,
      "activations/layer3_attention_weight_min": -78.21327209472656,
      "activations/layer4_attention_weight_max": 135.35037231445312,
      "activations/layer4_attention_weight_min": -121.73412322998047,
      "activations/layer5_attention_weight_max": 113.6409912109375,
      "activations/layer5_attention_weight_min": -95.55781555175781,
      "activations/layer6_attention_weight_max": 63.9871826171875,
      "activations/layer6_attention_weight_min": -59.04357147216797,
      "activations/layer7_attention_weight_max": 78.28441619873047,
      "activations/layer7_attention_weight_min": -69.82371520996094,
      "activations/layer8_attention_weight_max": 45.84150314331055,
      "activations/layer8_attention_weight_min": -39.8177375793457,
      "activations/layer9_attention_weight_max": 52.0474739074707,
      "activations/layer9_attention_weight_min": -44.553096771240234,
      "epoch": 2.99,
      "learning_rate": 0.0001320492424242424,
      "loss": 2.9139,
      "step": 51450
    },
    {
      "activations/layer0_attention_weight_max": 15.665061950683594,
      "activations/layer0_attention_weight_min": -12.202388763427734,
      "activations/layer10_attention_weight_max": 45.66708755493164,
      "activations/layer10_attention_weight_min": -34.26721954345703,
      "activations/layer11_attention_weight_max": 36.29130935668945,
      "activations/layer11_attention_weight_min": -32.12782287597656,
      "activations/layer12_attention_weight_max": 28.638553619384766,
      "activations/layer12_attention_weight_min": -26.098773956298828,
      "activations/layer13_attention_weight_max": 46.96154022216797,
      "activations/layer13_attention_weight_min": -33.46992492675781,
      "activations/layer14_attention_weight_max": 80.44273376464844,
      "activations/layer14_attention_weight_min": -58.533687591552734,
      "activations/layer15_attention_weight_max": 46.80897903442383,
      "activations/layer15_attention_weight_min": -30.246116638183594,
      "activations/layer16_attention_weight_max": 43.79731369018555,
      "activations/layer16_attention_weight_min": -31.902849197387695,
      "activations/layer17_attention_weight_max": 47.67448043823242,
      "activations/layer17_attention_weight_min": -27.477827072143555,
      "activations/layer18_attention_weight_max": 32.681663513183594,
      "activations/layer18_attention_weight_min": -17.764219284057617,
      "activations/layer19_attention_weight_max": 44.66585922241211,
      "activations/layer19_attention_weight_min": -28.862802505493164,
      "activations/layer1_attention_weight_max": 17.027179718017578,
      "activations/layer1_attention_weight_min": -12.754016876220703,
      "activations/layer20_attention_weight_max": 33.713409423828125,
      "activations/layer20_attention_weight_min": -22.36161994934082,
      "activations/layer21_attention_weight_max": 31.609045028686523,
      "activations/layer21_attention_weight_min": -19.93136978149414,
      "activations/layer22_attention_weight_max": 53.69819641113281,
      "activations/layer22_attention_weight_min": -25.934186935424805,
      "activations/layer23_attention_weight_max": 28.489871978759766,
      "activations/layer23_attention_weight_min": -17.270042419433594,
      "activations/layer2_attention_weight_max": 30.35733413696289,
      "activations/layer2_attention_weight_min": -31.658178329467773,
      "activations/layer3_attention_weight_max": 82.14310455322266,
      "activations/layer3_attention_weight_min": -86.47786712646484,
      "activations/layer4_attention_weight_max": 132.65138244628906,
      "activations/layer4_attention_weight_min": -130.98703002929688,
      "activations/layer5_attention_weight_max": 104.9287109375,
      "activations/layer5_attention_weight_min": -97.93053436279297,
      "activations/layer6_attention_weight_max": 58.32393264770508,
      "activations/layer6_attention_weight_min": -56.02231979370117,
      "activations/layer7_attention_weight_max": 69.61448669433594,
      "activations/layer7_attention_weight_min": -69.32964324951172,
      "activations/layer8_attention_weight_max": 41.360877990722656,
      "activations/layer8_attention_weight_min": -37.26747512817383,
      "activations/layer9_attention_weight_max": 43.19134521484375,
      "activations/layer9_attention_weight_min": -41.50065231323242,
      "epoch": 2.99,
      "learning_rate": 0.00013203030303030302,
      "loss": 2.9,
      "step": 51500
    },
    {
      "activations/layer0_attention_weight_max": 17.400053024291992,
      "activations/layer0_attention_weight_min": -12.38379192352295,
      "activations/layer10_attention_weight_max": 41.51319122314453,
      "activations/layer10_attention_weight_min": -33.21897506713867,
      "activations/layer11_attention_weight_max": 37.33272933959961,
      "activations/layer11_attention_weight_min": -28.829235076904297,
      "activations/layer12_attention_weight_max": 31.361984252929688,
      "activations/layer12_attention_weight_min": -25.068519592285156,
      "activations/layer13_attention_weight_max": 45.788150787353516,
      "activations/layer13_attention_weight_min": -34.740570068359375,
      "activations/layer14_attention_weight_max": 70.37639617919922,
      "activations/layer14_attention_weight_min": -51.629371643066406,
      "activations/layer15_attention_weight_max": 42.83929443359375,
      "activations/layer15_attention_weight_min": -30.62260627746582,
      "activations/layer16_attention_weight_max": 53.98936462402344,
      "activations/layer16_attention_weight_min": -38.46812057495117,
      "activations/layer17_attention_weight_max": 40.39581298828125,
      "activations/layer17_attention_weight_min": -27.79644203186035,
      "activations/layer18_attention_weight_max": 28.558269500732422,
      "activations/layer18_attention_weight_min": -21.107730865478516,
      "activations/layer19_attention_weight_max": 44.42477798461914,
      "activations/layer19_attention_weight_min": -32.285396575927734,
      "activations/layer1_attention_weight_max": 16.68850326538086,
      "activations/layer1_attention_weight_min": -13.187111854553223,
      "activations/layer20_attention_weight_max": 29.84857177734375,
      "activations/layer20_attention_weight_min": -24.7663516998291,
      "activations/layer21_attention_weight_max": 34.99967956542969,
      "activations/layer21_attention_weight_min": -21.022581100463867,
      "activations/layer22_attention_weight_max": 44.66208267211914,
      "activations/layer22_attention_weight_min": -30.4353084564209,
      "activations/layer23_attention_weight_max": 26.16598892211914,
      "activations/layer23_attention_weight_min": -17.58824920654297,
      "activations/layer2_attention_weight_max": 29.71141815185547,
      "activations/layer2_attention_weight_min": -29.950584411621094,
      "activations/layer3_attention_weight_max": 77.9289779663086,
      "activations/layer3_attention_weight_min": -82.9761962890625,
      "activations/layer4_attention_weight_max": 124.00296783447266,
      "activations/layer4_attention_weight_min": -126.45783996582031,
      "activations/layer5_attention_weight_max": 101.96611022949219,
      "activations/layer5_attention_weight_min": -93.8565673828125,
      "activations/layer6_attention_weight_max": 57.458160400390625,
      "activations/layer6_attention_weight_min": -49.94370651245117,
      "activations/layer7_attention_weight_max": 70.22886657714844,
      "activations/layer7_attention_weight_min": -67.21326446533203,
      "activations/layer8_attention_weight_max": 43.230316162109375,
      "activations/layer8_attention_weight_min": -36.123931884765625,
      "activations/layer9_attention_weight_max": 46.18837356567383,
      "activations/layer9_attention_weight_min": -39.03325271606445,
      "epoch": 3.0,
      "learning_rate": 0.00013201136363636364,
      "loss": 2.8985,
      "step": 51550
    },
    {
      "activations/layer0_attention_weight_max": 17.184125900268555,
      "activations/layer0_attention_weight_min": -11.987546920776367,
      "activations/layer10_attention_weight_max": 40.86155700683594,
      "activations/layer10_attention_weight_min": -36.08154296875,
      "activations/layer11_attention_weight_max": 36.60047149658203,
      "activations/layer11_attention_weight_min": -30.08881950378418,
      "activations/layer12_attention_weight_max": 40.45292282104492,
      "activations/layer12_attention_weight_min": -30.330663681030273,
      "activations/layer13_attention_weight_max": 54.63220977783203,
      "activations/layer13_attention_weight_min": -38.781837463378906,
      "activations/layer14_attention_weight_max": 76.51451110839844,
      "activations/layer14_attention_weight_min": -56.834930419921875,
      "activations/layer15_attention_weight_max": 52.05697250366211,
      "activations/layer15_attention_weight_min": -36.245765686035156,
      "activations/layer16_attention_weight_max": 44.93236541748047,
      "activations/layer16_attention_weight_min": -29.260848999023438,
      "activations/layer17_attention_weight_max": 48.384254455566406,
      "activations/layer17_attention_weight_min": -26.50400733947754,
      "activations/layer18_attention_weight_max": 29.642545700073242,
      "activations/layer18_attention_weight_min": -17.569398880004883,
      "activations/layer19_attention_weight_max": 42.57230758666992,
      "activations/layer19_attention_weight_min": -27.94235610961914,
      "activations/layer1_attention_weight_max": 18.6995906829834,
      "activations/layer1_attention_weight_min": -13.480133056640625,
      "activations/layer20_attention_weight_max": 31.08783531188965,
      "activations/layer20_attention_weight_min": -23.053218841552734,
      "activations/layer21_attention_weight_max": 29.841794967651367,
      "activations/layer21_attention_weight_min": -22.95895767211914,
      "activations/layer22_attention_weight_max": 43.98968505859375,
      "activations/layer22_attention_weight_min": -30.557708740234375,
      "activations/layer23_attention_weight_max": 26.506200790405273,
      "activations/layer23_attention_weight_min": -17.585773468017578,
      "activations/layer2_attention_weight_max": 31.40229606628418,
      "activations/layer2_attention_weight_min": -30.02065658569336,
      "activations/layer3_attention_weight_max": 84.89857482910156,
      "activations/layer3_attention_weight_min": -82.50474548339844,
      "activations/layer4_attention_weight_max": 132.5036163330078,
      "activations/layer4_attention_weight_min": -128.086181640625,
      "activations/layer5_attention_weight_max": 101.72535705566406,
      "activations/layer5_attention_weight_min": -92.04265594482422,
      "activations/layer6_attention_weight_max": 59.08070373535156,
      "activations/layer6_attention_weight_min": -56.31708526611328,
      "activations/layer7_attention_weight_max": 72.6219253540039,
      "activations/layer7_attention_weight_min": -70.83882141113281,
      "activations/layer8_attention_weight_max": 40.30070877075195,
      "activations/layer8_attention_weight_min": -37.060707092285156,
      "activations/layer9_attention_weight_max": 48.791316986083984,
      "activations/layer9_attention_weight_min": -44.572967529296875,
      "epoch": 3.0,
      "learning_rate": 0.00013199242424242422,
      "loss": 2.913,
      "step": 51600
    },
    {
      "activations/layer0_attention_weight_max": 16.979549407958984,
      "activations/layer0_attention_weight_min": -12.373658180236816,
      "activations/layer10_attention_weight_max": 42.181678771972656,
      "activations/layer10_attention_weight_min": -33.973976135253906,
      "activations/layer11_attention_weight_max": 38.38822937011719,
      "activations/layer11_attention_weight_min": -33.53221893310547,
      "activations/layer12_attention_weight_max": 34.834320068359375,
      "activations/layer12_attention_weight_min": -25.564340591430664,
      "activations/layer13_attention_weight_max": 54.21098709106445,
      "activations/layer13_attention_weight_min": -37.977230072021484,
      "activations/layer14_attention_weight_max": 66.64591979980469,
      "activations/layer14_attention_weight_min": -50.57364273071289,
      "activations/layer15_attention_weight_max": 43.646018981933594,
      "activations/layer15_attention_weight_min": -32.9865837097168,
      "activations/layer16_attention_weight_max": 41.04371643066406,
      "activations/layer16_attention_weight_min": -33.950294494628906,
      "activations/layer17_attention_weight_max": 45.06489562988281,
      "activations/layer17_attention_weight_min": -27.609548568725586,
      "activations/layer18_attention_weight_max": 30.804048538208008,
      "activations/layer18_attention_weight_min": -21.030385971069336,
      "activations/layer19_attention_weight_max": 43.80887985229492,
      "activations/layer19_attention_weight_min": -29.04144287109375,
      "activations/layer1_attention_weight_max": 18.323993682861328,
      "activations/layer1_attention_weight_min": -12.18466854095459,
      "activations/layer20_attention_weight_max": 38.844581604003906,
      "activations/layer20_attention_weight_min": -23.84614372253418,
      "activations/layer21_attention_weight_max": 33.6676025390625,
      "activations/layer21_attention_weight_min": -20.033615112304688,
      "activations/layer22_attention_weight_max": 51.152706146240234,
      "activations/layer22_attention_weight_min": -28.519184112548828,
      "activations/layer23_attention_weight_max": 30.806570053100586,
      "activations/layer23_attention_weight_min": -17.391868591308594,
      "activations/layer2_attention_weight_max": 30.698223114013672,
      "activations/layer2_attention_weight_min": -27.97688865661621,
      "activations/layer3_attention_weight_max": 80.94933319091797,
      "activations/layer3_attention_weight_min": -80.35182189941406,
      "activations/layer4_attention_weight_max": 149.91014099121094,
      "activations/layer4_attention_weight_min": -128.11312866210938,
      "activations/layer5_attention_weight_max": 107.78929138183594,
      "activations/layer5_attention_weight_min": -92.2772445678711,
      "activations/layer6_attention_weight_max": 61.45102310180664,
      "activations/layer6_attention_weight_min": -53.14194869995117,
      "activations/layer7_attention_weight_max": 80.26522827148438,
      "activations/layer7_attention_weight_min": -69.54193115234375,
      "activations/layer8_attention_weight_max": 42.92091751098633,
      "activations/layer8_attention_weight_min": -36.45128631591797,
      "activations/layer9_attention_weight_max": 53.576026916503906,
      "activations/layer9_attention_weight_min": -40.46150588989258,
      "epoch": 3.0,
      "learning_rate": 0.00013197348484848484,
      "loss": 2.9551,
      "step": 51650
    },
    {
      "activations/layer0_attention_weight_max": 17.75270652770996,
      "activations/layer0_attention_weight_min": -11.382561683654785,
      "activations/layer10_attention_weight_max": 44.9436149597168,
      "activations/layer10_attention_weight_min": -34.71794891357422,
      "activations/layer11_attention_weight_max": 38.17934799194336,
      "activations/layer11_attention_weight_min": -32.07209777832031,
      "activations/layer12_attention_weight_max": 49.99433517456055,
      "activations/layer12_attention_weight_min": -29.752727508544922,
      "activations/layer13_attention_weight_max": 65.89981079101562,
      "activations/layer13_attention_weight_min": -45.02027130126953,
      "activations/layer14_attention_weight_max": 80.58104705810547,
      "activations/layer14_attention_weight_min": -54.23373031616211,
      "activations/layer15_attention_weight_max": 62.705299377441406,
      "activations/layer15_attention_weight_min": -41.24258804321289,
      "activations/layer16_attention_weight_max": 47.813201904296875,
      "activations/layer16_attention_weight_min": -33.38456344604492,
      "activations/layer17_attention_weight_max": 45.520545959472656,
      "activations/layer17_attention_weight_min": -27.1802921295166,
      "activations/layer18_attention_weight_max": 33.2203254699707,
      "activations/layer18_attention_weight_min": -22.231515884399414,
      "activations/layer19_attention_weight_max": 49.16148376464844,
      "activations/layer19_attention_weight_min": -31.551681518554688,
      "activations/layer1_attention_weight_max": 18.098825454711914,
      "activations/layer1_attention_weight_min": -13.01831340789795,
      "activations/layer20_attention_weight_max": 40.59842300415039,
      "activations/layer20_attention_weight_min": -23.631540298461914,
      "activations/layer21_attention_weight_max": 39.87744140625,
      "activations/layer21_attention_weight_min": -21.986825942993164,
      "activations/layer22_attention_weight_max": 53.10386276245117,
      "activations/layer22_attention_weight_min": -27.365156173706055,
      "activations/layer23_attention_weight_max": 30.97964859008789,
      "activations/layer23_attention_weight_min": -17.800643920898438,
      "activations/layer2_attention_weight_max": 30.197736740112305,
      "activations/layer2_attention_weight_min": -29.471492767333984,
      "activations/layer3_attention_weight_max": 79.82428741455078,
      "activations/layer3_attention_weight_min": -79.96699523925781,
      "activations/layer4_attention_weight_max": 128.94015502929688,
      "activations/layer4_attention_weight_min": -122.72517395019531,
      "activations/layer5_attention_weight_max": 98.26811218261719,
      "activations/layer5_attention_weight_min": -92.64439392089844,
      "activations/layer6_attention_weight_max": 59.619266510009766,
      "activations/layer6_attention_weight_min": -56.29319381713867,
      "activations/layer7_attention_weight_max": 77.11188507080078,
      "activations/layer7_attention_weight_min": -72.64564514160156,
      "activations/layer8_attention_weight_max": 42.93769454956055,
      "activations/layer8_attention_weight_min": -37.87453079223633,
      "activations/layer9_attention_weight_max": 52.117366790771484,
      "activations/layer9_attention_weight_min": -43.16246032714844,
      "epoch": 3.0,
      "learning_rate": 0.00013195454545454543,
      "loss": 2.9021,
      "step": 51700
    },
    {
      "activations/layer0_attention_weight_max": 17.824155807495117,
      "activations/layer0_attention_weight_min": -13.100679397583008,
      "activations/layer10_attention_weight_max": 41.1375846862793,
      "activations/layer10_attention_weight_min": -34.28471755981445,
      "activations/layer11_attention_weight_max": 39.630943298339844,
      "activations/layer11_attention_weight_min": -32.045528411865234,
      "activations/layer12_attention_weight_max": 29.66387176513672,
      "activations/layer12_attention_weight_min": -27.536849975585938,
      "activations/layer13_attention_weight_max": 50.72042465209961,
      "activations/layer13_attention_weight_min": -35.44743347167969,
      "activations/layer14_attention_weight_max": 72.78021240234375,
      "activations/layer14_attention_weight_min": -51.80439758300781,
      "activations/layer15_attention_weight_max": 45.093570709228516,
      "activations/layer15_attention_weight_min": -32.14297103881836,
      "activations/layer16_attention_weight_max": 47.64140319824219,
      "activations/layer16_attention_weight_min": -31.82651710510254,
      "activations/layer17_attention_weight_max": 42.57695388793945,
      "activations/layer17_attention_weight_min": -28.200462341308594,
      "activations/layer18_attention_weight_max": 31.271989822387695,
      "activations/layer18_attention_weight_min": -18.76274871826172,
      "activations/layer19_attention_weight_max": 45.032554626464844,
      "activations/layer19_attention_weight_min": -28.62885856628418,
      "activations/layer1_attention_weight_max": 18.09099769592285,
      "activations/layer1_attention_weight_min": -12.228489875793457,
      "activations/layer20_attention_weight_max": 34.79964065551758,
      "activations/layer20_attention_weight_min": -23.35795783996582,
      "activations/layer21_attention_weight_max": 29.36117935180664,
      "activations/layer21_attention_weight_min": -20.924541473388672,
      "activations/layer22_attention_weight_max": 54.87826919555664,
      "activations/layer22_attention_weight_min": -26.864641189575195,
      "activations/layer23_attention_weight_max": 28.312278747558594,
      "activations/layer23_attention_weight_min": -16.831623077392578,
      "activations/layer2_attention_weight_max": 29.594867706298828,
      "activations/layer2_attention_weight_min": -28.217222213745117,
      "activations/layer3_attention_weight_max": 78.9933090209961,
      "activations/layer3_attention_weight_min": -79.03752899169922,
      "activations/layer4_attention_weight_max": 128.66278076171875,
      "activations/layer4_attention_weight_min": -119.76079559326172,
      "activations/layer5_attention_weight_max": 101.07588195800781,
      "activations/layer5_attention_weight_min": -88.79924774169922,
      "activations/layer6_attention_weight_max": 56.83316421508789,
      "activations/layer6_attention_weight_min": -55.95112228393555,
      "activations/layer7_attention_weight_max": 73.60787963867188,
      "activations/layer7_attention_weight_min": -68.16016387939453,
      "activations/layer8_attention_weight_max": 40.004425048828125,
      "activations/layer8_attention_weight_min": -39.15858840942383,
      "activations/layer9_attention_weight_max": 46.849822998046875,
      "activations/layer9_attention_weight_min": -38.01999282836914,
      "epoch": 3.01,
      "learning_rate": 0.00013193560606060604,
      "loss": 2.9047,
      "step": 51750
    },
    {
      "activations/layer0_attention_weight_max": 17.45379066467285,
      "activations/layer0_attention_weight_min": -11.917818069458008,
      "activations/layer10_attention_weight_max": 44.67634201049805,
      "activations/layer10_attention_weight_min": -32.092689514160156,
      "activations/layer11_attention_weight_max": 38.09553527832031,
      "activations/layer11_attention_weight_min": -31.048686981201172,
      "activations/layer12_attention_weight_max": 32.7950325012207,
      "activations/layer12_attention_weight_min": -26.154457092285156,
      "activations/layer13_attention_weight_max": 55.06377029418945,
      "activations/layer13_attention_weight_min": -37.37791442871094,
      "activations/layer14_attention_weight_max": 75.99938201904297,
      "activations/layer14_attention_weight_min": -50.236305236816406,
      "activations/layer15_attention_weight_max": 51.84169006347656,
      "activations/layer15_attention_weight_min": -33.127418518066406,
      "activations/layer16_attention_weight_max": 45.522926330566406,
      "activations/layer16_attention_weight_min": -34.22364044189453,
      "activations/layer17_attention_weight_max": 48.05117416381836,
      "activations/layer17_attention_weight_min": -25.72352409362793,
      "activations/layer18_attention_weight_max": 29.48369789123535,
      "activations/layer18_attention_weight_min": -19.145816802978516,
      "activations/layer19_attention_weight_max": 49.912635803222656,
      "activations/layer19_attention_weight_min": -30.6513729095459,
      "activations/layer1_attention_weight_max": 17.587146759033203,
      "activations/layer1_attention_weight_min": -15.351866722106934,
      "activations/layer20_attention_weight_max": 37.49399185180664,
      "activations/layer20_attention_weight_min": -25.197914123535156,
      "activations/layer21_attention_weight_max": 30.49669647216797,
      "activations/layer21_attention_weight_min": -21.163976669311523,
      "activations/layer22_attention_weight_max": 54.0407600402832,
      "activations/layer22_attention_weight_min": -28.071279525756836,
      "activations/layer23_attention_weight_max": 29.52553939819336,
      "activations/layer23_attention_weight_min": -17.3151798248291,
      "activations/layer2_attention_weight_max": 30.337745666503906,
      "activations/layer2_attention_weight_min": -29.659290313720703,
      "activations/layer3_attention_weight_max": 81.8431167602539,
      "activations/layer3_attention_weight_min": -79.82051849365234,
      "activations/layer4_attention_weight_max": 130.39256286621094,
      "activations/layer4_attention_weight_min": -117.34815979003906,
      "activations/layer5_attention_weight_max": 102.97886657714844,
      "activations/layer5_attention_weight_min": -89.07860565185547,
      "activations/layer6_attention_weight_max": 58.82572555541992,
      "activations/layer6_attention_weight_min": -49.37527084350586,
      "activations/layer7_attention_weight_max": 70.75789642333984,
      "activations/layer7_attention_weight_min": -70.43646240234375,
      "activations/layer8_attention_weight_max": 39.21485137939453,
      "activations/layer8_attention_weight_min": -34.7061653137207,
      "activations/layer9_attention_weight_max": 48.86701965332031,
      "activations/layer9_attention_weight_min": -37.941673278808594,
      "epoch": 3.01,
      "learning_rate": 0.00013191666666666666,
      "loss": 2.9102,
      "step": 51800
    },
    {
      "activations/layer0_attention_weight_max": 15.283263206481934,
      "activations/layer0_attention_weight_min": -11.695609092712402,
      "activations/layer10_attention_weight_max": 46.89383316040039,
      "activations/layer10_attention_weight_min": -38.914485931396484,
      "activations/layer11_attention_weight_max": 45.543697357177734,
      "activations/layer11_attention_weight_min": -34.49732208251953,
      "activations/layer12_attention_weight_max": 42.46215057373047,
      "activations/layer12_attention_weight_min": -34.36756134033203,
      "activations/layer13_attention_weight_max": 61.69148635864258,
      "activations/layer13_attention_weight_min": -39.997398376464844,
      "activations/layer14_attention_weight_max": 76.2961196899414,
      "activations/layer14_attention_weight_min": -53.34674835205078,
      "activations/layer15_attention_weight_max": 59.463905334472656,
      "activations/layer15_attention_weight_min": -34.67244338989258,
      "activations/layer16_attention_weight_max": 44.56050491333008,
      "activations/layer16_attention_weight_min": -31.36980438232422,
      "activations/layer17_attention_weight_max": 49.01582717895508,
      "activations/layer17_attention_weight_min": -25.389019012451172,
      "activations/layer18_attention_weight_max": 28.073246002197266,
      "activations/layer18_attention_weight_min": -18.848770141601562,
      "activations/layer19_attention_weight_max": 53.89437484741211,
      "activations/layer19_attention_weight_min": -28.406545639038086,
      "activations/layer1_attention_weight_max": 19.037288665771484,
      "activations/layer1_attention_weight_min": -14.659933090209961,
      "activations/layer20_attention_weight_max": 37.07229995727539,
      "activations/layer20_attention_weight_min": -24.61797523498535,
      "activations/layer21_attention_weight_max": 28.06248664855957,
      "activations/layer21_attention_weight_min": -18.634878158569336,
      "activations/layer22_attention_weight_max": 54.25725555419922,
      "activations/layer22_attention_weight_min": -27.861787796020508,
      "activations/layer23_attention_weight_max": 29.153915405273438,
      "activations/layer23_attention_weight_min": -16.65224266052246,
      "activations/layer2_attention_weight_max": 30.077083587646484,
      "activations/layer2_attention_weight_min": -30.236328125,
      "activations/layer3_attention_weight_max": 79.0604476928711,
      "activations/layer3_attention_weight_min": -81.58544158935547,
      "activations/layer4_attention_weight_max": 137.24478149414062,
      "activations/layer4_attention_weight_min": -118.9728775024414,
      "activations/layer5_attention_weight_max": 108.16921997070312,
      "activations/layer5_attention_weight_min": -93.05380249023438,
      "activations/layer6_attention_weight_max": 62.06953048706055,
      "activations/layer6_attention_weight_min": -51.743064880371094,
      "activations/layer7_attention_weight_max": 77.7688980102539,
      "activations/layer7_attention_weight_min": -70.59284210205078,
      "activations/layer8_attention_weight_max": 44.88975143432617,
      "activations/layer8_attention_weight_min": -37.89190673828125,
      "activations/layer9_attention_weight_max": 64.97651672363281,
      "activations/layer9_attention_weight_min": -53.73577880859375,
      "epoch": 3.01,
      "learning_rate": 0.00013189772727272724,
      "loss": 2.9177,
      "step": 51850
    },
    {
      "activations/layer0_attention_weight_max": 17.19385528564453,
      "activations/layer0_attention_weight_min": -11.524895668029785,
      "activations/layer10_attention_weight_max": 39.6721076965332,
      "activations/layer10_attention_weight_min": -35.1689453125,
      "activations/layer11_attention_weight_max": 36.06298065185547,
      "activations/layer11_attention_weight_min": -28.302913665771484,
      "activations/layer12_attention_weight_max": 37.84101104736328,
      "activations/layer12_attention_weight_min": -28.602380752563477,
      "activations/layer13_attention_weight_max": 53.08843231201172,
      "activations/layer13_attention_weight_min": -36.571250915527344,
      "activations/layer14_attention_weight_max": 75.78741455078125,
      "activations/layer14_attention_weight_min": -54.42872619628906,
      "activations/layer15_attention_weight_max": 50.38617706298828,
      "activations/layer15_attention_weight_min": -31.92620086669922,
      "activations/layer16_attention_weight_max": 47.329612731933594,
      "activations/layer16_attention_weight_min": -32.03828811645508,
      "activations/layer17_attention_weight_max": 40.13829803466797,
      "activations/layer17_attention_weight_min": -26.966209411621094,
      "activations/layer18_attention_weight_max": 35.000282287597656,
      "activations/layer18_attention_weight_min": -17.765308380126953,
      "activations/layer19_attention_weight_max": 42.27775573730469,
      "activations/layer19_attention_weight_min": -28.135440826416016,
      "activations/layer1_attention_weight_max": 17.919784545898438,
      "activations/layer1_attention_weight_min": -12.452584266662598,
      "activations/layer20_attention_weight_max": 33.07030487060547,
      "activations/layer20_attention_weight_min": -23.45673179626465,
      "activations/layer21_attention_weight_max": 37.079769134521484,
      "activations/layer21_attention_weight_min": -19.414756774902344,
      "activations/layer22_attention_weight_max": 44.73709487915039,
      "activations/layer22_attention_weight_min": -24.912975311279297,
      "activations/layer23_attention_weight_max": 27.09992790222168,
      "activations/layer23_attention_weight_min": -18.31001853942871,
      "activations/layer2_attention_weight_max": 30.100677490234375,
      "activations/layer2_attention_weight_min": -30.651670455932617,
      "activations/layer3_attention_weight_max": 79.88872528076172,
      "activations/layer3_attention_weight_min": -79.96194458007812,
      "activations/layer4_attention_weight_max": 129.8456573486328,
      "activations/layer4_attention_weight_min": -123.15370178222656,
      "activations/layer5_attention_weight_max": 101.66177368164062,
      "activations/layer5_attention_weight_min": -91.8601303100586,
      "activations/layer6_attention_weight_max": 58.18924331665039,
      "activations/layer6_attention_weight_min": -52.22624588012695,
      "activations/layer7_attention_weight_max": 72.19953918457031,
      "activations/layer7_attention_weight_min": -71.0444564819336,
      "activations/layer8_attention_weight_max": 39.83608627319336,
      "activations/layer8_attention_weight_min": -35.40483474731445,
      "activations/layer9_attention_weight_max": 47.73872756958008,
      "activations/layer9_attention_weight_min": -40.25695037841797,
      "epoch": 3.02,
      "learning_rate": 0.00013187878787878786,
      "loss": 2.8948,
      "step": 51900
    },
    {
      "activations/layer0_attention_weight_max": 16.974637985229492,
      "activations/layer0_attention_weight_min": -12.678495407104492,
      "activations/layer10_attention_weight_max": 42.2636604309082,
      "activations/layer10_attention_weight_min": -33.75006103515625,
      "activations/layer11_attention_weight_max": 35.13545227050781,
      "activations/layer11_attention_weight_min": -28.928363800048828,
      "activations/layer12_attention_weight_max": 36.021034240722656,
      "activations/layer12_attention_weight_min": -29.25193977355957,
      "activations/layer13_attention_weight_max": 50.693824768066406,
      "activations/layer13_attention_weight_min": -34.019832611083984,
      "activations/layer14_attention_weight_max": 77.74803924560547,
      "activations/layer14_attention_weight_min": -61.96969985961914,
      "activations/layer15_attention_weight_max": 47.89726638793945,
      "activations/layer15_attention_weight_min": -31.146591186523438,
      "activations/layer16_attention_weight_max": 45.542236328125,
      "activations/layer16_attention_weight_min": -33.754844665527344,
      "activations/layer17_attention_weight_max": 46.78791046142578,
      "activations/layer17_attention_weight_min": -26.20918846130371,
      "activations/layer18_attention_weight_max": 31.1060848236084,
      "activations/layer18_attention_weight_min": -19.791296005249023,
      "activations/layer19_attention_weight_max": 48.3881950378418,
      "activations/layer19_attention_weight_min": -29.63504409790039,
      "activations/layer1_attention_weight_max": 17.93896484375,
      "activations/layer1_attention_weight_min": -14.0680513381958,
      "activations/layer20_attention_weight_max": 35.33107376098633,
      "activations/layer20_attention_weight_min": -22.848796844482422,
      "activations/layer21_attention_weight_max": 39.54462432861328,
      "activations/layer21_attention_weight_min": -19.0943660736084,
      "activations/layer22_attention_weight_max": 50.53546905517578,
      "activations/layer22_attention_weight_min": -24.607160568237305,
      "activations/layer23_attention_weight_max": 30.49672508239746,
      "activations/layer23_attention_weight_min": -16.51580047607422,
      "activations/layer2_attention_weight_max": 31.085783004760742,
      "activations/layer2_attention_weight_min": -31.554668426513672,
      "activations/layer3_attention_weight_max": 82.59683227539062,
      "activations/layer3_attention_weight_min": -81.42889404296875,
      "activations/layer4_attention_weight_max": 131.9566192626953,
      "activations/layer4_attention_weight_min": -126.08074188232422,
      "activations/layer5_attention_weight_max": 101.58665466308594,
      "activations/layer5_attention_weight_min": -94.85865783691406,
      "activations/layer6_attention_weight_max": 60.288818359375,
      "activations/layer6_attention_weight_min": -53.9120979309082,
      "activations/layer7_attention_weight_max": 71.19564819335938,
      "activations/layer7_attention_weight_min": -68.79405975341797,
      "activations/layer8_attention_weight_max": 41.6973991394043,
      "activations/layer8_attention_weight_min": -36.60042953491211,
      "activations/layer9_attention_weight_max": 50.102230072021484,
      "activations/layer9_attention_weight_min": -40.85563278198242,
      "epoch": 3.02,
      "learning_rate": 0.00013185984848484847,
      "loss": 2.9198,
      "step": 51950
    },
    {
      "activations/layer0_attention_weight_max": 17.286026000976562,
      "activations/layer0_attention_weight_min": -11.939901351928711,
      "activations/layer10_attention_weight_max": 42.90403747558594,
      "activations/layer10_attention_weight_min": -33.15850830078125,
      "activations/layer11_attention_weight_max": 35.89239501953125,
      "activations/layer11_attention_weight_min": -29.36353302001953,
      "activations/layer12_attention_weight_max": 34.009735107421875,
      "activations/layer12_attention_weight_min": -26.963905334472656,
      "activations/layer13_attention_weight_max": 51.16861343383789,
      "activations/layer13_attention_weight_min": -35.74249267578125,
      "activations/layer14_attention_weight_max": 80.05751037597656,
      "activations/layer14_attention_weight_min": -60.512306213378906,
      "activations/layer15_attention_weight_max": 48.87401580810547,
      "activations/layer15_attention_weight_min": -34.272369384765625,
      "activations/layer16_attention_weight_max": 46.04100036621094,
      "activations/layer16_attention_weight_min": -31.53782844543457,
      "activations/layer17_attention_weight_max": 42.45547866821289,
      "activations/layer17_attention_weight_min": -25.701478958129883,
      "activations/layer18_attention_weight_max": 33.485260009765625,
      "activations/layer18_attention_weight_min": -21.556312561035156,
      "activations/layer19_attention_weight_max": 47.16359329223633,
      "activations/layer19_attention_weight_min": -30.69989013671875,
      "activations/layer1_attention_weight_max": 17.44830894470215,
      "activations/layer1_attention_weight_min": -13.904635429382324,
      "activations/layer20_attention_weight_max": 39.49484634399414,
      "activations/layer20_attention_weight_min": -22.73601722717285,
      "activations/layer21_attention_weight_max": 39.65620803833008,
      "activations/layer21_attention_weight_min": -24.10317039489746,
      "activations/layer22_attention_weight_max": 51.39812469482422,
      "activations/layer22_attention_weight_min": -28.872726440429688,
      "activations/layer23_attention_weight_max": 27.884708404541016,
      "activations/layer23_attention_weight_min": -18.07624626159668,
      "activations/layer2_attention_weight_max": 31.056896209716797,
      "activations/layer2_attention_weight_min": -29.34243392944336,
      "activations/layer3_attention_weight_max": 77.0289535522461,
      "activations/layer3_attention_weight_min": -83.20672607421875,
      "activations/layer4_attention_weight_max": 125.21051025390625,
      "activations/layer4_attention_weight_min": -127.6814193725586,
      "activations/layer5_attention_weight_max": 102.35015869140625,
      "activations/layer5_attention_weight_min": -98.30880737304688,
      "activations/layer6_attention_weight_max": 57.369972229003906,
      "activations/layer6_attention_weight_min": -56.867576599121094,
      "activations/layer7_attention_weight_max": 74.05199432373047,
      "activations/layer7_attention_weight_min": -72.11566162109375,
      "activations/layer8_attention_weight_max": 42.665687561035156,
      "activations/layer8_attention_weight_min": -37.660797119140625,
      "activations/layer9_attention_weight_max": 48.88553237915039,
      "activations/layer9_attention_weight_min": -38.17747116088867,
      "epoch": 3.02,
      "learning_rate": 0.00013184090909090906,
      "loss": 2.8939,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_loss": 2.859375,
      "eval_runtime": 8.53,
      "eval_samples_per_second": 503.399,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_openwebtext_loss": 2.859375,
      "eval_openwebtext_ppl": 17.45061689198871,
      "eval_openwebtext_runtime": 8.53,
      "eval_openwebtext_samples_per_second": 503.399,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_wikitext_loss": 3.1171875,
      "eval_wikitext_ppl": 22.582776184718522,
      "eval_wikitext_runtime": 2.008,
      "eval_wikitext_samples_per_second": 227.09,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_lambada_loss": 2.880859375,
      "eval_lambada_ppl": 17.829588900657228,
      "eval_lambada_runtime": 9.5927,
      "eval_lambada_samples_per_second": 507.574,
      "step": 52000
    },
    {
      "activations/layer0_attention_weight_max": 17.16170310974121,
      "activations/layer0_attention_weight_min": -13.289912223815918,
      "activations/layer10_attention_weight_max": 43.12198257446289,
      "activations/layer10_attention_weight_min": -34.77818298339844,
      "activations/layer11_attention_weight_max": 37.79993438720703,
      "activations/layer11_attention_weight_min": -29.983434677124023,
      "activations/layer12_attention_weight_max": 35.139747619628906,
      "activations/layer12_attention_weight_min": -28.109363555908203,
      "activations/layer13_attention_weight_max": 57.34382629394531,
      "activations/layer13_attention_weight_min": -32.04308319091797,
      "activations/layer14_attention_weight_max": 96.57893371582031,
      "activations/layer14_attention_weight_min": -56.99232864379883,
      "activations/layer15_attention_weight_max": 55.39755630493164,
      "activations/layer15_attention_weight_min": -32.348106384277344,
      "activations/layer16_attention_weight_max": 57.127418518066406,
      "activations/layer16_attention_weight_min": -34.27582931518555,
      "activations/layer17_attention_weight_max": 55.60206604003906,
      "activations/layer17_attention_weight_min": -27.4953670501709,
      "activations/layer18_attention_weight_max": 33.13084411621094,
      "activations/layer18_attention_weight_min": -18.315889358520508,
      "activations/layer19_attention_weight_max": 54.103912353515625,
      "activations/layer19_attention_weight_min": -29.2753963470459,
      "activations/layer1_attention_weight_max": 17.49036979675293,
      "activations/layer1_attention_weight_min": -15.51951789855957,
      "activations/layer20_attention_weight_max": 39.605438232421875,
      "activations/layer20_attention_weight_min": -23.773632049560547,
      "activations/layer21_attention_weight_max": 35.304176330566406,
      "activations/layer21_attention_weight_min": -19.66990852355957,
      "activations/layer22_attention_weight_max": 60.29507827758789,
      "activations/layer22_attention_weight_min": -24.919282913208008,
      "activations/layer23_attention_weight_max": 31.793012619018555,
      "activations/layer23_attention_weight_min": -15.681467056274414,
      "activations/layer2_attention_weight_max": 30.668235778808594,
      "activations/layer2_attention_weight_min": -28.880882263183594,
      "activations/layer3_attention_weight_max": 80.89873504638672,
      "activations/layer3_attention_weight_min": -80.85284423828125,
      "activations/layer4_attention_weight_max": 131.27474975585938,
      "activations/layer4_attention_weight_min": -119.5197525024414,
      "activations/layer5_attention_weight_max": 105.87155151367188,
      "activations/layer5_attention_weight_min": -90.30805969238281,
      "activations/layer6_attention_weight_max": 59.973365783691406,
      "activations/layer6_attention_weight_min": -51.35635757446289,
      "activations/layer7_attention_weight_max": 73.4736328125,
      "activations/layer7_attention_weight_min": -67.54840850830078,
      "activations/layer8_attention_weight_max": 41.63539505004883,
      "activations/layer8_attention_weight_min": -35.808956146240234,
      "activations/layer9_attention_weight_max": 50.199947357177734,
      "activations/layer9_attention_weight_min": -38.515071868896484,
      "epoch": 3.02,
      "learning_rate": 0.00013182196969696968,
      "loss": 2.909,
      "step": 52050
    },
    {
      "activations/layer0_attention_weight_max": 16.261882781982422,
      "activations/layer0_attention_weight_min": -10.885809898376465,
      "activations/layer10_attention_weight_max": 42.215972900390625,
      "activations/layer10_attention_weight_min": -35.05092239379883,
      "activations/layer11_attention_weight_max": 34.32036590576172,
      "activations/layer11_attention_weight_min": -27.273006439208984,
      "activations/layer12_attention_weight_max": 50.30720901489258,
      "activations/layer12_attention_weight_min": -30.61044692993164,
      "activations/layer13_attention_weight_max": 49.5909309387207,
      "activations/layer13_attention_weight_min": -37.40094757080078,
      "activations/layer14_attention_weight_max": 78.15946197509766,
      "activations/layer14_attention_weight_min": -58.3426513671875,
      "activations/layer15_attention_weight_max": 49.83000183105469,
      "activations/layer15_attention_weight_min": -34.29734420776367,
      "activations/layer16_attention_weight_max": 45.012752532958984,
      "activations/layer16_attention_weight_min": -32.7819709777832,
      "activations/layer17_attention_weight_max": 46.262596130371094,
      "activations/layer17_attention_weight_min": -31.480873107910156,
      "activations/layer18_attention_weight_max": 34.609954833984375,
      "activations/layer18_attention_weight_min": -22.546173095703125,
      "activations/layer19_attention_weight_max": 50.989803314208984,
      "activations/layer19_attention_weight_min": -30.942554473876953,
      "activations/layer1_attention_weight_max": 19.042505264282227,
      "activations/layer1_attention_weight_min": -12.956886291503906,
      "activations/layer20_attention_weight_max": 35.098392486572266,
      "activations/layer20_attention_weight_min": -24.904178619384766,
      "activations/layer21_attention_weight_max": 36.93059158325195,
      "activations/layer21_attention_weight_min": -20.629676818847656,
      "activations/layer22_attention_weight_max": 53.97970199584961,
      "activations/layer22_attention_weight_min": -28.01253318786621,
      "activations/layer23_attention_weight_max": 28.247724533081055,
      "activations/layer23_attention_weight_min": -16.347070693969727,
      "activations/layer2_attention_weight_max": 28.9798583984375,
      "activations/layer2_attention_weight_min": -30.393003463745117,
      "activations/layer3_attention_weight_max": 76.9168472290039,
      "activations/layer3_attention_weight_min": -80.06398010253906,
      "activations/layer4_attention_weight_max": 129.9358673095703,
      "activations/layer4_attention_weight_min": -123.9429702758789,
      "activations/layer5_attention_weight_max": 102.22771453857422,
      "activations/layer5_attention_weight_min": -91.43067932128906,
      "activations/layer6_attention_weight_max": 57.914100646972656,
      "activations/layer6_attention_weight_min": -52.668697357177734,
      "activations/layer7_attention_weight_max": 73.57887268066406,
      "activations/layer7_attention_weight_min": -69.00582122802734,
      "activations/layer8_attention_weight_max": 40.33447265625,
      "activations/layer8_attention_weight_min": -36.114479064941406,
      "activations/layer9_attention_weight_max": 49.609920501708984,
      "activations/layer9_attention_weight_min": -40.85890579223633,
      "epoch": 3.03,
      "learning_rate": 0.0001318030303030303,
      "loss": 2.8972,
      "step": 52100
    },
    {
      "activations/layer0_attention_weight_max": 17.36182975769043,
      "activations/layer0_attention_weight_min": -12.557143211364746,
      "activations/layer10_attention_weight_max": 41.63615417480469,
      "activations/layer10_attention_weight_min": -32.339874267578125,
      "activations/layer11_attention_weight_max": 38.28965377807617,
      "activations/layer11_attention_weight_min": -29.1920223236084,
      "activations/layer12_attention_weight_max": 32.777931213378906,
      "activations/layer12_attention_weight_min": -29.07499885559082,
      "activations/layer13_attention_weight_max": 47.30708694458008,
      "activations/layer13_attention_weight_min": -35.18345642089844,
      "activations/layer14_attention_weight_max": 68.43171691894531,
      "activations/layer14_attention_weight_min": -50.80723190307617,
      "activations/layer15_attention_weight_max": 43.741031646728516,
      "activations/layer15_attention_weight_min": -30.110145568847656,
      "activations/layer16_attention_weight_max": 42.89756774902344,
      "activations/layer16_attention_weight_min": -31.803401947021484,
      "activations/layer17_attention_weight_max": 43.597381591796875,
      "activations/layer17_attention_weight_min": -28.06388282775879,
      "activations/layer18_attention_weight_max": 33.455413818359375,
      "activations/layer18_attention_weight_min": -20.1815242767334,
      "activations/layer19_attention_weight_max": 43.783226013183594,
      "activations/layer19_attention_weight_min": -32.83768081665039,
      "activations/layer1_attention_weight_max": 17.255178451538086,
      "activations/layer1_attention_weight_min": -13.468124389648438,
      "activations/layer20_attention_weight_max": 36.45865249633789,
      "activations/layer20_attention_weight_min": -23.75285530090332,
      "activations/layer21_attention_weight_max": 33.65485763549805,
      "activations/layer21_attention_weight_min": -21.770462036132812,
      "activations/layer22_attention_weight_max": 48.86789321899414,
      "activations/layer22_attention_weight_min": -26.471311569213867,
      "activations/layer23_attention_weight_max": 26.04542350769043,
      "activations/layer23_attention_weight_min": -19.937021255493164,
      "activations/layer2_attention_weight_max": 28.8067626953125,
      "activations/layer2_attention_weight_min": -29.89154624938965,
      "activations/layer3_attention_weight_max": 78.6068344116211,
      "activations/layer3_attention_weight_min": -84.15422821044922,
      "activations/layer4_attention_weight_max": 127.46556854248047,
      "activations/layer4_attention_weight_min": -129.21324157714844,
      "activations/layer5_attention_weight_max": 108.1864013671875,
      "activations/layer5_attention_weight_min": -96.04463958740234,
      "activations/layer6_attention_weight_max": 62.92582702636719,
      "activations/layer6_attention_weight_min": -54.295040130615234,
      "activations/layer7_attention_weight_max": 76.26185607910156,
      "activations/layer7_attention_weight_min": -70.35132598876953,
      "activations/layer8_attention_weight_max": 39.90436553955078,
      "activations/layer8_attention_weight_min": -37.97828674316406,
      "activations/layer9_attention_weight_max": 46.703548431396484,
      "activations/layer9_attention_weight_min": -34.53132247924805,
      "epoch": 3.03,
      "learning_rate": 0.00013178446969696968,
      "loss": 2.9014,
      "step": 52150
    },
    {
      "activations/layer0_attention_weight_max": 17.540855407714844,
      "activations/layer0_attention_weight_min": -11.488593101501465,
      "activations/layer10_attention_weight_max": 45.73423385620117,
      "activations/layer10_attention_weight_min": -33.8963623046875,
      "activations/layer11_attention_weight_max": 40.335594177246094,
      "activations/layer11_attention_weight_min": -30.93843650817871,
      "activations/layer12_attention_weight_max": 64.451904296875,
      "activations/layer12_attention_weight_min": -31.151412963867188,
      "activations/layer13_attention_weight_max": 54.80348587036133,
      "activations/layer13_attention_weight_min": -35.69261169433594,
      "activations/layer14_attention_weight_max": 91.47718048095703,
      "activations/layer14_attention_weight_min": -60.073429107666016,
      "activations/layer15_attention_weight_max": 63.59065246582031,
      "activations/layer15_attention_weight_min": -34.34031677246094,
      "activations/layer16_attention_weight_max": 62.04643630981445,
      "activations/layer16_attention_weight_min": -34.2332878112793,
      "activations/layer17_attention_weight_max": 43.138343811035156,
      "activations/layer17_attention_weight_min": -27.25164222717285,
      "activations/layer18_attention_weight_max": 35.75983428955078,
      "activations/layer18_attention_weight_min": -20.285661697387695,
      "activations/layer19_attention_weight_max": 53.312957763671875,
      "activations/layer19_attention_weight_min": -32.02169418334961,
      "activations/layer1_attention_weight_max": 18.46055793762207,
      "activations/layer1_attention_weight_min": -13.013136863708496,
      "activations/layer20_attention_weight_max": 44.40459442138672,
      "activations/layer20_attention_weight_min": -22.794389724731445,
      "activations/layer21_attention_weight_max": 37.073211669921875,
      "activations/layer21_attention_weight_min": -20.13672637939453,
      "activations/layer22_attention_weight_max": 49.15212631225586,
      "activations/layer22_attention_weight_min": -25.874387741088867,
      "activations/layer23_attention_weight_max": 27.532817840576172,
      "activations/layer23_attention_weight_min": -17.433094024658203,
      "activations/layer2_attention_weight_max": 29.114538192749023,
      "activations/layer2_attention_weight_min": -27.87417221069336,
      "activations/layer3_attention_weight_max": 77.35067749023438,
      "activations/layer3_attention_weight_min": -83.13784790039062,
      "activations/layer4_attention_weight_max": 130.91697692871094,
      "activations/layer4_attention_weight_min": -126.96569061279297,
      "activations/layer5_attention_weight_max": 104.76676940917969,
      "activations/layer5_attention_weight_min": -94.63567352294922,
      "activations/layer6_attention_weight_max": 61.162986755371094,
      "activations/layer6_attention_weight_min": -54.346290588378906,
      "activations/layer7_attention_weight_max": 87.1478271484375,
      "activations/layer7_attention_weight_min": -78.74862670898438,
      "activations/layer8_attention_weight_max": 46.61565399169922,
      "activations/layer8_attention_weight_min": -40.08818054199219,
      "activations/layer9_attention_weight_max": 61.04926681518555,
      "activations/layer9_attention_weight_min": -45.09825897216797,
      "epoch": 3.03,
      "learning_rate": 0.0001317655303030303,
      "loss": 2.9084,
      "step": 52200
    },
    {
      "activations/layer0_attention_weight_max": 15.913914680480957,
      "activations/layer0_attention_weight_min": -11.988607406616211,
      "activations/layer10_attention_weight_max": 40.41725540161133,
      "activations/layer10_attention_weight_min": -34.47835159301758,
      "activations/layer11_attention_weight_max": 34.742244720458984,
      "activations/layer11_attention_weight_min": -28.532291412353516,
      "activations/layer12_attention_weight_max": 48.25346374511719,
      "activations/layer12_attention_weight_min": -27.857568740844727,
      "activations/layer13_attention_weight_max": 47.30660629272461,
      "activations/layer13_attention_weight_min": -33.841243743896484,
      "activations/layer14_attention_weight_max": 64.76564025878906,
      "activations/layer14_attention_weight_min": -49.741859436035156,
      "activations/layer15_attention_weight_max": 51.38844299316406,
      "activations/layer15_attention_weight_min": -39.989471435546875,
      "activations/layer16_attention_weight_max": 42.71671676635742,
      "activations/layer16_attention_weight_min": -31.85797119140625,
      "activations/layer17_attention_weight_max": 42.2574348449707,
      "activations/layer17_attention_weight_min": -25.95377540588379,
      "activations/layer18_attention_weight_max": 28.39324378967285,
      "activations/layer18_attention_weight_min": -19.395666122436523,
      "activations/layer19_attention_weight_max": 39.77946090698242,
      "activations/layer19_attention_weight_min": -29.485401153564453,
      "activations/layer1_attention_weight_max": 18.191179275512695,
      "activations/layer1_attention_weight_min": -13.007784843444824,
      "activations/layer20_attention_weight_max": 36.16537857055664,
      "activations/layer20_attention_weight_min": -21.948898315429688,
      "activations/layer21_attention_weight_max": 33.194576263427734,
      "activations/layer21_attention_weight_min": -20.080005645751953,
      "activations/layer22_attention_weight_max": 45.46319580078125,
      "activations/layer22_attention_weight_min": -23.98392677307129,
      "activations/layer23_attention_weight_max": 28.729969024658203,
      "activations/layer23_attention_weight_min": -15.807901382446289,
      "activations/layer2_attention_weight_max": 29.135150909423828,
      "activations/layer2_attention_weight_min": -29.090347290039062,
      "activations/layer3_attention_weight_max": 78.33293914794922,
      "activations/layer3_attention_weight_min": -82.27518463134766,
      "activations/layer4_attention_weight_max": 127.81119537353516,
      "activations/layer4_attention_weight_min": -122.55069732666016,
      "activations/layer5_attention_weight_max": 97.21598815917969,
      "activations/layer5_attention_weight_min": -91.30908203125,
      "activations/layer6_attention_weight_max": 58.08830261230469,
      "activations/layer6_attention_weight_min": -51.791080474853516,
      "activations/layer7_attention_weight_max": 73.44300079345703,
      "activations/layer7_attention_weight_min": -65.80209350585938,
      "activations/layer8_attention_weight_max": 37.66078186035156,
      "activations/layer8_attention_weight_min": -34.844032287597656,
      "activations/layer9_attention_weight_max": 46.3570671081543,
      "activations/layer9_attention_weight_min": -48.72121047973633,
      "epoch": 3.04,
      "learning_rate": 0.0001317465909090909,
      "loss": 2.8923,
      "step": 52250
    },
    {
      "activations/layer0_attention_weight_max": 17.751468658447266,
      "activations/layer0_attention_weight_min": -12.216445922851562,
      "activations/layer10_attention_weight_max": 40.54777908325195,
      "activations/layer10_attention_weight_min": -38.40156173706055,
      "activations/layer11_attention_weight_max": 37.444664001464844,
      "activations/layer11_attention_weight_min": -32.7305908203125,
      "activations/layer12_attention_weight_max": 32.077030181884766,
      "activations/layer12_attention_weight_min": -28.714553833007812,
      "activations/layer13_attention_weight_max": 51.457542419433594,
      "activations/layer13_attention_weight_min": -38.6383171081543,
      "activations/layer14_attention_weight_max": 81.41593170166016,
      "activations/layer14_attention_weight_min": -61.71147155761719,
      "activations/layer15_attention_weight_max": 45.605003356933594,
      "activations/layer15_attention_weight_min": -31.020183563232422,
      "activations/layer16_attention_weight_max": 53.819976806640625,
      "activations/layer16_attention_weight_min": -31.435392379760742,
      "activations/layer17_attention_weight_max": 45.88663101196289,
      "activations/layer17_attention_weight_min": -26.164016723632812,
      "activations/layer18_attention_weight_max": 34.11024475097656,
      "activations/layer18_attention_weight_min": -22.823963165283203,
      "activations/layer19_attention_weight_max": 53.12852096557617,
      "activations/layer19_attention_weight_min": -31.434133529663086,
      "activations/layer1_attention_weight_max": 17.643922805786133,
      "activations/layer1_attention_weight_min": -15.49773120880127,
      "activations/layer20_attention_weight_max": 33.78248977661133,
      "activations/layer20_attention_weight_min": -24.633705139160156,
      "activations/layer21_attention_weight_max": 38.488094329833984,
      "activations/layer21_attention_weight_min": -22.615392684936523,
      "activations/layer22_attention_weight_max": 49.32394790649414,
      "activations/layer22_attention_weight_min": -32.96824264526367,
      "activations/layer23_attention_weight_max": 28.89017105102539,
      "activations/layer23_attention_weight_min": -18.085166931152344,
      "activations/layer2_attention_weight_max": 30.670278549194336,
      "activations/layer2_attention_weight_min": -30.673725128173828,
      "activations/layer3_attention_weight_max": 79.04811096191406,
      "activations/layer3_attention_weight_min": -86.2487564086914,
      "activations/layer4_attention_weight_max": 133.67996215820312,
      "activations/layer4_attention_weight_min": -124.1779556274414,
      "activations/layer5_attention_weight_max": 108.0870361328125,
      "activations/layer5_attention_weight_min": -89.90919494628906,
      "activations/layer6_attention_weight_max": 58.571189880371094,
      "activations/layer6_attention_weight_min": -51.442176818847656,
      "activations/layer7_attention_weight_max": 73.10983276367188,
      "activations/layer7_attention_weight_min": -70.79228210449219,
      "activations/layer8_attention_weight_max": 43.54668045043945,
      "activations/layer8_attention_weight_min": -37.96883010864258,
      "activations/layer9_attention_weight_max": 48.602108001708984,
      "activations/layer9_attention_weight_min": -42.75763702392578,
      "epoch": 3.04,
      "learning_rate": 0.0001317276515151515,
      "loss": 2.8973,
      "step": 52300
    },
    {
      "activations/layer0_attention_weight_max": 16.567800521850586,
      "activations/layer0_attention_weight_min": -13.249086380004883,
      "activations/layer10_attention_weight_max": 39.355812072753906,
      "activations/layer10_attention_weight_min": -31.376285552978516,
      "activations/layer11_attention_weight_max": 38.764095306396484,
      "activations/layer11_attention_weight_min": -29.405624389648438,
      "activations/layer12_attention_weight_max": 37.559600830078125,
      "activations/layer12_attention_weight_min": -28.348360061645508,
      "activations/layer13_attention_weight_max": 49.59541320800781,
      "activations/layer13_attention_weight_min": -36.13862609863281,
      "activations/layer14_attention_weight_max": 77.25787353515625,
      "activations/layer14_attention_weight_min": -52.52406692504883,
      "activations/layer15_attention_weight_max": 50.282005310058594,
      "activations/layer15_attention_weight_min": -32.626712799072266,
      "activations/layer16_attention_weight_max": 51.36033630371094,
      "activations/layer16_attention_weight_min": -29.404428482055664,
      "activations/layer17_attention_weight_max": 42.69150924682617,
      "activations/layer17_attention_weight_min": -27.943622589111328,
      "activations/layer18_attention_weight_max": 29.494108200073242,
      "activations/layer18_attention_weight_min": -20.56767463684082,
      "activations/layer19_attention_weight_max": 54.9354133605957,
      "activations/layer19_attention_weight_min": -29.63416862487793,
      "activations/layer1_attention_weight_max": 17.94561767578125,
      "activations/layer1_attention_weight_min": -13.551937103271484,
      "activations/layer20_attention_weight_max": 40.56766128540039,
      "activations/layer20_attention_weight_min": -21.67441177368164,
      "activations/layer21_attention_weight_max": 36.74882507324219,
      "activations/layer21_attention_weight_min": -20.823999404907227,
      "activations/layer22_attention_weight_max": 54.30099868774414,
      "activations/layer22_attention_weight_min": -28.036184310913086,
      "activations/layer23_attention_weight_max": 30.821613311767578,
      "activations/layer23_attention_weight_min": -17.862375259399414,
      "activations/layer2_attention_weight_max": 31.991512298583984,
      "activations/layer2_attention_weight_min": -32.23133850097656,
      "activations/layer3_attention_weight_max": 83.90638732910156,
      "activations/layer3_attention_weight_min": -89.58478546142578,
      "activations/layer4_attention_weight_max": 142.2445831298828,
      "activations/layer4_attention_weight_min": -135.2188262939453,
      "activations/layer5_attention_weight_max": 115.05269622802734,
      "activations/layer5_attention_weight_min": -97.56535339355469,
      "activations/layer6_attention_weight_max": 62.77203369140625,
      "activations/layer6_attention_weight_min": -56.92267608642578,
      "activations/layer7_attention_weight_max": 75.40546417236328,
      "activations/layer7_attention_weight_min": -71.24815368652344,
      "activations/layer8_attention_weight_max": 43.61670684814453,
      "activations/layer8_attention_weight_min": -37.05662536621094,
      "activations/layer9_attention_weight_max": 49.753116607666016,
      "activations/layer9_attention_weight_min": -34.55633544921875,
      "epoch": 3.04,
      "learning_rate": 0.00013170871212121212,
      "loss": 2.8876,
      "step": 52350
    },
    {
      "activations/layer0_attention_weight_max": 17.393962860107422,
      "activations/layer0_attention_weight_min": -12.976574897766113,
      "activations/layer10_attention_weight_max": 60.39093017578125,
      "activations/layer10_attention_weight_min": -36.61720657348633,
      "activations/layer11_attention_weight_max": 54.35565948486328,
      "activations/layer11_attention_weight_min": -33.23232650756836,
      "activations/layer12_attention_weight_max": 51.134429931640625,
      "activations/layer12_attention_weight_min": -31.872072219848633,
      "activations/layer13_attention_weight_max": 68.61558532714844,
      "activations/layer13_attention_weight_min": -35.580631256103516,
      "activations/layer14_attention_weight_max": 109.82747650146484,
      "activations/layer14_attention_weight_min": -55.282657623291016,
      "activations/layer15_attention_weight_max": 55.40885543823242,
      "activations/layer15_attention_weight_min": -29.984420776367188,
      "activations/layer16_attention_weight_max": 47.72416305541992,
      "activations/layer16_attention_weight_min": -31.742155075073242,
      "activations/layer17_attention_weight_max": 45.53954315185547,
      "activations/layer17_attention_weight_min": -26.970373153686523,
      "activations/layer18_attention_weight_max": 29.082523345947266,
      "activations/layer18_attention_weight_min": -18.799976348876953,
      "activations/layer19_attention_weight_max": 43.778072357177734,
      "activations/layer19_attention_weight_min": -28.623132705688477,
      "activations/layer1_attention_weight_max": 19.3724365234375,
      "activations/layer1_attention_weight_min": -15.11023235321045,
      "activations/layer20_attention_weight_max": 33.32805633544922,
      "activations/layer20_attention_weight_min": -21.974546432495117,
      "activations/layer21_attention_weight_max": 32.09673309326172,
      "activations/layer21_attention_weight_min": -19.01914405822754,
      "activations/layer22_attention_weight_max": 49.00882339477539,
      "activations/layer22_attention_weight_min": -25.50992774963379,
      "activations/layer23_attention_weight_max": 27.470266342163086,
      "activations/layer23_attention_weight_min": -16.75647735595703,
      "activations/layer2_attention_weight_max": 32.44378662109375,
      "activations/layer2_attention_weight_min": -31.252790451049805,
      "activations/layer3_attention_weight_max": 92.50331115722656,
      "activations/layer3_attention_weight_min": -93.09033203125,
      "activations/layer4_attention_weight_max": 163.3287353515625,
      "activations/layer4_attention_weight_min": -143.1365966796875,
      "activations/layer5_attention_weight_max": 130.95594787597656,
      "activations/layer5_attention_weight_min": -108.19099426269531,
      "activations/layer6_attention_weight_max": 73.92366027832031,
      "activations/layer6_attention_weight_min": -57.724674224853516,
      "activations/layer7_attention_weight_max": 87.9488754272461,
      "activations/layer7_attention_weight_min": -74.50496673583984,
      "activations/layer8_attention_weight_max": 55.00880813598633,
      "activations/layer8_attention_weight_min": -42.92368698120117,
      "activations/layer9_attention_weight_max": 62.0906982421875,
      "activations/layer9_attention_weight_min": -46.85235595703125,
      "epoch": 3.04,
      "learning_rate": 0.00013168977272727273,
      "loss": 2.8804,
      "step": 52400
    },
    {
      "activations/layer0_attention_weight_max": 15.48437786102295,
      "activations/layer0_attention_weight_min": -13.642058372497559,
      "activations/layer10_attention_weight_max": 41.064334869384766,
      "activations/layer10_attention_weight_min": -32.53200149536133,
      "activations/layer11_attention_weight_max": 39.98394775390625,
      "activations/layer11_attention_weight_min": -31.806196212768555,
      "activations/layer12_attention_weight_max": 41.0978889465332,
      "activations/layer12_attention_weight_min": -26.153348922729492,
      "activations/layer13_attention_weight_max": 52.350364685058594,
      "activations/layer13_attention_weight_min": -36.81139373779297,
      "activations/layer14_attention_weight_max": 69.23390197753906,
      "activations/layer14_attention_weight_min": -49.955223083496094,
      "activations/layer15_attention_weight_max": 48.27621841430664,
      "activations/layer15_attention_weight_min": -31.637798309326172,
      "activations/layer16_attention_weight_max": 44.434532165527344,
      "activations/layer16_attention_weight_min": -33.1759147644043,
      "activations/layer17_attention_weight_max": 41.69869613647461,
      "activations/layer17_attention_weight_min": -27.29189109802246,
      "activations/layer18_attention_weight_max": 30.178123474121094,
      "activations/layer18_attention_weight_min": -18.34860610961914,
      "activations/layer19_attention_weight_max": 43.48696517944336,
      "activations/layer19_attention_weight_min": -29.656082153320312,
      "activations/layer1_attention_weight_max": 18.373552322387695,
      "activations/layer1_attention_weight_min": -14.136685371398926,
      "activations/layer20_attention_weight_max": 34.83168411254883,
      "activations/layer20_attention_weight_min": -23.311800003051758,
      "activations/layer21_attention_weight_max": 35.958858489990234,
      "activations/layer21_attention_weight_min": -18.282638549804688,
      "activations/layer22_attention_weight_max": 52.889644622802734,
      "activations/layer22_attention_weight_min": -25.979084014892578,
      "activations/layer23_attention_weight_max": 27.843666076660156,
      "activations/layer23_attention_weight_min": -18.095294952392578,
      "activations/layer2_attention_weight_max": 31.202939987182617,
      "activations/layer2_attention_weight_min": -30.143329620361328,
      "activations/layer3_attention_weight_max": 78.88716125488281,
      "activations/layer3_attention_weight_min": -83.97083282470703,
      "activations/layer4_attention_weight_max": 136.39291381835938,
      "activations/layer4_attention_weight_min": -125.3259048461914,
      "activations/layer5_attention_weight_max": 106.38864135742188,
      "activations/layer5_attention_weight_min": -91.87615966796875,
      "activations/layer6_attention_weight_max": 60.75647735595703,
      "activations/layer6_attention_weight_min": -52.05758285522461,
      "activations/layer7_attention_weight_max": 74.67804718017578,
      "activations/layer7_attention_weight_min": -68.97948455810547,
      "activations/layer8_attention_weight_max": 44.815460205078125,
      "activations/layer8_attention_weight_min": -37.01104736328125,
      "activations/layer9_attention_weight_max": 57.703338623046875,
      "activations/layer9_attention_weight_min": -38.69062042236328,
      "epoch": 3.05,
      "learning_rate": 0.00013167083333333332,
      "loss": 2.8943,
      "step": 52450
    },
    {
      "activations/layer0_attention_weight_max": 16.840864181518555,
      "activations/layer0_attention_weight_min": -11.142659187316895,
      "activations/layer10_attention_weight_max": 38.507102966308594,
      "activations/layer10_attention_weight_min": -31.955215454101562,
      "activations/layer11_attention_weight_max": 39.03798294067383,
      "activations/layer11_attention_weight_min": -30.145469665527344,
      "activations/layer12_attention_weight_max": 37.07698440551758,
      "activations/layer12_attention_weight_min": -32.341758728027344,
      "activations/layer13_attention_weight_max": 47.3998908996582,
      "activations/layer13_attention_weight_min": -38.4541015625,
      "activations/layer14_attention_weight_max": 69.61542510986328,
      "activations/layer14_attention_weight_min": -49.20170593261719,
      "activations/layer15_attention_weight_max": 50.092308044433594,
      "activations/layer15_attention_weight_min": -34.74363327026367,
      "activations/layer16_attention_weight_max": 39.447654724121094,
      "activations/layer16_attention_weight_min": -29.577926635742188,
      "activations/layer17_attention_weight_max": 42.588584899902344,
      "activations/layer17_attention_weight_min": -25.95166778564453,
      "activations/layer18_attention_weight_max": 33.54627227783203,
      "activations/layer18_attention_weight_min": -20.916210174560547,
      "activations/layer19_attention_weight_max": 43.70072937011719,
      "activations/layer19_attention_weight_min": -29.544729232788086,
      "activations/layer1_attention_weight_max": 18.210214614868164,
      "activations/layer1_attention_weight_min": -15.233954429626465,
      "activations/layer20_attention_weight_max": 36.096927642822266,
      "activations/layer20_attention_weight_min": -23.9962100982666,
      "activations/layer21_attention_weight_max": 36.08769607543945,
      "activations/layer21_attention_weight_min": -21.804489135742188,
      "activations/layer22_attention_weight_max": 55.629459381103516,
      "activations/layer22_attention_weight_min": -28.10779571533203,
      "activations/layer23_attention_weight_max": 28.635967254638672,
      "activations/layer23_attention_weight_min": -17.513885498046875,
      "activations/layer2_attention_weight_max": 29.785324096679688,
      "activations/layer2_attention_weight_min": -29.136356353759766,
      "activations/layer3_attention_weight_max": 80.20240020751953,
      "activations/layer3_attention_weight_min": -81.16272735595703,
      "activations/layer4_attention_weight_max": 132.57754516601562,
      "activations/layer4_attention_weight_min": -126.25029754638672,
      "activations/layer5_attention_weight_max": 98.9432601928711,
      "activations/layer5_attention_weight_min": -87.73939514160156,
      "activations/layer6_attention_weight_max": 57.07878875732422,
      "activations/layer6_attention_weight_min": -50.315696716308594,
      "activations/layer7_attention_weight_max": 73.29950714111328,
      "activations/layer7_attention_weight_min": -72.23931121826172,
      "activations/layer8_attention_weight_max": 40.42610549926758,
      "activations/layer8_attention_weight_min": -34.87596130371094,
      "activations/layer9_attention_weight_max": 57.38967514038086,
      "activations/layer9_attention_weight_min": -42.53499984741211,
      "epoch": 3.05,
      "learning_rate": 0.00013165189393939393,
      "loss": 2.9063,
      "step": 52500
    },
    {
      "activations/layer0_attention_weight_max": 16.703521728515625,
      "activations/layer0_attention_weight_min": -12.153915405273438,
      "activations/layer10_attention_weight_max": 40.33564376831055,
      "activations/layer10_attention_weight_min": -34.625946044921875,
      "activations/layer11_attention_weight_max": 34.42041015625,
      "activations/layer11_attention_weight_min": -30.762290954589844,
      "activations/layer12_attention_weight_max": 33.01789855957031,
      "activations/layer12_attention_weight_min": -26.481342315673828,
      "activations/layer13_attention_weight_max": 48.74405288696289,
      "activations/layer13_attention_weight_min": -38.03795623779297,
      "activations/layer14_attention_weight_max": 74.02945709228516,
      "activations/layer14_attention_weight_min": -59.561065673828125,
      "activations/layer15_attention_weight_max": 43.70492935180664,
      "activations/layer15_attention_weight_min": -31.858118057250977,
      "activations/layer16_attention_weight_max": 40.166015625,
      "activations/layer16_attention_weight_min": -30.913000106811523,
      "activations/layer17_attention_weight_max": 41.75642776489258,
      "activations/layer17_attention_weight_min": -29.57230567932129,
      "activations/layer18_attention_weight_max": 33.244415283203125,
      "activations/layer18_attention_weight_min": -21.078392028808594,
      "activations/layer19_attention_weight_max": 45.30857467651367,
      "activations/layer19_attention_weight_min": -29.812549591064453,
      "activations/layer1_attention_weight_max": 18.953731536865234,
      "activations/layer1_attention_weight_min": -13.545461654663086,
      "activations/layer20_attention_weight_max": 31.152326583862305,
      "activations/layer20_attention_weight_min": -25.306804656982422,
      "activations/layer21_attention_weight_max": 34.124488830566406,
      "activations/layer21_attention_weight_min": -21.285839080810547,
      "activations/layer22_attention_weight_max": 48.30610656738281,
      "activations/layer22_attention_weight_min": -31.029735565185547,
      "activations/layer23_attention_weight_max": 29.32207489013672,
      "activations/layer23_attention_weight_min": -17.01346206665039,
      "activations/layer2_attention_weight_max": 30.070106506347656,
      "activations/layer2_attention_weight_min": -29.87746810913086,
      "activations/layer3_attention_weight_max": 78.19757843017578,
      "activations/layer3_attention_weight_min": -80.71659851074219,
      "activations/layer4_attention_weight_max": 123.8408432006836,
      "activations/layer4_attention_weight_min": -122.4814224243164,
      "activations/layer5_attention_weight_max": 94.06832122802734,
      "activations/layer5_attention_weight_min": -88.09512329101562,
      "activations/layer6_attention_weight_max": 55.606178283691406,
      "activations/layer6_attention_weight_min": -51.959041595458984,
      "activations/layer7_attention_weight_max": 71.87106323242188,
      "activations/layer7_attention_weight_min": -71.52564239501953,
      "activations/layer8_attention_weight_max": 38.591888427734375,
      "activations/layer8_attention_weight_min": -37.48395538330078,
      "activations/layer9_attention_weight_max": 51.62899398803711,
      "activations/layer9_attention_weight_min": -39.175315856933594,
      "epoch": 3.05,
      "learning_rate": 0.00013163295454545455,
      "loss": 2.9075,
      "step": 52550
    },
    {
      "activations/layer0_attention_weight_max": 16.440998077392578,
      "activations/layer0_attention_weight_min": -12.734810829162598,
      "activations/layer10_attention_weight_max": 41.67771530151367,
      "activations/layer10_attention_weight_min": -32.07300567626953,
      "activations/layer11_attention_weight_max": 38.03195571899414,
      "activations/layer11_attention_weight_min": -28.999065399169922,
      "activations/layer12_attention_weight_max": 71.39456939697266,
      "activations/layer12_attention_weight_min": -32.4485969543457,
      "activations/layer13_attention_weight_max": 51.48879623413086,
      "activations/layer13_attention_weight_min": -35.6966438293457,
      "activations/layer14_attention_weight_max": 63.72015380859375,
      "activations/layer14_attention_weight_min": -52.00844955444336,
      "activations/layer15_attention_weight_max": 47.093936920166016,
      "activations/layer15_attention_weight_min": -32.43123245239258,
      "activations/layer16_attention_weight_max": 46.20861053466797,
      "activations/layer16_attention_weight_min": -34.14662170410156,
      "activations/layer17_attention_weight_max": 41.90613555908203,
      "activations/layer17_attention_weight_min": -26.7415714263916,
      "activations/layer18_attention_weight_max": 29.102176666259766,
      "activations/layer18_attention_weight_min": -18.09303855895996,
      "activations/layer19_attention_weight_max": 44.18978500366211,
      "activations/layer19_attention_weight_min": -30.676908493041992,
      "activations/layer1_attention_weight_max": 17.723438262939453,
      "activations/layer1_attention_weight_min": -13.253411293029785,
      "activations/layer20_attention_weight_max": 36.58743667602539,
      "activations/layer20_attention_weight_min": -24.235382080078125,
      "activations/layer21_attention_weight_max": 35.07306671142578,
      "activations/layer21_attention_weight_min": -22.247861862182617,
      "activations/layer22_attention_weight_max": 46.11105728149414,
      "activations/layer22_attention_weight_min": -27.40522575378418,
      "activations/layer23_attention_weight_max": 26.871686935424805,
      "activations/layer23_attention_weight_min": -16.149581909179688,
      "activations/layer2_attention_weight_max": 29.704893112182617,
      "activations/layer2_attention_weight_min": -29.70416259765625,
      "activations/layer3_attention_weight_max": 75.96995544433594,
      "activations/layer3_attention_weight_min": -78.03538513183594,
      "activations/layer4_attention_weight_max": 127.79607391357422,
      "activations/layer4_attention_weight_min": -122.4072494506836,
      "activations/layer5_attention_weight_max": 104.26158905029297,
      "activations/layer5_attention_weight_min": -94.069580078125,
      "activations/layer6_attention_weight_max": 58.43779373168945,
      "activations/layer6_attention_weight_min": -52.688865661621094,
      "activations/layer7_attention_weight_max": 73.92762756347656,
      "activations/layer7_attention_weight_min": -68.19540405273438,
      "activations/layer8_attention_weight_max": 39.21077346801758,
      "activations/layer8_attention_weight_min": -36.14284896850586,
      "activations/layer9_attention_weight_max": 49.590946197509766,
      "activations/layer9_attention_weight_min": -41.589290618896484,
      "epoch": 3.06,
      "learning_rate": 0.00013161401515151514,
      "loss": 2.9035,
      "step": 52600
    },
    {
      "activations/layer0_attention_weight_max": 16.668684005737305,
      "activations/layer0_attention_weight_min": -12.319968223571777,
      "activations/layer10_attention_weight_max": 44.94009780883789,
      "activations/layer10_attention_weight_min": -32.05220413208008,
      "activations/layer11_attention_weight_max": 38.57157516479492,
      "activations/layer11_attention_weight_min": -33.62353515625,
      "activations/layer12_attention_weight_max": 44.65558624267578,
      "activations/layer12_attention_weight_min": -40.87678909301758,
      "activations/layer13_attention_weight_max": 50.03206253051758,
      "activations/layer13_attention_weight_min": -32.27323913574219,
      "activations/layer14_attention_weight_max": 85.20361328125,
      "activations/layer14_attention_weight_min": -59.69729232788086,
      "activations/layer15_attention_weight_max": 46.61436080932617,
      "activations/layer15_attention_weight_min": -31.235342025756836,
      "activations/layer16_attention_weight_max": 47.14374923706055,
      "activations/layer16_attention_weight_min": -32.48627471923828,
      "activations/layer17_attention_weight_max": 46.963130950927734,
      "activations/layer17_attention_weight_min": -26.149333953857422,
      "activations/layer18_attention_weight_max": 35.17842483520508,
      "activations/layer18_attention_weight_min": -19.00586700439453,
      "activations/layer19_attention_weight_max": 50.326026916503906,
      "activations/layer19_attention_weight_min": -27.933273315429688,
      "activations/layer1_attention_weight_max": 17.609148025512695,
      "activations/layer1_attention_weight_min": -12.682352066040039,
      "activations/layer20_attention_weight_max": 36.21076202392578,
      "activations/layer20_attention_weight_min": -23.8037166595459,
      "activations/layer21_attention_weight_max": 35.4436149597168,
      "activations/layer21_attention_weight_min": -20.57816505432129,
      "activations/layer22_attention_weight_max": 49.930362701416016,
      "activations/layer22_attention_weight_min": -27.049964904785156,
      "activations/layer23_attention_weight_max": 29.29050064086914,
      "activations/layer23_attention_weight_min": -16.851825714111328,
      "activations/layer2_attention_weight_max": 27.70918083190918,
      "activations/layer2_attention_weight_min": -27.392606735229492,
      "activations/layer3_attention_weight_max": 73.41785430908203,
      "activations/layer3_attention_weight_min": -79.01585388183594,
      "activations/layer4_attention_weight_max": 128.32595825195312,
      "activations/layer4_attention_weight_min": -120.62686920166016,
      "activations/layer5_attention_weight_max": 103.7928695678711,
      "activations/layer5_attention_weight_min": -89.56835174560547,
      "activations/layer6_attention_weight_max": 57.68785095214844,
      "activations/layer6_attention_weight_min": -52.06196975708008,
      "activations/layer7_attention_weight_max": 76.93988037109375,
      "activations/layer7_attention_weight_min": -69.5143051147461,
      "activations/layer8_attention_weight_max": 38.064903259277344,
      "activations/layer8_attention_weight_min": -35.26149368286133,
      "activations/layer9_attention_weight_max": 53.77463912963867,
      "activations/layer9_attention_weight_min": -51.021568298339844,
      "epoch": 3.06,
      "learning_rate": 0.00013159507575757575,
      "loss": 2.9014,
      "step": 52650
    },
    {
      "activations/layer0_attention_weight_max": 17.726451873779297,
      "activations/layer0_attention_weight_min": -11.510481834411621,
      "activations/layer10_attention_weight_max": 38.55890655517578,
      "activations/layer10_attention_weight_min": -33.74552917480469,
      "activations/layer11_attention_weight_max": 36.82735061645508,
      "activations/layer11_attention_weight_min": -28.335262298583984,
      "activations/layer12_attention_weight_max": 37.79607009887695,
      "activations/layer12_attention_weight_min": -30.16054916381836,
      "activations/layer13_attention_weight_max": 49.20703887939453,
      "activations/layer13_attention_weight_min": -37.19297790527344,
      "activations/layer14_attention_weight_max": 56.91689682006836,
      "activations/layer14_attention_weight_min": -49.51182556152344,
      "activations/layer15_attention_weight_max": 45.130027770996094,
      "activations/layer15_attention_weight_min": -30.53163719177246,
      "activations/layer16_attention_weight_max": 39.47743225097656,
      "activations/layer16_attention_weight_min": -27.96718978881836,
      "activations/layer17_attention_weight_max": 45.216094970703125,
      "activations/layer17_attention_weight_min": -27.77639389038086,
      "activations/layer18_attention_weight_max": 27.047218322753906,
      "activations/layer18_attention_weight_min": -18.89919662475586,
      "activations/layer19_attention_weight_max": 40.998390197753906,
      "activations/layer19_attention_weight_min": -27.716049194335938,
      "activations/layer1_attention_weight_max": 18.4658203125,
      "activations/layer1_attention_weight_min": -13.629171371459961,
      "activations/layer20_attention_weight_max": 32.090675354003906,
      "activations/layer20_attention_weight_min": -22.938945770263672,
      "activations/layer21_attention_weight_max": 32.53264617919922,
      "activations/layer21_attention_weight_min": -20.839792251586914,
      "activations/layer22_attention_weight_max": 45.63262939453125,
      "activations/layer22_attention_weight_min": -26.33506965637207,
      "activations/layer23_attention_weight_max": 27.847610473632812,
      "activations/layer23_attention_weight_min": -16.325942993164062,
      "activations/layer2_attention_weight_max": 30.097946166992188,
      "activations/layer2_attention_weight_min": -29.08803939819336,
      "activations/layer3_attention_weight_max": 78.29442596435547,
      "activations/layer3_attention_weight_min": -82.14692687988281,
      "activations/layer4_attention_weight_max": 125.23980712890625,
      "activations/layer4_attention_weight_min": -125.65248107910156,
      "activations/layer5_attention_weight_max": 101.55677795410156,
      "activations/layer5_attention_weight_min": -94.97178649902344,
      "activations/layer6_attention_weight_max": 57.79119110107422,
      "activations/layer6_attention_weight_min": -53.238948822021484,
      "activations/layer7_attention_weight_max": 68.67021179199219,
      "activations/layer7_attention_weight_min": -66.1634292602539,
      "activations/layer8_attention_weight_max": 39.3385009765625,
      "activations/layer8_attention_weight_min": -34.2153434753418,
      "activations/layer9_attention_weight_max": 45.30677032470703,
      "activations/layer9_attention_weight_min": -37.03280258178711,
      "epoch": 3.06,
      "learning_rate": 0.00013157613636363637,
      "loss": 2.9058,
      "step": 52700
    },
    {
      "activations/layer0_attention_weight_max": 17.064573287963867,
      "activations/layer0_attention_weight_min": -11.560948371887207,
      "activations/layer10_attention_weight_max": 39.483402252197266,
      "activations/layer10_attention_weight_min": -34.362518310546875,
      "activations/layer11_attention_weight_max": 34.241729736328125,
      "activations/layer11_attention_weight_min": -28.518871307373047,
      "activations/layer12_attention_weight_max": 30.459531784057617,
      "activations/layer12_attention_weight_min": -28.056686401367188,
      "activations/layer13_attention_weight_max": 44.61735534667969,
      "activations/layer13_attention_weight_min": -35.30755615234375,
      "activations/layer14_attention_weight_max": 70.3817138671875,
      "activations/layer14_attention_weight_min": -57.58837127685547,
      "activations/layer15_attention_weight_max": 43.05805206298828,
      "activations/layer15_attention_weight_min": -31.45821762084961,
      "activations/layer16_attention_weight_max": 39.4246940612793,
      "activations/layer16_attention_weight_min": -30.905000686645508,
      "activations/layer17_attention_weight_max": 40.494117736816406,
      "activations/layer17_attention_weight_min": -27.28099250793457,
      "activations/layer18_attention_weight_max": 28.209274291992188,
      "activations/layer18_attention_weight_min": -19.66402244567871,
      "activations/layer19_attention_weight_max": 43.34339904785156,
      "activations/layer19_attention_weight_min": -29.77657127380371,
      "activations/layer1_attention_weight_max": 19.4481258392334,
      "activations/layer1_attention_weight_min": -16.014564514160156,
      "activations/layer20_attention_weight_max": 30.726627349853516,
      "activations/layer20_attention_weight_min": -22.839868545532227,
      "activations/layer21_attention_weight_max": 31.272329330444336,
      "activations/layer21_attention_weight_min": -20.524980545043945,
      "activations/layer22_attention_weight_max": 46.39644241333008,
      "activations/layer22_attention_weight_min": -32.663875579833984,
      "activations/layer23_attention_weight_max": 25.276241302490234,
      "activations/layer23_attention_weight_min": -20.078289031982422,
      "activations/layer2_attention_weight_max": 28.96711540222168,
      "activations/layer2_attention_weight_min": -28.7635498046875,
      "activations/layer3_attention_weight_max": 79.01224517822266,
      "activations/layer3_attention_weight_min": -79.46659851074219,
      "activations/layer4_attention_weight_max": 128.85836791992188,
      "activations/layer4_attention_weight_min": -122.24539947509766,
      "activations/layer5_attention_weight_max": 103.67292785644531,
      "activations/layer5_attention_weight_min": -96.70613098144531,
      "activations/layer6_attention_weight_max": 57.2719841003418,
      "activations/layer6_attention_weight_min": -58.0277214050293,
      "activations/layer7_attention_weight_max": 68.7091064453125,
      "activations/layer7_attention_weight_min": -68.87604522705078,
      "activations/layer8_attention_weight_max": 39.02301788330078,
      "activations/layer8_attention_weight_min": -37.37834167480469,
      "activations/layer9_attention_weight_max": 50.45652389526367,
      "activations/layer9_attention_weight_min": -36.173377990722656,
      "epoch": 3.07,
      "learning_rate": 0.00013155719696969696,
      "loss": 2.9128,
      "step": 52750
    },
    {
      "activations/layer0_attention_weight_max": 17.07356834411621,
      "activations/layer0_attention_weight_min": -11.886723518371582,
      "activations/layer10_attention_weight_max": 42.6910514831543,
      "activations/layer10_attention_weight_min": -33.84282302856445,
      "activations/layer11_attention_weight_max": 36.1875,
      "activations/layer11_attention_weight_min": -29.693798065185547,
      "activations/layer12_attention_weight_max": 35.580543518066406,
      "activations/layer12_attention_weight_min": -27.07062530517578,
      "activations/layer13_attention_weight_max": 54.9081916809082,
      "activations/layer13_attention_weight_min": -37.177242279052734,
      "activations/layer14_attention_weight_max": 83.601806640625,
      "activations/layer14_attention_weight_min": -62.422916412353516,
      "activations/layer15_attention_weight_max": 47.599334716796875,
      "activations/layer15_attention_weight_min": -30.6754150390625,
      "activations/layer16_attention_weight_max": 42.38991165161133,
      "activations/layer16_attention_weight_min": -29.912370681762695,
      "activations/layer17_attention_weight_max": 42.023590087890625,
      "activations/layer17_attention_weight_min": -28.124980926513672,
      "activations/layer18_attention_weight_max": 31.492868423461914,
      "activations/layer18_attention_weight_min": -19.01682472229004,
      "activations/layer19_attention_weight_max": 42.590877532958984,
      "activations/layer19_attention_weight_min": -26.45362663269043,
      "activations/layer1_attention_weight_max": 17.64402198791504,
      "activations/layer1_attention_weight_min": -14.403764724731445,
      "activations/layer20_attention_weight_max": 35.59281539916992,
      "activations/layer20_attention_weight_min": -22.535472869873047,
      "activations/layer21_attention_weight_max": 35.285552978515625,
      "activations/layer21_attention_weight_min": -20.592111587524414,
      "activations/layer22_attention_weight_max": 49.75570297241211,
      "activations/layer22_attention_weight_min": -25.979368209838867,
      "activations/layer23_attention_weight_max": 28.25392723083496,
      "activations/layer23_attention_weight_min": -16.04204559326172,
      "activations/layer2_attention_weight_max": 31.107391357421875,
      "activations/layer2_attention_weight_min": -32.53937911987305,
      "activations/layer3_attention_weight_max": 85.73518371582031,
      "activations/layer3_attention_weight_min": -89.03445434570312,
      "activations/layer4_attention_weight_max": 138.5657196044922,
      "activations/layer4_attention_weight_min": -132.7562255859375,
      "activations/layer5_attention_weight_max": 112.18733215332031,
      "activations/layer5_attention_weight_min": -99.34599304199219,
      "activations/layer6_attention_weight_max": 61.68715286254883,
      "activations/layer6_attention_weight_min": -58.589019775390625,
      "activations/layer7_attention_weight_max": 80.51419067382812,
      "activations/layer7_attention_weight_min": -70.22409057617188,
      "activations/layer8_attention_weight_max": 47.66908645629883,
      "activations/layer8_attention_weight_min": -41.621429443359375,
      "activations/layer9_attention_weight_max": 49.456512451171875,
      "activations/layer9_attention_weight_min": -38.337459564208984,
      "epoch": 3.07,
      "learning_rate": 0.00013153825757575757,
      "loss": 2.8938,
      "step": 52800
    },
    {
      "activations/layer0_attention_weight_max": 17.402286529541016,
      "activations/layer0_attention_weight_min": -11.140401840209961,
      "activations/layer10_attention_weight_max": 45.48994064331055,
      "activations/layer10_attention_weight_min": -32.282684326171875,
      "activations/layer11_attention_weight_max": 42.05045700073242,
      "activations/layer11_attention_weight_min": -28.084003448486328,
      "activations/layer12_attention_weight_max": 32.884403228759766,
      "activations/layer12_attention_weight_min": -32.32529830932617,
      "activations/layer13_attention_weight_max": 45.79994201660156,
      "activations/layer13_attention_weight_min": -35.76323318481445,
      "activations/layer14_attention_weight_max": 64.39192199707031,
      "activations/layer14_attention_weight_min": -52.427852630615234,
      "activations/layer15_attention_weight_max": 48.287139892578125,
      "activations/layer15_attention_weight_min": -32.82386016845703,
      "activations/layer16_attention_weight_max": 44.58661651611328,
      "activations/layer16_attention_weight_min": -32.76894760131836,
      "activations/layer17_attention_weight_max": 49.52925491333008,
      "activations/layer17_attention_weight_min": -29.82131004333496,
      "activations/layer18_attention_weight_max": 28.924489974975586,
      "activations/layer18_attention_weight_min": -20.44639778137207,
      "activations/layer19_attention_weight_max": 45.44968032836914,
      "activations/layer19_attention_weight_min": -33.39887237548828,
      "activations/layer1_attention_weight_max": 17.76694679260254,
      "activations/layer1_attention_weight_min": -12.992768287658691,
      "activations/layer20_attention_weight_max": 36.96107864379883,
      "activations/layer20_attention_weight_min": -23.647594451904297,
      "activations/layer21_attention_weight_max": 30.09017562866211,
      "activations/layer21_attention_weight_min": -19.956222534179688,
      "activations/layer22_attention_weight_max": 46.79949188232422,
      "activations/layer22_attention_weight_min": -27.90778160095215,
      "activations/layer23_attention_weight_max": 29.25579833984375,
      "activations/layer23_attention_weight_min": -17.561607360839844,
      "activations/layer2_attention_weight_max": 31.09479331970215,
      "activations/layer2_attention_weight_min": -30.415876388549805,
      "activations/layer3_attention_weight_max": 79.57229614257812,
      "activations/layer3_attention_weight_min": -84.74169921875,
      "activations/layer4_attention_weight_max": 125.94701385498047,
      "activations/layer4_attention_weight_min": -130.39036560058594,
      "activations/layer5_attention_weight_max": 99.08740234375,
      "activations/layer5_attention_weight_min": -96.49270629882812,
      "activations/layer6_attention_weight_max": 56.43373107910156,
      "activations/layer6_attention_weight_min": -56.442569732666016,
      "activations/layer7_attention_weight_max": 71.21117401123047,
      "activations/layer7_attention_weight_min": -69.59772491455078,
      "activations/layer8_attention_weight_max": 41.81707763671875,
      "activations/layer8_attention_weight_min": -36.24201202392578,
      "activations/layer9_attention_weight_max": 46.3626823425293,
      "activations/layer9_attention_weight_min": -38.20956039428711,
      "epoch": 3.07,
      "learning_rate": 0.00013151931818181819,
      "loss": 2.9086,
      "step": 52850
    },
    {
      "activations/layer0_attention_weight_max": 13.914915084838867,
      "activations/layer0_attention_weight_min": -11.445382118225098,
      "activations/layer10_attention_weight_max": 39.70663070678711,
      "activations/layer10_attention_weight_min": -32.472198486328125,
      "activations/layer11_attention_weight_max": 35.08938217163086,
      "activations/layer11_attention_weight_min": -28.13312530517578,
      "activations/layer12_attention_weight_max": 48.887481689453125,
      "activations/layer12_attention_weight_min": -34.602699279785156,
      "activations/layer13_attention_weight_max": 63.68583679199219,
      "activations/layer13_attention_weight_min": -56.64607238769531,
      "activations/layer14_attention_weight_max": 64.02926635742188,
      "activations/layer14_attention_weight_min": -53.922245025634766,
      "activations/layer15_attention_weight_max": 43.40752029418945,
      "activations/layer15_attention_weight_min": -33.94437789916992,
      "activations/layer16_attention_weight_max": 43.502559661865234,
      "activations/layer16_attention_weight_min": -29.61534881591797,
      "activations/layer17_attention_weight_max": 44.11259841918945,
      "activations/layer17_attention_weight_min": -25.662967681884766,
      "activations/layer18_attention_weight_max": 26.620288848876953,
      "activations/layer18_attention_weight_min": -22.65227508544922,
      "activations/layer19_attention_weight_max": 46.12803649902344,
      "activations/layer19_attention_weight_min": -34.0057487487793,
      "activations/layer1_attention_weight_max": 17.81242561340332,
      "activations/layer1_attention_weight_min": -13.155208587646484,
      "activations/layer20_attention_weight_max": 34.595359802246094,
      "activations/layer20_attention_weight_min": -24.977476119995117,
      "activations/layer21_attention_weight_max": 25.70684242248535,
      "activations/layer21_attention_weight_min": -18.06507110595703,
      "activations/layer22_attention_weight_max": 47.79402160644531,
      "activations/layer22_attention_weight_min": -25.318513870239258,
      "activations/layer23_attention_weight_max": 26.445531845092773,
      "activations/layer23_attention_weight_min": -17.4215087890625,
      "activations/layer2_attention_weight_max": 30.90499496459961,
      "activations/layer2_attention_weight_min": -29.807491302490234,
      "activations/layer3_attention_weight_max": 78.57783508300781,
      "activations/layer3_attention_weight_min": -80.1327133178711,
      "activations/layer4_attention_weight_max": 128.93194580078125,
      "activations/layer4_attention_weight_min": -128.0483856201172,
      "activations/layer5_attention_weight_max": 100.9034423828125,
      "activations/layer5_attention_weight_min": -94.92347717285156,
      "activations/layer6_attention_weight_max": 58.41090393066406,
      "activations/layer6_attention_weight_min": -54.928627014160156,
      "activations/layer7_attention_weight_max": 71.54969787597656,
      "activations/layer7_attention_weight_min": -72.02977752685547,
      "activations/layer8_attention_weight_max": 42.2974739074707,
      "activations/layer8_attention_weight_min": -39.06084442138672,
      "activations/layer9_attention_weight_max": 46.04417419433594,
      "activations/layer9_attention_weight_min": -37.29896926879883,
      "epoch": 3.07,
      "learning_rate": 0.00013150037878787877,
      "loss": 2.9029,
      "step": 52900
    },
    {
      "activations/layer0_attention_weight_max": 16.98781967163086,
      "activations/layer0_attention_weight_min": -11.311444282531738,
      "activations/layer10_attention_weight_max": 47.23471450805664,
      "activations/layer10_attention_weight_min": -38.01203918457031,
      "activations/layer11_attention_weight_max": 39.04624938964844,
      "activations/layer11_attention_weight_min": -29.178970336914062,
      "activations/layer12_attention_weight_max": 34.96337127685547,
      "activations/layer12_attention_weight_min": -28.93236541748047,
      "activations/layer13_attention_weight_max": 49.81714630126953,
      "activations/layer13_attention_weight_min": -36.2635498046875,
      "activations/layer14_attention_weight_max": 68.8328857421875,
      "activations/layer14_attention_weight_min": -51.124412536621094,
      "activations/layer15_attention_weight_max": 49.15414047241211,
      "activations/layer15_attention_weight_min": -33.716270446777344,
      "activations/layer16_attention_weight_max": 41.94133377075195,
      "activations/layer16_attention_weight_min": -30.951330184936523,
      "activations/layer17_attention_weight_max": 43.28677749633789,
      "activations/layer17_attention_weight_min": -28.28144073486328,
      "activations/layer18_attention_weight_max": 32.97221755981445,
      "activations/layer18_attention_weight_min": -20.229740142822266,
      "activations/layer19_attention_weight_max": 50.57943344116211,
      "activations/layer19_attention_weight_min": -29.684728622436523,
      "activations/layer1_attention_weight_max": 17.203123092651367,
      "activations/layer1_attention_weight_min": -13.739810943603516,
      "activations/layer20_attention_weight_max": 32.409603118896484,
      "activations/layer20_attention_weight_min": -22.793338775634766,
      "activations/layer21_attention_weight_max": 33.6031494140625,
      "activations/layer21_attention_weight_min": -20.26583480834961,
      "activations/layer22_attention_weight_max": 50.77728271484375,
      "activations/layer22_attention_weight_min": -27.17765998840332,
      "activations/layer23_attention_weight_max": 29.686386108398438,
      "activations/layer23_attention_weight_min": -15.92501449584961,
      "activations/layer2_attention_weight_max": 31.770736694335938,
      "activations/layer2_attention_weight_min": -30.80794906616211,
      "activations/layer3_attention_weight_max": 82.36109924316406,
      "activations/layer3_attention_weight_min": -83.56478881835938,
      "activations/layer4_attention_weight_max": 134.32144165039062,
      "activations/layer4_attention_weight_min": -126.42935943603516,
      "activations/layer5_attention_weight_max": 106.40243530273438,
      "activations/layer5_attention_weight_min": -92.07307434082031,
      "activations/layer6_attention_weight_max": 57.63111114501953,
      "activations/layer6_attention_weight_min": -54.945594787597656,
      "activations/layer7_attention_weight_max": 73.41558837890625,
      "activations/layer7_attention_weight_min": -69.98583221435547,
      "activations/layer8_attention_weight_max": 40.84022903442383,
      "activations/layer8_attention_weight_min": -38.38494110107422,
      "activations/layer9_attention_weight_max": 54.460906982421875,
      "activations/layer9_attention_weight_min": -47.93281173706055,
      "epoch": 3.08,
      "learning_rate": 0.0001314814393939394,
      "loss": 2.9062,
      "step": 52950
    },
    {
      "activations/layer0_attention_weight_max": 17.186412811279297,
      "activations/layer0_attention_weight_min": -11.155594825744629,
      "activations/layer10_attention_weight_max": 40.978973388671875,
      "activations/layer10_attention_weight_min": -34.168495178222656,
      "activations/layer11_attention_weight_max": 34.088653564453125,
      "activations/layer11_attention_weight_min": -28.480213165283203,
      "activations/layer12_attention_weight_max": 37.051025390625,
      "activations/layer12_attention_weight_min": -27.425430297851562,
      "activations/layer13_attention_weight_max": 52.617713928222656,
      "activations/layer13_attention_weight_min": -34.35382843017578,
      "activations/layer14_attention_weight_max": 74.8274154663086,
      "activations/layer14_attention_weight_min": -54.01205062866211,
      "activations/layer15_attention_weight_max": 51.1784553527832,
      "activations/layer15_attention_weight_min": -32.114837646484375,
      "activations/layer16_attention_weight_max": 48.94523620605469,
      "activations/layer16_attention_weight_min": -32.63800048828125,
      "activations/layer17_attention_weight_max": 43.75293731689453,
      "activations/layer17_attention_weight_min": -26.161500930786133,
      "activations/layer18_attention_weight_max": 30.99983024597168,
      "activations/layer18_attention_weight_min": -19.653778076171875,
      "activations/layer19_attention_weight_max": 52.719940185546875,
      "activations/layer19_attention_weight_min": -29.46121597290039,
      "activations/layer1_attention_weight_max": 17.817302703857422,
      "activations/layer1_attention_weight_min": -13.746712684631348,
      "activations/layer20_attention_weight_max": 35.07254409790039,
      "activations/layer20_attention_weight_min": -23.975688934326172,
      "activations/layer21_attention_weight_max": 33.404884338378906,
      "activations/layer21_attention_weight_min": -20.22903823852539,
      "activations/layer22_attention_weight_max": 50.4712028503418,
      "activations/layer22_attention_weight_min": -29.9196834564209,
      "activations/layer23_attention_weight_max": 30.239646911621094,
      "activations/layer23_attention_weight_min": -21.372289657592773,
      "activations/layer2_attention_weight_max": 31.371864318847656,
      "activations/layer2_attention_weight_min": -29.89946746826172,
      "activations/layer3_attention_weight_max": 79.65931701660156,
      "activations/layer3_attention_weight_min": -80.90979766845703,
      "activations/layer4_attention_weight_max": 132.1930389404297,
      "activations/layer4_attention_weight_min": -127.4405288696289,
      "activations/layer5_attention_weight_max": 97.17394256591797,
      "activations/layer5_attention_weight_min": -93.82301330566406,
      "activations/layer6_attention_weight_max": 54.03777313232422,
      "activations/layer6_attention_weight_min": -59.20808029174805,
      "activations/layer7_attention_weight_max": 71.4953842163086,
      "activations/layer7_attention_weight_min": -73.06355285644531,
      "activations/layer8_attention_weight_max": 40.2092170715332,
      "activations/layer8_attention_weight_min": -37.58818435668945,
      "activations/layer9_attention_weight_max": 44.97028732299805,
      "activations/layer9_attention_weight_min": -38.713043212890625,
      "epoch": 3.08,
      "learning_rate": 0.00013146249999999998,
      "loss": 2.9007,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_loss": 2.857421875,
      "eval_runtime": 8.513,
      "eval_samples_per_second": 504.406,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_openwebtext_loss": 2.857421875,
      "eval_openwebtext_ppl": 17.416566918622852,
      "eval_openwebtext_runtime": 8.513,
      "eval_openwebtext_samples_per_second": 504.406,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_wikitext_loss": 3.111328125,
      "eval_wikitext_ppl": 22.450842133517945,
      "eval_wikitext_runtime": 2.0147,
      "eval_wikitext_samples_per_second": 226.338,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_lambada_loss": 2.978515625,
      "eval_lambada_ppl": 19.658614220966218,
      "eval_lambada_runtime": 9.5921,
      "eval_lambada_samples_per_second": 507.603,
      "step": 53000
    },
    {
      "activations/layer0_attention_weight_max": 16.182212829589844,
      "activations/layer0_attention_weight_min": -12.299991607666016,
      "activations/layer10_attention_weight_max": 47.5044059753418,
      "activations/layer10_attention_weight_min": -33.7525520324707,
      "activations/layer11_attention_weight_max": 42.11762237548828,
      "activations/layer11_attention_weight_min": -31.104633331298828,
      "activations/layer12_attention_weight_max": 64.45214080810547,
      "activations/layer12_attention_weight_min": -31.435657501220703,
      "activations/layer13_attention_weight_max": 68.33834838867188,
      "activations/layer13_attention_weight_min": -41.99700164794922,
      "activations/layer14_attention_weight_max": 102.80608367919922,
      "activations/layer14_attention_weight_min": -57.44165802001953,
      "activations/layer15_attention_weight_max": 60.0139274597168,
      "activations/layer15_attention_weight_min": -32.69370651245117,
      "activations/layer16_attention_weight_max": 54.287071228027344,
      "activations/layer16_attention_weight_min": -33.59735870361328,
      "activations/layer17_attention_weight_max": 47.20366668701172,
      "activations/layer17_attention_weight_min": -25.3040828704834,
      "activations/layer18_attention_weight_max": 33.42914581298828,
      "activations/layer18_attention_weight_min": -19.390670776367188,
      "activations/layer19_attention_weight_max": 55.40863800048828,
      "activations/layer19_attention_weight_min": -30.333791732788086,
      "activations/layer1_attention_weight_max": 18.074295043945312,
      "activations/layer1_attention_weight_min": -13.47840690612793,
      "activations/layer20_attention_weight_max": 41.33279800415039,
      "activations/layer20_attention_weight_min": -22.455228805541992,
      "activations/layer21_attention_weight_max": 36.57188415527344,
      "activations/layer21_attention_weight_min": -20.64457893371582,
      "activations/layer22_attention_weight_max": 56.6872673034668,
      "activations/layer22_attention_weight_min": -25.840242385864258,
      "activations/layer23_attention_weight_max": 30.265539169311523,
      "activations/layer23_attention_weight_min": -14.855071067810059,
      "activations/layer2_attention_weight_max": 30.51346206665039,
      "activations/layer2_attention_weight_min": -29.479686737060547,
      "activations/layer3_attention_weight_max": 79.76058959960938,
      "activations/layer3_attention_weight_min": -82.16065216064453,
      "activations/layer4_attention_weight_max": 131.68295288085938,
      "activations/layer4_attention_weight_min": -124.28871154785156,
      "activations/layer5_attention_weight_max": 106.85807800292969,
      "activations/layer5_attention_weight_min": -96.47669982910156,
      "activations/layer6_attention_weight_max": 61.55278778076172,
      "activations/layer6_attention_weight_min": -57.26337814331055,
      "activations/layer7_attention_weight_max": 74.3957748413086,
      "activations/layer7_attention_weight_min": -72.11322021484375,
      "activations/layer8_attention_weight_max": 43.618839263916016,
      "activations/layer8_attention_weight_min": -38.22029495239258,
      "activations/layer9_attention_weight_max": 48.998207092285156,
      "activations/layer9_attention_weight_min": -39.529052734375,
      "epoch": 3.08,
      "learning_rate": 0.0001314435606060606,
      "loss": 2.9007,
      "step": 53050
    },
    {
      "activations/layer0_attention_weight_max": 17.230045318603516,
      "activations/layer0_attention_weight_min": -11.141621589660645,
      "activations/layer10_attention_weight_max": 40.104366302490234,
      "activations/layer10_attention_weight_min": -32.370365142822266,
      "activations/layer11_attention_weight_max": 36.95914840698242,
      "activations/layer11_attention_weight_min": -30.063209533691406,
      "activations/layer12_attention_weight_max": 37.29097366333008,
      "activations/layer12_attention_weight_min": -29.057897567749023,
      "activations/layer13_attention_weight_max": 49.66459655761719,
      "activations/layer13_attention_weight_min": -36.27924346923828,
      "activations/layer14_attention_weight_max": 74.17635345458984,
      "activations/layer14_attention_weight_min": -50.54852294921875,
      "activations/layer15_attention_weight_max": 48.98334503173828,
      "activations/layer15_attention_weight_min": -31.606403350830078,
      "activations/layer16_attention_weight_max": 45.26374053955078,
      "activations/layer16_attention_weight_min": -31.442501068115234,
      "activations/layer17_attention_weight_max": 41.78407669067383,
      "activations/layer17_attention_weight_min": -26.632226943969727,
      "activations/layer18_attention_weight_max": 32.37083435058594,
      "activations/layer18_attention_weight_min": -19.386072158813477,
      "activations/layer19_attention_weight_max": 48.681941986083984,
      "activations/layer19_attention_weight_min": -30.686538696289062,
      "activations/layer1_attention_weight_max": 18.59535789489746,
      "activations/layer1_attention_weight_min": -11.882633209228516,
      "activations/layer20_attention_weight_max": 34.92313003540039,
      "activations/layer20_attention_weight_min": -25.647022247314453,
      "activations/layer21_attention_weight_max": 31.244529724121094,
      "activations/layer21_attention_weight_min": -21.043346405029297,
      "activations/layer22_attention_weight_max": 49.37188720703125,
      "activations/layer22_attention_weight_min": -26.80058479309082,
      "activations/layer23_attention_weight_max": 29.197011947631836,
      "activations/layer23_attention_weight_min": -18.857372283935547,
      "activations/layer2_attention_weight_max": 28.606077194213867,
      "activations/layer2_attention_weight_min": -28.836212158203125,
      "activations/layer3_attention_weight_max": 77.72035217285156,
      "activations/layer3_attention_weight_min": -80.80728912353516,
      "activations/layer4_attention_weight_max": 127.29898834228516,
      "activations/layer4_attention_weight_min": -124.81610107421875,
      "activations/layer5_attention_weight_max": 101.35574340820312,
      "activations/layer5_attention_weight_min": -94.81242370605469,
      "activations/layer6_attention_weight_max": 59.02641296386719,
      "activations/layer6_attention_weight_min": -55.16697692871094,
      "activations/layer7_attention_weight_max": 75.59961700439453,
      "activations/layer7_attention_weight_min": -72.28250122070312,
      "activations/layer8_attention_weight_max": 40.12309646606445,
      "activations/layer8_attention_weight_min": -36.259464263916016,
      "activations/layer9_attention_weight_max": 47.46726608276367,
      "activations/layer9_attention_weight_min": -38.200984954833984,
      "epoch": 3.09,
      "learning_rate": 0.0001314246212121212,
      "loss": 2.8856,
      "step": 53100
    },
    {
      "activations/layer0_attention_weight_max": 17.072214126586914,
      "activations/layer0_attention_weight_min": -11.383785247802734,
      "activations/layer10_attention_weight_max": 40.24113464355469,
      "activations/layer10_attention_weight_min": -32.563331604003906,
      "activations/layer11_attention_weight_max": 36.042396545410156,
      "activations/layer11_attention_weight_min": -29.615604400634766,
      "activations/layer12_attention_weight_max": 67.44806671142578,
      "activations/layer12_attention_weight_min": -33.374847412109375,
      "activations/layer13_attention_weight_max": 44.670066833496094,
      "activations/layer13_attention_weight_min": -34.06140899658203,
      "activations/layer14_attention_weight_max": 67.20854949951172,
      "activations/layer14_attention_weight_min": -49.65143585205078,
      "activations/layer15_attention_weight_max": 46.136131286621094,
      "activations/layer15_attention_weight_min": -33.238826751708984,
      "activations/layer16_attention_weight_max": 43.256839752197266,
      "activations/layer16_attention_weight_min": -30.528154373168945,
      "activations/layer17_attention_weight_max": 41.866493225097656,
      "activations/layer17_attention_weight_min": -25.259803771972656,
      "activations/layer18_attention_weight_max": 28.677690505981445,
      "activations/layer18_attention_weight_min": -19.611005783081055,
      "activations/layer19_attention_weight_max": 50.459495544433594,
      "activations/layer19_attention_weight_min": -31.04810905456543,
      "activations/layer1_attention_weight_max": 18.80917739868164,
      "activations/layer1_attention_weight_min": -13.800297737121582,
      "activations/layer20_attention_weight_max": 37.45161056518555,
      "activations/layer20_attention_weight_min": -22.62236785888672,
      "activations/layer21_attention_weight_max": 36.91463851928711,
      "activations/layer21_attention_weight_min": -21.314485549926758,
      "activations/layer22_attention_weight_max": 51.72971725463867,
      "activations/layer22_attention_weight_min": -26.21529197692871,
      "activations/layer23_attention_weight_max": 29.096158981323242,
      "activations/layer23_attention_weight_min": -18.40857696533203,
      "activations/layer2_attention_weight_max": 29.458450317382812,
      "activations/layer2_attention_weight_min": -29.160024642944336,
      "activations/layer3_attention_weight_max": 81.7982177734375,
      "activations/layer3_attention_weight_min": -80.88426208496094,
      "activations/layer4_attention_weight_max": 134.07711791992188,
      "activations/layer4_attention_weight_min": -124.29936981201172,
      "activations/layer5_attention_weight_max": 102.44194030761719,
      "activations/layer5_attention_weight_min": -93.7550048828125,
      "activations/layer6_attention_weight_max": 58.237274169921875,
      "activations/layer6_attention_weight_min": -51.442020416259766,
      "activations/layer7_attention_weight_max": 70.47222137451172,
      "activations/layer7_attention_weight_min": -72.0145263671875,
      "activations/layer8_attention_weight_max": 41.270267486572266,
      "activations/layer8_attention_weight_min": -36.472694396972656,
      "activations/layer9_attention_weight_max": 47.95502471923828,
      "activations/layer9_attention_weight_min": -40.01073455810547,
      "epoch": 3.09,
      "learning_rate": 0.0001314056818181818,
      "loss": 2.9059,
      "step": 53150
    },
    {
      "activations/layer0_attention_weight_max": 17.890714645385742,
      "activations/layer0_attention_weight_min": -11.039701461791992,
      "activations/layer10_attention_weight_max": 41.457496643066406,
      "activations/layer10_attention_weight_min": -33.12583923339844,
      "activations/layer11_attention_weight_max": 38.836830139160156,
      "activations/layer11_attention_weight_min": -30.119657516479492,
      "activations/layer12_attention_weight_max": 62.32822036743164,
      "activations/layer12_attention_weight_min": -45.34859848022461,
      "activations/layer13_attention_weight_max": 56.68290328979492,
      "activations/layer13_attention_weight_min": -43.251033782958984,
      "activations/layer14_attention_weight_max": 70.13713073730469,
      "activations/layer14_attention_weight_min": -51.14091491699219,
      "activations/layer15_attention_weight_max": 47.51761245727539,
      "activations/layer15_attention_weight_min": -33.924560546875,
      "activations/layer16_attention_weight_max": 45.10581588745117,
      "activations/layer16_attention_weight_min": -31.66217803955078,
      "activations/layer17_attention_weight_max": 48.822635650634766,
      "activations/layer17_attention_weight_min": -28.641780853271484,
      "activations/layer18_attention_weight_max": 35.08028793334961,
      "activations/layer18_attention_weight_min": -19.584428787231445,
      "activations/layer19_attention_weight_max": 54.513099670410156,
      "activations/layer19_attention_weight_min": -30.974517822265625,
      "activations/layer1_attention_weight_max": 17.401805877685547,
      "activations/layer1_attention_weight_min": -12.518806457519531,
      "activations/layer20_attention_weight_max": 45.82855987548828,
      "activations/layer20_attention_weight_min": -23.314977645874023,
      "activations/layer21_attention_weight_max": 41.22233963012695,
      "activations/layer21_attention_weight_min": -23.812713623046875,
      "activations/layer22_attention_weight_max": 60.02268981933594,
      "activations/layer22_attention_weight_min": -30.332658767700195,
      "activations/layer23_attention_weight_max": 29.68445587158203,
      "activations/layer23_attention_weight_min": -17.29415512084961,
      "activations/layer2_attention_weight_max": 31.664649963378906,
      "activations/layer2_attention_weight_min": -30.09136962890625,
      "activations/layer3_attention_weight_max": 77.18473052978516,
      "activations/layer3_attention_weight_min": -80.7935791015625,
      "activations/layer4_attention_weight_max": 133.51657104492188,
      "activations/layer4_attention_weight_min": -125.85106658935547,
      "activations/layer5_attention_weight_max": 109.10433959960938,
      "activations/layer5_attention_weight_min": -94.94271087646484,
      "activations/layer6_attention_weight_max": 63.86650466918945,
      "activations/layer6_attention_weight_min": -55.9522590637207,
      "activations/layer7_attention_weight_max": 72.84862518310547,
      "activations/layer7_attention_weight_min": -73.03656005859375,
      "activations/layer8_attention_weight_max": 43.1444206237793,
      "activations/layer8_attention_weight_min": -39.46341323852539,
      "activations/layer9_attention_weight_max": 59.16386795043945,
      "activations/layer9_attention_weight_min": -47.717437744140625,
      "epoch": 3.09,
      "learning_rate": 0.0001313867424242424,
      "loss": 2.8973,
      "step": 53200
    },
    {
      "activations/layer0_attention_weight_max": 17.59980010986328,
      "activations/layer0_attention_weight_min": -11.477116584777832,
      "activations/layer10_attention_weight_max": 51.260887145996094,
      "activations/layer10_attention_weight_min": -39.987022399902344,
      "activations/layer11_attention_weight_max": 41.401241302490234,
      "activations/layer11_attention_weight_min": -28.085927963256836,
      "activations/layer12_attention_weight_max": 32.21796798706055,
      "activations/layer12_attention_weight_min": -26.96581268310547,
      "activations/layer13_attention_weight_max": 51.647789001464844,
      "activations/layer13_attention_weight_min": -34.828773498535156,
      "activations/layer14_attention_weight_max": 78.3554458618164,
      "activations/layer14_attention_weight_min": -54.67903518676758,
      "activations/layer15_attention_weight_max": 57.07569122314453,
      "activations/layer15_attention_weight_min": -34.31389617919922,
      "activations/layer16_attention_weight_max": 44.0046272277832,
      "activations/layer16_attention_weight_min": -31.363752365112305,
      "activations/layer17_attention_weight_max": 46.76478958129883,
      "activations/layer17_attention_weight_min": -28.155437469482422,
      "activations/layer18_attention_weight_max": 31.233713150024414,
      "activations/layer18_attention_weight_min": -19.291662216186523,
      "activations/layer19_attention_weight_max": 43.03806686401367,
      "activations/layer19_attention_weight_min": -30.09018325805664,
      "activations/layer1_attention_weight_max": 17.156227111816406,
      "activations/layer1_attention_weight_min": -13.91575813293457,
      "activations/layer20_attention_weight_max": 35.956485748291016,
      "activations/layer20_attention_weight_min": -23.94749641418457,
      "activations/layer21_attention_weight_max": 35.967384338378906,
      "activations/layer21_attention_weight_min": -19.999267578125,
      "activations/layer22_attention_weight_max": 50.888179779052734,
      "activations/layer22_attention_weight_min": -25.48811912536621,
      "activations/layer23_attention_weight_max": 28.154579162597656,
      "activations/layer23_attention_weight_min": -16.17395782470703,
      "activations/layer2_attention_weight_max": 29.715734481811523,
      "activations/layer2_attention_weight_min": -30.28946304321289,
      "activations/layer3_attention_weight_max": 76.68714904785156,
      "activations/layer3_attention_weight_min": -79.22655487060547,
      "activations/layer4_attention_weight_max": 132.0753631591797,
      "activations/layer4_attention_weight_min": -128.51451110839844,
      "activations/layer5_attention_weight_max": 109.77364349365234,
      "activations/layer5_attention_weight_min": -96.94041442871094,
      "activations/layer6_attention_weight_max": 58.580894470214844,
      "activations/layer6_attention_weight_min": -52.72597122192383,
      "activations/layer7_attention_weight_max": 72.73184967041016,
      "activations/layer7_attention_weight_min": -72.01410675048828,
      "activations/layer8_attention_weight_max": 41.63716506958008,
      "activations/layer8_attention_weight_min": -36.8243522644043,
      "activations/layer9_attention_weight_max": 48.178672790527344,
      "activations/layer9_attention_weight_min": -35.02540588378906,
      "epoch": 3.09,
      "learning_rate": 0.00013136780303030302,
      "loss": 2.8966,
      "step": 53250
    },
    {
      "activations/layer0_attention_weight_max": 17.41907501220703,
      "activations/layer0_attention_weight_min": -11.652255058288574,
      "activations/layer10_attention_weight_max": 43.8919792175293,
      "activations/layer10_attention_weight_min": -34.61079788208008,
      "activations/layer11_attention_weight_max": 41.03233337402344,
      "activations/layer11_attention_weight_min": -36.50979232788086,
      "activations/layer12_attention_weight_max": 76.83733367919922,
      "activations/layer12_attention_weight_min": -33.33884048461914,
      "activations/layer13_attention_weight_max": 62.8268928527832,
      "activations/layer13_attention_weight_min": -37.20009994506836,
      "activations/layer14_attention_weight_max": 73.64202880859375,
      "activations/layer14_attention_weight_min": -55.49888229370117,
      "activations/layer15_attention_weight_max": 46.73192596435547,
      "activations/layer15_attention_weight_min": -30.70064353942871,
      "activations/layer16_attention_weight_max": 46.387535095214844,
      "activations/layer16_attention_weight_min": -30.8134765625,
      "activations/layer17_attention_weight_max": 42.85587692260742,
      "activations/layer17_attention_weight_min": -26.005857467651367,
      "activations/layer18_attention_weight_max": 39.09710693359375,
      "activations/layer18_attention_weight_min": -20.99403190612793,
      "activations/layer19_attention_weight_max": 46.011436462402344,
      "activations/layer19_attention_weight_min": -31.772037506103516,
      "activations/layer1_attention_weight_max": 18.46700668334961,
      "activations/layer1_attention_weight_min": -13.724888801574707,
      "activations/layer20_attention_weight_max": 36.235923767089844,
      "activations/layer20_attention_weight_min": -23.910503387451172,
      "activations/layer21_attention_weight_max": 34.72514343261719,
      "activations/layer21_attention_weight_min": -20.719762802124023,
      "activations/layer22_attention_weight_max": 49.563472747802734,
      "activations/layer22_attention_weight_min": -27.093830108642578,
      "activations/layer23_attention_weight_max": 28.36653709411621,
      "activations/layer23_attention_weight_min": -17.616840362548828,
      "activations/layer2_attention_weight_max": 31.172107696533203,
      "activations/layer2_attention_weight_min": -32.69576644897461,
      "activations/layer3_attention_weight_max": 81.00682067871094,
      "activations/layer3_attention_weight_min": -85.34574890136719,
      "activations/layer4_attention_weight_max": 130.4023895263672,
      "activations/layer4_attention_weight_min": -133.28468322753906,
      "activations/layer5_attention_weight_max": 100.34779357910156,
      "activations/layer5_attention_weight_min": -92.65098571777344,
      "activations/layer6_attention_weight_max": 57.37903594970703,
      "activations/layer6_attention_weight_min": -54.44204330444336,
      "activations/layer7_attention_weight_max": 76.72132873535156,
      "activations/layer7_attention_weight_min": -70.52425384521484,
      "activations/layer8_attention_weight_max": 40.33158874511719,
      "activations/layer8_attention_weight_min": -35.42898941040039,
      "activations/layer9_attention_weight_max": 57.94312286376953,
      "activations/layer9_attention_weight_min": -42.78654098510742,
      "epoch": 3.1,
      "learning_rate": 0.0001313488636363636,
      "loss": 2.8881,
      "step": 53300
    },
    {
      "activations/layer0_attention_weight_max": 17.433347702026367,
      "activations/layer0_attention_weight_min": -11.081470489501953,
      "activations/layer10_attention_weight_max": 38.61640167236328,
      "activations/layer10_attention_weight_min": -32.695884704589844,
      "activations/layer11_attention_weight_max": 31.94781494140625,
      "activations/layer11_attention_weight_min": -30.113746643066406,
      "activations/layer12_attention_weight_max": 29.671831130981445,
      "activations/layer12_attention_weight_min": -24.094343185424805,
      "activations/layer13_attention_weight_max": 44.85557556152344,
      "activations/layer13_attention_weight_min": -33.87220001220703,
      "activations/layer14_attention_weight_max": 68.09002685546875,
      "activations/layer14_attention_weight_min": -52.53278732299805,
      "activations/layer15_attention_weight_max": 51.41592025756836,
      "activations/layer15_attention_weight_min": -34.97294235229492,
      "activations/layer16_attention_weight_max": 40.89323425292969,
      "activations/layer16_attention_weight_min": -31.206880569458008,
      "activations/layer17_attention_weight_max": 40.50544357299805,
      "activations/layer17_attention_weight_min": -28.603145599365234,
      "activations/layer18_attention_weight_max": 28.4567928314209,
      "activations/layer18_attention_weight_min": -18.679353713989258,
      "activations/layer19_attention_weight_max": 40.16264724731445,
      "activations/layer19_attention_weight_min": -30.253110885620117,
      "activations/layer1_attention_weight_max": 17.36915397644043,
      "activations/layer1_attention_weight_min": -16.1258487701416,
      "activations/layer20_attention_weight_max": 37.43019485473633,
      "activations/layer20_attention_weight_min": -24.90873908996582,
      "activations/layer21_attention_weight_max": 27.54532814025879,
      "activations/layer21_attention_weight_min": -20.931238174438477,
      "activations/layer22_attention_weight_max": 46.85397720336914,
      "activations/layer22_attention_weight_min": -28.422666549682617,
      "activations/layer23_attention_weight_max": 27.29738426208496,
      "activations/layer23_attention_weight_min": -17.63465118408203,
      "activations/layer2_attention_weight_max": 30.057205200195312,
      "activations/layer2_attention_weight_min": -31.428688049316406,
      "activations/layer3_attention_weight_max": 83.1606216430664,
      "activations/layer3_attention_weight_min": -85.28909301757812,
      "activations/layer4_attention_weight_max": 128.35960388183594,
      "activations/layer4_attention_weight_min": -133.1998748779297,
      "activations/layer5_attention_weight_max": 103.9934310913086,
      "activations/layer5_attention_weight_min": -98.6578598022461,
      "activations/layer6_attention_weight_max": 57.7941780090332,
      "activations/layer6_attention_weight_min": -56.74934005737305,
      "activations/layer7_attention_weight_max": 73.64083099365234,
      "activations/layer7_attention_weight_min": -76.08810424804688,
      "activations/layer8_attention_weight_max": 39.376930236816406,
      "activations/layer8_attention_weight_min": -40.94749450683594,
      "activations/layer9_attention_weight_max": 41.49330520629883,
      "activations/layer9_attention_weight_min": -35.64667892456055,
      "epoch": 3.1,
      "learning_rate": 0.00013132992424242423,
      "loss": 2.8853,
      "step": 53350
    },
    {
      "activations/layer0_attention_weight_max": 16.93855094909668,
      "activations/layer0_attention_weight_min": -11.371938705444336,
      "activations/layer10_attention_weight_max": 44.294891357421875,
      "activations/layer10_attention_weight_min": -33.551002502441406,
      "activations/layer11_attention_weight_max": 35.51253128051758,
      "activations/layer11_attention_weight_min": -27.358022689819336,
      "activations/layer12_attention_weight_max": 33.63975143432617,
      "activations/layer12_attention_weight_min": -25.725399017333984,
      "activations/layer13_attention_weight_max": 58.94035339355469,
      "activations/layer13_attention_weight_min": -41.29652786254883,
      "activations/layer14_attention_weight_max": 73.82658386230469,
      "activations/layer14_attention_weight_min": -51.353843688964844,
      "activations/layer15_attention_weight_max": 52.37234115600586,
      "activations/layer15_attention_weight_min": -30.714529037475586,
      "activations/layer16_attention_weight_max": 50.18721389770508,
      "activations/layer16_attention_weight_min": -30.24211311340332,
      "activations/layer17_attention_weight_max": 50.663761138916016,
      "activations/layer17_attention_weight_min": -26.646169662475586,
      "activations/layer18_attention_weight_max": 31.09963607788086,
      "activations/layer18_attention_weight_min": -19.227550506591797,
      "activations/layer19_attention_weight_max": 49.13435745239258,
      "activations/layer19_attention_weight_min": -30.202096939086914,
      "activations/layer1_attention_weight_max": 17.96141815185547,
      "activations/layer1_attention_weight_min": -13.772972106933594,
      "activations/layer20_attention_weight_max": 40.679744720458984,
      "activations/layer20_attention_weight_min": -23.383281707763672,
      "activations/layer21_attention_weight_max": 29.803817749023438,
      "activations/layer21_attention_weight_min": -16.887903213500977,
      "activations/layer22_attention_weight_max": 49.80592346191406,
      "activations/layer22_attention_weight_min": -25.09347915649414,
      "activations/layer23_attention_weight_max": 27.224857330322266,
      "activations/layer23_attention_weight_min": -16.055282592773438,
      "activations/layer2_attention_weight_max": 29.638229370117188,
      "activations/layer2_attention_weight_min": -29.28670883178711,
      "activations/layer3_attention_weight_max": 79.32099914550781,
      "activations/layer3_attention_weight_min": -83.46329498291016,
      "activations/layer4_attention_weight_max": 138.38890075683594,
      "activations/layer4_attention_weight_min": -130.4653778076172,
      "activations/layer5_attention_weight_max": 104.47676086425781,
      "activations/layer5_attention_weight_min": -95.88753509521484,
      "activations/layer6_attention_weight_max": 60.887882232666016,
      "activations/layer6_attention_weight_min": -54.91391372680664,
      "activations/layer7_attention_weight_max": 73.84089660644531,
      "activations/layer7_attention_weight_min": -69.2140884399414,
      "activations/layer8_attention_weight_max": 42.66157531738281,
      "activations/layer8_attention_weight_min": -37.34066390991211,
      "activations/layer9_attention_weight_max": 43.558128356933594,
      "activations/layer9_attention_weight_min": -39.31843566894531,
      "epoch": 3.1,
      "learning_rate": 0.00013131098484848482,
      "loss": 2.9057,
      "step": 53400
    },
    {
      "activations/layer0_attention_weight_max": 15.317538261413574,
      "activations/layer0_attention_weight_min": -12.284467697143555,
      "activations/layer10_attention_weight_max": 45.57061767578125,
      "activations/layer10_attention_weight_min": -34.817771911621094,
      "activations/layer11_attention_weight_max": 37.49367141723633,
      "activations/layer11_attention_weight_min": -31.41725730895996,
      "activations/layer12_attention_weight_max": 40.08769607543945,
      "activations/layer12_attention_weight_min": -26.619234085083008,
      "activations/layer13_attention_weight_max": 55.6756706237793,
      "activations/layer13_attention_weight_min": -35.36433410644531,
      "activations/layer14_attention_weight_max": 86.5391845703125,
      "activations/layer14_attention_weight_min": -52.21968078613281,
      "activations/layer15_attention_weight_max": 50.547645568847656,
      "activations/layer15_attention_weight_min": -33.403564453125,
      "activations/layer16_attention_weight_max": 52.81658935546875,
      "activations/layer16_attention_weight_min": -31.672895431518555,
      "activations/layer17_attention_weight_max": 50.247337341308594,
      "activations/layer17_attention_weight_min": -27.954360961914062,
      "activations/layer18_attention_weight_max": 33.688663482666016,
      "activations/layer18_attention_weight_min": -17.30770492553711,
      "activations/layer19_attention_weight_max": 46.56657028198242,
      "activations/layer19_attention_weight_min": -30.487701416015625,
      "activations/layer1_attention_weight_max": 17.666868209838867,
      "activations/layer1_attention_weight_min": -14.200593948364258,
      "activations/layer20_attention_weight_max": 36.96223068237305,
      "activations/layer20_attention_weight_min": -24.898235321044922,
      "activations/layer21_attention_weight_max": 35.04404830932617,
      "activations/layer21_attention_weight_min": -19.929397583007812,
      "activations/layer22_attention_weight_max": 53.70772933959961,
      "activations/layer22_attention_weight_min": -25.020200729370117,
      "activations/layer23_attention_weight_max": 30.342540740966797,
      "activations/layer23_attention_weight_min": -15.332626342773438,
      "activations/layer2_attention_weight_max": 31.83794403076172,
      "activations/layer2_attention_weight_min": -32.088462829589844,
      "activations/layer3_attention_weight_max": 87.9395523071289,
      "activations/layer3_attention_weight_min": -83.63031768798828,
      "activations/layer4_attention_weight_max": 146.2275390625,
      "activations/layer4_attention_weight_min": -136.33152770996094,
      "activations/layer5_attention_weight_max": 111.10872650146484,
      "activations/layer5_attention_weight_min": -98.12972259521484,
      "activations/layer6_attention_weight_max": 64.55252075195312,
      "activations/layer6_attention_weight_min": -59.458595275878906,
      "activations/layer7_attention_weight_max": 78.95647430419922,
      "activations/layer7_attention_weight_min": -74.44667053222656,
      "activations/layer8_attention_weight_max": 42.531558990478516,
      "activations/layer8_attention_weight_min": -43.38205337524414,
      "activations/layer9_attention_weight_max": 62.601985931396484,
      "activations/layer9_attention_weight_min": -43.13904571533203,
      "epoch": 3.11,
      "learning_rate": 0.00013129204545454543,
      "loss": 2.8947,
      "step": 53450
    },
    {
      "activations/layer0_attention_weight_max": 16.049442291259766,
      "activations/layer0_attention_weight_min": -12.098855018615723,
      "activations/layer10_attention_weight_max": 53.68450927734375,
      "activations/layer10_attention_weight_min": -34.468624114990234,
      "activations/layer11_attention_weight_max": 54.896202087402344,
      "activations/layer11_attention_weight_min": -38.747955322265625,
      "activations/layer12_attention_weight_max": 52.214576721191406,
      "activations/layer12_attention_weight_min": -30.638835906982422,
      "activations/layer13_attention_weight_max": 60.32798385620117,
      "activations/layer13_attention_weight_min": -35.52513885498047,
      "activations/layer14_attention_weight_max": 127.47535705566406,
      "activations/layer14_attention_weight_min": -77.84687042236328,
      "activations/layer15_attention_weight_max": 48.180809020996094,
      "activations/layer15_attention_weight_min": -30.56056785583496,
      "activations/layer16_attention_weight_max": 49.826759338378906,
      "activations/layer16_attention_weight_min": -33.279815673828125,
      "activations/layer17_attention_weight_max": 40.59128189086914,
      "activations/layer17_attention_weight_min": -27.1381778717041,
      "activations/layer18_attention_weight_max": 31.0756893157959,
      "activations/layer18_attention_weight_min": -18.320598602294922,
      "activations/layer19_attention_weight_max": 54.0279426574707,
      "activations/layer19_attention_weight_min": -30.4068660736084,
      "activations/layer1_attention_weight_max": 18.068050384521484,
      "activations/layer1_attention_weight_min": -13.630086898803711,
      "activations/layer20_attention_weight_max": 32.79279708862305,
      "activations/layer20_attention_weight_min": -22.571006774902344,
      "activations/layer21_attention_weight_max": 33.459503173828125,
      "activations/layer21_attention_weight_min": -19.6196346282959,
      "activations/layer22_attention_weight_max": 47.764739990234375,
      "activations/layer22_attention_weight_min": -27.32032012939453,
      "activations/layer23_attention_weight_max": 30.59132194519043,
      "activations/layer23_attention_weight_min": -16.869630813598633,
      "activations/layer2_attention_weight_max": 30.262561798095703,
      "activations/layer2_attention_weight_min": -29.04205322265625,
      "activations/layer3_attention_weight_max": 80.27852630615234,
      "activations/layer3_attention_weight_min": -82.3243179321289,
      "activations/layer4_attention_weight_max": 131.9711151123047,
      "activations/layer4_attention_weight_min": -130.5370635986328,
      "activations/layer5_attention_weight_max": 104.72344970703125,
      "activations/layer5_attention_weight_min": -96.18356323242188,
      "activations/layer6_attention_weight_max": 64.34197235107422,
      "activations/layer6_attention_weight_min": -57.566184997558594,
      "activations/layer7_attention_weight_max": 87.02091217041016,
      "activations/layer7_attention_weight_min": -76.4884262084961,
      "activations/layer8_attention_weight_max": 52.36667251586914,
      "activations/layer8_attention_weight_min": -38.24287414550781,
      "activations/layer9_attention_weight_max": 65.53280639648438,
      "activations/layer9_attention_weight_min": -49.19443130493164,
      "epoch": 3.11,
      "learning_rate": 0.00013127310606060605,
      "loss": 2.9047,
      "step": 53500
    },
    {
      "activations/layer0_attention_weight_max": 16.52740478515625,
      "activations/layer0_attention_weight_min": -11.430474281311035,
      "activations/layer10_attention_weight_max": 41.864952087402344,
      "activations/layer10_attention_weight_min": -33.441158294677734,
      "activations/layer11_attention_weight_max": 35.31425476074219,
      "activations/layer11_attention_weight_min": -28.805686950683594,
      "activations/layer12_attention_weight_max": 31.449108123779297,
      "activations/layer12_attention_weight_min": -27.114110946655273,
      "activations/layer13_attention_weight_max": 57.22154998779297,
      "activations/layer13_attention_weight_min": -35.70953369140625,
      "activations/layer14_attention_weight_max": 79.82260131835938,
      "activations/layer14_attention_weight_min": -54.6660270690918,
      "activations/layer15_attention_weight_max": 52.077816009521484,
      "activations/layer15_attention_weight_min": -32.764102935791016,
      "activations/layer16_attention_weight_max": 41.24386978149414,
      "activations/layer16_attention_weight_min": -31.479276657104492,
      "activations/layer17_attention_weight_max": 46.117279052734375,
      "activations/layer17_attention_weight_min": -28.256162643432617,
      "activations/layer18_attention_weight_max": 29.42254066467285,
      "activations/layer18_attention_weight_min": -19.72390365600586,
      "activations/layer19_attention_weight_max": 42.50046920776367,
      "activations/layer19_attention_weight_min": -34.415653228759766,
      "activations/layer1_attention_weight_max": 16.80549430847168,
      "activations/layer1_attention_weight_min": -13.347565650939941,
      "activations/layer20_attention_weight_max": 31.0152530670166,
      "activations/layer20_attention_weight_min": -23.165891647338867,
      "activations/layer21_attention_weight_max": 27.473106384277344,
      "activations/layer21_attention_weight_min": -18.636863708496094,
      "activations/layer22_attention_weight_max": 49.54469299316406,
      "activations/layer22_attention_weight_min": -25.97919273376465,
      "activations/layer23_attention_weight_max": 29.180496215820312,
      "activations/layer23_attention_weight_min": -17.849056243896484,
      "activations/layer2_attention_weight_max": 29.514150619506836,
      "activations/layer2_attention_weight_min": -29.11562728881836,
      "activations/layer3_attention_weight_max": 80.26911926269531,
      "activations/layer3_attention_weight_min": -82.04299926757812,
      "activations/layer4_attention_weight_max": 139.6534881591797,
      "activations/layer4_attention_weight_min": -128.68020629882812,
      "activations/layer5_attention_weight_max": 109.64363861083984,
      "activations/layer5_attention_weight_min": -95.2503662109375,
      "activations/layer6_attention_weight_max": 61.01124572753906,
      "activations/layer6_attention_weight_min": -54.03128433227539,
      "activations/layer7_attention_weight_max": 75.56360626220703,
      "activations/layer7_attention_weight_min": -74.67233276367188,
      "activations/layer8_attention_weight_max": 44.24884033203125,
      "activations/layer8_attention_weight_min": -38.30878829956055,
      "activations/layer9_attention_weight_max": 46.397789001464844,
      "activations/layer9_attention_weight_min": -34.57168960571289,
      "epoch": 3.11,
      "learning_rate": 0.00013125416666666666,
      "loss": 2.9047,
      "step": 53550
    },
    {
      "activations/layer0_attention_weight_max": 16.556428909301758,
      "activations/layer0_attention_weight_min": -12.414206504821777,
      "activations/layer10_attention_weight_max": 37.67954635620117,
      "activations/layer10_attention_weight_min": -33.11018371582031,
      "activations/layer11_attention_weight_max": 35.78631591796875,
      "activations/layer11_attention_weight_min": -30.643163681030273,
      "activations/layer12_attention_weight_max": 39.03102111816406,
      "activations/layer12_attention_weight_min": -31.762147903442383,
      "activations/layer13_attention_weight_max": 43.85388946533203,
      "activations/layer13_attention_weight_min": -33.96788787841797,
      "activations/layer14_attention_weight_max": 75.83314514160156,
      "activations/layer14_attention_weight_min": -55.04190444946289,
      "activations/layer15_attention_weight_max": 42.76442337036133,
      "activations/layer15_attention_weight_min": -33.34065246582031,
      "activations/layer16_attention_weight_max": 40.76420974731445,
      "activations/layer16_attention_weight_min": -31.21614646911621,
      "activations/layer17_attention_weight_max": 39.956504821777344,
      "activations/layer17_attention_weight_min": -27.0908145904541,
      "activations/layer18_attention_weight_max": 30.441940307617188,
      "activations/layer18_attention_weight_min": -19.70223617553711,
      "activations/layer19_attention_weight_max": 45.36394500732422,
      "activations/layer19_attention_weight_min": -31.33705711364746,
      "activations/layer1_attention_weight_max": 18.130807876586914,
      "activations/layer1_attention_weight_min": -14.972359657287598,
      "activations/layer20_attention_weight_max": 34.18961715698242,
      "activations/layer20_attention_weight_min": -23.321491241455078,
      "activations/layer21_attention_weight_max": 29.313594818115234,
      "activations/layer21_attention_weight_min": -20.19827651977539,
      "activations/layer22_attention_weight_max": 46.75860595703125,
      "activations/layer22_attention_weight_min": -27.064573287963867,
      "activations/layer23_attention_weight_max": 30.445114135742188,
      "activations/layer23_attention_weight_min": -18.14598846435547,
      "activations/layer2_attention_weight_max": 30.563344955444336,
      "activations/layer2_attention_weight_min": -30.440521240234375,
      "activations/layer3_attention_weight_max": 79.15144348144531,
      "activations/layer3_attention_weight_min": -83.45160675048828,
      "activations/layer4_attention_weight_max": 130.75735473632812,
      "activations/layer4_attention_weight_min": -122.05517578125,
      "activations/layer5_attention_weight_max": 96.77556610107422,
      "activations/layer5_attention_weight_min": -87.817626953125,
      "activations/layer6_attention_weight_max": 55.49905776977539,
      "activations/layer6_attention_weight_min": -50.759071350097656,
      "activations/layer7_attention_weight_max": 70.54446411132812,
      "activations/layer7_attention_weight_min": -65.68673706054688,
      "activations/layer8_attention_weight_max": 40.64430236816406,
      "activations/layer8_attention_weight_min": -36.683658599853516,
      "activations/layer9_attention_weight_max": 48.52005386352539,
      "activations/layer9_attention_weight_min": -51.171058654785156,
      "epoch": 3.11,
      "learning_rate": 0.00013123522727272725,
      "loss": 2.9017,
      "step": 53600
    },
    {
      "activations/layer0_attention_weight_max": 17.736106872558594,
      "activations/layer0_attention_weight_min": -12.14664077758789,
      "activations/layer10_attention_weight_max": 47.09721755981445,
      "activations/layer10_attention_weight_min": -35.00448226928711,
      "activations/layer11_attention_weight_max": 42.6175422668457,
      "activations/layer11_attention_weight_min": -31.836231231689453,
      "activations/layer12_attention_weight_max": 53.2336540222168,
      "activations/layer12_attention_weight_min": -27.52842903137207,
      "activations/layer13_attention_weight_max": 71.61345672607422,
      "activations/layer13_attention_weight_min": -44.21662139892578,
      "activations/layer14_attention_weight_max": 78.74032592773438,
      "activations/layer14_attention_weight_min": -57.319087982177734,
      "activations/layer15_attention_weight_max": 63.49163818359375,
      "activations/layer15_attention_weight_min": -37.39090347290039,
      "activations/layer16_attention_weight_max": 43.77532196044922,
      "activations/layer16_attention_weight_min": -30.272232055664062,
      "activations/layer17_attention_weight_max": 39.80467987060547,
      "activations/layer17_attention_weight_min": -26.511869430541992,
      "activations/layer18_attention_weight_max": 29.672218322753906,
      "activations/layer18_attention_weight_min": -21.1800594329834,
      "activations/layer19_attention_weight_max": 41.3900032043457,
      "activations/layer19_attention_weight_min": -27.155750274658203,
      "activations/layer1_attention_weight_max": 17.052547454833984,
      "activations/layer1_attention_weight_min": -14.756875038146973,
      "activations/layer20_attention_weight_max": 34.447898864746094,
      "activations/layer20_attention_weight_min": -21.83936309814453,
      "activations/layer21_attention_weight_max": 38.146095275878906,
      "activations/layer21_attention_weight_min": -20.31046485900879,
      "activations/layer22_attention_weight_max": 41.092498779296875,
      "activations/layer22_attention_weight_min": -26.70791244506836,
      "activations/layer23_attention_weight_max": 24.00314712524414,
      "activations/layer23_attention_weight_min": -17.003572463989258,
      "activations/layer2_attention_weight_max": 28.891401290893555,
      "activations/layer2_attention_weight_min": -29.04034423828125,
      "activations/layer3_attention_weight_max": 77.4229736328125,
      "activations/layer3_attention_weight_min": -81.9035415649414,
      "activations/layer4_attention_weight_max": 132.35264587402344,
      "activations/layer4_attention_weight_min": -128.34225463867188,
      "activations/layer5_attention_weight_max": 105.20320129394531,
      "activations/layer5_attention_weight_min": -89.37965393066406,
      "activations/layer6_attention_weight_max": 65.29314422607422,
      "activations/layer6_attention_weight_min": -53.202659606933594,
      "activations/layer7_attention_weight_max": 83.28878784179688,
      "activations/layer7_attention_weight_min": -67.99938201904297,
      "activations/layer8_attention_weight_max": 48.92397689819336,
      "activations/layer8_attention_weight_min": -37.15013885498047,
      "activations/layer9_attention_weight_max": 68.52324676513672,
      "activations/layer9_attention_weight_min": -49.4307746887207,
      "epoch": 3.12,
      "learning_rate": 0.00013121628787878786,
      "loss": 2.9091,
      "step": 53650
    },
    {
      "activations/layer0_attention_weight_max": 17.524682998657227,
      "activations/layer0_attention_weight_min": -11.751418113708496,
      "activations/layer10_attention_weight_max": 35.89881134033203,
      "activations/layer10_attention_weight_min": -33.0788688659668,
      "activations/layer11_attention_weight_max": 34.200016021728516,
      "activations/layer11_attention_weight_min": -26.858383178710938,
      "activations/layer12_attention_weight_max": 48.93487548828125,
      "activations/layer12_attention_weight_min": -29.40546417236328,
      "activations/layer13_attention_weight_max": 47.69413375854492,
      "activations/layer13_attention_weight_min": -35.543540954589844,
      "activations/layer14_attention_weight_max": 66.33540344238281,
      "activations/layer14_attention_weight_min": -47.02915573120117,
      "activations/layer15_attention_weight_max": 45.603858947753906,
      "activations/layer15_attention_weight_min": -31.3939151763916,
      "activations/layer16_attention_weight_max": 41.72883987426758,
      "activations/layer16_attention_weight_min": -30.319286346435547,
      "activations/layer17_attention_weight_max": 41.72967529296875,
      "activations/layer17_attention_weight_min": -26.046737670898438,
      "activations/layer18_attention_weight_max": 36.87332534790039,
      "activations/layer18_attention_weight_min": -19.812461853027344,
      "activations/layer19_attention_weight_max": 42.395206451416016,
      "activations/layer19_attention_weight_min": -29.586511611938477,
      "activations/layer1_attention_weight_max": 17.962236404418945,
      "activations/layer1_attention_weight_min": -15.320088386535645,
      "activations/layer20_attention_weight_max": 33.901241302490234,
      "activations/layer20_attention_weight_min": -21.89812660217285,
      "activations/layer21_attention_weight_max": 29.15829086303711,
      "activations/layer21_attention_weight_min": -19.541990280151367,
      "activations/layer22_attention_weight_max": 48.742061614990234,
      "activations/layer22_attention_weight_min": -28.59379005432129,
      "activations/layer23_attention_weight_max": 26.98121452331543,
      "activations/layer23_attention_weight_min": -18.418643951416016,
      "activations/layer2_attention_weight_max": 30.490535736083984,
      "activations/layer2_attention_weight_min": -29.74300765991211,
      "activations/layer3_attention_weight_max": 78.62470245361328,
      "activations/layer3_attention_weight_min": -81.61747741699219,
      "activations/layer4_attention_weight_max": 125.19527435302734,
      "activations/layer4_attention_weight_min": -121.28221893310547,
      "activations/layer5_attention_weight_max": 97.15425109863281,
      "activations/layer5_attention_weight_min": -94.81429290771484,
      "activations/layer6_attention_weight_max": 55.70579147338867,
      "activations/layer6_attention_weight_min": -55.054588317871094,
      "activations/layer7_attention_weight_max": 69.80760192871094,
      "activations/layer7_attention_weight_min": -71.01277923583984,
      "activations/layer8_attention_weight_max": 38.206722259521484,
      "activations/layer8_attention_weight_min": -35.06555938720703,
      "activations/layer9_attention_weight_max": 52.4160270690918,
      "activations/layer9_attention_weight_min": -41.59658432006836,
      "epoch": 3.12,
      "learning_rate": 0.00013119734848484848,
      "loss": 2.8972,
      "step": 53700
    },
    {
      "activations/layer0_attention_weight_max": 17.258575439453125,
      "activations/layer0_attention_weight_min": -11.706846237182617,
      "activations/layer10_attention_weight_max": 44.24285888671875,
      "activations/layer10_attention_weight_min": -35.16571807861328,
      "activations/layer11_attention_weight_max": 41.353233337402344,
      "activations/layer11_attention_weight_min": -34.00993347167969,
      "activations/layer12_attention_weight_max": 29.463973999023438,
      "activations/layer12_attention_weight_min": -26.848739624023438,
      "activations/layer13_attention_weight_max": 53.34714126586914,
      "activations/layer13_attention_weight_min": -35.36003875732422,
      "activations/layer14_attention_weight_max": 88.31925201416016,
      "activations/layer14_attention_weight_min": -58.90291213989258,
      "activations/layer15_attention_weight_max": 52.46974563598633,
      "activations/layer15_attention_weight_min": -31.03945541381836,
      "activations/layer16_attention_weight_max": 52.311466217041016,
      "activations/layer16_attention_weight_min": -35.84803771972656,
      "activations/layer17_attention_weight_max": 48.689083099365234,
      "activations/layer17_attention_weight_min": -27.576196670532227,
      "activations/layer18_attention_weight_max": 36.363868713378906,
      "activations/layer18_attention_weight_min": -18.695898056030273,
      "activations/layer19_attention_weight_max": 50.01554870605469,
      "activations/layer19_attention_weight_min": -29.02398109436035,
      "activations/layer1_attention_weight_max": 17.101364135742188,
      "activations/layer1_attention_weight_min": -12.70434284210205,
      "activations/layer20_attention_weight_max": 36.94987487792969,
      "activations/layer20_attention_weight_min": -21.88094139099121,
      "activations/layer21_attention_weight_max": 36.38421630859375,
      "activations/layer21_attention_weight_min": -19.752216339111328,
      "activations/layer22_attention_weight_max": 49.9332389831543,
      "activations/layer22_attention_weight_min": -28.853147506713867,
      "activations/layer23_attention_weight_max": 28.964611053466797,
      "activations/layer23_attention_weight_min": -17.640281677246094,
      "activations/layer2_attention_weight_max": 30.19241714477539,
      "activations/layer2_attention_weight_min": -28.341657638549805,
      "activations/layer3_attention_weight_max": 80.89403533935547,
      "activations/layer3_attention_weight_min": -82.38848876953125,
      "activations/layer4_attention_weight_max": 130.41116333007812,
      "activations/layer4_attention_weight_min": -121.8251724243164,
      "activations/layer5_attention_weight_max": 107.76495361328125,
      "activations/layer5_attention_weight_min": -90.04714965820312,
      "activations/layer6_attention_weight_max": 59.32231140136719,
      "activations/layer6_attention_weight_min": -54.24660110473633,
      "activations/layer7_attention_weight_max": 72.90861511230469,
      "activations/layer7_attention_weight_min": -65.56423950195312,
      "activations/layer8_attention_weight_max": 43.41508483886719,
      "activations/layer8_attention_weight_min": -34.928653717041016,
      "activations/layer9_attention_weight_max": 51.597164154052734,
      "activations/layer9_attention_weight_min": -42.629329681396484,
      "epoch": 3.12,
      "learning_rate": 0.0001311784090909091,
      "loss": 2.9125,
      "step": 53750
    },
    {
      "activations/layer0_attention_weight_max": 16.52741813659668,
      "activations/layer0_attention_weight_min": -11.248571395874023,
      "activations/layer10_attention_weight_max": 41.0046501159668,
      "activations/layer10_attention_weight_min": -33.06421661376953,
      "activations/layer11_attention_weight_max": 32.855838775634766,
      "activations/layer11_attention_weight_min": -27.18695831298828,
      "activations/layer12_attention_weight_max": 27.50774574279785,
      "activations/layer12_attention_weight_min": -26.740598678588867,
      "activations/layer13_attention_weight_max": 49.02688217163086,
      "activations/layer13_attention_weight_min": -38.44757080078125,
      "activations/layer14_attention_weight_max": 68.54544830322266,
      "activations/layer14_attention_weight_min": -55.78520965576172,
      "activations/layer15_attention_weight_max": 41.85216522216797,
      "activations/layer15_attention_weight_min": -31.111783981323242,
      "activations/layer16_attention_weight_max": 42.47863006591797,
      "activations/layer16_attention_weight_min": -28.804574966430664,
      "activations/layer17_attention_weight_max": 44.639827728271484,
      "activations/layer17_attention_weight_min": -28.183353424072266,
      "activations/layer18_attention_weight_max": 28.138916015625,
      "activations/layer18_attention_weight_min": -21.977907180786133,
      "activations/layer19_attention_weight_max": 41.11991500854492,
      "activations/layer19_attention_weight_min": -29.63899803161621,
      "activations/layer1_attention_weight_max": 18.234506607055664,
      "activations/layer1_attention_weight_min": -13.520282745361328,
      "activations/layer20_attention_weight_max": 29.821331024169922,
      "activations/layer20_attention_weight_min": -23.490154266357422,
      "activations/layer21_attention_weight_max": 36.401939392089844,
      "activations/layer21_attention_weight_min": -20.563133239746094,
      "activations/layer22_attention_weight_max": 51.83064270019531,
      "activations/layer22_attention_weight_min": -27.894550323486328,
      "activations/layer23_attention_weight_max": 29.11941909790039,
      "activations/layer23_attention_weight_min": -19.662166595458984,
      "activations/layer2_attention_weight_max": 30.490768432617188,
      "activations/layer2_attention_weight_min": -30.957006454467773,
      "activations/layer3_attention_weight_max": 78.21733856201172,
      "activations/layer3_attention_weight_min": -85.52046203613281,
      "activations/layer4_attention_weight_max": 126.60216522216797,
      "activations/layer4_attention_weight_min": -132.55105590820312,
      "activations/layer5_attention_weight_max": 99.44229125976562,
      "activations/layer5_attention_weight_min": -93.60967254638672,
      "activations/layer6_attention_weight_max": 58.69173049926758,
      "activations/layer6_attention_weight_min": -56.940032958984375,
      "activations/layer7_attention_weight_max": 70.78993225097656,
      "activations/layer7_attention_weight_min": -74.43804931640625,
      "activations/layer8_attention_weight_max": 38.57242202758789,
      "activations/layer8_attention_weight_min": -38.90434265136719,
      "activations/layer9_attention_weight_max": 44.43708419799805,
      "activations/layer9_attention_weight_min": -38.481319427490234,
      "epoch": 3.13,
      "learning_rate": 0.00013115946969696968,
      "loss": 2.9007,
      "step": 53800
    },
    {
      "activations/layer0_attention_weight_max": 17.014986038208008,
      "activations/layer0_attention_weight_min": -11.804286003112793,
      "activations/layer10_attention_weight_max": 38.3661994934082,
      "activations/layer10_attention_weight_min": -34.777740478515625,
      "activations/layer11_attention_weight_max": 34.950382232666016,
      "activations/layer11_attention_weight_min": -28.04662322998047,
      "activations/layer12_attention_weight_max": 29.25409507751465,
      "activations/layer12_attention_weight_min": -25.264596939086914,
      "activations/layer13_attention_weight_max": 52.66554641723633,
      "activations/layer13_attention_weight_min": -39.26704788208008,
      "activations/layer14_attention_weight_max": 68.85487365722656,
      "activations/layer14_attention_weight_min": -55.44118118286133,
      "activations/layer15_attention_weight_max": 44.20248794555664,
      "activations/layer15_attention_weight_min": -33.015541076660156,
      "activations/layer16_attention_weight_max": 45.5726318359375,
      "activations/layer16_attention_weight_min": -33.26910400390625,
      "activations/layer17_attention_weight_max": 46.49903106689453,
      "activations/layer17_attention_weight_min": -27.64948272705078,
      "activations/layer18_attention_weight_max": 28.039588928222656,
      "activations/layer18_attention_weight_min": -20.158981323242188,
      "activations/layer19_attention_weight_max": 48.02212142944336,
      "activations/layer19_attention_weight_min": -31.63994789123535,
      "activations/layer1_attention_weight_max": 18.169185638427734,
      "activations/layer1_attention_weight_min": -14.837100982666016,
      "activations/layer20_attention_weight_max": 36.41318130493164,
      "activations/layer20_attention_weight_min": -23.544736862182617,
      "activations/layer21_attention_weight_max": 32.67472839355469,
      "activations/layer21_attention_weight_min": -20.118362426757812,
      "activations/layer22_attention_weight_max": 50.41778564453125,
      "activations/layer22_attention_weight_min": -29.747230529785156,
      "activations/layer23_attention_weight_max": 31.56557846069336,
      "activations/layer23_attention_weight_min": -16.337881088256836,
      "activations/layer2_attention_weight_max": 31.03803825378418,
      "activations/layer2_attention_weight_min": -30.6331787109375,
      "activations/layer3_attention_weight_max": 80.16213989257812,
      "activations/layer3_attention_weight_min": -82.88731384277344,
      "activations/layer4_attention_weight_max": 128.32266235351562,
      "activations/layer4_attention_weight_min": -124.7374267578125,
      "activations/layer5_attention_weight_max": 95.14315032958984,
      "activations/layer5_attention_weight_min": -92.18318939208984,
      "activations/layer6_attention_weight_max": 55.122093200683594,
      "activations/layer6_attention_weight_min": -55.567649841308594,
      "activations/layer7_attention_weight_max": 72.18721771240234,
      "activations/layer7_attention_weight_min": -73.08509826660156,
      "activations/layer8_attention_weight_max": 37.82686996459961,
      "activations/layer8_attention_weight_min": -36.42927932739258,
      "activations/layer9_attention_weight_max": 46.49497604370117,
      "activations/layer9_attention_weight_min": -37.218345642089844,
      "epoch": 3.13,
      "learning_rate": 0.0001311405303030303,
      "loss": 2.9228,
      "step": 53850
    },
    {
      "activations/layer0_attention_weight_max": 16.56050682067871,
      "activations/layer0_attention_weight_min": -11.909512519836426,
      "activations/layer10_attention_weight_max": 41.17932891845703,
      "activations/layer10_attention_weight_min": -32.633026123046875,
      "activations/layer11_attention_weight_max": 37.86195373535156,
      "activations/layer11_attention_weight_min": -30.797298431396484,
      "activations/layer12_attention_weight_max": 31.351242065429688,
      "activations/layer12_attention_weight_min": -28.713340759277344,
      "activations/layer13_attention_weight_max": 47.57308578491211,
      "activations/layer13_attention_weight_min": -36.11045837402344,
      "activations/layer14_attention_weight_max": 76.75249481201172,
      "activations/layer14_attention_weight_min": -56.774444580078125,
      "activations/layer15_attention_weight_max": 42.178253173828125,
      "activations/layer15_attention_weight_min": -30.126644134521484,
      "activations/layer16_attention_weight_max": 37.79058837890625,
      "activations/layer16_attention_weight_min": -30.88562774658203,
      "activations/layer17_attention_weight_max": 44.30335998535156,
      "activations/layer17_attention_weight_min": -25.18600082397461,
      "activations/layer18_attention_weight_max": 30.922565460205078,
      "activations/layer18_attention_weight_min": -21.365144729614258,
      "activations/layer19_attention_weight_max": 46.80995178222656,
      "activations/layer19_attention_weight_min": -27.739683151245117,
      "activations/layer1_attention_weight_max": 17.667728424072266,
      "activations/layer1_attention_weight_min": -12.528779983520508,
      "activations/layer20_attention_weight_max": 29.60409164428711,
      "activations/layer20_attention_weight_min": -25.38408660888672,
      "activations/layer21_attention_weight_max": 35.77518081665039,
      "activations/layer21_attention_weight_min": -20.322046279907227,
      "activations/layer22_attention_weight_max": 48.4407958984375,
      "activations/layer22_attention_weight_min": -26.79914093017578,
      "activations/layer23_attention_weight_max": 26.532615661621094,
      "activations/layer23_attention_weight_min": -18.039016723632812,
      "activations/layer2_attention_weight_max": 30.733623504638672,
      "activations/layer2_attention_weight_min": -30.344100952148438,
      "activations/layer3_attention_weight_max": 80.23934936523438,
      "activations/layer3_attention_weight_min": -86.3472671508789,
      "activations/layer4_attention_weight_max": 126.93463134765625,
      "activations/layer4_attention_weight_min": -128.22488403320312,
      "activations/layer5_attention_weight_max": 97.7345199584961,
      "activations/layer5_attention_weight_min": -91.7021713256836,
      "activations/layer6_attention_weight_max": 55.40974426269531,
      "activations/layer6_attention_weight_min": -54.187862396240234,
      "activations/layer7_attention_weight_max": 70.22196197509766,
      "activations/layer7_attention_weight_min": -68.62226104736328,
      "activations/layer8_attention_weight_max": 37.914886474609375,
      "activations/layer8_attention_weight_min": -36.86821746826172,
      "activations/layer9_attention_weight_max": 43.18462371826172,
      "activations/layer9_attention_weight_min": -33.93347930908203,
      "epoch": 3.13,
      "learning_rate": 0.0001311215909090909,
      "loss": 2.8968,
      "step": 53900
    },
    {
      "activations/layer0_attention_weight_max": 16.672565460205078,
      "activations/layer0_attention_weight_min": -11.440006256103516,
      "activations/layer10_attention_weight_max": 45.396793365478516,
      "activations/layer10_attention_weight_min": -33.031959533691406,
      "activations/layer11_attention_weight_max": 39.51600646972656,
      "activations/layer11_attention_weight_min": -32.26126480102539,
      "activations/layer12_attention_weight_max": 36.5530891418457,
      "activations/layer12_attention_weight_min": -25.291297912597656,
      "activations/layer13_attention_weight_max": 53.14811706542969,
      "activations/layer13_attention_weight_min": -35.536216735839844,
      "activations/layer14_attention_weight_max": 81.55742645263672,
      "activations/layer14_attention_weight_min": -58.618717193603516,
      "activations/layer15_attention_weight_max": 53.035823822021484,
      "activations/layer15_attention_weight_min": -33.17660140991211,
      "activations/layer16_attention_weight_max": 46.39229202270508,
      "activations/layer16_attention_weight_min": -33.63101577758789,
      "activations/layer17_attention_weight_max": 50.173797607421875,
      "activations/layer17_attention_weight_min": -28.52032470703125,
      "activations/layer18_attention_weight_max": 32.55988693237305,
      "activations/layer18_attention_weight_min": -18.630340576171875,
      "activations/layer19_attention_weight_max": 48.86771774291992,
      "activations/layer19_attention_weight_min": -33.54903030395508,
      "activations/layer1_attention_weight_max": 16.671363830566406,
      "activations/layer1_attention_weight_min": -14.651260375976562,
      "activations/layer20_attention_weight_max": 42.41385269165039,
      "activations/layer20_attention_weight_min": -27.188936233520508,
      "activations/layer21_attention_weight_max": 40.56082534790039,
      "activations/layer21_attention_weight_min": -23.0338077545166,
      "activations/layer22_attention_weight_max": 55.97401428222656,
      "activations/layer22_attention_weight_min": -25.360198974609375,
      "activations/layer23_attention_weight_max": 30.897418975830078,
      "activations/layer23_attention_weight_min": -16.125057220458984,
      "activations/layer2_attention_weight_max": 31.575345993041992,
      "activations/layer2_attention_weight_min": -31.136627197265625,
      "activations/layer3_attention_weight_max": 81.41267395019531,
      "activations/layer3_attention_weight_min": -85.43727111816406,
      "activations/layer4_attention_weight_max": 142.0077362060547,
      "activations/layer4_attention_weight_min": -132.1885528564453,
      "activations/layer5_attention_weight_max": 113.3532943725586,
      "activations/layer5_attention_weight_min": -96.37226867675781,
      "activations/layer6_attention_weight_max": 61.16072082519531,
      "activations/layer6_attention_weight_min": -55.8404655456543,
      "activations/layer7_attention_weight_max": 79.82405853271484,
      "activations/layer7_attention_weight_min": -69.88334655761719,
      "activations/layer8_attention_weight_max": 41.009788513183594,
      "activations/layer8_attention_weight_min": -36.91388702392578,
      "activations/layer9_attention_weight_max": 56.07281494140625,
      "activations/layer9_attention_weight_min": -53.85725784301758,
      "epoch": 3.13,
      "learning_rate": 0.0001311026515151515,
      "loss": 2.9112,
      "step": 53950
    },
    {
      "activations/layer0_attention_weight_max": 16.117759704589844,
      "activations/layer0_attention_weight_min": -11.39134693145752,
      "activations/layer10_attention_weight_max": 39.993408203125,
      "activations/layer10_attention_weight_min": -32.25042724609375,
      "activations/layer11_attention_weight_max": 41.46647644042969,
      "activations/layer11_attention_weight_min": -29.493072509765625,
      "activations/layer12_attention_weight_max": 29.62173080444336,
      "activations/layer12_attention_weight_min": -28.836841583251953,
      "activations/layer13_attention_weight_max": 49.42195129394531,
      "activations/layer13_attention_weight_min": -34.68770217895508,
      "activations/layer14_attention_weight_max": 75.66122436523438,
      "activations/layer14_attention_weight_min": -56.500526428222656,
      "activations/layer15_attention_weight_max": 53.46910858154297,
      "activations/layer15_attention_weight_min": -33.25068664550781,
      "activations/layer16_attention_weight_max": 49.39720153808594,
      "activations/layer16_attention_weight_min": -34.806209564208984,
      "activations/layer17_attention_weight_max": 45.35889434814453,
      "activations/layer17_attention_weight_min": -26.90667152404785,
      "activations/layer18_attention_weight_max": 36.39915084838867,
      "activations/layer18_attention_weight_min": -20.327104568481445,
      "activations/layer19_attention_weight_max": 51.2552604675293,
      "activations/layer19_attention_weight_min": -29.695676803588867,
      "activations/layer1_attention_weight_max": 17.226945877075195,
      "activations/layer1_attention_weight_min": -13.952102661132812,
      "activations/layer20_attention_weight_max": 38.83423614501953,
      "activations/layer20_attention_weight_min": -23.011211395263672,
      "activations/layer21_attention_weight_max": 37.154945373535156,
      "activations/layer21_attention_weight_min": -22.72420883178711,
      "activations/layer22_attention_weight_max": 49.01811981201172,
      "activations/layer22_attention_weight_min": -26.595226287841797,
      "activations/layer23_attention_weight_max": 29.153282165527344,
      "activations/layer23_attention_weight_min": -17.060230255126953,
      "activations/layer2_attention_weight_max": 31.17972183227539,
      "activations/layer2_attention_weight_min": -30.34121322631836,
      "activations/layer3_attention_weight_max": 81.79948425292969,
      "activations/layer3_attention_weight_min": -81.6802978515625,
      "activations/layer4_attention_weight_max": 133.08811950683594,
      "activations/layer4_attention_weight_min": -130.52703857421875,
      "activations/layer5_attention_weight_max": 110.06957244873047,
      "activations/layer5_attention_weight_min": -96.23137664794922,
      "activations/layer6_attention_weight_max": 61.53227233886719,
      "activations/layer6_attention_weight_min": -55.21469497680664,
      "activations/layer7_attention_weight_max": 76.27869415283203,
      "activations/layer7_attention_weight_min": -70.29398345947266,
      "activations/layer8_attention_weight_max": 42.829261779785156,
      "activations/layer8_attention_weight_min": -37.977867126464844,
      "activations/layer9_attention_weight_max": 50.19083786010742,
      "activations/layer9_attention_weight_min": -37.35127639770508,
      "epoch": 3.14,
      "learning_rate": 0.00013108371212121212,
      "loss": 2.8867,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_loss": 2.85546875,
      "eval_runtime": 8.511,
      "eval_samples_per_second": 504.521,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_openwebtext_loss": 2.85546875,
      "eval_openwebtext_ppl": 17.382583384208317,
      "eval_openwebtext_runtime": 8.511,
      "eval_openwebtext_samples_per_second": 504.521,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_wikitext_loss": 3.11328125,
      "eval_wikitext_ppl": 22.494734284035275,
      "eval_wikitext_runtime": 1.9924,
      "eval_wikitext_samples_per_second": 228.868,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_lambada_loss": 2.947265625,
      "eval_lambada_ppl": 19.053782246709744,
      "eval_lambada_runtime": 9.6049,
      "eval_lambada_samples_per_second": 506.928,
      "step": 54000
    },
    {
      "activations/layer0_attention_weight_max": 15.056060791015625,
      "activations/layer0_attention_weight_min": -12.938342094421387,
      "activations/layer10_attention_weight_max": 43.931243896484375,
      "activations/layer10_attention_weight_min": -35.2479248046875,
      "activations/layer11_attention_weight_max": 37.34967803955078,
      "activations/layer11_attention_weight_min": -30.144887924194336,
      "activations/layer12_attention_weight_max": 38.46800994873047,
      "activations/layer12_attention_weight_min": -26.98410987854004,
      "activations/layer13_attention_weight_max": 51.77364730834961,
      "activations/layer13_attention_weight_min": -35.94096374511719,
      "activations/layer14_attention_weight_max": 86.1649398803711,
      "activations/layer14_attention_weight_min": -61.29833984375,
      "activations/layer15_attention_weight_max": 45.57423400878906,
      "activations/layer15_attention_weight_min": -33.205169677734375,
      "activations/layer16_attention_weight_max": 42.65143966674805,
      "activations/layer16_attention_weight_min": -29.190738677978516,
      "activations/layer17_attention_weight_max": 43.52985382080078,
      "activations/layer17_attention_weight_min": -26.017839431762695,
      "activations/layer18_attention_weight_max": 29.316320419311523,
      "activations/layer18_attention_weight_min": -19.033138275146484,
      "activations/layer19_attention_weight_max": 44.35578155517578,
      "activations/layer19_attention_weight_min": -28.677824020385742,
      "activations/layer1_attention_weight_max": 17.5894775390625,
      "activations/layer1_attention_weight_min": -13.333078384399414,
      "activations/layer20_attention_weight_max": 32.77527618408203,
      "activations/layer20_attention_weight_min": -25.077980041503906,
      "activations/layer21_attention_weight_max": 30.470876693725586,
      "activations/layer21_attention_weight_min": -18.526784896850586,
      "activations/layer22_attention_weight_max": 49.75284194946289,
      "activations/layer22_attention_weight_min": -28.758575439453125,
      "activations/layer23_attention_weight_max": 28.87727165222168,
      "activations/layer23_attention_weight_min": -19.216291427612305,
      "activations/layer2_attention_weight_max": 31.48676872253418,
      "activations/layer2_attention_weight_min": -31.137733459472656,
      "activations/layer3_attention_weight_max": 79.70892333984375,
      "activations/layer3_attention_weight_min": -84.8725357055664,
      "activations/layer4_attention_weight_max": 131.9480743408203,
      "activations/layer4_attention_weight_min": -128.91122436523438,
      "activations/layer5_attention_weight_max": 103.5609359741211,
      "activations/layer5_attention_weight_min": -99.43315124511719,
      "activations/layer6_attention_weight_max": 58.622764587402344,
      "activations/layer6_attention_weight_min": -52.59589767456055,
      "activations/layer7_attention_weight_max": 74.26525115966797,
      "activations/layer7_attention_weight_min": -72.72815704345703,
      "activations/layer8_attention_weight_max": 40.742332458496094,
      "activations/layer8_attention_weight_min": -38.8665885925293,
      "activations/layer9_attention_weight_max": 46.064022064208984,
      "activations/layer9_attention_weight_min": -41.71677017211914,
      "epoch": 3.14,
      "learning_rate": 0.00013106477272727273,
      "loss": 2.8934,
      "step": 54050
    },
    {
      "activations/layer0_attention_weight_max": 17.188873291015625,
      "activations/layer0_attention_weight_min": -12.489412307739258,
      "activations/layer10_attention_weight_max": 40.73347091674805,
      "activations/layer10_attention_weight_min": -30.651592254638672,
      "activations/layer11_attention_weight_max": 36.721527099609375,
      "activations/layer11_attention_weight_min": -30.11687469482422,
      "activations/layer12_attention_weight_max": 30.06734848022461,
      "activations/layer12_attention_weight_min": -29.787364959716797,
      "activations/layer13_attention_weight_max": 46.066104888916016,
      "activations/layer13_attention_weight_min": -33.03606414794922,
      "activations/layer14_attention_weight_max": 84.43394470214844,
      "activations/layer14_attention_weight_min": -57.61373519897461,
      "activations/layer15_attention_weight_max": 44.87226867675781,
      "activations/layer15_attention_weight_min": -31.155197143554688,
      "activations/layer16_attention_weight_max": 44.500640869140625,
      "activations/layer16_attention_weight_min": -30.998998641967773,
      "activations/layer17_attention_weight_max": 38.69576644897461,
      "activations/layer17_attention_weight_min": -23.909080505371094,
      "activations/layer18_attention_weight_max": 30.209148406982422,
      "activations/layer18_attention_weight_min": -18.491992950439453,
      "activations/layer19_attention_weight_max": 47.45903015136719,
      "activations/layer19_attention_weight_min": -29.649871826171875,
      "activations/layer1_attention_weight_max": 18.439847946166992,
      "activations/layer1_attention_weight_min": -15.638736724853516,
      "activations/layer20_attention_weight_max": 33.39419937133789,
      "activations/layer20_attention_weight_min": -21.77497673034668,
      "activations/layer21_attention_weight_max": 32.00351333618164,
      "activations/layer21_attention_weight_min": -17.841276168823242,
      "activations/layer22_attention_weight_max": 52.85517883300781,
      "activations/layer22_attention_weight_min": -25.82489585876465,
      "activations/layer23_attention_weight_max": 31.472320556640625,
      "activations/layer23_attention_weight_min": -16.767616271972656,
      "activations/layer2_attention_weight_max": 31.30923080444336,
      "activations/layer2_attention_weight_min": -29.472442626953125,
      "activations/layer3_attention_weight_max": 78.82080078125,
      "activations/layer3_attention_weight_min": -83.11611938476562,
      "activations/layer4_attention_weight_max": 138.57948303222656,
      "activations/layer4_attention_weight_min": -124.92718505859375,
      "activations/layer5_attention_weight_max": 106.77916717529297,
      "activations/layer5_attention_weight_min": -87.80963134765625,
      "activations/layer6_attention_weight_max": 58.1520881652832,
      "activations/layer6_attention_weight_min": -50.562232971191406,
      "activations/layer7_attention_weight_max": 77.9719467163086,
      "activations/layer7_attention_weight_min": -74.00849151611328,
      "activations/layer8_attention_weight_max": 43.22964859008789,
      "activations/layer8_attention_weight_min": -35.961368560791016,
      "activations/layer9_attention_weight_max": 55.540401458740234,
      "activations/layer9_attention_weight_min": -40.5772590637207,
      "epoch": 3.14,
      "learning_rate": 0.00013104621212121212,
      "loss": 2.914,
      "step": 54100
    },
    {
      "activations/layer0_attention_weight_max": 17.227127075195312,
      "activations/layer0_attention_weight_min": -11.879388809204102,
      "activations/layer10_attention_weight_max": 44.12177276611328,
      "activations/layer10_attention_weight_min": -34.35914611816406,
      "activations/layer11_attention_weight_max": 40.70781326293945,
      "activations/layer11_attention_weight_min": -30.159292221069336,
      "activations/layer12_attention_weight_max": 28.813756942749023,
      "activations/layer12_attention_weight_min": -25.98719024658203,
      "activations/layer13_attention_weight_max": 56.63023376464844,
      "activations/layer13_attention_weight_min": -37.02251434326172,
      "activations/layer14_attention_weight_max": 81.61976623535156,
      "activations/layer14_attention_weight_min": -60.98923110961914,
      "activations/layer15_attention_weight_max": 48.972782135009766,
      "activations/layer15_attention_weight_min": -35.2410774230957,
      "activations/layer16_attention_weight_max": 47.027427673339844,
      "activations/layer16_attention_weight_min": -32.85603713989258,
      "activations/layer17_attention_weight_max": 46.36113739013672,
      "activations/layer17_attention_weight_min": -27.08067512512207,
      "activations/layer18_attention_weight_max": 37.06414031982422,
      "activations/layer18_attention_weight_min": -22.352190017700195,
      "activations/layer19_attention_weight_max": 47.77386474609375,
      "activations/layer19_attention_weight_min": -29.617258071899414,
      "activations/layer1_attention_weight_max": 16.99317741394043,
      "activations/layer1_attention_weight_min": -13.535087585449219,
      "activations/layer20_attention_weight_max": 36.021636962890625,
      "activations/layer20_attention_weight_min": -25.222915649414062,
      "activations/layer21_attention_weight_max": 37.588504791259766,
      "activations/layer21_attention_weight_min": -20.63620948791504,
      "activations/layer22_attention_weight_max": 50.02715301513672,
      "activations/layer22_attention_weight_min": -27.42283821105957,
      "activations/layer23_attention_weight_max": 27.955337524414062,
      "activations/layer23_attention_weight_min": -18.009706497192383,
      "activations/layer2_attention_weight_max": 32.331153869628906,
      "activations/layer2_attention_weight_min": -30.762466430664062,
      "activations/layer3_attention_weight_max": 82.54647064208984,
      "activations/layer3_attention_weight_min": -84.38763427734375,
      "activations/layer4_attention_weight_max": 133.65538024902344,
      "activations/layer4_attention_weight_min": -126.08342742919922,
      "activations/layer5_attention_weight_max": 106.53285217285156,
      "activations/layer5_attention_weight_min": -93.14635467529297,
      "activations/layer6_attention_weight_max": 60.04155349731445,
      "activations/layer6_attention_weight_min": -55.5506477355957,
      "activations/layer7_attention_weight_max": 77.22152709960938,
      "activations/layer7_attention_weight_min": -71.01184844970703,
      "activations/layer8_attention_weight_max": 44.500709533691406,
      "activations/layer8_attention_weight_min": -37.60515213012695,
      "activations/layer9_attention_weight_max": 46.67953109741211,
      "activations/layer9_attention_weight_min": -39.86904525756836,
      "epoch": 3.15,
      "learning_rate": 0.0001310272727272727,
      "loss": 2.9037,
      "step": 54150
    },
    {
      "activations/layer0_attention_weight_max": 16.585657119750977,
      "activations/layer0_attention_weight_min": -12.128575325012207,
      "activations/layer10_attention_weight_max": 43.95697784423828,
      "activations/layer10_attention_weight_min": -33.33852005004883,
      "activations/layer11_attention_weight_max": 33.407432556152344,
      "activations/layer11_attention_weight_min": -28.926170349121094,
      "activations/layer12_attention_weight_max": 23.396095275878906,
      "activations/layer12_attention_weight_min": -26.49798011779785,
      "activations/layer13_attention_weight_max": 45.66114807128906,
      "activations/layer13_attention_weight_min": -32.64162826538086,
      "activations/layer14_attention_weight_max": 80.58230590820312,
      "activations/layer14_attention_weight_min": -65.66930389404297,
      "activations/layer15_attention_weight_max": 44.833621978759766,
      "activations/layer15_attention_weight_min": -32.983360290527344,
      "activations/layer16_attention_weight_max": 44.64451599121094,
      "activations/layer16_attention_weight_min": -31.325159072875977,
      "activations/layer17_attention_weight_max": 40.5736198425293,
      "activations/layer17_attention_weight_min": -28.69939613342285,
      "activations/layer18_attention_weight_max": 31.565580368041992,
      "activations/layer18_attention_weight_min": -20.518400192260742,
      "activations/layer19_attention_weight_max": 37.78215408325195,
      "activations/layer19_attention_weight_min": -30.945968627929688,
      "activations/layer1_attention_weight_max": 17.29636573791504,
      "activations/layer1_attention_weight_min": -12.210906982421875,
      "activations/layer20_attention_weight_max": 31.234825134277344,
      "activations/layer20_attention_weight_min": -24.057329177856445,
      "activations/layer21_attention_weight_max": 32.150230407714844,
      "activations/layer21_attention_weight_min": -20.8026180267334,
      "activations/layer22_attention_weight_max": 46.798622131347656,
      "activations/layer22_attention_weight_min": -29.57855987548828,
      "activations/layer23_attention_weight_max": 24.525625228881836,
      "activations/layer23_attention_weight_min": -16.38668441772461,
      "activations/layer2_attention_weight_max": 27.024799346923828,
      "activations/layer2_attention_weight_min": -30.289810180664062,
      "activations/layer3_attention_weight_max": 69.34851837158203,
      "activations/layer3_attention_weight_min": -69.84785461425781,
      "activations/layer4_attention_weight_max": 107.40277099609375,
      "activations/layer4_attention_weight_min": -114.5355453491211,
      "activations/layer5_attention_weight_max": 89.3860092163086,
      "activations/layer5_attention_weight_min": -107.11514282226562,
      "activations/layer6_attention_weight_max": 56.45800018310547,
      "activations/layer6_attention_weight_min": -57.28160858154297,
      "activations/layer7_attention_weight_max": 73.69013977050781,
      "activations/layer7_attention_weight_min": -77.28522491455078,
      "activations/layer8_attention_weight_max": 40.02775573730469,
      "activations/layer8_attention_weight_min": -40.672542572021484,
      "activations/layer9_attention_weight_max": 49.19751739501953,
      "activations/layer9_attention_weight_min": -36.345584869384766,
      "epoch": 3.15,
      "learning_rate": 0.00013100871212121213,
      "loss": 2.9377,
      "step": 54200
    },
    {
      "activations/layer0_attention_weight_max": 16.361434936523438,
      "activations/layer0_attention_weight_min": -12.731731414794922,
      "activations/layer10_attention_weight_max": 40.5894775390625,
      "activations/layer10_attention_weight_min": -31.178964614868164,
      "activations/layer11_attention_weight_max": 36.059844970703125,
      "activations/layer11_attention_weight_min": -31.92568016052246,
      "activations/layer12_attention_weight_max": 47.3070068359375,
      "activations/layer12_attention_weight_min": -29.272323608398438,
      "activations/layer13_attention_weight_max": 61.16917037963867,
      "activations/layer13_attention_weight_min": -38.60979080200195,
      "activations/layer14_attention_weight_max": 93.50592803955078,
      "activations/layer14_attention_weight_min": -62.301822662353516,
      "activations/layer15_attention_weight_max": 51.657264709472656,
      "activations/layer15_attention_weight_min": -34.08603286743164,
      "activations/layer16_attention_weight_max": 40.86865234375,
      "activations/layer16_attention_weight_min": -30.108442306518555,
      "activations/layer17_attention_weight_max": 44.78349685668945,
      "activations/layer17_attention_weight_min": -27.995481491088867,
      "activations/layer18_attention_weight_max": 34.086944580078125,
      "activations/layer18_attention_weight_min": -22.553386688232422,
      "activations/layer19_attention_weight_max": 41.71187210083008,
      "activations/layer19_attention_weight_min": -31.274457931518555,
      "activations/layer1_attention_weight_max": 18.51471519470215,
      "activations/layer1_attention_weight_min": -13.001242637634277,
      "activations/layer20_attention_weight_max": 33.907527923583984,
      "activations/layer20_attention_weight_min": -21.069473266601562,
      "activations/layer21_attention_weight_max": 30.003707885742188,
      "activations/layer21_attention_weight_min": -22.39275550842285,
      "activations/layer22_attention_weight_max": 50.574161529541016,
      "activations/layer22_attention_weight_min": -28.27510643005371,
      "activations/layer23_attention_weight_max": 28.09732437133789,
      "activations/layer23_attention_weight_min": -17.33672332763672,
      "activations/layer2_attention_weight_max": 28.444446563720703,
      "activations/layer2_attention_weight_min": -30.30708885192871,
      "activations/layer3_attention_weight_max": 73.6493911743164,
      "activations/layer3_attention_weight_min": -76.08572387695312,
      "activations/layer4_attention_weight_max": 120.3166275024414,
      "activations/layer4_attention_weight_min": -118.74016571044922,
      "activations/layer5_attention_weight_max": 99.19230651855469,
      "activations/layer5_attention_weight_min": -95.45962524414062,
      "activations/layer6_attention_weight_max": 57.362632751464844,
      "activations/layer6_attention_weight_min": -53.25112533569336,
      "activations/layer7_attention_weight_max": 69.51466369628906,
      "activations/layer7_attention_weight_min": -73.1444091796875,
      "activations/layer8_attention_weight_max": 42.5427131652832,
      "activations/layer8_attention_weight_min": -38.533817291259766,
      "activations/layer9_attention_weight_max": 53.81740951538086,
      "activations/layer9_attention_weight_min": -43.19696807861328,
      "epoch": 3.15,
      "learning_rate": 0.00013098977272727271,
      "loss": 2.9206,
      "step": 54250
    },
    {
      "activations/layer0_attention_weight_max": 17.626007080078125,
      "activations/layer0_attention_weight_min": -13.059410095214844,
      "activations/layer10_attention_weight_max": 42.5101318359375,
      "activations/layer10_attention_weight_min": -31.686864852905273,
      "activations/layer11_attention_weight_max": 42.560672760009766,
      "activations/layer11_attention_weight_min": -32.01927947998047,
      "activations/layer12_attention_weight_max": 36.65059280395508,
      "activations/layer12_attention_weight_min": -23.977556228637695,
      "activations/layer13_attention_weight_max": 56.15189743041992,
      "activations/layer13_attention_weight_min": -33.97324752807617,
      "activations/layer14_attention_weight_max": 75.25933074951172,
      "activations/layer14_attention_weight_min": -50.015811920166016,
      "activations/layer15_attention_weight_max": 48.71491622924805,
      "activations/layer15_attention_weight_min": -31.8891544342041,
      "activations/layer16_attention_weight_max": 43.01411437988281,
      "activations/layer16_attention_weight_min": -31.184839248657227,
      "activations/layer17_attention_weight_max": 51.1006965637207,
      "activations/layer17_attention_weight_min": -27.029800415039062,
      "activations/layer18_attention_weight_max": 32.1789436340332,
      "activations/layer18_attention_weight_min": -18.15785026550293,
      "activations/layer19_attention_weight_max": 48.84842300415039,
      "activations/layer19_attention_weight_min": -28.293420791625977,
      "activations/layer1_attention_weight_max": 17.71370506286621,
      "activations/layer1_attention_weight_min": -13.396706581115723,
      "activations/layer20_attention_weight_max": 38.6822509765625,
      "activations/layer20_attention_weight_min": -23.10955810546875,
      "activations/layer21_attention_weight_max": 33.3891716003418,
      "activations/layer21_attention_weight_min": -22.27097511291504,
      "activations/layer22_attention_weight_max": 59.70750045776367,
      "activations/layer22_attention_weight_min": -26.9239501953125,
      "activations/layer23_attention_weight_max": 28.898046493530273,
      "activations/layer23_attention_weight_min": -17.51643180847168,
      "activations/layer2_attention_weight_max": 28.40070915222168,
      "activations/layer2_attention_weight_min": -29.705509185791016,
      "activations/layer3_attention_weight_max": 74.28520965576172,
      "activations/layer3_attention_weight_min": -79.66202545166016,
      "activations/layer4_attention_weight_max": 126.3603286743164,
      "activations/layer4_attention_weight_min": -120.73286437988281,
      "activations/layer5_attention_weight_max": 101.53106689453125,
      "activations/layer5_attention_weight_min": -96.13606262207031,
      "activations/layer6_attention_weight_max": 60.847476959228516,
      "activations/layer6_attention_weight_min": -55.22880935668945,
      "activations/layer7_attention_weight_max": 71.9426498413086,
      "activations/layer7_attention_weight_min": -71.50000762939453,
      "activations/layer8_attention_weight_max": 39.732276916503906,
      "activations/layer8_attention_weight_min": -36.636348724365234,
      "activations/layer9_attention_weight_max": 41.49801254272461,
      "activations/layer9_attention_weight_min": -40.4001350402832,
      "epoch": 3.16,
      "learning_rate": 0.00013097083333333333,
      "loss": 2.9068,
      "step": 54300
    },
    {
      "activations/layer0_attention_weight_max": 16.786048889160156,
      "activations/layer0_attention_weight_min": -12.180944442749023,
      "activations/layer10_attention_weight_max": 42.57320022583008,
      "activations/layer10_attention_weight_min": -32.85520935058594,
      "activations/layer11_attention_weight_max": 37.84228515625,
      "activations/layer11_attention_weight_min": -29.264883041381836,
      "activations/layer12_attention_weight_max": 40.02473831176758,
      "activations/layer12_attention_weight_min": -26.32598114013672,
      "activations/layer13_attention_weight_max": 49.27878189086914,
      "activations/layer13_attention_weight_min": -33.85655975341797,
      "activations/layer14_attention_weight_max": 77.22439575195312,
      "activations/layer14_attention_weight_min": -51.71638107299805,
      "activations/layer15_attention_weight_max": 53.684879302978516,
      "activations/layer15_attention_weight_min": -33.22679138183594,
      "activations/layer16_attention_weight_max": 43.8002815246582,
      "activations/layer16_attention_weight_min": -32.4649658203125,
      "activations/layer17_attention_weight_max": 45.31108474731445,
      "activations/layer17_attention_weight_min": -27.636072158813477,
      "activations/layer18_attention_weight_max": 29.617300033569336,
      "activations/layer18_attention_weight_min": -20.63806915283203,
      "activations/layer19_attention_weight_max": 50.487308502197266,
      "activations/layer19_attention_weight_min": -30.336057662963867,
      "activations/layer1_attention_weight_max": 18.46552276611328,
      "activations/layer1_attention_weight_min": -14.657944679260254,
      "activations/layer20_attention_weight_max": 37.23865509033203,
      "activations/layer20_attention_weight_min": -23.918113708496094,
      "activations/layer21_attention_weight_max": 35.28786087036133,
      "activations/layer21_attention_weight_min": -20.81542205810547,
      "activations/layer22_attention_weight_max": 53.596580505371094,
      "activations/layer22_attention_weight_min": -26.26728057861328,
      "activations/layer23_attention_weight_max": 33.566654205322266,
      "activations/layer23_attention_weight_min": -17.763671875,
      "activations/layer2_attention_weight_max": 29.379756927490234,
      "activations/layer2_attention_weight_min": -29.864727020263672,
      "activations/layer3_attention_weight_max": 77.23676300048828,
      "activations/layer3_attention_weight_min": -78.87171173095703,
      "activations/layer4_attention_weight_max": 129.52073669433594,
      "activations/layer4_attention_weight_min": -122.89453125,
      "activations/layer5_attention_weight_max": 101.3282470703125,
      "activations/layer5_attention_weight_min": -93.92019653320312,
      "activations/layer6_attention_weight_max": 59.308738708496094,
      "activations/layer6_attention_weight_min": -55.122352600097656,
      "activations/layer7_attention_weight_max": 74.34269714355469,
      "activations/layer7_attention_weight_min": -74.87403106689453,
      "activations/layer8_attention_weight_max": 45.574195861816406,
      "activations/layer8_attention_weight_min": -41.95307540893555,
      "activations/layer9_attention_weight_max": 55.099090576171875,
      "activations/layer9_attention_weight_min": -43.246604919433594,
      "epoch": 3.16,
      "learning_rate": 0.00013095189393939392,
      "loss": 2.887,
      "step": 54350
    },
    {
      "activations/layer0_attention_weight_max": 16.110244750976562,
      "activations/layer0_attention_weight_min": -12.651090621948242,
      "activations/layer10_attention_weight_max": 42.32698440551758,
      "activations/layer10_attention_weight_min": -34.880340576171875,
      "activations/layer11_attention_weight_max": 34.62861633300781,
      "activations/layer11_attention_weight_min": -28.909740447998047,
      "activations/layer12_attention_weight_max": 36.23625564575195,
      "activations/layer12_attention_weight_min": -27.051481246948242,
      "activations/layer13_attention_weight_max": 52.104270935058594,
      "activations/layer13_attention_weight_min": -35.31306076049805,
      "activations/layer14_attention_weight_max": 89.74757385253906,
      "activations/layer14_attention_weight_min": -62.36663055419922,
      "activations/layer15_attention_weight_max": 48.18817901611328,
      "activations/layer15_attention_weight_min": -30.84355354309082,
      "activations/layer16_attention_weight_max": 42.53803253173828,
      "activations/layer16_attention_weight_min": -29.573217391967773,
      "activations/layer17_attention_weight_max": 44.01460266113281,
      "activations/layer17_attention_weight_min": -26.243038177490234,
      "activations/layer18_attention_weight_max": 31.352684020996094,
      "activations/layer18_attention_weight_min": -23.894704818725586,
      "activations/layer19_attention_weight_max": 42.524288177490234,
      "activations/layer19_attention_weight_min": -27.083297729492188,
      "activations/layer1_attention_weight_max": 18.603309631347656,
      "activations/layer1_attention_weight_min": -12.996097564697266,
      "activations/layer20_attention_weight_max": 35.40766906738281,
      "activations/layer20_attention_weight_min": -21.980913162231445,
      "activations/layer21_attention_weight_max": 37.92782211303711,
      "activations/layer21_attention_weight_min": -19.460010528564453,
      "activations/layer22_attention_weight_max": 50.48142623901367,
      "activations/layer22_attention_weight_min": -26.47478485107422,
      "activations/layer23_attention_weight_max": 32.379249572753906,
      "activations/layer23_attention_weight_min": -17.27610969543457,
      "activations/layer2_attention_weight_max": 28.484128952026367,
      "activations/layer2_attention_weight_min": -29.218894958496094,
      "activations/layer3_attention_weight_max": 73.0058364868164,
      "activations/layer3_attention_weight_min": -77.91822052001953,
      "activations/layer4_attention_weight_max": 127.70668029785156,
      "activations/layer4_attention_weight_min": -126.55034637451172,
      "activations/layer5_attention_weight_max": 101.85305786132812,
      "activations/layer5_attention_weight_min": -99.4159164428711,
      "activations/layer6_attention_weight_max": 55.50546646118164,
      "activations/layer6_attention_weight_min": -52.79167175292969,
      "activations/layer7_attention_weight_max": 69.38780975341797,
      "activations/layer7_attention_weight_min": -71.94334411621094,
      "activations/layer8_attention_weight_max": 40.6308479309082,
      "activations/layer8_attention_weight_min": -41.67984390258789,
      "activations/layer9_attention_weight_max": 42.50400924682617,
      "activations/layer9_attention_weight_min": -38.47010040283203,
      "epoch": 3.16,
      "learning_rate": 0.00013093295454545453,
      "loss": 2.903,
      "step": 54400
    },
    {
      "activations/layer0_attention_weight_max": 17.399696350097656,
      "activations/layer0_attention_weight_min": -13.848596572875977,
      "activations/layer10_attention_weight_max": 35.467918395996094,
      "activations/layer10_attention_weight_min": -31.97087860107422,
      "activations/layer11_attention_weight_max": 33.94134521484375,
      "activations/layer11_attention_weight_min": -29.86840057373047,
      "activations/layer12_attention_weight_max": 28.157014846801758,
      "activations/layer12_attention_weight_min": -30.481990814208984,
      "activations/layer13_attention_weight_max": 48.659358978271484,
      "activations/layer13_attention_weight_min": -35.39802551269531,
      "activations/layer14_attention_weight_max": 72.48756408691406,
      "activations/layer14_attention_weight_min": -59.782623291015625,
      "activations/layer15_attention_weight_max": 49.73779296875,
      "activations/layer15_attention_weight_min": -32.86838912963867,
      "activations/layer16_attention_weight_max": 43.984588623046875,
      "activations/layer16_attention_weight_min": -32.46550750732422,
      "activations/layer17_attention_weight_max": 45.93357467651367,
      "activations/layer17_attention_weight_min": -26.498443603515625,
      "activations/layer18_attention_weight_max": 30.922828674316406,
      "activations/layer18_attention_weight_min": -20.206974029541016,
      "activations/layer19_attention_weight_max": 46.73954772949219,
      "activations/layer19_attention_weight_min": -31.801721572875977,
      "activations/layer1_attention_weight_max": 18.068422317504883,
      "activations/layer1_attention_weight_min": -15.277654647827148,
      "activations/layer20_attention_weight_max": 34.21498489379883,
      "activations/layer20_attention_weight_min": -23.355388641357422,
      "activations/layer21_attention_weight_max": 30.766904830932617,
      "activations/layer21_attention_weight_min": -19.465085983276367,
      "activations/layer22_attention_weight_max": 50.04602813720703,
      "activations/layer22_attention_weight_min": -26.925025939941406,
      "activations/layer23_attention_weight_max": 27.499792098999023,
      "activations/layer23_attention_weight_min": -16.366886138916016,
      "activations/layer2_attention_weight_max": 29.566146850585938,
      "activations/layer2_attention_weight_min": -29.587520599365234,
      "activations/layer3_attention_weight_max": 76.49964141845703,
      "activations/layer3_attention_weight_min": -80.03905487060547,
      "activations/layer4_attention_weight_max": 120.26268768310547,
      "activations/layer4_attention_weight_min": -117.4758071899414,
      "activations/layer5_attention_weight_max": 95.61256408691406,
      "activations/layer5_attention_weight_min": -86.08651733398438,
      "activations/layer6_attention_weight_max": 55.627017974853516,
      "activations/layer6_attention_weight_min": -53.40660858154297,
      "activations/layer7_attention_weight_max": 69.41341400146484,
      "activations/layer7_attention_weight_min": -66.61161804199219,
      "activations/layer8_attention_weight_max": 37.527103424072266,
      "activations/layer8_attention_weight_min": -35.26445770263672,
      "activations/layer9_attention_weight_max": 50.30739974975586,
      "activations/layer9_attention_weight_min": -37.48365020751953,
      "epoch": 3.16,
      "learning_rate": 0.00013091401515151515,
      "loss": 2.9051,
      "step": 54450
    },
    {
      "activations/layer0_attention_weight_max": 17.607206344604492,
      "activations/layer0_attention_weight_min": -11.49358081817627,
      "activations/layer10_attention_weight_max": 51.12910079956055,
      "activations/layer10_attention_weight_min": -37.317359924316406,
      "activations/layer11_attention_weight_max": 45.156795501708984,
      "activations/layer11_attention_weight_min": -28.5697078704834,
      "activations/layer12_attention_weight_max": 38.39128112792969,
      "activations/layer12_attention_weight_min": -31.159358978271484,
      "activations/layer13_attention_weight_max": 57.926734924316406,
      "activations/layer13_attention_weight_min": -37.65212631225586,
      "activations/layer14_attention_weight_max": 91.23236083984375,
      "activations/layer14_attention_weight_min": -56.06423568725586,
      "activations/layer15_attention_weight_max": 64.18338012695312,
      "activations/layer15_attention_weight_min": -35.530494689941406,
      "activations/layer16_attention_weight_max": 52.67202377319336,
      "activations/layer16_attention_weight_min": -31.503751754760742,
      "activations/layer17_attention_weight_max": 51.44594955444336,
      "activations/layer17_attention_weight_min": -26.034870147705078,
      "activations/layer18_attention_weight_max": 34.228031158447266,
      "activations/layer18_attention_weight_min": -18.920101165771484,
      "activations/layer19_attention_weight_max": 55.19981002807617,
      "activations/layer19_attention_weight_min": -28.636871337890625,
      "activations/layer1_attention_weight_max": 18.088207244873047,
      "activations/layer1_attention_weight_min": -14.18979263305664,
      "activations/layer20_attention_weight_max": 41.26685333251953,
      "activations/layer20_attention_weight_min": -22.84084129333496,
      "activations/layer21_attention_weight_max": 39.41754150390625,
      "activations/layer21_attention_weight_min": -19.96099853515625,
      "activations/layer22_attention_weight_max": 55.18729782104492,
      "activations/layer22_attention_weight_min": -25.915931701660156,
      "activations/layer23_attention_weight_max": 30.397926330566406,
      "activations/layer23_attention_weight_min": -16.60729217529297,
      "activations/layer2_attention_weight_max": 30.01905059814453,
      "activations/layer2_attention_weight_min": -31.298648834228516,
      "activations/layer3_attention_weight_max": 83.88096618652344,
      "activations/layer3_attention_weight_min": -86.16661834716797,
      "activations/layer4_attention_weight_max": 136.27352905273438,
      "activations/layer4_attention_weight_min": -137.77822875976562,
      "activations/layer5_attention_weight_max": 111.00018310546875,
      "activations/layer5_attention_weight_min": -106.83078002929688,
      "activations/layer6_attention_weight_max": 61.64313507080078,
      "activations/layer6_attention_weight_min": -61.24037170410156,
      "activations/layer7_attention_weight_max": 82.0587387084961,
      "activations/layer7_attention_weight_min": -77.43843078613281,
      "activations/layer8_attention_weight_max": 47.94917678833008,
      "activations/layer8_attention_weight_min": -38.99248123168945,
      "activations/layer9_attention_weight_max": 59.02959060668945,
      "activations/layer9_attention_weight_min": -47.46393966674805,
      "epoch": 3.17,
      "learning_rate": 0.00013089507575757574,
      "loss": 2.9044,
      "step": 54500
    },
    {
      "activations/layer0_attention_weight_max": 17.580562591552734,
      "activations/layer0_attention_weight_min": -13.090407371520996,
      "activations/layer10_attention_weight_max": 38.23698043823242,
      "activations/layer10_attention_weight_min": -31.547542572021484,
      "activations/layer11_attention_weight_max": 37.315513610839844,
      "activations/layer11_attention_weight_min": -28.428260803222656,
      "activations/layer12_attention_weight_max": 30.978300094604492,
      "activations/layer12_attention_weight_min": -28.836828231811523,
      "activations/layer13_attention_weight_max": 46.48081970214844,
      "activations/layer13_attention_weight_min": -35.36671447753906,
      "activations/layer14_attention_weight_max": 62.48867416381836,
      "activations/layer14_attention_weight_min": -50.97685241699219,
      "activations/layer15_attention_weight_max": 49.2730598449707,
      "activations/layer15_attention_weight_min": -29.734907150268555,
      "activations/layer16_attention_weight_max": 39.885589599609375,
      "activations/layer16_attention_weight_min": -30.320877075195312,
      "activations/layer17_attention_weight_max": 40.0885124206543,
      "activations/layer17_attention_weight_min": -25.64190673828125,
      "activations/layer18_attention_weight_max": 27.766963958740234,
      "activations/layer18_attention_weight_min": -19.818185806274414,
      "activations/layer19_attention_weight_max": 42.40963363647461,
      "activations/layer19_attention_weight_min": -31.260873794555664,
      "activations/layer1_attention_weight_max": 17.62174415588379,
      "activations/layer1_attention_weight_min": -14.173994064331055,
      "activations/layer20_attention_weight_max": 33.29165267944336,
      "activations/layer20_attention_weight_min": -24.312740325927734,
      "activations/layer21_attention_weight_max": 35.1607551574707,
      "activations/layer21_attention_weight_min": -19.818775177001953,
      "activations/layer22_attention_weight_max": 46.47401428222656,
      "activations/layer22_attention_weight_min": -27.259809494018555,
      "activations/layer23_attention_weight_max": 27.80107879638672,
      "activations/layer23_attention_weight_min": -19.367053985595703,
      "activations/layer2_attention_weight_max": 29.411237716674805,
      "activations/layer2_attention_weight_min": -28.306926727294922,
      "activations/layer3_attention_weight_max": 76.419921875,
      "activations/layer3_attention_weight_min": -78.78997802734375,
      "activations/layer4_attention_weight_max": 123.6793212890625,
      "activations/layer4_attention_weight_min": -123.140625,
      "activations/layer5_attention_weight_max": 95.07421112060547,
      "activations/layer5_attention_weight_min": -90.07160949707031,
      "activations/layer6_attention_weight_max": 56.98029327392578,
      "activations/layer6_attention_weight_min": -53.64236831665039,
      "activations/layer7_attention_weight_max": 69.75911712646484,
      "activations/layer7_attention_weight_min": -71.95223999023438,
      "activations/layer8_attention_weight_max": 38.07518768310547,
      "activations/layer8_attention_weight_min": -38.04742431640625,
      "activations/layer9_attention_weight_max": 44.015235900878906,
      "activations/layer9_attention_weight_min": -40.3234977722168,
      "epoch": 3.17,
      "learning_rate": 0.00013087613636363635,
      "loss": 2.8985,
      "step": 54550
    },
    {
      "activations/layer0_attention_weight_max": 17.21369743347168,
      "activations/layer0_attention_weight_min": -12.874252319335938,
      "activations/layer10_attention_weight_max": 46.59834671020508,
      "activations/layer10_attention_weight_min": -34.2797737121582,
      "activations/layer11_attention_weight_max": 47.37126159667969,
      "activations/layer11_attention_weight_min": -32.402469635009766,
      "activations/layer12_attention_weight_max": 41.70543670654297,
      "activations/layer12_attention_weight_min": -29.384281158447266,
      "activations/layer13_attention_weight_max": 55.18880081176758,
      "activations/layer13_attention_weight_min": -48.25063705444336,
      "activations/layer14_attention_weight_max": 79.4622573852539,
      "activations/layer14_attention_weight_min": -58.661773681640625,
      "activations/layer15_attention_weight_max": 51.06035232543945,
      "activations/layer15_attention_weight_min": -36.28003692626953,
      "activations/layer16_attention_weight_max": 46.1075439453125,
      "activations/layer16_attention_weight_min": -31.217411041259766,
      "activations/layer17_attention_weight_max": 44.090309143066406,
      "activations/layer17_attention_weight_min": -23.624433517456055,
      "activations/layer18_attention_weight_max": 24.221702575683594,
      "activations/layer18_attention_weight_min": -18.635700225830078,
      "activations/layer19_attention_weight_max": 43.56778335571289,
      "activations/layer19_attention_weight_min": -29.16328239440918,
      "activations/layer1_attention_weight_max": 17.858869552612305,
      "activations/layer1_attention_weight_min": -12.994209289550781,
      "activations/layer20_attention_weight_max": 34.319820404052734,
      "activations/layer20_attention_weight_min": -23.685009002685547,
      "activations/layer21_attention_weight_max": 28.788644790649414,
      "activations/layer21_attention_weight_min": -19.115825653076172,
      "activations/layer22_attention_weight_max": 44.59230041503906,
      "activations/layer22_attention_weight_min": -26.364398956298828,
      "activations/layer23_attention_weight_max": 26.311931610107422,
      "activations/layer23_attention_weight_min": -16.563228607177734,
      "activations/layer2_attention_weight_max": 29.69642448425293,
      "activations/layer2_attention_weight_min": -30.78472137451172,
      "activations/layer3_attention_weight_max": 81.11032104492188,
      "activations/layer3_attention_weight_min": -89.27862548828125,
      "activations/layer4_attention_weight_max": 142.5242156982422,
      "activations/layer4_attention_weight_min": -130.64639282226562,
      "activations/layer5_attention_weight_max": 104.609375,
      "activations/layer5_attention_weight_min": -89.95098876953125,
      "activations/layer6_attention_weight_max": 61.224327087402344,
      "activations/layer6_attention_weight_min": -55.27947998046875,
      "activations/layer7_attention_weight_max": 81.46227264404297,
      "activations/layer7_attention_weight_min": -76.1749496459961,
      "activations/layer8_attention_weight_max": 43.77883529663086,
      "activations/layer8_attention_weight_min": -38.09208679199219,
      "activations/layer9_attention_weight_max": 60.275001525878906,
      "activations/layer9_attention_weight_min": -42.18886184692383,
      "epoch": 3.17,
      "learning_rate": 0.00013085719696969697,
      "loss": 2.895,
      "step": 54600
    },
    {
      "activations/layer0_attention_weight_max": 17.696001052856445,
      "activations/layer0_attention_weight_min": -11.112541198730469,
      "activations/layer10_attention_weight_max": 47.05118942260742,
      "activations/layer10_attention_weight_min": -36.402923583984375,
      "activations/layer11_attention_weight_max": 38.344242095947266,
      "activations/layer11_attention_weight_min": -29.693281173706055,
      "activations/layer12_attention_weight_max": 34.45330810546875,
      "activations/layer12_attention_weight_min": -29.469221115112305,
      "activations/layer13_attention_weight_max": 48.51914978027344,
      "activations/layer13_attention_weight_min": -33.76988220214844,
      "activations/layer14_attention_weight_max": 71.74958038330078,
      "activations/layer14_attention_weight_min": -49.291725158691406,
      "activations/layer15_attention_weight_max": 58.49521255493164,
      "activations/layer15_attention_weight_min": -34.446937561035156,
      "activations/layer16_attention_weight_max": 42.9662971496582,
      "activations/layer16_attention_weight_min": -30.37626075744629,
      "activations/layer17_attention_weight_max": 45.718231201171875,
      "activations/layer17_attention_weight_min": -26.96941375732422,
      "activations/layer18_attention_weight_max": 30.960073471069336,
      "activations/layer18_attention_weight_min": -19.18832015991211,
      "activations/layer19_attention_weight_max": 44.77999496459961,
      "activations/layer19_attention_weight_min": -31.207433700561523,
      "activations/layer1_attention_weight_max": 17.43915557861328,
      "activations/layer1_attention_weight_min": -12.52635669708252,
      "activations/layer20_attention_weight_max": 31.75370979309082,
      "activations/layer20_attention_weight_min": -24.37590789794922,
      "activations/layer21_attention_weight_max": 30.46923065185547,
      "activations/layer21_attention_weight_min": -19.333925247192383,
      "activations/layer22_attention_weight_max": 45.47614288330078,
      "activations/layer22_attention_weight_min": -27.283899307250977,
      "activations/layer23_attention_weight_max": 27.5966796875,
      "activations/layer23_attention_weight_min": -18.98174476623535,
      "activations/layer2_attention_weight_max": 29.24885368347168,
      "activations/layer2_attention_weight_min": -28.3339900970459,
      "activations/layer3_attention_weight_max": 79.58114624023438,
      "activations/layer3_attention_weight_min": -82.25508117675781,
      "activations/layer4_attention_weight_max": 132.48651123046875,
      "activations/layer4_attention_weight_min": -125.8321762084961,
      "activations/layer5_attention_weight_max": 98.9316635131836,
      "activations/layer5_attention_weight_min": -92.00145721435547,
      "activations/layer6_attention_weight_max": 54.95204162597656,
      "activations/layer6_attention_weight_min": -49.76431655883789,
      "activations/layer7_attention_weight_max": 68.90855407714844,
      "activations/layer7_attention_weight_min": -70.79103088378906,
      "activations/layer8_attention_weight_max": 40.39460372924805,
      "activations/layer8_attention_weight_min": -39.343589782714844,
      "activations/layer9_attention_weight_max": 47.988136291503906,
      "activations/layer9_attention_weight_min": -36.35451126098633,
      "epoch": 3.18,
      "learning_rate": 0.00013083825757575755,
      "loss": 2.8943,
      "step": 54650
    },
    {
      "activations/layer0_attention_weight_max": 15.740144729614258,
      "activations/layer0_attention_weight_min": -11.66149616241455,
      "activations/layer10_attention_weight_max": 42.90304946899414,
      "activations/layer10_attention_weight_min": -32.778846740722656,
      "activations/layer11_attention_weight_max": 38.07044982910156,
      "activations/layer11_attention_weight_min": -30.224468231201172,
      "activations/layer12_attention_weight_max": 41.10254669189453,
      "activations/layer12_attention_weight_min": -26.555723190307617,
      "activations/layer13_attention_weight_max": 52.473045349121094,
      "activations/layer13_attention_weight_min": -36.099098205566406,
      "activations/layer14_attention_weight_max": 91.8954849243164,
      "activations/layer14_attention_weight_min": -63.232444763183594,
      "activations/layer15_attention_weight_max": 52.180545806884766,
      "activations/layer15_attention_weight_min": -30.649686813354492,
      "activations/layer16_attention_weight_max": 46.79568862915039,
      "activations/layer16_attention_weight_min": -32.57554626464844,
      "activations/layer17_attention_weight_max": 49.567989349365234,
      "activations/layer17_attention_weight_min": -26.136756896972656,
      "activations/layer18_attention_weight_max": 31.278827667236328,
      "activations/layer18_attention_weight_min": -18.210399627685547,
      "activations/layer19_attention_weight_max": 51.90181350708008,
      "activations/layer19_attention_weight_min": -31.68232536315918,
      "activations/layer1_attention_weight_max": 17.98236083984375,
      "activations/layer1_attention_weight_min": -13.343368530273438,
      "activations/layer20_attention_weight_max": 44.81138229370117,
      "activations/layer20_attention_weight_min": -23.28209114074707,
      "activations/layer21_attention_weight_max": 33.10358428955078,
      "activations/layer21_attention_weight_min": -22.997421264648438,
      "activations/layer22_attention_weight_max": 57.59084701538086,
      "activations/layer22_attention_weight_min": -27.518842697143555,
      "activations/layer23_attention_weight_max": 33.39067077636719,
      "activations/layer23_attention_weight_min": -17.310522079467773,
      "activations/layer2_attention_weight_max": 29.11408042907715,
      "activations/layer2_attention_weight_min": -28.782424926757812,
      "activations/layer3_attention_weight_max": 80.20597839355469,
      "activations/layer3_attention_weight_min": -83.65787506103516,
      "activations/layer4_attention_weight_max": 131.1709442138672,
      "activations/layer4_attention_weight_min": -124.34366607666016,
      "activations/layer5_attention_weight_max": 104.6419906616211,
      "activations/layer5_attention_weight_min": -89.73149871826172,
      "activations/layer6_attention_weight_max": 58.86137771606445,
      "activations/layer6_attention_weight_min": -52.98991012573242,
      "activations/layer7_attention_weight_max": 76.37469482421875,
      "activations/layer7_attention_weight_min": -71.30675506591797,
      "activations/layer8_attention_weight_max": 42.06978225708008,
      "activations/layer8_attention_weight_min": -36.84618377685547,
      "activations/layer9_attention_weight_max": 51.01258087158203,
      "activations/layer9_attention_weight_min": -38.73094940185547,
      "epoch": 3.18,
      "learning_rate": 0.00013081931818181817,
      "loss": 2.9051,
      "step": 54700
    },
    {
      "activations/layer0_attention_weight_max": 17.023399353027344,
      "activations/layer0_attention_weight_min": -12.001739501953125,
      "activations/layer10_attention_weight_max": 42.070762634277344,
      "activations/layer10_attention_weight_min": -33.772705078125,
      "activations/layer11_attention_weight_max": 52.16302490234375,
      "activations/layer11_attention_weight_min": -33.41790771484375,
      "activations/layer12_attention_weight_max": 35.08583450317383,
      "activations/layer12_attention_weight_min": -25.0693359375,
      "activations/layer13_attention_weight_max": 71.11930084228516,
      "activations/layer13_attention_weight_min": -40.45119094848633,
      "activations/layer14_attention_weight_max": 84.07206726074219,
      "activations/layer14_attention_weight_min": -51.132205963134766,
      "activations/layer15_attention_weight_max": 59.391502380371094,
      "activations/layer15_attention_weight_min": -36.11327362060547,
      "activations/layer16_attention_weight_max": 44.986915588378906,
      "activations/layer16_attention_weight_min": -31.82036590576172,
      "activations/layer17_attention_weight_max": 41.9705696105957,
      "activations/layer17_attention_weight_min": -28.453927993774414,
      "activations/layer18_attention_weight_max": 38.17992401123047,
      "activations/layer18_attention_weight_min": -21.8514461517334,
      "activations/layer19_attention_weight_max": 42.34672927856445,
      "activations/layer19_attention_weight_min": -30.4408016204834,
      "activations/layer1_attention_weight_max": 17.617605209350586,
      "activations/layer1_attention_weight_min": -17.566606521606445,
      "activations/layer20_attention_weight_max": 35.995235443115234,
      "activations/layer20_attention_weight_min": -22.67127799987793,
      "activations/layer21_attention_weight_max": 30.03192710876465,
      "activations/layer21_attention_weight_min": -17.540971755981445,
      "activations/layer22_attention_weight_max": 45.081851959228516,
      "activations/layer22_attention_weight_min": -29.5535945892334,
      "activations/layer23_attention_weight_max": 28.8671932220459,
      "activations/layer23_attention_weight_min": -18.159278869628906,
      "activations/layer2_attention_weight_max": 31.737777709960938,
      "activations/layer2_attention_weight_min": -31.608837127685547,
      "activations/layer3_attention_weight_max": 86.6011734008789,
      "activations/layer3_attention_weight_min": -85.56927490234375,
      "activations/layer4_attention_weight_max": 134.09254455566406,
      "activations/layer4_attention_weight_min": -127.73846435546875,
      "activations/layer5_attention_weight_max": 105.53072357177734,
      "activations/layer5_attention_weight_min": -96.3277816772461,
      "activations/layer6_attention_weight_max": 62.29228591918945,
      "activations/layer6_attention_weight_min": -56.410308837890625,
      "activations/layer7_attention_weight_max": 77.02010345458984,
      "activations/layer7_attention_weight_min": -74.11373138427734,
      "activations/layer8_attention_weight_max": 45.64152908325195,
      "activations/layer8_attention_weight_min": -38.340545654296875,
      "activations/layer9_attention_weight_max": 51.55411911010742,
      "activations/layer9_attention_weight_min": -43.84864044189453,
      "epoch": 3.18,
      "learning_rate": 0.00013080037878787878,
      "loss": 2.8912,
      "step": 54750
    },
    {
      "activations/layer0_attention_weight_max": 17.631755828857422,
      "activations/layer0_attention_weight_min": -11.799827575683594,
      "activations/layer10_attention_weight_max": 45.39464569091797,
      "activations/layer10_attention_weight_min": -33.31866455078125,
      "activations/layer11_attention_weight_max": 46.988563537597656,
      "activations/layer11_attention_weight_min": -31.96091079711914,
      "activations/layer12_attention_weight_max": 36.95417404174805,
      "activations/layer12_attention_weight_min": -29.529800415039062,
      "activations/layer13_attention_weight_max": 65.31327056884766,
      "activations/layer13_attention_weight_min": -36.39889144897461,
      "activations/layer14_attention_weight_max": 91.2960205078125,
      "activations/layer14_attention_weight_min": -53.83992004394531,
      "activations/layer15_attention_weight_max": 59.618370056152344,
      "activations/layer15_attention_weight_min": -34.975563049316406,
      "activations/layer16_attention_weight_max": 51.529823303222656,
      "activations/layer16_attention_weight_min": -30.788650512695312,
      "activations/layer17_attention_weight_max": 49.81352233886719,
      "activations/layer17_attention_weight_min": -26.190244674682617,
      "activations/layer18_attention_weight_max": 33.85101318359375,
      "activations/layer18_attention_weight_min": -18.793859481811523,
      "activations/layer19_attention_weight_max": 50.3538703918457,
      "activations/layer19_attention_weight_min": -29.438756942749023,
      "activations/layer1_attention_weight_max": 16.90001678466797,
      "activations/layer1_attention_weight_min": -17.975547790527344,
      "activations/layer20_attention_weight_max": 40.6135368347168,
      "activations/layer20_attention_weight_min": -26.75197982788086,
      "activations/layer21_attention_weight_max": 30.821590423583984,
      "activations/layer21_attention_weight_min": -19.765995025634766,
      "activations/layer22_attention_weight_max": 61.54854965209961,
      "activations/layer22_attention_weight_min": -31.91501235961914,
      "activations/layer23_attention_weight_max": 27.599510192871094,
      "activations/layer23_attention_weight_min": -21.216873168945312,
      "activations/layer2_attention_weight_max": 31.339130401611328,
      "activations/layer2_attention_weight_min": -29.85567283630371,
      "activations/layer3_attention_weight_max": 82.55168151855469,
      "activations/layer3_attention_weight_min": -82.29368591308594,
      "activations/layer4_attention_weight_max": 142.89149475097656,
      "activations/layer4_attention_weight_min": -133.661865234375,
      "activations/layer5_attention_weight_max": 112.52822875976562,
      "activations/layer5_attention_weight_min": -93.3573226928711,
      "activations/layer6_attention_weight_max": 63.29376983642578,
      "activations/layer6_attention_weight_min": -52.620399475097656,
      "activations/layer7_attention_weight_max": 81.37272644042969,
      "activations/layer7_attention_weight_min": -75.60717010498047,
      "activations/layer8_attention_weight_max": 49.21660232543945,
      "activations/layer8_attention_weight_min": -38.70331573486328,
      "activations/layer9_attention_weight_max": 71.91325378417969,
      "activations/layer9_attention_weight_min": -50.700347900390625,
      "epoch": 3.18,
      "learning_rate": 0.00013078143939393937,
      "loss": 2.9008,
      "step": 54800
    },
    {
      "activations/layer0_attention_weight_max": 15.299665451049805,
      "activations/layer0_attention_weight_min": -13.401240348815918,
      "activations/layer10_attention_weight_max": 42.511436462402344,
      "activations/layer10_attention_weight_min": -35.017704010009766,
      "activations/layer11_attention_weight_max": 37.78639221191406,
      "activations/layer11_attention_weight_min": -28.93704605102539,
      "activations/layer12_attention_weight_max": 54.8488655090332,
      "activations/layer12_attention_weight_min": -31.477033615112305,
      "activations/layer13_attention_weight_max": 54.70155334472656,
      "activations/layer13_attention_weight_min": -39.07656478881836,
      "activations/layer14_attention_weight_max": 78.91435241699219,
      "activations/layer14_attention_weight_min": -58.951576232910156,
      "activations/layer15_attention_weight_max": 50.2862548828125,
      "activations/layer15_attention_weight_min": -34.56891632080078,
      "activations/layer16_attention_weight_max": 42.12329864501953,
      "activations/layer16_attention_weight_min": -32.425167083740234,
      "activations/layer17_attention_weight_max": 41.27650833129883,
      "activations/layer17_attention_weight_min": -28.337772369384766,
      "activations/layer18_attention_weight_max": 31.138395309448242,
      "activations/layer18_attention_weight_min": -21.206378936767578,
      "activations/layer19_attention_weight_max": 45.077152252197266,
      "activations/layer19_attention_weight_min": -31.909122467041016,
      "activations/layer1_attention_weight_max": 18.12519645690918,
      "activations/layer1_attention_weight_min": -16.230255126953125,
      "activations/layer20_attention_weight_max": 35.020103454589844,
      "activations/layer20_attention_weight_min": -26.635723114013672,
      "activations/layer21_attention_weight_max": 34.228939056396484,
      "activations/layer21_attention_weight_min": -26.129863739013672,
      "activations/layer22_attention_weight_max": 46.95032501220703,
      "activations/layer22_attention_weight_min": -32.213497161865234,
      "activations/layer23_attention_weight_max": 26.38903045654297,
      "activations/layer23_attention_weight_min": -18.157827377319336,
      "activations/layer2_attention_weight_max": 30.548368453979492,
      "activations/layer2_attention_weight_min": -30.379962921142578,
      "activations/layer3_attention_weight_max": 83.62422943115234,
      "activations/layer3_attention_weight_min": -83.68304443359375,
      "activations/layer4_attention_weight_max": 131.58859252929688,
      "activations/layer4_attention_weight_min": -122.91175842285156,
      "activations/layer5_attention_weight_max": 98.39981079101562,
      "activations/layer5_attention_weight_min": -87.41763305664062,
      "activations/layer6_attention_weight_max": 58.41389846801758,
      "activations/layer6_attention_weight_min": -52.02675247192383,
      "activations/layer7_attention_weight_max": 71.3302993774414,
      "activations/layer7_attention_weight_min": -70.37850952148438,
      "activations/layer8_attention_weight_max": 38.9293212890625,
      "activations/layer8_attention_weight_min": -36.193580627441406,
      "activations/layer9_attention_weight_max": 49.996639251708984,
      "activations/layer9_attention_weight_min": -40.351097106933594,
      "epoch": 3.19,
      "learning_rate": 0.0001307625,
      "loss": 2.8955,
      "step": 54850
    },
    {
      "activations/layer0_attention_weight_max": 17.132713317871094,
      "activations/layer0_attention_weight_min": -12.158061027526855,
      "activations/layer10_attention_weight_max": 37.781307220458984,
      "activations/layer10_attention_weight_min": -32.538536071777344,
      "activations/layer11_attention_weight_max": 36.70243453979492,
      "activations/layer11_attention_weight_min": -29.827543258666992,
      "activations/layer12_attention_weight_max": 39.2281494140625,
      "activations/layer12_attention_weight_min": -28.318315505981445,
      "activations/layer13_attention_weight_max": 66.41925811767578,
      "activations/layer13_attention_weight_min": -47.11592102050781,
      "activations/layer14_attention_weight_max": 78.61761474609375,
      "activations/layer14_attention_weight_min": -57.79182434082031,
      "activations/layer15_attention_weight_max": 62.28711700439453,
      "activations/layer15_attention_weight_min": -41.90199279785156,
      "activations/layer16_attention_weight_max": 48.47760772705078,
      "activations/layer16_attention_weight_min": -32.1933708190918,
      "activations/layer17_attention_weight_max": 48.42610549926758,
      "activations/layer17_attention_weight_min": -28.130538940429688,
      "activations/layer18_attention_weight_max": 30.93902587890625,
      "activations/layer18_attention_weight_min": -22.701120376586914,
      "activations/layer19_attention_weight_max": 45.712860107421875,
      "activations/layer19_attention_weight_min": -30.151565551757812,
      "activations/layer1_attention_weight_max": 17.4454288482666,
      "activations/layer1_attention_weight_min": -14.139947891235352,
      "activations/layer20_attention_weight_max": 38.51860809326172,
      "activations/layer20_attention_weight_min": -23.1501522064209,
      "activations/layer21_attention_weight_max": 32.336544036865234,
      "activations/layer21_attention_weight_min": -21.334558486938477,
      "activations/layer22_attention_weight_max": 49.67374801635742,
      "activations/layer22_attention_weight_min": -25.427642822265625,
      "activations/layer23_attention_weight_max": 30.4223690032959,
      "activations/layer23_attention_weight_min": -18.926300048828125,
      "activations/layer2_attention_weight_max": 31.04254150390625,
      "activations/layer2_attention_weight_min": -30.808223724365234,
      "activations/layer3_attention_weight_max": 81.30750274658203,
      "activations/layer3_attention_weight_min": -81.9013900756836,
      "activations/layer4_attention_weight_max": 134.55491638183594,
      "activations/layer4_attention_weight_min": -120.81401824951172,
      "activations/layer5_attention_weight_max": 101.24922943115234,
      "activations/layer5_attention_weight_min": -87.6295166015625,
      "activations/layer6_attention_weight_max": 55.75084686279297,
      "activations/layer6_attention_weight_min": -52.510093688964844,
      "activations/layer7_attention_weight_max": 76.01729583740234,
      "activations/layer7_attention_weight_min": -68.85028076171875,
      "activations/layer8_attention_weight_max": 40.25716018676758,
      "activations/layer8_attention_weight_min": -34.81641387939453,
      "activations/layer9_attention_weight_max": 47.276580810546875,
      "activations/layer9_attention_weight_min": -45.408504486083984,
      "epoch": 3.19,
      "learning_rate": 0.00013074356060606057,
      "loss": 2.8985,
      "step": 54900
    },
    {
      "activations/layer0_attention_weight_max": 15.852433204650879,
      "activations/layer0_attention_weight_min": -11.97529125213623,
      "activations/layer10_attention_weight_max": 41.99821853637695,
      "activations/layer10_attention_weight_min": -31.825115203857422,
      "activations/layer11_attention_weight_max": 34.87691116333008,
      "activations/layer11_attention_weight_min": -30.660289764404297,
      "activations/layer12_attention_weight_max": 34.4791259765625,
      "activations/layer12_attention_weight_min": -24.49686050415039,
      "activations/layer13_attention_weight_max": 45.64885711669922,
      "activations/layer13_attention_weight_min": -33.41844177246094,
      "activations/layer14_attention_weight_max": 81.87915802001953,
      "activations/layer14_attention_weight_min": -63.924625396728516,
      "activations/layer15_attention_weight_max": 44.51622009277344,
      "activations/layer15_attention_weight_min": -31.156015396118164,
      "activations/layer16_attention_weight_max": 53.942359924316406,
      "activations/layer16_attention_weight_min": -32.198570251464844,
      "activations/layer17_attention_weight_max": 45.457130432128906,
      "activations/layer17_attention_weight_min": -28.968217849731445,
      "activations/layer18_attention_weight_max": 27.954484939575195,
      "activations/layer18_attention_weight_min": -18.45075225830078,
      "activations/layer19_attention_weight_max": 43.786373138427734,
      "activations/layer19_attention_weight_min": -27.96196937561035,
      "activations/layer1_attention_weight_max": 18.361343383789062,
      "activations/layer1_attention_weight_min": -13.495887756347656,
      "activations/layer20_attention_weight_max": 31.249422073364258,
      "activations/layer20_attention_weight_min": -23.14039421081543,
      "activations/layer21_attention_weight_max": 30.39077377319336,
      "activations/layer21_attention_weight_min": -22.247549057006836,
      "activations/layer22_attention_weight_max": 45.00012969970703,
      "activations/layer22_attention_weight_min": -32.59470748901367,
      "activations/layer23_attention_weight_max": 30.223037719726562,
      "activations/layer23_attention_weight_min": -18.798343658447266,
      "activations/layer2_attention_weight_max": 29.759052276611328,
      "activations/layer2_attention_weight_min": -28.986595153808594,
      "activations/layer3_attention_weight_max": 83.14120483398438,
      "activations/layer3_attention_weight_min": -82.65911865234375,
      "activations/layer4_attention_weight_max": 129.6779022216797,
      "activations/layer4_attention_weight_min": -124.971923828125,
      "activations/layer5_attention_weight_max": 98.52578735351562,
      "activations/layer5_attention_weight_min": -91.62837982177734,
      "activations/layer6_attention_weight_max": 56.7058219909668,
      "activations/layer6_attention_weight_min": -56.64900588989258,
      "activations/layer7_attention_weight_max": 74.380615234375,
      "activations/layer7_attention_weight_min": -73.92748260498047,
      "activations/layer8_attention_weight_max": 43.244991302490234,
      "activations/layer8_attention_weight_min": -38.302513122558594,
      "activations/layer9_attention_weight_max": 48.14907455444336,
      "activations/layer9_attention_weight_min": -38.616146087646484,
      "epoch": 3.19,
      "learning_rate": 0.0001307246212121212,
      "loss": 2.8991,
      "step": 54950
    },
    {
      "activations/layer0_attention_weight_max": 17.306652069091797,
      "activations/layer0_attention_weight_min": -11.526091575622559,
      "activations/layer10_attention_weight_max": 39.80814743041992,
      "activations/layer10_attention_weight_min": -32.029815673828125,
      "activations/layer11_attention_weight_max": 34.59832000732422,
      "activations/layer11_attention_weight_min": -27.634634017944336,
      "activations/layer12_attention_weight_max": 52.10506820678711,
      "activations/layer12_attention_weight_min": -31.977272033691406,
      "activations/layer13_attention_weight_max": 50.65890884399414,
      "activations/layer13_attention_weight_min": -36.465938568115234,
      "activations/layer14_attention_weight_max": 78.97576141357422,
      "activations/layer14_attention_weight_min": -63.11445617675781,
      "activations/layer15_attention_weight_max": 47.17963790893555,
      "activations/layer15_attention_weight_min": -32.7491455078125,
      "activations/layer16_attention_weight_max": 45.75032424926758,
      "activations/layer16_attention_weight_min": -31.344133377075195,
      "activations/layer17_attention_weight_max": 45.206085205078125,
      "activations/layer17_attention_weight_min": -27.509788513183594,
      "activations/layer18_attention_weight_max": 39.666744232177734,
      "activations/layer18_attention_weight_min": -21.373783111572266,
      "activations/layer19_attention_weight_max": 48.75214767456055,
      "activations/layer19_attention_weight_min": -29.131027221679688,
      "activations/layer1_attention_weight_max": 17.250431060791016,
      "activations/layer1_attention_weight_min": -13.490204811096191,
      "activations/layer20_attention_weight_max": 39.54138946533203,
      "activations/layer20_attention_weight_min": -23.040355682373047,
      "activations/layer21_attention_weight_max": 38.90434646606445,
      "activations/layer21_attention_weight_min": -19.891870498657227,
      "activations/layer22_attention_weight_max": 54.4340705871582,
      "activations/layer22_attention_weight_min": -25.79715919494629,
      "activations/layer23_attention_weight_max": 26.89044761657715,
      "activations/layer23_attention_weight_min": -18.098094940185547,
      "activations/layer2_attention_weight_max": 30.53510856628418,
      "activations/layer2_attention_weight_min": -30.83576011657715,
      "activations/layer3_attention_weight_max": 80.12128448486328,
      "activations/layer3_attention_weight_min": -83.40666198730469,
      "activations/layer4_attention_weight_max": 129.66146850585938,
      "activations/layer4_attention_weight_min": -127.13048553466797,
      "activations/layer5_attention_weight_max": 98.36519622802734,
      "activations/layer5_attention_weight_min": -93.35980224609375,
      "activations/layer6_attention_weight_max": 60.8769645690918,
      "activations/layer6_attention_weight_min": -54.75830841064453,
      "activations/layer7_attention_weight_max": 72.45501708984375,
      "activations/layer7_attention_weight_min": -70.26766204833984,
      "activations/layer8_attention_weight_max": 40.96598434448242,
      "activations/layer8_attention_weight_min": -37.90966796875,
      "activations/layer9_attention_weight_max": 47.428672790527344,
      "activations/layer9_attention_weight_min": -43.90433120727539,
      "epoch": 3.2,
      "learning_rate": 0.0001307056818181818,
      "loss": 2.9061,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_loss": 2.8515625,
      "eval_runtime": 8.561,
      "eval_samples_per_second": 501.575,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_openwebtext_loss": 2.8515625,
      "eval_openwebtext_ppl": 17.314815113938415,
      "eval_openwebtext_runtime": 8.561,
      "eval_openwebtext_samples_per_second": 501.575,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_wikitext_loss": 3.08203125,
      "eval_wikitext_ppl": 21.802644068801417,
      "eval_wikitext_runtime": 2.0248,
      "eval_wikitext_samples_per_second": 225.207,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_lambada_loss": 2.8984375,
      "eval_lambada_ppl": 18.145770440989054,
      "eval_lambada_runtime": 9.6021,
      "eval_lambada_samples_per_second": 507.077,
      "step": 55000
    },
    {
      "activations/layer0_attention_weight_max": 15.06994915008545,
      "activations/layer0_attention_weight_min": -13.608696937561035,
      "activations/layer10_attention_weight_max": 43.22467041015625,
      "activations/layer10_attention_weight_min": -35.02466583251953,
      "activations/layer11_attention_weight_max": 38.73065948486328,
      "activations/layer11_attention_weight_min": -29.77878189086914,
      "activations/layer12_attention_weight_max": 47.33666229248047,
      "activations/layer12_attention_weight_min": -28.535110473632812,
      "activations/layer13_attention_weight_max": 67.30094909667969,
      "activations/layer13_attention_weight_min": -43.80265426635742,
      "activations/layer14_attention_weight_max": 73.40296936035156,
      "activations/layer14_attention_weight_min": -45.982486724853516,
      "activations/layer15_attention_weight_max": 51.985862731933594,
      "activations/layer15_attention_weight_min": -31.984121322631836,
      "activations/layer16_attention_weight_max": 48.73164749145508,
      "activations/layer16_attention_weight_min": -29.041587829589844,
      "activations/layer17_attention_weight_max": 44.60487747192383,
      "activations/layer17_attention_weight_min": -24.789562225341797,
      "activations/layer18_attention_weight_max": 30.230607986450195,
      "activations/layer18_attention_weight_min": -19.233154296875,
      "activations/layer19_attention_weight_max": 47.21185302734375,
      "activations/layer19_attention_weight_min": -30.256864547729492,
      "activations/layer1_attention_weight_max": 16.82497215270996,
      "activations/layer1_attention_weight_min": -13.959410667419434,
      "activations/layer20_attention_weight_max": 36.864295959472656,
      "activations/layer20_attention_weight_min": -23.125709533691406,
      "activations/layer21_attention_weight_max": 36.1135368347168,
      "activations/layer21_attention_weight_min": -19.48065185546875,
      "activations/layer22_attention_weight_max": 51.85804748535156,
      "activations/layer22_attention_weight_min": -24.383319854736328,
      "activations/layer23_attention_weight_max": 30.883052825927734,
      "activations/layer23_attention_weight_min": -16.75550079345703,
      "activations/layer2_attention_weight_max": 28.68866539001465,
      "activations/layer2_attention_weight_min": -30.032394409179688,
      "activations/layer3_attention_weight_max": 77.9067153930664,
      "activations/layer3_attention_weight_min": -84.8067398071289,
      "activations/layer4_attention_weight_max": 124.8336410522461,
      "activations/layer4_attention_weight_min": -125.9744644165039,
      "activations/layer5_attention_weight_max": 99.77857971191406,
      "activations/layer5_attention_weight_min": -89.94326782226562,
      "activations/layer6_attention_weight_max": 55.239341735839844,
      "activations/layer6_attention_weight_min": -52.21919631958008,
      "activations/layer7_attention_weight_max": 71.53497314453125,
      "activations/layer7_attention_weight_min": -70.64808654785156,
      "activations/layer8_attention_weight_max": 39.917442321777344,
      "activations/layer8_attention_weight_min": -37.54478454589844,
      "activations/layer9_attention_weight_max": 51.69597244262695,
      "activations/layer9_attention_weight_min": -40.72113037109375,
      "epoch": 3.2,
      "learning_rate": 0.00013068674242424242,
      "loss": 2.918,
      "step": 55050
    },
    {
      "activations/layer0_attention_weight_max": 16.895023345947266,
      "activations/layer0_attention_weight_min": -11.521072387695312,
      "activations/layer10_attention_weight_max": 39.53076171875,
      "activations/layer10_attention_weight_min": -34.8016242980957,
      "activations/layer11_attention_weight_max": 35.61585235595703,
      "activations/layer11_attention_weight_min": -28.047874450683594,
      "activations/layer12_attention_weight_max": 31.055965423583984,
      "activations/layer12_attention_weight_min": -25.836261749267578,
      "activations/layer13_attention_weight_max": 43.84542465209961,
      "activations/layer13_attention_weight_min": -33.53095626831055,
      "activations/layer14_attention_weight_max": 78.53556823730469,
      "activations/layer14_attention_weight_min": -57.15509033203125,
      "activations/layer15_attention_weight_max": 40.52088165283203,
      "activations/layer15_attention_weight_min": -30.872922897338867,
      "activations/layer16_attention_weight_max": 38.22919845581055,
      "activations/layer16_attention_weight_min": -31.010160446166992,
      "activations/layer17_attention_weight_max": 40.01751708984375,
      "activations/layer17_attention_weight_min": -27.518518447875977,
      "activations/layer18_attention_weight_max": 30.570846557617188,
      "activations/layer18_attention_weight_min": -19.70577621459961,
      "activations/layer19_attention_weight_max": 39.63120651245117,
      "activations/layer19_attention_weight_min": -27.43851089477539,
      "activations/layer1_attention_weight_max": 16.87472152709961,
      "activations/layer1_attention_weight_min": -15.610737800598145,
      "activations/layer20_attention_weight_max": 29.24358367919922,
      "activations/layer20_attention_weight_min": -22.581871032714844,
      "activations/layer21_attention_weight_max": 31.848081588745117,
      "activations/layer21_attention_weight_min": -20.67531394958496,
      "activations/layer22_attention_weight_max": 41.77751922607422,
      "activations/layer22_attention_weight_min": -25.2890625,
      "activations/layer23_attention_weight_max": 26.814990997314453,
      "activations/layer23_attention_weight_min": -16.441162109375,
      "activations/layer2_attention_weight_max": 30.607872009277344,
      "activations/layer2_attention_weight_min": -30.30618667602539,
      "activations/layer3_attention_weight_max": 79.19132995605469,
      "activations/layer3_attention_weight_min": -83.53074645996094,
      "activations/layer4_attention_weight_max": 123.11394500732422,
      "activations/layer4_attention_weight_min": -120.94049072265625,
      "activations/layer5_attention_weight_max": 94.72528076171875,
      "activations/layer5_attention_weight_min": -90.58934783935547,
      "activations/layer6_attention_weight_max": 54.55781936645508,
      "activations/layer6_attention_weight_min": -54.10869598388672,
      "activations/layer7_attention_weight_max": 72.00270080566406,
      "activations/layer7_attention_weight_min": -68.88976287841797,
      "activations/layer8_attention_weight_max": 37.13404083251953,
      "activations/layer8_attention_weight_min": -35.51062774658203,
      "activations/layer9_attention_weight_max": 44.550926208496094,
      "activations/layer9_attention_weight_min": -38.899574279785156,
      "epoch": 3.2,
      "learning_rate": 0.000130667803030303,
      "loss": 2.8963,
      "step": 55100
    },
    {
      "activations/layer0_attention_weight_max": 17.283296585083008,
      "activations/layer0_attention_weight_min": -12.320294380187988,
      "activations/layer10_attention_weight_max": 43.43581008911133,
      "activations/layer10_attention_weight_min": -31.932653427124023,
      "activations/layer11_attention_weight_max": 38.300331115722656,
      "activations/layer11_attention_weight_min": -31.109561920166016,
      "activations/layer12_attention_weight_max": 42.36039352416992,
      "activations/layer12_attention_weight_min": -32.233741760253906,
      "activations/layer13_attention_weight_max": 55.3341064453125,
      "activations/layer13_attention_weight_min": -35.77801513671875,
      "activations/layer14_attention_weight_max": 70.04438781738281,
      "activations/layer14_attention_weight_min": -47.022212982177734,
      "activations/layer15_attention_weight_max": 50.54549789428711,
      "activations/layer15_attention_weight_min": -33.23917007446289,
      "activations/layer16_attention_weight_max": 45.874454498291016,
      "activations/layer16_attention_weight_min": -29.957975387573242,
      "activations/layer17_attention_weight_max": 47.58133316040039,
      "activations/layer17_attention_weight_min": -25.75107765197754,
      "activations/layer18_attention_weight_max": 34.830257415771484,
      "activations/layer18_attention_weight_min": -18.74977684020996,
      "activations/layer19_attention_weight_max": 48.74625778198242,
      "activations/layer19_attention_weight_min": -29.591169357299805,
      "activations/layer1_attention_weight_max": 19.426565170288086,
      "activations/layer1_attention_weight_min": -14.219632148742676,
      "activations/layer20_attention_weight_max": 41.16379165649414,
      "activations/layer20_attention_weight_min": -25.814311981201172,
      "activations/layer21_attention_weight_max": 34.13970947265625,
      "activations/layer21_attention_weight_min": -24.033885955810547,
      "activations/layer22_attention_weight_max": 51.00727844238281,
      "activations/layer22_attention_weight_min": -28.3140811920166,
      "activations/layer23_attention_weight_max": 29.7822265625,
      "activations/layer23_attention_weight_min": -19.216411590576172,
      "activations/layer2_attention_weight_max": 31.853923797607422,
      "activations/layer2_attention_weight_min": -30.853450775146484,
      "activations/layer3_attention_weight_max": 86.40520477294922,
      "activations/layer3_attention_weight_min": -85.83887481689453,
      "activations/layer4_attention_weight_max": 132.4014129638672,
      "activations/layer4_attention_weight_min": -126.1909408569336,
      "activations/layer5_attention_weight_max": 102.51976013183594,
      "activations/layer5_attention_weight_min": -93.53924560546875,
      "activations/layer6_attention_weight_max": 57.177757263183594,
      "activations/layer6_attention_weight_min": -52.26023864746094,
      "activations/layer7_attention_weight_max": 72.81964874267578,
      "activations/layer7_attention_weight_min": -72.33873748779297,
      "activations/layer8_attention_weight_max": 40.79486083984375,
      "activations/layer8_attention_weight_min": -36.46718215942383,
      "activations/layer9_attention_weight_max": 50.7920036315918,
      "activations/layer9_attention_weight_min": -38.39814376831055,
      "epoch": 3.2,
      "learning_rate": 0.00013064886363636362,
      "loss": 2.8844,
      "step": 55150
    },
    {
      "activations/layer0_attention_weight_max": 16.95948028564453,
      "activations/layer0_attention_weight_min": -11.678750991821289,
      "activations/layer10_attention_weight_max": 42.946197509765625,
      "activations/layer10_attention_weight_min": -38.95332717895508,
      "activations/layer11_attention_weight_max": 35.92185974121094,
      "activations/layer11_attention_weight_min": -29.529003143310547,
      "activations/layer12_attention_weight_max": 31.097312927246094,
      "activations/layer12_attention_weight_min": -30.623205184936523,
      "activations/layer13_attention_weight_max": 47.68013000488281,
      "activations/layer13_attention_weight_min": -33.51213073730469,
      "activations/layer14_attention_weight_max": 75.18539428710938,
      "activations/layer14_attention_weight_min": -53.67164993286133,
      "activations/layer15_attention_weight_max": 45.15925216674805,
      "activations/layer15_attention_weight_min": -30.558406829833984,
      "activations/layer16_attention_weight_max": 42.373226165771484,
      "activations/layer16_attention_weight_min": -29.241987228393555,
      "activations/layer17_attention_weight_max": 49.04451370239258,
      "activations/layer17_attention_weight_min": -25.38242530822754,
      "activations/layer18_attention_weight_max": 34.61067199707031,
      "activations/layer18_attention_weight_min": -19.954784393310547,
      "activations/layer19_attention_weight_max": 46.78515625,
      "activations/layer19_attention_weight_min": -30.3742733001709,
      "activations/layer1_attention_weight_max": 17.08339500427246,
      "activations/layer1_attention_weight_min": -15.976981163024902,
      "activations/layer20_attention_weight_max": 30.413408279418945,
      "activations/layer20_attention_weight_min": -22.15774917602539,
      "activations/layer21_attention_weight_max": 33.17715072631836,
      "activations/layer21_attention_weight_min": -18.6143798828125,
      "activations/layer22_attention_weight_max": 56.38429260253906,
      "activations/layer22_attention_weight_min": -27.62973976135254,
      "activations/layer23_attention_weight_max": 30.46589469909668,
      "activations/layer23_attention_weight_min": -16.224498748779297,
      "activations/layer2_attention_weight_max": 31.513105392456055,
      "activations/layer2_attention_weight_min": -30.712364196777344,
      "activations/layer3_attention_weight_max": 82.84241485595703,
      "activations/layer3_attention_weight_min": -84.19364166259766,
      "activations/layer4_attention_weight_max": 129.52621459960938,
      "activations/layer4_attention_weight_min": -125.0533447265625,
      "activations/layer5_attention_weight_max": 97.14983367919922,
      "activations/layer5_attention_weight_min": -89.89620971679688,
      "activations/layer6_attention_weight_max": 54.79758071899414,
      "activations/layer6_attention_weight_min": -51.71803665161133,
      "activations/layer7_attention_weight_max": 68.76666259765625,
      "activations/layer7_attention_weight_min": -66.99566650390625,
      "activations/layer8_attention_weight_max": 38.72655487060547,
      "activations/layer8_attention_weight_min": -35.222312927246094,
      "activations/layer9_attention_weight_max": 48.56465530395508,
      "activations/layer9_attention_weight_min": -36.599544525146484,
      "epoch": 3.21,
      "learning_rate": 0.00013062992424242424,
      "loss": 2.8976,
      "step": 55200
    },
    {
      "activations/layer0_attention_weight_max": 14.693806648254395,
      "activations/layer0_attention_weight_min": -12.459181785583496,
      "activations/layer10_attention_weight_max": 37.36423110961914,
      "activations/layer10_attention_weight_min": -31.885698318481445,
      "activations/layer11_attention_weight_max": 41.97169876098633,
      "activations/layer11_attention_weight_min": -32.91659927368164,
      "activations/layer12_attention_weight_max": 64.4536361694336,
      "activations/layer12_attention_weight_min": -27.61503028869629,
      "activations/layer13_attention_weight_max": 53.559593200683594,
      "activations/layer13_attention_weight_min": -37.8654899597168,
      "activations/layer14_attention_weight_max": 77.1650619506836,
      "activations/layer14_attention_weight_min": -57.16289520263672,
      "activations/layer15_attention_weight_max": 48.49020004272461,
      "activations/layer15_attention_weight_min": -35.5268440246582,
      "activations/layer16_attention_weight_max": 42.36959457397461,
      "activations/layer16_attention_weight_min": -30.809463500976562,
      "activations/layer17_attention_weight_max": 41.99855041503906,
      "activations/layer17_attention_weight_min": -28.349472045898438,
      "activations/layer18_attention_weight_max": 30.2147216796875,
      "activations/layer18_attention_weight_min": -18.301578521728516,
      "activations/layer19_attention_weight_max": 43.828346252441406,
      "activations/layer19_attention_weight_min": -28.14764404296875,
      "activations/layer1_attention_weight_max": 18.288827896118164,
      "activations/layer1_attention_weight_min": -13.712729454040527,
      "activations/layer20_attention_weight_max": 35.92427444458008,
      "activations/layer20_attention_weight_min": -22.459117889404297,
      "activations/layer21_attention_weight_max": 39.02995300292969,
      "activations/layer21_attention_weight_min": -21.101415634155273,
      "activations/layer22_attention_weight_max": 44.25734329223633,
      "activations/layer22_attention_weight_min": -29.35294532775879,
      "activations/layer23_attention_weight_max": 25.911497116088867,
      "activations/layer23_attention_weight_min": -17.237173080444336,
      "activations/layer2_attention_weight_max": 31.38353729248047,
      "activations/layer2_attention_weight_min": -30.167394638061523,
      "activations/layer3_attention_weight_max": 81.53582763671875,
      "activations/layer3_attention_weight_min": -86.40068817138672,
      "activations/layer4_attention_weight_max": 126.94258880615234,
      "activations/layer4_attention_weight_min": -122.22469329833984,
      "activations/layer5_attention_weight_max": 98.71060180664062,
      "activations/layer5_attention_weight_min": -88.38909912109375,
      "activations/layer6_attention_weight_max": 54.83140182495117,
      "activations/layer6_attention_weight_min": -52.01578140258789,
      "activations/layer7_attention_weight_max": 75.73750305175781,
      "activations/layer7_attention_weight_min": -69.48941040039062,
      "activations/layer8_attention_weight_max": 39.07530975341797,
      "activations/layer8_attention_weight_min": -35.97331619262695,
      "activations/layer9_attention_weight_max": 51.917720794677734,
      "activations/layer9_attention_weight_min": -40.314369201660156,
      "epoch": 3.21,
      "learning_rate": 0.00013061098484848485,
      "loss": 2.9064,
      "step": 55250
    },
    {
      "activations/layer0_attention_weight_max": 16.996925354003906,
      "activations/layer0_attention_weight_min": -11.845877647399902,
      "activations/layer10_attention_weight_max": 40.163002014160156,
      "activations/layer10_attention_weight_min": -33.435142517089844,
      "activations/layer11_attention_weight_max": 39.8869514465332,
      "activations/layer11_attention_weight_min": -31.454566955566406,
      "activations/layer12_attention_weight_max": 57.333038330078125,
      "activations/layer12_attention_weight_min": -27.157541275024414,
      "activations/layer13_attention_weight_max": 56.299930572509766,
      "activations/layer13_attention_weight_min": -33.38047409057617,
      "activations/layer14_attention_weight_max": 79.36360168457031,
      "activations/layer14_attention_weight_min": -51.62648010253906,
      "activations/layer15_attention_weight_max": 44.66211700439453,
      "activations/layer15_attention_weight_min": -29.30470848083496,
      "activations/layer16_attention_weight_max": 44.353763580322266,
      "activations/layer16_attention_weight_min": -30.500476837158203,
      "activations/layer17_attention_weight_max": 40.7516975402832,
      "activations/layer17_attention_weight_min": -27.940139770507812,
      "activations/layer18_attention_weight_max": 32.29277801513672,
      "activations/layer18_attention_weight_min": -20.38018035888672,
      "activations/layer19_attention_weight_max": 42.195552825927734,
      "activations/layer19_attention_weight_min": -29.74358558654785,
      "activations/layer1_attention_weight_max": 17.801984786987305,
      "activations/layer1_attention_weight_min": -15.597702026367188,
      "activations/layer20_attention_weight_max": 35.48588943481445,
      "activations/layer20_attention_weight_min": -21.59079933166504,
      "activations/layer21_attention_weight_max": 39.27894592285156,
      "activations/layer21_attention_weight_min": -22.3706111907959,
      "activations/layer22_attention_weight_max": 48.82204818725586,
      "activations/layer22_attention_weight_min": -28.43304443359375,
      "activations/layer23_attention_weight_max": 26.613088607788086,
      "activations/layer23_attention_weight_min": -18.251819610595703,
      "activations/layer2_attention_weight_max": 29.843915939331055,
      "activations/layer2_attention_weight_min": -30.060985565185547,
      "activations/layer3_attention_weight_max": 80.89409637451172,
      "activations/layer3_attention_weight_min": -83.69586181640625,
      "activations/layer4_attention_weight_max": 129.20555114746094,
      "activations/layer4_attention_weight_min": -123.37358856201172,
      "activations/layer5_attention_weight_max": 97.37152099609375,
      "activations/layer5_attention_weight_min": -91.91307067871094,
      "activations/layer6_attention_weight_max": 58.30912780761719,
      "activations/layer6_attention_weight_min": -55.06498336791992,
      "activations/layer7_attention_weight_max": 76.75361633300781,
      "activations/layer7_attention_weight_min": -70.90148162841797,
      "activations/layer8_attention_weight_max": 39.353694915771484,
      "activations/layer8_attention_weight_min": -36.809974670410156,
      "activations/layer9_attention_weight_max": 49.771873474121094,
      "activations/layer9_attention_weight_min": -37.60945129394531,
      "epoch": 3.21,
      "learning_rate": 0.00013059204545454544,
      "loss": 2.8841,
      "step": 55300
    },
    {
      "activations/layer0_attention_weight_max": 14.433832168579102,
      "activations/layer0_attention_weight_min": -11.962261199951172,
      "activations/layer10_attention_weight_max": 55.314208984375,
      "activations/layer10_attention_weight_min": -35.38015365600586,
      "activations/layer11_attention_weight_max": 55.089820861816406,
      "activations/layer11_attention_weight_min": -35.20128631591797,
      "activations/layer12_attention_weight_max": 65.16658782958984,
      "activations/layer12_attention_weight_min": -30.90093231201172,
      "activations/layer13_attention_weight_max": 93.1993637084961,
      "activations/layer13_attention_weight_min": -46.9089469909668,
      "activations/layer14_attention_weight_max": 100.14856719970703,
      "activations/layer14_attention_weight_min": -54.07448959350586,
      "activations/layer15_attention_weight_max": 64.38224029541016,
      "activations/layer15_attention_weight_min": -31.593685150146484,
      "activations/layer16_attention_weight_max": 48.96786880493164,
      "activations/layer16_attention_weight_min": -31.380603790283203,
      "activations/layer17_attention_weight_max": 46.258094787597656,
      "activations/layer17_attention_weight_min": -26.930994033813477,
      "activations/layer18_attention_weight_max": 37.83049774169922,
      "activations/layer18_attention_weight_min": -21.220685958862305,
      "activations/layer19_attention_weight_max": 51.6533203125,
      "activations/layer19_attention_weight_min": -31.32069969177246,
      "activations/layer1_attention_weight_max": 17.26961326599121,
      "activations/layer1_attention_weight_min": -14.75639533996582,
      "activations/layer20_attention_weight_max": 40.031673431396484,
      "activations/layer20_attention_weight_min": -23.29766082763672,
      "activations/layer21_attention_weight_max": 33.749732971191406,
      "activations/layer21_attention_weight_min": -19.580284118652344,
      "activations/layer22_attention_weight_max": 53.38302230834961,
      "activations/layer22_attention_weight_min": -27.28546905517578,
      "activations/layer23_attention_weight_max": 27.660337448120117,
      "activations/layer23_attention_weight_min": -17.843202590942383,
      "activations/layer2_attention_weight_max": 32.51557159423828,
      "activations/layer2_attention_weight_min": -29.71884536743164,
      "activations/layer3_attention_weight_max": 88.58736419677734,
      "activations/layer3_attention_weight_min": -86.15007019042969,
      "activations/layer4_attention_weight_max": 145.30052185058594,
      "activations/layer4_attention_weight_min": -131.4852752685547,
      "activations/layer5_attention_weight_max": 113.47256469726562,
      "activations/layer5_attention_weight_min": -92.80723571777344,
      "activations/layer6_attention_weight_max": 67.8372802734375,
      "activations/layer6_attention_weight_min": -57.42145919799805,
      "activations/layer7_attention_weight_max": 87.13583374023438,
      "activations/layer7_attention_weight_min": -74.66206359863281,
      "activations/layer8_attention_weight_max": 52.55435562133789,
      "activations/layer8_attention_weight_min": -39.45657730102539,
      "activations/layer9_attention_weight_max": 71.0184097290039,
      "activations/layer9_attention_weight_min": -48.22731018066406,
      "epoch": 3.22,
      "learning_rate": 0.00013057310606060606,
      "loss": 2.8926,
      "step": 55350
    },
    {
      "activations/layer0_attention_weight_max": 16.01641273498535,
      "activations/layer0_attention_weight_min": -10.890727996826172,
      "activations/layer10_attention_weight_max": 38.84541320800781,
      "activations/layer10_attention_weight_min": -31.756975173950195,
      "activations/layer11_attention_weight_max": 36.609615325927734,
      "activations/layer11_attention_weight_min": -29.490345001220703,
      "activations/layer12_attention_weight_max": 34.21390151977539,
      "activations/layer12_attention_weight_min": -28.622041702270508,
      "activations/layer13_attention_weight_max": 45.751895904541016,
      "activations/layer13_attention_weight_min": -33.40343475341797,
      "activations/layer14_attention_weight_max": 70.96849822998047,
      "activations/layer14_attention_weight_min": -50.830482482910156,
      "activations/layer15_attention_weight_max": 47.54634475708008,
      "activations/layer15_attention_weight_min": -31.747201919555664,
      "activations/layer16_attention_weight_max": 42.575416564941406,
      "activations/layer16_attention_weight_min": -32.41230392456055,
      "activations/layer17_attention_weight_max": 40.37968444824219,
      "activations/layer17_attention_weight_min": -26.671993255615234,
      "activations/layer18_attention_weight_max": 33.849273681640625,
      "activations/layer18_attention_weight_min": -19.213119506835938,
      "activations/layer19_attention_weight_max": 39.40952682495117,
      "activations/layer19_attention_weight_min": -31.786758422851562,
      "activations/layer1_attention_weight_max": 17.013198852539062,
      "activations/layer1_attention_weight_min": -16.069664001464844,
      "activations/layer20_attention_weight_max": 34.66056442260742,
      "activations/layer20_attention_weight_min": -25.042301177978516,
      "activations/layer21_attention_weight_max": 33.41183853149414,
      "activations/layer21_attention_weight_min": -23.374284744262695,
      "activations/layer22_attention_weight_max": 48.31241226196289,
      "activations/layer22_attention_weight_min": -30.98999786376953,
      "activations/layer23_attention_weight_max": 26.45886993408203,
      "activations/layer23_attention_weight_min": -20.29193687438965,
      "activations/layer2_attention_weight_max": 30.733266830444336,
      "activations/layer2_attention_weight_min": -29.684118270874023,
      "activations/layer3_attention_weight_max": 83.30353546142578,
      "activations/layer3_attention_weight_min": -84.38797760009766,
      "activations/layer4_attention_weight_max": 129.01036071777344,
      "activations/layer4_attention_weight_min": -123.9735107421875,
      "activations/layer5_attention_weight_max": 97.76564025878906,
      "activations/layer5_attention_weight_min": -92.61599731445312,
      "activations/layer6_attention_weight_max": 57.144020080566406,
      "activations/layer6_attention_weight_min": -53.31346130371094,
      "activations/layer7_attention_weight_max": 69.29680633544922,
      "activations/layer7_attention_weight_min": -68.88123321533203,
      "activations/layer8_attention_weight_max": 38.626094818115234,
      "activations/layer8_attention_weight_min": -36.01607131958008,
      "activations/layer9_attention_weight_max": 43.25233840942383,
      "activations/layer9_attention_weight_min": -40.05960464477539,
      "epoch": 3.22,
      "learning_rate": 0.00013055416666666667,
      "loss": 2.8816,
      "step": 55400
    },
    {
      "activations/layer0_attention_weight_max": 16.901899337768555,
      "activations/layer0_attention_weight_min": -12.72092342376709,
      "activations/layer10_attention_weight_max": 39.00190353393555,
      "activations/layer10_attention_weight_min": -32.6044921875,
      "activations/layer11_attention_weight_max": 32.95576095581055,
      "activations/layer11_attention_weight_min": -32.350460052490234,
      "activations/layer12_attention_weight_max": 56.0750617980957,
      "activations/layer12_attention_weight_min": -42.86549758911133,
      "activations/layer13_attention_weight_max": 48.63935470581055,
      "activations/layer13_attention_weight_min": -36.04974365234375,
      "activations/layer14_attention_weight_max": 72.37162017822266,
      "activations/layer14_attention_weight_min": -52.608158111572266,
      "activations/layer15_attention_weight_max": 49.947933197021484,
      "activations/layer15_attention_weight_min": -30.4980525970459,
      "activations/layer16_attention_weight_max": 42.54823303222656,
      "activations/layer16_attention_weight_min": -31.880081176757812,
      "activations/layer17_attention_weight_max": 42.46773910522461,
      "activations/layer17_attention_weight_min": -25.6737117767334,
      "activations/layer18_attention_weight_max": 30.466501235961914,
      "activations/layer18_attention_weight_min": -20.129690170288086,
      "activations/layer19_attention_weight_max": 48.31118392944336,
      "activations/layer19_attention_weight_min": -30.289045333862305,
      "activations/layer1_attention_weight_max": 17.519763946533203,
      "activations/layer1_attention_weight_min": -12.662699699401855,
      "activations/layer20_attention_weight_max": 34.9862174987793,
      "activations/layer20_attention_weight_min": -22.900327682495117,
      "activations/layer21_attention_weight_max": 36.885311126708984,
      "activations/layer21_attention_weight_min": -20.405920028686523,
      "activations/layer22_attention_weight_max": 53.35966873168945,
      "activations/layer22_attention_weight_min": -28.98468780517578,
      "activations/layer23_attention_weight_max": 28.780515670776367,
      "activations/layer23_attention_weight_min": -19.00721549987793,
      "activations/layer2_attention_weight_max": 31.826719284057617,
      "activations/layer2_attention_weight_min": -30.430736541748047,
      "activations/layer3_attention_weight_max": 86.38885498046875,
      "activations/layer3_attention_weight_min": -85.00078582763672,
      "activations/layer4_attention_weight_max": 131.86802673339844,
      "activations/layer4_attention_weight_min": -126.90211486816406,
      "activations/layer5_attention_weight_max": 102.79393768310547,
      "activations/layer5_attention_weight_min": -96.01134490966797,
      "activations/layer6_attention_weight_max": 57.35905075073242,
      "activations/layer6_attention_weight_min": -53.893043518066406,
      "activations/layer7_attention_weight_max": 75.80879211425781,
      "activations/layer7_attention_weight_min": -70.93366241455078,
      "activations/layer8_attention_weight_max": 39.69789505004883,
      "activations/layer8_attention_weight_min": -38.74336242675781,
      "activations/layer9_attention_weight_max": 54.02522659301758,
      "activations/layer9_attention_weight_min": -39.675113677978516,
      "epoch": 3.22,
      "learning_rate": 0.00013053522727272726,
      "loss": 2.9086,
      "step": 55450
    },
    {
      "activations/layer0_attention_weight_max": 14.577049255371094,
      "activations/layer0_attention_weight_min": -11.696176528930664,
      "activations/layer10_attention_weight_max": 41.631988525390625,
      "activations/layer10_attention_weight_min": -35.27790832519531,
      "activations/layer11_attention_weight_max": 37.92338943481445,
      "activations/layer11_attention_weight_min": -28.068111419677734,
      "activations/layer12_attention_weight_max": 34.72581100463867,
      "activations/layer12_attention_weight_min": -27.963136672973633,
      "activations/layer13_attention_weight_max": 50.25891876220703,
      "activations/layer13_attention_weight_min": -36.400455474853516,
      "activations/layer14_attention_weight_max": 78.6055679321289,
      "activations/layer14_attention_weight_min": -54.51771545410156,
      "activations/layer15_attention_weight_max": 48.92597198486328,
      "activations/layer15_attention_weight_min": -32.661956787109375,
      "activations/layer16_attention_weight_max": 46.52030944824219,
      "activations/layer16_attention_weight_min": -31.509782791137695,
      "activations/layer17_attention_weight_max": 45.23286819458008,
      "activations/layer17_attention_weight_min": -26.005727767944336,
      "activations/layer18_attention_weight_max": 31.118000030517578,
      "activations/layer18_attention_weight_min": -18.809919357299805,
      "activations/layer19_attention_weight_max": 46.958580017089844,
      "activations/layer19_attention_weight_min": -29.928129196166992,
      "activations/layer1_attention_weight_max": 16.335731506347656,
      "activations/layer1_attention_weight_min": -14.028176307678223,
      "activations/layer20_attention_weight_max": 33.959991455078125,
      "activations/layer20_attention_weight_min": -23.363740921020508,
      "activations/layer21_attention_weight_max": 39.344154357910156,
      "activations/layer21_attention_weight_min": -22.555944442749023,
      "activations/layer22_attention_weight_max": 49.16508102416992,
      "activations/layer22_attention_weight_min": -28.325037002563477,
      "activations/layer23_attention_weight_max": 29.318984985351562,
      "activations/layer23_attention_weight_min": -18.128511428833008,
      "activations/layer2_attention_weight_max": 31.452253341674805,
      "activations/layer2_attention_weight_min": -31.041885375976562,
      "activations/layer3_attention_weight_max": 83.4823226928711,
      "activations/layer3_attention_weight_min": -85.87220764160156,
      "activations/layer4_attention_weight_max": 132.03602600097656,
      "activations/layer4_attention_weight_min": -126.1900634765625,
      "activations/layer5_attention_weight_max": 96.56332397460938,
      "activations/layer5_attention_weight_min": -89.53390502929688,
      "activations/layer6_attention_weight_max": 55.26096725463867,
      "activations/layer6_attention_weight_min": -51.44560241699219,
      "activations/layer7_attention_weight_max": 70.694091796875,
      "activations/layer7_attention_weight_min": -70.23954010009766,
      "activations/layer8_attention_weight_max": 37.88833236694336,
      "activations/layer8_attention_weight_min": -35.8401985168457,
      "activations/layer9_attention_weight_max": 42.250999450683594,
      "activations/layer9_attention_weight_min": -36.54671859741211,
      "epoch": 3.22,
      "learning_rate": 0.00013051628787878787,
      "loss": 2.8962,
      "step": 55500
    },
    {
      "activations/layer0_attention_weight_max": 16.744409561157227,
      "activations/layer0_attention_weight_min": -11.903402328491211,
      "activations/layer10_attention_weight_max": 37.592620849609375,
      "activations/layer10_attention_weight_min": -31.0341796875,
      "activations/layer11_attention_weight_max": 34.5472412109375,
      "activations/layer11_attention_weight_min": -28.53862762451172,
      "activations/layer12_attention_weight_max": 36.76585006713867,
      "activations/layer12_attention_weight_min": -26.16228675842285,
      "activations/layer13_attention_weight_max": 45.02143096923828,
      "activations/layer13_attention_weight_min": -34.18110656738281,
      "activations/layer14_attention_weight_max": 67.33349609375,
      "activations/layer14_attention_weight_min": -45.40574645996094,
      "activations/layer15_attention_weight_max": 45.67607498168945,
      "activations/layer15_attention_weight_min": -31.27733612060547,
      "activations/layer16_attention_weight_max": 45.60066223144531,
      "activations/layer16_attention_weight_min": -30.109569549560547,
      "activations/layer17_attention_weight_max": 41.24618911743164,
      "activations/layer17_attention_weight_min": -25.38425064086914,
      "activations/layer18_attention_weight_max": 32.34964370727539,
      "activations/layer18_attention_weight_min": -20.76656723022461,
      "activations/layer19_attention_weight_max": 43.5538330078125,
      "activations/layer19_attention_weight_min": -31.547704696655273,
      "activations/layer1_attention_weight_max": 17.99846649169922,
      "activations/layer1_attention_weight_min": -14.161791801452637,
      "activations/layer20_attention_weight_max": 33.906673431396484,
      "activations/layer20_attention_weight_min": -23.221839904785156,
      "activations/layer21_attention_weight_max": 37.93979263305664,
      "activations/layer21_attention_weight_min": -20.044891357421875,
      "activations/layer22_attention_weight_max": 49.71786880493164,
      "activations/layer22_attention_weight_min": -27.35785675048828,
      "activations/layer23_attention_weight_max": 27.178558349609375,
      "activations/layer23_attention_weight_min": -18.835933685302734,
      "activations/layer2_attention_weight_max": 30.802968978881836,
      "activations/layer2_attention_weight_min": -31.301944732666016,
      "activations/layer3_attention_weight_max": 81.83637237548828,
      "activations/layer3_attention_weight_min": -85.62017822265625,
      "activations/layer4_attention_weight_max": 126.95568084716797,
      "activations/layer4_attention_weight_min": -123.96624755859375,
      "activations/layer5_attention_weight_max": 99.59716033935547,
      "activations/layer5_attention_weight_min": -85.72161102294922,
      "activations/layer6_attention_weight_max": 54.96768569946289,
      "activations/layer6_attention_weight_min": -52.77729797363281,
      "activations/layer7_attention_weight_max": 69.41140747070312,
      "activations/layer7_attention_weight_min": -74.32896423339844,
      "activations/layer8_attention_weight_max": 37.8515625,
      "activations/layer8_attention_weight_min": -35.80907440185547,
      "activations/layer9_attention_weight_max": 42.545387268066406,
      "activations/layer9_attention_weight_min": -33.902748107910156,
      "epoch": 3.23,
      "learning_rate": 0.0001304973484848485,
      "loss": 2.8886,
      "step": 55550
    },
    {
      "activations/layer0_attention_weight_max": 15.164892196655273,
      "activations/layer0_attention_weight_min": -10.52006721496582,
      "activations/layer10_attention_weight_max": 43.14666748046875,
      "activations/layer10_attention_weight_min": -33.68935012817383,
      "activations/layer11_attention_weight_max": 39.1303825378418,
      "activations/layer11_attention_weight_min": -28.76250457763672,
      "activations/layer12_attention_weight_max": 35.172386169433594,
      "activations/layer12_attention_weight_min": -28.1607723236084,
      "activations/layer13_attention_weight_max": 49.24703598022461,
      "activations/layer13_attention_weight_min": -34.64439010620117,
      "activations/layer14_attention_weight_max": 66.34012603759766,
      "activations/layer14_attention_weight_min": -51.49225997924805,
      "activations/layer15_attention_weight_max": 45.94956588745117,
      "activations/layer15_attention_weight_min": -31.64825439453125,
      "activations/layer16_attention_weight_max": 48.774497985839844,
      "activations/layer16_attention_weight_min": -35.203338623046875,
      "activations/layer17_attention_weight_max": 46.9699821472168,
      "activations/layer17_attention_weight_min": -26.74907112121582,
      "activations/layer18_attention_weight_max": 32.731815338134766,
      "activations/layer18_attention_weight_min": -18.69603729248047,
      "activations/layer19_attention_weight_max": 49.80326843261719,
      "activations/layer19_attention_weight_min": -32.261959075927734,
      "activations/layer1_attention_weight_max": 17.710107803344727,
      "activations/layer1_attention_weight_min": -15.19298267364502,
      "activations/layer20_attention_weight_max": 38.04701614379883,
      "activations/layer20_attention_weight_min": -24.1079158782959,
      "activations/layer21_attention_weight_max": 35.26913833618164,
      "activations/layer21_attention_weight_min": -19.8096923828125,
      "activations/layer22_attention_weight_max": 54.60869216918945,
      "activations/layer22_attention_weight_min": -27.87677764892578,
      "activations/layer23_attention_weight_max": 33.66893005371094,
      "activations/layer23_attention_weight_min": -17.55496597290039,
      "activations/layer2_attention_weight_max": 31.575305938720703,
      "activations/layer2_attention_weight_min": -30.271350860595703,
      "activations/layer3_attention_weight_max": 78.59077453613281,
      "activations/layer3_attention_weight_min": -80.03095245361328,
      "activations/layer4_attention_weight_max": 124.31903076171875,
      "activations/layer4_attention_weight_min": -115.35863494873047,
      "activations/layer5_attention_weight_max": 95.48094177246094,
      "activations/layer5_attention_weight_min": -88.5766830444336,
      "activations/layer6_attention_weight_max": 55.45349884033203,
      "activations/layer6_attention_weight_min": -52.08171844482422,
      "activations/layer7_attention_weight_max": 70.98912048339844,
      "activations/layer7_attention_weight_min": -69.31997680664062,
      "activations/layer8_attention_weight_max": 36.92258834838867,
      "activations/layer8_attention_weight_min": -35.64510726928711,
      "activations/layer9_attention_weight_max": 50.968299865722656,
      "activations/layer9_attention_weight_min": -41.73352813720703,
      "epoch": 3.23,
      "learning_rate": 0.00013047840909090908,
      "loss": 2.906,
      "step": 55600
    },
    {
      "activations/layer0_attention_weight_max": 17.055030822753906,
      "activations/layer0_attention_weight_min": -11.531150817871094,
      "activations/layer10_attention_weight_max": 40.27626037597656,
      "activations/layer10_attention_weight_min": -34.1112060546875,
      "activations/layer11_attention_weight_max": 34.26811981201172,
      "activations/layer11_attention_weight_min": -30.74203109741211,
      "activations/layer12_attention_weight_max": 36.741085052490234,
      "activations/layer12_attention_weight_min": -27.140003204345703,
      "activations/layer13_attention_weight_max": 49.97313690185547,
      "activations/layer13_attention_weight_min": -36.2492561340332,
      "activations/layer14_attention_weight_max": 66.39488983154297,
      "activations/layer14_attention_weight_min": -53.279788970947266,
      "activations/layer15_attention_weight_max": 44.423194885253906,
      "activations/layer15_attention_weight_min": -32.30942153930664,
      "activations/layer16_attention_weight_max": 39.12918472290039,
      "activations/layer16_attention_weight_min": -35.89942932128906,
      "activations/layer17_attention_weight_max": 39.95485305786133,
      "activations/layer17_attention_weight_min": -30.008691787719727,
      "activations/layer18_attention_weight_max": 28.821456909179688,
      "activations/layer18_attention_weight_min": -22.071611404418945,
      "activations/layer19_attention_weight_max": 39.49919128417969,
      "activations/layer19_attention_weight_min": -30.744333267211914,
      "activations/layer1_attention_weight_max": 16.987958908081055,
      "activations/layer1_attention_weight_min": -12.811031341552734,
      "activations/layer20_attention_weight_max": 31.083633422851562,
      "activations/layer20_attention_weight_min": -22.803495407104492,
      "activations/layer21_attention_weight_max": 30.42655372619629,
      "activations/layer21_attention_weight_min": -20.8818302154541,
      "activations/layer22_attention_weight_max": 47.001949310302734,
      "activations/layer22_attention_weight_min": -29.337141036987305,
      "activations/layer23_attention_weight_max": 25.512222290039062,
      "activations/layer23_attention_weight_min": -17.904834747314453,
      "activations/layer2_attention_weight_max": 31.376113891601562,
      "activations/layer2_attention_weight_min": -30.06970977783203,
      "activations/layer3_attention_weight_max": 81.70165252685547,
      "activations/layer3_attention_weight_min": -85.31302642822266,
      "activations/layer4_attention_weight_max": 131.8417510986328,
      "activations/layer4_attention_weight_min": -126.70140838623047,
      "activations/layer5_attention_weight_max": 100.79792785644531,
      "activations/layer5_attention_weight_min": -92.14412689208984,
      "activations/layer6_attention_weight_max": 56.5999870300293,
      "activations/layer6_attention_weight_min": -51.64458084106445,
      "activations/layer7_attention_weight_max": 71.7406997680664,
      "activations/layer7_attention_weight_min": -73.084716796875,
      "activations/layer8_attention_weight_max": 40.727272033691406,
      "activations/layer8_attention_weight_min": -37.21150588989258,
      "activations/layer9_attention_weight_max": 47.30482864379883,
      "activations/layer9_attention_weight_min": -40.75509262084961,
      "epoch": 3.23,
      "learning_rate": 0.0001304594696969697,
      "loss": 2.8988,
      "step": 55650
    },
    {
      "activations/layer0_attention_weight_max": 16.452486038208008,
      "activations/layer0_attention_weight_min": -11.71796989440918,
      "activations/layer10_attention_weight_max": 43.06428146362305,
      "activations/layer10_attention_weight_min": -32.65179443359375,
      "activations/layer11_attention_weight_max": 38.14238357543945,
      "activations/layer11_attention_weight_min": -27.25922393798828,
      "activations/layer12_attention_weight_max": 37.11383056640625,
      "activations/layer12_attention_weight_min": -26.570051193237305,
      "activations/layer13_attention_weight_max": 51.7122917175293,
      "activations/layer13_attention_weight_min": -35.24152755737305,
      "activations/layer14_attention_weight_max": 90.47672271728516,
      "activations/layer14_attention_weight_min": -57.068660736083984,
      "activations/layer15_attention_weight_max": 46.83677673339844,
      "activations/layer15_attention_weight_min": -31.67056655883789,
      "activations/layer16_attention_weight_max": 49.97511291503906,
      "activations/layer16_attention_weight_min": -32.57953643798828,
      "activations/layer17_attention_weight_max": 46.129024505615234,
      "activations/layer17_attention_weight_min": -26.562381744384766,
      "activations/layer18_attention_weight_max": 31.91961097717285,
      "activations/layer18_attention_weight_min": -18.68104362487793,
      "activations/layer19_attention_weight_max": 49.82589340209961,
      "activations/layer19_attention_weight_min": -32.4235954284668,
      "activations/layer1_attention_weight_max": 17.03450584411621,
      "activations/layer1_attention_weight_min": -13.899312019348145,
      "activations/layer20_attention_weight_max": 33.441673278808594,
      "activations/layer20_attention_weight_min": -22.716838836669922,
      "activations/layer21_attention_weight_max": 35.692626953125,
      "activations/layer21_attention_weight_min": -19.456336975097656,
      "activations/layer22_attention_weight_max": 51.43593978881836,
      "activations/layer22_attention_weight_min": -26.87921905517578,
      "activations/layer23_attention_weight_max": 26.6980037689209,
      "activations/layer23_attention_weight_min": -15.028278350830078,
      "activations/layer2_attention_weight_max": 30.119136810302734,
      "activations/layer2_attention_weight_min": -30.35287857055664,
      "activations/layer3_attention_weight_max": 76.62474822998047,
      "activations/layer3_attention_weight_min": -81.02699279785156,
      "activations/layer4_attention_weight_max": 131.46986389160156,
      "activations/layer4_attention_weight_min": -126.05655670166016,
      "activations/layer5_attention_weight_max": 101.91676330566406,
      "activations/layer5_attention_weight_min": -92.28109741210938,
      "activations/layer6_attention_weight_max": 62.85444259643555,
      "activations/layer6_attention_weight_min": -53.977603912353516,
      "activations/layer7_attention_weight_max": 80.16177368164062,
      "activations/layer7_attention_weight_min": -69.56033325195312,
      "activations/layer8_attention_weight_max": 45.81064987182617,
      "activations/layer8_attention_weight_min": -37.077392578125,
      "activations/layer9_attention_weight_max": 51.75458908081055,
      "activations/layer9_attention_weight_min": -35.473812103271484,
      "epoch": 3.24,
      "learning_rate": 0.0001304405303030303,
      "loss": 2.9041,
      "step": 55700
    },
    {
      "activations/layer0_attention_weight_max": 13.794157028198242,
      "activations/layer0_attention_weight_min": -11.444710731506348,
      "activations/layer10_attention_weight_max": 38.422821044921875,
      "activations/layer10_attention_weight_min": -31.528432846069336,
      "activations/layer11_attention_weight_max": 44.497650146484375,
      "activations/layer11_attention_weight_min": -34.513946533203125,
      "activations/layer12_attention_weight_max": 50.54682540893555,
      "activations/layer12_attention_weight_min": -33.05406951904297,
      "activations/layer13_attention_weight_max": 49.293617248535156,
      "activations/layer13_attention_weight_min": -42.79603958129883,
      "activations/layer14_attention_weight_max": 66.21585845947266,
      "activations/layer14_attention_weight_min": -53.50244903564453,
      "activations/layer15_attention_weight_max": 48.15744400024414,
      "activations/layer15_attention_weight_min": -36.218017578125,
      "activations/layer16_attention_weight_max": 42.57669448852539,
      "activations/layer16_attention_weight_min": -30.608610153198242,
      "activations/layer17_attention_weight_max": 44.02585220336914,
      "activations/layer17_attention_weight_min": -24.23763656616211,
      "activations/layer18_attention_weight_max": 31.122182846069336,
      "activations/layer18_attention_weight_min": -19.17776107788086,
      "activations/layer19_attention_weight_max": 45.52394104003906,
      "activations/layer19_attention_weight_min": -28.347366333007812,
      "activations/layer1_attention_weight_max": 17.475223541259766,
      "activations/layer1_attention_weight_min": -12.760138511657715,
      "activations/layer20_attention_weight_max": 33.622535705566406,
      "activations/layer20_attention_weight_min": -22.8094539642334,
      "activations/layer21_attention_weight_max": 38.595130920410156,
      "activations/layer21_attention_weight_min": -18.54252052307129,
      "activations/layer22_attention_weight_max": 48.9914665222168,
      "activations/layer22_attention_weight_min": -27.638357162475586,
      "activations/layer23_attention_weight_max": 29.672042846679688,
      "activations/layer23_attention_weight_min": -16.055734634399414,
      "activations/layer2_attention_weight_max": 31.25200653076172,
      "activations/layer2_attention_weight_min": -29.73096466064453,
      "activations/layer3_attention_weight_max": 81.81795501708984,
      "activations/layer3_attention_weight_min": -86.16937255859375,
      "activations/layer4_attention_weight_max": 129.28404235839844,
      "activations/layer4_attention_weight_min": -125.65538787841797,
      "activations/layer5_attention_weight_max": 99.44393920898438,
      "activations/layer5_attention_weight_min": -88.54069519042969,
      "activations/layer6_attention_weight_max": 58.67251968383789,
      "activations/layer6_attention_weight_min": -54.5575065612793,
      "activations/layer7_attention_weight_max": 75.9519271850586,
      "activations/layer7_attention_weight_min": -82.9971923828125,
      "activations/layer8_attention_weight_max": 41.4707145690918,
      "activations/layer8_attention_weight_min": -41.40675354003906,
      "activations/layer9_attention_weight_max": 53.4221305847168,
      "activations/layer9_attention_weight_min": -47.14370346069336,
      "epoch": 3.24,
      "learning_rate": 0.0001304215909090909,
      "loss": 2.9065,
      "step": 55750
    },
    {
      "activations/layer0_attention_weight_max": 17.493061065673828,
      "activations/layer0_attention_weight_min": -11.531691551208496,
      "activations/layer10_attention_weight_max": 46.35813522338867,
      "activations/layer10_attention_weight_min": -35.36276626586914,
      "activations/layer11_attention_weight_max": 46.966243743896484,
      "activations/layer11_attention_weight_min": -33.056854248046875,
      "activations/layer12_attention_weight_max": 57.27475357055664,
      "activations/layer12_attention_weight_min": -32.187530517578125,
      "activations/layer13_attention_weight_max": 66.93123626708984,
      "activations/layer13_attention_weight_min": -40.81828689575195,
      "activations/layer14_attention_weight_max": 91.62362670898438,
      "activations/layer14_attention_weight_min": -47.722412109375,
      "activations/layer15_attention_weight_max": 48.442596435546875,
      "activations/layer15_attention_weight_min": -29.297958374023438,
      "activations/layer16_attention_weight_max": 47.070587158203125,
      "activations/layer16_attention_weight_min": -30.909603118896484,
      "activations/layer17_attention_weight_max": 37.544551849365234,
      "activations/layer17_attention_weight_min": -23.753883361816406,
      "activations/layer18_attention_weight_max": 29.702171325683594,
      "activations/layer18_attention_weight_min": -22.755516052246094,
      "activations/layer19_attention_weight_max": 47.46027755737305,
      "activations/layer19_attention_weight_min": -32.50054168701172,
      "activations/layer1_attention_weight_max": 18.835474014282227,
      "activations/layer1_attention_weight_min": -13.475356101989746,
      "activations/layer20_attention_weight_max": 33.984928131103516,
      "activations/layer20_attention_weight_min": -23.636606216430664,
      "activations/layer21_attention_weight_max": 30.93466567993164,
      "activations/layer21_attention_weight_min": -19.366077423095703,
      "activations/layer22_attention_weight_max": 51.678993225097656,
      "activations/layer22_attention_weight_min": -26.636398315429688,
      "activations/layer23_attention_weight_max": 30.853076934814453,
      "activations/layer23_attention_weight_min": -22.18224334716797,
      "activations/layer2_attention_weight_max": 32.3671875,
      "activations/layer2_attention_weight_min": -31.186626434326172,
      "activations/layer3_attention_weight_max": 90.38948059082031,
      "activations/layer3_attention_weight_min": -91.78831481933594,
      "activations/layer4_attention_weight_max": 155.9159393310547,
      "activations/layer4_attention_weight_min": -136.33421325683594,
      "activations/layer5_attention_weight_max": 121.10552215576172,
      "activations/layer5_attention_weight_min": -101.06814575195312,
      "activations/layer6_attention_weight_max": 65.792724609375,
      "activations/layer6_attention_weight_min": -54.286624908447266,
      "activations/layer7_attention_weight_max": 79.63201141357422,
      "activations/layer7_attention_weight_min": -75.81476593017578,
      "activations/layer8_attention_weight_max": 46.74454116821289,
      "activations/layer8_attention_weight_min": -39.960147857666016,
      "activations/layer9_attention_weight_max": 53.8985710144043,
      "activations/layer9_attention_weight_min": -44.28135681152344,
      "epoch": 3.24,
      "learning_rate": 0.0001304026515151515,
      "loss": 2.9026,
      "step": 55800
    },
    {
      "activations/layer0_attention_weight_max": 16.60343360900879,
      "activations/layer0_attention_weight_min": -12.256855010986328,
      "activations/layer10_attention_weight_max": 48.59152603149414,
      "activations/layer10_attention_weight_min": -35.96980285644531,
      "activations/layer11_attention_weight_max": 55.34481430053711,
      "activations/layer11_attention_weight_min": -36.38368606567383,
      "activations/layer12_attention_weight_max": 95.22230529785156,
      "activations/layer12_attention_weight_min": -38.909149169921875,
      "activations/layer13_attention_weight_max": 102.07605743408203,
      "activations/layer13_attention_weight_min": -55.61552047729492,
      "activations/layer14_attention_weight_max": 81.88419342041016,
      "activations/layer14_attention_weight_min": -58.14990234375,
      "activations/layer15_attention_weight_max": 73.09752655029297,
      "activations/layer15_attention_weight_min": -36.60840606689453,
      "activations/layer16_attention_weight_max": 43.84294891357422,
      "activations/layer16_attention_weight_min": -28.497480392456055,
      "activations/layer17_attention_weight_max": 43.29248046875,
      "activations/layer17_attention_weight_min": -24.95462989807129,
      "activations/layer18_attention_weight_max": 28.736032485961914,
      "activations/layer18_attention_weight_min": -19.22672462463379,
      "activations/layer19_attention_weight_max": 43.881404876708984,
      "activations/layer19_attention_weight_min": -28.167882919311523,
      "activations/layer1_attention_weight_max": 17.85041046142578,
      "activations/layer1_attention_weight_min": -13.53147029876709,
      "activations/layer20_attention_weight_max": 36.11981964111328,
      "activations/layer20_attention_weight_min": -24.558326721191406,
      "activations/layer21_attention_weight_max": 34.60065460205078,
      "activations/layer21_attention_weight_min": -19.714998245239258,
      "activations/layer22_attention_weight_max": 48.31231689453125,
      "activations/layer22_attention_weight_min": -28.052827835083008,
      "activations/layer23_attention_weight_max": 29.015670776367188,
      "activations/layer23_attention_weight_min": -17.881099700927734,
      "activations/layer2_attention_weight_max": 31.97871208190918,
      "activations/layer2_attention_weight_min": -31.68865203857422,
      "activations/layer3_attention_weight_max": 83.0753173828125,
      "activations/layer3_attention_weight_min": -89.21218872070312,
      "activations/layer4_attention_weight_max": 132.58958435058594,
      "activations/layer4_attention_weight_min": -124.16606903076172,
      "activations/layer5_attention_weight_max": 106.4521713256836,
      "activations/layer5_attention_weight_min": -92.42776489257812,
      "activations/layer6_attention_weight_max": 60.416419982910156,
      "activations/layer6_attention_weight_min": -55.841461181640625,
      "activations/layer7_attention_weight_max": 89.5819091796875,
      "activations/layer7_attention_weight_min": -75.25004577636719,
      "activations/layer8_attention_weight_max": 48.27925491333008,
      "activations/layer8_attention_weight_min": -39.380672454833984,
      "activations/layer9_attention_weight_max": 58.57886505126953,
      "activations/layer9_attention_weight_min": -43.47279357910156,
      "epoch": 3.25,
      "learning_rate": 0.0001303837121212121,
      "loss": 2.894,
      "step": 55850
    },
    {
      "activations/layer0_attention_weight_max": 15.688162803649902,
      "activations/layer0_attention_weight_min": -11.029425621032715,
      "activations/layer10_attention_weight_max": 42.7759895324707,
      "activations/layer10_attention_weight_min": -33.90000534057617,
      "activations/layer11_attention_weight_max": 37.802913665771484,
      "activations/layer11_attention_weight_min": -28.82447624206543,
      "activations/layer12_attention_weight_max": 33.53204345703125,
      "activations/layer12_attention_weight_min": -28.529741287231445,
      "activations/layer13_attention_weight_max": 51.050758361816406,
      "activations/layer13_attention_weight_min": -34.745201110839844,
      "activations/layer14_attention_weight_max": 77.02537536621094,
      "activations/layer14_attention_weight_min": -58.6435546875,
      "activations/layer15_attention_weight_max": 52.242252349853516,
      "activations/layer15_attention_weight_min": -33.52925491333008,
      "activations/layer16_attention_weight_max": 50.64410400390625,
      "activations/layer16_attention_weight_min": -32.135196685791016,
      "activations/layer17_attention_weight_max": 46.785743713378906,
      "activations/layer17_attention_weight_min": -29.58089828491211,
      "activations/layer18_attention_weight_max": 31.904415130615234,
      "activations/layer18_attention_weight_min": -19.032323837280273,
      "activations/layer19_attention_weight_max": 48.96973419189453,
      "activations/layer19_attention_weight_min": -29.617923736572266,
      "activations/layer1_attention_weight_max": 17.551015853881836,
      "activations/layer1_attention_weight_min": -13.712635040283203,
      "activations/layer20_attention_weight_max": 36.18703842163086,
      "activations/layer20_attention_weight_min": -24.467594146728516,
      "activations/layer21_attention_weight_max": 34.84373474121094,
      "activations/layer21_attention_weight_min": -20.13697052001953,
      "activations/layer22_attention_weight_max": 51.98026657104492,
      "activations/layer22_attention_weight_min": -28.986602783203125,
      "activations/layer23_attention_weight_max": 28.28229331970215,
      "activations/layer23_attention_weight_min": -19.706985473632812,
      "activations/layer2_attention_weight_max": 31.506301879882812,
      "activations/layer2_attention_weight_min": -31.24111557006836,
      "activations/layer3_attention_weight_max": 85.96892547607422,
      "activations/layer3_attention_weight_min": -86.52767944335938,
      "activations/layer4_attention_weight_max": 131.11618041992188,
      "activations/layer4_attention_weight_min": -126.4962158203125,
      "activations/layer5_attention_weight_max": 100.70426940917969,
      "activations/layer5_attention_weight_min": -89.93675994873047,
      "activations/layer6_attention_weight_max": 57.66328048706055,
      "activations/layer6_attention_weight_min": -54.536781311035156,
      "activations/layer7_attention_weight_max": 76.42361450195312,
      "activations/layer7_attention_weight_min": -71.6107177734375,
      "activations/layer8_attention_weight_max": 42.65102005004883,
      "activations/layer8_attention_weight_min": -36.65495681762695,
      "activations/layer9_attention_weight_max": 44.94279861450195,
      "activations/layer9_attention_weight_min": -40.22042465209961,
      "epoch": 3.25,
      "learning_rate": 0.0001303647727272727,
      "loss": 2.9048,
      "step": 55900
    },
    {
      "activations/layer0_attention_weight_max": 17.311376571655273,
      "activations/layer0_attention_weight_min": -11.326753616333008,
      "activations/layer10_attention_weight_max": 43.73494338989258,
      "activations/layer10_attention_weight_min": -33.8436393737793,
      "activations/layer11_attention_weight_max": 34.44090270996094,
      "activations/layer11_attention_weight_min": -29.672107696533203,
      "activations/layer12_attention_weight_max": 34.09000778198242,
      "activations/layer12_attention_weight_min": -27.968544006347656,
      "activations/layer13_attention_weight_max": 49.654396057128906,
      "activations/layer13_attention_weight_min": -33.69857406616211,
      "activations/layer14_attention_weight_max": 77.64376831054688,
      "activations/layer14_attention_weight_min": -49.87437438964844,
      "activations/layer15_attention_weight_max": 51.70878982543945,
      "activations/layer15_attention_weight_min": -32.86271667480469,
      "activations/layer16_attention_weight_max": 48.23150634765625,
      "activations/layer16_attention_weight_min": -32.88187789916992,
      "activations/layer17_attention_weight_max": 43.99342727661133,
      "activations/layer17_attention_weight_min": -29.387128829956055,
      "activations/layer18_attention_weight_max": 35.66322708129883,
      "activations/layer18_attention_weight_min": -20.465566635131836,
      "activations/layer19_attention_weight_max": 49.56890869140625,
      "activations/layer19_attention_weight_min": -29.135953903198242,
      "activations/layer1_attention_weight_max": 18.404756546020508,
      "activations/layer1_attention_weight_min": -13.896717071533203,
      "activations/layer20_attention_weight_max": 36.921077728271484,
      "activations/layer20_attention_weight_min": -24.559648513793945,
      "activations/layer21_attention_weight_max": 36.306365966796875,
      "activations/layer21_attention_weight_min": -21.844491958618164,
      "activations/layer22_attention_weight_max": 52.913612365722656,
      "activations/layer22_attention_weight_min": -26.786184310913086,
      "activations/layer23_attention_weight_max": 31.996379852294922,
      "activations/layer23_attention_weight_min": -16.670440673828125,
      "activations/layer2_attention_weight_max": 30.570926666259766,
      "activations/layer2_attention_weight_min": -30.914051055908203,
      "activations/layer3_attention_weight_max": 80.31366729736328,
      "activations/layer3_attention_weight_min": -82.65924835205078,
      "activations/layer4_attention_weight_max": 125.27986907958984,
      "activations/layer4_attention_weight_min": -125.42447662353516,
      "activations/layer5_attention_weight_max": 94.35966491699219,
      "activations/layer5_attention_weight_min": -89.39797973632812,
      "activations/layer6_attention_weight_max": 57.01866912841797,
      "activations/layer6_attention_weight_min": -53.77405548095703,
      "activations/layer7_attention_weight_max": 70.05626678466797,
      "activations/layer7_attention_weight_min": -70.11276245117188,
      "activations/layer8_attention_weight_max": 37.59674835205078,
      "activations/layer8_attention_weight_min": -37.11237335205078,
      "activations/layer9_attention_weight_max": 44.20549011230469,
      "activations/layer9_attention_weight_min": -45.757102966308594,
      "epoch": 3.25,
      "learning_rate": 0.00013034583333333333,
      "loss": 2.8929,
      "step": 55950
    },
    {
      "activations/layer0_attention_weight_max": 14.838744163513184,
      "activations/layer0_attention_weight_min": -11.793453216552734,
      "activations/layer10_attention_weight_max": 45.9960823059082,
      "activations/layer10_attention_weight_min": -30.75949478149414,
      "activations/layer11_attention_weight_max": 38.74994659423828,
      "activations/layer11_attention_weight_min": -28.13282012939453,
      "activations/layer12_attention_weight_max": 68.07258605957031,
      "activations/layer12_attention_weight_min": -32.6600456237793,
      "activations/layer13_attention_weight_max": 65.83748626708984,
      "activations/layer13_attention_weight_min": -43.1484489440918,
      "activations/layer14_attention_weight_max": 95.38308715820312,
      "activations/layer14_attention_weight_min": -60.26962661743164,
      "activations/layer15_attention_weight_max": 51.8530387878418,
      "activations/layer15_attention_weight_min": -30.241689682006836,
      "activations/layer16_attention_weight_max": 50.618858337402344,
      "activations/layer16_attention_weight_min": -28.84232521057129,
      "activations/layer17_attention_weight_max": 46.21409606933594,
      "activations/layer17_attention_weight_min": -24.239194869995117,
      "activations/layer18_attention_weight_max": 37.64312744140625,
      "activations/layer18_attention_weight_min": -21.848840713500977,
      "activations/layer19_attention_weight_max": 45.512413024902344,
      "activations/layer19_attention_weight_min": -28.343585968017578,
      "activations/layer1_attention_weight_max": 17.68558120727539,
      "activations/layer1_attention_weight_min": -15.884033203125,
      "activations/layer20_attention_weight_max": 36.45392608642578,
      "activations/layer20_attention_weight_min": -21.12635612487793,
      "activations/layer21_attention_weight_max": 30.43840217590332,
      "activations/layer21_attention_weight_min": -19.6109561920166,
      "activations/layer22_attention_weight_max": 53.489349365234375,
      "activations/layer22_attention_weight_min": -25.774553298950195,
      "activations/layer23_attention_weight_max": 28.373085021972656,
      "activations/layer23_attention_weight_min": -18.223949432373047,
      "activations/layer2_attention_weight_max": 29.391700744628906,
      "activations/layer2_attention_weight_min": -30.68073081970215,
      "activations/layer3_attention_weight_max": 78.31766510009766,
      "activations/layer3_attention_weight_min": -81.82527923583984,
      "activations/layer4_attention_weight_max": 130.08285522460938,
      "activations/layer4_attention_weight_min": -125.4892578125,
      "activations/layer5_attention_weight_max": 98.30498504638672,
      "activations/layer5_attention_weight_min": -91.89080047607422,
      "activations/layer6_attention_weight_max": 59.02775573730469,
      "activations/layer6_attention_weight_min": -51.745304107666016,
      "activations/layer7_attention_weight_max": 74.45555877685547,
      "activations/layer7_attention_weight_min": -71.28538513183594,
      "activations/layer8_attention_weight_max": 39.221893310546875,
      "activations/layer8_attention_weight_min": -37.56660842895508,
      "activations/layer9_attention_weight_max": 54.108882904052734,
      "activations/layer9_attention_weight_min": -39.11351776123047,
      "epoch": 3.25,
      "learning_rate": 0.00013032689393939392,
      "loss": 2.9,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_loss": 2.849609375,
      "eval_runtime": 8.466,
      "eval_samples_per_second": 507.208,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_openwebtext_loss": 2.849609375,
      "eval_openwebtext_ppl": 17.28103011956753,
      "eval_openwebtext_runtime": 8.466,
      "eval_openwebtext_samples_per_second": 507.208,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_wikitext_loss": 3.0859375,
      "eval_wikitext_ppl": 21.887977204970085,
      "eval_wikitext_runtime": 2.0611,
      "eval_wikitext_samples_per_second": 221.24,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_lambada_loss": 2.87890625,
      "eval_lambada_ppl": 17.79479946994832,
      "eval_lambada_runtime": 9.6028,
      "eval_lambada_samples_per_second": 507.041,
      "step": 56000
    },
    {
      "activations/layer0_attention_weight_max": 16.908464431762695,
      "activations/layer0_attention_weight_min": -10.717025756835938,
      "activations/layer10_attention_weight_max": 39.58062744140625,
      "activations/layer10_attention_weight_min": -32.0955696105957,
      "activations/layer11_attention_weight_max": 34.82870101928711,
      "activations/layer11_attention_weight_min": -31.493505477905273,
      "activations/layer12_attention_weight_max": 32.62409210205078,
      "activations/layer12_attention_weight_min": -28.1530818939209,
      "activations/layer13_attention_weight_max": 52.62569808959961,
      "activations/layer13_attention_weight_min": -38.84455871582031,
      "activations/layer14_attention_weight_max": 75.75110626220703,
      "activations/layer14_attention_weight_min": -62.18471908569336,
      "activations/layer15_attention_weight_max": 45.69782638549805,
      "activations/layer15_attention_weight_min": -33.56759262084961,
      "activations/layer16_attention_weight_max": 42.722843170166016,
      "activations/layer16_attention_weight_min": -30.773664474487305,
      "activations/layer17_attention_weight_max": 43.36623001098633,
      "activations/layer17_attention_weight_min": -27.818157196044922,
      "activations/layer18_attention_weight_max": 28.548357009887695,
      "activations/layer18_attention_weight_min": -19.362380981445312,
      "activations/layer19_attention_weight_max": 45.063045501708984,
      "activations/layer19_attention_weight_min": -30.285627365112305,
      "activations/layer1_attention_weight_max": 18.817216873168945,
      "activations/layer1_attention_weight_min": -13.273143768310547,
      "activations/layer20_attention_weight_max": 35.813053131103516,
      "activations/layer20_attention_weight_min": -22.196880340576172,
      "activations/layer21_attention_weight_max": 33.14534378051758,
      "activations/layer21_attention_weight_min": -20.385101318359375,
      "activations/layer22_attention_weight_max": 51.992103576660156,
      "activations/layer22_attention_weight_min": -26.813793182373047,
      "activations/layer23_attention_weight_max": 28.818754196166992,
      "activations/layer23_attention_weight_min": -16.747501373291016,
      "activations/layer2_attention_weight_max": 31.5941219329834,
      "activations/layer2_attention_weight_min": -29.908613204956055,
      "activations/layer3_attention_weight_max": 86.64442443847656,
      "activations/layer3_attention_weight_min": -85.54439544677734,
      "activations/layer4_attention_weight_max": 140.3949432373047,
      "activations/layer4_attention_weight_min": -124.71868133544922,
      "activations/layer5_attention_weight_max": 101.71004486083984,
      "activations/layer5_attention_weight_min": -93.1856689453125,
      "activations/layer6_attention_weight_max": 58.342742919921875,
      "activations/layer6_attention_weight_min": -56.0168342590332,
      "activations/layer7_attention_weight_max": 72.98436737060547,
      "activations/layer7_attention_weight_min": -70.90557098388672,
      "activations/layer8_attention_weight_max": 41.95094680786133,
      "activations/layer8_attention_weight_min": -38.19964599609375,
      "activations/layer9_attention_weight_max": 42.71938705444336,
      "activations/layer9_attention_weight_min": -39.80781555175781,
      "epoch": 3.26,
      "learning_rate": 0.00013030795454545453,
      "loss": 2.9149,
      "step": 56050
    },
    {
      "activations/layer0_attention_weight_max": 17.524307250976562,
      "activations/layer0_attention_weight_min": -11.637151718139648,
      "activations/layer10_attention_weight_max": 39.5633430480957,
      "activations/layer10_attention_weight_min": -33.15265655517578,
      "activations/layer11_attention_weight_max": 34.640621185302734,
      "activations/layer11_attention_weight_min": -29.365966796875,
      "activations/layer12_attention_weight_max": 40.40689468383789,
      "activations/layer12_attention_weight_min": -29.171239852905273,
      "activations/layer13_attention_weight_max": 46.8355598449707,
      "activations/layer13_attention_weight_min": -38.875244140625,
      "activations/layer14_attention_weight_max": 74.76972961425781,
      "activations/layer14_attention_weight_min": -58.55286407470703,
      "activations/layer15_attention_weight_max": 45.021183013916016,
      "activations/layer15_attention_weight_min": -30.52777671813965,
      "activations/layer16_attention_weight_max": 47.965728759765625,
      "activations/layer16_attention_weight_min": -33.87018585205078,
      "activations/layer17_attention_weight_max": 46.97868728637695,
      "activations/layer17_attention_weight_min": -26.56414794921875,
      "activations/layer18_attention_weight_max": 31.754533767700195,
      "activations/layer18_attention_weight_min": -19.269611358642578,
      "activations/layer19_attention_weight_max": 47.00046920776367,
      "activations/layer19_attention_weight_min": -28.493478775024414,
      "activations/layer1_attention_weight_max": 17.817461013793945,
      "activations/layer1_attention_weight_min": -13.511260032653809,
      "activations/layer20_attention_weight_max": 33.41054153442383,
      "activations/layer20_attention_weight_min": -25.111665725708008,
      "activations/layer21_attention_weight_max": 32.86421585083008,
      "activations/layer21_attention_weight_min": -21.912199020385742,
      "activations/layer22_attention_weight_max": 48.288818359375,
      "activations/layer22_attention_weight_min": -25.83296012878418,
      "activations/layer23_attention_weight_max": 25.53639030456543,
      "activations/layer23_attention_weight_min": -15.872305870056152,
      "activations/layer2_attention_weight_max": 30.086225509643555,
      "activations/layer2_attention_weight_min": -29.671669006347656,
      "activations/layer3_attention_weight_max": 77.58092498779297,
      "activations/layer3_attention_weight_min": -80.0842056274414,
      "activations/layer4_attention_weight_max": 126.1122817993164,
      "activations/layer4_attention_weight_min": -120.14580535888672,
      "activations/layer5_attention_weight_max": 96.42250061035156,
      "activations/layer5_attention_weight_min": -84.42823791503906,
      "activations/layer6_attention_weight_max": 56.73322677612305,
      "activations/layer6_attention_weight_min": -51.26711654663086,
      "activations/layer7_attention_weight_max": 73.5374755859375,
      "activations/layer7_attention_weight_min": -70.8938980102539,
      "activations/layer8_attention_weight_max": 40.467044830322266,
      "activations/layer8_attention_weight_min": -40.12529754638672,
      "activations/layer9_attention_weight_max": 46.36888885498047,
      "activations/layer9_attention_weight_min": -33.96329116821289,
      "epoch": 3.26,
      "learning_rate": 0.00013028901515151515,
      "loss": 2.904,
      "step": 56100
    },
    {
      "activations/layer0_attention_weight_max": 16.787025451660156,
      "activations/layer0_attention_weight_min": -12.058870315551758,
      "activations/layer10_attention_weight_max": 37.95022964477539,
      "activations/layer10_attention_weight_min": -32.1276741027832,
      "activations/layer11_attention_weight_max": 34.222694396972656,
      "activations/layer11_attention_weight_min": -30.10213279724121,
      "activations/layer12_attention_weight_max": 34.194087982177734,
      "activations/layer12_attention_weight_min": -28.700294494628906,
      "activations/layer13_attention_weight_max": 45.96531677246094,
      "activations/layer13_attention_weight_min": -34.928733825683594,
      "activations/layer14_attention_weight_max": 71.81832122802734,
      "activations/layer14_attention_weight_min": -46.86996841430664,
      "activations/layer15_attention_weight_max": 45.80145263671875,
      "activations/layer15_attention_weight_min": -31.421724319458008,
      "activations/layer16_attention_weight_max": 46.85184860229492,
      "activations/layer16_attention_weight_min": -32.823509216308594,
      "activations/layer17_attention_weight_max": 43.42951583862305,
      "activations/layer17_attention_weight_min": -27.800546646118164,
      "activations/layer18_attention_weight_max": 34.756805419921875,
      "activations/layer18_attention_weight_min": -21.33189582824707,
      "activations/layer19_attention_weight_max": 46.3603630065918,
      "activations/layer19_attention_weight_min": -29.656219482421875,
      "activations/layer1_attention_weight_max": 17.800708770751953,
      "activations/layer1_attention_weight_min": -13.13422966003418,
      "activations/layer20_attention_weight_max": 35.446266174316406,
      "activations/layer20_attention_weight_min": -24.797941207885742,
      "activations/layer21_attention_weight_max": 34.0072135925293,
      "activations/layer21_attention_weight_min": -19.91924476623535,
      "activations/layer22_attention_weight_max": 51.572021484375,
      "activations/layer22_attention_weight_min": -28.288238525390625,
      "activations/layer23_attention_weight_max": 29.31719970703125,
      "activations/layer23_attention_weight_min": -18.44099235534668,
      "activations/layer2_attention_weight_max": 29.965200424194336,
      "activations/layer2_attention_weight_min": -30.51797103881836,
      "activations/layer3_attention_weight_max": 84.03131866455078,
      "activations/layer3_attention_weight_min": -86.16950225830078,
      "activations/layer4_attention_weight_max": 126.7237548828125,
      "activations/layer4_attention_weight_min": -124.93668365478516,
      "activations/layer5_attention_weight_max": 96.9945297241211,
      "activations/layer5_attention_weight_min": -86.93367004394531,
      "activations/layer6_attention_weight_max": 54.47297286987305,
      "activations/layer6_attention_weight_min": -50.43315887451172,
      "activations/layer7_attention_weight_max": 70.73138427734375,
      "activations/layer7_attention_weight_min": -66.89505767822266,
      "activations/layer8_attention_weight_max": 38.96818923950195,
      "activations/layer8_attention_weight_min": -35.276451110839844,
      "activations/layer9_attention_weight_max": 44.03146743774414,
      "activations/layer9_attention_weight_min": -34.69790267944336,
      "epoch": 3.26,
      "learning_rate": 0.00013027007575757573,
      "loss": 2.8986,
      "step": 56150
    },
    {
      "activations/layer0_attention_weight_max": 17.479501724243164,
      "activations/layer0_attention_weight_min": -11.47786808013916,
      "activations/layer10_attention_weight_max": 41.97854995727539,
      "activations/layer10_attention_weight_min": -31.90243911743164,
      "activations/layer11_attention_weight_max": 41.71885681152344,
      "activations/layer11_attention_weight_min": -29.094738006591797,
      "activations/layer12_attention_weight_max": 33.22853469848633,
      "activations/layer12_attention_weight_min": -26.119056701660156,
      "activations/layer13_attention_weight_max": 59.09419631958008,
      "activations/layer13_attention_weight_min": -38.088111877441406,
      "activations/layer14_attention_weight_max": 75.82935333251953,
      "activations/layer14_attention_weight_min": -52.29335403442383,
      "activations/layer15_attention_weight_max": 55.201595306396484,
      "activations/layer15_attention_weight_min": -31.851627349853516,
      "activations/layer16_attention_weight_max": 46.39063262939453,
      "activations/layer16_attention_weight_min": -30.638639450073242,
      "activations/layer17_attention_weight_max": 52.29323196411133,
      "activations/layer17_attention_weight_min": -30.30015754699707,
      "activations/layer18_attention_weight_max": 33.639801025390625,
      "activations/layer18_attention_weight_min": -18.795984268188477,
      "activations/layer19_attention_weight_max": 53.680274963378906,
      "activations/layer19_attention_weight_min": -28.7170467376709,
      "activations/layer1_attention_weight_max": 17.278589248657227,
      "activations/layer1_attention_weight_min": -13.312013626098633,
      "activations/layer20_attention_weight_max": 45.523529052734375,
      "activations/layer20_attention_weight_min": -24.449710845947266,
      "activations/layer21_attention_weight_max": 45.46207809448242,
      "activations/layer21_attention_weight_min": -22.493408203125,
      "activations/layer22_attention_weight_max": 58.96988296508789,
      "activations/layer22_attention_weight_min": -25.22534942626953,
      "activations/layer23_attention_weight_max": 29.915420532226562,
      "activations/layer23_attention_weight_min": -16.708602905273438,
      "activations/layer2_attention_weight_max": 30.796188354492188,
      "activations/layer2_attention_weight_min": -30.738615036010742,
      "activations/layer3_attention_weight_max": 83.78720092773438,
      "activations/layer3_attention_weight_min": -85.53263092041016,
      "activations/layer4_attention_weight_max": 134.56607055664062,
      "activations/layer4_attention_weight_min": -128.8606414794922,
      "activations/layer5_attention_weight_max": 102.31688690185547,
      "activations/layer5_attention_weight_min": -92.76931762695312,
      "activations/layer6_attention_weight_max": 60.20161819458008,
      "activations/layer6_attention_weight_min": -54.56748962402344,
      "activations/layer7_attention_weight_max": 75.64063262939453,
      "activations/layer7_attention_weight_min": -70.22390747070312,
      "activations/layer8_attention_weight_max": 41.12941360473633,
      "activations/layer8_attention_weight_min": -37.052852630615234,
      "activations/layer9_attention_weight_max": 47.1998291015625,
      "activations/layer9_attention_weight_min": -32.364768981933594,
      "epoch": 3.27,
      "learning_rate": 0.00013025113636363635,
      "loss": 2.8918,
      "step": 56200
    },
    {
      "activations/layer0_attention_weight_max": 14.142617225646973,
      "activations/layer0_attention_weight_min": -12.3622465133667,
      "activations/layer10_attention_weight_max": 41.26110076904297,
      "activations/layer10_attention_weight_min": -34.447113037109375,
      "activations/layer11_attention_weight_max": 42.599056243896484,
      "activations/layer11_attention_weight_min": -33.00035095214844,
      "activations/layer12_attention_weight_max": 33.892852783203125,
      "activations/layer12_attention_weight_min": -29.823169708251953,
      "activations/layer13_attention_weight_max": 57.25581741333008,
      "activations/layer13_attention_weight_min": -34.824119567871094,
      "activations/layer14_attention_weight_max": 87.57682800292969,
      "activations/layer14_attention_weight_min": -56.855682373046875,
      "activations/layer15_attention_weight_max": 55.0751953125,
      "activations/layer15_attention_weight_min": -29.392663955688477,
      "activations/layer16_attention_weight_max": 48.12187576293945,
      "activations/layer16_attention_weight_min": -30.687604904174805,
      "activations/layer17_attention_weight_max": 47.64177703857422,
      "activations/layer17_attention_weight_min": -26.05922508239746,
      "activations/layer18_attention_weight_max": 38.67797088623047,
      "activations/layer18_attention_weight_min": -19.329252243041992,
      "activations/layer19_attention_weight_max": 50.54375076293945,
      "activations/layer19_attention_weight_min": -29.07594108581543,
      "activations/layer1_attention_weight_max": 18.417551040649414,
      "activations/layer1_attention_weight_min": -13.042543411254883,
      "activations/layer20_attention_weight_max": 37.357845306396484,
      "activations/layer20_attention_weight_min": -22.747739791870117,
      "activations/layer21_attention_weight_max": 38.094970703125,
      "activations/layer21_attention_weight_min": -19.778276443481445,
      "activations/layer22_attention_weight_max": 59.67390823364258,
      "activations/layer22_attention_weight_min": -29.024335861206055,
      "activations/layer23_attention_weight_max": 31.36829376220703,
      "activations/layer23_attention_weight_min": -17.185102462768555,
      "activations/layer2_attention_weight_max": 30.88909912109375,
      "activations/layer2_attention_weight_min": -29.63873291015625,
      "activations/layer3_attention_weight_max": 81.80957794189453,
      "activations/layer3_attention_weight_min": -84.45047760009766,
      "activations/layer4_attention_weight_max": 128.61050415039062,
      "activations/layer4_attention_weight_min": -122.85823822021484,
      "activations/layer5_attention_weight_max": 96.02166748046875,
      "activations/layer5_attention_weight_min": -87.40056610107422,
      "activations/layer6_attention_weight_max": 58.40964889526367,
      "activations/layer6_attention_weight_min": -52.737911224365234,
      "activations/layer7_attention_weight_max": 85.27617645263672,
      "activations/layer7_attention_weight_min": -71.86656188964844,
      "activations/layer8_attention_weight_max": 42.238101959228516,
      "activations/layer8_attention_weight_min": -37.711917877197266,
      "activations/layer9_attention_weight_max": 57.33668899536133,
      "activations/layer9_attention_weight_min": -41.055118560791016,
      "epoch": 3.27,
      "learning_rate": 0.00013023219696969694,
      "loss": 2.9107,
      "step": 56250
    },
    {
      "activations/layer0_attention_weight_max": 16.9291934967041,
      "activations/layer0_attention_weight_min": -11.517790794372559,
      "activations/layer10_attention_weight_max": 40.870670318603516,
      "activations/layer10_attention_weight_min": -33.08404541015625,
      "activations/layer11_attention_weight_max": 39.780799865722656,
      "activations/layer11_attention_weight_min": -30.529708862304688,
      "activations/layer12_attention_weight_max": 30.066675186157227,
      "activations/layer12_attention_weight_min": -26.33090591430664,
      "activations/layer13_attention_weight_max": 51.034568786621094,
      "activations/layer13_attention_weight_min": -34.93778991699219,
      "activations/layer14_attention_weight_max": 74.1129379272461,
      "activations/layer14_attention_weight_min": -56.21917724609375,
      "activations/layer15_attention_weight_max": 49.90331268310547,
      "activations/layer15_attention_weight_min": -30.971426010131836,
      "activations/layer16_attention_weight_max": 41.69915771484375,
      "activations/layer16_attention_weight_min": -30.88097381591797,
      "activations/layer17_attention_weight_max": 47.186702728271484,
      "activations/layer17_attention_weight_min": -26.09996795654297,
      "activations/layer18_attention_weight_max": 33.831153869628906,
      "activations/layer18_attention_weight_min": -18.775705337524414,
      "activations/layer19_attention_weight_max": 47.72089767456055,
      "activations/layer19_attention_weight_min": -29.248620986938477,
      "activations/layer1_attention_weight_max": 17.463483810424805,
      "activations/layer1_attention_weight_min": -14.687471389770508,
      "activations/layer20_attention_weight_max": 35.79414367675781,
      "activations/layer20_attention_weight_min": -24.06264877319336,
      "activations/layer21_attention_weight_max": 33.8264274597168,
      "activations/layer21_attention_weight_min": -23.33599281311035,
      "activations/layer22_attention_weight_max": 53.509910583496094,
      "activations/layer22_attention_weight_min": -27.320995330810547,
      "activations/layer23_attention_weight_max": 30.399593353271484,
      "activations/layer23_attention_weight_min": -16.436172485351562,
      "activations/layer2_attention_weight_max": 30.945669174194336,
      "activations/layer2_attention_weight_min": -30.80231285095215,
      "activations/layer3_attention_weight_max": 81.87681579589844,
      "activations/layer3_attention_weight_min": -86.65016174316406,
      "activations/layer4_attention_weight_max": 124.2852554321289,
      "activations/layer4_attention_weight_min": -122.928466796875,
      "activations/layer5_attention_weight_max": 97.97661590576172,
      "activations/layer5_attention_weight_min": -88.71031951904297,
      "activations/layer6_attention_weight_max": 61.439056396484375,
      "activations/layer6_attention_weight_min": -53.098262786865234,
      "activations/layer7_attention_weight_max": 71.38700866699219,
      "activations/layer7_attention_weight_min": -75.73152160644531,
      "activations/layer8_attention_weight_max": 40.07767868041992,
      "activations/layer8_attention_weight_min": -37.65868377685547,
      "activations/layer9_attention_weight_max": 47.20427703857422,
      "activations/layer9_attention_weight_min": -37.559871673583984,
      "epoch": 3.27,
      "learning_rate": 0.00013021325757575755,
      "loss": 2.8905,
      "step": 56300
    },
    {
      "activations/layer0_attention_weight_max": 17.042558670043945,
      "activations/layer0_attention_weight_min": -11.546895980834961,
      "activations/layer10_attention_weight_max": 39.11231231689453,
      "activations/layer10_attention_weight_min": -30.986576080322266,
      "activations/layer11_attention_weight_max": 37.136192321777344,
      "activations/layer11_attention_weight_min": -29.365375518798828,
      "activations/layer12_attention_weight_max": 35.24497985839844,
      "activations/layer12_attention_weight_min": -25.129852294921875,
      "activations/layer13_attention_weight_max": 51.389076232910156,
      "activations/layer13_attention_weight_min": -36.36114501953125,
      "activations/layer14_attention_weight_max": 69.8879623413086,
      "activations/layer14_attention_weight_min": -49.775665283203125,
      "activations/layer15_attention_weight_max": 45.59170150756836,
      "activations/layer15_attention_weight_min": -30.44925880432129,
      "activations/layer16_attention_weight_max": 47.27304458618164,
      "activations/layer16_attention_weight_min": -31.23543357849121,
      "activations/layer17_attention_weight_max": 45.246952056884766,
      "activations/layer17_attention_weight_min": -27.443811416625977,
      "activations/layer18_attention_weight_max": 35.9331169128418,
      "activations/layer18_attention_weight_min": -19.573589324951172,
      "activations/layer19_attention_weight_max": 53.19257354736328,
      "activations/layer19_attention_weight_min": -28.8417911529541,
      "activations/layer1_attention_weight_max": 17.165943145751953,
      "activations/layer1_attention_weight_min": -13.463458061218262,
      "activations/layer20_attention_weight_max": 38.93877410888672,
      "activations/layer20_attention_weight_min": -22.223804473876953,
      "activations/layer21_attention_weight_max": 39.73895263671875,
      "activations/layer21_attention_weight_min": -21.74068260192871,
      "activations/layer22_attention_weight_max": 54.59654235839844,
      "activations/layer22_attention_weight_min": -26.74208641052246,
      "activations/layer23_attention_weight_max": 31.477048873901367,
      "activations/layer23_attention_weight_min": -18.456966400146484,
      "activations/layer2_attention_weight_max": 30.642484664916992,
      "activations/layer2_attention_weight_min": -31.046178817749023,
      "activations/layer3_attention_weight_max": 82.4380111694336,
      "activations/layer3_attention_weight_min": -85.38088989257812,
      "activations/layer4_attention_weight_max": 130.947509765625,
      "activations/layer4_attention_weight_min": -120.32330322265625,
      "activations/layer5_attention_weight_max": 95.86795043945312,
      "activations/layer5_attention_weight_min": -87.77165222167969,
      "activations/layer6_attention_weight_max": 57.084007263183594,
      "activations/layer6_attention_weight_min": -50.95268249511719,
      "activations/layer7_attention_weight_max": 79.6695785522461,
      "activations/layer7_attention_weight_min": -74.97511291503906,
      "activations/layer8_attention_weight_max": 40.0677604675293,
      "activations/layer8_attention_weight_min": -35.39986038208008,
      "activations/layer9_attention_weight_max": 43.850685119628906,
      "activations/layer9_attention_weight_min": -39.94717025756836,
      "epoch": 3.27,
      "learning_rate": 0.00013019431818181817,
      "loss": 2.8926,
      "step": 56350
    },
    {
      "activations/layer0_attention_weight_max": 16.94384765625,
      "activations/layer0_attention_weight_min": -12.667593955993652,
      "activations/layer10_attention_weight_max": 38.8773307800293,
      "activations/layer10_attention_weight_min": -33.54948806762695,
      "activations/layer11_attention_weight_max": 36.46388626098633,
      "activations/layer11_attention_weight_min": -29.941993713378906,
      "activations/layer12_attention_weight_max": 34.670509338378906,
      "activations/layer12_attention_weight_min": -25.72776222229004,
      "activations/layer13_attention_weight_max": 41.82301712036133,
      "activations/layer13_attention_weight_min": -31.921342849731445,
      "activations/layer14_attention_weight_max": 62.329532623291016,
      "activations/layer14_attention_weight_min": -49.56122970581055,
      "activations/layer15_attention_weight_max": 43.46497344970703,
      "activations/layer15_attention_weight_min": -34.40005111694336,
      "activations/layer16_attention_weight_max": 39.9787712097168,
      "activations/layer16_attention_weight_min": -29.921972274780273,
      "activations/layer17_attention_weight_max": 39.19496536254883,
      "activations/layer17_attention_weight_min": -28.013416290283203,
      "activations/layer18_attention_weight_max": 33.90877914428711,
      "activations/layer18_attention_weight_min": -21.94247817993164,
      "activations/layer19_attention_weight_max": 39.63308334350586,
      "activations/layer19_attention_weight_min": -31.1163387298584,
      "activations/layer1_attention_weight_max": 16.924468994140625,
      "activations/layer1_attention_weight_min": -14.132734298706055,
      "activations/layer20_attention_weight_max": 35.09958267211914,
      "activations/layer20_attention_weight_min": -22.235328674316406,
      "activations/layer21_attention_weight_max": 33.18959045410156,
      "activations/layer21_attention_weight_min": -21.77644920349121,
      "activations/layer22_attention_weight_max": 47.53325271606445,
      "activations/layer22_attention_weight_min": -29.786415100097656,
      "activations/layer23_attention_weight_max": 28.22210693359375,
      "activations/layer23_attention_weight_min": -17.560243606567383,
      "activations/layer2_attention_weight_max": 29.591999053955078,
      "activations/layer2_attention_weight_min": -29.342601776123047,
      "activations/layer3_attention_weight_max": 77.60611724853516,
      "activations/layer3_attention_weight_min": -81.7094955444336,
      "activations/layer4_attention_weight_max": 122.5786361694336,
      "activations/layer4_attention_weight_min": -118.2695541381836,
      "activations/layer5_attention_weight_max": 93.59814453125,
      "activations/layer5_attention_weight_min": -86.481201171875,
      "activations/layer6_attention_weight_max": 54.71104431152344,
      "activations/layer6_attention_weight_min": -52.35338592529297,
      "activations/layer7_attention_weight_max": 67.16127014160156,
      "activations/layer7_attention_weight_min": -71.28883361816406,
      "activations/layer8_attention_weight_max": 37.8423957824707,
      "activations/layer8_attention_weight_min": -37.35744857788086,
      "activations/layer9_attention_weight_max": 42.29315185546875,
      "activations/layer9_attention_weight_min": -35.626461029052734,
      "epoch": 3.28,
      "learning_rate": 0.00013017537878787878,
      "loss": 2.8952,
      "step": 56400
    },
    {
      "activations/layer0_attention_weight_max": 14.497812271118164,
      "activations/layer0_attention_weight_min": -11.151089668273926,
      "activations/layer10_attention_weight_max": 39.231666564941406,
      "activations/layer10_attention_weight_min": -30.84331512451172,
      "activations/layer11_attention_weight_max": 34.653892517089844,
      "activations/layer11_attention_weight_min": -27.220434188842773,
      "activations/layer12_attention_weight_max": 29.959945678710938,
      "activations/layer12_attention_weight_min": -25.354772567749023,
      "activations/layer13_attention_weight_max": 46.03229904174805,
      "activations/layer13_attention_weight_min": -35.46187210083008,
      "activations/layer14_attention_weight_max": 73.97527313232422,
      "activations/layer14_attention_weight_min": -59.569427490234375,
      "activations/layer15_attention_weight_max": 50.74121856689453,
      "activations/layer15_attention_weight_min": -35.67299270629883,
      "activations/layer16_attention_weight_max": 43.36701583862305,
      "activations/layer16_attention_weight_min": -30.412248611450195,
      "activations/layer17_attention_weight_max": 41.79505157470703,
      "activations/layer17_attention_weight_min": -26.96166229248047,
      "activations/layer18_attention_weight_max": 32.22268295288086,
      "activations/layer18_attention_weight_min": -21.06524658203125,
      "activations/layer19_attention_weight_max": 41.806270599365234,
      "activations/layer19_attention_weight_min": -29.82122230529785,
      "activations/layer1_attention_weight_max": 17.979022979736328,
      "activations/layer1_attention_weight_min": -13.136327743530273,
      "activations/layer20_attention_weight_max": 36.43898391723633,
      "activations/layer20_attention_weight_min": -25.426639556884766,
      "activations/layer21_attention_weight_max": 41.88475036621094,
      "activations/layer21_attention_weight_min": -25.221275329589844,
      "activations/layer22_attention_weight_max": 50.59463882446289,
      "activations/layer22_attention_weight_min": -27.72690773010254,
      "activations/layer23_attention_weight_max": 28.892581939697266,
      "activations/layer23_attention_weight_min": -18.19412612915039,
      "activations/layer2_attention_weight_max": 29.549781799316406,
      "activations/layer2_attention_weight_min": -29.89920425415039,
      "activations/layer3_attention_weight_max": 79.31547546386719,
      "activations/layer3_attention_weight_min": -81.66508483886719,
      "activations/layer4_attention_weight_max": 133.2559814453125,
      "activations/layer4_attention_weight_min": -122.56729888916016,
      "activations/layer5_attention_weight_max": 101.61521911621094,
      "activations/layer5_attention_weight_min": -86.42858123779297,
      "activations/layer6_attention_weight_max": 61.240577697753906,
      "activations/layer6_attention_weight_min": -51.799251556396484,
      "activations/layer7_attention_weight_max": 78.67688751220703,
      "activations/layer7_attention_weight_min": -68.92959594726562,
      "activations/layer8_attention_weight_max": 39.54358673095703,
      "activations/layer8_attention_weight_min": -36.616607666015625,
      "activations/layer9_attention_weight_max": 58.15388107299805,
      "activations/layer9_attention_weight_min": -38.7389030456543,
      "epoch": 3.28,
      "learning_rate": 0.00013015643939393937,
      "loss": 2.9052,
      "step": 56450
    },
    {
      "activations/layer0_attention_weight_max": 17.03926658630371,
      "activations/layer0_attention_weight_min": -13.199588775634766,
      "activations/layer10_attention_weight_max": 36.40837478637695,
      "activations/layer10_attention_weight_min": -30.528873443603516,
      "activations/layer11_attention_weight_max": 35.11575698852539,
      "activations/layer11_attention_weight_min": -28.797231674194336,
      "activations/layer12_attention_weight_max": 29.523679733276367,
      "activations/layer12_attention_weight_min": -29.061721801757812,
      "activations/layer13_attention_weight_max": 44.06223678588867,
      "activations/layer13_attention_weight_min": -33.9031982421875,
      "activations/layer14_attention_weight_max": 63.2927131652832,
      "activations/layer14_attention_weight_min": -47.47652816772461,
      "activations/layer15_attention_weight_max": 41.58005142211914,
      "activations/layer15_attention_weight_min": -31.079015731811523,
      "activations/layer16_attention_weight_max": 39.070674896240234,
      "activations/layer16_attention_weight_min": -30.751420974731445,
      "activations/layer17_attention_weight_max": 38.05507278442383,
      "activations/layer17_attention_weight_min": -26.989933013916016,
      "activations/layer18_attention_weight_max": 29.96063804626465,
      "activations/layer18_attention_weight_min": -18.850675582885742,
      "activations/layer19_attention_weight_max": 41.147605895996094,
      "activations/layer19_attention_weight_min": -32.01826858520508,
      "activations/layer1_attention_weight_max": 17.70415496826172,
      "activations/layer1_attention_weight_min": -14.23520565032959,
      "activations/layer20_attention_weight_max": 31.466524124145508,
      "activations/layer20_attention_weight_min": -22.997638702392578,
      "activations/layer21_attention_weight_max": 30.10190773010254,
      "activations/layer21_attention_weight_min": -21.040447235107422,
      "activations/layer22_attention_weight_max": 44.09046173095703,
      "activations/layer22_attention_weight_min": -26.949954986572266,
      "activations/layer23_attention_weight_max": 26.062999725341797,
      "activations/layer23_attention_weight_min": -17.995147705078125,
      "activations/layer2_attention_weight_max": 30.35462760925293,
      "activations/layer2_attention_weight_min": -30.78639030456543,
      "activations/layer3_attention_weight_max": 80.2889633178711,
      "activations/layer3_attention_weight_min": -81.59453582763672,
      "activations/layer4_attention_weight_max": 131.83216857910156,
      "activations/layer4_attention_weight_min": -124.0915298461914,
      "activations/layer5_attention_weight_max": 102.57683563232422,
      "activations/layer5_attention_weight_min": -91.92974853515625,
      "activations/layer6_attention_weight_max": 57.388614654541016,
      "activations/layer6_attention_weight_min": -52.383548736572266,
      "activations/layer7_attention_weight_max": 72.56314086914062,
      "activations/layer7_attention_weight_min": -68.06922912597656,
      "activations/layer8_attention_weight_max": 37.85100173950195,
      "activations/layer8_attention_weight_min": -34.98405075073242,
      "activations/layer9_attention_weight_max": 47.75958251953125,
      "activations/layer9_attention_weight_min": -41.24712371826172,
      "epoch": 3.28,
      "learning_rate": 0.00013013749999999999,
      "loss": 2.8855,
      "step": 56500
    },
    {
      "activations/layer0_attention_weight_max": 17.024673461914062,
      "activations/layer0_attention_weight_min": -11.844026565551758,
      "activations/layer10_attention_weight_max": 40.98268508911133,
      "activations/layer10_attention_weight_min": -31.93086814880371,
      "activations/layer11_attention_weight_max": 34.09020233154297,
      "activations/layer11_attention_weight_min": -31.031248092651367,
      "activations/layer12_attention_weight_max": 30.50839614868164,
      "activations/layer12_attention_weight_min": -25.43598747253418,
      "activations/layer13_attention_weight_max": 44.843082427978516,
      "activations/layer13_attention_weight_min": -32.785343170166016,
      "activations/layer14_attention_weight_max": 72.052490234375,
      "activations/layer14_attention_weight_min": -56.163177490234375,
      "activations/layer15_attention_weight_max": 44.32501983642578,
      "activations/layer15_attention_weight_min": -30.947107315063477,
      "activations/layer16_attention_weight_max": 43.03603744506836,
      "activations/layer16_attention_weight_min": -32.43696594238281,
      "activations/layer17_attention_weight_max": 40.75453186035156,
      "activations/layer17_attention_weight_min": -25.286270141601562,
      "activations/layer18_attention_weight_max": 28.780160903930664,
      "activations/layer18_attention_weight_min": -19.40721321105957,
      "activations/layer19_attention_weight_max": 42.0930061340332,
      "activations/layer19_attention_weight_min": -28.013187408447266,
      "activations/layer1_attention_weight_max": 17.287193298339844,
      "activations/layer1_attention_weight_min": -15.051875114440918,
      "activations/layer20_attention_weight_max": 35.77958679199219,
      "activations/layer20_attention_weight_min": -23.303611755371094,
      "activations/layer21_attention_weight_max": 38.332374572753906,
      "activations/layer21_attention_weight_min": -22.23619842529297,
      "activations/layer22_attention_weight_max": 49.83240509033203,
      "activations/layer22_attention_weight_min": -27.21573829650879,
      "activations/layer23_attention_weight_max": 29.808753967285156,
      "activations/layer23_attention_weight_min": -16.93094825744629,
      "activations/layer2_attention_weight_max": 30.817106246948242,
      "activations/layer2_attention_weight_min": -31.080947875976562,
      "activations/layer3_attention_weight_max": 79.8003158569336,
      "activations/layer3_attention_weight_min": -83.82662200927734,
      "activations/layer4_attention_weight_max": 123.7066421508789,
      "activations/layer4_attention_weight_min": -123.11376190185547,
      "activations/layer5_attention_weight_max": 96.37371826171875,
      "activations/layer5_attention_weight_min": -94.126708984375,
      "activations/layer6_attention_weight_max": 54.040138244628906,
      "activations/layer6_attention_weight_min": -54.2192268371582,
      "activations/layer7_attention_weight_max": 73.07095336914062,
      "activations/layer7_attention_weight_min": -69.28734588623047,
      "activations/layer8_attention_weight_max": 39.49934387207031,
      "activations/layer8_attention_weight_min": -36.31378936767578,
      "activations/layer9_attention_weight_max": 45.035011291503906,
      "activations/layer9_attention_weight_min": -38.949493408203125,
      "epoch": 3.29,
      "learning_rate": 0.0001301185606060606,
      "loss": 2.8925,
      "step": 56550
    },
    {
      "activations/layer0_attention_weight_max": 17.11980438232422,
      "activations/layer0_attention_weight_min": -11.282700538635254,
      "activations/layer10_attention_weight_max": 40.02153396606445,
      "activations/layer10_attention_weight_min": -32.03754806518555,
      "activations/layer11_attention_weight_max": 35.32756042480469,
      "activations/layer11_attention_weight_min": -27.206546783447266,
      "activations/layer12_attention_weight_max": 30.56946563720703,
      "activations/layer12_attention_weight_min": -26.96076011657715,
      "activations/layer13_attention_weight_max": 45.65833282470703,
      "activations/layer13_attention_weight_min": -33.90357971191406,
      "activations/layer14_attention_weight_max": 67.68345642089844,
      "activations/layer14_attention_weight_min": -49.98310852050781,
      "activations/layer15_attention_weight_max": 45.91770553588867,
      "activations/layer15_attention_weight_min": -29.439620971679688,
      "activations/layer16_attention_weight_max": 40.70082092285156,
      "activations/layer16_attention_weight_min": -28.784828186035156,
      "activations/layer17_attention_weight_max": 43.7887077331543,
      "activations/layer17_attention_weight_min": -26.403148651123047,
      "activations/layer18_attention_weight_max": 26.23454475402832,
      "activations/layer18_attention_weight_min": -17.89106559753418,
      "activations/layer19_attention_weight_max": 44.013790130615234,
      "activations/layer19_attention_weight_min": -28.032629013061523,
      "activations/layer1_attention_weight_max": 17.230180740356445,
      "activations/layer1_attention_weight_min": -13.446882247924805,
      "activations/layer20_attention_weight_max": 32.219078063964844,
      "activations/layer20_attention_weight_min": -23.781396865844727,
      "activations/layer21_attention_weight_max": 36.128387451171875,
      "activations/layer21_attention_weight_min": -22.689977645874023,
      "activations/layer22_attention_weight_max": 43.94242477416992,
      "activations/layer22_attention_weight_min": -28.35894203186035,
      "activations/layer23_attention_weight_max": 29.809749603271484,
      "activations/layer23_attention_weight_min": -18.71515655517578,
      "activations/layer2_attention_weight_max": 30.955917358398438,
      "activations/layer2_attention_weight_min": -30.304729461669922,
      "activations/layer3_attention_weight_max": 81.37430572509766,
      "activations/layer3_attention_weight_min": -84.97692108154297,
      "activations/layer4_attention_weight_max": 127.07193756103516,
      "activations/layer4_attention_weight_min": -121.9394302368164,
      "activations/layer5_attention_weight_max": 97.03254699707031,
      "activations/layer5_attention_weight_min": -88.01617431640625,
      "activations/layer6_attention_weight_max": 56.6688117980957,
      "activations/layer6_attention_weight_min": -53.3496208190918,
      "activations/layer7_attention_weight_max": 69.88969421386719,
      "activations/layer7_attention_weight_min": -69.95706176757812,
      "activations/layer8_attention_weight_max": 39.12385177612305,
      "activations/layer8_attention_weight_min": -40.007625579833984,
      "activations/layer9_attention_weight_max": 45.017696380615234,
      "activations/layer9_attention_weight_min": -34.10148239135742,
      "epoch": 3.29,
      "learning_rate": 0.00013009962121212122,
      "loss": 2.8971,
      "step": 56600
    },
    {
      "activations/layer0_attention_weight_max": 17.139799118041992,
      "activations/layer0_attention_weight_min": -11.54713249206543,
      "activations/layer10_attention_weight_max": 38.92279052734375,
      "activations/layer10_attention_weight_min": -31.799497604370117,
      "activations/layer11_attention_weight_max": 37.31108093261719,
      "activations/layer11_attention_weight_min": -28.40903091430664,
      "activations/layer12_attention_weight_max": 34.4346809387207,
      "activations/layer12_attention_weight_min": -25.236629486083984,
      "activations/layer13_attention_weight_max": 51.87578582763672,
      "activations/layer13_attention_weight_min": -36.1168212890625,
      "activations/layer14_attention_weight_max": 104.34931945800781,
      "activations/layer14_attention_weight_min": -64.64171600341797,
      "activations/layer15_attention_weight_max": 47.219627380371094,
      "activations/layer15_attention_weight_min": -31.142051696777344,
      "activations/layer16_attention_weight_max": 47.34897232055664,
      "activations/layer16_attention_weight_min": -30.171451568603516,
      "activations/layer17_attention_weight_max": 45.264320373535156,
      "activations/layer17_attention_weight_min": -25.23725700378418,
      "activations/layer18_attention_weight_max": 38.43752670288086,
      "activations/layer18_attention_weight_min": -18.446428298950195,
      "activations/layer19_attention_weight_max": 49.095516204833984,
      "activations/layer19_attention_weight_min": -28.051071166992188,
      "activations/layer1_attention_weight_max": 17.32123374938965,
      "activations/layer1_attention_weight_min": -15.081360816955566,
      "activations/layer20_attention_weight_max": 36.104061126708984,
      "activations/layer20_attention_weight_min": -22.198957443237305,
      "activations/layer21_attention_weight_max": 40.09746170043945,
      "activations/layer21_attention_weight_min": -18.27726936340332,
      "activations/layer22_attention_weight_max": 62.80023956298828,
      "activations/layer22_attention_weight_min": -25.124441146850586,
      "activations/layer23_attention_weight_max": 36.05101013183594,
      "activations/layer23_attention_weight_min": -16.842445373535156,
      "activations/layer2_attention_weight_max": 31.649084091186523,
      "activations/layer2_attention_weight_min": -31.06451416015625,
      "activations/layer3_attention_weight_max": 83.1061019897461,
      "activations/layer3_attention_weight_min": -83.17464447021484,
      "activations/layer4_attention_weight_max": 129.49319458007812,
      "activations/layer4_attention_weight_min": -125.81440734863281,
      "activations/layer5_attention_weight_max": 98.87926483154297,
      "activations/layer5_attention_weight_min": -92.33309936523438,
      "activations/layer6_attention_weight_max": 57.79909133911133,
      "activations/layer6_attention_weight_min": -55.68979263305664,
      "activations/layer7_attention_weight_max": 74.90223693847656,
      "activations/layer7_attention_weight_min": -70.7375259399414,
      "activations/layer8_attention_weight_max": 38.92654800415039,
      "activations/layer8_attention_weight_min": -40.3205451965332,
      "activations/layer9_attention_weight_max": 45.14298629760742,
      "activations/layer9_attention_weight_min": -34.564117431640625,
      "epoch": 3.29,
      "learning_rate": 0.0001300806818181818,
      "loss": 2.8952,
      "step": 56650
    },
    {
      "activations/layer0_attention_weight_max": 16.406492233276367,
      "activations/layer0_attention_weight_min": -12.609762191772461,
      "activations/layer10_attention_weight_max": 42.84733963012695,
      "activations/layer10_attention_weight_min": -36.8293342590332,
      "activations/layer11_attention_weight_max": 33.782859802246094,
      "activations/layer11_attention_weight_min": -29.984981536865234,
      "activations/layer12_attention_weight_max": 28.033567428588867,
      "activations/layer12_attention_weight_min": -33.17488479614258,
      "activations/layer13_attention_weight_max": 43.33681869506836,
      "activations/layer13_attention_weight_min": -35.39042663574219,
      "activations/layer14_attention_weight_max": 72.32725524902344,
      "activations/layer14_attention_weight_min": -54.904212951660156,
      "activations/layer15_attention_weight_max": 42.22776794433594,
      "activations/layer15_attention_weight_min": -31.510536193847656,
      "activations/layer16_attention_weight_max": 48.310951232910156,
      "activations/layer16_attention_weight_min": -32.2963752746582,
      "activations/layer17_attention_weight_max": 43.14708709716797,
      "activations/layer17_attention_weight_min": -27.788326263427734,
      "activations/layer18_attention_weight_max": 33.556400299072266,
      "activations/layer18_attention_weight_min": -21.132381439208984,
      "activations/layer19_attention_weight_max": 45.84972381591797,
      "activations/layer19_attention_weight_min": -32.85729217529297,
      "activations/layer1_attention_weight_max": 17.98413848876953,
      "activations/layer1_attention_weight_min": -12.795238494873047,
      "activations/layer20_attention_weight_max": 33.94806671142578,
      "activations/layer20_attention_weight_min": -25.397266387939453,
      "activations/layer21_attention_weight_max": 33.97926712036133,
      "activations/layer21_attention_weight_min": -24.249923706054688,
      "activations/layer22_attention_weight_max": 50.15695571899414,
      "activations/layer22_attention_weight_min": -28.126054763793945,
      "activations/layer23_attention_weight_max": 25.7955379486084,
      "activations/layer23_attention_weight_min": -17.707996368408203,
      "activations/layer2_attention_weight_max": 30.948293685913086,
      "activations/layer2_attention_weight_min": -30.56867218017578,
      "activations/layer3_attention_weight_max": 84.93809509277344,
      "activations/layer3_attention_weight_min": -88.9401626586914,
      "activations/layer4_attention_weight_max": 136.13771057128906,
      "activations/layer4_attention_weight_min": -129.97105407714844,
      "activations/layer5_attention_weight_max": 96.3463134765625,
      "activations/layer5_attention_weight_min": -90.34527587890625,
      "activations/layer6_attention_weight_max": 55.97382736206055,
      "activations/layer6_attention_weight_min": -51.87730026245117,
      "activations/layer7_attention_weight_max": 75.38662719726562,
      "activations/layer7_attention_weight_min": -73.29586029052734,
      "activations/layer8_attention_weight_max": 40.25014114379883,
      "activations/layer8_attention_weight_min": -36.30693817138672,
      "activations/layer9_attention_weight_max": 56.33040237426758,
      "activations/layer9_attention_weight_min": -46.20225143432617,
      "epoch": 3.29,
      "learning_rate": 0.00013006174242424242,
      "loss": 2.9003,
      "step": 56700
    },
    {
      "activations/layer0_attention_weight_max": 16.062095642089844,
      "activations/layer0_attention_weight_min": -12.693862915039062,
      "activations/layer10_attention_weight_max": 43.11832809448242,
      "activations/layer10_attention_weight_min": -32.69831085205078,
      "activations/layer11_attention_weight_max": 44.865047454833984,
      "activations/layer11_attention_weight_min": -34.26594924926758,
      "activations/layer12_attention_weight_max": 36.78841781616211,
      "activations/layer12_attention_weight_min": -27.43983268737793,
      "activations/layer13_attention_weight_max": 55.80693435668945,
      "activations/layer13_attention_weight_min": -35.11603546142578,
      "activations/layer14_attention_weight_max": 79.13349914550781,
      "activations/layer14_attention_weight_min": -52.911163330078125,
      "activations/layer15_attention_weight_max": 53.281761169433594,
      "activations/layer15_attention_weight_min": -33.10892105102539,
      "activations/layer16_attention_weight_max": 50.62617874145508,
      "activations/layer16_attention_weight_min": -31.349925994873047,
      "activations/layer17_attention_weight_max": 41.57651138305664,
      "activations/layer17_attention_weight_min": -26.554136276245117,
      "activations/layer18_attention_weight_max": 30.953922271728516,
      "activations/layer18_attention_weight_min": -18.65959358215332,
      "activations/layer19_attention_weight_max": 46.20400619506836,
      "activations/layer19_attention_weight_min": -28.946582794189453,
      "activations/layer1_attention_weight_max": 17.3839054107666,
      "activations/layer1_attention_weight_min": -14.748207092285156,
      "activations/layer20_attention_weight_max": 35.167808532714844,
      "activations/layer20_attention_weight_min": -22.351058959960938,
      "activations/layer21_attention_weight_max": 41.059814453125,
      "activations/layer21_attention_weight_min": -19.750825881958008,
      "activations/layer22_attention_weight_max": 50.94281768798828,
      "activations/layer22_attention_weight_min": -25.175134658813477,
      "activations/layer23_attention_weight_max": 28.477497100830078,
      "activations/layer23_attention_weight_min": -16.890647888183594,
      "activations/layer2_attention_weight_max": 32.098548889160156,
      "activations/layer2_attention_weight_min": -30.247053146362305,
      "activations/layer3_attention_weight_max": 83.65811920166016,
      "activations/layer3_attention_weight_min": -83.27361297607422,
      "activations/layer4_attention_weight_max": 129.99058532714844,
      "activations/layer4_attention_weight_min": -119.75487518310547,
      "activations/layer5_attention_weight_max": 100.98540496826172,
      "activations/layer5_attention_weight_min": -87.6861572265625,
      "activations/layer6_attention_weight_max": 56.92347717285156,
      "activations/layer6_attention_weight_min": -53.98688888549805,
      "activations/layer7_attention_weight_max": 78.83782196044922,
      "activations/layer7_attention_weight_min": -77.4013671875,
      "activations/layer8_attention_weight_max": 40.26457214355469,
      "activations/layer8_attention_weight_min": -35.540550231933594,
      "activations/layer9_attention_weight_max": 45.554325103759766,
      "activations/layer9_attention_weight_min": -38.541595458984375,
      "epoch": 3.3,
      "learning_rate": 0.00013004280303030303,
      "loss": 2.8894,
      "step": 56750
    },
    {
      "activations/layer0_attention_weight_max": 16.06336784362793,
      "activations/layer0_attention_weight_min": -12.513903617858887,
      "activations/layer10_attention_weight_max": 38.35070037841797,
      "activations/layer10_attention_weight_min": -30.920040130615234,
      "activations/layer11_attention_weight_max": 41.961753845214844,
      "activations/layer11_attention_weight_min": -30.486671447753906,
      "activations/layer12_attention_weight_max": 30.207216262817383,
      "activations/layer12_attention_weight_min": -26.615524291992188,
      "activations/layer13_attention_weight_max": 46.453773498535156,
      "activations/layer13_attention_weight_min": -33.7171630859375,
      "activations/layer14_attention_weight_max": 81.19276428222656,
      "activations/layer14_attention_weight_min": -55.772029876708984,
      "activations/layer15_attention_weight_max": 51.91796875,
      "activations/layer15_attention_weight_min": -29.948692321777344,
      "activations/layer16_attention_weight_max": 52.312503814697266,
      "activations/layer16_attention_weight_min": -32.00309753417969,
      "activations/layer17_attention_weight_max": 46.05161666870117,
      "activations/layer17_attention_weight_min": -28.16411781311035,
      "activations/layer18_attention_weight_max": 32.297088623046875,
      "activations/layer18_attention_weight_min": -21.067794799804688,
      "activations/layer19_attention_weight_max": 50.36048126220703,
      "activations/layer19_attention_weight_min": -29.052518844604492,
      "activations/layer1_attention_weight_max": 18.627206802368164,
      "activations/layer1_attention_weight_min": -13.564506530761719,
      "activations/layer20_attention_weight_max": 32.34162902832031,
      "activations/layer20_attention_weight_min": -23.88072395324707,
      "activations/layer21_attention_weight_max": 36.962501525878906,
      "activations/layer21_attention_weight_min": -21.838172912597656,
      "activations/layer22_attention_weight_max": 49.6620979309082,
      "activations/layer22_attention_weight_min": -28.491025924682617,
      "activations/layer23_attention_weight_max": 28.469654083251953,
      "activations/layer23_attention_weight_min": -17.07098388671875,
      "activations/layer2_attention_weight_max": 30.661161422729492,
      "activations/layer2_attention_weight_min": -28.56527328491211,
      "activations/layer3_attention_weight_max": 80.60076141357422,
      "activations/layer3_attention_weight_min": -86.13330078125,
      "activations/layer4_attention_weight_max": 130.4755859375,
      "activations/layer4_attention_weight_min": -124.80809783935547,
      "activations/layer5_attention_weight_max": 95.61546325683594,
      "activations/layer5_attention_weight_min": -87.26860809326172,
      "activations/layer6_attention_weight_max": 57.01740646362305,
      "activations/layer6_attention_weight_min": -52.2208366394043,
      "activations/layer7_attention_weight_max": 72.56169128417969,
      "activations/layer7_attention_weight_min": -68.64169311523438,
      "activations/layer8_attention_weight_max": 41.013092041015625,
      "activations/layer8_attention_weight_min": -37.31916427612305,
      "activations/layer9_attention_weight_max": 50.16878890991211,
      "activations/layer9_attention_weight_min": -45.333797454833984,
      "epoch": 3.3,
      "learning_rate": 0.00013002386363636362,
      "loss": 2.908,
      "step": 56800
    },
    {
      "activations/layer0_attention_weight_max": 16.0880126953125,
      "activations/layer0_attention_weight_min": -11.711655616760254,
      "activations/layer10_attention_weight_max": 44.29411697387695,
      "activations/layer10_attention_weight_min": -31.218074798583984,
      "activations/layer11_attention_weight_max": 37.973365783691406,
      "activations/layer11_attention_weight_min": -29.155231475830078,
      "activations/layer12_attention_weight_max": 27.365488052368164,
      "activations/layer12_attention_weight_min": -26.332576751708984,
      "activations/layer13_attention_weight_max": 49.93508529663086,
      "activations/layer13_attention_weight_min": -33.9281120300293,
      "activations/layer14_attention_weight_max": 72.46026611328125,
      "activations/layer14_attention_weight_min": -49.22416305541992,
      "activations/layer15_attention_weight_max": 51.8958740234375,
      "activations/layer15_attention_weight_min": -33.66409683227539,
      "activations/layer16_attention_weight_max": 52.00583267211914,
      "activations/layer16_attention_weight_min": -30.500499725341797,
      "activations/layer17_attention_weight_max": 44.331703186035156,
      "activations/layer17_attention_weight_min": -24.907236099243164,
      "activations/layer18_attention_weight_max": 32.47039031982422,
      "activations/layer18_attention_weight_min": -18.186416625976562,
      "activations/layer19_attention_weight_max": 59.44550323486328,
      "activations/layer19_attention_weight_min": -31.234418869018555,
      "activations/layer1_attention_weight_max": 17.400190353393555,
      "activations/layer1_attention_weight_min": -13.821000099182129,
      "activations/layer20_attention_weight_max": 40.3389892578125,
      "activations/layer20_attention_weight_min": -24.58092498779297,
      "activations/layer21_attention_weight_max": 37.12831497192383,
      "activations/layer21_attention_weight_min": -21.071693420410156,
      "activations/layer22_attention_weight_max": 54.075439453125,
      "activations/layer22_attention_weight_min": -25.49485969543457,
      "activations/layer23_attention_weight_max": 31.891277313232422,
      "activations/layer23_attention_weight_min": -17.569509506225586,
      "activations/layer2_attention_weight_max": 31.866031646728516,
      "activations/layer2_attention_weight_min": -29.497207641601562,
      "activations/layer3_attention_weight_max": 82.59817504882812,
      "activations/layer3_attention_weight_min": -84.93832397460938,
      "activations/layer4_attention_weight_max": 133.90774536132812,
      "activations/layer4_attention_weight_min": -121.54619598388672,
      "activations/layer5_attention_weight_max": 104.19937133789062,
      "activations/layer5_attention_weight_min": -89.20335388183594,
      "activations/layer6_attention_weight_max": 63.587158203125,
      "activations/layer6_attention_weight_min": -53.86342239379883,
      "activations/layer7_attention_weight_max": 76.1766586303711,
      "activations/layer7_attention_weight_min": -77.44255065917969,
      "activations/layer8_attention_weight_max": 45.0555419921875,
      "activations/layer8_attention_weight_min": -37.22575759887695,
      "activations/layer9_attention_weight_max": 43.03683853149414,
      "activations/layer9_attention_weight_min": -34.65748596191406,
      "epoch": 3.3,
      "learning_rate": 0.00013000492424242424,
      "loss": 2.8968,
      "step": 56850
    },
    {
      "activations/layer0_attention_weight_max": 16.876995086669922,
      "activations/layer0_attention_weight_min": -11.73124885559082,
      "activations/layer10_attention_weight_max": 37.004127502441406,
      "activations/layer10_attention_weight_min": -30.508237838745117,
      "activations/layer11_attention_weight_max": 34.76883316040039,
      "activations/layer11_attention_weight_min": -28.63751220703125,
      "activations/layer12_attention_weight_max": 30.519872665405273,
      "activations/layer12_attention_weight_min": -23.77478790283203,
      "activations/layer13_attention_weight_max": 52.50455093383789,
      "activations/layer13_attention_weight_min": -44.63957977294922,
      "activations/layer14_attention_weight_max": 64.69705200195312,
      "activations/layer14_attention_weight_min": -54.187076568603516,
      "activations/layer15_attention_weight_max": 42.21001052856445,
      "activations/layer15_attention_weight_min": -30.499286651611328,
      "activations/layer16_attention_weight_max": 45.01120376586914,
      "activations/layer16_attention_weight_min": -33.091949462890625,
      "activations/layer17_attention_weight_max": 37.08582305908203,
      "activations/layer17_attention_weight_min": -25.776350021362305,
      "activations/layer18_attention_weight_max": 31.32394027709961,
      "activations/layer18_attention_weight_min": -20.795429229736328,
      "activations/layer19_attention_weight_max": 40.318302154541016,
      "activations/layer19_attention_weight_min": -29.822580337524414,
      "activations/layer1_attention_weight_max": 18.313228607177734,
      "activations/layer1_attention_weight_min": -14.664409637451172,
      "activations/layer20_attention_weight_max": 31.29888153076172,
      "activations/layer20_attention_weight_min": -22.854642868041992,
      "activations/layer21_attention_weight_max": 29.965604782104492,
      "activations/layer21_attention_weight_min": -20.6345272064209,
      "activations/layer22_attention_weight_max": 43.69275665283203,
      "activations/layer22_attention_weight_min": -27.0413818359375,
      "activations/layer23_attention_weight_max": 26.52168083190918,
      "activations/layer23_attention_weight_min": -17.78894805908203,
      "activations/layer2_attention_weight_max": 30.695301055908203,
      "activations/layer2_attention_weight_min": -30.66743278503418,
      "activations/layer3_attention_weight_max": 85.33358001708984,
      "activations/layer3_attention_weight_min": -83.57855224609375,
      "activations/layer4_attention_weight_max": 129.9328155517578,
      "activations/layer4_attention_weight_min": -125.5130844116211,
      "activations/layer5_attention_weight_max": 96.63976287841797,
      "activations/layer5_attention_weight_min": -89.19744873046875,
      "activations/layer6_attention_weight_max": 58.69373321533203,
      "activations/layer6_attention_weight_min": -54.45865249633789,
      "activations/layer7_attention_weight_max": 76.5747299194336,
      "activations/layer7_attention_weight_min": -71.49246215820312,
      "activations/layer8_attention_weight_max": 40.27144241333008,
      "activations/layer8_attention_weight_min": -35.41083908081055,
      "activations/layer9_attention_weight_max": 51.623085021972656,
      "activations/layer9_attention_weight_min": -42.61929702758789,
      "epoch": 3.31,
      "learning_rate": 0.00012998598484848485,
      "loss": 2.9059,
      "step": 56900
    },
    {
      "activations/layer0_attention_weight_max": 16.632780075073242,
      "activations/layer0_attention_weight_min": -10.844179153442383,
      "activations/layer10_attention_weight_max": 45.34940719604492,
      "activations/layer10_attention_weight_min": -32.82822799682617,
      "activations/layer11_attention_weight_max": 45.89568328857422,
      "activations/layer11_attention_weight_min": -28.606351852416992,
      "activations/layer12_attention_weight_max": 46.564056396484375,
      "activations/layer12_attention_weight_min": -29.907711029052734,
      "activations/layer13_attention_weight_max": 69.2433090209961,
      "activations/layer13_attention_weight_min": -39.95159149169922,
      "activations/layer14_attention_weight_max": 94.97822570800781,
      "activations/layer14_attention_weight_min": -53.720401763916016,
      "activations/layer15_attention_weight_max": 50.33705520629883,
      "activations/layer15_attention_weight_min": -31.829002380371094,
      "activations/layer16_attention_weight_max": 46.57377624511719,
      "activations/layer16_attention_weight_min": -31.239599227905273,
      "activations/layer17_attention_weight_max": 45.67359161376953,
      "activations/layer17_attention_weight_min": -27.29915428161621,
      "activations/layer18_attention_weight_max": 29.4888858795166,
      "activations/layer18_attention_weight_min": -20.779769897460938,
      "activations/layer19_attention_weight_max": 46.954078674316406,
      "activations/layer19_attention_weight_min": -32.02615737915039,
      "activations/layer1_attention_weight_max": 17.143447875976562,
      "activations/layer1_attention_weight_min": -16.432126998901367,
      "activations/layer20_attention_weight_max": 34.98335266113281,
      "activations/layer20_attention_weight_min": -24.191587448120117,
      "activations/layer21_attention_weight_max": 37.84749984741211,
      "activations/layer21_attention_weight_min": -20.1629581451416,
      "activations/layer22_attention_weight_max": 49.60564422607422,
      "activations/layer22_attention_weight_min": -28.819652557373047,
      "activations/layer23_attention_weight_max": 29.73560905456543,
      "activations/layer23_attention_weight_min": -18.451641082763672,
      "activations/layer2_attention_weight_max": 31.50714111328125,
      "activations/layer2_attention_weight_min": -32.19233703613281,
      "activations/layer3_attention_weight_max": 87.55692291259766,
      "activations/layer3_attention_weight_min": -89.76823425292969,
      "activations/layer4_attention_weight_max": 139.47869873046875,
      "activations/layer4_attention_weight_min": -128.1629638671875,
      "activations/layer5_attention_weight_max": 105.76724243164062,
      "activations/layer5_attention_weight_min": -94.39828491210938,
      "activations/layer6_attention_weight_max": 60.98677444458008,
      "activations/layer6_attention_weight_min": -56.48337936401367,
      "activations/layer7_attention_weight_max": 84.33534240722656,
      "activations/layer7_attention_weight_min": -72.48943328857422,
      "activations/layer8_attention_weight_max": 44.27680969238281,
      "activations/layer8_attention_weight_min": -38.81404495239258,
      "activations/layer9_attention_weight_max": 60.653907775878906,
      "activations/layer9_attention_weight_min": -43.67229461669922,
      "epoch": 3.31,
      "learning_rate": 0.00012996704545454544,
      "loss": 2.8948,
      "step": 56950
    },
    {
      "activations/layer0_attention_weight_max": 17.033933639526367,
      "activations/layer0_attention_weight_min": -11.814906120300293,
      "activations/layer10_attention_weight_max": 40.18698501586914,
      "activations/layer10_attention_weight_min": -32.20407485961914,
      "activations/layer11_attention_weight_max": 38.62411880493164,
      "activations/layer11_attention_weight_min": -30.07834243774414,
      "activations/layer12_attention_weight_max": 45.920379638671875,
      "activations/layer12_attention_weight_min": -32.10896682739258,
      "activations/layer13_attention_weight_max": 47.98741149902344,
      "activations/layer13_attention_weight_min": -32.44857406616211,
      "activations/layer14_attention_weight_max": 76.27291107177734,
      "activations/layer14_attention_weight_min": -54.589141845703125,
      "activations/layer15_attention_weight_max": 58.56510925292969,
      "activations/layer15_attention_weight_min": -36.139766693115234,
      "activations/layer16_attention_weight_max": 49.72296142578125,
      "activations/layer16_attention_weight_min": -28.76854133605957,
      "activations/layer17_attention_weight_max": 42.26108932495117,
      "activations/layer17_attention_weight_min": -25.74321937561035,
      "activations/layer18_attention_weight_max": 31.5222110748291,
      "activations/layer18_attention_weight_min": -18.577621459960938,
      "activations/layer19_attention_weight_max": 54.606258392333984,
      "activations/layer19_attention_weight_min": -29.691892623901367,
      "activations/layer1_attention_weight_max": 18.646377563476562,
      "activations/layer1_attention_weight_min": -13.751862525939941,
      "activations/layer20_attention_weight_max": 40.87140655517578,
      "activations/layer20_attention_weight_min": -20.801603317260742,
      "activations/layer21_attention_weight_max": 37.56962966918945,
      "activations/layer21_attention_weight_min": -21.01418113708496,
      "activations/layer22_attention_weight_max": 49.928245544433594,
      "activations/layer22_attention_weight_min": -27.413869857788086,
      "activations/layer23_attention_weight_max": 31.290029525756836,
      "activations/layer23_attention_weight_min": -19.14931869506836,
      "activations/layer2_attention_weight_max": 31.80462646484375,
      "activations/layer2_attention_weight_min": -31.48571014404297,
      "activations/layer3_attention_weight_max": 82.44648742675781,
      "activations/layer3_attention_weight_min": -90.15235900878906,
      "activations/layer4_attention_weight_max": 126.28529357910156,
      "activations/layer4_attention_weight_min": -128.55528259277344,
      "activations/layer5_attention_weight_max": 91.72286224365234,
      "activations/layer5_attention_weight_min": -85.13351440429688,
      "activations/layer6_attention_weight_max": 55.35424041748047,
      "activations/layer6_attention_weight_min": -55.391929626464844,
      "activations/layer7_attention_weight_max": 70.65589904785156,
      "activations/layer7_attention_weight_min": -70.27580261230469,
      "activations/layer8_attention_weight_max": 38.77126693725586,
      "activations/layer8_attention_weight_min": -36.10902404785156,
      "activations/layer9_attention_weight_max": 48.54887008666992,
      "activations/layer9_attention_weight_min": -52.060176849365234,
      "epoch": 3.31,
      "learning_rate": 0.00012994810606060605,
      "loss": 2.8944,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_loss": 2.84765625,
      "eval_runtime": 8.5103,
      "eval_samples_per_second": 504.564,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_openwebtext_loss": 2.84765625,
      "eval_openwebtext_ppl": 17.247311047115943,
      "eval_openwebtext_runtime": 8.5103,
      "eval_openwebtext_samples_per_second": 504.564,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_wikitext_loss": 3.09375,
      "eval_wikitext_ppl": 22.059646737841184,
      "eval_wikitext_runtime": 1.9892,
      "eval_wikitext_samples_per_second": 229.242,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_lambada_loss": 2.86328125,
      "eval_lambada_ppl": 17.51891667538985,
      "eval_lambada_runtime": 9.598,
      "eval_lambada_samples_per_second": 507.294,
      "step": 57000
    },
    {
      "activations/layer0_attention_weight_max": 14.804129600524902,
      "activations/layer0_attention_weight_min": -11.9222993850708,
      "activations/layer10_attention_weight_max": 41.90144348144531,
      "activations/layer10_attention_weight_min": -32.36998748779297,
      "activations/layer11_attention_weight_max": 36.465492248535156,
      "activations/layer11_attention_weight_min": -28.693172454833984,
      "activations/layer12_attention_weight_max": 43.401580810546875,
      "activations/layer12_attention_weight_min": -26.86569595336914,
      "activations/layer13_attention_weight_max": 51.85872268676758,
      "activations/layer13_attention_weight_min": -37.18989181518555,
      "activations/layer14_attention_weight_max": 68.218017578125,
      "activations/layer14_attention_weight_min": -47.153873443603516,
      "activations/layer15_attention_weight_max": 42.932884216308594,
      "activations/layer15_attention_weight_min": -29.93582534790039,
      "activations/layer16_attention_weight_max": 39.5991096496582,
      "activations/layer16_attention_weight_min": -30.139631271362305,
      "activations/layer17_attention_weight_max": 45.63440704345703,
      "activations/layer17_attention_weight_min": -26.390954971313477,
      "activations/layer18_attention_weight_max": 31.900924682617188,
      "activations/layer18_attention_weight_min": -20.210105895996094,
      "activations/layer19_attention_weight_max": 49.17310333251953,
      "activations/layer19_attention_weight_min": -29.00773048400879,
      "activations/layer1_attention_weight_max": 17.549808502197266,
      "activations/layer1_attention_weight_min": -16.037294387817383,
      "activations/layer20_attention_weight_max": 32.61343002319336,
      "activations/layer20_attention_weight_min": -24.326879501342773,
      "activations/layer21_attention_weight_max": 32.410335540771484,
      "activations/layer21_attention_weight_min": -21.651350021362305,
      "activations/layer22_attention_weight_max": 52.47208786010742,
      "activations/layer22_attention_weight_min": -25.3261661529541,
      "activations/layer23_attention_weight_max": 30.583881378173828,
      "activations/layer23_attention_weight_min": -17.528276443481445,
      "activations/layer2_attention_weight_max": 31.132863998413086,
      "activations/layer2_attention_weight_min": -31.006900787353516,
      "activations/layer3_attention_weight_max": 81.48157501220703,
      "activations/layer3_attention_weight_min": -85.31968688964844,
      "activations/layer4_attention_weight_max": 133.57432556152344,
      "activations/layer4_attention_weight_min": -125.6961441040039,
      "activations/layer5_attention_weight_max": 97.69830322265625,
      "activations/layer5_attention_weight_min": -91.64009094238281,
      "activations/layer6_attention_weight_max": 57.22936248779297,
      "activations/layer6_attention_weight_min": -55.337005615234375,
      "activations/layer7_attention_weight_max": 75.56179809570312,
      "activations/layer7_attention_weight_min": -71.89447021484375,
      "activations/layer8_attention_weight_max": 44.048484802246094,
      "activations/layer8_attention_weight_min": -38.1510009765625,
      "activations/layer9_attention_weight_max": 42.84640121459961,
      "activations/layer9_attention_weight_min": -34.1961555480957,
      "epoch": 3.31,
      "learning_rate": 0.00012992916666666667,
      "loss": 2.8797,
      "step": 57050
    },
    {
      "activations/layer0_attention_weight_max": 15.336564064025879,
      "activations/layer0_attention_weight_min": -11.661781311035156,
      "activations/layer10_attention_weight_max": 45.5080680847168,
      "activations/layer10_attention_weight_min": -34.39949417114258,
      "activations/layer11_attention_weight_max": 39.187225341796875,
      "activations/layer11_attention_weight_min": -29.911388397216797,
      "activations/layer12_attention_weight_max": 40.417354583740234,
      "activations/layer12_attention_weight_min": -27.015731811523438,
      "activations/layer13_attention_weight_max": 61.020599365234375,
      "activations/layer13_attention_weight_min": -36.93777084350586,
      "activations/layer14_attention_weight_max": 89.17781066894531,
      "activations/layer14_attention_weight_min": -56.43632888793945,
      "activations/layer15_attention_weight_max": 52.58444595336914,
      "activations/layer15_attention_weight_min": -31.4588623046875,
      "activations/layer16_attention_weight_max": 52.67282485961914,
      "activations/layer16_attention_weight_min": -31.434465408325195,
      "activations/layer17_attention_weight_max": 54.09978103637695,
      "activations/layer17_attention_weight_min": -25.76589584350586,
      "activations/layer18_attention_weight_max": 40.28620910644531,
      "activations/layer18_attention_weight_min": -18.989694595336914,
      "activations/layer19_attention_weight_max": 53.35639190673828,
      "activations/layer19_attention_weight_min": -29.713834762573242,
      "activations/layer1_attention_weight_max": 18.424238204956055,
      "activations/layer1_attention_weight_min": -14.23471736907959,
      "activations/layer20_attention_weight_max": 44.93412780761719,
      "activations/layer20_attention_weight_min": -21.94733238220215,
      "activations/layer21_attention_weight_max": 53.75656509399414,
      "activations/layer21_attention_weight_min": -22.803165435791016,
      "activations/layer22_attention_weight_max": 65.91231536865234,
      "activations/layer22_attention_weight_min": -27.1885986328125,
      "activations/layer23_attention_weight_max": 30.47463035583496,
      "activations/layer23_attention_weight_min": -15.290202140808105,
      "activations/layer2_attention_weight_max": 31.45852279663086,
      "activations/layer2_attention_weight_min": -29.65497589111328,
      "activations/layer3_attention_weight_max": 85.37276458740234,
      "activations/layer3_attention_weight_min": -84.36254119873047,
      "activations/layer4_attention_weight_max": 138.4724884033203,
      "activations/layer4_attention_weight_min": -129.35891723632812,
      "activations/layer5_attention_weight_max": 105.28807830810547,
      "activations/layer5_attention_weight_min": -90.08916473388672,
      "activations/layer6_attention_weight_max": 60.287662506103516,
      "activations/layer6_attention_weight_min": -54.5111198425293,
      "activations/layer7_attention_weight_max": 84.19133758544922,
      "activations/layer7_attention_weight_min": -76.95404052734375,
      "activations/layer8_attention_weight_max": 43.04747009277344,
      "activations/layer8_attention_weight_min": -38.94789505004883,
      "activations/layer9_attention_weight_max": 50.528564453125,
      "activations/layer9_attention_weight_min": -48.623844146728516,
      "epoch": 3.32,
      "learning_rate": 0.00012991022727272726,
      "loss": 2.8878,
      "step": 57100
    },
    {
      "activations/layer0_attention_weight_max": 15.075446128845215,
      "activations/layer0_attention_weight_min": -12.694889068603516,
      "activations/layer10_attention_weight_max": 38.16381072998047,
      "activations/layer10_attention_weight_min": -31.88010025024414,
      "activations/layer11_attention_weight_max": 37.08740997314453,
      "activations/layer11_attention_weight_min": -28.982839584350586,
      "activations/layer12_attention_weight_max": 61.005313873291016,
      "activations/layer12_attention_weight_min": -29.647968292236328,
      "activations/layer13_attention_weight_max": 69.30889129638672,
      "activations/layer13_attention_weight_min": -42.889400482177734,
      "activations/layer14_attention_weight_max": 65.74656677246094,
      "activations/layer14_attention_weight_min": -49.976417541503906,
      "activations/layer15_attention_weight_max": 47.4477653503418,
      "activations/layer15_attention_weight_min": -30.245506286621094,
      "activations/layer16_attention_weight_max": 46.663822174072266,
      "activations/layer16_attention_weight_min": -29.46595573425293,
      "activations/layer17_attention_weight_max": 44.76362228393555,
      "activations/layer17_attention_weight_min": -24.176475524902344,
      "activations/layer18_attention_weight_max": 32.53982162475586,
      "activations/layer18_attention_weight_min": -22.192209243774414,
      "activations/layer19_attention_weight_max": 48.786834716796875,
      "activations/layer19_attention_weight_min": -28.262659072875977,
      "activations/layer1_attention_weight_max": 17.778913497924805,
      "activations/layer1_attention_weight_min": -15.166207313537598,
      "activations/layer20_attention_weight_max": 30.96779441833496,
      "activations/layer20_attention_weight_min": -21.682199478149414,
      "activations/layer21_attention_weight_max": 33.35091018676758,
      "activations/layer21_attention_weight_min": -19.22586441040039,
      "activations/layer22_attention_weight_max": 46.824092864990234,
      "activations/layer22_attention_weight_min": -25.64623260498047,
      "activations/layer23_attention_weight_max": 27.230491638183594,
      "activations/layer23_attention_weight_min": -20.59231185913086,
      "activations/layer2_attention_weight_max": 31.40955352783203,
      "activations/layer2_attention_weight_min": -31.093624114990234,
      "activations/layer3_attention_weight_max": 86.04350280761719,
      "activations/layer3_attention_weight_min": -83.7722396850586,
      "activations/layer4_attention_weight_max": 132.72581481933594,
      "activations/layer4_attention_weight_min": -121.42750549316406,
      "activations/layer5_attention_weight_max": 98.14308166503906,
      "activations/layer5_attention_weight_min": -85.19931030273438,
      "activations/layer6_attention_weight_max": 57.6332893371582,
      "activations/layer6_attention_weight_min": -51.499942779541016,
      "activations/layer7_attention_weight_max": 72.08995056152344,
      "activations/layer7_attention_weight_min": -71.41556549072266,
      "activations/layer8_attention_weight_max": 40.731361389160156,
      "activations/layer8_attention_weight_min": -37.07966232299805,
      "activations/layer9_attention_weight_max": 48.67186737060547,
      "activations/layer9_attention_weight_min": -38.4860954284668,
      "epoch": 3.32,
      "learning_rate": 0.00012989128787878787,
      "loss": 2.8943,
      "step": 57150
    },
    {
      "activations/layer0_attention_weight_max": 16.053863525390625,
      "activations/layer0_attention_weight_min": -12.688652038574219,
      "activations/layer10_attention_weight_max": 41.8455696105957,
      "activations/layer10_attention_weight_min": -34.72951126098633,
      "activations/layer11_attention_weight_max": 37.02083206176758,
      "activations/layer11_attention_weight_min": -31.381393432617188,
      "activations/layer12_attention_weight_max": 34.88158416748047,
      "activations/layer12_attention_weight_min": -30.890790939331055,
      "activations/layer13_attention_weight_max": 48.31340789794922,
      "activations/layer13_attention_weight_min": -33.56873321533203,
      "activations/layer14_attention_weight_max": 76.08810424804688,
      "activations/layer14_attention_weight_min": -52.199676513671875,
      "activations/layer15_attention_weight_max": 56.37791442871094,
      "activations/layer15_attention_weight_min": -32.59988784790039,
      "activations/layer16_attention_weight_max": 50.27448654174805,
      "activations/layer16_attention_weight_min": -32.13356018066406,
      "activations/layer17_attention_weight_max": 48.1684455871582,
      "activations/layer17_attention_weight_min": -26.946802139282227,
      "activations/layer18_attention_weight_max": 34.97487258911133,
      "activations/layer18_attention_weight_min": -18.98051643371582,
      "activations/layer19_attention_weight_max": 51.286109924316406,
      "activations/layer19_attention_weight_min": -31.447660446166992,
      "activations/layer1_attention_weight_max": 17.991186141967773,
      "activations/layer1_attention_weight_min": -16.380355834960938,
      "activations/layer20_attention_weight_max": 34.271263122558594,
      "activations/layer20_attention_weight_min": -21.92552375793457,
      "activations/layer21_attention_weight_max": 40.625587463378906,
      "activations/layer21_attention_weight_min": -19.9195556640625,
      "activations/layer22_attention_weight_max": 50.41437530517578,
      "activations/layer22_attention_weight_min": -28.06667709350586,
      "activations/layer23_attention_weight_max": 32.1408805847168,
      "activations/layer23_attention_weight_min": -18.476099014282227,
      "activations/layer2_attention_weight_max": 31.166906356811523,
      "activations/layer2_attention_weight_min": -33.488067626953125,
      "activations/layer3_attention_weight_max": 86.07052612304688,
      "activations/layer3_attention_weight_min": -95.42805480957031,
      "activations/layer4_attention_weight_max": 139.47128295898438,
      "activations/layer4_attention_weight_min": -144.3260040283203,
      "activations/layer5_attention_weight_max": 101.37496185302734,
      "activations/layer5_attention_weight_min": -97.50334930419922,
      "activations/layer6_attention_weight_max": 59.725914001464844,
      "activations/layer6_attention_weight_min": -56.76215744018555,
      "activations/layer7_attention_weight_max": 78.18114471435547,
      "activations/layer7_attention_weight_min": -81.01982116699219,
      "activations/layer8_attention_weight_max": 40.838661193847656,
      "activations/layer8_attention_weight_min": -42.036033630371094,
      "activations/layer9_attention_weight_max": 49.72441482543945,
      "activations/layer9_attention_weight_min": -40.228206634521484,
      "epoch": 3.32,
      "learning_rate": 0.00012987234848484846,
      "loss": 2.8897,
      "step": 57200
    },
    {
      "activations/layer0_attention_weight_max": 16.31709861755371,
      "activations/layer0_attention_weight_min": -12.495871543884277,
      "activations/layer10_attention_weight_max": 42.40779495239258,
      "activations/layer10_attention_weight_min": -32.38969421386719,
      "activations/layer11_attention_weight_max": 33.68075942993164,
      "activations/layer11_attention_weight_min": -27.200088500976562,
      "activations/layer12_attention_weight_max": 33.05946731567383,
      "activations/layer12_attention_weight_min": -28.88236427307129,
      "activations/layer13_attention_weight_max": 48.0819206237793,
      "activations/layer13_attention_weight_min": -36.480224609375,
      "activations/layer14_attention_weight_max": 73.66946411132812,
      "activations/layer14_attention_weight_min": -55.07669448852539,
      "activations/layer15_attention_weight_max": 43.713436126708984,
      "activations/layer15_attention_weight_min": -30.96754264831543,
      "activations/layer16_attention_weight_max": 40.24987030029297,
      "activations/layer16_attention_weight_min": -31.36429786682129,
      "activations/layer17_attention_weight_max": 43.7961311340332,
      "activations/layer17_attention_weight_min": -26.597660064697266,
      "activations/layer18_attention_weight_max": 29.789823532104492,
      "activations/layer18_attention_weight_min": -19.01508140563965,
      "activations/layer19_attention_weight_max": 43.1145133972168,
      "activations/layer19_attention_weight_min": -31.32025146484375,
      "activations/layer1_attention_weight_max": 17.227516174316406,
      "activations/layer1_attention_weight_min": -14.195267677307129,
      "activations/layer20_attention_weight_max": 32.052879333496094,
      "activations/layer20_attention_weight_min": -22.73436737060547,
      "activations/layer21_attention_weight_max": 36.212913513183594,
      "activations/layer21_attention_weight_min": -19.70975112915039,
      "activations/layer22_attention_weight_max": 49.85422897338867,
      "activations/layer22_attention_weight_min": -27.719350814819336,
      "activations/layer23_attention_weight_max": 29.877540588378906,
      "activations/layer23_attention_weight_min": -16.92249298095703,
      "activations/layer2_attention_weight_max": 29.988569259643555,
      "activations/layer2_attention_weight_min": -32.44624328613281,
      "activations/layer3_attention_weight_max": 86.24829864501953,
      "activations/layer3_attention_weight_min": -90.25276184082031,
      "activations/layer4_attention_weight_max": 129.2716522216797,
      "activations/layer4_attention_weight_min": -124.34221649169922,
      "activations/layer5_attention_weight_max": 95.68804168701172,
      "activations/layer5_attention_weight_min": -90.17280578613281,
      "activations/layer6_attention_weight_max": 57.64082717895508,
      "activations/layer6_attention_weight_min": -56.35210418701172,
      "activations/layer7_attention_weight_max": 71.52828979492188,
      "activations/layer7_attention_weight_min": -74.34140014648438,
      "activations/layer8_attention_weight_max": 41.45006561279297,
      "activations/layer8_attention_weight_min": -38.94207763671875,
      "activations/layer9_attention_weight_max": 52.63661575317383,
      "activations/layer9_attention_weight_min": -37.84366989135742,
      "epoch": 3.33,
      "learning_rate": 0.00012985340909090908,
      "loss": 2.8931,
      "step": 57250
    },
    {
      "activations/layer0_attention_weight_max": 15.483463287353516,
      "activations/layer0_attention_weight_min": -11.743159294128418,
      "activations/layer10_attention_weight_max": 40.53981399536133,
      "activations/layer10_attention_weight_min": -32.6187744140625,
      "activations/layer11_attention_weight_max": 39.420867919921875,
      "activations/layer11_attention_weight_min": -29.372739791870117,
      "activations/layer12_attention_weight_max": 40.38912582397461,
      "activations/layer12_attention_weight_min": -28.485918045043945,
      "activations/layer13_attention_weight_max": 47.08364486694336,
      "activations/layer13_attention_weight_min": -34.152496337890625,
      "activations/layer14_attention_weight_max": 75.37169647216797,
      "activations/layer14_attention_weight_min": -50.678375244140625,
      "activations/layer15_attention_weight_max": 50.645626068115234,
      "activations/layer15_attention_weight_min": -32.99803924560547,
      "activations/layer16_attention_weight_max": 54.57267761230469,
      "activations/layer16_attention_weight_min": -31.795581817626953,
      "activations/layer17_attention_weight_max": 47.9929313659668,
      "activations/layer17_attention_weight_min": -28.529531478881836,
      "activations/layer18_attention_weight_max": 35.080711364746094,
      "activations/layer18_attention_weight_min": -19.28634262084961,
      "activations/layer19_attention_weight_max": 48.96933364868164,
      "activations/layer19_attention_weight_min": -31.18558120727539,
      "activations/layer1_attention_weight_max": 17.454877853393555,
      "activations/layer1_attention_weight_min": -13.337421417236328,
      "activations/layer20_attention_weight_max": 39.28278350830078,
      "activations/layer20_attention_weight_min": -25.4095401763916,
      "activations/layer21_attention_weight_max": 39.66353225708008,
      "activations/layer21_attention_weight_min": -21.718412399291992,
      "activations/layer22_attention_weight_max": 59.32819366455078,
      "activations/layer22_attention_weight_min": -26.432493209838867,
      "activations/layer23_attention_weight_max": 31.080989837646484,
      "activations/layer23_attention_weight_min": -19.25947380065918,
      "activations/layer2_attention_weight_max": 30.714900970458984,
      "activations/layer2_attention_weight_min": -30.119728088378906,
      "activations/layer3_attention_weight_max": 80.3446044921875,
      "activations/layer3_attention_weight_min": -83.70599365234375,
      "activations/layer4_attention_weight_max": 127.13761138916016,
      "activations/layer4_attention_weight_min": -124.62196350097656,
      "activations/layer5_attention_weight_max": 97.00536346435547,
      "activations/layer5_attention_weight_min": -88.26336669921875,
      "activations/layer6_attention_weight_max": 61.23432159423828,
      "activations/layer6_attention_weight_min": -52.96525573730469,
      "activations/layer7_attention_weight_max": 74.70233154296875,
      "activations/layer7_attention_weight_min": -71.06517791748047,
      "activations/layer8_attention_weight_max": 42.429351806640625,
      "activations/layer8_attention_weight_min": -36.58202362060547,
      "activations/layer9_attention_weight_max": 39.9873161315918,
      "activations/layer9_attention_weight_min": -34.645362854003906,
      "epoch": 3.33,
      "learning_rate": 0.0001298344696969697,
      "loss": 2.8847,
      "step": 57300
    },
    {
      "activations/layer0_attention_weight_max": 15.95226001739502,
      "activations/layer0_attention_weight_min": -12.468490600585938,
      "activations/layer10_attention_weight_max": 37.563629150390625,
      "activations/layer10_attention_weight_min": -32.353759765625,
      "activations/layer11_attention_weight_max": 35.24932861328125,
      "activations/layer11_attention_weight_min": -32.51216125488281,
      "activations/layer12_attention_weight_max": 41.984779357910156,
      "activations/layer12_attention_weight_min": -34.1208381652832,
      "activations/layer13_attention_weight_max": 58.24080276489258,
      "activations/layer13_attention_weight_min": -42.409202575683594,
      "activations/layer14_attention_weight_max": 83.86634063720703,
      "activations/layer14_attention_weight_min": -61.46327209472656,
      "activations/layer15_attention_weight_max": 45.34143829345703,
      "activations/layer15_attention_weight_min": -31.07305145263672,
      "activations/layer16_attention_weight_max": 43.027610778808594,
      "activations/layer16_attention_weight_min": -29.281145095825195,
      "activations/layer17_attention_weight_max": 41.04298782348633,
      "activations/layer17_attention_weight_min": -25.17174530029297,
      "activations/layer18_attention_weight_max": 28.436067581176758,
      "activations/layer18_attention_weight_min": -17.839000701904297,
      "activations/layer19_attention_weight_max": 39.807273864746094,
      "activations/layer19_attention_weight_min": -29.912015914916992,
      "activations/layer1_attention_weight_max": 18.693918228149414,
      "activations/layer1_attention_weight_min": -14.183341979980469,
      "activations/layer20_attention_weight_max": 31.552993774414062,
      "activations/layer20_attention_weight_min": -22.06452178955078,
      "activations/layer21_attention_weight_max": 29.109691619873047,
      "activations/layer21_attention_weight_min": -18.42626953125,
      "activations/layer22_attention_weight_max": 43.69187927246094,
      "activations/layer22_attention_weight_min": -28.11140251159668,
      "activations/layer23_attention_weight_max": 25.709203720092773,
      "activations/layer23_attention_weight_min": -17.328887939453125,
      "activations/layer2_attention_weight_max": 31.591001510620117,
      "activations/layer2_attention_weight_min": -30.65584373474121,
      "activations/layer3_attention_weight_max": 85.67607116699219,
      "activations/layer3_attention_weight_min": -86.67306518554688,
      "activations/layer4_attention_weight_max": 136.6763153076172,
      "activations/layer4_attention_weight_min": -125.25504302978516,
      "activations/layer5_attention_weight_max": 97.98574829101562,
      "activations/layer5_attention_weight_min": -87.96250915527344,
      "activations/layer6_attention_weight_max": 55.059940338134766,
      "activations/layer6_attention_weight_min": -54.687416076660156,
      "activations/layer7_attention_weight_max": 74.32255554199219,
      "activations/layer7_attention_weight_min": -72.31651306152344,
      "activations/layer8_attention_weight_max": 40.550140380859375,
      "activations/layer8_attention_weight_min": -36.5677375793457,
      "activations/layer9_attention_weight_max": 45.861751556396484,
      "activations/layer9_attention_weight_min": -41.567569732666016,
      "epoch": 3.33,
      "learning_rate": 0.00012981553030303028,
      "loss": 2.9,
      "step": 57350
    },
    {
      "activations/layer0_attention_weight_max": 16.3713436126709,
      "activations/layer0_attention_weight_min": -11.5772705078125,
      "activations/layer10_attention_weight_max": 39.75859451293945,
      "activations/layer10_attention_weight_min": -33.61299133300781,
      "activations/layer11_attention_weight_max": 36.867767333984375,
      "activations/layer11_attention_weight_min": -31.132404327392578,
      "activations/layer12_attention_weight_max": 36.435096740722656,
      "activations/layer12_attention_weight_min": -27.77252769470215,
      "activations/layer13_attention_weight_max": 50.1391487121582,
      "activations/layer13_attention_weight_min": -36.700836181640625,
      "activations/layer14_attention_weight_max": 65.02108001708984,
      "activations/layer14_attention_weight_min": -49.57394790649414,
      "activations/layer15_attention_weight_max": 45.460445404052734,
      "activations/layer15_attention_weight_min": -32.12549591064453,
      "activations/layer16_attention_weight_max": 42.27830123901367,
      "activations/layer16_attention_weight_min": -30.023014068603516,
      "activations/layer17_attention_weight_max": 45.03303909301758,
      "activations/layer17_attention_weight_min": -25.615238189697266,
      "activations/layer18_attention_weight_max": 32.66966247558594,
      "activations/layer18_attention_weight_min": -20.994836807250977,
      "activations/layer19_attention_weight_max": 50.636756896972656,
      "activations/layer19_attention_weight_min": -30.803878784179688,
      "activations/layer1_attention_weight_max": 17.70670509338379,
      "activations/layer1_attention_weight_min": -14.487316131591797,
      "activations/layer20_attention_weight_max": 43.85618591308594,
      "activations/layer20_attention_weight_min": -23.09208869934082,
      "activations/layer21_attention_weight_max": 38.75242614746094,
      "activations/layer21_attention_weight_min": -22.39089584350586,
      "activations/layer22_attention_weight_max": 52.476192474365234,
      "activations/layer22_attention_weight_min": -26.346025466918945,
      "activations/layer23_attention_weight_max": 30.05690574645996,
      "activations/layer23_attention_weight_min": -18.02069854736328,
      "activations/layer2_attention_weight_max": 30.154464721679688,
      "activations/layer2_attention_weight_min": -31.25819969177246,
      "activations/layer3_attention_weight_max": 80.24491119384766,
      "activations/layer3_attention_weight_min": -86.61512756347656,
      "activations/layer4_attention_weight_max": 127.33685302734375,
      "activations/layer4_attention_weight_min": -125.3439712524414,
      "activations/layer5_attention_weight_max": 96.01571655273438,
      "activations/layer5_attention_weight_min": -88.02684020996094,
      "activations/layer6_attention_weight_max": 56.95342254638672,
      "activations/layer6_attention_weight_min": -54.18544006347656,
      "activations/layer7_attention_weight_max": 72.79244995117188,
      "activations/layer7_attention_weight_min": -70.57420349121094,
      "activations/layer8_attention_weight_max": 40.93415832519531,
      "activations/layer8_attention_weight_min": -37.858970642089844,
      "activations/layer9_attention_weight_max": 50.03609848022461,
      "activations/layer9_attention_weight_min": -39.77680969238281,
      "epoch": 3.34,
      "learning_rate": 0.0001297965909090909,
      "loss": 2.8868,
      "step": 57400
    },
    {
      "activations/layer0_attention_weight_max": 15.615632057189941,
      "activations/layer0_attention_weight_min": -11.868349075317383,
      "activations/layer10_attention_weight_max": 44.57326126098633,
      "activations/layer10_attention_weight_min": -30.266069412231445,
      "activations/layer11_attention_weight_max": 39.03883361816406,
      "activations/layer11_attention_weight_min": -30.02862548828125,
      "activations/layer12_attention_weight_max": 40.51088333129883,
      "activations/layer12_attention_weight_min": -27.624774932861328,
      "activations/layer13_attention_weight_max": 49.87730026245117,
      "activations/layer13_attention_weight_min": -35.368682861328125,
      "activations/layer14_attention_weight_max": 67.8115005493164,
      "activations/layer14_attention_weight_min": -46.17822265625,
      "activations/layer15_attention_weight_max": 54.89283752441406,
      "activations/layer15_attention_weight_min": -32.369789123535156,
      "activations/layer16_attention_weight_max": 44.75041198730469,
      "activations/layer16_attention_weight_min": -28.41103172302246,
      "activations/layer17_attention_weight_max": 43.620704650878906,
      "activations/layer17_attention_weight_min": -26.116455078125,
      "activations/layer18_attention_weight_max": 31.920265197753906,
      "activations/layer18_attention_weight_min": -21.57155990600586,
      "activations/layer19_attention_weight_max": 48.089630126953125,
      "activations/layer19_attention_weight_min": -30.693204879760742,
      "activations/layer1_attention_weight_max": 17.444156646728516,
      "activations/layer1_attention_weight_min": -13.551836967468262,
      "activations/layer20_attention_weight_max": 35.61868667602539,
      "activations/layer20_attention_weight_min": -22.457050323486328,
      "activations/layer21_attention_weight_max": 35.337371826171875,
      "activations/layer21_attention_weight_min": -21.208759307861328,
      "activations/layer22_attention_weight_max": 49.319698333740234,
      "activations/layer22_attention_weight_min": -27.81291961669922,
      "activations/layer23_attention_weight_max": 33.285072326660156,
      "activations/layer23_attention_weight_min": -20.36478042602539,
      "activations/layer2_attention_weight_max": 31.212650299072266,
      "activations/layer2_attention_weight_min": -31.510406494140625,
      "activations/layer3_attention_weight_max": 84.29129028320312,
      "activations/layer3_attention_weight_min": -87.64696502685547,
      "activations/layer4_attention_weight_max": 140.11648559570312,
      "activations/layer4_attention_weight_min": -124.93944549560547,
      "activations/layer5_attention_weight_max": 101.63475799560547,
      "activations/layer5_attention_weight_min": -89.17924499511719,
      "activations/layer6_attention_weight_max": 57.85680389404297,
      "activations/layer6_attention_weight_min": -52.72432327270508,
      "activations/layer7_attention_weight_max": 75.17536163330078,
      "activations/layer7_attention_weight_min": -70.00643157958984,
      "activations/layer8_attention_weight_max": 41.966468811035156,
      "activations/layer8_attention_weight_min": -37.3897590637207,
      "activations/layer9_attention_weight_max": 48.295108795166016,
      "activations/layer9_attention_weight_min": -40.81647872924805,
      "epoch": 3.34,
      "learning_rate": 0.0001297776515151515,
      "loss": 2.89,
      "step": 57450
    },
    {
      "activations/layer0_attention_weight_max": 16.24806785583496,
      "activations/layer0_attention_weight_min": -11.362835884094238,
      "activations/layer10_attention_weight_max": 40.431575775146484,
      "activations/layer10_attention_weight_min": -33.29479217529297,
      "activations/layer11_attention_weight_max": 36.650325775146484,
      "activations/layer11_attention_weight_min": -28.719562530517578,
      "activations/layer12_attention_weight_max": 35.55918884277344,
      "activations/layer12_attention_weight_min": -26.090328216552734,
      "activations/layer13_attention_weight_max": 47.591426849365234,
      "activations/layer13_attention_weight_min": -34.87156677246094,
      "activations/layer14_attention_weight_max": 68.5368423461914,
      "activations/layer14_attention_weight_min": -52.08123779296875,
      "activations/layer15_attention_weight_max": 46.911441802978516,
      "activations/layer15_attention_weight_min": -29.928525924682617,
      "activations/layer16_attention_weight_max": 42.26447296142578,
      "activations/layer16_attention_weight_min": -28.5953311920166,
      "activations/layer17_attention_weight_max": 43.56093215942383,
      "activations/layer17_attention_weight_min": -25.234394073486328,
      "activations/layer18_attention_weight_max": 32.83843231201172,
      "activations/layer18_attention_weight_min": -20.855487823486328,
      "activations/layer19_attention_weight_max": 41.00628662109375,
      "activations/layer19_attention_weight_min": -27.407211303710938,
      "activations/layer1_attention_weight_max": 17.514440536499023,
      "activations/layer1_attention_weight_min": -14.956507682800293,
      "activations/layer20_attention_weight_max": 34.8073616027832,
      "activations/layer20_attention_weight_min": -23.701725006103516,
      "activations/layer21_attention_weight_max": 38.8109130859375,
      "activations/layer21_attention_weight_min": -21.110567092895508,
      "activations/layer22_attention_weight_max": 46.86006546020508,
      "activations/layer22_attention_weight_min": -26.091999053955078,
      "activations/layer23_attention_weight_max": 26.844627380371094,
      "activations/layer23_attention_weight_min": -15.644105911254883,
      "activations/layer2_attention_weight_max": 29.595535278320312,
      "activations/layer2_attention_weight_min": -29.2950439453125,
      "activations/layer3_attention_weight_max": 83.08854675292969,
      "activations/layer3_attention_weight_min": -82.3215560913086,
      "activations/layer4_attention_weight_max": 128.68174743652344,
      "activations/layer4_attention_weight_min": -123.04766845703125,
      "activations/layer5_attention_weight_max": 95.91836547851562,
      "activations/layer5_attention_weight_min": -87.77348327636719,
      "activations/layer6_attention_weight_max": 59.5771598815918,
      "activations/layer6_attention_weight_min": -52.94673538208008,
      "activations/layer7_attention_weight_max": 76.90242004394531,
      "activations/layer7_attention_weight_min": -72.10350036621094,
      "activations/layer8_attention_weight_max": 40.32978439331055,
      "activations/layer8_attention_weight_min": -37.291015625,
      "activations/layer9_attention_weight_max": 51.04746627807617,
      "activations/layer9_attention_weight_min": -36.35636901855469,
      "epoch": 3.34,
      "learning_rate": 0.0001297587121212121,
      "loss": 2.8885,
      "step": 57500
    },
    {
      "activations/layer0_attention_weight_max": 15.718454360961914,
      "activations/layer0_attention_weight_min": -12.83772087097168,
      "activations/layer10_attention_weight_max": 41.3377799987793,
      "activations/layer10_attention_weight_min": -34.90154266357422,
      "activations/layer11_attention_weight_max": 35.26032257080078,
      "activations/layer11_attention_weight_min": -27.796051025390625,
      "activations/layer12_attention_weight_max": 55.706520080566406,
      "activations/layer12_attention_weight_min": -29.961088180541992,
      "activations/layer13_attention_weight_max": 54.21341323852539,
      "activations/layer13_attention_weight_min": -38.23252868652344,
      "activations/layer14_attention_weight_max": 71.44306945800781,
      "activations/layer14_attention_weight_min": -52.075950622558594,
      "activations/layer15_attention_weight_max": 48.640960693359375,
      "activations/layer15_attention_weight_min": -35.076229095458984,
      "activations/layer16_attention_weight_max": 44.373008728027344,
      "activations/layer16_attention_weight_min": -29.680978775024414,
      "activations/layer17_attention_weight_max": 42.60994338989258,
      "activations/layer17_attention_weight_min": -27.37324333190918,
      "activations/layer18_attention_weight_max": 33.018436431884766,
      "activations/layer18_attention_weight_min": -19.40049934387207,
      "activations/layer19_attention_weight_max": 49.038047790527344,
      "activations/layer19_attention_weight_min": -29.035491943359375,
      "activations/layer1_attention_weight_max": 18.237085342407227,
      "activations/layer1_attention_weight_min": -14.410282135009766,
      "activations/layer20_attention_weight_max": 32.390567779541016,
      "activations/layer20_attention_weight_min": -24.603193283081055,
      "activations/layer21_attention_weight_max": 37.36720657348633,
      "activations/layer21_attention_weight_min": -21.98134422302246,
      "activations/layer22_attention_weight_max": 51.8179931640625,
      "activations/layer22_attention_weight_min": -26.592012405395508,
      "activations/layer23_attention_weight_max": 29.814334869384766,
      "activations/layer23_attention_weight_min": -20.835365295410156,
      "activations/layer2_attention_weight_max": 30.472454071044922,
      "activations/layer2_attention_weight_min": -29.448753356933594,
      "activations/layer3_attention_weight_max": 83.9340591430664,
      "activations/layer3_attention_weight_min": -82.43159484863281,
      "activations/layer4_attention_weight_max": 131.6736297607422,
      "activations/layer4_attention_weight_min": -133.5916748046875,
      "activations/layer5_attention_weight_max": 98.17849731445312,
      "activations/layer5_attention_weight_min": -93.82323455810547,
      "activations/layer6_attention_weight_max": 59.778541564941406,
      "activations/layer6_attention_weight_min": -55.92884826660156,
      "activations/layer7_attention_weight_max": 77.93366241455078,
      "activations/layer7_attention_weight_min": -70.67504119873047,
      "activations/layer8_attention_weight_max": 40.59818649291992,
      "activations/layer8_attention_weight_min": -38.55048751831055,
      "activations/layer9_attention_weight_max": 44.39487838745117,
      "activations/layer9_attention_weight_min": -37.65781021118164,
      "epoch": 3.34,
      "learning_rate": 0.0001297397727272727,
      "loss": 2.9126,
      "step": 57550
    },
    {
      "activations/layer0_attention_weight_max": 17.068090438842773,
      "activations/layer0_attention_weight_min": -10.996026039123535,
      "activations/layer10_attention_weight_max": 38.78404235839844,
      "activations/layer10_attention_weight_min": -35.61537551879883,
      "activations/layer11_attention_weight_max": 34.78887176513672,
      "activations/layer11_attention_weight_min": -28.72945213317871,
      "activations/layer12_attention_weight_max": 34.34189987182617,
      "activations/layer12_attention_weight_min": -24.55329132080078,
      "activations/layer13_attention_weight_max": 50.909122467041016,
      "activations/layer13_attention_weight_min": -37.44163513183594,
      "activations/layer14_attention_weight_max": 75.59123992919922,
      "activations/layer14_attention_weight_min": -58.24786376953125,
      "activations/layer15_attention_weight_max": 44.96215057373047,
      "activations/layer15_attention_weight_min": -35.974727630615234,
      "activations/layer16_attention_weight_max": 46.43450164794922,
      "activations/layer16_attention_weight_min": -33.9527473449707,
      "activations/layer17_attention_weight_max": 41.707763671875,
      "activations/layer17_attention_weight_min": -28.498165130615234,
      "activations/layer18_attention_weight_max": 32.77870178222656,
      "activations/layer18_attention_weight_min": -19.132566452026367,
      "activations/layer19_attention_weight_max": 41.53169250488281,
      "activations/layer19_attention_weight_min": -31.166126251220703,
      "activations/layer1_attention_weight_max": 16.731233596801758,
      "activations/layer1_attention_weight_min": -14.383278846740723,
      "activations/layer20_attention_weight_max": 35.26287078857422,
      "activations/layer20_attention_weight_min": -24.164077758789062,
      "activations/layer21_attention_weight_max": 30.90737533569336,
      "activations/layer21_attention_weight_min": -20.447853088378906,
      "activations/layer22_attention_weight_max": 49.91380310058594,
      "activations/layer22_attention_weight_min": -28.969377517700195,
      "activations/layer23_attention_weight_max": 26.922941207885742,
      "activations/layer23_attention_weight_min": -20.293027877807617,
      "activations/layer2_attention_weight_max": 30.89887809753418,
      "activations/layer2_attention_weight_min": -31.210247039794922,
      "activations/layer3_attention_weight_max": 84.54814147949219,
      "activations/layer3_attention_weight_min": -84.62633514404297,
      "activations/layer4_attention_weight_max": 133.68406677246094,
      "activations/layer4_attention_weight_min": -129.4437713623047,
      "activations/layer5_attention_weight_max": 97.43511199951172,
      "activations/layer5_attention_weight_min": -90.56124877929688,
      "activations/layer6_attention_weight_max": 57.66370391845703,
      "activations/layer6_attention_weight_min": -54.499637603759766,
      "activations/layer7_attention_weight_max": 71.240478515625,
      "activations/layer7_attention_weight_min": -71.07547760009766,
      "activations/layer8_attention_weight_max": 40.33997344970703,
      "activations/layer8_attention_weight_min": -37.79867172241211,
      "activations/layer9_attention_weight_max": 43.614784240722656,
      "activations/layer9_attention_weight_min": -35.0658073425293,
      "epoch": 3.35,
      "learning_rate": 0.00012972083333333333,
      "loss": 2.8821,
      "step": 57600
    },
    {
      "activations/layer0_attention_weight_max": 14.684563636779785,
      "activations/layer0_attention_weight_min": -12.455360412597656,
      "activations/layer10_attention_weight_max": 45.626529693603516,
      "activations/layer10_attention_weight_min": -34.81447982788086,
      "activations/layer11_attention_weight_max": 39.003395080566406,
      "activations/layer11_attention_weight_min": -31.940204620361328,
      "activations/layer12_attention_weight_max": 35.012264251708984,
      "activations/layer12_attention_weight_min": -31.499223709106445,
      "activations/layer13_attention_weight_max": 48.553714752197266,
      "activations/layer13_attention_weight_min": -34.190547943115234,
      "activations/layer14_attention_weight_max": 69.32363891601562,
      "activations/layer14_attention_weight_min": -49.69718551635742,
      "activations/layer15_attention_weight_max": 43.48556900024414,
      "activations/layer15_attention_weight_min": -29.488805770874023,
      "activations/layer16_attention_weight_max": 38.79563522338867,
      "activations/layer16_attention_weight_min": -27.628393173217773,
      "activations/layer17_attention_weight_max": 39.52507400512695,
      "activations/layer17_attention_weight_min": -27.099348068237305,
      "activations/layer18_attention_weight_max": 28.678451538085938,
      "activations/layer18_attention_weight_min": -22.994342803955078,
      "activations/layer19_attention_weight_max": 41.708858489990234,
      "activations/layer19_attention_weight_min": -30.215002059936523,
      "activations/layer1_attention_weight_max": 18.700952529907227,
      "activations/layer1_attention_weight_min": -14.387359619140625,
      "activations/layer20_attention_weight_max": 36.548927307128906,
      "activations/layer20_attention_weight_min": -26.050168991088867,
      "activations/layer21_attention_weight_max": 38.58865737915039,
      "activations/layer21_attention_weight_min": -23.69248390197754,
      "activations/layer22_attention_weight_max": 47.5540657043457,
      "activations/layer22_attention_weight_min": -29.527416229248047,
      "activations/layer23_attention_weight_max": 28.446823120117188,
      "activations/layer23_attention_weight_min": -18.36932373046875,
      "activations/layer2_attention_weight_max": 32.687564849853516,
      "activations/layer2_attention_weight_min": -31.190956115722656,
      "activations/layer3_attention_weight_max": 87.56603240966797,
      "activations/layer3_attention_weight_min": -90.22203063964844,
      "activations/layer4_attention_weight_max": 138.2639923095703,
      "activations/layer4_attention_weight_min": -129.3714141845703,
      "activations/layer5_attention_weight_max": 104.8160629272461,
      "activations/layer5_attention_weight_min": -92.70155334472656,
      "activations/layer6_attention_weight_max": 56.58860397338867,
      "activations/layer6_attention_weight_min": -57.59976577758789,
      "activations/layer7_attention_weight_max": 71.79166412353516,
      "activations/layer7_attention_weight_min": -70.85652160644531,
      "activations/layer8_attention_weight_max": 37.65629196166992,
      "activations/layer8_attention_weight_min": -37.46489334106445,
      "activations/layer9_attention_weight_max": 48.26514434814453,
      "activations/layer9_attention_weight_min": -42.84238815307617,
      "epoch": 3.35,
      "learning_rate": 0.00012970189393939391,
      "loss": 2.8926,
      "step": 57650
    },
    {
      "activations/layer0_attention_weight_max": 16.862476348876953,
      "activations/layer0_attention_weight_min": -11.789972305297852,
      "activations/layer10_attention_weight_max": 37.48454666137695,
      "activations/layer10_attention_weight_min": -31.130544662475586,
      "activations/layer11_attention_weight_max": 32.47819137573242,
      "activations/layer11_attention_weight_min": -28.26900863647461,
      "activations/layer12_attention_weight_max": 43.37352752685547,
      "activations/layer12_attention_weight_min": -25.871992111206055,
      "activations/layer13_attention_weight_max": 50.77536392211914,
      "activations/layer13_attention_weight_min": -34.8736686706543,
      "activations/layer14_attention_weight_max": 72.56169128417969,
      "activations/layer14_attention_weight_min": -52.57289123535156,
      "activations/layer15_attention_weight_max": 47.62970733642578,
      "activations/layer15_attention_weight_min": -31.813413619995117,
      "activations/layer16_attention_weight_max": 40.58436584472656,
      "activations/layer16_attention_weight_min": -30.539628982543945,
      "activations/layer17_attention_weight_max": 39.44667434692383,
      "activations/layer17_attention_weight_min": -25.59450340270996,
      "activations/layer18_attention_weight_max": 35.23539733886719,
      "activations/layer18_attention_weight_min": -22.210994720458984,
      "activations/layer19_attention_weight_max": 43.04914474487305,
      "activations/layer19_attention_weight_min": -31.37790298461914,
      "activations/layer1_attention_weight_max": 16.684499740600586,
      "activations/layer1_attention_weight_min": -13.392314910888672,
      "activations/layer20_attention_weight_max": 32.02975845336914,
      "activations/layer20_attention_weight_min": -22.855276107788086,
      "activations/layer21_attention_weight_max": 34.49434280395508,
      "activations/layer21_attention_weight_min": -20.99829864501953,
      "activations/layer22_attention_weight_max": 46.170570373535156,
      "activations/layer22_attention_weight_min": -25.40932273864746,
      "activations/layer23_attention_weight_max": 25.58771324157715,
      "activations/layer23_attention_weight_min": -16.81183624267578,
      "activations/layer2_attention_weight_max": 30.459531784057617,
      "activations/layer2_attention_weight_min": -30.27461814880371,
      "activations/layer3_attention_weight_max": 83.20549774169922,
      "activations/layer3_attention_weight_min": -81.82488250732422,
      "activations/layer4_attention_weight_max": 132.39727783203125,
      "activations/layer4_attention_weight_min": -120.0467758178711,
      "activations/layer5_attention_weight_max": 96.38851928710938,
      "activations/layer5_attention_weight_min": -84.88954162597656,
      "activations/layer6_attention_weight_max": 57.465736389160156,
      "activations/layer6_attention_weight_min": -51.8487663269043,
      "activations/layer7_attention_weight_max": 71.52465057373047,
      "activations/layer7_attention_weight_min": -68.66154479980469,
      "activations/layer8_attention_weight_max": 38.523216247558594,
      "activations/layer8_attention_weight_min": -35.33444595336914,
      "activations/layer9_attention_weight_max": 45.544551849365234,
      "activations/layer9_attention_weight_min": -37.329532623291016,
      "epoch": 3.35,
      "learning_rate": 0.00012968295454545453,
      "loss": 2.8734,
      "step": 57700
    },
    {
      "activations/layer0_attention_weight_max": 16.726057052612305,
      "activations/layer0_attention_weight_min": -12.005852699279785,
      "activations/layer10_attention_weight_max": 37.528236389160156,
      "activations/layer10_attention_weight_min": -31.420608520507812,
      "activations/layer11_attention_weight_max": 35.92999267578125,
      "activations/layer11_attention_weight_min": -28.1763973236084,
      "activations/layer12_attention_weight_max": 31.19828987121582,
      "activations/layer12_attention_weight_min": -24.872364044189453,
      "activations/layer13_attention_weight_max": 52.715816497802734,
      "activations/layer13_attention_weight_min": -36.67030715942383,
      "activations/layer14_attention_weight_max": 68.04375457763672,
      "activations/layer14_attention_weight_min": -47.102882385253906,
      "activations/layer15_attention_weight_max": 48.051700592041016,
      "activations/layer15_attention_weight_min": -30.031232833862305,
      "activations/layer16_attention_weight_max": 44.07919692993164,
      "activations/layer16_attention_weight_min": -32.12697982788086,
      "activations/layer17_attention_weight_max": 46.6197395324707,
      "activations/layer17_attention_weight_min": -26.566333770751953,
      "activations/layer18_attention_weight_max": 31.987741470336914,
      "activations/layer18_attention_weight_min": -19.551834106445312,
      "activations/layer19_attention_weight_max": 46.77470397949219,
      "activations/layer19_attention_weight_min": -30.236541748046875,
      "activations/layer1_attention_weight_max": 17.438037872314453,
      "activations/layer1_attention_weight_min": -16.575151443481445,
      "activations/layer20_attention_weight_max": 42.86418533325195,
      "activations/layer20_attention_weight_min": -23.608224868774414,
      "activations/layer21_attention_weight_max": 34.79079055786133,
      "activations/layer21_attention_weight_min": -21.58053970336914,
      "activations/layer22_attention_weight_max": 54.80745315551758,
      "activations/layer22_attention_weight_min": -27.10631561279297,
      "activations/layer23_attention_weight_max": 28.56343650817871,
      "activations/layer23_attention_weight_min": -16.071500778198242,
      "activations/layer2_attention_weight_max": 31.459774017333984,
      "activations/layer2_attention_weight_min": -31.114200592041016,
      "activations/layer3_attention_weight_max": 84.49452209472656,
      "activations/layer3_attention_weight_min": -85.6962890625,
      "activations/layer4_attention_weight_max": 134.20994567871094,
      "activations/layer4_attention_weight_min": -123.41202545166016,
      "activations/layer5_attention_weight_max": 96.3743896484375,
      "activations/layer5_attention_weight_min": -86.1890869140625,
      "activations/layer6_attention_weight_max": 57.921539306640625,
      "activations/layer6_attention_weight_min": -54.63511276245117,
      "activations/layer7_attention_weight_max": 74.57415008544922,
      "activations/layer7_attention_weight_min": -71.76024627685547,
      "activations/layer8_attention_weight_max": 40.77985382080078,
      "activations/layer8_attention_weight_min": -37.92630386352539,
      "activations/layer9_attention_weight_max": 40.49930191040039,
      "activations/layer9_attention_weight_min": -37.464332580566406,
      "epoch": 3.36,
      "learning_rate": 0.00012966401515151514,
      "loss": 2.9147,
      "step": 57750
    },
    {
      "activations/layer0_attention_weight_max": 17.112062454223633,
      "activations/layer0_attention_weight_min": -10.912334442138672,
      "activations/layer10_attention_weight_max": 36.05327224731445,
      "activations/layer10_attention_weight_min": -33.83586883544922,
      "activations/layer11_attention_weight_max": 36.95732116699219,
      "activations/layer11_attention_weight_min": -29.180967330932617,
      "activations/layer12_attention_weight_max": 31.15563201904297,
      "activations/layer12_attention_weight_min": -30.863277435302734,
      "activations/layer13_attention_weight_max": 43.332523345947266,
      "activations/layer13_attention_weight_min": -34.686431884765625,
      "activations/layer14_attention_weight_max": 70.64556121826172,
      "activations/layer14_attention_weight_min": -55.63287353515625,
      "activations/layer15_attention_weight_max": 41.63656997680664,
      "activations/layer15_attention_weight_min": -29.37284278869629,
      "activations/layer16_attention_weight_max": 37.21258544921875,
      "activations/layer16_attention_weight_min": -29.458742141723633,
      "activations/layer17_attention_weight_max": 40.69226837158203,
      "activations/layer17_attention_weight_min": -25.227005004882812,
      "activations/layer18_attention_weight_max": 28.3941650390625,
      "activations/layer18_attention_weight_min": -18.240081787109375,
      "activations/layer19_attention_weight_max": 39.8128547668457,
      "activations/layer19_attention_weight_min": -29.271936416625977,
      "activations/layer1_attention_weight_max": 18.432750701904297,
      "activations/layer1_attention_weight_min": -12.787898063659668,
      "activations/layer20_attention_weight_max": 29.163469314575195,
      "activations/layer20_attention_weight_min": -22.629497528076172,
      "activations/layer21_attention_weight_max": 31.41426658630371,
      "activations/layer21_attention_weight_min": -18.511228561401367,
      "activations/layer22_attention_weight_max": 48.239051818847656,
      "activations/layer22_attention_weight_min": -27.419572830200195,
      "activations/layer23_attention_weight_max": 25.79018211364746,
      "activations/layer23_attention_weight_min": -17.80548095703125,
      "activations/layer2_attention_weight_max": 29.565765380859375,
      "activations/layer2_attention_weight_min": -28.56688690185547,
      "activations/layer3_attention_weight_max": 80.75743865966797,
      "activations/layer3_attention_weight_min": -84.65947723388672,
      "activations/layer4_attention_weight_max": 124.74272918701172,
      "activations/layer4_attention_weight_min": -126.3160629272461,
      "activations/layer5_attention_weight_max": 99.62225341796875,
      "activations/layer5_attention_weight_min": -90.86931610107422,
      "activations/layer6_attention_weight_max": 57.98587417602539,
      "activations/layer6_attention_weight_min": -55.5859260559082,
      "activations/layer7_attention_weight_max": 72.66183471679688,
      "activations/layer7_attention_weight_min": -71.38164520263672,
      "activations/layer8_attention_weight_max": 40.54805374145508,
      "activations/layer8_attention_weight_min": -39.30241012573242,
      "activations/layer9_attention_weight_max": 43.35368728637695,
      "activations/layer9_attention_weight_min": -33.76359558105469,
      "epoch": 3.36,
      "learning_rate": 0.00012964507575757573,
      "loss": 2.8937,
      "step": 57800
    },
    {
      "activations/layer0_attention_weight_max": 17.505029678344727,
      "activations/layer0_attention_weight_min": -11.946178436279297,
      "activations/layer10_attention_weight_max": 37.661895751953125,
      "activations/layer10_attention_weight_min": -31.503759384155273,
      "activations/layer11_attention_weight_max": 35.678592681884766,
      "activations/layer11_attention_weight_min": -29.765676498413086,
      "activations/layer12_attention_weight_max": 30.151517868041992,
      "activations/layer12_attention_weight_min": -29.7106990814209,
      "activations/layer13_attention_weight_max": 46.541996002197266,
      "activations/layer13_attention_weight_min": -33.5390625,
      "activations/layer14_attention_weight_max": 71.41604614257812,
      "activations/layer14_attention_weight_min": -54.909942626953125,
      "activations/layer15_attention_weight_max": 44.52151870727539,
      "activations/layer15_attention_weight_min": -31.052228927612305,
      "activations/layer16_attention_weight_max": 42.44510269165039,
      "activations/layer16_attention_weight_min": -30.68201446533203,
      "activations/layer17_attention_weight_max": 45.15956497192383,
      "activations/layer17_attention_weight_min": -26.142847061157227,
      "activations/layer18_attention_weight_max": 37.55903244018555,
      "activations/layer18_attention_weight_min": -19.392118453979492,
      "activations/layer19_attention_weight_max": 48.63540267944336,
      "activations/layer19_attention_weight_min": -28.693267822265625,
      "activations/layer1_attention_weight_max": 17.495731353759766,
      "activations/layer1_attention_weight_min": -14.234564781188965,
      "activations/layer20_attention_weight_max": 38.46991729736328,
      "activations/layer20_attention_weight_min": -24.32117462158203,
      "activations/layer21_attention_weight_max": 37.938316345214844,
      "activations/layer21_attention_weight_min": -19.563861846923828,
      "activations/layer22_attention_weight_max": 52.67681884765625,
      "activations/layer22_attention_weight_min": -27.77619171142578,
      "activations/layer23_attention_weight_max": 29.32747459411621,
      "activations/layer23_attention_weight_min": -18.90069007873535,
      "activations/layer2_attention_weight_max": 31.888080596923828,
      "activations/layer2_attention_weight_min": -29.060749053955078,
      "activations/layer3_attention_weight_max": 84.47007751464844,
      "activations/layer3_attention_weight_min": -82.89278411865234,
      "activations/layer4_attention_weight_max": 128.06834411621094,
      "activations/layer4_attention_weight_min": -125.5279541015625,
      "activations/layer5_attention_weight_max": 93.19859313964844,
      "activations/layer5_attention_weight_min": -86.02412414550781,
      "activations/layer6_attention_weight_max": 52.821998596191406,
      "activations/layer6_attention_weight_min": -52.30256271362305,
      "activations/layer7_attention_weight_max": 70.14718627929688,
      "activations/layer7_attention_weight_min": -71.51266479492188,
      "activations/layer8_attention_weight_max": 41.32515335083008,
      "activations/layer8_attention_weight_min": -37.454246520996094,
      "activations/layer9_attention_weight_max": 45.87213897705078,
      "activations/layer9_attention_weight_min": -34.10447692871094,
      "epoch": 3.36,
      "learning_rate": 0.00012962613636363635,
      "loss": 2.8902,
      "step": 57850
    },
    {
      "activations/layer0_attention_weight_max": 16.385038375854492,
      "activations/layer0_attention_weight_min": -10.94433307647705,
      "activations/layer10_attention_weight_max": 37.69718933105469,
      "activations/layer10_attention_weight_min": -30.775920867919922,
      "activations/layer11_attention_weight_max": 33.261329650878906,
      "activations/layer11_attention_weight_min": -27.757701873779297,
      "activations/layer12_attention_weight_max": 57.86510467529297,
      "activations/layer12_attention_weight_min": -28.06390953063965,
      "activations/layer13_attention_weight_max": 43.369075775146484,
      "activations/layer13_attention_weight_min": -33.83969497680664,
      "activations/layer14_attention_weight_max": 65.2896728515625,
      "activations/layer14_attention_weight_min": -50.690101623535156,
      "activations/layer15_attention_weight_max": 43.95798873901367,
      "activations/layer15_attention_weight_min": -33.38178253173828,
      "activations/layer16_attention_weight_max": 45.69862747192383,
      "activations/layer16_attention_weight_min": -34.29681396484375,
      "activations/layer17_attention_weight_max": 37.75122833251953,
      "activations/layer17_attention_weight_min": -26.24558448791504,
      "activations/layer18_attention_weight_max": 31.057405471801758,
      "activations/layer18_attention_weight_min": -21.317466735839844,
      "activations/layer19_attention_weight_max": 49.601924896240234,
      "activations/layer19_attention_weight_min": -30.494964599609375,
      "activations/layer1_attention_weight_max": 17.419403076171875,
      "activations/layer1_attention_weight_min": -13.46895980834961,
      "activations/layer20_attention_weight_max": 35.749691009521484,
      "activations/layer20_attention_weight_min": -21.9387149810791,
      "activations/layer21_attention_weight_max": 33.40755081176758,
      "activations/layer21_attention_weight_min": -21.839221954345703,
      "activations/layer22_attention_weight_max": 50.44233703613281,
      "activations/layer22_attention_weight_min": -26.388731002807617,
      "activations/layer23_attention_weight_max": 32.34989929199219,
      "activations/layer23_attention_weight_min": -18.8740234375,
      "activations/layer2_attention_weight_max": 30.547101974487305,
      "activations/layer2_attention_weight_min": -28.712751388549805,
      "activations/layer3_attention_weight_max": 79.78778076171875,
      "activations/layer3_attention_weight_min": -79.61939239501953,
      "activations/layer4_attention_weight_max": 129.87106323242188,
      "activations/layer4_attention_weight_min": -126.4291763305664,
      "activations/layer5_attention_weight_max": 96.47206115722656,
      "activations/layer5_attention_weight_min": -86.43663787841797,
      "activations/layer6_attention_weight_max": 61.17768859863281,
      "activations/layer6_attention_weight_min": -52.17716979980469,
      "activations/layer7_attention_weight_max": 73.5957260131836,
      "activations/layer7_attention_weight_min": -74.2679672241211,
      "activations/layer8_attention_weight_max": 39.28538131713867,
      "activations/layer8_attention_weight_min": -36.58842849731445,
      "activations/layer9_attention_weight_max": 44.51353454589844,
      "activations/layer9_attention_weight_min": -35.81793212890625,
      "epoch": 3.36,
      "learning_rate": 0.00012960719696969696,
      "loss": 2.8937,
      "step": 57900
    },
    {
      "activations/layer0_attention_weight_max": 17.333528518676758,
      "activations/layer0_attention_weight_min": -11.741439819335938,
      "activations/layer10_attention_weight_max": 35.37382888793945,
      "activations/layer10_attention_weight_min": -29.97148323059082,
      "activations/layer11_attention_weight_max": 34.95494079589844,
      "activations/layer11_attention_weight_min": -28.501564025878906,
      "activations/layer12_attention_weight_max": 37.89658737182617,
      "activations/layer12_attention_weight_min": -25.643110275268555,
      "activations/layer13_attention_weight_max": 47.18052291870117,
      "activations/layer13_attention_weight_min": -33.522377014160156,
      "activations/layer14_attention_weight_max": 70.96479797363281,
      "activations/layer14_attention_weight_min": -47.9339485168457,
      "activations/layer15_attention_weight_max": 46.25368118286133,
      "activations/layer15_attention_weight_min": -30.651779174804688,
      "activations/layer16_attention_weight_max": 49.01655960083008,
      "activations/layer16_attention_weight_min": -28.95092010498047,
      "activations/layer17_attention_weight_max": 44.564048767089844,
      "activations/layer17_attention_weight_min": -25.279205322265625,
      "activations/layer18_attention_weight_max": 32.2760124206543,
      "activations/layer18_attention_weight_min": -19.080429077148438,
      "activations/layer19_attention_weight_max": 47.24753189086914,
      "activations/layer19_attention_weight_min": -29.159015655517578,
      "activations/layer1_attention_weight_max": 17.505504608154297,
      "activations/layer1_attention_weight_min": -14.148591041564941,
      "activations/layer20_attention_weight_max": 37.041717529296875,
      "activations/layer20_attention_weight_min": -23.471166610717773,
      "activations/layer21_attention_weight_max": 36.6379280090332,
      "activations/layer21_attention_weight_min": -21.51677894592285,
      "activations/layer22_attention_weight_max": 47.02478790283203,
      "activations/layer22_attention_weight_min": -26.65662956237793,
      "activations/layer23_attention_weight_max": 27.91462516784668,
      "activations/layer23_attention_weight_min": -21.492895126342773,
      "activations/layer2_attention_weight_max": 31.86463165283203,
      "activations/layer2_attention_weight_min": -31.53070640563965,
      "activations/layer3_attention_weight_max": 83.75624084472656,
      "activations/layer3_attention_weight_min": -86.92082977294922,
      "activations/layer4_attention_weight_max": 125.95287322998047,
      "activations/layer4_attention_weight_min": -130.5719451904297,
      "activations/layer5_attention_weight_max": 93.75588989257812,
      "activations/layer5_attention_weight_min": -89.16737365722656,
      "activations/layer6_attention_weight_max": 57.744300842285156,
      "activations/layer6_attention_weight_min": -52.35658264160156,
      "activations/layer7_attention_weight_max": 73.446533203125,
      "activations/layer7_attention_weight_min": -71.84141540527344,
      "activations/layer8_attention_weight_max": 41.68717575073242,
      "activations/layer8_attention_weight_min": -36.18990707397461,
      "activations/layer9_attention_weight_max": 42.322444915771484,
      "activations/layer9_attention_weight_min": -35.84684371948242,
      "epoch": 3.37,
      "learning_rate": 0.00012958825757575758,
      "loss": 2.8987,
      "step": 57950
    },
    {
      "activations/layer0_attention_weight_max": 18.151620864868164,
      "activations/layer0_attention_weight_min": -12.148314476013184,
      "activations/layer10_attention_weight_max": 37.51170349121094,
      "activations/layer10_attention_weight_min": -33.56605911254883,
      "activations/layer11_attention_weight_max": 35.02051544189453,
      "activations/layer11_attention_weight_min": -29.597267150878906,
      "activations/layer12_attention_weight_max": 27.98959732055664,
      "activations/layer12_attention_weight_min": -29.539405822753906,
      "activations/layer13_attention_weight_max": 44.89213180541992,
      "activations/layer13_attention_weight_min": -33.991275787353516,
      "activations/layer14_attention_weight_max": 68.828369140625,
      "activations/layer14_attention_weight_min": -54.76871109008789,
      "activations/layer15_attention_weight_max": 41.46242904663086,
      "activations/layer15_attention_weight_min": -31.399320602416992,
      "activations/layer16_attention_weight_max": 37.652618408203125,
      "activations/layer16_attention_weight_min": -29.8674373626709,
      "activations/layer17_attention_weight_max": 42.776954650878906,
      "activations/layer17_attention_weight_min": -29.053462982177734,
      "activations/layer18_attention_weight_max": 31.747718811035156,
      "activations/layer18_attention_weight_min": -21.45671844482422,
      "activations/layer19_attention_weight_max": 38.60561752319336,
      "activations/layer19_attention_weight_min": -28.814712524414062,
      "activations/layer1_attention_weight_max": 17.936038970947266,
      "activations/layer1_attention_weight_min": -13.765292167663574,
      "activations/layer20_attention_weight_max": 32.81904602050781,
      "activations/layer20_attention_weight_min": -23.080659866333008,
      "activations/layer21_attention_weight_max": 33.6588134765625,
      "activations/layer21_attention_weight_min": -20.5477352142334,
      "activations/layer22_attention_weight_max": 45.59592056274414,
      "activations/layer22_attention_weight_min": -29.63069725036621,
      "activations/layer23_attention_weight_max": 27.161151885986328,
      "activations/layer23_attention_weight_min": -19.873149871826172,
      "activations/layer2_attention_weight_max": 29.792163848876953,
      "activations/layer2_attention_weight_min": -30.122608184814453,
      "activations/layer3_attention_weight_max": 80.98041534423828,
      "activations/layer3_attention_weight_min": -85.75340270996094,
      "activations/layer4_attention_weight_max": 129.1261444091797,
      "activations/layer4_attention_weight_min": -129.76487731933594,
      "activations/layer5_attention_weight_max": 100.12803649902344,
      "activations/layer5_attention_weight_min": -90.34068298339844,
      "activations/layer6_attention_weight_max": 56.92401885986328,
      "activations/layer6_attention_weight_min": -56.61247634887695,
      "activations/layer7_attention_weight_max": 75.31732177734375,
      "activations/layer7_attention_weight_min": -72.90654754638672,
      "activations/layer8_attention_weight_max": 40.76232147216797,
      "activations/layer8_attention_weight_min": -38.48139572143555,
      "activations/layer9_attention_weight_max": 42.69942092895508,
      "activations/layer9_attention_weight_min": -36.48005294799805,
      "epoch": 3.37,
      "learning_rate": 0.00012956969696969697,
      "loss": 2.9021,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_loss": 2.845703125,
      "eval_runtime": 8.5183,
      "eval_samples_per_second": 504.089,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_openwebtext_loss": 2.845703125,
      "eval_openwebtext_ppl": 17.213657767955564,
      "eval_openwebtext_runtime": 8.5183,
      "eval_openwebtext_samples_per_second": 504.089,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_wikitext_loss": 3.09765625,
      "eval_wikitext_ppl": 22.145985754016134,
      "eval_wikitext_runtime": 2.0254,
      "eval_wikitext_samples_per_second": 225.143,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_lambada_loss": 2.849609375,
      "eval_lambada_ppl": 17.28103011956753,
      "eval_lambada_runtime": 9.5851,
      "eval_lambada_samples_per_second": 507.978,
      "step": 58000
    },
    {
      "activations/layer0_attention_weight_max": 16.86623191833496,
      "activations/layer0_attention_weight_min": -11.043456077575684,
      "activations/layer10_attention_weight_max": 39.41083526611328,
      "activations/layer10_attention_weight_min": -34.74348449707031,
      "activations/layer11_attention_weight_max": 38.917877197265625,
      "activations/layer11_attention_weight_min": -32.10700607299805,
      "activations/layer12_attention_weight_max": 52.921688079833984,
      "activations/layer12_attention_weight_min": -29.561115264892578,
      "activations/layer13_attention_weight_max": 43.627723693847656,
      "activations/layer13_attention_weight_min": -35.99761199951172,
      "activations/layer14_attention_weight_max": 67.4322738647461,
      "activations/layer14_attention_weight_min": -50.37466049194336,
      "activations/layer15_attention_weight_max": 46.6678352355957,
      "activations/layer15_attention_weight_min": -35.567481994628906,
      "activations/layer16_attention_weight_max": 41.30088424682617,
      "activations/layer16_attention_weight_min": -32.26092529296875,
      "activations/layer17_attention_weight_max": 43.54647445678711,
      "activations/layer17_attention_weight_min": -26.910194396972656,
      "activations/layer18_attention_weight_max": 30.444740295410156,
      "activations/layer18_attention_weight_min": -19.111202239990234,
      "activations/layer19_attention_weight_max": 42.933929443359375,
      "activations/layer19_attention_weight_min": -31.039276123046875,
      "activations/layer1_attention_weight_max": 18.203413009643555,
      "activations/layer1_attention_weight_min": -14.605567932128906,
      "activations/layer20_attention_weight_max": 32.88294219970703,
      "activations/layer20_attention_weight_min": -24.537551879882812,
      "activations/layer21_attention_weight_max": 33.25693130493164,
      "activations/layer21_attention_weight_min": -19.434946060180664,
      "activations/layer22_attention_weight_max": 51.80176544189453,
      "activations/layer22_attention_weight_min": -30.07779312133789,
      "activations/layer23_attention_weight_max": 26.856060028076172,
      "activations/layer23_attention_weight_min": -18.469614028930664,
      "activations/layer2_attention_weight_max": 30.538246154785156,
      "activations/layer2_attention_weight_min": -29.95761489868164,
      "activations/layer3_attention_weight_max": 79.97505187988281,
      "activations/layer3_attention_weight_min": -81.80831146240234,
      "activations/layer4_attention_weight_max": 123.39842987060547,
      "activations/layer4_attention_weight_min": -131.07150268554688,
      "activations/layer5_attention_weight_max": 94.06080627441406,
      "activations/layer5_attention_weight_min": -89.99895477294922,
      "activations/layer6_attention_weight_max": 56.9775390625,
      "activations/layer6_attention_weight_min": -57.23038864135742,
      "activations/layer7_attention_weight_max": 70.82369995117188,
      "activations/layer7_attention_weight_min": -70.7577896118164,
      "activations/layer8_attention_weight_max": 39.58125686645508,
      "activations/layer8_attention_weight_min": -37.45964431762695,
      "activations/layer9_attention_weight_max": 42.70006561279297,
      "activations/layer9_attention_weight_min": -34.0783805847168,
      "epoch": 3.37,
      "learning_rate": 0.00012955075757575756,
      "loss": 2.8682,
      "step": 58050
    },
    {
      "activations/layer0_attention_weight_max": 15.448034286499023,
      "activations/layer0_attention_weight_min": -13.088543891906738,
      "activations/layer10_attention_weight_max": 41.53207778930664,
      "activations/layer10_attention_weight_min": -32.83407974243164,
      "activations/layer11_attention_weight_max": 36.597862243652344,
      "activations/layer11_attention_weight_min": -26.297096252441406,
      "activations/layer12_attention_weight_max": 36.90376663208008,
      "activations/layer12_attention_weight_min": -27.140287399291992,
      "activations/layer13_attention_weight_max": 49.33639907836914,
      "activations/layer13_attention_weight_min": -34.68891525268555,
      "activations/layer14_attention_weight_max": 67.80606079101562,
      "activations/layer14_attention_weight_min": -50.062339782714844,
      "activations/layer15_attention_weight_max": 50.65548324584961,
      "activations/layer15_attention_weight_min": -32.522705078125,
      "activations/layer16_attention_weight_max": 41.320308685302734,
      "activations/layer16_attention_weight_min": -28.622459411621094,
      "activations/layer17_attention_weight_max": 44.10198211669922,
      "activations/layer17_attention_weight_min": -24.931224822998047,
      "activations/layer18_attention_weight_max": 33.67674255371094,
      "activations/layer18_attention_weight_min": -19.403894424438477,
      "activations/layer19_attention_weight_max": 40.74235153198242,
      "activations/layer19_attention_weight_min": -28.718042373657227,
      "activations/layer1_attention_weight_max": 17.341035842895508,
      "activations/layer1_attention_weight_min": -13.297085762023926,
      "activations/layer20_attention_weight_max": 32.784507751464844,
      "activations/layer20_attention_weight_min": -23.127197265625,
      "activations/layer21_attention_weight_max": 35.10263442993164,
      "activations/layer21_attention_weight_min": -18.8524112701416,
      "activations/layer22_attention_weight_max": 49.03782653808594,
      "activations/layer22_attention_weight_min": -26.274274826049805,
      "activations/layer23_attention_weight_max": 26.9138240814209,
      "activations/layer23_attention_weight_min": -18.44561767578125,
      "activations/layer2_attention_weight_max": 32.917991638183594,
      "activations/layer2_attention_weight_min": -33.027076721191406,
      "activations/layer3_attention_weight_max": 88.0707778930664,
      "activations/layer3_attention_weight_min": -90.51736450195312,
      "activations/layer4_attention_weight_max": 130.4761199951172,
      "activations/layer4_attention_weight_min": -132.9270782470703,
      "activations/layer5_attention_weight_max": 92.16380310058594,
      "activations/layer5_attention_weight_min": -88.8638916015625,
      "activations/layer6_attention_weight_max": 56.45310974121094,
      "activations/layer6_attention_weight_min": -58.602481842041016,
      "activations/layer7_attention_weight_max": 73.25508880615234,
      "activations/layer7_attention_weight_min": -72.02454376220703,
      "activations/layer8_attention_weight_max": 38.898624420166016,
      "activations/layer8_attention_weight_min": -37.124114990234375,
      "activations/layer9_attention_weight_max": 46.3123779296875,
      "activations/layer9_attention_weight_min": -37.71177673339844,
      "epoch": 3.38,
      "learning_rate": 0.00012953181818181817,
      "loss": 2.9127,
      "step": 58100
    },
    {
      "activations/layer0_attention_weight_max": 15.695683479309082,
      "activations/layer0_attention_weight_min": -13.416790962219238,
      "activations/layer10_attention_weight_max": 42.659751892089844,
      "activations/layer10_attention_weight_min": -34.5074462890625,
      "activations/layer11_attention_weight_max": 35.321800231933594,
      "activations/layer11_attention_weight_min": -27.900501251220703,
      "activations/layer12_attention_weight_max": 41.6705207824707,
      "activations/layer12_attention_weight_min": -30.746051788330078,
      "activations/layer13_attention_weight_max": 49.13478469848633,
      "activations/layer13_attention_weight_min": -39.61647033691406,
      "activations/layer14_attention_weight_max": 64.86731719970703,
      "activations/layer14_attention_weight_min": -52.563804626464844,
      "activations/layer15_attention_weight_max": 49.61146545410156,
      "activations/layer15_attention_weight_min": -31.6608943939209,
      "activations/layer16_attention_weight_max": 40.260284423828125,
      "activations/layer16_attention_weight_min": -28.152467727661133,
      "activations/layer17_attention_weight_max": 46.11110305786133,
      "activations/layer17_attention_weight_min": -23.942113876342773,
      "activations/layer18_attention_weight_max": 35.7031135559082,
      "activations/layer18_attention_weight_min": -19.395015716552734,
      "activations/layer19_attention_weight_max": 40.801761627197266,
      "activations/layer19_attention_weight_min": -27.396732330322266,
      "activations/layer1_attention_weight_max": 18.11739158630371,
      "activations/layer1_attention_weight_min": -14.510451316833496,
      "activations/layer20_attention_weight_max": 32.994503021240234,
      "activations/layer20_attention_weight_min": -22.92021369934082,
      "activations/layer21_attention_weight_max": 32.6053581237793,
      "activations/layer21_attention_weight_min": -20.58521842956543,
      "activations/layer22_attention_weight_max": 45.215274810791016,
      "activations/layer22_attention_weight_min": -31.061540603637695,
      "activations/layer23_attention_weight_max": 27.430553436279297,
      "activations/layer23_attention_weight_min": -18.14690589904785,
      "activations/layer2_attention_weight_max": 31.920682907104492,
      "activations/layer2_attention_weight_min": -31.52655029296875,
      "activations/layer3_attention_weight_max": 84.61113739013672,
      "activations/layer3_attention_weight_min": -84.11538696289062,
      "activations/layer4_attention_weight_max": 128.5050048828125,
      "activations/layer4_attention_weight_min": -125.74710845947266,
      "activations/layer5_attention_weight_max": 96.03605651855469,
      "activations/layer5_attention_weight_min": -95.96617889404297,
      "activations/layer6_attention_weight_max": 58.24897384643555,
      "activations/layer6_attention_weight_min": -61.59260940551758,
      "activations/layer7_attention_weight_max": 72.56758880615234,
      "activations/layer7_attention_weight_min": -72.76698303222656,
      "activations/layer8_attention_weight_max": 40.39817428588867,
      "activations/layer8_attention_weight_min": -36.24814224243164,
      "activations/layer9_attention_weight_max": 40.75415802001953,
      "activations/layer9_attention_weight_min": -37.88272476196289,
      "epoch": 3.38,
      "learning_rate": 0.00012951287878787879,
      "loss": 2.9119,
      "step": 58150
    },
    {
      "activations/layer0_attention_weight_max": 17.910846710205078,
      "activations/layer0_attention_weight_min": -12.283991813659668,
      "activations/layer10_attention_weight_max": 37.9505500793457,
      "activations/layer10_attention_weight_min": -31.72886085510254,
      "activations/layer11_attention_weight_max": 35.219268798828125,
      "activations/layer11_attention_weight_min": -27.62569236755371,
      "activations/layer12_attention_weight_max": 75.78034973144531,
      "activations/layer12_attention_weight_min": -38.78644943237305,
      "activations/layer13_attention_weight_max": 54.23782730102539,
      "activations/layer13_attention_weight_min": -35.909515380859375,
      "activations/layer14_attention_weight_max": 66.6658935546875,
      "activations/layer14_attention_weight_min": -50.024166107177734,
      "activations/layer15_attention_weight_max": 43.67513656616211,
      "activations/layer15_attention_weight_min": -30.142961502075195,
      "activations/layer16_attention_weight_max": 40.395286560058594,
      "activations/layer16_attention_weight_min": -29.166994094848633,
      "activations/layer17_attention_weight_max": 47.48549270629883,
      "activations/layer17_attention_weight_min": -26.53537940979004,
      "activations/layer18_attention_weight_max": 31.466936111450195,
      "activations/layer18_attention_weight_min": -21.091121673583984,
      "activations/layer19_attention_weight_max": 46.262123107910156,
      "activations/layer19_attention_weight_min": -29.967382431030273,
      "activations/layer1_attention_weight_max": 18.679576873779297,
      "activations/layer1_attention_weight_min": -14.083589553833008,
      "activations/layer20_attention_weight_max": 36.465553283691406,
      "activations/layer20_attention_weight_min": -22.65462875366211,
      "activations/layer21_attention_weight_max": 33.80434036254883,
      "activations/layer21_attention_weight_min": -21.825881958007812,
      "activations/layer22_attention_weight_max": 50.21064758300781,
      "activations/layer22_attention_weight_min": -27.095844268798828,
      "activations/layer23_attention_weight_max": 28.353042602539062,
      "activations/layer23_attention_weight_min": -17.250444412231445,
      "activations/layer2_attention_weight_max": 29.766761779785156,
      "activations/layer2_attention_weight_min": -29.364791870117188,
      "activations/layer3_attention_weight_max": 81.8099365234375,
      "activations/layer3_attention_weight_min": -86.09453582763672,
      "activations/layer4_attention_weight_max": 127.11943817138672,
      "activations/layer4_attention_weight_min": -127.2618179321289,
      "activations/layer5_attention_weight_max": 95.75531005859375,
      "activations/layer5_attention_weight_min": -88.4154281616211,
      "activations/layer6_attention_weight_max": 55.327964782714844,
      "activations/layer6_attention_weight_min": -54.23366165161133,
      "activations/layer7_attention_weight_max": 75.29469299316406,
      "activations/layer7_attention_weight_min": -70.47273254394531,
      "activations/layer8_attention_weight_max": 41.9839973449707,
      "activations/layer8_attention_weight_min": -37.10142517089844,
      "activations/layer9_attention_weight_max": 39.564815521240234,
      "activations/layer9_attention_weight_min": -34.738548278808594,
      "epoch": 3.38,
      "learning_rate": 0.0001294939393939394,
      "loss": 2.8846,
      "step": 58200
    },
    {
      "activations/layer0_attention_weight_max": 17.23918914794922,
      "activations/layer0_attention_weight_min": -12.579743385314941,
      "activations/layer10_attention_weight_max": 37.00190734863281,
      "activations/layer10_attention_weight_min": -30.80740737915039,
      "activations/layer11_attention_weight_max": 36.46219253540039,
      "activations/layer11_attention_weight_min": -29.95197296142578,
      "activations/layer12_attention_weight_max": 42.015533447265625,
      "activations/layer12_attention_weight_min": -28.85770606994629,
      "activations/layer13_attention_weight_max": 43.861690521240234,
      "activations/layer13_attention_weight_min": -32.44218444824219,
      "activations/layer14_attention_weight_max": 63.900352478027344,
      "activations/layer14_attention_weight_min": -50.57672882080078,
      "activations/layer15_attention_weight_max": 44.762481689453125,
      "activations/layer15_attention_weight_min": -32.28490447998047,
      "activations/layer16_attention_weight_max": 39.453731536865234,
      "activations/layer16_attention_weight_min": -30.795331954956055,
      "activations/layer17_attention_weight_max": 37.89406204223633,
      "activations/layer17_attention_weight_min": -25.234155654907227,
      "activations/layer18_attention_weight_max": 28.522117614746094,
      "activations/layer18_attention_weight_min": -18.354589462280273,
      "activations/layer19_attention_weight_max": 40.792686462402344,
      "activations/layer19_attention_weight_min": -27.81549072265625,
      "activations/layer1_attention_weight_max": 17.541316986083984,
      "activations/layer1_attention_weight_min": -15.876249313354492,
      "activations/layer20_attention_weight_max": 36.28007507324219,
      "activations/layer20_attention_weight_min": -21.907224655151367,
      "activations/layer21_attention_weight_max": 31.862457275390625,
      "activations/layer21_attention_weight_min": -19.992328643798828,
      "activations/layer22_attention_weight_max": 45.18840789794922,
      "activations/layer22_attention_weight_min": -25.68819808959961,
      "activations/layer23_attention_weight_max": 28.363027572631836,
      "activations/layer23_attention_weight_min": -17.911293029785156,
      "activations/layer2_attention_weight_max": 30.984760284423828,
      "activations/layer2_attention_weight_min": -31.492183685302734,
      "activations/layer3_attention_weight_max": 81.862548828125,
      "activations/layer3_attention_weight_min": -85.35963439941406,
      "activations/layer4_attention_weight_max": 130.7784881591797,
      "activations/layer4_attention_weight_min": -123.87393951416016,
      "activations/layer5_attention_weight_max": 97.49928283691406,
      "activations/layer5_attention_weight_min": -88.38896942138672,
      "activations/layer6_attention_weight_max": 59.022708892822266,
      "activations/layer6_attention_weight_min": -53.13145065307617,
      "activations/layer7_attention_weight_max": 76.5645751953125,
      "activations/layer7_attention_weight_min": -72.29386901855469,
      "activations/layer8_attention_weight_max": 38.15009307861328,
      "activations/layer8_attention_weight_min": -37.498043060302734,
      "activations/layer9_attention_weight_max": 43.07524871826172,
      "activations/layer9_attention_weight_min": -33.785316467285156,
      "epoch": 3.38,
      "learning_rate": 0.000129475,
      "loss": 2.8838,
      "step": 58250
    },
    {
      "activations/layer0_attention_weight_max": 16.824676513671875,
      "activations/layer0_attention_weight_min": -11.57201862335205,
      "activations/layer10_attention_weight_max": 42.90250778198242,
      "activations/layer10_attention_weight_min": -33.31203842163086,
      "activations/layer11_attention_weight_max": 35.60615539550781,
      "activations/layer11_attention_weight_min": -27.599044799804688,
      "activations/layer12_attention_weight_max": 25.750715255737305,
      "activations/layer12_attention_weight_min": -24.87131118774414,
      "activations/layer13_attention_weight_max": 47.598594665527344,
      "activations/layer13_attention_weight_min": -33.667755126953125,
      "activations/layer14_attention_weight_max": 64.23162841796875,
      "activations/layer14_attention_weight_min": -46.5352897644043,
      "activations/layer15_attention_weight_max": 50.485504150390625,
      "activations/layer15_attention_weight_min": -30.313371658325195,
      "activations/layer16_attention_weight_max": 45.209442138671875,
      "activations/layer16_attention_weight_min": -30.00577163696289,
      "activations/layer17_attention_weight_max": 47.339290618896484,
      "activations/layer17_attention_weight_min": -26.903085708618164,
      "activations/layer18_attention_weight_max": 31.091928482055664,
      "activations/layer18_attention_weight_min": -19.792177200317383,
      "activations/layer19_attention_weight_max": 46.97585678100586,
      "activations/layer19_attention_weight_min": -28.393070220947266,
      "activations/layer1_attention_weight_max": 17.67389488220215,
      "activations/layer1_attention_weight_min": -14.665035247802734,
      "activations/layer20_attention_weight_max": 36.26512145996094,
      "activations/layer20_attention_weight_min": -25.172618865966797,
      "activations/layer21_attention_weight_max": 34.09482955932617,
      "activations/layer21_attention_weight_min": -22.826038360595703,
      "activations/layer22_attention_weight_max": 48.8414192199707,
      "activations/layer22_attention_weight_min": -27.897384643554688,
      "activations/layer23_attention_weight_max": 28.39731216430664,
      "activations/layer23_attention_weight_min": -15.90515422821045,
      "activations/layer2_attention_weight_max": 33.153892517089844,
      "activations/layer2_attention_weight_min": -31.721843719482422,
      "activations/layer3_attention_weight_max": 87.51362609863281,
      "activations/layer3_attention_weight_min": -89.81793212890625,
      "activations/layer4_attention_weight_max": 138.77024841308594,
      "activations/layer4_attention_weight_min": -127.32816314697266,
      "activations/layer5_attention_weight_max": 99.76561737060547,
      "activations/layer5_attention_weight_min": -92.69966888427734,
      "activations/layer6_attention_weight_max": 61.11607360839844,
      "activations/layer6_attention_weight_min": -55.59926223754883,
      "activations/layer7_attention_weight_max": 77.35128784179688,
      "activations/layer7_attention_weight_min": -76.23974609375,
      "activations/layer8_attention_weight_max": 43.56508255004883,
      "activations/layer8_attention_weight_min": -37.402671813964844,
      "activations/layer9_attention_weight_max": 41.82841873168945,
      "activations/layer9_attention_weight_min": -37.890193939208984,
      "epoch": 3.39,
      "learning_rate": 0.0001294560606060606,
      "loss": 2.9056,
      "step": 58300
    },
    {
      "activations/layer0_attention_weight_max": 17.332304000854492,
      "activations/layer0_attention_weight_min": -12.00912857055664,
      "activations/layer10_attention_weight_max": 40.03773880004883,
      "activations/layer10_attention_weight_min": -32.1208381652832,
      "activations/layer11_attention_weight_max": 37.59735107421875,
      "activations/layer11_attention_weight_min": -28.107345581054688,
      "activations/layer12_attention_weight_max": 35.45451354980469,
      "activations/layer12_attention_weight_min": -36.362548828125,
      "activations/layer13_attention_weight_max": 44.81507873535156,
      "activations/layer13_attention_weight_min": -32.945159912109375,
      "activations/layer14_attention_weight_max": 63.51852798461914,
      "activations/layer14_attention_weight_min": -46.15586471557617,
      "activations/layer15_attention_weight_max": 42.96864700317383,
      "activations/layer15_attention_weight_min": -32.02248001098633,
      "activations/layer16_attention_weight_max": 42.55567169189453,
      "activations/layer16_attention_weight_min": -30.485530853271484,
      "activations/layer17_attention_weight_max": 39.25252914428711,
      "activations/layer17_attention_weight_min": -25.349939346313477,
      "activations/layer18_attention_weight_max": 27.99772834777832,
      "activations/layer18_attention_weight_min": -17.832107543945312,
      "activations/layer19_attention_weight_max": 40.004398345947266,
      "activations/layer19_attention_weight_min": -26.379169464111328,
      "activations/layer1_attention_weight_max": 17.93132209777832,
      "activations/layer1_attention_weight_min": -13.693072319030762,
      "activations/layer20_attention_weight_max": 34.47720718383789,
      "activations/layer20_attention_weight_min": -23.115942001342773,
      "activations/layer21_attention_weight_max": 33.229591369628906,
      "activations/layer21_attention_weight_min": -19.97484588623047,
      "activations/layer22_attention_weight_max": 49.48970031738281,
      "activations/layer22_attention_weight_min": -28.16501235961914,
      "activations/layer23_attention_weight_max": 27.97992706298828,
      "activations/layer23_attention_weight_min": -17.181873321533203,
      "activations/layer2_attention_weight_max": 31.77750015258789,
      "activations/layer2_attention_weight_min": -31.97308349609375,
      "activations/layer3_attention_weight_max": 87.80261993408203,
      "activations/layer3_attention_weight_min": -87.59590148925781,
      "activations/layer4_attention_weight_max": 135.57424926757812,
      "activations/layer4_attention_weight_min": -126.44676971435547,
      "activations/layer5_attention_weight_max": 93.30709075927734,
      "activations/layer5_attention_weight_min": -87.61180114746094,
      "activations/layer6_attention_weight_max": 58.53841018676758,
      "activations/layer6_attention_weight_min": -50.62487030029297,
      "activations/layer7_attention_weight_max": 82.93006896972656,
      "activations/layer7_attention_weight_min": -77.36365509033203,
      "activations/layer8_attention_weight_max": 36.78300857543945,
      "activations/layer8_attention_weight_min": -37.527339935302734,
      "activations/layer9_attention_weight_max": 54.544471740722656,
      "activations/layer9_attention_weight_min": -48.45015335083008,
      "epoch": 3.39,
      "learning_rate": 0.00012943712121212122,
      "loss": 2.8801,
      "step": 58350
    },
    {
      "activations/layer0_attention_weight_max": 17.486371994018555,
      "activations/layer0_attention_weight_min": -11.542693138122559,
      "activations/layer10_attention_weight_max": 39.064884185791016,
      "activations/layer10_attention_weight_min": -31.388687133789062,
      "activations/layer11_attention_weight_max": 35.74132537841797,
      "activations/layer11_attention_weight_min": -27.08730125427246,
      "activations/layer12_attention_weight_max": 56.16950988769531,
      "activations/layer12_attention_weight_min": -29.00885772705078,
      "activations/layer13_attention_weight_max": 49.37416458129883,
      "activations/layer13_attention_weight_min": -36.32265090942383,
      "activations/layer14_attention_weight_max": 70.79949951171875,
      "activations/layer14_attention_weight_min": -56.54582977294922,
      "activations/layer15_attention_weight_max": 56.361873626708984,
      "activations/layer15_attention_weight_min": -36.87338638305664,
      "activations/layer16_attention_weight_max": 44.4809684753418,
      "activations/layer16_attention_weight_min": -32.871681213378906,
      "activations/layer17_attention_weight_max": 45.6656494140625,
      "activations/layer17_attention_weight_min": -24.67894172668457,
      "activations/layer18_attention_weight_max": 31.8835391998291,
      "activations/layer18_attention_weight_min": -20.079822540283203,
      "activations/layer19_attention_weight_max": 44.306304931640625,
      "activations/layer19_attention_weight_min": -29.250768661499023,
      "activations/layer1_attention_weight_max": 17.177898406982422,
      "activations/layer1_attention_weight_min": -13.61109447479248,
      "activations/layer20_attention_weight_max": 33.617279052734375,
      "activations/layer20_attention_weight_min": -23.286197662353516,
      "activations/layer21_attention_weight_max": 31.753646850585938,
      "activations/layer21_attention_weight_min": -20.940256118774414,
      "activations/layer22_attention_weight_max": 54.14268493652344,
      "activations/layer22_attention_weight_min": -27.410968780517578,
      "activations/layer23_attention_weight_max": 30.357023239135742,
      "activations/layer23_attention_weight_min": -18.218257904052734,
      "activations/layer2_attention_weight_max": 31.017627716064453,
      "activations/layer2_attention_weight_min": -29.739990234375,
      "activations/layer3_attention_weight_max": 80.41150665283203,
      "activations/layer3_attention_weight_min": -85.6759033203125,
      "activations/layer4_attention_weight_max": 130.56890869140625,
      "activations/layer4_attention_weight_min": -121.1063003540039,
      "activations/layer5_attention_weight_max": 95.26640319824219,
      "activations/layer5_attention_weight_min": -84.98011779785156,
      "activations/layer6_attention_weight_max": 56.07155227661133,
      "activations/layer6_attention_weight_min": -55.80051040649414,
      "activations/layer7_attention_weight_max": 75.35321044921875,
      "activations/layer7_attention_weight_min": -70.75404357910156,
      "activations/layer8_attention_weight_max": 39.42385482788086,
      "activations/layer8_attention_weight_min": -37.265869140625,
      "activations/layer9_attention_weight_max": 49.080291748046875,
      "activations/layer9_attention_weight_min": -35.648555755615234,
      "epoch": 3.39,
      "learning_rate": 0.0001294181818181818,
      "loss": 2.9035,
      "step": 58400
    },
    {
      "activations/layer0_attention_weight_max": 16.905319213867188,
      "activations/layer0_attention_weight_min": -11.037284851074219,
      "activations/layer10_attention_weight_max": 47.562583923339844,
      "activations/layer10_attention_weight_min": -35.16943359375,
      "activations/layer11_attention_weight_max": 44.73731994628906,
      "activations/layer11_attention_weight_min": -29.948057174682617,
      "activations/layer12_attention_weight_max": 41.20820617675781,
      "activations/layer12_attention_weight_min": -27.573068618774414,
      "activations/layer13_attention_weight_max": 62.55381774902344,
      "activations/layer13_attention_weight_min": -35.684791564941406,
      "activations/layer14_attention_weight_max": 93.30480194091797,
      "activations/layer14_attention_weight_min": -57.5355224609375,
      "activations/layer15_attention_weight_max": 55.185184478759766,
      "activations/layer15_attention_weight_min": -34.07632827758789,
      "activations/layer16_attention_weight_max": 51.616065979003906,
      "activations/layer16_attention_weight_min": -34.209659576416016,
      "activations/layer17_attention_weight_max": 58.62156295776367,
      "activations/layer17_attention_weight_min": -28.14701271057129,
      "activations/layer18_attention_weight_max": 41.74958038330078,
      "activations/layer18_attention_weight_min": -22.085771560668945,
      "activations/layer19_attention_weight_max": 53.4101676940918,
      "activations/layer19_attention_weight_min": -29.48055076599121,
      "activations/layer1_attention_weight_max": 17.156187057495117,
      "activations/layer1_attention_weight_min": -16.501476287841797,
      "activations/layer20_attention_weight_max": 41.639984130859375,
      "activations/layer20_attention_weight_min": -22.774873733520508,
      "activations/layer21_attention_weight_max": 47.46128845214844,
      "activations/layer21_attention_weight_min": -21.352275848388672,
      "activations/layer22_attention_weight_max": 64.8102035522461,
      "activations/layer22_attention_weight_min": -28.228670120239258,
      "activations/layer23_attention_weight_max": 35.44138717651367,
      "activations/layer23_attention_weight_min": -17.904603958129883,
      "activations/layer2_attention_weight_max": 31.87389373779297,
      "activations/layer2_attention_weight_min": -31.045255661010742,
      "activations/layer3_attention_weight_max": 83.89143371582031,
      "activations/layer3_attention_weight_min": -87.30351257324219,
      "activations/layer4_attention_weight_max": 133.6769256591797,
      "activations/layer4_attention_weight_min": -130.1136932373047,
      "activations/layer5_attention_weight_max": 105.34004211425781,
      "activations/layer5_attention_weight_min": -95.81099700927734,
      "activations/layer6_attention_weight_max": 60.75736618041992,
      "activations/layer6_attention_weight_min": -60.45171356201172,
      "activations/layer7_attention_weight_max": 78.02262115478516,
      "activations/layer7_attention_weight_min": -77.5062484741211,
      "activations/layer8_attention_weight_max": 45.198280334472656,
      "activations/layer8_attention_weight_min": -41.593177795410156,
      "activations/layer9_attention_weight_max": 45.99340057373047,
      "activations/layer9_attention_weight_min": -38.13539123535156,
      "epoch": 3.4,
      "learning_rate": 0.00012939924242424242,
      "loss": 2.861,
      "step": 58450
    },
    {
      "activations/layer0_attention_weight_max": 16.915691375732422,
      "activations/layer0_attention_weight_min": -11.246139526367188,
      "activations/layer10_attention_weight_max": 48.96258544921875,
      "activations/layer10_attention_weight_min": -33.67045974731445,
      "activations/layer11_attention_weight_max": 50.702117919921875,
      "activations/layer11_attention_weight_min": -33.448951721191406,
      "activations/layer12_attention_weight_max": 36.631752014160156,
      "activations/layer12_attention_weight_min": -30.39280891418457,
      "activations/layer13_attention_weight_max": 63.761905670166016,
      "activations/layer13_attention_weight_min": -39.032684326171875,
      "activations/layer14_attention_weight_max": 103.47722625732422,
      "activations/layer14_attention_weight_min": -61.27845001220703,
      "activations/layer15_attention_weight_max": 70.0523452758789,
      "activations/layer15_attention_weight_min": -32.940940856933594,
      "activations/layer16_attention_weight_max": 67.1031723022461,
      "activations/layer16_attention_weight_min": -31.43218231201172,
      "activations/layer17_attention_weight_max": 50.82206726074219,
      "activations/layer17_attention_weight_min": -25.02176856994629,
      "activations/layer18_attention_weight_max": 38.6099739074707,
      "activations/layer18_attention_weight_min": -19.78594970703125,
      "activations/layer19_attention_weight_max": 75.55663299560547,
      "activations/layer19_attention_weight_min": -30.531686782836914,
      "activations/layer1_attention_weight_max": 17.453224182128906,
      "activations/layer1_attention_weight_min": -12.846144676208496,
      "activations/layer20_attention_weight_max": 55.33576965332031,
      "activations/layer20_attention_weight_min": -23.069774627685547,
      "activations/layer21_attention_weight_max": 35.34504318237305,
      "activations/layer21_attention_weight_min": -19.979087829589844,
      "activations/layer22_attention_weight_max": 61.78762435913086,
      "activations/layer22_attention_weight_min": -25.92919158935547,
      "activations/layer23_attention_weight_max": 33.5544548034668,
      "activations/layer23_attention_weight_min": -18.329500198364258,
      "activations/layer2_attention_weight_max": 32.989959716796875,
      "activations/layer2_attention_weight_min": -32.43557357788086,
      "activations/layer3_attention_weight_max": 89.93889617919922,
      "activations/layer3_attention_weight_min": -90.76409912109375,
      "activations/layer4_attention_weight_max": 143.37905883789062,
      "activations/layer4_attention_weight_min": -137.68931579589844,
      "activations/layer5_attention_weight_max": 105.04393768310547,
      "activations/layer5_attention_weight_min": -95.9071273803711,
      "activations/layer6_attention_weight_max": 65.08918762207031,
      "activations/layer6_attention_weight_min": -61.20033264160156,
      "activations/layer7_attention_weight_max": 82.3934555053711,
      "activations/layer7_attention_weight_min": -78.8021011352539,
      "activations/layer8_attention_weight_max": 48.54367446899414,
      "activations/layer8_attention_weight_min": -41.10591506958008,
      "activations/layer9_attention_weight_max": 47.938472747802734,
      "activations/layer9_attention_weight_min": -41.597740173339844,
      "epoch": 3.4,
      "learning_rate": 0.000129380303030303,
      "loss": 2.8929,
      "step": 58500
    },
    {
      "activations/layer0_attention_weight_max": 16.344482421875,
      "activations/layer0_attention_weight_min": -12.171626091003418,
      "activations/layer10_attention_weight_max": 38.17747116088867,
      "activations/layer10_attention_weight_min": -32.365570068359375,
      "activations/layer11_attention_weight_max": 40.142578125,
      "activations/layer11_attention_weight_min": -30.08808708190918,
      "activations/layer12_attention_weight_max": 31.86579132080078,
      "activations/layer12_attention_weight_min": -29.398746490478516,
      "activations/layer13_attention_weight_max": 59.734535217285156,
      "activations/layer13_attention_weight_min": -38.258460998535156,
      "activations/layer14_attention_weight_max": 75.94268035888672,
      "activations/layer14_attention_weight_min": -51.40935516357422,
      "activations/layer15_attention_weight_max": 48.220394134521484,
      "activations/layer15_attention_weight_min": -30.820741653442383,
      "activations/layer16_attention_weight_max": 45.842769622802734,
      "activations/layer16_attention_weight_min": -29.62313461303711,
      "activations/layer17_attention_weight_max": 46.73747253417969,
      "activations/layer17_attention_weight_min": -25.971689224243164,
      "activations/layer18_attention_weight_max": 29.60698127746582,
      "activations/layer18_attention_weight_min": -18.072181701660156,
      "activations/layer19_attention_weight_max": 46.19035720825195,
      "activations/layer19_attention_weight_min": -27.037038803100586,
      "activations/layer1_attention_weight_max": 16.588857650756836,
      "activations/layer1_attention_weight_min": -14.981219291687012,
      "activations/layer20_attention_weight_max": 34.58042526245117,
      "activations/layer20_attention_weight_min": -24.001476287841797,
      "activations/layer21_attention_weight_max": 34.515525817871094,
      "activations/layer21_attention_weight_min": -21.488861083984375,
      "activations/layer22_attention_weight_max": 50.89451217651367,
      "activations/layer22_attention_weight_min": -28.41486930847168,
      "activations/layer23_attention_weight_max": 32.63199996948242,
      "activations/layer23_attention_weight_min": -18.034534454345703,
      "activations/layer2_attention_weight_max": 30.623807907104492,
      "activations/layer2_attention_weight_min": -30.22452163696289,
      "activations/layer3_attention_weight_max": 84.47954559326172,
      "activations/layer3_attention_weight_min": -85.91976928710938,
      "activations/layer4_attention_weight_max": 139.9564666748047,
      "activations/layer4_attention_weight_min": -131.15240478515625,
      "activations/layer5_attention_weight_max": 97.51934051513672,
      "activations/layer5_attention_weight_min": -87.98880004882812,
      "activations/layer6_attention_weight_max": 57.16209411621094,
      "activations/layer6_attention_weight_min": -53.602542877197266,
      "activations/layer7_attention_weight_max": 75.94954681396484,
      "activations/layer7_attention_weight_min": -67.90422821044922,
      "activations/layer8_attention_weight_max": 37.41197204589844,
      "activations/layer8_attention_weight_min": -34.87873458862305,
      "activations/layer9_attention_weight_max": 57.14719772338867,
      "activations/layer9_attention_weight_min": -41.101837158203125,
      "epoch": 3.4,
      "learning_rate": 0.00012936136363636363,
      "loss": 2.8879,
      "step": 58550
    },
    {
      "activations/layer0_attention_weight_max": 16.442317962646484,
      "activations/layer0_attention_weight_min": -11.699198722839355,
      "activations/layer10_attention_weight_max": 42.39605712890625,
      "activations/layer10_attention_weight_min": -32.5268440246582,
      "activations/layer11_attention_weight_max": 35.51442337036133,
      "activations/layer11_attention_weight_min": -27.647619247436523,
      "activations/layer12_attention_weight_max": 59.936588287353516,
      "activations/layer12_attention_weight_min": -34.81011962890625,
      "activations/layer13_attention_weight_max": 47.562355041503906,
      "activations/layer13_attention_weight_min": -34.234352111816406,
      "activations/layer14_attention_weight_max": 67.45984649658203,
      "activations/layer14_attention_weight_min": -52.01333999633789,
      "activations/layer15_attention_weight_max": 48.031898498535156,
      "activations/layer15_attention_weight_min": -31.963024139404297,
      "activations/layer16_attention_weight_max": 45.217899322509766,
      "activations/layer16_attention_weight_min": -30.87241554260254,
      "activations/layer17_attention_weight_max": 42.85005569458008,
      "activations/layer17_attention_weight_min": -26.873857498168945,
      "activations/layer18_attention_weight_max": 31.082130432128906,
      "activations/layer18_attention_weight_min": -19.232311248779297,
      "activations/layer19_attention_weight_max": 41.195369720458984,
      "activations/layer19_attention_weight_min": -27.61433982849121,
      "activations/layer1_attention_weight_max": 17.381237030029297,
      "activations/layer1_attention_weight_min": -12.66215705871582,
      "activations/layer20_attention_weight_max": 33.668331146240234,
      "activations/layer20_attention_weight_min": -22.70323371887207,
      "activations/layer21_attention_weight_max": 28.885032653808594,
      "activations/layer21_attention_weight_min": -20.63067626953125,
      "activations/layer22_attention_weight_max": 49.40472412109375,
      "activations/layer22_attention_weight_min": -27.59148597717285,
      "activations/layer23_attention_weight_max": 28.821327209472656,
      "activations/layer23_attention_weight_min": -18.904071807861328,
      "activations/layer2_attention_weight_max": 31.611576080322266,
      "activations/layer2_attention_weight_min": -30.787580490112305,
      "activations/layer3_attention_weight_max": 82.37733459472656,
      "activations/layer3_attention_weight_min": -87.40692138671875,
      "activations/layer4_attention_weight_max": 132.3084259033203,
      "activations/layer4_attention_weight_min": -125.42252349853516,
      "activations/layer5_attention_weight_max": 99.1828842163086,
      "activations/layer5_attention_weight_min": -89.25227355957031,
      "activations/layer6_attention_weight_max": 57.08830261230469,
      "activations/layer6_attention_weight_min": -55.9232063293457,
      "activations/layer7_attention_weight_max": 78.37689208984375,
      "activations/layer7_attention_weight_min": -69.84089660644531,
      "activations/layer8_attention_weight_max": 39.89680862426758,
      "activations/layer8_attention_weight_min": -38.344844818115234,
      "activations/layer9_attention_weight_max": 49.40597915649414,
      "activations/layer9_attention_weight_min": -38.81358337402344,
      "epoch": 3.4,
      "learning_rate": 0.00012934242424242424,
      "loss": 2.8968,
      "step": 58600
    },
    {
      "activations/layer0_attention_weight_max": 16.818544387817383,
      "activations/layer0_attention_weight_min": -10.316906929016113,
      "activations/layer10_attention_weight_max": 45.20331573486328,
      "activations/layer10_attention_weight_min": -31.94997215270996,
      "activations/layer11_attention_weight_max": 38.48252868652344,
      "activations/layer11_attention_weight_min": -30.32837677001953,
      "activations/layer12_attention_weight_max": 39.936397552490234,
      "activations/layer12_attention_weight_min": -24.22182846069336,
      "activations/layer13_attention_weight_max": 53.370914459228516,
      "activations/layer13_attention_weight_min": -35.247745513916016,
      "activations/layer14_attention_weight_max": 80.42569732666016,
      "activations/layer14_attention_weight_min": -53.226318359375,
      "activations/layer15_attention_weight_max": 57.50421905517578,
      "activations/layer15_attention_weight_min": -33.27069091796875,
      "activations/layer16_attention_weight_max": 47.64192581176758,
      "activations/layer16_attention_weight_min": -28.956289291381836,
      "activations/layer17_attention_weight_max": 43.65654373168945,
      "activations/layer17_attention_weight_min": -27.289440155029297,
      "activations/layer18_attention_weight_max": 35.997676849365234,
      "activations/layer18_attention_weight_min": -19.33199119567871,
      "activations/layer19_attention_weight_max": 51.719390869140625,
      "activations/layer19_attention_weight_min": -33.84901809692383,
      "activations/layer1_attention_weight_max": 16.622230529785156,
      "activations/layer1_attention_weight_min": -13.835824012756348,
      "activations/layer20_attention_weight_max": 45.30634689331055,
      "activations/layer20_attention_weight_min": -22.172622680664062,
      "activations/layer21_attention_weight_max": 39.34827423095703,
      "activations/layer21_attention_weight_min": -20.54575538635254,
      "activations/layer22_attention_weight_max": 56.72248840332031,
      "activations/layer22_attention_weight_min": -28.649906158447266,
      "activations/layer23_attention_weight_max": 33.86397171020508,
      "activations/layer23_attention_weight_min": -16.504718780517578,
      "activations/layer2_attention_weight_max": 30.76559829711914,
      "activations/layer2_attention_weight_min": -30.571393966674805,
      "activations/layer3_attention_weight_max": 82.11524963378906,
      "activations/layer3_attention_weight_min": -82.37972259521484,
      "activations/layer4_attention_weight_max": 136.0590057373047,
      "activations/layer4_attention_weight_min": -127.74161529541016,
      "activations/layer5_attention_weight_max": 102.47966766357422,
      "activations/layer5_attention_weight_min": -92.48149871826172,
      "activations/layer6_attention_weight_max": 63.955848693847656,
      "activations/layer6_attention_weight_min": -58.33023452758789,
      "activations/layer7_attention_weight_max": 80.25074005126953,
      "activations/layer7_attention_weight_min": -71.60546875,
      "activations/layer8_attention_weight_max": 44.57411575317383,
      "activations/layer8_attention_weight_min": -38.79607009887695,
      "activations/layer9_attention_weight_max": 43.14506149291992,
      "activations/layer9_attention_weight_min": -38.83638000488281,
      "epoch": 3.41,
      "learning_rate": 0.00012932348484848483,
      "loss": 2.8955,
      "step": 58650
    },
    {
      "activations/layer0_attention_weight_max": 17.328750610351562,
      "activations/layer0_attention_weight_min": -12.687585830688477,
      "activations/layer10_attention_weight_max": 48.81209182739258,
      "activations/layer10_attention_weight_min": -32.93162536621094,
      "activations/layer11_attention_weight_max": 43.16361999511719,
      "activations/layer11_attention_weight_min": -31.32384490966797,
      "activations/layer12_attention_weight_max": 34.099769592285156,
      "activations/layer12_attention_weight_min": -30.1126651763916,
      "activations/layer13_attention_weight_max": 58.4029655456543,
      "activations/layer13_attention_weight_min": -36.608619689941406,
      "activations/layer14_attention_weight_max": 100.29084777832031,
      "activations/layer14_attention_weight_min": -65.74462890625,
      "activations/layer15_attention_weight_max": 55.26394271850586,
      "activations/layer15_attention_weight_min": -30.757827758789062,
      "activations/layer16_attention_weight_max": 50.956119537353516,
      "activations/layer16_attention_weight_min": -29.258201599121094,
      "activations/layer17_attention_weight_max": 44.2383918762207,
      "activations/layer17_attention_weight_min": -25.45338249206543,
      "activations/layer18_attention_weight_max": 28.835201263427734,
      "activations/layer18_attention_weight_min": -18.46176528930664,
      "activations/layer19_attention_weight_max": 53.45502471923828,
      "activations/layer19_attention_weight_min": -29.17957878112793,
      "activations/layer1_attention_weight_max": 17.530881881713867,
      "activations/layer1_attention_weight_min": -13.533783912658691,
      "activations/layer20_attention_weight_max": 42.255184173583984,
      "activations/layer20_attention_weight_min": -23.82135581970215,
      "activations/layer21_attention_weight_max": 36.64564895629883,
      "activations/layer21_attention_weight_min": -22.4035587310791,
      "activations/layer22_attention_weight_max": 58.50027847290039,
      "activations/layer22_attention_weight_min": -27.258100509643555,
      "activations/layer23_attention_weight_max": 32.28279113769531,
      "activations/layer23_attention_weight_min": -17.566057205200195,
      "activations/layer2_attention_weight_max": 31.036664962768555,
      "activations/layer2_attention_weight_min": -31.053077697753906,
      "activations/layer3_attention_weight_max": 82.718017578125,
      "activations/layer3_attention_weight_min": -86.70620727539062,
      "activations/layer4_attention_weight_max": 135.3603515625,
      "activations/layer4_attention_weight_min": -127.2643814086914,
      "activations/layer5_attention_weight_max": 101.99085998535156,
      "activations/layer5_attention_weight_min": -88.49605560302734,
      "activations/layer6_attention_weight_max": 61.31703567504883,
      "activations/layer6_attention_weight_min": -57.73790740966797,
      "activations/layer7_attention_weight_max": 77.33887481689453,
      "activations/layer7_attention_weight_min": -71.23554229736328,
      "activations/layer8_attention_weight_max": 41.50750732421875,
      "activations/layer8_attention_weight_min": -36.91880416870117,
      "activations/layer9_attention_weight_max": 46.78481674194336,
      "activations/layer9_attention_weight_min": -44.797462463378906,
      "epoch": 3.41,
      "learning_rate": 0.00012930454545454544,
      "loss": 2.876,
      "step": 58700
    },
    {
      "activations/layer0_attention_weight_max": 15.104840278625488,
      "activations/layer0_attention_weight_min": -12.196998596191406,
      "activations/layer10_attention_weight_max": 39.2521858215332,
      "activations/layer10_attention_weight_min": -33.60065460205078,
      "activations/layer11_attention_weight_max": 35.62949752807617,
      "activations/layer11_attention_weight_min": -28.933834075927734,
      "activations/layer12_attention_weight_max": 31.41399383544922,
      "activations/layer12_attention_weight_min": -25.721769332885742,
      "activations/layer13_attention_weight_max": 52.2912483215332,
      "activations/layer13_attention_weight_min": -35.86737060546875,
      "activations/layer14_attention_weight_max": 77.37488555908203,
      "activations/layer14_attention_weight_min": -55.29030227661133,
      "activations/layer15_attention_weight_max": 52.84788513183594,
      "activations/layer15_attention_weight_min": -34.9737663269043,
      "activations/layer16_attention_weight_max": 46.41620635986328,
      "activations/layer16_attention_weight_min": -29.044239044189453,
      "activations/layer17_attention_weight_max": 46.248992919921875,
      "activations/layer17_attention_weight_min": -25.156612396240234,
      "activations/layer18_attention_weight_max": 38.63349151611328,
      "activations/layer18_attention_weight_min": -21.610387802124023,
      "activations/layer19_attention_weight_max": 48.497108459472656,
      "activations/layer19_attention_weight_min": -29.779958724975586,
      "activations/layer1_attention_weight_max": 17.688505172729492,
      "activations/layer1_attention_weight_min": -12.81187629699707,
      "activations/layer20_attention_weight_max": 35.711788177490234,
      "activations/layer20_attention_weight_min": -22.4571533203125,
      "activations/layer21_attention_weight_max": 42.90507888793945,
      "activations/layer21_attention_weight_min": -21.802377700805664,
      "activations/layer22_attention_weight_max": 55.007930755615234,
      "activations/layer22_attention_weight_min": -31.786096572875977,
      "activations/layer23_attention_weight_max": 30.92193603515625,
      "activations/layer23_attention_weight_min": -19.036436080932617,
      "activations/layer2_attention_weight_max": 28.813474655151367,
      "activations/layer2_attention_weight_min": -30.024925231933594,
      "activations/layer3_attention_weight_max": 82.68387603759766,
      "activations/layer3_attention_weight_min": -85.51383972167969,
      "activations/layer4_attention_weight_max": 125.4625244140625,
      "activations/layer4_attention_weight_min": -127.32840728759766,
      "activations/layer5_attention_weight_max": 94.33565521240234,
      "activations/layer5_attention_weight_min": -91.84404754638672,
      "activations/layer6_attention_weight_max": 58.61414337158203,
      "activations/layer6_attention_weight_min": -57.91508483886719,
      "activations/layer7_attention_weight_max": 77.69001007080078,
      "activations/layer7_attention_weight_min": -75.12580108642578,
      "activations/layer8_attention_weight_max": 39.57276153564453,
      "activations/layer8_attention_weight_min": -39.56422805786133,
      "activations/layer9_attention_weight_max": 40.80753707885742,
      "activations/layer9_attention_weight_min": -35.13467025756836,
      "epoch": 3.41,
      "learning_rate": 0.00012928560606060606,
      "loss": 2.9018,
      "step": 58750
    },
    {
      "activations/layer0_attention_weight_max": 14.93869686126709,
      "activations/layer0_attention_weight_min": -12.552701950073242,
      "activations/layer10_attention_weight_max": 40.225196838378906,
      "activations/layer10_attention_weight_min": -31.984922409057617,
      "activations/layer11_attention_weight_max": 36.853275299072266,
      "activations/layer11_attention_weight_min": -28.346118927001953,
      "activations/layer12_attention_weight_max": 34.49908447265625,
      "activations/layer12_attention_weight_min": -29.449607849121094,
      "activations/layer13_attention_weight_max": 55.43102264404297,
      "activations/layer13_attention_weight_min": -39.92950439453125,
      "activations/layer14_attention_weight_max": 77.4723129272461,
      "activations/layer14_attention_weight_min": -60.10149383544922,
      "activations/layer15_attention_weight_max": 51.54402160644531,
      "activations/layer15_attention_weight_min": -31.814767837524414,
      "activations/layer16_attention_weight_max": 41.314491271972656,
      "activations/layer16_attention_weight_min": -28.40175437927246,
      "activations/layer17_attention_weight_max": 46.07112503051758,
      "activations/layer17_attention_weight_min": -26.250818252563477,
      "activations/layer18_attention_weight_max": 30.102603912353516,
      "activations/layer18_attention_weight_min": -19.804012298583984,
      "activations/layer19_attention_weight_max": 45.94012451171875,
      "activations/layer19_attention_weight_min": -28.007970809936523,
      "activations/layer1_attention_weight_max": 17.568754196166992,
      "activations/layer1_attention_weight_min": -13.788066864013672,
      "activations/layer20_attention_weight_max": 35.06682586669922,
      "activations/layer20_attention_weight_min": -23.589139938354492,
      "activations/layer21_attention_weight_max": 38.419498443603516,
      "activations/layer21_attention_weight_min": -22.634159088134766,
      "activations/layer22_attention_weight_max": 50.31570816040039,
      "activations/layer22_attention_weight_min": -29.182416915893555,
      "activations/layer23_attention_weight_max": 29.614261627197266,
      "activations/layer23_attention_weight_min": -20.293968200683594,
      "activations/layer2_attention_weight_max": 31.241973876953125,
      "activations/layer2_attention_weight_min": -30.050460815429688,
      "activations/layer3_attention_weight_max": 81.44327545166016,
      "activations/layer3_attention_weight_min": -89.16534423828125,
      "activations/layer4_attention_weight_max": 125.8729476928711,
      "activations/layer4_attention_weight_min": -128.87722778320312,
      "activations/layer5_attention_weight_max": 94.05519104003906,
      "activations/layer5_attention_weight_min": -96.78456115722656,
      "activations/layer6_attention_weight_max": 53.369407653808594,
      "activations/layer6_attention_weight_min": -54.32703399658203,
      "activations/layer7_attention_weight_max": 73.67866516113281,
      "activations/layer7_attention_weight_min": -75.48039245605469,
      "activations/layer8_attention_weight_max": 37.34455871582031,
      "activations/layer8_attention_weight_min": -39.440975189208984,
      "activations/layer9_attention_weight_max": 41.96063232421875,
      "activations/layer9_attention_weight_min": -33.29863739013672,
      "epoch": 3.42,
      "learning_rate": 0.00012926666666666665,
      "loss": 2.9,
      "step": 58800
    },
    {
      "activations/layer0_attention_weight_max": 17.493789672851562,
      "activations/layer0_attention_weight_min": -11.206116676330566,
      "activations/layer10_attention_weight_max": 37.73006820678711,
      "activations/layer10_attention_weight_min": -30.855772018432617,
      "activations/layer11_attention_weight_max": 32.031402587890625,
      "activations/layer11_attention_weight_min": -28.713550567626953,
      "activations/layer12_attention_weight_max": 49.55937194824219,
      "activations/layer12_attention_weight_min": -27.732322692871094,
      "activations/layer13_attention_weight_max": 49.326576232910156,
      "activations/layer13_attention_weight_min": -34.85354232788086,
      "activations/layer14_attention_weight_max": 88.65250396728516,
      "activations/layer14_attention_weight_min": -57.17770767211914,
      "activations/layer15_attention_weight_max": 49.58717346191406,
      "activations/layer15_attention_weight_min": -32.82166290283203,
      "activations/layer16_attention_weight_max": 48.58966827392578,
      "activations/layer16_attention_weight_min": -30.58470916748047,
      "activations/layer17_attention_weight_max": 44.682186126708984,
      "activations/layer17_attention_weight_min": -27.488643646240234,
      "activations/layer18_attention_weight_max": 36.90319061279297,
      "activations/layer18_attention_weight_min": -22.153127670288086,
      "activations/layer19_attention_weight_max": 50.128387451171875,
      "activations/layer19_attention_weight_min": -29.568761825561523,
      "activations/layer1_attention_weight_max": 17.38819694519043,
      "activations/layer1_attention_weight_min": -15.501687049865723,
      "activations/layer20_attention_weight_max": 39.763736724853516,
      "activations/layer20_attention_weight_min": -22.68551254272461,
      "activations/layer21_attention_weight_max": 42.095245361328125,
      "activations/layer21_attention_weight_min": -19.785140991210938,
      "activations/layer22_attention_weight_max": 52.71335983276367,
      "activations/layer22_attention_weight_min": -25.838544845581055,
      "activations/layer23_attention_weight_max": 31.354629516601562,
      "activations/layer23_attention_weight_min": -18.547210693359375,
      "activations/layer2_attention_weight_max": 30.54181671142578,
      "activations/layer2_attention_weight_min": -31.099369049072266,
      "activations/layer3_attention_weight_max": 82.57032775878906,
      "activations/layer3_attention_weight_min": -87.19702911376953,
      "activations/layer4_attention_weight_max": 131.13980102539062,
      "activations/layer4_attention_weight_min": -127.55549621582031,
      "activations/layer5_attention_weight_max": 98.59008026123047,
      "activations/layer5_attention_weight_min": -88.09414672851562,
      "activations/layer6_attention_weight_max": 57.2678108215332,
      "activations/layer6_attention_weight_min": -55.845157623291016,
      "activations/layer7_attention_weight_max": 71.9326171875,
      "activations/layer7_attention_weight_min": -74.17267608642578,
      "activations/layer8_attention_weight_max": 38.94708251953125,
      "activations/layer8_attention_weight_min": -37.83674621582031,
      "activations/layer9_attention_weight_max": 43.44823455810547,
      "activations/layer9_attention_weight_min": -36.90789031982422,
      "epoch": 3.42,
      "learning_rate": 0.00012924772727272726,
      "loss": 2.9083,
      "step": 58850
    },
    {
      "activations/layer0_attention_weight_max": 17.346689224243164,
      "activations/layer0_attention_weight_min": -10.705399513244629,
      "activations/layer10_attention_weight_max": 42.59571075439453,
      "activations/layer10_attention_weight_min": -33.885135650634766,
      "activations/layer11_attention_weight_max": 38.78059005737305,
      "activations/layer11_attention_weight_min": -28.505001068115234,
      "activations/layer12_attention_weight_max": 31.43622398376465,
      "activations/layer12_attention_weight_min": -26.246427536010742,
      "activations/layer13_attention_weight_max": 57.90470886230469,
      "activations/layer13_attention_weight_min": -39.614280700683594,
      "activations/layer14_attention_weight_max": 76.86395263671875,
      "activations/layer14_attention_weight_min": -50.91533279418945,
      "activations/layer15_attention_weight_max": 53.85365295410156,
      "activations/layer15_attention_weight_min": -35.19580078125,
      "activations/layer16_attention_weight_max": 44.01346206665039,
      "activations/layer16_attention_weight_min": -31.386363983154297,
      "activations/layer17_attention_weight_max": 47.56917190551758,
      "activations/layer17_attention_weight_min": -30.103206634521484,
      "activations/layer18_attention_weight_max": 32.0012321472168,
      "activations/layer18_attention_weight_min": -19.782941818237305,
      "activations/layer19_attention_weight_max": 49.51947021484375,
      "activations/layer19_attention_weight_min": -29.536640167236328,
      "activations/layer1_attention_weight_max": 17.37941551208496,
      "activations/layer1_attention_weight_min": -14.497930526733398,
      "activations/layer20_attention_weight_max": 42.270484924316406,
      "activations/layer20_attention_weight_min": -24.694805145263672,
      "activations/layer21_attention_weight_max": 36.489749908447266,
      "activations/layer21_attention_weight_min": -19.643728256225586,
      "activations/layer22_attention_weight_max": 51.795475006103516,
      "activations/layer22_attention_weight_min": -27.567136764526367,
      "activations/layer23_attention_weight_max": 30.84435272216797,
      "activations/layer23_attention_weight_min": -18.085838317871094,
      "activations/layer2_attention_weight_max": 33.64984893798828,
      "activations/layer2_attention_weight_min": -32.76861572265625,
      "activations/layer3_attention_weight_max": 87.92288208007812,
      "activations/layer3_attention_weight_min": -94.02064514160156,
      "activations/layer4_attention_weight_max": 141.2501678466797,
      "activations/layer4_attention_weight_min": -132.68475341796875,
      "activations/layer5_attention_weight_max": 104.62527465820312,
      "activations/layer5_attention_weight_min": -95.74691009521484,
      "activations/layer6_attention_weight_max": 62.811492919921875,
      "activations/layer6_attention_weight_min": -59.00153350830078,
      "activations/layer7_attention_weight_max": 85.53047180175781,
      "activations/layer7_attention_weight_min": -79.21247863769531,
      "activations/layer8_attention_weight_max": 44.33198547363281,
      "activations/layer8_attention_weight_min": -41.39097595214844,
      "activations/layer9_attention_weight_max": 41.163352966308594,
      "activations/layer9_attention_weight_min": -34.83112716674805,
      "epoch": 3.42,
      "learning_rate": 0.00012922878787878785,
      "loss": 2.8902,
      "step": 58900
    },
    {
      "activations/layer0_attention_weight_max": 16.98943519592285,
      "activations/layer0_attention_weight_min": -10.651281356811523,
      "activations/layer10_attention_weight_max": 38.08209991455078,
      "activations/layer10_attention_weight_min": -30.098012924194336,
      "activations/layer11_attention_weight_max": 33.235145568847656,
      "activations/layer11_attention_weight_min": -27.645030975341797,
      "activations/layer12_attention_weight_max": 30.795181274414062,
      "activations/layer12_attention_weight_min": -24.12045669555664,
      "activations/layer13_attention_weight_max": 44.93323516845703,
      "activations/layer13_attention_weight_min": -34.55386734008789,
      "activations/layer14_attention_weight_max": 65.97171783447266,
      "activations/layer14_attention_weight_min": -47.132389068603516,
      "activations/layer15_attention_weight_max": 44.620384216308594,
      "activations/layer15_attention_weight_min": -31.611371994018555,
      "activations/layer16_attention_weight_max": 41.63132095336914,
      "activations/layer16_attention_weight_min": -29.20442008972168,
      "activations/layer17_attention_weight_max": 39.9895133972168,
      "activations/layer17_attention_weight_min": -24.157503128051758,
      "activations/layer18_attention_weight_max": 30.4891414642334,
      "activations/layer18_attention_weight_min": -19.825519561767578,
      "activations/layer19_attention_weight_max": 39.557552337646484,
      "activations/layer19_attention_weight_min": -28.259275436401367,
      "activations/layer1_attention_weight_max": 17.7210636138916,
      "activations/layer1_attention_weight_min": -14.856878280639648,
      "activations/layer20_attention_weight_max": 35.0897331237793,
      "activations/layer20_attention_weight_min": -25.10655403137207,
      "activations/layer21_attention_weight_max": 30.077468872070312,
      "activations/layer21_attention_weight_min": -19.54121208190918,
      "activations/layer22_attention_weight_max": 51.79326629638672,
      "activations/layer22_attention_weight_min": -29.21074104309082,
      "activations/layer23_attention_weight_max": 26.616086959838867,
      "activations/layer23_attention_weight_min": -18.39059829711914,
      "activations/layer2_attention_weight_max": 31.50092124938965,
      "activations/layer2_attention_weight_min": -29.77435874938965,
      "activations/layer3_attention_weight_max": 83.03760528564453,
      "activations/layer3_attention_weight_min": -84.08331298828125,
      "activations/layer4_attention_weight_max": 131.4641876220703,
      "activations/layer4_attention_weight_min": -126.73724365234375,
      "activations/layer5_attention_weight_max": 96.50631713867188,
      "activations/layer5_attention_weight_min": -89.49549865722656,
      "activations/layer6_attention_weight_max": 57.870853424072266,
      "activations/layer6_attention_weight_min": -53.20769119262695,
      "activations/layer7_attention_weight_max": 73.23600769042969,
      "activations/layer7_attention_weight_min": -71.502197265625,
      "activations/layer8_attention_weight_max": 37.67625427246094,
      "activations/layer8_attention_weight_min": -37.630516052246094,
      "activations/layer9_attention_weight_max": 38.13227462768555,
      "activations/layer9_attention_weight_min": -34.868125915527344,
      "epoch": 3.43,
      "learning_rate": 0.00012920984848484846,
      "loss": 2.8734,
      "step": 58950
    },
    {
      "activations/layer0_attention_weight_max": 17.359188079833984,
      "activations/layer0_attention_weight_min": -11.693724632263184,
      "activations/layer10_attention_weight_max": 38.150691986083984,
      "activations/layer10_attention_weight_min": -30.035350799560547,
      "activations/layer11_attention_weight_max": 29.92988395690918,
      "activations/layer11_attention_weight_min": -26.80801773071289,
      "activations/layer12_attention_weight_max": 34.924041748046875,
      "activations/layer12_attention_weight_min": -26.03464698791504,
      "activations/layer13_attention_weight_max": 43.983951568603516,
      "activations/layer13_attention_weight_min": -38.91960906982422,
      "activations/layer14_attention_weight_max": 60.10163879394531,
      "activations/layer14_attention_weight_min": -46.34267807006836,
      "activations/layer15_attention_weight_max": 41.703773498535156,
      "activations/layer15_attention_weight_min": -32.11888122558594,
      "activations/layer16_attention_weight_max": 37.79274368286133,
      "activations/layer16_attention_weight_min": -30.91324806213379,
      "activations/layer17_attention_weight_max": 38.886138916015625,
      "activations/layer17_attention_weight_min": -26.48883056640625,
      "activations/layer18_attention_weight_max": 28.476049423217773,
      "activations/layer18_attention_weight_min": -20.87110137939453,
      "activations/layer19_attention_weight_max": 40.63288497924805,
      "activations/layer19_attention_weight_min": -32.77106857299805,
      "activations/layer1_attention_weight_max": 18.603803634643555,
      "activations/layer1_attention_weight_min": -13.666232109069824,
      "activations/layer20_attention_weight_max": 32.613216400146484,
      "activations/layer20_attention_weight_min": -22.56260108947754,
      "activations/layer21_attention_weight_max": 30.39800262451172,
      "activations/layer21_attention_weight_min": -21.01152992248535,
      "activations/layer22_attention_weight_max": 42.03034210205078,
      "activations/layer22_attention_weight_min": -34.592960357666016,
      "activations/layer23_attention_weight_max": 24.276397705078125,
      "activations/layer23_attention_weight_min": -20.543621063232422,
      "activations/layer2_attention_weight_max": 29.580711364746094,
      "activations/layer2_attention_weight_min": -28.90206527709961,
      "activations/layer3_attention_weight_max": 78.59658813476562,
      "activations/layer3_attention_weight_min": -83.56376647949219,
      "activations/layer4_attention_weight_max": 123.9972152709961,
      "activations/layer4_attention_weight_min": -120.48482513427734,
      "activations/layer5_attention_weight_max": 92.28976440429688,
      "activations/layer5_attention_weight_min": -84.27680206298828,
      "activations/layer6_attention_weight_max": 53.202701568603516,
      "activations/layer6_attention_weight_min": -54.652584075927734,
      "activations/layer7_attention_weight_max": 72.5724868774414,
      "activations/layer7_attention_weight_min": -68.4364242553711,
      "activations/layer8_attention_weight_max": 35.403961181640625,
      "activations/layer8_attention_weight_min": -36.150115966796875,
      "activations/layer9_attention_weight_max": 38.1002311706543,
      "activations/layer9_attention_weight_min": -33.215126037597656,
      "epoch": 3.43,
      "learning_rate": 0.00012919090909090908,
      "loss": 2.8809,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_loss": 2.84375,
      "eval_runtime": 8.4601,
      "eval_samples_per_second": 507.562,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_openwebtext_loss": 2.84375,
      "eval_openwebtext_ppl": 17.180070153709277,
      "eval_openwebtext_runtime": 8.4601,
      "eval_openwebtext_samples_per_second": 507.562,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_wikitext_loss": 3.087890625,
      "eval_wikitext_ppl": 21.930768935644906,
      "eval_wikitext_runtime": 2.0136,
      "eval_wikitext_samples_per_second": 226.457,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_lambada_loss": 2.93359375,
      "eval_lambada_ppl": 18.79505399780919,
      "eval_lambada_runtime": 9.6204,
      "eval_lambada_samples_per_second": 506.113,
      "step": 59000
    },
    {
      "activations/layer0_attention_weight_max": 16.48295783996582,
      "activations/layer0_attention_weight_min": -10.539069175720215,
      "activations/layer10_attention_weight_max": 39.81825637817383,
      "activations/layer10_attention_weight_min": -30.746856689453125,
      "activations/layer11_attention_weight_max": 44.958778381347656,
      "activations/layer11_attention_weight_min": -32.851226806640625,
      "activations/layer12_attention_weight_max": 29.85576629638672,
      "activations/layer12_attention_weight_min": -28.008310317993164,
      "activations/layer13_attention_weight_max": 51.91427230834961,
      "activations/layer13_attention_weight_min": -38.33717727661133,
      "activations/layer14_attention_weight_max": 61.14104080200195,
      "activations/layer14_attention_weight_min": -45.61772537231445,
      "activations/layer15_attention_weight_max": 43.74972915649414,
      "activations/layer15_attention_weight_min": -30.421913146972656,
      "activations/layer16_attention_weight_max": 44.59847640991211,
      "activations/layer16_attention_weight_min": -29.38838768005371,
      "activations/layer17_attention_weight_max": 43.28383255004883,
      "activations/layer17_attention_weight_min": -25.755868911743164,
      "activations/layer18_attention_weight_max": 30.75825309753418,
      "activations/layer18_attention_weight_min": -21.606258392333984,
      "activations/layer19_attention_weight_max": 50.79038619995117,
      "activations/layer19_attention_weight_min": -28.4793758392334,
      "activations/layer1_attention_weight_max": 17.993009567260742,
      "activations/layer1_attention_weight_min": -14.30831241607666,
      "activations/layer20_attention_weight_max": 33.6906623840332,
      "activations/layer20_attention_weight_min": -23.2635440826416,
      "activations/layer21_attention_weight_max": 35.177589416503906,
      "activations/layer21_attention_weight_min": -21.98296546936035,
      "activations/layer22_attention_weight_max": 49.04345703125,
      "activations/layer22_attention_weight_min": -26.52960205078125,
      "activations/layer23_attention_weight_max": 29.43195343017578,
      "activations/layer23_attention_weight_min": -16.783056259155273,
      "activations/layer2_attention_weight_max": 30.918292999267578,
      "activations/layer2_attention_weight_min": -29.715900421142578,
      "activations/layer3_attention_weight_max": 83.29672241210938,
      "activations/layer3_attention_weight_min": -89.10163116455078,
      "activations/layer4_attention_weight_max": 133.74510192871094,
      "activations/layer4_attention_weight_min": -128.6599578857422,
      "activations/layer5_attention_weight_max": 101.85163879394531,
      "activations/layer5_attention_weight_min": -95.13380432128906,
      "activations/layer6_attention_weight_max": 58.642234802246094,
      "activations/layer6_attention_weight_min": -54.47203826904297,
      "activations/layer7_attention_weight_max": 79.80015563964844,
      "activations/layer7_attention_weight_min": -76.1836929321289,
      "activations/layer8_attention_weight_max": 42.66762161254883,
      "activations/layer8_attention_weight_min": -39.54158401489258,
      "activations/layer9_attention_weight_max": 52.01386642456055,
      "activations/layer9_attention_weight_min": -47.46430206298828,
      "epoch": 3.43,
      "learning_rate": 0.00012917196969696967,
      "loss": 2.8995,
      "step": 59050
    },
    {
      "activations/layer0_attention_weight_max": 16.42886734008789,
      "activations/layer0_attention_weight_min": -11.48510456085205,
      "activations/layer10_attention_weight_max": 42.023048400878906,
      "activations/layer10_attention_weight_min": -34.62485885620117,
      "activations/layer11_attention_weight_max": 36.214630126953125,
      "activations/layer11_attention_weight_min": -30.408315658569336,
      "activations/layer12_attention_weight_max": 38.70594787597656,
      "activations/layer12_attention_weight_min": -27.14381217956543,
      "activations/layer13_attention_weight_max": 51.56721496582031,
      "activations/layer13_attention_weight_min": -35.955501556396484,
      "activations/layer14_attention_weight_max": 76.91751861572266,
      "activations/layer14_attention_weight_min": -50.042423248291016,
      "activations/layer15_attention_weight_max": 55.753173828125,
      "activations/layer15_attention_weight_min": -35.849021911621094,
      "activations/layer16_attention_weight_max": 38.41716384887695,
      "activations/layer16_attention_weight_min": -31.547622680664062,
      "activations/layer17_attention_weight_max": 42.1363410949707,
      "activations/layer17_attention_weight_min": -27.670896530151367,
      "activations/layer18_attention_weight_max": 32.57610321044922,
      "activations/layer18_attention_weight_min": -19.14688491821289,
      "activations/layer19_attention_weight_max": 43.14821243286133,
      "activations/layer19_attention_weight_min": -30.371906280517578,
      "activations/layer1_attention_weight_max": 16.8289737701416,
      "activations/layer1_attention_weight_min": -14.276228904724121,
      "activations/layer20_attention_weight_max": 31.404571533203125,
      "activations/layer20_attention_weight_min": -23.747886657714844,
      "activations/layer21_attention_weight_max": 30.6218318939209,
      "activations/layer21_attention_weight_min": -21.1683406829834,
      "activations/layer22_attention_weight_max": 48.53591537475586,
      "activations/layer22_attention_weight_min": -27.808889389038086,
      "activations/layer23_attention_weight_max": 27.620351791381836,
      "activations/layer23_attention_weight_min": -16.692033767700195,
      "activations/layer2_attention_weight_max": 30.06391143798828,
      "activations/layer2_attention_weight_min": -29.60950469970703,
      "activations/layer3_attention_weight_max": 84.23149108886719,
      "activations/layer3_attention_weight_min": -84.51466369628906,
      "activations/layer4_attention_weight_max": 130.35171508789062,
      "activations/layer4_attention_weight_min": -129.07398986816406,
      "activations/layer5_attention_weight_max": 101.72341918945312,
      "activations/layer5_attention_weight_min": -94.65167236328125,
      "activations/layer6_attention_weight_max": 57.95118713378906,
      "activations/layer6_attention_weight_min": -56.639122009277344,
      "activations/layer7_attention_weight_max": 80.15914916992188,
      "activations/layer7_attention_weight_min": -74.5517349243164,
      "activations/layer8_attention_weight_max": 42.155006408691406,
      "activations/layer8_attention_weight_min": -39.67100524902344,
      "activations/layer9_attention_weight_max": 48.39316940307617,
      "activations/layer9_attention_weight_min": -35.380741119384766,
      "epoch": 3.43,
      "learning_rate": 0.00012915303030303028,
      "loss": 2.9018,
      "step": 59100
    },
    {
      "activations/layer0_attention_weight_max": 14.061151504516602,
      "activations/layer0_attention_weight_min": -12.537504196166992,
      "activations/layer10_attention_weight_max": 44.517269134521484,
      "activations/layer10_attention_weight_min": -31.500333786010742,
      "activations/layer11_attention_weight_max": 36.25990676879883,
      "activations/layer11_attention_weight_min": -29.679096221923828,
      "activations/layer12_attention_weight_max": 29.697057723999023,
      "activations/layer12_attention_weight_min": -25.639850616455078,
      "activations/layer13_attention_weight_max": 51.03158187866211,
      "activations/layer13_attention_weight_min": -35.9148063659668,
      "activations/layer14_attention_weight_max": 78.25990295410156,
      "activations/layer14_attention_weight_min": -51.90570068359375,
      "activations/layer15_attention_weight_max": 45.946346282958984,
      "activations/layer15_attention_weight_min": -30.631969451904297,
      "activations/layer16_attention_weight_max": 44.61471176147461,
      "activations/layer16_attention_weight_min": -29.756460189819336,
      "activations/layer17_attention_weight_max": 46.36947250366211,
      "activations/layer17_attention_weight_min": -26.622413635253906,
      "activations/layer18_attention_weight_max": 32.456451416015625,
      "activations/layer18_attention_weight_min": -19.266498565673828,
      "activations/layer19_attention_weight_max": 51.40871810913086,
      "activations/layer19_attention_weight_min": -29.51763916015625,
      "activations/layer1_attention_weight_max": 17.982906341552734,
      "activations/layer1_attention_weight_min": -16.7291202545166,
      "activations/layer20_attention_weight_max": 39.641868591308594,
      "activations/layer20_attention_weight_min": -23.04363250732422,
      "activations/layer21_attention_weight_max": 40.544620513916016,
      "activations/layer21_attention_weight_min": -20.828079223632812,
      "activations/layer22_attention_weight_max": 54.55144119262695,
      "activations/layer22_attention_weight_min": -27.10158348083496,
      "activations/layer23_attention_weight_max": 31.5478515625,
      "activations/layer23_attention_weight_min": -18.275983810424805,
      "activations/layer2_attention_weight_max": 30.68932342529297,
      "activations/layer2_attention_weight_min": -30.485435485839844,
      "activations/layer3_attention_weight_max": 86.88156127929688,
      "activations/layer3_attention_weight_min": -87.31194305419922,
      "activations/layer4_attention_weight_max": 133.49949645996094,
      "activations/layer4_attention_weight_min": -132.94223022460938,
      "activations/layer5_attention_weight_max": 97.74556732177734,
      "activations/layer5_attention_weight_min": -94.67359924316406,
      "activations/layer6_attention_weight_max": 59.62255859375,
      "activations/layer6_attention_weight_min": -57.171104431152344,
      "activations/layer7_attention_weight_max": 75.84201049804688,
      "activations/layer7_attention_weight_min": -78.45073699951172,
      "activations/layer8_attention_weight_max": 41.07445526123047,
      "activations/layer8_attention_weight_min": -39.130714416503906,
      "activations/layer9_attention_weight_max": 43.40452194213867,
      "activations/layer9_attention_weight_min": -35.95991134643555,
      "epoch": 3.44,
      "learning_rate": 0.0001291340909090909,
      "loss": 2.9003,
      "step": 59150
    },
    {
      "activations/layer0_attention_weight_max": 16.077983856201172,
      "activations/layer0_attention_weight_min": -12.36600399017334,
      "activations/layer10_attention_weight_max": 37.43290328979492,
      "activations/layer10_attention_weight_min": -31.20433807373047,
      "activations/layer11_attention_weight_max": 32.77585983276367,
      "activations/layer11_attention_weight_min": -28.86194610595703,
      "activations/layer12_attention_weight_max": 29.13292121887207,
      "activations/layer12_attention_weight_min": -28.041847229003906,
      "activations/layer13_attention_weight_max": 44.461524963378906,
      "activations/layer13_attention_weight_min": -35.381595611572266,
      "activations/layer14_attention_weight_max": 62.01024627685547,
      "activations/layer14_attention_weight_min": -48.837162017822266,
      "activations/layer15_attention_weight_max": 44.32023239135742,
      "activations/layer15_attention_weight_min": -33.50979232788086,
      "activations/layer16_attention_weight_max": 41.536033630371094,
      "activations/layer16_attention_weight_min": -30.67387580871582,
      "activations/layer17_attention_weight_max": 43.848724365234375,
      "activations/layer17_attention_weight_min": -26.89033317565918,
      "activations/layer18_attention_weight_max": 32.3254280090332,
      "activations/layer18_attention_weight_min": -21.160526275634766,
      "activations/layer19_attention_weight_max": 42.60955810546875,
      "activations/layer19_attention_weight_min": -30.66792869567871,
      "activations/layer1_attention_weight_max": 16.330982208251953,
      "activations/layer1_attention_weight_min": -13.626782417297363,
      "activations/layer20_attention_weight_max": 38.678062438964844,
      "activations/layer20_attention_weight_min": -27.911651611328125,
      "activations/layer21_attention_weight_max": 32.625465393066406,
      "activations/layer21_attention_weight_min": -21.877107620239258,
      "activations/layer22_attention_weight_max": 57.959999084472656,
      "activations/layer22_attention_weight_min": -31.118053436279297,
      "activations/layer23_attention_weight_max": 30.692411422729492,
      "activations/layer23_attention_weight_min": -21.04486846923828,
      "activations/layer2_attention_weight_max": 31.789871215820312,
      "activations/layer2_attention_weight_min": -31.851781845092773,
      "activations/layer3_attention_weight_max": 81.0605239868164,
      "activations/layer3_attention_weight_min": -86.1270523071289,
      "activations/layer4_attention_weight_max": 129.81480407714844,
      "activations/layer4_attention_weight_min": -123.8763198852539,
      "activations/layer5_attention_weight_max": 94.47442626953125,
      "activations/layer5_attention_weight_min": -90.35623168945312,
      "activations/layer6_attention_weight_max": 55.66464614868164,
      "activations/layer6_attention_weight_min": -54.69948196411133,
      "activations/layer7_attention_weight_max": 73.75308990478516,
      "activations/layer7_attention_weight_min": -72.79043579101562,
      "activations/layer8_attention_weight_max": 36.703365325927734,
      "activations/layer8_attention_weight_min": -36.537601470947266,
      "activations/layer9_attention_weight_max": 44.05765914916992,
      "activations/layer9_attention_weight_min": -33.54014587402344,
      "epoch": 3.44,
      "learning_rate": 0.0001291151515151515,
      "loss": 2.8877,
      "step": 59200
    },
    {
      "activations/layer0_attention_weight_max": 16.053829193115234,
      "activations/layer0_attention_weight_min": -12.08974552154541,
      "activations/layer10_attention_weight_max": 38.756683349609375,
      "activations/layer10_attention_weight_min": -30.64784812927246,
      "activations/layer11_attention_weight_max": 42.083290100097656,
      "activations/layer11_attention_weight_min": -29.792430877685547,
      "activations/layer12_attention_weight_max": 47.20500946044922,
      "activations/layer12_attention_weight_min": -28.76003646850586,
      "activations/layer13_attention_weight_max": 56.047691345214844,
      "activations/layer13_attention_weight_min": -38.5078239440918,
      "activations/layer14_attention_weight_max": 86.64312744140625,
      "activations/layer14_attention_weight_min": -59.20718002319336,
      "activations/layer15_attention_weight_max": 44.92189407348633,
      "activations/layer15_attention_weight_min": -29.623022079467773,
      "activations/layer16_attention_weight_max": 42.81747055053711,
      "activations/layer16_attention_weight_min": -28.7260684967041,
      "activations/layer17_attention_weight_max": 42.15989303588867,
      "activations/layer17_attention_weight_min": -25.268896102905273,
      "activations/layer18_attention_weight_max": 32.60712432861328,
      "activations/layer18_attention_weight_min": -19.017715454101562,
      "activations/layer19_attention_weight_max": 44.221675872802734,
      "activations/layer19_attention_weight_min": -31.28909683227539,
      "activations/layer1_attention_weight_max": 18.42331314086914,
      "activations/layer1_attention_weight_min": -15.44698429107666,
      "activations/layer20_attention_weight_max": 36.17966842651367,
      "activations/layer20_attention_weight_min": -22.67277717590332,
      "activations/layer21_attention_weight_max": 36.227882385253906,
      "activations/layer21_attention_weight_min": -20.003931045532227,
      "activations/layer22_attention_weight_max": 53.0799560546875,
      "activations/layer22_attention_weight_min": -27.618074417114258,
      "activations/layer23_attention_weight_max": 31.912418365478516,
      "activations/layer23_attention_weight_min": -16.424999237060547,
      "activations/layer2_attention_weight_max": 32.5672607421875,
      "activations/layer2_attention_weight_min": -33.280582427978516,
      "activations/layer3_attention_weight_max": 87.06735229492188,
      "activations/layer3_attention_weight_min": -87.74768829345703,
      "activations/layer4_attention_weight_max": 137.44863891601562,
      "activations/layer4_attention_weight_min": -128.6227264404297,
      "activations/layer5_attention_weight_max": 99.15078735351562,
      "activations/layer5_attention_weight_min": -88.5468978881836,
      "activations/layer6_attention_weight_max": 59.181846618652344,
      "activations/layer6_attention_weight_min": -55.66737747192383,
      "activations/layer7_attention_weight_max": 74.55706024169922,
      "activations/layer7_attention_weight_min": -73.57769775390625,
      "activations/layer8_attention_weight_max": 41.19729995727539,
      "activations/layer8_attention_weight_min": -37.02542495727539,
      "activations/layer9_attention_weight_max": 38.55946350097656,
      "activations/layer9_attention_weight_min": -34.607704162597656,
      "epoch": 3.44,
      "learning_rate": 0.0001290962121212121,
      "loss": 2.8896,
      "step": 59250
    },
    {
      "activations/layer0_attention_weight_max": 15.532424926757812,
      "activations/layer0_attention_weight_min": -12.700957298278809,
      "activations/layer10_attention_weight_max": 49.04951095581055,
      "activations/layer10_attention_weight_min": -36.21848678588867,
      "activations/layer11_attention_weight_max": 54.66498947143555,
      "activations/layer11_attention_weight_min": -34.03445053100586,
      "activations/layer12_attention_weight_max": 39.70854568481445,
      "activations/layer12_attention_weight_min": -27.078269958496094,
      "activations/layer13_attention_weight_max": 69.06566619873047,
      "activations/layer13_attention_weight_min": -40.390647888183594,
      "activations/layer14_attention_weight_max": 79.35582733154297,
      "activations/layer14_attention_weight_min": -51.76789093017578,
      "activations/layer15_attention_weight_max": 53.580108642578125,
      "activations/layer15_attention_weight_min": -31.793039321899414,
      "activations/layer16_attention_weight_max": 43.51713943481445,
      "activations/layer16_attention_weight_min": -30.387527465820312,
      "activations/layer17_attention_weight_max": 37.848533630371094,
      "activations/layer17_attention_weight_min": -25.51945686340332,
      "activations/layer18_attention_weight_max": 30.882694244384766,
      "activations/layer18_attention_weight_min": -23.303363800048828,
      "activations/layer19_attention_weight_max": 41.477352142333984,
      "activations/layer19_attention_weight_min": -31.71376609802246,
      "activations/layer1_attention_weight_max": 17.85190773010254,
      "activations/layer1_attention_weight_min": -14.077571868896484,
      "activations/layer20_attention_weight_max": 29.076433181762695,
      "activations/layer20_attention_weight_min": -22.44070053100586,
      "activations/layer21_attention_weight_max": 28.303768157958984,
      "activations/layer21_attention_weight_min": -24.282760620117188,
      "activations/layer22_attention_weight_max": 49.59012985229492,
      "activations/layer22_attention_weight_min": -29.882478713989258,
      "activations/layer23_attention_weight_max": 28.788818359375,
      "activations/layer23_attention_weight_min": -19.710342407226562,
      "activations/layer2_attention_weight_max": 31.252174377441406,
      "activations/layer2_attention_weight_min": -32.82999801635742,
      "activations/layer3_attention_weight_max": 87.3458251953125,
      "activations/layer3_attention_weight_min": -92.99378204345703,
      "activations/layer4_attention_weight_max": 141.16831970214844,
      "activations/layer4_attention_weight_min": -133.78524780273438,
      "activations/layer5_attention_weight_max": 106.4193344116211,
      "activations/layer5_attention_weight_min": -99.29582977294922,
      "activations/layer6_attention_weight_max": 61.66847610473633,
      "activations/layer6_attention_weight_min": -56.98516845703125,
      "activations/layer7_attention_weight_max": 88.2405776977539,
      "activations/layer7_attention_weight_min": -85.50480651855469,
      "activations/layer8_attention_weight_max": 46.448158264160156,
      "activations/layer8_attention_weight_min": -41.994022369384766,
      "activations/layer9_attention_weight_max": 54.110633850097656,
      "activations/layer9_attention_weight_min": -49.102298736572266,
      "epoch": 3.45,
      "learning_rate": 0.0001290776515151515,
      "loss": 2.8965,
      "step": 59300
    },
    {
      "activations/layer0_attention_weight_max": 16.721464157104492,
      "activations/layer0_attention_weight_min": -11.795354843139648,
      "activations/layer10_attention_weight_max": 37.86305618286133,
      "activations/layer10_attention_weight_min": -31.958959579467773,
      "activations/layer11_attention_weight_max": 39.50316619873047,
      "activations/layer11_attention_weight_min": -31.727367401123047,
      "activations/layer12_attention_weight_max": 28.454856872558594,
      "activations/layer12_attention_weight_min": -30.352148056030273,
      "activations/layer13_attention_weight_max": 46.916847229003906,
      "activations/layer13_attention_weight_min": -33.58824920654297,
      "activations/layer14_attention_weight_max": 73.77862548828125,
      "activations/layer14_attention_weight_min": -49.87623596191406,
      "activations/layer15_attention_weight_max": 43.59872055053711,
      "activations/layer15_attention_weight_min": -29.4958438873291,
      "activations/layer16_attention_weight_max": 45.31832504272461,
      "activations/layer16_attention_weight_min": -30.545604705810547,
      "activations/layer17_attention_weight_max": 47.909820556640625,
      "activations/layer17_attention_weight_min": -24.74290657043457,
      "activations/layer18_attention_weight_max": 30.733903884887695,
      "activations/layer18_attention_weight_min": -18.161649703979492,
      "activations/layer19_attention_weight_max": 48.531009674072266,
      "activations/layer19_attention_weight_min": -29.1409854888916,
      "activations/layer1_attention_weight_max": 17.031064987182617,
      "activations/layer1_attention_weight_min": -14.947647094726562,
      "activations/layer20_attention_weight_max": 36.16611099243164,
      "activations/layer20_attention_weight_min": -21.705379486083984,
      "activations/layer21_attention_weight_max": 40.92067337036133,
      "activations/layer21_attention_weight_min": -19.985565185546875,
      "activations/layer22_attention_weight_max": 48.938663482666016,
      "activations/layer22_attention_weight_min": -25.62139892578125,
      "activations/layer23_attention_weight_max": 31.355712890625,
      "activations/layer23_attention_weight_min": -18.55287742614746,
      "activations/layer2_attention_weight_max": 32.395416259765625,
      "activations/layer2_attention_weight_min": -30.370868682861328,
      "activations/layer3_attention_weight_max": 82.81001281738281,
      "activations/layer3_attention_weight_min": -86.83351135253906,
      "activations/layer4_attention_weight_max": 128.12899780273438,
      "activations/layer4_attention_weight_min": -124.3261947631836,
      "activations/layer5_attention_weight_max": 95.36132049560547,
      "activations/layer5_attention_weight_min": -86.65719604492188,
      "activations/layer6_attention_weight_max": 57.314456939697266,
      "activations/layer6_attention_weight_min": -55.04949188232422,
      "activations/layer7_attention_weight_max": 76.1986312866211,
      "activations/layer7_attention_weight_min": -74.32793426513672,
      "activations/layer8_attention_weight_max": 41.123050689697266,
      "activations/layer8_attention_weight_min": -36.9507942199707,
      "activations/layer9_attention_weight_max": 43.6214714050293,
      "activations/layer9_attention_weight_min": -39.32140350341797,
      "epoch": 3.45,
      "learning_rate": 0.0001290587121212121,
      "loss": 2.8781,
      "step": 59350
    },
    {
      "activations/layer0_attention_weight_max": 16.814044952392578,
      "activations/layer0_attention_weight_min": -12.89834976196289,
      "activations/layer10_attention_weight_max": 38.46778106689453,
      "activations/layer10_attention_weight_min": -31.83926010131836,
      "activations/layer11_attention_weight_max": 33.362300872802734,
      "activations/layer11_attention_weight_min": -28.229270935058594,
      "activations/layer12_attention_weight_max": 43.44245147705078,
      "activations/layer12_attention_weight_min": -30.489089965820312,
      "activations/layer13_attention_weight_max": 51.162696838378906,
      "activations/layer13_attention_weight_min": -36.623653411865234,
      "activations/layer14_attention_weight_max": 87.48140716552734,
      "activations/layer14_attention_weight_min": -61.57651138305664,
      "activations/layer15_attention_weight_max": 46.445899963378906,
      "activations/layer15_attention_weight_min": -32.65468978881836,
      "activations/layer16_attention_weight_max": 46.54082107543945,
      "activations/layer16_attention_weight_min": -31.019412994384766,
      "activations/layer17_attention_weight_max": 45.10108947753906,
      "activations/layer17_attention_weight_min": -26.98394012451172,
      "activations/layer18_attention_weight_max": 31.014482498168945,
      "activations/layer18_attention_weight_min": -19.952367782592773,
      "activations/layer19_attention_weight_max": 52.24528884887695,
      "activations/layer19_attention_weight_min": -28.55478858947754,
      "activations/layer1_attention_weight_max": 17.19632911682129,
      "activations/layer1_attention_weight_min": -13.295027732849121,
      "activations/layer20_attention_weight_max": 35.22380828857422,
      "activations/layer20_attention_weight_min": -24.546287536621094,
      "activations/layer21_attention_weight_max": 38.30506896972656,
      "activations/layer21_attention_weight_min": -23.40214729309082,
      "activations/layer22_attention_weight_max": 55.11949157714844,
      "activations/layer22_attention_weight_min": -31.326908111572266,
      "activations/layer23_attention_weight_max": 31.09993553161621,
      "activations/layer23_attention_weight_min": -18.959976196289062,
      "activations/layer2_attention_weight_max": 29.199392318725586,
      "activations/layer2_attention_weight_min": -28.924747467041016,
      "activations/layer3_attention_weight_max": 81.61156463623047,
      "activations/layer3_attention_weight_min": -83.056640625,
      "activations/layer4_attention_weight_max": 130.34742736816406,
      "activations/layer4_attention_weight_min": -126.85945892333984,
      "activations/layer5_attention_weight_max": 94.37349700927734,
      "activations/layer5_attention_weight_min": -88.53347778320312,
      "activations/layer6_attention_weight_max": 58.76247024536133,
      "activations/layer6_attention_weight_min": -56.86860275268555,
      "activations/layer7_attention_weight_max": 74.36647033691406,
      "activations/layer7_attention_weight_min": -77.69927215576172,
      "activations/layer8_attention_weight_max": 38.82484436035156,
      "activations/layer8_attention_weight_min": -39.924076080322266,
      "activations/layer9_attention_weight_max": 45.171852111816406,
      "activations/layer9_attention_weight_min": -36.656776428222656,
      "epoch": 3.45,
      "learning_rate": 0.00012903977272727272,
      "loss": 2.9043,
      "step": 59400
    },
    {
      "activations/layer0_attention_weight_max": 16.01624298095703,
      "activations/layer0_attention_weight_min": -10.90395736694336,
      "activations/layer10_attention_weight_max": 36.24988555908203,
      "activations/layer10_attention_weight_min": -32.96521759033203,
      "activations/layer11_attention_weight_max": 36.100650787353516,
      "activations/layer11_attention_weight_min": -28.17300796508789,
      "activations/layer12_attention_weight_max": 28.83989715576172,
      "activations/layer12_attention_weight_min": -29.076465606689453,
      "activations/layer13_attention_weight_max": 50.05046844482422,
      "activations/layer13_attention_weight_min": -36.267189025878906,
      "activations/layer14_attention_weight_max": 73.84654235839844,
      "activations/layer14_attention_weight_min": -48.16304397583008,
      "activations/layer15_attention_weight_max": 48.31043243408203,
      "activations/layer15_attention_weight_min": -29.61519432067871,
      "activations/layer16_attention_weight_max": 44.08602523803711,
      "activations/layer16_attention_weight_min": -28.102285385131836,
      "activations/layer17_attention_weight_max": 48.41920852661133,
      "activations/layer17_attention_weight_min": -24.655725479125977,
      "activations/layer18_attention_weight_max": 30.321866989135742,
      "activations/layer18_attention_weight_min": -21.341426849365234,
      "activations/layer19_attention_weight_max": 51.761409759521484,
      "activations/layer19_attention_weight_min": -28.66873550415039,
      "activations/layer1_attention_weight_max": 17.835254669189453,
      "activations/layer1_attention_weight_min": -14.357969284057617,
      "activations/layer20_attention_weight_max": 37.478511810302734,
      "activations/layer20_attention_weight_min": -22.6980037689209,
      "activations/layer21_attention_weight_max": 33.032161712646484,
      "activations/layer21_attention_weight_min": -20.558673858642578,
      "activations/layer22_attention_weight_max": 57.14450454711914,
      "activations/layer22_attention_weight_min": -27.99306297302246,
      "activations/layer23_attention_weight_max": 30.45736312866211,
      "activations/layer23_attention_weight_min": -18.705677032470703,
      "activations/layer2_attention_weight_max": 29.821372985839844,
      "activations/layer2_attention_weight_min": -29.545818328857422,
      "activations/layer3_attention_weight_max": 81.5908432006836,
      "activations/layer3_attention_weight_min": -84.94682312011719,
      "activations/layer4_attention_weight_max": 131.21588134765625,
      "activations/layer4_attention_weight_min": -123.23172760009766,
      "activations/layer5_attention_weight_max": 99.65159606933594,
      "activations/layer5_attention_weight_min": -88.0564193725586,
      "activations/layer6_attention_weight_max": 56.49928283691406,
      "activations/layer6_attention_weight_min": -53.72404479980469,
      "activations/layer7_attention_weight_max": 73.36604309082031,
      "activations/layer7_attention_weight_min": -79.84613800048828,
      "activations/layer8_attention_weight_max": 38.30391311645508,
      "activations/layer8_attention_weight_min": -36.44146728515625,
      "activations/layer9_attention_weight_max": 44.59131622314453,
      "activations/layer9_attention_weight_min": -33.3680305480957,
      "epoch": 3.45,
      "learning_rate": 0.00012902083333333334,
      "loss": 2.8772,
      "step": 59450
    },
    {
      "activations/layer0_attention_weight_max": 15.276098251342773,
      "activations/layer0_attention_weight_min": -10.828981399536133,
      "activations/layer10_attention_weight_max": 50.877159118652344,
      "activations/layer10_attention_weight_min": -36.86381912231445,
      "activations/layer11_attention_weight_max": 47.00331115722656,
      "activations/layer11_attention_weight_min": -31.222929000854492,
      "activations/layer12_attention_weight_max": 70.13428497314453,
      "activations/layer12_attention_weight_min": -33.53826141357422,
      "activations/layer13_attention_weight_max": 103.3065414428711,
      "activations/layer13_attention_weight_min": -53.083003997802734,
      "activations/layer14_attention_weight_max": 107.6961669921875,
      "activations/layer14_attention_weight_min": -64.38818359375,
      "activations/layer15_attention_weight_max": 77.70645904541016,
      "activations/layer15_attention_weight_min": -43.72907257080078,
      "activations/layer16_attention_weight_max": 46.10268783569336,
      "activations/layer16_attention_weight_min": -28.65922737121582,
      "activations/layer17_attention_weight_max": 45.5960578918457,
      "activations/layer17_attention_weight_min": -23.81654930114746,
      "activations/layer18_attention_weight_max": 32.7989501953125,
      "activations/layer18_attention_weight_min": -20.49867820739746,
      "activations/layer19_attention_weight_max": 49.98474884033203,
      "activations/layer19_attention_weight_min": -29.9736328125,
      "activations/layer1_attention_weight_max": 18.67387580871582,
      "activations/layer1_attention_weight_min": -16.223730087280273,
      "activations/layer20_attention_weight_max": 38.872459411621094,
      "activations/layer20_attention_weight_min": -21.694358825683594,
      "activations/layer21_attention_weight_max": 33.2462158203125,
      "activations/layer21_attention_weight_min": -20.269824981689453,
      "activations/layer22_attention_weight_max": 52.38455581665039,
      "activations/layer22_attention_weight_min": -29.489253997802734,
      "activations/layer23_attention_weight_max": 31.29287338256836,
      "activations/layer23_attention_weight_min": -17.42876434326172,
      "activations/layer2_attention_weight_max": 31.514328002929688,
      "activations/layer2_attention_weight_min": -30.95206069946289,
      "activations/layer3_attention_weight_max": 84.06175231933594,
      "activations/layer3_attention_weight_min": -88.25150299072266,
      "activations/layer4_attention_weight_max": 134.4930877685547,
      "activations/layer4_attention_weight_min": -132.38375854492188,
      "activations/layer5_attention_weight_max": 101.3280258178711,
      "activations/layer5_attention_weight_min": -93.72712707519531,
      "activations/layer6_attention_weight_max": 62.01022720336914,
      "activations/layer6_attention_weight_min": -56.414344787597656,
      "activations/layer7_attention_weight_max": 91.20464324951172,
      "activations/layer7_attention_weight_min": -82.6624984741211,
      "activations/layer8_attention_weight_max": 46.189964294433594,
      "activations/layer8_attention_weight_min": -41.8292350769043,
      "activations/layer9_attention_weight_max": 71.89636993408203,
      "activations/layer9_attention_weight_min": -47.989952087402344,
      "epoch": 3.46,
      "learning_rate": 0.00012900189393939392,
      "loss": 2.8962,
      "step": 59500
    },
    {
      "activations/layer0_attention_weight_max": 16.493215560913086,
      "activations/layer0_attention_weight_min": -12.051031112670898,
      "activations/layer10_attention_weight_max": 42.39452362060547,
      "activations/layer10_attention_weight_min": -33.92562484741211,
      "activations/layer11_attention_weight_max": 38.70286560058594,
      "activations/layer11_attention_weight_min": -28.867055892944336,
      "activations/layer12_attention_weight_max": 51.18620681762695,
      "activations/layer12_attention_weight_min": -29.36577796936035,
      "activations/layer13_attention_weight_max": 62.61195373535156,
      "activations/layer13_attention_weight_min": -43.766536712646484,
      "activations/layer14_attention_weight_max": 75.87968444824219,
      "activations/layer14_attention_weight_min": -48.29636001586914,
      "activations/layer15_attention_weight_max": 44.5627555847168,
      "activations/layer15_attention_weight_min": -31.85801124572754,
      "activations/layer16_attention_weight_max": 44.58258056640625,
      "activations/layer16_attention_weight_min": -32.035423278808594,
      "activations/layer17_attention_weight_max": 45.02610397338867,
      "activations/layer17_attention_weight_min": -25.73661994934082,
      "activations/layer18_attention_weight_max": 37.74725341796875,
      "activations/layer18_attention_weight_min": -21.129732131958008,
      "activations/layer19_attention_weight_max": 49.89246368408203,
      "activations/layer19_attention_weight_min": -31.442941665649414,
      "activations/layer1_attention_weight_max": 18.517702102661133,
      "activations/layer1_attention_weight_min": -13.573942184448242,
      "activations/layer20_attention_weight_max": 36.39054489135742,
      "activations/layer20_attention_weight_min": -24.927783966064453,
      "activations/layer21_attention_weight_max": 41.18707275390625,
      "activations/layer21_attention_weight_min": -21.005216598510742,
      "activations/layer22_attention_weight_max": 50.12568664550781,
      "activations/layer22_attention_weight_min": -24.613176345825195,
      "activations/layer23_attention_weight_max": 32.80557632446289,
      "activations/layer23_attention_weight_min": -18.116695404052734,
      "activations/layer2_attention_weight_max": 33.1964225769043,
      "activations/layer2_attention_weight_min": -31.366863250732422,
      "activations/layer3_attention_weight_max": 86.1069107055664,
      "activations/layer3_attention_weight_min": -89.9664306640625,
      "activations/layer4_attention_weight_max": 139.16319274902344,
      "activations/layer4_attention_weight_min": -134.48855590820312,
      "activations/layer5_attention_weight_max": 100.78746795654297,
      "activations/layer5_attention_weight_min": -97.24349212646484,
      "activations/layer6_attention_weight_max": 61.19218063354492,
      "activations/layer6_attention_weight_min": -60.837005615234375,
      "activations/layer7_attention_weight_max": 76.9161148071289,
      "activations/layer7_attention_weight_min": -79.09967041015625,
      "activations/layer8_attention_weight_max": 42.566226959228516,
      "activations/layer8_attention_weight_min": -40.065364837646484,
      "activations/layer9_attention_weight_max": 49.42404556274414,
      "activations/layer9_attention_weight_min": -42.2000732421875,
      "epoch": 3.46,
      "learning_rate": 0.00012898295454545454,
      "loss": 2.8817,
      "step": 59550
    },
    {
      "activations/layer0_attention_weight_max": 16.489608764648438,
      "activations/layer0_attention_weight_min": -10.93938159942627,
      "activations/layer10_attention_weight_max": 39.902008056640625,
      "activations/layer10_attention_weight_min": -31.758243560791016,
      "activations/layer11_attention_weight_max": 34.84282302856445,
      "activations/layer11_attention_weight_min": -29.6683406829834,
      "activations/layer12_attention_weight_max": 29.816591262817383,
      "activations/layer12_attention_weight_min": -25.374618530273438,
      "activations/layer13_attention_weight_max": 49.30411911010742,
      "activations/layer13_attention_weight_min": -37.42388916015625,
      "activations/layer14_attention_weight_max": 78.9691390991211,
      "activations/layer14_attention_weight_min": -60.293190002441406,
      "activations/layer15_attention_weight_max": 45.38869857788086,
      "activations/layer15_attention_weight_min": -31.894262313842773,
      "activations/layer16_attention_weight_max": 43.04328155517578,
      "activations/layer16_attention_weight_min": -31.715959548950195,
      "activations/layer17_attention_weight_max": 43.152706146240234,
      "activations/layer17_attention_weight_min": -26.389925003051758,
      "activations/layer18_attention_weight_max": 32.36428451538086,
      "activations/layer18_attention_weight_min": -20.701366424560547,
      "activations/layer19_attention_weight_max": 49.655662536621094,
      "activations/layer19_attention_weight_min": -29.026880264282227,
      "activations/layer1_attention_weight_max": 17.311796188354492,
      "activations/layer1_attention_weight_min": -13.73945140838623,
      "activations/layer20_attention_weight_max": 36.64910125732422,
      "activations/layer20_attention_weight_min": -22.9575252532959,
      "activations/layer21_attention_weight_max": 33.603363037109375,
      "activations/layer21_attention_weight_min": -20.48567008972168,
      "activations/layer22_attention_weight_max": 55.70255661010742,
      "activations/layer22_attention_weight_min": -30.611665725708008,
      "activations/layer23_attention_weight_max": 30.770627975463867,
      "activations/layer23_attention_weight_min": -17.764114379882812,
      "activations/layer2_attention_weight_max": 33.19502639770508,
      "activations/layer2_attention_weight_min": -33.52830505371094,
      "activations/layer3_attention_weight_max": 88.8730239868164,
      "activations/layer3_attention_weight_min": -98.13302612304688,
      "activations/layer4_attention_weight_max": 138.0727996826172,
      "activations/layer4_attention_weight_min": -149.67568969726562,
      "activations/layer5_attention_weight_max": 103.4173812866211,
      "activations/layer5_attention_weight_min": -102.26351165771484,
      "activations/layer6_attention_weight_max": 61.732242584228516,
      "activations/layer6_attention_weight_min": -60.26811981201172,
      "activations/layer7_attention_weight_max": 81.27632141113281,
      "activations/layer7_attention_weight_min": -77.91545104980469,
      "activations/layer8_attention_weight_max": 39.21983337402344,
      "activations/layer8_attention_weight_min": -38.891357421875,
      "activations/layer9_attention_weight_max": 40.36509323120117,
      "activations/layer9_attention_weight_min": -36.21279525756836,
      "epoch": 3.46,
      "learning_rate": 0.00012896401515151515,
      "loss": 2.8842,
      "step": 59600
    },
    {
      "activations/layer0_attention_weight_max": 16.119991302490234,
      "activations/layer0_attention_weight_min": -11.055609703063965,
      "activations/layer10_attention_weight_max": 40.27006912231445,
      "activations/layer10_attention_weight_min": -31.95180320739746,
      "activations/layer11_attention_weight_max": 35.42230224609375,
      "activations/layer11_attention_weight_min": -26.764129638671875,
      "activations/layer12_attention_weight_max": 35.664344787597656,
      "activations/layer12_attention_weight_min": -26.160022735595703,
      "activations/layer13_attention_weight_max": 49.295772552490234,
      "activations/layer13_attention_weight_min": -36.968746185302734,
      "activations/layer14_attention_weight_max": 67.76925659179688,
      "activations/layer14_attention_weight_min": -49.0758056640625,
      "activations/layer15_attention_weight_max": 49.34796905517578,
      "activations/layer15_attention_weight_min": -34.84691619873047,
      "activations/layer16_attention_weight_max": 41.64665985107422,
      "activations/layer16_attention_weight_min": -30.983646392822266,
      "activations/layer17_attention_weight_max": 40.64606857299805,
      "activations/layer17_attention_weight_min": -26.555871963500977,
      "activations/layer18_attention_weight_max": 31.01432991027832,
      "activations/layer18_attention_weight_min": -20.92790985107422,
      "activations/layer19_attention_weight_max": 42.27375411987305,
      "activations/layer19_attention_weight_min": -30.513687133789062,
      "activations/layer1_attention_weight_max": 18.19547462463379,
      "activations/layer1_attention_weight_min": -14.89123249053955,
      "activations/layer20_attention_weight_max": 36.95652770996094,
      "activations/layer20_attention_weight_min": -24.753917694091797,
      "activations/layer21_attention_weight_max": 34.25896072387695,
      "activations/layer21_attention_weight_min": -25.100284576416016,
      "activations/layer22_attention_weight_max": 50.26943588256836,
      "activations/layer22_attention_weight_min": -31.988258361816406,
      "activations/layer23_attention_weight_max": 30.988142013549805,
      "activations/layer23_attention_weight_min": -22.49068832397461,
      "activations/layer2_attention_weight_max": 31.547109603881836,
      "activations/layer2_attention_weight_min": -33.07379150390625,
      "activations/layer3_attention_weight_max": 81.11758422851562,
      "activations/layer3_attention_weight_min": -89.48724365234375,
      "activations/layer4_attention_weight_max": 128.38128662109375,
      "activations/layer4_attention_weight_min": -133.63491821289062,
      "activations/layer5_attention_weight_max": 93.72038269042969,
      "activations/layer5_attention_weight_min": -88.21914672851562,
      "activations/layer6_attention_weight_max": 55.57428741455078,
      "activations/layer6_attention_weight_min": -57.49699020385742,
      "activations/layer7_attention_weight_max": 74.80288696289062,
      "activations/layer7_attention_weight_min": -75.05559539794922,
      "activations/layer8_attention_weight_max": 38.52969741821289,
      "activations/layer8_attention_weight_min": -37.86918258666992,
      "activations/layer9_attention_weight_max": 51.679969787597656,
      "activations/layer9_attention_weight_min": -36.962608337402344,
      "epoch": 3.47,
      "learning_rate": 0.00012894507575757574,
      "loss": 2.9117,
      "step": 59650
    },
    {
      "activations/layer0_attention_weight_max": 16.88400650024414,
      "activations/layer0_attention_weight_min": -12.090435028076172,
      "activations/layer10_attention_weight_max": 40.6605224609375,
      "activations/layer10_attention_weight_min": -36.27019500732422,
      "activations/layer11_attention_weight_max": 43.27711486816406,
      "activations/layer11_attention_weight_min": -35.82380676269531,
      "activations/layer12_attention_weight_max": 41.74566650390625,
      "activations/layer12_attention_weight_min": -28.31534767150879,
      "activations/layer13_attention_weight_max": 53.05534744262695,
      "activations/layer13_attention_weight_min": -34.855369567871094,
      "activations/layer14_attention_weight_max": 77.62235260009766,
      "activations/layer14_attention_weight_min": -52.81600570678711,
      "activations/layer15_attention_weight_max": 45.894046783447266,
      "activations/layer15_attention_weight_min": -29.851394653320312,
      "activations/layer16_attention_weight_max": 49.27610778808594,
      "activations/layer16_attention_weight_min": -32.38617706298828,
      "activations/layer17_attention_weight_max": 42.9200439453125,
      "activations/layer17_attention_weight_min": -23.006776809692383,
      "activations/layer18_attention_weight_max": 33.62385177612305,
      "activations/layer18_attention_weight_min": -21.554262161254883,
      "activations/layer19_attention_weight_max": 51.00341033935547,
      "activations/layer19_attention_weight_min": -29.8814640045166,
      "activations/layer1_attention_weight_max": 17.9593505859375,
      "activations/layer1_attention_weight_min": -15.799230575561523,
      "activations/layer20_attention_weight_max": 36.96794509887695,
      "activations/layer20_attention_weight_min": -22.25543785095215,
      "activations/layer21_attention_weight_max": 35.35911560058594,
      "activations/layer21_attention_weight_min": -19.355310440063477,
      "activations/layer22_attention_weight_max": 50.139686584472656,
      "activations/layer22_attention_weight_min": -28.59752655029297,
      "activations/layer23_attention_weight_max": 29.35599136352539,
      "activations/layer23_attention_weight_min": -17.513248443603516,
      "activations/layer2_attention_weight_max": 31.830869674682617,
      "activations/layer2_attention_weight_min": -29.59381103515625,
      "activations/layer3_attention_weight_max": 87.08794403076172,
      "activations/layer3_attention_weight_min": -86.00804138183594,
      "activations/layer4_attention_weight_max": 136.18116760253906,
      "activations/layer4_attention_weight_min": -122.15911102294922,
      "activations/layer5_attention_weight_max": 99.94308471679688,
      "activations/layer5_attention_weight_min": -85.57077026367188,
      "activations/layer6_attention_weight_max": 60.65831756591797,
      "activations/layer6_attention_weight_min": -53.122371673583984,
      "activations/layer7_attention_weight_max": 87.12439727783203,
      "activations/layer7_attention_weight_min": -73.72516632080078,
      "activations/layer8_attention_weight_max": 42.26819610595703,
      "activations/layer8_attention_weight_min": -39.6021728515625,
      "activations/layer9_attention_weight_max": 63.806556701660156,
      "activations/layer9_attention_weight_min": -45.53607177734375,
      "epoch": 3.47,
      "learning_rate": 0.00012892613636363636,
      "loss": 2.8776,
      "step": 59700
    },
    {
      "activations/layer0_attention_weight_max": 16.797130584716797,
      "activations/layer0_attention_weight_min": -12.84990406036377,
      "activations/layer10_attention_weight_max": 38.118751525878906,
      "activations/layer10_attention_weight_min": -31.021970748901367,
      "activations/layer11_attention_weight_max": 35.94192123413086,
      "activations/layer11_attention_weight_min": -30.82502555847168,
      "activations/layer12_attention_weight_max": 38.244136810302734,
      "activations/layer12_attention_weight_min": -28.89201545715332,
      "activations/layer13_attention_weight_max": 54.37931823730469,
      "activations/layer13_attention_weight_min": -42.76798629760742,
      "activations/layer14_attention_weight_max": 71.3122787475586,
      "activations/layer14_attention_weight_min": -52.87704849243164,
      "activations/layer15_attention_weight_max": 48.201175689697266,
      "activations/layer15_attention_weight_min": -34.431339263916016,
      "activations/layer16_attention_weight_max": 41.3486328125,
      "activations/layer16_attention_weight_min": -30.684558868408203,
      "activations/layer17_attention_weight_max": 44.78947830200195,
      "activations/layer17_attention_weight_min": -26.2984561920166,
      "activations/layer18_attention_weight_max": 29.270540237426758,
      "activations/layer18_attention_weight_min": -19.290985107421875,
      "activations/layer19_attention_weight_max": 43.4501953125,
      "activations/layer19_attention_weight_min": -29.736459732055664,
      "activations/layer1_attention_weight_max": 17.93379783630371,
      "activations/layer1_attention_weight_min": -13.851868629455566,
      "activations/layer20_attention_weight_max": 35.73935317993164,
      "activations/layer20_attention_weight_min": -24.079004287719727,
      "activations/layer21_attention_weight_max": 31.495250701904297,
      "activations/layer21_attention_weight_min": -20.244773864746094,
      "activations/layer22_attention_weight_max": 51.322547912597656,
      "activations/layer22_attention_weight_min": -28.05847930908203,
      "activations/layer23_attention_weight_max": 28.12454605102539,
      "activations/layer23_attention_weight_min": -16.832782745361328,
      "activations/layer2_attention_weight_max": 30.822134017944336,
      "activations/layer2_attention_weight_min": -30.291515350341797,
      "activations/layer3_attention_weight_max": 81.52088165283203,
      "activations/layer3_attention_weight_min": -85.1373519897461,
      "activations/layer4_attention_weight_max": 132.14735412597656,
      "activations/layer4_attention_weight_min": -125.31427001953125,
      "activations/layer5_attention_weight_max": 98.82614135742188,
      "activations/layer5_attention_weight_min": -85.08897399902344,
      "activations/layer6_attention_weight_max": 57.49930953979492,
      "activations/layer6_attention_weight_min": -53.86408996582031,
      "activations/layer7_attention_weight_max": 78.57626342773438,
      "activations/layer7_attention_weight_min": -77.73768615722656,
      "activations/layer8_attention_weight_max": 39.49760437011719,
      "activations/layer8_attention_weight_min": -41.9166374206543,
      "activations/layer9_attention_weight_max": 49.799190521240234,
      "activations/layer9_attention_weight_min": -44.033294677734375,
      "epoch": 3.47,
      "learning_rate": 0.00012890719696969697,
      "loss": 2.8847,
      "step": 59750
    },
    {
      "activations/layer0_attention_weight_max": 15.000234603881836,
      "activations/layer0_attention_weight_min": -11.424428939819336,
      "activations/layer10_attention_weight_max": 36.6272087097168,
      "activations/layer10_attention_weight_min": -33.43581008911133,
      "activations/layer11_attention_weight_max": 35.97320556640625,
      "activations/layer11_attention_weight_min": -27.219999313354492,
      "activations/layer12_attention_weight_max": 28.572834014892578,
      "activations/layer12_attention_weight_min": -26.278364181518555,
      "activations/layer13_attention_weight_max": 45.96949005126953,
      "activations/layer13_attention_weight_min": -31.85561752319336,
      "activations/layer14_attention_weight_max": 71.16712951660156,
      "activations/layer14_attention_weight_min": -48.931053161621094,
      "activations/layer15_attention_weight_max": 46.958492279052734,
      "activations/layer15_attention_weight_min": -30.400672912597656,
      "activations/layer16_attention_weight_max": 44.85794448852539,
      "activations/layer16_attention_weight_min": -30.500036239624023,
      "activations/layer17_attention_weight_max": 43.8476448059082,
      "activations/layer17_attention_weight_min": -26.316511154174805,
      "activations/layer18_attention_weight_max": 36.75018310546875,
      "activations/layer18_attention_weight_min": -21.23516845703125,
      "activations/layer19_attention_weight_max": 47.61906051635742,
      "activations/layer19_attention_weight_min": -30.62308692932129,
      "activations/layer1_attention_weight_max": 17.376596450805664,
      "activations/layer1_attention_weight_min": -14.735013008117676,
      "activations/layer20_attention_weight_max": 35.285770416259766,
      "activations/layer20_attention_weight_min": -23.27378273010254,
      "activations/layer21_attention_weight_max": 33.920066833496094,
      "activations/layer21_attention_weight_min": -21.365966796875,
      "activations/layer22_attention_weight_max": 52.21211624145508,
      "activations/layer22_attention_weight_min": -27.413543701171875,
      "activations/layer23_attention_weight_max": 31.703445434570312,
      "activations/layer23_attention_weight_min": -18.977230072021484,
      "activations/layer2_attention_weight_max": 30.106603622436523,
      "activations/layer2_attention_weight_min": -30.054805755615234,
      "activations/layer3_attention_weight_max": 85.32469177246094,
      "activations/layer3_attention_weight_min": -87.99210357666016,
      "activations/layer4_attention_weight_max": 126.96832275390625,
      "activations/layer4_attention_weight_min": -130.5330047607422,
      "activations/layer5_attention_weight_max": 95.62541198730469,
      "activations/layer5_attention_weight_min": -92.94173431396484,
      "activations/layer6_attention_weight_max": 56.64414978027344,
      "activations/layer6_attention_weight_min": -55.13930130004883,
      "activations/layer7_attention_weight_max": 71.06629180908203,
      "activations/layer7_attention_weight_min": -74.07625579833984,
      "activations/layer8_attention_weight_max": 37.90385055541992,
      "activations/layer8_attention_weight_min": -38.00334167480469,
      "activations/layer9_attention_weight_max": 38.52986145019531,
      "activations/layer9_attention_weight_min": -33.67735290527344,
      "epoch": 3.47,
      "learning_rate": 0.00012888825757575756,
      "loss": 2.8949,
      "step": 59800
    },
    {
      "activations/layer0_attention_weight_max": 16.673566818237305,
      "activations/layer0_attention_weight_min": -12.610519409179688,
      "activations/layer10_attention_weight_max": 40.25493621826172,
      "activations/layer10_attention_weight_min": -31.165542602539062,
      "activations/layer11_attention_weight_max": 40.598026275634766,
      "activations/layer11_attention_weight_min": -29.205215454101562,
      "activations/layer12_attention_weight_max": 31.543672561645508,
      "activations/layer12_attention_weight_min": -25.879709243774414,
      "activations/layer13_attention_weight_max": 59.70689392089844,
      "activations/layer13_attention_weight_min": -38.45109939575195,
      "activations/layer14_attention_weight_max": 86.7593994140625,
      "activations/layer14_attention_weight_min": -58.340553283691406,
      "activations/layer15_attention_weight_max": 51.0225830078125,
      "activations/layer15_attention_weight_min": -31.21241569519043,
      "activations/layer16_attention_weight_max": 48.92927169799805,
      "activations/layer16_attention_weight_min": -29.09010887145996,
      "activations/layer17_attention_weight_max": 50.24937057495117,
      "activations/layer17_attention_weight_min": -25.84606170654297,
      "activations/layer18_attention_weight_max": 34.1674690246582,
      "activations/layer18_attention_weight_min": -20.005908966064453,
      "activations/layer19_attention_weight_max": 50.18844223022461,
      "activations/layer19_attention_weight_min": -30.061908721923828,
      "activations/layer1_attention_weight_max": 17.160812377929688,
      "activations/layer1_attention_weight_min": -14.725664138793945,
      "activations/layer20_attention_weight_max": 39.62508010864258,
      "activations/layer20_attention_weight_min": -23.477895736694336,
      "activations/layer21_attention_weight_max": 33.019630432128906,
      "activations/layer21_attention_weight_min": -22.016647338867188,
      "activations/layer22_attention_weight_max": 55.29680252075195,
      "activations/layer22_attention_weight_min": -27.837482452392578,
      "activations/layer23_attention_weight_max": 32.142372131347656,
      "activations/layer23_attention_weight_min": -17.197341918945312,
      "activations/layer2_attention_weight_max": 30.910198211669922,
      "activations/layer2_attention_weight_min": -31.840408325195312,
      "activations/layer3_attention_weight_max": 88.23019409179688,
      "activations/layer3_attention_weight_min": -91.4273681640625,
      "activations/layer4_attention_weight_max": 135.06700134277344,
      "activations/layer4_attention_weight_min": -129.1642608642578,
      "activations/layer5_attention_weight_max": 100.15950775146484,
      "activations/layer5_attention_weight_min": -94.32212829589844,
      "activations/layer6_attention_weight_max": 58.96890640258789,
      "activations/layer6_attention_weight_min": -57.94153594970703,
      "activations/layer7_attention_weight_max": 81.61215209960938,
      "activations/layer7_attention_weight_min": -83.13325500488281,
      "activations/layer8_attention_weight_max": 43.91804885864258,
      "activations/layer8_attention_weight_min": -40.8361701965332,
      "activations/layer9_attention_weight_max": 40.76473617553711,
      "activations/layer9_attention_weight_min": -33.681922912597656,
      "epoch": 3.48,
      "learning_rate": 0.00012886931818181818,
      "loss": 2.8753,
      "step": 59850
    },
    {
      "activations/layer0_attention_weight_max": 17.7452392578125,
      "activations/layer0_attention_weight_min": -12.216300964355469,
      "activations/layer10_attention_weight_max": 34.831539154052734,
      "activations/layer10_attention_weight_min": -31.048593521118164,
      "activations/layer11_attention_weight_max": 32.881500244140625,
      "activations/layer11_attention_weight_min": -29.12037467956543,
      "activations/layer12_attention_weight_max": 48.52077102661133,
      "activations/layer12_attention_weight_min": -32.0034294128418,
      "activations/layer13_attention_weight_max": 48.435977935791016,
      "activations/layer13_attention_weight_min": -35.35714340209961,
      "activations/layer14_attention_weight_max": 64.3124771118164,
      "activations/layer14_attention_weight_min": -47.80830001831055,
      "activations/layer15_attention_weight_max": 43.01548385620117,
      "activations/layer15_attention_weight_min": -29.798839569091797,
      "activations/layer16_attention_weight_max": 41.75632095336914,
      "activations/layer16_attention_weight_min": -29.973312377929688,
      "activations/layer17_attention_weight_max": 40.78740310668945,
      "activations/layer17_attention_weight_min": -27.704030990600586,
      "activations/layer18_attention_weight_max": 30.937374114990234,
      "activations/layer18_attention_weight_min": -18.430095672607422,
      "activations/layer19_attention_weight_max": 42.73963928222656,
      "activations/layer19_attention_weight_min": -30.419147491455078,
      "activations/layer1_attention_weight_max": 18.518457412719727,
      "activations/layer1_attention_weight_min": -14.940479278564453,
      "activations/layer20_attention_weight_max": 39.46360397338867,
      "activations/layer20_attention_weight_min": -24.342771530151367,
      "activations/layer21_attention_weight_max": 33.85962677001953,
      "activations/layer21_attention_weight_min": -18.907684326171875,
      "activations/layer22_attention_weight_max": 52.15619659423828,
      "activations/layer22_attention_weight_min": -26.04717445373535,
      "activations/layer23_attention_weight_max": 29.56525230407715,
      "activations/layer23_attention_weight_min": -16.51198387145996,
      "activations/layer2_attention_weight_max": 29.456443786621094,
      "activations/layer2_attention_weight_min": -30.538372039794922,
      "activations/layer3_attention_weight_max": 83.91802978515625,
      "activations/layer3_attention_weight_min": -87.01792907714844,
      "activations/layer4_attention_weight_max": 133.67544555664062,
      "activations/layer4_attention_weight_min": -131.34083557128906,
      "activations/layer5_attention_weight_max": 93.97274017333984,
      "activations/layer5_attention_weight_min": -88.13319396972656,
      "activations/layer6_attention_weight_max": 55.36368942260742,
      "activations/layer6_attention_weight_min": -55.00716018676758,
      "activations/layer7_attention_weight_max": 77.14617156982422,
      "activations/layer7_attention_weight_min": -70.54407501220703,
      "activations/layer8_attention_weight_max": 38.41568374633789,
      "activations/layer8_attention_weight_min": -37.088722229003906,
      "activations/layer9_attention_weight_max": 45.41456985473633,
      "activations/layer9_attention_weight_min": -35.48712921142578,
      "epoch": 3.48,
      "learning_rate": 0.0001288503787878788,
      "loss": 2.8919,
      "step": 59900
    },
    {
      "activations/layer0_attention_weight_max": 17.296741485595703,
      "activations/layer0_attention_weight_min": -11.880754470825195,
      "activations/layer10_attention_weight_max": 43.0531005859375,
      "activations/layer10_attention_weight_min": -34.02699661254883,
      "activations/layer11_attention_weight_max": 44.718231201171875,
      "activations/layer11_attention_weight_min": -34.023468017578125,
      "activations/layer12_attention_weight_max": 32.02705383300781,
      "activations/layer12_attention_weight_min": -28.195566177368164,
      "activations/layer13_attention_weight_max": 44.41250228881836,
      "activations/layer13_attention_weight_min": -31.1835880279541,
      "activations/layer14_attention_weight_max": 76.6731948852539,
      "activations/layer14_attention_weight_min": -48.8680534362793,
      "activations/layer15_attention_weight_max": 52.69599914550781,
      "activations/layer15_attention_weight_min": -33.470558166503906,
      "activations/layer16_attention_weight_max": 45.1850471496582,
      "activations/layer16_attention_weight_min": -29.333948135375977,
      "activations/layer17_attention_weight_max": 43.889549255371094,
      "activations/layer17_attention_weight_min": -23.55474281311035,
      "activations/layer18_attention_weight_max": 30.810659408569336,
      "activations/layer18_attention_weight_min": -22.612302780151367,
      "activations/layer19_attention_weight_max": 44.7554817199707,
      "activations/layer19_attention_weight_min": -27.933990478515625,
      "activations/layer1_attention_weight_max": 17.06922721862793,
      "activations/layer1_attention_weight_min": -14.767816543579102,
      "activations/layer20_attention_weight_max": 35.4135856628418,
      "activations/layer20_attention_weight_min": -23.025903701782227,
      "activations/layer21_attention_weight_max": 34.22297286987305,
      "activations/layer21_attention_weight_min": -18.804302215576172,
      "activations/layer22_attention_weight_max": 52.32016372680664,
      "activations/layer22_attention_weight_min": -24.428945541381836,
      "activations/layer23_attention_weight_max": 29.010971069335938,
      "activations/layer23_attention_weight_min": -15.823298454284668,
      "activations/layer2_attention_weight_max": 31.27017593383789,
      "activations/layer2_attention_weight_min": -29.946453094482422,
      "activations/layer3_attention_weight_max": 86.91138458251953,
      "activations/layer3_attention_weight_min": -84.33267974853516,
      "activations/layer4_attention_weight_max": 133.6206817626953,
      "activations/layer4_attention_weight_min": -124.24002838134766,
      "activations/layer5_attention_weight_max": 98.7952880859375,
      "activations/layer5_attention_weight_min": -91.2071533203125,
      "activations/layer6_attention_weight_max": 56.580081939697266,
      "activations/layer6_attention_weight_min": -54.90885925292969,
      "activations/layer7_attention_weight_max": 82.51143646240234,
      "activations/layer7_attention_weight_min": -77.5254898071289,
      "activations/layer8_attention_weight_max": 41.2521858215332,
      "activations/layer8_attention_weight_min": -37.29106903076172,
      "activations/layer9_attention_weight_max": 54.581398010253906,
      "activations/layer9_attention_weight_min": -40.37405014038086,
      "epoch": 3.48,
      "learning_rate": 0.00012883143939393938,
      "loss": 2.8886,
      "step": 59950
    },
    {
      "activations/layer0_attention_weight_max": 16.916996002197266,
      "activations/layer0_attention_weight_min": -10.784936904907227,
      "activations/layer10_attention_weight_max": 42.00617599487305,
      "activations/layer10_attention_weight_min": -31.53390121459961,
      "activations/layer11_attention_weight_max": 41.70539093017578,
      "activations/layer11_attention_weight_min": -30.9292049407959,
      "activations/layer12_attention_weight_max": 62.1712760925293,
      "activations/layer12_attention_weight_min": -32.068084716796875,
      "activations/layer13_attention_weight_max": 51.85171890258789,
      "activations/layer13_attention_weight_min": -38.81016159057617,
      "activations/layer14_attention_weight_max": 66.2303466796875,
      "activations/layer14_attention_weight_min": -44.87916564941406,
      "activations/layer15_attention_weight_max": 43.909400939941406,
      "activations/layer15_attention_weight_min": -32.05101776123047,
      "activations/layer16_attention_weight_max": 40.144622802734375,
      "activations/layer16_attention_weight_min": -28.068872451782227,
      "activations/layer17_attention_weight_max": 39.66780471801758,
      "activations/layer17_attention_weight_min": -27.117164611816406,
      "activations/layer18_attention_weight_max": 30.5043888092041,
      "activations/layer18_attention_weight_min": -20.334064483642578,
      "activations/layer19_attention_weight_max": 38.135009765625,
      "activations/layer19_attention_weight_min": -27.42316246032715,
      "activations/layer1_attention_weight_max": 17.92862319946289,
      "activations/layer1_attention_weight_min": -15.352493286132812,
      "activations/layer20_attention_weight_max": 33.468631744384766,
      "activations/layer20_attention_weight_min": -23.30829429626465,
      "activations/layer21_attention_weight_max": 33.6024169921875,
      "activations/layer21_attention_weight_min": -23.374731063842773,
      "activations/layer22_attention_weight_max": 51.097965240478516,
      "activations/layer22_attention_weight_min": -26.712039947509766,
      "activations/layer23_attention_weight_max": 27.629592895507812,
      "activations/layer23_attention_weight_min": -19.30537986755371,
      "activations/layer2_attention_weight_max": 32.34989547729492,
      "activations/layer2_attention_weight_min": -31.856916427612305,
      "activations/layer3_attention_weight_max": 90.75575256347656,
      "activations/layer3_attention_weight_min": -89.59197998046875,
      "activations/layer4_attention_weight_max": 143.1680145263672,
      "activations/layer4_attention_weight_min": -138.6780242919922,
      "activations/layer5_attention_weight_max": 104.27007293701172,
      "activations/layer5_attention_weight_min": -90.13030242919922,
      "activations/layer6_attention_weight_max": 63.701934814453125,
      "activations/layer6_attention_weight_min": -57.02054977416992,
      "activations/layer7_attention_weight_max": 88.59262084960938,
      "activations/layer7_attention_weight_min": -77.31282043457031,
      "activations/layer8_attention_weight_max": 45.28174591064453,
      "activations/layer8_attention_weight_min": -38.6365852355957,
      "activations/layer9_attention_weight_max": 52.535614013671875,
      "activations/layer9_attention_weight_min": -40.95256423950195,
      "epoch": 3.49,
      "learning_rate": 0.0001288125,
      "loss": 2.8856,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_loss": 2.841796875,
      "eval_runtime": 8.5419,
      "eval_samples_per_second": 502.701,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_openwebtext_loss": 2.841796875,
      "eval_openwebtext_ppl": 17.14654807625046,
      "eval_openwebtext_runtime": 8.5419,
      "eval_openwebtext_samples_per_second": 502.701,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_wikitext_loss": 3.09375,
      "eval_wikitext_ppl": 22.059646737841184,
      "eval_wikitext_runtime": 2.0132,
      "eval_wikitext_samples_per_second": 226.509,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_lambada_loss": 2.91796875,
      "eval_lambada_ppl": 18.503663694167408,
      "eval_lambada_runtime": 9.5755,
      "eval_lambada_samples_per_second": 508.485,
      "step": 60000
    },
    {
      "activations/layer0_attention_weight_max": 17.019081115722656,
      "activations/layer0_attention_weight_min": -12.106727600097656,
      "activations/layer10_attention_weight_max": 37.98825454711914,
      "activations/layer10_attention_weight_min": -31.222299575805664,
      "activations/layer11_attention_weight_max": 36.59946823120117,
      "activations/layer11_attention_weight_min": -27.85995864868164,
      "activations/layer12_attention_weight_max": 44.13990020751953,
      "activations/layer12_attention_weight_min": -32.47346878051758,
      "activations/layer13_attention_weight_max": 49.668357849121094,
      "activations/layer13_attention_weight_min": -36.188438415527344,
      "activations/layer14_attention_weight_max": 81.72112274169922,
      "activations/layer14_attention_weight_min": -58.59661102294922,
      "activations/layer15_attention_weight_max": 48.50078201293945,
      "activations/layer15_attention_weight_min": -34.43138885498047,
      "activations/layer16_attention_weight_max": 43.22077941894531,
      "activations/layer16_attention_weight_min": -28.672866821289062,
      "activations/layer17_attention_weight_max": 52.179439544677734,
      "activations/layer17_attention_weight_min": -28.05978012084961,
      "activations/layer18_attention_weight_max": 33.25537872314453,
      "activations/layer18_attention_weight_min": -19.559329986572266,
      "activations/layer19_attention_weight_max": 41.463077545166016,
      "activations/layer19_attention_weight_min": -29.952117919921875,
      "activations/layer1_attention_weight_max": 18.159523010253906,
      "activations/layer1_attention_weight_min": -15.312324523925781,
      "activations/layer20_attention_weight_max": 33.43102264404297,
      "activations/layer20_attention_weight_min": -21.71570587158203,
      "activations/layer21_attention_weight_max": 29.731258392333984,
      "activations/layer21_attention_weight_min": -20.18649673461914,
      "activations/layer22_attention_weight_max": 51.677242279052734,
      "activations/layer22_attention_weight_min": -26.35400390625,
      "activations/layer23_attention_weight_max": 28.363508224487305,
      "activations/layer23_attention_weight_min": -17.822341918945312,
      "activations/layer2_attention_weight_max": 29.28572654724121,
      "activations/layer2_attention_weight_min": -29.445404052734375,
      "activations/layer3_attention_weight_max": 82.51388549804688,
      "activations/layer3_attention_weight_min": -87.32975006103516,
      "activations/layer4_attention_weight_max": 131.1669921875,
      "activations/layer4_attention_weight_min": -129.9576873779297,
      "activations/layer5_attention_weight_max": 92.76937866210938,
      "activations/layer5_attention_weight_min": -88.08743286132812,
      "activations/layer6_attention_weight_max": 58.998046875,
      "activations/layer6_attention_weight_min": -53.210838317871094,
      "activations/layer7_attention_weight_max": 70.97526550292969,
      "activations/layer7_attention_weight_min": -70.9694595336914,
      "activations/layer8_attention_weight_max": 38.23381423950195,
      "activations/layer8_attention_weight_min": -35.72987747192383,
      "activations/layer9_attention_weight_max": 48.06143569946289,
      "activations/layer9_attention_weight_min": -34.96805953979492,
      "epoch": 3.49,
      "learning_rate": 0.0001287935606060606,
      "loss": 2.883,
      "step": 60050
    },
    {
      "activations/layer0_attention_weight_max": 17.08608627319336,
      "activations/layer0_attention_weight_min": -13.023422241210938,
      "activations/layer10_attention_weight_max": 36.90094757080078,
      "activations/layer10_attention_weight_min": -30.411380767822266,
      "activations/layer11_attention_weight_max": 38.11912155151367,
      "activations/layer11_attention_weight_min": -29.731346130371094,
      "activations/layer12_attention_weight_max": 37.581390380859375,
      "activations/layer12_attention_weight_min": -29.405473709106445,
      "activations/layer13_attention_weight_max": 58.13941955566406,
      "activations/layer13_attention_weight_min": -33.20884323120117,
      "activations/layer14_attention_weight_max": 79.14749145507812,
      "activations/layer14_attention_weight_min": -48.76655960083008,
      "activations/layer15_attention_weight_max": 56.51802062988281,
      "activations/layer15_attention_weight_min": -29.30666160583496,
      "activations/layer16_attention_weight_max": 47.02268981933594,
      "activations/layer16_attention_weight_min": -29.95506477355957,
      "activations/layer17_attention_weight_max": 45.75297546386719,
      "activations/layer17_attention_weight_min": -24.610366821289062,
      "activations/layer18_attention_weight_max": 34.27880096435547,
      "activations/layer18_attention_weight_min": -19.71054458618164,
      "activations/layer19_attention_weight_max": 51.665802001953125,
      "activations/layer19_attention_weight_min": -28.198368072509766,
      "activations/layer1_attention_weight_max": 17.68564796447754,
      "activations/layer1_attention_weight_min": -14.377785682678223,
      "activations/layer20_attention_weight_max": 37.44721221923828,
      "activations/layer20_attention_weight_min": -21.22464370727539,
      "activations/layer21_attention_weight_max": 38.340843200683594,
      "activations/layer21_attention_weight_min": -23.029157638549805,
      "activations/layer22_attention_weight_max": 53.53313064575195,
      "activations/layer22_attention_weight_min": -26.76219367980957,
      "activations/layer23_attention_weight_max": 31.31313133239746,
      "activations/layer23_attention_weight_min": -17.912757873535156,
      "activations/layer2_attention_weight_max": 30.364032745361328,
      "activations/layer2_attention_weight_min": -30.492446899414062,
      "activations/layer3_attention_weight_max": 83.0569076538086,
      "activations/layer3_attention_weight_min": -86.59544372558594,
      "activations/layer4_attention_weight_max": 129.78326416015625,
      "activations/layer4_attention_weight_min": -127.8129653930664,
      "activations/layer5_attention_weight_max": 96.19842529296875,
      "activations/layer5_attention_weight_min": -91.12675476074219,
      "activations/layer6_attention_weight_max": 58.38847732543945,
      "activations/layer6_attention_weight_min": -57.41318893432617,
      "activations/layer7_attention_weight_max": 79.16459655761719,
      "activations/layer7_attention_weight_min": -75.82737731933594,
      "activations/layer8_attention_weight_max": 41.682769775390625,
      "activations/layer8_attention_weight_min": -39.340248107910156,
      "activations/layer9_attention_weight_max": 44.629310607910156,
      "activations/layer9_attention_weight_min": -39.00950241088867,
      "epoch": 3.49,
      "learning_rate": 0.0001287746212121212,
      "loss": 2.885,
      "step": 60100
    },
    {
      "activations/layer0_attention_weight_max": 17.55615234375,
      "activations/layer0_attention_weight_min": -11.919544219970703,
      "activations/layer10_attention_weight_max": 42.51313400268555,
      "activations/layer10_attention_weight_min": -32.13789749145508,
      "activations/layer11_attention_weight_max": 34.82645797729492,
      "activations/layer11_attention_weight_min": -28.59807586669922,
      "activations/layer12_attention_weight_max": 37.40087890625,
      "activations/layer12_attention_weight_min": -26.92740249633789,
      "activations/layer13_attention_weight_max": 47.81999206542969,
      "activations/layer13_attention_weight_min": -32.52349090576172,
      "activations/layer14_attention_weight_max": 76.91789245605469,
      "activations/layer14_attention_weight_min": -52.195735931396484,
      "activations/layer15_attention_weight_max": 47.115997314453125,
      "activations/layer15_attention_weight_min": -32.812198638916016,
      "activations/layer16_attention_weight_max": 43.5654411315918,
      "activations/layer16_attention_weight_min": -31.69133186340332,
      "activations/layer17_attention_weight_max": 39.47719192504883,
      "activations/layer17_attention_weight_min": -27.537233352661133,
      "activations/layer18_attention_weight_max": 30.690378189086914,
      "activations/layer18_attention_weight_min": -20.448017120361328,
      "activations/layer19_attention_weight_max": 45.922542572021484,
      "activations/layer19_attention_weight_min": -28.379714965820312,
      "activations/layer1_attention_weight_max": 18.174564361572266,
      "activations/layer1_attention_weight_min": -14.870400428771973,
      "activations/layer20_attention_weight_max": 31.957258224487305,
      "activations/layer20_attention_weight_min": -22.296829223632812,
      "activations/layer21_attention_weight_max": 34.917816162109375,
      "activations/layer21_attention_weight_min": -22.516874313354492,
      "activations/layer22_attention_weight_max": 50.75411605834961,
      "activations/layer22_attention_weight_min": -28.936737060546875,
      "activations/layer23_attention_weight_max": 30.505529403686523,
      "activations/layer23_attention_weight_min": -16.725568771362305,
      "activations/layer2_attention_weight_max": 30.779415130615234,
      "activations/layer2_attention_weight_min": -30.710691452026367,
      "activations/layer3_attention_weight_max": 89.25189208984375,
      "activations/layer3_attention_weight_min": -86.76919555664062,
      "activations/layer4_attention_weight_max": 136.91647338867188,
      "activations/layer4_attention_weight_min": -127.8492431640625,
      "activations/layer5_attention_weight_max": 98.66949462890625,
      "activations/layer5_attention_weight_min": -89.71382141113281,
      "activations/layer6_attention_weight_max": 58.42455291748047,
      "activations/layer6_attention_weight_min": -54.61787414550781,
      "activations/layer7_attention_weight_max": 78.833740234375,
      "activations/layer7_attention_weight_min": -76.59029388427734,
      "activations/layer8_attention_weight_max": 41.66493225097656,
      "activations/layer8_attention_weight_min": -39.588401794433594,
      "activations/layer9_attention_weight_max": 44.072879791259766,
      "activations/layer9_attention_weight_min": -33.92765426635742,
      "epoch": 3.5,
      "learning_rate": 0.0001287556818181818,
      "loss": 2.8819,
      "step": 60150
    },
    {
      "activations/layer0_attention_weight_max": 17.408849716186523,
      "activations/layer0_attention_weight_min": -11.450033187866211,
      "activations/layer10_attention_weight_max": 42.87125778198242,
      "activations/layer10_attention_weight_min": -33.3517951965332,
      "activations/layer11_attention_weight_max": 43.91448974609375,
      "activations/layer11_attention_weight_min": -31.40308380126953,
      "activations/layer12_attention_weight_max": 47.54436492919922,
      "activations/layer12_attention_weight_min": -27.37961769104004,
      "activations/layer13_attention_weight_max": 57.13343048095703,
      "activations/layer13_attention_weight_min": -36.38630294799805,
      "activations/layer14_attention_weight_max": 90.37506103515625,
      "activations/layer14_attention_weight_min": -56.906917572021484,
      "activations/layer15_attention_weight_max": 46.1632080078125,
      "activations/layer15_attention_weight_min": -29.06755828857422,
      "activations/layer16_attention_weight_max": 46.541378021240234,
      "activations/layer16_attention_weight_min": -31.91777229309082,
      "activations/layer17_attention_weight_max": 39.49843215942383,
      "activations/layer17_attention_weight_min": -23.418764114379883,
      "activations/layer18_attention_weight_max": 30.485795974731445,
      "activations/layer18_attention_weight_min": -17.977975845336914,
      "activations/layer19_attention_weight_max": 51.69063949584961,
      "activations/layer19_attention_weight_min": -28.831607818603516,
      "activations/layer1_attention_weight_max": 17.202411651611328,
      "activations/layer1_attention_weight_min": -14.410338401794434,
      "activations/layer20_attention_weight_max": 32.135250091552734,
      "activations/layer20_attention_weight_min": -21.315128326416016,
      "activations/layer21_attention_weight_max": 32.09823989868164,
      "activations/layer21_attention_weight_min": -18.13813018798828,
      "activations/layer22_attention_weight_max": 46.73698806762695,
      "activations/layer22_attention_weight_min": -26.171554565429688,
      "activations/layer23_attention_weight_max": 26.115110397338867,
      "activations/layer23_attention_weight_min": -17.807098388671875,
      "activations/layer2_attention_weight_max": 32.927921295166016,
      "activations/layer2_attention_weight_min": -34.094303131103516,
      "activations/layer3_attention_weight_max": 95.05340576171875,
      "activations/layer3_attention_weight_min": -97.89971160888672,
      "activations/layer4_attention_weight_max": 142.86984252929688,
      "activations/layer4_attention_weight_min": -139.0528106689453,
      "activations/layer5_attention_weight_max": 105.78437805175781,
      "activations/layer5_attention_weight_min": -96.30809020996094,
      "activations/layer6_attention_weight_max": 61.02510070800781,
      "activations/layer6_attention_weight_min": -57.50531005859375,
      "activations/layer7_attention_weight_max": 84.34696960449219,
      "activations/layer7_attention_weight_min": -82.50597381591797,
      "activations/layer8_attention_weight_max": 46.64091110229492,
      "activations/layer8_attention_weight_min": -41.4730110168457,
      "activations/layer9_attention_weight_max": 51.260955810546875,
      "activations/layer9_attention_weight_min": -43.657588958740234,
      "epoch": 3.5,
      "learning_rate": 0.0001287367424242424,
      "loss": 2.8788,
      "step": 60200
    },
    {
      "activations/layer0_attention_weight_max": 15.561721801757812,
      "activations/layer0_attention_weight_min": -11.115687370300293,
      "activations/layer10_attention_weight_max": 42.826759338378906,
      "activations/layer10_attention_weight_min": -34.549564361572266,
      "activations/layer11_attention_weight_max": 39.7138786315918,
      "activations/layer11_attention_weight_min": -28.159650802612305,
      "activations/layer12_attention_weight_max": 32.85755157470703,
      "activations/layer12_attention_weight_min": -26.013532638549805,
      "activations/layer13_attention_weight_max": 55.554264068603516,
      "activations/layer13_attention_weight_min": -34.22002410888672,
      "activations/layer14_attention_weight_max": 73.4868392944336,
      "activations/layer14_attention_weight_min": -53.42079544067383,
      "activations/layer15_attention_weight_max": 44.67811965942383,
      "activations/layer15_attention_weight_min": -31.22850799560547,
      "activations/layer16_attention_weight_max": 38.759437561035156,
      "activations/layer16_attention_weight_min": -30.843944549560547,
      "activations/layer17_attention_weight_max": 44.71754837036133,
      "activations/layer17_attention_weight_min": -26.52566909790039,
      "activations/layer18_attention_weight_max": 29.858976364135742,
      "activations/layer18_attention_weight_min": -22.43638038635254,
      "activations/layer19_attention_weight_max": 43.628475189208984,
      "activations/layer19_attention_weight_min": -29.54490089416504,
      "activations/layer1_attention_weight_max": 17.237459182739258,
      "activations/layer1_attention_weight_min": -15.84176254272461,
      "activations/layer20_attention_weight_max": 34.51738739013672,
      "activations/layer20_attention_weight_min": -23.706499099731445,
      "activations/layer21_attention_weight_max": 31.559825897216797,
      "activations/layer21_attention_weight_min": -20.68509292602539,
      "activations/layer22_attention_weight_max": 53.33252716064453,
      "activations/layer22_attention_weight_min": -29.521352767944336,
      "activations/layer23_attention_weight_max": 30.605024337768555,
      "activations/layer23_attention_weight_min": -18.930042266845703,
      "activations/layer2_attention_weight_max": 30.35708999633789,
      "activations/layer2_attention_weight_min": -29.432409286499023,
      "activations/layer3_attention_weight_max": 81.05213928222656,
      "activations/layer3_attention_weight_min": -85.5883560180664,
      "activations/layer4_attention_weight_max": 130.69284057617188,
      "activations/layer4_attention_weight_min": -122.49138641357422,
      "activations/layer5_attention_weight_max": 91.75726318359375,
      "activations/layer5_attention_weight_min": -84.84550476074219,
      "activations/layer6_attention_weight_max": 57.2451286315918,
      "activations/layer6_attention_weight_min": -52.666282653808594,
      "activations/layer7_attention_weight_max": 72.93196105957031,
      "activations/layer7_attention_weight_min": -70.16145324707031,
      "activations/layer8_attention_weight_max": 42.647186279296875,
      "activations/layer8_attention_weight_min": -37.89640426635742,
      "activations/layer9_attention_weight_max": 41.51248550415039,
      "activations/layer9_attention_weight_min": -35.664024353027344,
      "epoch": 3.5,
      "learning_rate": 0.00012871780303030301,
      "loss": 2.904,
      "step": 60250
    },
    {
      "activations/layer0_attention_weight_max": 17.25790786743164,
      "activations/layer0_attention_weight_min": -12.048369407653809,
      "activations/layer10_attention_weight_max": 40.06421661376953,
      "activations/layer10_attention_weight_min": -33.3249626159668,
      "activations/layer11_attention_weight_max": 33.281585693359375,
      "activations/layer11_attention_weight_min": -29.822784423828125,
      "activations/layer12_attention_weight_max": 31.018386840820312,
      "activations/layer12_attention_weight_min": -27.83975601196289,
      "activations/layer13_attention_weight_max": 44.0241584777832,
      "activations/layer13_attention_weight_min": -33.202728271484375,
      "activations/layer14_attention_weight_max": 63.21334457397461,
      "activations/layer14_attention_weight_min": -47.23649215698242,
      "activations/layer15_attention_weight_max": 41.301300048828125,
      "activations/layer15_attention_weight_min": -30.09707260131836,
      "activations/layer16_attention_weight_max": 39.937034606933594,
      "activations/layer16_attention_weight_min": -31.4810791015625,
      "activations/layer17_attention_weight_max": 40.1649055480957,
      "activations/layer17_attention_weight_min": -26.44214630126953,
      "activations/layer18_attention_weight_max": 27.987812042236328,
      "activations/layer18_attention_weight_min": -20.55513572692871,
      "activations/layer19_attention_weight_max": 41.6824836730957,
      "activations/layer19_attention_weight_min": -30.501373291015625,
      "activations/layer1_attention_weight_max": 18.196012496948242,
      "activations/layer1_attention_weight_min": -13.82790470123291,
      "activations/layer20_attention_weight_max": 32.40241241455078,
      "activations/layer20_attention_weight_min": -23.600749969482422,
      "activations/layer21_attention_weight_max": 33.06855010986328,
      "activations/layer21_attention_weight_min": -20.555004119873047,
      "activations/layer22_attention_weight_max": 50.03346252441406,
      "activations/layer22_attention_weight_min": -28.81256675720215,
      "activations/layer23_attention_weight_max": 30.609050750732422,
      "activations/layer23_attention_weight_min": -18.739471435546875,
      "activations/layer2_attention_weight_max": 30.368846893310547,
      "activations/layer2_attention_weight_min": -29.44101333618164,
      "activations/layer3_attention_weight_max": 82.55120849609375,
      "activations/layer3_attention_weight_min": -83.79407501220703,
      "activations/layer4_attention_weight_max": 129.16014099121094,
      "activations/layer4_attention_weight_min": -128.33119201660156,
      "activations/layer5_attention_weight_max": 96.4293441772461,
      "activations/layer5_attention_weight_min": -95.84505462646484,
      "activations/layer6_attention_weight_max": 55.905982971191406,
      "activations/layer6_attention_weight_min": -54.6061897277832,
      "activations/layer7_attention_weight_max": 71.76799011230469,
      "activations/layer7_attention_weight_min": -74.30659484863281,
      "activations/layer8_attention_weight_max": 38.518863677978516,
      "activations/layer8_attention_weight_min": -38.17686462402344,
      "activations/layer9_attention_weight_max": 38.53113555908203,
      "activations/layer9_attention_weight_min": -34.94620895385742,
      "epoch": 3.5,
      "learning_rate": 0.00012869886363636363,
      "loss": 2.9024,
      "step": 60300
    },
    {
      "activations/layer0_attention_weight_max": 17.528831481933594,
      "activations/layer0_attention_weight_min": -11.212503433227539,
      "activations/layer10_attention_weight_max": 41.335330963134766,
      "activations/layer10_attention_weight_min": -31.962890625,
      "activations/layer11_attention_weight_max": 35.12212371826172,
      "activations/layer11_attention_weight_min": -27.260478973388672,
      "activations/layer12_attention_weight_max": 30.21847915649414,
      "activations/layer12_attention_weight_min": -25.471763610839844,
      "activations/layer13_attention_weight_max": 52.385032653808594,
      "activations/layer13_attention_weight_min": -35.32352066040039,
      "activations/layer14_attention_weight_max": 71.83757019042969,
      "activations/layer14_attention_weight_min": -49.50043869018555,
      "activations/layer15_attention_weight_max": 48.248958587646484,
      "activations/layer15_attention_weight_min": -33.2434196472168,
      "activations/layer16_attention_weight_max": 46.53864288330078,
      "activations/layer16_attention_weight_min": -30.868925094604492,
      "activations/layer17_attention_weight_max": 44.98442459106445,
      "activations/layer17_attention_weight_min": -25.284692764282227,
      "activations/layer18_attention_weight_max": 37.609588623046875,
      "activations/layer18_attention_weight_min": -19.144376754760742,
      "activations/layer19_attention_weight_max": 49.18825912475586,
      "activations/layer19_attention_weight_min": -30.23883056640625,
      "activations/layer1_attention_weight_max": 17.263607025146484,
      "activations/layer1_attention_weight_min": -15.96194839477539,
      "activations/layer20_attention_weight_max": 32.12410354614258,
      "activations/layer20_attention_weight_min": -24.5169620513916,
      "activations/layer21_attention_weight_max": 37.98524856567383,
      "activations/layer21_attention_weight_min": -22.064157485961914,
      "activations/layer22_attention_weight_max": 55.02353286743164,
      "activations/layer22_attention_weight_min": -26.800399780273438,
      "activations/layer23_attention_weight_max": 30.08614730834961,
      "activations/layer23_attention_weight_min": -19.60079574584961,
      "activations/layer2_attention_weight_max": 31.220014572143555,
      "activations/layer2_attention_weight_min": -29.66132164001465,
      "activations/layer3_attention_weight_max": 84.32107543945312,
      "activations/layer3_attention_weight_min": -86.28951263427734,
      "activations/layer4_attention_weight_max": 132.795654296875,
      "activations/layer4_attention_weight_min": -132.29327392578125,
      "activations/layer5_attention_weight_max": 100.47048950195312,
      "activations/layer5_attention_weight_min": -93.51155090332031,
      "activations/layer6_attention_weight_max": 58.53166198730469,
      "activations/layer6_attention_weight_min": -56.77302932739258,
      "activations/layer7_attention_weight_max": 74.72509765625,
      "activations/layer7_attention_weight_min": -72.95118713378906,
      "activations/layer8_attention_weight_max": 42.42951202392578,
      "activations/layer8_attention_weight_min": -38.16536331176758,
      "activations/layer9_attention_weight_max": 42.85905838012695,
      "activations/layer9_attention_weight_min": -34.83431625366211,
      "epoch": 3.51,
      "learning_rate": 0.00012867992424242422,
      "loss": 2.8962,
      "step": 60350
    },
    {
      "activations/layer0_attention_weight_max": 16.42245864868164,
      "activations/layer0_attention_weight_min": -11.687467575073242,
      "activations/layer10_attention_weight_max": 34.98707580566406,
      "activations/layer10_attention_weight_min": -28.424697875976562,
      "activations/layer11_attention_weight_max": 35.11729049682617,
      "activations/layer11_attention_weight_min": -28.437393188476562,
      "activations/layer12_attention_weight_max": 54.60430145263672,
      "activations/layer12_attention_weight_min": -34.660465240478516,
      "activations/layer13_attention_weight_max": 56.20610809326172,
      "activations/layer13_attention_weight_min": -40.322139739990234,
      "activations/layer14_attention_weight_max": 82.27075958251953,
      "activations/layer14_attention_weight_min": -55.391231536865234,
      "activations/layer15_attention_weight_max": 42.43313980102539,
      "activations/layer15_attention_weight_min": -31.14433479309082,
      "activations/layer16_attention_weight_max": 51.528228759765625,
      "activations/layer16_attention_weight_min": -31.55678367614746,
      "activations/layer17_attention_weight_max": 47.898624420166016,
      "activations/layer17_attention_weight_min": -24.639652252197266,
      "activations/layer18_attention_weight_max": 34.629817962646484,
      "activations/layer18_attention_weight_min": -19.99793815612793,
      "activations/layer19_attention_weight_max": 52.32436752319336,
      "activations/layer19_attention_weight_min": -28.8868465423584,
      "activations/layer1_attention_weight_max": 17.221160888671875,
      "activations/layer1_attention_weight_min": -15.577905654907227,
      "activations/layer20_attention_weight_max": 38.157623291015625,
      "activations/layer20_attention_weight_min": -22.33576011657715,
      "activations/layer21_attention_weight_max": 35.95611572265625,
      "activations/layer21_attention_weight_min": -20.277021408081055,
      "activations/layer22_attention_weight_max": 49.64043045043945,
      "activations/layer22_attention_weight_min": -27.86835289001465,
      "activations/layer23_attention_weight_max": 30.337413787841797,
      "activations/layer23_attention_weight_min": -17.845565795898438,
      "activations/layer2_attention_weight_max": 31.953311920166016,
      "activations/layer2_attention_weight_min": -32.00966262817383,
      "activations/layer3_attention_weight_max": 87.88133239746094,
      "activations/layer3_attention_weight_min": -89.8412094116211,
      "activations/layer4_attention_weight_max": 143.0665740966797,
      "activations/layer4_attention_weight_min": -133.51181030273438,
      "activations/layer5_attention_weight_max": 99.31680297851562,
      "activations/layer5_attention_weight_min": -89.64926147460938,
      "activations/layer6_attention_weight_max": 59.18212890625,
      "activations/layer6_attention_weight_min": -54.20700454711914,
      "activations/layer7_attention_weight_max": 73.4148941040039,
      "activations/layer7_attention_weight_min": -74.65066528320312,
      "activations/layer8_attention_weight_max": 39.524898529052734,
      "activations/layer8_attention_weight_min": -38.47539138793945,
      "activations/layer9_attention_weight_max": 46.55596160888672,
      "activations/layer9_attention_weight_min": -41.85755157470703,
      "epoch": 3.51,
      "learning_rate": 0.00012866098484848483,
      "loss": 2.8857,
      "step": 60400
    },
    {
      "activations/layer0_attention_weight_max": 16.469053268432617,
      "activations/layer0_attention_weight_min": -11.378642082214355,
      "activations/layer10_attention_weight_max": 41.74219512939453,
      "activations/layer10_attention_weight_min": -33.019649505615234,
      "activations/layer11_attention_weight_max": 41.129554748535156,
      "activations/layer11_attention_weight_min": -33.99687957763672,
      "activations/layer12_attention_weight_max": 34.06988525390625,
      "activations/layer12_attention_weight_min": -28.24068832397461,
      "activations/layer13_attention_weight_max": 56.810306549072266,
      "activations/layer13_attention_weight_min": -33.795265197753906,
      "activations/layer14_attention_weight_max": 104.25279998779297,
      "activations/layer14_attention_weight_min": -60.52083206176758,
      "activations/layer15_attention_weight_max": 52.43404769897461,
      "activations/layer15_attention_weight_min": -28.915874481201172,
      "activations/layer16_attention_weight_max": 47.212703704833984,
      "activations/layer16_attention_weight_min": -31.03955078125,
      "activations/layer17_attention_weight_max": 51.26848220825195,
      "activations/layer17_attention_weight_min": -24.066715240478516,
      "activations/layer18_attention_weight_max": 32.71467208862305,
      "activations/layer18_attention_weight_min": -21.945146560668945,
      "activations/layer19_attention_weight_max": 49.711090087890625,
      "activations/layer19_attention_weight_min": -29.25188636779785,
      "activations/layer1_attention_weight_max": 17.534343719482422,
      "activations/layer1_attention_weight_min": -14.285079956054688,
      "activations/layer20_attention_weight_max": 37.447662353515625,
      "activations/layer20_attention_weight_min": -22.918012619018555,
      "activations/layer21_attention_weight_max": 37.571292877197266,
      "activations/layer21_attention_weight_min": -20.94831657409668,
      "activations/layer22_attention_weight_max": 56.273136138916016,
      "activations/layer22_attention_weight_min": -28.335710525512695,
      "activations/layer23_attention_weight_max": 35.22637176513672,
      "activations/layer23_attention_weight_min": -17.46038055419922,
      "activations/layer2_attention_weight_max": 31.20684051513672,
      "activations/layer2_attention_weight_min": -31.211666107177734,
      "activations/layer3_attention_weight_max": 82.74628448486328,
      "activations/layer3_attention_weight_min": -84.39469146728516,
      "activations/layer4_attention_weight_max": 130.6881561279297,
      "activations/layer4_attention_weight_min": -128.43399047851562,
      "activations/layer5_attention_weight_max": 95.58641815185547,
      "activations/layer5_attention_weight_min": -90.471923828125,
      "activations/layer6_attention_weight_max": 58.529327392578125,
      "activations/layer6_attention_weight_min": -57.093894958496094,
      "activations/layer7_attention_weight_max": 78.29097747802734,
      "activations/layer7_attention_weight_min": -83.87930297851562,
      "activations/layer8_attention_weight_max": 39.6917724609375,
      "activations/layer8_attention_weight_min": -39.90757369995117,
      "activations/layer9_attention_weight_max": 44.27085876464844,
      "activations/layer9_attention_weight_min": -39.20151138305664,
      "epoch": 3.51,
      "learning_rate": 0.00012864204545454545,
      "loss": 2.902,
      "step": 60450
    },
    {
      "activations/layer0_attention_weight_max": 16.44110679626465,
      "activations/layer0_attention_weight_min": -12.17705249786377,
      "activations/layer10_attention_weight_max": 38.01736068725586,
      "activations/layer10_attention_weight_min": -32.523651123046875,
      "activations/layer11_attention_weight_max": 37.84637451171875,
      "activations/layer11_attention_weight_min": -29.083446502685547,
      "activations/layer12_attention_weight_max": 36.00969314575195,
      "activations/layer12_attention_weight_min": -26.926668167114258,
      "activations/layer13_attention_weight_max": 54.42604446411133,
      "activations/layer13_attention_weight_min": -36.99461364746094,
      "activations/layer14_attention_weight_max": 86.03926086425781,
      "activations/layer14_attention_weight_min": -54.66266632080078,
      "activations/layer15_attention_weight_max": 49.38241195678711,
      "activations/layer15_attention_weight_min": -30.437355041503906,
      "activations/layer16_attention_weight_max": 49.77881622314453,
      "activations/layer16_attention_weight_min": -30.870080947875977,
      "activations/layer17_attention_weight_max": 47.78292465209961,
      "activations/layer17_attention_weight_min": -26.269346237182617,
      "activations/layer18_attention_weight_max": 36.60248947143555,
      "activations/layer18_attention_weight_min": -19.33279800415039,
      "activations/layer19_attention_weight_max": 57.49494552612305,
      "activations/layer19_attention_weight_min": -28.27707290649414,
      "activations/layer1_attention_weight_max": 18.092819213867188,
      "activations/layer1_attention_weight_min": -15.817089080810547,
      "activations/layer20_attention_weight_max": 39.771934509277344,
      "activations/layer20_attention_weight_min": -22.585311889648438,
      "activations/layer21_attention_weight_max": 37.95636749267578,
      "activations/layer21_attention_weight_min": -21.238229751586914,
      "activations/layer22_attention_weight_max": 53.78961181640625,
      "activations/layer22_attention_weight_min": -27.078372955322266,
      "activations/layer23_attention_weight_max": 32.65963363647461,
      "activations/layer23_attention_weight_min": -17.05282974243164,
      "activations/layer2_attention_weight_max": 32.10773468017578,
      "activations/layer2_attention_weight_min": -29.979312896728516,
      "activations/layer3_attention_weight_max": 81.8946762084961,
      "activations/layer3_attention_weight_min": -86.13282012939453,
      "activations/layer4_attention_weight_max": 135.06521606445312,
      "activations/layer4_attention_weight_min": -126.7529525756836,
      "activations/layer5_attention_weight_max": 96.89474487304688,
      "activations/layer5_attention_weight_min": -88.7528076171875,
      "activations/layer6_attention_weight_max": 57.71791458129883,
      "activations/layer6_attention_weight_min": -55.7850227355957,
      "activations/layer7_attention_weight_max": 77.23619079589844,
      "activations/layer7_attention_weight_min": -72.6282958984375,
      "activations/layer8_attention_weight_max": 38.77192306518555,
      "activations/layer8_attention_weight_min": -38.32649612426758,
      "activations/layer9_attention_weight_max": 44.515560150146484,
      "activations/layer9_attention_weight_min": -37.94718551635742,
      "epoch": 3.52,
      "learning_rate": 0.00012862310606060604,
      "loss": 2.8929,
      "step": 60500
    },
    {
      "activations/layer0_attention_weight_max": 16.874805450439453,
      "activations/layer0_attention_weight_min": -13.039128303527832,
      "activations/layer10_attention_weight_max": 41.65349578857422,
      "activations/layer10_attention_weight_min": -29.383310317993164,
      "activations/layer11_attention_weight_max": 36.06300354003906,
      "activations/layer11_attention_weight_min": -26.60059928894043,
      "activations/layer12_attention_weight_max": 27.67129898071289,
      "activations/layer12_attention_weight_min": -26.709735870361328,
      "activations/layer13_attention_weight_max": 55.58167266845703,
      "activations/layer13_attention_weight_min": -34.09711456298828,
      "activations/layer14_attention_weight_max": 82.05924987792969,
      "activations/layer14_attention_weight_min": -55.582340240478516,
      "activations/layer15_attention_weight_max": 51.988014221191406,
      "activations/layer15_attention_weight_min": -29.823246002197266,
      "activations/layer16_attention_weight_max": 48.63294982910156,
      "activations/layer16_attention_weight_min": -32.35798263549805,
      "activations/layer17_attention_weight_max": 46.122493743896484,
      "activations/layer17_attention_weight_min": -25.992630004882812,
      "activations/layer18_attention_weight_max": 34.02244186401367,
      "activations/layer18_attention_weight_min": -20.025516510009766,
      "activations/layer19_attention_weight_max": 52.516876220703125,
      "activations/layer19_attention_weight_min": -28.03462791442871,
      "activations/layer1_attention_weight_max": 17.354206085205078,
      "activations/layer1_attention_weight_min": -13.181214332580566,
      "activations/layer20_attention_weight_max": 37.63340377807617,
      "activations/layer20_attention_weight_min": -22.87616729736328,
      "activations/layer21_attention_weight_max": 34.68144607543945,
      "activations/layer21_attention_weight_min": -19.16463279724121,
      "activations/layer22_attention_weight_max": 58.22787094116211,
      "activations/layer22_attention_weight_min": -24.78281593322754,
      "activations/layer23_attention_weight_max": 34.06849670410156,
      "activations/layer23_attention_weight_min": -17.938915252685547,
      "activations/layer2_attention_weight_max": 31.74224090576172,
      "activations/layer2_attention_weight_min": -31.81501007080078,
      "activations/layer3_attention_weight_max": 79.24121856689453,
      "activations/layer3_attention_weight_min": -81.93994140625,
      "activations/layer4_attention_weight_max": 130.73208618164062,
      "activations/layer4_attention_weight_min": -121.8925552368164,
      "activations/layer5_attention_weight_max": 96.86061096191406,
      "activations/layer5_attention_weight_min": -87.77174377441406,
      "activations/layer6_attention_weight_max": 58.720497131347656,
      "activations/layer6_attention_weight_min": -57.80881118774414,
      "activations/layer7_attention_weight_max": 72.83428192138672,
      "activations/layer7_attention_weight_min": -73.86532592773438,
      "activations/layer8_attention_weight_max": 39.069305419921875,
      "activations/layer8_attention_weight_min": -36.21689987182617,
      "activations/layer9_attention_weight_max": 40.65383529663086,
      "activations/layer9_attention_weight_min": -35.75788497924805,
      "epoch": 3.52,
      "learning_rate": 0.00012860416666666665,
      "loss": 2.9036,
      "step": 60550
    },
    {
      "activations/layer0_attention_weight_max": 16.207508087158203,
      "activations/layer0_attention_weight_min": -11.643363952636719,
      "activations/layer10_attention_weight_max": 40.21843719482422,
      "activations/layer10_attention_weight_min": -34.19963836669922,
      "activations/layer11_attention_weight_max": 46.53230285644531,
      "activations/layer11_attention_weight_min": -33.663726806640625,
      "activations/layer12_attention_weight_max": 33.669986724853516,
      "activations/layer12_attention_weight_min": -29.384538650512695,
      "activations/layer13_attention_weight_max": 48.912864685058594,
      "activations/layer13_attention_weight_min": -33.9959831237793,
      "activations/layer14_attention_weight_max": 81.65985870361328,
      "activations/layer14_attention_weight_min": -53.63129806518555,
      "activations/layer15_attention_weight_max": 50.25883865356445,
      "activations/layer15_attention_weight_min": -33.03794479370117,
      "activations/layer16_attention_weight_max": 45.9615364074707,
      "activations/layer16_attention_weight_min": -28.927644729614258,
      "activations/layer17_attention_weight_max": 41.8682975769043,
      "activations/layer17_attention_weight_min": -25.146982192993164,
      "activations/layer18_attention_weight_max": 33.21037673950195,
      "activations/layer18_attention_weight_min": -22.286605834960938,
      "activations/layer19_attention_weight_max": 48.97537612915039,
      "activations/layer19_attention_weight_min": -28.864713668823242,
      "activations/layer1_attention_weight_max": 18.145191192626953,
      "activations/layer1_attention_weight_min": -14.387203216552734,
      "activations/layer20_attention_weight_max": 34.648277282714844,
      "activations/layer20_attention_weight_min": -23.31962776184082,
      "activations/layer21_attention_weight_max": 36.75444793701172,
      "activations/layer21_attention_weight_min": -21.149511337280273,
      "activations/layer22_attention_weight_max": 49.93448257446289,
      "activations/layer22_attention_weight_min": -27.160303115844727,
      "activations/layer23_attention_weight_max": 31.275856018066406,
      "activations/layer23_attention_weight_min": -19.578874588012695,
      "activations/layer2_attention_weight_max": 29.946765899658203,
      "activations/layer2_attention_weight_min": -29.072935104370117,
      "activations/layer3_attention_weight_max": 85.04737854003906,
      "activations/layer3_attention_weight_min": -84.48773193359375,
      "activations/layer4_attention_weight_max": 131.61053466796875,
      "activations/layer4_attention_weight_min": -122.73565673828125,
      "activations/layer5_attention_weight_max": 96.25591278076172,
      "activations/layer5_attention_weight_min": -85.04012298583984,
      "activations/layer6_attention_weight_max": 58.90819549560547,
      "activations/layer6_attention_weight_min": -54.31001281738281,
      "activations/layer7_attention_weight_max": 75.2089614868164,
      "activations/layer7_attention_weight_min": -77.92332458496094,
      "activations/layer8_attention_weight_max": 41.13698196411133,
      "activations/layer8_attention_weight_min": -37.73622512817383,
      "activations/layer9_attention_weight_max": 43.95913314819336,
      "activations/layer9_attention_weight_min": -46.161041259765625,
      "epoch": 3.52,
      "learning_rate": 0.00012858522727272727,
      "loss": 2.906,
      "step": 60600
    },
    {
      "activations/layer0_attention_weight_max": 15.900062561035156,
      "activations/layer0_attention_weight_min": -11.538169860839844,
      "activations/layer10_attention_weight_max": 39.35466384887695,
      "activations/layer10_attention_weight_min": -31.49449920654297,
      "activations/layer11_attention_weight_max": 34.95843505859375,
      "activations/layer11_attention_weight_min": -28.53811264038086,
      "activations/layer12_attention_weight_max": 68.39739990234375,
      "activations/layer12_attention_weight_min": -32.53828811645508,
      "activations/layer13_attention_weight_max": 51.24540328979492,
      "activations/layer13_attention_weight_min": -32.368473052978516,
      "activations/layer14_attention_weight_max": 74.58871459960938,
      "activations/layer14_attention_weight_min": -54.06056213378906,
      "activations/layer15_attention_weight_max": 46.4124641418457,
      "activations/layer15_attention_weight_min": -31.789121627807617,
      "activations/layer16_attention_weight_max": 43.87411117553711,
      "activations/layer16_attention_weight_min": -30.824926376342773,
      "activations/layer17_attention_weight_max": 42.94914245605469,
      "activations/layer17_attention_weight_min": -27.43865394592285,
      "activations/layer18_attention_weight_max": 33.662899017333984,
      "activations/layer18_attention_weight_min": -20.5601863861084,
      "activations/layer19_attention_weight_max": 51.65916061401367,
      "activations/layer19_attention_weight_min": -31.49086570739746,
      "activations/layer1_attention_weight_max": 17.69464874267578,
      "activations/layer1_attention_weight_min": -14.110442161560059,
      "activations/layer20_attention_weight_max": 38.788230895996094,
      "activations/layer20_attention_weight_min": -24.961580276489258,
      "activations/layer21_attention_weight_max": 43.26271438598633,
      "activations/layer21_attention_weight_min": -21.47531509399414,
      "activations/layer22_attention_weight_max": 53.35166549682617,
      "activations/layer22_attention_weight_min": -27.316003799438477,
      "activations/layer23_attention_weight_max": 32.99524688720703,
      "activations/layer23_attention_weight_min": -15.616146087646484,
      "activations/layer2_attention_weight_max": 31.155010223388672,
      "activations/layer2_attention_weight_min": -29.760358810424805,
      "activations/layer3_attention_weight_max": 83.42937469482422,
      "activations/layer3_attention_weight_min": -85.67542266845703,
      "activations/layer4_attention_weight_max": 132.58021545410156,
      "activations/layer4_attention_weight_min": -132.65025329589844,
      "activations/layer5_attention_weight_max": 98.04198455810547,
      "activations/layer5_attention_weight_min": -90.95848846435547,
      "activations/layer6_attention_weight_max": 60.69931411743164,
      "activations/layer6_attention_weight_min": -54.732723236083984,
      "activations/layer7_attention_weight_max": 79.21394348144531,
      "activations/layer7_attention_weight_min": -77.85614776611328,
      "activations/layer8_attention_weight_max": 40.19361114501953,
      "activations/layer8_attention_weight_min": -37.85320281982422,
      "activations/layer9_attention_weight_max": 45.64298629760742,
      "activations/layer9_attention_weight_min": -39.5439338684082,
      "epoch": 3.52,
      "learning_rate": 0.00012856628787878785,
      "loss": 2.9083,
      "step": 60650
    },
    {
      "activations/layer0_attention_weight_max": 15.869946479797363,
      "activations/layer0_attention_weight_min": -12.236666679382324,
      "activations/layer10_attention_weight_max": 43.17601013183594,
      "activations/layer10_attention_weight_min": -32.65728759765625,
      "activations/layer11_attention_weight_max": 39.052345275878906,
      "activations/layer11_attention_weight_min": -32.44327163696289,
      "activations/layer12_attention_weight_max": 33.44704055786133,
      "activations/layer12_attention_weight_min": -24.953832626342773,
      "activations/layer13_attention_weight_max": 49.645633697509766,
      "activations/layer13_attention_weight_min": -35.08592987060547,
      "activations/layer14_attention_weight_max": 76.28125762939453,
      "activations/layer14_attention_weight_min": -54.92753219604492,
      "activations/layer15_attention_weight_max": 44.63410949707031,
      "activations/layer15_attention_weight_min": -31.343820571899414,
      "activations/layer16_attention_weight_max": 39.64180374145508,
      "activations/layer16_attention_weight_min": -28.617267608642578,
      "activations/layer17_attention_weight_max": 42.33667755126953,
      "activations/layer17_attention_weight_min": -25.55919075012207,
      "activations/layer18_attention_weight_max": 30.880163192749023,
      "activations/layer18_attention_weight_min": -19.952266693115234,
      "activations/layer19_attention_weight_max": 44.46773910522461,
      "activations/layer19_attention_weight_min": -29.446826934814453,
      "activations/layer1_attention_weight_max": 18.222688674926758,
      "activations/layer1_attention_weight_min": -16.0945987701416,
      "activations/layer20_attention_weight_max": 33.110103607177734,
      "activations/layer20_attention_weight_min": -22.192646026611328,
      "activations/layer21_attention_weight_max": 37.4364128112793,
      "activations/layer21_attention_weight_min": -24.275699615478516,
      "activations/layer22_attention_weight_max": 53.75056838989258,
      "activations/layer22_attention_weight_min": -28.202911376953125,
      "activations/layer23_attention_weight_max": 29.00617790222168,
      "activations/layer23_attention_weight_min": -18.032672882080078,
      "activations/layer2_attention_weight_max": 31.284645080566406,
      "activations/layer2_attention_weight_min": -29.8797664642334,
      "activations/layer3_attention_weight_max": 85.91059112548828,
      "activations/layer3_attention_weight_min": -89.09346771240234,
      "activations/layer4_attention_weight_max": 134.1774139404297,
      "activations/layer4_attention_weight_min": -127.7931900024414,
      "activations/layer5_attention_weight_max": 101.69127655029297,
      "activations/layer5_attention_weight_min": -88.28800964355469,
      "activations/layer6_attention_weight_max": 62.20892333984375,
      "activations/layer6_attention_weight_min": -56.027305603027344,
      "activations/layer7_attention_weight_max": 82.5450668334961,
      "activations/layer7_attention_weight_min": -83.65327453613281,
      "activations/layer8_attention_weight_max": 43.001522064208984,
      "activations/layer8_attention_weight_min": -38.940521240234375,
      "activations/layer9_attention_weight_max": 45.19837188720703,
      "activations/layer9_attention_weight_min": -37.00133514404297,
      "epoch": 3.53,
      "learning_rate": 0.00012854734848484847,
      "loss": 2.8885,
      "step": 60700
    },
    {
      "activations/layer0_attention_weight_max": 15.899301528930664,
      "activations/layer0_attention_weight_min": -11.290528297424316,
      "activations/layer10_attention_weight_max": 40.769474029541016,
      "activations/layer10_attention_weight_min": -31.458450317382812,
      "activations/layer11_attention_weight_max": 34.71527862548828,
      "activations/layer11_attention_weight_min": -26.665647506713867,
      "activations/layer12_attention_weight_max": 29.644285202026367,
      "activations/layer12_attention_weight_min": -27.482694625854492,
      "activations/layer13_attention_weight_max": 48.450618743896484,
      "activations/layer13_attention_weight_min": -32.346675872802734,
      "activations/layer14_attention_weight_max": 76.75017547607422,
      "activations/layer14_attention_weight_min": -48.12739181518555,
      "activations/layer15_attention_weight_max": 48.08621597290039,
      "activations/layer15_attention_weight_min": -31.280757904052734,
      "activations/layer16_attention_weight_max": 41.321380615234375,
      "activations/layer16_attention_weight_min": -30.772417068481445,
      "activations/layer17_attention_weight_max": 46.31918716430664,
      "activations/layer17_attention_weight_min": -28.114334106445312,
      "activations/layer18_attention_weight_max": 31.434101104736328,
      "activations/layer18_attention_weight_min": -19.074602127075195,
      "activations/layer19_attention_weight_max": 45.76416778564453,
      "activations/layer19_attention_weight_min": -28.84499168395996,
      "activations/layer1_attention_weight_max": 17.68384552001953,
      "activations/layer1_attention_weight_min": -14.789612770080566,
      "activations/layer20_attention_weight_max": 40.76896286010742,
      "activations/layer20_attention_weight_min": -23.85363006591797,
      "activations/layer21_attention_weight_max": 36.53093338012695,
      "activations/layer21_attention_weight_min": -22.27755355834961,
      "activations/layer22_attention_weight_max": 53.07401657104492,
      "activations/layer22_attention_weight_min": -36.00831985473633,
      "activations/layer23_attention_weight_max": 29.23043441772461,
      "activations/layer23_attention_weight_min": -19.877872467041016,
      "activations/layer2_attention_weight_max": 31.519268035888672,
      "activations/layer2_attention_weight_min": -30.96797752380371,
      "activations/layer3_attention_weight_max": 86.86673736572266,
      "activations/layer3_attention_weight_min": -82.28180694580078,
      "activations/layer4_attention_weight_max": 138.36524963378906,
      "activations/layer4_attention_weight_min": -126.02935791015625,
      "activations/layer5_attention_weight_max": 96.01953125,
      "activations/layer5_attention_weight_min": -91.39543151855469,
      "activations/layer6_attention_weight_max": 57.18218231201172,
      "activations/layer6_attention_weight_min": -56.2850456237793,
      "activations/layer7_attention_weight_max": 78.37588500976562,
      "activations/layer7_attention_weight_min": -73.4052963256836,
      "activations/layer8_attention_weight_max": 41.911128997802734,
      "activations/layer8_attention_weight_min": -39.17961883544922,
      "activations/layer9_attention_weight_max": 39.92768478393555,
      "activations/layer9_attention_weight_min": -32.311641693115234,
      "epoch": 3.53,
      "learning_rate": 0.00012852840909090908,
      "loss": 2.9199,
      "step": 60750
    },
    {
      "activations/layer0_attention_weight_max": 15.195717811584473,
      "activations/layer0_attention_weight_min": -12.907323837280273,
      "activations/layer10_attention_weight_max": 39.02616882324219,
      "activations/layer10_attention_weight_min": -31.16789436340332,
      "activations/layer11_attention_weight_max": 36.38187026977539,
      "activations/layer11_attention_weight_min": -29.090747833251953,
      "activations/layer12_attention_weight_max": 75.73439025878906,
      "activations/layer12_attention_weight_min": -40.13905334472656,
      "activations/layer13_attention_weight_max": 71.57453918457031,
      "activations/layer13_attention_weight_min": -41.594017028808594,
      "activations/layer14_attention_weight_max": 85.3187484741211,
      "activations/layer14_attention_weight_min": -55.91410827636719,
      "activations/layer15_attention_weight_max": 46.01996612548828,
      "activations/layer15_attention_weight_min": -29.00465202331543,
      "activations/layer16_attention_weight_max": 42.83514404296875,
      "activations/layer16_attention_weight_min": -29.97359275817871,
      "activations/layer17_attention_weight_max": 46.026241302490234,
      "activations/layer17_attention_weight_min": -25.963525772094727,
      "activations/layer18_attention_weight_max": 31.045625686645508,
      "activations/layer18_attention_weight_min": -17.63125228881836,
      "activations/layer19_attention_weight_max": 45.03764724731445,
      "activations/layer19_attention_weight_min": -25.210302352905273,
      "activations/layer1_attention_weight_max": 19.04465675354004,
      "activations/layer1_attention_weight_min": -13.788410186767578,
      "activations/layer20_attention_weight_max": 40.449764251708984,
      "activations/layer20_attention_weight_min": -20.911344528198242,
      "activations/layer21_attention_weight_max": 35.8462028503418,
      "activations/layer21_attention_weight_min": -17.86587905883789,
      "activations/layer22_attention_weight_max": 53.778358459472656,
      "activations/layer22_attention_weight_min": -26.740020751953125,
      "activations/layer23_attention_weight_max": 31.124862670898438,
      "activations/layer23_attention_weight_min": -16.40791130065918,
      "activations/layer2_attention_weight_max": 31.343971252441406,
      "activations/layer2_attention_weight_min": -31.023923873901367,
      "activations/layer3_attention_weight_max": 89.90725708007812,
      "activations/layer3_attention_weight_min": -91.4035415649414,
      "activations/layer4_attention_weight_max": 135.99066162109375,
      "activations/layer4_attention_weight_min": -125.38716888427734,
      "activations/layer5_attention_weight_max": 96.6728744506836,
      "activations/layer5_attention_weight_min": -91.15577697753906,
      "activations/layer6_attention_weight_max": 56.99366760253906,
      "activations/layer6_attention_weight_min": -55.785865783691406,
      "activations/layer7_attention_weight_max": 78.49185943603516,
      "activations/layer7_attention_weight_min": -77.6422119140625,
      "activations/layer8_attention_weight_max": 39.43086624145508,
      "activations/layer8_attention_weight_min": -39.15618896484375,
      "activations/layer9_attention_weight_max": 46.11992263793945,
      "activations/layer9_attention_weight_min": -37.16384506225586,
      "epoch": 3.53,
      "learning_rate": 0.0001285094696969697,
      "loss": 2.8953,
      "step": 60800
    },
    {
      "activations/layer0_attention_weight_max": 15.038702011108398,
      "activations/layer0_attention_weight_min": -10.88744068145752,
      "activations/layer10_attention_weight_max": 37.1439208984375,
      "activations/layer10_attention_weight_min": -31.677989959716797,
      "activations/layer11_attention_weight_max": 34.32222366333008,
      "activations/layer11_attention_weight_min": -27.40620231628418,
      "activations/layer12_attention_weight_max": 32.92327117919922,
      "activations/layer12_attention_weight_min": -29.243715286254883,
      "activations/layer13_attention_weight_max": 45.954654693603516,
      "activations/layer13_attention_weight_min": -32.76524353027344,
      "activations/layer14_attention_weight_max": 69.19822692871094,
      "activations/layer14_attention_weight_min": -48.99577713012695,
      "activations/layer15_attention_weight_max": 50.11598205566406,
      "activations/layer15_attention_weight_min": -33.87399673461914,
      "activations/layer16_attention_weight_max": 49.12167739868164,
      "activations/layer16_attention_weight_min": -31.93902587890625,
      "activations/layer17_attention_weight_max": 44.695884704589844,
      "activations/layer17_attention_weight_min": -29.926944732666016,
      "activations/layer18_attention_weight_max": 34.52394104003906,
      "activations/layer18_attention_weight_min": -20.483966827392578,
      "activations/layer19_attention_weight_max": 47.0876579284668,
      "activations/layer19_attention_weight_min": -29.808042526245117,
      "activations/layer1_attention_weight_max": 17.66864013671875,
      "activations/layer1_attention_weight_min": -13.887309074401855,
      "activations/layer20_attention_weight_max": 36.82695388793945,
      "activations/layer20_attention_weight_min": -25.311168670654297,
      "activations/layer21_attention_weight_max": 34.65485382080078,
      "activations/layer21_attention_weight_min": -21.0697078704834,
      "activations/layer22_attention_weight_max": 50.81841278076172,
      "activations/layer22_attention_weight_min": -27.16225814819336,
      "activations/layer23_attention_weight_max": 29.714460372924805,
      "activations/layer23_attention_weight_min": -17.19769287109375,
      "activations/layer2_attention_weight_max": 30.304851531982422,
      "activations/layer2_attention_weight_min": -31.90396499633789,
      "activations/layer3_attention_weight_max": 81.97793579101562,
      "activations/layer3_attention_weight_min": -87.85142517089844,
      "activations/layer4_attention_weight_max": 129.9557342529297,
      "activations/layer4_attention_weight_min": -124.61226654052734,
      "activations/layer5_attention_weight_max": 96.4580078125,
      "activations/layer5_attention_weight_min": -88.26480102539062,
      "activations/layer6_attention_weight_max": 58.866249084472656,
      "activations/layer6_attention_weight_min": -58.266395568847656,
      "activations/layer7_attention_weight_max": 77.81607055664062,
      "activations/layer7_attention_weight_min": -73.64827728271484,
      "activations/layer8_attention_weight_max": 39.831912994384766,
      "activations/layer8_attention_weight_min": -37.2049560546875,
      "activations/layer9_attention_weight_max": 44.5380973815918,
      "activations/layer9_attention_weight_min": -39.81501007080078,
      "epoch": 3.54,
      "learning_rate": 0.0001284905303030303,
      "loss": 2.9031,
      "step": 60850
    },
    {
      "activations/layer0_attention_weight_max": 15.75089168548584,
      "activations/layer0_attention_weight_min": -11.45566177368164,
      "activations/layer10_attention_weight_max": 45.50193405151367,
      "activations/layer10_attention_weight_min": -32.59223175048828,
      "activations/layer11_attention_weight_max": 37.94811248779297,
      "activations/layer11_attention_weight_min": -32.46442794799805,
      "activations/layer12_attention_weight_max": 42.939483642578125,
      "activations/layer12_attention_weight_min": -27.919082641601562,
      "activations/layer13_attention_weight_max": 61.082984924316406,
      "activations/layer13_attention_weight_min": -41.21470642089844,
      "activations/layer14_attention_weight_max": 83.81204986572266,
      "activations/layer14_attention_weight_min": -60.84247970581055,
      "activations/layer15_attention_weight_max": 48.75835037231445,
      "activations/layer15_attention_weight_min": -32.166080474853516,
      "activations/layer16_attention_weight_max": 43.026580810546875,
      "activations/layer16_attention_weight_min": -35.99567413330078,
      "activations/layer17_attention_weight_max": 41.90058898925781,
      "activations/layer17_attention_weight_min": -25.164464950561523,
      "activations/layer18_attention_weight_max": 29.462936401367188,
      "activations/layer18_attention_weight_min": -20.887218475341797,
      "activations/layer19_attention_weight_max": 44.546146392822266,
      "activations/layer19_attention_weight_min": -29.113569259643555,
      "activations/layer1_attention_weight_max": 18.732486724853516,
      "activations/layer1_attention_weight_min": -14.255720138549805,
      "activations/layer20_attention_weight_max": 33.44619369506836,
      "activations/layer20_attention_weight_min": -22.906089782714844,
      "activations/layer21_attention_weight_max": 30.71358299255371,
      "activations/layer21_attention_weight_min": -19.954483032226562,
      "activations/layer22_attention_weight_max": 48.41858673095703,
      "activations/layer22_attention_weight_min": -29.49503517150879,
      "activations/layer23_attention_weight_max": 28.551002502441406,
      "activations/layer23_attention_weight_min": -18.74993133544922,
      "activations/layer2_attention_weight_max": 31.924345016479492,
      "activations/layer2_attention_weight_min": -30.274261474609375,
      "activations/layer3_attention_weight_max": 90.84613037109375,
      "activations/layer3_attention_weight_min": -93.18020629882812,
      "activations/layer4_attention_weight_max": 141.22300720214844,
      "activations/layer4_attention_weight_min": -145.5167694091797,
      "activations/layer5_attention_weight_max": 102.96240234375,
      "activations/layer5_attention_weight_min": -98.23974609375,
      "activations/layer6_attention_weight_max": 62.28570556640625,
      "activations/layer6_attention_weight_min": -62.4738883972168,
      "activations/layer7_attention_weight_max": 82.01444244384766,
      "activations/layer7_attention_weight_min": -85.73872375488281,
      "activations/layer8_attention_weight_max": 43.98175811767578,
      "activations/layer8_attention_weight_min": -41.65571975708008,
      "activations/layer9_attention_weight_max": 44.43143844604492,
      "activations/layer9_attention_weight_min": -39.6450309753418,
      "epoch": 3.54,
      "learning_rate": 0.0001284715909090909,
      "loss": 2.8857,
      "step": 60900
    },
    {
      "activations/layer0_attention_weight_max": 15.407429695129395,
      "activations/layer0_attention_weight_min": -11.641680717468262,
      "activations/layer10_attention_weight_max": 43.6031494140625,
      "activations/layer10_attention_weight_min": -31.749507904052734,
      "activations/layer11_attention_weight_max": 57.06960678100586,
      "activations/layer11_attention_weight_min": -34.302669525146484,
      "activations/layer12_attention_weight_max": 50.08407211303711,
      "activations/layer12_attention_weight_min": -29.14927101135254,
      "activations/layer13_attention_weight_max": 78.88259887695312,
      "activations/layer13_attention_weight_min": -40.850341796875,
      "activations/layer14_attention_weight_max": 108.02295684814453,
      "activations/layer14_attention_weight_min": -59.73134231567383,
      "activations/layer15_attention_weight_max": 66.97716522216797,
      "activations/layer15_attention_weight_min": -32.36817169189453,
      "activations/layer16_attention_weight_max": 45.2448844909668,
      "activations/layer16_attention_weight_min": -28.530508041381836,
      "activations/layer17_attention_weight_max": 42.15445327758789,
      "activations/layer17_attention_weight_min": -25.60757064819336,
      "activations/layer18_attention_weight_max": 35.48461151123047,
      "activations/layer18_attention_weight_min": -21.268644332885742,
      "activations/layer19_attention_weight_max": 49.36499786376953,
      "activations/layer19_attention_weight_min": -29.81414794921875,
      "activations/layer1_attention_weight_max": 17.329633712768555,
      "activations/layer1_attention_weight_min": -13.790079116821289,
      "activations/layer20_attention_weight_max": 39.6683464050293,
      "activations/layer20_attention_weight_min": -23.147409439086914,
      "activations/layer21_attention_weight_max": 31.58757781982422,
      "activations/layer21_attention_weight_min": -20.6966609954834,
      "activations/layer22_attention_weight_max": 44.41599655151367,
      "activations/layer22_attention_weight_min": -26.12990379333496,
      "activations/layer23_attention_weight_max": 27.637866973876953,
      "activations/layer23_attention_weight_min": -16.443513870239258,
      "activations/layer2_attention_weight_max": 30.392349243164062,
      "activations/layer2_attention_weight_min": -28.710742950439453,
      "activations/layer3_attention_weight_max": 82.17147827148438,
      "activations/layer3_attention_weight_min": -84.209716796875,
      "activations/layer4_attention_weight_max": 127.77384185791016,
      "activations/layer4_attention_weight_min": -127.68917083740234,
      "activations/layer5_attention_weight_max": 96.14525604248047,
      "activations/layer5_attention_weight_min": -90.93370056152344,
      "activations/layer6_attention_weight_max": 58.275428771972656,
      "activations/layer6_attention_weight_min": -54.28192901611328,
      "activations/layer7_attention_weight_max": 77.26091766357422,
      "activations/layer7_attention_weight_min": -74.19470977783203,
      "activations/layer8_attention_weight_max": 41.63724899291992,
      "activations/layer8_attention_weight_min": -37.00205612182617,
      "activations/layer9_attention_weight_max": 52.10305404663086,
      "activations/layer9_attention_weight_min": -39.24982833862305,
      "epoch": 3.54,
      "learning_rate": 0.00012845265151515152,
      "loss": 2.8813,
      "step": 60950
    },
    {
      "activations/layer0_attention_weight_max": 15.897117614746094,
      "activations/layer0_attention_weight_min": -10.926020622253418,
      "activations/layer10_attention_weight_max": 47.73887634277344,
      "activations/layer10_attention_weight_min": -32.31643295288086,
      "activations/layer11_attention_weight_max": 53.435523986816406,
      "activations/layer11_attention_weight_min": -31.436511993408203,
      "activations/layer12_attention_weight_max": 69.37481689453125,
      "activations/layer12_attention_weight_min": -30.131202697753906,
      "activations/layer13_attention_weight_max": 72.71833801269531,
      "activations/layer13_attention_weight_min": -38.69188690185547,
      "activations/layer14_attention_weight_max": 77.07891845703125,
      "activations/layer14_attention_weight_min": -50.573307037353516,
      "activations/layer15_attention_weight_max": 55.322669982910156,
      "activations/layer15_attention_weight_min": -28.53438377380371,
      "activations/layer16_attention_weight_max": 48.20317077636719,
      "activations/layer16_attention_weight_min": -29.756698608398438,
      "activations/layer17_attention_weight_max": 44.28046417236328,
      "activations/layer17_attention_weight_min": -24.709871292114258,
      "activations/layer18_attention_weight_max": 30.842082977294922,
      "activations/layer18_attention_weight_min": -18.077388763427734,
      "activations/layer19_attention_weight_max": 46.344688415527344,
      "activations/layer19_attention_weight_min": -29.125125885009766,
      "activations/layer1_attention_weight_max": 17.115583419799805,
      "activations/layer1_attention_weight_min": -14.366530418395996,
      "activations/layer20_attention_weight_max": 37.40038299560547,
      "activations/layer20_attention_weight_min": -22.659318923950195,
      "activations/layer21_attention_weight_max": 32.198509216308594,
      "activations/layer21_attention_weight_min": -18.378799438476562,
      "activations/layer22_attention_weight_max": 50.15338897705078,
      "activations/layer22_attention_weight_min": -25.60490608215332,
      "activations/layer23_attention_weight_max": 29.879894256591797,
      "activations/layer23_attention_weight_min": -17.043777465820312,
      "activations/layer2_attention_weight_max": 33.06841278076172,
      "activations/layer2_attention_weight_min": -33.647300720214844,
      "activations/layer3_attention_weight_max": 91.08683013916016,
      "activations/layer3_attention_weight_min": -90.20140838623047,
      "activations/layer4_attention_weight_max": 140.02537536621094,
      "activations/layer4_attention_weight_min": -137.37081909179688,
      "activations/layer5_attention_weight_max": 103.79981994628906,
      "activations/layer5_attention_weight_min": -91.6488037109375,
      "activations/layer6_attention_weight_max": 61.87391662597656,
      "activations/layer6_attention_weight_min": -54.01625442504883,
      "activations/layer7_attention_weight_max": 90.9385757446289,
      "activations/layer7_attention_weight_min": -76.60651397705078,
      "activations/layer8_attention_weight_max": 47.537906646728516,
      "activations/layer8_attention_weight_min": -42.44511795043945,
      "activations/layer9_attention_weight_max": 55.0048713684082,
      "activations/layer9_attention_weight_min": -43.84907150268555,
      "epoch": 3.54,
      "learning_rate": 0.0001284337121212121,
      "loss": 2.881,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_loss": 2.83984375,
      "eval_runtime": 8.5388,
      "eval_samples_per_second": 502.88,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_openwebtext_loss": 2.83984375,
      "eval_openwebtext_ppl": 17.113091407702502,
      "eval_openwebtext_runtime": 8.5388,
      "eval_openwebtext_samples_per_second": 502.88,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_wikitext_loss": 3.087890625,
      "eval_wikitext_ppl": 21.930768935644906,
      "eval_wikitext_runtime": 2.0186,
      "eval_wikitext_samples_per_second": 225.894,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_lambada_loss": 2.794921875,
      "eval_lambada_ppl": 16.36135047271935,
      "eval_lambada_runtime": 9.6223,
      "eval_lambada_samples_per_second": 506.014,
      "step": 61000
    },
    {
      "activations/layer0_attention_weight_max": 16.99085807800293,
      "activations/layer0_attention_weight_min": -11.421048164367676,
      "activations/layer10_attention_weight_max": 36.209503173828125,
      "activations/layer10_attention_weight_min": -30.783620834350586,
      "activations/layer11_attention_weight_max": 33.87019729614258,
      "activations/layer11_attention_weight_min": -27.890846252441406,
      "activations/layer12_attention_weight_max": 27.063371658325195,
      "activations/layer12_attention_weight_min": -31.587156295776367,
      "activations/layer13_attention_weight_max": 42.1004638671875,
      "activations/layer13_attention_weight_min": -33.05656051635742,
      "activations/layer14_attention_weight_max": 66.35292053222656,
      "activations/layer14_attention_weight_min": -48.682003021240234,
      "activations/layer15_attention_weight_max": 45.67995071411133,
      "activations/layer15_attention_weight_min": -28.4643497467041,
      "activations/layer16_attention_weight_max": 39.933738708496094,
      "activations/layer16_attention_weight_min": -28.21727180480957,
      "activations/layer17_attention_weight_max": 42.551082611083984,
      "activations/layer17_attention_weight_min": -25.461828231811523,
      "activations/layer18_attention_weight_max": 33.18753433227539,
      "activations/layer18_attention_weight_min": -18.191192626953125,
      "activations/layer19_attention_weight_max": 39.84598922729492,
      "activations/layer19_attention_weight_min": -28.502538681030273,
      "activations/layer1_attention_weight_max": 17.381526947021484,
      "activations/layer1_attention_weight_min": -14.006525993347168,
      "activations/layer20_attention_weight_max": 35.237640380859375,
      "activations/layer20_attention_weight_min": -22.392894744873047,
      "activations/layer21_attention_weight_max": 33.800697326660156,
      "activations/layer21_attention_weight_min": -20.390661239624023,
      "activations/layer22_attention_weight_max": 49.71369934082031,
      "activations/layer22_attention_weight_min": -27.728595733642578,
      "activations/layer23_attention_weight_max": 28.30413055419922,
      "activations/layer23_attention_weight_min": -19.465940475463867,
      "activations/layer2_attention_weight_max": 30.561460494995117,
      "activations/layer2_attention_weight_min": -30.303878784179688,
      "activations/layer3_attention_weight_max": 83.10345458984375,
      "activations/layer3_attention_weight_min": -83.0356674194336,
      "activations/layer4_attention_weight_max": 127.47960662841797,
      "activations/layer4_attention_weight_min": -123.57540130615234,
      "activations/layer5_attention_weight_max": 92.38765716552734,
      "activations/layer5_attention_weight_min": -85.63878631591797,
      "activations/layer6_attention_weight_max": 57.62196731567383,
      "activations/layer6_attention_weight_min": -53.42705535888672,
      "activations/layer7_attention_weight_max": 77.5233154296875,
      "activations/layer7_attention_weight_min": -76.07176208496094,
      "activations/layer8_attention_weight_max": 36.65159606933594,
      "activations/layer8_attention_weight_min": -34.765220642089844,
      "activations/layer9_attention_weight_max": 40.530452728271484,
      "activations/layer9_attention_weight_min": -40.02706527709961,
      "epoch": 3.55,
      "learning_rate": 0.00012841477272727272,
      "loss": 2.8754,
      "step": 61050
    },
    {
      "activations/layer0_attention_weight_max": 15.287151336669922,
      "activations/layer0_attention_weight_min": -11.430777549743652,
      "activations/layer10_attention_weight_max": 34.31367111206055,
      "activations/layer10_attention_weight_min": -29.747344970703125,
      "activations/layer11_attention_weight_max": 31.305105209350586,
      "activations/layer11_attention_weight_min": -29.09144401550293,
      "activations/layer12_attention_weight_max": 31.495071411132812,
      "activations/layer12_attention_weight_min": -27.9167537689209,
      "activations/layer13_attention_weight_max": 47.40711975097656,
      "activations/layer13_attention_weight_min": -36.5936279296875,
      "activations/layer14_attention_weight_max": 63.11964416503906,
      "activations/layer14_attention_weight_min": -52.296024322509766,
      "activations/layer15_attention_weight_max": 42.018985748291016,
      "activations/layer15_attention_weight_min": -31.068605422973633,
      "activations/layer16_attention_weight_max": 42.95962905883789,
      "activations/layer16_attention_weight_min": -31.006872177124023,
      "activations/layer17_attention_weight_max": 42.54801559448242,
      "activations/layer17_attention_weight_min": -24.32164764404297,
      "activations/layer18_attention_weight_max": 28.736337661743164,
      "activations/layer18_attention_weight_min": -20.084022521972656,
      "activations/layer19_attention_weight_max": 46.009464263916016,
      "activations/layer19_attention_weight_min": -29.370059967041016,
      "activations/layer1_attention_weight_max": 18.091215133666992,
      "activations/layer1_attention_weight_min": -13.480621337890625,
      "activations/layer20_attention_weight_max": 39.76397705078125,
      "activations/layer20_attention_weight_min": -23.237567901611328,
      "activations/layer21_attention_weight_max": 29.20731544494629,
      "activations/layer21_attention_weight_min": -19.14497184753418,
      "activations/layer22_attention_weight_max": 48.643009185791016,
      "activations/layer22_attention_weight_min": -25.66448211669922,
      "activations/layer23_attention_weight_max": 28.37799644470215,
      "activations/layer23_attention_weight_min": -18.116397857666016,
      "activations/layer2_attention_weight_max": 30.18048095703125,
      "activations/layer2_attention_weight_min": -29.100017547607422,
      "activations/layer3_attention_weight_max": 84.6043701171875,
      "activations/layer3_attention_weight_min": -85.17024993896484,
      "activations/layer4_attention_weight_max": 134.2186737060547,
      "activations/layer4_attention_weight_min": -128.7247772216797,
      "activations/layer5_attention_weight_max": 95.50399780273438,
      "activations/layer5_attention_weight_min": -89.200439453125,
      "activations/layer6_attention_weight_max": 61.506996154785156,
      "activations/layer6_attention_weight_min": -61.916900634765625,
      "activations/layer7_attention_weight_max": 73.29927062988281,
      "activations/layer7_attention_weight_min": -80.04212188720703,
      "activations/layer8_attention_weight_max": 40.55339050292969,
      "activations/layer8_attention_weight_min": -39.33551025390625,
      "activations/layer9_attention_weight_max": 43.055782318115234,
      "activations/layer9_attention_weight_min": -36.12338638305664,
      "epoch": 3.55,
      "learning_rate": 0.00012839583333333333,
      "loss": 2.8919,
      "step": 61100
    },
    {
      "activations/layer0_attention_weight_max": 16.361953735351562,
      "activations/layer0_attention_weight_min": -11.780558586120605,
      "activations/layer10_attention_weight_max": 36.197540283203125,
      "activations/layer10_attention_weight_min": -30.306230545043945,
      "activations/layer11_attention_weight_max": 35.45269012451172,
      "activations/layer11_attention_weight_min": -26.382068634033203,
      "activations/layer12_attention_weight_max": 34.992034912109375,
      "activations/layer12_attention_weight_min": -27.21497344970703,
      "activations/layer13_attention_weight_max": 50.34724426269531,
      "activations/layer13_attention_weight_min": -35.26592254638672,
      "activations/layer14_attention_weight_max": 64.18778228759766,
      "activations/layer14_attention_weight_min": -48.74811935424805,
      "activations/layer15_attention_weight_max": 39.421470642089844,
      "activations/layer15_attention_weight_min": -28.273235321044922,
      "activations/layer16_attention_weight_max": 36.83051300048828,
      "activations/layer16_attention_weight_min": -26.849687576293945,
      "activations/layer17_attention_weight_max": 40.17778015136719,
      "activations/layer17_attention_weight_min": -25.58932876586914,
      "activations/layer18_attention_weight_max": 32.846492767333984,
      "activations/layer18_attention_weight_min": -19.477628707885742,
      "activations/layer19_attention_weight_max": 40.87543487548828,
      "activations/layer19_attention_weight_min": -26.773656845092773,
      "activations/layer1_attention_weight_max": 17.277658462524414,
      "activations/layer1_attention_weight_min": -13.842257499694824,
      "activations/layer20_attention_weight_max": 34.045040130615234,
      "activations/layer20_attention_weight_min": -21.042322158813477,
      "activations/layer21_attention_weight_max": 40.295047760009766,
      "activations/layer21_attention_weight_min": -20.99541664123535,
      "activations/layer22_attention_weight_max": 52.036521911621094,
      "activations/layer22_attention_weight_min": -29.01766586303711,
      "activations/layer23_attention_weight_max": 30.60013198852539,
      "activations/layer23_attention_weight_min": -19.670711517333984,
      "activations/layer2_attention_weight_max": 29.9915828704834,
      "activations/layer2_attention_weight_min": -30.07906723022461,
      "activations/layer3_attention_weight_max": 87.02848052978516,
      "activations/layer3_attention_weight_min": -87.82025146484375,
      "activations/layer4_attention_weight_max": 126.64388275146484,
      "activations/layer4_attention_weight_min": -125.81743621826172,
      "activations/layer5_attention_weight_max": 93.13280487060547,
      "activations/layer5_attention_weight_min": -84.13719177246094,
      "activations/layer6_attention_weight_max": 54.47761535644531,
      "activations/layer6_attention_weight_min": -55.076175689697266,
      "activations/layer7_attention_weight_max": 70.0429916381836,
      "activations/layer7_attention_weight_min": -71.2833023071289,
      "activations/layer8_attention_weight_max": 36.64864730834961,
      "activations/layer8_attention_weight_min": -36.08108139038086,
      "activations/layer9_attention_weight_max": 38.23056411743164,
      "activations/layer9_attention_weight_min": -31.7889347076416,
      "epoch": 3.55,
      "learning_rate": 0.00012837689393939392,
      "loss": 2.8871,
      "step": 61150
    },
    {
      "activations/layer0_attention_weight_max": 13.639810562133789,
      "activations/layer0_attention_weight_min": -11.207952499389648,
      "activations/layer10_attention_weight_max": 39.359596252441406,
      "activations/layer10_attention_weight_min": -33.81360626220703,
      "activations/layer11_attention_weight_max": 34.067569732666016,
      "activations/layer11_attention_weight_min": -29.12623405456543,
      "activations/layer12_attention_weight_max": 37.11593246459961,
      "activations/layer12_attention_weight_min": -36.46159744262695,
      "activations/layer13_attention_weight_max": 51.7690315246582,
      "activations/layer13_attention_weight_min": -35.6558952331543,
      "activations/layer14_attention_weight_max": 90.49470520019531,
      "activations/layer14_attention_weight_min": -62.288185119628906,
      "activations/layer15_attention_weight_max": 42.652984619140625,
      "activations/layer15_attention_weight_min": -29.663619995117188,
      "activations/layer16_attention_weight_max": 36.75585174560547,
      "activations/layer16_attention_weight_min": -28.43726921081543,
      "activations/layer17_attention_weight_max": 39.255027770996094,
      "activations/layer17_attention_weight_min": -26.288951873779297,
      "activations/layer18_attention_weight_max": 29.789377212524414,
      "activations/layer18_attention_weight_min": -18.74704933166504,
      "activations/layer19_attention_weight_max": 41.037879943847656,
      "activations/layer19_attention_weight_min": -29.543598175048828,
      "activations/layer1_attention_weight_max": 17.104782104492188,
      "activations/layer1_attention_weight_min": -14.68382453918457,
      "activations/layer20_attention_weight_max": 34.90950012207031,
      "activations/layer20_attention_weight_min": -22.49266815185547,
      "activations/layer21_attention_weight_max": 31.110759735107422,
      "activations/layer21_attention_weight_min": -19.288406372070312,
      "activations/layer22_attention_weight_max": 48.04576110839844,
      "activations/layer22_attention_weight_min": -27.67745590209961,
      "activations/layer23_attention_weight_max": 30.05133819580078,
      "activations/layer23_attention_weight_min": -20.006248474121094,
      "activations/layer2_attention_weight_max": 31.126384735107422,
      "activations/layer2_attention_weight_min": -31.110807418823242,
      "activations/layer3_attention_weight_max": 85.61249542236328,
      "activations/layer3_attention_weight_min": -86.14083099365234,
      "activations/layer4_attention_weight_max": 135.6455535888672,
      "activations/layer4_attention_weight_min": -129.93406677246094,
      "activations/layer5_attention_weight_max": 100.89122009277344,
      "activations/layer5_attention_weight_min": -93.99382781982422,
      "activations/layer6_attention_weight_max": 59.33367156982422,
      "activations/layer6_attention_weight_min": -57.90386962890625,
      "activations/layer7_attention_weight_max": 80.29899597167969,
      "activations/layer7_attention_weight_min": -73.70523834228516,
      "activations/layer8_attention_weight_max": 42.76494598388672,
      "activations/layer8_attention_weight_min": -43.06731414794922,
      "activations/layer9_attention_weight_max": 42.4129524230957,
      "activations/layer9_attention_weight_min": -37.852657318115234,
      "epoch": 3.56,
      "learning_rate": 0.00012835795454545454,
      "loss": 2.872,
      "step": 61200
    },
    {
      "activations/layer0_attention_weight_max": 16.60983657836914,
      "activations/layer0_attention_weight_min": -11.693183898925781,
      "activations/layer10_attention_weight_max": 36.98472595214844,
      "activations/layer10_attention_weight_min": -29.968332290649414,
      "activations/layer11_attention_weight_max": 33.19462966918945,
      "activations/layer11_attention_weight_min": -27.951087951660156,
      "activations/layer12_attention_weight_max": 47.873016357421875,
      "activations/layer12_attention_weight_min": -31.869773864746094,
      "activations/layer13_attention_weight_max": 46.127906799316406,
      "activations/layer13_attention_weight_min": -31.684728622436523,
      "activations/layer14_attention_weight_max": 79.44984436035156,
      "activations/layer14_attention_weight_min": -52.78025436401367,
      "activations/layer15_attention_weight_max": 44.333560943603516,
      "activations/layer15_attention_weight_min": -28.962865829467773,
      "activations/layer16_attention_weight_max": 39.469635009765625,
      "activations/layer16_attention_weight_min": -28.824312210083008,
      "activations/layer17_attention_weight_max": 42.41136169433594,
      "activations/layer17_attention_weight_min": -25.816097259521484,
      "activations/layer18_attention_weight_max": 30.484106063842773,
      "activations/layer18_attention_weight_min": -18.779508590698242,
      "activations/layer19_attention_weight_max": 45.341129302978516,
      "activations/layer19_attention_weight_min": -30.216232299804688,
      "activations/layer1_attention_weight_max": 18.37088966369629,
      "activations/layer1_attention_weight_min": -14.120421409606934,
      "activations/layer20_attention_weight_max": 37.999290466308594,
      "activations/layer20_attention_weight_min": -23.958538055419922,
      "activations/layer21_attention_weight_max": 35.742496490478516,
      "activations/layer21_attention_weight_min": -19.945077896118164,
      "activations/layer22_attention_weight_max": 53.183021545410156,
      "activations/layer22_attention_weight_min": -25.676971435546875,
      "activations/layer23_attention_weight_max": 27.032140731811523,
      "activations/layer23_attention_weight_min": -15.765542984008789,
      "activations/layer2_attention_weight_max": 30.330442428588867,
      "activations/layer2_attention_weight_min": -31.177488327026367,
      "activations/layer3_attention_weight_max": 83.62801361083984,
      "activations/layer3_attention_weight_min": -91.81858825683594,
      "activations/layer4_attention_weight_max": 129.17819213867188,
      "activations/layer4_attention_weight_min": -133.468505859375,
      "activations/layer5_attention_weight_max": 93.35736083984375,
      "activations/layer5_attention_weight_min": -91.00984954833984,
      "activations/layer6_attention_weight_max": 57.56154251098633,
      "activations/layer6_attention_weight_min": -55.88312530517578,
      "activations/layer7_attention_weight_max": 74.98977661132812,
      "activations/layer7_attention_weight_min": -79.72300720214844,
      "activations/layer8_attention_weight_max": 38.33384704589844,
      "activations/layer8_attention_weight_min": -37.924530029296875,
      "activations/layer9_attention_weight_max": 42.6880989074707,
      "activations/layer9_attention_weight_min": -33.05047607421875,
      "epoch": 3.56,
      "learning_rate": 0.00012833901515151515,
      "loss": 2.8915,
      "step": 61250
    },
    {
      "activations/layer0_attention_weight_max": 15.25656795501709,
      "activations/layer0_attention_weight_min": -11.557032585144043,
      "activations/layer10_attention_weight_max": 38.37119674682617,
      "activations/layer10_attention_weight_min": -32.36707305908203,
      "activations/layer11_attention_weight_max": 43.27601623535156,
      "activations/layer11_attention_weight_min": -31.298404693603516,
      "activations/layer12_attention_weight_max": 43.83808898925781,
      "activations/layer12_attention_weight_min": -28.591232299804688,
      "activations/layer13_attention_weight_max": 60.8355598449707,
      "activations/layer13_attention_weight_min": -41.588134765625,
      "activations/layer14_attention_weight_max": 92.75665283203125,
      "activations/layer14_attention_weight_min": -64.07115173339844,
      "activations/layer15_attention_weight_max": 52.619625091552734,
      "activations/layer15_attention_weight_min": -32.613040924072266,
      "activations/layer16_attention_weight_max": 46.84650802612305,
      "activations/layer16_attention_weight_min": -32.61703872680664,
      "activations/layer17_attention_weight_max": 42.745121002197266,
      "activations/layer17_attention_weight_min": -24.869571685791016,
      "activations/layer18_attention_weight_max": 33.43738555908203,
      "activations/layer18_attention_weight_min": -21.49850845336914,
      "activations/layer19_attention_weight_max": 44.85508346557617,
      "activations/layer19_attention_weight_min": -28.657384872436523,
      "activations/layer1_attention_weight_max": 17.529481887817383,
      "activations/layer1_attention_weight_min": -13.87855052947998,
      "activations/layer20_attention_weight_max": 33.88249206542969,
      "activations/layer20_attention_weight_min": -24.002134323120117,
      "activations/layer21_attention_weight_max": 31.00006103515625,
      "activations/layer21_attention_weight_min": -23.782712936401367,
      "activations/layer22_attention_weight_max": 48.388126373291016,
      "activations/layer22_attention_weight_min": -28.746017456054688,
      "activations/layer23_attention_weight_max": 32.282466888427734,
      "activations/layer23_attention_weight_min": -19.56385040283203,
      "activations/layer2_attention_weight_max": 30.399242401123047,
      "activations/layer2_attention_weight_min": -31.341026306152344,
      "activations/layer3_attention_weight_max": 88.39112091064453,
      "activations/layer3_attention_weight_min": -87.82711029052734,
      "activations/layer4_attention_weight_max": 132.75404357910156,
      "activations/layer4_attention_weight_min": -128.30052185058594,
      "activations/layer5_attention_weight_max": 96.7169418334961,
      "activations/layer5_attention_weight_min": -88.0816879272461,
      "activations/layer6_attention_weight_max": 57.72711944580078,
      "activations/layer6_attention_weight_min": -56.251678466796875,
      "activations/layer7_attention_weight_max": 74.51756286621094,
      "activations/layer7_attention_weight_min": -76.42428588867188,
      "activations/layer8_attention_weight_max": 39.04358673095703,
      "activations/layer8_attention_weight_min": -37.678138732910156,
      "activations/layer9_attention_weight_max": 46.461483001708984,
      "activations/layer9_attention_weight_min": -35.81351089477539,
      "epoch": 3.56,
      "learning_rate": 0.00012832045454545454,
      "loss": 2.8817,
      "step": 61300
    },
    {
      "activations/layer0_attention_weight_max": 14.65096378326416,
      "activations/layer0_attention_weight_min": -11.556721687316895,
      "activations/layer10_attention_weight_max": 39.36546325683594,
      "activations/layer10_attention_weight_min": -31.816709518432617,
      "activations/layer11_attention_weight_max": 35.702178955078125,
      "activations/layer11_attention_weight_min": -29.053497314453125,
      "activations/layer12_attention_weight_max": 51.70418167114258,
      "activations/layer12_attention_weight_min": -28.271432876586914,
      "activations/layer13_attention_weight_max": 49.195560455322266,
      "activations/layer13_attention_weight_min": -32.787105560302734,
      "activations/layer14_attention_weight_max": 86.53553771972656,
      "activations/layer14_attention_weight_min": -59.02992248535156,
      "activations/layer15_attention_weight_max": 46.74607467651367,
      "activations/layer15_attention_weight_min": -30.2311954498291,
      "activations/layer16_attention_weight_max": 41.5203971862793,
      "activations/layer16_attention_weight_min": -30.200239181518555,
      "activations/layer17_attention_weight_max": 41.714290618896484,
      "activations/layer17_attention_weight_min": -26.429006576538086,
      "activations/layer18_attention_weight_max": 34.38070297241211,
      "activations/layer18_attention_weight_min": -18.990310668945312,
      "activations/layer19_attention_weight_max": 50.494102478027344,
      "activations/layer19_attention_weight_min": -30.174726486206055,
      "activations/layer1_attention_weight_max": 20.052783966064453,
      "activations/layer1_attention_weight_min": -15.168471336364746,
      "activations/layer20_attention_weight_max": 38.444053649902344,
      "activations/layer20_attention_weight_min": -23.998241424560547,
      "activations/layer21_attention_weight_max": 33.756107330322266,
      "activations/layer21_attention_weight_min": -20.20818328857422,
      "activations/layer22_attention_weight_max": 49.60370635986328,
      "activations/layer22_attention_weight_min": -26.74247169494629,
      "activations/layer23_attention_weight_max": 31.874645233154297,
      "activations/layer23_attention_weight_min": -18.617305755615234,
      "activations/layer2_attention_weight_max": 31.877277374267578,
      "activations/layer2_attention_weight_min": -32.675819396972656,
      "activations/layer3_attention_weight_max": 86.16160583496094,
      "activations/layer3_attention_weight_min": -87.29952239990234,
      "activations/layer4_attention_weight_max": 134.0391845703125,
      "activations/layer4_attention_weight_min": -129.29981994628906,
      "activations/layer5_attention_weight_max": 98.33500671386719,
      "activations/layer5_attention_weight_min": -88.73834228515625,
      "activations/layer6_attention_weight_max": 59.9387092590332,
      "activations/layer6_attention_weight_min": -57.004051208496094,
      "activations/layer7_attention_weight_max": 78.89146423339844,
      "activations/layer7_attention_weight_min": -78.65081024169922,
      "activations/layer8_attention_weight_max": 41.98068618774414,
      "activations/layer8_attention_weight_min": -42.04567337036133,
      "activations/layer9_attention_weight_max": 43.125694274902344,
      "activations/layer9_attention_weight_min": -40.96896743774414,
      "epoch": 3.56,
      "learning_rate": 0.00012830151515151513,
      "loss": 2.8926,
      "step": 61350
    },
    {
      "activations/layer0_attention_weight_max": 16.77119255065918,
      "activations/layer0_attention_weight_min": -12.106698036193848,
      "activations/layer10_attention_weight_max": 47.269805908203125,
      "activations/layer10_attention_weight_min": -32.12260818481445,
      "activations/layer11_attention_weight_max": 49.22030258178711,
      "activations/layer11_attention_weight_min": -31.662124633789062,
      "activations/layer12_attention_weight_max": 50.33488845825195,
      "activations/layer12_attention_weight_min": -27.796545028686523,
      "activations/layer13_attention_weight_max": 72.17476654052734,
      "activations/layer13_attention_weight_min": -39.89772033691406,
      "activations/layer14_attention_weight_max": 109.05414581298828,
      "activations/layer14_attention_weight_min": -56.363834381103516,
      "activations/layer15_attention_weight_max": 52.026920318603516,
      "activations/layer15_attention_weight_min": -30.03523063659668,
      "activations/layer16_attention_weight_max": 49.37714385986328,
      "activations/layer16_attention_weight_min": -31.606887817382812,
      "activations/layer17_attention_weight_max": 43.493995666503906,
      "activations/layer17_attention_weight_min": -24.2383975982666,
      "activations/layer18_attention_weight_max": 32.99138641357422,
      "activations/layer18_attention_weight_min": -22.831357955932617,
      "activations/layer19_attention_weight_max": 49.89308166503906,
      "activations/layer19_attention_weight_min": -29.891345977783203,
      "activations/layer1_attention_weight_max": 18.751672744750977,
      "activations/layer1_attention_weight_min": -15.267450332641602,
      "activations/layer20_attention_weight_max": 37.00401306152344,
      "activations/layer20_attention_weight_min": -21.857065200805664,
      "activations/layer21_attention_weight_max": 31.55703353881836,
      "activations/layer21_attention_weight_min": -19.22348976135254,
      "activations/layer22_attention_weight_max": 50.71677017211914,
      "activations/layer22_attention_weight_min": -26.58283233642578,
      "activations/layer23_attention_weight_max": 27.666595458984375,
      "activations/layer23_attention_weight_min": -19.243608474731445,
      "activations/layer2_attention_weight_max": 32.511234283447266,
      "activations/layer2_attention_weight_min": -30.67251968383789,
      "activations/layer3_attention_weight_max": 87.72447967529297,
      "activations/layer3_attention_weight_min": -86.42430114746094,
      "activations/layer4_attention_weight_max": 145.03646850585938,
      "activations/layer4_attention_weight_min": -129.37579345703125,
      "activations/layer5_attention_weight_max": 110.02676391601562,
      "activations/layer5_attention_weight_min": -90.78792572021484,
      "activations/layer6_attention_weight_max": 65.68708801269531,
      "activations/layer6_attention_weight_min": -55.56855010986328,
      "activations/layer7_attention_weight_max": 88.48953247070312,
      "activations/layer7_attention_weight_min": -80.72931671142578,
      "activations/layer8_attention_weight_max": 49.419246673583984,
      "activations/layer8_attention_weight_min": -37.89928436279297,
      "activations/layer9_attention_weight_max": 55.5833740234375,
      "activations/layer9_attention_weight_min": -42.310794830322266,
      "epoch": 3.57,
      "learning_rate": 0.00012828257575757575,
      "loss": 2.8971,
      "step": 61400
    },
    {
      "activations/layer0_attention_weight_max": 16.790069580078125,
      "activations/layer0_attention_weight_min": -10.772711753845215,
      "activations/layer10_attention_weight_max": 34.15088653564453,
      "activations/layer10_attention_weight_min": -30.13112449645996,
      "activations/layer11_attention_weight_max": 36.19027328491211,
      "activations/layer11_attention_weight_min": -28.089031219482422,
      "activations/layer12_attention_weight_max": 28.180055618286133,
      "activations/layer12_attention_weight_min": -28.118494033813477,
      "activations/layer13_attention_weight_max": 49.13212585449219,
      "activations/layer13_attention_weight_min": -35.47405242919922,
      "activations/layer14_attention_weight_max": 62.9566650390625,
      "activations/layer14_attention_weight_min": -49.07514953613281,
      "activations/layer15_attention_weight_max": 44.4473991394043,
      "activations/layer15_attention_weight_min": -31.713329315185547,
      "activations/layer16_attention_weight_max": 37.93101119995117,
      "activations/layer16_attention_weight_min": -29.91463851928711,
      "activations/layer17_attention_weight_max": 39.97425079345703,
      "activations/layer17_attention_weight_min": -26.329866409301758,
      "activations/layer18_attention_weight_max": 29.13215446472168,
      "activations/layer18_attention_weight_min": -20.392732620239258,
      "activations/layer19_attention_weight_max": 39.78961944580078,
      "activations/layer19_attention_weight_min": -30.029714584350586,
      "activations/layer1_attention_weight_max": 17.7358455657959,
      "activations/layer1_attention_weight_min": -14.07305908203125,
      "activations/layer20_attention_weight_max": 32.67554473876953,
      "activations/layer20_attention_weight_min": -24.110069274902344,
      "activations/layer21_attention_weight_max": 29.231277465820312,
      "activations/layer21_attention_weight_min": -20.161039352416992,
      "activations/layer22_attention_weight_max": 47.788936614990234,
      "activations/layer22_attention_weight_min": -29.601078033447266,
      "activations/layer23_attention_weight_max": 26.63488006591797,
      "activations/layer23_attention_weight_min": -18.31138038635254,
      "activations/layer2_attention_weight_max": 31.622739791870117,
      "activations/layer2_attention_weight_min": -31.085477828979492,
      "activations/layer3_attention_weight_max": 88.04232788085938,
      "activations/layer3_attention_weight_min": -83.54541778564453,
      "activations/layer4_attention_weight_max": 132.8610076904297,
      "activations/layer4_attention_weight_min": -130.27505493164062,
      "activations/layer5_attention_weight_max": 97.89449310302734,
      "activations/layer5_attention_weight_min": -89.28996276855469,
      "activations/layer6_attention_weight_max": 58.91071319580078,
      "activations/layer6_attention_weight_min": -55.02378463745117,
      "activations/layer7_attention_weight_max": 78.18169403076172,
      "activations/layer7_attention_weight_min": -72.12044525146484,
      "activations/layer8_attention_weight_max": 40.43886184692383,
      "activations/layer8_attention_weight_min": -35.85723876953125,
      "activations/layer9_attention_weight_max": 48.55950164794922,
      "activations/layer9_attention_weight_min": -39.691463470458984,
      "epoch": 3.57,
      "learning_rate": 0.00012826363636363636,
      "loss": 2.8835,
      "step": 61450
    },
    {
      "activations/layer0_attention_weight_max": 17.097200393676758,
      "activations/layer0_attention_weight_min": -11.736165046691895,
      "activations/layer10_attention_weight_max": 41.03535842895508,
      "activations/layer10_attention_weight_min": -31.502695083618164,
      "activations/layer11_attention_weight_max": 38.85649108886719,
      "activations/layer11_attention_weight_min": -30.32236099243164,
      "activations/layer12_attention_weight_max": 37.16509246826172,
      "activations/layer12_attention_weight_min": -25.533910751342773,
      "activations/layer13_attention_weight_max": 49.50999450683594,
      "activations/layer13_attention_weight_min": -33.448726654052734,
      "activations/layer14_attention_weight_max": 73.23460388183594,
      "activations/layer14_attention_weight_min": -47.89419937133789,
      "activations/layer15_attention_weight_max": 53.874359130859375,
      "activations/layer15_attention_weight_min": -31.921260833740234,
      "activations/layer16_attention_weight_max": 51.15932083129883,
      "activations/layer16_attention_weight_min": -30.32036590576172,
      "activations/layer17_attention_weight_max": 46.283199310302734,
      "activations/layer17_attention_weight_min": -26.017864227294922,
      "activations/layer18_attention_weight_max": 35.807674407958984,
      "activations/layer18_attention_weight_min": -18.312116622924805,
      "activations/layer19_attention_weight_max": 48.66864776611328,
      "activations/layer19_attention_weight_min": -27.265989303588867,
      "activations/layer1_attention_weight_max": 17.123281478881836,
      "activations/layer1_attention_weight_min": -15.735347747802734,
      "activations/layer20_attention_weight_max": 36.10960006713867,
      "activations/layer20_attention_weight_min": -23.31707763671875,
      "activations/layer21_attention_weight_max": 38.93153381347656,
      "activations/layer21_attention_weight_min": -19.507461547851562,
      "activations/layer22_attention_weight_max": 51.45380401611328,
      "activations/layer22_attention_weight_min": -27.55878257751465,
      "activations/layer23_attention_weight_max": 29.30327606201172,
      "activations/layer23_attention_weight_min": -16.68916130065918,
      "activations/layer2_attention_weight_max": 31.502735137939453,
      "activations/layer2_attention_weight_min": -30.414060592651367,
      "activations/layer3_attention_weight_max": 84.20735931396484,
      "activations/layer3_attention_weight_min": -85.45314025878906,
      "activations/layer4_attention_weight_max": 131.8714141845703,
      "activations/layer4_attention_weight_min": -129.71800231933594,
      "activations/layer5_attention_weight_max": 96.86279296875,
      "activations/layer5_attention_weight_min": -90.59716796875,
      "activations/layer6_attention_weight_max": 60.51341247558594,
      "activations/layer6_attention_weight_min": -55.08505630493164,
      "activations/layer7_attention_weight_max": 78.93745422363281,
      "activations/layer7_attention_weight_min": -77.78705596923828,
      "activations/layer8_attention_weight_max": 42.128746032714844,
      "activations/layer8_attention_weight_min": -38.551307678222656,
      "activations/layer9_attention_weight_max": 43.35313034057617,
      "activations/layer9_attention_weight_min": -36.707664489746094,
      "epoch": 3.57,
      "learning_rate": 0.00012824469696969695,
      "loss": 2.8956,
      "step": 61500
    },
    {
      "activations/layer0_attention_weight_max": 15.867231369018555,
      "activations/layer0_attention_weight_min": -10.887297630310059,
      "activations/layer10_attention_weight_max": 39.2513313293457,
      "activations/layer10_attention_weight_min": -31.50126838684082,
      "activations/layer11_attention_weight_max": 36.124046325683594,
      "activations/layer11_attention_weight_min": -28.445297241210938,
      "activations/layer12_attention_weight_max": 29.76789665222168,
      "activations/layer12_attention_weight_min": -25.174375534057617,
      "activations/layer13_attention_weight_max": 47.83315658569336,
      "activations/layer13_attention_weight_min": -36.58184051513672,
      "activations/layer14_attention_weight_max": 81.36991882324219,
      "activations/layer14_attention_weight_min": -57.57651138305664,
      "activations/layer15_attention_weight_max": 51.49568176269531,
      "activations/layer15_attention_weight_min": -32.132713317871094,
      "activations/layer16_attention_weight_max": 47.34764862060547,
      "activations/layer16_attention_weight_min": -34.2559814453125,
      "activations/layer17_attention_weight_max": 48.632240295410156,
      "activations/layer17_attention_weight_min": -28.321853637695312,
      "activations/layer18_attention_weight_max": 34.52378463745117,
      "activations/layer18_attention_weight_min": -18.41600227355957,
      "activations/layer19_attention_weight_max": 44.66324996948242,
      "activations/layer19_attention_weight_min": -30.725662231445312,
      "activations/layer1_attention_weight_max": 17.383440017700195,
      "activations/layer1_attention_weight_min": -17.374631881713867,
      "activations/layer20_attention_weight_max": 34.398033142089844,
      "activations/layer20_attention_weight_min": -23.75670051574707,
      "activations/layer21_attention_weight_max": 45.3845100402832,
      "activations/layer21_attention_weight_min": -22.807096481323242,
      "activations/layer22_attention_weight_max": 55.18006134033203,
      "activations/layer22_attention_weight_min": -30.687641143798828,
      "activations/layer23_attention_weight_max": 31.31280517578125,
      "activations/layer23_attention_weight_min": -18.339094161987305,
      "activations/layer2_attention_weight_max": 31.56496810913086,
      "activations/layer2_attention_weight_min": -31.761173248291016,
      "activations/layer3_attention_weight_max": 84.55947875976562,
      "activations/layer3_attention_weight_min": -89.05253601074219,
      "activations/layer4_attention_weight_max": 136.2512664794922,
      "activations/layer4_attention_weight_min": -130.1772003173828,
      "activations/layer5_attention_weight_max": 96.5272216796875,
      "activations/layer5_attention_weight_min": -91.99388122558594,
      "activations/layer6_attention_weight_max": 60.77180480957031,
      "activations/layer6_attention_weight_min": -56.12097930908203,
      "activations/layer7_attention_weight_max": 75.72589111328125,
      "activations/layer7_attention_weight_min": -75.73210144042969,
      "activations/layer8_attention_weight_max": 40.61647033691406,
      "activations/layer8_attention_weight_min": -40.4864387512207,
      "activations/layer9_attention_weight_max": 49.88607406616211,
      "activations/layer9_attention_weight_min": -32.24691390991211,
      "epoch": 3.58,
      "learning_rate": 0.00012822575757575756,
      "loss": 2.9085,
      "step": 61550
    },
    {
      "activations/layer0_attention_weight_max": 15.319108009338379,
      "activations/layer0_attention_weight_min": -12.074679374694824,
      "activations/layer10_attention_weight_max": 45.366294860839844,
      "activations/layer10_attention_weight_min": -31.588823318481445,
      "activations/layer11_attention_weight_max": 36.4657096862793,
      "activations/layer11_attention_weight_min": -26.952301025390625,
      "activations/layer12_attention_weight_max": 46.053382873535156,
      "activations/layer12_attention_weight_min": -29.062213897705078,
      "activations/layer13_attention_weight_max": 51.21427536010742,
      "activations/layer13_attention_weight_min": -33.413421630859375,
      "activations/layer14_attention_weight_max": 73.65301513671875,
      "activations/layer14_attention_weight_min": -48.707420349121094,
      "activations/layer15_attention_weight_max": 48.84123229980469,
      "activations/layer15_attention_weight_min": -31.39418601989746,
      "activations/layer16_attention_weight_max": 41.979148864746094,
      "activations/layer16_attention_weight_min": -31.65744972229004,
      "activations/layer17_attention_weight_max": 47.0052375793457,
      "activations/layer17_attention_weight_min": -26.521875381469727,
      "activations/layer18_attention_weight_max": 33.11437225341797,
      "activations/layer18_attention_weight_min": -17.744665145874023,
      "activations/layer19_attention_weight_max": 49.03779220581055,
      "activations/layer19_attention_weight_min": -29.13840675354004,
      "activations/layer1_attention_weight_max": 18.149188995361328,
      "activations/layer1_attention_weight_min": -14.168445587158203,
      "activations/layer20_attention_weight_max": 37.33134460449219,
      "activations/layer20_attention_weight_min": -24.291616439819336,
      "activations/layer21_attention_weight_max": 29.915128707885742,
      "activations/layer21_attention_weight_min": -19.542953491210938,
      "activations/layer22_attention_weight_max": 51.17182159423828,
      "activations/layer22_attention_weight_min": -28.05069923400879,
      "activations/layer23_attention_weight_max": 30.47401237487793,
      "activations/layer23_attention_weight_min": -16.748004913330078,
      "activations/layer2_attention_weight_max": 30.053939819335938,
      "activations/layer2_attention_weight_min": -30.468101501464844,
      "activations/layer3_attention_weight_max": 88.09843444824219,
      "activations/layer3_attention_weight_min": -89.8462142944336,
      "activations/layer4_attention_weight_max": 137.66915893554688,
      "activations/layer4_attention_weight_min": -130.9578857421875,
      "activations/layer5_attention_weight_max": 105.37960815429688,
      "activations/layer5_attention_weight_min": -95.03353118896484,
      "activations/layer6_attention_weight_max": 60.16514587402344,
      "activations/layer6_attention_weight_min": -57.76372146606445,
      "activations/layer7_attention_weight_max": 81.28108215332031,
      "activations/layer7_attention_weight_min": -77.34233093261719,
      "activations/layer8_attention_weight_max": 45.13611602783203,
      "activations/layer8_attention_weight_min": -45.437538146972656,
      "activations/layer9_attention_weight_max": 45.090816497802734,
      "activations/layer9_attention_weight_min": -36.35667419433594,
      "epoch": 3.58,
      "learning_rate": 0.00012820681818181818,
      "loss": 2.8958,
      "step": 61600
    },
    {
      "activations/layer0_attention_weight_max": 16.54812240600586,
      "activations/layer0_attention_weight_min": -11.326056480407715,
      "activations/layer10_attention_weight_max": 37.276058197021484,
      "activations/layer10_attention_weight_min": -30.542699813842773,
      "activations/layer11_attention_weight_max": 36.68902587890625,
      "activations/layer11_attention_weight_min": -31.887989044189453,
      "activations/layer12_attention_weight_max": 38.207252502441406,
      "activations/layer12_attention_weight_min": -28.351295471191406,
      "activations/layer13_attention_weight_max": 70.08704376220703,
      "activations/layer13_attention_weight_min": -48.29609680175781,
      "activations/layer14_attention_weight_max": 80.8692398071289,
      "activations/layer14_attention_weight_min": -55.93359375,
      "activations/layer15_attention_weight_max": 42.72492599487305,
      "activations/layer15_attention_weight_min": -28.13824462890625,
      "activations/layer16_attention_weight_max": 42.43045425415039,
      "activations/layer16_attention_weight_min": -30.43297004699707,
      "activations/layer17_attention_weight_max": 38.36958694458008,
      "activations/layer17_attention_weight_min": -28.505605697631836,
      "activations/layer18_attention_weight_max": 30.98753547668457,
      "activations/layer18_attention_weight_min": -20.166234970092773,
      "activations/layer19_attention_weight_max": 43.67760467529297,
      "activations/layer19_attention_weight_min": -31.141986846923828,
      "activations/layer1_attention_weight_max": 17.824039459228516,
      "activations/layer1_attention_weight_min": -14.939339637756348,
      "activations/layer20_attention_weight_max": 32.04736328125,
      "activations/layer20_attention_weight_min": -21.855928421020508,
      "activations/layer21_attention_weight_max": 33.04023361206055,
      "activations/layer21_attention_weight_min": -20.248191833496094,
      "activations/layer22_attention_weight_max": 47.428550720214844,
      "activations/layer22_attention_weight_min": -27.268022537231445,
      "activations/layer23_attention_weight_max": 30.860027313232422,
      "activations/layer23_attention_weight_min": -19.0379638671875,
      "activations/layer2_attention_weight_max": 32.73307418823242,
      "activations/layer2_attention_weight_min": -31.436071395874023,
      "activations/layer3_attention_weight_max": 82.97687530517578,
      "activations/layer3_attention_weight_min": -89.18621826171875,
      "activations/layer4_attention_weight_max": 137.52224731445312,
      "activations/layer4_attention_weight_min": -130.86959838867188,
      "activations/layer5_attention_weight_max": 98.95156860351562,
      "activations/layer5_attention_weight_min": -90.83189392089844,
      "activations/layer6_attention_weight_max": 58.02904510498047,
      "activations/layer6_attention_weight_min": -55.22773361206055,
      "activations/layer7_attention_weight_max": 74.44123077392578,
      "activations/layer7_attention_weight_min": -83.43819427490234,
      "activations/layer8_attention_weight_max": 39.758182525634766,
      "activations/layer8_attention_weight_min": -38.3870849609375,
      "activations/layer9_attention_weight_max": 53.8009147644043,
      "activations/layer9_attention_weight_min": -44.0352897644043,
      "epoch": 3.58,
      "learning_rate": 0.00012818787878787877,
      "loss": 2.8741,
      "step": 61650
    },
    {
      "activations/layer0_attention_weight_max": 15.757254600524902,
      "activations/layer0_attention_weight_min": -10.746761322021484,
      "activations/layer10_attention_weight_max": 48.87458419799805,
      "activations/layer10_attention_weight_min": -32.20960235595703,
      "activations/layer11_attention_weight_max": 45.39508819580078,
      "activations/layer11_attention_weight_min": -28.84725570678711,
      "activations/layer12_attention_weight_max": 56.52055358886719,
      "activations/layer12_attention_weight_min": -26.415164947509766,
      "activations/layer13_attention_weight_max": 80.45121765136719,
      "activations/layer13_attention_weight_min": -40.42790603637695,
      "activations/layer14_attention_weight_max": 78.43555450439453,
      "activations/layer14_attention_weight_min": -50.2774658203125,
      "activations/layer15_attention_weight_max": 52.213584899902344,
      "activations/layer15_attention_weight_min": -31.62567138671875,
      "activations/layer16_attention_weight_max": 44.48100662231445,
      "activations/layer16_attention_weight_min": -29.266725540161133,
      "activations/layer17_attention_weight_max": 43.06266403198242,
      "activations/layer17_attention_weight_min": -25.406579971313477,
      "activations/layer18_attention_weight_max": 32.818878173828125,
      "activations/layer18_attention_weight_min": -19.6750431060791,
      "activations/layer19_attention_weight_max": 46.44240951538086,
      "activations/layer19_attention_weight_min": -28.279386520385742,
      "activations/layer1_attention_weight_max": 17.79143524169922,
      "activations/layer1_attention_weight_min": -14.131022453308105,
      "activations/layer20_attention_weight_max": 35.01279830932617,
      "activations/layer20_attention_weight_min": -23.314151763916016,
      "activations/layer21_attention_weight_max": 41.53000259399414,
      "activations/layer21_attention_weight_min": -20.98569679260254,
      "activations/layer22_attention_weight_max": 51.45825958251953,
      "activations/layer22_attention_weight_min": -27.31940460205078,
      "activations/layer23_attention_weight_max": 27.460411071777344,
      "activations/layer23_attention_weight_min": -17.568973541259766,
      "activations/layer2_attention_weight_max": 33.16960906982422,
      "activations/layer2_attention_weight_min": -32.54560089111328,
      "activations/layer3_attention_weight_max": 91.78836059570312,
      "activations/layer3_attention_weight_min": -89.74697875976562,
      "activations/layer4_attention_weight_max": 147.30186462402344,
      "activations/layer4_attention_weight_min": -134.5547332763672,
      "activations/layer5_attention_weight_max": 106.65142822265625,
      "activations/layer5_attention_weight_min": -95.48851013183594,
      "activations/layer6_attention_weight_max": 66.39058685302734,
      "activations/layer6_attention_weight_min": -62.18891525268555,
      "activations/layer7_attention_weight_max": 91.51907348632812,
      "activations/layer7_attention_weight_min": -79.78173828125,
      "activations/layer8_attention_weight_max": 47.09462356567383,
      "activations/layer8_attention_weight_min": -44.63652038574219,
      "activations/layer9_attention_weight_max": 60.2382926940918,
      "activations/layer9_attention_weight_min": -44.02526092529297,
      "epoch": 3.59,
      "learning_rate": 0.00012816893939393938,
      "loss": 2.8856,
      "step": 61700
    },
    {
      "activations/layer0_attention_weight_max": 16.280593872070312,
      "activations/layer0_attention_weight_min": -11.726519584655762,
      "activations/layer10_attention_weight_max": 41.54966354370117,
      "activations/layer10_attention_weight_min": -30.480566024780273,
      "activations/layer11_attention_weight_max": 39.34843063354492,
      "activations/layer11_attention_weight_min": -28.898780822753906,
      "activations/layer12_attention_weight_max": 70.82102966308594,
      "activations/layer12_attention_weight_min": -33.71247100830078,
      "activations/layer13_attention_weight_max": 79.03878021240234,
      "activations/layer13_attention_weight_min": -44.07823181152344,
      "activations/layer14_attention_weight_max": 82.35429382324219,
      "activations/layer14_attention_weight_min": -56.28449249267578,
      "activations/layer15_attention_weight_max": 63.375728607177734,
      "activations/layer15_attention_weight_min": -32.30509567260742,
      "activations/layer16_attention_weight_max": 42.315792083740234,
      "activations/layer16_attention_weight_min": -30.20851707458496,
      "activations/layer17_attention_weight_max": 40.43753433227539,
      "activations/layer17_attention_weight_min": -27.881074905395508,
      "activations/layer18_attention_weight_max": 34.17818832397461,
      "activations/layer18_attention_weight_min": -21.043073654174805,
      "activations/layer19_attention_weight_max": 48.26980972290039,
      "activations/layer19_attention_weight_min": -29.879858016967773,
      "activations/layer1_attention_weight_max": 18.31124496459961,
      "activations/layer1_attention_weight_min": -12.66423511505127,
      "activations/layer20_attention_weight_max": 34.64633560180664,
      "activations/layer20_attention_weight_min": -24.619483947753906,
      "activations/layer21_attention_weight_max": 33.759822845458984,
      "activations/layer21_attention_weight_min": -19.068883895874023,
      "activations/layer22_attention_weight_max": 51.92707824707031,
      "activations/layer22_attention_weight_min": -30.678485870361328,
      "activations/layer23_attention_weight_max": 31.52261734008789,
      "activations/layer23_attention_weight_min": -20.219511032104492,
      "activations/layer2_attention_weight_max": 31.858243942260742,
      "activations/layer2_attention_weight_min": -32.37445831298828,
      "activations/layer3_attention_weight_max": 86.16021728515625,
      "activations/layer3_attention_weight_min": -86.7335433959961,
      "activations/layer4_attention_weight_max": 144.33914184570312,
      "activations/layer4_attention_weight_min": -133.98651123046875,
      "activations/layer5_attention_weight_max": 103.30023193359375,
      "activations/layer5_attention_weight_min": -90.37383270263672,
      "activations/layer6_attention_weight_max": 59.812355041503906,
      "activations/layer6_attention_weight_min": -54.456417083740234,
      "activations/layer7_attention_weight_max": 87.21712493896484,
      "activations/layer7_attention_weight_min": -80.46751403808594,
      "activations/layer8_attention_weight_max": 40.93910217285156,
      "activations/layer8_attention_weight_min": -36.30632400512695,
      "activations/layer9_attention_weight_max": 48.157169342041016,
      "activations/layer9_attention_weight_min": -42.232112884521484,
      "epoch": 3.59,
      "learning_rate": 0.00012814999999999997,
      "loss": 2.8746,
      "step": 61750
    },
    {
      "activations/layer0_attention_weight_max": 13.888803482055664,
      "activations/layer0_attention_weight_min": -12.254189491271973,
      "activations/layer10_attention_weight_max": 40.2745246887207,
      "activations/layer10_attention_weight_min": -32.37273025512695,
      "activations/layer11_attention_weight_max": 35.3031005859375,
      "activations/layer11_attention_weight_min": -27.055709838867188,
      "activations/layer12_attention_weight_max": 32.1590576171875,
      "activations/layer12_attention_weight_min": -23.544673919677734,
      "activations/layer13_attention_weight_max": 51.24827194213867,
      "activations/layer13_attention_weight_min": -34.78389358520508,
      "activations/layer14_attention_weight_max": 66.86860656738281,
      "activations/layer14_attention_weight_min": -48.03482437133789,
      "activations/layer15_attention_weight_max": 52.379234313964844,
      "activations/layer15_attention_weight_min": -32.87739562988281,
      "activations/layer16_attention_weight_max": 42.683982849121094,
      "activations/layer16_attention_weight_min": -30.19097900390625,
      "activations/layer17_attention_weight_max": 47.278926849365234,
      "activations/layer17_attention_weight_min": -26.82977867126465,
      "activations/layer18_attention_weight_max": 34.46184539794922,
      "activations/layer18_attention_weight_min": -19.51000213623047,
      "activations/layer19_attention_weight_max": 46.63703918457031,
      "activations/layer19_attention_weight_min": -30.154743194580078,
      "activations/layer1_attention_weight_max": 16.731487274169922,
      "activations/layer1_attention_weight_min": -14.03664779663086,
      "activations/layer20_attention_weight_max": 38.694149017333984,
      "activations/layer20_attention_weight_min": -26.189163208007812,
      "activations/layer21_attention_weight_max": 31.499839782714844,
      "activations/layer21_attention_weight_min": -21.930572509765625,
      "activations/layer22_attention_weight_max": 53.44296646118164,
      "activations/layer22_attention_weight_min": -29.942922592163086,
      "activations/layer23_attention_weight_max": 32.548744201660156,
      "activations/layer23_attention_weight_min": -16.69252586364746,
      "activations/layer2_attention_weight_max": 32.28882598876953,
      "activations/layer2_attention_weight_min": -30.264881134033203,
      "activations/layer3_attention_weight_max": 80.93006134033203,
      "activations/layer3_attention_weight_min": -84.00457000732422,
      "activations/layer4_attention_weight_max": 129.86241149902344,
      "activations/layer4_attention_weight_min": -125.32173919677734,
      "activations/layer5_attention_weight_max": 94.74607849121094,
      "activations/layer5_attention_weight_min": -86.929443359375,
      "activations/layer6_attention_weight_max": 59.13493728637695,
      "activations/layer6_attention_weight_min": -57.09119415283203,
      "activations/layer7_attention_weight_max": 79.54519653320312,
      "activations/layer7_attention_weight_min": -74.79190063476562,
      "activations/layer8_attention_weight_max": 38.81588363647461,
      "activations/layer8_attention_weight_min": -38.932918548583984,
      "activations/layer9_attention_weight_max": 40.01264572143555,
      "activations/layer9_attention_weight_min": -34.44213104248047,
      "epoch": 3.59,
      "learning_rate": 0.00012813106060606059,
      "loss": 2.9065,
      "step": 61800
    },
    {
      "activations/layer0_attention_weight_max": 14.781103134155273,
      "activations/layer0_attention_weight_min": -11.165045738220215,
      "activations/layer10_attention_weight_max": 37.87944412231445,
      "activations/layer10_attention_weight_min": -33.49665069580078,
      "activations/layer11_attention_weight_max": 33.90983581542969,
      "activations/layer11_attention_weight_min": -29.178035736083984,
      "activations/layer12_attention_weight_max": 29.077865600585938,
      "activations/layer12_attention_weight_min": -31.563899993896484,
      "activations/layer13_attention_weight_max": 48.173927307128906,
      "activations/layer13_attention_weight_min": -36.076454162597656,
      "activations/layer14_attention_weight_max": 66.70977783203125,
      "activations/layer14_attention_weight_min": -53.48295211791992,
      "activations/layer15_attention_weight_max": 47.11076354980469,
      "activations/layer15_attention_weight_min": -31.539854049682617,
      "activations/layer16_attention_weight_max": 44.34219741821289,
      "activations/layer16_attention_weight_min": -30.63191795349121,
      "activations/layer17_attention_weight_max": 48.983558654785156,
      "activations/layer17_attention_weight_min": -25.89328384399414,
      "activations/layer18_attention_weight_max": 31.692363739013672,
      "activations/layer18_attention_weight_min": -17.928733825683594,
      "activations/layer19_attention_weight_max": 52.10554122924805,
      "activations/layer19_attention_weight_min": -30.208383560180664,
      "activations/layer1_attention_weight_max": 17.866426467895508,
      "activations/layer1_attention_weight_min": -12.840736389160156,
      "activations/layer20_attention_weight_max": 37.48357391357422,
      "activations/layer20_attention_weight_min": -23.895793914794922,
      "activations/layer21_attention_weight_max": 39.5465202331543,
      "activations/layer21_attention_weight_min": -20.562740325927734,
      "activations/layer22_attention_weight_max": 57.7244758605957,
      "activations/layer22_attention_weight_min": -28.903791427612305,
      "activations/layer23_attention_weight_max": 37.51774215698242,
      "activations/layer23_attention_weight_min": -17.627431869506836,
      "activations/layer2_attention_weight_max": 31.240575790405273,
      "activations/layer2_attention_weight_min": -31.044723510742188,
      "activations/layer3_attention_weight_max": 84.2606430053711,
      "activations/layer3_attention_weight_min": -86.14118194580078,
      "activations/layer4_attention_weight_max": 132.18582153320312,
      "activations/layer4_attention_weight_min": -128.20713806152344,
      "activations/layer5_attention_weight_max": 98.66386413574219,
      "activations/layer5_attention_weight_min": -90.99641418457031,
      "activations/layer6_attention_weight_max": 57.76653289794922,
      "activations/layer6_attention_weight_min": -57.97719192504883,
      "activations/layer7_attention_weight_max": 77.72099304199219,
      "activations/layer7_attention_weight_min": -77.51443481445312,
      "activations/layer8_attention_weight_max": 38.7835693359375,
      "activations/layer8_attention_weight_min": -37.75870895385742,
      "activations/layer9_attention_weight_max": 39.142860412597656,
      "activations/layer9_attention_weight_min": -40.36623001098633,
      "epoch": 3.59,
      "learning_rate": 0.0001281121212121212,
      "loss": 2.8759,
      "step": 61850
    },
    {
      "activations/layer0_attention_weight_max": 16.544414520263672,
      "activations/layer0_attention_weight_min": -10.795650482177734,
      "activations/layer10_attention_weight_max": 35.7135009765625,
      "activations/layer10_attention_weight_min": -32.07923889160156,
      "activations/layer11_attention_weight_max": 39.32414245605469,
      "activations/layer11_attention_weight_min": -28.676023483276367,
      "activations/layer12_attention_weight_max": 74.9328384399414,
      "activations/layer12_attention_weight_min": -30.08167266845703,
      "activations/layer13_attention_weight_max": 54.31364440917969,
      "activations/layer13_attention_weight_min": -33.14396286010742,
      "activations/layer14_attention_weight_max": 72.67357635498047,
      "activations/layer14_attention_weight_min": -48.00197219848633,
      "activations/layer15_attention_weight_max": 47.90291213989258,
      "activations/layer15_attention_weight_min": -31.495147705078125,
      "activations/layer16_attention_weight_max": 47.356773376464844,
      "activations/layer16_attention_weight_min": -29.028703689575195,
      "activations/layer17_attention_weight_max": 42.05131149291992,
      "activations/layer17_attention_weight_min": -23.974782943725586,
      "activations/layer18_attention_weight_max": 30.00653076171875,
      "activations/layer18_attention_weight_min": -19.142929077148438,
      "activations/layer19_attention_weight_max": 46.65058517456055,
      "activations/layer19_attention_weight_min": -28.065397262573242,
      "activations/layer1_attention_weight_max": 16.507463455200195,
      "activations/layer1_attention_weight_min": -13.622535705566406,
      "activations/layer20_attention_weight_max": 35.116905212402344,
      "activations/layer20_attention_weight_min": -21.93440055847168,
      "activations/layer21_attention_weight_max": 30.267032623291016,
      "activations/layer21_attention_weight_min": -20.1382999420166,
      "activations/layer22_attention_weight_max": 51.53772735595703,
      "activations/layer22_attention_weight_min": -24.46204376220703,
      "activations/layer23_attention_weight_max": 29.006935119628906,
      "activations/layer23_attention_weight_min": -17.396331787109375,
      "activations/layer2_attention_weight_max": 30.570310592651367,
      "activations/layer2_attention_weight_min": -31.63454246520996,
      "activations/layer3_attention_weight_max": 81.58335876464844,
      "activations/layer3_attention_weight_min": -85.73625946044922,
      "activations/layer4_attention_weight_max": 130.61077880859375,
      "activations/layer4_attention_weight_min": -124.800048828125,
      "activations/layer5_attention_weight_max": 93.6894760131836,
      "activations/layer5_attention_weight_min": -91.36288452148438,
      "activations/layer6_attention_weight_max": 57.4698486328125,
      "activations/layer6_attention_weight_min": -54.10202407836914,
      "activations/layer7_attention_weight_max": 74.72714233398438,
      "activations/layer7_attention_weight_min": -73.19351196289062,
      "activations/layer8_attention_weight_max": 37.03449249267578,
      "activations/layer8_attention_weight_min": -40.05739212036133,
      "activations/layer9_attention_weight_max": 44.0366325378418,
      "activations/layer9_attention_weight_min": -33.89128112792969,
      "epoch": 3.6,
      "learning_rate": 0.0001280931818181818,
      "loss": 2.8784,
      "step": 61900
    },
    {
      "activations/layer0_attention_weight_max": 15.661935806274414,
      "activations/layer0_attention_weight_min": -10.937630653381348,
      "activations/layer10_attention_weight_max": 43.43556594848633,
      "activations/layer10_attention_weight_min": -30.656808853149414,
      "activations/layer11_attention_weight_max": 39.557464599609375,
      "activations/layer11_attention_weight_min": -31.55232810974121,
      "activations/layer12_attention_weight_max": 34.97631072998047,
      "activations/layer12_attention_weight_min": -25.139583587646484,
      "activations/layer13_attention_weight_max": 60.4197883605957,
      "activations/layer13_attention_weight_min": -37.16164779663086,
      "activations/layer14_attention_weight_max": 76.89446258544922,
      "activations/layer14_attention_weight_min": -50.94989776611328,
      "activations/layer15_attention_weight_max": 49.804019927978516,
      "activations/layer15_attention_weight_min": -33.62803268432617,
      "activations/layer16_attention_weight_max": 49.0677604675293,
      "activations/layer16_attention_weight_min": -30.995677947998047,
      "activations/layer17_attention_weight_max": 44.964080810546875,
      "activations/layer17_attention_weight_min": -25.944412231445312,
      "activations/layer18_attention_weight_max": 36.71452331542969,
      "activations/layer18_attention_weight_min": -21.6571102142334,
      "activations/layer19_attention_weight_max": 56.522823333740234,
      "activations/layer19_attention_weight_min": -29.498844146728516,
      "activations/layer1_attention_weight_max": 16.79544448852539,
      "activations/layer1_attention_weight_min": -11.970324516296387,
      "activations/layer20_attention_weight_max": 46.91374969482422,
      "activations/layer20_attention_weight_min": -22.735332489013672,
      "activations/layer21_attention_weight_max": 35.57440185546875,
      "activations/layer21_attention_weight_min": -23.851482391357422,
      "activations/layer22_attention_weight_max": 67.77436828613281,
      "activations/layer22_attention_weight_min": -28.54978370666504,
      "activations/layer23_attention_weight_max": 31.835424423217773,
      "activations/layer23_attention_weight_min": -18.72153663635254,
      "activations/layer2_attention_weight_max": 31.18362045288086,
      "activations/layer2_attention_weight_min": -30.817607879638672,
      "activations/layer3_attention_weight_max": 81.74012756347656,
      "activations/layer3_attention_weight_min": -83.94526672363281,
      "activations/layer4_attention_weight_max": 121.18688201904297,
      "activations/layer4_attention_weight_min": -126.89250946044922,
      "activations/layer5_attention_weight_max": 91.29216766357422,
      "activations/layer5_attention_weight_min": -92.72892761230469,
      "activations/layer6_attention_weight_max": 61.74809646606445,
      "activations/layer6_attention_weight_min": -56.445838928222656,
      "activations/layer7_attention_weight_max": 75.40308380126953,
      "activations/layer7_attention_weight_min": -74.6021499633789,
      "activations/layer8_attention_weight_max": 39.70637130737305,
      "activations/layer8_attention_weight_min": -37.62820053100586,
      "activations/layer9_attention_weight_max": 47.80644989013672,
      "activations/layer9_attention_weight_min": -36.69741439819336,
      "epoch": 3.6,
      "learning_rate": 0.0001280746212121212,
      "loss": 2.8818,
      "step": 61950
    },
    {
      "activations/layer0_attention_weight_max": 16.381860733032227,
      "activations/layer0_attention_weight_min": -12.298985481262207,
      "activations/layer10_attention_weight_max": 35.58100509643555,
      "activations/layer10_attention_weight_min": -29.633983612060547,
      "activations/layer11_attention_weight_max": 33.561981201171875,
      "activations/layer11_attention_weight_min": -27.75231170654297,
      "activations/layer12_attention_weight_max": 36.553871154785156,
      "activations/layer12_attention_weight_min": -25.719892501831055,
      "activations/layer13_attention_weight_max": 48.537269592285156,
      "activations/layer13_attention_weight_min": -38.57186508178711,
      "activations/layer14_attention_weight_max": 61.940330505371094,
      "activations/layer14_attention_weight_min": -47.38330841064453,
      "activations/layer15_attention_weight_max": 40.23508071899414,
      "activations/layer15_attention_weight_min": -28.30890655517578,
      "activations/layer16_attention_weight_max": 37.74690628051758,
      "activations/layer16_attention_weight_min": -28.521997451782227,
      "activations/layer17_attention_weight_max": 38.781978607177734,
      "activations/layer17_attention_weight_min": -24.54956817626953,
      "activations/layer18_attention_weight_max": 30.560705184936523,
      "activations/layer18_attention_weight_min": -21.359607696533203,
      "activations/layer19_attention_weight_max": 40.351112365722656,
      "activations/layer19_attention_weight_min": -30.63412857055664,
      "activations/layer1_attention_weight_max": 17.69011688232422,
      "activations/layer1_attention_weight_min": -14.529884338378906,
      "activations/layer20_attention_weight_max": 35.226192474365234,
      "activations/layer20_attention_weight_min": -24.2789363861084,
      "activations/layer21_attention_weight_max": 29.603988647460938,
      "activations/layer21_attention_weight_min": -21.62639808654785,
      "activations/layer22_attention_weight_max": 47.76829528808594,
      "activations/layer22_attention_weight_min": -27.899805068969727,
      "activations/layer23_attention_weight_max": 26.806480407714844,
      "activations/layer23_attention_weight_min": -18.379764556884766,
      "activations/layer2_attention_weight_max": 32.21141052246094,
      "activations/layer2_attention_weight_min": -31.139755249023438,
      "activations/layer3_attention_weight_max": 79.14006042480469,
      "activations/layer3_attention_weight_min": -82.91132354736328,
      "activations/layer4_attention_weight_max": 123.5707015991211,
      "activations/layer4_attention_weight_min": -123.4953384399414,
      "activations/layer5_attention_weight_max": 91.70782470703125,
      "activations/layer5_attention_weight_min": -89.10671997070312,
      "activations/layer6_attention_weight_max": 54.08377456665039,
      "activations/layer6_attention_weight_min": -54.93806076049805,
      "activations/layer7_attention_weight_max": 72.24992370605469,
      "activations/layer7_attention_weight_min": -73.33085632324219,
      "activations/layer8_attention_weight_max": 38.27590560913086,
      "activations/layer8_attention_weight_min": -37.11660385131836,
      "activations/layer9_attention_weight_max": 38.223384857177734,
      "activations/layer9_attention_weight_min": -35.67692184448242,
      "epoch": 3.6,
      "learning_rate": 0.0001280556818181818,
      "loss": 2.8816,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_loss": 2.837890625,
      "eval_runtime": 8.521,
      "eval_samples_per_second": 503.929,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_openwebtext_loss": 2.837890625,
      "eval_openwebtext_ppl": 17.079700020438295,
      "eval_openwebtext_runtime": 8.521,
      "eval_openwebtext_samples_per_second": 503.929,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_wikitext_loss": 3.078125,
      "eval_wikitext_ppl": 21.717643615002626,
      "eval_wikitext_runtime": 2.0198,
      "eval_wikitext_samples_per_second": 225.766,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_lambada_loss": 2.779296875,
      "eval_lambada_ppl": 16.107691244988956,
      "eval_lambada_runtime": 9.6207,
      "eval_lambada_samples_per_second": 506.098,
      "step": 62000
    },
    {
      "activations/layer0_attention_weight_max": 15.996101379394531,
      "activations/layer0_attention_weight_min": -11.59116268157959,
      "activations/layer10_attention_weight_max": 40.31876754760742,
      "activations/layer10_attention_weight_min": -31.216533660888672,
      "activations/layer11_attention_weight_max": 37.50361633300781,
      "activations/layer11_attention_weight_min": -27.47193717956543,
      "activations/layer12_attention_weight_max": 35.54497146606445,
      "activations/layer12_attention_weight_min": -27.593767166137695,
      "activations/layer13_attention_weight_max": 54.98942565917969,
      "activations/layer13_attention_weight_min": -39.31711959838867,
      "activations/layer14_attention_weight_max": 76.74720764160156,
      "activations/layer14_attention_weight_min": -55.717018127441406,
      "activations/layer15_attention_weight_max": 51.40869903564453,
      "activations/layer15_attention_weight_min": -29.887243270874023,
      "activations/layer16_attention_weight_max": 50.11125183105469,
      "activations/layer16_attention_weight_min": -29.86269760131836,
      "activations/layer17_attention_weight_max": 46.46968078613281,
      "activations/layer17_attention_weight_min": -26.557140350341797,
      "activations/layer18_attention_weight_max": 35.87276840209961,
      "activations/layer18_attention_weight_min": -20.263994216918945,
      "activations/layer19_attention_weight_max": 52.31740951538086,
      "activations/layer19_attention_weight_min": -28.231204986572266,
      "activations/layer1_attention_weight_max": 17.634614944458008,
      "activations/layer1_attention_weight_min": -12.971673011779785,
      "activations/layer20_attention_weight_max": 38.57202911376953,
      "activations/layer20_attention_weight_min": -23.493650436401367,
      "activations/layer21_attention_weight_max": 42.644012451171875,
      "activations/layer21_attention_weight_min": -21.2547664642334,
      "activations/layer22_attention_weight_max": 61.304481506347656,
      "activations/layer22_attention_weight_min": -25.70466423034668,
      "activations/layer23_attention_weight_max": 34.89436340332031,
      "activations/layer23_attention_weight_min": -16.852062225341797,
      "activations/layer2_attention_weight_max": 31.8463134765625,
      "activations/layer2_attention_weight_min": -29.857131958007812,
      "activations/layer3_attention_weight_max": 84.70185089111328,
      "activations/layer3_attention_weight_min": -86.7519760131836,
      "activations/layer4_attention_weight_max": 134.1715545654297,
      "activations/layer4_attention_weight_min": -125.50057220458984,
      "activations/layer5_attention_weight_max": 100.11634826660156,
      "activations/layer5_attention_weight_min": -94.37934875488281,
      "activations/layer6_attention_weight_max": 59.00171661376953,
      "activations/layer6_attention_weight_min": -55.31181335449219,
      "activations/layer7_attention_weight_max": 74.89701080322266,
      "activations/layer7_attention_weight_min": -71.54878997802734,
      "activations/layer8_attention_weight_max": 39.322635650634766,
      "activations/layer8_attention_weight_min": -37.58671951293945,
      "activations/layer9_attention_weight_max": 38.437599182128906,
      "activations/layer9_attention_weight_min": -38.72939682006836,
      "epoch": 3.61,
      "learning_rate": 0.0001280367424242424,
      "loss": 2.8909,
      "step": 62050
    },
    {
      "activations/layer0_attention_weight_max": 16.681358337402344,
      "activations/layer0_attention_weight_min": -11.379204750061035,
      "activations/layer10_attention_weight_max": 37.9404296875,
      "activations/layer10_attention_weight_min": -32.997314453125,
      "activations/layer11_attention_weight_max": 34.194541931152344,
      "activations/layer11_attention_weight_min": -28.907392501831055,
      "activations/layer12_attention_weight_max": 30.490001678466797,
      "activations/layer12_attention_weight_min": -25.447954177856445,
      "activations/layer13_attention_weight_max": 54.22364044189453,
      "activations/layer13_attention_weight_min": -35.633506774902344,
      "activations/layer14_attention_weight_max": 83.69622039794922,
      "activations/layer14_attention_weight_min": -55.350181579589844,
      "activations/layer15_attention_weight_max": 46.754695892333984,
      "activations/layer15_attention_weight_min": -28.6333065032959,
      "activations/layer16_attention_weight_max": 45.91868591308594,
      "activations/layer16_attention_weight_min": -31.47894287109375,
      "activations/layer17_attention_weight_max": 45.6335334777832,
      "activations/layer17_attention_weight_min": -25.838708877563477,
      "activations/layer18_attention_weight_max": 31.436702728271484,
      "activations/layer18_attention_weight_min": -18.188674926757812,
      "activations/layer19_attention_weight_max": 51.497596740722656,
      "activations/layer19_attention_weight_min": -29.215957641601562,
      "activations/layer1_attention_weight_max": 16.969938278198242,
      "activations/layer1_attention_weight_min": -14.059980392456055,
      "activations/layer20_attention_weight_max": 38.79950714111328,
      "activations/layer20_attention_weight_min": -22.846960067749023,
      "activations/layer21_attention_weight_max": 32.769371032714844,
      "activations/layer21_attention_weight_min": -20.663904190063477,
      "activations/layer22_attention_weight_max": 54.52641677856445,
      "activations/layer22_attention_weight_min": -26.705177307128906,
      "activations/layer23_attention_weight_max": 32.450408935546875,
      "activations/layer23_attention_weight_min": -20.015708923339844,
      "activations/layer2_attention_weight_max": 32.652244567871094,
      "activations/layer2_attention_weight_min": -30.694217681884766,
      "activations/layer3_attention_weight_max": 87.91900634765625,
      "activations/layer3_attention_weight_min": -88.39226531982422,
      "activations/layer4_attention_weight_max": 145.1337127685547,
      "activations/layer4_attention_weight_min": -129.78121948242188,
      "activations/layer5_attention_weight_max": 103.39370727539062,
      "activations/layer5_attention_weight_min": -93.7607650756836,
      "activations/layer6_attention_weight_max": 62.200340270996094,
      "activations/layer6_attention_weight_min": -60.03325271606445,
      "activations/layer7_attention_weight_max": 80.6880874633789,
      "activations/layer7_attention_weight_min": -81.737548828125,
      "activations/layer8_attention_weight_max": 43.208274841308594,
      "activations/layer8_attention_weight_min": -39.36285400390625,
      "activations/layer9_attention_weight_max": 46.91231155395508,
      "activations/layer9_attention_weight_min": -33.56026840209961,
      "epoch": 3.61,
      "learning_rate": 0.00012801780303030302,
      "loss": 2.8724,
      "step": 62100
    },
    {
      "activations/layer0_attention_weight_max": 15.85316276550293,
      "activations/layer0_attention_weight_min": -11.497099876403809,
      "activations/layer10_attention_weight_max": 36.58555603027344,
      "activations/layer10_attention_weight_min": -32.411556243896484,
      "activations/layer11_attention_weight_max": 44.44953918457031,
      "activations/layer11_attention_weight_min": -31.47609519958496,
      "activations/layer12_attention_weight_max": 30.261131286621094,
      "activations/layer12_attention_weight_min": -28.89061737060547,
      "activations/layer13_attention_weight_max": 54.47132110595703,
      "activations/layer13_attention_weight_min": -37.60825729370117,
      "activations/layer14_attention_weight_max": 63.760807037353516,
      "activations/layer14_attention_weight_min": -47.41721725463867,
      "activations/layer15_attention_weight_max": 44.674476623535156,
      "activations/layer15_attention_weight_min": -30.100908279418945,
      "activations/layer16_attention_weight_max": 44.289276123046875,
      "activations/layer16_attention_weight_min": -31.46452522277832,
      "activations/layer17_attention_weight_max": 43.60832977294922,
      "activations/layer17_attention_weight_min": -25.238258361816406,
      "activations/layer18_attention_weight_max": 28.103317260742188,
      "activations/layer18_attention_weight_min": -18.722055435180664,
      "activations/layer19_attention_weight_max": 41.27326965332031,
      "activations/layer19_attention_weight_min": -28.379270553588867,
      "activations/layer1_attention_weight_max": 19.017295837402344,
      "activations/layer1_attention_weight_min": -15.191441535949707,
      "activations/layer20_attention_weight_max": 36.72177505493164,
      "activations/layer20_attention_weight_min": -23.270620346069336,
      "activations/layer21_attention_weight_max": 31.13749885559082,
      "activations/layer21_attention_weight_min": -19.733234405517578,
      "activations/layer22_attention_weight_max": 47.52790069580078,
      "activations/layer22_attention_weight_min": -26.84487533569336,
      "activations/layer23_attention_weight_max": 27.757749557495117,
      "activations/layer23_attention_weight_min": -19.331283569335938,
      "activations/layer2_attention_weight_max": 32.5515251159668,
      "activations/layer2_attention_weight_min": -30.69629669189453,
      "activations/layer3_attention_weight_max": 86.92110443115234,
      "activations/layer3_attention_weight_min": -85.43888854980469,
      "activations/layer4_attention_weight_max": 134.188720703125,
      "activations/layer4_attention_weight_min": -126.03783416748047,
      "activations/layer5_attention_weight_max": 98.92949676513672,
      "activations/layer5_attention_weight_min": -87.57528686523438,
      "activations/layer6_attention_weight_max": 60.20456314086914,
      "activations/layer6_attention_weight_min": -54.495887756347656,
      "activations/layer7_attention_weight_max": 92.925048828125,
      "activations/layer7_attention_weight_min": -75.6817398071289,
      "activations/layer8_attention_weight_max": 44.385963439941406,
      "activations/layer8_attention_weight_min": -37.2999153137207,
      "activations/layer9_attention_weight_max": 51.997657775878906,
      "activations/layer9_attention_weight_min": -42.7528076171875,
      "epoch": 3.61,
      "learning_rate": 0.0001279988636363636,
      "loss": 2.8852,
      "step": 62150
    },
    {
      "activations/layer0_attention_weight_max": 16.05379295349121,
      "activations/layer0_attention_weight_min": -11.619194984436035,
      "activations/layer10_attention_weight_max": 37.624481201171875,
      "activations/layer10_attention_weight_min": -32.99826431274414,
      "activations/layer11_attention_weight_max": 32.358909606933594,
      "activations/layer11_attention_weight_min": -26.16183090209961,
      "activations/layer12_attention_weight_max": 33.20732879638672,
      "activations/layer12_attention_weight_min": -26.104652404785156,
      "activations/layer13_attention_weight_max": 48.218894958496094,
      "activations/layer13_attention_weight_min": -32.357025146484375,
      "activations/layer14_attention_weight_max": 71.27977752685547,
      "activations/layer14_attention_weight_min": -47.67742919921875,
      "activations/layer15_attention_weight_max": 44.61669921875,
      "activations/layer15_attention_weight_min": -29.61788558959961,
      "activations/layer16_attention_weight_max": 40.24968338012695,
      "activations/layer16_attention_weight_min": -29.35430908203125,
      "activations/layer17_attention_weight_max": 43.5488166809082,
      "activations/layer17_attention_weight_min": -23.83107566833496,
      "activations/layer18_attention_weight_max": 36.368228912353516,
      "activations/layer18_attention_weight_min": -19.945110321044922,
      "activations/layer19_attention_weight_max": 41.30182647705078,
      "activations/layer19_attention_weight_min": -29.056289672851562,
      "activations/layer1_attention_weight_max": 18.291431427001953,
      "activations/layer1_attention_weight_min": -14.060982704162598,
      "activations/layer20_attention_weight_max": 35.411094665527344,
      "activations/layer20_attention_weight_min": -22.140806198120117,
      "activations/layer21_attention_weight_max": 44.61869430541992,
      "activations/layer21_attention_weight_min": -21.354520797729492,
      "activations/layer22_attention_weight_max": 50.90857696533203,
      "activations/layer22_attention_weight_min": -27.92378807067871,
      "activations/layer23_attention_weight_max": 28.468910217285156,
      "activations/layer23_attention_weight_min": -17.831640243530273,
      "activations/layer2_attention_weight_max": 29.564159393310547,
      "activations/layer2_attention_weight_min": -28.719417572021484,
      "activations/layer3_attention_weight_max": 80.44733428955078,
      "activations/layer3_attention_weight_min": -87.20226287841797,
      "activations/layer4_attention_weight_max": 123.6689224243164,
      "activations/layer4_attention_weight_min": -122.66082000732422,
      "activations/layer5_attention_weight_max": 90.67766571044922,
      "activations/layer5_attention_weight_min": -84.292236328125,
      "activations/layer6_attention_weight_max": 53.537227630615234,
      "activations/layer6_attention_weight_min": -54.23745346069336,
      "activations/layer7_attention_weight_max": 74.8494644165039,
      "activations/layer7_attention_weight_min": -72.40362548828125,
      "activations/layer8_attention_weight_max": 37.22019958496094,
      "activations/layer8_attention_weight_min": -35.83655548095703,
      "activations/layer9_attention_weight_max": 38.75774002075195,
      "activations/layer9_attention_weight_min": -31.695632934570312,
      "epoch": 3.61,
      "learning_rate": 0.00012797992424242423,
      "loss": 2.8982,
      "step": 62200
    },
    {
      "activations/layer0_attention_weight_max": 15.857049942016602,
      "activations/layer0_attention_weight_min": -11.536781311035156,
      "activations/layer10_attention_weight_max": 33.841800689697266,
      "activations/layer10_attention_weight_min": -28.883703231811523,
      "activations/layer11_attention_weight_max": 31.182735443115234,
      "activations/layer11_attention_weight_min": -28.009750366210938,
      "activations/layer12_attention_weight_max": 26.81202507019043,
      "activations/layer12_attention_weight_min": -32.556251525878906,
      "activations/layer13_attention_weight_max": 41.50167465209961,
      "activations/layer13_attention_weight_min": -32.53691482543945,
      "activations/layer14_attention_weight_max": 57.045677185058594,
      "activations/layer14_attention_weight_min": -50.73914337158203,
      "activations/layer15_attention_weight_max": 44.83400344848633,
      "activations/layer15_attention_weight_min": -29.275650024414062,
      "activations/layer16_attention_weight_max": 36.99716567993164,
      "activations/layer16_attention_weight_min": -29.392578125,
      "activations/layer17_attention_weight_max": 37.67805099487305,
      "activations/layer17_attention_weight_min": -25.445642471313477,
      "activations/layer18_attention_weight_max": 29.864702224731445,
      "activations/layer18_attention_weight_min": -21.97308349609375,
      "activations/layer19_attention_weight_max": 40.8975830078125,
      "activations/layer19_attention_weight_min": -30.54961585998535,
      "activations/layer1_attention_weight_max": 18.048505783081055,
      "activations/layer1_attention_weight_min": -14.731905937194824,
      "activations/layer20_attention_weight_max": 34.37983322143555,
      "activations/layer20_attention_weight_min": -23.43470001220703,
      "activations/layer21_attention_weight_max": 36.411643981933594,
      "activations/layer21_attention_weight_min": -23.255685806274414,
      "activations/layer22_attention_weight_max": 45.423892974853516,
      "activations/layer22_attention_weight_min": -26.221208572387695,
      "activations/layer23_attention_weight_max": 27.94874382019043,
      "activations/layer23_attention_weight_min": -18.20928192138672,
      "activations/layer2_attention_weight_max": 31.082569122314453,
      "activations/layer2_attention_weight_min": -31.558841705322266,
      "activations/layer3_attention_weight_max": 86.7966537475586,
      "activations/layer3_attention_weight_min": -89.03439331054688,
      "activations/layer4_attention_weight_max": 136.03646850585938,
      "activations/layer4_attention_weight_min": -132.2715606689453,
      "activations/layer5_attention_weight_max": 97.52669525146484,
      "activations/layer5_attention_weight_min": -89.21066284179688,
      "activations/layer6_attention_weight_max": 60.50945281982422,
      "activations/layer6_attention_weight_min": -60.05841827392578,
      "activations/layer7_attention_weight_max": 77.5194091796875,
      "activations/layer7_attention_weight_min": -77.0542984008789,
      "activations/layer8_attention_weight_max": 39.172882080078125,
      "activations/layer8_attention_weight_min": -38.10471725463867,
      "activations/layer9_attention_weight_max": 38.75931930541992,
      "activations/layer9_attention_weight_min": -44.8214225769043,
      "epoch": 3.62,
      "learning_rate": 0.00012796098484848484,
      "loss": 2.8818,
      "step": 62250
    },
    {
      "activations/layer0_attention_weight_max": 16.81556510925293,
      "activations/layer0_attention_weight_min": -11.170605659484863,
      "activations/layer10_attention_weight_max": 36.546165466308594,
      "activations/layer10_attention_weight_min": -30.75717544555664,
      "activations/layer11_attention_weight_max": 32.89466857910156,
      "activations/layer11_attention_weight_min": -26.284343719482422,
      "activations/layer12_attention_weight_max": 32.94403076171875,
      "activations/layer12_attention_weight_min": -27.565505981445312,
      "activations/layer13_attention_weight_max": 46.08366012573242,
      "activations/layer13_attention_weight_min": -35.28546905517578,
      "activations/layer14_attention_weight_max": 65.52845764160156,
      "activations/layer14_attention_weight_min": -45.60767364501953,
      "activations/layer15_attention_weight_max": 45.8359489440918,
      "activations/layer15_attention_weight_min": -30.471269607543945,
      "activations/layer16_attention_weight_max": 39.03565979003906,
      "activations/layer16_attention_weight_min": -28.487075805664062,
      "activations/layer17_attention_weight_max": 42.97380447387695,
      "activations/layer17_attention_weight_min": -23.389347076416016,
      "activations/layer18_attention_weight_max": 27.442420959472656,
      "activations/layer18_attention_weight_min": -18.831552505493164,
      "activations/layer19_attention_weight_max": 40.64545440673828,
      "activations/layer19_attention_weight_min": -28.247543334960938,
      "activations/layer1_attention_weight_max": 18.387754440307617,
      "activations/layer1_attention_weight_min": -16.428672790527344,
      "activations/layer20_attention_weight_max": 34.34572219848633,
      "activations/layer20_attention_weight_min": -22.605764389038086,
      "activations/layer21_attention_weight_max": 28.73632049560547,
      "activations/layer21_attention_weight_min": -18.31987953186035,
      "activations/layer22_attention_weight_max": 49.15377426147461,
      "activations/layer22_attention_weight_min": -27.09394645690918,
      "activations/layer23_attention_weight_max": 27.72918701171875,
      "activations/layer23_attention_weight_min": -23.260107040405273,
      "activations/layer2_attention_weight_max": 31.774635314941406,
      "activations/layer2_attention_weight_min": -30.397483825683594,
      "activations/layer3_attention_weight_max": 80.72925567626953,
      "activations/layer3_attention_weight_min": -81.15856170654297,
      "activations/layer4_attention_weight_max": 130.69654846191406,
      "activations/layer4_attention_weight_min": -126.88545227050781,
      "activations/layer5_attention_weight_max": 94.87545776367188,
      "activations/layer5_attention_weight_min": -87.38729858398438,
      "activations/layer6_attention_weight_max": 56.542633056640625,
      "activations/layer6_attention_weight_min": -52.704124450683594,
      "activations/layer7_attention_weight_max": 71.84234619140625,
      "activations/layer7_attention_weight_min": -74.49693298339844,
      "activations/layer8_attention_weight_max": 39.781005859375,
      "activations/layer8_attention_weight_min": -36.98085403442383,
      "activations/layer9_attention_weight_max": 40.099491119384766,
      "activations/layer9_attention_weight_min": -35.5941276550293,
      "epoch": 3.62,
      "learning_rate": 0.00012794204545454546,
      "loss": 2.8899,
      "step": 62300
    },
    {
      "activations/layer0_attention_weight_max": 16.238197326660156,
      "activations/layer0_attention_weight_min": -10.964897155761719,
      "activations/layer10_attention_weight_max": 36.22490310668945,
      "activations/layer10_attention_weight_min": -30.398168563842773,
      "activations/layer11_attention_weight_max": 35.48208236694336,
      "activations/layer11_attention_weight_min": -28.507503509521484,
      "activations/layer12_attention_weight_max": 43.63797378540039,
      "activations/layer12_attention_weight_min": -28.77543830871582,
      "activations/layer13_attention_weight_max": 51.94015884399414,
      "activations/layer13_attention_weight_min": -36.76266098022461,
      "activations/layer14_attention_weight_max": 69.62328338623047,
      "activations/layer14_attention_weight_min": -56.201194763183594,
      "activations/layer15_attention_weight_max": 45.34523010253906,
      "activations/layer15_attention_weight_min": -31.73664093017578,
      "activations/layer16_attention_weight_max": 41.5079345703125,
      "activations/layer16_attention_weight_min": -32.10298156738281,
      "activations/layer17_attention_weight_max": 41.98809051513672,
      "activations/layer17_attention_weight_min": -26.350114822387695,
      "activations/layer18_attention_weight_max": 30.900653839111328,
      "activations/layer18_attention_weight_min": -22.548185348510742,
      "activations/layer19_attention_weight_max": 42.373634338378906,
      "activations/layer19_attention_weight_min": -30.70883560180664,
      "activations/layer1_attention_weight_max": 17.42635154724121,
      "activations/layer1_attention_weight_min": -12.54288387298584,
      "activations/layer20_attention_weight_max": 36.524410247802734,
      "activations/layer20_attention_weight_min": -23.813413619995117,
      "activations/layer21_attention_weight_max": 35.03276062011719,
      "activations/layer21_attention_weight_min": -23.90948486328125,
      "activations/layer22_attention_weight_max": 53.05078125,
      "activations/layer22_attention_weight_min": -28.32317352294922,
      "activations/layer23_attention_weight_max": 29.723979949951172,
      "activations/layer23_attention_weight_min": -19.566532135009766,
      "activations/layer2_attention_weight_max": 31.09174919128418,
      "activations/layer2_attention_weight_min": -28.765743255615234,
      "activations/layer3_attention_weight_max": 81.37915802001953,
      "activations/layer3_attention_weight_min": -83.41941833496094,
      "activations/layer4_attention_weight_max": 132.37091064453125,
      "activations/layer4_attention_weight_min": -127.28946685791016,
      "activations/layer5_attention_weight_max": 96.13744354248047,
      "activations/layer5_attention_weight_min": -90.37178039550781,
      "activations/layer6_attention_weight_max": 58.8167724609375,
      "activations/layer6_attention_weight_min": -53.26670455932617,
      "activations/layer7_attention_weight_max": 78.93667602539062,
      "activations/layer7_attention_weight_min": -76.24131774902344,
      "activations/layer8_attention_weight_max": 39.19240188598633,
      "activations/layer8_attention_weight_min": -37.29074478149414,
      "activations/layer9_attention_weight_max": 50.64167022705078,
      "activations/layer9_attention_weight_min": -33.777767181396484,
      "epoch": 3.62,
      "learning_rate": 0.00012792310606060605,
      "loss": 2.9045,
      "step": 62350
    },
    {
      "activations/layer0_attention_weight_max": 17.643056869506836,
      "activations/layer0_attention_weight_min": -11.967019081115723,
      "activations/layer10_attention_weight_max": 40.5052375793457,
      "activations/layer10_attention_weight_min": -34.611366271972656,
      "activations/layer11_attention_weight_max": 38.6511344909668,
      "activations/layer11_attention_weight_min": -28.956069946289062,
      "activations/layer12_attention_weight_max": 41.77198791503906,
      "activations/layer12_attention_weight_min": -25.875205993652344,
      "activations/layer13_attention_weight_max": 52.601924896240234,
      "activations/layer13_attention_weight_min": -37.0001220703125,
      "activations/layer14_attention_weight_max": 76.03585052490234,
      "activations/layer14_attention_weight_min": -52.82478713989258,
      "activations/layer15_attention_weight_max": 47.14289474487305,
      "activations/layer15_attention_weight_min": -30.194440841674805,
      "activations/layer16_attention_weight_max": 43.98482894897461,
      "activations/layer16_attention_weight_min": -32.33472442626953,
      "activations/layer17_attention_weight_max": 44.65578079223633,
      "activations/layer17_attention_weight_min": -26.67969512939453,
      "activations/layer18_attention_weight_max": 35.5778694152832,
      "activations/layer18_attention_weight_min": -20.637447357177734,
      "activations/layer19_attention_weight_max": 43.56318283081055,
      "activations/layer19_attention_weight_min": -28.968618392944336,
      "activations/layer1_attention_weight_max": 17.928363800048828,
      "activations/layer1_attention_weight_min": -13.348645210266113,
      "activations/layer20_attention_weight_max": 35.19845962524414,
      "activations/layer20_attention_weight_min": -24.854955673217773,
      "activations/layer21_attention_weight_max": 39.66987991333008,
      "activations/layer21_attention_weight_min": -23.614030838012695,
      "activations/layer22_attention_weight_max": 51.58053970336914,
      "activations/layer22_attention_weight_min": -28.116106033325195,
      "activations/layer23_attention_weight_max": 30.50341033935547,
      "activations/layer23_attention_weight_min": -18.385608673095703,
      "activations/layer2_attention_weight_max": 31.192855834960938,
      "activations/layer2_attention_weight_min": -30.29092788696289,
      "activations/layer3_attention_weight_max": 85.6687240600586,
      "activations/layer3_attention_weight_min": -86.37699127197266,
      "activations/layer4_attention_weight_max": 127.55326080322266,
      "activations/layer4_attention_weight_min": -132.23561096191406,
      "activations/layer5_attention_weight_max": 96.73945617675781,
      "activations/layer5_attention_weight_min": -92.99761199951172,
      "activations/layer6_attention_weight_max": 62.19011688232422,
      "activations/layer6_attention_weight_min": -58.47891616821289,
      "activations/layer7_attention_weight_max": 81.30149841308594,
      "activations/layer7_attention_weight_min": -84.6338882446289,
      "activations/layer8_attention_weight_max": 45.06438064575195,
      "activations/layer8_attention_weight_min": -41.891422271728516,
      "activations/layer9_attention_weight_max": 42.20803451538086,
      "activations/layer9_attention_weight_min": -36.9098014831543,
      "epoch": 3.63,
      "learning_rate": 0.00012790416666666666,
      "loss": 2.8985,
      "step": 62400
    },
    {
      "activations/layer0_attention_weight_max": 16.227886199951172,
      "activations/layer0_attention_weight_min": -11.54532241821289,
      "activations/layer10_attention_weight_max": 35.25154495239258,
      "activations/layer10_attention_weight_min": -31.707975387573242,
      "activations/layer11_attention_weight_max": 34.3864631652832,
      "activations/layer11_attention_weight_min": -27.634326934814453,
      "activations/layer12_attention_weight_max": 33.97965621948242,
      "activations/layer12_attention_weight_min": -26.940263748168945,
      "activations/layer13_attention_weight_max": 46.25695037841797,
      "activations/layer13_attention_weight_min": -33.37235641479492,
      "activations/layer14_attention_weight_max": 67.00919342041016,
      "activations/layer14_attention_weight_min": -57.325069427490234,
      "activations/layer15_attention_weight_max": 43.23310470581055,
      "activations/layer15_attention_weight_min": -29.45340347290039,
      "activations/layer16_attention_weight_max": 40.06267166137695,
      "activations/layer16_attention_weight_min": -28.720901489257812,
      "activations/layer17_attention_weight_max": 42.480926513671875,
      "activations/layer17_attention_weight_min": -27.463727951049805,
      "activations/layer18_attention_weight_max": 30.26876449584961,
      "activations/layer18_attention_weight_min": -21.859670639038086,
      "activations/layer19_attention_weight_max": 44.0020751953125,
      "activations/layer19_attention_weight_min": -29.20572853088379,
      "activations/layer1_attention_weight_max": 17.633583068847656,
      "activations/layer1_attention_weight_min": -15.762840270996094,
      "activations/layer20_attention_weight_max": 31.044071197509766,
      "activations/layer20_attention_weight_min": -21.540637969970703,
      "activations/layer21_attention_weight_max": 33.67963790893555,
      "activations/layer21_attention_weight_min": -20.825340270996094,
      "activations/layer22_attention_weight_max": 46.842857360839844,
      "activations/layer22_attention_weight_min": -28.92505645751953,
      "activations/layer23_attention_weight_max": 27.404706954956055,
      "activations/layer23_attention_weight_min": -18.271942138671875,
      "activations/layer2_attention_weight_max": 30.51366424560547,
      "activations/layer2_attention_weight_min": -30.043506622314453,
      "activations/layer3_attention_weight_max": 80.68287658691406,
      "activations/layer3_attention_weight_min": -82.90377807617188,
      "activations/layer4_attention_weight_max": 133.8164520263672,
      "activations/layer4_attention_weight_min": -124.83780670166016,
      "activations/layer5_attention_weight_max": 90.8431625366211,
      "activations/layer5_attention_weight_min": -87.77835083007812,
      "activations/layer6_attention_weight_max": 56.09839630126953,
      "activations/layer6_attention_weight_min": -57.97731399536133,
      "activations/layer7_attention_weight_max": 78.98744201660156,
      "activations/layer7_attention_weight_min": -74.94535827636719,
      "activations/layer8_attention_weight_max": 36.948814392089844,
      "activations/layer8_attention_weight_min": -36.86845397949219,
      "activations/layer9_attention_weight_max": 42.30179977416992,
      "activations/layer9_attention_weight_min": -33.95588302612305,
      "epoch": 3.63,
      "learning_rate": 0.00012788522727272728,
      "loss": 2.8835,
      "step": 62450
    },
    {
      "activations/layer0_attention_weight_max": 16.712928771972656,
      "activations/layer0_attention_weight_min": -11.467782020568848,
      "activations/layer10_attention_weight_max": 37.015174865722656,
      "activations/layer10_attention_weight_min": -30.53001594543457,
      "activations/layer11_attention_weight_max": 35.66749954223633,
      "activations/layer11_attention_weight_min": -27.41139030456543,
      "activations/layer12_attention_weight_max": 27.235370635986328,
      "activations/layer12_attention_weight_min": -26.79700469970703,
      "activations/layer13_attention_weight_max": 47.10060119628906,
      "activations/layer13_attention_weight_min": -34.85133743286133,
      "activations/layer14_attention_weight_max": 76.88607788085938,
      "activations/layer14_attention_weight_min": -53.36601257324219,
      "activations/layer15_attention_weight_max": 45.074127197265625,
      "activations/layer15_attention_weight_min": -30.256607055664062,
      "activations/layer16_attention_weight_max": 46.29819107055664,
      "activations/layer16_attention_weight_min": -30.024799346923828,
      "activations/layer17_attention_weight_max": 42.503204345703125,
      "activations/layer17_attention_weight_min": -28.532880783081055,
      "activations/layer18_attention_weight_max": 34.80839538574219,
      "activations/layer18_attention_weight_min": -20.17275619506836,
      "activations/layer19_attention_weight_max": 45.17927169799805,
      "activations/layer19_attention_weight_min": -30.84174919128418,
      "activations/layer1_attention_weight_max": 17.265907287597656,
      "activations/layer1_attention_weight_min": -14.298283576965332,
      "activations/layer20_attention_weight_max": 36.490726470947266,
      "activations/layer20_attention_weight_min": -22.451616287231445,
      "activations/layer21_attention_weight_max": 37.45478820800781,
      "activations/layer21_attention_weight_min": -18.35274887084961,
      "activations/layer22_attention_weight_max": 55.0692253112793,
      "activations/layer22_attention_weight_min": -30.179645538330078,
      "activations/layer23_attention_weight_max": 30.840415954589844,
      "activations/layer23_attention_weight_min": -17.76891326904297,
      "activations/layer2_attention_weight_max": 32.5269660949707,
      "activations/layer2_attention_weight_min": -31.718481063842773,
      "activations/layer3_attention_weight_max": 87.34812927246094,
      "activations/layer3_attention_weight_min": -85.70149993896484,
      "activations/layer4_attention_weight_max": 135.27574157714844,
      "activations/layer4_attention_weight_min": -130.0442352294922,
      "activations/layer5_attention_weight_max": 98.54029846191406,
      "activations/layer5_attention_weight_min": -88.12881469726562,
      "activations/layer6_attention_weight_max": 58.658878326416016,
      "activations/layer6_attention_weight_min": -56.95671081542969,
      "activations/layer7_attention_weight_max": 81.81117248535156,
      "activations/layer7_attention_weight_min": -76.89437866210938,
      "activations/layer8_attention_weight_max": 41.85334777832031,
      "activations/layer8_attention_weight_min": -41.29892349243164,
      "activations/layer9_attention_weight_max": 41.17308044433594,
      "activations/layer9_attention_weight_min": -33.84663391113281,
      "epoch": 3.63,
      "learning_rate": 0.00012786628787878786,
      "loss": 2.899,
      "step": 62500
    },
    {
      "activations/layer0_attention_weight_max": 16.991506576538086,
      "activations/layer0_attention_weight_min": -11.872245788574219,
      "activations/layer10_attention_weight_max": 38.353919982910156,
      "activations/layer10_attention_weight_min": -30.034915924072266,
      "activations/layer11_attention_weight_max": 35.2435302734375,
      "activations/layer11_attention_weight_min": -28.481876373291016,
      "activations/layer12_attention_weight_max": 32.64876937866211,
      "activations/layer12_attention_weight_min": -29.50587272644043,
      "activations/layer13_attention_weight_max": 49.61025619506836,
      "activations/layer13_attention_weight_min": -33.99364471435547,
      "activations/layer14_attention_weight_max": 73.50871276855469,
      "activations/layer14_attention_weight_min": -51.425262451171875,
      "activations/layer15_attention_weight_max": 41.87045669555664,
      "activations/layer15_attention_weight_min": -28.73259925842285,
      "activations/layer16_attention_weight_max": 43.82497787475586,
      "activations/layer16_attention_weight_min": -28.46695899963379,
      "activations/layer17_attention_weight_max": 39.329593658447266,
      "activations/layer17_attention_weight_min": -23.94516372680664,
      "activations/layer18_attention_weight_max": 30.86754035949707,
      "activations/layer18_attention_weight_min": -20.28423500061035,
      "activations/layer19_attention_weight_max": 47.21615982055664,
      "activations/layer19_attention_weight_min": -29.701068878173828,
      "activations/layer1_attention_weight_max": 17.684768676757812,
      "activations/layer1_attention_weight_min": -14.564705848693848,
      "activations/layer20_attention_weight_max": 32.72457504272461,
      "activations/layer20_attention_weight_min": -22.990671157836914,
      "activations/layer21_attention_weight_max": 31.07295799255371,
      "activations/layer21_attention_weight_min": -23.007801055908203,
      "activations/layer22_attention_weight_max": 55.0932731628418,
      "activations/layer22_attention_weight_min": -28.842458724975586,
      "activations/layer23_attention_weight_max": 29.83318328857422,
      "activations/layer23_attention_weight_min": -17.692386627197266,
      "activations/layer2_attention_weight_max": 30.289569854736328,
      "activations/layer2_attention_weight_min": -30.410995483398438,
      "activations/layer3_attention_weight_max": 83.93583679199219,
      "activations/layer3_attention_weight_min": -85.43836212158203,
      "activations/layer4_attention_weight_max": 128.2581024169922,
      "activations/layer4_attention_weight_min": -128.9396514892578,
      "activations/layer5_attention_weight_max": 98.65774536132812,
      "activations/layer5_attention_weight_min": -88.68864440917969,
      "activations/layer6_attention_weight_max": 62.4893913269043,
      "activations/layer6_attention_weight_min": -54.81098556518555,
      "activations/layer7_attention_weight_max": 83.42430114746094,
      "activations/layer7_attention_weight_min": -77.00521087646484,
      "activations/layer8_attention_weight_max": 41.98712158203125,
      "activations/layer8_attention_weight_min": -37.529212951660156,
      "activations/layer9_attention_weight_max": 42.68635559082031,
      "activations/layer9_attention_weight_min": -33.3514518737793,
      "epoch": 3.63,
      "learning_rate": 0.00012784772727272728,
      "loss": 2.8651,
      "step": 62550
    },
    {
      "activations/layer0_attention_weight_max": 16.709415435791016,
      "activations/layer0_attention_weight_min": -11.459967613220215,
      "activations/layer10_attention_weight_max": 43.825523376464844,
      "activations/layer10_attention_weight_min": -36.609046936035156,
      "activations/layer11_attention_weight_max": 44.825374603271484,
      "activations/layer11_attention_weight_min": -35.79438781738281,
      "activations/layer12_attention_weight_max": 53.92643737792969,
      "activations/layer12_attention_weight_min": -35.765193939208984,
      "activations/layer13_attention_weight_max": 70.20291900634766,
      "activations/layer13_attention_weight_min": -50.52467727661133,
      "activations/layer14_attention_weight_max": 73.48834228515625,
      "activations/layer14_attention_weight_min": -49.30997848510742,
      "activations/layer15_attention_weight_max": 56.302616119384766,
      "activations/layer15_attention_weight_min": -33.49471664428711,
      "activations/layer16_attention_weight_max": 46.732303619384766,
      "activations/layer16_attention_weight_min": -28.6184139251709,
      "activations/layer17_attention_weight_max": 46.278892517089844,
      "activations/layer17_attention_weight_min": -24.544633865356445,
      "activations/layer18_attention_weight_max": 33.194488525390625,
      "activations/layer18_attention_weight_min": -22.067758560180664,
      "activations/layer19_attention_weight_max": 47.6151237487793,
      "activations/layer19_attention_weight_min": -30.24027442932129,
      "activations/layer1_attention_weight_max": 20.17930793762207,
      "activations/layer1_attention_weight_min": -14.821560859680176,
      "activations/layer20_attention_weight_max": 44.064414978027344,
      "activations/layer20_attention_weight_min": -25.441072463989258,
      "activations/layer21_attention_weight_max": 37.750465393066406,
      "activations/layer21_attention_weight_min": -25.03355598449707,
      "activations/layer22_attention_weight_max": 54.41791534423828,
      "activations/layer22_attention_weight_min": -26.79745101928711,
      "activations/layer23_attention_weight_max": 30.353595733642578,
      "activations/layer23_attention_weight_min": -16.98680877685547,
      "activations/layer2_attention_weight_max": 31.786901473999023,
      "activations/layer2_attention_weight_min": -32.963905334472656,
      "activations/layer3_attention_weight_max": 93.78394317626953,
      "activations/layer3_attention_weight_min": -94.70222473144531,
      "activations/layer4_attention_weight_max": 145.45590209960938,
      "activations/layer4_attention_weight_min": -140.09078979492188,
      "activations/layer5_attention_weight_max": 106.25727844238281,
      "activations/layer5_attention_weight_min": -104.47071838378906,
      "activations/layer6_attention_weight_max": 66.59264373779297,
      "activations/layer6_attention_weight_min": -67.14620971679688,
      "activations/layer7_attention_weight_max": 99.47659301757812,
      "activations/layer7_attention_weight_min": -89.77876281738281,
      "activations/layer8_attention_weight_max": 52.74150466918945,
      "activations/layer8_attention_weight_min": -49.05177688598633,
      "activations/layer9_attention_weight_max": 54.959510803222656,
      "activations/layer9_attention_weight_min": -45.00620651245117,
      "epoch": 3.64,
      "learning_rate": 0.00012782878787878787,
      "loss": 2.8574,
      "step": 62600
    },
    {
      "activations/layer0_attention_weight_max": 17.13705062866211,
      "activations/layer0_attention_weight_min": -12.884692192077637,
      "activations/layer10_attention_weight_max": 37.46541213989258,
      "activations/layer10_attention_weight_min": -30.33568000793457,
      "activations/layer11_attention_weight_max": 37.98786544799805,
      "activations/layer11_attention_weight_min": -27.336851119995117,
      "activations/layer12_attention_weight_max": 45.7185173034668,
      "activations/layer12_attention_weight_min": -29.350370407104492,
      "activations/layer13_attention_weight_max": 50.01545333862305,
      "activations/layer13_attention_weight_min": -33.031036376953125,
      "activations/layer14_attention_weight_max": 79.3713607788086,
      "activations/layer14_attention_weight_min": -48.77643966674805,
      "activations/layer15_attention_weight_max": 42.052738189697266,
      "activations/layer15_attention_weight_min": -27.525131225585938,
      "activations/layer16_attention_weight_max": 39.558738708496094,
      "activations/layer16_attention_weight_min": -30.368301391601562,
      "activations/layer17_attention_weight_max": 41.79766082763672,
      "activations/layer17_attention_weight_min": -25.858713150024414,
      "activations/layer18_attention_weight_max": 35.07492446899414,
      "activations/layer18_attention_weight_min": -18.38078498840332,
      "activations/layer19_attention_weight_max": 49.505680084228516,
      "activations/layer19_attention_weight_min": -31.04984474182129,
      "activations/layer1_attention_weight_max": 18.296695709228516,
      "activations/layer1_attention_weight_min": -14.4053955078125,
      "activations/layer20_attention_weight_max": 34.45574951171875,
      "activations/layer20_attention_weight_min": -24.44957733154297,
      "activations/layer21_attention_weight_max": 36.328189849853516,
      "activations/layer21_attention_weight_min": -21.977720260620117,
      "activations/layer22_attention_weight_max": 52.107177734375,
      "activations/layer22_attention_weight_min": -27.923826217651367,
      "activations/layer23_attention_weight_max": 31.05866813659668,
      "activations/layer23_attention_weight_min": -19.647953033447266,
      "activations/layer2_attention_weight_max": 31.719451904296875,
      "activations/layer2_attention_weight_min": -30.641403198242188,
      "activations/layer3_attention_weight_max": 84.68860626220703,
      "activations/layer3_attention_weight_min": -88.01783752441406,
      "activations/layer4_attention_weight_max": 128.5408172607422,
      "activations/layer4_attention_weight_min": -133.9855194091797,
      "activations/layer5_attention_weight_max": 91.80741882324219,
      "activations/layer5_attention_weight_min": -87.21218872070312,
      "activations/layer6_attention_weight_max": 56.664154052734375,
      "activations/layer6_attention_weight_min": -56.952484130859375,
      "activations/layer7_attention_weight_max": 81.60566711425781,
      "activations/layer7_attention_weight_min": -74.83300018310547,
      "activations/layer8_attention_weight_max": 38.695613861083984,
      "activations/layer8_attention_weight_min": -38.21260452270508,
      "activations/layer9_attention_weight_max": 38.469364166259766,
      "activations/layer9_attention_weight_min": -33.81679916381836,
      "epoch": 3.64,
      "learning_rate": 0.00012780984848484848,
      "loss": 2.8839,
      "step": 62650
    },
    {
      "activations/layer0_attention_weight_max": 15.945669174194336,
      "activations/layer0_attention_weight_min": -11.141925811767578,
      "activations/layer10_attention_weight_max": 38.199214935302734,
      "activations/layer10_attention_weight_min": -31.18254280090332,
      "activations/layer11_attention_weight_max": 38.58720779418945,
      "activations/layer11_attention_weight_min": -27.871437072753906,
      "activations/layer12_attention_weight_max": 34.14577102661133,
      "activations/layer12_attention_weight_min": -26.88337516784668,
      "activations/layer13_attention_weight_max": 46.96974563598633,
      "activations/layer13_attention_weight_min": -34.78536605834961,
      "activations/layer14_attention_weight_max": 66.97191619873047,
      "activations/layer14_attention_weight_min": -48.182804107666016,
      "activations/layer15_attention_weight_max": 38.782466888427734,
      "activations/layer15_attention_weight_min": -28.09576988220215,
      "activations/layer16_attention_weight_max": 42.736488342285156,
      "activations/layer16_attention_weight_min": -28.602516174316406,
      "activations/layer17_attention_weight_max": 40.54353713989258,
      "activations/layer17_attention_weight_min": -24.51610565185547,
      "activations/layer18_attention_weight_max": 35.06534194946289,
      "activations/layer18_attention_weight_min": -19.154802322387695,
      "activations/layer19_attention_weight_max": 40.71836853027344,
      "activations/layer19_attention_weight_min": -28.26153564453125,
      "activations/layer1_attention_weight_max": 17.585233688354492,
      "activations/layer1_attention_weight_min": -14.81786060333252,
      "activations/layer20_attention_weight_max": 32.12295913696289,
      "activations/layer20_attention_weight_min": -21.91594696044922,
      "activations/layer21_attention_weight_max": 32.017372131347656,
      "activations/layer21_attention_weight_min": -20.194976806640625,
      "activations/layer22_attention_weight_max": 47.79575729370117,
      "activations/layer22_attention_weight_min": -29.37956428527832,
      "activations/layer23_attention_weight_max": 30.13134765625,
      "activations/layer23_attention_weight_min": -19.320648193359375,
      "activations/layer2_attention_weight_max": 30.45265769958496,
      "activations/layer2_attention_weight_min": -29.738990783691406,
      "activations/layer3_attention_weight_max": 81.80794525146484,
      "activations/layer3_attention_weight_min": -82.87109375,
      "activations/layer4_attention_weight_max": 127.0930404663086,
      "activations/layer4_attention_weight_min": -118.93498992919922,
      "activations/layer5_attention_weight_max": 95.83647155761719,
      "activations/layer5_attention_weight_min": -85.27464294433594,
      "activations/layer6_attention_weight_max": 58.58064651489258,
      "activations/layer6_attention_weight_min": -53.5788688659668,
      "activations/layer7_attention_weight_max": 75.42926788330078,
      "activations/layer7_attention_weight_min": -74.23290252685547,
      "activations/layer8_attention_weight_max": 43.19659423828125,
      "activations/layer8_attention_weight_min": -36.577720642089844,
      "activations/layer9_attention_weight_max": 56.79268264770508,
      "activations/layer9_attention_weight_min": -38.23482131958008,
      "epoch": 3.64,
      "learning_rate": 0.00012779090909090907,
      "loss": 2.9001,
      "step": 62700
    },
    {
      "activations/layer0_attention_weight_max": 17.382551193237305,
      "activations/layer0_attention_weight_min": -11.337617874145508,
      "activations/layer10_attention_weight_max": 42.637542724609375,
      "activations/layer10_attention_weight_min": -33.4291877746582,
      "activations/layer11_attention_weight_max": 44.13523864746094,
      "activations/layer11_attention_weight_min": -29.570751190185547,
      "activations/layer12_attention_weight_max": 62.284664154052734,
      "activations/layer12_attention_weight_min": -30.599143981933594,
      "activations/layer13_attention_weight_max": 71.10659790039062,
      "activations/layer13_attention_weight_min": -39.10185241699219,
      "activations/layer14_attention_weight_max": 105.37918853759766,
      "activations/layer14_attention_weight_min": -63.37171173095703,
      "activations/layer15_attention_weight_max": 52.589054107666016,
      "activations/layer15_attention_weight_min": -28.758913040161133,
      "activations/layer16_attention_weight_max": 49.50642013549805,
      "activations/layer16_attention_weight_min": -28.908628463745117,
      "activations/layer17_attention_weight_max": 61.36073303222656,
      "activations/layer17_attention_weight_min": -25.72772216796875,
      "activations/layer18_attention_weight_max": 56.5346565246582,
      "activations/layer18_attention_weight_min": -20.6800594329834,
      "activations/layer19_attention_weight_max": 52.7115592956543,
      "activations/layer19_attention_weight_min": -26.492544174194336,
      "activations/layer1_attention_weight_max": 17.715017318725586,
      "activations/layer1_attention_weight_min": -16.08313751220703,
      "activations/layer20_attention_weight_max": 36.931541442871094,
      "activations/layer20_attention_weight_min": -22.4388484954834,
      "activations/layer21_attention_weight_max": 37.913063049316406,
      "activations/layer21_attention_weight_min": -20.03246307373047,
      "activations/layer22_attention_weight_max": 53.26166534423828,
      "activations/layer22_attention_weight_min": -25.0123233795166,
      "activations/layer23_attention_weight_max": 29.57748031616211,
      "activations/layer23_attention_weight_min": -17.197925567626953,
      "activations/layer2_attention_weight_max": 33.048728942871094,
      "activations/layer2_attention_weight_min": -31.806753158569336,
      "activations/layer3_attention_weight_max": 87.49432373046875,
      "activations/layer3_attention_weight_min": -91.10772705078125,
      "activations/layer4_attention_weight_max": 134.4261016845703,
      "activations/layer4_attention_weight_min": -132.03355407714844,
      "activations/layer5_attention_weight_max": 101.50070190429688,
      "activations/layer5_attention_weight_min": -94.28886413574219,
      "activations/layer6_attention_weight_max": 60.66545104980469,
      "activations/layer6_attention_weight_min": -57.32608413696289,
      "activations/layer7_attention_weight_max": 86.60993194580078,
      "activations/layer7_attention_weight_min": -79.65150451660156,
      "activations/layer8_attention_weight_max": 46.37424850463867,
      "activations/layer8_attention_weight_min": -40.9095573425293,
      "activations/layer9_attention_weight_max": 49.52901077270508,
      "activations/layer9_attention_weight_min": -37.33991622924805,
      "epoch": 3.65,
      "learning_rate": 0.0001277719696969697,
      "loss": 2.894,
      "step": 62750
    },
    {
      "activations/layer0_attention_weight_max": 16.93130874633789,
      "activations/layer0_attention_weight_min": -11.733981132507324,
      "activations/layer10_attention_weight_max": 37.74900817871094,
      "activations/layer10_attention_weight_min": -30.287010192871094,
      "activations/layer11_attention_weight_max": 35.49118423461914,
      "activations/layer11_attention_weight_min": -28.228076934814453,
      "activations/layer12_attention_weight_max": 31.864965438842773,
      "activations/layer12_attention_weight_min": -29.53135108947754,
      "activations/layer13_attention_weight_max": 52.7250862121582,
      "activations/layer13_attention_weight_min": -33.36457824707031,
      "activations/layer14_attention_weight_max": 82.96182250976562,
      "activations/layer14_attention_weight_min": -51.71769714355469,
      "activations/layer15_attention_weight_max": 54.57093811035156,
      "activations/layer15_attention_weight_min": -31.944805145263672,
      "activations/layer16_attention_weight_max": 54.31100845336914,
      "activations/layer16_attention_weight_min": -32.00033187866211,
      "activations/layer17_attention_weight_max": 50.340885162353516,
      "activations/layer17_attention_weight_min": -27.66254425048828,
      "activations/layer18_attention_weight_max": 30.261032104492188,
      "activations/layer18_attention_weight_min": -18.845827102661133,
      "activations/layer19_attention_weight_max": 46.38731384277344,
      "activations/layer19_attention_weight_min": -29.26889419555664,
      "activations/layer1_attention_weight_max": 19.148822784423828,
      "activations/layer1_attention_weight_min": -12.988752365112305,
      "activations/layer20_attention_weight_max": 41.200862884521484,
      "activations/layer20_attention_weight_min": -23.810192108154297,
      "activations/layer21_attention_weight_max": 38.116127014160156,
      "activations/layer21_attention_weight_min": -21.371368408203125,
      "activations/layer22_attention_weight_max": 53.94685745239258,
      "activations/layer22_attention_weight_min": -25.991058349609375,
      "activations/layer23_attention_weight_max": 30.71576499938965,
      "activations/layer23_attention_weight_min": -15.997852325439453,
      "activations/layer2_attention_weight_max": 31.653060913085938,
      "activations/layer2_attention_weight_min": -30.987987518310547,
      "activations/layer3_attention_weight_max": 86.46356201171875,
      "activations/layer3_attention_weight_min": -88.1042709350586,
      "activations/layer4_attention_weight_max": 133.3285675048828,
      "activations/layer4_attention_weight_min": -132.0088653564453,
      "activations/layer5_attention_weight_max": 94.84941864013672,
      "activations/layer5_attention_weight_min": -90.43014526367188,
      "activations/layer6_attention_weight_max": 57.43288040161133,
      "activations/layer6_attention_weight_min": -55.871707916259766,
      "activations/layer7_attention_weight_max": 81.20648193359375,
      "activations/layer7_attention_weight_min": -77.25949096679688,
      "activations/layer8_attention_weight_max": 41.70004653930664,
      "activations/layer8_attention_weight_min": -39.48794174194336,
      "activations/layer9_attention_weight_max": 44.301597595214844,
      "activations/layer9_attention_weight_min": -32.678367614746094,
      "epoch": 3.65,
      "learning_rate": 0.0001277530303030303,
      "loss": 2.8882,
      "step": 62800
    },
    {
      "activations/layer0_attention_weight_max": 16.50150489807129,
      "activations/layer0_attention_weight_min": -11.085144996643066,
      "activations/layer10_attention_weight_max": 40.76655960083008,
      "activations/layer10_attention_weight_min": -32.110877990722656,
      "activations/layer11_attention_weight_max": 42.39034652709961,
      "activations/layer11_attention_weight_min": -32.5101203918457,
      "activations/layer12_attention_weight_max": 40.9548225402832,
      "activations/layer12_attention_weight_min": -28.420494079589844,
      "activations/layer13_attention_weight_max": 59.01718521118164,
      "activations/layer13_attention_weight_min": -40.192134857177734,
      "activations/layer14_attention_weight_max": 84.65997314453125,
      "activations/layer14_attention_weight_min": -62.3961067199707,
      "activations/layer15_attention_weight_max": 48.635746002197266,
      "activations/layer15_attention_weight_min": -30.49315071105957,
      "activations/layer16_attention_weight_max": 48.56499481201172,
      "activations/layer16_attention_weight_min": -32.30727767944336,
      "activations/layer17_attention_weight_max": 48.20900344848633,
      "activations/layer17_attention_weight_min": -26.89262580871582,
      "activations/layer18_attention_weight_max": 35.07087326049805,
      "activations/layer18_attention_weight_min": -20.073022842407227,
      "activations/layer19_attention_weight_max": 48.0393180847168,
      "activations/layer19_attention_weight_min": -28.109304428100586,
      "activations/layer1_attention_weight_max": 19.325429916381836,
      "activations/layer1_attention_weight_min": -14.934173583984375,
      "activations/layer20_attention_weight_max": 39.094791412353516,
      "activations/layer20_attention_weight_min": -27.21027183532715,
      "activations/layer21_attention_weight_max": 39.12100601196289,
      "activations/layer21_attention_weight_min": -18.63722801208496,
      "activations/layer22_attention_weight_max": 59.55337142944336,
      "activations/layer22_attention_weight_min": -28.093769073486328,
      "activations/layer23_attention_weight_max": 31.913190841674805,
      "activations/layer23_attention_weight_min": -18.64340591430664,
      "activations/layer2_attention_weight_max": 32.63653564453125,
      "activations/layer2_attention_weight_min": -32.00205993652344,
      "activations/layer3_attention_weight_max": 85.03814697265625,
      "activations/layer3_attention_weight_min": -87.98994445800781,
      "activations/layer4_attention_weight_max": 129.1848602294922,
      "activations/layer4_attention_weight_min": -130.41932678222656,
      "activations/layer5_attention_weight_max": 95.80471801757812,
      "activations/layer5_attention_weight_min": -93.96955871582031,
      "activations/layer6_attention_weight_max": 58.96794891357422,
      "activations/layer6_attention_weight_min": -55.193267822265625,
      "activations/layer7_attention_weight_max": 76.61311340332031,
      "activations/layer7_attention_weight_min": -82.0320053100586,
      "activations/layer8_attention_weight_max": 43.02573013305664,
      "activations/layer8_attention_weight_min": -39.21721267700195,
      "activations/layer9_attention_weight_max": 45.22892379760742,
      "activations/layer9_attention_weight_min": -45.75263595581055,
      "epoch": 3.65,
      "learning_rate": 0.0001277340909090909,
      "loss": 2.8938,
      "step": 62850
    },
    {
      "activations/layer0_attention_weight_max": 15.739350318908691,
      "activations/layer0_attention_weight_min": -11.875876426696777,
      "activations/layer10_attention_weight_max": 39.38785171508789,
      "activations/layer10_attention_weight_min": -30.89889907836914,
      "activations/layer11_attention_weight_max": 38.04237365722656,
      "activations/layer11_attention_weight_min": -30.411144256591797,
      "activations/layer12_attention_weight_max": 42.55274200439453,
      "activations/layer12_attention_weight_min": -28.196531295776367,
      "activations/layer13_attention_weight_max": 53.111427307128906,
      "activations/layer13_attention_weight_min": -36.70563888549805,
      "activations/layer14_attention_weight_max": 69.8825454711914,
      "activations/layer14_attention_weight_min": -51.24320983886719,
      "activations/layer15_attention_weight_max": 49.67930603027344,
      "activations/layer15_attention_weight_min": -33.79826736450195,
      "activations/layer16_attention_weight_max": 42.44688415527344,
      "activations/layer16_attention_weight_min": -29.853858947753906,
      "activations/layer17_attention_weight_max": 44.874202728271484,
      "activations/layer17_attention_weight_min": -26.550390243530273,
      "activations/layer18_attention_weight_max": 36.83174133300781,
      "activations/layer18_attention_weight_min": -20.272075653076172,
      "activations/layer19_attention_weight_max": 51.53883743286133,
      "activations/layer19_attention_weight_min": -30.193578720092773,
      "activations/layer1_attention_weight_max": 17.455659866333008,
      "activations/layer1_attention_weight_min": -15.228875160217285,
      "activations/layer20_attention_weight_max": 44.092037200927734,
      "activations/layer20_attention_weight_min": -23.74155616760254,
      "activations/layer21_attention_weight_max": 33.661293029785156,
      "activations/layer21_attention_weight_min": -21.285879135131836,
      "activations/layer22_attention_weight_max": 54.00081253051758,
      "activations/layer22_attention_weight_min": -29.013887405395508,
      "activations/layer23_attention_weight_max": 30.854915618896484,
      "activations/layer23_attention_weight_min": -17.85309600830078,
      "activations/layer2_attention_weight_max": 32.677059173583984,
      "activations/layer2_attention_weight_min": -30.185604095458984,
      "activations/layer3_attention_weight_max": 83.3571548461914,
      "activations/layer3_attention_weight_min": -86.29296112060547,
      "activations/layer4_attention_weight_max": 136.1815948486328,
      "activations/layer4_attention_weight_min": -133.0341796875,
      "activations/layer5_attention_weight_max": 95.46383666992188,
      "activations/layer5_attention_weight_min": -86.83891296386719,
      "activations/layer6_attention_weight_max": 56.015769958496094,
      "activations/layer6_attention_weight_min": -52.818538665771484,
      "activations/layer7_attention_weight_max": 74.97472381591797,
      "activations/layer7_attention_weight_min": -73.20578002929688,
      "activations/layer8_attention_weight_max": 38.51163864135742,
      "activations/layer8_attention_weight_min": -36.254337310791016,
      "activations/layer9_attention_weight_max": 40.82658386230469,
      "activations/layer9_attention_weight_min": -36.157447814941406,
      "epoch": 3.65,
      "learning_rate": 0.0001277151515151515,
      "loss": 2.8844,
      "step": 62900
    },
    {
      "activations/layer0_attention_weight_max": 15.098794937133789,
      "activations/layer0_attention_weight_min": -11.531487464904785,
      "activations/layer10_attention_weight_max": 40.86191177368164,
      "activations/layer10_attention_weight_min": -33.776607513427734,
      "activations/layer11_attention_weight_max": 36.96086120605469,
      "activations/layer11_attention_weight_min": -30.390586853027344,
      "activations/layer12_attention_weight_max": 44.07164001464844,
      "activations/layer12_attention_weight_min": -25.866456985473633,
      "activations/layer13_attention_weight_max": 50.085533142089844,
      "activations/layer13_attention_weight_min": -34.349365234375,
      "activations/layer14_attention_weight_max": 82.99603271484375,
      "activations/layer14_attention_weight_min": -57.670841217041016,
      "activations/layer15_attention_weight_max": 49.622772216796875,
      "activations/layer15_attention_weight_min": -31.518043518066406,
      "activations/layer16_attention_weight_max": 45.60879135131836,
      "activations/layer16_attention_weight_min": -29.302783966064453,
      "activations/layer17_attention_weight_max": 50.46257400512695,
      "activations/layer17_attention_weight_min": -25.2479190826416,
      "activations/layer18_attention_weight_max": 33.81591033935547,
      "activations/layer18_attention_weight_min": -19.269832611083984,
      "activations/layer19_attention_weight_max": 58.47561264038086,
      "activations/layer19_attention_weight_min": -30.638315200805664,
      "activations/layer1_attention_weight_max": 17.386198043823242,
      "activations/layer1_attention_weight_min": -14.280267715454102,
      "activations/layer20_attention_weight_max": 43.23024368286133,
      "activations/layer20_attention_weight_min": -22.933916091918945,
      "activations/layer21_attention_weight_max": 35.08991241455078,
      "activations/layer21_attention_weight_min": -20.008790969848633,
      "activations/layer22_attention_weight_max": 59.22180938720703,
      "activations/layer22_attention_weight_min": -30.412860870361328,
      "activations/layer23_attention_weight_max": 35.214229583740234,
      "activations/layer23_attention_weight_min": -19.449954986572266,
      "activations/layer2_attention_weight_max": 31.199542999267578,
      "activations/layer2_attention_weight_min": -30.302038192749023,
      "activations/layer3_attention_weight_max": 84.58548736572266,
      "activations/layer3_attention_weight_min": -87.06747436523438,
      "activations/layer4_attention_weight_max": 130.5216064453125,
      "activations/layer4_attention_weight_min": -125.6933364868164,
      "activations/layer5_attention_weight_max": 97.27505493164062,
      "activations/layer5_attention_weight_min": -88.7791748046875,
      "activations/layer6_attention_weight_max": 62.106727600097656,
      "activations/layer6_attention_weight_min": -56.48042678833008,
      "activations/layer7_attention_weight_max": 82.45524597167969,
      "activations/layer7_attention_weight_min": -81.46198272705078,
      "activations/layer8_attention_weight_max": 41.836875915527344,
      "activations/layer8_attention_weight_min": -41.33778381347656,
      "activations/layer9_attention_weight_max": 39.15851593017578,
      "activations/layer9_attention_weight_min": -32.56124496459961,
      "epoch": 3.66,
      "learning_rate": 0.00012769621212121212,
      "loss": 2.8699,
      "step": 62950
    },
    {
      "activations/layer0_attention_weight_max": 16.202783584594727,
      "activations/layer0_attention_weight_min": -10.967668533325195,
      "activations/layer10_attention_weight_max": 35.964115142822266,
      "activations/layer10_attention_weight_min": -30.874711990356445,
      "activations/layer11_attention_weight_max": 33.76608657836914,
      "activations/layer11_attention_weight_min": -26.93105697631836,
      "activations/layer12_attention_weight_max": 30.652286529541016,
      "activations/layer12_attention_weight_min": -30.027503967285156,
      "activations/layer13_attention_weight_max": 43.54084396362305,
      "activations/layer13_attention_weight_min": -32.249786376953125,
      "activations/layer14_attention_weight_max": 65.94021606445312,
      "activations/layer14_attention_weight_min": -47.748077392578125,
      "activations/layer15_attention_weight_max": 51.68003845214844,
      "activations/layer15_attention_weight_min": -31.291589736938477,
      "activations/layer16_attention_weight_max": 42.9801025390625,
      "activations/layer16_attention_weight_min": -29.937725067138672,
      "activations/layer17_attention_weight_max": 47.288352966308594,
      "activations/layer17_attention_weight_min": -27.817590713500977,
      "activations/layer18_attention_weight_max": 32.919464111328125,
      "activations/layer18_attention_weight_min": -19.964862823486328,
      "activations/layer19_attention_weight_max": 50.91160583496094,
      "activations/layer19_attention_weight_min": -28.0141544342041,
      "activations/layer1_attention_weight_max": 18.391477584838867,
      "activations/layer1_attention_weight_min": -13.577244758605957,
      "activations/layer20_attention_weight_max": 33.67261505126953,
      "activations/layer20_attention_weight_min": -23.69508934020996,
      "activations/layer21_attention_weight_max": 34.83576965332031,
      "activations/layer21_attention_weight_min": -20.46101951599121,
      "activations/layer22_attention_weight_max": 54.219261169433594,
      "activations/layer22_attention_weight_min": -28.6054630279541,
      "activations/layer23_attention_weight_max": 30.51369285583496,
      "activations/layer23_attention_weight_min": -19.30908966064453,
      "activations/layer2_attention_weight_max": 30.662044525146484,
      "activations/layer2_attention_weight_min": -31.109882354736328,
      "activations/layer3_attention_weight_max": 82.77909088134766,
      "activations/layer3_attention_weight_min": -82.16200256347656,
      "activations/layer4_attention_weight_max": 129.59547424316406,
      "activations/layer4_attention_weight_min": -128.19407653808594,
      "activations/layer5_attention_weight_max": 88.53685760498047,
      "activations/layer5_attention_weight_min": -85.31509399414062,
      "activations/layer6_attention_weight_max": 57.55429458618164,
      "activations/layer6_attention_weight_min": -54.74174118041992,
      "activations/layer7_attention_weight_max": 75.64153289794922,
      "activations/layer7_attention_weight_min": -73.98827362060547,
      "activations/layer8_attention_weight_max": 39.04705810546875,
      "activations/layer8_attention_weight_min": -39.77870178222656,
      "activations/layer9_attention_weight_max": 48.631649017333984,
      "activations/layer9_attention_weight_min": -35.29395294189453,
      "epoch": 3.66,
      "learning_rate": 0.0001276772727272727,
      "loss": 2.892,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_loss": 2.8359375,
      "eval_runtime": 8.5165,
      "eval_samples_per_second": 504.2,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_openwebtext_loss": 2.8359375,
      "eval_openwebtext_ppl": 17.046373787079762,
      "eval_openwebtext_runtime": 8.5165,
      "eval_openwebtext_samples_per_second": 504.2,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_wikitext_loss": 3.072265625,
      "eval_wikitext_ppl": 21.590763877931302,
      "eval_wikitext_runtime": 1.9961,
      "eval_wikitext_samples_per_second": 228.45,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_lambada_loss": 2.818359375,
      "eval_lambada_ppl": 16.74934871787156,
      "eval_lambada_runtime": 9.6461,
      "eval_lambada_samples_per_second": 504.765,
      "step": 63000
    },
    {
      "activations/layer0_attention_weight_max": 16.637107849121094,
      "activations/layer0_attention_weight_min": -12.352198600769043,
      "activations/layer10_attention_weight_max": 38.55971145629883,
      "activations/layer10_attention_weight_min": -35.7183952331543,
      "activations/layer11_attention_weight_max": 40.25090026855469,
      "activations/layer11_attention_weight_min": -35.66382598876953,
      "activations/layer12_attention_weight_max": 33.459228515625,
      "activations/layer12_attention_weight_min": -29.884401321411133,
      "activations/layer13_attention_weight_max": 60.93312072753906,
      "activations/layer13_attention_weight_min": -39.9815673828125,
      "activations/layer14_attention_weight_max": 64.46085357666016,
      "activations/layer14_attention_weight_min": -53.77094650268555,
      "activations/layer15_attention_weight_max": 43.000389099121094,
      "activations/layer15_attention_weight_min": -29.746627807617188,
      "activations/layer16_attention_weight_max": 42.28498077392578,
      "activations/layer16_attention_weight_min": -29.2974796295166,
      "activations/layer17_attention_weight_max": 42.93446350097656,
      "activations/layer17_attention_weight_min": -26.015975952148438,
      "activations/layer18_attention_weight_max": 31.066118240356445,
      "activations/layer18_attention_weight_min": -20.014888763427734,
      "activations/layer19_attention_weight_max": 45.63636016845703,
      "activations/layer19_attention_weight_min": -30.373886108398438,
      "activations/layer1_attention_weight_max": 17.342330932617188,
      "activations/layer1_attention_weight_min": -13.299610137939453,
      "activations/layer20_attention_weight_max": 32.05067443847656,
      "activations/layer20_attention_weight_min": -24.593965530395508,
      "activations/layer21_attention_weight_max": 30.723752975463867,
      "activations/layer21_attention_weight_min": -21.29184913635254,
      "activations/layer22_attention_weight_max": 46.53699493408203,
      "activations/layer22_attention_weight_min": -25.52248764038086,
      "activations/layer23_attention_weight_max": 27.931638717651367,
      "activations/layer23_attention_weight_min": -20.39151954650879,
      "activations/layer2_attention_weight_max": 31.005077362060547,
      "activations/layer2_attention_weight_min": -30.753387451171875,
      "activations/layer3_attention_weight_max": 84.11294555664062,
      "activations/layer3_attention_weight_min": -87.41129302978516,
      "activations/layer4_attention_weight_max": 134.5769805908203,
      "activations/layer4_attention_weight_min": -135.42747497558594,
      "activations/layer5_attention_weight_max": 98.95207214355469,
      "activations/layer5_attention_weight_min": -96.9853744506836,
      "activations/layer6_attention_weight_max": 57.351173400878906,
      "activations/layer6_attention_weight_min": -61.00636291503906,
      "activations/layer7_attention_weight_max": 74.44203186035156,
      "activations/layer7_attention_weight_min": -78.40646362304688,
      "activations/layer8_attention_weight_max": 41.96593475341797,
      "activations/layer8_attention_weight_min": -39.28508377075195,
      "activations/layer9_attention_weight_max": 43.5643196105957,
      "activations/layer9_attention_weight_min": -35.47093963623047,
      "epoch": 3.66,
      "learning_rate": 0.00012765833333333332,
      "loss": 2.8803,
      "step": 63050
    },
    {
      "activations/layer0_attention_weight_max": 15.142095565795898,
      "activations/layer0_attention_weight_min": -11.553531646728516,
      "activations/layer10_attention_weight_max": 37.149192810058594,
      "activations/layer10_attention_weight_min": -31.635398864746094,
      "activations/layer11_attention_weight_max": 36.923492431640625,
      "activations/layer11_attention_weight_min": -27.38486099243164,
      "activations/layer12_attention_weight_max": 58.93032455444336,
      "activations/layer12_attention_weight_min": -32.49126434326172,
      "activations/layer13_attention_weight_max": 51.217674255371094,
      "activations/layer13_attention_weight_min": -34.887611389160156,
      "activations/layer14_attention_weight_max": 95.5699234008789,
      "activations/layer14_attention_weight_min": -58.73721694946289,
      "activations/layer15_attention_weight_max": 50.867916107177734,
      "activations/layer15_attention_weight_min": -31.120512008666992,
      "activations/layer16_attention_weight_max": 50.150146484375,
      "activations/layer16_attention_weight_min": -30.47832489013672,
      "activations/layer17_attention_weight_max": 49.491302490234375,
      "activations/layer17_attention_weight_min": -26.419157028198242,
      "activations/layer18_attention_weight_max": 39.27920150756836,
      "activations/layer18_attention_weight_min": -21.103744506835938,
      "activations/layer19_attention_weight_max": 61.35538864135742,
      "activations/layer19_attention_weight_min": -28.433881759643555,
      "activations/layer1_attention_weight_max": 17.39939308166504,
      "activations/layer1_attention_weight_min": -13.84160327911377,
      "activations/layer20_attention_weight_max": 39.8354606628418,
      "activations/layer20_attention_weight_min": -23.55219841003418,
      "activations/layer21_attention_weight_max": 39.06956100463867,
      "activations/layer21_attention_weight_min": -19.829994201660156,
      "activations/layer22_attention_weight_max": 58.74073791503906,
      "activations/layer22_attention_weight_min": -28.096303939819336,
      "activations/layer23_attention_weight_max": 38.620758056640625,
      "activations/layer23_attention_weight_min": -20.73837661743164,
      "activations/layer2_attention_weight_max": 32.56088638305664,
      "activations/layer2_attention_weight_min": -30.74830436706543,
      "activations/layer3_attention_weight_max": 88.48529815673828,
      "activations/layer3_attention_weight_min": -87.65953063964844,
      "activations/layer4_attention_weight_max": 140.4493865966797,
      "activations/layer4_attention_weight_min": -128.4609832763672,
      "activations/layer5_attention_weight_max": 100.67473602294922,
      "activations/layer5_attention_weight_min": -86.11592864990234,
      "activations/layer6_attention_weight_max": 60.45502853393555,
      "activations/layer6_attention_weight_min": -56.286170959472656,
      "activations/layer7_attention_weight_max": 75.8495101928711,
      "activations/layer7_attention_weight_min": -74.22349548339844,
      "activations/layer8_attention_weight_max": 42.629703521728516,
      "activations/layer8_attention_weight_min": -37.30479431152344,
      "activations/layer9_attention_weight_max": 43.57847595214844,
      "activations/layer9_attention_weight_min": -33.427059173583984,
      "epoch": 3.67,
      "learning_rate": 0.00012763939393939394,
      "loss": 2.893,
      "step": 63100
    },
    {
      "activations/layer0_attention_weight_max": 16.563575744628906,
      "activations/layer0_attention_weight_min": -11.003262519836426,
      "activations/layer10_attention_weight_max": 37.68732452392578,
      "activations/layer10_attention_weight_min": -34.71698760986328,
      "activations/layer11_attention_weight_max": 37.3211555480957,
      "activations/layer11_attention_weight_min": -30.1176815032959,
      "activations/layer12_attention_weight_max": 31.437122344970703,
      "activations/layer12_attention_weight_min": -30.164051055908203,
      "activations/layer13_attention_weight_max": 66.47007751464844,
      "activations/layer13_attention_weight_min": -43.3060188293457,
      "activations/layer14_attention_weight_max": 68.98876953125,
      "activations/layer14_attention_weight_min": -52.23528289794922,
      "activations/layer15_attention_weight_max": 55.511146545410156,
      "activations/layer15_attention_weight_min": -34.09846878051758,
      "activations/layer16_attention_weight_max": 42.6596565246582,
      "activations/layer16_attention_weight_min": -32.24440002441406,
      "activations/layer17_attention_weight_max": 44.143959045410156,
      "activations/layer17_attention_weight_min": -26.8673095703125,
      "activations/layer18_attention_weight_max": 35.50858688354492,
      "activations/layer18_attention_weight_min": -23.996110916137695,
      "activations/layer19_attention_weight_max": 47.00944137573242,
      "activations/layer19_attention_weight_min": -30.113880157470703,
      "activations/layer1_attention_weight_max": 17.808517456054688,
      "activations/layer1_attention_weight_min": -14.372118949890137,
      "activations/layer20_attention_weight_max": 34.84148025512695,
      "activations/layer20_attention_weight_min": -25.007434844970703,
      "activations/layer21_attention_weight_max": 31.812150955200195,
      "activations/layer21_attention_weight_min": -19.46582794189453,
      "activations/layer22_attention_weight_max": 52.56491470336914,
      "activations/layer22_attention_weight_min": -27.11642837524414,
      "activations/layer23_attention_weight_max": 35.44280242919922,
      "activations/layer23_attention_weight_min": -17.73073959350586,
      "activations/layer2_attention_weight_max": 33.79795455932617,
      "activations/layer2_attention_weight_min": -32.681209564208984,
      "activations/layer3_attention_weight_max": 87.7142105102539,
      "activations/layer3_attention_weight_min": -86.56442260742188,
      "activations/layer4_attention_weight_max": 136.1807098388672,
      "activations/layer4_attention_weight_min": -130.9794464111328,
      "activations/layer5_attention_weight_max": 99.21974182128906,
      "activations/layer5_attention_weight_min": -92.67909240722656,
      "activations/layer6_attention_weight_max": 60.709381103515625,
      "activations/layer6_attention_weight_min": -61.36674118041992,
      "activations/layer7_attention_weight_max": 82.07530975341797,
      "activations/layer7_attention_weight_min": -84.95559692382812,
      "activations/layer8_attention_weight_max": 41.495399475097656,
      "activations/layer8_attention_weight_min": -41.723167419433594,
      "activations/layer9_attention_weight_max": 42.981502532958984,
      "activations/layer9_attention_weight_min": -37.68558883666992,
      "epoch": 3.67,
      "learning_rate": 0.00012762045454545453,
      "loss": 2.8792,
      "step": 63150
    },
    {
      "activations/layer0_attention_weight_max": 14.525843620300293,
      "activations/layer0_attention_weight_min": -11.027484893798828,
      "activations/layer10_attention_weight_max": 39.05495834350586,
      "activations/layer10_attention_weight_min": -30.717477798461914,
      "activations/layer11_attention_weight_max": 37.20213317871094,
      "activations/layer11_attention_weight_min": -30.89901351928711,
      "activations/layer12_attention_weight_max": 72.72975158691406,
      "activations/layer12_attention_weight_min": -35.885833740234375,
      "activations/layer13_attention_weight_max": 48.73562240600586,
      "activations/layer13_attention_weight_min": -36.1667594909668,
      "activations/layer14_attention_weight_max": 69.8208999633789,
      "activations/layer14_attention_weight_min": -52.49456787109375,
      "activations/layer15_attention_weight_max": 43.41849899291992,
      "activations/layer15_attention_weight_min": -30.10474967956543,
      "activations/layer16_attention_weight_max": 40.39665985107422,
      "activations/layer16_attention_weight_min": -30.694841384887695,
      "activations/layer17_attention_weight_max": 40.22547912597656,
      "activations/layer17_attention_weight_min": -24.705530166625977,
      "activations/layer18_attention_weight_max": 35.95489501953125,
      "activations/layer18_attention_weight_min": -20.983190536499023,
      "activations/layer19_attention_weight_max": 43.760921478271484,
      "activations/layer19_attention_weight_min": -29.320627212524414,
      "activations/layer1_attention_weight_max": 18.391122817993164,
      "activations/layer1_attention_weight_min": -16.80914878845215,
      "activations/layer20_attention_weight_max": 37.70241928100586,
      "activations/layer20_attention_weight_min": -22.590391159057617,
      "activations/layer21_attention_weight_max": 40.11608123779297,
      "activations/layer21_attention_weight_min": -21.68088722229004,
      "activations/layer22_attention_weight_max": 49.71404266357422,
      "activations/layer22_attention_weight_min": -25.738855361938477,
      "activations/layer23_attention_weight_max": 30.142210006713867,
      "activations/layer23_attention_weight_min": -16.47601890563965,
      "activations/layer2_attention_weight_max": 30.130971908569336,
      "activations/layer2_attention_weight_min": -30.24323844909668,
      "activations/layer3_attention_weight_max": 84.01122283935547,
      "activations/layer3_attention_weight_min": -90.08722686767578,
      "activations/layer4_attention_weight_max": 136.7482147216797,
      "activations/layer4_attention_weight_min": -131.30003356933594,
      "activations/layer5_attention_weight_max": 98.282958984375,
      "activations/layer5_attention_weight_min": -92.0178451538086,
      "activations/layer6_attention_weight_max": 58.07606506347656,
      "activations/layer6_attention_weight_min": -56.0643310546875,
      "activations/layer7_attention_weight_max": 76.48120880126953,
      "activations/layer7_attention_weight_min": -76.5803451538086,
      "activations/layer8_attention_weight_max": 42.02408218383789,
      "activations/layer8_attention_weight_min": -39.22150802612305,
      "activations/layer9_attention_weight_max": 45.6728630065918,
      "activations/layer9_attention_weight_min": -34.63505935668945,
      "epoch": 3.67,
      "learning_rate": 0.00012760151515151514,
      "loss": 2.902,
      "step": 63200
    },
    {
      "activations/layer0_attention_weight_max": 14.813817024230957,
      "activations/layer0_attention_weight_min": -11.323431015014648,
      "activations/layer10_attention_weight_max": 35.69984817504883,
      "activations/layer10_attention_weight_min": -31.01085090637207,
      "activations/layer11_attention_weight_max": 38.5863037109375,
      "activations/layer11_attention_weight_min": -32.35306930541992,
      "activations/layer12_attention_weight_max": 32.37354278564453,
      "activations/layer12_attention_weight_min": -30.6386775970459,
      "activations/layer13_attention_weight_max": 51.26114273071289,
      "activations/layer13_attention_weight_min": -34.168914794921875,
      "activations/layer14_attention_weight_max": 70.154296875,
      "activations/layer14_attention_weight_min": -48.296409606933594,
      "activations/layer15_attention_weight_max": 46.45629119873047,
      "activations/layer15_attention_weight_min": -32.49919891357422,
      "activations/layer16_attention_weight_max": 45.52670669555664,
      "activations/layer16_attention_weight_min": -30.12420654296875,
      "activations/layer17_attention_weight_max": 44.54302215576172,
      "activations/layer17_attention_weight_min": -28.469844818115234,
      "activations/layer18_attention_weight_max": 33.9184684753418,
      "activations/layer18_attention_weight_min": -18.941116333007812,
      "activations/layer19_attention_weight_max": 50.73866653442383,
      "activations/layer19_attention_weight_min": -29.479598999023438,
      "activations/layer1_attention_weight_max": 17.855243682861328,
      "activations/layer1_attention_weight_min": -15.68117618560791,
      "activations/layer20_attention_weight_max": 37.45072937011719,
      "activations/layer20_attention_weight_min": -24.227603912353516,
      "activations/layer21_attention_weight_max": 35.63991165161133,
      "activations/layer21_attention_weight_min": -22.009164810180664,
      "activations/layer22_attention_weight_max": 53.74353790283203,
      "activations/layer22_attention_weight_min": -27.37964630126953,
      "activations/layer23_attention_weight_max": 31.36254119873047,
      "activations/layer23_attention_weight_min": -17.654346466064453,
      "activations/layer2_attention_weight_max": 32.247230529785156,
      "activations/layer2_attention_weight_min": -30.218969345092773,
      "activations/layer3_attention_weight_max": 85.39462280273438,
      "activations/layer3_attention_weight_min": -86.9596176147461,
      "activations/layer4_attention_weight_max": 132.86485290527344,
      "activations/layer4_attention_weight_min": -126.75662994384766,
      "activations/layer5_attention_weight_max": 97.50959777832031,
      "activations/layer5_attention_weight_min": -88.12957000732422,
      "activations/layer6_attention_weight_max": 58.830413818359375,
      "activations/layer6_attention_weight_min": -54.90168762207031,
      "activations/layer7_attention_weight_max": 87.17909240722656,
      "activations/layer7_attention_weight_min": -79.18509674072266,
      "activations/layer8_attention_weight_max": 39.1911506652832,
      "activations/layer8_attention_weight_min": -38.229827880859375,
      "activations/layer9_attention_weight_max": 38.570186614990234,
      "activations/layer9_attention_weight_min": -39.869686126708984,
      "epoch": 3.68,
      "learning_rate": 0.00012758257575757573,
      "loss": 2.8827,
      "step": 63250
    },
    {
      "activations/layer0_attention_weight_max": 15.609297752380371,
      "activations/layer0_attention_weight_min": -11.709364891052246,
      "activations/layer10_attention_weight_max": 43.37310791015625,
      "activations/layer10_attention_weight_min": -33.95978546142578,
      "activations/layer11_attention_weight_max": 40.2476921081543,
      "activations/layer11_attention_weight_min": -29.02072525024414,
      "activations/layer12_attention_weight_max": 31.28529167175293,
      "activations/layer12_attention_weight_min": -27.213233947753906,
      "activations/layer13_attention_weight_max": 51.658592224121094,
      "activations/layer13_attention_weight_min": -34.60333251953125,
      "activations/layer14_attention_weight_max": 72.93889617919922,
      "activations/layer14_attention_weight_min": -56.077274322509766,
      "activations/layer15_attention_weight_max": 66.37692260742188,
      "activations/layer15_attention_weight_min": -34.99204635620117,
      "activations/layer16_attention_weight_max": 45.47053527832031,
      "activations/layer16_attention_weight_min": -29.06869125366211,
      "activations/layer17_attention_weight_max": 46.75385665893555,
      "activations/layer17_attention_weight_min": -26.38361930847168,
      "activations/layer18_attention_weight_max": 31.597862243652344,
      "activations/layer18_attention_weight_min": -19.762052536010742,
      "activations/layer19_attention_weight_max": 49.23221969604492,
      "activations/layer19_attention_weight_min": -28.383886337280273,
      "activations/layer1_attention_weight_max": 17.538434982299805,
      "activations/layer1_attention_weight_min": -15.875524520874023,
      "activations/layer20_attention_weight_max": 34.04231262207031,
      "activations/layer20_attention_weight_min": -22.37774658203125,
      "activations/layer21_attention_weight_max": 32.85801315307617,
      "activations/layer21_attention_weight_min": -20.025150299072266,
      "activations/layer22_attention_weight_max": 53.41685104370117,
      "activations/layer22_attention_weight_min": -27.479965209960938,
      "activations/layer23_attention_weight_max": 29.035133361816406,
      "activations/layer23_attention_weight_min": -17.285953521728516,
      "activations/layer2_attention_weight_max": 32.56542205810547,
      "activations/layer2_attention_weight_min": -31.61475944519043,
      "activations/layer3_attention_weight_max": 88.81010437011719,
      "activations/layer3_attention_weight_min": -91.0429458618164,
      "activations/layer4_attention_weight_max": 138.9913787841797,
      "activations/layer4_attention_weight_min": -132.7659149169922,
      "activations/layer5_attention_weight_max": 100.11167907714844,
      "activations/layer5_attention_weight_min": -92.89791107177734,
      "activations/layer6_attention_weight_max": 58.40135955810547,
      "activations/layer6_attention_weight_min": -58.55678176879883,
      "activations/layer7_attention_weight_max": 87.46790313720703,
      "activations/layer7_attention_weight_min": -73.73702239990234,
      "activations/layer8_attention_weight_max": 43.3235969543457,
      "activations/layer8_attention_weight_min": -39.12733840942383,
      "activations/layer9_attention_weight_max": 44.306270599365234,
      "activations/layer9_attention_weight_min": -35.339752197265625,
      "epoch": 3.68,
      "learning_rate": 0.00012756363636363634,
      "loss": 2.8814,
      "step": 63300
    },
    {
      "activations/layer0_attention_weight_max": 15.880935668945312,
      "activations/layer0_attention_weight_min": -11.299966812133789,
      "activations/layer10_attention_weight_max": 37.80323791503906,
      "activations/layer10_attention_weight_min": -28.341772079467773,
      "activations/layer11_attention_weight_max": 34.76884078979492,
      "activations/layer11_attention_weight_min": -27.275978088378906,
      "activations/layer12_attention_weight_max": 42.193115234375,
      "activations/layer12_attention_weight_min": -28.878664016723633,
      "activations/layer13_attention_weight_max": 47.220096588134766,
      "activations/layer13_attention_weight_min": -37.00531005859375,
      "activations/layer14_attention_weight_max": 66.95916748046875,
      "activations/layer14_attention_weight_min": -47.50372314453125,
      "activations/layer15_attention_weight_max": 46.15343475341797,
      "activations/layer15_attention_weight_min": -30.829090118408203,
      "activations/layer16_attention_weight_max": 42.72470474243164,
      "activations/layer16_attention_weight_min": -28.578571319580078,
      "activations/layer17_attention_weight_max": 44.65210723876953,
      "activations/layer17_attention_weight_min": -27.169191360473633,
      "activations/layer18_attention_weight_max": 34.320621490478516,
      "activations/layer18_attention_weight_min": -20.394363403320312,
      "activations/layer19_attention_weight_max": 47.74090576171875,
      "activations/layer19_attention_weight_min": -29.442121505737305,
      "activations/layer1_attention_weight_max": 17.9576473236084,
      "activations/layer1_attention_weight_min": -14.145512580871582,
      "activations/layer20_attention_weight_max": 40.760868072509766,
      "activations/layer20_attention_weight_min": -23.183624267578125,
      "activations/layer21_attention_weight_max": 41.80433654785156,
      "activations/layer21_attention_weight_min": -19.965173721313477,
      "activations/layer22_attention_weight_max": 54.96718978881836,
      "activations/layer22_attention_weight_min": -26.89093589782715,
      "activations/layer23_attention_weight_max": 31.526798248291016,
      "activations/layer23_attention_weight_min": -18.255502700805664,
      "activations/layer2_attention_weight_max": 30.956083297729492,
      "activations/layer2_attention_weight_min": -30.039321899414062,
      "activations/layer3_attention_weight_max": 83.48957824707031,
      "activations/layer3_attention_weight_min": -86.96257781982422,
      "activations/layer4_attention_weight_max": 133.57211303710938,
      "activations/layer4_attention_weight_min": -133.33242797851562,
      "activations/layer5_attention_weight_max": 92.3335189819336,
      "activations/layer5_attention_weight_min": -85.20735931396484,
      "activations/layer6_attention_weight_max": 60.29588317871094,
      "activations/layer6_attention_weight_min": -53.41365051269531,
      "activations/layer7_attention_weight_max": 79.68896484375,
      "activations/layer7_attention_weight_min": -75.09537506103516,
      "activations/layer8_attention_weight_max": 42.12400436401367,
      "activations/layer8_attention_weight_min": -37.76491165161133,
      "activations/layer9_attention_weight_max": 38.84232711791992,
      "activations/layer9_attention_weight_min": -30.917688369750977,
      "epoch": 3.68,
      "learning_rate": 0.00012754469696969696,
      "loss": 2.8981,
      "step": 63350
    },
    {
      "activations/layer0_attention_weight_max": 15.971978187561035,
      "activations/layer0_attention_weight_min": -12.6874361038208,
      "activations/layer10_attention_weight_max": 37.776222229003906,
      "activations/layer10_attention_weight_min": -31.522594451904297,
      "activations/layer11_attention_weight_max": 35.75370788574219,
      "activations/layer11_attention_weight_min": -27.81875228881836,
      "activations/layer12_attention_weight_max": 53.928863525390625,
      "activations/layer12_attention_weight_min": -28.52387237548828,
      "activations/layer13_attention_weight_max": 68.2420654296875,
      "activations/layer13_attention_weight_min": -52.0915412902832,
      "activations/layer14_attention_weight_max": 76.39397430419922,
      "activations/layer14_attention_weight_min": -59.71564483642578,
      "activations/layer15_attention_weight_max": 44.942893981933594,
      "activations/layer15_attention_weight_min": -33.9986686706543,
      "activations/layer16_attention_weight_max": 44.44578170776367,
      "activations/layer16_attention_weight_min": -29.165523529052734,
      "activations/layer17_attention_weight_max": 44.80243682861328,
      "activations/layer17_attention_weight_min": -25.525623321533203,
      "activations/layer18_attention_weight_max": 30.39315414428711,
      "activations/layer18_attention_weight_min": -22.29126739501953,
      "activations/layer19_attention_weight_max": 41.195945739746094,
      "activations/layer19_attention_weight_min": -29.309207916259766,
      "activations/layer1_attention_weight_max": 17.77494239807129,
      "activations/layer1_attention_weight_min": -14.634029388427734,
      "activations/layer20_attention_weight_max": 34.792110443115234,
      "activations/layer20_attention_weight_min": -21.953380584716797,
      "activations/layer21_attention_weight_max": 34.524169921875,
      "activations/layer21_attention_weight_min": -21.140188217163086,
      "activations/layer22_attention_weight_max": 51.31251525878906,
      "activations/layer22_attention_weight_min": -26.59819793701172,
      "activations/layer23_attention_weight_max": 35.53515625,
      "activations/layer23_attention_weight_min": -19.110885620117188,
      "activations/layer2_attention_weight_max": 32.54620361328125,
      "activations/layer2_attention_weight_min": -31.361385345458984,
      "activations/layer3_attention_weight_max": 85.50081634521484,
      "activations/layer3_attention_weight_min": -85.50991821289062,
      "activations/layer4_attention_weight_max": 131.83689880371094,
      "activations/layer4_attention_weight_min": -127.56999969482422,
      "activations/layer5_attention_weight_max": 90.23082733154297,
      "activations/layer5_attention_weight_min": -85.38919067382812,
      "activations/layer6_attention_weight_max": 56.4234733581543,
      "activations/layer6_attention_weight_min": -54.03269577026367,
      "activations/layer7_attention_weight_max": 73.20394134521484,
      "activations/layer7_attention_weight_min": -74.67562103271484,
      "activations/layer8_attention_weight_max": 38.70796585083008,
      "activations/layer8_attention_weight_min": -36.94527053833008,
      "activations/layer9_attention_weight_max": 42.16690444946289,
      "activations/layer9_attention_weight_min": -39.73789596557617,
      "epoch": 3.68,
      "learning_rate": 0.00012752575757575755,
      "loss": 2.8985,
      "step": 63400
    },
    {
      "activations/layer0_attention_weight_max": 16.472692489624023,
      "activations/layer0_attention_weight_min": -11.135086059570312,
      "activations/layer10_attention_weight_max": 39.448646545410156,
      "activations/layer10_attention_weight_min": -32.834957122802734,
      "activations/layer11_attention_weight_max": 39.13375473022461,
      "activations/layer11_attention_weight_min": -28.20618438720703,
      "activations/layer12_attention_weight_max": 38.45519256591797,
      "activations/layer12_attention_weight_min": -27.614383697509766,
      "activations/layer13_attention_weight_max": 64.83659362792969,
      "activations/layer13_attention_weight_min": -39.48553466796875,
      "activations/layer14_attention_weight_max": 111.25040435791016,
      "activations/layer14_attention_weight_min": -79.76940155029297,
      "activations/layer15_attention_weight_max": 44.26455307006836,
      "activations/layer15_attention_weight_min": -31.16632843017578,
      "activations/layer16_attention_weight_max": 47.39290237426758,
      "activations/layer16_attention_weight_min": -29.516571044921875,
      "activations/layer17_attention_weight_max": 54.87997055053711,
      "activations/layer17_attention_weight_min": -25.755355834960938,
      "activations/layer18_attention_weight_max": 37.61127853393555,
      "activations/layer18_attention_weight_min": -22.29857063293457,
      "activations/layer19_attention_weight_max": 45.39469528198242,
      "activations/layer19_attention_weight_min": -29.655935287475586,
      "activations/layer1_attention_weight_max": 17.35929298400879,
      "activations/layer1_attention_weight_min": -15.101644515991211,
      "activations/layer20_attention_weight_max": 40.55029296875,
      "activations/layer20_attention_weight_min": -23.05201530456543,
      "activations/layer21_attention_weight_max": 32.52457046508789,
      "activations/layer21_attention_weight_min": -22.129125595092773,
      "activations/layer22_attention_weight_max": 54.6724853515625,
      "activations/layer22_attention_weight_min": -26.79559898376465,
      "activations/layer23_attention_weight_max": 30.135623931884766,
      "activations/layer23_attention_weight_min": -18.306760787963867,
      "activations/layer2_attention_weight_max": 32.627403259277344,
      "activations/layer2_attention_weight_min": -31.124446868896484,
      "activations/layer3_attention_weight_max": 90.38335418701172,
      "activations/layer3_attention_weight_min": -89.80511474609375,
      "activations/layer4_attention_weight_max": 143.67735290527344,
      "activations/layer4_attention_weight_min": -137.53050231933594,
      "activations/layer5_attention_weight_max": 108.69711303710938,
      "activations/layer5_attention_weight_min": -99.3150863647461,
      "activations/layer6_attention_weight_max": 68.96862030029297,
      "activations/layer6_attention_weight_min": -58.75053787231445,
      "activations/layer7_attention_weight_max": 84.35721588134766,
      "activations/layer7_attention_weight_min": -82.71759033203125,
      "activations/layer8_attention_weight_max": 45.98220443725586,
      "activations/layer8_attention_weight_min": -39.64143753051758,
      "activations/layer9_attention_weight_max": 39.74146270751953,
      "activations/layer9_attention_weight_min": -35.68098068237305,
      "epoch": 3.69,
      "learning_rate": 0.00012750681818181816,
      "loss": 2.8852,
      "step": 63450
    },
    {
      "activations/layer0_attention_weight_max": 16.306209564208984,
      "activations/layer0_attention_weight_min": -12.518001556396484,
      "activations/layer10_attention_weight_max": 38.12107467651367,
      "activations/layer10_attention_weight_min": -31.20861053466797,
      "activations/layer11_attention_weight_max": 34.29580307006836,
      "activations/layer11_attention_weight_min": -28.664539337158203,
      "activations/layer12_attention_weight_max": 34.4952507019043,
      "activations/layer12_attention_weight_min": -27.77560043334961,
      "activations/layer13_attention_weight_max": 49.49016571044922,
      "activations/layer13_attention_weight_min": -34.87220764160156,
      "activations/layer14_attention_weight_max": 72.75914001464844,
      "activations/layer14_attention_weight_min": -54.78135681152344,
      "activations/layer15_attention_weight_max": 45.06385040283203,
      "activations/layer15_attention_weight_min": -27.808788299560547,
      "activations/layer16_attention_weight_max": 45.021480560302734,
      "activations/layer16_attention_weight_min": -27.288366317749023,
      "activations/layer17_attention_weight_max": 45.22140121459961,
      "activations/layer17_attention_weight_min": -25.881567001342773,
      "activations/layer18_attention_weight_max": 31.935579299926758,
      "activations/layer18_attention_weight_min": -19.98729705810547,
      "activations/layer19_attention_weight_max": 44.005210876464844,
      "activations/layer19_attention_weight_min": -28.338592529296875,
      "activations/layer1_attention_weight_max": 17.63288116455078,
      "activations/layer1_attention_weight_min": -14.892706871032715,
      "activations/layer20_attention_weight_max": 36.28268814086914,
      "activations/layer20_attention_weight_min": -24.95220375061035,
      "activations/layer21_attention_weight_max": 32.94741439819336,
      "activations/layer21_attention_weight_min": -20.253528594970703,
      "activations/layer22_attention_weight_max": 51.7928352355957,
      "activations/layer22_attention_weight_min": -29.626794815063477,
      "activations/layer23_attention_weight_max": 30.78364372253418,
      "activations/layer23_attention_weight_min": -17.46007537841797,
      "activations/layer2_attention_weight_max": 32.02191162109375,
      "activations/layer2_attention_weight_min": -33.24376678466797,
      "activations/layer3_attention_weight_max": 87.93379974365234,
      "activations/layer3_attention_weight_min": -88.93668365478516,
      "activations/layer4_attention_weight_max": 133.4968719482422,
      "activations/layer4_attention_weight_min": -128.82923889160156,
      "activations/layer5_attention_weight_max": 103.27906799316406,
      "activations/layer5_attention_weight_min": -92.23374938964844,
      "activations/layer6_attention_weight_max": 60.87495422363281,
      "activations/layer6_attention_weight_min": -53.159183502197266,
      "activations/layer7_attention_weight_max": 73.81095886230469,
      "activations/layer7_attention_weight_min": -74.5568618774414,
      "activations/layer8_attention_weight_max": 40.89238357543945,
      "activations/layer8_attention_weight_min": -37.2016487121582,
      "activations/layer9_attention_weight_max": 38.34597396850586,
      "activations/layer9_attention_weight_min": -34.27194595336914,
      "epoch": 3.69,
      "learning_rate": 0.00012748787878787878,
      "loss": 2.8787,
      "step": 63500
    },
    {
      "activations/layer0_attention_weight_max": 16.309791564941406,
      "activations/layer0_attention_weight_min": -11.698746681213379,
      "activations/layer10_attention_weight_max": 34.70966720581055,
      "activations/layer10_attention_weight_min": -32.337215423583984,
      "activations/layer11_attention_weight_max": 37.98112106323242,
      "activations/layer11_attention_weight_min": -26.49152183532715,
      "activations/layer12_attention_weight_max": 30.192974090576172,
      "activations/layer12_attention_weight_min": -28.434131622314453,
      "activations/layer13_attention_weight_max": 39.13999938964844,
      "activations/layer13_attention_weight_min": -31.34949493408203,
      "activations/layer14_attention_weight_max": 56.863990783691406,
      "activations/layer14_attention_weight_min": -47.30084991455078,
      "activations/layer15_attention_weight_max": 43.524078369140625,
      "activations/layer15_attention_weight_min": -29.783090591430664,
      "activations/layer16_attention_weight_max": 40.710853576660156,
      "activations/layer16_attention_weight_min": -28.9060115814209,
      "activations/layer17_attention_weight_max": 38.156768798828125,
      "activations/layer17_attention_weight_min": -25.908029556274414,
      "activations/layer18_attention_weight_max": 33.31703567504883,
      "activations/layer18_attention_weight_min": -19.3275146484375,
      "activations/layer19_attention_weight_max": 41.72056198120117,
      "activations/layer19_attention_weight_min": -28.721399307250977,
      "activations/layer1_attention_weight_max": 17.33175277709961,
      "activations/layer1_attention_weight_min": -14.108176231384277,
      "activations/layer20_attention_weight_max": 30.651559829711914,
      "activations/layer20_attention_weight_min": -22.701364517211914,
      "activations/layer21_attention_weight_max": 36.42107009887695,
      "activations/layer21_attention_weight_min": -23.459667205810547,
      "activations/layer22_attention_weight_max": 49.91242218017578,
      "activations/layer22_attention_weight_min": -28.67708396911621,
      "activations/layer23_attention_weight_max": 30.037885665893555,
      "activations/layer23_attention_weight_min": -19.342086791992188,
      "activations/layer2_attention_weight_max": 32.165504455566406,
      "activations/layer2_attention_weight_min": -31.448461532592773,
      "activations/layer3_attention_weight_max": 86.24104309082031,
      "activations/layer3_attention_weight_min": -86.23131561279297,
      "activations/layer4_attention_weight_max": 136.4273223876953,
      "activations/layer4_attention_weight_min": -128.95761108398438,
      "activations/layer5_attention_weight_max": 96.1474838256836,
      "activations/layer5_attention_weight_min": -90.66500854492188,
      "activations/layer6_attention_weight_max": 56.35148620605469,
      "activations/layer6_attention_weight_min": -56.77790832519531,
      "activations/layer7_attention_weight_max": 77.58190155029297,
      "activations/layer7_attention_weight_min": -73.89683532714844,
      "activations/layer8_attention_weight_max": 39.62522506713867,
      "activations/layer8_attention_weight_min": -38.88252258300781,
      "activations/layer9_attention_weight_max": 37.91164016723633,
      "activations/layer9_attention_weight_min": -33.649166107177734,
      "epoch": 3.69,
      "learning_rate": 0.00012746893939393937,
      "loss": 2.8865,
      "step": 63550
    },
    {
      "activations/layer0_attention_weight_max": 16.483863830566406,
      "activations/layer0_attention_weight_min": -12.195326805114746,
      "activations/layer10_attention_weight_max": 39.7501335144043,
      "activations/layer10_attention_weight_min": -29.398494720458984,
      "activations/layer11_attention_weight_max": 36.01474380493164,
      "activations/layer11_attention_weight_min": -26.3618221282959,
      "activations/layer12_attention_weight_max": 43.897186279296875,
      "activations/layer12_attention_weight_min": -29.3609676361084,
      "activations/layer13_attention_weight_max": 46.9565315246582,
      "activations/layer13_attention_weight_min": -31.24881362915039,
      "activations/layer14_attention_weight_max": 92.20235443115234,
      "activations/layer14_attention_weight_min": -52.206886291503906,
      "activations/layer15_attention_weight_max": 45.45000457763672,
      "activations/layer15_attention_weight_min": -30.502649307250977,
      "activations/layer16_attention_weight_max": 44.84221649169922,
      "activations/layer16_attention_weight_min": -28.967485427856445,
      "activations/layer17_attention_weight_max": 43.686981201171875,
      "activations/layer17_attention_weight_min": -25.376876831054688,
      "activations/layer18_attention_weight_max": 39.486656188964844,
      "activations/layer18_attention_weight_min": -20.338045120239258,
      "activations/layer19_attention_weight_max": 51.444923400878906,
      "activations/layer19_attention_weight_min": -28.431913375854492,
      "activations/layer1_attention_weight_max": 18.84331703186035,
      "activations/layer1_attention_weight_min": -13.366890907287598,
      "activations/layer20_attention_weight_max": 37.88584518432617,
      "activations/layer20_attention_weight_min": -22.77926254272461,
      "activations/layer21_attention_weight_max": 33.89160919189453,
      "activations/layer21_attention_weight_min": -21.18330955505371,
      "activations/layer22_attention_weight_max": 55.69365310668945,
      "activations/layer22_attention_weight_min": -27.264482498168945,
      "activations/layer23_attention_weight_max": 32.5579948425293,
      "activations/layer23_attention_weight_min": -20.910003662109375,
      "activations/layer2_attention_weight_max": 31.527423858642578,
      "activations/layer2_attention_weight_min": -30.727163314819336,
      "activations/layer3_attention_weight_max": 85.1852798461914,
      "activations/layer3_attention_weight_min": -85.06475830078125,
      "activations/layer4_attention_weight_max": 142.160400390625,
      "activations/layer4_attention_weight_min": -121.8888931274414,
      "activations/layer5_attention_weight_max": 100.03636932373047,
      "activations/layer5_attention_weight_min": -86.12284851074219,
      "activations/layer6_attention_weight_max": 54.0292854309082,
      "activations/layer6_attention_weight_min": -54.270103454589844,
      "activations/layer7_attention_weight_max": 77.74568176269531,
      "activations/layer7_attention_weight_min": -75.61580657958984,
      "activations/layer8_attention_weight_max": 41.63062286376953,
      "activations/layer8_attention_weight_min": -36.10987091064453,
      "activations/layer9_attention_weight_max": 42.632957458496094,
      "activations/layer9_attention_weight_min": -39.47568893432617,
      "epoch": 3.7,
      "learning_rate": 0.00012744999999999998,
      "loss": 2.8931,
      "step": 63600
    },
    {
      "activations/layer0_attention_weight_max": 16.613588333129883,
      "activations/layer0_attention_weight_min": -11.333244323730469,
      "activations/layer10_attention_weight_max": 34.96135330200195,
      "activations/layer10_attention_weight_min": -31.81975555419922,
      "activations/layer11_attention_weight_max": 36.716033935546875,
      "activations/layer11_attention_weight_min": -30.728391647338867,
      "activations/layer12_attention_weight_max": 31.659225463867188,
      "activations/layer12_attention_weight_min": -28.497602462768555,
      "activations/layer13_attention_weight_max": 52.24095916748047,
      "activations/layer13_attention_weight_min": -42.034812927246094,
      "activations/layer14_attention_weight_max": 68.6693115234375,
      "activations/layer14_attention_weight_min": -52.8798942565918,
      "activations/layer15_attention_weight_max": 40.834415435791016,
      "activations/layer15_attention_weight_min": -29.934139251708984,
      "activations/layer16_attention_weight_max": 40.922332763671875,
      "activations/layer16_attention_weight_min": -28.34208106994629,
      "activations/layer17_attention_weight_max": 43.4474983215332,
      "activations/layer17_attention_weight_min": -24.008567810058594,
      "activations/layer18_attention_weight_max": 33.19435119628906,
      "activations/layer18_attention_weight_min": -20.280595779418945,
      "activations/layer19_attention_weight_max": 41.85227584838867,
      "activations/layer19_attention_weight_min": -29.57232666015625,
      "activations/layer1_attention_weight_max": 17.140037536621094,
      "activations/layer1_attention_weight_min": -14.798638343811035,
      "activations/layer20_attention_weight_max": 35.30335998535156,
      "activations/layer20_attention_weight_min": -22.926652908325195,
      "activations/layer21_attention_weight_max": 32.048423767089844,
      "activations/layer21_attention_weight_min": -20.592933654785156,
      "activations/layer22_attention_weight_max": 51.421993255615234,
      "activations/layer22_attention_weight_min": -26.33427619934082,
      "activations/layer23_attention_weight_max": 30.23663330078125,
      "activations/layer23_attention_weight_min": -17.06303596496582,
      "activations/layer2_attention_weight_max": 30.827194213867188,
      "activations/layer2_attention_weight_min": -31.308094024658203,
      "activations/layer3_attention_weight_max": 87.40398406982422,
      "activations/layer3_attention_weight_min": -88.71548461914062,
      "activations/layer4_attention_weight_max": 139.63919067382812,
      "activations/layer4_attention_weight_min": -136.04153442382812,
      "activations/layer5_attention_weight_max": 98.92867279052734,
      "activations/layer5_attention_weight_min": -93.14581298828125,
      "activations/layer6_attention_weight_max": 57.45317077636719,
      "activations/layer6_attention_weight_min": -55.14389419555664,
      "activations/layer7_attention_weight_max": 79.08010864257812,
      "activations/layer7_attention_weight_min": -85.92599487304688,
      "activations/layer8_attention_weight_max": 39.81105422973633,
      "activations/layer8_attention_weight_min": -42.5108528137207,
      "activations/layer9_attention_weight_max": 47.47593307495117,
      "activations/layer9_attention_weight_min": -38.43539810180664,
      "epoch": 3.7,
      "learning_rate": 0.0001274310606060606,
      "loss": 2.8986,
      "step": 63650
    },
    {
      "activations/layer0_attention_weight_max": 15.993640899658203,
      "activations/layer0_attention_weight_min": -12.16690444946289,
      "activations/layer10_attention_weight_max": 37.14906692504883,
      "activations/layer10_attention_weight_min": -31.83565902709961,
      "activations/layer11_attention_weight_max": 35.72633361816406,
      "activations/layer11_attention_weight_min": -27.907550811767578,
      "activations/layer12_attention_weight_max": 29.62076187133789,
      "activations/layer12_attention_weight_min": -30.886247634887695,
      "activations/layer13_attention_weight_max": 45.72494888305664,
      "activations/layer13_attention_weight_min": -35.242279052734375,
      "activations/layer14_attention_weight_max": 78.88079833984375,
      "activations/layer14_attention_weight_min": -58.85375213623047,
      "activations/layer15_attention_weight_max": 42.2746467590332,
      "activations/layer15_attention_weight_min": -31.61880874633789,
      "activations/layer16_attention_weight_max": 39.653507232666016,
      "activations/layer16_attention_weight_min": -31.192989349365234,
      "activations/layer17_attention_weight_max": 39.48979187011719,
      "activations/layer17_attention_weight_min": -26.13640785217285,
      "activations/layer18_attention_weight_max": 32.443878173828125,
      "activations/layer18_attention_weight_min": -20.02284049987793,
      "activations/layer19_attention_weight_max": 44.00905227661133,
      "activations/layer19_attention_weight_min": -30.92984390258789,
      "activations/layer1_attention_weight_max": 19.613801956176758,
      "activations/layer1_attention_weight_min": -14.491290092468262,
      "activations/layer20_attention_weight_max": 34.33003616333008,
      "activations/layer20_attention_weight_min": -22.22039222717285,
      "activations/layer21_attention_weight_max": 32.914634704589844,
      "activations/layer21_attention_weight_min": -20.08316993713379,
      "activations/layer22_attention_weight_max": 46.1313362121582,
      "activations/layer22_attention_weight_min": -26.694623947143555,
      "activations/layer23_attention_weight_max": 28.19354820251465,
      "activations/layer23_attention_weight_min": -20.47127914428711,
      "activations/layer2_attention_weight_max": 29.959022521972656,
      "activations/layer2_attention_weight_min": -29.32079315185547,
      "activations/layer3_attention_weight_max": 83.64034271240234,
      "activations/layer3_attention_weight_min": -82.69815063476562,
      "activations/layer4_attention_weight_max": 132.28221130371094,
      "activations/layer4_attention_weight_min": -128.64451599121094,
      "activations/layer5_attention_weight_max": 94.8390121459961,
      "activations/layer5_attention_weight_min": -86.42929077148438,
      "activations/layer6_attention_weight_max": 57.449378967285156,
      "activations/layer6_attention_weight_min": -54.8818244934082,
      "activations/layer7_attention_weight_max": 78.94218444824219,
      "activations/layer7_attention_weight_min": -77.74029541015625,
      "activations/layer8_attention_weight_max": 42.56071853637695,
      "activations/layer8_attention_weight_min": -40.091094970703125,
      "activations/layer9_attention_weight_max": 37.46021270751953,
      "activations/layer9_attention_weight_min": -32.61174774169922,
      "epoch": 3.7,
      "learning_rate": 0.0001274121212121212,
      "loss": 2.8911,
      "step": 63700
    },
    {
      "activations/layer0_attention_weight_max": 16.897626876831055,
      "activations/layer0_attention_weight_min": -12.036303520202637,
      "activations/layer10_attention_weight_max": 45.9676628112793,
      "activations/layer10_attention_weight_min": -32.744232177734375,
      "activations/layer11_attention_weight_max": 47.07377243041992,
      "activations/layer11_attention_weight_min": -33.74158477783203,
      "activations/layer12_attention_weight_max": 38.71870803833008,
      "activations/layer12_attention_weight_min": -32.262847900390625,
      "activations/layer13_attention_weight_max": 47.557254791259766,
      "activations/layer13_attention_weight_min": -37.851131439208984,
      "activations/layer14_attention_weight_max": 65.05482482910156,
      "activations/layer14_attention_weight_min": -53.875579833984375,
      "activations/layer15_attention_weight_max": 45.86545944213867,
      "activations/layer15_attention_weight_min": -33.68613815307617,
      "activations/layer16_attention_weight_max": 41.44896697998047,
      "activations/layer16_attention_weight_min": -31.69720458984375,
      "activations/layer17_attention_weight_max": 38.49072265625,
      "activations/layer17_attention_weight_min": -25.10723876953125,
      "activations/layer18_attention_weight_max": 30.743942260742188,
      "activations/layer18_attention_weight_min": -20.0739688873291,
      "activations/layer19_attention_weight_max": 38.438011169433594,
      "activations/layer19_attention_weight_min": -29.968564987182617,
      "activations/layer1_attention_weight_max": 17.50930404663086,
      "activations/layer1_attention_weight_min": -16.691919326782227,
      "activations/layer20_attention_weight_max": 28.796459197998047,
      "activations/layer20_attention_weight_min": -23.110179901123047,
      "activations/layer21_attention_weight_max": 35.99109649658203,
      "activations/layer21_attention_weight_min": -20.822389602661133,
      "activations/layer22_attention_weight_max": 48.65866470336914,
      "activations/layer22_attention_weight_min": -26.504718780517578,
      "activations/layer23_attention_weight_max": 29.03107261657715,
      "activations/layer23_attention_weight_min": -18.83840560913086,
      "activations/layer2_attention_weight_max": 32.55778884887695,
      "activations/layer2_attention_weight_min": -31.91584014892578,
      "activations/layer3_attention_weight_max": 89.13138580322266,
      "activations/layer3_attention_weight_min": -87.11292266845703,
      "activations/layer4_attention_weight_max": 149.21067810058594,
      "activations/layer4_attention_weight_min": -135.8753662109375,
      "activations/layer5_attention_weight_max": 110.99137878417969,
      "activations/layer5_attention_weight_min": -102.25959777832031,
      "activations/layer6_attention_weight_max": 63.2029914855957,
      "activations/layer6_attention_weight_min": -58.84434509277344,
      "activations/layer7_attention_weight_max": 96.88502502441406,
      "activations/layer7_attention_weight_min": -82.94989013671875,
      "activations/layer8_attention_weight_max": 47.849430084228516,
      "activations/layer8_attention_weight_min": -40.57917404174805,
      "activations/layer9_attention_weight_max": 68.36144256591797,
      "activations/layer9_attention_weight_min": -44.54792785644531,
      "epoch": 3.7,
      "learning_rate": 0.0001273931818181818,
      "loss": 2.8967,
      "step": 63750
    },
    {
      "activations/layer0_attention_weight_max": 16.799026489257812,
      "activations/layer0_attention_weight_min": -12.071760177612305,
      "activations/layer10_attention_weight_max": 37.9339599609375,
      "activations/layer10_attention_weight_min": -31.241575241088867,
      "activations/layer11_attention_weight_max": 38.7418212890625,
      "activations/layer11_attention_weight_min": -29.470191955566406,
      "activations/layer12_attention_weight_max": 30.154193878173828,
      "activations/layer12_attention_weight_min": -28.363550186157227,
      "activations/layer13_attention_weight_max": 47.45024490356445,
      "activations/layer13_attention_weight_min": -33.885677337646484,
      "activations/layer14_attention_weight_max": 70.3203125,
      "activations/layer14_attention_weight_min": -51.299835205078125,
      "activations/layer15_attention_weight_max": 42.06570053100586,
      "activations/layer15_attention_weight_min": -33.315338134765625,
      "activations/layer16_attention_weight_max": 41.97822189331055,
      "activations/layer16_attention_weight_min": -28.96833038330078,
      "activations/layer17_attention_weight_max": 42.9507942199707,
      "activations/layer17_attention_weight_min": -24.725505828857422,
      "activations/layer18_attention_weight_max": 26.92115020751953,
      "activations/layer18_attention_weight_min": -21.727800369262695,
      "activations/layer19_attention_weight_max": 43.54852294921875,
      "activations/layer19_attention_weight_min": -31.24164390563965,
      "activations/layer1_attention_weight_max": 18.45781135559082,
      "activations/layer1_attention_weight_min": -14.8230619430542,
      "activations/layer20_attention_weight_max": 36.45098876953125,
      "activations/layer20_attention_weight_min": -23.69525718688965,
      "activations/layer21_attention_weight_max": 31.530241012573242,
      "activations/layer21_attention_weight_min": -18.676620483398438,
      "activations/layer22_attention_weight_max": 55.937320709228516,
      "activations/layer22_attention_weight_min": -27.1322078704834,
      "activations/layer23_attention_weight_max": 31.12519645690918,
      "activations/layer23_attention_weight_min": -17.44036102294922,
      "activations/layer2_attention_weight_max": 31.98784637451172,
      "activations/layer2_attention_weight_min": -30.534381866455078,
      "activations/layer3_attention_weight_max": 84.57673645019531,
      "activations/layer3_attention_weight_min": -84.61028289794922,
      "activations/layer4_attention_weight_max": 133.62686157226562,
      "activations/layer4_attention_weight_min": -122.53670501708984,
      "activations/layer5_attention_weight_max": 100.0172119140625,
      "activations/layer5_attention_weight_min": -87.6611557006836,
      "activations/layer6_attention_weight_max": 59.1938591003418,
      "activations/layer6_attention_weight_min": -55.314666748046875,
      "activations/layer7_attention_weight_max": 80.10134887695312,
      "activations/layer7_attention_weight_min": -73.36454010009766,
      "activations/layer8_attention_weight_max": 40.48196029663086,
      "activations/layer8_attention_weight_min": -36.875064849853516,
      "activations/layer9_attention_weight_max": 38.839962005615234,
      "activations/layer9_attention_weight_min": -36.21653366088867,
      "epoch": 3.71,
      "learning_rate": 0.00012737424242424241,
      "loss": 2.8729,
      "step": 63800
    },
    {
      "activations/layer0_attention_weight_max": 16.868488311767578,
      "activations/layer0_attention_weight_min": -12.687862396240234,
      "activations/layer10_attention_weight_max": 41.233001708984375,
      "activations/layer10_attention_weight_min": -30.287338256835938,
      "activations/layer11_attention_weight_max": 37.798912048339844,
      "activations/layer11_attention_weight_min": -27.90544891357422,
      "activations/layer12_attention_weight_max": 31.534313201904297,
      "activations/layer12_attention_weight_min": -28.625452041625977,
      "activations/layer13_attention_weight_max": 44.151004791259766,
      "activations/layer13_attention_weight_min": -32.776763916015625,
      "activations/layer14_attention_weight_max": 65.71288299560547,
      "activations/layer14_attention_weight_min": -50.657249450683594,
      "activations/layer15_attention_weight_max": 46.25741195678711,
      "activations/layer15_attention_weight_min": -33.5962028503418,
      "activations/layer16_attention_weight_max": 47.17305374145508,
      "activations/layer16_attention_weight_min": -29.36359977722168,
      "activations/layer17_attention_weight_max": 45.2305908203125,
      "activations/layer17_attention_weight_min": -25.990943908691406,
      "activations/layer18_attention_weight_max": 32.44706726074219,
      "activations/layer18_attention_weight_min": -18.531917572021484,
      "activations/layer19_attention_weight_max": 49.447513580322266,
      "activations/layer19_attention_weight_min": -28.271093368530273,
      "activations/layer1_attention_weight_max": 16.862306594848633,
      "activations/layer1_attention_weight_min": -16.046159744262695,
      "activations/layer20_attention_weight_max": 39.971370697021484,
      "activations/layer20_attention_weight_min": -23.07111167907715,
      "activations/layer21_attention_weight_max": 40.2021598815918,
      "activations/layer21_attention_weight_min": -21.231130599975586,
      "activations/layer22_attention_weight_max": 51.759613037109375,
      "activations/layer22_attention_weight_min": -27.353670120239258,
      "activations/layer23_attention_weight_max": 30.271160125732422,
      "activations/layer23_attention_weight_min": -18.578025817871094,
      "activations/layer2_attention_weight_max": 31.274198532104492,
      "activations/layer2_attention_weight_min": -31.285757064819336,
      "activations/layer3_attention_weight_max": 82.59907531738281,
      "activations/layer3_attention_weight_min": -81.85859680175781,
      "activations/layer4_attention_weight_max": 129.10641479492188,
      "activations/layer4_attention_weight_min": -125.62755584716797,
      "activations/layer5_attention_weight_max": 92.67889404296875,
      "activations/layer5_attention_weight_min": -88.93663024902344,
      "activations/layer6_attention_weight_max": 57.02009963989258,
      "activations/layer6_attention_weight_min": -58.63649368286133,
      "activations/layer7_attention_weight_max": 75.61505889892578,
      "activations/layer7_attention_weight_min": -74.2077865600586,
      "activations/layer8_attention_weight_max": 37.31175231933594,
      "activations/layer8_attention_weight_min": -37.7087516784668,
      "activations/layer9_attention_weight_max": 37.02069091796875,
      "activations/layer9_attention_weight_min": -33.204559326171875,
      "epoch": 3.71,
      "learning_rate": 0.00012735530303030303,
      "loss": 2.8799,
      "step": 63850
    },
    {
      "activations/layer0_attention_weight_max": 16.924999237060547,
      "activations/layer0_attention_weight_min": -11.185585975646973,
      "activations/layer10_attention_weight_max": 39.08245849609375,
      "activations/layer10_attention_weight_min": -30.478055953979492,
      "activations/layer11_attention_weight_max": 36.34088897705078,
      "activations/layer11_attention_weight_min": -27.868091583251953,
      "activations/layer12_attention_weight_max": 29.090530395507812,
      "activations/layer12_attention_weight_min": -27.782718658447266,
      "activations/layer13_attention_weight_max": 46.856056213378906,
      "activations/layer13_attention_weight_min": -33.60026168823242,
      "activations/layer14_attention_weight_max": 64.76699829101562,
      "activations/layer14_attention_weight_min": -46.10466384887695,
      "activations/layer15_attention_weight_max": 45.25693130493164,
      "activations/layer15_attention_weight_min": -29.331539154052734,
      "activations/layer16_attention_weight_max": 40.76756286621094,
      "activations/layer16_attention_weight_min": -30.333459854125977,
      "activations/layer17_attention_weight_max": 43.863224029541016,
      "activations/layer17_attention_weight_min": -23.619569778442383,
      "activations/layer18_attention_weight_max": 28.480558395385742,
      "activations/layer18_attention_weight_min": -19.782922744750977,
      "activations/layer19_attention_weight_max": 43.67440414428711,
      "activations/layer19_attention_weight_min": -30.475921630859375,
      "activations/layer1_attention_weight_max": 17.293601989746094,
      "activations/layer1_attention_weight_min": -14.855527877807617,
      "activations/layer20_attention_weight_max": 34.767269134521484,
      "activations/layer20_attention_weight_min": -23.451805114746094,
      "activations/layer21_attention_weight_max": 30.448251724243164,
      "activations/layer21_attention_weight_min": -19.546804428100586,
      "activations/layer22_attention_weight_max": 51.231048583984375,
      "activations/layer22_attention_weight_min": -29.228227615356445,
      "activations/layer23_attention_weight_max": 31.727466583251953,
      "activations/layer23_attention_weight_min": -18.10698127746582,
      "activations/layer2_attention_weight_max": 31.137067794799805,
      "activations/layer2_attention_weight_min": -30.722183227539062,
      "activations/layer3_attention_weight_max": 83.4364013671875,
      "activations/layer3_attention_weight_min": -88.75699615478516,
      "activations/layer4_attention_weight_max": 136.9232177734375,
      "activations/layer4_attention_weight_min": -131.07728576660156,
      "activations/layer5_attention_weight_max": 99.40776824951172,
      "activations/layer5_attention_weight_min": -92.0755844116211,
      "activations/layer6_attention_weight_max": 58.490989685058594,
      "activations/layer6_attention_weight_min": -55.67486572265625,
      "activations/layer7_attention_weight_max": 79.74378204345703,
      "activations/layer7_attention_weight_min": -76.62244415283203,
      "activations/layer8_attention_weight_max": 41.49733352661133,
      "activations/layer8_attention_weight_min": -41.22859191894531,
      "activations/layer9_attention_weight_max": 39.26167678833008,
      "activations/layer9_attention_weight_min": -33.997222900390625,
      "epoch": 3.71,
      "learning_rate": 0.00012733636363636364,
      "loss": 2.9117,
      "step": 63900
    },
    {
      "activations/layer0_attention_weight_max": 17.30125617980957,
      "activations/layer0_attention_weight_min": -11.576168060302734,
      "activations/layer10_attention_weight_max": 33.89103317260742,
      "activations/layer10_attention_weight_min": -30.201555252075195,
      "activations/layer11_attention_weight_max": 33.12846755981445,
      "activations/layer11_attention_weight_min": -28.41571617126465,
      "activations/layer12_attention_weight_max": 31.760879516601562,
      "activations/layer12_attention_weight_min": -30.9161319732666,
      "activations/layer13_attention_weight_max": 48.72757339477539,
      "activations/layer13_attention_weight_min": -35.68431854248047,
      "activations/layer14_attention_weight_max": 64.13037109375,
      "activations/layer14_attention_weight_min": -49.85859680175781,
      "activations/layer15_attention_weight_max": 38.866973876953125,
      "activations/layer15_attention_weight_min": -27.592683792114258,
      "activations/layer16_attention_weight_max": 39.53342056274414,
      "activations/layer16_attention_weight_min": -27.565271377563477,
      "activations/layer17_attention_weight_max": 41.609352111816406,
      "activations/layer17_attention_weight_min": -24.602190017700195,
      "activations/layer18_attention_weight_max": 32.94533920288086,
      "activations/layer18_attention_weight_min": -20.6130428314209,
      "activations/layer19_attention_weight_max": 44.99995040893555,
      "activations/layer19_attention_weight_min": -28.858097076416016,
      "activations/layer1_attention_weight_max": 17.222631454467773,
      "activations/layer1_attention_weight_min": -16.84662437438965,
      "activations/layer20_attention_weight_max": 30.139163970947266,
      "activations/layer20_attention_weight_min": -22.88836097717285,
      "activations/layer21_attention_weight_max": 35.414634704589844,
      "activations/layer21_attention_weight_min": -20.275022506713867,
      "activations/layer22_attention_weight_max": 52.258216857910156,
      "activations/layer22_attention_weight_min": -29.168794631958008,
      "activations/layer23_attention_weight_max": 29.236860275268555,
      "activations/layer23_attention_weight_min": -17.89952278137207,
      "activations/layer2_attention_weight_max": 30.395484924316406,
      "activations/layer2_attention_weight_min": -28.957759857177734,
      "activations/layer3_attention_weight_max": 82.26110076904297,
      "activations/layer3_attention_weight_min": -85.13661193847656,
      "activations/layer4_attention_weight_max": 125.21154022216797,
      "activations/layer4_attention_weight_min": -128.4973602294922,
      "activations/layer5_attention_weight_max": 91.363525390625,
      "activations/layer5_attention_weight_min": -91.2419662475586,
      "activations/layer6_attention_weight_max": 55.18653869628906,
      "activations/layer6_attention_weight_min": -57.76856994628906,
      "activations/layer7_attention_weight_max": 71.90960693359375,
      "activations/layer7_attention_weight_min": -75.44123077392578,
      "activations/layer8_attention_weight_max": 36.8459587097168,
      "activations/layer8_attention_weight_min": -41.18383026123047,
      "activations/layer9_attention_weight_max": 35.9919319152832,
      "activations/layer9_attention_weight_min": -33.416725158691406,
      "epoch": 3.72,
      "learning_rate": 0.00012731742424242423,
      "loss": 2.8754,
      "step": 63950
    },
    {
      "activations/layer0_attention_weight_max": 16.801651000976562,
      "activations/layer0_attention_weight_min": -12.214902877807617,
      "activations/layer10_attention_weight_max": 36.03059768676758,
      "activations/layer10_attention_weight_min": -30.764970779418945,
      "activations/layer11_attention_weight_max": 31.03167724609375,
      "activations/layer11_attention_weight_min": -28.401952743530273,
      "activations/layer12_attention_weight_max": 34.86098861694336,
      "activations/layer12_attention_weight_min": -29.09373664855957,
      "activations/layer13_attention_weight_max": 48.99137878417969,
      "activations/layer13_attention_weight_min": -32.04728317260742,
      "activations/layer14_attention_weight_max": 62.38440704345703,
      "activations/layer14_attention_weight_min": -44.99123764038086,
      "activations/layer15_attention_weight_max": 44.847537994384766,
      "activations/layer15_attention_weight_min": -28.998620986938477,
      "activations/layer16_attention_weight_max": 41.436241149902344,
      "activations/layer16_attention_weight_min": -27.874439239501953,
      "activations/layer17_attention_weight_max": 41.75483703613281,
      "activations/layer17_attention_weight_min": -23.813867568969727,
      "activations/layer18_attention_weight_max": 31.46725082397461,
      "activations/layer18_attention_weight_min": -19.775943756103516,
      "activations/layer19_attention_weight_max": 42.31293869018555,
      "activations/layer19_attention_weight_min": -28.568912506103516,
      "activations/layer1_attention_weight_max": 18.288366317749023,
      "activations/layer1_attention_weight_min": -16.681310653686523,
      "activations/layer20_attention_weight_max": 33.531005859375,
      "activations/layer20_attention_weight_min": -23.55122947692871,
      "activations/layer21_attention_weight_max": 30.961883544921875,
      "activations/layer21_attention_weight_min": -20.459360122680664,
      "activations/layer22_attention_weight_max": 49.16472625732422,
      "activations/layer22_attention_weight_min": -30.671342849731445,
      "activations/layer23_attention_weight_max": 28.111852645874023,
      "activations/layer23_attention_weight_min": -17.373973846435547,
      "activations/layer2_attention_weight_max": 32.3426628112793,
      "activations/layer2_attention_weight_min": -30.378158569335938,
      "activations/layer3_attention_weight_max": 84.5066146850586,
      "activations/layer3_attention_weight_min": -88.019775390625,
      "activations/layer4_attention_weight_max": 131.13951110839844,
      "activations/layer4_attention_weight_min": -128.21812438964844,
      "activations/layer5_attention_weight_max": 91.22216796875,
      "activations/layer5_attention_weight_min": -86.93197631835938,
      "activations/layer6_attention_weight_max": 56.0829963684082,
      "activations/layer6_attention_weight_min": -56.978031158447266,
      "activations/layer7_attention_weight_max": 75.41416931152344,
      "activations/layer7_attention_weight_min": -74.00042724609375,
      "activations/layer8_attention_weight_max": 40.33123779296875,
      "activations/layer8_attention_weight_min": -37.04903030395508,
      "activations/layer9_attention_weight_max": 43.87934494018555,
      "activations/layer9_attention_weight_min": -34.11653137207031,
      "epoch": 3.72,
      "learning_rate": 0.00012729848484848485,
      "loss": 2.8855,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_loss": 2.833984375,
      "eval_runtime": 8.5008,
      "eval_samples_per_second": 505.131,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_openwebtext_loss": 2.833984375,
      "eval_openwebtext_ppl": 17.013112580497378,
      "eval_openwebtext_runtime": 8.5008,
      "eval_openwebtext_samples_per_second": 505.131,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_wikitext_loss": 3.080078125,
      "eval_wikitext_ppl": 21.760102337787384,
      "eval_wikitext_runtime": 1.9652,
      "eval_wikitext_samples_per_second": 232.036,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_lambada_loss": 2.859375,
      "eval_lambada_ppl": 17.45061689198871,
      "eval_lambada_runtime": 9.5838,
      "eval_lambada_samples_per_second": 508.045,
      "step": 64000
    },
    {
      "activations/layer0_attention_weight_max": 17.31913948059082,
      "activations/layer0_attention_weight_min": -11.540140151977539,
      "activations/layer10_attention_weight_max": 39.51701736450195,
      "activations/layer10_attention_weight_min": -29.83132553100586,
      "activations/layer11_attention_weight_max": 34.7318115234375,
      "activations/layer11_attention_weight_min": -30.348766326904297,
      "activations/layer12_attention_weight_max": 23.746356964111328,
      "activations/layer12_attention_weight_min": -25.76523208618164,
      "activations/layer13_attention_weight_max": 45.36629104614258,
      "activations/layer13_attention_weight_min": -33.205318450927734,
      "activations/layer14_attention_weight_max": 67.86735534667969,
      "activations/layer14_attention_weight_min": -50.53787612915039,
      "activations/layer15_attention_weight_max": 41.71031188964844,
      "activations/layer15_attention_weight_min": -31.585468292236328,
      "activations/layer16_attention_weight_max": 41.77409744262695,
      "activations/layer16_attention_weight_min": -28.842836380004883,
      "activations/layer17_attention_weight_max": 40.28937530517578,
      "activations/layer17_attention_weight_min": -24.93895721435547,
      "activations/layer18_attention_weight_max": 30.03142738342285,
      "activations/layer18_attention_weight_min": -19.841386795043945,
      "activations/layer19_attention_weight_max": 44.016353607177734,
      "activations/layer19_attention_weight_min": -29.204944610595703,
      "activations/layer1_attention_weight_max": 18.377103805541992,
      "activations/layer1_attention_weight_min": -14.830318450927734,
      "activations/layer20_attention_weight_max": 32.30438995361328,
      "activations/layer20_attention_weight_min": -24.366811752319336,
      "activations/layer21_attention_weight_max": 29.21671485900879,
      "activations/layer21_attention_weight_min": -22.0083065032959,
      "activations/layer22_attention_weight_max": 47.26177215576172,
      "activations/layer22_attention_weight_min": -26.65158462524414,
      "activations/layer23_attention_weight_max": 32.200294494628906,
      "activations/layer23_attention_weight_min": -16.543081283569336,
      "activations/layer2_attention_weight_max": 32.95793151855469,
      "activations/layer2_attention_weight_min": -33.40689468383789,
      "activations/layer3_attention_weight_max": 85.87014770507812,
      "activations/layer3_attention_weight_min": -89.3072509765625,
      "activations/layer4_attention_weight_max": 128.15472412109375,
      "activations/layer4_attention_weight_min": -133.5640411376953,
      "activations/layer5_attention_weight_max": 96.21981048583984,
      "activations/layer5_attention_weight_min": -97.22161102294922,
      "activations/layer6_attention_weight_max": 60.08824157714844,
      "activations/layer6_attention_weight_min": -58.76471710205078,
      "activations/layer7_attention_weight_max": 73.84931945800781,
      "activations/layer7_attention_weight_min": -77.24079132080078,
      "activations/layer8_attention_weight_max": 41.86661911010742,
      "activations/layer8_attention_weight_min": -38.68539810180664,
      "activations/layer9_attention_weight_max": 43.756832122802734,
      "activations/layer9_attention_weight_min": -35.8579216003418,
      "epoch": 3.72,
      "learning_rate": 0.00012727954545454546,
      "loss": 2.8999,
      "step": 64050
    },
    {
      "activations/layer0_attention_weight_max": 16.631492614746094,
      "activations/layer0_attention_weight_min": -12.31472110748291,
      "activations/layer10_attention_weight_max": 37.11588668823242,
      "activations/layer10_attention_weight_min": -30.74019432067871,
      "activations/layer11_attention_weight_max": 33.56733703613281,
      "activations/layer11_attention_weight_min": -27.13558578491211,
      "activations/layer12_attention_weight_max": 31.304325103759766,
      "activations/layer12_attention_weight_min": -30.163368225097656,
      "activations/layer13_attention_weight_max": 52.565956115722656,
      "activations/layer13_attention_weight_min": -32.958221435546875,
      "activations/layer14_attention_weight_max": 65.81082153320312,
      "activations/layer14_attention_weight_min": -47.35406494140625,
      "activations/layer15_attention_weight_max": 46.86771774291992,
      "activations/layer15_attention_weight_min": -33.562530517578125,
      "activations/layer16_attention_weight_max": 46.09644317626953,
      "activations/layer16_attention_weight_min": -29.583045959472656,
      "activations/layer17_attention_weight_max": 43.285736083984375,
      "activations/layer17_attention_weight_min": -27.40180015563965,
      "activations/layer18_attention_weight_max": 36.69609451293945,
      "activations/layer18_attention_weight_min": -19.687040328979492,
      "activations/layer19_attention_weight_max": 42.70615005493164,
      "activations/layer19_attention_weight_min": -28.735366821289062,
      "activations/layer1_attention_weight_max": 17.990863800048828,
      "activations/layer1_attention_weight_min": -13.237256050109863,
      "activations/layer20_attention_weight_max": 34.594757080078125,
      "activations/layer20_attention_weight_min": -24.60284996032715,
      "activations/layer21_attention_weight_max": 38.08309555053711,
      "activations/layer21_attention_weight_min": -21.486963272094727,
      "activations/layer22_attention_weight_max": 51.9508171081543,
      "activations/layer22_attention_weight_min": -27.139869689941406,
      "activations/layer23_attention_weight_max": 31.322208404541016,
      "activations/layer23_attention_weight_min": -17.59593391418457,
      "activations/layer2_attention_weight_max": 29.60317611694336,
      "activations/layer2_attention_weight_min": -28.112045288085938,
      "activations/layer3_attention_weight_max": 79.8759765625,
      "activations/layer3_attention_weight_min": -83.17251586914062,
      "activations/layer4_attention_weight_max": 129.05027770996094,
      "activations/layer4_attention_weight_min": -128.2693634033203,
      "activations/layer5_attention_weight_max": 97.28350830078125,
      "activations/layer5_attention_weight_min": -86.75845336914062,
      "activations/layer6_attention_weight_max": 59.43443298339844,
      "activations/layer6_attention_weight_min": -54.04890441894531,
      "activations/layer7_attention_weight_max": 78.18143463134766,
      "activations/layer7_attention_weight_min": -74.09517669677734,
      "activations/layer8_attention_weight_max": 42.944217681884766,
      "activations/layer8_attention_weight_min": -35.403167724609375,
      "activations/layer9_attention_weight_max": 43.31651306152344,
      "activations/layer9_attention_weight_min": -33.35438537597656,
      "epoch": 3.72,
      "learning_rate": 0.00012726060606060605,
      "loss": 2.9029,
      "step": 64100
    },
    {
      "activations/layer0_attention_weight_max": 16.56869888305664,
      "activations/layer0_attention_weight_min": -10.70874309539795,
      "activations/layer10_attention_weight_max": 39.465492248535156,
      "activations/layer10_attention_weight_min": -33.41474914550781,
      "activations/layer11_attention_weight_max": 35.68902587890625,
      "activations/layer11_attention_weight_min": -28.433393478393555,
      "activations/layer12_attention_weight_max": 32.65097427368164,
      "activations/layer12_attention_weight_min": -29.308963775634766,
      "activations/layer13_attention_weight_max": 62.43122482299805,
      "activations/layer13_attention_weight_min": -41.872650146484375,
      "activations/layer14_attention_weight_max": 72.53971099853516,
      "activations/layer14_attention_weight_min": -57.68009567260742,
      "activations/layer15_attention_weight_max": 46.99171829223633,
      "activations/layer15_attention_weight_min": -32.15256881713867,
      "activations/layer16_attention_weight_max": 39.182220458984375,
      "activations/layer16_attention_weight_min": -28.123313903808594,
      "activations/layer17_attention_weight_max": 42.40291213989258,
      "activations/layer17_attention_weight_min": -27.99623680114746,
      "activations/layer18_attention_weight_max": 33.7514762878418,
      "activations/layer18_attention_weight_min": -20.859073638916016,
      "activations/layer19_attention_weight_max": 39.05234909057617,
      "activations/layer19_attention_weight_min": -28.90143394470215,
      "activations/layer1_attention_weight_max": 17.142742156982422,
      "activations/layer1_attention_weight_min": -15.505688667297363,
      "activations/layer20_attention_weight_max": 35.91633224487305,
      "activations/layer20_attention_weight_min": -24.069591522216797,
      "activations/layer21_attention_weight_max": 38.71708297729492,
      "activations/layer21_attention_weight_min": -22.27082633972168,
      "activations/layer22_attention_weight_max": 50.04493713378906,
      "activations/layer22_attention_weight_min": -29.36202049255371,
      "activations/layer23_attention_weight_max": 30.772216796875,
      "activations/layer23_attention_weight_min": -21.737089157104492,
      "activations/layer2_attention_weight_max": 31.5330810546875,
      "activations/layer2_attention_weight_min": -33.10982894897461,
      "activations/layer3_attention_weight_max": 84.00039672851562,
      "activations/layer3_attention_weight_min": -89.77092742919922,
      "activations/layer4_attention_weight_max": 130.3802490234375,
      "activations/layer4_attention_weight_min": -126.63175201416016,
      "activations/layer5_attention_weight_max": 88.93235778808594,
      "activations/layer5_attention_weight_min": -89.0794906616211,
      "activations/layer6_attention_weight_max": 54.30063247680664,
      "activations/layer6_attention_weight_min": -54.3773078918457,
      "activations/layer7_attention_weight_max": 75.29707336425781,
      "activations/layer7_attention_weight_min": -76.26969909667969,
      "activations/layer8_attention_weight_max": 40.216949462890625,
      "activations/layer8_attention_weight_min": -37.57233810424805,
      "activations/layer9_attention_weight_max": 44.19758224487305,
      "activations/layer9_attention_weight_min": -40.70085525512695,
      "epoch": 3.73,
      "learning_rate": 0.00012724166666666667,
      "loss": 2.8941,
      "step": 64150
    },
    {
      "activations/layer0_attention_weight_max": 16.779123306274414,
      "activations/layer0_attention_weight_min": -11.169986724853516,
      "activations/layer10_attention_weight_max": 37.111839294433594,
      "activations/layer10_attention_weight_min": -30.583242416381836,
      "activations/layer11_attention_weight_max": 34.51422882080078,
      "activations/layer11_attention_weight_min": -30.932310104370117,
      "activations/layer12_attention_weight_max": 43.62959289550781,
      "activations/layer12_attention_weight_min": -27.06192970275879,
      "activations/layer13_attention_weight_max": 45.771366119384766,
      "activations/layer13_attention_weight_min": -32.33676528930664,
      "activations/layer14_attention_weight_max": 69.27761840820312,
      "activations/layer14_attention_weight_min": -52.16648864746094,
      "activations/layer15_attention_weight_max": 42.545867919921875,
      "activations/layer15_attention_weight_min": -30.718975067138672,
      "activations/layer16_attention_weight_max": 42.15962600708008,
      "activations/layer16_attention_weight_min": -31.624616622924805,
      "activations/layer17_attention_weight_max": 42.88289260864258,
      "activations/layer17_attention_weight_min": -25.58835220336914,
      "activations/layer18_attention_weight_max": 37.06853485107422,
      "activations/layer18_attention_weight_min": -18.41966438293457,
      "activations/layer19_attention_weight_max": 44.8598747253418,
      "activations/layer19_attention_weight_min": -30.1854190826416,
      "activations/layer1_attention_weight_max": 17.230051040649414,
      "activations/layer1_attention_weight_min": -14.290420532226562,
      "activations/layer20_attention_weight_max": 32.81663513183594,
      "activations/layer20_attention_weight_min": -22.384618759155273,
      "activations/layer21_attention_weight_max": 30.07614517211914,
      "activations/layer21_attention_weight_min": -17.81826400756836,
      "activations/layer22_attention_weight_max": 54.09541702270508,
      "activations/layer22_attention_weight_min": -28.110029220581055,
      "activations/layer23_attention_weight_max": 32.49137878417969,
      "activations/layer23_attention_weight_min": -16.23708724975586,
      "activations/layer2_attention_weight_max": 31.378196716308594,
      "activations/layer2_attention_weight_min": -31.668989181518555,
      "activations/layer3_attention_weight_max": 85.73231506347656,
      "activations/layer3_attention_weight_min": -87.3738784790039,
      "activations/layer4_attention_weight_max": 135.41006469726562,
      "activations/layer4_attention_weight_min": -128.4366455078125,
      "activations/layer5_attention_weight_max": 97.48814392089844,
      "activations/layer5_attention_weight_min": -92.18478393554688,
      "activations/layer6_attention_weight_max": 58.72505187988281,
      "activations/layer6_attention_weight_min": -58.85029602050781,
      "activations/layer7_attention_weight_max": 86.08560943603516,
      "activations/layer7_attention_weight_min": -78.49463653564453,
      "activations/layer8_attention_weight_max": 42.631404876708984,
      "activations/layer8_attention_weight_min": -40.192420959472656,
      "activations/layer9_attention_weight_max": 37.55426788330078,
      "activations/layer9_attention_weight_min": -38.51211166381836,
      "epoch": 3.73,
      "learning_rate": 0.00012722272727272725,
      "loss": 2.8814,
      "step": 64200
    },
    {
      "activations/layer0_attention_weight_max": 16.93916893005371,
      "activations/layer0_attention_weight_min": -10.619848251342773,
      "activations/layer10_attention_weight_max": 35.294673919677734,
      "activations/layer10_attention_weight_min": -29.739974975585938,
      "activations/layer11_attention_weight_max": 31.56594467163086,
      "activations/layer11_attention_weight_min": -27.134384155273438,
      "activations/layer12_attention_weight_max": 34.075714111328125,
      "activations/layer12_attention_weight_min": -27.532514572143555,
      "activations/layer13_attention_weight_max": 43.1604118347168,
      "activations/layer13_attention_weight_min": -32.43934631347656,
      "activations/layer14_attention_weight_max": 70.09356689453125,
      "activations/layer14_attention_weight_min": -50.14155197143555,
      "activations/layer15_attention_weight_max": 45.39860916137695,
      "activations/layer15_attention_weight_min": -29.616182327270508,
      "activations/layer16_attention_weight_max": 42.4109992980957,
      "activations/layer16_attention_weight_min": -30.164762496948242,
      "activations/layer17_attention_weight_max": 46.27873992919922,
      "activations/layer17_attention_weight_min": -23.204530715942383,
      "activations/layer18_attention_weight_max": 35.728179931640625,
      "activations/layer18_attention_weight_min": -19.3315486907959,
      "activations/layer19_attention_weight_max": 45.66401290893555,
      "activations/layer19_attention_weight_min": -30.221616744995117,
      "activations/layer1_attention_weight_max": 18.07737922668457,
      "activations/layer1_attention_weight_min": -16.287534713745117,
      "activations/layer20_attention_weight_max": 32.71533203125,
      "activations/layer20_attention_weight_min": -22.917617797851562,
      "activations/layer21_attention_weight_max": 38.42304992675781,
      "activations/layer21_attention_weight_min": -19.481708526611328,
      "activations/layer22_attention_weight_max": 54.563838958740234,
      "activations/layer22_attention_weight_min": -26.630126953125,
      "activations/layer23_attention_weight_max": 34.399009704589844,
      "activations/layer23_attention_weight_min": -17.922000885009766,
      "activations/layer2_attention_weight_max": 30.081798553466797,
      "activations/layer2_attention_weight_min": -30.652074813842773,
      "activations/layer3_attention_weight_max": 82.6224365234375,
      "activations/layer3_attention_weight_min": -87.93489837646484,
      "activations/layer4_attention_weight_max": 131.2897491455078,
      "activations/layer4_attention_weight_min": -131.54983520507812,
      "activations/layer5_attention_weight_max": 92.5414047241211,
      "activations/layer5_attention_weight_min": -91.4209213256836,
      "activations/layer6_attention_weight_max": 54.938140869140625,
      "activations/layer6_attention_weight_min": -55.83953857421875,
      "activations/layer7_attention_weight_max": 73.7408447265625,
      "activations/layer7_attention_weight_min": -75.1501235961914,
      "activations/layer8_attention_weight_max": 37.66014099121094,
      "activations/layer8_attention_weight_min": -38.947261810302734,
      "activations/layer9_attention_weight_max": 37.22433090209961,
      "activations/layer9_attention_weight_min": -34.349632263183594,
      "epoch": 3.73,
      "learning_rate": 0.00012720416666666667,
      "loss": 2.8795,
      "step": 64250
    },
    {
      "activations/layer0_attention_weight_max": 16.49745750427246,
      "activations/layer0_attention_weight_min": -10.941207885742188,
      "activations/layer10_attention_weight_max": 37.43010330200195,
      "activations/layer10_attention_weight_min": -32.85205841064453,
      "activations/layer11_attention_weight_max": 34.107337951660156,
      "activations/layer11_attention_weight_min": -33.32111358642578,
      "activations/layer12_attention_weight_max": 27.723251342773438,
      "activations/layer12_attention_weight_min": -24.392391204833984,
      "activations/layer13_attention_weight_max": 47.47770309448242,
      "activations/layer13_attention_weight_min": -32.97779846191406,
      "activations/layer14_attention_weight_max": 75.40975952148438,
      "activations/layer14_attention_weight_min": -48.98775100708008,
      "activations/layer15_attention_weight_max": 47.766727447509766,
      "activations/layer15_attention_weight_min": -29.323305130004883,
      "activations/layer16_attention_weight_max": 48.74285125732422,
      "activations/layer16_attention_weight_min": -29.30271339416504,
      "activations/layer17_attention_weight_max": 44.66440963745117,
      "activations/layer17_attention_weight_min": -25.080303192138672,
      "activations/layer18_attention_weight_max": 40.367774963378906,
      "activations/layer18_attention_weight_min": -19.809236526489258,
      "activations/layer19_attention_weight_max": 48.992584228515625,
      "activations/layer19_attention_weight_min": -29.160669326782227,
      "activations/layer1_attention_weight_max": 17.353099822998047,
      "activations/layer1_attention_weight_min": -15.173354148864746,
      "activations/layer20_attention_weight_max": 39.42797088623047,
      "activations/layer20_attention_weight_min": -24.881969451904297,
      "activations/layer21_attention_weight_max": 35.909690856933594,
      "activations/layer21_attention_weight_min": -20.573413848876953,
      "activations/layer22_attention_weight_max": 55.78068923950195,
      "activations/layer22_attention_weight_min": -28.20944595336914,
      "activations/layer23_attention_weight_max": 29.733091354370117,
      "activations/layer23_attention_weight_min": -19.329866409301758,
      "activations/layer2_attention_weight_max": 31.464887619018555,
      "activations/layer2_attention_weight_min": -33.40293884277344,
      "activations/layer3_attention_weight_max": 80.53050231933594,
      "activations/layer3_attention_weight_min": -82.97311401367188,
      "activations/layer4_attention_weight_max": 127.39849853515625,
      "activations/layer4_attention_weight_min": -122.18658447265625,
      "activations/layer5_attention_weight_max": 91.69930267333984,
      "activations/layer5_attention_weight_min": -85.15153503417969,
      "activations/layer6_attention_weight_max": 56.43333435058594,
      "activations/layer6_attention_weight_min": -53.760860443115234,
      "activations/layer7_attention_weight_max": 75.06661987304688,
      "activations/layer7_attention_weight_min": -75.62457275390625,
      "activations/layer8_attention_weight_max": 37.85665512084961,
      "activations/layer8_attention_weight_min": -35.8484001159668,
      "activations/layer9_attention_weight_max": 42.28231430053711,
      "activations/layer9_attention_weight_min": -34.25672912597656,
      "epoch": 3.74,
      "learning_rate": 0.00012718522727272726,
      "loss": 2.8744,
      "step": 64300
    },
    {
      "activations/layer0_attention_weight_max": 17.61359214782715,
      "activations/layer0_attention_weight_min": -11.548360824584961,
      "activations/layer10_attention_weight_max": 37.53679656982422,
      "activations/layer10_attention_weight_min": -30.39935302734375,
      "activations/layer11_attention_weight_max": 35.55900573730469,
      "activations/layer11_attention_weight_min": -30.825111389160156,
      "activations/layer12_attention_weight_max": 28.3601131439209,
      "activations/layer12_attention_weight_min": -31.886371612548828,
      "activations/layer13_attention_weight_max": 48.64646530151367,
      "activations/layer13_attention_weight_min": -30.842763900756836,
      "activations/layer14_attention_weight_max": 68.3228530883789,
      "activations/layer14_attention_weight_min": -52.63536071777344,
      "activations/layer15_attention_weight_max": 55.978187561035156,
      "activations/layer15_attention_weight_min": -30.482900619506836,
      "activations/layer16_attention_weight_max": 40.96266174316406,
      "activations/layer16_attention_weight_min": -29.29062271118164,
      "activations/layer17_attention_weight_max": 44.288291931152344,
      "activations/layer17_attention_weight_min": -25.04127311706543,
      "activations/layer18_attention_weight_max": 30.669658660888672,
      "activations/layer18_attention_weight_min": -19.492158889770508,
      "activations/layer19_attention_weight_max": 44.15876388549805,
      "activations/layer19_attention_weight_min": -26.704238891601562,
      "activations/layer1_attention_weight_max": 18.293121337890625,
      "activations/layer1_attention_weight_min": -13.911406517028809,
      "activations/layer20_attention_weight_max": 31.83062171936035,
      "activations/layer20_attention_weight_min": -23.807756423950195,
      "activations/layer21_attention_weight_max": 30.386211395263672,
      "activations/layer21_attention_weight_min": -20.45355796813965,
      "activations/layer22_attention_weight_max": 46.392112731933594,
      "activations/layer22_attention_weight_min": -27.222919464111328,
      "activations/layer23_attention_weight_max": 30.250961303710938,
      "activations/layer23_attention_weight_min": -18.60554313659668,
      "activations/layer2_attention_weight_max": 32.37828063964844,
      "activations/layer2_attention_weight_min": -33.30759048461914,
      "activations/layer3_attention_weight_max": 91.19596862792969,
      "activations/layer3_attention_weight_min": -93.2271728515625,
      "activations/layer4_attention_weight_max": 140.3013153076172,
      "activations/layer4_attention_weight_min": -137.80007934570312,
      "activations/layer5_attention_weight_max": 101.70687103271484,
      "activations/layer5_attention_weight_min": -91.99404907226562,
      "activations/layer6_attention_weight_max": 59.29695510864258,
      "activations/layer6_attention_weight_min": -59.18495178222656,
      "activations/layer7_attention_weight_max": 78.6938247680664,
      "activations/layer7_attention_weight_min": -78.65006256103516,
      "activations/layer8_attention_weight_max": 41.947853088378906,
      "activations/layer8_attention_weight_min": -39.10563278198242,
      "activations/layer9_attention_weight_max": 42.194278717041016,
      "activations/layer9_attention_weight_min": -36.15241241455078,
      "epoch": 3.74,
      "learning_rate": 0.00012716628787878787,
      "loss": 2.8813,
      "step": 64350
    },
    {
      "activations/layer0_attention_weight_max": 16.89873504638672,
      "activations/layer0_attention_weight_min": -12.203126907348633,
      "activations/layer10_attention_weight_max": 36.32927703857422,
      "activations/layer10_attention_weight_min": -28.64786720275879,
      "activations/layer11_attention_weight_max": 32.44378662109375,
      "activations/layer11_attention_weight_min": -27.77832794189453,
      "activations/layer12_attention_weight_max": 31.079801559448242,
      "activations/layer12_attention_weight_min": -24.64765739440918,
      "activations/layer13_attention_weight_max": 43.95460510253906,
      "activations/layer13_attention_weight_min": -31.321155548095703,
      "activations/layer14_attention_weight_max": 61.26253128051758,
      "activations/layer14_attention_weight_min": -46.516448974609375,
      "activations/layer15_attention_weight_max": 43.16532897949219,
      "activations/layer15_attention_weight_min": -29.98536491394043,
      "activations/layer16_attention_weight_max": 47.08460235595703,
      "activations/layer16_attention_weight_min": -29.414615631103516,
      "activations/layer17_attention_weight_max": 42.54632568359375,
      "activations/layer17_attention_weight_min": -24.486480712890625,
      "activations/layer18_attention_weight_max": 28.929052352905273,
      "activations/layer18_attention_weight_min": -19.98566436767578,
      "activations/layer19_attention_weight_max": 42.11172103881836,
      "activations/layer19_attention_weight_min": -32.07866287231445,
      "activations/layer1_attention_weight_max": 17.11796760559082,
      "activations/layer1_attention_weight_min": -15.996549606323242,
      "activations/layer20_attention_weight_max": 31.423587799072266,
      "activations/layer20_attention_weight_min": -23.118627548217773,
      "activations/layer21_attention_weight_max": 30.526823043823242,
      "activations/layer21_attention_weight_min": -20.78739356994629,
      "activations/layer22_attention_weight_max": 49.766082763671875,
      "activations/layer22_attention_weight_min": -26.250167846679688,
      "activations/layer23_attention_weight_max": 29.39017105102539,
      "activations/layer23_attention_weight_min": -18.155635833740234,
      "activations/layer2_attention_weight_max": 29.772659301757812,
      "activations/layer2_attention_weight_min": -29.832754135131836,
      "activations/layer3_attention_weight_max": 82.32637786865234,
      "activations/layer3_attention_weight_min": -86.93533325195312,
      "activations/layer4_attention_weight_max": 134.6128692626953,
      "activations/layer4_attention_weight_min": -132.4855194091797,
      "activations/layer5_attention_weight_max": 93.06834411621094,
      "activations/layer5_attention_weight_min": -89.29095458984375,
      "activations/layer6_attention_weight_max": 56.41314697265625,
      "activations/layer6_attention_weight_min": -55.56298828125,
      "activations/layer7_attention_weight_max": 75.10728454589844,
      "activations/layer7_attention_weight_min": -76.6670913696289,
      "activations/layer8_attention_weight_max": 37.65443801879883,
      "activations/layer8_attention_weight_min": -39.727081298828125,
      "activations/layer9_attention_weight_max": 37.14996337890625,
      "activations/layer9_attention_weight_min": -36.762306213378906,
      "epoch": 3.74,
      "learning_rate": 0.00012714734848484846,
      "loss": 2.8948,
      "step": 64400
    },
    {
      "activations/layer0_attention_weight_max": 17.00288200378418,
      "activations/layer0_attention_weight_min": -11.55830192565918,
      "activations/layer10_attention_weight_max": 36.309322357177734,
      "activations/layer10_attention_weight_min": -31.084720611572266,
      "activations/layer11_attention_weight_max": 36.11747360229492,
      "activations/layer11_attention_weight_min": -31.930728912353516,
      "activations/layer12_attention_weight_max": 34.17752456665039,
      "activations/layer12_attention_weight_min": -28.213150024414062,
      "activations/layer13_attention_weight_max": 44.67142868041992,
      "activations/layer13_attention_weight_min": -35.28912353515625,
      "activations/layer14_attention_weight_max": 58.38322830200195,
      "activations/layer14_attention_weight_min": -44.55064010620117,
      "activations/layer15_attention_weight_max": 41.50654220581055,
      "activations/layer15_attention_weight_min": -31.366443634033203,
      "activations/layer16_attention_weight_max": 37.09568786621094,
      "activations/layer16_attention_weight_min": -29.58091926574707,
      "activations/layer17_attention_weight_max": 37.899627685546875,
      "activations/layer17_attention_weight_min": -24.91461753845215,
      "activations/layer18_attention_weight_max": 28.5588436126709,
      "activations/layer18_attention_weight_min": -22.193992614746094,
      "activations/layer19_attention_weight_max": 39.837371826171875,
      "activations/layer19_attention_weight_min": -30.49143409729004,
      "activations/layer1_attention_weight_max": 17.291561126708984,
      "activations/layer1_attention_weight_min": -15.071355819702148,
      "activations/layer20_attention_weight_max": 34.475528717041016,
      "activations/layer20_attention_weight_min": -24.415014266967773,
      "activations/layer21_attention_weight_max": 34.161033630371094,
      "activations/layer21_attention_weight_min": -22.14339256286621,
      "activations/layer22_attention_weight_max": 48.31494903564453,
      "activations/layer22_attention_weight_min": -29.80782699584961,
      "activations/layer23_attention_weight_max": 29.7000732421875,
      "activations/layer23_attention_weight_min": -20.88864517211914,
      "activations/layer2_attention_weight_max": 31.3634033203125,
      "activations/layer2_attention_weight_min": -31.381698608398438,
      "activations/layer3_attention_weight_max": 85.21656036376953,
      "activations/layer3_attention_weight_min": -97.20442199707031,
      "activations/layer4_attention_weight_max": 146.7574005126953,
      "activations/layer4_attention_weight_min": -151.7148895263672,
      "activations/layer5_attention_weight_max": 97.10759735107422,
      "activations/layer5_attention_weight_min": -95.13085174560547,
      "activations/layer6_attention_weight_max": 56.680137634277344,
      "activations/layer6_attention_weight_min": -55.31608200073242,
      "activations/layer7_attention_weight_max": 72.03758239746094,
      "activations/layer7_attention_weight_min": -76.43412780761719,
      "activations/layer8_attention_weight_max": 37.95650863647461,
      "activations/layer8_attention_weight_min": -37.58393478393555,
      "activations/layer9_attention_weight_max": 38.71073913574219,
      "activations/layer9_attention_weight_min": -34.01475143432617,
      "epoch": 3.74,
      "learning_rate": 0.00012712840909090908,
      "loss": 2.9011,
      "step": 64450
    },
    {
      "activations/layer0_attention_weight_max": 17.471223831176758,
      "activations/layer0_attention_weight_min": -12.475738525390625,
      "activations/layer10_attention_weight_max": 35.4520263671875,
      "activations/layer10_attention_weight_min": -30.291189193725586,
      "activations/layer11_attention_weight_max": 33.8341064453125,
      "activations/layer11_attention_weight_min": -28.980724334716797,
      "activations/layer12_attention_weight_max": 30.842222213745117,
      "activations/layer12_attention_weight_min": -28.754966735839844,
      "activations/layer13_attention_weight_max": 44.18000793457031,
      "activations/layer13_attention_weight_min": -34.095462799072266,
      "activations/layer14_attention_weight_max": 56.8852424621582,
      "activations/layer14_attention_weight_min": -48.05723190307617,
      "activations/layer15_attention_weight_max": 46.94511032104492,
      "activations/layer15_attention_weight_min": -32.327003479003906,
      "activations/layer16_attention_weight_max": 37.88680648803711,
      "activations/layer16_attention_weight_min": -30.225610733032227,
      "activations/layer17_attention_weight_max": 43.74734115600586,
      "activations/layer17_attention_weight_min": -24.945531845092773,
      "activations/layer18_attention_weight_max": 32.22633361816406,
      "activations/layer18_attention_weight_min": -20.061996459960938,
      "activations/layer19_attention_weight_max": 42.1684455871582,
      "activations/layer19_attention_weight_min": -29.736743927001953,
      "activations/layer1_attention_weight_max": 17.666709899902344,
      "activations/layer1_attention_weight_min": -12.881196022033691,
      "activations/layer20_attention_weight_max": 35.18213653564453,
      "activations/layer20_attention_weight_min": -23.30068016052246,
      "activations/layer21_attention_weight_max": 36.48390579223633,
      "activations/layer21_attention_weight_min": -21.58213233947754,
      "activations/layer22_attention_weight_max": 48.70859909057617,
      "activations/layer22_attention_weight_min": -28.1832218170166,
      "activations/layer23_attention_weight_max": 31.662641525268555,
      "activations/layer23_attention_weight_min": -19.42792320251465,
      "activations/layer2_attention_weight_max": 31.30563735961914,
      "activations/layer2_attention_weight_min": -30.500804901123047,
      "activations/layer3_attention_weight_max": 86.0433349609375,
      "activations/layer3_attention_weight_min": -88.0393295288086,
      "activations/layer4_attention_weight_max": 129.09930419921875,
      "activations/layer4_attention_weight_min": -126.16390228271484,
      "activations/layer5_attention_weight_max": 91.76774597167969,
      "activations/layer5_attention_weight_min": -90.8624267578125,
      "activations/layer6_attention_weight_max": 59.213951110839844,
      "activations/layer6_attention_weight_min": -58.50217056274414,
      "activations/layer7_attention_weight_max": 77.44775390625,
      "activations/layer7_attention_weight_min": -73.97810363769531,
      "activations/layer8_attention_weight_max": 40.4581298828125,
      "activations/layer8_attention_weight_min": -38.04045104980469,
      "activations/layer9_attention_weight_max": 36.95306396484375,
      "activations/layer9_attention_weight_min": -32.09427261352539,
      "epoch": 3.75,
      "learning_rate": 0.0001271094696969697,
      "loss": 2.8883,
      "step": 64500
    },
    {
      "activations/layer0_attention_weight_max": 15.313334465026855,
      "activations/layer0_attention_weight_min": -11.720396041870117,
      "activations/layer10_attention_weight_max": 35.97309112548828,
      "activations/layer10_attention_weight_min": -30.758878707885742,
      "activations/layer11_attention_weight_max": 32.50318908691406,
      "activations/layer11_attention_weight_min": -27.490766525268555,
      "activations/layer12_attention_weight_max": 58.52464294433594,
      "activations/layer12_attention_weight_min": -30.603282928466797,
      "activations/layer13_attention_weight_max": 49.932273864746094,
      "activations/layer13_attention_weight_min": -32.86265182495117,
      "activations/layer14_attention_weight_max": 62.11020278930664,
      "activations/layer14_attention_weight_min": -46.933860778808594,
      "activations/layer15_attention_weight_max": 44.5822868347168,
      "activations/layer15_attention_weight_min": -34.656761169433594,
      "activations/layer16_attention_weight_max": 41.29349136352539,
      "activations/layer16_attention_weight_min": -32.27585220336914,
      "activations/layer17_attention_weight_max": 47.70879364013672,
      "activations/layer17_attention_weight_min": -30.724971771240234,
      "activations/layer18_attention_weight_max": 31.215633392333984,
      "activations/layer18_attention_weight_min": -19.680347442626953,
      "activations/layer19_attention_weight_max": 51.559688568115234,
      "activations/layer19_attention_weight_min": -29.560989379882812,
      "activations/layer1_attention_weight_max": 17.81730079650879,
      "activations/layer1_attention_weight_min": -14.600920677185059,
      "activations/layer20_attention_weight_max": 37.749717712402344,
      "activations/layer20_attention_weight_min": -23.809328079223633,
      "activations/layer21_attention_weight_max": 31.654714584350586,
      "activations/layer21_attention_weight_min": -20.629697799682617,
      "activations/layer22_attention_weight_max": 52.8270149230957,
      "activations/layer22_attention_weight_min": -26.54917335510254,
      "activations/layer23_attention_weight_max": 28.975215911865234,
      "activations/layer23_attention_weight_min": -17.63447380065918,
      "activations/layer2_attention_weight_max": 33.56568908691406,
      "activations/layer2_attention_weight_min": -33.56787109375,
      "activations/layer3_attention_weight_max": 92.36894989013672,
      "activations/layer3_attention_weight_min": -95.2103042602539,
      "activations/layer4_attention_weight_max": 136.54432678222656,
      "activations/layer4_attention_weight_min": -130.14134216308594,
      "activations/layer5_attention_weight_max": 91.78265380859375,
      "activations/layer5_attention_weight_min": -92.63339233398438,
      "activations/layer6_attention_weight_max": 57.54483413696289,
      "activations/layer6_attention_weight_min": -54.73563766479492,
      "activations/layer7_attention_weight_max": 78.0396957397461,
      "activations/layer7_attention_weight_min": -76.82872009277344,
      "activations/layer8_attention_weight_max": 37.57848358154297,
      "activations/layer8_attention_weight_min": -37.26625061035156,
      "activations/layer9_attention_weight_max": 39.63530731201172,
      "activations/layer9_attention_weight_min": -34.17604446411133,
      "epoch": 3.75,
      "learning_rate": 0.00012709053030303028,
      "loss": 2.8661,
      "step": 64550
    },
    {
      "activations/layer0_attention_weight_max": 17.136659622192383,
      "activations/layer0_attention_weight_min": -11.269380569458008,
      "activations/layer10_attention_weight_max": 38.53374099731445,
      "activations/layer10_attention_weight_min": -29.23797607421875,
      "activations/layer11_attention_weight_max": 37.663753509521484,
      "activations/layer11_attention_weight_min": -29.318830490112305,
      "activations/layer12_attention_weight_max": 30.670581817626953,
      "activations/layer12_attention_weight_min": -28.700044631958008,
      "activations/layer13_attention_weight_max": 59.04450225830078,
      "activations/layer13_attention_weight_min": -41.89811706542969,
      "activations/layer14_attention_weight_max": 77.68755340576172,
      "activations/layer14_attention_weight_min": -51.71445083618164,
      "activations/layer15_attention_weight_max": 47.66083526611328,
      "activations/layer15_attention_weight_min": -29.032255172729492,
      "activations/layer16_attention_weight_max": 43.95280838012695,
      "activations/layer16_attention_weight_min": -28.251861572265625,
      "activations/layer17_attention_weight_max": 43.84876251220703,
      "activations/layer17_attention_weight_min": -23.827682495117188,
      "activations/layer18_attention_weight_max": 32.77535629272461,
      "activations/layer18_attention_weight_min": -19.470718383789062,
      "activations/layer19_attention_weight_max": 43.745662689208984,
      "activations/layer19_attention_weight_min": -27.71286392211914,
      "activations/layer1_attention_weight_max": 17.943580627441406,
      "activations/layer1_attention_weight_min": -14.377095222473145,
      "activations/layer20_attention_weight_max": 34.65214538574219,
      "activations/layer20_attention_weight_min": -22.907230377197266,
      "activations/layer21_attention_weight_max": 40.62188720703125,
      "activations/layer21_attention_weight_min": -20.485164642333984,
      "activations/layer22_attention_weight_max": 51.04605484008789,
      "activations/layer22_attention_weight_min": -27.15379524230957,
      "activations/layer23_attention_weight_max": 29.449783325195312,
      "activations/layer23_attention_weight_min": -17.66107177734375,
      "activations/layer2_attention_weight_max": 31.79261589050293,
      "activations/layer2_attention_weight_min": -30.409137725830078,
      "activations/layer3_attention_weight_max": 83.93872833251953,
      "activations/layer3_attention_weight_min": -87.15623474121094,
      "activations/layer4_attention_weight_max": 132.65843200683594,
      "activations/layer4_attention_weight_min": -127.84217834472656,
      "activations/layer5_attention_weight_max": 96.2227783203125,
      "activations/layer5_attention_weight_min": -85.39562225341797,
      "activations/layer6_attention_weight_max": 58.456520080566406,
      "activations/layer6_attention_weight_min": -54.96601104736328,
      "activations/layer7_attention_weight_max": 80.85182189941406,
      "activations/layer7_attention_weight_min": -74.77067565917969,
      "activations/layer8_attention_weight_max": 42.1121711730957,
      "activations/layer8_attention_weight_min": -38.69131088256836,
      "activations/layer9_attention_weight_max": 40.38828659057617,
      "activations/layer9_attention_weight_min": -31.953577041625977,
      "epoch": 3.75,
      "learning_rate": 0.0001270715909090909,
      "loss": 2.8808,
      "step": 64600
    },
    {
      "activations/layer0_attention_weight_max": 16.49416160583496,
      "activations/layer0_attention_weight_min": -11.737347602844238,
      "activations/layer10_attention_weight_max": 37.05498504638672,
      "activations/layer10_attention_weight_min": -31.84215545654297,
      "activations/layer11_attention_weight_max": 33.11978530883789,
      "activations/layer11_attention_weight_min": -26.184402465820312,
      "activations/layer12_attention_weight_max": 27.856664657592773,
      "activations/layer12_attention_weight_min": -28.4371395111084,
      "activations/layer13_attention_weight_max": 43.910465240478516,
      "activations/layer13_attention_weight_min": -32.29328155517578,
      "activations/layer14_attention_weight_max": 63.00131607055664,
      "activations/layer14_attention_weight_min": -50.492671966552734,
      "activations/layer15_attention_weight_max": 41.80882263183594,
      "activations/layer15_attention_weight_min": -30.46967124938965,
      "activations/layer16_attention_weight_max": 40.21036911010742,
      "activations/layer16_attention_weight_min": -29.330900192260742,
      "activations/layer17_attention_weight_max": 43.9911003112793,
      "activations/layer17_attention_weight_min": -26.64566993713379,
      "activations/layer18_attention_weight_max": 31.464641571044922,
      "activations/layer18_attention_weight_min": -20.039382934570312,
      "activations/layer19_attention_weight_max": 41.8874397277832,
      "activations/layer19_attention_weight_min": -28.776248931884766,
      "activations/layer1_attention_weight_max": 17.256803512573242,
      "activations/layer1_attention_weight_min": -14.541581153869629,
      "activations/layer20_attention_weight_max": 33.4705924987793,
      "activations/layer20_attention_weight_min": -24.385292053222656,
      "activations/layer21_attention_weight_max": 37.341312408447266,
      "activations/layer21_attention_weight_min": -20.103137969970703,
      "activations/layer22_attention_weight_max": 53.36534118652344,
      "activations/layer22_attention_weight_min": -28.494966506958008,
      "activations/layer23_attention_weight_max": 32.180213928222656,
      "activations/layer23_attention_weight_min": -21.18616485595703,
      "activations/layer2_attention_weight_max": 30.48871612548828,
      "activations/layer2_attention_weight_min": -31.815814971923828,
      "activations/layer3_attention_weight_max": 83.8433609008789,
      "activations/layer3_attention_weight_min": -90.52877807617188,
      "activations/layer4_attention_weight_max": 134.82557678222656,
      "activations/layer4_attention_weight_min": -140.47569274902344,
      "activations/layer5_attention_weight_max": 99.51659393310547,
      "activations/layer5_attention_weight_min": -92.12044525146484,
      "activations/layer6_attention_weight_max": 58.33686065673828,
      "activations/layer6_attention_weight_min": -58.57954788208008,
      "activations/layer7_attention_weight_max": 71.62909698486328,
      "activations/layer7_attention_weight_min": -74.9808349609375,
      "activations/layer8_attention_weight_max": 37.70791244506836,
      "activations/layer8_attention_weight_min": -37.461063385009766,
      "activations/layer9_attention_weight_max": 42.55141830444336,
      "activations/layer9_attention_weight_min": -34.170284271240234,
      "epoch": 3.76,
      "learning_rate": 0.0001270526515151515,
      "loss": 2.8873,
      "step": 64650
    },
    {
      "activations/layer0_attention_weight_max": 16.766345977783203,
      "activations/layer0_attention_weight_min": -11.212589263916016,
      "activations/layer10_attention_weight_max": 36.55703353881836,
      "activations/layer10_attention_weight_min": -30.4468994140625,
      "activations/layer11_attention_weight_max": 33.545982360839844,
      "activations/layer11_attention_weight_min": -30.0941162109375,
      "activations/layer12_attention_weight_max": 34.46466827392578,
      "activations/layer12_attention_weight_min": -25.558509826660156,
      "activations/layer13_attention_weight_max": 52.358768463134766,
      "activations/layer13_attention_weight_min": -36.683860778808594,
      "activations/layer14_attention_weight_max": 66.91834259033203,
      "activations/layer14_attention_weight_min": -46.39511489868164,
      "activations/layer15_attention_weight_max": 42.96046447753906,
      "activations/layer15_attention_weight_min": -29.26496124267578,
      "activations/layer16_attention_weight_max": 43.27134704589844,
      "activations/layer16_attention_weight_min": -31.301729202270508,
      "activations/layer17_attention_weight_max": 42.41224670410156,
      "activations/layer17_attention_weight_min": -27.054651260375977,
      "activations/layer18_attention_weight_max": 34.655364990234375,
      "activations/layer18_attention_weight_min": -20.426828384399414,
      "activations/layer19_attention_weight_max": 45.254886627197266,
      "activations/layer19_attention_weight_min": -30.977523803710938,
      "activations/layer1_attention_weight_max": 17.189746856689453,
      "activations/layer1_attention_weight_min": -16.1881160736084,
      "activations/layer20_attention_weight_max": 34.755001068115234,
      "activations/layer20_attention_weight_min": -24.050212860107422,
      "activations/layer21_attention_weight_max": 35.90966033935547,
      "activations/layer21_attention_weight_min": -20.85454750061035,
      "activations/layer22_attention_weight_max": 56.686519622802734,
      "activations/layer22_attention_weight_min": -29.284944534301758,
      "activations/layer23_attention_weight_max": 30.904144287109375,
      "activations/layer23_attention_weight_min": -19.545475006103516,
      "activations/layer2_attention_weight_max": 32.13679885864258,
      "activations/layer2_attention_weight_min": -33.706298828125,
      "activations/layer3_attention_weight_max": 85.47723388671875,
      "activations/layer3_attention_weight_min": -89.22250366210938,
      "activations/layer4_attention_weight_max": 133.2223663330078,
      "activations/layer4_attention_weight_min": -127.6963882446289,
      "activations/layer5_attention_weight_max": 95.19052124023438,
      "activations/layer5_attention_weight_min": -86.83538818359375,
      "activations/layer6_attention_weight_max": 58.770751953125,
      "activations/layer6_attention_weight_min": -55.601722717285156,
      "activations/layer7_attention_weight_max": 77.89439392089844,
      "activations/layer7_attention_weight_min": -77.71886444091797,
      "activations/layer8_attention_weight_max": 38.252952575683594,
      "activations/layer8_attention_weight_min": -38.33829879760742,
      "activations/layer9_attention_weight_max": 44.773033142089844,
      "activations/layer9_attention_weight_min": -38.374507904052734,
      "epoch": 3.76,
      "learning_rate": 0.0001270337121212121,
      "loss": 2.9027,
      "step": 64700
    },
    {
      "activations/layer0_attention_weight_max": 15.067426681518555,
      "activations/layer0_attention_weight_min": -13.90925407409668,
      "activations/layer10_attention_weight_max": 37.46958923339844,
      "activations/layer10_attention_weight_min": -30.803085327148438,
      "activations/layer11_attention_weight_max": 32.26518630981445,
      "activations/layer11_attention_weight_min": -27.30115509033203,
      "activations/layer12_attention_weight_max": 24.018325805664062,
      "activations/layer12_attention_weight_min": -28.380273818969727,
      "activations/layer13_attention_weight_max": 44.39026641845703,
      "activations/layer13_attention_weight_min": -36.04254150390625,
      "activations/layer14_attention_weight_max": 56.36843490600586,
      "activations/layer14_attention_weight_min": -46.45416259765625,
      "activations/layer15_attention_weight_max": 43.083335876464844,
      "activations/layer15_attention_weight_min": -31.39899253845215,
      "activations/layer16_attention_weight_max": 36.546653747558594,
      "activations/layer16_attention_weight_min": -29.339292526245117,
      "activations/layer17_attention_weight_max": 41.685150146484375,
      "activations/layer17_attention_weight_min": -25.243701934814453,
      "activations/layer18_attention_weight_max": 32.32796859741211,
      "activations/layer18_attention_weight_min": -22.14141845703125,
      "activations/layer19_attention_weight_max": 39.71574401855469,
      "activations/layer19_attention_weight_min": -29.164066314697266,
      "activations/layer1_attention_weight_max": 16.330984115600586,
      "activations/layer1_attention_weight_min": -17.64564323425293,
      "activations/layer20_attention_weight_max": 34.52009582519531,
      "activations/layer20_attention_weight_min": -25.5690860748291,
      "activations/layer21_attention_weight_max": 30.984477996826172,
      "activations/layer21_attention_weight_min": -19.9168701171875,
      "activations/layer22_attention_weight_max": 46.87317657470703,
      "activations/layer22_attention_weight_min": -27.240949630737305,
      "activations/layer23_attention_weight_max": 27.274009704589844,
      "activations/layer23_attention_weight_min": -17.54193878173828,
      "activations/layer2_attention_weight_max": 31.622360229492188,
      "activations/layer2_attention_weight_min": -31.553844451904297,
      "activations/layer3_attention_weight_max": 81.96088409423828,
      "activations/layer3_attention_weight_min": -82.0672836303711,
      "activations/layer4_attention_weight_max": 139.3230743408203,
      "activations/layer4_attention_weight_min": -125.88934326171875,
      "activations/layer5_attention_weight_max": 102.30513763427734,
      "activations/layer5_attention_weight_min": -90.41114807128906,
      "activations/layer6_attention_weight_max": 66.79033660888672,
      "activations/layer6_attention_weight_min": -56.23697280883789,
      "activations/layer7_attention_weight_max": 70.3755111694336,
      "activations/layer7_attention_weight_min": -78.81718444824219,
      "activations/layer8_attention_weight_max": 39.300296783447266,
      "activations/layer8_attention_weight_min": -38.29164505004883,
      "activations/layer9_attention_weight_max": 44.8348388671875,
      "activations/layer9_attention_weight_min": -33.29951095581055,
      "epoch": 3.76,
      "learning_rate": 0.0001270147727272727,
      "loss": 2.9398,
      "step": 64750
    },
    {
      "activations/layer0_attention_weight_max": 15.186068534851074,
      "activations/layer0_attention_weight_min": -13.472529411315918,
      "activations/layer10_attention_weight_max": 37.48478698730469,
      "activations/layer10_attention_weight_min": -30.862028121948242,
      "activations/layer11_attention_weight_max": 36.656890869140625,
      "activations/layer11_attention_weight_min": -30.407535552978516,
      "activations/layer12_attention_weight_max": 26.34125328063965,
      "activations/layer12_attention_weight_min": -26.646339416503906,
      "activations/layer13_attention_weight_max": 46.104591369628906,
      "activations/layer13_attention_weight_min": -34.2669677734375,
      "activations/layer14_attention_weight_max": 83.0471420288086,
      "activations/layer14_attention_weight_min": -67.74043273925781,
      "activations/layer15_attention_weight_max": 45.005699157714844,
      "activations/layer15_attention_weight_min": -33.13431167602539,
      "activations/layer16_attention_weight_max": 41.34873580932617,
      "activations/layer16_attention_weight_min": -29.84272575378418,
      "activations/layer17_attention_weight_max": 40.63407897949219,
      "activations/layer17_attention_weight_min": -26.9182186126709,
      "activations/layer18_attention_weight_max": 31.298982620239258,
      "activations/layer18_attention_weight_min": -19.030471801757812,
      "activations/layer19_attention_weight_max": 41.69376754760742,
      "activations/layer19_attention_weight_min": -28.698623657226562,
      "activations/layer1_attention_weight_max": 17.80304527282715,
      "activations/layer1_attention_weight_min": -17.501728057861328,
      "activations/layer20_attention_weight_max": 32.04066848754883,
      "activations/layer20_attention_weight_min": -25.250328063964844,
      "activations/layer21_attention_weight_max": 30.54526710510254,
      "activations/layer21_attention_weight_min": -20.764493942260742,
      "activations/layer22_attention_weight_max": 47.80710983276367,
      "activations/layer22_attention_weight_min": -26.922937393188477,
      "activations/layer23_attention_weight_max": 31.556713104248047,
      "activations/layer23_attention_weight_min": -17.73934555053711,
      "activations/layer2_attention_weight_max": 30.65882682800293,
      "activations/layer2_attention_weight_min": -30.702003479003906,
      "activations/layer3_attention_weight_max": 81.04085540771484,
      "activations/layer3_attention_weight_min": -82.4813461303711,
      "activations/layer4_attention_weight_max": 133.20797729492188,
      "activations/layer4_attention_weight_min": -122.45027160644531,
      "activations/layer5_attention_weight_max": 94.2854232788086,
      "activations/layer5_attention_weight_min": -86.88410949707031,
      "activations/layer6_attention_weight_max": 60.1258659362793,
      "activations/layer6_attention_weight_min": -54.68547439575195,
      "activations/layer7_attention_weight_max": 77.74518585205078,
      "activations/layer7_attention_weight_min": -75.40288543701172,
      "activations/layer8_attention_weight_max": 41.22774124145508,
      "activations/layer8_attention_weight_min": -39.468971252441406,
      "activations/layer9_attention_weight_max": 46.18465805053711,
      "activations/layer9_attention_weight_min": -32.42308044433594,
      "epoch": 3.77,
      "learning_rate": 0.00012699583333333333,
      "loss": 2.8822,
      "step": 64800
    },
    {
      "activations/layer0_attention_weight_max": 14.137165069580078,
      "activations/layer0_attention_weight_min": -12.78542709350586,
      "activations/layer10_attention_weight_max": 42.19388198852539,
      "activations/layer10_attention_weight_min": -33.629085540771484,
      "activations/layer11_attention_weight_max": 40.41899108886719,
      "activations/layer11_attention_weight_min": -28.63349151611328,
      "activations/layer12_attention_weight_max": 30.497928619384766,
      "activations/layer12_attention_weight_min": -36.31736755371094,
      "activations/layer13_attention_weight_max": 46.1722297668457,
      "activations/layer13_attention_weight_min": -31.531190872192383,
      "activations/layer14_attention_weight_max": 65.98583984375,
      "activations/layer14_attention_weight_min": -50.525455474853516,
      "activations/layer15_attention_weight_max": 48.02793884277344,
      "activations/layer15_attention_weight_min": -32.258941650390625,
      "activations/layer16_attention_weight_max": 42.1341667175293,
      "activations/layer16_attention_weight_min": -30.491695404052734,
      "activations/layer17_attention_weight_max": 41.590450286865234,
      "activations/layer17_attention_weight_min": -27.787057876586914,
      "activations/layer18_attention_weight_max": 34.695648193359375,
      "activations/layer18_attention_weight_min": -19.696868896484375,
      "activations/layer19_attention_weight_max": 39.50688934326172,
      "activations/layer19_attention_weight_min": -30.004179000854492,
      "activations/layer1_attention_weight_max": 18.021892547607422,
      "activations/layer1_attention_weight_min": -15.585193634033203,
      "activations/layer20_attention_weight_max": 37.54669952392578,
      "activations/layer20_attention_weight_min": -25.48000717163086,
      "activations/layer21_attention_weight_max": 30.759506225585938,
      "activations/layer21_attention_weight_min": -21.125574111938477,
      "activations/layer22_attention_weight_max": 50.06528091430664,
      "activations/layer22_attention_weight_min": -27.53704071044922,
      "activations/layer23_attention_weight_max": 26.86681365966797,
      "activations/layer23_attention_weight_min": -18.998653411865234,
      "activations/layer2_attention_weight_max": 32.028770446777344,
      "activations/layer2_attention_weight_min": -30.538671493530273,
      "activations/layer3_attention_weight_max": 81.960205078125,
      "activations/layer3_attention_weight_min": -82.40312194824219,
      "activations/layer4_attention_weight_max": 127.6581039428711,
      "activations/layer4_attention_weight_min": -123.85552215576172,
      "activations/layer5_attention_weight_max": 93.39350891113281,
      "activations/layer5_attention_weight_min": -84.9550552368164,
      "activations/layer6_attention_weight_max": 55.8023681640625,
      "activations/layer6_attention_weight_min": -58.26365280151367,
      "activations/layer7_attention_weight_max": 77.47412109375,
      "activations/layer7_attention_weight_min": -73.40926361083984,
      "activations/layer8_attention_weight_max": 41.271053314208984,
      "activations/layer8_attention_weight_min": -38.28778839111328,
      "activations/layer9_attention_weight_max": 51.328346252441406,
      "activations/layer9_attention_weight_min": -45.22455596923828,
      "epoch": 3.77,
      "learning_rate": 0.00012697689393939392,
      "loss": 2.8919,
      "step": 64850
    },
    {
      "activations/layer0_attention_weight_max": 15.551498413085938,
      "activations/layer0_attention_weight_min": -12.087249755859375,
      "activations/layer10_attention_weight_max": 39.0678596496582,
      "activations/layer10_attention_weight_min": -32.112701416015625,
      "activations/layer11_attention_weight_max": 33.971656799316406,
      "activations/layer11_attention_weight_min": -28.169551849365234,
      "activations/layer12_attention_weight_max": 30.52354621887207,
      "activations/layer12_attention_weight_min": -26.921911239624023,
      "activations/layer13_attention_weight_max": 46.84233856201172,
      "activations/layer13_attention_weight_min": -32.68444061279297,
      "activations/layer14_attention_weight_max": 57.31062698364258,
      "activations/layer14_attention_weight_min": -42.260887145996094,
      "activations/layer15_attention_weight_max": 46.035831451416016,
      "activations/layer15_attention_weight_min": -29.318065643310547,
      "activations/layer16_attention_weight_max": 40.37020492553711,
      "activations/layer16_attention_weight_min": -29.430809020996094,
      "activations/layer17_attention_weight_max": 43.21971893310547,
      "activations/layer17_attention_weight_min": -26.263290405273438,
      "activations/layer18_attention_weight_max": 31.28228759765625,
      "activations/layer18_attention_weight_min": -20.47246742248535,
      "activations/layer19_attention_weight_max": 39.56087875366211,
      "activations/layer19_attention_weight_min": -29.304906845092773,
      "activations/layer1_attention_weight_max": 16.828189849853516,
      "activations/layer1_attention_weight_min": -16.874666213989258,
      "activations/layer20_attention_weight_max": 34.89453887939453,
      "activations/layer20_attention_weight_min": -24.155269622802734,
      "activations/layer21_attention_weight_max": 34.519317626953125,
      "activations/layer21_attention_weight_min": -22.339536666870117,
      "activations/layer22_attention_weight_max": 48.76370620727539,
      "activations/layer22_attention_weight_min": -27.824785232543945,
      "activations/layer23_attention_weight_max": 28.750289916992188,
      "activations/layer23_attention_weight_min": -19.739770889282227,
      "activations/layer2_attention_weight_max": 31.056190490722656,
      "activations/layer2_attention_weight_min": -29.990604400634766,
      "activations/layer3_attention_weight_max": 80.53751373291016,
      "activations/layer3_attention_weight_min": -83.39518737792969,
      "activations/layer4_attention_weight_max": 126.38329315185547,
      "activations/layer4_attention_weight_min": -125.6435317993164,
      "activations/layer5_attention_weight_max": 91.82341003417969,
      "activations/layer5_attention_weight_min": -85.5882568359375,
      "activations/layer6_attention_weight_max": 55.1163215637207,
      "activations/layer6_attention_weight_min": -54.84280014038086,
      "activations/layer7_attention_weight_max": 74.22322082519531,
      "activations/layer7_attention_weight_min": -71.63656616210938,
      "activations/layer8_attention_weight_max": 41.7016487121582,
      "activations/layer8_attention_weight_min": -37.77073287963867,
      "activations/layer9_attention_weight_max": 40.498714447021484,
      "activations/layer9_attention_weight_min": -34.376155853271484,
      "epoch": 3.77,
      "learning_rate": 0.00012695795454545453,
      "loss": 2.8847,
      "step": 64900
    },
    {
      "activations/layer0_attention_weight_max": 13.734248161315918,
      "activations/layer0_attention_weight_min": -11.66404914855957,
      "activations/layer10_attention_weight_max": 36.39406204223633,
      "activations/layer10_attention_weight_min": -28.613555908203125,
      "activations/layer11_attention_weight_max": 46.505279541015625,
      "activations/layer11_attention_weight_min": -33.92485046386719,
      "activations/layer12_attention_weight_max": 32.88264846801758,
      "activations/layer12_attention_weight_min": -27.353511810302734,
      "activations/layer13_attention_weight_max": 60.49897384643555,
      "activations/layer13_attention_weight_min": -42.0384407043457,
      "activations/layer14_attention_weight_max": 75.10035705566406,
      "activations/layer14_attention_weight_min": -56.75041580200195,
      "activations/layer15_attention_weight_max": 43.959415435791016,
      "activations/layer15_attention_weight_min": -29.86290740966797,
      "activations/layer16_attention_weight_max": 41.497005462646484,
      "activations/layer16_attention_weight_min": -27.936254501342773,
      "activations/layer17_attention_weight_max": 42.9205207824707,
      "activations/layer17_attention_weight_min": -26.16922378540039,
      "activations/layer18_attention_weight_max": 32.64946746826172,
      "activations/layer18_attention_weight_min": -19.50802993774414,
      "activations/layer19_attention_weight_max": 47.493385314941406,
      "activations/layer19_attention_weight_min": -28.447128295898438,
      "activations/layer1_attention_weight_max": 16.862276077270508,
      "activations/layer1_attention_weight_min": -14.401463508605957,
      "activations/layer20_attention_weight_max": 35.927772521972656,
      "activations/layer20_attention_weight_min": -24.280311584472656,
      "activations/layer21_attention_weight_max": 38.002315521240234,
      "activations/layer21_attention_weight_min": -20.51328468322754,
      "activations/layer22_attention_weight_max": 52.48420715332031,
      "activations/layer22_attention_weight_min": -28.576799392700195,
      "activations/layer23_attention_weight_max": 32.245269775390625,
      "activations/layer23_attention_weight_min": -17.71196746826172,
      "activations/layer2_attention_weight_max": 32.047607421875,
      "activations/layer2_attention_weight_min": -31.90748405456543,
      "activations/layer3_attention_weight_max": 80.07183837890625,
      "activations/layer3_attention_weight_min": -86.73763275146484,
      "activations/layer4_attention_weight_max": 128.92576599121094,
      "activations/layer4_attention_weight_min": -121.7923355102539,
      "activations/layer5_attention_weight_max": 88.36669158935547,
      "activations/layer5_attention_weight_min": -84.3813705444336,
      "activations/layer6_attention_weight_max": 57.92902374267578,
      "activations/layer6_attention_weight_min": -56.299705505371094,
      "activations/layer7_attention_weight_max": 77.79493713378906,
      "activations/layer7_attention_weight_min": -76.34368133544922,
      "activations/layer8_attention_weight_max": 40.01133728027344,
      "activations/layer8_attention_weight_min": -37.378665924072266,
      "activations/layer9_attention_weight_max": 58.99211502075195,
      "activations/layer9_attention_weight_min": -37.280879974365234,
      "epoch": 3.77,
      "learning_rate": 0.00012693901515151515,
      "loss": 2.9008,
      "step": 64950
    },
    {
      "activations/layer0_attention_weight_max": 16.085338592529297,
      "activations/layer0_attention_weight_min": -11.523788452148438,
      "activations/layer10_attention_weight_max": 34.589500427246094,
      "activations/layer10_attention_weight_min": -30.191015243530273,
      "activations/layer11_attention_weight_max": 33.16387176513672,
      "activations/layer11_attention_weight_min": -28.58929443359375,
      "activations/layer12_attention_weight_max": 30.651500701904297,
      "activations/layer12_attention_weight_min": -29.015241622924805,
      "activations/layer13_attention_weight_max": 43.53835678100586,
      "activations/layer13_attention_weight_min": -34.503746032714844,
      "activations/layer14_attention_weight_max": 63.425323486328125,
      "activations/layer14_attention_weight_min": -43.8494873046875,
      "activations/layer15_attention_weight_max": 45.61741638183594,
      "activations/layer15_attention_weight_min": -30.473203659057617,
      "activations/layer16_attention_weight_max": 40.939754486083984,
      "activations/layer16_attention_weight_min": -28.742202758789062,
      "activations/layer17_attention_weight_max": 41.31199645996094,
      "activations/layer17_attention_weight_min": -24.536596298217773,
      "activations/layer18_attention_weight_max": 30.20973014831543,
      "activations/layer18_attention_weight_min": -18.983552932739258,
      "activations/layer19_attention_weight_max": 41.31834030151367,
      "activations/layer19_attention_weight_min": -27.379972457885742,
      "activations/layer1_attention_weight_max": 17.08617401123047,
      "activations/layer1_attention_weight_min": -16.195524215698242,
      "activations/layer20_attention_weight_max": 32.031009674072266,
      "activations/layer20_attention_weight_min": -24.17104721069336,
      "activations/layer21_attention_weight_max": 33.791893005371094,
      "activations/layer21_attention_weight_min": -21.902070999145508,
      "activations/layer22_attention_weight_max": 44.542762756347656,
      "activations/layer22_attention_weight_min": -27.429433822631836,
      "activations/layer23_attention_weight_max": 27.47382926940918,
      "activations/layer23_attention_weight_min": -20.657451629638672,
      "activations/layer2_attention_weight_max": 31.191593170166016,
      "activations/layer2_attention_weight_min": -31.70138931274414,
      "activations/layer3_attention_weight_max": 81.02732849121094,
      "activations/layer3_attention_weight_min": -82.18573760986328,
      "activations/layer4_attention_weight_max": 131.07945251464844,
      "activations/layer4_attention_weight_min": -125.42760467529297,
      "activations/layer5_attention_weight_max": 90.53106689453125,
      "activations/layer5_attention_weight_min": -83.3221435546875,
      "activations/layer6_attention_weight_max": 55.087120056152344,
      "activations/layer6_attention_weight_min": -54.378517150878906,
      "activations/layer7_attention_weight_max": 73.21578216552734,
      "activations/layer7_attention_weight_min": -74.79518127441406,
      "activations/layer8_attention_weight_max": 37.62731170654297,
      "activations/layer8_attention_weight_min": -38.068477630615234,
      "activations/layer9_attention_weight_max": 39.16952896118164,
      "activations/layer9_attention_weight_min": -39.133113861083984,
      "epoch": 3.78,
      "learning_rate": 0.00012692007575757573,
      "loss": 2.8888,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_loss": 2.830078125,
      "eval_runtime": 8.4806,
      "eval_samples_per_second": 506.333,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_openwebtext_loss": 2.830078125,
      "eval_openwebtext_ppl": 16.94678474038272,
      "eval_openwebtext_runtime": 8.4806,
      "eval_openwebtext_samples_per_second": 506.333,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_wikitext_loss": 3.056640625,
      "eval_wikitext_ppl": 21.256030110048442,
      "eval_wikitext_runtime": 2.0043,
      "eval_wikitext_samples_per_second": 227.516,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_lambada_loss": 2.814453125,
      "eval_lambada_ppl": 16.684049195604416,
      "eval_lambada_runtime": 9.6051,
      "eval_lambada_samples_per_second": 506.918,
      "step": 65000
    },
    {
      "activations/layer0_attention_weight_max": 15.911731719970703,
      "activations/layer0_attention_weight_min": -12.548118591308594,
      "activations/layer10_attention_weight_max": 33.39649963378906,
      "activations/layer10_attention_weight_min": -29.527555465698242,
      "activations/layer11_attention_weight_max": 31.06696128845215,
      "activations/layer11_attention_weight_min": -27.454418182373047,
      "activations/layer12_attention_weight_max": 53.886329650878906,
      "activations/layer12_attention_weight_min": -36.59069061279297,
      "activations/layer13_attention_weight_max": 45.79318618774414,
      "activations/layer13_attention_weight_min": -34.55648422241211,
      "activations/layer14_attention_weight_max": 79.00293731689453,
      "activations/layer14_attention_weight_min": -59.605587005615234,
      "activations/layer15_attention_weight_max": 40.837337493896484,
      "activations/layer15_attention_weight_min": -28.470991134643555,
      "activations/layer16_attention_weight_max": 35.05096435546875,
      "activations/layer16_attention_weight_min": -29.31092643737793,
      "activations/layer17_attention_weight_max": 38.61051559448242,
      "activations/layer17_attention_weight_min": -25.356626510620117,
      "activations/layer18_attention_weight_max": 33.526611328125,
      "activations/layer18_attention_weight_min": -22.23735237121582,
      "activations/layer19_attention_weight_max": 40.04189682006836,
      "activations/layer19_attention_weight_min": -30.415502548217773,
      "activations/layer1_attention_weight_max": 17.766677856445312,
      "activations/layer1_attention_weight_min": -16.950088500976562,
      "activations/layer20_attention_weight_max": 34.691280364990234,
      "activations/layer20_attention_weight_min": -22.454978942871094,
      "activations/layer21_attention_weight_max": 35.52339553833008,
      "activations/layer21_attention_weight_min": -21.115617752075195,
      "activations/layer22_attention_weight_max": 50.71915817260742,
      "activations/layer22_attention_weight_min": -27.96977996826172,
      "activations/layer23_attention_weight_max": 30.999801635742188,
      "activations/layer23_attention_weight_min": -17.805030822753906,
      "activations/layer2_attention_weight_max": 32.103294372558594,
      "activations/layer2_attention_weight_min": -30.870681762695312,
      "activations/layer3_attention_weight_max": 85.43183135986328,
      "activations/layer3_attention_weight_min": -84.20474243164062,
      "activations/layer4_attention_weight_max": 128.35394287109375,
      "activations/layer4_attention_weight_min": -128.4318084716797,
      "activations/layer5_attention_weight_max": 89.5550765991211,
      "activations/layer5_attention_weight_min": -83.26441955566406,
      "activations/layer6_attention_weight_max": 53.772457122802734,
      "activations/layer6_attention_weight_min": -54.94743728637695,
      "activations/layer7_attention_weight_max": 71.01930236816406,
      "activations/layer7_attention_weight_min": -76.53638458251953,
      "activations/layer8_attention_weight_max": 37.10567855834961,
      "activations/layer8_attention_weight_min": -36.7802848815918,
      "activations/layer9_attention_weight_max": 36.6819953918457,
      "activations/layer9_attention_weight_min": -33.87726974487305,
      "epoch": 3.78,
      "learning_rate": 0.00012690113636363635,
      "loss": 2.8861,
      "step": 65050
    },
    {
      "activations/layer0_attention_weight_max": 15.029221534729004,
      "activations/layer0_attention_weight_min": -11.371086120605469,
      "activations/layer10_attention_weight_max": 36.35774612426758,
      "activations/layer10_attention_weight_min": -31.637582778930664,
      "activations/layer11_attention_weight_max": 38.52410888671875,
      "activations/layer11_attention_weight_min": -30.693092346191406,
      "activations/layer12_attention_weight_max": 34.415992736816406,
      "activations/layer12_attention_weight_min": -29.17066192626953,
      "activations/layer13_attention_weight_max": 47.56525802612305,
      "activations/layer13_attention_weight_min": -32.911102294921875,
      "activations/layer14_attention_weight_max": 63.53390884399414,
      "activations/layer14_attention_weight_min": -47.022064208984375,
      "activations/layer15_attention_weight_max": 46.25299072265625,
      "activations/layer15_attention_weight_min": -35.88530349731445,
      "activations/layer16_attention_weight_max": 44.46048355102539,
      "activations/layer16_attention_weight_min": -28.71141815185547,
      "activations/layer17_attention_weight_max": 45.58180618286133,
      "activations/layer17_attention_weight_min": -26.57611656188965,
      "activations/layer18_attention_weight_max": 34.00278854370117,
      "activations/layer18_attention_weight_min": -19.8879337310791,
      "activations/layer19_attention_weight_max": 47.861656188964844,
      "activations/layer19_attention_weight_min": -28.495641708374023,
      "activations/layer1_attention_weight_max": 19.402849197387695,
      "activations/layer1_attention_weight_min": -17.741931915283203,
      "activations/layer20_attention_weight_max": 42.05937576293945,
      "activations/layer20_attention_weight_min": -22.664628982543945,
      "activations/layer21_attention_weight_max": 42.829071044921875,
      "activations/layer21_attention_weight_min": -21.23383903503418,
      "activations/layer22_attention_weight_max": 54.24530029296875,
      "activations/layer22_attention_weight_min": -30.90321922302246,
      "activations/layer23_attention_weight_max": 31.981924057006836,
      "activations/layer23_attention_weight_min": -18.812053680419922,
      "activations/layer2_attention_weight_max": 32.53394317626953,
      "activations/layer2_attention_weight_min": -31.066055297851562,
      "activations/layer3_attention_weight_max": 82.97038269042969,
      "activations/layer3_attention_weight_min": -82.46336364746094,
      "activations/layer4_attention_weight_max": 127.1799545288086,
      "activations/layer4_attention_weight_min": -122.37096405029297,
      "activations/layer5_attention_weight_max": 93.55474853515625,
      "activations/layer5_attention_weight_min": -84.02259063720703,
      "activations/layer6_attention_weight_max": 58.8175163269043,
      "activations/layer6_attention_weight_min": -55.55938720703125,
      "activations/layer7_attention_weight_max": 76.7726058959961,
      "activations/layer7_attention_weight_min": -74.53134155273438,
      "activations/layer8_attention_weight_max": 41.411773681640625,
      "activations/layer8_attention_weight_min": -38.618614196777344,
      "activations/layer9_attention_weight_max": 37.01309585571289,
      "activations/layer9_attention_weight_min": -38.477989196777344,
      "epoch": 3.78,
      "learning_rate": 0.00012688219696969696,
      "loss": 2.8596,
      "step": 65100
    },
    {
      "activations/layer0_attention_weight_max": 16.50377082824707,
      "activations/layer0_attention_weight_min": -11.621084213256836,
      "activations/layer10_attention_weight_max": 38.59538269042969,
      "activations/layer10_attention_weight_min": -30.782283782958984,
      "activations/layer11_attention_weight_max": 36.130313873291016,
      "activations/layer11_attention_weight_min": -27.42817497253418,
      "activations/layer12_attention_weight_max": 40.297515869140625,
      "activations/layer12_attention_weight_min": -29.382160186767578,
      "activations/layer13_attention_weight_max": 54.50503158569336,
      "activations/layer13_attention_weight_min": -39.01466751098633,
      "activations/layer14_attention_weight_max": 65.99644470214844,
      "activations/layer14_attention_weight_min": -45.53642272949219,
      "activations/layer15_attention_weight_max": 43.7857780456543,
      "activations/layer15_attention_weight_min": -28.905746459960938,
      "activations/layer16_attention_weight_max": 41.159000396728516,
      "activations/layer16_attention_weight_min": -29.01093101501465,
      "activations/layer17_attention_weight_max": 49.972957611083984,
      "activations/layer17_attention_weight_min": -24.761194229125977,
      "activations/layer18_attention_weight_max": 34.85322189331055,
      "activations/layer18_attention_weight_min": -19.890106201171875,
      "activations/layer19_attention_weight_max": 53.22996139526367,
      "activations/layer19_attention_weight_min": -29.295400619506836,
      "activations/layer1_attention_weight_max": 18.082002639770508,
      "activations/layer1_attention_weight_min": -16.091535568237305,
      "activations/layer20_attention_weight_max": 36.426734924316406,
      "activations/layer20_attention_weight_min": -22.21845817565918,
      "activations/layer21_attention_weight_max": 37.200347900390625,
      "activations/layer21_attention_weight_min": -19.831058502197266,
      "activations/layer22_attention_weight_max": 53.3630256652832,
      "activations/layer22_attention_weight_min": -28.128904342651367,
      "activations/layer23_attention_weight_max": 32.01292419433594,
      "activations/layer23_attention_weight_min": -16.9171142578125,
      "activations/layer2_attention_weight_max": 32.41148376464844,
      "activations/layer2_attention_weight_min": -32.13125991821289,
      "activations/layer3_attention_weight_max": 84.57351684570312,
      "activations/layer3_attention_weight_min": -89.04710388183594,
      "activations/layer4_attention_weight_max": 131.84841918945312,
      "activations/layer4_attention_weight_min": -130.31344604492188,
      "activations/layer5_attention_weight_max": 103.85711669921875,
      "activations/layer5_attention_weight_min": -91.85829162597656,
      "activations/layer6_attention_weight_max": 58.2957878112793,
      "activations/layer6_attention_weight_min": -56.55136489868164,
      "activations/layer7_attention_weight_max": 77.17937469482422,
      "activations/layer7_attention_weight_min": -79.01236724853516,
      "activations/layer8_attention_weight_max": 40.49611282348633,
      "activations/layer8_attention_weight_min": -39.65163803100586,
      "activations/layer9_attention_weight_max": 42.77068328857422,
      "activations/layer9_attention_weight_min": -33.613040924072266,
      "epoch": 3.79,
      "learning_rate": 0.00012686325757575755,
      "loss": 2.8799,
      "step": 65150
    },
    {
      "activations/layer0_attention_weight_max": 15.540656089782715,
      "activations/layer0_attention_weight_min": -11.051974296569824,
      "activations/layer10_attention_weight_max": 36.878578186035156,
      "activations/layer10_attention_weight_min": -33.096923828125,
      "activations/layer11_attention_weight_max": 35.53826904296875,
      "activations/layer11_attention_weight_min": -29.1254940032959,
      "activations/layer12_attention_weight_max": 31.390522003173828,
      "activations/layer12_attention_weight_min": -24.30023765563965,
      "activations/layer13_attention_weight_max": 49.25699234008789,
      "activations/layer13_attention_weight_min": -33.30149841308594,
      "activations/layer14_attention_weight_max": 83.98585510253906,
      "activations/layer14_attention_weight_min": -53.186851501464844,
      "activations/layer15_attention_weight_max": 47.28104782104492,
      "activations/layer15_attention_weight_min": -28.351238250732422,
      "activations/layer16_attention_weight_max": 46.449462890625,
      "activations/layer16_attention_weight_min": -28.978242874145508,
      "activations/layer17_attention_weight_max": 51.484493255615234,
      "activations/layer17_attention_weight_min": -25.801223754882812,
      "activations/layer18_attention_weight_max": 47.105003356933594,
      "activations/layer18_attention_weight_min": -21.5540828704834,
      "activations/layer19_attention_weight_max": 51.41677474975586,
      "activations/layer19_attention_weight_min": -28.360403060913086,
      "activations/layer1_attention_weight_max": 17.316598892211914,
      "activations/layer1_attention_weight_min": -17.93701934814453,
      "activations/layer20_attention_weight_max": 43.170223236083984,
      "activations/layer20_attention_weight_min": -24.353723526000977,
      "activations/layer21_attention_weight_max": 40.47074508666992,
      "activations/layer21_attention_weight_min": -23.235305786132812,
      "activations/layer22_attention_weight_max": 63.32828903198242,
      "activations/layer22_attention_weight_min": -26.0399227142334,
      "activations/layer23_attention_weight_max": 36.067169189453125,
      "activations/layer23_attention_weight_min": -16.079404830932617,
      "activations/layer2_attention_weight_max": 29.946897506713867,
      "activations/layer2_attention_weight_min": -30.693798065185547,
      "activations/layer3_attention_weight_max": 86.2496109008789,
      "activations/layer3_attention_weight_min": -83.718994140625,
      "activations/layer4_attention_weight_max": 133.24696350097656,
      "activations/layer4_attention_weight_min": -122.10652923583984,
      "activations/layer5_attention_weight_max": 91.88668060302734,
      "activations/layer5_attention_weight_min": -86.72859954833984,
      "activations/layer6_attention_weight_max": 58.4241943359375,
      "activations/layer6_attention_weight_min": -57.005863189697266,
      "activations/layer7_attention_weight_max": 75.36170196533203,
      "activations/layer7_attention_weight_min": -76.9128646850586,
      "activations/layer8_attention_weight_max": 42.90462875366211,
      "activations/layer8_attention_weight_min": -39.1142463684082,
      "activations/layer9_attention_weight_max": 47.68764114379883,
      "activations/layer9_attention_weight_min": -36.76009750366211,
      "epoch": 3.79,
      "learning_rate": 0.00012684431818181817,
      "loss": 2.8719,
      "step": 65200
    },
    {
      "activations/layer0_attention_weight_max": 14.981003761291504,
      "activations/layer0_attention_weight_min": -11.419332504272461,
      "activations/layer10_attention_weight_max": 35.347511291503906,
      "activations/layer10_attention_weight_min": -32.04145050048828,
      "activations/layer11_attention_weight_max": 36.72745132446289,
      "activations/layer11_attention_weight_min": -28.571882247924805,
      "activations/layer12_attention_weight_max": 28.13048553466797,
      "activations/layer12_attention_weight_min": -26.579898834228516,
      "activations/layer13_attention_weight_max": 47.93281173706055,
      "activations/layer13_attention_weight_min": -34.01286315917969,
      "activations/layer14_attention_weight_max": 64.83479309082031,
      "activations/layer14_attention_weight_min": -48.25069046020508,
      "activations/layer15_attention_weight_max": 44.950660705566406,
      "activations/layer15_attention_weight_min": -30.15504264831543,
      "activations/layer16_attention_weight_max": 41.368064880371094,
      "activations/layer16_attention_weight_min": -30.36754035949707,
      "activations/layer17_attention_weight_max": 41.52425003051758,
      "activations/layer17_attention_weight_min": -26.431623458862305,
      "activations/layer18_attention_weight_max": 30.118160247802734,
      "activations/layer18_attention_weight_min": -19.284658432006836,
      "activations/layer19_attention_weight_max": 47.515865325927734,
      "activations/layer19_attention_weight_min": -31.648305892944336,
      "activations/layer1_attention_weight_max": 16.573904037475586,
      "activations/layer1_attention_weight_min": -16.540679931640625,
      "activations/layer20_attention_weight_max": 34.211483001708984,
      "activations/layer20_attention_weight_min": -23.492338180541992,
      "activations/layer21_attention_weight_max": 33.010536193847656,
      "activations/layer21_attention_weight_min": -22.195880889892578,
      "activations/layer22_attention_weight_max": 49.522377014160156,
      "activations/layer22_attention_weight_min": -27.299083709716797,
      "activations/layer23_attention_weight_max": 29.98921775817871,
      "activations/layer23_attention_weight_min": -17.57831573486328,
      "activations/layer2_attention_weight_max": 30.73615837097168,
      "activations/layer2_attention_weight_min": -31.067663192749023,
      "activations/layer3_attention_weight_max": 83.77698516845703,
      "activations/layer3_attention_weight_min": -85.00871276855469,
      "activations/layer4_attention_weight_max": 129.21783447265625,
      "activations/layer4_attention_weight_min": -124.48571014404297,
      "activations/layer5_attention_weight_max": 91.96044921875,
      "activations/layer5_attention_weight_min": -88.14604187011719,
      "activations/layer6_attention_weight_max": 57.81666946411133,
      "activations/layer6_attention_weight_min": -56.41835403442383,
      "activations/layer7_attention_weight_max": 78.00599670410156,
      "activations/layer7_attention_weight_min": -79.60016632080078,
      "activations/layer8_attention_weight_max": 38.92616653442383,
      "activations/layer8_attention_weight_min": -41.28225326538086,
      "activations/layer9_attention_weight_max": 41.81840133666992,
      "activations/layer9_attention_weight_min": -33.95977020263672,
      "epoch": 3.79,
      "learning_rate": 0.00012682537878787878,
      "loss": 2.8694,
      "step": 65250
    },
    {
      "activations/layer0_attention_weight_max": 16.070741653442383,
      "activations/layer0_attention_weight_min": -11.676284790039062,
      "activations/layer10_attention_weight_max": 36.86554718017578,
      "activations/layer10_attention_weight_min": -31.565946578979492,
      "activations/layer11_attention_weight_max": 32.50373458862305,
      "activations/layer11_attention_weight_min": -28.106260299682617,
      "activations/layer12_attention_weight_max": 33.20220947265625,
      "activations/layer12_attention_weight_min": -25.86916732788086,
      "activations/layer13_attention_weight_max": 47.41048049926758,
      "activations/layer13_attention_weight_min": -36.011573791503906,
      "activations/layer14_attention_weight_max": 63.0678596496582,
      "activations/layer14_attention_weight_min": -48.55543518066406,
      "activations/layer15_attention_weight_max": 41.33993911743164,
      "activations/layer15_attention_weight_min": -30.753767013549805,
      "activations/layer16_attention_weight_max": 39.293827056884766,
      "activations/layer16_attention_weight_min": -28.692535400390625,
      "activations/layer17_attention_weight_max": 38.91017150878906,
      "activations/layer17_attention_weight_min": -24.10209083557129,
      "activations/layer18_attention_weight_max": 29.031715393066406,
      "activations/layer18_attention_weight_min": -18.333932876586914,
      "activations/layer19_attention_weight_max": 39.45844650268555,
      "activations/layer19_attention_weight_min": -29.47980308532715,
      "activations/layer1_attention_weight_max": 17.970911026000977,
      "activations/layer1_attention_weight_min": -18.9908390045166,
      "activations/layer20_attention_weight_max": 32.61463165283203,
      "activations/layer20_attention_weight_min": -25.067537307739258,
      "activations/layer21_attention_weight_max": 30.951988220214844,
      "activations/layer21_attention_weight_min": -18.71219253540039,
      "activations/layer22_attention_weight_max": 46.4412727355957,
      "activations/layer22_attention_weight_min": -27.361318588256836,
      "activations/layer23_attention_weight_max": 28.703197479248047,
      "activations/layer23_attention_weight_min": -18.652830123901367,
      "activations/layer2_attention_weight_max": 30.67190170288086,
      "activations/layer2_attention_weight_min": -30.699600219726562,
      "activations/layer3_attention_weight_max": 84.58780670166016,
      "activations/layer3_attention_weight_min": -85.43125915527344,
      "activations/layer4_attention_weight_max": 131.51473999023438,
      "activations/layer4_attention_weight_min": -129.0151824951172,
      "activations/layer5_attention_weight_max": 92.24102783203125,
      "activations/layer5_attention_weight_min": -87.76071166992188,
      "activations/layer6_attention_weight_max": 57.73322296142578,
      "activations/layer6_attention_weight_min": -58.358428955078125,
      "activations/layer7_attention_weight_max": 78.08547973632812,
      "activations/layer7_attention_weight_min": -74.92388153076172,
      "activations/layer8_attention_weight_max": 39.36050796508789,
      "activations/layer8_attention_weight_min": -39.92647171020508,
      "activations/layer9_attention_weight_max": 38.7048454284668,
      "activations/layer9_attention_weight_min": -34.410133361816406,
      "epoch": 3.79,
      "learning_rate": 0.0001268064393939394,
      "loss": 2.8682,
      "step": 65300
    },
    {
      "activations/layer0_attention_weight_max": 15.34231185913086,
      "activations/layer0_attention_weight_min": -11.70706558227539,
      "activations/layer10_attention_weight_max": 41.11943054199219,
      "activations/layer10_attention_weight_min": -33.333988189697266,
      "activations/layer11_attention_weight_max": 38.38156509399414,
      "activations/layer11_attention_weight_min": -28.24762535095215,
      "activations/layer12_attention_weight_max": 33.36945343017578,
      "activations/layer12_attention_weight_min": -24.446857452392578,
      "activations/layer13_attention_weight_max": 61.38481140136719,
      "activations/layer13_attention_weight_min": -37.99890899658203,
      "activations/layer14_attention_weight_max": 89.72225952148438,
      "activations/layer14_attention_weight_min": -57.67823791503906,
      "activations/layer15_attention_weight_max": 54.855350494384766,
      "activations/layer15_attention_weight_min": -29.822437286376953,
      "activations/layer16_attention_weight_max": 45.817420959472656,
      "activations/layer16_attention_weight_min": -27.972917556762695,
      "activations/layer17_attention_weight_max": 49.676170349121094,
      "activations/layer17_attention_weight_min": -25.32017707824707,
      "activations/layer18_attention_weight_max": 36.26906204223633,
      "activations/layer18_attention_weight_min": -20.388097763061523,
      "activations/layer19_attention_weight_max": 50.155174255371094,
      "activations/layer19_attention_weight_min": -28.314977645874023,
      "activations/layer1_attention_weight_max": 17.425159454345703,
      "activations/layer1_attention_weight_min": -16.2330322265625,
      "activations/layer20_attention_weight_max": 36.17149353027344,
      "activations/layer20_attention_weight_min": -24.162216186523438,
      "activations/layer21_attention_weight_max": 44.320533752441406,
      "activations/layer21_attention_weight_min": -19.51653480529785,
      "activations/layer22_attention_weight_max": 58.62689971923828,
      "activations/layer22_attention_weight_min": -25.44936752319336,
      "activations/layer23_attention_weight_max": 37.27191925048828,
      "activations/layer23_attention_weight_min": -19.236217498779297,
      "activations/layer2_attention_weight_max": 31.370624542236328,
      "activations/layer2_attention_weight_min": -31.35900115966797,
      "activations/layer3_attention_weight_max": 86.98213958740234,
      "activations/layer3_attention_weight_min": -84.07379150390625,
      "activations/layer4_attention_weight_max": 130.25856018066406,
      "activations/layer4_attention_weight_min": -127.12672424316406,
      "activations/layer5_attention_weight_max": 98.57139587402344,
      "activations/layer5_attention_weight_min": -91.24249267578125,
      "activations/layer6_attention_weight_max": 57.58499526977539,
      "activations/layer6_attention_weight_min": -61.09526062011719,
      "activations/layer7_attention_weight_max": 78.67776489257812,
      "activations/layer7_attention_weight_min": -80.09107208251953,
      "activations/layer8_attention_weight_max": 41.77664566040039,
      "activations/layer8_attention_weight_min": -40.57886505126953,
      "activations/layer9_attention_weight_max": 44.22369384765625,
      "activations/layer9_attention_weight_min": -37.65327835083008,
      "epoch": 3.8,
      "learning_rate": 0.00012678749999999999,
      "loss": 2.867,
      "step": 65350
    },
    {
      "activations/layer0_attention_weight_max": 15.418261528015137,
      "activations/layer0_attention_weight_min": -11.771769523620605,
      "activations/layer10_attention_weight_max": 41.40742492675781,
      "activations/layer10_attention_weight_min": -33.9881591796875,
      "activations/layer11_attention_weight_max": 49.63672637939453,
      "activations/layer11_attention_weight_min": -34.33048629760742,
      "activations/layer12_attention_weight_max": 44.34589385986328,
      "activations/layer12_attention_weight_min": -31.495084762573242,
      "activations/layer13_attention_weight_max": 53.415260314941406,
      "activations/layer13_attention_weight_min": -37.654441833496094,
      "activations/layer14_attention_weight_max": 85.2619857788086,
      "activations/layer14_attention_weight_min": -58.59611892700195,
      "activations/layer15_attention_weight_max": 46.27163314819336,
      "activations/layer15_attention_weight_min": -29.598735809326172,
      "activations/layer16_attention_weight_max": 42.74003601074219,
      "activations/layer16_attention_weight_min": -29.0206241607666,
      "activations/layer17_attention_weight_max": 42.905677795410156,
      "activations/layer17_attention_weight_min": -23.76395606994629,
      "activations/layer18_attention_weight_max": 33.047447204589844,
      "activations/layer18_attention_weight_min": -19.22748374938965,
      "activations/layer19_attention_weight_max": 45.06594467163086,
      "activations/layer19_attention_weight_min": -28.669071197509766,
      "activations/layer1_attention_weight_max": 17.299631118774414,
      "activations/layer1_attention_weight_min": -16.86452293395996,
      "activations/layer20_attention_weight_max": 34.80065155029297,
      "activations/layer20_attention_weight_min": -21.870323181152344,
      "activations/layer21_attention_weight_max": 38.9326286315918,
      "activations/layer21_attention_weight_min": -20.325620651245117,
      "activations/layer22_attention_weight_max": 53.89387512207031,
      "activations/layer22_attention_weight_min": -30.001964569091797,
      "activations/layer23_attention_weight_max": 33.46512985229492,
      "activations/layer23_attention_weight_min": -19.58909034729004,
      "activations/layer2_attention_weight_max": 30.88372802734375,
      "activations/layer2_attention_weight_min": -31.01034164428711,
      "activations/layer3_attention_weight_max": 83.79524993896484,
      "activations/layer3_attention_weight_min": -89.36504364013672,
      "activations/layer4_attention_weight_max": 128.03965759277344,
      "activations/layer4_attention_weight_min": -131.92970275878906,
      "activations/layer5_attention_weight_max": 91.28451538085938,
      "activations/layer5_attention_weight_min": -92.62635803222656,
      "activations/layer6_attention_weight_max": 56.99742889404297,
      "activations/layer6_attention_weight_min": -55.41301727294922,
      "activations/layer7_attention_weight_max": 89.13472747802734,
      "activations/layer7_attention_weight_min": -78.85415649414062,
      "activations/layer8_attention_weight_max": 41.41240310668945,
      "activations/layer8_attention_weight_min": -40.977962493896484,
      "activations/layer9_attention_weight_max": 56.82963943481445,
      "activations/layer9_attention_weight_min": -41.85030746459961,
      "epoch": 3.8,
      "learning_rate": 0.0001267685606060606,
      "loss": 2.8585,
      "step": 65400
    },
    {
      "activations/layer0_attention_weight_max": 15.663578987121582,
      "activations/layer0_attention_weight_min": -10.738170623779297,
      "activations/layer10_attention_weight_max": 38.446022033691406,
      "activations/layer10_attention_weight_min": -32.98351287841797,
      "activations/layer11_attention_weight_max": 37.25879669189453,
      "activations/layer11_attention_weight_min": -28.81473159790039,
      "activations/layer12_attention_weight_max": 43.7159423828125,
      "activations/layer12_attention_weight_min": -28.722227096557617,
      "activations/layer13_attention_weight_max": 57.947845458984375,
      "activations/layer13_attention_weight_min": -40.43042755126953,
      "activations/layer14_attention_weight_max": 89.44474792480469,
      "activations/layer14_attention_weight_min": -59.996437072753906,
      "activations/layer15_attention_weight_max": 46.410091400146484,
      "activations/layer15_attention_weight_min": -30.580978393554688,
      "activations/layer16_attention_weight_max": 45.37044906616211,
      "activations/layer16_attention_weight_min": -29.363525390625,
      "activations/layer17_attention_weight_max": 45.99427032470703,
      "activations/layer17_attention_weight_min": -26.117036819458008,
      "activations/layer18_attention_weight_max": 34.42052459716797,
      "activations/layer18_attention_weight_min": -19.127965927124023,
      "activations/layer19_attention_weight_max": 48.99293899536133,
      "activations/layer19_attention_weight_min": -32.09989929199219,
      "activations/layer1_attention_weight_max": 16.116472244262695,
      "activations/layer1_attention_weight_min": -17.0432071685791,
      "activations/layer20_attention_weight_max": 34.587860107421875,
      "activations/layer20_attention_weight_min": -27.38246726989746,
      "activations/layer21_attention_weight_max": 32.7605094909668,
      "activations/layer21_attention_weight_min": -20.607830047607422,
      "activations/layer22_attention_weight_max": 55.61503219604492,
      "activations/layer22_attention_weight_min": -25.95627212524414,
      "activations/layer23_attention_weight_max": 34.80464172363281,
      "activations/layer23_attention_weight_min": -17.382503509521484,
      "activations/layer2_attention_weight_max": 32.327938079833984,
      "activations/layer2_attention_weight_min": -32.35494613647461,
      "activations/layer3_attention_weight_max": 86.27218627929688,
      "activations/layer3_attention_weight_min": -85.7955093383789,
      "activations/layer4_attention_weight_max": 132.6160125732422,
      "activations/layer4_attention_weight_min": -131.12942504882812,
      "activations/layer5_attention_weight_max": 95.9737548828125,
      "activations/layer5_attention_weight_min": -88.56539154052734,
      "activations/layer6_attention_weight_max": 58.84062957763672,
      "activations/layer6_attention_weight_min": -56.36130142211914,
      "activations/layer7_attention_weight_max": 79.0514907836914,
      "activations/layer7_attention_weight_min": -76.69336700439453,
      "activations/layer8_attention_weight_max": 42.904022216796875,
      "activations/layer8_attention_weight_min": -41.18759536743164,
      "activations/layer9_attention_weight_max": 39.32870864868164,
      "activations/layer9_attention_weight_min": -34.76487350463867,
      "epoch": 3.8,
      "learning_rate": 0.00012674962121212122,
      "loss": 2.8775,
      "step": 65450
    },
    {
      "activations/layer0_attention_weight_max": 16.03240966796875,
      "activations/layer0_attention_weight_min": -11.4369478225708,
      "activations/layer10_attention_weight_max": 55.652706146240234,
      "activations/layer10_attention_weight_min": -41.08127212524414,
      "activations/layer11_attention_weight_max": 48.55650329589844,
      "activations/layer11_attention_weight_min": -34.88313293457031,
      "activations/layer12_attention_weight_max": 99.74908447265625,
      "activations/layer12_attention_weight_min": -39.9539794921875,
      "activations/layer13_attention_weight_max": 97.89936828613281,
      "activations/layer13_attention_weight_min": -52.34314727783203,
      "activations/layer14_attention_weight_max": 78.86993408203125,
      "activations/layer14_attention_weight_min": -48.8358039855957,
      "activations/layer15_attention_weight_max": 65.2258071899414,
      "activations/layer15_attention_weight_min": -35.916255950927734,
      "activations/layer16_attention_weight_max": 42.025943756103516,
      "activations/layer16_attention_weight_min": -31.173709869384766,
      "activations/layer17_attention_weight_max": 44.96624755859375,
      "activations/layer17_attention_weight_min": -27.743661880493164,
      "activations/layer18_attention_weight_max": 34.87883758544922,
      "activations/layer18_attention_weight_min": -21.613481521606445,
      "activations/layer19_attention_weight_max": 46.79413986206055,
      "activations/layer19_attention_weight_min": -30.64646339416504,
      "activations/layer1_attention_weight_max": 17.231311798095703,
      "activations/layer1_attention_weight_min": -14.242814064025879,
      "activations/layer20_attention_weight_max": 35.36247253417969,
      "activations/layer20_attention_weight_min": -23.33892059326172,
      "activations/layer21_attention_weight_max": 36.03877258300781,
      "activations/layer21_attention_weight_min": -21.28986167907715,
      "activations/layer22_attention_weight_max": 58.815006256103516,
      "activations/layer22_attention_weight_min": -28.576780319213867,
      "activations/layer23_attention_weight_max": 32.46037673950195,
      "activations/layer23_attention_weight_min": -19.254066467285156,
      "activations/layer2_attention_weight_max": 31.682729721069336,
      "activations/layer2_attention_weight_min": -30.241819381713867,
      "activations/layer3_attention_weight_max": 82.30847930908203,
      "activations/layer3_attention_weight_min": -89.1585922241211,
      "activations/layer4_attention_weight_max": 128.938720703125,
      "activations/layer4_attention_weight_min": -130.09075927734375,
      "activations/layer5_attention_weight_max": 98.05024719238281,
      "activations/layer5_attention_weight_min": -88.42439270019531,
      "activations/layer6_attention_weight_max": 64.47007751464844,
      "activations/layer6_attention_weight_min": -54.58209991455078,
      "activations/layer7_attention_weight_max": 99.0284423828125,
      "activations/layer7_attention_weight_min": -76.604248046875,
      "activations/layer8_attention_weight_max": 52.47649002075195,
      "activations/layer8_attention_weight_min": -45.93879699707031,
      "activations/layer9_attention_weight_max": 66.89004516601562,
      "activations/layer9_attention_weight_min": -43.22658920288086,
      "epoch": 3.81,
      "learning_rate": 0.0001267306818181818,
      "loss": 2.8573,
      "step": 65500
    },
    {
      "activations/layer0_attention_weight_max": 15.977505683898926,
      "activations/layer0_attention_weight_min": -10.349364280700684,
      "activations/layer10_attention_weight_max": 40.5706672668457,
      "activations/layer10_attention_weight_min": -31.411460876464844,
      "activations/layer11_attention_weight_max": 37.97462844848633,
      "activations/layer11_attention_weight_min": -31.042545318603516,
      "activations/layer12_attention_weight_max": 36.222068786621094,
      "activations/layer12_attention_weight_min": -25.663753509521484,
      "activations/layer13_attention_weight_max": 63.05678176879883,
      "activations/layer13_attention_weight_min": -47.039390563964844,
      "activations/layer14_attention_weight_max": 79.17093658447266,
      "activations/layer14_attention_weight_min": -59.0467414855957,
      "activations/layer15_attention_weight_max": 44.482086181640625,
      "activations/layer15_attention_weight_min": -30.88724708557129,
      "activations/layer16_attention_weight_max": 38.47871780395508,
      "activations/layer16_attention_weight_min": -29.621173858642578,
      "activations/layer17_attention_weight_max": 44.18995666503906,
      "activations/layer17_attention_weight_min": -26.57378578186035,
      "activations/layer18_attention_weight_max": 28.649898529052734,
      "activations/layer18_attention_weight_min": -19.53068733215332,
      "activations/layer19_attention_weight_max": 48.574371337890625,
      "activations/layer19_attention_weight_min": -28.784347534179688,
      "activations/layer1_attention_weight_max": 17.645835876464844,
      "activations/layer1_attention_weight_min": -14.270633697509766,
      "activations/layer20_attention_weight_max": 34.05122375488281,
      "activations/layer20_attention_weight_min": -22.819568634033203,
      "activations/layer21_attention_weight_max": 30.63347625732422,
      "activations/layer21_attention_weight_min": -21.23041534423828,
      "activations/layer22_attention_weight_max": 47.79170608520508,
      "activations/layer22_attention_weight_min": -26.79499626159668,
      "activations/layer23_attention_weight_max": 29.393226623535156,
      "activations/layer23_attention_weight_min": -20.31915283203125,
      "activations/layer2_attention_weight_max": 30.547801971435547,
      "activations/layer2_attention_weight_min": -30.542701721191406,
      "activations/layer3_attention_weight_max": 85.67259216308594,
      "activations/layer3_attention_weight_min": -88.69053649902344,
      "activations/layer4_attention_weight_max": 125.2637939453125,
      "activations/layer4_attention_weight_min": -133.65728759765625,
      "activations/layer5_attention_weight_max": 94.30500793457031,
      "activations/layer5_attention_weight_min": -91.83807373046875,
      "activations/layer6_attention_weight_max": 56.18006896972656,
      "activations/layer6_attention_weight_min": -58.46366882324219,
      "activations/layer7_attention_weight_max": 75.98052215576172,
      "activations/layer7_attention_weight_min": -79.04386138916016,
      "activations/layer8_attention_weight_max": 41.0568962097168,
      "activations/layer8_attention_weight_min": -38.61927032470703,
      "activations/layer9_attention_weight_max": 43.6812858581543,
      "activations/layer9_attention_weight_min": -39.062625885009766,
      "epoch": 3.81,
      "learning_rate": 0.00012671174242424242,
      "loss": 2.8653,
      "step": 65550
    },
    {
      "activations/layer0_attention_weight_max": 14.705901145935059,
      "activations/layer0_attention_weight_min": -11.200201988220215,
      "activations/layer10_attention_weight_max": 36.84047317504883,
      "activations/layer10_attention_weight_min": -30.20754051208496,
      "activations/layer11_attention_weight_max": 38.0037727355957,
      "activations/layer11_attention_weight_min": -32.242862701416016,
      "activations/layer12_attention_weight_max": 36.22211837768555,
      "activations/layer12_attention_weight_min": -28.922365188598633,
      "activations/layer13_attention_weight_max": 46.126869201660156,
      "activations/layer13_attention_weight_min": -34.697967529296875,
      "activations/layer14_attention_weight_max": 70.5351333618164,
      "activations/layer14_attention_weight_min": -51.54505920410156,
      "activations/layer15_attention_weight_max": 45.65799331665039,
      "activations/layer15_attention_weight_min": -30.236173629760742,
      "activations/layer16_attention_weight_max": 45.376319885253906,
      "activations/layer16_attention_weight_min": -33.28512954711914,
      "activations/layer17_attention_weight_max": 43.2680549621582,
      "activations/layer17_attention_weight_min": -27.213197708129883,
      "activations/layer18_attention_weight_max": 32.36018371582031,
      "activations/layer18_attention_weight_min": -21.58063316345215,
      "activations/layer19_attention_weight_max": 45.44537353515625,
      "activations/layer19_attention_weight_min": -30.709014892578125,
      "activations/layer1_attention_weight_max": 16.843538284301758,
      "activations/layer1_attention_weight_min": -15.313943862915039,
      "activations/layer20_attention_weight_max": 34.433998107910156,
      "activations/layer20_attention_weight_min": -23.499588012695312,
      "activations/layer21_attention_weight_max": 32.75528335571289,
      "activations/layer21_attention_weight_min": -19.758615493774414,
      "activations/layer22_attention_weight_max": 53.65264892578125,
      "activations/layer22_attention_weight_min": -30.655181884765625,
      "activations/layer23_attention_weight_max": 30.452329635620117,
      "activations/layer23_attention_weight_min": -19.28470230102539,
      "activations/layer2_attention_weight_max": 31.997404098510742,
      "activations/layer2_attention_weight_min": -30.998136520385742,
      "activations/layer3_attention_weight_max": 90.19976043701172,
      "activations/layer3_attention_weight_min": -84.7608871459961,
      "activations/layer4_attention_weight_max": 136.47862243652344,
      "activations/layer4_attention_weight_min": -128.55181884765625,
      "activations/layer5_attention_weight_max": 95.72322082519531,
      "activations/layer5_attention_weight_min": -85.13238525390625,
      "activations/layer6_attention_weight_max": 56.857452392578125,
      "activations/layer6_attention_weight_min": -55.16586685180664,
      "activations/layer7_attention_weight_max": 77.56714630126953,
      "activations/layer7_attention_weight_min": -74.55146026611328,
      "activations/layer8_attention_weight_max": 40.0084342956543,
      "activations/layer8_attention_weight_min": -39.712501525878906,
      "activations/layer9_attention_weight_max": 39.17508316040039,
      "activations/layer9_attention_weight_min": -36.583126068115234,
      "epoch": 3.81,
      "learning_rate": 0.00012669280303030303,
      "loss": 2.8701,
      "step": 65600
    },
    {
      "activations/layer0_attention_weight_max": 15.7198486328125,
      "activations/layer0_attention_weight_min": -11.13270092010498,
      "activations/layer10_attention_weight_max": 37.3660888671875,
      "activations/layer10_attention_weight_min": -31.731456756591797,
      "activations/layer11_attention_weight_max": 39.18386459350586,
      "activations/layer11_attention_weight_min": -31.314929962158203,
      "activations/layer12_attention_weight_max": 38.90871810913086,
      "activations/layer12_attention_weight_min": -28.690628051757812,
      "activations/layer13_attention_weight_max": 65.97029113769531,
      "activations/layer13_attention_weight_min": -45.3759880065918,
      "activations/layer14_attention_weight_max": 79.23194885253906,
      "activations/layer14_attention_weight_min": -56.093910217285156,
      "activations/layer15_attention_weight_max": 49.79790115356445,
      "activations/layer15_attention_weight_min": -36.02481460571289,
      "activations/layer16_attention_weight_max": 41.358665466308594,
      "activations/layer16_attention_weight_min": -31.018217086791992,
      "activations/layer17_attention_weight_max": 39.532737731933594,
      "activations/layer17_attention_weight_min": -28.022615432739258,
      "activations/layer18_attention_weight_max": 30.34304428100586,
      "activations/layer18_attention_weight_min": -20.35885238647461,
      "activations/layer19_attention_weight_max": 43.32242202758789,
      "activations/layer19_attention_weight_min": -29.73746109008789,
      "activations/layer1_attention_weight_max": 18.314029693603516,
      "activations/layer1_attention_weight_min": -14.754393577575684,
      "activations/layer20_attention_weight_max": 33.444026947021484,
      "activations/layer20_attention_weight_min": -22.20330047607422,
      "activations/layer21_attention_weight_max": 31.143234252929688,
      "activations/layer21_attention_weight_min": -19.799768447875977,
      "activations/layer22_attention_weight_max": 48.115570068359375,
      "activations/layer22_attention_weight_min": -29.18821907043457,
      "activations/layer23_attention_weight_max": 30.644561767578125,
      "activations/layer23_attention_weight_min": -16.89341926574707,
      "activations/layer2_attention_weight_max": 31.43902587890625,
      "activations/layer2_attention_weight_min": -30.921903610229492,
      "activations/layer3_attention_weight_max": 86.08731079101562,
      "activations/layer3_attention_weight_min": -86.37970733642578,
      "activations/layer4_attention_weight_max": 137.09454345703125,
      "activations/layer4_attention_weight_min": -127.4279556274414,
      "activations/layer5_attention_weight_max": 93.78377532958984,
      "activations/layer5_attention_weight_min": -85.74470520019531,
      "activations/layer6_attention_weight_max": 57.52813720703125,
      "activations/layer6_attention_weight_min": -56.052406311035156,
      "activations/layer7_attention_weight_max": 87.29070281982422,
      "activations/layer7_attention_weight_min": -78.42218780517578,
      "activations/layer8_attention_weight_max": 42.29878234863281,
      "activations/layer8_attention_weight_min": -39.22822570800781,
      "activations/layer9_attention_weight_max": 53.78725051879883,
      "activations/layer9_attention_weight_min": -43.29209518432617,
      "epoch": 3.81,
      "learning_rate": 0.00012667386363636362,
      "loss": 2.8568,
      "step": 65650
    },
    {
      "activations/layer0_attention_weight_max": 14.27195930480957,
      "activations/layer0_attention_weight_min": -12.117182731628418,
      "activations/layer10_attention_weight_max": 37.153480529785156,
      "activations/layer10_attention_weight_min": -32.030094146728516,
      "activations/layer11_attention_weight_max": 33.650875091552734,
      "activations/layer11_attention_weight_min": -29.400068283081055,
      "activations/layer12_attention_weight_max": 31.503618240356445,
      "activations/layer12_attention_weight_min": -24.778806686401367,
      "activations/layer13_attention_weight_max": 53.824764251708984,
      "activations/layer13_attention_weight_min": -35.7344970703125,
      "activations/layer14_attention_weight_max": 68.70262145996094,
      "activations/layer14_attention_weight_min": -49.52169418334961,
      "activations/layer15_attention_weight_max": 46.21959686279297,
      "activations/layer15_attention_weight_min": -30.72134017944336,
      "activations/layer16_attention_weight_max": 44.591590881347656,
      "activations/layer16_attention_weight_min": -28.56889533996582,
      "activations/layer17_attention_weight_max": 47.67466735839844,
      "activations/layer17_attention_weight_min": -24.76668930053711,
      "activations/layer18_attention_weight_max": 31.198726654052734,
      "activations/layer18_attention_weight_min": -18.99801254272461,
      "activations/layer19_attention_weight_max": 46.207305908203125,
      "activations/layer19_attention_weight_min": -27.402036666870117,
      "activations/layer1_attention_weight_max": 17.60114288330078,
      "activations/layer1_attention_weight_min": -14.358055114746094,
      "activations/layer20_attention_weight_max": 39.67167663574219,
      "activations/layer20_attention_weight_min": -24.755863189697266,
      "activations/layer21_attention_weight_max": 34.839996337890625,
      "activations/layer21_attention_weight_min": -19.880342483520508,
      "activations/layer22_attention_weight_max": 55.67759704589844,
      "activations/layer22_attention_weight_min": -26.231510162353516,
      "activations/layer23_attention_weight_max": 33.85346221923828,
      "activations/layer23_attention_weight_min": -18.062488555908203,
      "activations/layer2_attention_weight_max": 30.62131118774414,
      "activations/layer2_attention_weight_min": -29.753448486328125,
      "activations/layer3_attention_weight_max": 87.33878326416016,
      "activations/layer3_attention_weight_min": -86.36174011230469,
      "activations/layer4_attention_weight_max": 126.73162078857422,
      "activations/layer4_attention_weight_min": -130.23887634277344,
      "activations/layer5_attention_weight_max": 90.15739440917969,
      "activations/layer5_attention_weight_min": -87.85411071777344,
      "activations/layer6_attention_weight_max": 55.72163391113281,
      "activations/layer6_attention_weight_min": -56.449462890625,
      "activations/layer7_attention_weight_max": 74.30459594726562,
      "activations/layer7_attention_weight_min": -75.86310577392578,
      "activations/layer8_attention_weight_max": 39.37604522705078,
      "activations/layer8_attention_weight_min": -39.13591766357422,
      "activations/layer9_attention_weight_max": 40.54222869873047,
      "activations/layer9_attention_weight_min": -33.56282043457031,
      "epoch": 3.82,
      "learning_rate": 0.00012665492424242424,
      "loss": 2.8662,
      "step": 65700
    },
    {
      "activations/layer0_attention_weight_max": 14.41820240020752,
      "activations/layer0_attention_weight_min": -11.318290710449219,
      "activations/layer10_attention_weight_max": 41.5254020690918,
      "activations/layer10_attention_weight_min": -32.06522750854492,
      "activations/layer11_attention_weight_max": 33.67264175415039,
      "activations/layer11_attention_weight_min": -31.294424057006836,
      "activations/layer12_attention_weight_max": 25.599048614501953,
      "activations/layer12_attention_weight_min": -25.911840438842773,
      "activations/layer13_attention_weight_max": 44.34347152709961,
      "activations/layer13_attention_weight_min": -34.32084274291992,
      "activations/layer14_attention_weight_max": 59.674644470214844,
      "activations/layer14_attention_weight_min": -46.36463928222656,
      "activations/layer15_attention_weight_max": 44.3964729309082,
      "activations/layer15_attention_weight_min": -28.335508346557617,
      "activations/layer16_attention_weight_max": 43.57065200805664,
      "activations/layer16_attention_weight_min": -28.01409912109375,
      "activations/layer17_attention_weight_max": 42.686248779296875,
      "activations/layer17_attention_weight_min": -25.08403968811035,
      "activations/layer18_attention_weight_max": 28.64291000366211,
      "activations/layer18_attention_weight_min": -20.487258911132812,
      "activations/layer19_attention_weight_max": 41.11336898803711,
      "activations/layer19_attention_weight_min": -27.741540908813477,
      "activations/layer1_attention_weight_max": 16.844635009765625,
      "activations/layer1_attention_weight_min": -16.337257385253906,
      "activations/layer20_attention_weight_max": 32.47156524658203,
      "activations/layer20_attention_weight_min": -23.246164321899414,
      "activations/layer21_attention_weight_max": 36.331146240234375,
      "activations/layer21_attention_weight_min": -22.717504501342773,
      "activations/layer22_attention_weight_max": 50.46790313720703,
      "activations/layer22_attention_weight_min": -26.39798927307129,
      "activations/layer23_attention_weight_max": 29.095584869384766,
      "activations/layer23_attention_weight_min": -21.201942443847656,
      "activations/layer2_attention_weight_max": 30.407094955444336,
      "activations/layer2_attention_weight_min": -29.7200984954834,
      "activations/layer3_attention_weight_max": 85.47137451171875,
      "activations/layer3_attention_weight_min": -89.31747436523438,
      "activations/layer4_attention_weight_max": 129.4622344970703,
      "activations/layer4_attention_weight_min": -132.2310333251953,
      "activations/layer5_attention_weight_max": 93.62782287597656,
      "activations/layer5_attention_weight_min": -86.60872650146484,
      "activations/layer6_attention_weight_max": 59.60199737548828,
      "activations/layer6_attention_weight_min": -56.821434020996094,
      "activations/layer7_attention_weight_max": 77.32923889160156,
      "activations/layer7_attention_weight_min": -75.31237030029297,
      "activations/layer8_attention_weight_max": 39.330963134765625,
      "activations/layer8_attention_weight_min": -36.55496597290039,
      "activations/layer9_attention_weight_max": 38.20447540283203,
      "activations/layer9_attention_weight_min": -33.3724250793457,
      "epoch": 3.82,
      "learning_rate": 0.00012663598484848485,
      "loss": 2.8761,
      "step": 65750
    },
    {
      "activations/layer0_attention_weight_max": 16.944995880126953,
      "activations/layer0_attention_weight_min": -11.577458381652832,
      "activations/layer10_attention_weight_max": 38.77692794799805,
      "activations/layer10_attention_weight_min": -31.03879165649414,
      "activations/layer11_attention_weight_max": 34.44316864013672,
      "activations/layer11_attention_weight_min": -27.65439224243164,
      "activations/layer12_attention_weight_max": 40.3641471862793,
      "activations/layer12_attention_weight_min": -30.79279327392578,
      "activations/layer13_attention_weight_max": 53.489383697509766,
      "activations/layer13_attention_weight_min": -38.64134216308594,
      "activations/layer14_attention_weight_max": 74.52210235595703,
      "activations/layer14_attention_weight_min": -52.9793701171875,
      "activations/layer15_attention_weight_max": 43.676902770996094,
      "activations/layer15_attention_weight_min": -31.919082641601562,
      "activations/layer16_attention_weight_max": 38.923404693603516,
      "activations/layer16_attention_weight_min": -28.051097869873047,
      "activations/layer17_attention_weight_max": 41.35947036743164,
      "activations/layer17_attention_weight_min": -26.326311111450195,
      "activations/layer18_attention_weight_max": 31.237417221069336,
      "activations/layer18_attention_weight_min": -19.505508422851562,
      "activations/layer19_attention_weight_max": 41.89450454711914,
      "activations/layer19_attention_weight_min": -30.90199851989746,
      "activations/layer1_attention_weight_max": 17.810413360595703,
      "activations/layer1_attention_weight_min": -16.058618545532227,
      "activations/layer20_attention_weight_max": 29.650779724121094,
      "activations/layer20_attention_weight_min": -22.248485565185547,
      "activations/layer21_attention_weight_max": 31.741302490234375,
      "activations/layer21_attention_weight_min": -20.028396606445312,
      "activations/layer22_attention_weight_max": 47.96807098388672,
      "activations/layer22_attention_weight_min": -29.069292068481445,
      "activations/layer23_attention_weight_max": 28.483367919921875,
      "activations/layer23_attention_weight_min": -20.763097763061523,
      "activations/layer2_attention_weight_max": 32.20538330078125,
      "activations/layer2_attention_weight_min": -34.38063430786133,
      "activations/layer3_attention_weight_max": 88.59210968017578,
      "activations/layer3_attention_weight_min": -94.00499725341797,
      "activations/layer4_attention_weight_max": 136.89283752441406,
      "activations/layer4_attention_weight_min": -137.63621520996094,
      "activations/layer5_attention_weight_max": 97.83867645263672,
      "activations/layer5_attention_weight_min": -95.11226654052734,
      "activations/layer6_attention_weight_max": 65.8870849609375,
      "activations/layer6_attention_weight_min": -61.27879333496094,
      "activations/layer7_attention_weight_max": 78.23088073730469,
      "activations/layer7_attention_weight_min": -81.42196655273438,
      "activations/layer8_attention_weight_max": 40.28416442871094,
      "activations/layer8_attention_weight_min": -37.34227752685547,
      "activations/layer9_attention_weight_max": 39.39765167236328,
      "activations/layer9_attention_weight_min": -37.51387023925781,
      "epoch": 3.82,
      "learning_rate": 0.00012661704545454544,
      "loss": 2.8647,
      "step": 65800
    },
    {
      "activations/layer0_attention_weight_max": 16.215513229370117,
      "activations/layer0_attention_weight_min": -11.810736656188965,
      "activations/layer10_attention_weight_max": 37.922882080078125,
      "activations/layer10_attention_weight_min": -31.435256958007812,
      "activations/layer11_attention_weight_max": 34.022396087646484,
      "activations/layer11_attention_weight_min": -27.592390060424805,
      "activations/layer12_attention_weight_max": 28.994401931762695,
      "activations/layer12_attention_weight_min": -28.34272003173828,
      "activations/layer13_attention_weight_max": 54.12466812133789,
      "activations/layer13_attention_weight_min": -34.768707275390625,
      "activations/layer14_attention_weight_max": 70.6158676147461,
      "activations/layer14_attention_weight_min": -47.8753547668457,
      "activations/layer15_attention_weight_max": 50.01937484741211,
      "activations/layer15_attention_weight_min": -28.8741455078125,
      "activations/layer16_attention_weight_max": 50.64255142211914,
      "activations/layer16_attention_weight_min": -29.815540313720703,
      "activations/layer17_attention_weight_max": 45.30107498168945,
      "activations/layer17_attention_weight_min": -25.260896682739258,
      "activations/layer18_attention_weight_max": 36.73961639404297,
      "activations/layer18_attention_weight_min": -20.177024841308594,
      "activations/layer19_attention_weight_max": 54.08049392700195,
      "activations/layer19_attention_weight_min": -29.702625274658203,
      "activations/layer1_attention_weight_max": 16.441986083984375,
      "activations/layer1_attention_weight_min": -17.834888458251953,
      "activations/layer20_attention_weight_max": 41.080928802490234,
      "activations/layer20_attention_weight_min": -23.635343551635742,
      "activations/layer21_attention_weight_max": 42.54493713378906,
      "activations/layer21_attention_weight_min": -20.474441528320312,
      "activations/layer22_attention_weight_max": 58.560882568359375,
      "activations/layer22_attention_weight_min": -26.88599967956543,
      "activations/layer23_attention_weight_max": 35.069759368896484,
      "activations/layer23_attention_weight_min": -16.277286529541016,
      "activations/layer2_attention_weight_max": 32.3349494934082,
      "activations/layer2_attention_weight_min": -33.25138854980469,
      "activations/layer3_attention_weight_max": 88.4663314819336,
      "activations/layer3_attention_weight_min": -86.31358337402344,
      "activations/layer4_attention_weight_max": 134.1368408203125,
      "activations/layer4_attention_weight_min": -130.70266723632812,
      "activations/layer5_attention_weight_max": 97.27046966552734,
      "activations/layer5_attention_weight_min": -90.65648651123047,
      "activations/layer6_attention_weight_max": 58.02958297729492,
      "activations/layer6_attention_weight_min": -60.03510665893555,
      "activations/layer7_attention_weight_max": 76.56989288330078,
      "activations/layer7_attention_weight_min": -79.5694808959961,
      "activations/layer8_attention_weight_max": 41.07308578491211,
      "activations/layer8_attention_weight_min": -40.73856735229492,
      "activations/layer9_attention_weight_max": 40.537864685058594,
      "activations/layer9_attention_weight_min": -38.44365310668945,
      "epoch": 3.83,
      "learning_rate": 0.00012659810606060605,
      "loss": 2.8779,
      "step": 65850
    },
    {
      "activations/layer0_attention_weight_max": 15.960652351379395,
      "activations/layer0_attention_weight_min": -11.918292999267578,
      "activations/layer10_attention_weight_max": 42.30813217163086,
      "activations/layer10_attention_weight_min": -32.75468826293945,
      "activations/layer11_attention_weight_max": 35.97230529785156,
      "activations/layer11_attention_weight_min": -30.09860610961914,
      "activations/layer12_attention_weight_max": 34.63153076171875,
      "activations/layer12_attention_weight_min": -32.0455436706543,
      "activations/layer13_attention_weight_max": 48.659236907958984,
      "activations/layer13_attention_weight_min": -33.03297424316406,
      "activations/layer14_attention_weight_max": 70.60880279541016,
      "activations/layer14_attention_weight_min": -50.45762252807617,
      "activations/layer15_attention_weight_max": 42.19622039794922,
      "activations/layer15_attention_weight_min": -29.15127182006836,
      "activations/layer16_attention_weight_max": 40.771060943603516,
      "activations/layer16_attention_weight_min": -29.729215621948242,
      "activations/layer17_attention_weight_max": 44.61587905883789,
      "activations/layer17_attention_weight_min": -26.886240005493164,
      "activations/layer18_attention_weight_max": 30.46233558654785,
      "activations/layer18_attention_weight_min": -20.616252899169922,
      "activations/layer19_attention_weight_max": 42.20564651489258,
      "activations/layer19_attention_weight_min": -27.990161895751953,
      "activations/layer1_attention_weight_max": 17.651018142700195,
      "activations/layer1_attention_weight_min": -16.096664428710938,
      "activations/layer20_attention_weight_max": 34.79640197753906,
      "activations/layer20_attention_weight_min": -24.1344051361084,
      "activations/layer21_attention_weight_max": 36.554325103759766,
      "activations/layer21_attention_weight_min": -20.005573272705078,
      "activations/layer22_attention_weight_max": 50.343257904052734,
      "activations/layer22_attention_weight_min": -27.56550407409668,
      "activations/layer23_attention_weight_max": 32.81817626953125,
      "activations/layer23_attention_weight_min": -18.777502059936523,
      "activations/layer2_attention_weight_max": 31.03365707397461,
      "activations/layer2_attention_weight_min": -30.83576202392578,
      "activations/layer3_attention_weight_max": 84.32766723632812,
      "activations/layer3_attention_weight_min": -87.85803985595703,
      "activations/layer4_attention_weight_max": 129.31594848632812,
      "activations/layer4_attention_weight_min": -127.96826171875,
      "activations/layer5_attention_weight_max": 91.80528259277344,
      "activations/layer5_attention_weight_min": -82.65737915039062,
      "activations/layer6_attention_weight_max": 55.88160705566406,
      "activations/layer6_attention_weight_min": -54.351505279541016,
      "activations/layer7_attention_weight_max": 82.76301574707031,
      "activations/layer7_attention_weight_min": -75.3377685546875,
      "activations/layer8_attention_weight_max": 42.067996978759766,
      "activations/layer8_attention_weight_min": -36.876461029052734,
      "activations/layer9_attention_weight_max": 42.37519454956055,
      "activations/layer9_attention_weight_min": -33.8104362487793,
      "epoch": 3.83,
      "learning_rate": 0.00012657916666666664,
      "loss": 2.856,
      "step": 65900
    },
    {
      "activations/layer0_attention_weight_max": 14.185917854309082,
      "activations/layer0_attention_weight_min": -11.587986946105957,
      "activations/layer10_attention_weight_max": 38.037864685058594,
      "activations/layer10_attention_weight_min": -30.29315757751465,
      "activations/layer11_attention_weight_max": 34.235660552978516,
      "activations/layer11_attention_weight_min": -29.47513198852539,
      "activations/layer12_attention_weight_max": 40.6246337890625,
      "activations/layer12_attention_weight_min": -29.891942977905273,
      "activations/layer13_attention_weight_max": 51.016990661621094,
      "activations/layer13_attention_weight_min": -33.59031677246094,
      "activations/layer14_attention_weight_max": 81.36666107177734,
      "activations/layer14_attention_weight_min": -55.468162536621094,
      "activations/layer15_attention_weight_max": 45.99440383911133,
      "activations/layer15_attention_weight_min": -28.3646183013916,
      "activations/layer16_attention_weight_max": 46.45035171508789,
      "activations/layer16_attention_weight_min": -30.095888137817383,
      "activations/layer17_attention_weight_max": 45.55373001098633,
      "activations/layer17_attention_weight_min": -24.390798568725586,
      "activations/layer18_attention_weight_max": 37.468570709228516,
      "activations/layer18_attention_weight_min": -19.016908645629883,
      "activations/layer19_attention_weight_max": 51.12042236328125,
      "activations/layer19_attention_weight_min": -29.577024459838867,
      "activations/layer1_attention_weight_max": 16.61365509033203,
      "activations/layer1_attention_weight_min": -18.66200828552246,
      "activations/layer20_attention_weight_max": 35.293636322021484,
      "activations/layer20_attention_weight_min": -23.961759567260742,
      "activations/layer21_attention_weight_max": 36.93566131591797,
      "activations/layer21_attention_weight_min": -20.2404842376709,
      "activations/layer22_attention_weight_max": 52.98433303833008,
      "activations/layer22_attention_weight_min": -29.1883602142334,
      "activations/layer23_attention_weight_max": 31.374486923217773,
      "activations/layer23_attention_weight_min": -20.43734359741211,
      "activations/layer2_attention_weight_max": 32.21310806274414,
      "activations/layer2_attention_weight_min": -30.64132308959961,
      "activations/layer3_attention_weight_max": 88.4669189453125,
      "activations/layer3_attention_weight_min": -90.37564086914062,
      "activations/layer4_attention_weight_max": 133.97283935546875,
      "activations/layer4_attention_weight_min": -130.53594970703125,
      "activations/layer5_attention_weight_max": 99.56884765625,
      "activations/layer5_attention_weight_min": -90.40618896484375,
      "activations/layer6_attention_weight_max": 58.6805305480957,
      "activations/layer6_attention_weight_min": -58.309974670410156,
      "activations/layer7_attention_weight_max": 77.54219055175781,
      "activations/layer7_attention_weight_min": -81.2334976196289,
      "activations/layer8_attention_weight_max": 42.313262939453125,
      "activations/layer8_attention_weight_min": -40.862449645996094,
      "activations/layer9_attention_weight_max": 39.76973342895508,
      "activations/layer9_attention_weight_min": -41.27690124511719,
      "epoch": 3.83,
      "learning_rate": 0.00012656022727272726,
      "loss": 2.8583,
      "step": 65950
    },
    {
      "activations/layer0_attention_weight_max": 16.24929428100586,
      "activations/layer0_attention_weight_min": -11.952719688415527,
      "activations/layer10_attention_weight_max": 35.64752197265625,
      "activations/layer10_attention_weight_min": -30.728313446044922,
      "activations/layer11_attention_weight_max": 37.667449951171875,
      "activations/layer11_attention_weight_min": -31.545074462890625,
      "activations/layer12_attention_weight_max": 33.18483352661133,
      "activations/layer12_attention_weight_min": -26.540998458862305,
      "activations/layer13_attention_weight_max": 43.26881790161133,
      "activations/layer13_attention_weight_min": -30.98753547668457,
      "activations/layer14_attention_weight_max": 72.15094757080078,
      "activations/layer14_attention_weight_min": -53.752628326416016,
      "activations/layer15_attention_weight_max": 43.51383972167969,
      "activations/layer15_attention_weight_min": -30.3398380279541,
      "activations/layer16_attention_weight_max": 43.225494384765625,
      "activations/layer16_attention_weight_min": -32.61007308959961,
      "activations/layer17_attention_weight_max": 39.05181121826172,
      "activations/layer17_attention_weight_min": -25.76107406616211,
      "activations/layer18_attention_weight_max": 30.852642059326172,
      "activations/layer18_attention_weight_min": -19.27273178100586,
      "activations/layer19_attention_weight_max": 45.329490661621094,
      "activations/layer19_attention_weight_min": -29.8798828125,
      "activations/layer1_attention_weight_max": 16.197389602661133,
      "activations/layer1_attention_weight_min": -15.461566925048828,
      "activations/layer20_attention_weight_max": 35.90506362915039,
      "activations/layer20_attention_weight_min": -24.753915786743164,
      "activations/layer21_attention_weight_max": 30.130279541015625,
      "activations/layer21_attention_weight_min": -18.151037216186523,
      "activations/layer22_attention_weight_max": 50.52854919433594,
      "activations/layer22_attention_weight_min": -24.99795150756836,
      "activations/layer23_attention_weight_max": 31.332075119018555,
      "activations/layer23_attention_weight_min": -17.21550750732422,
      "activations/layer2_attention_weight_max": 32.08330535888672,
      "activations/layer2_attention_weight_min": -31.348064422607422,
      "activations/layer3_attention_weight_max": 89.2022705078125,
      "activations/layer3_attention_weight_min": -89.64897155761719,
      "activations/layer4_attention_weight_max": 138.49200439453125,
      "activations/layer4_attention_weight_min": -135.87657165527344,
      "activations/layer5_attention_weight_max": 97.08320617675781,
      "activations/layer5_attention_weight_min": -87.44527435302734,
      "activations/layer6_attention_weight_max": 60.000797271728516,
      "activations/layer6_attention_weight_min": -57.4056396484375,
      "activations/layer7_attention_weight_max": 84.68488311767578,
      "activations/layer7_attention_weight_min": -76.79167938232422,
      "activations/layer8_attention_weight_max": 40.503662109375,
      "activations/layer8_attention_weight_min": -38.70125198364258,
      "activations/layer9_attention_weight_max": 47.63249206542969,
      "activations/layer9_attention_weight_min": -34.05602264404297,
      "epoch": 3.83,
      "learning_rate": 0.00012654128787878787,
      "loss": 2.8677,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_loss": 2.8203125,
      "eval_runtime": 8.554,
      "eval_samples_per_second": 501.989,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_openwebtext_loss": 2.8203125,
      "eval_openwebtext_ppl": 16.782094257242417,
      "eval_openwebtext_runtime": 8.554,
      "eval_openwebtext_samples_per_second": 501.989,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_wikitext_loss": 3.04296875,
      "eval_wikitext_ppl": 20.967397891114594,
      "eval_wikitext_runtime": 1.9558,
      "eval_wikitext_samples_per_second": 233.15,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_lambada_loss": 2.796875,
      "eval_lambada_ppl": 16.39333746248769,
      "eval_lambada_runtime": 9.8578,
      "eval_lambada_samples_per_second": 493.923,
      "step": 66000
    },
    {
      "activations/layer0_attention_weight_max": 15.827260971069336,
      "activations/layer0_attention_weight_min": -11.297250747680664,
      "activations/layer10_attention_weight_max": 36.955039978027344,
      "activations/layer10_attention_weight_min": -31.588321685791016,
      "activations/layer11_attention_weight_max": 35.22755813598633,
      "activations/layer11_attention_weight_min": -31.63014793395996,
      "activations/layer12_attention_weight_max": 32.37743377685547,
      "activations/layer12_attention_weight_min": -24.111955642700195,
      "activations/layer13_attention_weight_max": 51.54936218261719,
      "activations/layer13_attention_weight_min": -36.204280853271484,
      "activations/layer14_attention_weight_max": 68.32145690917969,
      "activations/layer14_attention_weight_min": -46.84159469604492,
      "activations/layer15_attention_weight_max": 42.066158294677734,
      "activations/layer15_attention_weight_min": -31.3535213470459,
      "activations/layer16_attention_weight_max": 40.8636474609375,
      "activations/layer16_attention_weight_min": -31.752578735351562,
      "activations/layer17_attention_weight_max": 40.968631744384766,
      "activations/layer17_attention_weight_min": -25.691570281982422,
      "activations/layer18_attention_weight_max": 32.19085693359375,
      "activations/layer18_attention_weight_min": -21.60847282409668,
      "activations/layer19_attention_weight_max": 43.01029968261719,
      "activations/layer19_attention_weight_min": -31.56699562072754,
      "activations/layer1_attention_weight_max": 17.31548500061035,
      "activations/layer1_attention_weight_min": -14.44837760925293,
      "activations/layer20_attention_weight_max": 32.48857879638672,
      "activations/layer20_attention_weight_min": -22.66946029663086,
      "activations/layer21_attention_weight_max": 30.285354614257812,
      "activations/layer21_attention_weight_min": -21.479909896850586,
      "activations/layer22_attention_weight_max": 47.94240951538086,
      "activations/layer22_attention_weight_min": -29.16118049621582,
      "activations/layer23_attention_weight_max": 30.703460693359375,
      "activations/layer23_attention_weight_min": -22.80150604248047,
      "activations/layer2_attention_weight_max": 31.384990692138672,
      "activations/layer2_attention_weight_min": -32.364112854003906,
      "activations/layer3_attention_weight_max": 86.16681671142578,
      "activations/layer3_attention_weight_min": -87.73309326171875,
      "activations/layer4_attention_weight_max": 133.05389404296875,
      "activations/layer4_attention_weight_min": -127.1203842163086,
      "activations/layer5_attention_weight_max": 89.92930603027344,
      "activations/layer5_attention_weight_min": -87.51525115966797,
      "activations/layer6_attention_weight_max": 58.033531188964844,
      "activations/layer6_attention_weight_min": -55.387516021728516,
      "activations/layer7_attention_weight_max": 74.2325668334961,
      "activations/layer7_attention_weight_min": -76.52009582519531,
      "activations/layer8_attention_weight_max": 39.09293746948242,
      "activations/layer8_attention_weight_min": -40.32655334472656,
      "activations/layer9_attention_weight_max": 39.9233512878418,
      "activations/layer9_attention_weight_min": -34.04389953613281,
      "epoch": 3.84,
      "learning_rate": 0.00012652234848484846,
      "loss": 2.8659,
      "step": 66050
    },
    {
      "activations/layer0_attention_weight_max": 15.729316711425781,
      "activations/layer0_attention_weight_min": -12.012253761291504,
      "activations/layer10_attention_weight_max": 37.191673278808594,
      "activations/layer10_attention_weight_min": -31.13344955444336,
      "activations/layer11_attention_weight_max": 34.64460754394531,
      "activations/layer11_attention_weight_min": -27.60140037536621,
      "activations/layer12_attention_weight_max": 30.282453536987305,
      "activations/layer12_attention_weight_min": -27.805028915405273,
      "activations/layer13_attention_weight_max": 45.47410583496094,
      "activations/layer13_attention_weight_min": -33.386104583740234,
      "activations/layer14_attention_weight_max": 65.24176788330078,
      "activations/layer14_attention_weight_min": -49.250980377197266,
      "activations/layer15_attention_weight_max": 40.682281494140625,
      "activations/layer15_attention_weight_min": -30.3302001953125,
      "activations/layer16_attention_weight_max": 40.2633171081543,
      "activations/layer16_attention_weight_min": -30.0225887298584,
      "activations/layer17_attention_weight_max": 39.84461212158203,
      "activations/layer17_attention_weight_min": -28.311817169189453,
      "activations/layer18_attention_weight_max": 32.479549407958984,
      "activations/layer18_attention_weight_min": -19.792945861816406,
      "activations/layer19_attention_weight_max": 44.05551528930664,
      "activations/layer19_attention_weight_min": -29.57224464416504,
      "activations/layer1_attention_weight_max": 17.50832748413086,
      "activations/layer1_attention_weight_min": -14.330927848815918,
      "activations/layer20_attention_weight_max": 35.21369171142578,
      "activations/layer20_attention_weight_min": -23.520397186279297,
      "activations/layer21_attention_weight_max": 28.479154586791992,
      "activations/layer21_attention_weight_min": -19.108495712280273,
      "activations/layer22_attention_weight_max": 48.251461029052734,
      "activations/layer22_attention_weight_min": -27.63730239868164,
      "activations/layer23_attention_weight_max": 29.898839950561523,
      "activations/layer23_attention_weight_min": -17.37814712524414,
      "activations/layer2_attention_weight_max": 33.33262634277344,
      "activations/layer2_attention_weight_min": -31.24488639831543,
      "activations/layer3_attention_weight_max": 88.81982421875,
      "activations/layer3_attention_weight_min": -89.59589385986328,
      "activations/layer4_attention_weight_max": 137.884521484375,
      "activations/layer4_attention_weight_min": -131.5232391357422,
      "activations/layer5_attention_weight_max": 95.17835998535156,
      "activations/layer5_attention_weight_min": -86.5329818725586,
      "activations/layer6_attention_weight_max": 61.48056411743164,
      "activations/layer6_attention_weight_min": -54.73086929321289,
      "activations/layer7_attention_weight_max": 78.9281005859375,
      "activations/layer7_attention_weight_min": -76.32010650634766,
      "activations/layer8_attention_weight_max": 40.93239974975586,
      "activations/layer8_attention_weight_min": -38.20820999145508,
      "activations/layer9_attention_weight_max": 40.74183654785156,
      "activations/layer9_attention_weight_min": -39.95827102661133,
      "epoch": 3.84,
      "learning_rate": 0.00012650340909090908,
      "loss": 2.885,
      "step": 66100
    },
    {
      "activations/layer0_attention_weight_max": 14.49567985534668,
      "activations/layer0_attention_weight_min": -12.107526779174805,
      "activations/layer10_attention_weight_max": 34.06341552734375,
      "activations/layer10_attention_weight_min": -29.186199188232422,
      "activations/layer11_attention_weight_max": 35.52693176269531,
      "activations/layer11_attention_weight_min": -28.75063133239746,
      "activations/layer12_attention_weight_max": 28.360652923583984,
      "activations/layer12_attention_weight_min": -26.429292678833008,
      "activations/layer13_attention_weight_max": 55.51805877685547,
      "activations/layer13_attention_weight_min": -41.80064010620117,
      "activations/layer14_attention_weight_max": 70.8427505493164,
      "activations/layer14_attention_weight_min": -49.82920455932617,
      "activations/layer15_attention_weight_max": 45.80141830444336,
      "activations/layer15_attention_weight_min": -28.922100067138672,
      "activations/layer16_attention_weight_max": 42.28240203857422,
      "activations/layer16_attention_weight_min": -29.456377029418945,
      "activations/layer17_attention_weight_max": 43.490177154541016,
      "activations/layer17_attention_weight_min": -25.356380462646484,
      "activations/layer18_attention_weight_max": 29.29542350769043,
      "activations/layer18_attention_weight_min": -18.61600112915039,
      "activations/layer19_attention_weight_max": 47.024383544921875,
      "activations/layer19_attention_weight_min": -28.211292266845703,
      "activations/layer1_attention_weight_max": 17.19068717956543,
      "activations/layer1_attention_weight_min": -16.48014259338379,
      "activations/layer20_attention_weight_max": 33.139469146728516,
      "activations/layer20_attention_weight_min": -22.66629409790039,
      "activations/layer21_attention_weight_max": 36.03755569458008,
      "activations/layer21_attention_weight_min": -20.999755859375,
      "activations/layer22_attention_weight_max": 48.789207458496094,
      "activations/layer22_attention_weight_min": -30.773176193237305,
      "activations/layer23_attention_weight_max": 31.604347229003906,
      "activations/layer23_attention_weight_min": -19.0133056640625,
      "activations/layer2_attention_weight_max": 32.5191650390625,
      "activations/layer2_attention_weight_min": -30.70651626586914,
      "activations/layer3_attention_weight_max": 82.14830780029297,
      "activations/layer3_attention_weight_min": -87.49407196044922,
      "activations/layer4_attention_weight_max": 127.63382720947266,
      "activations/layer4_attention_weight_min": -123.1454086303711,
      "activations/layer5_attention_weight_max": 89.32476806640625,
      "activations/layer5_attention_weight_min": -79.83944702148438,
      "activations/layer6_attention_weight_max": 55.29551315307617,
      "activations/layer6_attention_weight_min": -57.85518264770508,
      "activations/layer7_attention_weight_max": 80.05101776123047,
      "activations/layer7_attention_weight_min": -77.55968475341797,
      "activations/layer8_attention_weight_max": 38.58475112915039,
      "activations/layer8_attention_weight_min": -36.2304573059082,
      "activations/layer9_attention_weight_max": 39.6868896484375,
      "activations/layer9_attention_weight_min": -34.44016647338867,
      "epoch": 3.84,
      "learning_rate": 0.0001264844696969697,
      "loss": 2.8623,
      "step": 66150
    },
    {
      "activations/layer0_attention_weight_max": 16.265657424926758,
      "activations/layer0_attention_weight_min": -11.37951374053955,
      "activations/layer10_attention_weight_max": 36.557037353515625,
      "activations/layer10_attention_weight_min": -31.971603393554688,
      "activations/layer11_attention_weight_max": 33.48485565185547,
      "activations/layer11_attention_weight_min": -30.437232971191406,
      "activations/layer12_attention_weight_max": 30.837116241455078,
      "activations/layer12_attention_weight_min": -25.53795623779297,
      "activations/layer13_attention_weight_max": 46.66081237792969,
      "activations/layer13_attention_weight_min": -32.50862121582031,
      "activations/layer14_attention_weight_max": 71.98834991455078,
      "activations/layer14_attention_weight_min": -52.899192810058594,
      "activations/layer15_attention_weight_max": 38.62730407714844,
      "activations/layer15_attention_weight_min": -28.445310592651367,
      "activations/layer16_attention_weight_max": 37.932559967041016,
      "activations/layer16_attention_weight_min": -32.12192153930664,
      "activations/layer17_attention_weight_max": 43.938995361328125,
      "activations/layer17_attention_weight_min": -26.46316146850586,
      "activations/layer18_attention_weight_max": 29.263587951660156,
      "activations/layer18_attention_weight_min": -19.750713348388672,
      "activations/layer19_attention_weight_max": 42.07029342651367,
      "activations/layer19_attention_weight_min": -29.322961807250977,
      "activations/layer1_attention_weight_max": 18.2725830078125,
      "activations/layer1_attention_weight_min": -14.750767707824707,
      "activations/layer20_attention_weight_max": 34.09028244018555,
      "activations/layer20_attention_weight_min": -22.886327743530273,
      "activations/layer21_attention_weight_max": 32.70652389526367,
      "activations/layer21_attention_weight_min": -18.76601219177246,
      "activations/layer22_attention_weight_max": 46.86990737915039,
      "activations/layer22_attention_weight_min": -27.73116111755371,
      "activations/layer23_attention_weight_max": 30.124671936035156,
      "activations/layer23_attention_weight_min": -18.545604705810547,
      "activations/layer2_attention_weight_max": 31.701108932495117,
      "activations/layer2_attention_weight_min": -30.286270141601562,
      "activations/layer3_attention_weight_max": 88.56605529785156,
      "activations/layer3_attention_weight_min": -87.13562774658203,
      "activations/layer4_attention_weight_max": 135.33163452148438,
      "activations/layer4_attention_weight_min": -133.6211700439453,
      "activations/layer5_attention_weight_max": 98.60643768310547,
      "activations/layer5_attention_weight_min": -92.25047302246094,
      "activations/layer6_attention_weight_max": 60.291175842285156,
      "activations/layer6_attention_weight_min": -59.502906799316406,
      "activations/layer7_attention_weight_max": 81.5265121459961,
      "activations/layer7_attention_weight_min": -85.28141784667969,
      "activations/layer8_attention_weight_max": 43.021018981933594,
      "activations/layer8_attention_weight_min": -42.25585174560547,
      "activations/layer9_attention_weight_max": 38.89212417602539,
      "activations/layer9_attention_weight_min": -38.09560775756836,
      "epoch": 3.85,
      "learning_rate": 0.00012646553030303028,
      "loss": 2.8745,
      "step": 66200
    },
    {
      "activations/layer0_attention_weight_max": 16.544763565063477,
      "activations/layer0_attention_weight_min": -11.075441360473633,
      "activations/layer10_attention_weight_max": 38.141422271728516,
      "activations/layer10_attention_weight_min": -29.74994468688965,
      "activations/layer11_attention_weight_max": 34.35651397705078,
      "activations/layer11_attention_weight_min": -31.88800048828125,
      "activations/layer12_attention_weight_max": 32.19675827026367,
      "activations/layer12_attention_weight_min": -30.364320755004883,
      "activations/layer13_attention_weight_max": 51.27109146118164,
      "activations/layer13_attention_weight_min": -34.716796875,
      "activations/layer14_attention_weight_max": 72.10139465332031,
      "activations/layer14_attention_weight_min": -53.5816764831543,
      "activations/layer15_attention_weight_max": 46.49536895751953,
      "activations/layer15_attention_weight_min": -28.77535057067871,
      "activations/layer16_attention_weight_max": 40.152854919433594,
      "activations/layer16_attention_weight_min": -28.200191497802734,
      "activations/layer17_attention_weight_max": 42.96588897705078,
      "activations/layer17_attention_weight_min": -25.96095848083496,
      "activations/layer18_attention_weight_max": 32.690608978271484,
      "activations/layer18_attention_weight_min": -19.571096420288086,
      "activations/layer19_attention_weight_max": 52.92300033569336,
      "activations/layer19_attention_weight_min": -28.950183868408203,
      "activations/layer1_attention_weight_max": 16.79706382751465,
      "activations/layer1_attention_weight_min": -17.45160484313965,
      "activations/layer20_attention_weight_max": 35.470394134521484,
      "activations/layer20_attention_weight_min": -25.25844955444336,
      "activations/layer21_attention_weight_max": 34.121788024902344,
      "activations/layer21_attention_weight_min": -21.44260597229004,
      "activations/layer22_attention_weight_max": 50.209617614746094,
      "activations/layer22_attention_weight_min": -27.23333740234375,
      "activations/layer23_attention_weight_max": 29.735036849975586,
      "activations/layer23_attention_weight_min": -17.51453971862793,
      "activations/layer2_attention_weight_max": 30.27216339111328,
      "activations/layer2_attention_weight_min": -29.327861785888672,
      "activations/layer3_attention_weight_max": 81.95826721191406,
      "activations/layer3_attention_weight_min": -88.88544464111328,
      "activations/layer4_attention_weight_max": 133.19154357910156,
      "activations/layer4_attention_weight_min": -132.69992065429688,
      "activations/layer5_attention_weight_max": 91.00567626953125,
      "activations/layer5_attention_weight_min": -84.78556060791016,
      "activations/layer6_attention_weight_max": 62.67328643798828,
      "activations/layer6_attention_weight_min": -58.32942581176758,
      "activations/layer7_attention_weight_max": 85.5584487915039,
      "activations/layer7_attention_weight_min": -73.7958984375,
      "activations/layer8_attention_weight_max": 39.89208221435547,
      "activations/layer8_attention_weight_min": -38.41179275512695,
      "activations/layer9_attention_weight_max": 42.97003173828125,
      "activations/layer9_attention_weight_min": -36.15837097167969,
      "epoch": 3.85,
      "learning_rate": 0.0001264465909090909,
      "loss": 2.8612,
      "step": 66250
    },
    {
      "activations/layer0_attention_weight_max": 15.379586219787598,
      "activations/layer0_attention_weight_min": -11.85446548461914,
      "activations/layer10_attention_weight_max": 36.152198791503906,
      "activations/layer10_attention_weight_min": -31.319229125976562,
      "activations/layer11_attention_weight_max": 36.44288635253906,
      "activations/layer11_attention_weight_min": -29.20494842529297,
      "activations/layer12_attention_weight_max": 44.207759857177734,
      "activations/layer12_attention_weight_min": -28.07982063293457,
      "activations/layer13_attention_weight_max": 47.055843353271484,
      "activations/layer13_attention_weight_min": -33.7894287109375,
      "activations/layer14_attention_weight_max": 86.78791046142578,
      "activations/layer14_attention_weight_min": -59.650177001953125,
      "activations/layer15_attention_weight_max": 43.18907165527344,
      "activations/layer15_attention_weight_min": -31.156185150146484,
      "activations/layer16_attention_weight_max": 44.76941680908203,
      "activations/layer16_attention_weight_min": -29.434404373168945,
      "activations/layer17_attention_weight_max": 42.1902961730957,
      "activations/layer17_attention_weight_min": -26.612424850463867,
      "activations/layer18_attention_weight_max": 34.46675491333008,
      "activations/layer18_attention_weight_min": -19.534847259521484,
      "activations/layer19_attention_weight_max": 47.62508773803711,
      "activations/layer19_attention_weight_min": -29.064804077148438,
      "activations/layer1_attention_weight_max": 17.94876480102539,
      "activations/layer1_attention_weight_min": -15.895493507385254,
      "activations/layer20_attention_weight_max": 35.535762786865234,
      "activations/layer20_attention_weight_min": -22.40883445739746,
      "activations/layer21_attention_weight_max": 36.769920349121094,
      "activations/layer21_attention_weight_min": -22.191511154174805,
      "activations/layer22_attention_weight_max": 51.59455108642578,
      "activations/layer22_attention_weight_min": -27.37769317626953,
      "activations/layer23_attention_weight_max": 35.6414794921875,
      "activations/layer23_attention_weight_min": -18.514265060424805,
      "activations/layer2_attention_weight_max": 32.431922912597656,
      "activations/layer2_attention_weight_min": -31.155067443847656,
      "activations/layer3_attention_weight_max": 88.77892303466797,
      "activations/layer3_attention_weight_min": -87.37487030029297,
      "activations/layer4_attention_weight_max": 134.7107696533203,
      "activations/layer4_attention_weight_min": -132.37913513183594,
      "activations/layer5_attention_weight_max": 96.81608581542969,
      "activations/layer5_attention_weight_min": -89.50839233398438,
      "activations/layer6_attention_weight_max": 59.90867233276367,
      "activations/layer6_attention_weight_min": -56.458091735839844,
      "activations/layer7_attention_weight_max": 77.34550476074219,
      "activations/layer7_attention_weight_min": -77.07466888427734,
      "activations/layer8_attention_weight_max": 41.53487014770508,
      "activations/layer8_attention_weight_min": -40.418399810791016,
      "activations/layer9_attention_weight_max": 41.657894134521484,
      "activations/layer9_attention_weight_min": -32.3033332824707,
      "epoch": 3.85,
      "learning_rate": 0.0001264276515151515,
      "loss": 2.8782,
      "step": 66300
    },
    {
      "activations/layer0_attention_weight_max": 16.283344268798828,
      "activations/layer0_attention_weight_min": -11.59398365020752,
      "activations/layer10_attention_weight_max": 38.31711959838867,
      "activations/layer10_attention_weight_min": -30.768020629882812,
      "activations/layer11_attention_weight_max": 40.21935272216797,
      "activations/layer11_attention_weight_min": -29.985713958740234,
      "activations/layer12_attention_weight_max": 61.222557067871094,
      "activations/layer12_attention_weight_min": -28.98893928527832,
      "activations/layer13_attention_weight_max": 58.41062927246094,
      "activations/layer13_attention_weight_min": -34.331512451171875,
      "activations/layer14_attention_weight_max": 80.10090637207031,
      "activations/layer14_attention_weight_min": -50.27216339111328,
      "activations/layer15_attention_weight_max": 45.813385009765625,
      "activations/layer15_attention_weight_min": -29.879390716552734,
      "activations/layer16_attention_weight_max": 41.87174606323242,
      "activations/layer16_attention_weight_min": -30.131969451904297,
      "activations/layer17_attention_weight_max": 38.04484176635742,
      "activations/layer17_attention_weight_min": -25.931596755981445,
      "activations/layer18_attention_weight_max": 30.26953125,
      "activations/layer18_attention_weight_min": -19.295265197753906,
      "activations/layer19_attention_weight_max": 46.570003509521484,
      "activations/layer19_attention_weight_min": -29.957019805908203,
      "activations/layer1_attention_weight_max": 17.356605529785156,
      "activations/layer1_attention_weight_min": -15.034496307373047,
      "activations/layer20_attention_weight_max": 37.35110092163086,
      "activations/layer20_attention_weight_min": -24.14295196533203,
      "activations/layer21_attention_weight_max": 29.836015701293945,
      "activations/layer21_attention_weight_min": -21.538816452026367,
      "activations/layer22_attention_weight_max": 45.30393600463867,
      "activations/layer22_attention_weight_min": -29.134517669677734,
      "activations/layer23_attention_weight_max": 26.498266220092773,
      "activations/layer23_attention_weight_min": -18.712371826171875,
      "activations/layer2_attention_weight_max": 31.360923767089844,
      "activations/layer2_attention_weight_min": -30.606016159057617,
      "activations/layer3_attention_weight_max": 83.64816284179688,
      "activations/layer3_attention_weight_min": -86.2495346069336,
      "activations/layer4_attention_weight_max": 130.42918395996094,
      "activations/layer4_attention_weight_min": -126.4449691772461,
      "activations/layer5_attention_weight_max": 92.42269134521484,
      "activations/layer5_attention_weight_min": -89.12511444091797,
      "activations/layer6_attention_weight_max": 57.88134765625,
      "activations/layer6_attention_weight_min": -55.95500564575195,
      "activations/layer7_attention_weight_max": 89.09623718261719,
      "activations/layer7_attention_weight_min": -79.97539520263672,
      "activations/layer8_attention_weight_max": 42.88676452636719,
      "activations/layer8_attention_weight_min": -37.91228485107422,
      "activations/layer9_attention_weight_max": 43.2776985168457,
      "activations/layer9_attention_weight_min": -38.501564025878906,
      "epoch": 3.86,
      "learning_rate": 0.0001264087121212121,
      "loss": 2.8805,
      "step": 66350
    },
    {
      "activations/layer0_attention_weight_max": 14.490344047546387,
      "activations/layer0_attention_weight_min": -11.603160858154297,
      "activations/layer10_attention_weight_max": 35.40802001953125,
      "activations/layer10_attention_weight_min": -32.403419494628906,
      "activations/layer11_attention_weight_max": 32.379241943359375,
      "activations/layer11_attention_weight_min": -29.035741806030273,
      "activations/layer12_attention_weight_max": 31.022249221801758,
      "activations/layer12_attention_weight_min": -28.739351272583008,
      "activations/layer13_attention_weight_max": 50.22748947143555,
      "activations/layer13_attention_weight_min": -36.76800537109375,
      "activations/layer14_attention_weight_max": 69.2691421508789,
      "activations/layer14_attention_weight_min": -52.272430419921875,
      "activations/layer15_attention_weight_max": 39.374996185302734,
      "activations/layer15_attention_weight_min": -28.417308807373047,
      "activations/layer16_attention_weight_max": 41.883079528808594,
      "activations/layer16_attention_weight_min": -29.451248168945312,
      "activations/layer17_attention_weight_max": 44.413455963134766,
      "activations/layer17_attention_weight_min": -27.234956741333008,
      "activations/layer18_attention_weight_max": 31.42686653137207,
      "activations/layer18_attention_weight_min": -21.18161964416504,
      "activations/layer19_attention_weight_max": 49.3205451965332,
      "activations/layer19_attention_weight_min": -28.69632339477539,
      "activations/layer1_attention_weight_max": 18.00119400024414,
      "activations/layer1_attention_weight_min": -14.934015274047852,
      "activations/layer20_attention_weight_max": 32.88319778442383,
      "activations/layer20_attention_weight_min": -23.679479598999023,
      "activations/layer21_attention_weight_max": 34.15416717529297,
      "activations/layer21_attention_weight_min": -19.95325469970703,
      "activations/layer22_attention_weight_max": 50.498409271240234,
      "activations/layer22_attention_weight_min": -27.489078521728516,
      "activations/layer23_attention_weight_max": 32.266212463378906,
      "activations/layer23_attention_weight_min": -18.642688751220703,
      "activations/layer2_attention_weight_max": 32.71544647216797,
      "activations/layer2_attention_weight_min": -31.414165496826172,
      "activations/layer3_attention_weight_max": 87.02832794189453,
      "activations/layer3_attention_weight_min": -85.77010345458984,
      "activations/layer4_attention_weight_max": 133.08058166503906,
      "activations/layer4_attention_weight_min": -127.16617584228516,
      "activations/layer5_attention_weight_max": 92.86812591552734,
      "activations/layer5_attention_weight_min": -84.71531677246094,
      "activations/layer6_attention_weight_max": 61.753543853759766,
      "activations/layer6_attention_weight_min": -54.853477478027344,
      "activations/layer7_attention_weight_max": 80.42171478271484,
      "activations/layer7_attention_weight_min": -77.60104370117188,
      "activations/layer8_attention_weight_max": 40.85890579223633,
      "activations/layer8_attention_weight_min": -36.60476303100586,
      "activations/layer9_attention_weight_max": 38.21468734741211,
      "activations/layer9_attention_weight_min": -33.393985748291016,
      "epoch": 3.86,
      "learning_rate": 0.0001263897727272727,
      "loss": 2.8774,
      "step": 66400
    },
    {
      "activations/layer0_attention_weight_max": 15.831513404846191,
      "activations/layer0_attention_weight_min": -11.800603866577148,
      "activations/layer10_attention_weight_max": 37.44832992553711,
      "activations/layer10_attention_weight_min": -31.040842056274414,
      "activations/layer11_attention_weight_max": 32.21234130859375,
      "activations/layer11_attention_weight_min": -28.591064453125,
      "activations/layer12_attention_weight_max": 27.833065032958984,
      "activations/layer12_attention_weight_min": -27.86009407043457,
      "activations/layer13_attention_weight_max": 64.80907440185547,
      "activations/layer13_attention_weight_min": -48.5047721862793,
      "activations/layer14_attention_weight_max": 66.30001068115234,
      "activations/layer14_attention_weight_min": -48.356388092041016,
      "activations/layer15_attention_weight_max": 60.31649398803711,
      "activations/layer15_attention_weight_min": -34.24849319458008,
      "activations/layer16_attention_weight_max": 39.26608657836914,
      "activations/layer16_attention_weight_min": -29.9776554107666,
      "activations/layer17_attention_weight_max": 43.17837905883789,
      "activations/layer17_attention_weight_min": -25.39204978942871,
      "activations/layer18_attention_weight_max": 30.394638061523438,
      "activations/layer18_attention_weight_min": -21.123703002929688,
      "activations/layer19_attention_weight_max": 44.9126091003418,
      "activations/layer19_attention_weight_min": -30.094572067260742,
      "activations/layer1_attention_weight_max": 17.066713333129883,
      "activations/layer1_attention_weight_min": -15.16526985168457,
      "activations/layer20_attention_weight_max": 32.396812438964844,
      "activations/layer20_attention_weight_min": -22.33724021911621,
      "activations/layer21_attention_weight_max": 28.066856384277344,
      "activations/layer21_attention_weight_min": -20.025714874267578,
      "activations/layer22_attention_weight_max": 53.17538070678711,
      "activations/layer22_attention_weight_min": -28.809432983398438,
      "activations/layer23_attention_weight_max": 28.660655975341797,
      "activations/layer23_attention_weight_min": -18.72968292236328,
      "activations/layer2_attention_weight_max": 31.79085922241211,
      "activations/layer2_attention_weight_min": -30.989377975463867,
      "activations/layer3_attention_weight_max": 88.51791381835938,
      "activations/layer3_attention_weight_min": -87.40105438232422,
      "activations/layer4_attention_weight_max": 136.14657592773438,
      "activations/layer4_attention_weight_min": -127.8019027709961,
      "activations/layer5_attention_weight_max": 97.3833236694336,
      "activations/layer5_attention_weight_min": -88.25119018554688,
      "activations/layer6_attention_weight_max": 60.97211456298828,
      "activations/layer6_attention_weight_min": -57.97529983520508,
      "activations/layer7_attention_weight_max": 77.52640533447266,
      "activations/layer7_attention_weight_min": -80.10381317138672,
      "activations/layer8_attention_weight_max": 38.92244338989258,
      "activations/layer8_attention_weight_min": -38.4255256652832,
      "activations/layer9_attention_weight_max": 37.52817916870117,
      "activations/layer9_attention_weight_min": -40.65293502807617,
      "epoch": 3.86,
      "learning_rate": 0.00012637083333333333,
      "loss": 2.8905,
      "step": 66450
    },
    {
      "activations/layer0_attention_weight_max": 14.825155258178711,
      "activations/layer0_attention_weight_min": -10.393259048461914,
      "activations/layer10_attention_weight_max": 48.9765739440918,
      "activations/layer10_attention_weight_min": -34.90953826904297,
      "activations/layer11_attention_weight_max": 39.524574279785156,
      "activations/layer11_attention_weight_min": -27.091693878173828,
      "activations/layer12_attention_weight_max": 29.810504913330078,
      "activations/layer12_attention_weight_min": -30.221708297729492,
      "activations/layer13_attention_weight_max": 59.11224365234375,
      "activations/layer13_attention_weight_min": -40.5814208984375,
      "activations/layer14_attention_weight_max": 81.38447570800781,
      "activations/layer14_attention_weight_min": -51.04745101928711,
      "activations/layer15_attention_weight_max": 59.895164489746094,
      "activations/layer15_attention_weight_min": -33.33305740356445,
      "activations/layer16_attention_weight_max": 44.98492431640625,
      "activations/layer16_attention_weight_min": -29.74851417541504,
      "activations/layer17_attention_weight_max": 48.88597106933594,
      "activations/layer17_attention_weight_min": -24.348663330078125,
      "activations/layer18_attention_weight_max": 35.768531799316406,
      "activations/layer18_attention_weight_min": -21.313005447387695,
      "activations/layer19_attention_weight_max": 42.97434616088867,
      "activations/layer19_attention_weight_min": -29.871671676635742,
      "activations/layer1_attention_weight_max": 16.99138641357422,
      "activations/layer1_attention_weight_min": -16.53307342529297,
      "activations/layer20_attention_weight_max": 36.78856658935547,
      "activations/layer20_attention_weight_min": -21.864383697509766,
      "activations/layer21_attention_weight_max": 36.038490295410156,
      "activations/layer21_attention_weight_min": -18.98848533630371,
      "activations/layer22_attention_weight_max": 48.90777587890625,
      "activations/layer22_attention_weight_min": -27.726730346679688,
      "activations/layer23_attention_weight_max": 27.025869369506836,
      "activations/layer23_attention_weight_min": -17.548263549804688,
      "activations/layer2_attention_weight_max": 30.538860321044922,
      "activations/layer2_attention_weight_min": -30.2161865234375,
      "activations/layer3_attention_weight_max": 88.60482788085938,
      "activations/layer3_attention_weight_min": -87.98243713378906,
      "activations/layer4_attention_weight_max": 140.75730895996094,
      "activations/layer4_attention_weight_min": -130.88719177246094,
      "activations/layer5_attention_weight_max": 101.66011047363281,
      "activations/layer5_attention_weight_min": -87.96707153320312,
      "activations/layer6_attention_weight_max": 65.43574523925781,
      "activations/layer6_attention_weight_min": -58.39727020263672,
      "activations/layer7_attention_weight_max": 89.00018310546875,
      "activations/layer7_attention_weight_min": -79.16210174560547,
      "activations/layer8_attention_weight_max": 46.19371795654297,
      "activations/layer8_attention_weight_min": -40.76640319824219,
      "activations/layer9_attention_weight_max": 46.25651168823242,
      "activations/layer9_attention_weight_min": -37.85578918457031,
      "epoch": 3.86,
      "learning_rate": 0.00012635189393939391,
      "loss": 2.8796,
      "step": 66500
    },
    {
      "activations/layer0_attention_weight_max": 15.61117172241211,
      "activations/layer0_attention_weight_min": -11.419180870056152,
      "activations/layer10_attention_weight_max": 36.96278762817383,
      "activations/layer10_attention_weight_min": -30.49411392211914,
      "activations/layer11_attention_weight_max": 39.104209899902344,
      "activations/layer11_attention_weight_min": -30.571773529052734,
      "activations/layer12_attention_weight_max": 38.455867767333984,
      "activations/layer12_attention_weight_min": -30.959562301635742,
      "activations/layer13_attention_weight_max": 77.93966674804688,
      "activations/layer13_attention_weight_min": -46.286048889160156,
      "activations/layer14_attention_weight_max": 84.9711685180664,
      "activations/layer14_attention_weight_min": -51.0330696105957,
      "activations/layer15_attention_weight_max": 53.999732971191406,
      "activations/layer15_attention_weight_min": -30.648235321044922,
      "activations/layer16_attention_weight_max": 44.332515716552734,
      "activations/layer16_attention_weight_min": -29.153623580932617,
      "activations/layer17_attention_weight_max": 43.03510665893555,
      "activations/layer17_attention_weight_min": -25.59861183166504,
      "activations/layer18_attention_weight_max": 33.420318603515625,
      "activations/layer18_attention_weight_min": -19.311885833740234,
      "activations/layer19_attention_weight_max": 42.2736701965332,
      "activations/layer19_attention_weight_min": -30.403076171875,
      "activations/layer1_attention_weight_max": 17.63518714904785,
      "activations/layer1_attention_weight_min": -16.92945098876953,
      "activations/layer20_attention_weight_max": 35.7247314453125,
      "activations/layer20_attention_weight_min": -26.995744705200195,
      "activations/layer21_attention_weight_max": 36.01709747314453,
      "activations/layer21_attention_weight_min": -20.117464065551758,
      "activations/layer22_attention_weight_max": 50.93946075439453,
      "activations/layer22_attention_weight_min": -27.700788497924805,
      "activations/layer23_attention_weight_max": 30.745201110839844,
      "activations/layer23_attention_weight_min": -20.797622680664062,
      "activations/layer2_attention_weight_max": 31.389883041381836,
      "activations/layer2_attention_weight_min": -30.509307861328125,
      "activations/layer3_attention_weight_max": 85.98136901855469,
      "activations/layer3_attention_weight_min": -87.40491485595703,
      "activations/layer4_attention_weight_max": 134.32347106933594,
      "activations/layer4_attention_weight_min": -126.66329193115234,
      "activations/layer5_attention_weight_max": 97.07514953613281,
      "activations/layer5_attention_weight_min": -86.49163818359375,
      "activations/layer6_attention_weight_max": 58.356842041015625,
      "activations/layer6_attention_weight_min": -55.55091857910156,
      "activations/layer7_attention_weight_max": 75.8776626586914,
      "activations/layer7_attention_weight_min": -78.25521087646484,
      "activations/layer8_attention_weight_max": 39.928707122802734,
      "activations/layer8_attention_weight_min": -39.55171585083008,
      "activations/layer9_attention_weight_max": 41.68430709838867,
      "activations/layer9_attention_weight_min": -40.28508377075195,
      "epoch": 3.87,
      "learning_rate": 0.00012633295454545453,
      "loss": 2.8686,
      "step": 66550
    },
    {
      "activations/layer0_attention_weight_max": 16.282785415649414,
      "activations/layer0_attention_weight_min": -11.435659408569336,
      "activations/layer10_attention_weight_max": 37.981163024902344,
      "activations/layer10_attention_weight_min": -31.591222763061523,
      "activations/layer11_attention_weight_max": 32.706077575683594,
      "activations/layer11_attention_weight_min": -29.69154167175293,
      "activations/layer12_attention_weight_max": 30.376163482666016,
      "activations/layer12_attention_weight_min": -28.9179744720459,
      "activations/layer13_attention_weight_max": 48.50010299682617,
      "activations/layer13_attention_weight_min": -32.73690414428711,
      "activations/layer14_attention_weight_max": 72.86895751953125,
      "activations/layer14_attention_weight_min": -56.51982498168945,
      "activations/layer15_attention_weight_max": 45.29905319213867,
      "activations/layer15_attention_weight_min": -31.557865142822266,
      "activations/layer16_attention_weight_max": 42.88286590576172,
      "activations/layer16_attention_weight_min": -28.015480041503906,
      "activations/layer17_attention_weight_max": 43.033958435058594,
      "activations/layer17_attention_weight_min": -24.76957130432129,
      "activations/layer18_attention_weight_max": 33.76430130004883,
      "activations/layer18_attention_weight_min": -22.30848503112793,
      "activations/layer19_attention_weight_max": 45.56916427612305,
      "activations/layer19_attention_weight_min": -29.118803024291992,
      "activations/layer1_attention_weight_max": 17.47661781311035,
      "activations/layer1_attention_weight_min": -16.034564971923828,
      "activations/layer20_attention_weight_max": 34.32706832885742,
      "activations/layer20_attention_weight_min": -23.020599365234375,
      "activations/layer21_attention_weight_max": 39.1219482421875,
      "activations/layer21_attention_weight_min": -20.88836669921875,
      "activations/layer22_attention_weight_max": 50.63597106933594,
      "activations/layer22_attention_weight_min": -28.6436824798584,
      "activations/layer23_attention_weight_max": 29.964523315429688,
      "activations/layer23_attention_weight_min": -17.225467681884766,
      "activations/layer2_attention_weight_max": 33.14171600341797,
      "activations/layer2_attention_weight_min": -30.80451202392578,
      "activations/layer3_attention_weight_max": 85.77181243896484,
      "activations/layer3_attention_weight_min": -86.2688217163086,
      "activations/layer4_attention_weight_max": 131.47610473632812,
      "activations/layer4_attention_weight_min": -127.63115692138672,
      "activations/layer5_attention_weight_max": 91.42266845703125,
      "activations/layer5_attention_weight_min": -85.59622192382812,
      "activations/layer6_attention_weight_max": 58.24626541137695,
      "activations/layer6_attention_weight_min": -58.18309783935547,
      "activations/layer7_attention_weight_max": 76.36593627929688,
      "activations/layer7_attention_weight_min": -80.3547592163086,
      "activations/layer8_attention_weight_max": 41.208824157714844,
      "activations/layer8_attention_weight_min": -40.04619598388672,
      "activations/layer9_attention_weight_max": 41.711761474609375,
      "activations/layer9_attention_weight_min": -36.6352653503418,
      "epoch": 3.87,
      "learning_rate": 0.00012631401515151514,
      "loss": 2.8734,
      "step": 66600
    },
    {
      "activations/layer0_attention_weight_max": 15.955415725708008,
      "activations/layer0_attention_weight_min": -11.734436988830566,
      "activations/layer10_attention_weight_max": 39.33557891845703,
      "activations/layer10_attention_weight_min": -31.42192268371582,
      "activations/layer11_attention_weight_max": 33.87710952758789,
      "activations/layer11_attention_weight_min": -28.772228240966797,
      "activations/layer12_attention_weight_max": 27.205936431884766,
      "activations/layer12_attention_weight_min": -33.197479248046875,
      "activations/layer13_attention_weight_max": 50.31228256225586,
      "activations/layer13_attention_weight_min": -34.48100280761719,
      "activations/layer14_attention_weight_max": 98.41668701171875,
      "activations/layer14_attention_weight_min": -65.9779281616211,
      "activations/layer15_attention_weight_max": 47.07960510253906,
      "activations/layer15_attention_weight_min": -28.96640396118164,
      "activations/layer16_attention_weight_max": 41.08854293823242,
      "activations/layer16_attention_weight_min": -28.13210105895996,
      "activations/layer17_attention_weight_max": 43.06196594238281,
      "activations/layer17_attention_weight_min": -25.876258850097656,
      "activations/layer18_attention_weight_max": 34.6299934387207,
      "activations/layer18_attention_weight_min": -19.054277420043945,
      "activations/layer19_attention_weight_max": 45.52037048339844,
      "activations/layer19_attention_weight_min": -29.022375106811523,
      "activations/layer1_attention_weight_max": 18.115415573120117,
      "activations/layer1_attention_weight_min": -15.28986644744873,
      "activations/layer20_attention_weight_max": 33.4728889465332,
      "activations/layer20_attention_weight_min": -22.569957733154297,
      "activations/layer21_attention_weight_max": 30.392492294311523,
      "activations/layer21_attention_weight_min": -20.904720306396484,
      "activations/layer22_attention_weight_max": 52.78548049926758,
      "activations/layer22_attention_weight_min": -25.05742073059082,
      "activations/layer23_attention_weight_max": 32.27251434326172,
      "activations/layer23_attention_weight_min": -18.351913452148438,
      "activations/layer2_attention_weight_max": 31.821311950683594,
      "activations/layer2_attention_weight_min": -30.24551773071289,
      "activations/layer3_attention_weight_max": 88.80146789550781,
      "activations/layer3_attention_weight_min": -89.45487213134766,
      "activations/layer4_attention_weight_max": 139.8218231201172,
      "activations/layer4_attention_weight_min": -136.77720642089844,
      "activations/layer5_attention_weight_max": 96.05438232421875,
      "activations/layer5_attention_weight_min": -88.84981536865234,
      "activations/layer6_attention_weight_max": 59.69504928588867,
      "activations/layer6_attention_weight_min": -59.86579513549805,
      "activations/layer7_attention_weight_max": 78.54917907714844,
      "activations/layer7_attention_weight_min": -78.61982727050781,
      "activations/layer8_attention_weight_max": 42.18743133544922,
      "activations/layer8_attention_weight_min": -41.295875549316406,
      "activations/layer9_attention_weight_max": 39.30619812011719,
      "activations/layer9_attention_weight_min": -37.81907272338867,
      "epoch": 3.87,
      "learning_rate": 0.00012629507575757576,
      "loss": 2.8855,
      "step": 66650
    },
    {
      "activations/layer0_attention_weight_max": 16.580453872680664,
      "activations/layer0_attention_weight_min": -11.474502563476562,
      "activations/layer10_attention_weight_max": 32.370121002197266,
      "activations/layer10_attention_weight_min": -30.89533042907715,
      "activations/layer11_attention_weight_max": 33.75395584106445,
      "activations/layer11_attention_weight_min": -27.013036727905273,
      "activations/layer12_attention_weight_max": 28.520105361938477,
      "activations/layer12_attention_weight_min": -29.319454193115234,
      "activations/layer13_attention_weight_max": 41.010581970214844,
      "activations/layer13_attention_weight_min": -31.46729278564453,
      "activations/layer14_attention_weight_max": 69.93862915039062,
      "activations/layer14_attention_weight_min": -54.377052307128906,
      "activations/layer15_attention_weight_max": 43.769920349121094,
      "activations/layer15_attention_weight_min": -29.803754806518555,
      "activations/layer16_attention_weight_max": 38.1430549621582,
      "activations/layer16_attention_weight_min": -28.545324325561523,
      "activations/layer17_attention_weight_max": 41.61933517456055,
      "activations/layer17_attention_weight_min": -25.960716247558594,
      "activations/layer18_attention_weight_max": 32.13261413574219,
      "activations/layer18_attention_weight_min": -18.456092834472656,
      "activations/layer19_attention_weight_max": 39.415225982666016,
      "activations/layer19_attention_weight_min": -27.394376754760742,
      "activations/layer1_attention_weight_max": 18.01331901550293,
      "activations/layer1_attention_weight_min": -14.850998878479004,
      "activations/layer20_attention_weight_max": 30.535688400268555,
      "activations/layer20_attention_weight_min": -22.082956314086914,
      "activations/layer21_attention_weight_max": 28.065824508666992,
      "activations/layer21_attention_weight_min": -19.424083709716797,
      "activations/layer22_attention_weight_max": 43.710838317871094,
      "activations/layer22_attention_weight_min": -28.101539611816406,
      "activations/layer23_attention_weight_max": 26.359760284423828,
      "activations/layer23_attention_weight_min": -19.217510223388672,
      "activations/layer2_attention_weight_max": 32.63340377807617,
      "activations/layer2_attention_weight_min": -31.47732162475586,
      "activations/layer3_attention_weight_max": 88.06485748291016,
      "activations/layer3_attention_weight_min": -88.23845672607422,
      "activations/layer4_attention_weight_max": 134.4147186279297,
      "activations/layer4_attention_weight_min": -128.61231994628906,
      "activations/layer5_attention_weight_max": 91.23848724365234,
      "activations/layer5_attention_weight_min": -87.41508483886719,
      "activations/layer6_attention_weight_max": 63.72711944580078,
      "activations/layer6_attention_weight_min": -57.9742431640625,
      "activations/layer7_attention_weight_max": 80.48247528076172,
      "activations/layer7_attention_weight_min": -77.38862609863281,
      "activations/layer8_attention_weight_max": 38.98138427734375,
      "activations/layer8_attention_weight_min": -37.251338958740234,
      "activations/layer9_attention_weight_max": 34.65602493286133,
      "activations/layer9_attention_weight_min": -33.53375244140625,
      "epoch": 3.88,
      "learning_rate": 0.00012627613636363635,
      "loss": 2.883,
      "step": 66700
    },
    {
      "activations/layer0_attention_weight_max": 16.036548614501953,
      "activations/layer0_attention_weight_min": -10.756478309631348,
      "activations/layer10_attention_weight_max": 37.80677032470703,
      "activations/layer10_attention_weight_min": -32.0043830871582,
      "activations/layer11_attention_weight_max": 34.80352783203125,
      "activations/layer11_attention_weight_min": -27.056135177612305,
      "activations/layer12_attention_weight_max": 81.05230712890625,
      "activations/layer12_attention_weight_min": -29.308568954467773,
      "activations/layer13_attention_weight_max": 60.480255126953125,
      "activations/layer13_attention_weight_min": -34.363853454589844,
      "activations/layer14_attention_weight_max": 66.10816955566406,
      "activations/layer14_attention_weight_min": -46.86865997314453,
      "activations/layer15_attention_weight_max": 46.756874084472656,
      "activations/layer15_attention_weight_min": -29.788135528564453,
      "activations/layer16_attention_weight_max": 49.516014099121094,
      "activations/layer16_attention_weight_min": -30.558277130126953,
      "activations/layer17_attention_weight_max": 42.88813400268555,
      "activations/layer17_attention_weight_min": -26.045352935791016,
      "activations/layer18_attention_weight_max": 34.97661590576172,
      "activations/layer18_attention_weight_min": -19.703157424926758,
      "activations/layer19_attention_weight_max": 49.732479095458984,
      "activations/layer19_attention_weight_min": -31.53523063659668,
      "activations/layer1_attention_weight_max": 17.688879013061523,
      "activations/layer1_attention_weight_min": -14.925463676452637,
      "activations/layer20_attention_weight_max": 34.97596740722656,
      "activations/layer20_attention_weight_min": -24.549907684326172,
      "activations/layer21_attention_weight_max": 35.1940803527832,
      "activations/layer21_attention_weight_min": -20.949188232421875,
      "activations/layer22_attention_weight_max": 51.07830810546875,
      "activations/layer22_attention_weight_min": -27.35214614868164,
      "activations/layer23_attention_weight_max": 31.879796981811523,
      "activations/layer23_attention_weight_min": -19.793407440185547,
      "activations/layer2_attention_weight_max": 32.32411575317383,
      "activations/layer2_attention_weight_min": -32.45140075683594,
      "activations/layer3_attention_weight_max": 84.78035736083984,
      "activations/layer3_attention_weight_min": -88.90575408935547,
      "activations/layer4_attention_weight_max": 124.60057830810547,
      "activations/layer4_attention_weight_min": -128.15487670898438,
      "activations/layer5_attention_weight_max": 91.09669494628906,
      "activations/layer5_attention_weight_min": -85.22171020507812,
      "activations/layer6_attention_weight_max": 56.79417419433594,
      "activations/layer6_attention_weight_min": -56.8839111328125,
      "activations/layer7_attention_weight_max": 82.61263275146484,
      "activations/layer7_attention_weight_min": -78.2671890258789,
      "activations/layer8_attention_weight_max": 38.52625274658203,
      "activations/layer8_attention_weight_min": -38.84939193725586,
      "activations/layer9_attention_weight_max": 38.74985122680664,
      "activations/layer9_attention_weight_min": -38.464847564697266,
      "epoch": 3.88,
      "learning_rate": 0.00012625719696969696,
      "loss": 2.8814,
      "step": 66750
    },
    {
      "activations/layer0_attention_weight_max": 15.485114097595215,
      "activations/layer0_attention_weight_min": -12.490191459655762,
      "activations/layer10_attention_weight_max": 40.44216537475586,
      "activations/layer10_attention_weight_min": -32.14583206176758,
      "activations/layer11_attention_weight_max": 46.03150939941406,
      "activations/layer11_attention_weight_min": -29.771276473999023,
      "activations/layer12_attention_weight_max": 42.43354797363281,
      "activations/layer12_attention_weight_min": -28.727943420410156,
      "activations/layer13_attention_weight_max": 51.59040069580078,
      "activations/layer13_attention_weight_min": -33.57136154174805,
      "activations/layer14_attention_weight_max": 79.68804168701172,
      "activations/layer14_attention_weight_min": -49.655094146728516,
      "activations/layer15_attention_weight_max": 45.300048828125,
      "activations/layer15_attention_weight_min": -26.85854148864746,
      "activations/layer16_attention_weight_max": 44.76156234741211,
      "activations/layer16_attention_weight_min": -29.856578826904297,
      "activations/layer17_attention_weight_max": 48.83861541748047,
      "activations/layer17_attention_weight_min": -25.60172462463379,
      "activations/layer18_attention_weight_max": 33.3132438659668,
      "activations/layer18_attention_weight_min": -20.02931785583496,
      "activations/layer19_attention_weight_max": 43.886138916015625,
      "activations/layer19_attention_weight_min": -27.481077194213867,
      "activations/layer1_attention_weight_max": 18.237245559692383,
      "activations/layer1_attention_weight_min": -13.882747650146484,
      "activations/layer20_attention_weight_max": 32.80622100830078,
      "activations/layer20_attention_weight_min": -21.54686737060547,
      "activations/layer21_attention_weight_max": 32.75008010864258,
      "activations/layer21_attention_weight_min": -19.256256103515625,
      "activations/layer22_attention_weight_max": 51.12720489501953,
      "activations/layer22_attention_weight_min": -25.360376358032227,
      "activations/layer23_attention_weight_max": 29.56038475036621,
      "activations/layer23_attention_weight_min": -16.801654815673828,
      "activations/layer2_attention_weight_max": 31.577590942382812,
      "activations/layer2_attention_weight_min": -30.595699310302734,
      "activations/layer3_attention_weight_max": 87.38990783691406,
      "activations/layer3_attention_weight_min": -87.36326599121094,
      "activations/layer4_attention_weight_max": 133.67626953125,
      "activations/layer4_attention_weight_min": -123.62792205810547,
      "activations/layer5_attention_weight_max": 92.26558685302734,
      "activations/layer5_attention_weight_min": -81.33564758300781,
      "activations/layer6_attention_weight_max": 57.67428970336914,
      "activations/layer6_attention_weight_min": -54.86875534057617,
      "activations/layer7_attention_weight_max": 81.69515991210938,
      "activations/layer7_attention_weight_min": -78.02238464355469,
      "activations/layer8_attention_weight_max": 44.620765686035156,
      "activations/layer8_attention_weight_min": -38.15861892700195,
      "activations/layer9_attention_weight_max": 47.301918029785156,
      "activations/layer9_attention_weight_min": -41.79304122924805,
      "epoch": 3.88,
      "learning_rate": 0.00012623825757575758,
      "loss": 2.8706,
      "step": 66800
    },
    {
      "activations/layer0_attention_weight_max": 16.018356323242188,
      "activations/layer0_attention_weight_min": -11.6395263671875,
      "activations/layer10_attention_weight_max": 38.77674102783203,
      "activations/layer10_attention_weight_min": -30.38088035583496,
      "activations/layer11_attention_weight_max": 35.14257049560547,
      "activations/layer11_attention_weight_min": -30.679794311523438,
      "activations/layer12_attention_weight_max": 50.870765686035156,
      "activations/layer12_attention_weight_min": -27.500093460083008,
      "activations/layer13_attention_weight_max": 66.53758239746094,
      "activations/layer13_attention_weight_min": -48.019874572753906,
      "activations/layer14_attention_weight_max": 66.11531829833984,
      "activations/layer14_attention_weight_min": -49.69633865356445,
      "activations/layer15_attention_weight_max": 46.4438591003418,
      "activations/layer15_attention_weight_min": -32.732444763183594,
      "activations/layer16_attention_weight_max": 47.713035583496094,
      "activations/layer16_attention_weight_min": -30.26582908630371,
      "activations/layer17_attention_weight_max": 43.38499450683594,
      "activations/layer17_attention_weight_min": -25.847578048706055,
      "activations/layer18_attention_weight_max": 30.198514938354492,
      "activations/layer18_attention_weight_min": -20.38495445251465,
      "activations/layer19_attention_weight_max": 45.49273681640625,
      "activations/layer19_attention_weight_min": -28.422780990600586,
      "activations/layer1_attention_weight_max": 17.17613983154297,
      "activations/layer1_attention_weight_min": -17.161054611206055,
      "activations/layer20_attention_weight_max": 31.69610023498535,
      "activations/layer20_attention_weight_min": -23.401567459106445,
      "activations/layer21_attention_weight_max": 31.205169677734375,
      "activations/layer21_attention_weight_min": -18.1739501953125,
      "activations/layer22_attention_weight_max": 51.84830856323242,
      "activations/layer22_attention_weight_min": -25.60201072692871,
      "activations/layer23_attention_weight_max": 31.72052764892578,
      "activations/layer23_attention_weight_min": -17.52110481262207,
      "activations/layer2_attention_weight_max": 34.01512908935547,
      "activations/layer2_attention_weight_min": -32.415122985839844,
      "activations/layer3_attention_weight_max": 90.10002136230469,
      "activations/layer3_attention_weight_min": -91.1423568725586,
      "activations/layer4_attention_weight_max": 134.9130859375,
      "activations/layer4_attention_weight_min": -134.8272705078125,
      "activations/layer5_attention_weight_max": 94.79350280761719,
      "activations/layer5_attention_weight_min": -87.76766967773438,
      "activations/layer6_attention_weight_max": 58.89885330200195,
      "activations/layer6_attention_weight_min": -57.550357818603516,
      "activations/layer7_attention_weight_max": 83.03132629394531,
      "activations/layer7_attention_weight_min": -76.75667572021484,
      "activations/layer8_attention_weight_max": 42.612060546875,
      "activations/layer8_attention_weight_min": -42.1023063659668,
      "activations/layer9_attention_weight_max": 41.66526412963867,
      "activations/layer9_attention_weight_min": -34.70869445800781,
      "epoch": 3.88,
      "learning_rate": 0.00012621931818181817,
      "loss": 2.8909,
      "step": 66850
    },
    {
      "activations/layer0_attention_weight_max": 16.288381576538086,
      "activations/layer0_attention_weight_min": -10.75627613067627,
      "activations/layer10_attention_weight_max": 37.49964904785156,
      "activations/layer10_attention_weight_min": -30.848106384277344,
      "activations/layer11_attention_weight_max": 35.471702575683594,
      "activations/layer11_attention_weight_min": -28.012258529663086,
      "activations/layer12_attention_weight_max": 36.48930358886719,
      "activations/layer12_attention_weight_min": -25.30859375,
      "activations/layer13_attention_weight_max": 54.262474060058594,
      "activations/layer13_attention_weight_min": -33.134735107421875,
      "activations/layer14_attention_weight_max": 86.79900360107422,
      "activations/layer14_attention_weight_min": -51.32633590698242,
      "activations/layer15_attention_weight_max": 47.873992919921875,
      "activations/layer15_attention_weight_min": -28.09626579284668,
      "activations/layer16_attention_weight_max": 41.38063430786133,
      "activations/layer16_attention_weight_min": -30.602733612060547,
      "activations/layer17_attention_weight_max": 46.81822967529297,
      "activations/layer17_attention_weight_min": -23.745094299316406,
      "activations/layer18_attention_weight_max": 32.099090576171875,
      "activations/layer18_attention_weight_min": -19.419300079345703,
      "activations/layer19_attention_weight_max": 47.0134391784668,
      "activations/layer19_attention_weight_min": -28.45417594909668,
      "activations/layer1_attention_weight_max": 17.90531349182129,
      "activations/layer1_attention_weight_min": -15.146430015563965,
      "activations/layer20_attention_weight_max": 37.35575866699219,
      "activations/layer20_attention_weight_min": -22.38239097595215,
      "activations/layer21_attention_weight_max": 39.87461853027344,
      "activations/layer21_attention_weight_min": -21.751012802124023,
      "activations/layer22_attention_weight_max": 59.57149887084961,
      "activations/layer22_attention_weight_min": -26.59688377380371,
      "activations/layer23_attention_weight_max": 33.49727249145508,
      "activations/layer23_attention_weight_min": -18.239768981933594,
      "activations/layer2_attention_weight_max": 30.45669174194336,
      "activations/layer2_attention_weight_min": -30.720409393310547,
      "activations/layer3_attention_weight_max": 84.09811401367188,
      "activations/layer3_attention_weight_min": -84.93263244628906,
      "activations/layer4_attention_weight_max": 131.22251892089844,
      "activations/layer4_attention_weight_min": -124.91999816894531,
      "activations/layer5_attention_weight_max": 88.61152648925781,
      "activations/layer5_attention_weight_min": -85.067626953125,
      "activations/layer6_attention_weight_max": 58.8099479675293,
      "activations/layer6_attention_weight_min": -59.35464859008789,
      "activations/layer7_attention_weight_max": 76.12749481201172,
      "activations/layer7_attention_weight_min": -77.62335205078125,
      "activations/layer8_attention_weight_max": 40.1514778137207,
      "activations/layer8_attention_weight_min": -39.42833709716797,
      "activations/layer9_attention_weight_max": 40.7406120300293,
      "activations/layer9_attention_weight_min": -35.251956939697266,
      "epoch": 3.89,
      "learning_rate": 0.00012620037878787878,
      "loss": 2.8916,
      "step": 66900
    },
    {
      "activations/layer0_attention_weight_max": 15.254314422607422,
      "activations/layer0_attention_weight_min": -11.922712326049805,
      "activations/layer10_attention_weight_max": 36.90109634399414,
      "activations/layer10_attention_weight_min": -31.608041763305664,
      "activations/layer11_attention_weight_max": 36.22003936767578,
      "activations/layer11_attention_weight_min": -31.408842086791992,
      "activations/layer12_attention_weight_max": 29.682052612304688,
      "activations/layer12_attention_weight_min": -30.58233070373535,
      "activations/layer13_attention_weight_max": 59.50276565551758,
      "activations/layer13_attention_weight_min": -51.07258224487305,
      "activations/layer14_attention_weight_max": 69.16268157958984,
      "activations/layer14_attention_weight_min": -56.23849868774414,
      "activations/layer15_attention_weight_max": 47.99893569946289,
      "activations/layer15_attention_weight_min": -35.81005096435547,
      "activations/layer16_attention_weight_max": 39.45722198486328,
      "activations/layer16_attention_weight_min": -29.675567626953125,
      "activations/layer17_attention_weight_max": 39.92129898071289,
      "activations/layer17_attention_weight_min": -24.418710708618164,
      "activations/layer18_attention_weight_max": 36.23430633544922,
      "activations/layer18_attention_weight_min": -20.479530334472656,
      "activations/layer19_attention_weight_max": 43.926692962646484,
      "activations/layer19_attention_weight_min": -27.82425308227539,
      "activations/layer1_attention_weight_max": 17.468944549560547,
      "activations/layer1_attention_weight_min": -15.100866317749023,
      "activations/layer20_attention_weight_max": 31.211780548095703,
      "activations/layer20_attention_weight_min": -20.845550537109375,
      "activations/layer21_attention_weight_max": 34.20634460449219,
      "activations/layer21_attention_weight_min": -20.73642349243164,
      "activations/layer22_attention_weight_max": 45.72593688964844,
      "activations/layer22_attention_weight_min": -28.07560157775879,
      "activations/layer23_attention_weight_max": 27.221981048583984,
      "activations/layer23_attention_weight_min": -18.730823516845703,
      "activations/layer2_attention_weight_max": 31.86250877380371,
      "activations/layer2_attention_weight_min": -31.852783203125,
      "activations/layer3_attention_weight_max": 87.02259826660156,
      "activations/layer3_attention_weight_min": -88.8886947631836,
      "activations/layer4_attention_weight_max": 130.478515625,
      "activations/layer4_attention_weight_min": -127.4868392944336,
      "activations/layer5_attention_weight_max": 89.43183898925781,
      "activations/layer5_attention_weight_min": -83.8252944946289,
      "activations/layer6_attention_weight_max": 54.98419952392578,
      "activations/layer6_attention_weight_min": -54.60780334472656,
      "activations/layer7_attention_weight_max": 75.62612915039062,
      "activations/layer7_attention_weight_min": -75.84100341796875,
      "activations/layer8_attention_weight_max": 37.78553771972656,
      "activations/layer8_attention_weight_min": -39.8521614074707,
      "activations/layer9_attention_weight_max": 40.1696662902832,
      "activations/layer9_attention_weight_min": -33.8334846496582,
      "epoch": 3.89,
      "learning_rate": 0.0001261814393939394,
      "loss": 2.8815,
      "step": 66950
    },
    {
      "activations/layer0_attention_weight_max": 17.069398880004883,
      "activations/layer0_attention_weight_min": -10.84736442565918,
      "activations/layer10_attention_weight_max": 37.37005615234375,
      "activations/layer10_attention_weight_min": -30.148265838623047,
      "activations/layer11_attention_weight_max": 34.13337707519531,
      "activations/layer11_attention_weight_min": -28.534765243530273,
      "activations/layer12_attention_weight_max": 29.666833877563477,
      "activations/layer12_attention_weight_min": -25.862777709960938,
      "activations/layer13_attention_weight_max": 46.3426399230957,
      "activations/layer13_attention_weight_min": -33.626007080078125,
      "activations/layer14_attention_weight_max": 68.35248565673828,
      "activations/layer14_attention_weight_min": -51.23571014404297,
      "activations/layer15_attention_weight_max": 41.214752197265625,
      "activations/layer15_attention_weight_min": -28.77204132080078,
      "activations/layer16_attention_weight_max": 44.568359375,
      "activations/layer16_attention_weight_min": -28.297060012817383,
      "activations/layer17_attention_weight_max": 42.790489196777344,
      "activations/layer17_attention_weight_min": -24.30588722229004,
      "activations/layer18_attention_weight_max": 35.17220687866211,
      "activations/layer18_attention_weight_min": -20.643705368041992,
      "activations/layer19_attention_weight_max": 41.30311584472656,
      "activations/layer19_attention_weight_min": -27.84023094177246,
      "activations/layer1_attention_weight_max": 17.450294494628906,
      "activations/layer1_attention_weight_min": -15.183918952941895,
      "activations/layer20_attention_weight_max": 37.55353546142578,
      "activations/layer20_attention_weight_min": -22.032960891723633,
      "activations/layer21_attention_weight_max": 29.62868309020996,
      "activations/layer21_attention_weight_min": -19.570810317993164,
      "activations/layer22_attention_weight_max": 52.60866928100586,
      "activations/layer22_attention_weight_min": -27.077423095703125,
      "activations/layer23_attention_weight_max": 31.419925689697266,
      "activations/layer23_attention_weight_min": -19.001110076904297,
      "activations/layer2_attention_weight_max": 32.146636962890625,
      "activations/layer2_attention_weight_min": -29.47960662841797,
      "activations/layer3_attention_weight_max": 82.28388214111328,
      "activations/layer3_attention_weight_min": -87.11952209472656,
      "activations/layer4_attention_weight_max": 130.21524047851562,
      "activations/layer4_attention_weight_min": -130.29298400878906,
      "activations/layer5_attention_weight_max": 87.57820892333984,
      "activations/layer5_attention_weight_min": -81.15602111816406,
      "activations/layer6_attention_weight_max": 59.62775802612305,
      "activations/layer6_attention_weight_min": -54.82106018066406,
      "activations/layer7_attention_weight_max": 74.7836685180664,
      "activations/layer7_attention_weight_min": -72.42159271240234,
      "activations/layer8_attention_weight_max": 38.78346252441406,
      "activations/layer8_attention_weight_min": -36.19978713989258,
      "activations/layer9_attention_weight_max": 36.77584457397461,
      "activations/layer9_attention_weight_min": -33.35932540893555,
      "epoch": 3.89,
      "learning_rate": 0.00012616249999999998,
      "loss": 2.8952,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4858,
      "eval_samples_per_second": 506.021,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4858,
      "eval_openwebtext_samples_per_second": 506.021,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_wikitext_loss": 3.048828125,
      "eval_wikitext_ppl": 21.090614371389922,
      "eval_wikitext_runtime": 1.9833,
      "eval_wikitext_samples_per_second": 229.922,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_lambada_loss": 2.748046875,
      "eval_lambada_ppl": 15.612109685326937,
      "eval_lambada_runtime": 9.5277,
      "eval_lambada_samples_per_second": 511.034,
      "step": 67000
    },
    {
      "activations/layer0_attention_weight_max": 14.311823844909668,
      "activations/layer0_attention_weight_min": -10.84391975402832,
      "activations/layer10_attention_weight_max": 40.22707748413086,
      "activations/layer10_attention_weight_min": -28.89306640625,
      "activations/layer11_attention_weight_max": 41.54755783081055,
      "activations/layer11_attention_weight_min": -29.12518310546875,
      "activations/layer12_attention_weight_max": 79.30460357666016,
      "activations/layer12_attention_weight_min": -36.373165130615234,
      "activations/layer13_attention_weight_max": 97.17214965820312,
      "activations/layer13_attention_weight_min": -51.840660095214844,
      "activations/layer14_attention_weight_max": 90.86905670166016,
      "activations/layer14_attention_weight_min": -50.847782135009766,
      "activations/layer15_attention_weight_max": 60.14377975463867,
      "activations/layer15_attention_weight_min": -32.803592681884766,
      "activations/layer16_attention_weight_max": 44.506126403808594,
      "activations/layer16_attention_weight_min": -29.238805770874023,
      "activations/layer17_attention_weight_max": 43.32820510864258,
      "activations/layer17_attention_weight_min": -24.261959075927734,
      "activations/layer18_attention_weight_max": 29.486469268798828,
      "activations/layer18_attention_weight_min": -17.9869327545166,
      "activations/layer19_attention_weight_max": 39.6710205078125,
      "activations/layer19_attention_weight_min": -28.243844985961914,
      "activations/layer1_attention_weight_max": 17.713729858398438,
      "activations/layer1_attention_weight_min": -14.845078468322754,
      "activations/layer20_attention_weight_max": 38.17655944824219,
      "activations/layer20_attention_weight_min": -24.136070251464844,
      "activations/layer21_attention_weight_max": 34.73572540283203,
      "activations/layer21_attention_weight_min": -21.122608184814453,
      "activations/layer22_attention_weight_max": 43.92109298706055,
      "activations/layer22_attention_weight_min": -26.596778869628906,
      "activations/layer23_attention_weight_max": 27.238079071044922,
      "activations/layer23_attention_weight_min": -16.723312377929688,
      "activations/layer2_attention_weight_max": 33.304893493652344,
      "activations/layer2_attention_weight_min": -33.53040313720703,
      "activations/layer3_attention_weight_max": 84.69734191894531,
      "activations/layer3_attention_weight_min": -86.81304168701172,
      "activations/layer4_attention_weight_max": 137.13925170898438,
      "activations/layer4_attention_weight_min": -130.40237426757812,
      "activations/layer5_attention_weight_max": 97.74886322021484,
      "activations/layer5_attention_weight_min": -86.56364440917969,
      "activations/layer6_attention_weight_max": 61.03798294067383,
      "activations/layer6_attention_weight_min": -54.76998519897461,
      "activations/layer7_attention_weight_max": 88.60655975341797,
      "activations/layer7_attention_weight_min": -79.11512756347656,
      "activations/layer8_attention_weight_max": 42.046653747558594,
      "activations/layer8_attention_weight_min": -36.14919662475586,
      "activations/layer9_attention_weight_max": 44.27566146850586,
      "activations/layer9_attention_weight_min": -42.63963317871094,
      "epoch": 3.9,
      "learning_rate": 0.0001261435606060606,
      "loss": 2.8736,
      "step": 67050
    },
    {
      "activations/layer0_attention_weight_max": 18.357389450073242,
      "activations/layer0_attention_weight_min": -11.187773704528809,
      "activations/layer10_attention_weight_max": 37.951759338378906,
      "activations/layer10_attention_weight_min": -31.046855926513672,
      "activations/layer11_attention_weight_max": 35.85226058959961,
      "activations/layer11_attention_weight_min": -28.971233367919922,
      "activations/layer12_attention_weight_max": 27.83599281311035,
      "activations/layer12_attention_weight_min": -32.076087951660156,
      "activations/layer13_attention_weight_max": 42.688621520996094,
      "activations/layer13_attention_weight_min": -32.946693420410156,
      "activations/layer14_attention_weight_max": 69.31805419921875,
      "activations/layer14_attention_weight_min": -49.22096633911133,
      "activations/layer15_attention_weight_max": 56.178192138671875,
      "activations/layer15_attention_weight_min": -34.21399688720703,
      "activations/layer16_attention_weight_max": 39.355159759521484,
      "activations/layer16_attention_weight_min": -29.755496978759766,
      "activations/layer17_attention_weight_max": 42.883811950683594,
      "activations/layer17_attention_weight_min": -29.152917861938477,
      "activations/layer18_attention_weight_max": 40.69857406616211,
      "activations/layer18_attention_weight_min": -22.212936401367188,
      "activations/layer19_attention_weight_max": 47.248252868652344,
      "activations/layer19_attention_weight_min": -29.523679733276367,
      "activations/layer1_attention_weight_max": 16.958274841308594,
      "activations/layer1_attention_weight_min": -15.557656288146973,
      "activations/layer20_attention_weight_max": 35.662654876708984,
      "activations/layer20_attention_weight_min": -21.647441864013672,
      "activations/layer21_attention_weight_max": 32.54135513305664,
      "activations/layer21_attention_weight_min": -21.0544490814209,
      "activations/layer22_attention_weight_max": 60.680118560791016,
      "activations/layer22_attention_weight_min": -27.60173797607422,
      "activations/layer23_attention_weight_max": 33.669593811035156,
      "activations/layer23_attention_weight_min": -17.891311645507812,
      "activations/layer2_attention_weight_max": 32.89297866821289,
      "activations/layer2_attention_weight_min": -31.315732955932617,
      "activations/layer3_attention_weight_max": 90.7916030883789,
      "activations/layer3_attention_weight_min": -89.70096588134766,
      "activations/layer4_attention_weight_max": 141.471435546875,
      "activations/layer4_attention_weight_min": -136.8088836669922,
      "activations/layer5_attention_weight_max": 97.42367553710938,
      "activations/layer5_attention_weight_min": -90.28610229492188,
      "activations/layer6_attention_weight_max": 58.482330322265625,
      "activations/layer6_attention_weight_min": -57.171913146972656,
      "activations/layer7_attention_weight_max": 78.81658935546875,
      "activations/layer7_attention_weight_min": -76.22714233398438,
      "activations/layer8_attention_weight_max": 41.227577209472656,
      "activations/layer8_attention_weight_min": -39.503440856933594,
      "activations/layer9_attention_weight_max": 47.246055603027344,
      "activations/layer9_attention_weight_min": -37.67557144165039,
      "epoch": 3.9,
      "learning_rate": 0.0001261246212121212,
      "loss": 2.8742,
      "step": 67100
    },
    {
      "activations/layer0_attention_weight_max": 16.42768096923828,
      "activations/layer0_attention_weight_min": -11.398816108703613,
      "activations/layer10_attention_weight_max": 39.585357666015625,
      "activations/layer10_attention_weight_min": -33.37908172607422,
      "activations/layer11_attention_weight_max": 38.13307571411133,
      "activations/layer11_attention_weight_min": -28.679012298583984,
      "activations/layer12_attention_weight_max": 47.9468994140625,
      "activations/layer12_attention_weight_min": -28.00547218322754,
      "activations/layer13_attention_weight_max": 50.44571304321289,
      "activations/layer13_attention_weight_min": -33.67351531982422,
      "activations/layer14_attention_weight_max": 73.88994598388672,
      "activations/layer14_attention_weight_min": -52.255615234375,
      "activations/layer15_attention_weight_max": 46.31963348388672,
      "activations/layer15_attention_weight_min": -29.945690155029297,
      "activations/layer16_attention_weight_max": 43.327632904052734,
      "activations/layer16_attention_weight_min": -31.338809967041016,
      "activations/layer17_attention_weight_max": 48.99374008178711,
      "activations/layer17_attention_weight_min": -24.657455444335938,
      "activations/layer18_attention_weight_max": 33.29511642456055,
      "activations/layer18_attention_weight_min": -19.88594627380371,
      "activations/layer19_attention_weight_max": 50.40516662597656,
      "activations/layer19_attention_weight_min": -29.730438232421875,
      "activations/layer1_attention_weight_max": 16.71207046508789,
      "activations/layer1_attention_weight_min": -14.911731719970703,
      "activations/layer20_attention_weight_max": 41.44391632080078,
      "activations/layer20_attention_weight_min": -22.449016571044922,
      "activations/layer21_attention_weight_max": 34.85068130493164,
      "activations/layer21_attention_weight_min": -20.01777458190918,
      "activations/layer22_attention_weight_max": 59.47927474975586,
      "activations/layer22_attention_weight_min": -27.664440155029297,
      "activations/layer23_attention_weight_max": 32.7886848449707,
      "activations/layer23_attention_weight_min": -17.06772232055664,
      "activations/layer2_attention_weight_max": 32.179847717285156,
      "activations/layer2_attention_weight_min": -31.381118774414062,
      "activations/layer3_attention_weight_max": 84.46552276611328,
      "activations/layer3_attention_weight_min": -85.65521240234375,
      "activations/layer4_attention_weight_max": 134.63255310058594,
      "activations/layer4_attention_weight_min": -126.42866516113281,
      "activations/layer5_attention_weight_max": 90.45268249511719,
      "activations/layer5_attention_weight_min": -84.83518981933594,
      "activations/layer6_attention_weight_max": 58.66972351074219,
      "activations/layer6_attention_weight_min": -57.02570724487305,
      "activations/layer7_attention_weight_max": 80.11029052734375,
      "activations/layer7_attention_weight_min": -81.49815368652344,
      "activations/layer8_attention_weight_max": 39.78630828857422,
      "activations/layer8_attention_weight_min": -39.30329132080078,
      "activations/layer9_attention_weight_max": 39.94480514526367,
      "activations/layer9_attention_weight_min": -36.1077766418457,
      "epoch": 3.9,
      "learning_rate": 0.0001261056818181818,
      "loss": 2.8756,
      "step": 67150
    },
    {
      "activations/layer0_attention_weight_max": 16.67306900024414,
      "activations/layer0_attention_weight_min": -11.018896102905273,
      "activations/layer10_attention_weight_max": 36.60927200317383,
      "activations/layer10_attention_weight_min": -32.096351623535156,
      "activations/layer11_attention_weight_max": 34.30186080932617,
      "activations/layer11_attention_weight_min": -27.435848236083984,
      "activations/layer12_attention_weight_max": 29.46501350402832,
      "activations/layer12_attention_weight_min": -29.164756774902344,
      "activations/layer13_attention_weight_max": 54.39913558959961,
      "activations/layer13_attention_weight_min": -40.82018280029297,
      "activations/layer14_attention_weight_max": 60.956207275390625,
      "activations/layer14_attention_weight_min": -44.83529281616211,
      "activations/layer15_attention_weight_max": 42.56276321411133,
      "activations/layer15_attention_weight_min": -32.14409255981445,
      "activations/layer16_attention_weight_max": 38.74616622924805,
      "activations/layer16_attention_weight_min": -27.82592010498047,
      "activations/layer17_attention_weight_max": 43.58297348022461,
      "activations/layer17_attention_weight_min": -25.67160987854004,
      "activations/layer18_attention_weight_max": 28.900718688964844,
      "activations/layer18_attention_weight_min": -21.301345825195312,
      "activations/layer19_attention_weight_max": 42.433349609375,
      "activations/layer19_attention_weight_min": -27.976459503173828,
      "activations/layer1_attention_weight_max": 17.330846786499023,
      "activations/layer1_attention_weight_min": -13.500828742980957,
      "activations/layer20_attention_weight_max": 34.162052154541016,
      "activations/layer20_attention_weight_min": -23.881696701049805,
      "activations/layer21_attention_weight_max": 30.27920150756836,
      "activations/layer21_attention_weight_min": -19.029268264770508,
      "activations/layer22_attention_weight_max": 47.44858932495117,
      "activations/layer22_attention_weight_min": -25.80341148376465,
      "activations/layer23_attention_weight_max": 29.50719451904297,
      "activations/layer23_attention_weight_min": -17.868663787841797,
      "activations/layer2_attention_weight_max": 31.80331802368164,
      "activations/layer2_attention_weight_min": -30.80437469482422,
      "activations/layer3_attention_weight_max": 87.84992980957031,
      "activations/layer3_attention_weight_min": -85.95695495605469,
      "activations/layer4_attention_weight_max": 136.98312377929688,
      "activations/layer4_attention_weight_min": -128.27487182617188,
      "activations/layer5_attention_weight_max": 93.88209533691406,
      "activations/layer5_attention_weight_min": -89.03425598144531,
      "activations/layer6_attention_weight_max": 60.73331832885742,
      "activations/layer6_attention_weight_min": -57.751346588134766,
      "activations/layer7_attention_weight_max": 79.053955078125,
      "activations/layer7_attention_weight_min": -81.07304382324219,
      "activations/layer8_attention_weight_max": 40.15473175048828,
      "activations/layer8_attention_weight_min": -38.75041580200195,
      "activations/layer9_attention_weight_max": 40.264320373535156,
      "activations/layer9_attention_weight_min": -33.456634521484375,
      "epoch": 3.9,
      "learning_rate": 0.0001260871212121212,
      "loss": 2.8666,
      "step": 67200
    },
    {
      "activations/layer0_attention_weight_max": 17.066144943237305,
      "activations/layer0_attention_weight_min": -11.838791847229004,
      "activations/layer10_attention_weight_max": 32.559043884277344,
      "activations/layer10_attention_weight_min": -30.16367530822754,
      "activations/layer11_attention_weight_max": 31.458995819091797,
      "activations/layer11_attention_weight_min": -25.584545135498047,
      "activations/layer12_attention_weight_max": 32.943092346191406,
      "activations/layer12_attention_weight_min": -25.88506507873535,
      "activations/layer13_attention_weight_max": 42.75273895263672,
      "activations/layer13_attention_weight_min": -33.18857192993164,
      "activations/layer14_attention_weight_max": 69.65299224853516,
      "activations/layer14_attention_weight_min": -49.81990432739258,
      "activations/layer15_attention_weight_max": 45.27315902709961,
      "activations/layer15_attention_weight_min": -32.40281295776367,
      "activations/layer16_attention_weight_max": 42.72542190551758,
      "activations/layer16_attention_weight_min": -29.270978927612305,
      "activations/layer17_attention_weight_max": 37.30504608154297,
      "activations/layer17_attention_weight_min": -23.717632293701172,
      "activations/layer18_attention_weight_max": 35.44775390625,
      "activations/layer18_attention_weight_min": -19.697315216064453,
      "activations/layer19_attention_weight_max": 42.46598434448242,
      "activations/layer19_attention_weight_min": -30.30165672302246,
      "activations/layer1_attention_weight_max": 16.952999114990234,
      "activations/layer1_attention_weight_min": -16.320476531982422,
      "activations/layer20_attention_weight_max": 30.5871524810791,
      "activations/layer20_attention_weight_min": -23.50941276550293,
      "activations/layer21_attention_weight_max": 32.72475814819336,
      "activations/layer21_attention_weight_min": -20.021398544311523,
      "activations/layer22_attention_weight_max": 44.38742446899414,
      "activations/layer22_attention_weight_min": -28.92021942138672,
      "activations/layer23_attention_weight_max": 29.592697143554688,
      "activations/layer23_attention_weight_min": -18.79339599609375,
      "activations/layer2_attention_weight_max": 31.702617645263672,
      "activations/layer2_attention_weight_min": -32.01685333251953,
      "activations/layer3_attention_weight_max": 86.16160583496094,
      "activations/layer3_attention_weight_min": -84.86775970458984,
      "activations/layer4_attention_weight_max": 130.4970703125,
      "activations/layer4_attention_weight_min": -124.11553192138672,
      "activations/layer5_attention_weight_max": 88.97779846191406,
      "activations/layer5_attention_weight_min": -81.72433471679688,
      "activations/layer6_attention_weight_max": 58.492515563964844,
      "activations/layer6_attention_weight_min": -54.00019073486328,
      "activations/layer7_attention_weight_max": 77.69505310058594,
      "activations/layer7_attention_weight_min": -76.92433166503906,
      "activations/layer8_attention_weight_max": 38.66179275512695,
      "activations/layer8_attention_weight_min": -35.57536697387695,
      "activations/layer9_attention_weight_max": 37.004940032958984,
      "activations/layer9_attention_weight_min": -34.28828048706055,
      "epoch": 3.91,
      "learning_rate": 0.0001260681818181818,
      "loss": 2.8717,
      "step": 67250
    },
    {
      "activations/layer0_attention_weight_max": 16.645177841186523,
      "activations/layer0_attention_weight_min": -10.941010475158691,
      "activations/layer10_attention_weight_max": 41.930999755859375,
      "activations/layer10_attention_weight_min": -34.72568893432617,
      "activations/layer11_attention_weight_max": 38.387149810791016,
      "activations/layer11_attention_weight_min": -28.81129264831543,
      "activations/layer12_attention_weight_max": 73.22467041015625,
      "activations/layer12_attention_weight_min": -30.621078491210938,
      "activations/layer13_attention_weight_max": 91.07324981689453,
      "activations/layer13_attention_weight_min": -43.915199279785156,
      "activations/layer14_attention_weight_max": 77.04724884033203,
      "activations/layer14_attention_weight_min": -47.661537170410156,
      "activations/layer15_attention_weight_max": 71.74540710449219,
      "activations/layer15_attention_weight_min": -30.95981788635254,
      "activations/layer16_attention_weight_max": 46.14539337158203,
      "activations/layer16_attention_weight_min": -26.456268310546875,
      "activations/layer17_attention_weight_max": 47.580780029296875,
      "activations/layer17_attention_weight_min": -26.627696990966797,
      "activations/layer18_attention_weight_max": 36.56648635864258,
      "activations/layer18_attention_weight_min": -20.117752075195312,
      "activations/layer19_attention_weight_max": 43.729923248291016,
      "activations/layer19_attention_weight_min": -27.5179500579834,
      "activations/layer1_attention_weight_max": 17.351970672607422,
      "activations/layer1_attention_weight_min": -16.232067108154297,
      "activations/layer20_attention_weight_max": 33.10358428955078,
      "activations/layer20_attention_weight_min": -21.40135955810547,
      "activations/layer21_attention_weight_max": 29.00502586364746,
      "activations/layer21_attention_weight_min": -18.52093505859375,
      "activations/layer22_attention_weight_max": 52.54636001586914,
      "activations/layer22_attention_weight_min": -27.408065795898438,
      "activations/layer23_attention_weight_max": 28.614591598510742,
      "activations/layer23_attention_weight_min": -18.08251190185547,
      "activations/layer2_attention_weight_max": 32.50798797607422,
      "activations/layer2_attention_weight_min": -31.25606918334961,
      "activations/layer3_attention_weight_max": 91.00914001464844,
      "activations/layer3_attention_weight_min": -90.69343566894531,
      "activations/layer4_attention_weight_max": 147.3116455078125,
      "activations/layer4_attention_weight_min": -136.1571502685547,
      "activations/layer5_attention_weight_max": 106.06993103027344,
      "activations/layer5_attention_weight_min": -95.91136169433594,
      "activations/layer6_attention_weight_max": 67.39065551757812,
      "activations/layer6_attention_weight_min": -61.40447998046875,
      "activations/layer7_attention_weight_max": 96.47112274169922,
      "activations/layer7_attention_weight_min": -86.01861572265625,
      "activations/layer8_attention_weight_max": 52.42372512817383,
      "activations/layer8_attention_weight_min": -47.07771301269531,
      "activations/layer9_attention_weight_max": 53.25027084350586,
      "activations/layer9_attention_weight_min": -38.27982711791992,
      "epoch": 3.91,
      "learning_rate": 0.00012604924242424242,
      "loss": 2.8884,
      "step": 67300
    },
    {
      "activations/layer0_attention_weight_max": 16.418869018554688,
      "activations/layer0_attention_weight_min": -11.442635536193848,
      "activations/layer10_attention_weight_max": 37.73383331298828,
      "activations/layer10_attention_weight_min": -30.556657791137695,
      "activations/layer11_attention_weight_max": 35.951690673828125,
      "activations/layer11_attention_weight_min": -28.74098777770996,
      "activations/layer12_attention_weight_max": 30.237638473510742,
      "activations/layer12_attention_weight_min": -24.323211669921875,
      "activations/layer13_attention_weight_max": 49.47252655029297,
      "activations/layer13_attention_weight_min": -32.58353805541992,
      "activations/layer14_attention_weight_max": 72.01966857910156,
      "activations/layer14_attention_weight_min": -49.82608413696289,
      "activations/layer15_attention_weight_max": 45.03193283081055,
      "activations/layer15_attention_weight_min": -31.608354568481445,
      "activations/layer16_attention_weight_max": 41.49293518066406,
      "activations/layer16_attention_weight_min": -28.011493682861328,
      "activations/layer17_attention_weight_max": 40.95283126831055,
      "activations/layer17_attention_weight_min": -25.87317657470703,
      "activations/layer18_attention_weight_max": 31.67786979675293,
      "activations/layer18_attention_weight_min": -19.12330436706543,
      "activations/layer19_attention_weight_max": 45.50707244873047,
      "activations/layer19_attention_weight_min": -26.159849166870117,
      "activations/layer1_attention_weight_max": 16.994518280029297,
      "activations/layer1_attention_weight_min": -16.00918197631836,
      "activations/layer20_attention_weight_max": 33.96629333496094,
      "activations/layer20_attention_weight_min": -23.258378982543945,
      "activations/layer21_attention_weight_max": 36.21131896972656,
      "activations/layer21_attention_weight_min": -19.580707550048828,
      "activations/layer22_attention_weight_max": 57.756752014160156,
      "activations/layer22_attention_weight_min": -28.28035545349121,
      "activations/layer23_attention_weight_max": 32.986663818359375,
      "activations/layer23_attention_weight_min": -17.136396408081055,
      "activations/layer2_attention_weight_max": 32.18538284301758,
      "activations/layer2_attention_weight_min": -31.32503318786621,
      "activations/layer3_attention_weight_max": 85.70948791503906,
      "activations/layer3_attention_weight_min": -90.9084701538086,
      "activations/layer4_attention_weight_max": 125.49271392822266,
      "activations/layer4_attention_weight_min": -129.38511657714844,
      "activations/layer5_attention_weight_max": 86.88264465332031,
      "activations/layer5_attention_weight_min": -85.0577392578125,
      "activations/layer6_attention_weight_max": 55.70878601074219,
      "activations/layer6_attention_weight_min": -56.3715705871582,
      "activations/layer7_attention_weight_max": 78.10255432128906,
      "activations/layer7_attention_weight_min": -79.58209991455078,
      "activations/layer8_attention_weight_max": 39.903045654296875,
      "activations/layer8_attention_weight_min": -38.974525451660156,
      "activations/layer9_attention_weight_max": 36.91444778442383,
      "activations/layer9_attention_weight_min": -34.10670852661133,
      "epoch": 3.91,
      "learning_rate": 0.000126030303030303,
      "loss": 2.8708,
      "step": 67350
    },
    {
      "activations/layer0_attention_weight_max": 16.08017349243164,
      "activations/layer0_attention_weight_min": -10.964115142822266,
      "activations/layer10_attention_weight_max": 38.66170120239258,
      "activations/layer10_attention_weight_min": -30.239477157592773,
      "activations/layer11_attention_weight_max": 37.728485107421875,
      "activations/layer11_attention_weight_min": -27.864063262939453,
      "activations/layer12_attention_weight_max": 61.940269470214844,
      "activations/layer12_attention_weight_min": -30.843061447143555,
      "activations/layer13_attention_weight_max": 60.32234573364258,
      "activations/layer13_attention_weight_min": -36.867103576660156,
      "activations/layer14_attention_weight_max": 64.91179656982422,
      "activations/layer14_attention_weight_min": -49.541770935058594,
      "activations/layer15_attention_weight_max": 39.103599548339844,
      "activations/layer15_attention_weight_min": -30.534317016601562,
      "activations/layer16_attention_weight_max": 41.0290641784668,
      "activations/layer16_attention_weight_min": -29.51511573791504,
      "activations/layer17_attention_weight_max": 39.18646240234375,
      "activations/layer17_attention_weight_min": -25.792375564575195,
      "activations/layer18_attention_weight_max": 29.36994743347168,
      "activations/layer18_attention_weight_min": -20.502164840698242,
      "activations/layer19_attention_weight_max": 40.67377853393555,
      "activations/layer19_attention_weight_min": -30.438085556030273,
      "activations/layer1_attention_weight_max": 17.208765029907227,
      "activations/layer1_attention_weight_min": -14.163695335388184,
      "activations/layer20_attention_weight_max": 33.25319290161133,
      "activations/layer20_attention_weight_min": -23.341306686401367,
      "activations/layer21_attention_weight_max": 36.4577522277832,
      "activations/layer21_attention_weight_min": -23.828128814697266,
      "activations/layer22_attention_weight_max": 47.146724700927734,
      "activations/layer22_attention_weight_min": -29.589679718017578,
      "activations/layer23_attention_weight_max": 27.938785552978516,
      "activations/layer23_attention_weight_min": -20.095958709716797,
      "activations/layer2_attention_weight_max": 32.3253173828125,
      "activations/layer2_attention_weight_min": -31.970138549804688,
      "activations/layer3_attention_weight_max": 89.87999725341797,
      "activations/layer3_attention_weight_min": -91.68284606933594,
      "activations/layer4_attention_weight_max": 133.31370544433594,
      "activations/layer4_attention_weight_min": -128.7992401123047,
      "activations/layer5_attention_weight_max": 91.29069519042969,
      "activations/layer5_attention_weight_min": -84.72091674804688,
      "activations/layer6_attention_weight_max": 59.44331741333008,
      "activations/layer6_attention_weight_min": -59.1089973449707,
      "activations/layer7_attention_weight_max": 79.54296875,
      "activations/layer7_attention_weight_min": -80.35321044921875,
      "activations/layer8_attention_weight_max": 42.30584716796875,
      "activations/layer8_attention_weight_min": -39.995880126953125,
      "activations/layer9_attention_weight_max": 47.89315414428711,
      "activations/layer9_attention_weight_min": -34.59144592285156,
      "epoch": 3.92,
      "learning_rate": 0.00012601136363636363,
      "loss": 2.863,
      "step": 67400
    },
    {
      "activations/layer0_attention_weight_max": 15.207836151123047,
      "activations/layer0_attention_weight_min": -10.949782371520996,
      "activations/layer10_attention_weight_max": 38.18416213989258,
      "activations/layer10_attention_weight_min": -30.43634033203125,
      "activations/layer11_attention_weight_max": 36.322181701660156,
      "activations/layer11_attention_weight_min": -26.318037033081055,
      "activations/layer12_attention_weight_max": 30.606060028076172,
      "activations/layer12_attention_weight_min": -27.960641860961914,
      "activations/layer13_attention_weight_max": 47.6114616394043,
      "activations/layer13_attention_weight_min": -36.13512420654297,
      "activations/layer14_attention_weight_max": 86.16828918457031,
      "activations/layer14_attention_weight_min": -56.8560905456543,
      "activations/layer15_attention_weight_max": 48.148319244384766,
      "activations/layer15_attention_weight_min": -33.13587188720703,
      "activations/layer16_attention_weight_max": 42.509544372558594,
      "activations/layer16_attention_weight_min": -29.546955108642578,
      "activations/layer17_attention_weight_max": 47.474369049072266,
      "activations/layer17_attention_weight_min": -26.667428970336914,
      "activations/layer18_attention_weight_max": 36.952247619628906,
      "activations/layer18_attention_weight_min": -19.6293888092041,
      "activations/layer19_attention_weight_max": 47.76067352294922,
      "activations/layer19_attention_weight_min": -31.968290328979492,
      "activations/layer1_attention_weight_max": 17.499513626098633,
      "activations/layer1_attention_weight_min": -15.779207229614258,
      "activations/layer20_attention_weight_max": 39.55921173095703,
      "activations/layer20_attention_weight_min": -23.617650985717773,
      "activations/layer21_attention_weight_max": 35.717620849609375,
      "activations/layer21_attention_weight_min": -20.064048767089844,
      "activations/layer22_attention_weight_max": 52.7550163269043,
      "activations/layer22_attention_weight_min": -30.422008514404297,
      "activations/layer23_attention_weight_max": 31.74111557006836,
      "activations/layer23_attention_weight_min": -17.506389617919922,
      "activations/layer2_attention_weight_max": 31.413345336914062,
      "activations/layer2_attention_weight_min": -30.385183334350586,
      "activations/layer3_attention_weight_max": 84.55724334716797,
      "activations/layer3_attention_weight_min": -87.634521484375,
      "activations/layer4_attention_weight_max": 129.1802215576172,
      "activations/layer4_attention_weight_min": -129.51715087890625,
      "activations/layer5_attention_weight_max": 92.1483383178711,
      "activations/layer5_attention_weight_min": -84.54573059082031,
      "activations/layer6_attention_weight_max": 58.76976776123047,
      "activations/layer6_attention_weight_min": -55.76704025268555,
      "activations/layer7_attention_weight_max": 78.59557342529297,
      "activations/layer7_attention_weight_min": -85.48117065429688,
      "activations/layer8_attention_weight_max": 40.55873489379883,
      "activations/layer8_attention_weight_min": -37.98229217529297,
      "activations/layer9_attention_weight_max": 39.065494537353516,
      "activations/layer9_attention_weight_min": -33.7171516418457,
      "epoch": 3.92,
      "learning_rate": 0.00012599242424242424,
      "loss": 2.8832,
      "step": 67450
    },
    {
      "activations/layer0_attention_weight_max": 16.5521240234375,
      "activations/layer0_attention_weight_min": -12.119558334350586,
      "activations/layer10_attention_weight_max": 37.90884780883789,
      "activations/layer10_attention_weight_min": -31.0841064453125,
      "activations/layer11_attention_weight_max": 32.685264587402344,
      "activations/layer11_attention_weight_min": -29.357879638671875,
      "activations/layer12_attention_weight_max": 31.316408157348633,
      "activations/layer12_attention_weight_min": -25.320632934570312,
      "activations/layer13_attention_weight_max": 43.43233108520508,
      "activations/layer13_attention_weight_min": -33.75071716308594,
      "activations/layer14_attention_weight_max": 71.73898315429688,
      "activations/layer14_attention_weight_min": -52.930843353271484,
      "activations/layer15_attention_weight_max": 41.25577926635742,
      "activations/layer15_attention_weight_min": -31.644062042236328,
      "activations/layer16_attention_weight_max": 40.868770599365234,
      "activations/layer16_attention_weight_min": -29.29045295715332,
      "activations/layer17_attention_weight_max": 38.56709289550781,
      "activations/layer17_attention_weight_min": -24.93277931213379,
      "activations/layer18_attention_weight_max": 29.954307556152344,
      "activations/layer18_attention_weight_min": -19.97966766357422,
      "activations/layer19_attention_weight_max": 43.5900993347168,
      "activations/layer19_attention_weight_min": -29.9559326171875,
      "activations/layer1_attention_weight_max": 17.154884338378906,
      "activations/layer1_attention_weight_min": -15.834671020507812,
      "activations/layer20_attention_weight_max": 32.70762252807617,
      "activations/layer20_attention_weight_min": -22.182498931884766,
      "activations/layer21_attention_weight_max": 30.739036560058594,
      "activations/layer21_attention_weight_min": -20.961576461791992,
      "activations/layer22_attention_weight_max": 49.383392333984375,
      "activations/layer22_attention_weight_min": -27.44489288330078,
      "activations/layer23_attention_weight_max": 30.63457489013672,
      "activations/layer23_attention_weight_min": -18.272998809814453,
      "activations/layer2_attention_weight_max": 33.79874038696289,
      "activations/layer2_attention_weight_min": -32.79250717163086,
      "activations/layer3_attention_weight_max": 86.77804565429688,
      "activations/layer3_attention_weight_min": -89.87554168701172,
      "activations/layer4_attention_weight_max": 129.2816925048828,
      "activations/layer4_attention_weight_min": -126.4854507446289,
      "activations/layer5_attention_weight_max": 90.30543518066406,
      "activations/layer5_attention_weight_min": -84.19721221923828,
      "activations/layer6_attention_weight_max": 58.725101470947266,
      "activations/layer6_attention_weight_min": -56.94865036010742,
      "activations/layer7_attention_weight_max": 77.06651306152344,
      "activations/layer7_attention_weight_min": -77.6905288696289,
      "activations/layer8_attention_weight_max": 38.03234100341797,
      "activations/layer8_attention_weight_min": -40.437801361083984,
      "activations/layer9_attention_weight_max": 37.29651641845703,
      "activations/layer9_attention_weight_min": -33.573463439941406,
      "epoch": 3.92,
      "learning_rate": 0.00012597348484848483,
      "loss": 2.8727,
      "step": 67500
    },
    {
      "activations/layer0_attention_weight_max": 16.20697784423828,
      "activations/layer0_attention_weight_min": -11.18265438079834,
      "activations/layer10_attention_weight_max": 35.21051788330078,
      "activations/layer10_attention_weight_min": -29.594621658325195,
      "activations/layer11_attention_weight_max": 32.572021484375,
      "activations/layer11_attention_weight_min": -26.546415328979492,
      "activations/layer12_attention_weight_max": 29.170385360717773,
      "activations/layer12_attention_weight_min": -27.651140213012695,
      "activations/layer13_attention_weight_max": 44.922115325927734,
      "activations/layer13_attention_weight_min": -31.932748794555664,
      "activations/layer14_attention_weight_max": 61.00830078125,
      "activations/layer14_attention_weight_min": -48.90037155151367,
      "activations/layer15_attention_weight_max": 39.45256805419922,
      "activations/layer15_attention_weight_min": -29.772951126098633,
      "activations/layer16_attention_weight_max": 38.75614929199219,
      "activations/layer16_attention_weight_min": -30.4876708984375,
      "activations/layer17_attention_weight_max": 35.292457580566406,
      "activations/layer17_attention_weight_min": -25.54556655883789,
      "activations/layer18_attention_weight_max": 29.323026657104492,
      "activations/layer18_attention_weight_min": -22.50656509399414,
      "activations/layer19_attention_weight_max": 39.073123931884766,
      "activations/layer19_attention_weight_min": -29.282873153686523,
      "activations/layer1_attention_weight_max": 16.709918975830078,
      "activations/layer1_attention_weight_min": -16.01983642578125,
      "activations/layer20_attention_weight_max": 30.638599395751953,
      "activations/layer20_attention_weight_min": -22.9001522064209,
      "activations/layer21_attention_weight_max": 32.65943145751953,
      "activations/layer21_attention_weight_min": -19.64963150024414,
      "activations/layer22_attention_weight_max": 45.91276931762695,
      "activations/layer22_attention_weight_min": -25.46828269958496,
      "activations/layer23_attention_weight_max": 29.47594451904297,
      "activations/layer23_attention_weight_min": -20.154014587402344,
      "activations/layer2_attention_weight_max": 30.76070213317871,
      "activations/layer2_attention_weight_min": -31.200511932373047,
      "activations/layer3_attention_weight_max": 82.25354766845703,
      "activations/layer3_attention_weight_min": -87.58048248291016,
      "activations/layer4_attention_weight_max": 130.08128356933594,
      "activations/layer4_attention_weight_min": -123.73207092285156,
      "activations/layer5_attention_weight_max": 90.33627319335938,
      "activations/layer5_attention_weight_min": -83.68521118164062,
      "activations/layer6_attention_weight_max": 55.947959899902344,
      "activations/layer6_attention_weight_min": -53.02591323852539,
      "activations/layer7_attention_weight_max": 74.48417663574219,
      "activations/layer7_attention_weight_min": -73.75712585449219,
      "activations/layer8_attention_weight_max": 37.24521255493164,
      "activations/layer8_attention_weight_min": -36.563541412353516,
      "activations/layer9_attention_weight_max": 38.64023208618164,
      "activations/layer9_attention_weight_min": -33.5713005065918,
      "epoch": 3.92,
      "learning_rate": 0.00012595454545454544,
      "loss": 2.8752,
      "step": 67550
    },
    {
      "activations/layer0_attention_weight_max": 15.712732315063477,
      "activations/layer0_attention_weight_min": -10.99278736114502,
      "activations/layer10_attention_weight_max": 35.686614990234375,
      "activations/layer10_attention_weight_min": -34.311702728271484,
      "activations/layer11_attention_weight_max": 31.992229461669922,
      "activations/layer11_attention_weight_min": -26.679550170898438,
      "activations/layer12_attention_weight_max": 42.48532485961914,
      "activations/layer12_attention_weight_min": -27.71269989013672,
      "activations/layer13_attention_weight_max": 39.984344482421875,
      "activations/layer13_attention_weight_min": -33.01371765136719,
      "activations/layer14_attention_weight_max": 58.497406005859375,
      "activations/layer14_attention_weight_min": -48.69466018676758,
      "activations/layer15_attention_weight_max": 44.10273361206055,
      "activations/layer15_attention_weight_min": -28.361757278442383,
      "activations/layer16_attention_weight_max": 37.219913482666016,
      "activations/layer16_attention_weight_min": -30.18170166015625,
      "activations/layer17_attention_weight_max": 38.35586166381836,
      "activations/layer17_attention_weight_min": -25.070798873901367,
      "activations/layer18_attention_weight_max": 32.84465789794922,
      "activations/layer18_attention_weight_min": -18.882122039794922,
      "activations/layer19_attention_weight_max": 41.830684661865234,
      "activations/layer19_attention_weight_min": -27.12998390197754,
      "activations/layer1_attention_weight_max": 16.452659606933594,
      "activations/layer1_attention_weight_min": -14.32292366027832,
      "activations/layer20_attention_weight_max": 30.998178482055664,
      "activations/layer20_attention_weight_min": -21.540470123291016,
      "activations/layer21_attention_weight_max": 36.12229537963867,
      "activations/layer21_attention_weight_min": -19.788705825805664,
      "activations/layer22_attention_weight_max": 52.57853317260742,
      "activations/layer22_attention_weight_min": -26.428796768188477,
      "activations/layer23_attention_weight_max": 35.83570098876953,
      "activations/layer23_attention_weight_min": -17.16791534423828,
      "activations/layer2_attention_weight_max": 33.022705078125,
      "activations/layer2_attention_weight_min": -30.017959594726562,
      "activations/layer3_attention_weight_max": 85.17752075195312,
      "activations/layer3_attention_weight_min": -85.7405776977539,
      "activations/layer4_attention_weight_max": 130.4087371826172,
      "activations/layer4_attention_weight_min": -126.4052505493164,
      "activations/layer5_attention_weight_max": 89.72001647949219,
      "activations/layer5_attention_weight_min": -85.48516082763672,
      "activations/layer6_attention_weight_max": 57.02178955078125,
      "activations/layer6_attention_weight_min": -58.08894729614258,
      "activations/layer7_attention_weight_max": 74.4450454711914,
      "activations/layer7_attention_weight_min": -74.10045623779297,
      "activations/layer8_attention_weight_max": 37.4568977355957,
      "activations/layer8_attention_weight_min": -38.750911712646484,
      "activations/layer9_attention_weight_max": 40.18968200683594,
      "activations/layer9_attention_weight_min": -35.62014389038086,
      "epoch": 3.93,
      "learning_rate": 0.00012593560606060603,
      "loss": 2.8685,
      "step": 67600
    },
    {
      "activations/layer0_attention_weight_max": 16.660749435424805,
      "activations/layer0_attention_weight_min": -11.219062805175781,
      "activations/layer10_attention_weight_max": 45.1397705078125,
      "activations/layer10_attention_weight_min": -31.49800682067871,
      "activations/layer11_attention_weight_max": 49.17538070678711,
      "activations/layer11_attention_weight_min": -32.03050231933594,
      "activations/layer12_attention_weight_max": 32.681827545166016,
      "activations/layer12_attention_weight_min": -28.611207962036133,
      "activations/layer13_attention_weight_max": 47.71118927001953,
      "activations/layer13_attention_weight_min": -32.60725402832031,
      "activations/layer14_attention_weight_max": 65.9568862915039,
      "activations/layer14_attention_weight_min": -48.464351654052734,
      "activations/layer15_attention_weight_max": 52.4577522277832,
      "activations/layer15_attention_weight_min": -35.84508514404297,
      "activations/layer16_attention_weight_max": 38.77351760864258,
      "activations/layer16_attention_weight_min": -27.409343719482422,
      "activations/layer17_attention_weight_max": 43.24419021606445,
      "activations/layer17_attention_weight_min": -26.523128509521484,
      "activations/layer18_attention_weight_max": 35.32123947143555,
      "activations/layer18_attention_weight_min": -20.77499008178711,
      "activations/layer19_attention_weight_max": 45.24921417236328,
      "activations/layer19_attention_weight_min": -30.30411720275879,
      "activations/layer1_attention_weight_max": 17.183204650878906,
      "activations/layer1_attention_weight_min": -13.451811790466309,
      "activations/layer20_attention_weight_max": 41.29020309448242,
      "activations/layer20_attention_weight_min": -22.539384841918945,
      "activations/layer21_attention_weight_max": 32.93220520019531,
      "activations/layer21_attention_weight_min": -20.486053466796875,
      "activations/layer22_attention_weight_max": 50.63433074951172,
      "activations/layer22_attention_weight_min": -29.469806671142578,
      "activations/layer23_attention_weight_max": 34.9372673034668,
      "activations/layer23_attention_weight_min": -17.371849060058594,
      "activations/layer2_attention_weight_max": 32.85875701904297,
      "activations/layer2_attention_weight_min": -31.859783172607422,
      "activations/layer3_attention_weight_max": 88.17827606201172,
      "activations/layer3_attention_weight_min": -91.02759552001953,
      "activations/layer4_attention_weight_max": 149.4748992919922,
      "activations/layer4_attention_weight_min": -136.83468627929688,
      "activations/layer5_attention_weight_max": 107.5924301147461,
      "activations/layer5_attention_weight_min": -91.93251037597656,
      "activations/layer6_attention_weight_max": 66.89326477050781,
      "activations/layer6_attention_weight_min": -57.368370056152344,
      "activations/layer7_attention_weight_max": 102.30232238769531,
      "activations/layer7_attention_weight_min": -81.48064422607422,
      "activations/layer8_attention_weight_max": 50.31200408935547,
      "activations/layer8_attention_weight_min": -42.774566650390625,
      "activations/layer9_attention_weight_max": 53.700714111328125,
      "activations/layer9_attention_weight_min": -44.40517807006836,
      "epoch": 3.93,
      "learning_rate": 0.00012591666666666665,
      "loss": 2.8831,
      "step": 67650
    },
    {
      "activations/layer0_attention_weight_max": 16.842235565185547,
      "activations/layer0_attention_weight_min": -12.062289237976074,
      "activations/layer10_attention_weight_max": 36.143638610839844,
      "activations/layer10_attention_weight_min": -31.07424545288086,
      "activations/layer11_attention_weight_max": 31.849342346191406,
      "activations/layer11_attention_weight_min": -28.920835494995117,
      "activations/layer12_attention_weight_max": 25.837085723876953,
      "activations/layer12_attention_weight_min": -28.857030868530273,
      "activations/layer13_attention_weight_max": 51.229915618896484,
      "activations/layer13_attention_weight_min": -33.73257827758789,
      "activations/layer14_attention_weight_max": 77.86369323730469,
      "activations/layer14_attention_weight_min": -49.20186996459961,
      "activations/layer15_attention_weight_max": 46.55088806152344,
      "activations/layer15_attention_weight_min": -30.39423942565918,
      "activations/layer16_attention_weight_max": 43.5015869140625,
      "activations/layer16_attention_weight_min": -28.934829711914062,
      "activations/layer17_attention_weight_max": 40.39271545410156,
      "activations/layer17_attention_weight_min": -26.052017211914062,
      "activations/layer18_attention_weight_max": 33.13597106933594,
      "activations/layer18_attention_weight_min": -18.826202392578125,
      "activations/layer19_attention_weight_max": 47.1618766784668,
      "activations/layer19_attention_weight_min": -29.264774322509766,
      "activations/layer1_attention_weight_max": 17.948762893676758,
      "activations/layer1_attention_weight_min": -14.980345726013184,
      "activations/layer20_attention_weight_max": 35.9765510559082,
      "activations/layer20_attention_weight_min": -23.964406967163086,
      "activations/layer21_attention_weight_max": 34.893707275390625,
      "activations/layer21_attention_weight_min": -18.99676513671875,
      "activations/layer22_attention_weight_max": 54.10218811035156,
      "activations/layer22_attention_weight_min": -26.951650619506836,
      "activations/layer23_attention_weight_max": 30.987991333007812,
      "activations/layer23_attention_weight_min": -21.763263702392578,
      "activations/layer2_attention_weight_max": 33.57788848876953,
      "activations/layer2_attention_weight_min": -30.96404266357422,
      "activations/layer3_attention_weight_max": 86.2555160522461,
      "activations/layer3_attention_weight_min": -89.53350830078125,
      "activations/layer4_attention_weight_max": 134.59661865234375,
      "activations/layer4_attention_weight_min": -130.98098754882812,
      "activations/layer5_attention_weight_max": 93.35884094238281,
      "activations/layer5_attention_weight_min": -83.13565063476562,
      "activations/layer6_attention_weight_max": 56.12419891357422,
      "activations/layer6_attention_weight_min": -56.02268600463867,
      "activations/layer7_attention_weight_max": 74.98271942138672,
      "activations/layer7_attention_weight_min": -75.17668151855469,
      "activations/layer8_attention_weight_max": 38.90996170043945,
      "activations/layer8_attention_weight_min": -37.42152786254883,
      "activations/layer9_attention_weight_max": 37.21432113647461,
      "activations/layer9_attention_weight_min": -34.40980529785156,
      "epoch": 3.93,
      "learning_rate": 0.00012589772727272726,
      "loss": 2.9006,
      "step": 67700
    },
    {
      "activations/layer0_attention_weight_max": 16.540536880493164,
      "activations/layer0_attention_weight_min": -12.594575881958008,
      "activations/layer10_attention_weight_max": 42.27860641479492,
      "activations/layer10_attention_weight_min": -30.940940856933594,
      "activations/layer11_attention_weight_max": 35.59770202636719,
      "activations/layer11_attention_weight_min": -27.010040283203125,
      "activations/layer12_attention_weight_max": 40.72025680541992,
      "activations/layer12_attention_weight_min": -29.121137619018555,
      "activations/layer13_attention_weight_max": 57.75912857055664,
      "activations/layer13_attention_weight_min": -33.330055236816406,
      "activations/layer14_attention_weight_max": 87.11544799804688,
      "activations/layer14_attention_weight_min": -48.87222671508789,
      "activations/layer15_attention_weight_max": 53.68819046020508,
      "activations/layer15_attention_weight_min": -27.94432830810547,
      "activations/layer16_attention_weight_max": 55.98719787597656,
      "activations/layer16_attention_weight_min": -28.670743942260742,
      "activations/layer17_attention_weight_max": 54.19580078125,
      "activations/layer17_attention_weight_min": -24.315410614013672,
      "activations/layer18_attention_weight_max": 48.66926193237305,
      "activations/layer18_attention_weight_min": -19.7371883392334,
      "activations/layer19_attention_weight_max": 56.93307113647461,
      "activations/layer19_attention_weight_min": -29.367212295532227,
      "activations/layer1_attention_weight_max": 17.662843704223633,
      "activations/layer1_attention_weight_min": -14.924376487731934,
      "activations/layer20_attention_weight_max": 42.49899673461914,
      "activations/layer20_attention_weight_min": -23.10882568359375,
      "activations/layer21_attention_weight_max": 44.51530075073242,
      "activations/layer21_attention_weight_min": -21.033090591430664,
      "activations/layer22_attention_weight_max": 74.74781036376953,
      "activations/layer22_attention_weight_min": -27.838850021362305,
      "activations/layer23_attention_weight_max": 38.03054428100586,
      "activations/layer23_attention_weight_min": -18.687519073486328,
      "activations/layer2_attention_weight_max": 33.625675201416016,
      "activations/layer2_attention_weight_min": -31.02861785888672,
      "activations/layer3_attention_weight_max": 87.86683654785156,
      "activations/layer3_attention_weight_min": -86.39005279541016,
      "activations/layer4_attention_weight_max": 129.93551635742188,
      "activations/layer4_attention_weight_min": -128.4900360107422,
      "activations/layer5_attention_weight_max": 91.58869934082031,
      "activations/layer5_attention_weight_min": -85.01609802246094,
      "activations/layer6_attention_weight_max": 57.381744384765625,
      "activations/layer6_attention_weight_min": -58.23505783081055,
      "activations/layer7_attention_weight_max": 81.7900619506836,
      "activations/layer7_attention_weight_min": -78.88046264648438,
      "activations/layer8_attention_weight_max": 40.184146881103516,
      "activations/layer8_attention_weight_min": -39.222740173339844,
      "activations/layer9_attention_weight_max": 40.07180404663086,
      "activations/layer9_attention_weight_min": -33.49430847167969,
      "epoch": 3.94,
      "learning_rate": 0.00012587878787878785,
      "loss": 2.8956,
      "step": 67750
    },
    {
      "activations/layer0_attention_weight_max": 16.255664825439453,
      "activations/layer0_attention_weight_min": -11.399870872497559,
      "activations/layer10_attention_weight_max": 35.3836555480957,
      "activations/layer10_attention_weight_min": -29.73113250732422,
      "activations/layer11_attention_weight_max": 31.626789093017578,
      "activations/layer11_attention_weight_min": -25.484333038330078,
      "activations/layer12_attention_weight_max": 28.214876174926758,
      "activations/layer12_attention_weight_min": -29.189165115356445,
      "activations/layer13_attention_weight_max": 45.28612518310547,
      "activations/layer13_attention_weight_min": -33.88941192626953,
      "activations/layer14_attention_weight_max": 56.72163009643555,
      "activations/layer14_attention_weight_min": -44.11498260498047,
      "activations/layer15_attention_weight_max": 40.93101501464844,
      "activations/layer15_attention_weight_min": -28.560321807861328,
      "activations/layer16_attention_weight_max": 39.69599914550781,
      "activations/layer16_attention_weight_min": -28.10001564025879,
      "activations/layer17_attention_weight_max": 39.11579895019531,
      "activations/layer17_attention_weight_min": -24.17680549621582,
      "activations/layer18_attention_weight_max": 34.38606643676758,
      "activations/layer18_attention_weight_min": -20.391321182250977,
      "activations/layer19_attention_weight_max": 42.259193420410156,
      "activations/layer19_attention_weight_min": -27.239561080932617,
      "activations/layer1_attention_weight_max": 16.394664764404297,
      "activations/layer1_attention_weight_min": -12.755823135375977,
      "activations/layer20_attention_weight_max": 33.319820404052734,
      "activations/layer20_attention_weight_min": -21.472904205322266,
      "activations/layer21_attention_weight_max": 33.857845306396484,
      "activations/layer21_attention_weight_min": -19.90907096862793,
      "activations/layer22_attention_weight_max": 51.15946578979492,
      "activations/layer22_attention_weight_min": -28.31698226928711,
      "activations/layer23_attention_weight_max": 30.988136291503906,
      "activations/layer23_attention_weight_min": -18.364099502563477,
      "activations/layer2_attention_weight_max": 31.371021270751953,
      "activations/layer2_attention_weight_min": -32.91242599487305,
      "activations/layer3_attention_weight_max": 82.79874420166016,
      "activations/layer3_attention_weight_min": -89.1182861328125,
      "activations/layer4_attention_weight_max": 123.29422760009766,
      "activations/layer4_attention_weight_min": -124.73103332519531,
      "activations/layer5_attention_weight_max": 83.99948120117188,
      "activations/layer5_attention_weight_min": -83.52594757080078,
      "activations/layer6_attention_weight_max": 52.96470642089844,
      "activations/layer6_attention_weight_min": -55.536949157714844,
      "activations/layer7_attention_weight_max": 72.28842163085938,
      "activations/layer7_attention_weight_min": -74.70359802246094,
      "activations/layer8_attention_weight_max": 36.2364501953125,
      "activations/layer8_attention_weight_min": -40.88578796386719,
      "activations/layer9_attention_weight_max": 35.074310302734375,
      "activations/layer9_attention_weight_min": -32.29081344604492,
      "epoch": 3.94,
      "learning_rate": 0.00012585984848484846,
      "loss": 2.8591,
      "step": 67800
    },
    {
      "activations/layer0_attention_weight_max": 16.736501693725586,
      "activations/layer0_attention_weight_min": -11.115245819091797,
      "activations/layer10_attention_weight_max": 40.31000518798828,
      "activations/layer10_attention_weight_min": -33.23976135253906,
      "activations/layer11_attention_weight_max": 45.09741973876953,
      "activations/layer11_attention_weight_min": -30.394184112548828,
      "activations/layer12_attention_weight_max": 43.75700759887695,
      "activations/layer12_attention_weight_min": -31.46338653564453,
      "activations/layer13_attention_weight_max": 64.79885864257812,
      "activations/layer13_attention_weight_min": -39.59253692626953,
      "activations/layer14_attention_weight_max": 84.07546997070312,
      "activations/layer14_attention_weight_min": -53.059478759765625,
      "activations/layer15_attention_weight_max": 57.76810073852539,
      "activations/layer15_attention_weight_min": -34.91401672363281,
      "activations/layer16_attention_weight_max": 49.217262268066406,
      "activations/layer16_attention_weight_min": -28.554336547851562,
      "activations/layer17_attention_weight_max": 41.668636322021484,
      "activations/layer17_attention_weight_min": -25.015682220458984,
      "activations/layer18_attention_weight_max": 33.98210525512695,
      "activations/layer18_attention_weight_min": -19.268083572387695,
      "activations/layer19_attention_weight_max": 48.40751647949219,
      "activations/layer19_attention_weight_min": -29.10225486755371,
      "activations/layer1_attention_weight_max": 16.789714813232422,
      "activations/layer1_attention_weight_min": -13.99664306640625,
      "activations/layer20_attention_weight_max": 42.231441497802734,
      "activations/layer20_attention_weight_min": -23.10914421081543,
      "activations/layer21_attention_weight_max": 31.554718017578125,
      "activations/layer21_attention_weight_min": -19.830669403076172,
      "activations/layer22_attention_weight_max": 53.16481018066406,
      "activations/layer22_attention_weight_min": -28.47614288330078,
      "activations/layer23_attention_weight_max": 32.6255989074707,
      "activations/layer23_attention_weight_min": -18.7293701171875,
      "activations/layer2_attention_weight_max": 34.79248046875,
      "activations/layer2_attention_weight_min": -32.35516357421875,
      "activations/layer3_attention_weight_max": 96.7687759399414,
      "activations/layer3_attention_weight_min": -96.20365905761719,
      "activations/layer4_attention_weight_max": 159.86949157714844,
      "activations/layer4_attention_weight_min": -144.947021484375,
      "activations/layer5_attention_weight_max": 105.57794189453125,
      "activations/layer5_attention_weight_min": -89.31575012207031,
      "activations/layer6_attention_weight_max": 66.43232727050781,
      "activations/layer6_attention_weight_min": -57.47909927368164,
      "activations/layer7_attention_weight_max": 90.195068359375,
      "activations/layer7_attention_weight_min": -79.93926239013672,
      "activations/layer8_attention_weight_max": 45.47169876098633,
      "activations/layer8_attention_weight_min": -39.93468475341797,
      "activations/layer9_attention_weight_max": 43.24036407470703,
      "activations/layer9_attention_weight_min": -38.27379608154297,
      "epoch": 3.94,
      "learning_rate": 0.00012584090909090908,
      "loss": 2.8674,
      "step": 67850
    },
    {
      "activations/layer0_attention_weight_max": 17.307048797607422,
      "activations/layer0_attention_weight_min": -10.830206871032715,
      "activations/layer10_attention_weight_max": 34.58103561401367,
      "activations/layer10_attention_weight_min": -29.800371170043945,
      "activations/layer11_attention_weight_max": 31.881431579589844,
      "activations/layer11_attention_weight_min": -27.151538848876953,
      "activations/layer12_attention_weight_max": 29.837400436401367,
      "activations/layer12_attention_weight_min": -27.74262809753418,
      "activations/layer13_attention_weight_max": 47.13553237915039,
      "activations/layer13_attention_weight_min": -36.09735107421875,
      "activations/layer14_attention_weight_max": 63.564544677734375,
      "activations/layer14_attention_weight_min": -53.625892639160156,
      "activations/layer15_attention_weight_max": 42.73768997192383,
      "activations/layer15_attention_weight_min": -28.091440200805664,
      "activations/layer16_attention_weight_max": 39.022056579589844,
      "activations/layer16_attention_weight_min": -28.12176513671875,
      "activations/layer17_attention_weight_max": 37.67543411254883,
      "activations/layer17_attention_weight_min": -24.145090103149414,
      "activations/layer18_attention_weight_max": 30.159330368041992,
      "activations/layer18_attention_weight_min": -20.08893394470215,
      "activations/layer19_attention_weight_max": 38.228729248046875,
      "activations/layer19_attention_weight_min": -29.406055450439453,
      "activations/layer1_attention_weight_max": 16.697959899902344,
      "activations/layer1_attention_weight_min": -13.885462760925293,
      "activations/layer20_attention_weight_max": 29.491369247436523,
      "activations/layer20_attention_weight_min": -22.13174057006836,
      "activations/layer21_attention_weight_max": 29.62709617614746,
      "activations/layer21_attention_weight_min": -22.64995765686035,
      "activations/layer22_attention_weight_max": 45.61906051635742,
      "activations/layer22_attention_weight_min": -26.19344711303711,
      "activations/layer23_attention_weight_max": 27.78759002685547,
      "activations/layer23_attention_weight_min": -17.603639602661133,
      "activations/layer2_attention_weight_max": 31.40506362915039,
      "activations/layer2_attention_weight_min": -31.971975326538086,
      "activations/layer3_attention_weight_max": 83.7216796875,
      "activations/layer3_attention_weight_min": -85.98532104492188,
      "activations/layer4_attention_weight_max": 128.4927215576172,
      "activations/layer4_attention_weight_min": -130.48880004882812,
      "activations/layer5_attention_weight_max": 86.1036376953125,
      "activations/layer5_attention_weight_min": -84.84361267089844,
      "activations/layer6_attention_weight_max": 54.829341888427734,
      "activations/layer6_attention_weight_min": -53.60139846801758,
      "activations/layer7_attention_weight_max": 73.80606842041016,
      "activations/layer7_attention_weight_min": -73.55777740478516,
      "activations/layer8_attention_weight_max": 36.50297927856445,
      "activations/layer8_attention_weight_min": -36.47832489013672,
      "activations/layer9_attention_weight_max": 36.39656448364258,
      "activations/layer9_attention_weight_min": -36.427608489990234,
      "epoch": 3.95,
      "learning_rate": 0.0001258219696969697,
      "loss": 2.8846,
      "step": 67900
    },
    {
      "activations/layer0_attention_weight_max": 15.771380424499512,
      "activations/layer0_attention_weight_min": -12.469645500183105,
      "activations/layer10_attention_weight_max": 44.3469352722168,
      "activations/layer10_attention_weight_min": -35.91024398803711,
      "activations/layer11_attention_weight_max": 39.809635162353516,
      "activations/layer11_attention_weight_min": -31.211767196655273,
      "activations/layer12_attention_weight_max": 38.59394073486328,
      "activations/layer12_attention_weight_min": -24.858535766601562,
      "activations/layer13_attention_weight_max": 70.31841278076172,
      "activations/layer13_attention_weight_min": -45.96626281738281,
      "activations/layer14_attention_weight_max": 71.25257873535156,
      "activations/layer14_attention_weight_min": -47.8101806640625,
      "activations/layer15_attention_weight_max": 46.606689453125,
      "activations/layer15_attention_weight_min": -31.866907119750977,
      "activations/layer16_attention_weight_max": 42.852569580078125,
      "activations/layer16_attention_weight_min": -29.787281036376953,
      "activations/layer17_attention_weight_max": 48.09279251098633,
      "activations/layer17_attention_weight_min": -24.47215461730957,
      "activations/layer18_attention_weight_max": 35.20293426513672,
      "activations/layer18_attention_weight_min": -18.842409133911133,
      "activations/layer19_attention_weight_max": 47.24343490600586,
      "activations/layer19_attention_weight_min": -29.971948623657227,
      "activations/layer1_attention_weight_max": 19.33045196533203,
      "activations/layer1_attention_weight_min": -15.232170104980469,
      "activations/layer20_attention_weight_max": 38.11341094970703,
      "activations/layer20_attention_weight_min": -22.90138053894043,
      "activations/layer21_attention_weight_max": 37.90431594848633,
      "activations/layer21_attention_weight_min": -19.71107292175293,
      "activations/layer22_attention_weight_max": 62.6033935546875,
      "activations/layer22_attention_weight_min": -26.31040382385254,
      "activations/layer23_attention_weight_max": 33.49595260620117,
      "activations/layer23_attention_weight_min": -19.017616271972656,
      "activations/layer2_attention_weight_max": 33.80548858642578,
      "activations/layer2_attention_weight_min": -32.255409240722656,
      "activations/layer3_attention_weight_max": 90.31129455566406,
      "activations/layer3_attention_weight_min": -95.53619384765625,
      "activations/layer4_attention_weight_max": 138.1659698486328,
      "activations/layer4_attention_weight_min": -131.8341522216797,
      "activations/layer5_attention_weight_max": 92.80471801757812,
      "activations/layer5_attention_weight_min": -91.14629364013672,
      "activations/layer6_attention_weight_max": 58.6226806640625,
      "activations/layer6_attention_weight_min": -62.43217468261719,
      "activations/layer7_attention_weight_max": 86.59324645996094,
      "activations/layer7_attention_weight_min": -84.30412292480469,
      "activations/layer8_attention_weight_max": 44.232418060302734,
      "activations/layer8_attention_weight_min": -43.257633209228516,
      "activations/layer9_attention_weight_max": 43.65745162963867,
      "activations/layer9_attention_weight_min": -36.835052490234375,
      "epoch": 3.95,
      "learning_rate": 0.00012580303030303028,
      "loss": 2.8768,
      "step": 67950
    },
    {
      "activations/layer0_attention_weight_max": 16.79421615600586,
      "activations/layer0_attention_weight_min": -12.019375801086426,
      "activations/layer10_attention_weight_max": 38.45891189575195,
      "activations/layer10_attention_weight_min": -28.742904663085938,
      "activations/layer11_attention_weight_max": 36.386695861816406,
      "activations/layer11_attention_weight_min": -25.237781524658203,
      "activations/layer12_attention_weight_max": 32.615901947021484,
      "activations/layer12_attention_weight_min": -25.904115676879883,
      "activations/layer13_attention_weight_max": 53.75910186767578,
      "activations/layer13_attention_weight_min": -35.18315887451172,
      "activations/layer14_attention_weight_max": 71.464599609375,
      "activations/layer14_attention_weight_min": -45.53857421875,
      "activations/layer15_attention_weight_max": 44.4113655090332,
      "activations/layer15_attention_weight_min": -29.902523040771484,
      "activations/layer16_attention_weight_max": 42.303524017333984,
      "activations/layer16_attention_weight_min": -32.269046783447266,
      "activations/layer17_attention_weight_max": 43.77848434448242,
      "activations/layer17_attention_weight_min": -26.57025146484375,
      "activations/layer18_attention_weight_max": 32.30133819580078,
      "activations/layer18_attention_weight_min": -21.80182647705078,
      "activations/layer19_attention_weight_max": 44.149234771728516,
      "activations/layer19_attention_weight_min": -29.36643409729004,
      "activations/layer1_attention_weight_max": 16.818618774414062,
      "activations/layer1_attention_weight_min": -13.31696605682373,
      "activations/layer20_attention_weight_max": 33.65971374511719,
      "activations/layer20_attention_weight_min": -23.443838119506836,
      "activations/layer21_attention_weight_max": 34.24117660522461,
      "activations/layer21_attention_weight_min": -23.192867279052734,
      "activations/layer22_attention_weight_max": 53.89017868041992,
      "activations/layer22_attention_weight_min": -27.679672241210938,
      "activations/layer23_attention_weight_max": 31.348623275756836,
      "activations/layer23_attention_weight_min": -18.610767364501953,
      "activations/layer2_attention_weight_max": 31.986942291259766,
      "activations/layer2_attention_weight_min": -31.419652938842773,
      "activations/layer3_attention_weight_max": 87.6834716796875,
      "activations/layer3_attention_weight_min": -88.23191833496094,
      "activations/layer4_attention_weight_max": 131.8998260498047,
      "activations/layer4_attention_weight_min": -131.93283081054688,
      "activations/layer5_attention_weight_max": 93.00395202636719,
      "activations/layer5_attention_weight_min": -88.77361297607422,
      "activations/layer6_attention_weight_max": 56.884891510009766,
      "activations/layer6_attention_weight_min": -55.897308349609375,
      "activations/layer7_attention_weight_max": 74.35520935058594,
      "activations/layer7_attention_weight_min": -79.05043029785156,
      "activations/layer8_attention_weight_max": 37.49211120605469,
      "activations/layer8_attention_weight_min": -37.803531646728516,
      "activations/layer9_attention_weight_max": 40.24704360961914,
      "activations/layer9_attention_weight_min": -32.9941520690918,
      "epoch": 3.95,
      "learning_rate": 0.0001257840909090909,
      "loss": 2.8731,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4909,
      "eval_samples_per_second": 505.719,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4909,
      "eval_openwebtext_samples_per_second": 505.719,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_wikitext_loss": 3.0546875,
      "eval_wikitext_ppl": 21.214554942517648,
      "eval_wikitext_runtime": 1.9657,
      "eval_wikitext_samples_per_second": 231.977,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_lambada_loss": 2.833984375,
      "eval_lambada_ppl": 17.013112580497378,
      "eval_lambada_runtime": 9.5643,
      "eval_lambada_samples_per_second": 509.081,
      "step": 68000
    },
    {
      "activations/layer0_attention_weight_max": 16.942209243774414,
      "activations/layer0_attention_weight_min": -11.177803039550781,
      "activations/layer10_attention_weight_max": 36.19441604614258,
      "activations/layer10_attention_weight_min": -31.449682235717773,
      "activations/layer11_attention_weight_max": 35.28581237792969,
      "activations/layer11_attention_weight_min": -28.16647720336914,
      "activations/layer12_attention_weight_max": 33.79873275756836,
      "activations/layer12_attention_weight_min": -30.519832611083984,
      "activations/layer13_attention_weight_max": 48.082828521728516,
      "activations/layer13_attention_weight_min": -31.698965072631836,
      "activations/layer14_attention_weight_max": 65.7671890258789,
      "activations/layer14_attention_weight_min": -47.84529113769531,
      "activations/layer15_attention_weight_max": 43.332454681396484,
      "activations/layer15_attention_weight_min": -28.712169647216797,
      "activations/layer16_attention_weight_max": 47.3939323425293,
      "activations/layer16_attention_weight_min": -28.903139114379883,
      "activations/layer17_attention_weight_max": 47.2711067199707,
      "activations/layer17_attention_weight_min": -24.984819412231445,
      "activations/layer18_attention_weight_max": 32.773902893066406,
      "activations/layer18_attention_weight_min": -20.21748924255371,
      "activations/layer19_attention_weight_max": 45.00102615356445,
      "activations/layer19_attention_weight_min": -27.550601959228516,
      "activations/layer1_attention_weight_max": 16.52618408203125,
      "activations/layer1_attention_weight_min": -13.820055961608887,
      "activations/layer20_attention_weight_max": 35.42138671875,
      "activations/layer20_attention_weight_min": -22.688222885131836,
      "activations/layer21_attention_weight_max": 32.81679153442383,
      "activations/layer21_attention_weight_min": -19.222087860107422,
      "activations/layer22_attention_weight_max": 49.0926628112793,
      "activations/layer22_attention_weight_min": -30.622676849365234,
      "activations/layer23_attention_weight_max": 29.94208335876465,
      "activations/layer23_attention_weight_min": -18.342041015625,
      "activations/layer2_attention_weight_max": 31.399181365966797,
      "activations/layer2_attention_weight_min": -30.523662567138672,
      "activations/layer3_attention_weight_max": 86.42715454101562,
      "activations/layer3_attention_weight_min": -88.41471862792969,
      "activations/layer4_attention_weight_max": 132.373779296875,
      "activations/layer4_attention_weight_min": -126.15882873535156,
      "activations/layer5_attention_weight_max": 91.67430877685547,
      "activations/layer5_attention_weight_min": -86.11100769042969,
      "activations/layer6_attention_weight_max": 58.45661544799805,
      "activations/layer6_attention_weight_min": -55.977943420410156,
      "activations/layer7_attention_weight_max": 79.4609603881836,
      "activations/layer7_attention_weight_min": -74.3956527709961,
      "activations/layer8_attention_weight_max": 41.656429290771484,
      "activations/layer8_attention_weight_min": -39.12095642089844,
      "activations/layer9_attention_weight_max": 39.69950485229492,
      "activations/layer9_attention_weight_min": -35.75144577026367,
      "epoch": 3.95,
      "learning_rate": 0.0001257651515151515,
      "loss": 2.8885,
      "step": 68050
    },
    {
      "activations/layer0_attention_weight_max": 16.929723739624023,
      "activations/layer0_attention_weight_min": -11.614537239074707,
      "activations/layer10_attention_weight_max": 32.1077766418457,
      "activations/layer10_attention_weight_min": -28.21719741821289,
      "activations/layer11_attention_weight_max": 31.3269100189209,
      "activations/layer11_attention_weight_min": -26.191343307495117,
      "activations/layer12_attention_weight_max": 28.618709564208984,
      "activations/layer12_attention_weight_min": -30.443233489990234,
      "activations/layer13_attention_weight_max": 41.60615539550781,
      "activations/layer13_attention_weight_min": -32.15441131591797,
      "activations/layer14_attention_weight_max": 71.28807067871094,
      "activations/layer14_attention_weight_min": -50.520057678222656,
      "activations/layer15_attention_weight_max": 46.0733642578125,
      "activations/layer15_attention_weight_min": -27.289291381835938,
      "activations/layer16_attention_weight_max": 40.455692291259766,
      "activations/layer16_attention_weight_min": -28.0804386138916,
      "activations/layer17_attention_weight_max": 48.45231246948242,
      "activations/layer17_attention_weight_min": -23.607227325439453,
      "activations/layer18_attention_weight_max": 33.62450408935547,
      "activations/layer18_attention_weight_min": -17.949281692504883,
      "activations/layer19_attention_weight_max": 39.79811096191406,
      "activations/layer19_attention_weight_min": -26.828702926635742,
      "activations/layer1_attention_weight_max": 17.73111915588379,
      "activations/layer1_attention_weight_min": -13.890754699707031,
      "activations/layer20_attention_weight_max": 35.33079528808594,
      "activations/layer20_attention_weight_min": -23.165599822998047,
      "activations/layer21_attention_weight_max": 35.62046432495117,
      "activations/layer21_attention_weight_min": -19.470998764038086,
      "activations/layer22_attention_weight_max": 47.98215866088867,
      "activations/layer22_attention_weight_min": -25.353668212890625,
      "activations/layer23_attention_weight_max": 28.029003143310547,
      "activations/layer23_attention_weight_min": -18.572864532470703,
      "activations/layer2_attention_weight_max": 30.353269577026367,
      "activations/layer2_attention_weight_min": -29.821142196655273,
      "activations/layer3_attention_weight_max": 81.66897583007812,
      "activations/layer3_attention_weight_min": -83.70934295654297,
      "activations/layer4_attention_weight_max": 128.8330535888672,
      "activations/layer4_attention_weight_min": -122.53607177734375,
      "activations/layer5_attention_weight_max": 90.73100280761719,
      "activations/layer5_attention_weight_min": -80.74746704101562,
      "activations/layer6_attention_weight_max": 58.45336151123047,
      "activations/layer6_attention_weight_min": -54.98615646362305,
      "activations/layer7_attention_weight_max": 78.07427215576172,
      "activations/layer7_attention_weight_min": -72.24375915527344,
      "activations/layer8_attention_weight_max": 38.71551513671875,
      "activations/layer8_attention_weight_min": -37.13589859008789,
      "activations/layer9_attention_weight_max": 41.848628997802734,
      "activations/layer9_attention_weight_min": -34.30349349975586,
      "epoch": 3.96,
      "learning_rate": 0.00012574621212121213,
      "loss": 2.8471,
      "step": 68100
    },
    {
      "activations/layer0_attention_weight_max": 15.87886905670166,
      "activations/layer0_attention_weight_min": -10.9699125289917,
      "activations/layer10_attention_weight_max": 40.137306213378906,
      "activations/layer10_attention_weight_min": -31.105894088745117,
      "activations/layer11_attention_weight_max": 40.05934524536133,
      "activations/layer11_attention_weight_min": -31.992704391479492,
      "activations/layer12_attention_weight_max": 29.77035140991211,
      "activations/layer12_attention_weight_min": -29.578447341918945,
      "activations/layer13_attention_weight_max": 52.0377311706543,
      "activations/layer13_attention_weight_min": -32.96997833251953,
      "activations/layer14_attention_weight_max": 74.34724426269531,
      "activations/layer14_attention_weight_min": -46.7236328125,
      "activations/layer15_attention_weight_max": 39.748477935791016,
      "activations/layer15_attention_weight_min": -28.978078842163086,
      "activations/layer16_attention_weight_max": 49.74424362182617,
      "activations/layer16_attention_weight_min": -29.53057098388672,
      "activations/layer17_attention_weight_max": 37.78592300415039,
      "activations/layer17_attention_weight_min": -24.99956703186035,
      "activations/layer18_attention_weight_max": 32.11707305908203,
      "activations/layer18_attention_weight_min": -20.15955352783203,
      "activations/layer19_attention_weight_max": 48.837982177734375,
      "activations/layer19_attention_weight_min": -33.00788497924805,
      "activations/layer1_attention_weight_max": 17.77853775024414,
      "activations/layer1_attention_weight_min": -14.904064178466797,
      "activations/layer20_attention_weight_max": 33.05122756958008,
      "activations/layer20_attention_weight_min": -24.20084571838379,
      "activations/layer21_attention_weight_max": 31.204368591308594,
      "activations/layer21_attention_weight_min": -20.38395881652832,
      "activations/layer22_attention_weight_max": 52.121742248535156,
      "activations/layer22_attention_weight_min": -27.783702850341797,
      "activations/layer23_attention_weight_max": 33.87788391113281,
      "activations/layer23_attention_weight_min": -19.083324432373047,
      "activations/layer2_attention_weight_max": 32.29999542236328,
      "activations/layer2_attention_weight_min": -33.80075454711914,
      "activations/layer3_attention_weight_max": 85.72130584716797,
      "activations/layer3_attention_weight_min": -86.8180160522461,
      "activations/layer4_attention_weight_max": 130.74964904785156,
      "activations/layer4_attention_weight_min": -129.10752868652344,
      "activations/layer5_attention_weight_max": 87.38407897949219,
      "activations/layer5_attention_weight_min": -84.3351821899414,
      "activations/layer6_attention_weight_max": 57.1719856262207,
      "activations/layer6_attention_weight_min": -55.99616622924805,
      "activations/layer7_attention_weight_max": 77.13392639160156,
      "activations/layer7_attention_weight_min": -77.41500091552734,
      "activations/layer8_attention_weight_max": 37.89204025268555,
      "activations/layer8_attention_weight_min": -37.47187805175781,
      "activations/layer9_attention_weight_max": 41.3032341003418,
      "activations/layer9_attention_weight_min": -36.10910415649414,
      "epoch": 3.96,
      "learning_rate": 0.00012572727272727272,
      "loss": 2.8771,
      "step": 68150
    },
    {
      "activations/layer0_attention_weight_max": 14.80860710144043,
      "activations/layer0_attention_weight_min": -12.325935363769531,
      "activations/layer10_attention_weight_max": 35.38496017456055,
      "activations/layer10_attention_weight_min": -30.459012985229492,
      "activations/layer11_attention_weight_max": 31.810733795166016,
      "activations/layer11_attention_weight_min": -26.078514099121094,
      "activations/layer12_attention_weight_max": 27.048912048339844,
      "activations/layer12_attention_weight_min": -30.937776565551758,
      "activations/layer13_attention_weight_max": 46.387813568115234,
      "activations/layer13_attention_weight_min": -32.74528121948242,
      "activations/layer14_attention_weight_max": 67.37812805175781,
      "activations/layer14_attention_weight_min": -54.303619384765625,
      "activations/layer15_attention_weight_max": 43.61201095581055,
      "activations/layer15_attention_weight_min": -30.02475929260254,
      "activations/layer16_attention_weight_max": 43.606197357177734,
      "activations/layer16_attention_weight_min": -31.82384490966797,
      "activations/layer17_attention_weight_max": 46.31454086303711,
      "activations/layer17_attention_weight_min": -25.797134399414062,
      "activations/layer18_attention_weight_max": 33.62432861328125,
      "activations/layer18_attention_weight_min": -19.339065551757812,
      "activations/layer19_attention_weight_max": 43.821964263916016,
      "activations/layer19_attention_weight_min": -29.58009147644043,
      "activations/layer1_attention_weight_max": 17.322973251342773,
      "activations/layer1_attention_weight_min": -15.863578796386719,
      "activations/layer20_attention_weight_max": 32.94016647338867,
      "activations/layer20_attention_weight_min": -22.851238250732422,
      "activations/layer21_attention_weight_max": 36.81726837158203,
      "activations/layer21_attention_weight_min": -21.623878479003906,
      "activations/layer22_attention_weight_max": 56.4487190246582,
      "activations/layer22_attention_weight_min": -29.421092987060547,
      "activations/layer23_attention_weight_max": 33.85792541503906,
      "activations/layer23_attention_weight_min": -21.079547882080078,
      "activations/layer2_attention_weight_max": 31.06891441345215,
      "activations/layer2_attention_weight_min": -30.45783233642578,
      "activations/layer3_attention_weight_max": 83.99246215820312,
      "activations/layer3_attention_weight_min": -87.45806121826172,
      "activations/layer4_attention_weight_max": 133.94093322753906,
      "activations/layer4_attention_weight_min": -128.6868133544922,
      "activations/layer5_attention_weight_max": 90.26611328125,
      "activations/layer5_attention_weight_min": -86.3512191772461,
      "activations/layer6_attention_weight_max": 58.502197265625,
      "activations/layer6_attention_weight_min": -54.94451904296875,
      "activations/layer7_attention_weight_max": 80.01852416992188,
      "activations/layer7_attention_weight_min": -77.063720703125,
      "activations/layer8_attention_weight_max": 40.979610443115234,
      "activations/layer8_attention_weight_min": -37.46176528930664,
      "activations/layer9_attention_weight_max": 41.17985534667969,
      "activations/layer9_attention_weight_min": -36.05263900756836,
      "epoch": 3.96,
      "learning_rate": 0.00012570833333333333,
      "loss": 2.8829,
      "step": 68200
    },
    {
      "activations/layer0_attention_weight_max": 15.829294204711914,
      "activations/layer0_attention_weight_min": -10.519126892089844,
      "activations/layer10_attention_weight_max": 50.21245193481445,
      "activations/layer10_attention_weight_min": -34.62561798095703,
      "activations/layer11_attention_weight_max": 51.37822723388672,
      "activations/layer11_attention_weight_min": -32.81183624267578,
      "activations/layer12_attention_weight_max": 71.52726745605469,
      "activations/layer12_attention_weight_min": -31.11286735534668,
      "activations/layer13_attention_weight_max": 86.88484191894531,
      "activations/layer13_attention_weight_min": -42.094505310058594,
      "activations/layer14_attention_weight_max": 88.10105895996094,
      "activations/layer14_attention_weight_min": -53.73512649536133,
      "activations/layer15_attention_weight_max": 73.84298706054688,
      "activations/layer15_attention_weight_min": -39.25175094604492,
      "activations/layer16_attention_weight_max": 43.705806732177734,
      "activations/layer16_attention_weight_min": -27.755144119262695,
      "activations/layer17_attention_weight_max": 42.56406021118164,
      "activations/layer17_attention_weight_min": -24.63229751586914,
      "activations/layer18_attention_weight_max": 32.81435775756836,
      "activations/layer18_attention_weight_min": -21.405628204345703,
      "activations/layer19_attention_weight_max": 46.45047378540039,
      "activations/layer19_attention_weight_min": -26.08725929260254,
      "activations/layer1_attention_weight_max": 17.028766632080078,
      "activations/layer1_attention_weight_min": -13.618430137634277,
      "activations/layer20_attention_weight_max": 36.39982604980469,
      "activations/layer20_attention_weight_min": -23.188840866088867,
      "activations/layer21_attention_weight_max": 31.714574813842773,
      "activations/layer21_attention_weight_min": -18.40018653869629,
      "activations/layer22_attention_weight_max": 47.930824279785156,
      "activations/layer22_attention_weight_min": -26.562744140625,
      "activations/layer23_attention_weight_max": 27.722707748413086,
      "activations/layer23_attention_weight_min": -18.369895935058594,
      "activations/layer2_attention_weight_max": 35.16893768310547,
      "activations/layer2_attention_weight_min": -32.10863494873047,
      "activations/layer3_attention_weight_max": 94.76624298095703,
      "activations/layer3_attention_weight_min": -93.31414794921875,
      "activations/layer4_attention_weight_max": 154.039306640625,
      "activations/layer4_attention_weight_min": -130.7646942138672,
      "activations/layer5_attention_weight_max": 102.93818664550781,
      "activations/layer5_attention_weight_min": -87.0550765991211,
      "activations/layer6_attention_weight_max": 66.60565948486328,
      "activations/layer6_attention_weight_min": -54.28789138793945,
      "activations/layer7_attention_weight_max": 100.01954650878906,
      "activations/layer7_attention_weight_min": -81.79730987548828,
      "activations/layer8_attention_weight_max": 51.8485107421875,
      "activations/layer8_attention_weight_min": -43.55961608886719,
      "activations/layer9_attention_weight_max": 52.163570404052734,
      "activations/layer9_attention_weight_min": -40.60811233520508,
      "epoch": 3.97,
      "learning_rate": 0.00012568939393939395,
      "loss": 2.8794,
      "step": 68250
    },
    {
      "activations/layer0_attention_weight_max": 16.943050384521484,
      "activations/layer0_attention_weight_min": -11.621219635009766,
      "activations/layer10_attention_weight_max": 36.29775619506836,
      "activations/layer10_attention_weight_min": -31.06169319152832,
      "activations/layer11_attention_weight_max": 36.49791717529297,
      "activations/layer11_attention_weight_min": -29.624120712280273,
      "activations/layer12_attention_weight_max": 46.050086975097656,
      "activations/layer12_attention_weight_min": -30.09955406188965,
      "activations/layer13_attention_weight_max": 60.0693244934082,
      "activations/layer13_attention_weight_min": -42.790504455566406,
      "activations/layer14_attention_weight_max": 72.38247680664062,
      "activations/layer14_attention_weight_min": -51.965999603271484,
      "activations/layer15_attention_weight_max": 42.169036865234375,
      "activations/layer15_attention_weight_min": -30.883705139160156,
      "activations/layer16_attention_weight_max": 38.481529235839844,
      "activations/layer16_attention_weight_min": -28.65488624572754,
      "activations/layer17_attention_weight_max": 39.7021484375,
      "activations/layer17_attention_weight_min": -25.2227840423584,
      "activations/layer18_attention_weight_max": 33.41758728027344,
      "activations/layer18_attention_weight_min": -20.288528442382812,
      "activations/layer19_attention_weight_max": 43.057411193847656,
      "activations/layer19_attention_weight_min": -28.81089210510254,
      "activations/layer1_attention_weight_max": 18.324464797973633,
      "activations/layer1_attention_weight_min": -14.735363960266113,
      "activations/layer20_attention_weight_max": 29.94862937927246,
      "activations/layer20_attention_weight_min": -24.04000473022461,
      "activations/layer21_attention_weight_max": 27.680021286010742,
      "activations/layer21_attention_weight_min": -19.436058044433594,
      "activations/layer22_attention_weight_max": 44.062339782714844,
      "activations/layer22_attention_weight_min": -28.204309463500977,
      "activations/layer23_attention_weight_max": 29.398717880249023,
      "activations/layer23_attention_weight_min": -18.053741455078125,
      "activations/layer2_attention_weight_max": 32.97275924682617,
      "activations/layer2_attention_weight_min": -31.909526824951172,
      "activations/layer3_attention_weight_max": 86.92778778076172,
      "activations/layer3_attention_weight_min": -88.4427719116211,
      "activations/layer4_attention_weight_max": 134.85829162597656,
      "activations/layer4_attention_weight_min": -135.54745483398438,
      "activations/layer5_attention_weight_max": 89.31188201904297,
      "activations/layer5_attention_weight_min": -88.12889862060547,
      "activations/layer6_attention_weight_max": 57.366127014160156,
      "activations/layer6_attention_weight_min": -57.07377624511719,
      "activations/layer7_attention_weight_max": 80.18882751464844,
      "activations/layer7_attention_weight_min": -82.05804443359375,
      "activations/layer8_attention_weight_max": 39.441436767578125,
      "activations/layer8_attention_weight_min": -37.47587966918945,
      "activations/layer9_attention_weight_max": 39.9299430847168,
      "activations/layer9_attention_weight_min": -35.45230484008789,
      "epoch": 3.97,
      "learning_rate": 0.00012567045454545453,
      "loss": 2.8727,
      "step": 68300
    },
    {
      "activations/layer0_attention_weight_max": 16.793926239013672,
      "activations/layer0_attention_weight_min": -11.499124526977539,
      "activations/layer10_attention_weight_max": 39.52935028076172,
      "activations/layer10_attention_weight_min": -31.92865562438965,
      "activations/layer11_attention_weight_max": 40.65874481201172,
      "activations/layer11_attention_weight_min": -29.846317291259766,
      "activations/layer12_attention_weight_max": 68.85478210449219,
      "activations/layer12_attention_weight_min": -28.477577209472656,
      "activations/layer13_attention_weight_max": 64.30736541748047,
      "activations/layer13_attention_weight_min": -35.02656173706055,
      "activations/layer14_attention_weight_max": 81.09237670898438,
      "activations/layer14_attention_weight_min": -49.45429229736328,
      "activations/layer15_attention_weight_max": 54.97536087036133,
      "activations/layer15_attention_weight_min": -29.475908279418945,
      "activations/layer16_attention_weight_max": 46.560325622558594,
      "activations/layer16_attention_weight_min": -29.13089942932129,
      "activations/layer17_attention_weight_max": 46.53834915161133,
      "activations/layer17_attention_weight_min": -23.752283096313477,
      "activations/layer18_attention_weight_max": 39.291351318359375,
      "activations/layer18_attention_weight_min": -21.147680282592773,
      "activations/layer19_attention_weight_max": 49.20698928833008,
      "activations/layer19_attention_weight_min": -28.245145797729492,
      "activations/layer1_attention_weight_max": 17.278091430664062,
      "activations/layer1_attention_weight_min": -14.736101150512695,
      "activations/layer20_attention_weight_max": 39.50965881347656,
      "activations/layer20_attention_weight_min": -23.63524055480957,
      "activations/layer21_attention_weight_max": 36.89302062988281,
      "activations/layer21_attention_weight_min": -22.853378295898438,
      "activations/layer22_attention_weight_max": 58.364131927490234,
      "activations/layer22_attention_weight_min": -26.550582885742188,
      "activations/layer23_attention_weight_max": 33.18512725830078,
      "activations/layer23_attention_weight_min": -16.116073608398438,
      "activations/layer2_attention_weight_max": 31.782934188842773,
      "activations/layer2_attention_weight_min": -30.539321899414062,
      "activations/layer3_attention_weight_max": 88.10808563232422,
      "activations/layer3_attention_weight_min": -91.3587875366211,
      "activations/layer4_attention_weight_max": 137.1945037841797,
      "activations/layer4_attention_weight_min": -130.86070251464844,
      "activations/layer5_attention_weight_max": 96.16317749023438,
      "activations/layer5_attention_weight_min": -85.71917724609375,
      "activations/layer6_attention_weight_max": 59.6906852722168,
      "activations/layer6_attention_weight_min": -57.28363800048828,
      "activations/layer7_attention_weight_max": 85.69095611572266,
      "activations/layer7_attention_weight_min": -81.24352264404297,
      "activations/layer8_attention_weight_max": 42.22316360473633,
      "activations/layer8_attention_weight_min": -40.79753494262695,
      "activations/layer9_attention_weight_max": 41.10519027709961,
      "activations/layer9_attention_weight_min": -36.02737808227539,
      "epoch": 3.97,
      "learning_rate": 0.00012565151515151515,
      "loss": 2.8883,
      "step": 68350
    },
    {
      "activations/layer0_attention_weight_max": 17.42281150817871,
      "activations/layer0_attention_weight_min": -11.979286193847656,
      "activations/layer10_attention_weight_max": 36.72627639770508,
      "activations/layer10_attention_weight_min": -31.28725242614746,
      "activations/layer11_attention_weight_max": 37.11676788330078,
      "activations/layer11_attention_weight_min": -27.67349624633789,
      "activations/layer12_attention_weight_max": 30.362625122070312,
      "activations/layer12_attention_weight_min": -36.123260498046875,
      "activations/layer13_attention_weight_max": 42.662559509277344,
      "activations/layer13_attention_weight_min": -34.78160858154297,
      "activations/layer14_attention_weight_max": 63.40203857421875,
      "activations/layer14_attention_weight_min": -49.36469268798828,
      "activations/layer15_attention_weight_max": 40.6184196472168,
      "activations/layer15_attention_weight_min": -27.400957107543945,
      "activations/layer16_attention_weight_max": 37.844573974609375,
      "activations/layer16_attention_weight_min": -28.96963119506836,
      "activations/layer17_attention_weight_max": 42.400997161865234,
      "activations/layer17_attention_weight_min": -24.96295166015625,
      "activations/layer18_attention_weight_max": 31.611356735229492,
      "activations/layer18_attention_weight_min": -19.179719924926758,
      "activations/layer19_attention_weight_max": 40.726356506347656,
      "activations/layer19_attention_weight_min": -28.518991470336914,
      "activations/layer1_attention_weight_max": 17.71272850036621,
      "activations/layer1_attention_weight_min": -13.938136100769043,
      "activations/layer20_attention_weight_max": 34.53569793701172,
      "activations/layer20_attention_weight_min": -21.820079803466797,
      "activations/layer21_attention_weight_max": 33.877891540527344,
      "activations/layer21_attention_weight_min": -19.661298751831055,
      "activations/layer22_attention_weight_max": 51.108055114746094,
      "activations/layer22_attention_weight_min": -25.678470611572266,
      "activations/layer23_attention_weight_max": 28.871910095214844,
      "activations/layer23_attention_weight_min": -18.557842254638672,
      "activations/layer2_attention_weight_max": 32.068992614746094,
      "activations/layer2_attention_weight_min": -31.250411987304688,
      "activations/layer3_attention_weight_max": 89.87360382080078,
      "activations/layer3_attention_weight_min": -90.76332092285156,
      "activations/layer4_attention_weight_max": 138.63232421875,
      "activations/layer4_attention_weight_min": -137.7608184814453,
      "activations/layer5_attention_weight_max": 95.86476135253906,
      "activations/layer5_attention_weight_min": -91.02717590332031,
      "activations/layer6_attention_weight_max": 62.323673248291016,
      "activations/layer6_attention_weight_min": -58.39615249633789,
      "activations/layer7_attention_weight_max": 85.77252197265625,
      "activations/layer7_attention_weight_min": -80.67572784423828,
      "activations/layer8_attention_weight_max": 40.38698196411133,
      "activations/layer8_attention_weight_min": -38.44779968261719,
      "activations/layer9_attention_weight_max": 40.13923263549805,
      "activations/layer9_attention_weight_min": -38.0743522644043,
      "epoch": 3.97,
      "learning_rate": 0.00012563257575757576,
      "loss": 2.8777,
      "step": 68400
    },
    {
      "activations/layer0_attention_weight_max": 15.749832153320312,
      "activations/layer0_attention_weight_min": -11.887855529785156,
      "activations/layer10_attention_weight_max": 36.68861389160156,
      "activations/layer10_attention_weight_min": -31.287307739257812,
      "activations/layer11_attention_weight_max": 34.19121170043945,
      "activations/layer11_attention_weight_min": -27.184593200683594,
      "activations/layer12_attention_weight_max": 32.63531494140625,
      "activations/layer12_attention_weight_min": -26.09090805053711,
      "activations/layer13_attention_weight_max": 46.8891716003418,
      "activations/layer13_attention_weight_min": -34.35749816894531,
      "activations/layer14_attention_weight_max": 76.42101287841797,
      "activations/layer14_attention_weight_min": -53.7058219909668,
      "activations/layer15_attention_weight_max": 40.65929412841797,
      "activations/layer15_attention_weight_min": -28.27524757385254,
      "activations/layer16_attention_weight_max": 39.89745330810547,
      "activations/layer16_attention_weight_min": -26.98012924194336,
      "activations/layer17_attention_weight_max": 40.80771255493164,
      "activations/layer17_attention_weight_min": -24.49663543701172,
      "activations/layer18_attention_weight_max": 32.74726486206055,
      "activations/layer18_attention_weight_min": -19.77066993713379,
      "activations/layer19_attention_weight_max": 41.60136413574219,
      "activations/layer19_attention_weight_min": -28.242618560791016,
      "activations/layer1_attention_weight_max": 16.825119018554688,
      "activations/layer1_attention_weight_min": -14.390536308288574,
      "activations/layer20_attention_weight_max": 34.239585876464844,
      "activations/layer20_attention_weight_min": -23.981304168701172,
      "activations/layer21_attention_weight_max": 34.32733917236328,
      "activations/layer21_attention_weight_min": -20.312702178955078,
      "activations/layer22_attention_weight_max": 50.61210250854492,
      "activations/layer22_attention_weight_min": -27.76181983947754,
      "activations/layer23_attention_weight_max": 32.32449722290039,
      "activations/layer23_attention_weight_min": -17.753299713134766,
      "activations/layer2_attention_weight_max": 31.426982879638672,
      "activations/layer2_attention_weight_min": -30.310779571533203,
      "activations/layer3_attention_weight_max": 86.79216003417969,
      "activations/layer3_attention_weight_min": -81.80714416503906,
      "activations/layer4_attention_weight_max": 132.4278564453125,
      "activations/layer4_attention_weight_min": -123.92280578613281,
      "activations/layer5_attention_weight_max": 89.20291137695312,
      "activations/layer5_attention_weight_min": -84.16012573242188,
      "activations/layer6_attention_weight_max": 57.71011734008789,
      "activations/layer6_attention_weight_min": -59.01387405395508,
      "activations/layer7_attention_weight_max": 80.88480377197266,
      "activations/layer7_attention_weight_min": -76.61392974853516,
      "activations/layer8_attention_weight_max": 44.30759048461914,
      "activations/layer8_attention_weight_min": -40.5562629699707,
      "activations/layer9_attention_weight_max": 39.57607650756836,
      "activations/layer9_attention_weight_min": -33.15910339355469,
      "epoch": 3.98,
      "learning_rate": 0.00012561363636363635,
      "loss": 2.8832,
      "step": 68450
    },
    {
      "activations/layer0_attention_weight_max": 17.309762954711914,
      "activations/layer0_attention_weight_min": -12.00603199005127,
      "activations/layer10_attention_weight_max": 34.803306579589844,
      "activations/layer10_attention_weight_min": -30.70808982849121,
      "activations/layer11_attention_weight_max": 34.97998809814453,
      "activations/layer11_attention_weight_min": -28.801626205444336,
      "activations/layer12_attention_weight_max": 27.170291900634766,
      "activations/layer12_attention_weight_min": -25.83879280090332,
      "activations/layer13_attention_weight_max": 44.60374069213867,
      "activations/layer13_attention_weight_min": -31.571277618408203,
      "activations/layer14_attention_weight_max": 66.3909683227539,
      "activations/layer14_attention_weight_min": -44.756805419921875,
      "activations/layer15_attention_weight_max": 44.10380172729492,
      "activations/layer15_attention_weight_min": -29.951324462890625,
      "activations/layer16_attention_weight_max": 39.07052230834961,
      "activations/layer16_attention_weight_min": -28.837934494018555,
      "activations/layer17_attention_weight_max": 41.324893951416016,
      "activations/layer17_attention_weight_min": -26.15458869934082,
      "activations/layer18_attention_weight_max": 34.49089431762695,
      "activations/layer18_attention_weight_min": -19.28607749938965,
      "activations/layer19_attention_weight_max": 43.375545501708984,
      "activations/layer19_attention_weight_min": -29.25946617126465,
      "activations/layer1_attention_weight_max": 17.137073516845703,
      "activations/layer1_attention_weight_min": -16.03215980529785,
      "activations/layer20_attention_weight_max": 34.922157287597656,
      "activations/layer20_attention_weight_min": -22.514429092407227,
      "activations/layer21_attention_weight_max": 38.952789306640625,
      "activations/layer21_attention_weight_min": -20.022113800048828,
      "activations/layer22_attention_weight_max": 58.31597900390625,
      "activations/layer22_attention_weight_min": -28.07265853881836,
      "activations/layer23_attention_weight_max": 32.79947280883789,
      "activations/layer23_attention_weight_min": -16.486114501953125,
      "activations/layer2_attention_weight_max": 30.704099655151367,
      "activations/layer2_attention_weight_min": -30.092628479003906,
      "activations/layer3_attention_weight_max": 88.35954284667969,
      "activations/layer3_attention_weight_min": -85.57764434814453,
      "activations/layer4_attention_weight_max": 127.82160949707031,
      "activations/layer4_attention_weight_min": -124.46229553222656,
      "activations/layer5_attention_weight_max": 87.25405883789062,
      "activations/layer5_attention_weight_min": -80.84976196289062,
      "activations/layer6_attention_weight_max": 56.60963439941406,
      "activations/layer6_attention_weight_min": -53.97701644897461,
      "activations/layer7_attention_weight_max": 77.22057342529297,
      "activations/layer7_attention_weight_min": -76.0110092163086,
      "activations/layer8_attention_weight_max": 39.52744674682617,
      "activations/layer8_attention_weight_min": -36.80105972290039,
      "activations/layer9_attention_weight_max": 36.69707489013672,
      "activations/layer9_attention_weight_min": -34.768455505371094,
      "epoch": 3.98,
      "learning_rate": 0.00012559469696969697,
      "loss": 2.87,
      "step": 68500
    },
    {
      "activations/layer0_attention_weight_max": 16.506608963012695,
      "activations/layer0_attention_weight_min": -11.641616821289062,
      "activations/layer10_attention_weight_max": 42.85307312011719,
      "activations/layer10_attention_weight_min": -31.738773345947266,
      "activations/layer11_attention_weight_max": 43.83293151855469,
      "activations/layer11_attention_weight_min": -31.569923400878906,
      "activations/layer12_attention_weight_max": 35.83879470825195,
      "activations/layer12_attention_weight_min": -32.23276901245117,
      "activations/layer13_attention_weight_max": 77.63533020019531,
      "activations/layer13_attention_weight_min": -43.7731819152832,
      "activations/layer14_attention_weight_max": 105.13996887207031,
      "activations/layer14_attention_weight_min": -59.5125732421875,
      "activations/layer15_attention_weight_max": 55.99812316894531,
      "activations/layer15_attention_weight_min": -28.827880859375,
      "activations/layer16_attention_weight_max": 51.508846282958984,
      "activations/layer16_attention_weight_min": -29.009641647338867,
      "activations/layer17_attention_weight_max": 53.74938201904297,
      "activations/layer17_attention_weight_min": -24.171228408813477,
      "activations/layer18_attention_weight_max": 40.249237060546875,
      "activations/layer18_attention_weight_min": -20.397993087768555,
      "activations/layer19_attention_weight_max": 56.839508056640625,
      "activations/layer19_attention_weight_min": -30.306982040405273,
      "activations/layer1_attention_weight_max": 18.160789489746094,
      "activations/layer1_attention_weight_min": -13.737231254577637,
      "activations/layer20_attention_weight_max": 43.64425277709961,
      "activations/layer20_attention_weight_min": -23.541217803955078,
      "activations/layer21_attention_weight_max": 36.76325225830078,
      "activations/layer21_attention_weight_min": -20.276782989501953,
      "activations/layer22_attention_weight_max": 63.70254898071289,
      "activations/layer22_attention_weight_min": -27.51154899597168,
      "activations/layer23_attention_weight_max": 41.693477630615234,
      "activations/layer23_attention_weight_min": -20.4139404296875,
      "activations/layer2_attention_weight_max": 33.51602554321289,
      "activations/layer2_attention_weight_min": -31.068769454956055,
      "activations/layer3_attention_weight_max": 88.93912506103516,
      "activations/layer3_attention_weight_min": -85.16177368164062,
      "activations/layer4_attention_weight_max": 135.23544311523438,
      "activations/layer4_attention_weight_min": -127.21056365966797,
      "activations/layer5_attention_weight_max": 92.64418029785156,
      "activations/layer5_attention_weight_min": -81.803955078125,
      "activations/layer6_attention_weight_max": 57.93076705932617,
      "activations/layer6_attention_weight_min": -54.470558166503906,
      "activations/layer7_attention_weight_max": 79.91004180908203,
      "activations/layer7_attention_weight_min": -80.80693817138672,
      "activations/layer8_attention_weight_max": 42.412540435791016,
      "activations/layer8_attention_weight_min": -39.96784973144531,
      "activations/layer9_attention_weight_max": 43.75175094604492,
      "activations/layer9_attention_weight_min": -32.76026153564453,
      "epoch": 3.98,
      "learning_rate": 0.00012557575757575755,
      "loss": 2.8668,
      "step": 68550
    },
    {
      "activations/layer0_attention_weight_max": 16.759380340576172,
      "activations/layer0_attention_weight_min": -11.698630332946777,
      "activations/layer10_attention_weight_max": 36.28036880493164,
      "activations/layer10_attention_weight_min": -31.485252380371094,
      "activations/layer11_attention_weight_max": 33.61445999145508,
      "activations/layer11_attention_weight_min": -30.272747039794922,
      "activations/layer12_attention_weight_max": 25.916427612304688,
      "activations/layer12_attention_weight_min": -24.461286544799805,
      "activations/layer13_attention_weight_max": 48.258338928222656,
      "activations/layer13_attention_weight_min": -32.558998107910156,
      "activations/layer14_attention_weight_max": 69.28209686279297,
      "activations/layer14_attention_weight_min": -50.60429763793945,
      "activations/layer15_attention_weight_max": 41.48681640625,
      "activations/layer15_attention_weight_min": -28.058866500854492,
      "activations/layer16_attention_weight_max": 39.77591323852539,
      "activations/layer16_attention_weight_min": -28.25590705871582,
      "activations/layer17_attention_weight_max": 40.70402908325195,
      "activations/layer17_attention_weight_min": -25.094697952270508,
      "activations/layer18_attention_weight_max": 28.034183502197266,
      "activations/layer18_attention_weight_min": -21.5913143157959,
      "activations/layer19_attention_weight_max": 44.286502838134766,
      "activations/layer19_attention_weight_min": -28.791934967041016,
      "activations/layer1_attention_weight_max": 17.947423934936523,
      "activations/layer1_attention_weight_min": -16.347946166992188,
      "activations/layer20_attention_weight_max": 33.343467712402344,
      "activations/layer20_attention_weight_min": -22.019115447998047,
      "activations/layer21_attention_weight_max": 27.267044067382812,
      "activations/layer21_attention_weight_min": -19.590316772460938,
      "activations/layer22_attention_weight_max": 51.602508544921875,
      "activations/layer22_attention_weight_min": -27.40355110168457,
      "activations/layer23_attention_weight_max": 29.847768783569336,
      "activations/layer23_attention_weight_min": -18.60321807861328,
      "activations/layer2_attention_weight_max": 34.70193862915039,
      "activations/layer2_attention_weight_min": -32.106502532958984,
      "activations/layer3_attention_weight_max": 90.70128631591797,
      "activations/layer3_attention_weight_min": -89.23497009277344,
      "activations/layer4_attention_weight_max": 137.8255157470703,
      "activations/layer4_attention_weight_min": -130.14901733398438,
      "activations/layer5_attention_weight_max": 90.76849365234375,
      "activations/layer5_attention_weight_min": -83.15912628173828,
      "activations/layer6_attention_weight_max": 61.1995964050293,
      "activations/layer6_attention_weight_min": -58.62890625,
      "activations/layer7_attention_weight_max": 84.25093841552734,
      "activations/layer7_attention_weight_min": -77.43392181396484,
      "activations/layer8_attention_weight_max": 41.88966751098633,
      "activations/layer8_attention_weight_min": -39.8259162902832,
      "activations/layer9_attention_weight_max": 38.956398010253906,
      "activations/layer9_attention_weight_min": -35.13359832763672,
      "epoch": 3.99,
      "learning_rate": 0.00012555681818181817,
      "loss": 2.8753,
      "step": 68600
    },
    {
      "activations/layer0_attention_weight_max": 15.084638595581055,
      "activations/layer0_attention_weight_min": -12.153335571289062,
      "activations/layer10_attention_weight_max": 39.30220031738281,
      "activations/layer10_attention_weight_min": -30.06789207458496,
      "activations/layer11_attention_weight_max": 36.511436462402344,
      "activations/layer11_attention_weight_min": -29.37173080444336,
      "activations/layer12_attention_weight_max": 48.75509262084961,
      "activations/layer12_attention_weight_min": -29.66646957397461,
      "activations/layer13_attention_weight_max": 70.65066528320312,
      "activations/layer13_attention_weight_min": -50.53351593017578,
      "activations/layer14_attention_weight_max": 73.8618392944336,
      "activations/layer14_attention_weight_min": -50.31978988647461,
      "activations/layer15_attention_weight_max": 51.9236946105957,
      "activations/layer15_attention_weight_min": -37.629886627197266,
      "activations/layer16_attention_weight_max": 44.56377029418945,
      "activations/layer16_attention_weight_min": -29.499216079711914,
      "activations/layer17_attention_weight_max": 47.636051177978516,
      "activations/layer17_attention_weight_min": -25.152681350708008,
      "activations/layer18_attention_weight_max": 36.81298828125,
      "activations/layer18_attention_weight_min": -19.649194717407227,
      "activations/layer19_attention_weight_max": 53.89784622192383,
      "activations/layer19_attention_weight_min": -30.828046798706055,
      "activations/layer1_attention_weight_max": 17.143169403076172,
      "activations/layer1_attention_weight_min": -14.608314514160156,
      "activations/layer20_attention_weight_max": 36.76578140258789,
      "activations/layer20_attention_weight_min": -27.09481430053711,
      "activations/layer21_attention_weight_max": 38.65082550048828,
      "activations/layer21_attention_weight_min": -21.083959579467773,
      "activations/layer22_attention_weight_max": 59.52210235595703,
      "activations/layer22_attention_weight_min": -27.44554328918457,
      "activations/layer23_attention_weight_max": 40.385093688964844,
      "activations/layer23_attention_weight_min": -17.31341552734375,
      "activations/layer2_attention_weight_max": 34.715545654296875,
      "activations/layer2_attention_weight_min": -31.588163375854492,
      "activations/layer3_attention_weight_max": 94.9842529296875,
      "activations/layer3_attention_weight_min": -90.65458679199219,
      "activations/layer4_attention_weight_max": 138.4026336669922,
      "activations/layer4_attention_weight_min": -130.19635009765625,
      "activations/layer5_attention_weight_max": 90.5355453491211,
      "activations/layer5_attention_weight_min": -84.09441375732422,
      "activations/layer6_attention_weight_max": 60.80840301513672,
      "activations/layer6_attention_weight_min": -58.12088394165039,
      "activations/layer7_attention_weight_max": 86.90606689453125,
      "activations/layer7_attention_weight_min": -78.11582946777344,
      "activations/layer8_attention_weight_max": 42.04048538208008,
      "activations/layer8_attention_weight_min": -41.02663803100586,
      "activations/layer9_attention_weight_max": 42.10447692871094,
      "activations/layer9_attention_weight_min": -40.0814323425293,
      "epoch": 3.99,
      "learning_rate": 0.00012553787878787878,
      "loss": 2.882,
      "step": 68650
    },
    {
      "activations/layer0_attention_weight_max": 17.980817794799805,
      "activations/layer0_attention_weight_min": -10.99262809753418,
      "activations/layer10_attention_weight_max": 36.622337341308594,
      "activations/layer10_attention_weight_min": -30.187162399291992,
      "activations/layer11_attention_weight_max": 33.364418029785156,
      "activations/layer11_attention_weight_min": -26.261030197143555,
      "activations/layer12_attention_weight_max": 28.946386337280273,
      "activations/layer12_attention_weight_min": -28.93673324584961,
      "activations/layer13_attention_weight_max": 48.010494232177734,
      "activations/layer13_attention_weight_min": -33.80481719970703,
      "activations/layer14_attention_weight_max": 85.07625579833984,
      "activations/layer14_attention_weight_min": -55.606693267822266,
      "activations/layer15_attention_weight_max": 43.862239837646484,
      "activations/layer15_attention_weight_min": -31.10220718383789,
      "activations/layer16_attention_weight_max": 40.19024658203125,
      "activations/layer16_attention_weight_min": -28.642656326293945,
      "activations/layer17_attention_weight_max": 40.300655364990234,
      "activations/layer17_attention_weight_min": -24.747486114501953,
      "activations/layer18_attention_weight_max": 31.893451690673828,
      "activations/layer18_attention_weight_min": -19.958635330200195,
      "activations/layer19_attention_weight_max": 44.18886947631836,
      "activations/layer19_attention_weight_min": -27.998361587524414,
      "activations/layer1_attention_weight_max": 17.176759719848633,
      "activations/layer1_attention_weight_min": -14.82429313659668,
      "activations/layer20_attention_weight_max": 34.50163269042969,
      "activations/layer20_attention_weight_min": -23.62977409362793,
      "activations/layer21_attention_weight_max": 36.159244537353516,
      "activations/layer21_attention_weight_min": -21.87021255493164,
      "activations/layer22_attention_weight_max": 48.9894905090332,
      "activations/layer22_attention_weight_min": -27.710948944091797,
      "activations/layer23_attention_weight_max": 30.40506362915039,
      "activations/layer23_attention_weight_min": -18.89695167541504,
      "activations/layer2_attention_weight_max": 31.536712646484375,
      "activations/layer2_attention_weight_min": -30.441680908203125,
      "activations/layer3_attention_weight_max": 86.2663803100586,
      "activations/layer3_attention_weight_min": -87.46585845947266,
      "activations/layer4_attention_weight_max": 132.9558868408203,
      "activations/layer4_attention_weight_min": -129.8478546142578,
      "activations/layer5_attention_weight_max": 98.27685546875,
      "activations/layer5_attention_weight_min": -83.50318908691406,
      "activations/layer6_attention_weight_max": 55.40665054321289,
      "activations/layer6_attention_weight_min": -56.92517852783203,
      "activations/layer7_attention_weight_max": 75.95735931396484,
      "activations/layer7_attention_weight_min": -72.33847045898438,
      "activations/layer8_attention_weight_max": 40.65798568725586,
      "activations/layer8_attention_weight_min": -38.705345153808594,
      "activations/layer9_attention_weight_max": 37.29129409790039,
      "activations/layer9_attention_weight_min": -31.862192153930664,
      "epoch": 3.99,
      "learning_rate": 0.00012551893939393937,
      "loss": 2.8769,
      "step": 68700
    },
    {
      "activations/layer0_attention_weight_max": 16.791915893554688,
      "activations/layer0_attention_weight_min": -13.229415893554688,
      "activations/layer10_attention_weight_max": 36.29671859741211,
      "activations/layer10_attention_weight_min": -31.23185157775879,
      "activations/layer11_attention_weight_max": 33.40974044799805,
      "activations/layer11_attention_weight_min": -30.15673065185547,
      "activations/layer12_attention_weight_max": 31.779191970825195,
      "activations/layer12_attention_weight_min": -25.89103126525879,
      "activations/layer13_attention_weight_max": 44.3744010925293,
      "activations/layer13_attention_weight_min": -34.67784881591797,
      "activations/layer14_attention_weight_max": 72.23209381103516,
      "activations/layer14_attention_weight_min": -49.69606018066406,
      "activations/layer15_attention_weight_max": 49.25865173339844,
      "activations/layer15_attention_weight_min": -30.685523986816406,
      "activations/layer16_attention_weight_max": 40.57072830200195,
      "activations/layer16_attention_weight_min": -29.995594024658203,
      "activations/layer17_attention_weight_max": 40.42371368408203,
      "activations/layer17_attention_weight_min": -24.70194435119629,
      "activations/layer18_attention_weight_max": 32.0401725769043,
      "activations/layer18_attention_weight_min": -19.285642623901367,
      "activations/layer19_attention_weight_max": 42.40709686279297,
      "activations/layer19_attention_weight_min": -29.136783599853516,
      "activations/layer1_attention_weight_max": 17.06608772277832,
      "activations/layer1_attention_weight_min": -15.04674243927002,
      "activations/layer20_attention_weight_max": 33.62343215942383,
      "activations/layer20_attention_weight_min": -21.739681243896484,
      "activations/layer21_attention_weight_max": 32.936492919921875,
      "activations/layer21_attention_weight_min": -18.531768798828125,
      "activations/layer22_attention_weight_max": 48.97373580932617,
      "activations/layer22_attention_weight_min": -25.68257713317871,
      "activations/layer23_attention_weight_max": 29.622272491455078,
      "activations/layer23_attention_weight_min": -17.762107849121094,
      "activations/layer2_attention_weight_max": 31.01698875427246,
      "activations/layer2_attention_weight_min": -31.020099639892578,
      "activations/layer3_attention_weight_max": 85.83260345458984,
      "activations/layer3_attention_weight_min": -85.68408966064453,
      "activations/layer4_attention_weight_max": 130.55747985839844,
      "activations/layer4_attention_weight_min": -125.99462127685547,
      "activations/layer5_attention_weight_max": 89.13687133789062,
      "activations/layer5_attention_weight_min": -81.35101318359375,
      "activations/layer6_attention_weight_max": 56.358036041259766,
      "activations/layer6_attention_weight_min": -55.0604133605957,
      "activations/layer7_attention_weight_max": 72.22492980957031,
      "activations/layer7_attention_weight_min": -74.3909683227539,
      "activations/layer8_attention_weight_max": 40.90128707885742,
      "activations/layer8_attention_weight_min": -39.22211837768555,
      "activations/layer9_attention_weight_max": 39.034671783447266,
      "activations/layer9_attention_weight_min": -35.10322189331055,
      "epoch": 3.99,
      "learning_rate": 0.0001255,
      "loss": 2.8787,
      "step": 68750
    },
    {
      "activations/layer0_attention_weight_max": 17.406356811523438,
      "activations/layer0_attention_weight_min": -11.966501235961914,
      "activations/layer10_attention_weight_max": 40.351783752441406,
      "activations/layer10_attention_weight_min": -31.997037887573242,
      "activations/layer11_attention_weight_max": 37.6475830078125,
      "activations/layer11_attention_weight_min": -30.12000846862793,
      "activations/layer12_attention_weight_max": 58.81135940551758,
      "activations/layer12_attention_weight_min": -26.02056121826172,
      "activations/layer13_attention_weight_max": 51.30976867675781,
      "activations/layer13_attention_weight_min": -33.49887466430664,
      "activations/layer14_attention_weight_max": 77.13626098632812,
      "activations/layer14_attention_weight_min": -50.16693878173828,
      "activations/layer15_attention_weight_max": 50.722068786621094,
      "activations/layer15_attention_weight_min": -29.00739860534668,
      "activations/layer16_attention_weight_max": 48.55541229248047,
      "activations/layer16_attention_weight_min": -28.48943519592285,
      "activations/layer17_attention_weight_max": 48.244537353515625,
      "activations/layer17_attention_weight_min": -25.8528995513916,
      "activations/layer18_attention_weight_max": 31.924875259399414,
      "activations/layer18_attention_weight_min": -19.71839714050293,
      "activations/layer19_attention_weight_max": 52.33659744262695,
      "activations/layer19_attention_weight_min": -29.032995223999023,
      "activations/layer1_attention_weight_max": 17.23275375366211,
      "activations/layer1_attention_weight_min": -16.74750518798828,
      "activations/layer20_attention_weight_max": 38.769317626953125,
      "activations/layer20_attention_weight_min": -22.874011993408203,
      "activations/layer21_attention_weight_max": 41.127227783203125,
      "activations/layer21_attention_weight_min": -22.39920425415039,
      "activations/layer22_attention_weight_max": 54.85993194580078,
      "activations/layer22_attention_weight_min": -34.63611602783203,
      "activations/layer23_attention_weight_max": 33.97514724731445,
      "activations/layer23_attention_weight_min": -22.800344467163086,
      "activations/layer2_attention_weight_max": 32.049896240234375,
      "activations/layer2_attention_weight_min": -32.025264739990234,
      "activations/layer3_attention_weight_max": 87.6278076171875,
      "activations/layer3_attention_weight_min": -86.57767486572266,
      "activations/layer4_attention_weight_max": 132.3448028564453,
      "activations/layer4_attention_weight_min": -128.35906982421875,
      "activations/layer5_attention_weight_max": 89.22785949707031,
      "activations/layer5_attention_weight_min": -85.15887451171875,
      "activations/layer6_attention_weight_max": 57.1653938293457,
      "activations/layer6_attention_weight_min": -59.24187088012695,
      "activations/layer7_attention_weight_max": 83.13220977783203,
      "activations/layer7_attention_weight_min": -78.52779388427734,
      "activations/layer8_attention_weight_max": 41.839630126953125,
      "activations/layer8_attention_weight_min": -39.86832046508789,
      "activations/layer9_attention_weight_max": 40.407379150390625,
      "activations/layer9_attention_weight_min": -34.819881439208984,
      "epoch": 4.0,
      "learning_rate": 0.0001254810606060606,
      "loss": 2.8878,
      "step": 68800
    },
    {
      "activations/layer0_attention_weight_max": 16.28163719177246,
      "activations/layer0_attention_weight_min": -12.716362953186035,
      "activations/layer10_attention_weight_max": 38.172508239746094,
      "activations/layer10_attention_weight_min": -29.133230209350586,
      "activations/layer11_attention_weight_max": 42.45716857910156,
      "activations/layer11_attention_weight_min": -29.480342864990234,
      "activations/layer12_attention_weight_max": 38.9690055847168,
      "activations/layer12_attention_weight_min": -35.54343032836914,
      "activations/layer13_attention_weight_max": 60.625762939453125,
      "activations/layer13_attention_weight_min": -40.58745574951172,
      "activations/layer14_attention_weight_max": 89.19747924804688,
      "activations/layer14_attention_weight_min": -51.36713790893555,
      "activations/layer15_attention_weight_max": 45.57481002807617,
      "activations/layer15_attention_weight_min": -28.351957321166992,
      "activations/layer16_attention_weight_max": 42.088382720947266,
      "activations/layer16_attention_weight_min": -29.517181396484375,
      "activations/layer17_attention_weight_max": 41.510475158691406,
      "activations/layer17_attention_weight_min": -25.09513282775879,
      "activations/layer18_attention_weight_max": 30.41119384765625,
      "activations/layer18_attention_weight_min": -19.021743774414062,
      "activations/layer19_attention_weight_max": 46.93739318847656,
      "activations/layer19_attention_weight_min": -29.05213737487793,
      "activations/layer1_attention_weight_max": 18.067737579345703,
      "activations/layer1_attention_weight_min": -14.044991493225098,
      "activations/layer20_attention_weight_max": 33.18540954589844,
      "activations/layer20_attention_weight_min": -23.421260833740234,
      "activations/layer21_attention_weight_max": 32.325897216796875,
      "activations/layer21_attention_weight_min": -20.689172744750977,
      "activations/layer22_attention_weight_max": 47.996742248535156,
      "activations/layer22_attention_weight_min": -26.831485748291016,
      "activations/layer23_attention_weight_max": 29.49359130859375,
      "activations/layer23_attention_weight_min": -19.384384155273438,
      "activations/layer2_attention_weight_max": 31.500072479248047,
      "activations/layer2_attention_weight_min": -30.253469467163086,
      "activations/layer3_attention_weight_max": 85.89389038085938,
      "activations/layer3_attention_weight_min": -85.74978637695312,
      "activations/layer4_attention_weight_max": 132.45834350585938,
      "activations/layer4_attention_weight_min": -122.15811920166016,
      "activations/layer5_attention_weight_max": 93.109130859375,
      "activations/layer5_attention_weight_min": -85.74313354492188,
      "activations/layer6_attention_weight_max": 63.5853385925293,
      "activations/layer6_attention_weight_min": -55.0064697265625,
      "activations/layer7_attention_weight_max": 78.57308197021484,
      "activations/layer7_attention_weight_min": -79.74495697021484,
      "activations/layer8_attention_weight_max": 43.29275894165039,
      "activations/layer8_attention_weight_min": -36.96107864379883,
      "activations/layer9_attention_weight_max": 46.890132904052734,
      "activations/layer9_attention_weight_min": -40.72370529174805,
      "epoch": 4.0,
      "learning_rate": 0.0001254621212121212,
      "loss": 2.9358,
      "step": 68850
    },
    {
      "activations/layer0_attention_weight_max": 15.861915588378906,
      "activations/layer0_attention_weight_min": -11.329306602478027,
      "activations/layer10_attention_weight_max": 40.73807907104492,
      "activations/layer10_attention_weight_min": -31.251873016357422,
      "activations/layer11_attention_weight_max": 39.00657272338867,
      "activations/layer11_attention_weight_min": -30.383319854736328,
      "activations/layer12_attention_weight_max": 40.09381103515625,
      "activations/layer12_attention_weight_min": -26.811052322387695,
      "activations/layer13_attention_weight_max": 59.177268981933594,
      "activations/layer13_attention_weight_min": -38.44974899291992,
      "activations/layer14_attention_weight_max": 73.7884750366211,
      "activations/layer14_attention_weight_min": -51.579158782958984,
      "activations/layer15_attention_weight_max": 42.600460052490234,
      "activations/layer15_attention_weight_min": -34.648799896240234,
      "activations/layer16_attention_weight_max": 38.785152435302734,
      "activations/layer16_attention_weight_min": -29.17650032043457,
      "activations/layer17_attention_weight_max": 45.05379104614258,
      "activations/layer17_attention_weight_min": -27.32300567626953,
      "activations/layer18_attention_weight_max": 30.76534652709961,
      "activations/layer18_attention_weight_min": -19.88698959350586,
      "activations/layer19_attention_weight_max": 45.86359405517578,
      "activations/layer19_attention_weight_min": -30.20857048034668,
      "activations/layer1_attention_weight_max": 17.568077087402344,
      "activations/layer1_attention_weight_min": -15.284505844116211,
      "activations/layer20_attention_weight_max": 33.608558654785156,
      "activations/layer20_attention_weight_min": -23.297677993774414,
      "activations/layer21_attention_weight_max": 31.29935073852539,
      "activations/layer21_attention_weight_min": -20.267982482910156,
      "activations/layer22_attention_weight_max": 55.896087646484375,
      "activations/layer22_attention_weight_min": -27.5777645111084,
      "activations/layer23_attention_weight_max": 31.473196029663086,
      "activations/layer23_attention_weight_min": -21.48099136352539,
      "activations/layer2_attention_weight_max": 32.13176727294922,
      "activations/layer2_attention_weight_min": -32.37959289550781,
      "activations/layer3_attention_weight_max": 88.205078125,
      "activations/layer3_attention_weight_min": -87.7052001953125,
      "activations/layer4_attention_weight_max": 137.50013732910156,
      "activations/layer4_attention_weight_min": -130.54942321777344,
      "activations/layer5_attention_weight_max": 98.54280090332031,
      "activations/layer5_attention_weight_min": -89.75192260742188,
      "activations/layer6_attention_weight_max": 60.58414077758789,
      "activations/layer6_attention_weight_min": -61.70165252685547,
      "activations/layer7_attention_weight_max": 79.347412109375,
      "activations/layer7_attention_weight_min": -81.25901794433594,
      "activations/layer8_attention_weight_max": 42.17424392700195,
      "activations/layer8_attention_weight_min": -41.73019027709961,
      "activations/layer9_attention_weight_max": 40.69145584106445,
      "activations/layer9_attention_weight_min": -37.04243469238281,
      "epoch": 4.0,
      "learning_rate": 0.0001254431818181818,
      "loss": 2.8644,
      "step": 68900
    },
    {
      "activations/layer0_attention_weight_max": 16.282930374145508,
      "activations/layer0_attention_weight_min": -11.218509674072266,
      "activations/layer10_attention_weight_max": 35.87987518310547,
      "activations/layer10_attention_weight_min": -30.631032943725586,
      "activations/layer11_attention_weight_max": 33.81380081176758,
      "activations/layer11_attention_weight_min": -26.392019271850586,
      "activations/layer12_attention_weight_max": 37.14278793334961,
      "activations/layer12_attention_weight_min": -27.140478134155273,
      "activations/layer13_attention_weight_max": 49.34644317626953,
      "activations/layer13_attention_weight_min": -33.022483825683594,
      "activations/layer14_attention_weight_max": 69.86746215820312,
      "activations/layer14_attention_weight_min": -50.851890563964844,
      "activations/layer15_attention_weight_max": 42.830928802490234,
      "activations/layer15_attention_weight_min": -28.67395782470703,
      "activations/layer16_attention_weight_max": 43.02912139892578,
      "activations/layer16_attention_weight_min": -26.867130279541016,
      "activations/layer17_attention_weight_max": 44.39191818237305,
      "activations/layer17_attention_weight_min": -25.331262588500977,
      "activations/layer18_attention_weight_max": 35.6400260925293,
      "activations/layer18_attention_weight_min": -23.036882400512695,
      "activations/layer19_attention_weight_max": 51.037052154541016,
      "activations/layer19_attention_weight_min": -28.071701049804688,
      "activations/layer1_attention_weight_max": 17.11713981628418,
      "activations/layer1_attention_weight_min": -16.935121536254883,
      "activations/layer20_attention_weight_max": 40.188079833984375,
      "activations/layer20_attention_weight_min": -23.0896053314209,
      "activations/layer21_attention_weight_max": 36.796234130859375,
      "activations/layer21_attention_weight_min": -21.674596786499023,
      "activations/layer22_attention_weight_max": 52.02390670776367,
      "activations/layer22_attention_weight_min": -29.35650634765625,
      "activations/layer23_attention_weight_max": 31.649795532226562,
      "activations/layer23_attention_weight_min": -23.3188533782959,
      "activations/layer2_attention_weight_max": 29.614986419677734,
      "activations/layer2_attention_weight_min": -29.58041763305664,
      "activations/layer3_attention_weight_max": 84.04424285888672,
      "activations/layer3_attention_weight_min": -84.16044616699219,
      "activations/layer4_attention_weight_max": 125.78145599365234,
      "activations/layer4_attention_weight_min": -124.7931137084961,
      "activations/layer5_attention_weight_max": 88.60567474365234,
      "activations/layer5_attention_weight_min": -84.99918365478516,
      "activations/layer6_attention_weight_max": 57.157047271728516,
      "activations/layer6_attention_weight_min": -58.746253967285156,
      "activations/layer7_attention_weight_max": 74.00701141357422,
      "activations/layer7_attention_weight_min": -83.58355712890625,
      "activations/layer8_attention_weight_max": 37.807071685791016,
      "activations/layer8_attention_weight_min": -38.43617630004883,
      "activations/layer9_attention_weight_max": 37.07612228393555,
      "activations/layer9_attention_weight_min": -32.37534713745117,
      "epoch": 4.01,
      "learning_rate": 0.00012542424242424242,
      "loss": 2.8604,
      "step": 68950
    },
    {
      "activations/layer0_attention_weight_max": 16.187034606933594,
      "activations/layer0_attention_weight_min": -10.855978965759277,
      "activations/layer10_attention_weight_max": 38.524356842041016,
      "activations/layer10_attention_weight_min": -31.228668212890625,
      "activations/layer11_attention_weight_max": 38.85198974609375,
      "activations/layer11_attention_weight_min": -26.46828269958496,
      "activations/layer12_attention_weight_max": 39.36561965942383,
      "activations/layer12_attention_weight_min": -25.76467514038086,
      "activations/layer13_attention_weight_max": 51.3642692565918,
      "activations/layer13_attention_weight_min": -36.03306579589844,
      "activations/layer14_attention_weight_max": 75.76206970214844,
      "activations/layer14_attention_weight_min": -54.07223129272461,
      "activations/layer15_attention_weight_max": 49.56293869018555,
      "activations/layer15_attention_weight_min": -30.553884506225586,
      "activations/layer16_attention_weight_max": 42.522762298583984,
      "activations/layer16_attention_weight_min": -27.860383987426758,
      "activations/layer17_attention_weight_max": 43.70924758911133,
      "activations/layer17_attention_weight_min": -26.019330978393555,
      "activations/layer18_attention_weight_max": 32.03876876831055,
      "activations/layer18_attention_weight_min": -19.854063034057617,
      "activations/layer19_attention_weight_max": 43.167110443115234,
      "activations/layer19_attention_weight_min": -28.138263702392578,
      "activations/layer1_attention_weight_max": 17.883472442626953,
      "activations/layer1_attention_weight_min": -15.743569374084473,
      "activations/layer20_attention_weight_max": 32.02158737182617,
      "activations/layer20_attention_weight_min": -25.8940372467041,
      "activations/layer21_attention_weight_max": 32.5997200012207,
      "activations/layer21_attention_weight_min": -21.2922420501709,
      "activations/layer22_attention_weight_max": 49.0895881652832,
      "activations/layer22_attention_weight_min": -29.631134033203125,
      "activations/layer23_attention_weight_max": 31.192960739135742,
      "activations/layer23_attention_weight_min": -19.70595359802246,
      "activations/layer2_attention_weight_max": 31.622188568115234,
      "activations/layer2_attention_weight_min": -29.571264266967773,
      "activations/layer3_attention_weight_max": 89.322998046875,
      "activations/layer3_attention_weight_min": -88.52149963378906,
      "activations/layer4_attention_weight_max": 140.46397399902344,
      "activations/layer4_attention_weight_min": -134.26316833496094,
      "activations/layer5_attention_weight_max": 94.97177124023438,
      "activations/layer5_attention_weight_min": -84.53720092773438,
      "activations/layer6_attention_weight_max": 55.528770446777344,
      "activations/layer6_attention_weight_min": -58.9734992980957,
      "activations/layer7_attention_weight_max": 76.70177459716797,
      "activations/layer7_attention_weight_min": -76.1828842163086,
      "activations/layer8_attention_weight_max": 42.539649963378906,
      "activations/layer8_attention_weight_min": -40.95883560180664,
      "activations/layer9_attention_weight_max": 40.27619552612305,
      "activations/layer9_attention_weight_min": -33.66372299194336,
      "epoch": 4.01,
      "learning_rate": 0.000125405303030303,
      "loss": 2.8728,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4144,
      "eval_samples_per_second": 510.316,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4144,
      "eval_openwebtext_samples_per_second": 510.316,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_wikitext_loss": 3.056640625,
      "eval_wikitext_ppl": 21.256030110048442,
      "eval_wikitext_runtime": 1.982,
      "eval_wikitext_samples_per_second": 230.065,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_lambada_loss": 2.759765625,
      "eval_lambada_ppl": 15.796140293991145,
      "eval_lambada_runtime": 9.5557,
      "eval_lambada_samples_per_second": 509.539,
      "step": 69000
    },
    {
      "activations/layer0_attention_weight_max": 16.650890350341797,
      "activations/layer0_attention_weight_min": -12.30897045135498,
      "activations/layer10_attention_weight_max": 35.14276885986328,
      "activations/layer10_attention_weight_min": -28.476327896118164,
      "activations/layer11_attention_weight_max": 31.047773361206055,
      "activations/layer11_attention_weight_min": -28.382009506225586,
      "activations/layer12_attention_weight_max": 38.74785614013672,
      "activations/layer12_attention_weight_min": -27.778013229370117,
      "activations/layer13_attention_weight_max": 44.56877136230469,
      "activations/layer13_attention_weight_min": -32.839622497558594,
      "activations/layer14_attention_weight_max": 69.7066650390625,
      "activations/layer14_attention_weight_min": -52.39736557006836,
      "activations/layer15_attention_weight_max": 43.86439895629883,
      "activations/layer15_attention_weight_min": -30.260251998901367,
      "activations/layer16_attention_weight_max": 43.58010482788086,
      "activations/layer16_attention_weight_min": -33.079437255859375,
      "activations/layer17_attention_weight_max": 50.41466522216797,
      "activations/layer17_attention_weight_min": -26.347265243530273,
      "activations/layer18_attention_weight_max": 39.55113983154297,
      "activations/layer18_attention_weight_min": -21.558696746826172,
      "activations/layer19_attention_weight_max": 40.69810104370117,
      "activations/layer19_attention_weight_min": -31.069854736328125,
      "activations/layer1_attention_weight_max": 16.021785736083984,
      "activations/layer1_attention_weight_min": -17.38054656982422,
      "activations/layer20_attention_weight_max": 34.198585510253906,
      "activations/layer20_attention_weight_min": -24.410066604614258,
      "activations/layer21_attention_weight_max": 34.07782745361328,
      "activations/layer21_attention_weight_min": -20.315750122070312,
      "activations/layer22_attention_weight_max": 44.388206481933594,
      "activations/layer22_attention_weight_min": -28.32659339904785,
      "activations/layer23_attention_weight_max": 30.261402130126953,
      "activations/layer23_attention_weight_min": -18.333412170410156,
      "activations/layer2_attention_weight_max": 31.97491455078125,
      "activations/layer2_attention_weight_min": -31.096725463867188,
      "activations/layer3_attention_weight_max": 87.82066345214844,
      "activations/layer3_attention_weight_min": -87.67494201660156,
      "activations/layer4_attention_weight_max": 128.53738403320312,
      "activations/layer4_attention_weight_min": -121.38372039794922,
      "activations/layer5_attention_weight_max": 87.62477111816406,
      "activations/layer5_attention_weight_min": -79.12184143066406,
      "activations/layer6_attention_weight_max": 56.343505859375,
      "activations/layer6_attention_weight_min": -56.55845642089844,
      "activations/layer7_attention_weight_max": 77.8988265991211,
      "activations/layer7_attention_weight_min": -71.14147186279297,
      "activations/layer8_attention_weight_max": 38.13448715209961,
      "activations/layer8_attention_weight_min": -36.537559509277344,
      "activations/layer9_attention_weight_max": 38.77505111694336,
      "activations/layer9_attention_weight_min": -33.7414665222168,
      "epoch": 4.01,
      "learning_rate": 0.00012538636363636362,
      "loss": 2.8675,
      "step": 69050
    },
    {
      "activations/layer0_attention_weight_max": 16.267683029174805,
      "activations/layer0_attention_weight_min": -12.222806930541992,
      "activations/layer10_attention_weight_max": 35.30829620361328,
      "activations/layer10_attention_weight_min": -30.31073570251465,
      "activations/layer11_attention_weight_max": 30.276424407958984,
      "activations/layer11_attention_weight_min": -26.286300659179688,
      "activations/layer12_attention_weight_max": 48.86530303955078,
      "activations/layer12_attention_weight_min": -29.508676528930664,
      "activations/layer13_attention_weight_max": 43.15605545043945,
      "activations/layer13_attention_weight_min": -32.75844955444336,
      "activations/layer14_attention_weight_max": 67.20220184326172,
      "activations/layer14_attention_weight_min": -52.846683502197266,
      "activations/layer15_attention_weight_max": 44.856727600097656,
      "activations/layer15_attention_weight_min": -30.464075088500977,
      "activations/layer16_attention_weight_max": 38.30415725708008,
      "activations/layer16_attention_weight_min": -28.55690574645996,
      "activations/layer17_attention_weight_max": 41.96526336669922,
      "activations/layer17_attention_weight_min": -27.020105361938477,
      "activations/layer18_attention_weight_max": 30.601036071777344,
      "activations/layer18_attention_weight_min": -21.151330947875977,
      "activations/layer19_attention_weight_max": 43.4587516784668,
      "activations/layer19_attention_weight_min": -32.0865364074707,
      "activations/layer1_attention_weight_max": 17.38549041748047,
      "activations/layer1_attention_weight_min": -13.899194717407227,
      "activations/layer20_attention_weight_max": 32.7886848449707,
      "activations/layer20_attention_weight_min": -25.327985763549805,
      "activations/layer21_attention_weight_max": 38.0689697265625,
      "activations/layer21_attention_weight_min": -20.7227783203125,
      "activations/layer22_attention_weight_max": 48.772010803222656,
      "activations/layer22_attention_weight_min": -30.836734771728516,
      "activations/layer23_attention_weight_max": 29.86573028564453,
      "activations/layer23_attention_weight_min": -17.335628509521484,
      "activations/layer2_attention_weight_max": 31.063934326171875,
      "activations/layer2_attention_weight_min": -30.814414978027344,
      "activations/layer3_attention_weight_max": 85.58389282226562,
      "activations/layer3_attention_weight_min": -85.70319366455078,
      "activations/layer4_attention_weight_max": 128.9962921142578,
      "activations/layer4_attention_weight_min": -124.50152587890625,
      "activations/layer5_attention_weight_max": 89.97930908203125,
      "activations/layer5_attention_weight_min": -86.12255859375,
      "activations/layer6_attention_weight_max": 56.19339370727539,
      "activations/layer6_attention_weight_min": -56.93722915649414,
      "activations/layer7_attention_weight_max": 77.3470687866211,
      "activations/layer7_attention_weight_min": -77.30445098876953,
      "activations/layer8_attention_weight_max": 38.23191833496094,
      "activations/layer8_attention_weight_min": -38.241703033447266,
      "activations/layer9_attention_weight_max": 36.32937240600586,
      "activations/layer9_attention_weight_min": -32.85752487182617,
      "epoch": 4.02,
      "learning_rate": 0.0001253674242424242,
      "loss": 2.8706,
      "step": 69100
    },
    {
      "activations/layer0_attention_weight_max": 16.20354652404785,
      "activations/layer0_attention_weight_min": -11.216801643371582,
      "activations/layer10_attention_weight_max": 37.076759338378906,
      "activations/layer10_attention_weight_min": -33.50947952270508,
      "activations/layer11_attention_weight_max": 34.68720245361328,
      "activations/layer11_attention_weight_min": -29.77168083190918,
      "activations/layer12_attention_weight_max": 45.721771240234375,
      "activations/layer12_attention_weight_min": -32.60871505737305,
      "activations/layer13_attention_weight_max": 45.228939056396484,
      "activations/layer13_attention_weight_min": -32.065208435058594,
      "activations/layer14_attention_weight_max": 81.16919708251953,
      "activations/layer14_attention_weight_min": -52.585487365722656,
      "activations/layer15_attention_weight_max": 39.057403564453125,
      "activations/layer15_attention_weight_min": -28.564163208007812,
      "activations/layer16_attention_weight_max": 42.61786651611328,
      "activations/layer16_attention_weight_min": -35.53662872314453,
      "activations/layer17_attention_weight_max": 37.394039154052734,
      "activations/layer17_attention_weight_min": -25.244037628173828,
      "activations/layer18_attention_weight_max": 31.42623519897461,
      "activations/layer18_attention_weight_min": -21.10140609741211,
      "activations/layer19_attention_weight_max": 41.142478942871094,
      "activations/layer19_attention_weight_min": -31.975133895874023,
      "activations/layer1_attention_weight_max": 16.8560791015625,
      "activations/layer1_attention_weight_min": -15.776203155517578,
      "activations/layer20_attention_weight_max": 31.629716873168945,
      "activations/layer20_attention_weight_min": -22.59825897216797,
      "activations/layer21_attention_weight_max": 31.974700927734375,
      "activations/layer21_attention_weight_min": -20.01910400390625,
      "activations/layer22_attention_weight_max": 50.5611457824707,
      "activations/layer22_attention_weight_min": -27.24690055847168,
      "activations/layer23_attention_weight_max": 28.719558715820312,
      "activations/layer23_attention_weight_min": -17.00979232788086,
      "activations/layer2_attention_weight_max": 32.11183166503906,
      "activations/layer2_attention_weight_min": -32.46147918701172,
      "activations/layer3_attention_weight_max": 84.73193359375,
      "activations/layer3_attention_weight_min": -86.0767593383789,
      "activations/layer4_attention_weight_max": 130.2202606201172,
      "activations/layer4_attention_weight_min": -127.40349578857422,
      "activations/layer5_attention_weight_max": 92.15107727050781,
      "activations/layer5_attention_weight_min": -85.42452239990234,
      "activations/layer6_attention_weight_max": 56.936038970947266,
      "activations/layer6_attention_weight_min": -55.489540100097656,
      "activations/layer7_attention_weight_max": 79.87338256835938,
      "activations/layer7_attention_weight_min": -79.80427551269531,
      "activations/layer8_attention_weight_max": 40.572853088378906,
      "activations/layer8_attention_weight_min": -39.86626052856445,
      "activations/layer9_attention_weight_max": 40.41260528564453,
      "activations/layer9_attention_weight_min": -35.51649856567383,
      "epoch": 4.02,
      "learning_rate": 0.00012534848484848483,
      "loss": 2.8492,
      "step": 69150
    },
    {
      "activations/layer0_attention_weight_max": 16.510122299194336,
      "activations/layer0_attention_weight_min": -11.319267272949219,
      "activations/layer10_attention_weight_max": 35.781211853027344,
      "activations/layer10_attention_weight_min": -32.23694610595703,
      "activations/layer11_attention_weight_max": 37.59703826904297,
      "activations/layer11_attention_weight_min": -29.097248077392578,
      "activations/layer12_attention_weight_max": 34.552879333496094,
      "activations/layer12_attention_weight_min": -26.171663284301758,
      "activations/layer13_attention_weight_max": 46.756832122802734,
      "activations/layer13_attention_weight_min": -32.89887619018555,
      "activations/layer14_attention_weight_max": 74.48146057128906,
      "activations/layer14_attention_weight_min": -50.847652435302734,
      "activations/layer15_attention_weight_max": 45.899654388427734,
      "activations/layer15_attention_weight_min": -29.41534996032715,
      "activations/layer16_attention_weight_max": 49.08290100097656,
      "activations/layer16_attention_weight_min": -28.498977661132812,
      "activations/layer17_attention_weight_max": 38.45462417602539,
      "activations/layer17_attention_weight_min": -24.92753791809082,
      "activations/layer18_attention_weight_max": 28.341243743896484,
      "activations/layer18_attention_weight_min": -18.80842399597168,
      "activations/layer19_attention_weight_max": 57.06281661987305,
      "activations/layer19_attention_weight_min": -28.342782974243164,
      "activations/layer1_attention_weight_max": 17.575069427490234,
      "activations/layer1_attention_weight_min": -14.72614574432373,
      "activations/layer20_attention_weight_max": 37.77503204345703,
      "activations/layer20_attention_weight_min": -22.43537712097168,
      "activations/layer21_attention_weight_max": 30.109045028686523,
      "activations/layer21_attention_weight_min": -18.020429611206055,
      "activations/layer22_attention_weight_max": 47.4751091003418,
      "activations/layer22_attention_weight_min": -26.496965408325195,
      "activations/layer23_attention_weight_max": 30.697917938232422,
      "activations/layer23_attention_weight_min": -18.428197860717773,
      "activations/layer2_attention_weight_max": 33.20314407348633,
      "activations/layer2_attention_weight_min": -32.335506439208984,
      "activations/layer3_attention_weight_max": 87.65623474121094,
      "activations/layer3_attention_weight_min": -87.93257141113281,
      "activations/layer4_attention_weight_max": 132.0504150390625,
      "activations/layer4_attention_weight_min": -123.93013000488281,
      "activations/layer5_attention_weight_max": 93.34075164794922,
      "activations/layer5_attention_weight_min": -83.61323547363281,
      "activations/layer6_attention_weight_max": 60.8543586730957,
      "activations/layer6_attention_weight_min": -57.33304214477539,
      "activations/layer7_attention_weight_max": 79.67066192626953,
      "activations/layer7_attention_weight_min": -76.13429260253906,
      "activations/layer8_attention_weight_max": 41.35007858276367,
      "activations/layer8_attention_weight_min": -38.61153030395508,
      "activations/layer9_attention_weight_max": 41.62620162963867,
      "activations/layer9_attention_weight_min": -34.442745208740234,
      "epoch": 4.02,
      "learning_rate": 0.00012532992424242422,
      "loss": 2.8695,
      "step": 69200
    },
    {
      "activations/layer0_attention_weight_max": 14.897794723510742,
      "activations/layer0_attention_weight_min": -11.502595901489258,
      "activations/layer10_attention_weight_max": 39.473201751708984,
      "activations/layer10_attention_weight_min": -32.268310546875,
      "activations/layer11_attention_weight_max": 36.08641815185547,
      "activations/layer11_attention_weight_min": -30.030914306640625,
      "activations/layer12_attention_weight_max": 29.892620086669922,
      "activations/layer12_attention_weight_min": -29.183412551879883,
      "activations/layer13_attention_weight_max": 47.90486526489258,
      "activations/layer13_attention_weight_min": -36.49750900268555,
      "activations/layer14_attention_weight_max": 70.30618286132812,
      "activations/layer14_attention_weight_min": -50.779178619384766,
      "activations/layer15_attention_weight_max": 46.082279205322266,
      "activations/layer15_attention_weight_min": -27.100095748901367,
      "activations/layer16_attention_weight_max": 40.82851028442383,
      "activations/layer16_attention_weight_min": -27.386150360107422,
      "activations/layer17_attention_weight_max": 49.54442596435547,
      "activations/layer17_attention_weight_min": -24.612979888916016,
      "activations/layer18_attention_weight_max": 33.85386276245117,
      "activations/layer18_attention_weight_min": -20.704748153686523,
      "activations/layer19_attention_weight_max": 44.605587005615234,
      "activations/layer19_attention_weight_min": -31.38471031188965,
      "activations/layer1_attention_weight_max": 18.031484603881836,
      "activations/layer1_attention_weight_min": -16.805896759033203,
      "activations/layer20_attention_weight_max": 38.949119567871094,
      "activations/layer20_attention_weight_min": -23.334257125854492,
      "activations/layer21_attention_weight_max": 34.46588134765625,
      "activations/layer21_attention_weight_min": -18.940631866455078,
      "activations/layer22_attention_weight_max": 56.36933135986328,
      "activations/layer22_attention_weight_min": -27.37022590637207,
      "activations/layer23_attention_weight_max": 33.68872833251953,
      "activations/layer23_attention_weight_min": -18.464426040649414,
      "activations/layer2_attention_weight_max": 32.02919006347656,
      "activations/layer2_attention_weight_min": -32.52237319946289,
      "activations/layer3_attention_weight_max": 85.74213409423828,
      "activations/layer3_attention_weight_min": -88.5654296875,
      "activations/layer4_attention_weight_max": 143.6887664794922,
      "activations/layer4_attention_weight_min": -137.49752807617188,
      "activations/layer5_attention_weight_max": 90.97161865234375,
      "activations/layer5_attention_weight_min": -95.53777313232422,
      "activations/layer6_attention_weight_max": 57.97581100463867,
      "activations/layer6_attention_weight_min": -61.24070739746094,
      "activations/layer7_attention_weight_max": 79.12117004394531,
      "activations/layer7_attention_weight_min": -84.15998077392578,
      "activations/layer8_attention_weight_max": 40.98070526123047,
      "activations/layer8_attention_weight_min": -41.318443298339844,
      "activations/layer9_attention_weight_max": 48.15877914428711,
      "activations/layer9_attention_weight_min": -39.32400131225586,
      "epoch": 4.02,
      "learning_rate": 0.00012531098484848483,
      "loss": 2.8748,
      "step": 69250
    },
    {
      "activations/layer0_attention_weight_max": 16.837081909179688,
      "activations/layer0_attention_weight_min": -11.391120910644531,
      "activations/layer10_attention_weight_max": 35.23741912841797,
      "activations/layer10_attention_weight_min": -30.27733612060547,
      "activations/layer11_attention_weight_max": 38.80998229980469,
      "activations/layer11_attention_weight_min": -29.459178924560547,
      "activations/layer12_attention_weight_max": 52.60663604736328,
      "activations/layer12_attention_weight_min": -31.1558895111084,
      "activations/layer13_attention_weight_max": 83.69776916503906,
      "activations/layer13_attention_weight_min": -50.57759094238281,
      "activations/layer14_attention_weight_max": 80.24669647216797,
      "activations/layer14_attention_weight_min": -51.567630767822266,
      "activations/layer15_attention_weight_max": 55.02279281616211,
      "activations/layer15_attention_weight_min": -31.855838775634766,
      "activations/layer16_attention_weight_max": 41.65726089477539,
      "activations/layer16_attention_weight_min": -27.064363479614258,
      "activations/layer17_attention_weight_max": 37.866085052490234,
      "activations/layer17_attention_weight_min": -24.86493682861328,
      "activations/layer18_attention_weight_max": 29.981571197509766,
      "activations/layer18_attention_weight_min": -19.829309463500977,
      "activations/layer19_attention_weight_max": 44.30301284790039,
      "activations/layer19_attention_weight_min": -28.711557388305664,
      "activations/layer1_attention_weight_max": 18.389509201049805,
      "activations/layer1_attention_weight_min": -14.313254356384277,
      "activations/layer20_attention_weight_max": 36.9864616394043,
      "activations/layer20_attention_weight_min": -22.78361701965332,
      "activations/layer21_attention_weight_max": 34.4691047668457,
      "activations/layer21_attention_weight_min": -21.800312042236328,
      "activations/layer22_attention_weight_max": 47.553855895996094,
      "activations/layer22_attention_weight_min": -26.95680809020996,
      "activations/layer23_attention_weight_max": 28.39775848388672,
      "activations/layer23_attention_weight_min": -17.201446533203125,
      "activations/layer2_attention_weight_max": 30.791349411010742,
      "activations/layer2_attention_weight_min": -30.872112274169922,
      "activations/layer3_attention_weight_max": 84.80227661132812,
      "activations/layer3_attention_weight_min": -89.9428482055664,
      "activations/layer4_attention_weight_max": 124.03582763671875,
      "activations/layer4_attention_weight_min": -129.10423278808594,
      "activations/layer5_attention_weight_max": 87.69356536865234,
      "activations/layer5_attention_weight_min": -81.50202941894531,
      "activations/layer6_attention_weight_max": 59.205284118652344,
      "activations/layer6_attention_weight_min": -55.95233154296875,
      "activations/layer7_attention_weight_max": 77.2540283203125,
      "activations/layer7_attention_weight_min": -76.46836853027344,
      "activations/layer8_attention_weight_max": 37.15242385864258,
      "activations/layer8_attention_weight_min": -36.87810516357422,
      "activations/layer9_attention_weight_max": 43.32609176635742,
      "activations/layer9_attention_weight_min": -40.02817153930664,
      "epoch": 4.03,
      "learning_rate": 0.00012529204545454545,
      "loss": 2.8628,
      "step": 69300
    },
    {
      "activations/layer0_attention_weight_max": 15.878199577331543,
      "activations/layer0_attention_weight_min": -10.851848602294922,
      "activations/layer10_attention_weight_max": 35.91279220581055,
      "activations/layer10_attention_weight_min": -29.878021240234375,
      "activations/layer11_attention_weight_max": 32.732383728027344,
      "activations/layer11_attention_weight_min": -28.978864669799805,
      "activations/layer12_attention_weight_max": 26.87467384338379,
      "activations/layer12_attention_weight_min": -29.742557525634766,
      "activations/layer13_attention_weight_max": 45.65595626831055,
      "activations/layer13_attention_weight_min": -31.433032989501953,
      "activations/layer14_attention_weight_max": 69.38162994384766,
      "activations/layer14_attention_weight_min": -48.761260986328125,
      "activations/layer15_attention_weight_max": 59.8834228515625,
      "activations/layer15_attention_weight_min": -29.684144973754883,
      "activations/layer16_attention_weight_max": 42.0257682800293,
      "activations/layer16_attention_weight_min": -28.440139770507812,
      "activations/layer17_attention_weight_max": 40.78593444824219,
      "activations/layer17_attention_weight_min": -26.276941299438477,
      "activations/layer18_attention_weight_max": 33.04814147949219,
      "activations/layer18_attention_weight_min": -22.252071380615234,
      "activations/layer19_attention_weight_max": 44.1370735168457,
      "activations/layer19_attention_weight_min": -30.155317306518555,
      "activations/layer1_attention_weight_max": 17.711315155029297,
      "activations/layer1_attention_weight_min": -16.51087188720703,
      "activations/layer20_attention_weight_max": 34.834556579589844,
      "activations/layer20_attention_weight_min": -24.56194305419922,
      "activations/layer21_attention_weight_max": 39.35935974121094,
      "activations/layer21_attention_weight_min": -19.550613403320312,
      "activations/layer22_attention_weight_max": 56.8182258605957,
      "activations/layer22_attention_weight_min": -27.8795166015625,
      "activations/layer23_attention_weight_max": 32.85093307495117,
      "activations/layer23_attention_weight_min": -17.137388229370117,
      "activations/layer2_attention_weight_max": 31.389137268066406,
      "activations/layer2_attention_weight_min": -31.020740509033203,
      "activations/layer3_attention_weight_max": 89.42479705810547,
      "activations/layer3_attention_weight_min": -90.689453125,
      "activations/layer4_attention_weight_max": 130.59213256835938,
      "activations/layer4_attention_weight_min": -126.02278137207031,
      "activations/layer5_attention_weight_max": 92.46031188964844,
      "activations/layer5_attention_weight_min": -87.27839660644531,
      "activations/layer6_attention_weight_max": 57.53599548339844,
      "activations/layer6_attention_weight_min": -55.40972900390625,
      "activations/layer7_attention_weight_max": 77.58878326416016,
      "activations/layer7_attention_weight_min": -74.80835723876953,
      "activations/layer8_attention_weight_max": 39.31594467163086,
      "activations/layer8_attention_weight_min": -39.2814826965332,
      "activations/layer9_attention_weight_max": 36.90214538574219,
      "activations/layer9_attention_weight_min": -41.480979919433594,
      "epoch": 4.03,
      "learning_rate": 0.00012527310606060604,
      "loss": 2.8704,
      "step": 69350
    },
    {
      "activations/layer0_attention_weight_max": 15.929800033569336,
      "activations/layer0_attention_weight_min": -10.699609756469727,
      "activations/layer10_attention_weight_max": 39.8681526184082,
      "activations/layer10_attention_weight_min": -30.67453956604004,
      "activations/layer11_attention_weight_max": 35.33591079711914,
      "activations/layer11_attention_weight_min": -29.443662643432617,
      "activations/layer12_attention_weight_max": 47.6322135925293,
      "activations/layer12_attention_weight_min": -24.34172248840332,
      "activations/layer13_attention_weight_max": 52.08234786987305,
      "activations/layer13_attention_weight_min": -35.59907531738281,
      "activations/layer14_attention_weight_max": 86.14715576171875,
      "activations/layer14_attention_weight_min": -54.260009765625,
      "activations/layer15_attention_weight_max": 47.56527328491211,
      "activations/layer15_attention_weight_min": -29.087873458862305,
      "activations/layer16_attention_weight_max": 44.919891357421875,
      "activations/layer16_attention_weight_min": -31.46360206604004,
      "activations/layer17_attention_weight_max": 44.045440673828125,
      "activations/layer17_attention_weight_min": -25.329893112182617,
      "activations/layer18_attention_weight_max": 32.99541473388672,
      "activations/layer18_attention_weight_min": -20.935253143310547,
      "activations/layer19_attention_weight_max": 46.46683883666992,
      "activations/layer19_attention_weight_min": -27.921049118041992,
      "activations/layer1_attention_weight_max": 16.868946075439453,
      "activations/layer1_attention_weight_min": -15.77710247039795,
      "activations/layer20_attention_weight_max": 36.2443733215332,
      "activations/layer20_attention_weight_min": -23.273574829101562,
      "activations/layer21_attention_weight_max": 35.67695999145508,
      "activations/layer21_attention_weight_min": -19.802644729614258,
      "activations/layer22_attention_weight_max": 51.307247161865234,
      "activations/layer22_attention_weight_min": -26.68057632446289,
      "activations/layer23_attention_weight_max": 33.27458190917969,
      "activations/layer23_attention_weight_min": -18.2994441986084,
      "activations/layer2_attention_weight_max": 31.745588302612305,
      "activations/layer2_attention_weight_min": -31.520599365234375,
      "activations/layer3_attention_weight_max": 85.9090805053711,
      "activations/layer3_attention_weight_min": -88.21704864501953,
      "activations/layer4_attention_weight_max": 125.20183563232422,
      "activations/layer4_attention_weight_min": -127.44283294677734,
      "activations/layer5_attention_weight_max": 85.41575622558594,
      "activations/layer5_attention_weight_min": -83.06604766845703,
      "activations/layer6_attention_weight_max": 56.598663330078125,
      "activations/layer6_attention_weight_min": -57.09227752685547,
      "activations/layer7_attention_weight_max": 75.97196960449219,
      "activations/layer7_attention_weight_min": -78.60875701904297,
      "activations/layer8_attention_weight_max": 39.53842544555664,
      "activations/layer8_attention_weight_min": -39.66158676147461,
      "activations/layer9_attention_weight_max": 38.882808685302734,
      "activations/layer9_attention_weight_min": -34.68801498413086,
      "epoch": 4.03,
      "learning_rate": 0.00012525416666666665,
      "loss": 2.873,
      "step": 69400
    },
    {
      "activations/layer0_attention_weight_max": 15.042964935302734,
      "activations/layer0_attention_weight_min": -10.636627197265625,
      "activations/layer10_attention_weight_max": 37.0232048034668,
      "activations/layer10_attention_weight_min": -30.71764373779297,
      "activations/layer11_attention_weight_max": 32.65930938720703,
      "activations/layer11_attention_weight_min": -26.933837890625,
      "activations/layer12_attention_weight_max": 52.22014617919922,
      "activations/layer12_attention_weight_min": -27.4650936126709,
      "activations/layer13_attention_weight_max": 45.71950149536133,
      "activations/layer13_attention_weight_min": -33.14925003051758,
      "activations/layer14_attention_weight_max": 67.1821517944336,
      "activations/layer14_attention_weight_min": -47.76032257080078,
      "activations/layer15_attention_weight_max": 53.61106491088867,
      "activations/layer15_attention_weight_min": -33.51872634887695,
      "activations/layer16_attention_weight_max": 41.19282913208008,
      "activations/layer16_attention_weight_min": -29.60063362121582,
      "activations/layer17_attention_weight_max": 44.66279983520508,
      "activations/layer17_attention_weight_min": -26.682254791259766,
      "activations/layer18_attention_weight_max": 35.71599578857422,
      "activations/layer18_attention_weight_min": -19.525272369384766,
      "activations/layer19_attention_weight_max": 46.694515228271484,
      "activations/layer19_attention_weight_min": -29.653228759765625,
      "activations/layer1_attention_weight_max": 17.21360969543457,
      "activations/layer1_attention_weight_min": -16.506134033203125,
      "activations/layer20_attention_weight_max": 36.83994674682617,
      "activations/layer20_attention_weight_min": -24.563941955566406,
      "activations/layer21_attention_weight_max": 41.11614227294922,
      "activations/layer21_attention_weight_min": -20.395790100097656,
      "activations/layer22_attention_weight_max": 52.827266693115234,
      "activations/layer22_attention_weight_min": -29.38796043395996,
      "activations/layer23_attention_weight_max": 34.927635192871094,
      "activations/layer23_attention_weight_min": -18.19681739807129,
      "activations/layer2_attention_weight_max": 32.559547424316406,
      "activations/layer2_attention_weight_min": -32.577667236328125,
      "activations/layer3_attention_weight_max": 90.53203582763672,
      "activations/layer3_attention_weight_min": -87.67877960205078,
      "activations/layer4_attention_weight_max": 130.31178283691406,
      "activations/layer4_attention_weight_min": -128.2752227783203,
      "activations/layer5_attention_weight_max": 89.40239715576172,
      "activations/layer5_attention_weight_min": -87.10293579101562,
      "activations/layer6_attention_weight_max": 56.89580535888672,
      "activations/layer6_attention_weight_min": -57.56855392456055,
      "activations/layer7_attention_weight_max": 77.16618347167969,
      "activations/layer7_attention_weight_min": -76.23255920410156,
      "activations/layer8_attention_weight_max": 39.05940246582031,
      "activations/layer8_attention_weight_min": -39.79974365234375,
      "activations/layer9_attention_weight_max": 38.856475830078125,
      "activations/layer9_attention_weight_min": -35.181331634521484,
      "epoch": 4.04,
      "learning_rate": 0.00012523522727272727,
      "loss": 2.8752,
      "step": 69450
    },
    {
      "activations/layer0_attention_weight_max": 17.168907165527344,
      "activations/layer0_attention_weight_min": -11.630931854248047,
      "activations/layer10_attention_weight_max": 37.03229522705078,
      "activations/layer10_attention_weight_min": -31.293338775634766,
      "activations/layer11_attention_weight_max": 37.070804595947266,
      "activations/layer11_attention_weight_min": -27.436063766479492,
      "activations/layer12_attention_weight_max": 51.60196304321289,
      "activations/layer12_attention_weight_min": -26.851972579956055,
      "activations/layer13_attention_weight_max": 62.27854537963867,
      "activations/layer13_attention_weight_min": -35.16254425048828,
      "activations/layer14_attention_weight_max": 102.41056060791016,
      "activations/layer14_attention_weight_min": -58.7034797668457,
      "activations/layer15_attention_weight_max": 65.26493072509766,
      "activations/layer15_attention_weight_min": -30.11966896057129,
      "activations/layer16_attention_weight_max": 54.517250061035156,
      "activations/layer16_attention_weight_min": -30.56614875793457,
      "activations/layer17_attention_weight_max": 61.82365798950195,
      "activations/layer17_attention_weight_min": -25.594058990478516,
      "activations/layer18_attention_weight_max": 52.99251937866211,
      "activations/layer18_attention_weight_min": -20.80303382873535,
      "activations/layer19_attention_weight_max": 60.816951751708984,
      "activations/layer19_attention_weight_min": -30.09726905822754,
      "activations/layer1_attention_weight_max": 18.55365753173828,
      "activations/layer1_attention_weight_min": -15.088668823242188,
      "activations/layer20_attention_weight_max": 50.2014274597168,
      "activations/layer20_attention_weight_min": -23.07274055480957,
      "activations/layer21_attention_weight_max": 48.837215423583984,
      "activations/layer21_attention_weight_min": -20.346454620361328,
      "activations/layer22_attention_weight_max": 72.90216827392578,
      "activations/layer22_attention_weight_min": -27.507619857788086,
      "activations/layer23_attention_weight_max": 45.48499298095703,
      "activations/layer23_attention_weight_min": -17.215986251831055,
      "activations/layer2_attention_weight_max": 30.909557342529297,
      "activations/layer2_attention_weight_min": -31.072269439697266,
      "activations/layer3_attention_weight_max": 87.91861724853516,
      "activations/layer3_attention_weight_min": -85.537841796875,
      "activations/layer4_attention_weight_max": 137.47389221191406,
      "activations/layer4_attention_weight_min": -129.774169921875,
      "activations/layer5_attention_weight_max": 93.13516235351562,
      "activations/layer5_attention_weight_min": -87.46952819824219,
      "activations/layer6_attention_weight_max": 64.92245483398438,
      "activations/layer6_attention_weight_min": -56.413089752197266,
      "activations/layer7_attention_weight_max": 81.00260162353516,
      "activations/layer7_attention_weight_min": -78.02194213867188,
      "activations/layer8_attention_weight_max": 40.30624771118164,
      "activations/layer8_attention_weight_min": -37.75593948364258,
      "activations/layer9_attention_weight_max": 38.757808685302734,
      "activations/layer9_attention_weight_min": -34.63431930541992,
      "epoch": 4.04,
      "learning_rate": 0.00012521628787878788,
      "loss": 2.875,
      "step": 69500
    },
    {
      "activations/layer0_attention_weight_max": 16.30891990661621,
      "activations/layer0_attention_weight_min": -12.108945846557617,
      "activations/layer10_attention_weight_max": 34.59911346435547,
      "activations/layer10_attention_weight_min": -30.57977294921875,
      "activations/layer11_attention_weight_max": 32.05895233154297,
      "activations/layer11_attention_weight_min": -27.648387908935547,
      "activations/layer12_attention_weight_max": 40.33616256713867,
      "activations/layer12_attention_weight_min": -28.367664337158203,
      "activations/layer13_attention_weight_max": 42.30476379394531,
      "activations/layer13_attention_weight_min": -32.21318054199219,
      "activations/layer14_attention_weight_max": 56.40406036376953,
      "activations/layer14_attention_weight_min": -43.74489212036133,
      "activations/layer15_attention_weight_max": 42.6008415222168,
      "activations/layer15_attention_weight_min": -27.095125198364258,
      "activations/layer16_attention_weight_max": 44.66118621826172,
      "activations/layer16_attention_weight_min": -29.146560668945312,
      "activations/layer17_attention_weight_max": 45.299049377441406,
      "activations/layer17_attention_weight_min": -24.382896423339844,
      "activations/layer18_attention_weight_max": 31.274038314819336,
      "activations/layer18_attention_weight_min": -19.566890716552734,
      "activations/layer19_attention_weight_max": 42.59945297241211,
      "activations/layer19_attention_weight_min": -29.035032272338867,
      "activations/layer1_attention_weight_max": 17.798919677734375,
      "activations/layer1_attention_weight_min": -14.266242027282715,
      "activations/layer20_attention_weight_max": 36.99326705932617,
      "activations/layer20_attention_weight_min": -23.160770416259766,
      "activations/layer21_attention_weight_max": 34.74267578125,
      "activations/layer21_attention_weight_min": -20.037078857421875,
      "activations/layer22_attention_weight_max": 48.52503204345703,
      "activations/layer22_attention_weight_min": -27.27485466003418,
      "activations/layer23_attention_weight_max": 37.201210021972656,
      "activations/layer23_attention_weight_min": -17.032546997070312,
      "activations/layer2_attention_weight_max": 31.07709503173828,
      "activations/layer2_attention_weight_min": -32.08341979980469,
      "activations/layer3_attention_weight_max": 84.48339080810547,
      "activations/layer3_attention_weight_min": -87.02429962158203,
      "activations/layer4_attention_weight_max": 124.89552307128906,
      "activations/layer4_attention_weight_min": -124.787353515625,
      "activations/layer5_attention_weight_max": 86.33686828613281,
      "activations/layer5_attention_weight_min": -80.62818145751953,
      "activations/layer6_attention_weight_max": 57.50264358520508,
      "activations/layer6_attention_weight_min": -55.525699615478516,
      "activations/layer7_attention_weight_max": 76.17569732666016,
      "activations/layer7_attention_weight_min": -73.55059051513672,
      "activations/layer8_attention_weight_max": 37.549583435058594,
      "activations/layer8_attention_weight_min": -36.52214050292969,
      "activations/layer9_attention_weight_max": 36.310142517089844,
      "activations/layer9_attention_weight_min": -35.188262939453125,
      "epoch": 4.04,
      "learning_rate": 0.00012519734848484847,
      "loss": 2.8763,
      "step": 69550
    },
    {
      "activations/layer0_attention_weight_max": 15.965600967407227,
      "activations/layer0_attention_weight_min": -12.381056785583496,
      "activations/layer10_attention_weight_max": 41.19279861450195,
      "activations/layer10_attention_weight_min": -31.58516502380371,
      "activations/layer11_attention_weight_max": 37.3907470703125,
      "activations/layer11_attention_weight_min": -29.947891235351562,
      "activations/layer12_attention_weight_max": 33.9033203125,
      "activations/layer12_attention_weight_min": -28.53297233581543,
      "activations/layer13_attention_weight_max": 50.66499328613281,
      "activations/layer13_attention_weight_min": -32.96623229980469,
      "activations/layer14_attention_weight_max": 74.84473419189453,
      "activations/layer14_attention_weight_min": -54.8490104675293,
      "activations/layer15_attention_weight_max": 42.352542877197266,
      "activations/layer15_attention_weight_min": -28.61918830871582,
      "activations/layer16_attention_weight_max": 43.43898391723633,
      "activations/layer16_attention_weight_min": -27.556230545043945,
      "activations/layer17_attention_weight_max": 43.432254791259766,
      "activations/layer17_attention_weight_min": -25.723758697509766,
      "activations/layer18_attention_weight_max": 32.72968673706055,
      "activations/layer18_attention_weight_min": -18.747217178344727,
      "activations/layer19_attention_weight_max": 43.685455322265625,
      "activations/layer19_attention_weight_min": -27.572843551635742,
      "activations/layer1_attention_weight_max": 17.406522750854492,
      "activations/layer1_attention_weight_min": -13.847489356994629,
      "activations/layer20_attention_weight_max": 35.86532211303711,
      "activations/layer20_attention_weight_min": -23.447507858276367,
      "activations/layer21_attention_weight_max": 33.10197448730469,
      "activations/layer21_attention_weight_min": -18.282350540161133,
      "activations/layer22_attention_weight_max": 55.852294921875,
      "activations/layer22_attention_weight_min": -27.790225982666016,
      "activations/layer23_attention_weight_max": 31.900442123413086,
      "activations/layer23_attention_weight_min": -18.90477752685547,
      "activations/layer2_attention_weight_max": 33.406639099121094,
      "activations/layer2_attention_weight_min": -31.730382919311523,
      "activations/layer3_attention_weight_max": 89.73737335205078,
      "activations/layer3_attention_weight_min": -90.32714080810547,
      "activations/layer4_attention_weight_max": 143.7899627685547,
      "activations/layer4_attention_weight_min": -129.71588134765625,
      "activations/layer5_attention_weight_max": 96.07320404052734,
      "activations/layer5_attention_weight_min": -85.24574279785156,
      "activations/layer6_attention_weight_max": 59.561546325683594,
      "activations/layer6_attention_weight_min": -56.02473831176758,
      "activations/layer7_attention_weight_max": 88.7041015625,
      "activations/layer7_attention_weight_min": -81.70980834960938,
      "activations/layer8_attention_weight_max": 44.15853500366211,
      "activations/layer8_attention_weight_min": -38.59926223754883,
      "activations/layer9_attention_weight_max": 39.164669036865234,
      "activations/layer9_attention_weight_min": -34.64285659790039,
      "epoch": 4.04,
      "learning_rate": 0.00012517840909090908,
      "loss": 2.8831,
      "step": 69600
    },
    {
      "activations/layer0_attention_weight_max": 16.468257904052734,
      "activations/layer0_attention_weight_min": -10.679800033569336,
      "activations/layer10_attention_weight_max": 32.548194885253906,
      "activations/layer10_attention_weight_min": -30.689231872558594,
      "activations/layer11_attention_weight_max": 31.961727142333984,
      "activations/layer11_attention_weight_min": -29.301719665527344,
      "activations/layer12_attention_weight_max": 29.343788146972656,
      "activations/layer12_attention_weight_min": -25.40719223022461,
      "activations/layer13_attention_weight_max": 48.04285430908203,
      "activations/layer13_attention_weight_min": -36.30570983886719,
      "activations/layer14_attention_weight_max": 64.81942749023438,
      "activations/layer14_attention_weight_min": -47.9444694519043,
      "activations/layer15_attention_weight_max": 43.977447509765625,
      "activations/layer15_attention_weight_min": -30.203651428222656,
      "activations/layer16_attention_weight_max": 40.981407165527344,
      "activations/layer16_attention_weight_min": -28.525941848754883,
      "activations/layer17_attention_weight_max": 45.626773834228516,
      "activations/layer17_attention_weight_min": -27.024520874023438,
      "activations/layer18_attention_weight_max": 30.838266372680664,
      "activations/layer18_attention_weight_min": -19.285768508911133,
      "activations/layer19_attention_weight_max": 38.05487060546875,
      "activations/layer19_attention_weight_min": -31.729938507080078,
      "activations/layer1_attention_weight_max": 18.02533531188965,
      "activations/layer1_attention_weight_min": -14.764914512634277,
      "activations/layer20_attention_weight_max": 30.097747802734375,
      "activations/layer20_attention_weight_min": -23.042577743530273,
      "activations/layer21_attention_weight_max": 30.44127655029297,
      "activations/layer21_attention_weight_min": -20.990610122680664,
      "activations/layer22_attention_weight_max": 50.90625762939453,
      "activations/layer22_attention_weight_min": -30.418752670288086,
      "activations/layer23_attention_weight_max": 29.14716339111328,
      "activations/layer23_attention_weight_min": -18.975337982177734,
      "activations/layer2_attention_weight_max": 30.507495880126953,
      "activations/layer2_attention_weight_min": -29.860706329345703,
      "activations/layer3_attention_weight_max": 88.20109558105469,
      "activations/layer3_attention_weight_min": -89.18067932128906,
      "activations/layer4_attention_weight_max": 125.0013427734375,
      "activations/layer4_attention_weight_min": -124.238037109375,
      "activations/layer5_attention_weight_max": 84.43025970458984,
      "activations/layer5_attention_weight_min": -81.82371520996094,
      "activations/layer6_attention_weight_max": 55.05604553222656,
      "activations/layer6_attention_weight_min": -54.996978759765625,
      "activations/layer7_attention_weight_max": 74.9430923461914,
      "activations/layer7_attention_weight_min": -74.09709167480469,
      "activations/layer8_attention_weight_max": 37.2604866027832,
      "activations/layer8_attention_weight_min": -37.28692626953125,
      "activations/layer9_attention_weight_max": 36.241371154785156,
      "activations/layer9_attention_weight_min": -39.19918441772461,
      "epoch": 4.05,
      "learning_rate": 0.0001251594696969697,
      "loss": 2.8867,
      "step": 69650
    },
    {
      "activations/layer0_attention_weight_max": 16.715438842773438,
      "activations/layer0_attention_weight_min": -12.818168640136719,
      "activations/layer10_attention_weight_max": 39.05818557739258,
      "activations/layer10_attention_weight_min": -33.64530944824219,
      "activations/layer11_attention_weight_max": 30.519088745117188,
      "activations/layer11_attention_weight_min": -26.74897575378418,
      "activations/layer12_attention_weight_max": 25.525222778320312,
      "activations/layer12_attention_weight_min": -30.489774703979492,
      "activations/layer13_attention_weight_max": 41.7948112487793,
      "activations/layer13_attention_weight_min": -32.11338424682617,
      "activations/layer14_attention_weight_max": 61.898597717285156,
      "activations/layer14_attention_weight_min": -45.308990478515625,
      "activations/layer15_attention_weight_max": 42.84054946899414,
      "activations/layer15_attention_weight_min": -29.1057186126709,
      "activations/layer16_attention_weight_max": 40.6719970703125,
      "activations/layer16_attention_weight_min": -32.40940856933594,
      "activations/layer17_attention_weight_max": 40.75279235839844,
      "activations/layer17_attention_weight_min": -26.36222267150879,
      "activations/layer18_attention_weight_max": 34.92665481567383,
      "activations/layer18_attention_weight_min": -20.951725006103516,
      "activations/layer19_attention_weight_max": 43.019493103027344,
      "activations/layer19_attention_weight_min": -32.5155029296875,
      "activations/layer1_attention_weight_max": 17.852373123168945,
      "activations/layer1_attention_weight_min": -13.969134330749512,
      "activations/layer20_attention_weight_max": 37.714351654052734,
      "activations/layer20_attention_weight_min": -23.113197326660156,
      "activations/layer21_attention_weight_max": 33.317928314208984,
      "activations/layer21_attention_weight_min": -20.896387100219727,
      "activations/layer22_attention_weight_max": 51.80742263793945,
      "activations/layer22_attention_weight_min": -27.36141586303711,
      "activations/layer23_attention_weight_max": 32.340538024902344,
      "activations/layer23_attention_weight_min": -18.981304168701172,
      "activations/layer2_attention_weight_max": 30.815288543701172,
      "activations/layer2_attention_weight_min": -31.225889205932617,
      "activations/layer3_attention_weight_max": 85.20716857910156,
      "activations/layer3_attention_weight_min": -87.48970031738281,
      "activations/layer4_attention_weight_max": 133.11683654785156,
      "activations/layer4_attention_weight_min": -133.76885986328125,
      "activations/layer5_attention_weight_max": 90.03526306152344,
      "activations/layer5_attention_weight_min": -88.08876037597656,
      "activations/layer6_attention_weight_max": 56.24991226196289,
      "activations/layer6_attention_weight_min": -55.64815139770508,
      "activations/layer7_attention_weight_max": 78.81280517578125,
      "activations/layer7_attention_weight_min": -77.71123504638672,
      "activations/layer8_attention_weight_max": 39.30222702026367,
      "activations/layer8_attention_weight_min": -40.08435821533203,
      "activations/layer9_attention_weight_max": 43.49937438964844,
      "activations/layer9_attention_weight_min": -44.3430290222168,
      "epoch": 4.05,
      "learning_rate": 0.0001251405303030303,
      "loss": 2.8728,
      "step": 69700
    },
    {
      "activations/layer0_attention_weight_max": 16.205846786499023,
      "activations/layer0_attention_weight_min": -12.660039901733398,
      "activations/layer10_attention_weight_max": 38.92150115966797,
      "activations/layer10_attention_weight_min": -31.810420989990234,
      "activations/layer11_attention_weight_max": 35.15492248535156,
      "activations/layer11_attention_weight_min": -28.101383209228516,
      "activations/layer12_attention_weight_max": 40.84775161743164,
      "activations/layer12_attention_weight_min": -27.3284854888916,
      "activations/layer13_attention_weight_max": 50.66486358642578,
      "activations/layer13_attention_weight_min": -35.47698974609375,
      "activations/layer14_attention_weight_max": 87.01605987548828,
      "activations/layer14_attention_weight_min": -57.23615264892578,
      "activations/layer15_attention_weight_max": 43.064212799072266,
      "activations/layer15_attention_weight_min": -28.19150733947754,
      "activations/layer16_attention_weight_max": 43.09180450439453,
      "activations/layer16_attention_weight_min": -29.655467987060547,
      "activations/layer17_attention_weight_max": 44.93777847290039,
      "activations/layer17_attention_weight_min": -25.69547462463379,
      "activations/layer18_attention_weight_max": 31.02924156188965,
      "activations/layer18_attention_weight_min": -18.925020217895508,
      "activations/layer19_attention_weight_max": 42.64105224609375,
      "activations/layer19_attention_weight_min": -29.729211807250977,
      "activations/layer1_attention_weight_max": 17.873600006103516,
      "activations/layer1_attention_weight_min": -13.787972450256348,
      "activations/layer20_attention_weight_max": 34.7939453125,
      "activations/layer20_attention_weight_min": -21.835247039794922,
      "activations/layer21_attention_weight_max": 38.726009368896484,
      "activations/layer21_attention_weight_min": -21.01680564880371,
      "activations/layer22_attention_weight_max": 52.72946548461914,
      "activations/layer22_attention_weight_min": -28.806562423706055,
      "activations/layer23_attention_weight_max": 30.09737205505371,
      "activations/layer23_attention_weight_min": -17.13397979736328,
      "activations/layer2_attention_weight_max": 32.496070861816406,
      "activations/layer2_attention_weight_min": -31.76852035522461,
      "activations/layer3_attention_weight_max": 86.37468719482422,
      "activations/layer3_attention_weight_min": -86.94223022460938,
      "activations/layer4_attention_weight_max": 136.14317321777344,
      "activations/layer4_attention_weight_min": -123.686767578125,
      "activations/layer5_attention_weight_max": 95.44564819335938,
      "activations/layer5_attention_weight_min": -81.7745132446289,
      "activations/layer6_attention_weight_max": 60.86569595336914,
      "activations/layer6_attention_weight_min": -60.89745330810547,
      "activations/layer7_attention_weight_max": 86.43798828125,
      "activations/layer7_attention_weight_min": -78.65409088134766,
      "activations/layer8_attention_weight_max": 43.499874114990234,
      "activations/layer8_attention_weight_min": -41.59697723388672,
      "activations/layer9_attention_weight_max": 42.17021942138672,
      "activations/layer9_attention_weight_min": -34.21923828125,
      "epoch": 4.05,
      "learning_rate": 0.0001251215909090909,
      "loss": 2.8316,
      "step": 69750
    },
    {
      "activations/layer0_attention_weight_max": 14.937849044799805,
      "activations/layer0_attention_weight_min": -12.748305320739746,
      "activations/layer10_attention_weight_max": 36.40040969848633,
      "activations/layer10_attention_weight_min": -29.735626220703125,
      "activations/layer11_attention_weight_max": 35.34723663330078,
      "activations/layer11_attention_weight_min": -25.72671890258789,
      "activations/layer12_attention_weight_max": 26.843053817749023,
      "activations/layer12_attention_weight_min": -24.910629272460938,
      "activations/layer13_attention_weight_max": 54.81787872314453,
      "activations/layer13_attention_weight_min": -35.954010009765625,
      "activations/layer14_attention_weight_max": 56.91371154785156,
      "activations/layer14_attention_weight_min": -44.71053695678711,
      "activations/layer15_attention_weight_max": 39.357940673828125,
      "activations/layer15_attention_weight_min": -28.420583724975586,
      "activations/layer16_attention_weight_max": 36.60858917236328,
      "activations/layer16_attention_weight_min": -29.72480010986328,
      "activations/layer17_attention_weight_max": 39.4169807434082,
      "activations/layer17_attention_weight_min": -24.411046981811523,
      "activations/layer18_attention_weight_max": 29.6986141204834,
      "activations/layer18_attention_weight_min": -19.18607521057129,
      "activations/layer19_attention_weight_max": 37.0230827331543,
      "activations/layer19_attention_weight_min": -28.355192184448242,
      "activations/layer1_attention_weight_max": 17.68499183654785,
      "activations/layer1_attention_weight_min": -14.077013969421387,
      "activations/layer20_attention_weight_max": 30.41201400756836,
      "activations/layer20_attention_weight_min": -23.316749572753906,
      "activations/layer21_attention_weight_max": 30.69589614868164,
      "activations/layer21_attention_weight_min": -21.270055770874023,
      "activations/layer22_attention_weight_max": 44.81243133544922,
      "activations/layer22_attention_weight_min": -29.553361892700195,
      "activations/layer23_attention_weight_max": 25.61884880065918,
      "activations/layer23_attention_weight_min": -17.420625686645508,
      "activations/layer2_attention_weight_max": 33.40224075317383,
      "activations/layer2_attention_weight_min": -31.96734619140625,
      "activations/layer3_attention_weight_max": 83.6574478149414,
      "activations/layer3_attention_weight_min": -86.28965759277344,
      "activations/layer4_attention_weight_max": 130.26951599121094,
      "activations/layer4_attention_weight_min": -125.77108001708984,
      "activations/layer5_attention_weight_max": 86.58673095703125,
      "activations/layer5_attention_weight_min": -81.09081268310547,
      "activations/layer6_attention_weight_max": 56.23566436767578,
      "activations/layer6_attention_weight_min": -55.0505485534668,
      "activations/layer7_attention_weight_max": 81.93102264404297,
      "activations/layer7_attention_weight_min": -77.15513610839844,
      "activations/layer8_attention_weight_max": 37.804481506347656,
      "activations/layer8_attention_weight_min": -38.71028518676758,
      "activations/layer9_attention_weight_max": 37.239967346191406,
      "activations/layer9_attention_weight_min": -34.42070388793945,
      "epoch": 4.06,
      "learning_rate": 0.00012510265151515152,
      "loss": 2.8448,
      "step": 69800
    },
    {
      "activations/layer0_attention_weight_max": 17.511991500854492,
      "activations/layer0_attention_weight_min": -11.817647933959961,
      "activations/layer10_attention_weight_max": 36.40643310546875,
      "activations/layer10_attention_weight_min": -30.863853454589844,
      "activations/layer11_attention_weight_max": 35.154136657714844,
      "activations/layer11_attention_weight_min": -26.873149871826172,
      "activations/layer12_attention_weight_max": 35.515289306640625,
      "activations/layer12_attention_weight_min": -27.116090774536133,
      "activations/layer13_attention_weight_max": 50.329891204833984,
      "activations/layer13_attention_weight_min": -34.546871185302734,
      "activations/layer14_attention_weight_max": 68.72762298583984,
      "activations/layer14_attention_weight_min": -45.49386215209961,
      "activations/layer15_attention_weight_max": 49.95335388183594,
      "activations/layer15_attention_weight_min": -32.35126876831055,
      "activations/layer16_attention_weight_max": 44.77816390991211,
      "activations/layer16_attention_weight_min": -32.37879180908203,
      "activations/layer17_attention_weight_max": 44.43123245239258,
      "activations/layer17_attention_weight_min": -27.277790069580078,
      "activations/layer18_attention_weight_max": 35.077701568603516,
      "activations/layer18_attention_weight_min": -18.896968841552734,
      "activations/layer19_attention_weight_max": 43.966346740722656,
      "activations/layer19_attention_weight_min": -31.014896392822266,
      "activations/layer1_attention_weight_max": 17.8602352142334,
      "activations/layer1_attention_weight_min": -13.70924186706543,
      "activations/layer20_attention_weight_max": 36.169578552246094,
      "activations/layer20_attention_weight_min": -23.439699172973633,
      "activations/layer21_attention_weight_max": 37.565589904785156,
      "activations/layer21_attention_weight_min": -20.386337280273438,
      "activations/layer22_attention_weight_max": 53.60462951660156,
      "activations/layer22_attention_weight_min": -26.743879318237305,
      "activations/layer23_attention_weight_max": 31.600749969482422,
      "activations/layer23_attention_weight_min": -17.54375457763672,
      "activations/layer2_attention_weight_max": 32.000885009765625,
      "activations/layer2_attention_weight_min": -32.12373733520508,
      "activations/layer3_attention_weight_max": 87.95455932617188,
      "activations/layer3_attention_weight_min": -92.3935317993164,
      "activations/layer4_attention_weight_max": 130.2761688232422,
      "activations/layer4_attention_weight_min": -135.35826110839844,
      "activations/layer5_attention_weight_max": 88.85860443115234,
      "activations/layer5_attention_weight_min": -87.45692443847656,
      "activations/layer6_attention_weight_max": 58.901939392089844,
      "activations/layer6_attention_weight_min": -58.331974029541016,
      "activations/layer7_attention_weight_max": 81.93820190429688,
      "activations/layer7_attention_weight_min": -80.36530303955078,
      "activations/layer8_attention_weight_max": 39.62959671020508,
      "activations/layer8_attention_weight_min": -38.33155059814453,
      "activations/layer9_attention_weight_max": 38.82191848754883,
      "activations/layer9_attention_weight_min": -33.0759162902832,
      "epoch": 4.06,
      "learning_rate": 0.0001250837121212121,
      "loss": 2.8899,
      "step": 69850
    },
    {
      "activations/layer0_attention_weight_max": 17.493221282958984,
      "activations/layer0_attention_weight_min": -11.78213119506836,
      "activations/layer10_attention_weight_max": 36.75281524658203,
      "activations/layer10_attention_weight_min": -31.554784774780273,
      "activations/layer11_attention_weight_max": 34.85735321044922,
      "activations/layer11_attention_weight_min": -27.801979064941406,
      "activations/layer12_attention_weight_max": 31.338443756103516,
      "activations/layer12_attention_weight_min": -27.732784271240234,
      "activations/layer13_attention_weight_max": 63.556915283203125,
      "activations/layer13_attention_weight_min": -40.1501350402832,
      "activations/layer14_attention_weight_max": 85.18518829345703,
      "activations/layer14_attention_weight_min": -56.53995132446289,
      "activations/layer15_attention_weight_max": 53.24550247192383,
      "activations/layer15_attention_weight_min": -34.08414840698242,
      "activations/layer16_attention_weight_max": 46.66935729980469,
      "activations/layer16_attention_weight_min": -29.670883178710938,
      "activations/layer17_attention_weight_max": 44.58232116699219,
      "activations/layer17_attention_weight_min": -25.01580810546875,
      "activations/layer18_attention_weight_max": 34.52653503417969,
      "activations/layer18_attention_weight_min": -21.001848220825195,
      "activations/layer19_attention_weight_max": 49.0007209777832,
      "activations/layer19_attention_weight_min": -30.519729614257812,
      "activations/layer1_attention_weight_max": 17.253097534179688,
      "activations/layer1_attention_weight_min": -14.85698127746582,
      "activations/layer20_attention_weight_max": 37.77145767211914,
      "activations/layer20_attention_weight_min": -24.185157775878906,
      "activations/layer21_attention_weight_max": 33.368019104003906,
      "activations/layer21_attention_weight_min": -19.531843185424805,
      "activations/layer22_attention_weight_max": 57.65875244140625,
      "activations/layer22_attention_weight_min": -29.24178695678711,
      "activations/layer23_attention_weight_max": 37.694366455078125,
      "activations/layer23_attention_weight_min": -19.9534854888916,
      "activations/layer2_attention_weight_max": 34.144920349121094,
      "activations/layer2_attention_weight_min": -31.133438110351562,
      "activations/layer3_attention_weight_max": 90.67134094238281,
      "activations/layer3_attention_weight_min": -89.22943115234375,
      "activations/layer4_attention_weight_max": 139.4536590576172,
      "activations/layer4_attention_weight_min": -133.6853485107422,
      "activations/layer5_attention_weight_max": 95.79149627685547,
      "activations/layer5_attention_weight_min": -83.6649169921875,
      "activations/layer6_attention_weight_max": 59.64912796020508,
      "activations/layer6_attention_weight_min": -60.12754440307617,
      "activations/layer7_attention_weight_max": 76.47418212890625,
      "activations/layer7_attention_weight_min": -80.0286865234375,
      "activations/layer8_attention_weight_max": 44.135799407958984,
      "activations/layer8_attention_weight_min": -39.8317985534668,
      "activations/layer9_attention_weight_max": 39.65909957885742,
      "activations/layer9_attention_weight_min": -35.229854583740234,
      "epoch": 4.06,
      "learning_rate": 0.00012506477272727272,
      "loss": 2.8743,
      "step": 69900
    },
    {
      "activations/layer0_attention_weight_max": 17.155597686767578,
      "activations/layer0_attention_weight_min": -11.455586433410645,
      "activations/layer10_attention_weight_max": 36.17027282714844,
      "activations/layer10_attention_weight_min": -30.085594177246094,
      "activations/layer11_attention_weight_max": 34.42973709106445,
      "activations/layer11_attention_weight_min": -27.35174560546875,
      "activations/layer12_attention_weight_max": 45.99933624267578,
      "activations/layer12_attention_weight_min": -30.926029205322266,
      "activations/layer13_attention_weight_max": 48.8360710144043,
      "activations/layer13_attention_weight_min": -33.440303802490234,
      "activations/layer14_attention_weight_max": 69.83634948730469,
      "activations/layer14_attention_weight_min": -50.996482849121094,
      "activations/layer15_attention_weight_max": 45.71521759033203,
      "activations/layer15_attention_weight_min": -28.95340919494629,
      "activations/layer16_attention_weight_max": 44.333831787109375,
      "activations/layer16_attention_weight_min": -30.007505416870117,
      "activations/layer17_attention_weight_max": 43.48965072631836,
      "activations/layer17_attention_weight_min": -25.82112693786621,
      "activations/layer18_attention_weight_max": 33.61466979980469,
      "activations/layer18_attention_weight_min": -21.0391902923584,
      "activations/layer19_attention_weight_max": 46.1898307800293,
      "activations/layer19_attention_weight_min": -29.40171241760254,
      "activations/layer1_attention_weight_max": 17.405866622924805,
      "activations/layer1_attention_weight_min": -13.667153358459473,
      "activations/layer20_attention_weight_max": 34.072444915771484,
      "activations/layer20_attention_weight_min": -24.233800888061523,
      "activations/layer21_attention_weight_max": 33.668006896972656,
      "activations/layer21_attention_weight_min": -21.331220626831055,
      "activations/layer22_attention_weight_max": 48.956050872802734,
      "activations/layer22_attention_weight_min": -28.396453857421875,
      "activations/layer23_attention_weight_max": 28.664175033569336,
      "activations/layer23_attention_weight_min": -20.980127334594727,
      "activations/layer2_attention_weight_max": 33.41497802734375,
      "activations/layer2_attention_weight_min": -31.542339324951172,
      "activations/layer3_attention_weight_max": 88.94133758544922,
      "activations/layer3_attention_weight_min": -88.38885498046875,
      "activations/layer4_attention_weight_max": 139.77377319335938,
      "activations/layer4_attention_weight_min": -128.79888916015625,
      "activations/layer5_attention_weight_max": 93.94981384277344,
      "activations/layer5_attention_weight_min": -82.02662658691406,
      "activations/layer6_attention_weight_max": 58.80082321166992,
      "activations/layer6_attention_weight_min": -55.108360290527344,
      "activations/layer7_attention_weight_max": 83.179443359375,
      "activations/layer7_attention_weight_min": -78.08496856689453,
      "activations/layer8_attention_weight_max": 41.349090576171875,
      "activations/layer8_attention_weight_min": -37.952571868896484,
      "activations/layer9_attention_weight_max": 41.5821418762207,
      "activations/layer9_attention_weight_min": -32.742591857910156,
      "epoch": 4.06,
      "learning_rate": 0.00012504583333333333,
      "loss": 2.8655,
      "step": 69950
    },
    {
      "activations/layer0_attention_weight_max": 17.447616577148438,
      "activations/layer0_attention_weight_min": -11.543819427490234,
      "activations/layer10_attention_weight_max": 36.397586822509766,
      "activations/layer10_attention_weight_min": -29.584081649780273,
      "activations/layer11_attention_weight_max": 33.939247131347656,
      "activations/layer11_attention_weight_min": -28.647741317749023,
      "activations/layer12_attention_weight_max": 33.55715560913086,
      "activations/layer12_attention_weight_min": -27.683069229125977,
      "activations/layer13_attention_weight_max": 47.36436080932617,
      "activations/layer13_attention_weight_min": -32.0139045715332,
      "activations/layer14_attention_weight_max": 64.54998016357422,
      "activations/layer14_attention_weight_min": -44.09321594238281,
      "activations/layer15_attention_weight_max": 43.40792465209961,
      "activations/layer15_attention_weight_min": -28.338071823120117,
      "activations/layer16_attention_weight_max": 39.70048141479492,
      "activations/layer16_attention_weight_min": -29.31035041809082,
      "activations/layer17_attention_weight_max": 43.17021560668945,
      "activations/layer17_attention_weight_min": -25.35738754272461,
      "activations/layer18_attention_weight_max": 34.59054946899414,
      "activations/layer18_attention_weight_min": -20.70738983154297,
      "activations/layer19_attention_weight_max": 47.7407341003418,
      "activations/layer19_attention_weight_min": -28.738895416259766,
      "activations/layer1_attention_weight_max": 18.39299201965332,
      "activations/layer1_attention_weight_min": -13.795578002929688,
      "activations/layer20_attention_weight_max": 33.93627166748047,
      "activations/layer20_attention_weight_min": -24.408052444458008,
      "activations/layer21_attention_weight_max": 38.262489318847656,
      "activations/layer21_attention_weight_min": -21.280927658081055,
      "activations/layer22_attention_weight_max": 55.277503967285156,
      "activations/layer22_attention_weight_min": -27.342561721801758,
      "activations/layer23_attention_weight_max": 36.31543731689453,
      "activations/layer23_attention_weight_min": -18.405010223388672,
      "activations/layer2_attention_weight_max": 31.950489044189453,
      "activations/layer2_attention_weight_min": -32.25769805908203,
      "activations/layer3_attention_weight_max": 96.05758666992188,
      "activations/layer3_attention_weight_min": -91.00111389160156,
      "activations/layer4_attention_weight_max": 137.95237731933594,
      "activations/layer4_attention_weight_min": -127.00716400146484,
      "activations/layer5_attention_weight_max": 89.02449035644531,
      "activations/layer5_attention_weight_min": -80.11344146728516,
      "activations/layer6_attention_weight_max": 56.24750900268555,
      "activations/layer6_attention_weight_min": -57.27642059326172,
      "activations/layer7_attention_weight_max": 80.78750610351562,
      "activations/layer7_attention_weight_min": -75.54674530029297,
      "activations/layer8_attention_weight_max": 40.64509963989258,
      "activations/layer8_attention_weight_min": -38.27119445800781,
      "activations/layer9_attention_weight_max": 36.208473205566406,
      "activations/layer9_attention_weight_min": -38.62177658081055,
      "epoch": 4.07,
      "learning_rate": 0.00012502689393939392,
      "loss": 2.8773,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4158,
      "eval_samples_per_second": 510.23,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4158,
      "eval_openwebtext_samples_per_second": 510.23,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_wikitext_loss": 3.060546875,
      "eval_wikitext_ppl": 21.33922385967176,
      "eval_wikitext_runtime": 1.9959,
      "eval_wikitext_samples_per_second": 228.466,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_lambada_loss": 2.853515625,
      "eval_lambada_ppl": 17.348666159108166,
      "eval_lambada_runtime": 9.5736,
      "eval_lambada_samples_per_second": 508.589,
      "step": 70000
    },
    {
      "activations/layer0_attention_weight_max": 16.86480140686035,
      "activations/layer0_attention_weight_min": -12.567450523376465,
      "activations/layer10_attention_weight_max": 35.351043701171875,
      "activations/layer10_attention_weight_min": -31.587804794311523,
      "activations/layer11_attention_weight_max": 31.54480743408203,
      "activations/layer11_attention_weight_min": -27.18890380859375,
      "activations/layer12_attention_weight_max": 24.395748138427734,
      "activations/layer12_attention_weight_min": -26.761598587036133,
      "activations/layer13_attention_weight_max": 49.026573181152344,
      "activations/layer13_attention_weight_min": -35.52262878417969,
      "activations/layer14_attention_weight_max": 64.1329116821289,
      "activations/layer14_attention_weight_min": -47.52973175048828,
      "activations/layer15_attention_weight_max": 41.151729583740234,
      "activations/layer15_attention_weight_min": -29.256423950195312,
      "activations/layer16_attention_weight_max": 38.11287307739258,
      "activations/layer16_attention_weight_min": -27.299327850341797,
      "activations/layer17_attention_weight_max": 44.03843688964844,
      "activations/layer17_attention_weight_min": -25.571823120117188,
      "activations/layer18_attention_weight_max": 32.2847785949707,
      "activations/layer18_attention_weight_min": -20.612728118896484,
      "activations/layer19_attention_weight_max": 44.5782470703125,
      "activations/layer19_attention_weight_min": -28.35890769958496,
      "activations/layer1_attention_weight_max": 18.2862491607666,
      "activations/layer1_attention_weight_min": -15.2746000289917,
      "activations/layer20_attention_weight_max": 36.04578399658203,
      "activations/layer20_attention_weight_min": -22.683433532714844,
      "activations/layer21_attention_weight_max": 32.17633056640625,
      "activations/layer21_attention_weight_min": -19.544437408447266,
      "activations/layer22_attention_weight_max": 49.566444396972656,
      "activations/layer22_attention_weight_min": -30.705902099609375,
      "activations/layer23_attention_weight_max": 31.69054412841797,
      "activations/layer23_attention_weight_min": -18.39099884033203,
      "activations/layer2_attention_weight_max": 31.30707550048828,
      "activations/layer2_attention_weight_min": -30.884445190429688,
      "activations/layer3_attention_weight_max": 87.16876220703125,
      "activations/layer3_attention_weight_min": -86.65677642822266,
      "activations/layer4_attention_weight_max": 132.17855834960938,
      "activations/layer4_attention_weight_min": -128.23825073242188,
      "activations/layer5_attention_weight_max": 88.74618530273438,
      "activations/layer5_attention_weight_min": -84.39382934570312,
      "activations/layer6_attention_weight_max": 56.738502502441406,
      "activations/layer6_attention_weight_min": -59.28934097290039,
      "activations/layer7_attention_weight_max": 77.33279418945312,
      "activations/layer7_attention_weight_min": -79.89361572265625,
      "activations/layer8_attention_weight_max": 40.24860763549805,
      "activations/layer8_attention_weight_min": -39.84665298461914,
      "activations/layer9_attention_weight_max": 39.740989685058594,
      "activations/layer9_attention_weight_min": -34.934940338134766,
      "epoch": 4.07,
      "learning_rate": 0.00012500795454545454,
      "loss": 2.8691,
      "step": 70050
    },
    {
      "activations/layer0_attention_weight_max": 17.066036224365234,
      "activations/layer0_attention_weight_min": -11.77235221862793,
      "activations/layer10_attention_weight_max": 37.99949645996094,
      "activations/layer10_attention_weight_min": -29.335886001586914,
      "activations/layer11_attention_weight_max": 34.218505859375,
      "activations/layer11_attention_weight_min": -28.512296676635742,
      "activations/layer12_attention_weight_max": 23.546300888061523,
      "activations/layer12_attention_weight_min": -24.954179763793945,
      "activations/layer13_attention_weight_max": 41.3267936706543,
      "activations/layer13_attention_weight_min": -32.00960159301758,
      "activations/layer14_attention_weight_max": 59.63071060180664,
      "activations/layer14_attention_weight_min": -49.96842956542969,
      "activations/layer15_attention_weight_max": 42.534339904785156,
      "activations/layer15_attention_weight_min": -32.40529251098633,
      "activations/layer16_attention_weight_max": 42.871124267578125,
      "activations/layer16_attention_weight_min": -28.847949981689453,
      "activations/layer17_attention_weight_max": 39.9340705871582,
      "activations/layer17_attention_weight_min": -26.126989364624023,
      "activations/layer18_attention_weight_max": 27.153879165649414,
      "activations/layer18_attention_weight_min": -20.80674171447754,
      "activations/layer19_attention_weight_max": 42.271663665771484,
      "activations/layer19_attention_weight_min": -28.519821166992188,
      "activations/layer1_attention_weight_max": 17.573780059814453,
      "activations/layer1_attention_weight_min": -13.829768180847168,
      "activations/layer20_attention_weight_max": 29.394214630126953,
      "activations/layer20_attention_weight_min": -22.931089401245117,
      "activations/layer21_attention_weight_max": 30.305526733398438,
      "activations/layer21_attention_weight_min": -19.833518981933594,
      "activations/layer22_attention_weight_max": 43.9698371887207,
      "activations/layer22_attention_weight_min": -26.644651412963867,
      "activations/layer23_attention_weight_max": 27.832408905029297,
      "activations/layer23_attention_weight_min": -19.17362403869629,
      "activations/layer2_attention_weight_max": 31.228981018066406,
      "activations/layer2_attention_weight_min": -31.51803207397461,
      "activations/layer3_attention_weight_max": 86.05506896972656,
      "activations/layer3_attention_weight_min": -87.62969970703125,
      "activations/layer4_attention_weight_max": 136.15440368652344,
      "activations/layer4_attention_weight_min": -134.89418029785156,
      "activations/layer5_attention_weight_max": 91.73406982421875,
      "activations/layer5_attention_weight_min": -87.55313110351562,
      "activations/layer6_attention_weight_max": 58.002418518066406,
      "activations/layer6_attention_weight_min": -55.928070068359375,
      "activations/layer7_attention_weight_max": 77.90936279296875,
      "activations/layer7_attention_weight_min": -75.61512756347656,
      "activations/layer8_attention_weight_max": 39.68950271606445,
      "activations/layer8_attention_weight_min": -40.784324645996094,
      "activations/layer9_attention_weight_max": 41.85049057006836,
      "activations/layer9_attention_weight_min": -34.62702560424805,
      "epoch": 4.07,
      "learning_rate": 0.00012498901515151515,
      "loss": 2.8676,
      "step": 70100
    },
    {
      "activations/layer0_attention_weight_max": 16.09061050415039,
      "activations/layer0_attention_weight_min": -12.60200309753418,
      "activations/layer10_attention_weight_max": 35.85889434814453,
      "activations/layer10_attention_weight_min": -31.122995376586914,
      "activations/layer11_attention_weight_max": 34.45549774169922,
      "activations/layer11_attention_weight_min": -27.68794059753418,
      "activations/layer12_attention_weight_max": 37.902191162109375,
      "activations/layer12_attention_weight_min": -27.551918029785156,
      "activations/layer13_attention_weight_max": 48.262046813964844,
      "activations/layer13_attention_weight_min": -34.0247802734375,
      "activations/layer14_attention_weight_max": 97.37397003173828,
      "activations/layer14_attention_weight_min": -63.06377029418945,
      "activations/layer15_attention_weight_max": 44.88418197631836,
      "activations/layer15_attention_weight_min": -30.254810333251953,
      "activations/layer16_attention_weight_max": 43.98085021972656,
      "activations/layer16_attention_weight_min": -31.40247344970703,
      "activations/layer17_attention_weight_max": 42.831302642822266,
      "activations/layer17_attention_weight_min": -25.628828048706055,
      "activations/layer18_attention_weight_max": 31.560977935791016,
      "activations/layer18_attention_weight_min": -22.585325241088867,
      "activations/layer19_attention_weight_max": 46.57481002807617,
      "activations/layer19_attention_weight_min": -29.228025436401367,
      "activations/layer1_attention_weight_max": 16.611379623413086,
      "activations/layer1_attention_weight_min": -13.505294799804688,
      "activations/layer20_attention_weight_max": 36.667686462402344,
      "activations/layer20_attention_weight_min": -23.33879852294922,
      "activations/layer21_attention_weight_max": 30.97868537902832,
      "activations/layer21_attention_weight_min": -21.954580307006836,
      "activations/layer22_attention_weight_max": 52.59563064575195,
      "activations/layer22_attention_weight_min": -29.365341186523438,
      "activations/layer23_attention_weight_max": 34.5857048034668,
      "activations/layer23_attention_weight_min": -19.06148338317871,
      "activations/layer2_attention_weight_max": 33.29667282104492,
      "activations/layer2_attention_weight_min": -30.927265167236328,
      "activations/layer3_attention_weight_max": 89.98444366455078,
      "activations/layer3_attention_weight_min": -90.2320327758789,
      "activations/layer4_attention_weight_max": 137.3226318359375,
      "activations/layer4_attention_weight_min": -132.57672119140625,
      "activations/layer5_attention_weight_max": 96.36705780029297,
      "activations/layer5_attention_weight_min": -84.9400634765625,
      "activations/layer6_attention_weight_max": 60.353187561035156,
      "activations/layer6_attention_weight_min": -57.6524543762207,
      "activations/layer7_attention_weight_max": 79.77291107177734,
      "activations/layer7_attention_weight_min": -77.72555541992188,
      "activations/layer8_attention_weight_max": 39.80403137207031,
      "activations/layer8_attention_weight_min": -36.81923294067383,
      "activations/layer9_attention_weight_max": 40.71585464477539,
      "activations/layer9_attention_weight_min": -34.22682571411133,
      "epoch": 4.08,
      "learning_rate": 0.00012497007575757574,
      "loss": 2.8535,
      "step": 70150
    },
    {
      "activations/layer0_attention_weight_max": 17.595502853393555,
      "activations/layer0_attention_weight_min": -12.433502197265625,
      "activations/layer10_attention_weight_max": 40.8216552734375,
      "activations/layer10_attention_weight_min": -33.43360900878906,
      "activations/layer11_attention_weight_max": 39.120121002197266,
      "activations/layer11_attention_weight_min": -30.143339157104492,
      "activations/layer12_attention_weight_max": 38.41981506347656,
      "activations/layer12_attention_weight_min": -31.419286727905273,
      "activations/layer13_attention_weight_max": 50.978538513183594,
      "activations/layer13_attention_weight_min": -35.75267028808594,
      "activations/layer14_attention_weight_max": 67.28970336914062,
      "activations/layer14_attention_weight_min": -46.601051330566406,
      "activations/layer15_attention_weight_max": 57.258567810058594,
      "activations/layer15_attention_weight_min": -38.27682113647461,
      "activations/layer16_attention_weight_max": 38.067386627197266,
      "activations/layer16_attention_weight_min": -28.823495864868164,
      "activations/layer17_attention_weight_max": 38.29317092895508,
      "activations/layer17_attention_weight_min": -26.0909366607666,
      "activations/layer18_attention_weight_max": 33.049407958984375,
      "activations/layer18_attention_weight_min": -22.62360191345215,
      "activations/layer19_attention_weight_max": 41.635520935058594,
      "activations/layer19_attention_weight_min": -30.89216423034668,
      "activations/layer1_attention_weight_max": 17.890384674072266,
      "activations/layer1_attention_weight_min": -16.26184844970703,
      "activations/layer20_attention_weight_max": 34.20025634765625,
      "activations/layer20_attention_weight_min": -22.43865394592285,
      "activations/layer21_attention_weight_max": 30.659696578979492,
      "activations/layer21_attention_weight_min": -20.375244140625,
      "activations/layer22_attention_weight_max": 51.19997787475586,
      "activations/layer22_attention_weight_min": -27.655258178710938,
      "activations/layer23_attention_weight_max": 36.280975341796875,
      "activations/layer23_attention_weight_min": -21.797443389892578,
      "activations/layer2_attention_weight_max": 35.182899475097656,
      "activations/layer2_attention_weight_min": -32.45648193359375,
      "activations/layer3_attention_weight_max": 92.74363708496094,
      "activations/layer3_attention_weight_min": -91.80785369873047,
      "activations/layer4_attention_weight_max": 139.03387451171875,
      "activations/layer4_attention_weight_min": -144.07923889160156,
      "activations/layer5_attention_weight_max": 94.22726440429688,
      "activations/layer5_attention_weight_min": -87.10717010498047,
      "activations/layer6_attention_weight_max": 57.644439697265625,
      "activations/layer6_attention_weight_min": -58.44597625732422,
      "activations/layer7_attention_weight_max": 86.05313873291016,
      "activations/layer7_attention_weight_min": -84.0251235961914,
      "activations/layer8_attention_weight_max": 42.92880630493164,
      "activations/layer8_attention_weight_min": -39.33209991455078,
      "activations/layer9_attention_weight_max": 42.0172119140625,
      "activations/layer9_attention_weight_min": -35.246395111083984,
      "epoch": 4.08,
      "learning_rate": 0.00012495151515151513,
      "loss": 2.8536,
      "step": 70200
    },
    {
      "activations/layer0_attention_weight_max": 17.325815200805664,
      "activations/layer0_attention_weight_min": -12.37206745147705,
      "activations/layer10_attention_weight_max": 33.196258544921875,
      "activations/layer10_attention_weight_min": -29.398921966552734,
      "activations/layer11_attention_weight_max": 31.315948486328125,
      "activations/layer11_attention_weight_min": -29.018653869628906,
      "activations/layer12_attention_weight_max": 27.838529586791992,
      "activations/layer12_attention_weight_min": -29.623531341552734,
      "activations/layer13_attention_weight_max": 42.00666809082031,
      "activations/layer13_attention_weight_min": -32.12590789794922,
      "activations/layer14_attention_weight_max": 63.07880783081055,
      "activations/layer14_attention_weight_min": -42.68427658081055,
      "activations/layer15_attention_weight_max": 46.08854675292969,
      "activations/layer15_attention_weight_min": -29.041423797607422,
      "activations/layer16_attention_weight_max": 41.12074279785156,
      "activations/layer16_attention_weight_min": -25.9692325592041,
      "activations/layer17_attention_weight_max": 37.6880989074707,
      "activations/layer17_attention_weight_min": -23.219499588012695,
      "activations/layer18_attention_weight_max": 30.339567184448242,
      "activations/layer18_attention_weight_min": -20.54582977294922,
      "activations/layer19_attention_weight_max": 42.47267150878906,
      "activations/layer19_attention_weight_min": -28.78877067565918,
      "activations/layer1_attention_weight_max": 17.347318649291992,
      "activations/layer1_attention_weight_min": -15.250136375427246,
      "activations/layer20_attention_weight_max": 36.7148323059082,
      "activations/layer20_attention_weight_min": -24.667861938476562,
      "activations/layer21_attention_weight_max": 29.954509735107422,
      "activations/layer21_attention_weight_min": -20.911222457885742,
      "activations/layer22_attention_weight_max": 48.17934799194336,
      "activations/layer22_attention_weight_min": -29.61884117126465,
      "activations/layer23_attention_weight_max": 29.755603790283203,
      "activations/layer23_attention_weight_min": -19.98378562927246,
      "activations/layer2_attention_weight_max": 31.14263916015625,
      "activations/layer2_attention_weight_min": -31.138723373413086,
      "activations/layer3_attention_weight_max": 83.68347930908203,
      "activations/layer3_attention_weight_min": -87.68892669677734,
      "activations/layer4_attention_weight_max": 128.79180908203125,
      "activations/layer4_attention_weight_min": -125.53749084472656,
      "activations/layer5_attention_weight_max": 86.07443237304688,
      "activations/layer5_attention_weight_min": -80.62467956542969,
      "activations/layer6_attention_weight_max": 54.016178131103516,
      "activations/layer6_attention_weight_min": -54.63071060180664,
      "activations/layer7_attention_weight_max": 76.29167175292969,
      "activations/layer7_attention_weight_min": -75.19943237304688,
      "activations/layer8_attention_weight_max": 39.22563552856445,
      "activations/layer8_attention_weight_min": -37.48789596557617,
      "activations/layer9_attention_weight_max": 36.049861907958984,
      "activations/layer9_attention_weight_min": -33.166927337646484,
      "epoch": 4.08,
      "learning_rate": 0.00012493257575757575,
      "loss": 2.8644,
      "step": 70250
    },
    {
      "activations/layer0_attention_weight_max": 16.568649291992188,
      "activations/layer0_attention_weight_min": -12.879218101501465,
      "activations/layer10_attention_weight_max": 39.26350021362305,
      "activations/layer10_attention_weight_min": -29.389354705810547,
      "activations/layer11_attention_weight_max": 38.42588806152344,
      "activations/layer11_attention_weight_min": -26.54698944091797,
      "activations/layer12_attention_weight_max": 33.27870559692383,
      "activations/layer12_attention_weight_min": -27.05769157409668,
      "activations/layer13_attention_weight_max": 54.86316680908203,
      "activations/layer13_attention_weight_min": -32.694366455078125,
      "activations/layer14_attention_weight_max": 75.25898742675781,
      "activations/layer14_attention_weight_min": -48.07544708251953,
      "activations/layer15_attention_weight_max": 47.03593826293945,
      "activations/layer15_attention_weight_min": -29.886383056640625,
      "activations/layer16_attention_weight_max": 48.01704406738281,
      "activations/layer16_attention_weight_min": -28.504241943359375,
      "activations/layer17_attention_weight_max": 43.409095764160156,
      "activations/layer17_attention_weight_min": -24.678211212158203,
      "activations/layer18_attention_weight_max": 33.812137603759766,
      "activations/layer18_attention_weight_min": -20.160789489746094,
      "activations/layer19_attention_weight_max": 43.01305389404297,
      "activations/layer19_attention_weight_min": -27.011886596679688,
      "activations/layer1_attention_weight_max": 18.195812225341797,
      "activations/layer1_attention_weight_min": -16.18958282470703,
      "activations/layer20_attention_weight_max": 35.214725494384766,
      "activations/layer20_attention_weight_min": -22.98151206970215,
      "activations/layer21_attention_weight_max": 35.417640686035156,
      "activations/layer21_attention_weight_min": -22.909353256225586,
      "activations/layer22_attention_weight_max": 56.361148834228516,
      "activations/layer22_attention_weight_min": -26.4404354095459,
      "activations/layer23_attention_weight_max": 32.9833984375,
      "activations/layer23_attention_weight_min": -18.954761505126953,
      "activations/layer2_attention_weight_max": 30.63234519958496,
      "activations/layer2_attention_weight_min": -30.39901351928711,
      "activations/layer3_attention_weight_max": 82.5771713256836,
      "activations/layer3_attention_weight_min": -84.17568969726562,
      "activations/layer4_attention_weight_max": 132.07528686523438,
      "activations/layer4_attention_weight_min": -126.5429458618164,
      "activations/layer5_attention_weight_max": 91.3294677734375,
      "activations/layer5_attention_weight_min": -85.87249755859375,
      "activations/layer6_attention_weight_max": 59.23345184326172,
      "activations/layer6_attention_weight_min": -56.411563873291016,
      "activations/layer7_attention_weight_max": 77.60186767578125,
      "activations/layer7_attention_weight_min": -73.05130767822266,
      "activations/layer8_attention_weight_max": 42.06721496582031,
      "activations/layer8_attention_weight_min": -37.1376953125,
      "activations/layer9_attention_weight_max": 39.659088134765625,
      "activations/layer9_attention_weight_min": -32.067344665527344,
      "epoch": 4.08,
      "learning_rate": 0.00012491363636363636,
      "loss": 2.8827,
      "step": 70300
    },
    {
      "activations/layer0_attention_weight_max": 14.561793327331543,
      "activations/layer0_attention_weight_min": -12.786840438842773,
      "activations/layer10_attention_weight_max": 34.62720489501953,
      "activations/layer10_attention_weight_min": -30.433164596557617,
      "activations/layer11_attention_weight_max": 31.5052490234375,
      "activations/layer11_attention_weight_min": -26.217628479003906,
      "activations/layer12_attention_weight_max": 25.48472785949707,
      "activations/layer12_attention_weight_min": -24.947463989257812,
      "activations/layer13_attention_weight_max": 47.66724395751953,
      "activations/layer13_attention_weight_min": -32.965362548828125,
      "activations/layer14_attention_weight_max": 63.355167388916016,
      "activations/layer14_attention_weight_min": -49.970458984375,
      "activations/layer15_attention_weight_max": 44.55140686035156,
      "activations/layer15_attention_weight_min": -30.020551681518555,
      "activations/layer16_attention_weight_max": 39.21971893310547,
      "activations/layer16_attention_weight_min": -28.212909698486328,
      "activations/layer17_attention_weight_max": 49.325130462646484,
      "activations/layer17_attention_weight_min": -24.500892639160156,
      "activations/layer18_attention_weight_max": 36.25673294067383,
      "activations/layer18_attention_weight_min": -18.917144775390625,
      "activations/layer19_attention_weight_max": 48.45587158203125,
      "activations/layer19_attention_weight_min": -29.52359390258789,
      "activations/layer1_attention_weight_max": 17.723346710205078,
      "activations/layer1_attention_weight_min": -14.70886516571045,
      "activations/layer20_attention_weight_max": 32.31181716918945,
      "activations/layer20_attention_weight_min": -23.552810668945312,
      "activations/layer21_attention_weight_max": 32.56145477294922,
      "activations/layer21_attention_weight_min": -19.732166290283203,
      "activations/layer22_attention_weight_max": 56.27324295043945,
      "activations/layer22_attention_weight_min": -27.46344757080078,
      "activations/layer23_attention_weight_max": 31.13849639892578,
      "activations/layer23_attention_weight_min": -17.449676513671875,
      "activations/layer2_attention_weight_max": 31.378881454467773,
      "activations/layer2_attention_weight_min": -31.83570098876953,
      "activations/layer3_attention_weight_max": 84.96043395996094,
      "activations/layer3_attention_weight_min": -86.16636657714844,
      "activations/layer4_attention_weight_max": 125.7329330444336,
      "activations/layer4_attention_weight_min": -127.79188537597656,
      "activations/layer5_attention_weight_max": 86.22999572753906,
      "activations/layer5_attention_weight_min": -84.92703247070312,
      "activations/layer6_attention_weight_max": 55.26762008666992,
      "activations/layer6_attention_weight_min": -56.115848541259766,
      "activations/layer7_attention_weight_max": 73.77918243408203,
      "activations/layer7_attention_weight_min": -75.58270263671875,
      "activations/layer8_attention_weight_max": 37.8301887512207,
      "activations/layer8_attention_weight_min": -39.90371322631836,
      "activations/layer9_attention_weight_max": 35.03611373901367,
      "activations/layer9_attention_weight_min": -32.952247619628906,
      "epoch": 4.09,
      "learning_rate": 0.00012489469696969695,
      "loss": 2.8616,
      "step": 70350
    },
    {
      "activations/layer0_attention_weight_max": 16.400766372680664,
      "activations/layer0_attention_weight_min": -11.38855266571045,
      "activations/layer10_attention_weight_max": 38.17666244506836,
      "activations/layer10_attention_weight_min": -31.491973876953125,
      "activations/layer11_attention_weight_max": 36.13514709472656,
      "activations/layer11_attention_weight_min": -29.322357177734375,
      "activations/layer12_attention_weight_max": 28.230369567871094,
      "activations/layer12_attention_weight_min": -26.232492446899414,
      "activations/layer13_attention_weight_max": 55.357826232910156,
      "activations/layer13_attention_weight_min": -38.619834899902344,
      "activations/layer14_attention_weight_max": 60.37163162231445,
      "activations/layer14_attention_weight_min": -42.91709518432617,
      "activations/layer15_attention_weight_max": 47.94062042236328,
      "activations/layer15_attention_weight_min": -27.707130432128906,
      "activations/layer16_attention_weight_max": 45.33650588989258,
      "activations/layer16_attention_weight_min": -29.784305572509766,
      "activations/layer17_attention_weight_max": 45.074283599853516,
      "activations/layer17_attention_weight_min": -25.650583267211914,
      "activations/layer18_attention_weight_max": 34.30989456176758,
      "activations/layer18_attention_weight_min": -20.703893661499023,
      "activations/layer19_attention_weight_max": 45.3140869140625,
      "activations/layer19_attention_weight_min": -28.135915756225586,
      "activations/layer1_attention_weight_max": 17.343900680541992,
      "activations/layer1_attention_weight_min": -13.770413398742676,
      "activations/layer20_attention_weight_max": 36.879188537597656,
      "activations/layer20_attention_weight_min": -22.61474609375,
      "activations/layer21_attention_weight_max": 31.87942123413086,
      "activations/layer21_attention_weight_min": -19.426044464111328,
      "activations/layer22_attention_weight_max": 51.74870681762695,
      "activations/layer22_attention_weight_min": -26.696308135986328,
      "activations/layer23_attention_weight_max": 29.74889373779297,
      "activations/layer23_attention_weight_min": -19.410808563232422,
      "activations/layer2_attention_weight_max": 31.186994552612305,
      "activations/layer2_attention_weight_min": -31.535133361816406,
      "activations/layer3_attention_weight_max": 85.06672668457031,
      "activations/layer3_attention_weight_min": -83.73342895507812,
      "activations/layer4_attention_weight_max": 127.1617660522461,
      "activations/layer4_attention_weight_min": -125.80948638916016,
      "activations/layer5_attention_weight_max": 88.08718872070312,
      "activations/layer5_attention_weight_min": -82.62373352050781,
      "activations/layer6_attention_weight_max": 57.51411437988281,
      "activations/layer6_attention_weight_min": -55.52855682373047,
      "activations/layer7_attention_weight_max": 79.5588607788086,
      "activations/layer7_attention_weight_min": -74.12458801269531,
      "activations/layer8_attention_weight_max": 39.35206985473633,
      "activations/layer8_attention_weight_min": -38.34743881225586,
      "activations/layer9_attention_weight_max": 38.904014587402344,
      "activations/layer9_attention_weight_min": -34.79030990600586,
      "epoch": 4.09,
      "learning_rate": 0.00012487575757575756,
      "loss": 2.8873,
      "step": 70400
    },
    {
      "activations/layer0_attention_weight_max": 16.346147537231445,
      "activations/layer0_attention_weight_min": -11.972352027893066,
      "activations/layer10_attention_weight_max": 35.79163360595703,
      "activations/layer10_attention_weight_min": -31.9257869720459,
      "activations/layer11_attention_weight_max": 35.327213287353516,
      "activations/layer11_attention_weight_min": -31.18179702758789,
      "activations/layer12_attention_weight_max": 36.96895980834961,
      "activations/layer12_attention_weight_min": -33.46947479248047,
      "activations/layer13_attention_weight_max": 50.545047760009766,
      "activations/layer13_attention_weight_min": -33.947818756103516,
      "activations/layer14_attention_weight_max": 83.05072784423828,
      "activations/layer14_attention_weight_min": -57.80915451049805,
      "activations/layer15_attention_weight_max": 63.25679016113281,
      "activations/layer15_attention_weight_min": -36.75053405761719,
      "activations/layer16_attention_weight_max": 41.30582809448242,
      "activations/layer16_attention_weight_min": -28.271799087524414,
      "activations/layer17_attention_weight_max": 43.98991775512695,
      "activations/layer17_attention_weight_min": -25.10093879699707,
      "activations/layer18_attention_weight_max": 33.13755798339844,
      "activations/layer18_attention_weight_min": -19.107730865478516,
      "activations/layer19_attention_weight_max": 47.78921890258789,
      "activations/layer19_attention_weight_min": -29.195486068725586,
      "activations/layer1_attention_weight_max": 17.35210609436035,
      "activations/layer1_attention_weight_min": -14.975462913513184,
      "activations/layer20_attention_weight_max": 32.77134704589844,
      "activations/layer20_attention_weight_min": -22.425579071044922,
      "activations/layer21_attention_weight_max": 31.021299362182617,
      "activations/layer21_attention_weight_min": -20.13628387451172,
      "activations/layer22_attention_weight_max": 50.34707260131836,
      "activations/layer22_attention_weight_min": -25.520221710205078,
      "activations/layer23_attention_weight_max": 31.758441925048828,
      "activations/layer23_attention_weight_min": -19.82187843322754,
      "activations/layer2_attention_weight_max": 32.07312774658203,
      "activations/layer2_attention_weight_min": -34.25389099121094,
      "activations/layer3_attention_weight_max": 90.59474182128906,
      "activations/layer3_attention_weight_min": -97.45887756347656,
      "activations/layer4_attention_weight_max": 136.50392150878906,
      "activations/layer4_attention_weight_min": -138.89559936523438,
      "activations/layer5_attention_weight_max": 96.98677825927734,
      "activations/layer5_attention_weight_min": -94.65538024902344,
      "activations/layer6_attention_weight_max": 60.329837799072266,
      "activations/layer6_attention_weight_min": -64.06462860107422,
      "activations/layer7_attention_weight_max": 87.25352478027344,
      "activations/layer7_attention_weight_min": -84.99734497070312,
      "activations/layer8_attention_weight_max": 42.65644073486328,
      "activations/layer8_attention_weight_min": -41.30768966674805,
      "activations/layer9_attention_weight_max": 41.19829177856445,
      "activations/layer9_attention_weight_min": -35.92374801635742,
      "epoch": 4.09,
      "learning_rate": 0.00012485681818181815,
      "loss": 2.869,
      "step": 70450
    },
    {
      "activations/layer0_attention_weight_max": 14.736614227294922,
      "activations/layer0_attention_weight_min": -12.564717292785645,
      "activations/layer10_attention_weight_max": 37.65818405151367,
      "activations/layer10_attention_weight_min": -31.856468200683594,
      "activations/layer11_attention_weight_max": 34.55973815917969,
      "activations/layer11_attention_weight_min": -27.556060791015625,
      "activations/layer12_attention_weight_max": 70.59000396728516,
      "activations/layer12_attention_weight_min": -32.819068908691406,
      "activations/layer13_attention_weight_max": 61.71738815307617,
      "activations/layer13_attention_weight_min": -39.770751953125,
      "activations/layer14_attention_weight_max": 85.364013671875,
      "activations/layer14_attention_weight_min": -53.03457260131836,
      "activations/layer15_attention_weight_max": 43.47624588012695,
      "activations/layer15_attention_weight_min": -30.775569915771484,
      "activations/layer16_attention_weight_max": 49.309303283691406,
      "activations/layer16_attention_weight_min": -30.554386138916016,
      "activations/layer17_attention_weight_max": 45.990474700927734,
      "activations/layer17_attention_weight_min": -27.27911949157715,
      "activations/layer18_attention_weight_max": 35.75395584106445,
      "activations/layer18_attention_weight_min": -20.353073120117188,
      "activations/layer19_attention_weight_max": 50.901851654052734,
      "activations/layer19_attention_weight_min": -29.795324325561523,
      "activations/layer1_attention_weight_max": 17.626535415649414,
      "activations/layer1_attention_weight_min": -16.632089614868164,
      "activations/layer20_attention_weight_max": 42.96073913574219,
      "activations/layer20_attention_weight_min": -23.425458908081055,
      "activations/layer21_attention_weight_max": 31.498300552368164,
      "activations/layer21_attention_weight_min": -20.808610916137695,
      "activations/layer22_attention_weight_max": 54.69211196899414,
      "activations/layer22_attention_weight_min": -27.993793487548828,
      "activations/layer23_attention_weight_max": 32.25659942626953,
      "activations/layer23_attention_weight_min": -20.212318420410156,
      "activations/layer2_attention_weight_max": 31.744827270507812,
      "activations/layer2_attention_weight_min": -32.9337043762207,
      "activations/layer3_attention_weight_max": 88.3780288696289,
      "activations/layer3_attention_weight_min": -89.63993835449219,
      "activations/layer4_attention_weight_max": 130.1668243408203,
      "activations/layer4_attention_weight_min": -131.2985382080078,
      "activations/layer5_attention_weight_max": 89.89144134521484,
      "activations/layer5_attention_weight_min": -87.21699523925781,
      "activations/layer6_attention_weight_max": 57.30009078979492,
      "activations/layer6_attention_weight_min": -60.05303955078125,
      "activations/layer7_attention_weight_max": 79.06035614013672,
      "activations/layer7_attention_weight_min": -81.42024230957031,
      "activations/layer8_attention_weight_max": 41.239662170410156,
      "activations/layer8_attention_weight_min": -38.431697845458984,
      "activations/layer9_attention_weight_max": 38.64626693725586,
      "activations/layer9_attention_weight_min": -33.15324401855469,
      "epoch": 4.1,
      "learning_rate": 0.00012483787878787877,
      "loss": 2.8739,
      "step": 70500
    },
    {
      "activations/layer0_attention_weight_max": 16.596744537353516,
      "activations/layer0_attention_weight_min": -11.843420028686523,
      "activations/layer10_attention_weight_max": 39.02560043334961,
      "activations/layer10_attention_weight_min": -31.70608901977539,
      "activations/layer11_attention_weight_max": 36.83650207519531,
      "activations/layer11_attention_weight_min": -26.782630920410156,
      "activations/layer12_attention_weight_max": 27.54610824584961,
      "activations/layer12_attention_weight_min": -26.05491065979004,
      "activations/layer13_attention_weight_max": 53.33332443237305,
      "activations/layer13_attention_weight_min": -34.407135009765625,
      "activations/layer14_attention_weight_max": 91.40322875976562,
      "activations/layer14_attention_weight_min": -60.25861358642578,
      "activations/layer15_attention_weight_max": 41.63378143310547,
      "activations/layer15_attention_weight_min": -27.816200256347656,
      "activations/layer16_attention_weight_max": 43.42684555053711,
      "activations/layer16_attention_weight_min": -29.443586349487305,
      "activations/layer17_attention_weight_max": 45.864356994628906,
      "activations/layer17_attention_weight_min": -27.25196075439453,
      "activations/layer18_attention_weight_max": 34.33112335205078,
      "activations/layer18_attention_weight_min": -21.021766662597656,
      "activations/layer19_attention_weight_max": 45.08811950683594,
      "activations/layer19_attention_weight_min": -27.882572174072266,
      "activations/layer1_attention_weight_max": 17.642457962036133,
      "activations/layer1_attention_weight_min": -15.675004959106445,
      "activations/layer20_attention_weight_max": 37.242515563964844,
      "activations/layer20_attention_weight_min": -22.23880386352539,
      "activations/layer21_attention_weight_max": 35.38337326049805,
      "activations/layer21_attention_weight_min": -20.780380249023438,
      "activations/layer22_attention_weight_max": 59.76274871826172,
      "activations/layer22_attention_weight_min": -28.648653030395508,
      "activations/layer23_attention_weight_max": 35.60308837890625,
      "activations/layer23_attention_weight_min": -19.04755973815918,
      "activations/layer2_attention_weight_max": 30.457088470458984,
      "activations/layer2_attention_weight_min": -30.655118942260742,
      "activations/layer3_attention_weight_max": 85.40544891357422,
      "activations/layer3_attention_weight_min": -89.2773666381836,
      "activations/layer4_attention_weight_max": 128.22715759277344,
      "activations/layer4_attention_weight_min": -130.47994995117188,
      "activations/layer5_attention_weight_max": 90.12724304199219,
      "activations/layer5_attention_weight_min": -85.32440185546875,
      "activations/layer6_attention_weight_max": 58.300052642822266,
      "activations/layer6_attention_weight_min": -58.76313400268555,
      "activations/layer7_attention_weight_max": 85.87738037109375,
      "activations/layer7_attention_weight_min": -86.99851989746094,
      "activations/layer8_attention_weight_max": 44.4493522644043,
      "activations/layer8_attention_weight_min": -42.91022491455078,
      "activations/layer9_attention_weight_max": 40.02971267700195,
      "activations/layer9_attention_weight_min": -39.5444450378418,
      "epoch": 4.1,
      "learning_rate": 0.00012481893939393938,
      "loss": 2.874,
      "step": 70550
    },
    {
      "activations/layer0_attention_weight_max": 16.753156661987305,
      "activations/layer0_attention_weight_min": -10.801387786865234,
      "activations/layer10_attention_weight_max": 33.595314025878906,
      "activations/layer10_attention_weight_min": -31.299428939819336,
      "activations/layer11_attention_weight_max": 30.993431091308594,
      "activations/layer11_attention_weight_min": -29.33564567565918,
      "activations/layer12_attention_weight_max": 25.772785186767578,
      "activations/layer12_attention_weight_min": -31.547481536865234,
      "activations/layer13_attention_weight_max": 40.11802673339844,
      "activations/layer13_attention_weight_min": -31.186466217041016,
      "activations/layer14_attention_weight_max": 52.19164276123047,
      "activations/layer14_attention_weight_min": -41.88650894165039,
      "activations/layer15_attention_weight_max": 41.97859191894531,
      "activations/layer15_attention_weight_min": -31.07718276977539,
      "activations/layer16_attention_weight_max": 44.186161041259766,
      "activations/layer16_attention_weight_min": -27.0205020904541,
      "activations/layer17_attention_weight_max": 42.994537353515625,
      "activations/layer17_attention_weight_min": -23.616090774536133,
      "activations/layer18_attention_weight_max": 31.329200744628906,
      "activations/layer18_attention_weight_min": -21.35938835144043,
      "activations/layer19_attention_weight_max": 41.37595748901367,
      "activations/layer19_attention_weight_min": -27.47116470336914,
      "activations/layer1_attention_weight_max": 17.391010284423828,
      "activations/layer1_attention_weight_min": -15.108902931213379,
      "activations/layer20_attention_weight_max": 29.73882484436035,
      "activations/layer20_attention_weight_min": -23.65142059326172,
      "activations/layer21_attention_weight_max": 32.468505859375,
      "activations/layer21_attention_weight_min": -19.679988861083984,
      "activations/layer22_attention_weight_max": 47.57658004760742,
      "activations/layer22_attention_weight_min": -26.027362823486328,
      "activations/layer23_attention_weight_max": 29.62783432006836,
      "activations/layer23_attention_weight_min": -18.247177124023438,
      "activations/layer2_attention_weight_max": 31.07675552368164,
      "activations/layer2_attention_weight_min": -31.02634048461914,
      "activations/layer3_attention_weight_max": 87.40489959716797,
      "activations/layer3_attention_weight_min": -89.82714080810547,
      "activations/layer4_attention_weight_max": 129.5775909423828,
      "activations/layer4_attention_weight_min": -127.68131256103516,
      "activations/layer5_attention_weight_max": 88.04573822021484,
      "activations/layer5_attention_weight_min": -82.7203598022461,
      "activations/layer6_attention_weight_max": 55.72330093383789,
      "activations/layer6_attention_weight_min": -56.1063117980957,
      "activations/layer7_attention_weight_max": 78.14555358886719,
      "activations/layer7_attention_weight_min": -77.4940185546875,
      "activations/layer8_attention_weight_max": 39.11286163330078,
      "activations/layer8_attention_weight_min": -40.71186828613281,
      "activations/layer9_attention_weight_max": 38.66548538208008,
      "activations/layer9_attention_weight_min": -36.44560241699219,
      "epoch": 4.1,
      "learning_rate": 0.00012479999999999997,
      "loss": 2.8727,
      "step": 70600
    },
    {
      "activations/layer0_attention_weight_max": 16.239662170410156,
      "activations/layer0_attention_weight_min": -11.489017486572266,
      "activations/layer10_attention_weight_max": 39.3614501953125,
      "activations/layer10_attention_weight_min": -33.6727180480957,
      "activations/layer11_attention_weight_max": 33.988426208496094,
      "activations/layer11_attention_weight_min": -27.36911392211914,
      "activations/layer12_attention_weight_max": 58.861236572265625,
      "activations/layer12_attention_weight_min": -34.02326965332031,
      "activations/layer13_attention_weight_max": 64.45280456542969,
      "activations/layer13_attention_weight_min": -39.15178680419922,
      "activations/layer14_attention_weight_max": 70.37594604492188,
      "activations/layer14_attention_weight_min": -49.30704116821289,
      "activations/layer15_attention_weight_max": 48.810302734375,
      "activations/layer15_attention_weight_min": -29.65070915222168,
      "activations/layer16_attention_weight_max": 43.65346908569336,
      "activations/layer16_attention_weight_min": -30.383581161499023,
      "activations/layer17_attention_weight_max": 47.25670623779297,
      "activations/layer17_attention_weight_min": -25.74650001525879,
      "activations/layer18_attention_weight_max": 30.9411563873291,
      "activations/layer18_attention_weight_min": -20.61638069152832,
      "activations/layer19_attention_weight_max": 46.105587005615234,
      "activations/layer19_attention_weight_min": -29.638809204101562,
      "activations/layer1_attention_weight_max": 17.041006088256836,
      "activations/layer1_attention_weight_min": -14.915925025939941,
      "activations/layer20_attention_weight_max": 33.44365310668945,
      "activations/layer20_attention_weight_min": -24.823734283447266,
      "activations/layer21_attention_weight_max": 32.56715774536133,
      "activations/layer21_attention_weight_min": -20.49098777770996,
      "activations/layer22_attention_weight_max": 50.28409957885742,
      "activations/layer22_attention_weight_min": -30.540756225585938,
      "activations/layer23_attention_weight_max": 30.37389373779297,
      "activations/layer23_attention_weight_min": -18.779109954833984,
      "activations/layer2_attention_weight_max": 34.345916748046875,
      "activations/layer2_attention_weight_min": -32.54560852050781,
      "activations/layer3_attention_weight_max": 93.2743911743164,
      "activations/layer3_attention_weight_min": -92.7351303100586,
      "activations/layer4_attention_weight_max": 143.31597900390625,
      "activations/layer4_attention_weight_min": -142.47457885742188,
      "activations/layer5_attention_weight_max": 98.3138427734375,
      "activations/layer5_attention_weight_min": -91.06948852539062,
      "activations/layer6_attention_weight_max": 58.74543380737305,
      "activations/layer6_attention_weight_min": -60.56739044189453,
      "activations/layer7_attention_weight_max": 79.96967315673828,
      "activations/layer7_attention_weight_min": -83.06941223144531,
      "activations/layer8_attention_weight_max": 43.299598693847656,
      "activations/layer8_attention_weight_min": -40.073612213134766,
      "activations/layer9_attention_weight_max": 41.70563888549805,
      "activations/layer9_attention_weight_min": -34.727603912353516,
      "epoch": 4.11,
      "learning_rate": 0.0001247814393939394,
      "loss": 2.8635,
      "step": 70650
    },
    {
      "activations/layer0_attention_weight_max": 17.63753890991211,
      "activations/layer0_attention_weight_min": -11.44411849975586,
      "activations/layer10_attention_weight_max": 37.84108352661133,
      "activations/layer10_attention_weight_min": -29.769956588745117,
      "activations/layer11_attention_weight_max": 34.537445068359375,
      "activations/layer11_attention_weight_min": -29.22547149658203,
      "activations/layer12_attention_weight_max": 33.84220504760742,
      "activations/layer12_attention_weight_min": -29.20958709716797,
      "activations/layer13_attention_weight_max": 43.89209747314453,
      "activations/layer13_attention_weight_min": -34.399627685546875,
      "activations/layer14_attention_weight_max": 71.49922943115234,
      "activations/layer14_attention_weight_min": -53.82975769042969,
      "activations/layer15_attention_weight_max": 41.27043533325195,
      "activations/layer15_attention_weight_min": -29.02549171447754,
      "activations/layer16_attention_weight_max": 38.075714111328125,
      "activations/layer16_attention_weight_min": -26.622352600097656,
      "activations/layer17_attention_weight_max": 38.34071731567383,
      "activations/layer17_attention_weight_min": -25.418010711669922,
      "activations/layer18_attention_weight_max": 30.9736270904541,
      "activations/layer18_attention_weight_min": -19.200973510742188,
      "activations/layer19_attention_weight_max": 43.312469482421875,
      "activations/layer19_attention_weight_min": -28.59511375427246,
      "activations/layer1_attention_weight_max": 18.041847229003906,
      "activations/layer1_attention_weight_min": -14.07680606842041,
      "activations/layer20_attention_weight_max": 36.5594482421875,
      "activations/layer20_attention_weight_min": -24.704504013061523,
      "activations/layer21_attention_weight_max": 32.22773361206055,
      "activations/layer21_attention_weight_min": -19.613508224487305,
      "activations/layer22_attention_weight_max": 53.31178283691406,
      "activations/layer22_attention_weight_min": -30.56055450439453,
      "activations/layer23_attention_weight_max": 32.00883865356445,
      "activations/layer23_attention_weight_min": -18.898717880249023,
      "activations/layer2_attention_weight_max": 30.121145248413086,
      "activations/layer2_attention_weight_min": -29.751033782958984,
      "activations/layer3_attention_weight_max": 81.1932144165039,
      "activations/layer3_attention_weight_min": -82.1220932006836,
      "activations/layer4_attention_weight_max": 133.14488220214844,
      "activations/layer4_attention_weight_min": -123.64290618896484,
      "activations/layer5_attention_weight_max": 90.9763412475586,
      "activations/layer5_attention_weight_min": -84.00823974609375,
      "activations/layer6_attention_weight_max": 57.97552490234375,
      "activations/layer6_attention_weight_min": -56.75346374511719,
      "activations/layer7_attention_weight_max": 77.8958740234375,
      "activations/layer7_attention_weight_min": -77.57069396972656,
      "activations/layer8_attention_weight_max": 39.85017013549805,
      "activations/layer8_attention_weight_min": -41.60356903076172,
      "activations/layer9_attention_weight_max": 41.28683090209961,
      "activations/layer9_attention_weight_min": -34.075653076171875,
      "epoch": 4.11,
      "learning_rate": 0.00012476249999999998,
      "loss": 2.8622,
      "step": 70700
    },
    {
      "activations/layer0_attention_weight_max": 15.733628273010254,
      "activations/layer0_attention_weight_min": -11.078536987304688,
      "activations/layer10_attention_weight_max": 45.78455352783203,
      "activations/layer10_attention_weight_min": -34.078060150146484,
      "activations/layer11_attention_weight_max": 42.90833282470703,
      "activations/layer11_attention_weight_min": -30.856748580932617,
      "activations/layer12_attention_weight_max": 37.24628829956055,
      "activations/layer12_attention_weight_min": -30.910274505615234,
      "activations/layer13_attention_weight_max": 73.31492614746094,
      "activations/layer13_attention_weight_min": -52.60445785522461,
      "activations/layer14_attention_weight_max": 83.83377838134766,
      "activations/layer14_attention_weight_min": -59.715030670166016,
      "activations/layer15_attention_weight_max": 57.273460388183594,
      "activations/layer15_attention_weight_min": -41.54808807373047,
      "activations/layer16_attention_weight_max": 41.80867385864258,
      "activations/layer16_attention_weight_min": -29.61061668395996,
      "activations/layer17_attention_weight_max": 39.65494918823242,
      "activations/layer17_attention_weight_min": -28.05276870727539,
      "activations/layer18_attention_weight_max": 42.839725494384766,
      "activations/layer18_attention_weight_min": -22.283723831176758,
      "activations/layer19_attention_weight_max": 46.80720138549805,
      "activations/layer19_attention_weight_min": -32.887916564941406,
      "activations/layer1_attention_weight_max": 18.643524169921875,
      "activations/layer1_attention_weight_min": -14.87746810913086,
      "activations/layer20_attention_weight_max": 34.41914749145508,
      "activations/layer20_attention_weight_min": -29.017398834228516,
      "activations/layer21_attention_weight_max": 31.116666793823242,
      "activations/layer21_attention_weight_min": -22.917264938354492,
      "activations/layer22_attention_weight_max": 48.217525482177734,
      "activations/layer22_attention_weight_min": -28.977943420410156,
      "activations/layer23_attention_weight_max": 28.78162384033203,
      "activations/layer23_attention_weight_min": -19.431882858276367,
      "activations/layer2_attention_weight_max": 35.88276290893555,
      "activations/layer2_attention_weight_min": -33.02946853637695,
      "activations/layer3_attention_weight_max": 102.34230041503906,
      "activations/layer3_attention_weight_min": -94.40261840820312,
      "activations/layer4_attention_weight_max": 158.04220581054688,
      "activations/layer4_attention_weight_min": -139.91380310058594,
      "activations/layer5_attention_weight_max": 99.03985595703125,
      "activations/layer5_attention_weight_min": -87.75652313232422,
      "activations/layer6_attention_weight_max": 63.70905303955078,
      "activations/layer6_attention_weight_min": -59.191410064697266,
      "activations/layer7_attention_weight_max": 100.87860870361328,
      "activations/layer7_attention_weight_min": -86.77448272705078,
      "activations/layer8_attention_weight_max": 50.59840774536133,
      "activations/layer8_attention_weight_min": -45.27534866333008,
      "activations/layer9_attention_weight_max": 45.15610885620117,
      "activations/layer9_attention_weight_min": -37.19387435913086,
      "epoch": 4.11,
      "learning_rate": 0.0001247435606060606,
      "loss": 2.8767,
      "step": 70750
    },
    {
      "activations/layer0_attention_weight_max": 16.583036422729492,
      "activations/layer0_attention_weight_min": -12.115652084350586,
      "activations/layer10_attention_weight_max": 32.40376663208008,
      "activations/layer10_attention_weight_min": -28.619537353515625,
      "activations/layer11_attention_weight_max": 32.559425354003906,
      "activations/layer11_attention_weight_min": -24.641315460205078,
      "activations/layer12_attention_weight_max": 31.815616607666016,
      "activations/layer12_attention_weight_min": -28.77582359313965,
      "activations/layer13_attention_weight_max": 47.49924087524414,
      "activations/layer13_attention_weight_min": -33.827178955078125,
      "activations/layer14_attention_weight_max": 61.75421142578125,
      "activations/layer14_attention_weight_min": -54.72319793701172,
      "activations/layer15_attention_weight_max": 40.67425537109375,
      "activations/layer15_attention_weight_min": -30.960269927978516,
      "activations/layer16_attention_weight_max": 36.24784469604492,
      "activations/layer16_attention_weight_min": -27.233671188354492,
      "activations/layer17_attention_weight_max": 38.36613845825195,
      "activations/layer17_attention_weight_min": -24.658193588256836,
      "activations/layer18_attention_weight_max": 34.11388397216797,
      "activations/layer18_attention_weight_min": -22.03801918029785,
      "activations/layer19_attention_weight_max": 36.98741149902344,
      "activations/layer19_attention_weight_min": -27.290685653686523,
      "activations/layer1_attention_weight_max": 17.635265350341797,
      "activations/layer1_attention_weight_min": -14.542524337768555,
      "activations/layer20_attention_weight_max": 30.098026275634766,
      "activations/layer20_attention_weight_min": -23.681913375854492,
      "activations/layer21_attention_weight_max": 31.805240631103516,
      "activations/layer21_attention_weight_min": -20.42780303955078,
      "activations/layer22_attention_weight_max": 47.005191802978516,
      "activations/layer22_attention_weight_min": -26.18307113647461,
      "activations/layer23_attention_weight_max": 29.48320770263672,
      "activations/layer23_attention_weight_min": -21.734661102294922,
      "activations/layer2_attention_weight_max": 32.16923904418945,
      "activations/layer2_attention_weight_min": -30.926555633544922,
      "activations/layer3_attention_weight_max": 87.23552703857422,
      "activations/layer3_attention_weight_min": -88.84864044189453,
      "activations/layer4_attention_weight_max": 128.41514587402344,
      "activations/layer4_attention_weight_min": -125.10887908935547,
      "activations/layer5_attention_weight_max": 90.36347961425781,
      "activations/layer5_attention_weight_min": -85.06355285644531,
      "activations/layer6_attention_weight_max": 59.798255920410156,
      "activations/layer6_attention_weight_min": -54.424556732177734,
      "activations/layer7_attention_weight_max": 75.38858032226562,
      "activations/layer7_attention_weight_min": -76.38560485839844,
      "activations/layer8_attention_weight_max": 38.517879486083984,
      "activations/layer8_attention_weight_min": -39.74568557739258,
      "activations/layer9_attention_weight_max": 39.32746505737305,
      "activations/layer9_attention_weight_min": -34.160003662109375,
      "epoch": 4.11,
      "learning_rate": 0.0001247246212121212,
      "loss": 2.8648,
      "step": 70800
    },
    {
      "activations/layer0_attention_weight_max": 17.061634063720703,
      "activations/layer0_attention_weight_min": -12.473664283752441,
      "activations/layer10_attention_weight_max": 38.053409576416016,
      "activations/layer10_attention_weight_min": -29.85502815246582,
      "activations/layer11_attention_weight_max": 35.57750701904297,
      "activations/layer11_attention_weight_min": -27.329113006591797,
      "activations/layer12_attention_weight_max": 45.092994689941406,
      "activations/layer12_attention_weight_min": -25.953039169311523,
      "activations/layer13_attention_weight_max": 50.5621223449707,
      "activations/layer13_attention_weight_min": -35.802406311035156,
      "activations/layer14_attention_weight_max": 69.32206726074219,
      "activations/layer14_attention_weight_min": -51.033546447753906,
      "activations/layer15_attention_weight_max": 44.708133697509766,
      "activations/layer15_attention_weight_min": -29.5875244140625,
      "activations/layer16_attention_weight_max": 42.49419403076172,
      "activations/layer16_attention_weight_min": -28.7070369720459,
      "activations/layer17_attention_weight_max": 44.1492805480957,
      "activations/layer17_attention_weight_min": -24.39440155029297,
      "activations/layer18_attention_weight_max": 32.31707000732422,
      "activations/layer18_attention_weight_min": -20.459667205810547,
      "activations/layer19_attention_weight_max": 42.257469177246094,
      "activations/layer19_attention_weight_min": -30.259464263916016,
      "activations/layer1_attention_weight_max": 18.012889862060547,
      "activations/layer1_attention_weight_min": -15.203353881835938,
      "activations/layer20_attention_weight_max": 36.5047721862793,
      "activations/layer20_attention_weight_min": -23.75014305114746,
      "activations/layer21_attention_weight_max": 33.504638671875,
      "activations/layer21_attention_weight_min": -22.009408950805664,
      "activations/layer22_attention_weight_max": 55.43317413330078,
      "activations/layer22_attention_weight_min": -32.392879486083984,
      "activations/layer23_attention_weight_max": 30.058822631835938,
      "activations/layer23_attention_weight_min": -21.135944366455078,
      "activations/layer2_attention_weight_max": 31.160673141479492,
      "activations/layer2_attention_weight_min": -29.42790985107422,
      "activations/layer3_attention_weight_max": 83.72274017333984,
      "activations/layer3_attention_weight_min": -88.03923034667969,
      "activations/layer4_attention_weight_max": 124.89131927490234,
      "activations/layer4_attention_weight_min": -126.17684173583984,
      "activations/layer5_attention_weight_max": 88.0332260131836,
      "activations/layer5_attention_weight_min": -82.94659423828125,
      "activations/layer6_attention_weight_max": 55.68821334838867,
      "activations/layer6_attention_weight_min": -57.364776611328125,
      "activations/layer7_attention_weight_max": 80.20114135742188,
      "activations/layer7_attention_weight_min": -76.25927734375,
      "activations/layer8_attention_weight_max": 37.516448974609375,
      "activations/layer8_attention_weight_min": -39.98691940307617,
      "activations/layer9_attention_weight_max": 36.993892669677734,
      "activations/layer9_attention_weight_min": -33.525543212890625,
      "epoch": 4.12,
      "learning_rate": 0.0001247056818181818,
      "loss": 2.868,
      "step": 70850
    },
    {
      "activations/layer0_attention_weight_max": 15.397302627563477,
      "activations/layer0_attention_weight_min": -13.588981628417969,
      "activations/layer10_attention_weight_max": 36.28071594238281,
      "activations/layer10_attention_weight_min": -30.806747436523438,
      "activations/layer11_attention_weight_max": 38.541954040527344,
      "activations/layer11_attention_weight_min": -29.133764266967773,
      "activations/layer12_attention_weight_max": 27.505720138549805,
      "activations/layer12_attention_weight_min": -31.6348819732666,
      "activations/layer13_attention_weight_max": 46.6036491394043,
      "activations/layer13_attention_weight_min": -31.93877410888672,
      "activations/layer14_attention_weight_max": 69.63996887207031,
      "activations/layer14_attention_weight_min": -43.80289077758789,
      "activations/layer15_attention_weight_max": 41.559532165527344,
      "activations/layer15_attention_weight_min": -29.725862503051758,
      "activations/layer16_attention_weight_max": 44.48924255371094,
      "activations/layer16_attention_weight_min": -29.474233627319336,
      "activations/layer17_attention_weight_max": 37.21871566772461,
      "activations/layer17_attention_weight_min": -24.860654830932617,
      "activations/layer18_attention_weight_max": 29.402742385864258,
      "activations/layer18_attention_weight_min": -18.778993606567383,
      "activations/layer19_attention_weight_max": 41.24042892456055,
      "activations/layer19_attention_weight_min": -29.06168556213379,
      "activations/layer1_attention_weight_max": 18.062484741210938,
      "activations/layer1_attention_weight_min": -13.476863861083984,
      "activations/layer20_attention_weight_max": 28.26573371887207,
      "activations/layer20_attention_weight_min": -23.6352596282959,
      "activations/layer21_attention_weight_max": 33.1311149597168,
      "activations/layer21_attention_weight_min": -19.439289093017578,
      "activations/layer22_attention_weight_max": 50.92717361450195,
      "activations/layer22_attention_weight_min": -29.093767166137695,
      "activations/layer23_attention_weight_max": 30.2587890625,
      "activations/layer23_attention_weight_min": -17.58848762512207,
      "activations/layer2_attention_weight_max": 32.73429870605469,
      "activations/layer2_attention_weight_min": -33.18730545043945,
      "activations/layer3_attention_weight_max": 93.73117065429688,
      "activations/layer3_attention_weight_min": -91.51813507080078,
      "activations/layer4_attention_weight_max": 141.99221801757812,
      "activations/layer4_attention_weight_min": -133.087890625,
      "activations/layer5_attention_weight_max": 97.24004364013672,
      "activations/layer5_attention_weight_min": -87.12679290771484,
      "activations/layer6_attention_weight_max": 58.361942291259766,
      "activations/layer6_attention_weight_min": -56.05999755859375,
      "activations/layer7_attention_weight_max": 86.00428771972656,
      "activations/layer7_attention_weight_min": -81.44490051269531,
      "activations/layer8_attention_weight_max": 43.2043342590332,
      "activations/layer8_attention_weight_min": -40.4031867980957,
      "activations/layer9_attention_weight_max": 41.89047622680664,
      "activations/layer9_attention_weight_min": -40.226531982421875,
      "epoch": 4.12,
      "learning_rate": 0.0001246867424242424,
      "loss": 2.8413,
      "step": 70900
    },
    {
      "activations/layer0_attention_weight_max": 16.62160301208496,
      "activations/layer0_attention_weight_min": -12.246960639953613,
      "activations/layer10_attention_weight_max": 34.81173324584961,
      "activations/layer10_attention_weight_min": -27.627660751342773,
      "activations/layer11_attention_weight_max": 35.8342170715332,
      "activations/layer11_attention_weight_min": -29.10781478881836,
      "activations/layer12_attention_weight_max": 30.896690368652344,
      "activations/layer12_attention_weight_min": -36.717803955078125,
      "activations/layer13_attention_weight_max": 54.326133728027344,
      "activations/layer13_attention_weight_min": -43.01998519897461,
      "activations/layer14_attention_weight_max": 66.91046142578125,
      "activations/layer14_attention_weight_min": -53.071346282958984,
      "activations/layer15_attention_weight_max": 41.050315856933594,
      "activations/layer15_attention_weight_min": -31.652389526367188,
      "activations/layer16_attention_weight_max": 38.11685562133789,
      "activations/layer16_attention_weight_min": -29.541900634765625,
      "activations/layer17_attention_weight_max": 37.7821044921875,
      "activations/layer17_attention_weight_min": -24.840845108032227,
      "activations/layer18_attention_weight_max": 35.44464874267578,
      "activations/layer18_attention_weight_min": -21.99351692199707,
      "activations/layer19_attention_weight_max": 44.22587203979492,
      "activations/layer19_attention_weight_min": -27.993072509765625,
      "activations/layer1_attention_weight_max": 18.004669189453125,
      "activations/layer1_attention_weight_min": -15.055127143859863,
      "activations/layer20_attention_weight_max": 33.816246032714844,
      "activations/layer20_attention_weight_min": -21.375917434692383,
      "activations/layer21_attention_weight_max": 32.735748291015625,
      "activations/layer21_attention_weight_min": -22.261518478393555,
      "activations/layer22_attention_weight_max": 53.0351676940918,
      "activations/layer22_attention_weight_min": -28.355873107910156,
      "activations/layer23_attention_weight_max": 33.53233337402344,
      "activations/layer23_attention_weight_min": -21.45100212097168,
      "activations/layer2_attention_weight_max": 31.71847152709961,
      "activations/layer2_attention_weight_min": -29.7557373046875,
      "activations/layer3_attention_weight_max": 90.74747467041016,
      "activations/layer3_attention_weight_min": -90.28218841552734,
      "activations/layer4_attention_weight_max": 137.1619415283203,
      "activations/layer4_attention_weight_min": -132.04104614257812,
      "activations/layer5_attention_weight_max": 96.13502502441406,
      "activations/layer5_attention_weight_min": -85.41482543945312,
      "activations/layer6_attention_weight_max": 57.102386474609375,
      "activations/layer6_attention_weight_min": -58.23322677612305,
      "activations/layer7_attention_weight_max": 86.32229614257812,
      "activations/layer7_attention_weight_min": -78.25834655761719,
      "activations/layer8_attention_weight_max": 42.00700759887695,
      "activations/layer8_attention_weight_min": -38.46013259887695,
      "activations/layer9_attention_weight_max": 41.610015869140625,
      "activations/layer9_attention_weight_min": -51.36279296875,
      "epoch": 4.12,
      "learning_rate": 0.00012466780303030302,
      "loss": 2.8715,
      "step": 70950
    },
    {
      "activations/layer0_attention_weight_max": 16.88228988647461,
      "activations/layer0_attention_weight_min": -12.563158988952637,
      "activations/layer10_attention_weight_max": 42.0979118347168,
      "activations/layer10_attention_weight_min": -31.738449096679688,
      "activations/layer11_attention_weight_max": 45.74059295654297,
      "activations/layer11_attention_weight_min": -30.60273551940918,
      "activations/layer12_attention_weight_max": 66.71484375,
      "activations/layer12_attention_weight_min": -33.132102966308594,
      "activations/layer13_attention_weight_max": 69.88513946533203,
      "activations/layer13_attention_weight_min": -47.26615524291992,
      "activations/layer14_attention_weight_max": 96.20867919921875,
      "activations/layer14_attention_weight_min": -65.71013641357422,
      "activations/layer15_attention_weight_max": 70.68617248535156,
      "activations/layer15_attention_weight_min": -37.86865997314453,
      "activations/layer16_attention_weight_max": 50.3724250793457,
      "activations/layer16_attention_weight_min": -29.354612350463867,
      "activations/layer17_attention_weight_max": 54.42066192626953,
      "activations/layer17_attention_weight_min": -28.527936935424805,
      "activations/layer18_attention_weight_max": 44.723201751708984,
      "activations/layer18_attention_weight_min": -22.604537963867188,
      "activations/layer19_attention_weight_max": 41.96661376953125,
      "activations/layer19_attention_weight_min": -27.413854598999023,
      "activations/layer1_attention_weight_max": 17.88572883605957,
      "activations/layer1_attention_weight_min": -14.836555480957031,
      "activations/layer20_attention_weight_max": 37.83889389038086,
      "activations/layer20_attention_weight_min": -21.592227935791016,
      "activations/layer21_attention_weight_max": 33.24388885498047,
      "activations/layer21_attention_weight_min": -19.896345138549805,
      "activations/layer22_attention_weight_max": 50.86685562133789,
      "activations/layer22_attention_weight_min": -25.552026748657227,
      "activations/layer23_attention_weight_max": 31.90011978149414,
      "activations/layer23_attention_weight_min": -18.324573516845703,
      "activations/layer2_attention_weight_max": 33.546470642089844,
      "activations/layer2_attention_weight_min": -33.65544128417969,
      "activations/layer3_attention_weight_max": 87.8261947631836,
      "activations/layer3_attention_weight_min": -89.2391128540039,
      "activations/layer4_attention_weight_max": 132.93531799316406,
      "activations/layer4_attention_weight_min": -124.08928680419922,
      "activations/layer5_attention_weight_max": 91.66583251953125,
      "activations/layer5_attention_weight_min": -81.16018676757812,
      "activations/layer6_attention_weight_max": 57.95679473876953,
      "activations/layer6_attention_weight_min": -56.57035827636719,
      "activations/layer7_attention_weight_max": 82.4421615600586,
      "activations/layer7_attention_weight_min": -82.39796447753906,
      "activations/layer8_attention_weight_max": 42.2165412902832,
      "activations/layer8_attention_weight_min": -39.53902816772461,
      "activations/layer9_attention_weight_max": 44.15928649902344,
      "activations/layer9_attention_weight_min": -35.09847640991211,
      "epoch": 4.13,
      "learning_rate": 0.00012464886363636364,
      "loss": 2.8636,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_loss": 2.822265625,
      "eval_runtime": 8.4723,
      "eval_samples_per_second": 506.826,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_openwebtext_loss": 2.822265625,
      "eval_openwebtext_ppl": 16.8149038152427,
      "eval_openwebtext_runtime": 8.4723,
      "eval_openwebtext_samples_per_second": 506.826,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_wikitext_loss": 3.052734375,
      "eval_wikitext_ppl": 21.173160702117308,
      "eval_wikitext_runtime": 1.9829,
      "eval_wikitext_samples_per_second": 229.971,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_lambada_loss": 2.708984375,
      "eval_lambada_ppl": 15.01401915561195,
      "eval_lambada_runtime": 9.5523,
      "eval_lambada_samples_per_second": 509.723,
      "step": 71000
    },
    {
      "activations/layer0_attention_weight_max": 17.22911834716797,
      "activations/layer0_attention_weight_min": -11.786632537841797,
      "activations/layer10_attention_weight_max": 37.48540496826172,
      "activations/layer10_attention_weight_min": -28.21901512145996,
      "activations/layer11_attention_weight_max": 40.337127685546875,
      "activations/layer11_attention_weight_min": -26.792572021484375,
      "activations/layer12_attention_weight_max": 29.95697593688965,
      "activations/layer12_attention_weight_min": -27.669736862182617,
      "activations/layer13_attention_weight_max": 53.27035903930664,
      "activations/layer13_attention_weight_min": -29.854055404663086,
      "activations/layer14_attention_weight_max": 76.36848449707031,
      "activations/layer14_attention_weight_min": -51.1881103515625,
      "activations/layer15_attention_weight_max": 47.75041961669922,
      "activations/layer15_attention_weight_min": -27.89466667175293,
      "activations/layer16_attention_weight_max": 45.2782096862793,
      "activations/layer16_attention_weight_min": -30.56004524230957,
      "activations/layer17_attention_weight_max": 41.00613784790039,
      "activations/layer17_attention_weight_min": -27.07659149169922,
      "activations/layer18_attention_weight_max": 30.674156188964844,
      "activations/layer18_attention_weight_min": -20.639671325683594,
      "activations/layer19_attention_weight_max": 42.03493118286133,
      "activations/layer19_attention_weight_min": -29.68208122253418,
      "activations/layer1_attention_weight_max": 17.887569427490234,
      "activations/layer1_attention_weight_min": -15.406586647033691,
      "activations/layer20_attention_weight_max": 36.62486267089844,
      "activations/layer20_attention_weight_min": -24.677160263061523,
      "activations/layer21_attention_weight_max": 35.285736083984375,
      "activations/layer21_attention_weight_min": -20.323566436767578,
      "activations/layer22_attention_weight_max": 54.247501373291016,
      "activations/layer22_attention_weight_min": -27.91460609436035,
      "activations/layer23_attention_weight_max": 33.85129165649414,
      "activations/layer23_attention_weight_min": -20.345508575439453,
      "activations/layer2_attention_weight_max": 32.098609924316406,
      "activations/layer2_attention_weight_min": -31.930522918701172,
      "activations/layer3_attention_weight_max": 91.36180877685547,
      "activations/layer3_attention_weight_min": -92.20323944091797,
      "activations/layer4_attention_weight_max": 136.81373596191406,
      "activations/layer4_attention_weight_min": -131.5500946044922,
      "activations/layer5_attention_weight_max": 93.059326171875,
      "activations/layer5_attention_weight_min": -85.21601104736328,
      "activations/layer6_attention_weight_max": 61.2315788269043,
      "activations/layer6_attention_weight_min": -57.177772521972656,
      "activations/layer7_attention_weight_max": 86.99858093261719,
      "activations/layer7_attention_weight_min": -79.4104232788086,
      "activations/layer8_attention_weight_max": 41.74357986450195,
      "activations/layer8_attention_weight_min": -39.703277587890625,
      "activations/layer9_attention_weight_max": 40.93039321899414,
      "activations/layer9_attention_weight_min": -33.33521270751953,
      "epoch": 4.13,
      "learning_rate": 0.00012462992424242423,
      "loss": 2.8606,
      "step": 71050
    },
    {
      "activations/layer0_attention_weight_max": 14.503726959228516,
      "activations/layer0_attention_weight_min": -12.148422241210938,
      "activations/layer10_attention_weight_max": 34.893917083740234,
      "activations/layer10_attention_weight_min": -29.79111671447754,
      "activations/layer11_attention_weight_max": 39.25403594970703,
      "activations/layer11_attention_weight_min": -29.911243438720703,
      "activations/layer12_attention_weight_max": 31.759679794311523,
      "activations/layer12_attention_weight_min": -29.632957458496094,
      "activations/layer13_attention_weight_max": 47.96579360961914,
      "activations/layer13_attention_weight_min": -34.94190216064453,
      "activations/layer14_attention_weight_max": 63.14799499511719,
      "activations/layer14_attention_weight_min": -45.411712646484375,
      "activations/layer15_attention_weight_max": 42.38230514526367,
      "activations/layer15_attention_weight_min": -30.761180877685547,
      "activations/layer16_attention_weight_max": 39.3736457824707,
      "activations/layer16_attention_weight_min": -28.221120834350586,
      "activations/layer17_attention_weight_max": 41.875186920166016,
      "activations/layer17_attention_weight_min": -24.498395919799805,
      "activations/layer18_attention_weight_max": 33.026371002197266,
      "activations/layer18_attention_weight_min": -20.59206199645996,
      "activations/layer19_attention_weight_max": 41.45587158203125,
      "activations/layer19_attention_weight_min": -27.857934951782227,
      "activations/layer1_attention_weight_max": 17.489349365234375,
      "activations/layer1_attention_weight_min": -15.817755699157715,
      "activations/layer20_attention_weight_max": 35.38277816772461,
      "activations/layer20_attention_weight_min": -23.375011444091797,
      "activations/layer21_attention_weight_max": 34.93330383300781,
      "activations/layer21_attention_weight_min": -21.478256225585938,
      "activations/layer22_attention_weight_max": 48.95647430419922,
      "activations/layer22_attention_weight_min": -29.78804588317871,
      "activations/layer23_attention_weight_max": 32.7634391784668,
      "activations/layer23_attention_weight_min": -18.97351837158203,
      "activations/layer2_attention_weight_max": 30.503215789794922,
      "activations/layer2_attention_weight_min": -33.02055358886719,
      "activations/layer3_attention_weight_max": 88.03905487060547,
      "activations/layer3_attention_weight_min": -87.7106704711914,
      "activations/layer4_attention_weight_max": 130.3292694091797,
      "activations/layer4_attention_weight_min": -124.9649429321289,
      "activations/layer5_attention_weight_max": 87.39170837402344,
      "activations/layer5_attention_weight_min": -85.73283386230469,
      "activations/layer6_attention_weight_max": 58.068687438964844,
      "activations/layer6_attention_weight_min": -59.58046340942383,
      "activations/layer7_attention_weight_max": 80.79387664794922,
      "activations/layer7_attention_weight_min": -79.96521759033203,
      "activations/layer8_attention_weight_max": 40.71729278564453,
      "activations/layer8_attention_weight_min": -38.19984436035156,
      "activations/layer9_attention_weight_max": 52.5473747253418,
      "activations/layer9_attention_weight_min": -40.072914123535156,
      "epoch": 4.13,
      "learning_rate": 0.00012461098484848484,
      "loss": 2.8578,
      "step": 71100
    },
    {
      "activations/layer0_attention_weight_max": 16.714021682739258,
      "activations/layer0_attention_weight_min": -12.069725036621094,
      "activations/layer10_attention_weight_max": 35.51176452636719,
      "activations/layer10_attention_weight_min": -29.525705337524414,
      "activations/layer11_attention_weight_max": 37.0916748046875,
      "activations/layer11_attention_weight_min": -30.013044357299805,
      "activations/layer12_attention_weight_max": 68.19332122802734,
      "activations/layer12_attention_weight_min": -31.629776000976562,
      "activations/layer13_attention_weight_max": 58.59159469604492,
      "activations/layer13_attention_weight_min": -41.20412063598633,
      "activations/layer14_attention_weight_max": 74.49598693847656,
      "activations/layer14_attention_weight_min": -49.052249908447266,
      "activations/layer15_attention_weight_max": 54.074974060058594,
      "activations/layer15_attention_weight_min": -31.0612735748291,
      "activations/layer16_attention_weight_max": 48.00602340698242,
      "activations/layer16_attention_weight_min": -29.52123260498047,
      "activations/layer17_attention_weight_max": 46.96088790893555,
      "activations/layer17_attention_weight_min": -26.69740104675293,
      "activations/layer18_attention_weight_max": 33.3293342590332,
      "activations/layer18_attention_weight_min": -19.175628662109375,
      "activations/layer19_attention_weight_max": 47.033729553222656,
      "activations/layer19_attention_weight_min": -29.3677978515625,
      "activations/layer1_attention_weight_max": 17.740922927856445,
      "activations/layer1_attention_weight_min": -18.39337730407715,
      "activations/layer20_attention_weight_max": 41.34840774536133,
      "activations/layer20_attention_weight_min": -24.65069007873535,
      "activations/layer21_attention_weight_max": 38.978538513183594,
      "activations/layer21_attention_weight_min": -19.567577362060547,
      "activations/layer22_attention_weight_max": 57.6340446472168,
      "activations/layer22_attention_weight_min": -25.580829620361328,
      "activations/layer23_attention_weight_max": 32.366729736328125,
      "activations/layer23_attention_weight_min": -20.48917579650879,
      "activations/layer2_attention_weight_max": 31.38153648376465,
      "activations/layer2_attention_weight_min": -29.29242515563965,
      "activations/layer3_attention_weight_max": 88.7251968383789,
      "activations/layer3_attention_weight_min": -86.78488159179688,
      "activations/layer4_attention_weight_max": 131.3838348388672,
      "activations/layer4_attention_weight_min": -125.43167877197266,
      "activations/layer5_attention_weight_max": 90.29912567138672,
      "activations/layer5_attention_weight_min": -83.85392761230469,
      "activations/layer6_attention_weight_max": 56.424652099609375,
      "activations/layer6_attention_weight_min": -54.88797378540039,
      "activations/layer7_attention_weight_max": 76.46004486083984,
      "activations/layer7_attention_weight_min": -74.86858367919922,
      "activations/layer8_attention_weight_max": 41.60890197753906,
      "activations/layer8_attention_weight_min": -38.91774368286133,
      "activations/layer9_attention_weight_max": 38.075801849365234,
      "activations/layer9_attention_weight_min": -33.585227966308594,
      "epoch": 4.13,
      "learning_rate": 0.00012459204545454546,
      "loss": 2.8643,
      "step": 71150
    },
    {
      "activations/layer0_attention_weight_max": 17.431438446044922,
      "activations/layer0_attention_weight_min": -12.995672225952148,
      "activations/layer10_attention_weight_max": 31.98617172241211,
      "activations/layer10_attention_weight_min": -28.734678268432617,
      "activations/layer11_attention_weight_max": 30.181163787841797,
      "activations/layer11_attention_weight_min": -27.064952850341797,
      "activations/layer12_attention_weight_max": 26.414691925048828,
      "activations/layer12_attention_weight_min": -25.748207092285156,
      "activations/layer13_attention_weight_max": 46.978477478027344,
      "activations/layer13_attention_weight_min": -40.27501678466797,
      "activations/layer14_attention_weight_max": 63.30104446411133,
      "activations/layer14_attention_weight_min": -47.93989562988281,
      "activations/layer15_attention_weight_max": 37.90991973876953,
      "activations/layer15_attention_weight_min": -28.22380828857422,
      "activations/layer16_attention_weight_max": 33.526729583740234,
      "activations/layer16_attention_weight_min": -29.430564880371094,
      "activations/layer17_attention_weight_max": 33.42189407348633,
      "activations/layer17_attention_weight_min": -25.679759979248047,
      "activations/layer18_attention_weight_max": 29.35338020324707,
      "activations/layer18_attention_weight_min": -20.634634017944336,
      "activations/layer19_attention_weight_max": 39.0263557434082,
      "activations/layer19_attention_weight_min": -28.765933990478516,
      "activations/layer1_attention_weight_max": 17.177915573120117,
      "activations/layer1_attention_weight_min": -13.336934089660645,
      "activations/layer20_attention_weight_max": 30.0711727142334,
      "activations/layer20_attention_weight_min": -22.264432907104492,
      "activations/layer21_attention_weight_max": 33.27511978149414,
      "activations/layer21_attention_weight_min": -24.10428237915039,
      "activations/layer22_attention_weight_max": 48.19339370727539,
      "activations/layer22_attention_weight_min": -26.69036865234375,
      "activations/layer23_attention_weight_max": 30.483213424682617,
      "activations/layer23_attention_weight_min": -18.27985954284668,
      "activations/layer2_attention_weight_max": 31.418004989624023,
      "activations/layer2_attention_weight_min": -31.441951751708984,
      "activations/layer3_attention_weight_max": 85.37605285644531,
      "activations/layer3_attention_weight_min": -87.59485626220703,
      "activations/layer4_attention_weight_max": 127.67394256591797,
      "activations/layer4_attention_weight_min": -124.57806396484375,
      "activations/layer5_attention_weight_max": 83.17662048339844,
      "activations/layer5_attention_weight_min": -83.10464477539062,
      "activations/layer6_attention_weight_max": 52.64097213745117,
      "activations/layer6_attention_weight_min": -57.34588623046875,
      "activations/layer7_attention_weight_max": 71.967529296875,
      "activations/layer7_attention_weight_min": -78.70661926269531,
      "activations/layer8_attention_weight_max": 37.77822494506836,
      "activations/layer8_attention_weight_min": -35.46088790893555,
      "activations/layer9_attention_weight_max": 40.36281967163086,
      "activations/layer9_attention_weight_min": -31.349151611328125,
      "epoch": 4.14,
      "learning_rate": 0.00012457310606060605,
      "loss": 2.8523,
      "step": 71200
    },
    {
      "activations/layer0_attention_weight_max": 16.89666175842285,
      "activations/layer0_attention_weight_min": -11.838812828063965,
      "activations/layer10_attention_weight_max": 33.655494689941406,
      "activations/layer10_attention_weight_min": -28.465118408203125,
      "activations/layer11_attention_weight_max": 30.45973014831543,
      "activations/layer11_attention_weight_min": -26.40594482421875,
      "activations/layer12_attention_weight_max": 36.36451721191406,
      "activations/layer12_attention_weight_min": -26.011459350585938,
      "activations/layer13_attention_weight_max": 54.244441986083984,
      "activations/layer13_attention_weight_min": -37.31616973876953,
      "activations/layer14_attention_weight_max": 62.73344421386719,
      "activations/layer14_attention_weight_min": -45.92874526977539,
      "activations/layer15_attention_weight_max": 42.20154571533203,
      "activations/layer15_attention_weight_min": -27.843870162963867,
      "activations/layer16_attention_weight_max": 40.985233306884766,
      "activations/layer16_attention_weight_min": -28.463760375976562,
      "activations/layer17_attention_weight_max": 39.34278869628906,
      "activations/layer17_attention_weight_min": -25.343069076538086,
      "activations/layer18_attention_weight_max": 32.499881744384766,
      "activations/layer18_attention_weight_min": -20.174654006958008,
      "activations/layer19_attention_weight_max": 46.685001373291016,
      "activations/layer19_attention_weight_min": -28.765573501586914,
      "activations/layer1_attention_weight_max": 17.393875122070312,
      "activations/layer1_attention_weight_min": -14.821515083312988,
      "activations/layer20_attention_weight_max": 35.62693786621094,
      "activations/layer20_attention_weight_min": -23.307720184326172,
      "activations/layer21_attention_weight_max": 37.65467834472656,
      "activations/layer21_attention_weight_min": -19.45289421081543,
      "activations/layer22_attention_weight_max": 51.40924072265625,
      "activations/layer22_attention_weight_min": -28.47938346862793,
      "activations/layer23_attention_weight_max": 31.161224365234375,
      "activations/layer23_attention_weight_min": -19.542728424072266,
      "activations/layer2_attention_weight_max": 31.43086051940918,
      "activations/layer2_attention_weight_min": -30.442943572998047,
      "activations/layer3_attention_weight_max": 82.81820678710938,
      "activations/layer3_attention_weight_min": -84.5009765625,
      "activations/layer4_attention_weight_max": 131.26187133789062,
      "activations/layer4_attention_weight_min": -126.85909271240234,
      "activations/layer5_attention_weight_max": 87.77714538574219,
      "activations/layer5_attention_weight_min": -84.29280090332031,
      "activations/layer6_attention_weight_max": 53.77002716064453,
      "activations/layer6_attention_weight_min": -56.274139404296875,
      "activations/layer7_attention_weight_max": 74.60003662109375,
      "activations/layer7_attention_weight_min": -76.79427337646484,
      "activations/layer8_attention_weight_max": 38.59853744506836,
      "activations/layer8_attention_weight_min": -37.41646194458008,
      "activations/layer9_attention_weight_max": 34.76396179199219,
      "activations/layer9_attention_weight_min": -30.965045928955078,
      "epoch": 4.14,
      "learning_rate": 0.00012455416666666666,
      "loss": 2.8863,
      "step": 71250
    },
    {
      "activations/layer0_attention_weight_max": 16.844959259033203,
      "activations/layer0_attention_weight_min": -12.66218090057373,
      "activations/layer10_attention_weight_max": 36.89002227783203,
      "activations/layer10_attention_weight_min": -30.52382469177246,
      "activations/layer11_attention_weight_max": 33.53033447265625,
      "activations/layer11_attention_weight_min": -30.027196884155273,
      "activations/layer12_attention_weight_max": 28.969253540039062,
      "activations/layer12_attention_weight_min": -25.93987274169922,
      "activations/layer13_attention_weight_max": 45.88302230834961,
      "activations/layer13_attention_weight_min": -32.671722412109375,
      "activations/layer14_attention_weight_max": 67.42376708984375,
      "activations/layer14_attention_weight_min": -49.090003967285156,
      "activations/layer15_attention_weight_max": 42.707801818847656,
      "activations/layer15_attention_weight_min": -29.13039779663086,
      "activations/layer16_attention_weight_max": 38.53830337524414,
      "activations/layer16_attention_weight_min": -27.558740615844727,
      "activations/layer17_attention_weight_max": 44.70488357543945,
      "activations/layer17_attention_weight_min": -27.31119155883789,
      "activations/layer18_attention_weight_max": 33.40720748901367,
      "activations/layer18_attention_weight_min": -20.345104217529297,
      "activations/layer19_attention_weight_max": 47.70546340942383,
      "activations/layer19_attention_weight_min": -28.679397583007812,
      "activations/layer1_attention_weight_max": 18.40621566772461,
      "activations/layer1_attention_weight_min": -14.220014572143555,
      "activations/layer20_attention_weight_max": 34.66676712036133,
      "activations/layer20_attention_weight_min": -23.708837509155273,
      "activations/layer21_attention_weight_max": 35.58130645751953,
      "activations/layer21_attention_weight_min": -20.26155662536621,
      "activations/layer22_attention_weight_max": 52.87469482421875,
      "activations/layer22_attention_weight_min": -28.307018280029297,
      "activations/layer23_attention_weight_max": 33.3486442565918,
      "activations/layer23_attention_weight_min": -19.353939056396484,
      "activations/layer2_attention_weight_max": 31.690176010131836,
      "activations/layer2_attention_weight_min": -31.86298179626465,
      "activations/layer3_attention_weight_max": 85.4198226928711,
      "activations/layer3_attention_weight_min": -90.19970703125,
      "activations/layer4_attention_weight_max": 129.5388946533203,
      "activations/layer4_attention_weight_min": -126.8582763671875,
      "activations/layer5_attention_weight_max": 88.82618713378906,
      "activations/layer5_attention_weight_min": -83.01419830322266,
      "activations/layer6_attention_weight_max": 56.08772659301758,
      "activations/layer6_attention_weight_min": -56.25205612182617,
      "activations/layer7_attention_weight_max": 77.17198181152344,
      "activations/layer7_attention_weight_min": -80.23271179199219,
      "activations/layer8_attention_weight_max": 39.197845458984375,
      "activations/layer8_attention_weight_min": -38.21623611450195,
      "activations/layer9_attention_weight_max": 36.65083694458008,
      "activations/layer9_attention_weight_min": -33.073768615722656,
      "epoch": 4.14,
      "learning_rate": 0.00012453522727272728,
      "loss": 2.8802,
      "step": 71300
    },
    {
      "activations/layer0_attention_weight_max": 17.285249710083008,
      "activations/layer0_attention_weight_min": -13.554614067077637,
      "activations/layer10_attention_weight_max": 36.78911209106445,
      "activations/layer10_attention_weight_min": -31.283504486083984,
      "activations/layer11_attention_weight_max": 31.941844940185547,
      "activations/layer11_attention_weight_min": -27.118335723876953,
      "activations/layer12_attention_weight_max": 31.305461883544922,
      "activations/layer12_attention_weight_min": -30.81262969970703,
      "activations/layer13_attention_weight_max": 44.40738296508789,
      "activations/layer13_attention_weight_min": -32.9122428894043,
      "activations/layer14_attention_weight_max": 64.0240478515625,
      "activations/layer14_attention_weight_min": -47.91964340209961,
      "activations/layer15_attention_weight_max": 45.105342864990234,
      "activations/layer15_attention_weight_min": -30.794178009033203,
      "activations/layer16_attention_weight_max": 41.496585845947266,
      "activations/layer16_attention_weight_min": -29.574922561645508,
      "activations/layer17_attention_weight_max": 47.33002853393555,
      "activations/layer17_attention_weight_min": -28.36083221435547,
      "activations/layer18_attention_weight_max": 32.97944641113281,
      "activations/layer18_attention_weight_min": -20.342750549316406,
      "activations/layer19_attention_weight_max": 42.678470611572266,
      "activations/layer19_attention_weight_min": -29.752775192260742,
      "activations/layer1_attention_weight_max": 18.29526138305664,
      "activations/layer1_attention_weight_min": -14.610570907592773,
      "activations/layer20_attention_weight_max": 34.1867561340332,
      "activations/layer20_attention_weight_min": -25.490800857543945,
      "activations/layer21_attention_weight_max": 36.551177978515625,
      "activations/layer21_attention_weight_min": -21.11537742614746,
      "activations/layer22_attention_weight_max": 49.72007369995117,
      "activations/layer22_attention_weight_min": -32.60532760620117,
      "activations/layer23_attention_weight_max": 31.100963592529297,
      "activations/layer23_attention_weight_min": -22.306842803955078,
      "activations/layer2_attention_weight_max": 33.47920227050781,
      "activations/layer2_attention_weight_min": -31.358572006225586,
      "activations/layer3_attention_weight_max": 88.75306701660156,
      "activations/layer3_attention_weight_min": -87.0555191040039,
      "activations/layer4_attention_weight_max": 130.91285705566406,
      "activations/layer4_attention_weight_min": -130.98391723632812,
      "activations/layer5_attention_weight_max": 89.73814392089844,
      "activations/layer5_attention_weight_min": -82.09557342529297,
      "activations/layer6_attention_weight_max": 60.07136917114258,
      "activations/layer6_attention_weight_min": -56.74028778076172,
      "activations/layer7_attention_weight_max": 82.32931518554688,
      "activations/layer7_attention_weight_min": -79.3728256225586,
      "activations/layer8_attention_weight_max": 41.1129264831543,
      "activations/layer8_attention_weight_min": -37.0814323425293,
      "activations/layer9_attention_weight_max": 37.782623291015625,
      "activations/layer9_attention_weight_min": -37.1440544128418,
      "epoch": 4.15,
      "learning_rate": 0.00012451628787878786,
      "loss": 2.866,
      "step": 71350
    },
    {
      "activations/layer0_attention_weight_max": 16.67103385925293,
      "activations/layer0_attention_weight_min": -10.872669219970703,
      "activations/layer10_attention_weight_max": 31.986160278320312,
      "activations/layer10_attention_weight_min": -28.374704360961914,
      "activations/layer11_attention_weight_max": 31.65682029724121,
      "activations/layer11_attention_weight_min": -25.418256759643555,
      "activations/layer12_attention_weight_max": 27.121721267700195,
      "activations/layer12_attention_weight_min": -26.2181453704834,
      "activations/layer13_attention_weight_max": 42.39619827270508,
      "activations/layer13_attention_weight_min": -34.13334655761719,
      "activations/layer14_attention_weight_max": 58.882545471191406,
      "activations/layer14_attention_weight_min": -46.87051010131836,
      "activations/layer15_attention_weight_max": 41.699825286865234,
      "activations/layer15_attention_weight_min": -29.26712417602539,
      "activations/layer16_attention_weight_max": 36.38835144042969,
      "activations/layer16_attention_weight_min": -26.813209533691406,
      "activations/layer17_attention_weight_max": 35.98645782470703,
      "activations/layer17_attention_weight_min": -24.220836639404297,
      "activations/layer18_attention_weight_max": 28.463354110717773,
      "activations/layer18_attention_weight_min": -20.017959594726562,
      "activations/layer19_attention_weight_max": 38.21043014526367,
      "activations/layer19_attention_weight_min": -29.868783950805664,
      "activations/layer1_attention_weight_max": 19.686418533325195,
      "activations/layer1_attention_weight_min": -14.564897537231445,
      "activations/layer20_attention_weight_max": 30.592988967895508,
      "activations/layer20_attention_weight_min": -24.412755966186523,
      "activations/layer21_attention_weight_max": 30.269432067871094,
      "activations/layer21_attention_weight_min": -21.124475479125977,
      "activations/layer22_attention_weight_max": 48.57976150512695,
      "activations/layer22_attention_weight_min": -29.250852584838867,
      "activations/layer23_attention_weight_max": 28.773853302001953,
      "activations/layer23_attention_weight_min": -19.92327880859375,
      "activations/layer2_attention_weight_max": 31.556007385253906,
      "activations/layer2_attention_weight_min": -30.520917892456055,
      "activations/layer3_attention_weight_max": 86.7576904296875,
      "activations/layer3_attention_weight_min": -85.68228912353516,
      "activations/layer4_attention_weight_max": 132.6949920654297,
      "activations/layer4_attention_weight_min": -125.19051361083984,
      "activations/layer5_attention_weight_max": 90.11296844482422,
      "activations/layer5_attention_weight_min": -83.48644256591797,
      "activations/layer6_attention_weight_max": 56.22233963012695,
      "activations/layer6_attention_weight_min": -57.87460708618164,
      "activations/layer7_attention_weight_max": 84.90953826904297,
      "activations/layer7_attention_weight_min": -74.36769104003906,
      "activations/layer8_attention_weight_max": 36.41054153442383,
      "activations/layer8_attention_weight_min": -38.139076232910156,
      "activations/layer9_attention_weight_max": 37.55495834350586,
      "activations/layer9_attention_weight_min": -32.69150924682617,
      "epoch": 4.15,
      "learning_rate": 0.00012449734848484848,
      "loss": 2.8479,
      "step": 71400
    },
    {
      "activations/layer0_attention_weight_max": 15.800286293029785,
      "activations/layer0_attention_weight_min": -12.139190673828125,
      "activations/layer10_attention_weight_max": 34.98780059814453,
      "activations/layer10_attention_weight_min": -30.207164764404297,
      "activations/layer11_attention_weight_max": 35.46756362915039,
      "activations/layer11_attention_weight_min": -27.383686065673828,
      "activations/layer12_attention_weight_max": 30.59853744506836,
      "activations/layer12_attention_weight_min": -26.93004608154297,
      "activations/layer13_attention_weight_max": 47.74629211425781,
      "activations/layer13_attention_weight_min": -33.25633239746094,
      "activations/layer14_attention_weight_max": 80.66517639160156,
      "activations/layer14_attention_weight_min": -61.96479415893555,
      "activations/layer15_attention_weight_max": 39.13603591918945,
      "activations/layer15_attention_weight_min": -30.92586326599121,
      "activations/layer16_attention_weight_max": 39.9135856628418,
      "activations/layer16_attention_weight_min": -30.53308868408203,
      "activations/layer17_attention_weight_max": 40.12141799926758,
      "activations/layer17_attention_weight_min": -25.0908145904541,
      "activations/layer18_attention_weight_max": 30.43879508972168,
      "activations/layer18_attention_weight_min": -18.527767181396484,
      "activations/layer19_attention_weight_max": 42.20585250854492,
      "activations/layer19_attention_weight_min": -28.940717697143555,
      "activations/layer1_attention_weight_max": 17.810640335083008,
      "activations/layer1_attention_weight_min": -13.429484367370605,
      "activations/layer20_attention_weight_max": 32.246124267578125,
      "activations/layer20_attention_weight_min": -23.734031677246094,
      "activations/layer21_attention_weight_max": 34.883384704589844,
      "activations/layer21_attention_weight_min": -19.251611709594727,
      "activations/layer22_attention_weight_max": 52.48678207397461,
      "activations/layer22_attention_weight_min": -27.639076232910156,
      "activations/layer23_attention_weight_max": 32.10538864135742,
      "activations/layer23_attention_weight_min": -17.57377815246582,
      "activations/layer2_attention_weight_max": 32.00275421142578,
      "activations/layer2_attention_weight_min": -32.2874870300293,
      "activations/layer3_attention_weight_max": 91.3729019165039,
      "activations/layer3_attention_weight_min": -94.60078430175781,
      "activations/layer4_attention_weight_max": 144.02711486816406,
      "activations/layer4_attention_weight_min": -129.1678009033203,
      "activations/layer5_attention_weight_max": 91.91722106933594,
      "activations/layer5_attention_weight_min": -85.92912292480469,
      "activations/layer6_attention_weight_max": 57.682106018066406,
      "activations/layer6_attention_weight_min": -56.53227996826172,
      "activations/layer7_attention_weight_max": 78.86288452148438,
      "activations/layer7_attention_weight_min": -78.49321746826172,
      "activations/layer8_attention_weight_max": 42.49003219604492,
      "activations/layer8_attention_weight_min": -40.116455078125,
      "activations/layer9_attention_weight_max": 36.675045013427734,
      "activations/layer9_attention_weight_min": -33.09755325317383,
      "epoch": 4.15,
      "learning_rate": 0.0001244784090909091,
      "loss": 2.8591,
      "step": 71450
    },
    {
      "activations/layer0_attention_weight_max": 16.592132568359375,
      "activations/layer0_attention_weight_min": -11.891423225402832,
      "activations/layer10_attention_weight_max": 38.04533004760742,
      "activations/layer10_attention_weight_min": -30.188003540039062,
      "activations/layer11_attention_weight_max": 35.72052001953125,
      "activations/layer11_attention_weight_min": -27.000417709350586,
      "activations/layer12_attention_weight_max": 34.06975555419922,
      "activations/layer12_attention_weight_min": -27.954200744628906,
      "activations/layer13_attention_weight_max": 45.635746002197266,
      "activations/layer13_attention_weight_min": -33.14420700073242,
      "activations/layer14_attention_weight_max": 63.6743049621582,
      "activations/layer14_attention_weight_min": -50.4261360168457,
      "activations/layer15_attention_weight_max": 39.14331817626953,
      "activations/layer15_attention_weight_min": -30.584880828857422,
      "activations/layer16_attention_weight_max": 39.16474151611328,
      "activations/layer16_attention_weight_min": -33.28242492675781,
      "activations/layer17_attention_weight_max": 39.52069091796875,
      "activations/layer17_attention_weight_min": -23.43413543701172,
      "activations/layer18_attention_weight_max": 28.799259185791016,
      "activations/layer18_attention_weight_min": -19.1683406829834,
      "activations/layer19_attention_weight_max": 38.43190383911133,
      "activations/layer19_attention_weight_min": -30.17356300354004,
      "activations/layer1_attention_weight_max": 17.06644058227539,
      "activations/layer1_attention_weight_min": -17.343708038330078,
      "activations/layer20_attention_weight_max": 34.46719741821289,
      "activations/layer20_attention_weight_min": -23.86908531188965,
      "activations/layer21_attention_weight_max": 28.469533920288086,
      "activations/layer21_attention_weight_min": -20.154569625854492,
      "activations/layer22_attention_weight_max": 46.012474060058594,
      "activations/layer22_attention_weight_min": -27.19241714477539,
      "activations/layer23_attention_weight_max": 30.998891830444336,
      "activations/layer23_attention_weight_min": -19.074487686157227,
      "activations/layer2_attention_weight_max": 31.529754638671875,
      "activations/layer2_attention_weight_min": -30.545822143554688,
      "activations/layer3_attention_weight_max": 87.33661651611328,
      "activations/layer3_attention_weight_min": -86.9480972290039,
      "activations/layer4_attention_weight_max": 130.1594696044922,
      "activations/layer4_attention_weight_min": -125.63917541503906,
      "activations/layer5_attention_weight_max": 88.32156372070312,
      "activations/layer5_attention_weight_min": -80.30294036865234,
      "activations/layer6_attention_weight_max": 54.53227233886719,
      "activations/layer6_attention_weight_min": -55.582740783691406,
      "activations/layer7_attention_weight_max": 79.60015106201172,
      "activations/layer7_attention_weight_min": -78.68740844726562,
      "activations/layer8_attention_weight_max": 37.97416687011719,
      "activations/layer8_attention_weight_min": -39.24592971801758,
      "activations/layer9_attention_weight_max": 37.68452072143555,
      "activations/layer9_attention_weight_min": -33.15914535522461,
      "epoch": 4.15,
      "learning_rate": 0.00012445946969696968,
      "loss": 2.8644,
      "step": 71500
    },
    {
      "activations/layer0_attention_weight_max": 15.677148818969727,
      "activations/layer0_attention_weight_min": -10.852778434753418,
      "activations/layer10_attention_weight_max": 34.36384201049805,
      "activations/layer10_attention_weight_min": -30.46933937072754,
      "activations/layer11_attention_weight_max": 32.683013916015625,
      "activations/layer11_attention_weight_min": -27.98050308227539,
      "activations/layer12_attention_weight_max": 41.62580108642578,
      "activations/layer12_attention_weight_min": -28.26590347290039,
      "activations/layer13_attention_weight_max": 48.40332794189453,
      "activations/layer13_attention_weight_min": -38.151588439941406,
      "activations/layer14_attention_weight_max": 60.588191986083984,
      "activations/layer14_attention_weight_min": -50.196739196777344,
      "activations/layer15_attention_weight_max": 44.85129928588867,
      "activations/layer15_attention_weight_min": -27.87479591369629,
      "activations/layer16_attention_weight_max": 35.50729751586914,
      "activations/layer16_attention_weight_min": -27.18299674987793,
      "activations/layer17_attention_weight_max": 44.74642562866211,
      "activations/layer17_attention_weight_min": -24.97590446472168,
      "activations/layer18_attention_weight_max": 33.85322189331055,
      "activations/layer18_attention_weight_min": -19.258756637573242,
      "activations/layer19_attention_weight_max": 37.26258850097656,
      "activations/layer19_attention_weight_min": -28.60646629333496,
      "activations/layer1_attention_weight_max": 17.90985679626465,
      "activations/layer1_attention_weight_min": -15.1658296585083,
      "activations/layer20_attention_weight_max": 32.82703399658203,
      "activations/layer20_attention_weight_min": -24.561124801635742,
      "activations/layer21_attention_weight_max": 29.286954879760742,
      "activations/layer21_attention_weight_min": -21.082590103149414,
      "activations/layer22_attention_weight_max": 46.77974319458008,
      "activations/layer22_attention_weight_min": -31.32830810546875,
      "activations/layer23_attention_weight_max": 30.573692321777344,
      "activations/layer23_attention_weight_min": -19.048355102539062,
      "activations/layer2_attention_weight_max": 30.81244468688965,
      "activations/layer2_attention_weight_min": -30.900381088256836,
      "activations/layer3_attention_weight_max": 83.74042510986328,
      "activations/layer3_attention_weight_min": -88.81835174560547,
      "activations/layer4_attention_weight_max": 129.0735321044922,
      "activations/layer4_attention_weight_min": -125.59966278076172,
      "activations/layer5_attention_weight_max": 91.6905517578125,
      "activations/layer5_attention_weight_min": -80.64336395263672,
      "activations/layer6_attention_weight_max": 57.82765197753906,
      "activations/layer6_attention_weight_min": -56.8303337097168,
      "activations/layer7_attention_weight_max": 78.5698013305664,
      "activations/layer7_attention_weight_min": -73.64694213867188,
      "activations/layer8_attention_weight_max": 38.22779083251953,
      "activations/layer8_attention_weight_min": -37.40742111206055,
      "activations/layer9_attention_weight_max": 36.765899658203125,
      "activations/layer9_attention_weight_min": -32.05476760864258,
      "epoch": 4.16,
      "learning_rate": 0.0001244405303030303,
      "loss": 2.8737,
      "step": 71550
    },
    {
      "activations/layer0_attention_weight_max": 16.6447811126709,
      "activations/layer0_attention_weight_min": -10.861346244812012,
      "activations/layer10_attention_weight_max": 35.05878829956055,
      "activations/layer10_attention_weight_min": -30.28915786743164,
      "activations/layer11_attention_weight_max": 35.32493591308594,
      "activations/layer11_attention_weight_min": -29.545135498046875,
      "activations/layer12_attention_weight_max": 30.328750610351562,
      "activations/layer12_attention_weight_min": -23.95469856262207,
      "activations/layer13_attention_weight_max": 53.814632415771484,
      "activations/layer13_attention_weight_min": -37.383785247802734,
      "activations/layer14_attention_weight_max": 60.356258392333984,
      "activations/layer14_attention_weight_min": -43.08434295654297,
      "activations/layer15_attention_weight_max": 40.02526092529297,
      "activations/layer15_attention_weight_min": -28.406110763549805,
      "activations/layer16_attention_weight_max": 38.48278045654297,
      "activations/layer16_attention_weight_min": -27.836183547973633,
      "activations/layer17_attention_weight_max": 42.0444221496582,
      "activations/layer17_attention_weight_min": -30.053781509399414,
      "activations/layer18_attention_weight_max": 35.826629638671875,
      "activations/layer18_attention_weight_min": -23.583444595336914,
      "activations/layer19_attention_weight_max": 45.01811981201172,
      "activations/layer19_attention_weight_min": -28.875965118408203,
      "activations/layer1_attention_weight_max": 17.647357940673828,
      "activations/layer1_attention_weight_min": -14.447312355041504,
      "activations/layer20_attention_weight_max": 33.85245895385742,
      "activations/layer20_attention_weight_min": -22.342973709106445,
      "activations/layer21_attention_weight_max": 38.31887435913086,
      "activations/layer21_attention_weight_min": -22.109268188476562,
      "activations/layer22_attention_weight_max": 51.65338897705078,
      "activations/layer22_attention_weight_min": -30.22309684753418,
      "activations/layer23_attention_weight_max": 30.76767349243164,
      "activations/layer23_attention_weight_min": -18.634666442871094,
      "activations/layer2_attention_weight_max": 31.584732055664062,
      "activations/layer2_attention_weight_min": -32.46916198730469,
      "activations/layer3_attention_weight_max": 88.80181121826172,
      "activations/layer3_attention_weight_min": -93.85111236572266,
      "activations/layer4_attention_weight_max": 129.928466796875,
      "activations/layer4_attention_weight_min": -127.0571517944336,
      "activations/layer5_attention_weight_max": 88.07540130615234,
      "activations/layer5_attention_weight_min": -84.61026000976562,
      "activations/layer6_attention_weight_max": 57.29644775390625,
      "activations/layer6_attention_weight_min": -60.15882873535156,
      "activations/layer7_attention_weight_max": 80.96820831298828,
      "activations/layer7_attention_weight_min": -75.41871643066406,
      "activations/layer8_attention_weight_max": 38.75994873046875,
      "activations/layer8_attention_weight_min": -40.16439437866211,
      "activations/layer9_attention_weight_max": 37.592124938964844,
      "activations/layer9_attention_weight_min": -37.32581329345703,
      "epoch": 4.16,
      "learning_rate": 0.00012442159090909088,
      "loss": 2.8803,
      "step": 71600
    },
    {
      "activations/layer0_attention_weight_max": 15.921894073486328,
      "activations/layer0_attention_weight_min": -12.428462982177734,
      "activations/layer10_attention_weight_max": 39.16987991333008,
      "activations/layer10_attention_weight_min": -29.580223083496094,
      "activations/layer11_attention_weight_max": 44.63373947143555,
      "activations/layer11_attention_weight_min": -29.736465454101562,
      "activations/layer12_attention_weight_max": 53.027687072753906,
      "activations/layer12_attention_weight_min": -28.03723907470703,
      "activations/layer13_attention_weight_max": 66.76024627685547,
      "activations/layer13_attention_weight_min": -40.99089813232422,
      "activations/layer14_attention_weight_max": 78.81376647949219,
      "activations/layer14_attention_weight_min": -51.47287368774414,
      "activations/layer15_attention_weight_max": 58.384822845458984,
      "activations/layer15_attention_weight_min": -28.658283233642578,
      "activations/layer16_attention_weight_max": 50.35531997680664,
      "activations/layer16_attention_weight_min": -29.186676025390625,
      "activations/layer17_attention_weight_max": 43.36890411376953,
      "activations/layer17_attention_weight_min": -24.63277244567871,
      "activations/layer18_attention_weight_max": 31.557390213012695,
      "activations/layer18_attention_weight_min": -21.669048309326172,
      "activations/layer19_attention_weight_max": 52.35402297973633,
      "activations/layer19_attention_weight_min": -29.69852066040039,
      "activations/layer1_attention_weight_max": 17.24826431274414,
      "activations/layer1_attention_weight_min": -15.375056266784668,
      "activations/layer20_attention_weight_max": 37.92759704589844,
      "activations/layer20_attention_weight_min": -22.993091583251953,
      "activations/layer21_attention_weight_max": 31.427213668823242,
      "activations/layer21_attention_weight_min": -23.666399002075195,
      "activations/layer22_attention_weight_max": 55.67838668823242,
      "activations/layer22_attention_weight_min": -30.80681037902832,
      "activations/layer23_attention_weight_max": 34.523529052734375,
      "activations/layer23_attention_weight_min": -21.580890655517578,
      "activations/layer2_attention_weight_max": 32.455596923828125,
      "activations/layer2_attention_weight_min": -30.85407829284668,
      "activations/layer3_attention_weight_max": 87.77686309814453,
      "activations/layer3_attention_weight_min": -88.74000549316406,
      "activations/layer4_attention_weight_max": 138.17564392089844,
      "activations/layer4_attention_weight_min": -133.8566436767578,
      "activations/layer5_attention_weight_max": 93.03115844726562,
      "activations/layer5_attention_weight_min": -92.94124603271484,
      "activations/layer6_attention_weight_max": 61.330509185791016,
      "activations/layer6_attention_weight_min": -56.09028244018555,
      "activations/layer7_attention_weight_max": 91.46804809570312,
      "activations/layer7_attention_weight_min": -75.41948699951172,
      "activations/layer8_attention_weight_max": 44.4339599609375,
      "activations/layer8_attention_weight_min": -38.61943817138672,
      "activations/layer9_attention_weight_max": 51.580230712890625,
      "activations/layer9_attention_weight_min": -36.232688903808594,
      "epoch": 4.16,
      "learning_rate": 0.0001244026515151515,
      "loss": 2.8724,
      "step": 71650
    },
    {
      "activations/layer0_attention_weight_max": 14.497530937194824,
      "activations/layer0_attention_weight_min": -12.500015258789062,
      "activations/layer10_attention_weight_max": 36.215702056884766,
      "activations/layer10_attention_weight_min": -32.32704162597656,
      "activations/layer11_attention_weight_max": 37.32514190673828,
      "activations/layer11_attention_weight_min": -30.47650146484375,
      "activations/layer12_attention_weight_max": 34.32279968261719,
      "activations/layer12_attention_weight_min": -25.075244903564453,
      "activations/layer13_attention_weight_max": 64.05844116210938,
      "activations/layer13_attention_weight_min": -43.20072937011719,
      "activations/layer14_attention_weight_max": 75.85731506347656,
      "activations/layer14_attention_weight_min": -54.37649917602539,
      "activations/layer15_attention_weight_max": 50.68721389770508,
      "activations/layer15_attention_weight_min": -32.35578918457031,
      "activations/layer16_attention_weight_max": 39.57749557495117,
      "activations/layer16_attention_weight_min": -29.28615951538086,
      "activations/layer17_attention_weight_max": 39.72584915161133,
      "activations/layer17_attention_weight_min": -23.733264923095703,
      "activations/layer18_attention_weight_max": 38.432960510253906,
      "activations/layer18_attention_weight_min": -20.13888168334961,
      "activations/layer19_attention_weight_max": 45.0308723449707,
      "activations/layer19_attention_weight_min": -29.533878326416016,
      "activations/layer1_attention_weight_max": 17.829286575317383,
      "activations/layer1_attention_weight_min": -15.222259521484375,
      "activations/layer20_attention_weight_max": 35.94621276855469,
      "activations/layer20_attention_weight_min": -24.53514862060547,
      "activations/layer21_attention_weight_max": 36.98105239868164,
      "activations/layer21_attention_weight_min": -20.252782821655273,
      "activations/layer22_attention_weight_max": 51.58637237548828,
      "activations/layer22_attention_weight_min": -26.764997482299805,
      "activations/layer23_attention_weight_max": 33.74399948120117,
      "activations/layer23_attention_weight_min": -20.820323944091797,
      "activations/layer2_attention_weight_max": 31.500198364257812,
      "activations/layer2_attention_weight_min": -31.080337524414062,
      "activations/layer3_attention_weight_max": 90.25379180908203,
      "activations/layer3_attention_weight_min": -91.06236267089844,
      "activations/layer4_attention_weight_max": 131.9344940185547,
      "activations/layer4_attention_weight_min": -132.1129913330078,
      "activations/layer5_attention_weight_max": 90.01763153076172,
      "activations/layer5_attention_weight_min": -85.83808898925781,
      "activations/layer6_attention_weight_max": 60.82317352294922,
      "activations/layer6_attention_weight_min": -58.69382858276367,
      "activations/layer7_attention_weight_max": 82.6219711303711,
      "activations/layer7_attention_weight_min": -81.89930725097656,
      "activations/layer8_attention_weight_max": 40.80424880981445,
      "activations/layer8_attention_weight_min": -41.8701171875,
      "activations/layer9_attention_weight_max": 44.255672454833984,
      "activations/layer9_attention_weight_min": -33.77314758300781,
      "epoch": 4.17,
      "learning_rate": 0.00012438371212121211,
      "loss": 2.8839,
      "step": 71700
    },
    {
      "activations/layer0_attention_weight_max": 16.4697265625,
      "activations/layer0_attention_weight_min": -12.155174255371094,
      "activations/layer10_attention_weight_max": 36.905555725097656,
      "activations/layer10_attention_weight_min": -30.85295295715332,
      "activations/layer11_attention_weight_max": 35.99394226074219,
      "activations/layer11_attention_weight_min": -27.19428825378418,
      "activations/layer12_attention_weight_max": 26.836715698242188,
      "activations/layer12_attention_weight_min": -27.576444625854492,
      "activations/layer13_attention_weight_max": 48.25798034667969,
      "activations/layer13_attention_weight_min": -35.0087776184082,
      "activations/layer14_attention_weight_max": 78.35047149658203,
      "activations/layer14_attention_weight_min": -52.851863861083984,
      "activations/layer15_attention_weight_max": 47.821346282958984,
      "activations/layer15_attention_weight_min": -29.431058883666992,
      "activations/layer16_attention_weight_max": 45.01333236694336,
      "activations/layer16_attention_weight_min": -28.924755096435547,
      "activations/layer17_attention_weight_max": 46.50453186035156,
      "activations/layer17_attention_weight_min": -24.219324111938477,
      "activations/layer18_attention_weight_max": 34.06364059448242,
      "activations/layer18_attention_weight_min": -20.374629974365234,
      "activations/layer19_attention_weight_max": 51.72397994995117,
      "activations/layer19_attention_weight_min": -31.720563888549805,
      "activations/layer1_attention_weight_max": 18.411237716674805,
      "activations/layer1_attention_weight_min": -15.26964282989502,
      "activations/layer20_attention_weight_max": 39.88822555541992,
      "activations/layer20_attention_weight_min": -22.478063583374023,
      "activations/layer21_attention_weight_max": 33.57484817504883,
      "activations/layer21_attention_weight_min": -19.7154541015625,
      "activations/layer22_attention_weight_max": 57.88919448852539,
      "activations/layer22_attention_weight_min": -26.44782829284668,
      "activations/layer23_attention_weight_max": 36.46742630004883,
      "activations/layer23_attention_weight_min": -17.512603759765625,
      "activations/layer2_attention_weight_max": 31.42833709716797,
      "activations/layer2_attention_weight_min": -31.324241638183594,
      "activations/layer3_attention_weight_max": 87.7568588256836,
      "activations/layer3_attention_weight_min": -91.59081268310547,
      "activations/layer4_attention_weight_max": 130.70388793945312,
      "activations/layer4_attention_weight_min": -129.92991638183594,
      "activations/layer5_attention_weight_max": 92.01094818115234,
      "activations/layer5_attention_weight_min": -86.32628631591797,
      "activations/layer6_attention_weight_max": 59.66789245605469,
      "activations/layer6_attention_weight_min": -56.08018112182617,
      "activations/layer7_attention_weight_max": 78.12217712402344,
      "activations/layer7_attention_weight_min": -82.21424865722656,
      "activations/layer8_attention_weight_max": 39.88113021850586,
      "activations/layer8_attention_weight_min": -40.46129608154297,
      "activations/layer9_attention_weight_max": 39.05187225341797,
      "activations/layer9_attention_weight_min": -42.339378356933594,
      "epoch": 4.17,
      "learning_rate": 0.0001243647727272727,
      "loss": 2.8746,
      "step": 71750
    },
    {
      "activations/layer0_attention_weight_max": 16.303508758544922,
      "activations/layer0_attention_weight_min": -11.054548263549805,
      "activations/layer10_attention_weight_max": 39.183746337890625,
      "activations/layer10_attention_weight_min": -30.795202255249023,
      "activations/layer11_attention_weight_max": 45.30101013183594,
      "activations/layer11_attention_weight_min": -30.296255111694336,
      "activations/layer12_attention_weight_max": 36.29020309448242,
      "activations/layer12_attention_weight_min": -31.611478805541992,
      "activations/layer13_attention_weight_max": 45.61774444580078,
      "activations/layer13_attention_weight_min": -32.99793243408203,
      "activations/layer14_attention_weight_max": 66.95127868652344,
      "activations/layer14_attention_weight_min": -41.174190521240234,
      "activations/layer15_attention_weight_max": 44.2807502746582,
      "activations/layer15_attention_weight_min": -28.69618034362793,
      "activations/layer16_attention_weight_max": 44.211822509765625,
      "activations/layer16_attention_weight_min": -27.501888275146484,
      "activations/layer17_attention_weight_max": 42.35297393798828,
      "activations/layer17_attention_weight_min": -24.687114715576172,
      "activations/layer18_attention_weight_max": 36.855804443359375,
      "activations/layer18_attention_weight_min": -19.857418060302734,
      "activations/layer19_attention_weight_max": 44.77170181274414,
      "activations/layer19_attention_weight_min": -28.3324031829834,
      "activations/layer1_attention_weight_max": 17.416349411010742,
      "activations/layer1_attention_weight_min": -14.825119018554688,
      "activations/layer20_attention_weight_max": 36.35327911376953,
      "activations/layer20_attention_weight_min": -25.601272583007812,
      "activations/layer21_attention_weight_max": 38.441917419433594,
      "activations/layer21_attention_weight_min": -22.609092712402344,
      "activations/layer22_attention_weight_max": 50.08358383178711,
      "activations/layer22_attention_weight_min": -27.616785049438477,
      "activations/layer23_attention_weight_max": 32.37354278564453,
      "activations/layer23_attention_weight_min": -18.10822296142578,
      "activations/layer2_attention_weight_max": 32.437889099121094,
      "activations/layer2_attention_weight_min": -31.822790145874023,
      "activations/layer3_attention_weight_max": 90.44329071044922,
      "activations/layer3_attention_weight_min": -93.89208221435547,
      "activations/layer4_attention_weight_max": 140.01974487304688,
      "activations/layer4_attention_weight_min": -139.204833984375,
      "activations/layer5_attention_weight_max": 102.20722961425781,
      "activations/layer5_attention_weight_min": -93.4854736328125,
      "activations/layer6_attention_weight_max": 62.29267883300781,
      "activations/layer6_attention_weight_min": -62.25204849243164,
      "activations/layer7_attention_weight_max": 85.85443115234375,
      "activations/layer7_attention_weight_min": -81.62712097167969,
      "activations/layer8_attention_weight_max": 45.44385528564453,
      "activations/layer8_attention_weight_min": -39.54927444458008,
      "activations/layer9_attention_weight_max": 45.23105239868164,
      "activations/layer9_attention_weight_min": -38.13003158569336,
      "epoch": 4.17,
      "learning_rate": 0.00012434583333333332,
      "loss": 2.8561,
      "step": 71800
    },
    {
      "activations/layer0_attention_weight_max": 16.595834732055664,
      "activations/layer0_attention_weight_min": -12.224175453186035,
      "activations/layer10_attention_weight_max": 35.80366134643555,
      "activations/layer10_attention_weight_min": -29.517229080200195,
      "activations/layer11_attention_weight_max": 35.72405242919922,
      "activations/layer11_attention_weight_min": -27.755741119384766,
      "activations/layer12_attention_weight_max": 32.6861457824707,
      "activations/layer12_attention_weight_min": -27.942853927612305,
      "activations/layer13_attention_weight_max": 57.0543212890625,
      "activations/layer13_attention_weight_min": -36.65070343017578,
      "activations/layer14_attention_weight_max": 70.3625259399414,
      "activations/layer14_attention_weight_min": -49.30437088012695,
      "activations/layer15_attention_weight_max": 46.4821891784668,
      "activations/layer15_attention_weight_min": -26.723234176635742,
      "activations/layer16_attention_weight_max": 41.237571716308594,
      "activations/layer16_attention_weight_min": -29.129987716674805,
      "activations/layer17_attention_weight_max": 44.854496002197266,
      "activations/layer17_attention_weight_min": -26.27762222290039,
      "activations/layer18_attention_weight_max": 32.80462646484375,
      "activations/layer18_attention_weight_min": -20.96084976196289,
      "activations/layer19_attention_weight_max": 47.03046798706055,
      "activations/layer19_attention_weight_min": -31.118499755859375,
      "activations/layer1_attention_weight_max": 17.046842575073242,
      "activations/layer1_attention_weight_min": -14.650273323059082,
      "activations/layer20_attention_weight_max": 37.90985870361328,
      "activations/layer20_attention_weight_min": -26.101022720336914,
      "activations/layer21_attention_weight_max": 39.534889221191406,
      "activations/layer21_attention_weight_min": -20.351825714111328,
      "activations/layer22_attention_weight_max": 56.03342819213867,
      "activations/layer22_attention_weight_min": -27.81198501586914,
      "activations/layer23_attention_weight_max": 35.55638885498047,
      "activations/layer23_attention_weight_min": -18.42523193359375,
      "activations/layer2_attention_weight_max": 31.826786041259766,
      "activations/layer2_attention_weight_min": -31.239248275756836,
      "activations/layer3_attention_weight_max": 86.27896118164062,
      "activations/layer3_attention_weight_min": -88.85709381103516,
      "activations/layer4_attention_weight_max": 130.6648712158203,
      "activations/layer4_attention_weight_min": -131.1343994140625,
      "activations/layer5_attention_weight_max": 87.89543914794922,
      "activations/layer5_attention_weight_min": -81.70161437988281,
      "activations/layer6_attention_weight_max": 57.74767303466797,
      "activations/layer6_attention_weight_min": -56.39264678955078,
      "activations/layer7_attention_weight_max": 79.14952087402344,
      "activations/layer7_attention_weight_min": -78.50261688232422,
      "activations/layer8_attention_weight_max": 41.4993896484375,
      "activations/layer8_attention_weight_min": -39.391258239746094,
      "activations/layer9_attention_weight_max": 38.272621154785156,
      "activations/layer9_attention_weight_min": -34.9380989074707,
      "epoch": 4.17,
      "learning_rate": 0.00012432689393939393,
      "loss": 2.8804,
      "step": 71850
    },
    {
      "activations/layer0_attention_weight_max": 16.118698120117188,
      "activations/layer0_attention_weight_min": -12.779058456420898,
      "activations/layer10_attention_weight_max": 35.66079330444336,
      "activations/layer10_attention_weight_min": -29.94492530822754,
      "activations/layer11_attention_weight_max": 29.70635223388672,
      "activations/layer11_attention_weight_min": -27.497474670410156,
      "activations/layer12_attention_weight_max": 27.45507049560547,
      "activations/layer12_attention_weight_min": -30.788372039794922,
      "activations/layer13_attention_weight_max": 44.11141586303711,
      "activations/layer13_attention_weight_min": -35.1120719909668,
      "activations/layer14_attention_weight_max": 79.99508666992188,
      "activations/layer14_attention_weight_min": -51.76072692871094,
      "activations/layer15_attention_weight_max": 38.277671813964844,
      "activations/layer15_attention_weight_min": -29.16136360168457,
      "activations/layer16_attention_weight_max": 43.80696105957031,
      "activations/layer16_attention_weight_min": -28.962261199951172,
      "activations/layer17_attention_weight_max": 39.07630157470703,
      "activations/layer17_attention_weight_min": -23.93157386779785,
      "activations/layer18_attention_weight_max": 33.2302131652832,
      "activations/layer18_attention_weight_min": -19.336936950683594,
      "activations/layer19_attention_weight_max": 52.17928695678711,
      "activations/layer19_attention_weight_min": -30.01380729675293,
      "activations/layer1_attention_weight_max": 16.630985260009766,
      "activations/layer1_attention_weight_min": -13.816328048706055,
      "activations/layer20_attention_weight_max": 35.58504867553711,
      "activations/layer20_attention_weight_min": -24.4803409576416,
      "activations/layer21_attention_weight_max": 35.5598258972168,
      "activations/layer21_attention_weight_min": -22.1883544921875,
      "activations/layer22_attention_weight_max": 57.4328727722168,
      "activations/layer22_attention_weight_min": -25.128578186035156,
      "activations/layer23_attention_weight_max": 33.12385559082031,
      "activations/layer23_attention_weight_min": -17.227205276489258,
      "activations/layer2_attention_weight_max": 30.766027450561523,
      "activations/layer2_attention_weight_min": -30.81134033203125,
      "activations/layer3_attention_weight_max": 84.45366668701172,
      "activations/layer3_attention_weight_min": -88.71041107177734,
      "activations/layer4_attention_weight_max": 126.9797592163086,
      "activations/layer4_attention_weight_min": -127.2684326171875,
      "activations/layer5_attention_weight_max": 88.14111328125,
      "activations/layer5_attention_weight_min": -85.65459442138672,
      "activations/layer6_attention_weight_max": 57.50801467895508,
      "activations/layer6_attention_weight_min": -54.41738510131836,
      "activations/layer7_attention_weight_max": 79.03131103515625,
      "activations/layer7_attention_weight_min": -82.04399108886719,
      "activations/layer8_attention_weight_max": 38.610687255859375,
      "activations/layer8_attention_weight_min": -39.35616683959961,
      "activations/layer9_attention_weight_max": 38.53285598754883,
      "activations/layer9_attention_weight_min": -33.01616668701172,
      "epoch": 4.18,
      "learning_rate": 0.00012430795454545452,
      "loss": 2.8626,
      "step": 71900
    },
    {
      "activations/layer0_attention_weight_max": 16.504680633544922,
      "activations/layer0_attention_weight_min": -12.335237503051758,
      "activations/layer10_attention_weight_max": 34.00709915161133,
      "activations/layer10_attention_weight_min": -29.90462303161621,
      "activations/layer11_attention_weight_max": 33.44108581542969,
      "activations/layer11_attention_weight_min": -29.021461486816406,
      "activations/layer12_attention_weight_max": 33.87101364135742,
      "activations/layer12_attention_weight_min": -26.2684383392334,
      "activations/layer13_attention_weight_max": 43.18767547607422,
      "activations/layer13_attention_weight_min": -34.880470275878906,
      "activations/layer14_attention_weight_max": 67.18234252929688,
      "activations/layer14_attention_weight_min": -48.812538146972656,
      "activations/layer15_attention_weight_max": 40.70022964477539,
      "activations/layer15_attention_weight_min": -28.54775619506836,
      "activations/layer16_attention_weight_max": 41.35361099243164,
      "activations/layer16_attention_weight_min": -28.360754013061523,
      "activations/layer17_attention_weight_max": 38.48640823364258,
      "activations/layer17_attention_weight_min": -26.027631759643555,
      "activations/layer18_attention_weight_max": 31.093833923339844,
      "activations/layer18_attention_weight_min": -20.45368003845215,
      "activations/layer19_attention_weight_max": 43.79755783081055,
      "activations/layer19_attention_weight_min": -29.289112091064453,
      "activations/layer1_attention_weight_max": 17.46636390686035,
      "activations/layer1_attention_weight_min": -13.470986366271973,
      "activations/layer20_attention_weight_max": 31.2686767578125,
      "activations/layer20_attention_weight_min": -22.671438217163086,
      "activations/layer21_attention_weight_max": 31.3288631439209,
      "activations/layer21_attention_weight_min": -19.932621002197266,
      "activations/layer22_attention_weight_max": 47.67543029785156,
      "activations/layer22_attention_weight_min": -28.14323616027832,
      "activations/layer23_attention_weight_max": 31.789196014404297,
      "activations/layer23_attention_weight_min": -18.421611785888672,
      "activations/layer2_attention_weight_max": 30.850372314453125,
      "activations/layer2_attention_weight_min": -30.21215057373047,
      "activations/layer3_attention_weight_max": 84.32296752929688,
      "activations/layer3_attention_weight_min": -86.99662017822266,
      "activations/layer4_attention_weight_max": 127.23628997802734,
      "activations/layer4_attention_weight_min": -128.79367065429688,
      "activations/layer5_attention_weight_max": 85.48199462890625,
      "activations/layer5_attention_weight_min": -79.36012268066406,
      "activations/layer6_attention_weight_max": 54.7307243347168,
      "activations/layer6_attention_weight_min": -55.54077911376953,
      "activations/layer7_attention_weight_max": 75.83876037597656,
      "activations/layer7_attention_weight_min": -76.53973388671875,
      "activations/layer8_attention_weight_max": 37.63803482055664,
      "activations/layer8_attention_weight_min": -35.61308288574219,
      "activations/layer9_attention_weight_max": 36.246681213378906,
      "activations/layer9_attention_weight_min": -30.95491600036621,
      "epoch": 4.18,
      "learning_rate": 0.00012428901515151514,
      "loss": 2.898,
      "step": 71950
    },
    {
      "activations/layer0_attention_weight_max": 16.389232635498047,
      "activations/layer0_attention_weight_min": -11.469377517700195,
      "activations/layer10_attention_weight_max": 36.048587799072266,
      "activations/layer10_attention_weight_min": -29.848363876342773,
      "activations/layer11_attention_weight_max": 35.901161193847656,
      "activations/layer11_attention_weight_min": -27.670989990234375,
      "activations/layer12_attention_weight_max": 27.583778381347656,
      "activations/layer12_attention_weight_min": -34.03376007080078,
      "activations/layer13_attention_weight_max": 42.83908462524414,
      "activations/layer13_attention_weight_min": -31.507287979125977,
      "activations/layer14_attention_weight_max": 64.25680541992188,
      "activations/layer14_attention_weight_min": -48.07032775878906,
      "activations/layer15_attention_weight_max": 39.14593505859375,
      "activations/layer15_attention_weight_min": -28.66731071472168,
      "activations/layer16_attention_weight_max": 41.28749465942383,
      "activations/layer16_attention_weight_min": -29.134157180786133,
      "activations/layer17_attention_weight_max": 39.0738410949707,
      "activations/layer17_attention_weight_min": -24.611942291259766,
      "activations/layer18_attention_weight_max": 29.636751174926758,
      "activations/layer18_attention_weight_min": -20.669078826904297,
      "activations/layer19_attention_weight_max": 44.46882247924805,
      "activations/layer19_attention_weight_min": -27.99146842956543,
      "activations/layer1_attention_weight_max": 16.762678146362305,
      "activations/layer1_attention_weight_min": -14.044499397277832,
      "activations/layer20_attention_weight_max": 36.173057556152344,
      "activations/layer20_attention_weight_min": -22.812725067138672,
      "activations/layer21_attention_weight_max": 35.98567199707031,
      "activations/layer21_attention_weight_min": -19.82077980041504,
      "activations/layer22_attention_weight_max": 52.88286209106445,
      "activations/layer22_attention_weight_min": -29.49112892150879,
      "activations/layer23_attention_weight_max": 36.55223083496094,
      "activations/layer23_attention_weight_min": -22.32906150817871,
      "activations/layer2_attention_weight_max": 31.82614517211914,
      "activations/layer2_attention_weight_min": -32.313228607177734,
      "activations/layer3_attention_weight_max": 89.1224136352539,
      "activations/layer3_attention_weight_min": -90.52249908447266,
      "activations/layer4_attention_weight_max": 130.9687042236328,
      "activations/layer4_attention_weight_min": -128.91685485839844,
      "activations/layer5_attention_weight_max": 89.21788024902344,
      "activations/layer5_attention_weight_min": -85.02701568603516,
      "activations/layer6_attention_weight_max": 56.831939697265625,
      "activations/layer6_attention_weight_min": -58.39567565917969,
      "activations/layer7_attention_weight_max": 77.49008178710938,
      "activations/layer7_attention_weight_min": -78.20462036132812,
      "activations/layer8_attention_weight_max": 36.97316360473633,
      "activations/layer8_attention_weight_min": -37.915504455566406,
      "activations/layer9_attention_weight_max": 37.48374938964844,
      "activations/layer9_attention_weight_min": -33.19867706298828,
      "epoch": 4.18,
      "learning_rate": 0.00012427007575757575,
      "loss": 2.875,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_loss": 2.8203125,
      "eval_runtime": 8.453,
      "eval_samples_per_second": 507.988,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_openwebtext_loss": 2.8203125,
      "eval_openwebtext_ppl": 16.782094257242417,
      "eval_openwebtext_runtime": 8.453,
      "eval_openwebtext_samples_per_second": 507.988,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_wikitext_loss": 3.056640625,
      "eval_wikitext_ppl": 21.256030110048442,
      "eval_wikitext_runtime": 2.0126,
      "eval_wikitext_samples_per_second": 226.571,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_lambada_loss": 2.7421875,
      "eval_lambada_ppl": 15.520899957093135,
      "eval_lambada_runtime": 9.5498,
      "eval_lambada_samples_per_second": 509.852,
      "step": 72000
    },
    {
      "activations/layer0_attention_weight_max": 16.74361801147461,
      "activations/layer0_attention_weight_min": -12.207979202270508,
      "activations/layer10_attention_weight_max": 37.55707550048828,
      "activations/layer10_attention_weight_min": -31.353708267211914,
      "activations/layer11_attention_weight_max": 32.10611343383789,
      "activations/layer11_attention_weight_min": -27.165233612060547,
      "activations/layer12_attention_weight_max": 31.24818229675293,
      "activations/layer12_attention_weight_min": -31.251935958862305,
      "activations/layer13_attention_weight_max": 43.28152847290039,
      "activations/layer13_attention_weight_min": -33.26850509643555,
      "activations/layer14_attention_weight_max": 84.8556900024414,
      "activations/layer14_attention_weight_min": -63.27528762817383,
      "activations/layer15_attention_weight_max": 42.0018196105957,
      "activations/layer15_attention_weight_min": -29.80319595336914,
      "activations/layer16_attention_weight_max": 38.18545150756836,
      "activations/layer16_attention_weight_min": -26.784910202026367,
      "activations/layer17_attention_weight_max": 41.673980712890625,
      "activations/layer17_attention_weight_min": -25.294729232788086,
      "activations/layer18_attention_weight_max": 40.37450408935547,
      "activations/layer18_attention_weight_min": -19.285493850708008,
      "activations/layer19_attention_weight_max": 44.08853530883789,
      "activations/layer19_attention_weight_min": -27.683929443359375,
      "activations/layer1_attention_weight_max": 18.36499786376953,
      "activations/layer1_attention_weight_min": -17.672060012817383,
      "activations/layer20_attention_weight_max": 41.154170989990234,
      "activations/layer20_attention_weight_min": -22.958593368530273,
      "activations/layer21_attention_weight_max": 37.686031341552734,
      "activations/layer21_attention_weight_min": -20.77630615234375,
      "activations/layer22_attention_weight_max": 49.25889205932617,
      "activations/layer22_attention_weight_min": -29.33608627319336,
      "activations/layer23_attention_weight_max": 31.798742294311523,
      "activations/layer23_attention_weight_min": -19.63174819946289,
      "activations/layer2_attention_weight_max": 31.28841781616211,
      "activations/layer2_attention_weight_min": -30.70757293701172,
      "activations/layer3_attention_weight_max": 86.54904174804688,
      "activations/layer3_attention_weight_min": -89.59425354003906,
      "activations/layer4_attention_weight_max": 128.37376403808594,
      "activations/layer4_attention_weight_min": -127.59696197509766,
      "activations/layer5_attention_weight_max": 92.60104370117188,
      "activations/layer5_attention_weight_min": -87.43792724609375,
      "activations/layer6_attention_weight_max": 62.76118850708008,
      "activations/layer6_attention_weight_min": -61.066253662109375,
      "activations/layer7_attention_weight_max": 78.33949279785156,
      "activations/layer7_attention_weight_min": -83.4751968383789,
      "activations/layer8_attention_weight_max": 41.91144561767578,
      "activations/layer8_attention_weight_min": -39.2120246887207,
      "activations/layer9_attention_weight_max": 38.57036590576172,
      "activations/layer9_attention_weight_min": -33.43872833251953,
      "epoch": 4.19,
      "learning_rate": 0.00012425113636363634,
      "loss": 2.866,
      "step": 72050
    },
    {
      "activations/layer0_attention_weight_max": 14.209634780883789,
      "activations/layer0_attention_weight_min": -10.59206771850586,
      "activations/layer10_attention_weight_max": 35.62604522705078,
      "activations/layer10_attention_weight_min": -31.040512084960938,
      "activations/layer11_attention_weight_max": 36.59022521972656,
      "activations/layer11_attention_weight_min": -31.846927642822266,
      "activations/layer12_attention_weight_max": 55.6779670715332,
      "activations/layer12_attention_weight_min": -26.86675453186035,
      "activations/layer13_attention_weight_max": 62.53501892089844,
      "activations/layer13_attention_weight_min": -40.3754997253418,
      "activations/layer14_attention_weight_max": 71.49923706054688,
      "activations/layer14_attention_weight_min": -49.40460968017578,
      "activations/layer15_attention_weight_max": 41.748634338378906,
      "activations/layer15_attention_weight_min": -28.58152198791504,
      "activations/layer16_attention_weight_max": 43.008331298828125,
      "activations/layer16_attention_weight_min": -28.915292739868164,
      "activations/layer17_attention_weight_max": 41.290679931640625,
      "activations/layer17_attention_weight_min": -25.634286880493164,
      "activations/layer18_attention_weight_max": 30.82183265686035,
      "activations/layer18_attention_weight_min": -18.85308074951172,
      "activations/layer19_attention_weight_max": 47.55794143676758,
      "activations/layer19_attention_weight_min": -29.151111602783203,
      "activations/layer1_attention_weight_max": 17.287561416625977,
      "activations/layer1_attention_weight_min": -13.901265144348145,
      "activations/layer20_attention_weight_max": 37.187252044677734,
      "activations/layer20_attention_weight_min": -24.659990310668945,
      "activations/layer21_attention_weight_max": 37.767913818359375,
      "activations/layer21_attention_weight_min": -21.528011322021484,
      "activations/layer22_attention_weight_max": 57.670204162597656,
      "activations/layer22_attention_weight_min": -29.35690689086914,
      "activations/layer23_attention_weight_max": 37.68572235107422,
      "activations/layer23_attention_weight_min": -16.91011619567871,
      "activations/layer2_attention_weight_max": 33.91642761230469,
      "activations/layer2_attention_weight_min": -31.976261138916016,
      "activations/layer3_attention_weight_max": 90.10708618164062,
      "activations/layer3_attention_weight_min": -92.29247283935547,
      "activations/layer4_attention_weight_max": 138.52633666992188,
      "activations/layer4_attention_weight_min": -133.6352996826172,
      "activations/layer5_attention_weight_max": 97.07258605957031,
      "activations/layer5_attention_weight_min": -87.38191986083984,
      "activations/layer6_attention_weight_max": 63.46131134033203,
      "activations/layer6_attention_weight_min": -57.46674728393555,
      "activations/layer7_attention_weight_max": 79.72472381591797,
      "activations/layer7_attention_weight_min": -78.69803619384766,
      "activations/layer8_attention_weight_max": 41.65974044799805,
      "activations/layer8_attention_weight_min": -38.774417877197266,
      "activations/layer9_attention_weight_max": 37.7502555847168,
      "activations/layer9_attention_weight_min": -34.39082336425781,
      "epoch": 4.19,
      "learning_rate": 0.00012423219696969695,
      "loss": 2.8654,
      "step": 72100
    },
    {
      "activations/layer0_attention_weight_max": 16.38811492919922,
      "activations/layer0_attention_weight_min": -12.102864265441895,
      "activations/layer10_attention_weight_max": 41.733524322509766,
      "activations/layer10_attention_weight_min": -33.04262161254883,
      "activations/layer11_attention_weight_max": 38.74222946166992,
      "activations/layer11_attention_weight_min": -28.82574462890625,
      "activations/layer12_attention_weight_max": 33.07110595703125,
      "activations/layer12_attention_weight_min": -28.280210494995117,
      "activations/layer13_attention_weight_max": 62.386558532714844,
      "activations/layer13_attention_weight_min": -34.78335189819336,
      "activations/layer14_attention_weight_max": 92.761474609375,
      "activations/layer14_attention_weight_min": -54.640289306640625,
      "activations/layer15_attention_weight_max": 53.11172103881836,
      "activations/layer15_attention_weight_min": -30.030393600463867,
      "activations/layer16_attention_weight_max": 50.59126281738281,
      "activations/layer16_attention_weight_min": -31.12017822265625,
      "activations/layer17_attention_weight_max": 43.04018020629883,
      "activations/layer17_attention_weight_min": -26.075973510742188,
      "activations/layer18_attention_weight_max": 34.962005615234375,
      "activations/layer18_attention_weight_min": -20.018136978149414,
      "activations/layer19_attention_weight_max": 52.11406326293945,
      "activations/layer19_attention_weight_min": -29.29437828063965,
      "activations/layer1_attention_weight_max": 17.446836471557617,
      "activations/layer1_attention_weight_min": -15.94090747833252,
      "activations/layer20_attention_weight_max": 41.99858474731445,
      "activations/layer20_attention_weight_min": -23.953676223754883,
      "activations/layer21_attention_weight_max": 38.73685073852539,
      "activations/layer21_attention_weight_min": -20.673980712890625,
      "activations/layer22_attention_weight_max": 56.8095817565918,
      "activations/layer22_attention_weight_min": -29.111421585083008,
      "activations/layer23_attention_weight_max": 35.798912048339844,
      "activations/layer23_attention_weight_min": -19.003005981445312,
      "activations/layer2_attention_weight_max": 32.4179801940918,
      "activations/layer2_attention_weight_min": -31.556615829467773,
      "activations/layer3_attention_weight_max": 84.01715087890625,
      "activations/layer3_attention_weight_min": -87.0938720703125,
      "activations/layer4_attention_weight_max": 132.51145935058594,
      "activations/layer4_attention_weight_min": -125.19258117675781,
      "activations/layer5_attention_weight_max": 92.72222900390625,
      "activations/layer5_attention_weight_min": -82.462158203125,
      "activations/layer6_attention_weight_max": 58.5179328918457,
      "activations/layer6_attention_weight_min": -56.48688507080078,
      "activations/layer7_attention_weight_max": 80.30660247802734,
      "activations/layer7_attention_weight_min": -76.68090057373047,
      "activations/layer8_attention_weight_max": 40.83234786987305,
      "activations/layer8_attention_weight_min": -39.62986755371094,
      "activations/layer9_attention_weight_max": 41.59628677368164,
      "activations/layer9_attention_weight_min": -32.713321685791016,
      "epoch": 4.19,
      "learning_rate": 0.00012421325757575757,
      "loss": 2.8698,
      "step": 72150
    },
    {
      "activations/layer0_attention_weight_max": 16.705101013183594,
      "activations/layer0_attention_weight_min": -10.636327743530273,
      "activations/layer10_attention_weight_max": 32.7745246887207,
      "activations/layer10_attention_weight_min": -30.931102752685547,
      "activations/layer11_attention_weight_max": 32.700286865234375,
      "activations/layer11_attention_weight_min": -26.98944664001465,
      "activations/layer12_attention_weight_max": 29.139076232910156,
      "activations/layer12_attention_weight_min": -25.947519302368164,
      "activations/layer13_attention_weight_max": 45.73445129394531,
      "activations/layer13_attention_weight_min": -36.66228103637695,
      "activations/layer14_attention_weight_max": 60.32307815551758,
      "activations/layer14_attention_weight_min": -53.19760513305664,
      "activations/layer15_attention_weight_max": 43.421871185302734,
      "activations/layer15_attention_weight_min": -29.609952926635742,
      "activations/layer16_attention_weight_max": 35.17625045776367,
      "activations/layer16_attention_weight_min": -29.154129028320312,
      "activations/layer17_attention_weight_max": 40.500152587890625,
      "activations/layer17_attention_weight_min": -25.382539749145508,
      "activations/layer18_attention_weight_max": 31.08690071105957,
      "activations/layer18_attention_weight_min": -23.48343849182129,
      "activations/layer19_attention_weight_max": 39.734554290771484,
      "activations/layer19_attention_weight_min": -30.488510131835938,
      "activations/layer1_attention_weight_max": 17.29629135131836,
      "activations/layer1_attention_weight_min": -14.070358276367188,
      "activations/layer20_attention_weight_max": 36.30475616455078,
      "activations/layer20_attention_weight_min": -26.544414520263672,
      "activations/layer21_attention_weight_max": 31.835630416870117,
      "activations/layer21_attention_weight_min": -21.929346084594727,
      "activations/layer22_attention_weight_max": 51.784217834472656,
      "activations/layer22_attention_weight_min": -28.31451988220215,
      "activations/layer23_attention_weight_max": 30.164772033691406,
      "activations/layer23_attention_weight_min": -18.297157287597656,
      "activations/layer2_attention_weight_max": 31.536664962768555,
      "activations/layer2_attention_weight_min": -31.254364013671875,
      "activations/layer3_attention_weight_max": 87.29216003417969,
      "activations/layer3_attention_weight_min": -94.2094955444336,
      "activations/layer4_attention_weight_max": 129.2102508544922,
      "activations/layer4_attention_weight_min": -130.5875701904297,
      "activations/layer5_attention_weight_max": 88.17057800292969,
      "activations/layer5_attention_weight_min": -82.8343276977539,
      "activations/layer6_attention_weight_max": 55.744384765625,
      "activations/layer6_attention_weight_min": -57.24665069580078,
      "activations/layer7_attention_weight_max": 76.3979721069336,
      "activations/layer7_attention_weight_min": -79.5726089477539,
      "activations/layer8_attention_weight_max": 37.752864837646484,
      "activations/layer8_attention_weight_min": -37.15511703491211,
      "activations/layer9_attention_weight_max": 37.52665328979492,
      "activations/layer9_attention_weight_min": -33.568870544433594,
      "epoch": 4.2,
      "learning_rate": 0.00012419431818181816,
      "loss": 2.8688,
      "step": 72200
    },
    {
      "activations/layer0_attention_weight_max": 15.374640464782715,
      "activations/layer0_attention_weight_min": -11.210346221923828,
      "activations/layer10_attention_weight_max": 35.82622528076172,
      "activations/layer10_attention_weight_min": -31.52866554260254,
      "activations/layer11_attention_weight_max": 35.1552619934082,
      "activations/layer11_attention_weight_min": -32.053001403808594,
      "activations/layer12_attention_weight_max": 29.22661018371582,
      "activations/layer12_attention_weight_min": -26.83263397216797,
      "activations/layer13_attention_weight_max": 50.94572448730469,
      "activations/layer13_attention_weight_min": -33.96196365356445,
      "activations/layer14_attention_weight_max": 78.59138488769531,
      "activations/layer14_attention_weight_min": -50.75442886352539,
      "activations/layer15_attention_weight_max": 42.212223052978516,
      "activations/layer15_attention_weight_min": -28.14857292175293,
      "activations/layer16_attention_weight_max": 39.6903076171875,
      "activations/layer16_attention_weight_min": -28.40135955810547,
      "activations/layer17_attention_weight_max": 40.55339431762695,
      "activations/layer17_attention_weight_min": -24.891244888305664,
      "activations/layer18_attention_weight_max": 33.80643081665039,
      "activations/layer18_attention_weight_min": -19.4791202545166,
      "activations/layer19_attention_weight_max": 43.415828704833984,
      "activations/layer19_attention_weight_min": -30.929061889648438,
      "activations/layer1_attention_weight_max": 17.8250732421875,
      "activations/layer1_attention_weight_min": -14.50153923034668,
      "activations/layer20_attention_weight_max": 35.206600189208984,
      "activations/layer20_attention_weight_min": -22.554903030395508,
      "activations/layer21_attention_weight_max": 33.61626052856445,
      "activations/layer21_attention_weight_min": -20.46658706665039,
      "activations/layer22_attention_weight_max": 50.88530731201172,
      "activations/layer22_attention_weight_min": -26.049171447753906,
      "activations/layer23_attention_weight_max": 30.624683380126953,
      "activations/layer23_attention_weight_min": -18.241304397583008,
      "activations/layer2_attention_weight_max": 32.99068069458008,
      "activations/layer2_attention_weight_min": -31.97675323486328,
      "activations/layer3_attention_weight_max": 86.5623550415039,
      "activations/layer3_attention_weight_min": -88.975341796875,
      "activations/layer4_attention_weight_max": 134.7838592529297,
      "activations/layer4_attention_weight_min": -127.48241424560547,
      "activations/layer5_attention_weight_max": 87.7270736694336,
      "activations/layer5_attention_weight_min": -85.7254867553711,
      "activations/layer6_attention_weight_max": 58.9270133972168,
      "activations/layer6_attention_weight_min": -56.52256393432617,
      "activations/layer7_attention_weight_max": 81.21527099609375,
      "activations/layer7_attention_weight_min": -83.03692626953125,
      "activations/layer8_attention_weight_max": 37.492919921875,
      "activations/layer8_attention_weight_min": -37.90646743774414,
      "activations/layer9_attention_weight_max": 40.58992385864258,
      "activations/layer9_attention_weight_min": -35.472991943359375,
      "epoch": 4.2,
      "learning_rate": 0.00012417537878787877,
      "loss": 2.862,
      "step": 72250
    },
    {
      "activations/layer0_attention_weight_max": 17.49758529663086,
      "activations/layer0_attention_weight_min": -12.22413444519043,
      "activations/layer10_attention_weight_max": 39.381649017333984,
      "activations/layer10_attention_weight_min": -31.861324310302734,
      "activations/layer11_attention_weight_max": 43.15121078491211,
      "activations/layer11_attention_weight_min": -26.909507751464844,
      "activations/layer12_attention_weight_max": 35.82765197753906,
      "activations/layer12_attention_weight_min": -28.103639602661133,
      "activations/layer13_attention_weight_max": 49.55545425415039,
      "activations/layer13_attention_weight_min": -34.36956024169922,
      "activations/layer14_attention_weight_max": 75.35625457763672,
      "activations/layer14_attention_weight_min": -51.449581146240234,
      "activations/layer15_attention_weight_max": 53.05978775024414,
      "activations/layer15_attention_weight_min": -30.750015258789062,
      "activations/layer16_attention_weight_max": 43.10329818725586,
      "activations/layer16_attention_weight_min": -29.45384407043457,
      "activations/layer17_attention_weight_max": 45.71162796020508,
      "activations/layer17_attention_weight_min": -24.585432052612305,
      "activations/layer18_attention_weight_max": 33.6496696472168,
      "activations/layer18_attention_weight_min": -20.471527099609375,
      "activations/layer19_attention_weight_max": 45.863616943359375,
      "activations/layer19_attention_weight_min": -27.93159294128418,
      "activations/layer1_attention_weight_max": 17.073593139648438,
      "activations/layer1_attention_weight_min": -17.559261322021484,
      "activations/layer20_attention_weight_max": 32.327606201171875,
      "activations/layer20_attention_weight_min": -22.145227432250977,
      "activations/layer21_attention_weight_max": 34.20585250854492,
      "activations/layer21_attention_weight_min": -20.573841094970703,
      "activations/layer22_attention_weight_max": 50.69252014160156,
      "activations/layer22_attention_weight_min": -30.559581756591797,
      "activations/layer23_attention_weight_max": 31.9553279876709,
      "activations/layer23_attention_weight_min": -19.88271141052246,
      "activations/layer2_attention_weight_max": 33.23958206176758,
      "activations/layer2_attention_weight_min": -31.956260681152344,
      "activations/layer3_attention_weight_max": 90.98709106445312,
      "activations/layer3_attention_weight_min": -91.38993072509766,
      "activations/layer4_attention_weight_max": 142.40103149414062,
      "activations/layer4_attention_weight_min": -138.30755615234375,
      "activations/layer5_attention_weight_max": 97.17916107177734,
      "activations/layer5_attention_weight_min": -84.68448638916016,
      "activations/layer6_attention_weight_max": 63.35501480102539,
      "activations/layer6_attention_weight_min": -54.80082702636719,
      "activations/layer7_attention_weight_max": 95.66889190673828,
      "activations/layer7_attention_weight_min": -87.17029571533203,
      "activations/layer8_attention_weight_max": 46.73563766479492,
      "activations/layer8_attention_weight_min": -43.57919692993164,
      "activations/layer9_attention_weight_max": 38.816558837890625,
      "activations/layer9_attention_weight_min": -39.50166702270508,
      "epoch": 4.2,
      "learning_rate": 0.0001241564393939394,
      "loss": 2.876,
      "step": 72300
    },
    {
      "activations/layer0_attention_weight_max": 16.433568954467773,
      "activations/layer0_attention_weight_min": -12.630732536315918,
      "activations/layer10_attention_weight_max": 33.39965057373047,
      "activations/layer10_attention_weight_min": -29.171123504638672,
      "activations/layer11_attention_weight_max": 30.835098266601562,
      "activations/layer11_attention_weight_min": -25.972576141357422,
      "activations/layer12_attention_weight_max": 33.10667037963867,
      "activations/layer12_attention_weight_min": -27.61833381652832,
      "activations/layer13_attention_weight_max": 49.0756950378418,
      "activations/layer13_attention_weight_min": -37.408382415771484,
      "activations/layer14_attention_weight_max": 63.95102310180664,
      "activations/layer14_attention_weight_min": -48.961360931396484,
      "activations/layer15_attention_weight_max": 39.99935531616211,
      "activations/layer15_attention_weight_min": -30.118558883666992,
      "activations/layer16_attention_weight_max": 39.284950256347656,
      "activations/layer16_attention_weight_min": -27.720815658569336,
      "activations/layer17_attention_weight_max": 39.01531219482422,
      "activations/layer17_attention_weight_min": -24.64086151123047,
      "activations/layer18_attention_weight_max": 29.079687118530273,
      "activations/layer18_attention_weight_min": -18.546749114990234,
      "activations/layer19_attention_weight_max": 44.049537658691406,
      "activations/layer19_attention_weight_min": -28.446691513061523,
      "activations/layer1_attention_weight_max": 17.684783935546875,
      "activations/layer1_attention_weight_min": -14.973203659057617,
      "activations/layer20_attention_weight_max": 36.58872604370117,
      "activations/layer20_attention_weight_min": -21.409683227539062,
      "activations/layer21_attention_weight_max": 33.396820068359375,
      "activations/layer21_attention_weight_min": -19.6833553314209,
      "activations/layer22_attention_weight_max": 51.23575973510742,
      "activations/layer22_attention_weight_min": -27.54178810119629,
      "activations/layer23_attention_weight_max": 31.383434295654297,
      "activations/layer23_attention_weight_min": -18.390350341796875,
      "activations/layer2_attention_weight_max": 30.96509552001953,
      "activations/layer2_attention_weight_min": -30.113414764404297,
      "activations/layer3_attention_weight_max": 83.50291442871094,
      "activations/layer3_attention_weight_min": -85.32919311523438,
      "activations/layer4_attention_weight_max": 125.80155181884766,
      "activations/layer4_attention_weight_min": -128.36471557617188,
      "activations/layer5_attention_weight_max": 86.23654174804688,
      "activations/layer5_attention_weight_min": -82.61940002441406,
      "activations/layer6_attention_weight_max": 56.684165954589844,
      "activations/layer6_attention_weight_min": -55.18739700317383,
      "activations/layer7_attention_weight_max": 78.11553955078125,
      "activations/layer7_attention_weight_min": -80.37455749511719,
      "activations/layer8_attention_weight_max": 38.88954544067383,
      "activations/layer8_attention_weight_min": -36.87740707397461,
      "activations/layer9_attention_weight_max": 37.692543029785156,
      "activations/layer9_attention_weight_min": -32.07297897338867,
      "epoch": 4.2,
      "learning_rate": 0.00012413825757575755,
      "loss": 2.8599,
      "step": 72350
    },
    {
      "activations/layer0_attention_weight_max": 16.26004409790039,
      "activations/layer0_attention_weight_min": -12.174932479858398,
      "activations/layer10_attention_weight_max": 37.084537506103516,
      "activations/layer10_attention_weight_min": -30.130386352539062,
      "activations/layer11_attention_weight_max": 35.4718132019043,
      "activations/layer11_attention_weight_min": -26.79987144470215,
      "activations/layer12_attention_weight_max": 30.411766052246094,
      "activations/layer12_attention_weight_min": -26.993539810180664,
      "activations/layer13_attention_weight_max": 46.444034576416016,
      "activations/layer13_attention_weight_min": -32.54651641845703,
      "activations/layer14_attention_weight_max": 61.28594207763672,
      "activations/layer14_attention_weight_min": -46.44779968261719,
      "activations/layer15_attention_weight_max": 40.11628341674805,
      "activations/layer15_attention_weight_min": -28.780054092407227,
      "activations/layer16_attention_weight_max": 40.175804138183594,
      "activations/layer16_attention_weight_min": -29.38633155822754,
      "activations/layer17_attention_weight_max": 40.41365051269531,
      "activations/layer17_attention_weight_min": -23.530099868774414,
      "activations/layer18_attention_weight_max": 40.810791015625,
      "activations/layer18_attention_weight_min": -21.728900909423828,
      "activations/layer19_attention_weight_max": 47.79291915893555,
      "activations/layer19_attention_weight_min": -29.755966186523438,
      "activations/layer1_attention_weight_max": 17.095293045043945,
      "activations/layer1_attention_weight_min": -13.759599685668945,
      "activations/layer20_attention_weight_max": 37.63960266113281,
      "activations/layer20_attention_weight_min": -24.865510940551758,
      "activations/layer21_attention_weight_max": 37.083045959472656,
      "activations/layer21_attention_weight_min": -21.11894416809082,
      "activations/layer22_attention_weight_max": 57.91632080078125,
      "activations/layer22_attention_weight_min": -26.981348037719727,
      "activations/layer23_attention_weight_max": 36.9467887878418,
      "activations/layer23_attention_weight_min": -18.84820556640625,
      "activations/layer2_attention_weight_max": 32.66568374633789,
      "activations/layer2_attention_weight_min": -31.138198852539062,
      "activations/layer3_attention_weight_max": 85.92505645751953,
      "activations/layer3_attention_weight_min": -88.20645904541016,
      "activations/layer4_attention_weight_max": 134.39308166503906,
      "activations/layer4_attention_weight_min": -125.45391845703125,
      "activations/layer5_attention_weight_max": 92.46286010742188,
      "activations/layer5_attention_weight_min": -86.59825897216797,
      "activations/layer6_attention_weight_max": 61.98652648925781,
      "activations/layer6_attention_weight_min": -61.82734298706055,
      "activations/layer7_attention_weight_max": 78.8961410522461,
      "activations/layer7_attention_weight_min": -76.3680191040039,
      "activations/layer8_attention_weight_max": 38.741817474365234,
      "activations/layer8_attention_weight_min": -39.017826080322266,
      "activations/layer9_attention_weight_max": 37.931209564208984,
      "activations/layer9_attention_weight_min": -37.645320892333984,
      "epoch": 4.21,
      "learning_rate": 0.00012411931818181817,
      "loss": 2.8735,
      "step": 72400
    },
    {
      "activations/layer0_attention_weight_max": 16.617883682250977,
      "activations/layer0_attention_weight_min": -11.25395679473877,
      "activations/layer10_attention_weight_max": 33.12281036376953,
      "activations/layer10_attention_weight_min": -30.762968063354492,
      "activations/layer11_attention_weight_max": 32.186309814453125,
      "activations/layer11_attention_weight_min": -25.57644271850586,
      "activations/layer12_attention_weight_max": 29.526063919067383,
      "activations/layer12_attention_weight_min": -27.661930084228516,
      "activations/layer13_attention_weight_max": 45.81792068481445,
      "activations/layer13_attention_weight_min": -33.727378845214844,
      "activations/layer14_attention_weight_max": 71.12226104736328,
      "activations/layer14_attention_weight_min": -49.557533264160156,
      "activations/layer15_attention_weight_max": 40.1697998046875,
      "activations/layer15_attention_weight_min": -27.51630401611328,
      "activations/layer16_attention_weight_max": 40.05122375488281,
      "activations/layer16_attention_weight_min": -28.2730712890625,
      "activations/layer17_attention_weight_max": 42.03805923461914,
      "activations/layer17_attention_weight_min": -26.432905197143555,
      "activations/layer18_attention_weight_max": 32.99094772338867,
      "activations/layer18_attention_weight_min": -19.351003646850586,
      "activations/layer19_attention_weight_max": 41.561279296875,
      "activations/layer19_attention_weight_min": -28.88825798034668,
      "activations/layer1_attention_weight_max": 17.082260131835938,
      "activations/layer1_attention_weight_min": -17.169986724853516,
      "activations/layer20_attention_weight_max": 33.20121765136719,
      "activations/layer20_attention_weight_min": -23.751131057739258,
      "activations/layer21_attention_weight_max": 36.93470001220703,
      "activations/layer21_attention_weight_min": -20.281457901000977,
      "activations/layer22_attention_weight_max": 49.635250091552734,
      "activations/layer22_attention_weight_min": -28.207321166992188,
      "activations/layer23_attention_weight_max": 32.76642608642578,
      "activations/layer23_attention_weight_min": -20.807619094848633,
      "activations/layer2_attention_weight_max": 31.35757827758789,
      "activations/layer2_attention_weight_min": -30.738521575927734,
      "activations/layer3_attention_weight_max": 85.7931900024414,
      "activations/layer3_attention_weight_min": -86.37491607666016,
      "activations/layer4_attention_weight_max": 125.44921875,
      "activations/layer4_attention_weight_min": -128.4412384033203,
      "activations/layer5_attention_weight_max": 90.39799499511719,
      "activations/layer5_attention_weight_min": -88.58470153808594,
      "activations/layer6_attention_weight_max": 54.28816604614258,
      "activations/layer6_attention_weight_min": -56.866981506347656,
      "activations/layer7_attention_weight_max": 74.61376190185547,
      "activations/layer7_attention_weight_min": -76.51318359375,
      "activations/layer8_attention_weight_max": 38.59549331665039,
      "activations/layer8_attention_weight_min": -38.6121711730957,
      "activations/layer9_attention_weight_max": 35.736934661865234,
      "activations/layer9_attention_weight_min": -32.8538703918457,
      "epoch": 4.21,
      "learning_rate": 0.00012410037878787878,
      "loss": 2.8618,
      "step": 72450
    },
    {
      "activations/layer0_attention_weight_max": 16.15279769897461,
      "activations/layer0_attention_weight_min": -13.076494216918945,
      "activations/layer10_attention_weight_max": 32.51218032836914,
      "activations/layer10_attention_weight_min": -29.036649703979492,
      "activations/layer11_attention_weight_max": 30.70082664489746,
      "activations/layer11_attention_weight_min": -26.956268310546875,
      "activations/layer12_attention_weight_max": 39.20356369018555,
      "activations/layer12_attention_weight_min": -28.369409561157227,
      "activations/layer13_attention_weight_max": 56.272132873535156,
      "activations/layer13_attention_weight_min": -41.9451789855957,
      "activations/layer14_attention_weight_max": 69.84233093261719,
      "activations/layer14_attention_weight_min": -46.38912582397461,
      "activations/layer15_attention_weight_max": 43.27681350708008,
      "activations/layer15_attention_weight_min": -28.169519424438477,
      "activations/layer16_attention_weight_max": 44.61239242553711,
      "activations/layer16_attention_weight_min": -30.28682518005371,
      "activations/layer17_attention_weight_max": 39.156341552734375,
      "activations/layer17_attention_weight_min": -24.8008975982666,
      "activations/layer18_attention_weight_max": 38.05998992919922,
      "activations/layer18_attention_weight_min": -20.910213470458984,
      "activations/layer19_attention_weight_max": 47.10282516479492,
      "activations/layer19_attention_weight_min": -30.405622482299805,
      "activations/layer1_attention_weight_max": 17.483173370361328,
      "activations/layer1_attention_weight_min": -15.494539260864258,
      "activations/layer20_attention_weight_max": 33.5418701171875,
      "activations/layer20_attention_weight_min": -24.163402557373047,
      "activations/layer21_attention_weight_max": 35.848907470703125,
      "activations/layer21_attention_weight_min": -20.511438369750977,
      "activations/layer22_attention_weight_max": 52.9570198059082,
      "activations/layer22_attention_weight_min": -31.330198287963867,
      "activations/layer23_attention_weight_max": 35.66768264770508,
      "activations/layer23_attention_weight_min": -20.89487075805664,
      "activations/layer2_attention_weight_max": 32.41987609863281,
      "activations/layer2_attention_weight_min": -31.998489379882812,
      "activations/layer3_attention_weight_max": 88.04389190673828,
      "activations/layer3_attention_weight_min": -90.190185546875,
      "activations/layer4_attention_weight_max": 132.5205535888672,
      "activations/layer4_attention_weight_min": -128.22679138183594,
      "activations/layer5_attention_weight_max": 87.30409240722656,
      "activations/layer5_attention_weight_min": -81.99043273925781,
      "activations/layer6_attention_weight_max": 55.40134048461914,
      "activations/layer6_attention_weight_min": -58.06871795654297,
      "activations/layer7_attention_weight_max": 77.8099594116211,
      "activations/layer7_attention_weight_min": -76.48218536376953,
      "activations/layer8_attention_weight_max": 39.294090270996094,
      "activations/layer8_attention_weight_min": -36.678253173828125,
      "activations/layer9_attention_weight_max": 36.57353973388672,
      "activations/layer9_attention_weight_min": -33.98991012573242,
      "epoch": 4.21,
      "learning_rate": 0.0001240814393939394,
      "loss": 2.8786,
      "step": 72500
    },
    {
      "activations/layer0_attention_weight_max": 17.785280227661133,
      "activations/layer0_attention_weight_min": -12.114657402038574,
      "activations/layer10_attention_weight_max": 44.88458251953125,
      "activations/layer10_attention_weight_min": -32.445255279541016,
      "activations/layer11_attention_weight_max": 43.48676300048828,
      "activations/layer11_attention_weight_min": -27.980512619018555,
      "activations/layer12_attention_weight_max": 43.74085235595703,
      "activations/layer12_attention_weight_min": -31.76658821105957,
      "activations/layer13_attention_weight_max": 57.753971099853516,
      "activations/layer13_attention_weight_min": -33.90177536010742,
      "activations/layer14_attention_weight_max": 85.07642364501953,
      "activations/layer14_attention_weight_min": -50.39833450317383,
      "activations/layer15_attention_weight_max": 45.5245246887207,
      "activations/layer15_attention_weight_min": -26.772579193115234,
      "activations/layer16_attention_weight_max": 46.32542419433594,
      "activations/layer16_attention_weight_min": -28.69049644470215,
      "activations/layer17_attention_weight_max": 44.4749641418457,
      "activations/layer17_attention_weight_min": -23.92588233947754,
      "activations/layer18_attention_weight_max": 37.03447341918945,
      "activations/layer18_attention_weight_min": -19.333511352539062,
      "activations/layer19_attention_weight_max": 45.944026947021484,
      "activations/layer19_attention_weight_min": -29.332639694213867,
      "activations/layer1_attention_weight_max": 17.61304473876953,
      "activations/layer1_attention_weight_min": -14.520930290222168,
      "activations/layer20_attention_weight_max": 40.43878936767578,
      "activations/layer20_attention_weight_min": -23.79893684387207,
      "activations/layer21_attention_weight_max": 35.203426361083984,
      "activations/layer21_attention_weight_min": -19.2432918548584,
      "activations/layer22_attention_weight_max": 56.22115707397461,
      "activations/layer22_attention_weight_min": -29.3909912109375,
      "activations/layer23_attention_weight_max": 34.091487884521484,
      "activations/layer23_attention_weight_min": -20.180105209350586,
      "activations/layer2_attention_weight_max": 33.631065368652344,
      "activations/layer2_attention_weight_min": -32.29046630859375,
      "activations/layer3_attention_weight_max": 92.24691772460938,
      "activations/layer3_attention_weight_min": -89.41154479980469,
      "activations/layer4_attention_weight_max": 138.2504425048828,
      "activations/layer4_attention_weight_min": -134.57135009765625,
      "activations/layer5_attention_weight_max": 95.7938461303711,
      "activations/layer5_attention_weight_min": -84.39179992675781,
      "activations/layer6_attention_weight_max": 61.0446662902832,
      "activations/layer6_attention_weight_min": -57.508792877197266,
      "activations/layer7_attention_weight_max": 89.60717010498047,
      "activations/layer7_attention_weight_min": -85.2486801147461,
      "activations/layer8_attention_weight_max": 43.23396682739258,
      "activations/layer8_attention_weight_min": -38.87126922607422,
      "activations/layer9_attention_weight_max": 44.45040512084961,
      "activations/layer9_attention_weight_min": -37.391963958740234,
      "epoch": 4.22,
      "learning_rate": 0.0001240625,
      "loss": 2.8757,
      "step": 72550
    },
    {
      "activations/layer0_attention_weight_max": 15.55424690246582,
      "activations/layer0_attention_weight_min": -12.432625770568848,
      "activations/layer10_attention_weight_max": 36.834083557128906,
      "activations/layer10_attention_weight_min": -30.33419418334961,
      "activations/layer11_attention_weight_max": 35.85074996948242,
      "activations/layer11_attention_weight_min": -28.081838607788086,
      "activations/layer12_attention_weight_max": 50.98442459106445,
      "activations/layer12_attention_weight_min": -25.985448837280273,
      "activations/layer13_attention_weight_max": 46.21493911743164,
      "activations/layer13_attention_weight_min": -31.55413055419922,
      "activations/layer14_attention_weight_max": 64.28009033203125,
      "activations/layer14_attention_weight_min": -50.500057220458984,
      "activations/layer15_attention_weight_max": 41.767948150634766,
      "activations/layer15_attention_weight_min": -29.621267318725586,
      "activations/layer16_attention_weight_max": 40.49696731567383,
      "activations/layer16_attention_weight_min": -28.922563552856445,
      "activations/layer17_attention_weight_max": 40.091896057128906,
      "activations/layer17_attention_weight_min": -25.350000381469727,
      "activations/layer18_attention_weight_max": 32.814796447753906,
      "activations/layer18_attention_weight_min": -19.787302017211914,
      "activations/layer19_attention_weight_max": 38.948612213134766,
      "activations/layer19_attention_weight_min": -28.21099281311035,
      "activations/layer1_attention_weight_max": 16.873552322387695,
      "activations/layer1_attention_weight_min": -15.276163101196289,
      "activations/layer20_attention_weight_max": 33.31523132324219,
      "activations/layer20_attention_weight_min": -24.032859802246094,
      "activations/layer21_attention_weight_max": 40.51166915893555,
      "activations/layer21_attention_weight_min": -20.472501754760742,
      "activations/layer22_attention_weight_max": 52.57073211669922,
      "activations/layer22_attention_weight_min": -27.92788314819336,
      "activations/layer23_attention_weight_max": 31.706283569335938,
      "activations/layer23_attention_weight_min": -19.935150146484375,
      "activations/layer2_attention_weight_max": 31.167293548583984,
      "activations/layer2_attention_weight_min": -30.764019012451172,
      "activations/layer3_attention_weight_max": 85.49512481689453,
      "activations/layer3_attention_weight_min": -88.04521179199219,
      "activations/layer4_attention_weight_max": 130.9194793701172,
      "activations/layer4_attention_weight_min": -131.32037353515625,
      "activations/layer5_attention_weight_max": 87.02706146240234,
      "activations/layer5_attention_weight_min": -87.1068115234375,
      "activations/layer6_attention_weight_max": 54.21043395996094,
      "activations/layer6_attention_weight_min": -58.32696533203125,
      "activations/layer7_attention_weight_max": 81.91803741455078,
      "activations/layer7_attention_weight_min": -82.43983459472656,
      "activations/layer8_attention_weight_max": 39.46563720703125,
      "activations/layer8_attention_weight_min": -41.61415481567383,
      "activations/layer9_attention_weight_max": 37.481422424316406,
      "activations/layer9_attention_weight_min": -32.798065185546875,
      "epoch": 4.22,
      "learning_rate": 0.0001240435606060606,
      "loss": 2.864,
      "step": 72600
    },
    {
      "activations/layer0_attention_weight_max": 16.540485382080078,
      "activations/layer0_attention_weight_min": -12.533559799194336,
      "activations/layer10_attention_weight_max": 35.20720672607422,
      "activations/layer10_attention_weight_min": -29.915225982666016,
      "activations/layer11_attention_weight_max": 33.76588439941406,
      "activations/layer11_attention_weight_min": -27.213279724121094,
      "activations/layer12_attention_weight_max": 26.416696548461914,
      "activations/layer12_attention_weight_min": -27.941307067871094,
      "activations/layer13_attention_weight_max": 45.76680374145508,
      "activations/layer13_attention_weight_min": -34.58343505859375,
      "activations/layer14_attention_weight_max": 68.36150360107422,
      "activations/layer14_attention_weight_min": -49.87614440917969,
      "activations/layer15_attention_weight_max": 42.69470977783203,
      "activations/layer15_attention_weight_min": -29.219329833984375,
      "activations/layer16_attention_weight_max": 45.365577697753906,
      "activations/layer16_attention_weight_min": -29.18444061279297,
      "activations/layer17_attention_weight_max": 40.55550765991211,
      "activations/layer17_attention_weight_min": -27.12399673461914,
      "activations/layer18_attention_weight_max": 37.854270935058594,
      "activations/layer18_attention_weight_min": -23.56988525390625,
      "activations/layer19_attention_weight_max": 41.929256439208984,
      "activations/layer19_attention_weight_min": -30.081716537475586,
      "activations/layer1_attention_weight_max": 17.813392639160156,
      "activations/layer1_attention_weight_min": -16.043737411499023,
      "activations/layer20_attention_weight_max": 34.613643646240234,
      "activations/layer20_attention_weight_min": -23.354135513305664,
      "activations/layer21_attention_weight_max": 41.3276481628418,
      "activations/layer21_attention_weight_min": -20.72928810119629,
      "activations/layer22_attention_weight_max": 50.38138198852539,
      "activations/layer22_attention_weight_min": -28.1129150390625,
      "activations/layer23_attention_weight_max": 29.53083038330078,
      "activations/layer23_attention_weight_min": -18.424863815307617,
      "activations/layer2_attention_weight_max": 31.440526962280273,
      "activations/layer2_attention_weight_min": -30.562061309814453,
      "activations/layer3_attention_weight_max": 87.42250061035156,
      "activations/layer3_attention_weight_min": -89.08089447021484,
      "activations/layer4_attention_weight_max": 134.7895965576172,
      "activations/layer4_attention_weight_min": -128.17459106445312,
      "activations/layer5_attention_weight_max": 86.12588500976562,
      "activations/layer5_attention_weight_min": -83.5624771118164,
      "activations/layer6_attention_weight_max": 58.08414077758789,
      "activations/layer6_attention_weight_min": -56.05974197387695,
      "activations/layer7_attention_weight_max": 86.4213638305664,
      "activations/layer7_attention_weight_min": -78.70346069335938,
      "activations/layer8_attention_weight_max": 40.26052474975586,
      "activations/layer8_attention_weight_min": -36.19733428955078,
      "activations/layer9_attention_weight_max": 35.477657318115234,
      "activations/layer9_attention_weight_min": -34.27227020263672,
      "epoch": 4.22,
      "learning_rate": 0.00012402462121212122,
      "loss": 2.8613,
      "step": 72650
    },
    {
      "activations/layer0_attention_weight_max": 16.173856735229492,
      "activations/layer0_attention_weight_min": -12.736258506774902,
      "activations/layer10_attention_weight_max": 36.792381286621094,
      "activations/layer10_attention_weight_min": -30.733768463134766,
      "activations/layer11_attention_weight_max": 37.68764114379883,
      "activations/layer11_attention_weight_min": -28.091014862060547,
      "activations/layer12_attention_weight_max": 27.223861694335938,
      "activations/layer12_attention_weight_min": -28.05023956298828,
      "activations/layer13_attention_weight_max": 55.884666442871094,
      "activations/layer13_attention_weight_min": -38.72151184082031,
      "activations/layer14_attention_weight_max": 66.45269775390625,
      "activations/layer14_attention_weight_min": -49.945926666259766,
      "activations/layer15_attention_weight_max": 47.615386962890625,
      "activations/layer15_attention_weight_min": -32.40934371948242,
      "activations/layer16_attention_weight_max": 40.96857452392578,
      "activations/layer16_attention_weight_min": -29.72708511352539,
      "activations/layer17_attention_weight_max": 41.835784912109375,
      "activations/layer17_attention_weight_min": -27.61412239074707,
      "activations/layer18_attention_weight_max": 31.320817947387695,
      "activations/layer18_attention_weight_min": -22.1610107421875,
      "activations/layer19_attention_weight_max": 43.90262985229492,
      "activations/layer19_attention_weight_min": -30.150604248046875,
      "activations/layer1_attention_weight_max": 16.759855270385742,
      "activations/layer1_attention_weight_min": -14.941234588623047,
      "activations/layer20_attention_weight_max": 41.36390686035156,
      "activations/layer20_attention_weight_min": -24.451358795166016,
      "activations/layer21_attention_weight_max": 38.55119705200195,
      "activations/layer21_attention_weight_min": -22.295743942260742,
      "activations/layer22_attention_weight_max": 54.626220703125,
      "activations/layer22_attention_weight_min": -26.60181999206543,
      "activations/layer23_attention_weight_max": 32.814361572265625,
      "activations/layer23_attention_weight_min": -18.82666778564453,
      "activations/layer2_attention_weight_max": 31.790115356445312,
      "activations/layer2_attention_weight_min": -32.0516357421875,
      "activations/layer3_attention_weight_max": 87.80585479736328,
      "activations/layer3_attention_weight_min": -88.68265533447266,
      "activations/layer4_attention_weight_max": 136.41439819335938,
      "activations/layer4_attention_weight_min": -128.2340545654297,
      "activations/layer5_attention_weight_max": 93.23262023925781,
      "activations/layer5_attention_weight_min": -86.35260009765625,
      "activations/layer6_attention_weight_max": 59.28309631347656,
      "activations/layer6_attention_weight_min": -59.647186279296875,
      "activations/layer7_attention_weight_max": 78.92111206054688,
      "activations/layer7_attention_weight_min": -80.78834533691406,
      "activations/layer8_attention_weight_max": 40.673065185546875,
      "activations/layer8_attention_weight_min": -37.19087600708008,
      "activations/layer9_attention_weight_max": 38.1627082824707,
      "activations/layer9_attention_weight_min": -35.61790084838867,
      "epoch": 4.22,
      "learning_rate": 0.0001240056818181818,
      "loss": 2.8596,
      "step": 72700
    },
    {
      "activations/layer0_attention_weight_max": 16.747447967529297,
      "activations/layer0_attention_weight_min": -12.737786293029785,
      "activations/layer10_attention_weight_max": 34.12567138671875,
      "activations/layer10_attention_weight_min": -28.203983306884766,
      "activations/layer11_attention_weight_max": 30.492950439453125,
      "activations/layer11_attention_weight_min": -26.65558624267578,
      "activations/layer12_attention_weight_max": 37.52399826049805,
      "activations/layer12_attention_weight_min": -27.209476470947266,
      "activations/layer13_attention_weight_max": 44.911624908447266,
      "activations/layer13_attention_weight_min": -32.896629333496094,
      "activations/layer14_attention_weight_max": 57.52619934082031,
      "activations/layer14_attention_weight_min": -44.16778564453125,
      "activations/layer15_attention_weight_max": 41.5610237121582,
      "activations/layer15_attention_weight_min": -30.737743377685547,
      "activations/layer16_attention_weight_max": 39.369510650634766,
      "activations/layer16_attention_weight_min": -28.08135986328125,
      "activations/layer17_attention_weight_max": 40.44853591918945,
      "activations/layer17_attention_weight_min": -25.31218910217285,
      "activations/layer18_attention_weight_max": 35.15336608886719,
      "activations/layer18_attention_weight_min": -19.901599884033203,
      "activations/layer19_attention_weight_max": 42.77452087402344,
      "activations/layer19_attention_weight_min": -28.300424575805664,
      "activations/layer1_attention_weight_max": 17.14325714111328,
      "activations/layer1_attention_weight_min": -16.155229568481445,
      "activations/layer20_attention_weight_max": 37.36450958251953,
      "activations/layer20_attention_weight_min": -24.069721221923828,
      "activations/layer21_attention_weight_max": 40.222408294677734,
      "activations/layer21_attention_weight_min": -21.955690383911133,
      "activations/layer22_attention_weight_max": 55.35247802734375,
      "activations/layer22_attention_weight_min": -28.127094268798828,
      "activations/layer23_attention_weight_max": 35.78369140625,
      "activations/layer23_attention_weight_min": -19.140209197998047,
      "activations/layer2_attention_weight_max": 33.45677185058594,
      "activations/layer2_attention_weight_min": -31.158004760742188,
      "activations/layer3_attention_weight_max": 88.98946380615234,
      "activations/layer3_attention_weight_min": -91.57793426513672,
      "activations/layer4_attention_weight_max": 132.5629425048828,
      "activations/layer4_attention_weight_min": -130.87506103515625,
      "activations/layer5_attention_weight_max": 89.5577621459961,
      "activations/layer5_attention_weight_min": -84.93130493164062,
      "activations/layer6_attention_weight_max": 57.86846923828125,
      "activations/layer6_attention_weight_min": -58.01704406738281,
      "activations/layer7_attention_weight_max": 76.58173370361328,
      "activations/layer7_attention_weight_min": -73.28387451171875,
      "activations/layer8_attention_weight_max": 39.58182907104492,
      "activations/layer8_attention_weight_min": -38.60853958129883,
      "activations/layer9_attention_weight_max": 36.51496887207031,
      "activations/layer9_attention_weight_min": -40.09440231323242,
      "epoch": 4.23,
      "learning_rate": 0.00012398674242424242,
      "loss": 2.8722,
      "step": 72750
    },
    {
      "activations/layer0_attention_weight_max": 16.503450393676758,
      "activations/layer0_attention_weight_min": -11.895569801330566,
      "activations/layer10_attention_weight_max": 36.4813232421875,
      "activations/layer10_attention_weight_min": -32.72649002075195,
      "activations/layer11_attention_weight_max": 36.11647033691406,
      "activations/layer11_attention_weight_min": -27.83700942993164,
      "activations/layer12_attention_weight_max": 27.098304748535156,
      "activations/layer12_attention_weight_min": -28.144453048706055,
      "activations/layer13_attention_weight_max": 52.47450256347656,
      "activations/layer13_attention_weight_min": -35.26384353637695,
      "activations/layer14_attention_weight_max": 71.30998992919922,
      "activations/layer14_attention_weight_min": -48.97969436645508,
      "activations/layer15_attention_weight_max": 50.881195068359375,
      "activations/layer15_attention_weight_min": -31.59133529663086,
      "activations/layer16_attention_weight_max": 38.084232330322266,
      "activations/layer16_attention_weight_min": -27.52066993713379,
      "activations/layer17_attention_weight_max": 43.37221908569336,
      "activations/layer17_attention_weight_min": -25.019460678100586,
      "activations/layer18_attention_weight_max": 31.308382034301758,
      "activations/layer18_attention_weight_min": -20.541522979736328,
      "activations/layer19_attention_weight_max": 45.442745208740234,
      "activations/layer19_attention_weight_min": -27.84800148010254,
      "activations/layer1_attention_weight_max": 17.685646057128906,
      "activations/layer1_attention_weight_min": -15.994477272033691,
      "activations/layer20_attention_weight_max": 35.91817092895508,
      "activations/layer20_attention_weight_min": -22.940561294555664,
      "activations/layer21_attention_weight_max": 33.27437210083008,
      "activations/layer21_attention_weight_min": -19.451416015625,
      "activations/layer22_attention_weight_max": 55.748653411865234,
      "activations/layer22_attention_weight_min": -26.313323974609375,
      "activations/layer23_attention_weight_max": 33.539913177490234,
      "activations/layer23_attention_weight_min": -19.63849639892578,
      "activations/layer2_attention_weight_max": 30.236419677734375,
      "activations/layer2_attention_weight_min": -30.547958374023438,
      "activations/layer3_attention_weight_max": 86.0949478149414,
      "activations/layer3_attention_weight_min": -87.18047332763672,
      "activations/layer4_attention_weight_max": 133.83470153808594,
      "activations/layer4_attention_weight_min": -131.53807067871094,
      "activations/layer5_attention_weight_max": 87.82820129394531,
      "activations/layer5_attention_weight_min": -84.91078186035156,
      "activations/layer6_attention_weight_max": 63.3675651550293,
      "activations/layer6_attention_weight_min": -57.7222785949707,
      "activations/layer7_attention_weight_max": 79.36190032958984,
      "activations/layer7_attention_weight_min": -77.21389770507812,
      "activations/layer8_attention_weight_max": 39.36589431762695,
      "activations/layer8_attention_weight_min": -39.71583938598633,
      "activations/layer9_attention_weight_max": 37.60460662841797,
      "activations/layer9_attention_weight_min": -34.56187438964844,
      "epoch": 4.23,
      "learning_rate": 0.00012396780303030303,
      "loss": 2.8813,
      "step": 72800
    },
    {
      "activations/layer0_attention_weight_max": 16.532245635986328,
      "activations/layer0_attention_weight_min": -10.986358642578125,
      "activations/layer10_attention_weight_max": 34.610015869140625,
      "activations/layer10_attention_weight_min": -30.129369735717773,
      "activations/layer11_attention_weight_max": 33.12352752685547,
      "activations/layer11_attention_weight_min": -27.986419677734375,
      "activations/layer12_attention_weight_max": 35.257144927978516,
      "activations/layer12_attention_weight_min": -27.742816925048828,
      "activations/layer13_attention_weight_max": 49.11912155151367,
      "activations/layer13_attention_weight_min": -35.50429916381836,
      "activations/layer14_attention_weight_max": 78.55238342285156,
      "activations/layer14_attention_weight_min": -60.354583740234375,
      "activations/layer15_attention_weight_max": 42.75543975830078,
      "activations/layer15_attention_weight_min": -30.361345291137695,
      "activations/layer16_attention_weight_max": 41.93428039550781,
      "activations/layer16_attention_weight_min": -29.090858459472656,
      "activations/layer17_attention_weight_max": 46.78053665161133,
      "activations/layer17_attention_weight_min": -25.773292541503906,
      "activations/layer18_attention_weight_max": 33.85601043701172,
      "activations/layer18_attention_weight_min": -20.20471954345703,
      "activations/layer19_attention_weight_max": 50.53107833862305,
      "activations/layer19_attention_weight_min": -27.235197067260742,
      "activations/layer1_attention_weight_max": 17.614755630493164,
      "activations/layer1_attention_weight_min": -13.397366523742676,
      "activations/layer20_attention_weight_max": 31.97072410583496,
      "activations/layer20_attention_weight_min": -22.217531204223633,
      "activations/layer21_attention_weight_max": 38.966007232666016,
      "activations/layer21_attention_weight_min": -22.500091552734375,
      "activations/layer22_attention_weight_max": 50.95988845825195,
      "activations/layer22_attention_weight_min": -28.125003814697266,
      "activations/layer23_attention_weight_max": 31.638797760009766,
      "activations/layer23_attention_weight_min": -17.720849990844727,
      "activations/layer2_attention_weight_max": 30.098201751708984,
      "activations/layer2_attention_weight_min": -31.063339233398438,
      "activations/layer3_attention_weight_max": 89.49148559570312,
      "activations/layer3_attention_weight_min": -88.6725082397461,
      "activations/layer4_attention_weight_max": 130.60191345214844,
      "activations/layer4_attention_weight_min": -129.16221618652344,
      "activations/layer5_attention_weight_max": 87.71836853027344,
      "activations/layer5_attention_weight_min": -86.25109100341797,
      "activations/layer6_attention_weight_max": 56.14619827270508,
      "activations/layer6_attention_weight_min": -59.31797790527344,
      "activations/layer7_attention_weight_max": 77.42533874511719,
      "activations/layer7_attention_weight_min": -77.9937973022461,
      "activations/layer8_attention_weight_max": 38.17326736450195,
      "activations/layer8_attention_weight_min": -40.15659713745117,
      "activations/layer9_attention_weight_max": 36.31344985961914,
      "activations/layer9_attention_weight_min": -33.66598892211914,
      "epoch": 4.23,
      "learning_rate": 0.00012394886363636362,
      "loss": 2.8525,
      "step": 72850
    },
    {
      "activations/layer0_attention_weight_max": 17.18236541748047,
      "activations/layer0_attention_weight_min": -12.119751930236816,
      "activations/layer10_attention_weight_max": 33.9139518737793,
      "activations/layer10_attention_weight_min": -30.193099975585938,
      "activations/layer11_attention_weight_max": 32.67911148071289,
      "activations/layer11_attention_weight_min": -26.097715377807617,
      "activations/layer12_attention_weight_max": 27.49985694885254,
      "activations/layer12_attention_weight_min": -22.576879501342773,
      "activations/layer13_attention_weight_max": 47.20193099975586,
      "activations/layer13_attention_weight_min": -34.3276481628418,
      "activations/layer14_attention_weight_max": 61.15325164794922,
      "activations/layer14_attention_weight_min": -47.90824890136719,
      "activations/layer15_attention_weight_max": 38.869285583496094,
      "activations/layer15_attention_weight_min": -28.750804901123047,
      "activations/layer16_attention_weight_max": 36.71140670776367,
      "activations/layer16_attention_weight_min": -29.934720993041992,
      "activations/layer17_attention_weight_max": 42.163230895996094,
      "activations/layer17_attention_weight_min": -27.594141006469727,
      "activations/layer18_attention_weight_max": 34.861167907714844,
      "activations/layer18_attention_weight_min": -21.18157958984375,
      "activations/layer19_attention_weight_max": 38.189697265625,
      "activations/layer19_attention_weight_min": -28.269733428955078,
      "activations/layer1_attention_weight_max": 16.713775634765625,
      "activations/layer1_attention_weight_min": -15.20389461517334,
      "activations/layer20_attention_weight_max": 34.2947883605957,
      "activations/layer20_attention_weight_min": -23.142961502075195,
      "activations/layer21_attention_weight_max": 33.49761199951172,
      "activations/layer21_attention_weight_min": -20.6741886138916,
      "activations/layer22_attention_weight_max": 52.0915641784668,
      "activations/layer22_attention_weight_min": -26.82712173461914,
      "activations/layer23_attention_weight_max": 33.562583923339844,
      "activations/layer23_attention_weight_min": -17.459840774536133,
      "activations/layer2_attention_weight_max": 30.31818962097168,
      "activations/layer2_attention_weight_min": -29.992778778076172,
      "activations/layer3_attention_weight_max": 81.35232543945312,
      "activations/layer3_attention_weight_min": -85.8116455078125,
      "activations/layer4_attention_weight_max": 125.66609954833984,
      "activations/layer4_attention_weight_min": -129.5957794189453,
      "activations/layer5_attention_weight_max": 88.44247436523438,
      "activations/layer5_attention_weight_min": -85.32818603515625,
      "activations/layer6_attention_weight_max": 58.09877395629883,
      "activations/layer6_attention_weight_min": -57.25299835205078,
      "activations/layer7_attention_weight_max": 76.52391052246094,
      "activations/layer7_attention_weight_min": -79.54769134521484,
      "activations/layer8_attention_weight_max": 38.26131820678711,
      "activations/layer8_attention_weight_min": -39.678279876708984,
      "activations/layer9_attention_weight_max": 36.05360794067383,
      "activations/layer9_attention_weight_min": -34.62260055541992,
      "epoch": 4.24,
      "learning_rate": 0.00012392992424242424,
      "loss": 2.8612,
      "step": 72900
    },
    {
      "activations/layer0_attention_weight_max": 14.389497756958008,
      "activations/layer0_attention_weight_min": -11.218093872070312,
      "activations/layer10_attention_weight_max": 34.282470703125,
      "activations/layer10_attention_weight_min": -30.166147232055664,
      "activations/layer11_attention_weight_max": 35.27415466308594,
      "activations/layer11_attention_weight_min": -27.46302604675293,
      "activations/layer12_attention_weight_max": 49.492488861083984,
      "activations/layer12_attention_weight_min": -25.421586990356445,
      "activations/layer13_attention_weight_max": 40.851234436035156,
      "activations/layer13_attention_weight_min": -31.85620880126953,
      "activations/layer14_attention_weight_max": 69.2271499633789,
      "activations/layer14_attention_weight_min": -43.01852798461914,
      "activations/layer15_attention_weight_max": 40.731781005859375,
      "activations/layer15_attention_weight_min": -26.885986328125,
      "activations/layer16_attention_weight_max": 43.102622985839844,
      "activations/layer16_attention_weight_min": -26.734683990478516,
      "activations/layer17_attention_weight_max": 39.882938385009766,
      "activations/layer17_attention_weight_min": -23.28875160217285,
      "activations/layer18_attention_weight_max": 28.646995544433594,
      "activations/layer18_attention_weight_min": -19.785655975341797,
      "activations/layer19_attention_weight_max": 48.76948928833008,
      "activations/layer19_attention_weight_min": -28.272058486938477,
      "activations/layer1_attention_weight_max": 17.186717987060547,
      "activations/layer1_attention_weight_min": -13.364827156066895,
      "activations/layer20_attention_weight_max": 40.30455780029297,
      "activations/layer20_attention_weight_min": -22.796037673950195,
      "activations/layer21_attention_weight_max": 32.72072219848633,
      "activations/layer21_attention_weight_min": -21.064292907714844,
      "activations/layer22_attention_weight_max": 49.066349029541016,
      "activations/layer22_attention_weight_min": -29.10945701599121,
      "activations/layer23_attention_weight_max": 31.844322204589844,
      "activations/layer23_attention_weight_min": -20.156465530395508,
      "activations/layer2_attention_weight_max": 31.31978988647461,
      "activations/layer2_attention_weight_min": -31.32013511657715,
      "activations/layer3_attention_weight_max": 83.31412506103516,
      "activations/layer3_attention_weight_min": -86.42621612548828,
      "activations/layer4_attention_weight_max": 130.19837951660156,
      "activations/layer4_attention_weight_min": -128.59873962402344,
      "activations/layer5_attention_weight_max": 87.96662902832031,
      "activations/layer5_attention_weight_min": -83.95616149902344,
      "activations/layer6_attention_weight_max": 58.520477294921875,
      "activations/layer6_attention_weight_min": -55.03123474121094,
      "activations/layer7_attention_weight_max": 79.70855712890625,
      "activations/layer7_attention_weight_min": -73.72124481201172,
      "activations/layer8_attention_weight_max": 38.77437973022461,
      "activations/layer8_attention_weight_min": -35.92214584350586,
      "activations/layer9_attention_weight_max": 38.45852279663086,
      "activations/layer9_attention_weight_min": -34.50103759765625,
      "epoch": 4.24,
      "learning_rate": 0.00012391098484848485,
      "loss": 2.8791,
      "step": 72950
    },
    {
      "activations/layer0_attention_weight_max": 15.624545097351074,
      "activations/layer0_attention_weight_min": -12.593405723571777,
      "activations/layer10_attention_weight_max": 37.36390686035156,
      "activations/layer10_attention_weight_min": -29.955814361572266,
      "activations/layer11_attention_weight_max": 35.37441635131836,
      "activations/layer11_attention_weight_min": -26.594640731811523,
      "activations/layer12_attention_weight_max": 28.415525436401367,
      "activations/layer12_attention_weight_min": -27.59320640563965,
      "activations/layer13_attention_weight_max": 48.38917541503906,
      "activations/layer13_attention_weight_min": -33.9827880859375,
      "activations/layer14_attention_weight_max": 71.35778045654297,
      "activations/layer14_attention_weight_min": -48.09296417236328,
      "activations/layer15_attention_weight_max": 40.139530181884766,
      "activations/layer15_attention_weight_min": -29.99363899230957,
      "activations/layer16_attention_weight_max": 37.8004035949707,
      "activations/layer16_attention_weight_min": -28.306476593017578,
      "activations/layer17_attention_weight_max": 38.6583251953125,
      "activations/layer17_attention_weight_min": -23.67021369934082,
      "activations/layer18_attention_weight_max": 36.31818771362305,
      "activations/layer18_attention_weight_min": -19.239721298217773,
      "activations/layer19_attention_weight_max": 38.41260528564453,
      "activations/layer19_attention_weight_min": -27.32578468322754,
      "activations/layer1_attention_weight_max": 18.06621742248535,
      "activations/layer1_attention_weight_min": -14.942317962646484,
      "activations/layer20_attention_weight_max": 33.30318069458008,
      "activations/layer20_attention_weight_min": -22.22486686706543,
      "activations/layer21_attention_weight_max": 38.677101135253906,
      "activations/layer21_attention_weight_min": -18.629669189453125,
      "activations/layer22_attention_weight_max": 47.00687789916992,
      "activations/layer22_attention_weight_min": -26.762062072753906,
      "activations/layer23_attention_weight_max": 29.080732345581055,
      "activations/layer23_attention_weight_min": -18.677875518798828,
      "activations/layer2_attention_weight_max": 30.58785629272461,
      "activations/layer2_attention_weight_min": -30.974403381347656,
      "activations/layer3_attention_weight_max": 84.3130111694336,
      "activations/layer3_attention_weight_min": -89.34290313720703,
      "activations/layer4_attention_weight_max": 125.4180908203125,
      "activations/layer4_attention_weight_min": -126.21754455566406,
      "activations/layer5_attention_weight_max": 85.12334442138672,
      "activations/layer5_attention_weight_min": -85.22358703613281,
      "activations/layer6_attention_weight_max": 59.45183181762695,
      "activations/layer6_attention_weight_min": -59.13042449951172,
      "activations/layer7_attention_weight_max": 80.62682342529297,
      "activations/layer7_attention_weight_min": -80.19844818115234,
      "activations/layer8_attention_weight_max": 38.902225494384766,
      "activations/layer8_attention_weight_min": -42.09620666503906,
      "activations/layer9_attention_weight_max": 41.62891387939453,
      "activations/layer9_attention_weight_min": -36.942466735839844,
      "epoch": 4.24,
      "learning_rate": 0.00012389204545454544,
      "loss": 2.8602,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_loss": 2.8203125,
      "eval_runtime": 8.463,
      "eval_samples_per_second": 507.385,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_openwebtext_loss": 2.8203125,
      "eval_openwebtext_ppl": 16.782094257242417,
      "eval_openwebtext_runtime": 8.463,
      "eval_openwebtext_samples_per_second": 507.385,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_wikitext_loss": 3.0546875,
      "eval_wikitext_ppl": 21.214554942517648,
      "eval_wikitext_runtime": 1.9858,
      "eval_wikitext_samples_per_second": 229.634,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_lambada_loss": 2.802734375,
      "eval_lambada_ppl": 16.489674134905655,
      "eval_lambada_runtime": 9.5503,
      "eval_lambada_samples_per_second": 509.829,
      "step": 73000
    },
    {
      "activations/layer0_attention_weight_max": 15.824127197265625,
      "activations/layer0_attention_weight_min": -11.35360336303711,
      "activations/layer10_attention_weight_max": 33.66288375854492,
      "activations/layer10_attention_weight_min": -29.874797821044922,
      "activations/layer11_attention_weight_max": 35.84413146972656,
      "activations/layer11_attention_weight_min": -27.311412811279297,
      "activations/layer12_attention_weight_max": 26.69788360595703,
      "activations/layer12_attention_weight_min": -26.331968307495117,
      "activations/layer13_attention_weight_max": 47.038700103759766,
      "activations/layer13_attention_weight_min": -33.14677047729492,
      "activations/layer14_attention_weight_max": 67.28189086914062,
      "activations/layer14_attention_weight_min": -47.97314453125,
      "activations/layer15_attention_weight_max": 40.98163986206055,
      "activations/layer15_attention_weight_min": -28.400733947753906,
      "activations/layer16_attention_weight_max": 38.999061584472656,
      "activations/layer16_attention_weight_min": -28.71402359008789,
      "activations/layer17_attention_weight_max": 43.07881164550781,
      "activations/layer17_attention_weight_min": -24.41568946838379,
      "activations/layer18_attention_weight_max": 37.288055419921875,
      "activations/layer18_attention_weight_min": -19.392669677734375,
      "activations/layer19_attention_weight_max": 43.67037582397461,
      "activations/layer19_attention_weight_min": -28.82893943786621,
      "activations/layer1_attention_weight_max": 17.85931396484375,
      "activations/layer1_attention_weight_min": -15.298460006713867,
      "activations/layer20_attention_weight_max": 33.2137565612793,
      "activations/layer20_attention_weight_min": -23.907320022583008,
      "activations/layer21_attention_weight_max": 32.59601974487305,
      "activations/layer21_attention_weight_min": -22.598379135131836,
      "activations/layer22_attention_weight_max": 54.54324722290039,
      "activations/layer22_attention_weight_min": -28.138933181762695,
      "activations/layer23_attention_weight_max": 32.411720275878906,
      "activations/layer23_attention_weight_min": -21.868986129760742,
      "activations/layer2_attention_weight_max": 32.09345245361328,
      "activations/layer2_attention_weight_min": -31.358051300048828,
      "activations/layer3_attention_weight_max": 85.88744354248047,
      "activations/layer3_attention_weight_min": -88.91083526611328,
      "activations/layer4_attention_weight_max": 131.81639099121094,
      "activations/layer4_attention_weight_min": -130.49240112304688,
      "activations/layer5_attention_weight_max": 86.70671844482422,
      "activations/layer5_attention_weight_min": -81.42105102539062,
      "activations/layer6_attention_weight_max": 56.29145812988281,
      "activations/layer6_attention_weight_min": -56.093223571777344,
      "activations/layer7_attention_weight_max": 77.36902618408203,
      "activations/layer7_attention_weight_min": -82.84312438964844,
      "activations/layer8_attention_weight_max": 38.70256805419922,
      "activations/layer8_attention_weight_min": -37.14978790283203,
      "activations/layer9_attention_weight_max": 37.72381591796875,
      "activations/layer9_attention_weight_min": -37.68717575073242,
      "epoch": 4.24,
      "learning_rate": 0.00012387310606060606,
      "loss": 2.8683,
      "step": 73050
    },
    {
      "activations/layer0_attention_weight_max": 17.377403259277344,
      "activations/layer0_attention_weight_min": -12.059922218322754,
      "activations/layer10_attention_weight_max": 35.7206916809082,
      "activations/layer10_attention_weight_min": -28.484603881835938,
      "activations/layer11_attention_weight_max": 33.090980529785156,
      "activations/layer11_attention_weight_min": -25.878278732299805,
      "activations/layer12_attention_weight_max": 39.91316604614258,
      "activations/layer12_attention_weight_min": -26.617204666137695,
      "activations/layer13_attention_weight_max": 47.302589416503906,
      "activations/layer13_attention_weight_min": -36.106224060058594,
      "activations/layer14_attention_weight_max": 56.50235366821289,
      "activations/layer14_attention_weight_min": -44.47360610961914,
      "activations/layer15_attention_weight_max": 52.51639175415039,
      "activations/layer15_attention_weight_min": -34.04121398925781,
      "activations/layer16_attention_weight_max": 36.730560302734375,
      "activations/layer16_attention_weight_min": -26.769765853881836,
      "activations/layer17_attention_weight_max": 39.17192840576172,
      "activations/layer17_attention_weight_min": -24.88106346130371,
      "activations/layer18_attention_weight_max": 32.6408805847168,
      "activations/layer18_attention_weight_min": -19.325742721557617,
      "activations/layer19_attention_weight_max": 43.61233139038086,
      "activations/layer19_attention_weight_min": -27.593921661376953,
      "activations/layer1_attention_weight_max": 17.44846534729004,
      "activations/layer1_attention_weight_min": -13.597831726074219,
      "activations/layer20_attention_weight_max": 32.868080139160156,
      "activations/layer20_attention_weight_min": -23.040626525878906,
      "activations/layer21_attention_weight_max": 30.872526168823242,
      "activations/layer21_attention_weight_min": -19.513216018676758,
      "activations/layer22_attention_weight_max": 48.713951110839844,
      "activations/layer22_attention_weight_min": -26.57431411743164,
      "activations/layer23_attention_weight_max": 31.672409057617188,
      "activations/layer23_attention_weight_min": -18.095169067382812,
      "activations/layer2_attention_weight_max": 31.931846618652344,
      "activations/layer2_attention_weight_min": -31.16124725341797,
      "activations/layer3_attention_weight_max": 87.01580810546875,
      "activations/layer3_attention_weight_min": -90.2374038696289,
      "activations/layer4_attention_weight_max": 129.35496520996094,
      "activations/layer4_attention_weight_min": -130.6704559326172,
      "activations/layer5_attention_weight_max": 86.4622802734375,
      "activations/layer5_attention_weight_min": -84.12238311767578,
      "activations/layer6_attention_weight_max": 57.202362060546875,
      "activations/layer6_attention_weight_min": -56.066253662109375,
      "activations/layer7_attention_weight_max": 78.9987564086914,
      "activations/layer7_attention_weight_min": -79.89866638183594,
      "activations/layer8_attention_weight_max": 37.23440170288086,
      "activations/layer8_attention_weight_min": -38.3567008972168,
      "activations/layer9_attention_weight_max": 35.37201690673828,
      "activations/layer9_attention_weight_min": -32.29061508178711,
      "epoch": 4.25,
      "learning_rate": 0.00012385416666666664,
      "loss": 2.8737,
      "step": 73100
    },
    {
      "activations/layer0_attention_weight_max": 16.581010818481445,
      "activations/layer0_attention_weight_min": -11.415992736816406,
      "activations/layer10_attention_weight_max": 37.93871307373047,
      "activations/layer10_attention_weight_min": -30.079864501953125,
      "activations/layer11_attention_weight_max": 33.889957427978516,
      "activations/layer11_attention_weight_min": -26.3660945892334,
      "activations/layer12_attention_weight_max": 29.995420455932617,
      "activations/layer12_attention_weight_min": -24.102691650390625,
      "activations/layer13_attention_weight_max": 55.870304107666016,
      "activations/layer13_attention_weight_min": -34.26984786987305,
      "activations/layer14_attention_weight_max": 66.23081970214844,
      "activations/layer14_attention_weight_min": -47.9874382019043,
      "activations/layer15_attention_weight_max": 40.15594482421875,
      "activations/layer15_attention_weight_min": -28.71516227722168,
      "activations/layer16_attention_weight_max": 39.060569763183594,
      "activations/layer16_attention_weight_min": -27.277097702026367,
      "activations/layer17_attention_weight_max": 39.66512680053711,
      "activations/layer17_attention_weight_min": -23.50113296508789,
      "activations/layer18_attention_weight_max": 32.321285247802734,
      "activations/layer18_attention_weight_min": -19.590744018554688,
      "activations/layer19_attention_weight_max": 39.48757553100586,
      "activations/layer19_attention_weight_min": -28.134130477905273,
      "activations/layer1_attention_weight_max": 17.2499942779541,
      "activations/layer1_attention_weight_min": -15.858443260192871,
      "activations/layer20_attention_weight_max": 33.54930114746094,
      "activations/layer20_attention_weight_min": -22.51926040649414,
      "activations/layer21_attention_weight_max": 31.820512771606445,
      "activations/layer21_attention_weight_min": -21.251779556274414,
      "activations/layer22_attention_weight_max": 47.81422424316406,
      "activations/layer22_attention_weight_min": -27.87934112548828,
      "activations/layer23_attention_weight_max": 29.98354721069336,
      "activations/layer23_attention_weight_min": -19.587034225463867,
      "activations/layer2_attention_weight_max": 31.032760620117188,
      "activations/layer2_attention_weight_min": -30.838407516479492,
      "activations/layer3_attention_weight_max": 83.25601959228516,
      "activations/layer3_attention_weight_min": -88.60493469238281,
      "activations/layer4_attention_weight_max": 125.10604095458984,
      "activations/layer4_attention_weight_min": -126.79415130615234,
      "activations/layer5_attention_weight_max": 87.54011535644531,
      "activations/layer5_attention_weight_min": -79.7142562866211,
      "activations/layer6_attention_weight_max": 55.49382400512695,
      "activations/layer6_attention_weight_min": -51.57387924194336,
      "activations/layer7_attention_weight_max": 77.90421295166016,
      "activations/layer7_attention_weight_min": -76.09031677246094,
      "activations/layer8_attention_weight_max": 36.69106674194336,
      "activations/layer8_attention_weight_min": -37.751014709472656,
      "activations/layer9_attention_weight_max": 39.3380126953125,
      "activations/layer9_attention_weight_min": -31.310102462768555,
      "epoch": 4.25,
      "learning_rate": 0.00012383522727272726,
      "loss": 2.8474,
      "step": 73150
    },
    {
      "activations/layer0_attention_weight_max": 15.914532661437988,
      "activations/layer0_attention_weight_min": -11.846535682678223,
      "activations/layer10_attention_weight_max": 35.285606384277344,
      "activations/layer10_attention_weight_min": -30.184749603271484,
      "activations/layer11_attention_weight_max": 35.006683349609375,
      "activations/layer11_attention_weight_min": -27.532794952392578,
      "activations/layer12_attention_weight_max": 31.778322219848633,
      "activations/layer12_attention_weight_min": -27.174936294555664,
      "activations/layer13_attention_weight_max": 62.10858917236328,
      "activations/layer13_attention_weight_min": -43.27678680419922,
      "activations/layer14_attention_weight_max": 66.95278930664062,
      "activations/layer14_attention_weight_min": -48.081390380859375,
      "activations/layer15_attention_weight_max": 39.9898567199707,
      "activations/layer15_attention_weight_min": -29.057039260864258,
      "activations/layer16_attention_weight_max": 41.155662536621094,
      "activations/layer16_attention_weight_min": -28.308765411376953,
      "activations/layer17_attention_weight_max": 41.23018264770508,
      "activations/layer17_attention_weight_min": -25.446203231811523,
      "activations/layer18_attention_weight_max": 35.162261962890625,
      "activations/layer18_attention_weight_min": -19.4341983795166,
      "activations/layer19_attention_weight_max": 48.76200866699219,
      "activations/layer19_attention_weight_min": -27.906347274780273,
      "activations/layer1_attention_weight_max": 17.662914276123047,
      "activations/layer1_attention_weight_min": -14.782663345336914,
      "activations/layer20_attention_weight_max": 34.94337463378906,
      "activations/layer20_attention_weight_min": -25.31456756591797,
      "activations/layer21_attention_weight_max": 33.7430534362793,
      "activations/layer21_attention_weight_min": -20.732839584350586,
      "activations/layer22_attention_weight_max": 48.089012145996094,
      "activations/layer22_attention_weight_min": -31.765424728393555,
      "activations/layer23_attention_weight_max": 29.111427307128906,
      "activations/layer23_attention_weight_min": -22.03127098083496,
      "activations/layer2_attention_weight_max": 31.11251449584961,
      "activations/layer2_attention_weight_min": -31.271780014038086,
      "activations/layer3_attention_weight_max": 85.80606079101562,
      "activations/layer3_attention_weight_min": -86.7401351928711,
      "activations/layer4_attention_weight_max": 138.97312927246094,
      "activations/layer4_attention_weight_min": -127.58943939208984,
      "activations/layer5_attention_weight_max": 91.20000457763672,
      "activations/layer5_attention_weight_min": -80.6905288696289,
      "activations/layer6_attention_weight_max": 57.59827423095703,
      "activations/layer6_attention_weight_min": -57.854339599609375,
      "activations/layer7_attention_weight_max": 79.74166870117188,
      "activations/layer7_attention_weight_min": -79.67069244384766,
      "activations/layer8_attention_weight_max": 40.97523498535156,
      "activations/layer8_attention_weight_min": -40.67490768432617,
      "activations/layer9_attention_weight_max": 36.95357131958008,
      "activations/layer9_attention_weight_min": -33.15195846557617,
      "epoch": 4.25,
      "learning_rate": 0.00012381628787878787,
      "loss": 2.8618,
      "step": 73200
    },
    {
      "activations/layer0_attention_weight_max": 16.630647659301758,
      "activations/layer0_attention_weight_min": -12.78023910522461,
      "activations/layer10_attention_weight_max": 33.4320068359375,
      "activations/layer10_attention_weight_min": -28.468740463256836,
      "activations/layer11_attention_weight_max": 33.13490676879883,
      "activations/layer11_attention_weight_min": -28.495513916015625,
      "activations/layer12_attention_weight_max": 34.04494857788086,
      "activations/layer12_attention_weight_min": -26.37382698059082,
      "activations/layer13_attention_weight_max": 43.84947204589844,
      "activations/layer13_attention_weight_min": -30.990419387817383,
      "activations/layer14_attention_weight_max": 68.3774642944336,
      "activations/layer14_attention_weight_min": -48.303794860839844,
      "activations/layer15_attention_weight_max": 43.24968338012695,
      "activations/layer15_attention_weight_min": -27.47304344177246,
      "activations/layer16_attention_weight_max": 39.894630432128906,
      "activations/layer16_attention_weight_min": -30.57359504699707,
      "activations/layer17_attention_weight_max": 43.9808464050293,
      "activations/layer17_attention_weight_min": -24.70123863220215,
      "activations/layer18_attention_weight_max": 36.550846099853516,
      "activations/layer18_attention_weight_min": -20.025548934936523,
      "activations/layer19_attention_weight_max": 44.490821838378906,
      "activations/layer19_attention_weight_min": -29.648435592651367,
      "activations/layer1_attention_weight_max": 17.922590255737305,
      "activations/layer1_attention_weight_min": -14.048148155212402,
      "activations/layer20_attention_weight_max": 37.82272720336914,
      "activations/layer20_attention_weight_min": -23.766244888305664,
      "activations/layer21_attention_weight_max": 35.93779754638672,
      "activations/layer21_attention_weight_min": -21.955638885498047,
      "activations/layer22_attention_weight_max": 47.87910842895508,
      "activations/layer22_attention_weight_min": -27.794872283935547,
      "activations/layer23_attention_weight_max": 32.375,
      "activations/layer23_attention_weight_min": -19.871187210083008,
      "activations/layer2_attention_weight_max": 30.262882232666016,
      "activations/layer2_attention_weight_min": -29.819578170776367,
      "activations/layer3_attention_weight_max": 86.8135986328125,
      "activations/layer3_attention_weight_min": -88.53377532958984,
      "activations/layer4_attention_weight_max": 130.45045471191406,
      "activations/layer4_attention_weight_min": -127.76202392578125,
      "activations/layer5_attention_weight_max": 87.55473327636719,
      "activations/layer5_attention_weight_min": -83.22734832763672,
      "activations/layer6_attention_weight_max": 57.16019821166992,
      "activations/layer6_attention_weight_min": -54.389583587646484,
      "activations/layer7_attention_weight_max": 82.11454772949219,
      "activations/layer7_attention_weight_min": -80.94635009765625,
      "activations/layer8_attention_weight_max": 39.819580078125,
      "activations/layer8_attention_weight_min": -41.419342041015625,
      "activations/layer9_attention_weight_max": 36.07380294799805,
      "activations/layer9_attention_weight_min": -33.52362823486328,
      "epoch": 4.26,
      "learning_rate": 0.00012379734848484846,
      "loss": 2.8733,
      "step": 73250
    },
    {
      "activations/layer0_attention_weight_max": 16.335208892822266,
      "activations/layer0_attention_weight_min": -11.124499320983887,
      "activations/layer10_attention_weight_max": 35.71516799926758,
      "activations/layer10_attention_weight_min": -32.175418853759766,
      "activations/layer11_attention_weight_max": 33.41194152832031,
      "activations/layer11_attention_weight_min": -27.36922836303711,
      "activations/layer12_attention_weight_max": 28.00930404663086,
      "activations/layer12_attention_weight_min": -29.18767738342285,
      "activations/layer13_attention_weight_max": 42.77696990966797,
      "activations/layer13_attention_weight_min": -34.860347747802734,
      "activations/layer14_attention_weight_max": 64.61541748046875,
      "activations/layer14_attention_weight_min": -48.6596565246582,
      "activations/layer15_attention_weight_max": 38.99265670776367,
      "activations/layer15_attention_weight_min": -26.84332847595215,
      "activations/layer16_attention_weight_max": 37.64414596557617,
      "activations/layer16_attention_weight_min": -28.12933921813965,
      "activations/layer17_attention_weight_max": 35.8142204284668,
      "activations/layer17_attention_weight_min": -24.88266944885254,
      "activations/layer18_attention_weight_max": 31.35491371154785,
      "activations/layer18_attention_weight_min": -18.594858169555664,
      "activations/layer19_attention_weight_max": 35.8084716796875,
      "activations/layer19_attention_weight_min": -30.484012603759766,
      "activations/layer1_attention_weight_max": 16.950607299804688,
      "activations/layer1_attention_weight_min": -17.253511428833008,
      "activations/layer20_attention_weight_max": 28.484386444091797,
      "activations/layer20_attention_weight_min": -22.923185348510742,
      "activations/layer21_attention_weight_max": 30.44454002380371,
      "activations/layer21_attention_weight_min": -21.83345603942871,
      "activations/layer22_attention_weight_max": 45.65447998046875,
      "activations/layer22_attention_weight_min": -31.4771728515625,
      "activations/layer23_attention_weight_max": 28.790388107299805,
      "activations/layer23_attention_weight_min": -20.70796775817871,
      "activations/layer2_attention_weight_max": 30.53565216064453,
      "activations/layer2_attention_weight_min": -31.827970504760742,
      "activations/layer3_attention_weight_max": 86.31483459472656,
      "activations/layer3_attention_weight_min": -87.99065399169922,
      "activations/layer4_attention_weight_max": 129.43272399902344,
      "activations/layer4_attention_weight_min": -126.64007568359375,
      "activations/layer5_attention_weight_max": 90.64201354980469,
      "activations/layer5_attention_weight_min": -82.04483795166016,
      "activations/layer6_attention_weight_max": 54.63133239746094,
      "activations/layer6_attention_weight_min": -56.620330810546875,
      "activations/layer7_attention_weight_max": 75.7592544555664,
      "activations/layer7_attention_weight_min": -75.51592254638672,
      "activations/layer8_attention_weight_max": 39.62340545654297,
      "activations/layer8_attention_weight_min": -40.38865661621094,
      "activations/layer9_attention_weight_max": 37.83317565917969,
      "activations/layer9_attention_weight_min": -35.469139099121094,
      "epoch": 4.26,
      "learning_rate": 0.00012377840909090908,
      "loss": 2.8805,
      "step": 73300
    },
    {
      "activations/layer0_attention_weight_max": 16.7397403717041,
      "activations/layer0_attention_weight_min": -11.452795028686523,
      "activations/layer10_attention_weight_max": 31.709732055664062,
      "activations/layer10_attention_weight_min": -28.577856063842773,
      "activations/layer11_attention_weight_max": 35.81559371948242,
      "activations/layer11_attention_weight_min": -29.23731803894043,
      "activations/layer12_attention_weight_max": 30.331724166870117,
      "activations/layer12_attention_weight_min": -27.05813980102539,
      "activations/layer13_attention_weight_max": 45.184146881103516,
      "activations/layer13_attention_weight_min": -33.049537658691406,
      "activations/layer14_attention_weight_max": 63.85170364379883,
      "activations/layer14_attention_weight_min": -45.38425827026367,
      "activations/layer15_attention_weight_max": 43.93769454956055,
      "activations/layer15_attention_weight_min": -28.24534797668457,
      "activations/layer16_attention_weight_max": 41.34989547729492,
      "activations/layer16_attention_weight_min": -28.148487091064453,
      "activations/layer17_attention_weight_max": 42.381553649902344,
      "activations/layer17_attention_weight_min": -25.38749122619629,
      "activations/layer18_attention_weight_max": 35.81688690185547,
      "activations/layer18_attention_weight_min": -19.540132522583008,
      "activations/layer19_attention_weight_max": 39.30023956298828,
      "activations/layer19_attention_weight_min": -30.140172958374023,
      "activations/layer1_attention_weight_max": 18.72243309020996,
      "activations/layer1_attention_weight_min": -16.515859603881836,
      "activations/layer20_attention_weight_max": 36.69808578491211,
      "activations/layer20_attention_weight_min": -24.721261978149414,
      "activations/layer21_attention_weight_max": 38.229217529296875,
      "activations/layer21_attention_weight_min": -21.421615600585938,
      "activations/layer22_attention_weight_max": 52.93905258178711,
      "activations/layer22_attention_weight_min": -28.346290588378906,
      "activations/layer23_attention_weight_max": 33.63957977294922,
      "activations/layer23_attention_weight_min": -18.22484016418457,
      "activations/layer2_attention_weight_max": 31.76849365234375,
      "activations/layer2_attention_weight_min": -30.00798797607422,
      "activations/layer3_attention_weight_max": 86.26610565185547,
      "activations/layer3_attention_weight_min": -88.49217224121094,
      "activations/layer4_attention_weight_max": 131.5308837890625,
      "activations/layer4_attention_weight_min": -122.86808013916016,
      "activations/layer5_attention_weight_max": 86.02009582519531,
      "activations/layer5_attention_weight_min": -80.3793716430664,
      "activations/layer6_attention_weight_max": 55.53138732910156,
      "activations/layer6_attention_weight_min": -55.2325553894043,
      "activations/layer7_attention_weight_max": 81.21417236328125,
      "activations/layer7_attention_weight_min": -76.90837097167969,
      "activations/layer8_attention_weight_max": 40.23445510864258,
      "activations/layer8_attention_weight_min": -36.42149353027344,
      "activations/layer9_attention_weight_max": 40.57246398925781,
      "activations/layer9_attention_weight_min": -40.93926239013672,
      "epoch": 4.26,
      "learning_rate": 0.0001237594696969697,
      "loss": 2.8843,
      "step": 73350
    },
    {
      "activations/layer0_attention_weight_max": 16.934999465942383,
      "activations/layer0_attention_weight_min": -12.127789497375488,
      "activations/layer10_attention_weight_max": 33.47483825683594,
      "activations/layer10_attention_weight_min": -30.579471588134766,
      "activations/layer11_attention_weight_max": 31.590171813964844,
      "activations/layer11_attention_weight_min": -27.499134063720703,
      "activations/layer12_attention_weight_max": 27.454545974731445,
      "activations/layer12_attention_weight_min": -26.291187286376953,
      "activations/layer13_attention_weight_max": 52.308589935302734,
      "activations/layer13_attention_weight_min": -34.1788330078125,
      "activations/layer14_attention_weight_max": 58.464969635009766,
      "activations/layer14_attention_weight_min": -44.793190002441406,
      "activations/layer15_attention_weight_max": 44.53422546386719,
      "activations/layer15_attention_weight_min": -30.34792137145996,
      "activations/layer16_attention_weight_max": 39.243350982666016,
      "activations/layer16_attention_weight_min": -29.31908416748047,
      "activations/layer17_attention_weight_max": 48.87553787231445,
      "activations/layer17_attention_weight_min": -24.818635940551758,
      "activations/layer18_attention_weight_max": 29.467016220092773,
      "activations/layer18_attention_weight_min": -18.543182373046875,
      "activations/layer19_attention_weight_max": 42.611873626708984,
      "activations/layer19_attention_weight_min": -28.64472770690918,
      "activations/layer1_attention_weight_max": 17.357784271240234,
      "activations/layer1_attention_weight_min": -14.540526390075684,
      "activations/layer20_attention_weight_max": 32.96696472167969,
      "activations/layer20_attention_weight_min": -24.199810028076172,
      "activations/layer21_attention_weight_max": 31.087448120117188,
      "activations/layer21_attention_weight_min": -21.046234130859375,
      "activations/layer22_attention_weight_max": 50.37143325805664,
      "activations/layer22_attention_weight_min": -27.511411666870117,
      "activations/layer23_attention_weight_max": 30.800941467285156,
      "activations/layer23_attention_weight_min": -18.230484008789062,
      "activations/layer2_attention_weight_max": 33.77423095703125,
      "activations/layer2_attention_weight_min": -31.803394317626953,
      "activations/layer3_attention_weight_max": 89.03135681152344,
      "activations/layer3_attention_weight_min": -92.40767669677734,
      "activations/layer4_attention_weight_max": 137.14462280273438,
      "activations/layer4_attention_weight_min": -136.83419799804688,
      "activations/layer5_attention_weight_max": 91.34147644042969,
      "activations/layer5_attention_weight_min": -88.208251953125,
      "activations/layer6_attention_weight_max": 58.60777282714844,
      "activations/layer6_attention_weight_min": -61.872215270996094,
      "activations/layer7_attention_weight_max": 78.24433898925781,
      "activations/layer7_attention_weight_min": -81.25196075439453,
      "activations/layer8_attention_weight_max": 39.1373291015625,
      "activations/layer8_attention_weight_min": -38.74159622192383,
      "activations/layer9_attention_weight_max": 35.8338737487793,
      "activations/layer9_attention_weight_min": -34.22695541381836,
      "epoch": 4.26,
      "learning_rate": 0.00012374053030303028,
      "loss": 2.8772,
      "step": 73400
    },
    {
      "activations/layer0_attention_weight_max": 16.014162063598633,
      "activations/layer0_attention_weight_min": -11.992140769958496,
      "activations/layer10_attention_weight_max": 36.423492431640625,
      "activations/layer10_attention_weight_min": -30.224369049072266,
      "activations/layer11_attention_weight_max": 32.11449432373047,
      "activations/layer11_attention_weight_min": -28.865859985351562,
      "activations/layer12_attention_weight_max": 28.49784278869629,
      "activations/layer12_attention_weight_min": -29.479637145996094,
      "activations/layer13_attention_weight_max": 40.84176254272461,
      "activations/layer13_attention_weight_min": -32.234046936035156,
      "activations/layer14_attention_weight_max": 60.412540435791016,
      "activations/layer14_attention_weight_min": -49.2099723815918,
      "activations/layer15_attention_weight_max": 40.46748352050781,
      "activations/layer15_attention_weight_min": -28.253694534301758,
      "activations/layer16_attention_weight_max": 38.480438232421875,
      "activations/layer16_attention_weight_min": -27.715559005737305,
      "activations/layer17_attention_weight_max": 39.711307525634766,
      "activations/layer17_attention_weight_min": -23.763227462768555,
      "activations/layer18_attention_weight_max": 27.53126335144043,
      "activations/layer18_attention_weight_min": -20.37578010559082,
      "activations/layer19_attention_weight_max": 39.384891510009766,
      "activations/layer19_attention_weight_min": -27.513486862182617,
      "activations/layer1_attention_weight_max": 17.544586181640625,
      "activations/layer1_attention_weight_min": -13.759392738342285,
      "activations/layer20_attention_weight_max": 31.337303161621094,
      "activations/layer20_attention_weight_min": -23.04768943786621,
      "activations/layer21_attention_weight_max": 28.18064308166504,
      "activations/layer21_attention_weight_min": -21.15464210510254,
      "activations/layer22_attention_weight_max": 51.00558853149414,
      "activations/layer22_attention_weight_min": -28.37238883972168,
      "activations/layer23_attention_weight_max": 29.31049346923828,
      "activations/layer23_attention_weight_min": -19.048690795898438,
      "activations/layer2_attention_weight_max": 32.97754669189453,
      "activations/layer2_attention_weight_min": -32.645896911621094,
      "activations/layer3_attention_weight_max": 89.09310913085938,
      "activations/layer3_attention_weight_min": -89.29727935791016,
      "activations/layer4_attention_weight_max": 132.95863342285156,
      "activations/layer4_attention_weight_min": -130.71292114257812,
      "activations/layer5_attention_weight_max": 90.16731262207031,
      "activations/layer5_attention_weight_min": -87.93999481201172,
      "activations/layer6_attention_weight_max": 57.31951904296875,
      "activations/layer6_attention_weight_min": -60.4298095703125,
      "activations/layer7_attention_weight_max": 81.09957885742188,
      "activations/layer7_attention_weight_min": -77.0808334350586,
      "activations/layer8_attention_weight_max": 39.62770080566406,
      "activations/layer8_attention_weight_min": -37.944034576416016,
      "activations/layer9_attention_weight_max": 36.87969970703125,
      "activations/layer9_attention_weight_min": -32.65785598754883,
      "epoch": 4.27,
      "learning_rate": 0.0001237215909090909,
      "loss": 2.8724,
      "step": 73450
    },
    {
      "activations/layer0_attention_weight_max": 14.062052726745605,
      "activations/layer0_attention_weight_min": -11.998839378356934,
      "activations/layer10_attention_weight_max": 30.826705932617188,
      "activations/layer10_attention_weight_min": -28.80055809020996,
      "activations/layer11_attention_weight_max": 28.892051696777344,
      "activations/layer11_attention_weight_min": -27.021251678466797,
      "activations/layer12_attention_weight_max": 25.304832458496094,
      "activations/layer12_attention_weight_min": -28.228870391845703,
      "activations/layer13_attention_weight_max": 41.24806594848633,
      "activations/layer13_attention_weight_min": -35.035526275634766,
      "activations/layer14_attention_weight_max": 62.65236282348633,
      "activations/layer14_attention_weight_min": -49.1853141784668,
      "activations/layer15_attention_weight_max": 44.132598876953125,
      "activations/layer15_attention_weight_min": -29.449867248535156,
      "activations/layer16_attention_weight_max": 39.07138442993164,
      "activations/layer16_attention_weight_min": -29.020042419433594,
      "activations/layer17_attention_weight_max": 42.87025833129883,
      "activations/layer17_attention_weight_min": -26.802762985229492,
      "activations/layer18_attention_weight_max": 35.996212005615234,
      "activations/layer18_attention_weight_min": -20.62636375427246,
      "activations/layer19_attention_weight_max": 40.92002868652344,
      "activations/layer19_attention_weight_min": -31.64957046508789,
      "activations/layer1_attention_weight_max": 19.32622528076172,
      "activations/layer1_attention_weight_min": -14.820526123046875,
      "activations/layer20_attention_weight_max": 38.501380920410156,
      "activations/layer20_attention_weight_min": -26.654478073120117,
      "activations/layer21_attention_weight_max": 38.47999572753906,
      "activations/layer21_attention_weight_min": -22.536544799804688,
      "activations/layer22_attention_weight_max": 56.522117614746094,
      "activations/layer22_attention_weight_min": -31.270893096923828,
      "activations/layer23_attention_weight_max": 32.624420166015625,
      "activations/layer23_attention_weight_min": -20.036731719970703,
      "activations/layer2_attention_weight_max": 30.785099029541016,
      "activations/layer2_attention_weight_min": -33.4018669128418,
      "activations/layer3_attention_weight_max": 87.6585922241211,
      "activations/layer3_attention_weight_min": -91.2433853149414,
      "activations/layer4_attention_weight_max": 116.5991439819336,
      "activations/layer4_attention_weight_min": -126.63353729248047,
      "activations/layer5_attention_weight_max": 78.32186889648438,
      "activations/layer5_attention_weight_min": -84.15037536621094,
      "activations/layer6_attention_weight_max": 52.919837951660156,
      "activations/layer6_attention_weight_min": -56.5758171081543,
      "activations/layer7_attention_weight_max": 74.13423919677734,
      "activations/layer7_attention_weight_min": -78.92337799072266,
      "activations/layer8_attention_weight_max": 38.150733947753906,
      "activations/layer8_attention_weight_min": -39.02321243286133,
      "activations/layer9_attention_weight_max": 33.57377624511719,
      "activations/layer9_attention_weight_min": -33.35186767578125,
      "epoch": 4.27,
      "learning_rate": 0.0001237026515151515,
      "loss": 2.8536,
      "step": 73500
    },
    {
      "activations/layer0_attention_weight_max": 16.948989868164062,
      "activations/layer0_attention_weight_min": -12.547489166259766,
      "activations/layer10_attention_weight_max": 33.13697814941406,
      "activations/layer10_attention_weight_min": -29.020402908325195,
      "activations/layer11_attention_weight_max": 33.31874084472656,
      "activations/layer11_attention_weight_min": -28.792049407958984,
      "activations/layer12_attention_weight_max": 43.40742111206055,
      "activations/layer12_attention_weight_min": -28.272262573242188,
      "activations/layer13_attention_weight_max": 52.062564849853516,
      "activations/layer13_attention_weight_min": -35.264488220214844,
      "activations/layer14_attention_weight_max": 72.3847885131836,
      "activations/layer14_attention_weight_min": -56.18402862548828,
      "activations/layer15_attention_weight_max": 44.24090576171875,
      "activations/layer15_attention_weight_min": -28.896167755126953,
      "activations/layer16_attention_weight_max": 44.577457427978516,
      "activations/layer16_attention_weight_min": -32.306114196777344,
      "activations/layer17_attention_weight_max": 41.80413818359375,
      "activations/layer17_attention_weight_min": -25.80942153930664,
      "activations/layer18_attention_weight_max": 30.832250595092773,
      "activations/layer18_attention_weight_min": -19.653499603271484,
      "activations/layer19_attention_weight_max": 44.15543746948242,
      "activations/layer19_attention_weight_min": -28.997888565063477,
      "activations/layer1_attention_weight_max": 18.015451431274414,
      "activations/layer1_attention_weight_min": -14.55255126953125,
      "activations/layer20_attention_weight_max": 34.33476257324219,
      "activations/layer20_attention_weight_min": -22.157934188842773,
      "activations/layer21_attention_weight_max": 31.436452865600586,
      "activations/layer21_attention_weight_min": -22.008359909057617,
      "activations/layer22_attention_weight_max": 54.43760299682617,
      "activations/layer22_attention_weight_min": -28.815143585205078,
      "activations/layer23_attention_weight_max": 34.536067962646484,
      "activations/layer23_attention_weight_min": -21.148204803466797,
      "activations/layer2_attention_weight_max": 30.349546432495117,
      "activations/layer2_attention_weight_min": -29.27785873413086,
      "activations/layer3_attention_weight_max": 81.93798828125,
      "activations/layer3_attention_weight_min": -83.3645248413086,
      "activations/layer4_attention_weight_max": 127.40303802490234,
      "activations/layer4_attention_weight_min": -126.32567596435547,
      "activations/layer5_attention_weight_max": 86.93711853027344,
      "activations/layer5_attention_weight_min": -81.88465881347656,
      "activations/layer6_attention_weight_max": 57.84185791015625,
      "activations/layer6_attention_weight_min": -56.76431655883789,
      "activations/layer7_attention_weight_max": 75.59380340576172,
      "activations/layer7_attention_weight_min": -77.5907211303711,
      "activations/layer8_attention_weight_max": 38.91627883911133,
      "activations/layer8_attention_weight_min": -37.83586120605469,
      "activations/layer9_attention_weight_max": 39.177608489990234,
      "activations/layer9_attention_weight_min": -31.923782348632812,
      "epoch": 4.27,
      "learning_rate": 0.0001236837121212121,
      "loss": 2.8639,
      "step": 73550
    },
    {
      "activations/layer0_attention_weight_max": 15.81637191772461,
      "activations/layer0_attention_weight_min": -11.928693771362305,
      "activations/layer10_attention_weight_max": 36.944122314453125,
      "activations/layer10_attention_weight_min": -30.005905151367188,
      "activations/layer11_attention_weight_max": 40.07915115356445,
      "activations/layer11_attention_weight_min": -29.039485931396484,
      "activations/layer12_attention_weight_max": 29.932994842529297,
      "activations/layer12_attention_weight_min": -28.515108108520508,
      "activations/layer13_attention_weight_max": 45.28990173339844,
      "activations/layer13_attention_weight_min": -33.536006927490234,
      "activations/layer14_attention_weight_max": 62.455623626708984,
      "activations/layer14_attention_weight_min": -49.05280303955078,
      "activations/layer15_attention_weight_max": 45.72130584716797,
      "activations/layer15_attention_weight_min": -30.783401489257812,
      "activations/layer16_attention_weight_max": 40.84926986694336,
      "activations/layer16_attention_weight_min": -28.87321662902832,
      "activations/layer17_attention_weight_max": 41.88875198364258,
      "activations/layer17_attention_weight_min": -26.09620475769043,
      "activations/layer18_attention_weight_max": 31.326223373413086,
      "activations/layer18_attention_weight_min": -21.113908767700195,
      "activations/layer19_attention_weight_max": 44.362525939941406,
      "activations/layer19_attention_weight_min": -28.226810455322266,
      "activations/layer1_attention_weight_max": 18.213539123535156,
      "activations/layer1_attention_weight_min": -16.083106994628906,
      "activations/layer20_attention_weight_max": 34.26654815673828,
      "activations/layer20_attention_weight_min": -22.82589340209961,
      "activations/layer21_attention_weight_max": 33.79180908203125,
      "activations/layer21_attention_weight_min": -20.259525299072266,
      "activations/layer22_attention_weight_max": 51.219627380371094,
      "activations/layer22_attention_weight_min": -30.640676498413086,
      "activations/layer23_attention_weight_max": 31.534818649291992,
      "activations/layer23_attention_weight_min": -20.426097869873047,
      "activations/layer2_attention_weight_max": 33.19654083251953,
      "activations/layer2_attention_weight_min": -31.06008529663086,
      "activations/layer3_attention_weight_max": 87.16963958740234,
      "activations/layer3_attention_weight_min": -88.2373275756836,
      "activations/layer4_attention_weight_max": 133.0357208251953,
      "activations/layer4_attention_weight_min": -132.33509826660156,
      "activations/layer5_attention_weight_max": 90.09500122070312,
      "activations/layer5_attention_weight_min": -82.48291778564453,
      "activations/layer6_attention_weight_max": 61.72230529785156,
      "activations/layer6_attention_weight_min": -56.55704879760742,
      "activations/layer7_attention_weight_max": 82.62442779541016,
      "activations/layer7_attention_weight_min": -78.16333770751953,
      "activations/layer8_attention_weight_max": 41.51912307739258,
      "activations/layer8_attention_weight_min": -39.7619514465332,
      "activations/layer9_attention_weight_max": 41.5484504699707,
      "activations/layer9_attention_weight_min": -33.86286544799805,
      "epoch": 4.28,
      "learning_rate": 0.0001236647727272727,
      "loss": 2.8779,
      "step": 73600
    },
    {
      "activations/layer0_attention_weight_max": 16.022666931152344,
      "activations/layer0_attention_weight_min": -12.325876235961914,
      "activations/layer10_attention_weight_max": 49.09324645996094,
      "activations/layer10_attention_weight_min": -35.00031280517578,
      "activations/layer11_attention_weight_max": 47.64446258544922,
      "activations/layer11_attention_weight_min": -28.966514587402344,
      "activations/layer12_attention_weight_max": 37.555843353271484,
      "activations/layer12_attention_weight_min": -37.06401443481445,
      "activations/layer13_attention_weight_max": 56.1434326171875,
      "activations/layer13_attention_weight_min": -32.99612045288086,
      "activations/layer14_attention_weight_max": 81.90934753417969,
      "activations/layer14_attention_weight_min": -50.531986236572266,
      "activations/layer15_attention_weight_max": 46.93799591064453,
      "activations/layer15_attention_weight_min": -29.307510375976562,
      "activations/layer16_attention_weight_max": 45.327415466308594,
      "activations/layer16_attention_weight_min": -30.35036277770996,
      "activations/layer17_attention_weight_max": 43.05751037597656,
      "activations/layer17_attention_weight_min": -24.81318473815918,
      "activations/layer18_attention_weight_max": 33.19856643676758,
      "activations/layer18_attention_weight_min": -19.54596710205078,
      "activations/layer19_attention_weight_max": 48.98601150512695,
      "activations/layer19_attention_weight_min": -28.935632705688477,
      "activations/layer1_attention_weight_max": 20.3520450592041,
      "activations/layer1_attention_weight_min": -18.3671875,
      "activations/layer20_attention_weight_max": 37.11248779296875,
      "activations/layer20_attention_weight_min": -24.374725341796875,
      "activations/layer21_attention_weight_max": 34.2059211730957,
      "activations/layer21_attention_weight_min": -18.931018829345703,
      "activations/layer22_attention_weight_max": 59.33684539794922,
      "activations/layer22_attention_weight_min": -28.58173942565918,
      "activations/layer23_attention_weight_max": 33.536407470703125,
      "activations/layer23_attention_weight_min": -20.40072250366211,
      "activations/layer2_attention_weight_max": 31.753103256225586,
      "activations/layer2_attention_weight_min": -31.987491607666016,
      "activations/layer3_attention_weight_max": 85.37454986572266,
      "activations/layer3_attention_weight_min": -90.60816192626953,
      "activations/layer4_attention_weight_max": 136.0759735107422,
      "activations/layer4_attention_weight_min": -134.20077514648438,
      "activations/layer5_attention_weight_max": 95.4179916381836,
      "activations/layer5_attention_weight_min": -86.5727767944336,
      "activations/layer6_attention_weight_max": 60.07904052734375,
      "activations/layer6_attention_weight_min": -57.09032440185547,
      "activations/layer7_attention_weight_max": 85.2025375366211,
      "activations/layer7_attention_weight_min": -80.17797088623047,
      "activations/layer8_attention_weight_max": 45.887325286865234,
      "activations/layer8_attention_weight_min": -41.14796447753906,
      "activations/layer9_attention_weight_max": 43.470420837402344,
      "activations/layer9_attention_weight_min": -37.5120735168457,
      "epoch": 4.28,
      "learning_rate": 0.0001236462121212121,
      "loss": 2.862,
      "step": 73650
    },
    {
      "activations/layer0_attention_weight_max": 16.55689811706543,
      "activations/layer0_attention_weight_min": -12.563030242919922,
      "activations/layer10_attention_weight_max": 33.09516525268555,
      "activations/layer10_attention_weight_min": -28.806180953979492,
      "activations/layer11_attention_weight_max": 30.945369720458984,
      "activations/layer11_attention_weight_min": -27.228322982788086,
      "activations/layer12_attention_weight_max": 30.365646362304688,
      "activations/layer12_attention_weight_min": -26.717348098754883,
      "activations/layer13_attention_weight_max": 43.731849670410156,
      "activations/layer13_attention_weight_min": -34.22597885131836,
      "activations/layer14_attention_weight_max": 60.86752700805664,
      "activations/layer14_attention_weight_min": -47.10529708862305,
      "activations/layer15_attention_weight_max": 39.3599739074707,
      "activations/layer15_attention_weight_min": -31.81404685974121,
      "activations/layer16_attention_weight_max": 39.621070861816406,
      "activations/layer16_attention_weight_min": -27.402624130249023,
      "activations/layer17_attention_weight_max": 35.53517150878906,
      "activations/layer17_attention_weight_min": -24.46275520324707,
      "activations/layer18_attention_weight_max": 32.86094665527344,
      "activations/layer18_attention_weight_min": -20.59444808959961,
      "activations/layer19_attention_weight_max": 40.601070404052734,
      "activations/layer19_attention_weight_min": -28.319311141967773,
      "activations/layer1_attention_weight_max": 17.632125854492188,
      "activations/layer1_attention_weight_min": -14.797481536865234,
      "activations/layer20_attention_weight_max": 31.688676834106445,
      "activations/layer20_attention_weight_min": -24.468542098999023,
      "activations/layer21_attention_weight_max": 31.847623825073242,
      "activations/layer21_attention_weight_min": -20.25876235961914,
      "activations/layer22_attention_weight_max": 45.254173278808594,
      "activations/layer22_attention_weight_min": -28.107297897338867,
      "activations/layer23_attention_weight_max": 29.722007751464844,
      "activations/layer23_attention_weight_min": -19.216196060180664,
      "activations/layer2_attention_weight_max": 31.267337799072266,
      "activations/layer2_attention_weight_min": -30.50001335144043,
      "activations/layer3_attention_weight_max": 84.3416976928711,
      "activations/layer3_attention_weight_min": -84.70755004882812,
      "activations/layer4_attention_weight_max": 127.9009017944336,
      "activations/layer4_attention_weight_min": -128.88673400878906,
      "activations/layer5_attention_weight_max": 87.34848022460938,
      "activations/layer5_attention_weight_min": -84.31773376464844,
      "activations/layer6_attention_weight_max": 59.17869186401367,
      "activations/layer6_attention_weight_min": -54.50102615356445,
      "activations/layer7_attention_weight_max": 79.3900146484375,
      "activations/layer7_attention_weight_min": -75.91268157958984,
      "activations/layer8_attention_weight_max": 38.25578308105469,
      "activations/layer8_attention_weight_min": -38.56911087036133,
      "activations/layer9_attention_weight_max": 35.653358459472656,
      "activations/layer9_attention_weight_min": -34.1782112121582,
      "epoch": 4.28,
      "learning_rate": 0.00012362727272727272,
      "loss": 2.8591,
      "step": 73700
    },
    {
      "activations/layer0_attention_weight_max": 16.396976470947266,
      "activations/layer0_attention_weight_min": -12.989712715148926,
      "activations/layer10_attention_weight_max": 36.70684814453125,
      "activations/layer10_attention_weight_min": -31.234214782714844,
      "activations/layer11_attention_weight_max": 30.748502731323242,
      "activations/layer11_attention_weight_min": -26.000057220458984,
      "activations/layer12_attention_weight_max": 24.699939727783203,
      "activations/layer12_attention_weight_min": -26.24612808227539,
      "activations/layer13_attention_weight_max": 44.00204086303711,
      "activations/layer13_attention_weight_min": -32.32080078125,
      "activations/layer14_attention_weight_max": 67.45103454589844,
      "activations/layer14_attention_weight_min": -48.35295104980469,
      "activations/layer15_attention_weight_max": 39.316436767578125,
      "activations/layer15_attention_weight_min": -27.19441032409668,
      "activations/layer16_attention_weight_max": 38.16017532348633,
      "activations/layer16_attention_weight_min": -26.53082275390625,
      "activations/layer17_attention_weight_max": 39.17538833618164,
      "activations/layer17_attention_weight_min": -24.67603874206543,
      "activations/layer18_attention_weight_max": 30.179874420166016,
      "activations/layer18_attention_weight_min": -19.403196334838867,
      "activations/layer19_attention_weight_max": 38.726375579833984,
      "activations/layer19_attention_weight_min": -28.03163719177246,
      "activations/layer1_attention_weight_max": 16.81452751159668,
      "activations/layer1_attention_weight_min": -15.190674781799316,
      "activations/layer20_attention_weight_max": 32.54726028442383,
      "activations/layer20_attention_weight_min": -22.875389099121094,
      "activations/layer21_attention_weight_max": 29.519638061523438,
      "activations/layer21_attention_weight_min": -18.586406707763672,
      "activations/layer22_attention_weight_max": 52.33938217163086,
      "activations/layer22_attention_weight_min": -26.13794708251953,
      "activations/layer23_attention_weight_max": 32.84679412841797,
      "activations/layer23_attention_weight_min": -17.016130447387695,
      "activations/layer2_attention_weight_max": 32.295345306396484,
      "activations/layer2_attention_weight_min": -30.275495529174805,
      "activations/layer3_attention_weight_max": 84.75857543945312,
      "activations/layer3_attention_weight_min": -86.20723724365234,
      "activations/layer4_attention_weight_max": 131.59239196777344,
      "activations/layer4_attention_weight_min": -125.70245361328125,
      "activations/layer5_attention_weight_max": 91.47991180419922,
      "activations/layer5_attention_weight_min": -84.25521850585938,
      "activations/layer6_attention_weight_max": 59.464847564697266,
      "activations/layer6_attention_weight_min": -59.810428619384766,
      "activations/layer7_attention_weight_max": 80.79167175292969,
      "activations/layer7_attention_weight_min": -84.0728988647461,
      "activations/layer8_attention_weight_max": 39.85076141357422,
      "activations/layer8_attention_weight_min": -39.515541076660156,
      "activations/layer9_attention_weight_max": 37.452762603759766,
      "activations/layer9_attention_weight_min": -34.375911712646484,
      "epoch": 4.29,
      "learning_rate": 0.00012360833333333333,
      "loss": 2.8821,
      "step": 73750
    },
    {
      "activations/layer0_attention_weight_max": 16.524370193481445,
      "activations/layer0_attention_weight_min": -11.451672554016113,
      "activations/layer10_attention_weight_max": 33.057716369628906,
      "activations/layer10_attention_weight_min": -31.042198181152344,
      "activations/layer11_attention_weight_max": 37.378658294677734,
      "activations/layer11_attention_weight_min": -28.06525993347168,
      "activations/layer12_attention_weight_max": 26.63229751586914,
      "activations/layer12_attention_weight_min": -26.38680648803711,
      "activations/layer13_attention_weight_max": 37.831275939941406,
      "activations/layer13_attention_weight_min": -30.847339630126953,
      "activations/layer14_attention_weight_max": 59.954811096191406,
      "activations/layer14_attention_weight_min": -46.88034439086914,
      "activations/layer15_attention_weight_max": 37.67706298828125,
      "activations/layer15_attention_weight_min": -27.983001708984375,
      "activations/layer16_attention_weight_max": 38.787471771240234,
      "activations/layer16_attention_weight_min": -27.214473724365234,
      "activations/layer17_attention_weight_max": 33.640953063964844,
      "activations/layer17_attention_weight_min": -24.021474838256836,
      "activations/layer18_attention_weight_max": 29.226119995117188,
      "activations/layer18_attention_weight_min": -19.22199058532715,
      "activations/layer19_attention_weight_max": 37.89206314086914,
      "activations/layer19_attention_weight_min": -28.96901512145996,
      "activations/layer1_attention_weight_max": 17.58358383178711,
      "activations/layer1_attention_weight_min": -15.168119430541992,
      "activations/layer20_attention_weight_max": 32.0152702331543,
      "activations/layer20_attention_weight_min": -23.160083770751953,
      "activations/layer21_attention_weight_max": 27.83669090270996,
      "activations/layer21_attention_weight_min": -20.762561798095703,
      "activations/layer22_attention_weight_max": 45.513877868652344,
      "activations/layer22_attention_weight_min": -26.820363998413086,
      "activations/layer23_attention_weight_max": 28.094470977783203,
      "activations/layer23_attention_weight_min": -19.166215896606445,
      "activations/layer2_attention_weight_max": 33.524871826171875,
      "activations/layer2_attention_weight_min": -31.18310546875,
      "activations/layer3_attention_weight_max": 89.03739929199219,
      "activations/layer3_attention_weight_min": -91.3623046875,
      "activations/layer4_attention_weight_max": 136.96502685546875,
      "activations/layer4_attention_weight_min": -134.4714813232422,
      "activations/layer5_attention_weight_max": 87.72637939453125,
      "activations/layer5_attention_weight_min": -89.356689453125,
      "activations/layer6_attention_weight_max": 57.81275939941406,
      "activations/layer6_attention_weight_min": -55.79389953613281,
      "activations/layer7_attention_weight_max": 80.06519317626953,
      "activations/layer7_attention_weight_min": -77.62348175048828,
      "activations/layer8_attention_weight_max": 39.64128494262695,
      "activations/layer8_attention_weight_min": -39.9570198059082,
      "activations/layer9_attention_weight_max": 40.127864837646484,
      "activations/layer9_attention_weight_min": -32.41982650756836,
      "epoch": 4.29,
      "learning_rate": 0.00012358939393939392,
      "loss": 2.8702,
      "step": 73800
    },
    {
      "activations/layer0_attention_weight_max": 16.862964630126953,
      "activations/layer0_attention_weight_min": -13.622406005859375,
      "activations/layer10_attention_weight_max": 38.58250427246094,
      "activations/layer10_attention_weight_min": -29.742918014526367,
      "activations/layer11_attention_weight_max": 46.810157775878906,
      "activations/layer11_attention_weight_min": -29.24713134765625,
      "activations/layer12_attention_weight_max": 55.64757537841797,
      "activations/layer12_attention_weight_min": -25.385644912719727,
      "activations/layer13_attention_weight_max": 73.36638641357422,
      "activations/layer13_attention_weight_min": -39.57013702392578,
      "activations/layer14_attention_weight_max": 73.06930541992188,
      "activations/layer14_attention_weight_min": -52.796470642089844,
      "activations/layer15_attention_weight_max": 45.80731964111328,
      "activations/layer15_attention_weight_min": -30.253969192504883,
      "activations/layer16_attention_weight_max": 43.83964157104492,
      "activations/layer16_attention_weight_min": -28.2247257232666,
      "activations/layer17_attention_weight_max": 47.49904251098633,
      "activations/layer17_attention_weight_min": -25.06761932373047,
      "activations/layer18_attention_weight_max": 32.973350524902344,
      "activations/layer18_attention_weight_min": -20.36408805847168,
      "activations/layer19_attention_weight_max": 43.71754455566406,
      "activations/layer19_attention_weight_min": -29.564794540405273,
      "activations/layer1_attention_weight_max": 18.228818893432617,
      "activations/layer1_attention_weight_min": -13.555927276611328,
      "activations/layer20_attention_weight_max": 33.899391174316406,
      "activations/layer20_attention_weight_min": -22.60087776184082,
      "activations/layer21_attention_weight_max": 36.449913024902344,
      "activations/layer21_attention_weight_min": -22.88645362854004,
      "activations/layer22_attention_weight_max": 51.7636604309082,
      "activations/layer22_attention_weight_min": -28.792675018310547,
      "activations/layer23_attention_weight_max": 34.35099792480469,
      "activations/layer23_attention_weight_min": -18.088346481323242,
      "activations/layer2_attention_weight_max": 31.12093734741211,
      "activations/layer2_attention_weight_min": -29.948631286621094,
      "activations/layer3_attention_weight_max": 85.35086822509766,
      "activations/layer3_attention_weight_min": -86.37141418457031,
      "activations/layer4_attention_weight_max": 132.02613830566406,
      "activations/layer4_attention_weight_min": -130.2644500732422,
      "activations/layer5_attention_weight_max": 88.49275207519531,
      "activations/layer5_attention_weight_min": -83.865478515625,
      "activations/layer6_attention_weight_max": 58.76950454711914,
      "activations/layer6_attention_weight_min": -56.641273498535156,
      "activations/layer7_attention_weight_max": 87.94730377197266,
      "activations/layer7_attention_weight_min": -77.31884002685547,
      "activations/layer8_attention_weight_max": 43.718021392822266,
      "activations/layer8_attention_weight_min": -40.1548957824707,
      "activations/layer9_attention_weight_max": 52.63446044921875,
      "activations/layer9_attention_weight_min": -37.19575119018555,
      "epoch": 4.29,
      "learning_rate": 0.00012357045454545454,
      "loss": 2.8705,
      "step": 73850
    },
    {
      "activations/layer0_attention_weight_max": 15.85660171508789,
      "activations/layer0_attention_weight_min": -11.735435485839844,
      "activations/layer10_attention_weight_max": 40.778594970703125,
      "activations/layer10_attention_weight_min": -30.389793395996094,
      "activations/layer11_attention_weight_max": 38.66505432128906,
      "activations/layer11_attention_weight_min": -28.20343780517578,
      "activations/layer12_attention_weight_max": 31.74384880065918,
      "activations/layer12_attention_weight_min": -25.267868041992188,
      "activations/layer13_attention_weight_max": 52.41889572143555,
      "activations/layer13_attention_weight_min": -33.31136703491211,
      "activations/layer14_attention_weight_max": 71.40318298339844,
      "activations/layer14_attention_weight_min": -43.80812454223633,
      "activations/layer15_attention_weight_max": 46.616241455078125,
      "activations/layer15_attention_weight_min": -29.599821090698242,
      "activations/layer16_attention_weight_max": 42.15949249267578,
      "activations/layer16_attention_weight_min": -27.774417877197266,
      "activations/layer17_attention_weight_max": 46.78680419921875,
      "activations/layer17_attention_weight_min": -25.2891788482666,
      "activations/layer18_attention_weight_max": 34.73756408691406,
      "activations/layer18_attention_weight_min": -20.374574661254883,
      "activations/layer19_attention_weight_max": 46.645198822021484,
      "activations/layer19_attention_weight_min": -29.9213924407959,
      "activations/layer1_attention_weight_max": 17.36158561706543,
      "activations/layer1_attention_weight_min": -14.759940147399902,
      "activations/layer20_attention_weight_max": 37.21506118774414,
      "activations/layer20_attention_weight_min": -24.973466873168945,
      "activations/layer21_attention_weight_max": 34.357696533203125,
      "activations/layer21_attention_weight_min": -22.099430084228516,
      "activations/layer22_attention_weight_max": 59.64564514160156,
      "activations/layer22_attention_weight_min": -26.578197479248047,
      "activations/layer23_attention_weight_max": 34.547279357910156,
      "activations/layer23_attention_weight_min": -19.280364990234375,
      "activations/layer2_attention_weight_max": 32.01243209838867,
      "activations/layer2_attention_weight_min": -31.43789291381836,
      "activations/layer3_attention_weight_max": 87.75763702392578,
      "activations/layer3_attention_weight_min": -89.68795013427734,
      "activations/layer4_attention_weight_max": 130.53248596191406,
      "activations/layer4_attention_weight_min": -128.35818481445312,
      "activations/layer5_attention_weight_max": 92.26765441894531,
      "activations/layer5_attention_weight_min": -84.95762634277344,
      "activations/layer6_attention_weight_max": 56.46202087402344,
      "activations/layer6_attention_weight_min": -58.10190200805664,
      "activations/layer7_attention_weight_max": 76.60984802246094,
      "activations/layer7_attention_weight_min": -76.50127410888672,
      "activations/layer8_attention_weight_max": 39.14918899536133,
      "activations/layer8_attention_weight_min": -38.72765350341797,
      "activations/layer9_attention_weight_max": 41.41285705566406,
      "activations/layer9_attention_weight_min": -33.067203521728516,
      "epoch": 4.29,
      "learning_rate": 0.00012355151515151515,
      "loss": 2.8758,
      "step": 73900
    },
    {
      "activations/layer0_attention_weight_max": 16.702543258666992,
      "activations/layer0_attention_weight_min": -11.398215293884277,
      "activations/layer10_attention_weight_max": 37.24462890625,
      "activations/layer10_attention_weight_min": -32.63818359375,
      "activations/layer11_attention_weight_max": 35.44915008544922,
      "activations/layer11_attention_weight_min": -29.569265365600586,
      "activations/layer12_attention_weight_max": 39.703365325927734,
      "activations/layer12_attention_weight_min": -28.932701110839844,
      "activations/layer13_attention_weight_max": 64.98673248291016,
      "activations/layer13_attention_weight_min": -37.74669647216797,
      "activations/layer14_attention_weight_max": 88.55433654785156,
      "activations/layer14_attention_weight_min": -55.16510772705078,
      "activations/layer15_attention_weight_max": 50.276912689208984,
      "activations/layer15_attention_weight_min": -33.001338958740234,
      "activations/layer16_attention_weight_max": 51.54723358154297,
      "activations/layer16_attention_weight_min": -28.297693252563477,
      "activations/layer17_attention_weight_max": 44.54098892211914,
      "activations/layer17_attention_weight_min": -26.102203369140625,
      "activations/layer18_attention_weight_max": 35.02440643310547,
      "activations/layer18_attention_weight_min": -20.807527542114258,
      "activations/layer19_attention_weight_max": 50.58295822143555,
      "activations/layer19_attention_weight_min": -28.691247940063477,
      "activations/layer1_attention_weight_max": 18.635971069335938,
      "activations/layer1_attention_weight_min": -13.741382598876953,
      "activations/layer20_attention_weight_max": 38.71381378173828,
      "activations/layer20_attention_weight_min": -23.75575065612793,
      "activations/layer21_attention_weight_max": 32.287750244140625,
      "activations/layer21_attention_weight_min": -21.800649642944336,
      "activations/layer22_attention_weight_max": 57.76760482788086,
      "activations/layer22_attention_weight_min": -29.799375534057617,
      "activations/layer23_attention_weight_max": 33.7472038269043,
      "activations/layer23_attention_weight_min": -21.79104995727539,
      "activations/layer2_attention_weight_max": 31.260005950927734,
      "activations/layer2_attention_weight_min": -30.888595581054688,
      "activations/layer3_attention_weight_max": 84.44152069091797,
      "activations/layer3_attention_weight_min": -89.67789459228516,
      "activations/layer4_attention_weight_max": 129.92575073242188,
      "activations/layer4_attention_weight_min": -125.16133880615234,
      "activations/layer5_attention_weight_max": 87.74541473388672,
      "activations/layer5_attention_weight_min": -81.90459442138672,
      "activations/layer6_attention_weight_max": 55.68633270263672,
      "activations/layer6_attention_weight_min": -58.51718521118164,
      "activations/layer7_attention_weight_max": 77.13607025146484,
      "activations/layer7_attention_weight_min": -79.87606811523438,
      "activations/layer8_attention_weight_max": 40.148529052734375,
      "activations/layer8_attention_weight_min": -41.3511848449707,
      "activations/layer9_attention_weight_max": 37.12700271606445,
      "activations/layer9_attention_weight_min": -33.440311431884766,
      "epoch": 4.3,
      "learning_rate": 0.00012353257575757574,
      "loss": 2.8654,
      "step": 73950
    },
    {
      "activations/layer0_attention_weight_max": 16.421714782714844,
      "activations/layer0_attention_weight_min": -11.866172790527344,
      "activations/layer10_attention_weight_max": 35.082481384277344,
      "activations/layer10_attention_weight_min": -31.353939056396484,
      "activations/layer11_attention_weight_max": 32.428104400634766,
      "activations/layer11_attention_weight_min": -28.11420440673828,
      "activations/layer12_attention_weight_max": 27.0565128326416,
      "activations/layer12_attention_weight_min": -32.636077880859375,
      "activations/layer13_attention_weight_max": 48.00233840942383,
      "activations/layer13_attention_weight_min": -36.58030700683594,
      "activations/layer14_attention_weight_max": 68.41693878173828,
      "activations/layer14_attention_weight_min": -53.24544143676758,
      "activations/layer15_attention_weight_max": 43.07110595703125,
      "activations/layer15_attention_weight_min": -27.678346633911133,
      "activations/layer16_attention_weight_max": 36.45048141479492,
      "activations/layer16_attention_weight_min": -28.214181900024414,
      "activations/layer17_attention_weight_max": 38.418128967285156,
      "activations/layer17_attention_weight_min": -23.87171745300293,
      "activations/layer18_attention_weight_max": 29.595125198364258,
      "activations/layer18_attention_weight_min": -20.24467658996582,
      "activations/layer19_attention_weight_max": 40.11011505126953,
      "activations/layer19_attention_weight_min": -30.8537540435791,
      "activations/layer1_attention_weight_max": 17.63397979736328,
      "activations/layer1_attention_weight_min": -16.021318435668945,
      "activations/layer20_attention_weight_max": 32.48552322387695,
      "activations/layer20_attention_weight_min": -24.47493553161621,
      "activations/layer21_attention_weight_max": 33.1761474609375,
      "activations/layer21_attention_weight_min": -22.227188110351562,
      "activations/layer22_attention_weight_max": 48.966129302978516,
      "activations/layer22_attention_weight_min": -30.20656967163086,
      "activations/layer23_attention_weight_max": 30.69471549987793,
      "activations/layer23_attention_weight_min": -22.34315299987793,
      "activations/layer2_attention_weight_max": 32.201988220214844,
      "activations/layer2_attention_weight_min": -31.135337829589844,
      "activations/layer3_attention_weight_max": 84.40186309814453,
      "activations/layer3_attention_weight_min": -83.35635375976562,
      "activations/layer4_attention_weight_max": 131.38446044921875,
      "activations/layer4_attention_weight_min": -124.04931640625,
      "activations/layer5_attention_weight_max": 89.28021240234375,
      "activations/layer5_attention_weight_min": -84.42015075683594,
      "activations/layer6_attention_weight_max": 55.45344543457031,
      "activations/layer6_attention_weight_min": -55.42247772216797,
      "activations/layer7_attention_weight_max": 78.09538269042969,
      "activations/layer7_attention_weight_min": -76.46330261230469,
      "activations/layer8_attention_weight_max": 41.56830596923828,
      "activations/layer8_attention_weight_min": -36.8814697265625,
      "activations/layer9_attention_weight_max": 45.96356201171875,
      "activations/layer9_attention_weight_min": -33.62046432495117,
      "epoch": 4.3,
      "learning_rate": 0.00012351363636363635,
      "loss": 2.8625,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_loss": 2.818359375,
      "eval_runtime": 8.4909,
      "eval_samples_per_second": 505.72,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_openwebtext_loss": 2.818359375,
      "eval_openwebtext_ppl": 16.74934871787156,
      "eval_openwebtext_runtime": 8.4909,
      "eval_openwebtext_samples_per_second": 505.72,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_wikitext_loss": 3.0546875,
      "eval_wikitext_ppl": 21.214554942517648,
      "eval_wikitext_runtime": 1.9756,
      "eval_wikitext_samples_per_second": 230.814,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_lambada_loss": 2.8671875,
      "eval_lambada_ppl": 17.587483776585056,
      "eval_lambada_runtime": 9.56,
      "eval_lambada_samples_per_second": 509.312,
      "step": 74000
    },
    {
      "activations/layer0_attention_weight_max": 17.09372901916504,
      "activations/layer0_attention_weight_min": -12.793340682983398,
      "activations/layer10_attention_weight_max": 33.27558517456055,
      "activations/layer10_attention_weight_min": -30.82962989807129,
      "activations/layer11_attention_weight_max": 31.397247314453125,
      "activations/layer11_attention_weight_min": -27.26126480102539,
      "activations/layer12_attention_weight_max": 23.75821876525879,
      "activations/layer12_attention_weight_min": -26.49699592590332,
      "activations/layer13_attention_weight_max": 39.726112365722656,
      "activations/layer13_attention_weight_min": -33.33372497558594,
      "activations/layer14_attention_weight_max": 60.16889953613281,
      "activations/layer14_attention_weight_min": -48.05341339111328,
      "activations/layer15_attention_weight_max": 40.02297592163086,
      "activations/layer15_attention_weight_min": -29.470258712768555,
      "activations/layer16_attention_weight_max": 36.45755386352539,
      "activations/layer16_attention_weight_min": -27.206172943115234,
      "activations/layer17_attention_weight_max": 39.213436126708984,
      "activations/layer17_attention_weight_min": -25.935203552246094,
      "activations/layer18_attention_weight_max": 31.994342803955078,
      "activations/layer18_attention_weight_min": -20.54192543029785,
      "activations/layer19_attention_weight_max": 42.04718780517578,
      "activations/layer19_attention_weight_min": -29.882898330688477,
      "activations/layer1_attention_weight_max": 16.468820571899414,
      "activations/layer1_attention_weight_min": -13.659730911254883,
      "activations/layer20_attention_weight_max": 36.67410659790039,
      "activations/layer20_attention_weight_min": -23.192350387573242,
      "activations/layer21_attention_weight_max": 31.315092086791992,
      "activations/layer21_attention_weight_min": -19.712936401367188,
      "activations/layer22_attention_weight_max": 51.299652099609375,
      "activations/layer22_attention_weight_min": -27.99574851989746,
      "activations/layer23_attention_weight_max": 29.112957000732422,
      "activations/layer23_attention_weight_min": -18.406600952148438,
      "activations/layer2_attention_weight_max": 34.326480865478516,
      "activations/layer2_attention_weight_min": -31.67229461669922,
      "activations/layer3_attention_weight_max": 92.22142791748047,
      "activations/layer3_attention_weight_min": -92.21454620361328,
      "activations/layer4_attention_weight_max": 139.97714233398438,
      "activations/layer4_attention_weight_min": -131.8093719482422,
      "activations/layer5_attention_weight_max": 88.41372680664062,
      "activations/layer5_attention_weight_min": -82.89805603027344,
      "activations/layer6_attention_weight_max": 60.373619079589844,
      "activations/layer6_attention_weight_min": -55.541343688964844,
      "activations/layer7_attention_weight_max": 80.69989776611328,
      "activations/layer7_attention_weight_min": -78.22642517089844,
      "activations/layer8_attention_weight_max": 37.99150466918945,
      "activations/layer8_attention_weight_min": -37.3748893737793,
      "activations/layer9_attention_weight_max": 37.892486572265625,
      "activations/layer9_attention_weight_min": -33.609710693359375,
      "epoch": 4.3,
      "learning_rate": 0.00012349469696969697,
      "loss": 2.8685,
      "step": 74050
    },
    {
      "activations/layer0_attention_weight_max": 16.44630241394043,
      "activations/layer0_attention_weight_min": -12.235395431518555,
      "activations/layer10_attention_weight_max": 38.289764404296875,
      "activations/layer10_attention_weight_min": -29.747703552246094,
      "activations/layer11_attention_weight_max": 40.5792236328125,
      "activations/layer11_attention_weight_min": -29.488142013549805,
      "activations/layer12_attention_weight_max": 45.07237243652344,
      "activations/layer12_attention_weight_min": -28.427648544311523,
      "activations/layer13_attention_weight_max": 61.80119323730469,
      "activations/layer13_attention_weight_min": -41.23640060424805,
      "activations/layer14_attention_weight_max": 80.61412811279297,
      "activations/layer14_attention_weight_min": -57.112422943115234,
      "activations/layer15_attention_weight_max": 42.294471740722656,
      "activations/layer15_attention_weight_min": -30.935394287109375,
      "activations/layer16_attention_weight_max": 42.69597625732422,
      "activations/layer16_attention_weight_min": -29.58249282836914,
      "activations/layer17_attention_weight_max": 42.5638427734375,
      "activations/layer17_attention_weight_min": -26.346336364746094,
      "activations/layer18_attention_weight_max": 31.926002502441406,
      "activations/layer18_attention_weight_min": -22.037179946899414,
      "activations/layer19_attention_weight_max": 42.85557556152344,
      "activations/layer19_attention_weight_min": -29.327896118164062,
      "activations/layer1_attention_weight_max": 17.575681686401367,
      "activations/layer1_attention_weight_min": -18.118993759155273,
      "activations/layer20_attention_weight_max": 32.691505432128906,
      "activations/layer20_attention_weight_min": -24.585050582885742,
      "activations/layer21_attention_weight_max": 37.736427307128906,
      "activations/layer21_attention_weight_min": -22.742874145507812,
      "activations/layer22_attention_weight_max": 52.24653625488281,
      "activations/layer22_attention_weight_min": -34.16342544555664,
      "activations/layer23_attention_weight_max": 29.610105514526367,
      "activations/layer23_attention_weight_min": -21.787092208862305,
      "activations/layer2_attention_weight_max": 33.09092330932617,
      "activations/layer2_attention_weight_min": -31.820903778076172,
      "activations/layer3_attention_weight_max": 89.14960479736328,
      "activations/layer3_attention_weight_min": -88.57710266113281,
      "activations/layer4_attention_weight_max": 131.2894287109375,
      "activations/layer4_attention_weight_min": -134.61056518554688,
      "activations/layer5_attention_weight_max": 87.2876205444336,
      "activations/layer5_attention_weight_min": -87.23451232910156,
      "activations/layer6_attention_weight_max": 56.34403610229492,
      "activations/layer6_attention_weight_min": -55.375946044921875,
      "activations/layer7_attention_weight_max": 79.80644226074219,
      "activations/layer7_attention_weight_min": -79.27729034423828,
      "activations/layer8_attention_weight_max": 45.10493087768555,
      "activations/layer8_attention_weight_min": -40.027183532714844,
      "activations/layer9_attention_weight_max": 46.56050109863281,
      "activations/layer9_attention_weight_min": -40.26622772216797,
      "epoch": 4.31,
      "learning_rate": 0.00012347575757575758,
      "loss": 2.8561,
      "step": 74100
    },
    {
      "activations/layer0_attention_weight_max": 16.29281234741211,
      "activations/layer0_attention_weight_min": -11.710619926452637,
      "activations/layer10_attention_weight_max": 36.48933029174805,
      "activations/layer10_attention_weight_min": -28.858688354492188,
      "activations/layer11_attention_weight_max": 32.88560485839844,
      "activations/layer11_attention_weight_min": -27.496856689453125,
      "activations/layer12_attention_weight_max": 33.02745056152344,
      "activations/layer12_attention_weight_min": -27.285093307495117,
      "activations/layer13_attention_weight_max": 45.367801666259766,
      "activations/layer13_attention_weight_min": -32.454322814941406,
      "activations/layer14_attention_weight_max": 64.77001953125,
      "activations/layer14_attention_weight_min": -50.35280227661133,
      "activations/layer15_attention_weight_max": 43.72886657714844,
      "activations/layer15_attention_weight_min": -27.375307083129883,
      "activations/layer16_attention_weight_max": 47.15968322753906,
      "activations/layer16_attention_weight_min": -27.617807388305664,
      "activations/layer17_attention_weight_max": 42.510372161865234,
      "activations/layer17_attention_weight_min": -23.785972595214844,
      "activations/layer18_attention_weight_max": 33.3873405456543,
      "activations/layer18_attention_weight_min": -19.48996353149414,
      "activations/layer19_attention_weight_max": 48.56193923950195,
      "activations/layer19_attention_weight_min": -27.916580200195312,
      "activations/layer1_attention_weight_max": 17.447298049926758,
      "activations/layer1_attention_weight_min": -13.934136390686035,
      "activations/layer20_attention_weight_max": 33.78604507446289,
      "activations/layer20_attention_weight_min": -22.849637985229492,
      "activations/layer21_attention_weight_max": 35.59541702270508,
      "activations/layer21_attention_weight_min": -20.44508934020996,
      "activations/layer22_attention_weight_max": 48.97264099121094,
      "activations/layer22_attention_weight_min": -25.758760452270508,
      "activations/layer23_attention_weight_max": 34.56852340698242,
      "activations/layer23_attention_weight_min": -17.849624633789062,
      "activations/layer2_attention_weight_max": 31.741943359375,
      "activations/layer2_attention_weight_min": -31.256925582885742,
      "activations/layer3_attention_weight_max": 85.40330505371094,
      "activations/layer3_attention_weight_min": -90.68011474609375,
      "activations/layer4_attention_weight_max": 127.04927062988281,
      "activations/layer4_attention_weight_min": -127.31107330322266,
      "activations/layer5_attention_weight_max": 86.4363021850586,
      "activations/layer5_attention_weight_min": -84.08123779296875,
      "activations/layer6_attention_weight_max": 57.01958084106445,
      "activations/layer6_attention_weight_min": -59.88842010498047,
      "activations/layer7_attention_weight_max": 87.76569366455078,
      "activations/layer7_attention_weight_min": -76.24065399169922,
      "activations/layer8_attention_weight_max": 40.209320068359375,
      "activations/layer8_attention_weight_min": -38.90205383300781,
      "activations/layer9_attention_weight_max": 35.28731918334961,
      "activations/layer9_attention_weight_min": -34.496185302734375,
      "epoch": 4.31,
      "learning_rate": 0.00012345681818181817,
      "loss": 2.8664,
      "step": 74150
    },
    {
      "activations/layer0_attention_weight_max": 16.786029815673828,
      "activations/layer0_attention_weight_min": -12.314249038696289,
      "activations/layer10_attention_weight_max": 34.95004653930664,
      "activations/layer10_attention_weight_min": -29.09564781188965,
      "activations/layer11_attention_weight_max": 33.53059768676758,
      "activations/layer11_attention_weight_min": -27.138351440429688,
      "activations/layer12_attention_weight_max": 26.03846549987793,
      "activations/layer12_attention_weight_min": -26.107877731323242,
      "activations/layer13_attention_weight_max": 44.599727630615234,
      "activations/layer13_attention_weight_min": -35.306495666503906,
      "activations/layer14_attention_weight_max": 69.5509262084961,
      "activations/layer14_attention_weight_min": -48.39011764526367,
      "activations/layer15_attention_weight_max": 43.15903091430664,
      "activations/layer15_attention_weight_min": -31.858728408813477,
      "activations/layer16_attention_weight_max": 36.53922653198242,
      "activations/layer16_attention_weight_min": -29.538860321044922,
      "activations/layer17_attention_weight_max": 38.734649658203125,
      "activations/layer17_attention_weight_min": -27.005573272705078,
      "activations/layer18_attention_weight_max": 31.735881805419922,
      "activations/layer18_attention_weight_min": -21.018321990966797,
      "activations/layer19_attention_weight_max": 39.30801010131836,
      "activations/layer19_attention_weight_min": -29.301687240600586,
      "activations/layer1_attention_weight_max": 17.89908790588379,
      "activations/layer1_attention_weight_min": -14.417901039123535,
      "activations/layer20_attention_weight_max": 29.92528533935547,
      "activations/layer20_attention_weight_min": -23.880733489990234,
      "activations/layer21_attention_weight_max": 29.792476654052734,
      "activations/layer21_attention_weight_min": -20.72710609436035,
      "activations/layer22_attention_weight_max": 46.175777435302734,
      "activations/layer22_attention_weight_min": -28.753435134887695,
      "activations/layer23_attention_weight_max": 31.928144454956055,
      "activations/layer23_attention_weight_min": -18.74004364013672,
      "activations/layer2_attention_weight_max": 32.22212600708008,
      "activations/layer2_attention_weight_min": -31.544782638549805,
      "activations/layer3_attention_weight_max": 87.14231872558594,
      "activations/layer3_attention_weight_min": -87.76008605957031,
      "activations/layer4_attention_weight_max": 134.95828247070312,
      "activations/layer4_attention_weight_min": -128.15501403808594,
      "activations/layer5_attention_weight_max": 87.73101043701172,
      "activations/layer5_attention_weight_min": -81.99410247802734,
      "activations/layer6_attention_weight_max": 58.24753189086914,
      "activations/layer6_attention_weight_min": -56.56220626831055,
      "activations/layer7_attention_weight_max": 76.99637603759766,
      "activations/layer7_attention_weight_min": -77.83943176269531,
      "activations/layer8_attention_weight_max": 37.819271087646484,
      "activations/layer8_attention_weight_min": -38.72618103027344,
      "activations/layer9_attention_weight_max": 37.718231201171875,
      "activations/layer9_attention_weight_min": -38.13657760620117,
      "epoch": 4.31,
      "learning_rate": 0.0001234378787878788,
      "loss": 2.8743,
      "step": 74200
    },
    {
      "activations/layer0_attention_weight_max": 15.981951713562012,
      "activations/layer0_attention_weight_min": -11.545161247253418,
      "activations/layer10_attention_weight_max": 37.554779052734375,
      "activations/layer10_attention_weight_min": -33.23754119873047,
      "activations/layer11_attention_weight_max": 33.9857063293457,
      "activations/layer11_attention_weight_min": -31.708084106445312,
      "activations/layer12_attention_weight_max": 57.715328216552734,
      "activations/layer12_attention_weight_min": -34.45981979370117,
      "activations/layer13_attention_weight_max": 65.37065887451172,
      "activations/layer13_attention_weight_min": -46.83063507080078,
      "activations/layer14_attention_weight_max": 76.8799819946289,
      "activations/layer14_attention_weight_min": -60.38569259643555,
      "activations/layer15_attention_weight_max": 42.83799743652344,
      "activations/layer15_attention_weight_min": -31.352354049682617,
      "activations/layer16_attention_weight_max": 37.93820571899414,
      "activations/layer16_attention_weight_min": -28.342041015625,
      "activations/layer17_attention_weight_max": 40.102657318115234,
      "activations/layer17_attention_weight_min": -27.113380432128906,
      "activations/layer18_attention_weight_max": 29.179786682128906,
      "activations/layer18_attention_weight_min": -20.417530059814453,
      "activations/layer19_attention_weight_max": 48.218990325927734,
      "activations/layer19_attention_weight_min": -28.373205184936523,
      "activations/layer1_attention_weight_max": 17.145349502563477,
      "activations/layer1_attention_weight_min": -15.193501472473145,
      "activations/layer20_attention_weight_max": 33.44876480102539,
      "activations/layer20_attention_weight_min": -21.782390594482422,
      "activations/layer21_attention_weight_max": 29.88553810119629,
      "activations/layer21_attention_weight_min": -19.567873001098633,
      "activations/layer22_attention_weight_max": 52.90769577026367,
      "activations/layer22_attention_weight_min": -29.091489791870117,
      "activations/layer23_attention_weight_max": 36.067501068115234,
      "activations/layer23_attention_weight_min": -20.902576446533203,
      "activations/layer2_attention_weight_max": 33.02256774902344,
      "activations/layer2_attention_weight_min": -32.99871063232422,
      "activations/layer3_attention_weight_max": 90.26626586914062,
      "activations/layer3_attention_weight_min": -92.45622253417969,
      "activations/layer4_attention_weight_max": 136.7830352783203,
      "activations/layer4_attention_weight_min": -138.02613830566406,
      "activations/layer5_attention_weight_max": 91.53392028808594,
      "activations/layer5_attention_weight_min": -90.46331024169922,
      "activations/layer6_attention_weight_max": 57.28215789794922,
      "activations/layer6_attention_weight_min": -59.21683883666992,
      "activations/layer7_attention_weight_max": 81.50012969970703,
      "activations/layer7_attention_weight_min": -79.92389678955078,
      "activations/layer8_attention_weight_max": 39.80859375,
      "activations/layer8_attention_weight_min": -37.93375015258789,
      "activations/layer9_attention_weight_max": 36.53904342651367,
      "activations/layer9_attention_weight_min": -33.640159606933594,
      "epoch": 4.31,
      "learning_rate": 0.00012341893939393938,
      "loss": 2.8563,
      "step": 74250
    },
    {
      "activations/layer0_attention_weight_max": 16.941852569580078,
      "activations/layer0_attention_weight_min": -11.731542587280273,
      "activations/layer10_attention_weight_max": 40.282142639160156,
      "activations/layer10_attention_weight_min": -32.26538848876953,
      "activations/layer11_attention_weight_max": 42.834373474121094,
      "activations/layer11_attention_weight_min": -34.81256103515625,
      "activations/layer12_attention_weight_max": 32.858428955078125,
      "activations/layer12_attention_weight_min": -31.519506454467773,
      "activations/layer13_attention_weight_max": 50.27438735961914,
      "activations/layer13_attention_weight_min": -33.001522064208984,
      "activations/layer14_attention_weight_max": 71.29613494873047,
      "activations/layer14_attention_weight_min": -55.95280075073242,
      "activations/layer15_attention_weight_max": 42.18799591064453,
      "activations/layer15_attention_weight_min": -29.337677001953125,
      "activations/layer16_attention_weight_max": 45.811100006103516,
      "activations/layer16_attention_weight_min": -27.541521072387695,
      "activations/layer17_attention_weight_max": 40.648136138916016,
      "activations/layer17_attention_weight_min": -25.655431747436523,
      "activations/layer18_attention_weight_max": 32.250396728515625,
      "activations/layer18_attention_weight_min": -18.6442813873291,
      "activations/layer19_attention_weight_max": 39.9239501953125,
      "activations/layer19_attention_weight_min": -26.94821548461914,
      "activations/layer1_attention_weight_max": 16.971908569335938,
      "activations/layer1_attention_weight_min": -14.580137252807617,
      "activations/layer20_attention_weight_max": 29.588083267211914,
      "activations/layer20_attention_weight_min": -21.624109268188477,
      "activations/layer21_attention_weight_max": 29.948898315429688,
      "activations/layer21_attention_weight_min": -18.453060150146484,
      "activations/layer22_attention_weight_max": 48.51972961425781,
      "activations/layer22_attention_weight_min": -25.97219467163086,
      "activations/layer23_attention_weight_max": 29.9860897064209,
      "activations/layer23_attention_weight_min": -18.54574966430664,
      "activations/layer2_attention_weight_max": 33.41802978515625,
      "activations/layer2_attention_weight_min": -32.58735275268555,
      "activations/layer3_attention_weight_max": 95.08256530761719,
      "activations/layer3_attention_weight_min": -95.31271362304688,
      "activations/layer4_attention_weight_max": 142.26280212402344,
      "activations/layer4_attention_weight_min": -127.538818359375,
      "activations/layer5_attention_weight_max": 92.93260192871094,
      "activations/layer5_attention_weight_min": -85.98248291015625,
      "activations/layer6_attention_weight_max": 64.3199462890625,
      "activations/layer6_attention_weight_min": -61.4874382019043,
      "activations/layer7_attention_weight_max": 98.8907699584961,
      "activations/layer7_attention_weight_min": -81.67033386230469,
      "activations/layer8_attention_weight_max": 46.06031799316406,
      "activations/layer8_attention_weight_min": -42.34423065185547,
      "activations/layer9_attention_weight_max": 53.4824104309082,
      "activations/layer9_attention_weight_min": -42.70627975463867,
      "epoch": 4.32,
      "learning_rate": 0.0001234,
      "loss": 2.8521,
      "step": 74300
    },
    {
      "activations/layer0_attention_weight_max": 17.032442092895508,
      "activations/layer0_attention_weight_min": -12.268796920776367,
      "activations/layer10_attention_weight_max": 36.18634796142578,
      "activations/layer10_attention_weight_min": -29.344968795776367,
      "activations/layer11_attention_weight_max": 32.980804443359375,
      "activations/layer11_attention_weight_min": -26.190664291381836,
      "activations/layer12_attention_weight_max": 32.864959716796875,
      "activations/layer12_attention_weight_min": -26.75301742553711,
      "activations/layer13_attention_weight_max": 43.72550582885742,
      "activations/layer13_attention_weight_min": -31.150760650634766,
      "activations/layer14_attention_weight_max": 66.31845092773438,
      "activations/layer14_attention_weight_min": -45.434085845947266,
      "activations/layer15_attention_weight_max": 48.526859283447266,
      "activations/layer15_attention_weight_min": -30.665042877197266,
      "activations/layer16_attention_weight_max": 41.260467529296875,
      "activations/layer16_attention_weight_min": -28.101001739501953,
      "activations/layer17_attention_weight_max": 44.864192962646484,
      "activations/layer17_attention_weight_min": -25.985990524291992,
      "activations/layer18_attention_weight_max": 38.19297409057617,
      "activations/layer18_attention_weight_min": -20.770610809326172,
      "activations/layer19_attention_weight_max": 41.2016716003418,
      "activations/layer19_attention_weight_min": -30.160459518432617,
      "activations/layer1_attention_weight_max": 16.837879180908203,
      "activations/layer1_attention_weight_min": -16.585525512695312,
      "activations/layer20_attention_weight_max": 36.93890380859375,
      "activations/layer20_attention_weight_min": -24.6833438873291,
      "activations/layer21_attention_weight_max": 35.34123611450195,
      "activations/layer21_attention_weight_min": -22.097248077392578,
      "activations/layer22_attention_weight_max": 56.09818649291992,
      "activations/layer22_attention_weight_min": -29.69679069519043,
      "activations/layer23_attention_weight_max": 32.96723937988281,
      "activations/layer23_attention_weight_min": -20.842803955078125,
      "activations/layer2_attention_weight_max": 31.634967803955078,
      "activations/layer2_attention_weight_min": -30.612720489501953,
      "activations/layer3_attention_weight_max": 89.13593292236328,
      "activations/layer3_attention_weight_min": -90.10311126708984,
      "activations/layer4_attention_weight_max": 131.78123474121094,
      "activations/layer4_attention_weight_min": -126.03301239013672,
      "activations/layer5_attention_weight_max": 89.31224060058594,
      "activations/layer5_attention_weight_min": -85.02783203125,
      "activations/layer6_attention_weight_max": 57.27013397216797,
      "activations/layer6_attention_weight_min": -59.80973815917969,
      "activations/layer7_attention_weight_max": 79.68894958496094,
      "activations/layer7_attention_weight_min": -79.0651626586914,
      "activations/layer8_attention_weight_max": 41.19841766357422,
      "activations/layer8_attention_weight_min": -37.47877883911133,
      "activations/layer9_attention_weight_max": 39.75651168823242,
      "activations/layer9_attention_weight_min": -34.650089263916016,
      "epoch": 4.32,
      "learning_rate": 0.0001233810606060606,
      "loss": 2.8555,
      "step": 74350
    },
    {
      "activations/layer0_attention_weight_max": 16.66023826599121,
      "activations/layer0_attention_weight_min": -11.731338500976562,
      "activations/layer10_attention_weight_max": 39.22310256958008,
      "activations/layer10_attention_weight_min": -29.02245330810547,
      "activations/layer11_attention_weight_max": 35.479957580566406,
      "activations/layer11_attention_weight_min": -26.43075942993164,
      "activations/layer12_attention_weight_max": 28.6372013092041,
      "activations/layer12_attention_weight_min": -28.275875091552734,
      "activations/layer13_attention_weight_max": 55.18474578857422,
      "activations/layer13_attention_weight_min": -33.458797454833984,
      "activations/layer14_attention_weight_max": 69.55558013916016,
      "activations/layer14_attention_weight_min": -49.6284065246582,
      "activations/layer15_attention_weight_max": 46.25234603881836,
      "activations/layer15_attention_weight_min": -27.711496353149414,
      "activations/layer16_attention_weight_max": 43.35310363769531,
      "activations/layer16_attention_weight_min": -28.954580307006836,
      "activations/layer17_attention_weight_max": 52.59086227416992,
      "activations/layer17_attention_weight_min": -24.531248092651367,
      "activations/layer18_attention_weight_max": 37.641143798828125,
      "activations/layer18_attention_weight_min": -20.73761558532715,
      "activations/layer19_attention_weight_max": 42.33644104003906,
      "activations/layer19_attention_weight_min": -31.79962730407715,
      "activations/layer1_attention_weight_max": 17.893421173095703,
      "activations/layer1_attention_weight_min": -13.646344184875488,
      "activations/layer20_attention_weight_max": 35.680301666259766,
      "activations/layer20_attention_weight_min": -23.321887969970703,
      "activations/layer21_attention_weight_max": 36.26690673828125,
      "activations/layer21_attention_weight_min": -22.292831420898438,
      "activations/layer22_attention_weight_max": 52.07740020751953,
      "activations/layer22_attention_weight_min": -29.09177017211914,
      "activations/layer23_attention_weight_max": 32.985809326171875,
      "activations/layer23_attention_weight_min": -21.668190002441406,
      "activations/layer2_attention_weight_max": 33.1078987121582,
      "activations/layer2_attention_weight_min": -30.22467803955078,
      "activations/layer3_attention_weight_max": 85.19212341308594,
      "activations/layer3_attention_weight_min": -87.58560180664062,
      "activations/layer4_attention_weight_max": 129.7161102294922,
      "activations/layer4_attention_weight_min": -121.83988189697266,
      "activations/layer5_attention_weight_max": 87.1922607421875,
      "activations/layer5_attention_weight_min": -81.19420623779297,
      "activations/layer6_attention_weight_max": 59.87077331542969,
      "activations/layer6_attention_weight_min": -57.31188201904297,
      "activations/layer7_attention_weight_max": 80.62989044189453,
      "activations/layer7_attention_weight_min": -79.42570495605469,
      "activations/layer8_attention_weight_max": 41.20058059692383,
      "activations/layer8_attention_weight_min": -37.44887161254883,
      "activations/layer9_attention_weight_max": 38.603912353515625,
      "activations/layer9_attention_weight_min": -32.38251876831055,
      "epoch": 4.32,
      "learning_rate": 0.0001233621212121212,
      "loss": 2.8733,
      "step": 74400
    },
    {
      "activations/layer0_attention_weight_max": 14.820333480834961,
      "activations/layer0_attention_weight_min": -12.395888328552246,
      "activations/layer10_attention_weight_max": 35.23881530761719,
      "activations/layer10_attention_weight_min": -30.64079475402832,
      "activations/layer11_attention_weight_max": 36.86767578125,
      "activations/layer11_attention_weight_min": -28.771190643310547,
      "activations/layer12_attention_weight_max": 33.70259475708008,
      "activations/layer12_attention_weight_min": -28.07369613647461,
      "activations/layer13_attention_weight_max": 50.78963851928711,
      "activations/layer13_attention_weight_min": -34.457637786865234,
      "activations/layer14_attention_weight_max": 69.27994537353516,
      "activations/layer14_attention_weight_min": -48.30341339111328,
      "activations/layer15_attention_weight_max": 45.814029693603516,
      "activations/layer15_attention_weight_min": -32.27849578857422,
      "activations/layer16_attention_weight_max": 42.47502136230469,
      "activations/layer16_attention_weight_min": -27.325271606445312,
      "activations/layer17_attention_weight_max": 42.74216842651367,
      "activations/layer17_attention_weight_min": -24.786087036132812,
      "activations/layer18_attention_weight_max": 30.713045120239258,
      "activations/layer18_attention_weight_min": -19.409317016601562,
      "activations/layer19_attention_weight_max": 47.861419677734375,
      "activations/layer19_attention_weight_min": -28.213998794555664,
      "activations/layer1_attention_weight_max": 17.76917266845703,
      "activations/layer1_attention_weight_min": -17.142772674560547,
      "activations/layer20_attention_weight_max": 41.48710250854492,
      "activations/layer20_attention_weight_min": -22.313390731811523,
      "activations/layer21_attention_weight_max": 28.478618621826172,
      "activations/layer21_attention_weight_min": -18.919370651245117,
      "activations/layer22_attention_weight_max": 48.36924743652344,
      "activations/layer22_attention_weight_min": -27.269487380981445,
      "activations/layer23_attention_weight_max": 30.665904998779297,
      "activations/layer23_attention_weight_min": -18.14821434020996,
      "activations/layer2_attention_weight_max": 31.532913208007812,
      "activations/layer2_attention_weight_min": -30.73786163330078,
      "activations/layer3_attention_weight_max": 88.09262084960938,
      "activations/layer3_attention_weight_min": -87.61949157714844,
      "activations/layer4_attention_weight_max": 144.0886688232422,
      "activations/layer4_attention_weight_min": -129.3153076171875,
      "activations/layer5_attention_weight_max": 90.28924560546875,
      "activations/layer5_attention_weight_min": -84.70645141601562,
      "activations/layer6_attention_weight_max": 59.626808166503906,
      "activations/layer6_attention_weight_min": -57.240867614746094,
      "activations/layer7_attention_weight_max": 81.85160064697266,
      "activations/layer7_attention_weight_min": -82.49340057373047,
      "activations/layer8_attention_weight_max": 40.204185485839844,
      "activations/layer8_attention_weight_min": -38.99702072143555,
      "activations/layer9_attention_weight_max": 38.19382095336914,
      "activations/layer9_attention_weight_min": -33.1187629699707,
      "epoch": 4.33,
      "learning_rate": 0.0001233431818181818,
      "loss": 2.8473,
      "step": 74450
    },
    {
      "activations/layer0_attention_weight_max": 16.95408058166504,
      "activations/layer0_attention_weight_min": -12.421278953552246,
      "activations/layer10_attention_weight_max": 31.616361618041992,
      "activations/layer10_attention_weight_min": -29.764965057373047,
      "activations/layer11_attention_weight_max": 31.9620418548584,
      "activations/layer11_attention_weight_min": -26.887603759765625,
      "activations/layer12_attention_weight_max": 27.17058563232422,
      "activations/layer12_attention_weight_min": -29.140644073486328,
      "activations/layer13_attention_weight_max": 41.660911560058594,
      "activations/layer13_attention_weight_min": -33.21525192260742,
      "activations/layer14_attention_weight_max": 50.994606018066406,
      "activations/layer14_attention_weight_min": -40.490386962890625,
      "activations/layer15_attention_weight_max": 38.704044342041016,
      "activations/layer15_attention_weight_min": -28.98240089416504,
      "activations/layer16_attention_weight_max": 39.08686447143555,
      "activations/layer16_attention_weight_min": -29.272523880004883,
      "activations/layer17_attention_weight_max": 35.18376541137695,
      "activations/layer17_attention_weight_min": -24.058198928833008,
      "activations/layer18_attention_weight_max": 30.9191837310791,
      "activations/layer18_attention_weight_min": -20.646976470947266,
      "activations/layer19_attention_weight_max": 44.31291961669922,
      "activations/layer19_attention_weight_min": -29.416400909423828,
      "activations/layer1_attention_weight_max": 17.428796768188477,
      "activations/layer1_attention_weight_min": -15.173165321350098,
      "activations/layer20_attention_weight_max": 31.82384490966797,
      "activations/layer20_attention_weight_min": -23.959959030151367,
      "activations/layer21_attention_weight_max": 27.611534118652344,
      "activations/layer21_attention_weight_min": -19.78876304626465,
      "activations/layer22_attention_weight_max": 46.20138931274414,
      "activations/layer22_attention_weight_min": -27.34000587463379,
      "activations/layer23_attention_weight_max": 32.33702850341797,
      "activations/layer23_attention_weight_min": -17.654884338378906,
      "activations/layer2_attention_weight_max": 31.464567184448242,
      "activations/layer2_attention_weight_min": -30.83914566040039,
      "activations/layer3_attention_weight_max": 86.86078643798828,
      "activations/layer3_attention_weight_min": -86.06524658203125,
      "activations/layer4_attention_weight_max": 124.1171646118164,
      "activations/layer4_attention_weight_min": -124.03730773925781,
      "activations/layer5_attention_weight_max": 82.89850616455078,
      "activations/layer5_attention_weight_min": -80.43557739257812,
      "activations/layer6_attention_weight_max": 58.857059478759766,
      "activations/layer6_attention_weight_min": -58.215606689453125,
      "activations/layer7_attention_weight_max": 77.30317687988281,
      "activations/layer7_attention_weight_min": -79.37906646728516,
      "activations/layer8_attention_weight_max": 37.357975006103516,
      "activations/layer8_attention_weight_min": -38.011871337890625,
      "activations/layer9_attention_weight_max": 39.76248550415039,
      "activations/layer9_attention_weight_min": -42.05195999145508,
      "epoch": 4.33,
      "learning_rate": 0.00012332424242424242,
      "loss": 2.851,
      "step": 74500
    },
    {
      "activations/layer0_attention_weight_max": 17.04697608947754,
      "activations/layer0_attention_weight_min": -12.441394805908203,
      "activations/layer10_attention_weight_max": 41.7088508605957,
      "activations/layer10_attention_weight_min": -29.702327728271484,
      "activations/layer11_attention_weight_max": 41.38975524902344,
      "activations/layer11_attention_weight_min": -28.631710052490234,
      "activations/layer12_attention_weight_max": 49.974517822265625,
      "activations/layer12_attention_weight_min": -27.347986221313477,
      "activations/layer13_attention_weight_max": 73.9559097290039,
      "activations/layer13_attention_weight_min": -41.55388641357422,
      "activations/layer14_attention_weight_max": 105.14868927001953,
      "activations/layer14_attention_weight_min": -59.976417541503906,
      "activations/layer15_attention_weight_max": 67.71626281738281,
      "activations/layer15_attention_weight_min": -34.86933135986328,
      "activations/layer16_attention_weight_max": 45.596702575683594,
      "activations/layer16_attention_weight_min": -28.799184799194336,
      "activations/layer17_attention_weight_max": 53.01778030395508,
      "activations/layer17_attention_weight_min": -27.77598762512207,
      "activations/layer18_attention_weight_max": 41.63105773925781,
      "activations/layer18_attention_weight_min": -20.17053985595703,
      "activations/layer19_attention_weight_max": 44.792171478271484,
      "activations/layer19_attention_weight_min": -29.18338966369629,
      "activations/layer1_attention_weight_max": 17.23615074157715,
      "activations/layer1_attention_weight_min": -14.08327579498291,
      "activations/layer20_attention_weight_max": 31.740520477294922,
      "activations/layer20_attention_weight_min": -22.11846923828125,
      "activations/layer21_attention_weight_max": 31.902427673339844,
      "activations/layer21_attention_weight_min": -20.419057846069336,
      "activations/layer22_attention_weight_max": 44.9593620300293,
      "activations/layer22_attention_weight_min": -25.85232162475586,
      "activations/layer23_attention_weight_max": 31.197538375854492,
      "activations/layer23_attention_weight_min": -16.938173294067383,
      "activations/layer2_attention_weight_max": 32.03691101074219,
      "activations/layer2_attention_weight_min": -31.443960189819336,
      "activations/layer3_attention_weight_max": 84.55730438232422,
      "activations/layer3_attention_weight_min": -86.19660186767578,
      "activations/layer4_attention_weight_max": 129.7263641357422,
      "activations/layer4_attention_weight_min": -128.4075164794922,
      "activations/layer5_attention_weight_max": 87.252197265625,
      "activations/layer5_attention_weight_min": -84.07864379882812,
      "activations/layer6_attention_weight_max": 61.712493896484375,
      "activations/layer6_attention_weight_min": -60.78629684448242,
      "activations/layer7_attention_weight_max": 84.39907836914062,
      "activations/layer7_attention_weight_min": -80.18437194824219,
      "activations/layer8_attention_weight_max": 42.4872932434082,
      "activations/layer8_attention_weight_min": -38.786903381347656,
      "activations/layer9_attention_weight_max": 39.92119598388672,
      "activations/layer9_attention_weight_min": -40.145118713378906,
      "epoch": 4.33,
      "learning_rate": 0.000123305303030303,
      "loss": 2.8659,
      "step": 74550
    },
    {
      "activations/layer0_attention_weight_max": 16.34336280822754,
      "activations/layer0_attention_weight_min": -11.773509979248047,
      "activations/layer10_attention_weight_max": 37.4507942199707,
      "activations/layer10_attention_weight_min": -29.61910629272461,
      "activations/layer11_attention_weight_max": 38.45824432373047,
      "activations/layer11_attention_weight_min": -27.17082977294922,
      "activations/layer12_attention_weight_max": 40.67034149169922,
      "activations/layer12_attention_weight_min": -28.45321273803711,
      "activations/layer13_attention_weight_max": 54.76737594604492,
      "activations/layer13_attention_weight_min": -36.925045013427734,
      "activations/layer14_attention_weight_max": 66.56336975097656,
      "activations/layer14_attention_weight_min": -47.03961944580078,
      "activations/layer15_attention_weight_max": 42.92393112182617,
      "activations/layer15_attention_weight_min": -30.443124771118164,
      "activations/layer16_attention_weight_max": 39.251197814941406,
      "activations/layer16_attention_weight_min": -28.131141662597656,
      "activations/layer17_attention_weight_max": 42.03030776977539,
      "activations/layer17_attention_weight_min": -24.832443237304688,
      "activations/layer18_attention_weight_max": 32.713680267333984,
      "activations/layer18_attention_weight_min": -21.297441482543945,
      "activations/layer19_attention_weight_max": 39.60401153564453,
      "activations/layer19_attention_weight_min": -28.84571647644043,
      "activations/layer1_attention_weight_max": 17.96820831298828,
      "activations/layer1_attention_weight_min": -14.045454025268555,
      "activations/layer20_attention_weight_max": 34.935882568359375,
      "activations/layer20_attention_weight_min": -23.589305877685547,
      "activations/layer21_attention_weight_max": 31.652446746826172,
      "activations/layer21_attention_weight_min": -20.881378173828125,
      "activations/layer22_attention_weight_max": 49.06536102294922,
      "activations/layer22_attention_weight_min": -27.241607666015625,
      "activations/layer23_attention_weight_max": 30.767059326171875,
      "activations/layer23_attention_weight_min": -19.98676300048828,
      "activations/layer2_attention_weight_max": 31.186447143554688,
      "activations/layer2_attention_weight_min": -31.765342712402344,
      "activations/layer3_attention_weight_max": 88.17523193359375,
      "activations/layer3_attention_weight_min": -88.66796875,
      "activations/layer4_attention_weight_max": 128.00384521484375,
      "activations/layer4_attention_weight_min": -126.9561538696289,
      "activations/layer5_attention_weight_max": 85.45897674560547,
      "activations/layer5_attention_weight_min": -82.68531799316406,
      "activations/layer6_attention_weight_max": 55.4707145690918,
      "activations/layer6_attention_weight_min": -55.93608474731445,
      "activations/layer7_attention_weight_max": 81.45440673828125,
      "activations/layer7_attention_weight_min": -76.78142547607422,
      "activations/layer8_attention_weight_max": 41.10150909423828,
      "activations/layer8_attention_weight_min": -36.95711135864258,
      "activations/layer9_attention_weight_max": 45.88456344604492,
      "activations/layer9_attention_weight_min": -34.57436752319336,
      "epoch": 4.33,
      "learning_rate": 0.00012328636363636363,
      "loss": 2.8592,
      "step": 74600
    },
    {
      "activations/layer0_attention_weight_max": 16.664777755737305,
      "activations/layer0_attention_weight_min": -11.977471351623535,
      "activations/layer10_attention_weight_max": 34.16676712036133,
      "activations/layer10_attention_weight_min": -31.818265914916992,
      "activations/layer11_attention_weight_max": 32.7461051940918,
      "activations/layer11_attention_weight_min": -28.880626678466797,
      "activations/layer12_attention_weight_max": 27.00041961669922,
      "activations/layer12_attention_weight_min": -27.55735969543457,
      "activations/layer13_attention_weight_max": 48.531070709228516,
      "activations/layer13_attention_weight_min": -32.43030548095703,
      "activations/layer14_attention_weight_max": 61.58853530883789,
      "activations/layer14_attention_weight_min": -42.02375030517578,
      "activations/layer15_attention_weight_max": 41.182106018066406,
      "activations/layer15_attention_weight_min": -28.46571922302246,
      "activations/layer16_attention_weight_max": 35.57129669189453,
      "activations/layer16_attention_weight_min": -28.733173370361328,
      "activations/layer17_attention_weight_max": 42.417198181152344,
      "activations/layer17_attention_weight_min": -23.848987579345703,
      "activations/layer18_attention_weight_max": 31.834857940673828,
      "activations/layer18_attention_weight_min": -21.573095321655273,
      "activations/layer19_attention_weight_max": 39.0756950378418,
      "activations/layer19_attention_weight_min": -34.10325241088867,
      "activations/layer1_attention_weight_max": 17.205982208251953,
      "activations/layer1_attention_weight_min": -15.411240577697754,
      "activations/layer20_attention_weight_max": 32.337032318115234,
      "activations/layer20_attention_weight_min": -23.42111587524414,
      "activations/layer21_attention_weight_max": 32.27029037475586,
      "activations/layer21_attention_weight_min": -19.045063018798828,
      "activations/layer22_attention_weight_max": 54.89435577392578,
      "activations/layer22_attention_weight_min": -27.382986068725586,
      "activations/layer23_attention_weight_max": 32.396400451660156,
      "activations/layer23_attention_weight_min": -19.58438491821289,
      "activations/layer2_attention_weight_max": 33.204559326171875,
      "activations/layer2_attention_weight_min": -32.298866271972656,
      "activations/layer3_attention_weight_max": 88.51376342773438,
      "activations/layer3_attention_weight_min": -90.75646209716797,
      "activations/layer4_attention_weight_max": 132.5830535888672,
      "activations/layer4_attention_weight_min": -126.65386199951172,
      "activations/layer5_attention_weight_max": 86.81941223144531,
      "activations/layer5_attention_weight_min": -83.69807434082031,
      "activations/layer6_attention_weight_max": 57.7224235534668,
      "activations/layer6_attention_weight_min": -58.10192108154297,
      "activations/layer7_attention_weight_max": 74.58745574951172,
      "activations/layer7_attention_weight_min": -83.2298355102539,
      "activations/layer8_attention_weight_max": 37.74079132080078,
      "activations/layer8_attention_weight_min": -39.23914337158203,
      "activations/layer9_attention_weight_max": 36.77303695678711,
      "activations/layer9_attention_weight_min": -34.62919998168945,
      "epoch": 4.34,
      "learning_rate": 0.00012326742424242424,
      "loss": 2.8725,
      "step": 74650
    },
    {
      "activations/layer0_attention_weight_max": 16.492752075195312,
      "activations/layer0_attention_weight_min": -12.526174545288086,
      "activations/layer10_attention_weight_max": 35.45781326293945,
      "activations/layer10_attention_weight_min": -32.20064163208008,
      "activations/layer11_attention_weight_max": 33.34977340698242,
      "activations/layer11_attention_weight_min": -27.321325302124023,
      "activations/layer12_attention_weight_max": 26.835718154907227,
      "activations/layer12_attention_weight_min": -28.881147384643555,
      "activations/layer13_attention_weight_max": 45.968013763427734,
      "activations/layer13_attention_weight_min": -34.230072021484375,
      "activations/layer14_attention_weight_max": 64.93995666503906,
      "activations/layer14_attention_weight_min": -48.634220123291016,
      "activations/layer15_attention_weight_max": 37.595603942871094,
      "activations/layer15_attention_weight_min": -26.14769172668457,
      "activations/layer16_attention_weight_max": 38.726341247558594,
      "activations/layer16_attention_weight_min": -26.741235733032227,
      "activations/layer17_attention_weight_max": 37.2996711730957,
      "activations/layer17_attention_weight_min": -24.1347599029541,
      "activations/layer18_attention_weight_max": 31.419025421142578,
      "activations/layer18_attention_weight_min": -20.900266647338867,
      "activations/layer19_attention_weight_max": 40.10869598388672,
      "activations/layer19_attention_weight_min": -28.693973541259766,
      "activations/layer1_attention_weight_max": 17.203163146972656,
      "activations/layer1_attention_weight_min": -16.032400131225586,
      "activations/layer20_attention_weight_max": 31.982027053833008,
      "activations/layer20_attention_weight_min": -25.970008850097656,
      "activations/layer21_attention_weight_max": 32.983734130859375,
      "activations/layer21_attention_weight_min": -20.41803741455078,
      "activations/layer22_attention_weight_max": 48.46480178833008,
      "activations/layer22_attention_weight_min": -28.7877140045166,
      "activations/layer23_attention_weight_max": 31.80310821533203,
      "activations/layer23_attention_weight_min": -18.030345916748047,
      "activations/layer2_attention_weight_max": 32.85530090332031,
      "activations/layer2_attention_weight_min": -30.737407684326172,
      "activations/layer3_attention_weight_max": 88.74747467041016,
      "activations/layer3_attention_weight_min": -90.00608825683594,
      "activations/layer4_attention_weight_max": 138.05393981933594,
      "activations/layer4_attention_weight_min": -137.55224609375,
      "activations/layer5_attention_weight_max": 90.3403549194336,
      "activations/layer5_attention_weight_min": -87.64297485351562,
      "activations/layer6_attention_weight_max": 61.835182189941406,
      "activations/layer6_attention_weight_min": -58.69694137573242,
      "activations/layer7_attention_weight_max": 81.45292663574219,
      "activations/layer7_attention_weight_min": -82.90672302246094,
      "activations/layer8_attention_weight_max": 41.5788459777832,
      "activations/layer8_attention_weight_min": -42.84123611450195,
      "activations/layer9_attention_weight_max": 38.157989501953125,
      "activations/layer9_attention_weight_min": -36.093223571777344,
      "epoch": 4.34,
      "learning_rate": 0.00012324848484848483,
      "loss": 2.8657,
      "step": 74700
    },
    {
      "activations/layer0_attention_weight_max": 17.417137145996094,
      "activations/layer0_attention_weight_min": -11.461663246154785,
      "activations/layer10_attention_weight_max": 32.42216873168945,
      "activations/layer10_attention_weight_min": -33.030330657958984,
      "activations/layer11_attention_weight_max": 29.439109802246094,
      "activations/layer11_attention_weight_min": -27.920108795166016,
      "activations/layer12_attention_weight_max": 27.441593170166016,
      "activations/layer12_attention_weight_min": -27.860095977783203,
      "activations/layer13_attention_weight_max": 40.48271942138672,
      "activations/layer13_attention_weight_min": -32.227169036865234,
      "activations/layer14_attention_weight_max": 54.610618591308594,
      "activations/layer14_attention_weight_min": -43.933387756347656,
      "activations/layer15_attention_weight_max": 43.650821685791016,
      "activations/layer15_attention_weight_min": -27.211658477783203,
      "activations/layer16_attention_weight_max": 41.752445220947266,
      "activations/layer16_attention_weight_min": -29.843433380126953,
      "activations/layer17_attention_weight_max": 37.173770904541016,
      "activations/layer17_attention_weight_min": -25.112594604492188,
      "activations/layer18_attention_weight_max": 35.57510757446289,
      "activations/layer18_attention_weight_min": -21.110307693481445,
      "activations/layer19_attention_weight_max": 38.42618179321289,
      "activations/layer19_attention_weight_min": -30.205698013305664,
      "activations/layer1_attention_weight_max": 16.43096351623535,
      "activations/layer1_attention_weight_min": -14.41176986694336,
      "activations/layer20_attention_weight_max": 30.648818969726562,
      "activations/layer20_attention_weight_min": -23.648027420043945,
      "activations/layer21_attention_weight_max": 35.617759704589844,
      "activations/layer21_attention_weight_min": -20.540790557861328,
      "activations/layer22_attention_weight_max": 48.21863555908203,
      "activations/layer22_attention_weight_min": -25.216230392456055,
      "activations/layer23_attention_weight_max": 31.835174560546875,
      "activations/layer23_attention_weight_min": -19.420082092285156,
      "activations/layer2_attention_weight_max": 33.98725128173828,
      "activations/layer2_attention_weight_min": -31.83034324645996,
      "activations/layer3_attention_weight_max": 87.55011749267578,
      "activations/layer3_attention_weight_min": -92.85110473632812,
      "activations/layer4_attention_weight_max": 134.0120086669922,
      "activations/layer4_attention_weight_min": -130.5633544921875,
      "activations/layer5_attention_weight_max": 89.29862976074219,
      "activations/layer5_attention_weight_min": -89.08678436279297,
      "activations/layer6_attention_weight_max": 58.04375457763672,
      "activations/layer6_attention_weight_min": -58.59912872314453,
      "activations/layer7_attention_weight_max": 80.94426727294922,
      "activations/layer7_attention_weight_min": -81.3626480102539,
      "activations/layer8_attention_weight_max": 40.568931579589844,
      "activations/layer8_attention_weight_min": -41.05928039550781,
      "activations/layer9_attention_weight_max": 46.1630744934082,
      "activations/layer9_attention_weight_min": -43.33887481689453,
      "epoch": 4.34,
      "learning_rate": 0.00012322954545454544,
      "loss": 2.8579,
      "step": 74750
    },
    {
      "activations/layer0_attention_weight_max": 16.71847915649414,
      "activations/layer0_attention_weight_min": -13.503341674804688,
      "activations/layer10_attention_weight_max": 32.67315673828125,
      "activations/layer10_attention_weight_min": -30.27037239074707,
      "activations/layer11_attention_weight_max": 32.78734588623047,
      "activations/layer11_attention_weight_min": -27.389860153198242,
      "activations/layer12_attention_weight_max": 28.320390701293945,
      "activations/layer12_attention_weight_min": -26.27100372314453,
      "activations/layer13_attention_weight_max": 41.76707458496094,
      "activations/layer13_attention_weight_min": -33.76689910888672,
      "activations/layer14_attention_weight_max": 72.0271224975586,
      "activations/layer14_attention_weight_min": -49.11122131347656,
      "activations/layer15_attention_weight_max": 38.703582763671875,
      "activations/layer15_attention_weight_min": -29.825960159301758,
      "activations/layer16_attention_weight_max": 35.67805480957031,
      "activations/layer16_attention_weight_min": -31.403898239135742,
      "activations/layer17_attention_weight_max": 40.183494567871094,
      "activations/layer17_attention_weight_min": -25.27351951599121,
      "activations/layer18_attention_weight_max": 32.085025787353516,
      "activations/layer18_attention_weight_min": -20.16869354248047,
      "activations/layer19_attention_weight_max": 45.0513801574707,
      "activations/layer19_attention_weight_min": -29.511154174804688,
      "activations/layer1_attention_weight_max": 17.110260009765625,
      "activations/layer1_attention_weight_min": -13.917284965515137,
      "activations/layer20_attention_weight_max": 31.315738677978516,
      "activations/layer20_attention_weight_min": -22.928245544433594,
      "activations/layer21_attention_weight_max": 34.507598876953125,
      "activations/layer21_attention_weight_min": -19.791217803955078,
      "activations/layer22_attention_weight_max": 50.60773468017578,
      "activations/layer22_attention_weight_min": -29.035581588745117,
      "activations/layer23_attention_weight_max": 31.129127502441406,
      "activations/layer23_attention_weight_min": -18.243911743164062,
      "activations/layer2_attention_weight_max": 34.34159851074219,
      "activations/layer2_attention_weight_min": -32.323692321777344,
      "activations/layer3_attention_weight_max": 95.05812072753906,
      "activations/layer3_attention_weight_min": -94.01885986328125,
      "activations/layer4_attention_weight_max": 139.7078094482422,
      "activations/layer4_attention_weight_min": -132.1819610595703,
      "activations/layer5_attention_weight_max": 95.2662582397461,
      "activations/layer5_attention_weight_min": -83.13314819335938,
      "activations/layer6_attention_weight_max": 59.490318298339844,
      "activations/layer6_attention_weight_min": -57.45990753173828,
      "activations/layer7_attention_weight_max": 79.64750671386719,
      "activations/layer7_attention_weight_min": -82.74229431152344,
      "activations/layer8_attention_weight_max": 39.45703125,
      "activations/layer8_attention_weight_min": -36.84182357788086,
      "activations/layer9_attention_weight_max": 37.59151077270508,
      "activations/layer9_attention_weight_min": -33.57716369628906,
      "epoch": 4.35,
      "learning_rate": 0.00012321060606060603,
      "loss": 2.8895,
      "step": 74800
    },
    {
      "activations/layer0_attention_weight_max": 16.775537490844727,
      "activations/layer0_attention_weight_min": -11.4401216506958,
      "activations/layer10_attention_weight_max": 47.258811950683594,
      "activations/layer10_attention_weight_min": -33.35783004760742,
      "activations/layer11_attention_weight_max": 53.10688400268555,
      "activations/layer11_attention_weight_min": -34.00019073486328,
      "activations/layer12_attention_weight_max": 78.99853515625,
      "activations/layer12_attention_weight_min": -30.575639724731445,
      "activations/layer13_attention_weight_max": 81.87288665771484,
      "activations/layer13_attention_weight_min": -41.120216369628906,
      "activations/layer14_attention_weight_max": 88.15354919433594,
      "activations/layer14_attention_weight_min": -50.81265640258789,
      "activations/layer15_attention_weight_max": 46.43040084838867,
      "activations/layer15_attention_weight_min": -29.529489517211914,
      "activations/layer16_attention_weight_max": 44.01856231689453,
      "activations/layer16_attention_weight_min": -31.36717414855957,
      "activations/layer17_attention_weight_max": 39.98262405395508,
      "activations/layer17_attention_weight_min": -25.07205581665039,
      "activations/layer18_attention_weight_max": 31.643009185791016,
      "activations/layer18_attention_weight_min": -20.359025955200195,
      "activations/layer19_attention_weight_max": 41.470184326171875,
      "activations/layer19_attention_weight_min": -30.068817138671875,
      "activations/layer1_attention_weight_max": 17.679723739624023,
      "activations/layer1_attention_weight_min": -15.409879684448242,
      "activations/layer20_attention_weight_max": 31.654027938842773,
      "activations/layer20_attention_weight_min": -21.81595230102539,
      "activations/layer21_attention_weight_max": 33.23388671875,
      "activations/layer21_attention_weight_min": -20.628772735595703,
      "activations/layer22_attention_weight_max": 48.21108627319336,
      "activations/layer22_attention_weight_min": -28.04314422607422,
      "activations/layer23_attention_weight_max": 31.596525192260742,
      "activations/layer23_attention_weight_min": -18.683361053466797,
      "activations/layer2_attention_weight_max": 34.784889221191406,
      "activations/layer2_attention_weight_min": -34.620609283447266,
      "activations/layer3_attention_weight_max": 94.4834213256836,
      "activations/layer3_attention_weight_min": -98.96469116210938,
      "activations/layer4_attention_weight_max": 145.7637176513672,
      "activations/layer4_attention_weight_min": -136.84182739257812,
      "activations/layer5_attention_weight_max": 93.91043090820312,
      "activations/layer5_attention_weight_min": -87.832275390625,
      "activations/layer6_attention_weight_max": 61.26163864135742,
      "activations/layer6_attention_weight_min": -60.756195068359375,
      "activations/layer7_attention_weight_max": 97.93896484375,
      "activations/layer7_attention_weight_min": -86.95440673828125,
      "activations/layer8_attention_weight_max": 50.901885986328125,
      "activations/layer8_attention_weight_min": -41.9814338684082,
      "activations/layer9_attention_weight_max": 45.73529052734375,
      "activations/layer9_attention_weight_min": -41.01076126098633,
      "epoch": 4.35,
      "learning_rate": 0.00012319166666666665,
      "loss": 2.8699,
      "step": 74850
    },
    {
      "activations/layer0_attention_weight_max": 17.018922805786133,
      "activations/layer0_attention_weight_min": -12.890799522399902,
      "activations/layer10_attention_weight_max": 32.86384582519531,
      "activations/layer10_attention_weight_min": -28.908145904541016,
      "activations/layer11_attention_weight_max": 31.02927017211914,
      "activations/layer11_attention_weight_min": -27.394941329956055,
      "activations/layer12_attention_weight_max": 28.920913696289062,
      "activations/layer12_attention_weight_min": -26.319629669189453,
      "activations/layer13_attention_weight_max": 39.31978225708008,
      "activations/layer13_attention_weight_min": -30.619792938232422,
      "activations/layer14_attention_weight_max": 59.04584503173828,
      "activations/layer14_attention_weight_min": -47.72890853881836,
      "activations/layer15_attention_weight_max": 43.487823486328125,
      "activations/layer15_attention_weight_min": -28.755382537841797,
      "activations/layer16_attention_weight_max": 38.809661865234375,
      "activations/layer16_attention_weight_min": -28.11135482788086,
      "activations/layer17_attention_weight_max": 38.50243377685547,
      "activations/layer17_attention_weight_min": -25.94057273864746,
      "activations/layer18_attention_weight_max": 31.171911239624023,
      "activations/layer18_attention_weight_min": -21.296855926513672,
      "activations/layer19_attention_weight_max": 40.21702194213867,
      "activations/layer19_attention_weight_min": -29.568479537963867,
      "activations/layer1_attention_weight_max": 17.62028694152832,
      "activations/layer1_attention_weight_min": -14.991907119750977,
      "activations/layer20_attention_weight_max": 33.452877044677734,
      "activations/layer20_attention_weight_min": -24.378459930419922,
      "activations/layer21_attention_weight_max": 31.358171463012695,
      "activations/layer21_attention_weight_min": -21.740880966186523,
      "activations/layer22_attention_weight_max": 47.5134162902832,
      "activations/layer22_attention_weight_min": -27.931447982788086,
      "activations/layer23_attention_weight_max": 28.135318756103516,
      "activations/layer23_attention_weight_min": -19.115558624267578,
      "activations/layer2_attention_weight_max": 32.31657028198242,
      "activations/layer2_attention_weight_min": -30.99365234375,
      "activations/layer3_attention_weight_max": 83.90949249267578,
      "activations/layer3_attention_weight_min": -85.15876007080078,
      "activations/layer4_attention_weight_max": 123.86701965332031,
      "activations/layer4_attention_weight_min": -123.94112396240234,
      "activations/layer5_attention_weight_max": 83.99716186523438,
      "activations/layer5_attention_weight_min": -81.011962890625,
      "activations/layer6_attention_weight_max": 52.367008209228516,
      "activations/layer6_attention_weight_min": -54.141380310058594,
      "activations/layer7_attention_weight_max": 73.49385070800781,
      "activations/layer7_attention_weight_min": -75.81375885009766,
      "activations/layer8_attention_weight_max": 36.953407287597656,
      "activations/layer8_attention_weight_min": -36.30258560180664,
      "activations/layer9_attention_weight_max": 32.87343978881836,
      "activations/layer9_attention_weight_min": -31.36162757873535,
      "epoch": 4.35,
      "learning_rate": 0.00012317272727272726,
      "loss": 2.8675,
      "step": 74900
    },
    {
      "activations/layer0_attention_weight_max": 14.363444328308105,
      "activations/layer0_attention_weight_min": -11.915703773498535,
      "activations/layer10_attention_weight_max": 35.9083366394043,
      "activations/layer10_attention_weight_min": -30.625701904296875,
      "activations/layer11_attention_weight_max": 37.51247024536133,
      "activations/layer11_attention_weight_min": -28.752531051635742,
      "activations/layer12_attention_weight_max": 59.388397216796875,
      "activations/layer12_attention_weight_min": -28.1994686126709,
      "activations/layer13_attention_weight_max": 54.15837478637695,
      "activations/layer13_attention_weight_min": -36.96488571166992,
      "activations/layer14_attention_weight_max": 60.935752868652344,
      "activations/layer14_attention_weight_min": -44.51829528808594,
      "activations/layer15_attention_weight_max": 48.1546516418457,
      "activations/layer15_attention_weight_min": -33.13035583496094,
      "activations/layer16_attention_weight_max": 41.18280029296875,
      "activations/layer16_attention_weight_min": -28.224475860595703,
      "activations/layer17_attention_weight_max": 39.089813232421875,
      "activations/layer17_attention_weight_min": -24.663585662841797,
      "activations/layer18_attention_weight_max": 29.074012756347656,
      "activations/layer18_attention_weight_min": -21.125524520874023,
      "activations/layer19_attention_weight_max": 43.145565032958984,
      "activations/layer19_attention_weight_min": -28.8516902923584,
      "activations/layer1_attention_weight_max": 17.360490798950195,
      "activations/layer1_attention_weight_min": -15.331704139709473,
      "activations/layer20_attention_weight_max": 32.21946716308594,
      "activations/layer20_attention_weight_min": -23.819002151489258,
      "activations/layer21_attention_weight_max": 30.003402709960938,
      "activations/layer21_attention_weight_min": -20.408201217651367,
      "activations/layer22_attention_weight_max": 51.1573600769043,
      "activations/layer22_attention_weight_min": -29.034164428710938,
      "activations/layer23_attention_weight_max": 29.362197875976562,
      "activations/layer23_attention_weight_min": -17.20895004272461,
      "activations/layer2_attention_weight_max": 31.30615234375,
      "activations/layer2_attention_weight_min": -31.585233688354492,
      "activations/layer3_attention_weight_max": 91.05268859863281,
      "activations/layer3_attention_weight_min": -90.13848876953125,
      "activations/layer4_attention_weight_max": 131.49310302734375,
      "activations/layer4_attention_weight_min": -129.2413787841797,
      "activations/layer5_attention_weight_max": 88.42440795898438,
      "activations/layer5_attention_weight_min": -87.7392807006836,
      "activations/layer6_attention_weight_max": 55.72275161743164,
      "activations/layer6_attention_weight_min": -57.616214752197266,
      "activations/layer7_attention_weight_max": 77.38410949707031,
      "activations/layer7_attention_weight_min": -78.86778259277344,
      "activations/layer8_attention_weight_max": 40.01941680908203,
      "activations/layer8_attention_weight_min": -36.92369079589844,
      "activations/layer9_attention_weight_max": 37.86599349975586,
      "activations/layer9_attention_weight_min": -32.668338775634766,
      "epoch": 4.36,
      "learning_rate": 0.00012315378787878785,
      "loss": 2.8592,
      "step": 74950
    },
    {
      "activations/layer0_attention_weight_max": 14.794753074645996,
      "activations/layer0_attention_weight_min": -11.575689315795898,
      "activations/layer10_attention_weight_max": 37.3670654296875,
      "activations/layer10_attention_weight_min": -29.037858963012695,
      "activations/layer11_attention_weight_max": 33.756011962890625,
      "activations/layer11_attention_weight_min": -26.501537322998047,
      "activations/layer12_attention_weight_max": 28.422449111938477,
      "activations/layer12_attention_weight_min": -25.87128257751465,
      "activations/layer13_attention_weight_max": 46.478675842285156,
      "activations/layer13_attention_weight_min": -31.179222106933594,
      "activations/layer14_attention_weight_max": 67.47074127197266,
      "activations/layer14_attention_weight_min": -50.82416915893555,
      "activations/layer15_attention_weight_max": 43.970298767089844,
      "activations/layer15_attention_weight_min": -29.25516128540039,
      "activations/layer16_attention_weight_max": 41.672489166259766,
      "activations/layer16_attention_weight_min": -28.615602493286133,
      "activations/layer17_attention_weight_max": 48.196571350097656,
      "activations/layer17_attention_weight_min": -25.137929916381836,
      "activations/layer18_attention_weight_max": 34.4037971496582,
      "activations/layer18_attention_weight_min": -20.244380950927734,
      "activations/layer19_attention_weight_max": 47.790767669677734,
      "activations/layer19_attention_weight_min": -29.00971794128418,
      "activations/layer1_attention_weight_max": 17.59442138671875,
      "activations/layer1_attention_weight_min": -15.347283363342285,
      "activations/layer20_attention_weight_max": 39.41647720336914,
      "activations/layer20_attention_weight_min": -23.335596084594727,
      "activations/layer21_attention_weight_max": 39.70897674560547,
      "activations/layer21_attention_weight_min": -20.470178604125977,
      "activations/layer22_attention_weight_max": 52.8275032043457,
      "activations/layer22_attention_weight_min": -30.360898971557617,
      "activations/layer23_attention_weight_max": 33.386653900146484,
      "activations/layer23_attention_weight_min": -21.75680160522461,
      "activations/layer2_attention_weight_max": 33.221927642822266,
      "activations/layer2_attention_weight_min": -31.286518096923828,
      "activations/layer3_attention_weight_max": 85.7601318359375,
      "activations/layer3_attention_weight_min": -87.16918182373047,
      "activations/layer4_attention_weight_max": 131.57882690429688,
      "activations/layer4_attention_weight_min": -124.05370330810547,
      "activations/layer5_attention_weight_max": 87.83007049560547,
      "activations/layer5_attention_weight_min": -81.24635314941406,
      "activations/layer6_attention_weight_max": 56.58765411376953,
      "activations/layer6_attention_weight_min": -57.624332427978516,
      "activations/layer7_attention_weight_max": 75.0402603149414,
      "activations/layer7_attention_weight_min": -81.49678039550781,
      "activations/layer8_attention_weight_max": 37.21379089355469,
      "activations/layer8_attention_weight_min": -37.74169921875,
      "activations/layer9_attention_weight_max": 34.53860855102539,
      "activations/layer9_attention_weight_min": -31.22612953186035,
      "epoch": 4.36,
      "learning_rate": 0.00012313484848484847,
      "loss": 2.863,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_loss": 2.81640625,
      "eval_runtime": 8.4996,
      "eval_samples_per_second": 505.199,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_openwebtext_loss": 2.81640625,
      "eval_openwebtext_ppl": 16.716667072215767,
      "eval_openwebtext_runtime": 8.4996,
      "eval_openwebtext_samples_per_second": 505.199,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_wikitext_loss": 3.068359375,
      "eval_wikitext_ppl": 21.506589466713315,
      "eval_wikitext_runtime": 1.9959,
      "eval_wikitext_samples_per_second": 228.463,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_lambada_loss": 2.740234375,
      "eval_lambada_ppl": 15.490615283867873,
      "eval_lambada_runtime": 9.5774,
      "eval_lambada_samples_per_second": 508.383,
      "step": 75000
    },
    {
      "activations/layer0_attention_weight_max": 16.67085838317871,
      "activations/layer0_attention_weight_min": -12.240275382995605,
      "activations/layer10_attention_weight_max": 34.978973388671875,
      "activations/layer10_attention_weight_min": -30.08388900756836,
      "activations/layer11_attention_weight_max": 34.3423957824707,
      "activations/layer11_attention_weight_min": -27.42182159423828,
      "activations/layer12_attention_weight_max": 30.686628341674805,
      "activations/layer12_attention_weight_min": -29.242774963378906,
      "activations/layer13_attention_weight_max": 42.208778381347656,
      "activations/layer13_attention_weight_min": -32.930519104003906,
      "activations/layer14_attention_weight_max": 65.01012420654297,
      "activations/layer14_attention_weight_min": -48.28760528564453,
      "activations/layer15_attention_weight_max": 43.69982147216797,
      "activations/layer15_attention_weight_min": -30.33481788635254,
      "activations/layer16_attention_weight_max": 38.160667419433594,
      "activations/layer16_attention_weight_min": -28.889965057373047,
      "activations/layer17_attention_weight_max": 38.526123046875,
      "activations/layer17_attention_weight_min": -26.633148193359375,
      "activations/layer18_attention_weight_max": 32.757999420166016,
      "activations/layer18_attention_weight_min": -19.52935218811035,
      "activations/layer19_attention_weight_max": 38.24342346191406,
      "activations/layer19_attention_weight_min": -29.881977081298828,
      "activations/layer1_attention_weight_max": 18.378177642822266,
      "activations/layer1_attention_weight_min": -14.751038551330566,
      "activations/layer20_attention_weight_max": 35.87748718261719,
      "activations/layer20_attention_weight_min": -24.15176010131836,
      "activations/layer21_attention_weight_max": 40.27153778076172,
      "activations/layer21_attention_weight_min": -22.16631317138672,
      "activations/layer22_attention_weight_max": 49.011051177978516,
      "activations/layer22_attention_weight_min": -28.15992546081543,
      "activations/layer23_attention_weight_max": 31.062986373901367,
      "activations/layer23_attention_weight_min": -18.65357208251953,
      "activations/layer2_attention_weight_max": 32.09541320800781,
      "activations/layer2_attention_weight_min": -31.572307586669922,
      "activations/layer3_attention_weight_max": 84.16210174560547,
      "activations/layer3_attention_weight_min": -85.59347534179688,
      "activations/layer4_attention_weight_max": 126.9594955444336,
      "activations/layer4_attention_weight_min": -119.1803970336914,
      "activations/layer5_attention_weight_max": 83.96460723876953,
      "activations/layer5_attention_weight_min": -82.43585205078125,
      "activations/layer6_attention_weight_max": 54.270015716552734,
      "activations/layer6_attention_weight_min": -55.43568420410156,
      "activations/layer7_attention_weight_max": 78.75438690185547,
      "activations/layer7_attention_weight_min": -78.92257690429688,
      "activations/layer8_attention_weight_max": 39.1722297668457,
      "activations/layer8_attention_weight_min": -38.21866989135742,
      "activations/layer9_attention_weight_max": 36.89228057861328,
      "activations/layer9_attention_weight_min": -33.491294860839844,
      "epoch": 4.36,
      "learning_rate": 0.00012311590909090908,
      "loss": 2.8683,
      "step": 75050
    },
    {
      "activations/layer0_attention_weight_max": 16.42266845703125,
      "activations/layer0_attention_weight_min": -12.42556095123291,
      "activations/layer10_attention_weight_max": 36.83182144165039,
      "activations/layer10_attention_weight_min": -28.798295974731445,
      "activations/layer11_attention_weight_max": 35.538429260253906,
      "activations/layer11_attention_weight_min": -27.342008590698242,
      "activations/layer12_attention_weight_max": 40.35483169555664,
      "activations/layer12_attention_weight_min": -27.821969985961914,
      "activations/layer13_attention_weight_max": 47.59574508666992,
      "activations/layer13_attention_weight_min": -34.20010757446289,
      "activations/layer14_attention_weight_max": 74.50245666503906,
      "activations/layer14_attention_weight_min": -51.759281158447266,
      "activations/layer15_attention_weight_max": 41.231910705566406,
      "activations/layer15_attention_weight_min": -29.27299690246582,
      "activations/layer16_attention_weight_max": 40.10448455810547,
      "activations/layer16_attention_weight_min": -28.76698875427246,
      "activations/layer17_attention_weight_max": 42.09568786621094,
      "activations/layer17_attention_weight_min": -26.768356323242188,
      "activations/layer18_attention_weight_max": 30.197362899780273,
      "activations/layer18_attention_weight_min": -22.7088680267334,
      "activations/layer19_attention_weight_max": 47.044219970703125,
      "activations/layer19_attention_weight_min": -28.98728370666504,
      "activations/layer1_attention_weight_max": 17.049560546875,
      "activations/layer1_attention_weight_min": -14.049034118652344,
      "activations/layer20_attention_weight_max": 34.967655181884766,
      "activations/layer20_attention_weight_min": -23.08078956604004,
      "activations/layer21_attention_weight_max": 29.94973373413086,
      "activations/layer21_attention_weight_min": -21.343950271606445,
      "activations/layer22_attention_weight_max": 49.60409927368164,
      "activations/layer22_attention_weight_min": -28.52362060546875,
      "activations/layer23_attention_weight_max": 33.409297943115234,
      "activations/layer23_attention_weight_min": -19.801570892333984,
      "activations/layer2_attention_weight_max": 33.26513671875,
      "activations/layer2_attention_weight_min": -31.066539764404297,
      "activations/layer3_attention_weight_max": 87.48277282714844,
      "activations/layer3_attention_weight_min": -90.20203399658203,
      "activations/layer4_attention_weight_max": 129.1088104248047,
      "activations/layer4_attention_weight_min": -127.57328033447266,
      "activations/layer5_attention_weight_max": 84.94668579101562,
      "activations/layer5_attention_weight_min": -81.01133728027344,
      "activations/layer6_attention_weight_max": 55.8975944519043,
      "activations/layer6_attention_weight_min": -56.68467330932617,
      "activations/layer7_attention_weight_max": 80.75919342041016,
      "activations/layer7_attention_weight_min": -78.41340637207031,
      "activations/layer8_attention_weight_max": 39.6390495300293,
      "activations/layer8_attention_weight_min": -37.8320198059082,
      "activations/layer9_attention_weight_max": 40.16892623901367,
      "activations/layer9_attention_weight_min": -33.830970764160156,
      "epoch": 4.36,
      "learning_rate": 0.0001230969696969697,
      "loss": 2.8587,
      "step": 75100
    },
    {
      "activations/layer0_attention_weight_max": 14.665603637695312,
      "activations/layer0_attention_weight_min": -11.810482025146484,
      "activations/layer10_attention_weight_max": 36.48860168457031,
      "activations/layer10_attention_weight_min": -32.53434753417969,
      "activations/layer11_attention_weight_max": 35.45270919799805,
      "activations/layer11_attention_weight_min": -28.445693969726562,
      "activations/layer12_attention_weight_max": 65.3434066772461,
      "activations/layer12_attention_weight_min": -28.425209045410156,
      "activations/layer13_attention_weight_max": 72.65019989013672,
      "activations/layer13_attention_weight_min": -40.667842864990234,
      "activations/layer14_attention_weight_max": 88.70603942871094,
      "activations/layer14_attention_weight_min": -52.95616912841797,
      "activations/layer15_attention_weight_max": 48.244178771972656,
      "activations/layer15_attention_weight_min": -29.432514190673828,
      "activations/layer16_attention_weight_max": 41.9722900390625,
      "activations/layer16_attention_weight_min": -28.64496612548828,
      "activations/layer17_attention_weight_max": 41.039794921875,
      "activations/layer17_attention_weight_min": -26.416139602661133,
      "activations/layer18_attention_weight_max": 33.20451736450195,
      "activations/layer18_attention_weight_min": -19.204252243041992,
      "activations/layer19_attention_weight_max": 43.25149917602539,
      "activations/layer19_attention_weight_min": -29.887983322143555,
      "activations/layer1_attention_weight_max": 16.694828033447266,
      "activations/layer1_attention_weight_min": -14.86729621887207,
      "activations/layer20_attention_weight_max": 35.426448822021484,
      "activations/layer20_attention_weight_min": -24.44939422607422,
      "activations/layer21_attention_weight_max": 37.86079406738281,
      "activations/layer21_attention_weight_min": -19.56966781616211,
      "activations/layer22_attention_weight_max": 54.83467102050781,
      "activations/layer22_attention_weight_min": -27.560274124145508,
      "activations/layer23_attention_weight_max": 30.227811813354492,
      "activations/layer23_attention_weight_min": -19.517093658447266,
      "activations/layer2_attention_weight_max": 32.88149642944336,
      "activations/layer2_attention_weight_min": -30.591041564941406,
      "activations/layer3_attention_weight_max": 89.25997924804688,
      "activations/layer3_attention_weight_min": -88.55899810791016,
      "activations/layer4_attention_weight_max": 126.09252166748047,
      "activations/layer4_attention_weight_min": -130.5635528564453,
      "activations/layer5_attention_weight_max": 88.89112854003906,
      "activations/layer5_attention_weight_min": -89.43626403808594,
      "activations/layer6_attention_weight_max": 58.33979034423828,
      "activations/layer6_attention_weight_min": -58.9696159362793,
      "activations/layer7_attention_weight_max": 85.00967407226562,
      "activations/layer7_attention_weight_min": -81.99431610107422,
      "activations/layer8_attention_weight_max": 39.26754379272461,
      "activations/layer8_attention_weight_min": -39.33540725708008,
      "activations/layer9_attention_weight_max": 38.113853454589844,
      "activations/layer9_attention_weight_min": -35.509639739990234,
      "epoch": 4.37,
      "learning_rate": 0.00012307803030303028,
      "loss": 2.8585,
      "step": 75150
    },
    {
      "activations/layer0_attention_weight_max": 16.774538040161133,
      "activations/layer0_attention_weight_min": -12.450953483581543,
      "activations/layer10_attention_weight_max": 37.62523651123047,
      "activations/layer10_attention_weight_min": -31.799758911132812,
      "activations/layer11_attention_weight_max": 43.495601654052734,
      "activations/layer11_attention_weight_min": -28.606884002685547,
      "activations/layer12_attention_weight_max": 44.69310760498047,
      "activations/layer12_attention_weight_min": -26.157703399658203,
      "activations/layer13_attention_weight_max": 53.547508239746094,
      "activations/layer13_attention_weight_min": -37.51456832885742,
      "activations/layer14_attention_weight_max": 68.57694244384766,
      "activations/layer14_attention_weight_min": -49.03592300415039,
      "activations/layer15_attention_weight_max": 44.29911422729492,
      "activations/layer15_attention_weight_min": -30.142995834350586,
      "activations/layer16_attention_weight_max": 40.07746505737305,
      "activations/layer16_attention_weight_min": -27.103397369384766,
      "activations/layer17_attention_weight_max": 39.743202209472656,
      "activations/layer17_attention_weight_min": -24.39292335510254,
      "activations/layer18_attention_weight_max": 31.157939910888672,
      "activations/layer18_attention_weight_min": -19.248184204101562,
      "activations/layer19_attention_weight_max": 40.98606491088867,
      "activations/layer19_attention_weight_min": -29.27314567565918,
      "activations/layer1_attention_weight_max": 18.011625289916992,
      "activations/layer1_attention_weight_min": -17.365690231323242,
      "activations/layer20_attention_weight_max": 34.43027114868164,
      "activations/layer20_attention_weight_min": -22.477310180664062,
      "activations/layer21_attention_weight_max": 29.19759750366211,
      "activations/layer21_attention_weight_min": -21.739038467407227,
      "activations/layer22_attention_weight_max": 52.26409912109375,
      "activations/layer22_attention_weight_min": -30.831298828125,
      "activations/layer23_attention_weight_max": 33.388427734375,
      "activations/layer23_attention_weight_min": -22.409683227539062,
      "activations/layer2_attention_weight_max": 34.42626953125,
      "activations/layer2_attention_weight_min": -31.701316833496094,
      "activations/layer3_attention_weight_max": 88.96446228027344,
      "activations/layer3_attention_weight_min": -90.91764068603516,
      "activations/layer4_attention_weight_max": 141.02235412597656,
      "activations/layer4_attention_weight_min": -128.38035583496094,
      "activations/layer5_attention_weight_max": 90.46939086914062,
      "activations/layer5_attention_weight_min": -83.90888977050781,
      "activations/layer6_attention_weight_max": 57.13451385498047,
      "activations/layer6_attention_weight_min": -54.963836669921875,
      "activations/layer7_attention_weight_max": 80.98023986816406,
      "activations/layer7_attention_weight_min": -80.16752624511719,
      "activations/layer8_attention_weight_max": 40.415958404541016,
      "activations/layer8_attention_weight_min": -39.42430114746094,
      "activations/layer9_attention_weight_max": 37.50645065307617,
      "activations/layer9_attention_weight_min": -39.34696960449219,
      "epoch": 4.37,
      "learning_rate": 0.00012305946969696967,
      "loss": 2.8807,
      "step": 75200
    },
    {
      "activations/layer0_attention_weight_max": 17.046253204345703,
      "activations/layer0_attention_weight_min": -12.150970458984375,
      "activations/layer10_attention_weight_max": 38.564659118652344,
      "activations/layer10_attention_weight_min": -29.93474578857422,
      "activations/layer11_attention_weight_max": 35.14171600341797,
      "activations/layer11_attention_weight_min": -27.59668731689453,
      "activations/layer12_attention_weight_max": 30.14975357055664,
      "activations/layer12_attention_weight_min": -29.473819732666016,
      "activations/layer13_attention_weight_max": 52.798194885253906,
      "activations/layer13_attention_weight_min": -34.89689636230469,
      "activations/layer14_attention_weight_max": 61.964683532714844,
      "activations/layer14_attention_weight_min": -42.61391067504883,
      "activations/layer15_attention_weight_max": 43.66852951049805,
      "activations/layer15_attention_weight_min": -28.35891342163086,
      "activations/layer16_attention_weight_max": 38.78456497192383,
      "activations/layer16_attention_weight_min": -27.098373413085938,
      "activations/layer17_attention_weight_max": 44.600345611572266,
      "activations/layer17_attention_weight_min": -24.477252960205078,
      "activations/layer18_attention_weight_max": 35.89637756347656,
      "activations/layer18_attention_weight_min": -19.841815948486328,
      "activations/layer19_attention_weight_max": 41.699974060058594,
      "activations/layer19_attention_weight_min": -29.481332778930664,
      "activations/layer1_attention_weight_max": 17.35959243774414,
      "activations/layer1_attention_weight_min": -15.219697952270508,
      "activations/layer20_attention_weight_max": 38.09843826293945,
      "activations/layer20_attention_weight_min": -26.623435974121094,
      "activations/layer21_attention_weight_max": 33.5430793762207,
      "activations/layer21_attention_weight_min": -21.16908836364746,
      "activations/layer22_attention_weight_max": 53.181785583496094,
      "activations/layer22_attention_weight_min": -28.75476837158203,
      "activations/layer23_attention_weight_max": 31.841480255126953,
      "activations/layer23_attention_weight_min": -25.6124324798584,
      "activations/layer2_attention_weight_max": 32.424991607666016,
      "activations/layer2_attention_weight_min": -30.649219512939453,
      "activations/layer3_attention_weight_max": 87.64225006103516,
      "activations/layer3_attention_weight_min": -94.35646057128906,
      "activations/layer4_attention_weight_max": 136.06655883789062,
      "activations/layer4_attention_weight_min": -127.21693420410156,
      "activations/layer5_attention_weight_max": 91.53544616699219,
      "activations/layer5_attention_weight_min": -81.74828338623047,
      "activations/layer6_attention_weight_max": 58.1638298034668,
      "activations/layer6_attention_weight_min": -58.693580627441406,
      "activations/layer7_attention_weight_max": 81.6263656616211,
      "activations/layer7_attention_weight_min": -76.26347351074219,
      "activations/layer8_attention_weight_max": 39.83268737792969,
      "activations/layer8_attention_weight_min": -38.84437942504883,
      "activations/layer9_attention_weight_max": 37.247474670410156,
      "activations/layer9_attention_weight_min": -33.75489044189453,
      "epoch": 4.37,
      "learning_rate": 0.0001230405303030303,
      "loss": 2.8635,
      "step": 75250
    },
    {
      "activations/layer0_attention_weight_max": 16.47884750366211,
      "activations/layer0_attention_weight_min": -12.232583045959473,
      "activations/layer10_attention_weight_max": 35.42882537841797,
      "activations/layer10_attention_weight_min": -32.36358642578125,
      "activations/layer11_attention_weight_max": 35.474388122558594,
      "activations/layer11_attention_weight_min": -27.739316940307617,
      "activations/layer12_attention_weight_max": 29.277973175048828,
      "activations/layer12_attention_weight_min": -24.160572052001953,
      "activations/layer13_attention_weight_max": 56.76143264770508,
      "activations/layer13_attention_weight_min": -35.50547409057617,
      "activations/layer14_attention_weight_max": 66.91678619384766,
      "activations/layer14_attention_weight_min": -46.3221435546875,
      "activations/layer15_attention_weight_max": 49.371742248535156,
      "activations/layer15_attention_weight_min": -26.960046768188477,
      "activations/layer16_attention_weight_max": 41.06519317626953,
      "activations/layer16_attention_weight_min": -27.233749389648438,
      "activations/layer17_attention_weight_max": 44.434017181396484,
      "activations/layer17_attention_weight_min": -25.63288688659668,
      "activations/layer18_attention_weight_max": 34.567108154296875,
      "activations/layer18_attention_weight_min": -22.0869197845459,
      "activations/layer19_attention_weight_max": 44.16366195678711,
      "activations/layer19_attention_weight_min": -27.83912467956543,
      "activations/layer1_attention_weight_max": 16.942495346069336,
      "activations/layer1_attention_weight_min": -14.47081184387207,
      "activations/layer20_attention_weight_max": 32.54275894165039,
      "activations/layer20_attention_weight_min": -23.60491943359375,
      "activations/layer21_attention_weight_max": 31.52182960510254,
      "activations/layer21_attention_weight_min": -20.71530532836914,
      "activations/layer22_attention_weight_max": 53.8630256652832,
      "activations/layer22_attention_weight_min": -26.761091232299805,
      "activations/layer23_attention_weight_max": 32.08716583251953,
      "activations/layer23_attention_weight_min": -20.308387756347656,
      "activations/layer2_attention_weight_max": 33.04352569580078,
      "activations/layer2_attention_weight_min": -31.235774993896484,
      "activations/layer3_attention_weight_max": 85.18225860595703,
      "activations/layer3_attention_weight_min": -91.36456298828125,
      "activations/layer4_attention_weight_max": 133.03042602539062,
      "activations/layer4_attention_weight_min": -130.01319885253906,
      "activations/layer5_attention_weight_max": 89.38195037841797,
      "activations/layer5_attention_weight_min": -81.59931945800781,
      "activations/layer6_attention_weight_max": 57.32670211791992,
      "activations/layer6_attention_weight_min": -56.53053283691406,
      "activations/layer7_attention_weight_max": 80.09487915039062,
      "activations/layer7_attention_weight_min": -78.21757507324219,
      "activations/layer8_attention_weight_max": 42.418312072753906,
      "activations/layer8_attention_weight_min": -42.886539459228516,
      "activations/layer9_attention_weight_max": 40.07999801635742,
      "activations/layer9_attention_weight_min": -35.9022331237793,
      "epoch": 4.38,
      "learning_rate": 0.0001230215909090909,
      "loss": 2.8735,
      "step": 75300
    },
    {
      "activations/layer0_attention_weight_max": 16.60577964782715,
      "activations/layer0_attention_weight_min": -11.435670852661133,
      "activations/layer10_attention_weight_max": 34.81446075439453,
      "activations/layer10_attention_weight_min": -31.955419540405273,
      "activations/layer11_attention_weight_max": 35.705078125,
      "activations/layer11_attention_weight_min": -30.516666412353516,
      "activations/layer12_attention_weight_max": 26.967872619628906,
      "activations/layer12_attention_weight_min": -34.430908203125,
      "activations/layer13_attention_weight_max": 43.891536712646484,
      "activations/layer13_attention_weight_min": -33.132869720458984,
      "activations/layer14_attention_weight_max": 63.46208953857422,
      "activations/layer14_attention_weight_min": -44.56895446777344,
      "activations/layer15_attention_weight_max": 45.137062072753906,
      "activations/layer15_attention_weight_min": -31.766815185546875,
      "activations/layer16_attention_weight_max": 43.976016998291016,
      "activations/layer16_attention_weight_min": -28.227489471435547,
      "activations/layer17_attention_weight_max": 48.97807693481445,
      "activations/layer17_attention_weight_min": -26.76214027404785,
      "activations/layer18_attention_weight_max": 31.66375160217285,
      "activations/layer18_attention_weight_min": -22.564781188964844,
      "activations/layer19_attention_weight_max": 49.62760543823242,
      "activations/layer19_attention_weight_min": -28.875640869140625,
      "activations/layer1_attention_weight_max": 17.00788116455078,
      "activations/layer1_attention_weight_min": -14.682233810424805,
      "activations/layer20_attention_weight_max": 41.805240631103516,
      "activations/layer20_attention_weight_min": -23.282604217529297,
      "activations/layer21_attention_weight_max": 33.32368087768555,
      "activations/layer21_attention_weight_min": -22.01667022705078,
      "activations/layer22_attention_weight_max": 61.677433013916016,
      "activations/layer22_attention_weight_min": -26.53986358642578,
      "activations/layer23_attention_weight_max": 32.24354553222656,
      "activations/layer23_attention_weight_min": -20.398605346679688,
      "activations/layer2_attention_weight_max": 31.981563568115234,
      "activations/layer2_attention_weight_min": -31.685195922851562,
      "activations/layer3_attention_weight_max": 89.3344955444336,
      "activations/layer3_attention_weight_min": -98.3957290649414,
      "activations/layer4_attention_weight_max": 136.41738891601562,
      "activations/layer4_attention_weight_min": -140.7018280029297,
      "activations/layer5_attention_weight_max": 92.216064453125,
      "activations/layer5_attention_weight_min": -90.76262664794922,
      "activations/layer6_attention_weight_max": 59.09495162963867,
      "activations/layer6_attention_weight_min": -59.60524368286133,
      "activations/layer7_attention_weight_max": 87.26451110839844,
      "activations/layer7_attention_weight_min": -82.30513763427734,
      "activations/layer8_attention_weight_max": 42.48383712768555,
      "activations/layer8_attention_weight_min": -38.26540756225586,
      "activations/layer9_attention_weight_max": 38.91297912597656,
      "activations/layer9_attention_weight_min": -40.95391845703125,
      "epoch": 4.38,
      "learning_rate": 0.00012300265151515152,
      "loss": 2.8567,
      "step": 75350
    },
    {
      "activations/layer0_attention_weight_max": 17.585464477539062,
      "activations/layer0_attention_weight_min": -11.604399681091309,
      "activations/layer10_attention_weight_max": 36.931556701660156,
      "activations/layer10_attention_weight_min": -30.91552734375,
      "activations/layer11_attention_weight_max": 33.0615119934082,
      "activations/layer11_attention_weight_min": -30.018394470214844,
      "activations/layer12_attention_weight_max": 26.87562370300293,
      "activations/layer12_attention_weight_min": -27.344724655151367,
      "activations/layer13_attention_weight_max": 53.72763442993164,
      "activations/layer13_attention_weight_min": -34.14713668823242,
      "activations/layer14_attention_weight_max": 62.39780044555664,
      "activations/layer14_attention_weight_min": -45.257999420166016,
      "activations/layer15_attention_weight_max": 48.00604248046875,
      "activations/layer15_attention_weight_min": -30.593679428100586,
      "activations/layer16_attention_weight_max": 39.824337005615234,
      "activations/layer16_attention_weight_min": -27.782817840576172,
      "activations/layer17_attention_weight_max": 46.58296203613281,
      "activations/layer17_attention_weight_min": -27.335025787353516,
      "activations/layer18_attention_weight_max": 34.11945724487305,
      "activations/layer18_attention_weight_min": -19.945566177368164,
      "activations/layer19_attention_weight_max": 43.93225860595703,
      "activations/layer19_attention_weight_min": -28.641820907592773,
      "activations/layer1_attention_weight_max": 17.72939109802246,
      "activations/layer1_attention_weight_min": -16.909236907958984,
      "activations/layer20_attention_weight_max": 39.557926177978516,
      "activations/layer20_attention_weight_min": -24.896574020385742,
      "activations/layer21_attention_weight_max": 35.5853385925293,
      "activations/layer21_attention_weight_min": -20.775211334228516,
      "activations/layer22_attention_weight_max": 60.87663650512695,
      "activations/layer22_attention_weight_min": -28.61990737915039,
      "activations/layer23_attention_weight_max": 32.33474349975586,
      "activations/layer23_attention_weight_min": -18.75345230102539,
      "activations/layer2_attention_weight_max": 32.040489196777344,
      "activations/layer2_attention_weight_min": -31.31222915649414,
      "activations/layer3_attention_weight_max": 86.22956085205078,
      "activations/layer3_attention_weight_min": -87.51758575439453,
      "activations/layer4_attention_weight_max": 127.17813873291016,
      "activations/layer4_attention_weight_min": -127.46492004394531,
      "activations/layer5_attention_weight_max": 91.94070434570312,
      "activations/layer5_attention_weight_min": -85.38695526123047,
      "activations/layer6_attention_weight_max": 59.769012451171875,
      "activations/layer6_attention_weight_min": -59.97368240356445,
      "activations/layer7_attention_weight_max": 81.89324951171875,
      "activations/layer7_attention_weight_min": -79.77655792236328,
      "activations/layer8_attention_weight_max": 41.7283935546875,
      "activations/layer8_attention_weight_min": -37.015647888183594,
      "activations/layer9_attention_weight_max": 36.26188659667969,
      "activations/layer9_attention_weight_min": -37.618072509765625,
      "epoch": 4.38,
      "learning_rate": 0.0001229837121212121,
      "loss": 2.8713,
      "step": 75400
    },
    {
      "activations/layer0_attention_weight_max": 16.419570922851562,
      "activations/layer0_attention_weight_min": -12.594609260559082,
      "activations/layer10_attention_weight_max": 33.96480178833008,
      "activations/layer10_attention_weight_min": -27.62813949584961,
      "activations/layer11_attention_weight_max": 34.9793701171875,
      "activations/layer11_attention_weight_min": -27.020477294921875,
      "activations/layer12_attention_weight_max": 46.59355545043945,
      "activations/layer12_attention_weight_min": -27.48318862915039,
      "activations/layer13_attention_weight_max": 41.86143493652344,
      "activations/layer13_attention_weight_min": -31.966806411743164,
      "activations/layer14_attention_weight_max": 66.60769653320312,
      "activations/layer14_attention_weight_min": -49.722511291503906,
      "activations/layer15_attention_weight_max": 44.91062545776367,
      "activations/layer15_attention_weight_min": -28.701679229736328,
      "activations/layer16_attention_weight_max": 37.474185943603516,
      "activations/layer16_attention_weight_min": -28.429590225219727,
      "activations/layer17_attention_weight_max": 38.98564529418945,
      "activations/layer17_attention_weight_min": -25.04094886779785,
      "activations/layer18_attention_weight_max": 31.528911590576172,
      "activations/layer18_attention_weight_min": -20.295000076293945,
      "activations/layer19_attention_weight_max": 40.018310546875,
      "activations/layer19_attention_weight_min": -30.256336212158203,
      "activations/layer1_attention_weight_max": 17.137977600097656,
      "activations/layer1_attention_weight_min": -13.866241455078125,
      "activations/layer20_attention_weight_max": 31.824567794799805,
      "activations/layer20_attention_weight_min": -22.10872459411621,
      "activations/layer21_attention_weight_max": 40.15008544921875,
      "activations/layer21_attention_weight_min": -20.4971981048584,
      "activations/layer22_attention_weight_max": 47.88568878173828,
      "activations/layer22_attention_weight_min": -27.64585304260254,
      "activations/layer23_attention_weight_max": 29.919261932373047,
      "activations/layer23_attention_weight_min": -18.66038703918457,
      "activations/layer2_attention_weight_max": 33.16382598876953,
      "activations/layer2_attention_weight_min": -30.940277099609375,
      "activations/layer3_attention_weight_max": 90.09318542480469,
      "activations/layer3_attention_weight_min": -90.77947235107422,
      "activations/layer4_attention_weight_max": 130.28477478027344,
      "activations/layer4_attention_weight_min": -124.45613098144531,
      "activations/layer5_attention_weight_max": 89.061767578125,
      "activations/layer5_attention_weight_min": -80.29466247558594,
      "activations/layer6_attention_weight_max": 57.57456588745117,
      "activations/layer6_attention_weight_min": -55.31657409667969,
      "activations/layer7_attention_weight_max": 83.52084350585938,
      "activations/layer7_attention_weight_min": -79.6961441040039,
      "activations/layer8_attention_weight_max": 40.17108917236328,
      "activations/layer8_attention_weight_min": -38.07174301147461,
      "activations/layer9_attention_weight_max": 35.46403121948242,
      "activations/layer9_attention_weight_min": -35.135841369628906,
      "epoch": 4.38,
      "learning_rate": 0.00012296477272727272,
      "loss": 2.8689,
      "step": 75450
    },
    {
      "activations/layer0_attention_weight_max": 16.823951721191406,
      "activations/layer0_attention_weight_min": -11.94136905670166,
      "activations/layer10_attention_weight_max": 35.25004959106445,
      "activations/layer10_attention_weight_min": -29.404958724975586,
      "activations/layer11_attention_weight_max": 34.13682174682617,
      "activations/layer11_attention_weight_min": -29.91291618347168,
      "activations/layer12_attention_weight_max": 40.828887939453125,
      "activations/layer12_attention_weight_min": -28.78040885925293,
      "activations/layer13_attention_weight_max": 51.255897521972656,
      "activations/layer13_attention_weight_min": -34.95925521850586,
      "activations/layer14_attention_weight_max": 72.07951354980469,
      "activations/layer14_attention_weight_min": -53.87995910644531,
      "activations/layer15_attention_weight_max": 43.81791687011719,
      "activations/layer15_attention_weight_min": -29.80171775817871,
      "activations/layer16_attention_weight_max": 49.03276443481445,
      "activations/layer16_attention_weight_min": -29.468582153320312,
      "activations/layer17_attention_weight_max": 37.53362274169922,
      "activations/layer17_attention_weight_min": -23.20561981201172,
      "activations/layer18_attention_weight_max": 29.516704559326172,
      "activations/layer18_attention_weight_min": -19.69742202758789,
      "activations/layer19_attention_weight_max": 52.12137985229492,
      "activations/layer19_attention_weight_min": -27.591571807861328,
      "activations/layer1_attention_weight_max": 18.247514724731445,
      "activations/layer1_attention_weight_min": -15.018513679504395,
      "activations/layer20_attention_weight_max": 31.946664810180664,
      "activations/layer20_attention_weight_min": -23.23513412475586,
      "activations/layer21_attention_weight_max": 32.436912536621094,
      "activations/layer21_attention_weight_min": -20.6556339263916,
      "activations/layer22_attention_weight_max": 46.09764099121094,
      "activations/layer22_attention_weight_min": -28.446863174438477,
      "activations/layer23_attention_weight_max": 31.365158081054688,
      "activations/layer23_attention_weight_min": -20.3453426361084,
      "activations/layer2_attention_weight_max": 32.45224380493164,
      "activations/layer2_attention_weight_min": -32.13328170776367,
      "activations/layer3_attention_weight_max": 86.72276306152344,
      "activations/layer3_attention_weight_min": -88.79469299316406,
      "activations/layer4_attention_weight_max": 127.5602798461914,
      "activations/layer4_attention_weight_min": -129.43907165527344,
      "activations/layer5_attention_weight_max": 87.19242858886719,
      "activations/layer5_attention_weight_min": -82.51316833496094,
      "activations/layer6_attention_weight_max": 57.13268280029297,
      "activations/layer6_attention_weight_min": -56.07243728637695,
      "activations/layer7_attention_weight_max": 82.25341033935547,
      "activations/layer7_attention_weight_min": -82.38728332519531,
      "activations/layer8_attention_weight_max": 40.199588775634766,
      "activations/layer8_attention_weight_min": -38.88676071166992,
      "activations/layer9_attention_weight_max": 38.5574836730957,
      "activations/layer9_attention_weight_min": -39.209022521972656,
      "epoch": 4.39,
      "learning_rate": 0.00012294583333333334,
      "loss": 2.8597,
      "step": 75500
    },
    {
      "activations/layer0_attention_weight_max": 15.712174415588379,
      "activations/layer0_attention_weight_min": -10.83149528503418,
      "activations/layer10_attention_weight_max": 35.97389602661133,
      "activations/layer10_attention_weight_min": -30.627819061279297,
      "activations/layer11_attention_weight_max": 33.9307746887207,
      "activations/layer11_attention_weight_min": -29.622251510620117,
      "activations/layer12_attention_weight_max": 28.80872917175293,
      "activations/layer12_attention_weight_min": -25.69639015197754,
      "activations/layer13_attention_weight_max": 45.09537887573242,
      "activations/layer13_attention_weight_min": -30.97698402404785,
      "activations/layer14_attention_weight_max": 73.62718200683594,
      "activations/layer14_attention_weight_min": -49.7333869934082,
      "activations/layer15_attention_weight_max": 45.710235595703125,
      "activations/layer15_attention_weight_min": -30.378616333007812,
      "activations/layer16_attention_weight_max": 44.883113861083984,
      "activations/layer16_attention_weight_min": -27.65984344482422,
      "activations/layer17_attention_weight_max": 42.32454299926758,
      "activations/layer17_attention_weight_min": -23.666339874267578,
      "activations/layer18_attention_weight_max": 30.884376525878906,
      "activations/layer18_attention_weight_min": -19.305456161499023,
      "activations/layer19_attention_weight_max": 44.8771858215332,
      "activations/layer19_attention_weight_min": -30.394775390625,
      "activations/layer1_attention_weight_max": 18.098941802978516,
      "activations/layer1_attention_weight_min": -14.081695556640625,
      "activations/layer20_attention_weight_max": 38.83555603027344,
      "activations/layer20_attention_weight_min": -23.537817001342773,
      "activations/layer21_attention_weight_max": 42.16511154174805,
      "activations/layer21_attention_weight_min": -19.665191650390625,
      "activations/layer22_attention_weight_max": 51.1251220703125,
      "activations/layer22_attention_weight_min": -32.27157974243164,
      "activations/layer23_attention_weight_max": 33.883304595947266,
      "activations/layer23_attention_weight_min": -20.04536247253418,
      "activations/layer2_attention_weight_max": 31.67945671081543,
      "activations/layer2_attention_weight_min": -31.30335807800293,
      "activations/layer3_attention_weight_max": 89.1517333984375,
      "activations/layer3_attention_weight_min": -91.91375732421875,
      "activations/layer4_attention_weight_max": 131.27545166015625,
      "activations/layer4_attention_weight_min": -136.24700927734375,
      "activations/layer5_attention_weight_max": 90.50454711914062,
      "activations/layer5_attention_weight_min": -89.61504364013672,
      "activations/layer6_attention_weight_max": 60.86258316040039,
      "activations/layer6_attention_weight_min": -59.19523620605469,
      "activations/layer7_attention_weight_max": 86.88579559326172,
      "activations/layer7_attention_weight_min": -89.24019622802734,
      "activations/layer8_attention_weight_max": 48.14578628540039,
      "activations/layer8_attention_weight_min": -44.477516174316406,
      "activations/layer9_attention_weight_max": 41.40612030029297,
      "activations/layer9_attention_weight_min": -36.89044189453125,
      "epoch": 4.39,
      "learning_rate": 0.00012292689393939393,
      "loss": 2.8778,
      "step": 75550
    },
    {
      "activations/layer0_attention_weight_max": 16.718551635742188,
      "activations/layer0_attention_weight_min": -11.01973819732666,
      "activations/layer10_attention_weight_max": 36.83710861206055,
      "activations/layer10_attention_weight_min": -29.208646774291992,
      "activations/layer11_attention_weight_max": 36.47026062011719,
      "activations/layer11_attention_weight_min": -26.404775619506836,
      "activations/layer12_attention_weight_max": 35.172882080078125,
      "activations/layer12_attention_weight_min": -33.18528366088867,
      "activations/layer13_attention_weight_max": 42.698768615722656,
      "activations/layer13_attention_weight_min": -32.11506652832031,
      "activations/layer14_attention_weight_max": 65.98097229003906,
      "activations/layer14_attention_weight_min": -48.200260162353516,
      "activations/layer15_attention_weight_max": 36.85369873046875,
      "activations/layer15_attention_weight_min": -26.967180252075195,
      "activations/layer16_attention_weight_max": 37.23798751831055,
      "activations/layer16_attention_weight_min": -30.95792579650879,
      "activations/layer17_attention_weight_max": 37.257625579833984,
      "activations/layer17_attention_weight_min": -23.896495819091797,
      "activations/layer18_attention_weight_max": 29.011850357055664,
      "activations/layer18_attention_weight_min": -19.244474411010742,
      "activations/layer19_attention_weight_max": 38.47342300415039,
      "activations/layer19_attention_weight_min": -29.7703800201416,
      "activations/layer1_attention_weight_max": 17.97774887084961,
      "activations/layer1_attention_weight_min": -19.909652709960938,
      "activations/layer20_attention_weight_max": 30.3151912689209,
      "activations/layer20_attention_weight_min": -25.683738708496094,
      "activations/layer21_attention_weight_max": 28.96920394897461,
      "activations/layer21_attention_weight_min": -21.893884658813477,
      "activations/layer22_attention_weight_max": 52.317596435546875,
      "activations/layer22_attention_weight_min": -30.049924850463867,
      "activations/layer23_attention_weight_max": 30.35485076904297,
      "activations/layer23_attention_weight_min": -23.801321029663086,
      "activations/layer2_attention_weight_max": 34.182960510253906,
      "activations/layer2_attention_weight_min": -33.63752365112305,
      "activations/layer3_attention_weight_max": 94.8656005859375,
      "activations/layer3_attention_weight_min": -94.4684066772461,
      "activations/layer4_attention_weight_max": 139.73963928222656,
      "activations/layer4_attention_weight_min": -133.99867248535156,
      "activations/layer5_attention_weight_max": 90.50994110107422,
      "activations/layer5_attention_weight_min": -84.41602325439453,
      "activations/layer6_attention_weight_max": 55.64863967895508,
      "activations/layer6_attention_weight_min": -54.74638748168945,
      "activations/layer7_attention_weight_max": 82.20803833007812,
      "activations/layer7_attention_weight_min": -78.13148498535156,
      "activations/layer8_attention_weight_max": 44.861473083496094,
      "activations/layer8_attention_weight_min": -38.67827224731445,
      "activations/layer9_attention_weight_max": 43.816585540771484,
      "activations/layer9_attention_weight_min": -40.033897399902344,
      "epoch": 4.39,
      "learning_rate": 0.00012290795454545454,
      "loss": 2.8791,
      "step": 75600
    },
    {
      "activations/layer0_attention_weight_max": 16.05392837524414,
      "activations/layer0_attention_weight_min": -12.556400299072266,
      "activations/layer10_attention_weight_max": 36.037872314453125,
      "activations/layer10_attention_weight_min": -29.678112030029297,
      "activations/layer11_attention_weight_max": 34.628047943115234,
      "activations/layer11_attention_weight_min": -26.449520111083984,
      "activations/layer12_attention_weight_max": 27.917030334472656,
      "activations/layer12_attention_weight_min": -24.668106079101562,
      "activations/layer13_attention_weight_max": 50.96521759033203,
      "activations/layer13_attention_weight_min": -35.91545486450195,
      "activations/layer14_attention_weight_max": 70.04293823242188,
      "activations/layer14_attention_weight_min": -49.59436798095703,
      "activations/layer15_attention_weight_max": 45.51953125,
      "activations/layer15_attention_weight_min": -30.335355758666992,
      "activations/layer16_attention_weight_max": 40.28768539428711,
      "activations/layer16_attention_weight_min": -28.506277084350586,
      "activations/layer17_attention_weight_max": 40.57329177856445,
      "activations/layer17_attention_weight_min": -24.759868621826172,
      "activations/layer18_attention_weight_max": 35.059268951416016,
      "activations/layer18_attention_weight_min": -22.493146896362305,
      "activations/layer19_attention_weight_max": 41.55170440673828,
      "activations/layer19_attention_weight_min": -28.610013961791992,
      "activations/layer1_attention_weight_max": 17.25186538696289,
      "activations/layer1_attention_weight_min": -14.097883224487305,
      "activations/layer20_attention_weight_max": 32.13955307006836,
      "activations/layer20_attention_weight_min": -22.799114227294922,
      "activations/layer21_attention_weight_max": 31.032175064086914,
      "activations/layer21_attention_weight_min": -20.63088035583496,
      "activations/layer22_attention_weight_max": 51.36003494262695,
      "activations/layer22_attention_weight_min": -28.56468391418457,
      "activations/layer23_attention_weight_max": 32.1048469543457,
      "activations/layer23_attention_weight_min": -19.666126251220703,
      "activations/layer2_attention_weight_max": 32.10562515258789,
      "activations/layer2_attention_weight_min": -30.107053756713867,
      "activations/layer3_attention_weight_max": 88.15018463134766,
      "activations/layer3_attention_weight_min": -88.24903869628906,
      "activations/layer4_attention_weight_max": 134.08203125,
      "activations/layer4_attention_weight_min": -124.64189910888672,
      "activations/layer5_attention_weight_max": 86.37425994873047,
      "activations/layer5_attention_weight_min": -81.63416290283203,
      "activations/layer6_attention_weight_max": 56.3455924987793,
      "activations/layer6_attention_weight_min": -56.35413360595703,
      "activations/layer7_attention_weight_max": 79.97181701660156,
      "activations/layer7_attention_weight_min": -83.97760772705078,
      "activations/layer8_attention_weight_max": 41.018367767333984,
      "activations/layer8_attention_weight_min": -41.50792694091797,
      "activations/layer9_attention_weight_max": 36.55393981933594,
      "activations/layer9_attention_weight_min": -34.05976104736328,
      "epoch": 4.4,
      "learning_rate": 0.00012288901515151516,
      "loss": 2.865,
      "step": 75650
    },
    {
      "activations/layer0_attention_weight_max": 14.988909721374512,
      "activations/layer0_attention_weight_min": -12.324856758117676,
      "activations/layer10_attention_weight_max": 39.928016662597656,
      "activations/layer10_attention_weight_min": -31.3480281829834,
      "activations/layer11_attention_weight_max": 37.22412872314453,
      "activations/layer11_attention_weight_min": -30.696922302246094,
      "activations/layer12_attention_weight_max": 36.696067810058594,
      "activations/layer12_attention_weight_min": -28.039005279541016,
      "activations/layer13_attention_weight_max": 53.11876678466797,
      "activations/layer13_attention_weight_min": -36.96236038208008,
      "activations/layer14_attention_weight_max": 68.85192108154297,
      "activations/layer14_attention_weight_min": -46.29790496826172,
      "activations/layer15_attention_weight_max": 42.89882278442383,
      "activations/layer15_attention_weight_min": -30.1352481842041,
      "activations/layer16_attention_weight_max": 41.451229095458984,
      "activations/layer16_attention_weight_min": -26.912511825561523,
      "activations/layer17_attention_weight_max": 43.7650032043457,
      "activations/layer17_attention_weight_min": -24.010194778442383,
      "activations/layer18_attention_weight_max": 31.669681549072266,
      "activations/layer18_attention_weight_min": -20.442718505859375,
      "activations/layer19_attention_weight_max": 42.3809814453125,
      "activations/layer19_attention_weight_min": -26.790775299072266,
      "activations/layer1_attention_weight_max": 18.664316177368164,
      "activations/layer1_attention_weight_min": -15.385174751281738,
      "activations/layer20_attention_weight_max": 34.45078659057617,
      "activations/layer20_attention_weight_min": -22.470481872558594,
      "activations/layer21_attention_weight_max": 29.780973434448242,
      "activations/layer21_attention_weight_min": -19.057092666625977,
      "activations/layer22_attention_weight_max": 50.85812759399414,
      "activations/layer22_attention_weight_min": -25.97794532775879,
      "activations/layer23_attention_weight_max": 36.563812255859375,
      "activations/layer23_attention_weight_min": -18.80516242980957,
      "activations/layer2_attention_weight_max": 32.07466506958008,
      "activations/layer2_attention_weight_min": -30.977577209472656,
      "activations/layer3_attention_weight_max": 90.03775787353516,
      "activations/layer3_attention_weight_min": -88.42866516113281,
      "activations/layer4_attention_weight_max": 139.84071350097656,
      "activations/layer4_attention_weight_min": -131.15769958496094,
      "activations/layer5_attention_weight_max": 97.74175262451172,
      "activations/layer5_attention_weight_min": -88.60945129394531,
      "activations/layer6_attention_weight_max": 62.185882568359375,
      "activations/layer6_attention_weight_min": -58.930137634277344,
      "activations/layer7_attention_weight_max": 83.96223449707031,
      "activations/layer7_attention_weight_min": -81.13180541992188,
      "activations/layer8_attention_weight_max": 44.82938003540039,
      "activations/layer8_attention_weight_min": -41.02256393432617,
      "activations/layer9_attention_weight_max": 41.87470626831055,
      "activations/layer9_attention_weight_min": -36.58665466308594,
      "epoch": 4.4,
      "learning_rate": 0.00012287007575757574,
      "loss": 2.876,
      "step": 75700
    },
    {
      "activations/layer0_attention_weight_max": 16.527311325073242,
      "activations/layer0_attention_weight_min": -11.734712600708008,
      "activations/layer10_attention_weight_max": 37.28328323364258,
      "activations/layer10_attention_weight_min": -30.068058013916016,
      "activations/layer11_attention_weight_max": 33.77097702026367,
      "activations/layer11_attention_weight_min": -27.655136108398438,
      "activations/layer12_attention_weight_max": 25.751663208007812,
      "activations/layer12_attention_weight_min": -28.693180084228516,
      "activations/layer13_attention_weight_max": 42.06944274902344,
      "activations/layer13_attention_weight_min": -34.46196746826172,
      "activations/layer14_attention_weight_max": 68.82909393310547,
      "activations/layer14_attention_weight_min": -47.42645263671875,
      "activations/layer15_attention_weight_max": 38.038047790527344,
      "activations/layer15_attention_weight_min": -27.49834442138672,
      "activations/layer16_attention_weight_max": 36.844322204589844,
      "activations/layer16_attention_weight_min": -28.97477149963379,
      "activations/layer17_attention_weight_max": 40.1357307434082,
      "activations/layer17_attention_weight_min": -24.596851348876953,
      "activations/layer18_attention_weight_max": 32.09214401245117,
      "activations/layer18_attention_weight_min": -20.8602237701416,
      "activations/layer19_attention_weight_max": 39.425804138183594,
      "activations/layer19_attention_weight_min": -29.937780380249023,
      "activations/layer1_attention_weight_max": 17.34148406982422,
      "activations/layer1_attention_weight_min": -15.544594764709473,
      "activations/layer20_attention_weight_max": 32.268367767333984,
      "activations/layer20_attention_weight_min": -25.555784225463867,
      "activations/layer21_attention_weight_max": 32.67244338989258,
      "activations/layer21_attention_weight_min": -23.13506507873535,
      "activations/layer22_attention_weight_max": 56.863494873046875,
      "activations/layer22_attention_weight_min": -31.299238204956055,
      "activations/layer23_attention_weight_max": 33.807701110839844,
      "activations/layer23_attention_weight_min": -20.299842834472656,
      "activations/layer2_attention_weight_max": 32.232460021972656,
      "activations/layer2_attention_weight_min": -30.670650482177734,
      "activations/layer3_attention_weight_max": 87.78229522705078,
      "activations/layer3_attention_weight_min": -89.42318725585938,
      "activations/layer4_attention_weight_max": 131.3021240234375,
      "activations/layer4_attention_weight_min": -133.38909912109375,
      "activations/layer5_attention_weight_max": 85.63742065429688,
      "activations/layer5_attention_weight_min": -84.12846374511719,
      "activations/layer6_attention_weight_max": 55.91519546508789,
      "activations/layer6_attention_weight_min": -56.032325744628906,
      "activations/layer7_attention_weight_max": 79.25607299804688,
      "activations/layer7_attention_weight_min": -77.95357513427734,
      "activations/layer8_attention_weight_max": 39.74788284301758,
      "activations/layer8_attention_weight_min": -39.673561096191406,
      "activations/layer9_attention_weight_max": 38.894710540771484,
      "activations/layer9_attention_weight_min": -35.73442459106445,
      "epoch": 4.4,
      "learning_rate": 0.00012285113636363636,
      "loss": 2.8574,
      "step": 75750
    },
    {
      "activations/layer0_attention_weight_max": 15.044692039489746,
      "activations/layer0_attention_weight_min": -11.229092597961426,
      "activations/layer10_attention_weight_max": 41.09489822387695,
      "activations/layer10_attention_weight_min": -33.25375747680664,
      "activations/layer11_attention_weight_max": 42.24440002441406,
      "activations/layer11_attention_weight_min": -26.107162475585938,
      "activations/layer12_attention_weight_max": 29.655052185058594,
      "activations/layer12_attention_weight_min": -28.597043991088867,
      "activations/layer13_attention_weight_max": 53.751285552978516,
      "activations/layer13_attention_weight_min": -33.85078048706055,
      "activations/layer14_attention_weight_max": 76.55301666259766,
      "activations/layer14_attention_weight_min": -48.57349395751953,
      "activations/layer15_attention_weight_max": 54.13098907470703,
      "activations/layer15_attention_weight_min": -31.774507522583008,
      "activations/layer16_attention_weight_max": 39.89657211303711,
      "activations/layer16_attention_weight_min": -27.319334030151367,
      "activations/layer17_attention_weight_max": 44.76633071899414,
      "activations/layer17_attention_weight_min": -23.586753845214844,
      "activations/layer18_attention_weight_max": 32.96809768676758,
      "activations/layer18_attention_weight_min": -21.05739974975586,
      "activations/layer19_attention_weight_max": 37.919288635253906,
      "activations/layer19_attention_weight_min": -27.824323654174805,
      "activations/layer1_attention_weight_max": 17.15976905822754,
      "activations/layer1_attention_weight_min": -14.997604370117188,
      "activations/layer20_attention_weight_max": 32.68541717529297,
      "activations/layer20_attention_weight_min": -22.416410446166992,
      "activations/layer21_attention_weight_max": 37.84115219116211,
      "activations/layer21_attention_weight_min": -20.92228126525879,
      "activations/layer22_attention_weight_max": 50.42373275756836,
      "activations/layer22_attention_weight_min": -28.371938705444336,
      "activations/layer23_attention_weight_max": 29.565704345703125,
      "activations/layer23_attention_weight_min": -19.82807159423828,
      "activations/layer2_attention_weight_max": 34.55120086669922,
      "activations/layer2_attention_weight_min": -33.39039611816406,
      "activations/layer3_attention_weight_max": 93.27693176269531,
      "activations/layer3_attention_weight_min": -95.56199645996094,
      "activations/layer4_attention_weight_max": 136.7745361328125,
      "activations/layer4_attention_weight_min": -133.50927734375,
      "activations/layer5_attention_weight_max": 96.53056335449219,
      "activations/layer5_attention_weight_min": -93.63992309570312,
      "activations/layer6_attention_weight_max": 62.10449981689453,
      "activations/layer6_attention_weight_min": -62.2714729309082,
      "activations/layer7_attention_weight_max": 88.03472900390625,
      "activations/layer7_attention_weight_min": -88.06717681884766,
      "activations/layer8_attention_weight_max": 43.53519821166992,
      "activations/layer8_attention_weight_min": -43.84083938598633,
      "activations/layer9_attention_weight_max": 41.903385162353516,
      "activations/layer9_attention_weight_min": -35.73915100097656,
      "epoch": 4.4,
      "learning_rate": 0.00012283219696969697,
      "loss": 2.8588,
      "step": 75800
    },
    {
      "activations/layer0_attention_weight_max": 16.429214477539062,
      "activations/layer0_attention_weight_min": -12.769491195678711,
      "activations/layer10_attention_weight_max": 37.60541534423828,
      "activations/layer10_attention_weight_min": -30.88121795654297,
      "activations/layer11_attention_weight_max": 35.49537658691406,
      "activations/layer11_attention_weight_min": -27.44512176513672,
      "activations/layer12_attention_weight_max": 27.00857925415039,
      "activations/layer12_attention_weight_min": -26.105100631713867,
      "activations/layer13_attention_weight_max": 48.000728607177734,
      "activations/layer13_attention_weight_min": -35.85480499267578,
      "activations/layer14_attention_weight_max": 80.5108413696289,
      "activations/layer14_attention_weight_min": -56.3552360534668,
      "activations/layer15_attention_weight_max": 45.61362075805664,
      "activations/layer15_attention_weight_min": -28.86431884765625,
      "activations/layer16_attention_weight_max": 44.250892639160156,
      "activations/layer16_attention_weight_min": -27.80181884765625,
      "activations/layer17_attention_weight_max": 42.9627685546875,
      "activations/layer17_attention_weight_min": -26.393138885498047,
      "activations/layer18_attention_weight_max": 37.365318298339844,
      "activations/layer18_attention_weight_min": -21.748933792114258,
      "activations/layer19_attention_weight_max": 46.64379119873047,
      "activations/layer19_attention_weight_min": -28.8082275390625,
      "activations/layer1_attention_weight_max": 18.086368560791016,
      "activations/layer1_attention_weight_min": -17.523174285888672,
      "activations/layer20_attention_weight_max": 35.09635543823242,
      "activations/layer20_attention_weight_min": -24.633623123168945,
      "activations/layer21_attention_weight_max": 33.49515151977539,
      "activations/layer21_attention_weight_min": -21.094757080078125,
      "activations/layer22_attention_weight_max": 51.27806091308594,
      "activations/layer22_attention_weight_min": -29.20564079284668,
      "activations/layer23_attention_weight_max": 34.43983459472656,
      "activations/layer23_attention_weight_min": -19.178565979003906,
      "activations/layer2_attention_weight_max": 33.64089584350586,
      "activations/layer2_attention_weight_min": -31.052879333496094,
      "activations/layer3_attention_weight_max": 93.11922454833984,
      "activations/layer3_attention_weight_min": -90.64286041259766,
      "activations/layer4_attention_weight_max": 136.49269104003906,
      "activations/layer4_attention_weight_min": -124.9707260131836,
      "activations/layer5_attention_weight_max": 89.48873138427734,
      "activations/layer5_attention_weight_min": -84.0777587890625,
      "activations/layer6_attention_weight_max": 57.218048095703125,
      "activations/layer6_attention_weight_min": -55.6156120300293,
      "activations/layer7_attention_weight_max": 82.40943145751953,
      "activations/layer7_attention_weight_min": -77.8227310180664,
      "activations/layer8_attention_weight_max": 41.312644958496094,
      "activations/layer8_attention_weight_min": -40.40489959716797,
      "activations/layer9_attention_weight_max": 37.738059997558594,
      "activations/layer9_attention_weight_min": -33.390926361083984,
      "epoch": 4.41,
      "learning_rate": 0.00012281325757575756,
      "loss": 2.8658,
      "step": 75850
    },
    {
      "activations/layer0_attention_weight_max": 14.348713874816895,
      "activations/layer0_attention_weight_min": -11.59800910949707,
      "activations/layer10_attention_weight_max": 35.9787483215332,
      "activations/layer10_attention_weight_min": -28.887351989746094,
      "activations/layer11_attention_weight_max": 33.610130310058594,
      "activations/layer11_attention_weight_min": -26.89346694946289,
      "activations/layer12_attention_weight_max": 49.7700309753418,
      "activations/layer12_attention_weight_min": -28.449417114257812,
      "activations/layer13_attention_weight_max": 55.762245178222656,
      "activations/layer13_attention_weight_min": -33.168296813964844,
      "activations/layer14_attention_weight_max": 71.82223510742188,
      "activations/layer14_attention_weight_min": -45.98637008666992,
      "activations/layer15_attention_weight_max": 51.41982650756836,
      "activations/layer15_attention_weight_min": -28.925195693969727,
      "activations/layer16_attention_weight_max": 45.8014030456543,
      "activations/layer16_attention_weight_min": -27.374479293823242,
      "activations/layer17_attention_weight_max": 52.59309005737305,
      "activations/layer17_attention_weight_min": -25.823667526245117,
      "activations/layer18_attention_weight_max": 36.006771087646484,
      "activations/layer18_attention_weight_min": -19.682079315185547,
      "activations/layer19_attention_weight_max": 50.51313018798828,
      "activations/layer19_attention_weight_min": -30.158496856689453,
      "activations/layer1_attention_weight_max": 17.259140014648438,
      "activations/layer1_attention_weight_min": -13.907424926757812,
      "activations/layer20_attention_weight_max": 42.34505081176758,
      "activations/layer20_attention_weight_min": -22.920001983642578,
      "activations/layer21_attention_weight_max": 38.50726318359375,
      "activations/layer21_attention_weight_min": -21.071802139282227,
      "activations/layer22_attention_weight_max": 57.68269348144531,
      "activations/layer22_attention_weight_min": -28.977754592895508,
      "activations/layer23_attention_weight_max": 35.600677490234375,
      "activations/layer23_attention_weight_min": -19.390714645385742,
      "activations/layer2_attention_weight_max": 32.03076934814453,
      "activations/layer2_attention_weight_min": -31.42726707458496,
      "activations/layer3_attention_weight_max": 92.17830657958984,
      "activations/layer3_attention_weight_min": -93.3692855834961,
      "activations/layer4_attention_weight_max": 131.0962371826172,
      "activations/layer4_attention_weight_min": -126.58670806884766,
      "activations/layer5_attention_weight_max": 86.9547348022461,
      "activations/layer5_attention_weight_min": -82.9046859741211,
      "activations/layer6_attention_weight_max": 56.358097076416016,
      "activations/layer6_attention_weight_min": -55.78289794921875,
      "activations/layer7_attention_weight_max": 78.0816421508789,
      "activations/layer7_attention_weight_min": -77.2718276977539,
      "activations/layer8_attention_weight_max": 37.447574615478516,
      "activations/layer8_attention_weight_min": -38.3442268371582,
      "activations/layer9_attention_weight_max": 39.19221115112305,
      "activations/layer9_attention_weight_min": -33.39493942260742,
      "epoch": 4.41,
      "learning_rate": 0.00012279431818181818,
      "loss": 2.8617,
      "step": 75900
    },
    {
      "activations/layer0_attention_weight_max": 16.31764030456543,
      "activations/layer0_attention_weight_min": -11.323091506958008,
      "activations/layer10_attention_weight_max": 35.99434280395508,
      "activations/layer10_attention_weight_min": -33.165130615234375,
      "activations/layer11_attention_weight_max": 36.458282470703125,
      "activations/layer11_attention_weight_min": -26.722349166870117,
      "activations/layer12_attention_weight_max": 51.3781623840332,
      "activations/layer12_attention_weight_min": -23.685195922851562,
      "activations/layer13_attention_weight_max": 57.1450080871582,
      "activations/layer13_attention_weight_min": -35.78042221069336,
      "activations/layer14_attention_weight_max": 69.136474609375,
      "activations/layer14_attention_weight_min": -44.7257080078125,
      "activations/layer15_attention_weight_max": 44.23138427734375,
      "activations/layer15_attention_weight_min": -29.25878143310547,
      "activations/layer16_attention_weight_max": 48.646366119384766,
      "activations/layer16_attention_weight_min": -27.844005584716797,
      "activations/layer17_attention_weight_max": 42.073570251464844,
      "activations/layer17_attention_weight_min": -24.68978500366211,
      "activations/layer18_attention_weight_max": 36.51812744140625,
      "activations/layer18_attention_weight_min": -21.117033004760742,
      "activations/layer19_attention_weight_max": 50.39863967895508,
      "activations/layer19_attention_weight_min": -30.726205825805664,
      "activations/layer1_attention_weight_max": 17.12078285217285,
      "activations/layer1_attention_weight_min": -14.225483894348145,
      "activations/layer20_attention_weight_max": 37.1123161315918,
      "activations/layer20_attention_weight_min": -23.30460548400879,
      "activations/layer21_attention_weight_max": 35.766998291015625,
      "activations/layer21_attention_weight_min": -22.3868465423584,
      "activations/layer22_attention_weight_max": 57.54690170288086,
      "activations/layer22_attention_weight_min": -29.07994842529297,
      "activations/layer23_attention_weight_max": 33.201629638671875,
      "activations/layer23_attention_weight_min": -19.351661682128906,
      "activations/layer2_attention_weight_max": 32.449161529541016,
      "activations/layer2_attention_weight_min": -32.28963088989258,
      "activations/layer3_attention_weight_max": 89.44548797607422,
      "activations/layer3_attention_weight_min": -90.06892395019531,
      "activations/layer4_attention_weight_max": 132.88560485839844,
      "activations/layer4_attention_weight_min": -127.47261810302734,
      "activations/layer5_attention_weight_max": 91.53018188476562,
      "activations/layer5_attention_weight_min": -81.93401336669922,
      "activations/layer6_attention_weight_max": 66.8819351196289,
      "activations/layer6_attention_weight_min": -59.71684265136719,
      "activations/layer7_attention_weight_max": 88.67320251464844,
      "activations/layer7_attention_weight_min": -79.62349700927734,
      "activations/layer8_attention_weight_max": 42.09242630004883,
      "activations/layer8_attention_weight_min": -41.240989685058594,
      "activations/layer9_attention_weight_max": 37.473358154296875,
      "activations/layer9_attention_weight_min": -33.722862243652344,
      "epoch": 4.41,
      "learning_rate": 0.00012277537878787876,
      "loss": 2.8607,
      "step": 75950
    },
    {
      "activations/layer0_attention_weight_max": 17.53166389465332,
      "activations/layer0_attention_weight_min": -12.823592185974121,
      "activations/layer10_attention_weight_max": 36.06212615966797,
      "activations/layer10_attention_weight_min": -31.070066452026367,
      "activations/layer11_attention_weight_max": 32.27815246582031,
      "activations/layer11_attention_weight_min": -26.679622650146484,
      "activations/layer12_attention_weight_max": 31.861515045166016,
      "activations/layer12_attention_weight_min": -26.461381912231445,
      "activations/layer13_attention_weight_max": 42.540550231933594,
      "activations/layer13_attention_weight_min": -33.59528350830078,
      "activations/layer14_attention_weight_max": 72.44541931152344,
      "activations/layer14_attention_weight_min": -55.33753967285156,
      "activations/layer15_attention_weight_max": 40.77215576171875,
      "activations/layer15_attention_weight_min": -28.904176712036133,
      "activations/layer16_attention_weight_max": 39.85176086425781,
      "activations/layer16_attention_weight_min": -28.453664779663086,
      "activations/layer17_attention_weight_max": 40.035743713378906,
      "activations/layer17_attention_weight_min": -23.0267276763916,
      "activations/layer18_attention_weight_max": 33.99359130859375,
      "activations/layer18_attention_weight_min": -20.64168930053711,
      "activations/layer19_attention_weight_max": 42.04198455810547,
      "activations/layer19_attention_weight_min": -30.544021606445312,
      "activations/layer1_attention_weight_max": 18.21122932434082,
      "activations/layer1_attention_weight_min": -16.562925338745117,
      "activations/layer20_attention_weight_max": 34.73398208618164,
      "activations/layer20_attention_weight_min": -24.40065574645996,
      "activations/layer21_attention_weight_max": 33.963809967041016,
      "activations/layer21_attention_weight_min": -21.71518325805664,
      "activations/layer22_attention_weight_max": 53.04423141479492,
      "activations/layer22_attention_weight_min": -30.655534744262695,
      "activations/layer23_attention_weight_max": 33.47706985473633,
      "activations/layer23_attention_weight_min": -21.918033599853516,
      "activations/layer2_attention_weight_max": 32.85615158081055,
      "activations/layer2_attention_weight_min": -31.2645320892334,
      "activations/layer3_attention_weight_max": 88.25514221191406,
      "activations/layer3_attention_weight_min": -89.65257263183594,
      "activations/layer4_attention_weight_max": 131.0289306640625,
      "activations/layer4_attention_weight_min": -127.30413818359375,
      "activations/layer5_attention_weight_max": 88.29405975341797,
      "activations/layer5_attention_weight_min": -83.81812286376953,
      "activations/layer6_attention_weight_max": 57.35716247558594,
      "activations/layer6_attention_weight_min": -60.89366912841797,
      "activations/layer7_attention_weight_max": 82.2978744506836,
      "activations/layer7_attention_weight_min": -74.47509002685547,
      "activations/layer8_attention_weight_max": 41.173946380615234,
      "activations/layer8_attention_weight_min": -39.47039794921875,
      "activations/layer9_attention_weight_max": 37.79164123535156,
      "activations/layer9_attention_weight_min": -36.60524368286133,
      "epoch": 4.42,
      "learning_rate": 0.00012275643939393938,
      "loss": 2.8524,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_loss": 2.81640625,
      "eval_runtime": 8.5185,
      "eval_samples_per_second": 504.079,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_openwebtext_loss": 2.81640625,
      "eval_openwebtext_ppl": 16.716667072215767,
      "eval_openwebtext_runtime": 8.5185,
      "eval_openwebtext_samples_per_second": 504.079,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_wikitext_loss": 3.064453125,
      "eval_wikitext_ppl": 21.42274322042474,
      "eval_wikitext_runtime": 1.9846,
      "eval_wikitext_samples_per_second": 229.764,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_lambada_loss": 2.84765625,
      "eval_lambada_ppl": 17.247311047115943,
      "eval_lambada_runtime": 9.5646,
      "eval_lambada_samples_per_second": 509.063,
      "step": 76000
    },
    {
      "activations/layer0_attention_weight_max": 17.448001861572266,
      "activations/layer0_attention_weight_min": -12.907307624816895,
      "activations/layer10_attention_weight_max": 35.20103073120117,
      "activations/layer10_attention_weight_min": -31.754253387451172,
      "activations/layer11_attention_weight_max": 35.036231994628906,
      "activations/layer11_attention_weight_min": -29.37586784362793,
      "activations/layer12_attention_weight_max": 43.13777160644531,
      "activations/layer12_attention_weight_min": -27.43333625793457,
      "activations/layer13_attention_weight_max": 51.65424728393555,
      "activations/layer13_attention_weight_min": -34.13456344604492,
      "activations/layer14_attention_weight_max": 79.12962341308594,
      "activations/layer14_attention_weight_min": -51.67742919921875,
      "activations/layer15_attention_weight_max": 43.07764434814453,
      "activations/layer15_attention_weight_min": -27.668249130249023,
      "activations/layer16_attention_weight_max": 40.84958267211914,
      "activations/layer16_attention_weight_min": -26.811752319335938,
      "activations/layer17_attention_weight_max": 39.56409454345703,
      "activations/layer17_attention_weight_min": -24.477869033813477,
      "activations/layer18_attention_weight_max": 35.51530838012695,
      "activations/layer18_attention_weight_min": -19.997831344604492,
      "activations/layer19_attention_weight_max": 44.583255767822266,
      "activations/layer19_attention_weight_min": -27.24164390563965,
      "activations/layer1_attention_weight_max": 17.900800704956055,
      "activations/layer1_attention_weight_min": -15.992576599121094,
      "activations/layer20_attention_weight_max": 31.62895393371582,
      "activations/layer20_attention_weight_min": -23.501256942749023,
      "activations/layer21_attention_weight_max": 33.14063262939453,
      "activations/layer21_attention_weight_min": -22.525739669799805,
      "activations/layer22_attention_weight_max": 59.13597869873047,
      "activations/layer22_attention_weight_min": -33.14439392089844,
      "activations/layer23_attention_weight_max": 32.07813262939453,
      "activations/layer23_attention_weight_min": -20.367280960083008,
      "activations/layer2_attention_weight_max": 32.87623977661133,
      "activations/layer2_attention_weight_min": -33.221290588378906,
      "activations/layer3_attention_weight_max": 93.50093078613281,
      "activations/layer3_attention_weight_min": -92.7635269165039,
      "activations/layer4_attention_weight_max": 133.53004455566406,
      "activations/layer4_attention_weight_min": -129.51348876953125,
      "activations/layer5_attention_weight_max": 93.47252655029297,
      "activations/layer5_attention_weight_min": -88.70643615722656,
      "activations/layer6_attention_weight_max": 58.23611831665039,
      "activations/layer6_attention_weight_min": -55.758262634277344,
      "activations/layer7_attention_weight_max": 82.49907684326172,
      "activations/layer7_attention_weight_min": -78.76111602783203,
      "activations/layer8_attention_weight_max": 40.579498291015625,
      "activations/layer8_attention_weight_min": -40.29686737060547,
      "activations/layer9_attention_weight_max": 39.49678421020508,
      "activations/layer9_attention_weight_min": -34.12781524658203,
      "epoch": 4.42,
      "learning_rate": 0.0001227375,
      "loss": 2.8477,
      "step": 76050
    },
    {
      "activations/layer0_attention_weight_max": 16.991960525512695,
      "activations/layer0_attention_weight_min": -12.247502326965332,
      "activations/layer10_attention_weight_max": 37.29416275024414,
      "activations/layer10_attention_weight_min": -29.578353881835938,
      "activations/layer11_attention_weight_max": 34.33019256591797,
      "activations/layer11_attention_weight_min": -30.409818649291992,
      "activations/layer12_attention_weight_max": 30.547243118286133,
      "activations/layer12_attention_weight_min": -35.95862579345703,
      "activations/layer13_attention_weight_max": 48.711631774902344,
      "activations/layer13_attention_weight_min": -33.56777572631836,
      "activations/layer14_attention_weight_max": 66.94185638427734,
      "activations/layer14_attention_weight_min": -47.93210983276367,
      "activations/layer15_attention_weight_max": 40.770206451416016,
      "activations/layer15_attention_weight_min": -28.021106719970703,
      "activations/layer16_attention_weight_max": 39.89333724975586,
      "activations/layer16_attention_weight_min": -27.658784866333008,
      "activations/layer17_attention_weight_max": 40.804847717285156,
      "activations/layer17_attention_weight_min": -25.983884811401367,
      "activations/layer18_attention_weight_max": 33.80826950073242,
      "activations/layer18_attention_weight_min": -22.552188873291016,
      "activations/layer19_attention_weight_max": 45.962867736816406,
      "activations/layer19_attention_weight_min": -28.3537654876709,
      "activations/layer1_attention_weight_max": 17.387157440185547,
      "activations/layer1_attention_weight_min": -14.945561408996582,
      "activations/layer20_attention_weight_max": 32.152530670166016,
      "activations/layer20_attention_weight_min": -24.17967987060547,
      "activations/layer21_attention_weight_max": 33.609344482421875,
      "activations/layer21_attention_weight_min": -20.097013473510742,
      "activations/layer22_attention_weight_max": 52.63750076293945,
      "activations/layer22_attention_weight_min": -29.692304611206055,
      "activations/layer23_attention_weight_max": 33.3525390625,
      "activations/layer23_attention_weight_min": -19.168865203857422,
      "activations/layer2_attention_weight_max": 30.487628936767578,
      "activations/layer2_attention_weight_min": -30.689573287963867,
      "activations/layer3_attention_weight_max": 86.74678039550781,
      "activations/layer3_attention_weight_min": -88.55889892578125,
      "activations/layer4_attention_weight_max": 125.32787322998047,
      "activations/layer4_attention_weight_min": -122.31412506103516,
      "activations/layer5_attention_weight_max": 81.25621032714844,
      "activations/layer5_attention_weight_min": -79.2626953125,
      "activations/layer6_attention_weight_max": 56.19721603393555,
      "activations/layer6_attention_weight_min": -53.00953674316406,
      "activations/layer7_attention_weight_max": 73.81745147705078,
      "activations/layer7_attention_weight_min": -73.5201187133789,
      "activations/layer8_attention_weight_max": 36.50774383544922,
      "activations/layer8_attention_weight_min": -38.796226501464844,
      "activations/layer9_attention_weight_max": 35.60187911987305,
      "activations/layer9_attention_weight_min": -33.90571594238281,
      "epoch": 4.42,
      "learning_rate": 0.00012271856060606058,
      "loss": 2.8568,
      "step": 76100
    },
    {
      "activations/layer0_attention_weight_max": 16.307275772094727,
      "activations/layer0_attention_weight_min": -11.282622337341309,
      "activations/layer10_attention_weight_max": 38.397300720214844,
      "activations/layer10_attention_weight_min": -30.97922134399414,
      "activations/layer11_attention_weight_max": 36.9326057434082,
      "activations/layer11_attention_weight_min": -27.82265853881836,
      "activations/layer12_attention_weight_max": 66.41143798828125,
      "activations/layer12_attention_weight_min": -28.995315551757812,
      "activations/layer13_attention_weight_max": 49.358951568603516,
      "activations/layer13_attention_weight_min": -36.41465377807617,
      "activations/layer14_attention_weight_max": 66.20246124267578,
      "activations/layer14_attention_weight_min": -47.61151885986328,
      "activations/layer15_attention_weight_max": 41.819480895996094,
      "activations/layer15_attention_weight_min": -30.626928329467773,
      "activations/layer16_attention_weight_max": 40.5144157409668,
      "activations/layer16_attention_weight_min": -28.968158721923828,
      "activations/layer17_attention_weight_max": 41.91347122192383,
      "activations/layer17_attention_weight_min": -26.7742862701416,
      "activations/layer18_attention_weight_max": 35.39704513549805,
      "activations/layer18_attention_weight_min": -19.455848693847656,
      "activations/layer19_attention_weight_max": 41.831172943115234,
      "activations/layer19_attention_weight_min": -28.44529151916504,
      "activations/layer1_attention_weight_max": 18.074146270751953,
      "activations/layer1_attention_weight_min": -15.218087196350098,
      "activations/layer20_attention_weight_max": 33.2767219543457,
      "activations/layer20_attention_weight_min": -22.977449417114258,
      "activations/layer21_attention_weight_max": 33.20658493041992,
      "activations/layer21_attention_weight_min": -21.592437744140625,
      "activations/layer22_attention_weight_max": 51.27735137939453,
      "activations/layer22_attention_weight_min": -25.891334533691406,
      "activations/layer23_attention_weight_max": 36.32350540161133,
      "activations/layer23_attention_weight_min": -20.098892211914062,
      "activations/layer2_attention_weight_max": 33.74182891845703,
      "activations/layer2_attention_weight_min": -32.88132095336914,
      "activations/layer3_attention_weight_max": 94.63040161132812,
      "activations/layer3_attention_weight_min": -97.7132568359375,
      "activations/layer4_attention_weight_max": 133.2884979248047,
      "activations/layer4_attention_weight_min": -131.8929901123047,
      "activations/layer5_attention_weight_max": 91.67576599121094,
      "activations/layer5_attention_weight_min": -87.23198699951172,
      "activations/layer6_attention_weight_max": 58.563575744628906,
      "activations/layer6_attention_weight_min": -61.5316276550293,
      "activations/layer7_attention_weight_max": 83.91587829589844,
      "activations/layer7_attention_weight_min": -77.76825714111328,
      "activations/layer8_attention_weight_max": 40.10393142700195,
      "activations/layer8_attention_weight_min": -40.075443267822266,
      "activations/layer9_attention_weight_max": 39.9868049621582,
      "activations/layer9_attention_weight_min": -34.02578353881836,
      "epoch": 4.42,
      "learning_rate": 0.0001226996212121212,
      "loss": 2.8653,
      "step": 76150
    },
    {
      "activations/layer0_attention_weight_max": 16.436946868896484,
      "activations/layer0_attention_weight_min": -11.373174667358398,
      "activations/layer10_attention_weight_max": 36.06703186035156,
      "activations/layer10_attention_weight_min": -30.40427017211914,
      "activations/layer11_attention_weight_max": 33.01322937011719,
      "activations/layer11_attention_weight_min": -27.730886459350586,
      "activations/layer12_attention_weight_max": 24.32554054260254,
      "activations/layer12_attention_weight_min": -27.83157730102539,
      "activations/layer13_attention_weight_max": 42.79710388183594,
      "activations/layer13_attention_weight_min": -30.27908706665039,
      "activations/layer14_attention_weight_max": 69.13438415527344,
      "activations/layer14_attention_weight_min": -52.66061782836914,
      "activations/layer15_attention_weight_max": 50.3056640625,
      "activations/layer15_attention_weight_min": -32.2212028503418,
      "activations/layer16_attention_weight_max": 39.47797393798828,
      "activations/layer16_attention_weight_min": -26.964719772338867,
      "activations/layer17_attention_weight_max": 42.24228286743164,
      "activations/layer17_attention_weight_min": -24.977611541748047,
      "activations/layer18_attention_weight_max": 36.586151123046875,
      "activations/layer18_attention_weight_min": -23.82442855834961,
      "activations/layer19_attention_weight_max": 44.9715690612793,
      "activations/layer19_attention_weight_min": -29.29617691040039,
      "activations/layer1_attention_weight_max": 17.00634002685547,
      "activations/layer1_attention_weight_min": -14.465142250061035,
      "activations/layer20_attention_weight_max": 34.09473419189453,
      "activations/layer20_attention_weight_min": -22.876705169677734,
      "activations/layer21_attention_weight_max": 33.68646240234375,
      "activations/layer21_attention_weight_min": -18.404611587524414,
      "activations/layer22_attention_weight_max": 52.63662338256836,
      "activations/layer22_attention_weight_min": -27.706201553344727,
      "activations/layer23_attention_weight_max": 31.072948455810547,
      "activations/layer23_attention_weight_min": -16.79322052001953,
      "activations/layer2_attention_weight_max": 33.44951629638672,
      "activations/layer2_attention_weight_min": -31.243488311767578,
      "activations/layer3_attention_weight_max": 88.68660736083984,
      "activations/layer3_attention_weight_min": -87.64350128173828,
      "activations/layer4_attention_weight_max": 135.7285919189453,
      "activations/layer4_attention_weight_min": -127.17138671875,
      "activations/layer5_attention_weight_max": 95.62828826904297,
      "activations/layer5_attention_weight_min": -85.19137573242188,
      "activations/layer6_attention_weight_max": 60.11873245239258,
      "activations/layer6_attention_weight_min": -59.53046798706055,
      "activations/layer7_attention_weight_max": 85.57135009765625,
      "activations/layer7_attention_weight_min": -79.53092956542969,
      "activations/layer8_attention_weight_max": 42.041297912597656,
      "activations/layer8_attention_weight_min": -40.61775588989258,
      "activations/layer9_attention_weight_max": 37.92882537841797,
      "activations/layer9_attention_weight_min": -37.20637893676758,
      "epoch": 4.43,
      "learning_rate": 0.0001226806818181818,
      "loss": 2.85,
      "step": 76200
    },
    {
      "activations/layer0_attention_weight_max": 15.783269882202148,
      "activations/layer0_attention_weight_min": -12.823491096496582,
      "activations/layer10_attention_weight_max": 35.728271484375,
      "activations/layer10_attention_weight_min": -31.97691535949707,
      "activations/layer11_attention_weight_max": 37.009334564208984,
      "activations/layer11_attention_weight_min": -28.397008895874023,
      "activations/layer12_attention_weight_max": 48.76837921142578,
      "activations/layer12_attention_weight_min": -26.09602165222168,
      "activations/layer13_attention_weight_max": 50.11384582519531,
      "activations/layer13_attention_weight_min": -33.86328887939453,
      "activations/layer14_attention_weight_max": 87.35530090332031,
      "activations/layer14_attention_weight_min": -61.137264251708984,
      "activations/layer15_attention_weight_max": 42.928741455078125,
      "activations/layer15_attention_weight_min": -28.07515525817871,
      "activations/layer16_attention_weight_max": 42.339569091796875,
      "activations/layer16_attention_weight_min": -28.328649520874023,
      "activations/layer17_attention_weight_max": 41.13125991821289,
      "activations/layer17_attention_weight_min": -24.444992065429688,
      "activations/layer18_attention_weight_max": 34.66712188720703,
      "activations/layer18_attention_weight_min": -19.342025756835938,
      "activations/layer19_attention_weight_max": 46.84481430053711,
      "activations/layer19_attention_weight_min": -29.812353134155273,
      "activations/layer1_attention_weight_max": 17.39940071105957,
      "activations/layer1_attention_weight_min": -13.514697074890137,
      "activations/layer20_attention_weight_max": 35.399959564208984,
      "activations/layer20_attention_weight_min": -23.316478729248047,
      "activations/layer21_attention_weight_max": 33.11072540283203,
      "activations/layer21_attention_weight_min": -21.249269485473633,
      "activations/layer22_attention_weight_max": 52.329872131347656,
      "activations/layer22_attention_weight_min": -28.20566177368164,
      "activations/layer23_attention_weight_max": 34.74539566040039,
      "activations/layer23_attention_weight_min": -20.526153564453125,
      "activations/layer2_attention_weight_max": 32.762451171875,
      "activations/layer2_attention_weight_min": -31.43079948425293,
      "activations/layer3_attention_weight_max": 89.16532135009766,
      "activations/layer3_attention_weight_min": -91.68778228759766,
      "activations/layer4_attention_weight_max": 131.57847595214844,
      "activations/layer4_attention_weight_min": -131.7102508544922,
      "activations/layer5_attention_weight_max": 93.26730346679688,
      "activations/layer5_attention_weight_min": -84.1452865600586,
      "activations/layer6_attention_weight_max": 58.851051330566406,
      "activations/layer6_attention_weight_min": -57.05193328857422,
      "activations/layer7_attention_weight_max": 81.36863708496094,
      "activations/layer7_attention_weight_min": -84.03611755371094,
      "activations/layer8_attention_weight_max": 40.16585922241211,
      "activations/layer8_attention_weight_min": -40.924495697021484,
      "activations/layer9_attention_weight_max": 36.41104507446289,
      "activations/layer9_attention_weight_min": -33.316802978515625,
      "epoch": 4.43,
      "learning_rate": 0.0001226617424242424,
      "loss": 2.8723,
      "step": 76250
    },
    {
      "activations/layer0_attention_weight_max": 14.067530632019043,
      "activations/layer0_attention_weight_min": -12.325400352478027,
      "activations/layer10_attention_weight_max": 35.228580474853516,
      "activations/layer10_attention_weight_min": -30.008296966552734,
      "activations/layer11_attention_weight_max": 39.995445251464844,
      "activations/layer11_attention_weight_min": -28.791854858398438,
      "activations/layer12_attention_weight_max": 45.61834716796875,
      "activations/layer12_attention_weight_min": -28.77878761291504,
      "activations/layer13_attention_weight_max": 49.571109771728516,
      "activations/layer13_attention_weight_min": -33.69533920288086,
      "activations/layer14_attention_weight_max": 62.232784271240234,
      "activations/layer14_attention_weight_min": -47.09041213989258,
      "activations/layer15_attention_weight_max": 47.008697509765625,
      "activations/layer15_attention_weight_min": -27.2681884765625,
      "activations/layer16_attention_weight_max": 44.10464859008789,
      "activations/layer16_attention_weight_min": -27.729812622070312,
      "activations/layer17_attention_weight_max": 41.581947326660156,
      "activations/layer17_attention_weight_min": -24.36195182800293,
      "activations/layer18_attention_weight_max": 39.3966178894043,
      "activations/layer18_attention_weight_min": -22.198747634887695,
      "activations/layer19_attention_weight_max": 40.26205062866211,
      "activations/layer19_attention_weight_min": -27.8892765045166,
      "activations/layer1_attention_weight_max": 19.0574951171875,
      "activations/layer1_attention_weight_min": -16.157039642333984,
      "activations/layer20_attention_weight_max": 36.77179718017578,
      "activations/layer20_attention_weight_min": -25.60634994506836,
      "activations/layer21_attention_weight_max": 40.521095275878906,
      "activations/layer21_attention_weight_min": -21.061721801757812,
      "activations/layer22_attention_weight_max": 52.89478302001953,
      "activations/layer22_attention_weight_min": -27.0079402923584,
      "activations/layer23_attention_weight_max": 30.866605758666992,
      "activations/layer23_attention_weight_min": -18.58975601196289,
      "activations/layer2_attention_weight_max": 33.37786102294922,
      "activations/layer2_attention_weight_min": -32.229652404785156,
      "activations/layer3_attention_weight_max": 90.72606658935547,
      "activations/layer3_attention_weight_min": -91.56398010253906,
      "activations/layer4_attention_weight_max": 137.18109130859375,
      "activations/layer4_attention_weight_min": -133.31698608398438,
      "activations/layer5_attention_weight_max": 90.84410858154297,
      "activations/layer5_attention_weight_min": -81.52838134765625,
      "activations/layer6_attention_weight_max": 58.45582580566406,
      "activations/layer6_attention_weight_min": -56.60270309448242,
      "activations/layer7_attention_weight_max": 89.25987243652344,
      "activations/layer7_attention_weight_min": -78.53492736816406,
      "activations/layer8_attention_weight_max": 43.09067916870117,
      "activations/layer8_attention_weight_min": -38.5723762512207,
      "activations/layer9_attention_weight_max": 39.415340423583984,
      "activations/layer9_attention_weight_min": -37.54670715332031,
      "epoch": 4.43,
      "learning_rate": 0.00012264280303030302,
      "loss": 2.8545,
      "step": 76300
    },
    {
      "activations/layer0_attention_weight_max": 16.399303436279297,
      "activations/layer0_attention_weight_min": -11.546258926391602,
      "activations/layer10_attention_weight_max": 34.113956451416016,
      "activations/layer10_attention_weight_min": -30.825368881225586,
      "activations/layer11_attention_weight_max": 32.990753173828125,
      "activations/layer11_attention_weight_min": -27.709693908691406,
      "activations/layer12_attention_weight_max": 34.323551177978516,
      "activations/layer12_attention_weight_min": -28.01213836669922,
      "activations/layer13_attention_weight_max": 57.30413818359375,
      "activations/layer13_attention_weight_min": -35.71061325073242,
      "activations/layer14_attention_weight_max": 69.72161102294922,
      "activations/layer14_attention_weight_min": -46.940128326416016,
      "activations/layer15_attention_weight_max": 55.41102981567383,
      "activations/layer15_attention_weight_min": -33.211273193359375,
      "activations/layer16_attention_weight_max": 44.522029876708984,
      "activations/layer16_attention_weight_min": -31.903141021728516,
      "activations/layer17_attention_weight_max": 47.59048843383789,
      "activations/layer17_attention_weight_min": -25.3045711517334,
      "activations/layer18_attention_weight_max": 32.544124603271484,
      "activations/layer18_attention_weight_min": -21.763507843017578,
      "activations/layer19_attention_weight_max": 44.93818664550781,
      "activations/layer19_attention_weight_min": -28.835529327392578,
      "activations/layer1_attention_weight_max": 17.557844161987305,
      "activations/layer1_attention_weight_min": -13.548951148986816,
      "activations/layer20_attention_weight_max": 33.67314529418945,
      "activations/layer20_attention_weight_min": -22.920326232910156,
      "activations/layer21_attention_weight_max": 34.52811813354492,
      "activations/layer21_attention_weight_min": -20.58319664001465,
      "activations/layer22_attention_weight_max": 54.75564193725586,
      "activations/layer22_attention_weight_min": -27.809175491333008,
      "activations/layer23_attention_weight_max": 35.45336151123047,
      "activations/layer23_attention_weight_min": -22.21620750427246,
      "activations/layer2_attention_weight_max": 32.09066390991211,
      "activations/layer2_attention_weight_min": -32.74753952026367,
      "activations/layer3_attention_weight_max": 88.22904968261719,
      "activations/layer3_attention_weight_min": -88.24018096923828,
      "activations/layer4_attention_weight_max": 128.66477966308594,
      "activations/layer4_attention_weight_min": -127.05256652832031,
      "activations/layer5_attention_weight_max": 83.27690124511719,
      "activations/layer5_attention_weight_min": -83.34042358398438,
      "activations/layer6_attention_weight_max": 57.178672790527344,
      "activations/layer6_attention_weight_min": -55.89582824707031,
      "activations/layer7_attention_weight_max": 83.69319152832031,
      "activations/layer7_attention_weight_min": -78.7296371459961,
      "activations/layer8_attention_weight_max": 41.52278137207031,
      "activations/layer8_attention_weight_min": -41.044395446777344,
      "activations/layer9_attention_weight_max": 38.82238006591797,
      "activations/layer9_attention_weight_min": -36.664974212646484,
      "epoch": 4.44,
      "learning_rate": 0.0001226238636363636,
      "loss": 2.8625,
      "step": 76350
    },
    {
      "activations/layer0_attention_weight_max": 15.207670211791992,
      "activations/layer0_attention_weight_min": -12.908367156982422,
      "activations/layer10_attention_weight_max": 43.49192810058594,
      "activations/layer10_attention_weight_min": -32.79282760620117,
      "activations/layer11_attention_weight_max": 48.648338317871094,
      "activations/layer11_attention_weight_min": -31.800987243652344,
      "activations/layer12_attention_weight_max": 70.92009735107422,
      "activations/layer12_attention_weight_min": -34.346099853515625,
      "activations/layer13_attention_weight_max": 72.47860717773438,
      "activations/layer13_attention_weight_min": -45.964237213134766,
      "activations/layer14_attention_weight_max": 75.39447784423828,
      "activations/layer14_attention_weight_min": -50.34687805175781,
      "activations/layer15_attention_weight_max": 56.237274169921875,
      "activations/layer15_attention_weight_min": -33.55970764160156,
      "activations/layer16_attention_weight_max": 41.590003967285156,
      "activations/layer16_attention_weight_min": -27.430896759033203,
      "activations/layer17_attention_weight_max": 39.65331268310547,
      "activations/layer17_attention_weight_min": -22.62249183654785,
      "activations/layer18_attention_weight_max": 33.05085372924805,
      "activations/layer18_attention_weight_min": -18.482833862304688,
      "activations/layer19_attention_weight_max": 45.462806701660156,
      "activations/layer19_attention_weight_min": -27.065052032470703,
      "activations/layer1_attention_weight_max": 17.15799903869629,
      "activations/layer1_attention_weight_min": -16.5074405670166,
      "activations/layer20_attention_weight_max": 35.22916030883789,
      "activations/layer20_attention_weight_min": -21.763195037841797,
      "activations/layer21_attention_weight_max": 31.380027770996094,
      "activations/layer21_attention_weight_min": -17.199176788330078,
      "activations/layer22_attention_weight_max": 52.9174690246582,
      "activations/layer22_attention_weight_min": -27.57545280456543,
      "activations/layer23_attention_weight_max": 31.21080780029297,
      "activations/layer23_attention_weight_min": -19.63134765625,
      "activations/layer2_attention_weight_max": 32.93335723876953,
      "activations/layer2_attention_weight_min": -32.57509231567383,
      "activations/layer3_attention_weight_max": 92.03433990478516,
      "activations/layer3_attention_weight_min": -94.02195739746094,
      "activations/layer4_attention_weight_max": 134.86643981933594,
      "activations/layer4_attention_weight_min": -131.3096466064453,
      "activations/layer5_attention_weight_max": 87.36781311035156,
      "activations/layer5_attention_weight_min": -86.58512878417969,
      "activations/layer6_attention_weight_max": 61.12297058105469,
      "activations/layer6_attention_weight_min": -55.17698669433594,
      "activations/layer7_attention_weight_max": 108.71522521972656,
      "activations/layer7_attention_weight_min": -85.33879089355469,
      "activations/layer8_attention_weight_max": 47.59437942504883,
      "activations/layer8_attention_weight_min": -42.356590270996094,
      "activations/layer9_attention_weight_max": 47.3283805847168,
      "activations/layer9_attention_weight_min": -52.851898193359375,
      "epoch": 4.44,
      "learning_rate": 0.00012260492424242422,
      "loss": 2.8642,
      "step": 76400
    },
    {
      "activations/layer0_attention_weight_max": 16.934307098388672,
      "activations/layer0_attention_weight_min": -12.669927597045898,
      "activations/layer10_attention_weight_max": 36.28926467895508,
      "activations/layer10_attention_weight_min": -30.6113338470459,
      "activations/layer11_attention_weight_max": 31.96529197692871,
      "activations/layer11_attention_weight_min": -27.204389572143555,
      "activations/layer12_attention_weight_max": 33.81470489501953,
      "activations/layer12_attention_weight_min": -26.941341400146484,
      "activations/layer13_attention_weight_max": 49.94078063964844,
      "activations/layer13_attention_weight_min": -30.517303466796875,
      "activations/layer14_attention_weight_max": 70.95223999023438,
      "activations/layer14_attention_weight_min": -50.099647521972656,
      "activations/layer15_attention_weight_max": 51.25896453857422,
      "activations/layer15_attention_weight_min": -30.627012252807617,
      "activations/layer16_attention_weight_max": 43.906005859375,
      "activations/layer16_attention_weight_min": -29.398950576782227,
      "activations/layer17_attention_weight_max": 42.12052917480469,
      "activations/layer17_attention_weight_min": -24.94428253173828,
      "activations/layer18_attention_weight_max": 31.976694107055664,
      "activations/layer18_attention_weight_min": -19.18236541748047,
      "activations/layer19_attention_weight_max": 45.00167465209961,
      "activations/layer19_attention_weight_min": -28.16306495666504,
      "activations/layer1_attention_weight_max": 18.05742835998535,
      "activations/layer1_attention_weight_min": -15.792328834533691,
      "activations/layer20_attention_weight_max": 36.573143005371094,
      "activations/layer20_attention_weight_min": -22.88139533996582,
      "activations/layer21_attention_weight_max": 36.781494140625,
      "activations/layer21_attention_weight_min": -21.798797607421875,
      "activations/layer22_attention_weight_max": 53.316158294677734,
      "activations/layer22_attention_weight_min": -27.17377471923828,
      "activations/layer23_attention_weight_max": 37.52330017089844,
      "activations/layer23_attention_weight_min": -18.435081481933594,
      "activations/layer2_attention_weight_max": 32.20430374145508,
      "activations/layer2_attention_weight_min": -31.612255096435547,
      "activations/layer3_attention_weight_max": 91.58223724365234,
      "activations/layer3_attention_weight_min": -90.69389343261719,
      "activations/layer4_attention_weight_max": 131.0430145263672,
      "activations/layer4_attention_weight_min": -131.4341583251953,
      "activations/layer5_attention_weight_max": 86.69041442871094,
      "activations/layer5_attention_weight_min": -82.12271881103516,
      "activations/layer6_attention_weight_max": 56.932044982910156,
      "activations/layer6_attention_weight_min": -56.23662567138672,
      "activations/layer7_attention_weight_max": 79.27259063720703,
      "activations/layer7_attention_weight_min": -78.05989074707031,
      "activations/layer8_attention_weight_max": 39.558834075927734,
      "activations/layer8_attention_weight_min": -37.264251708984375,
      "activations/layer9_attention_weight_max": 35.760955810546875,
      "activations/layer9_attention_weight_min": -33.286861419677734,
      "epoch": 4.44,
      "learning_rate": 0.00012258598484848483,
      "loss": 2.8606,
      "step": 76450
    },
    {
      "activations/layer0_attention_weight_max": 15.789783477783203,
      "activations/layer0_attention_weight_min": -12.171308517456055,
      "activations/layer10_attention_weight_max": 34.045257568359375,
      "activations/layer10_attention_weight_min": -30.76073455810547,
      "activations/layer11_attention_weight_max": 34.133758544921875,
      "activations/layer11_attention_weight_min": -28.49323081970215,
      "activations/layer12_attention_weight_max": 35.87824630737305,
      "activations/layer12_attention_weight_min": -26.304887771606445,
      "activations/layer13_attention_weight_max": 49.07929992675781,
      "activations/layer13_attention_weight_min": -37.71345520019531,
      "activations/layer14_attention_weight_max": 71.620849609375,
      "activations/layer14_attention_weight_min": -50.22268295288086,
      "activations/layer15_attention_weight_max": 39.584224700927734,
      "activations/layer15_attention_weight_min": -28.26947784423828,
      "activations/layer16_attention_weight_max": 40.13669967651367,
      "activations/layer16_attention_weight_min": -28.569339752197266,
      "activations/layer17_attention_weight_max": 40.19862747192383,
      "activations/layer17_attention_weight_min": -24.114017486572266,
      "activations/layer18_attention_weight_max": 34.73918533325195,
      "activations/layer18_attention_weight_min": -19.12258529663086,
      "activations/layer19_attention_weight_max": 42.81402587890625,
      "activations/layer19_attention_weight_min": -28.778228759765625,
      "activations/layer1_attention_weight_max": 17.027402877807617,
      "activations/layer1_attention_weight_min": -14.425864219665527,
      "activations/layer20_attention_weight_max": 37.187286376953125,
      "activations/layer20_attention_weight_min": -21.831111907958984,
      "activations/layer21_attention_weight_max": 35.34773635864258,
      "activations/layer21_attention_weight_min": -19.368581771850586,
      "activations/layer22_attention_weight_max": 50.55501937866211,
      "activations/layer22_attention_weight_min": -28.505979537963867,
      "activations/layer23_attention_weight_max": 33.056129455566406,
      "activations/layer23_attention_weight_min": -20.03281021118164,
      "activations/layer2_attention_weight_max": 32.25189971923828,
      "activations/layer2_attention_weight_min": -32.70585250854492,
      "activations/layer3_attention_weight_max": 86.86282348632812,
      "activations/layer3_attention_weight_min": -88.11786651611328,
      "activations/layer4_attention_weight_max": 127.10665130615234,
      "activations/layer4_attention_weight_min": -124.8396987915039,
      "activations/layer5_attention_weight_max": 84.57913970947266,
      "activations/layer5_attention_weight_min": -82.22360229492188,
      "activations/layer6_attention_weight_max": 56.176177978515625,
      "activations/layer6_attention_weight_min": -58.851463317871094,
      "activations/layer7_attention_weight_max": 81.9926986694336,
      "activations/layer7_attention_weight_min": -80.6822738647461,
      "activations/layer8_attention_weight_max": 38.06159973144531,
      "activations/layer8_attention_weight_min": -40.07037353515625,
      "activations/layer9_attention_weight_max": 37.049232482910156,
      "activations/layer9_attention_weight_min": -35.18071365356445,
      "epoch": 4.45,
      "learning_rate": 0.00012256704545454545,
      "loss": 2.8596,
      "step": 76500
    },
    {
      "activations/layer0_attention_weight_max": 15.519392013549805,
      "activations/layer0_attention_weight_min": -12.881195068359375,
      "activations/layer10_attention_weight_max": 37.190757751464844,
      "activations/layer10_attention_weight_min": -30.74526596069336,
      "activations/layer11_attention_weight_max": 40.87339401245117,
      "activations/layer11_attention_weight_min": -31.850706100463867,
      "activations/layer12_attention_weight_max": 35.668495178222656,
      "activations/layer12_attention_weight_min": -34.16252899169922,
      "activations/layer13_attention_weight_max": 50.283653259277344,
      "activations/layer13_attention_weight_min": -37.29246520996094,
      "activations/layer14_attention_weight_max": 83.10842895507812,
      "activations/layer14_attention_weight_min": -59.562103271484375,
      "activations/layer15_attention_weight_max": 48.19697570800781,
      "activations/layer15_attention_weight_min": -31.12340545654297,
      "activations/layer16_attention_weight_max": 41.265865325927734,
      "activations/layer16_attention_weight_min": -28.363000869750977,
      "activations/layer17_attention_weight_max": 42.02859115600586,
      "activations/layer17_attention_weight_min": -26.36577796936035,
      "activations/layer18_attention_weight_max": 30.078655242919922,
      "activations/layer18_attention_weight_min": -19.605802536010742,
      "activations/layer19_attention_weight_max": 39.56134033203125,
      "activations/layer19_attention_weight_min": -29.108917236328125,
      "activations/layer1_attention_weight_max": 17.65239906311035,
      "activations/layer1_attention_weight_min": -15.105615615844727,
      "activations/layer20_attention_weight_max": 34.60795974731445,
      "activations/layer20_attention_weight_min": -22.20509147644043,
      "activations/layer21_attention_weight_max": 32.769935607910156,
      "activations/layer21_attention_weight_min": -18.50534439086914,
      "activations/layer22_attention_weight_max": 52.63631057739258,
      "activations/layer22_attention_weight_min": -25.19361114501953,
      "activations/layer23_attention_weight_max": 28.55731964111328,
      "activations/layer23_attention_weight_min": -16.72958755493164,
      "activations/layer2_attention_weight_max": 30.323009490966797,
      "activations/layer2_attention_weight_min": -29.3692626953125,
      "activations/layer3_attention_weight_max": 88.67295837402344,
      "activations/layer3_attention_weight_min": -90.44184875488281,
      "activations/layer4_attention_weight_max": 135.31216430664062,
      "activations/layer4_attention_weight_min": -126.14876556396484,
      "activations/layer5_attention_weight_max": 86.73140716552734,
      "activations/layer5_attention_weight_min": -82.4391860961914,
      "activations/layer6_attention_weight_max": 59.5244255065918,
      "activations/layer6_attention_weight_min": -56.41865158081055,
      "activations/layer7_attention_weight_max": 87.51634216308594,
      "activations/layer7_attention_weight_min": -84.75997924804688,
      "activations/layer8_attention_weight_max": 42.82719802856445,
      "activations/layer8_attention_weight_min": -39.87349319458008,
      "activations/layer9_attention_weight_max": 41.54221725463867,
      "activations/layer9_attention_weight_min": -37.89347457885742,
      "epoch": 4.45,
      "learning_rate": 0.00012254810606060604,
      "loss": 2.8941,
      "step": 76550
    },
    {
      "activations/layer0_attention_weight_max": 15.540709495544434,
      "activations/layer0_attention_weight_min": -11.296541213989258,
      "activations/layer10_attention_weight_max": 38.07512283325195,
      "activations/layer10_attention_weight_min": -30.835172653198242,
      "activations/layer11_attention_weight_max": 35.884910583496094,
      "activations/layer11_attention_weight_min": -27.999515533447266,
      "activations/layer12_attention_weight_max": 44.83367919921875,
      "activations/layer12_attention_weight_min": -34.535423278808594,
      "activations/layer13_attention_weight_max": 54.557411193847656,
      "activations/layer13_attention_weight_min": -41.714927673339844,
      "activations/layer14_attention_weight_max": 71.62495422363281,
      "activations/layer14_attention_weight_min": -49.03976821899414,
      "activations/layer15_attention_weight_max": 45.29256057739258,
      "activations/layer15_attention_weight_min": -30.226280212402344,
      "activations/layer16_attention_weight_max": 37.721534729003906,
      "activations/layer16_attention_weight_min": -27.539871215820312,
      "activations/layer17_attention_weight_max": 38.810462951660156,
      "activations/layer17_attention_weight_min": -26.673852920532227,
      "activations/layer18_attention_weight_max": 35.05126190185547,
      "activations/layer18_attention_weight_min": -20.013240814208984,
      "activations/layer19_attention_weight_max": 41.75390625,
      "activations/layer19_attention_weight_min": -28.751968383789062,
      "activations/layer1_attention_weight_max": 16.794897079467773,
      "activations/layer1_attention_weight_min": -14.480545997619629,
      "activations/layer20_attention_weight_max": 32.16355895996094,
      "activations/layer20_attention_weight_min": -22.31041145324707,
      "activations/layer21_attention_weight_max": 32.8594970703125,
      "activations/layer21_attention_weight_min": -21.988605499267578,
      "activations/layer22_attention_weight_max": 46.00625991821289,
      "activations/layer22_attention_weight_min": -26.17228889465332,
      "activations/layer23_attention_weight_max": 28.097675323486328,
      "activations/layer23_attention_weight_min": -19.171390533447266,
      "activations/layer2_attention_weight_max": 33.250125885009766,
      "activations/layer2_attention_weight_min": -31.993101119995117,
      "activations/layer3_attention_weight_max": 97.53225708007812,
      "activations/layer3_attention_weight_min": -95.58392333984375,
      "activations/layer4_attention_weight_max": 157.07945251464844,
      "activations/layer4_attention_weight_min": -140.4034881591797,
      "activations/layer5_attention_weight_max": 105.38059997558594,
      "activations/layer5_attention_weight_min": -93.73235321044922,
      "activations/layer6_attention_weight_max": 67.98373413085938,
      "activations/layer6_attention_weight_min": -64.18892669677734,
      "activations/layer7_attention_weight_max": 88.70525360107422,
      "activations/layer7_attention_weight_min": -87.09139251708984,
      "activations/layer8_attention_weight_max": 47.76768112182617,
      "activations/layer8_attention_weight_min": -41.75811767578125,
      "activations/layer9_attention_weight_max": 40.82998275756836,
      "activations/layer9_attention_weight_min": -35.049076080322266,
      "epoch": 4.45,
      "learning_rate": 0.00012252916666666665,
      "loss": 2.8605,
      "step": 76600
    },
    {
      "activations/layer0_attention_weight_max": 16.20245361328125,
      "activations/layer0_attention_weight_min": -11.981795310974121,
      "activations/layer10_attention_weight_max": 35.42759704589844,
      "activations/layer10_attention_weight_min": -29.563772201538086,
      "activations/layer11_attention_weight_max": 34.150306701660156,
      "activations/layer11_attention_weight_min": -27.7900447845459,
      "activations/layer12_attention_weight_max": 26.287166595458984,
      "activations/layer12_attention_weight_min": -26.36723518371582,
      "activations/layer13_attention_weight_max": 44.893680572509766,
      "activations/layer13_attention_weight_min": -34.78990173339844,
      "activations/layer14_attention_weight_max": 75.58917999267578,
      "activations/layer14_attention_weight_min": -49.9019660949707,
      "activations/layer15_attention_weight_max": 43.713504791259766,
      "activations/layer15_attention_weight_min": -29.704442977905273,
      "activations/layer16_attention_weight_max": 38.006038665771484,
      "activations/layer16_attention_weight_min": -28.754621505737305,
      "activations/layer17_attention_weight_max": 38.93470001220703,
      "activations/layer17_attention_weight_min": -25.149127960205078,
      "activations/layer18_attention_weight_max": 32.07184982299805,
      "activations/layer18_attention_weight_min": -19.387699127197266,
      "activations/layer19_attention_weight_max": 42.13528060913086,
      "activations/layer19_attention_weight_min": -29.733667373657227,
      "activations/layer1_attention_weight_max": 17.589412689208984,
      "activations/layer1_attention_weight_min": -15.777918815612793,
      "activations/layer20_attention_weight_max": 32.22188186645508,
      "activations/layer20_attention_weight_min": -26.485868453979492,
      "activations/layer21_attention_weight_max": 35.451019287109375,
      "activations/layer21_attention_weight_min": -25.875059127807617,
      "activations/layer22_attention_weight_max": 52.47654724121094,
      "activations/layer22_attention_weight_min": -31.62980079650879,
      "activations/layer23_attention_weight_max": 32.701019287109375,
      "activations/layer23_attention_weight_min": -18.931148529052734,
      "activations/layer2_attention_weight_max": 33.86790466308594,
      "activations/layer2_attention_weight_min": -31.592880249023438,
      "activations/layer3_attention_weight_max": 89.76807403564453,
      "activations/layer3_attention_weight_min": -88.63972473144531,
      "activations/layer4_attention_weight_max": 129.97921752929688,
      "activations/layer4_attention_weight_min": -121.66753387451172,
      "activations/layer5_attention_weight_max": 88.82876586914062,
      "activations/layer5_attention_weight_min": -80.70051574707031,
      "activations/layer6_attention_weight_max": 55.09928512573242,
      "activations/layer6_attention_weight_min": -56.63228225708008,
      "activations/layer7_attention_weight_max": 79.2177963256836,
      "activations/layer7_attention_weight_min": -82.98706817626953,
      "activations/layer8_attention_weight_max": 39.471702575683594,
      "activations/layer8_attention_weight_min": -38.99678039550781,
      "activations/layer9_attention_weight_max": 38.432342529296875,
      "activations/layer9_attention_weight_min": -31.903839111328125,
      "epoch": 4.45,
      "learning_rate": 0.00012251022727272727,
      "loss": 2.8599,
      "step": 76650
    },
    {
      "activations/layer0_attention_weight_max": 16.919343948364258,
      "activations/layer0_attention_weight_min": -11.692450523376465,
      "activations/layer10_attention_weight_max": 34.4813346862793,
      "activations/layer10_attention_weight_min": -30.44590950012207,
      "activations/layer11_attention_weight_max": 32.95804977416992,
      "activations/layer11_attention_weight_min": -28.081401824951172,
      "activations/layer12_attention_weight_max": 28.43509864807129,
      "activations/layer12_attention_weight_min": -32.79088592529297,
      "activations/layer13_attention_weight_max": 44.54270935058594,
      "activations/layer13_attention_weight_min": -35.30619430541992,
      "activations/layer14_attention_weight_max": 70.97347259521484,
      "activations/layer14_attention_weight_min": -51.196781158447266,
      "activations/layer15_attention_weight_max": 41.714229583740234,
      "activations/layer15_attention_weight_min": -28.155874252319336,
      "activations/layer16_attention_weight_max": 44.09297561645508,
      "activations/layer16_attention_weight_min": -27.817007064819336,
      "activations/layer17_attention_weight_max": 42.64799499511719,
      "activations/layer17_attention_weight_min": -24.496965408325195,
      "activations/layer18_attention_weight_max": 31.532089233398438,
      "activations/layer18_attention_weight_min": -19.180557250976562,
      "activations/layer19_attention_weight_max": 41.85956573486328,
      "activations/layer19_attention_weight_min": -28.835346221923828,
      "activations/layer1_attention_weight_max": 18.47224998474121,
      "activations/layer1_attention_weight_min": -14.658576965332031,
      "activations/layer20_attention_weight_max": 31.453062057495117,
      "activations/layer20_attention_weight_min": -23.440031051635742,
      "activations/layer21_attention_weight_max": 33.0352897644043,
      "activations/layer21_attention_weight_min": -21.326635360717773,
      "activations/layer22_attention_weight_max": 49.80064010620117,
      "activations/layer22_attention_weight_min": -27.87179183959961,
      "activations/layer23_attention_weight_max": 32.707733154296875,
      "activations/layer23_attention_weight_min": -21.6805419921875,
      "activations/layer2_attention_weight_max": 31.606544494628906,
      "activations/layer2_attention_weight_min": -30.054092407226562,
      "activations/layer3_attention_weight_max": 87.93247985839844,
      "activations/layer3_attention_weight_min": -90.6523208618164,
      "activations/layer4_attention_weight_max": 130.7306671142578,
      "activations/layer4_attention_weight_min": -123.08025360107422,
      "activations/layer5_attention_weight_max": 85.10441589355469,
      "activations/layer5_attention_weight_min": -80.20306396484375,
      "activations/layer6_attention_weight_max": 56.24150848388672,
      "activations/layer6_attention_weight_min": -57.802879333496094,
      "activations/layer7_attention_weight_max": 78.64753723144531,
      "activations/layer7_attention_weight_min": -77.31149291992188,
      "activations/layer8_attention_weight_max": 39.0067024230957,
      "activations/layer8_attention_weight_min": -37.833255767822266,
      "activations/layer9_attention_weight_max": 36.524932861328125,
      "activations/layer9_attention_weight_min": -31.80037498474121,
      "epoch": 4.46,
      "learning_rate": 0.00012249128787878788,
      "loss": 2.866,
      "step": 76700
    },
    {
      "activations/layer0_attention_weight_max": 15.663176536560059,
      "activations/layer0_attention_weight_min": -11.52881145477295,
      "activations/layer10_attention_weight_max": 36.04750061035156,
      "activations/layer10_attention_weight_min": -30.334196090698242,
      "activations/layer11_attention_weight_max": 31.399198532104492,
      "activations/layer11_attention_weight_min": -27.39139175415039,
      "activations/layer12_attention_weight_max": 37.015838623046875,
      "activations/layer12_attention_weight_min": -29.162555694580078,
      "activations/layer13_attention_weight_max": 39.46562576293945,
      "activations/layer13_attention_weight_min": -32.534305572509766,
      "activations/layer14_attention_weight_max": 61.537254333496094,
      "activations/layer14_attention_weight_min": -40.85225296020508,
      "activations/layer15_attention_weight_max": 43.507328033447266,
      "activations/layer15_attention_weight_min": -28.60696792602539,
      "activations/layer16_attention_weight_max": 40.24003601074219,
      "activations/layer16_attention_weight_min": -26.844478607177734,
      "activations/layer17_attention_weight_max": 43.36030960083008,
      "activations/layer17_attention_weight_min": -25.631540298461914,
      "activations/layer18_attention_weight_max": 30.72247886657715,
      "activations/layer18_attention_weight_min": -21.521751403808594,
      "activations/layer19_attention_weight_max": 39.6799201965332,
      "activations/layer19_attention_weight_min": -29.059438705444336,
      "activations/layer1_attention_weight_max": 17.81710433959961,
      "activations/layer1_attention_weight_min": -15.58516788482666,
      "activations/layer20_attention_weight_max": 31.791841506958008,
      "activations/layer20_attention_weight_min": -21.866710662841797,
      "activations/layer21_attention_weight_max": 33.91142272949219,
      "activations/layer21_attention_weight_min": -22.27846336364746,
      "activations/layer22_attention_weight_max": 51.696231842041016,
      "activations/layer22_attention_weight_min": -28.937578201293945,
      "activations/layer23_attention_weight_max": 30.90985679626465,
      "activations/layer23_attention_weight_min": -21.744182586669922,
      "activations/layer2_attention_weight_max": 31.879901885986328,
      "activations/layer2_attention_weight_min": -31.91534423828125,
      "activations/layer3_attention_weight_max": 87.18102264404297,
      "activations/layer3_attention_weight_min": -89.31179809570312,
      "activations/layer4_attention_weight_max": 127.75367736816406,
      "activations/layer4_attention_weight_min": -127.20455169677734,
      "activations/layer5_attention_weight_max": 85.28935241699219,
      "activations/layer5_attention_weight_min": -84.50521850585938,
      "activations/layer6_attention_weight_max": 57.77162551879883,
      "activations/layer6_attention_weight_min": -56.94767379760742,
      "activations/layer7_attention_weight_max": 83.74990844726562,
      "activations/layer7_attention_weight_min": -80.6824722290039,
      "activations/layer8_attention_weight_max": 41.99203109741211,
      "activations/layer8_attention_weight_min": -40.476829528808594,
      "activations/layer9_attention_weight_max": 40.58917236328125,
      "activations/layer9_attention_weight_min": -35.05204391479492,
      "epoch": 4.46,
      "learning_rate": 0.00012247234848484847,
      "loss": 2.8694,
      "step": 76750
    },
    {
      "activations/layer0_attention_weight_max": 16.466623306274414,
      "activations/layer0_attention_weight_min": -11.835593223571777,
      "activations/layer10_attention_weight_max": 38.156375885009766,
      "activations/layer10_attention_weight_min": -30.76092529296875,
      "activations/layer11_attention_weight_max": 33.62145233154297,
      "activations/layer11_attention_weight_min": -26.37616539001465,
      "activations/layer12_attention_weight_max": 31.088594436645508,
      "activations/layer12_attention_weight_min": -28.205968856811523,
      "activations/layer13_attention_weight_max": 53.647789001464844,
      "activations/layer13_attention_weight_min": -33.237388610839844,
      "activations/layer14_attention_weight_max": 79.20797729492188,
      "activations/layer14_attention_weight_min": -49.56108474731445,
      "activations/layer15_attention_weight_max": 44.4566650390625,
      "activations/layer15_attention_weight_min": -27.183006286621094,
      "activations/layer16_attention_weight_max": 41.82977294921875,
      "activations/layer16_attention_weight_min": -26.549915313720703,
      "activations/layer17_attention_weight_max": 45.69947814941406,
      "activations/layer17_attention_weight_min": -22.81024169921875,
      "activations/layer18_attention_weight_max": 39.71137237548828,
      "activations/layer18_attention_weight_min": -19.64925765991211,
      "activations/layer19_attention_weight_max": 52.37664794921875,
      "activations/layer19_attention_weight_min": -28.502161026000977,
      "activations/layer1_attention_weight_max": 17.219938278198242,
      "activations/layer1_attention_weight_min": -13.654715538024902,
      "activations/layer20_attention_weight_max": 40.58971405029297,
      "activations/layer20_attention_weight_min": -22.29350471496582,
      "activations/layer21_attention_weight_max": 40.34922790527344,
      "activations/layer21_attention_weight_min": -20.571786880493164,
      "activations/layer22_attention_weight_max": 62.64004898071289,
      "activations/layer22_attention_weight_min": -25.156288146972656,
      "activations/layer23_attention_weight_max": 36.493980407714844,
      "activations/layer23_attention_weight_min": -18.639766693115234,
      "activations/layer2_attention_weight_max": 32.2857666015625,
      "activations/layer2_attention_weight_min": -30.835521697998047,
      "activations/layer3_attention_weight_max": 90.09004211425781,
      "activations/layer3_attention_weight_min": -94.93637084960938,
      "activations/layer4_attention_weight_max": 133.67453002929688,
      "activations/layer4_attention_weight_min": -140.2115020751953,
      "activations/layer5_attention_weight_max": 91.76531982421875,
      "activations/layer5_attention_weight_min": -85.74739837646484,
      "activations/layer6_attention_weight_max": 57.16411590576172,
      "activations/layer6_attention_weight_min": -60.13743591308594,
      "activations/layer7_attention_weight_max": 86.03761291503906,
      "activations/layer7_attention_weight_min": -80.1524658203125,
      "activations/layer8_attention_weight_max": 42.415348052978516,
      "activations/layer8_attention_weight_min": -41.64015579223633,
      "activations/layer9_attention_weight_max": 39.10123825073242,
      "activations/layer9_attention_weight_min": -34.304656982421875,
      "epoch": 4.46,
      "learning_rate": 0.00012245340909090909,
      "loss": 2.8612,
      "step": 76800
    },
    {
      "activations/layer0_attention_weight_max": 17.059715270996094,
      "activations/layer0_attention_weight_min": -12.062294960021973,
      "activations/layer10_attention_weight_max": 34.30043411254883,
      "activations/layer10_attention_weight_min": -30.539072036743164,
      "activations/layer11_attention_weight_max": 31.963300704956055,
      "activations/layer11_attention_weight_min": -26.958141326904297,
      "activations/layer12_attention_weight_max": 30.49656867980957,
      "activations/layer12_attention_weight_min": -24.94244384765625,
      "activations/layer13_attention_weight_max": 44.598472595214844,
      "activations/layer13_attention_weight_min": -31.528976440429688,
      "activations/layer14_attention_weight_max": 59.68032455444336,
      "activations/layer14_attention_weight_min": -47.35639572143555,
      "activations/layer15_attention_weight_max": 40.55137252807617,
      "activations/layer15_attention_weight_min": -28.285484313964844,
      "activations/layer16_attention_weight_max": 36.1510009765625,
      "activations/layer16_attention_weight_min": -27.106645584106445,
      "activations/layer17_attention_weight_max": 35.96205520629883,
      "activations/layer17_attention_weight_min": -25.18056869506836,
      "activations/layer18_attention_weight_max": 29.801610946655273,
      "activations/layer18_attention_weight_min": -19.642297744750977,
      "activations/layer19_attention_weight_max": 41.7677116394043,
      "activations/layer19_attention_weight_min": -29.045316696166992,
      "activations/layer1_attention_weight_max": 18.519943237304688,
      "activations/layer1_attention_weight_min": -13.565658569335938,
      "activations/layer20_attention_weight_max": 34.27167510986328,
      "activations/layer20_attention_weight_min": -25.21599769592285,
      "activations/layer21_attention_weight_max": 33.026554107666016,
      "activations/layer21_attention_weight_min": -23.97784996032715,
      "activations/layer22_attention_weight_max": 52.92869186401367,
      "activations/layer22_attention_weight_min": -28.77973747253418,
      "activations/layer23_attention_weight_max": 35.62327575683594,
      "activations/layer23_attention_weight_min": -20.744762420654297,
      "activations/layer2_attention_weight_max": 31.606231689453125,
      "activations/layer2_attention_weight_min": -30.17933464050293,
      "activations/layer3_attention_weight_max": 87.62210083007812,
      "activations/layer3_attention_weight_min": -87.88175964355469,
      "activations/layer4_attention_weight_max": 128.4318389892578,
      "activations/layer4_attention_weight_min": -121.3065185546875,
      "activations/layer5_attention_weight_max": 83.7381362915039,
      "activations/layer5_attention_weight_min": -82.04766845703125,
      "activations/layer6_attention_weight_max": 56.86521911621094,
      "activations/layer6_attention_weight_min": -54.688575744628906,
      "activations/layer7_attention_weight_max": 77.69717407226562,
      "activations/layer7_attention_weight_min": -78.19126892089844,
      "activations/layer8_attention_weight_max": 38.62211990356445,
      "activations/layer8_attention_weight_min": -36.941200256347656,
      "activations/layer9_attention_weight_max": 35.5846061706543,
      "activations/layer9_attention_weight_min": -34.63209915161133,
      "epoch": 4.47,
      "learning_rate": 0.0001224344696969697,
      "loss": 2.8515,
      "step": 76850
    },
    {
      "activations/layer0_attention_weight_max": 16.070831298828125,
      "activations/layer0_attention_weight_min": -11.945189476013184,
      "activations/layer10_attention_weight_max": 33.742164611816406,
      "activations/layer10_attention_weight_min": -31.411392211914062,
      "activations/layer11_attention_weight_max": 30.42610740661621,
      "activations/layer11_attention_weight_min": -28.928152084350586,
      "activations/layer12_attention_weight_max": 23.83310317993164,
      "activations/layer12_attention_weight_min": -25.6861515045166,
      "activations/layer13_attention_weight_max": 45.2630500793457,
      "activations/layer13_attention_weight_min": -35.08561706542969,
      "activations/layer14_attention_weight_max": 66.96287536621094,
      "activations/layer14_attention_weight_min": -49.621131896972656,
      "activations/layer15_attention_weight_max": 41.28339767456055,
      "activations/layer15_attention_weight_min": -28.53862762451172,
      "activations/layer16_attention_weight_max": 42.228641510009766,
      "activations/layer16_attention_weight_min": -27.37342071533203,
      "activations/layer17_attention_weight_max": 42.23134231567383,
      "activations/layer17_attention_weight_min": -25.31490135192871,
      "activations/layer18_attention_weight_max": 31.958900451660156,
      "activations/layer18_attention_weight_min": -19.90978240966797,
      "activations/layer19_attention_weight_max": 44.63706970214844,
      "activations/layer19_attention_weight_min": -29.475645065307617,
      "activations/layer1_attention_weight_max": 17.18309783935547,
      "activations/layer1_attention_weight_min": -17.11285972595215,
      "activations/layer20_attention_weight_max": 31.423667907714844,
      "activations/layer20_attention_weight_min": -23.52235984802246,
      "activations/layer21_attention_weight_max": 33.24574661254883,
      "activations/layer21_attention_weight_min": -23.07221794128418,
      "activations/layer22_attention_weight_max": 50.316104888916016,
      "activations/layer22_attention_weight_min": -32.6118278503418,
      "activations/layer23_attention_weight_max": 29.142627716064453,
      "activations/layer23_attention_weight_min": -21.344144821166992,
      "activations/layer2_attention_weight_max": 33.3477897644043,
      "activations/layer2_attention_weight_min": -31.381446838378906,
      "activations/layer3_attention_weight_max": 90.00006103515625,
      "activations/layer3_attention_weight_min": -88.74658203125,
      "activations/layer4_attention_weight_max": 129.22158813476562,
      "activations/layer4_attention_weight_min": -131.15869140625,
      "activations/layer5_attention_weight_max": 88.07101440429688,
      "activations/layer5_attention_weight_min": -87.22139739990234,
      "activations/layer6_attention_weight_max": 55.99794387817383,
      "activations/layer6_attention_weight_min": -57.53273391723633,
      "activations/layer7_attention_weight_max": 77.9131088256836,
      "activations/layer7_attention_weight_min": -78.15135955810547,
      "activations/layer8_attention_weight_max": 41.07960891723633,
      "activations/layer8_attention_weight_min": -40.23540496826172,
      "activations/layer9_attention_weight_max": 36.382049560546875,
      "activations/layer9_attention_weight_min": -34.05703353881836,
      "epoch": 4.47,
      "learning_rate": 0.0001224155303030303,
      "loss": 2.8562,
      "step": 76900
    },
    {
      "activations/layer0_attention_weight_max": 17.10544204711914,
      "activations/layer0_attention_weight_min": -11.402758598327637,
      "activations/layer10_attention_weight_max": 33.40639114379883,
      "activations/layer10_attention_weight_min": -29.282943725585938,
      "activations/layer11_attention_weight_max": 32.339134216308594,
      "activations/layer11_attention_weight_min": -29.341039657592773,
      "activations/layer12_attention_weight_max": 25.36196517944336,
      "activations/layer12_attention_weight_min": -27.45450210571289,
      "activations/layer13_attention_weight_max": 42.83779525756836,
      "activations/layer13_attention_weight_min": -30.638940811157227,
      "activations/layer14_attention_weight_max": 59.790687561035156,
      "activations/layer14_attention_weight_min": -42.45802307128906,
      "activations/layer15_attention_weight_max": 39.04655075073242,
      "activations/layer15_attention_weight_min": -28.07207679748535,
      "activations/layer16_attention_weight_max": 37.40797424316406,
      "activations/layer16_attention_weight_min": -26.27250862121582,
      "activations/layer17_attention_weight_max": 39.0023078918457,
      "activations/layer17_attention_weight_min": -24.61447525024414,
      "activations/layer18_attention_weight_max": 32.972232818603516,
      "activations/layer18_attention_weight_min": -19.800195693969727,
      "activations/layer19_attention_weight_max": 39.50480270385742,
      "activations/layer19_attention_weight_min": -28.610761642456055,
      "activations/layer1_attention_weight_max": 17.839326858520508,
      "activations/layer1_attention_weight_min": -14.494721412658691,
      "activations/layer20_attention_weight_max": 31.449987411499023,
      "activations/layer20_attention_weight_min": -22.938825607299805,
      "activations/layer21_attention_weight_max": 32.90914535522461,
      "activations/layer21_attention_weight_min": -19.289846420288086,
      "activations/layer22_attention_weight_max": 47.98697280883789,
      "activations/layer22_attention_weight_min": -29.99506187438965,
      "activations/layer23_attention_weight_max": 29.930295944213867,
      "activations/layer23_attention_weight_min": -18.500965118408203,
      "activations/layer2_attention_weight_max": 31.446630477905273,
      "activations/layer2_attention_weight_min": -31.245525360107422,
      "activations/layer3_attention_weight_max": 88.0439682006836,
      "activations/layer3_attention_weight_min": -89.53223419189453,
      "activations/layer4_attention_weight_max": 129.21432495117188,
      "activations/layer4_attention_weight_min": -126.78160858154297,
      "activations/layer5_attention_weight_max": 84.85441589355469,
      "activations/layer5_attention_weight_min": -80.86323547363281,
      "activations/layer6_attention_weight_max": 56.04795837402344,
      "activations/layer6_attention_weight_min": -55.02318572998047,
      "activations/layer7_attention_weight_max": 80.58202362060547,
      "activations/layer7_attention_weight_min": -79.74334716796875,
      "activations/layer8_attention_weight_max": 41.87590026855469,
      "activations/layer8_attention_weight_min": -38.29964828491211,
      "activations/layer9_attention_weight_max": 35.843509674072266,
      "activations/layer9_attention_weight_min": -38.98832321166992,
      "epoch": 4.47,
      "learning_rate": 0.0001223965909090909,
      "loss": 2.855,
      "step": 76950
    },
    {
      "activations/layer0_attention_weight_max": 17.444082260131836,
      "activations/layer0_attention_weight_min": -11.677119255065918,
      "activations/layer10_attention_weight_max": 33.770259857177734,
      "activations/layer10_attention_weight_min": -32.074867248535156,
      "activations/layer11_attention_weight_max": 34.50507736206055,
      "activations/layer11_attention_weight_min": -28.99862289428711,
      "activations/layer12_attention_weight_max": 26.553321838378906,
      "activations/layer12_attention_weight_min": -26.299325942993164,
      "activations/layer13_attention_weight_max": 51.5262336730957,
      "activations/layer13_attention_weight_min": -39.703609466552734,
      "activations/layer14_attention_weight_max": 59.378753662109375,
      "activations/layer14_attention_weight_min": -49.721370697021484,
      "activations/layer15_attention_weight_max": 40.96004104614258,
      "activations/layer15_attention_weight_min": -31.147680282592773,
      "activations/layer16_attention_weight_max": 33.98101806640625,
      "activations/layer16_attention_weight_min": -29.97214126586914,
      "activations/layer17_attention_weight_max": 34.820133209228516,
      "activations/layer17_attention_weight_min": -26.08444595336914,
      "activations/layer18_attention_weight_max": 29.94424057006836,
      "activations/layer18_attention_weight_min": -20.066085815429688,
      "activations/layer19_attention_weight_max": 36.643978118896484,
      "activations/layer19_attention_weight_min": -29.96864891052246,
      "activations/layer1_attention_weight_max": 17.394620895385742,
      "activations/layer1_attention_weight_min": -13.904812812805176,
      "activations/layer20_attention_weight_max": 32.02473831176758,
      "activations/layer20_attention_weight_min": -24.304655075073242,
      "activations/layer21_attention_weight_max": 28.112869262695312,
      "activations/layer21_attention_weight_min": -20.77002716064453,
      "activations/layer22_attention_weight_max": 44.24601364135742,
      "activations/layer22_attention_weight_min": -26.96848487854004,
      "activations/layer23_attention_weight_max": 29.665267944335938,
      "activations/layer23_attention_weight_min": -19.36300277709961,
      "activations/layer2_attention_weight_max": 32.3511962890625,
      "activations/layer2_attention_weight_min": -32.897499084472656,
      "activations/layer3_attention_weight_max": 92.56953430175781,
      "activations/layer3_attention_weight_min": -94.08785247802734,
      "activations/layer4_attention_weight_max": 131.65380859375,
      "activations/layer4_attention_weight_min": -127.68544006347656,
      "activations/layer5_attention_weight_max": 87.8720932006836,
      "activations/layer5_attention_weight_min": -82.55511474609375,
      "activations/layer6_attention_weight_max": 58.08314514160156,
      "activations/layer6_attention_weight_min": -55.88593292236328,
      "activations/layer7_attention_weight_max": 79.97281646728516,
      "activations/layer7_attention_weight_min": -77.53385925292969,
      "activations/layer8_attention_weight_max": 39.804473876953125,
      "activations/layer8_attention_weight_min": -40.49459457397461,
      "activations/layer9_attention_weight_max": 36.19367599487305,
      "activations/layer9_attention_weight_min": -34.68380355834961,
      "epoch": 4.47,
      "learning_rate": 0.00012237765151515152,
      "loss": 2.8655,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_loss": 2.814453125,
      "eval_runtime": 8.476,
      "eval_samples_per_second": 506.605,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_openwebtext_loss": 2.814453125,
      "eval_openwebtext_ppl": 16.684049195604416,
      "eval_openwebtext_runtime": 8.476,
      "eval_openwebtext_samples_per_second": 506.605,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_wikitext_loss": 3.05859375,
      "eval_wikitext_ppl": 21.297586362924953,
      "eval_wikitext_runtime": 2.007,
      "eval_wikitext_samples_per_second": 227.206,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_lambada_loss": 2.77734375,
      "eval_lambada_ppl": 16.076261613516785,
      "eval_lambada_runtime": 9.5556,
      "eval_lambada_samples_per_second": 509.543,
      "step": 77000
    },
    {
      "activations/layer0_attention_weight_max": 14.900206565856934,
      "activations/layer0_attention_weight_min": -11.80388069152832,
      "activations/layer10_attention_weight_max": 37.752506256103516,
      "activations/layer10_attention_weight_min": -32.35858917236328,
      "activations/layer11_attention_weight_max": 31.01856231689453,
      "activations/layer11_attention_weight_min": -28.000186920166016,
      "activations/layer12_attention_weight_max": 31.15609359741211,
      "activations/layer12_attention_weight_min": -26.517972946166992,
      "activations/layer13_attention_weight_max": 47.038414001464844,
      "activations/layer13_attention_weight_min": -32.47975158691406,
      "activations/layer14_attention_weight_max": 75.01264190673828,
      "activations/layer14_attention_weight_min": -51.12702178955078,
      "activations/layer15_attention_weight_max": 48.836524963378906,
      "activations/layer15_attention_weight_min": -27.1357479095459,
      "activations/layer16_attention_weight_max": 40.99213790893555,
      "activations/layer16_attention_weight_min": -28.175634384155273,
      "activations/layer17_attention_weight_max": 45.764373779296875,
      "activations/layer17_attention_weight_min": -26.336151123046875,
      "activations/layer18_attention_weight_max": 37.10232925415039,
      "activations/layer18_attention_weight_min": -19.1973819732666,
      "activations/layer19_attention_weight_max": 40.07733154296875,
      "activations/layer19_attention_weight_min": -28.872507095336914,
      "activations/layer1_attention_weight_max": 17.950794219970703,
      "activations/layer1_attention_weight_min": -15.821853637695312,
      "activations/layer20_attention_weight_max": 35.832096099853516,
      "activations/layer20_attention_weight_min": -24.127588272094727,
      "activations/layer21_attention_weight_max": 38.11348342895508,
      "activations/layer21_attention_weight_min": -21.050979614257812,
      "activations/layer22_attention_weight_max": 54.2987174987793,
      "activations/layer22_attention_weight_min": -26.656330108642578,
      "activations/layer23_attention_weight_max": 31.451244354248047,
      "activations/layer23_attention_weight_min": -19.71666717529297,
      "activations/layer2_attention_weight_max": 32.59617614746094,
      "activations/layer2_attention_weight_min": -33.01948928833008,
      "activations/layer3_attention_weight_max": 90.43074798583984,
      "activations/layer3_attention_weight_min": -93.55724334716797,
      "activations/layer4_attention_weight_max": 133.72744750976562,
      "activations/layer4_attention_weight_min": -130.3386993408203,
      "activations/layer5_attention_weight_max": 91.22735595703125,
      "activations/layer5_attention_weight_min": -90.00991821289062,
      "activations/layer6_attention_weight_max": 59.29880905151367,
      "activations/layer6_attention_weight_min": -63.30620574951172,
      "activations/layer7_attention_weight_max": 84.11540222167969,
      "activations/layer7_attention_weight_min": -89.05593872070312,
      "activations/layer8_attention_weight_max": 43.306182861328125,
      "activations/layer8_attention_weight_min": -41.46703338623047,
      "activations/layer9_attention_weight_max": 38.25204086303711,
      "activations/layer9_attention_weight_min": -35.25244903564453,
      "epoch": 4.48,
      "learning_rate": 0.0001223587121212121,
      "loss": 2.8479,
      "step": 77050
    },
    {
      "activations/layer0_attention_weight_max": 16.264780044555664,
      "activations/layer0_attention_weight_min": -12.579391479492188,
      "activations/layer10_attention_weight_max": 33.09697341918945,
      "activations/layer10_attention_weight_min": -29.32737159729004,
      "activations/layer11_attention_weight_max": 34.169944763183594,
      "activations/layer11_attention_weight_min": -26.742849349975586,
      "activations/layer12_attention_weight_max": 33.058170318603516,
      "activations/layer12_attention_weight_min": -26.82280158996582,
      "activations/layer13_attention_weight_max": 43.27657699584961,
      "activations/layer13_attention_weight_min": -31.11647605895996,
      "activations/layer14_attention_weight_max": 60.251949310302734,
      "activations/layer14_attention_weight_min": -43.20448684692383,
      "activations/layer15_attention_weight_max": 42.93303680419922,
      "activations/layer15_attention_weight_min": -27.021635055541992,
      "activations/layer16_attention_weight_max": 40.02116012573242,
      "activations/layer16_attention_weight_min": -30.64677619934082,
      "activations/layer17_attention_weight_max": 40.39185333251953,
      "activations/layer17_attention_weight_min": -24.60757064819336,
      "activations/layer18_attention_weight_max": 33.80259704589844,
      "activations/layer18_attention_weight_min": -20.558269500732422,
      "activations/layer19_attention_weight_max": 48.28973388671875,
      "activations/layer19_attention_weight_min": -30.178537368774414,
      "activations/layer1_attention_weight_max": 17.069828033447266,
      "activations/layer1_attention_weight_min": -16.378156661987305,
      "activations/layer20_attention_weight_max": 35.27153015136719,
      "activations/layer20_attention_weight_min": -24.375951766967773,
      "activations/layer21_attention_weight_max": 35.59172058105469,
      "activations/layer21_attention_weight_min": -19.65070152282715,
      "activations/layer22_attention_weight_max": 51.35296630859375,
      "activations/layer22_attention_weight_min": -27.13327980041504,
      "activations/layer23_attention_weight_max": 37.46757507324219,
      "activations/layer23_attention_weight_min": -21.164003372192383,
      "activations/layer2_attention_weight_max": 31.950889587402344,
      "activations/layer2_attention_weight_min": -30.490734100341797,
      "activations/layer3_attention_weight_max": 89.18301391601562,
      "activations/layer3_attention_weight_min": -89.80166625976562,
      "activations/layer4_attention_weight_max": 132.75047302246094,
      "activations/layer4_attention_weight_min": -126.6963119506836,
      "activations/layer5_attention_weight_max": 87.62324523925781,
      "activations/layer5_attention_weight_min": -81.49761962890625,
      "activations/layer6_attention_weight_max": 56.24869918823242,
      "activations/layer6_attention_weight_min": -61.21176528930664,
      "activations/layer7_attention_weight_max": 78.45694732666016,
      "activations/layer7_attention_weight_min": -83.06421661376953,
      "activations/layer8_attention_weight_max": 38.7864875793457,
      "activations/layer8_attention_weight_min": -39.622859954833984,
      "activations/layer9_attention_weight_max": 36.235713958740234,
      "activations/layer9_attention_weight_min": -35.16605758666992,
      "epoch": 4.48,
      "learning_rate": 0.00012233977272727272,
      "loss": 2.8648,
      "step": 77100
    },
    {
      "activations/layer0_attention_weight_max": 16.234670639038086,
      "activations/layer0_attention_weight_min": -11.418421745300293,
      "activations/layer10_attention_weight_max": 35.681182861328125,
      "activations/layer10_attention_weight_min": -29.673282623291016,
      "activations/layer11_attention_weight_max": 34.09099197387695,
      "activations/layer11_attention_weight_min": -26.27345848083496,
      "activations/layer12_attention_weight_max": 31.934494018554688,
      "activations/layer12_attention_weight_min": -31.03974723815918,
      "activations/layer13_attention_weight_max": 49.07982635498047,
      "activations/layer13_attention_weight_min": -33.056968688964844,
      "activations/layer14_attention_weight_max": 73.56684112548828,
      "activations/layer14_attention_weight_min": -45.33164596557617,
      "activations/layer15_attention_weight_max": 48.85707473754883,
      "activations/layer15_attention_weight_min": -28.66107940673828,
      "activations/layer16_attention_weight_max": 38.48694610595703,
      "activations/layer16_attention_weight_min": -29.06647300720215,
      "activations/layer17_attention_weight_max": 45.23067092895508,
      "activations/layer17_attention_weight_min": -24.069076538085938,
      "activations/layer18_attention_weight_max": 41.12193298339844,
      "activations/layer18_attention_weight_min": -19.503063201904297,
      "activations/layer19_attention_weight_max": 39.5639762878418,
      "activations/layer19_attention_weight_min": -28.70758628845215,
      "activations/layer1_attention_weight_max": 17.315656661987305,
      "activations/layer1_attention_weight_min": -15.044776916503906,
      "activations/layer20_attention_weight_max": 37.26129913330078,
      "activations/layer20_attention_weight_min": -25.040056228637695,
      "activations/layer21_attention_weight_max": 42.967041015625,
      "activations/layer21_attention_weight_min": -20.67072868347168,
      "activations/layer22_attention_weight_max": 54.458736419677734,
      "activations/layer22_attention_weight_min": -27.50290870666504,
      "activations/layer23_attention_weight_max": 32.02734375,
      "activations/layer23_attention_weight_min": -18.681459426879883,
      "activations/layer2_attention_weight_max": 32.632972717285156,
      "activations/layer2_attention_weight_min": -30.66083335876465,
      "activations/layer3_attention_weight_max": 87.30809020996094,
      "activations/layer3_attention_weight_min": -85.0501480102539,
      "activations/layer4_attention_weight_max": 129.85044860839844,
      "activations/layer4_attention_weight_min": -121.37479400634766,
      "activations/layer5_attention_weight_max": 89.19236755371094,
      "activations/layer5_attention_weight_min": -80.45265197753906,
      "activations/layer6_attention_weight_max": 57.28883743286133,
      "activations/layer6_attention_weight_min": -55.167503356933594,
      "activations/layer7_attention_weight_max": 79.52822875976562,
      "activations/layer7_attention_weight_min": -77.4699478149414,
      "activations/layer8_attention_weight_max": 40.451908111572266,
      "activations/layer8_attention_weight_min": -38.01716232299805,
      "activations/layer9_attention_weight_max": 36.939735412597656,
      "activations/layer9_attention_weight_min": -33.663780212402344,
      "epoch": 4.48,
      "learning_rate": 0.00012232083333333334,
      "loss": 2.8617,
      "step": 77150
    },
    {
      "activations/layer0_attention_weight_max": 16.567214965820312,
      "activations/layer0_attention_weight_min": -10.804901123046875,
      "activations/layer10_attention_weight_max": 33.557525634765625,
      "activations/layer10_attention_weight_min": -29.39133644104004,
      "activations/layer11_attention_weight_max": 33.358524322509766,
      "activations/layer11_attention_weight_min": -26.382041931152344,
      "activations/layer12_attention_weight_max": 44.354671478271484,
      "activations/layer12_attention_weight_min": -26.49736785888672,
      "activations/layer13_attention_weight_max": 49.820552825927734,
      "activations/layer13_attention_weight_min": -32.98200225830078,
      "activations/layer14_attention_weight_max": 57.869964599609375,
      "activations/layer14_attention_weight_min": -41.191383361816406,
      "activations/layer15_attention_weight_max": 40.6064567565918,
      "activations/layer15_attention_weight_min": -27.06670379638672,
      "activations/layer16_attention_weight_max": 36.6841926574707,
      "activations/layer16_attention_weight_min": -27.10841178894043,
      "activations/layer17_attention_weight_max": 40.0517578125,
      "activations/layer17_attention_weight_min": -26.27610206604004,
      "activations/layer18_attention_weight_max": 32.490726470947266,
      "activations/layer18_attention_weight_min": -22.521347045898438,
      "activations/layer19_attention_weight_max": 40.235103607177734,
      "activations/layer19_attention_weight_min": -27.782438278198242,
      "activations/layer1_attention_weight_max": 16.57202911376953,
      "activations/layer1_attention_weight_min": -14.192276954650879,
      "activations/layer20_attention_weight_max": 39.025508880615234,
      "activations/layer20_attention_weight_min": -22.944997787475586,
      "activations/layer21_attention_weight_max": 34.44022750854492,
      "activations/layer21_attention_weight_min": -20.141891479492188,
      "activations/layer22_attention_weight_max": 49.12363052368164,
      "activations/layer22_attention_weight_min": -30.48526954650879,
      "activations/layer23_attention_weight_max": 28.968786239624023,
      "activations/layer23_attention_weight_min": -18.568859100341797,
      "activations/layer2_attention_weight_max": 32.57451629638672,
      "activations/layer2_attention_weight_min": -30.241430282592773,
      "activations/layer3_attention_weight_max": 90.02897644042969,
      "activations/layer3_attention_weight_min": -87.97576904296875,
      "activations/layer4_attention_weight_max": 129.19522094726562,
      "activations/layer4_attention_weight_min": -127.37379455566406,
      "activations/layer5_attention_weight_max": 84.37554931640625,
      "activations/layer5_attention_weight_min": -79.20431518554688,
      "activations/layer6_attention_weight_max": 55.19844055175781,
      "activations/layer6_attention_weight_min": -59.34153366088867,
      "activations/layer7_attention_weight_max": 76.42719268798828,
      "activations/layer7_attention_weight_min": -80.0949935913086,
      "activations/layer8_attention_weight_max": 41.157352447509766,
      "activations/layer8_attention_weight_min": -38.89217758178711,
      "activations/layer9_attention_weight_max": 38.84394836425781,
      "activations/layer9_attention_weight_min": -34.70580291748047,
      "epoch": 4.49,
      "learning_rate": 0.00012230189393939392,
      "loss": 2.8782,
      "step": 77200
    },
    {
      "activations/layer0_attention_weight_max": 16.133453369140625,
      "activations/layer0_attention_weight_min": -11.644043922424316,
      "activations/layer10_attention_weight_max": 35.8575439453125,
      "activations/layer10_attention_weight_min": -29.98110580444336,
      "activations/layer11_attention_weight_max": 33.59718322753906,
      "activations/layer11_attention_weight_min": -27.835548400878906,
      "activations/layer12_attention_weight_max": 31.5369930267334,
      "activations/layer12_attention_weight_min": -28.578752517700195,
      "activations/layer13_attention_weight_max": 54.1923713684082,
      "activations/layer13_attention_weight_min": -36.31103515625,
      "activations/layer14_attention_weight_max": 66.28948974609375,
      "activations/layer14_attention_weight_min": -44.632568359375,
      "activations/layer15_attention_weight_max": 44.51643753051758,
      "activations/layer15_attention_weight_min": -28.055788040161133,
      "activations/layer16_attention_weight_max": 45.6308708190918,
      "activations/layer16_attention_weight_min": -26.955726623535156,
      "activations/layer17_attention_weight_max": 40.215885162353516,
      "activations/layer17_attention_weight_min": -24.748188018798828,
      "activations/layer18_attention_weight_max": 30.780126571655273,
      "activations/layer18_attention_weight_min": -22.03258514404297,
      "activations/layer19_attention_weight_max": 47.9859733581543,
      "activations/layer19_attention_weight_min": -29.326663970947266,
      "activations/layer1_attention_weight_max": 17.261390686035156,
      "activations/layer1_attention_weight_min": -16.294307708740234,
      "activations/layer20_attention_weight_max": 36.828128814697266,
      "activations/layer20_attention_weight_min": -24.43889045715332,
      "activations/layer21_attention_weight_max": 31.272348403930664,
      "activations/layer21_attention_weight_min": -19.55569076538086,
      "activations/layer22_attention_weight_max": 50.60271072387695,
      "activations/layer22_attention_weight_min": -29.126821517944336,
      "activations/layer23_attention_weight_max": 34.24447250366211,
      "activations/layer23_attention_weight_min": -20.42080307006836,
      "activations/layer2_attention_weight_max": 34.1511344909668,
      "activations/layer2_attention_weight_min": -31.39668846130371,
      "activations/layer3_attention_weight_max": 87.38866424560547,
      "activations/layer3_attention_weight_min": -90.59033203125,
      "activations/layer4_attention_weight_max": 131.55226135253906,
      "activations/layer4_attention_weight_min": -128.67617797851562,
      "activations/layer5_attention_weight_max": 85.99346160888672,
      "activations/layer5_attention_weight_min": -82.24573516845703,
      "activations/layer6_attention_weight_max": 58.864959716796875,
      "activations/layer6_attention_weight_min": -56.38078689575195,
      "activations/layer7_attention_weight_max": 81.06261444091797,
      "activations/layer7_attention_weight_min": -83.6591796875,
      "activations/layer8_attention_weight_max": 40.24544143676758,
      "activations/layer8_attention_weight_min": -38.164634704589844,
      "activations/layer9_attention_weight_max": 39.31863021850586,
      "activations/layer9_attention_weight_min": -33.032718658447266,
      "epoch": 4.49,
      "learning_rate": 0.00012228295454545454,
      "loss": 2.8628,
      "step": 77250
    },
    {
      "activations/layer0_attention_weight_max": 15.928311347961426,
      "activations/layer0_attention_weight_min": -12.447568893432617,
      "activations/layer10_attention_weight_max": 37.380592346191406,
      "activations/layer10_attention_weight_min": -31.351871490478516,
      "activations/layer11_attention_weight_max": 36.5340576171875,
      "activations/layer11_attention_weight_min": -27.34601593017578,
      "activations/layer12_attention_weight_max": 29.765043258666992,
      "activations/layer12_attention_weight_min": -29.098432540893555,
      "activations/layer13_attention_weight_max": 50.73221969604492,
      "activations/layer13_attention_weight_min": -31.494625091552734,
      "activations/layer14_attention_weight_max": 75.04115295410156,
      "activations/layer14_attention_weight_min": -51.503753662109375,
      "activations/layer15_attention_weight_max": 50.701786041259766,
      "activations/layer15_attention_weight_min": -28.133764266967773,
      "activations/layer16_attention_weight_max": 43.13884353637695,
      "activations/layer16_attention_weight_min": -29.233123779296875,
      "activations/layer17_attention_weight_max": 46.45063781738281,
      "activations/layer17_attention_weight_min": -26.40034294128418,
      "activations/layer18_attention_weight_max": 34.76144027709961,
      "activations/layer18_attention_weight_min": -22.01837730407715,
      "activations/layer19_attention_weight_max": 45.78907012939453,
      "activations/layer19_attention_weight_min": -29.327640533447266,
      "activations/layer1_attention_weight_max": 17.45328712463379,
      "activations/layer1_attention_weight_min": -15.634288787841797,
      "activations/layer20_attention_weight_max": 41.511436462402344,
      "activations/layer20_attention_weight_min": -24.65704345703125,
      "activations/layer21_attention_weight_max": 35.277523040771484,
      "activations/layer21_attention_weight_min": -20.608144760131836,
      "activations/layer22_attention_weight_max": 59.17479705810547,
      "activations/layer22_attention_weight_min": -25.85666847229004,
      "activations/layer23_attention_weight_max": 35.713104248046875,
      "activations/layer23_attention_weight_min": -19.209095001220703,
      "activations/layer2_attention_weight_max": 34.142642974853516,
      "activations/layer2_attention_weight_min": -32.81288528442383,
      "activations/layer3_attention_weight_max": 89.01736450195312,
      "activations/layer3_attention_weight_min": -94.25525665283203,
      "activations/layer4_attention_weight_max": 137.48097229003906,
      "activations/layer4_attention_weight_min": -130.13629150390625,
      "activations/layer5_attention_weight_max": 90.9419174194336,
      "activations/layer5_attention_weight_min": -82.67218017578125,
      "activations/layer6_attention_weight_max": 59.12749481201172,
      "activations/layer6_attention_weight_min": -59.11438751220703,
      "activations/layer7_attention_weight_max": 81.38761138916016,
      "activations/layer7_attention_weight_min": -81.01602172851562,
      "activations/layer8_attention_weight_max": 42.357757568359375,
      "activations/layer8_attention_weight_min": -41.36929702758789,
      "activations/layer9_attention_weight_max": 40.75517272949219,
      "activations/layer9_attention_weight_min": -34.97267150878906,
      "epoch": 4.49,
      "learning_rate": 0.00012226401515151515,
      "loss": 2.866,
      "step": 77300
    },
    {
      "activations/layer0_attention_weight_max": 17.373550415039062,
      "activations/layer0_attention_weight_min": -11.004593849182129,
      "activations/layer10_attention_weight_max": 37.151615142822266,
      "activations/layer10_attention_weight_min": -29.7364559173584,
      "activations/layer11_attention_weight_max": 31.860172271728516,
      "activations/layer11_attention_weight_min": -26.62214469909668,
      "activations/layer12_attention_weight_max": 25.316951751708984,
      "activations/layer12_attention_weight_min": -27.223764419555664,
      "activations/layer13_attention_weight_max": 43.991493225097656,
      "activations/layer13_attention_weight_min": -32.707977294921875,
      "activations/layer14_attention_weight_max": 62.77992630004883,
      "activations/layer14_attention_weight_min": -43.433658599853516,
      "activations/layer15_attention_weight_max": 40.804481506347656,
      "activations/layer15_attention_weight_min": -28.05336570739746,
      "activations/layer16_attention_weight_max": 44.81357192993164,
      "activations/layer16_attention_weight_min": -27.31476593017578,
      "activations/layer17_attention_weight_max": 43.66291427612305,
      "activations/layer17_attention_weight_min": -25.156850814819336,
      "activations/layer18_attention_weight_max": 29.832759857177734,
      "activations/layer18_attention_weight_min": -19.43852424621582,
      "activations/layer19_attention_weight_max": 42.030941009521484,
      "activations/layer19_attention_weight_min": -28.353851318359375,
      "activations/layer1_attention_weight_max": 17.192968368530273,
      "activations/layer1_attention_weight_min": -14.281867027282715,
      "activations/layer20_attention_weight_max": 34.08130645751953,
      "activations/layer20_attention_weight_min": -24.364463806152344,
      "activations/layer21_attention_weight_max": 35.33161163330078,
      "activations/layer21_attention_weight_min": -19.930150985717773,
      "activations/layer22_attention_weight_max": 50.93893051147461,
      "activations/layer22_attention_weight_min": -27.75522232055664,
      "activations/layer23_attention_weight_max": 34.05210876464844,
      "activations/layer23_attention_weight_min": -20.675323486328125,
      "activations/layer2_attention_weight_max": 34.38561248779297,
      "activations/layer2_attention_weight_min": -33.549381256103516,
      "activations/layer3_attention_weight_max": 93.55189514160156,
      "activations/layer3_attention_weight_min": -91.63693237304688,
      "activations/layer4_attention_weight_max": 142.9069061279297,
      "activations/layer4_attention_weight_min": -135.07574462890625,
      "activations/layer5_attention_weight_max": 95.91273498535156,
      "activations/layer5_attention_weight_min": -86.91695404052734,
      "activations/layer6_attention_weight_max": 58.82733154296875,
      "activations/layer6_attention_weight_min": -60.963294982910156,
      "activations/layer7_attention_weight_max": 87.68487548828125,
      "activations/layer7_attention_weight_min": -82.3166275024414,
      "activations/layer8_attention_weight_max": 41.70194625854492,
      "activations/layer8_attention_weight_min": -40.20122146606445,
      "activations/layer9_attention_weight_max": 40.25018310546875,
      "activations/layer9_attention_weight_min": -32.95077133178711,
      "epoch": 4.49,
      "learning_rate": 0.00012224545454545455,
      "loss": 2.8678,
      "step": 77350
    },
    {
      "activations/layer0_attention_weight_max": 16.985990524291992,
      "activations/layer0_attention_weight_min": -11.892748832702637,
      "activations/layer10_attention_weight_max": 36.41981887817383,
      "activations/layer10_attention_weight_min": -31.159908294677734,
      "activations/layer11_attention_weight_max": 36.862525939941406,
      "activations/layer11_attention_weight_min": -27.002792358398438,
      "activations/layer12_attention_weight_max": 37.13572692871094,
      "activations/layer12_attention_weight_min": -27.82872200012207,
      "activations/layer13_attention_weight_max": 55.35828399658203,
      "activations/layer13_attention_weight_min": -36.84843444824219,
      "activations/layer14_attention_weight_max": 66.64531707763672,
      "activations/layer14_attention_weight_min": -51.143619537353516,
      "activations/layer15_attention_weight_max": 47.530967712402344,
      "activations/layer15_attention_weight_min": -28.34124755859375,
      "activations/layer16_attention_weight_max": 37.82222366333008,
      "activations/layer16_attention_weight_min": -29.518627166748047,
      "activations/layer17_attention_weight_max": 44.03269958496094,
      "activations/layer17_attention_weight_min": -25.700326919555664,
      "activations/layer18_attention_weight_max": 41.196258544921875,
      "activations/layer18_attention_weight_min": -23.203365325927734,
      "activations/layer19_attention_weight_max": 45.30717849731445,
      "activations/layer19_attention_weight_min": -32.70156478881836,
      "activations/layer1_attention_weight_max": 17.977657318115234,
      "activations/layer1_attention_weight_min": -13.699695587158203,
      "activations/layer20_attention_weight_max": 39.71154022216797,
      "activations/layer20_attention_weight_min": -23.635522842407227,
      "activations/layer21_attention_weight_max": 39.411373138427734,
      "activations/layer21_attention_weight_min": -21.8782958984375,
      "activations/layer22_attention_weight_max": 56.263153076171875,
      "activations/layer22_attention_weight_min": -28.472293853759766,
      "activations/layer23_attention_weight_max": 36.94797897338867,
      "activations/layer23_attention_weight_min": -20.255102157592773,
      "activations/layer2_attention_weight_max": 33.58257293701172,
      "activations/layer2_attention_weight_min": -30.489341735839844,
      "activations/layer3_attention_weight_max": 86.00640869140625,
      "activations/layer3_attention_weight_min": -87.290283203125,
      "activations/layer4_attention_weight_max": 124.93743133544922,
      "activations/layer4_attention_weight_min": -123.62457275390625,
      "activations/layer5_attention_weight_max": 82.24199676513672,
      "activations/layer5_attention_weight_min": -80.68321228027344,
      "activations/layer6_attention_weight_max": 55.68745040893555,
      "activations/layer6_attention_weight_min": -55.14312744140625,
      "activations/layer7_attention_weight_max": 78.32693481445312,
      "activations/layer7_attention_weight_min": -81.59201049804688,
      "activations/layer8_attention_weight_max": 40.303035736083984,
      "activations/layer8_attention_weight_min": -37.0650749206543,
      "activations/layer9_attention_weight_max": 37.65713119506836,
      "activations/layer9_attention_weight_min": -33.412376403808594,
      "epoch": 4.5,
      "learning_rate": 0.00012222651515151513,
      "loss": 2.8802,
      "step": 77400
    },
    {
      "activations/layer0_attention_weight_max": 16.997678756713867,
      "activations/layer0_attention_weight_min": -10.81021499633789,
      "activations/layer10_attention_weight_max": 31.64662742614746,
      "activations/layer10_attention_weight_min": -29.419979095458984,
      "activations/layer11_attention_weight_max": 31.161338806152344,
      "activations/layer11_attention_weight_min": -26.169095993041992,
      "activations/layer12_attention_weight_max": 25.627933502197266,
      "activations/layer12_attention_weight_min": -27.261287689208984,
      "activations/layer13_attention_weight_max": 40.25986862182617,
      "activations/layer13_attention_weight_min": -30.05620002746582,
      "activations/layer14_attention_weight_max": 58.25062561035156,
      "activations/layer14_attention_weight_min": -46.07693862915039,
      "activations/layer15_attention_weight_max": 39.64669418334961,
      "activations/layer15_attention_weight_min": -29.50107765197754,
      "activations/layer16_attention_weight_max": 37.141422271728516,
      "activations/layer16_attention_weight_min": -25.87608528137207,
      "activations/layer17_attention_weight_max": 39.23995590209961,
      "activations/layer17_attention_weight_min": -24.14499282836914,
      "activations/layer18_attention_weight_max": 33.88532638549805,
      "activations/layer18_attention_weight_min": -19.409439086914062,
      "activations/layer19_attention_weight_max": 41.37534713745117,
      "activations/layer19_attention_weight_min": -29.20699119567871,
      "activations/layer1_attention_weight_max": 17.743288040161133,
      "activations/layer1_attention_weight_min": -13.07482624053955,
      "activations/layer20_attention_weight_max": 31.939186096191406,
      "activations/layer20_attention_weight_min": -25.973665237426758,
      "activations/layer21_attention_weight_max": 34.66474533081055,
      "activations/layer21_attention_weight_min": -19.362756729125977,
      "activations/layer22_attention_weight_max": 48.595943450927734,
      "activations/layer22_attention_weight_min": -26.21111297607422,
      "activations/layer23_attention_weight_max": 32.47061538696289,
      "activations/layer23_attention_weight_min": -17.22175407409668,
      "activations/layer2_attention_weight_max": 33.7161750793457,
      "activations/layer2_attention_weight_min": -29.721355438232422,
      "activations/layer3_attention_weight_max": 86.36705780029297,
      "activations/layer3_attention_weight_min": -85.00106048583984,
      "activations/layer4_attention_weight_max": 131.0353240966797,
      "activations/layer4_attention_weight_min": -126.24059295654297,
      "activations/layer5_attention_weight_max": 85.94132995605469,
      "activations/layer5_attention_weight_min": -84.38557434082031,
      "activations/layer6_attention_weight_max": 56.13998794555664,
      "activations/layer6_attention_weight_min": -55.47456741333008,
      "activations/layer7_attention_weight_max": 79.6432113647461,
      "activations/layer7_attention_weight_min": -75.74578857421875,
      "activations/layer8_attention_weight_max": 37.87675476074219,
      "activations/layer8_attention_weight_min": -37.64097595214844,
      "activations/layer9_attention_weight_max": 35.53450393676758,
      "activations/layer9_attention_weight_min": -31.617206573486328,
      "epoch": 4.5,
      "learning_rate": 0.00012220757575757575,
      "loss": 2.8716,
      "step": 77450
    },
    {
      "activations/layer0_attention_weight_max": 17.39686393737793,
      "activations/layer0_attention_weight_min": -12.507749557495117,
      "activations/layer10_attention_weight_max": 35.2222900390625,
      "activations/layer10_attention_weight_min": -30.86556625366211,
      "activations/layer11_attention_weight_max": 32.35142517089844,
      "activations/layer11_attention_weight_min": -26.96413803100586,
      "activations/layer12_attention_weight_max": 52.506526947021484,
      "activations/layer12_attention_weight_min": -26.676095962524414,
      "activations/layer13_attention_weight_max": 47.12979507446289,
      "activations/layer13_attention_weight_min": -34.92877197265625,
      "activations/layer14_attention_weight_max": 67.85086059570312,
      "activations/layer14_attention_weight_min": -50.46220779418945,
      "activations/layer15_attention_weight_max": 43.14323043823242,
      "activations/layer15_attention_weight_min": -29.946393966674805,
      "activations/layer16_attention_weight_max": 37.403175354003906,
      "activations/layer16_attention_weight_min": -29.54248046875,
      "activations/layer17_attention_weight_max": 42.727882385253906,
      "activations/layer17_attention_weight_min": -26.371910095214844,
      "activations/layer18_attention_weight_max": 30.405038833618164,
      "activations/layer18_attention_weight_min": -19.134994506835938,
      "activations/layer19_attention_weight_max": 40.264244079589844,
      "activations/layer19_attention_weight_min": -31.5244083404541,
      "activations/layer1_attention_weight_max": 17.313201904296875,
      "activations/layer1_attention_weight_min": -15.870720863342285,
      "activations/layer20_attention_weight_max": 34.36618423461914,
      "activations/layer20_attention_weight_min": -24.085939407348633,
      "activations/layer21_attention_weight_max": 32.73367691040039,
      "activations/layer21_attention_weight_min": -20.09672737121582,
      "activations/layer22_attention_weight_max": 50.75625991821289,
      "activations/layer22_attention_weight_min": -29.76067352294922,
      "activations/layer23_attention_weight_max": 35.243621826171875,
      "activations/layer23_attention_weight_min": -21.66061782836914,
      "activations/layer2_attention_weight_max": 32.23846435546875,
      "activations/layer2_attention_weight_min": -31.216154098510742,
      "activations/layer3_attention_weight_max": 90.00532531738281,
      "activations/layer3_attention_weight_min": -89.58091735839844,
      "activations/layer4_attention_weight_max": 128.52708435058594,
      "activations/layer4_attention_weight_min": -124.11344146728516,
      "activations/layer5_attention_weight_max": 87.48662567138672,
      "activations/layer5_attention_weight_min": -80.42606353759766,
      "activations/layer6_attention_weight_max": 56.179935455322266,
      "activations/layer6_attention_weight_min": -56.65446090698242,
      "activations/layer7_attention_weight_max": 81.51425170898438,
      "activations/layer7_attention_weight_min": -79.9051284790039,
      "activations/layer8_attention_weight_max": 39.98653030395508,
      "activations/layer8_attention_weight_min": -38.80373001098633,
      "activations/layer9_attention_weight_max": 39.42243576049805,
      "activations/layer9_attention_weight_min": -33.583248138427734,
      "epoch": 4.5,
      "learning_rate": 0.00012218863636363636,
      "loss": 2.8764,
      "step": 77500
    },
    {
      "activations/layer0_attention_weight_max": 16.482080459594727,
      "activations/layer0_attention_weight_min": -12.920462608337402,
      "activations/layer10_attention_weight_max": 33.237457275390625,
      "activations/layer10_attention_weight_min": -29.591970443725586,
      "activations/layer11_attention_weight_max": 31.984716415405273,
      "activations/layer11_attention_weight_min": -26.966276168823242,
      "activations/layer12_attention_weight_max": 38.24288558959961,
      "activations/layer12_attention_weight_min": -28.472837448120117,
      "activations/layer13_attention_weight_max": 47.73416519165039,
      "activations/layer13_attention_weight_min": -31.748844146728516,
      "activations/layer14_attention_weight_max": 78.56156158447266,
      "activations/layer14_attention_weight_min": -48.572566986083984,
      "activations/layer15_attention_weight_max": 41.182456970214844,
      "activations/layer15_attention_weight_min": -27.445234298706055,
      "activations/layer16_attention_weight_max": 38.59803771972656,
      "activations/layer16_attention_weight_min": -31.49726104736328,
      "activations/layer17_attention_weight_max": 39.5423469543457,
      "activations/layer17_attention_weight_min": -25.726076126098633,
      "activations/layer18_attention_weight_max": 35.64358901977539,
      "activations/layer18_attention_weight_min": -20.960506439208984,
      "activations/layer19_attention_weight_max": 39.0930290222168,
      "activations/layer19_attention_weight_min": -28.91725730895996,
      "activations/layer1_attention_weight_max": 17.059009552001953,
      "activations/layer1_attention_weight_min": -15.033625602722168,
      "activations/layer20_attention_weight_max": 32.385921478271484,
      "activations/layer20_attention_weight_min": -23.836101531982422,
      "activations/layer21_attention_weight_max": 35.0390510559082,
      "activations/layer21_attention_weight_min": -25.118480682373047,
      "activations/layer22_attention_weight_max": 52.58262634277344,
      "activations/layer22_attention_weight_min": -28.67581558227539,
      "activations/layer23_attention_weight_max": 32.124595642089844,
      "activations/layer23_attention_weight_min": -20.287874221801758,
      "activations/layer2_attention_weight_max": 34.382415771484375,
      "activations/layer2_attention_weight_min": -31.239683151245117,
      "activations/layer3_attention_weight_max": 87.99073028564453,
      "activations/layer3_attention_weight_min": -87.79518127441406,
      "activations/layer4_attention_weight_max": 133.58367919921875,
      "activations/layer4_attention_weight_min": -128.59445190429688,
      "activations/layer5_attention_weight_max": 90.48312377929688,
      "activations/layer5_attention_weight_min": -87.85159301757812,
      "activations/layer6_attention_weight_max": 59.010963439941406,
      "activations/layer6_attention_weight_min": -58.393218994140625,
      "activations/layer7_attention_weight_max": 79.44571685791016,
      "activations/layer7_attention_weight_min": -82.58679962158203,
      "activations/layer8_attention_weight_max": 40.071739196777344,
      "activations/layer8_attention_weight_min": -39.015480041503906,
      "activations/layer9_attention_weight_max": 37.832759857177734,
      "activations/layer9_attention_weight_min": -34.40110397338867,
      "epoch": 4.51,
      "learning_rate": 0.00012216969696969695,
      "loss": 2.8582,
      "step": 77550
    },
    {
      "activations/layer0_attention_weight_max": 15.157366752624512,
      "activations/layer0_attention_weight_min": -12.111895561218262,
      "activations/layer10_attention_weight_max": 37.792152404785156,
      "activations/layer10_attention_weight_min": -32.31806182861328,
      "activations/layer11_attention_weight_max": 37.34075927734375,
      "activations/layer11_attention_weight_min": -28.12068748474121,
      "activations/layer12_attention_weight_max": 37.48838424682617,
      "activations/layer12_attention_weight_min": -27.795852661132812,
      "activations/layer13_attention_weight_max": 52.874019622802734,
      "activations/layer13_attention_weight_min": -35.649349212646484,
      "activations/layer14_attention_weight_max": 86.59207153320312,
      "activations/layer14_attention_weight_min": -52.564945220947266,
      "activations/layer15_attention_weight_max": 53.23431396484375,
      "activations/layer15_attention_weight_min": -32.114776611328125,
      "activations/layer16_attention_weight_max": 43.391319274902344,
      "activations/layer16_attention_weight_min": -27.07269287109375,
      "activations/layer17_attention_weight_max": 46.280025482177734,
      "activations/layer17_attention_weight_min": -28.425182342529297,
      "activations/layer18_attention_weight_max": 35.10004806518555,
      "activations/layer18_attention_weight_min": -20.348539352416992,
      "activations/layer19_attention_weight_max": 53.236873626708984,
      "activations/layer19_attention_weight_min": -30.899662017822266,
      "activations/layer1_attention_weight_max": 16.517375946044922,
      "activations/layer1_attention_weight_min": -14.088418960571289,
      "activations/layer20_attention_weight_max": 36.42112731933594,
      "activations/layer20_attention_weight_min": -29.310283660888672,
      "activations/layer21_attention_weight_max": 34.87955856323242,
      "activations/layer21_attention_weight_min": -22.82842254638672,
      "activations/layer22_attention_weight_max": 59.42664337158203,
      "activations/layer22_attention_weight_min": -28.628293991088867,
      "activations/layer23_attention_weight_max": 35.73094940185547,
      "activations/layer23_attention_weight_min": -17.473230361938477,
      "activations/layer2_attention_weight_max": 32.581092834472656,
      "activations/layer2_attention_weight_min": -31.5216121673584,
      "activations/layer3_attention_weight_max": 88.51952362060547,
      "activations/layer3_attention_weight_min": -92.16433715820312,
      "activations/layer4_attention_weight_max": 131.17849731445312,
      "activations/layer4_attention_weight_min": -130.41845703125,
      "activations/layer5_attention_weight_max": 92.1638412475586,
      "activations/layer5_attention_weight_min": -83.69912719726562,
      "activations/layer6_attention_weight_max": 56.51609802246094,
      "activations/layer6_attention_weight_min": -59.94677734375,
      "activations/layer7_attention_weight_max": 84.07754516601562,
      "activations/layer7_attention_weight_min": -85.02397155761719,
      "activations/layer8_attention_weight_max": 41.495670318603516,
      "activations/layer8_attention_weight_min": -40.12196350097656,
      "activations/layer9_attention_weight_max": 40.671539306640625,
      "activations/layer9_attention_weight_min": -35.499996185302734,
      "epoch": 4.51,
      "learning_rate": 0.00012215075757575757,
      "loss": 2.8641,
      "step": 77600
    },
    {
      "activations/layer0_attention_weight_max": 16.509798049926758,
      "activations/layer0_attention_weight_min": -13.162217140197754,
      "activations/layer10_attention_weight_max": 33.324562072753906,
      "activations/layer10_attention_weight_min": -29.297466278076172,
      "activations/layer11_attention_weight_max": 33.05813217163086,
      "activations/layer11_attention_weight_min": -27.809799194335938,
      "activations/layer12_attention_weight_max": 32.897926330566406,
      "activations/layer12_attention_weight_min": -27.60584259033203,
      "activations/layer13_attention_weight_max": 43.44804763793945,
      "activations/layer13_attention_weight_min": -33.43892288208008,
      "activations/layer14_attention_weight_max": 59.1013069152832,
      "activations/layer14_attention_weight_min": -47.7732048034668,
      "activations/layer15_attention_weight_max": 41.31414794921875,
      "activations/layer15_attention_weight_min": -28.839784622192383,
      "activations/layer16_attention_weight_max": 35.95193862915039,
      "activations/layer16_attention_weight_min": -28.349422454833984,
      "activations/layer17_attention_weight_max": 41.856563568115234,
      "activations/layer17_attention_weight_min": -24.1931095123291,
      "activations/layer18_attention_weight_max": 31.14433479309082,
      "activations/layer18_attention_weight_min": -20.899171829223633,
      "activations/layer19_attention_weight_max": 37.3526725769043,
      "activations/layer19_attention_weight_min": -30.7094783782959,
      "activations/layer1_attention_weight_max": 17.7590274810791,
      "activations/layer1_attention_weight_min": -13.701229095458984,
      "activations/layer20_attention_weight_max": 28.859207153320312,
      "activations/layer20_attention_weight_min": -22.480443954467773,
      "activations/layer21_attention_weight_max": 30.581436157226562,
      "activations/layer21_attention_weight_min": -20.654672622680664,
      "activations/layer22_attention_weight_max": 46.2634391784668,
      "activations/layer22_attention_weight_min": -30.853694915771484,
      "activations/layer23_attention_weight_max": 31.119983673095703,
      "activations/layer23_attention_weight_min": -23.478567123413086,
      "activations/layer2_attention_weight_max": 32.85486602783203,
      "activations/layer2_attention_weight_min": -30.317825317382812,
      "activations/layer3_attention_weight_max": 89.8358154296875,
      "activations/layer3_attention_weight_min": -90.52508544921875,
      "activations/layer4_attention_weight_max": 133.080078125,
      "activations/layer4_attention_weight_min": -128.8684539794922,
      "activations/layer5_attention_weight_max": 90.47509002685547,
      "activations/layer5_attention_weight_min": -85.54900360107422,
      "activations/layer6_attention_weight_max": 58.45529556274414,
      "activations/layer6_attention_weight_min": -59.281280517578125,
      "activations/layer7_attention_weight_max": 78.13135528564453,
      "activations/layer7_attention_weight_min": -80.02202606201172,
      "activations/layer8_attention_weight_max": 38.09706497192383,
      "activations/layer8_attention_weight_min": -39.584571838378906,
      "activations/layer9_attention_weight_max": 34.911285400390625,
      "activations/layer9_attention_weight_min": -34.69718933105469,
      "epoch": 4.51,
      "learning_rate": 0.00012213181818181815,
      "loss": 2.8642,
      "step": 77650
    },
    {
      "activations/layer0_attention_weight_max": 16.8303165435791,
      "activations/layer0_attention_weight_min": -12.56550121307373,
      "activations/layer10_attention_weight_max": 35.12700653076172,
      "activations/layer10_attention_weight_min": -31.62912368774414,
      "activations/layer11_attention_weight_max": 31.789823532104492,
      "activations/layer11_attention_weight_min": -26.34307098388672,
      "activations/layer12_attention_weight_max": 27.441974639892578,
      "activations/layer12_attention_weight_min": -26.76477813720703,
      "activations/layer13_attention_weight_max": 41.70372009277344,
      "activations/layer13_attention_weight_min": -34.05036163330078,
      "activations/layer14_attention_weight_max": 63.62175369262695,
      "activations/layer14_attention_weight_min": -46.856624603271484,
      "activations/layer15_attention_weight_max": 37.268104553222656,
      "activations/layer15_attention_weight_min": -28.86871910095215,
      "activations/layer16_attention_weight_max": 40.19285583496094,
      "activations/layer16_attention_weight_min": -28.47511863708496,
      "activations/layer17_attention_weight_max": 37.84688949584961,
      "activations/layer17_attention_weight_min": -24.92313575744629,
      "activations/layer18_attention_weight_max": 28.449308395385742,
      "activations/layer18_attention_weight_min": -18.679683685302734,
      "activations/layer19_attention_weight_max": 43.22745895385742,
      "activations/layer19_attention_weight_min": -28.111536026000977,
      "activations/layer1_attention_weight_max": 17.23040199279785,
      "activations/layer1_attention_weight_min": -13.265318870544434,
      "activations/layer20_attention_weight_max": 33.96437072753906,
      "activations/layer20_attention_weight_min": -24.797649383544922,
      "activations/layer21_attention_weight_max": 30.992919921875,
      "activations/layer21_attention_weight_min": -20.361305236816406,
      "activations/layer22_attention_weight_max": 49.59580612182617,
      "activations/layer22_attention_weight_min": -27.89676856994629,
      "activations/layer23_attention_weight_max": 30.808334350585938,
      "activations/layer23_attention_weight_min": -19.491880416870117,
      "activations/layer2_attention_weight_max": 31.913198471069336,
      "activations/layer2_attention_weight_min": -30.034404754638672,
      "activations/layer3_attention_weight_max": 88.20310974121094,
      "activations/layer3_attention_weight_min": -88.8885498046875,
      "activations/layer4_attention_weight_max": 126.03934478759766,
      "activations/layer4_attention_weight_min": -123.93563079833984,
      "activations/layer5_attention_weight_max": 82.14218139648438,
      "activations/layer5_attention_weight_min": -79.75061798095703,
      "activations/layer6_attention_weight_max": 54.157527923583984,
      "activations/layer6_attention_weight_min": -58.423561096191406,
      "activations/layer7_attention_weight_max": 85.6427230834961,
      "activations/layer7_attention_weight_min": -84.50163269042969,
      "activations/layer8_attention_weight_max": 40.86049270629883,
      "activations/layer8_attention_weight_min": -41.721534729003906,
      "activations/layer9_attention_weight_max": 36.21134567260742,
      "activations/layer9_attention_weight_min": -33.57255172729492,
      "epoch": 4.51,
      "learning_rate": 0.00012211287878787877,
      "loss": 2.8688,
      "step": 77700
    },
    {
      "activations/layer0_attention_weight_max": 16.19775390625,
      "activations/layer0_attention_weight_min": -12.349382400512695,
      "activations/layer10_attention_weight_max": 37.83681106567383,
      "activations/layer10_attention_weight_min": -31.47439193725586,
      "activations/layer11_attention_weight_max": 40.493003845214844,
      "activations/layer11_attention_weight_min": -29.597389221191406,
      "activations/layer12_attention_weight_max": 40.20615005493164,
      "activations/layer12_attention_weight_min": -24.653400421142578,
      "activations/layer13_attention_weight_max": 48.53498840332031,
      "activations/layer13_attention_weight_min": -34.1329345703125,
      "activations/layer14_attention_weight_max": 62.84705352783203,
      "activations/layer14_attention_weight_min": -49.496849060058594,
      "activations/layer15_attention_weight_max": 54.652931213378906,
      "activations/layer15_attention_weight_min": -30.982528686523438,
      "activations/layer16_attention_weight_max": 41.39241027832031,
      "activations/layer16_attention_weight_min": -29.427343368530273,
      "activations/layer17_attention_weight_max": 38.736236572265625,
      "activations/layer17_attention_weight_min": -25.079269409179688,
      "activations/layer18_attention_weight_max": 29.53483772277832,
      "activations/layer18_attention_weight_min": -20.168312072753906,
      "activations/layer19_attention_weight_max": 41.463436126708984,
      "activations/layer19_attention_weight_min": -28.488401412963867,
      "activations/layer1_attention_weight_max": 17.47212028503418,
      "activations/layer1_attention_weight_min": -15.847898483276367,
      "activations/layer20_attention_weight_max": 28.40144920349121,
      "activations/layer20_attention_weight_min": -22.42905616760254,
      "activations/layer21_attention_weight_max": 29.59836196899414,
      "activations/layer21_attention_weight_min": -20.471969604492188,
      "activations/layer22_attention_weight_max": 46.19732666015625,
      "activations/layer22_attention_weight_min": -25.68121910095215,
      "activations/layer23_attention_weight_max": 31.21603012084961,
      "activations/layer23_attention_weight_min": -20.188213348388672,
      "activations/layer2_attention_weight_max": 34.34241485595703,
      "activations/layer2_attention_weight_min": -34.60343933105469,
      "activations/layer3_attention_weight_max": 90.47178649902344,
      "activations/layer3_attention_weight_min": -93.76530456542969,
      "activations/layer4_attention_weight_max": 137.61923217773438,
      "activations/layer4_attention_weight_min": -135.05056762695312,
      "activations/layer5_attention_weight_max": 91.03981018066406,
      "activations/layer5_attention_weight_min": -84.86131286621094,
      "activations/layer6_attention_weight_max": 57.45243835449219,
      "activations/layer6_attention_weight_min": -60.95134353637695,
      "activations/layer7_attention_weight_max": 91.31889343261719,
      "activations/layer7_attention_weight_min": -83.5625228881836,
      "activations/layer8_attention_weight_max": 41.95764923095703,
      "activations/layer8_attention_weight_min": -40.09328079223633,
      "activations/layer9_attention_weight_max": 40.5064811706543,
      "activations/layer9_attention_weight_min": -37.87644577026367,
      "epoch": 4.52,
      "learning_rate": 0.00012209393939393938,
      "loss": 2.8602,
      "step": 77750
    },
    {
      "activations/layer0_attention_weight_max": 16.99179458618164,
      "activations/layer0_attention_weight_min": -11.374275207519531,
      "activations/layer10_attention_weight_max": 34.989444732666016,
      "activations/layer10_attention_weight_min": -31.974802017211914,
      "activations/layer11_attention_weight_max": 31.727996826171875,
      "activations/layer11_attention_weight_min": -26.83901023864746,
      "activations/layer12_attention_weight_max": 28.046186447143555,
      "activations/layer12_attention_weight_min": -26.28791046142578,
      "activations/layer13_attention_weight_max": 41.288917541503906,
      "activations/layer13_attention_weight_min": -34.23143005371094,
      "activations/layer14_attention_weight_max": 56.83481216430664,
      "activations/layer14_attention_weight_min": -46.44833755493164,
      "activations/layer15_attention_weight_max": 41.02741241455078,
      "activations/layer15_attention_weight_min": -28.034387588500977,
      "activations/layer16_attention_weight_max": 36.621517181396484,
      "activations/layer16_attention_weight_min": -30.512311935424805,
      "activations/layer17_attention_weight_max": 40.0955696105957,
      "activations/layer17_attention_weight_min": -26.978944778442383,
      "activations/layer18_attention_weight_max": 37.909000396728516,
      "activations/layer18_attention_weight_min": -21.15339469909668,
      "activations/layer19_attention_weight_max": 38.10542678833008,
      "activations/layer19_attention_weight_min": -29.508283615112305,
      "activations/layer1_attention_weight_max": 17.05415153503418,
      "activations/layer1_attention_weight_min": -12.284309387207031,
      "activations/layer20_attention_weight_max": 30.55809211730957,
      "activations/layer20_attention_weight_min": -24.200916290283203,
      "activations/layer21_attention_weight_max": 33.95968246459961,
      "activations/layer21_attention_weight_min": -20.470870971679688,
      "activations/layer22_attention_weight_max": 48.734130859375,
      "activations/layer22_attention_weight_min": -27.368743896484375,
      "activations/layer23_attention_weight_max": 29.9864559173584,
      "activations/layer23_attention_weight_min": -19.43409538269043,
      "activations/layer2_attention_weight_max": 31.74664306640625,
      "activations/layer2_attention_weight_min": -30.24036979675293,
      "activations/layer3_attention_weight_max": 86.83688354492188,
      "activations/layer3_attention_weight_min": -88.90039825439453,
      "activations/layer4_attention_weight_max": 135.3514404296875,
      "activations/layer4_attention_weight_min": -129.90623474121094,
      "activations/layer5_attention_weight_max": 90.81624603271484,
      "activations/layer5_attention_weight_min": -86.29659271240234,
      "activations/layer6_attention_weight_max": 58.83231735229492,
      "activations/layer6_attention_weight_min": -58.0360221862793,
      "activations/layer7_attention_weight_max": 83.8704605102539,
      "activations/layer7_attention_weight_min": -86.96546173095703,
      "activations/layer8_attention_weight_max": 41.21107864379883,
      "activations/layer8_attention_weight_min": -38.23483657836914,
      "activations/layer9_attention_weight_max": 37.487548828125,
      "activations/layer9_attention_weight_min": -33.292598724365234,
      "epoch": 4.52,
      "learning_rate": 0.00012207499999999997,
      "loss": 2.8552,
      "step": 77800
    },
    {
      "activations/layer0_attention_weight_max": 16.136789321899414,
      "activations/layer0_attention_weight_min": -12.098854064941406,
      "activations/layer10_attention_weight_max": 31.59479331970215,
      "activations/layer10_attention_weight_min": -31.215890884399414,
      "activations/layer11_attention_weight_max": 32.62725830078125,
      "activations/layer11_attention_weight_min": -27.690013885498047,
      "activations/layer12_attention_weight_max": 32.215110778808594,
      "activations/layer12_attention_weight_min": -25.83309555053711,
      "activations/layer13_attention_weight_max": 51.45111846923828,
      "activations/layer13_attention_weight_min": -33.23136520385742,
      "activations/layer14_attention_weight_max": 67.04012298583984,
      "activations/layer14_attention_weight_min": -51.09402084350586,
      "activations/layer15_attention_weight_max": 42.88771438598633,
      "activations/layer15_attention_weight_min": -30.662433624267578,
      "activations/layer16_attention_weight_max": 40.006526947021484,
      "activations/layer16_attention_weight_min": -27.810882568359375,
      "activations/layer17_attention_weight_max": 40.404541015625,
      "activations/layer17_attention_weight_min": -25.259355545043945,
      "activations/layer18_attention_weight_max": 33.97057342529297,
      "activations/layer18_attention_weight_min": -19.667692184448242,
      "activations/layer19_attention_weight_max": 44.73920440673828,
      "activations/layer19_attention_weight_min": -31.229156494140625,
      "activations/layer1_attention_weight_max": 17.0759334564209,
      "activations/layer1_attention_weight_min": -13.973851203918457,
      "activations/layer20_attention_weight_max": 39.749900817871094,
      "activations/layer20_attention_weight_min": -24.51227378845215,
      "activations/layer21_attention_weight_max": 35.345985412597656,
      "activations/layer21_attention_weight_min": -22.002779006958008,
      "activations/layer22_attention_weight_max": 50.95956039428711,
      "activations/layer22_attention_weight_min": -26.81490707397461,
      "activations/layer23_attention_weight_max": 31.52851676940918,
      "activations/layer23_attention_weight_min": -20.668649673461914,
      "activations/layer2_attention_weight_max": 31.703510284423828,
      "activations/layer2_attention_weight_min": -31.068029403686523,
      "activations/layer3_attention_weight_max": 89.4457778930664,
      "activations/layer3_attention_weight_min": -93.60697937011719,
      "activations/layer4_attention_weight_max": 136.98828125,
      "activations/layer4_attention_weight_min": -133.27838134765625,
      "activations/layer5_attention_weight_max": 87.15472412109375,
      "activations/layer5_attention_weight_min": -88.11773681640625,
      "activations/layer6_attention_weight_max": 57.177467346191406,
      "activations/layer6_attention_weight_min": -57.90239334106445,
      "activations/layer7_attention_weight_max": 78.19767761230469,
      "activations/layer7_attention_weight_min": -88.37017059326172,
      "activations/layer8_attention_weight_max": 39.77239227294922,
      "activations/layer8_attention_weight_min": -39.486209869384766,
      "activations/layer9_attention_weight_max": 36.58986282348633,
      "activations/layer9_attention_weight_min": -34.69156265258789,
      "epoch": 4.52,
      "learning_rate": 0.0001220560606060606,
      "loss": 2.864,
      "step": 77850
    },
    {
      "activations/layer0_attention_weight_max": 15.620588302612305,
      "activations/layer0_attention_weight_min": -11.790919303894043,
      "activations/layer10_attention_weight_max": 36.769622802734375,
      "activations/layer10_attention_weight_min": -32.06416702270508,
      "activations/layer11_attention_weight_max": 35.1375846862793,
      "activations/layer11_attention_weight_min": -27.70429229736328,
      "activations/layer12_attention_weight_max": 41.3571891784668,
      "activations/layer12_attention_weight_min": -28.509559631347656,
      "activations/layer13_attention_weight_max": 52.97187042236328,
      "activations/layer13_attention_weight_min": -36.90975570678711,
      "activations/layer14_attention_weight_max": 71.3538589477539,
      "activations/layer14_attention_weight_min": -53.3447265625,
      "activations/layer15_attention_weight_max": 49.08317947387695,
      "activations/layer15_attention_weight_min": -28.58518409729004,
      "activations/layer16_attention_weight_max": 39.79138946533203,
      "activations/layer16_attention_weight_min": -29.588455200195312,
      "activations/layer17_attention_weight_max": 43.992889404296875,
      "activations/layer17_attention_weight_min": -24.321640014648438,
      "activations/layer18_attention_weight_max": 38.8289680480957,
      "activations/layer18_attention_weight_min": -19.73067855834961,
      "activations/layer19_attention_weight_max": 46.9414176940918,
      "activations/layer19_attention_weight_min": -28.44392967224121,
      "activations/layer1_attention_weight_max": 16.979955673217773,
      "activations/layer1_attention_weight_min": -14.582677841186523,
      "activations/layer20_attention_weight_max": 37.288028717041016,
      "activations/layer20_attention_weight_min": -23.405458450317383,
      "activations/layer21_attention_weight_max": 41.41855239868164,
      "activations/layer21_attention_weight_min": -20.681480407714844,
      "activations/layer22_attention_weight_max": 55.82041549682617,
      "activations/layer22_attention_weight_min": -27.292945861816406,
      "activations/layer23_attention_weight_max": 36.198204040527344,
      "activations/layer23_attention_weight_min": -18.402002334594727,
      "activations/layer2_attention_weight_max": 31.8654842376709,
      "activations/layer2_attention_weight_min": -30.47609519958496,
      "activations/layer3_attention_weight_max": 89.38689422607422,
      "activations/layer3_attention_weight_min": -89.69512939453125,
      "activations/layer4_attention_weight_max": 134.59495544433594,
      "activations/layer4_attention_weight_min": -130.02597045898438,
      "activations/layer5_attention_weight_max": 91.02733612060547,
      "activations/layer5_attention_weight_min": -88.11922454833984,
      "activations/layer6_attention_weight_max": 64.19914245605469,
      "activations/layer6_attention_weight_min": -59.772613525390625,
      "activations/layer7_attention_weight_max": 85.22085571289062,
      "activations/layer7_attention_weight_min": -83.45003509521484,
      "activations/layer8_attention_weight_max": 43.32852554321289,
      "activations/layer8_attention_weight_min": -43.27004623413086,
      "activations/layer9_attention_weight_max": 39.16487121582031,
      "activations/layer9_attention_weight_min": -35.69569396972656,
      "epoch": 4.53,
      "learning_rate": 0.0001220371212121212,
      "loss": 2.8663,
      "step": 77900
    },
    {
      "activations/layer0_attention_weight_max": 16.630311965942383,
      "activations/layer0_attention_weight_min": -12.441035270690918,
      "activations/layer10_attention_weight_max": 38.60226058959961,
      "activations/layer10_attention_weight_min": -30.30666732788086,
      "activations/layer11_attention_weight_max": 35.348289489746094,
      "activations/layer11_attention_weight_min": -27.657791137695312,
      "activations/layer12_attention_weight_max": 42.4121208190918,
      "activations/layer12_attention_weight_min": -26.77372932434082,
      "activations/layer13_attention_weight_max": 40.980438232421875,
      "activations/layer13_attention_weight_min": -32.466400146484375,
      "activations/layer14_attention_weight_max": 76.85781860351562,
      "activations/layer14_attention_weight_min": -60.985023498535156,
      "activations/layer15_attention_weight_max": 39.45315170288086,
      "activations/layer15_attention_weight_min": -27.452579498291016,
      "activations/layer16_attention_weight_max": 39.8004150390625,
      "activations/layer16_attention_weight_min": -28.12978744506836,
      "activations/layer17_attention_weight_max": 37.389617919921875,
      "activations/layer17_attention_weight_min": -25.347200393676758,
      "activations/layer18_attention_weight_max": 31.019229888916016,
      "activations/layer18_attention_weight_min": -19.31798553466797,
      "activations/layer19_attention_weight_max": 39.1284294128418,
      "activations/layer19_attention_weight_min": -30.40105628967285,
      "activations/layer1_attention_weight_max": 17.36233901977539,
      "activations/layer1_attention_weight_min": -15.521405220031738,
      "activations/layer20_attention_weight_max": 31.44501304626465,
      "activations/layer20_attention_weight_min": -22.529983520507812,
      "activations/layer21_attention_weight_max": 30.45029640197754,
      "activations/layer21_attention_weight_min": -19.647216796875,
      "activations/layer22_attention_weight_max": 51.82649612426758,
      "activations/layer22_attention_weight_min": -29.14910888671875,
      "activations/layer23_attention_weight_max": 32.77279281616211,
      "activations/layer23_attention_weight_min": -18.8454647064209,
      "activations/layer2_attention_weight_max": 32.6749153137207,
      "activations/layer2_attention_weight_min": -32.52211380004883,
      "activations/layer3_attention_weight_max": 93.56241607666016,
      "activations/layer3_attention_weight_min": -94.25149536132812,
      "activations/layer4_attention_weight_max": 140.06529235839844,
      "activations/layer4_attention_weight_min": -141.61964416503906,
      "activations/layer5_attention_weight_max": 94.68798828125,
      "activations/layer5_attention_weight_min": -85.9222412109375,
      "activations/layer6_attention_weight_max": 64.5050277709961,
      "activations/layer6_attention_weight_min": -63.08277893066406,
      "activations/layer7_attention_weight_max": 91.6185302734375,
      "activations/layer7_attention_weight_min": -86.39273834228516,
      "activations/layer8_attention_weight_max": 46.001800537109375,
      "activations/layer8_attention_weight_min": -40.31937789916992,
      "activations/layer9_attention_weight_max": 40.709503173828125,
      "activations/layer9_attention_weight_min": -39.4757194519043,
      "epoch": 4.53,
      "learning_rate": 0.0001220181818181818,
      "loss": 2.8637,
      "step": 77950
    },
    {
      "activations/layer0_attention_weight_max": 16.468618392944336,
      "activations/layer0_attention_weight_min": -12.649374008178711,
      "activations/layer10_attention_weight_max": 40.14389419555664,
      "activations/layer10_attention_weight_min": -31.891902923583984,
      "activations/layer11_attention_weight_max": 45.78715133666992,
      "activations/layer11_attention_weight_min": -29.537458419799805,
      "activations/layer12_attention_weight_max": 26.890771865844727,
      "activations/layer12_attention_weight_min": -25.68852424621582,
      "activations/layer13_attention_weight_max": 44.1832160949707,
      "activations/layer13_attention_weight_min": -34.87793731689453,
      "activations/layer14_attention_weight_max": 66.23332214355469,
      "activations/layer14_attention_weight_min": -46.22049331665039,
      "activations/layer15_attention_weight_max": 46.3111572265625,
      "activations/layer15_attention_weight_min": -29.443565368652344,
      "activations/layer16_attention_weight_max": 37.44939422607422,
      "activations/layer16_attention_weight_min": -26.86817741394043,
      "activations/layer17_attention_weight_max": 42.98090744018555,
      "activations/layer17_attention_weight_min": -24.09168815612793,
      "activations/layer18_attention_weight_max": 30.947120666503906,
      "activations/layer18_attention_weight_min": -18.612577438354492,
      "activations/layer19_attention_weight_max": 40.316036224365234,
      "activations/layer19_attention_weight_min": -27.721826553344727,
      "activations/layer1_attention_weight_max": 17.258493423461914,
      "activations/layer1_attention_weight_min": -14.719098091125488,
      "activations/layer20_attention_weight_max": 37.72776794433594,
      "activations/layer20_attention_weight_min": -22.286046981811523,
      "activations/layer21_attention_weight_max": 33.9068603515625,
      "activations/layer21_attention_weight_min": -20.890281677246094,
      "activations/layer22_attention_weight_max": 50.09992218017578,
      "activations/layer22_attention_weight_min": -25.46367835998535,
      "activations/layer23_attention_weight_max": 33.271629333496094,
      "activations/layer23_attention_weight_min": -20.569578170776367,
      "activations/layer2_attention_weight_max": 32.70512771606445,
      "activations/layer2_attention_weight_min": -31.14025115966797,
      "activations/layer3_attention_weight_max": 89.58552551269531,
      "activations/layer3_attention_weight_min": -86.11557006835938,
      "activations/layer4_attention_weight_max": 128.32034301757812,
      "activations/layer4_attention_weight_min": -123.72408294677734,
      "activations/layer5_attention_weight_max": 86.91728973388672,
      "activations/layer5_attention_weight_min": -83.22358703613281,
      "activations/layer6_attention_weight_max": 59.39181900024414,
      "activations/layer6_attention_weight_min": -58.68107223510742,
      "activations/layer7_attention_weight_max": 87.23687744140625,
      "activations/layer7_attention_weight_min": -84.17098236083984,
      "activations/layer8_attention_weight_max": 41.93624496459961,
      "activations/layer8_attention_weight_min": -45.94947052001953,
      "activations/layer9_attention_weight_max": 41.98958969116211,
      "activations/layer9_attention_weight_min": -37.09849548339844,
      "epoch": 4.53,
      "learning_rate": 0.00012199924242424242,
      "loss": 2.8551,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_loss": 2.8125,
      "eval_runtime": 8.4567,
      "eval_samples_per_second": 507.765,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_openwebtext_loss": 2.8125,
      "eval_openwebtext_ppl": 16.651494963610144,
      "eval_openwebtext_runtime": 8.4567,
      "eval_openwebtext_samples_per_second": 507.765,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_wikitext_loss": 3.056640625,
      "eval_wikitext_ppl": 21.256030110048442,
      "eval_wikitext_runtime": 2.0039,
      "eval_wikitext_samples_per_second": 227.553,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_lambada_loss": 2.767578125,
      "eval_lambada_ppl": 15.920030957803608,
      "eval_lambada_runtime": 9.5298,
      "eval_lambada_samples_per_second": 510.925,
      "step": 78000
    },
    {
      "activations/layer0_attention_weight_max": 17.981584548950195,
      "activations/layer0_attention_weight_min": -11.861802101135254,
      "activations/layer10_attention_weight_max": 36.46505355834961,
      "activations/layer10_attention_weight_min": -29.913524627685547,
      "activations/layer11_attention_weight_max": 36.9108772277832,
      "activations/layer11_attention_weight_min": -29.114477157592773,
      "activations/layer12_attention_weight_max": 50.66618347167969,
      "activations/layer12_attention_weight_min": -33.77163314819336,
      "activations/layer13_attention_weight_max": 62.07188415527344,
      "activations/layer13_attention_weight_min": -37.52938461303711,
      "activations/layer14_attention_weight_max": 63.56826400756836,
      "activations/layer14_attention_weight_min": -43.792442321777344,
      "activations/layer15_attention_weight_max": 47.59688949584961,
      "activations/layer15_attention_weight_min": -28.855243682861328,
      "activations/layer16_attention_weight_max": 44.371829986572266,
      "activations/layer16_attention_weight_min": -28.592859268188477,
      "activations/layer17_attention_weight_max": 41.31800842285156,
      "activations/layer17_attention_weight_min": -25.467086791992188,
      "activations/layer18_attention_weight_max": 30.763216018676758,
      "activations/layer18_attention_weight_min": -20.91229820251465,
      "activations/layer19_attention_weight_max": 48.07807159423828,
      "activations/layer19_attention_weight_min": -28.015888214111328,
      "activations/layer1_attention_weight_max": 17.149158477783203,
      "activations/layer1_attention_weight_min": -13.592852592468262,
      "activations/layer20_attention_weight_max": 37.53732681274414,
      "activations/layer20_attention_weight_min": -24.08827018737793,
      "activations/layer21_attention_weight_max": 31.72432518005371,
      "activations/layer21_attention_weight_min": -20.31154441833496,
      "activations/layer22_attention_weight_max": 50.636470794677734,
      "activations/layer22_attention_weight_min": -28.754657745361328,
      "activations/layer23_attention_weight_max": 32.82048034667969,
      "activations/layer23_attention_weight_min": -17.940414428710938,
      "activations/layer2_attention_weight_max": 30.967573165893555,
      "activations/layer2_attention_weight_min": -31.256546020507812,
      "activations/layer3_attention_weight_max": 85.37625885009766,
      "activations/layer3_attention_weight_min": -87.83462524414062,
      "activations/layer4_attention_weight_max": 132.40196228027344,
      "activations/layer4_attention_weight_min": -129.5924530029297,
      "activations/layer5_attention_weight_max": 87.44207000732422,
      "activations/layer5_attention_weight_min": -82.30758666992188,
      "activations/layer6_attention_weight_max": 55.329261779785156,
      "activations/layer6_attention_weight_min": -56.00248718261719,
      "activations/layer7_attention_weight_max": 85.60414123535156,
      "activations/layer7_attention_weight_min": -78.44873046875,
      "activations/layer8_attention_weight_max": 40.01335525512695,
      "activations/layer8_attention_weight_min": -37.49229431152344,
      "activations/layer9_attention_weight_max": 41.51499938964844,
      "activations/layer9_attention_weight_min": -31.215641021728516,
      "epoch": 4.54,
      "learning_rate": 0.000121980303030303,
      "loss": 2.8568,
      "step": 78050
    },
    {
      "activations/layer0_attention_weight_max": 15.871176719665527,
      "activations/layer0_attention_weight_min": -12.673768043518066,
      "activations/layer10_attention_weight_max": 34.84538650512695,
      "activations/layer10_attention_weight_min": -28.64224624633789,
      "activations/layer11_attention_weight_max": 33.67848205566406,
      "activations/layer11_attention_weight_min": -28.6903133392334,
      "activations/layer12_attention_weight_max": 30.20853042602539,
      "activations/layer12_attention_weight_min": -26.878746032714844,
      "activations/layer13_attention_weight_max": 43.885047912597656,
      "activations/layer13_attention_weight_min": -31.759572982788086,
      "activations/layer14_attention_weight_max": 72.49606323242188,
      "activations/layer14_attention_weight_min": -50.26864242553711,
      "activations/layer15_attention_weight_max": 43.36225891113281,
      "activations/layer15_attention_weight_min": -29.1470890045166,
      "activations/layer16_attention_weight_max": 40.556514739990234,
      "activations/layer16_attention_weight_min": -28.208656311035156,
      "activations/layer17_attention_weight_max": 43.71711349487305,
      "activations/layer17_attention_weight_min": -27.531625747680664,
      "activations/layer18_attention_weight_max": 33.68342208862305,
      "activations/layer18_attention_weight_min": -20.861955642700195,
      "activations/layer19_attention_weight_max": 50.064517974853516,
      "activations/layer19_attention_weight_min": -29.405838012695312,
      "activations/layer1_attention_weight_max": 18.21315574645996,
      "activations/layer1_attention_weight_min": -15.714811325073242,
      "activations/layer20_attention_weight_max": 33.63376235961914,
      "activations/layer20_attention_weight_min": -25.056804656982422,
      "activations/layer21_attention_weight_max": 37.34754180908203,
      "activations/layer21_attention_weight_min": -22.332717895507812,
      "activations/layer22_attention_weight_max": 55.110416412353516,
      "activations/layer22_attention_weight_min": -30.134811401367188,
      "activations/layer23_attention_weight_max": 37.301639556884766,
      "activations/layer23_attention_weight_min": -19.07048988342285,
      "activations/layer2_attention_weight_max": 33.850563049316406,
      "activations/layer2_attention_weight_min": -31.353918075561523,
      "activations/layer3_attention_weight_max": 93.43753051757812,
      "activations/layer3_attention_weight_min": -90.3720932006836,
      "activations/layer4_attention_weight_max": 137.2505340576172,
      "activations/layer4_attention_weight_min": -136.21763610839844,
      "activations/layer5_attention_weight_max": 88.33605194091797,
      "activations/layer5_attention_weight_min": -86.51124572753906,
      "activations/layer6_attention_weight_max": 57.09358596801758,
      "activations/layer6_attention_weight_min": -56.553565979003906,
      "activations/layer7_attention_weight_max": 79.553466796875,
      "activations/layer7_attention_weight_min": -81.62323760986328,
      "activations/layer8_attention_weight_max": 41.34352111816406,
      "activations/layer8_attention_weight_min": -41.73261642456055,
      "activations/layer9_attention_weight_max": 41.66621017456055,
      "activations/layer9_attention_weight_min": -35.059200286865234,
      "epoch": 4.54,
      "learning_rate": 0.00012196136363636362,
      "loss": 2.8667,
      "step": 78100
    },
    {
      "activations/layer0_attention_weight_max": 16.79593849182129,
      "activations/layer0_attention_weight_min": -12.035284042358398,
      "activations/layer10_attention_weight_max": 36.005577087402344,
      "activations/layer10_attention_weight_min": -29.955007553100586,
      "activations/layer11_attention_weight_max": 31.235639572143555,
      "activations/layer11_attention_weight_min": -27.433225631713867,
      "activations/layer12_attention_weight_max": 30.899124145507812,
      "activations/layer12_attention_weight_min": -28.330734252929688,
      "activations/layer13_attention_weight_max": 40.070838928222656,
      "activations/layer13_attention_weight_min": -31.700328826904297,
      "activations/layer14_attention_weight_max": 58.61563491821289,
      "activations/layer14_attention_weight_min": -42.6523551940918,
      "activations/layer15_attention_weight_max": 38.32096481323242,
      "activations/layer15_attention_weight_min": -27.180885314941406,
      "activations/layer16_attention_weight_max": 37.08407211303711,
      "activations/layer16_attention_weight_min": -28.88732147216797,
      "activations/layer17_attention_weight_max": 37.46626663208008,
      "activations/layer17_attention_weight_min": -23.114803314208984,
      "activations/layer18_attention_weight_max": 34.4593620300293,
      "activations/layer18_attention_weight_min": -19.8626708984375,
      "activations/layer19_attention_weight_max": 41.80541229248047,
      "activations/layer19_attention_weight_min": -29.159109115600586,
      "activations/layer1_attention_weight_max": 17.243057250976562,
      "activations/layer1_attention_weight_min": -16.60601234436035,
      "activations/layer20_attention_weight_max": 33.898704528808594,
      "activations/layer20_attention_weight_min": -22.852130889892578,
      "activations/layer21_attention_weight_max": 37.42267990112305,
      "activations/layer21_attention_weight_min": -21.416912078857422,
      "activations/layer22_attention_weight_max": 50.47618103027344,
      "activations/layer22_attention_weight_min": -25.98118019104004,
      "activations/layer23_attention_weight_max": 28.55836296081543,
      "activations/layer23_attention_weight_min": -18.365459442138672,
      "activations/layer2_attention_weight_max": 32.290626525878906,
      "activations/layer2_attention_weight_min": -30.5251522064209,
      "activations/layer3_attention_weight_max": 84.25493621826172,
      "activations/layer3_attention_weight_min": -88.9049301147461,
      "activations/layer4_attention_weight_max": 127.6747055053711,
      "activations/layer4_attention_weight_min": -125.26496887207031,
      "activations/layer5_attention_weight_max": 90.83097839355469,
      "activations/layer5_attention_weight_min": -81.31887817382812,
      "activations/layer6_attention_weight_max": 55.14369583129883,
      "activations/layer6_attention_weight_min": -58.40439987182617,
      "activations/layer7_attention_weight_max": 84.00895690917969,
      "activations/layer7_attention_weight_min": -79.9383544921875,
      "activations/layer8_attention_weight_max": 39.668067932128906,
      "activations/layer8_attention_weight_min": -37.37842559814453,
      "activations/layer9_attention_weight_max": 36.12955856323242,
      "activations/layer9_attention_weight_min": -37.0932731628418,
      "epoch": 4.54,
      "learning_rate": 0.00012194242424242424,
      "loss": 2.8764,
      "step": 78150
    },
    {
      "activations/layer0_attention_weight_max": 15.106245994567871,
      "activations/layer0_attention_weight_min": -12.481609344482422,
      "activations/layer10_attention_weight_max": 37.4801025390625,
      "activations/layer10_attention_weight_min": -30.442888259887695,
      "activations/layer11_attention_weight_max": 41.320411682128906,
      "activations/layer11_attention_weight_min": -29.99957847595215,
      "activations/layer12_attention_weight_max": 41.53231430053711,
      "activations/layer12_attention_weight_min": -29.481077194213867,
      "activations/layer13_attention_weight_max": 45.12971115112305,
      "activations/layer13_attention_weight_min": -33.94328689575195,
      "activations/layer14_attention_weight_max": 70.62252044677734,
      "activations/layer14_attention_weight_min": -46.84480667114258,
      "activations/layer15_attention_weight_max": 41.769020080566406,
      "activations/layer15_attention_weight_min": -27.297286987304688,
      "activations/layer16_attention_weight_max": 42.38386154174805,
      "activations/layer16_attention_weight_min": -29.885942459106445,
      "activations/layer17_attention_weight_max": 41.18865203857422,
      "activations/layer17_attention_weight_min": -25.620830535888672,
      "activations/layer18_attention_weight_max": 33.694637298583984,
      "activations/layer18_attention_weight_min": -19.48580551147461,
      "activations/layer19_attention_weight_max": 41.408905029296875,
      "activations/layer19_attention_weight_min": -29.892658233642578,
      "activations/layer1_attention_weight_max": 16.98318099975586,
      "activations/layer1_attention_weight_min": -15.666383743286133,
      "activations/layer20_attention_weight_max": 36.43275451660156,
      "activations/layer20_attention_weight_min": -22.89643096923828,
      "activations/layer21_attention_weight_max": 36.67506408691406,
      "activations/layer21_attention_weight_min": -20.26849365234375,
      "activations/layer22_attention_weight_max": 63.829261779785156,
      "activations/layer22_attention_weight_min": -29.189565658569336,
      "activations/layer23_attention_weight_max": 34.677711486816406,
      "activations/layer23_attention_weight_min": -23.59716033935547,
      "activations/layer2_attention_weight_max": 32.568870544433594,
      "activations/layer2_attention_weight_min": -31.597965240478516,
      "activations/layer3_attention_weight_max": 87.83104705810547,
      "activations/layer3_attention_weight_min": -89.74774169921875,
      "activations/layer4_attention_weight_max": 136.3072509765625,
      "activations/layer4_attention_weight_min": -126.76460266113281,
      "activations/layer5_attention_weight_max": 84.34256744384766,
      "activations/layer5_attention_weight_min": -83.81413269042969,
      "activations/layer6_attention_weight_max": 56.343971252441406,
      "activations/layer6_attention_weight_min": -60.129154205322266,
      "activations/layer7_attention_weight_max": 82.56475830078125,
      "activations/layer7_attention_weight_min": -84.25630950927734,
      "activations/layer8_attention_weight_max": 41.15592575073242,
      "activations/layer8_attention_weight_min": -40.66416931152344,
      "activations/layer9_attention_weight_max": 40.20768356323242,
      "activations/layer9_attention_weight_min": -38.88145446777344,
      "epoch": 4.54,
      "learning_rate": 0.00012192348484848482,
      "loss": 2.8784,
      "step": 78200
    },
    {
      "activations/layer0_attention_weight_max": 17.157949447631836,
      "activations/layer0_attention_weight_min": -11.876557350158691,
      "activations/layer10_attention_weight_max": 36.87604904174805,
      "activations/layer10_attention_weight_min": -30.216875076293945,
      "activations/layer11_attention_weight_max": 38.703529357910156,
      "activations/layer11_attention_weight_min": -28.587818145751953,
      "activations/layer12_attention_weight_max": 34.509422302246094,
      "activations/layer12_attention_weight_min": -32.97721481323242,
      "activations/layer13_attention_weight_max": 50.049644470214844,
      "activations/layer13_attention_weight_min": -37.52640914916992,
      "activations/layer14_attention_weight_max": 58.980255126953125,
      "activations/layer14_attention_weight_min": -49.69633865356445,
      "activations/layer15_attention_weight_max": 42.4366340637207,
      "activations/layer15_attention_weight_min": -28.545154571533203,
      "activations/layer16_attention_weight_max": 37.564910888671875,
      "activations/layer16_attention_weight_min": -28.19882583618164,
      "activations/layer17_attention_weight_max": 41.46480178833008,
      "activations/layer17_attention_weight_min": -23.31407928466797,
      "activations/layer18_attention_weight_max": 30.639751434326172,
      "activations/layer18_attention_weight_min": -21.469770431518555,
      "activations/layer19_attention_weight_max": 40.03609085083008,
      "activations/layer19_attention_weight_min": -29.164709091186523,
      "activations/layer1_attention_weight_max": 16.8367919921875,
      "activations/layer1_attention_weight_min": -16.287250518798828,
      "activations/layer20_attention_weight_max": 34.881622314453125,
      "activations/layer20_attention_weight_min": -21.868812561035156,
      "activations/layer21_attention_weight_max": 29.016332626342773,
      "activations/layer21_attention_weight_min": -20.313486099243164,
      "activations/layer22_attention_weight_max": 49.399314880371094,
      "activations/layer22_attention_weight_min": -25.696870803833008,
      "activations/layer23_attention_weight_max": 29.808170318603516,
      "activations/layer23_attention_weight_min": -20.822711944580078,
      "activations/layer2_attention_weight_max": 31.167036056518555,
      "activations/layer2_attention_weight_min": -30.098072052001953,
      "activations/layer3_attention_weight_max": 82.00679779052734,
      "activations/layer3_attention_weight_min": -85.19586181640625,
      "activations/layer4_attention_weight_max": 128.9091339111328,
      "activations/layer4_attention_weight_min": -126.00538635253906,
      "activations/layer5_attention_weight_max": 85.18375396728516,
      "activations/layer5_attention_weight_min": -79.50532531738281,
      "activations/layer6_attention_weight_max": 56.984527587890625,
      "activations/layer6_attention_weight_min": -56.6320915222168,
      "activations/layer7_attention_weight_max": 89.23614501953125,
      "activations/layer7_attention_weight_min": -77.33352661132812,
      "activations/layer8_attention_weight_max": 42.69118118286133,
      "activations/layer8_attention_weight_min": -39.4525032043457,
      "activations/layer9_attention_weight_max": 43.02817916870117,
      "activations/layer9_attention_weight_min": -36.25111389160156,
      "epoch": 4.55,
      "learning_rate": 0.00012190454545454544,
      "loss": 2.8555,
      "step": 78250
    },
    {
      "activations/layer0_attention_weight_max": 16.52655029296875,
      "activations/layer0_attention_weight_min": -11.1083345413208,
      "activations/layer10_attention_weight_max": 37.07514572143555,
      "activations/layer10_attention_weight_min": -32.52663040161133,
      "activations/layer11_attention_weight_max": 38.197303771972656,
      "activations/layer11_attention_weight_min": -28.876007080078125,
      "activations/layer12_attention_weight_max": 26.236780166625977,
      "activations/layer12_attention_weight_min": -26.282087326049805,
      "activations/layer13_attention_weight_max": 43.202640533447266,
      "activations/layer13_attention_weight_min": -31.051734924316406,
      "activations/layer14_attention_weight_max": 63.32193374633789,
      "activations/layer14_attention_weight_min": -45.76250076293945,
      "activations/layer15_attention_weight_max": 39.306514739990234,
      "activations/layer15_attention_weight_min": -28.8902645111084,
      "activations/layer16_attention_weight_max": 37.3246955871582,
      "activations/layer16_attention_weight_min": -26.37357521057129,
      "activations/layer17_attention_weight_max": 41.02259826660156,
      "activations/layer17_attention_weight_min": -26.762849807739258,
      "activations/layer18_attention_weight_max": 33.32947540283203,
      "activations/layer18_attention_weight_min": -22.078161239624023,
      "activations/layer19_attention_weight_max": 42.89497375488281,
      "activations/layer19_attention_weight_min": -27.63972282409668,
      "activations/layer1_attention_weight_max": 17.6783390045166,
      "activations/layer1_attention_weight_min": -15.474894523620605,
      "activations/layer20_attention_weight_max": 36.16624450683594,
      "activations/layer20_attention_weight_min": -22.252641677856445,
      "activations/layer21_attention_weight_max": 32.15538787841797,
      "activations/layer21_attention_weight_min": -19.703540802001953,
      "activations/layer22_attention_weight_max": 50.21646499633789,
      "activations/layer22_attention_weight_min": -26.48941993713379,
      "activations/layer23_attention_weight_max": 34.07323455810547,
      "activations/layer23_attention_weight_min": -22.53204917907715,
      "activations/layer2_attention_weight_max": 33.23002624511719,
      "activations/layer2_attention_weight_min": -31.87384796142578,
      "activations/layer3_attention_weight_max": 90.02776336669922,
      "activations/layer3_attention_weight_min": -93.40573120117188,
      "activations/layer4_attention_weight_max": 135.32244873046875,
      "activations/layer4_attention_weight_min": -132.04531860351562,
      "activations/layer5_attention_weight_max": 87.29021453857422,
      "activations/layer5_attention_weight_min": -83.61918640136719,
      "activations/layer6_attention_weight_max": 60.48963165283203,
      "activations/layer6_attention_weight_min": -57.02569580078125,
      "activations/layer7_attention_weight_max": 94.45507049560547,
      "activations/layer7_attention_weight_min": -85.32230377197266,
      "activations/layer8_attention_weight_max": 42.62861251831055,
      "activations/layer8_attention_weight_min": -38.74460220336914,
      "activations/layer9_attention_weight_max": 40.66255569458008,
      "activations/layer9_attention_weight_min": -38.00673294067383,
      "epoch": 4.55,
      "learning_rate": 0.00012188560606060605,
      "loss": 2.8706,
      "step": 78300
    },
    {
      "activations/layer0_attention_weight_max": 15.964741706848145,
      "activations/layer0_attention_weight_min": -12.301149368286133,
      "activations/layer10_attention_weight_max": 38.99020767211914,
      "activations/layer10_attention_weight_min": -32.196800231933594,
      "activations/layer11_attention_weight_max": 36.73857498168945,
      "activations/layer11_attention_weight_min": -28.601669311523438,
      "activations/layer12_attention_weight_max": 27.986162185668945,
      "activations/layer12_attention_weight_min": -29.527454376220703,
      "activations/layer13_attention_weight_max": 50.03287124633789,
      "activations/layer13_attention_weight_min": -32.81929016113281,
      "activations/layer14_attention_weight_max": 86.76361846923828,
      "activations/layer14_attention_weight_min": -55.77089309692383,
      "activations/layer15_attention_weight_max": 48.32063293457031,
      "activations/layer15_attention_weight_min": -32.375587463378906,
      "activations/layer16_attention_weight_max": 42.22732925415039,
      "activations/layer16_attention_weight_min": -28.26820182800293,
      "activations/layer17_attention_weight_max": 41.03383255004883,
      "activations/layer17_attention_weight_min": -24.907255172729492,
      "activations/layer18_attention_weight_max": 31.73634147644043,
      "activations/layer18_attention_weight_min": -20.47889518737793,
      "activations/layer19_attention_weight_max": 44.905330657958984,
      "activations/layer19_attention_weight_min": -29.21122169494629,
      "activations/layer1_attention_weight_max": 17.716869354248047,
      "activations/layer1_attention_weight_min": -13.683048248291016,
      "activations/layer20_attention_weight_max": 35.86566925048828,
      "activations/layer20_attention_weight_min": -24.054502487182617,
      "activations/layer21_attention_weight_max": 30.942684173583984,
      "activations/layer21_attention_weight_min": -20.31951332092285,
      "activations/layer22_attention_weight_max": 58.185787200927734,
      "activations/layer22_attention_weight_min": -29.666805267333984,
      "activations/layer23_attention_weight_max": 34.06480407714844,
      "activations/layer23_attention_weight_min": -19.43337059020996,
      "activations/layer2_attention_weight_max": 32.52067947387695,
      "activations/layer2_attention_weight_min": -32.24837112426758,
      "activations/layer3_attention_weight_max": 92.85318756103516,
      "activations/layer3_attention_weight_min": -93.96832275390625,
      "activations/layer4_attention_weight_max": 136.44732666015625,
      "activations/layer4_attention_weight_min": -134.0983123779297,
      "activations/layer5_attention_weight_max": 93.24056243896484,
      "activations/layer5_attention_weight_min": -90.99302673339844,
      "activations/layer6_attention_weight_max": 63.314430236816406,
      "activations/layer6_attention_weight_min": -64.43301391601562,
      "activations/layer7_attention_weight_max": 87.80284881591797,
      "activations/layer7_attention_weight_min": -85.53233337402344,
      "activations/layer8_attention_weight_max": 46.56650924682617,
      "activations/layer8_attention_weight_min": -42.33146667480469,
      "activations/layer9_attention_weight_max": 40.72929763793945,
      "activations/layer9_attention_weight_min": -35.778934478759766,
      "epoch": 4.55,
      "learning_rate": 0.00012186666666666666,
      "loss": 2.8529,
      "step": 78350
    },
    {
      "activations/layer0_attention_weight_max": 16.91205406188965,
      "activations/layer0_attention_weight_min": -12.049954414367676,
      "activations/layer10_attention_weight_max": 36.0384521484375,
      "activations/layer10_attention_weight_min": -32.560909271240234,
      "activations/layer11_attention_weight_max": 34.23884201049805,
      "activations/layer11_attention_weight_min": -27.993331909179688,
      "activations/layer12_attention_weight_max": 29.259370803833008,
      "activations/layer12_attention_weight_min": -28.486970901489258,
      "activations/layer13_attention_weight_max": 58.10675811767578,
      "activations/layer13_attention_weight_min": -35.38905334472656,
      "activations/layer14_attention_weight_max": 77.43000793457031,
      "activations/layer14_attention_weight_min": -52.34648132324219,
      "activations/layer15_attention_weight_max": 44.783660888671875,
      "activations/layer15_attention_weight_min": -29.151927947998047,
      "activations/layer16_attention_weight_max": 38.384117126464844,
      "activations/layer16_attention_weight_min": -28.770708084106445,
      "activations/layer17_attention_weight_max": 41.05316925048828,
      "activations/layer17_attention_weight_min": -25.009227752685547,
      "activations/layer18_attention_weight_max": 40.995845794677734,
      "activations/layer18_attention_weight_min": -21.071382522583008,
      "activations/layer19_attention_weight_max": 42.1767692565918,
      "activations/layer19_attention_weight_min": -28.90007972717285,
      "activations/layer1_attention_weight_max": 16.83220672607422,
      "activations/layer1_attention_weight_min": -16.050996780395508,
      "activations/layer20_attention_weight_max": 32.71991729736328,
      "activations/layer20_attention_weight_min": -21.892210006713867,
      "activations/layer21_attention_weight_max": 35.40782928466797,
      "activations/layer21_attention_weight_min": -20.593332290649414,
      "activations/layer22_attention_weight_max": 51.610225677490234,
      "activations/layer22_attention_weight_min": -30.180599212646484,
      "activations/layer23_attention_weight_max": 34.19077682495117,
      "activations/layer23_attention_weight_min": -19.935222625732422,
      "activations/layer2_attention_weight_max": 31.28693199157715,
      "activations/layer2_attention_weight_min": -31.548276901245117,
      "activations/layer3_attention_weight_max": 92.51200103759766,
      "activations/layer3_attention_weight_min": -96.19647216796875,
      "activations/layer4_attention_weight_max": 135.7519989013672,
      "activations/layer4_attention_weight_min": -131.29693603515625,
      "activations/layer5_attention_weight_max": 93.43438720703125,
      "activations/layer5_attention_weight_min": -84.62065124511719,
      "activations/layer6_attention_weight_max": 57.403446197509766,
      "activations/layer6_attention_weight_min": -59.56748580932617,
      "activations/layer7_attention_weight_max": 82.19895935058594,
      "activations/layer7_attention_weight_min": -81.34229278564453,
      "activations/layer8_attention_weight_max": 41.380313873291016,
      "activations/layer8_attention_weight_min": -40.67287063598633,
      "activations/layer9_attention_weight_max": 38.99607467651367,
      "activations/layer9_attention_weight_min": -37.193511962890625,
      "epoch": 4.56,
      "learning_rate": 0.00012184772727272726,
      "loss": 2.86,
      "step": 78400
    },
    {
      "activations/layer0_attention_weight_max": 17.213464736938477,
      "activations/layer0_attention_weight_min": -12.137846946716309,
      "activations/layer10_attention_weight_max": 34.186805725097656,
      "activations/layer10_attention_weight_min": -30.844635009765625,
      "activations/layer11_attention_weight_max": 32.061119079589844,
      "activations/layer11_attention_weight_min": -28.743988037109375,
      "activations/layer12_attention_weight_max": 46.20248794555664,
      "activations/layer12_attention_weight_min": -27.139249801635742,
      "activations/layer13_attention_weight_max": 43.19306182861328,
      "activations/layer13_attention_weight_min": -32.62501525878906,
      "activations/layer14_attention_weight_max": 60.53285598754883,
      "activations/layer14_attention_weight_min": -49.28892135620117,
      "activations/layer15_attention_weight_max": 38.051979064941406,
      "activations/layer15_attention_weight_min": -28.04188346862793,
      "activations/layer16_attention_weight_max": 37.76051330566406,
      "activations/layer16_attention_weight_min": -29.959135055541992,
      "activations/layer17_attention_weight_max": 39.25777053833008,
      "activations/layer17_attention_weight_min": -23.3095703125,
      "activations/layer18_attention_weight_max": 27.655128479003906,
      "activations/layer18_attention_weight_min": -19.730260848999023,
      "activations/layer19_attention_weight_max": 39.08540725708008,
      "activations/layer19_attention_weight_min": -29.362319946289062,
      "activations/layer1_attention_weight_max": 17.669532775878906,
      "activations/layer1_attention_weight_min": -16.099790573120117,
      "activations/layer20_attention_weight_max": 32.81496047973633,
      "activations/layer20_attention_weight_min": -23.36571502685547,
      "activations/layer21_attention_weight_max": 32.0704460144043,
      "activations/layer21_attention_weight_min": -19.168598175048828,
      "activations/layer22_attention_weight_max": 45.3390007019043,
      "activations/layer22_attention_weight_min": -27.317060470581055,
      "activations/layer23_attention_weight_max": 28.070659637451172,
      "activations/layer23_attention_weight_min": -18.444734573364258,
      "activations/layer2_attention_weight_max": 32.21382141113281,
      "activations/layer2_attention_weight_min": -31.82849884033203,
      "activations/layer3_attention_weight_max": 88.73577117919922,
      "activations/layer3_attention_weight_min": -89.8812026977539,
      "activations/layer4_attention_weight_max": 134.45384216308594,
      "activations/layer4_attention_weight_min": -134.18212890625,
      "activations/layer5_attention_weight_max": 89.72793579101562,
      "activations/layer5_attention_weight_min": -86.07969665527344,
      "activations/layer6_attention_weight_max": 56.94487762451172,
      "activations/layer6_attention_weight_min": -57.594017028808594,
      "activations/layer7_attention_weight_max": 85.95872497558594,
      "activations/layer7_attention_weight_min": -83.64393615722656,
      "activations/layer8_attention_weight_max": 42.35551834106445,
      "activations/layer8_attention_weight_min": -38.769508361816406,
      "activations/layer9_attention_weight_max": 39.38541793823242,
      "activations/layer9_attention_weight_min": -35.84811019897461,
      "epoch": 4.56,
      "learning_rate": 0.00012182878787878787,
      "loss": 2.8701,
      "step": 78450
    },
    {
      "activations/layer0_attention_weight_max": 15.299625396728516,
      "activations/layer0_attention_weight_min": -10.738747596740723,
      "activations/layer10_attention_weight_max": 35.31904983520508,
      "activations/layer10_attention_weight_min": -31.322391510009766,
      "activations/layer11_attention_weight_max": 32.57681655883789,
      "activations/layer11_attention_weight_min": -27.420957565307617,
      "activations/layer12_attention_weight_max": 24.44164276123047,
      "activations/layer12_attention_weight_min": -24.574743270874023,
      "activations/layer13_attention_weight_max": 44.529911041259766,
      "activations/layer13_attention_weight_min": -32.65092849731445,
      "activations/layer14_attention_weight_max": 64.08314514160156,
      "activations/layer14_attention_weight_min": -51.176414489746094,
      "activations/layer15_attention_weight_max": 40.5289421081543,
      "activations/layer15_attention_weight_min": -28.305315017700195,
      "activations/layer16_attention_weight_max": 34.8490104675293,
      "activations/layer16_attention_weight_min": -29.050148010253906,
      "activations/layer17_attention_weight_max": 39.70188903808594,
      "activations/layer17_attention_weight_min": -26.443504333496094,
      "activations/layer18_attention_weight_max": 29.631620407104492,
      "activations/layer18_attention_weight_min": -18.837221145629883,
      "activations/layer19_attention_weight_max": 37.40400314331055,
      "activations/layer19_attention_weight_min": -27.79633903503418,
      "activations/layer1_attention_weight_max": 18.22736930847168,
      "activations/layer1_attention_weight_min": -14.344470024108887,
      "activations/layer20_attention_weight_max": 32.91449737548828,
      "activations/layer20_attention_weight_min": -24.097082138061523,
      "activations/layer21_attention_weight_max": 32.325191497802734,
      "activations/layer21_attention_weight_min": -19.641422271728516,
      "activations/layer22_attention_weight_max": 51.65378189086914,
      "activations/layer22_attention_weight_min": -29.649024963378906,
      "activations/layer23_attention_weight_max": 30.709182739257812,
      "activations/layer23_attention_weight_min": -19.48029899597168,
      "activations/layer2_attention_weight_max": 31.315139770507812,
      "activations/layer2_attention_weight_min": -29.48335075378418,
      "activations/layer3_attention_weight_max": 86.60733032226562,
      "activations/layer3_attention_weight_min": -89.13966369628906,
      "activations/layer4_attention_weight_max": 129.42735290527344,
      "activations/layer4_attention_weight_min": -129.0742950439453,
      "activations/layer5_attention_weight_max": 85.70706176757812,
      "activations/layer5_attention_weight_min": -79.2344970703125,
      "activations/layer6_attention_weight_max": 56.52939987182617,
      "activations/layer6_attention_weight_min": -56.56218338012695,
      "activations/layer7_attention_weight_max": 79.62984466552734,
      "activations/layer7_attention_weight_min": -81.5963363647461,
      "activations/layer8_attention_weight_max": 43.81941604614258,
      "activations/layer8_attention_weight_min": -37.92994689941406,
      "activations/layer9_attention_weight_max": 37.210506439208984,
      "activations/layer9_attention_weight_min": -32.71970748901367,
      "epoch": 4.56,
      "learning_rate": 0.00012180984848484847,
      "loss": 2.8623,
      "step": 78500
    },
    {
      "activations/layer0_attention_weight_max": 16.937746047973633,
      "activations/layer0_attention_weight_min": -11.984942436218262,
      "activations/layer10_attention_weight_max": 35.55329895019531,
      "activations/layer10_attention_weight_min": -30.57485580444336,
      "activations/layer11_attention_weight_max": 33.832763671875,
      "activations/layer11_attention_weight_min": -29.75214385986328,
      "activations/layer12_attention_weight_max": 35.37547302246094,
      "activations/layer12_attention_weight_min": -27.109575271606445,
      "activations/layer13_attention_weight_max": 53.85374450683594,
      "activations/layer13_attention_weight_min": -45.16861343383789,
      "activations/layer14_attention_weight_max": 69.3184585571289,
      "activations/layer14_attention_weight_min": -56.10357666015625,
      "activations/layer15_attention_weight_max": 40.268062591552734,
      "activations/layer15_attention_weight_min": -27.03101348876953,
      "activations/layer16_attention_weight_max": 41.89163589477539,
      "activations/layer16_attention_weight_min": -26.83306121826172,
      "activations/layer17_attention_weight_max": 40.56452941894531,
      "activations/layer17_attention_weight_min": -25.511716842651367,
      "activations/layer18_attention_weight_max": 31.014205932617188,
      "activations/layer18_attention_weight_min": -19.90851402282715,
      "activations/layer19_attention_weight_max": 42.53438186645508,
      "activations/layer19_attention_weight_min": -29.6566219329834,
      "activations/layer1_attention_weight_max": 17.706289291381836,
      "activations/layer1_attention_weight_min": -16.093231201171875,
      "activations/layer20_attention_weight_max": 32.05451583862305,
      "activations/layer20_attention_weight_min": -24.391817092895508,
      "activations/layer21_attention_weight_max": 37.79227828979492,
      "activations/layer21_attention_weight_min": -24.08281135559082,
      "activations/layer22_attention_weight_max": 54.51529312133789,
      "activations/layer22_attention_weight_min": -28.804773330688477,
      "activations/layer23_attention_weight_max": 32.19314193725586,
      "activations/layer23_attention_weight_min": -17.51753807067871,
      "activations/layer2_attention_weight_max": 31.94260025024414,
      "activations/layer2_attention_weight_min": -31.490962982177734,
      "activations/layer3_attention_weight_max": 87.22248077392578,
      "activations/layer3_attention_weight_min": -90.48869323730469,
      "activations/layer4_attention_weight_max": 135.75367736816406,
      "activations/layer4_attention_weight_min": -130.12159729003906,
      "activations/layer5_attention_weight_max": 90.1249008178711,
      "activations/layer5_attention_weight_min": -84.25715637207031,
      "activations/layer6_attention_weight_max": 60.07041549682617,
      "activations/layer6_attention_weight_min": -61.00035858154297,
      "activations/layer7_attention_weight_max": 85.48552703857422,
      "activations/layer7_attention_weight_min": -86.32177734375,
      "activations/layer8_attention_weight_max": 40.85721206665039,
      "activations/layer8_attention_weight_min": -38.1039924621582,
      "activations/layer9_attention_weight_max": 39.044246673583984,
      "activations/layer9_attention_weight_min": -34.9268913269043,
      "epoch": 4.56,
      "learning_rate": 0.00012179090909090909,
      "loss": 2.8798,
      "step": 78550
    },
    {
      "activations/layer0_attention_weight_max": 16.423147201538086,
      "activations/layer0_attention_weight_min": -11.217293739318848,
      "activations/layer10_attention_weight_max": 35.86408615112305,
      "activations/layer10_attention_weight_min": -29.11453628540039,
      "activations/layer11_attention_weight_max": 35.244224548339844,
      "activations/layer11_attention_weight_min": -28.502803802490234,
      "activations/layer12_attention_weight_max": 25.655733108520508,
      "activations/layer12_attention_weight_min": -32.272605895996094,
      "activations/layer13_attention_weight_max": 42.48019027709961,
      "activations/layer13_attention_weight_min": -30.61804962158203,
      "activations/layer14_attention_weight_max": 61.97361755371094,
      "activations/layer14_attention_weight_min": -47.14644241333008,
      "activations/layer15_attention_weight_max": 45.45651626586914,
      "activations/layer15_attention_weight_min": -27.924251556396484,
      "activations/layer16_attention_weight_max": 38.472774505615234,
      "activations/layer16_attention_weight_min": -32.51283264160156,
      "activations/layer17_attention_weight_max": 44.1020622253418,
      "activations/layer17_attention_weight_min": -23.09458351135254,
      "activations/layer18_attention_weight_max": 38.794952392578125,
      "activations/layer18_attention_weight_min": -20.4113826751709,
      "activations/layer19_attention_weight_max": 38.660037994384766,
      "activations/layer19_attention_weight_min": -27.82802391052246,
      "activations/layer1_attention_weight_max": 17.9785099029541,
      "activations/layer1_attention_weight_min": -14.029845237731934,
      "activations/layer20_attention_weight_max": 30.7520751953125,
      "activations/layer20_attention_weight_min": -22.485597610473633,
      "activations/layer21_attention_weight_max": 31.40407371520996,
      "activations/layer21_attention_weight_min": -21.45584487915039,
      "activations/layer22_attention_weight_max": 48.4720573425293,
      "activations/layer22_attention_weight_min": -26.609813690185547,
      "activations/layer23_attention_weight_max": 30.330142974853516,
      "activations/layer23_attention_weight_min": -19.306114196777344,
      "activations/layer2_attention_weight_max": 30.53118133544922,
      "activations/layer2_attention_weight_min": -30.55805778503418,
      "activations/layer3_attention_weight_max": 89.42711639404297,
      "activations/layer3_attention_weight_min": -89.45911407470703,
      "activations/layer4_attention_weight_max": 129.92103576660156,
      "activations/layer4_attention_weight_min": -126.23699951171875,
      "activations/layer5_attention_weight_max": 84.0427017211914,
      "activations/layer5_attention_weight_min": -80.82903289794922,
      "activations/layer6_attention_weight_max": 53.194114685058594,
      "activations/layer6_attention_weight_min": -53.712886810302734,
      "activations/layer7_attention_weight_max": 85.08439636230469,
      "activations/layer7_attention_weight_min": -80.26541137695312,
      "activations/layer8_attention_weight_max": 42.22688293457031,
      "activations/layer8_attention_weight_min": -37.77922821044922,
      "activations/layer9_attention_weight_max": 36.392913818359375,
      "activations/layer9_attention_weight_min": -44.21123123168945,
      "epoch": 4.57,
      "learning_rate": 0.00012177196969696968,
      "loss": 2.8768,
      "step": 78600
    },
    {
      "activations/layer0_attention_weight_max": 15.349607467651367,
      "activations/layer0_attention_weight_min": -11.774102210998535,
      "activations/layer10_attention_weight_max": 34.151031494140625,
      "activations/layer10_attention_weight_min": -29.276182174682617,
      "activations/layer11_attention_weight_max": 32.530845642089844,
      "activations/layer11_attention_weight_min": -27.08885955810547,
      "activations/layer12_attention_weight_max": 26.99241828918457,
      "activations/layer12_attention_weight_min": -30.364971160888672,
      "activations/layer13_attention_weight_max": 39.870094299316406,
      "activations/layer13_attention_weight_min": -31.798036575317383,
      "activations/layer14_attention_weight_max": 70.49845123291016,
      "activations/layer14_attention_weight_min": -46.25107192993164,
      "activations/layer15_attention_weight_max": 38.8995246887207,
      "activations/layer15_attention_weight_min": -27.193767547607422,
      "activations/layer16_attention_weight_max": 36.082096099853516,
      "activations/layer16_attention_weight_min": -30.489774703979492,
      "activations/layer17_attention_weight_max": 38.1322021484375,
      "activations/layer17_attention_weight_min": -24.1591739654541,
      "activations/layer18_attention_weight_max": 30.106746673583984,
      "activations/layer18_attention_weight_min": -18.827476501464844,
      "activations/layer19_attention_weight_max": 39.09046173095703,
      "activations/layer19_attention_weight_min": -28.59125328063965,
      "activations/layer1_attention_weight_max": 17.792999267578125,
      "activations/layer1_attention_weight_min": -14.72335433959961,
      "activations/layer20_attention_weight_max": 34.408504486083984,
      "activations/layer20_attention_weight_min": -21.868270874023438,
      "activations/layer21_attention_weight_max": 34.73411178588867,
      "activations/layer21_attention_weight_min": -21.03743553161621,
      "activations/layer22_attention_weight_max": 49.82830810546875,
      "activations/layer22_attention_weight_min": -25.587646484375,
      "activations/layer23_attention_weight_max": 33.71967697143555,
      "activations/layer23_attention_weight_min": -19.267597198486328,
      "activations/layer2_attention_weight_max": 31.415945053100586,
      "activations/layer2_attention_weight_min": -30.056411743164062,
      "activations/layer3_attention_weight_max": 89.26556396484375,
      "activations/layer3_attention_weight_min": -88.6232681274414,
      "activations/layer4_attention_weight_max": 131.04776000976562,
      "activations/layer4_attention_weight_min": -132.50160217285156,
      "activations/layer5_attention_weight_max": 85.67445373535156,
      "activations/layer5_attention_weight_min": -81.23177337646484,
      "activations/layer6_attention_weight_max": 56.69145965576172,
      "activations/layer6_attention_weight_min": -54.90732192993164,
      "activations/layer7_attention_weight_max": 76.93347930908203,
      "activations/layer7_attention_weight_min": -75.62002563476562,
      "activations/layer8_attention_weight_max": 41.107887268066406,
      "activations/layer8_attention_weight_min": -38.81081008911133,
      "activations/layer9_attention_weight_max": 37.99338912963867,
      "activations/layer9_attention_weight_min": -34.15024185180664,
      "epoch": 4.57,
      "learning_rate": 0.00012175340909090908,
      "loss": 2.8695,
      "step": 78650
    },
    {
      "activations/layer0_attention_weight_max": 15.26526165008545,
      "activations/layer0_attention_weight_min": -11.060626983642578,
      "activations/layer10_attention_weight_max": 35.187416076660156,
      "activations/layer10_attention_weight_min": -32.177459716796875,
      "activations/layer11_attention_weight_max": 31.845104217529297,
      "activations/layer11_attention_weight_min": -28.551509857177734,
      "activations/layer12_attention_weight_max": 33.26900100708008,
      "activations/layer12_attention_weight_min": -25.60266876220703,
      "activations/layer13_attention_weight_max": 46.28763961791992,
      "activations/layer13_attention_weight_min": -30.91434097290039,
      "activations/layer14_attention_weight_max": 75.51142120361328,
      "activations/layer14_attention_weight_min": -53.23951721191406,
      "activations/layer15_attention_weight_max": 44.074951171875,
      "activations/layer15_attention_weight_min": -28.324464797973633,
      "activations/layer16_attention_weight_max": 40.146873474121094,
      "activations/layer16_attention_weight_min": -30.286653518676758,
      "activations/layer17_attention_weight_max": 41.51594924926758,
      "activations/layer17_attention_weight_min": -26.771142959594727,
      "activations/layer18_attention_weight_max": 31.423480987548828,
      "activations/layer18_attention_weight_min": -20.515193939208984,
      "activations/layer19_attention_weight_max": 54.644439697265625,
      "activations/layer19_attention_weight_min": -31.189340591430664,
      "activations/layer1_attention_weight_max": 17.591062545776367,
      "activations/layer1_attention_weight_min": -14.378385543823242,
      "activations/layer20_attention_weight_max": 40.34082794189453,
      "activations/layer20_attention_weight_min": -23.077123641967773,
      "activations/layer21_attention_weight_max": 44.68511962890625,
      "activations/layer21_attention_weight_min": -21.777368545532227,
      "activations/layer22_attention_weight_max": 57.69926834106445,
      "activations/layer22_attention_weight_min": -30.602020263671875,
      "activations/layer23_attention_weight_max": 37.213584899902344,
      "activations/layer23_attention_weight_min": -20.513938903808594,
      "activations/layer2_attention_weight_max": 33.63152313232422,
      "activations/layer2_attention_weight_min": -30.445730209350586,
      "activations/layer3_attention_weight_max": 88.7972412109375,
      "activations/layer3_attention_weight_min": -86.56713104248047,
      "activations/layer4_attention_weight_max": 131.5681610107422,
      "activations/layer4_attention_weight_min": -126.71919250488281,
      "activations/layer5_attention_weight_max": 84.99037170410156,
      "activations/layer5_attention_weight_min": -89.97722625732422,
      "activations/layer6_attention_weight_max": 58.509971618652344,
      "activations/layer6_attention_weight_min": -60.29660415649414,
      "activations/layer7_attention_weight_max": 84.4839859008789,
      "activations/layer7_attention_weight_min": -81.15099334716797,
      "activations/layer8_attention_weight_max": 42.21063232421875,
      "activations/layer8_attention_weight_min": -39.37446212768555,
      "activations/layer9_attention_weight_max": 37.77217102050781,
      "activations/layer9_attention_weight_min": -35.05184555053711,
      "epoch": 4.57,
      "learning_rate": 0.00012173446969696968,
      "loss": 2.8563,
      "step": 78700
    },
    {
      "activations/layer0_attention_weight_max": 16.74026107788086,
      "activations/layer0_attention_weight_min": -12.300872802734375,
      "activations/layer10_attention_weight_max": 35.933189392089844,
      "activations/layer10_attention_weight_min": -29.937068939208984,
      "activations/layer11_attention_weight_max": 32.877445220947266,
      "activations/layer11_attention_weight_min": -29.749889373779297,
      "activations/layer12_attention_weight_max": 28.14261817932129,
      "activations/layer12_attention_weight_min": -34.941558837890625,
      "activations/layer13_attention_weight_max": 45.19450759887695,
      "activations/layer13_attention_weight_min": -33.54656982421875,
      "activations/layer14_attention_weight_max": 76.75282287597656,
      "activations/layer14_attention_weight_min": -57.84252166748047,
      "activations/layer15_attention_weight_max": 46.090492248535156,
      "activations/layer15_attention_weight_min": -29.049142837524414,
      "activations/layer16_attention_weight_max": 39.451602935791016,
      "activations/layer16_attention_weight_min": -27.9422664642334,
      "activations/layer17_attention_weight_max": 41.798377990722656,
      "activations/layer17_attention_weight_min": -24.965919494628906,
      "activations/layer18_attention_weight_max": 37.85155487060547,
      "activations/layer18_attention_weight_min": -21.7231502532959,
      "activations/layer19_attention_weight_max": 42.0204963684082,
      "activations/layer19_attention_weight_min": -28.66482925415039,
      "activations/layer1_attention_weight_max": 17.331851959228516,
      "activations/layer1_attention_weight_min": -14.23075008392334,
      "activations/layer20_attention_weight_max": 37.31842803955078,
      "activations/layer20_attention_weight_min": -25.10970115661621,
      "activations/layer21_attention_weight_max": 36.03560256958008,
      "activations/layer21_attention_weight_min": -20.385391235351562,
      "activations/layer22_attention_weight_max": 49.741275787353516,
      "activations/layer22_attention_weight_min": -27.336837768554688,
      "activations/layer23_attention_weight_max": 32.266685485839844,
      "activations/layer23_attention_weight_min": -21.90622329711914,
      "activations/layer2_attention_weight_max": 31.22341537475586,
      "activations/layer2_attention_weight_min": -32.94441223144531,
      "activations/layer3_attention_weight_max": 88.35865020751953,
      "activations/layer3_attention_weight_min": -89.01317596435547,
      "activations/layer4_attention_weight_max": 132.97007751464844,
      "activations/layer4_attention_weight_min": -127.75599670410156,
      "activations/layer5_attention_weight_max": 81.832275390625,
      "activations/layer5_attention_weight_min": -79.54615783691406,
      "activations/layer6_attention_weight_max": 55.91495895385742,
      "activations/layer6_attention_weight_min": -58.12921142578125,
      "activations/layer7_attention_weight_max": 81.21621704101562,
      "activations/layer7_attention_weight_min": -81.05245208740234,
      "activations/layer8_attention_weight_max": 40.47264099121094,
      "activations/layer8_attention_weight_min": -38.73255920410156,
      "activations/layer9_attention_weight_max": 36.96250534057617,
      "activations/layer9_attention_weight_min": -34.122779846191406,
      "epoch": 4.58,
      "learning_rate": 0.0001217155303030303,
      "loss": 2.85,
      "step": 78750
    },
    {
      "activations/layer0_attention_weight_max": 17.050460815429688,
      "activations/layer0_attention_weight_min": -12.459656715393066,
      "activations/layer10_attention_weight_max": 32.3663444519043,
      "activations/layer10_attention_weight_min": -29.8187255859375,
      "activations/layer11_attention_weight_max": 30.840499877929688,
      "activations/layer11_attention_weight_min": -26.785911560058594,
      "activations/layer12_attention_weight_max": 25.40696907043457,
      "activations/layer12_attention_weight_min": -27.07611656188965,
      "activations/layer13_attention_weight_max": 39.25177001953125,
      "activations/layer13_attention_weight_min": -30.967952728271484,
      "activations/layer14_attention_weight_max": 61.127899169921875,
      "activations/layer14_attention_weight_min": -45.188690185546875,
      "activations/layer15_attention_weight_max": 41.730552673339844,
      "activations/layer15_attention_weight_min": -27.132566452026367,
      "activations/layer16_attention_weight_max": 35.671878814697266,
      "activations/layer16_attention_weight_min": -25.853689193725586,
      "activations/layer17_attention_weight_max": 37.859134674072266,
      "activations/layer17_attention_weight_min": -24.74838638305664,
      "activations/layer18_attention_weight_max": 28.5639591217041,
      "activations/layer18_attention_weight_min": -19.184078216552734,
      "activations/layer19_attention_weight_max": 39.59450149536133,
      "activations/layer19_attention_weight_min": -28.199676513671875,
      "activations/layer1_attention_weight_max": 17.708385467529297,
      "activations/layer1_attention_weight_min": -14.306403160095215,
      "activations/layer20_attention_weight_max": 30.560840606689453,
      "activations/layer20_attention_weight_min": -21.955307006835938,
      "activations/layer21_attention_weight_max": 26.75244140625,
      "activations/layer21_attention_weight_min": -20.60352325439453,
      "activations/layer22_attention_weight_max": 46.19300079345703,
      "activations/layer22_attention_weight_min": -28.01019287109375,
      "activations/layer23_attention_weight_max": 29.12898826599121,
      "activations/layer23_attention_weight_min": -20.12509536743164,
      "activations/layer2_attention_weight_max": 30.261280059814453,
      "activations/layer2_attention_weight_min": -30.30311393737793,
      "activations/layer3_attention_weight_max": 94.10303497314453,
      "activations/layer3_attention_weight_min": -92.34993743896484,
      "activations/layer4_attention_weight_max": 136.95069885253906,
      "activations/layer4_attention_weight_min": -130.00161743164062,
      "activations/layer5_attention_weight_max": 84.1808090209961,
      "activations/layer5_attention_weight_min": -84.95821380615234,
      "activations/layer6_attention_weight_max": 53.45520782470703,
      "activations/layer6_attention_weight_min": -53.656593322753906,
      "activations/layer7_attention_weight_max": 74.54891967773438,
      "activations/layer7_attention_weight_min": -75.16529083251953,
      "activations/layer8_attention_weight_max": 35.45878219604492,
      "activations/layer8_attention_weight_min": -38.011146545410156,
      "activations/layer9_attention_weight_max": 35.03641891479492,
      "activations/layer9_attention_weight_min": -36.131656646728516,
      "epoch": 4.58,
      "learning_rate": 0.00012169659090909089,
      "loss": 2.8803,
      "step": 78800
    },
    {
      "activations/layer0_attention_weight_max": 16.371183395385742,
      "activations/layer0_attention_weight_min": -12.122605323791504,
      "activations/layer10_attention_weight_max": 32.12437438964844,
      "activations/layer10_attention_weight_min": -29.37603759765625,
      "activations/layer11_attention_weight_max": 31.951648712158203,
      "activations/layer11_attention_weight_min": -27.978984832763672,
      "activations/layer12_attention_weight_max": 29.932857513427734,
      "activations/layer12_attention_weight_min": -26.739011764526367,
      "activations/layer13_attention_weight_max": 42.345558166503906,
      "activations/layer13_attention_weight_min": -32.26649856567383,
      "activations/layer14_attention_weight_max": 67.2764663696289,
      "activations/layer14_attention_weight_min": -47.926605224609375,
      "activations/layer15_attention_weight_max": 37.43191909790039,
      "activations/layer15_attention_weight_min": -29.17051887512207,
      "activations/layer16_attention_weight_max": 44.21072769165039,
      "activations/layer16_attention_weight_min": -31.707706451416016,
      "activations/layer17_attention_weight_max": 41.53962326049805,
      "activations/layer17_attention_weight_min": -24.616525650024414,
      "activations/layer18_attention_weight_max": 32.395755767822266,
      "activations/layer18_attention_weight_min": -20.45271873474121,
      "activations/layer19_attention_weight_max": 54.44932174682617,
      "activations/layer19_attention_weight_min": -30.4732723236084,
      "activations/layer1_attention_weight_max": 18.258834838867188,
      "activations/layer1_attention_weight_min": -14.848685264587402,
      "activations/layer20_attention_weight_max": 38.478904724121094,
      "activations/layer20_attention_weight_min": -23.68720054626465,
      "activations/layer21_attention_weight_max": 34.85128402709961,
      "activations/layer21_attention_weight_min": -19.30039405822754,
      "activations/layer22_attention_weight_max": 57.7717170715332,
      "activations/layer22_attention_weight_min": -25.76178741455078,
      "activations/layer23_attention_weight_max": 33.05809783935547,
      "activations/layer23_attention_weight_min": -17.257003784179688,
      "activations/layer2_attention_weight_max": 31.498855590820312,
      "activations/layer2_attention_weight_min": -31.094432830810547,
      "activations/layer3_attention_weight_max": 90.97701263427734,
      "activations/layer3_attention_weight_min": -88.18583679199219,
      "activations/layer4_attention_weight_max": 128.36634826660156,
      "activations/layer4_attention_weight_min": -127.57437896728516,
      "activations/layer5_attention_weight_max": 86.33983612060547,
      "activations/layer5_attention_weight_min": -83.40834045410156,
      "activations/layer6_attention_weight_max": 57.08683395385742,
      "activations/layer6_attention_weight_min": -58.52213668823242,
      "activations/layer7_attention_weight_max": 79.0079116821289,
      "activations/layer7_attention_weight_min": -82.0215835571289,
      "activations/layer8_attention_weight_max": 36.9820556640625,
      "activations/layer8_attention_weight_min": -36.96058654785156,
      "activations/layer9_attention_weight_max": 34.265525817871094,
      "activations/layer9_attention_weight_min": -33.337303161621094,
      "epoch": 4.58,
      "learning_rate": 0.0001216776515151515,
      "loss": 2.8733,
      "step": 78850
    },
    {
      "activations/layer0_attention_weight_max": 16.624610900878906,
      "activations/layer0_attention_weight_min": -11.895284652709961,
      "activations/layer10_attention_weight_max": 34.01982498168945,
      "activations/layer10_attention_weight_min": -29.839996337890625,
      "activations/layer11_attention_weight_max": 32.16775131225586,
      "activations/layer11_attention_weight_min": -26.859638214111328,
      "activations/layer12_attention_weight_max": 26.212099075317383,
      "activations/layer12_attention_weight_min": -25.95604133605957,
      "activations/layer13_attention_weight_max": 41.518768310546875,
      "activations/layer13_attention_weight_min": -33.40449142456055,
      "activations/layer14_attention_weight_max": 62.99671936035156,
      "activations/layer14_attention_weight_min": -46.32139587402344,
      "activations/layer15_attention_weight_max": 38.19659423828125,
      "activations/layer15_attention_weight_min": -28.724166870117188,
      "activations/layer16_attention_weight_max": 35.39844512939453,
      "activations/layer16_attention_weight_min": -27.084575653076172,
      "activations/layer17_attention_weight_max": 39.013145446777344,
      "activations/layer17_attention_weight_min": -24.285924911499023,
      "activations/layer18_attention_weight_max": 29.18461799621582,
      "activations/layer18_attention_weight_min": -18.129718780517578,
      "activations/layer19_attention_weight_max": 41.709938049316406,
      "activations/layer19_attention_weight_min": -30.789968490600586,
      "activations/layer1_attention_weight_max": 17.347688674926758,
      "activations/layer1_attention_weight_min": -14.015579223632812,
      "activations/layer20_attention_weight_max": 31.085411071777344,
      "activations/layer20_attention_weight_min": -24.119966506958008,
      "activations/layer21_attention_weight_max": 33.22015380859375,
      "activations/layer21_attention_weight_min": -20.279056549072266,
      "activations/layer22_attention_weight_max": 55.66553497314453,
      "activations/layer22_attention_weight_min": -26.856903076171875,
      "activations/layer23_attention_weight_max": 34.02574920654297,
      "activations/layer23_attention_weight_min": -17.13995361328125,
      "activations/layer2_attention_weight_max": 31.08349609375,
      "activations/layer2_attention_weight_min": -30.199447631835938,
      "activations/layer3_attention_weight_max": 87.46308898925781,
      "activations/layer3_attention_weight_min": -87.00421142578125,
      "activations/layer4_attention_weight_max": 130.6370086669922,
      "activations/layer4_attention_weight_min": -123.17008209228516,
      "activations/layer5_attention_weight_max": 86.05757141113281,
      "activations/layer5_attention_weight_min": -78.9278335571289,
      "activations/layer6_attention_weight_max": 57.66619873046875,
      "activations/layer6_attention_weight_min": -54.63926315307617,
      "activations/layer7_attention_weight_max": 80.2862777709961,
      "activations/layer7_attention_weight_min": -88.08098602294922,
      "activations/layer8_attention_weight_max": 40.1728401184082,
      "activations/layer8_attention_weight_min": -38.165016174316406,
      "activations/layer9_attention_weight_max": 36.8946418762207,
      "activations/layer9_attention_weight_min": -31.941879272460938,
      "epoch": 4.58,
      "learning_rate": 0.00012165909090909089,
      "loss": 2.8735,
      "step": 78900
    },
    {
      "activations/layer0_attention_weight_max": 16.07761573791504,
      "activations/layer0_attention_weight_min": -12.650830268859863,
      "activations/layer10_attention_weight_max": 34.066837310791016,
      "activations/layer10_attention_weight_min": -31.555212020874023,
      "activations/layer11_attention_weight_max": 34.93037796020508,
      "activations/layer11_attention_weight_min": -26.23847770690918,
      "activations/layer12_attention_weight_max": 29.221445083618164,
      "activations/layer12_attention_weight_min": -31.517805099487305,
      "activations/layer13_attention_weight_max": 45.79177474975586,
      "activations/layer13_attention_weight_min": -33.70260238647461,
      "activations/layer14_attention_weight_max": 65.48910522460938,
      "activations/layer14_attention_weight_min": -48.824005126953125,
      "activations/layer15_attention_weight_max": 39.30903244018555,
      "activations/layer15_attention_weight_min": -28.774389266967773,
      "activations/layer16_attention_weight_max": 38.87412643432617,
      "activations/layer16_attention_weight_min": -29.53398323059082,
      "activations/layer17_attention_weight_max": 37.929473876953125,
      "activations/layer17_attention_weight_min": -24.90740203857422,
      "activations/layer18_attention_weight_max": 30.05938148498535,
      "activations/layer18_attention_weight_min": -22.226608276367188,
      "activations/layer19_attention_weight_max": 42.309478759765625,
      "activations/layer19_attention_weight_min": -29.680692672729492,
      "activations/layer1_attention_weight_max": 17.636388778686523,
      "activations/layer1_attention_weight_min": -14.614912033081055,
      "activations/layer20_attention_weight_max": 31.355934143066406,
      "activations/layer20_attention_weight_min": -23.434764862060547,
      "activations/layer21_attention_weight_max": 33.73634719848633,
      "activations/layer21_attention_weight_min": -20.86868667602539,
      "activations/layer22_attention_weight_max": 48.09807205200195,
      "activations/layer22_attention_weight_min": -28.739208221435547,
      "activations/layer23_attention_weight_max": 30.73508071899414,
      "activations/layer23_attention_weight_min": -21.447040557861328,
      "activations/layer2_attention_weight_max": 32.543514251708984,
      "activations/layer2_attention_weight_min": -31.53791618347168,
      "activations/layer3_attention_weight_max": 92.8672103881836,
      "activations/layer3_attention_weight_min": -89.88446044921875,
      "activations/layer4_attention_weight_max": 141.5861053466797,
      "activations/layer4_attention_weight_min": -132.32989501953125,
      "activations/layer5_attention_weight_max": 91.18596649169922,
      "activations/layer5_attention_weight_min": -87.66217041015625,
      "activations/layer6_attention_weight_max": 57.60715103149414,
      "activations/layer6_attention_weight_min": -61.80906677246094,
      "activations/layer7_attention_weight_max": 83.88131713867188,
      "activations/layer7_attention_weight_min": -80.74555969238281,
      "activations/layer8_attention_weight_max": 41.812530517578125,
      "activations/layer8_attention_weight_min": -40.60391616821289,
      "activations/layer9_attention_weight_max": 38.712921142578125,
      "activations/layer9_attention_weight_min": -35.46242904663086,
      "epoch": 4.59,
      "learning_rate": 0.00012164015151515151,
      "loss": 2.8484,
      "step": 78950
    },
    {
      "activations/layer0_attention_weight_max": 16.534421920776367,
      "activations/layer0_attention_weight_min": -12.128478050231934,
      "activations/layer10_attention_weight_max": 37.035377502441406,
      "activations/layer10_attention_weight_min": -30.09605598449707,
      "activations/layer11_attention_weight_max": 36.36691665649414,
      "activations/layer11_attention_weight_min": -28.979543685913086,
      "activations/layer12_attention_weight_max": 33.80891799926758,
      "activations/layer12_attention_weight_min": -27.24417495727539,
      "activations/layer13_attention_weight_max": 43.29814147949219,
      "activations/layer13_attention_weight_min": -31.530677795410156,
      "activations/layer14_attention_weight_max": 63.80903625488281,
      "activations/layer14_attention_weight_min": -44.586917877197266,
      "activations/layer15_attention_weight_max": 41.467899322509766,
      "activations/layer15_attention_weight_min": -29.104862213134766,
      "activations/layer16_attention_weight_max": 35.76411819458008,
      "activations/layer16_attention_weight_min": -27.63421058654785,
      "activations/layer17_attention_weight_max": 41.83662796020508,
      "activations/layer17_attention_weight_min": -25.49047088623047,
      "activations/layer18_attention_weight_max": 30.842548370361328,
      "activations/layer18_attention_weight_min": -20.65096664428711,
      "activations/layer19_attention_weight_max": 41.834228515625,
      "activations/layer19_attention_weight_min": -28.71784019470215,
      "activations/layer1_attention_weight_max": 17.516443252563477,
      "activations/layer1_attention_weight_min": -13.849149703979492,
      "activations/layer20_attention_weight_max": 32.61974334716797,
      "activations/layer20_attention_weight_min": -22.06914520263672,
      "activations/layer21_attention_weight_max": 31.02042007446289,
      "activations/layer21_attention_weight_min": -20.14969825744629,
      "activations/layer22_attention_weight_max": 50.23637008666992,
      "activations/layer22_attention_weight_min": -29.707279205322266,
      "activations/layer23_attention_weight_max": 28.946752548217773,
      "activations/layer23_attention_weight_min": -18.136802673339844,
      "activations/layer2_attention_weight_max": 31.05850601196289,
      "activations/layer2_attention_weight_min": -29.345394134521484,
      "activations/layer3_attention_weight_max": 82.6619873046875,
      "activations/layer3_attention_weight_min": -80.91229248046875,
      "activations/layer4_attention_weight_max": 119.35322570800781,
      "activations/layer4_attention_weight_min": -117.61641693115234,
      "activations/layer5_attention_weight_max": 82.19508361816406,
      "activations/layer5_attention_weight_min": -78.96088409423828,
      "activations/layer6_attention_weight_max": 58.857059478759766,
      "activations/layer6_attention_weight_min": -54.11431121826172,
      "activations/layer7_attention_weight_max": 82.3090591430664,
      "activations/layer7_attention_weight_min": -77.72268676757812,
      "activations/layer8_attention_weight_max": 40.23905563354492,
      "activations/layer8_attention_weight_min": -38.053871154785156,
      "activations/layer9_attention_weight_max": 38.45039749145508,
      "activations/layer9_attention_weight_min": -33.02547073364258,
      "epoch": 4.59,
      "learning_rate": 0.00012162121212121211,
      "loss": 2.8727,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_loss": 2.814453125,
      "eval_runtime": 8.4825,
      "eval_samples_per_second": 506.219,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_openwebtext_loss": 2.814453125,
      "eval_openwebtext_ppl": 16.684049195604416,
      "eval_openwebtext_runtime": 8.4825,
      "eval_openwebtext_samples_per_second": 506.219,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_wikitext_loss": 3.068359375,
      "eval_wikitext_ppl": 21.506589466713315,
      "eval_wikitext_runtime": 1.994,
      "eval_wikitext_samples_per_second": 228.69,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_lambada_loss": 2.7421875,
      "eval_lambada_ppl": 15.520899957093135,
      "eval_lambada_runtime": 9.5725,
      "eval_lambada_samples_per_second": 508.642,
      "step": 79000
    },
    {
      "activations/layer0_attention_weight_max": 16.799678802490234,
      "activations/layer0_attention_weight_min": -12.160711288452148,
      "activations/layer10_attention_weight_max": 38.59357452392578,
      "activations/layer10_attention_weight_min": -31.10832977294922,
      "activations/layer11_attention_weight_max": 34.17292404174805,
      "activations/layer11_attention_weight_min": -27.67761993408203,
      "activations/layer12_attention_weight_max": 49.187530517578125,
      "activations/layer12_attention_weight_min": -30.39246368408203,
      "activations/layer13_attention_weight_max": 59.3514404296875,
      "activations/layer13_attention_weight_min": -36.42950439453125,
      "activations/layer14_attention_weight_max": 60.35042953491211,
      "activations/layer14_attention_weight_min": -43.11566162109375,
      "activations/layer15_attention_weight_max": 42.14567947387695,
      "activations/layer15_attention_weight_min": -28.90403938293457,
      "activations/layer16_attention_weight_max": 38.2908935546875,
      "activations/layer16_attention_weight_min": -28.87736701965332,
      "activations/layer17_attention_weight_max": 39.402069091796875,
      "activations/layer17_attention_weight_min": -25.6779727935791,
      "activations/layer18_attention_weight_max": 34.79755783081055,
      "activations/layer18_attention_weight_min": -18.407089233398438,
      "activations/layer19_attention_weight_max": 36.48966598510742,
      "activations/layer19_attention_weight_min": -28.41269874572754,
      "activations/layer1_attention_weight_max": 18.905630111694336,
      "activations/layer1_attention_weight_min": -13.90912914276123,
      "activations/layer20_attention_weight_max": 34.26485061645508,
      "activations/layer20_attention_weight_min": -23.44926643371582,
      "activations/layer21_attention_weight_max": 35.82792663574219,
      "activations/layer21_attention_weight_min": -19.96396827697754,
      "activations/layer22_attention_weight_max": 48.76609420776367,
      "activations/layer22_attention_weight_min": -28.70352554321289,
      "activations/layer23_attention_weight_max": 30.46858787536621,
      "activations/layer23_attention_weight_min": -19.12717628479004,
      "activations/layer2_attention_weight_max": 33.22513961791992,
      "activations/layer2_attention_weight_min": -32.9985237121582,
      "activations/layer3_attention_weight_max": 88.65857696533203,
      "activations/layer3_attention_weight_min": -94.36975860595703,
      "activations/layer4_attention_weight_max": 134.07337951660156,
      "activations/layer4_attention_weight_min": -131.80406188964844,
      "activations/layer5_attention_weight_max": 86.14220428466797,
      "activations/layer5_attention_weight_min": -83.69964599609375,
      "activations/layer6_attention_weight_max": 59.413475036621094,
      "activations/layer6_attention_weight_min": -56.12189483642578,
      "activations/layer7_attention_weight_max": 89.55303192138672,
      "activations/layer7_attention_weight_min": -82.69586181640625,
      "activations/layer8_attention_weight_max": 42.85186767578125,
      "activations/layer8_attention_weight_min": -39.55519485473633,
      "activations/layer9_attention_weight_max": 40.74296569824219,
      "activations/layer9_attention_weight_min": -34.697731018066406,
      "epoch": 4.59,
      "learning_rate": 0.00012160227272727271,
      "loss": 2.8661,
      "step": 79050
    },
    {
      "activations/layer0_attention_weight_max": 16.89101219177246,
      "activations/layer0_attention_weight_min": -13.114800453186035,
      "activations/layer10_attention_weight_max": 32.588401794433594,
      "activations/layer10_attention_weight_min": -30.499589920043945,
      "activations/layer11_attention_weight_max": 31.57168197631836,
      "activations/layer11_attention_weight_min": -27.773273468017578,
      "activations/layer12_attention_weight_max": 27.13117027282715,
      "activations/layer12_attention_weight_min": -26.501291275024414,
      "activations/layer13_attention_weight_max": 37.96084213256836,
      "activations/layer13_attention_weight_min": -29.890756607055664,
      "activations/layer14_attention_weight_max": 75.18677520751953,
      "activations/layer14_attention_weight_min": -54.904205322265625,
      "activations/layer15_attention_weight_max": 36.68633270263672,
      "activations/layer15_attention_weight_min": -28.026782989501953,
      "activations/layer16_attention_weight_max": 40.52088165283203,
      "activations/layer16_attention_weight_min": -26.65416717529297,
      "activations/layer17_attention_weight_max": 37.0732421875,
      "activations/layer17_attention_weight_min": -24.254335403442383,
      "activations/layer18_attention_weight_max": 27.747163772583008,
      "activations/layer18_attention_weight_min": -19.439607620239258,
      "activations/layer19_attention_weight_max": 44.057151794433594,
      "activations/layer19_attention_weight_min": -29.823755264282227,
      "activations/layer1_attention_weight_max": 17.848003387451172,
      "activations/layer1_attention_weight_min": -14.470059394836426,
      "activations/layer20_attention_weight_max": 30.808393478393555,
      "activations/layer20_attention_weight_min": -22.49510955810547,
      "activations/layer21_attention_weight_max": 26.40532112121582,
      "activations/layer21_attention_weight_min": -20.768447875976562,
      "activations/layer22_attention_weight_max": 48.5774040222168,
      "activations/layer22_attention_weight_min": -26.983051300048828,
      "activations/layer23_attention_weight_max": 31.886302947998047,
      "activations/layer23_attention_weight_min": -21.185033798217773,
      "activations/layer2_attention_weight_max": 32.799598693847656,
      "activations/layer2_attention_weight_min": -31.88744354248047,
      "activations/layer3_attention_weight_max": 88.72733306884766,
      "activations/layer3_attention_weight_min": -86.71088409423828,
      "activations/layer4_attention_weight_max": 129.2713165283203,
      "activations/layer4_attention_weight_min": -123.51478576660156,
      "activations/layer5_attention_weight_max": 87.8497543334961,
      "activations/layer5_attention_weight_min": -83.69669342041016,
      "activations/layer6_attention_weight_max": 59.58496856689453,
      "activations/layer6_attention_weight_min": -55.28363800048828,
      "activations/layer7_attention_weight_max": 78.96660614013672,
      "activations/layer7_attention_weight_min": -76.75666046142578,
      "activations/layer8_attention_weight_max": 37.58124923706055,
      "activations/layer8_attention_weight_min": -40.408809661865234,
      "activations/layer9_attention_weight_max": 37.76918029785156,
      "activations/layer9_attention_weight_min": -36.978363037109375,
      "epoch": 4.6,
      "learning_rate": 0.00012158333333333333,
      "loss": 2.8653,
      "step": 79100
    },
    {
      "activations/layer0_attention_weight_max": 16.576629638671875,
      "activations/layer0_attention_weight_min": -12.577131271362305,
      "activations/layer10_attention_weight_max": 33.76903533935547,
      "activations/layer10_attention_weight_min": -28.967235565185547,
      "activations/layer11_attention_weight_max": 30.225128173828125,
      "activations/layer11_attention_weight_min": -29.024295806884766,
      "activations/layer12_attention_weight_max": 26.099214553833008,
      "activations/layer12_attention_weight_min": -27.684310913085938,
      "activations/layer13_attention_weight_max": 43.78394317626953,
      "activations/layer13_attention_weight_min": -31.220787048339844,
      "activations/layer14_attention_weight_max": 59.96986389160156,
      "activations/layer14_attention_weight_min": -44.43589782714844,
      "activations/layer15_attention_weight_max": 42.28028869628906,
      "activations/layer15_attention_weight_min": -29.183910369873047,
      "activations/layer16_attention_weight_max": 36.1549186706543,
      "activations/layer16_attention_weight_min": -28.309249877929688,
      "activations/layer17_attention_weight_max": 39.92946243286133,
      "activations/layer17_attention_weight_min": -24.167455673217773,
      "activations/layer18_attention_weight_max": 35.234100341796875,
      "activations/layer18_attention_weight_min": -24.661191940307617,
      "activations/layer19_attention_weight_max": 36.98119354248047,
      "activations/layer19_attention_weight_min": -28.946094512939453,
      "activations/layer1_attention_weight_max": 17.306678771972656,
      "activations/layer1_attention_weight_min": -13.833844184875488,
      "activations/layer20_attention_weight_max": 38.36969757080078,
      "activations/layer20_attention_weight_min": -24.484413146972656,
      "activations/layer21_attention_weight_max": 35.77388381958008,
      "activations/layer21_attention_weight_min": -22.407466888427734,
      "activations/layer22_attention_weight_max": 48.46550750732422,
      "activations/layer22_attention_weight_min": -29.086742401123047,
      "activations/layer23_attention_weight_max": 31.96748161315918,
      "activations/layer23_attention_weight_min": -20.724563598632812,
      "activations/layer2_attention_weight_max": 32.220008850097656,
      "activations/layer2_attention_weight_min": -29.9798583984375,
      "activations/layer3_attention_weight_max": 89.89288330078125,
      "activations/layer3_attention_weight_min": -85.25931549072266,
      "activations/layer4_attention_weight_max": 132.94529724121094,
      "activations/layer4_attention_weight_min": -122.74126434326172,
      "activations/layer5_attention_weight_max": 83.14606475830078,
      "activations/layer5_attention_weight_min": -80.56786346435547,
      "activations/layer6_attention_weight_max": 58.871009826660156,
      "activations/layer6_attention_weight_min": -54.77432632446289,
      "activations/layer7_attention_weight_max": 83.97738647460938,
      "activations/layer7_attention_weight_min": -81.19324493408203,
      "activations/layer8_attention_weight_max": 40.90977096557617,
      "activations/layer8_attention_weight_min": -38.78446960449219,
      "activations/layer9_attention_weight_max": 36.692787170410156,
      "activations/layer9_attention_weight_min": -36.189178466796875,
      "epoch": 4.6,
      "learning_rate": 0.00012156439393939393,
      "loss": 2.8617,
      "step": 79150
    },
    {
      "activations/layer0_attention_weight_max": 15.02002239227295,
      "activations/layer0_attention_weight_min": -10.928438186645508,
      "activations/layer10_attention_weight_max": 35.443511962890625,
      "activations/layer10_attention_weight_min": -30.41349983215332,
      "activations/layer11_attention_weight_max": 34.73204803466797,
      "activations/layer11_attention_weight_min": -27.03921890258789,
      "activations/layer12_attention_weight_max": 37.71343994140625,
      "activations/layer12_attention_weight_min": -29.027708053588867,
      "activations/layer13_attention_weight_max": 49.33323287963867,
      "activations/layer13_attention_weight_min": -33.9593620300293,
      "activations/layer14_attention_weight_max": 66.53868865966797,
      "activations/layer14_attention_weight_min": -47.74490737915039,
      "activations/layer15_attention_weight_max": 49.29288864135742,
      "activations/layer15_attention_weight_min": -33.032005310058594,
      "activations/layer16_attention_weight_max": 38.12442398071289,
      "activations/layer16_attention_weight_min": -29.84870147705078,
      "activations/layer17_attention_weight_max": 43.39773941040039,
      "activations/layer17_attention_weight_min": -27.574552536010742,
      "activations/layer18_attention_weight_max": 31.872671127319336,
      "activations/layer18_attention_weight_min": -19.631267547607422,
      "activations/layer19_attention_weight_max": 44.18491744995117,
      "activations/layer19_attention_weight_min": -29.80948829650879,
      "activations/layer1_attention_weight_max": 17.96150779724121,
      "activations/layer1_attention_weight_min": -16.56711769104004,
      "activations/layer20_attention_weight_max": 31.7574405670166,
      "activations/layer20_attention_weight_min": -23.22081756591797,
      "activations/layer21_attention_weight_max": 31.498077392578125,
      "activations/layer21_attention_weight_min": -21.48174285888672,
      "activations/layer22_attention_weight_max": 50.40938186645508,
      "activations/layer22_attention_weight_min": -27.81943130493164,
      "activations/layer23_attention_weight_max": 31.85942840576172,
      "activations/layer23_attention_weight_min": -17.81033706665039,
      "activations/layer2_attention_weight_max": 33.742759704589844,
      "activations/layer2_attention_weight_min": -31.959320068359375,
      "activations/layer3_attention_weight_max": 88.12954711914062,
      "activations/layer3_attention_weight_min": -86.11568450927734,
      "activations/layer4_attention_weight_max": 130.2256622314453,
      "activations/layer4_attention_weight_min": -128.4957733154297,
      "activations/layer5_attention_weight_max": 88.7818603515625,
      "activations/layer5_attention_weight_min": -84.45719909667969,
      "activations/layer6_attention_weight_max": 59.217002868652344,
      "activations/layer6_attention_weight_min": -59.31220626831055,
      "activations/layer7_attention_weight_max": 84.64106750488281,
      "activations/layer7_attention_weight_min": -83.30359649658203,
      "activations/layer8_attention_weight_max": 40.26467514038086,
      "activations/layer8_attention_weight_min": -39.873451232910156,
      "activations/layer9_attention_weight_max": 40.567779541015625,
      "activations/layer9_attention_weight_min": -38.87000274658203,
      "epoch": 4.6,
      "learning_rate": 0.00012154545454545453,
      "loss": 2.8689,
      "step": 79200
    },
    {
      "activations/layer0_attention_weight_max": 16.80947494506836,
      "activations/layer0_attention_weight_min": -11.617999076843262,
      "activations/layer10_attention_weight_max": 38.68845748901367,
      "activations/layer10_attention_weight_min": -32.15501022338867,
      "activations/layer11_attention_weight_max": 39.7388916015625,
      "activations/layer11_attention_weight_min": -29.04034996032715,
      "activations/layer12_attention_weight_max": 47.25410842895508,
      "activations/layer12_attention_weight_min": -32.62845993041992,
      "activations/layer13_attention_weight_max": 72.52271270751953,
      "activations/layer13_attention_weight_min": -38.15168762207031,
      "activations/layer14_attention_weight_max": 102.02108001708984,
      "activations/layer14_attention_weight_min": -62.30327606201172,
      "activations/layer15_attention_weight_max": 68.01737976074219,
      "activations/layer15_attention_weight_min": -30.891355514526367,
      "activations/layer16_attention_weight_max": 57.574119567871094,
      "activations/layer16_attention_weight_min": -27.134294509887695,
      "activations/layer17_attention_weight_max": 52.882266998291016,
      "activations/layer17_attention_weight_min": -24.000835418701172,
      "activations/layer18_attention_weight_max": 39.9668083190918,
      "activations/layer18_attention_weight_min": -23.131149291992188,
      "activations/layer19_attention_weight_max": 56.105712890625,
      "activations/layer19_attention_weight_min": -26.800275802612305,
      "activations/layer1_attention_weight_max": 17.066635131835938,
      "activations/layer1_attention_weight_min": -13.520954132080078,
      "activations/layer20_attention_weight_max": 39.305416107177734,
      "activations/layer20_attention_weight_min": -21.705730438232422,
      "activations/layer21_attention_weight_max": 34.43804168701172,
      "activations/layer21_attention_weight_min": -20.196989059448242,
      "activations/layer22_attention_weight_max": 61.224945068359375,
      "activations/layer22_attention_weight_min": -24.97324562072754,
      "activations/layer23_attention_weight_max": 34.95238494873047,
      "activations/layer23_attention_weight_min": -18.23243522644043,
      "activations/layer2_attention_weight_max": 32.11840057373047,
      "activations/layer2_attention_weight_min": -30.99964141845703,
      "activations/layer3_attention_weight_max": 94.88423156738281,
      "activations/layer3_attention_weight_min": -97.23185729980469,
      "activations/layer4_attention_weight_max": 141.5048828125,
      "activations/layer4_attention_weight_min": -138.1472625732422,
      "activations/layer5_attention_weight_max": 89.14700317382812,
      "activations/layer5_attention_weight_min": -87.59465789794922,
      "activations/layer6_attention_weight_max": 59.849937438964844,
      "activations/layer6_attention_weight_min": -64.08041381835938,
      "activations/layer7_attention_weight_max": 86.9258804321289,
      "activations/layer7_attention_weight_min": -84.31736755371094,
      "activations/layer8_attention_weight_max": 42.07917022705078,
      "activations/layer8_attention_weight_min": -40.21824264526367,
      "activations/layer9_attention_weight_max": 38.594966888427734,
      "activations/layer9_attention_weight_min": -39.618019104003906,
      "epoch": 4.6,
      "learning_rate": 0.00012152651515151514,
      "loss": 2.8616,
      "step": 79250
    },
    {
      "activations/layer0_attention_weight_max": 15.658052444458008,
      "activations/layer0_attention_weight_min": -11.164673805236816,
      "activations/layer10_attention_weight_max": 37.483642578125,
      "activations/layer10_attention_weight_min": -31.799673080444336,
      "activations/layer11_attention_weight_max": 37.131813049316406,
      "activations/layer11_attention_weight_min": -26.096508026123047,
      "activations/layer12_attention_weight_max": 31.130531311035156,
      "activations/layer12_attention_weight_min": -28.439821243286133,
      "activations/layer13_attention_weight_max": 46.23982238769531,
      "activations/layer13_attention_weight_min": -35.94358825683594,
      "activations/layer14_attention_weight_max": 71.59248352050781,
      "activations/layer14_attention_weight_min": -44.77317810058594,
      "activations/layer15_attention_weight_max": 44.792396545410156,
      "activations/layer15_attention_weight_min": -27.788944244384766,
      "activations/layer16_attention_weight_max": 47.39839172363281,
      "activations/layer16_attention_weight_min": -29.544694900512695,
      "activations/layer17_attention_weight_max": 43.99775695800781,
      "activations/layer17_attention_weight_min": -23.37347984313965,
      "activations/layer18_attention_weight_max": 32.37189483642578,
      "activations/layer18_attention_weight_min": -20.217870712280273,
      "activations/layer19_attention_weight_max": 49.34592056274414,
      "activations/layer19_attention_weight_min": -31.599340438842773,
      "activations/layer1_attention_weight_max": 17.841127395629883,
      "activations/layer1_attention_weight_min": -16.31143569946289,
      "activations/layer20_attention_weight_max": 35.45917510986328,
      "activations/layer20_attention_weight_min": -25.336870193481445,
      "activations/layer21_attention_weight_max": 38.70381546020508,
      "activations/layer21_attention_weight_min": -20.747465133666992,
      "activations/layer22_attention_weight_max": 53.71099090576172,
      "activations/layer22_attention_weight_min": -30.628360748291016,
      "activations/layer23_attention_weight_max": 34.914955139160156,
      "activations/layer23_attention_weight_min": -20.538982391357422,
      "activations/layer2_attention_weight_max": 31.42498779296875,
      "activations/layer2_attention_weight_min": -30.32081413269043,
      "activations/layer3_attention_weight_max": 87.71224975585938,
      "activations/layer3_attention_weight_min": -90.12876892089844,
      "activations/layer4_attention_weight_max": 129.51902770996094,
      "activations/layer4_attention_weight_min": -131.84434509277344,
      "activations/layer5_attention_weight_max": 85.33729553222656,
      "activations/layer5_attention_weight_min": -81.28910827636719,
      "activations/layer6_attention_weight_max": 63.61744689941406,
      "activations/layer6_attention_weight_min": -57.93457794189453,
      "activations/layer7_attention_weight_max": 78.92078399658203,
      "activations/layer7_attention_weight_min": -83.45899200439453,
      "activations/layer8_attention_weight_max": 38.475975036621094,
      "activations/layer8_attention_weight_min": -40.64173126220703,
      "activations/layer9_attention_weight_max": 36.62644958496094,
      "activations/layer9_attention_weight_min": -34.597930908203125,
      "epoch": 4.61,
      "learning_rate": 0.00012150757575757574,
      "loss": 2.8618,
      "step": 79300
    },
    {
      "activations/layer0_attention_weight_max": 16.86212730407715,
      "activations/layer0_attention_weight_min": -11.818641662597656,
      "activations/layer10_attention_weight_max": 34.89898681640625,
      "activations/layer10_attention_weight_min": -32.23088073730469,
      "activations/layer11_attention_weight_max": 32.91192626953125,
      "activations/layer11_attention_weight_min": -28.501270294189453,
      "activations/layer12_attention_weight_max": 23.54972267150879,
      "activations/layer12_attention_weight_min": -25.273237228393555,
      "activations/layer13_attention_weight_max": 45.161537170410156,
      "activations/layer13_attention_weight_min": -34.25373077392578,
      "activations/layer14_attention_weight_max": 71.58999633789062,
      "activations/layer14_attention_weight_min": -47.49072265625,
      "activations/layer15_attention_weight_max": 43.881065368652344,
      "activations/layer15_attention_weight_min": -28.233963012695312,
      "activations/layer16_attention_weight_max": 39.34012222290039,
      "activations/layer16_attention_weight_min": -29.16769790649414,
      "activations/layer17_attention_weight_max": 40.45841979980469,
      "activations/layer17_attention_weight_min": -24.191577911376953,
      "activations/layer18_attention_weight_max": 41.10451889038086,
      "activations/layer18_attention_weight_min": -20.789751052856445,
      "activations/layer19_attention_weight_max": 43.344482421875,
      "activations/layer19_attention_weight_min": -31.974328994750977,
      "activations/layer1_attention_weight_max": 17.07733154296875,
      "activations/layer1_attention_weight_min": -14.550617218017578,
      "activations/layer20_attention_weight_max": 31.957773208618164,
      "activations/layer20_attention_weight_min": -24.944507598876953,
      "activations/layer21_attention_weight_max": 35.7950439453125,
      "activations/layer21_attention_weight_min": -21.494529724121094,
      "activations/layer22_attention_weight_max": 50.842594146728516,
      "activations/layer22_attention_weight_min": -27.416650772094727,
      "activations/layer23_attention_weight_max": 31.516088485717773,
      "activations/layer23_attention_weight_min": -20.772994995117188,
      "activations/layer2_attention_weight_max": 31.576904296875,
      "activations/layer2_attention_weight_min": -30.364208221435547,
      "activations/layer3_attention_weight_max": 87.72554779052734,
      "activations/layer3_attention_weight_min": -88.86442565917969,
      "activations/layer4_attention_weight_max": 130.81134033203125,
      "activations/layer4_attention_weight_min": -125.23807525634766,
      "activations/layer5_attention_weight_max": 89.7718505859375,
      "activations/layer5_attention_weight_min": -82.25902557373047,
      "activations/layer6_attention_weight_max": 60.067203521728516,
      "activations/layer6_attention_weight_min": -59.82220458984375,
      "activations/layer7_attention_weight_max": 87.51126861572266,
      "activations/layer7_attention_weight_min": -79.43041229248047,
      "activations/layer8_attention_weight_max": 43.06468200683594,
      "activations/layer8_attention_weight_min": -40.85250473022461,
      "activations/layer9_attention_weight_max": 42.18003463745117,
      "activations/layer9_attention_weight_min": -34.918601989746094,
      "epoch": 4.61,
      "learning_rate": 0.00012148863636363636,
      "loss": 2.8534,
      "step": 79350
    },
    {
      "activations/layer0_attention_weight_max": 16.36771011352539,
      "activations/layer0_attention_weight_min": -11.732474327087402,
      "activations/layer10_attention_weight_max": 37.064064025878906,
      "activations/layer10_attention_weight_min": -29.454105377197266,
      "activations/layer11_attention_weight_max": 41.291160583496094,
      "activations/layer11_attention_weight_min": -27.977415084838867,
      "activations/layer12_attention_weight_max": 47.94257354736328,
      "activations/layer12_attention_weight_min": -26.14800262451172,
      "activations/layer13_attention_weight_max": 49.723854064941406,
      "activations/layer13_attention_weight_min": -35.9837760925293,
      "activations/layer14_attention_weight_max": 71.68321990966797,
      "activations/layer14_attention_weight_min": -44.65723419189453,
      "activations/layer15_attention_weight_max": 49.77838134765625,
      "activations/layer15_attention_weight_min": -33.371639251708984,
      "activations/layer16_attention_weight_max": 42.278804779052734,
      "activations/layer16_attention_weight_min": -28.4678955078125,
      "activations/layer17_attention_weight_max": 47.32667922973633,
      "activations/layer17_attention_weight_min": -23.053089141845703,
      "activations/layer18_attention_weight_max": 33.65468215942383,
      "activations/layer18_attention_weight_min": -19.186450958251953,
      "activations/layer19_attention_weight_max": 46.413082122802734,
      "activations/layer19_attention_weight_min": -28.866397857666016,
      "activations/layer1_attention_weight_max": 17.462339401245117,
      "activations/layer1_attention_weight_min": -13.507894515991211,
      "activations/layer20_attention_weight_max": 36.09408187866211,
      "activations/layer20_attention_weight_min": -22.462121963500977,
      "activations/layer21_attention_weight_max": 35.79460525512695,
      "activations/layer21_attention_weight_min": -23.38805389404297,
      "activations/layer22_attention_weight_max": 57.991573333740234,
      "activations/layer22_attention_weight_min": -26.43256187438965,
      "activations/layer23_attention_weight_max": 31.348575592041016,
      "activations/layer23_attention_weight_min": -19.900785446166992,
      "activations/layer2_attention_weight_max": 29.65062713623047,
      "activations/layer2_attention_weight_min": -31.134183883666992,
      "activations/layer3_attention_weight_max": 89.75166320800781,
      "activations/layer3_attention_weight_min": -90.03800964355469,
      "activations/layer4_attention_weight_max": 130.51109313964844,
      "activations/layer4_attention_weight_min": -122.56085205078125,
      "activations/layer5_attention_weight_max": 80.86771392822266,
      "activations/layer5_attention_weight_min": -78.32823181152344,
      "activations/layer6_attention_weight_max": 53.67219543457031,
      "activations/layer6_attention_weight_min": -52.584449768066406,
      "activations/layer7_attention_weight_max": 84.81517028808594,
      "activations/layer7_attention_weight_min": -79.78656005859375,
      "activations/layer8_attention_weight_max": 40.6889762878418,
      "activations/layer8_attention_weight_min": -37.13320541381836,
      "activations/layer9_attention_weight_max": 39.06991195678711,
      "activations/layer9_attention_weight_min": -32.00288772583008,
      "epoch": 4.61,
      "learning_rate": 0.00012146969696969696,
      "loss": 2.8496,
      "step": 79400
    },
    {
      "activations/layer0_attention_weight_max": 15.872058868408203,
      "activations/layer0_attention_weight_min": -11.525032997131348,
      "activations/layer10_attention_weight_max": 40.528663635253906,
      "activations/layer10_attention_weight_min": -33.74385452270508,
      "activations/layer11_attention_weight_max": 42.1048469543457,
      "activations/layer11_attention_weight_min": -27.66278648376465,
      "activations/layer12_attention_weight_max": 57.56193542480469,
      "activations/layer12_attention_weight_min": -33.92332458496094,
      "activations/layer13_attention_weight_max": 55.54490280151367,
      "activations/layer13_attention_weight_min": -34.01939010620117,
      "activations/layer14_attention_weight_max": 67.25128936767578,
      "activations/layer14_attention_weight_min": -48.34275817871094,
      "activations/layer15_attention_weight_max": 46.699623107910156,
      "activations/layer15_attention_weight_min": -31.059429168701172,
      "activations/layer16_attention_weight_max": 45.532135009765625,
      "activations/layer16_attention_weight_min": -26.969547271728516,
      "activations/layer17_attention_weight_max": 43.537044525146484,
      "activations/layer17_attention_weight_min": -24.511016845703125,
      "activations/layer18_attention_weight_max": 31.398502349853516,
      "activations/layer18_attention_weight_min": -23.45195960998535,
      "activations/layer19_attention_weight_max": 40.8779411315918,
      "activations/layer19_attention_weight_min": -30.638822555541992,
      "activations/layer1_attention_weight_max": 20.202205657958984,
      "activations/layer1_attention_weight_min": -15.972521781921387,
      "activations/layer20_attention_weight_max": 32.31737518310547,
      "activations/layer20_attention_weight_min": -24.7703800201416,
      "activations/layer21_attention_weight_max": 34.839385986328125,
      "activations/layer21_attention_weight_min": -23.314189910888672,
      "activations/layer22_attention_weight_max": 54.387977600097656,
      "activations/layer22_attention_weight_min": -29.327457427978516,
      "activations/layer23_attention_weight_max": 35.16706848144531,
      "activations/layer23_attention_weight_min": -17.81344223022461,
      "activations/layer2_attention_weight_max": 34.11379623413086,
      "activations/layer2_attention_weight_min": -31.669790267944336,
      "activations/layer3_attention_weight_max": 96.24126434326172,
      "activations/layer3_attention_weight_min": -93.5114517211914,
      "activations/layer4_attention_weight_max": 144.6141357421875,
      "activations/layer4_attention_weight_min": -131.50360107421875,
      "activations/layer5_attention_weight_max": 92.63844299316406,
      "activations/layer5_attention_weight_min": -81.4891128540039,
      "activations/layer6_attention_weight_max": 57.43688201904297,
      "activations/layer6_attention_weight_min": -58.11882019042969,
      "activations/layer7_attention_weight_max": 91.90328979492188,
      "activations/layer7_attention_weight_min": -85.36588287353516,
      "activations/layer8_attention_weight_max": 45.52142333984375,
      "activations/layer8_attention_weight_min": -42.476749420166016,
      "activations/layer9_attention_weight_max": 42.35292053222656,
      "activations/layer9_attention_weight_min": -39.55012130737305,
      "epoch": 4.62,
      "learning_rate": 0.00012145075757575756,
      "loss": 2.8609,
      "step": 79450
    },
    {
      "activations/layer0_attention_weight_max": 16.73141098022461,
      "activations/layer0_attention_weight_min": -11.936098098754883,
      "activations/layer10_attention_weight_max": 38.1690673828125,
      "activations/layer10_attention_weight_min": -36.079315185546875,
      "activations/layer11_attention_weight_max": 38.797271728515625,
      "activations/layer11_attention_weight_min": -30.05878257751465,
      "activations/layer12_attention_weight_max": 54.272342681884766,
      "activations/layer12_attention_weight_min": -29.905115127563477,
      "activations/layer13_attention_weight_max": 58.07270050048828,
      "activations/layer13_attention_weight_min": -40.3245964050293,
      "activations/layer14_attention_weight_max": 79.34684753417969,
      "activations/layer14_attention_weight_min": -45.78596878051758,
      "activations/layer15_attention_weight_max": 44.072181701660156,
      "activations/layer15_attention_weight_min": -32.219764709472656,
      "activations/layer16_attention_weight_max": 42.367584228515625,
      "activations/layer16_attention_weight_min": -28.84843635559082,
      "activations/layer17_attention_weight_max": 43.69722366333008,
      "activations/layer17_attention_weight_min": -25.170272827148438,
      "activations/layer18_attention_weight_max": 33.52963638305664,
      "activations/layer18_attention_weight_min": -23.14777374267578,
      "activations/layer19_attention_weight_max": 43.34455108642578,
      "activations/layer19_attention_weight_min": -30.147794723510742,
      "activations/layer1_attention_weight_max": 17.46051597595215,
      "activations/layer1_attention_weight_min": -15.04719066619873,
      "activations/layer20_attention_weight_max": 35.33454895019531,
      "activations/layer20_attention_weight_min": -22.959692001342773,
      "activations/layer21_attention_weight_max": 33.23619842529297,
      "activations/layer21_attention_weight_min": -20.7467098236084,
      "activations/layer22_attention_weight_max": 49.52138900756836,
      "activations/layer22_attention_weight_min": -28.413803100585938,
      "activations/layer23_attention_weight_max": 32.141422271728516,
      "activations/layer23_attention_weight_min": -19.03201675415039,
      "activations/layer2_attention_weight_max": 33.294212341308594,
      "activations/layer2_attention_weight_min": -30.627952575683594,
      "activations/layer3_attention_weight_max": 92.62666320800781,
      "activations/layer3_attention_weight_min": -89.48394012451172,
      "activations/layer4_attention_weight_max": 144.89886474609375,
      "activations/layer4_attention_weight_min": -128.21832275390625,
      "activations/layer5_attention_weight_max": 94.00587463378906,
      "activations/layer5_attention_weight_min": -85.43836975097656,
      "activations/layer6_attention_weight_max": 63.177711486816406,
      "activations/layer6_attention_weight_min": -58.68220901489258,
      "activations/layer7_attention_weight_max": 91.33374786376953,
      "activations/layer7_attention_weight_min": -87.89330291748047,
      "activations/layer8_attention_weight_max": 45.98562240600586,
      "activations/layer8_attention_weight_min": -47.84632110595703,
      "activations/layer9_attention_weight_max": 40.67502975463867,
      "activations/layer9_attention_weight_min": -38.647666931152344,
      "epoch": 4.62,
      "learning_rate": 0.00012143181818181818,
      "loss": 2.8467,
      "step": 79500
    },
    {
      "activations/layer0_attention_weight_max": 16.854915618896484,
      "activations/layer0_attention_weight_min": -12.17643928527832,
      "activations/layer10_attention_weight_max": 29.801902770996094,
      "activations/layer10_attention_weight_min": -28.540822982788086,
      "activations/layer11_attention_weight_max": 30.50075912475586,
      "activations/layer11_attention_weight_min": -26.444683074951172,
      "activations/layer12_attention_weight_max": 23.92877769470215,
      "activations/layer12_attention_weight_min": -25.86579132080078,
      "activations/layer13_attention_weight_max": 39.988197326660156,
      "activations/layer13_attention_weight_min": -32.51726531982422,
      "activations/layer14_attention_weight_max": 57.3721923828125,
      "activations/layer14_attention_weight_min": -46.11968994140625,
      "activations/layer15_attention_weight_max": 40.758392333984375,
      "activations/layer15_attention_weight_min": -29.47759437561035,
      "activations/layer16_attention_weight_max": 38.69761657714844,
      "activations/layer16_attention_weight_min": -26.778644561767578,
      "activations/layer17_attention_weight_max": 35.309173583984375,
      "activations/layer17_attention_weight_min": -25.36759376525879,
      "activations/layer18_attention_weight_max": 35.54335403442383,
      "activations/layer18_attention_weight_min": -21.037946701049805,
      "activations/layer19_attention_weight_max": 42.54285430908203,
      "activations/layer19_attention_weight_min": -28.859983444213867,
      "activations/layer1_attention_weight_max": 17.603912353515625,
      "activations/layer1_attention_weight_min": -17.137861251831055,
      "activations/layer20_attention_weight_max": 34.02913284301758,
      "activations/layer20_attention_weight_min": -23.503938674926758,
      "activations/layer21_attention_weight_max": 34.68571472167969,
      "activations/layer21_attention_weight_min": -23.168216705322266,
      "activations/layer22_attention_weight_max": 50.047969818115234,
      "activations/layer22_attention_weight_min": -31.90966033935547,
      "activations/layer23_attention_weight_max": 31.622821807861328,
      "activations/layer23_attention_weight_min": -22.28390121459961,
      "activations/layer2_attention_weight_max": 30.773685455322266,
      "activations/layer2_attention_weight_min": -29.877050399780273,
      "activations/layer3_attention_weight_max": 87.85486602783203,
      "activations/layer3_attention_weight_min": -85.5072021484375,
      "activations/layer4_attention_weight_max": 130.28721618652344,
      "activations/layer4_attention_weight_min": -127.0959243774414,
      "activations/layer5_attention_weight_max": 83.91661834716797,
      "activations/layer5_attention_weight_min": -80.22491455078125,
      "activations/layer6_attention_weight_max": 54.80132293701172,
      "activations/layer6_attention_weight_min": -53.61030578613281,
      "activations/layer7_attention_weight_max": 79.08985137939453,
      "activations/layer7_attention_weight_min": -78.54878234863281,
      "activations/layer8_attention_weight_max": 35.58524703979492,
      "activations/layer8_attention_weight_min": -36.957923889160156,
      "activations/layer9_attention_weight_max": 31.960214614868164,
      "activations/layer9_attention_weight_min": -35.401512145996094,
      "epoch": 4.62,
      "learning_rate": 0.00012141287878787877,
      "loss": 2.859,
      "step": 79550
    },
    {
      "activations/layer0_attention_weight_max": 15.867576599121094,
      "activations/layer0_attention_weight_min": -12.184097290039062,
      "activations/layer10_attention_weight_max": 33.26694869995117,
      "activations/layer10_attention_weight_min": -29.48099136352539,
      "activations/layer11_attention_weight_max": 30.876659393310547,
      "activations/layer11_attention_weight_min": -27.370967864990234,
      "activations/layer12_attention_weight_max": 24.353967666625977,
      "activations/layer12_attention_weight_min": -25.158411026000977,
      "activations/layer13_attention_weight_max": 41.800933837890625,
      "activations/layer13_attention_weight_min": -32.40102767944336,
      "activations/layer14_attention_weight_max": 57.35672378540039,
      "activations/layer14_attention_weight_min": -47.24134063720703,
      "activations/layer15_attention_weight_max": 39.73707580566406,
      "activations/layer15_attention_weight_min": -29.885995864868164,
      "activations/layer16_attention_weight_max": 35.661834716796875,
      "activations/layer16_attention_weight_min": -28.49893569946289,
      "activations/layer17_attention_weight_max": 38.28181076049805,
      "activations/layer17_attention_weight_min": -26.286645889282227,
      "activations/layer18_attention_weight_max": 30.07550048828125,
      "activations/layer18_attention_weight_min": -21.72909164428711,
      "activations/layer19_attention_weight_max": 38.02288055419922,
      "activations/layer19_attention_weight_min": -28.427597045898438,
      "activations/layer1_attention_weight_max": 17.2285213470459,
      "activations/layer1_attention_weight_min": -14.69167423248291,
      "activations/layer20_attention_weight_max": 37.03762435913086,
      "activations/layer20_attention_weight_min": -24.694730758666992,
      "activations/layer21_attention_weight_max": 35.55070877075195,
      "activations/layer21_attention_weight_min": -21.29331398010254,
      "activations/layer22_attention_weight_max": 53.26396560668945,
      "activations/layer22_attention_weight_min": -27.63072395324707,
      "activations/layer23_attention_weight_max": 30.47210693359375,
      "activations/layer23_attention_weight_min": -20.519346237182617,
      "activations/layer2_attention_weight_max": 31.487659454345703,
      "activations/layer2_attention_weight_min": -29.730010986328125,
      "activations/layer3_attention_weight_max": 88.8721923828125,
      "activations/layer3_attention_weight_min": -90.93816375732422,
      "activations/layer4_attention_weight_max": 127.89111328125,
      "activations/layer4_attention_weight_min": -129.88584899902344,
      "activations/layer5_attention_weight_max": 82.95943450927734,
      "activations/layer5_attention_weight_min": -80.74526977539062,
      "activations/layer6_attention_weight_max": 55.99710464477539,
      "activations/layer6_attention_weight_min": -56.08058166503906,
      "activations/layer7_attention_weight_max": 83.00399780273438,
      "activations/layer7_attention_weight_min": -76.4200210571289,
      "activations/layer8_attention_weight_max": 38.903133392333984,
      "activations/layer8_attention_weight_min": -36.974159240722656,
      "activations/layer9_attention_weight_max": 40.24040603637695,
      "activations/layer9_attention_weight_min": -35.782283782958984,
      "epoch": 4.63,
      "learning_rate": 0.00012139393939393938,
      "loss": 2.8739,
      "step": 79600
    },
    {
      "activations/layer0_attention_weight_max": 17.40039825439453,
      "activations/layer0_attention_weight_min": -12.32666015625,
      "activations/layer10_attention_weight_max": 36.72804260253906,
      "activations/layer10_attention_weight_min": -29.64914894104004,
      "activations/layer11_attention_weight_max": 33.301151275634766,
      "activations/layer11_attention_weight_min": -26.875450134277344,
      "activations/layer12_attention_weight_max": 26.71360206604004,
      "activations/layer12_attention_weight_min": -29.449018478393555,
      "activations/layer13_attention_weight_max": 44.22415542602539,
      "activations/layer13_attention_weight_min": -31.8510799407959,
      "activations/layer14_attention_weight_max": 56.88239669799805,
      "activations/layer14_attention_weight_min": -44.62345504760742,
      "activations/layer15_attention_weight_max": 39.45635223388672,
      "activations/layer15_attention_weight_min": -28.14761734008789,
      "activations/layer16_attention_weight_max": 42.652503967285156,
      "activations/layer16_attention_weight_min": -27.654495239257812,
      "activations/layer17_attention_weight_max": 41.447059631347656,
      "activations/layer17_attention_weight_min": -24.249835968017578,
      "activations/layer18_attention_weight_max": 31.75596809387207,
      "activations/layer18_attention_weight_min": -23.18046760559082,
      "activations/layer19_attention_weight_max": 42.34501266479492,
      "activations/layer19_attention_weight_min": -27.66867446899414,
      "activations/layer1_attention_weight_max": 16.663490295410156,
      "activations/layer1_attention_weight_min": -15.006122589111328,
      "activations/layer20_attention_weight_max": 33.384342193603516,
      "activations/layer20_attention_weight_min": -22.802757263183594,
      "activations/layer21_attention_weight_max": 30.02518653869629,
      "activations/layer21_attention_weight_min": -20.612459182739258,
      "activations/layer22_attention_weight_max": 49.66887664794922,
      "activations/layer22_attention_weight_min": -27.705366134643555,
      "activations/layer23_attention_weight_max": 37.721092224121094,
      "activations/layer23_attention_weight_min": -19.986831665039062,
      "activations/layer2_attention_weight_max": 33.68128204345703,
      "activations/layer2_attention_weight_min": -32.84425354003906,
      "activations/layer3_attention_weight_max": 93.89849090576172,
      "activations/layer3_attention_weight_min": -94.54399108886719,
      "activations/layer4_attention_weight_max": 139.91456604003906,
      "activations/layer4_attention_weight_min": -134.12338256835938,
      "activations/layer5_attention_weight_max": 95.21592712402344,
      "activations/layer5_attention_weight_min": -84.98007202148438,
      "activations/layer6_attention_weight_max": 60.26966094970703,
      "activations/layer6_attention_weight_min": -61.75809860229492,
      "activations/layer7_attention_weight_max": 83.42037963867188,
      "activations/layer7_attention_weight_min": -86.31505584716797,
      "activations/layer8_attention_weight_max": 42.13749694824219,
      "activations/layer8_attention_weight_min": -40.15397262573242,
      "activations/layer9_attention_weight_max": 38.49205780029297,
      "activations/layer9_attention_weight_min": -34.25382614135742,
      "epoch": 4.63,
      "learning_rate": 0.000121375,
      "loss": 2.8511,
      "step": 79650
    },
    {
      "activations/layer0_attention_weight_max": 17.4094181060791,
      "activations/layer0_attention_weight_min": -12.52429485321045,
      "activations/layer10_attention_weight_max": 36.29168701171875,
      "activations/layer10_attention_weight_min": -31.659393310546875,
      "activations/layer11_attention_weight_max": 33.04719543457031,
      "activations/layer11_attention_weight_min": -28.729122161865234,
      "activations/layer12_attention_weight_max": 25.30170249938965,
      "activations/layer12_attention_weight_min": -31.6843204498291,
      "activations/layer13_attention_weight_max": 49.486717224121094,
      "activations/layer13_attention_weight_min": -36.34545135498047,
      "activations/layer14_attention_weight_max": 63.92414093017578,
      "activations/layer14_attention_weight_min": -47.02583312988281,
      "activations/layer15_attention_weight_max": 40.38748550415039,
      "activations/layer15_attention_weight_min": -29.44051170349121,
      "activations/layer16_attention_weight_max": 38.64385986328125,
      "activations/layer16_attention_weight_min": -28.40638542175293,
      "activations/layer17_attention_weight_max": 38.04037857055664,
      "activations/layer17_attention_weight_min": -24.72393035888672,
      "activations/layer18_attention_weight_max": 27.750417709350586,
      "activations/layer18_attention_weight_min": -19.260717391967773,
      "activations/layer19_attention_weight_max": 42.05427932739258,
      "activations/layer19_attention_weight_min": -29.88919448852539,
      "activations/layer1_attention_weight_max": 17.420785903930664,
      "activations/layer1_attention_weight_min": -16.274290084838867,
      "activations/layer20_attention_weight_max": 35.463340759277344,
      "activations/layer20_attention_weight_min": -22.980854034423828,
      "activations/layer21_attention_weight_max": 33.450965881347656,
      "activations/layer21_attention_weight_min": -21.484970092773438,
      "activations/layer22_attention_weight_max": 46.097923278808594,
      "activations/layer22_attention_weight_min": -28.409963607788086,
      "activations/layer23_attention_weight_max": 33.781646728515625,
      "activations/layer23_attention_weight_min": -19.905683517456055,
      "activations/layer2_attention_weight_max": 33.07151794433594,
      "activations/layer2_attention_weight_min": -31.961095809936523,
      "activations/layer3_attention_weight_max": 89.14266967773438,
      "activations/layer3_attention_weight_min": -93.24869537353516,
      "activations/layer4_attention_weight_max": 132.84164428710938,
      "activations/layer4_attention_weight_min": -131.07162475585938,
      "activations/layer5_attention_weight_max": 83.43707275390625,
      "activations/layer5_attention_weight_min": -83.30925750732422,
      "activations/layer6_attention_weight_max": 56.04368209838867,
      "activations/layer6_attention_weight_min": -57.24269104003906,
      "activations/layer7_attention_weight_max": 83.41079711914062,
      "activations/layer7_attention_weight_min": -87.51805877685547,
      "activations/layer8_attention_weight_max": 39.933189392089844,
      "activations/layer8_attention_weight_min": -39.68406295776367,
      "activations/layer9_attention_weight_max": 38.566993713378906,
      "activations/layer9_attention_weight_min": -33.85309600830078,
      "epoch": 4.63,
      "learning_rate": 0.00012135606060606058,
      "loss": 2.8758,
      "step": 79700
    },
    {
      "activations/layer0_attention_weight_max": 16.9475154876709,
      "activations/layer0_attention_weight_min": -13.394674301147461,
      "activations/layer10_attention_weight_max": 33.5623779296875,
      "activations/layer10_attention_weight_min": -29.968406677246094,
      "activations/layer11_attention_weight_max": 31.795995712280273,
      "activations/layer11_attention_weight_min": -25.970640182495117,
      "activations/layer12_attention_weight_max": 43.66002655029297,
      "activations/layer12_attention_weight_min": -28.851343154907227,
      "activations/layer13_attention_weight_max": 43.191165924072266,
      "activations/layer13_attention_weight_min": -33.06244659423828,
      "activations/layer14_attention_weight_max": 57.52582931518555,
      "activations/layer14_attention_weight_min": -40.956668853759766,
      "activations/layer15_attention_weight_max": 40.265533447265625,
      "activations/layer15_attention_weight_min": -27.613452911376953,
      "activations/layer16_attention_weight_max": 35.775146484375,
      "activations/layer16_attention_weight_min": -27.12934112548828,
      "activations/layer17_attention_weight_max": 37.7067985534668,
      "activations/layer17_attention_weight_min": -22.893333435058594,
      "activations/layer18_attention_weight_max": 38.739418029785156,
      "activations/layer18_attention_weight_min": -19.42205047607422,
      "activations/layer19_attention_weight_max": 38.668495178222656,
      "activations/layer19_attention_weight_min": -28.21339225769043,
      "activations/layer1_attention_weight_max": 18.447996139526367,
      "activations/layer1_attention_weight_min": -15.805776596069336,
      "activations/layer20_attention_weight_max": 35.21426773071289,
      "activations/layer20_attention_weight_min": -22.581640243530273,
      "activations/layer21_attention_weight_max": 37.12615203857422,
      "activations/layer21_attention_weight_min": -21.103351593017578,
      "activations/layer22_attention_weight_max": 46.49660873413086,
      "activations/layer22_attention_weight_min": -28.278913497924805,
      "activations/layer23_attention_weight_max": 33.347816467285156,
      "activations/layer23_attention_weight_min": -21.858566284179688,
      "activations/layer2_attention_weight_max": 33.88627624511719,
      "activations/layer2_attention_weight_min": -32.378875732421875,
      "activations/layer3_attention_weight_max": 92.44377136230469,
      "activations/layer3_attention_weight_min": -96.62493133544922,
      "activations/layer4_attention_weight_max": 133.82859802246094,
      "activations/layer4_attention_weight_min": -136.4363555908203,
      "activations/layer5_attention_weight_max": 85.109619140625,
      "activations/layer5_attention_weight_min": -81.76911926269531,
      "activations/layer6_attention_weight_max": 55.387969970703125,
      "activations/layer6_attention_weight_min": -58.40181350708008,
      "activations/layer7_attention_weight_max": 84.73773956298828,
      "activations/layer7_attention_weight_min": -81.8181381225586,
      "activations/layer8_attention_weight_max": 40.08951950073242,
      "activations/layer8_attention_weight_min": -41.01124954223633,
      "activations/layer9_attention_weight_max": 35.009456634521484,
      "activations/layer9_attention_weight_min": -34.0863151550293,
      "epoch": 4.63,
      "learning_rate": 0.0001213371212121212,
      "loss": 2.8724,
      "step": 79750
    },
    {
      "activations/layer0_attention_weight_max": 16.610397338867188,
      "activations/layer0_attention_weight_min": -11.916095733642578,
      "activations/layer10_attention_weight_max": 32.54946517944336,
      "activations/layer10_attention_weight_min": -29.66937828063965,
      "activations/layer11_attention_weight_max": 34.290000915527344,
      "activations/layer11_attention_weight_min": -28.085586547851562,
      "activations/layer12_attention_weight_max": 55.39228820800781,
      "activations/layer12_attention_weight_min": -28.619224548339844,
      "activations/layer13_attention_weight_max": 43.89194107055664,
      "activations/layer13_attention_weight_min": -30.899728775024414,
      "activations/layer14_attention_weight_max": 69.18836975097656,
      "activations/layer14_attention_weight_min": -47.777408599853516,
      "activations/layer15_attention_weight_max": 37.87649917602539,
      "activations/layer15_attention_weight_min": -26.163070678710938,
      "activations/layer16_attention_weight_max": 40.10132598876953,
      "activations/layer16_attention_weight_min": -30.074424743652344,
      "activations/layer17_attention_weight_max": 38.59709548950195,
      "activations/layer17_attention_weight_min": -23.407209396362305,
      "activations/layer18_attention_weight_max": 29.556827545166016,
      "activations/layer18_attention_weight_min": -19.57853126525879,
      "activations/layer19_attention_weight_max": 41.57876968383789,
      "activations/layer19_attention_weight_min": -29.639001846313477,
      "activations/layer1_attention_weight_max": 16.79707145690918,
      "activations/layer1_attention_weight_min": -14.425854682922363,
      "activations/layer20_attention_weight_max": 33.329524993896484,
      "activations/layer20_attention_weight_min": -22.987014770507812,
      "activations/layer21_attention_weight_max": 39.87514877319336,
      "activations/layer21_attention_weight_min": -19.026060104370117,
      "activations/layer22_attention_weight_max": 48.76102066040039,
      "activations/layer22_attention_weight_min": -28.03988265991211,
      "activations/layer23_attention_weight_max": 30.987407684326172,
      "activations/layer23_attention_weight_min": -22.003704071044922,
      "activations/layer2_attention_weight_max": 34.52241516113281,
      "activations/layer2_attention_weight_min": -30.51105308532715,
      "activations/layer3_attention_weight_max": 88.95111846923828,
      "activations/layer3_attention_weight_min": -88.81201171875,
      "activations/layer4_attention_weight_max": 125.46416473388672,
      "activations/layer4_attention_weight_min": -127.53504943847656,
      "activations/layer5_attention_weight_max": 82.90121459960938,
      "activations/layer5_attention_weight_min": -81.36923217773438,
      "activations/layer6_attention_weight_max": 55.826560974121094,
      "activations/layer6_attention_weight_min": -56.495914459228516,
      "activations/layer7_attention_weight_max": 79.07081604003906,
      "activations/layer7_attention_weight_min": -80.38247680664062,
      "activations/layer8_attention_weight_max": 39.7556037902832,
      "activations/layer8_attention_weight_min": -39.94069290161133,
      "activations/layer9_attention_weight_max": 37.285369873046875,
      "activations/layer9_attention_weight_min": -33.977989196777344,
      "epoch": 4.64,
      "learning_rate": 0.00012131818181818181,
      "loss": 2.867,
      "step": 79800
    },
    {
      "activations/layer0_attention_weight_max": 17.44643211364746,
      "activations/layer0_attention_weight_min": -13.048604965209961,
      "activations/layer10_attention_weight_max": 33.47677230834961,
      "activations/layer10_attention_weight_min": -33.376808166503906,
      "activations/layer11_attention_weight_max": 32.10528564453125,
      "activations/layer11_attention_weight_min": -27.519437789916992,
      "activations/layer12_attention_weight_max": 33.23131561279297,
      "activations/layer12_attention_weight_min": -25.126026153564453,
      "activations/layer13_attention_weight_max": 42.62428665161133,
      "activations/layer13_attention_weight_min": -32.3855094909668,
      "activations/layer14_attention_weight_max": 68.29129791259766,
      "activations/layer14_attention_weight_min": -45.920677185058594,
      "activations/layer15_attention_weight_max": 50.3489875793457,
      "activations/layer15_attention_weight_min": -26.996017456054688,
      "activations/layer16_attention_weight_max": 40.90947723388672,
      "activations/layer16_attention_weight_min": -28.031822204589844,
      "activations/layer17_attention_weight_max": 44.4354133605957,
      "activations/layer17_attention_weight_min": -23.072603225708008,
      "activations/layer18_attention_weight_max": 37.22502517700195,
      "activations/layer18_attention_weight_min": -20.263761520385742,
      "activations/layer19_attention_weight_max": 43.19480514526367,
      "activations/layer19_attention_weight_min": -27.887195587158203,
      "activations/layer1_attention_weight_max": 17.142444610595703,
      "activations/layer1_attention_weight_min": -13.6917724609375,
      "activations/layer20_attention_weight_max": 37.20232391357422,
      "activations/layer20_attention_weight_min": -24.812297821044922,
      "activations/layer21_attention_weight_max": 35.563270568847656,
      "activations/layer21_attention_weight_min": -20.572263717651367,
      "activations/layer22_attention_weight_max": 58.26188278198242,
      "activations/layer22_attention_weight_min": -26.104585647583008,
      "activations/layer23_attention_weight_max": 35.921875,
      "activations/layer23_attention_weight_min": -18.238929748535156,
      "activations/layer2_attention_weight_max": 32.12527084350586,
      "activations/layer2_attention_weight_min": -32.683311462402344,
      "activations/layer3_attention_weight_max": 90.99295806884766,
      "activations/layer3_attention_weight_min": -97.70366668701172,
      "activations/layer4_attention_weight_max": 136.1376190185547,
      "activations/layer4_attention_weight_min": -136.5648651123047,
      "activations/layer5_attention_weight_max": 87.36760711669922,
      "activations/layer5_attention_weight_min": -85.18002319335938,
      "activations/layer6_attention_weight_max": 57.943389892578125,
      "activations/layer6_attention_weight_min": -61.07566452026367,
      "activations/layer7_attention_weight_max": 83.49614715576172,
      "activations/layer7_attention_weight_min": -81.33756256103516,
      "activations/layer8_attention_weight_max": 38.33065414428711,
      "activations/layer8_attention_weight_min": -40.95368957519531,
      "activations/layer9_attention_weight_max": 34.58545684814453,
      "activations/layer9_attention_weight_min": -37.64576721191406,
      "epoch": 4.64,
      "learning_rate": 0.00012129924242424242,
      "loss": 2.8626,
      "step": 79850
    },
    {
      "activations/layer0_attention_weight_max": 16.361398696899414,
      "activations/layer0_attention_weight_min": -11.300058364868164,
      "activations/layer10_attention_weight_max": 42.341941833496094,
      "activations/layer10_attention_weight_min": -32.75390625,
      "activations/layer11_attention_weight_max": 42.369293212890625,
      "activations/layer11_attention_weight_min": -30.034507751464844,
      "activations/layer12_attention_weight_max": 50.58815002441406,
      "activations/layer12_attention_weight_min": -27.72467613220215,
      "activations/layer13_attention_weight_max": 64.31866455078125,
      "activations/layer13_attention_weight_min": -42.16447448730469,
      "activations/layer14_attention_weight_max": 62.97309494018555,
      "activations/layer14_attention_weight_min": -42.91999435424805,
      "activations/layer15_attention_weight_max": 42.54169464111328,
      "activations/layer15_attention_weight_min": -29.51270294189453,
      "activations/layer16_attention_weight_max": 41.48668670654297,
      "activations/layer16_attention_weight_min": -26.53050422668457,
      "activations/layer17_attention_weight_max": 46.48418045043945,
      "activations/layer17_attention_weight_min": -23.50980567932129,
      "activations/layer18_attention_weight_max": 31.49701499938965,
      "activations/layer18_attention_weight_min": -21.141176223754883,
      "activations/layer19_attention_weight_max": 42.73947525024414,
      "activations/layer19_attention_weight_min": -27.661657333374023,
      "activations/layer1_attention_weight_max": 19.207611083984375,
      "activations/layer1_attention_weight_min": -15.590170860290527,
      "activations/layer20_attention_weight_max": 32.36396026611328,
      "activations/layer20_attention_weight_min": -22.7561092376709,
      "activations/layer21_attention_weight_max": 30.29872703552246,
      "activations/layer21_attention_weight_min": -21.83517837524414,
      "activations/layer22_attention_weight_max": 55.69324493408203,
      "activations/layer22_attention_weight_min": -26.93929100036621,
      "activations/layer23_attention_weight_max": 35.06132507324219,
      "activations/layer23_attention_weight_min": -18.839221954345703,
      "activations/layer2_attention_weight_max": 33.22029113769531,
      "activations/layer2_attention_weight_min": -31.65334701538086,
      "activations/layer3_attention_weight_max": 90.63921356201172,
      "activations/layer3_attention_weight_min": -92.39996337890625,
      "activations/layer4_attention_weight_max": 132.20663452148438,
      "activations/layer4_attention_weight_min": -130.2668914794922,
      "activations/layer5_attention_weight_max": 86.74068450927734,
      "activations/layer5_attention_weight_min": -83.93801879882812,
      "activations/layer6_attention_weight_max": 60.173728942871094,
      "activations/layer6_attention_weight_min": -56.2591438293457,
      "activations/layer7_attention_weight_max": 89.13692474365234,
      "activations/layer7_attention_weight_min": -82.07130432128906,
      "activations/layer8_attention_weight_max": 46.04826736450195,
      "activations/layer8_attention_weight_min": -40.645843505859375,
      "activations/layer9_attention_weight_max": 46.39944076538086,
      "activations/layer9_attention_weight_min": -41.66731262207031,
      "epoch": 4.64,
      "learning_rate": 0.00012128030303030302,
      "loss": 2.8469,
      "step": 79900
    },
    {
      "activations/layer0_attention_weight_max": 18.077102661132812,
      "activations/layer0_attention_weight_min": -13.221113204956055,
      "activations/layer10_attention_weight_max": 33.313453674316406,
      "activations/layer10_attention_weight_min": -29.311553955078125,
      "activations/layer11_attention_weight_max": 34.3671875,
      "activations/layer11_attention_weight_min": -26.655454635620117,
      "activations/layer12_attention_weight_max": 29.743127822875977,
      "activations/layer12_attention_weight_min": -26.302839279174805,
      "activations/layer13_attention_weight_max": 44.39474868774414,
      "activations/layer13_attention_weight_min": -32.068206787109375,
      "activations/layer14_attention_weight_max": 59.652809143066406,
      "activations/layer14_attention_weight_min": -50.263282775878906,
      "activations/layer15_attention_weight_max": 38.66468811035156,
      "activations/layer15_attention_weight_min": -31.308462142944336,
      "activations/layer16_attention_weight_max": 37.60179138183594,
      "activations/layer16_attention_weight_min": -27.269847869873047,
      "activations/layer17_attention_weight_max": 39.6414680480957,
      "activations/layer17_attention_weight_min": -25.18768310546875,
      "activations/layer18_attention_weight_max": 30.188438415527344,
      "activations/layer18_attention_weight_min": -19.360267639160156,
      "activations/layer19_attention_weight_max": 35.62919235229492,
      "activations/layer19_attention_weight_min": -29.67097282409668,
      "activations/layer1_attention_weight_max": 18.212915420532227,
      "activations/layer1_attention_weight_min": -14.51777458190918,
      "activations/layer20_attention_weight_max": 32.22032165527344,
      "activations/layer20_attention_weight_min": -22.36713218688965,
      "activations/layer21_attention_weight_max": 30.26365089416504,
      "activations/layer21_attention_weight_min": -20.620817184448242,
      "activations/layer22_attention_weight_max": 48.44071578979492,
      "activations/layer22_attention_weight_min": -29.14369010925293,
      "activations/layer23_attention_weight_max": 28.102689743041992,
      "activations/layer23_attention_weight_min": -20.318126678466797,
      "activations/layer2_attention_weight_max": 32.04047393798828,
      "activations/layer2_attention_weight_min": -32.141929626464844,
      "activations/layer3_attention_weight_max": 85.6201400756836,
      "activations/layer3_attention_weight_min": -90.76947021484375,
      "activations/layer4_attention_weight_max": 129.33108520507812,
      "activations/layer4_attention_weight_min": -128.67486572265625,
      "activations/layer5_attention_weight_max": 81.06114959716797,
      "activations/layer5_attention_weight_min": -80.57449340820312,
      "activations/layer6_attention_weight_max": 54.99250793457031,
      "activations/layer6_attention_weight_min": -55.26179885864258,
      "activations/layer7_attention_weight_max": 78.69882202148438,
      "activations/layer7_attention_weight_min": -81.42097473144531,
      "activations/layer8_attention_weight_max": 37.35560989379883,
      "activations/layer8_attention_weight_min": -36.9217529296875,
      "activations/layer9_attention_weight_max": 35.81974411010742,
      "activations/layer9_attention_weight_min": -32.43047332763672,
      "epoch": 4.65,
      "learning_rate": 0.00012126136363636362,
      "loss": 2.841,
      "step": 79950
    },
    {
      "activations/layer0_attention_weight_max": 15.76928424835205,
      "activations/layer0_attention_weight_min": -11.781523704528809,
      "activations/layer10_attention_weight_max": 37.60896301269531,
      "activations/layer10_attention_weight_min": -31.155899047851562,
      "activations/layer11_attention_weight_max": 33.07904052734375,
      "activations/layer11_attention_weight_min": -26.935482025146484,
      "activations/layer12_attention_weight_max": 31.62047576904297,
      "activations/layer12_attention_weight_min": -28.140857696533203,
      "activations/layer13_attention_weight_max": 46.55520248413086,
      "activations/layer13_attention_weight_min": -31.99408721923828,
      "activations/layer14_attention_weight_max": 69.28598022460938,
      "activations/layer14_attention_weight_min": -47.75593948364258,
      "activations/layer15_attention_weight_max": 40.21839141845703,
      "activations/layer15_attention_weight_min": -28.020166397094727,
      "activations/layer16_attention_weight_max": 39.021209716796875,
      "activations/layer16_attention_weight_min": -27.30460548400879,
      "activations/layer17_attention_weight_max": 42.1617317199707,
      "activations/layer17_attention_weight_min": -24.742250442504883,
      "activations/layer18_attention_weight_max": 31.452617645263672,
      "activations/layer18_attention_weight_min": -21.266008377075195,
      "activations/layer19_attention_weight_max": 43.38630294799805,
      "activations/layer19_attention_weight_min": -30.75322914123535,
      "activations/layer1_attention_weight_max": 18.497705459594727,
      "activations/layer1_attention_weight_min": -14.484676361083984,
      "activations/layer20_attention_weight_max": 38.277557373046875,
      "activations/layer20_attention_weight_min": -22.676176071166992,
      "activations/layer21_attention_weight_max": 31.720542907714844,
      "activations/layer21_attention_weight_min": -19.789817810058594,
      "activations/layer22_attention_weight_max": 57.13092803955078,
      "activations/layer22_attention_weight_min": -29.045360565185547,
      "activations/layer23_attention_weight_max": 36.25847244262695,
      "activations/layer23_attention_weight_min": -19.313322067260742,
      "activations/layer2_attention_weight_max": 32.876800537109375,
      "activations/layer2_attention_weight_min": -30.3482723236084,
      "activations/layer3_attention_weight_max": 87.40504455566406,
      "activations/layer3_attention_weight_min": -89.5107650756836,
      "activations/layer4_attention_weight_max": 128.88157653808594,
      "activations/layer4_attention_weight_min": -125.7176742553711,
      "activations/layer5_attention_weight_max": 87.02019500732422,
      "activations/layer5_attention_weight_min": -84.80058288574219,
      "activations/layer6_attention_weight_max": 55.19765090942383,
      "activations/layer6_attention_weight_min": -57.946109771728516,
      "activations/layer7_attention_weight_max": 80.26814270019531,
      "activations/layer7_attention_weight_min": -81.62178802490234,
      "activations/layer8_attention_weight_max": 42.248905181884766,
      "activations/layer8_attention_weight_min": -41.252933502197266,
      "activations/layer9_attention_weight_max": 38.8100700378418,
      "activations/layer9_attention_weight_min": -35.21791076660156,
      "epoch": 4.65,
      "learning_rate": 0.00012124242424242423,
      "loss": 2.866,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_loss": 2.80859375,
      "eval_runtime": 8.3746,
      "eval_samples_per_second": 512.74,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_openwebtext_loss": 2.80859375,
      "eval_openwebtext_ppl": 16.58657693697683,
      "eval_openwebtext_runtime": 8.3746,
      "eval_openwebtext_samples_per_second": 512.74,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_wikitext_loss": 3.052734375,
      "eval_wikitext_ppl": 21.173160702117308,
      "eval_wikitext_runtime": 1.9993,
      "eval_wikitext_samples_per_second": 228.078,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_lambada_loss": 2.7265625,
      "eval_lambada_ppl": 15.280270706875589,
      "eval_lambada_runtime": 9.5531,
      "eval_lambada_samples_per_second": 509.68,
      "step": 80000
    },
    {
      "activations/layer0_attention_weight_max": 17.456466674804688,
      "activations/layer0_attention_weight_min": -11.921091079711914,
      "activations/layer10_attention_weight_max": 35.802886962890625,
      "activations/layer10_attention_weight_min": -30.599761962890625,
      "activations/layer11_attention_weight_max": 33.83095169067383,
      "activations/layer11_attention_weight_min": -30.242063522338867,
      "activations/layer12_attention_weight_max": 43.45599365234375,
      "activations/layer12_attention_weight_min": -25.945959091186523,
      "activations/layer13_attention_weight_max": 46.98503875732422,
      "activations/layer13_attention_weight_min": -34.35454559326172,
      "activations/layer14_attention_weight_max": 68.84254455566406,
      "activations/layer14_attention_weight_min": -48.657962799072266,
      "activations/layer15_attention_weight_max": 46.01573181152344,
      "activations/layer15_attention_weight_min": -30.05006217956543,
      "activations/layer16_attention_weight_max": 40.765708923339844,
      "activations/layer16_attention_weight_min": -29.334287643432617,
      "activations/layer17_attention_weight_max": 46.930912017822266,
      "activations/layer17_attention_weight_min": -27.57143783569336,
      "activations/layer18_attention_weight_max": 32.779327392578125,
      "activations/layer18_attention_weight_min": -19.400386810302734,
      "activations/layer19_attention_weight_max": 46.86726760864258,
      "activations/layer19_attention_weight_min": -29.488727569580078,
      "activations/layer1_attention_weight_max": 17.191877365112305,
      "activations/layer1_attention_weight_min": -15.697888374328613,
      "activations/layer20_attention_weight_max": 36.51499938964844,
      "activations/layer20_attention_weight_min": -25.458513259887695,
      "activations/layer21_attention_weight_max": 33.072166442871094,
      "activations/layer21_attention_weight_min": -19.91192054748535,
      "activations/layer22_attention_weight_max": 54.30767822265625,
      "activations/layer22_attention_weight_min": -28.41691780090332,
      "activations/layer23_attention_weight_max": 36.11152267456055,
      "activations/layer23_attention_weight_min": -21.264751434326172,
      "activations/layer2_attention_weight_max": 30.815710067749023,
      "activations/layer2_attention_weight_min": -32.973480224609375,
      "activations/layer3_attention_weight_max": 86.77210235595703,
      "activations/layer3_attention_weight_min": -87.32573699951172,
      "activations/layer4_attention_weight_max": 127.0260238647461,
      "activations/layer4_attention_weight_min": -124.57726287841797,
      "activations/layer5_attention_weight_max": 84.92324829101562,
      "activations/layer5_attention_weight_min": -80.10848999023438,
      "activations/layer6_attention_weight_max": 56.39441680908203,
      "activations/layer6_attention_weight_min": -55.81634521484375,
      "activations/layer7_attention_weight_max": 81.56407165527344,
      "activations/layer7_attention_weight_min": -82.74015045166016,
      "activations/layer8_attention_weight_max": 40.204124450683594,
      "activations/layer8_attention_weight_min": -39.36421585083008,
      "activations/layer9_attention_weight_max": 37.429134368896484,
      "activations/layer9_attention_weight_min": -39.251461029052734,
      "epoch": 4.65,
      "learning_rate": 0.00012122348484848485,
      "loss": 2.8611,
      "step": 80050
    },
    {
      "activations/layer0_attention_weight_max": 15.741047859191895,
      "activations/layer0_attention_weight_min": -12.041836738586426,
      "activations/layer10_attention_weight_max": 36.1512451171875,
      "activations/layer10_attention_weight_min": -29.949546813964844,
      "activations/layer11_attention_weight_max": 33.770835876464844,
      "activations/layer11_attention_weight_min": -27.27606773376465,
      "activations/layer12_attention_weight_max": 44.97087860107422,
      "activations/layer12_attention_weight_min": -26.851015090942383,
      "activations/layer13_attention_weight_max": 44.26079177856445,
      "activations/layer13_attention_weight_min": -30.265443801879883,
      "activations/layer14_attention_weight_max": 63.50261688232422,
      "activations/layer14_attention_weight_min": -42.992225646972656,
      "activations/layer15_attention_weight_max": 42.840797424316406,
      "activations/layer15_attention_weight_min": -26.4725341796875,
      "activations/layer16_attention_weight_max": 35.0391960144043,
      "activations/layer16_attention_weight_min": -25.276575088500977,
      "activations/layer17_attention_weight_max": 35.961341857910156,
      "activations/layer17_attention_weight_min": -22.349836349487305,
      "activations/layer18_attention_weight_max": 33.60485076904297,
      "activations/layer18_attention_weight_min": -20.4359188079834,
      "activations/layer19_attention_weight_max": 37.89713668823242,
      "activations/layer19_attention_weight_min": -25.66707420349121,
      "activations/layer1_attention_weight_max": 17.711219787597656,
      "activations/layer1_attention_weight_min": -15.954825401306152,
      "activations/layer20_attention_weight_max": 32.60830307006836,
      "activations/layer20_attention_weight_min": -21.40873146057129,
      "activations/layer21_attention_weight_max": 33.480403900146484,
      "activations/layer21_attention_weight_min": -20.13265609741211,
      "activations/layer22_attention_weight_max": 49.07917022705078,
      "activations/layer22_attention_weight_min": -26.769941329956055,
      "activations/layer23_attention_weight_max": 31.975643157958984,
      "activations/layer23_attention_weight_min": -21.335208892822266,
      "activations/layer2_attention_weight_max": 33.133731842041016,
      "activations/layer2_attention_weight_min": -29.730655670166016,
      "activations/layer3_attention_weight_max": 89.44055938720703,
      "activations/layer3_attention_weight_min": -87.5909423828125,
      "activations/layer4_attention_weight_max": 131.63186645507812,
      "activations/layer4_attention_weight_min": -126.78529357910156,
      "activations/layer5_attention_weight_max": 83.18562316894531,
      "activations/layer5_attention_weight_min": -81.3849105834961,
      "activations/layer6_attention_weight_max": 54.34382247924805,
      "activations/layer6_attention_weight_min": -59.75825500488281,
      "activations/layer7_attention_weight_max": 82.19583892822266,
      "activations/layer7_attention_weight_min": -79.53858947753906,
      "activations/layer8_attention_weight_max": 39.49110412597656,
      "activations/layer8_attention_weight_min": -38.40763473510742,
      "activations/layer9_attention_weight_max": 36.00642395019531,
      "activations/layer9_attention_weight_min": -32.392459869384766,
      "epoch": 4.65,
      "learning_rate": 0.00012120454545454544,
      "loss": 2.8688,
      "step": 80100
    },
    {
      "activations/layer0_attention_weight_max": 17.011329650878906,
      "activations/layer0_attention_weight_min": -12.323464393615723,
      "activations/layer10_attention_weight_max": 35.23187255859375,
      "activations/layer10_attention_weight_min": -30.747690200805664,
      "activations/layer11_attention_weight_max": 34.037357330322266,
      "activations/layer11_attention_weight_min": -25.800975799560547,
      "activations/layer12_attention_weight_max": 45.55438995361328,
      "activations/layer12_attention_weight_min": -32.26640701293945,
      "activations/layer13_attention_weight_max": 47.44390106201172,
      "activations/layer13_attention_weight_min": -32.005435943603516,
      "activations/layer14_attention_weight_max": 71.62540435791016,
      "activations/layer14_attention_weight_min": -49.04854202270508,
      "activations/layer15_attention_weight_max": 46.841339111328125,
      "activations/layer15_attention_weight_min": -29.632173538208008,
      "activations/layer16_attention_weight_max": 41.351219177246094,
      "activations/layer16_attention_weight_min": -28.104228973388672,
      "activations/layer17_attention_weight_max": 43.98698043823242,
      "activations/layer17_attention_weight_min": -24.357629776000977,
      "activations/layer18_attention_weight_max": 34.68440628051758,
      "activations/layer18_attention_weight_min": -21.45148277282715,
      "activations/layer19_attention_weight_max": 50.670753479003906,
      "activations/layer19_attention_weight_min": -30.298969268798828,
      "activations/layer1_attention_weight_max": 18.77349853515625,
      "activations/layer1_attention_weight_min": -15.386435508728027,
      "activations/layer20_attention_weight_max": 35.610137939453125,
      "activations/layer20_attention_weight_min": -22.952714920043945,
      "activations/layer21_attention_weight_max": 40.05113983154297,
      "activations/layer21_attention_weight_min": -21.050796508789062,
      "activations/layer22_attention_weight_max": 56.56970977783203,
      "activations/layer22_attention_weight_min": -29.02974510192871,
      "activations/layer23_attention_weight_max": 39.12441635131836,
      "activations/layer23_attention_weight_min": -23.28439712524414,
      "activations/layer2_attention_weight_max": 33.82369613647461,
      "activations/layer2_attention_weight_min": -31.13063621520996,
      "activations/layer3_attention_weight_max": 97.01763916015625,
      "activations/layer3_attention_weight_min": -93.41100311279297,
      "activations/layer4_attention_weight_max": 138.25047302246094,
      "activations/layer4_attention_weight_min": -134.56654357910156,
      "activations/layer5_attention_weight_max": 88.5329818725586,
      "activations/layer5_attention_weight_min": -88.5399169921875,
      "activations/layer6_attention_weight_max": 58.34779357910156,
      "activations/layer6_attention_weight_min": -58.852909088134766,
      "activations/layer7_attention_weight_max": 81.62178039550781,
      "activations/layer7_attention_weight_min": -85.0509262084961,
      "activations/layer8_attention_weight_max": 40.87882995605469,
      "activations/layer8_attention_weight_min": -40.54634475708008,
      "activations/layer9_attention_weight_max": 37.5605583190918,
      "activations/layer9_attention_weight_min": -33.761959075927734,
      "epoch": 4.66,
      "learning_rate": 0.00012118560606060605,
      "loss": 2.8766,
      "step": 80150
    },
    {
      "activations/layer0_attention_weight_max": 15.771845817565918,
      "activations/layer0_attention_weight_min": -13.000332832336426,
      "activations/layer10_attention_weight_max": 42.20849609375,
      "activations/layer10_attention_weight_min": -31.859283447265625,
      "activations/layer11_attention_weight_max": 38.418697357177734,
      "activations/layer11_attention_weight_min": -27.237449645996094,
      "activations/layer12_attention_weight_max": 25.17082405090332,
      "activations/layer12_attention_weight_min": -27.9154109954834,
      "activations/layer13_attention_weight_max": 60.46684646606445,
      "activations/layer13_attention_weight_min": -34.23633575439453,
      "activations/layer14_attention_weight_max": 97.12366485595703,
      "activations/layer14_attention_weight_min": -55.972808837890625,
      "activations/layer15_attention_weight_max": 42.76941680908203,
      "activations/layer15_attention_weight_min": -26.606002807617188,
      "activations/layer16_attention_weight_max": 45.0274543762207,
      "activations/layer16_attention_weight_min": -26.54262924194336,
      "activations/layer17_attention_weight_max": 40.75263595581055,
      "activations/layer17_attention_weight_min": -23.957929611206055,
      "activations/layer18_attention_weight_max": 33.77404022216797,
      "activations/layer18_attention_weight_min": -20.164831161499023,
      "activations/layer19_attention_weight_max": 50.31240463256836,
      "activations/layer19_attention_weight_min": -30.70847511291504,
      "activations/layer1_attention_weight_max": 18.082069396972656,
      "activations/layer1_attention_weight_min": -16.51375961303711,
      "activations/layer20_attention_weight_max": 41.999717712402344,
      "activations/layer20_attention_weight_min": -21.616077423095703,
      "activations/layer21_attention_weight_max": 32.10865020751953,
      "activations/layer21_attention_weight_min": -20.02716636657715,
      "activations/layer22_attention_weight_max": 53.468109130859375,
      "activations/layer22_attention_weight_min": -26.882043838500977,
      "activations/layer23_attention_weight_max": 32.37016677856445,
      "activations/layer23_attention_weight_min": -19.634069442749023,
      "activations/layer2_attention_weight_max": 35.517417907714844,
      "activations/layer2_attention_weight_min": -33.25205993652344,
      "activations/layer3_attention_weight_max": 98.56742095947266,
      "activations/layer3_attention_weight_min": -96.05021667480469,
      "activations/layer4_attention_weight_max": 160.64439392089844,
      "activations/layer4_attention_weight_min": -139.04808044433594,
      "activations/layer5_attention_weight_max": 97.43807983398438,
      "activations/layer5_attention_weight_min": -88.39799499511719,
      "activations/layer6_attention_weight_max": 63.15696716308594,
      "activations/layer6_attention_weight_min": -58.20623016357422,
      "activations/layer7_attention_weight_max": 98.2075424194336,
      "activations/layer7_attention_weight_min": -88.57582092285156,
      "activations/layer8_attention_weight_max": 49.604007720947266,
      "activations/layer8_attention_weight_min": -43.90168762207031,
      "activations/layer9_attention_weight_max": 42.986629486083984,
      "activations/layer9_attention_weight_min": -44.7408332824707,
      "epoch": 4.66,
      "learning_rate": 0.00012116666666666667,
      "loss": 2.8507,
      "step": 80200
    },
    {
      "activations/layer0_attention_weight_max": 16.778940200805664,
      "activations/layer0_attention_weight_min": -11.533990859985352,
      "activations/layer10_attention_weight_max": 35.101348876953125,
      "activations/layer10_attention_weight_min": -32.34645462036133,
      "activations/layer11_attention_weight_max": 32.98250198364258,
      "activations/layer11_attention_weight_min": -28.719345092773438,
      "activations/layer12_attention_weight_max": 49.4383659362793,
      "activations/layer12_attention_weight_min": -28.277082443237305,
      "activations/layer13_attention_weight_max": 54.340389251708984,
      "activations/layer13_attention_weight_min": -38.20916748046875,
      "activations/layer14_attention_weight_max": 76.82768249511719,
      "activations/layer14_attention_weight_min": -50.43997573852539,
      "activations/layer15_attention_weight_max": 47.478599548339844,
      "activations/layer15_attention_weight_min": -30.216096878051758,
      "activations/layer16_attention_weight_max": 40.257781982421875,
      "activations/layer16_attention_weight_min": -27.812414169311523,
      "activations/layer17_attention_weight_max": 43.304840087890625,
      "activations/layer17_attention_weight_min": -26.066682815551758,
      "activations/layer18_attention_weight_max": 31.802047729492188,
      "activations/layer18_attention_weight_min": -18.757434844970703,
      "activations/layer19_attention_weight_max": 45.075191497802734,
      "activations/layer19_attention_weight_min": -27.805349349975586,
      "activations/layer1_attention_weight_max": 16.984142303466797,
      "activations/layer1_attention_weight_min": -15.51567554473877,
      "activations/layer20_attention_weight_max": 39.49941635131836,
      "activations/layer20_attention_weight_min": -21.767459869384766,
      "activations/layer21_attention_weight_max": 31.708786010742188,
      "activations/layer21_attention_weight_min": -22.19622039794922,
      "activations/layer22_attention_weight_max": 56.048824310302734,
      "activations/layer22_attention_weight_min": -28.26683235168457,
      "activations/layer23_attention_weight_max": 31.67953109741211,
      "activations/layer23_attention_weight_min": -18.72910499572754,
      "activations/layer2_attention_weight_max": 32.83882522583008,
      "activations/layer2_attention_weight_min": -32.036014556884766,
      "activations/layer3_attention_weight_max": 87.32939910888672,
      "activations/layer3_attention_weight_min": -92.6566162109375,
      "activations/layer4_attention_weight_max": 130.4544677734375,
      "activations/layer4_attention_weight_min": -124.32673645019531,
      "activations/layer5_attention_weight_max": 83.43526458740234,
      "activations/layer5_attention_weight_min": -79.203369140625,
      "activations/layer6_attention_weight_max": 61.419761657714844,
      "activations/layer6_attention_weight_min": -55.745269775390625,
      "activations/layer7_attention_weight_max": 85.40808868408203,
      "activations/layer7_attention_weight_min": -79.17668151855469,
      "activations/layer8_attention_weight_max": 41.32566833496094,
      "activations/layer8_attention_weight_min": -40.4168586730957,
      "activations/layer9_attention_weight_max": 37.79490280151367,
      "activations/layer9_attention_weight_min": -35.71892547607422,
      "epoch": 4.66,
      "learning_rate": 0.00012114772727272725,
      "loss": 2.8605,
      "step": 80250
    },
    {
      "activations/layer0_attention_weight_max": 14.961308479309082,
      "activations/layer0_attention_weight_min": -11.995950698852539,
      "activations/layer10_attention_weight_max": 35.410369873046875,
      "activations/layer10_attention_weight_min": -31.98617935180664,
      "activations/layer11_attention_weight_max": 33.29468536376953,
      "activations/layer11_attention_weight_min": -26.90878677368164,
      "activations/layer12_attention_weight_max": 29.55540657043457,
      "activations/layer12_attention_weight_min": -28.48318862915039,
      "activations/layer13_attention_weight_max": 54.57753372192383,
      "activations/layer13_attention_weight_min": -37.12583923339844,
      "activations/layer14_attention_weight_max": 67.75054168701172,
      "activations/layer14_attention_weight_min": -52.21168518066406,
      "activations/layer15_attention_weight_max": 40.21501159667969,
      "activations/layer15_attention_weight_min": -28.573284149169922,
      "activations/layer16_attention_weight_max": 41.8587646484375,
      "activations/layer16_attention_weight_min": -26.841480255126953,
      "activations/layer17_attention_weight_max": 38.972782135009766,
      "activations/layer17_attention_weight_min": -24.843164443969727,
      "activations/layer18_attention_weight_max": 34.686153411865234,
      "activations/layer18_attention_weight_min": -21.561132431030273,
      "activations/layer19_attention_weight_max": 46.165523529052734,
      "activations/layer19_attention_weight_min": -29.336572647094727,
      "activations/layer1_attention_weight_max": 18.356962203979492,
      "activations/layer1_attention_weight_min": -15.35852336883545,
      "activations/layer20_attention_weight_max": 31.80340576171875,
      "activations/layer20_attention_weight_min": -23.006492614746094,
      "activations/layer21_attention_weight_max": 33.9927978515625,
      "activations/layer21_attention_weight_min": -21.07233238220215,
      "activations/layer22_attention_weight_max": 49.4454231262207,
      "activations/layer22_attention_weight_min": -28.78680992126465,
      "activations/layer23_attention_weight_max": 35.86809539794922,
      "activations/layer23_attention_weight_min": -21.694082260131836,
      "activations/layer2_attention_weight_max": 33.85298538208008,
      "activations/layer2_attention_weight_min": -31.93899917602539,
      "activations/layer3_attention_weight_max": 92.19515228271484,
      "activations/layer3_attention_weight_min": -97.44788360595703,
      "activations/layer4_attention_weight_max": 133.2583465576172,
      "activations/layer4_attention_weight_min": -134.97142028808594,
      "activations/layer5_attention_weight_max": 87.62942504882812,
      "activations/layer5_attention_weight_min": -85.0531234741211,
      "activations/layer6_attention_weight_max": 56.829437255859375,
      "activations/layer6_attention_weight_min": -58.8212776184082,
      "activations/layer7_attention_weight_max": 81.9962158203125,
      "activations/layer7_attention_weight_min": -83.35145568847656,
      "activations/layer8_attention_weight_max": 40.0658073425293,
      "activations/layer8_attention_weight_min": -40.768402099609375,
      "activations/layer9_attention_weight_max": 36.62876892089844,
      "activations/layer9_attention_weight_min": -35.146484375,
      "epoch": 4.67,
      "learning_rate": 0.00012112878787878787,
      "loss": 2.8599,
      "step": 80300
    },
    {
      "activations/layer0_attention_weight_max": 17.244037628173828,
      "activations/layer0_attention_weight_min": -11.680331230163574,
      "activations/layer10_attention_weight_max": 34.72882843017578,
      "activations/layer10_attention_weight_min": -30.67136573791504,
      "activations/layer11_attention_weight_max": 33.27246856689453,
      "activations/layer11_attention_weight_min": -28.522871017456055,
      "activations/layer12_attention_weight_max": 26.55492401123047,
      "activations/layer12_attention_weight_min": -27.46439552307129,
      "activations/layer13_attention_weight_max": 53.290443420410156,
      "activations/layer13_attention_weight_min": -34.00572204589844,
      "activations/layer14_attention_weight_max": 64.01847076416016,
      "activations/layer14_attention_weight_min": -46.12839126586914,
      "activations/layer15_attention_weight_max": 47.51185989379883,
      "activations/layer15_attention_weight_min": -27.486835479736328,
      "activations/layer16_attention_weight_max": 38.5518798828125,
      "activations/layer16_attention_weight_min": -26.265138626098633,
      "activations/layer17_attention_weight_max": 42.14337158203125,
      "activations/layer17_attention_weight_min": -25.999441146850586,
      "activations/layer18_attention_weight_max": 33.20960235595703,
      "activations/layer18_attention_weight_min": -22.619556427001953,
      "activations/layer19_attention_weight_max": 41.78653335571289,
      "activations/layer19_attention_weight_min": -26.70448112487793,
      "activations/layer1_attention_weight_max": 16.607440948486328,
      "activations/layer1_attention_weight_min": -13.96707534790039,
      "activations/layer20_attention_weight_max": 32.741607666015625,
      "activations/layer20_attention_weight_min": -22.087976455688477,
      "activations/layer21_attention_weight_max": 31.446529388427734,
      "activations/layer21_attention_weight_min": -19.16326141357422,
      "activations/layer22_attention_weight_max": 49.8565673828125,
      "activations/layer22_attention_weight_min": -26.529104232788086,
      "activations/layer23_attention_weight_max": 29.795690536499023,
      "activations/layer23_attention_weight_min": -18.25893783569336,
      "activations/layer2_attention_weight_max": 31.01926040649414,
      "activations/layer2_attention_weight_min": -31.35500717163086,
      "activations/layer3_attention_weight_max": 87.25590515136719,
      "activations/layer3_attention_weight_min": -90.04457092285156,
      "activations/layer4_attention_weight_max": 131.94528198242188,
      "activations/layer4_attention_weight_min": -134.52133178710938,
      "activations/layer5_attention_weight_max": 90.48351287841797,
      "activations/layer5_attention_weight_min": -86.30159759521484,
      "activations/layer6_attention_weight_max": 59.163822174072266,
      "activations/layer6_attention_weight_min": -60.50932312011719,
      "activations/layer7_attention_weight_max": 79.88534545898438,
      "activations/layer7_attention_weight_min": -81.03456115722656,
      "activations/layer8_attention_weight_max": 45.29658126831055,
      "activations/layer8_attention_weight_min": -43.25505065917969,
      "activations/layer9_attention_weight_max": 38.20528030395508,
      "activations/layer9_attention_weight_min": -38.77779006958008,
      "epoch": 4.67,
      "learning_rate": 0.00012110984848484848,
      "loss": 2.8646,
      "step": 80350
    },
    {
      "activations/layer0_attention_weight_max": 16.210281372070312,
      "activations/layer0_attention_weight_min": -12.946044921875,
      "activations/layer10_attention_weight_max": 36.576690673828125,
      "activations/layer10_attention_weight_min": -30.761831283569336,
      "activations/layer11_attention_weight_max": 33.66083908081055,
      "activations/layer11_attention_weight_min": -27.65221405029297,
      "activations/layer12_attention_weight_max": 28.756059646606445,
      "activations/layer12_attention_weight_min": -26.848337173461914,
      "activations/layer13_attention_weight_max": 48.5031623840332,
      "activations/layer13_attention_weight_min": -36.93235778808594,
      "activations/layer14_attention_weight_max": 61.871707916259766,
      "activations/layer14_attention_weight_min": -44.64839553833008,
      "activations/layer15_attention_weight_max": 42.09938049316406,
      "activations/layer15_attention_weight_min": -29.58807373046875,
      "activations/layer16_attention_weight_max": 40.0963249206543,
      "activations/layer16_attention_weight_min": -27.89850616455078,
      "activations/layer17_attention_weight_max": 39.45310974121094,
      "activations/layer17_attention_weight_min": -25.744678497314453,
      "activations/layer18_attention_weight_max": 32.90563201904297,
      "activations/layer18_attention_weight_min": -20.391117095947266,
      "activations/layer19_attention_weight_max": 43.5196533203125,
      "activations/layer19_attention_weight_min": -29.836393356323242,
      "activations/layer1_attention_weight_max": 17.734432220458984,
      "activations/layer1_attention_weight_min": -15.02070426940918,
      "activations/layer20_attention_weight_max": 33.731597900390625,
      "activations/layer20_attention_weight_min": -23.529560089111328,
      "activations/layer21_attention_weight_max": 31.112621307373047,
      "activations/layer21_attention_weight_min": -19.796859741210938,
      "activations/layer22_attention_weight_max": 52.645843505859375,
      "activations/layer22_attention_weight_min": -30.699153900146484,
      "activations/layer23_attention_weight_max": 32.90074920654297,
      "activations/layer23_attention_weight_min": -19.497318267822266,
      "activations/layer2_attention_weight_max": 31.19839096069336,
      "activations/layer2_attention_weight_min": -31.93617057800293,
      "activations/layer3_attention_weight_max": 90.25650024414062,
      "activations/layer3_attention_weight_min": -91.75870513916016,
      "activations/layer4_attention_weight_max": 131.0554656982422,
      "activations/layer4_attention_weight_min": -127.95157623291016,
      "activations/layer5_attention_weight_max": 84.37114715576172,
      "activations/layer5_attention_weight_min": -83.78275299072266,
      "activations/layer6_attention_weight_max": 60.19799041748047,
      "activations/layer6_attention_weight_min": -57.67557907104492,
      "activations/layer7_attention_weight_max": 80.09432983398438,
      "activations/layer7_attention_weight_min": -81.81629180908203,
      "activations/layer8_attention_weight_max": 41.273048400878906,
      "activations/layer8_attention_weight_min": -40.94880676269531,
      "activations/layer9_attention_weight_max": 36.615692138671875,
      "activations/layer9_attention_weight_min": -33.88513946533203,
      "epoch": 4.67,
      "learning_rate": 0.00012109090909090907,
      "loss": 2.8659,
      "step": 80400
    },
    {
      "activations/layer0_attention_weight_max": 16.327003479003906,
      "activations/layer0_attention_weight_min": -11.525110244750977,
      "activations/layer10_attention_weight_max": 35.6055908203125,
      "activations/layer10_attention_weight_min": -30.781431198120117,
      "activations/layer11_attention_weight_max": 34.854427337646484,
      "activations/layer11_attention_weight_min": -27.029102325439453,
      "activations/layer12_attention_weight_max": 33.926780700683594,
      "activations/layer12_attention_weight_min": -29.381874084472656,
      "activations/layer13_attention_weight_max": 46.906002044677734,
      "activations/layer13_attention_weight_min": -34.443355560302734,
      "activations/layer14_attention_weight_max": 66.58617401123047,
      "activations/layer14_attention_weight_min": -45.731117248535156,
      "activations/layer15_attention_weight_max": 44.22911071777344,
      "activations/layer15_attention_weight_min": -28.191864013671875,
      "activations/layer16_attention_weight_max": 50.97954177856445,
      "activations/layer16_attention_weight_min": -30.27863311767578,
      "activations/layer17_attention_weight_max": 44.09949493408203,
      "activations/layer17_attention_weight_min": -25.52706527709961,
      "activations/layer18_attention_weight_max": 39.427955627441406,
      "activations/layer18_attention_weight_min": -22.92276382446289,
      "activations/layer19_attention_weight_max": 49.28712844848633,
      "activations/layer19_attention_weight_min": -30.61411476135254,
      "activations/layer1_attention_weight_max": 17.726469039916992,
      "activations/layer1_attention_weight_min": -14.096390724182129,
      "activations/layer20_attention_weight_max": 37.7694206237793,
      "activations/layer20_attention_weight_min": -24.926183700561523,
      "activations/layer21_attention_weight_max": 36.43379592895508,
      "activations/layer21_attention_weight_min": -22.65761375427246,
      "activations/layer22_attention_weight_max": 56.16146469116211,
      "activations/layer22_attention_weight_min": -29.807777404785156,
      "activations/layer23_attention_weight_max": 34.42670822143555,
      "activations/layer23_attention_weight_min": -21.401744842529297,
      "activations/layer2_attention_weight_max": 32.61076354980469,
      "activations/layer2_attention_weight_min": -31.560623168945312,
      "activations/layer3_attention_weight_max": 88.4378433227539,
      "activations/layer3_attention_weight_min": -92.24175262451172,
      "activations/layer4_attention_weight_max": 131.6436767578125,
      "activations/layer4_attention_weight_min": -130.5527801513672,
      "activations/layer5_attention_weight_max": 83.27081298828125,
      "activations/layer5_attention_weight_min": -84.76443481445312,
      "activations/layer6_attention_weight_max": 56.99104690551758,
      "activations/layer6_attention_weight_min": -58.318302154541016,
      "activations/layer7_attention_weight_max": 80.37245178222656,
      "activations/layer7_attention_weight_min": -84.39820098876953,
      "activations/layer8_attention_weight_max": 38.96009063720703,
      "activations/layer8_attention_weight_min": -41.58598709106445,
      "activations/layer9_attention_weight_max": 36.030757904052734,
      "activations/layer9_attention_weight_min": -35.878116607666016,
      "epoch": 4.67,
      "learning_rate": 0.00012107196969696969,
      "loss": 2.8715,
      "step": 80450
    },
    {
      "activations/layer0_attention_weight_max": 17.833765029907227,
      "activations/layer0_attention_weight_min": -13.20749282836914,
      "activations/layer10_attention_weight_max": 35.62602615356445,
      "activations/layer10_attention_weight_min": -30.17313575744629,
      "activations/layer11_attention_weight_max": 34.904945373535156,
      "activations/layer11_attention_weight_min": -29.198476791381836,
      "activations/layer12_attention_weight_max": 33.64414596557617,
      "activations/layer12_attention_weight_min": -29.345705032348633,
      "activations/layer13_attention_weight_max": 44.39663314819336,
      "activations/layer13_attention_weight_min": -31.160268783569336,
      "activations/layer14_attention_weight_max": 60.738651275634766,
      "activations/layer14_attention_weight_min": -46.39345169067383,
      "activations/layer15_attention_weight_max": 36.65967559814453,
      "activations/layer15_attention_weight_min": -27.341426849365234,
      "activations/layer16_attention_weight_max": 36.54819869995117,
      "activations/layer16_attention_weight_min": -28.554672241210938,
      "activations/layer17_attention_weight_max": 35.32426834106445,
      "activations/layer17_attention_weight_min": -24.847896575927734,
      "activations/layer18_attention_weight_max": 32.1834831237793,
      "activations/layer18_attention_weight_min": -21.010528564453125,
      "activations/layer19_attention_weight_max": 39.236690521240234,
      "activations/layer19_attention_weight_min": -30.103506088256836,
      "activations/layer1_attention_weight_max": 18.12545394897461,
      "activations/layer1_attention_weight_min": -15.369817733764648,
      "activations/layer20_attention_weight_max": 29.933671951293945,
      "activations/layer20_attention_weight_min": -24.247669219970703,
      "activations/layer21_attention_weight_max": 37.616180419921875,
      "activations/layer21_attention_weight_min": -21.641578674316406,
      "activations/layer22_attention_weight_max": 45.83613586425781,
      "activations/layer22_attention_weight_min": -28.001264572143555,
      "activations/layer23_attention_weight_max": 32.708919525146484,
      "activations/layer23_attention_weight_min": -19.069849014282227,
      "activations/layer2_attention_weight_max": 31.93110466003418,
      "activations/layer2_attention_weight_min": -32.19713592529297,
      "activations/layer3_attention_weight_max": 85.09872436523438,
      "activations/layer3_attention_weight_min": -92.12008666992188,
      "activations/layer4_attention_weight_max": 125.9809799194336,
      "activations/layer4_attention_weight_min": -126.22782135009766,
      "activations/layer5_attention_weight_max": 84.69034576416016,
      "activations/layer5_attention_weight_min": -79.50714111328125,
      "activations/layer6_attention_weight_max": 59.60575485229492,
      "activations/layer6_attention_weight_min": -56.3558464050293,
      "activations/layer7_attention_weight_max": 84.8796615600586,
      "activations/layer7_attention_weight_min": -83.98189544677734,
      "activations/layer8_attention_weight_max": 40.82705307006836,
      "activations/layer8_attention_weight_min": -38.59214782714844,
      "activations/layer9_attention_weight_max": 37.39347457885742,
      "activations/layer9_attention_weight_min": -33.80272674560547,
      "epoch": 4.68,
      "learning_rate": 0.00012105303030303029,
      "loss": 2.8672,
      "step": 80500
    },
    {
      "activations/layer0_attention_weight_max": 17.136966705322266,
      "activations/layer0_attention_weight_min": -12.896245002746582,
      "activations/layer10_attention_weight_max": 39.18674087524414,
      "activations/layer10_attention_weight_min": -30.753007888793945,
      "activations/layer11_attention_weight_max": 39.639244079589844,
      "activations/layer11_attention_weight_min": -28.444217681884766,
      "activations/layer12_attention_weight_max": 23.64190101623535,
      "activations/layer12_attention_weight_min": -26.01478385925293,
      "activations/layer13_attention_weight_max": 37.93377685546875,
      "activations/layer13_attention_weight_min": -31.161968231201172,
      "activations/layer14_attention_weight_max": 53.29888153076172,
      "activations/layer14_attention_weight_min": -45.39157485961914,
      "activations/layer15_attention_weight_max": 37.71521759033203,
      "activations/layer15_attention_weight_min": -26.24526023864746,
      "activations/layer16_attention_weight_max": 37.62607192993164,
      "activations/layer16_attention_weight_min": -28.41908073425293,
      "activations/layer17_attention_weight_max": 36.0551872253418,
      "activations/layer17_attention_weight_min": -25.427019119262695,
      "activations/layer18_attention_weight_max": 28.110441207885742,
      "activations/layer18_attention_weight_min": -19.584348678588867,
      "activations/layer19_attention_weight_max": 40.51713943481445,
      "activations/layer19_attention_weight_min": -27.9146728515625,
      "activations/layer1_attention_weight_max": 17.893383026123047,
      "activations/layer1_attention_weight_min": -14.322469711303711,
      "activations/layer20_attention_weight_max": 31.393413543701172,
      "activations/layer20_attention_weight_min": -21.657028198242188,
      "activations/layer21_attention_weight_max": 28.0447998046875,
      "activations/layer21_attention_weight_min": -19.2144775390625,
      "activations/layer22_attention_weight_max": 47.823307037353516,
      "activations/layer22_attention_weight_min": -27.3538761138916,
      "activations/layer23_attention_weight_max": 30.830486297607422,
      "activations/layer23_attention_weight_min": -21.888748168945312,
      "activations/layer2_attention_weight_max": 33.36040115356445,
      "activations/layer2_attention_weight_min": -31.916664123535156,
      "activations/layer3_attention_weight_max": 93.86168670654297,
      "activations/layer3_attention_weight_min": -93.8121337890625,
      "activations/layer4_attention_weight_max": 130.58853149414062,
      "activations/layer4_attention_weight_min": -128.52688598632812,
      "activations/layer5_attention_weight_max": 81.81538391113281,
      "activations/layer5_attention_weight_min": -80.19252014160156,
      "activations/layer6_attention_weight_max": 52.9726448059082,
      "activations/layer6_attention_weight_min": -59.7260627746582,
      "activations/layer7_attention_weight_max": 82.91803741455078,
      "activations/layer7_attention_weight_min": -83.05909729003906,
      "activations/layer8_attention_weight_max": 42.95994567871094,
      "activations/layer8_attention_weight_min": -39.64155960083008,
      "activations/layer9_attention_weight_max": 39.73532485961914,
      "activations/layer9_attention_weight_min": -35.94038391113281,
      "epoch": 4.68,
      "learning_rate": 0.0001210340909090909,
      "loss": 2.863,
      "step": 80550
    },
    {
      "activations/layer0_attention_weight_max": 15.903837203979492,
      "activations/layer0_attention_weight_min": -12.3121976852417,
      "activations/layer10_attention_weight_max": 38.25757598876953,
      "activations/layer10_attention_weight_min": -33.673091888427734,
      "activations/layer11_attention_weight_max": 39.72930908203125,
      "activations/layer11_attention_weight_min": -32.863128662109375,
      "activations/layer12_attention_weight_max": 32.95343017578125,
      "activations/layer12_attention_weight_min": -26.84056282043457,
      "activations/layer13_attention_weight_max": 48.0238151550293,
      "activations/layer13_attention_weight_min": -35.52098083496094,
      "activations/layer14_attention_weight_max": 85.86553955078125,
      "activations/layer14_attention_weight_min": -56.49003982543945,
      "activations/layer15_attention_weight_max": 47.13151168823242,
      "activations/layer15_attention_weight_min": -30.08945655822754,
      "activations/layer16_attention_weight_max": 40.8712272644043,
      "activations/layer16_attention_weight_min": -28.55168342590332,
      "activations/layer17_attention_weight_max": 43.0604248046875,
      "activations/layer17_attention_weight_min": -26.635929107666016,
      "activations/layer18_attention_weight_max": 37.1632194519043,
      "activations/layer18_attention_weight_min": -21.2752742767334,
      "activations/layer19_attention_weight_max": 47.4404411315918,
      "activations/layer19_attention_weight_min": -29.085830688476562,
      "activations/layer1_attention_weight_max": 17.227642059326172,
      "activations/layer1_attention_weight_min": -14.388054847717285,
      "activations/layer20_attention_weight_max": 39.57501220703125,
      "activations/layer20_attention_weight_min": -25.116758346557617,
      "activations/layer21_attention_weight_max": 37.961483001708984,
      "activations/layer21_attention_weight_min": -19.98213005065918,
      "activations/layer22_attention_weight_max": 56.88116455078125,
      "activations/layer22_attention_weight_min": -29.47370147705078,
      "activations/layer23_attention_weight_max": 39.32489776611328,
      "activations/layer23_attention_weight_min": -21.157251358032227,
      "activations/layer2_attention_weight_max": 30.957599639892578,
      "activations/layer2_attention_weight_min": -32.66736602783203,
      "activations/layer3_attention_weight_max": 86.35768127441406,
      "activations/layer3_attention_weight_min": -90.84480285644531,
      "activations/layer4_attention_weight_max": 130.35508728027344,
      "activations/layer4_attention_weight_min": -129.5450897216797,
      "activations/layer5_attention_weight_max": 87.93777465820312,
      "activations/layer5_attention_weight_min": -82.7123031616211,
      "activations/layer6_attention_weight_max": 56.74308776855469,
      "activations/layer6_attention_weight_min": -56.165611267089844,
      "activations/layer7_attention_weight_max": 94.83702087402344,
      "activations/layer7_attention_weight_min": -82.90999603271484,
      "activations/layer8_attention_weight_max": 41.53892135620117,
      "activations/layer8_attention_weight_min": -41.3239860534668,
      "activations/layer9_attention_weight_max": 37.65480041503906,
      "activations/layer9_attention_weight_min": -36.330928802490234,
      "epoch": 4.68,
      "learning_rate": 0.0001210151515151515,
      "loss": 2.8644,
      "step": 80600
    },
    {
      "activations/layer0_attention_weight_max": 17.113147735595703,
      "activations/layer0_attention_weight_min": -12.123032569885254,
      "activations/layer10_attention_weight_max": 37.974910736083984,
      "activations/layer10_attention_weight_min": -29.794708251953125,
      "activations/layer11_attention_weight_max": 41.76085662841797,
      "activations/layer11_attention_weight_min": -28.102338790893555,
      "activations/layer12_attention_weight_max": 36.190399169921875,
      "activations/layer12_attention_weight_min": -31.502689361572266,
      "activations/layer13_attention_weight_max": 43.61892318725586,
      "activations/layer13_attention_weight_min": -31.39227867126465,
      "activations/layer14_attention_weight_max": 58.65630340576172,
      "activations/layer14_attention_weight_min": -44.5737419128418,
      "activations/layer15_attention_weight_max": 38.03884506225586,
      "activations/layer15_attention_weight_min": -27.921010971069336,
      "activations/layer16_attention_weight_max": 33.49873733520508,
      "activations/layer16_attention_weight_min": -27.50212287902832,
      "activations/layer17_attention_weight_max": 42.41020965576172,
      "activations/layer17_attention_weight_min": -25.095369338989258,
      "activations/layer18_attention_weight_max": 30.89442253112793,
      "activations/layer18_attention_weight_min": -20.183456420898438,
      "activations/layer19_attention_weight_max": 38.41567611694336,
      "activations/layer19_attention_weight_min": -28.895709991455078,
      "activations/layer1_attention_weight_max": 16.78575897216797,
      "activations/layer1_attention_weight_min": -17.501434326171875,
      "activations/layer20_attention_weight_max": 30.581432342529297,
      "activations/layer20_attention_weight_min": -25.547374725341797,
      "activations/layer21_attention_weight_max": 28.548490524291992,
      "activations/layer21_attention_weight_min": -20.776641845703125,
      "activations/layer22_attention_weight_max": 46.5546760559082,
      "activations/layer22_attention_weight_min": -27.7154483795166,
      "activations/layer23_attention_weight_max": 32.00602340698242,
      "activations/layer23_attention_weight_min": -20.71145248413086,
      "activations/layer2_attention_weight_max": 32.80310821533203,
      "activations/layer2_attention_weight_min": -32.563995361328125,
      "activations/layer3_attention_weight_max": 87.5594482421875,
      "activations/layer3_attention_weight_min": -91.19213104248047,
      "activations/layer4_attention_weight_max": 128.3225555419922,
      "activations/layer4_attention_weight_min": -127.3300552368164,
      "activations/layer5_attention_weight_max": 87.08879089355469,
      "activations/layer5_attention_weight_min": -85.6197509765625,
      "activations/layer6_attention_weight_max": 57.62482452392578,
      "activations/layer6_attention_weight_min": -59.05242919921875,
      "activations/layer7_attention_weight_max": 87.29153442382812,
      "activations/layer7_attention_weight_min": -83.91779327392578,
      "activations/layer8_attention_weight_max": 40.19084548950195,
      "activations/layer8_attention_weight_min": -40.086463928222656,
      "activations/layer9_attention_weight_max": 44.787559509277344,
      "activations/layer9_attention_weight_min": -34.26797103881836,
      "epoch": 4.69,
      "learning_rate": 0.00012099621212121211,
      "loss": 2.8674,
      "step": 80650
    },
    {
      "activations/layer0_attention_weight_max": 16.952964782714844,
      "activations/layer0_attention_weight_min": -12.55295467376709,
      "activations/layer10_attention_weight_max": 35.13233184814453,
      "activations/layer10_attention_weight_min": -30.824630737304688,
      "activations/layer11_attention_weight_max": 31.08147430419922,
      "activations/layer11_attention_weight_min": -26.98480224609375,
      "activations/layer12_attention_weight_max": 25.647117614746094,
      "activations/layer12_attention_weight_min": -27.732234954833984,
      "activations/layer13_attention_weight_max": 40.69111251831055,
      "activations/layer13_attention_weight_min": -32.80790710449219,
      "activations/layer14_attention_weight_max": 56.208946228027344,
      "activations/layer14_attention_weight_min": -47.75813674926758,
      "activations/layer15_attention_weight_max": 44.40949630737305,
      "activations/layer15_attention_weight_min": -32.508201599121094,
      "activations/layer16_attention_weight_max": 34.52918243408203,
      "activations/layer16_attention_weight_min": -28.515546798706055,
      "activations/layer17_attention_weight_max": 35.488162994384766,
      "activations/layer17_attention_weight_min": -25.51488494873047,
      "activations/layer18_attention_weight_max": 35.85591125488281,
      "activations/layer18_attention_weight_min": -19.65697479248047,
      "activations/layer19_attention_weight_max": 42.905452728271484,
      "activations/layer19_attention_weight_min": -31.212005615234375,
      "activations/layer1_attention_weight_max": 16.692779541015625,
      "activations/layer1_attention_weight_min": -15.267748832702637,
      "activations/layer20_attention_weight_max": 31.574373245239258,
      "activations/layer20_attention_weight_min": -23.181396484375,
      "activations/layer21_attention_weight_max": 27.970834732055664,
      "activations/layer21_attention_weight_min": -19.79285430908203,
      "activations/layer22_attention_weight_max": 50.37727355957031,
      "activations/layer22_attention_weight_min": -28.394882202148438,
      "activations/layer23_attention_weight_max": 30.392608642578125,
      "activations/layer23_attention_weight_min": -21.67144012451172,
      "activations/layer2_attention_weight_max": 31.297216415405273,
      "activations/layer2_attention_weight_min": -32.898963928222656,
      "activations/layer3_attention_weight_max": 87.34634399414062,
      "activations/layer3_attention_weight_min": -89.62052917480469,
      "activations/layer4_attention_weight_max": 126.7044448852539,
      "activations/layer4_attention_weight_min": -122.89705657958984,
      "activations/layer5_attention_weight_max": 80.33444213867188,
      "activations/layer5_attention_weight_min": -79.36293029785156,
      "activations/layer6_attention_weight_max": 56.78305435180664,
      "activations/layer6_attention_weight_min": -54.14628982543945,
      "activations/layer7_attention_weight_max": 80.09427642822266,
      "activations/layer7_attention_weight_min": -78.57354736328125,
      "activations/layer8_attention_weight_max": 40.1715202331543,
      "activations/layer8_attention_weight_min": -39.44297409057617,
      "activations/layer9_attention_weight_max": 34.98796081542969,
      "activations/layer9_attention_weight_min": -38.984004974365234,
      "epoch": 4.69,
      "learning_rate": 0.00012097727272727272,
      "loss": 2.8635,
      "step": 80700
    },
    {
      "activations/layer0_attention_weight_max": 15.441311836242676,
      "activations/layer0_attention_weight_min": -13.156929969787598,
      "activations/layer10_attention_weight_max": 34.52045822143555,
      "activations/layer10_attention_weight_min": -31.174936294555664,
      "activations/layer11_attention_weight_max": 34.496307373046875,
      "activations/layer11_attention_weight_min": -28.59459686279297,
      "activations/layer12_attention_weight_max": 24.402238845825195,
      "activations/layer12_attention_weight_min": -35.17204666137695,
      "activations/layer13_attention_weight_max": 43.07089614868164,
      "activations/layer13_attention_weight_min": -33.442440032958984,
      "activations/layer14_attention_weight_max": 67.29229736328125,
      "activations/layer14_attention_weight_min": -49.960845947265625,
      "activations/layer15_attention_weight_max": 41.99110794067383,
      "activations/layer15_attention_weight_min": -29.336456298828125,
      "activations/layer16_attention_weight_max": 35.96060562133789,
      "activations/layer16_attention_weight_min": -28.85765838623047,
      "activations/layer17_attention_weight_max": 42.06147003173828,
      "activations/layer17_attention_weight_min": -23.747339248657227,
      "activations/layer18_attention_weight_max": 30.570068359375,
      "activations/layer18_attention_weight_min": -19.78692054748535,
      "activations/layer19_attention_weight_max": 42.0497932434082,
      "activations/layer19_attention_weight_min": -31.33306884765625,
      "activations/layer1_attention_weight_max": 17.651538848876953,
      "activations/layer1_attention_weight_min": -15.087604522705078,
      "activations/layer20_attention_weight_max": 35.168113708496094,
      "activations/layer20_attention_weight_min": -24.803733825683594,
      "activations/layer21_attention_weight_max": 31.175079345703125,
      "activations/layer21_attention_weight_min": -21.60662269592285,
      "activations/layer22_attention_weight_max": 48.42247009277344,
      "activations/layer22_attention_weight_min": -28.16743278503418,
      "activations/layer23_attention_weight_max": 31.159400939941406,
      "activations/layer23_attention_weight_min": -21.408966064453125,
      "activations/layer2_attention_weight_max": 31.313228607177734,
      "activations/layer2_attention_weight_min": -31.750747680664062,
      "activations/layer3_attention_weight_max": 89.10001373291016,
      "activations/layer3_attention_weight_min": -89.89954376220703,
      "activations/layer4_attention_weight_max": 131.5409393310547,
      "activations/layer4_attention_weight_min": -128.50613403320312,
      "activations/layer5_attention_weight_max": 86.91381072998047,
      "activations/layer5_attention_weight_min": -80.42532348632812,
      "activations/layer6_attention_weight_max": 57.74018096923828,
      "activations/layer6_attention_weight_min": -55.82500076293945,
      "activations/layer7_attention_weight_max": 87.1504898071289,
      "activations/layer7_attention_weight_min": -82.52123260498047,
      "activations/layer8_attention_weight_max": 38.41669845581055,
      "activations/layer8_attention_weight_min": -40.976837158203125,
      "activations/layer9_attention_weight_max": 34.93968200683594,
      "activations/layer9_attention_weight_min": -41.491947174072266,
      "epoch": 4.69,
      "learning_rate": 0.00012095833333333332,
      "loss": 2.8664,
      "step": 80750
    },
    {
      "activations/layer0_attention_weight_max": 16.26093101501465,
      "activations/layer0_attention_weight_min": -13.639350891113281,
      "activations/layer10_attention_weight_max": 34.742095947265625,
      "activations/layer10_attention_weight_min": -29.779308319091797,
      "activations/layer11_attention_weight_max": 33.1448860168457,
      "activations/layer11_attention_weight_min": -28.629287719726562,
      "activations/layer12_attention_weight_max": 41.17313766479492,
      "activations/layer12_attention_weight_min": -27.9711856842041,
      "activations/layer13_attention_weight_max": 60.45145797729492,
      "activations/layer13_attention_weight_min": -40.152488708496094,
      "activations/layer14_attention_weight_max": 62.640838623046875,
      "activations/layer14_attention_weight_min": -48.825897216796875,
      "activations/layer15_attention_weight_max": 39.32223892211914,
      "activations/layer15_attention_weight_min": -29.341392517089844,
      "activations/layer16_attention_weight_max": 35.74752426147461,
      "activations/layer16_attention_weight_min": -31.27586555480957,
      "activations/layer17_attention_weight_max": 40.3286018371582,
      "activations/layer17_attention_weight_min": -26.88212013244629,
      "activations/layer18_attention_weight_max": 33.95241165161133,
      "activations/layer18_attention_weight_min": -21.34984588623047,
      "activations/layer19_attention_weight_max": 37.5495491027832,
      "activations/layer19_attention_weight_min": -30.73170280456543,
      "activations/layer1_attention_weight_max": 18.76515769958496,
      "activations/layer1_attention_weight_min": -17.978635787963867,
      "activations/layer20_attention_weight_max": 31.744422912597656,
      "activations/layer20_attention_weight_min": -24.844165802001953,
      "activations/layer21_attention_weight_max": 29.77487564086914,
      "activations/layer21_attention_weight_min": -22.49905776977539,
      "activations/layer22_attention_weight_max": 52.198238372802734,
      "activations/layer22_attention_weight_min": -28.396041870117188,
      "activations/layer23_attention_weight_max": 30.520774841308594,
      "activations/layer23_attention_weight_min": -22.10781478881836,
      "activations/layer2_attention_weight_max": 34.38148498535156,
      "activations/layer2_attention_weight_min": -32.137596130371094,
      "activations/layer3_attention_weight_max": 95.98921966552734,
      "activations/layer3_attention_weight_min": -92.10668182373047,
      "activations/layer4_attention_weight_max": 143.8997039794922,
      "activations/layer4_attention_weight_min": -132.8190155029297,
      "activations/layer5_attention_weight_max": 91.45299530029297,
      "activations/layer5_attention_weight_min": -84.51382446289062,
      "activations/layer6_attention_weight_max": 56.535648345947266,
      "activations/layer6_attention_weight_min": -63.01347351074219,
      "activations/layer7_attention_weight_max": 87.98123168945312,
      "activations/layer7_attention_weight_min": -85.04234313964844,
      "activations/layer8_attention_weight_max": 42.188011169433594,
      "activations/layer8_attention_weight_min": -40.20130157470703,
      "activations/layer9_attention_weight_max": 37.01742172241211,
      "activations/layer9_attention_weight_min": -33.30537796020508,
      "epoch": 4.69,
      "learning_rate": 0.00012093939393939393,
      "loss": 2.8558,
      "step": 80800
    },
    {
      "activations/layer0_attention_weight_max": 17.164134979248047,
      "activations/layer0_attention_weight_min": -12.512786865234375,
      "activations/layer10_attention_weight_max": 36.063961029052734,
      "activations/layer10_attention_weight_min": -32.400733947753906,
      "activations/layer11_attention_weight_max": 36.82054901123047,
      "activations/layer11_attention_weight_min": -30.763212203979492,
      "activations/layer12_attention_weight_max": 30.778156280517578,
      "activations/layer12_attention_weight_min": -27.17306137084961,
      "activations/layer13_attention_weight_max": 46.74900436401367,
      "activations/layer13_attention_weight_min": -36.35613250732422,
      "activations/layer14_attention_weight_max": 66.53202056884766,
      "activations/layer14_attention_weight_min": -48.823036193847656,
      "activations/layer15_attention_weight_max": 37.90564727783203,
      "activations/layer15_attention_weight_min": -28.318510055541992,
      "activations/layer16_attention_weight_max": 36.585784912109375,
      "activations/layer16_attention_weight_min": -29.42719078063965,
      "activations/layer17_attention_weight_max": 38.59059143066406,
      "activations/layer17_attention_weight_min": -23.666366577148438,
      "activations/layer18_attention_weight_max": 31.996662139892578,
      "activations/layer18_attention_weight_min": -21.34842300415039,
      "activations/layer19_attention_weight_max": 39.90707015991211,
      "activations/layer19_attention_weight_min": -30.34187889099121,
      "activations/layer1_attention_weight_max": 17.343841552734375,
      "activations/layer1_attention_weight_min": -14.920693397521973,
      "activations/layer20_attention_weight_max": 33.28976821899414,
      "activations/layer20_attention_weight_min": -24.22588348388672,
      "activations/layer21_attention_weight_max": 33.895423889160156,
      "activations/layer21_attention_weight_min": -20.791183471679688,
      "activations/layer22_attention_weight_max": 55.47342300415039,
      "activations/layer22_attention_weight_min": -29.25822639465332,
      "activations/layer23_attention_weight_max": 32.647247314453125,
      "activations/layer23_attention_weight_min": -20.321704864501953,
      "activations/layer2_attention_weight_max": 32.248558044433594,
      "activations/layer2_attention_weight_min": -32.209022521972656,
      "activations/layer3_attention_weight_max": 91.79293823242188,
      "activations/layer3_attention_weight_min": -93.45632934570312,
      "activations/layer4_attention_weight_max": 135.86148071289062,
      "activations/layer4_attention_weight_min": -136.34725952148438,
      "activations/layer5_attention_weight_max": 88.88752746582031,
      "activations/layer5_attention_weight_min": -85.91472625732422,
      "activations/layer6_attention_weight_max": 60.054466247558594,
      "activations/layer6_attention_weight_min": -59.203617095947266,
      "activations/layer7_attention_weight_max": 88.49012756347656,
      "activations/layer7_attention_weight_min": -84.51665496826172,
      "activations/layer8_attention_weight_max": 42.573707580566406,
      "activations/layer8_attention_weight_min": -41.059452056884766,
      "activations/layer9_attention_weight_max": 37.98836898803711,
      "activations/layer9_attention_weight_min": -34.27336120605469,
      "epoch": 4.7,
      "learning_rate": 0.00012092045454545454,
      "loss": 2.8803,
      "step": 80850
    },
    {
      "activations/layer0_attention_weight_max": 16.583051681518555,
      "activations/layer0_attention_weight_min": -11.337882995605469,
      "activations/layer10_attention_weight_max": 41.05781555175781,
      "activations/layer10_attention_weight_min": -32.11790084838867,
      "activations/layer11_attention_weight_max": 43.644187927246094,
      "activations/layer11_attention_weight_min": -29.659269332885742,
      "activations/layer12_attention_weight_max": 32.31071090698242,
      "activations/layer12_attention_weight_min": -31.012414932250977,
      "activations/layer13_attention_weight_max": 46.07341384887695,
      "activations/layer13_attention_weight_min": -33.247859954833984,
      "activations/layer14_attention_weight_max": 61.51632308959961,
      "activations/layer14_attention_weight_min": -47.354820251464844,
      "activations/layer15_attention_weight_max": 40.01033401489258,
      "activations/layer15_attention_weight_min": -27.402305603027344,
      "activations/layer16_attention_weight_max": 35.54218292236328,
      "activations/layer16_attention_weight_min": -27.173965454101562,
      "activations/layer17_attention_weight_max": 35.217308044433594,
      "activations/layer17_attention_weight_min": -24.85110855102539,
      "activations/layer18_attention_weight_max": 30.267778396606445,
      "activations/layer18_attention_weight_min": -19.56153106689453,
      "activations/layer19_attention_weight_max": 40.37043380737305,
      "activations/layer19_attention_weight_min": -29.95758056640625,
      "activations/layer1_attention_weight_max": 20.457271575927734,
      "activations/layer1_attention_weight_min": -15.131730079650879,
      "activations/layer20_attention_weight_max": 32.999446868896484,
      "activations/layer20_attention_weight_min": -25.961835861206055,
      "activations/layer21_attention_weight_max": 31.003795623779297,
      "activations/layer21_attention_weight_min": -19.395156860351562,
      "activations/layer22_attention_weight_max": 52.92261505126953,
      "activations/layer22_attention_weight_min": -27.909671783447266,
      "activations/layer23_attention_weight_max": 31.110523223876953,
      "activations/layer23_attention_weight_min": -18.457731246948242,
      "activations/layer2_attention_weight_max": 32.00617980957031,
      "activations/layer2_attention_weight_min": -30.950178146362305,
      "activations/layer3_attention_weight_max": 86.37107849121094,
      "activations/layer3_attention_weight_min": -88.6059341430664,
      "activations/layer4_attention_weight_max": 133.8443145751953,
      "activations/layer4_attention_weight_min": -128.11167907714844,
      "activations/layer5_attention_weight_max": 84.1788330078125,
      "activations/layer5_attention_weight_min": -80.35008239746094,
      "activations/layer6_attention_weight_max": 59.886077880859375,
      "activations/layer6_attention_weight_min": -54.76628112792969,
      "activations/layer7_attention_weight_max": 93.55523681640625,
      "activations/layer7_attention_weight_min": -87.21109008789062,
      "activations/layer8_attention_weight_max": 42.69086837768555,
      "activations/layer8_attention_weight_min": -40.82768630981445,
      "activations/layer9_attention_weight_max": 38.56338119506836,
      "activations/layer9_attention_weight_min": -40.442203521728516,
      "epoch": 4.7,
      "learning_rate": 0.00012090151515151516,
      "loss": 2.8692,
      "step": 80900
    },
    {
      "activations/layer0_attention_weight_max": 18.04586410522461,
      "activations/layer0_attention_weight_min": -12.482158660888672,
      "activations/layer10_attention_weight_max": 32.48948287963867,
      "activations/layer10_attention_weight_min": -29.44743537902832,
      "activations/layer11_attention_weight_max": 29.755704879760742,
      "activations/layer11_attention_weight_min": -26.364639282226562,
      "activations/layer12_attention_weight_max": 27.219642639160156,
      "activations/layer12_attention_weight_min": -26.705909729003906,
      "activations/layer13_attention_weight_max": 44.4793586730957,
      "activations/layer13_attention_weight_min": -31.64189910888672,
      "activations/layer14_attention_weight_max": 66.6536636352539,
      "activations/layer14_attention_weight_min": -58.165077209472656,
      "activations/layer15_attention_weight_max": 37.87310791015625,
      "activations/layer15_attention_weight_min": -27.38644027709961,
      "activations/layer16_attention_weight_max": 35.08023452758789,
      "activations/layer16_attention_weight_min": -27.47357940673828,
      "activations/layer17_attention_weight_max": 37.589813232421875,
      "activations/layer17_attention_weight_min": -23.88490104675293,
      "activations/layer18_attention_weight_max": 30.883378982543945,
      "activations/layer18_attention_weight_min": -19.90895652770996,
      "activations/layer19_attention_weight_max": 38.47697830200195,
      "activations/layer19_attention_weight_min": -28.521696090698242,
      "activations/layer1_attention_weight_max": 16.9888916015625,
      "activations/layer1_attention_weight_min": -15.055183410644531,
      "activations/layer20_attention_weight_max": 32.066646575927734,
      "activations/layer20_attention_weight_min": -22.84626579284668,
      "activations/layer21_attention_weight_max": 29.585521697998047,
      "activations/layer21_attention_weight_min": -20.11984634399414,
      "activations/layer22_attention_weight_max": 51.06182098388672,
      "activations/layer22_attention_weight_min": -27.107379913330078,
      "activations/layer23_attention_weight_max": 31.89665985107422,
      "activations/layer23_attention_weight_min": -18.84746551513672,
      "activations/layer2_attention_weight_max": 32.3736686706543,
      "activations/layer2_attention_weight_min": -31.85409927368164,
      "activations/layer3_attention_weight_max": 87.43656158447266,
      "activations/layer3_attention_weight_min": -87.59954071044922,
      "activations/layer4_attention_weight_max": 132.66688537597656,
      "activations/layer4_attention_weight_min": -124.50457763671875,
      "activations/layer5_attention_weight_max": 82.83325958251953,
      "activations/layer5_attention_weight_min": -80.43531799316406,
      "activations/layer6_attention_weight_max": 54.62696838378906,
      "activations/layer6_attention_weight_min": -58.92494583129883,
      "activations/layer7_attention_weight_max": 77.25975799560547,
      "activations/layer7_attention_weight_min": -80.56993865966797,
      "activations/layer8_attention_weight_max": 38.53609085083008,
      "activations/layer8_attention_weight_min": -39.10485076904297,
      "activations/layer9_attention_weight_max": 33.4909782409668,
      "activations/layer9_attention_weight_min": -33.32118606567383,
      "epoch": 4.7,
      "learning_rate": 0.00012088257575757574,
      "loss": 2.8579,
      "step": 80950
    },
    {
      "activations/layer0_attention_weight_max": 16.985912322998047,
      "activations/layer0_attention_weight_min": -11.534696578979492,
      "activations/layer10_attention_weight_max": 35.6683349609375,
      "activations/layer10_attention_weight_min": -32.149635314941406,
      "activations/layer11_attention_weight_max": 35.42779541015625,
      "activations/layer11_attention_weight_min": -29.242053985595703,
      "activations/layer12_attention_weight_max": 40.640602111816406,
      "activations/layer12_attention_weight_min": -27.73897933959961,
      "activations/layer13_attention_weight_max": 48.125282287597656,
      "activations/layer13_attention_weight_min": -35.09956359863281,
      "activations/layer14_attention_weight_max": 62.642578125,
      "activations/layer14_attention_weight_min": -45.978759765625,
      "activations/layer15_attention_weight_max": 40.978790283203125,
      "activations/layer15_attention_weight_min": -28.4152889251709,
      "activations/layer16_attention_weight_max": 34.98345947265625,
      "activations/layer16_attention_weight_min": -27.804006576538086,
      "activations/layer17_attention_weight_max": 39.22002410888672,
      "activations/layer17_attention_weight_min": -27.20149803161621,
      "activations/layer18_attention_weight_max": 31.9918155670166,
      "activations/layer18_attention_weight_min": -20.669572830200195,
      "activations/layer19_attention_weight_max": 44.2449951171875,
      "activations/layer19_attention_weight_min": -29.76356315612793,
      "activations/layer1_attention_weight_max": 18.15472412109375,
      "activations/layer1_attention_weight_min": -14.25340747833252,
      "activations/layer20_attention_weight_max": 36.70720672607422,
      "activations/layer20_attention_weight_min": -25.183860778808594,
      "activations/layer21_attention_weight_max": 30.28082847595215,
      "activations/layer21_attention_weight_min": -21.94771385192871,
      "activations/layer22_attention_weight_max": 52.26839065551758,
      "activations/layer22_attention_weight_min": -27.141368865966797,
      "activations/layer23_attention_weight_max": 34.49005126953125,
      "activations/layer23_attention_weight_min": -20.90089988708496,
      "activations/layer2_attention_weight_max": 32.84738540649414,
      "activations/layer2_attention_weight_min": -32.99455642700195,
      "activations/layer3_attention_weight_max": 91.57933807373047,
      "activations/layer3_attention_weight_min": -101.41334533691406,
      "activations/layer4_attention_weight_max": 136.99295043945312,
      "activations/layer4_attention_weight_min": -135.98460388183594,
      "activations/layer5_attention_weight_max": 86.96896362304688,
      "activations/layer5_attention_weight_min": -87.0186996459961,
      "activations/layer6_attention_weight_max": 60.68344497680664,
      "activations/layer6_attention_weight_min": -58.66754913330078,
      "activations/layer7_attention_weight_max": 84.61841583251953,
      "activations/layer7_attention_weight_min": -84.73828887939453,
      "activations/layer8_attention_weight_max": 42.39177703857422,
      "activations/layer8_attention_weight_min": -39.66584396362305,
      "activations/layer9_attention_weight_max": 38.32748031616211,
      "activations/layer9_attention_weight_min": -35.098941802978516,
      "epoch": 4.71,
      "learning_rate": 0.00012086363636363636,
      "loss": 2.8688,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_loss": 2.80859375,
      "eval_runtime": 8.4916,
      "eval_samples_per_second": 505.679,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_openwebtext_loss": 2.80859375,
      "eval_openwebtext_ppl": 16.58657693697683,
      "eval_openwebtext_runtime": 8.4916,
      "eval_openwebtext_samples_per_second": 505.679,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_wikitext_loss": 3.052734375,
      "eval_wikitext_ppl": 21.173160702117308,
      "eval_wikitext_runtime": 1.9749,
      "eval_wikitext_samples_per_second": 230.896,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_lambada_loss": 2.703125,
      "eval_lambada_ppl": 14.926303617194517,
      "eval_lambada_runtime": 9.5686,
      "eval_lambada_samples_per_second": 508.854,
      "step": 81000
    },
    {
      "activations/layer0_attention_weight_max": 17.245725631713867,
      "activations/layer0_attention_weight_min": -13.141115188598633,
      "activations/layer10_attention_weight_max": 35.470481872558594,
      "activations/layer10_attention_weight_min": -31.07286834716797,
      "activations/layer11_attention_weight_max": 41.39512634277344,
      "activations/layer11_attention_weight_min": -30.36425018310547,
      "activations/layer12_attention_weight_max": 39.369667053222656,
      "activations/layer12_attention_weight_min": -25.970645904541016,
      "activations/layer13_attention_weight_max": 47.54663848876953,
      "activations/layer13_attention_weight_min": -33.58864974975586,
      "activations/layer14_attention_weight_max": 69.88130950927734,
      "activations/layer14_attention_weight_min": -48.23969268798828,
      "activations/layer15_attention_weight_max": 48.057838439941406,
      "activations/layer15_attention_weight_min": -28.18914222717285,
      "activations/layer16_attention_weight_max": 40.22267532348633,
      "activations/layer16_attention_weight_min": -27.104389190673828,
      "activations/layer17_attention_weight_max": 40.9522819519043,
      "activations/layer17_attention_weight_min": -23.27167510986328,
      "activations/layer18_attention_weight_max": 33.91080093383789,
      "activations/layer18_attention_weight_min": -19.409908294677734,
      "activations/layer19_attention_weight_max": 42.62916946411133,
      "activations/layer19_attention_weight_min": -30.058063507080078,
      "activations/layer1_attention_weight_max": 16.967329025268555,
      "activations/layer1_attention_weight_min": -15.370549201965332,
      "activations/layer20_attention_weight_max": 32.41196060180664,
      "activations/layer20_attention_weight_min": -25.47347640991211,
      "activations/layer21_attention_weight_max": 32.97584915161133,
      "activations/layer21_attention_weight_min": -20.515295028686523,
      "activations/layer22_attention_weight_max": 51.59732437133789,
      "activations/layer22_attention_weight_min": -28.515838623046875,
      "activations/layer23_attention_weight_max": 34.970359802246094,
      "activations/layer23_attention_weight_min": -19.693803787231445,
      "activations/layer2_attention_weight_max": 34.17106628417969,
      "activations/layer2_attention_weight_min": -33.355194091796875,
      "activations/layer3_attention_weight_max": 90.8192367553711,
      "activations/layer3_attention_weight_min": -94.0552978515625,
      "activations/layer4_attention_weight_max": 136.12806701660156,
      "activations/layer4_attention_weight_min": -127.47124481201172,
      "activations/layer5_attention_weight_max": 92.5126953125,
      "activations/layer5_attention_weight_min": -82.20863342285156,
      "activations/layer6_attention_weight_max": 57.98115539550781,
      "activations/layer6_attention_weight_min": -56.30126190185547,
      "activations/layer7_attention_weight_max": 83.5380859375,
      "activations/layer7_attention_weight_min": -78.7455825805664,
      "activations/layer8_attention_weight_max": 41.485355377197266,
      "activations/layer8_attention_weight_min": -44.71830749511719,
      "activations/layer9_attention_weight_max": 39.01261520385742,
      "activations/layer9_attention_weight_min": -34.93480682373047,
      "epoch": 4.71,
      "learning_rate": 0.00012084469696969695,
      "loss": 2.8606,
      "step": 81050
    },
    {
      "activations/layer0_attention_weight_max": 15.459890365600586,
      "activations/layer0_attention_weight_min": -12.194403648376465,
      "activations/layer10_attention_weight_max": 35.438880920410156,
      "activations/layer10_attention_weight_min": -31.365947723388672,
      "activations/layer11_attention_weight_max": 33.80494689941406,
      "activations/layer11_attention_weight_min": -29.122455596923828,
      "activations/layer12_attention_weight_max": 40.9273567199707,
      "activations/layer12_attention_weight_min": -29.03354835510254,
      "activations/layer13_attention_weight_max": 59.63323211669922,
      "activations/layer13_attention_weight_min": -39.27989196777344,
      "activations/layer14_attention_weight_max": 82.18577575683594,
      "activations/layer14_attention_weight_min": -54.26958465576172,
      "activations/layer15_attention_weight_max": 48.38352966308594,
      "activations/layer15_attention_weight_min": -29.95671272277832,
      "activations/layer16_attention_weight_max": 44.948726654052734,
      "activations/layer16_attention_weight_min": -31.165966033935547,
      "activations/layer17_attention_weight_max": 55.03365707397461,
      "activations/layer17_attention_weight_min": -26.062429428100586,
      "activations/layer18_attention_weight_max": 39.492530822753906,
      "activations/layer18_attention_weight_min": -22.244300842285156,
      "activations/layer19_attention_weight_max": 52.474159240722656,
      "activations/layer19_attention_weight_min": -31.99323081970215,
      "activations/layer1_attention_weight_max": 17.335847854614258,
      "activations/layer1_attention_weight_min": -15.369307518005371,
      "activations/layer20_attention_weight_max": 43.83445739746094,
      "activations/layer20_attention_weight_min": -24.87349510192871,
      "activations/layer21_attention_weight_max": 38.24654769897461,
      "activations/layer21_attention_weight_min": -21.266803741455078,
      "activations/layer22_attention_weight_max": 72.80638885498047,
      "activations/layer22_attention_weight_min": -31.308467864990234,
      "activations/layer23_attention_weight_max": 47.4512939453125,
      "activations/layer23_attention_weight_min": -21.581623077392578,
      "activations/layer2_attention_weight_max": 32.211875915527344,
      "activations/layer2_attention_weight_min": -32.76506042480469,
      "activations/layer3_attention_weight_max": 87.00272369384766,
      "activations/layer3_attention_weight_min": -87.16655731201172,
      "activations/layer4_attention_weight_max": 130.94239807128906,
      "activations/layer4_attention_weight_min": -124.84307861328125,
      "activations/layer5_attention_weight_max": 85.55117797851562,
      "activations/layer5_attention_weight_min": -82.59815979003906,
      "activations/layer6_attention_weight_max": 57.81758117675781,
      "activations/layer6_attention_weight_min": -60.051544189453125,
      "activations/layer7_attention_weight_max": 84.4705810546875,
      "activations/layer7_attention_weight_min": -79.40949249267578,
      "activations/layer8_attention_weight_max": 39.132572174072266,
      "activations/layer8_attention_weight_min": -39.468505859375,
      "activations/layer9_attention_weight_max": 37.94089126586914,
      "activations/layer9_attention_weight_min": -33.779022216796875,
      "epoch": 4.71,
      "learning_rate": 0.00012082575757575756,
      "loss": 2.8681,
      "step": 81100
    },
    {
      "activations/layer0_attention_weight_max": 17.567119598388672,
      "activations/layer0_attention_weight_min": -11.044547080993652,
      "activations/layer10_attention_weight_max": 44.35222244262695,
      "activations/layer10_attention_weight_min": -33.405860900878906,
      "activations/layer11_attention_weight_max": 45.06708908081055,
      "activations/layer11_attention_weight_min": -31.773725509643555,
      "activations/layer12_attention_weight_max": 34.686195373535156,
      "activations/layer12_attention_weight_min": -28.03464698791504,
      "activations/layer13_attention_weight_max": 54.26992416381836,
      "activations/layer13_attention_weight_min": -33.81688690185547,
      "activations/layer14_attention_weight_max": 74.47595977783203,
      "activations/layer14_attention_weight_min": -46.019630432128906,
      "activations/layer15_attention_weight_max": 41.231563568115234,
      "activations/layer15_attention_weight_min": -28.571794509887695,
      "activations/layer16_attention_weight_max": 41.02701187133789,
      "activations/layer16_attention_weight_min": -27.805179595947266,
      "activations/layer17_attention_weight_max": 42.06291580200195,
      "activations/layer17_attention_weight_min": -24.211875915527344,
      "activations/layer18_attention_weight_max": 32.024803161621094,
      "activations/layer18_attention_weight_min": -19.939115524291992,
      "activations/layer19_attention_weight_max": 39.46086502075195,
      "activations/layer19_attention_weight_min": -29.900039672851562,
      "activations/layer1_attention_weight_max": 19.57596778869629,
      "activations/layer1_attention_weight_min": -14.113354682922363,
      "activations/layer20_attention_weight_max": 34.213558197021484,
      "activations/layer20_attention_weight_min": -23.371044158935547,
      "activations/layer21_attention_weight_max": 37.618507385253906,
      "activations/layer21_attention_weight_min": -19.524662017822266,
      "activations/layer22_attention_weight_max": 54.61698532104492,
      "activations/layer22_attention_weight_min": -29.268268585205078,
      "activations/layer23_attention_weight_max": 33.7947998046875,
      "activations/layer23_attention_weight_min": -18.936988830566406,
      "activations/layer2_attention_weight_max": 33.93367004394531,
      "activations/layer2_attention_weight_min": -34.09394836425781,
      "activations/layer3_attention_weight_max": 95.53756713867188,
      "activations/layer3_attention_weight_min": -96.54573059082031,
      "activations/layer4_attention_weight_max": 147.37742614746094,
      "activations/layer4_attention_weight_min": -138.19761657714844,
      "activations/layer5_attention_weight_max": 96.40206146240234,
      "activations/layer5_attention_weight_min": -88.82156372070312,
      "activations/layer6_attention_weight_max": 62.94898986816406,
      "activations/layer6_attention_weight_min": -64.22233581542969,
      "activations/layer7_attention_weight_max": 98.8972396850586,
      "activations/layer7_attention_weight_min": -89.46574401855469,
      "activations/layer8_attention_weight_max": 48.90591812133789,
      "activations/layer8_attention_weight_min": -45.17341995239258,
      "activations/layer9_attention_weight_max": 47.12337875366211,
      "activations/layer9_attention_weight_min": -40.02265548706055,
      "epoch": 4.72,
      "learning_rate": 0.00012080681818181818,
      "loss": 2.8619,
      "step": 81150
    },
    {
      "activations/layer0_attention_weight_max": 17.92660903930664,
      "activations/layer0_attention_weight_min": -11.28111743927002,
      "activations/layer10_attention_weight_max": 31.1263484954834,
      "activations/layer10_attention_weight_min": -28.786352157592773,
      "activations/layer11_attention_weight_max": 31.078350067138672,
      "activations/layer11_attention_weight_min": -25.93887710571289,
      "activations/layer12_attention_weight_max": 26.780776977539062,
      "activations/layer12_attention_weight_min": -29.98802947998047,
      "activations/layer13_attention_weight_max": 44.06970977783203,
      "activations/layer13_attention_weight_min": -31.230552673339844,
      "activations/layer14_attention_weight_max": 76.20741271972656,
      "activations/layer14_attention_weight_min": -58.329994201660156,
      "activations/layer15_attention_weight_max": 35.16892623901367,
      "activations/layer15_attention_weight_min": -26.828935623168945,
      "activations/layer16_attention_weight_max": 39.3447380065918,
      "activations/layer16_attention_weight_min": -28.291179656982422,
      "activations/layer17_attention_weight_max": 37.215118408203125,
      "activations/layer17_attention_weight_min": -23.306392669677734,
      "activations/layer18_attention_weight_max": 29.955745697021484,
      "activations/layer18_attention_weight_min": -20.191410064697266,
      "activations/layer19_attention_weight_max": 41.893985748291016,
      "activations/layer19_attention_weight_min": -28.869352340698242,
      "activations/layer1_attention_weight_max": 18.288009643554688,
      "activations/layer1_attention_weight_min": -16.154563903808594,
      "activations/layer20_attention_weight_max": 31.087678909301758,
      "activations/layer20_attention_weight_min": -24.044113159179688,
      "activations/layer21_attention_weight_max": 34.77589416503906,
      "activations/layer21_attention_weight_min": -19.89095687866211,
      "activations/layer22_attention_weight_max": 47.019676208496094,
      "activations/layer22_attention_weight_min": -27.471725463867188,
      "activations/layer23_attention_weight_max": 32.11804962158203,
      "activations/layer23_attention_weight_min": -20.236703872680664,
      "activations/layer2_attention_weight_max": 34.53826904296875,
      "activations/layer2_attention_weight_min": -32.94409942626953,
      "activations/layer3_attention_weight_max": 98.08163452148438,
      "activations/layer3_attention_weight_min": -97.29674530029297,
      "activations/layer4_attention_weight_max": 145.3590850830078,
      "activations/layer4_attention_weight_min": -139.58177185058594,
      "activations/layer5_attention_weight_max": 87.96975708007812,
      "activations/layer5_attention_weight_min": -81.79194641113281,
      "activations/layer6_attention_weight_max": 56.486656188964844,
      "activations/layer6_attention_weight_min": -55.64227294921875,
      "activations/layer7_attention_weight_max": 91.11215209960938,
      "activations/layer7_attention_weight_min": -86.11646270751953,
      "activations/layer8_attention_weight_max": 37.98811721801758,
      "activations/layer8_attention_weight_min": -37.8694953918457,
      "activations/layer9_attention_weight_max": 36.393898010253906,
      "activations/layer9_attention_weight_min": -35.61800003051758,
      "epoch": 4.72,
      "learning_rate": 0.00012078825757575757,
      "loss": 2.8552,
      "step": 81200
    },
    {
      "activations/layer0_attention_weight_max": 16.657278060913086,
      "activations/layer0_attention_weight_min": -11.243574142456055,
      "activations/layer10_attention_weight_max": 33.747352600097656,
      "activations/layer10_attention_weight_min": -29.70798110961914,
      "activations/layer11_attention_weight_max": 33.182716369628906,
      "activations/layer11_attention_weight_min": -28.498577117919922,
      "activations/layer12_attention_weight_max": 39.85605239868164,
      "activations/layer12_attention_weight_min": -25.503904342651367,
      "activations/layer13_attention_weight_max": 50.97666931152344,
      "activations/layer13_attention_weight_min": -36.4340934753418,
      "activations/layer14_attention_weight_max": 67.02835083007812,
      "activations/layer14_attention_weight_min": -49.44556427001953,
      "activations/layer15_attention_weight_max": 39.184600830078125,
      "activations/layer15_attention_weight_min": -29.686729431152344,
      "activations/layer16_attention_weight_max": 36.02320861816406,
      "activations/layer16_attention_weight_min": -27.141435623168945,
      "activations/layer17_attention_weight_max": 37.438907623291016,
      "activations/layer17_attention_weight_min": -25.710718154907227,
      "activations/layer18_attention_weight_max": 29.658464431762695,
      "activations/layer18_attention_weight_min": -20.319894790649414,
      "activations/layer19_attention_weight_max": 41.20664596557617,
      "activations/layer19_attention_weight_min": -26.59242057800293,
      "activations/layer1_attention_weight_max": 17.551559448242188,
      "activations/layer1_attention_weight_min": -17.339771270751953,
      "activations/layer20_attention_weight_max": 29.959972381591797,
      "activations/layer20_attention_weight_min": -22.430692672729492,
      "activations/layer21_attention_weight_max": 32.38435745239258,
      "activations/layer21_attention_weight_min": -19.39177894592285,
      "activations/layer22_attention_weight_max": 45.808956146240234,
      "activations/layer22_attention_weight_min": -31.231767654418945,
      "activations/layer23_attention_weight_max": 33.948001861572266,
      "activations/layer23_attention_weight_min": -21.465919494628906,
      "activations/layer2_attention_weight_max": 33.29949188232422,
      "activations/layer2_attention_weight_min": -31.48739242553711,
      "activations/layer3_attention_weight_max": 90.60298156738281,
      "activations/layer3_attention_weight_min": -92.39019775390625,
      "activations/layer4_attention_weight_max": 129.7628936767578,
      "activations/layer4_attention_weight_min": -123.03050994873047,
      "activations/layer5_attention_weight_max": 85.23583984375,
      "activations/layer5_attention_weight_min": -78.3963851928711,
      "activations/layer6_attention_weight_max": 58.52174377441406,
      "activations/layer6_attention_weight_min": -55.18757247924805,
      "activations/layer7_attention_weight_max": 78.76939392089844,
      "activations/layer7_attention_weight_min": -76.20762634277344,
      "activations/layer8_attention_weight_max": 38.87078857421875,
      "activations/layer8_attention_weight_min": -38.834312438964844,
      "activations/layer9_attention_weight_max": 36.35033416748047,
      "activations/layer9_attention_weight_min": -32.72527313232422,
      "epoch": 4.72,
      "learning_rate": 0.00012076931818181817,
      "loss": 2.8557,
      "step": 81250
    },
    {
      "activations/layer0_attention_weight_max": 16.435218811035156,
      "activations/layer0_attention_weight_min": -12.43431568145752,
      "activations/layer10_attention_weight_max": 37.549198150634766,
      "activations/layer10_attention_weight_min": -33.078670501708984,
      "activations/layer11_attention_weight_max": 41.5698127746582,
      "activations/layer11_attention_weight_min": -28.696826934814453,
      "activations/layer12_attention_weight_max": 30.013368606567383,
      "activations/layer12_attention_weight_min": -25.31699562072754,
      "activations/layer13_attention_weight_max": 46.11470413208008,
      "activations/layer13_attention_weight_min": -32.86555480957031,
      "activations/layer14_attention_weight_max": 66.99099731445312,
      "activations/layer14_attention_weight_min": -47.213043212890625,
      "activations/layer15_attention_weight_max": 40.24912643432617,
      "activations/layer15_attention_weight_min": -29.16338348388672,
      "activations/layer16_attention_weight_max": 43.21320724487305,
      "activations/layer16_attention_weight_min": -28.5810546875,
      "activations/layer17_attention_weight_max": 43.794368743896484,
      "activations/layer17_attention_weight_min": -24.703989028930664,
      "activations/layer18_attention_weight_max": 35.006160736083984,
      "activations/layer18_attention_weight_min": -22.02594757080078,
      "activations/layer19_attention_weight_max": 43.837493896484375,
      "activations/layer19_attention_weight_min": -29.268903732299805,
      "activations/layer1_attention_weight_max": 18.081241607666016,
      "activations/layer1_attention_weight_min": -15.89876651763916,
      "activations/layer20_attention_weight_max": 35.30731201171875,
      "activations/layer20_attention_weight_min": -23.489749908447266,
      "activations/layer21_attention_weight_max": 29.648595809936523,
      "activations/layer21_attention_weight_min": -20.732194900512695,
      "activations/layer22_attention_weight_max": 52.01264953613281,
      "activations/layer22_attention_weight_min": -28.96491050720215,
      "activations/layer23_attention_weight_max": 33.582008361816406,
      "activations/layer23_attention_weight_min": -20.000831604003906,
      "activations/layer2_attention_weight_max": 33.000038146972656,
      "activations/layer2_attention_weight_min": -31.991666793823242,
      "activations/layer3_attention_weight_max": 90.41983032226562,
      "activations/layer3_attention_weight_min": -92.83960723876953,
      "activations/layer4_attention_weight_max": 137.17556762695312,
      "activations/layer4_attention_weight_min": -130.54054260253906,
      "activations/layer5_attention_weight_max": 87.34603881835938,
      "activations/layer5_attention_weight_min": -81.98856353759766,
      "activations/layer6_attention_weight_max": 58.23664474487305,
      "activations/layer6_attention_weight_min": -61.22629928588867,
      "activations/layer7_attention_weight_max": 86.66575622558594,
      "activations/layer7_attention_weight_min": -89.42929077148438,
      "activations/layer8_attention_weight_max": 42.825531005859375,
      "activations/layer8_attention_weight_min": -39.01978302001953,
      "activations/layer9_attention_weight_max": 38.154178619384766,
      "activations/layer9_attention_weight_min": -35.69891357421875,
      "epoch": 4.72,
      "learning_rate": 0.00012075037878787877,
      "loss": 2.8692,
      "step": 81300
    },
    {
      "activations/layer0_attention_weight_max": 16.83021354675293,
      "activations/layer0_attention_weight_min": -11.425536155700684,
      "activations/layer10_attention_weight_max": 36.26308059692383,
      "activations/layer10_attention_weight_min": -29.731029510498047,
      "activations/layer11_attention_weight_max": 31.957592010498047,
      "activations/layer11_attention_weight_min": -27.355937957763672,
      "activations/layer12_attention_weight_max": 34.325130462646484,
      "activations/layer12_attention_weight_min": -25.064197540283203,
      "activations/layer13_attention_weight_max": 41.76319122314453,
      "activations/layer13_attention_weight_min": -29.006750106811523,
      "activations/layer14_attention_weight_max": 61.629886627197266,
      "activations/layer14_attention_weight_min": -45.37134552001953,
      "activations/layer15_attention_weight_max": 45.17331314086914,
      "activations/layer15_attention_weight_min": -27.289291381835938,
      "activations/layer16_attention_weight_max": 39.8957633972168,
      "activations/layer16_attention_weight_min": -29.491167068481445,
      "activations/layer17_attention_weight_max": 39.66625213623047,
      "activations/layer17_attention_weight_min": -23.82379913330078,
      "activations/layer18_attention_weight_max": 35.822593688964844,
      "activations/layer18_attention_weight_min": -19.507930755615234,
      "activations/layer19_attention_weight_max": 44.57522201538086,
      "activations/layer19_attention_weight_min": -29.4043025970459,
      "activations/layer1_attention_weight_max": 18.125873565673828,
      "activations/layer1_attention_weight_min": -14.893901824951172,
      "activations/layer20_attention_weight_max": 37.08940887451172,
      "activations/layer20_attention_weight_min": -24.676624298095703,
      "activations/layer21_attention_weight_max": 41.85664749145508,
      "activations/layer21_attention_weight_min": -21.614910125732422,
      "activations/layer22_attention_weight_max": 52.258731842041016,
      "activations/layer22_attention_weight_min": -29.77288055419922,
      "activations/layer23_attention_weight_max": 34.37641906738281,
      "activations/layer23_attention_weight_min": -20.864797592163086,
      "activations/layer2_attention_weight_max": 32.519615173339844,
      "activations/layer2_attention_weight_min": -33.19536209106445,
      "activations/layer3_attention_weight_max": 88.23139190673828,
      "activations/layer3_attention_weight_min": -88.65322875976562,
      "activations/layer4_attention_weight_max": 133.36148071289062,
      "activations/layer4_attention_weight_min": -123.73406982421875,
      "activations/layer5_attention_weight_max": 85.14839172363281,
      "activations/layer5_attention_weight_min": -80.48402404785156,
      "activations/layer6_attention_weight_max": 56.65806579589844,
      "activations/layer6_attention_weight_min": -55.06291961669922,
      "activations/layer7_attention_weight_max": 82.90818786621094,
      "activations/layer7_attention_weight_min": -81.61104583740234,
      "activations/layer8_attention_weight_max": 39.62070846557617,
      "activations/layer8_attention_weight_min": -39.46120834350586,
      "activations/layer9_attention_weight_max": 37.07596969604492,
      "activations/layer9_attention_weight_min": -33.6106071472168,
      "epoch": 4.73,
      "learning_rate": 0.00012073143939393938,
      "loss": 2.8657,
      "step": 81350
    },
    {
      "activations/layer0_attention_weight_max": 16.08484649658203,
      "activations/layer0_attention_weight_min": -13.505104064941406,
      "activations/layer10_attention_weight_max": 35.898681640625,
      "activations/layer10_attention_weight_min": -29.198944091796875,
      "activations/layer11_attention_weight_max": 32.8291130065918,
      "activations/layer11_attention_weight_min": -27.39883041381836,
      "activations/layer12_attention_weight_max": 35.43158721923828,
      "activations/layer12_attention_weight_min": -29.576658248901367,
      "activations/layer13_attention_weight_max": 60.89106750488281,
      "activations/layer13_attention_weight_min": -41.34992599487305,
      "activations/layer14_attention_weight_max": 67.99688720703125,
      "activations/layer14_attention_weight_min": -46.871253967285156,
      "activations/layer15_attention_weight_max": 49.5689697265625,
      "activations/layer15_attention_weight_min": -29.94339942932129,
      "activations/layer16_attention_weight_max": 40.120330810546875,
      "activations/layer16_attention_weight_min": -31.78098487854004,
      "activations/layer17_attention_weight_max": 45.49761199951172,
      "activations/layer17_attention_weight_min": -24.4967041015625,
      "activations/layer18_attention_weight_max": 37.718265533447266,
      "activations/layer18_attention_weight_min": -20.27347183227539,
      "activations/layer19_attention_weight_max": 46.49789047241211,
      "activations/layer19_attention_weight_min": -31.75029945373535,
      "activations/layer1_attention_weight_max": 17.662893295288086,
      "activations/layer1_attention_weight_min": -13.951791763305664,
      "activations/layer20_attention_weight_max": 37.663875579833984,
      "activations/layer20_attention_weight_min": -23.58559226989746,
      "activations/layer21_attention_weight_max": 37.58951950073242,
      "activations/layer21_attention_weight_min": -20.965944290161133,
      "activations/layer22_attention_weight_max": 61.788246154785156,
      "activations/layer22_attention_weight_min": -27.803438186645508,
      "activations/layer23_attention_weight_max": 39.36918258666992,
      "activations/layer23_attention_weight_min": -20.389881134033203,
      "activations/layer2_attention_weight_max": 32.23556900024414,
      "activations/layer2_attention_weight_min": -31.858078002929688,
      "activations/layer3_attention_weight_max": 90.22351837158203,
      "activations/layer3_attention_weight_min": -93.3171615600586,
      "activations/layer4_attention_weight_max": 128.92674255371094,
      "activations/layer4_attention_weight_min": -130.62586975097656,
      "activations/layer5_attention_weight_max": 84.39149475097656,
      "activations/layer5_attention_weight_min": -81.65753936767578,
      "activations/layer6_attention_weight_max": 56.80638885498047,
      "activations/layer6_attention_weight_min": -55.175907135009766,
      "activations/layer7_attention_weight_max": 81.18812561035156,
      "activations/layer7_attention_weight_min": -79.36442565917969,
      "activations/layer8_attention_weight_max": 40.48863983154297,
      "activations/layer8_attention_weight_min": -39.23057556152344,
      "activations/layer9_attention_weight_max": 35.33937454223633,
      "activations/layer9_attention_weight_min": -35.34377670288086,
      "epoch": 4.73,
      "learning_rate": 0.00012071249999999999,
      "loss": 2.8715,
      "step": 81400
    },
    {
      "activations/layer0_attention_weight_max": 17.597436904907227,
      "activations/layer0_attention_weight_min": -12.520625114440918,
      "activations/layer10_attention_weight_max": 40.69562530517578,
      "activations/layer10_attention_weight_min": -30.649858474731445,
      "activations/layer11_attention_weight_max": 35.315406799316406,
      "activations/layer11_attention_weight_min": -27.562885284423828,
      "activations/layer12_attention_weight_max": 27.814376831054688,
      "activations/layer12_attention_weight_min": -26.882373809814453,
      "activations/layer13_attention_weight_max": 50.37948989868164,
      "activations/layer13_attention_weight_min": -33.13026809692383,
      "activations/layer14_attention_weight_max": 81.46977233886719,
      "activations/layer14_attention_weight_min": -49.370487213134766,
      "activations/layer15_attention_weight_max": 42.33839797973633,
      "activations/layer15_attention_weight_min": -26.990507125854492,
      "activations/layer16_attention_weight_max": 40.78856658935547,
      "activations/layer16_attention_weight_min": -28.51093864440918,
      "activations/layer17_attention_weight_max": 42.01558303833008,
      "activations/layer17_attention_weight_min": -23.11569595336914,
      "activations/layer18_attention_weight_max": 41.77208709716797,
      "activations/layer18_attention_weight_min": -20.4892578125,
      "activations/layer19_attention_weight_max": 44.63457489013672,
      "activations/layer19_attention_weight_min": -30.681791305541992,
      "activations/layer1_attention_weight_max": 17.386445999145508,
      "activations/layer1_attention_weight_min": -15.181227684020996,
      "activations/layer20_attention_weight_max": 37.40744400024414,
      "activations/layer20_attention_weight_min": -23.298362731933594,
      "activations/layer21_attention_weight_max": 35.02683639526367,
      "activations/layer21_attention_weight_min": -21.33832359313965,
      "activations/layer22_attention_weight_max": 58.7100944519043,
      "activations/layer22_attention_weight_min": -29.690446853637695,
      "activations/layer23_attention_weight_max": 35.27149200439453,
      "activations/layer23_attention_weight_min": -21.181232452392578,
      "activations/layer2_attention_weight_max": 32.42855453491211,
      "activations/layer2_attention_weight_min": -31.201772689819336,
      "activations/layer3_attention_weight_max": 91.40938568115234,
      "activations/layer3_attention_weight_min": -90.73358154296875,
      "activations/layer4_attention_weight_max": 135.82310485839844,
      "activations/layer4_attention_weight_min": -131.2793426513672,
      "activations/layer5_attention_weight_max": 92.59960174560547,
      "activations/layer5_attention_weight_min": -81.97869873046875,
      "activations/layer6_attention_weight_max": 62.867279052734375,
      "activations/layer6_attention_weight_min": -60.10719299316406,
      "activations/layer7_attention_weight_max": 83.96293640136719,
      "activations/layer7_attention_weight_min": -83.85250091552734,
      "activations/layer8_attention_weight_max": 42.23124694824219,
      "activations/layer8_attention_weight_min": -43.75350570678711,
      "activations/layer9_attention_weight_max": 41.258182525634766,
      "activations/layer9_attention_weight_min": -35.14316177368164,
      "epoch": 4.73,
      "learning_rate": 0.0001206935606060606,
      "loss": 2.8653,
      "step": 81450
    },
    {
      "activations/layer0_attention_weight_max": 16.388032913208008,
      "activations/layer0_attention_weight_min": -11.347838401794434,
      "activations/layer10_attention_weight_max": 34.61355972290039,
      "activations/layer10_attention_weight_min": -30.63387680053711,
      "activations/layer11_attention_weight_max": 33.21242904663086,
      "activations/layer11_attention_weight_min": -27.542545318603516,
      "activations/layer12_attention_weight_max": 30.233287811279297,
      "activations/layer12_attention_weight_min": -28.979341506958008,
      "activations/layer13_attention_weight_max": 46.959327697753906,
      "activations/layer13_attention_weight_min": -34.17587661743164,
      "activations/layer14_attention_weight_max": 59.7293815612793,
      "activations/layer14_attention_weight_min": -47.899559020996094,
      "activations/layer15_attention_weight_max": 38.91350555419922,
      "activations/layer15_attention_weight_min": -30.612030029296875,
      "activations/layer16_attention_weight_max": 39.13960647583008,
      "activations/layer16_attention_weight_min": -31.858348846435547,
      "activations/layer17_attention_weight_max": 42.80174255371094,
      "activations/layer17_attention_weight_min": -27.416749954223633,
      "activations/layer18_attention_weight_max": 32.66572189331055,
      "activations/layer18_attention_weight_min": -19.440265655517578,
      "activations/layer19_attention_weight_max": 42.060577392578125,
      "activations/layer19_attention_weight_min": -30.163894653320312,
      "activations/layer1_attention_weight_max": 17.592058181762695,
      "activations/layer1_attention_weight_min": -14.898141860961914,
      "activations/layer20_attention_weight_max": 36.51152801513672,
      "activations/layer20_attention_weight_min": -25.706851959228516,
      "activations/layer21_attention_weight_max": 33.67225646972656,
      "activations/layer21_attention_weight_min": -23.31256675720215,
      "activations/layer22_attention_weight_max": 51.77454376220703,
      "activations/layer22_attention_weight_min": -28.8564395904541,
      "activations/layer23_attention_weight_max": 33.87997055053711,
      "activations/layer23_attention_weight_min": -20.238739013671875,
      "activations/layer2_attention_weight_max": 30.331226348876953,
      "activations/layer2_attention_weight_min": -30.160457611083984,
      "activations/layer3_attention_weight_max": 85.45072937011719,
      "activations/layer3_attention_weight_min": -89.62014770507812,
      "activations/layer4_attention_weight_max": 127.84124755859375,
      "activations/layer4_attention_weight_min": -128.71640014648438,
      "activations/layer5_attention_weight_max": 83.61759185791016,
      "activations/layer5_attention_weight_min": -82.98210144042969,
      "activations/layer6_attention_weight_max": 57.84663009643555,
      "activations/layer6_attention_weight_min": -57.530643463134766,
      "activations/layer7_attention_weight_max": 85.978515625,
      "activations/layer7_attention_weight_min": -81.98452758789062,
      "activations/layer8_attention_weight_max": 39.9424934387207,
      "activations/layer8_attention_weight_min": -37.86402893066406,
      "activations/layer9_attention_weight_max": 36.51394271850586,
      "activations/layer9_attention_weight_min": -36.51347351074219,
      "epoch": 4.74,
      "learning_rate": 0.0001206746212121212,
      "loss": 2.8672,
      "step": 81500
    },
    {
      "activations/layer0_attention_weight_max": 17.01181983947754,
      "activations/layer0_attention_weight_min": -12.17477798461914,
      "activations/layer10_attention_weight_max": 31.583059310913086,
      "activations/layer10_attention_weight_min": -30.29878807067871,
      "activations/layer11_attention_weight_max": 29.270248413085938,
      "activations/layer11_attention_weight_min": -26.751508712768555,
      "activations/layer12_attention_weight_max": 27.13945960998535,
      "activations/layer12_attention_weight_min": -29.13640785217285,
      "activations/layer13_attention_weight_max": 39.50105285644531,
      "activations/layer13_attention_weight_min": -31.233379364013672,
      "activations/layer14_attention_weight_max": 67.14119720458984,
      "activations/layer14_attention_weight_min": -48.59614181518555,
      "activations/layer15_attention_weight_max": 39.137351989746094,
      "activations/layer15_attention_weight_min": -27.069507598876953,
      "activations/layer16_attention_weight_max": 44.078208923339844,
      "activations/layer16_attention_weight_min": -29.180917739868164,
      "activations/layer17_attention_weight_max": 41.57194900512695,
      "activations/layer17_attention_weight_min": -24.126968383789062,
      "activations/layer18_attention_weight_max": 31.131967544555664,
      "activations/layer18_attention_weight_min": -21.180774688720703,
      "activations/layer19_attention_weight_max": 47.53586959838867,
      "activations/layer19_attention_weight_min": -31.494739532470703,
      "activations/layer1_attention_weight_max": 17.318315505981445,
      "activations/layer1_attention_weight_min": -14.349095344543457,
      "activations/layer20_attention_weight_max": 34.677093505859375,
      "activations/layer20_attention_weight_min": -24.817222595214844,
      "activations/layer21_attention_weight_max": 32.4715576171875,
      "activations/layer21_attention_weight_min": -20.744340896606445,
      "activations/layer22_attention_weight_max": 53.6175537109375,
      "activations/layer22_attention_weight_min": -28.69278907775879,
      "activations/layer23_attention_weight_max": 35.238792419433594,
      "activations/layer23_attention_weight_min": -21.54732894897461,
      "activations/layer2_attention_weight_max": 30.775352478027344,
      "activations/layer2_attention_weight_min": -31.936811447143555,
      "activations/layer3_attention_weight_max": 89.62483215332031,
      "activations/layer3_attention_weight_min": -94.23504638671875,
      "activations/layer4_attention_weight_max": 127.54769134521484,
      "activations/layer4_attention_weight_min": -129.06671142578125,
      "activations/layer5_attention_weight_max": 80.88937377929688,
      "activations/layer5_attention_weight_min": -77.57357788085938,
      "activations/layer6_attention_weight_max": 57.17826843261719,
      "activations/layer6_attention_weight_min": -55.78623962402344,
      "activations/layer7_attention_weight_max": 77.75109100341797,
      "activations/layer7_attention_weight_min": -78.15666961669922,
      "activations/layer8_attention_weight_max": 38.35033416748047,
      "activations/layer8_attention_weight_min": -36.33201217651367,
      "activations/layer9_attention_weight_max": 33.4052734375,
      "activations/layer9_attention_weight_min": -33.867584228515625,
      "epoch": 4.74,
      "learning_rate": 0.0001206556818181818,
      "loss": 2.847,
      "step": 81550
    },
    {
      "activations/layer0_attention_weight_max": 16.983890533447266,
      "activations/layer0_attention_weight_min": -13.504939079284668,
      "activations/layer10_attention_weight_max": 36.80269241333008,
      "activations/layer10_attention_weight_min": -29.468151092529297,
      "activations/layer11_attention_weight_max": 32.429325103759766,
      "activations/layer11_attention_weight_min": -26.53109359741211,
      "activations/layer12_attention_weight_max": 26.098291397094727,
      "activations/layer12_attention_weight_min": -25.118667602539062,
      "activations/layer13_attention_weight_max": 48.82505798339844,
      "activations/layer13_attention_weight_min": -31.920856475830078,
      "activations/layer14_attention_weight_max": 61.490779876708984,
      "activations/layer14_attention_weight_min": -42.32026672363281,
      "activations/layer15_attention_weight_max": 44.182212829589844,
      "activations/layer15_attention_weight_min": -28.430522918701172,
      "activations/layer16_attention_weight_max": 41.44082260131836,
      "activations/layer16_attention_weight_min": -27.25642204284668,
      "activations/layer17_attention_weight_max": 43.44968032836914,
      "activations/layer17_attention_weight_min": -24.43016815185547,
      "activations/layer18_attention_weight_max": 30.570547103881836,
      "activations/layer18_attention_weight_min": -19.219257354736328,
      "activations/layer19_attention_weight_max": 41.0683479309082,
      "activations/layer19_attention_weight_min": -29.91744041442871,
      "activations/layer1_attention_weight_max": 17.34984016418457,
      "activations/layer1_attention_weight_min": -14.777863502502441,
      "activations/layer20_attention_weight_max": 38.48345947265625,
      "activations/layer20_attention_weight_min": -23.445858001708984,
      "activations/layer21_attention_weight_max": 34.717376708984375,
      "activations/layer21_attention_weight_min": -20.72136878967285,
      "activations/layer22_attention_weight_max": 49.52484893798828,
      "activations/layer22_attention_weight_min": -27.287317276000977,
      "activations/layer23_attention_weight_max": 29.698972702026367,
      "activations/layer23_attention_weight_min": -19.135116577148438,
      "activations/layer2_attention_weight_max": 32.36543273925781,
      "activations/layer2_attention_weight_min": -30.913236618041992,
      "activations/layer3_attention_weight_max": 91.33207702636719,
      "activations/layer3_attention_weight_min": -92.15174865722656,
      "activations/layer4_attention_weight_max": 134.7866973876953,
      "activations/layer4_attention_weight_min": -129.42318725585938,
      "activations/layer5_attention_weight_max": 84.24700927734375,
      "activations/layer5_attention_weight_min": -80.6865234375,
      "activations/layer6_attention_weight_max": 59.31291961669922,
      "activations/layer6_attention_weight_min": -57.05514907836914,
      "activations/layer7_attention_weight_max": 81.82015228271484,
      "activations/layer7_attention_weight_min": -79.39434814453125,
      "activations/layer8_attention_weight_max": 41.20576477050781,
      "activations/layer8_attention_weight_min": -38.21678924560547,
      "activations/layer9_attention_weight_max": 38.722774505615234,
      "activations/layer9_attention_weight_min": -33.017330169677734,
      "epoch": 4.74,
      "learning_rate": 0.00012063674242424242,
      "loss": 2.852,
      "step": 81600
    },
    {
      "activations/layer0_attention_weight_max": 17.515727996826172,
      "activations/layer0_attention_weight_min": -11.343772888183594,
      "activations/layer10_attention_weight_max": 34.49904251098633,
      "activations/layer10_attention_weight_min": -30.04567527770996,
      "activations/layer11_attention_weight_max": 31.142995834350586,
      "activations/layer11_attention_weight_min": -28.310688018798828,
      "activations/layer12_attention_weight_max": 30.813451766967773,
      "activations/layer12_attention_weight_min": -26.251752853393555,
      "activations/layer13_attention_weight_max": 53.853145599365234,
      "activations/layer13_attention_weight_min": -37.18190002441406,
      "activations/layer14_attention_weight_max": 57.423587799072266,
      "activations/layer14_attention_weight_min": -45.21185302734375,
      "activations/layer15_attention_weight_max": 41.8954963684082,
      "activations/layer15_attention_weight_min": -27.472578048706055,
      "activations/layer16_attention_weight_max": 36.79366683959961,
      "activations/layer16_attention_weight_min": -26.86417579650879,
      "activations/layer17_attention_weight_max": 39.68946838378906,
      "activations/layer17_attention_weight_min": -24.647472381591797,
      "activations/layer18_attention_weight_max": 32.50261306762695,
      "activations/layer18_attention_weight_min": -19.4168758392334,
      "activations/layer19_attention_weight_max": 45.55527114868164,
      "activations/layer19_attention_weight_min": -28.366086959838867,
      "activations/layer1_attention_weight_max": 17.2085018157959,
      "activations/layer1_attention_weight_min": -15.995149612426758,
      "activations/layer20_attention_weight_max": 34.55010986328125,
      "activations/layer20_attention_weight_min": -23.921775817871094,
      "activations/layer21_attention_weight_max": 32.90032196044922,
      "activations/layer21_attention_weight_min": -20.29353904724121,
      "activations/layer22_attention_weight_max": 51.539791107177734,
      "activations/layer22_attention_weight_min": -27.762571334838867,
      "activations/layer23_attention_weight_max": 35.29777908325195,
      "activations/layer23_attention_weight_min": -20.558778762817383,
      "activations/layer2_attention_weight_max": 32.03553009033203,
      "activations/layer2_attention_weight_min": -30.42984390258789,
      "activations/layer3_attention_weight_max": 89.74933624267578,
      "activations/layer3_attention_weight_min": -89.72516632080078,
      "activations/layer4_attention_weight_max": 134.412841796875,
      "activations/layer4_attention_weight_min": -131.8954315185547,
      "activations/layer5_attention_weight_max": 84.30300903320312,
      "activations/layer5_attention_weight_min": -83.4747085571289,
      "activations/layer6_attention_weight_max": 57.34918975830078,
      "activations/layer6_attention_weight_min": -55.57087707519531,
      "activations/layer7_attention_weight_max": 82.2516098022461,
      "activations/layer7_attention_weight_min": -79.53963470458984,
      "activations/layer8_attention_weight_max": 39.732330322265625,
      "activations/layer8_attention_weight_min": -38.19972610473633,
      "activations/layer9_attention_weight_max": 35.60671615600586,
      "activations/layer9_attention_weight_min": -33.419273376464844,
      "epoch": 4.74,
      "learning_rate": 0.00012061780303030301,
      "loss": 2.8687,
      "step": 81650
    },
    {
      "activations/layer0_attention_weight_max": 17.474590301513672,
      "activations/layer0_attention_weight_min": -11.617924690246582,
      "activations/layer10_attention_weight_max": 34.02569580078125,
      "activations/layer10_attention_weight_min": -31.326133728027344,
      "activations/layer11_attention_weight_max": 32.00048065185547,
      "activations/layer11_attention_weight_min": -27.528621673583984,
      "activations/layer12_attention_weight_max": 34.75598907470703,
      "activations/layer12_attention_weight_min": -29.869007110595703,
      "activations/layer13_attention_weight_max": 43.38331604003906,
      "activations/layer13_attention_weight_min": -35.535743713378906,
      "activations/layer14_attention_weight_max": 59.49464797973633,
      "activations/layer14_attention_weight_min": -44.272430419921875,
      "activations/layer15_attention_weight_max": 39.55812072753906,
      "activations/layer15_attention_weight_min": -27.501590728759766,
      "activations/layer16_attention_weight_max": 40.179229736328125,
      "activations/layer16_attention_weight_min": -28.95663070678711,
      "activations/layer17_attention_weight_max": 39.781864166259766,
      "activations/layer17_attention_weight_min": -24.53232192993164,
      "activations/layer18_attention_weight_max": 35.56341552734375,
      "activations/layer18_attention_weight_min": -22.613340377807617,
      "activations/layer19_attention_weight_max": 43.5473518371582,
      "activations/layer19_attention_weight_min": -30.10872459411621,
      "activations/layer1_attention_weight_max": 16.76796531677246,
      "activations/layer1_attention_weight_min": -16.221576690673828,
      "activations/layer20_attention_weight_max": 34.67955780029297,
      "activations/layer20_attention_weight_min": -23.91426658630371,
      "activations/layer21_attention_weight_max": 32.65774917602539,
      "activations/layer21_attention_weight_min": -22.0355281829834,
      "activations/layer22_attention_weight_max": 53.73283004760742,
      "activations/layer22_attention_weight_min": -29.30824089050293,
      "activations/layer23_attention_weight_max": 34.71967315673828,
      "activations/layer23_attention_weight_min": -22.161540985107422,
      "activations/layer2_attention_weight_max": 34.0875244140625,
      "activations/layer2_attention_weight_min": -30.728382110595703,
      "activations/layer3_attention_weight_max": 93.12767028808594,
      "activations/layer3_attention_weight_min": -89.50241088867188,
      "activations/layer4_attention_weight_max": 136.438232421875,
      "activations/layer4_attention_weight_min": -130.5972900390625,
      "activations/layer5_attention_weight_max": 88.98259735107422,
      "activations/layer5_attention_weight_min": -86.12187957763672,
      "activations/layer6_attention_weight_max": 56.556495666503906,
      "activations/layer6_attention_weight_min": -57.46015548706055,
      "activations/layer7_attention_weight_max": 81.10803985595703,
      "activations/layer7_attention_weight_min": -81.9284896850586,
      "activations/layer8_attention_weight_max": 38.91712188720703,
      "activations/layer8_attention_weight_min": -39.86515808105469,
      "activations/layer9_attention_weight_max": 36.167259216308594,
      "activations/layer9_attention_weight_min": -34.93836212158203,
      "epoch": 4.75,
      "learning_rate": 0.00012059886363636362,
      "loss": 2.8456,
      "step": 81700
    },
    {
      "activations/layer0_attention_weight_max": 15.693663597106934,
      "activations/layer0_attention_weight_min": -12.118494987487793,
      "activations/layer10_attention_weight_max": 32.18303298950195,
      "activations/layer10_attention_weight_min": -27.89659881591797,
      "activations/layer11_attention_weight_max": 32.76073455810547,
      "activations/layer11_attention_weight_min": -25.701269149780273,
      "activations/layer12_attention_weight_max": 31.58016586303711,
      "activations/layer12_attention_weight_min": -29.197187423706055,
      "activations/layer13_attention_weight_max": 44.59212875366211,
      "activations/layer13_attention_weight_min": -33.728328704833984,
      "activations/layer14_attention_weight_max": 58.61965560913086,
      "activations/layer14_attention_weight_min": -44.98046112060547,
      "activations/layer15_attention_weight_max": 40.4212532043457,
      "activations/layer15_attention_weight_min": -28.016647338867188,
      "activations/layer16_attention_weight_max": 41.87466049194336,
      "activations/layer16_attention_weight_min": -28.233604431152344,
      "activations/layer17_attention_weight_max": 40.619529724121094,
      "activations/layer17_attention_weight_min": -24.61370277404785,
      "activations/layer18_attention_weight_max": 35.47539520263672,
      "activations/layer18_attention_weight_min": -20.394601821899414,
      "activations/layer19_attention_weight_max": 38.767974853515625,
      "activations/layer19_attention_weight_min": -29.23512840270996,
      "activations/layer1_attention_weight_max": 17.676069259643555,
      "activations/layer1_attention_weight_min": -15.599618911743164,
      "activations/layer20_attention_weight_max": 32.72843551635742,
      "activations/layer20_attention_weight_min": -22.69477081298828,
      "activations/layer21_attention_weight_max": 29.17538070678711,
      "activations/layer21_attention_weight_min": -21.432153701782227,
      "activations/layer22_attention_weight_max": 49.60395050048828,
      "activations/layer22_attention_weight_min": -28.84156036376953,
      "activations/layer23_attention_weight_max": 36.008811950683594,
      "activations/layer23_attention_weight_min": -18.926843643188477,
      "activations/layer2_attention_weight_max": 32.871421813964844,
      "activations/layer2_attention_weight_min": -32.56716537475586,
      "activations/layer3_attention_weight_max": 90.91475677490234,
      "activations/layer3_attention_weight_min": -90.27639770507812,
      "activations/layer4_attention_weight_max": 128.0087127685547,
      "activations/layer4_attention_weight_min": -123.87501525878906,
      "activations/layer5_attention_weight_max": 86.12004089355469,
      "activations/layer5_attention_weight_min": -81.21345520019531,
      "activations/layer6_attention_weight_max": 58.21034240722656,
      "activations/layer6_attention_weight_min": -55.944549560546875,
      "activations/layer7_attention_weight_max": 85.07595825195312,
      "activations/layer7_attention_weight_min": -78.64688873291016,
      "activations/layer8_attention_weight_max": 38.44754409790039,
      "activations/layer8_attention_weight_min": -37.91136169433594,
      "activations/layer9_attention_weight_max": 35.27021408081055,
      "activations/layer9_attention_weight_min": -31.91047477722168,
      "epoch": 4.75,
      "learning_rate": 0.00012057992424242424,
      "loss": 2.858,
      "step": 81750
    },
    {
      "activations/layer0_attention_weight_max": 17.179887771606445,
      "activations/layer0_attention_weight_min": -13.11755657196045,
      "activations/layer10_attention_weight_max": 35.34392166137695,
      "activations/layer10_attention_weight_min": -31.63690757751465,
      "activations/layer11_attention_weight_max": 34.67559051513672,
      "activations/layer11_attention_weight_min": -29.68011474609375,
      "activations/layer12_attention_weight_max": 29.6641788482666,
      "activations/layer12_attention_weight_min": -27.00963592529297,
      "activations/layer13_attention_weight_max": 44.79945373535156,
      "activations/layer13_attention_weight_min": -31.239477157592773,
      "activations/layer14_attention_weight_max": 64.00675964355469,
      "activations/layer14_attention_weight_min": -42.467044830322266,
      "activations/layer15_attention_weight_max": 43.21534729003906,
      "activations/layer15_attention_weight_min": -28.216672897338867,
      "activations/layer16_attention_weight_max": 40.06597900390625,
      "activations/layer16_attention_weight_min": -29.04524803161621,
      "activations/layer17_attention_weight_max": 38.7181396484375,
      "activations/layer17_attention_weight_min": -26.28432846069336,
      "activations/layer18_attention_weight_max": 30.760604858398438,
      "activations/layer18_attention_weight_min": -19.569660186767578,
      "activations/layer19_attention_weight_max": 43.197265625,
      "activations/layer19_attention_weight_min": -30.693485260009766,
      "activations/layer1_attention_weight_max": 17.291711807250977,
      "activations/layer1_attention_weight_min": -15.573152542114258,
      "activations/layer20_attention_weight_max": 32.070777893066406,
      "activations/layer20_attention_weight_min": -22.418848037719727,
      "activations/layer21_attention_weight_max": 30.362924575805664,
      "activations/layer21_attention_weight_min": -23.9985408782959,
      "activations/layer22_attention_weight_max": 50.701820373535156,
      "activations/layer22_attention_weight_min": -27.60410499572754,
      "activations/layer23_attention_weight_max": 32.262481689453125,
      "activations/layer23_attention_weight_min": -18.029327392578125,
      "activations/layer2_attention_weight_max": 33.134193420410156,
      "activations/layer2_attention_weight_min": -32.31214904785156,
      "activations/layer3_attention_weight_max": 94.4653549194336,
      "activations/layer3_attention_weight_min": -93.0970230102539,
      "activations/layer4_attention_weight_max": 142.48431396484375,
      "activations/layer4_attention_weight_min": -134.91189575195312,
      "activations/layer5_attention_weight_max": 94.07637786865234,
      "activations/layer5_attention_weight_min": -85.8646469116211,
      "activations/layer6_attention_weight_max": 59.950626373291016,
      "activations/layer6_attention_weight_min": -63.36151123046875,
      "activations/layer7_attention_weight_max": 89.38729095458984,
      "activations/layer7_attention_weight_min": -86.16686248779297,
      "activations/layer8_attention_weight_max": 42.624420166015625,
      "activations/layer8_attention_weight_min": -43.84752655029297,
      "activations/layer9_attention_weight_max": 39.00399398803711,
      "activations/layer9_attention_weight_min": -35.247589111328125,
      "epoch": 4.75,
      "learning_rate": 0.00012056098484848483,
      "loss": 2.8501,
      "step": 81800
    },
    {
      "activations/layer0_attention_weight_max": 17.02635955810547,
      "activations/layer0_attention_weight_min": -11.007401466369629,
      "activations/layer10_attention_weight_max": 33.67018127441406,
      "activations/layer10_attention_weight_min": -29.763057708740234,
      "activations/layer11_attention_weight_max": 32.90049743652344,
      "activations/layer11_attention_weight_min": -28.081912994384766,
      "activations/layer12_attention_weight_max": 24.470224380493164,
      "activations/layer12_attention_weight_min": -33.505821228027344,
      "activations/layer13_attention_weight_max": 45.72899627685547,
      "activations/layer13_attention_weight_min": -32.786285400390625,
      "activations/layer14_attention_weight_max": 60.29715347290039,
      "activations/layer14_attention_weight_min": -48.53168487548828,
      "activations/layer15_attention_weight_max": 42.358001708984375,
      "activations/layer15_attention_weight_min": -28.11238670349121,
      "activations/layer16_attention_weight_max": 37.318233489990234,
      "activations/layer16_attention_weight_min": -28.009244918823242,
      "activations/layer17_attention_weight_max": 40.23078918457031,
      "activations/layer17_attention_weight_min": -24.699846267700195,
      "activations/layer18_attention_weight_max": 33.099525451660156,
      "activations/layer18_attention_weight_min": -21.028520584106445,
      "activations/layer19_attention_weight_max": 41.64852523803711,
      "activations/layer19_attention_weight_min": -31.306833267211914,
      "activations/layer1_attention_weight_max": 17.696557998657227,
      "activations/layer1_attention_weight_min": -15.314887046813965,
      "activations/layer20_attention_weight_max": 36.6988639831543,
      "activations/layer20_attention_weight_min": -22.642976760864258,
      "activations/layer21_attention_weight_max": 33.00794982910156,
      "activations/layer21_attention_weight_min": -21.794658660888672,
      "activations/layer22_attention_weight_max": 53.8249626159668,
      "activations/layer22_attention_weight_min": -27.40292739868164,
      "activations/layer23_attention_weight_max": 34.00595474243164,
      "activations/layer23_attention_weight_min": -20.521442413330078,
      "activations/layer2_attention_weight_max": 32.14384841918945,
      "activations/layer2_attention_weight_min": -32.01322555541992,
      "activations/layer3_attention_weight_max": 86.82437133789062,
      "activations/layer3_attention_weight_min": -90.5810546875,
      "activations/layer4_attention_weight_max": 123.82198333740234,
      "activations/layer4_attention_weight_min": -124.35991668701172,
      "activations/layer5_attention_weight_max": 78.56295776367188,
      "activations/layer5_attention_weight_min": -78.63309478759766,
      "activations/layer6_attention_weight_max": 55.44462585449219,
      "activations/layer6_attention_weight_min": -56.182884216308594,
      "activations/layer7_attention_weight_max": 79.84701538085938,
      "activations/layer7_attention_weight_min": -81.6594009399414,
      "activations/layer8_attention_weight_max": 37.20023727416992,
      "activations/layer8_attention_weight_min": -39.62240219116211,
      "activations/layer9_attention_weight_max": 35.87716293334961,
      "activations/layer9_attention_weight_min": -33.21045684814453,
      "epoch": 4.76,
      "learning_rate": 0.00012054242424242423,
      "loss": 2.8648,
      "step": 81850
    },
    {
      "activations/layer0_attention_weight_max": 17.552734375,
      "activations/layer0_attention_weight_min": -11.942544937133789,
      "activations/layer10_attention_weight_max": 37.27481460571289,
      "activations/layer10_attention_weight_min": -32.96173095703125,
      "activations/layer11_attention_weight_max": 34.845462799072266,
      "activations/layer11_attention_weight_min": -26.745101928710938,
      "activations/layer12_attention_weight_max": 27.135046005249023,
      "activations/layer12_attention_weight_min": -25.15366554260254,
      "activations/layer13_attention_weight_max": 51.11310958862305,
      "activations/layer13_attention_weight_min": -32.82906723022461,
      "activations/layer14_attention_weight_max": 64.56853485107422,
      "activations/layer14_attention_weight_min": -46.09885787963867,
      "activations/layer15_attention_weight_max": 39.92963790893555,
      "activations/layer15_attention_weight_min": -28.06044578552246,
      "activations/layer16_attention_weight_max": 38.41388702392578,
      "activations/layer16_attention_weight_min": -27.97783660888672,
      "activations/layer17_attention_weight_max": 40.22650909423828,
      "activations/layer17_attention_weight_min": -24.673908233642578,
      "activations/layer18_attention_weight_max": 34.07493591308594,
      "activations/layer18_attention_weight_min": -21.424640655517578,
      "activations/layer19_attention_weight_max": 42.69889831542969,
      "activations/layer19_attention_weight_min": -27.84821128845215,
      "activations/layer1_attention_weight_max": 17.646133422851562,
      "activations/layer1_attention_weight_min": -14.519728660583496,
      "activations/layer20_attention_weight_max": 33.369728088378906,
      "activations/layer20_attention_weight_min": -23.00515365600586,
      "activations/layer21_attention_weight_max": 36.99207305908203,
      "activations/layer21_attention_weight_min": -20.398534774780273,
      "activations/layer22_attention_weight_max": 57.940887451171875,
      "activations/layer22_attention_weight_min": -26.066457748413086,
      "activations/layer23_attention_weight_max": 34.912235260009766,
      "activations/layer23_attention_weight_min": -23.456932067871094,
      "activations/layer2_attention_weight_max": 31.027616500854492,
      "activations/layer2_attention_weight_min": -32.57469177246094,
      "activations/layer3_attention_weight_max": 90.73234558105469,
      "activations/layer3_attention_weight_min": -88.77339172363281,
      "activations/layer4_attention_weight_max": 137.03176879882812,
      "activations/layer4_attention_weight_min": -127.01171875,
      "activations/layer5_attention_weight_max": 90.95199584960938,
      "activations/layer5_attention_weight_min": -81.96656799316406,
      "activations/layer6_attention_weight_max": 59.74030303955078,
      "activations/layer6_attention_weight_min": -58.336936950683594,
      "activations/layer7_attention_weight_max": 93.5036392211914,
      "activations/layer7_attention_weight_min": -87.58472442626953,
      "activations/layer8_attention_weight_max": 42.543155670166016,
      "activations/layer8_attention_weight_min": -42.470149993896484,
      "activations/layer9_attention_weight_max": 37.30501937866211,
      "activations/layer9_attention_weight_min": -36.76100158691406,
      "epoch": 4.76,
      "learning_rate": 0.00012052348484848483,
      "loss": 2.8577,
      "step": 81900
    },
    {
      "activations/layer0_attention_weight_max": 17.421072006225586,
      "activations/layer0_attention_weight_min": -11.767292022705078,
      "activations/layer10_attention_weight_max": 35.6685676574707,
      "activations/layer10_attention_weight_min": -30.155948638916016,
      "activations/layer11_attention_weight_max": 34.07257843017578,
      "activations/layer11_attention_weight_min": -26.52004623413086,
      "activations/layer12_attention_weight_max": 30.02672576904297,
      "activations/layer12_attention_weight_min": -28.112768173217773,
      "activations/layer13_attention_weight_max": 43.17744827270508,
      "activations/layer13_attention_weight_min": -33.14957809448242,
      "activations/layer14_attention_weight_max": 76.08792877197266,
      "activations/layer14_attention_weight_min": -48.887203216552734,
      "activations/layer15_attention_weight_max": 39.740970611572266,
      "activations/layer15_attention_weight_min": -27.500919342041016,
      "activations/layer16_attention_weight_max": 36.708343505859375,
      "activations/layer16_attention_weight_min": -28.78439712524414,
      "activations/layer17_attention_weight_max": 37.828163146972656,
      "activations/layer17_attention_weight_min": -24.903804779052734,
      "activations/layer18_attention_weight_max": 30.399803161621094,
      "activations/layer18_attention_weight_min": -22.064922332763672,
      "activations/layer19_attention_weight_max": 41.279605865478516,
      "activations/layer19_attention_weight_min": -30.077558517456055,
      "activations/layer1_attention_weight_max": 17.82115364074707,
      "activations/layer1_attention_weight_min": -14.198161125183105,
      "activations/layer20_attention_weight_max": 32.14247512817383,
      "activations/layer20_attention_weight_min": -21.76106071472168,
      "activations/layer21_attention_weight_max": 40.669090270996094,
      "activations/layer21_attention_weight_min": -21.603551864624023,
      "activations/layer22_attention_weight_max": 49.54880142211914,
      "activations/layer22_attention_weight_min": -26.07720947265625,
      "activations/layer23_attention_weight_max": 32.306983947753906,
      "activations/layer23_attention_weight_min": -21.205154418945312,
      "activations/layer2_attention_weight_max": 31.242368698120117,
      "activations/layer2_attention_weight_min": -31.31947898864746,
      "activations/layer3_attention_weight_max": 86.61907196044922,
      "activations/layer3_attention_weight_min": -87.82018280029297,
      "activations/layer4_attention_weight_max": 134.29237365722656,
      "activations/layer4_attention_weight_min": -123.3254623413086,
      "activations/layer5_attention_weight_max": 87.42803192138672,
      "activations/layer5_attention_weight_min": -78.83979034423828,
      "activations/layer6_attention_weight_max": 56.58943557739258,
      "activations/layer6_attention_weight_min": -56.190582275390625,
      "activations/layer7_attention_weight_max": 82.4816665649414,
      "activations/layer7_attention_weight_min": -84.1141128540039,
      "activations/layer8_attention_weight_max": 42.78898620605469,
      "activations/layer8_attention_weight_min": -43.53941345214844,
      "activations/layer9_attention_weight_max": 39.261436462402344,
      "activations/layer9_attention_weight_min": -35.14572525024414,
      "epoch": 4.76,
      "learning_rate": 0.00012050454545454545,
      "loss": 2.863,
      "step": 81950
    },
    {
      "activations/layer0_attention_weight_max": 16.4798526763916,
      "activations/layer0_attention_weight_min": -12.407618522644043,
      "activations/layer10_attention_weight_max": 33.30499267578125,
      "activations/layer10_attention_weight_min": -31.059707641601562,
      "activations/layer11_attention_weight_max": 32.796043395996094,
      "activations/layer11_attention_weight_min": -32.04193115234375,
      "activations/layer12_attention_weight_max": 32.706642150878906,
      "activations/layer12_attention_weight_min": -31.401025772094727,
      "activations/layer13_attention_weight_max": 43.18564224243164,
      "activations/layer13_attention_weight_min": -34.01668930053711,
      "activations/layer14_attention_weight_max": 71.25929260253906,
      "activations/layer14_attention_weight_min": -57.14280319213867,
      "activations/layer15_attention_weight_max": 40.93225860595703,
      "activations/layer15_attention_weight_min": -29.562976837158203,
      "activations/layer16_attention_weight_max": 40.68584442138672,
      "activations/layer16_attention_weight_min": -29.00240135192871,
      "activations/layer17_attention_weight_max": 36.760887145996094,
      "activations/layer17_attention_weight_min": -24.616052627563477,
      "activations/layer18_attention_weight_max": 35.02094650268555,
      "activations/layer18_attention_weight_min": -22.659217834472656,
      "activations/layer19_attention_weight_max": 42.83370590209961,
      "activations/layer19_attention_weight_min": -29.305923461914062,
      "activations/layer1_attention_weight_max": 17.841663360595703,
      "activations/layer1_attention_weight_min": -16.850645065307617,
      "activations/layer20_attention_weight_max": 29.307422637939453,
      "activations/layer20_attention_weight_min": -24.104475021362305,
      "activations/layer21_attention_weight_max": 31.861602783203125,
      "activations/layer21_attention_weight_min": -21.913345336914062,
      "activations/layer22_attention_weight_max": 44.38772201538086,
      "activations/layer22_attention_weight_min": -27.511859893798828,
      "activations/layer23_attention_weight_max": 30.202795028686523,
      "activations/layer23_attention_weight_min": -20.842151641845703,
      "activations/layer2_attention_weight_max": 34.64051818847656,
      "activations/layer2_attention_weight_min": -34.762657165527344,
      "activations/layer3_attention_weight_max": 93.47161102294922,
      "activations/layer3_attention_weight_min": -94.61830139160156,
      "activations/layer4_attention_weight_max": 144.22239685058594,
      "activations/layer4_attention_weight_min": -135.06602478027344,
      "activations/layer5_attention_weight_max": 91.11251068115234,
      "activations/layer5_attention_weight_min": -81.54144287109375,
      "activations/layer6_attention_weight_max": 55.98907470703125,
      "activations/layer6_attention_weight_min": -55.36518096923828,
      "activations/layer7_attention_weight_max": 87.7527847290039,
      "activations/layer7_attention_weight_min": -92.75101470947266,
      "activations/layer8_attention_weight_max": 41.42721939086914,
      "activations/layer8_attention_weight_min": -40.9158821105957,
      "activations/layer9_attention_weight_max": 36.996315002441406,
      "activations/layer9_attention_weight_min": -40.00862503051758,
      "epoch": 4.76,
      "learning_rate": 0.00012048560606060605,
      "loss": 2.8705,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_loss": 2.806640625,
      "eval_runtime": 8.4827,
      "eval_samples_per_second": 506.204,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_openwebtext_loss": 2.806640625,
      "eval_openwebtext_ppl": 16.554212894695087,
      "eval_openwebtext_runtime": 8.4827,
      "eval_openwebtext_samples_per_second": 506.204,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_wikitext_loss": 3.044921875,
      "eval_wikitext_ppl": 21.008389858307645,
      "eval_wikitext_runtime": 1.9936,
      "eval_wikitext_samples_per_second": 228.733,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5527,
      "eval_lambada_samples_per_second": 509.698,
      "step": 82000
    },
    {
      "activations/layer0_attention_weight_max": 15.851215362548828,
      "activations/layer0_attention_weight_min": -11.847607612609863,
      "activations/layer10_attention_weight_max": 36.297515869140625,
      "activations/layer10_attention_weight_min": -29.527217864990234,
      "activations/layer11_attention_weight_max": 32.96263122558594,
      "activations/layer11_attention_weight_min": -26.713340759277344,
      "activations/layer12_attention_weight_max": 30.754135131835938,
      "activations/layer12_attention_weight_min": -28.522001266479492,
      "activations/layer13_attention_weight_max": 54.130592346191406,
      "activations/layer13_attention_weight_min": -39.02967834472656,
      "activations/layer14_attention_weight_max": 67.9911117553711,
      "activations/layer14_attention_weight_min": -51.127685546875,
      "activations/layer15_attention_weight_max": 42.9945068359375,
      "activations/layer15_attention_weight_min": -29.518165588378906,
      "activations/layer16_attention_weight_max": 40.76484298706055,
      "activations/layer16_attention_weight_min": -27.3682804107666,
      "activations/layer17_attention_weight_max": 43.9255256652832,
      "activations/layer17_attention_weight_min": -25.094942092895508,
      "activations/layer18_attention_weight_max": 33.0763053894043,
      "activations/layer18_attention_weight_min": -20.367572784423828,
      "activations/layer19_attention_weight_max": 42.71540451049805,
      "activations/layer19_attention_weight_min": -29.525985717773438,
      "activations/layer1_attention_weight_max": 17.43679428100586,
      "activations/layer1_attention_weight_min": -13.695018768310547,
      "activations/layer20_attention_weight_max": 35.77859115600586,
      "activations/layer20_attention_weight_min": -25.127164840698242,
      "activations/layer21_attention_weight_max": 30.521392822265625,
      "activations/layer21_attention_weight_min": -19.129119873046875,
      "activations/layer22_attention_weight_max": 50.488250732421875,
      "activations/layer22_attention_weight_min": -29.81494903564453,
      "activations/layer23_attention_weight_max": 34.481689453125,
      "activations/layer23_attention_weight_min": -20.049560546875,
      "activations/layer2_attention_weight_max": 32.41289520263672,
      "activations/layer2_attention_weight_min": -31.364776611328125,
      "activations/layer3_attention_weight_max": 89.56768798828125,
      "activations/layer3_attention_weight_min": -93.573974609375,
      "activations/layer4_attention_weight_max": 129.0888214111328,
      "activations/layer4_attention_weight_min": -132.110107421875,
      "activations/layer5_attention_weight_max": 87.42771911621094,
      "activations/layer5_attention_weight_min": -82.02603912353516,
      "activations/layer6_attention_weight_max": 54.99000549316406,
      "activations/layer6_attention_weight_min": -58.974578857421875,
      "activations/layer7_attention_weight_max": 82.43800354003906,
      "activations/layer7_attention_weight_min": -86.87161254882812,
      "activations/layer8_attention_weight_max": 40.104454040527344,
      "activations/layer8_attention_weight_min": -44.39873123168945,
      "activations/layer9_attention_weight_max": 36.50692367553711,
      "activations/layer9_attention_weight_min": -38.103271484375,
      "epoch": 4.77,
      "learning_rate": 0.00012046666666666665,
      "loss": 2.8536,
      "step": 82050
    },
    {
      "activations/layer0_attention_weight_max": 17.88258934020996,
      "activations/layer0_attention_weight_min": -11.699823379516602,
      "activations/layer10_attention_weight_max": 35.518394470214844,
      "activations/layer10_attention_weight_min": -30.54262924194336,
      "activations/layer11_attention_weight_max": 33.34363555908203,
      "activations/layer11_attention_weight_min": -26.3057861328125,
      "activations/layer12_attention_weight_max": 39.3314323425293,
      "activations/layer12_attention_weight_min": -27.418245315551758,
      "activations/layer13_attention_weight_max": 68.56452941894531,
      "activations/layer13_attention_weight_min": -43.901371002197266,
      "activations/layer14_attention_weight_max": 84.87471771240234,
      "activations/layer14_attention_weight_min": -53.240787506103516,
      "activations/layer15_attention_weight_max": 57.661991119384766,
      "activations/layer15_attention_weight_min": -35.180564880371094,
      "activations/layer16_attention_weight_max": 44.62925720214844,
      "activations/layer16_attention_weight_min": -29.07962989807129,
      "activations/layer17_attention_weight_max": 41.43405532836914,
      "activations/layer17_attention_weight_min": -24.864238739013672,
      "activations/layer18_attention_weight_max": 33.20034408569336,
      "activations/layer18_attention_weight_min": -21.765993118286133,
      "activations/layer19_attention_weight_max": 47.07766342163086,
      "activations/layer19_attention_weight_min": -27.744384765625,
      "activations/layer1_attention_weight_max": 16.776212692260742,
      "activations/layer1_attention_weight_min": -14.975251197814941,
      "activations/layer20_attention_weight_max": 33.83338165283203,
      "activations/layer20_attention_weight_min": -22.967432022094727,
      "activations/layer21_attention_weight_max": 36.10966110229492,
      "activations/layer21_attention_weight_min": -21.691261291503906,
      "activations/layer22_attention_weight_max": 49.471473693847656,
      "activations/layer22_attention_weight_min": -30.254180908203125,
      "activations/layer23_attention_weight_max": 32.56951141357422,
      "activations/layer23_attention_weight_min": -20.998205184936523,
      "activations/layer2_attention_weight_max": 34.99272155761719,
      "activations/layer2_attention_weight_min": -32.528892517089844,
      "activations/layer3_attention_weight_max": 94.15977478027344,
      "activations/layer3_attention_weight_min": -91.14399719238281,
      "activations/layer4_attention_weight_max": 136.0772705078125,
      "activations/layer4_attention_weight_min": -125.07344055175781,
      "activations/layer5_attention_weight_max": 88.15567016601562,
      "activations/layer5_attention_weight_min": -82.22694396972656,
      "activations/layer6_attention_weight_max": 59.21870803833008,
      "activations/layer6_attention_weight_min": -58.91048049926758,
      "activations/layer7_attention_weight_max": 88.56757354736328,
      "activations/layer7_attention_weight_min": -79.84603118896484,
      "activations/layer8_attention_weight_max": 40.087467193603516,
      "activations/layer8_attention_weight_min": -40.326541900634766,
      "activations/layer9_attention_weight_max": 39.63174057006836,
      "activations/layer9_attention_weight_min": -37.25065231323242,
      "epoch": 4.77,
      "learning_rate": 0.00012044810606060605,
      "loss": 2.8697,
      "step": 82100
    },
    {
      "activations/layer0_attention_weight_max": 16.588146209716797,
      "activations/layer0_attention_weight_min": -12.571063041687012,
      "activations/layer10_attention_weight_max": 36.31950759887695,
      "activations/layer10_attention_weight_min": -33.00319290161133,
      "activations/layer11_attention_weight_max": 34.01890563964844,
      "activations/layer11_attention_weight_min": -28.602745056152344,
      "activations/layer12_attention_weight_max": 27.700777053833008,
      "activations/layer12_attention_weight_min": -27.748445510864258,
      "activations/layer13_attention_weight_max": 39.8612174987793,
      "activations/layer13_attention_weight_min": -33.32801818847656,
      "activations/layer14_attention_weight_max": 62.54582214355469,
      "activations/layer14_attention_weight_min": -48.105316162109375,
      "activations/layer15_attention_weight_max": 39.88704299926758,
      "activations/layer15_attention_weight_min": -29.267349243164062,
      "activations/layer16_attention_weight_max": 35.560218811035156,
      "activations/layer16_attention_weight_min": -28.346698760986328,
      "activations/layer17_attention_weight_max": 41.63823318481445,
      "activations/layer17_attention_weight_min": -25.1914005279541,
      "activations/layer18_attention_weight_max": 32.93447494506836,
      "activations/layer18_attention_weight_min": -19.36504554748535,
      "activations/layer19_attention_weight_max": 42.61927032470703,
      "activations/layer19_attention_weight_min": -29.371871948242188,
      "activations/layer1_attention_weight_max": 19.057907104492188,
      "activations/layer1_attention_weight_min": -14.695657730102539,
      "activations/layer20_attention_weight_max": 34.34698486328125,
      "activations/layer20_attention_weight_min": -23.598363876342773,
      "activations/layer21_attention_weight_max": 31.15913963317871,
      "activations/layer21_attention_weight_min": -20.5533390045166,
      "activations/layer22_attention_weight_max": 52.36969757080078,
      "activations/layer22_attention_weight_min": -28.701183319091797,
      "activations/layer23_attention_weight_max": 32.201663970947266,
      "activations/layer23_attention_weight_min": -17.610618591308594,
      "activations/layer2_attention_weight_max": 32.4776725769043,
      "activations/layer2_attention_weight_min": -30.943222045898438,
      "activations/layer3_attention_weight_max": 93.10908508300781,
      "activations/layer3_attention_weight_min": -90.29722595214844,
      "activations/layer4_attention_weight_max": 133.7362060546875,
      "activations/layer4_attention_weight_min": -127.6651611328125,
      "activations/layer5_attention_weight_max": 86.19115447998047,
      "activations/layer5_attention_weight_min": -83.12659454345703,
      "activations/layer6_attention_weight_max": 55.85095977783203,
      "activations/layer6_attention_weight_min": -55.233070373535156,
      "activations/layer7_attention_weight_max": 81.36898040771484,
      "activations/layer7_attention_weight_min": -81.08027648925781,
      "activations/layer8_attention_weight_max": 41.77827072143555,
      "activations/layer8_attention_weight_min": -38.55562210083008,
      "activations/layer9_attention_weight_max": 36.731346130371094,
      "activations/layer9_attention_weight_min": -34.064884185791016,
      "epoch": 4.77,
      "learning_rate": 0.00012042916666666666,
      "loss": 2.8724,
      "step": 82150
    },
    {
      "activations/layer0_attention_weight_max": 17.486053466796875,
      "activations/layer0_attention_weight_min": -12.021282196044922,
      "activations/layer10_attention_weight_max": 32.332725524902344,
      "activations/layer10_attention_weight_min": -29.29815673828125,
      "activations/layer11_attention_weight_max": 32.71379089355469,
      "activations/layer11_attention_weight_min": -27.84642791748047,
      "activations/layer12_attention_weight_max": 30.833906173706055,
      "activations/layer12_attention_weight_min": -28.578781127929688,
      "activations/layer13_attention_weight_max": 45.881465911865234,
      "activations/layer13_attention_weight_min": -31.388513565063477,
      "activations/layer14_attention_weight_max": 58.48337173461914,
      "activations/layer14_attention_weight_min": -44.13515853881836,
      "activations/layer15_attention_weight_max": 45.15340805053711,
      "activations/layer15_attention_weight_min": -26.277063369750977,
      "activations/layer16_attention_weight_max": 39.11589050292969,
      "activations/layer16_attention_weight_min": -26.769603729248047,
      "activations/layer17_attention_weight_max": 44.30097198486328,
      "activations/layer17_attention_weight_min": -23.796201705932617,
      "activations/layer18_attention_weight_max": 32.82609176635742,
      "activations/layer18_attention_weight_min": -19.1313533782959,
      "activations/layer19_attention_weight_max": 41.09268569946289,
      "activations/layer19_attention_weight_min": -27.516462326049805,
      "activations/layer1_attention_weight_max": 16.974655151367188,
      "activations/layer1_attention_weight_min": -16.14760398864746,
      "activations/layer20_attention_weight_max": 35.937835693359375,
      "activations/layer20_attention_weight_min": -22.89957618713379,
      "activations/layer21_attention_weight_max": 32.565284729003906,
      "activations/layer21_attention_weight_min": -20.17122459411621,
      "activations/layer22_attention_weight_max": 52.10041046142578,
      "activations/layer22_attention_weight_min": -26.26038932800293,
      "activations/layer23_attention_weight_max": 35.3731689453125,
      "activations/layer23_attention_weight_min": -20.064987182617188,
      "activations/layer2_attention_weight_max": 34.73110580444336,
      "activations/layer2_attention_weight_min": -32.213218688964844,
      "activations/layer3_attention_weight_max": 89.3053207397461,
      "activations/layer3_attention_weight_min": -90.93827056884766,
      "activations/layer4_attention_weight_max": 133.6312713623047,
      "activations/layer4_attention_weight_min": -125.89803314208984,
      "activations/layer5_attention_weight_max": 87.67662048339844,
      "activations/layer5_attention_weight_min": -80.44880676269531,
      "activations/layer6_attention_weight_max": 60.91934585571289,
      "activations/layer6_attention_weight_min": -56.169864654541016,
      "activations/layer7_attention_weight_max": 86.8275146484375,
      "activations/layer7_attention_weight_min": -79.84496307373047,
      "activations/layer8_attention_weight_max": 40.300872802734375,
      "activations/layer8_attention_weight_min": -36.911705017089844,
      "activations/layer9_attention_weight_max": 36.05291748046875,
      "activations/layer9_attention_weight_min": -36.65945816040039,
      "epoch": 4.78,
      "learning_rate": 0.00012041022727272726,
      "loss": 2.8501,
      "step": 82200
    },
    {
      "activations/layer0_attention_weight_max": 16.368635177612305,
      "activations/layer0_attention_weight_min": -11.928899765014648,
      "activations/layer10_attention_weight_max": 37.96430206298828,
      "activations/layer10_attention_weight_min": -33.683448791503906,
      "activations/layer11_attention_weight_max": 37.85564041137695,
      "activations/layer11_attention_weight_min": -31.80364227294922,
      "activations/layer12_attention_weight_max": 33.86399841308594,
      "activations/layer12_attention_weight_min": -28.030405044555664,
      "activations/layer13_attention_weight_max": 50.12324142456055,
      "activations/layer13_attention_weight_min": -35.54644012451172,
      "activations/layer14_attention_weight_max": 59.593299865722656,
      "activations/layer14_attention_weight_min": -44.83115768432617,
      "activations/layer15_attention_weight_max": 38.890499114990234,
      "activations/layer15_attention_weight_min": -32.449058532714844,
      "activations/layer16_attention_weight_max": 38.431610107421875,
      "activations/layer16_attention_weight_min": -27.477741241455078,
      "activations/layer17_attention_weight_max": 36.16378402709961,
      "activations/layer17_attention_weight_min": -23.184982299804688,
      "activations/layer18_attention_weight_max": 30.313146591186523,
      "activations/layer18_attention_weight_min": -19.952804565429688,
      "activations/layer19_attention_weight_max": 39.59163284301758,
      "activations/layer19_attention_weight_min": -30.577817916870117,
      "activations/layer1_attention_weight_max": 17.76479148864746,
      "activations/layer1_attention_weight_min": -16.211227416992188,
      "activations/layer20_attention_weight_max": 34.49786376953125,
      "activations/layer20_attention_weight_min": -25.59636688232422,
      "activations/layer21_attention_weight_max": 32.32500457763672,
      "activations/layer21_attention_weight_min": -21.4161434173584,
      "activations/layer22_attention_weight_max": 50.4763298034668,
      "activations/layer22_attention_weight_min": -28.091827392578125,
      "activations/layer23_attention_weight_max": 34.1688232421875,
      "activations/layer23_attention_weight_min": -22.42030906677246,
      "activations/layer2_attention_weight_max": 34.87580871582031,
      "activations/layer2_attention_weight_min": -31.414993286132812,
      "activations/layer3_attention_weight_max": 91.26274871826172,
      "activations/layer3_attention_weight_min": -91.96387481689453,
      "activations/layer4_attention_weight_max": 136.57464599609375,
      "activations/layer4_attention_weight_min": -130.36788940429688,
      "activations/layer5_attention_weight_max": 85.64918518066406,
      "activations/layer5_attention_weight_min": -84.64103698730469,
      "activations/layer6_attention_weight_max": 59.27875518798828,
      "activations/layer6_attention_weight_min": -60.37455368041992,
      "activations/layer7_attention_weight_max": 97.69341278076172,
      "activations/layer7_attention_weight_min": -87.14730072021484,
      "activations/layer8_attention_weight_max": 46.95137405395508,
      "activations/layer8_attention_weight_min": -43.3135871887207,
      "activations/layer9_attention_weight_max": 41.02970504760742,
      "activations/layer9_attention_weight_min": -37.12263107299805,
      "epoch": 4.78,
      "learning_rate": 0.00012039128787878787,
      "loss": 2.8517,
      "step": 82250
    },
    {
      "activations/layer0_attention_weight_max": 17.226167678833008,
      "activations/layer0_attention_weight_min": -11.110620498657227,
      "activations/layer10_attention_weight_max": 32.981773376464844,
      "activations/layer10_attention_weight_min": -30.48134422302246,
      "activations/layer11_attention_weight_max": 31.39282989501953,
      "activations/layer11_attention_weight_min": -27.18288230895996,
      "activations/layer12_attention_weight_max": 29.870851516723633,
      "activations/layer12_attention_weight_min": -27.23385238647461,
      "activations/layer13_attention_weight_max": 44.68962860107422,
      "activations/layer13_attention_weight_min": -33.5681037902832,
      "activations/layer14_attention_weight_max": 67.72624969482422,
      "activations/layer14_attention_weight_min": -42.91724395751953,
      "activations/layer15_attention_weight_max": 37.894718170166016,
      "activations/layer15_attention_weight_min": -26.886560440063477,
      "activations/layer16_attention_weight_max": 39.671146392822266,
      "activations/layer16_attention_weight_min": -28.346269607543945,
      "activations/layer17_attention_weight_max": 37.19767379760742,
      "activations/layer17_attention_weight_min": -24.091106414794922,
      "activations/layer18_attention_weight_max": 33.403717041015625,
      "activations/layer18_attention_weight_min": -22.044506072998047,
      "activations/layer19_attention_weight_max": 47.90312194824219,
      "activations/layer19_attention_weight_min": -29.90132713317871,
      "activations/layer1_attention_weight_max": 17.608400344848633,
      "activations/layer1_attention_weight_min": -18.183439254760742,
      "activations/layer20_attention_weight_max": 31.486005783081055,
      "activations/layer20_attention_weight_min": -24.52351951599121,
      "activations/layer21_attention_weight_max": 30.840885162353516,
      "activations/layer21_attention_weight_min": -20.83966064453125,
      "activations/layer22_attention_weight_max": 52.95828628540039,
      "activations/layer22_attention_weight_min": -28.063156127929688,
      "activations/layer23_attention_weight_max": 32.696693420410156,
      "activations/layer23_attention_weight_min": -19.16507339477539,
      "activations/layer2_attention_weight_max": 34.151153564453125,
      "activations/layer2_attention_weight_min": -32.18479919433594,
      "activations/layer3_attention_weight_max": 90.4195327758789,
      "activations/layer3_attention_weight_min": -91.33130645751953,
      "activations/layer4_attention_weight_max": 140.0238037109375,
      "activations/layer4_attention_weight_min": -130.1134796142578,
      "activations/layer5_attention_weight_max": 94.49655151367188,
      "activations/layer5_attention_weight_min": -88.33494567871094,
      "activations/layer6_attention_weight_max": 59.581642150878906,
      "activations/layer6_attention_weight_min": -63.01514434814453,
      "activations/layer7_attention_weight_max": 84.29637908935547,
      "activations/layer7_attention_weight_min": -81.45475769042969,
      "activations/layer8_attention_weight_max": 41.18034362792969,
      "activations/layer8_attention_weight_min": -39.08433151245117,
      "activations/layer9_attention_weight_max": 40.584922790527344,
      "activations/layer9_attention_weight_min": -34.14937973022461,
      "epoch": 4.78,
      "learning_rate": 0.00012037234848484847,
      "loss": 2.8728,
      "step": 82300
    },
    {
      "activations/layer0_attention_weight_max": 15.028966903686523,
      "activations/layer0_attention_weight_min": -10.81552505493164,
      "activations/layer10_attention_weight_max": 36.14808654785156,
      "activations/layer10_attention_weight_min": -29.86437225341797,
      "activations/layer11_attention_weight_max": 36.53981399536133,
      "activations/layer11_attention_weight_min": -25.82137680053711,
      "activations/layer12_attention_weight_max": 47.37129211425781,
      "activations/layer12_attention_weight_min": -32.95265579223633,
      "activations/layer13_attention_weight_max": 45.392391204833984,
      "activations/layer13_attention_weight_min": -31.967004776000977,
      "activations/layer14_attention_weight_max": 65.6656265258789,
      "activations/layer14_attention_weight_min": -46.11268997192383,
      "activations/layer15_attention_weight_max": 41.08619689941406,
      "activations/layer15_attention_weight_min": -25.623737335205078,
      "activations/layer16_attention_weight_max": 38.394798278808594,
      "activations/layer16_attention_weight_min": -26.375844955444336,
      "activations/layer17_attention_weight_max": 37.180694580078125,
      "activations/layer17_attention_weight_min": -23.262182235717773,
      "activations/layer18_attention_weight_max": 30.456905364990234,
      "activations/layer18_attention_weight_min": -19.820613861083984,
      "activations/layer19_attention_weight_max": 40.345802307128906,
      "activations/layer19_attention_weight_min": -26.670276641845703,
      "activations/layer1_attention_weight_max": 17.779146194458008,
      "activations/layer1_attention_weight_min": -13.670307159423828,
      "activations/layer20_attention_weight_max": 31.316749572753906,
      "activations/layer20_attention_weight_min": -21.339811325073242,
      "activations/layer21_attention_weight_max": 32.31049346923828,
      "activations/layer21_attention_weight_min": -20.78660011291504,
      "activations/layer22_attention_weight_max": 46.19890213012695,
      "activations/layer22_attention_weight_min": -27.71996307373047,
      "activations/layer23_attention_weight_max": 30.869047164916992,
      "activations/layer23_attention_weight_min": -19.03838539123535,
      "activations/layer2_attention_weight_max": 31.924617767333984,
      "activations/layer2_attention_weight_min": -31.754653930664062,
      "activations/layer3_attention_weight_max": 91.14567565917969,
      "activations/layer3_attention_weight_min": -88.90943908691406,
      "activations/layer4_attention_weight_max": 136.91354370117188,
      "activations/layer4_attention_weight_min": -127.00105285644531,
      "activations/layer5_attention_weight_max": 84.14268493652344,
      "activations/layer5_attention_weight_min": -82.08478546142578,
      "activations/layer6_attention_weight_max": 54.904266357421875,
      "activations/layer6_attention_weight_min": -54.94651412963867,
      "activations/layer7_attention_weight_max": 82.10785675048828,
      "activations/layer7_attention_weight_min": -83.53872680664062,
      "activations/layer8_attention_weight_max": 41.09163284301758,
      "activations/layer8_attention_weight_min": -38.77464294433594,
      "activations/layer9_attention_weight_max": 38.291019439697266,
      "activations/layer9_attention_weight_min": -40.93971252441406,
      "epoch": 4.78,
      "learning_rate": 0.00012035340909090907,
      "loss": 2.8702,
      "step": 82350
    },
    {
      "activations/layer0_attention_weight_max": 16.12314224243164,
      "activations/layer0_attention_weight_min": -12.651590347290039,
      "activations/layer10_attention_weight_max": 42.25815963745117,
      "activations/layer10_attention_weight_min": -32.42808532714844,
      "activations/layer11_attention_weight_max": 43.75879669189453,
      "activations/layer11_attention_weight_min": -27.736845016479492,
      "activations/layer12_attention_weight_max": 46.40878677368164,
      "activations/layer12_attention_weight_min": -26.72463607788086,
      "activations/layer13_attention_weight_max": 55.7750129699707,
      "activations/layer13_attention_weight_min": -33.557762145996094,
      "activations/layer14_attention_weight_max": 83.35745239257812,
      "activations/layer14_attention_weight_min": -51.21860122680664,
      "activations/layer15_attention_weight_max": 51.71995162963867,
      "activations/layer15_attention_weight_min": -30.993345260620117,
      "activations/layer16_attention_weight_max": 48.46499252319336,
      "activations/layer16_attention_weight_min": -28.72674560546875,
      "activations/layer17_attention_weight_max": 49.70389175415039,
      "activations/layer17_attention_weight_min": -24.700586318969727,
      "activations/layer18_attention_weight_max": 41.165924072265625,
      "activations/layer18_attention_weight_min": -22.647186279296875,
      "activations/layer19_attention_weight_max": 50.347042083740234,
      "activations/layer19_attention_weight_min": -31.542316436767578,
      "activations/layer1_attention_weight_max": 17.536602020263672,
      "activations/layer1_attention_weight_min": -15.605483055114746,
      "activations/layer20_attention_weight_max": 47.4287223815918,
      "activations/layer20_attention_weight_min": -26.79559898376465,
      "activations/layer21_attention_weight_max": 42.36024856567383,
      "activations/layer21_attention_weight_min": -20.508758544921875,
      "activations/layer22_attention_weight_max": 63.31614303588867,
      "activations/layer22_attention_weight_min": -27.00983238220215,
      "activations/layer23_attention_weight_max": 39.93828582763672,
      "activations/layer23_attention_weight_min": -19.467411041259766,
      "activations/layer2_attention_weight_max": 30.83662986755371,
      "activations/layer2_attention_weight_min": -30.23495101928711,
      "activations/layer3_attention_weight_max": 84.90475463867188,
      "activations/layer3_attention_weight_min": -88.00311279296875,
      "activations/layer4_attention_weight_max": 128.1611328125,
      "activations/layer4_attention_weight_min": -125.24737548828125,
      "activations/layer5_attention_weight_max": 82.41770935058594,
      "activations/layer5_attention_weight_min": -78.10531616210938,
      "activations/layer6_attention_weight_max": 59.489952087402344,
      "activations/layer6_attention_weight_min": -56.75181198120117,
      "activations/layer7_attention_weight_max": 83.84031677246094,
      "activations/layer7_attention_weight_min": -86.31714630126953,
      "activations/layer8_attention_weight_max": 43.92262268066406,
      "activations/layer8_attention_weight_min": -42.26247024536133,
      "activations/layer9_attention_weight_max": 41.6143913269043,
      "activations/layer9_attention_weight_min": -36.1423454284668,
      "epoch": 4.79,
      "learning_rate": 0.00012033446969696969,
      "loss": 2.8665,
      "step": 82400
    },
    {
      "activations/layer0_attention_weight_max": 16.783340454101562,
      "activations/layer0_attention_weight_min": -13.032411575317383,
      "activations/layer10_attention_weight_max": 33.8100471496582,
      "activations/layer10_attention_weight_min": -30.441890716552734,
      "activations/layer11_attention_weight_max": 31.78394889831543,
      "activations/layer11_attention_weight_min": -29.031909942626953,
      "activations/layer12_attention_weight_max": 42.30288314819336,
      "activations/layer12_attention_weight_min": -27.59412384033203,
      "activations/layer13_attention_weight_max": 42.44696044921875,
      "activations/layer13_attention_weight_min": -33.2078857421875,
      "activations/layer14_attention_weight_max": 75.44135284423828,
      "activations/layer14_attention_weight_min": -44.919921875,
      "activations/layer15_attention_weight_max": 40.33265686035156,
      "activations/layer15_attention_weight_min": -34.03786849975586,
      "activations/layer16_attention_weight_max": 36.81098175048828,
      "activations/layer16_attention_weight_min": -30.99846076965332,
      "activations/layer17_attention_weight_max": 36.5941047668457,
      "activations/layer17_attention_weight_min": -25.86833381652832,
      "activations/layer18_attention_weight_max": 30.759883880615234,
      "activations/layer18_attention_weight_min": -20.022762298583984,
      "activations/layer19_attention_weight_max": 39.235382080078125,
      "activations/layer19_attention_weight_min": -27.559879302978516,
      "activations/layer1_attention_weight_max": 18.154020309448242,
      "activations/layer1_attention_weight_min": -14.991024017333984,
      "activations/layer20_attention_weight_max": 32.412654876708984,
      "activations/layer20_attention_weight_min": -23.396459579467773,
      "activations/layer21_attention_weight_max": 34.45243835449219,
      "activations/layer21_attention_weight_min": -19.406028747558594,
      "activations/layer22_attention_weight_max": 50.58543014526367,
      "activations/layer22_attention_weight_min": -27.895776748657227,
      "activations/layer23_attention_weight_max": 31.982528686523438,
      "activations/layer23_attention_weight_min": -23.204586029052734,
      "activations/layer2_attention_weight_max": 30.46184539794922,
      "activations/layer2_attention_weight_min": -30.95712661743164,
      "activations/layer3_attention_weight_max": 90.00798034667969,
      "activations/layer3_attention_weight_min": -93.615234375,
      "activations/layer4_attention_weight_max": 132.19715881347656,
      "activations/layer4_attention_weight_min": -131.60751342773438,
      "activations/layer5_attention_weight_max": 85.60546875,
      "activations/layer5_attention_weight_min": -84.27886962890625,
      "activations/layer6_attention_weight_max": 56.935184478759766,
      "activations/layer6_attention_weight_min": -58.30767822265625,
      "activations/layer7_attention_weight_max": 83.68233489990234,
      "activations/layer7_attention_weight_min": -82.85411834716797,
      "activations/layer8_attention_weight_max": 39.5238037109375,
      "activations/layer8_attention_weight_min": -37.70098876953125,
      "activations/layer9_attention_weight_max": 39.94301986694336,
      "activations/layer9_attention_weight_min": -36.53009033203125,
      "epoch": 4.79,
      "learning_rate": 0.0001203155303030303,
      "loss": 2.8661,
      "step": 82450
    },
    {
      "activations/layer0_attention_weight_max": 17.61305046081543,
      "activations/layer0_attention_weight_min": -12.186352729797363,
      "activations/layer10_attention_weight_max": 32.413639068603516,
      "activations/layer10_attention_weight_min": -31.7436580657959,
      "activations/layer11_attention_weight_max": 30.512557983398438,
      "activations/layer11_attention_weight_min": -28.307109832763672,
      "activations/layer12_attention_weight_max": 44.986061096191406,
      "activations/layer12_attention_weight_min": -30.991350173950195,
      "activations/layer13_attention_weight_max": 56.227630615234375,
      "activations/layer13_attention_weight_min": -42.71064376831055,
      "activations/layer14_attention_weight_max": 78.66361236572266,
      "activations/layer14_attention_weight_min": -58.60969161987305,
      "activations/layer15_attention_weight_max": 46.752288818359375,
      "activations/layer15_attention_weight_min": -27.578731536865234,
      "activations/layer16_attention_weight_max": 38.00432586669922,
      "activations/layer16_attention_weight_min": -28.53917694091797,
      "activations/layer17_attention_weight_max": 42.869808197021484,
      "activations/layer17_attention_weight_min": -25.674278259277344,
      "activations/layer18_attention_weight_max": 30.18502426147461,
      "activations/layer18_attention_weight_min": -20.350542068481445,
      "activations/layer19_attention_weight_max": 40.860870361328125,
      "activations/layer19_attention_weight_min": -30.524723052978516,
      "activations/layer1_attention_weight_max": 17.926738739013672,
      "activations/layer1_attention_weight_min": -15.285595893859863,
      "activations/layer20_attention_weight_max": 35.66701889038086,
      "activations/layer20_attention_weight_min": -22.706392288208008,
      "activations/layer21_attention_weight_max": 30.889211654663086,
      "activations/layer21_attention_weight_min": -23.050403594970703,
      "activations/layer22_attention_weight_max": 45.42544937133789,
      "activations/layer22_attention_weight_min": -26.793415069580078,
      "activations/layer23_attention_weight_max": 30.23487663269043,
      "activations/layer23_attention_weight_min": -20.11772346496582,
      "activations/layer2_attention_weight_max": 30.37783432006836,
      "activations/layer2_attention_weight_min": -32.03450393676758,
      "activations/layer3_attention_weight_max": 90.55708312988281,
      "activations/layer3_attention_weight_min": -90.55024719238281,
      "activations/layer4_attention_weight_max": 130.43154907226562,
      "activations/layer4_attention_weight_min": -131.79852294921875,
      "activations/layer5_attention_weight_max": 85.53740692138672,
      "activations/layer5_attention_weight_min": -85.0702896118164,
      "activations/layer6_attention_weight_max": 56.89236831665039,
      "activations/layer6_attention_weight_min": -57.971961975097656,
      "activations/layer7_attention_weight_max": 89.10421752929688,
      "activations/layer7_attention_weight_min": -81.258056640625,
      "activations/layer8_attention_weight_max": 39.61348342895508,
      "activations/layer8_attention_weight_min": -41.45532989501953,
      "activations/layer9_attention_weight_max": 34.756832122802734,
      "activations/layer9_attention_weight_min": -34.71259689331055,
      "epoch": 4.79,
      "learning_rate": 0.00012029659090909089,
      "loss": 2.8558,
      "step": 82500
    },
    {
      "activations/layer0_attention_weight_max": 17.048202514648438,
      "activations/layer0_attention_weight_min": -11.765097618103027,
      "activations/layer10_attention_weight_max": 38.20701217651367,
      "activations/layer10_attention_weight_min": -31.381572723388672,
      "activations/layer11_attention_weight_max": 38.12340545654297,
      "activations/layer11_attention_weight_min": -30.067588806152344,
      "activations/layer12_attention_weight_max": 30.380414962768555,
      "activations/layer12_attention_weight_min": -25.522672653198242,
      "activations/layer13_attention_weight_max": 44.78994369506836,
      "activations/layer13_attention_weight_min": -32.66978454589844,
      "activations/layer14_attention_weight_max": 65.48764038085938,
      "activations/layer14_attention_weight_min": -44.69633865356445,
      "activations/layer15_attention_weight_max": 43.78765106201172,
      "activations/layer15_attention_weight_min": -29.397571563720703,
      "activations/layer16_attention_weight_max": 37.849037170410156,
      "activations/layer16_attention_weight_min": -29.569849014282227,
      "activations/layer17_attention_weight_max": 39.17051315307617,
      "activations/layer17_attention_weight_min": -24.97710609436035,
      "activations/layer18_attention_weight_max": 31.266569137573242,
      "activations/layer18_attention_weight_min": -19.361907958984375,
      "activations/layer19_attention_weight_max": 40.94798278808594,
      "activations/layer19_attention_weight_min": -30.828603744506836,
      "activations/layer1_attention_weight_max": 17.1843204498291,
      "activations/layer1_attention_weight_min": -14.681038856506348,
      "activations/layer20_attention_weight_max": 32.27540588378906,
      "activations/layer20_attention_weight_min": -24.144298553466797,
      "activations/layer21_attention_weight_max": 33.64630889892578,
      "activations/layer21_attention_weight_min": -21.170576095581055,
      "activations/layer22_attention_weight_max": 51.43860626220703,
      "activations/layer22_attention_weight_min": -30.56483268737793,
      "activations/layer23_attention_weight_max": 31.46725845336914,
      "activations/layer23_attention_weight_min": -20.882946014404297,
      "activations/layer2_attention_weight_max": 30.572654724121094,
      "activations/layer2_attention_weight_min": -31.7578182220459,
      "activations/layer3_attention_weight_max": 90.92491149902344,
      "activations/layer3_attention_weight_min": -89.82918548583984,
      "activations/layer4_attention_weight_max": 129.9653778076172,
      "activations/layer4_attention_weight_min": -129.18014526367188,
      "activations/layer5_attention_weight_max": 84.44482421875,
      "activations/layer5_attention_weight_min": -81.27276611328125,
      "activations/layer6_attention_weight_max": 58.86185836791992,
      "activations/layer6_attention_weight_min": -56.5709114074707,
      "activations/layer7_attention_weight_max": 82.45035552978516,
      "activations/layer7_attention_weight_min": -86.88583374023438,
      "activations/layer8_attention_weight_max": 42.229331970214844,
      "activations/layer8_attention_weight_min": -38.911529541015625,
      "activations/layer9_attention_weight_max": 41.25080871582031,
      "activations/layer9_attention_weight_min": -34.49072265625,
      "epoch": 4.8,
      "learning_rate": 0.00012027765151515151,
      "loss": 2.8607,
      "step": 82550
    },
    {
      "activations/layer0_attention_weight_max": 16.779651641845703,
      "activations/layer0_attention_weight_min": -11.581357955932617,
      "activations/layer10_attention_weight_max": 36.48884963989258,
      "activations/layer10_attention_weight_min": -29.840282440185547,
      "activations/layer11_attention_weight_max": 36.858154296875,
      "activations/layer11_attention_weight_min": -28.306835174560547,
      "activations/layer12_attention_weight_max": 39.885963439941406,
      "activations/layer12_attention_weight_min": -26.60517120361328,
      "activations/layer13_attention_weight_max": 47.98074722290039,
      "activations/layer13_attention_weight_min": -35.58198547363281,
      "activations/layer14_attention_weight_max": 75.56564331054688,
      "activations/layer14_attention_weight_min": -54.765380859375,
      "activations/layer15_attention_weight_max": 44.120628356933594,
      "activations/layer15_attention_weight_min": -26.44198989868164,
      "activations/layer16_attention_weight_max": 36.84577178955078,
      "activations/layer16_attention_weight_min": -28.93178367614746,
      "activations/layer17_attention_weight_max": 39.56062698364258,
      "activations/layer17_attention_weight_min": -25.010046005249023,
      "activations/layer18_attention_weight_max": 30.37261962890625,
      "activations/layer18_attention_weight_min": -19.975032806396484,
      "activations/layer19_attention_weight_max": 40.63526916503906,
      "activations/layer19_attention_weight_min": -27.707380294799805,
      "activations/layer1_attention_weight_max": 17.370685577392578,
      "activations/layer1_attention_weight_min": -13.596229553222656,
      "activations/layer20_attention_weight_max": 31.582046508789062,
      "activations/layer20_attention_weight_min": -23.09868049621582,
      "activations/layer21_attention_weight_max": 33.79976272583008,
      "activations/layer21_attention_weight_min": -19.87795639038086,
      "activations/layer22_attention_weight_max": 49.52643585205078,
      "activations/layer22_attention_weight_min": -28.705949783325195,
      "activations/layer23_attention_weight_max": 32.475341796875,
      "activations/layer23_attention_weight_min": -20.33545684814453,
      "activations/layer2_attention_weight_max": 31.19736099243164,
      "activations/layer2_attention_weight_min": -31.92041778564453,
      "activations/layer3_attention_weight_max": 86.13896179199219,
      "activations/layer3_attention_weight_min": -89.42999267578125,
      "activations/layer4_attention_weight_max": 133.5714874267578,
      "activations/layer4_attention_weight_min": -132.3674774169922,
      "activations/layer5_attention_weight_max": 86.76313781738281,
      "activations/layer5_attention_weight_min": -80.03498077392578,
      "activations/layer6_attention_weight_max": 56.6569709777832,
      "activations/layer6_attention_weight_min": -55.66316223144531,
      "activations/layer7_attention_weight_max": 82.18525695800781,
      "activations/layer7_attention_weight_min": -83.21859741210938,
      "activations/layer8_attention_weight_max": 41.37553787231445,
      "activations/layer8_attention_weight_min": -41.47719955444336,
      "activations/layer9_attention_weight_max": 38.407012939453125,
      "activations/layer9_attention_weight_min": -32.93143081665039,
      "epoch": 4.8,
      "learning_rate": 0.00012025871212121211,
      "loss": 2.856,
      "step": 82600
    },
    {
      "activations/layer0_attention_weight_max": 17.94062614440918,
      "activations/layer0_attention_weight_min": -11.87697696685791,
      "activations/layer10_attention_weight_max": 33.716102600097656,
      "activations/layer10_attention_weight_min": -29.368698120117188,
      "activations/layer11_attention_weight_max": 32.51634979248047,
      "activations/layer11_attention_weight_min": -27.79877281188965,
      "activations/layer12_attention_weight_max": 49.89865493774414,
      "activations/layer12_attention_weight_min": -31.19603157043457,
      "activations/layer13_attention_weight_max": 46.55177688598633,
      "activations/layer13_attention_weight_min": -30.137666702270508,
      "activations/layer14_attention_weight_max": 66.3951416015625,
      "activations/layer14_attention_weight_min": -44.71136474609375,
      "activations/layer15_attention_weight_max": 39.86992263793945,
      "activations/layer15_attention_weight_min": -26.707326889038086,
      "activations/layer16_attention_weight_max": 36.37881851196289,
      "activations/layer16_attention_weight_min": -28.42548942565918,
      "activations/layer17_attention_weight_max": 36.30131149291992,
      "activations/layer17_attention_weight_min": -24.325313568115234,
      "activations/layer18_attention_weight_max": 28.228708267211914,
      "activations/layer18_attention_weight_min": -22.350500106811523,
      "activations/layer19_attention_weight_max": 40.01176452636719,
      "activations/layer19_attention_weight_min": -29.656219482421875,
      "activations/layer1_attention_weight_max": 17.54530906677246,
      "activations/layer1_attention_weight_min": -14.50704288482666,
      "activations/layer20_attention_weight_max": 34.33949279785156,
      "activations/layer20_attention_weight_min": -23.7015438079834,
      "activations/layer21_attention_weight_max": 34.145423889160156,
      "activations/layer21_attention_weight_min": -21.063024520874023,
      "activations/layer22_attention_weight_max": 48.283782958984375,
      "activations/layer22_attention_weight_min": -27.928464889526367,
      "activations/layer23_attention_weight_max": 32.01542282104492,
      "activations/layer23_attention_weight_min": -19.6512451171875,
      "activations/layer2_attention_weight_max": 31.701976776123047,
      "activations/layer2_attention_weight_min": -30.79962730407715,
      "activations/layer3_attention_weight_max": 87.1054458618164,
      "activations/layer3_attention_weight_min": -91.13148498535156,
      "activations/layer4_attention_weight_max": 130.64739990234375,
      "activations/layer4_attention_weight_min": -126.51234436035156,
      "activations/layer5_attention_weight_max": 80.06819915771484,
      "activations/layer5_attention_weight_min": -79.77626037597656,
      "activations/layer6_attention_weight_max": 55.5386848449707,
      "activations/layer6_attention_weight_min": -55.151309967041016,
      "activations/layer7_attention_weight_max": 82.1082992553711,
      "activations/layer7_attention_weight_min": -86.22020721435547,
      "activations/layer8_attention_weight_max": 40.81694793701172,
      "activations/layer8_attention_weight_min": -40.44643020629883,
      "activations/layer9_attention_weight_max": 36.09593963623047,
      "activations/layer9_attention_weight_min": -35.933162689208984,
      "epoch": 4.8,
      "learning_rate": 0.00012023977272727271,
      "loss": 2.8471,
      "step": 82650
    },
    {
      "activations/layer0_attention_weight_max": 17.10384750366211,
      "activations/layer0_attention_weight_min": -11.597494125366211,
      "activations/layer10_attention_weight_max": 43.5604133605957,
      "activations/layer10_attention_weight_min": -35.73390579223633,
      "activations/layer11_attention_weight_max": 43.30169677734375,
      "activations/layer11_attention_weight_min": -32.952064514160156,
      "activations/layer12_attention_weight_max": 47.766815185546875,
      "activations/layer12_attention_weight_min": -26.352914810180664,
      "activations/layer13_attention_weight_max": 80.0772476196289,
      "activations/layer13_attention_weight_min": -47.2608757019043,
      "activations/layer14_attention_weight_max": 82.39279174804688,
      "activations/layer14_attention_weight_min": -47.91902160644531,
      "activations/layer15_attention_weight_max": 64.19629669189453,
      "activations/layer15_attention_weight_min": -34.280948638916016,
      "activations/layer16_attention_weight_max": 47.19439697265625,
      "activations/layer16_attention_weight_min": -29.08352279663086,
      "activations/layer17_attention_weight_max": 41.632530212402344,
      "activations/layer17_attention_weight_min": -27.363191604614258,
      "activations/layer18_attention_weight_max": 39.36557388305664,
      "activations/layer18_attention_weight_min": -20.96260643005371,
      "activations/layer19_attention_weight_max": 53.84501266479492,
      "activations/layer19_attention_weight_min": -28.837921142578125,
      "activations/layer1_attention_weight_max": 17.986085891723633,
      "activations/layer1_attention_weight_min": -15.831744194030762,
      "activations/layer20_attention_weight_max": 37.94872283935547,
      "activations/layer20_attention_weight_min": -24.012144088745117,
      "activations/layer21_attention_weight_max": 41.85151290893555,
      "activations/layer21_attention_weight_min": -21.087617874145508,
      "activations/layer22_attention_weight_max": 52.84819793701172,
      "activations/layer22_attention_weight_min": -31.375019073486328,
      "activations/layer23_attention_weight_max": 35.236534118652344,
      "activations/layer23_attention_weight_min": -20.381919860839844,
      "activations/layer2_attention_weight_max": 32.65879821777344,
      "activations/layer2_attention_weight_min": -33.15693664550781,
      "activations/layer3_attention_weight_max": 97.58827209472656,
      "activations/layer3_attention_weight_min": -99.77669525146484,
      "activations/layer4_attention_weight_max": 140.9388427734375,
      "activations/layer4_attention_weight_min": -148.23526000976562,
      "activations/layer5_attention_weight_max": 89.07536315917969,
      "activations/layer5_attention_weight_min": -88.48638916015625,
      "activations/layer6_attention_weight_max": 59.58650207519531,
      "activations/layer6_attention_weight_min": -56.7739143371582,
      "activations/layer7_attention_weight_max": 93.88326263427734,
      "activations/layer7_attention_weight_min": -89.18293762207031,
      "activations/layer8_attention_weight_max": 45.183876037597656,
      "activations/layer8_attention_weight_min": -45.21003723144531,
      "activations/layer9_attention_weight_max": 42.051780700683594,
      "activations/layer9_attention_weight_min": -37.53458786010742,
      "epoch": 4.81,
      "learning_rate": 0.00012022083333333333,
      "loss": 2.8546,
      "step": 82700
    },
    {
      "activations/layer0_attention_weight_max": 16.67104148864746,
      "activations/layer0_attention_weight_min": -12.679924011230469,
      "activations/layer10_attention_weight_max": 37.72725296020508,
      "activations/layer10_attention_weight_min": -33.160396575927734,
      "activations/layer11_attention_weight_max": 34.34904479980469,
      "activations/layer11_attention_weight_min": -30.97618865966797,
      "activations/layer12_attention_weight_max": 23.624128341674805,
      "activations/layer12_attention_weight_min": -29.01420021057129,
      "activations/layer13_attention_weight_max": 44.6767463684082,
      "activations/layer13_attention_weight_min": -33.02640151977539,
      "activations/layer14_attention_weight_max": 68.18560028076172,
      "activations/layer14_attention_weight_min": -48.78436279296875,
      "activations/layer15_attention_weight_max": 43.057926177978516,
      "activations/layer15_attention_weight_min": -26.690988540649414,
      "activations/layer16_attention_weight_max": 41.80174255371094,
      "activations/layer16_attention_weight_min": -30.90961456298828,
      "activations/layer17_attention_weight_max": 38.59589767456055,
      "activations/layer17_attention_weight_min": -24.788976669311523,
      "activations/layer18_attention_weight_max": 30.690954208374023,
      "activations/layer18_attention_weight_min": -19.52397346496582,
      "activations/layer19_attention_weight_max": 45.08448028564453,
      "activations/layer19_attention_weight_min": -31.58591079711914,
      "activations/layer1_attention_weight_max": 17.858184814453125,
      "activations/layer1_attention_weight_min": -14.740768432617188,
      "activations/layer20_attention_weight_max": 33.94755935668945,
      "activations/layer20_attention_weight_min": -24.030345916748047,
      "activations/layer21_attention_weight_max": 31.133056640625,
      "activations/layer21_attention_weight_min": -19.812450408935547,
      "activations/layer22_attention_weight_max": 53.71009063720703,
      "activations/layer22_attention_weight_min": -28.13041114807129,
      "activations/layer23_attention_weight_max": 31.27358055114746,
      "activations/layer23_attention_weight_min": -20.85836410522461,
      "activations/layer2_attention_weight_max": 32.679203033447266,
      "activations/layer2_attention_weight_min": -34.614715576171875,
      "activations/layer3_attention_weight_max": 89.54373931884766,
      "activations/layer3_attention_weight_min": -89.28221893310547,
      "activations/layer4_attention_weight_max": 134.0464324951172,
      "activations/layer4_attention_weight_min": -130.01904296875,
      "activations/layer5_attention_weight_max": 88.49571990966797,
      "activations/layer5_attention_weight_min": -83.45036315917969,
      "activations/layer6_attention_weight_max": 58.442230224609375,
      "activations/layer6_attention_weight_min": -58.585899353027344,
      "activations/layer7_attention_weight_max": 81.15913391113281,
      "activations/layer7_attention_weight_min": -86.8171157836914,
      "activations/layer8_attention_weight_max": 42.89169692993164,
      "activations/layer8_attention_weight_min": -39.01548767089844,
      "activations/layer9_attention_weight_max": 41.80477523803711,
      "activations/layer9_attention_weight_min": -34.957698822021484,
      "epoch": 4.81,
      "learning_rate": 0.00012020189393939393,
      "loss": 2.8668,
      "step": 82750
    },
    {
      "activations/layer0_attention_weight_max": 17.44484519958496,
      "activations/layer0_attention_weight_min": -11.357626914978027,
      "activations/layer10_attention_weight_max": 33.546844482421875,
      "activations/layer10_attention_weight_min": -29.656869888305664,
      "activations/layer11_attention_weight_max": 32.59345626831055,
      "activations/layer11_attention_weight_min": -25.456716537475586,
      "activations/layer12_attention_weight_max": 31.422183990478516,
      "activations/layer12_attention_weight_min": -29.427927017211914,
      "activations/layer13_attention_weight_max": 48.903724670410156,
      "activations/layer13_attention_weight_min": -35.27452850341797,
      "activations/layer14_attention_weight_max": 60.0804557800293,
      "activations/layer14_attention_weight_min": -47.045562744140625,
      "activations/layer15_attention_weight_max": 38.390525817871094,
      "activations/layer15_attention_weight_min": -26.41217803955078,
      "activations/layer16_attention_weight_max": 39.337406158447266,
      "activations/layer16_attention_weight_min": -27.23734474182129,
      "activations/layer17_attention_weight_max": 41.38154983520508,
      "activations/layer17_attention_weight_min": -24.9620361328125,
      "activations/layer18_attention_weight_max": 39.14480209350586,
      "activations/layer18_attention_weight_min": -19.65663719177246,
      "activations/layer19_attention_weight_max": 42.667259216308594,
      "activations/layer19_attention_weight_min": -28.678747177124023,
      "activations/layer1_attention_weight_max": 17.67608070373535,
      "activations/layer1_attention_weight_min": -14.925337791442871,
      "activations/layer20_attention_weight_max": 33.15019226074219,
      "activations/layer20_attention_weight_min": -26.054203033447266,
      "activations/layer21_attention_weight_max": 41.483734130859375,
      "activations/layer21_attention_weight_min": -21.52983856201172,
      "activations/layer22_attention_weight_max": 52.96051025390625,
      "activations/layer22_attention_weight_min": -28.42150115966797,
      "activations/layer23_attention_weight_max": 34.527244567871094,
      "activations/layer23_attention_weight_min": -20.06157684326172,
      "activations/layer2_attention_weight_max": 32.134666442871094,
      "activations/layer2_attention_weight_min": -30.134037017822266,
      "activations/layer3_attention_weight_max": 88.31421661376953,
      "activations/layer3_attention_weight_min": -84.89165496826172,
      "activations/layer4_attention_weight_max": 132.80511474609375,
      "activations/layer4_attention_weight_min": -125.25362396240234,
      "activations/layer5_attention_weight_max": 93.25957489013672,
      "activations/layer5_attention_weight_min": -82.7305908203125,
      "activations/layer6_attention_weight_max": 65.46167755126953,
      "activations/layer6_attention_weight_min": -57.22456741333008,
      "activations/layer7_attention_weight_max": 81.69073486328125,
      "activations/layer7_attention_weight_min": -86.72935485839844,
      "activations/layer8_attention_weight_max": 39.40437316894531,
      "activations/layer8_attention_weight_min": -38.50162887573242,
      "activations/layer9_attention_weight_max": 36.644073486328125,
      "activations/layer9_attention_weight_min": -33.33743667602539,
      "epoch": 4.81,
      "learning_rate": 0.00012018295454545453,
      "loss": 2.8543,
      "step": 82800
    },
    {
      "activations/layer0_attention_weight_max": 16.202661514282227,
      "activations/layer0_attention_weight_min": -12.656553268432617,
      "activations/layer10_attention_weight_max": 31.97738265991211,
      "activations/layer10_attention_weight_min": -30.255050659179688,
      "activations/layer11_attention_weight_max": 32.305755615234375,
      "activations/layer11_attention_weight_min": -27.64745330810547,
      "activations/layer12_attention_weight_max": 24.61090087890625,
      "activations/layer12_attention_weight_min": -33.434364318847656,
      "activations/layer13_attention_weight_max": 39.04949951171875,
      "activations/layer13_attention_weight_min": -31.122018814086914,
      "activations/layer14_attention_weight_max": 55.03329849243164,
      "activations/layer14_attention_weight_min": -44.63521957397461,
      "activations/layer15_attention_weight_max": 35.39908981323242,
      "activations/layer15_attention_weight_min": -26.915172576904297,
      "activations/layer16_attention_weight_max": 33.4215202331543,
      "activations/layer16_attention_weight_min": -28.52178382873535,
      "activations/layer17_attention_weight_max": 33.668331146240234,
      "activations/layer17_attention_weight_min": -23.788150787353516,
      "activations/layer18_attention_weight_max": 30.1025390625,
      "activations/layer18_attention_weight_min": -19.3751277923584,
      "activations/layer19_attention_weight_max": 37.16987991333008,
      "activations/layer19_attention_weight_min": -29.89581871032715,
      "activations/layer1_attention_weight_max": 17.023082733154297,
      "activations/layer1_attention_weight_min": -14.374553680419922,
      "activations/layer20_attention_weight_max": 28.711917877197266,
      "activations/layer20_attention_weight_min": -22.939369201660156,
      "activations/layer21_attention_weight_max": 29.924325942993164,
      "activations/layer21_attention_weight_min": -21.706132888793945,
      "activations/layer22_attention_weight_max": 46.251792907714844,
      "activations/layer22_attention_weight_min": -27.570234298706055,
      "activations/layer23_attention_weight_max": 31.306676864624023,
      "activations/layer23_attention_weight_min": -24.617971420288086,
      "activations/layer2_attention_weight_max": 33.17435073852539,
      "activations/layer2_attention_weight_min": -32.1173095703125,
      "activations/layer3_attention_weight_max": 91.39007568359375,
      "activations/layer3_attention_weight_min": -91.20172119140625,
      "activations/layer4_attention_weight_max": 132.81964111328125,
      "activations/layer4_attention_weight_min": -125.45560455322266,
      "activations/layer5_attention_weight_max": 87.86759948730469,
      "activations/layer5_attention_weight_min": -82.04396057128906,
      "activations/layer6_attention_weight_max": 62.80208206176758,
      "activations/layer6_attention_weight_min": -56.8780403137207,
      "activations/layer7_attention_weight_max": 86.4235610961914,
      "activations/layer7_attention_weight_min": -79.28489685058594,
      "activations/layer8_attention_weight_max": 41.591697692871094,
      "activations/layer8_attention_weight_min": -37.657161712646484,
      "activations/layer9_attention_weight_max": 37.400001525878906,
      "activations/layer9_attention_weight_min": -37.1717643737793,
      "epoch": 4.81,
      "learning_rate": 0.00012016401515151514,
      "loss": 2.8313,
      "step": 82850
    },
    {
      "activations/layer0_attention_weight_max": 17.47919273376465,
      "activations/layer0_attention_weight_min": -10.797286033630371,
      "activations/layer10_attention_weight_max": 35.42779541015625,
      "activations/layer10_attention_weight_min": -30.61534881591797,
      "activations/layer11_attention_weight_max": 39.33812713623047,
      "activations/layer11_attention_weight_min": -27.62177085876465,
      "activations/layer12_attention_weight_max": 29.95380210876465,
      "activations/layer12_attention_weight_min": -34.2560920715332,
      "activations/layer13_attention_weight_max": 49.782188415527344,
      "activations/layer13_attention_weight_min": -35.400611877441406,
      "activations/layer14_attention_weight_max": 70.8726806640625,
      "activations/layer14_attention_weight_min": -49.15205383300781,
      "activations/layer15_attention_weight_max": 45.39315414428711,
      "activations/layer15_attention_weight_min": -28.217710494995117,
      "activations/layer16_attention_weight_max": 43.2619743347168,
      "activations/layer16_attention_weight_min": -29.3142032623291,
      "activations/layer17_attention_weight_max": 41.85725402832031,
      "activations/layer17_attention_weight_min": -26.75433921813965,
      "activations/layer18_attention_weight_max": 33.63813400268555,
      "activations/layer18_attention_weight_min": -19.332626342773438,
      "activations/layer19_attention_weight_max": 45.65395736694336,
      "activations/layer19_attention_weight_min": -28.4473934173584,
      "activations/layer1_attention_weight_max": 17.010034561157227,
      "activations/layer1_attention_weight_min": -12.52393913269043,
      "activations/layer20_attention_weight_max": 31.557405471801758,
      "activations/layer20_attention_weight_min": -23.163875579833984,
      "activations/layer21_attention_weight_max": 30.178970336914062,
      "activations/layer21_attention_weight_min": -21.081947326660156,
      "activations/layer22_attention_weight_max": 52.4481201171875,
      "activations/layer22_attention_weight_min": -30.92070770263672,
      "activations/layer23_attention_weight_max": 35.196861267089844,
      "activations/layer23_attention_weight_min": -21.651248931884766,
      "activations/layer2_attention_weight_max": 30.81436538696289,
      "activations/layer2_attention_weight_min": -31.28309440612793,
      "activations/layer3_attention_weight_max": 91.5650405883789,
      "activations/layer3_attention_weight_min": -91.91015625,
      "activations/layer4_attention_weight_max": 130.07757568359375,
      "activations/layer4_attention_weight_min": -129.63746643066406,
      "activations/layer5_attention_weight_max": 81.41415405273438,
      "activations/layer5_attention_weight_min": -78.9857406616211,
      "activations/layer6_attention_weight_max": 55.2153434753418,
      "activations/layer6_attention_weight_min": -57.0662727355957,
      "activations/layer7_attention_weight_max": 85.7609634399414,
      "activations/layer7_attention_weight_min": -81.98123168945312,
      "activations/layer8_attention_weight_max": 39.50806427001953,
      "activations/layer8_attention_weight_min": -39.487083435058594,
      "activations/layer9_attention_weight_max": 37.532127380371094,
      "activations/layer9_attention_weight_min": -45.2265510559082,
      "epoch": 4.82,
      "learning_rate": 0.00012014507575757575,
      "loss": 2.8476,
      "step": 82900
    },
    {
      "activations/layer0_attention_weight_max": 17.262222290039062,
      "activations/layer0_attention_weight_min": -11.887248039245605,
      "activations/layer10_attention_weight_max": 33.028568267822266,
      "activations/layer10_attention_weight_min": -29.732194900512695,
      "activations/layer11_attention_weight_max": 30.823192596435547,
      "activations/layer11_attention_weight_min": -29.481801986694336,
      "activations/layer12_attention_weight_max": 31.817344665527344,
      "activations/layer12_attention_weight_min": -25.48855209350586,
      "activations/layer13_attention_weight_max": 43.70156478881836,
      "activations/layer13_attention_weight_min": -33.63460922241211,
      "activations/layer14_attention_weight_max": 67.5912094116211,
      "activations/layer14_attention_weight_min": -48.59718704223633,
      "activations/layer15_attention_weight_max": 43.86711502075195,
      "activations/layer15_attention_weight_min": -28.667255401611328,
      "activations/layer16_attention_weight_max": 40.6475944519043,
      "activations/layer16_attention_weight_min": -28.206392288208008,
      "activations/layer17_attention_weight_max": 40.799930572509766,
      "activations/layer17_attention_weight_min": -24.216529846191406,
      "activations/layer18_attention_weight_max": 31.904319763183594,
      "activations/layer18_attention_weight_min": -20.086288452148438,
      "activations/layer19_attention_weight_max": 45.78776550292969,
      "activations/layer19_attention_weight_min": -28.792875289916992,
      "activations/layer1_attention_weight_max": 17.022581100463867,
      "activations/layer1_attention_weight_min": -17.211246490478516,
      "activations/layer20_attention_weight_max": 34.52696228027344,
      "activations/layer20_attention_weight_min": -22.58245277404785,
      "activations/layer21_attention_weight_max": 33.75788116455078,
      "activations/layer21_attention_weight_min": -19.692962646484375,
      "activations/layer22_attention_weight_max": 58.30387496948242,
      "activations/layer22_attention_weight_min": -27.93679428100586,
      "activations/layer23_attention_weight_max": 33.51411056518555,
      "activations/layer23_attention_weight_min": -22.05621337890625,
      "activations/layer2_attention_weight_max": 32.140281677246094,
      "activations/layer2_attention_weight_min": -32.5023193359375,
      "activations/layer3_attention_weight_max": 87.25775146484375,
      "activations/layer3_attention_weight_min": -87.46991729736328,
      "activations/layer4_attention_weight_max": 130.54347229003906,
      "activations/layer4_attention_weight_min": -124.29451751708984,
      "activations/layer5_attention_weight_max": 82.03977966308594,
      "activations/layer5_attention_weight_min": -80.60395812988281,
      "activations/layer6_attention_weight_max": 52.89219665527344,
      "activations/layer6_attention_weight_min": -57.06547546386719,
      "activations/layer7_attention_weight_max": 79.46892547607422,
      "activations/layer7_attention_weight_min": -78.73348999023438,
      "activations/layer8_attention_weight_max": 38.97222900390625,
      "activations/layer8_attention_weight_min": -40.42531204223633,
      "activations/layer9_attention_weight_max": 41.06332015991211,
      "activations/layer9_attention_weight_min": -35.68559265136719,
      "epoch": 4.82,
      "learning_rate": 0.00012012613636363636,
      "loss": 2.8435,
      "step": 82950
    },
    {
      "activations/layer0_attention_weight_max": 15.80187702178955,
      "activations/layer0_attention_weight_min": -11.284435272216797,
      "activations/layer10_attention_weight_max": 36.0696907043457,
      "activations/layer10_attention_weight_min": -34.45198440551758,
      "activations/layer11_attention_weight_max": 38.365848541259766,
      "activations/layer11_attention_weight_min": -28.902006149291992,
      "activations/layer12_attention_weight_max": 35.73234176635742,
      "activations/layer12_attention_weight_min": -27.68683624267578,
      "activations/layer13_attention_weight_max": 45.07139587402344,
      "activations/layer13_attention_weight_min": -32.07172775268555,
      "activations/layer14_attention_weight_max": 58.73872756958008,
      "activations/layer14_attention_weight_min": -42.69647979736328,
      "activations/layer15_attention_weight_max": 46.094505310058594,
      "activations/layer15_attention_weight_min": -34.13575744628906,
      "activations/layer16_attention_weight_max": 38.41947937011719,
      "activations/layer16_attention_weight_min": -28.418434143066406,
      "activations/layer17_attention_weight_max": 44.145172119140625,
      "activations/layer17_attention_weight_min": -23.95262336730957,
      "activations/layer18_attention_weight_max": 33.714508056640625,
      "activations/layer18_attention_weight_min": -19.34000015258789,
      "activations/layer19_attention_weight_max": 44.343544006347656,
      "activations/layer19_attention_weight_min": -29.137014389038086,
      "activations/layer1_attention_weight_max": 18.54414939880371,
      "activations/layer1_attention_weight_min": -15.81346321105957,
      "activations/layer20_attention_weight_max": 32.36652374267578,
      "activations/layer20_attention_weight_min": -22.569725036621094,
      "activations/layer21_attention_weight_max": 31.11736297607422,
      "activations/layer21_attention_weight_min": -20.860410690307617,
      "activations/layer22_attention_weight_max": 50.52532196044922,
      "activations/layer22_attention_weight_min": -27.543155670166016,
      "activations/layer23_attention_weight_max": 33.45415496826172,
      "activations/layer23_attention_weight_min": -22.654958724975586,
      "activations/layer2_attention_weight_max": 34.60580062866211,
      "activations/layer2_attention_weight_min": -34.12828826904297,
      "activations/layer3_attention_weight_max": 94.8992691040039,
      "activations/layer3_attention_weight_min": -96.80286407470703,
      "activations/layer4_attention_weight_max": 139.9308319091797,
      "activations/layer4_attention_weight_min": -135.94834899902344,
      "activations/layer5_attention_weight_max": 90.36039733886719,
      "activations/layer5_attention_weight_min": -84.6360092163086,
      "activations/layer6_attention_weight_max": 62.44529342651367,
      "activations/layer6_attention_weight_min": -57.22002410888672,
      "activations/layer7_attention_weight_max": 99.2305908203125,
      "activations/layer7_attention_weight_min": -86.48052978515625,
      "activations/layer8_attention_weight_max": 47.30437088012695,
      "activations/layer8_attention_weight_min": -42.429447174072266,
      "activations/layer9_attention_weight_max": 42.15098190307617,
      "activations/layer9_attention_weight_min": -40.47420883178711,
      "epoch": 4.82,
      "learning_rate": 0.00012010719696969696,
      "loss": 2.8551,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_loss": 2.806640625,
      "eval_runtime": 8.506,
      "eval_samples_per_second": 504.822,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_openwebtext_loss": 2.806640625,
      "eval_openwebtext_ppl": 16.554212894695087,
      "eval_openwebtext_runtime": 8.506,
      "eval_openwebtext_samples_per_second": 504.822,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_wikitext_loss": 3.04296875,
      "eval_wikitext_ppl": 20.967397891114594,
      "eval_wikitext_runtime": 2.0249,
      "eval_wikitext_samples_per_second": 225.202,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_lambada_loss": 2.73046875,
      "eval_lambada_ppl": 15.340075995482293,
      "eval_lambada_runtime": 9.5663,
      "eval_lambada_samples_per_second": 508.972,
      "step": 83000
    },
    {
      "activations/layer0_attention_weight_max": 16.710552215576172,
      "activations/layer0_attention_weight_min": -11.886340141296387,
      "activations/layer10_attention_weight_max": 35.38323974609375,
      "activations/layer10_attention_weight_min": -31.577451705932617,
      "activations/layer11_attention_weight_max": 33.55787658691406,
      "activations/layer11_attention_weight_min": -27.23525047302246,
      "activations/layer12_attention_weight_max": 29.22054672241211,
      "activations/layer12_attention_weight_min": -26.392011642456055,
      "activations/layer13_attention_weight_max": 47.66725540161133,
      "activations/layer13_attention_weight_min": -34.42644500732422,
      "activations/layer14_attention_weight_max": 68.72232055664062,
      "activations/layer14_attention_weight_min": -46.85430908203125,
      "activations/layer15_attention_weight_max": 39.700660705566406,
      "activations/layer15_attention_weight_min": -27.29996681213379,
      "activations/layer16_attention_weight_max": 42.81887435913086,
      "activations/layer16_attention_weight_min": -29.061260223388672,
      "activations/layer17_attention_weight_max": 39.20764923095703,
      "activations/layer17_attention_weight_min": -24.032432556152344,
      "activations/layer18_attention_weight_max": 33.19686508178711,
      "activations/layer18_attention_weight_min": -21.128894805908203,
      "activations/layer19_attention_weight_max": 41.6740837097168,
      "activations/layer19_attention_weight_min": -32.33304977416992,
      "activations/layer1_attention_weight_max": 18.1097469329834,
      "activations/layer1_attention_weight_min": -14.45429801940918,
      "activations/layer20_attention_weight_max": 34.76512145996094,
      "activations/layer20_attention_weight_min": -26.34959602355957,
      "activations/layer21_attention_weight_max": 33.347412109375,
      "activations/layer21_attention_weight_min": -20.994741439819336,
      "activations/layer22_attention_weight_max": 60.49058151245117,
      "activations/layer22_attention_weight_min": -27.439247131347656,
      "activations/layer23_attention_weight_max": 34.81047821044922,
      "activations/layer23_attention_weight_min": -21.335529327392578,
      "activations/layer2_attention_weight_max": 33.44712829589844,
      "activations/layer2_attention_weight_min": -31.981887817382812,
      "activations/layer3_attention_weight_max": 89.97296905517578,
      "activations/layer3_attention_weight_min": -93.53270721435547,
      "activations/layer4_attention_weight_max": 132.97511291503906,
      "activations/layer4_attention_weight_min": -131.9214324951172,
      "activations/layer5_attention_weight_max": 91.13665008544922,
      "activations/layer5_attention_weight_min": -85.08902740478516,
      "activations/layer6_attention_weight_max": 60.98478317260742,
      "activations/layer6_attention_weight_min": -61.19633483886719,
      "activations/layer7_attention_weight_max": 84.66191864013672,
      "activations/layer7_attention_weight_min": -87.71756744384766,
      "activations/layer8_attention_weight_max": 44.10651397705078,
      "activations/layer8_attention_weight_min": -42.61249542236328,
      "activations/layer9_attention_weight_max": 41.26780319213867,
      "activations/layer9_attention_weight_min": -39.7585334777832,
      "epoch": 4.83,
      "learning_rate": 0.00012008825757575756,
      "loss": 2.8536,
      "step": 83050
    },
    {
      "activations/layer0_attention_weight_max": 17.307844161987305,
      "activations/layer0_attention_weight_min": -12.140595436096191,
      "activations/layer10_attention_weight_max": 31.670263290405273,
      "activations/layer10_attention_weight_min": -29.7497615814209,
      "activations/layer11_attention_weight_max": 29.492887496948242,
      "activations/layer11_attention_weight_min": -29.287687301635742,
      "activations/layer12_attention_weight_max": 38.66519546508789,
      "activations/layer12_attention_weight_min": -29.526973724365234,
      "activations/layer13_attention_weight_max": 55.98601150512695,
      "activations/layer13_attention_weight_min": -37.540992736816406,
      "activations/layer14_attention_weight_max": 55.587318420410156,
      "activations/layer14_attention_weight_min": -44.77859878540039,
      "activations/layer15_attention_weight_max": 38.59260177612305,
      "activations/layer15_attention_weight_min": -26.927513122558594,
      "activations/layer16_attention_weight_max": 40.129905700683594,
      "activations/layer16_attention_weight_min": -29.459857940673828,
      "activations/layer17_attention_weight_max": 38.815826416015625,
      "activations/layer17_attention_weight_min": -22.819828033447266,
      "activations/layer18_attention_weight_max": 31.452411651611328,
      "activations/layer18_attention_weight_min": -18.700653076171875,
      "activations/layer19_attention_weight_max": 39.672821044921875,
      "activations/layer19_attention_weight_min": -26.53000831604004,
      "activations/layer1_attention_weight_max": 17.446327209472656,
      "activations/layer1_attention_weight_min": -14.796454429626465,
      "activations/layer20_attention_weight_max": 34.47602844238281,
      "activations/layer20_attention_weight_min": -22.145368576049805,
      "activations/layer21_attention_weight_max": 31.00539779663086,
      "activations/layer21_attention_weight_min": -18.367603302001953,
      "activations/layer22_attention_weight_max": 50.05495834350586,
      "activations/layer22_attention_weight_min": -26.565067291259766,
      "activations/layer23_attention_weight_max": 32.57023239135742,
      "activations/layer23_attention_weight_min": -19.29520034790039,
      "activations/layer2_attention_weight_max": 31.295249938964844,
      "activations/layer2_attention_weight_min": -29.70478630065918,
      "activations/layer3_attention_weight_max": 83.81423950195312,
      "activations/layer3_attention_weight_min": -88.84577941894531,
      "activations/layer4_attention_weight_max": 124.38085174560547,
      "activations/layer4_attention_weight_min": -124.41204833984375,
      "activations/layer5_attention_weight_max": 81.58812713623047,
      "activations/layer5_attention_weight_min": -78.81766510009766,
      "activations/layer6_attention_weight_max": 54.69184494018555,
      "activations/layer6_attention_weight_min": -55.804752349853516,
      "activations/layer7_attention_weight_max": 78.43891906738281,
      "activations/layer7_attention_weight_min": -82.7874984741211,
      "activations/layer8_attention_weight_max": 39.916324615478516,
      "activations/layer8_attention_weight_min": -38.658714294433594,
      "activations/layer9_attention_weight_max": 36.07618713378906,
      "activations/layer9_attention_weight_min": -35.15346908569336,
      "epoch": 4.83,
      "learning_rate": 0.00012006931818181818,
      "loss": 2.8505,
      "step": 83100
    },
    {
      "activations/layer0_attention_weight_max": 17.24070930480957,
      "activations/layer0_attention_weight_min": -12.690201759338379,
      "activations/layer10_attention_weight_max": 31.313735961914062,
      "activations/layer10_attention_weight_min": -29.816301345825195,
      "activations/layer11_attention_weight_max": 29.91714096069336,
      "activations/layer11_attention_weight_min": -27.08059310913086,
      "activations/layer12_attention_weight_max": 27.498050689697266,
      "activations/layer12_attention_weight_min": -27.302030563354492,
      "activations/layer13_attention_weight_max": 39.14269256591797,
      "activations/layer13_attention_weight_min": -31.9793643951416,
      "activations/layer14_attention_weight_max": 59.825653076171875,
      "activations/layer14_attention_weight_min": -49.691917419433594,
      "activations/layer15_attention_weight_max": 37.01645278930664,
      "activations/layer15_attention_weight_min": -29.163015365600586,
      "activations/layer16_attention_weight_max": 35.86440658569336,
      "activations/layer16_attention_weight_min": -30.55255126953125,
      "activations/layer17_attention_weight_max": 34.356258392333984,
      "activations/layer17_attention_weight_min": -25.79313087463379,
      "activations/layer18_attention_weight_max": 32.09178161621094,
      "activations/layer18_attention_weight_min": -21.47101402282715,
      "activations/layer19_attention_weight_max": 42.210514068603516,
      "activations/layer19_attention_weight_min": -29.432493209838867,
      "activations/layer1_attention_weight_max": 17.342052459716797,
      "activations/layer1_attention_weight_min": -18.57016944885254,
      "activations/layer20_attention_weight_max": 32.39405059814453,
      "activations/layer20_attention_weight_min": -24.48797035217285,
      "activations/layer21_attention_weight_max": 31.454259872436523,
      "activations/layer21_attention_weight_min": -21.610450744628906,
      "activations/layer22_attention_weight_max": 48.06680679321289,
      "activations/layer22_attention_weight_min": -27.621721267700195,
      "activations/layer23_attention_weight_max": 31.2135009765625,
      "activations/layer23_attention_weight_min": -20.73563003540039,
      "activations/layer2_attention_weight_max": 32.11049270629883,
      "activations/layer2_attention_weight_min": -31.48836326599121,
      "activations/layer3_attention_weight_max": 89.12976837158203,
      "activations/layer3_attention_weight_min": -93.49007415771484,
      "activations/layer4_attention_weight_max": 131.6118621826172,
      "activations/layer4_attention_weight_min": -122.7168960571289,
      "activations/layer5_attention_weight_max": 82.95925903320312,
      "activations/layer5_attention_weight_min": -78.56621551513672,
      "activations/layer6_attention_weight_max": 57.527862548828125,
      "activations/layer6_attention_weight_min": -56.59089660644531,
      "activations/layer7_attention_weight_max": 82.18563842773438,
      "activations/layer7_attention_weight_min": -88.71216583251953,
      "activations/layer8_attention_weight_max": 38.69822692871094,
      "activations/layer8_attention_weight_min": -39.44334030151367,
      "activations/layer9_attention_weight_max": 36.1865348815918,
      "activations/layer9_attention_weight_min": -31.482013702392578,
      "epoch": 4.83,
      "learning_rate": 0.00012005037878787877,
      "loss": 2.8532,
      "step": 83150
    },
    {
      "activations/layer0_attention_weight_max": 17.0227108001709,
      "activations/layer0_attention_weight_min": -10.896602630615234,
      "activations/layer10_attention_weight_max": 39.709041595458984,
      "activations/layer10_attention_weight_min": -32.29837417602539,
      "activations/layer11_attention_weight_max": 42.487060546875,
      "activations/layer11_attention_weight_min": -32.24468231201172,
      "activations/layer12_attention_weight_max": 37.96223449707031,
      "activations/layer12_attention_weight_min": -27.140562057495117,
      "activations/layer13_attention_weight_max": 62.43393325805664,
      "activations/layer13_attention_weight_min": -37.11491394042969,
      "activations/layer14_attention_weight_max": 64.78667449951172,
      "activations/layer14_attention_weight_min": -45.4924201965332,
      "activations/layer15_attention_weight_max": 44.88325881958008,
      "activations/layer15_attention_weight_min": -28.944704055786133,
      "activations/layer16_attention_weight_max": 41.88936233520508,
      "activations/layer16_attention_weight_min": -28.24214744567871,
      "activations/layer17_attention_weight_max": 37.48264694213867,
      "activations/layer17_attention_weight_min": -24.311628341674805,
      "activations/layer18_attention_weight_max": 29.447189331054688,
      "activations/layer18_attention_weight_min": -19.624631881713867,
      "activations/layer19_attention_weight_max": 49.87348175048828,
      "activations/layer19_attention_weight_min": -30.267606735229492,
      "activations/layer1_attention_weight_max": 17.960779190063477,
      "activations/layer1_attention_weight_min": -14.645124435424805,
      "activations/layer20_attention_weight_max": 36.854148864746094,
      "activations/layer20_attention_weight_min": -24.224361419677734,
      "activations/layer21_attention_weight_max": 30.54954719543457,
      "activations/layer21_attention_weight_min": -20.249435424804688,
      "activations/layer22_attention_weight_max": 49.66302490234375,
      "activations/layer22_attention_weight_min": -28.626834869384766,
      "activations/layer23_attention_weight_max": 30.615392684936523,
      "activations/layer23_attention_weight_min": -18.151382446289062,
      "activations/layer2_attention_weight_max": 32.29517364501953,
      "activations/layer2_attention_weight_min": -32.024147033691406,
      "activations/layer3_attention_weight_max": 90.24662780761719,
      "activations/layer3_attention_weight_min": -92.24340057373047,
      "activations/layer4_attention_weight_max": 133.67906188964844,
      "activations/layer4_attention_weight_min": -129.70846557617188,
      "activations/layer5_attention_weight_max": 88.53435516357422,
      "activations/layer5_attention_weight_min": -85.24750518798828,
      "activations/layer6_attention_weight_max": 60.5413703918457,
      "activations/layer6_attention_weight_min": -59.141563415527344,
      "activations/layer7_attention_weight_max": 86.8731460571289,
      "activations/layer7_attention_weight_min": -84.9522705078125,
      "activations/layer8_attention_weight_max": 44.7509765625,
      "activations/layer8_attention_weight_min": -40.68301010131836,
      "activations/layer9_attention_weight_max": 39.74170684814453,
      "activations/layer9_attention_weight_min": -34.721824645996094,
      "epoch": 4.83,
      "learning_rate": 0.00012003143939393938,
      "loss": 2.8736,
      "step": 83200
    },
    {
      "activations/layer0_attention_weight_max": 16.87255096435547,
      "activations/layer0_attention_weight_min": -11.023187637329102,
      "activations/layer10_attention_weight_max": 57.289886474609375,
      "activations/layer10_attention_weight_min": -46.563377380371094,
      "activations/layer11_attention_weight_max": 56.61223602294922,
      "activations/layer11_attention_weight_min": -39.24053955078125,
      "activations/layer12_attention_weight_max": 81.36956787109375,
      "activations/layer12_attention_weight_min": -27.5855770111084,
      "activations/layer13_attention_weight_max": 101.20589447021484,
      "activations/layer13_attention_weight_min": -42.04054260253906,
      "activations/layer14_attention_weight_max": 98.06549072265625,
      "activations/layer14_attention_weight_min": -48.693031311035156,
      "activations/layer15_attention_weight_max": 82.40731811523438,
      "activations/layer15_attention_weight_min": -31.488317489624023,
      "activations/layer16_attention_weight_max": 41.64699172973633,
      "activations/layer16_attention_weight_min": -28.889421463012695,
      "activations/layer17_attention_weight_max": 41.48945999145508,
      "activations/layer17_attention_weight_min": -24.995319366455078,
      "activations/layer18_attention_weight_max": 35.21344757080078,
      "activations/layer18_attention_weight_min": -21.953937530517578,
      "activations/layer19_attention_weight_max": 41.099857330322266,
      "activations/layer19_attention_weight_min": -29.736433029174805,
      "activations/layer1_attention_weight_max": 18.753616333007812,
      "activations/layer1_attention_weight_min": -15.871488571166992,
      "activations/layer20_attention_weight_max": 37.44001388549805,
      "activations/layer20_attention_weight_min": -23.72437286376953,
      "activations/layer21_attention_weight_max": 34.72941970825195,
      "activations/layer21_attention_weight_min": -21.220842361450195,
      "activations/layer22_attention_weight_max": 51.80281448364258,
      "activations/layer22_attention_weight_min": -29.37477684020996,
      "activations/layer23_attention_weight_max": 37.688026428222656,
      "activations/layer23_attention_weight_min": -20.36785888671875,
      "activations/layer2_attention_weight_max": 37.668731689453125,
      "activations/layer2_attention_weight_min": -37.99662780761719,
      "activations/layer3_attention_weight_max": 114.17577362060547,
      "activations/layer3_attention_weight_min": -113.55081939697266,
      "activations/layer4_attention_weight_max": 167.04150390625,
      "activations/layer4_attention_weight_min": -150.47813415527344,
      "activations/layer5_attention_weight_max": 108.49766540527344,
      "activations/layer5_attention_weight_min": -92.34341430664062,
      "activations/layer6_attention_weight_max": 73.03189849853516,
      "activations/layer6_attention_weight_min": -59.6795654296875,
      "activations/layer7_attention_weight_max": 114.9288330078125,
      "activations/layer7_attention_weight_min": -89.44981384277344,
      "activations/layer8_attention_weight_max": 55.58792495727539,
      "activations/layer8_attention_weight_min": -47.93391036987305,
      "activations/layer9_attention_weight_max": 54.9091796875,
      "activations/layer9_attention_weight_min": -45.18949508666992,
      "epoch": 4.84,
      "learning_rate": 0.0001200125,
      "loss": 2.8612,
      "step": 83250
    },
    {
      "activations/layer0_attention_weight_max": 16.46910858154297,
      "activations/layer0_attention_weight_min": -11.664874076843262,
      "activations/layer10_attention_weight_max": 31.908042907714844,
      "activations/layer10_attention_weight_min": -30.576364517211914,
      "activations/layer11_attention_weight_max": 32.42540740966797,
      "activations/layer11_attention_weight_min": -27.047651290893555,
      "activations/layer12_attention_weight_max": 42.809547424316406,
      "activations/layer12_attention_weight_min": -26.666519165039062,
      "activations/layer13_attention_weight_max": 42.79994583129883,
      "activations/layer13_attention_weight_min": -31.879810333251953,
      "activations/layer14_attention_weight_max": 58.483314514160156,
      "activations/layer14_attention_weight_min": -43.10165023803711,
      "activations/layer15_attention_weight_max": 45.382972717285156,
      "activations/layer15_attention_weight_min": -28.172622680664062,
      "activations/layer16_attention_weight_max": 35.92636489868164,
      "activations/layer16_attention_weight_min": -26.547014236450195,
      "activations/layer17_attention_weight_max": 38.13941955566406,
      "activations/layer17_attention_weight_min": -23.386646270751953,
      "activations/layer18_attention_weight_max": 32.075927734375,
      "activations/layer18_attention_weight_min": -20.03076934814453,
      "activations/layer19_attention_weight_max": 34.988487243652344,
      "activations/layer19_attention_weight_min": -28.2823429107666,
      "activations/layer1_attention_weight_max": 17.463441848754883,
      "activations/layer1_attention_weight_min": -19.9174747467041,
      "activations/layer20_attention_weight_max": 28.656177520751953,
      "activations/layer20_attention_weight_min": -22.67232894897461,
      "activations/layer21_attention_weight_max": 28.03522300720215,
      "activations/layer21_attention_weight_min": -21.755958557128906,
      "activations/layer22_attention_weight_max": 45.9998779296875,
      "activations/layer22_attention_weight_min": -29.232025146484375,
      "activations/layer23_attention_weight_max": 31.35174560546875,
      "activations/layer23_attention_weight_min": -22.175243377685547,
      "activations/layer2_attention_weight_max": 32.180091857910156,
      "activations/layer2_attention_weight_min": -31.339691162109375,
      "activations/layer3_attention_weight_max": 89.42765045166016,
      "activations/layer3_attention_weight_min": -92.66083526611328,
      "activations/layer4_attention_weight_max": 132.9476776123047,
      "activations/layer4_attention_weight_min": -128.22647094726562,
      "activations/layer5_attention_weight_max": 86.96489715576172,
      "activations/layer5_attention_weight_min": -80.31332397460938,
      "activations/layer6_attention_weight_max": 56.740943908691406,
      "activations/layer6_attention_weight_min": -59.17131042480469,
      "activations/layer7_attention_weight_max": 82.62771606445312,
      "activations/layer7_attention_weight_min": -84.99783325195312,
      "activations/layer8_attention_weight_max": 40.540863037109375,
      "activations/layer8_attention_weight_min": -39.8022346496582,
      "activations/layer9_attention_weight_max": 35.479103088378906,
      "activations/layer9_attention_weight_min": -37.630977630615234,
      "epoch": 4.84,
      "learning_rate": 0.00011999356060606058,
      "loss": 2.8559,
      "step": 83300
    },
    {
      "activations/layer0_attention_weight_max": 15.856534957885742,
      "activations/layer0_attention_weight_min": -12.244698524475098,
      "activations/layer10_attention_weight_max": 35.018985748291016,
      "activations/layer10_attention_weight_min": -30.13445281982422,
      "activations/layer11_attention_weight_max": 32.362152099609375,
      "activations/layer11_attention_weight_min": -26.62702751159668,
      "activations/layer12_attention_weight_max": 29.100873947143555,
      "activations/layer12_attention_weight_min": -26.02863311767578,
      "activations/layer13_attention_weight_max": 44.30073547363281,
      "activations/layer13_attention_weight_min": -32.096824645996094,
      "activations/layer14_attention_weight_max": 65.48128509521484,
      "activations/layer14_attention_weight_min": -46.659603118896484,
      "activations/layer15_attention_weight_max": 37.526039123535156,
      "activations/layer15_attention_weight_min": -28.836206436157227,
      "activations/layer16_attention_weight_max": 37.014827728271484,
      "activations/layer16_attention_weight_min": -27.349620819091797,
      "activations/layer17_attention_weight_max": 39.82773971557617,
      "activations/layer17_attention_weight_min": -24.022897720336914,
      "activations/layer18_attention_weight_max": 39.8291130065918,
      "activations/layer18_attention_weight_min": -20.211177825927734,
      "activations/layer19_attention_weight_max": 40.25132369995117,
      "activations/layer19_attention_weight_min": -29.436094284057617,
      "activations/layer1_attention_weight_max": 17.296388626098633,
      "activations/layer1_attention_weight_min": -14.99538516998291,
      "activations/layer20_attention_weight_max": 33.306209564208984,
      "activations/layer20_attention_weight_min": -22.929494857788086,
      "activations/layer21_attention_weight_max": 33.81784439086914,
      "activations/layer21_attention_weight_min": -21.298500061035156,
      "activations/layer22_attention_weight_max": 52.303585052490234,
      "activations/layer22_attention_weight_min": -25.973888397216797,
      "activations/layer23_attention_weight_max": 32.88444519042969,
      "activations/layer23_attention_weight_min": -20.431568145751953,
      "activations/layer2_attention_weight_max": 34.2039909362793,
      "activations/layer2_attention_weight_min": -32.81035614013672,
      "activations/layer3_attention_weight_max": 93.62962341308594,
      "activations/layer3_attention_weight_min": -93.42078399658203,
      "activations/layer4_attention_weight_max": 135.3411102294922,
      "activations/layer4_attention_weight_min": -135.26748657226562,
      "activations/layer5_attention_weight_max": 88.37987518310547,
      "activations/layer5_attention_weight_min": -81.7894515991211,
      "activations/layer6_attention_weight_max": 58.70908737182617,
      "activations/layer6_attention_weight_min": -57.55034637451172,
      "activations/layer7_attention_weight_max": 90.18875122070312,
      "activations/layer7_attention_weight_min": -83.50557708740234,
      "activations/layer8_attention_weight_max": 42.5616340637207,
      "activations/layer8_attention_weight_min": -39.41725540161133,
      "activations/layer9_attention_weight_max": 40.29024887084961,
      "activations/layer9_attention_weight_min": -34.43947982788086,
      "epoch": 4.84,
      "learning_rate": 0.0001199746212121212,
      "loss": 2.8549,
      "step": 83350
    },
    {
      "activations/layer0_attention_weight_max": 17.263572692871094,
      "activations/layer0_attention_weight_min": -12.82555103302002,
      "activations/layer10_attention_weight_max": 40.38128662109375,
      "activations/layer10_attention_weight_min": -31.670257568359375,
      "activations/layer11_attention_weight_max": 39.19165802001953,
      "activations/layer11_attention_weight_min": -28.553089141845703,
      "activations/layer12_attention_weight_max": 61.267066955566406,
      "activations/layer12_attention_weight_min": -27.27228546142578,
      "activations/layer13_attention_weight_max": 50.53291702270508,
      "activations/layer13_attention_weight_min": -32.373146057128906,
      "activations/layer14_attention_weight_max": 64.13208770751953,
      "activations/layer14_attention_weight_min": -41.183834075927734,
      "activations/layer15_attention_weight_max": 43.78753662109375,
      "activations/layer15_attention_weight_min": -27.58184242248535,
      "activations/layer16_attention_weight_max": 42.796165466308594,
      "activations/layer16_attention_weight_min": -27.258377075195312,
      "activations/layer17_attention_weight_max": 39.346473693847656,
      "activations/layer17_attention_weight_min": -23.904157638549805,
      "activations/layer18_attention_weight_max": 30.077001571655273,
      "activations/layer18_attention_weight_min": -20.806503295898438,
      "activations/layer19_attention_weight_max": 40.54914855957031,
      "activations/layer19_attention_weight_min": -28.1434383392334,
      "activations/layer1_attention_weight_max": 17.829448699951172,
      "activations/layer1_attention_weight_min": -15.135597229003906,
      "activations/layer20_attention_weight_max": 31.140960693359375,
      "activations/layer20_attention_weight_min": -22.446739196777344,
      "activations/layer21_attention_weight_max": 29.410024642944336,
      "activations/layer21_attention_weight_min": -19.72222328186035,
      "activations/layer22_attention_weight_max": 50.88129425048828,
      "activations/layer22_attention_weight_min": -28.366729736328125,
      "activations/layer23_attention_weight_max": 34.18279266357422,
      "activations/layer23_attention_weight_min": -20.667253494262695,
      "activations/layer2_attention_weight_max": 33.182533264160156,
      "activations/layer2_attention_weight_min": -32.960411071777344,
      "activations/layer3_attention_weight_max": 92.03021240234375,
      "activations/layer3_attention_weight_min": -96.75621795654297,
      "activations/layer4_attention_weight_max": 142.17774963378906,
      "activations/layer4_attention_weight_min": -136.35533142089844,
      "activations/layer5_attention_weight_max": 86.64254760742188,
      "activations/layer5_attention_weight_min": -81.92587280273438,
      "activations/layer6_attention_weight_max": 58.931846618652344,
      "activations/layer6_attention_weight_min": -56.889244079589844,
      "activations/layer7_attention_weight_max": 94.12447357177734,
      "activations/layer7_attention_weight_min": -88.04428100585938,
      "activations/layer8_attention_weight_max": 46.997318267822266,
      "activations/layer8_attention_weight_min": -44.693355560302734,
      "activations/layer9_attention_weight_max": 40.43606948852539,
      "activations/layer9_attention_weight_min": -39.363643646240234,
      "epoch": 4.85,
      "learning_rate": 0.00011995606060606059,
      "loss": 2.8566,
      "step": 83400
    },
    {
      "activations/layer0_attention_weight_max": 16.73529624938965,
      "activations/layer0_attention_weight_min": -10.812772750854492,
      "activations/layer10_attention_weight_max": 52.45188903808594,
      "activations/layer10_attention_weight_min": -37.4178466796875,
      "activations/layer11_attention_weight_max": 50.831871032714844,
      "activations/layer11_attention_weight_min": -33.57239532470703,
      "activations/layer12_attention_weight_max": 68.69363403320312,
      "activations/layer12_attention_weight_min": -33.401222229003906,
      "activations/layer13_attention_weight_max": 77.4539566040039,
      "activations/layer13_attention_weight_min": -40.50593566894531,
      "activations/layer14_attention_weight_max": 78.16223907470703,
      "activations/layer14_attention_weight_min": -46.30472946166992,
      "activations/layer15_attention_weight_max": 73.26924133300781,
      "activations/layer15_attention_weight_min": -36.38555908203125,
      "activations/layer16_attention_weight_max": 50.886741638183594,
      "activations/layer16_attention_weight_min": -28.999345779418945,
      "activations/layer17_attention_weight_max": 41.50814437866211,
      "activations/layer17_attention_weight_min": -23.91593360900879,
      "activations/layer18_attention_weight_max": 34.397857666015625,
      "activations/layer18_attention_weight_min": -21.302160263061523,
      "activations/layer19_attention_weight_max": 41.345069885253906,
      "activations/layer19_attention_weight_min": -31.13321304321289,
      "activations/layer1_attention_weight_max": 18.214284896850586,
      "activations/layer1_attention_weight_min": -14.681289672851562,
      "activations/layer20_attention_weight_max": 35.02444076538086,
      "activations/layer20_attention_weight_min": -23.612215042114258,
      "activations/layer21_attention_weight_max": 37.024253845214844,
      "activations/layer21_attention_weight_min": -20.90753173828125,
      "activations/layer22_attention_weight_max": 53.9567756652832,
      "activations/layer22_attention_weight_min": -27.94656753540039,
      "activations/layer23_attention_weight_max": 31.498825073242188,
      "activations/layer23_attention_weight_min": -16.600879669189453,
      "activations/layer2_attention_weight_max": 33.89470672607422,
      "activations/layer2_attention_weight_min": -31.56630516052246,
      "activations/layer3_attention_weight_max": 93.29949951171875,
      "activations/layer3_attention_weight_min": -92.12435913085938,
      "activations/layer4_attention_weight_max": 146.09527587890625,
      "activations/layer4_attention_weight_min": -134.06700134277344,
      "activations/layer5_attention_weight_max": 92.80401611328125,
      "activations/layer5_attention_weight_min": -85.8617172241211,
      "activations/layer6_attention_weight_max": 65.90572357177734,
      "activations/layer6_attention_weight_min": -60.736053466796875,
      "activations/layer7_attention_weight_max": 111.44679260253906,
      "activations/layer7_attention_weight_min": -91.8881607055664,
      "activations/layer8_attention_weight_max": 53.57228088378906,
      "activations/layer8_attention_weight_min": -46.52875900268555,
      "activations/layer9_attention_weight_max": 49.37456130981445,
      "activations/layer9_attention_weight_min": -42.978153228759766,
      "epoch": 4.85,
      "learning_rate": 0.0001199371212121212,
      "loss": 2.8653,
      "step": 83450
    },
    {
      "activations/layer0_attention_weight_max": 16.08620262145996,
      "activations/layer0_attention_weight_min": -12.062931060791016,
      "activations/layer10_attention_weight_max": 35.041358947753906,
      "activations/layer10_attention_weight_min": -29.4410457611084,
      "activations/layer11_attention_weight_max": 37.12689208984375,
      "activations/layer11_attention_weight_min": -26.211063385009766,
      "activations/layer12_attention_weight_max": 41.68993377685547,
      "activations/layer12_attention_weight_min": -29.402114868164062,
      "activations/layer13_attention_weight_max": 49.85649871826172,
      "activations/layer13_attention_weight_min": -33.879058837890625,
      "activations/layer14_attention_weight_max": 56.80574417114258,
      "activations/layer14_attention_weight_min": -41.51079177856445,
      "activations/layer15_attention_weight_max": 42.965145111083984,
      "activations/layer15_attention_weight_min": -28.853515625,
      "activations/layer16_attention_weight_max": 40.28160858154297,
      "activations/layer16_attention_weight_min": -26.56413459777832,
      "activations/layer17_attention_weight_max": 35.6093635559082,
      "activations/layer17_attention_weight_min": -23.96173095703125,
      "activations/layer18_attention_weight_max": 27.732694625854492,
      "activations/layer18_attention_weight_min": -20.119369506835938,
      "activations/layer19_attention_weight_max": 40.33430480957031,
      "activations/layer19_attention_weight_min": -28.617816925048828,
      "activations/layer1_attention_weight_max": 16.434526443481445,
      "activations/layer1_attention_weight_min": -13.512847900390625,
      "activations/layer20_attention_weight_max": 30.20115852355957,
      "activations/layer20_attention_weight_min": -23.114944458007812,
      "activations/layer21_attention_weight_max": 29.25755500793457,
      "activations/layer21_attention_weight_min": -18.681947708129883,
      "activations/layer22_attention_weight_max": 45.143455505371094,
      "activations/layer22_attention_weight_min": -25.56785774230957,
      "activations/layer23_attention_weight_max": 32.10896301269531,
      "activations/layer23_attention_weight_min": -19.555904388427734,
      "activations/layer2_attention_weight_max": 35.06492614746094,
      "activations/layer2_attention_weight_min": -32.65678024291992,
      "activations/layer3_attention_weight_max": 94.67160034179688,
      "activations/layer3_attention_weight_min": -94.42699432373047,
      "activations/layer4_attention_weight_max": 130.0836639404297,
      "activations/layer4_attention_weight_min": -124.69481658935547,
      "activations/layer5_attention_weight_max": 84.73184967041016,
      "activations/layer5_attention_weight_min": -77.30647277832031,
      "activations/layer6_attention_weight_max": 57.06929016113281,
      "activations/layer6_attention_weight_min": -52.3519172668457,
      "activations/layer7_attention_weight_max": 85.17510986328125,
      "activations/layer7_attention_weight_min": -81.14571380615234,
      "activations/layer8_attention_weight_max": 42.043880462646484,
      "activations/layer8_attention_weight_min": -38.35886764526367,
      "activations/layer9_attention_weight_max": 39.67744445800781,
      "activations/layer9_attention_weight_min": -36.586544036865234,
      "epoch": 4.85,
      "learning_rate": 0.00011991818181818181,
      "loss": 2.8601,
      "step": 83500
    },
    {
      "activations/layer0_attention_weight_max": 17.212570190429688,
      "activations/layer0_attention_weight_min": -11.868194580078125,
      "activations/layer10_attention_weight_max": 36.90147018432617,
      "activations/layer10_attention_weight_min": -30.60759735107422,
      "activations/layer11_attention_weight_max": 39.660587310791016,
      "activations/layer11_attention_weight_min": -28.159032821655273,
      "activations/layer12_attention_weight_max": 24.360471725463867,
      "activations/layer12_attention_weight_min": -25.47877311706543,
      "activations/layer13_attention_weight_max": 46.15427017211914,
      "activations/layer13_attention_weight_min": -36.141239166259766,
      "activations/layer14_attention_weight_max": 60.61647415161133,
      "activations/layer14_attention_weight_min": -43.0079345703125,
      "activations/layer15_attention_weight_max": 41.72590255737305,
      "activations/layer15_attention_weight_min": -28.42245101928711,
      "activations/layer16_attention_weight_max": 37.146240234375,
      "activations/layer16_attention_weight_min": -29.875274658203125,
      "activations/layer17_attention_weight_max": 37.911468505859375,
      "activations/layer17_attention_weight_min": -26.109270095825195,
      "activations/layer18_attention_weight_max": 34.14238357543945,
      "activations/layer18_attention_weight_min": -19.464929580688477,
      "activations/layer19_attention_weight_max": 36.96336364746094,
      "activations/layer19_attention_weight_min": -29.417699813842773,
      "activations/layer1_attention_weight_max": 16.569929122924805,
      "activations/layer1_attention_weight_min": -13.989089012145996,
      "activations/layer20_attention_weight_max": 32.507869720458984,
      "activations/layer20_attention_weight_min": -25.04470443725586,
      "activations/layer21_attention_weight_max": 37.4175910949707,
      "activations/layer21_attention_weight_min": -24.016708374023438,
      "activations/layer22_attention_weight_max": 48.26716995239258,
      "activations/layer22_attention_weight_min": -30.92514419555664,
      "activations/layer23_attention_weight_max": 31.601425170898438,
      "activations/layer23_attention_weight_min": -20.77971076965332,
      "activations/layer2_attention_weight_max": 31.423484802246094,
      "activations/layer2_attention_weight_min": -30.239093780517578,
      "activations/layer3_attention_weight_max": 89.0628890991211,
      "activations/layer3_attention_weight_min": -89.56133270263672,
      "activations/layer4_attention_weight_max": 131.8482666015625,
      "activations/layer4_attention_weight_min": -128.498779296875,
      "activations/layer5_attention_weight_max": 84.79434204101562,
      "activations/layer5_attention_weight_min": -80.47350311279297,
      "activations/layer6_attention_weight_max": 57.99394226074219,
      "activations/layer6_attention_weight_min": -56.03540802001953,
      "activations/layer7_attention_weight_max": 90.98426055908203,
      "activations/layer7_attention_weight_min": -84.39275360107422,
      "activations/layer8_attention_weight_max": 41.478973388671875,
      "activations/layer8_attention_weight_min": -38.91669845581055,
      "activations/layer9_attention_weight_max": 37.13425064086914,
      "activations/layer9_attention_weight_min": -32.495750427246094,
      "epoch": 4.85,
      "learning_rate": 0.00011989924242424241,
      "loss": 2.8558,
      "step": 83550
    },
    {
      "activations/layer0_attention_weight_max": 17.01735496520996,
      "activations/layer0_attention_weight_min": -11.64268684387207,
      "activations/layer10_attention_weight_max": 31.577823638916016,
      "activations/layer10_attention_weight_min": -32.82560348510742,
      "activations/layer11_attention_weight_max": 29.930572509765625,
      "activations/layer11_attention_weight_min": -27.69066619873047,
      "activations/layer12_attention_weight_max": 30.77263641357422,
      "activations/layer12_attention_weight_min": -28.04074478149414,
      "activations/layer13_attention_weight_max": 44.26408004760742,
      "activations/layer13_attention_weight_min": -35.083892822265625,
      "activations/layer14_attention_weight_max": 53.90509796142578,
      "activations/layer14_attention_weight_min": -42.43531799316406,
      "activations/layer15_attention_weight_max": 44.44853973388672,
      "activations/layer15_attention_weight_min": -28.411802291870117,
      "activations/layer16_attention_weight_max": 35.27635955810547,
      "activations/layer16_attention_weight_min": -28.21061897277832,
      "activations/layer17_attention_weight_max": 41.37141418457031,
      "activations/layer17_attention_weight_min": -23.73767852783203,
      "activations/layer18_attention_weight_max": 30.365718841552734,
      "activations/layer18_attention_weight_min": -19.138103485107422,
      "activations/layer19_attention_weight_max": 41.72638702392578,
      "activations/layer19_attention_weight_min": -29.55105972290039,
      "activations/layer1_attention_weight_max": 18.30760383605957,
      "activations/layer1_attention_weight_min": -14.444236755371094,
      "activations/layer20_attention_weight_max": 33.042301177978516,
      "activations/layer20_attention_weight_min": -26.066129684448242,
      "activations/layer21_attention_weight_max": 30.40213966369629,
      "activations/layer21_attention_weight_min": -21.037538528442383,
      "activations/layer22_attention_weight_max": 47.3807487487793,
      "activations/layer22_attention_weight_min": -29.32772445678711,
      "activations/layer23_attention_weight_max": 32.17378234863281,
      "activations/layer23_attention_weight_min": -20.679412841796875,
      "activations/layer2_attention_weight_max": 32.067039489746094,
      "activations/layer2_attention_weight_min": -32.39613723754883,
      "activations/layer3_attention_weight_max": 89.16857147216797,
      "activations/layer3_attention_weight_min": -88.60796356201172,
      "activations/layer4_attention_weight_max": 127.61317443847656,
      "activations/layer4_attention_weight_min": -122.5354232788086,
      "activations/layer5_attention_weight_max": 82.10453033447266,
      "activations/layer5_attention_weight_min": -79.86949157714844,
      "activations/layer6_attention_weight_max": 55.117393493652344,
      "activations/layer6_attention_weight_min": -55.44386291503906,
      "activations/layer7_attention_weight_max": 85.19207763671875,
      "activations/layer7_attention_weight_min": -80.3976821899414,
      "activations/layer8_attention_weight_max": 39.381839752197266,
      "activations/layer8_attention_weight_min": -37.70689392089844,
      "activations/layer9_attention_weight_max": 40.10032272338867,
      "activations/layer9_attention_weight_min": -38.99924850463867,
      "epoch": 4.86,
      "learning_rate": 0.00011988030303030302,
      "loss": 2.8648,
      "step": 83600
    },
    {
      "activations/layer0_attention_weight_max": 17.56159782409668,
      "activations/layer0_attention_weight_min": -12.638008117675781,
      "activations/layer10_attention_weight_max": 37.346092224121094,
      "activations/layer10_attention_weight_min": -33.8929328918457,
      "activations/layer11_attention_weight_max": 34.64100646972656,
      "activations/layer11_attention_weight_min": -28.20423126220703,
      "activations/layer12_attention_weight_max": 30.119041442871094,
      "activations/layer12_attention_weight_min": -24.335710525512695,
      "activations/layer13_attention_weight_max": 49.82917022705078,
      "activations/layer13_attention_weight_min": -34.05673599243164,
      "activations/layer14_attention_weight_max": 68.83815002441406,
      "activations/layer14_attention_weight_min": -49.886268615722656,
      "activations/layer15_attention_weight_max": 40.479095458984375,
      "activations/layer15_attention_weight_min": -29.12700843811035,
      "activations/layer16_attention_weight_max": 40.62417221069336,
      "activations/layer16_attention_weight_min": -30.184017181396484,
      "activations/layer17_attention_weight_max": 42.79664993286133,
      "activations/layer17_attention_weight_min": -24.073606491088867,
      "activations/layer18_attention_weight_max": 32.499725341796875,
      "activations/layer18_attention_weight_min": -21.285903930664062,
      "activations/layer19_attention_weight_max": 46.42765808105469,
      "activations/layer19_attention_weight_min": -31.508420944213867,
      "activations/layer1_attention_weight_max": 17.52703857421875,
      "activations/layer1_attention_weight_min": -18.15631866455078,
      "activations/layer20_attention_weight_max": 34.28627014160156,
      "activations/layer20_attention_weight_min": -24.693185806274414,
      "activations/layer21_attention_weight_max": 31.6610107421875,
      "activations/layer21_attention_weight_min": -20.859655380249023,
      "activations/layer22_attention_weight_max": 51.87700271606445,
      "activations/layer22_attention_weight_min": -27.694719314575195,
      "activations/layer23_attention_weight_max": 32.758522033691406,
      "activations/layer23_attention_weight_min": -21.036088943481445,
      "activations/layer2_attention_weight_max": 35.41032791137695,
      "activations/layer2_attention_weight_min": -35.286251068115234,
      "activations/layer3_attention_weight_max": 100.0404052734375,
      "activations/layer3_attention_weight_min": -102.49713134765625,
      "activations/layer4_attention_weight_max": 147.0015411376953,
      "activations/layer4_attention_weight_min": -144.67515563964844,
      "activations/layer5_attention_weight_max": 94.89411926269531,
      "activations/layer5_attention_weight_min": -94.43641662597656,
      "activations/layer6_attention_weight_max": 64.14923095703125,
      "activations/layer6_attention_weight_min": -66.32418823242188,
      "activations/layer7_attention_weight_max": 92.68057250976562,
      "activations/layer7_attention_weight_min": -96.55133819580078,
      "activations/layer8_attention_weight_max": 47.594146728515625,
      "activations/layer8_attention_weight_min": -50.80635452270508,
      "activations/layer9_attention_weight_max": 41.67489242553711,
      "activations/layer9_attention_weight_min": -39.44593048095703,
      "epoch": 4.86,
      "learning_rate": 0.00011986136363636362,
      "loss": 2.8656,
      "step": 83650
    },
    {
      "activations/layer0_attention_weight_max": 17.235506057739258,
      "activations/layer0_attention_weight_min": -11.731425285339355,
      "activations/layer10_attention_weight_max": 33.00008010864258,
      "activations/layer10_attention_weight_min": -29.469436645507812,
      "activations/layer11_attention_weight_max": 29.92030906677246,
      "activations/layer11_attention_weight_min": -26.466537475585938,
      "activations/layer12_attention_weight_max": 27.015506744384766,
      "activations/layer12_attention_weight_min": -27.29770278930664,
      "activations/layer13_attention_weight_max": 38.67357635498047,
      "activations/layer13_attention_weight_min": -31.55449104309082,
      "activations/layer14_attention_weight_max": 57.870697021484375,
      "activations/layer14_attention_weight_min": -46.326019287109375,
      "activations/layer15_attention_weight_max": 37.91703414916992,
      "activations/layer15_attention_weight_min": -29.321531295776367,
      "activations/layer16_attention_weight_max": 38.14685821533203,
      "activations/layer16_attention_weight_min": -29.13805389404297,
      "activations/layer17_attention_weight_max": 40.56657791137695,
      "activations/layer17_attention_weight_min": -27.371522903442383,
      "activations/layer18_attention_weight_max": 34.99943923950195,
      "activations/layer18_attention_weight_min": -22.77802848815918,
      "activations/layer19_attention_weight_max": 40.20768356323242,
      "activations/layer19_attention_weight_min": -32.78241729736328,
      "activations/layer1_attention_weight_max": 17.709306716918945,
      "activations/layer1_attention_weight_min": -15.786831855773926,
      "activations/layer20_attention_weight_max": 32.64158630371094,
      "activations/layer20_attention_weight_min": -25.443204879760742,
      "activations/layer21_attention_weight_max": 33.02055740356445,
      "activations/layer21_attention_weight_min": -24.955286026000977,
      "activations/layer22_attention_weight_max": 50.09977340698242,
      "activations/layer22_attention_weight_min": -28.301660537719727,
      "activations/layer23_attention_weight_max": 31.681068420410156,
      "activations/layer23_attention_weight_min": -20.723527908325195,
      "activations/layer2_attention_weight_max": 32.01740264892578,
      "activations/layer2_attention_weight_min": -32.03755187988281,
      "activations/layer3_attention_weight_max": 90.90579223632812,
      "activations/layer3_attention_weight_min": -91.76683807373047,
      "activations/layer4_attention_weight_max": 129.3172607421875,
      "activations/layer4_attention_weight_min": -126.63262176513672,
      "activations/layer5_attention_weight_max": 83.50015258789062,
      "activations/layer5_attention_weight_min": -79.76710510253906,
      "activations/layer6_attention_weight_max": 54.229331970214844,
      "activations/layer6_attention_weight_min": -56.21752166748047,
      "activations/layer7_attention_weight_max": 79.42949676513672,
      "activations/layer7_attention_weight_min": -79.81983184814453,
      "activations/layer8_attention_weight_max": 38.96824645996094,
      "activations/layer8_attention_weight_min": -39.1793212890625,
      "activations/layer9_attention_weight_max": 33.696407318115234,
      "activations/layer9_attention_weight_min": -32.6727294921875,
      "epoch": 4.86,
      "learning_rate": 0.00011984242424242424,
      "loss": 2.8601,
      "step": 83700
    },
    {
      "activations/layer0_attention_weight_max": 17.132143020629883,
      "activations/layer0_attention_weight_min": -11.19914722442627,
      "activations/layer10_attention_weight_max": 38.66313552856445,
      "activations/layer10_attention_weight_min": -28.248680114746094,
      "activations/layer11_attention_weight_max": 31.88739013671875,
      "activations/layer11_attention_weight_min": -27.50057601928711,
      "activations/layer12_attention_weight_max": 28.504823684692383,
      "activations/layer12_attention_weight_min": -28.404748916625977,
      "activations/layer13_attention_weight_max": 48.4234504699707,
      "activations/layer13_attention_weight_min": -33.596229553222656,
      "activations/layer14_attention_weight_max": 57.375511169433594,
      "activations/layer14_attention_weight_min": -42.4576416015625,
      "activations/layer15_attention_weight_max": 50.70221710205078,
      "activations/layer15_attention_weight_min": -30.48200798034668,
      "activations/layer16_attention_weight_max": 38.930274963378906,
      "activations/layer16_attention_weight_min": -27.258634567260742,
      "activations/layer17_attention_weight_max": 38.811153411865234,
      "activations/layer17_attention_weight_min": -23.78801727294922,
      "activations/layer18_attention_weight_max": 37.281150817871094,
      "activations/layer18_attention_weight_min": -20.393489837646484,
      "activations/layer19_attention_weight_max": 40.23752975463867,
      "activations/layer19_attention_weight_min": -28.375503540039062,
      "activations/layer1_attention_weight_max": 17.707212448120117,
      "activations/layer1_attention_weight_min": -16.91435432434082,
      "activations/layer20_attention_weight_max": 31.173320770263672,
      "activations/layer20_attention_weight_min": -23.31572151184082,
      "activations/layer21_attention_weight_max": 32.494136810302734,
      "activations/layer21_attention_weight_min": -21.12966537475586,
      "activations/layer22_attention_weight_max": 53.9052848815918,
      "activations/layer22_attention_weight_min": -29.992992401123047,
      "activations/layer23_attention_weight_max": 30.28335189819336,
      "activations/layer23_attention_weight_min": -19.795146942138672,
      "activations/layer2_attention_weight_max": 31.013105392456055,
      "activations/layer2_attention_weight_min": -31.669538497924805,
      "activations/layer3_attention_weight_max": 88.71878814697266,
      "activations/layer3_attention_weight_min": -91.54090881347656,
      "activations/layer4_attention_weight_max": 132.37864685058594,
      "activations/layer4_attention_weight_min": -133.98779296875,
      "activations/layer5_attention_weight_max": 83.81077575683594,
      "activations/layer5_attention_weight_min": -80.61419677734375,
      "activations/layer6_attention_weight_max": 58.308311462402344,
      "activations/layer6_attention_weight_min": -53.38589859008789,
      "activations/layer7_attention_weight_max": 77.70494842529297,
      "activations/layer7_attention_weight_min": -85.25543975830078,
      "activations/layer8_attention_weight_max": 39.72501754760742,
      "activations/layer8_attention_weight_min": -36.77875900268555,
      "activations/layer9_attention_weight_max": 36.460689544677734,
      "activations/layer9_attention_weight_min": -36.12893295288086,
      "epoch": 4.87,
      "learning_rate": 0.00011982348484848483,
      "loss": 2.8544,
      "step": 83750
    },
    {
      "activations/layer0_attention_weight_max": 17.21277618408203,
      "activations/layer0_attention_weight_min": -11.266794204711914,
      "activations/layer10_attention_weight_max": 34.05521011352539,
      "activations/layer10_attention_weight_min": -31.24017333984375,
      "activations/layer11_attention_weight_max": 33.73686218261719,
      "activations/layer11_attention_weight_min": -30.757835388183594,
      "activations/layer12_attention_weight_max": 37.35366439819336,
      "activations/layer12_attention_weight_min": -27.090255737304688,
      "activations/layer13_attention_weight_max": 48.18874740600586,
      "activations/layer13_attention_weight_min": -35.47459030151367,
      "activations/layer14_attention_weight_max": 57.665855407714844,
      "activations/layer14_attention_weight_min": -41.48949432373047,
      "activations/layer15_attention_weight_max": 51.16347122192383,
      "activations/layer15_attention_weight_min": -31.903778076171875,
      "activations/layer16_attention_weight_max": 37.739532470703125,
      "activations/layer16_attention_weight_min": -27.543489456176758,
      "activations/layer17_attention_weight_max": 38.77350997924805,
      "activations/layer17_attention_weight_min": -24.516942977905273,
      "activations/layer18_attention_weight_max": 36.06035232543945,
      "activations/layer18_attention_weight_min": -20.5710506439209,
      "activations/layer19_attention_weight_max": 40.914588928222656,
      "activations/layer19_attention_weight_min": -27.118650436401367,
      "activations/layer1_attention_weight_max": 18.775527954101562,
      "activations/layer1_attention_weight_min": -14.693472862243652,
      "activations/layer20_attention_weight_max": 32.14528274536133,
      "activations/layer20_attention_weight_min": -22.706464767456055,
      "activations/layer21_attention_weight_max": 31.85807991027832,
      "activations/layer21_attention_weight_min": -20.349748611450195,
      "activations/layer22_attention_weight_max": 49.94623565673828,
      "activations/layer22_attention_weight_min": -28.692691802978516,
      "activations/layer23_attention_weight_max": 33.64871597290039,
      "activations/layer23_attention_weight_min": -21.286283493041992,
      "activations/layer2_attention_weight_max": 33.98863983154297,
      "activations/layer2_attention_weight_min": -33.066184997558594,
      "activations/layer3_attention_weight_max": 96.4433822631836,
      "activations/layer3_attention_weight_min": -99.50679779052734,
      "activations/layer4_attention_weight_max": 141.85635375976562,
      "activations/layer4_attention_weight_min": -134.57757568359375,
      "activations/layer5_attention_weight_max": 86.71043395996094,
      "activations/layer5_attention_weight_min": -83.7326431274414,
      "activations/layer6_attention_weight_max": 60.73551559448242,
      "activations/layer6_attention_weight_min": -56.66505813598633,
      "activations/layer7_attention_weight_max": 84.38899230957031,
      "activations/layer7_attention_weight_min": -85.0254898071289,
      "activations/layer8_attention_weight_max": 41.58412551879883,
      "activations/layer8_attention_weight_min": -39.81846237182617,
      "activations/layer9_attention_weight_max": 37.37617111206055,
      "activations/layer9_attention_weight_min": -42.94887924194336,
      "epoch": 4.87,
      "learning_rate": 0.00011980454545454544,
      "loss": 2.8534,
      "step": 83800
    },
    {
      "activations/layer0_attention_weight_max": 16.384355545043945,
      "activations/layer0_attention_weight_min": -12.35634708404541,
      "activations/layer10_attention_weight_max": 32.82777786254883,
      "activations/layer10_attention_weight_min": -29.510875701904297,
      "activations/layer11_attention_weight_max": 31.51938247680664,
      "activations/layer11_attention_weight_min": -26.85498809814453,
      "activations/layer12_attention_weight_max": 27.224454879760742,
      "activations/layer12_attention_weight_min": -24.52928352355957,
      "activations/layer13_attention_weight_max": 59.99429702758789,
      "activations/layer13_attention_weight_min": -39.90538787841797,
      "activations/layer14_attention_weight_max": 66.49542999267578,
      "activations/layer14_attention_weight_min": -45.99958419799805,
      "activations/layer15_attention_weight_max": 39.158443450927734,
      "activations/layer15_attention_weight_min": -30.526878356933594,
      "activations/layer16_attention_weight_max": 33.29551696777344,
      "activations/layer16_attention_weight_min": -30.098587036132812,
      "activations/layer17_attention_weight_max": 40.236000061035156,
      "activations/layer17_attention_weight_min": -23.994592666625977,
      "activations/layer18_attention_weight_max": 32.262603759765625,
      "activations/layer18_attention_weight_min": -20.459590911865234,
      "activations/layer19_attention_weight_max": 38.03615951538086,
      "activations/layer19_attention_weight_min": -33.211673736572266,
      "activations/layer1_attention_weight_max": 17.09496307373047,
      "activations/layer1_attention_weight_min": -15.370097160339355,
      "activations/layer20_attention_weight_max": 30.438398361206055,
      "activations/layer20_attention_weight_min": -24.199283599853516,
      "activations/layer21_attention_weight_max": 29.47813606262207,
      "activations/layer21_attention_weight_min": -20.70362091064453,
      "activations/layer22_attention_weight_max": 54.17729568481445,
      "activations/layer22_attention_weight_min": -31.871295928955078,
      "activations/layer23_attention_weight_max": 34.95948791503906,
      "activations/layer23_attention_weight_min": -19.797908782958984,
      "activations/layer2_attention_weight_max": 31.97360610961914,
      "activations/layer2_attention_weight_min": -29.803930282592773,
      "activations/layer3_attention_weight_max": 87.0679931640625,
      "activations/layer3_attention_weight_min": -90.52099609375,
      "activations/layer4_attention_weight_max": 129.07540893554688,
      "activations/layer4_attention_weight_min": -125.43133544921875,
      "activations/layer5_attention_weight_max": 81.53413391113281,
      "activations/layer5_attention_weight_min": -82.31717681884766,
      "activations/layer6_attention_weight_max": 58.83589553833008,
      "activations/layer6_attention_weight_min": -55.145877838134766,
      "activations/layer7_attention_weight_max": 80.66480255126953,
      "activations/layer7_attention_weight_min": -77.87562561035156,
      "activations/layer8_attention_weight_max": 39.522850036621094,
      "activations/layer8_attention_weight_min": -38.09370803833008,
      "activations/layer9_attention_weight_max": 35.02602005004883,
      "activations/layer9_attention_weight_min": -32.404823303222656,
      "epoch": 4.87,
      "learning_rate": 0.00011978560606060606,
      "loss": 2.8547,
      "step": 83850
    },
    {
      "activations/layer0_attention_weight_max": 15.29206371307373,
      "activations/layer0_attention_weight_min": -12.48440170288086,
      "activations/layer10_attention_weight_max": 32.980228424072266,
      "activations/layer10_attention_weight_min": -31.36142349243164,
      "activations/layer11_attention_weight_max": 31.507841110229492,
      "activations/layer11_attention_weight_min": -28.051359176635742,
      "activations/layer12_attention_weight_max": 34.39122772216797,
      "activations/layer12_attention_weight_min": -28.03036117553711,
      "activations/layer13_attention_weight_max": 44.2019157409668,
      "activations/layer13_attention_weight_min": -33.21664047241211,
      "activations/layer14_attention_weight_max": 57.969688415527344,
      "activations/layer14_attention_weight_min": -43.68941879272461,
      "activations/layer15_attention_weight_max": 40.89938735961914,
      "activations/layer15_attention_weight_min": -28.472307205200195,
      "activations/layer16_attention_weight_max": 36.16780471801758,
      "activations/layer16_attention_weight_min": -27.48512077331543,
      "activations/layer17_attention_weight_max": 36.838340759277344,
      "activations/layer17_attention_weight_min": -25.744752883911133,
      "activations/layer18_attention_weight_max": 34.70843505859375,
      "activations/layer18_attention_weight_min": -18.918935775756836,
      "activations/layer19_attention_weight_max": 38.124473571777344,
      "activations/layer19_attention_weight_min": -28.7490291595459,
      "activations/layer1_attention_weight_max": 16.316192626953125,
      "activations/layer1_attention_weight_min": -14.477018356323242,
      "activations/layer20_attention_weight_max": 34.33949279785156,
      "activations/layer20_attention_weight_min": -23.031333923339844,
      "activations/layer21_attention_weight_max": 31.371002197265625,
      "activations/layer21_attention_weight_min": -21.268125534057617,
      "activations/layer22_attention_weight_max": 47.247283935546875,
      "activations/layer22_attention_weight_min": -26.7077693939209,
      "activations/layer23_attention_weight_max": 33.32883071899414,
      "activations/layer23_attention_weight_min": -19.662416458129883,
      "activations/layer2_attention_weight_max": 31.352108001708984,
      "activations/layer2_attention_weight_min": -30.14350128173828,
      "activations/layer3_attention_weight_max": 89.48247528076172,
      "activations/layer3_attention_weight_min": -88.06918334960938,
      "activations/layer4_attention_weight_max": 137.1972198486328,
      "activations/layer4_attention_weight_min": -128.4608612060547,
      "activations/layer5_attention_weight_max": 91.12560272216797,
      "activations/layer5_attention_weight_min": -81.06349182128906,
      "activations/layer6_attention_weight_max": 59.69734573364258,
      "activations/layer6_attention_weight_min": -57.273990631103516,
      "activations/layer7_attention_weight_max": 89.44977569580078,
      "activations/layer7_attention_weight_min": -83.37440490722656,
      "activations/layer8_attention_weight_max": 39.77070999145508,
      "activations/layer8_attention_weight_min": -38.2475471496582,
      "activations/layer9_attention_weight_max": 37.64004898071289,
      "activations/layer9_attention_weight_min": -34.54233932495117,
      "epoch": 4.88,
      "learning_rate": 0.00011976666666666665,
      "loss": 2.8498,
      "step": 83900
    },
    {
      "activations/layer0_attention_weight_max": 17.17159080505371,
      "activations/layer0_attention_weight_min": -12.36811351776123,
      "activations/layer10_attention_weight_max": 35.16814041137695,
      "activations/layer10_attention_weight_min": -29.87436866760254,
      "activations/layer11_attention_weight_max": 32.91688537597656,
      "activations/layer11_attention_weight_min": -28.278629302978516,
      "activations/layer12_attention_weight_max": 25.33537483215332,
      "activations/layer12_attention_weight_min": -27.165124893188477,
      "activations/layer13_attention_weight_max": 48.17564392089844,
      "activations/layer13_attention_weight_min": -32.62795639038086,
      "activations/layer14_attention_weight_max": 92.01077270507812,
      "activations/layer14_attention_weight_min": -68.51119232177734,
      "activations/layer15_attention_weight_max": 40.79651641845703,
      "activations/layer15_attention_weight_min": -30.53223419189453,
      "activations/layer16_attention_weight_max": 33.35961151123047,
      "activations/layer16_attention_weight_min": -28.14493179321289,
      "activations/layer17_attention_weight_max": 40.61735153198242,
      "activations/layer17_attention_weight_min": -27.36235809326172,
      "activations/layer18_attention_weight_max": 30.961828231811523,
      "activations/layer18_attention_weight_min": -21.110727310180664,
      "activations/layer19_attention_weight_max": 40.28432846069336,
      "activations/layer19_attention_weight_min": -29.929174423217773,
      "activations/layer1_attention_weight_max": 17.790390014648438,
      "activations/layer1_attention_weight_min": -14.62622356414795,
      "activations/layer20_attention_weight_max": 33.86244583129883,
      "activations/layer20_attention_weight_min": -23.0629940032959,
      "activations/layer21_attention_weight_max": 33.978981018066406,
      "activations/layer21_attention_weight_min": -20.362503051757812,
      "activations/layer22_attention_weight_max": 51.690860748291016,
      "activations/layer22_attention_weight_min": -28.762250900268555,
      "activations/layer23_attention_weight_max": 32.727210998535156,
      "activations/layer23_attention_weight_min": -18.930519104003906,
      "activations/layer2_attention_weight_max": 31.77289390563965,
      "activations/layer2_attention_weight_min": -30.47182273864746,
      "activations/layer3_attention_weight_max": 89.70055389404297,
      "activations/layer3_attention_weight_min": -92.33380889892578,
      "activations/layer4_attention_weight_max": 130.63389587402344,
      "activations/layer4_attention_weight_min": -130.4467315673828,
      "activations/layer5_attention_weight_max": 85.92404174804688,
      "activations/layer5_attention_weight_min": -85.95167541503906,
      "activations/layer6_attention_weight_max": 59.75650405883789,
      "activations/layer6_attention_weight_min": -57.67921829223633,
      "activations/layer7_attention_weight_max": 88.72386932373047,
      "activations/layer7_attention_weight_min": -84.90104675292969,
      "activations/layer8_attention_weight_max": 41.57224655151367,
      "activations/layer8_attention_weight_min": -39.16033935546875,
      "activations/layer9_attention_weight_max": 38.13463592529297,
      "activations/layer9_attention_weight_min": -34.93888473510742,
      "epoch": 4.88,
      "learning_rate": 0.00011974772727272726,
      "loss": 2.8448,
      "step": 83950
    },
    {
      "activations/layer0_attention_weight_max": 16.911386489868164,
      "activations/layer0_attention_weight_min": -12.478832244873047,
      "activations/layer10_attention_weight_max": 35.082366943359375,
      "activations/layer10_attention_weight_min": -32.43415069580078,
      "activations/layer11_attention_weight_max": 34.73942947387695,
      "activations/layer11_attention_weight_min": -30.86458396911621,
      "activations/layer12_attention_weight_max": 36.1315803527832,
      "activations/layer12_attention_weight_min": -26.702747344970703,
      "activations/layer13_attention_weight_max": 45.36357498168945,
      "activations/layer13_attention_weight_min": -35.21211242675781,
      "activations/layer14_attention_weight_max": 80.51637268066406,
      "activations/layer14_attention_weight_min": -51.53901290893555,
      "activations/layer15_attention_weight_max": 42.25496292114258,
      "activations/layer15_attention_weight_min": -30.6842041015625,
      "activations/layer16_attention_weight_max": 49.19646453857422,
      "activations/layer16_attention_weight_min": -31.53336524963379,
      "activations/layer17_attention_weight_max": 39.25691604614258,
      "activations/layer17_attention_weight_min": -23.881988525390625,
      "activations/layer18_attention_weight_max": 31.245153427124023,
      "activations/layer18_attention_weight_min": -19.371761322021484,
      "activations/layer19_attention_weight_max": 53.882747650146484,
      "activations/layer19_attention_weight_min": -31.296934127807617,
      "activations/layer1_attention_weight_max": 17.483917236328125,
      "activations/layer1_attention_weight_min": -13.638493537902832,
      "activations/layer20_attention_weight_max": 35.27193069458008,
      "activations/layer20_attention_weight_min": -25.149375915527344,
      "activations/layer21_attention_weight_max": 34.400753021240234,
      "activations/layer21_attention_weight_min": -21.244565963745117,
      "activations/layer22_attention_weight_max": 58.312198638916016,
      "activations/layer22_attention_weight_min": -28.637754440307617,
      "activations/layer23_attention_weight_max": 36.24922180175781,
      "activations/layer23_attention_weight_min": -20.601957321166992,
      "activations/layer2_attention_weight_max": 32.615318298339844,
      "activations/layer2_attention_weight_min": -32.451202392578125,
      "activations/layer3_attention_weight_max": 89.95451354980469,
      "activations/layer3_attention_weight_min": -93.6625747680664,
      "activations/layer4_attention_weight_max": 131.09112548828125,
      "activations/layer4_attention_weight_min": -129.6355743408203,
      "activations/layer5_attention_weight_max": 82.8092041015625,
      "activations/layer5_attention_weight_min": -81.36732482910156,
      "activations/layer6_attention_weight_max": 55.63307189941406,
      "activations/layer6_attention_weight_min": -56.927574157714844,
      "activations/layer7_attention_weight_max": 85.09347534179688,
      "activations/layer7_attention_weight_min": -90.88447570800781,
      "activations/layer8_attention_weight_max": 39.69060134887695,
      "activations/layer8_attention_weight_min": -40.35419464111328,
      "activations/layer9_attention_weight_max": 36.19721603393555,
      "activations/layer9_attention_weight_min": -35.30416488647461,
      "epoch": 4.88,
      "learning_rate": 0.00011972878787878788,
      "loss": 2.8611,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_loss": 2.8046875,
      "eval_runtime": 8.5357,
      "eval_samples_per_second": 503.065,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_openwebtext_loss": 2.8046875,
      "eval_openwebtext_ppl": 16.521912001744084,
      "eval_openwebtext_runtime": 8.5357,
      "eval_openwebtext_samples_per_second": 503.065,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_wikitext_loss": 3.04296875,
      "eval_wikitext_ppl": 20.967397891114594,
      "eval_wikitext_runtime": 1.9944,
      "eval_wikitext_samples_per_second": 228.641,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_lambada_loss": 2.73828125,
      "eval_lambada_ppl": 15.460389702669161,
      "eval_lambada_runtime": 9.6209,
      "eval_lambada_samples_per_second": 506.084,
      "step": 84000
    },
    {
      "activations/layer0_attention_weight_max": 17.03738021850586,
      "activations/layer0_attention_weight_min": -11.673501014709473,
      "activations/layer10_attention_weight_max": 31.88055419921875,
      "activations/layer10_attention_weight_min": -29.582273483276367,
      "activations/layer11_attention_weight_max": 32.46350860595703,
      "activations/layer11_attention_weight_min": -26.382617950439453,
      "activations/layer12_attention_weight_max": 41.70268630981445,
      "activations/layer12_attention_weight_min": -25.88241195678711,
      "activations/layer13_attention_weight_max": 51.9943733215332,
      "activations/layer13_attention_weight_min": -33.99074935913086,
      "activations/layer14_attention_weight_max": 73.36360168457031,
      "activations/layer14_attention_weight_min": -50.080810546875,
      "activations/layer15_attention_weight_max": 51.443756103515625,
      "activations/layer15_attention_weight_min": -31.38609504699707,
      "activations/layer16_attention_weight_max": 46.20695114135742,
      "activations/layer16_attention_weight_min": -27.970596313476562,
      "activations/layer17_attention_weight_max": 43.05577087402344,
      "activations/layer17_attention_weight_min": -26.094064712524414,
      "activations/layer18_attention_weight_max": 30.592121124267578,
      "activations/layer18_attention_weight_min": -19.515975952148438,
      "activations/layer19_attention_weight_max": 51.69287109375,
      "activations/layer19_attention_weight_min": -27.92498779296875,
      "activations/layer1_attention_weight_max": 17.010536193847656,
      "activations/layer1_attention_weight_min": -16.15462875366211,
      "activations/layer20_attention_weight_max": 36.283103942871094,
      "activations/layer20_attention_weight_min": -22.188573837280273,
      "activations/layer21_attention_weight_max": 34.04199981689453,
      "activations/layer21_attention_weight_min": -21.12518882751465,
      "activations/layer22_attention_weight_max": 52.38690185546875,
      "activations/layer22_attention_weight_min": -28.143390655517578,
      "activations/layer23_attention_weight_max": 31.88216781616211,
      "activations/layer23_attention_weight_min": -21.525466918945312,
      "activations/layer2_attention_weight_max": 31.785184860229492,
      "activations/layer2_attention_weight_min": -31.239463806152344,
      "activations/layer3_attention_weight_max": 89.15872192382812,
      "activations/layer3_attention_weight_min": -91.19200134277344,
      "activations/layer4_attention_weight_max": 129.7405242919922,
      "activations/layer4_attention_weight_min": -129.29998779296875,
      "activations/layer5_attention_weight_max": 84.90471649169922,
      "activations/layer5_attention_weight_min": -82.29325866699219,
      "activations/layer6_attention_weight_max": 56.22983169555664,
      "activations/layer6_attention_weight_min": -59.692161560058594,
      "activations/layer7_attention_weight_max": 79.32432556152344,
      "activations/layer7_attention_weight_min": -83.60050201416016,
      "activations/layer8_attention_weight_max": 41.2024040222168,
      "activations/layer8_attention_weight_min": -43.435447692871094,
      "activations/layer9_attention_weight_max": 37.37191390991211,
      "activations/layer9_attention_weight_min": -33.174373626708984,
      "epoch": 4.88,
      "learning_rate": 0.00011970984848484846,
      "loss": 2.8483,
      "step": 84050
    },
    {
      "activations/layer0_attention_weight_max": 17.187763214111328,
      "activations/layer0_attention_weight_min": -12.58047866821289,
      "activations/layer10_attention_weight_max": 34.1135139465332,
      "activations/layer10_attention_weight_min": -29.723617553710938,
      "activations/layer11_attention_weight_max": 32.30833435058594,
      "activations/layer11_attention_weight_min": -27.362943649291992,
      "activations/layer12_attention_weight_max": 23.748592376708984,
      "activations/layer12_attention_weight_min": -24.992948532104492,
      "activations/layer13_attention_weight_max": 55.546485900878906,
      "activations/layer13_attention_weight_min": -36.99311065673828,
      "activations/layer14_attention_weight_max": 69.27045440673828,
      "activations/layer14_attention_weight_min": -47.55403137207031,
      "activations/layer15_attention_weight_max": 38.312705993652344,
      "activations/layer15_attention_weight_min": -28.30510711669922,
      "activations/layer16_attention_weight_max": 34.920433044433594,
      "activations/layer16_attention_weight_min": -26.515514373779297,
      "activations/layer17_attention_weight_max": 38.94022750854492,
      "activations/layer17_attention_weight_min": -24.825210571289062,
      "activations/layer18_attention_weight_max": 30.78575325012207,
      "activations/layer18_attention_weight_min": -20.461206436157227,
      "activations/layer19_attention_weight_max": 39.90266036987305,
      "activations/layer19_attention_weight_min": -29.350881576538086,
      "activations/layer1_attention_weight_max": 18.452917098999023,
      "activations/layer1_attention_weight_min": -14.203927040100098,
      "activations/layer20_attention_weight_max": 31.625457763671875,
      "activations/layer20_attention_weight_min": -22.356700897216797,
      "activations/layer21_attention_weight_max": 32.351318359375,
      "activations/layer21_attention_weight_min": -19.905176162719727,
      "activations/layer22_attention_weight_max": 49.66838836669922,
      "activations/layer22_attention_weight_min": -26.089332580566406,
      "activations/layer23_attention_weight_max": 31.91377067565918,
      "activations/layer23_attention_weight_min": -20.226625442504883,
      "activations/layer2_attention_weight_max": 31.548505783081055,
      "activations/layer2_attention_weight_min": -30.850543975830078,
      "activations/layer3_attention_weight_max": 89.14070892333984,
      "activations/layer3_attention_weight_min": -89.58505249023438,
      "activations/layer4_attention_weight_max": 129.3160400390625,
      "activations/layer4_attention_weight_min": -128.5565948486328,
      "activations/layer5_attention_weight_max": 83.95359802246094,
      "activations/layer5_attention_weight_min": -81.72775268554688,
      "activations/layer6_attention_weight_max": 55.13444519042969,
      "activations/layer6_attention_weight_min": -59.3116569519043,
      "activations/layer7_attention_weight_max": 83.6871337890625,
      "activations/layer7_attention_weight_min": -83.19998931884766,
      "activations/layer8_attention_weight_max": 40.50654602050781,
      "activations/layer8_attention_weight_min": -39.23399353027344,
      "activations/layer9_attention_weight_max": 35.89924240112305,
      "activations/layer9_attention_weight_min": -34.575557708740234,
      "epoch": 4.89,
      "learning_rate": 0.00011969090909090908,
      "loss": 2.8595,
      "step": 84100
    },
    {
      "activations/layer0_attention_weight_max": 16.705928802490234,
      "activations/layer0_attention_weight_min": -12.515298843383789,
      "activations/layer10_attention_weight_max": 35.36357879638672,
      "activations/layer10_attention_weight_min": -30.732242584228516,
      "activations/layer11_attention_weight_max": 33.217506408691406,
      "activations/layer11_attention_weight_min": -28.84368896484375,
      "activations/layer12_attention_weight_max": 29.559280395507812,
      "activations/layer12_attention_weight_min": -30.23535919189453,
      "activations/layer13_attention_weight_max": 50.08073425292969,
      "activations/layer13_attention_weight_min": -36.876644134521484,
      "activations/layer14_attention_weight_max": 57.61271286010742,
      "activations/layer14_attention_weight_min": -44.51290512084961,
      "activations/layer15_attention_weight_max": 40.96768569946289,
      "activations/layer15_attention_weight_min": -27.147663116455078,
      "activations/layer16_attention_weight_max": 39.32194900512695,
      "activations/layer16_attention_weight_min": -29.235401153564453,
      "activations/layer17_attention_weight_max": 40.72145080566406,
      "activations/layer17_attention_weight_min": -23.910776138305664,
      "activations/layer18_attention_weight_max": 33.65888595581055,
      "activations/layer18_attention_weight_min": -19.71141815185547,
      "activations/layer19_attention_weight_max": 43.54287338256836,
      "activations/layer19_attention_weight_min": -29.499969482421875,
      "activations/layer1_attention_weight_max": 18.11899757385254,
      "activations/layer1_attention_weight_min": -14.70152473449707,
      "activations/layer20_attention_weight_max": 35.87326431274414,
      "activations/layer20_attention_weight_min": -24.45131492614746,
      "activations/layer21_attention_weight_max": 30.855314254760742,
      "activations/layer21_attention_weight_min": -22.029571533203125,
      "activations/layer22_attention_weight_max": 50.16261291503906,
      "activations/layer22_attention_weight_min": -28.452884674072266,
      "activations/layer23_attention_weight_max": 35.463478088378906,
      "activations/layer23_attention_weight_min": -20.059680938720703,
      "activations/layer2_attention_weight_max": 32.11511993408203,
      "activations/layer2_attention_weight_min": -30.23458480834961,
      "activations/layer3_attention_weight_max": 89.23302459716797,
      "activations/layer3_attention_weight_min": -86.97596740722656,
      "activations/layer4_attention_weight_max": 138.4511260986328,
      "activations/layer4_attention_weight_min": -125.79107666015625,
      "activations/layer5_attention_weight_max": 89.80918884277344,
      "activations/layer5_attention_weight_min": -78.09065246582031,
      "activations/layer6_attention_weight_max": 58.076263427734375,
      "activations/layer6_attention_weight_min": -56.90679931640625,
      "activations/layer7_attention_weight_max": 84.64639282226562,
      "activations/layer7_attention_weight_min": -84.25724029541016,
      "activations/layer8_attention_weight_max": 42.730445861816406,
      "activations/layer8_attention_weight_min": -40.11957550048828,
      "activations/layer9_attention_weight_max": 37.160316467285156,
      "activations/layer9_attention_weight_min": -34.11969757080078,
      "epoch": 4.89,
      "learning_rate": 0.0001196719696969697,
      "loss": 2.8592,
      "step": 84150
    },
    {
      "activations/layer0_attention_weight_max": 17.535879135131836,
      "activations/layer0_attention_weight_min": -11.435257911682129,
      "activations/layer10_attention_weight_max": 36.414730072021484,
      "activations/layer10_attention_weight_min": -31.860605239868164,
      "activations/layer11_attention_weight_max": 31.893840789794922,
      "activations/layer11_attention_weight_min": -29.556066513061523,
      "activations/layer12_attention_weight_max": 52.992794036865234,
      "activations/layer12_attention_weight_min": -26.791589736938477,
      "activations/layer13_attention_weight_max": 71.50382995605469,
      "activations/layer13_attention_weight_min": -44.34528732299805,
      "activations/layer14_attention_weight_max": 60.98904037475586,
      "activations/layer14_attention_weight_min": -46.00353240966797,
      "activations/layer15_attention_weight_max": 50.87953567504883,
      "activations/layer15_attention_weight_min": -29.17082977294922,
      "activations/layer16_attention_weight_max": 36.04707336425781,
      "activations/layer16_attention_weight_min": -28.412992477416992,
      "activations/layer17_attention_weight_max": 35.46126937866211,
      "activations/layer17_attention_weight_min": -25.006134033203125,
      "activations/layer18_attention_weight_max": 30.218097686767578,
      "activations/layer18_attention_weight_min": -19.311803817749023,
      "activations/layer19_attention_weight_max": 35.15093994140625,
      "activations/layer19_attention_weight_min": -30.318145751953125,
      "activations/layer1_attention_weight_max": 17.22820281982422,
      "activations/layer1_attention_weight_min": -14.45053768157959,
      "activations/layer20_attention_weight_max": 29.129764556884766,
      "activations/layer20_attention_weight_min": -23.266326904296875,
      "activations/layer21_attention_weight_max": 26.251419067382812,
      "activations/layer21_attention_weight_min": -21.456398010253906,
      "activations/layer22_attention_weight_max": 49.70736312866211,
      "activations/layer22_attention_weight_min": -26.71796989440918,
      "activations/layer23_attention_weight_max": 32.369964599609375,
      "activations/layer23_attention_weight_min": -18.567333221435547,
      "activations/layer2_attention_weight_max": 33.36882019042969,
      "activations/layer2_attention_weight_min": -32.633766174316406,
      "activations/layer3_attention_weight_max": 94.03734588623047,
      "activations/layer3_attention_weight_min": -93.84989166259766,
      "activations/layer4_attention_weight_max": 131.7367706298828,
      "activations/layer4_attention_weight_min": -133.4855194091797,
      "activations/layer5_attention_weight_max": 85.72978210449219,
      "activations/layer5_attention_weight_min": -81.24653625488281,
      "activations/layer6_attention_weight_max": 59.72609329223633,
      "activations/layer6_attention_weight_min": -59.6570930480957,
      "activations/layer7_attention_weight_max": 84.49961853027344,
      "activations/layer7_attention_weight_min": -82.9642333984375,
      "activations/layer8_attention_weight_max": 41.09104537963867,
      "activations/layer8_attention_weight_min": -38.57545852661133,
      "activations/layer9_attention_weight_max": 36.886409759521484,
      "activations/layer9_attention_weight_min": -34.465736389160156,
      "epoch": 4.89,
      "learning_rate": 0.0001196530303030303,
      "loss": 2.8477,
      "step": 84200
    },
    {
      "activations/layer0_attention_weight_max": 14.28327751159668,
      "activations/layer0_attention_weight_min": -12.885658264160156,
      "activations/layer10_attention_weight_max": 34.617000579833984,
      "activations/layer10_attention_weight_min": -33.02708435058594,
      "activations/layer11_attention_weight_max": 31.999534606933594,
      "activations/layer11_attention_weight_min": -28.189083099365234,
      "activations/layer12_attention_weight_max": 25.702646255493164,
      "activations/layer12_attention_weight_min": -24.736169815063477,
      "activations/layer13_attention_weight_max": 48.84536361694336,
      "activations/layer13_attention_weight_min": -34.61223602294922,
      "activations/layer14_attention_weight_max": 75.80525207519531,
      "activations/layer14_attention_weight_min": -54.56013107299805,
      "activations/layer15_attention_weight_max": 44.7392692565918,
      "activations/layer15_attention_weight_min": -28.795255661010742,
      "activations/layer16_attention_weight_max": 39.98270034790039,
      "activations/layer16_attention_weight_min": -29.461450576782227,
      "activations/layer17_attention_weight_max": 43.84473419189453,
      "activations/layer17_attention_weight_min": -24.4180850982666,
      "activations/layer18_attention_weight_max": 34.41724395751953,
      "activations/layer18_attention_weight_min": -20.08674430847168,
      "activations/layer19_attention_weight_max": 47.0832405090332,
      "activations/layer19_attention_weight_min": -30.023996353149414,
      "activations/layer1_attention_weight_max": 17.175832748413086,
      "activations/layer1_attention_weight_min": -16.0892333984375,
      "activations/layer20_attention_weight_max": 35.35435485839844,
      "activations/layer20_attention_weight_min": -24.352998733520508,
      "activations/layer21_attention_weight_max": 34.48103713989258,
      "activations/layer21_attention_weight_min": -18.922029495239258,
      "activations/layer22_attention_weight_max": 50.63108825683594,
      "activations/layer22_attention_weight_min": -27.38007926940918,
      "activations/layer23_attention_weight_max": 35.40523910522461,
      "activations/layer23_attention_weight_min": -17.804840087890625,
      "activations/layer2_attention_weight_max": 32.98979949951172,
      "activations/layer2_attention_weight_min": -32.53791809082031,
      "activations/layer3_attention_weight_max": 92.07837677001953,
      "activations/layer3_attention_weight_min": -92.61814880371094,
      "activations/layer4_attention_weight_max": 135.3310089111328,
      "activations/layer4_attention_weight_min": -129.2853240966797,
      "activations/layer5_attention_weight_max": 85.90168762207031,
      "activations/layer5_attention_weight_min": -80.99877166748047,
      "activations/layer6_attention_weight_max": 59.71650695800781,
      "activations/layer6_attention_weight_min": -57.65818405151367,
      "activations/layer7_attention_weight_max": 83.59537506103516,
      "activations/layer7_attention_weight_min": -90.16917419433594,
      "activations/layer8_attention_weight_max": 40.36438751220703,
      "activations/layer8_attention_weight_min": -40.91151809692383,
      "activations/layer9_attention_weight_max": 37.39596176147461,
      "activations/layer9_attention_weight_min": -37.1632194519043,
      "epoch": 4.9,
      "learning_rate": 0.0001196340909090909,
      "loss": 2.8627,
      "step": 84250
    },
    {
      "activations/layer0_attention_weight_max": 16.535734176635742,
      "activations/layer0_attention_weight_min": -10.688028335571289,
      "activations/layer10_attention_weight_max": 34.42097854614258,
      "activations/layer10_attention_weight_min": -30.567520141601562,
      "activations/layer11_attention_weight_max": 33.58077621459961,
      "activations/layer11_attention_weight_min": -28.76888656616211,
      "activations/layer12_attention_weight_max": 42.07832717895508,
      "activations/layer12_attention_weight_min": -27.137407302856445,
      "activations/layer13_attention_weight_max": 57.39530563354492,
      "activations/layer13_attention_weight_min": -40.57650375366211,
      "activations/layer14_attention_weight_max": 69.14025115966797,
      "activations/layer14_attention_weight_min": -46.99782180786133,
      "activations/layer15_attention_weight_max": 37.441383361816406,
      "activations/layer15_attention_weight_min": -29.39858055114746,
      "activations/layer16_attention_weight_max": 34.44624710083008,
      "activations/layer16_attention_weight_min": -29.939437866210938,
      "activations/layer17_attention_weight_max": 37.449623107910156,
      "activations/layer17_attention_weight_min": -23.85266876220703,
      "activations/layer18_attention_weight_max": 32.68083953857422,
      "activations/layer18_attention_weight_min": -20.37837791442871,
      "activations/layer19_attention_weight_max": 38.871376037597656,
      "activations/layer19_attention_weight_min": -30.69732666015625,
      "activations/layer1_attention_weight_max": 17.809864044189453,
      "activations/layer1_attention_weight_min": -15.112602233886719,
      "activations/layer20_attention_weight_max": 31.12653160095215,
      "activations/layer20_attention_weight_min": -22.7941837310791,
      "activations/layer21_attention_weight_max": 33.1356086730957,
      "activations/layer21_attention_weight_min": -23.108919143676758,
      "activations/layer22_attention_weight_max": 50.602783203125,
      "activations/layer22_attention_weight_min": -30.3035945892334,
      "activations/layer23_attention_weight_max": 30.228862762451172,
      "activations/layer23_attention_weight_min": -20.115367889404297,
      "activations/layer2_attention_weight_max": 34.167152404785156,
      "activations/layer2_attention_weight_min": -34.821739196777344,
      "activations/layer3_attention_weight_max": 95.8543701171875,
      "activations/layer3_attention_weight_min": -94.10680389404297,
      "activations/layer4_attention_weight_max": 139.7610321044922,
      "activations/layer4_attention_weight_min": -141.7056121826172,
      "activations/layer5_attention_weight_max": 86.56320190429688,
      "activations/layer5_attention_weight_min": -87.43482971191406,
      "activations/layer6_attention_weight_max": 55.470497131347656,
      "activations/layer6_attention_weight_min": -61.5307731628418,
      "activations/layer7_attention_weight_max": 82.17219543457031,
      "activations/layer7_attention_weight_min": -87.63195037841797,
      "activations/layer8_attention_weight_max": 42.452911376953125,
      "activations/layer8_attention_weight_min": -43.5143928527832,
      "activations/layer9_attention_weight_max": 36.01650619506836,
      "activations/layer9_attention_weight_min": -35.84038162231445,
      "epoch": 4.9,
      "learning_rate": 0.0001196151515151515,
      "loss": 2.8501,
      "step": 84300
    },
    {
      "activations/layer0_attention_weight_max": 16.69635581970215,
      "activations/layer0_attention_weight_min": -13.182173728942871,
      "activations/layer10_attention_weight_max": 34.79029083251953,
      "activations/layer10_attention_weight_min": -34.12992477416992,
      "activations/layer11_attention_weight_max": 33.36083984375,
      "activations/layer11_attention_weight_min": -30.74268341064453,
      "activations/layer12_attention_weight_max": 31.60710334777832,
      "activations/layer12_attention_weight_min": -28.251060485839844,
      "activations/layer13_attention_weight_max": 48.563438415527344,
      "activations/layer13_attention_weight_min": -38.29464340209961,
      "activations/layer14_attention_weight_max": 64.24327850341797,
      "activations/layer14_attention_weight_min": -49.535850524902344,
      "activations/layer15_attention_weight_max": 44.0673828125,
      "activations/layer15_attention_weight_min": -32.575050354003906,
      "activations/layer16_attention_weight_max": 37.81038284301758,
      "activations/layer16_attention_weight_min": -28.696033477783203,
      "activations/layer17_attention_weight_max": 37.10245895385742,
      "activations/layer17_attention_weight_min": -27.056169509887695,
      "activations/layer18_attention_weight_max": 31.9982967376709,
      "activations/layer18_attention_weight_min": -20.759479522705078,
      "activations/layer19_attention_weight_max": 48.238189697265625,
      "activations/layer19_attention_weight_min": -30.120222091674805,
      "activations/layer1_attention_weight_max": 17.6453914642334,
      "activations/layer1_attention_weight_min": -13.833210945129395,
      "activations/layer20_attention_weight_max": 32.551918029785156,
      "activations/layer20_attention_weight_min": -25.4719181060791,
      "activations/layer21_attention_weight_max": 31.27592658996582,
      "activations/layer21_attention_weight_min": -21.71393394470215,
      "activations/layer22_attention_weight_max": 47.29694366455078,
      "activations/layer22_attention_weight_min": -28.695363998413086,
      "activations/layer23_attention_weight_max": 31.475284576416016,
      "activations/layer23_attention_weight_min": -22.483055114746094,
      "activations/layer2_attention_weight_max": 31.227935791015625,
      "activations/layer2_attention_weight_min": -30.220510482788086,
      "activations/layer3_attention_weight_max": 91.91277313232422,
      "activations/layer3_attention_weight_min": -88.29035949707031,
      "activations/layer4_attention_weight_max": 132.0640869140625,
      "activations/layer4_attention_weight_min": -122.7420883178711,
      "activations/layer5_attention_weight_max": 83.97156524658203,
      "activations/layer5_attention_weight_min": -77.49237060546875,
      "activations/layer6_attention_weight_max": 57.007164001464844,
      "activations/layer6_attention_weight_min": -59.01569747924805,
      "activations/layer7_attention_weight_max": 86.54339599609375,
      "activations/layer7_attention_weight_min": -85.17642974853516,
      "activations/layer8_attention_weight_max": 41.65083312988281,
      "activations/layer8_attention_weight_min": -40.44105911254883,
      "activations/layer9_attention_weight_max": 37.55152130126953,
      "activations/layer9_attention_weight_min": -35.05616760253906,
      "epoch": 4.9,
      "learning_rate": 0.00011959621212121211,
      "loss": 2.855,
      "step": 84350
    },
    {
      "activations/layer0_attention_weight_max": 16.173826217651367,
      "activations/layer0_attention_weight_min": -11.638996124267578,
      "activations/layer10_attention_weight_max": 42.75108337402344,
      "activations/layer10_attention_weight_min": -33.7357292175293,
      "activations/layer11_attention_weight_max": 48.782432556152344,
      "activations/layer11_attention_weight_min": -31.622440338134766,
      "activations/layer12_attention_weight_max": 43.18252944946289,
      "activations/layer12_attention_weight_min": -27.2614803314209,
      "activations/layer13_attention_weight_max": 52.586483001708984,
      "activations/layer13_attention_weight_min": -34.67394256591797,
      "activations/layer14_attention_weight_max": 62.57744598388672,
      "activations/layer14_attention_weight_min": -43.88362503051758,
      "activations/layer15_attention_weight_max": 44.76837921142578,
      "activations/layer15_attention_weight_min": -27.59062385559082,
      "activations/layer16_attention_weight_max": 38.1591796875,
      "activations/layer16_attention_weight_min": -29.077579498291016,
      "activations/layer17_attention_weight_max": 39.754390716552734,
      "activations/layer17_attention_weight_min": -23.05959701538086,
      "activations/layer18_attention_weight_max": 30.70333480834961,
      "activations/layer18_attention_weight_min": -18.861066818237305,
      "activations/layer19_attention_weight_max": 41.40834426879883,
      "activations/layer19_attention_weight_min": -29.992786407470703,
      "activations/layer1_attention_weight_max": 18.670225143432617,
      "activations/layer1_attention_weight_min": -14.949797630310059,
      "activations/layer20_attention_weight_max": 32.08060836791992,
      "activations/layer20_attention_weight_min": -21.85202407836914,
      "activations/layer21_attention_weight_max": 34.604454040527344,
      "activations/layer21_attention_weight_min": -21.815444946289062,
      "activations/layer22_attention_weight_max": 49.27719497680664,
      "activations/layer22_attention_weight_min": -26.84745979309082,
      "activations/layer23_attention_weight_max": 32.8104362487793,
      "activations/layer23_attention_weight_min": -20.22606658935547,
      "activations/layer2_attention_weight_max": 32.989112854003906,
      "activations/layer2_attention_weight_min": -33.57349395751953,
      "activations/layer3_attention_weight_max": 90.73333740234375,
      "activations/layer3_attention_weight_min": -96.40818786621094,
      "activations/layer4_attention_weight_max": 135.68389892578125,
      "activations/layer4_attention_weight_min": -132.53831481933594,
      "activations/layer5_attention_weight_max": 86.29241943359375,
      "activations/layer5_attention_weight_min": -81.66385650634766,
      "activations/layer6_attention_weight_max": 64.05062866210938,
      "activations/layer6_attention_weight_min": -56.08091735839844,
      "activations/layer7_attention_weight_max": 98.78986358642578,
      "activations/layer7_attention_weight_min": -95.95030212402344,
      "activations/layer8_attention_weight_max": 46.40442657470703,
      "activations/layer8_attention_weight_min": -43.09246826171875,
      "activations/layer9_attention_weight_max": 42.53727340698242,
      "activations/layer9_attention_weight_min": -46.086544036865234,
      "epoch": 4.9,
      "learning_rate": 0.00011957727272727272,
      "loss": 2.8465,
      "step": 84400
    },
    {
      "activations/layer0_attention_weight_max": 14.995689392089844,
      "activations/layer0_attention_weight_min": -11.855916976928711,
      "activations/layer10_attention_weight_max": 34.20459747314453,
      "activations/layer10_attention_weight_min": -30.554393768310547,
      "activations/layer11_attention_weight_max": 37.564247131347656,
      "activations/layer11_attention_weight_min": -30.30846405029297,
      "activations/layer12_attention_weight_max": 27.71993637084961,
      "activations/layer12_attention_weight_min": -26.864242553710938,
      "activations/layer13_attention_weight_max": 44.53030776977539,
      "activations/layer13_attention_weight_min": -35.09324645996094,
      "activations/layer14_attention_weight_max": 56.49956512451172,
      "activations/layer14_attention_weight_min": -47.291229248046875,
      "activations/layer15_attention_weight_max": 37.897640228271484,
      "activations/layer15_attention_weight_min": -27.793657302856445,
      "activations/layer16_attention_weight_max": 35.586673736572266,
      "activations/layer16_attention_weight_min": -30.08834457397461,
      "activations/layer17_attention_weight_max": 35.39022445678711,
      "activations/layer17_attention_weight_min": -25.259241104125977,
      "activations/layer18_attention_weight_max": 34.269203186035156,
      "activations/layer18_attention_weight_min": -18.218305587768555,
      "activations/layer19_attention_weight_max": 39.7844352722168,
      "activations/layer19_attention_weight_min": -30.149885177612305,
      "activations/layer1_attention_weight_max": 16.909639358520508,
      "activations/layer1_attention_weight_min": -14.015869140625,
      "activations/layer20_attention_weight_max": 33.72758865356445,
      "activations/layer20_attention_weight_min": -25.765478134155273,
      "activations/layer21_attention_weight_max": 32.23250198364258,
      "activations/layer21_attention_weight_min": -19.526535034179688,
      "activations/layer22_attention_weight_max": 52.08477020263672,
      "activations/layer22_attention_weight_min": -29.296859741210938,
      "activations/layer23_attention_weight_max": 32.88235855102539,
      "activations/layer23_attention_weight_min": -18.728958129882812,
      "activations/layer2_attention_weight_max": 31.705440521240234,
      "activations/layer2_attention_weight_min": -30.96540069580078,
      "activations/layer3_attention_weight_max": 88.11312103271484,
      "activations/layer3_attention_weight_min": -91.54720306396484,
      "activations/layer4_attention_weight_max": 126.96935272216797,
      "activations/layer4_attention_weight_min": -126.07939910888672,
      "activations/layer5_attention_weight_max": 80.64938354492188,
      "activations/layer5_attention_weight_min": -78.01860046386719,
      "activations/layer6_attention_weight_max": 58.41830825805664,
      "activations/layer6_attention_weight_min": -55.932071685791016,
      "activations/layer7_attention_weight_max": 78.77581787109375,
      "activations/layer7_attention_weight_min": -81.73225402832031,
      "activations/layer8_attention_weight_max": 37.27327346801758,
      "activations/layer8_attention_weight_min": -37.59297180175781,
      "activations/layer9_attention_weight_max": 35.435157775878906,
      "activations/layer9_attention_weight_min": -31.781375885009766,
      "epoch": 4.91,
      "learning_rate": 0.00011955833333333332,
      "loss": 2.8726,
      "step": 84450
    },
    {
      "activations/layer0_attention_weight_max": 16.84113311767578,
      "activations/layer0_attention_weight_min": -11.440458297729492,
      "activations/layer10_attention_weight_max": 32.68111038208008,
      "activations/layer10_attention_weight_min": -30.31767463684082,
      "activations/layer11_attention_weight_max": 33.1573486328125,
      "activations/layer11_attention_weight_min": -27.140518188476562,
      "activations/layer12_attention_weight_max": 25.624927520751953,
      "activations/layer12_attention_weight_min": -30.173038482666016,
      "activations/layer13_attention_weight_max": 48.67108917236328,
      "activations/layer13_attention_weight_min": -36.11068344116211,
      "activations/layer14_attention_weight_max": 71.01591491699219,
      "activations/layer14_attention_weight_min": -49.838531494140625,
      "activations/layer15_attention_weight_max": 40.329559326171875,
      "activations/layer15_attention_weight_min": -27.871471405029297,
      "activations/layer16_attention_weight_max": 38.6761360168457,
      "activations/layer16_attention_weight_min": -29.824312210083008,
      "activations/layer17_attention_weight_max": 40.706058502197266,
      "activations/layer17_attention_weight_min": -26.32639503479004,
      "activations/layer18_attention_weight_max": 32.14102554321289,
      "activations/layer18_attention_weight_min": -22.55445671081543,
      "activations/layer19_attention_weight_max": 40.58926773071289,
      "activations/layer19_attention_weight_min": -31.32887077331543,
      "activations/layer1_attention_weight_max": 17.309904098510742,
      "activations/layer1_attention_weight_min": -17.955106735229492,
      "activations/layer20_attention_weight_max": 32.983882904052734,
      "activations/layer20_attention_weight_min": -23.47539520263672,
      "activations/layer21_attention_weight_max": 39.564456939697266,
      "activations/layer21_attention_weight_min": -21.488317489624023,
      "activations/layer22_attention_weight_max": 52.3842658996582,
      "activations/layer22_attention_weight_min": -30.83445930480957,
      "activations/layer23_attention_weight_max": 32.17047882080078,
      "activations/layer23_attention_weight_min": -20.12104034423828,
      "activations/layer2_attention_weight_max": 32.34288787841797,
      "activations/layer2_attention_weight_min": -33.524810791015625,
      "activations/layer3_attention_weight_max": 90.28675079345703,
      "activations/layer3_attention_weight_min": -93.4113998413086,
      "activations/layer4_attention_weight_max": 127.33168029785156,
      "activations/layer4_attention_weight_min": -131.19383239746094,
      "activations/layer5_attention_weight_max": 80.71754455566406,
      "activations/layer5_attention_weight_min": -81.4603271484375,
      "activations/layer6_attention_weight_max": 55.64923095703125,
      "activations/layer6_attention_weight_min": -61.746910095214844,
      "activations/layer7_attention_weight_max": 82.5215835571289,
      "activations/layer7_attention_weight_min": -83.31672668457031,
      "activations/layer8_attention_weight_max": 38.53889846801758,
      "activations/layer8_attention_weight_min": -39.18586349487305,
      "activations/layer9_attention_weight_max": 35.76882553100586,
      "activations/layer9_attention_weight_min": -34.34302520751953,
      "epoch": 4.91,
      "learning_rate": 0.00011953939393939393,
      "loss": 2.869,
      "step": 84500
    },
    {
      "activations/layer0_attention_weight_max": 16.528640747070312,
      "activations/layer0_attention_weight_min": -11.721153259277344,
      "activations/layer10_attention_weight_max": 32.735687255859375,
      "activations/layer10_attention_weight_min": -30.718114852905273,
      "activations/layer11_attention_weight_max": 29.439666748046875,
      "activations/layer11_attention_weight_min": -26.72277069091797,
      "activations/layer12_attention_weight_max": 30.984827041625977,
      "activations/layer12_attention_weight_min": -25.88239288330078,
      "activations/layer13_attention_weight_max": 47.83867645263672,
      "activations/layer13_attention_weight_min": -35.44984436035156,
      "activations/layer14_attention_weight_max": 56.33963394165039,
      "activations/layer14_attention_weight_min": -46.992820739746094,
      "activations/layer15_attention_weight_max": 37.766178131103516,
      "activations/layer15_attention_weight_min": -28.56328773498535,
      "activations/layer16_attention_weight_max": 36.59475326538086,
      "activations/layer16_attention_weight_min": -27.696535110473633,
      "activations/layer17_attention_weight_max": 38.01649475097656,
      "activations/layer17_attention_weight_min": -24.472808837890625,
      "activations/layer18_attention_weight_max": 32.35142135620117,
      "activations/layer18_attention_weight_min": -19.658430099487305,
      "activations/layer19_attention_weight_max": 37.55415344238281,
      "activations/layer19_attention_weight_min": -28.359058380126953,
      "activations/layer1_attention_weight_max": 17.540382385253906,
      "activations/layer1_attention_weight_min": -16.077478408813477,
      "activations/layer20_attention_weight_max": 32.15262222290039,
      "activations/layer20_attention_weight_min": -23.11229133605957,
      "activations/layer21_attention_weight_max": 36.1064453125,
      "activations/layer21_attention_weight_min": -20.837501525878906,
      "activations/layer22_attention_weight_max": 45.64947509765625,
      "activations/layer22_attention_weight_min": -27.72723388671875,
      "activations/layer23_attention_weight_max": 31.64301300048828,
      "activations/layer23_attention_weight_min": -20.2413272857666,
      "activations/layer2_attention_weight_max": 28.18145751953125,
      "activations/layer2_attention_weight_min": -29.290569305419922,
      "activations/layer3_attention_weight_max": 88.43663787841797,
      "activations/layer3_attention_weight_min": -86.62989044189453,
      "activations/layer4_attention_weight_max": 120.78960418701172,
      "activations/layer4_attention_weight_min": -129.74302673339844,
      "activations/layer5_attention_weight_max": 79.58131408691406,
      "activations/layer5_attention_weight_min": -77.3192367553711,
      "activations/layer6_attention_weight_max": 54.57210159301758,
      "activations/layer6_attention_weight_min": -57.88935089111328,
      "activations/layer7_attention_weight_max": 77.34519958496094,
      "activations/layer7_attention_weight_min": -81.90970611572266,
      "activations/layer8_attention_weight_max": 39.38764572143555,
      "activations/layer8_attention_weight_min": -38.720096588134766,
      "activations/layer9_attention_weight_max": 34.1224250793457,
      "activations/layer9_attention_weight_min": -33.306095123291016,
      "epoch": 4.91,
      "learning_rate": 0.00011952045454545455,
      "loss": 2.8754,
      "step": 84550
    },
    {
      "activations/layer0_attention_weight_max": 16.72482681274414,
      "activations/layer0_attention_weight_min": -11.684524536132812,
      "activations/layer10_attention_weight_max": 33.80582046508789,
      "activations/layer10_attention_weight_min": -28.539684295654297,
      "activations/layer11_attention_weight_max": 32.294071197509766,
      "activations/layer11_attention_weight_min": -26.43451499938965,
      "activations/layer12_attention_weight_max": 41.486934661865234,
      "activations/layer12_attention_weight_min": -29.88487434387207,
      "activations/layer13_attention_weight_max": 51.23847198486328,
      "activations/layer13_attention_weight_min": -35.05242156982422,
      "activations/layer14_attention_weight_max": 59.819068908691406,
      "activations/layer14_attention_weight_min": -43.79123306274414,
      "activations/layer15_attention_weight_max": 38.88393783569336,
      "activations/layer15_attention_weight_min": -27.944076538085938,
      "activations/layer16_attention_weight_max": 37.72548294067383,
      "activations/layer16_attention_weight_min": -26.661043167114258,
      "activations/layer17_attention_weight_max": 37.31822204589844,
      "activations/layer17_attention_weight_min": -26.626550674438477,
      "activations/layer18_attention_weight_max": 38.4064826965332,
      "activations/layer18_attention_weight_min": -19.984371185302734,
      "activations/layer19_attention_weight_max": 38.15908432006836,
      "activations/layer19_attention_weight_min": -29.354217529296875,
      "activations/layer1_attention_weight_max": 18.68679428100586,
      "activations/layer1_attention_weight_min": -14.98701286315918,
      "activations/layer20_attention_weight_max": 33.33863067626953,
      "activations/layer20_attention_weight_min": -22.838735580444336,
      "activations/layer21_attention_weight_max": 36.9478645324707,
      "activations/layer21_attention_weight_min": -20.9012451171875,
      "activations/layer22_attention_weight_max": 53.87620162963867,
      "activations/layer22_attention_weight_min": -28.32563018798828,
      "activations/layer23_attention_weight_max": 33.08267593383789,
      "activations/layer23_attention_weight_min": -22.156742095947266,
      "activations/layer2_attention_weight_max": 31.24155044555664,
      "activations/layer2_attention_weight_min": -31.54762077331543,
      "activations/layer3_attention_weight_max": 93.62531280517578,
      "activations/layer3_attention_weight_min": -96.6109619140625,
      "activations/layer4_attention_weight_max": 133.8238067626953,
      "activations/layer4_attention_weight_min": -132.06109619140625,
      "activations/layer5_attention_weight_max": 80.2258529663086,
      "activations/layer5_attention_weight_min": -80.53825378417969,
      "activations/layer6_attention_weight_max": 57.06975555419922,
      "activations/layer6_attention_weight_min": -62.10508728027344,
      "activations/layer7_attention_weight_max": 92.82089233398438,
      "activations/layer7_attention_weight_min": -84.92328643798828,
      "activations/layer8_attention_weight_max": 40.488853454589844,
      "activations/layer8_attention_weight_min": -41.537410736083984,
      "activations/layer9_attention_weight_max": 36.50141143798828,
      "activations/layer9_attention_weight_min": -38.95941925048828,
      "epoch": 4.92,
      "learning_rate": 0.00011950151515151513,
      "loss": 2.8588,
      "step": 84600
    },
    {
      "activations/layer0_attention_weight_max": 17.188722610473633,
      "activations/layer0_attention_weight_min": -11.233935356140137,
      "activations/layer10_attention_weight_max": 34.81829833984375,
      "activations/layer10_attention_weight_min": -31.47511863708496,
      "activations/layer11_attention_weight_max": 38.558998107910156,
      "activations/layer11_attention_weight_min": -30.43484115600586,
      "activations/layer12_attention_weight_max": 25.539751052856445,
      "activations/layer12_attention_weight_min": -28.32693099975586,
      "activations/layer13_attention_weight_max": 39.16737365722656,
      "activations/layer13_attention_weight_min": -32.77287673950195,
      "activations/layer14_attention_weight_max": 66.39249420166016,
      "activations/layer14_attention_weight_min": -45.8143424987793,
      "activations/layer15_attention_weight_max": 40.23017883300781,
      "activations/layer15_attention_weight_min": -31.762800216674805,
      "activations/layer16_attention_weight_max": 36.49810028076172,
      "activations/layer16_attention_weight_min": -27.997814178466797,
      "activations/layer17_attention_weight_max": 38.72895812988281,
      "activations/layer17_attention_weight_min": -26.500452041625977,
      "activations/layer18_attention_weight_max": 31.249521255493164,
      "activations/layer18_attention_weight_min": -19.5317440032959,
      "activations/layer19_attention_weight_max": 39.9917106628418,
      "activations/layer19_attention_weight_min": -30.719074249267578,
      "activations/layer1_attention_weight_max": 17.242692947387695,
      "activations/layer1_attention_weight_min": -15.716675758361816,
      "activations/layer20_attention_weight_max": 33.86309051513672,
      "activations/layer20_attention_weight_min": -22.9268856048584,
      "activations/layer21_attention_weight_max": 31.877971649169922,
      "activations/layer21_attention_weight_min": -19.917715072631836,
      "activations/layer22_attention_weight_max": 51.2059440612793,
      "activations/layer22_attention_weight_min": -28.895978927612305,
      "activations/layer23_attention_weight_max": 33.44801712036133,
      "activations/layer23_attention_weight_min": -22.900157928466797,
      "activations/layer2_attention_weight_max": 30.74195671081543,
      "activations/layer2_attention_weight_min": -30.79513168334961,
      "activations/layer3_attention_weight_max": 86.58837890625,
      "activations/layer3_attention_weight_min": -89.235107421875,
      "activations/layer4_attention_weight_max": 126.63819885253906,
      "activations/layer4_attention_weight_min": -122.557373046875,
      "activations/layer5_attention_weight_max": 81.15701293945312,
      "activations/layer5_attention_weight_min": -74.14476013183594,
      "activations/layer6_attention_weight_max": 56.70945358276367,
      "activations/layer6_attention_weight_min": -54.91569519042969,
      "activations/layer7_attention_weight_max": 83.63805389404297,
      "activations/layer7_attention_weight_min": -84.87510681152344,
      "activations/layer8_attention_weight_max": 39.70229721069336,
      "activations/layer8_attention_weight_min": -36.11150360107422,
      "activations/layer9_attention_weight_max": 37.6341667175293,
      "activations/layer9_attention_weight_min": -33.32570266723633,
      "epoch": 4.92,
      "learning_rate": 0.00011948257575757575,
      "loss": 2.8562,
      "step": 84650
    },
    {
      "activations/layer0_attention_weight_max": 16.839710235595703,
      "activations/layer0_attention_weight_min": -12.94332504272461,
      "activations/layer10_attention_weight_max": 38.27734375,
      "activations/layer10_attention_weight_min": -31.849239349365234,
      "activations/layer11_attention_weight_max": 35.149208068847656,
      "activations/layer11_attention_weight_min": -28.36568260192871,
      "activations/layer12_attention_weight_max": 26.20526885986328,
      "activations/layer12_attention_weight_min": -28.402034759521484,
      "activations/layer13_attention_weight_max": 50.90531539916992,
      "activations/layer13_attention_weight_min": -36.26705551147461,
      "activations/layer14_attention_weight_max": 91.77070617675781,
      "activations/layer14_attention_weight_min": -62.976783752441406,
      "activations/layer15_attention_weight_max": 38.75727081298828,
      "activations/layer15_attention_weight_min": -27.723831176757812,
      "activations/layer16_attention_weight_max": 37.88825988769531,
      "activations/layer16_attention_weight_min": -28.0958194732666,
      "activations/layer17_attention_weight_max": 37.929073333740234,
      "activations/layer17_attention_weight_min": -24.55958366394043,
      "activations/layer18_attention_weight_max": 29.597042083740234,
      "activations/layer18_attention_weight_min": -19.35892677307129,
      "activations/layer19_attention_weight_max": 36.096824645996094,
      "activations/layer19_attention_weight_min": -29.8624324798584,
      "activations/layer1_attention_weight_max": 18.24864959716797,
      "activations/layer1_attention_weight_min": -16.002338409423828,
      "activations/layer20_attention_weight_max": 30.149089813232422,
      "activations/layer20_attention_weight_min": -25.048128128051758,
      "activations/layer21_attention_weight_max": 29.10414695739746,
      "activations/layer21_attention_weight_min": -20.630281448364258,
      "activations/layer22_attention_weight_max": 43.988765716552734,
      "activations/layer22_attention_weight_min": -29.559717178344727,
      "activations/layer23_attention_weight_max": 30.60519790649414,
      "activations/layer23_attention_weight_min": -22.164731979370117,
      "activations/layer2_attention_weight_max": 31.644390106201172,
      "activations/layer2_attention_weight_min": -30.11714744567871,
      "activations/layer3_attention_weight_max": 88.17597198486328,
      "activations/layer3_attention_weight_min": -88.5689697265625,
      "activations/layer4_attention_weight_max": 131.8098907470703,
      "activations/layer4_attention_weight_min": -128.03675842285156,
      "activations/layer5_attention_weight_max": 87.59599304199219,
      "activations/layer5_attention_weight_min": -76.86421966552734,
      "activations/layer6_attention_weight_max": 56.626102447509766,
      "activations/layer6_attention_weight_min": -56.0724983215332,
      "activations/layer7_attention_weight_max": 86.06959533691406,
      "activations/layer7_attention_weight_min": -89.8515853881836,
      "activations/layer8_attention_weight_max": 42.06236267089844,
      "activations/layer8_attention_weight_min": -40.57188034057617,
      "activations/layer9_attention_weight_max": 39.04251480102539,
      "activations/layer9_attention_weight_min": -35.04682159423828,
      "epoch": 4.92,
      "learning_rate": 0.00011946363636363636,
      "loss": 2.8539,
      "step": 84700
    },
    {
      "activations/layer0_attention_weight_max": 16.4012451171875,
      "activations/layer0_attention_weight_min": -11.576915740966797,
      "activations/layer10_attention_weight_max": 35.47682189941406,
      "activations/layer10_attention_weight_min": -29.855863571166992,
      "activations/layer11_attention_weight_max": 32.679351806640625,
      "activations/layer11_attention_weight_min": -26.522483825683594,
      "activations/layer12_attention_weight_max": 25.809497833251953,
      "activations/layer12_attention_weight_min": -27.61961555480957,
      "activations/layer13_attention_weight_max": 43.00117492675781,
      "activations/layer13_attention_weight_min": -31.30449867248535,
      "activations/layer14_attention_weight_max": 73.81183624267578,
      "activations/layer14_attention_weight_min": -53.03435516357422,
      "activations/layer15_attention_weight_max": 39.146705627441406,
      "activations/layer15_attention_weight_min": -27.103761672973633,
      "activations/layer16_attention_weight_max": 36.69401550292969,
      "activations/layer16_attention_weight_min": -27.43024253845215,
      "activations/layer17_attention_weight_max": 40.34047317504883,
      "activations/layer17_attention_weight_min": -24.171567916870117,
      "activations/layer18_attention_weight_max": 36.388755798339844,
      "activations/layer18_attention_weight_min": -20.956758499145508,
      "activations/layer19_attention_weight_max": 43.93770980834961,
      "activations/layer19_attention_weight_min": -29.41657066345215,
      "activations/layer1_attention_weight_max": 17.399452209472656,
      "activations/layer1_attention_weight_min": -14.636075019836426,
      "activations/layer20_attention_weight_max": 35.41710662841797,
      "activations/layer20_attention_weight_min": -23.964330673217773,
      "activations/layer21_attention_weight_max": 36.044189453125,
      "activations/layer21_attention_weight_min": -21.055755615234375,
      "activations/layer22_attention_weight_max": 52.18360137939453,
      "activations/layer22_attention_weight_min": -28.844791412353516,
      "activations/layer23_attention_weight_max": 35.26641082763672,
      "activations/layer23_attention_weight_min": -23.77359390258789,
      "activations/layer2_attention_weight_max": 32.052772521972656,
      "activations/layer2_attention_weight_min": -32.183685302734375,
      "activations/layer3_attention_weight_max": 86.59071350097656,
      "activations/layer3_attention_weight_min": -90.66255950927734,
      "activations/layer4_attention_weight_max": 121.57549285888672,
      "activations/layer4_attention_weight_min": -123.14497375488281,
      "activations/layer5_attention_weight_max": 79.39958190917969,
      "activations/layer5_attention_weight_min": -77.43798065185547,
      "activations/layer6_attention_weight_max": 52.58228302001953,
      "activations/layer6_attention_weight_min": -56.83396911621094,
      "activations/layer7_attention_weight_max": 82.08238220214844,
      "activations/layer7_attention_weight_min": -82.45899200439453,
      "activations/layer8_attention_weight_max": 41.13856887817383,
      "activations/layer8_attention_weight_min": -39.61854553222656,
      "activations/layer9_attention_weight_max": 37.44552993774414,
      "activations/layer9_attention_weight_min": -35.557552337646484,
      "epoch": 4.92,
      "learning_rate": 0.00011944469696969695,
      "loss": 2.849,
      "step": 84750
    },
    {
      "activations/layer0_attention_weight_max": 15.647881507873535,
      "activations/layer0_attention_weight_min": -10.808738708496094,
      "activations/layer10_attention_weight_max": 37.630348205566406,
      "activations/layer10_attention_weight_min": -32.91926574707031,
      "activations/layer11_attention_weight_max": 38.33353042602539,
      "activations/layer11_attention_weight_min": -31.43861198425293,
      "activations/layer12_attention_weight_max": 36.96915054321289,
      "activations/layer12_attention_weight_min": -30.969011306762695,
      "activations/layer13_attention_weight_max": 47.04800796508789,
      "activations/layer13_attention_weight_min": -32.258880615234375,
      "activations/layer14_attention_weight_max": 63.89704513549805,
      "activations/layer14_attention_weight_min": -47.114479064941406,
      "activations/layer15_attention_weight_max": 41.939842224121094,
      "activations/layer15_attention_weight_min": -28.221418380737305,
      "activations/layer16_attention_weight_max": 40.08588409423828,
      "activations/layer16_attention_weight_min": -27.315589904785156,
      "activations/layer17_attention_weight_max": 41.764888763427734,
      "activations/layer17_attention_weight_min": -24.53244400024414,
      "activations/layer18_attention_weight_max": 32.19518280029297,
      "activations/layer18_attention_weight_min": -20.736045837402344,
      "activations/layer19_attention_weight_max": 39.08382797241211,
      "activations/layer19_attention_weight_min": -29.5577335357666,
      "activations/layer1_attention_weight_max": 17.050251007080078,
      "activations/layer1_attention_weight_min": -15.181520462036133,
      "activations/layer20_attention_weight_max": 33.814449310302734,
      "activations/layer20_attention_weight_min": -23.981800079345703,
      "activations/layer21_attention_weight_max": 34.42741775512695,
      "activations/layer21_attention_weight_min": -21.212800979614258,
      "activations/layer22_attention_weight_max": 52.9395866394043,
      "activations/layer22_attention_weight_min": -29.92436408996582,
      "activations/layer23_attention_weight_max": 35.15366744995117,
      "activations/layer23_attention_weight_min": -22.655590057373047,
      "activations/layer2_attention_weight_max": 34.472679138183594,
      "activations/layer2_attention_weight_min": -35.22364044189453,
      "activations/layer3_attention_weight_max": 88.9551010131836,
      "activations/layer3_attention_weight_min": -94.60404205322266,
      "activations/layer4_attention_weight_max": 132.7425994873047,
      "activations/layer4_attention_weight_min": -122.20943450927734,
      "activations/layer5_attention_weight_max": 83.81321716308594,
      "activations/layer5_attention_weight_min": -79.09530639648438,
      "activations/layer6_attention_weight_max": 59.31085205078125,
      "activations/layer6_attention_weight_min": -55.80128479003906,
      "activations/layer7_attention_weight_max": 95.13816833496094,
      "activations/layer7_attention_weight_min": -89.789306640625,
      "activations/layer8_attention_weight_max": 45.71148681640625,
      "activations/layer8_attention_weight_min": -40.95056915283203,
      "activations/layer9_attention_weight_max": 41.660030364990234,
      "activations/layer9_attention_weight_min": -42.357418060302734,
      "epoch": 4.93,
      "learning_rate": 0.00011942575757575757,
      "loss": 2.8486,
      "step": 84800
    },
    {
      "activations/layer0_attention_weight_max": 17.400178909301758,
      "activations/layer0_attention_weight_min": -11.019325256347656,
      "activations/layer10_attention_weight_max": 35.76132583618164,
      "activations/layer10_attention_weight_min": -30.38882827758789,
      "activations/layer11_attention_weight_max": 36.23246383666992,
      "activations/layer11_attention_weight_min": -26.638748168945312,
      "activations/layer12_attention_weight_max": 40.00074005126953,
      "activations/layer12_attention_weight_min": -29.973880767822266,
      "activations/layer13_attention_weight_max": 43.26641082763672,
      "activations/layer13_attention_weight_min": -32.51728057861328,
      "activations/layer14_attention_weight_max": 59.883155822753906,
      "activations/layer14_attention_weight_min": -41.190818786621094,
      "activations/layer15_attention_weight_max": 49.83083724975586,
      "activations/layer15_attention_weight_min": -30.449031829833984,
      "activations/layer16_attention_weight_max": 38.276817321777344,
      "activations/layer16_attention_weight_min": -27.568065643310547,
      "activations/layer17_attention_weight_max": 39.51057815551758,
      "activations/layer17_attention_weight_min": -23.73222541809082,
      "activations/layer18_attention_weight_max": 31.150596618652344,
      "activations/layer18_attention_weight_min": -19.729434967041016,
      "activations/layer19_attention_weight_max": 41.95869064331055,
      "activations/layer19_attention_weight_min": -30.240142822265625,
      "activations/layer1_attention_weight_max": 18.02664566040039,
      "activations/layer1_attention_weight_min": -14.699596405029297,
      "activations/layer20_attention_weight_max": 39.38938903808594,
      "activations/layer20_attention_weight_min": -22.346538543701172,
      "activations/layer21_attention_weight_max": 36.60712432861328,
      "activations/layer21_attention_weight_min": -21.48790740966797,
      "activations/layer22_attention_weight_max": 51.6984748840332,
      "activations/layer22_attention_weight_min": -26.55124282836914,
      "activations/layer23_attention_weight_max": 34.39125442504883,
      "activations/layer23_attention_weight_min": -20.21689224243164,
      "activations/layer2_attention_weight_max": 32.89369201660156,
      "activations/layer2_attention_weight_min": -30.516986846923828,
      "activations/layer3_attention_weight_max": 86.67223358154297,
      "activations/layer3_attention_weight_min": -92.43355560302734,
      "activations/layer4_attention_weight_max": 127.50868225097656,
      "activations/layer4_attention_weight_min": -122.62239837646484,
      "activations/layer5_attention_weight_max": 79.90109252929688,
      "activations/layer5_attention_weight_min": -78.59571838378906,
      "activations/layer6_attention_weight_max": 55.9468879699707,
      "activations/layer6_attention_weight_min": -53.43446350097656,
      "activations/layer7_attention_weight_max": 86.03382110595703,
      "activations/layer7_attention_weight_min": -83.14805603027344,
      "activations/layer8_attention_weight_max": 38.67204666137695,
      "activations/layer8_attention_weight_min": -37.13459014892578,
      "activations/layer9_attention_weight_max": 36.683292388916016,
      "activations/layer9_attention_weight_min": -36.31863021850586,
      "epoch": 4.93,
      "learning_rate": 0.00011940681818181817,
      "loss": 2.8602,
      "step": 84850
    },
    {
      "activations/layer0_attention_weight_max": 16.350717544555664,
      "activations/layer0_attention_weight_min": -11.413512229919434,
      "activations/layer10_attention_weight_max": 34.15980529785156,
      "activations/layer10_attention_weight_min": -31.260374069213867,
      "activations/layer11_attention_weight_max": 31.961652755737305,
      "activations/layer11_attention_weight_min": -27.853378295898438,
      "activations/layer12_attention_weight_max": 29.140186309814453,
      "activations/layer12_attention_weight_min": -26.44776153564453,
      "activations/layer13_attention_weight_max": 45.37100601196289,
      "activations/layer13_attention_weight_min": -31.929931640625,
      "activations/layer14_attention_weight_max": 67.15975952148438,
      "activations/layer14_attention_weight_min": -49.08071517944336,
      "activations/layer15_attention_weight_max": 37.84877395629883,
      "activations/layer15_attention_weight_min": -29.195329666137695,
      "activations/layer16_attention_weight_max": 33.253170013427734,
      "activations/layer16_attention_weight_min": -28.393991470336914,
      "activations/layer17_attention_weight_max": 39.106964111328125,
      "activations/layer17_attention_weight_min": -23.51432228088379,
      "activations/layer18_attention_weight_max": 31.918025970458984,
      "activations/layer18_attention_weight_min": -19.45823097229004,
      "activations/layer19_attention_weight_max": 38.50356674194336,
      "activations/layer19_attention_weight_min": -29.251602172851562,
      "activations/layer1_attention_weight_max": 17.374900817871094,
      "activations/layer1_attention_weight_min": -14.04976749420166,
      "activations/layer20_attention_weight_max": 32.79570388793945,
      "activations/layer20_attention_weight_min": -23.02191734313965,
      "activations/layer21_attention_weight_max": 34.84108352661133,
      "activations/layer21_attention_weight_min": -20.404870986938477,
      "activations/layer22_attention_weight_max": 50.699581146240234,
      "activations/layer22_attention_weight_min": -26.60541343688965,
      "activations/layer23_attention_weight_max": 35.03284454345703,
      "activations/layer23_attention_weight_min": -20.448793411254883,
      "activations/layer2_attention_weight_max": 31.782978057861328,
      "activations/layer2_attention_weight_min": -30.633024215698242,
      "activations/layer3_attention_weight_max": 89.4858169555664,
      "activations/layer3_attention_weight_min": -92.78399658203125,
      "activations/layer4_attention_weight_max": 131.05386352539062,
      "activations/layer4_attention_weight_min": -130.15269470214844,
      "activations/layer5_attention_weight_max": 82.61883544921875,
      "activations/layer5_attention_weight_min": -82.84602355957031,
      "activations/layer6_attention_weight_max": 56.14203643798828,
      "activations/layer6_attention_weight_min": -58.201194763183594,
      "activations/layer7_attention_weight_max": 81.24591064453125,
      "activations/layer7_attention_weight_min": -84.56706237792969,
      "activations/layer8_attention_weight_max": 39.58427047729492,
      "activations/layer8_attention_weight_min": -39.94143295288086,
      "activations/layer9_attention_weight_max": 39.25600814819336,
      "activations/layer9_attention_weight_min": -34.22561264038086,
      "epoch": 4.93,
      "learning_rate": 0.00011938787878787877,
      "loss": 2.8454,
      "step": 84900
    },
    {
      "activations/layer0_attention_weight_max": 17.0426082611084,
      "activations/layer0_attention_weight_min": -11.548895835876465,
      "activations/layer10_attention_weight_max": 31.232755661010742,
      "activations/layer10_attention_weight_min": -29.30670738220215,
      "activations/layer11_attention_weight_max": 32.416812896728516,
      "activations/layer11_attention_weight_min": -27.434818267822266,
      "activations/layer12_attention_weight_max": 47.15736770629883,
      "activations/layer12_attention_weight_min": -28.916086196899414,
      "activations/layer13_attention_weight_max": 44.760826110839844,
      "activations/layer13_attention_weight_min": -31.911714553833008,
      "activations/layer14_attention_weight_max": 67.08221435546875,
      "activations/layer14_attention_weight_min": -44.44431686401367,
      "activations/layer15_attention_weight_max": 39.41733932495117,
      "activations/layer15_attention_weight_min": -26.333890914916992,
      "activations/layer16_attention_weight_max": 35.64344787597656,
      "activations/layer16_attention_weight_min": -26.89008331298828,
      "activations/layer17_attention_weight_max": 42.046287536621094,
      "activations/layer17_attention_weight_min": -23.43987274169922,
      "activations/layer18_attention_weight_max": 35.73102569580078,
      "activations/layer18_attention_weight_min": -21.75275230407715,
      "activations/layer19_attention_weight_max": 35.64722442626953,
      "activations/layer19_attention_weight_min": -27.962570190429688,
      "activations/layer1_attention_weight_max": 19.02149200439453,
      "activations/layer1_attention_weight_min": -16.342416763305664,
      "activations/layer20_attention_weight_max": 30.117586135864258,
      "activations/layer20_attention_weight_min": -22.380229949951172,
      "activations/layer21_attention_weight_max": 31.186582565307617,
      "activations/layer21_attention_weight_min": -20.981307983398438,
      "activations/layer22_attention_weight_max": 48.26886749267578,
      "activations/layer22_attention_weight_min": -27.041902542114258,
      "activations/layer23_attention_weight_max": 30.404277801513672,
      "activations/layer23_attention_weight_min": -20.528841018676758,
      "activations/layer2_attention_weight_max": 30.619369506835938,
      "activations/layer2_attention_weight_min": -30.16455078125,
      "activations/layer3_attention_weight_max": 87.6007080078125,
      "activations/layer3_attention_weight_min": -90.6080322265625,
      "activations/layer4_attention_weight_max": 126.43656158447266,
      "activations/layer4_attention_weight_min": -130.9949188232422,
      "activations/layer5_attention_weight_max": 79.53196716308594,
      "activations/layer5_attention_weight_min": -78.10525512695312,
      "activations/layer6_attention_weight_max": 56.81715774536133,
      "activations/layer6_attention_weight_min": -53.8056755065918,
      "activations/layer7_attention_weight_max": 91.11528015136719,
      "activations/layer7_attention_weight_min": -81.13865661621094,
      "activations/layer8_attention_weight_max": 36.07673263549805,
      "activations/layer8_attention_weight_min": -36.19984436035156,
      "activations/layer9_attention_weight_max": 33.374267578125,
      "activations/layer9_attention_weight_min": -35.024986267089844,
      "epoch": 4.94,
      "learning_rate": 0.00011936893939393939,
      "loss": 2.8515,
      "step": 84950
    },
    {
      "activations/layer0_attention_weight_max": 16.69469451904297,
      "activations/layer0_attention_weight_min": -11.02377986907959,
      "activations/layer10_attention_weight_max": 46.21809005737305,
      "activations/layer10_attention_weight_min": -37.42900848388672,
      "activations/layer11_attention_weight_max": 48.444557189941406,
      "activations/layer11_attention_weight_min": -35.39054870605469,
      "activations/layer12_attention_weight_max": 34.712493896484375,
      "activations/layer12_attention_weight_min": -27.79191780090332,
      "activations/layer13_attention_weight_max": 66.50874328613281,
      "activations/layer13_attention_weight_min": -39.77320861816406,
      "activations/layer14_attention_weight_max": 99.8013916015625,
      "activations/layer14_attention_weight_min": -56.293827056884766,
      "activations/layer15_attention_weight_max": 67.81497955322266,
      "activations/layer15_attention_weight_min": -31.76332664489746,
      "activations/layer16_attention_weight_max": 50.08335876464844,
      "activations/layer16_attention_weight_min": -30.129541397094727,
      "activations/layer17_attention_weight_max": 47.0789794921875,
      "activations/layer17_attention_weight_min": -25.076648712158203,
      "activations/layer18_attention_weight_max": 37.82411193847656,
      "activations/layer18_attention_weight_min": -20.05817985534668,
      "activations/layer19_attention_weight_max": 55.05670928955078,
      "activations/layer19_attention_weight_min": -29.18609619140625,
      "activations/layer1_attention_weight_max": 16.99602699279785,
      "activations/layer1_attention_weight_min": -13.637458801269531,
      "activations/layer20_attention_weight_max": 40.04092788696289,
      "activations/layer20_attention_weight_min": -22.287099838256836,
      "activations/layer21_attention_weight_max": 32.95339584350586,
      "activations/layer21_attention_weight_min": -19.925203323364258,
      "activations/layer22_attention_weight_max": 58.143524169921875,
      "activations/layer22_attention_weight_min": -27.467872619628906,
      "activations/layer23_attention_weight_max": 41.85468673706055,
      "activations/layer23_attention_weight_min": -20.47324562072754,
      "activations/layer2_attention_weight_max": 30.00452995300293,
      "activations/layer2_attention_weight_min": -30.300453186035156,
      "activations/layer3_attention_weight_max": 84.79122161865234,
      "activations/layer3_attention_weight_min": -88.01419830322266,
      "activations/layer4_attention_weight_max": 126.85002899169922,
      "activations/layer4_attention_weight_min": -129.78738403320312,
      "activations/layer5_attention_weight_max": 82.48452758789062,
      "activations/layer5_attention_weight_min": -77.25634002685547,
      "activations/layer6_attention_weight_max": 61.12918472290039,
      "activations/layer6_attention_weight_min": -56.381935119628906,
      "activations/layer7_attention_weight_max": 102.92849731445312,
      "activations/layer7_attention_weight_min": -90.86865997314453,
      "activations/layer8_attention_weight_max": 48.13722229003906,
      "activations/layer8_attention_weight_min": -44.89820861816406,
      "activations/layer9_attention_weight_max": 43.72849655151367,
      "activations/layer9_attention_weight_min": -37.36262893676758,
      "epoch": 4.94,
      "learning_rate": 0.00011934999999999999,
      "loss": 2.8783,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_loss": 2.8046875,
      "eval_runtime": 8.4597,
      "eval_samples_per_second": 507.586,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_openwebtext_loss": 2.8046875,
      "eval_openwebtext_ppl": 16.521912001744084,
      "eval_openwebtext_runtime": 8.4597,
      "eval_openwebtext_samples_per_second": 507.586,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_wikitext_loss": 3.037109375,
      "eval_wikitext_ppl": 20.84490127137742,
      "eval_wikitext_runtime": 1.9918,
      "eval_wikitext_samples_per_second": 228.941,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_lambada_loss": 2.732421875,
      "eval_lambada_ppl": 15.370066359341997,
      "eval_lambada_runtime": 9.5583,
      "eval_lambada_samples_per_second": 509.402,
      "step": 85000
    },
    {
      "activations/layer0_attention_weight_max": 14.82324504852295,
      "activations/layer0_attention_weight_min": -11.758499145507812,
      "activations/layer10_attention_weight_max": 37.54692077636719,
      "activations/layer10_attention_weight_min": -31.219499588012695,
      "activations/layer11_attention_weight_max": 40.748146057128906,
      "activations/layer11_attention_weight_min": -29.903793334960938,
      "activations/layer12_attention_weight_max": 28.36302375793457,
      "activations/layer12_attention_weight_min": -30.560447692871094,
      "activations/layer13_attention_weight_max": 50.36851501464844,
      "activations/layer13_attention_weight_min": -31.704919815063477,
      "activations/layer14_attention_weight_max": 61.03714370727539,
      "activations/layer14_attention_weight_min": -40.95176696777344,
      "activations/layer15_attention_weight_max": 37.701969146728516,
      "activations/layer15_attention_weight_min": -28.18358612060547,
      "activations/layer16_attention_weight_max": 34.18217849731445,
      "activations/layer16_attention_weight_min": -30.535673141479492,
      "activations/layer17_attention_weight_max": 37.88558578491211,
      "activations/layer17_attention_weight_min": -25.62040138244629,
      "activations/layer18_attention_weight_max": 32.51054763793945,
      "activations/layer18_attention_weight_min": -20.09395980834961,
      "activations/layer19_attention_weight_max": 42.19507598876953,
      "activations/layer19_attention_weight_min": -27.434186935424805,
      "activations/layer1_attention_weight_max": 18.318923950195312,
      "activations/layer1_attention_weight_min": -14.104050636291504,
      "activations/layer20_attention_weight_max": 34.419036865234375,
      "activations/layer20_attention_weight_min": -22.75908088684082,
      "activations/layer21_attention_weight_max": 34.05827713012695,
      "activations/layer21_attention_weight_min": -20.70836067199707,
      "activations/layer22_attention_weight_max": 55.86592483520508,
      "activations/layer22_attention_weight_min": -29.129663467407227,
      "activations/layer23_attention_weight_max": 35.57164001464844,
      "activations/layer23_attention_weight_min": -20.454139709472656,
      "activations/layer2_attention_weight_max": 32.69329833984375,
      "activations/layer2_attention_weight_min": -32.240787506103516,
      "activations/layer3_attention_weight_max": 92.9579849243164,
      "activations/layer3_attention_weight_min": -93.2888412475586,
      "activations/layer4_attention_weight_max": 135.83277893066406,
      "activations/layer4_attention_weight_min": -136.53707885742188,
      "activations/layer5_attention_weight_max": 84.92151641845703,
      "activations/layer5_attention_weight_min": -82.54344177246094,
      "activations/layer6_attention_weight_max": 59.29839324951172,
      "activations/layer6_attention_weight_min": -61.227020263671875,
      "activations/layer7_attention_weight_max": 90.90047454833984,
      "activations/layer7_attention_weight_min": -86.93601989746094,
      "activations/layer8_attention_weight_max": 43.3280029296875,
      "activations/layer8_attention_weight_min": -41.413230895996094,
      "activations/layer9_attention_weight_max": 40.476436614990234,
      "activations/layer9_attention_weight_min": -39.22462463378906,
      "epoch": 4.94,
      "learning_rate": 0.0001193310606060606,
      "loss": 2.8453,
      "step": 85050
    },
    {
      "activations/layer0_attention_weight_max": 16.725778579711914,
      "activations/layer0_attention_weight_min": -12.381556510925293,
      "activations/layer10_attention_weight_max": 34.937015533447266,
      "activations/layer10_attention_weight_min": -31.62095832824707,
      "activations/layer11_attention_weight_max": 31.335132598876953,
      "activations/layer11_attention_weight_min": -27.567798614501953,
      "activations/layer12_attention_weight_max": 30.932497024536133,
      "activations/layer12_attention_weight_min": -25.08413314819336,
      "activations/layer13_attention_weight_max": 45.04881286621094,
      "activations/layer13_attention_weight_min": -32.14688491821289,
      "activations/layer14_attention_weight_max": 66.45379638671875,
      "activations/layer14_attention_weight_min": -45.73006057739258,
      "activations/layer15_attention_weight_max": 40.576942443847656,
      "activations/layer15_attention_weight_min": -28.120311737060547,
      "activations/layer16_attention_weight_max": 45.16864776611328,
      "activations/layer16_attention_weight_min": -29.260108947753906,
      "activations/layer17_attention_weight_max": 39.852169036865234,
      "activations/layer17_attention_weight_min": -25.45725440979004,
      "activations/layer18_attention_weight_max": 35.09378433227539,
      "activations/layer18_attention_weight_min": -21.269031524658203,
      "activations/layer19_attention_weight_max": 47.26518249511719,
      "activations/layer19_attention_weight_min": -30.9205322265625,
      "activations/layer1_attention_weight_max": 17.182262420654297,
      "activations/layer1_attention_weight_min": -15.432076454162598,
      "activations/layer20_attention_weight_max": 34.22539520263672,
      "activations/layer20_attention_weight_min": -24.552350997924805,
      "activations/layer21_attention_weight_max": 35.47218704223633,
      "activations/layer21_attention_weight_min": -22.837249755859375,
      "activations/layer22_attention_weight_max": 50.8204460144043,
      "activations/layer22_attention_weight_min": -28.622468948364258,
      "activations/layer23_attention_weight_max": 36.935577392578125,
      "activations/layer23_attention_weight_min": -20.508621215820312,
      "activations/layer2_attention_weight_max": 31.89145278930664,
      "activations/layer2_attention_weight_min": -31.335330963134766,
      "activations/layer3_attention_weight_max": 88.868408203125,
      "activations/layer3_attention_weight_min": -92.1374282836914,
      "activations/layer4_attention_weight_max": 126.6383285522461,
      "activations/layer4_attention_weight_min": -133.67330932617188,
      "activations/layer5_attention_weight_max": 80.9202651977539,
      "activations/layer5_attention_weight_min": -78.24040985107422,
      "activations/layer6_attention_weight_max": 56.23979949951172,
      "activations/layer6_attention_weight_min": -58.41971969604492,
      "activations/layer7_attention_weight_max": 82.24532318115234,
      "activations/layer7_attention_weight_min": -86.93238830566406,
      "activations/layer8_attention_weight_max": 39.44791030883789,
      "activations/layer8_attention_weight_min": -38.612892150878906,
      "activations/layer9_attention_weight_max": 34.870182037353516,
      "activations/layer9_attention_weight_min": -34.45375442504883,
      "epoch": 4.94,
      "learning_rate": 0.0001193121212121212,
      "loss": 2.8578,
      "step": 85100
    },
    {
      "activations/layer0_attention_weight_max": 17.237958908081055,
      "activations/layer0_attention_weight_min": -11.121123313903809,
      "activations/layer10_attention_weight_max": 32.977657318115234,
      "activations/layer10_attention_weight_min": -28.824569702148438,
      "activations/layer11_attention_weight_max": 30.571470260620117,
      "activations/layer11_attention_weight_min": -27.6082763671875,
      "activations/layer12_attention_weight_max": 28.586441040039062,
      "activations/layer12_attention_weight_min": -26.509199142456055,
      "activations/layer13_attention_weight_max": 44.68449401855469,
      "activations/layer13_attention_weight_min": -32.287132263183594,
      "activations/layer14_attention_weight_max": 61.1656608581543,
      "activations/layer14_attention_weight_min": -44.6523551940918,
      "activations/layer15_attention_weight_max": 40.85657501220703,
      "activations/layer15_attention_weight_min": -28.184568405151367,
      "activations/layer16_attention_weight_max": 43.18444061279297,
      "activations/layer16_attention_weight_min": -26.68568992614746,
      "activations/layer17_attention_weight_max": 42.33002471923828,
      "activations/layer17_attention_weight_min": -24.186031341552734,
      "activations/layer18_attention_weight_max": 32.18251419067383,
      "activations/layer18_attention_weight_min": -20.025529861450195,
      "activations/layer19_attention_weight_max": 40.37250900268555,
      "activations/layer19_attention_weight_min": -30.04957389831543,
      "activations/layer1_attention_weight_max": 17.325273513793945,
      "activations/layer1_attention_weight_min": -15.116657257080078,
      "activations/layer20_attention_weight_max": 34.809200286865234,
      "activations/layer20_attention_weight_min": -22.330446243286133,
      "activations/layer21_attention_weight_max": 36.99324417114258,
      "activations/layer21_attention_weight_min": -21.233154296875,
      "activations/layer22_attention_weight_max": 49.303531646728516,
      "activations/layer22_attention_weight_min": -29.12153434753418,
      "activations/layer23_attention_weight_max": 32.938194274902344,
      "activations/layer23_attention_weight_min": -19.485755920410156,
      "activations/layer2_attention_weight_max": 33.39888000488281,
      "activations/layer2_attention_weight_min": -32.058387756347656,
      "activations/layer3_attention_weight_max": 87.90353393554688,
      "activations/layer3_attention_weight_min": -91.38845825195312,
      "activations/layer4_attention_weight_max": 128.7056121826172,
      "activations/layer4_attention_weight_min": -126.1402359008789,
      "activations/layer5_attention_weight_max": 86.33172607421875,
      "activations/layer5_attention_weight_min": -79.16734313964844,
      "activations/layer6_attention_weight_max": 57.04963302612305,
      "activations/layer6_attention_weight_min": -56.64653396606445,
      "activations/layer7_attention_weight_max": 86.10374450683594,
      "activations/layer7_attention_weight_min": -85.06084442138672,
      "activations/layer8_attention_weight_max": 39.89125061035156,
      "activations/layer8_attention_weight_min": -40.715816497802734,
      "activations/layer9_attention_weight_max": 35.09922790527344,
      "activations/layer9_attention_weight_min": -33.72043991088867,
      "epoch": 4.95,
      "learning_rate": 0.0001192931818181818,
      "loss": 2.8525,
      "step": 85150
    },
    {
      "activations/layer0_attention_weight_max": 15.060914039611816,
      "activations/layer0_attention_weight_min": -13.333942413330078,
      "activations/layer10_attention_weight_max": 36.10913848876953,
      "activations/layer10_attention_weight_min": -32.23283004760742,
      "activations/layer11_attention_weight_max": 33.331687927246094,
      "activations/layer11_attention_weight_min": -28.2277889251709,
      "activations/layer12_attention_weight_max": 32.83919143676758,
      "activations/layer12_attention_weight_min": -29.123178482055664,
      "activations/layer13_attention_weight_max": 48.316707611083984,
      "activations/layer13_attention_weight_min": -33.485206604003906,
      "activations/layer14_attention_weight_max": 60.79401397705078,
      "activations/layer14_attention_weight_min": -45.448387145996094,
      "activations/layer15_attention_weight_max": 48.17042541503906,
      "activations/layer15_attention_weight_min": -28.80988883972168,
      "activations/layer16_attention_weight_max": 39.140357971191406,
      "activations/layer16_attention_weight_min": -28.277019500732422,
      "activations/layer17_attention_weight_max": 44.20770263671875,
      "activations/layer17_attention_weight_min": -26.171506881713867,
      "activations/layer18_attention_weight_max": 40.57992935180664,
      "activations/layer18_attention_weight_min": -20.0604305267334,
      "activations/layer19_attention_weight_max": 40.54279708862305,
      "activations/layer19_attention_weight_min": -30.889286041259766,
      "activations/layer1_attention_weight_max": 17.02707290649414,
      "activations/layer1_attention_weight_min": -14.669641494750977,
      "activations/layer20_attention_weight_max": 33.07626724243164,
      "activations/layer20_attention_weight_min": -23.374845504760742,
      "activations/layer21_attention_weight_max": 39.4141845703125,
      "activations/layer21_attention_weight_min": -21.239185333251953,
      "activations/layer22_attention_weight_max": 55.04887008666992,
      "activations/layer22_attention_weight_min": -27.394073486328125,
      "activations/layer23_attention_weight_max": 38.72643280029297,
      "activations/layer23_attention_weight_min": -21.296720504760742,
      "activations/layer2_attention_weight_max": 32.759735107421875,
      "activations/layer2_attention_weight_min": -29.89327049255371,
      "activations/layer3_attention_weight_max": 88.79759216308594,
      "activations/layer3_attention_weight_min": -92.8657455444336,
      "activations/layer4_attention_weight_max": 131.9468994140625,
      "activations/layer4_attention_weight_min": -133.70751953125,
      "activations/layer5_attention_weight_max": 83.44503021240234,
      "activations/layer5_attention_weight_min": -80.5983657836914,
      "activations/layer6_attention_weight_max": 54.588111877441406,
      "activations/layer6_attention_weight_min": -58.897125244140625,
      "activations/layer7_attention_weight_max": 84.60807800292969,
      "activations/layer7_attention_weight_min": -85.61260986328125,
      "activations/layer8_attention_weight_max": 41.153358459472656,
      "activations/layer8_attention_weight_min": -41.169864654541016,
      "activations/layer9_attention_weight_max": 39.941627502441406,
      "activations/layer9_attention_weight_min": -34.185829162597656,
      "epoch": 4.95,
      "learning_rate": 0.00011927424242424242,
      "loss": 2.8449,
      "step": 85200
    },
    {
      "activations/layer0_attention_weight_max": 17.082490921020508,
      "activations/layer0_attention_weight_min": -11.776768684387207,
      "activations/layer10_attention_weight_max": 39.05799102783203,
      "activations/layer10_attention_weight_min": -31.96293830871582,
      "activations/layer11_attention_weight_max": 34.98480224609375,
      "activations/layer11_attention_weight_min": -27.612041473388672,
      "activations/layer12_attention_weight_max": 34.441017150878906,
      "activations/layer12_attention_weight_min": -30.021507263183594,
      "activations/layer13_attention_weight_max": 51.04124450683594,
      "activations/layer13_attention_weight_min": -34.27792739868164,
      "activations/layer14_attention_weight_max": 81.50038146972656,
      "activations/layer14_attention_weight_min": -54.46266555786133,
      "activations/layer15_attention_weight_max": 40.52313995361328,
      "activations/layer15_attention_weight_min": -29.8454532623291,
      "activations/layer16_attention_weight_max": 39.21556854248047,
      "activations/layer16_attention_weight_min": -27.98415184020996,
      "activations/layer17_attention_weight_max": 42.733158111572266,
      "activations/layer17_attention_weight_min": -25.714744567871094,
      "activations/layer18_attention_weight_max": 30.932668685913086,
      "activations/layer18_attention_weight_min": -19.781633377075195,
      "activations/layer19_attention_weight_max": 41.3788948059082,
      "activations/layer19_attention_weight_min": -30.4375,
      "activations/layer1_attention_weight_max": 17.248090744018555,
      "activations/layer1_attention_weight_min": -16.21360969543457,
      "activations/layer20_attention_weight_max": 33.68324661254883,
      "activations/layer20_attention_weight_min": -24.823772430419922,
      "activations/layer21_attention_weight_max": 36.5335807800293,
      "activations/layer21_attention_weight_min": -21.87412452697754,
      "activations/layer22_attention_weight_max": 55.324432373046875,
      "activations/layer22_attention_weight_min": -30.716632843017578,
      "activations/layer23_attention_weight_max": 36.0995979309082,
      "activations/layer23_attention_weight_min": -21.548690795898438,
      "activations/layer2_attention_weight_max": 32.07595443725586,
      "activations/layer2_attention_weight_min": -31.671127319335938,
      "activations/layer3_attention_weight_max": 92.12480926513672,
      "activations/layer3_attention_weight_min": -91.8050765991211,
      "activations/layer4_attention_weight_max": 132.6537628173828,
      "activations/layer4_attention_weight_min": -134.34425354003906,
      "activations/layer5_attention_weight_max": 85.27198028564453,
      "activations/layer5_attention_weight_min": -83.03971862792969,
      "activations/layer6_attention_weight_max": 60.45615768432617,
      "activations/layer6_attention_weight_min": -59.46376037597656,
      "activations/layer7_attention_weight_max": 89.6179428100586,
      "activations/layer7_attention_weight_min": -90.12691497802734,
      "activations/layer8_attention_weight_max": 43.239784240722656,
      "activations/layer8_attention_weight_min": -41.7115478515625,
      "activations/layer9_attention_weight_max": 40.01470184326172,
      "activations/layer9_attention_weight_min": -35.17469024658203,
      "epoch": 4.95,
      "learning_rate": 0.00011925530303030301,
      "loss": 2.8558,
      "step": 85250
    },
    {
      "activations/layer0_attention_weight_max": 15.148723602294922,
      "activations/layer0_attention_weight_min": -11.08934211730957,
      "activations/layer10_attention_weight_max": 38.473472595214844,
      "activations/layer10_attention_weight_min": -32.058555603027344,
      "activations/layer11_attention_weight_max": 40.809051513671875,
      "activations/layer11_attention_weight_min": -31.271804809570312,
      "activations/layer12_attention_weight_max": 70.0331802368164,
      "activations/layer12_attention_weight_min": -28.942289352416992,
      "activations/layer13_attention_weight_max": 90.54103088378906,
      "activations/layer13_attention_weight_min": -43.28950881958008,
      "activations/layer14_attention_weight_max": 79.69036102294922,
      "activations/layer14_attention_weight_min": -42.56373596191406,
      "activations/layer15_attention_weight_max": 70.96527099609375,
      "activations/layer15_attention_weight_min": -30.50940704345703,
      "activations/layer16_attention_weight_max": 37.00456237792969,
      "activations/layer16_attention_weight_min": -29.65843963623047,
      "activations/layer17_attention_weight_max": 38.27947998046875,
      "activations/layer17_attention_weight_min": -25.59486198425293,
      "activations/layer18_attention_weight_max": 35.47814178466797,
      "activations/layer18_attention_weight_min": -18.59185791015625,
      "activations/layer19_attention_weight_max": 39.60601806640625,
      "activations/layer19_attention_weight_min": -31.922163009643555,
      "activations/layer1_attention_weight_max": 18.39423942565918,
      "activations/layer1_attention_weight_min": -14.482168197631836,
      "activations/layer20_attention_weight_max": 34.374725341796875,
      "activations/layer20_attention_weight_min": -23.054019927978516,
      "activations/layer21_attention_weight_max": 33.81776809692383,
      "activations/layer21_attention_weight_min": -22.275850296020508,
      "activations/layer22_attention_weight_max": 47.50410461425781,
      "activations/layer22_attention_weight_min": -26.90860366821289,
      "activations/layer23_attention_weight_max": 34.09862518310547,
      "activations/layer23_attention_weight_min": -20.540109634399414,
      "activations/layer2_attention_weight_max": 30.9482479095459,
      "activations/layer2_attention_weight_min": -30.79213523864746,
      "activations/layer3_attention_weight_max": 87.69310760498047,
      "activations/layer3_attention_weight_min": -92.22696685791016,
      "activations/layer4_attention_weight_max": 132.6248016357422,
      "activations/layer4_attention_weight_min": -127.6577377319336,
      "activations/layer5_attention_weight_max": 85.23892211914062,
      "activations/layer5_attention_weight_min": -77.12657928466797,
      "activations/layer6_attention_weight_max": 60.29624557495117,
      "activations/layer6_attention_weight_min": -55.223899841308594,
      "activations/layer7_attention_weight_max": 93.1734619140625,
      "activations/layer7_attention_weight_min": -86.81005096435547,
      "activations/layer8_attention_weight_max": 42.13356399536133,
      "activations/layer8_attention_weight_min": -38.08070755004883,
      "activations/layer9_attention_weight_max": 38.1666259765625,
      "activations/layer9_attention_weight_min": -35.56660079956055,
      "epoch": 4.96,
      "learning_rate": 0.00011923636363636362,
      "loss": 2.8361,
      "step": 85300
    },
    {
      "activations/layer0_attention_weight_max": 16.830434799194336,
      "activations/layer0_attention_weight_min": -11.920060157775879,
      "activations/layer10_attention_weight_max": 38.10402297973633,
      "activations/layer10_attention_weight_min": -31.826208114624023,
      "activations/layer11_attention_weight_max": 36.29619216918945,
      "activations/layer11_attention_weight_min": -28.343116760253906,
      "activations/layer12_attention_weight_max": 34.61752700805664,
      "activations/layer12_attention_weight_min": -24.981101989746094,
      "activations/layer13_attention_weight_max": 48.818641662597656,
      "activations/layer13_attention_weight_min": -32.48200607299805,
      "activations/layer14_attention_weight_max": 79.88157653808594,
      "activations/layer14_attention_weight_min": -54.530460357666016,
      "activations/layer15_attention_weight_max": 46.2734489440918,
      "activations/layer15_attention_weight_min": -28.863428115844727,
      "activations/layer16_attention_weight_max": 43.75672149658203,
      "activations/layer16_attention_weight_min": -27.64713478088379,
      "activations/layer17_attention_weight_max": 48.98925018310547,
      "activations/layer17_attention_weight_min": -24.042451858520508,
      "activations/layer18_attention_weight_max": 36.145286560058594,
      "activations/layer18_attention_weight_min": -21.957006454467773,
      "activations/layer19_attention_weight_max": 49.17538070678711,
      "activations/layer19_attention_weight_min": -29.947805404663086,
      "activations/layer1_attention_weight_max": 17.432247161865234,
      "activations/layer1_attention_weight_min": -13.833338737487793,
      "activations/layer20_attention_weight_max": 40.74330520629883,
      "activations/layer20_attention_weight_min": -24.684120178222656,
      "activations/layer21_attention_weight_max": 36.17266845703125,
      "activations/layer21_attention_weight_min": -20.24091339111328,
      "activations/layer22_attention_weight_max": 55.04214859008789,
      "activations/layer22_attention_weight_min": -28.64370346069336,
      "activations/layer23_attention_weight_max": 38.471282958984375,
      "activations/layer23_attention_weight_min": -21.114315032958984,
      "activations/layer2_attention_weight_max": 30.882850646972656,
      "activations/layer2_attention_weight_min": -30.5540771484375,
      "activations/layer3_attention_weight_max": 87.09740447998047,
      "activations/layer3_attention_weight_min": -89.26203155517578,
      "activations/layer4_attention_weight_max": 132.55670166015625,
      "activations/layer4_attention_weight_min": -129.0025634765625,
      "activations/layer5_attention_weight_max": 83.14828491210938,
      "activations/layer5_attention_weight_min": -81.0657958984375,
      "activations/layer6_attention_weight_max": 57.98992156982422,
      "activations/layer6_attention_weight_min": -59.437381744384766,
      "activations/layer7_attention_weight_max": 88.36917114257812,
      "activations/layer7_attention_weight_min": -89.27733612060547,
      "activations/layer8_attention_weight_max": 40.71454620361328,
      "activations/layer8_attention_weight_min": -43.100685119628906,
      "activations/layer9_attention_weight_max": 39.83559799194336,
      "activations/layer9_attention_weight_min": -37.4818229675293,
      "epoch": 4.96,
      "learning_rate": 0.00011921742424242424,
      "loss": 2.8573,
      "step": 85350
    },
    {
      "activations/layer0_attention_weight_max": 16.63205909729004,
      "activations/layer0_attention_weight_min": -12.076642990112305,
      "activations/layer10_attention_weight_max": 34.48341751098633,
      "activations/layer10_attention_weight_min": -30.967971801757812,
      "activations/layer11_attention_weight_max": 34.389766693115234,
      "activations/layer11_attention_weight_min": -27.250244140625,
      "activations/layer12_attention_weight_max": 39.829750061035156,
      "activations/layer12_attention_weight_min": -26.508020401000977,
      "activations/layer13_attention_weight_max": 45.13286590576172,
      "activations/layer13_attention_weight_min": -36.9543342590332,
      "activations/layer14_attention_weight_max": 59.53943634033203,
      "activations/layer14_attention_weight_min": -47.82261276245117,
      "activations/layer15_attention_weight_max": 39.0767936706543,
      "activations/layer15_attention_weight_min": -28.44495964050293,
      "activations/layer16_attention_weight_max": 40.59649658203125,
      "activations/layer16_attention_weight_min": -30.311983108520508,
      "activations/layer17_attention_weight_max": 36.39197540283203,
      "activations/layer17_attention_weight_min": -24.81328773498535,
      "activations/layer18_attention_weight_max": 27.539175033569336,
      "activations/layer18_attention_weight_min": -22.540699005126953,
      "activations/layer19_attention_weight_max": 44.84821701049805,
      "activations/layer19_attention_weight_min": -32.42644500732422,
      "activations/layer1_attention_weight_max": 17.042783737182617,
      "activations/layer1_attention_weight_min": -16.460163116455078,
      "activations/layer20_attention_weight_max": 33.774959564208984,
      "activations/layer20_attention_weight_min": -26.198448181152344,
      "activations/layer21_attention_weight_max": 33.051795959472656,
      "activations/layer21_attention_weight_min": -23.79265594482422,
      "activations/layer22_attention_weight_max": 48.925048828125,
      "activations/layer22_attention_weight_min": -30.52790641784668,
      "activations/layer23_attention_weight_max": 32.81090545654297,
      "activations/layer23_attention_weight_min": -21.77206802368164,
      "activations/layer2_attention_weight_max": 32.14084243774414,
      "activations/layer2_attention_weight_min": -30.453330993652344,
      "activations/layer3_attention_weight_max": 88.66342163085938,
      "activations/layer3_attention_weight_min": -90.05654907226562,
      "activations/layer4_attention_weight_max": 127.02886962890625,
      "activations/layer4_attention_weight_min": -125.39151763916016,
      "activations/layer5_attention_weight_max": 82.55542755126953,
      "activations/layer5_attention_weight_min": -77.30136108398438,
      "activations/layer6_attention_weight_max": 57.441673278808594,
      "activations/layer6_attention_weight_min": -58.72087860107422,
      "activations/layer7_attention_weight_max": 87.07980346679688,
      "activations/layer7_attention_weight_min": -84.41392517089844,
      "activations/layer8_attention_weight_max": 41.88536834716797,
      "activations/layer8_attention_weight_min": -38.80335235595703,
      "activations/layer9_attention_weight_max": 37.598426818847656,
      "activations/layer9_attention_weight_min": -34.18560791015625,
      "epoch": 4.96,
      "learning_rate": 0.00011919848484848483,
      "loss": 2.8592,
      "step": 85400
    },
    {
      "activations/layer0_attention_weight_max": 17.225908279418945,
      "activations/layer0_attention_weight_min": -11.443370819091797,
      "activations/layer10_attention_weight_max": 38.07663345336914,
      "activations/layer10_attention_weight_min": -32.64625930786133,
      "activations/layer11_attention_weight_max": 33.96885681152344,
      "activations/layer11_attention_weight_min": -27.80652618408203,
      "activations/layer12_attention_weight_max": 25.016464233398438,
      "activations/layer12_attention_weight_min": -25.604646682739258,
      "activations/layer13_attention_weight_max": 45.36561965942383,
      "activations/layer13_attention_weight_min": -33.74013900756836,
      "activations/layer14_attention_weight_max": 70.547119140625,
      "activations/layer14_attention_weight_min": -48.858516693115234,
      "activations/layer15_attention_weight_max": 42.415950775146484,
      "activations/layer15_attention_weight_min": -28.57693862915039,
      "activations/layer16_attention_weight_max": 36.2317008972168,
      "activations/layer16_attention_weight_min": -27.855562210083008,
      "activations/layer17_attention_weight_max": 41.81868362426758,
      "activations/layer17_attention_weight_min": -25.704498291015625,
      "activations/layer18_attention_weight_max": 31.825468063354492,
      "activations/layer18_attention_weight_min": -22.711641311645508,
      "activations/layer19_attention_weight_max": 42.21051025390625,
      "activations/layer19_attention_weight_min": -31.00775718688965,
      "activations/layer1_attention_weight_max": 17.37321662902832,
      "activations/layer1_attention_weight_min": -14.384676933288574,
      "activations/layer20_attention_weight_max": 38.18705749511719,
      "activations/layer20_attention_weight_min": -24.564592361450195,
      "activations/layer21_attention_weight_max": 34.856807708740234,
      "activations/layer21_attention_weight_min": -23.20571517944336,
      "activations/layer22_attention_weight_max": 54.11880874633789,
      "activations/layer22_attention_weight_min": -29.86731719970703,
      "activations/layer23_attention_weight_max": 36.88216781616211,
      "activations/layer23_attention_weight_min": -22.59949493408203,
      "activations/layer2_attention_weight_max": 31.604442596435547,
      "activations/layer2_attention_weight_min": -30.441539764404297,
      "activations/layer3_attention_weight_max": 84.71544647216797,
      "activations/layer3_attention_weight_min": -88.56644439697266,
      "activations/layer4_attention_weight_max": 120.59178924560547,
      "activations/layer4_attention_weight_min": -119.87580871582031,
      "activations/layer5_attention_weight_max": 80.09307861328125,
      "activations/layer5_attention_weight_min": -77.46979522705078,
      "activations/layer6_attention_weight_max": 54.19209671020508,
      "activations/layer6_attention_weight_min": -55.97196960449219,
      "activations/layer7_attention_weight_max": 84.6015625,
      "activations/layer7_attention_weight_min": -86.76956176757812,
      "activations/layer8_attention_weight_max": 40.326602935791016,
      "activations/layer8_attention_weight_min": -40.6514778137207,
      "activations/layer9_attention_weight_max": 37.70058822631836,
      "activations/layer9_attention_weight_min": -34.192378997802734,
      "epoch": 4.97,
      "learning_rate": 0.00011917954545454544,
      "loss": 2.8731,
      "step": 85450
    },
    {
      "activations/layer0_attention_weight_max": 16.981098175048828,
      "activations/layer0_attention_weight_min": -11.186169624328613,
      "activations/layer10_attention_weight_max": 33.78792190551758,
      "activations/layer10_attention_weight_min": -29.69796371459961,
      "activations/layer11_attention_weight_max": 31.970151901245117,
      "activations/layer11_attention_weight_min": -30.99616241455078,
      "activations/layer12_attention_weight_max": 25.75492286682129,
      "activations/layer12_attention_weight_min": -26.81149673461914,
      "activations/layer13_attention_weight_max": 41.888275146484375,
      "activations/layer13_attention_weight_min": -34.07301330566406,
      "activations/layer14_attention_weight_max": 58.50788879394531,
      "activations/layer14_attention_weight_min": -43.843788146972656,
      "activations/layer15_attention_weight_max": 38.655067443847656,
      "activations/layer15_attention_weight_min": -27.56123924255371,
      "activations/layer16_attention_weight_max": 36.72203826904297,
      "activations/layer16_attention_weight_min": -28.397308349609375,
      "activations/layer17_attention_weight_max": 40.68312454223633,
      "activations/layer17_attention_weight_min": -24.531591415405273,
      "activations/layer18_attention_weight_max": 40.47231674194336,
      "activations/layer18_attention_weight_min": -19.5286922454834,
      "activations/layer19_attention_weight_max": 42.47931671142578,
      "activations/layer19_attention_weight_min": -28.913843154907227,
      "activations/layer1_attention_weight_max": 16.94231605529785,
      "activations/layer1_attention_weight_min": -12.934712409973145,
      "activations/layer20_attention_weight_max": 35.551448822021484,
      "activations/layer20_attention_weight_min": -23.05742645263672,
      "activations/layer21_attention_weight_max": 36.76416778564453,
      "activations/layer21_attention_weight_min": -21.381738662719727,
      "activations/layer22_attention_weight_max": 55.51725387573242,
      "activations/layer22_attention_weight_min": -29.254608154296875,
      "activations/layer23_attention_weight_max": 35.077964782714844,
      "activations/layer23_attention_weight_min": -20.912761688232422,
      "activations/layer2_attention_weight_max": 30.25743865966797,
      "activations/layer2_attention_weight_min": -30.90558433532715,
      "activations/layer3_attention_weight_max": 83.44488525390625,
      "activations/layer3_attention_weight_min": -88.11431121826172,
      "activations/layer4_attention_weight_max": 125.43358612060547,
      "activations/layer4_attention_weight_min": -124.7273178100586,
      "activations/layer5_attention_weight_max": 79.26382446289062,
      "activations/layer5_attention_weight_min": -78.19184875488281,
      "activations/layer6_attention_weight_max": 53.05363464355469,
      "activations/layer6_attention_weight_min": -55.36111831665039,
      "activations/layer7_attention_weight_max": 81.74431610107422,
      "activations/layer7_attention_weight_min": -81.41299438476562,
      "activations/layer8_attention_weight_max": 38.1976432800293,
      "activations/layer8_attention_weight_min": -37.195228576660156,
      "activations/layer9_attention_weight_max": 37.2296257019043,
      "activations/layer9_attention_weight_min": -33.36190414428711,
      "epoch": 4.97,
      "learning_rate": 0.00011916060606060606,
      "loss": 2.8648,
      "step": 85500
    },
    {
      "activations/layer0_attention_weight_max": 14.44981575012207,
      "activations/layer0_attention_weight_min": -11.636590003967285,
      "activations/layer10_attention_weight_max": 38.13482666015625,
      "activations/layer10_attention_weight_min": -31.463211059570312,
      "activations/layer11_attention_weight_max": 38.97045135498047,
      "activations/layer11_attention_weight_min": -28.381505966186523,
      "activations/layer12_attention_weight_max": 69.14646911621094,
      "activations/layer12_attention_weight_min": -39.82559585571289,
      "activations/layer13_attention_weight_max": 86.93379974365234,
      "activations/layer13_attention_weight_min": -44.89873504638672,
      "activations/layer14_attention_weight_max": 75.59471130371094,
      "activations/layer14_attention_weight_min": -45.57294845581055,
      "activations/layer15_attention_weight_max": 72.25232696533203,
      "activations/layer15_attention_weight_min": -31.880640029907227,
      "activations/layer16_attention_weight_max": 36.099266052246094,
      "activations/layer16_attention_weight_min": -29.62487030029297,
      "activations/layer17_attention_weight_max": 35.09217071533203,
      "activations/layer17_attention_weight_min": -24.948806762695312,
      "activations/layer18_attention_weight_max": 29.2979679107666,
      "activations/layer18_attention_weight_min": -20.629854202270508,
      "activations/layer19_attention_weight_max": 40.487918853759766,
      "activations/layer19_attention_weight_min": -29.98722267150879,
      "activations/layer1_attention_weight_max": 18.241418838500977,
      "activations/layer1_attention_weight_min": -13.483393669128418,
      "activations/layer20_attention_weight_max": 34.53337097167969,
      "activations/layer20_attention_weight_min": -22.48617172241211,
      "activations/layer21_attention_weight_max": 30.283451080322266,
      "activations/layer21_attention_weight_min": -20.62233543395996,
      "activations/layer22_attention_weight_max": 43.46357727050781,
      "activations/layer22_attention_weight_min": -25.075639724731445,
      "activations/layer23_attention_weight_max": 27.265531539916992,
      "activations/layer23_attention_weight_min": -17.518367767333984,
      "activations/layer2_attention_weight_max": 32.162193298339844,
      "activations/layer2_attention_weight_min": -31.109268188476562,
      "activations/layer3_attention_weight_max": 94.01021575927734,
      "activations/layer3_attention_weight_min": -92.46685028076172,
      "activations/layer4_attention_weight_max": 134.84397888183594,
      "activations/layer4_attention_weight_min": -130.51380920410156,
      "activations/layer5_attention_weight_max": 83.46414184570312,
      "activations/layer5_attention_weight_min": -81.0478744506836,
      "activations/layer6_attention_weight_max": 57.31087112426758,
      "activations/layer6_attention_weight_min": -58.32001495361328,
      "activations/layer7_attention_weight_max": 93.59615325927734,
      "activations/layer7_attention_weight_min": -86.89007568359375,
      "activations/layer8_attention_weight_max": 41.93245315551758,
      "activations/layer8_attention_weight_min": -41.92975616455078,
      "activations/layer9_attention_weight_max": 38.550804138183594,
      "activations/layer9_attention_weight_min": -43.9320182800293,
      "epoch": 4.97,
      "learning_rate": 0.00011914166666666666,
      "loss": 2.8539,
      "step": 85550
    },
    {
      "activations/layer0_attention_weight_max": 17.045713424682617,
      "activations/layer0_attention_weight_min": -11.740206718444824,
      "activations/layer10_attention_weight_max": 34.55567169189453,
      "activations/layer10_attention_weight_min": -31.783048629760742,
      "activations/layer11_attention_weight_max": 32.44654083251953,
      "activations/layer11_attention_weight_min": -29.747333526611328,
      "activations/layer12_attention_weight_max": 23.75653648376465,
      "activations/layer12_attention_weight_min": -29.29645347595215,
      "activations/layer13_attention_weight_max": 49.07654571533203,
      "activations/layer13_attention_weight_min": -36.66901779174805,
      "activations/layer14_attention_weight_max": 64.2713394165039,
      "activations/layer14_attention_weight_min": -49.355323791503906,
      "activations/layer15_attention_weight_max": 36.86204528808594,
      "activations/layer15_attention_weight_min": -29.192968368530273,
      "activations/layer16_attention_weight_max": 32.02259063720703,
      "activations/layer16_attention_weight_min": -26.73929214477539,
      "activations/layer17_attention_weight_max": 33.10654067993164,
      "activations/layer17_attention_weight_min": -23.060548782348633,
      "activations/layer18_attention_weight_max": 32.74197769165039,
      "activations/layer18_attention_weight_min": -20.105226516723633,
      "activations/layer19_attention_weight_max": 35.5103645324707,
      "activations/layer19_attention_weight_min": -27.466049194335938,
      "activations/layer1_attention_weight_max": 16.65578842163086,
      "activations/layer1_attention_weight_min": -14.55479907989502,
      "activations/layer20_attention_weight_max": 32.40090560913086,
      "activations/layer20_attention_weight_min": -23.924468994140625,
      "activations/layer21_attention_weight_max": 29.179954528808594,
      "activations/layer21_attention_weight_min": -22.605981826782227,
      "activations/layer22_attention_weight_max": 45.54164123535156,
      "activations/layer22_attention_weight_min": -26.852542877197266,
      "activations/layer23_attention_weight_max": 30.87958335876465,
      "activations/layer23_attention_weight_min": -22.179489135742188,
      "activations/layer2_attention_weight_max": 30.221580505371094,
      "activations/layer2_attention_weight_min": -29.98176383972168,
      "activations/layer3_attention_weight_max": 87.64424133300781,
      "activations/layer3_attention_weight_min": -89.26827239990234,
      "activations/layer4_attention_weight_max": 134.67698669433594,
      "activations/layer4_attention_weight_min": -128.0842742919922,
      "activations/layer5_attention_weight_max": 81.75493621826172,
      "activations/layer5_attention_weight_min": -82.40728759765625,
      "activations/layer6_attention_weight_max": 58.35816955566406,
      "activations/layer6_attention_weight_min": -57.13608932495117,
      "activations/layer7_attention_weight_max": 81.90924835205078,
      "activations/layer7_attention_weight_min": -84.10301971435547,
      "activations/layer8_attention_weight_max": 41.692726135253906,
      "activations/layer8_attention_weight_min": -40.89258575439453,
      "activations/layer9_attention_weight_max": 40.255950927734375,
      "activations/layer9_attention_weight_min": -35.1556510925293,
      "epoch": 4.97,
      "learning_rate": 0.00011912272727272726,
      "loss": 2.8403,
      "step": 85600
    },
    {
      "activations/layer0_attention_weight_max": 16.586158752441406,
      "activations/layer0_attention_weight_min": -11.78567123413086,
      "activations/layer10_attention_weight_max": 34.29619216918945,
      "activations/layer10_attention_weight_min": -32.73688507080078,
      "activations/layer11_attention_weight_max": 32.157066345214844,
      "activations/layer11_attention_weight_min": -30.07337188720703,
      "activations/layer12_attention_weight_max": 28.550495147705078,
      "activations/layer12_attention_weight_min": -26.87105941772461,
      "activations/layer13_attention_weight_max": 42.204246520996094,
      "activations/layer13_attention_weight_min": -33.540523529052734,
      "activations/layer14_attention_weight_max": 59.97370147705078,
      "activations/layer14_attention_weight_min": -44.9432258605957,
      "activations/layer15_attention_weight_max": 41.13936233520508,
      "activations/layer15_attention_weight_min": -28.07902717590332,
      "activations/layer16_attention_weight_max": 36.9988899230957,
      "activations/layer16_attention_weight_min": -28.21086311340332,
      "activations/layer17_attention_weight_max": 35.42386245727539,
      "activations/layer17_attention_weight_min": -25.8957462310791,
      "activations/layer18_attention_weight_max": 28.2225399017334,
      "activations/layer18_attention_weight_min": -19.375211715698242,
      "activations/layer19_attention_weight_max": 35.885738372802734,
      "activations/layer19_attention_weight_min": -28.225265502929688,
      "activations/layer1_attention_weight_max": 16.727792739868164,
      "activations/layer1_attention_weight_min": -13.739889144897461,
      "activations/layer20_attention_weight_max": 31.12835121154785,
      "activations/layer20_attention_weight_min": -22.767894744873047,
      "activations/layer21_attention_weight_max": 26.66315269470215,
      "activations/layer21_attention_weight_min": -19.331449508666992,
      "activations/layer22_attention_weight_max": 50.9055290222168,
      "activations/layer22_attention_weight_min": -29.41911506652832,
      "activations/layer23_attention_weight_max": 28.766616821289062,
      "activations/layer23_attention_weight_min": -21.470516204833984,
      "activations/layer2_attention_weight_max": 31.125736236572266,
      "activations/layer2_attention_weight_min": -29.70928192138672,
      "activations/layer3_attention_weight_max": 86.76054382324219,
      "activations/layer3_attention_weight_min": -85.34629821777344,
      "activations/layer4_attention_weight_max": 129.83790588378906,
      "activations/layer4_attention_weight_min": -123.4233627319336,
      "activations/layer5_attention_weight_max": 82.00836944580078,
      "activations/layer5_attention_weight_min": -82.23143768310547,
      "activations/layer6_attention_weight_max": 57.575355529785156,
      "activations/layer6_attention_weight_min": -57.11296463012695,
      "activations/layer7_attention_weight_max": 82.86766052246094,
      "activations/layer7_attention_weight_min": -83.52159881591797,
      "activations/layer8_attention_weight_max": 40.25934600830078,
      "activations/layer8_attention_weight_min": -38.22927474975586,
      "activations/layer9_attention_weight_max": 37.68791961669922,
      "activations/layer9_attention_weight_min": -33.14229202270508,
      "epoch": 4.98,
      "learning_rate": 0.00011910378787878787,
      "loss": 2.8558,
      "step": 85650
    },
    {
      "activations/layer0_attention_weight_max": 16.597545623779297,
      "activations/layer0_attention_weight_min": -11.733841896057129,
      "activations/layer10_attention_weight_max": 34.53802490234375,
      "activations/layer10_attention_weight_min": -33.14205551147461,
      "activations/layer11_attention_weight_max": 34.37400436401367,
      "activations/layer11_attention_weight_min": -26.090665817260742,
      "activations/layer12_attention_weight_max": 31.031089782714844,
      "activations/layer12_attention_weight_min": -26.01734161376953,
      "activations/layer13_attention_weight_max": 46.52592468261719,
      "activations/layer13_attention_weight_min": -32.210086822509766,
      "activations/layer14_attention_weight_max": 65.0920639038086,
      "activations/layer14_attention_weight_min": -44.12874221801758,
      "activations/layer15_attention_weight_max": 45.0380744934082,
      "activations/layer15_attention_weight_min": -28.356624603271484,
      "activations/layer16_attention_weight_max": 40.9112434387207,
      "activations/layer16_attention_weight_min": -30.086456298828125,
      "activations/layer17_attention_weight_max": 43.21855163574219,
      "activations/layer17_attention_weight_min": -23.7488956451416,
      "activations/layer18_attention_weight_max": 42.11298751831055,
      "activations/layer18_attention_weight_min": -20.34769630432129,
      "activations/layer19_attention_weight_max": 50.57660675048828,
      "activations/layer19_attention_weight_min": -29.692853927612305,
      "activations/layer1_attention_weight_max": 18.45525360107422,
      "activations/layer1_attention_weight_min": -17.03360366821289,
      "activations/layer20_attention_weight_max": 37.644813537597656,
      "activations/layer20_attention_weight_min": -25.895130157470703,
      "activations/layer21_attention_weight_max": 38.1459846496582,
      "activations/layer21_attention_weight_min": -21.499847412109375,
      "activations/layer22_attention_weight_max": 56.8682975769043,
      "activations/layer22_attention_weight_min": -29.77540397644043,
      "activations/layer23_attention_weight_max": 43.84421920776367,
      "activations/layer23_attention_weight_min": -23.40521240234375,
      "activations/layer2_attention_weight_max": 31.47999382019043,
      "activations/layer2_attention_weight_min": -32.65734100341797,
      "activations/layer3_attention_weight_max": 92.89046478271484,
      "activations/layer3_attention_weight_min": -95.07896423339844,
      "activations/layer4_attention_weight_max": 131.427490234375,
      "activations/layer4_attention_weight_min": -136.0924835205078,
      "activations/layer5_attention_weight_max": 83.40946960449219,
      "activations/layer5_attention_weight_min": -80.67488098144531,
      "activations/layer6_attention_weight_max": 58.82132339477539,
      "activations/layer6_attention_weight_min": -56.2144660949707,
      "activations/layer7_attention_weight_max": 90.6376953125,
      "activations/layer7_attention_weight_min": -88.7211685180664,
      "activations/layer8_attention_weight_max": 46.82210159301758,
      "activations/layer8_attention_weight_min": -42.808956146240234,
      "activations/layer9_attention_weight_max": 39.301910400390625,
      "activations/layer9_attention_weight_min": -34.4362678527832,
      "epoch": 4.98,
      "learning_rate": 0.00011908484848484848,
      "loss": 2.8541,
      "step": 85700
    },
    {
      "activations/layer0_attention_weight_max": 15.52467155456543,
      "activations/layer0_attention_weight_min": -11.292400360107422,
      "activations/layer10_attention_weight_max": 31.895803451538086,
      "activations/layer10_attention_weight_min": -32.376068115234375,
      "activations/layer11_attention_weight_max": 29.93648338317871,
      "activations/layer11_attention_weight_min": -29.067401885986328,
      "activations/layer12_attention_weight_max": 22.46652603149414,
      "activations/layer12_attention_weight_min": -26.345199584960938,
      "activations/layer13_attention_weight_max": 39.89152145385742,
      "activations/layer13_attention_weight_min": -36.85601043701172,
      "activations/layer14_attention_weight_max": 55.4830322265625,
      "activations/layer14_attention_weight_min": -46.38887405395508,
      "activations/layer15_attention_weight_max": 39.32075119018555,
      "activations/layer15_attention_weight_min": -28.7843017578125,
      "activations/layer16_attention_weight_max": 36.19974899291992,
      "activations/layer16_attention_weight_min": -30.076534271240234,
      "activations/layer17_attention_weight_max": 36.57020950317383,
      "activations/layer17_attention_weight_min": -24.873319625854492,
      "activations/layer18_attention_weight_max": 31.388652801513672,
      "activations/layer18_attention_weight_min": -20.246540069580078,
      "activations/layer19_attention_weight_max": 41.702781677246094,
      "activations/layer19_attention_weight_min": -29.905155181884766,
      "activations/layer1_attention_weight_max": 17.318843841552734,
      "activations/layer1_attention_weight_min": -13.082866668701172,
      "activations/layer20_attention_weight_max": 35.681861877441406,
      "activations/layer20_attention_weight_min": -29.185911178588867,
      "activations/layer21_attention_weight_max": 33.30443572998047,
      "activations/layer21_attention_weight_min": -21.36279296875,
      "activations/layer22_attention_weight_max": 54.169471740722656,
      "activations/layer22_attention_weight_min": -28.851810455322266,
      "activations/layer23_attention_weight_max": 30.865446090698242,
      "activations/layer23_attention_weight_min": -19.679119110107422,
      "activations/layer2_attention_weight_max": 32.94757080078125,
      "activations/layer2_attention_weight_min": -30.67548942565918,
      "activations/layer3_attention_weight_max": 91.30999755859375,
      "activations/layer3_attention_weight_min": -91.65177154541016,
      "activations/layer4_attention_weight_max": 130.84365844726562,
      "activations/layer4_attention_weight_min": -130.86402893066406,
      "activations/layer5_attention_weight_max": 81.26349639892578,
      "activations/layer5_attention_weight_min": -80.01093292236328,
      "activations/layer6_attention_weight_max": 53.20261001586914,
      "activations/layer6_attention_weight_min": -54.66451644897461,
      "activations/layer7_attention_weight_max": 78.72959899902344,
      "activations/layer7_attention_weight_min": -80.61399841308594,
      "activations/layer8_attention_weight_max": 37.18088150024414,
      "activations/layer8_attention_weight_min": -38.245487213134766,
      "activations/layer9_attention_weight_max": 36.530269622802734,
      "activations/layer9_attention_weight_min": -33.93910598754883,
      "epoch": 4.98,
      "learning_rate": 0.00011906590909090908,
      "loss": 2.8564,
      "step": 85750
    },
    {
      "activations/layer0_attention_weight_max": 16.528600692749023,
      "activations/layer0_attention_weight_min": -12.041627883911133,
      "activations/layer10_attention_weight_max": 34.516700744628906,
      "activations/layer10_attention_weight_min": -30.34735107421875,
      "activations/layer11_attention_weight_max": 33.598880767822266,
      "activations/layer11_attention_weight_min": -28.104339599609375,
      "activations/layer12_attention_weight_max": 33.20979690551758,
      "activations/layer12_attention_weight_min": -29.396732330322266,
      "activations/layer13_attention_weight_max": 41.87862014770508,
      "activations/layer13_attention_weight_min": -31.034791946411133,
      "activations/layer14_attention_weight_max": 67.37590789794922,
      "activations/layer14_attention_weight_min": -46.343624114990234,
      "activations/layer15_attention_weight_max": 38.75578308105469,
      "activations/layer15_attention_weight_min": -28.855424880981445,
      "activations/layer16_attention_weight_max": 36.6201286315918,
      "activations/layer16_attention_weight_min": -27.299551010131836,
      "activations/layer17_attention_weight_max": 36.16153335571289,
      "activations/layer17_attention_weight_min": -25.189056396484375,
      "activations/layer18_attention_weight_max": 30.428457260131836,
      "activations/layer18_attention_weight_min": -22.7873592376709,
      "activations/layer19_attention_weight_max": 39.752994537353516,
      "activations/layer19_attention_weight_min": -30.21986961364746,
      "activations/layer1_attention_weight_max": 17.173263549804688,
      "activations/layer1_attention_weight_min": -15.435076713562012,
      "activations/layer20_attention_weight_max": 33.78296661376953,
      "activations/layer20_attention_weight_min": -23.717378616333008,
      "activations/layer21_attention_weight_max": 29.84830665588379,
      "activations/layer21_attention_weight_min": -21.49881362915039,
      "activations/layer22_attention_weight_max": 46.332435607910156,
      "activations/layer22_attention_weight_min": -29.5521297454834,
      "activations/layer23_attention_weight_max": 31.440906524658203,
      "activations/layer23_attention_weight_min": -19.658504486083984,
      "activations/layer2_attention_weight_max": 33.22636032104492,
      "activations/layer2_attention_weight_min": -33.57408905029297,
      "activations/layer3_attention_weight_max": 90.3215560913086,
      "activations/layer3_attention_weight_min": -94.88825988769531,
      "activations/layer4_attention_weight_max": 130.4098358154297,
      "activations/layer4_attention_weight_min": -130.32611083984375,
      "activations/layer5_attention_weight_max": 83.26976013183594,
      "activations/layer5_attention_weight_min": -80.29003143310547,
      "activations/layer6_attention_weight_max": 58.28586959838867,
      "activations/layer6_attention_weight_min": -55.53569793701172,
      "activations/layer7_attention_weight_max": 84.9677734375,
      "activations/layer7_attention_weight_min": -82.57107543945312,
      "activations/layer8_attention_weight_max": 40.64241409301758,
      "activations/layer8_attention_weight_min": -38.18403625488281,
      "activations/layer9_attention_weight_max": 35.57770919799805,
      "activations/layer9_attention_weight_min": -32.684791564941406,
      "epoch": 4.99,
      "learning_rate": 0.00011904696969696968,
      "loss": 2.8431,
      "step": 85800
    },
    {
      "activations/layer0_attention_weight_max": 16.587806701660156,
      "activations/layer0_attention_weight_min": -12.576074600219727,
      "activations/layer10_attention_weight_max": 32.20591735839844,
      "activations/layer10_attention_weight_min": -30.77126693725586,
      "activations/layer11_attention_weight_max": 29.964920043945312,
      "activations/layer11_attention_weight_min": -27.956884384155273,
      "activations/layer12_attention_weight_max": 29.897151947021484,
      "activations/layer12_attention_weight_min": -28.351703643798828,
      "activations/layer13_attention_weight_max": 43.9513053894043,
      "activations/layer13_attention_weight_min": -31.797306060791016,
      "activations/layer14_attention_weight_max": 60.60966110229492,
      "activations/layer14_attention_weight_min": -51.85631561279297,
      "activations/layer15_attention_weight_max": 38.112728118896484,
      "activations/layer15_attention_weight_min": -31.379243850708008,
      "activations/layer16_attention_weight_max": 36.54257583618164,
      "activations/layer16_attention_weight_min": -30.59075164794922,
      "activations/layer17_attention_weight_max": 37.96880340576172,
      "activations/layer17_attention_weight_min": -26.20916175842285,
      "activations/layer18_attention_weight_max": 33.16869354248047,
      "activations/layer18_attention_weight_min": -19.837797164916992,
      "activations/layer19_attention_weight_max": 35.219966888427734,
      "activations/layer19_attention_weight_min": -30.04030418395996,
      "activations/layer1_attention_weight_max": 17.38030433654785,
      "activations/layer1_attention_weight_min": -13.506826400756836,
      "activations/layer20_attention_weight_max": 34.85234069824219,
      "activations/layer20_attention_weight_min": -23.774229049682617,
      "activations/layer21_attention_weight_max": 36.088382720947266,
      "activations/layer21_attention_weight_min": -22.010522842407227,
      "activations/layer22_attention_weight_max": 49.77707290649414,
      "activations/layer22_attention_weight_min": -28.940088272094727,
      "activations/layer23_attention_weight_max": 35.328956604003906,
      "activations/layer23_attention_weight_min": -21.300445556640625,
      "activations/layer2_attention_weight_max": 32.78484344482422,
      "activations/layer2_attention_weight_min": -32.13914489746094,
      "activations/layer3_attention_weight_max": 91.62947845458984,
      "activations/layer3_attention_weight_min": -94.4892578125,
      "activations/layer4_attention_weight_max": 135.03883361816406,
      "activations/layer4_attention_weight_min": -130.03868103027344,
      "activations/layer5_attention_weight_max": 82.08289337158203,
      "activations/layer5_attention_weight_min": -80.12406921386719,
      "activations/layer6_attention_weight_max": 56.73195266723633,
      "activations/layer6_attention_weight_min": -56.18986892700195,
      "activations/layer7_attention_weight_max": 80.76240539550781,
      "activations/layer7_attention_weight_min": -83.17855072021484,
      "activations/layer8_attention_weight_max": 39.64928436279297,
      "activations/layer8_attention_weight_min": -40.91375732421875,
      "activations/layer9_attention_weight_max": 34.839717864990234,
      "activations/layer9_attention_weight_min": -34.41318893432617,
      "epoch": 4.99,
      "learning_rate": 0.0001190280303030303,
      "loss": 2.8495,
      "step": 85850
    },
    {
      "activations/layer0_attention_weight_max": 16.332672119140625,
      "activations/layer0_attention_weight_min": -11.184340476989746,
      "activations/layer10_attention_weight_max": 36.49882888793945,
      "activations/layer10_attention_weight_min": -32.523014068603516,
      "activations/layer11_attention_weight_max": 33.084800720214844,
      "activations/layer11_attention_weight_min": -26.901416778564453,
      "activations/layer12_attention_weight_max": 33.10878372192383,
      "activations/layer12_attention_weight_min": -23.55862808227539,
      "activations/layer13_attention_weight_max": 47.04439163208008,
      "activations/layer13_attention_weight_min": -31.739864349365234,
      "activations/layer14_attention_weight_max": 64.53514862060547,
      "activations/layer14_attention_weight_min": -47.86133575439453,
      "activations/layer15_attention_weight_max": 39.64316940307617,
      "activations/layer15_attention_weight_min": -28.93548583984375,
      "activations/layer16_attention_weight_max": 37.106624603271484,
      "activations/layer16_attention_weight_min": -27.9845027923584,
      "activations/layer17_attention_weight_max": 40.243141174316406,
      "activations/layer17_attention_weight_min": -26.356494903564453,
      "activations/layer18_attention_weight_max": 34.888458251953125,
      "activations/layer18_attention_weight_min": -20.439804077148438,
      "activations/layer19_attention_weight_max": 44.776737213134766,
      "activations/layer19_attention_weight_min": -29.66117286682129,
      "activations/layer1_attention_weight_max": 18.349632263183594,
      "activations/layer1_attention_weight_min": -16.197206497192383,
      "activations/layer20_attention_weight_max": 38.294281005859375,
      "activations/layer20_attention_weight_min": -21.652727127075195,
      "activations/layer21_attention_weight_max": 42.56165313720703,
      "activations/layer21_attention_weight_min": -19.429494857788086,
      "activations/layer22_attention_weight_max": 56.49582290649414,
      "activations/layer22_attention_weight_min": -26.7327823638916,
      "activations/layer23_attention_weight_max": 34.62466049194336,
      "activations/layer23_attention_weight_min": -22.547948837280273,
      "activations/layer2_attention_weight_max": 31.700580596923828,
      "activations/layer2_attention_weight_min": -31.882366180419922,
      "activations/layer3_attention_weight_max": 87.96869659423828,
      "activations/layer3_attention_weight_min": -87.90013885498047,
      "activations/layer4_attention_weight_max": 132.0275421142578,
      "activations/layer4_attention_weight_min": -129.53598022460938,
      "activations/layer5_attention_weight_max": 87.49398040771484,
      "activations/layer5_attention_weight_min": -83.93447875976562,
      "activations/layer6_attention_weight_max": 55.83186721801758,
      "activations/layer6_attention_weight_min": -59.209014892578125,
      "activations/layer7_attention_weight_max": 82.33100891113281,
      "activations/layer7_attention_weight_min": -86.6904296875,
      "activations/layer8_attention_weight_max": 40.77040481567383,
      "activations/layer8_attention_weight_min": -40.36323547363281,
      "activations/layer9_attention_weight_max": 37.09563064575195,
      "activations/layer9_attention_weight_min": -36.783023834228516,
      "epoch": 4.99,
      "learning_rate": 0.00011900909090909091,
      "loss": 2.8595,
      "step": 85900
    },
    {
      "activations/layer0_attention_weight_max": 16.472776412963867,
      "activations/layer0_attention_weight_min": -12.214471817016602,
      "activations/layer10_attention_weight_max": 37.99176025390625,
      "activations/layer10_attention_weight_min": -31.502864837646484,
      "activations/layer11_attention_weight_max": 35.48553466796875,
      "activations/layer11_attention_weight_min": -27.161148071289062,
      "activations/layer12_attention_weight_max": 30.592817306518555,
      "activations/layer12_attention_weight_min": -30.663509368896484,
      "activations/layer13_attention_weight_max": 45.48661422729492,
      "activations/layer13_attention_weight_min": -34.36227035522461,
      "activations/layer14_attention_weight_max": 62.31149673461914,
      "activations/layer14_attention_weight_min": -43.71770477294922,
      "activations/layer15_attention_weight_max": 43.10307693481445,
      "activations/layer15_attention_weight_min": -28.817663192749023,
      "activations/layer16_attention_weight_max": 41.519588470458984,
      "activations/layer16_attention_weight_min": -28.263092041015625,
      "activations/layer17_attention_weight_max": 45.75710678100586,
      "activations/layer17_attention_weight_min": -23.726160049438477,
      "activations/layer18_attention_weight_max": 30.527427673339844,
      "activations/layer18_attention_weight_min": -23.40056610107422,
      "activations/layer19_attention_weight_max": 48.47181701660156,
      "activations/layer19_attention_weight_min": -29.906179428100586,
      "activations/layer1_attention_weight_max": 17.11172103881836,
      "activations/layer1_attention_weight_min": -14.368152618408203,
      "activations/layer20_attention_weight_max": 37.34663391113281,
      "activations/layer20_attention_weight_min": -21.99039077758789,
      "activations/layer21_attention_weight_max": 36.890777587890625,
      "activations/layer21_attention_weight_min": -19.971220016479492,
      "activations/layer22_attention_weight_max": 56.09712219238281,
      "activations/layer22_attention_weight_min": -28.075986862182617,
      "activations/layer23_attention_weight_max": 38.54410934448242,
      "activations/layer23_attention_weight_min": -19.8714542388916,
      "activations/layer2_attention_weight_max": 33.086795806884766,
      "activations/layer2_attention_weight_min": -31.432737350463867,
      "activations/layer3_attention_weight_max": 92.95781707763672,
      "activations/layer3_attention_weight_min": -90.53408813476562,
      "activations/layer4_attention_weight_max": 136.29556274414062,
      "activations/layer4_attention_weight_min": -129.06883239746094,
      "activations/layer5_attention_weight_max": 88.94097900390625,
      "activations/layer5_attention_weight_min": -79.77955627441406,
      "activations/layer6_attention_weight_max": 58.344459533691406,
      "activations/layer6_attention_weight_min": -56.05952453613281,
      "activations/layer7_attention_weight_max": 89.9079360961914,
      "activations/layer7_attention_weight_min": -78.72821044921875,
      "activations/layer8_attention_weight_max": 43.812191009521484,
      "activations/layer8_attention_weight_min": -39.20429992675781,
      "activations/layer9_attention_weight_max": 41.73341751098633,
      "activations/layer9_attention_weight_min": -35.0883903503418,
      "epoch": 4.99,
      "learning_rate": 0.0001189901515151515,
      "loss": 2.8666,
      "step": 85950
    },
    {
      "activations/layer0_attention_weight_max": 16.74784278869629,
      "activations/layer0_attention_weight_min": -11.085573196411133,
      "activations/layer10_attention_weight_max": 32.6547966003418,
      "activations/layer10_attention_weight_min": -29.585954666137695,
      "activations/layer11_attention_weight_max": 32.3620491027832,
      "activations/layer11_attention_weight_min": -26.814369201660156,
      "activations/layer12_attention_weight_max": 34.612483978271484,
      "activations/layer12_attention_weight_min": -28.66323471069336,
      "activations/layer13_attention_weight_max": 58.99116134643555,
      "activations/layer13_attention_weight_min": -36.89989471435547,
      "activations/layer14_attention_weight_max": 64.59784698486328,
      "activations/layer14_attention_weight_min": -45.34129333496094,
      "activations/layer15_attention_weight_max": 46.68803787231445,
      "activations/layer15_attention_weight_min": -30.241519927978516,
      "activations/layer16_attention_weight_max": 40.814876556396484,
      "activations/layer16_attention_weight_min": -27.835893630981445,
      "activations/layer17_attention_weight_max": 37.15684509277344,
      "activations/layer17_attention_weight_min": -24.628602981567383,
      "activations/layer18_attention_weight_max": 38.19065856933594,
      "activations/layer18_attention_weight_min": -24.08867073059082,
      "activations/layer19_attention_weight_max": 40.95306396484375,
      "activations/layer19_attention_weight_min": -29.065765380859375,
      "activations/layer1_attention_weight_max": 20.45067024230957,
      "activations/layer1_attention_weight_min": -15.431365013122559,
      "activations/layer20_attention_weight_max": 29.7022762298584,
      "activations/layer20_attention_weight_min": -23.12270736694336,
      "activations/layer21_attention_weight_max": 29.46818733215332,
      "activations/layer21_attention_weight_min": -20.727563858032227,
      "activations/layer22_attention_weight_max": 52.84169006347656,
      "activations/layer22_attention_weight_min": -29.425859451293945,
      "activations/layer23_attention_weight_max": 32.74384307861328,
      "activations/layer23_attention_weight_min": -21.533084869384766,
      "activations/layer2_attention_weight_max": 32.84556579589844,
      "activations/layer2_attention_weight_min": -32.38793182373047,
      "activations/layer3_attention_weight_max": 91.49066162109375,
      "activations/layer3_attention_weight_min": -93.10293579101562,
      "activations/layer4_attention_weight_max": 134.13137817382812,
      "activations/layer4_attention_weight_min": -133.9374237060547,
      "activations/layer5_attention_weight_max": 84.06167602539062,
      "activations/layer5_attention_weight_min": -81.00682067871094,
      "activations/layer6_attention_weight_max": 58.1958122253418,
      "activations/layer6_attention_weight_min": -59.63640213012695,
      "activations/layer7_attention_weight_max": 84.64645385742188,
      "activations/layer7_attention_weight_min": -84.46446990966797,
      "activations/layer8_attention_weight_max": 40.139434814453125,
      "activations/layer8_attention_weight_min": -40.360862731933594,
      "activations/layer9_attention_weight_max": 36.16605758666992,
      "activations/layer9_attention_weight_min": -33.50250244140625,
      "epoch": 5.0,
      "learning_rate": 0.00011897121212121211,
      "loss": 2.8484,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_loss": 2.802734375,
      "eval_runtime": 8.4744,
      "eval_samples_per_second": 506.704,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_openwebtext_loss": 2.802734375,
      "eval_openwebtext_ppl": 16.489674134905655,
      "eval_openwebtext_runtime": 8.4744,
      "eval_openwebtext_samples_per_second": 506.704,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_wikitext_loss": 3.041015625,
      "eval_wikitext_ppl": 20.92648590822237,
      "eval_wikitext_runtime": 1.9864,
      "eval_wikitext_samples_per_second": 229.566,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_lambada_loss": 2.76953125,
      "eval_lambada_ppl": 15.951155153095918,
      "eval_lambada_runtime": 9.5495,
      "eval_lambada_samples_per_second": 509.87,
      "step": 86000
    },
    {
      "activations/layer0_attention_weight_max": 15.340003967285156,
      "activations/layer0_attention_weight_min": -11.150067329406738,
      "activations/layer10_attention_weight_max": 38.72124099731445,
      "activations/layer10_attention_weight_min": -31.89743423461914,
      "activations/layer11_attention_weight_max": 35.752418518066406,
      "activations/layer11_attention_weight_min": -28.483299255371094,
      "activations/layer12_attention_weight_max": 26.599082946777344,
      "activations/layer12_attention_weight_min": -27.49577522277832,
      "activations/layer13_attention_weight_max": 44.66715621948242,
      "activations/layer13_attention_weight_min": -31.48851776123047,
      "activations/layer14_attention_weight_max": 59.3364143371582,
      "activations/layer14_attention_weight_min": -41.390708923339844,
      "activations/layer15_attention_weight_max": 43.986507415771484,
      "activations/layer15_attention_weight_min": -29.59967613220215,
      "activations/layer16_attention_weight_max": 36.38602066040039,
      "activations/layer16_attention_weight_min": -28.55084228515625,
      "activations/layer17_attention_weight_max": 38.997798919677734,
      "activations/layer17_attention_weight_min": -26.06302833557129,
      "activations/layer18_attention_weight_max": 33.28257369995117,
      "activations/layer18_attention_weight_min": -19.814882278442383,
      "activations/layer19_attention_weight_max": 40.41032409667969,
      "activations/layer19_attention_weight_min": -28.709543228149414,
      "activations/layer1_attention_weight_max": 17.224227905273438,
      "activations/layer1_attention_weight_min": -14.66465950012207,
      "activations/layer20_attention_weight_max": 31.254858016967773,
      "activations/layer20_attention_weight_min": -25.061616897583008,
      "activations/layer21_attention_weight_max": 33.338356018066406,
      "activations/layer21_attention_weight_min": -21.090314865112305,
      "activations/layer22_attention_weight_max": 50.540618896484375,
      "activations/layer22_attention_weight_min": -27.608213424682617,
      "activations/layer23_attention_weight_max": 36.01406478881836,
      "activations/layer23_attention_weight_min": -20.519546508789062,
      "activations/layer2_attention_weight_max": 32.759132385253906,
      "activations/layer2_attention_weight_min": -30.170867919921875,
      "activations/layer3_attention_weight_max": 87.50214385986328,
      "activations/layer3_attention_weight_min": -89.94432830810547,
      "activations/layer4_attention_weight_max": 126.55030822753906,
      "activations/layer4_attention_weight_min": -125.46678924560547,
      "activations/layer5_attention_weight_max": 81.29619598388672,
      "activations/layer5_attention_weight_min": -76.97621154785156,
      "activations/layer6_attention_weight_max": 58.528648376464844,
      "activations/layer6_attention_weight_min": -55.791656494140625,
      "activations/layer7_attention_weight_max": 84.29019165039062,
      "activations/layer7_attention_weight_min": -77.7989501953125,
      "activations/layer8_attention_weight_max": 42.47129821777344,
      "activations/layer8_attention_weight_min": -39.25726318359375,
      "activations/layer9_attention_weight_max": 42.50141525268555,
      "activations/layer9_attention_weight_min": -35.08700942993164,
      "epoch": 5.0,
      "learning_rate": 0.00011895227272727273,
      "loss": 2.8511,
      "step": 86050
    },
    {
      "activations/layer0_attention_weight_max": 16.822296142578125,
      "activations/layer0_attention_weight_min": -11.533061981201172,
      "activations/layer10_attention_weight_max": 33.21893310546875,
      "activations/layer10_attention_weight_min": -31.97844696044922,
      "activations/layer11_attention_weight_max": 33.021053314208984,
      "activations/layer11_attention_weight_min": -28.501644134521484,
      "activations/layer12_attention_weight_max": 25.07019805908203,
      "activations/layer12_attention_weight_min": -29.3185977935791,
      "activations/layer13_attention_weight_max": 44.72825622558594,
      "activations/layer13_attention_weight_min": -32.423973083496094,
      "activations/layer14_attention_weight_max": 69.97064971923828,
      "activations/layer14_attention_weight_min": -45.180686950683594,
      "activations/layer15_attention_weight_max": 41.21726989746094,
      "activations/layer15_attention_weight_min": -30.600574493408203,
      "activations/layer16_attention_weight_max": 38.26961135864258,
      "activations/layer16_attention_weight_min": -27.068058013916016,
      "activations/layer17_attention_weight_max": 41.88682174682617,
      "activations/layer17_attention_weight_min": -24.76859474182129,
      "activations/layer18_attention_weight_max": 34.821556091308594,
      "activations/layer18_attention_weight_min": -20.905805587768555,
      "activations/layer19_attention_weight_max": 43.23705291748047,
      "activations/layer19_attention_weight_min": -29.060983657836914,
      "activations/layer1_attention_weight_max": 17.54698371887207,
      "activations/layer1_attention_weight_min": -14.085533142089844,
      "activations/layer20_attention_weight_max": 34.872703552246094,
      "activations/layer20_attention_weight_min": -23.824769973754883,
      "activations/layer21_attention_weight_max": 33.188507080078125,
      "activations/layer21_attention_weight_min": -22.15462875366211,
      "activations/layer22_attention_weight_max": 58.8890495300293,
      "activations/layer22_attention_weight_min": -26.709890365600586,
      "activations/layer23_attention_weight_max": 37.81903076171875,
      "activations/layer23_attention_weight_min": -19.066532135009766,
      "activations/layer2_attention_weight_max": 31.565387725830078,
      "activations/layer2_attention_weight_min": -30.548969268798828,
      "activations/layer3_attention_weight_max": 89.54741668701172,
      "activations/layer3_attention_weight_min": -89.76783752441406,
      "activations/layer4_attention_weight_max": 126.6266098022461,
      "activations/layer4_attention_weight_min": -126.3377914428711,
      "activations/layer5_attention_weight_max": 78.22323608398438,
      "activations/layer5_attention_weight_min": -77.42445373535156,
      "activations/layer6_attention_weight_max": 54.70200729370117,
      "activations/layer6_attention_weight_min": -56.89451217651367,
      "activations/layer7_attention_weight_max": 80.34673309326172,
      "activations/layer7_attention_weight_min": -80.8104248046875,
      "activations/layer8_attention_weight_max": 40.44612503051758,
      "activations/layer8_attention_weight_min": -44.207305908203125,
      "activations/layer9_attention_weight_max": 37.30980682373047,
      "activations/layer9_attention_weight_min": -35.15577697753906,
      "epoch": 5.0,
      "learning_rate": 0.00011893295454545454,
      "loss": 2.9012,
      "step": 86100
    },
    {
      "activations/layer0_attention_weight_max": 17.131567001342773,
      "activations/layer0_attention_weight_min": -11.96738052368164,
      "activations/layer10_attention_weight_max": 32.15755844116211,
      "activations/layer10_attention_weight_min": -30.395614624023438,
      "activations/layer11_attention_weight_max": 31.381572723388672,
      "activations/layer11_attention_weight_min": -28.7387752532959,
      "activations/layer12_attention_weight_max": 28.543134689331055,
      "activations/layer12_attention_weight_min": -27.40665054321289,
      "activations/layer13_attention_weight_max": 44.78050231933594,
      "activations/layer13_attention_weight_min": -32.73320007324219,
      "activations/layer14_attention_weight_max": 58.57716751098633,
      "activations/layer14_attention_weight_min": -44.461647033691406,
      "activations/layer15_attention_weight_max": 40.45480728149414,
      "activations/layer15_attention_weight_min": -27.881366729736328,
      "activations/layer16_attention_weight_max": 35.65164566040039,
      "activations/layer16_attention_weight_min": -28.954792022705078,
      "activations/layer17_attention_weight_max": 39.7763671875,
      "activations/layer17_attention_weight_min": -24.613670349121094,
      "activations/layer18_attention_weight_max": 33.73684310913086,
      "activations/layer18_attention_weight_min": -21.1339111328125,
      "activations/layer19_attention_weight_max": 38.3251838684082,
      "activations/layer19_attention_weight_min": -30.9337158203125,
      "activations/layer1_attention_weight_max": 17.913923263549805,
      "activations/layer1_attention_weight_min": -14.339874267578125,
      "activations/layer20_attention_weight_max": 34.622493743896484,
      "activations/layer20_attention_weight_min": -23.271970748901367,
      "activations/layer21_attention_weight_max": 32.66482925415039,
      "activations/layer21_attention_weight_min": -20.189254760742188,
      "activations/layer22_attention_weight_max": 52.04209899902344,
      "activations/layer22_attention_weight_min": -26.984270095825195,
      "activations/layer23_attention_weight_max": 34.58803176879883,
      "activations/layer23_attention_weight_min": -20.329713821411133,
      "activations/layer2_attention_weight_max": 31.27758026123047,
      "activations/layer2_attention_weight_min": -30.9123592376709,
      "activations/layer3_attention_weight_max": 85.42877197265625,
      "activations/layer3_attention_weight_min": -85.19497680664062,
      "activations/layer4_attention_weight_max": 127.84815979003906,
      "activations/layer4_attention_weight_min": -123.24369049072266,
      "activations/layer5_attention_weight_max": 81.30961608886719,
      "activations/layer5_attention_weight_min": -77.85493469238281,
      "activations/layer6_attention_weight_max": 57.90270233154297,
      "activations/layer6_attention_weight_min": -55.902854919433594,
      "activations/layer7_attention_weight_max": 84.83138275146484,
      "activations/layer7_attention_weight_min": -83.68622589111328,
      "activations/layer8_attention_weight_max": 38.31510543823242,
      "activations/layer8_attention_weight_min": -38.61188507080078,
      "activations/layer9_attention_weight_max": 34.84962844848633,
      "activations/layer9_attention_weight_min": -35.812259674072266,
      "epoch": 5.01,
      "learning_rate": 0.00011891401515151514,
      "loss": 2.8596,
      "step": 86150
    },
    {
      "activations/layer0_attention_weight_max": 16.862815856933594,
      "activations/layer0_attention_weight_min": -12.915982246398926,
      "activations/layer10_attention_weight_max": 32.207794189453125,
      "activations/layer10_attention_weight_min": -31.938674926757812,
      "activations/layer11_attention_weight_max": 31.39785385131836,
      "activations/layer11_attention_weight_min": -27.60790252685547,
      "activations/layer12_attention_weight_max": 33.372802734375,
      "activations/layer12_attention_weight_min": -26.93917465209961,
      "activations/layer13_attention_weight_max": 39.912879943847656,
      "activations/layer13_attention_weight_min": -29.2171630859375,
      "activations/layer14_attention_weight_max": 55.3335075378418,
      "activations/layer14_attention_weight_min": -38.093109130859375,
      "activations/layer15_attention_weight_max": 36.76316833496094,
      "activations/layer15_attention_weight_min": -27.98052406311035,
      "activations/layer16_attention_weight_max": 33.474369049072266,
      "activations/layer16_attention_weight_min": -27.319137573242188,
      "activations/layer17_attention_weight_max": 36.160064697265625,
      "activations/layer17_attention_weight_min": -24.904279708862305,
      "activations/layer18_attention_weight_max": 27.362619400024414,
      "activations/layer18_attention_weight_min": -20.956584930419922,
      "activations/layer19_attention_weight_max": 37.53971862792969,
      "activations/layer19_attention_weight_min": -27.7803955078125,
      "activations/layer1_attention_weight_max": 18.369766235351562,
      "activations/layer1_attention_weight_min": -14.324469566345215,
      "activations/layer20_attention_weight_max": 31.598255157470703,
      "activations/layer20_attention_weight_min": -25.0799503326416,
      "activations/layer21_attention_weight_max": 31.627477645874023,
      "activations/layer21_attention_weight_min": -22.3630313873291,
      "activations/layer22_attention_weight_max": 47.365234375,
      "activations/layer22_attention_weight_min": -29.53867530822754,
      "activations/layer23_attention_weight_max": 31.790218353271484,
      "activations/layer23_attention_weight_min": -21.369457244873047,
      "activations/layer2_attention_weight_max": 31.648113250732422,
      "activations/layer2_attention_weight_min": -30.63245391845703,
      "activations/layer3_attention_weight_max": 88.71270751953125,
      "activations/layer3_attention_weight_min": -88.90789031982422,
      "activations/layer4_attention_weight_max": 133.79795837402344,
      "activations/layer4_attention_weight_min": -123.78437805175781,
      "activations/layer5_attention_weight_max": 83.9013442993164,
      "activations/layer5_attention_weight_min": -79.86325073242188,
      "activations/layer6_attention_weight_max": 53.43088912963867,
      "activations/layer6_attention_weight_min": -55.80583953857422,
      "activations/layer7_attention_weight_max": 78.9053955078125,
      "activations/layer7_attention_weight_min": -81.37715148925781,
      "activations/layer8_attention_weight_max": 39.45442581176758,
      "activations/layer8_attention_weight_min": -38.705902099609375,
      "activations/layer9_attention_weight_max": 36.9143180847168,
      "activations/layer9_attention_weight_min": -33.354034423828125,
      "epoch": 5.01,
      "learning_rate": 0.00011889507575757574,
      "loss": 2.8395,
      "step": 86200
    },
    {
      "activations/layer0_attention_weight_max": 16.543155670166016,
      "activations/layer0_attention_weight_min": -11.953985214233398,
      "activations/layer10_attention_weight_max": 33.20392608642578,
      "activations/layer10_attention_weight_min": -29.377864837646484,
      "activations/layer11_attention_weight_max": 30.81990623474121,
      "activations/layer11_attention_weight_min": -28.024524688720703,
      "activations/layer12_attention_weight_max": 30.482641220092773,
      "activations/layer12_attention_weight_min": -28.60804557800293,
      "activations/layer13_attention_weight_max": 45.82101821899414,
      "activations/layer13_attention_weight_min": -33.348609924316406,
      "activations/layer14_attention_weight_max": 59.28211975097656,
      "activations/layer14_attention_weight_min": -43.59288787841797,
      "activations/layer15_attention_weight_max": 42.14811706542969,
      "activations/layer15_attention_weight_min": -27.9284725189209,
      "activations/layer16_attention_weight_max": 38.267032623291016,
      "activations/layer16_attention_weight_min": -28.295366287231445,
      "activations/layer17_attention_weight_max": 37.47116470336914,
      "activations/layer17_attention_weight_min": -25.481082916259766,
      "activations/layer18_attention_weight_max": 30.319547653198242,
      "activations/layer18_attention_weight_min": -21.313045501708984,
      "activations/layer19_attention_weight_max": 40.33916473388672,
      "activations/layer19_attention_weight_min": -28.770099639892578,
      "activations/layer1_attention_weight_max": 17.01318359375,
      "activations/layer1_attention_weight_min": -13.78523063659668,
      "activations/layer20_attention_weight_max": 30.48723602294922,
      "activations/layer20_attention_weight_min": -23.380245208740234,
      "activations/layer21_attention_weight_max": 30.551212310791016,
      "activations/layer21_attention_weight_min": -20.48978042602539,
      "activations/layer22_attention_weight_max": 54.07073974609375,
      "activations/layer22_attention_weight_min": -28.95656394958496,
      "activations/layer23_attention_weight_max": 33.269508361816406,
      "activations/layer23_attention_weight_min": -22.092504501342773,
      "activations/layer2_attention_weight_max": 33.40931701660156,
      "activations/layer2_attention_weight_min": -29.68069076538086,
      "activations/layer3_attention_weight_max": 92.46300506591797,
      "activations/layer3_attention_weight_min": -89.25357818603516,
      "activations/layer4_attention_weight_max": 137.8730926513672,
      "activations/layer4_attention_weight_min": -124.0770492553711,
      "activations/layer5_attention_weight_max": 87.16236877441406,
      "activations/layer5_attention_weight_min": -78.90103149414062,
      "activations/layer6_attention_weight_max": 60.414459228515625,
      "activations/layer6_attention_weight_min": -57.42117691040039,
      "activations/layer7_attention_weight_max": 84.49996948242188,
      "activations/layer7_attention_weight_min": -80.614013671875,
      "activations/layer8_attention_weight_max": 40.63202667236328,
      "activations/layer8_attention_weight_min": -38.06697082519531,
      "activations/layer9_attention_weight_max": 36.191532135009766,
      "activations/layer9_attention_weight_min": -32.94692611694336,
      "epoch": 5.01,
      "learning_rate": 0.00011887613636363636,
      "loss": 2.8382,
      "step": 86250
    },
    {
      "activations/layer0_attention_weight_max": 17.007862091064453,
      "activations/layer0_attention_weight_min": -11.674278259277344,
      "activations/layer10_attention_weight_max": 33.85737991333008,
      "activations/layer10_attention_weight_min": -31.512235641479492,
      "activations/layer11_attention_weight_max": 31.76886558532715,
      "activations/layer11_attention_weight_min": -28.46798324584961,
      "activations/layer12_attention_weight_max": 27.422990798950195,
      "activations/layer12_attention_weight_min": -27.873567581176758,
      "activations/layer13_attention_weight_max": 44.86134719848633,
      "activations/layer13_attention_weight_min": -30.81093406677246,
      "activations/layer14_attention_weight_max": 64.17158508300781,
      "activations/layer14_attention_weight_min": -47.20795440673828,
      "activations/layer15_attention_weight_max": 37.60072326660156,
      "activations/layer15_attention_weight_min": -29.461984634399414,
      "activations/layer16_attention_weight_max": 40.66051483154297,
      "activations/layer16_attention_weight_min": -28.122095108032227,
      "activations/layer17_attention_weight_max": 40.49734878540039,
      "activations/layer17_attention_weight_min": -25.225412368774414,
      "activations/layer18_attention_weight_max": 31.779741287231445,
      "activations/layer18_attention_weight_min": -19.9080867767334,
      "activations/layer19_attention_weight_max": 39.46101379394531,
      "activations/layer19_attention_weight_min": -30.6571102142334,
      "activations/layer1_attention_weight_max": 17.919260025024414,
      "activations/layer1_attention_weight_min": -13.475062370300293,
      "activations/layer20_attention_weight_max": 30.52043914794922,
      "activations/layer20_attention_weight_min": -22.64531898498535,
      "activations/layer21_attention_weight_max": 37.69052505493164,
      "activations/layer21_attention_weight_min": -19.782283782958984,
      "activations/layer22_attention_weight_max": 46.76355743408203,
      "activations/layer22_attention_weight_min": -27.447547912597656,
      "activations/layer23_attention_weight_max": 37.615631103515625,
      "activations/layer23_attention_weight_min": -19.975126266479492,
      "activations/layer2_attention_weight_max": 31.6141357421875,
      "activations/layer2_attention_weight_min": -30.629653930664062,
      "activations/layer3_attention_weight_max": 88.1712646484375,
      "activations/layer3_attention_weight_min": -91.0312271118164,
      "activations/layer4_attention_weight_max": 129.78794860839844,
      "activations/layer4_attention_weight_min": -129.9579315185547,
      "activations/layer5_attention_weight_max": 86.83500671386719,
      "activations/layer5_attention_weight_min": -83.10111999511719,
      "activations/layer6_attention_weight_max": 55.26462936401367,
      "activations/layer6_attention_weight_min": -55.846961975097656,
      "activations/layer7_attention_weight_max": 90.54084014892578,
      "activations/layer7_attention_weight_min": -84.42317962646484,
      "activations/layer8_attention_weight_max": 42.3412971496582,
      "activations/layer8_attention_weight_min": -39.89397048950195,
      "activations/layer9_attention_weight_max": 37.01981735229492,
      "activations/layer9_attention_weight_min": -33.182647705078125,
      "epoch": 5.01,
      "learning_rate": 0.00011885719696969696,
      "loss": 2.8415,
      "step": 86300
    },
    {
      "activations/layer0_attention_weight_max": 16.092971801757812,
      "activations/layer0_attention_weight_min": -11.983352661132812,
      "activations/layer10_attention_weight_max": 36.52479934692383,
      "activations/layer10_attention_weight_min": -30.68254852294922,
      "activations/layer11_attention_weight_max": 34.65376663208008,
      "activations/layer11_attention_weight_min": -28.805557250976562,
      "activations/layer12_attention_weight_max": 25.209518432617188,
      "activations/layer12_attention_weight_min": -25.60651969909668,
      "activations/layer13_attention_weight_max": 46.805641174316406,
      "activations/layer13_attention_weight_min": -32.424285888671875,
      "activations/layer14_attention_weight_max": 66.57538604736328,
      "activations/layer14_attention_weight_min": -45.033470153808594,
      "activations/layer15_attention_weight_max": 46.14780044555664,
      "activations/layer15_attention_weight_min": -30.435327529907227,
      "activations/layer16_attention_weight_max": 38.789520263671875,
      "activations/layer16_attention_weight_min": -30.376188278198242,
      "activations/layer17_attention_weight_max": 42.20756912231445,
      "activations/layer17_attention_weight_min": -24.09630012512207,
      "activations/layer18_attention_weight_max": 34.70707321166992,
      "activations/layer18_attention_weight_min": -21.139419555664062,
      "activations/layer19_attention_weight_max": 44.5320930480957,
      "activations/layer19_attention_weight_min": -30.853483200073242,
      "activations/layer1_attention_weight_max": 16.683225631713867,
      "activations/layer1_attention_weight_min": -15.984421730041504,
      "activations/layer20_attention_weight_max": 34.360069274902344,
      "activations/layer20_attention_weight_min": -25.459293365478516,
      "activations/layer21_attention_weight_max": 35.22508239746094,
      "activations/layer21_attention_weight_min": -24.042810440063477,
      "activations/layer22_attention_weight_max": 56.24678421020508,
      "activations/layer22_attention_weight_min": -27.9074649810791,
      "activations/layer23_attention_weight_max": 35.0681266784668,
      "activations/layer23_attention_weight_min": -20.54100799560547,
      "activations/layer2_attention_weight_max": 32.886070251464844,
      "activations/layer2_attention_weight_min": -30.744102478027344,
      "activations/layer3_attention_weight_max": 92.64448547363281,
      "activations/layer3_attention_weight_min": -93.82867431640625,
      "activations/layer4_attention_weight_max": 128.79078674316406,
      "activations/layer4_attention_weight_min": -129.4869842529297,
      "activations/layer5_attention_weight_max": 83.06770324707031,
      "activations/layer5_attention_weight_min": -82.91253662109375,
      "activations/layer6_attention_weight_max": 64.1251220703125,
      "activations/layer6_attention_weight_min": -61.17668151855469,
      "activations/layer7_attention_weight_max": 86.64199829101562,
      "activations/layer7_attention_weight_min": -89.47718811035156,
      "activations/layer8_attention_weight_max": 40.462242126464844,
      "activations/layer8_attention_weight_min": -40.46401596069336,
      "activations/layer9_attention_weight_max": 36.3834114074707,
      "activations/layer9_attention_weight_min": -35.18449020385742,
      "epoch": 5.02,
      "learning_rate": 0.00011883825757575757,
      "loss": 2.8462,
      "step": 86350
    },
    {
      "activations/layer0_attention_weight_max": 16.699485778808594,
      "activations/layer0_attention_weight_min": -11.081557273864746,
      "activations/layer10_attention_weight_max": 35.3819694519043,
      "activations/layer10_attention_weight_min": -30.051259994506836,
      "activations/layer11_attention_weight_max": 31.94814109802246,
      "activations/layer11_attention_weight_min": -28.68586540222168,
      "activations/layer12_attention_weight_max": 31.493879318237305,
      "activations/layer12_attention_weight_min": -25.85120391845703,
      "activations/layer13_attention_weight_max": 47.42420959472656,
      "activations/layer13_attention_weight_min": -37.034339904785156,
      "activations/layer14_attention_weight_max": 54.7816162109375,
      "activations/layer14_attention_weight_min": -42.881919860839844,
      "activations/layer15_attention_weight_max": 42.59238815307617,
      "activations/layer15_attention_weight_min": -31.715993881225586,
      "activations/layer16_attention_weight_max": 39.87906265258789,
      "activations/layer16_attention_weight_min": -28.99393081665039,
      "activations/layer17_attention_weight_max": 37.830074310302734,
      "activations/layer17_attention_weight_min": -23.970483779907227,
      "activations/layer18_attention_weight_max": 33.69183349609375,
      "activations/layer18_attention_weight_min": -21.272668838500977,
      "activations/layer19_attention_weight_max": 44.49806213378906,
      "activations/layer19_attention_weight_min": -29.11982536315918,
      "activations/layer1_attention_weight_max": 17.446884155273438,
      "activations/layer1_attention_weight_min": -13.668272018432617,
      "activations/layer20_attention_weight_max": 31.61370849609375,
      "activations/layer20_attention_weight_min": -23.2939510345459,
      "activations/layer21_attention_weight_max": 27.759536743164062,
      "activations/layer21_attention_weight_min": -20.611677169799805,
      "activations/layer22_attention_weight_max": 48.94975662231445,
      "activations/layer22_attention_weight_min": -29.92997932434082,
      "activations/layer23_attention_weight_max": 36.995723724365234,
      "activations/layer23_attention_weight_min": -19.823497772216797,
      "activations/layer2_attention_weight_max": 32.103309631347656,
      "activations/layer2_attention_weight_min": -30.503217697143555,
      "activations/layer3_attention_weight_max": 88.26051330566406,
      "activations/layer3_attention_weight_min": -93.71249389648438,
      "activations/layer4_attention_weight_max": 134.9656219482422,
      "activations/layer4_attention_weight_min": -127.8874740600586,
      "activations/layer5_attention_weight_max": 81.50978088378906,
      "activations/layer5_attention_weight_min": -80.31198120117188,
      "activations/layer6_attention_weight_max": 54.20143508911133,
      "activations/layer6_attention_weight_min": -53.687740325927734,
      "activations/layer7_attention_weight_max": 87.73115539550781,
      "activations/layer7_attention_weight_min": -81.05042266845703,
      "activations/layer8_attention_weight_max": 39.81681442260742,
      "activations/layer8_attention_weight_min": -37.820621490478516,
      "activations/layer9_attention_weight_max": 37.89571762084961,
      "activations/layer9_attention_weight_min": -33.71946334838867,
      "epoch": 5.02,
      "learning_rate": 0.00011881969696969696,
      "loss": 2.8368,
      "step": 86400
    },
    {
      "activations/layer0_attention_weight_max": 17.340099334716797,
      "activations/layer0_attention_weight_min": -11.213920593261719,
      "activations/layer10_attention_weight_max": 34.699188232421875,
      "activations/layer10_attention_weight_min": -30.251474380493164,
      "activations/layer11_attention_weight_max": 35.3599739074707,
      "activations/layer11_attention_weight_min": -28.12672233581543,
      "activations/layer12_attention_weight_max": 34.141746520996094,
      "activations/layer12_attention_weight_min": -28.77556800842285,
      "activations/layer13_attention_weight_max": 53.101314544677734,
      "activations/layer13_attention_weight_min": -39.6901741027832,
      "activations/layer14_attention_weight_max": 67.464111328125,
      "activations/layer14_attention_weight_min": -50.35059356689453,
      "activations/layer15_attention_weight_max": 41.556514739990234,
      "activations/layer15_attention_weight_min": -29.747516632080078,
      "activations/layer16_attention_weight_max": 40.43042755126953,
      "activations/layer16_attention_weight_min": -28.555166244506836,
      "activations/layer17_attention_weight_max": 38.1828727722168,
      "activations/layer17_attention_weight_min": -24.384769439697266,
      "activations/layer18_attention_weight_max": 31.26793098449707,
      "activations/layer18_attention_weight_min": -20.137100219726562,
      "activations/layer19_attention_weight_max": 40.239715576171875,
      "activations/layer19_attention_weight_min": -28.23744773864746,
      "activations/layer1_attention_weight_max": 16.900476455688477,
      "activations/layer1_attention_weight_min": -15.00251579284668,
      "activations/layer20_attention_weight_max": 31.628162384033203,
      "activations/layer20_attention_weight_min": -25.5002384185791,
      "activations/layer21_attention_weight_max": 33.68288040161133,
      "activations/layer21_attention_weight_min": -23.20255470275879,
      "activations/layer22_attention_weight_max": 50.79867172241211,
      "activations/layer22_attention_weight_min": -28.1418514251709,
      "activations/layer23_attention_weight_max": 34.718482971191406,
      "activations/layer23_attention_weight_min": -21.998756408691406,
      "activations/layer2_attention_weight_max": 31.265079498291016,
      "activations/layer2_attention_weight_min": -33.012794494628906,
      "activations/layer3_attention_weight_max": 91.54370880126953,
      "activations/layer3_attention_weight_min": -92.12275695800781,
      "activations/layer4_attention_weight_max": 124.19298553466797,
      "activations/layer4_attention_weight_min": -124.98477935791016,
      "activations/layer5_attention_weight_max": 79.59149932861328,
      "activations/layer5_attention_weight_min": -77.76142883300781,
      "activations/layer6_attention_weight_max": 53.60389709472656,
      "activations/layer6_attention_weight_min": -58.708404541015625,
      "activations/layer7_attention_weight_max": 78.58723449707031,
      "activations/layer7_attention_weight_min": -83.8879623413086,
      "activations/layer8_attention_weight_max": 41.30451583862305,
      "activations/layer8_attention_weight_min": -37.85639953613281,
      "activations/layer9_attention_weight_max": 35.616722106933594,
      "activations/layer9_attention_weight_min": -33.563087463378906,
      "epoch": 5.02,
      "learning_rate": 0.00011880075757575757,
      "loss": 2.8455,
      "step": 86450
    },
    {
      "activations/layer0_attention_weight_max": 17.363889694213867,
      "activations/layer0_attention_weight_min": -12.040452003479004,
      "activations/layer10_attention_weight_max": 33.85536575317383,
      "activations/layer10_attention_weight_min": -30.997299194335938,
      "activations/layer11_attention_weight_max": 37.966339111328125,
      "activations/layer11_attention_weight_min": -29.961782455444336,
      "activations/layer12_attention_weight_max": 51.347599029541016,
      "activations/layer12_attention_weight_min": -29.414243698120117,
      "activations/layer13_attention_weight_max": 50.608882904052734,
      "activations/layer13_attention_weight_min": -33.43706512451172,
      "activations/layer14_attention_weight_max": 53.04521942138672,
      "activations/layer14_attention_weight_min": -43.321990966796875,
      "activations/layer15_attention_weight_max": 35.62184524536133,
      "activations/layer15_attention_weight_min": -28.553512573242188,
      "activations/layer16_attention_weight_max": 31.726314544677734,
      "activations/layer16_attention_weight_min": -27.147674560546875,
      "activations/layer17_attention_weight_max": 36.40531539916992,
      "activations/layer17_attention_weight_min": -24.421646118164062,
      "activations/layer18_attention_weight_max": 34.702484130859375,
      "activations/layer18_attention_weight_min": -22.356124877929688,
      "activations/layer19_attention_weight_max": 36.479766845703125,
      "activations/layer19_attention_weight_min": -28.760221481323242,
      "activations/layer1_attention_weight_max": 18.51358413696289,
      "activations/layer1_attention_weight_min": -13.322699546813965,
      "activations/layer20_attention_weight_max": 29.08030891418457,
      "activations/layer20_attention_weight_min": -23.943326950073242,
      "activations/layer21_attention_weight_max": 33.593509674072266,
      "activations/layer21_attention_weight_min": -19.699718475341797,
      "activations/layer22_attention_weight_max": 42.783599853515625,
      "activations/layer22_attention_weight_min": -27.35010528564453,
      "activations/layer23_attention_weight_max": 31.745914459228516,
      "activations/layer23_attention_weight_min": -20.291812896728516,
      "activations/layer2_attention_weight_max": 33.75637435913086,
      "activations/layer2_attention_weight_min": -32.67127990722656,
      "activations/layer3_attention_weight_max": 92.77794647216797,
      "activations/layer3_attention_weight_min": -96.77835083007812,
      "activations/layer4_attention_weight_max": 133.92620849609375,
      "activations/layer4_attention_weight_min": -132.51304626464844,
      "activations/layer5_attention_weight_max": 79.72618865966797,
      "activations/layer5_attention_weight_min": -81.86985778808594,
      "activations/layer6_attention_weight_max": 56.99028396606445,
      "activations/layer6_attention_weight_min": -54.68197250366211,
      "activations/layer7_attention_weight_max": 92.2159423828125,
      "activations/layer7_attention_weight_min": -83.24004364013672,
      "activations/layer8_attention_weight_max": 41.0833625793457,
      "activations/layer8_attention_weight_min": -39.48280715942383,
      "activations/layer9_attention_weight_max": 36.380836486816406,
      "activations/layer9_attention_weight_min": -41.494239807128906,
      "epoch": 5.03,
      "learning_rate": 0.00011878181818181817,
      "loss": 2.8311,
      "step": 86500
    },
    {
      "activations/layer0_attention_weight_max": 17.191368103027344,
      "activations/layer0_attention_weight_min": -13.251984596252441,
      "activations/layer10_attention_weight_max": 33.14739990234375,
      "activations/layer10_attention_weight_min": -29.716548919677734,
      "activations/layer11_attention_weight_max": 33.5574951171875,
      "activations/layer11_attention_weight_min": -28.026845932006836,
      "activations/layer12_attention_weight_max": 23.73887062072754,
      "activations/layer12_attention_weight_min": -32.282310485839844,
      "activations/layer13_attention_weight_max": 42.7414436340332,
      "activations/layer13_attention_weight_min": -30.555557250976562,
      "activations/layer14_attention_weight_max": 54.10429382324219,
      "activations/layer14_attention_weight_min": -42.437171936035156,
      "activations/layer15_attention_weight_max": 36.73418045043945,
      "activations/layer15_attention_weight_min": -27.475915908813477,
      "activations/layer16_attention_weight_max": 34.55128479003906,
      "activations/layer16_attention_weight_min": -29.118816375732422,
      "activations/layer17_attention_weight_max": 39.7455940246582,
      "activations/layer17_attention_weight_min": -23.072757720947266,
      "activations/layer18_attention_weight_max": 30.832462310791016,
      "activations/layer18_attention_weight_min": -22.661073684692383,
      "activations/layer19_attention_weight_max": 38.385475158691406,
      "activations/layer19_attention_weight_min": -28.740131378173828,
      "activations/layer1_attention_weight_max": 18.95467758178711,
      "activations/layer1_attention_weight_min": -15.537910461425781,
      "activations/layer20_attention_weight_max": 30.743675231933594,
      "activations/layer20_attention_weight_min": -24.501846313476562,
      "activations/layer21_attention_weight_max": 27.455162048339844,
      "activations/layer21_attention_weight_min": -19.538665771484375,
      "activations/layer22_attention_weight_max": 47.78656768798828,
      "activations/layer22_attention_weight_min": -30.475446701049805,
      "activations/layer23_attention_weight_max": 38.36992263793945,
      "activations/layer23_attention_weight_min": -21.49382781982422,
      "activations/layer2_attention_weight_max": 32.86529541015625,
      "activations/layer2_attention_weight_min": -31.84990692138672,
      "activations/layer3_attention_weight_max": 91.9952163696289,
      "activations/layer3_attention_weight_min": -91.82218933105469,
      "activations/layer4_attention_weight_max": 130.9722900390625,
      "activations/layer4_attention_weight_min": -135.19171142578125,
      "activations/layer5_attention_weight_max": 81.53620910644531,
      "activations/layer5_attention_weight_min": -83.71792602539062,
      "activations/layer6_attention_weight_max": 55.49360656738281,
      "activations/layer6_attention_weight_min": -60.41344451904297,
      "activations/layer7_attention_weight_max": 77.29117584228516,
      "activations/layer7_attention_weight_min": -84.41869354248047,
      "activations/layer8_attention_weight_max": 38.007625579833984,
      "activations/layer8_attention_weight_min": -40.75397491455078,
      "activations/layer9_attention_weight_max": 34.31229019165039,
      "activations/layer9_attention_weight_min": -37.5443000793457,
      "epoch": 5.03,
      "learning_rate": 0.00011876287878787878,
      "loss": 2.8411,
      "step": 86550
    },
    {
      "activations/layer0_attention_weight_max": 16.550567626953125,
      "activations/layer0_attention_weight_min": -11.758402824401855,
      "activations/layer10_attention_weight_max": 40.40894317626953,
      "activations/layer10_attention_weight_min": -31.50929832458496,
      "activations/layer11_attention_weight_max": 37.058197021484375,
      "activations/layer11_attention_weight_min": -29.89737319946289,
      "activations/layer12_attention_weight_max": 25.87236785888672,
      "activations/layer12_attention_weight_min": -29.283588409423828,
      "activations/layer13_attention_weight_max": 41.58399200439453,
      "activations/layer13_attention_weight_min": -32.58545684814453,
      "activations/layer14_attention_weight_max": 61.372310638427734,
      "activations/layer14_attention_weight_min": -44.69371032714844,
      "activations/layer15_attention_weight_max": 42.553916931152344,
      "activations/layer15_attention_weight_min": -29.91105079650879,
      "activations/layer16_attention_weight_max": 40.01182556152344,
      "activations/layer16_attention_weight_min": -29.945098876953125,
      "activations/layer17_attention_weight_max": 40.632686614990234,
      "activations/layer17_attention_weight_min": -27.504304885864258,
      "activations/layer18_attention_weight_max": 38.311767578125,
      "activations/layer18_attention_weight_min": -20.174251556396484,
      "activations/layer19_attention_weight_max": 40.330345153808594,
      "activations/layer19_attention_weight_min": -31.4672908782959,
      "activations/layer1_attention_weight_max": 16.82347869873047,
      "activations/layer1_attention_weight_min": -13.951765060424805,
      "activations/layer20_attention_weight_max": 36.465576171875,
      "activations/layer20_attention_weight_min": -24.85818099975586,
      "activations/layer21_attention_weight_max": 44.55060958862305,
      "activations/layer21_attention_weight_min": -20.61713409423828,
      "activations/layer22_attention_weight_max": 56.42826461791992,
      "activations/layer22_attention_weight_min": -28.622774124145508,
      "activations/layer23_attention_weight_max": 35.803749084472656,
      "activations/layer23_attention_weight_min": -19.080978393554688,
      "activations/layer2_attention_weight_max": 33.228214263916016,
      "activations/layer2_attention_weight_min": -31.666685104370117,
      "activations/layer3_attention_weight_max": 92.87814331054688,
      "activations/layer3_attention_weight_min": -96.64749908447266,
      "activations/layer4_attention_weight_max": 131.91307067871094,
      "activations/layer4_attention_weight_min": -135.43447875976562,
      "activations/layer5_attention_weight_max": 83.58904266357422,
      "activations/layer5_attention_weight_min": -84.84058380126953,
      "activations/layer6_attention_weight_max": 57.87451171875,
      "activations/layer6_attention_weight_min": -57.65666580200195,
      "activations/layer7_attention_weight_max": 88.09436798095703,
      "activations/layer7_attention_weight_min": -86.24758911132812,
      "activations/layer8_attention_weight_max": 41.24259567260742,
      "activations/layer8_attention_weight_min": -40.92224884033203,
      "activations/layer9_attention_weight_max": 40.45560836791992,
      "activations/layer9_attention_weight_min": -41.88106918334961,
      "epoch": 5.03,
      "learning_rate": 0.0001187439393939394,
      "loss": 2.8438,
      "step": 86600
    },
    {
      "activations/layer0_attention_weight_max": 17.098480224609375,
      "activations/layer0_attention_weight_min": -12.130876541137695,
      "activations/layer10_attention_weight_max": 34.67500686645508,
      "activations/layer10_attention_weight_min": -29.932964324951172,
      "activations/layer11_attention_weight_max": 32.831092834472656,
      "activations/layer11_attention_weight_min": -26.554454803466797,
      "activations/layer12_attention_weight_max": 24.980255126953125,
      "activations/layer12_attention_weight_min": -27.416173934936523,
      "activations/layer13_attention_weight_max": 42.92876052856445,
      "activations/layer13_attention_weight_min": -30.680017471313477,
      "activations/layer14_attention_weight_max": 67.94989013671875,
      "activations/layer14_attention_weight_min": -45.84992218017578,
      "activations/layer15_attention_weight_max": 39.55862045288086,
      "activations/layer15_attention_weight_min": -27.663619995117188,
      "activations/layer16_attention_weight_max": 38.368019104003906,
      "activations/layer16_attention_weight_min": -25.911148071289062,
      "activations/layer17_attention_weight_max": 40.68349075317383,
      "activations/layer17_attention_weight_min": -23.828153610229492,
      "activations/layer18_attention_weight_max": 34.228118896484375,
      "activations/layer18_attention_weight_min": -19.5909481048584,
      "activations/layer19_attention_weight_max": 41.15412521362305,
      "activations/layer19_attention_weight_min": -28.7747859954834,
      "activations/layer1_attention_weight_max": 17.034414291381836,
      "activations/layer1_attention_weight_min": -16.034500122070312,
      "activations/layer20_attention_weight_max": 36.529014587402344,
      "activations/layer20_attention_weight_min": -23.42417335510254,
      "activations/layer21_attention_weight_max": 34.93042755126953,
      "activations/layer21_attention_weight_min": -21.007226943969727,
      "activations/layer22_attention_weight_max": 50.28879928588867,
      "activations/layer22_attention_weight_min": -29.53791618347168,
      "activations/layer23_attention_weight_max": 36.55121994018555,
      "activations/layer23_attention_weight_min": -19.411434173583984,
      "activations/layer2_attention_weight_max": 31.378482818603516,
      "activations/layer2_attention_weight_min": -29.823348999023438,
      "activations/layer3_attention_weight_max": 87.30958557128906,
      "activations/layer3_attention_weight_min": -94.08719635009766,
      "activations/layer4_attention_weight_max": 133.14088439941406,
      "activations/layer4_attention_weight_min": -123.7327651977539,
      "activations/layer5_attention_weight_max": 84.79914855957031,
      "activations/layer5_attention_weight_min": -77.93958282470703,
      "activations/layer6_attention_weight_max": 56.28892135620117,
      "activations/layer6_attention_weight_min": -53.59785461425781,
      "activations/layer7_attention_weight_max": 81.22933197021484,
      "activations/layer7_attention_weight_min": -77.37232971191406,
      "activations/layer8_attention_weight_max": 39.23419952392578,
      "activations/layer8_attention_weight_min": -37.9901008605957,
      "activations/layer9_attention_weight_max": 37.30088806152344,
      "activations/layer9_attention_weight_min": -33.20797348022461,
      "epoch": 5.03,
      "learning_rate": 0.00011872499999999999,
      "loss": 2.851,
      "step": 86650
    },
    {
      "activations/layer0_attention_weight_max": 16.5524959564209,
      "activations/layer0_attention_weight_min": -12.400506973266602,
      "activations/layer10_attention_weight_max": 33.73682403564453,
      "activations/layer10_attention_weight_min": -30.03701400756836,
      "activations/layer11_attention_weight_max": 35.697288513183594,
      "activations/layer11_attention_weight_min": -28.958641052246094,
      "activations/layer12_attention_weight_max": 24.366697311401367,
      "activations/layer12_attention_weight_min": -27.72344207763672,
      "activations/layer13_attention_weight_max": 41.595497131347656,
      "activations/layer13_attention_weight_min": -31.239486694335938,
      "activations/layer14_attention_weight_max": 52.12990951538086,
      "activations/layer14_attention_weight_min": -40.81705093383789,
      "activations/layer15_attention_weight_max": 38.922306060791016,
      "activations/layer15_attention_weight_min": -28.61043357849121,
      "activations/layer16_attention_weight_max": 32.98569107055664,
      "activations/layer16_attention_weight_min": -28.905702590942383,
      "activations/layer17_attention_weight_max": 36.50629806518555,
      "activations/layer17_attention_weight_min": -23.596128463745117,
      "activations/layer18_attention_weight_max": 29.952472686767578,
      "activations/layer18_attention_weight_min": -21.920223236083984,
      "activations/layer19_attention_weight_max": 35.390071868896484,
      "activations/layer19_attention_weight_min": -30.16777992248535,
      "activations/layer1_attention_weight_max": 17.677595138549805,
      "activations/layer1_attention_weight_min": -16.633005142211914,
      "activations/layer20_attention_weight_max": 30.585725784301758,
      "activations/layer20_attention_weight_min": -23.50948715209961,
      "activations/layer21_attention_weight_max": 28.74838638305664,
      "activations/layer21_attention_weight_min": -22.248640060424805,
      "activations/layer22_attention_weight_max": 49.81315231323242,
      "activations/layer22_attention_weight_min": -27.26283073425293,
      "activations/layer23_attention_weight_max": 32.298797607421875,
      "activations/layer23_attention_weight_min": -20.523845672607422,
      "activations/layer2_attention_weight_max": 32.53033447265625,
      "activations/layer2_attention_weight_min": -31.82452392578125,
      "activations/layer3_attention_weight_max": 89.2155990600586,
      "activations/layer3_attention_weight_min": -88.18859100341797,
      "activations/layer4_attention_weight_max": 128.5952911376953,
      "activations/layer4_attention_weight_min": -125.08373260498047,
      "activations/layer5_attention_weight_max": 81.28945922851562,
      "activations/layer5_attention_weight_min": -77.79278564453125,
      "activations/layer6_attention_weight_max": 56.14111328125,
      "activations/layer6_attention_weight_min": -57.20953369140625,
      "activations/layer7_attention_weight_max": 82.76994323730469,
      "activations/layer7_attention_weight_min": -79.47748565673828,
      "activations/layer8_attention_weight_max": 38.6882209777832,
      "activations/layer8_attention_weight_min": -38.22521209716797,
      "activations/layer9_attention_weight_max": 34.21194076538086,
      "activations/layer9_attention_weight_min": -35.276405334472656,
      "epoch": 5.04,
      "learning_rate": 0.0001187060606060606,
      "loss": 2.8437,
      "step": 86700
    },
    {
      "activations/layer0_attention_weight_max": 17.69062042236328,
      "activations/layer0_attention_weight_min": -11.003458976745605,
      "activations/layer10_attention_weight_max": 33.93513870239258,
      "activations/layer10_attention_weight_min": -29.020389556884766,
      "activations/layer11_attention_weight_max": 35.57652282714844,
      "activations/layer11_attention_weight_min": -27.60786247253418,
      "activations/layer12_attention_weight_max": 56.12777328491211,
      "activations/layer12_attention_weight_min": -30.250703811645508,
      "activations/layer13_attention_weight_max": 47.63648986816406,
      "activations/layer13_attention_weight_min": -32.67134475708008,
      "activations/layer14_attention_weight_max": 62.39065170288086,
      "activations/layer14_attention_weight_min": -46.341758728027344,
      "activations/layer15_attention_weight_max": 42.42117691040039,
      "activations/layer15_attention_weight_min": -29.020177841186523,
      "activations/layer16_attention_weight_max": 38.90457534790039,
      "activations/layer16_attention_weight_min": -29.007524490356445,
      "activations/layer17_attention_weight_max": 40.07970428466797,
      "activations/layer17_attention_weight_min": -23.884248733520508,
      "activations/layer18_attention_weight_max": 34.8453369140625,
      "activations/layer18_attention_weight_min": -20.834001541137695,
      "activations/layer19_attention_weight_max": 42.48942184448242,
      "activations/layer19_attention_weight_min": -29.052045822143555,
      "activations/layer1_attention_weight_max": 18.93019676208496,
      "activations/layer1_attention_weight_min": -14.610753059387207,
      "activations/layer20_attention_weight_max": 32.011043548583984,
      "activations/layer20_attention_weight_min": -25.457805633544922,
      "activations/layer21_attention_weight_max": 31.916942596435547,
      "activations/layer21_attention_weight_min": -22.481603622436523,
      "activations/layer22_attention_weight_max": 54.28805923461914,
      "activations/layer22_attention_weight_min": -31.722904205322266,
      "activations/layer23_attention_weight_max": 37.38042449951172,
      "activations/layer23_attention_weight_min": -21.039794921875,
      "activations/layer2_attention_weight_max": 32.09403610229492,
      "activations/layer2_attention_weight_min": -31.962772369384766,
      "activations/layer3_attention_weight_max": 89.28279876708984,
      "activations/layer3_attention_weight_min": -88.25708770751953,
      "activations/layer4_attention_weight_max": 131.43138122558594,
      "activations/layer4_attention_weight_min": -122.83141326904297,
      "activations/layer5_attention_weight_max": 87.30056762695312,
      "activations/layer5_attention_weight_min": -77.66766357421875,
      "activations/layer6_attention_weight_max": 62.43080520629883,
      "activations/layer6_attention_weight_min": -54.85696792602539,
      "activations/layer7_attention_weight_max": 80.18636322021484,
      "activations/layer7_attention_weight_min": -79.75098419189453,
      "activations/layer8_attention_weight_max": 43.89327621459961,
      "activations/layer8_attention_weight_min": -36.24541473388672,
      "activations/layer9_attention_weight_max": 37.18181228637695,
      "activations/layer9_attention_weight_min": -31.497589111328125,
      "epoch": 5.04,
      "learning_rate": 0.00011868712121212119,
      "loss": 2.8454,
      "step": 86750
    },
    {
      "activations/layer0_attention_weight_max": 17.161346435546875,
      "activations/layer0_attention_weight_min": -12.25922679901123,
      "activations/layer10_attention_weight_max": 41.15923309326172,
      "activations/layer10_attention_weight_min": -34.99860382080078,
      "activations/layer11_attention_weight_max": 39.17203140258789,
      "activations/layer11_attention_weight_min": -31.072799682617188,
      "activations/layer12_attention_weight_max": 32.43936538696289,
      "activations/layer12_attention_weight_min": -27.655065536499023,
      "activations/layer13_attention_weight_max": 65.0503158569336,
      "activations/layer13_attention_weight_min": -46.18733596801758,
      "activations/layer14_attention_weight_max": 70.5037612915039,
      "activations/layer14_attention_weight_min": -52.680233001708984,
      "activations/layer15_attention_weight_max": 45.713558197021484,
      "activations/layer15_attention_weight_min": -32.13434982299805,
      "activations/layer16_attention_weight_max": 39.277626037597656,
      "activations/layer16_attention_weight_min": -30.52271270751953,
      "activations/layer17_attention_weight_max": 38.61342239379883,
      "activations/layer17_attention_weight_min": -25.083404541015625,
      "activations/layer18_attention_weight_max": 34.676048278808594,
      "activations/layer18_attention_weight_min": -21.956510543823242,
      "activations/layer19_attention_weight_max": 47.308204650878906,
      "activations/layer19_attention_weight_min": -29.637014389038086,
      "activations/layer1_attention_weight_max": 17.04828453063965,
      "activations/layer1_attention_weight_min": -14.653397560119629,
      "activations/layer20_attention_weight_max": 33.64514923095703,
      "activations/layer20_attention_weight_min": -27.267677307128906,
      "activations/layer21_attention_weight_max": 38.85096740722656,
      "activations/layer21_attention_weight_min": -22.732267379760742,
      "activations/layer22_attention_weight_max": 50.7104377746582,
      "activations/layer22_attention_weight_min": -29.27614974975586,
      "activations/layer23_attention_weight_max": 34.7120361328125,
      "activations/layer23_attention_weight_min": -21.544231414794922,
      "activations/layer2_attention_weight_max": 34.009151458740234,
      "activations/layer2_attention_weight_min": -30.54454803466797,
      "activations/layer3_attention_weight_max": 94.72154235839844,
      "activations/layer3_attention_weight_min": -92.44180297851562,
      "activations/layer4_attention_weight_max": 140.66943359375,
      "activations/layer4_attention_weight_min": -138.97325134277344,
      "activations/layer5_attention_weight_max": 89.73921203613281,
      "activations/layer5_attention_weight_min": -83.88739013671875,
      "activations/layer6_attention_weight_max": 62.06452941894531,
      "activations/layer6_attention_weight_min": -57.850379943847656,
      "activations/layer7_attention_weight_max": 99.5097885131836,
      "activations/layer7_attention_weight_min": -91.0289306640625,
      "activations/layer8_attention_weight_max": 49.30263900756836,
      "activations/layer8_attention_weight_min": -44.40085983276367,
      "activations/layer9_attention_weight_max": 46.15537643432617,
      "activations/layer9_attention_weight_min": -38.647422790527344,
      "epoch": 5.04,
      "learning_rate": 0.0001186681818181818,
      "loss": 2.8457,
      "step": 86800
    },
    {
      "activations/layer0_attention_weight_max": 16.580188751220703,
      "activations/layer0_attention_weight_min": -12.65990161895752,
      "activations/layer10_attention_weight_max": 33.76383590698242,
      "activations/layer10_attention_weight_min": -30.560501098632812,
      "activations/layer11_attention_weight_max": 34.01713943481445,
      "activations/layer11_attention_weight_min": -27.829364776611328,
      "activations/layer12_attention_weight_max": 66.78645324707031,
      "activations/layer12_attention_weight_min": -28.22501564025879,
      "activations/layer13_attention_weight_max": 51.68068313598633,
      "activations/layer13_attention_weight_min": -31.789445877075195,
      "activations/layer14_attention_weight_max": 76.92027282714844,
      "activations/layer14_attention_weight_min": -49.26736831665039,
      "activations/layer15_attention_weight_max": 43.475990295410156,
      "activations/layer15_attention_weight_min": -27.67779541015625,
      "activations/layer16_attention_weight_max": 43.95888137817383,
      "activations/layer16_attention_weight_min": -27.760648727416992,
      "activations/layer17_attention_weight_max": 44.32829284667969,
      "activations/layer17_attention_weight_min": -24.84834861755371,
      "activations/layer18_attention_weight_max": 37.290061950683594,
      "activations/layer18_attention_weight_min": -20.431543350219727,
      "activations/layer19_attention_weight_max": 49.13513946533203,
      "activations/layer19_attention_weight_min": -29.49000358581543,
      "activations/layer1_attention_weight_max": 17.545055389404297,
      "activations/layer1_attention_weight_min": -13.834939002990723,
      "activations/layer20_attention_weight_max": 39.47423553466797,
      "activations/layer20_attention_weight_min": -24.38341522216797,
      "activations/layer21_attention_weight_max": 37.29128646850586,
      "activations/layer21_attention_weight_min": -21.1419620513916,
      "activations/layer22_attention_weight_max": 55.64873123168945,
      "activations/layer22_attention_weight_min": -29.067949295043945,
      "activations/layer23_attention_weight_max": 40.749053955078125,
      "activations/layer23_attention_weight_min": -19.232135772705078,
      "activations/layer2_attention_weight_max": 34.40046691894531,
      "activations/layer2_attention_weight_min": -34.26002502441406,
      "activations/layer3_attention_weight_max": 96.02523803710938,
      "activations/layer3_attention_weight_min": -94.3020248413086,
      "activations/layer4_attention_weight_max": 130.94004821777344,
      "activations/layer4_attention_weight_min": -130.0403289794922,
      "activations/layer5_attention_weight_max": 80.62103271484375,
      "activations/layer5_attention_weight_min": -78.54147338867188,
      "activations/layer6_attention_weight_max": 54.32085037231445,
      "activations/layer6_attention_weight_min": -58.408714294433594,
      "activations/layer7_attention_weight_max": 84.55025482177734,
      "activations/layer7_attention_weight_min": -82.07197570800781,
      "activations/layer8_attention_weight_max": 39.2083854675293,
      "activations/layer8_attention_weight_min": -37.83191680908203,
      "activations/layer9_attention_weight_max": 36.8820686340332,
      "activations/layer9_attention_weight_min": -33.63017654418945,
      "epoch": 5.05,
      "learning_rate": 0.00011864924242424242,
      "loss": 2.834,
      "step": 86850
    },
    {
      "activations/layer0_attention_weight_max": 17.188068389892578,
      "activations/layer0_attention_weight_min": -13.03799819946289,
      "activations/layer10_attention_weight_max": 35.0455322265625,
      "activations/layer10_attention_weight_min": -29.030790328979492,
      "activations/layer11_attention_weight_max": 34.65106201171875,
      "activations/layer11_attention_weight_min": -26.677513122558594,
      "activations/layer12_attention_weight_max": 40.769447326660156,
      "activations/layer12_attention_weight_min": -28.731367111206055,
      "activations/layer13_attention_weight_max": 54.23206329345703,
      "activations/layer13_attention_weight_min": -33.50295639038086,
      "activations/layer14_attention_weight_max": 64.04690551757812,
      "activations/layer14_attention_weight_min": -45.0587043762207,
      "activations/layer15_attention_weight_max": 53.94383239746094,
      "activations/layer15_attention_weight_min": -33.85267639160156,
      "activations/layer16_attention_weight_max": 43.623939514160156,
      "activations/layer16_attention_weight_min": -28.039047241210938,
      "activations/layer17_attention_weight_max": 39.559261322021484,
      "activations/layer17_attention_weight_min": -25.026580810546875,
      "activations/layer18_attention_weight_max": 34.91441345214844,
      "activations/layer18_attention_weight_min": -20.212858200073242,
      "activations/layer19_attention_weight_max": 46.06901168823242,
      "activations/layer19_attention_weight_min": -29.970901489257812,
      "activations/layer1_attention_weight_max": 16.871509552001953,
      "activations/layer1_attention_weight_min": -13.99502182006836,
      "activations/layer20_attention_weight_max": 34.14706802368164,
      "activations/layer20_attention_weight_min": -22.849822998046875,
      "activations/layer21_attention_weight_max": 37.697296142578125,
      "activations/layer21_attention_weight_min": -21.288650512695312,
      "activations/layer22_attention_weight_max": 48.36845779418945,
      "activations/layer22_attention_weight_min": -30.77109146118164,
      "activations/layer23_attention_weight_max": 31.77846336364746,
      "activations/layer23_attention_weight_min": -19.66021728515625,
      "activations/layer2_attention_weight_max": 32.884490966796875,
      "activations/layer2_attention_weight_min": -30.779129028320312,
      "activations/layer3_attention_weight_max": 89.85599517822266,
      "activations/layer3_attention_weight_min": -90.54447937011719,
      "activations/layer4_attention_weight_max": 129.26080322265625,
      "activations/layer4_attention_weight_min": -126.74571990966797,
      "activations/layer5_attention_weight_max": 81.93152618408203,
      "activations/layer5_attention_weight_min": -79.4217758178711,
      "activations/layer6_attention_weight_max": 57.02238082885742,
      "activations/layer6_attention_weight_min": -56.169593811035156,
      "activations/layer7_attention_weight_max": 88.69803619384766,
      "activations/layer7_attention_weight_min": -86.04195404052734,
      "activations/layer8_attention_weight_max": 40.27774429321289,
      "activations/layer8_attention_weight_min": -40.861385345458984,
      "activations/layer9_attention_weight_max": 37.26652526855469,
      "activations/layer9_attention_weight_min": -32.86334228515625,
      "epoch": 5.05,
      "learning_rate": 0.00011863030303030302,
      "loss": 2.8449,
      "step": 86900
    },
    {
      "activations/layer0_attention_weight_max": 17.50200080871582,
      "activations/layer0_attention_weight_min": -12.347477912902832,
      "activations/layer10_attention_weight_max": 33.34467315673828,
      "activations/layer10_attention_weight_min": -33.32515335083008,
      "activations/layer11_attention_weight_max": 32.2618408203125,
      "activations/layer11_attention_weight_min": -28.140783309936523,
      "activations/layer12_attention_weight_max": 29.114444732666016,
      "activations/layer12_attention_weight_min": -26.121204376220703,
      "activations/layer13_attention_weight_max": 49.006771087646484,
      "activations/layer13_attention_weight_min": -32.896812438964844,
      "activations/layer14_attention_weight_max": 65.67300415039062,
      "activations/layer14_attention_weight_min": -43.375579833984375,
      "activations/layer15_attention_weight_max": 40.89696502685547,
      "activations/layer15_attention_weight_min": -28.08372688293457,
      "activations/layer16_attention_weight_max": 37.70633316040039,
      "activations/layer16_attention_weight_min": -27.40713119506836,
      "activations/layer17_attention_weight_max": 40.76591491699219,
      "activations/layer17_attention_weight_min": -24.166513442993164,
      "activations/layer18_attention_weight_max": 34.407501220703125,
      "activations/layer18_attention_weight_min": -20.948745727539062,
      "activations/layer19_attention_weight_max": 41.61330795288086,
      "activations/layer19_attention_weight_min": -29.871992111206055,
      "activations/layer1_attention_weight_max": 17.335811614990234,
      "activations/layer1_attention_weight_min": -13.956352233886719,
      "activations/layer20_attention_weight_max": 32.39030075073242,
      "activations/layer20_attention_weight_min": -24.68937873840332,
      "activations/layer21_attention_weight_max": 31.909454345703125,
      "activations/layer21_attention_weight_min": -21.58570098876953,
      "activations/layer22_attention_weight_max": 52.25394058227539,
      "activations/layer22_attention_weight_min": -28.345781326293945,
      "activations/layer23_attention_weight_max": 36.88740539550781,
      "activations/layer23_attention_weight_min": -20.992523193359375,
      "activations/layer2_attention_weight_max": 32.45359802246094,
      "activations/layer2_attention_weight_min": -31.953990936279297,
      "activations/layer3_attention_weight_max": 90.96865844726562,
      "activations/layer3_attention_weight_min": -89.75900268554688,
      "activations/layer4_attention_weight_max": 132.20347595214844,
      "activations/layer4_attention_weight_min": -133.75865173339844,
      "activations/layer5_attention_weight_max": 82.9969482421875,
      "activations/layer5_attention_weight_min": -85.19886016845703,
      "activations/layer6_attention_weight_max": 56.612762451171875,
      "activations/layer6_attention_weight_min": -59.12086868286133,
      "activations/layer7_attention_weight_max": 87.54550170898438,
      "activations/layer7_attention_weight_min": -82.83929443359375,
      "activations/layer8_attention_weight_max": 40.06760025024414,
      "activations/layer8_attention_weight_min": -40.03401184082031,
      "activations/layer9_attention_weight_max": 36.0279655456543,
      "activations/layer9_attention_weight_min": -34.31551742553711,
      "epoch": 5.05,
      "learning_rate": 0.00011861136363636362,
      "loss": 2.8564,
      "step": 86950
    },
    {
      "activations/layer0_attention_weight_max": 16.470596313476562,
      "activations/layer0_attention_weight_min": -13.282752990722656,
      "activations/layer10_attention_weight_max": 32.4633674621582,
      "activations/layer10_attention_weight_min": -29.428131103515625,
      "activations/layer11_attention_weight_max": 32.07039260864258,
      "activations/layer11_attention_weight_min": -27.05885887145996,
      "activations/layer12_attention_weight_max": 25.224096298217773,
      "activations/layer12_attention_weight_min": -28.160297393798828,
      "activations/layer13_attention_weight_max": 41.168426513671875,
      "activations/layer13_attention_weight_min": -31.370349884033203,
      "activations/layer14_attention_weight_max": 59.29031753540039,
      "activations/layer14_attention_weight_min": -46.3276481628418,
      "activations/layer15_attention_weight_max": 42.404972076416016,
      "activations/layer15_attention_weight_min": -30.501598358154297,
      "activations/layer16_attention_weight_max": 34.25208282470703,
      "activations/layer16_attention_weight_min": -29.71755027770996,
      "activations/layer17_attention_weight_max": 37.96909713745117,
      "activations/layer17_attention_weight_min": -24.94419288635254,
      "activations/layer18_attention_weight_max": 33.65415573120117,
      "activations/layer18_attention_weight_min": -20.89083480834961,
      "activations/layer19_attention_weight_max": 34.840389251708984,
      "activations/layer19_attention_weight_min": -30.00555419921875,
      "activations/layer1_attention_weight_max": 17.397626876831055,
      "activations/layer1_attention_weight_min": -14.102866172790527,
      "activations/layer20_attention_weight_max": 31.088518142700195,
      "activations/layer20_attention_weight_min": -23.167682647705078,
      "activations/layer21_attention_weight_max": 27.70433807373047,
      "activations/layer21_attention_weight_min": -22.296253204345703,
      "activations/layer22_attention_weight_max": 50.56626510620117,
      "activations/layer22_attention_weight_min": -27.38506317138672,
      "activations/layer23_attention_weight_max": 30.94289207458496,
      "activations/layer23_attention_weight_min": -21.432104110717773,
      "activations/layer2_attention_weight_max": 32.29602813720703,
      "activations/layer2_attention_weight_min": -31.91934585571289,
      "activations/layer3_attention_weight_max": 89.5387191772461,
      "activations/layer3_attention_weight_min": -88.51626586914062,
      "activations/layer4_attention_weight_max": 126.6374282836914,
      "activations/layer4_attention_weight_min": -124.12187957763672,
      "activations/layer5_attention_weight_max": 78.12709045410156,
      "activations/layer5_attention_weight_min": -74.95298767089844,
      "activations/layer6_attention_weight_max": 58.248779296875,
      "activations/layer6_attention_weight_min": -56.19648742675781,
      "activations/layer7_attention_weight_max": 84.67560577392578,
      "activations/layer7_attention_weight_min": -83.7165298461914,
      "activations/layer8_attention_weight_max": 38.19844436645508,
      "activations/layer8_attention_weight_min": -37.31730270385742,
      "activations/layer9_attention_weight_max": 35.76247787475586,
      "activations/layer9_attention_weight_min": -34.69625473022461,
      "epoch": 5.06,
      "learning_rate": 0.00011859242424242424,
      "loss": 2.8461,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_loss": 2.802734375,
      "eval_runtime": 8.5268,
      "eval_samples_per_second": 503.586,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_openwebtext_loss": 2.802734375,
      "eval_openwebtext_ppl": 16.489674134905655,
      "eval_openwebtext_runtime": 8.5268,
      "eval_openwebtext_samples_per_second": 503.586,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_wikitext_loss": 3.03125,
      "eval_wikitext_ppl": 20.723120306578686,
      "eval_wikitext_runtime": 2.0361,
      "eval_wikitext_samples_per_second": 223.958,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_lambada_loss": 2.7578125,
      "eval_lambada_ppl": 15.765318566620541,
      "eval_lambada_runtime": 9.5758,
      "eval_lambada_samples_per_second": 508.47,
      "step": 87000
    },
    {
      "activations/layer0_attention_weight_max": 16.69890022277832,
      "activations/layer0_attention_weight_min": -11.438342094421387,
      "activations/layer10_attention_weight_max": 34.01533889770508,
      "activations/layer10_attention_weight_min": -30.12527847290039,
      "activations/layer11_attention_weight_max": 29.967702865600586,
      "activations/layer11_attention_weight_min": -26.788021087646484,
      "activations/layer12_attention_weight_max": 32.18080520629883,
      "activations/layer12_attention_weight_min": -28.35907554626465,
      "activations/layer13_attention_weight_max": 43.037010192871094,
      "activations/layer13_attention_weight_min": -33.29478073120117,
      "activations/layer14_attention_weight_max": 57.152313232421875,
      "activations/layer14_attention_weight_min": -47.26002883911133,
      "activations/layer15_attention_weight_max": 38.9802131652832,
      "activations/layer15_attention_weight_min": -28.07762336730957,
      "activations/layer16_attention_weight_max": 35.10830307006836,
      "activations/layer16_attention_weight_min": -27.984771728515625,
      "activations/layer17_attention_weight_max": 35.60049819946289,
      "activations/layer17_attention_weight_min": -25.996807098388672,
      "activations/layer18_attention_weight_max": 31.694801330566406,
      "activations/layer18_attention_weight_min": -19.85914421081543,
      "activations/layer19_attention_weight_max": 36.635101318359375,
      "activations/layer19_attention_weight_min": -29.347509384155273,
      "activations/layer1_attention_weight_max": 16.46743392944336,
      "activations/layer1_attention_weight_min": -14.936694145202637,
      "activations/layer20_attention_weight_max": 32.56121063232422,
      "activations/layer20_attention_weight_min": -22.608152389526367,
      "activations/layer21_attention_weight_max": 34.63118362426758,
      "activations/layer21_attention_weight_min": -20.78730010986328,
      "activations/layer22_attention_weight_max": 46.07744216918945,
      "activations/layer22_attention_weight_min": -29.507001876831055,
      "activations/layer23_attention_weight_max": 30.255521774291992,
      "activations/layer23_attention_weight_min": -19.750852584838867,
      "activations/layer2_attention_weight_max": 31.767988204956055,
      "activations/layer2_attention_weight_min": -31.405675888061523,
      "activations/layer3_attention_weight_max": 87.8062515258789,
      "activations/layer3_attention_weight_min": -88.79656219482422,
      "activations/layer4_attention_weight_max": 130.1634979248047,
      "activations/layer4_attention_weight_min": -123.68280029296875,
      "activations/layer5_attention_weight_max": 80.68516540527344,
      "activations/layer5_attention_weight_min": -78.53062438964844,
      "activations/layer6_attention_weight_max": 56.811988830566406,
      "activations/layer6_attention_weight_min": -56.33753967285156,
      "activations/layer7_attention_weight_max": 86.30780792236328,
      "activations/layer7_attention_weight_min": -82.31725311279297,
      "activations/layer8_attention_weight_max": 39.77738952636719,
      "activations/layer8_attention_weight_min": -40.8734245300293,
      "activations/layer9_attention_weight_max": 34.63669204711914,
      "activations/layer9_attention_weight_min": -32.98814392089844,
      "epoch": 5.06,
      "learning_rate": 0.00011857348484848484,
      "loss": 2.8118,
      "step": 87050
    },
    {
      "activations/layer0_attention_weight_max": 14.72569751739502,
      "activations/layer0_attention_weight_min": -11.165441513061523,
      "activations/layer10_attention_weight_max": 32.43914031982422,
      "activations/layer10_attention_weight_min": -30.453611373901367,
      "activations/layer11_attention_weight_max": 34.28974151611328,
      "activations/layer11_attention_weight_min": -27.694990158081055,
      "activations/layer12_attention_weight_max": 44.03681182861328,
      "activations/layer12_attention_weight_min": -27.347166061401367,
      "activations/layer13_attention_weight_max": 47.00284194946289,
      "activations/layer13_attention_weight_min": -33.26734161376953,
      "activations/layer14_attention_weight_max": 68.41556549072266,
      "activations/layer14_attention_weight_min": -49.875484466552734,
      "activations/layer15_attention_weight_max": 38.507713317871094,
      "activations/layer15_attention_weight_min": -29.695573806762695,
      "activations/layer16_attention_weight_max": 38.54615020751953,
      "activations/layer16_attention_weight_min": -27.93376350402832,
      "activations/layer17_attention_weight_max": 38.61678695678711,
      "activations/layer17_attention_weight_min": -24.161970138549805,
      "activations/layer18_attention_weight_max": 30.933889389038086,
      "activations/layer18_attention_weight_min": -20.831066131591797,
      "activations/layer19_attention_weight_max": 39.24325942993164,
      "activations/layer19_attention_weight_min": -29.2966365814209,
      "activations/layer1_attention_weight_max": 16.963565826416016,
      "activations/layer1_attention_weight_min": -15.242806434631348,
      "activations/layer20_attention_weight_max": 32.31428909301758,
      "activations/layer20_attention_weight_min": -24.374446868896484,
      "activations/layer21_attention_weight_max": 34.560909271240234,
      "activations/layer21_attention_weight_min": -19.622211456298828,
      "activations/layer22_attention_weight_max": 51.379920959472656,
      "activations/layer22_attention_weight_min": -26.455739974975586,
      "activations/layer23_attention_weight_max": 33.97084426879883,
      "activations/layer23_attention_weight_min": -19.710020065307617,
      "activations/layer2_attention_weight_max": 30.98159408569336,
      "activations/layer2_attention_weight_min": -29.889320373535156,
      "activations/layer3_attention_weight_max": 86.53814697265625,
      "activations/layer3_attention_weight_min": -88.77718353271484,
      "activations/layer4_attention_weight_max": 132.45909118652344,
      "activations/layer4_attention_weight_min": -127.03081512451172,
      "activations/layer5_attention_weight_max": 80.5072250366211,
      "activations/layer5_attention_weight_min": -78.21100616455078,
      "activations/layer6_attention_weight_max": 57.63533020019531,
      "activations/layer6_attention_weight_min": -56.40106201171875,
      "activations/layer7_attention_weight_max": 81.507568359375,
      "activations/layer7_attention_weight_min": -83.40990447998047,
      "activations/layer8_attention_weight_max": 37.537235260009766,
      "activations/layer8_attention_weight_min": -37.53977966308594,
      "activations/layer9_attention_weight_max": 36.3709831237793,
      "activations/layer9_attention_weight_min": -33.969322204589844,
      "epoch": 5.06,
      "learning_rate": 0.00011855454545454545,
      "loss": 2.8485,
      "step": 87100
    },
    {
      "activations/layer0_attention_weight_max": 17.433795928955078,
      "activations/layer0_attention_weight_min": -12.706911087036133,
      "activations/layer10_attention_weight_max": 32.16284942626953,
      "activations/layer10_attention_weight_min": -29.774089813232422,
      "activations/layer11_attention_weight_max": 33.28961181640625,
      "activations/layer11_attention_weight_min": -27.42855453491211,
      "activations/layer12_attention_weight_max": 26.053918838500977,
      "activations/layer12_attention_weight_min": -26.39331817626953,
      "activations/layer13_attention_weight_max": 41.06212615966797,
      "activations/layer13_attention_weight_min": -32.78532409667969,
      "activations/layer14_attention_weight_max": 57.130760192871094,
      "activations/layer14_attention_weight_min": -45.17121505737305,
      "activations/layer15_attention_weight_max": 42.57746887207031,
      "activations/layer15_attention_weight_min": -34.30469512939453,
      "activations/layer16_attention_weight_max": 38.18400955200195,
      "activations/layer16_attention_weight_min": -33.096866607666016,
      "activations/layer17_attention_weight_max": 37.63331985473633,
      "activations/layer17_attention_weight_min": -23.87482452392578,
      "activations/layer18_attention_weight_max": 31.83864974975586,
      "activations/layer18_attention_weight_min": -20.051368713378906,
      "activations/layer19_attention_weight_max": 44.5162467956543,
      "activations/layer19_attention_weight_min": -35.34133529663086,
      "activations/layer1_attention_weight_max": 17.634401321411133,
      "activations/layer1_attention_weight_min": -17.169803619384766,
      "activations/layer20_attention_weight_max": 34.8258171081543,
      "activations/layer20_attention_weight_min": -23.773256301879883,
      "activations/layer21_attention_weight_max": 30.873538970947266,
      "activations/layer21_attention_weight_min": -21.340028762817383,
      "activations/layer22_attention_weight_max": 51.20081329345703,
      "activations/layer22_attention_weight_min": -27.2913761138916,
      "activations/layer23_attention_weight_max": 34.3399543762207,
      "activations/layer23_attention_weight_min": -18.64348030090332,
      "activations/layer2_attention_weight_max": 31.09316635131836,
      "activations/layer2_attention_weight_min": -28.606285095214844,
      "activations/layer3_attention_weight_max": 88.67747497558594,
      "activations/layer3_attention_weight_min": -86.14708709716797,
      "activations/layer4_attention_weight_max": 128.57887268066406,
      "activations/layer4_attention_weight_min": -124.12747955322266,
      "activations/layer5_attention_weight_max": 80.35320281982422,
      "activations/layer5_attention_weight_min": -75.42608642578125,
      "activations/layer6_attention_weight_max": 55.084136962890625,
      "activations/layer6_attention_weight_min": -55.18458557128906,
      "activations/layer7_attention_weight_max": 84.30521392822266,
      "activations/layer7_attention_weight_min": -84.44953155517578,
      "activations/layer8_attention_weight_max": 37.87670135498047,
      "activations/layer8_attention_weight_min": -38.140655517578125,
      "activations/layer9_attention_weight_max": 35.220855712890625,
      "activations/layer9_attention_weight_min": -35.88703536987305,
      "epoch": 5.06,
      "learning_rate": 0.00011853560606060604,
      "loss": 2.856,
      "step": 87150
    },
    {
      "activations/layer0_attention_weight_max": 16.95240020751953,
      "activations/layer0_attention_weight_min": -11.164447784423828,
      "activations/layer10_attention_weight_max": 38.89410400390625,
      "activations/layer10_attention_weight_min": -32.944122314453125,
      "activations/layer11_attention_weight_max": 36.66205596923828,
      "activations/layer11_attention_weight_min": -31.77715301513672,
      "activations/layer12_attention_weight_max": 37.85209655761719,
      "activations/layer12_attention_weight_min": -25.80817985534668,
      "activations/layer13_attention_weight_max": 56.00864028930664,
      "activations/layer13_attention_weight_min": -34.98728942871094,
      "activations/layer14_attention_weight_max": 88.71440887451172,
      "activations/layer14_attention_weight_min": -57.4351692199707,
      "activations/layer15_attention_weight_max": 44.73639678955078,
      "activations/layer15_attention_weight_min": -30.35276222229004,
      "activations/layer16_attention_weight_max": 39.86296081542969,
      "activations/layer16_attention_weight_min": -31.999589920043945,
      "activations/layer17_attention_weight_max": 44.468101501464844,
      "activations/layer17_attention_weight_min": -26.829713821411133,
      "activations/layer18_attention_weight_max": 36.91493606567383,
      "activations/layer18_attention_weight_min": -22.55077362060547,
      "activations/layer19_attention_weight_max": 50.82062911987305,
      "activations/layer19_attention_weight_min": -31.228864669799805,
      "activations/layer1_attention_weight_max": 18.365867614746094,
      "activations/layer1_attention_weight_min": -14.716110229492188,
      "activations/layer20_attention_weight_max": 40.90182876586914,
      "activations/layer20_attention_weight_min": -24.123958587646484,
      "activations/layer21_attention_weight_max": 38.36211013793945,
      "activations/layer21_attention_weight_min": -21.290067672729492,
      "activations/layer22_attention_weight_max": 73.03929901123047,
      "activations/layer22_attention_weight_min": -31.631725311279297,
      "activations/layer23_attention_weight_max": 44.0620231628418,
      "activations/layer23_attention_weight_min": -20.408130645751953,
      "activations/layer2_attention_weight_max": 32.640689849853516,
      "activations/layer2_attention_weight_min": -31.735044479370117,
      "activations/layer3_attention_weight_max": 93.84736633300781,
      "activations/layer3_attention_weight_min": -95.59639739990234,
      "activations/layer4_attention_weight_max": 137.702392578125,
      "activations/layer4_attention_weight_min": -132.34939575195312,
      "activations/layer5_attention_weight_max": 85.07127380371094,
      "activations/layer5_attention_weight_min": -82.85318756103516,
      "activations/layer6_attention_weight_max": 59.71834182739258,
      "activations/layer6_attention_weight_min": -58.56852340698242,
      "activations/layer7_attention_weight_max": 92.33576965332031,
      "activations/layer7_attention_weight_min": -86.62080383300781,
      "activations/layer8_attention_weight_max": 44.377620697021484,
      "activations/layer8_attention_weight_min": -40.44108200073242,
      "activations/layer9_attention_weight_max": 41.081573486328125,
      "activations/layer9_attention_weight_min": -35.877174377441406,
      "epoch": 5.07,
      "learning_rate": 0.00011851666666666666,
      "loss": 2.8404,
      "step": 87200
    },
    {
      "activations/layer0_attention_weight_max": 16.872039794921875,
      "activations/layer0_attention_weight_min": -11.560853958129883,
      "activations/layer10_attention_weight_max": 33.9327278137207,
      "activations/layer10_attention_weight_min": -31.079687118530273,
      "activations/layer11_attention_weight_max": 30.078102111816406,
      "activations/layer11_attention_weight_min": -26.917804718017578,
      "activations/layer12_attention_weight_max": 25.760740280151367,
      "activations/layer12_attention_weight_min": -25.40499496459961,
      "activations/layer13_attention_weight_max": 40.51245880126953,
      "activations/layer13_attention_weight_min": -32.18186569213867,
      "activations/layer14_attention_weight_max": 55.17870330810547,
      "activations/layer14_attention_weight_min": -40.938987731933594,
      "activations/layer15_attention_weight_max": 38.76820373535156,
      "activations/layer15_attention_weight_min": -29.429668426513672,
      "activations/layer16_attention_weight_max": 35.299896240234375,
      "activations/layer16_attention_weight_min": -27.230587005615234,
      "activations/layer17_attention_weight_max": 35.88808059692383,
      "activations/layer17_attention_weight_min": -25.06922721862793,
      "activations/layer18_attention_weight_max": 29.760398864746094,
      "activations/layer18_attention_weight_min": -20.2296199798584,
      "activations/layer19_attention_weight_max": 40.128692626953125,
      "activations/layer19_attention_weight_min": -29.92201042175293,
      "activations/layer1_attention_weight_max": 17.167865753173828,
      "activations/layer1_attention_weight_min": -17.583175659179688,
      "activations/layer20_attention_weight_max": 34.0619010925293,
      "activations/layer20_attention_weight_min": -24.295578002929688,
      "activations/layer21_attention_weight_max": 33.14671325683594,
      "activations/layer21_attention_weight_min": -21.887887954711914,
      "activations/layer22_attention_weight_max": 47.102970123291016,
      "activations/layer22_attention_weight_min": -29.38086700439453,
      "activations/layer23_attention_weight_max": 32.784095764160156,
      "activations/layer23_attention_weight_min": -21.085695266723633,
      "activations/layer2_attention_weight_max": 31.363990783691406,
      "activations/layer2_attention_weight_min": -32.09585952758789,
      "activations/layer3_attention_weight_max": 86.03241729736328,
      "activations/layer3_attention_weight_min": -89.6005630493164,
      "activations/layer4_attention_weight_max": 127.72981262207031,
      "activations/layer4_attention_weight_min": -127.0362777709961,
      "activations/layer5_attention_weight_max": 78.20655822753906,
      "activations/layer5_attention_weight_min": -79.39447021484375,
      "activations/layer6_attention_weight_max": 57.066532135009766,
      "activations/layer6_attention_weight_min": -55.745182037353516,
      "activations/layer7_attention_weight_max": 80.4999008178711,
      "activations/layer7_attention_weight_min": -87.01036071777344,
      "activations/layer8_attention_weight_max": 36.92900085449219,
      "activations/layer8_attention_weight_min": -39.837547302246094,
      "activations/layer9_attention_weight_max": 34.67788314819336,
      "activations/layer9_attention_weight_min": -34.055381774902344,
      "epoch": 5.07,
      "learning_rate": 0.00011849810606060605,
      "loss": 2.8516,
      "step": 87250
    },
    {
      "activations/layer0_attention_weight_max": 17.713489532470703,
      "activations/layer0_attention_weight_min": -12.393168449401855,
      "activations/layer10_attention_weight_max": 43.67860794067383,
      "activations/layer10_attention_weight_min": -31.924179077148438,
      "activations/layer11_attention_weight_max": 40.84404754638672,
      "activations/layer11_attention_weight_min": -28.571949005126953,
      "activations/layer12_attention_weight_max": 35.461402893066406,
      "activations/layer12_attention_weight_min": -31.31269645690918,
      "activations/layer13_attention_weight_max": 64.90823364257812,
      "activations/layer13_attention_weight_min": -35.51975631713867,
      "activations/layer14_attention_weight_max": 92.39305114746094,
      "activations/layer14_attention_weight_min": -48.08662414550781,
      "activations/layer15_attention_weight_max": 42.206932067871094,
      "activations/layer15_attention_weight_min": -29.568517684936523,
      "activations/layer16_attention_weight_max": 45.138362884521484,
      "activations/layer16_attention_weight_min": -27.606639862060547,
      "activations/layer17_attention_weight_max": 41.78485107421875,
      "activations/layer17_attention_weight_min": -24.918434143066406,
      "activations/layer18_attention_weight_max": 45.72812271118164,
      "activations/layer18_attention_weight_min": -22.61542320251465,
      "activations/layer19_attention_weight_max": 56.3283576965332,
      "activations/layer19_attention_weight_min": -30.734317779541016,
      "activations/layer1_attention_weight_max": 17.897451400756836,
      "activations/layer1_attention_weight_min": -17.76407241821289,
      "activations/layer20_attention_weight_max": 38.375755310058594,
      "activations/layer20_attention_weight_min": -22.904050827026367,
      "activations/layer21_attention_weight_max": 38.601234436035156,
      "activations/layer21_attention_weight_min": -22.638486862182617,
      "activations/layer22_attention_weight_max": 63.869529724121094,
      "activations/layer22_attention_weight_min": -28.187389373779297,
      "activations/layer23_attention_weight_max": 41.27152633666992,
      "activations/layer23_attention_weight_min": -21.01153564453125,
      "activations/layer2_attention_weight_max": 32.26877212524414,
      "activations/layer2_attention_weight_min": -31.336898803710938,
      "activations/layer3_attention_weight_max": 87.7392578125,
      "activations/layer3_attention_weight_min": -90.0555648803711,
      "activations/layer4_attention_weight_max": 129.7882537841797,
      "activations/layer4_attention_weight_min": -129.99522399902344,
      "activations/layer5_attention_weight_max": 85.64958190917969,
      "activations/layer5_attention_weight_min": -80.05950927734375,
      "activations/layer6_attention_weight_max": 61.86140441894531,
      "activations/layer6_attention_weight_min": -59.74138259887695,
      "activations/layer7_attention_weight_max": 100.48775482177734,
      "activations/layer7_attention_weight_min": -88.77719116210938,
      "activations/layer8_attention_weight_max": 51.979549407958984,
      "activations/layer8_attention_weight_min": -42.782470703125,
      "activations/layer9_attention_weight_max": 48.103546142578125,
      "activations/layer9_attention_weight_min": -36.46391296386719,
      "epoch": 5.07,
      "learning_rate": 0.00011847916666666666,
      "loss": 2.8544,
      "step": 87300
    },
    {
      "activations/layer0_attention_weight_max": 16.183517456054688,
      "activations/layer0_attention_weight_min": -12.481972694396973,
      "activations/layer10_attention_weight_max": 36.56740951538086,
      "activations/layer10_attention_weight_min": -30.806785583496094,
      "activations/layer11_attention_weight_max": 38.102256774902344,
      "activations/layer11_attention_weight_min": -31.691692352294922,
      "activations/layer12_attention_weight_max": 36.9951286315918,
      "activations/layer12_attention_weight_min": -30.8096981048584,
      "activations/layer13_attention_weight_max": 48.23515319824219,
      "activations/layer13_attention_weight_min": -34.953155517578125,
      "activations/layer14_attention_weight_max": 65.11624908447266,
      "activations/layer14_attention_weight_min": -50.1245231628418,
      "activations/layer15_attention_weight_max": 39.26987075805664,
      "activations/layer15_attention_weight_min": -29.92682647705078,
      "activations/layer16_attention_weight_max": 35.08882522583008,
      "activations/layer16_attention_weight_min": -27.158632278442383,
      "activations/layer17_attention_weight_max": 37.545406341552734,
      "activations/layer17_attention_weight_min": -23.98165512084961,
      "activations/layer18_attention_weight_max": 29.818645477294922,
      "activations/layer18_attention_weight_min": -20.113677978515625,
      "activations/layer19_attention_weight_max": 37.9202995300293,
      "activations/layer19_attention_weight_min": -30.784116744995117,
      "activations/layer1_attention_weight_max": 18.457048416137695,
      "activations/layer1_attention_weight_min": -15.669995307922363,
      "activations/layer20_attention_weight_max": 32.33563995361328,
      "activations/layer20_attention_weight_min": -24.61205291748047,
      "activations/layer21_attention_weight_max": 31.630157470703125,
      "activations/layer21_attention_weight_min": -20.33603858947754,
      "activations/layer22_attention_weight_max": 47.3021125793457,
      "activations/layer22_attention_weight_min": -31.65938377380371,
      "activations/layer23_attention_weight_max": 32.11760711669922,
      "activations/layer23_attention_weight_min": -22.508092880249023,
      "activations/layer2_attention_weight_max": 33.557376861572266,
      "activations/layer2_attention_weight_min": -31.457115173339844,
      "activations/layer3_attention_weight_max": 90.20714569091797,
      "activations/layer3_attention_weight_min": -91.901611328125,
      "activations/layer4_attention_weight_max": 134.20706176757812,
      "activations/layer4_attention_weight_min": -127.92243957519531,
      "activations/layer5_attention_weight_max": 86.63909149169922,
      "activations/layer5_attention_weight_min": -83.34391021728516,
      "activations/layer6_attention_weight_max": 56.34026336669922,
      "activations/layer6_attention_weight_min": -57.82486343383789,
      "activations/layer7_attention_weight_max": 83.72655487060547,
      "activations/layer7_attention_weight_min": -83.22270202636719,
      "activations/layer8_attention_weight_max": 42.752010345458984,
      "activations/layer8_attention_weight_min": -39.90373992919922,
      "activations/layer9_attention_weight_max": 37.71331787109375,
      "activations/layer9_attention_weight_min": -35.23379135131836,
      "epoch": 5.08,
      "learning_rate": 0.00011846022727272728,
      "loss": 2.8323,
      "step": 87350
    },
    {
      "activations/layer0_attention_weight_max": 17.431577682495117,
      "activations/layer0_attention_weight_min": -12.059288024902344,
      "activations/layer10_attention_weight_max": 36.625823974609375,
      "activations/layer10_attention_weight_min": -30.04904556274414,
      "activations/layer11_attention_weight_max": 37.99388885498047,
      "activations/layer11_attention_weight_min": -27.08711051940918,
      "activations/layer12_attention_weight_max": 44.101715087890625,
      "activations/layer12_attention_weight_min": -24.951311111450195,
      "activations/layer13_attention_weight_max": 47.1176643371582,
      "activations/layer13_attention_weight_min": -31.50428009033203,
      "activations/layer14_attention_weight_max": 69.06143951416016,
      "activations/layer14_attention_weight_min": -44.921630859375,
      "activations/layer15_attention_weight_max": 40.95835876464844,
      "activations/layer15_attention_weight_min": -28.28371238708496,
      "activations/layer16_attention_weight_max": 41.59709167480469,
      "activations/layer16_attention_weight_min": -29.787885665893555,
      "activations/layer17_attention_weight_max": 49.92251968383789,
      "activations/layer17_attention_weight_min": -26.582897186279297,
      "activations/layer18_attention_weight_max": 36.69216537475586,
      "activations/layer18_attention_weight_min": -19.92548179626465,
      "activations/layer19_attention_weight_max": 42.13117599487305,
      "activations/layer19_attention_weight_min": -29.519168853759766,
      "activations/layer1_attention_weight_max": 17.327417373657227,
      "activations/layer1_attention_weight_min": -15.684401512145996,
      "activations/layer20_attention_weight_max": 32.00348663330078,
      "activations/layer20_attention_weight_min": -22.640724182128906,
      "activations/layer21_attention_weight_max": 30.915401458740234,
      "activations/layer21_attention_weight_min": -19.03295135498047,
      "activations/layer22_attention_weight_max": 50.69253921508789,
      "activations/layer22_attention_weight_min": -28.934003829956055,
      "activations/layer23_attention_weight_max": 34.05111312866211,
      "activations/layer23_attention_weight_min": -20.246932983398438,
      "activations/layer2_attention_weight_max": 32.75990676879883,
      "activations/layer2_attention_weight_min": -31.757244110107422,
      "activations/layer3_attention_weight_max": 92.88731384277344,
      "activations/layer3_attention_weight_min": -93.19512176513672,
      "activations/layer4_attention_weight_max": 132.15199279785156,
      "activations/layer4_attention_weight_min": -129.32357788085938,
      "activations/layer5_attention_weight_max": 82.96366882324219,
      "activations/layer5_attention_weight_min": -77.53755187988281,
      "activations/layer6_attention_weight_max": 57.42081832885742,
      "activations/layer6_attention_weight_min": -57.97325897216797,
      "activations/layer7_attention_weight_max": 90.21427154541016,
      "activations/layer7_attention_weight_min": -85.48360443115234,
      "activations/layer8_attention_weight_max": 40.83778762817383,
      "activations/layer8_attention_weight_min": -39.105323791503906,
      "activations/layer9_attention_weight_max": 36.80862045288086,
      "activations/layer9_attention_weight_min": -32.28831100463867,
      "epoch": 5.08,
      "learning_rate": 0.00011844128787878787,
      "loss": 2.8419,
      "step": 87400
    },
    {
      "activations/layer0_attention_weight_max": 14.573285102844238,
      "activations/layer0_attention_weight_min": -11.437383651733398,
      "activations/layer10_attention_weight_max": 37.378971099853516,
      "activations/layer10_attention_weight_min": -31.308792114257812,
      "activations/layer11_attention_weight_max": 34.58133316040039,
      "activations/layer11_attention_weight_min": -28.047298431396484,
      "activations/layer12_attention_weight_max": 34.29221725463867,
      "activations/layer12_attention_weight_min": -27.449237823486328,
      "activations/layer13_attention_weight_max": 48.314979553222656,
      "activations/layer13_attention_weight_min": -33.04752731323242,
      "activations/layer14_attention_weight_max": 63.68632888793945,
      "activations/layer14_attention_weight_min": -44.67646408081055,
      "activations/layer15_attention_weight_max": 44.48133087158203,
      "activations/layer15_attention_weight_min": -29.91014289855957,
      "activations/layer16_attention_weight_max": 41.956050872802734,
      "activations/layer16_attention_weight_min": -27.577898025512695,
      "activations/layer17_attention_weight_max": 44.14155960083008,
      "activations/layer17_attention_weight_min": -25.981103897094727,
      "activations/layer18_attention_weight_max": 31.82607650756836,
      "activations/layer18_attention_weight_min": -20.71625518798828,
      "activations/layer19_attention_weight_max": 45.20831298828125,
      "activations/layer19_attention_weight_min": -30.90455436706543,
      "activations/layer1_attention_weight_max": 18.535737991333008,
      "activations/layer1_attention_weight_min": -12.032537460327148,
      "activations/layer20_attention_weight_max": 42.99103546142578,
      "activations/layer20_attention_weight_min": -24.82332992553711,
      "activations/layer21_attention_weight_max": 35.79545974731445,
      "activations/layer21_attention_weight_min": -21.25437355041504,
      "activations/layer22_attention_weight_max": 59.08158493041992,
      "activations/layer22_attention_weight_min": -30.021617889404297,
      "activations/layer23_attention_weight_max": 34.29663848876953,
      "activations/layer23_attention_weight_min": -22.679807662963867,
      "activations/layer2_attention_weight_max": 29.54077911376953,
      "activations/layer2_attention_weight_min": -32.332908630371094,
      "activations/layer3_attention_weight_max": 88.1412353515625,
      "activations/layer3_attention_weight_min": -85.52491760253906,
      "activations/layer4_attention_weight_max": 132.0696563720703,
      "activations/layer4_attention_weight_min": -130.36073303222656,
      "activations/layer5_attention_weight_max": 82.42854309082031,
      "activations/layer5_attention_weight_min": -81.8331298828125,
      "activations/layer6_attention_weight_max": 56.748600006103516,
      "activations/layer6_attention_weight_min": -56.967132568359375,
      "activations/layer7_attention_weight_max": 84.15251922607422,
      "activations/layer7_attention_weight_min": -85.55504608154297,
      "activations/layer8_attention_weight_max": 41.14177322387695,
      "activations/layer8_attention_weight_min": -39.21126174926758,
      "activations/layer9_attention_weight_max": 37.930503845214844,
      "activations/layer9_attention_weight_min": -34.83683395385742,
      "epoch": 5.08,
      "learning_rate": 0.00011842234848484848,
      "loss": 2.8433,
      "step": 87450
    },
    {
      "activations/layer0_attention_weight_max": 14.44433879852295,
      "activations/layer0_attention_weight_min": -12.037093162536621,
      "activations/layer10_attention_weight_max": 35.36848831176758,
      "activations/layer10_attention_weight_min": -32.32649612426758,
      "activations/layer11_attention_weight_max": 35.61052703857422,
      "activations/layer11_attention_weight_min": -28.903018951416016,
      "activations/layer12_attention_weight_max": 37.03044128417969,
      "activations/layer12_attention_weight_min": -30.88959503173828,
      "activations/layer13_attention_weight_max": 44.501399993896484,
      "activations/layer13_attention_weight_min": -34.971805572509766,
      "activations/layer14_attention_weight_max": 75.96065521240234,
      "activations/layer14_attention_weight_min": -52.120235443115234,
      "activations/layer15_attention_weight_max": 41.89657211303711,
      "activations/layer15_attention_weight_min": -31.8084659576416,
      "activations/layer16_attention_weight_max": 36.11094284057617,
      "activations/layer16_attention_weight_min": -28.422061920166016,
      "activations/layer17_attention_weight_max": 41.00920104980469,
      "activations/layer17_attention_weight_min": -27.6174259185791,
      "activations/layer18_attention_weight_max": 32.22795867919922,
      "activations/layer18_attention_weight_min": -20.118637084960938,
      "activations/layer19_attention_weight_max": 37.830902099609375,
      "activations/layer19_attention_weight_min": -31.114994049072266,
      "activations/layer1_attention_weight_max": 17.81897735595703,
      "activations/layer1_attention_weight_min": -15.105751991271973,
      "activations/layer20_attention_weight_max": 33.12441635131836,
      "activations/layer20_attention_weight_min": -25.066837310791016,
      "activations/layer21_attention_weight_max": 36.19921112060547,
      "activations/layer21_attention_weight_min": -21.724943161010742,
      "activations/layer22_attention_weight_max": 53.65966796875,
      "activations/layer22_attention_weight_min": -30.46529769897461,
      "activations/layer23_attention_weight_max": 34.071651458740234,
      "activations/layer23_attention_weight_min": -25.907424926757812,
      "activations/layer2_attention_weight_max": 32.56556701660156,
      "activations/layer2_attention_weight_min": -33.29003143310547,
      "activations/layer3_attention_weight_max": 84.35289764404297,
      "activations/layer3_attention_weight_min": -85.9596939086914,
      "activations/layer4_attention_weight_max": 126.46380615234375,
      "activations/layer4_attention_weight_min": -123.42463684082031,
      "activations/layer5_attention_weight_max": 78.494140625,
      "activations/layer5_attention_weight_min": -77.4820556640625,
      "activations/layer6_attention_weight_max": 53.082420349121094,
      "activations/layer6_attention_weight_min": -55.32581329345703,
      "activations/layer7_attention_weight_max": 88.26483917236328,
      "activations/layer7_attention_weight_min": -85.30492401123047,
      "activations/layer8_attention_weight_max": 38.258052825927734,
      "activations/layer8_attention_weight_min": -40.01524353027344,
      "activations/layer9_attention_weight_max": 37.029720306396484,
      "activations/layer9_attention_weight_min": -40.818359375,
      "epoch": 5.08,
      "learning_rate": 0.00011840340909090907,
      "loss": 2.8432,
      "step": 87500
    },
    {
      "activations/layer0_attention_weight_max": 16.868534088134766,
      "activations/layer0_attention_weight_min": -12.95691967010498,
      "activations/layer10_attention_weight_max": 36.06730651855469,
      "activations/layer10_attention_weight_min": -31.548112869262695,
      "activations/layer11_attention_weight_max": 33.7484130859375,
      "activations/layer11_attention_weight_min": -26.79212188720703,
      "activations/layer12_attention_weight_max": 28.8553409576416,
      "activations/layer12_attention_weight_min": -22.8466796875,
      "activations/layer13_attention_weight_max": 47.3709602355957,
      "activations/layer13_attention_weight_min": -34.98275375366211,
      "activations/layer14_attention_weight_max": 72.5477523803711,
      "activations/layer14_attention_weight_min": -50.563873291015625,
      "activations/layer15_attention_weight_max": 44.27119064331055,
      "activations/layer15_attention_weight_min": -28.44721221923828,
      "activations/layer16_attention_weight_max": 38.67070770263672,
      "activations/layer16_attention_weight_min": -28.106626510620117,
      "activations/layer17_attention_weight_max": 40.68043899536133,
      "activations/layer17_attention_weight_min": -24.511991500854492,
      "activations/layer18_attention_weight_max": 31.63503646850586,
      "activations/layer18_attention_weight_min": -22.273401260375977,
      "activations/layer19_attention_weight_max": 44.53009033203125,
      "activations/layer19_attention_weight_min": -30.217092514038086,
      "activations/layer1_attention_weight_max": 17.113622665405273,
      "activations/layer1_attention_weight_min": -13.513786315917969,
      "activations/layer20_attention_weight_max": 35.775245666503906,
      "activations/layer20_attention_weight_min": -24.179033279418945,
      "activations/layer21_attention_weight_max": 35.472381591796875,
      "activations/layer21_attention_weight_min": -20.133386611938477,
      "activations/layer22_attention_weight_max": 53.98728942871094,
      "activations/layer22_attention_weight_min": -27.777008056640625,
      "activations/layer23_attention_weight_max": 35.44688034057617,
      "activations/layer23_attention_weight_min": -19.64031219482422,
      "activations/layer2_attention_weight_max": 30.517684936523438,
      "activations/layer2_attention_weight_min": -30.878589630126953,
      "activations/layer3_attention_weight_max": 87.5501708984375,
      "activations/layer3_attention_weight_min": -87.90003967285156,
      "activations/layer4_attention_weight_max": 127.55313873291016,
      "activations/layer4_attention_weight_min": -122.93092346191406,
      "activations/layer5_attention_weight_max": 80.64522552490234,
      "activations/layer5_attention_weight_min": -76.193359375,
      "activations/layer6_attention_weight_max": 53.12180709838867,
      "activations/layer6_attention_weight_min": -60.38043212890625,
      "activations/layer7_attention_weight_max": 83.86166381835938,
      "activations/layer7_attention_weight_min": -89.2323989868164,
      "activations/layer8_attention_weight_max": 41.0651741027832,
      "activations/layer8_attention_weight_min": -40.4066276550293,
      "activations/layer9_attention_weight_max": 37.22172927856445,
      "activations/layer9_attention_weight_min": -33.88966751098633,
      "epoch": 5.09,
      "learning_rate": 0.00011838446969696968,
      "loss": 2.8365,
      "step": 87550
    },
    {
      "activations/layer0_attention_weight_max": 16.98142433166504,
      "activations/layer0_attention_weight_min": -11.704645156860352,
      "activations/layer10_attention_weight_max": 33.314903259277344,
      "activations/layer10_attention_weight_min": -31.630516052246094,
      "activations/layer11_attention_weight_max": 31.21210289001465,
      "activations/layer11_attention_weight_min": -27.342565536499023,
      "activations/layer12_attention_weight_max": 35.2746696472168,
      "activations/layer12_attention_weight_min": -26.581308364868164,
      "activations/layer13_attention_weight_max": 40.061004638671875,
      "activations/layer13_attention_weight_min": -35.002174377441406,
      "activations/layer14_attention_weight_max": 50.73822784423828,
      "activations/layer14_attention_weight_min": -40.43619918823242,
      "activations/layer15_attention_weight_max": 36.26871871948242,
      "activations/layer15_attention_weight_min": -26.97332191467285,
      "activations/layer16_attention_weight_max": 34.37623977661133,
      "activations/layer16_attention_weight_min": -25.89765167236328,
      "activations/layer17_attention_weight_max": 36.595394134521484,
      "activations/layer17_attention_weight_min": -24.13115692138672,
      "activations/layer18_attention_weight_max": 29.261911392211914,
      "activations/layer18_attention_weight_min": -19.27066993713379,
      "activations/layer19_attention_weight_max": 35.52347946166992,
      "activations/layer19_attention_weight_min": -28.419757843017578,
      "activations/layer1_attention_weight_max": 16.36129379272461,
      "activations/layer1_attention_weight_min": -14.04076862335205,
      "activations/layer20_attention_weight_max": 30.73165512084961,
      "activations/layer20_attention_weight_min": -22.240388870239258,
      "activations/layer21_attention_weight_max": 31.14398193359375,
      "activations/layer21_attention_weight_min": -21.824539184570312,
      "activations/layer22_attention_weight_max": 49.65947723388672,
      "activations/layer22_attention_weight_min": -29.12823486328125,
      "activations/layer23_attention_weight_max": 34.41000747680664,
      "activations/layer23_attention_weight_min": -21.537382125854492,
      "activations/layer2_attention_weight_max": 30.725317001342773,
      "activations/layer2_attention_weight_min": -28.537120819091797,
      "activations/layer3_attention_weight_max": 90.86344909667969,
      "activations/layer3_attention_weight_min": -85.64645385742188,
      "activations/layer4_attention_weight_max": 131.21035766601562,
      "activations/layer4_attention_weight_min": -121.30097961425781,
      "activations/layer5_attention_weight_max": 84.73806762695312,
      "activations/layer5_attention_weight_min": -77.7469482421875,
      "activations/layer6_attention_weight_max": 57.668399810791016,
      "activations/layer6_attention_weight_min": -54.00990295410156,
      "activations/layer7_attention_weight_max": 86.9625244140625,
      "activations/layer7_attention_weight_min": -82.76182556152344,
      "activations/layer8_attention_weight_max": 38.59223937988281,
      "activations/layer8_attention_weight_min": -39.858707427978516,
      "activations/layer9_attention_weight_max": 37.004127502441406,
      "activations/layer9_attention_weight_min": -36.156009674072266,
      "epoch": 5.09,
      "learning_rate": 0.0001183655303030303,
      "loss": 2.8566,
      "step": 87600
    },
    {
      "activations/layer0_attention_weight_max": 17.590862274169922,
      "activations/layer0_attention_weight_min": -12.19737434387207,
      "activations/layer10_attention_weight_max": 33.36243438720703,
      "activations/layer10_attention_weight_min": -30.583343505859375,
      "activations/layer11_attention_weight_max": 32.28173828125,
      "activations/layer11_attention_weight_min": -27.54863739013672,
      "activations/layer12_attention_weight_max": 27.473739624023438,
      "activations/layer12_attention_weight_min": -29.589393615722656,
      "activations/layer13_attention_weight_max": 44.671512603759766,
      "activations/layer13_attention_weight_min": -32.41276931762695,
      "activations/layer14_attention_weight_max": 60.72800827026367,
      "activations/layer14_attention_weight_min": -48.78804016113281,
      "activations/layer15_attention_weight_max": 38.09617233276367,
      "activations/layer15_attention_weight_min": -28.235984802246094,
      "activations/layer16_attention_weight_max": 36.72557067871094,
      "activations/layer16_attention_weight_min": -28.409727096557617,
      "activations/layer17_attention_weight_max": 38.61860656738281,
      "activations/layer17_attention_weight_min": -25.711650848388672,
      "activations/layer18_attention_weight_max": 35.84334945678711,
      "activations/layer18_attention_weight_min": -22.158899307250977,
      "activations/layer19_attention_weight_max": 42.3940544128418,
      "activations/layer19_attention_weight_min": -30.425382614135742,
      "activations/layer1_attention_weight_max": 17.01445770263672,
      "activations/layer1_attention_weight_min": -13.442606925964355,
      "activations/layer20_attention_weight_max": 30.119304656982422,
      "activations/layer20_attention_weight_min": -23.1551513671875,
      "activations/layer21_attention_weight_max": 30.399877548217773,
      "activations/layer21_attention_weight_min": -22.46521759033203,
      "activations/layer22_attention_weight_max": 50.63759994506836,
      "activations/layer22_attention_weight_min": -28.908973693847656,
      "activations/layer23_attention_weight_max": 36.91058349609375,
      "activations/layer23_attention_weight_min": -20.3339786529541,
      "activations/layer2_attention_weight_max": 32.10820007324219,
      "activations/layer2_attention_weight_min": -28.483318328857422,
      "activations/layer3_attention_weight_max": 90.91213989257812,
      "activations/layer3_attention_weight_min": -87.51354217529297,
      "activations/layer4_attention_weight_max": 137.33860778808594,
      "activations/layer4_attention_weight_min": -127.13338470458984,
      "activations/layer5_attention_weight_max": 85.00000762939453,
      "activations/layer5_attention_weight_min": -82.07171630859375,
      "activations/layer6_attention_weight_max": 61.993289947509766,
      "activations/layer6_attention_weight_min": -58.275184631347656,
      "activations/layer7_attention_weight_max": 84.83180236816406,
      "activations/layer7_attention_weight_min": -84.31468963623047,
      "activations/layer8_attention_weight_max": 42.52231216430664,
      "activations/layer8_attention_weight_min": -38.78453063964844,
      "activations/layer9_attention_weight_max": 37.929439544677734,
      "activations/layer9_attention_weight_min": -34.29550552368164,
      "epoch": 5.09,
      "learning_rate": 0.0001183465909090909,
      "loss": 2.8534,
      "step": 87650
    },
    {
      "activations/layer0_attention_weight_max": 17.032983779907227,
      "activations/layer0_attention_weight_min": -12.352543830871582,
      "activations/layer10_attention_weight_max": 32.81722640991211,
      "activations/layer10_attention_weight_min": -28.048185348510742,
      "activations/layer11_attention_weight_max": 30.351713180541992,
      "activations/layer11_attention_weight_min": -26.206493377685547,
      "activations/layer12_attention_weight_max": 27.92746353149414,
      "activations/layer12_attention_weight_min": -27.803348541259766,
      "activations/layer13_attention_weight_max": 39.119319915771484,
      "activations/layer13_attention_weight_min": -29.83415412902832,
      "activations/layer14_attention_weight_max": 58.55971145629883,
      "activations/layer14_attention_weight_min": -42.7303352355957,
      "activations/layer15_attention_weight_max": 40.9550895690918,
      "activations/layer15_attention_weight_min": -27.94035530090332,
      "activations/layer16_attention_weight_max": 35.12961196899414,
      "activations/layer16_attention_weight_min": -27.73301887512207,
      "activations/layer17_attention_weight_max": 38.95491409301758,
      "activations/layer17_attention_weight_min": -25.067977905273438,
      "activations/layer18_attention_weight_max": 33.04403305053711,
      "activations/layer18_attention_weight_min": -18.17827796936035,
      "activations/layer19_attention_weight_max": 36.710811614990234,
      "activations/layer19_attention_weight_min": -27.804319381713867,
      "activations/layer1_attention_weight_max": 18.85939598083496,
      "activations/layer1_attention_weight_min": -15.350942611694336,
      "activations/layer20_attention_weight_max": 32.84602355957031,
      "activations/layer20_attention_weight_min": -21.771759033203125,
      "activations/layer21_attention_weight_max": 30.777999877929688,
      "activations/layer21_attention_weight_min": -19.816574096679688,
      "activations/layer22_attention_weight_max": 52.68598175048828,
      "activations/layer22_attention_weight_min": -25.460800170898438,
      "activations/layer23_attention_weight_max": 32.31282424926758,
      "activations/layer23_attention_weight_min": -19.88226318359375,
      "activations/layer2_attention_weight_max": 29.504261016845703,
      "activations/layer2_attention_weight_min": -29.266136169433594,
      "activations/layer3_attention_weight_max": 85.741943359375,
      "activations/layer3_attention_weight_min": -88.723876953125,
      "activations/layer4_attention_weight_max": 128.80795288085938,
      "activations/layer4_attention_weight_min": -123.89867401123047,
      "activations/layer5_attention_weight_max": 80.5373764038086,
      "activations/layer5_attention_weight_min": -78.04641723632812,
      "activations/layer6_attention_weight_max": 55.11773681640625,
      "activations/layer6_attention_weight_min": -53.66191101074219,
      "activations/layer7_attention_weight_max": 79.88276672363281,
      "activations/layer7_attention_weight_min": -79.07156372070312,
      "activations/layer8_attention_weight_max": 39.60428237915039,
      "activations/layer8_attention_weight_min": -37.67539978027344,
      "activations/layer9_attention_weight_max": 33.95108413696289,
      "activations/layer9_attention_weight_min": -33.766902923583984,
      "epoch": 5.1,
      "learning_rate": 0.0001183276515151515,
      "loss": 2.844,
      "step": 87700
    },
    {
      "activations/layer0_attention_weight_max": 17.78643226623535,
      "activations/layer0_attention_weight_min": -11.952523231506348,
      "activations/layer10_attention_weight_max": 32.616729736328125,
      "activations/layer10_attention_weight_min": -30.78399658203125,
      "activations/layer11_attention_weight_max": 32.12323760986328,
      "activations/layer11_attention_weight_min": -28.567161560058594,
      "activations/layer12_attention_weight_max": 39.774871826171875,
      "activations/layer12_attention_weight_min": -28.68438148498535,
      "activations/layer13_attention_weight_max": 46.313655853271484,
      "activations/layer13_attention_weight_min": -33.977882385253906,
      "activations/layer14_attention_weight_max": 65.73173522949219,
      "activations/layer14_attention_weight_min": -43.25044250488281,
      "activations/layer15_attention_weight_max": 37.128326416015625,
      "activations/layer15_attention_weight_min": -28.6260986328125,
      "activations/layer16_attention_weight_max": 44.61675262451172,
      "activations/layer16_attention_weight_min": -28.74323081970215,
      "activations/layer17_attention_weight_max": 37.86054611206055,
      "activations/layer17_attention_weight_min": -24.650161743164062,
      "activations/layer18_attention_weight_max": 34.95878601074219,
      "activations/layer18_attention_weight_min": -21.99480438232422,
      "activations/layer19_attention_weight_max": 43.94540786743164,
      "activations/layer19_attention_weight_min": -29.041147232055664,
      "activations/layer1_attention_weight_max": 17.804702758789062,
      "activations/layer1_attention_weight_min": -14.571006774902344,
      "activations/layer20_attention_weight_max": 34.90193176269531,
      "activations/layer20_attention_weight_min": -23.224620819091797,
      "activations/layer21_attention_weight_max": 30.196678161621094,
      "activations/layer21_attention_weight_min": -21.27614402770996,
      "activations/layer22_attention_weight_max": 52.84893035888672,
      "activations/layer22_attention_weight_min": -28.9680233001709,
      "activations/layer23_attention_weight_max": 37.248687744140625,
      "activations/layer23_attention_weight_min": -22.765026092529297,
      "activations/layer2_attention_weight_max": 30.432241439819336,
      "activations/layer2_attention_weight_min": -29.706274032592773,
      "activations/layer3_attention_weight_max": 88.04478454589844,
      "activations/layer3_attention_weight_min": -87.63926696777344,
      "activations/layer4_attention_weight_max": 126.30175018310547,
      "activations/layer4_attention_weight_min": -128.67991638183594,
      "activations/layer5_attention_weight_max": 78.37321472167969,
      "activations/layer5_attention_weight_min": -76.00447082519531,
      "activations/layer6_attention_weight_max": 54.42655944824219,
      "activations/layer6_attention_weight_min": -55.31132888793945,
      "activations/layer7_attention_weight_max": 80.7598876953125,
      "activations/layer7_attention_weight_min": -87.40409851074219,
      "activations/layer8_attention_weight_max": 38.12763214111328,
      "activations/layer8_attention_weight_min": -38.13007354736328,
      "activations/layer9_attention_weight_max": 34.71152877807617,
      "activations/layer9_attention_weight_min": -34.22724533081055,
      "epoch": 5.1,
      "learning_rate": 0.00011830871212121212,
      "loss": 2.8479,
      "step": 87750
    },
    {
      "activations/layer0_attention_weight_max": 17.869964599609375,
      "activations/layer0_attention_weight_min": -12.98825454711914,
      "activations/layer10_attention_weight_max": 37.68959426879883,
      "activations/layer10_attention_weight_min": -32.29819107055664,
      "activations/layer11_attention_weight_max": 35.774024963378906,
      "activations/layer11_attention_weight_min": -27.475133895874023,
      "activations/layer12_attention_weight_max": 37.002540588378906,
      "activations/layer12_attention_weight_min": -27.937549591064453,
      "activations/layer13_attention_weight_max": 51.99864959716797,
      "activations/layer13_attention_weight_min": -33.016231536865234,
      "activations/layer14_attention_weight_max": 90.3086929321289,
      "activations/layer14_attention_weight_min": -55.737403869628906,
      "activations/layer15_attention_weight_max": 51.25446701049805,
      "activations/layer15_attention_weight_min": -29.443988800048828,
      "activations/layer16_attention_weight_max": 43.90204620361328,
      "activations/layer16_attention_weight_min": -29.143964767456055,
      "activations/layer17_attention_weight_max": 43.84477996826172,
      "activations/layer17_attention_weight_min": -24.049558639526367,
      "activations/layer18_attention_weight_max": 39.62804412841797,
      "activations/layer18_attention_weight_min": -20.5782527923584,
      "activations/layer19_attention_weight_max": 51.052207946777344,
      "activations/layer19_attention_weight_min": -28.893156051635742,
      "activations/layer1_attention_weight_max": 17.290611267089844,
      "activations/layer1_attention_weight_min": -15.181758880615234,
      "activations/layer20_attention_weight_max": 45.26923370361328,
      "activations/layer20_attention_weight_min": -23.74250602722168,
      "activations/layer21_attention_weight_max": 37.958858489990234,
      "activations/layer21_attention_weight_min": -22.367517471313477,
      "activations/layer22_attention_weight_max": 56.13628005981445,
      "activations/layer22_attention_weight_min": -27.228105545043945,
      "activations/layer23_attention_weight_max": 37.53639602661133,
      "activations/layer23_attention_weight_min": -18.775548934936523,
      "activations/layer2_attention_weight_max": 31.5758056640625,
      "activations/layer2_attention_weight_min": -31.105030059814453,
      "activations/layer3_attention_weight_max": 89.08318328857422,
      "activations/layer3_attention_weight_min": -90.21176147460938,
      "activations/layer4_attention_weight_max": 127.92656707763672,
      "activations/layer4_attention_weight_min": -131.17568969726562,
      "activations/layer5_attention_weight_max": 83.09563446044922,
      "activations/layer5_attention_weight_min": -80.8993148803711,
      "activations/layer6_attention_weight_max": 54.978485107421875,
      "activations/layer6_attention_weight_min": -58.40253829956055,
      "activations/layer7_attention_weight_max": 88.40701293945312,
      "activations/layer7_attention_weight_min": -87.28781127929688,
      "activations/layer8_attention_weight_max": 42.11750411987305,
      "activations/layer8_attention_weight_min": -43.02784729003906,
      "activations/layer9_attention_weight_max": 39.76105880737305,
      "activations/layer9_attention_weight_min": -35.4275016784668,
      "epoch": 5.1,
      "learning_rate": 0.00011828977272727272,
      "loss": 2.844,
      "step": 87800
    },
    {
      "activations/layer0_attention_weight_max": 15.8502836227417,
      "activations/layer0_attention_weight_min": -11.135924339294434,
      "activations/layer10_attention_weight_max": 35.464500427246094,
      "activations/layer10_attention_weight_min": -29.88524055480957,
      "activations/layer11_attention_weight_max": 35.90962219238281,
      "activations/layer11_attention_weight_min": -27.72589874267578,
      "activations/layer12_attention_weight_max": 25.198251724243164,
      "activations/layer12_attention_weight_min": -24.147701263427734,
      "activations/layer13_attention_weight_max": 45.07978439331055,
      "activations/layer13_attention_weight_min": -31.450679779052734,
      "activations/layer14_attention_weight_max": 62.31291198730469,
      "activations/layer14_attention_weight_min": -43.7332649230957,
      "activations/layer15_attention_weight_max": 44.948726654052734,
      "activations/layer15_attention_weight_min": -28.065597534179688,
      "activations/layer16_attention_weight_max": 41.418487548828125,
      "activations/layer16_attention_weight_min": -28.72351837158203,
      "activations/layer17_attention_weight_max": 41.435401916503906,
      "activations/layer17_attention_weight_min": -24.3840389251709,
      "activations/layer18_attention_weight_max": 33.63829803466797,
      "activations/layer18_attention_weight_min": -20.951080322265625,
      "activations/layer19_attention_weight_max": 40.21179962158203,
      "activations/layer19_attention_weight_min": -29.401351928710938,
      "activations/layer1_attention_weight_max": 19.164716720581055,
      "activations/layer1_attention_weight_min": -16.141027450561523,
      "activations/layer20_attention_weight_max": 30.105016708374023,
      "activations/layer20_attention_weight_min": -23.365718841552734,
      "activations/layer21_attention_weight_max": 37.597225189208984,
      "activations/layer21_attention_weight_min": -21.887208938598633,
      "activations/layer22_attention_weight_max": 50.247108459472656,
      "activations/layer22_attention_weight_min": -29.39134407043457,
      "activations/layer23_attention_weight_max": 33.068115234375,
      "activations/layer23_attention_weight_min": -20.39335823059082,
      "activations/layer2_attention_weight_max": 30.792526245117188,
      "activations/layer2_attention_weight_min": -29.758054733276367,
      "activations/layer3_attention_weight_max": 86.68244934082031,
      "activations/layer3_attention_weight_min": -87.16915893554688,
      "activations/layer4_attention_weight_max": 131.4073028564453,
      "activations/layer4_attention_weight_min": -126.65443420410156,
      "activations/layer5_attention_weight_max": 85.2569580078125,
      "activations/layer5_attention_weight_min": -80.66554260253906,
      "activations/layer6_attention_weight_max": 55.504642486572266,
      "activations/layer6_attention_weight_min": -57.62530517578125,
      "activations/layer7_attention_weight_max": 82.5931167602539,
      "activations/layer7_attention_weight_min": -85.926513671875,
      "activations/layer8_attention_weight_max": 42.608428955078125,
      "activations/layer8_attention_weight_min": -40.12904739379883,
      "activations/layer9_attention_weight_max": 39.91059875488281,
      "activations/layer9_attention_weight_min": -38.475433349609375,
      "epoch": 5.1,
      "learning_rate": 0.00011827083333333332,
      "loss": 2.8477,
      "step": 87850
    },
    {
      "activations/layer0_attention_weight_max": 16.78330421447754,
      "activations/layer0_attention_weight_min": -12.519886016845703,
      "activations/layer10_attention_weight_max": 36.751136779785156,
      "activations/layer10_attention_weight_min": -29.7706356048584,
      "activations/layer11_attention_weight_max": 36.20553207397461,
      "activations/layer11_attention_weight_min": -27.432662963867188,
      "activations/layer12_attention_weight_max": 25.777027130126953,
      "activations/layer12_attention_weight_min": -26.370162963867188,
      "activations/layer13_attention_weight_max": 41.858985900878906,
      "activations/layer13_attention_weight_min": -31.847511291503906,
      "activations/layer14_attention_weight_max": 56.10710525512695,
      "activations/layer14_attention_weight_min": -41.43769454956055,
      "activations/layer15_attention_weight_max": 40.046077728271484,
      "activations/layer15_attention_weight_min": -27.9356689453125,
      "activations/layer16_attention_weight_max": 37.69513702392578,
      "activations/layer16_attention_weight_min": -27.241348266601562,
      "activations/layer17_attention_weight_max": 39.51333999633789,
      "activations/layer17_attention_weight_min": -24.52092170715332,
      "activations/layer18_attention_weight_max": 32.93281555175781,
      "activations/layer18_attention_weight_min": -22.69432258605957,
      "activations/layer19_attention_weight_max": 36.9676628112793,
      "activations/layer19_attention_weight_min": -27.67624855041504,
      "activations/layer1_attention_weight_max": 17.35115623474121,
      "activations/layer1_attention_weight_min": -15.780014038085938,
      "activations/layer20_attention_weight_max": 32.65949630737305,
      "activations/layer20_attention_weight_min": -22.7143611907959,
      "activations/layer21_attention_weight_max": 32.48756790161133,
      "activations/layer21_attention_weight_min": -23.077707290649414,
      "activations/layer22_attention_weight_max": 51.55002212524414,
      "activations/layer22_attention_weight_min": -28.986345291137695,
      "activations/layer23_attention_weight_max": 34.28143310546875,
      "activations/layer23_attention_weight_min": -20.304962158203125,
      "activations/layer2_attention_weight_max": 30.658367156982422,
      "activations/layer2_attention_weight_min": -30.289770126342773,
      "activations/layer3_attention_weight_max": 86.17662811279297,
      "activations/layer3_attention_weight_min": -85.42423248291016,
      "activations/layer4_attention_weight_max": 124.53462219238281,
      "activations/layer4_attention_weight_min": -120.65216064453125,
      "activations/layer5_attention_weight_max": 78.83863067626953,
      "activations/layer5_attention_weight_min": -76.08953857421875,
      "activations/layer6_attention_weight_max": 54.98920440673828,
      "activations/layer6_attention_weight_min": -53.59070587158203,
      "activations/layer7_attention_weight_max": 84.8031997680664,
      "activations/layer7_attention_weight_min": -80.00336456298828,
      "activations/layer8_attention_weight_max": 40.85224533081055,
      "activations/layer8_attention_weight_min": -37.50017166137695,
      "activations/layer9_attention_weight_max": 37.7984733581543,
      "activations/layer9_attention_weight_min": -33.64677047729492,
      "epoch": 5.11,
      "learning_rate": 0.00011825189393939392,
      "loss": 2.8451,
      "step": 87900
    },
    {
      "activations/layer0_attention_weight_max": 17.53145980834961,
      "activations/layer0_attention_weight_min": -11.611944198608398,
      "activations/layer10_attention_weight_max": 33.83842849731445,
      "activations/layer10_attention_weight_min": -30.156578063964844,
      "activations/layer11_attention_weight_max": 29.748676300048828,
      "activations/layer11_attention_weight_min": -29.97182846069336,
      "activations/layer12_attention_weight_max": 29.557096481323242,
      "activations/layer12_attention_weight_min": -28.56523895263672,
      "activations/layer13_attention_weight_max": 46.03769302368164,
      "activations/layer13_attention_weight_min": -34.443050384521484,
      "activations/layer14_attention_weight_max": 55.60700607299805,
      "activations/layer14_attention_weight_min": -43.18562698364258,
      "activations/layer15_attention_weight_max": 37.217166900634766,
      "activations/layer15_attention_weight_min": -28.598472595214844,
      "activations/layer16_attention_weight_max": 36.783363342285156,
      "activations/layer16_attention_weight_min": -28.957433700561523,
      "activations/layer17_attention_weight_max": 36.461509704589844,
      "activations/layer17_attention_weight_min": -25.586353302001953,
      "activations/layer18_attention_weight_max": 31.147294998168945,
      "activations/layer18_attention_weight_min": -23.672603607177734,
      "activations/layer19_attention_weight_max": 40.54767608642578,
      "activations/layer19_attention_weight_min": -29.297927856445312,
      "activations/layer1_attention_weight_max": 17.259387969970703,
      "activations/layer1_attention_weight_min": -13.929469108581543,
      "activations/layer20_attention_weight_max": 32.37546920776367,
      "activations/layer20_attention_weight_min": -23.531112670898438,
      "activations/layer21_attention_weight_max": 32.409820556640625,
      "activations/layer21_attention_weight_min": -22.025651931762695,
      "activations/layer22_attention_weight_max": 50.20294952392578,
      "activations/layer22_attention_weight_min": -29.372535705566406,
      "activations/layer23_attention_weight_max": 36.0621223449707,
      "activations/layer23_attention_weight_min": -23.68143081665039,
      "activations/layer2_attention_weight_max": 30.84023666381836,
      "activations/layer2_attention_weight_min": -29.76175308227539,
      "activations/layer3_attention_weight_max": 86.40447235107422,
      "activations/layer3_attention_weight_min": -90.4680404663086,
      "activations/layer4_attention_weight_max": 126.41756439208984,
      "activations/layer4_attention_weight_min": -124.01793670654297,
      "activations/layer5_attention_weight_max": 79.90882873535156,
      "activations/layer5_attention_weight_min": -78.7343521118164,
      "activations/layer6_attention_weight_max": 53.64339065551758,
      "activations/layer6_attention_weight_min": -55.84930419921875,
      "activations/layer7_attention_weight_max": 88.47962188720703,
      "activations/layer7_attention_weight_min": -80.5015640258789,
      "activations/layer8_attention_weight_max": 41.20551681518555,
      "activations/layer8_attention_weight_min": -41.54861831665039,
      "activations/layer9_attention_weight_max": 38.06103515625,
      "activations/layer9_attention_weight_min": -33.43968963623047,
      "epoch": 5.11,
      "learning_rate": 0.00011823295454545454,
      "loss": 2.846,
      "step": 87950
    },
    {
      "activations/layer0_attention_weight_max": 16.769493103027344,
      "activations/layer0_attention_weight_min": -11.977396011352539,
      "activations/layer10_attention_weight_max": 34.88703155517578,
      "activations/layer10_attention_weight_min": -32.475345611572266,
      "activations/layer11_attention_weight_max": 34.37055969238281,
      "activations/layer11_attention_weight_min": -27.265422821044922,
      "activations/layer12_attention_weight_max": 30.826860427856445,
      "activations/layer12_attention_weight_min": -26.81714630126953,
      "activations/layer13_attention_weight_max": 45.14310073852539,
      "activations/layer13_attention_weight_min": -32.265499114990234,
      "activations/layer14_attention_weight_max": 59.000762939453125,
      "activations/layer14_attention_weight_min": -42.57814407348633,
      "activations/layer15_attention_weight_max": 47.07269287109375,
      "activations/layer15_attention_weight_min": -29.02055549621582,
      "activations/layer16_attention_weight_max": 36.820159912109375,
      "activations/layer16_attention_weight_min": -27.913118362426758,
      "activations/layer17_attention_weight_max": 37.83325958251953,
      "activations/layer17_attention_weight_min": -25.154085159301758,
      "activations/layer18_attention_weight_max": 30.424911499023438,
      "activations/layer18_attention_weight_min": -19.961849212646484,
      "activations/layer19_attention_weight_max": 40.13449478149414,
      "activations/layer19_attention_weight_min": -28.949542999267578,
      "activations/layer1_attention_weight_max": 16.983848571777344,
      "activations/layer1_attention_weight_min": -14.6907320022583,
      "activations/layer20_attention_weight_max": 34.130043029785156,
      "activations/layer20_attention_weight_min": -23.313457489013672,
      "activations/layer21_attention_weight_max": 33.9507942199707,
      "activations/layer21_attention_weight_min": -22.216636657714844,
      "activations/layer22_attention_weight_max": 51.620452880859375,
      "activations/layer22_attention_weight_min": -29.76230239868164,
      "activations/layer23_attention_weight_max": 33.806427001953125,
      "activations/layer23_attention_weight_min": -22.98292350769043,
      "activations/layer2_attention_weight_max": 32.51013946533203,
      "activations/layer2_attention_weight_min": -32.09725570678711,
      "activations/layer3_attention_weight_max": 88.32000732421875,
      "activations/layer3_attention_weight_min": -87.8515396118164,
      "activations/layer4_attention_weight_max": 131.48590087890625,
      "activations/layer4_attention_weight_min": -126.1839599609375,
      "activations/layer5_attention_weight_max": 82.55168914794922,
      "activations/layer5_attention_weight_min": -78.42833709716797,
      "activations/layer6_attention_weight_max": 58.20075225830078,
      "activations/layer6_attention_weight_min": -56.954925537109375,
      "activations/layer7_attention_weight_max": 88.446533203125,
      "activations/layer7_attention_weight_min": -87.78199005126953,
      "activations/layer8_attention_weight_max": 41.67523956298828,
      "activations/layer8_attention_weight_min": -39.695858001708984,
      "activations/layer9_attention_weight_max": 36.77732467651367,
      "activations/layer9_attention_weight_min": -35.27640914916992,
      "epoch": 5.11,
      "learning_rate": 0.00011821401515151515,
      "loss": 2.8324,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_loss": 2.80078125,
      "eval_runtime": 8.5736,
      "eval_samples_per_second": 500.841,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_openwebtext_loss": 2.80078125,
      "eval_openwebtext_ppl": 16.457499171202052,
      "eval_openwebtext_runtime": 8.5736,
      "eval_openwebtext_samples_per_second": 500.841,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_wikitext_loss": 3.029296875,
      "eval_wikitext_ppl": 20.68268496272443,
      "eval_wikitext_runtime": 2.0283,
      "eval_wikitext_samples_per_second": 224.817,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_lambada_loss": 2.751953125,
      "eval_lambada_ppl": 15.673213754973451,
      "eval_lambada_runtime": 9.613,
      "eval_lambada_samples_per_second": 506.503,
      "step": 88000
    },
    {
      "activations/layer0_attention_weight_max": 17.11594581604004,
      "activations/layer0_attention_weight_min": -11.739679336547852,
      "activations/layer10_attention_weight_max": 37.666603088378906,
      "activations/layer10_attention_weight_min": -32.925472259521484,
      "activations/layer11_attention_weight_max": 36.655242919921875,
      "activations/layer11_attention_weight_min": -30.1193904876709,
      "activations/layer12_attention_weight_max": 28.93482208251953,
      "activations/layer12_attention_weight_min": -25.727027893066406,
      "activations/layer13_attention_weight_max": 48.90047836303711,
      "activations/layer13_attention_weight_min": -34.0485725402832,
      "activations/layer14_attention_weight_max": 78.11556243896484,
      "activations/layer14_attention_weight_min": -48.10382080078125,
      "activations/layer15_attention_weight_max": 45.0738410949707,
      "activations/layer15_attention_weight_min": -29.733348846435547,
      "activations/layer16_attention_weight_max": 42.30400466918945,
      "activations/layer16_attention_weight_min": -29.00200080871582,
      "activations/layer17_attention_weight_max": 46.89252853393555,
      "activations/layer17_attention_weight_min": -27.63289451599121,
      "activations/layer18_attention_weight_max": 33.42616271972656,
      "activations/layer18_attention_weight_min": -20.653369903564453,
      "activations/layer19_attention_weight_max": 48.419639587402344,
      "activations/layer19_attention_weight_min": -29.377538681030273,
      "activations/layer1_attention_weight_max": 17.182666778564453,
      "activations/layer1_attention_weight_min": -14.104272842407227,
      "activations/layer20_attention_weight_max": 36.84051513671875,
      "activations/layer20_attention_weight_min": -23.933208465576172,
      "activations/layer21_attention_weight_max": 38.249420166015625,
      "activations/layer21_attention_weight_min": -21.25398826599121,
      "activations/layer22_attention_weight_max": 58.384803771972656,
      "activations/layer22_attention_weight_min": -28.002222061157227,
      "activations/layer23_attention_weight_max": 36.34173583984375,
      "activations/layer23_attention_weight_min": -18.988948822021484,
      "activations/layer2_attention_weight_max": 35.01715850830078,
      "activations/layer2_attention_weight_min": -32.84717559814453,
      "activations/layer3_attention_weight_max": 93.24354553222656,
      "activations/layer3_attention_weight_min": -95.0235595703125,
      "activations/layer4_attention_weight_max": 133.01954650878906,
      "activations/layer4_attention_weight_min": -132.05340576171875,
      "activations/layer5_attention_weight_max": 83.02227783203125,
      "activations/layer5_attention_weight_min": -84.5606689453125,
      "activations/layer6_attention_weight_max": 58.172019958496094,
      "activations/layer6_attention_weight_min": -58.14375686645508,
      "activations/layer7_attention_weight_max": 87.59508514404297,
      "activations/layer7_attention_weight_min": -86.10308837890625,
      "activations/layer8_attention_weight_max": 43.29079055786133,
      "activations/layer8_attention_weight_min": -41.45895004272461,
      "activations/layer9_attention_weight_max": 40.83137130737305,
      "activations/layer9_attention_weight_min": -37.20344161987305,
      "epoch": 5.12,
      "learning_rate": 0.00011819507575757574,
      "loss": 2.8668,
      "step": 88050
    },
    {
      "activations/layer0_attention_weight_max": 17.141773223876953,
      "activations/layer0_attention_weight_min": -12.947364807128906,
      "activations/layer10_attention_weight_max": 36.61687088012695,
      "activations/layer10_attention_weight_min": -30.026451110839844,
      "activations/layer11_attention_weight_max": 43.74404525756836,
      "activations/layer11_attention_weight_min": -27.912303924560547,
      "activations/layer12_attention_weight_max": 55.03706359863281,
      "activations/layer12_attention_weight_min": -28.202592849731445,
      "activations/layer13_attention_weight_max": 59.02244567871094,
      "activations/layer13_attention_weight_min": -33.01351547241211,
      "activations/layer14_attention_weight_max": 62.40146255493164,
      "activations/layer14_attention_weight_min": -42.34014892578125,
      "activations/layer15_attention_weight_max": 40.480045318603516,
      "activations/layer15_attention_weight_min": -27.472810745239258,
      "activations/layer16_attention_weight_max": 35.68904113769531,
      "activations/layer16_attention_weight_min": -28.631622314453125,
      "activations/layer17_attention_weight_max": 37.6722412109375,
      "activations/layer17_attention_weight_min": -24.69016456604004,
      "activations/layer18_attention_weight_max": 37.33159255981445,
      "activations/layer18_attention_weight_min": -22.03473663330078,
      "activations/layer19_attention_weight_max": 39.107242584228516,
      "activations/layer19_attention_weight_min": -27.801864624023438,
      "activations/layer1_attention_weight_max": 18.42586898803711,
      "activations/layer1_attention_weight_min": -14.698442459106445,
      "activations/layer20_attention_weight_max": 30.940439224243164,
      "activations/layer20_attention_weight_min": -22.360748291015625,
      "activations/layer21_attention_weight_max": 32.858154296875,
      "activations/layer21_attention_weight_min": -21.773845672607422,
      "activations/layer22_attention_weight_max": 50.95860290527344,
      "activations/layer22_attention_weight_min": -27.222599029541016,
      "activations/layer23_attention_weight_max": 32.63673400878906,
      "activations/layer23_attention_weight_min": -22.560422897338867,
      "activations/layer2_attention_weight_max": 32.79841613769531,
      "activations/layer2_attention_weight_min": -32.52366638183594,
      "activations/layer3_attention_weight_max": 89.08073425292969,
      "activations/layer3_attention_weight_min": -93.47429656982422,
      "activations/layer4_attention_weight_max": 127.014404296875,
      "activations/layer4_attention_weight_min": -123.5656509399414,
      "activations/layer5_attention_weight_max": 78.53716278076172,
      "activations/layer5_attention_weight_min": -80.37677001953125,
      "activations/layer6_attention_weight_max": 55.46784210205078,
      "activations/layer6_attention_weight_min": -60.19303512573242,
      "activations/layer7_attention_weight_max": 93.06375885009766,
      "activations/layer7_attention_weight_min": -85.56365203857422,
      "activations/layer8_attention_weight_max": 41.427276611328125,
      "activations/layer8_attention_weight_min": -40.07603454589844,
      "activations/layer9_attention_weight_max": 36.17582702636719,
      "activations/layer9_attention_weight_min": -42.36879348754883,
      "epoch": 5.12,
      "learning_rate": 0.00011817613636363635,
      "loss": 2.8418,
      "step": 88100
    },
    {
      "activations/layer0_attention_weight_max": 16.532073974609375,
      "activations/layer0_attention_weight_min": -12.929763793945312,
      "activations/layer10_attention_weight_max": 35.11684799194336,
      "activations/layer10_attention_weight_min": -30.45912742614746,
      "activations/layer11_attention_weight_max": 32.043739318847656,
      "activations/layer11_attention_weight_min": -25.99360466003418,
      "activations/layer12_attention_weight_max": 46.08672332763672,
      "activations/layer12_attention_weight_min": -27.11482048034668,
      "activations/layer13_attention_weight_max": 47.23374938964844,
      "activations/layer13_attention_weight_min": -31.509525299072266,
      "activations/layer14_attention_weight_max": 68.14495849609375,
      "activations/layer14_attention_weight_min": -49.52069854736328,
      "activations/layer15_attention_weight_max": 36.55427932739258,
      "activations/layer15_attention_weight_min": -27.9371280670166,
      "activations/layer16_attention_weight_max": 40.02809143066406,
      "activations/layer16_attention_weight_min": -28.91226577758789,
      "activations/layer17_attention_weight_max": 38.0241584777832,
      "activations/layer17_attention_weight_min": -25.27030372619629,
      "activations/layer18_attention_weight_max": 29.35930061340332,
      "activations/layer18_attention_weight_min": -22.4533634185791,
      "activations/layer19_attention_weight_max": 44.55913543701172,
      "activations/layer19_attention_weight_min": -30.813528060913086,
      "activations/layer1_attention_weight_max": 17.46101951599121,
      "activations/layer1_attention_weight_min": -14.397143363952637,
      "activations/layer20_attention_weight_max": 33.34170913696289,
      "activations/layer20_attention_weight_min": -25.691932678222656,
      "activations/layer21_attention_weight_max": 32.669715881347656,
      "activations/layer21_attention_weight_min": -21.30463218688965,
      "activations/layer22_attention_weight_max": 58.68777084350586,
      "activations/layer22_attention_weight_min": -27.32990074157715,
      "activations/layer23_attention_weight_max": 33.42174530029297,
      "activations/layer23_attention_weight_min": -20.276958465576172,
      "activations/layer2_attention_weight_max": 34.88349151611328,
      "activations/layer2_attention_weight_min": -32.057891845703125,
      "activations/layer3_attention_weight_max": 90.13050079345703,
      "activations/layer3_attention_weight_min": -91.81097412109375,
      "activations/layer4_attention_weight_max": 128.47305297851562,
      "activations/layer4_attention_weight_min": -129.10421752929688,
      "activations/layer5_attention_weight_max": 80.62437438964844,
      "activations/layer5_attention_weight_min": -76.2811508178711,
      "activations/layer6_attention_weight_max": 55.10481643676758,
      "activations/layer6_attention_weight_min": -55.57331466674805,
      "activations/layer7_attention_weight_max": 82.8724136352539,
      "activations/layer7_attention_weight_min": -82.57698059082031,
      "activations/layer8_attention_weight_max": 38.51853942871094,
      "activations/layer8_attention_weight_min": -37.579200744628906,
      "activations/layer9_attention_weight_max": 35.79023361206055,
      "activations/layer9_attention_weight_min": -36.69819641113281,
      "epoch": 5.12,
      "learning_rate": 0.00011815719696969697,
      "loss": 2.8453,
      "step": 88150
    },
    {
      "activations/layer0_attention_weight_max": 16.13719367980957,
      "activations/layer0_attention_weight_min": -12.617111206054688,
      "activations/layer10_attention_weight_max": 34.30807113647461,
      "activations/layer10_attention_weight_min": -30.568016052246094,
      "activations/layer11_attention_weight_max": 32.37580108642578,
      "activations/layer11_attention_weight_min": -27.008930206298828,
      "activations/layer12_attention_weight_max": 26.703824996948242,
      "activations/layer12_attention_weight_min": -27.59238624572754,
      "activations/layer13_attention_weight_max": 40.60328674316406,
      "activations/layer13_attention_weight_min": -31.630966186523438,
      "activations/layer14_attention_weight_max": 58.43751525878906,
      "activations/layer14_attention_weight_min": -42.508209228515625,
      "activations/layer15_attention_weight_max": 38.00291442871094,
      "activations/layer15_attention_weight_min": -26.872209548950195,
      "activations/layer16_attention_weight_max": 34.69575881958008,
      "activations/layer16_attention_weight_min": -26.64023780822754,
      "activations/layer17_attention_weight_max": 37.78081130981445,
      "activations/layer17_attention_weight_min": -24.18462562561035,
      "activations/layer18_attention_weight_max": 36.61699676513672,
      "activations/layer18_attention_weight_min": -21.300155639648438,
      "activations/layer19_attention_weight_max": 38.538597106933594,
      "activations/layer19_attention_weight_min": -29.27695655822754,
      "activations/layer1_attention_weight_max": 17.83902931213379,
      "activations/layer1_attention_weight_min": -13.382122039794922,
      "activations/layer20_attention_weight_max": 32.1053352355957,
      "activations/layer20_attention_weight_min": -24.378931045532227,
      "activations/layer21_attention_weight_max": 31.502023696899414,
      "activations/layer21_attention_weight_min": -22.257131576538086,
      "activations/layer22_attention_weight_max": 48.37632369995117,
      "activations/layer22_attention_weight_min": -26.457536697387695,
      "activations/layer23_attention_weight_max": 33.81606674194336,
      "activations/layer23_attention_weight_min": -20.16109275817871,
      "activations/layer2_attention_weight_max": 32.270843505859375,
      "activations/layer2_attention_weight_min": -30.01356315612793,
      "activations/layer3_attention_weight_max": 83.92053985595703,
      "activations/layer3_attention_weight_min": -88.5792236328125,
      "activations/layer4_attention_weight_max": 130.62847900390625,
      "activations/layer4_attention_weight_min": -129.09329223632812,
      "activations/layer5_attention_weight_max": 81.97561645507812,
      "activations/layer5_attention_weight_min": -82.13360595703125,
      "activations/layer6_attention_weight_max": 58.29924774169922,
      "activations/layer6_attention_weight_min": -59.36576461791992,
      "activations/layer7_attention_weight_max": 84.45845794677734,
      "activations/layer7_attention_weight_min": -86.25038146972656,
      "activations/layer8_attention_weight_max": 39.6379280090332,
      "activations/layer8_attention_weight_min": -41.61507034301758,
      "activations/layer9_attention_weight_max": 36.93276596069336,
      "activations/layer9_attention_weight_min": -34.686546325683594,
      "epoch": 5.12,
      "learning_rate": 0.00011813825757575756,
      "loss": 2.8526,
      "step": 88200
    },
    {
      "activations/layer0_attention_weight_max": 16.866653442382812,
      "activations/layer0_attention_weight_min": -11.97882080078125,
      "activations/layer10_attention_weight_max": 32.75153732299805,
      "activations/layer10_attention_weight_min": -29.495479583740234,
      "activations/layer11_attention_weight_max": 30.93938446044922,
      "activations/layer11_attention_weight_min": -26.74049949645996,
      "activations/layer12_attention_weight_max": 27.306119918823242,
      "activations/layer12_attention_weight_min": -28.4180850982666,
      "activations/layer13_attention_weight_max": 45.52581024169922,
      "activations/layer13_attention_weight_min": -34.27113342285156,
      "activations/layer14_attention_weight_max": 63.16170883178711,
      "activations/layer14_attention_weight_min": -47.83082580566406,
      "activations/layer15_attention_weight_max": 42.49858474731445,
      "activations/layer15_attention_weight_min": -27.57195281982422,
      "activations/layer16_attention_weight_max": 39.71042251586914,
      "activations/layer16_attention_weight_min": -28.1650390625,
      "activations/layer17_attention_weight_max": 38.4554328918457,
      "activations/layer17_attention_weight_min": -25.56283187866211,
      "activations/layer18_attention_weight_max": 38.65168380737305,
      "activations/layer18_attention_weight_min": -21.469194412231445,
      "activations/layer19_attention_weight_max": 41.50030517578125,
      "activations/layer19_attention_weight_min": -29.833871841430664,
      "activations/layer1_attention_weight_max": 17.21384048461914,
      "activations/layer1_attention_weight_min": -14.060335159301758,
      "activations/layer20_attention_weight_max": 32.776973724365234,
      "activations/layer20_attention_weight_min": -22.7999324798584,
      "activations/layer21_attention_weight_max": 36.141883850097656,
      "activations/layer21_attention_weight_min": -21.00336456298828,
      "activations/layer22_attention_weight_max": 51.801239013671875,
      "activations/layer22_attention_weight_min": -28.36349868774414,
      "activations/layer23_attention_weight_max": 33.00715637207031,
      "activations/layer23_attention_weight_min": -20.344179153442383,
      "activations/layer2_attention_weight_max": 30.187862396240234,
      "activations/layer2_attention_weight_min": -28.956579208374023,
      "activations/layer3_attention_weight_max": 84.17871856689453,
      "activations/layer3_attention_weight_min": -85.02278137207031,
      "activations/layer4_attention_weight_max": 126.29561614990234,
      "activations/layer4_attention_weight_min": -122.6675033569336,
      "activations/layer5_attention_weight_max": 80.5338134765625,
      "activations/layer5_attention_weight_min": -76.8197250366211,
      "activations/layer6_attention_weight_max": 55.992950439453125,
      "activations/layer6_attention_weight_min": -55.31166076660156,
      "activations/layer7_attention_weight_max": 87.77264404296875,
      "activations/layer7_attention_weight_min": -83.23623657226562,
      "activations/layer8_attention_weight_max": 37.31035232543945,
      "activations/layer8_attention_weight_min": -38.62969207763672,
      "activations/layer9_attention_weight_max": 35.894527435302734,
      "activations/layer9_attention_weight_min": -33.52666473388672,
      "epoch": 5.13,
      "learning_rate": 0.00011811931818181817,
      "loss": 2.8547,
      "step": 88250
    },
    {
      "activations/layer0_attention_weight_max": 17.02235984802246,
      "activations/layer0_attention_weight_min": -11.37637710571289,
      "activations/layer10_attention_weight_max": 32.895877838134766,
      "activations/layer10_attention_weight_min": -30.149370193481445,
      "activations/layer11_attention_weight_max": 35.587066650390625,
      "activations/layer11_attention_weight_min": -27.490402221679688,
      "activations/layer12_attention_weight_max": 36.49452590942383,
      "activations/layer12_attention_weight_min": -27.916858673095703,
      "activations/layer13_attention_weight_max": 50.500343322753906,
      "activations/layer13_attention_weight_min": -33.170745849609375,
      "activations/layer14_attention_weight_max": 85.2324447631836,
      "activations/layer14_attention_weight_min": -56.52595520019531,
      "activations/layer15_attention_weight_max": 49.4475212097168,
      "activations/layer15_attention_weight_min": -29.919300079345703,
      "activations/layer16_attention_weight_max": 44.739990234375,
      "activations/layer16_attention_weight_min": -30.93840217590332,
      "activations/layer17_attention_weight_max": 43.257938385009766,
      "activations/layer17_attention_weight_min": -25.336225509643555,
      "activations/layer18_attention_weight_max": 40.7567253112793,
      "activations/layer18_attention_weight_min": -21.4226016998291,
      "activations/layer19_attention_weight_max": 48.09066390991211,
      "activations/layer19_attention_weight_min": -32.404273986816406,
      "activations/layer1_attention_weight_max": 17.814577102661133,
      "activations/layer1_attention_weight_min": -13.821739196777344,
      "activations/layer20_attention_weight_max": 38.424888610839844,
      "activations/layer20_attention_weight_min": -23.821147918701172,
      "activations/layer21_attention_weight_max": 37.11420822143555,
      "activations/layer21_attention_weight_min": -24.160276412963867,
      "activations/layer22_attention_weight_max": 65.54338073730469,
      "activations/layer22_attention_weight_min": -29.32975959777832,
      "activations/layer23_attention_weight_max": 44.10264587402344,
      "activations/layer23_attention_weight_min": -21.156265258789062,
      "activations/layer2_attention_weight_max": 31.267784118652344,
      "activations/layer2_attention_weight_min": -30.966337203979492,
      "activations/layer3_attention_weight_max": 88.67908477783203,
      "activations/layer3_attention_weight_min": -90.43360137939453,
      "activations/layer4_attention_weight_max": 127.6457290649414,
      "activations/layer4_attention_weight_min": -129.20382690429688,
      "activations/layer5_attention_weight_max": 80.08721923828125,
      "activations/layer5_attention_weight_min": -79.25167083740234,
      "activations/layer6_attention_weight_max": 52.69306945800781,
      "activations/layer6_attention_weight_min": -60.63389205932617,
      "activations/layer7_attention_weight_max": 79.11822509765625,
      "activations/layer7_attention_weight_min": -86.94347381591797,
      "activations/layer8_attention_weight_max": 38.65394592285156,
      "activations/layer8_attention_weight_min": -38.49180221557617,
      "activations/layer9_attention_weight_max": 36.52532958984375,
      "activations/layer9_attention_weight_min": -34.4398193359375,
      "epoch": 5.13,
      "learning_rate": 0.00011810037878787879,
      "loss": 2.8391,
      "step": 88300
    },
    {
      "activations/layer0_attention_weight_max": 16.344545364379883,
      "activations/layer0_attention_weight_min": -12.44363021850586,
      "activations/layer10_attention_weight_max": 35.819786071777344,
      "activations/layer10_attention_weight_min": -30.28879165649414,
      "activations/layer11_attention_weight_max": 32.76306915283203,
      "activations/layer11_attention_weight_min": -27.84333038330078,
      "activations/layer12_attention_weight_max": 31.44373893737793,
      "activations/layer12_attention_weight_min": -31.948802947998047,
      "activations/layer13_attention_weight_max": 58.50988006591797,
      "activations/layer13_attention_weight_min": -45.66752624511719,
      "activations/layer14_attention_weight_max": 74.15650177001953,
      "activations/layer14_attention_weight_min": -56.37639236450195,
      "activations/layer15_attention_weight_max": 49.21235656738281,
      "activations/layer15_attention_weight_min": -34.110572814941406,
      "activations/layer16_attention_weight_max": 38.755062103271484,
      "activations/layer16_attention_weight_min": -26.511600494384766,
      "activations/layer17_attention_weight_max": 40.01472854614258,
      "activations/layer17_attention_weight_min": -24.498966217041016,
      "activations/layer18_attention_weight_max": 31.148460388183594,
      "activations/layer18_attention_weight_min": -23.35906410217285,
      "activations/layer19_attention_weight_max": 36.305747985839844,
      "activations/layer19_attention_weight_min": -27.942487716674805,
      "activations/layer1_attention_weight_max": 20.239856719970703,
      "activations/layer1_attention_weight_min": -13.711431503295898,
      "activations/layer20_attention_weight_max": 30.256994247436523,
      "activations/layer20_attention_weight_min": -23.188711166381836,
      "activations/layer21_attention_weight_max": 25.648448944091797,
      "activations/layer21_attention_weight_min": -18.929412841796875,
      "activations/layer22_attention_weight_max": 46.2476921081543,
      "activations/layer22_attention_weight_min": -28.734378814697266,
      "activations/layer23_attention_weight_max": 30.773412704467773,
      "activations/layer23_attention_weight_min": -18.545337677001953,
      "activations/layer2_attention_weight_max": 35.2645263671875,
      "activations/layer2_attention_weight_min": -32.48692321777344,
      "activations/layer3_attention_weight_max": 97.49217224121094,
      "activations/layer3_attention_weight_min": -95.51065826416016,
      "activations/layer4_attention_weight_max": 138.97901916503906,
      "activations/layer4_attention_weight_min": -129.85379028320312,
      "activations/layer5_attention_weight_max": 88.79344177246094,
      "activations/layer5_attention_weight_min": -79.61219024658203,
      "activations/layer6_attention_weight_max": 58.57941436767578,
      "activations/layer6_attention_weight_min": -58.61805725097656,
      "activations/layer7_attention_weight_max": 90.8966293334961,
      "activations/layer7_attention_weight_min": -85.33881378173828,
      "activations/layer8_attention_weight_max": 44.587867736816406,
      "activations/layer8_attention_weight_min": -39.619483947753906,
      "activations/layer9_attention_weight_max": 39.60268783569336,
      "activations/layer9_attention_weight_min": -37.318721771240234,
      "epoch": 5.13,
      "learning_rate": 0.00011808143939393937,
      "loss": 2.8422,
      "step": 88350
    },
    {
      "activations/layer0_attention_weight_max": 16.331958770751953,
      "activations/layer0_attention_weight_min": -11.4457426071167,
      "activations/layer10_attention_weight_max": 37.349754333496094,
      "activations/layer10_attention_weight_min": -30.494232177734375,
      "activations/layer11_attention_weight_max": 34.76184844970703,
      "activations/layer11_attention_weight_min": -28.26673698425293,
      "activations/layer12_attention_weight_max": 28.209131240844727,
      "activations/layer12_attention_weight_min": -30.811519622802734,
      "activations/layer13_attention_weight_max": 44.695579528808594,
      "activations/layer13_attention_weight_min": -32.745887756347656,
      "activations/layer14_attention_weight_max": 60.42328643798828,
      "activations/layer14_attention_weight_min": -41.7454719543457,
      "activations/layer15_attention_weight_max": 40.27467346191406,
      "activations/layer15_attention_weight_min": -28.219585418701172,
      "activations/layer16_attention_weight_max": 35.95687484741211,
      "activations/layer16_attention_weight_min": -27.77482795715332,
      "activations/layer17_attention_weight_max": 40.61393356323242,
      "activations/layer17_attention_weight_min": -24.31563949584961,
      "activations/layer18_attention_weight_max": 28.932819366455078,
      "activations/layer18_attention_weight_min": -20.54941749572754,
      "activations/layer19_attention_weight_max": 40.0187873840332,
      "activations/layer19_attention_weight_min": -31.25188636779785,
      "activations/layer1_attention_weight_max": 17.517892837524414,
      "activations/layer1_attention_weight_min": -13.766777992248535,
      "activations/layer20_attention_weight_max": 35.86602783203125,
      "activations/layer20_attention_weight_min": -25.46480941772461,
      "activations/layer21_attention_weight_max": 29.296646118164062,
      "activations/layer21_attention_weight_min": -20.821361541748047,
      "activations/layer22_attention_weight_max": 50.0977668762207,
      "activations/layer22_attention_weight_min": -28.03524398803711,
      "activations/layer23_attention_weight_max": 34.451019287109375,
      "activations/layer23_attention_weight_min": -20.376625061035156,
      "activations/layer2_attention_weight_max": 32.67935562133789,
      "activations/layer2_attention_weight_min": -31.67703628540039,
      "activations/layer3_attention_weight_max": 91.29396057128906,
      "activations/layer3_attention_weight_min": -96.19242095947266,
      "activations/layer4_attention_weight_max": 130.7154998779297,
      "activations/layer4_attention_weight_min": -130.0462646484375,
      "activations/layer5_attention_weight_max": 85.52105712890625,
      "activations/layer5_attention_weight_min": -81.53636169433594,
      "activations/layer6_attention_weight_max": 55.364688873291016,
      "activations/layer6_attention_weight_min": -64.03904724121094,
      "activations/layer7_attention_weight_max": 84.2729721069336,
      "activations/layer7_attention_weight_min": -86.89508819580078,
      "activations/layer8_attention_weight_max": 40.53715133666992,
      "activations/layer8_attention_weight_min": -40.8131103515625,
      "activations/layer9_attention_weight_max": 40.521846771240234,
      "activations/layer9_attention_weight_min": -34.777591705322266,
      "epoch": 5.14,
      "learning_rate": 0.00011806249999999999,
      "loss": 2.8469,
      "step": 88400
    },
    {
      "activations/layer0_attention_weight_max": 17.93673324584961,
      "activations/layer0_attention_weight_min": -11.642603874206543,
      "activations/layer10_attention_weight_max": 39.46099853515625,
      "activations/layer10_attention_weight_min": -32.17323684692383,
      "activations/layer11_attention_weight_max": 33.82597732543945,
      "activations/layer11_attention_weight_min": -30.12474822998047,
      "activations/layer12_attention_weight_max": 27.538694381713867,
      "activations/layer12_attention_weight_min": -25.859729766845703,
      "activations/layer13_attention_weight_max": 46.75596237182617,
      "activations/layer13_attention_weight_min": -36.138328552246094,
      "activations/layer14_attention_weight_max": 62.513919830322266,
      "activations/layer14_attention_weight_min": -45.947601318359375,
      "activations/layer15_attention_weight_max": 40.265933990478516,
      "activations/layer15_attention_weight_min": -29.273426055908203,
      "activations/layer16_attention_weight_max": 36.256473541259766,
      "activations/layer16_attention_weight_min": -28.06023406982422,
      "activations/layer17_attention_weight_max": 38.01286697387695,
      "activations/layer17_attention_weight_min": -26.125041961669922,
      "activations/layer18_attention_weight_max": 33.73588562011719,
      "activations/layer18_attention_weight_min": -21.20988655090332,
      "activations/layer19_attention_weight_max": 38.09958267211914,
      "activations/layer19_attention_weight_min": -31.631439208984375,
      "activations/layer1_attention_weight_max": 17.69571304321289,
      "activations/layer1_attention_weight_min": -15.28671932220459,
      "activations/layer20_attention_weight_max": 31.148059844970703,
      "activations/layer20_attention_weight_min": -26.063459396362305,
      "activations/layer21_attention_weight_max": 34.72315216064453,
      "activations/layer21_attention_weight_min": -21.95693016052246,
      "activations/layer22_attention_weight_max": 49.115257263183594,
      "activations/layer22_attention_weight_min": -32.219329833984375,
      "activations/layer23_attention_weight_max": 33.44683837890625,
      "activations/layer23_attention_weight_min": -21.53595542907715,
      "activations/layer2_attention_weight_max": 32.9652099609375,
      "activations/layer2_attention_weight_min": -31.898468017578125,
      "activations/layer3_attention_weight_max": 90.75975799560547,
      "activations/layer3_attention_weight_min": -88.93305206298828,
      "activations/layer4_attention_weight_max": 135.49057006835938,
      "activations/layer4_attention_weight_min": -131.41238403320312,
      "activations/layer5_attention_weight_max": 88.11801147460938,
      "activations/layer5_attention_weight_min": -84.02983093261719,
      "activations/layer6_attention_weight_max": 57.94614791870117,
      "activations/layer6_attention_weight_min": -58.43355178833008,
      "activations/layer7_attention_weight_max": 85.45298767089844,
      "activations/layer7_attention_weight_min": -87.7135009765625,
      "activations/layer8_attention_weight_max": 43.97531509399414,
      "activations/layer8_attention_weight_min": -41.89705276489258,
      "activations/layer9_attention_weight_max": 39.27199172973633,
      "activations/layer9_attention_weight_min": -34.40996170043945,
      "epoch": 5.14,
      "learning_rate": 0.00011804356060606059,
      "loss": 2.8542,
      "step": 88450
    },
    {
      "activations/layer0_attention_weight_max": 16.7340030670166,
      "activations/layer0_attention_weight_min": -11.997855186462402,
      "activations/layer10_attention_weight_max": 32.683841705322266,
      "activations/layer10_attention_weight_min": -30.34477424621582,
      "activations/layer11_attention_weight_max": 32.332279205322266,
      "activations/layer11_attention_weight_min": -28.25888442993164,
      "activations/layer12_attention_weight_max": 25.24541664123535,
      "activations/layer12_attention_weight_min": -30.463905334472656,
      "activations/layer13_attention_weight_max": 39.90388870239258,
      "activations/layer13_attention_weight_min": -29.901714324951172,
      "activations/layer14_attention_weight_max": 54.65949249267578,
      "activations/layer14_attention_weight_min": -44.446746826171875,
      "activations/layer15_attention_weight_max": 37.28533935546875,
      "activations/layer15_attention_weight_min": -26.50380516052246,
      "activations/layer16_attention_weight_max": 36.07551574707031,
      "activations/layer16_attention_weight_min": -27.261451721191406,
      "activations/layer17_attention_weight_max": 38.12020492553711,
      "activations/layer17_attention_weight_min": -22.744760513305664,
      "activations/layer18_attention_weight_max": 32.229713439941406,
      "activations/layer18_attention_weight_min": -17.95884895324707,
      "activations/layer19_attention_weight_max": 36.697608947753906,
      "activations/layer19_attention_weight_min": -28.181961059570312,
      "activations/layer1_attention_weight_max": 17.988933563232422,
      "activations/layer1_attention_weight_min": -14.916532516479492,
      "activations/layer20_attention_weight_max": 30.81026268005371,
      "activations/layer20_attention_weight_min": -21.67415428161621,
      "activations/layer21_attention_weight_max": 30.29568862915039,
      "activations/layer21_attention_weight_min": -18.912750244140625,
      "activations/layer22_attention_weight_max": 46.5760498046875,
      "activations/layer22_attention_weight_min": -26.6933650970459,
      "activations/layer23_attention_weight_max": 31.843303680419922,
      "activations/layer23_attention_weight_min": -18.95960807800293,
      "activations/layer2_attention_weight_max": 31.49837303161621,
      "activations/layer2_attention_weight_min": -31.363948822021484,
      "activations/layer3_attention_weight_max": 88.433837890625,
      "activations/layer3_attention_weight_min": -88.43877410888672,
      "activations/layer4_attention_weight_max": 125.81048583984375,
      "activations/layer4_attention_weight_min": -124.87591552734375,
      "activations/layer5_attention_weight_max": 76.91287994384766,
      "activations/layer5_attention_weight_min": -76.52095794677734,
      "activations/layer6_attention_weight_max": 55.94704055786133,
      "activations/layer6_attention_weight_min": -53.300880432128906,
      "activations/layer7_attention_weight_max": 84.7774887084961,
      "activations/layer7_attention_weight_min": -85.0288314819336,
      "activations/layer8_attention_weight_max": 40.509368896484375,
      "activations/layer8_attention_weight_min": -40.01238250732422,
      "activations/layer9_attention_weight_max": 36.2135009765625,
      "activations/layer9_attention_weight_min": -34.15033721923828,
      "epoch": 5.14,
      "learning_rate": 0.0001180246212121212,
      "loss": 2.8693,
      "step": 88500
    },
    {
      "activations/layer0_attention_weight_max": 16.739282608032227,
      "activations/layer0_attention_weight_min": -12.283843040466309,
      "activations/layer10_attention_weight_max": 34.46925354003906,
      "activations/layer10_attention_weight_min": -31.888151168823242,
      "activations/layer11_attention_weight_max": 33.15007019042969,
      "activations/layer11_attention_weight_min": -29.310537338256836,
      "activations/layer12_attention_weight_max": 29.28562355041504,
      "activations/layer12_attention_weight_min": -26.897323608398438,
      "activations/layer13_attention_weight_max": 48.45908737182617,
      "activations/layer13_attention_weight_min": -33.51280212402344,
      "activations/layer14_attention_weight_max": 71.30004119873047,
      "activations/layer14_attention_weight_min": -46.699798583984375,
      "activations/layer15_attention_weight_max": 42.014381408691406,
      "activations/layer15_attention_weight_min": -32.36477279663086,
      "activations/layer16_attention_weight_max": 40.30811309814453,
      "activations/layer16_attention_weight_min": -28.083600997924805,
      "activations/layer17_attention_weight_max": 48.54513168334961,
      "activations/layer17_attention_weight_min": -25.124225616455078,
      "activations/layer18_attention_weight_max": 35.95280838012695,
      "activations/layer18_attention_weight_min": -21.70420265197754,
      "activations/layer19_attention_weight_max": 45.347557067871094,
      "activations/layer19_attention_weight_min": -29.280996322631836,
      "activations/layer1_attention_weight_max": 17.011699676513672,
      "activations/layer1_attention_weight_min": -14.608562469482422,
      "activations/layer20_attention_weight_max": 39.50841522216797,
      "activations/layer20_attention_weight_min": -22.101276397705078,
      "activations/layer21_attention_weight_max": 37.90819549560547,
      "activations/layer21_attention_weight_min": -19.3377742767334,
      "activations/layer22_attention_weight_max": 60.328125,
      "activations/layer22_attention_weight_min": -27.30392074584961,
      "activations/layer23_attention_weight_max": 38.0709228515625,
      "activations/layer23_attention_weight_min": -20.699230194091797,
      "activations/layer2_attention_weight_max": 32.3573112487793,
      "activations/layer2_attention_weight_min": -32.24744415283203,
      "activations/layer3_attention_weight_max": 91.31114196777344,
      "activations/layer3_attention_weight_min": -92.80113220214844,
      "activations/layer4_attention_weight_max": 132.9807586669922,
      "activations/layer4_attention_weight_min": -135.0709686279297,
      "activations/layer5_attention_weight_max": 80.29668426513672,
      "activations/layer5_attention_weight_min": -79.50929260253906,
      "activations/layer6_attention_weight_max": 53.534950256347656,
      "activations/layer6_attention_weight_min": -55.80598449707031,
      "activations/layer7_attention_weight_max": 80.94358825683594,
      "activations/layer7_attention_weight_min": -83.64026641845703,
      "activations/layer8_attention_weight_max": 39.02774429321289,
      "activations/layer8_attention_weight_min": -39.23386001586914,
      "activations/layer9_attention_weight_max": 38.687110900878906,
      "activations/layer9_attention_weight_min": -33.150760650634766,
      "epoch": 5.15,
      "learning_rate": 0.00011800568181818181,
      "loss": 2.8396,
      "step": 88550
    },
    {
      "activations/layer0_attention_weight_max": 17.042097091674805,
      "activations/layer0_attention_weight_min": -12.140565872192383,
      "activations/layer10_attention_weight_max": 38.39933776855469,
      "activations/layer10_attention_weight_min": -32.702720642089844,
      "activations/layer11_attention_weight_max": 34.74488067626953,
      "activations/layer11_attention_weight_min": -27.816150665283203,
      "activations/layer12_attention_weight_max": 40.137489318847656,
      "activations/layer12_attention_weight_min": -29.821979522705078,
      "activations/layer13_attention_weight_max": 54.88047790527344,
      "activations/layer13_attention_weight_min": -43.89234924316406,
      "activations/layer14_attention_weight_max": 61.71487808227539,
      "activations/layer14_attention_weight_min": -44.538185119628906,
      "activations/layer15_attention_weight_max": 40.47562789916992,
      "activations/layer15_attention_weight_min": -28.31189727783203,
      "activations/layer16_attention_weight_max": 42.52593231201172,
      "activations/layer16_attention_weight_min": -28.689287185668945,
      "activations/layer17_attention_weight_max": 38.193119049072266,
      "activations/layer17_attention_weight_min": -25.48223304748535,
      "activations/layer18_attention_weight_max": 35.83760070800781,
      "activations/layer18_attention_weight_min": -20.452877044677734,
      "activations/layer19_attention_weight_max": 41.72665786743164,
      "activations/layer19_attention_weight_min": -28.9480037689209,
      "activations/layer1_attention_weight_max": 16.203968048095703,
      "activations/layer1_attention_weight_min": -13.896121978759766,
      "activations/layer20_attention_weight_max": 30.816375732421875,
      "activations/layer20_attention_weight_min": -23.70743751525879,
      "activations/layer21_attention_weight_max": 33.0909538269043,
      "activations/layer21_attention_weight_min": -20.53958511352539,
      "activations/layer22_attention_weight_max": 50.02693557739258,
      "activations/layer22_attention_weight_min": -28.42989730834961,
      "activations/layer23_attention_weight_max": 33.28709030151367,
      "activations/layer23_attention_weight_min": -20.604877471923828,
      "activations/layer2_attention_weight_max": 33.71664047241211,
      "activations/layer2_attention_weight_min": -33.61125183105469,
      "activations/layer3_attention_weight_max": 93.5670166015625,
      "activations/layer3_attention_weight_min": -96.7203140258789,
      "activations/layer4_attention_weight_max": 136.23501586914062,
      "activations/layer4_attention_weight_min": -136.42117309570312,
      "activations/layer5_attention_weight_max": 84.91551971435547,
      "activations/layer5_attention_weight_min": -83.73591613769531,
      "activations/layer6_attention_weight_max": 56.940650939941406,
      "activations/layer6_attention_weight_min": -56.42578887939453,
      "activations/layer7_attention_weight_max": 84.84827423095703,
      "activations/layer7_attention_weight_min": -87.80659484863281,
      "activations/layer8_attention_weight_max": 44.59363555908203,
      "activations/layer8_attention_weight_min": -40.08559036254883,
      "activations/layer9_attention_weight_max": 42.36085510253906,
      "activations/layer9_attention_weight_min": -35.80683517456055,
      "epoch": 5.15,
      "learning_rate": 0.00011798674242424241,
      "loss": 2.8556,
      "step": 88600
    },
    {
      "activations/layer0_attention_weight_max": 17.053762435913086,
      "activations/layer0_attention_weight_min": -12.096378326416016,
      "activations/layer10_attention_weight_max": 36.37175369262695,
      "activations/layer10_attention_weight_min": -31.31216812133789,
      "activations/layer11_attention_weight_max": 38.08882141113281,
      "activations/layer11_attention_weight_min": -28.575117111206055,
      "activations/layer12_attention_weight_max": 68.0810775756836,
      "activations/layer12_attention_weight_min": -28.093568801879883,
      "activations/layer13_attention_weight_max": 75.79725646972656,
      "activations/layer13_attention_weight_min": -39.1766471862793,
      "activations/layer14_attention_weight_max": 83.30064392089844,
      "activations/layer14_attention_weight_min": -46.86607360839844,
      "activations/layer15_attention_weight_max": 54.21389389038086,
      "activations/layer15_attention_weight_min": -29.82159423828125,
      "activations/layer16_attention_weight_max": 52.332275390625,
      "activations/layer16_attention_weight_min": -28.93367576599121,
      "activations/layer17_attention_weight_max": 45.67253112792969,
      "activations/layer17_attention_weight_min": -26.149593353271484,
      "activations/layer18_attention_weight_max": 36.506492614746094,
      "activations/layer18_attention_weight_min": -21.013656616210938,
      "activations/layer19_attention_weight_max": 59.8304328918457,
      "activations/layer19_attention_weight_min": -28.703536987304688,
      "activations/layer1_attention_weight_max": 18.89598846435547,
      "activations/layer1_attention_weight_min": -15.79576301574707,
      "activations/layer20_attention_weight_max": 39.616573333740234,
      "activations/layer20_attention_weight_min": -22.2371768951416,
      "activations/layer21_attention_weight_max": 33.04941940307617,
      "activations/layer21_attention_weight_min": -19.907146453857422,
      "activations/layer22_attention_weight_max": 63.76166534423828,
      "activations/layer22_attention_weight_min": -26.18548011779785,
      "activations/layer23_attention_weight_max": 42.560455322265625,
      "activations/layer23_attention_weight_min": -18.776893615722656,
      "activations/layer2_attention_weight_max": 32.86391830444336,
      "activations/layer2_attention_weight_min": -31.57662010192871,
      "activations/layer3_attention_weight_max": 91.19851684570312,
      "activations/layer3_attention_weight_min": -91.96389770507812,
      "activations/layer4_attention_weight_max": 135.406982421875,
      "activations/layer4_attention_weight_min": -128.02389526367188,
      "activations/layer5_attention_weight_max": 79.35874938964844,
      "activations/layer5_attention_weight_min": -76.4428482055664,
      "activations/layer6_attention_weight_max": 54.96909713745117,
      "activations/layer6_attention_weight_min": -54.99528884887695,
      "activations/layer7_attention_weight_max": 81.25836181640625,
      "activations/layer7_attention_weight_min": -87.65467834472656,
      "activations/layer8_attention_weight_max": 41.53059768676758,
      "activations/layer8_attention_weight_min": -40.606754302978516,
      "activations/layer9_attention_weight_max": 38.08930587768555,
      "activations/layer9_attention_weight_min": -32.90937423706055,
      "epoch": 5.15,
      "learning_rate": 0.00011796780303030302,
      "loss": 2.8435,
      "step": 88650
    },
    {
      "activations/layer0_attention_weight_max": 15.772926330566406,
      "activations/layer0_attention_weight_min": -12.531396865844727,
      "activations/layer10_attention_weight_max": 32.46430969238281,
      "activations/layer10_attention_weight_min": -31.195384979248047,
      "activations/layer11_attention_weight_max": 31.6806697845459,
      "activations/layer11_attention_weight_min": -28.03603744506836,
      "activations/layer12_attention_weight_max": 30.990663528442383,
      "activations/layer12_attention_weight_min": -26.268400192260742,
      "activations/layer13_attention_weight_max": 49.38734436035156,
      "activations/layer13_attention_weight_min": -37.0718879699707,
      "activations/layer14_attention_weight_max": 62.116676330566406,
      "activations/layer14_attention_weight_min": -44.25811767578125,
      "activations/layer15_attention_weight_max": 37.2734260559082,
      "activations/layer15_attention_weight_min": -27.635440826416016,
      "activations/layer16_attention_weight_max": 37.41659927368164,
      "activations/layer16_attention_weight_min": -29.027902603149414,
      "activations/layer17_attention_weight_max": 38.77388381958008,
      "activations/layer17_attention_weight_min": -23.675451278686523,
      "activations/layer18_attention_weight_max": 31.398210525512695,
      "activations/layer18_attention_weight_min": -19.6522216796875,
      "activations/layer19_attention_weight_max": 39.03555679321289,
      "activations/layer19_attention_weight_min": -28.403614044189453,
      "activations/layer1_attention_weight_max": 17.650001525878906,
      "activations/layer1_attention_weight_min": -14.249876976013184,
      "activations/layer20_attention_weight_max": 32.67256546020508,
      "activations/layer20_attention_weight_min": -24.14797592163086,
      "activations/layer21_attention_weight_max": 34.5267333984375,
      "activations/layer21_attention_weight_min": -20.99005699157715,
      "activations/layer22_attention_weight_max": 49.3533935546875,
      "activations/layer22_attention_weight_min": -28.471940994262695,
      "activations/layer23_attention_weight_max": 33.177467346191406,
      "activations/layer23_attention_weight_min": -21.29363250732422,
      "activations/layer2_attention_weight_max": 30.745731353759766,
      "activations/layer2_attention_weight_min": -30.172897338867188,
      "activations/layer3_attention_weight_max": 84.20558166503906,
      "activations/layer3_attention_weight_min": -88.44805908203125,
      "activations/layer4_attention_weight_max": 127.15621185302734,
      "activations/layer4_attention_weight_min": -124.64836883544922,
      "activations/layer5_attention_weight_max": 78.45819854736328,
      "activations/layer5_attention_weight_min": -75.88887023925781,
      "activations/layer6_attention_weight_max": 55.84354019165039,
      "activations/layer6_attention_weight_min": -53.29875946044922,
      "activations/layer7_attention_weight_max": 85.25627136230469,
      "activations/layer7_attention_weight_min": -80.9209213256836,
      "activations/layer8_attention_weight_max": 41.999107360839844,
      "activations/layer8_attention_weight_min": -35.246463775634766,
      "activations/layer9_attention_weight_max": 34.750736236572266,
      "activations/layer9_attention_weight_min": -33.55869674682617,
      "epoch": 5.15,
      "learning_rate": 0.00011794886363636364,
      "loss": 2.8432,
      "step": 88700
    },
    {
      "activations/layer0_attention_weight_max": 16.728490829467773,
      "activations/layer0_attention_weight_min": -11.341568946838379,
      "activations/layer10_attention_weight_max": 33.941654205322266,
      "activations/layer10_attention_weight_min": -30.577665328979492,
      "activations/layer11_attention_weight_max": 33.983497619628906,
      "activations/layer11_attention_weight_min": -28.72804069519043,
      "activations/layer12_attention_weight_max": 36.92121505737305,
      "activations/layer12_attention_weight_min": -27.00200843811035,
      "activations/layer13_attention_weight_max": 48.3232307434082,
      "activations/layer13_attention_weight_min": -31.430721282958984,
      "activations/layer14_attention_weight_max": 70.92446899414062,
      "activations/layer14_attention_weight_min": -42.87792205810547,
      "activations/layer15_attention_weight_max": 42.831626892089844,
      "activations/layer15_attention_weight_min": -27.640348434448242,
      "activations/layer16_attention_weight_max": 41.14063262939453,
      "activations/layer16_attention_weight_min": -27.95899200439453,
      "activations/layer17_attention_weight_max": 43.11237335205078,
      "activations/layer17_attention_weight_min": -23.89610481262207,
      "activations/layer18_attention_weight_max": 32.799957275390625,
      "activations/layer18_attention_weight_min": -19.520706176757812,
      "activations/layer19_attention_weight_max": 46.51913070678711,
      "activations/layer19_attention_weight_min": -30.077253341674805,
      "activations/layer1_attention_weight_max": 17.33488655090332,
      "activations/layer1_attention_weight_min": -13.450990676879883,
      "activations/layer20_attention_weight_max": 36.2132682800293,
      "activations/layer20_attention_weight_min": -24.789228439331055,
      "activations/layer21_attention_weight_max": 32.9774055480957,
      "activations/layer21_attention_weight_min": -20.806730270385742,
      "activations/layer22_attention_weight_max": 52.18461227416992,
      "activations/layer22_attention_weight_min": -26.621091842651367,
      "activations/layer23_attention_weight_max": 33.85879898071289,
      "activations/layer23_attention_weight_min": -20.149784088134766,
      "activations/layer2_attention_weight_max": 31.374446868896484,
      "activations/layer2_attention_weight_min": -30.30121612548828,
      "activations/layer3_attention_weight_max": 87.31171417236328,
      "activations/layer3_attention_weight_min": -88.87258911132812,
      "activations/layer4_attention_weight_max": 124.75328826904297,
      "activations/layer4_attention_weight_min": -121.3654556274414,
      "activations/layer5_attention_weight_max": 78.04492950439453,
      "activations/layer5_attention_weight_min": -73.73363494873047,
      "activations/layer6_attention_weight_max": 55.440433502197266,
      "activations/layer6_attention_weight_min": -53.50160598754883,
      "activations/layer7_attention_weight_max": 80.7444839477539,
      "activations/layer7_attention_weight_min": -82.16802215576172,
      "activations/layer8_attention_weight_max": 38.9450569152832,
      "activations/layer8_attention_weight_min": -39.12373733520508,
      "activations/layer9_attention_weight_max": 36.57347106933594,
      "activations/layer9_attention_weight_min": -34.143898010253906,
      "epoch": 5.16,
      "learning_rate": 0.00011792992424242423,
      "loss": 2.8317,
      "step": 88750
    },
    {
      "activations/layer0_attention_weight_max": 16.37128257751465,
      "activations/layer0_attention_weight_min": -12.705473899841309,
      "activations/layer10_attention_weight_max": 34.38874816894531,
      "activations/layer10_attention_weight_min": -30.013151168823242,
      "activations/layer11_attention_weight_max": 31.14149284362793,
      "activations/layer11_attention_weight_min": -27.092411041259766,
      "activations/layer12_attention_weight_max": 30.315507888793945,
      "activations/layer12_attention_weight_min": -27.38680648803711,
      "activations/layer13_attention_weight_max": 41.71734619140625,
      "activations/layer13_attention_weight_min": -32.40620422363281,
      "activations/layer14_attention_weight_max": 62.56952667236328,
      "activations/layer14_attention_weight_min": -47.20229721069336,
      "activations/layer15_attention_weight_max": 36.73464584350586,
      "activations/layer15_attention_weight_min": -29.189321517944336,
      "activations/layer16_attention_weight_max": 36.275672912597656,
      "activations/layer16_attention_weight_min": -28.338687896728516,
      "activations/layer17_attention_weight_max": 36.07270431518555,
      "activations/layer17_attention_weight_min": -24.409099578857422,
      "activations/layer18_attention_weight_max": 30.495576858520508,
      "activations/layer18_attention_weight_min": -20.169677734375,
      "activations/layer19_attention_weight_max": 42.2173957824707,
      "activations/layer19_attention_weight_min": -30.744253158569336,
      "activations/layer1_attention_weight_max": 16.279747009277344,
      "activations/layer1_attention_weight_min": -14.64315414428711,
      "activations/layer20_attention_weight_max": 31.88812828063965,
      "activations/layer20_attention_weight_min": -23.818403244018555,
      "activations/layer21_attention_weight_max": 32.226165771484375,
      "activations/layer21_attention_weight_min": -23.25356674194336,
      "activations/layer22_attention_weight_max": 54.854801177978516,
      "activations/layer22_attention_weight_min": -28.88016128540039,
      "activations/layer23_attention_weight_max": 33.745235443115234,
      "activations/layer23_attention_weight_min": -22.102157592773438,
      "activations/layer2_attention_weight_max": 33.60431671142578,
      "activations/layer2_attention_weight_min": -32.37382507324219,
      "activations/layer3_attention_weight_max": 91.8985595703125,
      "activations/layer3_attention_weight_min": -91.7457275390625,
      "activations/layer4_attention_weight_max": 133.90545654296875,
      "activations/layer4_attention_weight_min": -127.16117095947266,
      "activations/layer5_attention_weight_max": 80.12068939208984,
      "activations/layer5_attention_weight_min": -77.33627319335938,
      "activations/layer6_attention_weight_max": 55.378570556640625,
      "activations/layer6_attention_weight_min": -55.17875289916992,
      "activations/layer7_attention_weight_max": 87.2806625366211,
      "activations/layer7_attention_weight_min": -92.37281799316406,
      "activations/layer8_attention_weight_max": 38.938533782958984,
      "activations/layer8_attention_weight_min": -39.03927993774414,
      "activations/layer9_attention_weight_max": 37.384178161621094,
      "activations/layer9_attention_weight_min": -34.177398681640625,
      "epoch": 5.16,
      "learning_rate": 0.00011791098484848484,
      "loss": 2.8613,
      "step": 88800
    },
    {
      "activations/layer0_attention_weight_max": 15.465296745300293,
      "activations/layer0_attention_weight_min": -12.840548515319824,
      "activations/layer10_attention_weight_max": 35.96044921875,
      "activations/layer10_attention_weight_min": -32.45551300048828,
      "activations/layer11_attention_weight_max": 36.396888732910156,
      "activations/layer11_attention_weight_min": -28.044437408447266,
      "activations/layer12_attention_weight_max": 30.275346755981445,
      "activations/layer12_attention_weight_min": -25.91782569885254,
      "activations/layer13_attention_weight_max": 47.16948318481445,
      "activations/layer13_attention_weight_min": -34.049713134765625,
      "activations/layer14_attention_weight_max": 65.51986694335938,
      "activations/layer14_attention_weight_min": -54.119632720947266,
      "activations/layer15_attention_weight_max": 40.628746032714844,
      "activations/layer15_attention_weight_min": -28.07658576965332,
      "activations/layer16_attention_weight_max": 37.80979919433594,
      "activations/layer16_attention_weight_min": -27.082956314086914,
      "activations/layer17_attention_weight_max": 40.52791213989258,
      "activations/layer17_attention_weight_min": -23.754884719848633,
      "activations/layer18_attention_weight_max": 31.608373641967773,
      "activations/layer18_attention_weight_min": -20.10472297668457,
      "activations/layer19_attention_weight_max": 42.36710739135742,
      "activations/layer19_attention_weight_min": -29.749149322509766,
      "activations/layer1_attention_weight_max": 19.474348068237305,
      "activations/layer1_attention_weight_min": -14.902151107788086,
      "activations/layer20_attention_weight_max": 35.05812072753906,
      "activations/layer20_attention_weight_min": -23.090213775634766,
      "activations/layer21_attention_weight_max": 33.14014434814453,
      "activations/layer21_attention_weight_min": -20.240633010864258,
      "activations/layer22_attention_weight_max": 55.74424362182617,
      "activations/layer22_attention_weight_min": -25.691356658935547,
      "activations/layer23_attention_weight_max": 37.43988800048828,
      "activations/layer23_attention_weight_min": -19.515106201171875,
      "activations/layer2_attention_weight_max": 33.121315002441406,
      "activations/layer2_attention_weight_min": -32.02265167236328,
      "activations/layer3_attention_weight_max": 96.1968002319336,
      "activations/layer3_attention_weight_min": -95.12641143798828,
      "activations/layer4_attention_weight_max": 143.3738555908203,
      "activations/layer4_attention_weight_min": -138.41583251953125,
      "activations/layer5_attention_weight_max": 89.14349365234375,
      "activations/layer5_attention_weight_min": -81.92820739746094,
      "activations/layer6_attention_weight_max": 59.7419319152832,
      "activations/layer6_attention_weight_min": -56.55600357055664,
      "activations/layer7_attention_weight_max": 92.86329650878906,
      "activations/layer7_attention_weight_min": -90.37303161621094,
      "activations/layer8_attention_weight_max": 42.48495101928711,
      "activations/layer8_attention_weight_min": -41.2552490234375,
      "activations/layer9_attention_weight_max": 37.79169845581055,
      "activations/layer9_attention_weight_min": -34.39414596557617,
      "epoch": 5.16,
      "learning_rate": 0.00011789204545454543,
      "loss": 2.866,
      "step": 88850
    },
    {
      "activations/layer0_attention_weight_max": 17.204267501831055,
      "activations/layer0_attention_weight_min": -11.540058135986328,
      "activations/layer10_attention_weight_max": 35.5448112487793,
      "activations/layer10_attention_weight_min": -29.880435943603516,
      "activations/layer11_attention_weight_max": 35.206520080566406,
      "activations/layer11_attention_weight_min": -26.22406578063965,
      "activations/layer12_attention_weight_max": 38.77237319946289,
      "activations/layer12_attention_weight_min": -27.87296485900879,
      "activations/layer13_attention_weight_max": 44.945987701416016,
      "activations/layer13_attention_weight_min": -32.0654411315918,
      "activations/layer14_attention_weight_max": 62.5595588684082,
      "activations/layer14_attention_weight_min": -44.138519287109375,
      "activations/layer15_attention_weight_max": 45.747047424316406,
      "activations/layer15_attention_weight_min": -28.202482223510742,
      "activations/layer16_attention_weight_max": 40.272159576416016,
      "activations/layer16_attention_weight_min": -26.554412841796875,
      "activations/layer17_attention_weight_max": 41.43193817138672,
      "activations/layer17_attention_weight_min": -23.419755935668945,
      "activations/layer18_attention_weight_max": 36.8531379699707,
      "activations/layer18_attention_weight_min": -23.0418758392334,
      "activations/layer19_attention_weight_max": 43.75373077392578,
      "activations/layer19_attention_weight_min": -29.727842330932617,
      "activations/layer1_attention_weight_max": 17.35869026184082,
      "activations/layer1_attention_weight_min": -15.236922264099121,
      "activations/layer20_attention_weight_max": 35.884178161621094,
      "activations/layer20_attention_weight_min": -25.3898983001709,
      "activations/layer21_attention_weight_max": 33.50592803955078,
      "activations/layer21_attention_weight_min": -20.24126434326172,
      "activations/layer22_attention_weight_max": 56.461814880371094,
      "activations/layer22_attention_weight_min": -29.83510398864746,
      "activations/layer23_attention_weight_max": 37.366668701171875,
      "activations/layer23_attention_weight_min": -22.059070587158203,
      "activations/layer2_attention_weight_max": 32.146121978759766,
      "activations/layer2_attention_weight_min": -30.821287155151367,
      "activations/layer3_attention_weight_max": 87.87387084960938,
      "activations/layer3_attention_weight_min": -92.6202621459961,
      "activations/layer4_attention_weight_max": 130.63082885742188,
      "activations/layer4_attention_weight_min": -127.2123794555664,
      "activations/layer5_attention_weight_max": 81.42466735839844,
      "activations/layer5_attention_weight_min": -73.33808135986328,
      "activations/layer6_attention_weight_max": 56.373504638671875,
      "activations/layer6_attention_weight_min": -54.67544937133789,
      "activations/layer7_attention_weight_max": 85.36628723144531,
      "activations/layer7_attention_weight_min": -81.03176879882812,
      "activations/layer8_attention_weight_max": 42.16189956665039,
      "activations/layer8_attention_weight_min": -41.6387825012207,
      "activations/layer9_attention_weight_max": 36.068965911865234,
      "activations/layer9_attention_weight_min": -33.81334686279297,
      "epoch": 5.17,
      "learning_rate": 0.00011787310606060605,
      "loss": 2.8531,
      "step": 88900
    },
    {
      "activations/layer0_attention_weight_max": 15.479680061340332,
      "activations/layer0_attention_weight_min": -11.067159652709961,
      "activations/layer10_attention_weight_max": 41.512943267822266,
      "activations/layer10_attention_weight_min": -32.39182662963867,
      "activations/layer11_attention_weight_max": 37.932743072509766,
      "activations/layer11_attention_weight_min": -28.734272003173828,
      "activations/layer12_attention_weight_max": 36.09741973876953,
      "activations/layer12_attention_weight_min": -26.81342124938965,
      "activations/layer13_attention_weight_max": 47.68867111206055,
      "activations/layer13_attention_weight_min": -32.58146667480469,
      "activations/layer14_attention_weight_max": 66.06082153320312,
      "activations/layer14_attention_weight_min": -46.037696838378906,
      "activations/layer15_attention_weight_max": 43.52292251586914,
      "activations/layer15_attention_weight_min": -30.542936325073242,
      "activations/layer16_attention_weight_max": 35.713836669921875,
      "activations/layer16_attention_weight_min": -27.303672790527344,
      "activations/layer17_attention_weight_max": 37.72829818725586,
      "activations/layer17_attention_weight_min": -23.18827247619629,
      "activations/layer18_attention_weight_max": 29.876262664794922,
      "activations/layer18_attention_weight_min": -19.406152725219727,
      "activations/layer19_attention_weight_max": 43.138832092285156,
      "activations/layer19_attention_weight_min": -29.091562271118164,
      "activations/layer1_attention_weight_max": 17.625988006591797,
      "activations/layer1_attention_weight_min": -14.079583168029785,
      "activations/layer20_attention_weight_max": 31.60002899169922,
      "activations/layer20_attention_weight_min": -25.237638473510742,
      "activations/layer21_attention_weight_max": 30.229217529296875,
      "activations/layer21_attention_weight_min": -21.75087547302246,
      "activations/layer22_attention_weight_max": 47.26221466064453,
      "activations/layer22_attention_weight_min": -29.226905822753906,
      "activations/layer23_attention_weight_max": 33.2832145690918,
      "activations/layer23_attention_weight_min": -19.316680908203125,
      "activations/layer2_attention_weight_max": 31.61034393310547,
      "activations/layer2_attention_weight_min": -32.689910888671875,
      "activations/layer3_attention_weight_max": 91.27835845947266,
      "activations/layer3_attention_weight_min": -92.4031753540039,
      "activations/layer4_attention_weight_max": 135.11802673339844,
      "activations/layer4_attention_weight_min": -126.10977935791016,
      "activations/layer5_attention_weight_max": 87.73312377929688,
      "activations/layer5_attention_weight_min": -77.50477600097656,
      "activations/layer6_attention_weight_max": 57.58116912841797,
      "activations/layer6_attention_weight_min": -57.29615020751953,
      "activations/layer7_attention_weight_max": 92.71016693115234,
      "activations/layer7_attention_weight_min": -84.60247802734375,
      "activations/layer8_attention_weight_max": 49.069068908691406,
      "activations/layer8_attention_weight_min": -42.323448181152344,
      "activations/layer9_attention_weight_max": 44.54517364501953,
      "activations/layer9_attention_weight_min": -36.35355758666992,
      "epoch": 5.17,
      "learning_rate": 0.00011785416666666666,
      "loss": 2.841,
      "step": 88950
    },
    {
      "activations/layer0_attention_weight_max": 16.799013137817383,
      "activations/layer0_attention_weight_min": -11.920802116394043,
      "activations/layer10_attention_weight_max": 37.71493148803711,
      "activations/layer10_attention_weight_min": -30.59212303161621,
      "activations/layer11_attention_weight_max": 36.557403564453125,
      "activations/layer11_attention_weight_min": -26.49306297302246,
      "activations/layer12_attention_weight_max": 28.05271339416504,
      "activations/layer12_attention_weight_min": -27.539323806762695,
      "activations/layer13_attention_weight_max": 51.47427749633789,
      "activations/layer13_attention_weight_min": -31.37946319580078,
      "activations/layer14_attention_weight_max": 83.89832305908203,
      "activations/layer14_attention_weight_min": -50.957759857177734,
      "activations/layer15_attention_weight_max": 45.17726516723633,
      "activations/layer15_attention_weight_min": -27.202377319335938,
      "activations/layer16_attention_weight_max": 41.163047790527344,
      "activations/layer16_attention_weight_min": -26.08104133605957,
      "activations/layer17_attention_weight_max": 42.42586898803711,
      "activations/layer17_attention_weight_min": -24.228750228881836,
      "activations/layer18_attention_weight_max": 39.434574127197266,
      "activations/layer18_attention_weight_min": -20.939224243164062,
      "activations/layer19_attention_weight_max": 43.917266845703125,
      "activations/layer19_attention_weight_min": -29.848169326782227,
      "activations/layer1_attention_weight_max": 16.546632766723633,
      "activations/layer1_attention_weight_min": -14.582870483398438,
      "activations/layer20_attention_weight_max": 38.66176223754883,
      "activations/layer20_attention_weight_min": -26.12289047241211,
      "activations/layer21_attention_weight_max": 35.268226623535156,
      "activations/layer21_attention_weight_min": -21.623044967651367,
      "activations/layer22_attention_weight_max": 57.811397552490234,
      "activations/layer22_attention_weight_min": -28.78746223449707,
      "activations/layer23_attention_weight_max": 40.1477165222168,
      "activations/layer23_attention_weight_min": -19.072063446044922,
      "activations/layer2_attention_weight_max": 32.476585388183594,
      "activations/layer2_attention_weight_min": -32.902671813964844,
      "activations/layer3_attention_weight_max": 92.26383972167969,
      "activations/layer3_attention_weight_min": -95.48672485351562,
      "activations/layer4_attention_weight_max": 129.04298400878906,
      "activations/layer4_attention_weight_min": -132.0908660888672,
      "activations/layer5_attention_weight_max": 79.77069854736328,
      "activations/layer5_attention_weight_min": -77.6697998046875,
      "activations/layer6_attention_weight_max": 54.27969741821289,
      "activations/layer6_attention_weight_min": -54.56459045410156,
      "activations/layer7_attention_weight_max": 85.5595474243164,
      "activations/layer7_attention_weight_min": -80.097900390625,
      "activations/layer8_attention_weight_max": 41.51860046386719,
      "activations/layer8_attention_weight_min": -39.889923095703125,
      "activations/layer9_attention_weight_max": 40.04001998901367,
      "activations/layer9_attention_weight_min": -35.843135833740234,
      "epoch": 5.17,
      "learning_rate": 0.00011783522727272726,
      "loss": 2.8583,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.529,
      "eval_samples_per_second": 503.459,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.529,
      "eval_openwebtext_samples_per_second": 503.459,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 1.9975,
      "eval_wikitext_samples_per_second": 228.28,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_lambada_loss": 2.681640625,
      "eval_lambada_ppl": 14.609041604294383,
      "eval_lambada_runtime": 9.5915,
      "eval_lambada_samples_per_second": 507.637,
      "step": 89000
    },
    {
      "activations/layer0_attention_weight_max": 16.427457809448242,
      "activations/layer0_attention_weight_min": -11.114935874938965,
      "activations/layer10_attention_weight_max": 34.63809585571289,
      "activations/layer10_attention_weight_min": -33.50374221801758,
      "activations/layer11_attention_weight_max": 33.576377868652344,
      "activations/layer11_attention_weight_min": -28.366985321044922,
      "activations/layer12_attention_weight_max": 30.292116165161133,
      "activations/layer12_attention_weight_min": -25.334266662597656,
      "activations/layer13_attention_weight_max": 58.569522857666016,
      "activations/layer13_attention_weight_min": -39.597877502441406,
      "activations/layer14_attention_weight_max": 68.64842224121094,
      "activations/layer14_attention_weight_min": -52.342796325683594,
      "activations/layer15_attention_weight_max": 50.00428009033203,
      "activations/layer15_attention_weight_min": -32.59177780151367,
      "activations/layer16_attention_weight_max": 37.90351104736328,
      "activations/layer16_attention_weight_min": -29.00450325012207,
      "activations/layer17_attention_weight_max": 39.5652961730957,
      "activations/layer17_attention_weight_min": -25.629972457885742,
      "activations/layer18_attention_weight_max": 34.93047332763672,
      "activations/layer18_attention_weight_min": -20.690622329711914,
      "activations/layer19_attention_weight_max": 45.22184371948242,
      "activations/layer19_attention_weight_min": -34.448665618896484,
      "activations/layer1_attention_weight_max": 17.647274017333984,
      "activations/layer1_attention_weight_min": -14.096121788024902,
      "activations/layer20_attention_weight_max": 32.389305114746094,
      "activations/layer20_attention_weight_min": -25.14971351623535,
      "activations/layer21_attention_weight_max": 33.71598434448242,
      "activations/layer21_attention_weight_min": -22.2464656829834,
      "activations/layer22_attention_weight_max": 54.42560577392578,
      "activations/layer22_attention_weight_min": -27.36037254333496,
      "activations/layer23_attention_weight_max": 36.08503723144531,
      "activations/layer23_attention_weight_min": -23.53546905517578,
      "activations/layer2_attention_weight_max": 32.294227600097656,
      "activations/layer2_attention_weight_min": -33.61485290527344,
      "activations/layer3_attention_weight_max": 91.71686553955078,
      "activations/layer3_attention_weight_min": -96.80748748779297,
      "activations/layer4_attention_weight_max": 137.38433837890625,
      "activations/layer4_attention_weight_min": -133.25706481933594,
      "activations/layer5_attention_weight_max": 85.06907653808594,
      "activations/layer5_attention_weight_min": -84.32820892333984,
      "activations/layer6_attention_weight_max": 56.9188346862793,
      "activations/layer6_attention_weight_min": -56.45243453979492,
      "activations/layer7_attention_weight_max": 88.0495376586914,
      "activations/layer7_attention_weight_min": -85.53436279296875,
      "activations/layer8_attention_weight_max": 44.32280349731445,
      "activations/layer8_attention_weight_min": -41.15005874633789,
      "activations/layer9_attention_weight_max": 37.528221130371094,
      "activations/layer9_attention_weight_min": -36.32392501831055,
      "epoch": 5.17,
      "learning_rate": 0.00011781628787878786,
      "loss": 2.8734,
      "step": 89050
    },
    {
      "activations/layer0_attention_weight_max": 16.654834747314453,
      "activations/layer0_attention_weight_min": -10.742451667785645,
      "activations/layer10_attention_weight_max": 31.147079467773438,
      "activations/layer10_attention_weight_min": -30.012680053710938,
      "activations/layer11_attention_weight_max": 30.871902465820312,
      "activations/layer11_attention_weight_min": -27.6885986328125,
      "activations/layer12_attention_weight_max": 44.17478942871094,
      "activations/layer12_attention_weight_min": -26.399118423461914,
      "activations/layer13_attention_weight_max": 46.6888313293457,
      "activations/layer13_attention_weight_min": -32.93305206298828,
      "activations/layer14_attention_weight_max": 62.61125183105469,
      "activations/layer14_attention_weight_min": -47.47064971923828,
      "activations/layer15_attention_weight_max": 42.91280746459961,
      "activations/layer15_attention_weight_min": -27.308008193969727,
      "activations/layer16_attention_weight_max": 36.26068878173828,
      "activations/layer16_attention_weight_min": -27.434356689453125,
      "activations/layer17_attention_weight_max": 41.85830307006836,
      "activations/layer17_attention_weight_min": -24.07117462158203,
      "activations/layer18_attention_weight_max": 32.268348693847656,
      "activations/layer18_attention_weight_min": -21.647493362426758,
      "activations/layer19_attention_weight_max": 39.261165618896484,
      "activations/layer19_attention_weight_min": -30.3760986328125,
      "activations/layer1_attention_weight_max": 17.393199920654297,
      "activations/layer1_attention_weight_min": -14.20495891571045,
      "activations/layer20_attention_weight_max": 34.36357879638672,
      "activations/layer20_attention_weight_min": -23.761205673217773,
      "activations/layer21_attention_weight_max": 35.53606033325195,
      "activations/layer21_attention_weight_min": -21.366891860961914,
      "activations/layer22_attention_weight_max": 49.00960159301758,
      "activations/layer22_attention_weight_min": -27.260204315185547,
      "activations/layer23_attention_weight_max": 31.51190185546875,
      "activations/layer23_attention_weight_min": -21.998708724975586,
      "activations/layer2_attention_weight_max": 32.022335052490234,
      "activations/layer2_attention_weight_min": -31.65229034423828,
      "activations/layer3_attention_weight_max": 90.3741455078125,
      "activations/layer3_attention_weight_min": -88.96814727783203,
      "activations/layer4_attention_weight_max": 127.4375991821289,
      "activations/layer4_attention_weight_min": -125.6119155883789,
      "activations/layer5_attention_weight_max": 76.70527648925781,
      "activations/layer5_attention_weight_min": -76.16229248046875,
      "activations/layer6_attention_weight_max": 54.6483039855957,
      "activations/layer6_attention_weight_min": -55.0201530456543,
      "activations/layer7_attention_weight_max": 79.74994659423828,
      "activations/layer7_attention_weight_min": -83.74189758300781,
      "activations/layer8_attention_weight_max": 37.53377914428711,
      "activations/layer8_attention_weight_min": -37.337039947509766,
      "activations/layer9_attention_weight_max": 34.842594146728516,
      "activations/layer9_attention_weight_min": -31.798568725585938,
      "epoch": 5.18,
      "learning_rate": 0.00011779734848484848,
      "loss": 2.8424,
      "step": 89100
    },
    {
      "activations/layer0_attention_weight_max": 16.552209854125977,
      "activations/layer0_attention_weight_min": -12.010466575622559,
      "activations/layer10_attention_weight_max": 37.07741165161133,
      "activations/layer10_attention_weight_min": -31.644643783569336,
      "activations/layer11_attention_weight_max": 33.73997497558594,
      "activations/layer11_attention_weight_min": -30.911785125732422,
      "activations/layer12_attention_weight_max": 27.548267364501953,
      "activations/layer12_attention_weight_min": -25.474964141845703,
      "activations/layer13_attention_weight_max": 43.789894104003906,
      "activations/layer13_attention_weight_min": -31.552589416503906,
      "activations/layer14_attention_weight_max": 61.771240234375,
      "activations/layer14_attention_weight_min": -46.317466735839844,
      "activations/layer15_attention_weight_max": 41.89744186401367,
      "activations/layer15_attention_weight_min": -28.68099021911621,
      "activations/layer16_attention_weight_max": 38.16322326660156,
      "activations/layer16_attention_weight_min": -26.480504989624023,
      "activations/layer17_attention_weight_max": 41.21624755859375,
      "activations/layer17_attention_weight_min": -26.744518280029297,
      "activations/layer18_attention_weight_max": 31.649703979492188,
      "activations/layer18_attention_weight_min": -20.1347713470459,
      "activations/layer19_attention_weight_max": 43.895145416259766,
      "activations/layer19_attention_weight_min": -28.515039443969727,
      "activations/layer1_attention_weight_max": 16.701086044311523,
      "activations/layer1_attention_weight_min": -16.038793563842773,
      "activations/layer20_attention_weight_max": 37.64134216308594,
      "activations/layer20_attention_weight_min": -24.479698181152344,
      "activations/layer21_attention_weight_max": 36.4260368347168,
      "activations/layer21_attention_weight_min": -22.202293395996094,
      "activations/layer22_attention_weight_max": 55.62022018432617,
      "activations/layer22_attention_weight_min": -28.52362060546875,
      "activations/layer23_attention_weight_max": 41.33167266845703,
      "activations/layer23_attention_weight_min": -19.44211196899414,
      "activations/layer2_attention_weight_max": 32.46849060058594,
      "activations/layer2_attention_weight_min": -33.365386962890625,
      "activations/layer3_attention_weight_max": 91.87614440917969,
      "activations/layer3_attention_weight_min": -93.50819396972656,
      "activations/layer4_attention_weight_max": 135.0172576904297,
      "activations/layer4_attention_weight_min": -130.11167907714844,
      "activations/layer5_attention_weight_max": 84.48896789550781,
      "activations/layer5_attention_weight_min": -81.57260131835938,
      "activations/layer6_attention_weight_max": 56.454654693603516,
      "activations/layer6_attention_weight_min": -58.3065185546875,
      "activations/layer7_attention_weight_max": 90.18318176269531,
      "activations/layer7_attention_weight_min": -87.03789520263672,
      "activations/layer8_attention_weight_max": 43.76613235473633,
      "activations/layer8_attention_weight_min": -42.031368255615234,
      "activations/layer9_attention_weight_max": 39.551795959472656,
      "activations/layer9_attention_weight_min": -33.87263107299805,
      "epoch": 5.18,
      "learning_rate": 0.00011777840909090908,
      "loss": 2.8373,
      "step": 89150
    },
    {
      "activations/layer0_attention_weight_max": 15.419034004211426,
      "activations/layer0_attention_weight_min": -11.263731956481934,
      "activations/layer10_attention_weight_max": 41.906761169433594,
      "activations/layer10_attention_weight_min": -35.759063720703125,
      "activations/layer11_attention_weight_max": 38.6930046081543,
      "activations/layer11_attention_weight_min": -30.26366424560547,
      "activations/layer12_attention_weight_max": 37.170745849609375,
      "activations/layer12_attention_weight_min": -27.95037078857422,
      "activations/layer13_attention_weight_max": 51.665470123291016,
      "activations/layer13_attention_weight_min": -33.49440002441406,
      "activations/layer14_attention_weight_max": 85.56928253173828,
      "activations/layer14_attention_weight_min": -53.26913070678711,
      "activations/layer15_attention_weight_max": 50.97177505493164,
      "activations/layer15_attention_weight_min": -30.624244689941406,
      "activations/layer16_attention_weight_max": 44.61455535888672,
      "activations/layer16_attention_weight_min": -27.155197143554688,
      "activations/layer17_attention_weight_max": 42.6953010559082,
      "activations/layer17_attention_weight_min": -23.59502410888672,
      "activations/layer18_attention_weight_max": 32.2535514831543,
      "activations/layer18_attention_weight_min": -20.43514633178711,
      "activations/layer19_attention_weight_max": 51.57474899291992,
      "activations/layer19_attention_weight_min": -29.277673721313477,
      "activations/layer1_attention_weight_max": 18.51018714904785,
      "activations/layer1_attention_weight_min": -17.218303680419922,
      "activations/layer20_attention_weight_max": 39.174102783203125,
      "activations/layer20_attention_weight_min": -23.345504760742188,
      "activations/layer21_attention_weight_max": 34.664974212646484,
      "activations/layer21_attention_weight_min": -19.895950317382812,
      "activations/layer22_attention_weight_max": 58.682273864746094,
      "activations/layer22_attention_weight_min": -27.494749069213867,
      "activations/layer23_attention_weight_max": 40.492530822753906,
      "activations/layer23_attention_weight_min": -19.021709442138672,
      "activations/layer2_attention_weight_max": 32.65562438964844,
      "activations/layer2_attention_weight_min": -32.11162185668945,
      "activations/layer3_attention_weight_max": 91.7764892578125,
      "activations/layer3_attention_weight_min": -97.27960205078125,
      "activations/layer4_attention_weight_max": 134.25144958496094,
      "activations/layer4_attention_weight_min": -134.70579528808594,
      "activations/layer5_attention_weight_max": 86.42691040039062,
      "activations/layer5_attention_weight_min": -82.52867889404297,
      "activations/layer6_attention_weight_max": 59.60112762451172,
      "activations/layer6_attention_weight_min": -58.87635040283203,
      "activations/layer7_attention_weight_max": 92.23735046386719,
      "activations/layer7_attention_weight_min": -89.21764373779297,
      "activations/layer8_attention_weight_max": 44.6413688659668,
      "activations/layer8_attention_weight_min": -44.175804138183594,
      "activations/layer9_attention_weight_max": 43.519771575927734,
      "activations/layer9_attention_weight_min": -39.24152374267578,
      "epoch": 5.18,
      "learning_rate": 0.00011775946969696968,
      "loss": 2.8391,
      "step": 89200
    },
    {
      "activations/layer0_attention_weight_max": 17.207218170166016,
      "activations/layer0_attention_weight_min": -12.110751152038574,
      "activations/layer10_attention_weight_max": 36.854373931884766,
      "activations/layer10_attention_weight_min": -33.529449462890625,
      "activations/layer11_attention_weight_max": 34.118812561035156,
      "activations/layer11_attention_weight_min": -27.70159149169922,
      "activations/layer12_attention_weight_max": 25.868141174316406,
      "activations/layer12_attention_weight_min": -26.494840621948242,
      "activations/layer13_attention_weight_max": 49.34800338745117,
      "activations/layer13_attention_weight_min": -34.394893646240234,
      "activations/layer14_attention_weight_max": 63.571502685546875,
      "activations/layer14_attention_weight_min": -45.43417739868164,
      "activations/layer15_attention_weight_max": 55.42378234863281,
      "activations/layer15_attention_weight_min": -32.602500915527344,
      "activations/layer16_attention_weight_max": 38.515167236328125,
      "activations/layer16_attention_weight_min": -27.848894119262695,
      "activations/layer17_attention_weight_max": 41.772613525390625,
      "activations/layer17_attention_weight_min": -25.897340774536133,
      "activations/layer18_attention_weight_max": 32.37290954589844,
      "activations/layer18_attention_weight_min": -21.032487869262695,
      "activations/layer19_attention_weight_max": 39.48875045776367,
      "activations/layer19_attention_weight_min": -28.940805435180664,
      "activations/layer1_attention_weight_max": 16.796297073364258,
      "activations/layer1_attention_weight_min": -16.645292282104492,
      "activations/layer20_attention_weight_max": 30.038639068603516,
      "activations/layer20_attention_weight_min": -21.87891960144043,
      "activations/layer21_attention_weight_max": 27.459651947021484,
      "activations/layer21_attention_weight_min": -20.36881446838379,
      "activations/layer22_attention_weight_max": 48.33797073364258,
      "activations/layer22_attention_weight_min": -28.763147354125977,
      "activations/layer23_attention_weight_max": 35.370269775390625,
      "activations/layer23_attention_weight_min": -20.513885498046875,
      "activations/layer2_attention_weight_max": 34.67447280883789,
      "activations/layer2_attention_weight_min": -34.884368896484375,
      "activations/layer3_attention_weight_max": 99.71949005126953,
      "activations/layer3_attention_weight_min": -95.97129821777344,
      "activations/layer4_attention_weight_max": 139.6126708984375,
      "activations/layer4_attention_weight_min": -135.33169555664062,
      "activations/layer5_attention_weight_max": 85.7756576538086,
      "activations/layer5_attention_weight_min": -79.3769760131836,
      "activations/layer6_attention_weight_max": 56.78560256958008,
      "activations/layer6_attention_weight_min": -57.0894889831543,
      "activations/layer7_attention_weight_max": 86.64540100097656,
      "activations/layer7_attention_weight_min": -89.31676483154297,
      "activations/layer8_attention_weight_max": 41.142555236816406,
      "activations/layer8_attention_weight_min": -40.95457077026367,
      "activations/layer9_attention_weight_max": 38.00703048706055,
      "activations/layer9_attention_weight_min": -35.25965881347656,
      "epoch": 5.19,
      "learning_rate": 0.0001177405303030303,
      "loss": 2.8457,
      "step": 89250
    },
    {
      "activations/layer0_attention_weight_max": 17.084186553955078,
      "activations/layer0_attention_weight_min": -12.301810264587402,
      "activations/layer10_attention_weight_max": 32.87261962890625,
      "activations/layer10_attention_weight_min": -30.469707489013672,
      "activations/layer11_attention_weight_max": 29.50861167907715,
      "activations/layer11_attention_weight_min": -27.27173614501953,
      "activations/layer12_attention_weight_max": 26.596277236938477,
      "activations/layer12_attention_weight_min": -30.195032119750977,
      "activations/layer13_attention_weight_max": 37.86260986328125,
      "activations/layer13_attention_weight_min": -33.196231842041016,
      "activations/layer14_attention_weight_max": 57.3640022277832,
      "activations/layer14_attention_weight_min": -45.63979721069336,
      "activations/layer15_attention_weight_max": 36.414710998535156,
      "activations/layer15_attention_weight_min": -27.05775260925293,
      "activations/layer16_attention_weight_max": 36.42604446411133,
      "activations/layer16_attention_weight_min": -26.523460388183594,
      "activations/layer17_attention_weight_max": 34.67716979980469,
      "activations/layer17_attention_weight_min": -24.47664451599121,
      "activations/layer18_attention_weight_max": 33.19011688232422,
      "activations/layer18_attention_weight_min": -19.031898498535156,
      "activations/layer19_attention_weight_max": 37.93811798095703,
      "activations/layer19_attention_weight_min": -29.711095809936523,
      "activations/layer1_attention_weight_max": 17.25551986694336,
      "activations/layer1_attention_weight_min": -14.630446434020996,
      "activations/layer20_attention_weight_max": 29.43317222595215,
      "activations/layer20_attention_weight_min": -23.49877166748047,
      "activations/layer21_attention_weight_max": 31.486309051513672,
      "activations/layer21_attention_weight_min": -21.05100440979004,
      "activations/layer22_attention_weight_max": 51.572383880615234,
      "activations/layer22_attention_weight_min": -28.657516479492188,
      "activations/layer23_attention_weight_max": 32.837074279785156,
      "activations/layer23_attention_weight_min": -20.989543914794922,
      "activations/layer2_attention_weight_max": 31.44638442993164,
      "activations/layer2_attention_weight_min": -32.09986877441406,
      "activations/layer3_attention_weight_max": 88.78441619873047,
      "activations/layer3_attention_weight_min": -94.2704849243164,
      "activations/layer4_attention_weight_max": 128.02667236328125,
      "activations/layer4_attention_weight_min": -127.6320571899414,
      "activations/layer5_attention_weight_max": 79.42536163330078,
      "activations/layer5_attention_weight_min": -77.21646118164062,
      "activations/layer6_attention_weight_max": 55.22584915161133,
      "activations/layer6_attention_weight_min": -54.41536331176758,
      "activations/layer7_attention_weight_max": 84.15673065185547,
      "activations/layer7_attention_weight_min": -83.74012756347656,
      "activations/layer8_attention_weight_max": 39.06911849975586,
      "activations/layer8_attention_weight_min": -39.24930191040039,
      "activations/layer9_attention_weight_max": 35.88141632080078,
      "activations/layer9_attention_weight_min": -33.499053955078125,
      "epoch": 5.19,
      "learning_rate": 0.0001177215909090909,
      "loss": 2.8586,
      "step": 89300
    },
    {
      "activations/layer0_attention_weight_max": 17.405607223510742,
      "activations/layer0_attention_weight_min": -11.491802215576172,
      "activations/layer10_attention_weight_max": 32.975128173828125,
      "activations/layer10_attention_weight_min": -32.9371223449707,
      "activations/layer11_attention_weight_max": 32.22150802612305,
      "activations/layer11_attention_weight_min": -29.438243865966797,
      "activations/layer12_attention_weight_max": 24.983551025390625,
      "activations/layer12_attention_weight_min": -29.30182647705078,
      "activations/layer13_attention_weight_max": 49.41890335083008,
      "activations/layer13_attention_weight_min": -36.80061340332031,
      "activations/layer14_attention_weight_max": 68.25001525878906,
      "activations/layer14_attention_weight_min": -52.676937103271484,
      "activations/layer15_attention_weight_max": 37.577171325683594,
      "activations/layer15_attention_weight_min": -27.307781219482422,
      "activations/layer16_attention_weight_max": 37.84730911254883,
      "activations/layer16_attention_weight_min": -27.147550582885742,
      "activations/layer17_attention_weight_max": 37.34858703613281,
      "activations/layer17_attention_weight_min": -24.986379623413086,
      "activations/layer18_attention_weight_max": 33.05740737915039,
      "activations/layer18_attention_weight_min": -21.42951774597168,
      "activations/layer19_attention_weight_max": 38.848262786865234,
      "activations/layer19_attention_weight_min": -29.50050163269043,
      "activations/layer1_attention_weight_max": 17.078577041625977,
      "activations/layer1_attention_weight_min": -13.648971557617188,
      "activations/layer20_attention_weight_max": 33.81968688964844,
      "activations/layer20_attention_weight_min": -22.095548629760742,
      "activations/layer21_attention_weight_max": 34.26240158081055,
      "activations/layer21_attention_weight_min": -21.740468978881836,
      "activations/layer22_attention_weight_max": 46.29014587402344,
      "activations/layer22_attention_weight_min": -27.755544662475586,
      "activations/layer23_attention_weight_max": 34.237770080566406,
      "activations/layer23_attention_weight_min": -21.288387298583984,
      "activations/layer2_attention_weight_max": 31.654096603393555,
      "activations/layer2_attention_weight_min": -32.162681579589844,
      "activations/layer3_attention_weight_max": 88.45381164550781,
      "activations/layer3_attention_weight_min": -91.21831512451172,
      "activations/layer4_attention_weight_max": 127.9658203125,
      "activations/layer4_attention_weight_min": -126.626953125,
      "activations/layer5_attention_weight_max": 83.93888854980469,
      "activations/layer5_attention_weight_min": -81.332275390625,
      "activations/layer6_attention_weight_max": 58.05038833618164,
      "activations/layer6_attention_weight_min": -59.55629348754883,
      "activations/layer7_attention_weight_max": 87.53655242919922,
      "activations/layer7_attention_weight_min": -84.92735290527344,
      "activations/layer8_attention_weight_max": 39.027305603027344,
      "activations/layer8_attention_weight_min": -39.99693298339844,
      "activations/layer9_attention_weight_max": 35.29323196411133,
      "activations/layer9_attention_weight_min": -37.8162956237793,
      "epoch": 5.19,
      "learning_rate": 0.00011770303030303029,
      "loss": 2.8518,
      "step": 89350
    },
    {
      "activations/layer0_attention_weight_max": 13.864299774169922,
      "activations/layer0_attention_weight_min": -12.23853588104248,
      "activations/layer10_attention_weight_max": 34.44368362426758,
      "activations/layer10_attention_weight_min": -30.248937606811523,
      "activations/layer11_attention_weight_max": 30.879364013671875,
      "activations/layer11_attention_weight_min": -29.671592712402344,
      "activations/layer12_attention_weight_max": 24.86445426940918,
      "activations/layer12_attention_weight_min": -27.135719299316406,
      "activations/layer13_attention_weight_max": 47.64259338378906,
      "activations/layer13_attention_weight_min": -34.55903244018555,
      "activations/layer14_attention_weight_max": 71.6027603149414,
      "activations/layer14_attention_weight_min": -48.83421325683594,
      "activations/layer15_attention_weight_max": 40.780948638916016,
      "activations/layer15_attention_weight_min": -29.06121063232422,
      "activations/layer16_attention_weight_max": 49.22539520263672,
      "activations/layer16_attention_weight_min": -29.308738708496094,
      "activations/layer17_attention_weight_max": 41.86713790893555,
      "activations/layer17_attention_weight_min": -24.38013458251953,
      "activations/layer18_attention_weight_max": 32.957366943359375,
      "activations/layer18_attention_weight_min": -20.621252059936523,
      "activations/layer19_attention_weight_max": 50.664676666259766,
      "activations/layer19_attention_weight_min": -29.547210693359375,
      "activations/layer1_attention_weight_max": 17.57172966003418,
      "activations/layer1_attention_weight_min": -14.007123947143555,
      "activations/layer20_attention_weight_max": 39.20773696899414,
      "activations/layer20_attention_weight_min": -24.871030807495117,
      "activations/layer21_attention_weight_max": 39.060386657714844,
      "activations/layer21_attention_weight_min": -22.26771354675293,
      "activations/layer22_attention_weight_max": 56.28968811035156,
      "activations/layer22_attention_weight_min": -28.803104400634766,
      "activations/layer23_attention_weight_max": 37.14543914794922,
      "activations/layer23_attention_weight_min": -21.52593994140625,
      "activations/layer2_attention_weight_max": 32.17129898071289,
      "activations/layer2_attention_weight_min": -31.541656494140625,
      "activations/layer3_attention_weight_max": 89.68109893798828,
      "activations/layer3_attention_weight_min": -91.41175079345703,
      "activations/layer4_attention_weight_max": 132.43685913085938,
      "activations/layer4_attention_weight_min": -125.6440658569336,
      "activations/layer5_attention_weight_max": 82.72340393066406,
      "activations/layer5_attention_weight_min": -77.55132293701172,
      "activations/layer6_attention_weight_max": 56.08564376831055,
      "activations/layer6_attention_weight_min": -54.61848831176758,
      "activations/layer7_attention_weight_max": 84.34518432617188,
      "activations/layer7_attention_weight_min": -81.44867706298828,
      "activations/layer8_attention_weight_max": 39.232234954833984,
      "activations/layer8_attention_weight_min": -37.01115036010742,
      "activations/layer9_attention_weight_max": 37.72321701049805,
      "activations/layer9_attention_weight_min": -33.67146301269531,
      "epoch": 5.19,
      "learning_rate": 0.0001176840909090909,
      "loss": 2.8535,
      "step": 89400
    },
    {
      "activations/layer0_attention_weight_max": 16.767961502075195,
      "activations/layer0_attention_weight_min": -11.282434463500977,
      "activations/layer10_attention_weight_max": 33.03809356689453,
      "activations/layer10_attention_weight_min": -29.99003028869629,
      "activations/layer11_attention_weight_max": 33.0716552734375,
      "activations/layer11_attention_weight_min": -28.195812225341797,
      "activations/layer12_attention_weight_max": 29.761919021606445,
      "activations/layer12_attention_weight_min": -28.698928833007812,
      "activations/layer13_attention_weight_max": 44.03948211669922,
      "activations/layer13_attention_weight_min": -31.885272979736328,
      "activations/layer14_attention_weight_max": 59.681556701660156,
      "activations/layer14_attention_weight_min": -42.10236740112305,
      "activations/layer15_attention_weight_max": 50.1378059387207,
      "activations/layer15_attention_weight_min": -31.771114349365234,
      "activations/layer16_attention_weight_max": 38.88498306274414,
      "activations/layer16_attention_weight_min": -30.211210250854492,
      "activations/layer17_attention_weight_max": 41.85240173339844,
      "activations/layer17_attention_weight_min": -27.39553451538086,
      "activations/layer18_attention_weight_max": 35.12900924682617,
      "activations/layer18_attention_weight_min": -21.603769302368164,
      "activations/layer19_attention_weight_max": 41.606239318847656,
      "activations/layer19_attention_weight_min": -29.50455093383789,
      "activations/layer1_attention_weight_max": 17.726213455200195,
      "activations/layer1_attention_weight_min": -15.379024505615234,
      "activations/layer20_attention_weight_max": 35.79377365112305,
      "activations/layer20_attention_weight_min": -26.03697395324707,
      "activations/layer21_attention_weight_max": 32.42897033691406,
      "activations/layer21_attention_weight_min": -23.711557388305664,
      "activations/layer22_attention_weight_max": 53.722312927246094,
      "activations/layer22_attention_weight_min": -28.389970779418945,
      "activations/layer23_attention_weight_max": 33.300594329833984,
      "activations/layer23_attention_weight_min": -19.592330932617188,
      "activations/layer2_attention_weight_max": 32.413185119628906,
      "activations/layer2_attention_weight_min": -31.73833465576172,
      "activations/layer3_attention_weight_max": 91.54158020019531,
      "activations/layer3_attention_weight_min": -93.79463195800781,
      "activations/layer4_attention_weight_max": 131.7843475341797,
      "activations/layer4_attention_weight_min": -129.99148559570312,
      "activations/layer5_attention_weight_max": 82.05772399902344,
      "activations/layer5_attention_weight_min": -78.785888671875,
      "activations/layer6_attention_weight_max": 55.91751480102539,
      "activations/layer6_attention_weight_min": -57.38563919067383,
      "activations/layer7_attention_weight_max": 85.28427124023438,
      "activations/layer7_attention_weight_min": -83.04547882080078,
      "activations/layer8_attention_weight_max": 38.33386993408203,
      "activations/layer8_attention_weight_min": -39.29972457885742,
      "activations/layer9_attention_weight_max": 35.18806838989258,
      "activations/layer9_attention_weight_min": -35.42867660522461,
      "epoch": 5.2,
      "learning_rate": 0.00011766515151515152,
      "loss": 2.8577,
      "step": 89450
    },
    {
      "activations/layer0_attention_weight_max": 17.40401268005371,
      "activations/layer0_attention_weight_min": -12.471733093261719,
      "activations/layer10_attention_weight_max": 31.069509506225586,
      "activations/layer10_attention_weight_min": -29.690664291381836,
      "activations/layer11_attention_weight_max": 29.667251586914062,
      "activations/layer11_attention_weight_min": -29.626575469970703,
      "activations/layer12_attention_weight_max": 22.552112579345703,
      "activations/layer12_attention_weight_min": -29.44660758972168,
      "activations/layer13_attention_weight_max": 39.441734313964844,
      "activations/layer13_attention_weight_min": -32.0679931640625,
      "activations/layer14_attention_weight_max": 51.23790740966797,
      "activations/layer14_attention_weight_min": -42.03800964355469,
      "activations/layer15_attention_weight_max": 37.7103157043457,
      "activations/layer15_attention_weight_min": -26.95122718811035,
      "activations/layer16_attention_weight_max": 34.46208190917969,
      "activations/layer16_attention_weight_min": -26.006853103637695,
      "activations/layer17_attention_weight_max": 41.73700714111328,
      "activations/layer17_attention_weight_min": -24.15654182434082,
      "activations/layer18_attention_weight_max": 29.306631088256836,
      "activations/layer18_attention_weight_min": -19.646066665649414,
      "activations/layer19_attention_weight_max": 40.15731430053711,
      "activations/layer19_attention_weight_min": -30.11421012878418,
      "activations/layer1_attention_weight_max": 17.940969467163086,
      "activations/layer1_attention_weight_min": -16.681249618530273,
      "activations/layer20_attention_weight_max": 30.06297492980957,
      "activations/layer20_attention_weight_min": -23.774158477783203,
      "activations/layer21_attention_weight_max": 28.862834930419922,
      "activations/layer21_attention_weight_min": -20.165115356445312,
      "activations/layer22_attention_weight_max": 46.93095016479492,
      "activations/layer22_attention_weight_min": -28.46050453186035,
      "activations/layer23_attention_weight_max": 33.05781555175781,
      "activations/layer23_attention_weight_min": -21.207813262939453,
      "activations/layer2_attention_weight_max": 33.952362060546875,
      "activations/layer2_attention_weight_min": -32.928829193115234,
      "activations/layer3_attention_weight_max": 92.82457733154297,
      "activations/layer3_attention_weight_min": -97.69637298583984,
      "activations/layer4_attention_weight_max": 131.86741638183594,
      "activations/layer4_attention_weight_min": -134.81492614746094,
      "activations/layer5_attention_weight_max": 81.06956481933594,
      "activations/layer5_attention_weight_min": -79.05055236816406,
      "activations/layer6_attention_weight_max": 54.86360168457031,
      "activations/layer6_attention_weight_min": -57.117156982421875,
      "activations/layer7_attention_weight_max": 83.4775161743164,
      "activations/layer7_attention_weight_min": -84.2352066040039,
      "activations/layer8_attention_weight_max": 39.51169967651367,
      "activations/layer8_attention_weight_min": -41.46875,
      "activations/layer9_attention_weight_max": 34.791927337646484,
      "activations/layer9_attention_weight_min": -33.83919906616211,
      "epoch": 5.2,
      "learning_rate": 0.00011764621212121211,
      "loss": 2.8506,
      "step": 89500
    },
    {
      "activations/layer0_attention_weight_max": 15.810338020324707,
      "activations/layer0_attention_weight_min": -11.35497760772705,
      "activations/layer10_attention_weight_max": 33.87732696533203,
      "activations/layer10_attention_weight_min": -28.88327407836914,
      "activations/layer11_attention_weight_max": 32.053863525390625,
      "activations/layer11_attention_weight_min": -29.742610931396484,
      "activations/layer12_attention_weight_max": 31.469989776611328,
      "activations/layer12_attention_weight_min": -26.79464340209961,
      "activations/layer13_attention_weight_max": 43.832542419433594,
      "activations/layer13_attention_weight_min": -35.46739196777344,
      "activations/layer14_attention_weight_max": 63.58156204223633,
      "activations/layer14_attention_weight_min": -46.68516540527344,
      "activations/layer15_attention_weight_max": 40.287113189697266,
      "activations/layer15_attention_weight_min": -30.67988395690918,
      "activations/layer16_attention_weight_max": 35.03892517089844,
      "activations/layer16_attention_weight_min": -28.90190315246582,
      "activations/layer17_attention_weight_max": 39.11412811279297,
      "activations/layer17_attention_weight_min": -24.579723358154297,
      "activations/layer18_attention_weight_max": 35.64939880371094,
      "activations/layer18_attention_weight_min": -23.102373123168945,
      "activations/layer19_attention_weight_max": 39.37506866455078,
      "activations/layer19_attention_weight_min": -28.527912139892578,
      "activations/layer1_attention_weight_max": 17.76910400390625,
      "activations/layer1_attention_weight_min": -17.657224655151367,
      "activations/layer20_attention_weight_max": 33.89847183227539,
      "activations/layer20_attention_weight_min": -24.11734390258789,
      "activations/layer21_attention_weight_max": 34.86349105834961,
      "activations/layer21_attention_weight_min": -20.390583038330078,
      "activations/layer22_attention_weight_max": 51.11272430419922,
      "activations/layer22_attention_weight_min": -29.82985496520996,
      "activations/layer23_attention_weight_max": 34.93511962890625,
      "activations/layer23_attention_weight_min": -23.74335479736328,
      "activations/layer2_attention_weight_max": 31.996015548706055,
      "activations/layer2_attention_weight_min": -31.310352325439453,
      "activations/layer3_attention_weight_max": 91.03681182861328,
      "activations/layer3_attention_weight_min": -94.5496826171875,
      "activations/layer4_attention_weight_max": 133.0193634033203,
      "activations/layer4_attention_weight_min": -129.35653686523438,
      "activations/layer5_attention_weight_max": 84.21690368652344,
      "activations/layer5_attention_weight_min": -79.84994506835938,
      "activations/layer6_attention_weight_max": 54.61815643310547,
      "activations/layer6_attention_weight_min": -56.78627014160156,
      "activations/layer7_attention_weight_max": 82.78665161132812,
      "activations/layer7_attention_weight_min": -82.81824493408203,
      "activations/layer8_attention_weight_max": 40.70142364501953,
      "activations/layer8_attention_weight_min": -40.34716033935547,
      "activations/layer9_attention_weight_max": 38.36809539794922,
      "activations/layer9_attention_weight_min": -35.37360382080078,
      "epoch": 5.2,
      "learning_rate": 0.00011762727272727272,
      "loss": 2.8479,
      "step": 89550
    },
    {
      "activations/layer0_attention_weight_max": 16.638229370117188,
      "activations/layer0_attention_weight_min": -11.615591049194336,
      "activations/layer10_attention_weight_max": 31.903148651123047,
      "activations/layer10_attention_weight_min": -29.30147361755371,
      "activations/layer11_attention_weight_max": 31.907543182373047,
      "activations/layer11_attention_weight_min": -24.934324264526367,
      "activations/layer12_attention_weight_max": 26.401151657104492,
      "activations/layer12_attention_weight_min": -27.761367797851562,
      "activations/layer13_attention_weight_max": 40.35480499267578,
      "activations/layer13_attention_weight_min": -29.59188461303711,
      "activations/layer14_attention_weight_max": 58.710575103759766,
      "activations/layer14_attention_weight_min": -44.864898681640625,
      "activations/layer15_attention_weight_max": 37.51531219482422,
      "activations/layer15_attention_weight_min": -26.808046340942383,
      "activations/layer16_attention_weight_max": 37.144779205322266,
      "activations/layer16_attention_weight_min": -26.679956436157227,
      "activations/layer17_attention_weight_max": 40.751399993896484,
      "activations/layer17_attention_weight_min": -24.766658782958984,
      "activations/layer18_attention_weight_max": 33.56892395019531,
      "activations/layer18_attention_weight_min": -21.24650764465332,
      "activations/layer19_attention_weight_max": 37.51293182373047,
      "activations/layer19_attention_weight_min": -29.51544189453125,
      "activations/layer1_attention_weight_max": 17.483434677124023,
      "activations/layer1_attention_weight_min": -15.539505004882812,
      "activations/layer20_attention_weight_max": 30.21954345703125,
      "activations/layer20_attention_weight_min": -22.142513275146484,
      "activations/layer21_attention_weight_max": 29.46743392944336,
      "activations/layer21_attention_weight_min": -20.541025161743164,
      "activations/layer22_attention_weight_max": 48.442176818847656,
      "activations/layer22_attention_weight_min": -26.744741439819336,
      "activations/layer23_attention_weight_max": 34.189857482910156,
      "activations/layer23_attention_weight_min": -20.157512664794922,
      "activations/layer2_attention_weight_max": 31.859630584716797,
      "activations/layer2_attention_weight_min": -31.427825927734375,
      "activations/layer3_attention_weight_max": 88.26342010498047,
      "activations/layer3_attention_weight_min": -91.82756805419922,
      "activations/layer4_attention_weight_max": 127.88018035888672,
      "activations/layer4_attention_weight_min": -126.8499755859375,
      "activations/layer5_attention_weight_max": 75.93077850341797,
      "activations/layer5_attention_weight_min": -75.52959442138672,
      "activations/layer6_attention_weight_max": 52.07369613647461,
      "activations/layer6_attention_weight_min": -54.025794982910156,
      "activations/layer7_attention_weight_max": 74.79560089111328,
      "activations/layer7_attention_weight_min": -79.91242980957031,
      "activations/layer8_attention_weight_max": 38.2969970703125,
      "activations/layer8_attention_weight_min": -36.76091003417969,
      "activations/layer9_attention_weight_max": 34.38330841064453,
      "activations/layer9_attention_weight_min": -31.86383056640625,
      "epoch": 5.21,
      "learning_rate": 0.00011760833333333331,
      "loss": 2.8432,
      "step": 89600
    },
    {
      "activations/layer0_attention_weight_max": 16.807580947875977,
      "activations/layer0_attention_weight_min": -11.83972454071045,
      "activations/layer10_attention_weight_max": 42.99039840698242,
      "activations/layer10_attention_weight_min": -32.502723693847656,
      "activations/layer11_attention_weight_max": 41.711280822753906,
      "activations/layer11_attention_weight_min": -28.996124267578125,
      "activations/layer12_attention_weight_max": 39.11631393432617,
      "activations/layer12_attention_weight_min": -32.38335418701172,
      "activations/layer13_attention_weight_max": 73.8889389038086,
      "activations/layer13_attention_weight_min": -36.410091400146484,
      "activations/layer14_attention_weight_max": 114.97163391113281,
      "activations/layer14_attention_weight_min": -56.1601448059082,
      "activations/layer15_attention_weight_max": 60.61323165893555,
      "activations/layer15_attention_weight_min": -32.068702697753906,
      "activations/layer16_attention_weight_max": 52.75997543334961,
      "activations/layer16_attention_weight_min": -26.731487274169922,
      "activations/layer17_attention_weight_max": 57.0167121887207,
      "activations/layer17_attention_weight_min": -24.98767852783203,
      "activations/layer18_attention_weight_max": 55.558067321777344,
      "activations/layer18_attention_weight_min": -25.488956451416016,
      "activations/layer19_attention_weight_max": 61.523494720458984,
      "activations/layer19_attention_weight_min": -30.27791404724121,
      "activations/layer1_attention_weight_max": 17.339885711669922,
      "activations/layer1_attention_weight_min": -16.49026107788086,
      "activations/layer20_attention_weight_max": 54.22610855102539,
      "activations/layer20_attention_weight_min": -24.073749542236328,
      "activations/layer21_attention_weight_max": 54.36811447143555,
      "activations/layer21_attention_weight_min": -22.3724308013916,
      "activations/layer22_attention_weight_max": 91.7742919921875,
      "activations/layer22_attention_weight_min": -31.3617000579834,
      "activations/layer23_attention_weight_max": 56.62705993652344,
      "activations/layer23_attention_weight_min": -21.47529411315918,
      "activations/layer2_attention_weight_max": 33.04242706298828,
      "activations/layer2_attention_weight_min": -34.265892028808594,
      "activations/layer3_attention_weight_max": 92.00471496582031,
      "activations/layer3_attention_weight_min": -95.5032730102539,
      "activations/layer4_attention_weight_max": 126.81571197509766,
      "activations/layer4_attention_weight_min": -131.7349395751953,
      "activations/layer5_attention_weight_max": 82.16667175292969,
      "activations/layer5_attention_weight_min": -79.47158813476562,
      "activations/layer6_attention_weight_max": 56.55901336669922,
      "activations/layer6_attention_weight_min": -57.636749267578125,
      "activations/layer7_attention_weight_max": 88.75885009765625,
      "activations/layer7_attention_weight_min": -86.5933837890625,
      "activations/layer8_attention_weight_max": 44.16128158569336,
      "activations/layer8_attention_weight_min": -41.40148162841797,
      "activations/layer9_attention_weight_max": 41.98479080200195,
      "activations/layer9_attention_weight_min": -36.02618408203125,
      "epoch": 5.21,
      "learning_rate": 0.00011758939393939392,
      "loss": 2.8455,
      "step": 89650
    },
    {
      "activations/layer0_attention_weight_max": 16.283151626586914,
      "activations/layer0_attention_weight_min": -12.380566596984863,
      "activations/layer10_attention_weight_max": 39.173255920410156,
      "activations/layer10_attention_weight_min": -34.60991287231445,
      "activations/layer11_attention_weight_max": 34.426666259765625,
      "activations/layer11_attention_weight_min": -28.32134246826172,
      "activations/layer12_attention_weight_max": 28.879981994628906,
      "activations/layer12_attention_weight_min": -27.62360954284668,
      "activations/layer13_attention_weight_max": 44.63239669799805,
      "activations/layer13_attention_weight_min": -30.493654251098633,
      "activations/layer14_attention_weight_max": 57.59600830078125,
      "activations/layer14_attention_weight_min": -41.37498474121094,
      "activations/layer15_attention_weight_max": 44.15531921386719,
      "activations/layer15_attention_weight_min": -28.35103416442871,
      "activations/layer16_attention_weight_max": 41.55173873901367,
      "activations/layer16_attention_weight_min": -28.44717025756836,
      "activations/layer17_attention_weight_max": 44.667110443115234,
      "activations/layer17_attention_weight_min": -25.847692489624023,
      "activations/layer18_attention_weight_max": 34.17604064941406,
      "activations/layer18_attention_weight_min": -21.341365814208984,
      "activations/layer19_attention_weight_max": 43.34922790527344,
      "activations/layer19_attention_weight_min": -32.04659652709961,
      "activations/layer1_attention_weight_max": 17.167102813720703,
      "activations/layer1_attention_weight_min": -15.532228469848633,
      "activations/layer20_attention_weight_max": 34.868804931640625,
      "activations/layer20_attention_weight_min": -23.86505699157715,
      "activations/layer21_attention_weight_max": 35.829322814941406,
      "activations/layer21_attention_weight_min": -20.277820587158203,
      "activations/layer22_attention_weight_max": 50.424072265625,
      "activations/layer22_attention_weight_min": -26.76641082763672,
      "activations/layer23_attention_weight_max": 35.110782623291016,
      "activations/layer23_attention_weight_min": -20.028268814086914,
      "activations/layer2_attention_weight_max": 33.69910430908203,
      "activations/layer2_attention_weight_min": -32.090965270996094,
      "activations/layer3_attention_weight_max": 88.42155456542969,
      "activations/layer3_attention_weight_min": -90.28349304199219,
      "activations/layer4_attention_weight_max": 130.27671813964844,
      "activations/layer4_attention_weight_min": -126.60548400878906,
      "activations/layer5_attention_weight_max": 83.02604675292969,
      "activations/layer5_attention_weight_min": -81.00340270996094,
      "activations/layer6_attention_weight_max": 54.98289489746094,
      "activations/layer6_attention_weight_min": -61.32054138183594,
      "activations/layer7_attention_weight_max": 85.07241821289062,
      "activations/layer7_attention_weight_min": -87.38839721679688,
      "activations/layer8_attention_weight_max": 43.06938552856445,
      "activations/layer8_attention_weight_min": -40.25026321411133,
      "activations/layer9_attention_weight_max": 37.29855728149414,
      "activations/layer9_attention_weight_min": -33.6361198425293,
      "epoch": 5.21,
      "learning_rate": 0.00011757045454545454,
      "loss": 2.843,
      "step": 89700
    },
    {
      "activations/layer0_attention_weight_max": 16.150175094604492,
      "activations/layer0_attention_weight_min": -12.122173309326172,
      "activations/layer10_attention_weight_max": 33.96477508544922,
      "activations/layer10_attention_weight_min": -32.412784576416016,
      "activations/layer11_attention_weight_max": 33.01791763305664,
      "activations/layer11_attention_weight_min": -27.72028350830078,
      "activations/layer12_attention_weight_max": 24.820096969604492,
      "activations/layer12_attention_weight_min": -31.044418334960938,
      "activations/layer13_attention_weight_max": 41.7022705078125,
      "activations/layer13_attention_weight_min": -32.62614822387695,
      "activations/layer14_attention_weight_max": 54.79747009277344,
      "activations/layer14_attention_weight_min": -42.55915451049805,
      "activations/layer15_attention_weight_max": 38.816795349121094,
      "activations/layer15_attention_weight_min": -27.262258529663086,
      "activations/layer16_attention_weight_max": 37.16848373413086,
      "activations/layer16_attention_weight_min": -27.22736930847168,
      "activations/layer17_attention_weight_max": 40.7960090637207,
      "activations/layer17_attention_weight_min": -24.006994247436523,
      "activations/layer18_attention_weight_max": 37.36598587036133,
      "activations/layer18_attention_weight_min": -20.575122833251953,
      "activations/layer19_attention_weight_max": 41.50025939941406,
      "activations/layer19_attention_weight_min": -30.437891006469727,
      "activations/layer1_attention_weight_max": 17.3758487701416,
      "activations/layer1_attention_weight_min": -14.14956283569336,
      "activations/layer20_attention_weight_max": 31.372745513916016,
      "activations/layer20_attention_weight_min": -24.230148315429688,
      "activations/layer21_attention_weight_max": 32.83943176269531,
      "activations/layer21_attention_weight_min": -22.113319396972656,
      "activations/layer22_attention_weight_max": 46.42514419555664,
      "activations/layer22_attention_weight_min": -27.49976348876953,
      "activations/layer23_attention_weight_max": 33.96173095703125,
      "activations/layer23_attention_weight_min": -19.38348388671875,
      "activations/layer2_attention_weight_max": 32.20478057861328,
      "activations/layer2_attention_weight_min": -31.748220443725586,
      "activations/layer3_attention_weight_max": 94.56895446777344,
      "activations/layer3_attention_weight_min": -94.57743835449219,
      "activations/layer4_attention_weight_max": 133.8229217529297,
      "activations/layer4_attention_weight_min": -129.96311950683594,
      "activations/layer5_attention_weight_max": 86.96040344238281,
      "activations/layer5_attention_weight_min": -81.3530502319336,
      "activations/layer6_attention_weight_max": 56.880985260009766,
      "activations/layer6_attention_weight_min": -58.10234832763672,
      "activations/layer7_attention_weight_max": 88.24819946289062,
      "activations/layer7_attention_weight_min": -90.41051483154297,
      "activations/layer8_attention_weight_max": 41.7554817199707,
      "activations/layer8_attention_weight_min": -41.72759246826172,
      "activations/layer9_attention_weight_max": 37.85808563232422,
      "activations/layer9_attention_weight_min": -34.595924377441406,
      "epoch": 5.21,
      "learning_rate": 0.00011755151515151514,
      "loss": 2.8369,
      "step": 89750
    },
    {
      "activations/layer0_attention_weight_max": 15.810542106628418,
      "activations/layer0_attention_weight_min": -12.009255409240723,
      "activations/layer10_attention_weight_max": 32.95864486694336,
      "activations/layer10_attention_weight_min": -29.62936782836914,
      "activations/layer11_attention_weight_max": 31.993444442749023,
      "activations/layer11_attention_weight_min": -27.500083923339844,
      "activations/layer12_attention_weight_max": 58.76249694824219,
      "activations/layer12_attention_weight_min": -27.141077041625977,
      "activations/layer13_attention_weight_max": 45.357505798339844,
      "activations/layer13_attention_weight_min": -30.565521240234375,
      "activations/layer14_attention_weight_max": 62.16925811767578,
      "activations/layer14_attention_weight_min": -46.10295486450195,
      "activations/layer15_attention_weight_max": 40.50937271118164,
      "activations/layer15_attention_weight_min": -27.059173583984375,
      "activations/layer16_attention_weight_max": 42.71171569824219,
      "activations/layer16_attention_weight_min": -28.07613182067871,
      "activations/layer17_attention_weight_max": 36.63410949707031,
      "activations/layer17_attention_weight_min": -24.3139705657959,
      "activations/layer18_attention_weight_max": 33.916465759277344,
      "activations/layer18_attention_weight_min": -19.952266693115234,
      "activations/layer19_attention_weight_max": 44.13113021850586,
      "activations/layer19_attention_weight_min": -29.743066787719727,
      "activations/layer1_attention_weight_max": 16.99685287475586,
      "activations/layer1_attention_weight_min": -14.994593620300293,
      "activations/layer20_attention_weight_max": 33.15590286254883,
      "activations/layer20_attention_weight_min": -22.412778854370117,
      "activations/layer21_attention_weight_max": 37.186607360839844,
      "activations/layer21_attention_weight_min": -20.630857467651367,
      "activations/layer22_attention_weight_max": 49.262176513671875,
      "activations/layer22_attention_weight_min": -26.867841720581055,
      "activations/layer23_attention_weight_max": 31.10871696472168,
      "activations/layer23_attention_weight_min": -20.637889862060547,
      "activations/layer2_attention_weight_max": 32.734718322753906,
      "activations/layer2_attention_weight_min": -33.348140716552734,
      "activations/layer3_attention_weight_max": 89.76658630371094,
      "activations/layer3_attention_weight_min": -92.461181640625,
      "activations/layer4_attention_weight_max": 125.7050552368164,
      "activations/layer4_attention_weight_min": -123.44744110107422,
      "activations/layer5_attention_weight_max": 81.44697570800781,
      "activations/layer5_attention_weight_min": -77.00653839111328,
      "activations/layer6_attention_weight_max": 54.97013854980469,
      "activations/layer6_attention_weight_min": -56.95172882080078,
      "activations/layer7_attention_weight_max": 84.24708557128906,
      "activations/layer7_attention_weight_min": -84.240966796875,
      "activations/layer8_attention_weight_max": 38.27925491333008,
      "activations/layer8_attention_weight_min": -40.23353576660156,
      "activations/layer9_attention_weight_max": 34.21308517456055,
      "activations/layer9_attention_weight_min": -32.88358688354492,
      "epoch": 5.22,
      "learning_rate": 0.00011753257575757574,
      "loss": 2.8544,
      "step": 89800
    },
    {
      "activations/layer0_attention_weight_max": 16.720134735107422,
      "activations/layer0_attention_weight_min": -13.13749885559082,
      "activations/layer10_attention_weight_max": 32.39943313598633,
      "activations/layer10_attention_weight_min": -31.48881721496582,
      "activations/layer11_attention_weight_max": 29.68991470336914,
      "activations/layer11_attention_weight_min": -29.382526397705078,
      "activations/layer12_attention_weight_max": 27.686742782592773,
      "activations/layer12_attention_weight_min": -27.763221740722656,
      "activations/layer13_attention_weight_max": 40.087379455566406,
      "activations/layer13_attention_weight_min": -32.90693283081055,
      "activations/layer14_attention_weight_max": 55.66865539550781,
      "activations/layer14_attention_weight_min": -40.94691467285156,
      "activations/layer15_attention_weight_max": 37.33610153198242,
      "activations/layer15_attention_weight_min": -30.06387710571289,
      "activations/layer16_attention_weight_max": 38.36737060546875,
      "activations/layer16_attention_weight_min": -28.264841079711914,
      "activations/layer17_attention_weight_max": 36.620487213134766,
      "activations/layer17_attention_weight_min": -25.532188415527344,
      "activations/layer18_attention_weight_max": 32.010162353515625,
      "activations/layer18_attention_weight_min": -21.42839813232422,
      "activations/layer19_attention_weight_max": 39.34013366699219,
      "activations/layer19_attention_weight_min": -31.254501342773438,
      "activations/layer1_attention_weight_max": 16.501123428344727,
      "activations/layer1_attention_weight_min": -15.7507963180542,
      "activations/layer20_attention_weight_max": 31.969335556030273,
      "activations/layer20_attention_weight_min": -25.541013717651367,
      "activations/layer21_attention_weight_max": 30.212215423583984,
      "activations/layer21_attention_weight_min": -22.38762664794922,
      "activations/layer22_attention_weight_max": 46.26741409301758,
      "activations/layer22_attention_weight_min": -32.341861724853516,
      "activations/layer23_attention_weight_max": 32.977054595947266,
      "activations/layer23_attention_weight_min": -23.558307647705078,
      "activations/layer2_attention_weight_max": 31.69093132019043,
      "activations/layer2_attention_weight_min": -31.525754928588867,
      "activations/layer3_attention_weight_max": 88.09220123291016,
      "activations/layer3_attention_weight_min": -89.98171997070312,
      "activations/layer4_attention_weight_max": 129.6410369873047,
      "activations/layer4_attention_weight_min": -126.40958404541016,
      "activations/layer5_attention_weight_max": 84.20125579833984,
      "activations/layer5_attention_weight_min": -77.78255462646484,
      "activations/layer6_attention_weight_max": 57.2792854309082,
      "activations/layer6_attention_weight_min": -58.98111343383789,
      "activations/layer7_attention_weight_max": 88.97367858886719,
      "activations/layer7_attention_weight_min": -86.46786499023438,
      "activations/layer8_attention_weight_max": 41.05894470214844,
      "activations/layer8_attention_weight_min": -39.328346252441406,
      "activations/layer9_attention_weight_max": 36.498592376708984,
      "activations/layer9_attention_weight_min": -33.05400466918945,
      "epoch": 5.22,
      "learning_rate": 0.00011751401515151513,
      "loss": 2.8553,
      "step": 89850
    },
    {
      "activations/layer0_attention_weight_max": 15.775886535644531,
      "activations/layer0_attention_weight_min": -12.090324401855469,
      "activations/layer10_attention_weight_max": 35.108795166015625,
      "activations/layer10_attention_weight_min": -29.506486892700195,
      "activations/layer11_attention_weight_max": 34.72527313232422,
      "activations/layer11_attention_weight_min": -28.659378051757812,
      "activations/layer12_attention_weight_max": 36.355281829833984,
      "activations/layer12_attention_weight_min": -25.494165420532227,
      "activations/layer13_attention_weight_max": 49.392921447753906,
      "activations/layer13_attention_weight_min": -31.649879455566406,
      "activations/layer14_attention_weight_max": 70.4014663696289,
      "activations/layer14_attention_weight_min": -49.220699310302734,
      "activations/layer15_attention_weight_max": 41.91615676879883,
      "activations/layer15_attention_weight_min": -29.614253997802734,
      "activations/layer16_attention_weight_max": 40.187747955322266,
      "activations/layer16_attention_weight_min": -30.94578742980957,
      "activations/layer17_attention_weight_max": 39.03804397583008,
      "activations/layer17_attention_weight_min": -26.45036506652832,
      "activations/layer18_attention_weight_max": 32.57887649536133,
      "activations/layer18_attention_weight_min": -20.97881317138672,
      "activations/layer19_attention_weight_max": 42.302730560302734,
      "activations/layer19_attention_weight_min": -31.135990142822266,
      "activations/layer1_attention_weight_max": 17.46708869934082,
      "activations/layer1_attention_weight_min": -17.515159606933594,
      "activations/layer20_attention_weight_max": 37.64955139160156,
      "activations/layer20_attention_weight_min": -24.002735137939453,
      "activations/layer21_attention_weight_max": 32.16218948364258,
      "activations/layer21_attention_weight_min": -19.762226104736328,
      "activations/layer22_attention_weight_max": 52.8112907409668,
      "activations/layer22_attention_weight_min": -27.20343017578125,
      "activations/layer23_attention_weight_max": 33.68095397949219,
      "activations/layer23_attention_weight_min": -18.7889404296875,
      "activations/layer2_attention_weight_max": 31.42011260986328,
      "activations/layer2_attention_weight_min": -31.094799041748047,
      "activations/layer3_attention_weight_max": 89.6558837890625,
      "activations/layer3_attention_weight_min": -90.29718017578125,
      "activations/layer4_attention_weight_max": 127.6551513671875,
      "activations/layer4_attention_weight_min": -124.96410369873047,
      "activations/layer5_attention_weight_max": 81.96851348876953,
      "activations/layer5_attention_weight_min": -76.81886291503906,
      "activations/layer6_attention_weight_max": 55.59033966064453,
      "activations/layer6_attention_weight_min": -53.38032913208008,
      "activations/layer7_attention_weight_max": 82.0062484741211,
      "activations/layer7_attention_weight_min": -81.48563385009766,
      "activations/layer8_attention_weight_max": 38.57730484008789,
      "activations/layer8_attention_weight_min": -40.054927825927734,
      "activations/layer9_attention_weight_max": 35.98480224609375,
      "activations/layer9_attention_weight_min": -39.85432815551758,
      "epoch": 5.22,
      "learning_rate": 0.00011749507575757575,
      "loss": 2.8472,
      "step": 89900
    },
    {
      "activations/layer0_attention_weight_max": 17.049589157104492,
      "activations/layer0_attention_weight_min": -12.297236442565918,
      "activations/layer10_attention_weight_max": 33.603904724121094,
      "activations/layer10_attention_weight_min": -29.405120849609375,
      "activations/layer11_attention_weight_max": 29.690235137939453,
      "activations/layer11_attention_weight_min": -26.889211654663086,
      "activations/layer12_attention_weight_max": 27.36703109741211,
      "activations/layer12_attention_weight_min": -27.430910110473633,
      "activations/layer13_attention_weight_max": 40.09558868408203,
      "activations/layer13_attention_weight_min": -30.154983520507812,
      "activations/layer14_attention_weight_max": 55.58692169189453,
      "activations/layer14_attention_weight_min": -40.31519317626953,
      "activations/layer15_attention_weight_max": 37.1269416809082,
      "activations/layer15_attention_weight_min": -28.41023063659668,
      "activations/layer16_attention_weight_max": 36.6088752746582,
      "activations/layer16_attention_weight_min": -27.745046615600586,
      "activations/layer17_attention_weight_max": 35.50449752807617,
      "activations/layer17_attention_weight_min": -25.0788516998291,
      "activations/layer18_attention_weight_max": 31.507131576538086,
      "activations/layer18_attention_weight_min": -19.799678802490234,
      "activations/layer19_attention_weight_max": 37.842105865478516,
      "activations/layer19_attention_weight_min": -29.27235984802246,
      "activations/layer1_attention_weight_max": 16.693906784057617,
      "activations/layer1_attention_weight_min": -15.30001163482666,
      "activations/layer20_attention_weight_max": 30.649456024169922,
      "activations/layer20_attention_weight_min": -24.920743942260742,
      "activations/layer21_attention_weight_max": 28.82945442199707,
      "activations/layer21_attention_weight_min": -21.744441986083984,
      "activations/layer22_attention_weight_max": 48.360660552978516,
      "activations/layer22_attention_weight_min": -28.23174285888672,
      "activations/layer23_attention_weight_max": 33.57561492919922,
      "activations/layer23_attention_weight_min": -20.785934448242188,
      "activations/layer2_attention_weight_max": 31.994855880737305,
      "activations/layer2_attention_weight_min": -31.19715118408203,
      "activations/layer3_attention_weight_max": 88.02287292480469,
      "activations/layer3_attention_weight_min": -91.00296783447266,
      "activations/layer4_attention_weight_max": 125.56485748291016,
      "activations/layer4_attention_weight_min": -120.79418182373047,
      "activations/layer5_attention_weight_max": 76.68766784667969,
      "activations/layer5_attention_weight_min": -74.43946838378906,
      "activations/layer6_attention_weight_max": 53.72322082519531,
      "activations/layer6_attention_weight_min": -55.037017822265625,
      "activations/layer7_attention_weight_max": 80.25697326660156,
      "activations/layer7_attention_weight_min": -78.13050079345703,
      "activations/layer8_attention_weight_max": 36.9078369140625,
      "activations/layer8_attention_weight_min": -39.226806640625,
      "activations/layer9_attention_weight_max": 33.16651916503906,
      "activations/layer9_attention_weight_min": -35.02558517456055,
      "epoch": 5.23,
      "learning_rate": 0.00011747613636363635,
      "loss": 2.8484,
      "step": 89950
    },
    {
      "activations/layer0_attention_weight_max": 16.931110382080078,
      "activations/layer0_attention_weight_min": -11.712121963500977,
      "activations/layer10_attention_weight_max": 33.74343490600586,
      "activations/layer10_attention_weight_min": -30.306955337524414,
      "activations/layer11_attention_weight_max": 31.226613998413086,
      "activations/layer11_attention_weight_min": -28.029638290405273,
      "activations/layer12_attention_weight_max": 30.350326538085938,
      "activations/layer12_attention_weight_min": -27.74300765991211,
      "activations/layer13_attention_weight_max": 44.66321563720703,
      "activations/layer13_attention_weight_min": -34.2170524597168,
      "activations/layer14_attention_weight_max": 63.378143310546875,
      "activations/layer14_attention_weight_min": -45.61708450317383,
      "activations/layer15_attention_weight_max": 40.38629150390625,
      "activations/layer15_attention_weight_min": -28.438079833984375,
      "activations/layer16_attention_weight_max": 37.802642822265625,
      "activations/layer16_attention_weight_min": -30.506103515625,
      "activations/layer17_attention_weight_max": 39.79579162597656,
      "activations/layer17_attention_weight_min": -25.62791633605957,
      "activations/layer18_attention_weight_max": 31.804723739624023,
      "activations/layer18_attention_weight_min": -18.798227310180664,
      "activations/layer19_attention_weight_max": 42.219642639160156,
      "activations/layer19_attention_weight_min": -30.024826049804688,
      "activations/layer1_attention_weight_max": 17.092111587524414,
      "activations/layer1_attention_weight_min": -13.989832878112793,
      "activations/layer20_attention_weight_max": 34.85614776611328,
      "activations/layer20_attention_weight_min": -22.804973602294922,
      "activations/layer21_attention_weight_max": 37.06885528564453,
      "activations/layer21_attention_weight_min": -20.125160217285156,
      "activations/layer22_attention_weight_max": 51.17595291137695,
      "activations/layer22_attention_weight_min": -29.717041015625,
      "activations/layer23_attention_weight_max": 36.40385055541992,
      "activations/layer23_attention_weight_min": -20.851341247558594,
      "activations/layer2_attention_weight_max": 32.19047164916992,
      "activations/layer2_attention_weight_min": -30.516822814941406,
      "activations/layer3_attention_weight_max": 86.99237060546875,
      "activations/layer3_attention_weight_min": -90.3900146484375,
      "activations/layer4_attention_weight_max": 122.1875228881836,
      "activations/layer4_attention_weight_min": -125.333740234375,
      "activations/layer5_attention_weight_max": 75.28822326660156,
      "activations/layer5_attention_weight_min": -76.62198638916016,
      "activations/layer6_attention_weight_max": 54.9787483215332,
      "activations/layer6_attention_weight_min": -55.06355285644531,
      "activations/layer7_attention_weight_max": 80.26238250732422,
      "activations/layer7_attention_weight_min": -82.55892181396484,
      "activations/layer8_attention_weight_max": 38.64482879638672,
      "activations/layer8_attention_weight_min": -39.554962158203125,
      "activations/layer9_attention_weight_max": 36.58743667602539,
      "activations/layer9_attention_weight_min": -34.245357513427734,
      "epoch": 5.23,
      "learning_rate": 0.00011745719696969697,
      "loss": 2.8434,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.5687,
      "eval_samples_per_second": 501.128,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.5687,
      "eval_openwebtext_samples_per_second": 501.128,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 1.9994,
      "eval_wikitext_samples_per_second": 228.065,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_lambada_loss": 2.6953125,
      "eval_lambada_ppl": 14.81014620089916,
      "eval_lambada_runtime": 9.5769,
      "eval_lambada_samples_per_second": 508.411,
      "step": 90000
    },
    {
      "activations/layer0_attention_weight_max": 16.816776275634766,
      "activations/layer0_attention_weight_min": -11.188706398010254,
      "activations/layer10_attention_weight_max": 33.08554458618164,
      "activations/layer10_attention_weight_min": -30.73468017578125,
      "activations/layer11_attention_weight_max": 34.79690933227539,
      "activations/layer11_attention_weight_min": -29.22414207458496,
      "activations/layer12_attention_weight_max": 27.08942222595215,
      "activations/layer12_attention_weight_min": -26.242069244384766,
      "activations/layer13_attention_weight_max": 46.3598747253418,
      "activations/layer13_attention_weight_min": -34.18517303466797,
      "activations/layer14_attention_weight_max": 58.455387115478516,
      "activations/layer14_attention_weight_min": -47.013702392578125,
      "activations/layer15_attention_weight_max": 43.1947135925293,
      "activations/layer15_attention_weight_min": -30.427377700805664,
      "activations/layer16_attention_weight_max": 38.71610641479492,
      "activations/layer16_attention_weight_min": -27.48759651184082,
      "activations/layer17_attention_weight_max": 37.73575210571289,
      "activations/layer17_attention_weight_min": -25.499576568603516,
      "activations/layer18_attention_weight_max": 32.80405044555664,
      "activations/layer18_attention_weight_min": -20.478300094604492,
      "activations/layer19_attention_weight_max": 43.78290557861328,
      "activations/layer19_attention_weight_min": -29.27921485900879,
      "activations/layer1_attention_weight_max": 18.026315689086914,
      "activations/layer1_attention_weight_min": -14.551309585571289,
      "activations/layer20_attention_weight_max": 36.7954216003418,
      "activations/layer20_attention_weight_min": -22.251842498779297,
      "activations/layer21_attention_weight_max": 33.38484573364258,
      "activations/layer21_attention_weight_min": -20.187328338623047,
      "activations/layer22_attention_weight_max": 51.86451721191406,
      "activations/layer22_attention_weight_min": -29.317428588867188,
      "activations/layer23_attention_weight_max": 34.52284240722656,
      "activations/layer23_attention_weight_min": -21.344463348388672,
      "activations/layer2_attention_weight_max": 33.76166534423828,
      "activations/layer2_attention_weight_min": -31.306018829345703,
      "activations/layer3_attention_weight_max": 91.88984680175781,
      "activations/layer3_attention_weight_min": -90.85150146484375,
      "activations/layer4_attention_weight_max": 133.14291381835938,
      "activations/layer4_attention_weight_min": -127.5173110961914,
      "activations/layer5_attention_weight_max": 79.45112609863281,
      "activations/layer5_attention_weight_min": -78.11824035644531,
      "activations/layer6_attention_weight_max": 57.34765625,
      "activations/layer6_attention_weight_min": -60.21919631958008,
      "activations/layer7_attention_weight_max": 87.2191390991211,
      "activations/layer7_attention_weight_min": -88.66515350341797,
      "activations/layer8_attention_weight_max": 40.729793548583984,
      "activations/layer8_attention_weight_min": -39.47547149658203,
      "activations/layer9_attention_weight_max": 34.931373596191406,
      "activations/layer9_attention_weight_min": -36.27613067626953,
      "epoch": 5.23,
      "learning_rate": 0.00011743825757575757,
      "loss": 2.8424,
      "step": 90050
    },
    {
      "activations/layer0_attention_weight_max": 16.482040405273438,
      "activations/layer0_attention_weight_min": -11.345405578613281,
      "activations/layer10_attention_weight_max": 32.03267288208008,
      "activations/layer10_attention_weight_min": -29.01320457458496,
      "activations/layer11_attention_weight_max": 36.309600830078125,
      "activations/layer11_attention_weight_min": -27.816648483276367,
      "activations/layer12_attention_weight_max": 38.265419006347656,
      "activations/layer12_attention_weight_min": -28.618087768554688,
      "activations/layer13_attention_weight_max": 46.11467742919922,
      "activations/layer13_attention_weight_min": -33.30355453491211,
      "activations/layer14_attention_weight_max": 61.21235275268555,
      "activations/layer14_attention_weight_min": -46.70505905151367,
      "activations/layer15_attention_weight_max": 40.90605926513672,
      "activations/layer15_attention_weight_min": -28.78729248046875,
      "activations/layer16_attention_weight_max": 40.50540542602539,
      "activations/layer16_attention_weight_min": -27.60054588317871,
      "activations/layer17_attention_weight_max": 42.67698287963867,
      "activations/layer17_attention_weight_min": -24.892009735107422,
      "activations/layer18_attention_weight_max": 39.83171463012695,
      "activations/layer18_attention_weight_min": -21.99683380126953,
      "activations/layer19_attention_weight_max": 43.366641998291016,
      "activations/layer19_attention_weight_min": -29.76993751525879,
      "activations/layer1_attention_weight_max": 18.341522216796875,
      "activations/layer1_attention_weight_min": -15.340250015258789,
      "activations/layer20_attention_weight_max": 37.96621322631836,
      "activations/layer20_attention_weight_min": -23.680442810058594,
      "activations/layer21_attention_weight_max": 34.47967529296875,
      "activations/layer21_attention_weight_min": -23.11598777770996,
      "activations/layer22_attention_weight_max": 50.75241470336914,
      "activations/layer22_attention_weight_min": -27.981388092041016,
      "activations/layer23_attention_weight_max": 34.87535095214844,
      "activations/layer23_attention_weight_min": -20.31321144104004,
      "activations/layer2_attention_weight_max": 34.11359786987305,
      "activations/layer2_attention_weight_min": -32.2867317199707,
      "activations/layer3_attention_weight_max": 87.75895690917969,
      "activations/layer3_attention_weight_min": -90.17196655273438,
      "activations/layer4_attention_weight_max": 125.8730239868164,
      "activations/layer4_attention_weight_min": -125.84525299072266,
      "activations/layer5_attention_weight_max": 78.48465728759766,
      "activations/layer5_attention_weight_min": -76.66241455078125,
      "activations/layer6_attention_weight_max": 53.40117645263672,
      "activations/layer6_attention_weight_min": -55.49897003173828,
      "activations/layer7_attention_weight_max": 79.91799926757812,
      "activations/layer7_attention_weight_min": -81.81080627441406,
      "activations/layer8_attention_weight_max": 39.093109130859375,
      "activations/layer8_attention_weight_min": -38.25893020629883,
      "activations/layer9_attention_weight_max": 33.91961669921875,
      "activations/layer9_attention_weight_min": -32.996944427490234,
      "epoch": 5.24,
      "learning_rate": 0.00011741931818181817,
      "loss": 2.8413,
      "step": 90100
    },
    {
      "activations/layer0_attention_weight_max": 16.660429000854492,
      "activations/layer0_attention_weight_min": -11.766142845153809,
      "activations/layer10_attention_weight_max": 38.37604904174805,
      "activations/layer10_attention_weight_min": -33.35029602050781,
      "activations/layer11_attention_weight_max": 33.65943908691406,
      "activations/layer11_attention_weight_min": -26.613630294799805,
      "activations/layer12_attention_weight_max": 29.713232040405273,
      "activations/layer12_attention_weight_min": -28.683382034301758,
      "activations/layer13_attention_weight_max": 42.69121170043945,
      "activations/layer13_attention_weight_min": -30.48424530029297,
      "activations/layer14_attention_weight_max": 62.740142822265625,
      "activations/layer14_attention_weight_min": -44.29379653930664,
      "activations/layer15_attention_weight_max": 47.00838088989258,
      "activations/layer15_attention_weight_min": -30.04277992248535,
      "activations/layer16_attention_weight_max": 35.80042266845703,
      "activations/layer16_attention_weight_min": -26.56797981262207,
      "activations/layer17_attention_weight_max": 41.08756637573242,
      "activations/layer17_attention_weight_min": -25.07059669494629,
      "activations/layer18_attention_weight_max": 32.30358123779297,
      "activations/layer18_attention_weight_min": -21.203880310058594,
      "activations/layer19_attention_weight_max": 42.827327728271484,
      "activations/layer19_attention_weight_min": -28.170272827148438,
      "activations/layer1_attention_weight_max": 17.132457733154297,
      "activations/layer1_attention_weight_min": -14.051942825317383,
      "activations/layer20_attention_weight_max": 33.00641632080078,
      "activations/layer20_attention_weight_min": -22.34697723388672,
      "activations/layer21_attention_weight_max": 32.40865707397461,
      "activations/layer21_attention_weight_min": -19.89259910583496,
      "activations/layer22_attention_weight_max": 50.98157501220703,
      "activations/layer22_attention_weight_min": -26.05115509033203,
      "activations/layer23_attention_weight_max": 35.32779312133789,
      "activations/layer23_attention_weight_min": -20.938026428222656,
      "activations/layer2_attention_weight_max": 33.58969497680664,
      "activations/layer2_attention_weight_min": -33.776248931884766,
      "activations/layer3_attention_weight_max": 93.24796295166016,
      "activations/layer3_attention_weight_min": -93.78992462158203,
      "activations/layer4_attention_weight_max": 128.09449768066406,
      "activations/layer4_attention_weight_min": -125.42339324951172,
      "activations/layer5_attention_weight_max": 83.33198547363281,
      "activations/layer5_attention_weight_min": -76.48440551757812,
      "activations/layer6_attention_weight_max": 58.2184944152832,
      "activations/layer6_attention_weight_min": -57.46525192260742,
      "activations/layer7_attention_weight_max": 87.37931823730469,
      "activations/layer7_attention_weight_min": -83.62013244628906,
      "activations/layer8_attention_weight_max": 44.3135986328125,
      "activations/layer8_attention_weight_min": -41.523109436035156,
      "activations/layer9_attention_weight_max": 41.094398498535156,
      "activations/layer9_attention_weight_min": -36.9146614074707,
      "epoch": 5.24,
      "learning_rate": 0.00011740037878787878,
      "loss": 2.8411,
      "step": 90150
    },
    {
      "activations/layer0_attention_weight_max": 17.32039451599121,
      "activations/layer0_attention_weight_min": -12.500046730041504,
      "activations/layer10_attention_weight_max": 40.50545120239258,
      "activations/layer10_attention_weight_min": -33.38719177246094,
      "activations/layer11_attention_weight_max": 42.74642562866211,
      "activations/layer11_attention_weight_min": -30.681076049804688,
      "activations/layer12_attention_weight_max": 26.892765045166016,
      "activations/layer12_attention_weight_min": -28.49291229248047,
      "activations/layer13_attention_weight_max": 43.413917541503906,
      "activations/layer13_attention_weight_min": -32.48556900024414,
      "activations/layer14_attention_weight_max": 76.22937774658203,
      "activations/layer14_attention_weight_min": -48.41047286987305,
      "activations/layer15_attention_weight_max": 44.1782112121582,
      "activations/layer15_attention_weight_min": -28.181344985961914,
      "activations/layer16_attention_weight_max": 40.36612319946289,
      "activations/layer16_attention_weight_min": -29.106658935546875,
      "activations/layer17_attention_weight_max": 38.93266677856445,
      "activations/layer17_attention_weight_min": -24.672100067138672,
      "activations/layer18_attention_weight_max": 29.54730796813965,
      "activations/layer18_attention_weight_min": -21.12584686279297,
      "activations/layer19_attention_weight_max": 38.04227828979492,
      "activations/layer19_attention_weight_min": -31.847543716430664,
      "activations/layer1_attention_weight_max": 17.549501419067383,
      "activations/layer1_attention_weight_min": -16.392417907714844,
      "activations/layer20_attention_weight_max": 33.30542755126953,
      "activations/layer20_attention_weight_min": -23.70640754699707,
      "activations/layer21_attention_weight_max": 34.260677337646484,
      "activations/layer21_attention_weight_min": -20.158885955810547,
      "activations/layer22_attention_weight_max": 52.35747146606445,
      "activations/layer22_attention_weight_min": -27.03690528869629,
      "activations/layer23_attention_weight_max": 35.208946228027344,
      "activations/layer23_attention_weight_min": -18.996793746948242,
      "activations/layer2_attention_weight_max": 33.16719436645508,
      "activations/layer2_attention_weight_min": -32.893882751464844,
      "activations/layer3_attention_weight_max": 94.31980895996094,
      "activations/layer3_attention_weight_min": -97.26338195800781,
      "activations/layer4_attention_weight_max": 134.28411865234375,
      "activations/layer4_attention_weight_min": -139.81898498535156,
      "activations/layer5_attention_weight_max": 83.73649597167969,
      "activations/layer5_attention_weight_min": -84.2653579711914,
      "activations/layer6_attention_weight_max": 55.73345184326172,
      "activations/layer6_attention_weight_min": -58.85913848876953,
      "activations/layer7_attention_weight_max": 93.9231185913086,
      "activations/layer7_attention_weight_min": -88.40943908691406,
      "activations/layer8_attention_weight_max": 42.257389068603516,
      "activations/layer8_attention_weight_min": -41.16682434082031,
      "activations/layer9_attention_weight_max": 38.11716842651367,
      "activations/layer9_attention_weight_min": -35.57741928100586,
      "epoch": 5.24,
      "learning_rate": 0.00011738143939393938,
      "loss": 2.8507,
      "step": 90200
    },
    {
      "activations/layer0_attention_weight_max": 16.7833251953125,
      "activations/layer0_attention_weight_min": -12.45092487335205,
      "activations/layer10_attention_weight_max": 35.646453857421875,
      "activations/layer10_attention_weight_min": -30.413686752319336,
      "activations/layer11_attention_weight_max": 37.541038513183594,
      "activations/layer11_attention_weight_min": -27.982994079589844,
      "activations/layer12_attention_weight_max": 49.422767639160156,
      "activations/layer12_attention_weight_min": -33.69099807739258,
      "activations/layer13_attention_weight_max": 58.50139617919922,
      "activations/layer13_attention_weight_min": -35.96795654296875,
      "activations/layer14_attention_weight_max": 59.34897232055664,
      "activations/layer14_attention_weight_min": -43.997276306152344,
      "activations/layer15_attention_weight_max": 41.8614616394043,
      "activations/layer15_attention_weight_min": -29.42984390258789,
      "activations/layer16_attention_weight_max": 38.829566955566406,
      "activations/layer16_attention_weight_min": -27.277210235595703,
      "activations/layer17_attention_weight_max": 43.81438446044922,
      "activations/layer17_attention_weight_min": -23.92837905883789,
      "activations/layer18_attention_weight_max": 38.19609832763672,
      "activations/layer18_attention_weight_min": -21.32881736755371,
      "activations/layer19_attention_weight_max": 44.0976676940918,
      "activations/layer19_attention_weight_min": -29.341018676757812,
      "activations/layer1_attention_weight_max": 17.37251853942871,
      "activations/layer1_attention_weight_min": -13.750577926635742,
      "activations/layer20_attention_weight_max": 39.09618377685547,
      "activations/layer20_attention_weight_min": -25.736255645751953,
      "activations/layer21_attention_weight_max": 34.40535354614258,
      "activations/layer21_attention_weight_min": -19.952089309692383,
      "activations/layer22_attention_weight_max": 59.53086853027344,
      "activations/layer22_attention_weight_min": -26.95684242248535,
      "activations/layer23_attention_weight_max": 36.1233024597168,
      "activations/layer23_attention_weight_min": -18.45798110961914,
      "activations/layer2_attention_weight_max": 32.67279052734375,
      "activations/layer2_attention_weight_min": -31.951993942260742,
      "activations/layer3_attention_weight_max": 95.18447875976562,
      "activations/layer3_attention_weight_min": -92.57003021240234,
      "activations/layer4_attention_weight_max": 132.53248596191406,
      "activations/layer4_attention_weight_min": -128.54132080078125,
      "activations/layer5_attention_weight_max": 82.05778503417969,
      "activations/layer5_attention_weight_min": -73.85994720458984,
      "activations/layer6_attention_weight_max": 57.82376480102539,
      "activations/layer6_attention_weight_min": -53.888797760009766,
      "activations/layer7_attention_weight_max": 90.54849243164062,
      "activations/layer7_attention_weight_min": -84.33878326416016,
      "activations/layer8_attention_weight_max": 41.6674690246582,
      "activations/layer8_attention_weight_min": -39.51201629638672,
      "activations/layer9_attention_weight_max": 37.6006965637207,
      "activations/layer9_attention_weight_min": -41.17271041870117,
      "epoch": 5.24,
      "learning_rate": 0.00011736249999999999,
      "loss": 2.8513,
      "step": 90250
    },
    {
      "activations/layer0_attention_weight_max": 16.36493492126465,
      "activations/layer0_attention_weight_min": -12.463479042053223,
      "activations/layer10_attention_weight_max": 33.86128616333008,
      "activations/layer10_attention_weight_min": -30.464536666870117,
      "activations/layer11_attention_weight_max": 34.88739776611328,
      "activations/layer11_attention_weight_min": -28.206153869628906,
      "activations/layer12_attention_weight_max": 36.38713836669922,
      "activations/layer12_attention_weight_min": -27.658329010009766,
      "activations/layer13_attention_weight_max": 42.11837387084961,
      "activations/layer13_attention_weight_min": -32.820308685302734,
      "activations/layer14_attention_weight_max": 85.70884704589844,
      "activations/layer14_attention_weight_min": -57.579349517822266,
      "activations/layer15_attention_weight_max": 40.38483810424805,
      "activations/layer15_attention_weight_min": -28.54494285583496,
      "activations/layer16_attention_weight_max": 39.223575592041016,
      "activations/layer16_attention_weight_min": -28.477645874023438,
      "activations/layer17_attention_weight_max": 39.50865936279297,
      "activations/layer17_attention_weight_min": -25.070751190185547,
      "activations/layer18_attention_weight_max": 29.672592163085938,
      "activations/layer18_attention_weight_min": -19.99903106689453,
      "activations/layer19_attention_weight_max": 38.51435852050781,
      "activations/layer19_attention_weight_min": -27.214818954467773,
      "activations/layer1_attention_weight_max": 17.33877944946289,
      "activations/layer1_attention_weight_min": -14.88894271850586,
      "activations/layer20_attention_weight_max": 33.54554748535156,
      "activations/layer20_attention_weight_min": -22.385007858276367,
      "activations/layer21_attention_weight_max": 31.52947998046875,
      "activations/layer21_attention_weight_min": -20.711647033691406,
      "activations/layer22_attention_weight_max": 48.36627197265625,
      "activations/layer22_attention_weight_min": -27.50579833984375,
      "activations/layer23_attention_weight_max": 29.931156158447266,
      "activations/layer23_attention_weight_min": -20.399658203125,
      "activations/layer2_attention_weight_max": 32.02056884765625,
      "activations/layer2_attention_weight_min": -31.430477142333984,
      "activations/layer3_attention_weight_max": 88.18383026123047,
      "activations/layer3_attention_weight_min": -91.83548736572266,
      "activations/layer4_attention_weight_max": 129.26174926757812,
      "activations/layer4_attention_weight_min": -128.42271423339844,
      "activations/layer5_attention_weight_max": 79.30853271484375,
      "activations/layer5_attention_weight_min": -76.8485107421875,
      "activations/layer6_attention_weight_max": 55.266624450683594,
      "activations/layer6_attention_weight_min": -56.04301834106445,
      "activations/layer7_attention_weight_max": 91.05500030517578,
      "activations/layer7_attention_weight_min": -82.56754302978516,
      "activations/layer8_attention_weight_max": 43.10676193237305,
      "activations/layer8_attention_weight_min": -39.51654815673828,
      "activations/layer9_attention_weight_max": 37.508174896240234,
      "activations/layer9_attention_weight_min": -33.966190338134766,
      "epoch": 5.25,
      "learning_rate": 0.0001173435606060606,
      "loss": 2.8328,
      "step": 90300
    },
    {
      "activations/layer0_attention_weight_max": 16.731266021728516,
      "activations/layer0_attention_weight_min": -11.650203704833984,
      "activations/layer10_attention_weight_max": 33.951148986816406,
      "activations/layer10_attention_weight_min": -30.988126754760742,
      "activations/layer11_attention_weight_max": 32.12493133544922,
      "activations/layer11_attention_weight_min": -27.365863800048828,
      "activations/layer12_attention_weight_max": 30.943544387817383,
      "activations/layer12_attention_weight_min": -25.77229118347168,
      "activations/layer13_attention_weight_max": 40.85316848754883,
      "activations/layer13_attention_weight_min": -31.4916934967041,
      "activations/layer14_attention_weight_max": 69.0652847290039,
      "activations/layer14_attention_weight_min": -46.632259368896484,
      "activations/layer15_attention_weight_max": 46.24599075317383,
      "activations/layer15_attention_weight_min": -28.532756805419922,
      "activations/layer16_attention_weight_max": 42.733943939208984,
      "activations/layer16_attention_weight_min": -27.015336990356445,
      "activations/layer17_attention_weight_max": 43.1391487121582,
      "activations/layer17_attention_weight_min": -26.25811195373535,
      "activations/layer18_attention_weight_max": 35.91761779785156,
      "activations/layer18_attention_weight_min": -21.24698257446289,
      "activations/layer19_attention_weight_max": 43.454647064208984,
      "activations/layer19_attention_weight_min": -30.652090072631836,
      "activations/layer1_attention_weight_max": 18.29982566833496,
      "activations/layer1_attention_weight_min": -17.37158203125,
      "activations/layer20_attention_weight_max": 37.52732467651367,
      "activations/layer20_attention_weight_min": -23.530275344848633,
      "activations/layer21_attention_weight_max": 32.53130340576172,
      "activations/layer21_attention_weight_min": -21.361230850219727,
      "activations/layer22_attention_weight_max": 53.254703521728516,
      "activations/layer22_attention_weight_min": -28.72045135498047,
      "activations/layer23_attention_weight_max": 34.50840377807617,
      "activations/layer23_attention_weight_min": -21.151409149169922,
      "activations/layer2_attention_weight_max": 33.420654296875,
      "activations/layer2_attention_weight_min": -34.03132629394531,
      "activations/layer3_attention_weight_max": 89.22254943847656,
      "activations/layer3_attention_weight_min": -93.05261993408203,
      "activations/layer4_attention_weight_max": 130.56219482421875,
      "activations/layer4_attention_weight_min": -130.7084197998047,
      "activations/layer5_attention_weight_max": 81.43771362304688,
      "activations/layer5_attention_weight_min": -81.70701599121094,
      "activations/layer6_attention_weight_max": 55.83990478515625,
      "activations/layer6_attention_weight_min": -56.666690826416016,
      "activations/layer7_attention_weight_max": 88.26461029052734,
      "activations/layer7_attention_weight_min": -85.85673522949219,
      "activations/layer8_attention_weight_max": 40.12245559692383,
      "activations/layer8_attention_weight_min": -40.399078369140625,
      "activations/layer9_attention_weight_max": 36.15335464477539,
      "activations/layer9_attention_weight_min": -34.132408142089844,
      "epoch": 5.25,
      "learning_rate": 0.00011732462121212119,
      "loss": 2.8351,
      "step": 90350
    },
    {
      "activations/layer0_attention_weight_max": 16.928876876831055,
      "activations/layer0_attention_weight_min": -13.052980422973633,
      "activations/layer10_attention_weight_max": 33.73138427734375,
      "activations/layer10_attention_weight_min": -28.25750732421875,
      "activations/layer11_attention_weight_max": 31.826007843017578,
      "activations/layer11_attention_weight_min": -26.599584579467773,
      "activations/layer12_attention_weight_max": 27.571210861206055,
      "activations/layer12_attention_weight_min": -29.606586456298828,
      "activations/layer13_attention_weight_max": 45.08118438720703,
      "activations/layer13_attention_weight_min": -30.5191593170166,
      "activations/layer14_attention_weight_max": 55.017459869384766,
      "activations/layer14_attention_weight_min": -39.300445556640625,
      "activations/layer15_attention_weight_max": 42.31443405151367,
      "activations/layer15_attention_weight_min": -30.25724983215332,
      "activations/layer16_attention_weight_max": 35.41146469116211,
      "activations/layer16_attention_weight_min": -26.915205001831055,
      "activations/layer17_attention_weight_max": 38.73201370239258,
      "activations/layer17_attention_weight_min": -24.55347442626953,
      "activations/layer18_attention_weight_max": 29.48774528503418,
      "activations/layer18_attention_weight_min": -19.491226196289062,
      "activations/layer19_attention_weight_max": 38.79470443725586,
      "activations/layer19_attention_weight_min": -27.645971298217773,
      "activations/layer1_attention_weight_max": 16.764766693115234,
      "activations/layer1_attention_weight_min": -14.026637077331543,
      "activations/layer20_attention_weight_max": 30.81561279296875,
      "activations/layer20_attention_weight_min": -25.66524314880371,
      "activations/layer21_attention_weight_max": 29.80965232849121,
      "activations/layer21_attention_weight_min": -22.53350067138672,
      "activations/layer22_attention_weight_max": 54.259464263916016,
      "activations/layer22_attention_weight_min": -26.721914291381836,
      "activations/layer23_attention_weight_max": 33.193603515625,
      "activations/layer23_attention_weight_min": -21.008235931396484,
      "activations/layer2_attention_weight_max": 33.82191848754883,
      "activations/layer2_attention_weight_min": -32.19512176513672,
      "activations/layer3_attention_weight_max": 96.11060333251953,
      "activations/layer3_attention_weight_min": -94.27140808105469,
      "activations/layer4_attention_weight_max": 135.27398681640625,
      "activations/layer4_attention_weight_min": -137.54063415527344,
      "activations/layer5_attention_weight_max": 83.19290924072266,
      "activations/layer5_attention_weight_min": -82.65404510498047,
      "activations/layer6_attention_weight_max": 58.11994552612305,
      "activations/layer6_attention_weight_min": -57.3202018737793,
      "activations/layer7_attention_weight_max": 85.80361938476562,
      "activations/layer7_attention_weight_min": -83.7924575805664,
      "activations/layer8_attention_weight_max": 39.555721282958984,
      "activations/layer8_attention_weight_min": -39.09181594848633,
      "activations/layer9_attention_weight_max": 36.862152099609375,
      "activations/layer9_attention_weight_min": -34.59281539916992,
      "epoch": 5.25,
      "learning_rate": 0.0001173056818181818,
      "loss": 2.8431,
      "step": 90400
    },
    {
      "activations/layer0_attention_weight_max": 16.247032165527344,
      "activations/layer0_attention_weight_min": -10.695467948913574,
      "activations/layer10_attention_weight_max": 32.769691467285156,
      "activations/layer10_attention_weight_min": -29.12918472290039,
      "activations/layer11_attention_weight_max": 31.15180206298828,
      "activations/layer11_attention_weight_min": -25.693870544433594,
      "activations/layer12_attention_weight_max": 25.612110137939453,
      "activations/layer12_attention_weight_min": -24.830020904541016,
      "activations/layer13_attention_weight_max": 44.704078674316406,
      "activations/layer13_attention_weight_min": -31.876388549804688,
      "activations/layer14_attention_weight_max": 60.25242233276367,
      "activations/layer14_attention_weight_min": -44.8135871887207,
      "activations/layer15_attention_weight_max": 37.033050537109375,
      "activations/layer15_attention_weight_min": -27.35292625427246,
      "activations/layer16_attention_weight_max": 39.07256317138672,
      "activations/layer16_attention_weight_min": -27.582210540771484,
      "activations/layer17_attention_weight_max": 41.55226516723633,
      "activations/layer17_attention_weight_min": -25.791818618774414,
      "activations/layer18_attention_weight_max": 38.188438415527344,
      "activations/layer18_attention_weight_min": -21.132164001464844,
      "activations/layer19_attention_weight_max": 44.043704986572266,
      "activations/layer19_attention_weight_min": -29.17569351196289,
      "activations/layer1_attention_weight_max": 17.560588836669922,
      "activations/layer1_attention_weight_min": -16.150299072265625,
      "activations/layer20_attention_weight_max": 32.21157455444336,
      "activations/layer20_attention_weight_min": -25.475751876831055,
      "activations/layer21_attention_weight_max": 28.892135620117188,
      "activations/layer21_attention_weight_min": -22.08971405029297,
      "activations/layer22_attention_weight_max": 49.77434539794922,
      "activations/layer22_attention_weight_min": -30.41139793395996,
      "activations/layer23_attention_weight_max": 32.844913482666016,
      "activations/layer23_attention_weight_min": -23.48121452331543,
      "activations/layer2_attention_weight_max": 32.79853820800781,
      "activations/layer2_attention_weight_min": -32.03656768798828,
      "activations/layer3_attention_weight_max": 91.62914276123047,
      "activations/layer3_attention_weight_min": -90.76374053955078,
      "activations/layer4_attention_weight_max": 129.07225036621094,
      "activations/layer4_attention_weight_min": -130.99559020996094,
      "activations/layer5_attention_weight_max": 82.33981323242188,
      "activations/layer5_attention_weight_min": -80.89381408691406,
      "activations/layer6_attention_weight_max": 58.416893005371094,
      "activations/layer6_attention_weight_min": -57.7059211730957,
      "activations/layer7_attention_weight_max": 88.57269287109375,
      "activations/layer7_attention_weight_min": -81.69313049316406,
      "activations/layer8_attention_weight_max": 37.900856018066406,
      "activations/layer8_attention_weight_min": -39.49787139892578,
      "activations/layer9_attention_weight_max": 35.654300689697266,
      "activations/layer9_attention_weight_min": -32.3442497253418,
      "epoch": 5.26,
      "learning_rate": 0.00011728674242424242,
      "loss": 2.8465,
      "step": 90450
    },
    {
      "activations/layer0_attention_weight_max": 16.710603713989258,
      "activations/layer0_attention_weight_min": -11.714473724365234,
      "activations/layer10_attention_weight_max": 33.28919219970703,
      "activations/layer10_attention_weight_min": -31.531692504882812,
      "activations/layer11_attention_weight_max": 33.36442565917969,
      "activations/layer11_attention_weight_min": -28.563873291015625,
      "activations/layer12_attention_weight_max": 37.43235397338867,
      "activations/layer12_attention_weight_min": -27.821617126464844,
      "activations/layer13_attention_weight_max": 43.119022369384766,
      "activations/layer13_attention_weight_min": -31.082523345947266,
      "activations/layer14_attention_weight_max": 55.63390350341797,
      "activations/layer14_attention_weight_min": -39.52783966064453,
      "activations/layer15_attention_weight_max": 45.40072250366211,
      "activations/layer15_attention_weight_min": -28.5104923248291,
      "activations/layer16_attention_weight_max": 41.34461975097656,
      "activations/layer16_attention_weight_min": -28.8529109954834,
      "activations/layer17_attention_weight_max": 37.12349319458008,
      "activations/layer17_attention_weight_min": -24.8353271484375,
      "activations/layer18_attention_weight_max": 31.723257064819336,
      "activations/layer18_attention_weight_min": -19.725927352905273,
      "activations/layer19_attention_weight_max": 43.97220230102539,
      "activations/layer19_attention_weight_min": -29.080463409423828,
      "activations/layer1_attention_weight_max": 17.796100616455078,
      "activations/layer1_attention_weight_min": -16.58726692199707,
      "activations/layer20_attention_weight_max": 32.53789520263672,
      "activations/layer20_attention_weight_min": -23.48035430908203,
      "activations/layer21_attention_weight_max": 34.87445068359375,
      "activations/layer21_attention_weight_min": -20.767990112304688,
      "activations/layer22_attention_weight_max": 48.57228088378906,
      "activations/layer22_attention_weight_min": -30.05517578125,
      "activations/layer23_attention_weight_max": 30.934179306030273,
      "activations/layer23_attention_weight_min": -19.637649536132812,
      "activations/layer2_attention_weight_max": 31.422115325927734,
      "activations/layer2_attention_weight_min": -31.065587997436523,
      "activations/layer3_attention_weight_max": 90.92984771728516,
      "activations/layer3_attention_weight_min": -89.9947509765625,
      "activations/layer4_attention_weight_max": 130.47067260742188,
      "activations/layer4_attention_weight_min": -125.3216323852539,
      "activations/layer5_attention_weight_max": 80.42433166503906,
      "activations/layer5_attention_weight_min": -77.95423889160156,
      "activations/layer6_attention_weight_max": 56.59331130981445,
      "activations/layer6_attention_weight_min": -58.68763732910156,
      "activations/layer7_attention_weight_max": 81.91854095458984,
      "activations/layer7_attention_weight_min": -87.04241943359375,
      "activations/layer8_attention_weight_max": 39.47521209716797,
      "activations/layer8_attention_weight_min": -40.40872573852539,
      "activations/layer9_attention_weight_max": 35.11817169189453,
      "activations/layer9_attention_weight_min": -33.48435592651367,
      "epoch": 5.26,
      "learning_rate": 0.00011726780303030302,
      "loss": 2.8494,
      "step": 90500
    },
    {
      "activations/layer0_attention_weight_max": 16.557964324951172,
      "activations/layer0_attention_weight_min": -11.515120506286621,
      "activations/layer10_attention_weight_max": 32.79273223876953,
      "activations/layer10_attention_weight_min": -31.274703979492188,
      "activations/layer11_attention_weight_max": 33.98292922973633,
      "activations/layer11_attention_weight_min": -27.94530487060547,
      "activations/layer12_attention_weight_max": 25.6696834564209,
      "activations/layer12_attention_weight_min": -27.87224006652832,
      "activations/layer13_attention_weight_max": 48.881141662597656,
      "activations/layer13_attention_weight_min": -32.53822708129883,
      "activations/layer14_attention_weight_max": 54.89054489135742,
      "activations/layer14_attention_weight_min": -43.615440368652344,
      "activations/layer15_attention_weight_max": 41.71596145629883,
      "activations/layer15_attention_weight_min": -27.915739059448242,
      "activations/layer16_attention_weight_max": 36.328433990478516,
      "activations/layer16_attention_weight_min": -27.8875732421875,
      "activations/layer17_attention_weight_max": 37.168540954589844,
      "activations/layer17_attention_weight_min": -25.473108291625977,
      "activations/layer18_attention_weight_max": 35.63721466064453,
      "activations/layer18_attention_weight_min": -21.17103385925293,
      "activations/layer19_attention_weight_max": 42.15347671508789,
      "activations/layer19_attention_weight_min": -32.20991516113281,
      "activations/layer1_attention_weight_max": 18.553424835205078,
      "activations/layer1_attention_weight_min": -14.268190383911133,
      "activations/layer20_attention_weight_max": 34.72158432006836,
      "activations/layer20_attention_weight_min": -26.516380310058594,
      "activations/layer21_attention_weight_max": 32.70878601074219,
      "activations/layer21_attention_weight_min": -24.2462215423584,
      "activations/layer22_attention_weight_max": 50.54744338989258,
      "activations/layer22_attention_weight_min": -33.60166549682617,
      "activations/layer23_attention_weight_max": 37.35900115966797,
      "activations/layer23_attention_weight_min": -22.068546295166016,
      "activations/layer2_attention_weight_max": 33.683902740478516,
      "activations/layer2_attention_weight_min": -32.208736419677734,
      "activations/layer3_attention_weight_max": 92.46414184570312,
      "activations/layer3_attention_weight_min": -93.49634552001953,
      "activations/layer4_attention_weight_max": 139.80458068847656,
      "activations/layer4_attention_weight_min": -129.32017517089844,
      "activations/layer5_attention_weight_max": 85.0769271850586,
      "activations/layer5_attention_weight_min": -77.91770935058594,
      "activations/layer6_attention_weight_max": 60.027854919433594,
      "activations/layer6_attention_weight_min": -54.783206939697266,
      "activations/layer7_attention_weight_max": 90.1180648803711,
      "activations/layer7_attention_weight_min": -81.07467651367188,
      "activations/layer8_attention_weight_max": 39.48905563354492,
      "activations/layer8_attention_weight_min": -40.37733840942383,
      "activations/layer9_attention_weight_max": 36.04109573364258,
      "activations/layer9_attention_weight_min": -34.315364837646484,
      "epoch": 5.26,
      "learning_rate": 0.00011724886363636362,
      "loss": 2.843,
      "step": 90550
    },
    {
      "activations/layer0_attention_weight_max": 15.980179786682129,
      "activations/layer0_attention_weight_min": -11.838537216186523,
      "activations/layer10_attention_weight_max": 31.645122528076172,
      "activations/layer10_attention_weight_min": -28.957387924194336,
      "activations/layer11_attention_weight_max": 32.463905334472656,
      "activations/layer11_attention_weight_min": -27.41440200805664,
      "activations/layer12_attention_weight_max": 23.706344604492188,
      "activations/layer12_attention_weight_min": -27.366308212280273,
      "activations/layer13_attention_weight_max": 40.57941818237305,
      "activations/layer13_attention_weight_min": -31.399560928344727,
      "activations/layer14_attention_weight_max": 57.4174690246582,
      "activations/layer14_attention_weight_min": -45.86407470703125,
      "activations/layer15_attention_weight_max": 42.50649642944336,
      "activations/layer15_attention_weight_min": -27.183345794677734,
      "activations/layer16_attention_weight_max": 35.78133773803711,
      "activations/layer16_attention_weight_min": -28.297931671142578,
      "activations/layer17_attention_weight_max": 40.90718078613281,
      "activations/layer17_attention_weight_min": -23.560972213745117,
      "activations/layer18_attention_weight_max": 33.10956954956055,
      "activations/layer18_attention_weight_min": -19.31710433959961,
      "activations/layer19_attention_weight_max": 35.652191162109375,
      "activations/layer19_attention_weight_min": -30.13604164123535,
      "activations/layer1_attention_weight_max": 17.40525245666504,
      "activations/layer1_attention_weight_min": -15.390380859375,
      "activations/layer20_attention_weight_max": 31.663169860839844,
      "activations/layer20_attention_weight_min": -22.6335391998291,
      "activations/layer21_attention_weight_max": 31.398263931274414,
      "activations/layer21_attention_weight_min": -21.71969223022461,
      "activations/layer22_attention_weight_max": 52.14963150024414,
      "activations/layer22_attention_weight_min": -30.69100570678711,
      "activations/layer23_attention_weight_max": 30.583065032958984,
      "activations/layer23_attention_weight_min": -20.10076141357422,
      "activations/layer2_attention_weight_max": 32.85740661621094,
      "activations/layer2_attention_weight_min": -32.135353088378906,
      "activations/layer3_attention_weight_max": 88.01454162597656,
      "activations/layer3_attention_weight_min": -89.82150268554688,
      "activations/layer4_attention_weight_max": 126.03497314453125,
      "activations/layer4_attention_weight_min": -126.90775299072266,
      "activations/layer5_attention_weight_max": 77.39926147460938,
      "activations/layer5_attention_weight_min": -73.68934631347656,
      "activations/layer6_attention_weight_max": 57.24567794799805,
      "activations/layer6_attention_weight_min": -57.918556213378906,
      "activations/layer7_attention_weight_max": 84.48619842529297,
      "activations/layer7_attention_weight_min": -80.0226058959961,
      "activations/layer8_attention_weight_max": 36.6429328918457,
      "activations/layer8_attention_weight_min": -40.55213165283203,
      "activations/layer9_attention_weight_max": 36.59988784790039,
      "activations/layer9_attention_weight_min": -33.165409088134766,
      "epoch": 5.26,
      "learning_rate": 0.00011722992424242424,
      "loss": 2.8389,
      "step": 90600
    },
    {
      "activations/layer0_attention_weight_max": 16.307519912719727,
      "activations/layer0_attention_weight_min": -11.065234184265137,
      "activations/layer10_attention_weight_max": 34.7206916809082,
      "activations/layer10_attention_weight_min": -32.31117630004883,
      "activations/layer11_attention_weight_max": 33.5543327331543,
      "activations/layer11_attention_weight_min": -29.757551193237305,
      "activations/layer12_attention_weight_max": 30.713682174682617,
      "activations/layer12_attention_weight_min": -27.536222457885742,
      "activations/layer13_attention_weight_max": 38.24219512939453,
      "activations/layer13_attention_weight_min": -31.677265167236328,
      "activations/layer14_attention_weight_max": 52.10698699951172,
      "activations/layer14_attention_weight_min": -45.62382125854492,
      "activations/layer15_attention_weight_max": 36.292667388916016,
      "activations/layer15_attention_weight_min": -27.866756439208984,
      "activations/layer16_attention_weight_max": 31.691179275512695,
      "activations/layer16_attention_weight_min": -25.638591766357422,
      "activations/layer17_attention_weight_max": 33.918087005615234,
      "activations/layer17_attention_weight_min": -23.10344886779785,
      "activations/layer18_attention_weight_max": 28.607370376586914,
      "activations/layer18_attention_weight_min": -20.43462371826172,
      "activations/layer19_attention_weight_max": 37.84282302856445,
      "activations/layer19_attention_weight_min": -28.366384506225586,
      "activations/layer1_attention_weight_max": 18.026105880737305,
      "activations/layer1_attention_weight_min": -14.45624828338623,
      "activations/layer20_attention_weight_max": 27.811058044433594,
      "activations/layer20_attention_weight_min": -23.48586654663086,
      "activations/layer21_attention_weight_max": 30.83291244506836,
      "activations/layer21_attention_weight_min": -19.98282241821289,
      "activations/layer22_attention_weight_max": 42.29851150512695,
      "activations/layer22_attention_weight_min": -26.942626953125,
      "activations/layer23_attention_weight_max": 32.00798416137695,
      "activations/layer23_attention_weight_min": -20.999462127685547,
      "activations/layer2_attention_weight_max": 32.68409729003906,
      "activations/layer2_attention_weight_min": -32.69013595581055,
      "activations/layer3_attention_weight_max": 92.86312103271484,
      "activations/layer3_attention_weight_min": -99.45146942138672,
      "activations/layer4_attention_weight_max": 140.61907958984375,
      "activations/layer4_attention_weight_min": -134.51426696777344,
      "activations/layer5_attention_weight_max": 83.11370086669922,
      "activations/layer5_attention_weight_min": -82.90714263916016,
      "activations/layer6_attention_weight_max": 63.14463806152344,
      "activations/layer6_attention_weight_min": -60.69313430786133,
      "activations/layer7_attention_weight_max": 96.49592590332031,
      "activations/layer7_attention_weight_min": -94.03963470458984,
      "activations/layer8_attention_weight_max": 41.952110290527344,
      "activations/layer8_attention_weight_min": -42.90583419799805,
      "activations/layer9_attention_weight_max": 37.53274917602539,
      "activations/layer9_attention_weight_min": -36.476558685302734,
      "epoch": 5.27,
      "learning_rate": 0.00011721136363636363,
      "loss": 2.8504,
      "step": 90650
    },
    {
      "activations/layer0_attention_weight_max": 14.855234146118164,
      "activations/layer0_attention_weight_min": -11.957419395446777,
      "activations/layer10_attention_weight_max": 36.35202407836914,
      "activations/layer10_attention_weight_min": -30.86045265197754,
      "activations/layer11_attention_weight_max": 36.685585021972656,
      "activations/layer11_attention_weight_min": -28.348026275634766,
      "activations/layer12_attention_weight_max": 33.32575225830078,
      "activations/layer12_attention_weight_min": -29.866552352905273,
      "activations/layer13_attention_weight_max": 46.299598693847656,
      "activations/layer13_attention_weight_min": -35.6123161315918,
      "activations/layer14_attention_weight_max": 74.97261047363281,
      "activations/layer14_attention_weight_min": -54.46381759643555,
      "activations/layer15_attention_weight_max": 41.023468017578125,
      "activations/layer15_attention_weight_min": -31.915328979492188,
      "activations/layer16_attention_weight_max": 36.79388427734375,
      "activations/layer16_attention_weight_min": -33.51303482055664,
      "activations/layer17_attention_weight_max": 43.25687789916992,
      "activations/layer17_attention_weight_min": -27.98451805114746,
      "activations/layer18_attention_weight_max": 36.956661224365234,
      "activations/layer18_attention_weight_min": -22.114925384521484,
      "activations/layer19_attention_weight_max": 39.2176513671875,
      "activations/layer19_attention_weight_min": -33.88160705566406,
      "activations/layer1_attention_weight_max": 17.275978088378906,
      "activations/layer1_attention_weight_min": -14.50178337097168,
      "activations/layer20_attention_weight_max": 33.22402572631836,
      "activations/layer20_attention_weight_min": -25.727083206176758,
      "activations/layer21_attention_weight_max": 31.334299087524414,
      "activations/layer21_attention_weight_min": -22.992557525634766,
      "activations/layer22_attention_weight_max": 47.762393951416016,
      "activations/layer22_attention_weight_min": -31.55845832824707,
      "activations/layer23_attention_weight_max": 31.976425170898438,
      "activations/layer23_attention_weight_min": -20.952823638916016,
      "activations/layer2_attention_weight_max": 32.220428466796875,
      "activations/layer2_attention_weight_min": -31.768272399902344,
      "activations/layer3_attention_weight_max": 91.37567901611328,
      "activations/layer3_attention_weight_min": -91.84269714355469,
      "activations/layer4_attention_weight_max": 131.0692596435547,
      "activations/layer4_attention_weight_min": -127.47335052490234,
      "activations/layer5_attention_weight_max": 81.58828735351562,
      "activations/layer5_attention_weight_min": -76.52360534667969,
      "activations/layer6_attention_weight_max": 57.73131561279297,
      "activations/layer6_attention_weight_min": -55.63588333129883,
      "activations/layer7_attention_weight_max": 85.2264633178711,
      "activations/layer7_attention_weight_min": -82.87062072753906,
      "activations/layer8_attention_weight_max": 38.2270393371582,
      "activations/layer8_attention_weight_min": -39.34245300292969,
      "activations/layer9_attention_weight_max": 34.901424407958984,
      "activations/layer9_attention_weight_min": -40.123313903808594,
      "epoch": 5.27,
      "learning_rate": 0.00011719242424242423,
      "loss": 2.843,
      "step": 90700
    },
    {
      "activations/layer0_attention_weight_max": 16.53710174560547,
      "activations/layer0_attention_weight_min": -12.58757209777832,
      "activations/layer10_attention_weight_max": 34.776756286621094,
      "activations/layer10_attention_weight_min": -30.335296630859375,
      "activations/layer11_attention_weight_max": 33.70043182373047,
      "activations/layer11_attention_weight_min": -27.18524169921875,
      "activations/layer12_attention_weight_max": 27.352445602416992,
      "activations/layer12_attention_weight_min": -26.012313842773438,
      "activations/layer13_attention_weight_max": 47.57188034057617,
      "activations/layer13_attention_weight_min": -35.74816131591797,
      "activations/layer14_attention_weight_max": 68.3641586303711,
      "activations/layer14_attention_weight_min": -44.965858459472656,
      "activations/layer15_attention_weight_max": 41.59455871582031,
      "activations/layer15_attention_weight_min": -28.411094665527344,
      "activations/layer16_attention_weight_max": 44.02972412109375,
      "activations/layer16_attention_weight_min": -27.280073165893555,
      "activations/layer17_attention_weight_max": 42.7144660949707,
      "activations/layer17_attention_weight_min": -24.991453170776367,
      "activations/layer18_attention_weight_max": 40.1251335144043,
      "activations/layer18_attention_weight_min": -20.518583297729492,
      "activations/layer19_attention_weight_max": 50.4956169128418,
      "activations/layer19_attention_weight_min": -28.223922729492188,
      "activations/layer1_attention_weight_max": 17.40861701965332,
      "activations/layer1_attention_weight_min": -14.572550773620605,
      "activations/layer20_attention_weight_max": 46.32478332519531,
      "activations/layer20_attention_weight_min": -23.954362869262695,
      "activations/layer21_attention_weight_max": 45.088626861572266,
      "activations/layer21_attention_weight_min": -21.90353775024414,
      "activations/layer22_attention_weight_max": 65.69660949707031,
      "activations/layer22_attention_weight_min": -26.897024154663086,
      "activations/layer23_attention_weight_max": 41.87518310546875,
      "activations/layer23_attention_weight_min": -19.386363983154297,
      "activations/layer2_attention_weight_max": 33.26091384887695,
      "activations/layer2_attention_weight_min": -31.164831161499023,
      "activations/layer3_attention_weight_max": 92.28093719482422,
      "activations/layer3_attention_weight_min": -92.77530670166016,
      "activations/layer4_attention_weight_max": 134.102783203125,
      "activations/layer4_attention_weight_min": -127.81877899169922,
      "activations/layer5_attention_weight_max": 81.47219848632812,
      "activations/layer5_attention_weight_min": -76.696533203125,
      "activations/layer6_attention_weight_max": 54.13850021362305,
      "activations/layer6_attention_weight_min": -55.8771858215332,
      "activations/layer7_attention_weight_max": 82.65481567382812,
      "activations/layer7_attention_weight_min": -84.03224182128906,
      "activations/layer8_attention_weight_max": 40.856788635253906,
      "activations/layer8_attention_weight_min": -38.96782302856445,
      "activations/layer9_attention_weight_max": 37.16364288330078,
      "activations/layer9_attention_weight_min": -33.75834274291992,
      "epoch": 5.27,
      "learning_rate": 0.00011717348484848484,
      "loss": 2.841,
      "step": 90750
    },
    {
      "activations/layer0_attention_weight_max": 15.878676414489746,
      "activations/layer0_attention_weight_min": -12.157083511352539,
      "activations/layer10_attention_weight_max": 34.607810974121094,
      "activations/layer10_attention_weight_min": -30.585424423217773,
      "activations/layer11_attention_weight_max": 34.966163635253906,
      "activations/layer11_attention_weight_min": -27.434659957885742,
      "activations/layer12_attention_weight_max": 34.83845520019531,
      "activations/layer12_attention_weight_min": -29.781513214111328,
      "activations/layer13_attention_weight_max": 43.4542121887207,
      "activations/layer13_attention_weight_min": -32.18962097167969,
      "activations/layer14_attention_weight_max": 60.820228576660156,
      "activations/layer14_attention_weight_min": -48.24713134765625,
      "activations/layer15_attention_weight_max": 47.67533493041992,
      "activations/layer15_attention_weight_min": -30.631860733032227,
      "activations/layer16_attention_weight_max": 36.308837890625,
      "activations/layer16_attention_weight_min": -27.89047622680664,
      "activations/layer17_attention_weight_max": 37.70270538330078,
      "activations/layer17_attention_weight_min": -24.41631317138672,
      "activations/layer18_attention_weight_max": 34.9443244934082,
      "activations/layer18_attention_weight_min": -21.684362411499023,
      "activations/layer19_attention_weight_max": 42.35469055175781,
      "activations/layer19_attention_weight_min": -29.82282066345215,
      "activations/layer1_attention_weight_max": 17.628334045410156,
      "activations/layer1_attention_weight_min": -17.30915069580078,
      "activations/layer20_attention_weight_max": 35.795223236083984,
      "activations/layer20_attention_weight_min": -23.426986694335938,
      "activations/layer21_attention_weight_max": 33.564964294433594,
      "activations/layer21_attention_weight_min": -20.169437408447266,
      "activations/layer22_attention_weight_max": 51.333221435546875,
      "activations/layer22_attention_weight_min": -29.014848709106445,
      "activations/layer23_attention_weight_max": 34.75019836425781,
      "activations/layer23_attention_weight_min": -22.527481079101562,
      "activations/layer2_attention_weight_max": 31.704296112060547,
      "activations/layer2_attention_weight_min": -32.00568389892578,
      "activations/layer3_attention_weight_max": 88.8118667602539,
      "activations/layer3_attention_weight_min": -86.72893524169922,
      "activations/layer4_attention_weight_max": 123.38984680175781,
      "activations/layer4_attention_weight_min": -124.9485855102539,
      "activations/layer5_attention_weight_max": 74.84009552001953,
      "activations/layer5_attention_weight_min": -74.543212890625,
      "activations/layer6_attention_weight_max": 53.41073226928711,
      "activations/layer6_attention_weight_min": -57.83422088623047,
      "activations/layer7_attention_weight_max": 82.63329315185547,
      "activations/layer7_attention_weight_min": -88.14443969726562,
      "activations/layer8_attention_weight_max": 39.9367561340332,
      "activations/layer8_attention_weight_min": -38.64848709106445,
      "activations/layer9_attention_weight_max": 36.44819259643555,
      "activations/layer9_attention_weight_min": -37.690956115722656,
      "epoch": 5.28,
      "learning_rate": 0.00011715454545454545,
      "loss": 2.8495,
      "step": 90800
    },
    {
      "activations/layer0_attention_weight_max": 16.93170738220215,
      "activations/layer0_attention_weight_min": -12.725640296936035,
      "activations/layer10_attention_weight_max": 33.118499755859375,
      "activations/layer10_attention_weight_min": -30.11115074157715,
      "activations/layer11_attention_weight_max": 31.082372665405273,
      "activations/layer11_attention_weight_min": -28.29267120361328,
      "activations/layer12_attention_weight_max": 24.554306030273438,
      "activations/layer12_attention_weight_min": -27.16531753540039,
      "activations/layer13_attention_weight_max": 42.16747283935547,
      "activations/layer13_attention_weight_min": -32.906288146972656,
      "activations/layer14_attention_weight_max": 60.24026107788086,
      "activations/layer14_attention_weight_min": -46.5236930847168,
      "activations/layer15_attention_weight_max": 42.459556579589844,
      "activations/layer15_attention_weight_min": -27.592044830322266,
      "activations/layer16_attention_weight_max": 40.49471664428711,
      "activations/layer16_attention_weight_min": -26.67090606689453,
      "activations/layer17_attention_weight_max": 48.30615234375,
      "activations/layer17_attention_weight_min": -24.2503604888916,
      "activations/layer18_attention_weight_max": 31.925798416137695,
      "activations/layer18_attention_weight_min": -19.868465423583984,
      "activations/layer19_attention_weight_max": 42.034278869628906,
      "activations/layer19_attention_weight_min": -29.569833755493164,
      "activations/layer1_attention_weight_max": 17.99081039428711,
      "activations/layer1_attention_weight_min": -14.766974449157715,
      "activations/layer20_attention_weight_max": 32.8516845703125,
      "activations/layer20_attention_weight_min": -24.755807876586914,
      "activations/layer21_attention_weight_max": 30.168649673461914,
      "activations/layer21_attention_weight_min": -21.24753761291504,
      "activations/layer22_attention_weight_max": 58.374141693115234,
      "activations/layer22_attention_weight_min": -26.587175369262695,
      "activations/layer23_attention_weight_max": 34.17300033569336,
      "activations/layer23_attention_weight_min": -19.8491268157959,
      "activations/layer2_attention_weight_max": 33.38164520263672,
      "activations/layer2_attention_weight_min": -32.60383224487305,
      "activations/layer3_attention_weight_max": 94.90947723388672,
      "activations/layer3_attention_weight_min": -95.34751892089844,
      "activations/layer4_attention_weight_max": 133.3662567138672,
      "activations/layer4_attention_weight_min": -130.71078491210938,
      "activations/layer5_attention_weight_max": 78.4827880859375,
      "activations/layer5_attention_weight_min": -75.370849609375,
      "activations/layer6_attention_weight_max": 54.37565231323242,
      "activations/layer6_attention_weight_min": -54.18782043457031,
      "activations/layer7_attention_weight_max": 82.50680541992188,
      "activations/layer7_attention_weight_min": -82.39958190917969,
      "activations/layer8_attention_weight_max": 38.863399505615234,
      "activations/layer8_attention_weight_min": -36.54026412963867,
      "activations/layer9_attention_weight_max": 36.19495391845703,
      "activations/layer9_attention_weight_min": -33.268428802490234,
      "epoch": 5.28,
      "learning_rate": 0.00011713560606060605,
      "loss": 2.848,
      "step": 90850
    },
    {
      "activations/layer0_attention_weight_max": 17.405315399169922,
      "activations/layer0_attention_weight_min": -12.485513687133789,
      "activations/layer10_attention_weight_max": 38.070125579833984,
      "activations/layer10_attention_weight_min": -32.73197937011719,
      "activations/layer11_attention_weight_max": 34.54106903076172,
      "activations/layer11_attention_weight_min": -28.637128829956055,
      "activations/layer12_attention_weight_max": 26.136856079101562,
      "activations/layer12_attention_weight_min": -23.97584342956543,
      "activations/layer13_attention_weight_max": 44.857460021972656,
      "activations/layer13_attention_weight_min": -33.16459655761719,
      "activations/layer14_attention_weight_max": 62.74344253540039,
      "activations/layer14_attention_weight_min": -43.61240005493164,
      "activations/layer15_attention_weight_max": 38.00522232055664,
      "activations/layer15_attention_weight_min": -30.491535186767578,
      "activations/layer16_attention_weight_max": 37.63981628417969,
      "activations/layer16_attention_weight_min": -27.298450469970703,
      "activations/layer17_attention_weight_max": 41.594425201416016,
      "activations/layer17_attention_weight_min": -24.710254669189453,
      "activations/layer18_attention_weight_max": 33.81145477294922,
      "activations/layer18_attention_weight_min": -19.53011703491211,
      "activations/layer19_attention_weight_max": 39.118289947509766,
      "activations/layer19_attention_weight_min": -29.4060115814209,
      "activations/layer1_attention_weight_max": 17.556461334228516,
      "activations/layer1_attention_weight_min": -14.121855735778809,
      "activations/layer20_attention_weight_max": 34.08382797241211,
      "activations/layer20_attention_weight_min": -23.029090881347656,
      "activations/layer21_attention_weight_max": 31.216453552246094,
      "activations/layer21_attention_weight_min": -20.842432022094727,
      "activations/layer22_attention_weight_max": 54.87602615356445,
      "activations/layer22_attention_weight_min": -29.56999397277832,
      "activations/layer23_attention_weight_max": 35.75437545776367,
      "activations/layer23_attention_weight_min": -21.583202362060547,
      "activations/layer2_attention_weight_max": 32.49067306518555,
      "activations/layer2_attention_weight_min": -31.59906005859375,
      "activations/layer3_attention_weight_max": 92.74159240722656,
      "activations/layer3_attention_weight_min": -91.39706420898438,
      "activations/layer4_attention_weight_max": 129.11842346191406,
      "activations/layer4_attention_weight_min": -125.19441986083984,
      "activations/layer5_attention_weight_max": 79.9869384765625,
      "activations/layer5_attention_weight_min": -76.69551086425781,
      "activations/layer6_attention_weight_max": 55.69462966918945,
      "activations/layer6_attention_weight_min": -55.31671905517578,
      "activations/layer7_attention_weight_max": 88.13658142089844,
      "activations/layer7_attention_weight_min": -83.75695037841797,
      "activations/layer8_attention_weight_max": 44.0086784362793,
      "activations/layer8_attention_weight_min": -41.23591613769531,
      "activations/layer9_attention_weight_max": 40.8724479675293,
      "activations/layer9_attention_weight_min": -35.55434799194336,
      "epoch": 5.28,
      "learning_rate": 0.00011711666666666666,
      "loss": 2.8338,
      "step": 90900
    },
    {
      "activations/layer0_attention_weight_max": 17.081554412841797,
      "activations/layer0_attention_weight_min": -11.241539001464844,
      "activations/layer10_attention_weight_max": 34.895851135253906,
      "activations/layer10_attention_weight_min": -30.86520004272461,
      "activations/layer11_attention_weight_max": 34.59307098388672,
      "activations/layer11_attention_weight_min": -28.344131469726562,
      "activations/layer12_attention_weight_max": 27.006994247436523,
      "activations/layer12_attention_weight_min": -26.19502067565918,
      "activations/layer13_attention_weight_max": 46.80204772949219,
      "activations/layer13_attention_weight_min": -33.351524353027344,
      "activations/layer14_attention_weight_max": 51.368927001953125,
      "activations/layer14_attention_weight_min": -44.13709259033203,
      "activations/layer15_attention_weight_max": 49.4759407043457,
      "activations/layer15_attention_weight_min": -34.10703659057617,
      "activations/layer16_attention_weight_max": 35.095497131347656,
      "activations/layer16_attention_weight_min": -26.692045211791992,
      "activations/layer17_attention_weight_max": 38.279239654541016,
      "activations/layer17_attention_weight_min": -24.322967529296875,
      "activations/layer18_attention_weight_max": 36.0836296081543,
      "activations/layer18_attention_weight_min": -21.317703247070312,
      "activations/layer19_attention_weight_max": 37.685516357421875,
      "activations/layer19_attention_weight_min": -31.300302505493164,
      "activations/layer1_attention_weight_max": 17.80978012084961,
      "activations/layer1_attention_weight_min": -16.104633331298828,
      "activations/layer20_attention_weight_max": 30.82671356201172,
      "activations/layer20_attention_weight_min": -22.634227752685547,
      "activations/layer21_attention_weight_max": 32.14501953125,
      "activations/layer21_attention_weight_min": -22.483396530151367,
      "activations/layer22_attention_weight_max": 51.40412902832031,
      "activations/layer22_attention_weight_min": -28.37357521057129,
      "activations/layer23_attention_weight_max": 38.019264221191406,
      "activations/layer23_attention_weight_min": -22.186491012573242,
      "activations/layer2_attention_weight_max": 33.14989471435547,
      "activations/layer2_attention_weight_min": -32.16350173950195,
      "activations/layer3_attention_weight_max": 92.44196319580078,
      "activations/layer3_attention_weight_min": -91.837646484375,
      "activations/layer4_attention_weight_max": 130.79563903808594,
      "activations/layer4_attention_weight_min": -127.74715423583984,
      "activations/layer5_attention_weight_max": 78.56201171875,
      "activations/layer5_attention_weight_min": -78.59812927246094,
      "activations/layer6_attention_weight_max": 54.53634262084961,
      "activations/layer6_attention_weight_min": -58.67382049560547,
      "activations/layer7_attention_weight_max": 79.31492614746094,
      "activations/layer7_attention_weight_min": -85.9029541015625,
      "activations/layer8_attention_weight_max": 38.24557876586914,
      "activations/layer8_attention_weight_min": -39.90943145751953,
      "activations/layer9_attention_weight_max": 36.76814651489258,
      "activations/layer9_attention_weight_min": -35.150428771972656,
      "epoch": 5.28,
      "learning_rate": 0.00011709772727272725,
      "loss": 2.8443,
      "step": 90950
    },
    {
      "activations/layer0_attention_weight_max": 16.65327262878418,
      "activations/layer0_attention_weight_min": -11.274969100952148,
      "activations/layer10_attention_weight_max": 35.82435989379883,
      "activations/layer10_attention_weight_min": -31.781145095825195,
      "activations/layer11_attention_weight_max": 31.736425399780273,
      "activations/layer11_attention_weight_min": -30.17706298828125,
      "activations/layer12_attention_weight_max": 31.257644653320312,
      "activations/layer12_attention_weight_min": -30.350566864013672,
      "activations/layer13_attention_weight_max": 40.628074645996094,
      "activations/layer13_attention_weight_min": -31.38288116455078,
      "activations/layer14_attention_weight_max": 61.412994384765625,
      "activations/layer14_attention_weight_min": -45.983219146728516,
      "activations/layer15_attention_weight_max": 38.945701599121094,
      "activations/layer15_attention_weight_min": -28.45880889892578,
      "activations/layer16_attention_weight_max": 40.30669403076172,
      "activations/layer16_attention_weight_min": -26.705904006958008,
      "activations/layer17_attention_weight_max": 38.206199645996094,
      "activations/layer17_attention_weight_min": -25.661001205444336,
      "activations/layer18_attention_weight_max": 34.13871383666992,
      "activations/layer18_attention_weight_min": -20.877904891967773,
      "activations/layer19_attention_weight_max": 39.30581283569336,
      "activations/layer19_attention_weight_min": -28.1744441986084,
      "activations/layer1_attention_weight_max": 18.01577377319336,
      "activations/layer1_attention_weight_min": -14.532654762268066,
      "activations/layer20_attention_weight_max": 32.35264587402344,
      "activations/layer20_attention_weight_min": -23.610872268676758,
      "activations/layer21_attention_weight_max": 33.92436599731445,
      "activations/layer21_attention_weight_min": -23.618553161621094,
      "activations/layer22_attention_weight_max": 47.71704864501953,
      "activations/layer22_attention_weight_min": -28.284475326538086,
      "activations/layer23_attention_weight_max": 34.674652099609375,
      "activations/layer23_attention_weight_min": -21.162750244140625,
      "activations/layer2_attention_weight_max": 31.341398239135742,
      "activations/layer2_attention_weight_min": -31.2789363861084,
      "activations/layer3_attention_weight_max": 87.79376983642578,
      "activations/layer3_attention_weight_min": -91.20779418945312,
      "activations/layer4_attention_weight_max": 126.75404357910156,
      "activations/layer4_attention_weight_min": -126.40966033935547,
      "activations/layer5_attention_weight_max": 78.43600463867188,
      "activations/layer5_attention_weight_min": -78.26117706298828,
      "activations/layer6_attention_weight_max": 59.19129943847656,
      "activations/layer6_attention_weight_min": -58.149600982666016,
      "activations/layer7_attention_weight_max": 82.4280776977539,
      "activations/layer7_attention_weight_min": -90.8717269897461,
      "activations/layer8_attention_weight_max": 39.69761657714844,
      "activations/layer8_attention_weight_min": -39.141963958740234,
      "activations/layer9_attention_weight_max": 37.05177688598633,
      "activations/layer9_attention_weight_min": -33.159053802490234,
      "epoch": 5.29,
      "learning_rate": 0.00011707878787878787,
      "loss": 2.8398,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_loss": 2.796875,
      "eval_runtime": 8.5056,
      "eval_samples_per_second": 504.844,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_openwebtext_loss": 2.796875,
      "eval_openwebtext_ppl": 16.39333746248769,
      "eval_openwebtext_runtime": 8.5056,
      "eval_openwebtext_samples_per_second": 504.844,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 1.9633,
      "eval_wikitext_samples_per_second": 232.266,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_lambada_loss": 2.767578125,
      "eval_lambada_ppl": 15.920030957803608,
      "eval_lambada_runtime": 9.585,
      "eval_lambada_samples_per_second": 507.98,
      "step": 91000
    },
    {
      "activations/layer0_attention_weight_max": 16.516265869140625,
      "activations/layer0_attention_weight_min": -12.50368595123291,
      "activations/layer10_attention_weight_max": 35.5034294128418,
      "activations/layer10_attention_weight_min": -32.67111587524414,
      "activations/layer11_attention_weight_max": 35.1884880065918,
      "activations/layer11_attention_weight_min": -27.45248031616211,
      "activations/layer12_attention_weight_max": 24.3052921295166,
      "activations/layer12_attention_weight_min": -27.11125373840332,
      "activations/layer13_attention_weight_max": 42.33797836303711,
      "activations/layer13_attention_weight_min": -31.428260803222656,
      "activations/layer14_attention_weight_max": 74.06205749511719,
      "activations/layer14_attention_weight_min": -47.102882385253906,
      "activations/layer15_attention_weight_max": 39.65256881713867,
      "activations/layer15_attention_weight_min": -27.728525161743164,
      "activations/layer16_attention_weight_max": 42.10757827758789,
      "activations/layer16_attention_weight_min": -26.430633544921875,
      "activations/layer17_attention_weight_max": 37.3327522277832,
      "activations/layer17_attention_weight_min": -25.17751121520996,
      "activations/layer18_attention_weight_max": 28.937602996826172,
      "activations/layer18_attention_weight_min": -20.042951583862305,
      "activations/layer19_attention_weight_max": 41.54681396484375,
      "activations/layer19_attention_weight_min": -32.562862396240234,
      "activations/layer1_attention_weight_max": 17.680204391479492,
      "activations/layer1_attention_weight_min": -15.641559600830078,
      "activations/layer20_attention_weight_max": 32.80524826049805,
      "activations/layer20_attention_weight_min": -25.615947723388672,
      "activations/layer21_attention_weight_max": 31.388572692871094,
      "activations/layer21_attention_weight_min": -21.83806800842285,
      "activations/layer22_attention_weight_max": 48.91410446166992,
      "activations/layer22_attention_weight_min": -29.964828491210938,
      "activations/layer23_attention_weight_max": 34.12754821777344,
      "activations/layer23_attention_weight_min": -21.726078033447266,
      "activations/layer2_attention_weight_max": 30.992977142333984,
      "activations/layer2_attention_weight_min": -31.649534225463867,
      "activations/layer3_attention_weight_max": 93.00819396972656,
      "activations/layer3_attention_weight_min": -99.23210906982422,
      "activations/layer4_attention_weight_max": 135.8768310546875,
      "activations/layer4_attention_weight_min": -134.36668395996094,
      "activations/layer5_attention_weight_max": 81.76731872558594,
      "activations/layer5_attention_weight_min": -79.11251831054688,
      "activations/layer6_attention_weight_max": 56.10406494140625,
      "activations/layer6_attention_weight_min": -54.772544860839844,
      "activations/layer7_attention_weight_max": 82.81686401367188,
      "activations/layer7_attention_weight_min": -84.84148406982422,
      "activations/layer8_attention_weight_max": 40.089088439941406,
      "activations/layer8_attention_weight_min": -37.93910217285156,
      "activations/layer9_attention_weight_max": 37.49420166015625,
      "activations/layer9_attention_weight_min": -33.04977798461914,
      "epoch": 5.29,
      "learning_rate": 0.00011705984848484848,
      "loss": 2.834,
      "step": 91050
    },
    {
      "activations/layer0_attention_weight_max": 16.7522029876709,
      "activations/layer0_attention_weight_min": -11.841521263122559,
      "activations/layer10_attention_weight_max": 36.92314529418945,
      "activations/layer10_attention_weight_min": -30.877731323242188,
      "activations/layer11_attention_weight_max": 36.05061340332031,
      "activations/layer11_attention_weight_min": -29.49701499938965,
      "activations/layer12_attention_weight_max": 36.19139862060547,
      "activations/layer12_attention_weight_min": -31.001319885253906,
      "activations/layer13_attention_weight_max": 48.30739974975586,
      "activations/layer13_attention_weight_min": -31.55832862854004,
      "activations/layer14_attention_weight_max": 71.8331527709961,
      "activations/layer14_attention_weight_min": -44.977054595947266,
      "activations/layer15_attention_weight_max": 38.52114486694336,
      "activations/layer15_attention_weight_min": -28.17525291442871,
      "activations/layer16_attention_weight_max": 44.010948181152344,
      "activations/layer16_attention_weight_min": -28.59613037109375,
      "activations/layer17_attention_weight_max": 45.219207763671875,
      "activations/layer17_attention_weight_min": -24.98958969116211,
      "activations/layer18_attention_weight_max": 32.60224533081055,
      "activations/layer18_attention_weight_min": -21.87764549255371,
      "activations/layer19_attention_weight_max": 35.10453414916992,
      "activations/layer19_attention_weight_min": -29.6934871673584,
      "activations/layer1_attention_weight_max": 18.263099670410156,
      "activations/layer1_attention_weight_min": -13.619415283203125,
      "activations/layer20_attention_weight_max": 29.062997817993164,
      "activations/layer20_attention_weight_min": -22.399404525756836,
      "activations/layer21_attention_weight_max": 27.987838745117188,
      "activations/layer21_attention_weight_min": -21.56401824951172,
      "activations/layer22_attention_weight_max": 45.73194122314453,
      "activations/layer22_attention_weight_min": -26.787961959838867,
      "activations/layer23_attention_weight_max": 31.29327392578125,
      "activations/layer23_attention_weight_min": -20.986181259155273,
      "activations/layer2_attention_weight_max": 33.097328186035156,
      "activations/layer2_attention_weight_min": -32.951416015625,
      "activations/layer3_attention_weight_max": 97.84115600585938,
      "activations/layer3_attention_weight_min": -96.46660614013672,
      "activations/layer4_attention_weight_max": 135.86585998535156,
      "activations/layer4_attention_weight_min": -130.19683837890625,
      "activations/layer5_attention_weight_max": 78.70994567871094,
      "activations/layer5_attention_weight_min": -78.98423767089844,
      "activations/layer6_attention_weight_max": 54.9172477722168,
      "activations/layer6_attention_weight_min": -56.04878234863281,
      "activations/layer7_attention_weight_max": 95.50927734375,
      "activations/layer7_attention_weight_min": -82.49224090576172,
      "activations/layer8_attention_weight_max": 42.309532165527344,
      "activations/layer8_attention_weight_min": -39.09648895263672,
      "activations/layer9_attention_weight_max": 36.754268646240234,
      "activations/layer9_attention_weight_min": -35.247440338134766,
      "epoch": 5.29,
      "learning_rate": 0.00011704090909090907,
      "loss": 2.8437,
      "step": 91100
    },
    {
      "activations/layer0_attention_weight_max": 16.826074600219727,
      "activations/layer0_attention_weight_min": -11.17370891571045,
      "activations/layer10_attention_weight_max": 35.083961486816406,
      "activations/layer10_attention_weight_min": -32.001461029052734,
      "activations/layer11_attention_weight_max": 30.69048309326172,
      "activations/layer11_attention_weight_min": -29.023174285888672,
      "activations/layer12_attention_weight_max": 28.37958335876465,
      "activations/layer12_attention_weight_min": -26.006567001342773,
      "activations/layer13_attention_weight_max": 46.60047149658203,
      "activations/layer13_attention_weight_min": -36.56610870361328,
      "activations/layer14_attention_weight_max": 59.78995895385742,
      "activations/layer14_attention_weight_min": -46.98877716064453,
      "activations/layer15_attention_weight_max": 36.31950759887695,
      "activations/layer15_attention_weight_min": -28.09891700744629,
      "activations/layer16_attention_weight_max": 34.599212646484375,
      "activations/layer16_attention_weight_min": -29.079973220825195,
      "activations/layer17_attention_weight_max": 35.78453063964844,
      "activations/layer17_attention_weight_min": -24.52313995361328,
      "activations/layer18_attention_weight_max": 36.90581130981445,
      "activations/layer18_attention_weight_min": -20.281307220458984,
      "activations/layer19_attention_weight_max": 38.54137420654297,
      "activations/layer19_attention_weight_min": -29.397497177124023,
      "activations/layer1_attention_weight_max": 16.8022403717041,
      "activations/layer1_attention_weight_min": -15.620625495910645,
      "activations/layer20_attention_weight_max": 30.818086624145508,
      "activations/layer20_attention_weight_min": -23.60000991821289,
      "activations/layer21_attention_weight_max": 33.57947540283203,
      "activations/layer21_attention_weight_min": -21.876684188842773,
      "activations/layer22_attention_weight_max": 49.897987365722656,
      "activations/layer22_attention_weight_min": -30.848657608032227,
      "activations/layer23_attention_weight_max": 35.97126388549805,
      "activations/layer23_attention_weight_min": -24.351810455322266,
      "activations/layer2_attention_weight_max": 30.931819915771484,
      "activations/layer2_attention_weight_min": -32.36668014526367,
      "activations/layer3_attention_weight_max": 89.26807403564453,
      "activations/layer3_attention_weight_min": -92.67393493652344,
      "activations/layer4_attention_weight_max": 128.5596160888672,
      "activations/layer4_attention_weight_min": -130.3508758544922,
      "activations/layer5_attention_weight_max": 79.0435562133789,
      "activations/layer5_attention_weight_min": -78.41569519042969,
      "activations/layer6_attention_weight_max": 52.43844985961914,
      "activations/layer6_attention_weight_min": -55.50229263305664,
      "activations/layer7_attention_weight_max": 83.15222930908203,
      "activations/layer7_attention_weight_min": -83.91778564453125,
      "activations/layer8_attention_weight_max": 41.962303161621094,
      "activations/layer8_attention_weight_min": -39.866329193115234,
      "activations/layer9_attention_weight_max": 35.756874084472656,
      "activations/layer9_attention_weight_min": -34.22346496582031,
      "epoch": 5.3,
      "learning_rate": 0.00011702196969696968,
      "loss": 2.8561,
      "step": 91150
    },
    {
      "activations/layer0_attention_weight_max": 15.88458251953125,
      "activations/layer0_attention_weight_min": -11.863740921020508,
      "activations/layer10_attention_weight_max": 33.84130859375,
      "activations/layer10_attention_weight_min": -33.9134407043457,
      "activations/layer11_attention_weight_max": 31.645709991455078,
      "activations/layer11_attention_weight_min": -27.738285064697266,
      "activations/layer12_attention_weight_max": 27.04256820678711,
      "activations/layer12_attention_weight_min": -28.91702651977539,
      "activations/layer13_attention_weight_max": 39.15656280517578,
      "activations/layer13_attention_weight_min": -32.04486846923828,
      "activations/layer14_attention_weight_max": 70.632080078125,
      "activations/layer14_attention_weight_min": -51.6712646484375,
      "activations/layer15_attention_weight_max": 40.74102020263672,
      "activations/layer15_attention_weight_min": -29.205610275268555,
      "activations/layer16_attention_weight_max": 36.40868377685547,
      "activations/layer16_attention_weight_min": -28.37148094177246,
      "activations/layer17_attention_weight_max": 44.750587463378906,
      "activations/layer17_attention_weight_min": -24.183582305908203,
      "activations/layer18_attention_weight_max": 34.04927444458008,
      "activations/layer18_attention_weight_min": -21.176340103149414,
      "activations/layer19_attention_weight_max": 42.38129425048828,
      "activations/layer19_attention_weight_min": -30.727741241455078,
      "activations/layer1_attention_weight_max": 17.050094604492188,
      "activations/layer1_attention_weight_min": -14.735032081604004,
      "activations/layer20_attention_weight_max": 37.62377166748047,
      "activations/layer20_attention_weight_min": -27.065256118774414,
      "activations/layer21_attention_weight_max": 33.77611541748047,
      "activations/layer21_attention_weight_min": -23.50172996520996,
      "activations/layer22_attention_weight_max": 57.61062240600586,
      "activations/layer22_attention_weight_min": -35.04441452026367,
      "activations/layer23_attention_weight_max": 40.34563064575195,
      "activations/layer23_attention_weight_min": -24.375564575195312,
      "activations/layer2_attention_weight_max": 32.770164489746094,
      "activations/layer2_attention_weight_min": -32.38777160644531,
      "activations/layer3_attention_weight_max": 93.39549255371094,
      "activations/layer3_attention_weight_min": -96.22545623779297,
      "activations/layer4_attention_weight_max": 135.69200134277344,
      "activations/layer4_attention_weight_min": -134.65386962890625,
      "activations/layer5_attention_weight_max": 82.18684387207031,
      "activations/layer5_attention_weight_min": -84.55227661132812,
      "activations/layer6_attention_weight_max": 57.3606071472168,
      "activations/layer6_attention_weight_min": -58.17143630981445,
      "activations/layer7_attention_weight_max": 84.12571716308594,
      "activations/layer7_attention_weight_min": -88.94879913330078,
      "activations/layer8_attention_weight_max": 43.12245559692383,
      "activations/layer8_attention_weight_min": -43.1476936340332,
      "activations/layer9_attention_weight_max": 38.02708435058594,
      "activations/layer9_attention_weight_min": -35.091064453125,
      "epoch": 5.3,
      "learning_rate": 0.0001170030303030303,
      "loss": 2.8463,
      "step": 91200
    },
    {
      "activations/layer0_attention_weight_max": 16.284225463867188,
      "activations/layer0_attention_weight_min": -12.598834991455078,
      "activations/layer10_attention_weight_max": 36.71453857421875,
      "activations/layer10_attention_weight_min": -32.63137435913086,
      "activations/layer11_attention_weight_max": 39.35724639892578,
      "activations/layer11_attention_weight_min": -31.28989028930664,
      "activations/layer12_attention_weight_max": 28.183080673217773,
      "activations/layer12_attention_weight_min": -28.599546432495117,
      "activations/layer13_attention_weight_max": 46.12218475341797,
      "activations/layer13_attention_weight_min": -36.7525634765625,
      "activations/layer14_attention_weight_max": 61.512420654296875,
      "activations/layer14_attention_weight_min": -46.28691482543945,
      "activations/layer15_attention_weight_max": 37.48040008544922,
      "activations/layer15_attention_weight_min": -28.714126586914062,
      "activations/layer16_attention_weight_max": 36.98630142211914,
      "activations/layer16_attention_weight_min": -26.809110641479492,
      "activations/layer17_attention_weight_max": 40.03261184692383,
      "activations/layer17_attention_weight_min": -24.4083194732666,
      "activations/layer18_attention_weight_max": 25.777942657470703,
      "activations/layer18_attention_weight_min": -20.028589248657227,
      "activations/layer19_attention_weight_max": 39.714988708496094,
      "activations/layer19_attention_weight_min": -28.480045318603516,
      "activations/layer1_attention_weight_max": 18.155292510986328,
      "activations/layer1_attention_weight_min": -14.529212951660156,
      "activations/layer20_attention_weight_max": 28.509721755981445,
      "activations/layer20_attention_weight_min": -21.680004119873047,
      "activations/layer21_attention_weight_max": 30.43529510498047,
      "activations/layer21_attention_weight_min": -19.902862548828125,
      "activations/layer22_attention_weight_max": 47.96115493774414,
      "activations/layer22_attention_weight_min": -29.142597198486328,
      "activations/layer23_attention_weight_max": 32.29469680786133,
      "activations/layer23_attention_weight_min": -22.97540283203125,
      "activations/layer2_attention_weight_max": 31.48282241821289,
      "activations/layer2_attention_weight_min": -32.82789611816406,
      "activations/layer3_attention_weight_max": 94.06798553466797,
      "activations/layer3_attention_weight_min": -90.41561126708984,
      "activations/layer4_attention_weight_max": 130.55259704589844,
      "activations/layer4_attention_weight_min": -128.38096618652344,
      "activations/layer5_attention_weight_max": 82.77592468261719,
      "activations/layer5_attention_weight_min": -79.35691833496094,
      "activations/layer6_attention_weight_max": 57.77873611450195,
      "activations/layer6_attention_weight_min": -58.74840545654297,
      "activations/layer7_attention_weight_max": 88.4654769897461,
      "activations/layer7_attention_weight_min": -91.86912536621094,
      "activations/layer8_attention_weight_max": 41.379310607910156,
      "activations/layer8_attention_weight_min": -41.399532318115234,
      "activations/layer9_attention_weight_max": 38.5092887878418,
      "activations/layer9_attention_weight_min": -36.297767639160156,
      "epoch": 5.3,
      "learning_rate": 0.0001169840909090909,
      "loss": 2.841,
      "step": 91250
    },
    {
      "activations/layer0_attention_weight_max": 16.05329704284668,
      "activations/layer0_attention_weight_min": -12.712227821350098,
      "activations/layer10_attention_weight_max": 35.63667297363281,
      "activations/layer10_attention_weight_min": -31.593399047851562,
      "activations/layer11_attention_weight_max": 32.82441329956055,
      "activations/layer11_attention_weight_min": -27.439071655273438,
      "activations/layer12_attention_weight_max": 28.05451011657715,
      "activations/layer12_attention_weight_min": -24.896347045898438,
      "activations/layer13_attention_weight_max": 56.09808349609375,
      "activations/layer13_attention_weight_min": -38.98016357421875,
      "activations/layer14_attention_weight_max": 58.01325607299805,
      "activations/layer14_attention_weight_min": -47.784507751464844,
      "activations/layer15_attention_weight_max": 37.43985366821289,
      "activations/layer15_attention_weight_min": -28.822507858276367,
      "activations/layer16_attention_weight_max": 37.76044464111328,
      "activations/layer16_attention_weight_min": -26.943687438964844,
      "activations/layer17_attention_weight_max": 37.174617767333984,
      "activations/layer17_attention_weight_min": -25.853052139282227,
      "activations/layer18_attention_weight_max": 30.369474411010742,
      "activations/layer18_attention_weight_min": -19.66712760925293,
      "activations/layer19_attention_weight_max": 39.623348236083984,
      "activations/layer19_attention_weight_min": -27.827497482299805,
      "activations/layer1_attention_weight_max": 17.4212703704834,
      "activations/layer1_attention_weight_min": -14.247455596923828,
      "activations/layer20_attention_weight_max": 30.085620880126953,
      "activations/layer20_attention_weight_min": -22.353595733642578,
      "activations/layer21_attention_weight_max": 30.54030990600586,
      "activations/layer21_attention_weight_min": -20.995542526245117,
      "activations/layer22_attention_weight_max": 47.8775749206543,
      "activations/layer22_attention_weight_min": -28.290822982788086,
      "activations/layer23_attention_weight_max": 35.50053787231445,
      "activations/layer23_attention_weight_min": -21.84436798095703,
      "activations/layer2_attention_weight_max": 31.06252670288086,
      "activations/layer2_attention_weight_min": -30.213764190673828,
      "activations/layer3_attention_weight_max": 89.32306671142578,
      "activations/layer3_attention_weight_min": -88.59571838378906,
      "activations/layer4_attention_weight_max": 127.464111328125,
      "activations/layer4_attention_weight_min": -125.0448989868164,
      "activations/layer5_attention_weight_max": 77.20706939697266,
      "activations/layer5_attention_weight_min": -75.80583190917969,
      "activations/layer6_attention_weight_max": 55.72074508666992,
      "activations/layer6_attention_weight_min": -55.79127883911133,
      "activations/layer7_attention_weight_max": 84.72994995117188,
      "activations/layer7_attention_weight_min": -84.8399429321289,
      "activations/layer8_attention_weight_max": 40.262454986572266,
      "activations/layer8_attention_weight_min": -41.26820755004883,
      "activations/layer9_attention_weight_max": 37.57117462158203,
      "activations/layer9_attention_weight_min": -36.912841796875,
      "epoch": 5.31,
      "learning_rate": 0.0001169651515151515,
      "loss": 2.844,
      "step": 91300
    },
    {
      "activations/layer0_attention_weight_max": 16.786176681518555,
      "activations/layer0_attention_weight_min": -11.8611478805542,
      "activations/layer10_attention_weight_max": 32.63936996459961,
      "activations/layer10_attention_weight_min": -29.385971069335938,
      "activations/layer11_attention_weight_max": 32.23503875732422,
      "activations/layer11_attention_weight_min": -25.688526153564453,
      "activations/layer12_attention_weight_max": 40.286476135253906,
      "activations/layer12_attention_weight_min": -24.140766143798828,
      "activations/layer13_attention_weight_max": 43.45478820800781,
      "activations/layer13_attention_weight_min": -34.13079833984375,
      "activations/layer14_attention_weight_max": 60.40103530883789,
      "activations/layer14_attention_weight_min": -46.03187561035156,
      "activations/layer15_attention_weight_max": 37.10550308227539,
      "activations/layer15_attention_weight_min": -27.665071487426758,
      "activations/layer16_attention_weight_max": 40.310791015625,
      "activations/layer16_attention_weight_min": -28.02010154724121,
      "activations/layer17_attention_weight_max": 37.431907653808594,
      "activations/layer17_attention_weight_min": -23.095844268798828,
      "activations/layer18_attention_weight_max": 31.962528228759766,
      "activations/layer18_attention_weight_min": -19.776565551757812,
      "activations/layer19_attention_weight_max": 40.97018814086914,
      "activations/layer19_attention_weight_min": -28.69671058654785,
      "activations/layer1_attention_weight_max": 17.361034393310547,
      "activations/layer1_attention_weight_min": -13.526470184326172,
      "activations/layer20_attention_weight_max": 35.85837173461914,
      "activations/layer20_attention_weight_min": -24.284292221069336,
      "activations/layer21_attention_weight_max": 28.80219078063965,
      "activations/layer21_attention_weight_min": -22.084632873535156,
      "activations/layer22_attention_weight_max": 50.611419677734375,
      "activations/layer22_attention_weight_min": -27.41973876953125,
      "activations/layer23_attention_weight_max": 30.155019760131836,
      "activations/layer23_attention_weight_min": -19.76355743408203,
      "activations/layer2_attention_weight_max": 32.23927688598633,
      "activations/layer2_attention_weight_min": -32.278419494628906,
      "activations/layer3_attention_weight_max": 87.80261993408203,
      "activations/layer3_attention_weight_min": -89.79222106933594,
      "activations/layer4_attention_weight_max": 127.47110748291016,
      "activations/layer4_attention_weight_min": -131.3220672607422,
      "activations/layer5_attention_weight_max": 77.62211608886719,
      "activations/layer5_attention_weight_min": -76.9008560180664,
      "activations/layer6_attention_weight_max": 55.73591232299805,
      "activations/layer6_attention_weight_min": -54.40583038330078,
      "activations/layer7_attention_weight_max": 83.6013412475586,
      "activations/layer7_attention_weight_min": -89.62815856933594,
      "activations/layer8_attention_weight_max": 41.88441848754883,
      "activations/layer8_attention_weight_min": -38.52324295043945,
      "activations/layer9_attention_weight_max": 36.182674407958984,
      "activations/layer9_attention_weight_min": -33.07961654663086,
      "epoch": 5.31,
      "learning_rate": 0.00011694621212121212,
      "loss": 2.8322,
      "step": 91350
    },
    {
      "activations/layer0_attention_weight_max": 16.79306411743164,
      "activations/layer0_attention_weight_min": -11.514518737792969,
      "activations/layer10_attention_weight_max": 38.81854248046875,
      "activations/layer10_attention_weight_min": -34.0239372253418,
      "activations/layer11_attention_weight_max": 39.27894592285156,
      "activations/layer11_attention_weight_min": -33.734867095947266,
      "activations/layer12_attention_weight_max": 27.513999938964844,
      "activations/layer12_attention_weight_min": -27.639860153198242,
      "activations/layer13_attention_weight_max": 50.1398811340332,
      "activations/layer13_attention_weight_min": -33.934471130371094,
      "activations/layer14_attention_weight_max": 71.7881088256836,
      "activations/layer14_attention_weight_min": -49.452125549316406,
      "activations/layer15_attention_weight_max": 43.48292541503906,
      "activations/layer15_attention_weight_min": -27.893341064453125,
      "activations/layer16_attention_weight_max": 38.344913482666016,
      "activations/layer16_attention_weight_min": -27.044462203979492,
      "activations/layer17_attention_weight_max": 42.853912353515625,
      "activations/layer17_attention_weight_min": -23.716876983642578,
      "activations/layer18_attention_weight_max": 36.94279861450195,
      "activations/layer18_attention_weight_min": -19.85750961303711,
      "activations/layer19_attention_weight_max": 41.95140838623047,
      "activations/layer19_attention_weight_min": -27.842443466186523,
      "activations/layer1_attention_weight_max": 17.24476432800293,
      "activations/layer1_attention_weight_min": -13.946518898010254,
      "activations/layer20_attention_weight_max": 33.830692291259766,
      "activations/layer20_attention_weight_min": -22.577383041381836,
      "activations/layer21_attention_weight_max": 33.58036804199219,
      "activations/layer21_attention_weight_min": -18.901384353637695,
      "activations/layer22_attention_weight_max": 57.14204406738281,
      "activations/layer22_attention_weight_min": -27.295495986938477,
      "activations/layer23_attention_weight_max": 35.699527740478516,
      "activations/layer23_attention_weight_min": -21.471309661865234,
      "activations/layer2_attention_weight_max": 33.7940673828125,
      "activations/layer2_attention_weight_min": -35.13023376464844,
      "activations/layer3_attention_weight_max": 95.51798248291016,
      "activations/layer3_attention_weight_min": -98.44286346435547,
      "activations/layer4_attention_weight_max": 135.55413818359375,
      "activations/layer4_attention_weight_min": -138.39385986328125,
      "activations/layer5_attention_weight_max": 81.56584167480469,
      "activations/layer5_attention_weight_min": -83.22884368896484,
      "activations/layer6_attention_weight_max": 58.44971466064453,
      "activations/layer6_attention_weight_min": -57.916839599609375,
      "activations/layer7_attention_weight_max": 98.12411499023438,
      "activations/layer7_attention_weight_min": -88.40030670166016,
      "activations/layer8_attention_weight_max": 47.04539489746094,
      "activations/layer8_attention_weight_min": -41.86114501953125,
      "activations/layer9_attention_weight_max": 40.36738967895508,
      "activations/layer9_attention_weight_min": -36.157257080078125,
      "epoch": 5.31,
      "learning_rate": 0.00011692727272727272,
      "loss": 2.8362,
      "step": 91400
    },
    {
      "activations/layer0_attention_weight_max": 16.684011459350586,
      "activations/layer0_attention_weight_min": -11.946341514587402,
      "activations/layer10_attention_weight_max": 29.852975845336914,
      "activations/layer10_attention_weight_min": -31.628488540649414,
      "activations/layer11_attention_weight_max": 29.574806213378906,
      "activations/layer11_attention_weight_min": -27.257869720458984,
      "activations/layer12_attention_weight_max": 28.797624588012695,
      "activations/layer12_attention_weight_min": -30.62409210205078,
      "activations/layer13_attention_weight_max": 40.113643646240234,
      "activations/layer13_attention_weight_min": -33.84250259399414,
      "activations/layer14_attention_weight_max": 58.41996383666992,
      "activations/layer14_attention_weight_min": -42.83450698852539,
      "activations/layer15_attention_weight_max": 39.03417205810547,
      "activations/layer15_attention_weight_min": -29.923616409301758,
      "activations/layer16_attention_weight_max": 34.50215148925781,
      "activations/layer16_attention_weight_min": -27.405359268188477,
      "activations/layer17_attention_weight_max": 36.004051208496094,
      "activations/layer17_attention_weight_min": -23.6016788482666,
      "activations/layer18_attention_weight_max": 29.905550003051758,
      "activations/layer18_attention_weight_min": -19.42180061340332,
      "activations/layer19_attention_weight_max": 39.95553207397461,
      "activations/layer19_attention_weight_min": -28.34661293029785,
      "activations/layer1_attention_weight_max": 17.6527042388916,
      "activations/layer1_attention_weight_min": -14.59406852722168,
      "activations/layer20_attention_weight_max": 34.87190246582031,
      "activations/layer20_attention_weight_min": -21.89329719543457,
      "activations/layer21_attention_weight_max": 34.38027572631836,
      "activations/layer21_attention_weight_min": -20.606481552124023,
      "activations/layer22_attention_weight_max": 45.06745147705078,
      "activations/layer22_attention_weight_min": -31.02402687072754,
      "activations/layer23_attention_weight_max": 31.55572509765625,
      "activations/layer23_attention_weight_min": -19.97919464111328,
      "activations/layer2_attention_weight_max": 31.81183433532715,
      "activations/layer2_attention_weight_min": -33.70133590698242,
      "activations/layer3_attention_weight_max": 89.27561950683594,
      "activations/layer3_attention_weight_min": -95.44905090332031,
      "activations/layer4_attention_weight_max": 129.7324676513672,
      "activations/layer4_attention_weight_min": -130.0656280517578,
      "activations/layer5_attention_weight_max": 78.51113891601562,
      "activations/layer5_attention_weight_min": -74.3676986694336,
      "activations/layer6_attention_weight_max": 57.16505813598633,
      "activations/layer6_attention_weight_min": -55.19080352783203,
      "activations/layer7_attention_weight_max": 82.8340835571289,
      "activations/layer7_attention_weight_min": -85.13970184326172,
      "activations/layer8_attention_weight_max": 41.071861267089844,
      "activations/layer8_attention_weight_min": -41.86540603637695,
      "activations/layer9_attention_weight_max": 34.020084381103516,
      "activations/layer9_attention_weight_min": -38.6714973449707,
      "epoch": 5.31,
      "learning_rate": 0.00011690833333333332,
      "loss": 2.8541,
      "step": 91450
    },
    {
      "activations/layer0_attention_weight_max": 16.948453903198242,
      "activations/layer0_attention_weight_min": -12.099946975708008,
      "activations/layer10_attention_weight_max": 35.12671661376953,
      "activations/layer10_attention_weight_min": -31.262413024902344,
      "activations/layer11_attention_weight_max": 33.05322265625,
      "activations/layer11_attention_weight_min": -28.328838348388672,
      "activations/layer12_attention_weight_max": 41.48116683959961,
      "activations/layer12_attention_weight_min": -24.34364128112793,
      "activations/layer13_attention_weight_max": 47.80020523071289,
      "activations/layer13_attention_weight_min": -32.089656829833984,
      "activations/layer14_attention_weight_max": 67.7546615600586,
      "activations/layer14_attention_weight_min": -47.37539291381836,
      "activations/layer15_attention_weight_max": 42.53014373779297,
      "activations/layer15_attention_weight_min": -28.15957260131836,
      "activations/layer16_attention_weight_max": 43.17020034790039,
      "activations/layer16_attention_weight_min": -26.915050506591797,
      "activations/layer17_attention_weight_max": 42.70314407348633,
      "activations/layer17_attention_weight_min": -24.378582000732422,
      "activations/layer18_attention_weight_max": 33.81831741333008,
      "activations/layer18_attention_weight_min": -19.495363235473633,
      "activations/layer19_attention_weight_max": 41.83405303955078,
      "activations/layer19_attention_weight_min": -29.367542266845703,
      "activations/layer1_attention_weight_max": 17.314598083496094,
      "activations/layer1_attention_weight_min": -14.09285831451416,
      "activations/layer20_attention_weight_max": 36.60273361206055,
      "activations/layer20_attention_weight_min": -23.64066505432129,
      "activations/layer21_attention_weight_max": 34.89812469482422,
      "activations/layer21_attention_weight_min": -23.270681381225586,
      "activations/layer22_attention_weight_max": 53.10181427001953,
      "activations/layer22_attention_weight_min": -27.69005584716797,
      "activations/layer23_attention_weight_max": 36.14500045776367,
      "activations/layer23_attention_weight_min": -20.17569351196289,
      "activations/layer2_attention_weight_max": 33.101600646972656,
      "activations/layer2_attention_weight_min": -31.597518920898438,
      "activations/layer3_attention_weight_max": 94.149658203125,
      "activations/layer3_attention_weight_min": -91.01902770996094,
      "activations/layer4_attention_weight_max": 132.9032745361328,
      "activations/layer4_attention_weight_min": -131.1103973388672,
      "activations/layer5_attention_weight_max": 83.46139526367188,
      "activations/layer5_attention_weight_min": -80.83422088623047,
      "activations/layer6_attention_weight_max": 55.696746826171875,
      "activations/layer6_attention_weight_min": -56.30854034423828,
      "activations/layer7_attention_weight_max": 84.32703399658203,
      "activations/layer7_attention_weight_min": -83.03984832763672,
      "activations/layer8_attention_weight_max": 40.98277282714844,
      "activations/layer8_attention_weight_min": -39.62923812866211,
      "activations/layer9_attention_weight_max": 37.18449020385742,
      "activations/layer9_attention_weight_min": -34.208465576171875,
      "epoch": 5.32,
      "learning_rate": 0.00011688939393939392,
      "loss": 2.8412,
      "step": 91500
    },
    {
      "activations/layer0_attention_weight_max": 15.616345405578613,
      "activations/layer0_attention_weight_min": -12.510967254638672,
      "activations/layer10_attention_weight_max": 35.78910827636719,
      "activations/layer10_attention_weight_min": -33.0737190246582,
      "activations/layer11_attention_weight_max": 36.096778869628906,
      "activations/layer11_attention_weight_min": -33.90666198730469,
      "activations/layer12_attention_weight_max": 42.25774383544922,
      "activations/layer12_attention_weight_min": -27.84842300415039,
      "activations/layer13_attention_weight_max": 64.87090301513672,
      "activations/layer13_attention_weight_min": -46.191097259521484,
      "activations/layer14_attention_weight_max": 71.51628112792969,
      "activations/layer14_attention_weight_min": -47.083595275878906,
      "activations/layer15_attention_weight_max": 48.48193359375,
      "activations/layer15_attention_weight_min": -37.01070022583008,
      "activations/layer16_attention_weight_max": 38.86576843261719,
      "activations/layer16_attention_weight_min": -27.321435928344727,
      "activations/layer17_attention_weight_max": 38.60411834716797,
      "activations/layer17_attention_weight_min": -27.98152732849121,
      "activations/layer18_attention_weight_max": 33.455528259277344,
      "activations/layer18_attention_weight_min": -21.1562557220459,
      "activations/layer19_attention_weight_max": 42.84251403808594,
      "activations/layer19_attention_weight_min": -31.554075241088867,
      "activations/layer1_attention_weight_max": 17.98472023010254,
      "activations/layer1_attention_weight_min": -15.326994895935059,
      "activations/layer20_attention_weight_max": 34.234039306640625,
      "activations/layer20_attention_weight_min": -25.7127685546875,
      "activations/layer21_attention_weight_max": 31.04669189453125,
      "activations/layer21_attention_weight_min": -25.345260620117188,
      "activations/layer22_attention_weight_max": 56.71546936035156,
      "activations/layer22_attention_weight_min": -31.580108642578125,
      "activations/layer23_attention_weight_max": 36.21875,
      "activations/layer23_attention_weight_min": -22.01307487487793,
      "activations/layer2_attention_weight_max": 32.238590240478516,
      "activations/layer2_attention_weight_min": -31.327970504760742,
      "activations/layer3_attention_weight_max": 91.38565063476562,
      "activations/layer3_attention_weight_min": -91.52051544189453,
      "activations/layer4_attention_weight_max": 138.27122497558594,
      "activations/layer4_attention_weight_min": -125.83522033691406,
      "activations/layer5_attention_weight_max": 79.84748840332031,
      "activations/layer5_attention_weight_min": -74.1318359375,
      "activations/layer6_attention_weight_max": 56.95450973510742,
      "activations/layer6_attention_weight_min": -54.29788589477539,
      "activations/layer7_attention_weight_max": 99.6281509399414,
      "activations/layer7_attention_weight_min": -80.49015808105469,
      "activations/layer8_attention_weight_max": 39.621604919433594,
      "activations/layer8_attention_weight_min": -39.97134017944336,
      "activations/layer9_attention_weight_max": 39.96342086791992,
      "activations/layer9_attention_weight_min": -34.30207824707031,
      "epoch": 5.32,
      "learning_rate": 0.00011687045454545454,
      "loss": 2.8412,
      "step": 91550
    },
    {
      "activations/layer0_attention_weight_max": 15.778148651123047,
      "activations/layer0_attention_weight_min": -11.808283805847168,
      "activations/layer10_attention_weight_max": 41.1864013671875,
      "activations/layer10_attention_weight_min": -34.8758430480957,
      "activations/layer11_attention_weight_max": 43.19237518310547,
      "activations/layer11_attention_weight_min": -32.30595779418945,
      "activations/layer12_attention_weight_max": 26.98991584777832,
      "activations/layer12_attention_weight_min": -28.37738609313965,
      "activations/layer13_attention_weight_max": 43.41730499267578,
      "activations/layer13_attention_weight_min": -32.86095428466797,
      "activations/layer14_attention_weight_max": 69.62139129638672,
      "activations/layer14_attention_weight_min": -46.703880310058594,
      "activations/layer15_attention_weight_max": 39.317527770996094,
      "activations/layer15_attention_weight_min": -30.060874938964844,
      "activations/layer16_attention_weight_max": 39.34727478027344,
      "activations/layer16_attention_weight_min": -29.380352020263672,
      "activations/layer17_attention_weight_max": 42.4235725402832,
      "activations/layer17_attention_weight_min": -26.15061378479004,
      "activations/layer18_attention_weight_max": 31.44383430480957,
      "activations/layer18_attention_weight_min": -21.630184173583984,
      "activations/layer19_attention_weight_max": 41.254844665527344,
      "activations/layer19_attention_weight_min": -32.729434967041016,
      "activations/layer1_attention_weight_max": 18.530719757080078,
      "activations/layer1_attention_weight_min": -14.751442909240723,
      "activations/layer20_attention_weight_max": 31.265361785888672,
      "activations/layer20_attention_weight_min": -23.875080108642578,
      "activations/layer21_attention_weight_max": 34.953948974609375,
      "activations/layer21_attention_weight_min": -23.640356063842773,
      "activations/layer22_attention_weight_max": 55.33699417114258,
      "activations/layer22_attention_weight_min": -27.90646743774414,
      "activations/layer23_attention_weight_max": 35.220542907714844,
      "activations/layer23_attention_weight_min": -19.89240264892578,
      "activations/layer2_attention_weight_max": 34.542144775390625,
      "activations/layer2_attention_weight_min": -34.262939453125,
      "activations/layer3_attention_weight_max": 100.3656997680664,
      "activations/layer3_attention_weight_min": -101.02568054199219,
      "activations/layer4_attention_weight_max": 148.123779296875,
      "activations/layer4_attention_weight_min": -142.19671630859375,
      "activations/layer5_attention_weight_max": 93.68667602539062,
      "activations/layer5_attention_weight_min": -87.70065307617188,
      "activations/layer6_attention_weight_max": 66.80445861816406,
      "activations/layer6_attention_weight_min": -65.78855895996094,
      "activations/layer7_attention_weight_max": 105.67456817626953,
      "activations/layer7_attention_weight_min": -98.16536712646484,
      "activations/layer8_attention_weight_max": 49.689823150634766,
      "activations/layer8_attention_weight_min": -47.06504440307617,
      "activations/layer9_attention_weight_max": 43.41099166870117,
      "activations/layer9_attention_weight_min": -38.983333587646484,
      "epoch": 5.32,
      "learning_rate": 0.00011685151515151515,
      "loss": 2.8462,
      "step": 91600
    },
    {
      "activations/layer0_attention_weight_max": 16.7346248626709,
      "activations/layer0_attention_weight_min": -11.525066375732422,
      "activations/layer10_attention_weight_max": 36.05143356323242,
      "activations/layer10_attention_weight_min": -30.691123962402344,
      "activations/layer11_attention_weight_max": 31.88864517211914,
      "activations/layer11_attention_weight_min": -27.568328857421875,
      "activations/layer12_attention_weight_max": 22.528844833374023,
      "activations/layer12_attention_weight_min": -25.998775482177734,
      "activations/layer13_attention_weight_max": 42.886295318603516,
      "activations/layer13_attention_weight_min": -30.827600479125977,
      "activations/layer14_attention_weight_max": 60.38413619995117,
      "activations/layer14_attention_weight_min": -43.2242431640625,
      "activations/layer15_attention_weight_max": 38.973350524902344,
      "activations/layer15_attention_weight_min": -30.110294342041016,
      "activations/layer16_attention_weight_max": 38.50505828857422,
      "activations/layer16_attention_weight_min": -27.363252639770508,
      "activations/layer17_attention_weight_max": 36.1927375793457,
      "activations/layer17_attention_weight_min": -24.607463836669922,
      "activations/layer18_attention_weight_max": 30.956331253051758,
      "activations/layer18_attention_weight_min": -20.32897186279297,
      "activations/layer19_attention_weight_max": 38.83536148071289,
      "activations/layer19_attention_weight_min": -29.749353408813477,
      "activations/layer1_attention_weight_max": 18.159151077270508,
      "activations/layer1_attention_weight_min": -15.201679229736328,
      "activations/layer20_attention_weight_max": 33.77579116821289,
      "activations/layer20_attention_weight_min": -22.71146011352539,
      "activations/layer21_attention_weight_max": 34.21236801147461,
      "activations/layer21_attention_weight_min": -23.945812225341797,
      "activations/layer22_attention_weight_max": 54.2719612121582,
      "activations/layer22_attention_weight_min": -29.78159523010254,
      "activations/layer23_attention_weight_max": 34.071937561035156,
      "activations/layer23_attention_weight_min": -21.860275268554688,
      "activations/layer2_attention_weight_max": 32.350624084472656,
      "activations/layer2_attention_weight_min": -32.323890686035156,
      "activations/layer3_attention_weight_max": 89.60054779052734,
      "activations/layer3_attention_weight_min": -92.7855224609375,
      "activations/layer4_attention_weight_max": 128.43206787109375,
      "activations/layer4_attention_weight_min": -125.6187744140625,
      "activations/layer5_attention_weight_max": 79.76863861083984,
      "activations/layer5_attention_weight_min": -80.11196899414062,
      "activations/layer6_attention_weight_max": 56.563629150390625,
      "activations/layer6_attention_weight_min": -57.77214431762695,
      "activations/layer7_attention_weight_max": 83.5519027709961,
      "activations/layer7_attention_weight_min": -83.23725128173828,
      "activations/layer8_attention_weight_max": 39.65135955810547,
      "activations/layer8_attention_weight_min": -39.88252258300781,
      "activations/layer9_attention_weight_max": 37.703651428222656,
      "activations/layer9_attention_weight_min": -35.51438903808594,
      "epoch": 5.33,
      "learning_rate": 0.00011683257575757574,
      "loss": 2.8455,
      "step": 91650
    },
    {
      "activations/layer0_attention_weight_max": 16.444570541381836,
      "activations/layer0_attention_weight_min": -12.27745246887207,
      "activations/layer10_attention_weight_max": 33.42268753051758,
      "activations/layer10_attention_weight_min": -30.007497787475586,
      "activations/layer11_attention_weight_max": 31.533527374267578,
      "activations/layer11_attention_weight_min": -27.22791862487793,
      "activations/layer12_attention_weight_max": 24.03753089904785,
      "activations/layer12_attention_weight_min": -26.783639907836914,
      "activations/layer13_attention_weight_max": 41.559906005859375,
      "activations/layer13_attention_weight_min": -32.93479537963867,
      "activations/layer14_attention_weight_max": 59.19536590576172,
      "activations/layer14_attention_weight_min": -45.75531005859375,
      "activations/layer15_attention_weight_max": 40.020477294921875,
      "activations/layer15_attention_weight_min": -28.897823333740234,
      "activations/layer16_attention_weight_max": 37.53313064575195,
      "activations/layer16_attention_weight_min": -29.432214736938477,
      "activations/layer17_attention_weight_max": 38.47612380981445,
      "activations/layer17_attention_weight_min": -25.572978973388672,
      "activations/layer18_attention_weight_max": 30.554269790649414,
      "activations/layer18_attention_weight_min": -20.53911590576172,
      "activations/layer19_attention_weight_max": 38.16393280029297,
      "activations/layer19_attention_weight_min": -29.41355323791504,
      "activations/layer1_attention_weight_max": 17.61848258972168,
      "activations/layer1_attention_weight_min": -14.820501327514648,
      "activations/layer20_attention_weight_max": 30.7679500579834,
      "activations/layer20_attention_weight_min": -24.05267333984375,
      "activations/layer21_attention_weight_max": 31.61453628540039,
      "activations/layer21_attention_weight_min": -21.038902282714844,
      "activations/layer22_attention_weight_max": 45.9610595703125,
      "activations/layer22_attention_weight_min": -28.71280288696289,
      "activations/layer23_attention_weight_max": 32.80271530151367,
      "activations/layer23_attention_weight_min": -21.408605575561523,
      "activations/layer2_attention_weight_max": 32.193016052246094,
      "activations/layer2_attention_weight_min": -30.198467254638672,
      "activations/layer3_attention_weight_max": 92.39337158203125,
      "activations/layer3_attention_weight_min": -89.81209564208984,
      "activations/layer4_attention_weight_max": 133.9623260498047,
      "activations/layer4_attention_weight_min": -126.07621002197266,
      "activations/layer5_attention_weight_max": 77.22359466552734,
      "activations/layer5_attention_weight_min": -76.84759521484375,
      "activations/layer6_attention_weight_max": 54.43846893310547,
      "activations/layer6_attention_weight_min": -52.89816665649414,
      "activations/layer7_attention_weight_max": 86.7401123046875,
      "activations/layer7_attention_weight_min": -84.42247009277344,
      "activations/layer8_attention_weight_max": 37.094703674316406,
      "activations/layer8_attention_weight_min": -37.48638916015625,
      "activations/layer9_attention_weight_max": 34.6308708190918,
      "activations/layer9_attention_weight_min": -32.60944747924805,
      "epoch": 5.33,
      "learning_rate": 0.00011681363636363635,
      "loss": 2.851,
      "step": 91700
    },
    {
      "activations/layer0_attention_weight_max": 15.066490173339844,
      "activations/layer0_attention_weight_min": -11.901904106140137,
      "activations/layer10_attention_weight_max": 35.62785339355469,
      "activations/layer10_attention_weight_min": -32.009857177734375,
      "activations/layer11_attention_weight_max": 34.42958068847656,
      "activations/layer11_attention_weight_min": -28.889875411987305,
      "activations/layer12_attention_weight_max": 57.76573944091797,
      "activations/layer12_attention_weight_min": -28.967844009399414,
      "activations/layer13_attention_weight_max": 48.61254119873047,
      "activations/layer13_attention_weight_min": -32.123050689697266,
      "activations/layer14_attention_weight_max": 56.936458587646484,
      "activations/layer14_attention_weight_min": -43.9347038269043,
      "activations/layer15_attention_weight_max": 37.802127838134766,
      "activations/layer15_attention_weight_min": -29.05937385559082,
      "activations/layer16_attention_weight_max": 37.307891845703125,
      "activations/layer16_attention_weight_min": -27.402618408203125,
      "activations/layer17_attention_weight_max": 38.87029266357422,
      "activations/layer17_attention_weight_min": -25.37691307067871,
      "activations/layer18_attention_weight_max": 28.44219398498535,
      "activations/layer18_attention_weight_min": -18.78200912475586,
      "activations/layer19_attention_weight_max": 40.84908676147461,
      "activations/layer19_attention_weight_min": -29.235198974609375,
      "activations/layer1_attention_weight_max": 16.658159255981445,
      "activations/layer1_attention_weight_min": -14.645421028137207,
      "activations/layer20_attention_weight_max": 36.20892333984375,
      "activations/layer20_attention_weight_min": -25.972593307495117,
      "activations/layer21_attention_weight_max": 32.679405212402344,
      "activations/layer21_attention_weight_min": -20.46244239807129,
      "activations/layer22_attention_weight_max": 50.05418014526367,
      "activations/layer22_attention_weight_min": -28.193506240844727,
      "activations/layer23_attention_weight_max": 37.173118591308594,
      "activations/layer23_attention_weight_min": -20.507266998291016,
      "activations/layer2_attention_weight_max": 33.20578384399414,
      "activations/layer2_attention_weight_min": -32.842376708984375,
      "activations/layer3_attention_weight_max": 91.28472137451172,
      "activations/layer3_attention_weight_min": -92.1266098022461,
      "activations/layer4_attention_weight_max": 133.76097106933594,
      "activations/layer4_attention_weight_min": -132.5706024169922,
      "activations/layer5_attention_weight_max": 80.66046905517578,
      "activations/layer5_attention_weight_min": -77.37177276611328,
      "activations/layer6_attention_weight_max": 56.38701248168945,
      "activations/layer6_attention_weight_min": -60.629180908203125,
      "activations/layer7_attention_weight_max": 86.9632568359375,
      "activations/layer7_attention_weight_min": -93.21251678466797,
      "activations/layer8_attention_weight_max": 43.5154914855957,
      "activations/layer8_attention_weight_min": -44.81984329223633,
      "activations/layer9_attention_weight_max": 39.32027816772461,
      "activations/layer9_attention_weight_min": -40.313743591308594,
      "epoch": 5.33,
      "learning_rate": 0.00011679469696969697,
      "loss": 2.8388,
      "step": 91750
    },
    {
      "activations/layer0_attention_weight_max": 15.224088668823242,
      "activations/layer0_attention_weight_min": -12.042049407958984,
      "activations/layer10_attention_weight_max": 34.22906494140625,
      "activations/layer10_attention_weight_min": -31.088274002075195,
      "activations/layer11_attention_weight_max": 34.05719757080078,
      "activations/layer11_attention_weight_min": -29.339115142822266,
      "activations/layer12_attention_weight_max": 24.47539710998535,
      "activations/layer12_attention_weight_min": -28.76835060119629,
      "activations/layer13_attention_weight_max": 44.52549743652344,
      "activations/layer13_attention_weight_min": -34.11646270751953,
      "activations/layer14_attention_weight_max": 62.969234466552734,
      "activations/layer14_attention_weight_min": -46.50678634643555,
      "activations/layer15_attention_weight_max": 38.26529312133789,
      "activations/layer15_attention_weight_min": -28.993820190429688,
      "activations/layer16_attention_weight_max": 37.855499267578125,
      "activations/layer16_attention_weight_min": -27.524324417114258,
      "activations/layer17_attention_weight_max": 40.284271240234375,
      "activations/layer17_attention_weight_min": -24.030176162719727,
      "activations/layer18_attention_weight_max": 33.70622634887695,
      "activations/layer18_attention_weight_min": -20.333152770996094,
      "activations/layer19_attention_weight_max": 43.11068344116211,
      "activations/layer19_attention_weight_min": -29.617889404296875,
      "activations/layer1_attention_weight_max": 16.655580520629883,
      "activations/layer1_attention_weight_min": -13.68010139465332,
      "activations/layer20_attention_weight_max": 33.83103942871094,
      "activations/layer20_attention_weight_min": -22.471078872680664,
      "activations/layer21_attention_weight_max": 32.35574722290039,
      "activations/layer21_attention_weight_min": -21.61651611328125,
      "activations/layer22_attention_weight_max": 59.66893005371094,
      "activations/layer22_attention_weight_min": -30.01925277709961,
      "activations/layer23_attention_weight_max": 33.128761291503906,
      "activations/layer23_attention_weight_min": -20.353897094726562,
      "activations/layer2_attention_weight_max": 33.05511474609375,
      "activations/layer2_attention_weight_min": -34.26386642456055,
      "activations/layer3_attention_weight_max": 93.1265640258789,
      "activations/layer3_attention_weight_min": -98.84149932861328,
      "activations/layer4_attention_weight_max": 134.06317138671875,
      "activations/layer4_attention_weight_min": -133.42434692382812,
      "activations/layer5_attention_weight_max": 79.54048156738281,
      "activations/layer5_attention_weight_min": -79.79149627685547,
      "activations/layer6_attention_weight_max": 57.41322326660156,
      "activations/layer6_attention_weight_min": -56.87593460083008,
      "activations/layer7_attention_weight_max": 88.20911407470703,
      "activations/layer7_attention_weight_min": -90.26983642578125,
      "activations/layer8_attention_weight_max": 40.7189826965332,
      "activations/layer8_attention_weight_min": -41.123844146728516,
      "activations/layer9_attention_weight_max": 37.49705123901367,
      "activations/layer9_attention_weight_min": -36.26493453979492,
      "epoch": 5.33,
      "learning_rate": 0.00011677575757575756,
      "loss": 2.8436,
      "step": 91800
    },
    {
      "activations/layer0_attention_weight_max": 15.574198722839355,
      "activations/layer0_attention_weight_min": -11.407371520996094,
      "activations/layer10_attention_weight_max": 36.01563262939453,
      "activations/layer10_attention_weight_min": -30.597244262695312,
      "activations/layer11_attention_weight_max": 35.029197692871094,
      "activations/layer11_attention_weight_min": -29.924945831298828,
      "activations/layer12_attention_weight_max": 29.029666900634766,
      "activations/layer12_attention_weight_min": -27.458772659301758,
      "activations/layer13_attention_weight_max": 44.42786407470703,
      "activations/layer13_attention_weight_min": -32.52992248535156,
      "activations/layer14_attention_weight_max": 62.595455169677734,
      "activations/layer14_attention_weight_min": -45.94083023071289,
      "activations/layer15_attention_weight_max": 48.75572967529297,
      "activations/layer15_attention_weight_min": -30.78141212463379,
      "activations/layer16_attention_weight_max": 45.027069091796875,
      "activations/layer16_attention_weight_min": -28.885833740234375,
      "activations/layer17_attention_weight_max": 47.68613052368164,
      "activations/layer17_attention_weight_min": -27.250286102294922,
      "activations/layer18_attention_weight_max": 31.998647689819336,
      "activations/layer18_attention_weight_min": -19.135377883911133,
      "activations/layer19_attention_weight_max": 44.3557014465332,
      "activations/layer19_attention_weight_min": -29.470829010009766,
      "activations/layer1_attention_weight_max": 17.717065811157227,
      "activations/layer1_attention_weight_min": -16.273813247680664,
      "activations/layer20_attention_weight_max": 35.44301986694336,
      "activations/layer20_attention_weight_min": -23.338640213012695,
      "activations/layer21_attention_weight_max": 32.911956787109375,
      "activations/layer21_attention_weight_min": -21.852222442626953,
      "activations/layer22_attention_weight_max": 55.21952438354492,
      "activations/layer22_attention_weight_min": -31.571638107299805,
      "activations/layer23_attention_weight_max": 33.18434524536133,
      "activations/layer23_attention_weight_min": -19.977338790893555,
      "activations/layer2_attention_weight_max": 31.031341552734375,
      "activations/layer2_attention_weight_min": -31.864322662353516,
      "activations/layer3_attention_weight_max": 88.06381225585938,
      "activations/layer3_attention_weight_min": -89.74632263183594,
      "activations/layer4_attention_weight_max": 131.83255004882812,
      "activations/layer4_attention_weight_min": -129.86068725585938,
      "activations/layer5_attention_weight_max": 81.18760681152344,
      "activations/layer5_attention_weight_min": -78.53855895996094,
      "activations/layer6_attention_weight_max": 57.31663513183594,
      "activations/layer6_attention_weight_min": -57.14178466796875,
      "activations/layer7_attention_weight_max": 87.45648956298828,
      "activations/layer7_attention_weight_min": -91.406982421875,
      "activations/layer8_attention_weight_max": 44.333526611328125,
      "activations/layer8_attention_weight_min": -39.50654983520508,
      "activations/layer9_attention_weight_max": 41.59807205200195,
      "activations/layer9_attention_weight_min": -35.356990814208984,
      "epoch": 5.34,
      "learning_rate": 0.00011675681818181817,
      "loss": 2.8413,
      "step": 91850
    },
    {
      "activations/layer0_attention_weight_max": 16.716747283935547,
      "activations/layer0_attention_weight_min": -11.938667297363281,
      "activations/layer10_attention_weight_max": 33.58890914916992,
      "activations/layer10_attention_weight_min": -30.03908920288086,
      "activations/layer11_attention_weight_max": 30.60647201538086,
      "activations/layer11_attention_weight_min": -28.11420440673828,
      "activations/layer12_attention_weight_max": 30.08926010131836,
      "activations/layer12_attention_weight_min": -25.88971519470215,
      "activations/layer13_attention_weight_max": 42.85475158691406,
      "activations/layer13_attention_weight_min": -33.887969970703125,
      "activations/layer14_attention_weight_max": 61.024559020996094,
      "activations/layer14_attention_weight_min": -47.306854248046875,
      "activations/layer15_attention_weight_max": 39.26406478881836,
      "activations/layer15_attention_weight_min": -28.66207504272461,
      "activations/layer16_attention_weight_max": 44.87832260131836,
      "activations/layer16_attention_weight_min": -32.123416900634766,
      "activations/layer17_attention_weight_max": 41.09069061279297,
      "activations/layer17_attention_weight_min": -25.882369995117188,
      "activations/layer18_attention_weight_max": 34.45899963378906,
      "activations/layer18_attention_weight_min": -22.824108123779297,
      "activations/layer19_attention_weight_max": 48.80721664428711,
      "activations/layer19_attention_weight_min": -29.49348258972168,
      "activations/layer1_attention_weight_max": 16.628307342529297,
      "activations/layer1_attention_weight_min": -14.40621280670166,
      "activations/layer20_attention_weight_max": 38.163814544677734,
      "activations/layer20_attention_weight_min": -24.467449188232422,
      "activations/layer21_attention_weight_max": 35.85908508300781,
      "activations/layer21_attention_weight_min": -23.113800048828125,
      "activations/layer22_attention_weight_max": 52.44229507446289,
      "activations/layer22_attention_weight_min": -27.02075958251953,
      "activations/layer23_attention_weight_max": 36.4406852722168,
      "activations/layer23_attention_weight_min": -22.384695053100586,
      "activations/layer2_attention_weight_max": 31.201688766479492,
      "activations/layer2_attention_weight_min": -31.248451232910156,
      "activations/layer3_attention_weight_max": 89.60884857177734,
      "activations/layer3_attention_weight_min": -92.31038665771484,
      "activations/layer4_attention_weight_max": 128.10116577148438,
      "activations/layer4_attention_weight_min": -126.47016906738281,
      "activations/layer5_attention_weight_max": 79.94122314453125,
      "activations/layer5_attention_weight_min": -79.34581756591797,
      "activations/layer6_attention_weight_max": 56.60167694091797,
      "activations/layer6_attention_weight_min": -58.47834396362305,
      "activations/layer7_attention_weight_max": 82.4056625366211,
      "activations/layer7_attention_weight_min": -90.0572509765625,
      "activations/layer8_attention_weight_max": 41.211769104003906,
      "activations/layer8_attention_weight_min": -41.40739822387695,
      "activations/layer9_attention_weight_max": 38.63125991821289,
      "activations/layer9_attention_weight_min": -35.24665832519531,
      "epoch": 5.34,
      "learning_rate": 0.00011673787878787879,
      "loss": 2.8649,
      "step": 91900
    },
    {
      "activations/layer0_attention_weight_max": 16.851720809936523,
      "activations/layer0_attention_weight_min": -11.749299049377441,
      "activations/layer10_attention_weight_max": 33.54671096801758,
      "activations/layer10_attention_weight_min": -31.67574119567871,
      "activations/layer11_attention_weight_max": 30.610029220581055,
      "activations/layer11_attention_weight_min": -27.616825103759766,
      "activations/layer12_attention_weight_max": 26.514652252197266,
      "activations/layer12_attention_weight_min": -30.415864944458008,
      "activations/layer13_attention_weight_max": 41.69316482543945,
      "activations/layer13_attention_weight_min": -33.725013732910156,
      "activations/layer14_attention_weight_max": 52.547950744628906,
      "activations/layer14_attention_weight_min": -45.339385986328125,
      "activations/layer15_attention_weight_max": 40.770782470703125,
      "activations/layer15_attention_weight_min": -30.7751522064209,
      "activations/layer16_attention_weight_max": 39.602874755859375,
      "activations/layer16_attention_weight_min": -29.51662254333496,
      "activations/layer17_attention_weight_max": 41.96631622314453,
      "activations/layer17_attention_weight_min": -26.31806182861328,
      "activations/layer18_attention_weight_max": 34.39549255371094,
      "activations/layer18_attention_weight_min": -18.90921974182129,
      "activations/layer19_attention_weight_max": 43.43864822387695,
      "activations/layer19_attention_weight_min": -29.10223388671875,
      "activations/layer1_attention_weight_max": 17.543018341064453,
      "activations/layer1_attention_weight_min": -16.99079132080078,
      "activations/layer20_attention_weight_max": 33.248939514160156,
      "activations/layer20_attention_weight_min": -22.338529586791992,
      "activations/layer21_attention_weight_max": 34.23869323730469,
      "activations/layer21_attention_weight_min": -19.915515899658203,
      "activations/layer22_attention_weight_max": 53.74275588989258,
      "activations/layer22_attention_weight_min": -26.058916091918945,
      "activations/layer23_attention_weight_max": 33.2140007019043,
      "activations/layer23_attention_weight_min": -19.07382583618164,
      "activations/layer2_attention_weight_max": 32.62571334838867,
      "activations/layer2_attention_weight_min": -31.586891174316406,
      "activations/layer3_attention_weight_max": 89.98670959472656,
      "activations/layer3_attention_weight_min": -89.35980987548828,
      "activations/layer4_attention_weight_max": 131.63392639160156,
      "activations/layer4_attention_weight_min": -126.7735366821289,
      "activations/layer5_attention_weight_max": 81.32937622070312,
      "activations/layer5_attention_weight_min": -78.09733581542969,
      "activations/layer6_attention_weight_max": 59.252891540527344,
      "activations/layer6_attention_weight_min": -57.114349365234375,
      "activations/layer7_attention_weight_max": 88.0543212890625,
      "activations/layer7_attention_weight_min": -89.47432708740234,
      "activations/layer8_attention_weight_max": 40.72216796875,
      "activations/layer8_attention_weight_min": -38.9580192565918,
      "activations/layer9_attention_weight_max": 36.88470458984375,
      "activations/layer9_attention_weight_min": -34.7819938659668,
      "epoch": 5.34,
      "learning_rate": 0.00011671893939393938,
      "loss": 2.8351,
      "step": 91950
    },
    {
      "activations/layer0_attention_weight_max": 16.808015823364258,
      "activations/layer0_attention_weight_min": -13.055842399597168,
      "activations/layer10_attention_weight_max": 35.826393127441406,
      "activations/layer10_attention_weight_min": -32.530517578125,
      "activations/layer11_attention_weight_max": 33.525421142578125,
      "activations/layer11_attention_weight_min": -28.871482849121094,
      "activations/layer12_attention_weight_max": 27.64493179321289,
      "activations/layer12_attention_weight_min": -25.066802978515625,
      "activations/layer13_attention_weight_max": 52.20881271362305,
      "activations/layer13_attention_weight_min": -38.033241271972656,
      "activations/layer14_attention_weight_max": 77.42838287353516,
      "activations/layer14_attention_weight_min": -53.1695556640625,
      "activations/layer15_attention_weight_max": 45.0716552734375,
      "activations/layer15_attention_weight_min": -29.60072898864746,
      "activations/layer16_attention_weight_max": 37.263427734375,
      "activations/layer16_attention_weight_min": -27.573749542236328,
      "activations/layer17_attention_weight_max": 38.03007888793945,
      "activations/layer17_attention_weight_min": -23.77429962158203,
      "activations/layer18_attention_weight_max": 38.080387115478516,
      "activations/layer18_attention_weight_min": -19.93984603881836,
      "activations/layer19_attention_weight_max": 43.39424514770508,
      "activations/layer19_attention_weight_min": -32.57883834838867,
      "activations/layer1_attention_weight_max": 16.873003005981445,
      "activations/layer1_attention_weight_min": -15.14505672454834,
      "activations/layer20_attention_weight_max": 33.21927261352539,
      "activations/layer20_attention_weight_min": -25.364877700805664,
      "activations/layer21_attention_weight_max": 33.21122741699219,
      "activations/layer21_attention_weight_min": -21.88194465637207,
      "activations/layer22_attention_weight_max": 51.30874252319336,
      "activations/layer22_attention_weight_min": -28.30689239501953,
      "activations/layer23_attention_weight_max": 38.332191467285156,
      "activations/layer23_attention_weight_min": -20.634502410888672,
      "activations/layer2_attention_weight_max": 34.59532165527344,
      "activations/layer2_attention_weight_min": -32.93549728393555,
      "activations/layer3_attention_weight_max": 94.0593490600586,
      "activations/layer3_attention_weight_min": -93.50694274902344,
      "activations/layer4_attention_weight_max": 136.68312072753906,
      "activations/layer4_attention_weight_min": -132.69564819335938,
      "activations/layer5_attention_weight_max": 84.13262176513672,
      "activations/layer5_attention_weight_min": -82.8785171508789,
      "activations/layer6_attention_weight_max": 55.94292068481445,
      "activations/layer6_attention_weight_min": -58.32415771484375,
      "activations/layer7_attention_weight_max": 87.63419342041016,
      "activations/layer7_attention_weight_min": -94.7389907836914,
      "activations/layer8_attention_weight_max": 40.51258087158203,
      "activations/layer8_attention_weight_min": -41.56331253051758,
      "activations/layer9_attention_weight_max": 38.730682373046875,
      "activations/layer9_attention_weight_min": -37.74005889892578,
      "epoch": 5.35,
      "learning_rate": 0.00011669999999999999,
      "loss": 2.8637,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_loss": 2.796875,
      "eval_runtime": 8.5104,
      "eval_samples_per_second": 504.562,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_openwebtext_loss": 2.796875,
      "eval_openwebtext_ppl": 16.39333746248769,
      "eval_openwebtext_runtime": 8.5104,
      "eval_openwebtext_samples_per_second": 504.562,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_wikitext_loss": 3.025390625,
      "eval_wikitext_ppl": 20.602050815688813,
      "eval_wikitext_runtime": 2.0023,
      "eval_wikitext_samples_per_second": 227.74,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_lambada_loss": 2.68359375,
      "eval_lambada_ppl": 14.637602771363136,
      "eval_lambada_runtime": 9.6407,
      "eval_lambada_samples_per_second": 505.045,
      "step": 92000
    },
    {
      "activations/layer0_attention_weight_max": 17.03696632385254,
      "activations/layer0_attention_weight_min": -11.301340103149414,
      "activations/layer10_attention_weight_max": 36.61329650878906,
      "activations/layer10_attention_weight_min": -32.19537353515625,
      "activations/layer11_attention_weight_max": 38.472564697265625,
      "activations/layer11_attention_weight_min": -29.622364044189453,
      "activations/layer12_attention_weight_max": 47.96216583251953,
      "activations/layer12_attention_weight_min": -27.29747772216797,
      "activations/layer13_attention_weight_max": 71.86805725097656,
      "activations/layer13_attention_weight_min": -43.159297943115234,
      "activations/layer14_attention_weight_max": 69.9084701538086,
      "activations/layer14_attention_weight_min": -46.689903259277344,
      "activations/layer15_attention_weight_max": 48.64445114135742,
      "activations/layer15_attention_weight_min": -29.422077178955078,
      "activations/layer16_attention_weight_max": 40.25169372558594,
      "activations/layer16_attention_weight_min": -26.763113021850586,
      "activations/layer17_attention_weight_max": 38.08207702636719,
      "activations/layer17_attention_weight_min": -23.39608383178711,
      "activations/layer18_attention_weight_max": 33.08409118652344,
      "activations/layer18_attention_weight_min": -22.15609359741211,
      "activations/layer19_attention_weight_max": 44.13926696777344,
      "activations/layer19_attention_weight_min": -29.607248306274414,
      "activations/layer1_attention_weight_max": 16.792293548583984,
      "activations/layer1_attention_weight_min": -14.766222953796387,
      "activations/layer20_attention_weight_max": 35.26716995239258,
      "activations/layer20_attention_weight_min": -23.60721206665039,
      "activations/layer21_attention_weight_max": 34.93962478637695,
      "activations/layer21_attention_weight_min": -20.59610366821289,
      "activations/layer22_attention_weight_max": 51.71363830566406,
      "activations/layer22_attention_weight_min": -28.17756462097168,
      "activations/layer23_attention_weight_max": 34.48762130737305,
      "activations/layer23_attention_weight_min": -18.573030471801758,
      "activations/layer2_attention_weight_max": 35.582069396972656,
      "activations/layer2_attention_weight_min": -35.3690071105957,
      "activations/layer3_attention_weight_max": 92.65166473388672,
      "activations/layer3_attention_weight_min": -93.8973617553711,
      "activations/layer4_attention_weight_max": 135.68296813964844,
      "activations/layer4_attention_weight_min": -131.6693572998047,
      "activations/layer5_attention_weight_max": 85.14595031738281,
      "activations/layer5_attention_weight_min": -82.15536499023438,
      "activations/layer6_attention_weight_max": 56.80805206298828,
      "activations/layer6_attention_weight_min": -56.84800338745117,
      "activations/layer7_attention_weight_max": 90.17996978759766,
      "activations/layer7_attention_weight_min": -88.8434066772461,
      "activations/layer8_attention_weight_max": 43.164573669433594,
      "activations/layer8_attention_weight_min": -43.7039794921875,
      "activations/layer9_attention_weight_max": 38.492774963378906,
      "activations/layer9_attention_weight_min": -36.966304779052734,
      "epoch": 5.35,
      "learning_rate": 0.00011668106060606059,
      "loss": 2.8278,
      "step": 92050
    },
    {
      "activations/layer0_attention_weight_max": 15.456415176391602,
      "activations/layer0_attention_weight_min": -11.187531471252441,
      "activations/layer10_attention_weight_max": 35.584022521972656,
      "activations/layer10_attention_weight_min": -31.190166473388672,
      "activations/layer11_attention_weight_max": 37.563690185546875,
      "activations/layer11_attention_weight_min": -29.663938522338867,
      "activations/layer12_attention_weight_max": 37.625301361083984,
      "activations/layer12_attention_weight_min": -31.210163116455078,
      "activations/layer13_attention_weight_max": 61.31583786010742,
      "activations/layer13_attention_weight_min": -39.06075668334961,
      "activations/layer14_attention_weight_max": 69.39779663085938,
      "activations/layer14_attention_weight_min": -50.90224075317383,
      "activations/layer15_attention_weight_max": 49.0234375,
      "activations/layer15_attention_weight_min": -30.920818328857422,
      "activations/layer16_attention_weight_max": 38.04555130004883,
      "activations/layer16_attention_weight_min": -27.768367767333984,
      "activations/layer17_attention_weight_max": 41.87832260131836,
      "activations/layer17_attention_weight_min": -24.381492614746094,
      "activations/layer18_attention_weight_max": 36.99028778076172,
      "activations/layer18_attention_weight_min": -19.981416702270508,
      "activations/layer19_attention_weight_max": 44.08002853393555,
      "activations/layer19_attention_weight_min": -29.483434677124023,
      "activations/layer1_attention_weight_max": 17.609045028686523,
      "activations/layer1_attention_weight_min": -15.745841026306152,
      "activations/layer20_attention_weight_max": 38.80096435546875,
      "activations/layer20_attention_weight_min": -24.096086502075195,
      "activations/layer21_attention_weight_max": 38.073699951171875,
      "activations/layer21_attention_weight_min": -22.05669403076172,
      "activations/layer22_attention_weight_max": 55.19138717651367,
      "activations/layer22_attention_weight_min": -28.163713455200195,
      "activations/layer23_attention_weight_max": 38.47997283935547,
      "activations/layer23_attention_weight_min": -20.835485458374023,
      "activations/layer2_attention_weight_max": 33.7400016784668,
      "activations/layer2_attention_weight_min": -33.77383804321289,
      "activations/layer3_attention_weight_max": 91.69639587402344,
      "activations/layer3_attention_weight_min": -95.37537384033203,
      "activations/layer4_attention_weight_max": 137.86839294433594,
      "activations/layer4_attention_weight_min": -131.25045776367188,
      "activations/layer5_attention_weight_max": 84.33379364013672,
      "activations/layer5_attention_weight_min": -75.76924896240234,
      "activations/layer6_attention_weight_max": 60.13111114501953,
      "activations/layer6_attention_weight_min": -54.71803665161133,
      "activations/layer7_attention_weight_max": 91.99699401855469,
      "activations/layer7_attention_weight_min": -86.3973159790039,
      "activations/layer8_attention_weight_max": 42.570335388183594,
      "activations/layer8_attention_weight_min": -40.196998596191406,
      "activations/layer9_attention_weight_max": 37.2763786315918,
      "activations/layer9_attention_weight_min": -41.165672302246094,
      "epoch": 5.35,
      "learning_rate": 0.00011666212121212121,
      "loss": 2.844,
      "step": 92100
    },
    {
      "activations/layer0_attention_weight_max": 16.192224502563477,
      "activations/layer0_attention_weight_min": -10.642207145690918,
      "activations/layer10_attention_weight_max": 40.759193420410156,
      "activations/layer10_attention_weight_min": -32.68067932128906,
      "activations/layer11_attention_weight_max": 37.75741958618164,
      "activations/layer11_attention_weight_min": -29.856998443603516,
      "activations/layer12_attention_weight_max": 25.518898010253906,
      "activations/layer12_attention_weight_min": -27.685901641845703,
      "activations/layer13_attention_weight_max": 42.56311798095703,
      "activations/layer13_attention_weight_min": -33.126434326171875,
      "activations/layer14_attention_weight_max": 60.496795654296875,
      "activations/layer14_attention_weight_min": -49.294654846191406,
      "activations/layer15_attention_weight_max": 40.341060638427734,
      "activations/layer15_attention_weight_min": -28.00285530090332,
      "activations/layer16_attention_weight_max": 40.74951934814453,
      "activations/layer16_attention_weight_min": -26.958580017089844,
      "activations/layer17_attention_weight_max": 39.142757415771484,
      "activations/layer17_attention_weight_min": -24.90852165222168,
      "activations/layer18_attention_weight_max": 26.78156852722168,
      "activations/layer18_attention_weight_min": -19.1079044342041,
      "activations/layer19_attention_weight_max": 43.46499252319336,
      "activations/layer19_attention_weight_min": -29.71823501586914,
      "activations/layer1_attention_weight_max": 18.60931968688965,
      "activations/layer1_attention_weight_min": -15.57357406616211,
      "activations/layer20_attention_weight_max": 29.90484046936035,
      "activations/layer20_attention_weight_min": -24.383899688720703,
      "activations/layer21_attention_weight_max": 26.55942153930664,
      "activations/layer21_attention_weight_min": -21.59710121154785,
      "activations/layer22_attention_weight_max": 48.29356384277344,
      "activations/layer22_attention_weight_min": -26.79361915588379,
      "activations/layer23_attention_weight_max": 34.803443908691406,
      "activations/layer23_attention_weight_min": -19.51598358154297,
      "activations/layer2_attention_weight_max": 34.473018646240234,
      "activations/layer2_attention_weight_min": -35.329429626464844,
      "activations/layer3_attention_weight_max": 95.35885620117188,
      "activations/layer3_attention_weight_min": -101.50454711914062,
      "activations/layer4_attention_weight_max": 134.67066955566406,
      "activations/layer4_attention_weight_min": -142.09425354003906,
      "activations/layer5_attention_weight_max": 81.12018585205078,
      "activations/layer5_attention_weight_min": -82.15375518798828,
      "activations/layer6_attention_weight_max": 55.373477935791016,
      "activations/layer6_attention_weight_min": -61.40633010864258,
      "activations/layer7_attention_weight_max": 91.57962799072266,
      "activations/layer7_attention_weight_min": -91.1993637084961,
      "activations/layer8_attention_weight_max": 46.085872650146484,
      "activations/layer8_attention_weight_min": -42.31013107299805,
      "activations/layer9_attention_weight_max": 43.563167572021484,
      "activations/layer9_attention_weight_min": -37.220638275146484,
      "epoch": 5.35,
      "learning_rate": 0.00011664318181818181,
      "loss": 2.8433,
      "step": 92150
    },
    {
      "activations/layer0_attention_weight_max": 15.556145668029785,
      "activations/layer0_attention_weight_min": -12.409941673278809,
      "activations/layer10_attention_weight_max": 35.57768630981445,
      "activations/layer10_attention_weight_min": -30.261375427246094,
      "activations/layer11_attention_weight_max": 32.886837005615234,
      "activations/layer11_attention_weight_min": -27.310012817382812,
      "activations/layer12_attention_weight_max": 26.22875213623047,
      "activations/layer12_attention_weight_min": -24.50585174560547,
      "activations/layer13_attention_weight_max": 44.15180969238281,
      "activations/layer13_attention_weight_min": -32.17961120605469,
      "activations/layer14_attention_weight_max": 64.80147552490234,
      "activations/layer14_attention_weight_min": -40.98771286010742,
      "activations/layer15_attention_weight_max": 40.21297836303711,
      "activations/layer15_attention_weight_min": -27.11643409729004,
      "activations/layer16_attention_weight_max": 37.71283721923828,
      "activations/layer16_attention_weight_min": -26.273271560668945,
      "activations/layer17_attention_weight_max": 37.90345764160156,
      "activations/layer17_attention_weight_min": -24.409099578857422,
      "activations/layer18_attention_weight_max": 32.3036003112793,
      "activations/layer18_attention_weight_min": -18.219926834106445,
      "activations/layer19_attention_weight_max": 40.65530014038086,
      "activations/layer19_attention_weight_min": -28.549970626831055,
      "activations/layer1_attention_weight_max": 17.986757278442383,
      "activations/layer1_attention_weight_min": -15.198317527770996,
      "activations/layer20_attention_weight_max": 31.77810287475586,
      "activations/layer20_attention_weight_min": -22.765647888183594,
      "activations/layer21_attention_weight_max": 29.163686752319336,
      "activations/layer21_attention_weight_min": -20.26214027404785,
      "activations/layer22_attention_weight_max": 49.12785720825195,
      "activations/layer22_attention_weight_min": -26.20209503173828,
      "activations/layer23_attention_weight_max": 33.57312774658203,
      "activations/layer23_attention_weight_min": -19.39175796508789,
      "activations/layer2_attention_weight_max": 32.366050720214844,
      "activations/layer2_attention_weight_min": -31.8518123626709,
      "activations/layer3_attention_weight_max": 91.76089477539062,
      "activations/layer3_attention_weight_min": -91.506591796875,
      "activations/layer4_attention_weight_max": 129.81626892089844,
      "activations/layer4_attention_weight_min": -125.34759521484375,
      "activations/layer5_attention_weight_max": 76.73808288574219,
      "activations/layer5_attention_weight_min": -75.13826751708984,
      "activations/layer6_attention_weight_max": 60.51799011230469,
      "activations/layer6_attention_weight_min": -53.672855377197266,
      "activations/layer7_attention_weight_max": 84.51811218261719,
      "activations/layer7_attention_weight_min": -83.93525695800781,
      "activations/layer8_attention_weight_max": 42.09980392456055,
      "activations/layer8_attention_weight_min": -39.21233367919922,
      "activations/layer9_attention_weight_max": 38.926692962646484,
      "activations/layer9_attention_weight_min": -33.74522018432617,
      "epoch": 5.36,
      "learning_rate": 0.00011662424242424241,
      "loss": 2.8402,
      "step": 92200
    },
    {
      "activations/layer0_attention_weight_max": 16.672260284423828,
      "activations/layer0_attention_weight_min": -10.799747467041016,
      "activations/layer10_attention_weight_max": 34.01563262939453,
      "activations/layer10_attention_weight_min": -31.983993530273438,
      "activations/layer11_attention_weight_max": 35.03758239746094,
      "activations/layer11_attention_weight_min": -28.105695724487305,
      "activations/layer12_attention_weight_max": 30.734142303466797,
      "activations/layer12_attention_weight_min": -25.85038948059082,
      "activations/layer13_attention_weight_max": 41.77375793457031,
      "activations/layer13_attention_weight_min": -31.201557159423828,
      "activations/layer14_attention_weight_max": 61.133522033691406,
      "activations/layer14_attention_weight_min": -43.25847625732422,
      "activations/layer15_attention_weight_max": 36.265567779541016,
      "activations/layer15_attention_weight_min": -29.324626922607422,
      "activations/layer16_attention_weight_max": 34.642086029052734,
      "activations/layer16_attention_weight_min": -30.729036331176758,
      "activations/layer17_attention_weight_max": 34.23801803588867,
      "activations/layer17_attention_weight_min": -24.248281478881836,
      "activations/layer18_attention_weight_max": 33.40998458862305,
      "activations/layer18_attention_weight_min": -20.931819915771484,
      "activations/layer19_attention_weight_max": 40.90019226074219,
      "activations/layer19_attention_weight_min": -33.34797286987305,
      "activations/layer1_attention_weight_max": 16.405044555664062,
      "activations/layer1_attention_weight_min": -16.015825271606445,
      "activations/layer20_attention_weight_max": 29.634126663208008,
      "activations/layer20_attention_weight_min": -23.38572883605957,
      "activations/layer21_attention_weight_max": 28.36720848083496,
      "activations/layer21_attention_weight_min": -22.50954246520996,
      "activations/layer22_attention_weight_max": 48.13041305541992,
      "activations/layer22_attention_weight_min": -29.12554931640625,
      "activations/layer23_attention_weight_max": 36.642234802246094,
      "activations/layer23_attention_weight_min": -21.864120483398438,
      "activations/layer2_attention_weight_max": 33.193687438964844,
      "activations/layer2_attention_weight_min": -31.425689697265625,
      "activations/layer3_attention_weight_max": 94.85105895996094,
      "activations/layer3_attention_weight_min": -92.41290283203125,
      "activations/layer4_attention_weight_max": 136.2347412109375,
      "activations/layer4_attention_weight_min": -132.73606872558594,
      "activations/layer5_attention_weight_max": 79.52010345458984,
      "activations/layer5_attention_weight_min": -78.11087036132812,
      "activations/layer6_attention_weight_max": 54.86161422729492,
      "activations/layer6_attention_weight_min": -56.19715881347656,
      "activations/layer7_attention_weight_max": 87.0547866821289,
      "activations/layer7_attention_weight_min": -90.4564437866211,
      "activations/layer8_attention_weight_max": 39.13726043701172,
      "activations/layer8_attention_weight_min": -40.05322265625,
      "activations/layer9_attention_weight_max": 34.953407287597656,
      "activations/layer9_attention_weight_min": -33.38615036010742,
      "epoch": 5.36,
      "learning_rate": 0.00011660530303030303,
      "loss": 2.8497,
      "step": 92250
    },
    {
      "activations/layer0_attention_weight_max": 16.3017578125,
      "activations/layer0_attention_weight_min": -12.244388580322266,
      "activations/layer10_attention_weight_max": 32.228614807128906,
      "activations/layer10_attention_weight_min": -31.14129638671875,
      "activations/layer11_attention_weight_max": 30.336246490478516,
      "activations/layer11_attention_weight_min": -28.468875885009766,
      "activations/layer12_attention_weight_max": 40.810272216796875,
      "activations/layer12_attention_weight_min": -29.240232467651367,
      "activations/layer13_attention_weight_max": 50.87687301635742,
      "activations/layer13_attention_weight_min": -32.34986114501953,
      "activations/layer14_attention_weight_max": 65.22309112548828,
      "activations/layer14_attention_weight_min": -46.50764846801758,
      "activations/layer15_attention_weight_max": 40.25043487548828,
      "activations/layer15_attention_weight_min": -28.739301681518555,
      "activations/layer16_attention_weight_max": 39.359493255615234,
      "activations/layer16_attention_weight_min": -27.468379974365234,
      "activations/layer17_attention_weight_max": 39.236976623535156,
      "activations/layer17_attention_weight_min": -24.103214263916016,
      "activations/layer18_attention_weight_max": 30.497360229492188,
      "activations/layer18_attention_weight_min": -21.388568878173828,
      "activations/layer19_attention_weight_max": 39.77520751953125,
      "activations/layer19_attention_weight_min": -28.078235626220703,
      "activations/layer1_attention_weight_max": 17.431068420410156,
      "activations/layer1_attention_weight_min": -13.95096492767334,
      "activations/layer20_attention_weight_max": 35.71999740600586,
      "activations/layer20_attention_weight_min": -23.907381057739258,
      "activations/layer21_attention_weight_max": 35.60465621948242,
      "activations/layer21_attention_weight_min": -20.541818618774414,
      "activations/layer22_attention_weight_max": 52.30838394165039,
      "activations/layer22_attention_weight_min": -28.090085983276367,
      "activations/layer23_attention_weight_max": 35.20759582519531,
      "activations/layer23_attention_weight_min": -21.45905876159668,
      "activations/layer2_attention_weight_max": 32.162742614746094,
      "activations/layer2_attention_weight_min": -31.234046936035156,
      "activations/layer3_attention_weight_max": 90.2473373413086,
      "activations/layer3_attention_weight_min": -93.9634017944336,
      "activations/layer4_attention_weight_max": 136.5216522216797,
      "activations/layer4_attention_weight_min": -128.7742919921875,
      "activations/layer5_attention_weight_max": 76.387939453125,
      "activations/layer5_attention_weight_min": -73.76213073730469,
      "activations/layer6_attention_weight_max": 59.87516784667969,
      "activations/layer6_attention_weight_min": -55.081077575683594,
      "activations/layer7_attention_weight_max": 86.3328857421875,
      "activations/layer7_attention_weight_min": -82.70175170898438,
      "activations/layer8_attention_weight_max": 39.71240997314453,
      "activations/layer8_attention_weight_min": -39.36476516723633,
      "activations/layer9_attention_weight_max": 35.584320068359375,
      "activations/layer9_attention_weight_min": -34.78799819946289,
      "epoch": 5.36,
      "learning_rate": 0.00011658636363636363,
      "loss": 2.8428,
      "step": 92300
    },
    {
      "activations/layer0_attention_weight_max": 17.541034698486328,
      "activations/layer0_attention_weight_min": -13.520512580871582,
      "activations/layer10_attention_weight_max": 33.968955993652344,
      "activations/layer10_attention_weight_min": -30.470754623413086,
      "activations/layer11_attention_weight_max": 32.750770568847656,
      "activations/layer11_attention_weight_min": -27.616804122924805,
      "activations/layer12_attention_weight_max": 26.97638511657715,
      "activations/layer12_attention_weight_min": -24.699951171875,
      "activations/layer13_attention_weight_max": 48.14281463623047,
      "activations/layer13_attention_weight_min": -31.9322452545166,
      "activations/layer14_attention_weight_max": 57.20036697387695,
      "activations/layer14_attention_weight_min": -37.983333587646484,
      "activations/layer15_attention_weight_max": 42.962440490722656,
      "activations/layer15_attention_weight_min": -29.08192253112793,
      "activations/layer16_attention_weight_max": 42.478187561035156,
      "activations/layer16_attention_weight_min": -27.55542755126953,
      "activations/layer17_attention_weight_max": 43.082645416259766,
      "activations/layer17_attention_weight_min": -25.20522689819336,
      "activations/layer18_attention_weight_max": 32.11730194091797,
      "activations/layer18_attention_weight_min": -19.492448806762695,
      "activations/layer19_attention_weight_max": 41.87075424194336,
      "activations/layer19_attention_weight_min": -29.237558364868164,
      "activations/layer1_attention_weight_max": 17.643665313720703,
      "activations/layer1_attention_weight_min": -14.927067756652832,
      "activations/layer20_attention_weight_max": 37.089054107666016,
      "activations/layer20_attention_weight_min": -23.986003875732422,
      "activations/layer21_attention_weight_max": 36.3168830871582,
      "activations/layer21_attention_weight_min": -21.826278686523438,
      "activations/layer22_attention_weight_max": 51.45330047607422,
      "activations/layer22_attention_weight_min": -28.14740753173828,
      "activations/layer23_attention_weight_max": 34.315120697021484,
      "activations/layer23_attention_weight_min": -21.374324798583984,
      "activations/layer2_attention_weight_max": 32.562278747558594,
      "activations/layer2_attention_weight_min": -31.64761734008789,
      "activations/layer3_attention_weight_max": 93.64207458496094,
      "activations/layer3_attention_weight_min": -94.55441284179688,
      "activations/layer4_attention_weight_max": 134.99444580078125,
      "activations/layer4_attention_weight_min": -124.75218963623047,
      "activations/layer5_attention_weight_max": 77.48739624023438,
      "activations/layer5_attention_weight_min": -75.17938232421875,
      "activations/layer6_attention_weight_max": 54.72414016723633,
      "activations/layer6_attention_weight_min": -55.1640510559082,
      "activations/layer7_attention_weight_max": 81.26516723632812,
      "activations/layer7_attention_weight_min": -84.4092025756836,
      "activations/layer8_attention_weight_max": 38.8127326965332,
      "activations/layer8_attention_weight_min": -41.22479248046875,
      "activations/layer9_attention_weight_max": 37.2907600402832,
      "activations/layer9_attention_weight_min": -35.16763687133789,
      "epoch": 5.37,
      "learning_rate": 0.00011656742424242423,
      "loss": 2.8287,
      "step": 92350
    },
    {
      "activations/layer0_attention_weight_max": 15.57691764831543,
      "activations/layer0_attention_weight_min": -11.884703636169434,
      "activations/layer10_attention_weight_max": 36.042598724365234,
      "activations/layer10_attention_weight_min": -32.48811340332031,
      "activations/layer11_attention_weight_max": 41.564208984375,
      "activations/layer11_attention_weight_min": -28.66274070739746,
      "activations/layer12_attention_weight_max": 47.6806640625,
      "activations/layer12_attention_weight_min": -26.986892700195312,
      "activations/layer13_attention_weight_max": 69.40032196044922,
      "activations/layer13_attention_weight_min": -37.87162780761719,
      "activations/layer14_attention_weight_max": 65.48954772949219,
      "activations/layer14_attention_weight_min": -46.96627426147461,
      "activations/layer15_attention_weight_max": 67.21609497070312,
      "activations/layer15_attention_weight_min": -35.327903747558594,
      "activations/layer16_attention_weight_max": 38.5296630859375,
      "activations/layer16_attention_weight_min": -25.662817001342773,
      "activations/layer17_attention_weight_max": 41.20314025878906,
      "activations/layer17_attention_weight_min": -23.618574142456055,
      "activations/layer18_attention_weight_max": 35.87327575683594,
      "activations/layer18_attention_weight_min": -21.274028778076172,
      "activations/layer19_attention_weight_max": 44.02656173706055,
      "activations/layer19_attention_weight_min": -29.310789108276367,
      "activations/layer1_attention_weight_max": 16.75314712524414,
      "activations/layer1_attention_weight_min": -14.75423526763916,
      "activations/layer20_attention_weight_max": 33.85087966918945,
      "activations/layer20_attention_weight_min": -22.86971092224121,
      "activations/layer21_attention_weight_max": 37.804264068603516,
      "activations/layer21_attention_weight_min": -22.413814544677734,
      "activations/layer22_attention_weight_max": 52.96419143676758,
      "activations/layer22_attention_weight_min": -28.31852149963379,
      "activations/layer23_attention_weight_max": 36.263160705566406,
      "activations/layer23_attention_weight_min": -23.15980339050293,
      "activations/layer2_attention_weight_max": 32.90174102783203,
      "activations/layer2_attention_weight_min": -32.07041549682617,
      "activations/layer3_attention_weight_max": 91.89849090576172,
      "activations/layer3_attention_weight_min": -95.3597640991211,
      "activations/layer4_attention_weight_max": 132.26406860351562,
      "activations/layer4_attention_weight_min": -130.8028564453125,
      "activations/layer5_attention_weight_max": 81.9066390991211,
      "activations/layer5_attention_weight_min": -78.38021087646484,
      "activations/layer6_attention_weight_max": 57.48634338378906,
      "activations/layer6_attention_weight_min": -58.52392578125,
      "activations/layer7_attention_weight_max": 91.329833984375,
      "activations/layer7_attention_weight_min": -85.49240112304688,
      "activations/layer8_attention_weight_max": 44.543357849121094,
      "activations/layer8_attention_weight_min": -42.16023254394531,
      "activations/layer9_attention_weight_max": 41.539710998535156,
      "activations/layer9_attention_weight_min": -37.27291488647461,
      "epoch": 5.37,
      "learning_rate": 0.00011654848484848484,
      "loss": 2.8353,
      "step": 92400
    },
    {
      "activations/layer0_attention_weight_max": 16.176624298095703,
      "activations/layer0_attention_weight_min": -12.291207313537598,
      "activations/layer10_attention_weight_max": 31.962156295776367,
      "activations/layer10_attention_weight_min": -28.78688621520996,
      "activations/layer11_attention_weight_max": 34.06501007080078,
      "activations/layer11_attention_weight_min": -26.534610748291016,
      "activations/layer12_attention_weight_max": 26.589494705200195,
      "activations/layer12_attention_weight_min": -26.284915924072266,
      "activations/layer13_attention_weight_max": 40.96052551269531,
      "activations/layer13_attention_weight_min": -31.74491310119629,
      "activations/layer14_attention_weight_max": 52.033573150634766,
      "activations/layer14_attention_weight_min": -39.05216598510742,
      "activations/layer15_attention_weight_max": 39.3784294128418,
      "activations/layer15_attention_weight_min": -29.877012252807617,
      "activations/layer16_attention_weight_max": 36.8107795715332,
      "activations/layer16_attention_weight_min": -28.752050399780273,
      "activations/layer17_attention_weight_max": 35.39715576171875,
      "activations/layer17_attention_weight_min": -25.185344696044922,
      "activations/layer18_attention_weight_max": 31.322641372680664,
      "activations/layer18_attention_weight_min": -19.374835968017578,
      "activations/layer19_attention_weight_max": 36.66048049926758,
      "activations/layer19_attention_weight_min": -29.263639450073242,
      "activations/layer1_attention_weight_max": 17.362789154052734,
      "activations/layer1_attention_weight_min": -13.669519424438477,
      "activations/layer20_attention_weight_max": 30.574600219726562,
      "activations/layer20_attention_weight_min": -22.827281951904297,
      "activations/layer21_attention_weight_max": 31.912471771240234,
      "activations/layer21_attention_weight_min": -20.13770866394043,
      "activations/layer22_attention_weight_max": 46.42826461791992,
      "activations/layer22_attention_weight_min": -29.70573616027832,
      "activations/layer23_attention_weight_max": 33.20185089111328,
      "activations/layer23_attention_weight_min": -22.128690719604492,
      "activations/layer2_attention_weight_max": 32.81520080566406,
      "activations/layer2_attention_weight_min": -32.6241340637207,
      "activations/layer3_attention_weight_max": 88.66602325439453,
      "activations/layer3_attention_weight_min": -93.97955322265625,
      "activations/layer4_attention_weight_max": 129.49600219726562,
      "activations/layer4_attention_weight_min": -134.0380096435547,
      "activations/layer5_attention_weight_max": 76.97407531738281,
      "activations/layer5_attention_weight_min": -75.17643737792969,
      "activations/layer6_attention_weight_max": 52.65925979614258,
      "activations/layer6_attention_weight_min": -57.169803619384766,
      "activations/layer7_attention_weight_max": 85.74149322509766,
      "activations/layer7_attention_weight_min": -84.93603515625,
      "activations/layer8_attention_weight_max": 37.56196975708008,
      "activations/layer8_attention_weight_min": -40.02878189086914,
      "activations/layer9_attention_weight_max": 34.51151657104492,
      "activations/layer9_attention_weight_min": -38.87710189819336,
      "epoch": 5.37,
      "learning_rate": 0.00011652954545454543,
      "loss": 2.8426,
      "step": 92450
    },
    {
      "activations/layer0_attention_weight_max": 16.20455551147461,
      "activations/layer0_attention_weight_min": -11.702033996582031,
      "activations/layer10_attention_weight_max": 41.183677673339844,
      "activations/layer10_attention_weight_min": -32.49024200439453,
      "activations/layer11_attention_weight_max": 42.16110610961914,
      "activations/layer11_attention_weight_min": -30.64388084411621,
      "activations/layer12_attention_weight_max": 41.44623565673828,
      "activations/layer12_attention_weight_min": -27.47355842590332,
      "activations/layer13_attention_weight_max": 57.34743118286133,
      "activations/layer13_attention_weight_min": -38.85105895996094,
      "activations/layer14_attention_weight_max": 69.91001892089844,
      "activations/layer14_attention_weight_min": -44.338321685791016,
      "activations/layer15_attention_weight_max": 44.10735321044922,
      "activations/layer15_attention_weight_min": -27.517566680908203,
      "activations/layer16_attention_weight_max": 38.32524490356445,
      "activations/layer16_attention_weight_min": -26.67241668701172,
      "activations/layer17_attention_weight_max": 39.371299743652344,
      "activations/layer17_attention_weight_min": -23.9649658203125,
      "activations/layer18_attention_weight_max": 30.040264129638672,
      "activations/layer18_attention_weight_min": -20.160736083984375,
      "activations/layer19_attention_weight_max": 41.61683654785156,
      "activations/layer19_attention_weight_min": -31.483667373657227,
      "activations/layer1_attention_weight_max": 17.40093231201172,
      "activations/layer1_attention_weight_min": -16.224145889282227,
      "activations/layer20_attention_weight_max": 38.06592559814453,
      "activations/layer20_attention_weight_min": -23.344175338745117,
      "activations/layer21_attention_weight_max": 30.944372177124023,
      "activations/layer21_attention_weight_min": -21.72859764099121,
      "activations/layer22_attention_weight_max": 54.43899154663086,
      "activations/layer22_attention_weight_min": -27.54662322998047,
      "activations/layer23_attention_weight_max": 38.542816162109375,
      "activations/layer23_attention_weight_min": -21.618064880371094,
      "activations/layer2_attention_weight_max": 34.77756118774414,
      "activations/layer2_attention_weight_min": -32.42414093017578,
      "activations/layer3_attention_weight_max": 92.54086303710938,
      "activations/layer3_attention_weight_min": -92.16156768798828,
      "activations/layer4_attention_weight_max": 133.07012939453125,
      "activations/layer4_attention_weight_min": -128.09317016601562,
      "activations/layer5_attention_weight_max": 79.42071533203125,
      "activations/layer5_attention_weight_min": -75.3006820678711,
      "activations/layer6_attention_weight_max": 55.169517517089844,
      "activations/layer6_attention_weight_min": -54.7824592590332,
      "activations/layer7_attention_weight_max": 94.77649688720703,
      "activations/layer7_attention_weight_min": -84.4827651977539,
      "activations/layer8_attention_weight_max": 45.06216812133789,
      "activations/layer8_attention_weight_min": -38.63797378540039,
      "activations/layer9_attention_weight_max": 38.955570220947266,
      "activations/layer9_attention_weight_min": -42.713279724121094,
      "epoch": 5.37,
      "learning_rate": 0.00011651060606060605,
      "loss": 2.86,
      "step": 92500
    },
    {
      "activations/layer0_attention_weight_max": 16.43900489807129,
      "activations/layer0_attention_weight_min": -11.9129638671875,
      "activations/layer10_attention_weight_max": 34.44559860229492,
      "activations/layer10_attention_weight_min": -31.36820411682129,
      "activations/layer11_attention_weight_max": 32.45381546020508,
      "activations/layer11_attention_weight_min": -28.882831573486328,
      "activations/layer12_attention_weight_max": 35.201133728027344,
      "activations/layer12_attention_weight_min": -26.179765701293945,
      "activations/layer13_attention_weight_max": 45.36458969116211,
      "activations/layer13_attention_weight_min": -31.365535736083984,
      "activations/layer14_attention_weight_max": 58.53706741333008,
      "activations/layer14_attention_weight_min": -41.989097595214844,
      "activations/layer15_attention_weight_max": 42.42654037475586,
      "activations/layer15_attention_weight_min": -32.144100189208984,
      "activations/layer16_attention_weight_max": 37.80118179321289,
      "activations/layer16_attention_weight_min": -27.301902770996094,
      "activations/layer17_attention_weight_max": 40.85261917114258,
      "activations/layer17_attention_weight_min": -25.892459869384766,
      "activations/layer18_attention_weight_max": 31.73307991027832,
      "activations/layer18_attention_weight_min": -20.172075271606445,
      "activations/layer19_attention_weight_max": 45.07484436035156,
      "activations/layer19_attention_weight_min": -31.83730125427246,
      "activations/layer1_attention_weight_max": 16.56713104248047,
      "activations/layer1_attention_weight_min": -14.905306816101074,
      "activations/layer20_attention_weight_max": 34.75075149536133,
      "activations/layer20_attention_weight_min": -24.456546783447266,
      "activations/layer21_attention_weight_max": 36.36732864379883,
      "activations/layer21_attention_weight_min": -21.103111267089844,
      "activations/layer22_attention_weight_max": 62.036739349365234,
      "activations/layer22_attention_weight_min": -27.131874084472656,
      "activations/layer23_attention_weight_max": 39.196800231933594,
      "activations/layer23_attention_weight_min": -23.613006591796875,
      "activations/layer2_attention_weight_max": 31.971416473388672,
      "activations/layer2_attention_weight_min": -32.785743713378906,
      "activations/layer3_attention_weight_max": 89.47164154052734,
      "activations/layer3_attention_weight_min": -92.71542358398438,
      "activations/layer4_attention_weight_max": 131.75596618652344,
      "activations/layer4_attention_weight_min": -130.48362731933594,
      "activations/layer5_attention_weight_max": 79.3828125,
      "activations/layer5_attention_weight_min": -80.06212615966797,
      "activations/layer6_attention_weight_max": 54.36555862426758,
      "activations/layer6_attention_weight_min": -57.13151931762695,
      "activations/layer7_attention_weight_max": 87.61321258544922,
      "activations/layer7_attention_weight_min": -88.64369201660156,
      "activations/layer8_attention_weight_max": 39.44986343383789,
      "activations/layer8_attention_weight_min": -43.09271240234375,
      "activations/layer9_attention_weight_max": 38.33530807495117,
      "activations/layer9_attention_weight_min": -34.11702346801758,
      "epoch": 5.38,
      "learning_rate": 0.00011649166666666666,
      "loss": 2.8337,
      "step": 92550
    },
    {
      "activations/layer0_attention_weight_max": 16.0874080657959,
      "activations/layer0_attention_weight_min": -10.910892486572266,
      "activations/layer10_attention_weight_max": 33.33589553833008,
      "activations/layer10_attention_weight_min": -29.19991111755371,
      "activations/layer11_attention_weight_max": 34.04607009887695,
      "activations/layer11_attention_weight_min": -27.316471099853516,
      "activations/layer12_attention_weight_max": 34.96345901489258,
      "activations/layer12_attention_weight_min": -26.168954849243164,
      "activations/layer13_attention_weight_max": 39.71472930908203,
      "activations/layer13_attention_weight_min": -32.6643180847168,
      "activations/layer14_attention_weight_max": 52.88029098510742,
      "activations/layer14_attention_weight_min": -39.12685775756836,
      "activations/layer15_attention_weight_max": 39.85065841674805,
      "activations/layer15_attention_weight_min": -28.520915985107422,
      "activations/layer16_attention_weight_max": 33.8029670715332,
      "activations/layer16_attention_weight_min": -26.324426651000977,
      "activations/layer17_attention_weight_max": 35.30900955200195,
      "activations/layer17_attention_weight_min": -24.331523895263672,
      "activations/layer18_attention_weight_max": 33.62946701049805,
      "activations/layer18_attention_weight_min": -19.071796417236328,
      "activations/layer19_attention_weight_max": 38.67809295654297,
      "activations/layer19_attention_weight_min": -27.572355270385742,
      "activations/layer1_attention_weight_max": 17.526670455932617,
      "activations/layer1_attention_weight_min": -14.158347129821777,
      "activations/layer20_attention_weight_max": 31.215524673461914,
      "activations/layer20_attention_weight_min": -24.62007713317871,
      "activations/layer21_attention_weight_max": 37.024261474609375,
      "activations/layer21_attention_weight_min": -20.787511825561523,
      "activations/layer22_attention_weight_max": 55.029117584228516,
      "activations/layer22_attention_weight_min": -28.17912483215332,
      "activations/layer23_attention_weight_max": 35.79865646362305,
      "activations/layer23_attention_weight_min": -20.03056526184082,
      "activations/layer2_attention_weight_max": 31.118131637573242,
      "activations/layer2_attention_weight_min": -29.63802719116211,
      "activations/layer3_attention_weight_max": 92.18224334716797,
      "activations/layer3_attention_weight_min": -91.88516998291016,
      "activations/layer4_attention_weight_max": 132.3345184326172,
      "activations/layer4_attention_weight_min": -124.84700775146484,
      "activations/layer5_attention_weight_max": 81.09554290771484,
      "activations/layer5_attention_weight_min": -76.7966537475586,
      "activations/layer6_attention_weight_max": 58.19382858276367,
      "activations/layer6_attention_weight_min": -54.80349349975586,
      "activations/layer7_attention_weight_max": 85.01982116699219,
      "activations/layer7_attention_weight_min": -82.79186248779297,
      "activations/layer8_attention_weight_max": 43.4540901184082,
      "activations/layer8_attention_weight_min": -40.601768493652344,
      "activations/layer9_attention_weight_max": 37.69903564453125,
      "activations/layer9_attention_weight_min": -33.82900619506836,
      "epoch": 5.38,
      "learning_rate": 0.00011647272727272726,
      "loss": 2.8549,
      "step": 92600
    },
    {
      "activations/layer0_attention_weight_max": 15.959595680236816,
      "activations/layer0_attention_weight_min": -11.184407234191895,
      "activations/layer10_attention_weight_max": 35.514137268066406,
      "activations/layer10_attention_weight_min": -32.16123580932617,
      "activations/layer11_attention_weight_max": 31.37057113647461,
      "activations/layer11_attention_weight_min": -28.670202255249023,
      "activations/layer12_attention_weight_max": 45.127349853515625,
      "activations/layer12_attention_weight_min": -29.80695915222168,
      "activations/layer13_attention_weight_max": 72.49072265625,
      "activations/layer13_attention_weight_min": -48.692420959472656,
      "activations/layer14_attention_weight_max": 64.80264282226562,
      "activations/layer14_attention_weight_min": -44.92210006713867,
      "activations/layer15_attention_weight_max": 46.382022857666016,
      "activations/layer15_attention_weight_min": -30.39690399169922,
      "activations/layer16_attention_weight_max": 42.035362243652344,
      "activations/layer16_attention_weight_min": -28.797595977783203,
      "activations/layer17_attention_weight_max": 41.24013137817383,
      "activations/layer17_attention_weight_min": -25.160505294799805,
      "activations/layer18_attention_weight_max": 33.81396484375,
      "activations/layer18_attention_weight_min": -20.34479331970215,
      "activations/layer19_attention_weight_max": 51.795448303222656,
      "activations/layer19_attention_weight_min": -30.24273109436035,
      "activations/layer1_attention_weight_max": 17.09287452697754,
      "activations/layer1_attention_weight_min": -15.302417755126953,
      "activations/layer20_attention_weight_max": 33.329586029052734,
      "activations/layer20_attention_weight_min": -22.392498016357422,
      "activations/layer21_attention_weight_max": 33.39868927001953,
      "activations/layer21_attention_weight_min": -22.102224349975586,
      "activations/layer22_attention_weight_max": 53.729095458984375,
      "activations/layer22_attention_weight_min": -29.175289154052734,
      "activations/layer23_attention_weight_max": 40.30791473388672,
      "activations/layer23_attention_weight_min": -21.951034545898438,
      "activations/layer2_attention_weight_max": 32.48970413208008,
      "activations/layer2_attention_weight_min": -30.992706298828125,
      "activations/layer3_attention_weight_max": 89.16265106201172,
      "activations/layer3_attention_weight_min": -94.00951385498047,
      "activations/layer4_attention_weight_max": 131.98336791992188,
      "activations/layer4_attention_weight_min": -133.9700469970703,
      "activations/layer5_attention_weight_max": 82.60206604003906,
      "activations/layer5_attention_weight_min": -75.75581359863281,
      "activations/layer6_attention_weight_max": 55.819679260253906,
      "activations/layer6_attention_weight_min": -58.01138687133789,
      "activations/layer7_attention_weight_max": 90.4529800415039,
      "activations/layer7_attention_weight_min": -87.05364990234375,
      "activations/layer8_attention_weight_max": 42.545166015625,
      "activations/layer8_attention_weight_min": -40.43021011352539,
      "activations/layer9_attention_weight_max": 36.61422348022461,
      "activations/layer9_attention_weight_min": -40.843746185302734,
      "epoch": 5.38,
      "learning_rate": 0.00011645378787878786,
      "loss": 2.8548,
      "step": 92650
    },
    {
      "activations/layer0_attention_weight_max": 16.650636672973633,
      "activations/layer0_attention_weight_min": -12.479609489440918,
      "activations/layer10_attention_weight_max": 34.9892463684082,
      "activations/layer10_attention_weight_min": -31.289989471435547,
      "activations/layer11_attention_weight_max": 33.29188919067383,
      "activations/layer11_attention_weight_min": -27.894092559814453,
      "activations/layer12_attention_weight_max": 28.969886779785156,
      "activations/layer12_attention_weight_min": -30.016599655151367,
      "activations/layer13_attention_weight_max": 43.49690628051758,
      "activations/layer13_attention_weight_min": -33.577056884765625,
      "activations/layer14_attention_weight_max": 74.60259246826172,
      "activations/layer14_attention_weight_min": -53.580238342285156,
      "activations/layer15_attention_weight_max": 38.052913665771484,
      "activations/layer15_attention_weight_min": -27.264278411865234,
      "activations/layer16_attention_weight_max": 34.3403205871582,
      "activations/layer16_attention_weight_min": -27.132352828979492,
      "activations/layer17_attention_weight_max": 36.151580810546875,
      "activations/layer17_attention_weight_min": -24.26359748840332,
      "activations/layer18_attention_weight_max": 30.35016441345215,
      "activations/layer18_attention_weight_min": -20.051109313964844,
      "activations/layer19_attention_weight_max": 36.653724670410156,
      "activations/layer19_attention_weight_min": -28.53427505493164,
      "activations/layer1_attention_weight_max": 17.645246505737305,
      "activations/layer1_attention_weight_min": -13.50584602355957,
      "activations/layer20_attention_weight_max": 32.95848846435547,
      "activations/layer20_attention_weight_min": -23.988508224487305,
      "activations/layer21_attention_weight_max": 33.019317626953125,
      "activations/layer21_attention_weight_min": -20.040494918823242,
      "activations/layer22_attention_weight_max": 48.57708740234375,
      "activations/layer22_attention_weight_min": -28.31605339050293,
      "activations/layer23_attention_weight_max": 32.775081634521484,
      "activations/layer23_attention_weight_min": -20.22657585144043,
      "activations/layer2_attention_weight_max": 33.77963638305664,
      "activations/layer2_attention_weight_min": -32.92781448364258,
      "activations/layer3_attention_weight_max": 91.3519515991211,
      "activations/layer3_attention_weight_min": -93.61900329589844,
      "activations/layer4_attention_weight_max": 133.96885681152344,
      "activations/layer4_attention_weight_min": -125.247314453125,
      "activations/layer5_attention_weight_max": 82.09226989746094,
      "activations/layer5_attention_weight_min": -78.0450668334961,
      "activations/layer6_attention_weight_max": 57.32346725463867,
      "activations/layer6_attention_weight_min": -56.040489196777344,
      "activations/layer7_attention_weight_max": 90.33682250976562,
      "activations/layer7_attention_weight_min": -87.8816909790039,
      "activations/layer8_attention_weight_max": 43.26773452758789,
      "activations/layer8_attention_weight_min": -42.08439254760742,
      "activations/layer9_attention_weight_max": 38.5764274597168,
      "activations/layer9_attention_weight_min": -34.132205963134766,
      "epoch": 5.39,
      "learning_rate": 0.00011643484848484848,
      "loss": 2.8362,
      "step": 92700
    },
    {
      "activations/layer0_attention_weight_max": 17.55866241455078,
      "activations/layer0_attention_weight_min": -11.221427917480469,
      "activations/layer10_attention_weight_max": 34.97547912597656,
      "activations/layer10_attention_weight_min": -31.951974868774414,
      "activations/layer11_attention_weight_max": 34.33241653442383,
      "activations/layer11_attention_weight_min": -28.963699340820312,
      "activations/layer12_attention_weight_max": 39.04240036010742,
      "activations/layer12_attention_weight_min": -25.614601135253906,
      "activations/layer13_attention_weight_max": 52.465675354003906,
      "activations/layer13_attention_weight_min": -36.06513595581055,
      "activations/layer14_attention_weight_max": 66.10466766357422,
      "activations/layer14_attention_weight_min": -48.474639892578125,
      "activations/layer15_attention_weight_max": 52.8804931640625,
      "activations/layer15_attention_weight_min": -31.146528244018555,
      "activations/layer16_attention_weight_max": 38.5202751159668,
      "activations/layer16_attention_weight_min": -29.523067474365234,
      "activations/layer17_attention_weight_max": 41.90644836425781,
      "activations/layer17_attention_weight_min": -26.765777587890625,
      "activations/layer18_attention_weight_max": 30.197471618652344,
      "activations/layer18_attention_weight_min": -20.044532775878906,
      "activations/layer19_attention_weight_max": 39.672393798828125,
      "activations/layer19_attention_weight_min": -32.1026725769043,
      "activations/layer1_attention_weight_max": 17.418006896972656,
      "activations/layer1_attention_weight_min": -15.417610168457031,
      "activations/layer20_attention_weight_max": 35.70619201660156,
      "activations/layer20_attention_weight_min": -25.566816329956055,
      "activations/layer21_attention_weight_max": 34.37554168701172,
      "activations/layer21_attention_weight_min": -23.321680068969727,
      "activations/layer22_attention_weight_max": 50.34404754638672,
      "activations/layer22_attention_weight_min": -28.813114166259766,
      "activations/layer23_attention_weight_max": 35.13322448730469,
      "activations/layer23_attention_weight_min": -22.77166748046875,
      "activations/layer2_attention_weight_max": 35.87290954589844,
      "activations/layer2_attention_weight_min": -35.08935546875,
      "activations/layer3_attention_weight_max": 98.77989196777344,
      "activations/layer3_attention_weight_min": -101.88829803466797,
      "activations/layer4_attention_weight_max": 138.59518432617188,
      "activations/layer4_attention_weight_min": -140.3114776611328,
      "activations/layer5_attention_weight_max": 85.33663940429688,
      "activations/layer5_attention_weight_min": -81.03532409667969,
      "activations/layer6_attention_weight_max": 61.609161376953125,
      "activations/layer6_attention_weight_min": -56.273719787597656,
      "activations/layer7_attention_weight_max": 92.66376495361328,
      "activations/layer7_attention_weight_min": -88.57865905761719,
      "activations/layer8_attention_weight_max": 41.5428352355957,
      "activations/layer8_attention_weight_min": -40.57198715209961,
      "activations/layer9_attention_weight_max": 39.15135955810547,
      "activations/layer9_attention_weight_min": -34.579246520996094,
      "epoch": 5.39,
      "learning_rate": 0.00011641590909090908,
      "loss": 2.8449,
      "step": 92750
    },
    {
      "activations/layer0_attention_weight_max": 17.10968017578125,
      "activations/layer0_attention_weight_min": -11.252959251403809,
      "activations/layer10_attention_weight_max": 32.25300979614258,
      "activations/layer10_attention_weight_min": -33.061126708984375,
      "activations/layer11_attention_weight_max": 33.09117889404297,
      "activations/layer11_attention_weight_min": -28.67047882080078,
      "activations/layer12_attention_weight_max": 26.368976593017578,
      "activations/layer12_attention_weight_min": -25.389480590820312,
      "activations/layer13_attention_weight_max": 44.197811126708984,
      "activations/layer13_attention_weight_min": -34.223873138427734,
      "activations/layer14_attention_weight_max": 52.605796813964844,
      "activations/layer14_attention_weight_min": -44.25366973876953,
      "activations/layer15_attention_weight_max": 38.46240234375,
      "activations/layer15_attention_weight_min": -28.777387619018555,
      "activations/layer16_attention_weight_max": 33.21463394165039,
      "activations/layer16_attention_weight_min": -28.598567962646484,
      "activations/layer17_attention_weight_max": 34.38601303100586,
      "activations/layer17_attention_weight_min": -24.262041091918945,
      "activations/layer18_attention_weight_max": 30.878995895385742,
      "activations/layer18_attention_weight_min": -19.85677146911621,
      "activations/layer19_attention_weight_max": 36.20389938354492,
      "activations/layer19_attention_weight_min": -30.6248722076416,
      "activations/layer1_attention_weight_max": 17.9407901763916,
      "activations/layer1_attention_weight_min": -14.682625770568848,
      "activations/layer20_attention_weight_max": 30.133060455322266,
      "activations/layer20_attention_weight_min": -24.750646591186523,
      "activations/layer21_attention_weight_max": 35.987709045410156,
      "activations/layer21_attention_weight_min": -20.26483154296875,
      "activations/layer22_attention_weight_max": 45.560264587402344,
      "activations/layer22_attention_weight_min": -29.41422462463379,
      "activations/layer23_attention_weight_max": 30.879497528076172,
      "activations/layer23_attention_weight_min": -21.239906311035156,
      "activations/layer2_attention_weight_max": 32.34587860107422,
      "activations/layer2_attention_weight_min": -30.46039581298828,
      "activations/layer3_attention_weight_max": 88.12628173828125,
      "activations/layer3_attention_weight_min": -87.75394439697266,
      "activations/layer4_attention_weight_max": 130.83425903320312,
      "activations/layer4_attention_weight_min": -125.84950256347656,
      "activations/layer5_attention_weight_max": 77.15568542480469,
      "activations/layer5_attention_weight_min": -77.81295776367188,
      "activations/layer6_attention_weight_max": 53.053768157958984,
      "activations/layer6_attention_weight_min": -56.12504959106445,
      "activations/layer7_attention_weight_max": 87.55270385742188,
      "activations/layer7_attention_weight_min": -85.2089614868164,
      "activations/layer8_attention_weight_max": 38.04347229003906,
      "activations/layer8_attention_weight_min": -39.83940505981445,
      "activations/layer9_attention_weight_max": 35.14784240722656,
      "activations/layer9_attention_weight_min": -36.257137298583984,
      "epoch": 5.39,
      "learning_rate": 0.00011639696969696968,
      "loss": 2.8449,
      "step": 92800
    },
    {
      "activations/layer0_attention_weight_max": 16.543737411499023,
      "activations/layer0_attention_weight_min": -11.309507369995117,
      "activations/layer10_attention_weight_max": 43.41450119018555,
      "activations/layer10_attention_weight_min": -34.304588317871094,
      "activations/layer11_attention_weight_max": 48.24497985839844,
      "activations/layer11_attention_weight_min": -29.96565818786621,
      "activations/layer12_attention_weight_max": 41.767616271972656,
      "activations/layer12_attention_weight_min": -30.730812072753906,
      "activations/layer13_attention_weight_max": 73.51725769042969,
      "activations/layer13_attention_weight_min": -39.4835090637207,
      "activations/layer14_attention_weight_max": 110.59781646728516,
      "activations/layer14_attention_weight_min": -52.74854278564453,
      "activations/layer15_attention_weight_max": 69.15467071533203,
      "activations/layer15_attention_weight_min": -31.111299514770508,
      "activations/layer16_attention_weight_max": 47.594722747802734,
      "activations/layer16_attention_weight_min": -27.721458435058594,
      "activations/layer17_attention_weight_max": 48.717384338378906,
      "activations/layer17_attention_weight_min": -25.584985733032227,
      "activations/layer18_attention_weight_max": 40.198890686035156,
      "activations/layer18_attention_weight_min": -22.63001823425293,
      "activations/layer19_attention_weight_max": 52.109920501708984,
      "activations/layer19_attention_weight_min": -31.183929443359375,
      "activations/layer1_attention_weight_max": 17.4442138671875,
      "activations/layer1_attention_weight_min": -15.463769912719727,
      "activations/layer20_attention_weight_max": 44.83436584472656,
      "activations/layer20_attention_weight_min": -22.84428596496582,
      "activations/layer21_attention_weight_max": 41.19163513183594,
      "activations/layer21_attention_weight_min": -23.14784049987793,
      "activations/layer22_attention_weight_max": 70.38483428955078,
      "activations/layer22_attention_weight_min": -29.845924377441406,
      "activations/layer23_attention_weight_max": 45.03541564941406,
      "activations/layer23_attention_weight_min": -21.265865325927734,
      "activations/layer2_attention_weight_max": 35.602542877197266,
      "activations/layer2_attention_weight_min": -32.860530853271484,
      "activations/layer3_attention_weight_max": 96.62471008300781,
      "activations/layer3_attention_weight_min": -94.10812377929688,
      "activations/layer4_attention_weight_max": 138.1622772216797,
      "activations/layer4_attention_weight_min": -135.8412628173828,
      "activations/layer5_attention_weight_max": 84.54716491699219,
      "activations/layer5_attention_weight_min": -77.3674087524414,
      "activations/layer6_attention_weight_max": 61.68292236328125,
      "activations/layer6_attention_weight_min": -56.7169075012207,
      "activations/layer7_attention_weight_max": 91.46563720703125,
      "activations/layer7_attention_weight_min": -88.84539031982422,
      "activations/layer8_attention_weight_max": 44.941314697265625,
      "activations/layer8_attention_weight_min": -40.90067672729492,
      "activations/layer9_attention_weight_max": 40.5845832824707,
      "activations/layer9_attention_weight_min": -35.451690673828125,
      "epoch": 5.4,
      "learning_rate": 0.0001163780303030303,
      "loss": 2.844,
      "step": 92850
    },
    {
      "activations/layer0_attention_weight_max": 14.657419204711914,
      "activations/layer0_attention_weight_min": -12.207807540893555,
      "activations/layer10_attention_weight_max": 34.22829055786133,
      "activations/layer10_attention_weight_min": -31.53134536743164,
      "activations/layer11_attention_weight_max": 36.93670654296875,
      "activations/layer11_attention_weight_min": -29.53175163269043,
      "activations/layer12_attention_weight_max": 29.461301803588867,
      "activations/layer12_attention_weight_min": -31.734743118286133,
      "activations/layer13_attention_weight_max": 44.612586975097656,
      "activations/layer13_attention_weight_min": -34.30495834350586,
      "activations/layer14_attention_weight_max": 56.693416595458984,
      "activations/layer14_attention_weight_min": -48.98249816894531,
      "activations/layer15_attention_weight_max": 43.00007247924805,
      "activations/layer15_attention_weight_min": -31.579252243041992,
      "activations/layer16_attention_weight_max": 37.07352066040039,
      "activations/layer16_attention_weight_min": -29.01806640625,
      "activations/layer17_attention_weight_max": 40.84816360473633,
      "activations/layer17_attention_weight_min": -23.829448699951172,
      "activations/layer18_attention_weight_max": 32.04034423828125,
      "activations/layer18_attention_weight_min": -20.13026237487793,
      "activations/layer19_attention_weight_max": 40.16832733154297,
      "activations/layer19_attention_weight_min": -32.088401794433594,
      "activations/layer1_attention_weight_max": 17.954137802124023,
      "activations/layer1_attention_weight_min": -14.12371826171875,
      "activations/layer20_attention_weight_max": 33.95939636230469,
      "activations/layer20_attention_weight_min": -23.35344123840332,
      "activations/layer21_attention_weight_max": 33.71431350708008,
      "activations/layer21_attention_weight_min": -21.286630630493164,
      "activations/layer22_attention_weight_max": 49.38400650024414,
      "activations/layer22_attention_weight_min": -28.801355361938477,
      "activations/layer23_attention_weight_max": 31.2371826171875,
      "activations/layer23_attention_weight_min": -21.861499786376953,
      "activations/layer2_attention_weight_max": 32.68476104736328,
      "activations/layer2_attention_weight_min": -32.50250244140625,
      "activations/layer3_attention_weight_max": 88.66250610351562,
      "activations/layer3_attention_weight_min": -90.15899658203125,
      "activations/layer4_attention_weight_max": 129.5465850830078,
      "activations/layer4_attention_weight_min": -126.9625015258789,
      "activations/layer5_attention_weight_max": 77.29171752929688,
      "activations/layer5_attention_weight_min": -78.29190826416016,
      "activations/layer6_attention_weight_max": 55.57752227783203,
      "activations/layer6_attention_weight_min": -57.399234771728516,
      "activations/layer7_attention_weight_max": 90.01177978515625,
      "activations/layer7_attention_weight_min": -87.99320983886719,
      "activations/layer8_attention_weight_max": 42.20161437988281,
      "activations/layer8_attention_weight_min": -40.0152473449707,
      "activations/layer9_attention_weight_max": 34.99262237548828,
      "activations/layer9_attention_weight_min": -39.641273498535156,
      "epoch": 5.4,
      "learning_rate": 0.0001163590909090909,
      "loss": 2.8399,
      "step": 92900
    },
    {
      "activations/layer0_attention_weight_max": 16.284053802490234,
      "activations/layer0_attention_weight_min": -10.587895393371582,
      "activations/layer10_attention_weight_max": 32.7353630065918,
      "activations/layer10_attention_weight_min": -31.482479095458984,
      "activations/layer11_attention_weight_max": 30.576175689697266,
      "activations/layer11_attention_weight_min": -28.702388763427734,
      "activations/layer12_attention_weight_max": 24.277671813964844,
      "activations/layer12_attention_weight_min": -25.92399787902832,
      "activations/layer13_attention_weight_max": 41.3955078125,
      "activations/layer13_attention_weight_min": -33.552860260009766,
      "activations/layer14_attention_weight_max": 58.39993667602539,
      "activations/layer14_attention_weight_min": -42.07119369506836,
      "activations/layer15_attention_weight_max": 35.913116455078125,
      "activations/layer15_attention_weight_min": -28.824003219604492,
      "activations/layer16_attention_weight_max": 33.586875915527344,
      "activations/layer16_attention_weight_min": -28.04047966003418,
      "activations/layer17_attention_weight_max": 35.64706802368164,
      "activations/layer17_attention_weight_min": -25.487215042114258,
      "activations/layer18_attention_weight_max": 34.030757904052734,
      "activations/layer18_attention_weight_min": -21.320552825927734,
      "activations/layer19_attention_weight_max": 38.3464469909668,
      "activations/layer19_attention_weight_min": -28.85202407836914,
      "activations/layer1_attention_weight_max": 17.496578216552734,
      "activations/layer1_attention_weight_min": -14.602107048034668,
      "activations/layer20_attention_weight_max": 29.766756057739258,
      "activations/layer20_attention_weight_min": -22.729515075683594,
      "activations/layer21_attention_weight_max": 28.886594772338867,
      "activations/layer21_attention_weight_min": -20.303394317626953,
      "activations/layer22_attention_weight_max": 51.42700958251953,
      "activations/layer22_attention_weight_min": -26.247997283935547,
      "activations/layer23_attention_weight_max": 36.32927322387695,
      "activations/layer23_attention_weight_min": -20.787954330444336,
      "activations/layer2_attention_weight_max": 31.838558197021484,
      "activations/layer2_attention_weight_min": -30.99087142944336,
      "activations/layer3_attention_weight_max": 88.37017059326172,
      "activations/layer3_attention_weight_min": -91.0229263305664,
      "activations/layer4_attention_weight_max": 127.5976791381836,
      "activations/layer4_attention_weight_min": -132.2759246826172,
      "activations/layer5_attention_weight_max": 75.1474838256836,
      "activations/layer5_attention_weight_min": -79.50069427490234,
      "activations/layer6_attention_weight_max": 55.63640213012695,
      "activations/layer6_attention_weight_min": -55.36331558227539,
      "activations/layer7_attention_weight_max": 82.80432891845703,
      "activations/layer7_attention_weight_min": -83.74374389648438,
      "activations/layer8_attention_weight_max": 38.799678802490234,
      "activations/layer8_attention_weight_min": -39.43076705932617,
      "activations/layer9_attention_weight_max": 36.21829605102539,
      "activations/layer9_attention_weight_min": -34.24884033203125,
      "epoch": 5.4,
      "learning_rate": 0.00011634015151515151,
      "loss": 2.8407,
      "step": 92950
    },
    {
      "activations/layer0_attention_weight_max": 15.97177505493164,
      "activations/layer0_attention_weight_min": -12.55738639831543,
      "activations/layer10_attention_weight_max": 32.012786865234375,
      "activations/layer10_attention_weight_min": -30.606002807617188,
      "activations/layer11_attention_weight_max": 29.70099449157715,
      "activations/layer11_attention_weight_min": -26.963825225830078,
      "activations/layer12_attention_weight_max": 23.186174392700195,
      "activations/layer12_attention_weight_min": -33.15951156616211,
      "activations/layer13_attention_weight_max": 41.98433303833008,
      "activations/layer13_attention_weight_min": -31.93471908569336,
      "activations/layer14_attention_weight_max": 64.32860565185547,
      "activations/layer14_attention_weight_min": -49.994056701660156,
      "activations/layer15_attention_weight_max": 36.24338150024414,
      "activations/layer15_attention_weight_min": -28.124984741210938,
      "activations/layer16_attention_weight_max": 34.99442672729492,
      "activations/layer16_attention_weight_min": -29.045103073120117,
      "activations/layer17_attention_weight_max": 37.11018753051758,
      "activations/layer17_attention_weight_min": -24.43359375,
      "activations/layer18_attention_weight_max": 29.481407165527344,
      "activations/layer18_attention_weight_min": -20.619407653808594,
      "activations/layer19_attention_weight_max": 39.94070053100586,
      "activations/layer19_attention_weight_min": -29.576908111572266,
      "activations/layer1_attention_weight_max": 17.397502899169922,
      "activations/layer1_attention_weight_min": -12.849937438964844,
      "activations/layer20_attention_weight_max": 30.839630126953125,
      "activations/layer20_attention_weight_min": -23.257923126220703,
      "activations/layer21_attention_weight_max": 30.443771362304688,
      "activations/layer21_attention_weight_min": -22.647661209106445,
      "activations/layer22_attention_weight_max": 48.17483901977539,
      "activations/layer22_attention_weight_min": -29.104001998901367,
      "activations/layer23_attention_weight_max": 36.64186096191406,
      "activations/layer23_attention_weight_min": -22.77141571044922,
      "activations/layer2_attention_weight_max": 32.14171600341797,
      "activations/layer2_attention_weight_min": -31.48537254333496,
      "activations/layer3_attention_weight_max": 88.91928100585938,
      "activations/layer3_attention_weight_min": -90.2069320678711,
      "activations/layer4_attention_weight_max": 127.23250579833984,
      "activations/layer4_attention_weight_min": -124.25591278076172,
      "activations/layer5_attention_weight_max": 78.38493347167969,
      "activations/layer5_attention_weight_min": -74.03837585449219,
      "activations/layer6_attention_weight_max": 54.98881149291992,
      "activations/layer6_attention_weight_min": -54.056827545166016,
      "activations/layer7_attention_weight_max": 82.43695068359375,
      "activations/layer7_attention_weight_min": -84.87466430664062,
      "activations/layer8_attention_weight_max": 39.129844665527344,
      "activations/layer8_attention_weight_min": -38.75804138183594,
      "activations/layer9_attention_weight_max": 34.829708099365234,
      "activations/layer9_attention_weight_min": -33.03874969482422,
      "epoch": 5.4,
      "learning_rate": 0.0001163212121212121,
      "loss": 2.8415,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_loss": 2.794921875,
      "eval_runtime": 8.5552,
      "eval_samples_per_second": 501.915,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_openwebtext_loss": 2.794921875,
      "eval_openwebtext_ppl": 16.36135047271935,
      "eval_openwebtext_runtime": 8.5552,
      "eval_openwebtext_samples_per_second": 501.915,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 1.9669,
      "eval_wikitext_samples_per_second": 231.832,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_lambada_loss": 2.697265625,
      "eval_lambada_ppl": 14.83910053420958,
      "eval_lambada_runtime": 9.6062,
      "eval_lambada_samples_per_second": 506.86,
      "step": 93000
    },
    {
      "activations/layer0_attention_weight_max": 16.1007137298584,
      "activations/layer0_attention_weight_min": -11.106474876403809,
      "activations/layer10_attention_weight_max": 41.12164306640625,
      "activations/layer10_attention_weight_min": -37.97944641113281,
      "activations/layer11_attention_weight_max": 38.783531188964844,
      "activations/layer11_attention_weight_min": -34.65796661376953,
      "activations/layer12_attention_weight_max": 33.79175567626953,
      "activations/layer12_attention_weight_min": -34.04361343383789,
      "activations/layer13_attention_weight_max": 56.402347564697266,
      "activations/layer13_attention_weight_min": -40.29771423339844,
      "activations/layer14_attention_weight_max": 64.19976806640625,
      "activations/layer14_attention_weight_min": -45.12066650390625,
      "activations/layer15_attention_weight_max": 44.22142028808594,
      "activations/layer15_attention_weight_min": -32.83359909057617,
      "activations/layer16_attention_weight_max": 35.95423889160156,
      "activations/layer16_attention_weight_min": -29.164648056030273,
      "activations/layer17_attention_weight_max": 33.99238204956055,
      "activations/layer17_attention_weight_min": -23.291168212890625,
      "activations/layer18_attention_weight_max": 30.848318099975586,
      "activations/layer18_attention_weight_min": -19.704498291015625,
      "activations/layer19_attention_weight_max": 40.29665756225586,
      "activations/layer19_attention_weight_min": -29.149778366088867,
      "activations/layer1_attention_weight_max": 17.647932052612305,
      "activations/layer1_attention_weight_min": -15.312359809875488,
      "activations/layer20_attention_weight_max": 32.88740921020508,
      "activations/layer20_attention_weight_min": -22.828306198120117,
      "activations/layer21_attention_weight_max": 32.7419319152832,
      "activations/layer21_attention_weight_min": -21.37143898010254,
      "activations/layer22_attention_weight_max": 46.147789001464844,
      "activations/layer22_attention_weight_min": -28.078218460083008,
      "activations/layer23_attention_weight_max": 33.35215377807617,
      "activations/layer23_attention_weight_min": -21.094045639038086,
      "activations/layer2_attention_weight_max": 32.14044189453125,
      "activations/layer2_attention_weight_min": -30.57929229736328,
      "activations/layer3_attention_weight_max": 88.32848358154297,
      "activations/layer3_attention_weight_min": -86.13948059082031,
      "activations/layer4_attention_weight_max": 134.5723876953125,
      "activations/layer4_attention_weight_min": -121.12715148925781,
      "activations/layer5_attention_weight_max": 81.34199523925781,
      "activations/layer5_attention_weight_min": -78.13687133789062,
      "activations/layer6_attention_weight_max": 55.87996292114258,
      "activations/layer6_attention_weight_min": -57.902008056640625,
      "activations/layer7_attention_weight_max": 95.8346939086914,
      "activations/layer7_attention_weight_min": -89.04459381103516,
      "activations/layer8_attention_weight_max": 47.156890869140625,
      "activations/layer8_attention_weight_min": -43.47801208496094,
      "activations/layer9_attention_weight_max": 43.3476676940918,
      "activations/layer9_attention_weight_min": -37.60832214355469,
      "epoch": 5.41,
      "learning_rate": 0.00011630227272727272,
      "loss": 2.8358,
      "step": 93050
    },
    {
      "activations/layer0_attention_weight_max": 16.455564498901367,
      "activations/layer0_attention_weight_min": -11.821622848510742,
      "activations/layer10_attention_weight_max": 30.871829986572266,
      "activations/layer10_attention_weight_min": -31.12129783630371,
      "activations/layer11_attention_weight_max": 30.091236114501953,
      "activations/layer11_attention_weight_min": -27.793867111206055,
      "activations/layer12_attention_weight_max": 23.832862854003906,
      "activations/layer12_attention_weight_min": -27.152767181396484,
      "activations/layer13_attention_weight_max": 39.718963623046875,
      "activations/layer13_attention_weight_min": -30.47374725341797,
      "activations/layer14_attention_weight_max": 49.01530075073242,
      "activations/layer14_attention_weight_min": -38.982337951660156,
      "activations/layer15_attention_weight_max": 36.4373893737793,
      "activations/layer15_attention_weight_min": -28.185956954956055,
      "activations/layer16_attention_weight_max": 33.724266052246094,
      "activations/layer16_attention_weight_min": -27.48465347290039,
      "activations/layer17_attention_weight_max": 37.096744537353516,
      "activations/layer17_attention_weight_min": -24.756118774414062,
      "activations/layer18_attention_weight_max": 36.39183044433594,
      "activations/layer18_attention_weight_min": -20.873414993286133,
      "activations/layer19_attention_weight_max": 38.94811248779297,
      "activations/layer19_attention_weight_min": -29.45362663269043,
      "activations/layer1_attention_weight_max": 17.066896438598633,
      "activations/layer1_attention_weight_min": -13.936639785766602,
      "activations/layer20_attention_weight_max": 31.307952880859375,
      "activations/layer20_attention_weight_min": -24.19325828552246,
      "activations/layer21_attention_weight_max": 31.099437713623047,
      "activations/layer21_attention_weight_min": -21.897470474243164,
      "activations/layer22_attention_weight_max": 47.76848220825195,
      "activations/layer22_attention_weight_min": -30.619869232177734,
      "activations/layer23_attention_weight_max": 36.49061584472656,
      "activations/layer23_attention_weight_min": -21.552228927612305,
      "activations/layer2_attention_weight_max": 34.017005920410156,
      "activations/layer2_attention_weight_min": -32.121002197265625,
      "activations/layer3_attention_weight_max": 91.30699157714844,
      "activations/layer3_attention_weight_min": -87.52745819091797,
      "activations/layer4_attention_weight_max": 126.05012512207031,
      "activations/layer4_attention_weight_min": -127.85184478759766,
      "activations/layer5_attention_weight_max": 74.93891906738281,
      "activations/layer5_attention_weight_min": -73.81700897216797,
      "activations/layer6_attention_weight_max": 54.046390533447266,
      "activations/layer6_attention_weight_min": -58.02309036254883,
      "activations/layer7_attention_weight_max": 82.69646453857422,
      "activations/layer7_attention_weight_min": -85.60970306396484,
      "activations/layer8_attention_weight_max": 36.607635498046875,
      "activations/layer8_attention_weight_min": -39.1195182800293,
      "activations/layer9_attention_weight_max": 36.45844650268555,
      "activations/layer9_attention_weight_min": -31.600690841674805,
      "epoch": 5.41,
      "learning_rate": 0.00011628333333333333,
      "loss": 2.8604,
      "step": 93100
    },
    {
      "activations/layer0_attention_weight_max": 15.613706588745117,
      "activations/layer0_attention_weight_min": -13.059722900390625,
      "activations/layer10_attention_weight_max": 34.03917694091797,
      "activations/layer10_attention_weight_min": -30.558156967163086,
      "activations/layer11_attention_weight_max": 32.04981231689453,
      "activations/layer11_attention_weight_min": -29.111209869384766,
      "activations/layer12_attention_weight_max": 35.8239631652832,
      "activations/layer12_attention_weight_min": -27.39493751525879,
      "activations/layer13_attention_weight_max": 46.30373764038086,
      "activations/layer13_attention_weight_min": -38.2760009765625,
      "activations/layer14_attention_weight_max": 67.10102844238281,
      "activations/layer14_attention_weight_min": -44.63180160522461,
      "activations/layer15_attention_weight_max": 39.51768493652344,
      "activations/layer15_attention_weight_min": -28.16286277770996,
      "activations/layer16_attention_weight_max": 36.31911849975586,
      "activations/layer16_attention_weight_min": -27.523496627807617,
      "activations/layer17_attention_weight_max": 36.787498474121094,
      "activations/layer17_attention_weight_min": -25.0374698638916,
      "activations/layer18_attention_weight_max": 30.12189292907715,
      "activations/layer18_attention_weight_min": -20.818357467651367,
      "activations/layer19_attention_weight_max": 42.849674224853516,
      "activations/layer19_attention_weight_min": -30.331317901611328,
      "activations/layer1_attention_weight_max": 18.249536514282227,
      "activations/layer1_attention_weight_min": -17.266279220581055,
      "activations/layer20_attention_weight_max": 35.40377426147461,
      "activations/layer20_attention_weight_min": -23.938352584838867,
      "activations/layer21_attention_weight_max": 35.189178466796875,
      "activations/layer21_attention_weight_min": -19.83761978149414,
      "activations/layer22_attention_weight_max": 54.65077209472656,
      "activations/layer22_attention_weight_min": -26.92301368713379,
      "activations/layer23_attention_weight_max": 36.60301208496094,
      "activations/layer23_attention_weight_min": -21.207157135009766,
      "activations/layer2_attention_weight_max": 32.803619384765625,
      "activations/layer2_attention_weight_min": -31.341293334960938,
      "activations/layer3_attention_weight_max": 91.03999328613281,
      "activations/layer3_attention_weight_min": -91.00459289550781,
      "activations/layer4_attention_weight_max": 130.5734100341797,
      "activations/layer4_attention_weight_min": -121.5337142944336,
      "activations/layer5_attention_weight_max": 80.97289276123047,
      "activations/layer5_attention_weight_min": -77.12297058105469,
      "activations/layer6_attention_weight_max": 58.2731819152832,
      "activations/layer6_attention_weight_min": -55.014320373535156,
      "activations/layer7_attention_weight_max": 87.72489166259766,
      "activations/layer7_attention_weight_min": -86.43354797363281,
      "activations/layer8_attention_weight_max": 43.02167892456055,
      "activations/layer8_attention_weight_min": -40.55076599121094,
      "activations/layer9_attention_weight_max": 37.438453674316406,
      "activations/layer9_attention_weight_min": -34.82648849487305,
      "epoch": 5.41,
      "learning_rate": 0.00011626439393939392,
      "loss": 2.829,
      "step": 93150
    },
    {
      "activations/layer0_attention_weight_max": 16.17464828491211,
      "activations/layer0_attention_weight_min": -11.669973373413086,
      "activations/layer10_attention_weight_max": 32.33064651489258,
      "activations/layer10_attention_weight_min": -30.285337448120117,
      "activations/layer11_attention_weight_max": 32.45868682861328,
      "activations/layer11_attention_weight_min": -27.95381736755371,
      "activations/layer12_attention_weight_max": 24.377206802368164,
      "activations/layer12_attention_weight_min": -25.96172332763672,
      "activations/layer13_attention_weight_max": 40.941139221191406,
      "activations/layer13_attention_weight_min": -32.96657943725586,
      "activations/layer14_attention_weight_max": 70.18798065185547,
      "activations/layer14_attention_weight_min": -44.490821838378906,
      "activations/layer15_attention_weight_max": 38.05794906616211,
      "activations/layer15_attention_weight_min": -30.831363677978516,
      "activations/layer16_attention_weight_max": 37.10734176635742,
      "activations/layer16_attention_weight_min": -28.97236442565918,
      "activations/layer17_attention_weight_max": 36.25814437866211,
      "activations/layer17_attention_weight_min": -24.662410736083984,
      "activations/layer18_attention_weight_max": 31.515159606933594,
      "activations/layer18_attention_weight_min": -20.308366775512695,
      "activations/layer19_attention_weight_max": 35.83354187011719,
      "activations/layer19_attention_weight_min": -30.834247589111328,
      "activations/layer1_attention_weight_max": 17.48833465576172,
      "activations/layer1_attention_weight_min": -14.05888557434082,
      "activations/layer20_attention_weight_max": 29.845991134643555,
      "activations/layer20_attention_weight_min": -23.469228744506836,
      "activations/layer21_attention_weight_max": 29.238149642944336,
      "activations/layer21_attention_weight_min": -21.944467544555664,
      "activations/layer22_attention_weight_max": 43.75981521606445,
      "activations/layer22_attention_weight_min": -28.471046447753906,
      "activations/layer23_attention_weight_max": 30.110395431518555,
      "activations/layer23_attention_weight_min": -22.436912536621094,
      "activations/layer2_attention_weight_max": 30.766340255737305,
      "activations/layer2_attention_weight_min": -30.98849105834961,
      "activations/layer3_attention_weight_max": 88.28629302978516,
      "activations/layer3_attention_weight_min": -88.80473327636719,
      "activations/layer4_attention_weight_max": 125.3187255859375,
      "activations/layer4_attention_weight_min": -122.32213592529297,
      "activations/layer5_attention_weight_max": 74.07707214355469,
      "activations/layer5_attention_weight_min": -74.90321350097656,
      "activations/layer6_attention_weight_max": 55.4672966003418,
      "activations/layer6_attention_weight_min": -58.091339111328125,
      "activations/layer7_attention_weight_max": 85.5628433227539,
      "activations/layer7_attention_weight_min": -85.49261474609375,
      "activations/layer8_attention_weight_max": 39.03715896606445,
      "activations/layer8_attention_weight_min": -37.90962219238281,
      "activations/layer9_attention_weight_max": 33.72895812988281,
      "activations/layer9_attention_weight_min": -35.23747253417969,
      "epoch": 5.42,
      "learning_rate": 0.00011624545454545453,
      "loss": 2.8304,
      "step": 93200
    },
    {
      "activations/layer0_attention_weight_max": 16.90601348876953,
      "activations/layer0_attention_weight_min": -12.753961563110352,
      "activations/layer10_attention_weight_max": 36.56158447265625,
      "activations/layer10_attention_weight_min": -36.81837844848633,
      "activations/layer11_attention_weight_max": 37.23656463623047,
      "activations/layer11_attention_weight_min": -31.978343963623047,
      "activations/layer12_attention_weight_max": 37.727054595947266,
      "activations/layer12_attention_weight_min": -27.141416549682617,
      "activations/layer13_attention_weight_max": 60.47446823120117,
      "activations/layer13_attention_weight_min": -36.55366897583008,
      "activations/layer14_attention_weight_max": 71.55219268798828,
      "activations/layer14_attention_weight_min": -56.83871841430664,
      "activations/layer15_attention_weight_max": 52.20109176635742,
      "activations/layer15_attention_weight_min": -30.86361312866211,
      "activations/layer16_attention_weight_max": 37.84321594238281,
      "activations/layer16_attention_weight_min": -28.265544891357422,
      "activations/layer17_attention_weight_max": 37.514259338378906,
      "activations/layer17_attention_weight_min": -24.50419044494629,
      "activations/layer18_attention_weight_max": 35.27495574951172,
      "activations/layer18_attention_weight_min": -20.42646598815918,
      "activations/layer19_attention_weight_max": 43.79240036010742,
      "activations/layer19_attention_weight_min": -30.072219848632812,
      "activations/layer1_attention_weight_max": 21.24321937561035,
      "activations/layer1_attention_weight_min": -18.855392456054688,
      "activations/layer20_attention_weight_max": 33.624088287353516,
      "activations/layer20_attention_weight_min": -23.972570419311523,
      "activations/layer21_attention_weight_max": 38.1802864074707,
      "activations/layer21_attention_weight_min": -21.537656784057617,
      "activations/layer22_attention_weight_max": 51.4742546081543,
      "activations/layer22_attention_weight_min": -28.837549209594727,
      "activations/layer23_attention_weight_max": 40.055843353271484,
      "activations/layer23_attention_weight_min": -22.93524742126465,
      "activations/layer2_attention_weight_max": 32.83088302612305,
      "activations/layer2_attention_weight_min": -34.542388916015625,
      "activations/layer3_attention_weight_max": 95.24925231933594,
      "activations/layer3_attention_weight_min": -96.68102264404297,
      "activations/layer4_attention_weight_max": 136.7290496826172,
      "activations/layer4_attention_weight_min": -142.57888793945312,
      "activations/layer5_attention_weight_max": 82.50450897216797,
      "activations/layer5_attention_weight_min": -80.01974487304688,
      "activations/layer6_attention_weight_max": 59.50968551635742,
      "activations/layer6_attention_weight_min": -58.384193420410156,
      "activations/layer7_attention_weight_max": 94.25172424316406,
      "activations/layer7_attention_weight_min": -92.97364044189453,
      "activations/layer8_attention_weight_max": 43.78029251098633,
      "activations/layer8_attention_weight_min": -43.00497055053711,
      "activations/layer9_attention_weight_max": 37.50061798095703,
      "activations/layer9_attention_weight_min": -35.96219253540039,
      "epoch": 5.42,
      "learning_rate": 0.00011622689393939393,
      "loss": 2.8407,
      "step": 93250
    },
    {
      "activations/layer0_attention_weight_max": 16.773523330688477,
      "activations/layer0_attention_weight_min": -12.797743797302246,
      "activations/layer10_attention_weight_max": 33.649166107177734,
      "activations/layer10_attention_weight_min": -30.484939575195312,
      "activations/layer11_attention_weight_max": 33.45799255371094,
      "activations/layer11_attention_weight_min": -28.208797454833984,
      "activations/layer12_attention_weight_max": 31.683238983154297,
      "activations/layer12_attention_weight_min": -24.85490608215332,
      "activations/layer13_attention_weight_max": 43.349849700927734,
      "activations/layer13_attention_weight_min": -33.86903381347656,
      "activations/layer14_attention_weight_max": 72.41632080078125,
      "activations/layer14_attention_weight_min": -50.3782844543457,
      "activations/layer15_attention_weight_max": 43.996856689453125,
      "activations/layer15_attention_weight_min": -28.502992630004883,
      "activations/layer16_attention_weight_max": 41.91911315917969,
      "activations/layer16_attention_weight_min": -27.352296829223633,
      "activations/layer17_attention_weight_max": 42.71815490722656,
      "activations/layer17_attention_weight_min": -23.0589599609375,
      "activations/layer18_attention_weight_max": 37.548545837402344,
      "activations/layer18_attention_weight_min": -21.354698181152344,
      "activations/layer19_attention_weight_max": 46.61237335205078,
      "activations/layer19_attention_weight_min": -31.7728271484375,
      "activations/layer1_attention_weight_max": 16.565589904785156,
      "activations/layer1_attention_weight_min": -17.18446159362793,
      "activations/layer20_attention_weight_max": 39.629859924316406,
      "activations/layer20_attention_weight_min": -23.505155563354492,
      "activations/layer21_attention_weight_max": 40.5287971496582,
      "activations/layer21_attention_weight_min": -20.577198028564453,
      "activations/layer22_attention_weight_max": 54.45575714111328,
      "activations/layer22_attention_weight_min": -26.37700080871582,
      "activations/layer23_attention_weight_max": 35.135223388671875,
      "activations/layer23_attention_weight_min": -20.70134735107422,
      "activations/layer2_attention_weight_max": 30.28551483154297,
      "activations/layer2_attention_weight_min": -29.654964447021484,
      "activations/layer3_attention_weight_max": 86.26445007324219,
      "activations/layer3_attention_weight_min": -87.47467041015625,
      "activations/layer4_attention_weight_max": 125.41814422607422,
      "activations/layer4_attention_weight_min": -119.92911529541016,
      "activations/layer5_attention_weight_max": 74.44145965576172,
      "activations/layer5_attention_weight_min": -73.90642547607422,
      "activations/layer6_attention_weight_max": 53.71964645385742,
      "activations/layer6_attention_weight_min": -53.431026458740234,
      "activations/layer7_attention_weight_max": 86.33528900146484,
      "activations/layer7_attention_weight_min": -81.86167907714844,
      "activations/layer8_attention_weight_max": 40.578346252441406,
      "activations/layer8_attention_weight_min": -37.49459457397461,
      "activations/layer9_attention_weight_max": 36.69449996948242,
      "activations/layer9_attention_weight_min": -33.61205291748047,
      "epoch": 5.42,
      "learning_rate": 0.00011620795454545454,
      "loss": 2.8571,
      "step": 93300
    },
    {
      "activations/layer0_attention_weight_max": 15.970016479492188,
      "activations/layer0_attention_weight_min": -12.88554859161377,
      "activations/layer10_attention_weight_max": 31.242612838745117,
      "activations/layer10_attention_weight_min": -30.520721435546875,
      "activations/layer11_attention_weight_max": 32.72515869140625,
      "activations/layer11_attention_weight_min": -28.412023544311523,
      "activations/layer12_attention_weight_max": 26.219003677368164,
      "activations/layer12_attention_weight_min": -25.74394416809082,
      "activations/layer13_attention_weight_max": 41.13174057006836,
      "activations/layer13_attention_weight_min": -34.32130432128906,
      "activations/layer14_attention_weight_max": 56.82453155517578,
      "activations/layer14_attention_weight_min": -42.52790069580078,
      "activations/layer15_attention_weight_max": 39.90100860595703,
      "activations/layer15_attention_weight_min": -28.01762580871582,
      "activations/layer16_attention_weight_max": 38.5040397644043,
      "activations/layer16_attention_weight_min": -27.817195892333984,
      "activations/layer17_attention_weight_max": 37.205169677734375,
      "activations/layer17_attention_weight_min": -25.01982307434082,
      "activations/layer18_attention_weight_max": 37.510475158691406,
      "activations/layer18_attention_weight_min": -23.907955169677734,
      "activations/layer19_attention_weight_max": 41.19068908691406,
      "activations/layer19_attention_weight_min": -30.717514038085938,
      "activations/layer1_attention_weight_max": 17.920679092407227,
      "activations/layer1_attention_weight_min": -14.439014434814453,
      "activations/layer20_attention_weight_max": 34.3921012878418,
      "activations/layer20_attention_weight_min": -22.90643882751465,
      "activations/layer21_attention_weight_max": 31.442522048950195,
      "activations/layer21_attention_weight_min": -20.8855037689209,
      "activations/layer22_attention_weight_max": 48.55268096923828,
      "activations/layer22_attention_weight_min": -28.349729537963867,
      "activations/layer23_attention_weight_max": 33.611610412597656,
      "activations/layer23_attention_weight_min": -22.373350143432617,
      "activations/layer2_attention_weight_max": 32.52422332763672,
      "activations/layer2_attention_weight_min": -29.762256622314453,
      "activations/layer3_attention_weight_max": 86.0837173461914,
      "activations/layer3_attention_weight_min": -86.1196517944336,
      "activations/layer4_attention_weight_max": 120.16627502441406,
      "activations/layer4_attention_weight_min": -123.37071990966797,
      "activations/layer5_attention_weight_max": 75.66092681884766,
      "activations/layer5_attention_weight_min": -73.28892517089844,
      "activations/layer6_attention_weight_max": 54.320613861083984,
      "activations/layer6_attention_weight_min": -54.45961380004883,
      "activations/layer7_attention_weight_max": 82.23551940917969,
      "activations/layer7_attention_weight_min": -82.41996002197266,
      "activations/layer8_attention_weight_max": 37.96974563598633,
      "activations/layer8_attention_weight_min": -36.87918472290039,
      "activations/layer9_attention_weight_max": 33.28173828125,
      "activations/layer9_attention_weight_min": -33.91903305053711,
      "epoch": 5.42,
      "learning_rate": 0.00011618939393939393,
      "loss": 2.8525,
      "step": 93350
    },
    {
      "activations/layer0_attention_weight_max": 16.370098114013672,
      "activations/layer0_attention_weight_min": -12.453889846801758,
      "activations/layer10_attention_weight_max": 31.74045181274414,
      "activations/layer10_attention_weight_min": -30.251468658447266,
      "activations/layer11_attention_weight_max": 31.723087310791016,
      "activations/layer11_attention_weight_min": -29.02892303466797,
      "activations/layer12_attention_weight_max": 31.24127769470215,
      "activations/layer12_attention_weight_min": -28.666763305664062,
      "activations/layer13_attention_weight_max": 40.595191955566406,
      "activations/layer13_attention_weight_min": -31.138708114624023,
      "activations/layer14_attention_weight_max": 55.61505889892578,
      "activations/layer14_attention_weight_min": -42.82960891723633,
      "activations/layer15_attention_weight_max": 38.558536529541016,
      "activations/layer15_attention_weight_min": -27.293718338012695,
      "activations/layer16_attention_weight_max": 34.46145248413086,
      "activations/layer16_attention_weight_min": -27.268808364868164,
      "activations/layer17_attention_weight_max": 36.19142532348633,
      "activations/layer17_attention_weight_min": -24.541461944580078,
      "activations/layer18_attention_weight_max": 30.1196346282959,
      "activations/layer18_attention_weight_min": -19.418502807617188,
      "activations/layer19_attention_weight_max": 40.082061767578125,
      "activations/layer19_attention_weight_min": -28.00079917907715,
      "activations/layer1_attention_weight_max": 18.847923278808594,
      "activations/layer1_attention_weight_min": -14.14327621459961,
      "activations/layer20_attention_weight_max": 33.11552429199219,
      "activations/layer20_attention_weight_min": -21.66728973388672,
      "activations/layer21_attention_weight_max": 32.99725341796875,
      "activations/layer21_attention_weight_min": -22.382705688476562,
      "activations/layer22_attention_weight_max": 53.602508544921875,
      "activations/layer22_attention_weight_min": -29.703338623046875,
      "activations/layer23_attention_weight_max": 31.00361442565918,
      "activations/layer23_attention_weight_min": -21.542593002319336,
      "activations/layer2_attention_weight_max": 32.015037536621094,
      "activations/layer2_attention_weight_min": -30.606964111328125,
      "activations/layer3_attention_weight_max": 88.00444030761719,
      "activations/layer3_attention_weight_min": -89.50963592529297,
      "activations/layer4_attention_weight_max": 122.97987365722656,
      "activations/layer4_attention_weight_min": -121.8084945678711,
      "activations/layer5_attention_weight_max": 74.75326538085938,
      "activations/layer5_attention_weight_min": -72.7090835571289,
      "activations/layer6_attention_weight_max": 53.33430480957031,
      "activations/layer6_attention_weight_min": -53.20401382446289,
      "activations/layer7_attention_weight_max": 77.32792663574219,
      "activations/layer7_attention_weight_min": -82.58252716064453,
      "activations/layer8_attention_weight_max": 40.47209548950195,
      "activations/layer8_attention_weight_min": -41.4505500793457,
      "activations/layer9_attention_weight_max": 35.830711364746094,
      "activations/layer9_attention_weight_min": -35.07571029663086,
      "epoch": 5.43,
      "learning_rate": 0.00011617045454545453,
      "loss": 2.8309,
      "step": 93400
    },
    {
      "activations/layer0_attention_weight_max": 16.23172950744629,
      "activations/layer0_attention_weight_min": -12.682319641113281,
      "activations/layer10_attention_weight_max": 36.4373893737793,
      "activations/layer10_attention_weight_min": -31.72398567199707,
      "activations/layer11_attention_weight_max": 35.60944366455078,
      "activations/layer11_attention_weight_min": -28.85000228881836,
      "activations/layer12_attention_weight_max": 27.376859664916992,
      "activations/layer12_attention_weight_min": -27.321182250976562,
      "activations/layer13_attention_weight_max": 46.161128997802734,
      "activations/layer13_attention_weight_min": -33.173133850097656,
      "activations/layer14_attention_weight_max": 58.104827880859375,
      "activations/layer14_attention_weight_min": -43.19849395751953,
      "activations/layer15_attention_weight_max": 39.855072021484375,
      "activations/layer15_attention_weight_min": -28.557605743408203,
      "activations/layer16_attention_weight_max": 40.7121467590332,
      "activations/layer16_attention_weight_min": -28.369524002075195,
      "activations/layer17_attention_weight_max": 45.47114562988281,
      "activations/layer17_attention_weight_min": -26.40478515625,
      "activations/layer18_attention_weight_max": 33.53697967529297,
      "activations/layer18_attention_weight_min": -20.77760887145996,
      "activations/layer19_attention_weight_max": 47.11016082763672,
      "activations/layer19_attention_weight_min": -29.056365966796875,
      "activations/layer1_attention_weight_max": 16.99248695373535,
      "activations/layer1_attention_weight_min": -13.622177124023438,
      "activations/layer20_attention_weight_max": 37.287540435791016,
      "activations/layer20_attention_weight_min": -24.832677841186523,
      "activations/layer21_attention_weight_max": 33.960853576660156,
      "activations/layer21_attention_weight_min": -25.28476333618164,
      "activations/layer22_attention_weight_max": 53.22677993774414,
      "activations/layer22_attention_weight_min": -29.383277893066406,
      "activations/layer23_attention_weight_max": 41.136329650878906,
      "activations/layer23_attention_weight_min": -22.754676818847656,
      "activations/layer2_attention_weight_max": 31.29573631286621,
      "activations/layer2_attention_weight_min": -31.666133880615234,
      "activations/layer3_attention_weight_max": 89.1015625,
      "activations/layer3_attention_weight_min": -91.18396759033203,
      "activations/layer4_attention_weight_max": 126.04573059082031,
      "activations/layer4_attention_weight_min": -122.52668762207031,
      "activations/layer5_attention_weight_max": 78.4565200805664,
      "activations/layer5_attention_weight_min": -74.55850982666016,
      "activations/layer6_attention_weight_max": 57.407806396484375,
      "activations/layer6_attention_weight_min": -56.5283203125,
      "activations/layer7_attention_weight_max": 84.75370025634766,
      "activations/layer7_attention_weight_min": -82.27857971191406,
      "activations/layer8_attention_weight_max": 41.88956069946289,
      "activations/layer8_attention_weight_min": -43.09324645996094,
      "activations/layer9_attention_weight_max": 37.318180084228516,
      "activations/layer9_attention_weight_min": -36.06808090209961,
      "epoch": 5.43,
      "learning_rate": 0.00011615151515151513,
      "loss": 2.8523,
      "step": 93450
    },
    {
      "activations/layer0_attention_weight_max": 16.215511322021484,
      "activations/layer0_attention_weight_min": -11.806939125061035,
      "activations/layer10_attention_weight_max": 34.18260955810547,
      "activations/layer10_attention_weight_min": -29.40270233154297,
      "activations/layer11_attention_weight_max": 39.06370544433594,
      "activations/layer11_attention_weight_min": -29.897445678710938,
      "activations/layer12_attention_weight_max": 28.340486526489258,
      "activations/layer12_attention_weight_min": -28.680620193481445,
      "activations/layer13_attention_weight_max": 47.5560417175293,
      "activations/layer13_attention_weight_min": -35.2174072265625,
      "activations/layer14_attention_weight_max": 60.61863327026367,
      "activations/layer14_attention_weight_min": -45.33918762207031,
      "activations/layer15_attention_weight_max": 40.497230529785156,
      "activations/layer15_attention_weight_min": -28.124874114990234,
      "activations/layer16_attention_weight_max": 37.59010314941406,
      "activations/layer16_attention_weight_min": -26.688827514648438,
      "activations/layer17_attention_weight_max": 36.904449462890625,
      "activations/layer17_attention_weight_min": -25.259727478027344,
      "activations/layer18_attention_weight_max": 29.820768356323242,
      "activations/layer18_attention_weight_min": -19.504703521728516,
      "activations/layer19_attention_weight_max": 41.29201889038086,
      "activations/layer19_attention_weight_min": -31.401071548461914,
      "activations/layer1_attention_weight_max": 17.485496520996094,
      "activations/layer1_attention_weight_min": -14.420868873596191,
      "activations/layer20_attention_weight_max": 31.19013023376465,
      "activations/layer20_attention_weight_min": -25.030675888061523,
      "activations/layer21_attention_weight_max": 29.758342742919922,
      "activations/layer21_attention_weight_min": -20.3862361907959,
      "activations/layer22_attention_weight_max": 49.219627380371094,
      "activations/layer22_attention_weight_min": -28.65782928466797,
      "activations/layer23_attention_weight_max": 33.550323486328125,
      "activations/layer23_attention_weight_min": -22.29241943359375,
      "activations/layer2_attention_weight_max": 31.377796173095703,
      "activations/layer2_attention_weight_min": -31.074981689453125,
      "activations/layer3_attention_weight_max": 85.8301773071289,
      "activations/layer3_attention_weight_min": -90.94442749023438,
      "activations/layer4_attention_weight_max": 124.88423156738281,
      "activations/layer4_attention_weight_min": -125.99176025390625,
      "activations/layer5_attention_weight_max": 75.32381439208984,
      "activations/layer5_attention_weight_min": -71.32872009277344,
      "activations/layer6_attention_weight_max": 51.51929473876953,
      "activations/layer6_attention_weight_min": -53.6391716003418,
      "activations/layer7_attention_weight_max": 81.73612976074219,
      "activations/layer7_attention_weight_min": -80.77603149414062,
      "activations/layer8_attention_weight_max": 39.115352630615234,
      "activations/layer8_attention_weight_min": -39.397499084472656,
      "activations/layer9_attention_weight_max": 35.87607192993164,
      "activations/layer9_attention_weight_min": -31.26039695739746,
      "epoch": 5.43,
      "learning_rate": 0.00011613257575757575,
      "loss": 2.8404,
      "step": 93500
    },
    {
      "activations/layer0_attention_weight_max": 16.36786651611328,
      "activations/layer0_attention_weight_min": -12.544830322265625,
      "activations/layer10_attention_weight_max": 39.700321197509766,
      "activations/layer10_attention_weight_min": -34.07554626464844,
      "activations/layer11_attention_weight_max": 41.50471496582031,
      "activations/layer11_attention_weight_min": -32.18791198730469,
      "activations/layer12_attention_weight_max": 58.701820373535156,
      "activations/layer12_attention_weight_min": -28.35163116455078,
      "activations/layer13_attention_weight_max": 58.55610275268555,
      "activations/layer13_attention_weight_min": -34.253021240234375,
      "activations/layer14_attention_weight_max": 73.43936157226562,
      "activations/layer14_attention_weight_min": -45.9228630065918,
      "activations/layer15_attention_weight_max": 48.25703048706055,
      "activations/layer15_attention_weight_min": -29.389753341674805,
      "activations/layer16_attention_weight_max": 38.877567291259766,
      "activations/layer16_attention_weight_min": -26.654211044311523,
      "activations/layer17_attention_weight_max": 40.19276809692383,
      "activations/layer17_attention_weight_min": -23.70270538330078,
      "activations/layer18_attention_weight_max": 36.52034378051758,
      "activations/layer18_attention_weight_min": -21.131689071655273,
      "activations/layer19_attention_weight_max": 47.20473861694336,
      "activations/layer19_attention_weight_min": -29.433618545532227,
      "activations/layer1_attention_weight_max": 17.294147491455078,
      "activations/layer1_attention_weight_min": -16.10236930847168,
      "activations/layer20_attention_weight_max": 36.35621643066406,
      "activations/layer20_attention_weight_min": -24.949289321899414,
      "activations/layer21_attention_weight_max": 43.97494888305664,
      "activations/layer21_attention_weight_min": -20.67860221862793,
      "activations/layer22_attention_weight_max": 55.5866813659668,
      "activations/layer22_attention_weight_min": -27.553735733032227,
      "activations/layer23_attention_weight_max": 36.252403259277344,
      "activations/layer23_attention_weight_min": -21.205181121826172,
      "activations/layer2_attention_weight_max": 31.990474700927734,
      "activations/layer2_attention_weight_min": -31.980487823486328,
      "activations/layer3_attention_weight_max": 92.84262084960938,
      "activations/layer3_attention_weight_min": -96.95954895019531,
      "activations/layer4_attention_weight_max": 134.56532287597656,
      "activations/layer4_attention_weight_min": -138.79046630859375,
      "activations/layer5_attention_weight_max": 83.07537078857422,
      "activations/layer5_attention_weight_min": -78.75901794433594,
      "activations/layer6_attention_weight_max": 59.89508819580078,
      "activations/layer6_attention_weight_min": -59.5709114074707,
      "activations/layer7_attention_weight_max": 93.41520690917969,
      "activations/layer7_attention_weight_min": -91.073486328125,
      "activations/layer8_attention_weight_max": 44.179603576660156,
      "activations/layer8_attention_weight_min": -43.04835891723633,
      "activations/layer9_attention_weight_max": 44.51878356933594,
      "activations/layer9_attention_weight_min": -37.1041145324707,
      "epoch": 5.44,
      "learning_rate": 0.00011611363636363635,
      "loss": 2.8272,
      "step": 93550
    },
    {
      "activations/layer0_attention_weight_max": 16.94150733947754,
      "activations/layer0_attention_weight_min": -13.037313461303711,
      "activations/layer10_attention_weight_max": 35.284202575683594,
      "activations/layer10_attention_weight_min": -31.705678939819336,
      "activations/layer11_attention_weight_max": 31.415632247924805,
      "activations/layer11_attention_weight_min": -29.438581466674805,
      "activations/layer12_attention_weight_max": 22.670955657958984,
      "activations/layer12_attention_weight_min": -24.393314361572266,
      "activations/layer13_attention_weight_max": 42.68640899658203,
      "activations/layer13_attention_weight_min": -30.162141799926758,
      "activations/layer14_attention_weight_max": 56.764827728271484,
      "activations/layer14_attention_weight_min": -39.40336608886719,
      "activations/layer15_attention_weight_max": 38.51943588256836,
      "activations/layer15_attention_weight_min": -26.474618911743164,
      "activations/layer16_attention_weight_max": 36.42515182495117,
      "activations/layer16_attention_weight_min": -27.41911506652832,
      "activations/layer17_attention_weight_max": 37.11383056640625,
      "activations/layer17_attention_weight_min": -23.038724899291992,
      "activations/layer18_attention_weight_max": 30.635061264038086,
      "activations/layer18_attention_weight_min": -18.29702377319336,
      "activations/layer19_attention_weight_max": 39.23915481567383,
      "activations/layer19_attention_weight_min": -28.021574020385742,
      "activations/layer1_attention_weight_max": 17.50905418395996,
      "activations/layer1_attention_weight_min": -14.787078857421875,
      "activations/layer20_attention_weight_max": 32.30430603027344,
      "activations/layer20_attention_weight_min": -22.8530216217041,
      "activations/layer21_attention_weight_max": 28.891197204589844,
      "activations/layer21_attention_weight_min": -20.792037963867188,
      "activations/layer22_attention_weight_max": 48.77265167236328,
      "activations/layer22_attention_weight_min": -25.713687896728516,
      "activations/layer23_attention_weight_max": 32.03162384033203,
      "activations/layer23_attention_weight_min": -19.379131317138672,
      "activations/layer2_attention_weight_max": 31.116018295288086,
      "activations/layer2_attention_weight_min": -31.057483673095703,
      "activations/layer3_attention_weight_max": 87.08103942871094,
      "activations/layer3_attention_weight_min": -87.3445816040039,
      "activations/layer4_attention_weight_max": 130.93936157226562,
      "activations/layer4_attention_weight_min": -126.95647430419922,
      "activations/layer5_attention_weight_max": 79.62916564941406,
      "activations/layer5_attention_weight_min": -77.49713134765625,
      "activations/layer6_attention_weight_max": 53.86454391479492,
      "activations/layer6_attention_weight_min": -55.51677322387695,
      "activations/layer7_attention_weight_max": 83.83287048339844,
      "activations/layer7_attention_weight_min": -85.09172058105469,
      "activations/layer8_attention_weight_max": 40.43739700317383,
      "activations/layer8_attention_weight_min": -41.070152282714844,
      "activations/layer9_attention_weight_max": 39.4239387512207,
      "activations/layer9_attention_weight_min": -34.914390563964844,
      "epoch": 5.44,
      "learning_rate": 0.00011609469696969697,
      "loss": 2.8297,
      "step": 93600
    },
    {
      "activations/layer0_attention_weight_max": 15.278820037841797,
      "activations/layer0_attention_weight_min": -11.84786319732666,
      "activations/layer10_attention_weight_max": 34.009944915771484,
      "activations/layer10_attention_weight_min": -30.046371459960938,
      "activations/layer11_attention_weight_max": 30.19964599609375,
      "activations/layer11_attention_weight_min": -27.306310653686523,
      "activations/layer12_attention_weight_max": 43.476558685302734,
      "activations/layer12_attention_weight_min": -32.028629302978516,
      "activations/layer13_attention_weight_max": 42.71937561035156,
      "activations/layer13_attention_weight_min": -33.74045944213867,
      "activations/layer14_attention_weight_max": 61.94795227050781,
      "activations/layer14_attention_weight_min": -44.17414474487305,
      "activations/layer15_attention_weight_max": 39.55341720581055,
      "activations/layer15_attention_weight_min": -27.320541381835938,
      "activations/layer16_attention_weight_max": 37.094581604003906,
      "activations/layer16_attention_weight_min": -26.584178924560547,
      "activations/layer17_attention_weight_max": 37.597259521484375,
      "activations/layer17_attention_weight_min": -23.755807876586914,
      "activations/layer18_attention_weight_max": 30.194931030273438,
      "activations/layer18_attention_weight_min": -20.18864631652832,
      "activations/layer19_attention_weight_max": 37.95127487182617,
      "activations/layer19_attention_weight_min": -28.74310874938965,
      "activations/layer1_attention_weight_max": 17.07761001586914,
      "activations/layer1_attention_weight_min": -14.098109245300293,
      "activations/layer20_attention_weight_max": 32.23466873168945,
      "activations/layer20_attention_weight_min": -23.232379913330078,
      "activations/layer21_attention_weight_max": 31.400314331054688,
      "activations/layer21_attention_weight_min": -21.48795509338379,
      "activations/layer22_attention_weight_max": 51.515079498291016,
      "activations/layer22_attention_weight_min": -26.192970275878906,
      "activations/layer23_attention_weight_max": 35.231407165527344,
      "activations/layer23_attention_weight_min": -21.18705177307129,
      "activations/layer2_attention_weight_max": 32.618812561035156,
      "activations/layer2_attention_weight_min": -32.56108856201172,
      "activations/layer3_attention_weight_max": 91.07550811767578,
      "activations/layer3_attention_weight_min": -99.22286224365234,
      "activations/layer4_attention_weight_max": 128.41390991210938,
      "activations/layer4_attention_weight_min": -128.95909118652344,
      "activations/layer5_attention_weight_max": 75.81669616699219,
      "activations/layer5_attention_weight_min": -73.64591217041016,
      "activations/layer6_attention_weight_max": 55.60410690307617,
      "activations/layer6_attention_weight_min": -55.87256622314453,
      "activations/layer7_attention_weight_max": 82.645751953125,
      "activations/layer7_attention_weight_min": -86.44523620605469,
      "activations/layer8_attention_weight_max": 38.687156677246094,
      "activations/layer8_attention_weight_min": -37.778446197509766,
      "activations/layer9_attention_weight_max": 35.41209030151367,
      "activations/layer9_attention_weight_min": -33.09832000732422,
      "epoch": 5.44,
      "learning_rate": 0.00011607575757575757,
      "loss": 2.8387,
      "step": 93650
    },
    {
      "activations/layer0_attention_weight_max": 16.04903221130371,
      "activations/layer0_attention_weight_min": -12.124472618103027,
      "activations/layer10_attention_weight_max": 36.15628433227539,
      "activations/layer10_attention_weight_min": -30.696622848510742,
      "activations/layer11_attention_weight_max": 33.33637619018555,
      "activations/layer11_attention_weight_min": -28.034969329833984,
      "activations/layer12_attention_weight_max": 32.17781066894531,
      "activations/layer12_attention_weight_min": -29.746173858642578,
      "activations/layer13_attention_weight_max": 40.48552703857422,
      "activations/layer13_attention_weight_min": -32.75095748901367,
      "activations/layer14_attention_weight_max": 59.57672882080078,
      "activations/layer14_attention_weight_min": -45.209022521972656,
      "activations/layer15_attention_weight_max": 37.02256774902344,
      "activations/layer15_attention_weight_min": -30.446632385253906,
      "activations/layer16_attention_weight_max": 36.48484802246094,
      "activations/layer16_attention_weight_min": -27.701025009155273,
      "activations/layer17_attention_weight_max": 40.566402435302734,
      "activations/layer17_attention_weight_min": -24.995670318603516,
      "activations/layer18_attention_weight_max": 28.550106048583984,
      "activations/layer18_attention_weight_min": -19.80113983154297,
      "activations/layer19_attention_weight_max": 40.45915985107422,
      "activations/layer19_attention_weight_min": -29.91077995300293,
      "activations/layer1_attention_weight_max": 16.542095184326172,
      "activations/layer1_attention_weight_min": -17.694290161132812,
      "activations/layer20_attention_weight_max": 34.07761001586914,
      "activations/layer20_attention_weight_min": -24.200746536254883,
      "activations/layer21_attention_weight_max": 28.98806381225586,
      "activations/layer21_attention_weight_min": -21.561124801635742,
      "activations/layer22_attention_weight_max": 51.52099609375,
      "activations/layer22_attention_weight_min": -29.68124008178711,
      "activations/layer23_attention_weight_max": 35.59974670410156,
      "activations/layer23_attention_weight_min": -20.489967346191406,
      "activations/layer2_attention_weight_max": 32.50083923339844,
      "activations/layer2_attention_weight_min": -32.1596565246582,
      "activations/layer3_attention_weight_max": 95.8824691772461,
      "activations/layer3_attention_weight_min": -90.72908020019531,
      "activations/layer4_attention_weight_max": 130.42236328125,
      "activations/layer4_attention_weight_min": -127.01039123535156,
      "activations/layer5_attention_weight_max": 79.87423706054688,
      "activations/layer5_attention_weight_min": -77.10897827148438,
      "activations/layer6_attention_weight_max": 56.5936279296875,
      "activations/layer6_attention_weight_min": -56.54759216308594,
      "activations/layer7_attention_weight_max": 87.8885726928711,
      "activations/layer7_attention_weight_min": -89.12227630615234,
      "activations/layer8_attention_weight_max": 42.1396369934082,
      "activations/layer8_attention_weight_min": -41.161354064941406,
      "activations/layer9_attention_weight_max": 38.93348693847656,
      "activations/layer9_attention_weight_min": -34.46306610107422,
      "epoch": 5.44,
      "learning_rate": 0.00011605681818181817,
      "loss": 2.8384,
      "step": 93700
    },
    {
      "activations/layer0_attention_weight_max": 16.487632751464844,
      "activations/layer0_attention_weight_min": -11.572370529174805,
      "activations/layer10_attention_weight_max": 31.028656005859375,
      "activations/layer10_attention_weight_min": -28.632707595825195,
      "activations/layer11_attention_weight_max": 31.846715927124023,
      "activations/layer11_attention_weight_min": -26.561595916748047,
      "activations/layer12_attention_weight_max": 28.959413528442383,
      "activations/layer12_attention_weight_min": -29.130380630493164,
      "activations/layer13_attention_weight_max": 41.50880432128906,
      "activations/layer13_attention_weight_min": -31.68532943725586,
      "activations/layer14_attention_weight_max": 64.14436340332031,
      "activations/layer14_attention_weight_min": -49.81398010253906,
      "activations/layer15_attention_weight_max": 42.78946304321289,
      "activations/layer15_attention_weight_min": -28.38141632080078,
      "activations/layer16_attention_weight_max": 37.031394958496094,
      "activations/layer16_attention_weight_min": -27.09708023071289,
      "activations/layer17_attention_weight_max": 39.493534088134766,
      "activations/layer17_attention_weight_min": -24.755823135375977,
      "activations/layer18_attention_weight_max": 31.364084243774414,
      "activations/layer18_attention_weight_min": -19.999204635620117,
      "activations/layer19_attention_weight_max": 38.2051887512207,
      "activations/layer19_attention_weight_min": -27.97212791442871,
      "activations/layer1_attention_weight_max": 18.070547103881836,
      "activations/layer1_attention_weight_min": -14.302053451538086,
      "activations/layer20_attention_weight_max": 32.07504653930664,
      "activations/layer20_attention_weight_min": -22.393768310546875,
      "activations/layer21_attention_weight_max": 33.17913055419922,
      "activations/layer21_attention_weight_min": -21.569509506225586,
      "activations/layer22_attention_weight_max": 49.31509780883789,
      "activations/layer22_attention_weight_min": -27.685876846313477,
      "activations/layer23_attention_weight_max": 32.6980094909668,
      "activations/layer23_attention_weight_min": -21.326269149780273,
      "activations/layer2_attention_weight_max": 32.800819396972656,
      "activations/layer2_attention_weight_min": -31.436559677124023,
      "activations/layer3_attention_weight_max": 85.84745025634766,
      "activations/layer3_attention_weight_min": -86.52765655517578,
      "activations/layer4_attention_weight_max": 122.56450653076172,
      "activations/layer4_attention_weight_min": -120.18391418457031,
      "activations/layer5_attention_weight_max": 73.50836944580078,
      "activations/layer5_attention_weight_min": -73.42291259765625,
      "activations/layer6_attention_weight_max": 52.441261291503906,
      "activations/layer6_attention_weight_min": -55.78728485107422,
      "activations/layer7_attention_weight_max": 83.38751983642578,
      "activations/layer7_attention_weight_min": -80.42280578613281,
      "activations/layer8_attention_weight_max": 37.389007568359375,
      "activations/layer8_attention_weight_min": -37.50494384765625,
      "activations/layer9_attention_weight_max": 33.57727813720703,
      "activations/layer9_attention_weight_min": -32.16770553588867,
      "epoch": 5.45,
      "learning_rate": 0.00011603787878787878,
      "loss": 2.83,
      "step": 93750
    },
    {
      "activations/layer0_attention_weight_max": 15.214898109436035,
      "activations/layer0_attention_weight_min": -13.433276176452637,
      "activations/layer10_attention_weight_max": 35.43632125854492,
      "activations/layer10_attention_weight_min": -32.53950500488281,
      "activations/layer11_attention_weight_max": 35.23099136352539,
      "activations/layer11_attention_weight_min": -29.254222869873047,
      "activations/layer12_attention_weight_max": 28.216623306274414,
      "activations/layer12_attention_weight_min": -26.958864212036133,
      "activations/layer13_attention_weight_max": 47.062374114990234,
      "activations/layer13_attention_weight_min": -33.670318603515625,
      "activations/layer14_attention_weight_max": 58.866146087646484,
      "activations/layer14_attention_weight_min": -45.31667709350586,
      "activations/layer15_attention_weight_max": 44.927433013916016,
      "activations/layer15_attention_weight_min": -32.80313491821289,
      "activations/layer16_attention_weight_max": 40.489540100097656,
      "activations/layer16_attention_weight_min": -29.0799560546875,
      "activations/layer17_attention_weight_max": 44.56781005859375,
      "activations/layer17_attention_weight_min": -25.07110023498535,
      "activations/layer18_attention_weight_max": 35.51968002319336,
      "activations/layer18_attention_weight_min": -23.343629837036133,
      "activations/layer19_attention_weight_max": 43.72694778442383,
      "activations/layer19_attention_weight_min": -28.70368003845215,
      "activations/layer1_attention_weight_max": 18.313587188720703,
      "activations/layer1_attention_weight_min": -14.377729415893555,
      "activations/layer20_attention_weight_max": 35.08005142211914,
      "activations/layer20_attention_weight_min": -23.09032440185547,
      "activations/layer21_attention_weight_max": 35.41899108886719,
      "activations/layer21_attention_weight_min": -21.7683162689209,
      "activations/layer22_attention_weight_max": 54.85164260864258,
      "activations/layer22_attention_weight_min": -27.974853515625,
      "activations/layer23_attention_weight_max": 38.569602966308594,
      "activations/layer23_attention_weight_min": -24.0725154876709,
      "activations/layer2_attention_weight_max": 34.47688293457031,
      "activations/layer2_attention_weight_min": -32.02967071533203,
      "activations/layer3_attention_weight_max": 94.13554382324219,
      "activations/layer3_attention_weight_min": -93.14237976074219,
      "activations/layer4_attention_weight_max": 130.0558624267578,
      "activations/layer4_attention_weight_min": -127.63671875,
      "activations/layer5_attention_weight_max": 79.731201171875,
      "activations/layer5_attention_weight_min": -79.1102523803711,
      "activations/layer6_attention_weight_max": 57.64771270751953,
      "activations/layer6_attention_weight_min": -60.53785705566406,
      "activations/layer7_attention_weight_max": 86.84056091308594,
      "activations/layer7_attention_weight_min": -91.56532287597656,
      "activations/layer8_attention_weight_max": 43.11748123168945,
      "activations/layer8_attention_weight_min": -42.67390823364258,
      "activations/layer9_attention_weight_max": 38.32499694824219,
      "activations/layer9_attention_weight_min": -35.93899917602539,
      "epoch": 5.45,
      "learning_rate": 0.00011601893939393939,
      "loss": 2.8321,
      "step": 93800
    },
    {
      "activations/layer0_attention_weight_max": 16.439178466796875,
      "activations/layer0_attention_weight_min": -11.91392707824707,
      "activations/layer10_attention_weight_max": 38.2852668762207,
      "activations/layer10_attention_weight_min": -29.153371810913086,
      "activations/layer11_attention_weight_max": 38.71212387084961,
      "activations/layer11_attention_weight_min": -27.962451934814453,
      "activations/layer12_attention_weight_max": 43.38485336303711,
      "activations/layer12_attention_weight_min": -27.115345001220703,
      "activations/layer13_attention_weight_max": 42.56330871582031,
      "activations/layer13_attention_weight_min": -30.58107566833496,
      "activations/layer14_attention_weight_max": 57.038818359375,
      "activations/layer14_attention_weight_min": -46.35859298706055,
      "activations/layer15_attention_weight_max": 35.10671615600586,
      "activations/layer15_attention_weight_min": -28.394901275634766,
      "activations/layer16_attention_weight_max": 36.58823776245117,
      "activations/layer16_attention_weight_min": -28.32343864440918,
      "activations/layer17_attention_weight_max": 37.67151641845703,
      "activations/layer17_attention_weight_min": -25.178133010864258,
      "activations/layer18_attention_weight_max": 33.95690155029297,
      "activations/layer18_attention_weight_min": -20.779157638549805,
      "activations/layer19_attention_weight_max": 38.33171463012695,
      "activations/layer19_attention_weight_min": -28.96710205078125,
      "activations/layer1_attention_weight_max": 17.068349838256836,
      "activations/layer1_attention_weight_min": -14.196637153625488,
      "activations/layer20_attention_weight_max": 30.800430297851562,
      "activations/layer20_attention_weight_min": -23.591352462768555,
      "activations/layer21_attention_weight_max": 32.81035232543945,
      "activations/layer21_attention_weight_min": -20.258594512939453,
      "activations/layer22_attention_weight_max": 48.99563217163086,
      "activations/layer22_attention_weight_min": -28.13323974609375,
      "activations/layer23_attention_weight_max": 36.242340087890625,
      "activations/layer23_attention_weight_min": -20.23961639404297,
      "activations/layer2_attention_weight_max": 33.77882766723633,
      "activations/layer2_attention_weight_min": -32.825679779052734,
      "activations/layer3_attention_weight_max": 89.6386489868164,
      "activations/layer3_attention_weight_min": -92.47481536865234,
      "activations/layer4_attention_weight_max": 128.7338104248047,
      "activations/layer4_attention_weight_min": -129.12400817871094,
      "activations/layer5_attention_weight_max": 77.26852416992188,
      "activations/layer5_attention_weight_min": -76.08387756347656,
      "activations/layer6_attention_weight_max": 54.45603561401367,
      "activations/layer6_attention_weight_min": -56.22032928466797,
      "activations/layer7_attention_weight_max": 84.2171859741211,
      "activations/layer7_attention_weight_min": -83.26856231689453,
      "activations/layer8_attention_weight_max": 39.904518127441406,
      "activations/layer8_attention_weight_min": -39.07363510131836,
      "activations/layer9_attention_weight_max": 36.05060577392578,
      "activations/layer9_attention_weight_min": -35.46788024902344,
      "epoch": 5.45,
      "learning_rate": 0.00011599999999999999,
      "loss": 2.852,
      "step": 93850
    },
    {
      "activations/layer0_attention_weight_max": 16.18128204345703,
      "activations/layer0_attention_weight_min": -11.92354965209961,
      "activations/layer10_attention_weight_max": 34.4406852722168,
      "activations/layer10_attention_weight_min": -30.06771469116211,
      "activations/layer11_attention_weight_max": 32.484230041503906,
      "activations/layer11_attention_weight_min": -27.756736755371094,
      "activations/layer12_attention_weight_max": 52.187984466552734,
      "activations/layer12_attention_weight_min": -30.170913696289062,
      "activations/layer13_attention_weight_max": 46.28640365600586,
      "activations/layer13_attention_weight_min": -31.89324188232422,
      "activations/layer14_attention_weight_max": 68.77005004882812,
      "activations/layer14_attention_weight_min": -50.693084716796875,
      "activations/layer15_attention_weight_max": 40.287418365478516,
      "activations/layer15_attention_weight_min": -30.50192642211914,
      "activations/layer16_attention_weight_max": 38.0367317199707,
      "activations/layer16_attention_weight_min": -29.090938568115234,
      "activations/layer17_attention_weight_max": 39.963226318359375,
      "activations/layer17_attention_weight_min": -24.28090476989746,
      "activations/layer18_attention_weight_max": 33.12690734863281,
      "activations/layer18_attention_weight_min": -20.475175857543945,
      "activations/layer19_attention_weight_max": 39.53279113769531,
      "activations/layer19_attention_weight_min": -30.49936294555664,
      "activations/layer1_attention_weight_max": 17.02433204650879,
      "activations/layer1_attention_weight_min": -13.520942687988281,
      "activations/layer20_attention_weight_max": 32.70590591430664,
      "activations/layer20_attention_weight_min": -22.993791580200195,
      "activations/layer21_attention_weight_max": 30.53452491760254,
      "activations/layer21_attention_weight_min": -20.147253036499023,
      "activations/layer22_attention_weight_max": 49.158203125,
      "activations/layer22_attention_weight_min": -30.093862533569336,
      "activations/layer23_attention_weight_max": 34.275970458984375,
      "activations/layer23_attention_weight_min": -22.37093734741211,
      "activations/layer2_attention_weight_max": 32.130638122558594,
      "activations/layer2_attention_weight_min": -32.86741638183594,
      "activations/layer3_attention_weight_max": 87.01982116699219,
      "activations/layer3_attention_weight_min": -90.93756866455078,
      "activations/layer4_attention_weight_max": 125.23125457763672,
      "activations/layer4_attention_weight_min": -124.72242736816406,
      "activations/layer5_attention_weight_max": 76.55412292480469,
      "activations/layer5_attention_weight_min": -73.57070922851562,
      "activations/layer6_attention_weight_max": 54.26406478881836,
      "activations/layer6_attention_weight_min": -56.4586067199707,
      "activations/layer7_attention_weight_max": 85.17357635498047,
      "activations/layer7_attention_weight_min": -84.39476013183594,
      "activations/layer8_attention_weight_max": 42.02347183227539,
      "activations/layer8_attention_weight_min": -37.468387603759766,
      "activations/layer9_attention_weight_max": 35.52291488647461,
      "activations/layer9_attention_weight_min": -33.432865142822266,
      "epoch": 5.46,
      "learning_rate": 0.0001159810606060606,
      "loss": 2.844,
      "step": 93900
    },
    {
      "activations/layer0_attention_weight_max": 17.580947875976562,
      "activations/layer0_attention_weight_min": -11.724189758300781,
      "activations/layer10_attention_weight_max": 30.48274803161621,
      "activations/layer10_attention_weight_min": -30.4053955078125,
      "activations/layer11_attention_weight_max": 28.703649520874023,
      "activations/layer11_attention_weight_min": -27.429210662841797,
      "activations/layer12_attention_weight_max": 24.986845016479492,
      "activations/layer12_attention_weight_min": -25.820735931396484,
      "activations/layer13_attention_weight_max": 41.223182678222656,
      "activations/layer13_attention_weight_min": -34.32103729248047,
      "activations/layer14_attention_weight_max": 64.15062713623047,
      "activations/layer14_attention_weight_min": -48.07120132446289,
      "activations/layer15_attention_weight_max": 39.73875427246094,
      "activations/layer15_attention_weight_min": -27.362552642822266,
      "activations/layer16_attention_weight_max": 36.58932876586914,
      "activations/layer16_attention_weight_min": -28.19997787475586,
      "activations/layer17_attention_weight_max": 39.87407684326172,
      "activations/layer17_attention_weight_min": -23.93613624572754,
      "activations/layer18_attention_weight_max": 36.771759033203125,
      "activations/layer18_attention_weight_min": -21.458389282226562,
      "activations/layer19_attention_weight_max": 39.4369010925293,
      "activations/layer19_attention_weight_min": -30.357969284057617,
      "activations/layer1_attention_weight_max": 18.686330795288086,
      "activations/layer1_attention_weight_min": -14.752801895141602,
      "activations/layer20_attention_weight_max": 31.5156307220459,
      "activations/layer20_attention_weight_min": -24.346290588378906,
      "activations/layer21_attention_weight_max": 35.0545654296875,
      "activations/layer21_attention_weight_min": -21.035499572753906,
      "activations/layer22_attention_weight_max": 50.00556945800781,
      "activations/layer22_attention_weight_min": -27.79765510559082,
      "activations/layer23_attention_weight_max": 35.84770965576172,
      "activations/layer23_attention_weight_min": -21.78577995300293,
      "activations/layer2_attention_weight_max": 32.174930572509766,
      "activations/layer2_attention_weight_min": -31.141063690185547,
      "activations/layer3_attention_weight_max": 89.7621841430664,
      "activations/layer3_attention_weight_min": -92.74530029296875,
      "activations/layer4_attention_weight_max": 126.70377349853516,
      "activations/layer4_attention_weight_min": -123.25141906738281,
      "activations/layer5_attention_weight_max": 78.71166229248047,
      "activations/layer5_attention_weight_min": -73.92342376708984,
      "activations/layer6_attention_weight_max": 53.38962173461914,
      "activations/layer6_attention_weight_min": -55.77626419067383,
      "activations/layer7_attention_weight_max": 83.99334716796875,
      "activations/layer7_attention_weight_min": -85.78253936767578,
      "activations/layer8_attention_weight_max": 37.95012283325195,
      "activations/layer8_attention_weight_min": -37.206119537353516,
      "activations/layer9_attention_weight_max": 34.827632904052734,
      "activations/layer9_attention_weight_min": -31.199331283569336,
      "epoch": 5.46,
      "learning_rate": 0.00011596212121212119,
      "loss": 2.8427,
      "step": 93950
    },
    {
      "activations/layer0_attention_weight_max": 16.30512046813965,
      "activations/layer0_attention_weight_min": -11.663859367370605,
      "activations/layer10_attention_weight_max": 30.11727523803711,
      "activations/layer10_attention_weight_min": -32.141265869140625,
      "activations/layer11_attention_weight_max": 32.96571350097656,
      "activations/layer11_attention_weight_min": -27.945720672607422,
      "activations/layer12_attention_weight_max": 26.933032989501953,
      "activations/layer12_attention_weight_min": -29.477663040161133,
      "activations/layer13_attention_weight_max": 37.336204528808594,
      "activations/layer13_attention_weight_min": -31.689090728759766,
      "activations/layer14_attention_weight_max": 53.116573333740234,
      "activations/layer14_attention_weight_min": -41.974342346191406,
      "activations/layer15_attention_weight_max": 34.032466888427734,
      "activations/layer15_attention_weight_min": -28.29770278930664,
      "activations/layer16_attention_weight_max": 35.45951843261719,
      "activations/layer16_attention_weight_min": -29.298049926757812,
      "activations/layer17_attention_weight_max": 37.64883804321289,
      "activations/layer17_attention_weight_min": -25.877201080322266,
      "activations/layer18_attention_weight_max": 32.0984001159668,
      "activations/layer18_attention_weight_min": -21.0598201751709,
      "activations/layer19_attention_weight_max": 40.01542282104492,
      "activations/layer19_attention_weight_min": -27.99403190612793,
      "activations/layer1_attention_weight_max": 17.56329917907715,
      "activations/layer1_attention_weight_min": -15.904206275939941,
      "activations/layer20_attention_weight_max": 32.183998107910156,
      "activations/layer20_attention_weight_min": -22.549570083618164,
      "activations/layer21_attention_weight_max": 30.88393211364746,
      "activations/layer21_attention_weight_min": -19.894880294799805,
      "activations/layer22_attention_weight_max": 50.97517776489258,
      "activations/layer22_attention_weight_min": -27.459808349609375,
      "activations/layer23_attention_weight_max": 33.501548767089844,
      "activations/layer23_attention_weight_min": -20.43368911743164,
      "activations/layer2_attention_weight_max": 30.478893280029297,
      "activations/layer2_attention_weight_min": -30.604999542236328,
      "activations/layer3_attention_weight_max": 88.17097473144531,
      "activations/layer3_attention_weight_min": -93.17941284179688,
      "activations/layer4_attention_weight_max": 125.05012512207031,
      "activations/layer4_attention_weight_min": -125.7444076538086,
      "activations/layer5_attention_weight_max": 76.006103515625,
      "activations/layer5_attention_weight_min": -74.65762329101562,
      "activations/layer6_attention_weight_max": 53.27989959716797,
      "activations/layer6_attention_weight_min": -54.50965881347656,
      "activations/layer7_attention_weight_max": 80.22379302978516,
      "activations/layer7_attention_weight_min": -85.6881332397461,
      "activations/layer8_attention_weight_max": 37.75405502319336,
      "activations/layer8_attention_weight_min": -39.40700149536133,
      "activations/layer9_attention_weight_max": 34.15143966674805,
      "activations/layer9_attention_weight_min": -33.9451789855957,
      "epoch": 5.46,
      "learning_rate": 0.0001159431818181818,
      "loss": 2.8361,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_loss": 2.794921875,
      "eval_runtime": 8.3896,
      "eval_samples_per_second": 511.821,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_openwebtext_loss": 2.794921875,
      "eval_openwebtext_ppl": 16.36135047271935,
      "eval_openwebtext_runtime": 8.3896,
      "eval_openwebtext_samples_per_second": 511.821,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_wikitext_loss": 3.021484375,
      "eval_wikitext_ppl": 20.521731031400577,
      "eval_wikitext_runtime": 1.9992,
      "eval_wikitext_samples_per_second": 228.09,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_lambada_loss": 2.67578125,
      "eval_lambada_ppl": 14.523692042874359,
      "eval_lambada_runtime": 9.5898,
      "eval_lambada_samples_per_second": 507.726,
      "step": 94000
    },
    {
      "activations/layer0_attention_weight_max": 15.325187683105469,
      "activations/layer0_attention_weight_min": -11.983878135681152,
      "activations/layer10_attention_weight_max": 36.64425277709961,
      "activations/layer10_attention_weight_min": -32.976905822753906,
      "activations/layer11_attention_weight_max": 34.454463958740234,
      "activations/layer11_attention_weight_min": -28.859115600585938,
      "activations/layer12_attention_weight_max": 23.959522247314453,
      "activations/layer12_attention_weight_min": -24.037899017333984,
      "activations/layer13_attention_weight_max": 47.28084945678711,
      "activations/layer13_attention_weight_min": -32.27901077270508,
      "activations/layer14_attention_weight_max": 62.95761489868164,
      "activations/layer14_attention_weight_min": -42.41950225830078,
      "activations/layer15_attention_weight_max": 41.85468292236328,
      "activations/layer15_attention_weight_min": -28.396162033081055,
      "activations/layer16_attention_weight_max": 37.2813720703125,
      "activations/layer16_attention_weight_min": -27.64788818359375,
      "activations/layer17_attention_weight_max": 42.22015380859375,
      "activations/layer17_attention_weight_min": -25.900659561157227,
      "activations/layer18_attention_weight_max": 34.68836212158203,
      "activations/layer18_attention_weight_min": -21.694454193115234,
      "activations/layer19_attention_weight_max": 42.441162109375,
      "activations/layer19_attention_weight_min": -30.37851905822754,
      "activations/layer1_attention_weight_max": 20.825756072998047,
      "activations/layer1_attention_weight_min": -20.154071807861328,
      "activations/layer20_attention_weight_max": 33.44283676147461,
      "activations/layer20_attention_weight_min": -23.896209716796875,
      "activations/layer21_attention_weight_max": 34.83594512939453,
      "activations/layer21_attention_weight_min": -20.90253448486328,
      "activations/layer22_attention_weight_max": 56.76593780517578,
      "activations/layer22_attention_weight_min": -28.120527267456055,
      "activations/layer23_attention_weight_max": 41.670616149902344,
      "activations/layer23_attention_weight_min": -21.081066131591797,
      "activations/layer2_attention_weight_max": 33.482093811035156,
      "activations/layer2_attention_weight_min": -31.129348754882812,
      "activations/layer3_attention_weight_max": 89.7109146118164,
      "activations/layer3_attention_weight_min": -90.70519256591797,
      "activations/layer4_attention_weight_max": 131.5729217529297,
      "activations/layer4_attention_weight_min": -129.39044189453125,
      "activations/layer5_attention_weight_max": 81.87581634521484,
      "activations/layer5_attention_weight_min": -77.98523712158203,
      "activations/layer6_attention_weight_max": 59.13982009887695,
      "activations/layer6_attention_weight_min": -55.61265182495117,
      "activations/layer7_attention_weight_max": 86.83101654052734,
      "activations/layer7_attention_weight_min": -90.55158233642578,
      "activations/layer8_attention_weight_max": 42.069515228271484,
      "activations/layer8_attention_weight_min": -40.7796516418457,
      "activations/layer9_attention_weight_max": 40.48392105102539,
      "activations/layer9_attention_weight_min": -36.03947830200195,
      "epoch": 5.46,
      "learning_rate": 0.00011592424242424242,
      "loss": 2.8148,
      "step": 94050
    },
    {
      "activations/layer0_attention_weight_max": 16.198272705078125,
      "activations/layer0_attention_weight_min": -11.405535697937012,
      "activations/layer10_attention_weight_max": 38.629188537597656,
      "activations/layer10_attention_weight_min": -33.21699905395508,
      "activations/layer11_attention_weight_max": 34.520179748535156,
      "activations/layer11_attention_weight_min": -30.89415740966797,
      "activations/layer12_attention_weight_max": 25.763893127441406,
      "activations/layer12_attention_weight_min": -28.813007354736328,
      "activations/layer13_attention_weight_max": 47.409080505371094,
      "activations/layer13_attention_weight_min": -35.01708221435547,
      "activations/layer14_attention_weight_max": 78.47505187988281,
      "activations/layer14_attention_weight_min": -56.43865203857422,
      "activations/layer15_attention_weight_max": 39.58816146850586,
      "activations/layer15_attention_weight_min": -28.497690200805664,
      "activations/layer16_attention_weight_max": 41.63581085205078,
      "activations/layer16_attention_weight_min": -27.050521850585938,
      "activations/layer17_attention_weight_max": 43.4422492980957,
      "activations/layer17_attention_weight_min": -24.658172607421875,
      "activations/layer18_attention_weight_max": 37.19381332397461,
      "activations/layer18_attention_weight_min": -22.929067611694336,
      "activations/layer19_attention_weight_max": 47.599822998046875,
      "activations/layer19_attention_weight_min": -31.04241371154785,
      "activations/layer1_attention_weight_max": 16.891246795654297,
      "activations/layer1_attention_weight_min": -14.793901443481445,
      "activations/layer20_attention_weight_max": 36.74955749511719,
      "activations/layer20_attention_weight_min": -23.942367553710938,
      "activations/layer21_attention_weight_max": 37.94239044189453,
      "activations/layer21_attention_weight_min": -21.455190658569336,
      "activations/layer22_attention_weight_max": 62.69171905517578,
      "activations/layer22_attention_weight_min": -29.563922882080078,
      "activations/layer23_attention_weight_max": 40.25554656982422,
      "activations/layer23_attention_weight_min": -22.411924362182617,
      "activations/layer2_attention_weight_max": 32.624046325683594,
      "activations/layer2_attention_weight_min": -31.057159423828125,
      "activations/layer3_attention_weight_max": 90.29016876220703,
      "activations/layer3_attention_weight_min": -93.33930969238281,
      "activations/layer4_attention_weight_max": 135.6400146484375,
      "activations/layer4_attention_weight_min": -128.39353942871094,
      "activations/layer5_attention_weight_max": 82.40951538085938,
      "activations/layer5_attention_weight_min": -76.33320617675781,
      "activations/layer6_attention_weight_max": 56.513160705566406,
      "activations/layer6_attention_weight_min": -56.11592102050781,
      "activations/layer7_attention_weight_max": 91.58173370361328,
      "activations/layer7_attention_weight_min": -88.40827178955078,
      "activations/layer8_attention_weight_max": 43.541465759277344,
      "activations/layer8_attention_weight_min": -44.35100173950195,
      "activations/layer9_attention_weight_max": 40.276241302490234,
      "activations/layer9_attention_weight_min": -39.041603088378906,
      "epoch": 5.47,
      "learning_rate": 0.00011590530303030302,
      "loss": 2.8325,
      "step": 94100
    },
    {
      "activations/layer0_attention_weight_max": 15.397603034973145,
      "activations/layer0_attention_weight_min": -12.262843132019043,
      "activations/layer10_attention_weight_max": 33.83773422241211,
      "activations/layer10_attention_weight_min": -31.473297119140625,
      "activations/layer11_attention_weight_max": 31.893203735351562,
      "activations/layer11_attention_weight_min": -28.547409057617188,
      "activations/layer12_attention_weight_max": 33.48710250854492,
      "activations/layer12_attention_weight_min": -27.212541580200195,
      "activations/layer13_attention_weight_max": 60.97509002685547,
      "activations/layer13_attention_weight_min": -42.704124450683594,
      "activations/layer14_attention_weight_max": 70.48899841308594,
      "activations/layer14_attention_weight_min": -49.31333923339844,
      "activations/layer15_attention_weight_max": 42.475887298583984,
      "activations/layer15_attention_weight_min": -30.8974666595459,
      "activations/layer16_attention_weight_max": 38.73101806640625,
      "activations/layer16_attention_weight_min": -30.686735153198242,
      "activations/layer17_attention_weight_max": 37.14497756958008,
      "activations/layer17_attention_weight_min": -24.1904239654541,
      "activations/layer18_attention_weight_max": 31.86845588684082,
      "activations/layer18_attention_weight_min": -21.998191833496094,
      "activations/layer19_attention_weight_max": 41.917938232421875,
      "activations/layer19_attention_weight_min": -29.041868209838867,
      "activations/layer1_attention_weight_max": 17.22187042236328,
      "activations/layer1_attention_weight_min": -17.712100982666016,
      "activations/layer20_attention_weight_max": 31.72869300842285,
      "activations/layer20_attention_weight_min": -25.550107955932617,
      "activations/layer21_attention_weight_max": 35.057212829589844,
      "activations/layer21_attention_weight_min": -22.627843856811523,
      "activations/layer22_attention_weight_max": 51.6228141784668,
      "activations/layer22_attention_weight_min": -29.094255447387695,
      "activations/layer23_attention_weight_max": 35.336368560791016,
      "activations/layer23_attention_weight_min": -20.69971466064453,
      "activations/layer2_attention_weight_max": 32.556663513183594,
      "activations/layer2_attention_weight_min": -32.04646301269531,
      "activations/layer3_attention_weight_max": 91.71305084228516,
      "activations/layer3_attention_weight_min": -94.80158233642578,
      "activations/layer4_attention_weight_max": 133.12876892089844,
      "activations/layer4_attention_weight_min": -128.5432586669922,
      "activations/layer5_attention_weight_max": 78.74220275878906,
      "activations/layer5_attention_weight_min": -76.50323486328125,
      "activations/layer6_attention_weight_max": 57.85871887207031,
      "activations/layer6_attention_weight_min": -56.14293670654297,
      "activations/layer7_attention_weight_max": 83.72071838378906,
      "activations/layer7_attention_weight_min": -83.17953491210938,
      "activations/layer8_attention_weight_max": 40.56331253051758,
      "activations/layer8_attention_weight_min": -41.97951126098633,
      "activations/layer9_attention_weight_max": 37.252132415771484,
      "activations/layer9_attention_weight_min": -36.052452087402344,
      "epoch": 5.47,
      "learning_rate": 0.00011588636363636362,
      "loss": 2.8387,
      "step": 94150
    },
    {
      "activations/layer0_attention_weight_max": 16.508548736572266,
      "activations/layer0_attention_weight_min": -12.034168243408203,
      "activations/layer10_attention_weight_max": 30.54948616027832,
      "activations/layer10_attention_weight_min": -32.20973205566406,
      "activations/layer11_attention_weight_max": 31.04364585876465,
      "activations/layer11_attention_weight_min": -27.504650115966797,
      "activations/layer12_attention_weight_max": 24.546531677246094,
      "activations/layer12_attention_weight_min": -31.76919937133789,
      "activations/layer13_attention_weight_max": 37.439453125,
      "activations/layer13_attention_weight_min": -31.45388412475586,
      "activations/layer14_attention_weight_max": 57.752872467041016,
      "activations/layer14_attention_weight_min": -45.67191696166992,
      "activations/layer15_attention_weight_max": 36.427616119384766,
      "activations/layer15_attention_weight_min": -27.20523452758789,
      "activations/layer16_attention_weight_max": 32.54698944091797,
      "activations/layer16_attention_weight_min": -26.384334564208984,
      "activations/layer17_attention_weight_max": 38.0297966003418,
      "activations/layer17_attention_weight_min": -23.09966468811035,
      "activations/layer18_attention_weight_max": 30.243343353271484,
      "activations/layer18_attention_weight_min": -19.12638282775879,
      "activations/layer19_attention_weight_max": 38.4434700012207,
      "activations/layer19_attention_weight_min": -28.368894577026367,
      "activations/layer1_attention_weight_max": 17.08843421936035,
      "activations/layer1_attention_weight_min": -13.96776008605957,
      "activations/layer20_attention_weight_max": 31.92137336730957,
      "activations/layer20_attention_weight_min": -22.748369216918945,
      "activations/layer21_attention_weight_max": 30.8507080078125,
      "activations/layer21_attention_weight_min": -21.097084045410156,
      "activations/layer22_attention_weight_max": 51.02104949951172,
      "activations/layer22_attention_weight_min": -26.58098793029785,
      "activations/layer23_attention_weight_max": 34.6353759765625,
      "activations/layer23_attention_weight_min": -21.271347045898438,
      "activations/layer2_attention_weight_max": 31.04485321044922,
      "activations/layer2_attention_weight_min": -30.919376373291016,
      "activations/layer3_attention_weight_max": 91.93653869628906,
      "activations/layer3_attention_weight_min": -91.18880462646484,
      "activations/layer4_attention_weight_max": 124.88651275634766,
      "activations/layer4_attention_weight_min": -124.08099365234375,
      "activations/layer5_attention_weight_max": 77.76193237304688,
      "activations/layer5_attention_weight_min": -75.29048156738281,
      "activations/layer6_attention_weight_max": 55.291419982910156,
      "activations/layer6_attention_weight_min": -55.21593475341797,
      "activations/layer7_attention_weight_max": 89.04361724853516,
      "activations/layer7_attention_weight_min": -84.38851928710938,
      "activations/layer8_attention_weight_max": 37.4031982421875,
      "activations/layer8_attention_weight_min": -38.66013717651367,
      "activations/layer9_attention_weight_max": 34.3112907409668,
      "activations/layer9_attention_weight_min": -39.759498596191406,
      "epoch": 5.47,
      "learning_rate": 0.00011586742424242424,
      "loss": 2.8311,
      "step": 94200
    },
    {
      "activations/layer0_attention_weight_max": 15.775632858276367,
      "activations/layer0_attention_weight_min": -12.502068519592285,
      "activations/layer10_attention_weight_max": 37.797977447509766,
      "activations/layer10_attention_weight_min": -32.578102111816406,
      "activations/layer11_attention_weight_max": 34.62815856933594,
      "activations/layer11_attention_weight_min": -28.90044403076172,
      "activations/layer12_attention_weight_max": 25.89560890197754,
      "activations/layer12_attention_weight_min": -27.03888511657715,
      "activations/layer13_attention_weight_max": 49.26292037963867,
      "activations/layer13_attention_weight_min": -34.572662353515625,
      "activations/layer14_attention_weight_max": 60.232460021972656,
      "activations/layer14_attention_weight_min": -43.0369987487793,
      "activations/layer15_attention_weight_max": 39.905174255371094,
      "activations/layer15_attention_weight_min": -28.347219467163086,
      "activations/layer16_attention_weight_max": 41.373291015625,
      "activations/layer16_attention_weight_min": -28.8026123046875,
      "activations/layer17_attention_weight_max": 42.11555480957031,
      "activations/layer17_attention_weight_min": -24.891347885131836,
      "activations/layer18_attention_weight_max": 36.55805206298828,
      "activations/layer18_attention_weight_min": -21.69811248779297,
      "activations/layer19_attention_weight_max": 47.751914978027344,
      "activations/layer19_attention_weight_min": -29.504241943359375,
      "activations/layer1_attention_weight_max": 17.256366729736328,
      "activations/layer1_attention_weight_min": -18.301076889038086,
      "activations/layer20_attention_weight_max": 36.517601013183594,
      "activations/layer20_attention_weight_min": -22.985435485839844,
      "activations/layer21_attention_weight_max": 39.19499588012695,
      "activations/layer21_attention_weight_min": -22.673215866088867,
      "activations/layer22_attention_weight_max": 57.09373092651367,
      "activations/layer22_attention_weight_min": -27.240007400512695,
      "activations/layer23_attention_weight_max": 39.275089263916016,
      "activations/layer23_attention_weight_min": -24.388507843017578,
      "activations/layer2_attention_weight_max": 32.341064453125,
      "activations/layer2_attention_weight_min": -32.00917053222656,
      "activations/layer3_attention_weight_max": 94.75324249267578,
      "activations/layer3_attention_weight_min": -96.66773986816406,
      "activations/layer4_attention_weight_max": 135.48326110839844,
      "activations/layer4_attention_weight_min": -135.07086181640625,
      "activations/layer5_attention_weight_max": 84.29236602783203,
      "activations/layer5_attention_weight_min": -78.20631408691406,
      "activations/layer6_attention_weight_max": 59.51893997192383,
      "activations/layer6_attention_weight_min": -61.896095275878906,
      "activations/layer7_attention_weight_max": 97.35842895507812,
      "activations/layer7_attention_weight_min": -90.31216430664062,
      "activations/layer8_attention_weight_max": 47.001041412353516,
      "activations/layer8_attention_weight_min": -44.313072204589844,
      "activations/layer9_attention_weight_max": 38.79487228393555,
      "activations/layer9_attention_weight_min": -36.96172332763672,
      "epoch": 5.48,
      "learning_rate": 0.00011584848484848484,
      "loss": 2.8432,
      "step": 94250
    },
    {
      "activations/layer0_attention_weight_max": 16.247562408447266,
      "activations/layer0_attention_weight_min": -12.198643684387207,
      "activations/layer10_attention_weight_max": 38.537208557128906,
      "activations/layer10_attention_weight_min": -32.49126434326172,
      "activations/layer11_attention_weight_max": 35.86064147949219,
      "activations/layer11_attention_weight_min": -29.589080810546875,
      "activations/layer12_attention_weight_max": 35.42323684692383,
      "activations/layer12_attention_weight_min": -29.04064178466797,
      "activations/layer13_attention_weight_max": 44.293087005615234,
      "activations/layer13_attention_weight_min": -35.998023986816406,
      "activations/layer14_attention_weight_max": 64.5228042602539,
      "activations/layer14_attention_weight_min": -46.82725143432617,
      "activations/layer15_attention_weight_max": 38.922000885009766,
      "activations/layer15_attention_weight_min": -30.38007354736328,
      "activations/layer16_attention_weight_max": 39.1898193359375,
      "activations/layer16_attention_weight_min": -28.59934425354004,
      "activations/layer17_attention_weight_max": 42.66657638549805,
      "activations/layer17_attention_weight_min": -25.500965118408203,
      "activations/layer18_attention_weight_max": 31.805423736572266,
      "activations/layer18_attention_weight_min": -18.80928611755371,
      "activations/layer19_attention_weight_max": 44.25014877319336,
      "activations/layer19_attention_weight_min": -29.848896026611328,
      "activations/layer1_attention_weight_max": 17.534774780273438,
      "activations/layer1_attention_weight_min": -16.040565490722656,
      "activations/layer20_attention_weight_max": 40.24732971191406,
      "activations/layer20_attention_weight_min": -23.995777130126953,
      "activations/layer21_attention_weight_max": 38.118595123291016,
      "activations/layer21_attention_weight_min": -20.399320602416992,
      "activations/layer22_attention_weight_max": 52.02573013305664,
      "activations/layer22_attention_weight_min": -26.4642333984375,
      "activations/layer23_attention_weight_max": 36.337738037109375,
      "activations/layer23_attention_weight_min": -20.821758270263672,
      "activations/layer2_attention_weight_max": 31.960407257080078,
      "activations/layer2_attention_weight_min": -31.57132339477539,
      "activations/layer3_attention_weight_max": 93.04954528808594,
      "activations/layer3_attention_weight_min": -91.41285705566406,
      "activations/layer4_attention_weight_max": 135.78924560546875,
      "activations/layer4_attention_weight_min": -131.12506103515625,
      "activations/layer5_attention_weight_max": 85.75852966308594,
      "activations/layer5_attention_weight_min": -82.47795104980469,
      "activations/layer6_attention_weight_max": 60.8278923034668,
      "activations/layer6_attention_weight_min": -59.41707229614258,
      "activations/layer7_attention_weight_max": 94.18827819824219,
      "activations/layer7_attention_weight_min": -91.5869369506836,
      "activations/layer8_attention_weight_max": 44.792816162109375,
      "activations/layer8_attention_weight_min": -41.786983489990234,
      "activations/layer9_attention_weight_max": 38.773738861083984,
      "activations/layer9_attention_weight_min": -35.87607955932617,
      "epoch": 5.48,
      "learning_rate": 0.00011582954545454544,
      "loss": 2.8096,
      "step": 94300
    },
    {
      "activations/layer0_attention_weight_max": 16.26137351989746,
      "activations/layer0_attention_weight_min": -11.442591667175293,
      "activations/layer10_attention_weight_max": 36.324073791503906,
      "activations/layer10_attention_weight_min": -33.86421203613281,
      "activations/layer11_attention_weight_max": 35.107696533203125,
      "activations/layer11_attention_weight_min": -32.36900329589844,
      "activations/layer12_attention_weight_max": 30.88302230834961,
      "activations/layer12_attention_weight_min": -28.078126907348633,
      "activations/layer13_attention_weight_max": 45.86023712158203,
      "activations/layer13_attention_weight_min": -35.04092025756836,
      "activations/layer14_attention_weight_max": 70.93730926513672,
      "activations/layer14_attention_weight_min": -50.531822204589844,
      "activations/layer15_attention_weight_max": 41.46384048461914,
      "activations/layer15_attention_weight_min": -28.19213104248047,
      "activations/layer16_attention_weight_max": 39.32435989379883,
      "activations/layer16_attention_weight_min": -28.474924087524414,
      "activations/layer17_attention_weight_max": 40.08320617675781,
      "activations/layer17_attention_weight_min": -25.753747940063477,
      "activations/layer18_attention_weight_max": 33.27558517456055,
      "activations/layer18_attention_weight_min": -21.376575469970703,
      "activations/layer19_attention_weight_max": 40.83319091796875,
      "activations/layer19_attention_weight_min": -30.344274520874023,
      "activations/layer1_attention_weight_max": 17.084259033203125,
      "activations/layer1_attention_weight_min": -14.736599922180176,
      "activations/layer20_attention_weight_max": 34.8249397277832,
      "activations/layer20_attention_weight_min": -24.522266387939453,
      "activations/layer21_attention_weight_max": 31.473270416259766,
      "activations/layer21_attention_weight_min": -20.748523712158203,
      "activations/layer22_attention_weight_max": 50.31814956665039,
      "activations/layer22_attention_weight_min": -30.62505340576172,
      "activations/layer23_attention_weight_max": 35.04460525512695,
      "activations/layer23_attention_weight_min": -21.015201568603516,
      "activations/layer2_attention_weight_max": 33.151668548583984,
      "activations/layer2_attention_weight_min": -32.325843811035156,
      "activations/layer3_attention_weight_max": 91.78556060791016,
      "activations/layer3_attention_weight_min": -95.54344940185547,
      "activations/layer4_attention_weight_max": 131.7130889892578,
      "activations/layer4_attention_weight_min": -129.5952911376953,
      "activations/layer5_attention_weight_max": 80.97419738769531,
      "activations/layer5_attention_weight_min": -76.66909790039062,
      "activations/layer6_attention_weight_max": 58.54505157470703,
      "activations/layer6_attention_weight_min": -58.434940338134766,
      "activations/layer7_attention_weight_max": 90.66439056396484,
      "activations/layer7_attention_weight_min": -95.01591491699219,
      "activations/layer8_attention_weight_max": 43.001346588134766,
      "activations/layer8_attention_weight_min": -41.49653244018555,
      "activations/layer9_attention_weight_max": 40.14384078979492,
      "activations/layer9_attention_weight_min": -39.27203369140625,
      "epoch": 5.48,
      "learning_rate": 0.00011581060606060604,
      "loss": 2.8335,
      "step": 94350
    },
    {
      "activations/layer0_attention_weight_max": 16.77768325805664,
      "activations/layer0_attention_weight_min": -11.581061363220215,
      "activations/layer10_attention_weight_max": 33.3013801574707,
      "activations/layer10_attention_weight_min": -31.21507453918457,
      "activations/layer11_attention_weight_max": 31.80245590209961,
      "activations/layer11_attention_weight_min": -28.20183563232422,
      "activations/layer12_attention_weight_max": 25.592248916625977,
      "activations/layer12_attention_weight_min": -28.226224899291992,
      "activations/layer13_attention_weight_max": 45.45548629760742,
      "activations/layer13_attention_weight_min": -31.64775848388672,
      "activations/layer14_attention_weight_max": 57.07089614868164,
      "activations/layer14_attention_weight_min": -40.62329864501953,
      "activations/layer15_attention_weight_max": 41.99884033203125,
      "activations/layer15_attention_weight_min": -28.059051513671875,
      "activations/layer16_attention_weight_max": 34.77928161621094,
      "activations/layer16_attention_weight_min": -26.893871307373047,
      "activations/layer17_attention_weight_max": 36.12605667114258,
      "activations/layer17_attention_weight_min": -24.276927947998047,
      "activations/layer18_attention_weight_max": 30.558706283569336,
      "activations/layer18_attention_weight_min": -20.355642318725586,
      "activations/layer19_attention_weight_max": 44.341766357421875,
      "activations/layer19_attention_weight_min": -31.028244018554688,
      "activations/layer1_attention_weight_max": 17.203760147094727,
      "activations/layer1_attention_weight_min": -14.843131065368652,
      "activations/layer20_attention_weight_max": 33.72882843017578,
      "activations/layer20_attention_weight_min": -24.011577606201172,
      "activations/layer21_attention_weight_max": 30.54275131225586,
      "activations/layer21_attention_weight_min": -19.447534561157227,
      "activations/layer22_attention_weight_max": 52.82395935058594,
      "activations/layer22_attention_weight_min": -27.2574405670166,
      "activations/layer23_attention_weight_max": 35.31118392944336,
      "activations/layer23_attention_weight_min": -21.8349609375,
      "activations/layer2_attention_weight_max": 31.851314544677734,
      "activations/layer2_attention_weight_min": -31.752914428710938,
      "activations/layer3_attention_weight_max": 96.94796752929688,
      "activations/layer3_attention_weight_min": -93.68258666992188,
      "activations/layer4_attention_weight_max": 137.37033081054688,
      "activations/layer4_attention_weight_min": -128.29808044433594,
      "activations/layer5_attention_weight_max": 80.66509246826172,
      "activations/layer5_attention_weight_min": -75.0852279663086,
      "activations/layer6_attention_weight_max": 57.92094421386719,
      "activations/layer6_attention_weight_min": -57.551658630371094,
      "activations/layer7_attention_weight_max": 86.69497680664062,
      "activations/layer7_attention_weight_min": -89.28523254394531,
      "activations/layer8_attention_weight_max": 40.92500305175781,
      "activations/layer8_attention_weight_min": -40.44852828979492,
      "activations/layer9_attention_weight_max": 40.27439880371094,
      "activations/layer9_attention_weight_min": -35.178340911865234,
      "epoch": 5.49,
      "learning_rate": 0.00011579166666666666,
      "loss": 2.8113,
      "step": 94400
    },
    {
      "activations/layer0_attention_weight_max": 16.174806594848633,
      "activations/layer0_attention_weight_min": -13.252340316772461,
      "activations/layer10_attention_weight_max": 32.02312088012695,
      "activations/layer10_attention_weight_min": -30.786331176757812,
      "activations/layer11_attention_weight_max": 31.17036247253418,
      "activations/layer11_attention_weight_min": -26.556724548339844,
      "activations/layer12_attention_weight_max": 46.0638313293457,
      "activations/layer12_attention_weight_min": -28.846220016479492,
      "activations/layer13_attention_weight_max": 42.029258728027344,
      "activations/layer13_attention_weight_min": -30.77195930480957,
      "activations/layer14_attention_weight_max": 48.44804000854492,
      "activations/layer14_attention_weight_min": -39.193546295166016,
      "activations/layer15_attention_weight_max": 36.267242431640625,
      "activations/layer15_attention_weight_min": -28.573646545410156,
      "activations/layer16_attention_weight_max": 35.30492401123047,
      "activations/layer16_attention_weight_min": -28.658952713012695,
      "activations/layer17_attention_weight_max": 35.56748580932617,
      "activations/layer17_attention_weight_min": -23.39615821838379,
      "activations/layer18_attention_weight_max": 31.790287017822266,
      "activations/layer18_attention_weight_min": -20.291379928588867,
      "activations/layer19_attention_weight_max": 41.794036865234375,
      "activations/layer19_attention_weight_min": -28.76555824279785,
      "activations/layer1_attention_weight_max": 17.03014373779297,
      "activations/layer1_attention_weight_min": -15.488496780395508,
      "activations/layer20_attention_weight_max": 35.38535690307617,
      "activations/layer20_attention_weight_min": -22.331787109375,
      "activations/layer21_attention_weight_max": 33.40519332885742,
      "activations/layer21_attention_weight_min": -19.56107521057129,
      "activations/layer22_attention_weight_max": 49.35548400878906,
      "activations/layer22_attention_weight_min": -27.274921417236328,
      "activations/layer23_attention_weight_max": 35.47211456298828,
      "activations/layer23_attention_weight_min": -19.421789169311523,
      "activations/layer2_attention_weight_max": 31.958667755126953,
      "activations/layer2_attention_weight_min": -31.741313934326172,
      "activations/layer3_attention_weight_max": 95.4561996459961,
      "activations/layer3_attention_weight_min": -95.18852996826172,
      "activations/layer4_attention_weight_max": 131.73873901367188,
      "activations/layer4_attention_weight_min": -129.2752227783203,
      "activations/layer5_attention_weight_max": 76.44656372070312,
      "activations/layer5_attention_weight_min": -76.12533569335938,
      "activations/layer6_attention_weight_max": 53.2048225402832,
      "activations/layer6_attention_weight_min": -56.18278121948242,
      "activations/layer7_attention_weight_max": 87.73723602294922,
      "activations/layer7_attention_weight_min": -82.95276641845703,
      "activations/layer8_attention_weight_max": 38.715579986572266,
      "activations/layer8_attention_weight_min": -39.037479400634766,
      "activations/layer9_attention_weight_max": 36.02253723144531,
      "activations/layer9_attention_weight_min": -34.047889709472656,
      "epoch": 5.49,
      "learning_rate": 0.00011577310606060605,
      "loss": 2.8312,
      "step": 94450
    },
    {
      "activations/layer0_attention_weight_max": 16.29616355895996,
      "activations/layer0_attention_weight_min": -10.81470012664795,
      "activations/layer10_attention_weight_max": 34.30945587158203,
      "activations/layer10_attention_weight_min": -29.533933639526367,
      "activations/layer11_attention_weight_max": 31.8262939453125,
      "activations/layer11_attention_weight_min": -27.88675308227539,
      "activations/layer12_attention_weight_max": 27.544174194335938,
      "activations/layer12_attention_weight_min": -29.84502601623535,
      "activations/layer13_attention_weight_max": 46.880455017089844,
      "activations/layer13_attention_weight_min": -33.623348236083984,
      "activations/layer14_attention_weight_max": 51.4909782409668,
      "activations/layer14_attention_weight_min": -39.28316116333008,
      "activations/layer15_attention_weight_max": 39.49342346191406,
      "activations/layer15_attention_weight_min": -27.55191421508789,
      "activations/layer16_attention_weight_max": 36.213680267333984,
      "activations/layer16_attention_weight_min": -26.9854736328125,
      "activations/layer17_attention_weight_max": 39.26696014404297,
      "activations/layer17_attention_weight_min": -23.647361755371094,
      "activations/layer18_attention_weight_max": 37.78389358520508,
      "activations/layer18_attention_weight_min": -20.370479583740234,
      "activations/layer19_attention_weight_max": 42.89455032348633,
      "activations/layer19_attention_weight_min": -30.874130249023438,
      "activations/layer1_attention_weight_max": 17.39655303955078,
      "activations/layer1_attention_weight_min": -14.165792465209961,
      "activations/layer20_attention_weight_max": 35.48819351196289,
      "activations/layer20_attention_weight_min": -23.184978485107422,
      "activations/layer21_attention_weight_max": 36.717529296875,
      "activations/layer21_attention_weight_min": -20.44683265686035,
      "activations/layer22_attention_weight_max": 55.09478759765625,
      "activations/layer22_attention_weight_min": -28.23265838623047,
      "activations/layer23_attention_weight_max": 34.8643798828125,
      "activations/layer23_attention_weight_min": -22.21808624267578,
      "activations/layer2_attention_weight_max": 31.201915740966797,
      "activations/layer2_attention_weight_min": -30.69646453857422,
      "activations/layer3_attention_weight_max": 87.5879898071289,
      "activations/layer3_attention_weight_min": -89.41482543945312,
      "activations/layer4_attention_weight_max": 125.90618133544922,
      "activations/layer4_attention_weight_min": -123.65962982177734,
      "activations/layer5_attention_weight_max": 77.43034362792969,
      "activations/layer5_attention_weight_min": -77.02217102050781,
      "activations/layer6_attention_weight_max": 56.737388610839844,
      "activations/layer6_attention_weight_min": -55.036354064941406,
      "activations/layer7_attention_weight_max": 86.45162963867188,
      "activations/layer7_attention_weight_min": -86.63692474365234,
      "activations/layer8_attention_weight_max": 40.30458068847656,
      "activations/layer8_attention_weight_min": -39.68123245239258,
      "activations/layer9_attention_weight_max": 37.5212516784668,
      "activations/layer9_attention_weight_min": -32.414363861083984,
      "epoch": 5.49,
      "learning_rate": 0.00011575416666666666,
      "loss": 2.8244,
      "step": 94500
    },
    {
      "activations/layer0_attention_weight_max": 16.256322860717773,
      "activations/layer0_attention_weight_min": -12.336977005004883,
      "activations/layer10_attention_weight_max": 32.10783767700195,
      "activations/layer10_attention_weight_min": -33.00049591064453,
      "activations/layer11_attention_weight_max": 32.898128509521484,
      "activations/layer11_attention_weight_min": -28.44985580444336,
      "activations/layer12_attention_weight_max": 28.10672950744629,
      "activations/layer12_attention_weight_min": -28.232057571411133,
      "activations/layer13_attention_weight_max": 41.85469055175781,
      "activations/layer13_attention_weight_min": -31.84807014465332,
      "activations/layer14_attention_weight_max": 56.05607604980469,
      "activations/layer14_attention_weight_min": -41.397403717041016,
      "activations/layer15_attention_weight_max": 40.97140884399414,
      "activations/layer15_attention_weight_min": -29.035385131835938,
      "activations/layer16_attention_weight_max": 38.06303405761719,
      "activations/layer16_attention_weight_min": -27.703622817993164,
      "activations/layer17_attention_weight_max": 40.22496795654297,
      "activations/layer17_attention_weight_min": -24.375036239624023,
      "activations/layer18_attention_weight_max": 32.29039001464844,
      "activations/layer18_attention_weight_min": -19.639263153076172,
      "activations/layer19_attention_weight_max": 41.224609375,
      "activations/layer19_attention_weight_min": -27.58254051208496,
      "activations/layer1_attention_weight_max": 17.277193069458008,
      "activations/layer1_attention_weight_min": -15.76073932647705,
      "activations/layer20_attention_weight_max": 32.84056091308594,
      "activations/layer20_attention_weight_min": -21.987525939941406,
      "activations/layer21_attention_weight_max": 33.28215408325195,
      "activations/layer21_attention_weight_min": -20.93511962890625,
      "activations/layer22_attention_weight_max": 51.63208770751953,
      "activations/layer22_attention_weight_min": -26.50093650817871,
      "activations/layer23_attention_weight_max": 35.82261657714844,
      "activations/layer23_attention_weight_min": -20.881643295288086,
      "activations/layer2_attention_weight_max": 32.80406951904297,
      "activations/layer2_attention_weight_min": -31.644283294677734,
      "activations/layer3_attention_weight_max": 94.40428924560547,
      "activations/layer3_attention_weight_min": -96.36876678466797,
      "activations/layer4_attention_weight_max": 136.66539001464844,
      "activations/layer4_attention_weight_min": -128.86410522460938,
      "activations/layer5_attention_weight_max": 77.68836212158203,
      "activations/layer5_attention_weight_min": -75.38402557373047,
      "activations/layer6_attention_weight_max": 54.19657897949219,
      "activations/layer6_attention_weight_min": -54.93976974487305,
      "activations/layer7_attention_weight_max": 83.9170150756836,
      "activations/layer7_attention_weight_min": -90.71292114257812,
      "activations/layer8_attention_weight_max": 41.71050262451172,
      "activations/layer8_attention_weight_min": -42.30693435668945,
      "activations/layer9_attention_weight_max": 35.4371223449707,
      "activations/layer9_attention_weight_min": -36.20365524291992,
      "epoch": 5.49,
      "learning_rate": 0.00011573522727272725,
      "loss": 2.8158,
      "step": 94550
    },
    {
      "activations/layer0_attention_weight_max": 16.067106246948242,
      "activations/layer0_attention_weight_min": -12.077876091003418,
      "activations/layer10_attention_weight_max": 35.25262451171875,
      "activations/layer10_attention_weight_min": -31.22739028930664,
      "activations/layer11_attention_weight_max": 31.475522994995117,
      "activations/layer11_attention_weight_min": -28.43320083618164,
      "activations/layer12_attention_weight_max": 34.9768180847168,
      "activations/layer12_attention_weight_min": -29.705427169799805,
      "activations/layer13_attention_weight_max": 44.63434600830078,
      "activations/layer13_attention_weight_min": -34.33221435546875,
      "activations/layer14_attention_weight_max": 67.96083068847656,
      "activations/layer14_attention_weight_min": -44.27175521850586,
      "activations/layer15_attention_weight_max": 45.33918762207031,
      "activations/layer15_attention_weight_min": -29.74675178527832,
      "activations/layer16_attention_weight_max": 40.834503173828125,
      "activations/layer16_attention_weight_min": -27.894332885742188,
      "activations/layer17_attention_weight_max": 41.963443756103516,
      "activations/layer17_attention_weight_min": -24.460622787475586,
      "activations/layer18_attention_weight_max": 34.11994552612305,
      "activations/layer18_attention_weight_min": -21.87596321105957,
      "activations/layer19_attention_weight_max": 44.386199951171875,
      "activations/layer19_attention_weight_min": -30.217309951782227,
      "activations/layer1_attention_weight_max": 16.962263107299805,
      "activations/layer1_attention_weight_min": -14.48833179473877,
      "activations/layer20_attention_weight_max": 34.54085922241211,
      "activations/layer20_attention_weight_min": -21.992233276367188,
      "activations/layer21_attention_weight_max": 33.26472854614258,
      "activations/layer21_attention_weight_min": -19.738941192626953,
      "activations/layer22_attention_weight_max": 59.449947357177734,
      "activations/layer22_attention_weight_min": -26.523563385009766,
      "activations/layer23_attention_weight_max": 40.78189468383789,
      "activations/layer23_attention_weight_min": -19.170818328857422,
      "activations/layer2_attention_weight_max": 31.302379608154297,
      "activations/layer2_attention_weight_min": -31.472660064697266,
      "activations/layer3_attention_weight_max": 88.48484802246094,
      "activations/layer3_attention_weight_min": -91.61954498291016,
      "activations/layer4_attention_weight_max": 128.7353057861328,
      "activations/layer4_attention_weight_min": -127.8904800415039,
      "activations/layer5_attention_weight_max": 75.95965576171875,
      "activations/layer5_attention_weight_min": -76.64775085449219,
      "activations/layer6_attention_weight_max": 56.8325080871582,
      "activations/layer6_attention_weight_min": -57.57268142700195,
      "activations/layer7_attention_weight_max": 87.12031555175781,
      "activations/layer7_attention_weight_min": -83.68890380859375,
      "activations/layer8_attention_weight_max": 40.4055061340332,
      "activations/layer8_attention_weight_min": -39.37834548950195,
      "activations/layer9_attention_weight_max": 36.7083854675293,
      "activations/layer9_attention_weight_min": -34.9293327331543,
      "epoch": 5.5,
      "learning_rate": 0.00011571628787878787,
      "loss": 2.8262,
      "step": 94600
    },
    {
      "activations/layer0_attention_weight_max": 16.436721801757812,
      "activations/layer0_attention_weight_min": -11.1605806350708,
      "activations/layer10_attention_weight_max": 31.411569595336914,
      "activations/layer10_attention_weight_min": -29.83080291748047,
      "activations/layer11_attention_weight_max": 32.933799743652344,
      "activations/layer11_attention_weight_min": -26.261932373046875,
      "activations/layer12_attention_weight_max": 23.657135009765625,
      "activations/layer12_attention_weight_min": -26.43790626525879,
      "activations/layer13_attention_weight_max": 38.17884826660156,
      "activations/layer13_attention_weight_min": -30.885229110717773,
      "activations/layer14_attention_weight_max": 58.110652923583984,
      "activations/layer14_attention_weight_min": -49.118186950683594,
      "activations/layer15_attention_weight_max": 43.831642150878906,
      "activations/layer15_attention_weight_min": -27.174062728881836,
      "activations/layer16_attention_weight_max": 37.71127700805664,
      "activations/layer16_attention_weight_min": -28.262643814086914,
      "activations/layer17_attention_weight_max": 46.243316650390625,
      "activations/layer17_attention_weight_min": -24.772790908813477,
      "activations/layer18_attention_weight_max": 35.08954620361328,
      "activations/layer18_attention_weight_min": -21.711278915405273,
      "activations/layer19_attention_weight_max": 41.70689392089844,
      "activations/layer19_attention_weight_min": -30.32553482055664,
      "activations/layer1_attention_weight_max": 16.99910545349121,
      "activations/layer1_attention_weight_min": -15.52968692779541,
      "activations/layer20_attention_weight_max": 39.59831619262695,
      "activations/layer20_attention_weight_min": -22.299442291259766,
      "activations/layer21_attention_weight_max": 33.994781494140625,
      "activations/layer21_attention_weight_min": -22.911340713500977,
      "activations/layer22_attention_weight_max": 56.47757339477539,
      "activations/layer22_attention_weight_min": -30.10314178466797,
      "activations/layer23_attention_weight_max": 35.081417083740234,
      "activations/layer23_attention_weight_min": -21.077417373657227,
      "activations/layer2_attention_weight_max": 33.164283752441406,
      "activations/layer2_attention_weight_min": -31.30831527709961,
      "activations/layer3_attention_weight_max": 97.63583374023438,
      "activations/layer3_attention_weight_min": -96.88424682617188,
      "activations/layer4_attention_weight_max": 137.3049774169922,
      "activations/layer4_attention_weight_min": -128.2281951904297,
      "activations/layer5_attention_weight_max": 77.59883880615234,
      "activations/layer5_attention_weight_min": -73.6178207397461,
      "activations/layer6_attention_weight_max": 55.40455627441406,
      "activations/layer6_attention_weight_min": -53.7213020324707,
      "activations/layer7_attention_weight_max": 89.54181671142578,
      "activations/layer7_attention_weight_min": -83.98696899414062,
      "activations/layer8_attention_weight_max": 39.536643981933594,
      "activations/layer8_attention_weight_min": -38.15752410888672,
      "activations/layer9_attention_weight_max": 34.80084991455078,
      "activations/layer9_attention_weight_min": -33.86212158203125,
      "epoch": 5.5,
      "learning_rate": 0.00011569734848484848,
      "loss": 2.819,
      "step": 94650
    },
    {
      "activations/layer0_attention_weight_max": 16.370574951171875,
      "activations/layer0_attention_weight_min": -12.3439302444458,
      "activations/layer10_attention_weight_max": 32.51832962036133,
      "activations/layer10_attention_weight_min": -29.504314422607422,
      "activations/layer11_attention_weight_max": 30.4199275970459,
      "activations/layer11_attention_weight_min": -26.88532066345215,
      "activations/layer12_attention_weight_max": 30.531246185302734,
      "activations/layer12_attention_weight_min": -26.520137786865234,
      "activations/layer13_attention_weight_max": 41.00825881958008,
      "activations/layer13_attention_weight_min": -31.466407775878906,
      "activations/layer14_attention_weight_max": 65.53181457519531,
      "activations/layer14_attention_weight_min": -47.8294563293457,
      "activations/layer15_attention_weight_max": 38.11430358886719,
      "activations/layer15_attention_weight_min": -27.77054786682129,
      "activations/layer16_attention_weight_max": 37.096431732177734,
      "activations/layer16_attention_weight_min": -26.446304321289062,
      "activations/layer17_attention_weight_max": 37.5166015625,
      "activations/layer17_attention_weight_min": -23.46963119506836,
      "activations/layer18_attention_weight_max": 31.556888580322266,
      "activations/layer18_attention_weight_min": -19.898117065429688,
      "activations/layer19_attention_weight_max": 38.01850891113281,
      "activations/layer19_attention_weight_min": -28.366117477416992,
      "activations/layer1_attention_weight_max": 16.188640594482422,
      "activations/layer1_attention_weight_min": -14.069648742675781,
      "activations/layer20_attention_weight_max": 31.69145393371582,
      "activations/layer20_attention_weight_min": -23.40764045715332,
      "activations/layer21_attention_weight_max": 33.713134765625,
      "activations/layer21_attention_weight_min": -21.3162784576416,
      "activations/layer22_attention_weight_max": 50.65239715576172,
      "activations/layer22_attention_weight_min": -27.47266387939453,
      "activations/layer23_attention_weight_max": 32.71538162231445,
      "activations/layer23_attention_weight_min": -20.80453872680664,
      "activations/layer2_attention_weight_max": 30.771841049194336,
      "activations/layer2_attention_weight_min": -30.49532127380371,
      "activations/layer3_attention_weight_max": 91.23435974121094,
      "activations/layer3_attention_weight_min": -91.19548034667969,
      "activations/layer4_attention_weight_max": 134.29579162597656,
      "activations/layer4_attention_weight_min": -128.2625732421875,
      "activations/layer5_attention_weight_max": 78.8720703125,
      "activations/layer5_attention_weight_min": -75.99757385253906,
      "activations/layer6_attention_weight_max": 55.8713264465332,
      "activations/layer6_attention_weight_min": -56.041603088378906,
      "activations/layer7_attention_weight_max": 86.45030975341797,
      "activations/layer7_attention_weight_min": -84.54716491699219,
      "activations/layer8_attention_weight_max": 40.77933120727539,
      "activations/layer8_attention_weight_min": -42.40184783935547,
      "activations/layer9_attention_weight_max": 37.819400787353516,
      "activations/layer9_attention_weight_min": -34.536624908447266,
      "epoch": 5.5,
      "learning_rate": 0.00011567840909090907,
      "loss": 2.8364,
      "step": 94700
    },
    {
      "activations/layer0_attention_weight_max": 16.541067123413086,
      "activations/layer0_attention_weight_min": -10.877670288085938,
      "activations/layer10_attention_weight_max": 35.875244140625,
      "activations/layer10_attention_weight_min": -31.4688777923584,
      "activations/layer11_attention_weight_max": 34.90900802612305,
      "activations/layer11_attention_weight_min": -29.415111541748047,
      "activations/layer12_attention_weight_max": 41.17402648925781,
      "activations/layer12_attention_weight_min": -26.182437896728516,
      "activations/layer13_attention_weight_max": 62.85569763183594,
      "activations/layer13_attention_weight_min": -33.735252380371094,
      "activations/layer14_attention_weight_max": 57.27571105957031,
      "activations/layer14_attention_weight_min": -42.898597717285156,
      "activations/layer15_attention_weight_max": 45.16758728027344,
      "activations/layer15_attention_weight_min": -28.83194351196289,
      "activations/layer16_attention_weight_max": 39.689598083496094,
      "activations/layer16_attention_weight_min": -27.912446975708008,
      "activations/layer17_attention_weight_max": 40.27589797973633,
      "activations/layer17_attention_weight_min": -25.066064834594727,
      "activations/layer18_attention_weight_max": 30.72205924987793,
      "activations/layer18_attention_weight_min": -19.24982452392578,
      "activations/layer19_attention_weight_max": 40.668174743652344,
      "activations/layer19_attention_weight_min": -30.234018325805664,
      "activations/layer1_attention_weight_max": 18.021059036254883,
      "activations/layer1_attention_weight_min": -14.61694622039795,
      "activations/layer20_attention_weight_max": 35.137577056884766,
      "activations/layer20_attention_weight_min": -24.922889709472656,
      "activations/layer21_attention_weight_max": 31.954227447509766,
      "activations/layer21_attention_weight_min": -20.7652645111084,
      "activations/layer22_attention_weight_max": 52.44625473022461,
      "activations/layer22_attention_weight_min": -28.488889694213867,
      "activations/layer23_attention_weight_max": 34.56709289550781,
      "activations/layer23_attention_weight_min": -20.094707489013672,
      "activations/layer2_attention_weight_max": 31.31435203552246,
      "activations/layer2_attention_weight_min": -30.31746482849121,
      "activations/layer3_attention_weight_max": 89.75743865966797,
      "activations/layer3_attention_weight_min": -88.7753677368164,
      "activations/layer4_attention_weight_max": 126.47159576416016,
      "activations/layer4_attention_weight_min": -119.97428131103516,
      "activations/layer5_attention_weight_max": 77.71261596679688,
      "activations/layer5_attention_weight_min": -73.98593139648438,
      "activations/layer6_attention_weight_max": 56.69514846801758,
      "activations/layer6_attention_weight_min": -54.87403869628906,
      "activations/layer7_attention_weight_max": 83.72675323486328,
      "activations/layer7_attention_weight_min": -88.291015625,
      "activations/layer8_attention_weight_max": 39.207855224609375,
      "activations/layer8_attention_weight_min": -41.02894592285156,
      "activations/layer9_attention_weight_max": 34.356361389160156,
      "activations/layer9_attention_weight_min": -36.696659088134766,
      "epoch": 5.51,
      "learning_rate": 0.00011565946969696968,
      "loss": 2.8331,
      "step": 94750
    },
    {
      "activations/layer0_attention_weight_max": 16.259082794189453,
      "activations/layer0_attention_weight_min": -10.385008811950684,
      "activations/layer10_attention_weight_max": 32.88849639892578,
      "activations/layer10_attention_weight_min": -30.673490524291992,
      "activations/layer11_attention_weight_max": 33.1868782043457,
      "activations/layer11_attention_weight_min": -28.92873191833496,
      "activations/layer12_attention_weight_max": 38.89751434326172,
      "activations/layer12_attention_weight_min": -30.525285720825195,
      "activations/layer13_attention_weight_max": 52.18332290649414,
      "activations/layer13_attention_weight_min": -34.77730941772461,
      "activations/layer14_attention_weight_max": 54.561119079589844,
      "activations/layer14_attention_weight_min": -40.27524948120117,
      "activations/layer15_attention_weight_max": 42.69692611694336,
      "activations/layer15_attention_weight_min": -27.916196823120117,
      "activations/layer16_attention_weight_max": 35.93069839477539,
      "activations/layer16_attention_weight_min": -27.534793853759766,
      "activations/layer17_attention_weight_max": 35.353271484375,
      "activations/layer17_attention_weight_min": -23.67695426940918,
      "activations/layer18_attention_weight_max": 29.02003288269043,
      "activations/layer18_attention_weight_min": -19.53522491455078,
      "activations/layer19_attention_weight_max": 37.61697006225586,
      "activations/layer19_attention_weight_min": -30.246261596679688,
      "activations/layer1_attention_weight_max": 17.929052352905273,
      "activations/layer1_attention_weight_min": -14.559599876403809,
      "activations/layer20_attention_weight_max": 29.098970413208008,
      "activations/layer20_attention_weight_min": -22.18511199951172,
      "activations/layer21_attention_weight_max": 30.615446090698242,
      "activations/layer21_attention_weight_min": -21.376293182373047,
      "activations/layer22_attention_weight_max": 45.90217590332031,
      "activations/layer22_attention_weight_min": -28.051729202270508,
      "activations/layer23_attention_weight_max": 30.23623275756836,
      "activations/layer23_attention_weight_min": -19.25469970703125,
      "activations/layer2_attention_weight_max": 32.5419921875,
      "activations/layer2_attention_weight_min": -30.797842025756836,
      "activations/layer3_attention_weight_max": 91.7799301147461,
      "activations/layer3_attention_weight_min": -95.59703826904297,
      "activations/layer4_attention_weight_max": 140.7466278076172,
      "activations/layer4_attention_weight_min": -134.27330017089844,
      "activations/layer5_attention_weight_max": 80.40890502929688,
      "activations/layer5_attention_weight_min": -76.46907043457031,
      "activations/layer6_attention_weight_max": 54.96526336669922,
      "activations/layer6_attention_weight_min": -55.0279426574707,
      "activations/layer7_attention_weight_max": 88.27136993408203,
      "activations/layer7_attention_weight_min": -84.04136657714844,
      "activations/layer8_attention_weight_max": 40.30906295776367,
      "activations/layer8_attention_weight_min": -39.49226760864258,
      "activations/layer9_attention_weight_max": 36.989200592041016,
      "activations/layer9_attention_weight_min": -40.54146957397461,
      "epoch": 5.51,
      "learning_rate": 0.0001156405303030303,
      "loss": 2.839,
      "step": 94800
    },
    {
      "activations/layer0_attention_weight_max": 16.856199264526367,
      "activations/layer0_attention_weight_min": -11.457854270935059,
      "activations/layer10_attention_weight_max": 33.7996711730957,
      "activations/layer10_attention_weight_min": -31.30876922607422,
      "activations/layer11_attention_weight_max": 31.29505157470703,
      "activations/layer11_attention_weight_min": -29.150470733642578,
      "activations/layer12_attention_weight_max": 24.534927368164062,
      "activations/layer12_attention_weight_min": -26.351057052612305,
      "activations/layer13_attention_weight_max": 55.38929748535156,
      "activations/layer13_attention_weight_min": -38.3580436706543,
      "activations/layer14_attention_weight_max": 63.09435272216797,
      "activations/layer14_attention_weight_min": -50.366817474365234,
      "activations/layer15_attention_weight_max": 38.895111083984375,
      "activations/layer15_attention_weight_min": -28.955463409423828,
      "activations/layer16_attention_weight_max": 36.81325912475586,
      "activations/layer16_attention_weight_min": -27.725788116455078,
      "activations/layer17_attention_weight_max": 37.3604736328125,
      "activations/layer17_attention_weight_min": -24.338592529296875,
      "activations/layer18_attention_weight_max": 32.775108337402344,
      "activations/layer18_attention_weight_min": -24.808496475219727,
      "activations/layer19_attention_weight_max": 39.634124755859375,
      "activations/layer19_attention_weight_min": -31.76557731628418,
      "activations/layer1_attention_weight_max": 16.623069763183594,
      "activations/layer1_attention_weight_min": -15.35338020324707,
      "activations/layer20_attention_weight_max": 30.28490447998047,
      "activations/layer20_attention_weight_min": -23.672941207885742,
      "activations/layer21_attention_weight_max": 27.06584358215332,
      "activations/layer21_attention_weight_min": -20.011615753173828,
      "activations/layer22_attention_weight_max": 48.12973403930664,
      "activations/layer22_attention_weight_min": -28.60479736328125,
      "activations/layer23_attention_weight_max": 33.536155700683594,
      "activations/layer23_attention_weight_min": -21.299211502075195,
      "activations/layer2_attention_weight_max": 30.354278564453125,
      "activations/layer2_attention_weight_min": -30.551742553710938,
      "activations/layer3_attention_weight_max": 92.84514617919922,
      "activations/layer3_attention_weight_min": -91.28852844238281,
      "activations/layer4_attention_weight_max": 125.42134094238281,
      "activations/layer4_attention_weight_min": -123.97962951660156,
      "activations/layer5_attention_weight_max": 77.14151763916016,
      "activations/layer5_attention_weight_min": -73.79667663574219,
      "activations/layer6_attention_weight_max": 54.63119888305664,
      "activations/layer6_attention_weight_min": -55.67655944824219,
      "activations/layer7_attention_weight_max": 84.4280014038086,
      "activations/layer7_attention_weight_min": -92.05795288085938,
      "activations/layer8_attention_weight_max": 40.22602081298828,
      "activations/layer8_attention_weight_min": -41.32780075073242,
      "activations/layer9_attention_weight_max": 37.71730041503906,
      "activations/layer9_attention_weight_min": -36.2749137878418,
      "epoch": 5.51,
      "learning_rate": 0.00011562159090909089,
      "loss": 2.8426,
      "step": 94850
    },
    {
      "activations/layer0_attention_weight_max": 16.426902770996094,
      "activations/layer0_attention_weight_min": -12.322707176208496,
      "activations/layer10_attention_weight_max": 34.94267654418945,
      "activations/layer10_attention_weight_min": -32.85010528564453,
      "activations/layer11_attention_weight_max": 32.685935974121094,
      "activations/layer11_attention_weight_min": -30.26915740966797,
      "activations/layer12_attention_weight_max": 24.492919921875,
      "activations/layer12_attention_weight_min": -25.482213973999023,
      "activations/layer13_attention_weight_max": 44.32516860961914,
      "activations/layer13_attention_weight_min": -33.34954833984375,
      "activations/layer14_attention_weight_max": 59.29655075073242,
      "activations/layer14_attention_weight_min": -44.9812126159668,
      "activations/layer15_attention_weight_max": 49.37588119506836,
      "activations/layer15_attention_weight_min": -31.65904998779297,
      "activations/layer16_attention_weight_max": 37.93038558959961,
      "activations/layer16_attention_weight_min": -27.618816375732422,
      "activations/layer17_attention_weight_max": 36.85619354248047,
      "activations/layer17_attention_weight_min": -24.37912940979004,
      "activations/layer18_attention_weight_max": 35.84095764160156,
      "activations/layer18_attention_weight_min": -23.64616584777832,
      "activations/layer19_attention_weight_max": 37.91587448120117,
      "activations/layer19_attention_weight_min": -29.425954818725586,
      "activations/layer1_attention_weight_max": 17.48230743408203,
      "activations/layer1_attention_weight_min": -15.277990341186523,
      "activations/layer20_attention_weight_max": 32.561702728271484,
      "activations/layer20_attention_weight_min": -24.891345977783203,
      "activations/layer21_attention_weight_max": 31.18632698059082,
      "activations/layer21_attention_weight_min": -21.75925064086914,
      "activations/layer22_attention_weight_max": 46.49100112915039,
      "activations/layer22_attention_weight_min": -27.302793502807617,
      "activations/layer23_attention_weight_max": 33.82299041748047,
      "activations/layer23_attention_weight_min": -20.683502197265625,
      "activations/layer2_attention_weight_max": 31.950666427612305,
      "activations/layer2_attention_weight_min": -31.81366539001465,
      "activations/layer3_attention_weight_max": 92.54566955566406,
      "activations/layer3_attention_weight_min": -93.33221435546875,
      "activations/layer4_attention_weight_max": 131.7929229736328,
      "activations/layer4_attention_weight_min": -132.0169677734375,
      "activations/layer5_attention_weight_max": 81.5444564819336,
      "activations/layer5_attention_weight_min": -78.27677917480469,
      "activations/layer6_attention_weight_max": 55.82368850708008,
      "activations/layer6_attention_weight_min": -60.38145446777344,
      "activations/layer7_attention_weight_max": 91.70462036132812,
      "activations/layer7_attention_weight_min": -89.21865844726562,
      "activations/layer8_attention_weight_max": 41.457557678222656,
      "activations/layer8_attention_weight_min": -42.98476791381836,
      "activations/layer9_attention_weight_max": 38.64849853515625,
      "activations/layer9_attention_weight_min": -36.94688415527344,
      "epoch": 5.51,
      "learning_rate": 0.0001156026515151515,
      "loss": 2.8344,
      "step": 94900
    },
    {
      "activations/layer0_attention_weight_max": 16.746105194091797,
      "activations/layer0_attention_weight_min": -11.685284614562988,
      "activations/layer10_attention_weight_max": 31.85618019104004,
      "activations/layer10_attention_weight_min": -30.684127807617188,
      "activations/layer11_attention_weight_max": 32.18013000488281,
      "activations/layer11_attention_weight_min": -27.829792022705078,
      "activations/layer12_attention_weight_max": 25.704404830932617,
      "activations/layer12_attention_weight_min": -27.519485473632812,
      "activations/layer13_attention_weight_max": 44.2149543762207,
      "activations/layer13_attention_weight_min": -32.25473403930664,
      "activations/layer14_attention_weight_max": 57.19524002075195,
      "activations/layer14_attention_weight_min": -44.16212844848633,
      "activations/layer15_attention_weight_max": 35.462982177734375,
      "activations/layer15_attention_weight_min": -28.91488265991211,
      "activations/layer16_attention_weight_max": 33.046119689941406,
      "activations/layer16_attention_weight_min": -29.958845138549805,
      "activations/layer17_attention_weight_max": 35.25437545776367,
      "activations/layer17_attention_weight_min": -24.376529693603516,
      "activations/layer18_attention_weight_max": 35.120574951171875,
      "activations/layer18_attention_weight_min": -18.79660415649414,
      "activations/layer19_attention_weight_max": 36.39653015136719,
      "activations/layer19_attention_weight_min": -28.58527946472168,
      "activations/layer1_attention_weight_max": 16.593624114990234,
      "activations/layer1_attention_weight_min": -15.543927192687988,
      "activations/layer20_attention_weight_max": 31.95962142944336,
      "activations/layer20_attention_weight_min": -23.78724479675293,
      "activations/layer21_attention_weight_max": 34.562782287597656,
      "activations/layer21_attention_weight_min": -20.179702758789062,
      "activations/layer22_attention_weight_max": 48.19831848144531,
      "activations/layer22_attention_weight_min": -29.6154842376709,
      "activations/layer23_attention_weight_max": 33.61735916137695,
      "activations/layer23_attention_weight_min": -19.62489891052246,
      "activations/layer2_attention_weight_max": 32.879432678222656,
      "activations/layer2_attention_weight_min": -32.29533767700195,
      "activations/layer3_attention_weight_max": 94.97311401367188,
      "activations/layer3_attention_weight_min": -96.0130844116211,
      "activations/layer4_attention_weight_max": 135.67684936523438,
      "activations/layer4_attention_weight_min": -130.7145233154297,
      "activations/layer5_attention_weight_max": 79.35881805419922,
      "activations/layer5_attention_weight_min": -76.81986999511719,
      "activations/layer6_attention_weight_max": 57.77210998535156,
      "activations/layer6_attention_weight_min": -58.52289962768555,
      "activations/layer7_attention_weight_max": 88.87588500976562,
      "activations/layer7_attention_weight_min": -87.31456756591797,
      "activations/layer8_attention_weight_max": 41.436279296875,
      "activations/layer8_attention_weight_min": -40.16173553466797,
      "activations/layer9_attention_weight_max": 36.26450729370117,
      "activations/layer9_attention_weight_min": -34.5422477722168,
      "epoch": 5.52,
      "learning_rate": 0.00011558371212121212,
      "loss": 2.8303,
      "step": 94950
    },
    {
      "activations/layer0_attention_weight_max": 15.605361938476562,
      "activations/layer0_attention_weight_min": -11.90103530883789,
      "activations/layer10_attention_weight_max": 33.51899337768555,
      "activations/layer10_attention_weight_min": -31.439897537231445,
      "activations/layer11_attention_weight_max": 33.199588775634766,
      "activations/layer11_attention_weight_min": -30.3211727142334,
      "activations/layer12_attention_weight_max": 27.97486686706543,
      "activations/layer12_attention_weight_min": -28.078577041625977,
      "activations/layer13_attention_weight_max": 46.510074615478516,
      "activations/layer13_attention_weight_min": -34.42030334472656,
      "activations/layer14_attention_weight_max": 64.94442749023438,
      "activations/layer14_attention_weight_min": -45.70858383178711,
      "activations/layer15_attention_weight_max": 38.39057159423828,
      "activations/layer15_attention_weight_min": -29.69251251220703,
      "activations/layer16_attention_weight_max": 34.485469818115234,
      "activations/layer16_attention_weight_min": -30.496854782104492,
      "activations/layer17_attention_weight_max": 36.12828826904297,
      "activations/layer17_attention_weight_min": -25.47956657409668,
      "activations/layer18_attention_weight_max": 29.500572204589844,
      "activations/layer18_attention_weight_min": -21.984888076782227,
      "activations/layer19_attention_weight_max": 37.340824127197266,
      "activations/layer19_attention_weight_min": -31.508359909057617,
      "activations/layer1_attention_weight_max": 18.276878356933594,
      "activations/layer1_attention_weight_min": -15.092976570129395,
      "activations/layer20_attention_weight_max": 33.603267669677734,
      "activations/layer20_attention_weight_min": -24.338666915893555,
      "activations/layer21_attention_weight_max": 31.21811866760254,
      "activations/layer21_attention_weight_min": -20.96084976196289,
      "activations/layer22_attention_weight_max": 51.12179946899414,
      "activations/layer22_attention_weight_min": -29.130212783813477,
      "activations/layer23_attention_weight_max": 34.85549545288086,
      "activations/layer23_attention_weight_min": -22.48019790649414,
      "activations/layer2_attention_weight_max": 33.93682861328125,
      "activations/layer2_attention_weight_min": -31.909530639648438,
      "activations/layer3_attention_weight_max": 94.49516296386719,
      "activations/layer3_attention_weight_min": -98.55708312988281,
      "activations/layer4_attention_weight_max": 136.9471435546875,
      "activations/layer4_attention_weight_min": -131.86178588867188,
      "activations/layer5_attention_weight_max": 82.46438598632812,
      "activations/layer5_attention_weight_min": -78.5538558959961,
      "activations/layer6_attention_weight_max": 57.574039459228516,
      "activations/layer6_attention_weight_min": -57.74400329589844,
      "activations/layer7_attention_weight_max": 94.02765655517578,
      "activations/layer7_attention_weight_min": -87.82176971435547,
      "activations/layer8_attention_weight_max": 45.1645393371582,
      "activations/layer8_attention_weight_min": -42.79523468017578,
      "activations/layer9_attention_weight_max": 36.20524978637695,
      "activations/layer9_attention_weight_min": -36.42046356201172,
      "epoch": 5.52,
      "learning_rate": 0.00011556477272727272,
      "loss": 2.832,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.755,
      "eval_samples_per_second": 490.462,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.755,
      "eval_openwebtext_samples_per_second": 490.462,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_wikitext_loss": 3.013671875,
      "eval_wikitext_ppl": 20.36202965350754,
      "eval_wikitext_runtime": 1.9664,
      "eval_wikitext_samples_per_second": 231.895,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_lambada_loss": 2.626953125,
      "eval_lambada_ppl": 13.831562592310346,
      "eval_lambada_runtime": 9.5547,
      "eval_lambada_samples_per_second": 509.593,
      "step": 95000
    },
    {
      "activations/layer0_attention_weight_max": 16.84573745727539,
      "activations/layer0_attention_weight_min": -11.7159423828125,
      "activations/layer10_attention_weight_max": 33.98397445678711,
      "activations/layer10_attention_weight_min": -29.569150924682617,
      "activations/layer11_attention_weight_max": 32.38291931152344,
      "activations/layer11_attention_weight_min": -28.181339263916016,
      "activations/layer12_attention_weight_max": 23.276708602905273,
      "activations/layer12_attention_weight_min": -25.403961181640625,
      "activations/layer13_attention_weight_max": 40.8845100402832,
      "activations/layer13_attention_weight_min": -31.817729949951172,
      "activations/layer14_attention_weight_max": 55.12296676635742,
      "activations/layer14_attention_weight_min": -41.6518669128418,
      "activations/layer15_attention_weight_max": 40.62674331665039,
      "activations/layer15_attention_weight_min": -27.98641014099121,
      "activations/layer16_attention_weight_max": 38.00615310668945,
      "activations/layer16_attention_weight_min": -27.417381286621094,
      "activations/layer17_attention_weight_max": 41.047428131103516,
      "activations/layer17_attention_weight_min": -23.597272872924805,
      "activations/layer18_attention_weight_max": 32.24439239501953,
      "activations/layer18_attention_weight_min": -19.914669036865234,
      "activations/layer19_attention_weight_max": 41.14733123779297,
      "activations/layer19_attention_weight_min": -28.108566284179688,
      "activations/layer1_attention_weight_max": 16.771095275878906,
      "activations/layer1_attention_weight_min": -16.187515258789062,
      "activations/layer20_attention_weight_max": 31.92266845703125,
      "activations/layer20_attention_weight_min": -24.678783416748047,
      "activations/layer21_attention_weight_max": 35.052734375,
      "activations/layer21_attention_weight_min": -20.421382904052734,
      "activations/layer22_attention_weight_max": 52.35022735595703,
      "activations/layer22_attention_weight_min": -28.60743522644043,
      "activations/layer23_attention_weight_max": 34.73162841796875,
      "activations/layer23_attention_weight_min": -20.541019439697266,
      "activations/layer2_attention_weight_max": 31.732933044433594,
      "activations/layer2_attention_weight_min": -30.876480102539062,
      "activations/layer3_attention_weight_max": 90.45008850097656,
      "activations/layer3_attention_weight_min": -90.80738830566406,
      "activations/layer4_attention_weight_max": 128.8964385986328,
      "activations/layer4_attention_weight_min": -128.9087371826172,
      "activations/layer5_attention_weight_max": 79.18238830566406,
      "activations/layer5_attention_weight_min": -75.20054626464844,
      "activations/layer6_attention_weight_max": 56.45055389404297,
      "activations/layer6_attention_weight_min": -54.7062873840332,
      "activations/layer7_attention_weight_max": 85.24356079101562,
      "activations/layer7_attention_weight_min": -82.80953979492188,
      "activations/layer8_attention_weight_max": 40.37896728515625,
      "activations/layer8_attention_weight_min": -40.28595733642578,
      "activations/layer9_attention_weight_max": 35.45130920410156,
      "activations/layer9_attention_weight_min": -34.29830551147461,
      "epoch": 5.52,
      "learning_rate": 0.00011554583333333332,
      "loss": 2.8362,
      "step": 95050
    },
    {
      "activations/layer0_attention_weight_max": 15.846882820129395,
      "activations/layer0_attention_weight_min": -11.642398834228516,
      "activations/layer10_attention_weight_max": 32.69182205200195,
      "activations/layer10_attention_weight_min": -30.520309448242188,
      "activations/layer11_attention_weight_max": 32.63785171508789,
      "activations/layer11_attention_weight_min": -27.49471664428711,
      "activations/layer12_attention_weight_max": 21.496994018554688,
      "activations/layer12_attention_weight_min": -25.34309196472168,
      "activations/layer13_attention_weight_max": 41.87788772583008,
      "activations/layer13_attention_weight_min": -32.897560119628906,
      "activations/layer14_attention_weight_max": 54.37730407714844,
      "activations/layer14_attention_weight_min": -41.244361877441406,
      "activations/layer15_attention_weight_max": 39.794857025146484,
      "activations/layer15_attention_weight_min": -28.55553436279297,
      "activations/layer16_attention_weight_max": 33.96989822387695,
      "activations/layer16_attention_weight_min": -26.800615310668945,
      "activations/layer17_attention_weight_max": 38.36836624145508,
      "activations/layer17_attention_weight_min": -25.11069679260254,
      "activations/layer18_attention_weight_max": 28.888505935668945,
      "activations/layer18_attention_weight_min": -19.654029846191406,
      "activations/layer19_attention_weight_max": 38.08012771606445,
      "activations/layer19_attention_weight_min": -27.699935913085938,
      "activations/layer1_attention_weight_max": 18.388792037963867,
      "activations/layer1_attention_weight_min": -14.311352729797363,
      "activations/layer20_attention_weight_max": 30.910259246826172,
      "activations/layer20_attention_weight_min": -23.287813186645508,
      "activations/layer21_attention_weight_max": 30.412517547607422,
      "activations/layer21_attention_weight_min": -18.90545082092285,
      "activations/layer22_attention_weight_max": 48.27265930175781,
      "activations/layer22_attention_weight_min": -27.026123046875,
      "activations/layer23_attention_weight_max": 33.029296875,
      "activations/layer23_attention_weight_min": -19.802011489868164,
      "activations/layer2_attention_weight_max": 33.140037536621094,
      "activations/layer2_attention_weight_min": -31.607948303222656,
      "activations/layer3_attention_weight_max": 93.6979751586914,
      "activations/layer3_attention_weight_min": -92.57412719726562,
      "activations/layer4_attention_weight_max": 135.33908081054688,
      "activations/layer4_attention_weight_min": -130.33572387695312,
      "activations/layer5_attention_weight_max": 81.54083251953125,
      "activations/layer5_attention_weight_min": -76.91340637207031,
      "activations/layer6_attention_weight_max": 55.86632537841797,
      "activations/layer6_attention_weight_min": -58.68971252441406,
      "activations/layer7_attention_weight_max": 87.26033782958984,
      "activations/layer7_attention_weight_min": -89.83538055419922,
      "activations/layer8_attention_weight_max": 40.54115676879883,
      "activations/layer8_attention_weight_min": -39.233070373535156,
      "activations/layer9_attention_weight_max": 36.47513961791992,
      "activations/layer9_attention_weight_min": -33.67525863647461,
      "epoch": 5.53,
      "learning_rate": 0.00011552689393939392,
      "loss": 2.8487,
      "step": 95100
    },
    {
      "activations/layer0_attention_weight_max": 16.619070053100586,
      "activations/layer0_attention_weight_min": -11.430391311645508,
      "activations/layer10_attention_weight_max": 29.8377742767334,
      "activations/layer10_attention_weight_min": -29.11900520324707,
      "activations/layer11_attention_weight_max": 30.148094177246094,
      "activations/layer11_attention_weight_min": -26.22021484375,
      "activations/layer12_attention_weight_max": 21.72926139831543,
      "activations/layer12_attention_weight_min": -26.53266143798828,
      "activations/layer13_attention_weight_max": 39.58333969116211,
      "activations/layer13_attention_weight_min": -31.387500762939453,
      "activations/layer14_attention_weight_max": 50.344364166259766,
      "activations/layer14_attention_weight_min": -44.45261001586914,
      "activations/layer15_attention_weight_max": 34.96131896972656,
      "activations/layer15_attention_weight_min": -28.58751678466797,
      "activations/layer16_attention_weight_max": 33.33278274536133,
      "activations/layer16_attention_weight_min": -25.76850700378418,
      "activations/layer17_attention_weight_max": 35.100486755371094,
      "activations/layer17_attention_weight_min": -23.936405181884766,
      "activations/layer18_attention_weight_max": 31.306570053100586,
      "activations/layer18_attention_weight_min": -19.94058609008789,
      "activations/layer19_attention_weight_max": 40.4463005065918,
      "activations/layer19_attention_weight_min": -28.320072174072266,
      "activations/layer1_attention_weight_max": 17.20307159423828,
      "activations/layer1_attention_weight_min": -14.292247772216797,
      "activations/layer20_attention_weight_max": 30.369354248046875,
      "activations/layer20_attention_weight_min": -22.7147216796875,
      "activations/layer21_attention_weight_max": 32.67074203491211,
      "activations/layer21_attention_weight_min": -21.838603973388672,
      "activations/layer22_attention_weight_max": 45.2914924621582,
      "activations/layer22_attention_weight_min": -31.424551010131836,
      "activations/layer23_attention_weight_max": 33.84757995605469,
      "activations/layer23_attention_weight_min": -21.620849609375,
      "activations/layer2_attention_weight_max": 31.2832088470459,
      "activations/layer2_attention_weight_min": -29.896347045898438,
      "activations/layer3_attention_weight_max": 87.77644348144531,
      "activations/layer3_attention_weight_min": -90.87723541259766,
      "activations/layer4_attention_weight_max": 127.35808563232422,
      "activations/layer4_attention_weight_min": -122.68730926513672,
      "activations/layer5_attention_weight_max": 76.5823745727539,
      "activations/layer5_attention_weight_min": -72.712890625,
      "activations/layer6_attention_weight_max": 52.49248504638672,
      "activations/layer6_attention_weight_min": -53.53599166870117,
      "activations/layer7_attention_weight_max": 78.9871826171875,
      "activations/layer7_attention_weight_min": -85.5134048461914,
      "activations/layer8_attention_weight_max": 36.7742805480957,
      "activations/layer8_attention_weight_min": -36.597660064697266,
      "activations/layer9_attention_weight_max": 33.15099334716797,
      "activations/layer9_attention_weight_min": -32.013832092285156,
      "epoch": 5.53,
      "learning_rate": 0.00011550795454545454,
      "loss": 2.8476,
      "step": 95150
    },
    {
      "activations/layer0_attention_weight_max": 14.719330787658691,
      "activations/layer0_attention_weight_min": -12.134625434875488,
      "activations/layer10_attention_weight_max": 32.758392333984375,
      "activations/layer10_attention_weight_min": -32.474571228027344,
      "activations/layer11_attention_weight_max": 33.50215148925781,
      "activations/layer11_attention_weight_min": -30.76293182373047,
      "activations/layer12_attention_weight_max": 45.496337890625,
      "activations/layer12_attention_weight_min": -28.035945892333984,
      "activations/layer13_attention_weight_max": 43.07838821411133,
      "activations/layer13_attention_weight_min": -33.27107238769531,
      "activations/layer14_attention_weight_max": 52.80494689941406,
      "activations/layer14_attention_weight_min": -44.2704963684082,
      "activations/layer15_attention_weight_max": 39.1962776184082,
      "activations/layer15_attention_weight_min": -26.84256362915039,
      "activations/layer16_attention_weight_max": 35.44516372680664,
      "activations/layer16_attention_weight_min": -27.610218048095703,
      "activations/layer17_attention_weight_max": 40.34880065917969,
      "activations/layer17_attention_weight_min": -23.528223037719727,
      "activations/layer18_attention_weight_max": 30.419034957885742,
      "activations/layer18_attention_weight_min": -20.075252532958984,
      "activations/layer19_attention_weight_max": 34.677738189697266,
      "activations/layer19_attention_weight_min": -27.694562911987305,
      "activations/layer1_attention_weight_max": 16.851423263549805,
      "activations/layer1_attention_weight_min": -14.755000114440918,
      "activations/layer20_attention_weight_max": 31.514421463012695,
      "activations/layer20_attention_weight_min": -23.03837776184082,
      "activations/layer21_attention_weight_max": 29.508304595947266,
      "activations/layer21_attention_weight_min": -22.03252601623535,
      "activations/layer22_attention_weight_max": 48.97380447387695,
      "activations/layer22_attention_weight_min": -27.71930694580078,
      "activations/layer23_attention_weight_max": 31.47247886657715,
      "activations/layer23_attention_weight_min": -20.223413467407227,
      "activations/layer2_attention_weight_max": 32.652679443359375,
      "activations/layer2_attention_weight_min": -32.870033264160156,
      "activations/layer3_attention_weight_max": 95.87826538085938,
      "activations/layer3_attention_weight_min": -97.0739974975586,
      "activations/layer4_attention_weight_max": 134.51551818847656,
      "activations/layer4_attention_weight_min": -129.9254913330078,
      "activations/layer5_attention_weight_max": 79.84954833984375,
      "activations/layer5_attention_weight_min": -78.13094329833984,
      "activations/layer6_attention_weight_max": 54.23158645629883,
      "activations/layer6_attention_weight_min": -55.38983154296875,
      "activations/layer7_attention_weight_max": 90.26627349853516,
      "activations/layer7_attention_weight_min": -85.516845703125,
      "activations/layer8_attention_weight_max": 40.58183670043945,
      "activations/layer8_attention_weight_min": -39.00847244262695,
      "activations/layer9_attention_weight_max": 35.95351791381836,
      "activations/layer9_attention_weight_min": -36.1490364074707,
      "epoch": 5.53,
      "learning_rate": 0.00011548901515151515,
      "loss": 2.8323,
      "step": 95200
    },
    {
      "activations/layer0_attention_weight_max": 16.666099548339844,
      "activations/layer0_attention_weight_min": -11.419659614562988,
      "activations/layer10_attention_weight_max": 34.632843017578125,
      "activations/layer10_attention_weight_min": -32.814208984375,
      "activations/layer11_attention_weight_max": 34.56646728515625,
      "activations/layer11_attention_weight_min": -27.201101303100586,
      "activations/layer12_attention_weight_max": 21.839649200439453,
      "activations/layer12_attention_weight_min": -25.82427215576172,
      "activations/layer13_attention_weight_max": 42.85698318481445,
      "activations/layer13_attention_weight_min": -31.216304779052734,
      "activations/layer14_attention_weight_max": 57.43128204345703,
      "activations/layer14_attention_weight_min": -43.739871978759766,
      "activations/layer15_attention_weight_max": 40.333045959472656,
      "activations/layer15_attention_weight_min": -29.43971061706543,
      "activations/layer16_attention_weight_max": 38.18363952636719,
      "activations/layer16_attention_weight_min": -28.03317642211914,
      "activations/layer17_attention_weight_max": 38.990745544433594,
      "activations/layer17_attention_weight_min": -25.253570556640625,
      "activations/layer18_attention_weight_max": 33.12453842163086,
      "activations/layer18_attention_weight_min": -21.635587692260742,
      "activations/layer19_attention_weight_max": 42.336448669433594,
      "activations/layer19_attention_weight_min": -30.139501571655273,
      "activations/layer1_attention_weight_max": 17.631099700927734,
      "activations/layer1_attention_weight_min": -14.856252670288086,
      "activations/layer20_attention_weight_max": 34.50975036621094,
      "activations/layer20_attention_weight_min": -24.984983444213867,
      "activations/layer21_attention_weight_max": 34.284019470214844,
      "activations/layer21_attention_weight_min": -24.162506103515625,
      "activations/layer22_attention_weight_max": 51.84861755371094,
      "activations/layer22_attention_weight_min": -31.956357955932617,
      "activations/layer23_attention_weight_max": 32.95721435546875,
      "activations/layer23_attention_weight_min": -23.10645294189453,
      "activations/layer2_attention_weight_max": 32.579559326171875,
      "activations/layer2_attention_weight_min": -30.847496032714844,
      "activations/layer3_attention_weight_max": 91.57868957519531,
      "activations/layer3_attention_weight_min": -92.53762817382812,
      "activations/layer4_attention_weight_max": 130.8289031982422,
      "activations/layer4_attention_weight_min": -131.3037872314453,
      "activations/layer5_attention_weight_max": 79.72347259521484,
      "activations/layer5_attention_weight_min": -78.06206512451172,
      "activations/layer6_attention_weight_max": 57.24667739868164,
      "activations/layer6_attention_weight_min": -60.032615661621094,
      "activations/layer7_attention_weight_max": 86.96202087402344,
      "activations/layer7_attention_weight_min": -86.07918548583984,
      "activations/layer8_attention_weight_max": 40.76643753051758,
      "activations/layer8_attention_weight_min": -42.64729690551758,
      "activations/layer9_attention_weight_max": 36.99588394165039,
      "activations/layer9_attention_weight_min": -36.63853454589844,
      "epoch": 5.53,
      "learning_rate": 0.00011547007575757574,
      "loss": 2.8218,
      "step": 95250
    },
    {
      "activations/layer0_attention_weight_max": 15.232254028320312,
      "activations/layer0_attention_weight_min": -11.22185230255127,
      "activations/layer10_attention_weight_max": 34.16333770751953,
      "activations/layer10_attention_weight_min": -30.77610969543457,
      "activations/layer11_attention_weight_max": 31.876249313354492,
      "activations/layer11_attention_weight_min": -28.38484764099121,
      "activations/layer12_attention_weight_max": 26.472572326660156,
      "activations/layer12_attention_weight_min": -31.378887176513672,
      "activations/layer13_attention_weight_max": 38.82091522216797,
      "activations/layer13_attention_weight_min": -32.7952766418457,
      "activations/layer14_attention_weight_max": 57.426212310791016,
      "activations/layer14_attention_weight_min": -45.408775329589844,
      "activations/layer15_attention_weight_max": 40.59984588623047,
      "activations/layer15_attention_weight_min": -30.018638610839844,
      "activations/layer16_attention_weight_max": 38.42757034301758,
      "activations/layer16_attention_weight_min": -27.036474227905273,
      "activations/layer17_attention_weight_max": 36.63820266723633,
      "activations/layer17_attention_weight_min": -23.85988426208496,
      "activations/layer18_attention_weight_max": 29.860713958740234,
      "activations/layer18_attention_weight_min": -21.07285499572754,
      "activations/layer19_attention_weight_max": 38.29990768432617,
      "activations/layer19_attention_weight_min": -29.522851943969727,
      "activations/layer1_attention_weight_max": 17.42904281616211,
      "activations/layer1_attention_weight_min": -14.559401512145996,
      "activations/layer20_attention_weight_max": 32.32963943481445,
      "activations/layer20_attention_weight_min": -23.31667137145996,
      "activations/layer21_attention_weight_max": 33.40847396850586,
      "activations/layer21_attention_weight_min": -21.390172958374023,
      "activations/layer22_attention_weight_max": 50.80563735961914,
      "activations/layer22_attention_weight_min": -27.276832580566406,
      "activations/layer23_attention_weight_max": 35.42647171020508,
      "activations/layer23_attention_weight_min": -22.55043601989746,
      "activations/layer2_attention_weight_max": 32.59105682373047,
      "activations/layer2_attention_weight_min": -31.39974021911621,
      "activations/layer3_attention_weight_max": 92.21385955810547,
      "activations/layer3_attention_weight_min": -93.03370666503906,
      "activations/layer4_attention_weight_max": 133.6654815673828,
      "activations/layer4_attention_weight_min": -126.3628921508789,
      "activations/layer5_attention_weight_max": 79.99246215820312,
      "activations/layer5_attention_weight_min": -75.30081176757812,
      "activations/layer6_attention_weight_max": 54.736759185791016,
      "activations/layer6_attention_weight_min": -59.56009292602539,
      "activations/layer7_attention_weight_max": 85.5602798461914,
      "activations/layer7_attention_weight_min": -87.64913940429688,
      "activations/layer8_attention_weight_max": 40.664337158203125,
      "activations/layer8_attention_weight_min": -42.092430114746094,
      "activations/layer9_attention_weight_max": 37.864784240722656,
      "activations/layer9_attention_weight_min": -38.789031982421875,
      "epoch": 5.54,
      "learning_rate": 0.00011545113636363636,
      "loss": 2.8267,
      "step": 95300
    },
    {
      "activations/layer0_attention_weight_max": 16.475324630737305,
      "activations/layer0_attention_weight_min": -11.4926176071167,
      "activations/layer10_attention_weight_max": 34.73101043701172,
      "activations/layer10_attention_weight_min": -28.99530029296875,
      "activations/layer11_attention_weight_max": 30.5860595703125,
      "activations/layer11_attention_weight_min": -28.4355525970459,
      "activations/layer12_attention_weight_max": 45.3203239440918,
      "activations/layer12_attention_weight_min": -27.748498916625977,
      "activations/layer13_attention_weight_max": 44.099586486816406,
      "activations/layer13_attention_weight_min": -31.597484588623047,
      "activations/layer14_attention_weight_max": 63.947418212890625,
      "activations/layer14_attention_weight_min": -43.82805252075195,
      "activations/layer15_attention_weight_max": 39.742034912109375,
      "activations/layer15_attention_weight_min": -28.210132598876953,
      "activations/layer16_attention_weight_max": 38.66561508178711,
      "activations/layer16_attention_weight_min": -27.78376579284668,
      "activations/layer17_attention_weight_max": 39.67438888549805,
      "activations/layer17_attention_weight_min": -23.597454071044922,
      "activations/layer18_attention_weight_max": 32.144466400146484,
      "activations/layer18_attention_weight_min": -20.985471725463867,
      "activations/layer19_attention_weight_max": 41.10980224609375,
      "activations/layer19_attention_weight_min": -29.101032257080078,
      "activations/layer1_attention_weight_max": 16.767879486083984,
      "activations/layer1_attention_weight_min": -14.057741165161133,
      "activations/layer20_attention_weight_max": 35.13014602661133,
      "activations/layer20_attention_weight_min": -22.826213836669922,
      "activations/layer21_attention_weight_max": 33.47011947631836,
      "activations/layer21_attention_weight_min": -20.773086547851562,
      "activations/layer22_attention_weight_max": 50.860389709472656,
      "activations/layer22_attention_weight_min": -27.4100399017334,
      "activations/layer23_attention_weight_max": 36.71397399902344,
      "activations/layer23_attention_weight_min": -22.656173706054688,
      "activations/layer2_attention_weight_max": 32.01526641845703,
      "activations/layer2_attention_weight_min": -32.98796081542969,
      "activations/layer3_attention_weight_max": 91.7908935546875,
      "activations/layer3_attention_weight_min": -89.62769317626953,
      "activations/layer4_attention_weight_max": 130.9412078857422,
      "activations/layer4_attention_weight_min": -125.95172119140625,
      "activations/layer5_attention_weight_max": 77.67105865478516,
      "activations/layer5_attention_weight_min": -73.26323699951172,
      "activations/layer6_attention_weight_max": 53.81138229370117,
      "activations/layer6_attention_weight_min": -56.507972717285156,
      "activations/layer7_attention_weight_max": 85.07730102539062,
      "activations/layer7_attention_weight_min": -87.23161315917969,
      "activations/layer8_attention_weight_max": 41.19975280761719,
      "activations/layer8_attention_weight_min": -38.86186599731445,
      "activations/layer9_attention_weight_max": 35.8739128112793,
      "activations/layer9_attention_weight_min": -37.6542854309082,
      "epoch": 5.54,
      "learning_rate": 0.00011543219696969697,
      "loss": 2.8418,
      "step": 95350
    },
    {
      "activations/layer0_attention_weight_max": 16.868356704711914,
      "activations/layer0_attention_weight_min": -12.571122169494629,
      "activations/layer10_attention_weight_max": 35.98350524902344,
      "activations/layer10_attention_weight_min": -32.7923698425293,
      "activations/layer11_attention_weight_max": 31.559185028076172,
      "activations/layer11_attention_weight_min": -31.276378631591797,
      "activations/layer12_attention_weight_max": 26.92636489868164,
      "activations/layer12_attention_weight_min": -26.334896087646484,
      "activations/layer13_attention_weight_max": 38.1733512878418,
      "activations/layer13_attention_weight_min": -32.403221130371094,
      "activations/layer14_attention_weight_max": 67.36416625976562,
      "activations/layer14_attention_weight_min": -50.781005859375,
      "activations/layer15_attention_weight_max": 37.11988067626953,
      "activations/layer15_attention_weight_min": -28.456138610839844,
      "activations/layer16_attention_weight_max": 35.8022575378418,
      "activations/layer16_attention_weight_min": -28.57755470275879,
      "activations/layer17_attention_weight_max": 35.59286117553711,
      "activations/layer17_attention_weight_min": -24.804546356201172,
      "activations/layer18_attention_weight_max": 29.765615463256836,
      "activations/layer18_attention_weight_min": -18.963546752929688,
      "activations/layer19_attention_weight_max": 38.624969482421875,
      "activations/layer19_attention_weight_min": -28.696279525756836,
      "activations/layer1_attention_weight_max": 17.112489700317383,
      "activations/layer1_attention_weight_min": -15.070563316345215,
      "activations/layer20_attention_weight_max": 29.662307739257812,
      "activations/layer20_attention_weight_min": -24.736173629760742,
      "activations/layer21_attention_weight_max": 26.59303855895996,
      "activations/layer21_attention_weight_min": -21.815134048461914,
      "activations/layer22_attention_weight_max": 46.376708984375,
      "activations/layer22_attention_weight_min": -26.835174560546875,
      "activations/layer23_attention_weight_max": 30.54292869567871,
      "activations/layer23_attention_weight_min": -19.959064483642578,
      "activations/layer2_attention_weight_max": 33.091026306152344,
      "activations/layer2_attention_weight_min": -32.676979064941406,
      "activations/layer3_attention_weight_max": 93.48342895507812,
      "activations/layer3_attention_weight_min": -91.5499496459961,
      "activations/layer4_attention_weight_max": 135.8984832763672,
      "activations/layer4_attention_weight_min": -128.37875366210938,
      "activations/layer5_attention_weight_max": 81.28385925292969,
      "activations/layer5_attention_weight_min": -75.26899719238281,
      "activations/layer6_attention_weight_max": 58.20383071899414,
      "activations/layer6_attention_weight_min": -58.66327667236328,
      "activations/layer7_attention_weight_max": 87.64054107666016,
      "activations/layer7_attention_weight_min": -84.09602355957031,
      "activations/layer8_attention_weight_max": 42.87411880493164,
      "activations/layer8_attention_weight_min": -39.45487594604492,
      "activations/layer9_attention_weight_max": 39.58760070800781,
      "activations/layer9_attention_weight_min": -36.2201042175293,
      "epoch": 5.54,
      "learning_rate": 0.00011541325757575756,
      "loss": 2.8218,
      "step": 95400
    },
    {
      "activations/layer0_attention_weight_max": 14.978189468383789,
      "activations/layer0_attention_weight_min": -11.186424255371094,
      "activations/layer10_attention_weight_max": 32.182525634765625,
      "activations/layer10_attention_weight_min": -31.973995208740234,
      "activations/layer11_attention_weight_max": 33.64966583251953,
      "activations/layer11_attention_weight_min": -28.607681274414062,
      "activations/layer12_attention_weight_max": 36.426918029785156,
      "activations/layer12_attention_weight_min": -28.56764030456543,
      "activations/layer13_attention_weight_max": 44.18342208862305,
      "activations/layer13_attention_weight_min": -31.926218032836914,
      "activations/layer14_attention_weight_max": 71.77365112304688,
      "activations/layer14_attention_weight_min": -49.177337646484375,
      "activations/layer15_attention_weight_max": 37.18638229370117,
      "activations/layer15_attention_weight_min": -27.303869247436523,
      "activations/layer16_attention_weight_max": 34.495357513427734,
      "activations/layer16_attention_weight_min": -27.23028564453125,
      "activations/layer17_attention_weight_max": 37.505836486816406,
      "activations/layer17_attention_weight_min": -24.229904174804688,
      "activations/layer18_attention_weight_max": 28.5622501373291,
      "activations/layer18_attention_weight_min": -19.80843162536621,
      "activations/layer19_attention_weight_max": 41.69303512573242,
      "activations/layer19_attention_weight_min": -29.27935218811035,
      "activations/layer1_attention_weight_max": 18.621318817138672,
      "activations/layer1_attention_weight_min": -13.260890007019043,
      "activations/layer20_attention_weight_max": 35.5372314453125,
      "activations/layer20_attention_weight_min": -22.94355583190918,
      "activations/layer21_attention_weight_max": 28.779064178466797,
      "activations/layer21_attention_weight_min": -19.19547462463379,
      "activations/layer22_attention_weight_max": 55.41350555419922,
      "activations/layer22_attention_weight_min": -28.813385009765625,
      "activations/layer23_attention_weight_max": 37.105079650878906,
      "activations/layer23_attention_weight_min": -22.234487533569336,
      "activations/layer2_attention_weight_max": 33.67108917236328,
      "activations/layer2_attention_weight_min": -31.795820236206055,
      "activations/layer3_attention_weight_max": 92.78971862792969,
      "activations/layer3_attention_weight_min": -92.28329467773438,
      "activations/layer4_attention_weight_max": 135.50352478027344,
      "activations/layer4_attention_weight_min": -128.7035675048828,
      "activations/layer5_attention_weight_max": 80.62005615234375,
      "activations/layer5_attention_weight_min": -79.37667083740234,
      "activations/layer6_attention_weight_max": 57.656593322753906,
      "activations/layer6_attention_weight_min": -54.93784713745117,
      "activations/layer7_attention_weight_max": 89.40251159667969,
      "activations/layer7_attention_weight_min": -86.68126678466797,
      "activations/layer8_attention_weight_max": 39.82505416870117,
      "activations/layer8_attention_weight_min": -38.15185546875,
      "activations/layer9_attention_weight_max": 35.16282653808594,
      "activations/layer9_attention_weight_min": -37.38201904296875,
      "epoch": 5.55,
      "learning_rate": 0.00011539431818181817,
      "loss": 2.8399,
      "step": 95450
    },
    {
      "activations/layer0_attention_weight_max": 17.06052589416504,
      "activations/layer0_attention_weight_min": -11.863066673278809,
      "activations/layer10_attention_weight_max": 34.09482955932617,
      "activations/layer10_attention_weight_min": -31.482498168945312,
      "activations/layer11_attention_weight_max": 34.11284255981445,
      "activations/layer11_attention_weight_min": -29.466903686523438,
      "activations/layer12_attention_weight_max": 29.73007583618164,
      "activations/layer12_attention_weight_min": -43.06110382080078,
      "activations/layer13_attention_weight_max": 45.7807502746582,
      "activations/layer13_attention_weight_min": -31.919301986694336,
      "activations/layer14_attention_weight_max": 65.09712219238281,
      "activations/layer14_attention_weight_min": -46.632144927978516,
      "activations/layer15_attention_weight_max": 37.64936065673828,
      "activations/layer15_attention_weight_min": -28.772829055786133,
      "activations/layer16_attention_weight_max": 35.553070068359375,
      "activations/layer16_attention_weight_min": -26.842864990234375,
      "activations/layer17_attention_weight_max": 38.51845169067383,
      "activations/layer17_attention_weight_min": -26.17386817932129,
      "activations/layer18_attention_weight_max": 36.24565887451172,
      "activations/layer18_attention_weight_min": -20.323230743408203,
      "activations/layer19_attention_weight_max": 38.7275505065918,
      "activations/layer19_attention_weight_min": -29.91162109375,
      "activations/layer1_attention_weight_max": 17.073326110839844,
      "activations/layer1_attention_weight_min": -15.769888877868652,
      "activations/layer20_attention_weight_max": 32.65544891357422,
      "activations/layer20_attention_weight_min": -23.60787582397461,
      "activations/layer21_attention_weight_max": 35.518795013427734,
      "activations/layer21_attention_weight_min": -21.97393035888672,
      "activations/layer22_attention_weight_max": 53.18140411376953,
      "activations/layer22_attention_weight_min": -28.883787155151367,
      "activations/layer23_attention_weight_max": 37.10000228881836,
      "activations/layer23_attention_weight_min": -19.55238151550293,
      "activations/layer2_attention_weight_max": 32.954368591308594,
      "activations/layer2_attention_weight_min": -31.37104034423828,
      "activations/layer3_attention_weight_max": 92.81898498535156,
      "activations/layer3_attention_weight_min": -92.88672637939453,
      "activations/layer4_attention_weight_max": 133.62802124023438,
      "activations/layer4_attention_weight_min": -127.46004486083984,
      "activations/layer5_attention_weight_max": 77.84635925292969,
      "activations/layer5_attention_weight_min": -77.97709655761719,
      "activations/layer6_attention_weight_max": 55.82877731323242,
      "activations/layer6_attention_weight_min": -57.018341064453125,
      "activations/layer7_attention_weight_max": 89.46300506591797,
      "activations/layer7_attention_weight_min": -92.9472885131836,
      "activations/layer8_attention_weight_max": 38.40325927734375,
      "activations/layer8_attention_weight_min": -39.45494842529297,
      "activations/layer9_attention_weight_max": 37.54940414428711,
      "activations/layer9_attention_weight_min": -51.902687072753906,
      "epoch": 5.55,
      "learning_rate": 0.00011537537878787879,
      "loss": 2.8285,
      "step": 95500
    },
    {
      "activations/layer0_attention_weight_max": 16.331398010253906,
      "activations/layer0_attention_weight_min": -11.566032409667969,
      "activations/layer10_attention_weight_max": 35.384613037109375,
      "activations/layer10_attention_weight_min": -33.21421813964844,
      "activations/layer11_attention_weight_max": 33.58030700683594,
      "activations/layer11_attention_weight_min": -29.36907958984375,
      "activations/layer12_attention_weight_max": 22.199737548828125,
      "activations/layer12_attention_weight_min": -28.237449645996094,
      "activations/layer13_attention_weight_max": 43.60905456542969,
      "activations/layer13_attention_weight_min": -32.82124328613281,
      "activations/layer14_attention_weight_max": 60.677223205566406,
      "activations/layer14_attention_weight_min": -47.12574768066406,
      "activations/layer15_attention_weight_max": 41.920799255371094,
      "activations/layer15_attention_weight_min": -29.537904739379883,
      "activations/layer16_attention_weight_max": 35.847373962402344,
      "activations/layer16_attention_weight_min": -28.047382354736328,
      "activations/layer17_attention_weight_max": 39.0222282409668,
      "activations/layer17_attention_weight_min": -23.885282516479492,
      "activations/layer18_attention_weight_max": 31.79741096496582,
      "activations/layer18_attention_weight_min": -20.343318939208984,
      "activations/layer19_attention_weight_max": 41.19314956665039,
      "activations/layer19_attention_weight_min": -30.64177894592285,
      "activations/layer1_attention_weight_max": 17.578306198120117,
      "activations/layer1_attention_weight_min": -13.840510368347168,
      "activations/layer20_attention_weight_max": 32.08549118041992,
      "activations/layer20_attention_weight_min": -21.9494571685791,
      "activations/layer21_attention_weight_max": 36.45405197143555,
      "activations/layer21_attention_weight_min": -20.02483558654785,
      "activations/layer22_attention_weight_max": 51.44132614135742,
      "activations/layer22_attention_weight_min": -25.066234588623047,
      "activations/layer23_attention_weight_max": 33.02080535888672,
      "activations/layer23_attention_weight_min": -19.249202728271484,
      "activations/layer2_attention_weight_max": 33.62697982788086,
      "activations/layer2_attention_weight_min": -32.75060272216797,
      "activations/layer3_attention_weight_max": 96.91465759277344,
      "activations/layer3_attention_weight_min": -100.5440673828125,
      "activations/layer4_attention_weight_max": 143.21922302246094,
      "activations/layer4_attention_weight_min": -143.40020751953125,
      "activations/layer5_attention_weight_max": 83.95880126953125,
      "activations/layer5_attention_weight_min": -83.58137512207031,
      "activations/layer6_attention_weight_max": 58.932884216308594,
      "activations/layer6_attention_weight_min": -58.35379409790039,
      "activations/layer7_attention_weight_max": 92.86637115478516,
      "activations/layer7_attention_weight_min": -100.76953125,
      "activations/layer8_attention_weight_max": 46.108707427978516,
      "activations/layer8_attention_weight_min": -45.59300994873047,
      "activations/layer9_attention_weight_max": 40.30070877075195,
      "activations/layer9_attention_weight_min": -35.49193572998047,
      "epoch": 5.55,
      "learning_rate": 0.00011535643939393938,
      "loss": 2.846,
      "step": 95550
    },
    {
      "activations/layer0_attention_weight_max": 15.14237117767334,
      "activations/layer0_attention_weight_min": -11.666447639465332,
      "activations/layer10_attention_weight_max": 37.04290771484375,
      "activations/layer10_attention_weight_min": -34.82695388793945,
      "activations/layer11_attention_weight_max": 31.579246520996094,
      "activations/layer11_attention_weight_min": -28.54641342163086,
      "activations/layer12_attention_weight_max": 24.885330200195312,
      "activations/layer12_attention_weight_min": -26.758625030517578,
      "activations/layer13_attention_weight_max": 40.28064727783203,
      "activations/layer13_attention_weight_min": -32.58961486816406,
      "activations/layer14_attention_weight_max": 62.81630325317383,
      "activations/layer14_attention_weight_min": -47.49061965942383,
      "activations/layer15_attention_weight_max": 36.991546630859375,
      "activations/layer15_attention_weight_min": -28.355669021606445,
      "activations/layer16_attention_weight_max": 36.838687896728516,
      "activations/layer16_attention_weight_min": -29.550107955932617,
      "activations/layer17_attention_weight_max": 39.956321716308594,
      "activations/layer17_attention_weight_min": -24.456920623779297,
      "activations/layer18_attention_weight_max": 30.71758460998535,
      "activations/layer18_attention_weight_min": -20.500818252563477,
      "activations/layer19_attention_weight_max": 37.33877944946289,
      "activations/layer19_attention_weight_min": -32.08891296386719,
      "activations/layer1_attention_weight_max": 17.409034729003906,
      "activations/layer1_attention_weight_min": -15.750335693359375,
      "activations/layer20_attention_weight_max": 32.41706085205078,
      "activations/layer20_attention_weight_min": -25.554746627807617,
      "activations/layer21_attention_weight_max": 30.21341896057129,
      "activations/layer21_attention_weight_min": -21.001991271972656,
      "activations/layer22_attention_weight_max": 50.52010726928711,
      "activations/layer22_attention_weight_min": -27.972606658935547,
      "activations/layer23_attention_weight_max": 33.222686767578125,
      "activations/layer23_attention_weight_min": -22.761608123779297,
      "activations/layer2_attention_weight_max": 34.125465393066406,
      "activations/layer2_attention_weight_min": -33.94697189331055,
      "activations/layer3_attention_weight_max": 93.83597564697266,
      "activations/layer3_attention_weight_min": -96.2215347290039,
      "activations/layer4_attention_weight_max": 131.5874786376953,
      "activations/layer4_attention_weight_min": -131.9784393310547,
      "activations/layer5_attention_weight_max": 80.91991424560547,
      "activations/layer5_attention_weight_min": -76.88456726074219,
      "activations/layer6_attention_weight_max": 57.772804260253906,
      "activations/layer6_attention_weight_min": -60.70206832885742,
      "activations/layer7_attention_weight_max": 87.67984771728516,
      "activations/layer7_attention_weight_min": -93.51663970947266,
      "activations/layer8_attention_weight_max": 42.442440032958984,
      "activations/layer8_attention_weight_min": -42.80832290649414,
      "activations/layer9_attention_weight_max": 38.19673156738281,
      "activations/layer9_attention_weight_min": -36.45170211791992,
      "epoch": 5.55,
      "learning_rate": 0.00011533749999999999,
      "loss": 2.85,
      "step": 95600
    },
    {
      "activations/layer0_attention_weight_max": 16.460962295532227,
      "activations/layer0_attention_weight_min": -12.123676300048828,
      "activations/layer10_attention_weight_max": 32.088844299316406,
      "activations/layer10_attention_weight_min": -31.3745174407959,
      "activations/layer11_attention_weight_max": 31.856090545654297,
      "activations/layer11_attention_weight_min": -29.411466598510742,
      "activations/layer12_attention_weight_max": 25.408039093017578,
      "activations/layer12_attention_weight_min": -29.53392219543457,
      "activations/layer13_attention_weight_max": 42.55415344238281,
      "activations/layer13_attention_weight_min": -30.946191787719727,
      "activations/layer14_attention_weight_max": 57.96208572387695,
      "activations/layer14_attention_weight_min": -45.657562255859375,
      "activations/layer15_attention_weight_max": 34.95991897583008,
      "activations/layer15_attention_weight_min": -27.3331241607666,
      "activations/layer16_attention_weight_max": 36.31402587890625,
      "activations/layer16_attention_weight_min": -27.303895950317383,
      "activations/layer17_attention_weight_max": 32.429527282714844,
      "activations/layer17_attention_weight_min": -23.78223419189453,
      "activations/layer18_attention_weight_max": 30.701068878173828,
      "activations/layer18_attention_weight_min": -19.287633895874023,
      "activations/layer19_attention_weight_max": 34.456329345703125,
      "activations/layer19_attention_weight_min": -29.636951446533203,
      "activations/layer1_attention_weight_max": 17.248348236083984,
      "activations/layer1_attention_weight_min": -15.38421630859375,
      "activations/layer20_attention_weight_max": 27.866458892822266,
      "activations/layer20_attention_weight_min": -23.966001510620117,
      "activations/layer21_attention_weight_max": 29.551025390625,
      "activations/layer21_attention_weight_min": -22.224321365356445,
      "activations/layer22_attention_weight_max": 45.34114456176758,
      "activations/layer22_attention_weight_min": -28.54111671447754,
      "activations/layer23_attention_weight_max": 33.191062927246094,
      "activations/layer23_attention_weight_min": -24.56170654296875,
      "activations/layer2_attention_weight_max": 31.599544525146484,
      "activations/layer2_attention_weight_min": -31.38338279724121,
      "activations/layer3_attention_weight_max": 89.57257080078125,
      "activations/layer3_attention_weight_min": -91.88089752197266,
      "activations/layer4_attention_weight_max": 132.22531127929688,
      "activations/layer4_attention_weight_min": -129.98654174804688,
      "activations/layer5_attention_weight_max": 80.0854263305664,
      "activations/layer5_attention_weight_min": -75.22987365722656,
      "activations/layer6_attention_weight_max": 53.73123550415039,
      "activations/layer6_attention_weight_min": -54.31300735473633,
      "activations/layer7_attention_weight_max": 81.48902893066406,
      "activations/layer7_attention_weight_min": -83.9127426147461,
      "activations/layer8_attention_weight_max": 37.96429443359375,
      "activations/layer8_attention_weight_min": -40.48542404174805,
      "activations/layer9_attention_weight_max": 36.905696868896484,
      "activations/layer9_attention_weight_min": -34.33238983154297,
      "epoch": 5.56,
      "learning_rate": 0.00011531856060606059,
      "loss": 2.8522,
      "step": 95650
    },
    {
      "activations/layer0_attention_weight_max": 16.050235748291016,
      "activations/layer0_attention_weight_min": -12.308104515075684,
      "activations/layer10_attention_weight_max": 37.164306640625,
      "activations/layer10_attention_weight_min": -31.379613876342773,
      "activations/layer11_attention_weight_max": 34.543846130371094,
      "activations/layer11_attention_weight_min": -27.35738182067871,
      "activations/layer12_attention_weight_max": 24.173179626464844,
      "activations/layer12_attention_weight_min": -28.737459182739258,
      "activations/layer13_attention_weight_max": 43.32118606567383,
      "activations/layer13_attention_weight_min": -31.604907989501953,
      "activations/layer14_attention_weight_max": 64.43832397460938,
      "activations/layer14_attention_weight_min": -46.2052001953125,
      "activations/layer15_attention_weight_max": 36.79063034057617,
      "activations/layer15_attention_weight_min": -28.919178009033203,
      "activations/layer16_attention_weight_max": 36.38017272949219,
      "activations/layer16_attention_weight_min": -26.342235565185547,
      "activations/layer17_attention_weight_max": 43.25847244262695,
      "activations/layer17_attention_weight_min": -24.767024993896484,
      "activations/layer18_attention_weight_max": 30.741233825683594,
      "activations/layer18_attention_weight_min": -20.102293014526367,
      "activations/layer19_attention_weight_max": 38.753997802734375,
      "activations/layer19_attention_weight_min": -28.261463165283203,
      "activations/layer1_attention_weight_max": 17.253032684326172,
      "activations/layer1_attention_weight_min": -17.030731201171875,
      "activations/layer20_attention_weight_max": 33.223384857177734,
      "activations/layer20_attention_weight_min": -22.285505294799805,
      "activations/layer21_attention_weight_max": 31.845693588256836,
      "activations/layer21_attention_weight_min": -20.02522850036621,
      "activations/layer22_attention_weight_max": 51.65293502807617,
      "activations/layer22_attention_weight_min": -29.48860740661621,
      "activations/layer23_attention_weight_max": 34.068153381347656,
      "activations/layer23_attention_weight_min": -20.310562133789062,
      "activations/layer2_attention_weight_max": 34.02556228637695,
      "activations/layer2_attention_weight_min": -32.987911224365234,
      "activations/layer3_attention_weight_max": 96.84296417236328,
      "activations/layer3_attention_weight_min": -96.23522186279297,
      "activations/layer4_attention_weight_max": 138.24342346191406,
      "activations/layer4_attention_weight_min": -129.4742889404297,
      "activations/layer5_attention_weight_max": 82.04649353027344,
      "activations/layer5_attention_weight_min": -77.32667541503906,
      "activations/layer6_attention_weight_max": 57.37101745605469,
      "activations/layer6_attention_weight_min": -58.10254669189453,
      "activations/layer7_attention_weight_max": 91.90127563476562,
      "activations/layer7_attention_weight_min": -86.07795715332031,
      "activations/layer8_attention_weight_max": 42.22975540161133,
      "activations/layer8_attention_weight_min": -40.367454528808594,
      "activations/layer9_attention_weight_max": 41.74660873413086,
      "activations/layer9_attention_weight_min": -33.81306457519531,
      "epoch": 5.56,
      "learning_rate": 0.00011529962121212121,
      "loss": 2.8272,
      "step": 95700
    },
    {
      "activations/layer0_attention_weight_max": 16.36504364013672,
      "activations/layer0_attention_weight_min": -12.209495544433594,
      "activations/layer10_attention_weight_max": 30.496238708496094,
      "activations/layer10_attention_weight_min": -29.22359275817871,
      "activations/layer11_attention_weight_max": 30.088008880615234,
      "activations/layer11_attention_weight_min": -27.190101623535156,
      "activations/layer12_attention_weight_max": 20.863422393798828,
      "activations/layer12_attention_weight_min": -26.190311431884766,
      "activations/layer13_attention_weight_max": 44.39051055908203,
      "activations/layer13_attention_weight_min": -32.61320495605469,
      "activations/layer14_attention_weight_max": 51.460174560546875,
      "activations/layer14_attention_weight_min": -44.06793212890625,
      "activations/layer15_attention_weight_max": 33.55005645751953,
      "activations/layer15_attention_weight_min": -27.42728042602539,
      "activations/layer16_attention_weight_max": 32.996673583984375,
      "activations/layer16_attention_weight_min": -26.686832427978516,
      "activations/layer17_attention_weight_max": 34.89778518676758,
      "activations/layer17_attention_weight_min": -23.241634368896484,
      "activations/layer18_attention_weight_max": 29.89165687561035,
      "activations/layer18_attention_weight_min": -20.93256950378418,
      "activations/layer19_attention_weight_max": 33.7081413269043,
      "activations/layer19_attention_weight_min": -28.552642822265625,
      "activations/layer1_attention_weight_max": 17.241439819335938,
      "activations/layer1_attention_weight_min": -13.705987930297852,
      "activations/layer20_attention_weight_max": 28.12081527709961,
      "activations/layer20_attention_weight_min": -23.041229248046875,
      "activations/layer21_attention_weight_max": 29.844161987304688,
      "activations/layer21_attention_weight_min": -20.816761016845703,
      "activations/layer22_attention_weight_max": 44.39727783203125,
      "activations/layer22_attention_weight_min": -28.547183990478516,
      "activations/layer23_attention_weight_max": 32.62876510620117,
      "activations/layer23_attention_weight_min": -23.362056732177734,
      "activations/layer2_attention_weight_max": 31.581205368041992,
      "activations/layer2_attention_weight_min": -30.94363784790039,
      "activations/layer3_attention_weight_max": 91.41858673095703,
      "activations/layer3_attention_weight_min": -93.34453582763672,
      "activations/layer4_attention_weight_max": 127.4961166381836,
      "activations/layer4_attention_weight_min": -126.17766571044922,
      "activations/layer5_attention_weight_max": 79.38246154785156,
      "activations/layer5_attention_weight_min": -77.17681884765625,
      "activations/layer6_attention_weight_max": 52.98081588745117,
      "activations/layer6_attention_weight_min": -53.842716217041016,
      "activations/layer7_attention_weight_max": 86.88330078125,
      "activations/layer7_attention_weight_min": -83.24879455566406,
      "activations/layer8_attention_weight_max": 37.80503845214844,
      "activations/layer8_attention_weight_min": -36.65394973754883,
      "activations/layer9_attention_weight_max": 35.0981559753418,
      "activations/layer9_attention_weight_min": -36.718902587890625,
      "epoch": 5.56,
      "learning_rate": 0.00011528068181818181,
      "loss": 2.8199,
      "step": 95750
    },
    {
      "activations/layer0_attention_weight_max": 16.62152862548828,
      "activations/layer0_attention_weight_min": -12.652352333068848,
      "activations/layer10_attention_weight_max": 34.61560821533203,
      "activations/layer10_attention_weight_min": -31.01003646850586,
      "activations/layer11_attention_weight_max": 33.0032958984375,
      "activations/layer11_attention_weight_min": -27.26396942138672,
      "activations/layer12_attention_weight_max": 42.548583984375,
      "activations/layer12_attention_weight_min": -28.610614776611328,
      "activations/layer13_attention_weight_max": 57.06380081176758,
      "activations/layer13_attention_weight_min": -38.1726188659668,
      "activations/layer14_attention_weight_max": 63.12602615356445,
      "activations/layer14_attention_weight_min": -47.49733352661133,
      "activations/layer15_attention_weight_max": 42.274513244628906,
      "activations/layer15_attention_weight_min": -32.27821731567383,
      "activations/layer16_attention_weight_max": 35.749847412109375,
      "activations/layer16_attention_weight_min": -27.05950355529785,
      "activations/layer17_attention_weight_max": 36.08661651611328,
      "activations/layer17_attention_weight_min": -24.086923599243164,
      "activations/layer18_attention_weight_max": 37.51238250732422,
      "activations/layer18_attention_weight_min": -19.64958953857422,
      "activations/layer19_attention_weight_max": 41.55445098876953,
      "activations/layer19_attention_weight_min": -30.933080673217773,
      "activations/layer1_attention_weight_max": 17.6411075592041,
      "activations/layer1_attention_weight_min": -16.573997497558594,
      "activations/layer20_attention_weight_max": 33.088233947753906,
      "activations/layer20_attention_weight_min": -24.448625564575195,
      "activations/layer21_attention_weight_max": 34.50872802734375,
      "activations/layer21_attention_weight_min": -22.970848083496094,
      "activations/layer22_attention_weight_max": 52.40457534790039,
      "activations/layer22_attention_weight_min": -29.5391845703125,
      "activations/layer23_attention_weight_max": 36.98023223876953,
      "activations/layer23_attention_weight_min": -22.851165771484375,
      "activations/layer2_attention_weight_max": 30.87664794921875,
      "activations/layer2_attention_weight_min": -31.05593490600586,
      "activations/layer3_attention_weight_max": 90.11994171142578,
      "activations/layer3_attention_weight_min": -90.2251968383789,
      "activations/layer4_attention_weight_max": 129.26959228515625,
      "activations/layer4_attention_weight_min": -122.507568359375,
      "activations/layer5_attention_weight_max": 78.14204406738281,
      "activations/layer5_attention_weight_min": -73.96546936035156,
      "activations/layer6_attention_weight_max": 53.55758285522461,
      "activations/layer6_attention_weight_min": -55.91434860229492,
      "activations/layer7_attention_weight_max": 81.01634979248047,
      "activations/layer7_attention_weight_min": -85.63243865966797,
      "activations/layer8_attention_weight_max": 39.39958572387695,
      "activations/layer8_attention_weight_min": -37.00100326538086,
      "activations/layer9_attention_weight_max": 37.8266716003418,
      "activations/layer9_attention_weight_min": -33.27793884277344,
      "epoch": 5.57,
      "learning_rate": 0.00011526174242424241,
      "loss": 2.8327,
      "step": 95800
    },
    {
      "activations/layer0_attention_weight_max": 16.49923324584961,
      "activations/layer0_attention_weight_min": -11.807201385498047,
      "activations/layer10_attention_weight_max": 32.50677490234375,
      "activations/layer10_attention_weight_min": -32.253631591796875,
      "activations/layer11_attention_weight_max": 34.294925689697266,
      "activations/layer11_attention_weight_min": -31.16768455505371,
      "activations/layer12_attention_weight_max": 39.587799072265625,
      "activations/layer12_attention_weight_min": -30.617937088012695,
      "activations/layer13_attention_weight_max": 39.342132568359375,
      "activations/layer13_attention_weight_min": -31.454544067382812,
      "activations/layer14_attention_weight_max": 57.27306365966797,
      "activations/layer14_attention_weight_min": -38.05532455444336,
      "activations/layer15_attention_weight_max": 39.31719207763672,
      "activations/layer15_attention_weight_min": -27.98127555847168,
      "activations/layer16_attention_weight_max": 32.89889907836914,
      "activations/layer16_attention_weight_min": -28.54151153564453,
      "activations/layer17_attention_weight_max": 37.63334655761719,
      "activations/layer17_attention_weight_min": -23.682140350341797,
      "activations/layer18_attention_weight_max": 32.98069381713867,
      "activations/layer18_attention_weight_min": -20.883886337280273,
      "activations/layer19_attention_weight_max": 34.41947555541992,
      "activations/layer19_attention_weight_min": -30.292203903198242,
      "activations/layer1_attention_weight_max": 17.07101821899414,
      "activations/layer1_attention_weight_min": -14.441791534423828,
      "activations/layer20_attention_weight_max": 27.4282283782959,
      "activations/layer20_attention_weight_min": -22.015214920043945,
      "activations/layer21_attention_weight_max": 34.53953552246094,
      "activations/layer21_attention_weight_min": -20.580629348754883,
      "activations/layer22_attention_weight_max": 46.75236892700195,
      "activations/layer22_attention_weight_min": -29.142024993896484,
      "activations/layer23_attention_weight_max": 37.95555114746094,
      "activations/layer23_attention_weight_min": -21.302108764648438,
      "activations/layer2_attention_weight_max": 30.87238311767578,
      "activations/layer2_attention_weight_min": -31.524967193603516,
      "activations/layer3_attention_weight_max": 90.01676177978516,
      "activations/layer3_attention_weight_min": -88.39570617675781,
      "activations/layer4_attention_weight_max": 127.4827651977539,
      "activations/layer4_attention_weight_min": -125.44874572753906,
      "activations/layer5_attention_weight_max": 75.33377075195312,
      "activations/layer5_attention_weight_min": -76.03553771972656,
      "activations/layer6_attention_weight_max": 54.93849563598633,
      "activations/layer6_attention_weight_min": -57.57390594482422,
      "activations/layer7_attention_weight_max": 86.82166290283203,
      "activations/layer7_attention_weight_min": -87.29051208496094,
      "activations/layer8_attention_weight_max": 38.469913482666016,
      "activations/layer8_attention_weight_min": -37.30721664428711,
      "activations/layer9_attention_weight_max": 35.953792572021484,
      "activations/layer9_attention_weight_min": -38.85081481933594,
      "epoch": 5.57,
      "learning_rate": 0.00011524280303030303,
      "loss": 2.8361,
      "step": 95850
    },
    {
      "activations/layer0_attention_weight_max": 16.502004623413086,
      "activations/layer0_attention_weight_min": -11.814367294311523,
      "activations/layer10_attention_weight_max": 33.169769287109375,
      "activations/layer10_attention_weight_min": -29.50286102294922,
      "activations/layer11_attention_weight_max": 33.895484924316406,
      "activations/layer11_attention_weight_min": -26.53949546813965,
      "activations/layer12_attention_weight_max": 26.32124137878418,
      "activations/layer12_attention_weight_min": -25.794357299804688,
      "activations/layer13_attention_weight_max": 41.5247802734375,
      "activations/layer13_attention_weight_min": -32.18006134033203,
      "activations/layer14_attention_weight_max": 53.29258346557617,
      "activations/layer14_attention_weight_min": -37.69966125488281,
      "activations/layer15_attention_weight_max": 40.342002868652344,
      "activations/layer15_attention_weight_min": -26.769065856933594,
      "activations/layer16_attention_weight_max": 35.625457763671875,
      "activations/layer16_attention_weight_min": -28.938182830810547,
      "activations/layer17_attention_weight_max": 38.68288803100586,
      "activations/layer17_attention_weight_min": -23.199216842651367,
      "activations/layer18_attention_weight_max": 33.854644775390625,
      "activations/layer18_attention_weight_min": -20.02801513671875,
      "activations/layer19_attention_weight_max": 36.90062713623047,
      "activations/layer19_attention_weight_min": -29.363407135009766,
      "activations/layer1_attention_weight_max": 17.249513626098633,
      "activations/layer1_attention_weight_min": -15.009916305541992,
      "activations/layer20_attention_weight_max": 30.602399826049805,
      "activations/layer20_attention_weight_min": -22.656185150146484,
      "activations/layer21_attention_weight_max": 36.35672378540039,
      "activations/layer21_attention_weight_min": -20.371231079101562,
      "activations/layer22_attention_weight_max": 47.8162956237793,
      "activations/layer22_attention_weight_min": -28.82291030883789,
      "activations/layer23_attention_weight_max": 34.322654724121094,
      "activations/layer23_attention_weight_min": -20.959735870361328,
      "activations/layer2_attention_weight_max": 31.271820068359375,
      "activations/layer2_attention_weight_min": -30.40462875366211,
      "activations/layer3_attention_weight_max": 86.8309555053711,
      "activations/layer3_attention_weight_min": -93.21543884277344,
      "activations/layer4_attention_weight_max": 129.87399291992188,
      "activations/layer4_attention_weight_min": -122.4064712524414,
      "activations/layer5_attention_weight_max": 74.57794189453125,
      "activations/layer5_attention_weight_min": -71.58666229248047,
      "activations/layer6_attention_weight_max": 54.033302307128906,
      "activations/layer6_attention_weight_min": -54.046817779541016,
      "activations/layer7_attention_weight_max": 80.85142517089844,
      "activations/layer7_attention_weight_min": -84.56155395507812,
      "activations/layer8_attention_weight_max": 40.142452239990234,
      "activations/layer8_attention_weight_min": -38.61418533325195,
      "activations/layer9_attention_weight_max": 34.990440368652344,
      "activations/layer9_attention_weight_min": -32.86297607421875,
      "epoch": 5.57,
      "learning_rate": 0.00011522386363636363,
      "loss": 2.8364,
      "step": 95900
    },
    {
      "activations/layer0_attention_weight_max": 15.77347469329834,
      "activations/layer0_attention_weight_min": -12.26463794708252,
      "activations/layer10_attention_weight_max": 32.04317092895508,
      "activations/layer10_attention_weight_min": -30.04144287109375,
      "activations/layer11_attention_weight_max": 30.076330184936523,
      "activations/layer11_attention_weight_min": -26.191940307617188,
      "activations/layer12_attention_weight_max": 27.55927276611328,
      "activations/layer12_attention_weight_min": -27.307764053344727,
      "activations/layer13_attention_weight_max": 39.51386260986328,
      "activations/layer13_attention_weight_min": -31.6460018157959,
      "activations/layer14_attention_weight_max": 54.100433349609375,
      "activations/layer14_attention_weight_min": -41.33961868286133,
      "activations/layer15_attention_weight_max": 35.01197814941406,
      "activations/layer15_attention_weight_min": -28.64472007751465,
      "activations/layer16_attention_weight_max": 35.52440643310547,
      "activations/layer16_attention_weight_min": -28.217636108398438,
      "activations/layer17_attention_weight_max": 35.00507736206055,
      "activations/layer17_attention_weight_min": -25.48932456970215,
      "activations/layer18_attention_weight_max": 32.81437683105469,
      "activations/layer18_attention_weight_min": -21.324954986572266,
      "activations/layer19_attention_weight_max": 43.052947998046875,
      "activations/layer19_attention_weight_min": -29.15740394592285,
      "activations/layer1_attention_weight_max": 16.376842498779297,
      "activations/layer1_attention_weight_min": -14.86915397644043,
      "activations/layer20_attention_weight_max": 33.67688751220703,
      "activations/layer20_attention_weight_min": -24.716079711914062,
      "activations/layer21_attention_weight_max": 30.939104080200195,
      "activations/layer21_attention_weight_min": -20.39900779724121,
      "activations/layer22_attention_weight_max": 50.49346923828125,
      "activations/layer22_attention_weight_min": -29.53965187072754,
      "activations/layer23_attention_weight_max": 34.43012237548828,
      "activations/layer23_attention_weight_min": -21.97670555114746,
      "activations/layer2_attention_weight_max": 30.71182632446289,
      "activations/layer2_attention_weight_min": -30.9038143157959,
      "activations/layer3_attention_weight_max": 88.7992935180664,
      "activations/layer3_attention_weight_min": -91.5142593383789,
      "activations/layer4_attention_weight_max": 128.6072235107422,
      "activations/layer4_attention_weight_min": -125.02192687988281,
      "activations/layer5_attention_weight_max": 77.52493286132812,
      "activations/layer5_attention_weight_min": -75.8061294555664,
      "activations/layer6_attention_weight_max": 59.5262565612793,
      "activations/layer6_attention_weight_min": -56.61587905883789,
      "activations/layer7_attention_weight_max": 89.90391540527344,
      "activations/layer7_attention_weight_min": -82.70220184326172,
      "activations/layer8_attention_weight_max": 40.23186111450195,
      "activations/layer8_attention_weight_min": -43.10329055786133,
      "activations/layer9_attention_weight_max": 35.1425666809082,
      "activations/layer9_attention_weight_min": -34.0265007019043,
      "epoch": 5.58,
      "learning_rate": 0.00011520492424242423,
      "loss": 2.8386,
      "step": 95950
    },
    {
      "activations/layer0_attention_weight_max": 16.10973358154297,
      "activations/layer0_attention_weight_min": -11.115073204040527,
      "activations/layer10_attention_weight_max": 33.244544982910156,
      "activations/layer10_attention_weight_min": -30.405309677124023,
      "activations/layer11_attention_weight_max": 32.329959869384766,
      "activations/layer11_attention_weight_min": -28.756362915039062,
      "activations/layer12_attention_weight_max": 29.43274688720703,
      "activations/layer12_attention_weight_min": -30.133312225341797,
      "activations/layer13_attention_weight_max": 48.57430648803711,
      "activations/layer13_attention_weight_min": -33.19291305541992,
      "activations/layer14_attention_weight_max": 64.39617156982422,
      "activations/layer14_attention_weight_min": -49.83668899536133,
      "activations/layer15_attention_weight_max": 37.412498474121094,
      "activations/layer15_attention_weight_min": -28.043052673339844,
      "activations/layer16_attention_weight_max": 37.203311920166016,
      "activations/layer16_attention_weight_min": -27.381362915039062,
      "activations/layer17_attention_weight_max": 38.478397369384766,
      "activations/layer17_attention_weight_min": -24.22381019592285,
      "activations/layer18_attention_weight_max": 38.34910202026367,
      "activations/layer18_attention_weight_min": -20.047147750854492,
      "activations/layer19_attention_weight_max": 37.72417449951172,
      "activations/layer19_attention_weight_min": -27.591161727905273,
      "activations/layer1_attention_weight_max": 17.608226776123047,
      "activations/layer1_attention_weight_min": -15.228694915771484,
      "activations/layer20_attention_weight_max": 32.67006301879883,
      "activations/layer20_attention_weight_min": -22.528209686279297,
      "activations/layer21_attention_weight_max": 29.406478881835938,
      "activations/layer21_attention_weight_min": -20.362503051757812,
      "activations/layer22_attention_weight_max": 53.49338150024414,
      "activations/layer22_attention_weight_min": -26.419166564941406,
      "activations/layer23_attention_weight_max": 39.25031661987305,
      "activations/layer23_attention_weight_min": -21.49722671508789,
      "activations/layer2_attention_weight_max": 31.33529281616211,
      "activations/layer2_attention_weight_min": -31.430953979492188,
      "activations/layer3_attention_weight_max": 92.59561920166016,
      "activations/layer3_attention_weight_min": -93.50691223144531,
      "activations/layer4_attention_weight_max": 127.3215560913086,
      "activations/layer4_attention_weight_min": -129.1092529296875,
      "activations/layer5_attention_weight_max": 80.15615844726562,
      "activations/layer5_attention_weight_min": -79.48872375488281,
      "activations/layer6_attention_weight_max": 55.766761779785156,
      "activations/layer6_attention_weight_min": -61.9551887512207,
      "activations/layer7_attention_weight_max": 89.4986801147461,
      "activations/layer7_attention_weight_min": -93.85900115966797,
      "activations/layer8_attention_weight_max": 42.90150833129883,
      "activations/layer8_attention_weight_min": -44.96848678588867,
      "activations/layer9_attention_weight_max": 36.48470687866211,
      "activations/layer9_attention_weight_min": -35.40603256225586,
      "epoch": 5.58,
      "learning_rate": 0.00011518598484848484,
      "loss": 2.8454,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.4445,
      "eval_samples_per_second": 508.497,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.4445,
      "eval_openwebtext_samples_per_second": 508.497,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_wikitext_loss": 3.017578125,
      "eval_wikitext_ppl": 20.44172438427552,
      "eval_wikitext_runtime": 1.9265,
      "eval_wikitext_samples_per_second": 236.699,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5414,
      "eval_lambada_samples_per_second": 510.302,
      "step": 96000
    },
    {
      "activations/layer0_attention_weight_max": 16.312646865844727,
      "activations/layer0_attention_weight_min": -12.995111465454102,
      "activations/layer10_attention_weight_max": 36.24174880981445,
      "activations/layer10_attention_weight_min": -30.754955291748047,
      "activations/layer11_attention_weight_max": 35.169918060302734,
      "activations/layer11_attention_weight_min": -29.044675827026367,
      "activations/layer12_attention_weight_max": 63.62712860107422,
      "activations/layer12_attention_weight_min": -27.708335876464844,
      "activations/layer13_attention_weight_max": 59.07490921020508,
      "activations/layer13_attention_weight_min": -29.850994110107422,
      "activations/layer14_attention_weight_max": 64.08577728271484,
      "activations/layer14_attention_weight_min": -39.669578552246094,
      "activations/layer15_attention_weight_max": 41.43613815307617,
      "activations/layer15_attention_weight_min": -27.175310134887695,
      "activations/layer16_attention_weight_max": 36.12349319458008,
      "activations/layer16_attention_weight_min": -27.14451789855957,
      "activations/layer17_attention_weight_max": 42.78297805786133,
      "activations/layer17_attention_weight_min": -24.411270141601562,
      "activations/layer18_attention_weight_max": 29.120811462402344,
      "activations/layer18_attention_weight_min": -20.02473258972168,
      "activations/layer19_attention_weight_max": 37.881324768066406,
      "activations/layer19_attention_weight_min": -28.393207550048828,
      "activations/layer1_attention_weight_max": 17.230627059936523,
      "activations/layer1_attention_weight_min": -14.939833641052246,
      "activations/layer20_attention_weight_max": 33.91412353515625,
      "activations/layer20_attention_weight_min": -23.186552047729492,
      "activations/layer21_attention_weight_max": 33.0242919921875,
      "activations/layer21_attention_weight_min": -19.9153995513916,
      "activations/layer22_attention_weight_max": 50.15147018432617,
      "activations/layer22_attention_weight_min": -28.208154678344727,
      "activations/layer23_attention_weight_max": 33.792694091796875,
      "activations/layer23_attention_weight_min": -22.583599090576172,
      "activations/layer2_attention_weight_max": 32.73005676269531,
      "activations/layer2_attention_weight_min": -32.86217498779297,
      "activations/layer3_attention_weight_max": 92.18994903564453,
      "activations/layer3_attention_weight_min": -93.10822296142578,
      "activations/layer4_attention_weight_max": 131.7898406982422,
      "activations/layer4_attention_weight_min": -129.62646484375,
      "activations/layer5_attention_weight_max": 77.45834350585938,
      "activations/layer5_attention_weight_min": -79.13500213623047,
      "activations/layer6_attention_weight_max": 55.39744567871094,
      "activations/layer6_attention_weight_min": -58.751068115234375,
      "activations/layer7_attention_weight_max": 84.18273162841797,
      "activations/layer7_attention_weight_min": -90.84329986572266,
      "activations/layer8_attention_weight_max": 41.15498733520508,
      "activations/layer8_attention_weight_min": -42.15357208251953,
      "activations/layer9_attention_weight_max": 36.7506217956543,
      "activations/layer9_attention_weight_min": -40.95051574707031,
      "epoch": 5.58,
      "learning_rate": 0.00011516704545454543,
      "loss": 2.8428,
      "step": 96050
    },
    {
      "activations/layer0_attention_weight_max": 15.591403007507324,
      "activations/layer0_attention_weight_min": -12.485000610351562,
      "activations/layer10_attention_weight_max": 35.50758743286133,
      "activations/layer10_attention_weight_min": -33.735572814941406,
      "activations/layer11_attention_weight_max": 34.787322998046875,
      "activations/layer11_attention_weight_min": -28.76215362548828,
      "activations/layer12_attention_weight_max": 27.4809627532959,
      "activations/layer12_attention_weight_min": -27.301834106445312,
      "activations/layer13_attention_weight_max": 44.363121032714844,
      "activations/layer13_attention_weight_min": -34.90536880493164,
      "activations/layer14_attention_weight_max": 75.89907836914062,
      "activations/layer14_attention_weight_min": -56.00349044799805,
      "activations/layer15_attention_weight_max": 40.584571838378906,
      "activations/layer15_attention_weight_min": -28.36290740966797,
      "activations/layer16_attention_weight_max": 39.78756332397461,
      "activations/layer16_attention_weight_min": -27.265949249267578,
      "activations/layer17_attention_weight_max": 41.86400604248047,
      "activations/layer17_attention_weight_min": -25.017711639404297,
      "activations/layer18_attention_weight_max": 34.81128692626953,
      "activations/layer18_attention_weight_min": -18.801359176635742,
      "activations/layer19_attention_weight_max": 42.35244369506836,
      "activations/layer19_attention_weight_min": -28.063232421875,
      "activations/layer1_attention_weight_max": 17.51047134399414,
      "activations/layer1_attention_weight_min": -14.637537956237793,
      "activations/layer20_attention_weight_max": 32.35883331298828,
      "activations/layer20_attention_weight_min": -23.714391708374023,
      "activations/layer21_attention_weight_max": 39.30039978027344,
      "activations/layer21_attention_weight_min": -20.310993194580078,
      "activations/layer22_attention_weight_max": 57.98637771606445,
      "activations/layer22_attention_weight_min": -28.52357292175293,
      "activations/layer23_attention_weight_max": 35.388214111328125,
      "activations/layer23_attention_weight_min": -17.91828155517578,
      "activations/layer2_attention_weight_max": 32.964332580566406,
      "activations/layer2_attention_weight_min": -32.81165313720703,
      "activations/layer3_attention_weight_max": 93.7994613647461,
      "activations/layer3_attention_weight_min": -95.49758911132812,
      "activations/layer4_attention_weight_max": 134.02935791015625,
      "activations/layer4_attention_weight_min": -133.4855194091797,
      "activations/layer5_attention_weight_max": 83.04917907714844,
      "activations/layer5_attention_weight_min": -80.07231140136719,
      "activations/layer6_attention_weight_max": 55.640716552734375,
      "activations/layer6_attention_weight_min": -59.17816162109375,
      "activations/layer7_attention_weight_max": 89.29346466064453,
      "activations/layer7_attention_weight_min": -90.25276184082031,
      "activations/layer8_attention_weight_max": 42.33623123168945,
      "activations/layer8_attention_weight_min": -41.38031768798828,
      "activations/layer9_attention_weight_max": 39.692481994628906,
      "activations/layer9_attention_weight_min": -38.479549407958984,
      "epoch": 5.58,
      "learning_rate": 0.00011514810606060605,
      "loss": 2.8426,
      "step": 96100
    },
    {
      "activations/layer0_attention_weight_max": 17.202117919921875,
      "activations/layer0_attention_weight_min": -11.301955223083496,
      "activations/layer10_attention_weight_max": 34.903900146484375,
      "activations/layer10_attention_weight_min": -32.874603271484375,
      "activations/layer11_attention_weight_max": 32.88282775878906,
      "activations/layer11_attention_weight_min": -29.30572509765625,
      "activations/layer12_attention_weight_max": 30.48280143737793,
      "activations/layer12_attention_weight_min": -27.481098175048828,
      "activations/layer13_attention_weight_max": 45.194236755371094,
      "activations/layer13_attention_weight_min": -32.99183654785156,
      "activations/layer14_attention_weight_max": 61.58995056152344,
      "activations/layer14_attention_weight_min": -50.55377960205078,
      "activations/layer15_attention_weight_max": 34.41804122924805,
      "activations/layer15_attention_weight_min": -28.38759994506836,
      "activations/layer16_attention_weight_max": 37.280967712402344,
      "activations/layer16_attention_weight_min": -29.242029190063477,
      "activations/layer17_attention_weight_max": 35.16194534301758,
      "activations/layer17_attention_weight_min": -24.655319213867188,
      "activations/layer18_attention_weight_max": 31.82904052734375,
      "activations/layer18_attention_weight_min": -21.295698165893555,
      "activations/layer19_attention_weight_max": 37.381927490234375,
      "activations/layer19_attention_weight_min": -28.19129753112793,
      "activations/layer1_attention_weight_max": 18.976526260375977,
      "activations/layer1_attention_weight_min": -16.553482055664062,
      "activations/layer20_attention_weight_max": 34.08154296875,
      "activations/layer20_attention_weight_min": -23.107053756713867,
      "activations/layer21_attention_weight_max": 34.364376068115234,
      "activations/layer21_attention_weight_min": -21.011165618896484,
      "activations/layer22_attention_weight_max": 51.03260803222656,
      "activations/layer22_attention_weight_min": -29.570453643798828,
      "activations/layer23_attention_weight_max": 36.772796630859375,
      "activations/layer23_attention_weight_min": -20.969573974609375,
      "activations/layer2_attention_weight_max": 35.74611282348633,
      "activations/layer2_attention_weight_min": -33.768524169921875,
      "activations/layer3_attention_weight_max": 98.77945709228516,
      "activations/layer3_attention_weight_min": -101.00270080566406,
      "activations/layer4_attention_weight_max": 137.23585510253906,
      "activations/layer4_attention_weight_min": -132.90208435058594,
      "activations/layer5_attention_weight_max": 82.79015350341797,
      "activations/layer5_attention_weight_min": -78.90107727050781,
      "activations/layer6_attention_weight_max": 61.69835662841797,
      "activations/layer6_attention_weight_min": -59.04410171508789,
      "activations/layer7_attention_weight_max": 89.91607666015625,
      "activations/layer7_attention_weight_min": -89.61604309082031,
      "activations/layer8_attention_weight_max": 40.11549758911133,
      "activations/layer8_attention_weight_min": -43.36949157714844,
      "activations/layer9_attention_weight_max": 37.095706939697266,
      "activations/layer9_attention_weight_min": -37.899417877197266,
      "epoch": 5.59,
      "learning_rate": 0.00011512916666666666,
      "loss": 2.8363,
      "step": 96150
    },
    {
      "activations/layer0_attention_weight_max": 15.299915313720703,
      "activations/layer0_attention_weight_min": -12.996509552001953,
      "activations/layer10_attention_weight_max": 34.35992431640625,
      "activations/layer10_attention_weight_min": -32.6913948059082,
      "activations/layer11_attention_weight_max": 33.65890884399414,
      "activations/layer11_attention_weight_min": -29.8089656829834,
      "activations/layer12_attention_weight_max": 28.369247436523438,
      "activations/layer12_attention_weight_min": -25.78714942932129,
      "activations/layer13_attention_weight_max": 46.02172088623047,
      "activations/layer13_attention_weight_min": -35.041259765625,
      "activations/layer14_attention_weight_max": 76.64170837402344,
      "activations/layer14_attention_weight_min": -61.10741424560547,
      "activations/layer15_attention_weight_max": 41.55845260620117,
      "activations/layer15_attention_weight_min": -28.58321189880371,
      "activations/layer16_attention_weight_max": 38.54605484008789,
      "activations/layer16_attention_weight_min": -29.48813819885254,
      "activations/layer17_attention_weight_max": 41.00801086425781,
      "activations/layer17_attention_weight_min": -26.29796600341797,
      "activations/layer18_attention_weight_max": 31.162269592285156,
      "activations/layer18_attention_weight_min": -20.207597732543945,
      "activations/layer19_attention_weight_max": 41.72714614868164,
      "activations/layer19_attention_weight_min": -29.734119415283203,
      "activations/layer1_attention_weight_max": 17.33185577392578,
      "activations/layer1_attention_weight_min": -13.79194450378418,
      "activations/layer20_attention_weight_max": 33.6542854309082,
      "activations/layer20_attention_weight_min": -24.77151107788086,
      "activations/layer21_attention_weight_max": 31.927249908447266,
      "activations/layer21_attention_weight_min": -21.300918579101562,
      "activations/layer22_attention_weight_max": 52.0914421081543,
      "activations/layer22_attention_weight_min": -28.69244956970215,
      "activations/layer23_attention_weight_max": 35.33873748779297,
      "activations/layer23_attention_weight_min": -19.394840240478516,
      "activations/layer2_attention_weight_max": 31.409786224365234,
      "activations/layer2_attention_weight_min": -31.185802459716797,
      "activations/layer3_attention_weight_max": 91.44160461425781,
      "activations/layer3_attention_weight_min": -91.01244354248047,
      "activations/layer4_attention_weight_max": 129.9302215576172,
      "activations/layer4_attention_weight_min": -123.8059310913086,
      "activations/layer5_attention_weight_max": 75.12095642089844,
      "activations/layer5_attention_weight_min": -72.88478088378906,
      "activations/layer6_attention_weight_max": 55.681156158447266,
      "activations/layer6_attention_weight_min": -57.53395462036133,
      "activations/layer7_attention_weight_max": 84.65837860107422,
      "activations/layer7_attention_weight_min": -87.12274932861328,
      "activations/layer8_attention_weight_max": 40.307125091552734,
      "activations/layer8_attention_weight_min": -41.24646759033203,
      "activations/layer9_attention_weight_max": 36.813175201416016,
      "activations/layer9_attention_weight_min": -35.13334655761719,
      "epoch": 5.59,
      "learning_rate": 0.00011511022727272726,
      "loss": 2.85,
      "step": 96200
    },
    {
      "activations/layer0_attention_weight_max": 16.229183197021484,
      "activations/layer0_attention_weight_min": -13.08730411529541,
      "activations/layer10_attention_weight_max": 34.71381759643555,
      "activations/layer10_attention_weight_min": -30.266719818115234,
      "activations/layer11_attention_weight_max": 33.750282287597656,
      "activations/layer11_attention_weight_min": -27.36903190612793,
      "activations/layer12_attention_weight_max": 28.484533309936523,
      "activations/layer12_attention_weight_min": -29.276782989501953,
      "activations/layer13_attention_weight_max": 36.72383117675781,
      "activations/layer13_attention_weight_min": -30.748809814453125,
      "activations/layer14_attention_weight_max": 50.85870361328125,
      "activations/layer14_attention_weight_min": -42.427818298339844,
      "activations/layer15_attention_weight_max": 34.9281120300293,
      "activations/layer15_attention_weight_min": -28.455589294433594,
      "activations/layer16_attention_weight_max": 33.729942321777344,
      "activations/layer16_attention_weight_min": -26.80936622619629,
      "activations/layer17_attention_weight_max": 36.75794982910156,
      "activations/layer17_attention_weight_min": -24.120914459228516,
      "activations/layer18_attention_weight_max": 29.3394832611084,
      "activations/layer18_attention_weight_min": -21.02059555053711,
      "activations/layer19_attention_weight_max": 42.28639602661133,
      "activations/layer19_attention_weight_min": -31.26114273071289,
      "activations/layer1_attention_weight_max": 17.9306640625,
      "activations/layer1_attention_weight_min": -14.651073455810547,
      "activations/layer20_attention_weight_max": 31.83611297607422,
      "activations/layer20_attention_weight_min": -27.438608169555664,
      "activations/layer21_attention_weight_max": 28.63702964782715,
      "activations/layer21_attention_weight_min": -20.291851043701172,
      "activations/layer22_attention_weight_max": 51.19361877441406,
      "activations/layer22_attention_weight_min": -29.700477600097656,
      "activations/layer23_attention_weight_max": 39.852630615234375,
      "activations/layer23_attention_weight_min": -24.43053436279297,
      "activations/layer2_attention_weight_max": 31.142333984375,
      "activations/layer2_attention_weight_min": -31.236318588256836,
      "activations/layer3_attention_weight_max": 89.64501953125,
      "activations/layer3_attention_weight_min": -87.1832504272461,
      "activations/layer4_attention_weight_max": 124.02251434326172,
      "activations/layer4_attention_weight_min": -118.36925506591797,
      "activations/layer5_attention_weight_max": 74.45831298828125,
      "activations/layer5_attention_weight_min": -71.84784698486328,
      "activations/layer6_attention_weight_max": 55.34778594970703,
      "activations/layer6_attention_weight_min": -55.41063690185547,
      "activations/layer7_attention_weight_max": 88.49076843261719,
      "activations/layer7_attention_weight_min": -86.29505157470703,
      "activations/layer8_attention_weight_max": 41.680641174316406,
      "activations/layer8_attention_weight_min": -38.441978454589844,
      "activations/layer9_attention_weight_max": 37.12379837036133,
      "activations/layer9_attention_weight_min": -33.436370849609375,
      "epoch": 5.59,
      "learning_rate": 0.00011509128787878786,
      "loss": 2.8378,
      "step": 96250
    },
    {
      "activations/layer0_attention_weight_max": 16.931467056274414,
      "activations/layer0_attention_weight_min": -12.762391090393066,
      "activations/layer10_attention_weight_max": 33.956687927246094,
      "activations/layer10_attention_weight_min": -31.45664405822754,
      "activations/layer11_attention_weight_max": 34.67512130737305,
      "activations/layer11_attention_weight_min": -29.5151424407959,
      "activations/layer12_attention_weight_max": 40.35611343383789,
      "activations/layer12_attention_weight_min": -27.067859649658203,
      "activations/layer13_attention_weight_max": 43.551151275634766,
      "activations/layer13_attention_weight_min": -32.764530181884766,
      "activations/layer14_attention_weight_max": 59.63714599609375,
      "activations/layer14_attention_weight_min": -48.69953536987305,
      "activations/layer15_attention_weight_max": 35.77610397338867,
      "activations/layer15_attention_weight_min": -28.93975257873535,
      "activations/layer16_attention_weight_max": 39.688133239746094,
      "activations/layer16_attention_weight_min": -26.751462936401367,
      "activations/layer17_attention_weight_max": 37.04353713989258,
      "activations/layer17_attention_weight_min": -24.267690658569336,
      "activations/layer18_attention_weight_max": 31.054100036621094,
      "activations/layer18_attention_weight_min": -20.178054809570312,
      "activations/layer19_attention_weight_max": 44.85136795043945,
      "activations/layer19_attention_weight_min": -30.500268936157227,
      "activations/layer1_attention_weight_max": 17.665306091308594,
      "activations/layer1_attention_weight_min": -13.710661888122559,
      "activations/layer20_attention_weight_max": 32.33858871459961,
      "activations/layer20_attention_weight_min": -25.617259979248047,
      "activations/layer21_attention_weight_max": 32.34421920776367,
      "activations/layer21_attention_weight_min": -21.850317001342773,
      "activations/layer22_attention_weight_max": 53.11080551147461,
      "activations/layer22_attention_weight_min": -29.01447868347168,
      "activations/layer23_attention_weight_max": 41.433837890625,
      "activations/layer23_attention_weight_min": -22.078887939453125,
      "activations/layer2_attention_weight_max": 31.923999786376953,
      "activations/layer2_attention_weight_min": -30.90114974975586,
      "activations/layer3_attention_weight_max": 91.45481872558594,
      "activations/layer3_attention_weight_min": -90.91233825683594,
      "activations/layer4_attention_weight_max": 131.71339416503906,
      "activations/layer4_attention_weight_min": -126.1305160522461,
      "activations/layer5_attention_weight_max": 79.12818908691406,
      "activations/layer5_attention_weight_min": -76.7164306640625,
      "activations/layer6_attention_weight_max": 56.91523742675781,
      "activations/layer6_attention_weight_min": -60.39474868774414,
      "activations/layer7_attention_weight_max": 88.39425659179688,
      "activations/layer7_attention_weight_min": -88.32400512695312,
      "activations/layer8_attention_weight_max": 40.43010330200195,
      "activations/layer8_attention_weight_min": -42.614444732666016,
      "activations/layer9_attention_weight_max": 38.41521072387695,
      "activations/layer9_attention_weight_min": -34.165321350097656,
      "epoch": 5.6,
      "learning_rate": 0.00011507234848484848,
      "loss": 2.8468,
      "step": 96300
    },
    {
      "activations/layer0_attention_weight_max": 16.733360290527344,
      "activations/layer0_attention_weight_min": -12.012085914611816,
      "activations/layer10_attention_weight_max": 38.15328598022461,
      "activations/layer10_attention_weight_min": -35.98576736450195,
      "activations/layer11_attention_weight_max": 33.02234649658203,
      "activations/layer11_attention_weight_min": -29.771650314331055,
      "activations/layer12_attention_weight_max": 27.408401489257812,
      "activations/layer12_attention_weight_min": -28.621047973632812,
      "activations/layer13_attention_weight_max": 46.67862319946289,
      "activations/layer13_attention_weight_min": -33.570777893066406,
      "activations/layer14_attention_weight_max": 68.88453674316406,
      "activations/layer14_attention_weight_min": -48.02109909057617,
      "activations/layer15_attention_weight_max": 44.697147369384766,
      "activations/layer15_attention_weight_min": -28.791576385498047,
      "activations/layer16_attention_weight_max": 38.7658576965332,
      "activations/layer16_attention_weight_min": -31.60517120361328,
      "activations/layer17_attention_weight_max": 39.51493453979492,
      "activations/layer17_attention_weight_min": -25.89813804626465,
      "activations/layer18_attention_weight_max": 34.72697448730469,
      "activations/layer18_attention_weight_min": -20.041976928710938,
      "activations/layer19_attention_weight_max": 40.782257080078125,
      "activations/layer19_attention_weight_min": -30.237720489501953,
      "activations/layer1_attention_weight_max": 16.548337936401367,
      "activations/layer1_attention_weight_min": -15.595804214477539,
      "activations/layer20_attention_weight_max": 33.29989242553711,
      "activations/layer20_attention_weight_min": -24.337499618530273,
      "activations/layer21_attention_weight_max": 32.35783767700195,
      "activations/layer21_attention_weight_min": -21.514320373535156,
      "activations/layer22_attention_weight_max": 51.32730484008789,
      "activations/layer22_attention_weight_min": -29.540626525878906,
      "activations/layer23_attention_weight_max": 37.19978332519531,
      "activations/layer23_attention_weight_min": -22.72406578063965,
      "activations/layer2_attention_weight_max": 33.067787170410156,
      "activations/layer2_attention_weight_min": -32.458251953125,
      "activations/layer3_attention_weight_max": 98.67657470703125,
      "activations/layer3_attention_weight_min": -95.790771484375,
      "activations/layer4_attention_weight_max": 136.0349578857422,
      "activations/layer4_attention_weight_min": -131.41696166992188,
      "activations/layer5_attention_weight_max": 84.14091491699219,
      "activations/layer5_attention_weight_min": -79.02392578125,
      "activations/layer6_attention_weight_max": 58.173866271972656,
      "activations/layer6_attention_weight_min": -61.05400466918945,
      "activations/layer7_attention_weight_max": 94.62785339355469,
      "activations/layer7_attention_weight_min": -92.45333862304688,
      "activations/layer8_attention_weight_max": 45.58307647705078,
      "activations/layer8_attention_weight_min": -45.703521728515625,
      "activations/layer9_attention_weight_max": 41.71513366699219,
      "activations/layer9_attention_weight_min": -39.557804107666016,
      "epoch": 5.6,
      "learning_rate": 0.00011505340909090908,
      "loss": 2.844,
      "step": 96350
    },
    {
      "activations/layer0_attention_weight_max": 16.162410736083984,
      "activations/layer0_attention_weight_min": -13.059467315673828,
      "activations/layer10_attention_weight_max": 33.17262649536133,
      "activations/layer10_attention_weight_min": -31.827707290649414,
      "activations/layer11_attention_weight_max": 33.52428436279297,
      "activations/layer11_attention_weight_min": -27.865802764892578,
      "activations/layer12_attention_weight_max": 34.92558670043945,
      "activations/layer12_attention_weight_min": -26.2241153717041,
      "activations/layer13_attention_weight_max": 40.82719039916992,
      "activations/layer13_attention_weight_min": -32.93171691894531,
      "activations/layer14_attention_weight_max": 64.30645751953125,
      "activations/layer14_attention_weight_min": -48.1504020690918,
      "activations/layer15_attention_weight_max": 41.95499038696289,
      "activations/layer15_attention_weight_min": -27.442874908447266,
      "activations/layer16_attention_weight_max": 42.11519241333008,
      "activations/layer16_attention_weight_min": -27.37911033630371,
      "activations/layer17_attention_weight_max": 38.00808334350586,
      "activations/layer17_attention_weight_min": -24.37914276123047,
      "activations/layer18_attention_weight_max": 31.129413604736328,
      "activations/layer18_attention_weight_min": -21.238874435424805,
      "activations/layer19_attention_weight_max": 43.65825271606445,
      "activations/layer19_attention_weight_min": -29.784793853759766,
      "activations/layer1_attention_weight_max": 18.72203826904297,
      "activations/layer1_attention_weight_min": -14.70154094696045,
      "activations/layer20_attention_weight_max": 32.87309265136719,
      "activations/layer20_attention_weight_min": -23.16826629638672,
      "activations/layer21_attention_weight_max": 32.65911865234375,
      "activations/layer21_attention_weight_min": -21.86942481994629,
      "activations/layer22_attention_weight_max": 52.58250427246094,
      "activations/layer22_attention_weight_min": -29.99793243408203,
      "activations/layer23_attention_weight_max": 37.8046760559082,
      "activations/layer23_attention_weight_min": -21.602237701416016,
      "activations/layer2_attention_weight_max": 31.522647857666016,
      "activations/layer2_attention_weight_min": -33.18067932128906,
      "activations/layer3_attention_weight_max": 87.18407440185547,
      "activations/layer3_attention_weight_min": -87.92911529541016,
      "activations/layer4_attention_weight_max": 124.74971771240234,
      "activations/layer4_attention_weight_min": -122.44449615478516,
      "activations/layer5_attention_weight_max": 75.30845642089844,
      "activations/layer5_attention_weight_min": -77.6708984375,
      "activations/layer6_attention_weight_max": 54.80093002319336,
      "activations/layer6_attention_weight_min": -61.610233306884766,
      "activations/layer7_attention_weight_max": 88.29279327392578,
      "activations/layer7_attention_weight_min": -94.15496826171875,
      "activations/layer8_attention_weight_max": 38.364444732666016,
      "activations/layer8_attention_weight_min": -41.477516174316406,
      "activations/layer9_attention_weight_max": 35.81749725341797,
      "activations/layer9_attention_weight_min": -36.20494842529297,
      "epoch": 5.6,
      "learning_rate": 0.00011503446969696968,
      "loss": 2.8436,
      "step": 96400
    },
    {
      "activations/layer0_attention_weight_max": 15.937103271484375,
      "activations/layer0_attention_weight_min": -12.167012214660645,
      "activations/layer10_attention_weight_max": 33.76698684692383,
      "activations/layer10_attention_weight_min": -31.270748138427734,
      "activations/layer11_attention_weight_max": 32.48931884765625,
      "activations/layer11_attention_weight_min": -30.65300750732422,
      "activations/layer12_attention_weight_max": 27.723482131958008,
      "activations/layer12_attention_weight_min": -31.037416458129883,
      "activations/layer13_attention_weight_max": 46.099369049072266,
      "activations/layer13_attention_weight_min": -34.75030517578125,
      "activations/layer14_attention_weight_max": 50.66750717163086,
      "activations/layer14_attention_weight_min": -37.492366790771484,
      "activations/layer15_attention_weight_max": 38.58513641357422,
      "activations/layer15_attention_weight_min": -30.350549697875977,
      "activations/layer16_attention_weight_max": 35.26142120361328,
      "activations/layer16_attention_weight_min": -29.971328735351562,
      "activations/layer17_attention_weight_max": 33.3172607421875,
      "activations/layer17_attention_weight_min": -25.680307388305664,
      "activations/layer18_attention_weight_max": 27.110246658325195,
      "activations/layer18_attention_weight_min": -22.04439353942871,
      "activations/layer19_attention_weight_max": 35.164608001708984,
      "activations/layer19_attention_weight_min": -29.7482967376709,
      "activations/layer1_attention_weight_max": 18.530961990356445,
      "activations/layer1_attention_weight_min": -17.91389274597168,
      "activations/layer20_attention_weight_max": 29.58160400390625,
      "activations/layer20_attention_weight_min": -24.774106979370117,
      "activations/layer21_attention_weight_max": 30.61000633239746,
      "activations/layer21_attention_weight_min": -22.30201530456543,
      "activations/layer22_attention_weight_max": 47.8061408996582,
      "activations/layer22_attention_weight_min": -30.92708969116211,
      "activations/layer23_attention_weight_max": 30.56385612487793,
      "activations/layer23_attention_weight_min": -21.567245483398438,
      "activations/layer2_attention_weight_max": 31.303146362304688,
      "activations/layer2_attention_weight_min": -30.211135864257812,
      "activations/layer3_attention_weight_max": 79.73876190185547,
      "activations/layer3_attention_weight_min": -83.61241149902344,
      "activations/layer4_attention_weight_max": 119.65169525146484,
      "activations/layer4_attention_weight_min": -121.04402923583984,
      "activations/layer5_attention_weight_max": 73.67457580566406,
      "activations/layer5_attention_weight_min": -76.73165893554688,
      "activations/layer6_attention_weight_max": 53.68620300292969,
      "activations/layer6_attention_weight_min": -59.93626403808594,
      "activations/layer7_attention_weight_max": 83.78129577636719,
      "activations/layer7_attention_weight_min": -89.4539794921875,
      "activations/layer8_attention_weight_max": 38.15376663208008,
      "activations/layer8_attention_weight_min": -41.688194274902344,
      "activations/layer9_attention_weight_max": 35.32847595214844,
      "activations/layer9_attention_weight_min": -38.19614791870117,
      "epoch": 5.6,
      "learning_rate": 0.0001150155303030303,
      "loss": 2.831,
      "step": 96450
    },
    {
      "activations/layer0_attention_weight_max": 15.49366569519043,
      "activations/layer0_attention_weight_min": -11.853761672973633,
      "activations/layer10_attention_weight_max": 32.317771911621094,
      "activations/layer10_attention_weight_min": -30.396591186523438,
      "activations/layer11_attention_weight_max": 32.598358154296875,
      "activations/layer11_attention_weight_min": -28.751834869384766,
      "activations/layer12_attention_weight_max": 31.880996704101562,
      "activations/layer12_attention_weight_min": -26.816659927368164,
      "activations/layer13_attention_weight_max": 46.68427276611328,
      "activations/layer13_attention_weight_min": -33.8371467590332,
      "activations/layer14_attention_weight_max": 66.03766632080078,
      "activations/layer14_attention_weight_min": -47.40470504760742,
      "activations/layer15_attention_weight_max": 41.46015548706055,
      "activations/layer15_attention_weight_min": -30.283430099487305,
      "activations/layer16_attention_weight_max": 38.960994720458984,
      "activations/layer16_attention_weight_min": -27.822227478027344,
      "activations/layer17_attention_weight_max": 41.762908935546875,
      "activations/layer17_attention_weight_min": -24.54010772705078,
      "activations/layer18_attention_weight_max": 37.89496994018555,
      "activations/layer18_attention_weight_min": -20.310930252075195,
      "activations/layer19_attention_weight_max": 45.326515197753906,
      "activations/layer19_attention_weight_min": -28.1584529876709,
      "activations/layer1_attention_weight_max": 16.42203712463379,
      "activations/layer1_attention_weight_min": -12.973352432250977,
      "activations/layer20_attention_weight_max": 40.59895324707031,
      "activations/layer20_attention_weight_min": -22.853063583374023,
      "activations/layer21_attention_weight_max": 39.935035705566406,
      "activations/layer21_attention_weight_min": -20.0583553314209,
      "activations/layer22_attention_weight_max": 53.64920425415039,
      "activations/layer22_attention_weight_min": -29.787221908569336,
      "activations/layer23_attention_weight_max": 36.702606201171875,
      "activations/layer23_attention_weight_min": -21.2659912109375,
      "activations/layer2_attention_weight_max": 31.926616668701172,
      "activations/layer2_attention_weight_min": -30.960569381713867,
      "activations/layer3_attention_weight_max": 85.65185546875,
      "activations/layer3_attention_weight_min": -87.8964614868164,
      "activations/layer4_attention_weight_max": 119.27130126953125,
      "activations/layer4_attention_weight_min": -121.9092025756836,
      "activations/layer5_attention_weight_max": 71.86302947998047,
      "activations/layer5_attention_weight_min": -70.53236389160156,
      "activations/layer6_attention_weight_max": 55.21995544433594,
      "activations/layer6_attention_weight_min": -56.12723922729492,
      "activations/layer7_attention_weight_max": 83.41350555419922,
      "activations/layer7_attention_weight_min": -86.71863555908203,
      "activations/layer8_attention_weight_max": 38.87070846557617,
      "activations/layer8_attention_weight_min": -39.47246551513672,
      "activations/layer9_attention_weight_max": 35.86903762817383,
      "activations/layer9_attention_weight_min": -34.18741989135742,
      "epoch": 5.61,
      "learning_rate": 0.0001149965909090909,
      "loss": 2.8363,
      "step": 96500
    },
    {
      "activations/layer0_attention_weight_max": 15.90986442565918,
      "activations/layer0_attention_weight_min": -13.742928504943848,
      "activations/layer10_attention_weight_max": 31.314035415649414,
      "activations/layer10_attention_weight_min": -29.029563903808594,
      "activations/layer11_attention_weight_max": 32.92264175415039,
      "activations/layer11_attention_weight_min": -28.424232482910156,
      "activations/layer12_attention_weight_max": 35.50437927246094,
      "activations/layer12_attention_weight_min": -29.312244415283203,
      "activations/layer13_attention_weight_max": 42.315059661865234,
      "activations/layer13_attention_weight_min": -33.94729232788086,
      "activations/layer14_attention_weight_max": 57.341026306152344,
      "activations/layer14_attention_weight_min": -42.03221130371094,
      "activations/layer15_attention_weight_max": 43.17093276977539,
      "activations/layer15_attention_weight_min": -27.2884521484375,
      "activations/layer16_attention_weight_max": 36.91836929321289,
      "activations/layer16_attention_weight_min": -28.42608642578125,
      "activations/layer17_attention_weight_max": 41.027015686035156,
      "activations/layer17_attention_weight_min": -24.09090805053711,
      "activations/layer18_attention_weight_max": 29.346895217895508,
      "activations/layer18_attention_weight_min": -19.66583251953125,
      "activations/layer19_attention_weight_max": 38.53764724731445,
      "activations/layer19_attention_weight_min": -29.530719757080078,
      "activations/layer1_attention_weight_max": 18.30184555053711,
      "activations/layer1_attention_weight_min": -15.132952690124512,
      "activations/layer20_attention_weight_max": 32.799278259277344,
      "activations/layer20_attention_weight_min": -23.782882690429688,
      "activations/layer21_attention_weight_max": 32.20029067993164,
      "activations/layer21_attention_weight_min": -21.174583435058594,
      "activations/layer22_attention_weight_max": 50.36119079589844,
      "activations/layer22_attention_weight_min": -29.28838348388672,
      "activations/layer23_attention_weight_max": 33.24803161621094,
      "activations/layer23_attention_weight_min": -21.59182357788086,
      "activations/layer2_attention_weight_max": 32.16096496582031,
      "activations/layer2_attention_weight_min": -30.890546798706055,
      "activations/layer3_attention_weight_max": 91.75363159179688,
      "activations/layer3_attention_weight_min": -91.98573303222656,
      "activations/layer4_attention_weight_max": 131.14666748046875,
      "activations/layer4_attention_weight_min": -126.1334457397461,
      "activations/layer5_attention_weight_max": 75.74910736083984,
      "activations/layer5_attention_weight_min": -73.19662475585938,
      "activations/layer6_attention_weight_max": 55.28998947143555,
      "activations/layer6_attention_weight_min": -55.43320083618164,
      "activations/layer7_attention_weight_max": 84.57217407226562,
      "activations/layer7_attention_weight_min": -87.74076843261719,
      "activations/layer8_attention_weight_max": 38.794525146484375,
      "activations/layer8_attention_weight_min": -39.91033935546875,
      "activations/layer9_attention_weight_max": 34.365352630615234,
      "activations/layer9_attention_weight_min": -35.95512008666992,
      "epoch": 5.61,
      "learning_rate": 0.00011497765151515151,
      "loss": 2.847,
      "step": 96550
    },
    {
      "activations/layer0_attention_weight_max": 15.232460021972656,
      "activations/layer0_attention_weight_min": -13.465986251831055,
      "activations/layer10_attention_weight_max": 36.47401809692383,
      "activations/layer10_attention_weight_min": -30.69342613220215,
      "activations/layer11_attention_weight_max": 33.953514099121094,
      "activations/layer11_attention_weight_min": -28.524152755737305,
      "activations/layer12_attention_weight_max": 21.739721298217773,
      "activations/layer12_attention_weight_min": -26.9416561126709,
      "activations/layer13_attention_weight_max": 39.85292434692383,
      "activations/layer13_attention_weight_min": -31.34003448486328,
      "activations/layer14_attention_weight_max": 54.52174377441406,
      "activations/layer14_attention_weight_min": -43.79389572143555,
      "activations/layer15_attention_weight_max": 35.41899108886719,
      "activations/layer15_attention_weight_min": -28.1639461517334,
      "activations/layer16_attention_weight_max": 34.385005950927734,
      "activations/layer16_attention_weight_min": -28.088335037231445,
      "activations/layer17_attention_weight_max": 36.132747650146484,
      "activations/layer17_attention_weight_min": -23.90932846069336,
      "activations/layer18_attention_weight_max": 29.212556838989258,
      "activations/layer18_attention_weight_min": -20.384469985961914,
      "activations/layer19_attention_weight_max": 39.6889533996582,
      "activations/layer19_attention_weight_min": -29.078948974609375,
      "activations/layer1_attention_weight_max": 17.182083129882812,
      "activations/layer1_attention_weight_min": -14.327430725097656,
      "activations/layer20_attention_weight_max": 30.9561710357666,
      "activations/layer20_attention_weight_min": -24.337053298950195,
      "activations/layer21_attention_weight_max": 32.46207046508789,
      "activations/layer21_attention_weight_min": -21.05786895751953,
      "activations/layer22_attention_weight_max": 48.21120834350586,
      "activations/layer22_attention_weight_min": -28.792884826660156,
      "activations/layer23_attention_weight_max": 35.62984848022461,
      "activations/layer23_attention_weight_min": -21.691068649291992,
      "activations/layer2_attention_weight_max": 32.4550895690918,
      "activations/layer2_attention_weight_min": -30.42373275756836,
      "activations/layer3_attention_weight_max": 84.48819732666016,
      "activations/layer3_attention_weight_min": -84.94403076171875,
      "activations/layer4_attention_weight_max": 120.95515441894531,
      "activations/layer4_attention_weight_min": -117.5315170288086,
      "activations/layer5_attention_weight_max": 73.25764465332031,
      "activations/layer5_attention_weight_min": -70.2349624633789,
      "activations/layer6_attention_weight_max": 55.33192443847656,
      "activations/layer6_attention_weight_min": -56.56266403198242,
      "activations/layer7_attention_weight_max": 84.94966888427734,
      "activations/layer7_attention_weight_min": -83.52346801757812,
      "activations/layer8_attention_weight_max": 44.17559814453125,
      "activations/layer8_attention_weight_min": -42.314327239990234,
      "activations/layer9_attention_weight_max": 39.36907958984375,
      "activations/layer9_attention_weight_min": -36.24087142944336,
      "epoch": 5.61,
      "learning_rate": 0.0001149587121212121,
      "loss": 2.8395,
      "step": 96600
    },
    {
      "activations/layer0_attention_weight_max": 16.106035232543945,
      "activations/layer0_attention_weight_min": -13.35135269165039,
      "activations/layer10_attention_weight_max": 35.05461502075195,
      "activations/layer10_attention_weight_min": -30.931148529052734,
      "activations/layer11_attention_weight_max": 31.46504020690918,
      "activations/layer11_attention_weight_min": -27.283926010131836,
      "activations/layer12_attention_weight_max": 24.365055084228516,
      "activations/layer12_attention_weight_min": -32.745262145996094,
      "activations/layer13_attention_weight_max": 36.824790954589844,
      "activations/layer13_attention_weight_min": -30.799678802490234,
      "activations/layer14_attention_weight_max": 53.17745590209961,
      "activations/layer14_attention_weight_min": -44.79423522949219,
      "activations/layer15_attention_weight_max": 33.989620208740234,
      "activations/layer15_attention_weight_min": -27.319412231445312,
      "activations/layer16_attention_weight_max": 33.30582809448242,
      "activations/layer16_attention_weight_min": -27.351526260375977,
      "activations/layer17_attention_weight_max": 35.349159240722656,
      "activations/layer17_attention_weight_min": -23.054828643798828,
      "activations/layer18_attention_weight_max": 29.226171493530273,
      "activations/layer18_attention_weight_min": -19.925050735473633,
      "activations/layer19_attention_weight_max": 40.348758697509766,
      "activations/layer19_attention_weight_min": -30.043363571166992,
      "activations/layer1_attention_weight_max": 17.53673553466797,
      "activations/layer1_attention_weight_min": -14.055285453796387,
      "activations/layer20_attention_weight_max": 34.130855560302734,
      "activations/layer20_attention_weight_min": -22.186790466308594,
      "activations/layer21_attention_weight_max": 27.16648292541504,
      "activations/layer21_attention_weight_min": -20.139789581298828,
      "activations/layer22_attention_weight_max": 42.52532196044922,
      "activations/layer22_attention_weight_min": -29.89378547668457,
      "activations/layer23_attention_weight_max": 35.5435676574707,
      "activations/layer23_attention_weight_min": -23.107093811035156,
      "activations/layer2_attention_weight_max": 32.901710510253906,
      "activations/layer2_attention_weight_min": -31.206090927124023,
      "activations/layer3_attention_weight_max": 86.85246276855469,
      "activations/layer3_attention_weight_min": -86.72769165039062,
      "activations/layer4_attention_weight_max": 120.7115478515625,
      "activations/layer4_attention_weight_min": -121.50874328613281,
      "activations/layer5_attention_weight_max": 74.76880645751953,
      "activations/layer5_attention_weight_min": -70.68138122558594,
      "activations/layer6_attention_weight_max": 52.96169662475586,
      "activations/layer6_attention_weight_min": -53.671180725097656,
      "activations/layer7_attention_weight_max": 85.50450134277344,
      "activations/layer7_attention_weight_min": -81.05439758300781,
      "activations/layer8_attention_weight_max": 38.95241165161133,
      "activations/layer8_attention_weight_min": -41.66082000732422,
      "activations/layer9_attention_weight_max": 35.55717468261719,
      "activations/layer9_attention_weight_min": -32.69020080566406,
      "epoch": 5.62,
      "learning_rate": 0.00011493977272727272,
      "loss": 2.841,
      "step": 96650
    },
    {
      "activations/layer0_attention_weight_max": 16.94357681274414,
      "activations/layer0_attention_weight_min": -12.184794425964355,
      "activations/layer10_attention_weight_max": 38.30103302001953,
      "activations/layer10_attention_weight_min": -30.906370162963867,
      "activations/layer11_attention_weight_max": 37.34357452392578,
      "activations/layer11_attention_weight_min": -28.998937606811523,
      "activations/layer12_attention_weight_max": 27.441646575927734,
      "activations/layer12_attention_weight_min": -27.31466293334961,
      "activations/layer13_attention_weight_max": 49.59843444824219,
      "activations/layer13_attention_weight_min": -32.154136657714844,
      "activations/layer14_attention_weight_max": 63.93471908569336,
      "activations/layer14_attention_weight_min": -45.21395492553711,
      "activations/layer15_attention_weight_max": 38.3777961730957,
      "activations/layer15_attention_weight_min": -27.500377655029297,
      "activations/layer16_attention_weight_max": 35.65620422363281,
      "activations/layer16_attention_weight_min": -25.5006103515625,
      "activations/layer17_attention_weight_max": 37.57003402709961,
      "activations/layer17_attention_weight_min": -24.107059478759766,
      "activations/layer18_attention_weight_max": 31.14140510559082,
      "activations/layer18_attention_weight_min": -19.629268646240234,
      "activations/layer19_attention_weight_max": 36.56760025024414,
      "activations/layer19_attention_weight_min": -28.712177276611328,
      "activations/layer1_attention_weight_max": 16.867305755615234,
      "activations/layer1_attention_weight_min": -13.309991836547852,
      "activations/layer20_attention_weight_max": 31.867570877075195,
      "activations/layer20_attention_weight_min": -23.024738311767578,
      "activations/layer21_attention_weight_max": 30.38921356201172,
      "activations/layer21_attention_weight_min": -19.903568267822266,
      "activations/layer22_attention_weight_max": 46.79056167602539,
      "activations/layer22_attention_weight_min": -27.768537521362305,
      "activations/layer23_attention_weight_max": 33.58607482910156,
      "activations/layer23_attention_weight_min": -22.352705001831055,
      "activations/layer2_attention_weight_max": 31.121793746948242,
      "activations/layer2_attention_weight_min": -31.56153106689453,
      "activations/layer3_attention_weight_max": 92.357666015625,
      "activations/layer3_attention_weight_min": -87.5556869506836,
      "activations/layer4_attention_weight_max": 127.5379409790039,
      "activations/layer4_attention_weight_min": -122.5941162109375,
      "activations/layer5_attention_weight_max": 74.84095764160156,
      "activations/layer5_attention_weight_min": -73.26890563964844,
      "activations/layer6_attention_weight_max": 53.764644622802734,
      "activations/layer6_attention_weight_min": -54.436405181884766,
      "activations/layer7_attention_weight_max": 80.65021514892578,
      "activations/layer7_attention_weight_min": -87.56121826171875,
      "activations/layer8_attention_weight_max": 39.74559020996094,
      "activations/layer8_attention_weight_min": -38.73088073730469,
      "activations/layer9_attention_weight_max": 37.487300872802734,
      "activations/layer9_attention_weight_min": -33.31380844116211,
      "epoch": 5.62,
      "learning_rate": 0.00011492121212121211,
      "loss": 2.8326,
      "step": 96700
    },
    {
      "activations/layer0_attention_weight_max": 15.818039894104004,
      "activations/layer0_attention_weight_min": -12.194199562072754,
      "activations/layer10_attention_weight_max": 33.590675354003906,
      "activations/layer10_attention_weight_min": -30.111648559570312,
      "activations/layer11_attention_weight_max": 33.955562591552734,
      "activations/layer11_attention_weight_min": -27.434370040893555,
      "activations/layer12_attention_weight_max": 27.148958206176758,
      "activations/layer12_attention_weight_min": -28.39798355102539,
      "activations/layer13_attention_weight_max": 43.435298919677734,
      "activations/layer13_attention_weight_min": -32.657127380371094,
      "activations/layer14_attention_weight_max": 68.35076904296875,
      "activations/layer14_attention_weight_min": -49.57607650756836,
      "activations/layer15_attention_weight_max": 37.82615280151367,
      "activations/layer15_attention_weight_min": -28.87565040588379,
      "activations/layer16_attention_weight_max": 40.77314376831055,
      "activations/layer16_attention_weight_min": -27.936254501342773,
      "activations/layer17_attention_weight_max": 37.99391174316406,
      "activations/layer17_attention_weight_min": -26.100980758666992,
      "activations/layer18_attention_weight_max": 35.365379333496094,
      "activations/layer18_attention_weight_min": -19.752742767333984,
      "activations/layer19_attention_weight_max": 46.10232925415039,
      "activations/layer19_attention_weight_min": -30.692523956298828,
      "activations/layer1_attention_weight_max": 18.016220092773438,
      "activations/layer1_attention_weight_min": -13.788342475891113,
      "activations/layer20_attention_weight_max": 38.09653091430664,
      "activations/layer20_attention_weight_min": -23.985614776611328,
      "activations/layer21_attention_weight_max": 34.815162658691406,
      "activations/layer21_attention_weight_min": -20.232887268066406,
      "activations/layer22_attention_weight_max": 53.18638610839844,
      "activations/layer22_attention_weight_min": -29.062410354614258,
      "activations/layer23_attention_weight_max": 34.37484359741211,
      "activations/layer23_attention_weight_min": -21.38961410522461,
      "activations/layer2_attention_weight_max": 31.18368911743164,
      "activations/layer2_attention_weight_min": -33.10530471801758,
      "activations/layer3_attention_weight_max": 86.77362823486328,
      "activations/layer3_attention_weight_min": -89.08124542236328,
      "activations/layer4_attention_weight_max": 123.57293701171875,
      "activations/layer4_attention_weight_min": -123.79471588134766,
      "activations/layer5_attention_weight_max": 75.92510986328125,
      "activations/layer5_attention_weight_min": -71.4814453125,
      "activations/layer6_attention_weight_max": 54.78865051269531,
      "activations/layer6_attention_weight_min": -54.07633590698242,
      "activations/layer7_attention_weight_max": 85.99279022216797,
      "activations/layer7_attention_weight_min": -88.19947814941406,
      "activations/layer8_attention_weight_max": 40.20458984375,
      "activations/layer8_attention_weight_min": -40.33659744262695,
      "activations/layer9_attention_weight_max": 34.70856475830078,
      "activations/layer9_attention_weight_min": -35.039581298828125,
      "epoch": 5.62,
      "learning_rate": 0.00011490227272727272,
      "loss": 2.8449,
      "step": 96750
    },
    {
      "activations/layer0_attention_weight_max": 16.346067428588867,
      "activations/layer0_attention_weight_min": -13.139747619628906,
      "activations/layer10_attention_weight_max": 35.1606330871582,
      "activations/layer10_attention_weight_min": -32.32099151611328,
      "activations/layer11_attention_weight_max": 33.3729362487793,
      "activations/layer11_attention_weight_min": -27.782325744628906,
      "activations/layer12_attention_weight_max": 30.676387786865234,
      "activations/layer12_attention_weight_min": -26.520282745361328,
      "activations/layer13_attention_weight_max": 45.52171325683594,
      "activations/layer13_attention_weight_min": -32.41035461425781,
      "activations/layer14_attention_weight_max": 56.07145690917969,
      "activations/layer14_attention_weight_min": -40.62406921386719,
      "activations/layer15_attention_weight_max": 42.970184326171875,
      "activations/layer15_attention_weight_min": -28.960933685302734,
      "activations/layer16_attention_weight_max": 34.235496520996094,
      "activations/layer16_attention_weight_min": -26.346904754638672,
      "activations/layer17_attention_weight_max": 37.659175872802734,
      "activations/layer17_attention_weight_min": -24.759031295776367,
      "activations/layer18_attention_weight_max": 32.928749084472656,
      "activations/layer18_attention_weight_min": -21.362228393554688,
      "activations/layer19_attention_weight_max": 37.69254684448242,
      "activations/layer19_attention_weight_min": -29.730817794799805,
      "activations/layer1_attention_weight_max": 16.25446891784668,
      "activations/layer1_attention_weight_min": -12.696468353271484,
      "activations/layer20_attention_weight_max": 33.114402770996094,
      "activations/layer20_attention_weight_min": -26.98619270324707,
      "activations/layer21_attention_weight_max": 32.48431396484375,
      "activations/layer21_attention_weight_min": -22.35924530029297,
      "activations/layer22_attention_weight_max": 51.25584030151367,
      "activations/layer22_attention_weight_min": -28.75773811340332,
      "activations/layer23_attention_weight_max": 35.989776611328125,
      "activations/layer23_attention_weight_min": -22.240446090698242,
      "activations/layer2_attention_weight_max": 32.357566833496094,
      "activations/layer2_attention_weight_min": -31.098007202148438,
      "activations/layer3_attention_weight_max": 89.77387237548828,
      "activations/layer3_attention_weight_min": -91.0069580078125,
      "activations/layer4_attention_weight_max": 133.05902099609375,
      "activations/layer4_attention_weight_min": -122.79715728759766,
      "activations/layer5_attention_weight_max": 77.53889465332031,
      "activations/layer5_attention_weight_min": -72.87694549560547,
      "activations/layer6_attention_weight_max": 57.30924606323242,
      "activations/layer6_attention_weight_min": -56.08436965942383,
      "activations/layer7_attention_weight_max": 87.00444793701172,
      "activations/layer7_attention_weight_min": -87.24390411376953,
      "activations/layer8_attention_weight_max": 39.35841369628906,
      "activations/layer8_attention_weight_min": -37.63155746459961,
      "activations/layer9_attention_weight_max": 37.136756896972656,
      "activations/layer9_attention_weight_min": -33.624481201171875,
      "epoch": 5.62,
      "learning_rate": 0.00011488333333333331,
      "loss": 2.8234,
      "step": 96800
    },
    {
      "activations/layer0_attention_weight_max": 16.66362953186035,
      "activations/layer0_attention_weight_min": -12.125577926635742,
      "activations/layer10_attention_weight_max": 33.2457160949707,
      "activations/layer10_attention_weight_min": -31.401681900024414,
      "activations/layer11_attention_weight_max": 33.0395393371582,
      "activations/layer11_attention_weight_min": -32.01736831665039,
      "activations/layer12_attention_weight_max": 29.980754852294922,
      "activations/layer12_attention_weight_min": -28.587966918945312,
      "activations/layer13_attention_weight_max": 42.8032341003418,
      "activations/layer13_attention_weight_min": -34.88302230834961,
      "activations/layer14_attention_weight_max": 59.80842590332031,
      "activations/layer14_attention_weight_min": -44.38149642944336,
      "activations/layer15_attention_weight_max": 36.79338455200195,
      "activations/layer15_attention_weight_min": -28.519943237304688,
      "activations/layer16_attention_weight_max": 40.305702209472656,
      "activations/layer16_attention_weight_min": -29.028940200805664,
      "activations/layer17_attention_weight_max": 37.973297119140625,
      "activations/layer17_attention_weight_min": -25.7989444732666,
      "activations/layer18_attention_weight_max": 32.10696029663086,
      "activations/layer18_attention_weight_min": -20.260459899902344,
      "activations/layer19_attention_weight_max": 40.110595703125,
      "activations/layer19_attention_weight_min": -29.774606704711914,
      "activations/layer1_attention_weight_max": 17.457962036132812,
      "activations/layer1_attention_weight_min": -13.344011306762695,
      "activations/layer20_attention_weight_max": 32.91652297973633,
      "activations/layer20_attention_weight_min": -22.251171112060547,
      "activations/layer21_attention_weight_max": 30.9775447845459,
      "activations/layer21_attention_weight_min": -21.603673934936523,
      "activations/layer22_attention_weight_max": 49.7578239440918,
      "activations/layer22_attention_weight_min": -31.00959014892578,
      "activations/layer23_attention_weight_max": 34.27645492553711,
      "activations/layer23_attention_weight_min": -24.770292282104492,
      "activations/layer2_attention_weight_max": 32.195526123046875,
      "activations/layer2_attention_weight_min": -32.72449493408203,
      "activations/layer3_attention_weight_max": 89.91295623779297,
      "activations/layer3_attention_weight_min": -93.70816802978516,
      "activations/layer4_attention_weight_max": 127.21931457519531,
      "activations/layer4_attention_weight_min": -124.42726135253906,
      "activations/layer5_attention_weight_max": 78.83494567871094,
      "activations/layer5_attention_weight_min": -75.50234985351562,
      "activations/layer6_attention_weight_max": 55.263492584228516,
      "activations/layer6_attention_weight_min": -55.23976135253906,
      "activations/layer7_attention_weight_max": 86.18047332763672,
      "activations/layer7_attention_weight_min": -95.4550552368164,
      "activations/layer8_attention_weight_max": 39.49870681762695,
      "activations/layer8_attention_weight_min": -39.90314865112305,
      "activations/layer9_attention_weight_max": 36.77561569213867,
      "activations/layer9_attention_weight_min": -33.966426849365234,
      "epoch": 5.63,
      "learning_rate": 0.00011486439393939393,
      "loss": 2.861,
      "step": 96850
    },
    {
      "activations/layer0_attention_weight_max": 16.52349853515625,
      "activations/layer0_attention_weight_min": -13.104267120361328,
      "activations/layer10_attention_weight_max": 35.91657257080078,
      "activations/layer10_attention_weight_min": -31.899276733398438,
      "activations/layer11_attention_weight_max": 33.22624588012695,
      "activations/layer11_attention_weight_min": -28.466503143310547,
      "activations/layer12_attention_weight_max": 32.367061614990234,
      "activations/layer12_attention_weight_min": -24.633878707885742,
      "activations/layer13_attention_weight_max": 45.178409576416016,
      "activations/layer13_attention_weight_min": -34.223060607910156,
      "activations/layer14_attention_weight_max": 66.64371490478516,
      "activations/layer14_attention_weight_min": -49.30963897705078,
      "activations/layer15_attention_weight_max": 46.06453323364258,
      "activations/layer15_attention_weight_min": -28.915800094604492,
      "activations/layer16_attention_weight_max": 41.6779899597168,
      "activations/layer16_attention_weight_min": -28.926071166992188,
      "activations/layer17_attention_weight_max": 40.800575256347656,
      "activations/layer17_attention_weight_min": -24.222896575927734,
      "activations/layer18_attention_weight_max": 36.790618896484375,
      "activations/layer18_attention_weight_min": -19.985158920288086,
      "activations/layer19_attention_weight_max": 40.384071350097656,
      "activations/layer19_attention_weight_min": -29.518842697143555,
      "activations/layer1_attention_weight_max": 17.23265266418457,
      "activations/layer1_attention_weight_min": -14.301889419555664,
      "activations/layer20_attention_weight_max": 34.91143798828125,
      "activations/layer20_attention_weight_min": -23.116138458251953,
      "activations/layer21_attention_weight_max": 31.531070709228516,
      "activations/layer21_attention_weight_min": -20.661386489868164,
      "activations/layer22_attention_weight_max": 51.17978286743164,
      "activations/layer22_attention_weight_min": -28.14107894897461,
      "activations/layer23_attention_weight_max": 36.558414459228516,
      "activations/layer23_attention_weight_min": -21.404300689697266,
      "activations/layer2_attention_weight_max": 29.740171432495117,
      "activations/layer2_attention_weight_min": -30.071142196655273,
      "activations/layer3_attention_weight_max": 85.70511627197266,
      "activations/layer3_attention_weight_min": -90.95809173583984,
      "activations/layer4_attention_weight_max": 127.09740447998047,
      "activations/layer4_attention_weight_min": -128.89801025390625,
      "activations/layer5_attention_weight_max": 73.9627456665039,
      "activations/layer5_attention_weight_min": -75.58535766601562,
      "activations/layer6_attention_weight_max": 57.291595458984375,
      "activations/layer6_attention_weight_min": -57.89560317993164,
      "activations/layer7_attention_weight_max": 87.47899627685547,
      "activations/layer7_attention_weight_min": -88.27113342285156,
      "activations/layer8_attention_weight_max": 41.87379837036133,
      "activations/layer8_attention_weight_min": -41.21644973754883,
      "activations/layer9_attention_weight_max": 38.54538345336914,
      "activations/layer9_attention_weight_min": -36.60575485229492,
      "epoch": 5.63,
      "learning_rate": 0.00011484545454545454,
      "loss": 2.8444,
      "step": 96900
    },
    {
      "activations/layer0_attention_weight_max": 16.46457290649414,
      "activations/layer0_attention_weight_min": -12.98455810546875,
      "activations/layer10_attention_weight_max": 34.47633743286133,
      "activations/layer10_attention_weight_min": -32.51140213012695,
      "activations/layer11_attention_weight_max": 32.40180969238281,
      "activations/layer11_attention_weight_min": -29.467212677001953,
      "activations/layer12_attention_weight_max": 39.159175872802734,
      "activations/layer12_attention_weight_min": -25.03329849243164,
      "activations/layer13_attention_weight_max": 55.83200454711914,
      "activations/layer13_attention_weight_min": -34.83465576171875,
      "activations/layer14_attention_weight_max": 66.71959686279297,
      "activations/layer14_attention_weight_min": -45.536415100097656,
      "activations/layer15_attention_weight_max": 41.45372772216797,
      "activations/layer15_attention_weight_min": -28.565460205078125,
      "activations/layer16_attention_weight_max": 45.687198638916016,
      "activations/layer16_attention_weight_min": -29.506128311157227,
      "activations/layer17_attention_weight_max": 47.41325378417969,
      "activations/layer17_attention_weight_min": -26.904335021972656,
      "activations/layer18_attention_weight_max": 36.22557830810547,
      "activations/layer18_attention_weight_min": -24.03370475769043,
      "activations/layer19_attention_weight_max": 44.4769401550293,
      "activations/layer19_attention_weight_min": -30.758813858032227,
      "activations/layer1_attention_weight_max": 17.691831588745117,
      "activations/layer1_attention_weight_min": -13.31218433380127,
      "activations/layer20_attention_weight_max": 38.65639877319336,
      "activations/layer20_attention_weight_min": -27.087556838989258,
      "activations/layer21_attention_weight_max": 35.55424880981445,
      "activations/layer21_attention_weight_min": -25.245159149169922,
      "activations/layer22_attention_weight_max": 59.0588493347168,
      "activations/layer22_attention_weight_min": -31.194150924682617,
      "activations/layer23_attention_weight_max": 36.18623352050781,
      "activations/layer23_attention_weight_min": -23.184738159179688,
      "activations/layer2_attention_weight_max": 34.673072814941406,
      "activations/layer2_attention_weight_min": -33.74647521972656,
      "activations/layer3_attention_weight_max": 86.70459747314453,
      "activations/layer3_attention_weight_min": -87.46726989746094,
      "activations/layer4_attention_weight_max": 122.05885314941406,
      "activations/layer4_attention_weight_min": -121.83197021484375,
      "activations/layer5_attention_weight_max": 71.99774932861328,
      "activations/layer5_attention_weight_min": -73.3307876586914,
      "activations/layer6_attention_weight_max": 57.84375,
      "activations/layer6_attention_weight_min": -55.73487091064453,
      "activations/layer7_attention_weight_max": 81.17272186279297,
      "activations/layer7_attention_weight_min": -90.98924255371094,
      "activations/layer8_attention_weight_max": 42.34507369995117,
      "activations/layer8_attention_weight_min": -44.358673095703125,
      "activations/layer9_attention_weight_max": 37.192264556884766,
      "activations/layer9_attention_weight_min": -36.79952621459961,
      "epoch": 5.63,
      "learning_rate": 0.00011482651515151513,
      "loss": 2.8419,
      "step": 96950
    },
    {
      "activations/layer0_attention_weight_max": 15.514226913452148,
      "activations/layer0_attention_weight_min": -12.277471542358398,
      "activations/layer10_attention_weight_max": 33.2861442565918,
      "activations/layer10_attention_weight_min": -31.755680084228516,
      "activations/layer11_attention_weight_max": 37.237606048583984,
      "activations/layer11_attention_weight_min": -29.734045028686523,
      "activations/layer12_attention_weight_max": 25.2899227142334,
      "activations/layer12_attention_weight_min": -28.089765548706055,
      "activations/layer13_attention_weight_max": 50.31346130371094,
      "activations/layer13_attention_weight_min": -38.926448822021484,
      "activations/layer14_attention_weight_max": 57.30501937866211,
      "activations/layer14_attention_weight_min": -44.60515213012695,
      "activations/layer15_attention_weight_max": 35.14019012451172,
      "activations/layer15_attention_weight_min": -29.070903778076172,
      "activations/layer16_attention_weight_max": 37.34469985961914,
      "activations/layer16_attention_weight_min": -26.923173904418945,
      "activations/layer17_attention_weight_max": 34.11063766479492,
      "activations/layer17_attention_weight_min": -23.36393928527832,
      "activations/layer18_attention_weight_max": 32.99297332763672,
      "activations/layer18_attention_weight_min": -18.548879623413086,
      "activations/layer19_attention_weight_max": 36.920955657958984,
      "activations/layer19_attention_weight_min": -28.547216415405273,
      "activations/layer1_attention_weight_max": 17.97401237487793,
      "activations/layer1_attention_weight_min": -14.631150245666504,
      "activations/layer20_attention_weight_max": 29.38897705078125,
      "activations/layer20_attention_weight_min": -23.17424774169922,
      "activations/layer21_attention_weight_max": 28.31121826171875,
      "activations/layer21_attention_weight_min": -20.250459671020508,
      "activations/layer22_attention_weight_max": 46.45090866088867,
      "activations/layer22_attention_weight_min": -28.066743850708008,
      "activations/layer23_attention_weight_max": 31.47323989868164,
      "activations/layer23_attention_weight_min": -21.77901268005371,
      "activations/layer2_attention_weight_max": 33.7444953918457,
      "activations/layer2_attention_weight_min": -33.88025665283203,
      "activations/layer3_attention_weight_max": 94.1983413696289,
      "activations/layer3_attention_weight_min": -93.29551696777344,
      "activations/layer4_attention_weight_max": 128.666748046875,
      "activations/layer4_attention_weight_min": -122.9925765991211,
      "activations/layer5_attention_weight_max": 80.69710540771484,
      "activations/layer5_attention_weight_min": -72.11478424072266,
      "activations/layer6_attention_weight_max": 54.960060119628906,
      "activations/layer6_attention_weight_min": -53.93447494506836,
      "activations/layer7_attention_weight_max": 93.84551239013672,
      "activations/layer7_attention_weight_min": -84.71269989013672,
      "activations/layer8_attention_weight_max": 40.81388473510742,
      "activations/layer8_attention_weight_min": -40.365909576416016,
      "activations/layer9_attention_weight_max": 34.83631896972656,
      "activations/layer9_attention_weight_min": -42.828243255615234,
      "epoch": 5.64,
      "learning_rate": 0.00011480757575757574,
      "loss": 2.8514,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_loss": 2.791015625,
      "eval_runtime": 8.4803,
      "eval_samples_per_second": 506.352,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_openwebtext_loss": 2.791015625,
      "eval_openwebtext_ppl": 16.2975636122563,
      "eval_openwebtext_runtime": 8.4803,
      "eval_openwebtext_samples_per_second": 506.352,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 1.9467,
      "eval_wikitext_samples_per_second": 234.243,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_lambada_loss": 2.763671875,
      "eval_lambada_ppl": 15.85796463907484,
      "eval_lambada_runtime": 9.5218,
      "eval_lambada_samples_per_second": 511.355,
      "step": 97000
    },
    {
      "activations/layer0_attention_weight_max": 15.444095611572266,
      "activations/layer0_attention_weight_min": -13.297577857971191,
      "activations/layer10_attention_weight_max": 35.850189208984375,
      "activations/layer10_attention_weight_min": -31.245128631591797,
      "activations/layer11_attention_weight_max": 30.619712829589844,
      "activations/layer11_attention_weight_min": -28.399642944335938,
      "activations/layer12_attention_weight_max": 22.327167510986328,
      "activations/layer12_attention_weight_min": -31.691099166870117,
      "activations/layer13_attention_weight_max": 38.98024368286133,
      "activations/layer13_attention_weight_min": -30.945281982421875,
      "activations/layer14_attention_weight_max": 60.790374755859375,
      "activations/layer14_attention_weight_min": -43.541385650634766,
      "activations/layer15_attention_weight_max": 36.76148986816406,
      "activations/layer15_attention_weight_min": -26.646305084228516,
      "activations/layer16_attention_weight_max": 35.24888610839844,
      "activations/layer16_attention_weight_min": -28.801849365234375,
      "activations/layer17_attention_weight_max": 35.86023712158203,
      "activations/layer17_attention_weight_min": -24.122865676879883,
      "activations/layer18_attention_weight_max": 29.617919921875,
      "activations/layer18_attention_weight_min": -20.230327606201172,
      "activations/layer19_attention_weight_max": 36.286617279052734,
      "activations/layer19_attention_weight_min": -29.097888946533203,
      "activations/layer1_attention_weight_max": 17.7097110748291,
      "activations/layer1_attention_weight_min": -13.53487777709961,
      "activations/layer20_attention_weight_max": 33.40681838989258,
      "activations/layer20_attention_weight_min": -21.669404983520508,
      "activations/layer21_attention_weight_max": 30.45734214782715,
      "activations/layer21_attention_weight_min": -20.293479919433594,
      "activations/layer22_attention_weight_max": 51.51300811767578,
      "activations/layer22_attention_weight_min": -27.77591323852539,
      "activations/layer23_attention_weight_max": 31.51744842529297,
      "activations/layer23_attention_weight_min": -20.881986618041992,
      "activations/layer2_attention_weight_max": 31.25396728515625,
      "activations/layer2_attention_weight_min": -30.0551815032959,
      "activations/layer3_attention_weight_max": 88.45345306396484,
      "activations/layer3_attention_weight_min": -89.64737701416016,
      "activations/layer4_attention_weight_max": 122.08853912353516,
      "activations/layer4_attention_weight_min": -124.80712890625,
      "activations/layer5_attention_weight_max": 71.33216094970703,
      "activations/layer5_attention_weight_min": -70.68046569824219,
      "activations/layer6_attention_weight_max": 57.89060592651367,
      "activations/layer6_attention_weight_min": -55.30507278442383,
      "activations/layer7_attention_weight_max": 90.08377075195312,
      "activations/layer7_attention_weight_min": -88.90548706054688,
      "activations/layer8_attention_weight_max": 41.87662887573242,
      "activations/layer8_attention_weight_min": -40.22892379760742,
      "activations/layer9_attention_weight_max": 37.81124496459961,
      "activations/layer9_attention_weight_min": -33.7155876159668,
      "epoch": 5.64,
      "learning_rate": 0.00011478863636363636,
      "loss": 2.8404,
      "step": 97050
    },
    {
      "activations/layer0_attention_weight_max": 15.596285820007324,
      "activations/layer0_attention_weight_min": -12.505141258239746,
      "activations/layer10_attention_weight_max": 34.82546615600586,
      "activations/layer10_attention_weight_min": -31.47214698791504,
      "activations/layer11_attention_weight_max": 35.12172317504883,
      "activations/layer11_attention_weight_min": -29.28990936279297,
      "activations/layer12_attention_weight_max": 28.7023868560791,
      "activations/layer12_attention_weight_min": -30.580039978027344,
      "activations/layer13_attention_weight_max": 47.58657455444336,
      "activations/layer13_attention_weight_min": -32.21101379394531,
      "activations/layer14_attention_weight_max": 69.83575439453125,
      "activations/layer14_attention_weight_min": -44.27627182006836,
      "activations/layer15_attention_weight_max": 43.22086715698242,
      "activations/layer15_attention_weight_min": -28.382923126220703,
      "activations/layer16_attention_weight_max": 38.491310119628906,
      "activations/layer16_attention_weight_min": -27.887428283691406,
      "activations/layer17_attention_weight_max": 41.09524917602539,
      "activations/layer17_attention_weight_min": -24.80418586730957,
      "activations/layer18_attention_weight_max": 33.0837287902832,
      "activations/layer18_attention_weight_min": -19.91417694091797,
      "activations/layer19_attention_weight_max": 43.80015182495117,
      "activations/layer19_attention_weight_min": -29.973844528198242,
      "activations/layer1_attention_weight_max": 17.545116424560547,
      "activations/layer1_attention_weight_min": -14.228227615356445,
      "activations/layer20_attention_weight_max": 42.02739334106445,
      "activations/layer20_attention_weight_min": -22.777753829956055,
      "activations/layer21_attention_weight_max": 36.79302978515625,
      "activations/layer21_attention_weight_min": -21.278621673583984,
      "activations/layer22_attention_weight_max": 58.6519889831543,
      "activations/layer22_attention_weight_min": -25.94476318359375,
      "activations/layer23_attention_weight_max": 38.38626480102539,
      "activations/layer23_attention_weight_min": -19.8314151763916,
      "activations/layer2_attention_weight_max": 34.200828552246094,
      "activations/layer2_attention_weight_min": -33.725128173828125,
      "activations/layer3_attention_weight_max": 90.5399398803711,
      "activations/layer3_attention_weight_min": -93.7841796875,
      "activations/layer4_attention_weight_max": 122.94429016113281,
      "activations/layer4_attention_weight_min": -124.22550201416016,
      "activations/layer5_attention_weight_max": 76.05113220214844,
      "activations/layer5_attention_weight_min": -73.7395248413086,
      "activations/layer6_attention_weight_max": 56.60074996948242,
      "activations/layer6_attention_weight_min": -59.17820358276367,
      "activations/layer7_attention_weight_max": 84.32359313964844,
      "activations/layer7_attention_weight_min": -89.1126480102539,
      "activations/layer8_attention_weight_max": 42.520511627197266,
      "activations/layer8_attention_weight_min": -43.25516891479492,
      "activations/layer9_attention_weight_max": 38.05350875854492,
      "activations/layer9_attention_weight_min": -38.8519401550293,
      "epoch": 5.64,
      "learning_rate": 0.00011476969696969696,
      "loss": 2.8411,
      "step": 97100
    },
    {
      "activations/layer0_attention_weight_max": 15.308599472045898,
      "activations/layer0_attention_weight_min": -13.905599594116211,
      "activations/layer10_attention_weight_max": 34.48735427856445,
      "activations/layer10_attention_weight_min": -30.263242721557617,
      "activations/layer11_attention_weight_max": 32.62639236450195,
      "activations/layer11_attention_weight_min": -28.06945037841797,
      "activations/layer12_attention_weight_max": 23.135948181152344,
      "activations/layer12_attention_weight_min": -28.916854858398438,
      "activations/layer13_attention_weight_max": 42.66569900512695,
      "activations/layer13_attention_weight_min": -31.920021057128906,
      "activations/layer14_attention_weight_max": 52.26308059692383,
      "activations/layer14_attention_weight_min": -44.55851364135742,
      "activations/layer15_attention_weight_max": 38.48114776611328,
      "activations/layer15_attention_weight_min": -27.64574432373047,
      "activations/layer16_attention_weight_max": 32.4503288269043,
      "activations/layer16_attention_weight_min": -27.257997512817383,
      "activations/layer17_attention_weight_max": 36.118438720703125,
      "activations/layer17_attention_weight_min": -24.19405746459961,
      "activations/layer18_attention_weight_max": 28.950956344604492,
      "activations/layer18_attention_weight_min": -21.866573333740234,
      "activations/layer19_attention_weight_max": 35.459957122802734,
      "activations/layer19_attention_weight_min": -27.103229522705078,
      "activations/layer1_attention_weight_max": 17.49457359313965,
      "activations/layer1_attention_weight_min": -15.884493827819824,
      "activations/layer20_attention_weight_max": 28.88417625427246,
      "activations/layer20_attention_weight_min": -23.173002243041992,
      "activations/layer21_attention_weight_max": 25.861112594604492,
      "activations/layer21_attention_weight_min": -19.215124130249023,
      "activations/layer22_attention_weight_max": 43.66542053222656,
      "activations/layer22_attention_weight_min": -29.080984115600586,
      "activations/layer23_attention_weight_max": 30.030216217041016,
      "activations/layer23_attention_weight_min": -19.85637855529785,
      "activations/layer2_attention_weight_max": 31.98908805847168,
      "activations/layer2_attention_weight_min": -30.747325897216797,
      "activations/layer3_attention_weight_max": 93.56465911865234,
      "activations/layer3_attention_weight_min": -94.14620208740234,
      "activations/layer4_attention_weight_max": 128.38153076171875,
      "activations/layer4_attention_weight_min": -129.5286102294922,
      "activations/layer5_attention_weight_max": 77.04986572265625,
      "activations/layer5_attention_weight_min": -78.44914245605469,
      "activations/layer6_attention_weight_max": 57.580894470214844,
      "activations/layer6_attention_weight_min": -59.33363723754883,
      "activations/layer7_attention_weight_max": 88.6124267578125,
      "activations/layer7_attention_weight_min": -91.54871368408203,
      "activations/layer8_attention_weight_max": 41.903961181640625,
      "activations/layer8_attention_weight_min": -40.72282791137695,
      "activations/layer9_attention_weight_max": 38.298728942871094,
      "activations/layer9_attention_weight_min": -34.66971206665039,
      "epoch": 5.64,
      "learning_rate": 0.00011475075757575756,
      "loss": 2.8407,
      "step": 97150
    },
    {
      "activations/layer0_attention_weight_max": 16.2493953704834,
      "activations/layer0_attention_weight_min": -12.931805610656738,
      "activations/layer10_attention_weight_max": 33.65864944458008,
      "activations/layer10_attention_weight_min": -33.1690559387207,
      "activations/layer11_attention_weight_max": 32.06476593017578,
      "activations/layer11_attention_weight_min": -27.717466354370117,
      "activations/layer12_attention_weight_max": 29.404661178588867,
      "activations/layer12_attention_weight_min": -23.36008071899414,
      "activations/layer13_attention_weight_max": 43.41796112060547,
      "activations/layer13_attention_weight_min": -31.431386947631836,
      "activations/layer14_attention_weight_max": 62.795631408691406,
      "activations/layer14_attention_weight_min": -49.24692153930664,
      "activations/layer15_attention_weight_max": 37.443458557128906,
      "activations/layer15_attention_weight_min": -30.20305061340332,
      "activations/layer16_attention_weight_max": 35.73933410644531,
      "activations/layer16_attention_weight_min": -27.06995964050293,
      "activations/layer17_attention_weight_max": 43.85074234008789,
      "activations/layer17_attention_weight_min": -24.95375633239746,
      "activations/layer18_attention_weight_max": 33.55366134643555,
      "activations/layer18_attention_weight_min": -21.44112777709961,
      "activations/layer19_attention_weight_max": 40.40937042236328,
      "activations/layer19_attention_weight_min": -29.346723556518555,
      "activations/layer1_attention_weight_max": 16.52165412902832,
      "activations/layer1_attention_weight_min": -13.739935874938965,
      "activations/layer20_attention_weight_max": 36.72473907470703,
      "activations/layer20_attention_weight_min": -24.305816650390625,
      "activations/layer21_attention_weight_max": 32.28018569946289,
      "activations/layer21_attention_weight_min": -21.832778930664062,
      "activations/layer22_attention_weight_max": 51.70039749145508,
      "activations/layer22_attention_weight_min": -29.97252655029297,
      "activations/layer23_attention_weight_max": 32.55473709106445,
      "activations/layer23_attention_weight_min": -23.511259078979492,
      "activations/layer2_attention_weight_max": 30.751747131347656,
      "activations/layer2_attention_weight_min": -31.301021575927734,
      "activations/layer3_attention_weight_max": 90.60285949707031,
      "activations/layer3_attention_weight_min": -92.90706634521484,
      "activations/layer4_attention_weight_max": 126.87831115722656,
      "activations/layer4_attention_weight_min": -123.7832260131836,
      "activations/layer5_attention_weight_max": 75.93457794189453,
      "activations/layer5_attention_weight_min": -74.0879135131836,
      "activations/layer6_attention_weight_max": 54.27820587158203,
      "activations/layer6_attention_weight_min": -53.91812515258789,
      "activations/layer7_attention_weight_max": 81.26404571533203,
      "activations/layer7_attention_weight_min": -84.03984069824219,
      "activations/layer8_attention_weight_max": 39.72024154663086,
      "activations/layer8_attention_weight_min": -41.659915924072266,
      "activations/layer9_attention_weight_max": 37.48054885864258,
      "activations/layer9_attention_weight_min": -35.90497589111328,
      "epoch": 5.65,
      "learning_rate": 0.00011473181818181816,
      "loss": 2.8293,
      "step": 97200
    },
    {
      "activations/layer0_attention_weight_max": 15.2593412399292,
      "activations/layer0_attention_weight_min": -13.48658561706543,
      "activations/layer10_attention_weight_max": 32.7667236328125,
      "activations/layer10_attention_weight_min": -30.529022216796875,
      "activations/layer11_attention_weight_max": 33.24469757080078,
      "activations/layer11_attention_weight_min": -29.598997116088867,
      "activations/layer12_attention_weight_max": 25.583297729492188,
      "activations/layer12_attention_weight_min": -30.873350143432617,
      "activations/layer13_attention_weight_max": 35.28953170776367,
      "activations/layer13_attention_weight_min": -31.474239349365234,
      "activations/layer14_attention_weight_max": 54.56810760498047,
      "activations/layer14_attention_weight_min": -46.04989242553711,
      "activations/layer15_attention_weight_max": 33.94896697998047,
      "activations/layer15_attention_weight_min": -29.0098819732666,
      "activations/layer16_attention_weight_max": 32.82391357421875,
      "activations/layer16_attention_weight_min": -28.3195858001709,
      "activations/layer17_attention_weight_max": 36.286781311035156,
      "activations/layer17_attention_weight_min": -23.939661026000977,
      "activations/layer18_attention_weight_max": 30.114871978759766,
      "activations/layer18_attention_weight_min": -21.91014289855957,
      "activations/layer19_attention_weight_max": 37.7667350769043,
      "activations/layer19_attention_weight_min": -28.229602813720703,
      "activations/layer1_attention_weight_max": 17.021778106689453,
      "activations/layer1_attention_weight_min": -14.742496490478516,
      "activations/layer20_attention_weight_max": 31.345781326293945,
      "activations/layer20_attention_weight_min": -22.953325271606445,
      "activations/layer21_attention_weight_max": 28.14214324951172,
      "activations/layer21_attention_weight_min": -20.4202880859375,
      "activations/layer22_attention_weight_max": 52.11216735839844,
      "activations/layer22_attention_weight_min": -27.795835494995117,
      "activations/layer23_attention_weight_max": 32.463401794433594,
      "activations/layer23_attention_weight_min": -20.99101448059082,
      "activations/layer2_attention_weight_max": 31.832088470458984,
      "activations/layer2_attention_weight_min": -30.618993759155273,
      "activations/layer3_attention_weight_max": 90.7779541015625,
      "activations/layer3_attention_weight_min": -90.92216491699219,
      "activations/layer4_attention_weight_max": 126.52385711669922,
      "activations/layer4_attention_weight_min": -119.38492584228516,
      "activations/layer5_attention_weight_max": 77.54620361328125,
      "activations/layer5_attention_weight_min": -71.58447265625,
      "activations/layer6_attention_weight_max": 56.110626220703125,
      "activations/layer6_attention_weight_min": -55.317237854003906,
      "activations/layer7_attention_weight_max": 91.16169738769531,
      "activations/layer7_attention_weight_min": -87.05664825439453,
      "activations/layer8_attention_weight_max": 41.942108154296875,
      "activations/layer8_attention_weight_min": -40.05589294433594,
      "activations/layer9_attention_weight_max": 36.012245178222656,
      "activations/layer9_attention_weight_min": -34.378387451171875,
      "epoch": 5.65,
      "learning_rate": 0.00011471287878787878,
      "loss": 2.8362,
      "step": 97250
    },
    {
      "activations/layer0_attention_weight_max": 15.19003677368164,
      "activations/layer0_attention_weight_min": -12.417034149169922,
      "activations/layer10_attention_weight_max": 35.37895965576172,
      "activations/layer10_attention_weight_min": -31.74801254272461,
      "activations/layer11_attention_weight_max": 35.42634201049805,
      "activations/layer11_attention_weight_min": -31.454120635986328,
      "activations/layer12_attention_weight_max": 29.736600875854492,
      "activations/layer12_attention_weight_min": -26.7338924407959,
      "activations/layer13_attention_weight_max": 44.312015533447266,
      "activations/layer13_attention_weight_min": -33.00038146972656,
      "activations/layer14_attention_weight_max": 69.74838256835938,
      "activations/layer14_attention_weight_min": -53.05096435546875,
      "activations/layer15_attention_weight_max": 39.52166748046875,
      "activations/layer15_attention_weight_min": -28.66956329345703,
      "activations/layer16_attention_weight_max": 36.80577850341797,
      "activations/layer16_attention_weight_min": -29.447193145751953,
      "activations/layer17_attention_weight_max": 42.722991943359375,
      "activations/layer17_attention_weight_min": -25.001888275146484,
      "activations/layer18_attention_weight_max": 41.041072845458984,
      "activations/layer18_attention_weight_min": -20.324626922607422,
      "activations/layer19_attention_weight_max": 43.69530487060547,
      "activations/layer19_attention_weight_min": -31.83025550842285,
      "activations/layer1_attention_weight_max": 15.881482124328613,
      "activations/layer1_attention_weight_min": -14.411755561828613,
      "activations/layer20_attention_weight_max": 35.13288879394531,
      "activations/layer20_attention_weight_min": -25.315275192260742,
      "activations/layer21_attention_weight_max": 38.708988189697266,
      "activations/layer21_attention_weight_min": -21.618450164794922,
      "activations/layer22_attention_weight_max": 51.800575256347656,
      "activations/layer22_attention_weight_min": -28.473243713378906,
      "activations/layer23_attention_weight_max": 34.64506530761719,
      "activations/layer23_attention_weight_min": -22.061201095581055,
      "activations/layer2_attention_weight_max": 32.45973205566406,
      "activations/layer2_attention_weight_min": -29.99945640563965,
      "activations/layer3_attention_weight_max": 86.58760070800781,
      "activations/layer3_attention_weight_min": -88.58859252929688,
      "activations/layer4_attention_weight_max": 123.84101867675781,
      "activations/layer4_attention_weight_min": -119.71150970458984,
      "activations/layer5_attention_weight_max": 74.92726135253906,
      "activations/layer5_attention_weight_min": -72.84133911132812,
      "activations/layer6_attention_weight_max": 59.09665298461914,
      "activations/layer6_attention_weight_min": -57.4091796875,
      "activations/layer7_attention_weight_max": 87.98848724365234,
      "activations/layer7_attention_weight_min": -84.25581359863281,
      "activations/layer8_attention_weight_max": 41.52534103393555,
      "activations/layer8_attention_weight_min": -40.75101852416992,
      "activations/layer9_attention_weight_max": 38.44521713256836,
      "activations/layer9_attention_weight_min": -33.71897506713867,
      "epoch": 5.65,
      "learning_rate": 0.0001146939393939394,
      "loss": 2.8336,
      "step": 97300
    },
    {
      "activations/layer0_attention_weight_max": 15.455944061279297,
      "activations/layer0_attention_weight_min": -12.894637107849121,
      "activations/layer10_attention_weight_max": 33.430946350097656,
      "activations/layer10_attention_weight_min": -32.15594482421875,
      "activations/layer11_attention_weight_max": 32.084774017333984,
      "activations/layer11_attention_weight_min": -29.787107467651367,
      "activations/layer12_attention_weight_max": 25.735471725463867,
      "activations/layer12_attention_weight_min": -26.958860397338867,
      "activations/layer13_attention_weight_max": 39.88312911987305,
      "activations/layer13_attention_weight_min": -32.694759368896484,
      "activations/layer14_attention_weight_max": 61.81969451904297,
      "activations/layer14_attention_weight_min": -45.06757736206055,
      "activations/layer15_attention_weight_max": 35.677555084228516,
      "activations/layer15_attention_weight_min": -29.327144622802734,
      "activations/layer16_attention_weight_max": 36.9520263671875,
      "activations/layer16_attention_weight_min": -28.4825382232666,
      "activations/layer17_attention_weight_max": 39.63691329956055,
      "activations/layer17_attention_weight_min": -24.215456008911133,
      "activations/layer18_attention_weight_max": 40.34539031982422,
      "activations/layer18_attention_weight_min": -19.658109664916992,
      "activations/layer19_attention_weight_max": 40.404964447021484,
      "activations/layer19_attention_weight_min": -29.93147087097168,
      "activations/layer1_attention_weight_max": 16.58542823791504,
      "activations/layer1_attention_weight_min": -13.492717742919922,
      "activations/layer20_attention_weight_max": 33.507667541503906,
      "activations/layer20_attention_weight_min": -24.909263610839844,
      "activations/layer21_attention_weight_max": 32.364444732666016,
      "activations/layer21_attention_weight_min": -21.421875,
      "activations/layer22_attention_weight_max": 57.26084518432617,
      "activations/layer22_attention_weight_min": -28.295475006103516,
      "activations/layer23_attention_weight_max": 35.01409149169922,
      "activations/layer23_attention_weight_min": -20.246610641479492,
      "activations/layer2_attention_weight_max": 31.120494842529297,
      "activations/layer2_attention_weight_min": -31.144695281982422,
      "activations/layer3_attention_weight_max": 88.6526870727539,
      "activations/layer3_attention_weight_min": -88.21951293945312,
      "activations/layer4_attention_weight_max": 120.8566665649414,
      "activations/layer4_attention_weight_min": -121.07073974609375,
      "activations/layer5_attention_weight_max": 74.35420227050781,
      "activations/layer5_attention_weight_min": -74.18881225585938,
      "activations/layer6_attention_weight_max": 54.612281799316406,
      "activations/layer6_attention_weight_min": -54.68433380126953,
      "activations/layer7_attention_weight_max": 82.31803894042969,
      "activations/layer7_attention_weight_min": -87.78392791748047,
      "activations/layer8_attention_weight_max": 42.029747009277344,
      "activations/layer8_attention_weight_min": -41.30522537231445,
      "activations/layer9_attention_weight_max": 37.16715621948242,
      "activations/layer9_attention_weight_min": -34.308815002441406,
      "epoch": 5.66,
      "learning_rate": 0.00011467499999999998,
      "loss": 2.8354,
      "step": 97350
    },
    {
      "activations/layer0_attention_weight_max": 16.09049415588379,
      "activations/layer0_attention_weight_min": -12.646244049072266,
      "activations/layer10_attention_weight_max": 35.00142288208008,
      "activations/layer10_attention_weight_min": -29.90866470336914,
      "activations/layer11_attention_weight_max": 33.67729949951172,
      "activations/layer11_attention_weight_min": -26.816104888916016,
      "activations/layer12_attention_weight_max": 25.224912643432617,
      "activations/layer12_attention_weight_min": -26.73270034790039,
      "activations/layer13_attention_weight_max": 40.94300079345703,
      "activations/layer13_attention_weight_min": -29.776344299316406,
      "activations/layer14_attention_weight_max": 58.68496322631836,
      "activations/layer14_attention_weight_min": -43.64933395385742,
      "activations/layer15_attention_weight_max": 35.40199661254883,
      "activations/layer15_attention_weight_min": -28.617677688598633,
      "activations/layer16_attention_weight_max": 33.52383041381836,
      "activations/layer16_attention_weight_min": -26.69374656677246,
      "activations/layer17_attention_weight_max": 36.844512939453125,
      "activations/layer17_attention_weight_min": -24.950387954711914,
      "activations/layer18_attention_weight_max": 30.366188049316406,
      "activations/layer18_attention_weight_min": -20.466121673583984,
      "activations/layer19_attention_weight_max": 40.202545166015625,
      "activations/layer19_attention_weight_min": -29.016576766967773,
      "activations/layer1_attention_weight_max": 17.244203567504883,
      "activations/layer1_attention_weight_min": -13.788990020751953,
      "activations/layer20_attention_weight_max": 30.18852996826172,
      "activations/layer20_attention_weight_min": -23.1788330078125,
      "activations/layer21_attention_weight_max": 29.917409896850586,
      "activations/layer21_attention_weight_min": -23.840740203857422,
      "activations/layer22_attention_weight_max": 51.81325912475586,
      "activations/layer22_attention_weight_min": -28.265058517456055,
      "activations/layer23_attention_weight_max": 32.896812438964844,
      "activations/layer23_attention_weight_min": -20.513263702392578,
      "activations/layer2_attention_weight_max": 32.88760757446289,
      "activations/layer2_attention_weight_min": -31.43954086303711,
      "activations/layer3_attention_weight_max": 89.04127502441406,
      "activations/layer3_attention_weight_min": -91.53646087646484,
      "activations/layer4_attention_weight_max": 123.89995574951172,
      "activations/layer4_attention_weight_min": -126.24674224853516,
      "activations/layer5_attention_weight_max": 73.03551483154297,
      "activations/layer5_attention_weight_min": -81.89125061035156,
      "activations/layer6_attention_weight_max": 52.34421920776367,
      "activations/layer6_attention_weight_min": -55.1241569519043,
      "activations/layer7_attention_weight_max": 83.20478057861328,
      "activations/layer7_attention_weight_min": -81.43260192871094,
      "activations/layer8_attention_weight_max": 39.21038818359375,
      "activations/layer8_attention_weight_min": -38.702781677246094,
      "activations/layer9_attention_weight_max": 36.69563674926758,
      "activations/layer9_attention_weight_min": -34.08982849121094,
      "epoch": 5.66,
      "learning_rate": 0.0001146560606060606,
      "loss": 2.8328,
      "step": 97400
    },
    {
      "activations/layer0_attention_weight_max": 15.052591323852539,
      "activations/layer0_attention_weight_min": -12.872302055358887,
      "activations/layer10_attention_weight_max": 36.467323303222656,
      "activations/layer10_attention_weight_min": -31.076950073242188,
      "activations/layer11_attention_weight_max": 42.26150894165039,
      "activations/layer11_attention_weight_min": -28.49776840209961,
      "activations/layer12_attention_weight_max": 37.766273498535156,
      "activations/layer12_attention_weight_min": -27.43842124938965,
      "activations/layer13_attention_weight_max": 55.48127365112305,
      "activations/layer13_attention_weight_min": -39.1954345703125,
      "activations/layer14_attention_weight_max": 67.45411682128906,
      "activations/layer14_attention_weight_min": -51.36932373046875,
      "activations/layer15_attention_weight_max": 60.228553771972656,
      "activations/layer15_attention_weight_min": -35.04570388793945,
      "activations/layer16_attention_weight_max": 39.00530242919922,
      "activations/layer16_attention_weight_min": -27.824907302856445,
      "activations/layer17_attention_weight_max": 38.8776969909668,
      "activations/layer17_attention_weight_min": -25.94895362854004,
      "activations/layer18_attention_weight_max": 33.071014404296875,
      "activations/layer18_attention_weight_min": -22.265392303466797,
      "activations/layer19_attention_weight_max": 41.39010238647461,
      "activations/layer19_attention_weight_min": -29.29672622680664,
      "activations/layer1_attention_weight_max": 17.87452507019043,
      "activations/layer1_attention_weight_min": -13.53080940246582,
      "activations/layer20_attention_weight_max": 34.80838394165039,
      "activations/layer20_attention_weight_min": -23.821346282958984,
      "activations/layer21_attention_weight_max": 33.030799865722656,
      "activations/layer21_attention_weight_min": -21.347579956054688,
      "activations/layer22_attention_weight_max": 51.60434341430664,
      "activations/layer22_attention_weight_min": -29.830259323120117,
      "activations/layer23_attention_weight_max": 35.4034423828125,
      "activations/layer23_attention_weight_min": -22.870216369628906,
      "activations/layer2_attention_weight_max": 33.05837631225586,
      "activations/layer2_attention_weight_min": -31.447002410888672,
      "activations/layer3_attention_weight_max": 93.26251983642578,
      "activations/layer3_attention_weight_min": -93.85289001464844,
      "activations/layer4_attention_weight_max": 130.18946838378906,
      "activations/layer4_attention_weight_min": -131.3214569091797,
      "activations/layer5_attention_weight_max": 76.762939453125,
      "activations/layer5_attention_weight_min": -74.73912811279297,
      "activations/layer6_attention_weight_max": 55.88209915161133,
      "activations/layer6_attention_weight_min": -58.39485549926758,
      "activations/layer7_attention_weight_max": 93.09335327148438,
      "activations/layer7_attention_weight_min": -90.93124389648438,
      "activations/layer8_attention_weight_max": 41.22934341430664,
      "activations/layer8_attention_weight_min": -40.99519729614258,
      "activations/layer9_attention_weight_max": 37.60105514526367,
      "activations/layer9_attention_weight_min": -40.55989074707031,
      "epoch": 5.66,
      "learning_rate": 0.00011463712121212121,
      "loss": 2.8404,
      "step": 97450
    },
    {
      "activations/layer0_attention_weight_max": 15.361713409423828,
      "activations/layer0_attention_weight_min": -13.08215618133545,
      "activations/layer10_attention_weight_max": 36.3622932434082,
      "activations/layer10_attention_weight_min": -30.75906753540039,
      "activations/layer11_attention_weight_max": 37.106346130371094,
      "activations/layer11_attention_weight_min": -28.53523063659668,
      "activations/layer12_attention_weight_max": 26.57841682434082,
      "activations/layer12_attention_weight_min": -24.867551803588867,
      "activations/layer13_attention_weight_max": 45.52168273925781,
      "activations/layer13_attention_weight_min": -31.515464782714844,
      "activations/layer14_attention_weight_max": 65.00662994384766,
      "activations/layer14_attention_weight_min": -47.113712310791016,
      "activations/layer15_attention_weight_max": 39.26765823364258,
      "activations/layer15_attention_weight_min": -29.525054931640625,
      "activations/layer16_attention_weight_max": 37.45640182495117,
      "activations/layer16_attention_weight_min": -28.018634796142578,
      "activations/layer17_attention_weight_max": 37.922019958496094,
      "activations/layer17_attention_weight_min": -25.446563720703125,
      "activations/layer18_attention_weight_max": 32.556156158447266,
      "activations/layer18_attention_weight_min": -20.60508918762207,
      "activations/layer19_attention_weight_max": 38.546600341796875,
      "activations/layer19_attention_weight_min": -30.74258804321289,
      "activations/layer1_attention_weight_max": 17.094816207885742,
      "activations/layer1_attention_weight_min": -17.947917938232422,
      "activations/layer20_attention_weight_max": 31.454063415527344,
      "activations/layer20_attention_weight_min": -23.718130111694336,
      "activations/layer21_attention_weight_max": 32.754947662353516,
      "activations/layer21_attention_weight_min": -23.318038940429688,
      "activations/layer22_attention_weight_max": 48.78325653076172,
      "activations/layer22_attention_weight_min": -31.881181716918945,
      "activations/layer23_attention_weight_max": 34.82389450073242,
      "activations/layer23_attention_weight_min": -20.989994049072266,
      "activations/layer2_attention_weight_max": 33.6241340637207,
      "activations/layer2_attention_weight_min": -32.63200378417969,
      "activations/layer3_attention_weight_max": 89.26874542236328,
      "activations/layer3_attention_weight_min": -88.96406555175781,
      "activations/layer4_attention_weight_max": 129.83078002929688,
      "activations/layer4_attention_weight_min": -122.09163665771484,
      "activations/layer5_attention_weight_max": 79.22037506103516,
      "activations/layer5_attention_weight_min": -73.85596466064453,
      "activations/layer6_attention_weight_max": 55.9547233581543,
      "activations/layer6_attention_weight_min": -60.30921936035156,
      "activations/layer7_attention_weight_max": 88.59159088134766,
      "activations/layer7_attention_weight_min": -91.49888610839844,
      "activations/layer8_attention_weight_max": 42.10227966308594,
      "activations/layer8_attention_weight_min": -39.04561996459961,
      "activations/layer9_attention_weight_max": 36.42094039916992,
      "activations/layer9_attention_weight_min": -33.29835510253906,
      "epoch": 5.67,
      "learning_rate": 0.0001146181818181818,
      "loss": 2.8556,
      "step": 97500
    },
    {
      "activations/layer0_attention_weight_max": 16.326940536499023,
      "activations/layer0_attention_weight_min": -12.507326126098633,
      "activations/layer10_attention_weight_max": 35.30089569091797,
      "activations/layer10_attention_weight_min": -30.684799194335938,
      "activations/layer11_attention_weight_max": 34.58114242553711,
      "activations/layer11_attention_weight_min": -27.928951263427734,
      "activations/layer12_attention_weight_max": 30.834514617919922,
      "activations/layer12_attention_weight_min": -30.9007625579834,
      "activations/layer13_attention_weight_max": 45.18913269042969,
      "activations/layer13_attention_weight_min": -33.510231018066406,
      "activations/layer14_attention_weight_max": 68.75704956054688,
      "activations/layer14_attention_weight_min": -44.19721221923828,
      "activations/layer15_attention_weight_max": 39.17780303955078,
      "activations/layer15_attention_weight_min": -27.637563705444336,
      "activations/layer16_attention_weight_max": 39.35287857055664,
      "activations/layer16_attention_weight_min": -29.048891067504883,
      "activations/layer17_attention_weight_max": 38.32155227661133,
      "activations/layer17_attention_weight_min": -25.38165855407715,
      "activations/layer18_attention_weight_max": 29.82826042175293,
      "activations/layer18_attention_weight_min": -20.70063018798828,
      "activations/layer19_attention_weight_max": 44.29235076904297,
      "activations/layer19_attention_weight_min": -28.006956100463867,
      "activations/layer1_attention_weight_max": 16.650205612182617,
      "activations/layer1_attention_weight_min": -16.641437530517578,
      "activations/layer20_attention_weight_max": 35.419105529785156,
      "activations/layer20_attention_weight_min": -23.817768096923828,
      "activations/layer21_attention_weight_max": 34.90325164794922,
      "activations/layer21_attention_weight_min": -20.275020599365234,
      "activations/layer22_attention_weight_max": 54.85454177856445,
      "activations/layer22_attention_weight_min": -27.123451232910156,
      "activations/layer23_attention_weight_max": 34.845638275146484,
      "activations/layer23_attention_weight_min": -21.284561157226562,
      "activations/layer2_attention_weight_max": 34.97422409057617,
      "activations/layer2_attention_weight_min": -32.92317199707031,
      "activations/layer3_attention_weight_max": 93.55006408691406,
      "activations/layer3_attention_weight_min": -89.34059143066406,
      "activations/layer4_attention_weight_max": 130.0671844482422,
      "activations/layer4_attention_weight_min": -117.62257385253906,
      "activations/layer5_attention_weight_max": 74.87501525878906,
      "activations/layer5_attention_weight_min": -69.58369445800781,
      "activations/layer6_attention_weight_max": 57.68223190307617,
      "activations/layer6_attention_weight_min": -54.597801208496094,
      "activations/layer7_attention_weight_max": 88.3227310180664,
      "activations/layer7_attention_weight_min": -88.35426330566406,
      "activations/layer8_attention_weight_max": 42.327354431152344,
      "activations/layer8_attention_weight_min": -39.001556396484375,
      "activations/layer9_attention_weight_max": 37.27676010131836,
      "activations/layer9_attention_weight_min": -34.4002799987793,
      "epoch": 5.67,
      "learning_rate": 0.00011459924242424242,
      "loss": 2.8292,
      "step": 97550
    },
    {
      "activations/layer0_attention_weight_max": 16.00051498413086,
      "activations/layer0_attention_weight_min": -12.296435356140137,
      "activations/layer10_attention_weight_max": 33.75974655151367,
      "activations/layer10_attention_weight_min": -30.558530807495117,
      "activations/layer11_attention_weight_max": 33.94676208496094,
      "activations/layer11_attention_weight_min": -29.131542205810547,
      "activations/layer12_attention_weight_max": 49.881103515625,
      "activations/layer12_attention_weight_min": -25.70040512084961,
      "activations/layer13_attention_weight_max": 47.06742858886719,
      "activations/layer13_attention_weight_min": -34.296932220458984,
      "activations/layer14_attention_weight_max": 79.54932403564453,
      "activations/layer14_attention_weight_min": -56.1741943359375,
      "activations/layer15_attention_weight_max": 41.922916412353516,
      "activations/layer15_attention_weight_min": -28.632678985595703,
      "activations/layer16_attention_weight_max": 44.16972351074219,
      "activations/layer16_attention_weight_min": -28.518814086914062,
      "activations/layer17_attention_weight_max": 39.05080032348633,
      "activations/layer17_attention_weight_min": -25.433990478515625,
      "activations/layer18_attention_weight_max": 33.94086837768555,
      "activations/layer18_attention_weight_min": -20.072799682617188,
      "activations/layer19_attention_weight_max": 42.51287078857422,
      "activations/layer19_attention_weight_min": -31.68100929260254,
      "activations/layer1_attention_weight_max": 17.65909194946289,
      "activations/layer1_attention_weight_min": -15.374215126037598,
      "activations/layer20_attention_weight_max": 30.589231491088867,
      "activations/layer20_attention_weight_min": -23.709070205688477,
      "activations/layer21_attention_weight_max": 33.0955924987793,
      "activations/layer21_attention_weight_min": -23.610002517700195,
      "activations/layer22_attention_weight_max": 49.032997131347656,
      "activations/layer22_attention_weight_min": -27.892812728881836,
      "activations/layer23_attention_weight_max": 33.8746337890625,
      "activations/layer23_attention_weight_min": -21.621702194213867,
      "activations/layer2_attention_weight_max": 32.175682067871094,
      "activations/layer2_attention_weight_min": -31.795896530151367,
      "activations/layer3_attention_weight_max": 90.76400756835938,
      "activations/layer3_attention_weight_min": -90.09577941894531,
      "activations/layer4_attention_weight_max": 124.2513427734375,
      "activations/layer4_attention_weight_min": -121.4493408203125,
      "activations/layer5_attention_weight_max": 73.3807373046875,
      "activations/layer5_attention_weight_min": -72.95726013183594,
      "activations/layer6_attention_weight_max": 55.62775421142578,
      "activations/layer6_attention_weight_min": -53.14494323730469,
      "activations/layer7_attention_weight_max": 85.53924560546875,
      "activations/layer7_attention_weight_min": -82.94566345214844,
      "activations/layer8_attention_weight_max": 41.12736892700195,
      "activations/layer8_attention_weight_min": -39.49492263793945,
      "activations/layer9_attention_weight_max": 35.074462890625,
      "activations/layer9_attention_weight_min": -35.25336837768555,
      "epoch": 5.67,
      "learning_rate": 0.00011458030303030303,
      "loss": 2.8493,
      "step": 97600
    },
    {
      "activations/layer0_attention_weight_max": 16.757307052612305,
      "activations/layer0_attention_weight_min": -11.853256225585938,
      "activations/layer10_attention_weight_max": 36.049808502197266,
      "activations/layer10_attention_weight_min": -31.395164489746094,
      "activations/layer11_attention_weight_max": 41.129215240478516,
      "activations/layer11_attention_weight_min": -30.047094345092773,
      "activations/layer12_attention_weight_max": 36.7240104675293,
      "activations/layer12_attention_weight_min": -29.846437454223633,
      "activations/layer13_attention_weight_max": 58.909156799316406,
      "activations/layer13_attention_weight_min": -41.5213508605957,
      "activations/layer14_attention_weight_max": 59.77446746826172,
      "activations/layer14_attention_weight_min": -40.33481216430664,
      "activations/layer15_attention_weight_max": 57.67008972167969,
      "activations/layer15_attention_weight_min": -39.73899459838867,
      "activations/layer16_attention_weight_max": 36.71833801269531,
      "activations/layer16_attention_weight_min": -27.107250213623047,
      "activations/layer17_attention_weight_max": 34.25044631958008,
      "activations/layer17_attention_weight_min": -22.859556198120117,
      "activations/layer18_attention_weight_max": 31.584274291992188,
      "activations/layer18_attention_weight_min": -22.473190307617188,
      "activations/layer19_attention_weight_max": 33.843849182128906,
      "activations/layer19_attention_weight_min": -27.565305709838867,
      "activations/layer1_attention_weight_max": 17.449108123779297,
      "activations/layer1_attention_weight_min": -14.716276168823242,
      "activations/layer20_attention_weight_max": 30.02666473388672,
      "activations/layer20_attention_weight_min": -21.24805450439453,
      "activations/layer21_attention_weight_max": 26.971120834350586,
      "activations/layer21_attention_weight_min": -19.05668830871582,
      "activations/layer22_attention_weight_max": 43.022735595703125,
      "activations/layer22_attention_weight_min": -25.26209259033203,
      "activations/layer23_attention_weight_max": 31.16938018798828,
      "activations/layer23_attention_weight_min": -20.41661834716797,
      "activations/layer2_attention_weight_max": 34.4449462890625,
      "activations/layer2_attention_weight_min": -33.47858428955078,
      "activations/layer3_attention_weight_max": 99.05105590820312,
      "activations/layer3_attention_weight_min": -95.79827880859375,
      "activations/layer4_attention_weight_max": 131.87010192871094,
      "activations/layer4_attention_weight_min": -130.0645751953125,
      "activations/layer5_attention_weight_max": 75.91227722167969,
      "activations/layer5_attention_weight_min": -76.53573608398438,
      "activations/layer6_attention_weight_max": 58.217994689941406,
      "activations/layer6_attention_weight_min": -58.08710479736328,
      "activations/layer7_attention_weight_max": 94.30432891845703,
      "activations/layer7_attention_weight_min": -84.63556671142578,
      "activations/layer8_attention_weight_max": 39.43159866333008,
      "activations/layer8_attention_weight_min": -41.18708419799805,
      "activations/layer9_attention_weight_max": 35.96220016479492,
      "activations/layer9_attention_weight_min": -37.8701171875,
      "epoch": 5.67,
      "learning_rate": 0.00011456136363636362,
      "loss": 2.8427,
      "step": 97650
    },
    {
      "activations/layer0_attention_weight_max": 16.3795166015625,
      "activations/layer0_attention_weight_min": -12.301196098327637,
      "activations/layer10_attention_weight_max": 33.219268798828125,
      "activations/layer10_attention_weight_min": -32.5628662109375,
      "activations/layer11_attention_weight_max": 33.043006896972656,
      "activations/layer11_attention_weight_min": -28.40791893005371,
      "activations/layer12_attention_weight_max": 30.327350616455078,
      "activations/layer12_attention_weight_min": -30.02013397216797,
      "activations/layer13_attention_weight_max": 43.27354431152344,
      "activations/layer13_attention_weight_min": -33.21561813354492,
      "activations/layer14_attention_weight_max": 58.42934036254883,
      "activations/layer14_attention_weight_min": -42.62140655517578,
      "activations/layer15_attention_weight_max": 36.94902801513672,
      "activations/layer15_attention_weight_min": -29.429298400878906,
      "activations/layer16_attention_weight_max": 37.257720947265625,
      "activations/layer16_attention_weight_min": -30.326433181762695,
      "activations/layer17_attention_weight_max": 37.413700103759766,
      "activations/layer17_attention_weight_min": -26.5492000579834,
      "activations/layer18_attention_weight_max": 32.44914627075195,
      "activations/layer18_attention_weight_min": -21.898269653320312,
      "activations/layer19_attention_weight_max": 37.25334167480469,
      "activations/layer19_attention_weight_min": -30.274993896484375,
      "activations/layer1_attention_weight_max": 18.128173828125,
      "activations/layer1_attention_weight_min": -15.4052095413208,
      "activations/layer20_attention_weight_max": 31.190584182739258,
      "activations/layer20_attention_weight_min": -23.23422622680664,
      "activations/layer21_attention_weight_max": 34.808128356933594,
      "activations/layer21_attention_weight_min": -21.861318588256836,
      "activations/layer22_attention_weight_max": 45.72915267944336,
      "activations/layer22_attention_weight_min": -28.46373176574707,
      "activations/layer23_attention_weight_max": 32.779151916503906,
      "activations/layer23_attention_weight_min": -21.23538589477539,
      "activations/layer2_attention_weight_max": 32.432273864746094,
      "activations/layer2_attention_weight_min": -31.57996368408203,
      "activations/layer3_attention_weight_max": 88.90174865722656,
      "activations/layer3_attention_weight_min": -91.05445861816406,
      "activations/layer4_attention_weight_max": 131.11643981933594,
      "activations/layer4_attention_weight_min": -124.97151184082031,
      "activations/layer5_attention_weight_max": 78.47102355957031,
      "activations/layer5_attention_weight_min": -71.99812316894531,
      "activations/layer6_attention_weight_max": 56.40301513671875,
      "activations/layer6_attention_weight_min": -56.9675407409668,
      "activations/layer7_attention_weight_max": 88.49760437011719,
      "activations/layer7_attention_weight_min": -85.6143798828125,
      "activations/layer8_attention_weight_max": 41.875999450683594,
      "activations/layer8_attention_weight_min": -39.789852142333984,
      "activations/layer9_attention_weight_max": 34.6618537902832,
      "activations/layer9_attention_weight_min": -34.30625534057617,
      "epoch": 5.68,
      "learning_rate": 0.00011454242424242423,
      "loss": 2.8571,
      "step": 97700
    },
    {
      "activations/layer0_attention_weight_max": 16.416534423828125,
      "activations/layer0_attention_weight_min": -11.732802391052246,
      "activations/layer10_attention_weight_max": 36.97324752807617,
      "activations/layer10_attention_weight_min": -33.61278533935547,
      "activations/layer11_attention_weight_max": 32.85275650024414,
      "activations/layer11_attention_weight_min": -30.323301315307617,
      "activations/layer12_attention_weight_max": 29.089357376098633,
      "activations/layer12_attention_weight_min": -28.10163116455078,
      "activations/layer13_attention_weight_max": 47.27518844604492,
      "activations/layer13_attention_weight_min": -32.662593841552734,
      "activations/layer14_attention_weight_max": 66.63154602050781,
      "activations/layer14_attention_weight_min": -42.5031623840332,
      "activations/layer15_attention_weight_max": 42.23897171020508,
      "activations/layer15_attention_weight_min": -27.488191604614258,
      "activations/layer16_attention_weight_max": 38.917938232421875,
      "activations/layer16_attention_weight_min": -26.839216232299805,
      "activations/layer17_attention_weight_max": 37.97145080566406,
      "activations/layer17_attention_weight_min": -25.845367431640625,
      "activations/layer18_attention_weight_max": 35.69319534301758,
      "activations/layer18_attention_weight_min": -21.133705139160156,
      "activations/layer19_attention_weight_max": 41.798606872558594,
      "activations/layer19_attention_weight_min": -28.35109519958496,
      "activations/layer1_attention_weight_max": 17.040058135986328,
      "activations/layer1_attention_weight_min": -15.482125282287598,
      "activations/layer20_attention_weight_max": 33.35760498046875,
      "activations/layer20_attention_weight_min": -25.583677291870117,
      "activations/layer21_attention_weight_max": 35.91168212890625,
      "activations/layer21_attention_weight_min": -22.361797332763672,
      "activations/layer22_attention_weight_max": 53.7215576171875,
      "activations/layer22_attention_weight_min": -28.64031410217285,
      "activations/layer23_attention_weight_max": 36.95093536376953,
      "activations/layer23_attention_weight_min": -21.779081344604492,
      "activations/layer2_attention_weight_max": 32.91654968261719,
      "activations/layer2_attention_weight_min": -31.937274932861328,
      "activations/layer3_attention_weight_max": 88.46985626220703,
      "activations/layer3_attention_weight_min": -88.45062255859375,
      "activations/layer4_attention_weight_max": 127.79388427734375,
      "activations/layer4_attention_weight_min": -123.93453216552734,
      "activations/layer5_attention_weight_max": 76.30393981933594,
      "activations/layer5_attention_weight_min": -73.36611938476562,
      "activations/layer6_attention_weight_max": 54.509483337402344,
      "activations/layer6_attention_weight_min": -53.96794509887695,
      "activations/layer7_attention_weight_max": 89.007080078125,
      "activations/layer7_attention_weight_min": -83.62731170654297,
      "activations/layer8_attention_weight_max": 42.44460678100586,
      "activations/layer8_attention_weight_min": -41.05351638793945,
      "activations/layer9_attention_weight_max": 38.73104476928711,
      "activations/layer9_attention_weight_min": -35.675392150878906,
      "epoch": 5.68,
      "learning_rate": 0.00011452348484848483,
      "loss": 2.846,
      "step": 97750
    },
    {
      "activations/layer0_attention_weight_max": 16.29777717590332,
      "activations/layer0_attention_weight_min": -12.477564811706543,
      "activations/layer10_attention_weight_max": 33.42963409423828,
      "activations/layer10_attention_weight_min": -33.473899841308594,
      "activations/layer11_attention_weight_max": 35.07806396484375,
      "activations/layer11_attention_weight_min": -27.769834518432617,
      "activations/layer12_attention_weight_max": 31.58552360534668,
      "activations/layer12_attention_weight_min": -28.198543548583984,
      "activations/layer13_attention_weight_max": 48.13502502441406,
      "activations/layer13_attention_weight_min": -32.13264083862305,
      "activations/layer14_attention_weight_max": 56.80213928222656,
      "activations/layer14_attention_weight_min": -42.674644470214844,
      "activations/layer15_attention_weight_max": 46.6059455871582,
      "activations/layer15_attention_weight_min": -30.12568473815918,
      "activations/layer16_attention_weight_max": 36.804317474365234,
      "activations/layer16_attention_weight_min": -29.71760368347168,
      "activations/layer17_attention_weight_max": 37.50749969482422,
      "activations/layer17_attention_weight_min": -24.402734756469727,
      "activations/layer18_attention_weight_max": 36.947757720947266,
      "activations/layer18_attention_weight_min": -18.914093017578125,
      "activations/layer19_attention_weight_max": 39.034507751464844,
      "activations/layer19_attention_weight_min": -31.95595359802246,
      "activations/layer1_attention_weight_max": 16.642250061035156,
      "activations/layer1_attention_weight_min": -16.554794311523438,
      "activations/layer20_attention_weight_max": 32.57390213012695,
      "activations/layer20_attention_weight_min": -25.112842559814453,
      "activations/layer21_attention_weight_max": 35.03564453125,
      "activations/layer21_attention_weight_min": -23.702953338623047,
      "activations/layer22_attention_weight_max": 52.5438346862793,
      "activations/layer22_attention_weight_min": -27.61638069152832,
      "activations/layer23_attention_weight_max": 35.62487030029297,
      "activations/layer23_attention_weight_min": -21.151203155517578,
      "activations/layer2_attention_weight_max": 32.72072982788086,
      "activations/layer2_attention_weight_min": -31.985990524291992,
      "activations/layer3_attention_weight_max": 92.41651153564453,
      "activations/layer3_attention_weight_min": -91.9819564819336,
      "activations/layer4_attention_weight_max": 129.44482421875,
      "activations/layer4_attention_weight_min": -126.6725082397461,
      "activations/layer5_attention_weight_max": 75.43179321289062,
      "activations/layer5_attention_weight_min": -73.6285171508789,
      "activations/layer6_attention_weight_max": 52.858421325683594,
      "activations/layer6_attention_weight_min": -56.240543365478516,
      "activations/layer7_attention_weight_max": 85.71715545654297,
      "activations/layer7_attention_weight_min": -85.2296142578125,
      "activations/layer8_attention_weight_max": 39.151607513427734,
      "activations/layer8_attention_weight_min": -39.63397979736328,
      "activations/layer9_attention_weight_max": 33.58972930908203,
      "activations/layer9_attention_weight_min": -35.77334213256836,
      "epoch": 5.68,
      "learning_rate": 0.00011450454545454545,
      "loss": 2.8372,
      "step": 97800
    },
    {
      "activations/layer0_attention_weight_max": 15.05981159210205,
      "activations/layer0_attention_weight_min": -12.643172264099121,
      "activations/layer10_attention_weight_max": 36.92644500732422,
      "activations/layer10_attention_weight_min": -31.239700317382812,
      "activations/layer11_attention_weight_max": 38.716278076171875,
      "activations/layer11_attention_weight_min": -29.87828826904297,
      "activations/layer12_attention_weight_max": 26.475818634033203,
      "activations/layer12_attention_weight_min": -27.89722442626953,
      "activations/layer13_attention_weight_max": 53.461570739746094,
      "activations/layer13_attention_weight_min": -31.85318946838379,
      "activations/layer14_attention_weight_max": 74.1939468383789,
      "activations/layer14_attention_weight_min": -46.884830474853516,
      "activations/layer15_attention_weight_max": 42.70976638793945,
      "activations/layer15_attention_weight_min": -28.06100845336914,
      "activations/layer16_attention_weight_max": 45.41068649291992,
      "activations/layer16_attention_weight_min": -29.28319549560547,
      "activations/layer17_attention_weight_max": 46.62604904174805,
      "activations/layer17_attention_weight_min": -24.154647827148438,
      "activations/layer18_attention_weight_max": 42.3531379699707,
      "activations/layer18_attention_weight_min": -21.19513511657715,
      "activations/layer19_attention_weight_max": 45.23716354370117,
      "activations/layer19_attention_weight_min": -28.5773983001709,
      "activations/layer1_attention_weight_max": 17.142419815063477,
      "activations/layer1_attention_weight_min": -16.408491134643555,
      "activations/layer20_attention_weight_max": 39.291709899902344,
      "activations/layer20_attention_weight_min": -22.51875877380371,
      "activations/layer21_attention_weight_max": 39.11450958251953,
      "activations/layer21_attention_weight_min": -21.210725784301758,
      "activations/layer22_attention_weight_max": 60.71869659423828,
      "activations/layer22_attention_weight_min": -28.66670799255371,
      "activations/layer23_attention_weight_max": 36.97739791870117,
      "activations/layer23_attention_weight_min": -20.760562896728516,
      "activations/layer2_attention_weight_max": 32.42366409301758,
      "activations/layer2_attention_weight_min": -33.93146514892578,
      "activations/layer3_attention_weight_max": 95.00199127197266,
      "activations/layer3_attention_weight_min": -98.36688232421875,
      "activations/layer4_attention_weight_max": 127.58973693847656,
      "activations/layer4_attention_weight_min": -128.27487182617188,
      "activations/layer5_attention_weight_max": 75.95545959472656,
      "activations/layer5_attention_weight_min": -75.40396118164062,
      "activations/layer6_attention_weight_max": 53.01774215698242,
      "activations/layer6_attention_weight_min": -57.89421081542969,
      "activations/layer7_attention_weight_max": 86.87959289550781,
      "activations/layer7_attention_weight_min": -88.35332489013672,
      "activations/layer8_attention_weight_max": 43.05625534057617,
      "activations/layer8_attention_weight_min": -42.61686706542969,
      "activations/layer9_attention_weight_max": 38.0057373046875,
      "activations/layer9_attention_weight_min": -35.38444519042969,
      "epoch": 5.69,
      "learning_rate": 0.00011448560606060605,
      "loss": 2.8511,
      "step": 97850
    },
    {
      "activations/layer0_attention_weight_max": 14.443883895874023,
      "activations/layer0_attention_weight_min": -12.451790809631348,
      "activations/layer10_attention_weight_max": 35.40619659423828,
      "activations/layer10_attention_weight_min": -32.678043365478516,
      "activations/layer11_attention_weight_max": 38.00784683227539,
      "activations/layer11_attention_weight_min": -29.822593688964844,
      "activations/layer12_attention_weight_max": 24.320316314697266,
      "activations/layer12_attention_weight_min": -25.469364166259766,
      "activations/layer13_attention_weight_max": 46.693016052246094,
      "activations/layer13_attention_weight_min": -31.630752563476562,
      "activations/layer14_attention_weight_max": 74.2564926147461,
      "activations/layer14_attention_weight_min": -61.194480895996094,
      "activations/layer15_attention_weight_max": 42.278236389160156,
      "activations/layer15_attention_weight_min": -28.322181701660156,
      "activations/layer16_attention_weight_max": 38.67764663696289,
      "activations/layer16_attention_weight_min": -27.635969161987305,
      "activations/layer17_attention_weight_max": 41.419368743896484,
      "activations/layer17_attention_weight_min": -25.679443359375,
      "activations/layer18_attention_weight_max": 37.608924865722656,
      "activations/layer18_attention_weight_min": -22.12845802307129,
      "activations/layer19_attention_weight_max": 44.433284759521484,
      "activations/layer19_attention_weight_min": -30.910776138305664,
      "activations/layer1_attention_weight_max": 16.976200103759766,
      "activations/layer1_attention_weight_min": -15.403557777404785,
      "activations/layer20_attention_weight_max": 34.337371826171875,
      "activations/layer20_attention_weight_min": -24.625566482543945,
      "activations/layer21_attention_weight_max": 35.95973587036133,
      "activations/layer21_attention_weight_min": -21.309097290039062,
      "activations/layer22_attention_weight_max": 51.9507942199707,
      "activations/layer22_attention_weight_min": -28.703521728515625,
      "activations/layer23_attention_weight_max": 35.64021301269531,
      "activations/layer23_attention_weight_min": -22.528263092041016,
      "activations/layer2_attention_weight_max": 32.139495849609375,
      "activations/layer2_attention_weight_min": -31.06220245361328,
      "activations/layer3_attention_weight_max": 87.65992736816406,
      "activations/layer3_attention_weight_min": -88.63690948486328,
      "activations/layer4_attention_weight_max": 127.15811920166016,
      "activations/layer4_attention_weight_min": -124.8226318359375,
      "activations/layer5_attention_weight_max": 78.58123779296875,
      "activations/layer5_attention_weight_min": -76.35860443115234,
      "activations/layer6_attention_weight_max": 55.83222579956055,
      "activations/layer6_attention_weight_min": -60.90561294555664,
      "activations/layer7_attention_weight_max": 85.13003540039062,
      "activations/layer7_attention_weight_min": -84.02418518066406,
      "activations/layer8_attention_weight_max": 40.32942199707031,
      "activations/layer8_attention_weight_min": -41.2536506652832,
      "activations/layer9_attention_weight_max": 37.32706832885742,
      "activations/layer9_attention_weight_min": -34.533687591552734,
      "epoch": 5.69,
      "learning_rate": 0.00011446666666666665,
      "loss": 2.8235,
      "step": 97900
    },
    {
      "activations/layer0_attention_weight_max": 15.98585319519043,
      "activations/layer0_attention_weight_min": -12.73913288116455,
      "activations/layer10_attention_weight_max": 32.080543518066406,
      "activations/layer10_attention_weight_min": -30.20705223083496,
      "activations/layer11_attention_weight_max": 29.026533126831055,
      "activations/layer11_attention_weight_min": -27.130882263183594,
      "activations/layer12_attention_weight_max": 26.56843376159668,
      "activations/layer12_attention_weight_min": -29.590848922729492,
      "activations/layer13_attention_weight_max": 37.27739334106445,
      "activations/layer13_attention_weight_min": -32.761634826660156,
      "activations/layer14_attention_weight_max": 63.505130767822266,
      "activations/layer14_attention_weight_min": -45.25787353515625,
      "activations/layer15_attention_weight_max": 38.3586311340332,
      "activations/layer15_attention_weight_min": -29.250402450561523,
      "activations/layer16_attention_weight_max": 37.2729377746582,
      "activations/layer16_attention_weight_min": -27.591962814331055,
      "activations/layer17_attention_weight_max": 33.874046325683594,
      "activations/layer17_attention_weight_min": -23.294370651245117,
      "activations/layer18_attention_weight_max": 27.969688415527344,
      "activations/layer18_attention_weight_min": -19.981367111206055,
      "activations/layer19_attention_weight_max": 47.17245864868164,
      "activations/layer19_attention_weight_min": -29.07745933532715,
      "activations/layer1_attention_weight_max": 16.76886749267578,
      "activations/layer1_attention_weight_min": -14.798432350158691,
      "activations/layer20_attention_weight_max": 35.12815856933594,
      "activations/layer20_attention_weight_min": -23.791105270385742,
      "activations/layer21_attention_weight_max": 30.06610107421875,
      "activations/layer21_attention_weight_min": -21.567087173461914,
      "activations/layer22_attention_weight_max": 50.99534606933594,
      "activations/layer22_attention_weight_min": -28.31903839111328,
      "activations/layer23_attention_weight_max": 31.387428283691406,
      "activations/layer23_attention_weight_min": -24.17782211303711,
      "activations/layer2_attention_weight_max": 31.351186752319336,
      "activations/layer2_attention_weight_min": -29.851009368896484,
      "activations/layer3_attention_weight_max": 83.79773712158203,
      "activations/layer3_attention_weight_min": -88.185302734375,
      "activations/layer4_attention_weight_max": 121.29743957519531,
      "activations/layer4_attention_weight_min": -123.16151428222656,
      "activations/layer5_attention_weight_max": 76.49639892578125,
      "activations/layer5_attention_weight_min": -72.1364974975586,
      "activations/layer6_attention_weight_max": 55.0817985534668,
      "activations/layer6_attention_weight_min": -54.05012893676758,
      "activations/layer7_attention_weight_max": 83.4808578491211,
      "activations/layer7_attention_weight_min": -85.66732788085938,
      "activations/layer8_attention_weight_max": 38.73738098144531,
      "activations/layer8_attention_weight_min": -38.31629180908203,
      "activations/layer9_attention_weight_max": 33.66305160522461,
      "activations/layer9_attention_weight_min": -34.18608856201172,
      "epoch": 5.69,
      "learning_rate": 0.00011444772727272727,
      "loss": 2.8305,
      "step": 97950
    },
    {
      "activations/layer0_attention_weight_max": 16.66538429260254,
      "activations/layer0_attention_weight_min": -11.62751579284668,
      "activations/layer10_attention_weight_max": 34.805686950683594,
      "activations/layer10_attention_weight_min": -31.459012985229492,
      "activations/layer11_attention_weight_max": 30.1811580657959,
      "activations/layer11_attention_weight_min": -27.48768424987793,
      "activations/layer12_attention_weight_max": 24.052385330200195,
      "activations/layer12_attention_weight_min": -24.988204956054688,
      "activations/layer13_attention_weight_max": 42.042781829833984,
      "activations/layer13_attention_weight_min": -34.58906555175781,
      "activations/layer14_attention_weight_max": 53.93791198730469,
      "activations/layer14_attention_weight_min": -43.63829803466797,
      "activations/layer15_attention_weight_max": 34.811866760253906,
      "activations/layer15_attention_weight_min": -28.032190322875977,
      "activations/layer16_attention_weight_max": 35.21701431274414,
      "activations/layer16_attention_weight_min": -27.34055519104004,
      "activations/layer17_attention_weight_max": 34.27201843261719,
      "activations/layer17_attention_weight_min": -25.410737991333008,
      "activations/layer18_attention_weight_max": 31.401174545288086,
      "activations/layer18_attention_weight_min": -21.14918327331543,
      "activations/layer19_attention_weight_max": 39.816802978515625,
      "activations/layer19_attention_weight_min": -28.526723861694336,
      "activations/layer1_attention_weight_max": 17.641952514648438,
      "activations/layer1_attention_weight_min": -16.03094482421875,
      "activations/layer20_attention_weight_max": 32.04631805419922,
      "activations/layer20_attention_weight_min": -22.639402389526367,
      "activations/layer21_attention_weight_max": 30.957626342773438,
      "activations/layer21_attention_weight_min": -21.256549835205078,
      "activations/layer22_attention_weight_max": 50.23062515258789,
      "activations/layer22_attention_weight_min": -27.316532135009766,
      "activations/layer23_attention_weight_max": 31.711872100830078,
      "activations/layer23_attention_weight_min": -23.51552391052246,
      "activations/layer2_attention_weight_max": 32.60945510864258,
      "activations/layer2_attention_weight_min": -30.962440490722656,
      "activations/layer3_attention_weight_max": 91.48060607910156,
      "activations/layer3_attention_weight_min": -91.31287384033203,
      "activations/layer4_attention_weight_max": 133.82373046875,
      "activations/layer4_attention_weight_min": -121.74755859375,
      "activations/layer5_attention_weight_max": 78.86927032470703,
      "activations/layer5_attention_weight_min": -70.61412048339844,
      "activations/layer6_attention_weight_max": 56.3390998840332,
      "activations/layer6_attention_weight_min": -55.19725799560547,
      "activations/layer7_attention_weight_max": 86.63218688964844,
      "activations/layer7_attention_weight_min": -83.71348571777344,
      "activations/layer8_attention_weight_max": 38.41017532348633,
      "activations/layer8_attention_weight_min": -38.130950927734375,
      "activations/layer9_attention_weight_max": 35.450443267822266,
      "activations/layer9_attention_weight_min": -32.83480453491211,
      "epoch": 5.69,
      "learning_rate": 0.00011442878787878787,
      "loss": 2.8346,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.4539,
      "eval_samples_per_second": 507.934,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.4539,
      "eval_openwebtext_samples_per_second": 507.934,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 1.9472,
      "eval_wikitext_samples_per_second": 234.183,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5379,
      "eval_lambada_samples_per_second": 510.489,
      "step": 98000
    },
    {
      "activations/layer0_attention_weight_max": 15.744807243347168,
      "activations/layer0_attention_weight_min": -13.411731719970703,
      "activations/layer10_attention_weight_max": 41.5177116394043,
      "activations/layer10_attention_weight_min": -33.874908447265625,
      "activations/layer11_attention_weight_max": 41.2403678894043,
      "activations/layer11_attention_weight_min": -31.591636657714844,
      "activations/layer12_attention_weight_max": 36.05416488647461,
      "activations/layer12_attention_weight_min": -28.548667907714844,
      "activations/layer13_attention_weight_max": 60.02183532714844,
      "activations/layer13_attention_weight_min": -37.01422119140625,
      "activations/layer14_attention_weight_max": 66.0147933959961,
      "activations/layer14_attention_weight_min": -44.913673400878906,
      "activations/layer15_attention_weight_max": 58.4431266784668,
      "activations/layer15_attention_weight_min": -35.61683654785156,
      "activations/layer16_attention_weight_max": 40.76530456542969,
      "activations/layer16_attention_weight_min": -26.725683212280273,
      "activations/layer17_attention_weight_max": 38.61653137207031,
      "activations/layer17_attention_weight_min": -24.0670166015625,
      "activations/layer18_attention_weight_max": 38.12223434448242,
      "activations/layer18_attention_weight_min": -20.290653228759766,
      "activations/layer19_attention_weight_max": 45.038997650146484,
      "activations/layer19_attention_weight_min": -29.5219669342041,
      "activations/layer1_attention_weight_max": 16.3679141998291,
      "activations/layer1_attention_weight_min": -16.731887817382812,
      "activations/layer20_attention_weight_max": 37.888427734375,
      "activations/layer20_attention_weight_min": -22.32853889465332,
      "activations/layer21_attention_weight_max": 36.160682678222656,
      "activations/layer21_attention_weight_min": -20.88352394104004,
      "activations/layer22_attention_weight_max": 53.43813705444336,
      "activations/layer22_attention_weight_min": -26.1510066986084,
      "activations/layer23_attention_weight_max": 39.62580108642578,
      "activations/layer23_attention_weight_min": -20.731403350830078,
      "activations/layer2_attention_weight_max": 33.16872787475586,
      "activations/layer2_attention_weight_min": -31.443714141845703,
      "activations/layer3_attention_weight_max": 89.65312194824219,
      "activations/layer3_attention_weight_min": -89.02964782714844,
      "activations/layer4_attention_weight_max": 129.1806640625,
      "activations/layer4_attention_weight_min": -127.2392349243164,
      "activations/layer5_attention_weight_max": 78.77943420410156,
      "activations/layer5_attention_weight_min": -70.66779327392578,
      "activations/layer6_attention_weight_max": 59.862640380859375,
      "activations/layer6_attention_weight_min": -53.23887252807617,
      "activations/layer7_attention_weight_max": 99.64019012451172,
      "activations/layer7_attention_weight_min": -88.15240478515625,
      "activations/layer8_attention_weight_max": 49.61825942993164,
      "activations/layer8_attention_weight_min": -43.946571350097656,
      "activations/layer9_attention_weight_max": 40.46589279174805,
      "activations/layer9_attention_weight_min": -45.566776275634766,
      "epoch": 5.7,
      "learning_rate": 0.00011440984848484847,
      "loss": 2.8354,
      "step": 98050
    },
    {
      "activations/layer0_attention_weight_max": 15.404911041259766,
      "activations/layer0_attention_weight_min": -13.011666297912598,
      "activations/layer10_attention_weight_max": 29.780540466308594,
      "activations/layer10_attention_weight_min": -29.445993423461914,
      "activations/layer11_attention_weight_max": 31.04506492614746,
      "activations/layer11_attention_weight_min": -28.80614471435547,
      "activations/layer12_attention_weight_max": 37.847900390625,
      "activations/layer12_attention_weight_min": -25.77373695373535,
      "activations/layer13_attention_weight_max": 45.02436065673828,
      "activations/layer13_attention_weight_min": -34.98786544799805,
      "activations/layer14_attention_weight_max": 57.895416259765625,
      "activations/layer14_attention_weight_min": -45.27952575683594,
      "activations/layer15_attention_weight_max": 40.831417083740234,
      "activations/layer15_attention_weight_min": -28.439823150634766,
      "activations/layer16_attention_weight_max": 37.09564208984375,
      "activations/layer16_attention_weight_min": -28.14911460876465,
      "activations/layer17_attention_weight_max": 38.32959747314453,
      "activations/layer17_attention_weight_min": -24.87789535522461,
      "activations/layer18_attention_weight_max": 29.448331832885742,
      "activations/layer18_attention_weight_min": -19.41366958618164,
      "activations/layer19_attention_weight_max": 37.67280960083008,
      "activations/layer19_attention_weight_min": -30.59210777282715,
      "activations/layer1_attention_weight_max": 16.355514526367188,
      "activations/layer1_attention_weight_min": -17.40131378173828,
      "activations/layer20_attention_weight_max": 33.30213165283203,
      "activations/layer20_attention_weight_min": -24.13762664794922,
      "activations/layer21_attention_weight_max": 30.439340591430664,
      "activations/layer21_attention_weight_min": -23.24231719970703,
      "activations/layer22_attention_weight_max": 46.90424346923828,
      "activations/layer22_attention_weight_min": -27.39127540588379,
      "activations/layer23_attention_weight_max": 32.91249465942383,
      "activations/layer23_attention_weight_min": -19.95901107788086,
      "activations/layer2_attention_weight_max": 32.10878372192383,
      "activations/layer2_attention_weight_min": -32.25581359863281,
      "activations/layer3_attention_weight_max": 89.42420959472656,
      "activations/layer3_attention_weight_min": -92.64875793457031,
      "activations/layer4_attention_weight_max": 121.8687515258789,
      "activations/layer4_attention_weight_min": -124.28499603271484,
      "activations/layer5_attention_weight_max": 71.02993774414062,
      "activations/layer5_attention_weight_min": -71.75415802001953,
      "activations/layer6_attention_weight_max": 51.16460037231445,
      "activations/layer6_attention_weight_min": -50.84132766723633,
      "activations/layer7_attention_weight_max": 79.2986831665039,
      "activations/layer7_attention_weight_min": -87.07180786132812,
      "activations/layer8_attention_weight_max": 36.06976318359375,
      "activations/layer8_attention_weight_min": -36.37282180786133,
      "activations/layer9_attention_weight_max": 31.02390480041504,
      "activations/layer9_attention_weight_min": -34.28641891479492,
      "epoch": 5.7,
      "learning_rate": 0.00011439090909090909,
      "loss": 2.8246,
      "step": 98100
    },
    {
      "activations/layer0_attention_weight_max": 14.736661911010742,
      "activations/layer0_attention_weight_min": -12.893821716308594,
      "activations/layer10_attention_weight_max": 37.039859771728516,
      "activations/layer10_attention_weight_min": -31.833621978759766,
      "activations/layer11_attention_weight_max": 35.139564514160156,
      "activations/layer11_attention_weight_min": -27.91714859008789,
      "activations/layer12_attention_weight_max": 28.25101089477539,
      "activations/layer12_attention_weight_min": -30.43937873840332,
      "activations/layer13_attention_weight_max": 50.02412033081055,
      "activations/layer13_attention_weight_min": -31.276697158813477,
      "activations/layer14_attention_weight_max": 63.52873229980469,
      "activations/layer14_attention_weight_min": -46.813682556152344,
      "activations/layer15_attention_weight_max": 40.53218078613281,
      "activations/layer15_attention_weight_min": -28.482839584350586,
      "activations/layer16_attention_weight_max": 40.07667541503906,
      "activations/layer16_attention_weight_min": -30.553455352783203,
      "activations/layer17_attention_weight_max": 43.00184631347656,
      "activations/layer17_attention_weight_min": -24.349285125732422,
      "activations/layer18_attention_weight_max": 31.592924118041992,
      "activations/layer18_attention_weight_min": -22.235265731811523,
      "activations/layer19_attention_weight_max": 43.83639144897461,
      "activations/layer19_attention_weight_min": -31.553876876831055,
      "activations/layer1_attention_weight_max": 16.43869972229004,
      "activations/layer1_attention_weight_min": -14.37358570098877,
      "activations/layer20_attention_weight_max": 38.305267333984375,
      "activations/layer20_attention_weight_min": -23.954565048217773,
      "activations/layer21_attention_weight_max": 32.38821792602539,
      "activations/layer21_attention_weight_min": -19.963390350341797,
      "activations/layer22_attention_weight_max": 56.10337448120117,
      "activations/layer22_attention_weight_min": -25.973249435424805,
      "activations/layer23_attention_weight_max": 37.73249816894531,
      "activations/layer23_attention_weight_min": -19.05707550048828,
      "activations/layer2_attention_weight_max": 33.15630340576172,
      "activations/layer2_attention_weight_min": -32.396507263183594,
      "activations/layer3_attention_weight_max": 98.10417938232422,
      "activations/layer3_attention_weight_min": -92.08348846435547,
      "activations/layer4_attention_weight_max": 134.75376892089844,
      "activations/layer4_attention_weight_min": -129.41693115234375,
      "activations/layer5_attention_weight_max": 82.1878890991211,
      "activations/layer5_attention_weight_min": -75.88565063476562,
      "activations/layer6_attention_weight_max": 56.112613677978516,
      "activations/layer6_attention_weight_min": -54.02946090698242,
      "activations/layer7_attention_weight_max": 89.92072296142578,
      "activations/layer7_attention_weight_min": -83.41941833496094,
      "activations/layer8_attention_weight_max": 39.82183074951172,
      "activations/layer8_attention_weight_min": -38.516422271728516,
      "activations/layer9_attention_weight_max": 35.34547424316406,
      "activations/layer9_attention_weight_min": -35.257598876953125,
      "epoch": 5.7,
      "learning_rate": 0.0001143719696969697,
      "loss": 2.8369,
      "step": 98150
    },
    {
      "activations/layer0_attention_weight_max": 15.335728645324707,
      "activations/layer0_attention_weight_min": -11.85416030883789,
      "activations/layer10_attention_weight_max": 36.433624267578125,
      "activations/layer10_attention_weight_min": -31.53635597229004,
      "activations/layer11_attention_weight_max": 34.10943603515625,
      "activations/layer11_attention_weight_min": -29.52998161315918,
      "activations/layer12_attention_weight_max": 24.499616622924805,
      "activations/layer12_attention_weight_min": -25.027807235717773,
      "activations/layer13_attention_weight_max": 40.683631896972656,
      "activations/layer13_attention_weight_min": -32.89381408691406,
      "activations/layer14_attention_weight_max": 54.20806884765625,
      "activations/layer14_attention_weight_min": -43.87356948852539,
      "activations/layer15_attention_weight_max": 38.79658126831055,
      "activations/layer15_attention_weight_min": -28.037443161010742,
      "activations/layer16_attention_weight_max": 34.21703338623047,
      "activations/layer16_attention_weight_min": -26.837299346923828,
      "activations/layer17_attention_weight_max": 33.8437385559082,
      "activations/layer17_attention_weight_min": -25.114816665649414,
      "activations/layer18_attention_weight_max": 30.602426528930664,
      "activations/layer18_attention_weight_min": -20.1240177154541,
      "activations/layer19_attention_weight_max": 39.07853698730469,
      "activations/layer19_attention_weight_min": -31.241851806640625,
      "activations/layer1_attention_weight_max": 17.76580047607422,
      "activations/layer1_attention_weight_min": -15.540189743041992,
      "activations/layer20_attention_weight_max": 31.5765438079834,
      "activations/layer20_attention_weight_min": -24.026472091674805,
      "activations/layer21_attention_weight_max": 33.677303314208984,
      "activations/layer21_attention_weight_min": -21.607975006103516,
      "activations/layer22_attention_weight_max": 50.73495864868164,
      "activations/layer22_attention_weight_min": -30.508930206298828,
      "activations/layer23_attention_weight_max": 35.417518615722656,
      "activations/layer23_attention_weight_min": -20.53766441345215,
      "activations/layer2_attention_weight_max": 31.454803466796875,
      "activations/layer2_attention_weight_min": -30.48699951171875,
      "activations/layer3_attention_weight_max": 91.89157104492188,
      "activations/layer3_attention_weight_min": -90.15271759033203,
      "activations/layer4_attention_weight_max": 131.71481323242188,
      "activations/layer4_attention_weight_min": -123.41279602050781,
      "activations/layer5_attention_weight_max": 78.96784973144531,
      "activations/layer5_attention_weight_min": -74.04532623291016,
      "activations/layer6_attention_weight_max": 56.97061538696289,
      "activations/layer6_attention_weight_min": -56.50188064575195,
      "activations/layer7_attention_weight_max": 88.35384368896484,
      "activations/layer7_attention_weight_min": -84.50116729736328,
      "activations/layer8_attention_weight_max": 42.7650032043457,
      "activations/layer8_attention_weight_min": -41.0283317565918,
      "activations/layer9_attention_weight_max": 41.26539993286133,
      "activations/layer9_attention_weight_min": -35.087650299072266,
      "epoch": 5.71,
      "learning_rate": 0.00011435303030303029,
      "loss": 2.842,
      "step": 98200
    },
    {
      "activations/layer0_attention_weight_max": 16.849729537963867,
      "activations/layer0_attention_weight_min": -11.997550010681152,
      "activations/layer10_attention_weight_max": 30.957061767578125,
      "activations/layer10_attention_weight_min": -30.019750595092773,
      "activations/layer11_attention_weight_max": 29.086788177490234,
      "activations/layer11_attention_weight_min": -28.015918731689453,
      "activations/layer12_attention_weight_max": 35.14288330078125,
      "activations/layer12_attention_weight_min": -28.6157169342041,
      "activations/layer13_attention_weight_max": 38.853919982910156,
      "activations/layer13_attention_weight_min": -31.761674880981445,
      "activations/layer14_attention_weight_max": 63.4000358581543,
      "activations/layer14_attention_weight_min": -50.639896392822266,
      "activations/layer15_attention_weight_max": 37.465492248535156,
      "activations/layer15_attention_weight_min": -28.28879737854004,
      "activations/layer16_attention_weight_max": 33.91518783569336,
      "activations/layer16_attention_weight_min": -26.856975555419922,
      "activations/layer17_attention_weight_max": 37.023197174072266,
      "activations/layer17_attention_weight_min": -23.756641387939453,
      "activations/layer18_attention_weight_max": 33.431419372558594,
      "activations/layer18_attention_weight_min": -20.465272903442383,
      "activations/layer19_attention_weight_max": 36.94430923461914,
      "activations/layer19_attention_weight_min": -27.614484786987305,
      "activations/layer1_attention_weight_max": 16.643245697021484,
      "activations/layer1_attention_weight_min": -14.32748794555664,
      "activations/layer20_attention_weight_max": 30.522138595581055,
      "activations/layer20_attention_weight_min": -22.868942260742188,
      "activations/layer21_attention_weight_max": 32.37726974487305,
      "activations/layer21_attention_weight_min": -20.707067489624023,
      "activations/layer22_attention_weight_max": 49.17774963378906,
      "activations/layer22_attention_weight_min": -26.955978393554688,
      "activations/layer23_attention_weight_max": 33.195152282714844,
      "activations/layer23_attention_weight_min": -22.079809188842773,
      "activations/layer2_attention_weight_max": 30.754375457763672,
      "activations/layer2_attention_weight_min": -30.060211181640625,
      "activations/layer3_attention_weight_max": 87.73020935058594,
      "activations/layer3_attention_weight_min": -88.92193603515625,
      "activations/layer4_attention_weight_max": 124.99022674560547,
      "activations/layer4_attention_weight_min": -123.448974609375,
      "activations/layer5_attention_weight_max": 73.82612609863281,
      "activations/layer5_attention_weight_min": -68.9052734375,
      "activations/layer6_attention_weight_max": 54.66410446166992,
      "activations/layer6_attention_weight_min": -53.03764343261719,
      "activations/layer7_attention_weight_max": 82.32389831542969,
      "activations/layer7_attention_weight_min": -87.67103576660156,
      "activations/layer8_attention_weight_max": 37.85399627685547,
      "activations/layer8_attention_weight_min": -38.36334991455078,
      "activations/layer9_attention_weight_max": 33.309715270996094,
      "activations/layer9_attention_weight_min": -33.02088165283203,
      "epoch": 5.71,
      "learning_rate": 0.0001143340909090909,
      "loss": 2.8321,
      "step": 98250
    },
    {
      "activations/layer0_attention_weight_max": 16.252729415893555,
      "activations/layer0_attention_weight_min": -13.083250045776367,
      "activations/layer10_attention_weight_max": 32.84867858886719,
      "activations/layer10_attention_weight_min": -30.612058639526367,
      "activations/layer11_attention_weight_max": 32.24966812133789,
      "activations/layer11_attention_weight_min": -29.179258346557617,
      "activations/layer12_attention_weight_max": 23.48257064819336,
      "activations/layer12_attention_weight_min": -26.388717651367188,
      "activations/layer13_attention_weight_max": 44.416343688964844,
      "activations/layer13_attention_weight_min": -32.92559814453125,
      "activations/layer14_attention_weight_max": 59.19076919555664,
      "activations/layer14_attention_weight_min": -44.40435028076172,
      "activations/layer15_attention_weight_max": 37.88207244873047,
      "activations/layer15_attention_weight_min": -28.37246322631836,
      "activations/layer16_attention_weight_max": 34.84074783325195,
      "activations/layer16_attention_weight_min": -27.916072845458984,
      "activations/layer17_attention_weight_max": 39.372535705566406,
      "activations/layer17_attention_weight_min": -25.483346939086914,
      "activations/layer18_attention_weight_max": 37.068992614746094,
      "activations/layer18_attention_weight_min": -20.152095794677734,
      "activations/layer19_attention_weight_max": 46.15885925292969,
      "activations/layer19_attention_weight_min": -29.027204513549805,
      "activations/layer1_attention_weight_max": 17.57926368713379,
      "activations/layer1_attention_weight_min": -17.122299194335938,
      "activations/layer20_attention_weight_max": 34.804931640625,
      "activations/layer20_attention_weight_min": -22.702720642089844,
      "activations/layer21_attention_weight_max": 33.96033477783203,
      "activations/layer21_attention_weight_min": -21.790008544921875,
      "activations/layer22_attention_weight_max": 53.72724914550781,
      "activations/layer22_attention_weight_min": -29.71083641052246,
      "activations/layer23_attention_weight_max": 38.198570251464844,
      "activations/layer23_attention_weight_min": -21.544025421142578,
      "activations/layer2_attention_weight_max": 32.3204345703125,
      "activations/layer2_attention_weight_min": -32.767333984375,
      "activations/layer3_attention_weight_max": 90.47383117675781,
      "activations/layer3_attention_weight_min": -95.94132232666016,
      "activations/layer4_attention_weight_max": 125.77033996582031,
      "activations/layer4_attention_weight_min": -128.7845916748047,
      "activations/layer5_attention_weight_max": 74.17301940917969,
      "activations/layer5_attention_weight_min": -75.64830017089844,
      "activations/layer6_attention_weight_max": 53.91362380981445,
      "activations/layer6_attention_weight_min": -61.15314483642578,
      "activations/layer7_attention_weight_max": 85.30371856689453,
      "activations/layer7_attention_weight_min": -89.64675903320312,
      "activations/layer8_attention_weight_max": 39.13338088989258,
      "activations/layer8_attention_weight_min": -41.18669509887695,
      "activations/layer9_attention_weight_max": 36.00126266479492,
      "activations/layer9_attention_weight_min": -34.920536041259766,
      "epoch": 5.71,
      "learning_rate": 0.0001143151515151515,
      "loss": 2.8315,
      "step": 98300
    },
    {
      "activations/layer0_attention_weight_max": 16.961429595947266,
      "activations/layer0_attention_weight_min": -11.751812934875488,
      "activations/layer10_attention_weight_max": 34.75480651855469,
      "activations/layer10_attention_weight_min": -30.581851959228516,
      "activations/layer11_attention_weight_max": 32.55680847167969,
      "activations/layer11_attention_weight_min": -28.044065475463867,
      "activations/layer12_attention_weight_max": 21.904014587402344,
      "activations/layer12_attention_weight_min": -29.541902542114258,
      "activations/layer13_attention_weight_max": 40.39325714111328,
      "activations/layer13_attention_weight_min": -31.37645149230957,
      "activations/layer14_attention_weight_max": 55.308860778808594,
      "activations/layer14_attention_weight_min": -43.46163558959961,
      "activations/layer15_attention_weight_max": 39.55690002441406,
      "activations/layer15_attention_weight_min": -28.066431045532227,
      "activations/layer16_attention_weight_max": 33.917335510253906,
      "activations/layer16_attention_weight_min": -26.979116439819336,
      "activations/layer17_attention_weight_max": 34.493064880371094,
      "activations/layer17_attention_weight_min": -23.493202209472656,
      "activations/layer18_attention_weight_max": 39.62030029296875,
      "activations/layer18_attention_weight_min": -20.759057998657227,
      "activations/layer19_attention_weight_max": 38.37013244628906,
      "activations/layer19_attention_weight_min": -28.42333984375,
      "activations/layer1_attention_weight_max": 17.48784828186035,
      "activations/layer1_attention_weight_min": -16.003225326538086,
      "activations/layer20_attention_weight_max": 31.763504028320312,
      "activations/layer20_attention_weight_min": -23.16939926147461,
      "activations/layer21_attention_weight_max": 33.95315933227539,
      "activations/layer21_attention_weight_min": -21.308923721313477,
      "activations/layer22_attention_weight_max": 48.44696807861328,
      "activations/layer22_attention_weight_min": -26.1923828125,
      "activations/layer23_attention_weight_max": 33.38526916503906,
      "activations/layer23_attention_weight_min": -21.735820770263672,
      "activations/layer2_attention_weight_max": 32.13383483886719,
      "activations/layer2_attention_weight_min": -29.846654891967773,
      "activations/layer3_attention_weight_max": 91.93439483642578,
      "activations/layer3_attention_weight_min": -91.4864730834961,
      "activations/layer4_attention_weight_max": 127.3120346069336,
      "activations/layer4_attention_weight_min": -124.1774673461914,
      "activations/layer5_attention_weight_max": 75.84175872802734,
      "activations/layer5_attention_weight_min": -71.89649963378906,
      "activations/layer6_attention_weight_max": 54.71846008300781,
      "activations/layer6_attention_weight_min": -53.20365905761719,
      "activations/layer7_attention_weight_max": 81.02105712890625,
      "activations/layer7_attention_weight_min": -82.06367492675781,
      "activations/layer8_attention_weight_max": 39.84512710571289,
      "activations/layer8_attention_weight_min": -40.32529830932617,
      "activations/layer9_attention_weight_max": 37.33430099487305,
      "activations/layer9_attention_weight_min": -33.58380889892578,
      "epoch": 5.71,
      "learning_rate": 0.0001142962121212121,
      "loss": 2.8442,
      "step": 98350
    },
    {
      "activations/layer0_attention_weight_max": 16.27347755432129,
      "activations/layer0_attention_weight_min": -12.498785972595215,
      "activations/layer10_attention_weight_max": 37.911800384521484,
      "activations/layer10_attention_weight_min": -34.62440872192383,
      "activations/layer11_attention_weight_max": 36.45044708251953,
      "activations/layer11_attention_weight_min": -29.86172103881836,
      "activations/layer12_attention_weight_max": 29.42298698425293,
      "activations/layer12_attention_weight_min": -27.471250534057617,
      "activations/layer13_attention_weight_max": 40.17942810058594,
      "activations/layer13_attention_weight_min": -33.00217056274414,
      "activations/layer14_attention_weight_max": 62.41455078125,
      "activations/layer14_attention_weight_min": -44.850975036621094,
      "activations/layer15_attention_weight_max": 38.54743576049805,
      "activations/layer15_attention_weight_min": -30.34937858581543,
      "activations/layer16_attention_weight_max": 35.76190948486328,
      "activations/layer16_attention_weight_min": -29.529428482055664,
      "activations/layer17_attention_weight_max": 37.26127624511719,
      "activations/layer17_attention_weight_min": -23.06916046142578,
      "activations/layer18_attention_weight_max": 37.19512176513672,
      "activations/layer18_attention_weight_min": -20.046010971069336,
      "activations/layer19_attention_weight_max": 44.7938117980957,
      "activations/layer19_attention_weight_min": -29.253437042236328,
      "activations/layer1_attention_weight_max": 16.428516387939453,
      "activations/layer1_attention_weight_min": -15.414894104003906,
      "activations/layer20_attention_weight_max": 36.05457305908203,
      "activations/layer20_attention_weight_min": -22.130830764770508,
      "activations/layer21_attention_weight_max": 35.92737579345703,
      "activations/layer21_attention_weight_min": -20.751480102539062,
      "activations/layer22_attention_weight_max": 51.89478302001953,
      "activations/layer22_attention_weight_min": -27.26153564453125,
      "activations/layer23_attention_weight_max": 37.02409744262695,
      "activations/layer23_attention_weight_min": -19.78718376159668,
      "activations/layer2_attention_weight_max": 33.22882080078125,
      "activations/layer2_attention_weight_min": -32.33784866333008,
      "activations/layer3_attention_weight_max": 94.13206481933594,
      "activations/layer3_attention_weight_min": -96.627197265625,
      "activations/layer4_attention_weight_max": 126.98332977294922,
      "activations/layer4_attention_weight_min": -125.70433807373047,
      "activations/layer5_attention_weight_max": 75.69957733154297,
      "activations/layer5_attention_weight_min": -72.53269958496094,
      "activations/layer6_attention_weight_max": 53.597496032714844,
      "activations/layer6_attention_weight_min": -55.67817687988281,
      "activations/layer7_attention_weight_max": 88.52535247802734,
      "activations/layer7_attention_weight_min": -82.75459289550781,
      "activations/layer8_attention_weight_max": 41.272335052490234,
      "activations/layer8_attention_weight_min": -39.83205032348633,
      "activations/layer9_attention_weight_max": 38.58197784423828,
      "activations/layer9_attention_weight_min": -35.37440872192383,
      "epoch": 5.72,
      "learning_rate": 0.00011427727272727272,
      "loss": 2.8487,
      "step": 98400
    },
    {
      "activations/layer0_attention_weight_max": 16.610061645507812,
      "activations/layer0_attention_weight_min": -12.758055686950684,
      "activations/layer10_attention_weight_max": 31.823781967163086,
      "activations/layer10_attention_weight_min": -31.501230239868164,
      "activations/layer11_attention_weight_max": 34.033416748046875,
      "activations/layer11_attention_weight_min": -28.835281372070312,
      "activations/layer12_attention_weight_max": 29.36286163330078,
      "activations/layer12_attention_weight_min": -26.67795753479004,
      "activations/layer13_attention_weight_max": 44.11721420288086,
      "activations/layer13_attention_weight_min": -29.946697235107422,
      "activations/layer14_attention_weight_max": 58.32363510131836,
      "activations/layer14_attention_weight_min": -40.362117767333984,
      "activations/layer15_attention_weight_max": 44.287628173828125,
      "activations/layer15_attention_weight_min": -31.083852767944336,
      "activations/layer16_attention_weight_max": 33.56864929199219,
      "activations/layer16_attention_weight_min": -28.235464096069336,
      "activations/layer17_attention_weight_max": 35.68533706665039,
      "activations/layer17_attention_weight_min": -23.594064712524414,
      "activations/layer18_attention_weight_max": 30.188594818115234,
      "activations/layer18_attention_weight_min": -19.822154998779297,
      "activations/layer19_attention_weight_max": 37.7919807434082,
      "activations/layer19_attention_weight_min": -28.152292251586914,
      "activations/layer1_attention_weight_max": 17.20599937438965,
      "activations/layer1_attention_weight_min": -14.787847518920898,
      "activations/layer20_attention_weight_max": 32.52887725830078,
      "activations/layer20_attention_weight_min": -22.233659744262695,
      "activations/layer21_attention_weight_max": 34.670406341552734,
      "activations/layer21_attention_weight_min": -19.597105026245117,
      "activations/layer22_attention_weight_max": 48.00469970703125,
      "activations/layer22_attention_weight_min": -27.03703498840332,
      "activations/layer23_attention_weight_max": 32.58350372314453,
      "activations/layer23_attention_weight_min": -23.59784698486328,
      "activations/layer2_attention_weight_max": 34.0053596496582,
      "activations/layer2_attention_weight_min": -33.90221405029297,
      "activations/layer3_attention_weight_max": 93.33201599121094,
      "activations/layer3_attention_weight_min": -92.10932159423828,
      "activations/layer4_attention_weight_max": 124.05726623535156,
      "activations/layer4_attention_weight_min": -126.48613739013672,
      "activations/layer5_attention_weight_max": 73.75260925292969,
      "activations/layer5_attention_weight_min": -73.21441650390625,
      "activations/layer6_attention_weight_max": 52.62923812866211,
      "activations/layer6_attention_weight_min": -56.11078643798828,
      "activations/layer7_attention_weight_max": 84.8976058959961,
      "activations/layer7_attention_weight_min": -85.41593170166016,
      "activations/layer8_attention_weight_max": 40.28794479370117,
      "activations/layer8_attention_weight_min": -41.856204986572266,
      "activations/layer9_attention_weight_max": 36.66788864135742,
      "activations/layer9_attention_weight_min": -34.22188949584961,
      "epoch": 5.72,
      "learning_rate": 0.00011425833333333332,
      "loss": 2.8383,
      "step": 98450
    },
    {
      "activations/layer0_attention_weight_max": 16.37329864501953,
      "activations/layer0_attention_weight_min": -10.581153869628906,
      "activations/layer10_attention_weight_max": 31.562150955200195,
      "activations/layer10_attention_weight_min": -31.31709861755371,
      "activations/layer11_attention_weight_max": 31.232664108276367,
      "activations/layer11_attention_weight_min": -29.13460922241211,
      "activations/layer12_attention_weight_max": 23.055038452148438,
      "activations/layer12_attention_weight_min": -26.44631576538086,
      "activations/layer13_attention_weight_max": 40.26629638671875,
      "activations/layer13_attention_weight_min": -32.99506759643555,
      "activations/layer14_attention_weight_max": 50.73853302001953,
      "activations/layer14_attention_weight_min": -39.020755767822266,
      "activations/layer15_attention_weight_max": 45.41099548339844,
      "activations/layer15_attention_weight_min": -29.709123611450195,
      "activations/layer16_attention_weight_max": 35.80207061767578,
      "activations/layer16_attention_weight_min": -27.95351791381836,
      "activations/layer17_attention_weight_max": 40.9064826965332,
      "activations/layer17_attention_weight_min": -24.81687355041504,
      "activations/layer18_attention_weight_max": 29.690387725830078,
      "activations/layer18_attention_weight_min": -19.23652458190918,
      "activations/layer19_attention_weight_max": 38.609676361083984,
      "activations/layer19_attention_weight_min": -28.53810691833496,
      "activations/layer1_attention_weight_max": 17.434669494628906,
      "activations/layer1_attention_weight_min": -14.770451545715332,
      "activations/layer20_attention_weight_max": 35.03965377807617,
      "activations/layer20_attention_weight_min": -23.05052947998047,
      "activations/layer21_attention_weight_max": 35.048213958740234,
      "activations/layer21_attention_weight_min": -21.51473045349121,
      "activations/layer22_attention_weight_max": 47.66910171508789,
      "activations/layer22_attention_weight_min": -28.586341857910156,
      "activations/layer23_attention_weight_max": 33.356361389160156,
      "activations/layer23_attention_weight_min": -19.78732681274414,
      "activations/layer2_attention_weight_max": 32.779415130615234,
      "activations/layer2_attention_weight_min": -31.854711532592773,
      "activations/layer3_attention_weight_max": 89.8333740234375,
      "activations/layer3_attention_weight_min": -92.65977478027344,
      "activations/layer4_attention_weight_max": 122.074951171875,
      "activations/layer4_attention_weight_min": -121.91841888427734,
      "activations/layer5_attention_weight_max": 72.70130920410156,
      "activations/layer5_attention_weight_min": -70.96977996826172,
      "activations/layer6_attention_weight_max": 59.03845977783203,
      "activations/layer6_attention_weight_min": -53.21652603149414,
      "activations/layer7_attention_weight_max": 87.55653381347656,
      "activations/layer7_attention_weight_min": -82.05233764648438,
      "activations/layer8_attention_weight_max": 40.17288589477539,
      "activations/layer8_attention_weight_min": -41.85976791381836,
      "activations/layer9_attention_weight_max": 35.278438568115234,
      "activations/layer9_attention_weight_min": -35.466915130615234,
      "epoch": 5.72,
      "learning_rate": 0.00011423939393939392,
      "loss": 2.829,
      "step": 98500
    },
    {
      "activations/layer0_attention_weight_max": 15.592639923095703,
      "activations/layer0_attention_weight_min": -12.652240753173828,
      "activations/layer10_attention_weight_max": 32.48953628540039,
      "activations/layer10_attention_weight_min": -32.828834533691406,
      "activations/layer11_attention_weight_max": 30.445945739746094,
      "activations/layer11_attention_weight_min": -29.628177642822266,
      "activations/layer12_attention_weight_max": 35.96165466308594,
      "activations/layer12_attention_weight_min": -34.052093505859375,
      "activations/layer13_attention_weight_max": 40.03401565551758,
      "activations/layer13_attention_weight_min": -31.138185501098633,
      "activations/layer14_attention_weight_max": 64.41043853759766,
      "activations/layer14_attention_weight_min": -45.75288009643555,
      "activations/layer15_attention_weight_max": 37.16506576538086,
      "activations/layer15_attention_weight_min": -28.092914581298828,
      "activations/layer16_attention_weight_max": 35.42426300048828,
      "activations/layer16_attention_weight_min": -29.13987922668457,
      "activations/layer17_attention_weight_max": 36.784812927246094,
      "activations/layer17_attention_weight_min": -24.921005249023438,
      "activations/layer18_attention_weight_max": 34.50934600830078,
      "activations/layer18_attention_weight_min": -20.333410263061523,
      "activations/layer19_attention_weight_max": 38.91749572753906,
      "activations/layer19_attention_weight_min": -27.62979507446289,
      "activations/layer1_attention_weight_max": 18.083293914794922,
      "activations/layer1_attention_weight_min": -15.063456535339355,
      "activations/layer20_attention_weight_max": 33.73787307739258,
      "activations/layer20_attention_weight_min": -24.00695037841797,
      "activations/layer21_attention_weight_max": 44.09120559692383,
      "activations/layer21_attention_weight_min": -21.252304077148438,
      "activations/layer22_attention_weight_max": 55.39163589477539,
      "activations/layer22_attention_weight_min": -28.01972007751465,
      "activations/layer23_attention_weight_max": 34.35745620727539,
      "activations/layer23_attention_weight_min": -21.607112884521484,
      "activations/layer2_attention_weight_max": 32.03507614135742,
      "activations/layer2_attention_weight_min": -32.31589889526367,
      "activations/layer3_attention_weight_max": 90.8436279296875,
      "activations/layer3_attention_weight_min": -94.42120361328125,
      "activations/layer4_attention_weight_max": 127.52845764160156,
      "activations/layer4_attention_weight_min": -128.27926635742188,
      "activations/layer5_attention_weight_max": 75.82984161376953,
      "activations/layer5_attention_weight_min": -76.10581970214844,
      "activations/layer6_attention_weight_max": 54.25564956665039,
      "activations/layer6_attention_weight_min": -54.798851013183594,
      "activations/layer7_attention_weight_max": 81.60527801513672,
      "activations/layer7_attention_weight_min": -84.3002700805664,
      "activations/layer8_attention_weight_max": 42.30092239379883,
      "activations/layer8_attention_weight_min": -39.58534622192383,
      "activations/layer9_attention_weight_max": 37.4385986328125,
      "activations/layer9_attention_weight_min": -34.09995651245117,
      "epoch": 5.73,
      "learning_rate": 0.00011422045454545454,
      "loss": 2.8513,
      "step": 98550
    },
    {
      "activations/layer0_attention_weight_max": 16.47027587890625,
      "activations/layer0_attention_weight_min": -11.691533088684082,
      "activations/layer10_attention_weight_max": 37.23042297363281,
      "activations/layer10_attention_weight_min": -33.1025390625,
      "activations/layer11_attention_weight_max": 34.696075439453125,
      "activations/layer11_attention_weight_min": -28.073604583740234,
      "activations/layer12_attention_weight_max": 35.867740631103516,
      "activations/layer12_attention_weight_min": -29.300302505493164,
      "activations/layer13_attention_weight_max": 45.33892059326172,
      "activations/layer13_attention_weight_min": -32.064422607421875,
      "activations/layer14_attention_weight_max": 74.1317367553711,
      "activations/layer14_attention_weight_min": -46.42484664916992,
      "activations/layer15_attention_weight_max": 42.7061767578125,
      "activations/layer15_attention_weight_min": -29.153173446655273,
      "activations/layer16_attention_weight_max": 44.40266036987305,
      "activations/layer16_attention_weight_min": -27.02455711364746,
      "activations/layer17_attention_weight_max": 40.93768310546875,
      "activations/layer17_attention_weight_min": -24.957983016967773,
      "activations/layer18_attention_weight_max": 36.83005142211914,
      "activations/layer18_attention_weight_min": -20.355854034423828,
      "activations/layer19_attention_weight_max": 49.939205169677734,
      "activations/layer19_attention_weight_min": -28.731964111328125,
      "activations/layer1_attention_weight_max": 17.142667770385742,
      "activations/layer1_attention_weight_min": -14.616540908813477,
      "activations/layer20_attention_weight_max": 37.24647903442383,
      "activations/layer20_attention_weight_min": -24.049436569213867,
      "activations/layer21_attention_weight_max": 35.21967315673828,
      "activations/layer21_attention_weight_min": -20.189231872558594,
      "activations/layer22_attention_weight_max": 63.40680694580078,
      "activations/layer22_attention_weight_min": -27.40616798400879,
      "activations/layer23_attention_weight_max": 38.74342346191406,
      "activations/layer23_attention_weight_min": -20.794612884521484,
      "activations/layer2_attention_weight_max": 33.14320373535156,
      "activations/layer2_attention_weight_min": -33.24896240234375,
      "activations/layer3_attention_weight_max": 98.71408081054688,
      "activations/layer3_attention_weight_min": -97.94446563720703,
      "activations/layer4_attention_weight_max": 137.3457489013672,
      "activations/layer4_attention_weight_min": -134.61134338378906,
      "activations/layer5_attention_weight_max": 78.11856842041016,
      "activations/layer5_attention_weight_min": -80.53995513916016,
      "activations/layer6_attention_weight_max": 55.56946563720703,
      "activations/layer6_attention_weight_min": -62.00962829589844,
      "activations/layer7_attention_weight_max": 91.52940368652344,
      "activations/layer7_attention_weight_min": -93.71172332763672,
      "activations/layer8_attention_weight_max": 43.60029983520508,
      "activations/layer8_attention_weight_min": -46.95553207397461,
      "activations/layer9_attention_weight_max": 38.96754837036133,
      "activations/layer9_attention_weight_min": -38.01959228515625,
      "epoch": 5.73,
      "learning_rate": 0.00011420151515151514,
      "loss": 2.8279,
      "step": 98600
    },
    {
      "activations/layer0_attention_weight_max": 16.43794822692871,
      "activations/layer0_attention_weight_min": -11.570122718811035,
      "activations/layer10_attention_weight_max": 35.761653900146484,
      "activations/layer10_attention_weight_min": -36.70235061645508,
      "activations/layer11_attention_weight_max": 35.70245361328125,
      "activations/layer11_attention_weight_min": -34.45402526855469,
      "activations/layer12_attention_weight_max": 43.827083587646484,
      "activations/layer12_attention_weight_min": -27.29266929626465,
      "activations/layer13_attention_weight_max": 68.29145812988281,
      "activations/layer13_attention_weight_min": -44.589717864990234,
      "activations/layer14_attention_weight_max": 63.15306091308594,
      "activations/layer14_attention_weight_min": -48.19302749633789,
      "activations/layer15_attention_weight_max": 61.725486755371094,
      "activations/layer15_attention_weight_min": -40.99686050415039,
      "activations/layer16_attention_weight_max": 37.22809600830078,
      "activations/layer16_attention_weight_min": -26.11976432800293,
      "activations/layer17_attention_weight_max": 36.63222122192383,
      "activations/layer17_attention_weight_min": -23.83832359313965,
      "activations/layer18_attention_weight_max": 32.35440444946289,
      "activations/layer18_attention_weight_min": -20.468578338623047,
      "activations/layer19_attention_weight_max": 39.9959831237793,
      "activations/layer19_attention_weight_min": -32.525428771972656,
      "activations/layer1_attention_weight_max": 18.511417388916016,
      "activations/layer1_attention_weight_min": -17.272186279296875,
      "activations/layer20_attention_weight_max": 29.076936721801758,
      "activations/layer20_attention_weight_min": -22.566957473754883,
      "activations/layer21_attention_weight_max": 30.21236801147461,
      "activations/layer21_attention_weight_min": -20.919755935668945,
      "activations/layer22_attention_weight_max": 51.634315490722656,
      "activations/layer22_attention_weight_min": -32.38594055175781,
      "activations/layer23_attention_weight_max": 35.127532958984375,
      "activations/layer23_attention_weight_min": -21.78665542602539,
      "activations/layer2_attention_weight_max": 33.359710693359375,
      "activations/layer2_attention_weight_min": -32.34661102294922,
      "activations/layer3_attention_weight_max": 91.49759674072266,
      "activations/layer3_attention_weight_min": -92.35420227050781,
      "activations/layer4_attention_weight_max": 127.27081298828125,
      "activations/layer4_attention_weight_min": -124.15673065185547,
      "activations/layer5_attention_weight_max": 76.44160461425781,
      "activations/layer5_attention_weight_min": -73.43607330322266,
      "activations/layer6_attention_weight_max": 55.67315673828125,
      "activations/layer6_attention_weight_min": -57.36125564575195,
      "activations/layer7_attention_weight_max": 86.7864990234375,
      "activations/layer7_attention_weight_min": -94.14356994628906,
      "activations/layer8_attention_weight_max": 45.75641632080078,
      "activations/layer8_attention_weight_min": -44.196651458740234,
      "activations/layer9_attention_weight_max": 40.39716339111328,
      "activations/layer9_attention_weight_min": -37.51432800292969,
      "epoch": 5.73,
      "learning_rate": 0.00011418257575757576,
      "loss": 2.8289,
      "step": 98650
    },
    {
      "activations/layer0_attention_weight_max": 16.978130340576172,
      "activations/layer0_attention_weight_min": -11.875,
      "activations/layer10_attention_weight_max": 31.244415283203125,
      "activations/layer10_attention_weight_min": -29.660232543945312,
      "activations/layer11_attention_weight_max": 30.849241256713867,
      "activations/layer11_attention_weight_min": -27.579442977905273,
      "activations/layer12_attention_weight_max": 24.41354751586914,
      "activations/layer12_attention_weight_min": -25.183786392211914,
      "activations/layer13_attention_weight_max": 43.19379806518555,
      "activations/layer13_attention_weight_min": -31.73275375366211,
      "activations/layer14_attention_weight_max": 63.62240982055664,
      "activations/layer14_attention_weight_min": -48.63810348510742,
      "activations/layer15_attention_weight_max": 36.42991638183594,
      "activations/layer15_attention_weight_min": -27.654434204101562,
      "activations/layer16_attention_weight_max": 35.932437896728516,
      "activations/layer16_attention_weight_min": -26.033369064331055,
      "activations/layer17_attention_weight_max": 38.516021728515625,
      "activations/layer17_attention_weight_min": -23.78236961364746,
      "activations/layer18_attention_weight_max": 35.441795349121094,
      "activations/layer18_attention_weight_min": -19.4771785736084,
      "activations/layer19_attention_weight_max": 39.25014877319336,
      "activations/layer19_attention_weight_min": -28.544919967651367,
      "activations/layer1_attention_weight_max": 17.334928512573242,
      "activations/layer1_attention_weight_min": -15.653379440307617,
      "activations/layer20_attention_weight_max": 31.97008514404297,
      "activations/layer20_attention_weight_min": -23.656919479370117,
      "activations/layer21_attention_weight_max": 34.99064254760742,
      "activations/layer21_attention_weight_min": -22.142553329467773,
      "activations/layer22_attention_weight_max": 51.78044891357422,
      "activations/layer22_attention_weight_min": -28.54137420654297,
      "activations/layer23_attention_weight_max": 34.589332580566406,
      "activations/layer23_attention_weight_min": -21.842273712158203,
      "activations/layer2_attention_weight_max": 32.071678161621094,
      "activations/layer2_attention_weight_min": -31.49429702758789,
      "activations/layer3_attention_weight_max": 92.7459487915039,
      "activations/layer3_attention_weight_min": -93.66682434082031,
      "activations/layer4_attention_weight_max": 124.0336685180664,
      "activations/layer4_attention_weight_min": -127.7574462890625,
      "activations/layer5_attention_weight_max": 72.95364379882812,
      "activations/layer5_attention_weight_min": -72.76841735839844,
      "activations/layer6_attention_weight_max": 53.35463333129883,
      "activations/layer6_attention_weight_min": -56.8460807800293,
      "activations/layer7_attention_weight_max": 84.05528259277344,
      "activations/layer7_attention_weight_min": -84.21048736572266,
      "activations/layer8_attention_weight_max": 37.91506576538086,
      "activations/layer8_attention_weight_min": -38.76272964477539,
      "activations/layer9_attention_weight_max": 33.229461669921875,
      "activations/layer9_attention_weight_min": -32.13450622558594,
      "epoch": 5.74,
      "learning_rate": 0.00011416363636363634,
      "loss": 2.8465,
      "step": 98700
    },
    {
      "activations/layer0_attention_weight_max": 16.531890869140625,
      "activations/layer0_attention_weight_min": -11.461678504943848,
      "activations/layer10_attention_weight_max": 34.9857292175293,
      "activations/layer10_attention_weight_min": -32.612579345703125,
      "activations/layer11_attention_weight_max": 37.164093017578125,
      "activations/layer11_attention_weight_min": -27.0389461517334,
      "activations/layer12_attention_weight_max": 25.66187858581543,
      "activations/layer12_attention_weight_min": -29.10877227783203,
      "activations/layer13_attention_weight_max": 41.5402717590332,
      "activations/layer13_attention_weight_min": -32.936676025390625,
      "activations/layer14_attention_weight_max": 68.93253326416016,
      "activations/layer14_attention_weight_min": -47.6226921081543,
      "activations/layer15_attention_weight_max": 38.69491195678711,
      "activations/layer15_attention_weight_min": -27.470813751220703,
      "activations/layer16_attention_weight_max": 34.83518600463867,
      "activations/layer16_attention_weight_min": -26.514436721801758,
      "activations/layer17_attention_weight_max": 37.187137603759766,
      "activations/layer17_attention_weight_min": -24.476425170898438,
      "activations/layer18_attention_weight_max": 30.703886032104492,
      "activations/layer18_attention_weight_min": -21.581958770751953,
      "activations/layer19_attention_weight_max": 39.62789535522461,
      "activations/layer19_attention_weight_min": -30.132322311401367,
      "activations/layer1_attention_weight_max": 16.91508674621582,
      "activations/layer1_attention_weight_min": -13.727131843566895,
      "activations/layer20_attention_weight_max": 34.96699142456055,
      "activations/layer20_attention_weight_min": -26.579784393310547,
      "activations/layer21_attention_weight_max": 33.3795280456543,
      "activations/layer21_attention_weight_min": -22.911874771118164,
      "activations/layer22_attention_weight_max": 49.92167282104492,
      "activations/layer22_attention_weight_min": -31.379718780517578,
      "activations/layer23_attention_weight_max": 31.259204864501953,
      "activations/layer23_attention_weight_min": -20.161718368530273,
      "activations/layer2_attention_weight_max": 32.05219268798828,
      "activations/layer2_attention_weight_min": -31.316280364990234,
      "activations/layer3_attention_weight_max": 92.29224395751953,
      "activations/layer3_attention_weight_min": -89.36505889892578,
      "activations/layer4_attention_weight_max": 127.4142074584961,
      "activations/layer4_attention_weight_min": -122.364990234375,
      "activations/layer5_attention_weight_max": 78.33232116699219,
      "activations/layer5_attention_weight_min": -76.77823638916016,
      "activations/layer6_attention_weight_max": 56.97123336791992,
      "activations/layer6_attention_weight_min": -59.338958740234375,
      "activations/layer7_attention_weight_max": 95.0120620727539,
      "activations/layer7_attention_weight_min": -93.14872741699219,
      "activations/layer8_attention_weight_max": 43.72555923461914,
      "activations/layer8_attention_weight_min": -43.32883834838867,
      "activations/layer9_attention_weight_max": 38.76720428466797,
      "activations/layer9_attention_weight_min": -35.433326721191406,
      "epoch": 5.74,
      "learning_rate": 0.00011414469696969696,
      "loss": 2.8564,
      "step": 98750
    },
    {
      "activations/layer0_attention_weight_max": 16.863698959350586,
      "activations/layer0_attention_weight_min": -11.265030860900879,
      "activations/layer10_attention_weight_max": 36.88927459716797,
      "activations/layer10_attention_weight_min": -31.469907760620117,
      "activations/layer11_attention_weight_max": 32.35680389404297,
      "activations/layer11_attention_weight_min": -28.45920181274414,
      "activations/layer12_attention_weight_max": 29.450273513793945,
      "activations/layer12_attention_weight_min": -25.943500518798828,
      "activations/layer13_attention_weight_max": 43.6528205871582,
      "activations/layer13_attention_weight_min": -32.425594329833984,
      "activations/layer14_attention_weight_max": 58.900333404541016,
      "activations/layer14_attention_weight_min": -39.70052719116211,
      "activations/layer15_attention_weight_max": 44.26993942260742,
      "activations/layer15_attention_weight_min": -27.89901351928711,
      "activations/layer16_attention_weight_max": 38.992374420166016,
      "activations/layer16_attention_weight_min": -28.259124755859375,
      "activations/layer17_attention_weight_max": 40.91306686401367,
      "activations/layer17_attention_weight_min": -23.94463539123535,
      "activations/layer18_attention_weight_max": 32.78202819824219,
      "activations/layer18_attention_weight_min": -19.58428382873535,
      "activations/layer19_attention_weight_max": 40.73081970214844,
      "activations/layer19_attention_weight_min": -33.03438186645508,
      "activations/layer1_attention_weight_max": 17.499486923217773,
      "activations/layer1_attention_weight_min": -14.989960670471191,
      "activations/layer20_attention_weight_max": 33.5479736328125,
      "activations/layer20_attention_weight_min": -22.687767028808594,
      "activations/layer21_attention_weight_max": 29.32706642150879,
      "activations/layer21_attention_weight_min": -20.630830764770508,
      "activations/layer22_attention_weight_max": 54.05607986450195,
      "activations/layer22_attention_weight_min": -28.843124389648438,
      "activations/layer23_attention_weight_max": 35.272525787353516,
      "activations/layer23_attention_weight_min": -21.47426986694336,
      "activations/layer2_attention_weight_max": 33.24308776855469,
      "activations/layer2_attention_weight_min": -33.22161102294922,
      "activations/layer3_attention_weight_max": 92.5809326171875,
      "activations/layer3_attention_weight_min": -96.3641586303711,
      "activations/layer4_attention_weight_max": 127.20501708984375,
      "activations/layer4_attention_weight_min": -125.46227264404297,
      "activations/layer5_attention_weight_max": 75.81133270263672,
      "activations/layer5_attention_weight_min": -73.97856140136719,
      "activations/layer6_attention_weight_max": 54.44986343383789,
      "activations/layer6_attention_weight_min": -55.34053421020508,
      "activations/layer7_attention_weight_max": 83.06175994873047,
      "activations/layer7_attention_weight_min": -83.35831451416016,
      "activations/layer8_attention_weight_max": 39.90575408935547,
      "activations/layer8_attention_weight_min": -42.0683479309082,
      "activations/layer9_attention_weight_max": 39.502193450927734,
      "activations/layer9_attention_weight_min": -36.38510513305664,
      "epoch": 5.74,
      "learning_rate": 0.00011412575757575757,
      "loss": 2.8463,
      "step": 98800
    },
    {
      "activations/layer0_attention_weight_max": 15.11893081665039,
      "activations/layer0_attention_weight_min": -12.396171569824219,
      "activations/layer10_attention_weight_max": 32.529212951660156,
      "activations/layer10_attention_weight_min": -31.46552085876465,
      "activations/layer11_attention_weight_max": 31.889196395874023,
      "activations/layer11_attention_weight_min": -28.449966430664062,
      "activations/layer12_attention_weight_max": 29.675064086914062,
      "activations/layer12_attention_weight_min": -29.940879821777344,
      "activations/layer13_attention_weight_max": 46.34748077392578,
      "activations/layer13_attention_weight_min": -35.5542106628418,
      "activations/layer14_attention_weight_max": 70.23847961425781,
      "activations/layer14_attention_weight_min": -47.34855651855469,
      "activations/layer15_attention_weight_max": 40.58830261230469,
      "activations/layer15_attention_weight_min": -29.281692504882812,
      "activations/layer16_attention_weight_max": 34.80362319946289,
      "activations/layer16_attention_weight_min": -27.48503875732422,
      "activations/layer17_attention_weight_max": 39.6061897277832,
      "activations/layer17_attention_weight_min": -24.48118019104004,
      "activations/layer18_attention_weight_max": 31.61810302734375,
      "activations/layer18_attention_weight_min": -20.57172393798828,
      "activations/layer19_attention_weight_max": 42.483375549316406,
      "activations/layer19_attention_weight_min": -30.748868942260742,
      "activations/layer1_attention_weight_max": 17.300390243530273,
      "activations/layer1_attention_weight_min": -14.89101791381836,
      "activations/layer20_attention_weight_max": 35.44688034057617,
      "activations/layer20_attention_weight_min": -22.794580459594727,
      "activations/layer21_attention_weight_max": 32.011871337890625,
      "activations/layer21_attention_weight_min": -20.3575382232666,
      "activations/layer22_attention_weight_max": 57.77285385131836,
      "activations/layer22_attention_weight_min": -29.873451232910156,
      "activations/layer23_attention_weight_max": 34.1963996887207,
      "activations/layer23_attention_weight_min": -22.469406127929688,
      "activations/layer2_attention_weight_max": 33.07295227050781,
      "activations/layer2_attention_weight_min": -31.62677574157715,
      "activations/layer3_attention_weight_max": 93.87553405761719,
      "activations/layer3_attention_weight_min": -91.51294708251953,
      "activations/layer4_attention_weight_max": 130.7805938720703,
      "activations/layer4_attention_weight_min": -120.7879409790039,
      "activations/layer5_attention_weight_max": 77.24507904052734,
      "activations/layer5_attention_weight_min": -70.83687591552734,
      "activations/layer6_attention_weight_max": 54.463462829589844,
      "activations/layer6_attention_weight_min": -55.640869140625,
      "activations/layer7_attention_weight_max": 88.59547424316406,
      "activations/layer7_attention_weight_min": -88.69825744628906,
      "activations/layer8_attention_weight_max": 41.68922805786133,
      "activations/layer8_attention_weight_min": -40.45977020263672,
      "activations/layer9_attention_weight_max": 35.149253845214844,
      "activations/layer9_attention_weight_min": -34.035728454589844,
      "epoch": 5.74,
      "learning_rate": 0.00011410719696969697,
      "loss": 2.8464,
      "step": 98850
    },
    {
      "activations/layer0_attention_weight_max": 16.511438369750977,
      "activations/layer0_attention_weight_min": -11.859334945678711,
      "activations/layer10_attention_weight_max": 35.42825698852539,
      "activations/layer10_attention_weight_min": -33.73060607910156,
      "activations/layer11_attention_weight_max": 32.87739181518555,
      "activations/layer11_attention_weight_min": -29.544097900390625,
      "activations/layer12_attention_weight_max": 24.87627601623535,
      "activations/layer12_attention_weight_min": -28.22827911376953,
      "activations/layer13_attention_weight_max": 41.3491325378418,
      "activations/layer13_attention_weight_min": -34.21347427368164,
      "activations/layer14_attention_weight_max": 69.29597473144531,
      "activations/layer14_attention_weight_min": -59.1112060546875,
      "activations/layer15_attention_weight_max": 33.3648567199707,
      "activations/layer15_attention_weight_min": -28.4514102935791,
      "activations/layer16_attention_weight_max": 36.83318328857422,
      "activations/layer16_attention_weight_min": -28.323049545288086,
      "activations/layer17_attention_weight_max": 31.28692626953125,
      "activations/layer17_attention_weight_min": -23.728336334228516,
      "activations/layer18_attention_weight_max": 30.569808959960938,
      "activations/layer18_attention_weight_min": -19.639495849609375,
      "activations/layer19_attention_weight_max": 36.98446273803711,
      "activations/layer19_attention_weight_min": -30.247421264648438,
      "activations/layer1_attention_weight_max": 18.51645278930664,
      "activations/layer1_attention_weight_min": -17.9232234954834,
      "activations/layer20_attention_weight_max": 27.88239097595215,
      "activations/layer20_attention_weight_min": -24.606300354003906,
      "activations/layer21_attention_weight_max": 25.330949783325195,
      "activations/layer21_attention_weight_min": -22.370203018188477,
      "activations/layer22_attention_weight_max": 42.89606475830078,
      "activations/layer22_attention_weight_min": -28.44170570373535,
      "activations/layer23_attention_weight_max": 34.050804138183594,
      "activations/layer23_attention_weight_min": -22.919754028320312,
      "activations/layer2_attention_weight_max": 33.25492858886719,
      "activations/layer2_attention_weight_min": -32.8203125,
      "activations/layer3_attention_weight_max": 94.8946304321289,
      "activations/layer3_attention_weight_min": -95.69585418701172,
      "activations/layer4_attention_weight_max": 134.23780822753906,
      "activations/layer4_attention_weight_min": -129.90219116210938,
      "activations/layer5_attention_weight_max": 80.30296325683594,
      "activations/layer5_attention_weight_min": -74.52389526367188,
      "activations/layer6_attention_weight_max": 58.04862594604492,
      "activations/layer6_attention_weight_min": -57.32368850708008,
      "activations/layer7_attention_weight_max": 89.83216857910156,
      "activations/layer7_attention_weight_min": -94.37007904052734,
      "activations/layer8_attention_weight_max": 43.10628890991211,
      "activations/layer8_attention_weight_min": -45.66471481323242,
      "activations/layer9_attention_weight_max": 37.5535774230957,
      "activations/layer9_attention_weight_min": -37.78600311279297,
      "epoch": 5.75,
      "learning_rate": 0.00011408825757575755,
      "loss": 2.8308,
      "step": 98900
    },
    {
      "activations/layer0_attention_weight_max": 16.39148712158203,
      "activations/layer0_attention_weight_min": -12.47086238861084,
      "activations/layer10_attention_weight_max": 34.259029388427734,
      "activations/layer10_attention_weight_min": -30.976421356201172,
      "activations/layer11_attention_weight_max": 31.024551391601562,
      "activations/layer11_attention_weight_min": -27.56170654296875,
      "activations/layer12_attention_weight_max": 22.969175338745117,
      "activations/layer12_attention_weight_min": -26.579816818237305,
      "activations/layer13_attention_weight_max": 43.71440124511719,
      "activations/layer13_attention_weight_min": -31.82150650024414,
      "activations/layer14_attention_weight_max": 59.360355377197266,
      "activations/layer14_attention_weight_min": -41.17195129394531,
      "activations/layer15_attention_weight_max": 37.703617095947266,
      "activations/layer15_attention_weight_min": -27.482542037963867,
      "activations/layer16_attention_weight_max": 36.11395263671875,
      "activations/layer16_attention_weight_min": -28.61598014831543,
      "activations/layer17_attention_weight_max": 37.87752151489258,
      "activations/layer17_attention_weight_min": -23.82139015197754,
      "activations/layer18_attention_weight_max": 32.36909103393555,
      "activations/layer18_attention_weight_min": -22.021900177001953,
      "activations/layer19_attention_weight_max": 38.21184158325195,
      "activations/layer19_attention_weight_min": -30.181894302368164,
      "activations/layer1_attention_weight_max": 16.233970642089844,
      "activations/layer1_attention_weight_min": -15.270903587341309,
      "activations/layer20_attention_weight_max": 31.72998046875,
      "activations/layer20_attention_weight_min": -25.627790451049805,
      "activations/layer21_attention_weight_max": 34.97757339477539,
      "activations/layer21_attention_weight_min": -22.346914291381836,
      "activations/layer22_attention_weight_max": 53.158443450927734,
      "activations/layer22_attention_weight_min": -30.783227920532227,
      "activations/layer23_attention_weight_max": 32.4741096496582,
      "activations/layer23_attention_weight_min": -22.933752059936523,
      "activations/layer2_attention_weight_max": 32.404747009277344,
      "activations/layer2_attention_weight_min": -32.64949035644531,
      "activations/layer3_attention_weight_max": 90.78326416015625,
      "activations/layer3_attention_weight_min": -94.47030639648438,
      "activations/layer4_attention_weight_max": 127.8034439086914,
      "activations/layer4_attention_weight_min": -129.328369140625,
      "activations/layer5_attention_weight_max": 74.93254852294922,
      "activations/layer5_attention_weight_min": -74.91615295410156,
      "activations/layer6_attention_weight_max": 54.469276428222656,
      "activations/layer6_attention_weight_min": -55.21834182739258,
      "activations/layer7_attention_weight_max": 87.94957733154297,
      "activations/layer7_attention_weight_min": -85.05770111083984,
      "activations/layer8_attention_weight_max": 43.43511199951172,
      "activations/layer8_attention_weight_min": -40.200801849365234,
      "activations/layer9_attention_weight_max": 36.91813278198242,
      "activations/layer9_attention_weight_min": -35.03886413574219,
      "epoch": 5.75,
      "learning_rate": 0.00011406931818181817,
      "loss": 2.8452,
      "step": 98950
    },
    {
      "activations/layer0_attention_weight_max": 17.758499145507812,
      "activations/layer0_attention_weight_min": -11.429457664489746,
      "activations/layer10_attention_weight_max": 33.904945373535156,
      "activations/layer10_attention_weight_min": -31.90676498413086,
      "activations/layer11_attention_weight_max": 31.08538055419922,
      "activations/layer11_attention_weight_min": -28.578140258789062,
      "activations/layer12_attention_weight_max": 27.977617263793945,
      "activations/layer12_attention_weight_min": -25.095415115356445,
      "activations/layer13_attention_weight_max": 48.16079330444336,
      "activations/layer13_attention_weight_min": -35.59583282470703,
      "activations/layer14_attention_weight_max": 55.76070022583008,
      "activations/layer14_attention_weight_min": -43.32709503173828,
      "activations/layer15_attention_weight_max": 37.47174835205078,
      "activations/layer15_attention_weight_min": -26.96673583984375,
      "activations/layer16_attention_weight_max": 33.785648345947266,
      "activations/layer16_attention_weight_min": -26.349040985107422,
      "activations/layer17_attention_weight_max": 39.73872756958008,
      "activations/layer17_attention_weight_min": -23.92572021484375,
      "activations/layer18_attention_weight_max": 33.927330017089844,
      "activations/layer18_attention_weight_min": -20.143041610717773,
      "activations/layer19_attention_weight_max": 40.21968078613281,
      "activations/layer19_attention_weight_min": -30.515274047851562,
      "activations/layer1_attention_weight_max": 16.174999237060547,
      "activations/layer1_attention_weight_min": -14.514625549316406,
      "activations/layer20_attention_weight_max": 32.27840042114258,
      "activations/layer20_attention_weight_min": -23.606082916259766,
      "activations/layer21_attention_weight_max": 31.53511619567871,
      "activations/layer21_attention_weight_min": -21.62218475341797,
      "activations/layer22_attention_weight_max": 51.41069412231445,
      "activations/layer22_attention_weight_min": -26.888872146606445,
      "activations/layer23_attention_weight_max": 35.53099822998047,
      "activations/layer23_attention_weight_min": -19.6119384765625,
      "activations/layer2_attention_weight_max": 32.45387649536133,
      "activations/layer2_attention_weight_min": -30.289955139160156,
      "activations/layer3_attention_weight_max": 92.11505889892578,
      "activations/layer3_attention_weight_min": -91.34738159179688,
      "activations/layer4_attention_weight_max": 126.65284729003906,
      "activations/layer4_attention_weight_min": -125.08416748046875,
      "activations/layer5_attention_weight_max": 74.82737731933594,
      "activations/layer5_attention_weight_min": -74.70731353759766,
      "activations/layer6_attention_weight_max": 56.81093215942383,
      "activations/layer6_attention_weight_min": -55.14006042480469,
      "activations/layer7_attention_weight_max": 86.68136596679688,
      "activations/layer7_attention_weight_min": -86.82555389404297,
      "activations/layer8_attention_weight_max": 41.05202865600586,
      "activations/layer8_attention_weight_min": -39.261817932128906,
      "activations/layer9_attention_weight_max": 39.570621490478516,
      "activations/layer9_attention_weight_min": -35.479705810546875,
      "epoch": 5.75,
      "learning_rate": 0.00011405037878787878,
      "loss": 2.8499,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.4527,
      "eval_samples_per_second": 508.0,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.4527,
      "eval_openwebtext_samples_per_second": 508.0,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 1.9665,
      "eval_wikitext_samples_per_second": 231.88,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_lambada_loss": 2.708984375,
      "eval_lambada_ppl": 15.01401915561195,
      "eval_lambada_runtime": 9.5377,
      "eval_lambada_samples_per_second": 510.498,
      "step": 99000
    },
    {
      "activations/layer0_attention_weight_max": 15.822026252746582,
      "activations/layer0_attention_weight_min": -12.12154769897461,
      "activations/layer10_attention_weight_max": 33.82882308959961,
      "activations/layer10_attention_weight_min": -30.870372772216797,
      "activations/layer11_attention_weight_max": 31.922046661376953,
      "activations/layer11_attention_weight_min": -27.804561614990234,
      "activations/layer12_attention_weight_max": 25.255996704101562,
      "activations/layer12_attention_weight_min": -29.742395401000977,
      "activations/layer13_attention_weight_max": 49.58405303955078,
      "activations/layer13_attention_weight_min": -35.8065299987793,
      "activations/layer14_attention_weight_max": 64.14741516113281,
      "activations/layer14_attention_weight_min": -45.083473205566406,
      "activations/layer15_attention_weight_max": 39.36499786376953,
      "activations/layer15_attention_weight_min": -28.730712890625,
      "activations/layer16_attention_weight_max": 35.44816970825195,
      "activations/layer16_attention_weight_min": -27.945709228515625,
      "activations/layer17_attention_weight_max": 35.23307418823242,
      "activations/layer17_attention_weight_min": -25.59514808654785,
      "activations/layer18_attention_weight_max": 33.046417236328125,
      "activations/layer18_attention_weight_min": -20.84778594970703,
      "activations/layer19_attention_weight_max": 37.62082290649414,
      "activations/layer19_attention_weight_min": -28.937402725219727,
      "activations/layer1_attention_weight_max": 18.055828094482422,
      "activations/layer1_attention_weight_min": -16.50625228881836,
      "activations/layer20_attention_weight_max": 32.61592483520508,
      "activations/layer20_attention_weight_min": -23.240509033203125,
      "activations/layer21_attention_weight_max": 31.512557983398438,
      "activations/layer21_attention_weight_min": -21.35431480407715,
      "activations/layer22_attention_weight_max": 46.5103759765625,
      "activations/layer22_attention_weight_min": -28.577831268310547,
      "activations/layer23_attention_weight_max": 36.99253845214844,
      "activations/layer23_attention_weight_min": -22.684852600097656,
      "activations/layer2_attention_weight_max": 33.62506866455078,
      "activations/layer2_attention_weight_min": -33.86405944824219,
      "activations/layer3_attention_weight_max": 93.03243255615234,
      "activations/layer3_attention_weight_min": -92.45451354980469,
      "activations/layer4_attention_weight_max": 127.38478088378906,
      "activations/layer4_attention_weight_min": -123.57262420654297,
      "activations/layer5_attention_weight_max": 74.094970703125,
      "activations/layer5_attention_weight_min": -72.23638916015625,
      "activations/layer6_attention_weight_max": 52.75514221191406,
      "activations/layer6_attention_weight_min": -53.83889389038086,
      "activations/layer7_attention_weight_max": 88.80372619628906,
      "activations/layer7_attention_weight_min": -90.71847534179688,
      "activations/layer8_attention_weight_max": 44.67033767700195,
      "activations/layer8_attention_weight_min": -42.16325759887695,
      "activations/layer9_attention_weight_max": 39.151363372802734,
      "activations/layer9_attention_weight_min": -35.57735061645508,
      "epoch": 5.76,
      "learning_rate": 0.00011403143939393937,
      "loss": 2.8481,
      "step": 99050
    },
    {
      "activations/layer0_attention_weight_max": 15.537841796875,
      "activations/layer0_attention_weight_min": -11.591063499450684,
      "activations/layer10_attention_weight_max": 32.51900100708008,
      "activations/layer10_attention_weight_min": -31.7186336517334,
      "activations/layer11_attention_weight_max": 29.524185180664062,
      "activations/layer11_attention_weight_min": -28.007583618164062,
      "activations/layer12_attention_weight_max": 30.472003936767578,
      "activations/layer12_attention_weight_min": -27.038841247558594,
      "activations/layer13_attention_weight_max": 40.48555374145508,
      "activations/layer13_attention_weight_min": -31.473756790161133,
      "activations/layer14_attention_weight_max": 59.98881149291992,
      "activations/layer14_attention_weight_min": -47.33882141113281,
      "activations/layer15_attention_weight_max": 34.56892776489258,
      "activations/layer15_attention_weight_min": -27.599016189575195,
      "activations/layer16_attention_weight_max": 33.21064376831055,
      "activations/layer16_attention_weight_min": -28.189411163330078,
      "activations/layer17_attention_weight_max": 36.504432678222656,
      "activations/layer17_attention_weight_min": -23.50821304321289,
      "activations/layer18_attention_weight_max": 37.891666412353516,
      "activations/layer18_attention_weight_min": -19.67261505126953,
      "activations/layer19_attention_weight_max": 39.20806121826172,
      "activations/layer19_attention_weight_min": -28.999982833862305,
      "activations/layer1_attention_weight_max": 16.338468551635742,
      "activations/layer1_attention_weight_min": -15.636484146118164,
      "activations/layer20_attention_weight_max": 31.04216766357422,
      "activations/layer20_attention_weight_min": -22.323373794555664,
      "activations/layer21_attention_weight_max": 28.923803329467773,
      "activations/layer21_attention_weight_min": -20.207595825195312,
      "activations/layer22_attention_weight_max": 49.32310104370117,
      "activations/layer22_attention_weight_min": -26.28847312927246,
      "activations/layer23_attention_weight_max": 35.89457321166992,
      "activations/layer23_attention_weight_min": -22.125041961669922,
      "activations/layer2_attention_weight_max": 32.0452766418457,
      "activations/layer2_attention_weight_min": -31.04926109313965,
      "activations/layer3_attention_weight_max": 90.2657470703125,
      "activations/layer3_attention_weight_min": -93.21070098876953,
      "activations/layer4_attention_weight_max": 129.73593139648438,
      "activations/layer4_attention_weight_min": -128.13095092773438,
      "activations/layer5_attention_weight_max": 75.63851928710938,
      "activations/layer5_attention_weight_min": -74.44779205322266,
      "activations/layer6_attention_weight_max": 54.55727767944336,
      "activations/layer6_attention_weight_min": -58.53522491455078,
      "activations/layer7_attention_weight_max": 86.91979217529297,
      "activations/layer7_attention_weight_min": -86.39337921142578,
      "activations/layer8_attention_weight_max": 39.04133224487305,
      "activations/layer8_attention_weight_min": -37.756168365478516,
      "activations/layer9_attention_weight_max": 35.143585205078125,
      "activations/layer9_attention_weight_min": -36.92306900024414,
      "epoch": 5.76,
      "learning_rate": 0.00011401249999999999,
      "loss": 2.8382,
      "step": 99100
    },
    {
      "activations/layer0_attention_weight_max": 15.892570495605469,
      "activations/layer0_attention_weight_min": -11.492815971374512,
      "activations/layer10_attention_weight_max": 34.693603515625,
      "activations/layer10_attention_weight_min": -32.06250762939453,
      "activations/layer11_attention_weight_max": 32.73487091064453,
      "activations/layer11_attention_weight_min": -28.09027099609375,
      "activations/layer12_attention_weight_max": 46.1293830871582,
      "activations/layer12_attention_weight_min": -27.659156799316406,
      "activations/layer13_attention_weight_max": 44.703269958496094,
      "activations/layer13_attention_weight_min": -32.206153869628906,
      "activations/layer14_attention_weight_max": 56.08656692504883,
      "activations/layer14_attention_weight_min": -48.3194694519043,
      "activations/layer15_attention_weight_max": 38.08742141723633,
      "activations/layer15_attention_weight_min": -30.817777633666992,
      "activations/layer16_attention_weight_max": 37.90351104736328,
      "activations/layer16_attention_weight_min": -29.93387794494629,
      "activations/layer17_attention_weight_max": 37.67402648925781,
      "activations/layer17_attention_weight_min": -27.822389602661133,
      "activations/layer18_attention_weight_max": 33.0631103515625,
      "activations/layer18_attention_weight_min": -21.513931274414062,
      "activations/layer19_attention_weight_max": 44.60683059692383,
      "activations/layer19_attention_weight_min": -37.32107925415039,
      "activations/layer1_attention_weight_max": 17.494129180908203,
      "activations/layer1_attention_weight_min": -15.427938461303711,
      "activations/layer20_attention_weight_max": 34.21665954589844,
      "activations/layer20_attention_weight_min": -30.252172470092773,
      "activations/layer21_attention_weight_max": 32.76611328125,
      "activations/layer21_attention_weight_min": -26.046167373657227,
      "activations/layer22_attention_weight_max": 55.69868850708008,
      "activations/layer22_attention_weight_min": -34.71519088745117,
      "activations/layer23_attention_weight_max": 35.07018280029297,
      "activations/layer23_attention_weight_min": -23.701793670654297,
      "activations/layer2_attention_weight_max": 33.19723892211914,
      "activations/layer2_attention_weight_min": -31.546791076660156,
      "activations/layer3_attention_weight_max": 90.6827621459961,
      "activations/layer3_attention_weight_min": -93.37086486816406,
      "activations/layer4_attention_weight_max": 128.96279907226562,
      "activations/layer4_attention_weight_min": -128.3519744873047,
      "activations/layer5_attention_weight_max": 77.69219207763672,
      "activations/layer5_attention_weight_min": -79.03955841064453,
      "activations/layer6_attention_weight_max": 55.481956481933594,
      "activations/layer6_attention_weight_min": -57.52579116821289,
      "activations/layer7_attention_weight_max": 84.82112121582031,
      "activations/layer7_attention_weight_min": -86.34495544433594,
      "activations/layer8_attention_weight_max": 40.022972106933594,
      "activations/layer8_attention_weight_min": -38.510196685791016,
      "activations/layer9_attention_weight_max": 35.53028869628906,
      "activations/layer9_attention_weight_min": -33.75187301635742,
      "epoch": 5.76,
      "learning_rate": 0.0001139935606060606,
      "loss": 2.8311,
      "step": 99150
    },
    {
      "activations/layer0_attention_weight_max": 16.46128273010254,
      "activations/layer0_attention_weight_min": -11.876014709472656,
      "activations/layer10_attention_weight_max": 32.828739166259766,
      "activations/layer10_attention_weight_min": -27.907665252685547,
      "activations/layer11_attention_weight_max": 29.607994079589844,
      "activations/layer11_attention_weight_min": -27.964374542236328,
      "activations/layer12_attention_weight_max": 23.70401382446289,
      "activations/layer12_attention_weight_min": -35.85141372680664,
      "activations/layer13_attention_weight_max": 38.02812957763672,
      "activations/layer13_attention_weight_min": -31.875595092773438,
      "activations/layer14_attention_weight_max": 57.39927291870117,
      "activations/layer14_attention_weight_min": -47.384639739990234,
      "activations/layer15_attention_weight_max": 34.36924743652344,
      "activations/layer15_attention_weight_min": -26.536273956298828,
      "activations/layer16_attention_weight_max": 32.04167556762695,
      "activations/layer16_attention_weight_min": -26.63197135925293,
      "activations/layer17_attention_weight_max": 34.26411437988281,
      "activations/layer17_attention_weight_min": -23.1246280670166,
      "activations/layer18_attention_weight_max": 32.375431060791016,
      "activations/layer18_attention_weight_min": -19.001399993896484,
      "activations/layer19_attention_weight_max": 36.217281341552734,
      "activations/layer19_attention_weight_min": -29.009714126586914,
      "activations/layer1_attention_weight_max": 17.04533576965332,
      "activations/layer1_attention_weight_min": -15.510189056396484,
      "activations/layer20_attention_weight_max": 29.748538970947266,
      "activations/layer20_attention_weight_min": -23.576904296875,
      "activations/layer21_attention_weight_max": 28.866031646728516,
      "activations/layer21_attention_weight_min": -21.117115020751953,
      "activations/layer22_attention_weight_max": 48.71913146972656,
      "activations/layer22_attention_weight_min": -27.99077033996582,
      "activations/layer23_attention_weight_max": 31.976898193359375,
      "activations/layer23_attention_weight_min": -20.703746795654297,
      "activations/layer2_attention_weight_max": 34.236045837402344,
      "activations/layer2_attention_weight_min": -31.20145034790039,
      "activations/layer3_attention_weight_max": 98.96332550048828,
      "activations/layer3_attention_weight_min": -93.2294692993164,
      "activations/layer4_attention_weight_max": 134.53570556640625,
      "activations/layer4_attention_weight_min": -125.3576889038086,
      "activations/layer5_attention_weight_max": 76.90919494628906,
      "activations/layer5_attention_weight_min": -71.97696685791016,
      "activations/layer6_attention_weight_max": 56.32808303833008,
      "activations/layer6_attention_weight_min": -52.61442184448242,
      "activations/layer7_attention_weight_max": 89.0516586303711,
      "activations/layer7_attention_weight_min": -76.5946273803711,
      "activations/layer8_attention_weight_max": 41.41150665283203,
      "activations/layer8_attention_weight_min": -37.225624084472656,
      "activations/layer9_attention_weight_max": 35.0265007019043,
      "activations/layer9_attention_weight_min": -39.912899017333984,
      "epoch": 5.76,
      "learning_rate": 0.00011397499999999999,
      "loss": 2.8424,
      "step": 99200
    },
    {
      "activations/layer0_attention_weight_max": 16.847291946411133,
      "activations/layer0_attention_weight_min": -12.552803039550781,
      "activations/layer10_attention_weight_max": 32.95745849609375,
      "activations/layer10_attention_weight_min": -33.86088562011719,
      "activations/layer11_attention_weight_max": 33.72943878173828,
      "activations/layer11_attention_weight_min": -29.009353637695312,
      "activations/layer12_attention_weight_max": 37.225074768066406,
      "activations/layer12_attention_weight_min": -27.572153091430664,
      "activations/layer13_attention_weight_max": 42.32381820678711,
      "activations/layer13_attention_weight_min": -30.428133010864258,
      "activations/layer14_attention_weight_max": 58.36835479736328,
      "activations/layer14_attention_weight_min": -41.113807678222656,
      "activations/layer15_attention_weight_max": 41.79132843017578,
      "activations/layer15_attention_weight_min": -29.234296798706055,
      "activations/layer16_attention_weight_max": 35.563331604003906,
      "activations/layer16_attention_weight_min": -27.171627044677734,
      "activations/layer17_attention_weight_max": 36.622798919677734,
      "activations/layer17_attention_weight_min": -23.998289108276367,
      "activations/layer18_attention_weight_max": 32.942378997802734,
      "activations/layer18_attention_weight_min": -21.48821449279785,
      "activations/layer19_attention_weight_max": 36.467132568359375,
      "activations/layer19_attention_weight_min": -31.870635986328125,
      "activations/layer1_attention_weight_max": 19.28340721130371,
      "activations/layer1_attention_weight_min": -15.34934139251709,
      "activations/layer20_attention_weight_max": 34.158470153808594,
      "activations/layer20_attention_weight_min": -23.447612762451172,
      "activations/layer21_attention_weight_max": 30.939706802368164,
      "activations/layer21_attention_weight_min": -20.641864776611328,
      "activations/layer22_attention_weight_max": 49.811912536621094,
      "activations/layer22_attention_weight_min": -27.318031311035156,
      "activations/layer23_attention_weight_max": 36.31364822387695,
      "activations/layer23_attention_weight_min": -20.43740463256836,
      "activations/layer2_attention_weight_max": 33.59843444824219,
      "activations/layer2_attention_weight_min": -33.21172332763672,
      "activations/layer3_attention_weight_max": 93.02355194091797,
      "activations/layer3_attention_weight_min": -93.94152069091797,
      "activations/layer4_attention_weight_max": 133.83676147460938,
      "activations/layer4_attention_weight_min": -131.47267150878906,
      "activations/layer5_attention_weight_max": 77.26594543457031,
      "activations/layer5_attention_weight_min": -78.22626495361328,
      "activations/layer6_attention_weight_max": 57.1263313293457,
      "activations/layer6_attention_weight_min": -58.011085510253906,
      "activations/layer7_attention_weight_max": 88.9771499633789,
      "activations/layer7_attention_weight_min": -93.28591918945312,
      "activations/layer8_attention_weight_max": 41.32380294799805,
      "activations/layer8_attention_weight_min": -39.47886276245117,
      "activations/layer9_attention_weight_max": 35.46487808227539,
      "activations/layer9_attention_weight_min": -34.46569061279297,
      "epoch": 5.77,
      "learning_rate": 0.0001139560606060606,
      "loss": 2.835,
      "step": 99250
    },
    {
      "activations/layer0_attention_weight_max": 16.797510147094727,
      "activations/layer0_attention_weight_min": -11.799103736877441,
      "activations/layer10_attention_weight_max": 34.24957275390625,
      "activations/layer10_attention_weight_min": -31.157615661621094,
      "activations/layer11_attention_weight_max": 32.727142333984375,
      "activations/layer11_attention_weight_min": -29.165786743164062,
      "activations/layer12_attention_weight_max": 27.491487503051758,
      "activations/layer12_attention_weight_min": -26.537057876586914,
      "activations/layer13_attention_weight_max": 47.7483024597168,
      "activations/layer13_attention_weight_min": -33.7706298828125,
      "activations/layer14_attention_weight_max": 57.45667266845703,
      "activations/layer14_attention_weight_min": -48.38960647583008,
      "activations/layer15_attention_weight_max": 40.55599594116211,
      "activations/layer15_attention_weight_min": -28.94587516784668,
      "activations/layer16_attention_weight_max": 35.73473358154297,
      "activations/layer16_attention_weight_min": -28.222612380981445,
      "activations/layer17_attention_weight_max": 33.15993118286133,
      "activations/layer17_attention_weight_min": -26.015666961669922,
      "activations/layer18_attention_weight_max": 30.49139404296875,
      "activations/layer18_attention_weight_min": -20.695714950561523,
      "activations/layer19_attention_weight_max": 35.9193115234375,
      "activations/layer19_attention_weight_min": -31.600101470947266,
      "activations/layer1_attention_weight_max": 18.16877555847168,
      "activations/layer1_attention_weight_min": -15.096699714660645,
      "activations/layer20_attention_weight_max": 29.108001708984375,
      "activations/layer20_attention_weight_min": -26.146610260009766,
      "activations/layer21_attention_weight_max": 32.4649658203125,
      "activations/layer21_attention_weight_min": -24.136699676513672,
      "activations/layer22_attention_weight_max": 45.426578521728516,
      "activations/layer22_attention_weight_min": -30.9044246673584,
      "activations/layer23_attention_weight_max": 31.497526168823242,
      "activations/layer23_attention_weight_min": -22.294164657592773,
      "activations/layer2_attention_weight_max": 33.71927261352539,
      "activations/layer2_attention_weight_min": -30.78545379638672,
      "activations/layer3_attention_weight_max": 91.86424255371094,
      "activations/layer3_attention_weight_min": -94.14339447021484,
      "activations/layer4_attention_weight_max": 134.0245819091797,
      "activations/layer4_attention_weight_min": -128.52008056640625,
      "activations/layer5_attention_weight_max": 77.37644958496094,
      "activations/layer5_attention_weight_min": -73.39486694335938,
      "activations/layer6_attention_weight_max": 54.98378372192383,
      "activations/layer6_attention_weight_min": -55.41650390625,
      "activations/layer7_attention_weight_max": 89.8918685913086,
      "activations/layer7_attention_weight_min": -89.1474609375,
      "activations/layer8_attention_weight_max": 40.5528450012207,
      "activations/layer8_attention_weight_min": -40.84383773803711,
      "activations/layer9_attention_weight_max": 38.07905960083008,
      "activations/layer9_attention_weight_min": -37.62173080444336,
      "epoch": 5.77,
      "learning_rate": 0.0001139371212121212,
      "loss": 2.8433,
      "step": 99300
    },
    {
      "activations/layer0_attention_weight_max": 16.263198852539062,
      "activations/layer0_attention_weight_min": -11.665427207946777,
      "activations/layer10_attention_weight_max": 33.08577346801758,
      "activations/layer10_attention_weight_min": -31.057785034179688,
      "activations/layer11_attention_weight_max": 32.29658889770508,
      "activations/layer11_attention_weight_min": -28.31163787841797,
      "activations/layer12_attention_weight_max": 28.88857650756836,
      "activations/layer12_attention_weight_min": -27.40829086303711,
      "activations/layer13_attention_weight_max": 39.89649963378906,
      "activations/layer13_attention_weight_min": -30.890235900878906,
      "activations/layer14_attention_weight_max": 61.407989501953125,
      "activations/layer14_attention_weight_min": -45.5843505859375,
      "activations/layer15_attention_weight_max": 39.103939056396484,
      "activations/layer15_attention_weight_min": -27.987789154052734,
      "activations/layer16_attention_weight_max": 38.317752838134766,
      "activations/layer16_attention_weight_min": -29.71916389465332,
      "activations/layer17_attention_weight_max": 36.73377990722656,
      "activations/layer17_attention_weight_min": -24.587614059448242,
      "activations/layer18_attention_weight_max": 37.06528854370117,
      "activations/layer18_attention_weight_min": -21.9487361907959,
      "activations/layer19_attention_weight_max": 41.216976165771484,
      "activations/layer19_attention_weight_min": -28.434391021728516,
      "activations/layer1_attention_weight_max": 16.841503143310547,
      "activations/layer1_attention_weight_min": -17.589996337890625,
      "activations/layer20_attention_weight_max": 32.95993423461914,
      "activations/layer20_attention_weight_min": -22.661630630493164,
      "activations/layer21_attention_weight_max": 35.79890060424805,
      "activations/layer21_attention_weight_min": -19.992774963378906,
      "activations/layer22_attention_weight_max": 52.24787139892578,
      "activations/layer22_attention_weight_min": -27.318696975708008,
      "activations/layer23_attention_weight_max": 35.51761245727539,
      "activations/layer23_attention_weight_min": -24.00404930114746,
      "activations/layer2_attention_weight_max": 32.72069549560547,
      "activations/layer2_attention_weight_min": -31.022258758544922,
      "activations/layer3_attention_weight_max": 90.86949920654297,
      "activations/layer3_attention_weight_min": -92.0308837890625,
      "activations/layer4_attention_weight_max": 127.55598449707031,
      "activations/layer4_attention_weight_min": -125.45355224609375,
      "activations/layer5_attention_weight_max": 78.46232604980469,
      "activations/layer5_attention_weight_min": -74.77368927001953,
      "activations/layer6_attention_weight_max": 60.76001739501953,
      "activations/layer6_attention_weight_min": -56.08298110961914,
      "activations/layer7_attention_weight_max": 86.7297592163086,
      "activations/layer7_attention_weight_min": -88.66888427734375,
      "activations/layer8_attention_weight_max": 39.80519485473633,
      "activations/layer8_attention_weight_min": -40.11994552612305,
      "activations/layer9_attention_weight_max": 34.686119079589844,
      "activations/layer9_attention_weight_min": -36.40772247314453,
      "epoch": 5.77,
      "learning_rate": 0.00011391818181818181,
      "loss": 2.8278,
      "step": 99350
    },
    {
      "activations/layer0_attention_weight_max": 16.209491729736328,
      "activations/layer0_attention_weight_min": -11.930994033813477,
      "activations/layer10_attention_weight_max": 34.710609436035156,
      "activations/layer10_attention_weight_min": -33.20900344848633,
      "activations/layer11_attention_weight_max": 32.94517135620117,
      "activations/layer11_attention_weight_min": -30.000648498535156,
      "activations/layer12_attention_weight_max": 33.031578063964844,
      "activations/layer12_attention_weight_min": -27.757633209228516,
      "activations/layer13_attention_weight_max": 53.53467559814453,
      "activations/layer13_attention_weight_min": -38.52668380737305,
      "activations/layer14_attention_weight_max": 66.59261322021484,
      "activations/layer14_attention_weight_min": -52.48038101196289,
      "activations/layer15_attention_weight_max": 37.89604949951172,
      "activations/layer15_attention_weight_min": -28.188215255737305,
      "activations/layer16_attention_weight_max": 34.707855224609375,
      "activations/layer16_attention_weight_min": -28.995134353637695,
      "activations/layer17_attention_weight_max": 36.210418701171875,
      "activations/layer17_attention_weight_min": -24.90079689025879,
      "activations/layer18_attention_weight_max": 33.33690643310547,
      "activations/layer18_attention_weight_min": -22.313798904418945,
      "activations/layer19_attention_weight_max": 41.601043701171875,
      "activations/layer19_attention_weight_min": -29.299808502197266,
      "activations/layer1_attention_weight_max": 16.35207748413086,
      "activations/layer1_attention_weight_min": -13.903285026550293,
      "activations/layer20_attention_weight_max": 31.07916831970215,
      "activations/layer20_attention_weight_min": -24.116090774536133,
      "activations/layer21_attention_weight_max": 30.300382614135742,
      "activations/layer21_attention_weight_min": -21.81416130065918,
      "activations/layer22_attention_weight_max": 46.91990280151367,
      "activations/layer22_attention_weight_min": -28.873891830444336,
      "activations/layer23_attention_weight_max": 34.91341781616211,
      "activations/layer23_attention_weight_min": -22.82984733581543,
      "activations/layer2_attention_weight_max": 32.589378356933594,
      "activations/layer2_attention_weight_min": -29.324222564697266,
      "activations/layer3_attention_weight_max": 90.91429901123047,
      "activations/layer3_attention_weight_min": -89.77046966552734,
      "activations/layer4_attention_weight_max": 122.19115447998047,
      "activations/layer4_attention_weight_min": -126.23017883300781,
      "activations/layer5_attention_weight_max": 72.04946899414062,
      "activations/layer5_attention_weight_min": -70.00033569335938,
      "activations/layer6_attention_weight_max": 53.381954193115234,
      "activations/layer6_attention_weight_min": -58.21317672729492,
      "activations/layer7_attention_weight_max": 86.08386993408203,
      "activations/layer7_attention_weight_min": -92.67861938476562,
      "activations/layer8_attention_weight_max": 42.093387603759766,
      "activations/layer8_attention_weight_min": -44.97206497192383,
      "activations/layer9_attention_weight_max": 36.312049865722656,
      "activations/layer9_attention_weight_min": -35.12736511230469,
      "epoch": 5.78,
      "learning_rate": 0.00011389924242424241,
      "loss": 2.8357,
      "step": 99400
    },
    {
      "activations/layer0_attention_weight_max": 16.793376922607422,
      "activations/layer0_attention_weight_min": -11.93435287475586,
      "activations/layer10_attention_weight_max": 32.258995056152344,
      "activations/layer10_attention_weight_min": -30.39453125,
      "activations/layer11_attention_weight_max": 34.25004577636719,
      "activations/layer11_attention_weight_min": -27.65608787536621,
      "activations/layer12_attention_weight_max": 24.12247657775879,
      "activations/layer12_attention_weight_min": -23.433740615844727,
      "activations/layer13_attention_weight_max": 41.54444122314453,
      "activations/layer13_attention_weight_min": -35.524715423583984,
      "activations/layer14_attention_weight_max": 58.59754180908203,
      "activations/layer14_attention_weight_min": -42.39822006225586,
      "activations/layer15_attention_weight_max": 35.82439422607422,
      "activations/layer15_attention_weight_min": -29.654287338256836,
      "activations/layer16_attention_weight_max": 32.97529602050781,
      "activations/layer16_attention_weight_min": -27.1888427734375,
      "activations/layer17_attention_weight_max": 35.933349609375,
      "activations/layer17_attention_weight_min": -24.659759521484375,
      "activations/layer18_attention_weight_max": 29.65383529663086,
      "activations/layer18_attention_weight_min": -20.229021072387695,
      "activations/layer19_attention_weight_max": 33.85915756225586,
      "activations/layer19_attention_weight_min": -29.97771644592285,
      "activations/layer1_attention_weight_max": 16.756492614746094,
      "activations/layer1_attention_weight_min": -13.64367389678955,
      "activations/layer20_attention_weight_max": 30.671262741088867,
      "activations/layer20_attention_weight_min": -22.295324325561523,
      "activations/layer21_attention_weight_max": 34.46331787109375,
      "activations/layer21_attention_weight_min": -20.044818878173828,
      "activations/layer22_attention_weight_max": 47.77635192871094,
      "activations/layer22_attention_weight_min": -26.833648681640625,
      "activations/layer23_attention_weight_max": 31.376203536987305,
      "activations/layer23_attention_weight_min": -20.547042846679688,
      "activations/layer2_attention_weight_max": 31.729251861572266,
      "activations/layer2_attention_weight_min": -30.777721405029297,
      "activations/layer3_attention_weight_max": 90.4872817993164,
      "activations/layer3_attention_weight_min": -89.42586517333984,
      "activations/layer4_attention_weight_max": 121.97871398925781,
      "activations/layer4_attention_weight_min": -119.3415756225586,
      "activations/layer5_attention_weight_max": 69.23973083496094,
      "activations/layer5_attention_weight_min": -68.21595764160156,
      "activations/layer6_attention_weight_max": 52.57728576660156,
      "activations/layer6_attention_weight_min": -52.334068298339844,
      "activations/layer7_attention_weight_max": 83.7942886352539,
      "activations/layer7_attention_weight_min": -82.45307922363281,
      "activations/layer8_attention_weight_max": 39.45697784423828,
      "activations/layer8_attention_weight_min": -38.73667907714844,
      "activations/layer9_attention_weight_max": 35.0541877746582,
      "activations/layer9_attention_weight_min": -32.752986907958984,
      "epoch": 5.78,
      "learning_rate": 0.00011388030303030303,
      "loss": 2.8401,
      "step": 99450
    },
    {
      "activations/layer0_attention_weight_max": 15.847949028015137,
      "activations/layer0_attention_weight_min": -12.629422187805176,
      "activations/layer10_attention_weight_max": 38.80558776855469,
      "activations/layer10_attention_weight_min": -30.98228645324707,
      "activations/layer11_attention_weight_max": 44.57230758666992,
      "activations/layer11_attention_weight_min": -31.8476619720459,
      "activations/layer12_attention_weight_max": 27.94912338256836,
      "activations/layer12_attention_weight_min": -27.282161712646484,
      "activations/layer13_attention_weight_max": 40.48362350463867,
      "activations/layer13_attention_weight_min": -34.35179138183594,
      "activations/layer14_attention_weight_max": 56.091793060302734,
      "activations/layer14_attention_weight_min": -43.200462341308594,
      "activations/layer15_attention_weight_max": 36.61470413208008,
      "activations/layer15_attention_weight_min": -29.187885284423828,
      "activations/layer16_attention_weight_max": 34.33815002441406,
      "activations/layer16_attention_weight_min": -28.677270889282227,
      "activations/layer17_attention_weight_max": 34.691078186035156,
      "activations/layer17_attention_weight_min": -24.115427017211914,
      "activations/layer18_attention_weight_max": 33.53334045410156,
      "activations/layer18_attention_weight_min": -20.95594024658203,
      "activations/layer19_attention_weight_max": 37.234066009521484,
      "activations/layer19_attention_weight_min": -29.26881980895996,
      "activations/layer1_attention_weight_max": 17.190200805664062,
      "activations/layer1_attention_weight_min": -15.084155082702637,
      "activations/layer20_attention_weight_max": 30.135501861572266,
      "activations/layer20_attention_weight_min": -24.628360748291016,
      "activations/layer21_attention_weight_max": 36.983306884765625,
      "activations/layer21_attention_weight_min": -21.35701560974121,
      "activations/layer22_attention_weight_max": 48.7180290222168,
      "activations/layer22_attention_weight_min": -29.166624069213867,
      "activations/layer23_attention_weight_max": 35.003536224365234,
      "activations/layer23_attention_weight_min": -19.841732025146484,
      "activations/layer2_attention_weight_max": 31.919095993041992,
      "activations/layer2_attention_weight_min": -31.913055419921875,
      "activations/layer3_attention_weight_max": 87.07727813720703,
      "activations/layer3_attention_weight_min": -91.72418212890625,
      "activations/layer4_attention_weight_max": 127.95494842529297,
      "activations/layer4_attention_weight_min": -123.0552749633789,
      "activations/layer5_attention_weight_max": 74.84263610839844,
      "activations/layer5_attention_weight_min": -71.57356262207031,
      "activations/layer6_attention_weight_max": 53.29428482055664,
      "activations/layer6_attention_weight_min": -51.66216278076172,
      "activations/layer7_attention_weight_max": 91.13518524169922,
      "activations/layer7_attention_weight_min": -88.32833099365234,
      "activations/layer8_attention_weight_max": 43.68122100830078,
      "activations/layer8_attention_weight_min": -38.10377502441406,
      "activations/layer9_attention_weight_max": 36.602718353271484,
      "activations/layer9_attention_weight_min": -34.17646026611328,
      "epoch": 5.78,
      "learning_rate": 0.00011386136363636363,
      "loss": 2.8462,
      "step": 99500
    },
    {
      "activations/layer0_attention_weight_max": 16.454246520996094,
      "activations/layer0_attention_weight_min": -11.93903636932373,
      "activations/layer10_attention_weight_max": 34.34257507324219,
      "activations/layer10_attention_weight_min": -31.22924041748047,
      "activations/layer11_attention_weight_max": 38.69599533081055,
      "activations/layer11_attention_weight_min": -31.273029327392578,
      "activations/layer12_attention_weight_max": 39.295318603515625,
      "activations/layer12_attention_weight_min": -38.87983322143555,
      "activations/layer13_attention_weight_max": 58.86647415161133,
      "activations/layer13_attention_weight_min": -36.549442291259766,
      "activations/layer14_attention_weight_max": 66.88798522949219,
      "activations/layer14_attention_weight_min": -49.97592544555664,
      "activations/layer15_attention_weight_max": 57.00313949584961,
      "activations/layer15_attention_weight_min": -34.55479431152344,
      "activations/layer16_attention_weight_max": 36.15350341796875,
      "activations/layer16_attention_weight_min": -27.873863220214844,
      "activations/layer17_attention_weight_max": 40.97279357910156,
      "activations/layer17_attention_weight_min": -23.500812530517578,
      "activations/layer18_attention_weight_max": 28.684200286865234,
      "activations/layer18_attention_weight_min": -20.408830642700195,
      "activations/layer19_attention_weight_max": 39.09333801269531,
      "activations/layer19_attention_weight_min": -32.66215515136719,
      "activations/layer1_attention_weight_max": 17.561769485473633,
      "activations/layer1_attention_weight_min": -14.454388618469238,
      "activations/layer20_attention_weight_max": 36.3282470703125,
      "activations/layer20_attention_weight_min": -23.687360763549805,
      "activations/layer21_attention_weight_max": 30.375118255615234,
      "activations/layer21_attention_weight_min": -22.90850830078125,
      "activations/layer22_attention_weight_max": 48.036781311035156,
      "activations/layer22_attention_weight_min": -27.56903076171875,
      "activations/layer23_attention_weight_max": 34.18680191040039,
      "activations/layer23_attention_weight_min": -21.955970764160156,
      "activations/layer2_attention_weight_max": 31.633209228515625,
      "activations/layer2_attention_weight_min": -30.025463104248047,
      "activations/layer3_attention_weight_max": 91.63175201416016,
      "activations/layer3_attention_weight_min": -91.6869888305664,
      "activations/layer4_attention_weight_max": 130.69308471679688,
      "activations/layer4_attention_weight_min": -121.39520263671875,
      "activations/layer5_attention_weight_max": 78.36973571777344,
      "activations/layer5_attention_weight_min": -74.51246643066406,
      "activations/layer6_attention_weight_max": 56.72637176513672,
      "activations/layer6_attention_weight_min": -56.1026496887207,
      "activations/layer7_attention_weight_max": 93.13924407958984,
      "activations/layer7_attention_weight_min": -90.13928985595703,
      "activations/layer8_attention_weight_max": 42.57383728027344,
      "activations/layer8_attention_weight_min": -40.66719055175781,
      "activations/layer9_attention_weight_max": 36.14469909667969,
      "activations/layer9_attention_weight_min": -50.02505111694336,
      "epoch": 5.78,
      "learning_rate": 0.00011384242424242423,
      "loss": 2.8283,
      "step": 99550
    },
    {
      "activations/layer0_attention_weight_max": 16.379152297973633,
      "activations/layer0_attention_weight_min": -12.335047721862793,
      "activations/layer10_attention_weight_max": 33.97016143798828,
      "activations/layer10_attention_weight_min": -31.097368240356445,
      "activations/layer11_attention_weight_max": 31.597896575927734,
      "activations/layer11_attention_weight_min": -28.79443359375,
      "activations/layer12_attention_weight_max": 28.010934829711914,
      "activations/layer12_attention_weight_min": -27.52179527282715,
      "activations/layer13_attention_weight_max": 47.738136291503906,
      "activations/layer13_attention_weight_min": -33.292762756347656,
      "activations/layer14_attention_weight_max": 63.04212188720703,
      "activations/layer14_attention_weight_min": -41.171539306640625,
      "activations/layer15_attention_weight_max": 42.52399444580078,
      "activations/layer15_attention_weight_min": -29.04970932006836,
      "activations/layer16_attention_weight_max": 42.122802734375,
      "activations/layer16_attention_weight_min": -27.756702423095703,
      "activations/layer17_attention_weight_max": 42.63417053222656,
      "activations/layer17_attention_weight_min": -25.04656410217285,
      "activations/layer18_attention_weight_max": 36.025787353515625,
      "activations/layer18_attention_weight_min": -21.014177322387695,
      "activations/layer19_attention_weight_max": 42.6111946105957,
      "activations/layer19_attention_weight_min": -31.04294204711914,
      "activations/layer1_attention_weight_max": 16.877580642700195,
      "activations/layer1_attention_weight_min": -13.60958194732666,
      "activations/layer20_attention_weight_max": 35.046199798583984,
      "activations/layer20_attention_weight_min": -23.913536071777344,
      "activations/layer21_attention_weight_max": 36.255977630615234,
      "activations/layer21_attention_weight_min": -23.086654663085938,
      "activations/layer22_attention_weight_max": 57.06539535522461,
      "activations/layer22_attention_weight_min": -29.440170288085938,
      "activations/layer23_attention_weight_max": 35.55812072753906,
      "activations/layer23_attention_weight_min": -22.021350860595703,
      "activations/layer2_attention_weight_max": 32.019920349121094,
      "activations/layer2_attention_weight_min": -31.413536071777344,
      "activations/layer3_attention_weight_max": 84.58064270019531,
      "activations/layer3_attention_weight_min": -90.61504364013672,
      "activations/layer4_attention_weight_max": 120.81507873535156,
      "activations/layer4_attention_weight_min": -121.6340103149414,
      "activations/layer5_attention_weight_max": 70.38224029541016,
      "activations/layer5_attention_weight_min": -68.49717712402344,
      "activations/layer6_attention_weight_max": 55.68208694458008,
      "activations/layer6_attention_weight_min": -53.932865142822266,
      "activations/layer7_attention_weight_max": 84.7514419555664,
      "activations/layer7_attention_weight_min": -88.2199478149414,
      "activations/layer8_attention_weight_max": 39.47859573364258,
      "activations/layer8_attention_weight_min": -38.16747283935547,
      "activations/layer9_attention_weight_max": 35.83114242553711,
      "activations/layer9_attention_weight_min": -35.921817779541016,
      "epoch": 5.79,
      "learning_rate": 0.00011382348484848484,
      "loss": 2.8427,
      "step": 99600
    },
    {
      "activations/layer0_attention_weight_max": 15.597260475158691,
      "activations/layer0_attention_weight_min": -13.137906074523926,
      "activations/layer10_attention_weight_max": 35.3022346496582,
      "activations/layer10_attention_weight_min": -31.30339813232422,
      "activations/layer11_attention_weight_max": 32.89031219482422,
      "activations/layer11_attention_weight_min": -29.19198989868164,
      "activations/layer12_attention_weight_max": 28.5482234954834,
      "activations/layer12_attention_weight_min": -29.289857864379883,
      "activations/layer13_attention_weight_max": 46.70519256591797,
      "activations/layer13_attention_weight_min": -33.259124755859375,
      "activations/layer14_attention_weight_max": 66.3459243774414,
      "activations/layer14_attention_weight_min": -52.00392150878906,
      "activations/layer15_attention_weight_max": 41.86113357543945,
      "activations/layer15_attention_weight_min": -30.22429656982422,
      "activations/layer16_attention_weight_max": 36.60841751098633,
      "activations/layer16_attention_weight_min": -28.414339065551758,
      "activations/layer17_attention_weight_max": 39.22673416137695,
      "activations/layer17_attention_weight_min": -24.871112823486328,
      "activations/layer18_attention_weight_max": 29.42019271850586,
      "activations/layer18_attention_weight_min": -20.6213436126709,
      "activations/layer19_attention_weight_max": 40.91812515258789,
      "activations/layer19_attention_weight_min": -31.646894454956055,
      "activations/layer1_attention_weight_max": 17.236515045166016,
      "activations/layer1_attention_weight_min": -14.660079002380371,
      "activations/layer20_attention_weight_max": 32.15206527709961,
      "activations/layer20_attention_weight_min": -23.138277053833008,
      "activations/layer21_attention_weight_max": 31.2146053314209,
      "activations/layer21_attention_weight_min": -22.034011840820312,
      "activations/layer22_attention_weight_max": 51.49110412597656,
      "activations/layer22_attention_weight_min": -29.892372131347656,
      "activations/layer23_attention_weight_max": 34.76251220703125,
      "activations/layer23_attention_weight_min": -22.69241714477539,
      "activations/layer2_attention_weight_max": 30.914039611816406,
      "activations/layer2_attention_weight_min": -30.530698776245117,
      "activations/layer3_attention_weight_max": 89.19239807128906,
      "activations/layer3_attention_weight_min": -91.52188110351562,
      "activations/layer4_attention_weight_max": 127.80891418457031,
      "activations/layer4_attention_weight_min": -123.1406021118164,
      "activations/layer5_attention_weight_max": 77.23995971679688,
      "activations/layer5_attention_weight_min": -71.17385864257812,
      "activations/layer6_attention_weight_max": 56.313785552978516,
      "activations/layer6_attention_weight_min": -55.9827995300293,
      "activations/layer7_attention_weight_max": 87.44226837158203,
      "activations/layer7_attention_weight_min": -91.63919067382812,
      "activations/layer8_attention_weight_max": 38.85562515258789,
      "activations/layer8_attention_weight_min": -38.71637725830078,
      "activations/layer9_attention_weight_max": 36.166229248046875,
      "activations/layer9_attention_weight_min": -33.78237533569336,
      "epoch": 5.79,
      "learning_rate": 0.00011380454545454543,
      "loss": 2.8276,
      "step": 99650
    },
    {
      "activations/layer0_attention_weight_max": 15.752593040466309,
      "activations/layer0_attention_weight_min": -11.383466720581055,
      "activations/layer10_attention_weight_max": 36.71686935424805,
      "activations/layer10_attention_weight_min": -31.08843994140625,
      "activations/layer11_attention_weight_max": 35.67637252807617,
      "activations/layer11_attention_weight_min": -29.719120025634766,
      "activations/layer12_attention_weight_max": 46.1310920715332,
      "activations/layer12_attention_weight_min": -32.10356521606445,
      "activations/layer13_attention_weight_max": 44.791751861572266,
      "activations/layer13_attention_weight_min": -31.57040023803711,
      "activations/layer14_attention_weight_max": 61.73345184326172,
      "activations/layer14_attention_weight_min": -41.77354431152344,
      "activations/layer15_attention_weight_max": 39.221614837646484,
      "activations/layer15_attention_weight_min": -30.207273483276367,
      "activations/layer16_attention_weight_max": 38.87665939331055,
      "activations/layer16_attention_weight_min": -27.227449417114258,
      "activations/layer17_attention_weight_max": 39.906219482421875,
      "activations/layer17_attention_weight_min": -24.305694580078125,
      "activations/layer18_attention_weight_max": 41.9818115234375,
      "activations/layer18_attention_weight_min": -20.96370506286621,
      "activations/layer19_attention_weight_max": 46.108787536621094,
      "activations/layer19_attention_weight_min": -29.877151489257812,
      "activations/layer1_attention_weight_max": 17.686769485473633,
      "activations/layer1_attention_weight_min": -13.09095287322998,
      "activations/layer20_attention_weight_max": 34.95987319946289,
      "activations/layer20_attention_weight_min": -25.51327133178711,
      "activations/layer21_attention_weight_max": 36.45978927612305,
      "activations/layer21_attention_weight_min": -21.912263870239258,
      "activations/layer22_attention_weight_max": 57.13212203979492,
      "activations/layer22_attention_weight_min": -27.98849868774414,
      "activations/layer23_attention_weight_max": 42.733604431152344,
      "activations/layer23_attention_weight_min": -21.695873260498047,
      "activations/layer2_attention_weight_max": 30.902563095092773,
      "activations/layer2_attention_weight_min": -28.461551666259766,
      "activations/layer3_attention_weight_max": 85.4666976928711,
      "activations/layer3_attention_weight_min": -88.28823852539062,
      "activations/layer4_attention_weight_max": 129.44747924804688,
      "activations/layer4_attention_weight_min": -122.01683807373047,
      "activations/layer5_attention_weight_max": 74.6591567993164,
      "activations/layer5_attention_weight_min": -69.46469116210938,
      "activations/layer6_attention_weight_max": 53.77909469604492,
      "activations/layer6_attention_weight_min": -52.5837516784668,
      "activations/layer7_attention_weight_max": 88.08663177490234,
      "activations/layer7_attention_weight_min": -79.34380340576172,
      "activations/layer8_attention_weight_max": 41.55266189575195,
      "activations/layer8_attention_weight_min": -38.784664154052734,
      "activations/layer9_attention_weight_max": 38.105892181396484,
      "activations/layer9_attention_weight_min": -36.07420349121094,
      "epoch": 5.79,
      "learning_rate": 0.00011378560606060605,
      "loss": 2.8512,
      "step": 99700
    },
    {
      "activations/layer0_attention_weight_max": 15.884578704833984,
      "activations/layer0_attention_weight_min": -12.545760154724121,
      "activations/layer10_attention_weight_max": 29.566268920898438,
      "activations/layer10_attention_weight_min": -29.549808502197266,
      "activations/layer11_attention_weight_max": 28.35576629638672,
      "activations/layer11_attention_weight_min": -28.193866729736328,
      "activations/layer12_attention_weight_max": 28.396085739135742,
      "activations/layer12_attention_weight_min": -26.33762550354004,
      "activations/layer13_attention_weight_max": 38.151283264160156,
      "activations/layer13_attention_weight_min": -30.70491600036621,
      "activations/layer14_attention_weight_max": 49.219017028808594,
      "activations/layer14_attention_weight_min": -46.9482536315918,
      "activations/layer15_attention_weight_max": 32.41334915161133,
      "activations/layer15_attention_weight_min": -28.248079299926758,
      "activations/layer16_attention_weight_max": 30.533504486083984,
      "activations/layer16_attention_weight_min": -27.38260841369629,
      "activations/layer17_attention_weight_max": 34.64994812011719,
      "activations/layer17_attention_weight_min": -26.379558563232422,
      "activations/layer18_attention_weight_max": 33.77099609375,
      "activations/layer18_attention_weight_min": -20.885480880737305,
      "activations/layer19_attention_weight_max": 33.987606048583984,
      "activations/layer19_attention_weight_min": -28.708127975463867,
      "activations/layer1_attention_weight_max": 17.26426887512207,
      "activations/layer1_attention_weight_min": -12.181674003601074,
      "activations/layer20_attention_weight_max": 31.287391662597656,
      "activations/layer20_attention_weight_min": -24.13664436340332,
      "activations/layer21_attention_weight_max": 30.938507080078125,
      "activations/layer21_attention_weight_min": -20.17624855041504,
      "activations/layer22_attention_weight_max": 49.943939208984375,
      "activations/layer22_attention_weight_min": -29.245820999145508,
      "activations/layer23_attention_weight_max": 33.01248550415039,
      "activations/layer23_attention_weight_min": -23.1680965423584,
      "activations/layer2_attention_weight_max": 31.27603530883789,
      "activations/layer2_attention_weight_min": -30.248939514160156,
      "activations/layer3_attention_weight_max": 89.6765365600586,
      "activations/layer3_attention_weight_min": -89.73461151123047,
      "activations/layer4_attention_weight_max": 126.68761444091797,
      "activations/layer4_attention_weight_min": -124.5195541381836,
      "activations/layer5_attention_weight_max": 73.54442596435547,
      "activations/layer5_attention_weight_min": -73.08187103271484,
      "activations/layer6_attention_weight_max": 51.50038528442383,
      "activations/layer6_attention_weight_min": -54.77743911743164,
      "activations/layer7_attention_weight_max": 81.38761901855469,
      "activations/layer7_attention_weight_min": -84.65794372558594,
      "activations/layer8_attention_weight_max": 36.99684143066406,
      "activations/layer8_attention_weight_min": -39.56208801269531,
      "activations/layer9_attention_weight_max": 34.33375549316406,
      "activations/layer9_attention_weight_min": -32.36713409423828,
      "epoch": 5.8,
      "learning_rate": 0.00011376704545454544,
      "loss": 2.8309,
      "step": 99750
    },
    {
      "activations/layer0_attention_weight_max": 15.998741149902344,
      "activations/layer0_attention_weight_min": -12.617742538452148,
      "activations/layer10_attention_weight_max": 34.79344177246094,
      "activations/layer10_attention_weight_min": -32.95111846923828,
      "activations/layer11_attention_weight_max": 33.38431930541992,
      "activations/layer11_attention_weight_min": -31.140945434570312,
      "activations/layer12_attention_weight_max": 27.307268142700195,
      "activations/layer12_attention_weight_min": -24.97825813293457,
      "activations/layer13_attention_weight_max": 45.47065734863281,
      "activations/layer13_attention_weight_min": -34.81399154663086,
      "activations/layer14_attention_weight_max": 80.04022979736328,
      "activations/layer14_attention_weight_min": -54.42953872680664,
      "activations/layer15_attention_weight_max": 43.64527130126953,
      "activations/layer15_attention_weight_min": -29.604326248168945,
      "activations/layer16_attention_weight_max": 40.98684310913086,
      "activations/layer16_attention_weight_min": -28.2374267578125,
      "activations/layer17_attention_weight_max": 42.14631652832031,
      "activations/layer17_attention_weight_min": -24.436185836791992,
      "activations/layer18_attention_weight_max": 37.199745178222656,
      "activations/layer18_attention_weight_min": -20.149450302124023,
      "activations/layer19_attention_weight_max": 41.19716262817383,
      "activations/layer19_attention_weight_min": -29.773130416870117,
      "activations/layer1_attention_weight_max": 17.722288131713867,
      "activations/layer1_attention_weight_min": -14.882941246032715,
      "activations/layer20_attention_weight_max": 32.981910705566406,
      "activations/layer20_attention_weight_min": -23.820310592651367,
      "activations/layer21_attention_weight_max": 32.74198913574219,
      "activations/layer21_attention_weight_min": -21.371301651000977,
      "activations/layer22_attention_weight_max": 51.5945930480957,
      "activations/layer22_attention_weight_min": -29.317625045776367,
      "activations/layer23_attention_weight_max": 34.71540832519531,
      "activations/layer23_attention_weight_min": -22.36504364013672,
      "activations/layer2_attention_weight_max": 30.546733856201172,
      "activations/layer2_attention_weight_min": -28.738452911376953,
      "activations/layer3_attention_weight_max": 90.08727264404297,
      "activations/layer3_attention_weight_min": -86.6513442993164,
      "activations/layer4_attention_weight_max": 128.2639617919922,
      "activations/layer4_attention_weight_min": -120.3766860961914,
      "activations/layer5_attention_weight_max": 76.0401611328125,
      "activations/layer5_attention_weight_min": -71.2043685913086,
      "activations/layer6_attention_weight_max": 58.73444747924805,
      "activations/layer6_attention_weight_min": -55.4517707824707,
      "activations/layer7_attention_weight_max": 92.62364196777344,
      "activations/layer7_attention_weight_min": -92.50891876220703,
      "activations/layer8_attention_weight_max": 39.70349884033203,
      "activations/layer8_attention_weight_min": -42.24554443359375,
      "activations/layer9_attention_weight_max": 37.85273361206055,
      "activations/layer9_attention_weight_min": -37.25346755981445,
      "epoch": 5.8,
      "learning_rate": 0.00011374810606060605,
      "loss": 2.8438,
      "step": 99800
    },
    {
      "activations/layer0_attention_weight_max": 15.49292278289795,
      "activations/layer0_attention_weight_min": -11.727067947387695,
      "activations/layer10_attention_weight_max": 33.44121170043945,
      "activations/layer10_attention_weight_min": -30.499174118041992,
      "activations/layer11_attention_weight_max": 33.31189727783203,
      "activations/layer11_attention_weight_min": -27.115507125854492,
      "activations/layer12_attention_weight_max": 26.982662200927734,
      "activations/layer12_attention_weight_min": -25.290523529052734,
      "activations/layer13_attention_weight_max": 44.23115158081055,
      "activations/layer13_attention_weight_min": -31.390581130981445,
      "activations/layer14_attention_weight_max": 55.06203842163086,
      "activations/layer14_attention_weight_min": -43.23026657104492,
      "activations/layer15_attention_weight_max": 39.077430725097656,
      "activations/layer15_attention_weight_min": -26.895437240600586,
      "activations/layer16_attention_weight_max": 35.91729736328125,
      "activations/layer16_attention_weight_min": -27.38815689086914,
      "activations/layer17_attention_weight_max": 38.63386917114258,
      "activations/layer17_attention_weight_min": -22.956148147583008,
      "activations/layer18_attention_weight_max": 32.035125732421875,
      "activations/layer18_attention_weight_min": -19.595245361328125,
      "activations/layer19_attention_weight_max": 43.94287109375,
      "activations/layer19_attention_weight_min": -28.598438262939453,
      "activations/layer1_attention_weight_max": 17.520523071289062,
      "activations/layer1_attention_weight_min": -14.018637657165527,
      "activations/layer20_attention_weight_max": 35.83512496948242,
      "activations/layer20_attention_weight_min": -21.839963912963867,
      "activations/layer21_attention_weight_max": 32.75932693481445,
      "activations/layer21_attention_weight_min": -19.780385971069336,
      "activations/layer22_attention_weight_max": 49.9698600769043,
      "activations/layer22_attention_weight_min": -28.908248901367188,
      "activations/layer23_attention_weight_max": 38.51559829711914,
      "activations/layer23_attention_weight_min": -21.178775787353516,
      "activations/layer2_attention_weight_max": 31.836423873901367,
      "activations/layer2_attention_weight_min": -30.235774993896484,
      "activations/layer3_attention_weight_max": 85.96346282958984,
      "activations/layer3_attention_weight_min": -89.82173156738281,
      "activations/layer4_attention_weight_max": 118.80962371826172,
      "activations/layer4_attention_weight_min": -124.9548110961914,
      "activations/layer5_attention_weight_max": 69.93083953857422,
      "activations/layer5_attention_weight_min": -70.3552474975586,
      "activations/layer6_attention_weight_max": 53.739376068115234,
      "activations/layer6_attention_weight_min": -56.82942199707031,
      "activations/layer7_attention_weight_max": 84.65057373046875,
      "activations/layer7_attention_weight_min": -84.45704650878906,
      "activations/layer8_attention_weight_max": 40.51958465576172,
      "activations/layer8_attention_weight_min": -38.76236343383789,
      "activations/layer9_attention_weight_max": 35.4093132019043,
      "activations/layer9_attention_weight_min": -33.94791793823242,
      "epoch": 5.8,
      "learning_rate": 0.00011372916666666666,
      "loss": 2.8325,
      "step": 99850
    },
    {
      "activations/layer0_attention_weight_max": 15.5839262008667,
      "activations/layer0_attention_weight_min": -12.676295280456543,
      "activations/layer10_attention_weight_max": 35.50041580200195,
      "activations/layer10_attention_weight_min": -31.6978759765625,
      "activations/layer11_attention_weight_max": 34.36244201660156,
      "activations/layer11_attention_weight_min": -27.146642684936523,
      "activations/layer12_attention_weight_max": 25.873533248901367,
      "activations/layer12_attention_weight_min": -26.126127243041992,
      "activations/layer13_attention_weight_max": 44.49345397949219,
      "activations/layer13_attention_weight_min": -31.812061309814453,
      "activations/layer14_attention_weight_max": 64.0902099609375,
      "activations/layer14_attention_weight_min": -45.89040756225586,
      "activations/layer15_attention_weight_max": 39.2419319152832,
      "activations/layer15_attention_weight_min": -29.142980575561523,
      "activations/layer16_attention_weight_max": 38.91244888305664,
      "activations/layer16_attention_weight_min": -27.932098388671875,
      "activations/layer17_attention_weight_max": 43.18277359008789,
      "activations/layer17_attention_weight_min": -23.648420333862305,
      "activations/layer18_attention_weight_max": 33.25133514404297,
      "activations/layer18_attention_weight_min": -21.591806411743164,
      "activations/layer19_attention_weight_max": 42.762882232666016,
      "activations/layer19_attention_weight_min": -29.313627243041992,
      "activations/layer1_attention_weight_max": 17.533615112304688,
      "activations/layer1_attention_weight_min": -15.416951179504395,
      "activations/layer20_attention_weight_max": 39.269676208496094,
      "activations/layer20_attention_weight_min": -23.069772720336914,
      "activations/layer21_attention_weight_max": 32.932342529296875,
      "activations/layer21_attention_weight_min": -20.840492248535156,
      "activations/layer22_attention_weight_max": 53.3104133605957,
      "activations/layer22_attention_weight_min": -27.71573829650879,
      "activations/layer23_attention_weight_max": 36.271522521972656,
      "activations/layer23_attention_weight_min": -22.297382354736328,
      "activations/layer2_attention_weight_max": 32.12699890136719,
      "activations/layer2_attention_weight_min": -31.241310119628906,
      "activations/layer3_attention_weight_max": 90.81230163574219,
      "activations/layer3_attention_weight_min": -95.90062713623047,
      "activations/layer4_attention_weight_max": 127.6511459350586,
      "activations/layer4_attention_weight_min": -133.61033630371094,
      "activations/layer5_attention_weight_max": 73.61083984375,
      "activations/layer5_attention_weight_min": -73.83547973632812,
      "activations/layer6_attention_weight_max": 55.55161666870117,
      "activations/layer6_attention_weight_min": -55.18915939331055,
      "activations/layer7_attention_weight_max": 87.13545227050781,
      "activations/layer7_attention_weight_min": -83.95946502685547,
      "activations/layer8_attention_weight_max": 41.88687515258789,
      "activations/layer8_attention_weight_min": -40.40486526489258,
      "activations/layer9_attention_weight_max": 37.45402145385742,
      "activations/layer9_attention_weight_min": -34.521392822265625,
      "epoch": 5.8,
      "learning_rate": 0.00011371022727272726,
      "loss": 2.8261,
      "step": 99900
    },
    {
      "activations/layer0_attention_weight_max": 16.40777587890625,
      "activations/layer0_attention_weight_min": -10.987977981567383,
      "activations/layer10_attention_weight_max": 32.76448059082031,
      "activations/layer10_attention_weight_min": -30.355764389038086,
      "activations/layer11_attention_weight_max": 33.09458923339844,
      "activations/layer11_attention_weight_min": -27.383955001831055,
      "activations/layer12_attention_weight_max": 29.470523834228516,
      "activations/layer12_attention_weight_min": -36.75030517578125,
      "activations/layer13_attention_weight_max": 37.4434928894043,
      "activations/layer13_attention_weight_min": -32.303646087646484,
      "activations/layer14_attention_weight_max": 55.80107116699219,
      "activations/layer14_attention_weight_min": -44.72892379760742,
      "activations/layer15_attention_weight_max": 36.87453842163086,
      "activations/layer15_attention_weight_min": -27.808744430541992,
      "activations/layer16_attention_weight_max": 39.06496810913086,
      "activations/layer16_attention_weight_min": -27.91961097717285,
      "activations/layer17_attention_weight_max": 37.16196823120117,
      "activations/layer17_attention_weight_min": -24.096914291381836,
      "activations/layer18_attention_weight_max": 33.104671478271484,
      "activations/layer18_attention_weight_min": -20.8609561920166,
      "activations/layer19_attention_weight_max": 40.401798248291016,
      "activations/layer19_attention_weight_min": -30.960596084594727,
      "activations/layer1_attention_weight_max": 16.99068260192871,
      "activations/layer1_attention_weight_min": -16.221614837646484,
      "activations/layer20_attention_weight_max": 34.04068374633789,
      "activations/layer20_attention_weight_min": -23.681737899780273,
      "activations/layer21_attention_weight_max": 32.08769607543945,
      "activations/layer21_attention_weight_min": -19.76932144165039,
      "activations/layer22_attention_weight_max": 50.92112731933594,
      "activations/layer22_attention_weight_min": -31.233659744262695,
      "activations/layer23_attention_weight_max": 39.3407096862793,
      "activations/layer23_attention_weight_min": -24.888134002685547,
      "activations/layer2_attention_weight_max": 32.82572937011719,
      "activations/layer2_attention_weight_min": -31.412097930908203,
      "activations/layer3_attention_weight_max": 94.92488861083984,
      "activations/layer3_attention_weight_min": -95.49231719970703,
      "activations/layer4_attention_weight_max": 127.35359954833984,
      "activations/layer4_attention_weight_min": -128.15626525878906,
      "activations/layer5_attention_weight_max": 75.08757781982422,
      "activations/layer5_attention_weight_min": -72.49436950683594,
      "activations/layer6_attention_weight_max": 54.77461624145508,
      "activations/layer6_attention_weight_min": -56.899261474609375,
      "activations/layer7_attention_weight_max": 89.96715545654297,
      "activations/layer7_attention_weight_min": -87.22785949707031,
      "activations/layer8_attention_weight_max": 40.50130081176758,
      "activations/layer8_attention_weight_min": -39.932106018066406,
      "activations/layer9_attention_weight_max": 36.693115234375,
      "activations/layer9_attention_weight_min": -34.64651870727539,
      "epoch": 5.81,
      "learning_rate": 0.00011369128787878787,
      "loss": 2.8298,
      "step": 99950
    },
    {
      "activations/layer0_attention_weight_max": 17.109575271606445,
      "activations/layer0_attention_weight_min": -11.362504959106445,
      "activations/layer10_attention_weight_max": 33.78300476074219,
      "activations/layer10_attention_weight_min": -30.417428970336914,
      "activations/layer11_attention_weight_max": 33.31153106689453,
      "activations/layer11_attention_weight_min": -28.928081512451172,
      "activations/layer12_attention_weight_max": 36.59333419799805,
      "activations/layer12_attention_weight_min": -28.30442237854004,
      "activations/layer13_attention_weight_max": 46.746681213378906,
      "activations/layer13_attention_weight_min": -35.06932830810547,
      "activations/layer14_attention_weight_max": 59.401405334472656,
      "activations/layer14_attention_weight_min": -45.06660461425781,
      "activations/layer15_attention_weight_max": 43.38982009887695,
      "activations/layer15_attention_weight_min": -33.397647857666016,
      "activations/layer16_attention_weight_max": 37.440650939941406,
      "activations/layer16_attention_weight_min": -27.202037811279297,
      "activations/layer17_attention_weight_max": 41.547035217285156,
      "activations/layer17_attention_weight_min": -26.749065399169922,
      "activations/layer18_attention_weight_max": 36.128353118896484,
      "activations/layer18_attention_weight_min": -21.261648178100586,
      "activations/layer19_attention_weight_max": 41.02212905883789,
      "activations/layer19_attention_weight_min": -32.5518913269043,
      "activations/layer1_attention_weight_max": 17.194272994995117,
      "activations/layer1_attention_weight_min": -13.252732276916504,
      "activations/layer20_attention_weight_max": 37.41037368774414,
      "activations/layer20_attention_weight_min": -23.746173858642578,
      "activations/layer21_attention_weight_max": 38.370933532714844,
      "activations/layer21_attention_weight_min": -21.8067569732666,
      "activations/layer22_attention_weight_max": 56.17674255371094,
      "activations/layer22_attention_weight_min": -27.9298095703125,
      "activations/layer23_attention_weight_max": 38.128726959228516,
      "activations/layer23_attention_weight_min": -23.036466598510742,
      "activations/layer2_attention_weight_max": 29.841053009033203,
      "activations/layer2_attention_weight_min": -30.108787536621094,
      "activations/layer3_attention_weight_max": 83.37931060791016,
      "activations/layer3_attention_weight_min": -87.60585021972656,
      "activations/layer4_attention_weight_max": 118.5638656616211,
      "activations/layer4_attention_weight_min": -119.06825256347656,
      "activations/layer5_attention_weight_max": 71.46288299560547,
      "activations/layer5_attention_weight_min": -72.30606079101562,
      "activations/layer6_attention_weight_max": 52.46245574951172,
      "activations/layer6_attention_weight_min": -54.448421478271484,
      "activations/layer7_attention_weight_max": 82.79755401611328,
      "activations/layer7_attention_weight_min": -85.29032135009766,
      "activations/layer8_attention_weight_max": 39.945045471191406,
      "activations/layer8_attention_weight_min": -40.795536041259766,
      "activations/layer9_attention_weight_max": 35.677467346191406,
      "activations/layer9_attention_weight_min": -35.18500518798828,
      "epoch": 5.81,
      "learning_rate": 0.00011367234848484847,
      "loss": 2.8484,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.4961,
      "eval_samples_per_second": 505.407,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.4961,
      "eval_openwebtext_samples_per_second": 505.407,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_wikitext_loss": 3.021484375,
      "eval_wikitext_ppl": 20.521731031400577,
      "eval_wikitext_runtime": 1.9493,
      "eval_wikitext_samples_per_second": 233.936,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_lambada_loss": 2.662109375,
      "eval_lambada_ppl": 14.326477161274006,
      "eval_lambada_runtime": 9.526,
      "eval_lambada_samples_per_second": 511.13,
      "step": 100000
    },
    {
      "activations/layer0_attention_weight_max": 15.987674713134766,
      "activations/layer0_attention_weight_min": -13.02374267578125,
      "activations/layer10_attention_weight_max": 38.07137680053711,
      "activations/layer10_attention_weight_min": -32.718910217285156,
      "activations/layer11_attention_weight_max": 36.74631881713867,
      "activations/layer11_attention_weight_min": -29.83041000366211,
      "activations/layer12_attention_weight_max": 26.518991470336914,
      "activations/layer12_attention_weight_min": -24.54425048828125,
      "activations/layer13_attention_weight_max": 50.61643981933594,
      "activations/layer13_attention_weight_min": -33.81644821166992,
      "activations/layer14_attention_weight_max": 66.83464050292969,
      "activations/layer14_attention_weight_min": -43.21466064453125,
      "activations/layer15_attention_weight_max": 45.08779525756836,
      "activations/layer15_attention_weight_min": -30.842039108276367,
      "activations/layer16_attention_weight_max": 40.08454513549805,
      "activations/layer16_attention_weight_min": -28.481769561767578,
      "activations/layer17_attention_weight_max": 44.80324935913086,
      "activations/layer17_attention_weight_min": -27.216161727905273,
      "activations/layer18_attention_weight_max": 32.23467254638672,
      "activations/layer18_attention_weight_min": -21.090944290161133,
      "activations/layer19_attention_weight_max": 46.21073913574219,
      "activations/layer19_attention_weight_min": -28.127904891967773,
      "activations/layer1_attention_weight_max": 17.017086029052734,
      "activations/layer1_attention_weight_min": -15.87554931640625,
      "activations/layer20_attention_weight_max": 37.21237564086914,
      "activations/layer20_attention_weight_min": -21.836610794067383,
      "activations/layer21_attention_weight_max": 37.30030059814453,
      "activations/layer21_attention_weight_min": -20.53204345703125,
      "activations/layer22_attention_weight_max": 57.09931945800781,
      "activations/layer22_attention_weight_min": -26.10944175720215,
      "activations/layer23_attention_weight_max": 39.43323516845703,
      "activations/layer23_attention_weight_min": -20.762283325195312,
      "activations/layer2_attention_weight_max": 31.453777313232422,
      "activations/layer2_attention_weight_min": -31.911657333374023,
      "activations/layer3_attention_weight_max": 92.21865844726562,
      "activations/layer3_attention_weight_min": -92.3226547241211,
      "activations/layer4_attention_weight_max": 130.70677185058594,
      "activations/layer4_attention_weight_min": -128.71035766601562,
      "activations/layer5_attention_weight_max": 74.30931091308594,
      "activations/layer5_attention_weight_min": -76.17913818359375,
      "activations/layer6_attention_weight_max": 55.78742218017578,
      "activations/layer6_attention_weight_min": -63.07829666137695,
      "activations/layer7_attention_weight_max": 91.47532653808594,
      "activations/layer7_attention_weight_min": -97.4222412109375,
      "activations/layer8_attention_weight_max": 42.89265441894531,
      "activations/layer8_attention_weight_min": -44.44121170043945,
      "activations/layer9_attention_weight_max": 39.082115173339844,
      "activations/layer9_attention_weight_min": -35.55489730834961,
      "epoch": 5.81,
      "learning_rate": 0.00011365340909090907,
      "loss": 2.849,
      "step": 100050
    },
    {
      "activations/layer0_attention_weight_max": 15.13931655883789,
      "activations/layer0_attention_weight_min": -11.329194068908691,
      "activations/layer10_attention_weight_max": 43.65390396118164,
      "activations/layer10_attention_weight_min": -39.38812255859375,
      "activations/layer11_attention_weight_max": 45.850006103515625,
      "activations/layer11_attention_weight_min": -35.730979919433594,
      "activations/layer12_attention_weight_max": 30.2841854095459,
      "activations/layer12_attention_weight_min": -31.211750030517578,
      "activations/layer13_attention_weight_max": 64.36951446533203,
      "activations/layer13_attention_weight_min": -35.643646240234375,
      "activations/layer14_attention_weight_max": 86.98345184326172,
      "activations/layer14_attention_weight_min": -58.702392578125,
      "activations/layer15_attention_weight_max": 56.16707992553711,
      "activations/layer15_attention_weight_min": -33.151641845703125,
      "activations/layer16_attention_weight_max": 42.74513626098633,
      "activations/layer16_attention_weight_min": -28.1634578704834,
      "activations/layer17_attention_weight_max": 44.76453399658203,
      "activations/layer17_attention_weight_min": -26.556171417236328,
      "activations/layer18_attention_weight_max": 36.092777252197266,
      "activations/layer18_attention_weight_min": -23.038156509399414,
      "activations/layer19_attention_weight_max": 49.2435417175293,
      "activations/layer19_attention_weight_min": -29.4163761138916,
      "activations/layer1_attention_weight_max": 16.852319717407227,
      "activations/layer1_attention_weight_min": -14.849721908569336,
      "activations/layer20_attention_weight_max": 39.03713607788086,
      "activations/layer20_attention_weight_min": -21.902433395385742,
      "activations/layer21_attention_weight_max": 29.505741119384766,
      "activations/layer21_attention_weight_min": -22.47212028503418,
      "activations/layer22_attention_weight_max": 53.28199005126953,
      "activations/layer22_attention_weight_min": -27.99197006225586,
      "activations/layer23_attention_weight_max": 37.310455322265625,
      "activations/layer23_attention_weight_min": -19.343891143798828,
      "activations/layer2_attention_weight_max": 36.03718566894531,
      "activations/layer2_attention_weight_min": -33.489871978759766,
      "activations/layer3_attention_weight_max": 100.34056091308594,
      "activations/layer3_attention_weight_min": -97.05037689208984,
      "activations/layer4_attention_weight_max": 141.9755401611328,
      "activations/layer4_attention_weight_min": -130.99240112304688,
      "activations/layer5_attention_weight_max": 74.87379455566406,
      "activations/layer5_attention_weight_min": -73.06857299804688,
      "activations/layer6_attention_weight_max": 56.12853240966797,
      "activations/layer6_attention_weight_min": -54.40607452392578,
      "activations/layer7_attention_weight_max": 96.89937591552734,
      "activations/layer7_attention_weight_min": -92.42315673828125,
      "activations/layer8_attention_weight_max": 47.62434005737305,
      "activations/layer8_attention_weight_min": -44.95973587036133,
      "activations/layer9_attention_weight_max": 44.537113189697266,
      "activations/layer9_attention_weight_min": -39.53322982788086,
      "epoch": 5.82,
      "learning_rate": 0.00011363446969696969,
      "loss": 2.8379,
      "step": 100100
    },
    {
      "activations/layer0_attention_weight_max": 16.402385711669922,
      "activations/layer0_attention_weight_min": -12.661369323730469,
      "activations/layer10_attention_weight_max": 31.461454391479492,
      "activations/layer10_attention_weight_min": -29.328414916992188,
      "activations/layer11_attention_weight_max": 29.897747039794922,
      "activations/layer11_attention_weight_min": -28.852848052978516,
      "activations/layer12_attention_weight_max": 27.776504516601562,
      "activations/layer12_attention_weight_min": -27.565763473510742,
      "activations/layer13_attention_weight_max": 47.17045211791992,
      "activations/layer13_attention_weight_min": -35.495506286621094,
      "activations/layer14_attention_weight_max": 53.39683151245117,
      "activations/layer14_attention_weight_min": -44.8587760925293,
      "activations/layer15_attention_weight_max": 38.18998336791992,
      "activations/layer15_attention_weight_min": -31.0859432220459,
      "activations/layer16_attention_weight_max": 33.16969680786133,
      "activations/layer16_attention_weight_min": -28.68695831298828,
      "activations/layer17_attention_weight_max": 37.22198486328125,
      "activations/layer17_attention_weight_min": -26.071269989013672,
      "activations/layer18_attention_weight_max": 34.7470703125,
      "activations/layer18_attention_weight_min": -20.62959861755371,
      "activations/layer19_attention_weight_max": 36.37715148925781,
      "activations/layer19_attention_weight_min": -31.841684341430664,
      "activations/layer1_attention_weight_max": 17.830106735229492,
      "activations/layer1_attention_weight_min": -15.077169418334961,
      "activations/layer20_attention_weight_max": 29.449329376220703,
      "activations/layer20_attention_weight_min": -26.15666961669922,
      "activations/layer21_attention_weight_max": 31.088762283325195,
      "activations/layer21_attention_weight_min": -26.565349578857422,
      "activations/layer22_attention_weight_max": 47.865814208984375,
      "activations/layer22_attention_weight_min": -31.13349723815918,
      "activations/layer23_attention_weight_max": 35.17948913574219,
      "activations/layer23_attention_weight_min": -22.125028610229492,
      "activations/layer2_attention_weight_max": 31.542814254760742,
      "activations/layer2_attention_weight_min": -31.518125534057617,
      "activations/layer3_attention_weight_max": 88.5824203491211,
      "activations/layer3_attention_weight_min": -94.00343322753906,
      "activations/layer4_attention_weight_max": 123.08454895019531,
      "activations/layer4_attention_weight_min": -125.6173324584961,
      "activations/layer5_attention_weight_max": 72.25655364990234,
      "activations/layer5_attention_weight_min": -72.54312896728516,
      "activations/layer6_attention_weight_max": 54.068477630615234,
      "activations/layer6_attention_weight_min": -55.07594299316406,
      "activations/layer7_attention_weight_max": 80.3873519897461,
      "activations/layer7_attention_weight_min": -84.30052947998047,
      "activations/layer8_attention_weight_max": 39.637367248535156,
      "activations/layer8_attention_weight_min": -37.831993103027344,
      "activations/layer9_attention_weight_max": 34.43880844116211,
      "activations/layer9_attention_weight_min": -31.81776237487793,
      "epoch": 5.82,
      "learning_rate": 0.00011361553030303029,
      "loss": 2.8415,
      "step": 100150
    },
    {
      "activations/layer0_attention_weight_max": 16.51561737060547,
      "activations/layer0_attention_weight_min": -12.072402954101562,
      "activations/layer10_attention_weight_max": 33.070159912109375,
      "activations/layer10_attention_weight_min": -30.33683967590332,
      "activations/layer11_attention_weight_max": 32.391483306884766,
      "activations/layer11_attention_weight_min": -28.732280731201172,
      "activations/layer12_attention_weight_max": 22.143259048461914,
      "activations/layer12_attention_weight_min": -26.25244903564453,
      "activations/layer13_attention_weight_max": 42.3922119140625,
      "activations/layer13_attention_weight_min": -30.68844985961914,
      "activations/layer14_attention_weight_max": 56.313968658447266,
      "activations/layer14_attention_weight_min": -45.11777877807617,
      "activations/layer15_attention_weight_max": 34.677467346191406,
      "activations/layer15_attention_weight_min": -27.43008041381836,
      "activations/layer16_attention_weight_max": 35.06332778930664,
      "activations/layer16_attention_weight_min": -28.56450653076172,
      "activations/layer17_attention_weight_max": 37.950592041015625,
      "activations/layer17_attention_weight_min": -23.716360092163086,
      "activations/layer18_attention_weight_max": 34.72047805786133,
      "activations/layer18_attention_weight_min": -20.041379928588867,
      "activations/layer19_attention_weight_max": 37.74427032470703,
      "activations/layer19_attention_weight_min": -30.285959243774414,
      "activations/layer1_attention_weight_max": 16.200557708740234,
      "activations/layer1_attention_weight_min": -15.231858253479004,
      "activations/layer20_attention_weight_max": 30.63287353515625,
      "activations/layer20_attention_weight_min": -23.9318904876709,
      "activations/layer21_attention_weight_max": 36.06886672973633,
      "activations/layer21_attention_weight_min": -22.438138961791992,
      "activations/layer22_attention_weight_max": 51.527992248535156,
      "activations/layer22_attention_weight_min": -29.62578010559082,
      "activations/layer23_attention_weight_max": 35.04920959472656,
      "activations/layer23_attention_weight_min": -22.576778411865234,
      "activations/layer2_attention_weight_max": 32.36072540283203,
      "activations/layer2_attention_weight_min": -33.80461502075195,
      "activations/layer3_attention_weight_max": 90.87471008300781,
      "activations/layer3_attention_weight_min": -93.9982681274414,
      "activations/layer4_attention_weight_max": 121.80934143066406,
      "activations/layer4_attention_weight_min": -127.23811340332031,
      "activations/layer5_attention_weight_max": 72.7642822265625,
      "activations/layer5_attention_weight_min": -75.4854507446289,
      "activations/layer6_attention_weight_max": 54.548484802246094,
      "activations/layer6_attention_weight_min": -57.69258499145508,
      "activations/layer7_attention_weight_max": 84.32353210449219,
      "activations/layer7_attention_weight_min": -87.73220825195312,
      "activations/layer8_attention_weight_max": 39.38533401489258,
      "activations/layer8_attention_weight_min": -39.49152374267578,
      "activations/layer9_attention_weight_max": 37.5723876953125,
      "activations/layer9_attention_weight_min": -33.79622268676758,
      "epoch": 5.82,
      "learning_rate": 0.0001135965909090909,
      "loss": 2.8418,
      "step": 100200
    },
    {
      "activations/layer0_attention_weight_max": 14.670409202575684,
      "activations/layer0_attention_weight_min": -12.565181732177734,
      "activations/layer10_attention_weight_max": 39.066802978515625,
      "activations/layer10_attention_weight_min": -33.09857177734375,
      "activations/layer11_attention_weight_max": 33.49246597290039,
      "activations/layer11_attention_weight_min": -31.879989624023438,
      "activations/layer12_attention_weight_max": 43.43238067626953,
      "activations/layer12_attention_weight_min": -24.843971252441406,
      "activations/layer13_attention_weight_max": 46.45368194580078,
      "activations/layer13_attention_weight_min": -32.52175521850586,
      "activations/layer14_attention_weight_max": 60.93549346923828,
      "activations/layer14_attention_weight_min": -44.51530075073242,
      "activations/layer15_attention_weight_max": 40.74188995361328,
      "activations/layer15_attention_weight_min": -29.366870880126953,
      "activations/layer16_attention_weight_max": 37.76408767700195,
      "activations/layer16_attention_weight_min": -27.942716598510742,
      "activations/layer17_attention_weight_max": 43.10481643676758,
      "activations/layer17_attention_weight_min": -24.486347198486328,
      "activations/layer18_attention_weight_max": 32.26347732543945,
      "activations/layer18_attention_weight_min": -20.095380783081055,
      "activations/layer19_attention_weight_max": 44.47045135498047,
      "activations/layer19_attention_weight_min": -29.40130615234375,
      "activations/layer1_attention_weight_max": 17.30653953552246,
      "activations/layer1_attention_weight_min": -14.872145652770996,
      "activations/layer20_attention_weight_max": 36.160301208496094,
      "activations/layer20_attention_weight_min": -22.57523536682129,
      "activations/layer21_attention_weight_max": 31.896963119506836,
      "activations/layer21_attention_weight_min": -22.83447265625,
      "activations/layer22_attention_weight_max": 55.169822692871094,
      "activations/layer22_attention_weight_min": -30.32047462463379,
      "activations/layer23_attention_weight_max": 37.848915100097656,
      "activations/layer23_attention_weight_min": -21.18987274169922,
      "activations/layer2_attention_weight_max": 31.033985137939453,
      "activations/layer2_attention_weight_min": -30.029998779296875,
      "activations/layer3_attention_weight_max": 87.18956756591797,
      "activations/layer3_attention_weight_min": -95.62751770019531,
      "activations/layer4_attention_weight_max": 124.02174377441406,
      "activations/layer4_attention_weight_min": -122.73250579833984,
      "activations/layer5_attention_weight_max": 72.14971160888672,
      "activations/layer5_attention_weight_min": -73.4698257446289,
      "activations/layer6_attention_weight_max": 56.69021987915039,
      "activations/layer6_attention_weight_min": -55.237144470214844,
      "activations/layer7_attention_weight_max": 95.22663879394531,
      "activations/layer7_attention_weight_min": -87.66352844238281,
      "activations/layer8_attention_weight_max": 41.60478973388672,
      "activations/layer8_attention_weight_min": -43.4588737487793,
      "activations/layer9_attention_weight_max": 40.654335021972656,
      "activations/layer9_attention_weight_min": -36.36402130126953,
      "epoch": 5.83,
      "learning_rate": 0.00011357765151515151,
      "loss": 2.8505,
      "step": 100250
    },
    {
      "activations/layer0_attention_weight_max": 16.33106231689453,
      "activations/layer0_attention_weight_min": -11.534430503845215,
      "activations/layer10_attention_weight_max": 34.05282211303711,
      "activations/layer10_attention_weight_min": -31.652923583984375,
      "activations/layer11_attention_weight_max": 32.92406463623047,
      "activations/layer11_attention_weight_min": -29.37373924255371,
      "activations/layer12_attention_weight_max": 31.541135787963867,
      "activations/layer12_attention_weight_min": -22.752174377441406,
      "activations/layer13_attention_weight_max": 44.2808723449707,
      "activations/layer13_attention_weight_min": -34.03105926513672,
      "activations/layer14_attention_weight_max": 63.79592514038086,
      "activations/layer14_attention_weight_min": -48.02760696411133,
      "activations/layer15_attention_weight_max": 46.3338737487793,
      "activations/layer15_attention_weight_min": -29.79566764831543,
      "activations/layer16_attention_weight_max": 40.6790657043457,
      "activations/layer16_attention_weight_min": -28.569433212280273,
      "activations/layer17_attention_weight_max": 40.60185623168945,
      "activations/layer17_attention_weight_min": -24.048364639282227,
      "activations/layer18_attention_weight_max": 34.275997161865234,
      "activations/layer18_attention_weight_min": -21.827611923217773,
      "activations/layer19_attention_weight_max": 44.987789154052734,
      "activations/layer19_attention_weight_min": -30.316564559936523,
      "activations/layer1_attention_weight_max": 15.716681480407715,
      "activations/layer1_attention_weight_min": -14.309563636779785,
      "activations/layer20_attention_weight_max": 39.64113998413086,
      "activations/layer20_attention_weight_min": -24.588350296020508,
      "activations/layer21_attention_weight_max": 33.69261932373047,
      "activations/layer21_attention_weight_min": -22.01859474182129,
      "activations/layer22_attention_weight_max": 53.24956130981445,
      "activations/layer22_attention_weight_min": -28.10603904724121,
      "activations/layer23_attention_weight_max": 39.894981384277344,
      "activations/layer23_attention_weight_min": -23.551511764526367,
      "activations/layer2_attention_weight_max": 35.26898193359375,
      "activations/layer2_attention_weight_min": -31.877988815307617,
      "activations/layer3_attention_weight_max": 98.504150390625,
      "activations/layer3_attention_weight_min": -105.35718536376953,
      "activations/layer4_attention_weight_max": 143.52972412109375,
      "activations/layer4_attention_weight_min": -137.2574920654297,
      "activations/layer5_attention_weight_max": 76.84585571289062,
      "activations/layer5_attention_weight_min": -74.2892837524414,
      "activations/layer6_attention_weight_max": 59.764320373535156,
      "activations/layer6_attention_weight_min": -55.48577117919922,
      "activations/layer7_attention_weight_max": 88.31658935546875,
      "activations/layer7_attention_weight_min": -91.71672058105469,
      "activations/layer8_attention_weight_max": 40.11738586425781,
      "activations/layer8_attention_weight_min": -42.65918731689453,
      "activations/layer9_attention_weight_max": 36.31647872924805,
      "activations/layer9_attention_weight_min": -33.129817962646484,
      "epoch": 5.83,
      "learning_rate": 0.00011355871212121211,
      "loss": 2.8551,
      "step": 100300
    },
    {
      "activations/layer0_attention_weight_max": 16.334186553955078,
      "activations/layer0_attention_weight_min": -12.548118591308594,
      "activations/layer10_attention_weight_max": 31.515539169311523,
      "activations/layer10_attention_weight_min": -30.432964324951172,
      "activations/layer11_attention_weight_max": 31.87359619140625,
      "activations/layer11_attention_weight_min": -28.011741638183594,
      "activations/layer12_attention_weight_max": 27.175508499145508,
      "activations/layer12_attention_weight_min": -30.655675888061523,
      "activations/layer13_attention_weight_max": 40.27790832519531,
      "activations/layer13_attention_weight_min": -35.32941818237305,
      "activations/layer14_attention_weight_max": 54.99557876586914,
      "activations/layer14_attention_weight_min": -48.33216094970703,
      "activations/layer15_attention_weight_max": 33.327396392822266,
      "activations/layer15_attention_weight_min": -29.32227325439453,
      "activations/layer16_attention_weight_max": 32.08183670043945,
      "activations/layer16_attention_weight_min": -27.988067626953125,
      "activations/layer17_attention_weight_max": 31.772363662719727,
      "activations/layer17_attention_weight_min": -26.28562355041504,
      "activations/layer18_attention_weight_max": 31.600177764892578,
      "activations/layer18_attention_weight_min": -20.715665817260742,
      "activations/layer19_attention_weight_max": 34.32594299316406,
      "activations/layer19_attention_weight_min": -30.661649703979492,
      "activations/layer1_attention_weight_max": 17.001482009887695,
      "activations/layer1_attention_weight_min": -16.74024200439453,
      "activations/layer20_attention_weight_max": 28.618606567382812,
      "activations/layer20_attention_weight_min": -24.05828285217285,
      "activations/layer21_attention_weight_max": 26.586416244506836,
      "activations/layer21_attention_weight_min": -22.581830978393555,
      "activations/layer22_attention_weight_max": 41.46249771118164,
      "activations/layer22_attention_weight_min": -32.310020446777344,
      "activations/layer23_attention_weight_max": 32.36651611328125,
      "activations/layer23_attention_weight_min": -22.84769058227539,
      "activations/layer2_attention_weight_max": 31.817533493041992,
      "activations/layer2_attention_weight_min": -32.09773635864258,
      "activations/layer3_attention_weight_max": 90.82079315185547,
      "activations/layer3_attention_weight_min": -100.08104705810547,
      "activations/layer4_attention_weight_max": 122.92186737060547,
      "activations/layer4_attention_weight_min": -131.12408447265625,
      "activations/layer5_attention_weight_max": 70.00453186035156,
      "activations/layer5_attention_weight_min": -69.15119934082031,
      "activations/layer6_attention_weight_max": 51.22592544555664,
      "activations/layer6_attention_weight_min": -51.367252349853516,
      "activations/layer7_attention_weight_max": 83.74919128417969,
      "activations/layer7_attention_weight_min": -82.532958984375,
      "activations/layer8_attention_weight_max": 37.143775939941406,
      "activations/layer8_attention_weight_min": -39.395572662353516,
      "activations/layer9_attention_weight_max": 35.00128936767578,
      "activations/layer9_attention_weight_min": -37.08015823364258,
      "epoch": 5.83,
      "learning_rate": 0.00011353977272727272,
      "loss": 2.8505,
      "step": 100350
    },
    {
      "activations/layer0_attention_weight_max": 17.096487045288086,
      "activations/layer0_attention_weight_min": -12.205026626586914,
      "activations/layer10_attention_weight_max": 34.95219421386719,
      "activations/layer10_attention_weight_min": -30.541746139526367,
      "activations/layer11_attention_weight_max": 32.232398986816406,
      "activations/layer11_attention_weight_min": -28.166545867919922,
      "activations/layer12_attention_weight_max": 41.13357925415039,
      "activations/layer12_attention_weight_min": -35.14966583251953,
      "activations/layer13_attention_weight_max": 51.26026153564453,
      "activations/layer13_attention_weight_min": -31.974931716918945,
      "activations/layer14_attention_weight_max": 55.29241180419922,
      "activations/layer14_attention_weight_min": -39.01250457763672,
      "activations/layer15_attention_weight_max": 37.9539680480957,
      "activations/layer15_attention_weight_min": -28.592981338500977,
      "activations/layer16_attention_weight_max": 36.621055603027344,
      "activations/layer16_attention_weight_min": -28.371416091918945,
      "activations/layer17_attention_weight_max": 37.99660110473633,
      "activations/layer17_attention_weight_min": -23.821401596069336,
      "activations/layer18_attention_weight_max": 29.289934158325195,
      "activations/layer18_attention_weight_min": -21.789037704467773,
      "activations/layer19_attention_weight_max": 36.22893524169922,
      "activations/layer19_attention_weight_min": -29.097137451171875,
      "activations/layer1_attention_weight_max": 17.93185043334961,
      "activations/layer1_attention_weight_min": -18.934894561767578,
      "activations/layer20_attention_weight_max": 29.629966735839844,
      "activations/layer20_attention_weight_min": -24.772172927856445,
      "activations/layer21_attention_weight_max": 29.66396713256836,
      "activations/layer21_attention_weight_min": -20.61659812927246,
      "activations/layer22_attention_weight_max": 47.17423629760742,
      "activations/layer22_attention_weight_min": -26.581268310546875,
      "activations/layer23_attention_weight_max": 33.00849151611328,
      "activations/layer23_attention_weight_min": -20.814565658569336,
      "activations/layer2_attention_weight_max": 35.181976318359375,
      "activations/layer2_attention_weight_min": -32.67127990722656,
      "activations/layer3_attention_weight_max": 95.96318817138672,
      "activations/layer3_attention_weight_min": -98.22293090820312,
      "activations/layer4_attention_weight_max": 126.93060302734375,
      "activations/layer4_attention_weight_min": -126.02274322509766,
      "activations/layer5_attention_weight_max": 72.51553344726562,
      "activations/layer5_attention_weight_min": -69.61637115478516,
      "activations/layer6_attention_weight_max": 53.73368453979492,
      "activations/layer6_attention_weight_min": -55.30345153808594,
      "activations/layer7_attention_weight_max": 91.14763641357422,
      "activations/layer7_attention_weight_min": -81.62390899658203,
      "activations/layer8_attention_weight_max": 40.62221145629883,
      "activations/layer8_attention_weight_min": -39.006690979003906,
      "activations/layer9_attention_weight_max": 40.890933990478516,
      "activations/layer9_attention_weight_min": -36.16645812988281,
      "epoch": 5.83,
      "learning_rate": 0.00011352083333333331,
      "loss": 2.8247,
      "step": 100400
    },
    {
      "activations/layer0_attention_weight_max": 17.430255889892578,
      "activations/layer0_attention_weight_min": -12.089604377746582,
      "activations/layer10_attention_weight_max": 33.702430725097656,
      "activations/layer10_attention_weight_min": -30.037565231323242,
      "activations/layer11_attention_weight_max": 32.74315643310547,
      "activations/layer11_attention_weight_min": -27.076499938964844,
      "activations/layer12_attention_weight_max": 24.386550903320312,
      "activations/layer12_attention_weight_min": -31.41765022277832,
      "activations/layer13_attention_weight_max": 43.84520721435547,
      "activations/layer13_attention_weight_min": -31.23500633239746,
      "activations/layer14_attention_weight_max": 55.34196472167969,
      "activations/layer14_attention_weight_min": -38.45098114013672,
      "activations/layer15_attention_weight_max": 38.44569778442383,
      "activations/layer15_attention_weight_min": -27.66790771484375,
      "activations/layer16_attention_weight_max": 34.524784088134766,
      "activations/layer16_attention_weight_min": -26.868284225463867,
      "activations/layer17_attention_weight_max": 35.639137268066406,
      "activations/layer17_attention_weight_min": -24.21664810180664,
      "activations/layer18_attention_weight_max": 32.21027755737305,
      "activations/layer18_attention_weight_min": -20.53888511657715,
      "activations/layer19_attention_weight_max": 38.28538131713867,
      "activations/layer19_attention_weight_min": -28.886259078979492,
      "activations/layer1_attention_weight_max": 16.760461807250977,
      "activations/layer1_attention_weight_min": -14.51378345489502,
      "activations/layer20_attention_weight_max": 32.13239669799805,
      "activations/layer20_attention_weight_min": -24.52608299255371,
      "activations/layer21_attention_weight_max": 31.459280014038086,
      "activations/layer21_attention_weight_min": -20.60590934753418,
      "activations/layer22_attention_weight_max": 48.63930892944336,
      "activations/layer22_attention_weight_min": -28.14700698852539,
      "activations/layer23_attention_weight_max": 32.27088928222656,
      "activations/layer23_attention_weight_min": -20.75623321533203,
      "activations/layer2_attention_weight_max": 31.30934715270996,
      "activations/layer2_attention_weight_min": -32.733028411865234,
      "activations/layer3_attention_weight_max": 95.01461791992188,
      "activations/layer3_attention_weight_min": -100.7972640991211,
      "activations/layer4_attention_weight_max": 129.13079833984375,
      "activations/layer4_attention_weight_min": -133.15673828125,
      "activations/layer5_attention_weight_max": 74.0184097290039,
      "activations/layer5_attention_weight_min": -74.89578247070312,
      "activations/layer6_attention_weight_max": 54.292335510253906,
      "activations/layer6_attention_weight_min": -59.3256950378418,
      "activations/layer7_attention_weight_max": 83.1480484008789,
      "activations/layer7_attention_weight_min": -87.43425750732422,
      "activations/layer8_attention_weight_max": 39.21622848510742,
      "activations/layer8_attention_weight_min": -40.48661804199219,
      "activations/layer9_attention_weight_max": 36.26798629760742,
      "activations/layer9_attention_weight_min": -36.91712188720703,
      "epoch": 5.84,
      "learning_rate": 0.00011350189393939393,
      "loss": 2.8397,
      "step": 100450
    },
    {
      "activations/layer0_attention_weight_max": 16.509145736694336,
      "activations/layer0_attention_weight_min": -11.242880821228027,
      "activations/layer10_attention_weight_max": 36.91236114501953,
      "activations/layer10_attention_weight_min": -30.724191665649414,
      "activations/layer11_attention_weight_max": 36.296146392822266,
      "activations/layer11_attention_weight_min": -30.546300888061523,
      "activations/layer12_attention_weight_max": 64.6687240600586,
      "activations/layer12_attention_weight_min": -27.246131896972656,
      "activations/layer13_attention_weight_max": 57.141849517822266,
      "activations/layer13_attention_weight_min": -36.76874542236328,
      "activations/layer14_attention_weight_max": 63.05455780029297,
      "activations/layer14_attention_weight_min": -42.97203826904297,
      "activations/layer15_attention_weight_max": 44.556541442871094,
      "activations/layer15_attention_weight_min": -29.329242706298828,
      "activations/layer16_attention_weight_max": 38.477813720703125,
      "activations/layer16_attention_weight_min": -27.45551872253418,
      "activations/layer17_attention_weight_max": 38.09702682495117,
      "activations/layer17_attention_weight_min": -24.8468074798584,
      "activations/layer18_attention_weight_max": 32.24732208251953,
      "activations/layer18_attention_weight_min": -19.814882278442383,
      "activations/layer19_attention_weight_max": 41.99308395385742,
      "activations/layer19_attention_weight_min": -28.35066795349121,
      "activations/layer1_attention_weight_max": 17.18931007385254,
      "activations/layer1_attention_weight_min": -15.875263214111328,
      "activations/layer20_attention_weight_max": 33.52414321899414,
      "activations/layer20_attention_weight_min": -24.3178653717041,
      "activations/layer21_attention_weight_max": 31.047744750976562,
      "activations/layer21_attention_weight_min": -20.62068748474121,
      "activations/layer22_attention_weight_max": 51.11610412597656,
      "activations/layer22_attention_weight_min": -30.362445831298828,
      "activations/layer23_attention_weight_max": 34.24427795410156,
      "activations/layer23_attention_weight_min": -23.06658172607422,
      "activations/layer2_attention_weight_max": 33.7471923828125,
      "activations/layer2_attention_weight_min": -31.47093963623047,
      "activations/layer3_attention_weight_max": 94.14895629882812,
      "activations/layer3_attention_weight_min": -92.37977600097656,
      "activations/layer4_attention_weight_max": 126.58900451660156,
      "activations/layer4_attention_weight_min": -121.8200912475586,
      "activations/layer5_attention_weight_max": 75.67880249023438,
      "activations/layer5_attention_weight_min": -71.0259780883789,
      "activations/layer6_attention_weight_max": 55.813663482666016,
      "activations/layer6_attention_weight_min": -57.045814514160156,
      "activations/layer7_attention_weight_max": 90.87396240234375,
      "activations/layer7_attention_weight_min": -87.36441802978516,
      "activations/layer8_attention_weight_max": 41.119014739990234,
      "activations/layer8_attention_weight_min": -40.40707015991211,
      "activations/layer9_attention_weight_max": 39.77064895629883,
      "activations/layer9_attention_weight_min": -35.33217239379883,
      "epoch": 5.84,
      "learning_rate": 0.00011348295454545454,
      "loss": 2.8391,
      "step": 100500
    },
    {
      "activations/layer0_attention_weight_max": 15.662439346313477,
      "activations/layer0_attention_weight_min": -12.134603500366211,
      "activations/layer10_attention_weight_max": 33.99033737182617,
      "activations/layer10_attention_weight_min": -31.88724136352539,
      "activations/layer11_attention_weight_max": 35.49785614013672,
      "activations/layer11_attention_weight_min": -30.158599853515625,
      "activations/layer12_attention_weight_max": 24.08954620361328,
      "activations/layer12_attention_weight_min": -25.908212661743164,
      "activations/layer13_attention_weight_max": 39.72309875488281,
      "activations/layer13_attention_weight_min": -30.129653930664062,
      "activations/layer14_attention_weight_max": 60.59761047363281,
      "activations/layer14_attention_weight_min": -46.911834716796875,
      "activations/layer15_attention_weight_max": 37.43562698364258,
      "activations/layer15_attention_weight_min": -28.964488983154297,
      "activations/layer16_attention_weight_max": 37.811649322509766,
      "activations/layer16_attention_weight_min": -30.392959594726562,
      "activations/layer17_attention_weight_max": 37.191349029541016,
      "activations/layer17_attention_weight_min": -24.882965087890625,
      "activations/layer18_attention_weight_max": 38.09666061401367,
      "activations/layer18_attention_weight_min": -21.561491012573242,
      "activations/layer19_attention_weight_max": 38.867210388183594,
      "activations/layer19_attention_weight_min": -29.821613311767578,
      "activations/layer1_attention_weight_max": 16.6671142578125,
      "activations/layer1_attention_weight_min": -15.099081993103027,
      "activations/layer20_attention_weight_max": 34.703060150146484,
      "activations/layer20_attention_weight_min": -23.488908767700195,
      "activations/layer21_attention_weight_max": 38.55754852294922,
      "activations/layer21_attention_weight_min": -22.5679874420166,
      "activations/layer22_attention_weight_max": 53.7957649230957,
      "activations/layer22_attention_weight_min": -29.82319450378418,
      "activations/layer23_attention_weight_max": 35.54484939575195,
      "activations/layer23_attention_weight_min": -19.897584915161133,
      "activations/layer2_attention_weight_max": 33.58049011230469,
      "activations/layer2_attention_weight_min": -33.825439453125,
      "activations/layer3_attention_weight_max": 96.02704620361328,
      "activations/layer3_attention_weight_min": -97.1877212524414,
      "activations/layer4_attention_weight_max": 130.38182067871094,
      "activations/layer4_attention_weight_min": -133.24766540527344,
      "activations/layer5_attention_weight_max": 77.03099060058594,
      "activations/layer5_attention_weight_min": -72.08456420898438,
      "activations/layer6_attention_weight_max": 57.607852935791016,
      "activations/layer6_attention_weight_min": -53.21406173706055,
      "activations/layer7_attention_weight_max": 86.57059478759766,
      "activations/layer7_attention_weight_min": -84.4159927368164,
      "activations/layer8_attention_weight_max": 41.711326599121094,
      "activations/layer8_attention_weight_min": -39.42424011230469,
      "activations/layer9_attention_weight_max": 35.59092330932617,
      "activations/layer9_attention_weight_min": -37.62412643432617,
      "epoch": 5.84,
      "learning_rate": 0.00011346401515151513,
      "loss": 2.8378,
      "step": 100550
    },
    {
      "activations/layer0_attention_weight_max": 15.177821159362793,
      "activations/layer0_attention_weight_min": -12.254728317260742,
      "activations/layer10_attention_weight_max": 35.58333206176758,
      "activations/layer10_attention_weight_min": -33.92876434326172,
      "activations/layer11_attention_weight_max": 34.25471115112305,
      "activations/layer11_attention_weight_min": -29.376819610595703,
      "activations/layer12_attention_weight_max": 30.073383331298828,
      "activations/layer12_attention_weight_min": -26.772239685058594,
      "activations/layer13_attention_weight_max": 47.99354553222656,
      "activations/layer13_attention_weight_min": -32.779762268066406,
      "activations/layer14_attention_weight_max": 62.16100311279297,
      "activations/layer14_attention_weight_min": -48.981849670410156,
      "activations/layer15_attention_weight_max": 46.31094741821289,
      "activations/layer15_attention_weight_min": -30.57971954345703,
      "activations/layer16_attention_weight_max": 36.771846771240234,
      "activations/layer16_attention_weight_min": -29.675458908081055,
      "activations/layer17_attention_weight_max": 36.54703903198242,
      "activations/layer17_attention_weight_min": -25.443008422851562,
      "activations/layer18_attention_weight_max": 35.76826095581055,
      "activations/layer18_attention_weight_min": -20.200777053833008,
      "activations/layer19_attention_weight_max": 37.35915756225586,
      "activations/layer19_attention_weight_min": -31.632322311401367,
      "activations/layer1_attention_weight_max": 17.619226455688477,
      "activations/layer1_attention_weight_min": -14.670525550842285,
      "activations/layer20_attention_weight_max": 32.08180618286133,
      "activations/layer20_attention_weight_min": -23.429153442382812,
      "activations/layer21_attention_weight_max": 35.833099365234375,
      "activations/layer21_attention_weight_min": -21.335002899169922,
      "activations/layer22_attention_weight_max": 48.0085563659668,
      "activations/layer22_attention_weight_min": -27.65478515625,
      "activations/layer23_attention_weight_max": 32.666778564453125,
      "activations/layer23_attention_weight_min": -20.420339584350586,
      "activations/layer2_attention_weight_max": 33.28427505493164,
      "activations/layer2_attention_weight_min": -33.406646728515625,
      "activations/layer3_attention_weight_max": 91.11384582519531,
      "activations/layer3_attention_weight_min": -92.38948059082031,
      "activations/layer4_attention_weight_max": 125.4759292602539,
      "activations/layer4_attention_weight_min": -125.91131591796875,
      "activations/layer5_attention_weight_max": 74.3994140625,
      "activations/layer5_attention_weight_min": -75.49514770507812,
      "activations/layer6_attention_weight_max": 56.894046783447266,
      "activations/layer6_attention_weight_min": -57.13066101074219,
      "activations/layer7_attention_weight_max": 90.99969482421875,
      "activations/layer7_attention_weight_min": -88.72561645507812,
      "activations/layer8_attention_weight_max": 40.398658752441406,
      "activations/layer8_attention_weight_min": -43.265159606933594,
      "activations/layer9_attention_weight_max": 39.17908477783203,
      "activations/layer9_attention_weight_min": -36.02228927612305,
      "epoch": 5.85,
      "learning_rate": 0.00011344507575757575,
      "loss": 2.835,
      "step": 100600
    },
    {
      "activations/layer0_attention_weight_max": 16.976654052734375,
      "activations/layer0_attention_weight_min": -12.478957176208496,
      "activations/layer10_attention_weight_max": 31.5794734954834,
      "activations/layer10_attention_weight_min": -32.54424285888672,
      "activations/layer11_attention_weight_max": 32.13825225830078,
      "activations/layer11_attention_weight_min": -28.733427047729492,
      "activations/layer12_attention_weight_max": 30.017776489257812,
      "activations/layer12_attention_weight_min": -23.777652740478516,
      "activations/layer13_attention_weight_max": 42.837806701660156,
      "activations/layer13_attention_weight_min": -31.126903533935547,
      "activations/layer14_attention_weight_max": 62.89094543457031,
      "activations/layer14_attention_weight_min": -44.877197265625,
      "activations/layer15_attention_weight_max": 38.71419906616211,
      "activations/layer15_attention_weight_min": -27.9266300201416,
      "activations/layer16_attention_weight_max": 37.49283218383789,
      "activations/layer16_attention_weight_min": -27.758590698242188,
      "activations/layer17_attention_weight_max": 38.75798416137695,
      "activations/layer17_attention_weight_min": -25.470582962036133,
      "activations/layer18_attention_weight_max": 40.85498046875,
      "activations/layer18_attention_weight_min": -20.38755226135254,
      "activations/layer19_attention_weight_max": 43.998817443847656,
      "activations/layer19_attention_weight_min": -31.897857666015625,
      "activations/layer1_attention_weight_max": 18.83555030822754,
      "activations/layer1_attention_weight_min": -16.89336395263672,
      "activations/layer20_attention_weight_max": 37.102474212646484,
      "activations/layer20_attention_weight_min": -24.527385711669922,
      "activations/layer21_attention_weight_max": 38.130130767822266,
      "activations/layer21_attention_weight_min": -21.390010833740234,
      "activations/layer22_attention_weight_max": 58.28309631347656,
      "activations/layer22_attention_weight_min": -30.296979904174805,
      "activations/layer23_attention_weight_max": 39.95740509033203,
      "activations/layer23_attention_weight_min": -25.850204467773438,
      "activations/layer2_attention_weight_max": 31.7034969329834,
      "activations/layer2_attention_weight_min": -30.826969146728516,
      "activations/layer3_attention_weight_max": 85.91385650634766,
      "activations/layer3_attention_weight_min": -87.49871063232422,
      "activations/layer4_attention_weight_max": 124.77559661865234,
      "activations/layer4_attention_weight_min": -121.17644500732422,
      "activations/layer5_attention_weight_max": 71.5408706665039,
      "activations/layer5_attention_weight_min": -73.45526885986328,
      "activations/layer6_attention_weight_max": 55.42134475708008,
      "activations/layer6_attention_weight_min": -56.22382354736328,
      "activations/layer7_attention_weight_max": 87.86376953125,
      "activations/layer7_attention_weight_min": -86.35690307617188,
      "activations/layer8_attention_weight_max": 38.54904556274414,
      "activations/layer8_attention_weight_min": -39.114131927490234,
      "activations/layer9_attention_weight_max": 35.25569152832031,
      "activations/layer9_attention_weight_min": -34.759910583496094,
      "epoch": 5.85,
      "learning_rate": 0.00011342613636363636,
      "loss": 2.8369,
      "step": 100650
    },
    {
      "activations/layer0_attention_weight_max": 17.628358840942383,
      "activations/layer0_attention_weight_min": -12.336009979248047,
      "activations/layer10_attention_weight_max": 31.780406951904297,
      "activations/layer10_attention_weight_min": -28.603515625,
      "activations/layer11_attention_weight_max": 28.355789184570312,
      "activations/layer11_attention_weight_min": -26.64736557006836,
      "activations/layer12_attention_weight_max": 23.200340270996094,
      "activations/layer12_attention_weight_min": -24.68496322631836,
      "activations/layer13_attention_weight_max": 39.14104080200195,
      "activations/layer13_attention_weight_min": -31.71871566772461,
      "activations/layer14_attention_weight_max": 59.4925537109375,
      "activations/layer14_attention_weight_min": -47.39781188964844,
      "activations/layer15_attention_weight_max": 34.7061767578125,
      "activations/layer15_attention_weight_min": -28.09589385986328,
      "activations/layer16_attention_weight_max": 34.90707015991211,
      "activations/layer16_attention_weight_min": -27.914012908935547,
      "activations/layer17_attention_weight_max": 34.274009704589844,
      "activations/layer17_attention_weight_min": -24.7252140045166,
      "activations/layer18_attention_weight_max": 29.485307693481445,
      "activations/layer18_attention_weight_min": -20.779428482055664,
      "activations/layer19_attention_weight_max": 38.2558708190918,
      "activations/layer19_attention_weight_min": -30.530899047851562,
      "activations/layer1_attention_weight_max": 17.32461929321289,
      "activations/layer1_attention_weight_min": -14.545391082763672,
      "activations/layer20_attention_weight_max": 28.35595703125,
      "activations/layer20_attention_weight_min": -26.14312744140625,
      "activations/layer21_attention_weight_max": 29.997671127319336,
      "activations/layer21_attention_weight_min": -20.230093002319336,
      "activations/layer22_attention_weight_max": 44.53213882446289,
      "activations/layer22_attention_weight_min": -28.258460998535156,
      "activations/layer23_attention_weight_max": 36.080963134765625,
      "activations/layer23_attention_weight_min": -21.232656478881836,
      "activations/layer2_attention_weight_max": 31.048839569091797,
      "activations/layer2_attention_weight_min": -30.69460678100586,
      "activations/layer3_attention_weight_max": 85.59622192382812,
      "activations/layer3_attention_weight_min": -89.32337188720703,
      "activations/layer4_attention_weight_max": 119.65164947509766,
      "activations/layer4_attention_weight_min": -120.80841064453125,
      "activations/layer5_attention_weight_max": 69.09815979003906,
      "activations/layer5_attention_weight_min": -67.53886413574219,
      "activations/layer6_attention_weight_max": 50.68650436401367,
      "activations/layer6_attention_weight_min": -52.185646057128906,
      "activations/layer7_attention_weight_max": 77.7242431640625,
      "activations/layer7_attention_weight_min": -77.73004913330078,
      "activations/layer8_attention_weight_max": 37.16878128051758,
      "activations/layer8_attention_weight_min": -38.5032844543457,
      "activations/layer9_attention_weight_max": 33.39699935913086,
      "activations/layer9_attention_weight_min": -31.935546875,
      "epoch": 5.85,
      "learning_rate": 0.00011340719696969696,
      "loss": 2.8372,
      "step": 100700
    },
    {
      "activations/layer0_attention_weight_max": 15.811553001403809,
      "activations/layer0_attention_weight_min": -11.814913749694824,
      "activations/layer10_attention_weight_max": 36.779945373535156,
      "activations/layer10_attention_weight_min": -31.986825942993164,
      "activations/layer11_attention_weight_max": 34.769935607910156,
      "activations/layer11_attention_weight_min": -27.829824447631836,
      "activations/layer12_attention_weight_max": 39.527320861816406,
      "activations/layer12_attention_weight_min": -26.51049041748047,
      "activations/layer13_attention_weight_max": 42.441627502441406,
      "activations/layer13_attention_weight_min": -32.64287185668945,
      "activations/layer14_attention_weight_max": 53.840049743652344,
      "activations/layer14_attention_weight_min": -36.8714485168457,
      "activations/layer15_attention_weight_max": 44.2718505859375,
      "activations/layer15_attention_weight_min": -29.420570373535156,
      "activations/layer16_attention_weight_max": 37.44297409057617,
      "activations/layer16_attention_weight_min": -28.72865867614746,
      "activations/layer17_attention_weight_max": 40.9852294921875,
      "activations/layer17_attention_weight_min": -23.760873794555664,
      "activations/layer18_attention_weight_max": 30.560701370239258,
      "activations/layer18_attention_weight_min": -20.618770599365234,
      "activations/layer19_attention_weight_max": 43.53871154785156,
      "activations/layer19_attention_weight_min": -28.946836471557617,
      "activations/layer1_attention_weight_max": 16.904624938964844,
      "activations/layer1_attention_weight_min": -14.237000465393066,
      "activations/layer20_attention_weight_max": 38.34195327758789,
      "activations/layer20_attention_weight_min": -23.249492645263672,
      "activations/layer21_attention_weight_max": 35.64170455932617,
      "activations/layer21_attention_weight_min": -21.61933135986328,
      "activations/layer22_attention_weight_max": 52.271629333496094,
      "activations/layer22_attention_weight_min": -28.25894546508789,
      "activations/layer23_attention_weight_max": 31.9041805267334,
      "activations/layer23_attention_weight_min": -21.216419219970703,
      "activations/layer2_attention_weight_max": 32.344818115234375,
      "activations/layer2_attention_weight_min": -34.076499938964844,
      "activations/layer3_attention_weight_max": 91.10476684570312,
      "activations/layer3_attention_weight_min": -93.66069030761719,
      "activations/layer4_attention_weight_max": 125.09828186035156,
      "activations/layer4_attention_weight_min": -126.75563049316406,
      "activations/layer5_attention_weight_max": 72.21891784667969,
      "activations/layer5_attention_weight_min": -69.7616958618164,
      "activations/layer6_attention_weight_max": 53.39643859863281,
      "activations/layer6_attention_weight_min": -55.84678649902344,
      "activations/layer7_attention_weight_max": 87.80168151855469,
      "activations/layer7_attention_weight_min": -90.04447174072266,
      "activations/layer8_attention_weight_max": 38.23882293701172,
      "activations/layer8_attention_weight_min": -40.44742202758789,
      "activations/layer9_attention_weight_max": 38.0348014831543,
      "activations/layer9_attention_weight_min": -35.003902435302734,
      "epoch": 5.85,
      "learning_rate": 0.00011338863636363635,
      "loss": 2.8422,
      "step": 100750
    },
    {
      "activations/layer0_attention_weight_max": 17.577224731445312,
      "activations/layer0_attention_weight_min": -11.99052906036377,
      "activations/layer10_attention_weight_max": 32.88484573364258,
      "activations/layer10_attention_weight_min": -29.15789031982422,
      "activations/layer11_attention_weight_max": 31.35863494873047,
      "activations/layer11_attention_weight_min": -28.25800895690918,
      "activations/layer12_attention_weight_max": 27.756139755249023,
      "activations/layer12_attention_weight_min": -27.822105407714844,
      "activations/layer13_attention_weight_max": 43.841163635253906,
      "activations/layer13_attention_weight_min": -34.72520065307617,
      "activations/layer14_attention_weight_max": 63.4324951171875,
      "activations/layer14_attention_weight_min": -43.65637969970703,
      "activations/layer15_attention_weight_max": 46.020347595214844,
      "activations/layer15_attention_weight_min": -29.759262084960938,
      "activations/layer16_attention_weight_max": 41.03337478637695,
      "activations/layer16_attention_weight_min": -30.36458396911621,
      "activations/layer17_attention_weight_max": 39.62050247192383,
      "activations/layer17_attention_weight_min": -27.276294708251953,
      "activations/layer18_attention_weight_max": 31.697490692138672,
      "activations/layer18_attention_weight_min": -20.722997665405273,
      "activations/layer19_attention_weight_max": 41.638187408447266,
      "activations/layer19_attention_weight_min": -31.00111961364746,
      "activations/layer1_attention_weight_max": 18.354703903198242,
      "activations/layer1_attention_weight_min": -18.752737045288086,
      "activations/layer20_attention_weight_max": 32.89288330078125,
      "activations/layer20_attention_weight_min": -25.786935806274414,
      "activations/layer21_attention_weight_max": 34.71273422241211,
      "activations/layer21_attention_weight_min": -22.826873779296875,
      "activations/layer22_attention_weight_max": 48.93256378173828,
      "activations/layer22_attention_weight_min": -28.6035213470459,
      "activations/layer23_attention_weight_max": 34.655723571777344,
      "activations/layer23_attention_weight_min": -19.31377410888672,
      "activations/layer2_attention_weight_max": 29.927587509155273,
      "activations/layer2_attention_weight_min": -29.4754695892334,
      "activations/layer3_attention_weight_max": 88.60823059082031,
      "activations/layer3_attention_weight_min": -85.66029357910156,
      "activations/layer4_attention_weight_max": 119.3589096069336,
      "activations/layer4_attention_weight_min": -117.17291259765625,
      "activations/layer5_attention_weight_max": 72.07923126220703,
      "activations/layer5_attention_weight_min": -69.17001342773438,
      "activations/layer6_attention_weight_max": 55.31245422363281,
      "activations/layer6_attention_weight_min": -55.31856155395508,
      "activations/layer7_attention_weight_max": 83.4699478149414,
      "activations/layer7_attention_weight_min": -83.13368225097656,
      "activations/layer8_attention_weight_max": 37.66632080078125,
      "activations/layer8_attention_weight_min": -39.21055221557617,
      "activations/layer9_attention_weight_max": 33.342891693115234,
      "activations/layer9_attention_weight_min": -34.98698425292969,
      "epoch": 5.86,
      "learning_rate": 0.00011336969696969695,
      "loss": 2.8195,
      "step": 100800
    },
    {
      "activations/layer0_attention_weight_max": 15.724990844726562,
      "activations/layer0_attention_weight_min": -12.298001289367676,
      "activations/layer10_attention_weight_max": 34.2652702331543,
      "activations/layer10_attention_weight_min": -29.374454498291016,
      "activations/layer11_attention_weight_max": 35.662086486816406,
      "activations/layer11_attention_weight_min": -29.268369674682617,
      "activations/layer12_attention_weight_max": 31.196430206298828,
      "activations/layer12_attention_weight_min": -27.096513748168945,
      "activations/layer13_attention_weight_max": 49.143943786621094,
      "activations/layer13_attention_weight_min": -32.627220153808594,
      "activations/layer14_attention_weight_max": 67.0510025024414,
      "activations/layer14_attention_weight_min": -49.97495651245117,
      "activations/layer15_attention_weight_max": 43.68067932128906,
      "activations/layer15_attention_weight_min": -28.83791160583496,
      "activations/layer16_attention_weight_max": 36.73606491088867,
      "activations/layer16_attention_weight_min": -27.536413192749023,
      "activations/layer17_attention_weight_max": 37.79547882080078,
      "activations/layer17_attention_weight_min": -24.762929916381836,
      "activations/layer18_attention_weight_max": 36.72272491455078,
      "activations/layer18_attention_weight_min": -21.153730392456055,
      "activations/layer19_attention_weight_max": 41.760250091552734,
      "activations/layer19_attention_weight_min": -30.71710777282715,
      "activations/layer1_attention_weight_max": 19.040477752685547,
      "activations/layer1_attention_weight_min": -15.236225128173828,
      "activations/layer20_attention_weight_max": 35.73963165283203,
      "activations/layer20_attention_weight_min": -26.243207931518555,
      "activations/layer21_attention_weight_max": 36.894737243652344,
      "activations/layer21_attention_weight_min": -23.451793670654297,
      "activations/layer22_attention_weight_max": 49.144134521484375,
      "activations/layer22_attention_weight_min": -28.294410705566406,
      "activations/layer23_attention_weight_max": 34.74555587768555,
      "activations/layer23_attention_weight_min": -21.68543243408203,
      "activations/layer2_attention_weight_max": 31.944686889648438,
      "activations/layer2_attention_weight_min": -31.719234466552734,
      "activations/layer3_attention_weight_max": 86.1034164428711,
      "activations/layer3_attention_weight_min": -90.96133422851562,
      "activations/layer4_attention_weight_max": 126.7200698852539,
      "activations/layer4_attention_weight_min": -119.91275787353516,
      "activations/layer5_attention_weight_max": 73.59893798828125,
      "activations/layer5_attention_weight_min": -69.18108367919922,
      "activations/layer6_attention_weight_max": 52.836082458496094,
      "activations/layer6_attention_weight_min": -53.382076263427734,
      "activations/layer7_attention_weight_max": 90.9164047241211,
      "activations/layer7_attention_weight_min": -79.48176574707031,
      "activations/layer8_attention_weight_max": 39.919437408447266,
      "activations/layer8_attention_weight_min": -37.10397720336914,
      "activations/layer9_attention_weight_max": 32.16225814819336,
      "activations/layer9_attention_weight_min": -36.882266998291016,
      "epoch": 5.86,
      "learning_rate": 0.00011335075757575757,
      "loss": 2.8474,
      "step": 100850
    },
    {
      "activations/layer0_attention_weight_max": 16.540250778198242,
      "activations/layer0_attention_weight_min": -12.940566062927246,
      "activations/layer10_attention_weight_max": 33.203556060791016,
      "activations/layer10_attention_weight_min": -31.09481430053711,
      "activations/layer11_attention_weight_max": 33.574798583984375,
      "activations/layer11_attention_weight_min": -28.086109161376953,
      "activations/layer12_attention_weight_max": 24.606796264648438,
      "activations/layer12_attention_weight_min": -24.89951515197754,
      "activations/layer13_attention_weight_max": 52.354488372802734,
      "activations/layer13_attention_weight_min": -35.87372970581055,
      "activations/layer14_attention_weight_max": 64.55934143066406,
      "activations/layer14_attention_weight_min": -45.339359283447266,
      "activations/layer15_attention_weight_max": 41.729042053222656,
      "activations/layer15_attention_weight_min": -27.605907440185547,
      "activations/layer16_attention_weight_max": 39.53019714355469,
      "activations/layer16_attention_weight_min": -25.39225196838379,
      "activations/layer17_attention_weight_max": 34.6053466796875,
      "activations/layer17_attention_weight_min": -24.439102172851562,
      "activations/layer18_attention_weight_max": 34.64790344238281,
      "activations/layer18_attention_weight_min": -23.27606773376465,
      "activations/layer19_attention_weight_max": 41.19988250732422,
      "activations/layer19_attention_weight_min": -28.904874801635742,
      "activations/layer1_attention_weight_max": 18.168886184692383,
      "activations/layer1_attention_weight_min": -14.625910758972168,
      "activations/layer20_attention_weight_max": 34.90411376953125,
      "activations/layer20_attention_weight_min": -23.168132781982422,
      "activations/layer21_attention_weight_max": 33.74040222167969,
      "activations/layer21_attention_weight_min": -20.88809585571289,
      "activations/layer22_attention_weight_max": 48.13228225708008,
      "activations/layer22_attention_weight_min": -29.270076751708984,
      "activations/layer23_attention_weight_max": 32.987510681152344,
      "activations/layer23_attention_weight_min": -20.581653594970703,
      "activations/layer2_attention_weight_max": 33.785675048828125,
      "activations/layer2_attention_weight_min": -31.881736755371094,
      "activations/layer3_attention_weight_max": 95.93011474609375,
      "activations/layer3_attention_weight_min": -98.31949615478516,
      "activations/layer4_attention_weight_max": 140.9861297607422,
      "activations/layer4_attention_weight_min": -134.23387145996094,
      "activations/layer5_attention_weight_max": 82.33863830566406,
      "activations/layer5_attention_weight_min": -76.49629211425781,
      "activations/layer6_attention_weight_max": 59.04474639892578,
      "activations/layer6_attention_weight_min": -54.30238723754883,
      "activations/layer7_attention_weight_max": 88.0888900756836,
      "activations/layer7_attention_weight_min": -91.84886932373047,
      "activations/layer8_attention_weight_max": 42.80992889404297,
      "activations/layer8_attention_weight_min": -41.06572341918945,
      "activations/layer9_attention_weight_max": 36.90102005004883,
      "activations/layer9_attention_weight_min": -35.78221130371094,
      "epoch": 5.86,
      "learning_rate": 0.00011333181818181817,
      "loss": 2.8475,
      "step": 100900
    },
    {
      "activations/layer0_attention_weight_max": 15.678646087646484,
      "activations/layer0_attention_weight_min": -11.591048240661621,
      "activations/layer10_attention_weight_max": 35.68341827392578,
      "activations/layer10_attention_weight_min": -33.379146575927734,
      "activations/layer11_attention_weight_max": 33.09709930419922,
      "activations/layer11_attention_weight_min": -27.641843795776367,
      "activations/layer12_attention_weight_max": 34.729583740234375,
      "activations/layer12_attention_weight_min": -25.223398208618164,
      "activations/layer13_attention_weight_max": 47.11129379272461,
      "activations/layer13_attention_weight_min": -33.62604522705078,
      "activations/layer14_attention_weight_max": 71.81170654296875,
      "activations/layer14_attention_weight_min": -46.82473373413086,
      "activations/layer15_attention_weight_max": 39.199100494384766,
      "activations/layer15_attention_weight_min": -29.73697280883789,
      "activations/layer16_attention_weight_max": 36.255767822265625,
      "activations/layer16_attention_weight_min": -28.61737060546875,
      "activations/layer17_attention_weight_max": 36.04072570800781,
      "activations/layer17_attention_weight_min": -24.733165740966797,
      "activations/layer18_attention_weight_max": 32.80024719238281,
      "activations/layer18_attention_weight_min": -20.971332550048828,
      "activations/layer19_attention_weight_max": 42.688350677490234,
      "activations/layer19_attention_weight_min": -35.099822998046875,
      "activations/layer1_attention_weight_max": 17.16590690612793,
      "activations/layer1_attention_weight_min": -14.829197883605957,
      "activations/layer20_attention_weight_max": 33.71359634399414,
      "activations/layer20_attention_weight_min": -24.950576782226562,
      "activations/layer21_attention_weight_max": 32.121524810791016,
      "activations/layer21_attention_weight_min": -21.94566535949707,
      "activations/layer22_attention_weight_max": 54.482295989990234,
      "activations/layer22_attention_weight_min": -29.493093490600586,
      "activations/layer23_attention_weight_max": 34.65203094482422,
      "activations/layer23_attention_weight_min": -20.57745933532715,
      "activations/layer2_attention_weight_max": 32.15361404418945,
      "activations/layer2_attention_weight_min": -32.54705047607422,
      "activations/layer3_attention_weight_max": 92.782958984375,
      "activations/layer3_attention_weight_min": -95.9974594116211,
      "activations/layer4_attention_weight_max": 121.44847869873047,
      "activations/layer4_attention_weight_min": -124.92589569091797,
      "activations/layer5_attention_weight_max": 72.16546630859375,
      "activations/layer5_attention_weight_min": -72.13786315917969,
      "activations/layer6_attention_weight_max": 52.72401809692383,
      "activations/layer6_attention_weight_min": -55.309818267822266,
      "activations/layer7_attention_weight_max": 87.02681732177734,
      "activations/layer7_attention_weight_min": -90.15309143066406,
      "activations/layer8_attention_weight_max": 39.69216537475586,
      "activations/layer8_attention_weight_min": -41.77553939819336,
      "activations/layer9_attention_weight_max": 37.326053619384766,
      "activations/layer9_attention_weight_min": -36.54983901977539,
      "epoch": 5.87,
      "learning_rate": 0.00011331287878787879,
      "loss": 2.8379,
      "step": 100950
    },
    {
      "activations/layer0_attention_weight_max": 15.241399765014648,
      "activations/layer0_attention_weight_min": -12.855352401733398,
      "activations/layer10_attention_weight_max": 40.11275863647461,
      "activations/layer10_attention_weight_min": -35.21394348144531,
      "activations/layer11_attention_weight_max": 37.1927490234375,
      "activations/layer11_attention_weight_min": -29.97665023803711,
      "activations/layer12_attention_weight_max": 26.50166130065918,
      "activations/layer12_attention_weight_min": -23.36186981201172,
      "activations/layer13_attention_weight_max": 45.78188705444336,
      "activations/layer13_attention_weight_min": -35.124568939208984,
      "activations/layer14_attention_weight_max": 68.97383117675781,
      "activations/layer14_attention_weight_min": -45.294254302978516,
      "activations/layer15_attention_weight_max": 43.08208465576172,
      "activations/layer15_attention_weight_min": -28.998104095458984,
      "activations/layer16_attention_weight_max": 37.742271423339844,
      "activations/layer16_attention_weight_min": -27.64312171936035,
      "activations/layer17_attention_weight_max": 42.65053939819336,
      "activations/layer17_attention_weight_min": -23.808391571044922,
      "activations/layer18_attention_weight_max": 32.87265396118164,
      "activations/layer18_attention_weight_min": -20.05250358581543,
      "activations/layer19_attention_weight_max": 38.6867561340332,
      "activations/layer19_attention_weight_min": -29.55524253845215,
      "activations/layer1_attention_weight_max": 17.577693939208984,
      "activations/layer1_attention_weight_min": -16.4077091217041,
      "activations/layer20_attention_weight_max": 36.39616012573242,
      "activations/layer20_attention_weight_min": -24.443387985229492,
      "activations/layer21_attention_weight_max": 30.987350463867188,
      "activations/layer21_attention_weight_min": -21.057357788085938,
      "activations/layer22_attention_weight_max": 49.19216537475586,
      "activations/layer22_attention_weight_min": -30.6239013671875,
      "activations/layer23_attention_weight_max": 34.49619674682617,
      "activations/layer23_attention_weight_min": -23.497264862060547,
      "activations/layer2_attention_weight_max": 30.56438446044922,
      "activations/layer2_attention_weight_min": -30.359037399291992,
      "activations/layer3_attention_weight_max": 91.43093872070312,
      "activations/layer3_attention_weight_min": -98.5697021484375,
      "activations/layer4_attention_weight_max": 129.5504608154297,
      "activations/layer4_attention_weight_min": -127.5265121459961,
      "activations/layer5_attention_weight_max": 78.64742279052734,
      "activations/layer5_attention_weight_min": -73.49346923828125,
      "activations/layer6_attention_weight_max": 58.562503814697266,
      "activations/layer6_attention_weight_min": -56.57176208496094,
      "activations/layer7_attention_weight_max": 86.64559936523438,
      "activations/layer7_attention_weight_min": -91.16547393798828,
      "activations/layer8_attention_weight_max": 43.389034271240234,
      "activations/layer8_attention_weight_min": -44.70616149902344,
      "activations/layer9_attention_weight_max": 40.66518783569336,
      "activations/layer9_attention_weight_min": -40.30122756958008,
      "epoch": 5.87,
      "learning_rate": 0.00011329393939393937,
      "loss": 2.8396,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.5187,
      "eval_samples_per_second": 504.068,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.5187,
      "eval_openwebtext_samples_per_second": 504.068,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 1.952,
      "eval_wikitext_samples_per_second": 233.61,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.5505,
      "eval_lambada_samples_per_second": 509.814,
      "step": 101000
    },
    {
      "activations/layer0_attention_weight_max": 15.14116096496582,
      "activations/layer0_attention_weight_min": -11.58251953125,
      "activations/layer10_attention_weight_max": 33.397865295410156,
      "activations/layer10_attention_weight_min": -30.6593074798584,
      "activations/layer11_attention_weight_max": 32.22084045410156,
      "activations/layer11_attention_weight_min": -27.612472534179688,
      "activations/layer12_attention_weight_max": 29.646059036254883,
      "activations/layer12_attention_weight_min": -26.771554946899414,
      "activations/layer13_attention_weight_max": 41.7562255859375,
      "activations/layer13_attention_weight_min": -32.796817779541016,
      "activations/layer14_attention_weight_max": 65.52962493896484,
      "activations/layer14_attention_weight_min": -45.73176193237305,
      "activations/layer15_attention_weight_max": 39.206153869628906,
      "activations/layer15_attention_weight_min": -28.412485122680664,
      "activations/layer16_attention_weight_max": 43.20091247558594,
      "activations/layer16_attention_weight_min": -25.79704475402832,
      "activations/layer17_attention_weight_max": 37.35942840576172,
      "activations/layer17_attention_weight_min": -24.210796356201172,
      "activations/layer18_attention_weight_max": 31.833011627197266,
      "activations/layer18_attention_weight_min": -20.264389038085938,
      "activations/layer19_attention_weight_max": 40.60332107543945,
      "activations/layer19_attention_weight_min": -29.133045196533203,
      "activations/layer1_attention_weight_max": 17.042329788208008,
      "activations/layer1_attention_weight_min": -14.096099853515625,
      "activations/layer20_attention_weight_max": 31.40196990966797,
      "activations/layer20_attention_weight_min": -25.328784942626953,
      "activations/layer21_attention_weight_max": 31.036712646484375,
      "activations/layer21_attention_weight_min": -21.714401245117188,
      "activations/layer22_attention_weight_max": 53.293212890625,
      "activations/layer22_attention_weight_min": -29.194143295288086,
      "activations/layer23_attention_weight_max": 31.2950496673584,
      "activations/layer23_attention_weight_min": -22.380338668823242,
      "activations/layer2_attention_weight_max": 30.80677604675293,
      "activations/layer2_attention_weight_min": -28.807668685913086,
      "activations/layer3_attention_weight_max": 90.82611846923828,
      "activations/layer3_attention_weight_min": -88.33129119873047,
      "activations/layer4_attention_weight_max": 129.53053283691406,
      "activations/layer4_attention_weight_min": -117.23079681396484,
      "activations/layer5_attention_weight_max": 72.53810119628906,
      "activations/layer5_attention_weight_min": -67.5885238647461,
      "activations/layer6_attention_weight_max": 52.63595199584961,
      "activations/layer6_attention_weight_min": -53.47675704956055,
      "activations/layer7_attention_weight_max": 84.22789764404297,
      "activations/layer7_attention_weight_min": -91.99593353271484,
      "activations/layer8_attention_weight_max": 40.3147087097168,
      "activations/layer8_attention_weight_min": -41.134071350097656,
      "activations/layer9_attention_weight_max": 36.21665954589844,
      "activations/layer9_attention_weight_min": -34.455284118652344,
      "epoch": 5.87,
      "learning_rate": 0.00011327499999999999,
      "loss": 2.8417,
      "step": 101050
    },
    {
      "activations/layer0_attention_weight_max": 15.448369979858398,
      "activations/layer0_attention_weight_min": -11.999229431152344,
      "activations/layer10_attention_weight_max": 32.13691329956055,
      "activations/layer10_attention_weight_min": -31.325054168701172,
      "activations/layer11_attention_weight_max": 34.30555725097656,
      "activations/layer11_attention_weight_min": -28.602935791015625,
      "activations/layer12_attention_weight_max": 24.653385162353516,
      "activations/layer12_attention_weight_min": -28.28612518310547,
      "activations/layer13_attention_weight_max": 41.841461181640625,
      "activations/layer13_attention_weight_min": -30.675071716308594,
      "activations/layer14_attention_weight_max": 57.02753448486328,
      "activations/layer14_attention_weight_min": -41.806640625,
      "activations/layer15_attention_weight_max": 37.065975189208984,
      "activations/layer15_attention_weight_min": -29.325153350830078,
      "activations/layer16_attention_weight_max": 37.52769470214844,
      "activations/layer16_attention_weight_min": -27.475200653076172,
      "activations/layer17_attention_weight_max": 35.5780143737793,
      "activations/layer17_attention_weight_min": -23.053600311279297,
      "activations/layer18_attention_weight_max": 27.813936233520508,
      "activations/layer18_attention_weight_min": -19.155290603637695,
      "activations/layer19_attention_weight_max": 42.763221740722656,
      "activations/layer19_attention_weight_min": -28.45527458190918,
      "activations/layer1_attention_weight_max": 17.41672706604004,
      "activations/layer1_attention_weight_min": -13.981199264526367,
      "activations/layer20_attention_weight_max": 31.08637046813965,
      "activations/layer20_attention_weight_min": -23.577428817749023,
      "activations/layer21_attention_weight_max": 28.906871795654297,
      "activations/layer21_attention_weight_min": -19.58669662475586,
      "activations/layer22_attention_weight_max": 51.1615104675293,
      "activations/layer22_attention_weight_min": -27.1629638671875,
      "activations/layer23_attention_weight_max": 33.671630859375,
      "activations/layer23_attention_weight_min": -19.05793571472168,
      "activations/layer2_attention_weight_max": 30.342388153076172,
      "activations/layer2_attention_weight_min": -30.747610092163086,
      "activations/layer3_attention_weight_max": 92.0666732788086,
      "activations/layer3_attention_weight_min": -91.60202026367188,
      "activations/layer4_attention_weight_max": 126.34669494628906,
      "activations/layer4_attention_weight_min": -121.69210052490234,
      "activations/layer5_attention_weight_max": 74.58805847167969,
      "activations/layer5_attention_weight_min": -69.79334259033203,
      "activations/layer6_attention_weight_max": 53.41043472290039,
      "activations/layer6_attention_weight_min": -55.92844009399414,
      "activations/layer7_attention_weight_max": 84.7042236328125,
      "activations/layer7_attention_weight_min": -86.77137756347656,
      "activations/layer8_attention_weight_max": 39.99483108520508,
      "activations/layer8_attention_weight_min": -40.089447021484375,
      "activations/layer9_attention_weight_max": 37.365196228027344,
      "activations/layer9_attention_weight_min": -34.92722702026367,
      "epoch": 5.87,
      "learning_rate": 0.0001132560606060606,
      "loss": 2.8532,
      "step": 101100
    },
    {
      "activations/layer0_attention_weight_max": 14.710628509521484,
      "activations/layer0_attention_weight_min": -12.408988952636719,
      "activations/layer10_attention_weight_max": 34.27812957763672,
      "activations/layer10_attention_weight_min": -29.584897994995117,
      "activations/layer11_attention_weight_max": 32.28081130981445,
      "activations/layer11_attention_weight_min": -27.317502975463867,
      "activations/layer12_attention_weight_max": 22.815160751342773,
      "activations/layer12_attention_weight_min": -27.98297691345215,
      "activations/layer13_attention_weight_max": 38.29057312011719,
      "activations/layer13_attention_weight_min": -31.409183502197266,
      "activations/layer14_attention_weight_max": 50.78296661376953,
      "activations/layer14_attention_weight_min": -47.341697692871094,
      "activations/layer15_attention_weight_max": 36.73111343383789,
      "activations/layer15_attention_weight_min": -28.235544204711914,
      "activations/layer16_attention_weight_max": 34.062469482421875,
      "activations/layer16_attention_weight_min": -29.78221321105957,
      "activations/layer17_attention_weight_max": 36.514617919921875,
      "activations/layer17_attention_weight_min": -24.80342674255371,
      "activations/layer18_attention_weight_max": 31.128087997436523,
      "activations/layer18_attention_weight_min": -20.41841697692871,
      "activations/layer19_attention_weight_max": 36.91220474243164,
      "activations/layer19_attention_weight_min": -29.47035026550293,
      "activations/layer1_attention_weight_max": 17.331743240356445,
      "activations/layer1_attention_weight_min": -14.245254516601562,
      "activations/layer20_attention_weight_max": 30.88439178466797,
      "activations/layer20_attention_weight_min": -24.085765838623047,
      "activations/layer21_attention_weight_max": 31.38266944885254,
      "activations/layer21_attention_weight_min": -21.2764892578125,
      "activations/layer22_attention_weight_max": 49.57431411743164,
      "activations/layer22_attention_weight_min": -27.070234298706055,
      "activations/layer23_attention_weight_max": 33.137420654296875,
      "activations/layer23_attention_weight_min": -22.559322357177734,
      "activations/layer2_attention_weight_max": 30.373916625976562,
      "activations/layer2_attention_weight_min": -30.722049713134766,
      "activations/layer3_attention_weight_max": 89.85002136230469,
      "activations/layer3_attention_weight_min": -89.98200988769531,
      "activations/layer4_attention_weight_max": 124.80266571044922,
      "activations/layer4_attention_weight_min": -121.96351623535156,
      "activations/layer5_attention_weight_max": 71.15602111816406,
      "activations/layer5_attention_weight_min": -70.04426574707031,
      "activations/layer6_attention_weight_max": 51.29161071777344,
      "activations/layer6_attention_weight_min": -53.506404876708984,
      "activations/layer7_attention_weight_max": 83.5541000366211,
      "activations/layer7_attention_weight_min": -82.56385803222656,
      "activations/layer8_attention_weight_max": 39.02952194213867,
      "activations/layer8_attention_weight_min": -37.833900451660156,
      "activations/layer9_attention_weight_max": 34.98055648803711,
      "activations/layer9_attention_weight_min": -33.20901870727539,
      "epoch": 5.88,
      "learning_rate": 0.00011323712121212119,
      "loss": 2.8414,
      "step": 101150
    },
    {
      "activations/layer0_attention_weight_max": 14.77913761138916,
      "activations/layer0_attention_weight_min": -12.857708930969238,
      "activations/layer10_attention_weight_max": 34.905181884765625,
      "activations/layer10_attention_weight_min": -29.921316146850586,
      "activations/layer11_attention_weight_max": 34.42427444458008,
      "activations/layer11_attention_weight_min": -27.44771385192871,
      "activations/layer12_attention_weight_max": 31.878271102905273,
      "activations/layer12_attention_weight_min": -24.931949615478516,
      "activations/layer13_attention_weight_max": 44.889225006103516,
      "activations/layer13_attention_weight_min": -34.065181732177734,
      "activations/layer14_attention_weight_max": 68.51667785644531,
      "activations/layer14_attention_weight_min": -48.66727828979492,
      "activations/layer15_attention_weight_max": 38.259185791015625,
      "activations/layer15_attention_weight_min": -30.177074432373047,
      "activations/layer16_attention_weight_max": 38.07695007324219,
      "activations/layer16_attention_weight_min": -30.471328735351562,
      "activations/layer17_attention_weight_max": 40.669219970703125,
      "activations/layer17_attention_weight_min": -27.459545135498047,
      "activations/layer18_attention_weight_max": 34.13233947753906,
      "activations/layer18_attention_weight_min": -22.65479850769043,
      "activations/layer19_attention_weight_max": 41.47431564331055,
      "activations/layer19_attention_weight_min": -35.30561828613281,
      "activations/layer1_attention_weight_max": 17.564041137695312,
      "activations/layer1_attention_weight_min": -14.224200248718262,
      "activations/layer20_attention_weight_max": 34.97764587402344,
      "activations/layer20_attention_weight_min": -26.041940689086914,
      "activations/layer21_attention_weight_max": 34.110191345214844,
      "activations/layer21_attention_weight_min": -21.348758697509766,
      "activations/layer22_attention_weight_max": 52.09779357910156,
      "activations/layer22_attention_weight_min": -29.991641998291016,
      "activations/layer23_attention_weight_max": 36.903873443603516,
      "activations/layer23_attention_weight_min": -22.676883697509766,
      "activations/layer2_attention_weight_max": 30.313711166381836,
      "activations/layer2_attention_weight_min": -30.705734252929688,
      "activations/layer3_attention_weight_max": 89.79523468017578,
      "activations/layer3_attention_weight_min": -93.18096160888672,
      "activations/layer4_attention_weight_max": 123.18355560302734,
      "activations/layer4_attention_weight_min": -120.2179946899414,
      "activations/layer5_attention_weight_max": 74.25514221191406,
      "activations/layer5_attention_weight_min": -71.8032455444336,
      "activations/layer6_attention_weight_max": 53.67397689819336,
      "activations/layer6_attention_weight_min": -53.69502639770508,
      "activations/layer7_attention_weight_max": 86.43148803710938,
      "activations/layer7_attention_weight_min": -82.24437713623047,
      "activations/layer8_attention_weight_max": 41.13847732543945,
      "activations/layer8_attention_weight_min": -38.90810012817383,
      "activations/layer9_attention_weight_max": 36.34919357299805,
      "activations/layer9_attention_weight_min": -36.07640075683594,
      "epoch": 5.88,
      "learning_rate": 0.0001132181818181818,
      "loss": 2.8412,
      "step": 101200
    },
    {
      "activations/layer0_attention_weight_max": 15.40461254119873,
      "activations/layer0_attention_weight_min": -12.403975486755371,
      "activations/layer10_attention_weight_max": 31.663089752197266,
      "activations/layer10_attention_weight_min": -29.449417114257812,
      "activations/layer11_attention_weight_max": 32.92002868652344,
      "activations/layer11_attention_weight_min": -25.961200714111328,
      "activations/layer12_attention_weight_max": 48.890785217285156,
      "activations/layer12_attention_weight_min": -28.110658645629883,
      "activations/layer13_attention_weight_max": 60.29080581665039,
      "activations/layer13_attention_weight_min": -39.36348342895508,
      "activations/layer14_attention_weight_max": 62.18089294433594,
      "activations/layer14_attention_weight_min": -43.477684020996094,
      "activations/layer15_attention_weight_max": 52.16091537475586,
      "activations/layer15_attention_weight_min": -31.336196899414062,
      "activations/layer16_attention_weight_max": 38.066070556640625,
      "activations/layer16_attention_weight_min": -26.768022537231445,
      "activations/layer17_attention_weight_max": 37.50408935546875,
      "activations/layer17_attention_weight_min": -25.098098754882812,
      "activations/layer18_attention_weight_max": 36.122291564941406,
      "activations/layer18_attention_weight_min": -20.14592742919922,
      "activations/layer19_attention_weight_max": 41.70808410644531,
      "activations/layer19_attention_weight_min": -28.93667984008789,
      "activations/layer1_attention_weight_max": 16.79145050048828,
      "activations/layer1_attention_weight_min": -14.803437232971191,
      "activations/layer20_attention_weight_max": 34.17384719848633,
      "activations/layer20_attention_weight_min": -23.920141220092773,
      "activations/layer21_attention_weight_max": 32.92320251464844,
      "activations/layer21_attention_weight_min": -21.54689598083496,
      "activations/layer22_attention_weight_max": 50.13037109375,
      "activations/layer22_attention_weight_min": -28.87447738647461,
      "activations/layer23_attention_weight_max": 39.66413879394531,
      "activations/layer23_attention_weight_min": -23.418851852416992,
      "activations/layer2_attention_weight_max": 33.719356536865234,
      "activations/layer2_attention_weight_min": -30.327529907226562,
      "activations/layer3_attention_weight_max": 93.03900146484375,
      "activations/layer3_attention_weight_min": -96.24810791015625,
      "activations/layer4_attention_weight_max": 129.8553924560547,
      "activations/layer4_attention_weight_min": -121.86705780029297,
      "activations/layer5_attention_weight_max": 73.0979232788086,
      "activations/layer5_attention_weight_min": -71.00375366210938,
      "activations/layer6_attention_weight_max": 55.27790451049805,
      "activations/layer6_attention_weight_min": -53.659873962402344,
      "activations/layer7_attention_weight_max": 87.13111877441406,
      "activations/layer7_attention_weight_min": -83.15422058105469,
      "activations/layer8_attention_weight_max": 38.069637298583984,
      "activations/layer8_attention_weight_min": -38.4835205078125,
      "activations/layer9_attention_weight_max": 34.24485397338867,
      "activations/layer9_attention_weight_min": -33.99862289428711,
      "epoch": 5.88,
      "learning_rate": 0.00011319924242424242,
      "loss": 2.8344,
      "step": 101250
    },
    {
      "activations/layer0_attention_weight_max": 15.507159233093262,
      "activations/layer0_attention_weight_min": -12.695919036865234,
      "activations/layer10_attention_weight_max": 31.861295700073242,
      "activations/layer10_attention_weight_min": -30.52574348449707,
      "activations/layer11_attention_weight_max": 33.254150390625,
      "activations/layer11_attention_weight_min": -28.501392364501953,
      "activations/layer12_attention_weight_max": 25.666711807250977,
      "activations/layer12_attention_weight_min": -33.88362503051758,
      "activations/layer13_attention_weight_max": 46.48084259033203,
      "activations/layer13_attention_weight_min": -33.30317306518555,
      "activations/layer14_attention_weight_max": 53.836238861083984,
      "activations/layer14_attention_weight_min": -40.74602127075195,
      "activations/layer15_attention_weight_max": 35.95020294189453,
      "activations/layer15_attention_weight_min": -30.251605987548828,
      "activations/layer16_attention_weight_max": 36.460540771484375,
      "activations/layer16_attention_weight_min": -28.802452087402344,
      "activations/layer17_attention_weight_max": 36.19567108154297,
      "activations/layer17_attention_weight_min": -26.636253356933594,
      "activations/layer18_attention_weight_max": 36.566978454589844,
      "activations/layer18_attention_weight_min": -21.895824432373047,
      "activations/layer19_attention_weight_max": 38.25065231323242,
      "activations/layer19_attention_weight_min": -34.37208557128906,
      "activations/layer1_attention_weight_max": 16.985103607177734,
      "activations/layer1_attention_weight_min": -14.588447570800781,
      "activations/layer20_attention_weight_max": 34.86473846435547,
      "activations/layer20_attention_weight_min": -25.7262020111084,
      "activations/layer21_attention_weight_max": 32.03876495361328,
      "activations/layer21_attention_weight_min": -26.28329849243164,
      "activations/layer22_attention_weight_max": 46.6725959777832,
      "activations/layer22_attention_weight_min": -31.194644927978516,
      "activations/layer23_attention_weight_max": 35.00775909423828,
      "activations/layer23_attention_weight_min": -23.621749877929688,
      "activations/layer2_attention_weight_max": 31.373004913330078,
      "activations/layer2_attention_weight_min": -30.37679672241211,
      "activations/layer3_attention_weight_max": 92.9798812866211,
      "activations/layer3_attention_weight_min": -94.0289535522461,
      "activations/layer4_attention_weight_max": 129.20187377929688,
      "activations/layer4_attention_weight_min": -127.17311096191406,
      "activations/layer5_attention_weight_max": 70.1006851196289,
      "activations/layer5_attention_weight_min": -70.05154418945312,
      "activations/layer6_attention_weight_max": 56.64339065551758,
      "activations/layer6_attention_weight_min": -54.44157028198242,
      "activations/layer7_attention_weight_max": 89.5201416015625,
      "activations/layer7_attention_weight_min": -86.97184753417969,
      "activations/layer8_attention_weight_max": 38.48668670654297,
      "activations/layer8_attention_weight_min": -39.916954040527344,
      "activations/layer9_attention_weight_max": 35.551780700683594,
      "activations/layer9_attention_weight_min": -33.037776947021484,
      "epoch": 5.89,
      "learning_rate": 0.00011318030303030301,
      "loss": 2.8183,
      "step": 101300
    },
    {
      "activations/layer0_attention_weight_max": 15.869025230407715,
      "activations/layer0_attention_weight_min": -11.863602638244629,
      "activations/layer10_attention_weight_max": 36.407230377197266,
      "activations/layer10_attention_weight_min": -31.331857681274414,
      "activations/layer11_attention_weight_max": 33.69635009765625,
      "activations/layer11_attention_weight_min": -28.911041259765625,
      "activations/layer12_attention_weight_max": 25.267669677734375,
      "activations/layer12_attention_weight_min": -26.1275577545166,
      "activations/layer13_attention_weight_max": 45.720848083496094,
      "activations/layer13_attention_weight_min": -31.070825576782227,
      "activations/layer14_attention_weight_max": 62.8542366027832,
      "activations/layer14_attention_weight_min": -45.605628967285156,
      "activations/layer15_attention_weight_max": 41.29597854614258,
      "activations/layer15_attention_weight_min": -30.005706787109375,
      "activations/layer16_attention_weight_max": 38.69718933105469,
      "activations/layer16_attention_weight_min": -29.48298454284668,
      "activations/layer17_attention_weight_max": 40.62160110473633,
      "activations/layer17_attention_weight_min": -23.59663963317871,
      "activations/layer18_attention_weight_max": 39.48372268676758,
      "activations/layer18_attention_weight_min": -21.310543060302734,
      "activations/layer19_attention_weight_max": 41.60630416870117,
      "activations/layer19_attention_weight_min": -31.291961669921875,
      "activations/layer1_attention_weight_max": 16.388158798217773,
      "activations/layer1_attention_weight_min": -16.604393005371094,
      "activations/layer20_attention_weight_max": 35.11737823486328,
      "activations/layer20_attention_weight_min": -25.606412887573242,
      "activations/layer21_attention_weight_max": 34.65861511230469,
      "activations/layer21_attention_weight_min": -21.472949981689453,
      "activations/layer22_attention_weight_max": 57.37570571899414,
      "activations/layer22_attention_weight_min": -29.011892318725586,
      "activations/layer23_attention_weight_max": 36.97038269042969,
      "activations/layer23_attention_weight_min": -23.475021362304688,
      "activations/layer2_attention_weight_max": 32.11625289916992,
      "activations/layer2_attention_weight_min": -30.45331382751465,
      "activations/layer3_attention_weight_max": 92.51248931884766,
      "activations/layer3_attention_weight_min": -91.06358337402344,
      "activations/layer4_attention_weight_max": 133.76400756835938,
      "activations/layer4_attention_weight_min": -124.17522430419922,
      "activations/layer5_attention_weight_max": 76.69218444824219,
      "activations/layer5_attention_weight_min": -74.52113342285156,
      "activations/layer6_attention_weight_max": 56.216522216796875,
      "activations/layer6_attention_weight_min": -56.094364166259766,
      "activations/layer7_attention_weight_max": 88.21285247802734,
      "activations/layer7_attention_weight_min": -85.7123031616211,
      "activations/layer8_attention_weight_max": 42.430728912353516,
      "activations/layer8_attention_weight_min": -43.66377258300781,
      "activations/layer9_attention_weight_max": 37.75381088256836,
      "activations/layer9_attention_weight_min": -35.963104248046875,
      "epoch": 5.89,
      "learning_rate": 0.00011316136363636362,
      "loss": 2.8439,
      "step": 101350
    },
    {
      "activations/layer0_attention_weight_max": 15.540221214294434,
      "activations/layer0_attention_weight_min": -11.209373474121094,
      "activations/layer10_attention_weight_max": 33.667789459228516,
      "activations/layer10_attention_weight_min": -32.119110107421875,
      "activations/layer11_attention_weight_max": 31.496028900146484,
      "activations/layer11_attention_weight_min": -27.72273826599121,
      "activations/layer12_attention_weight_max": 23.092323303222656,
      "activations/layer12_attention_weight_min": -24.92223358154297,
      "activations/layer13_attention_weight_max": 41.83496856689453,
      "activations/layer13_attention_weight_min": -32.1634635925293,
      "activations/layer14_attention_weight_max": 63.58554458618164,
      "activations/layer14_attention_weight_min": -47.63560104370117,
      "activations/layer15_attention_weight_max": 36.782108306884766,
      "activations/layer15_attention_weight_min": -28.185436248779297,
      "activations/layer16_attention_weight_max": 35.14081954956055,
      "activations/layer16_attention_weight_min": -27.319538116455078,
      "activations/layer17_attention_weight_max": 36.105316162109375,
      "activations/layer17_attention_weight_min": -24.035573959350586,
      "activations/layer18_attention_weight_max": 33.59291076660156,
      "activations/layer18_attention_weight_min": -20.592361450195312,
      "activations/layer19_attention_weight_max": 39.66810989379883,
      "activations/layer19_attention_weight_min": -29.144855499267578,
      "activations/layer1_attention_weight_max": 18.004121780395508,
      "activations/layer1_attention_weight_min": -13.15782356262207,
      "activations/layer20_attention_weight_max": 31.474674224853516,
      "activations/layer20_attention_weight_min": -23.667875289916992,
      "activations/layer21_attention_weight_max": 33.78115463256836,
      "activations/layer21_attention_weight_min": -20.644365310668945,
      "activations/layer22_attention_weight_max": 48.9763069152832,
      "activations/layer22_attention_weight_min": -31.391700744628906,
      "activations/layer23_attention_weight_max": 36.5471076965332,
      "activations/layer23_attention_weight_min": -21.5870418548584,
      "activations/layer2_attention_weight_max": 34.008277893066406,
      "activations/layer2_attention_weight_min": -32.156734466552734,
      "activations/layer3_attention_weight_max": 91.51899719238281,
      "activations/layer3_attention_weight_min": -91.54518127441406,
      "activations/layer4_attention_weight_max": 120.3900146484375,
      "activations/layer4_attention_weight_min": -118.98323822021484,
      "activations/layer5_attention_weight_max": 70.01708984375,
      "activations/layer5_attention_weight_min": -67.9993896484375,
      "activations/layer6_attention_weight_max": 51.74667739868164,
      "activations/layer6_attention_weight_min": -52.585838317871094,
      "activations/layer7_attention_weight_max": 83.42620086669922,
      "activations/layer7_attention_weight_min": -82.91767120361328,
      "activations/layer8_attention_weight_max": 40.04667663574219,
      "activations/layer8_attention_weight_min": -39.86137390136719,
      "activations/layer9_attention_weight_max": 36.49076461791992,
      "activations/layer9_attention_weight_min": -34.8514404296875,
      "epoch": 5.89,
      "learning_rate": 0.00011314242424242424,
      "loss": 2.8426,
      "step": 101400
    },
    {
      "activations/layer0_attention_weight_max": 15.714692115783691,
      "activations/layer0_attention_weight_min": -12.202771186828613,
      "activations/layer10_attention_weight_max": 31.266098022460938,
      "activations/layer10_attention_weight_min": -32.93448257446289,
      "activations/layer11_attention_weight_max": 30.676319122314453,
      "activations/layer11_attention_weight_min": -26.864980697631836,
      "activations/layer12_attention_weight_max": 33.019287109375,
      "activations/layer12_attention_weight_min": -28.286657333374023,
      "activations/layer13_attention_weight_max": 40.01983642578125,
      "activations/layer13_attention_weight_min": -30.677446365356445,
      "activations/layer14_attention_weight_max": 62.37822723388672,
      "activations/layer14_attention_weight_min": -41.04330825805664,
      "activations/layer15_attention_weight_max": 36.854854583740234,
      "activations/layer15_attention_weight_min": -29.425823211669922,
      "activations/layer16_attention_weight_max": 36.224205017089844,
      "activations/layer16_attention_weight_min": -26.64947509765625,
      "activations/layer17_attention_weight_max": 39.2341423034668,
      "activations/layer17_attention_weight_min": -26.653610229492188,
      "activations/layer18_attention_weight_max": 37.258583068847656,
      "activations/layer18_attention_weight_min": -20.197214126586914,
      "activations/layer19_attention_weight_max": 38.20036697387695,
      "activations/layer19_attention_weight_min": -29.1307373046875,
      "activations/layer1_attention_weight_max": 17.434797286987305,
      "activations/layer1_attention_weight_min": -14.367083549499512,
      "activations/layer20_attention_weight_max": 37.15440368652344,
      "activations/layer20_attention_weight_min": -23.05855369567871,
      "activations/layer21_attention_weight_max": 35.127479553222656,
      "activations/layer21_attention_weight_min": -20.0717830657959,
      "activations/layer22_attention_weight_max": 46.751312255859375,
      "activations/layer22_attention_weight_min": -26.588830947875977,
      "activations/layer23_attention_weight_max": 34.78200912475586,
      "activations/layer23_attention_weight_min": -21.765256881713867,
      "activations/layer2_attention_weight_max": 30.763896942138672,
      "activations/layer2_attention_weight_min": -29.39189910888672,
      "activations/layer3_attention_weight_max": 90.77559661865234,
      "activations/layer3_attention_weight_min": -90.26273345947266,
      "activations/layer4_attention_weight_max": 129.21128845214844,
      "activations/layer4_attention_weight_min": -119.59598541259766,
      "activations/layer5_attention_weight_max": 75.89692687988281,
      "activations/layer5_attention_weight_min": -69.38871765136719,
      "activations/layer6_attention_weight_max": 53.478275299072266,
      "activations/layer6_attention_weight_min": -55.2266845703125,
      "activations/layer7_attention_weight_max": 92.73157501220703,
      "activations/layer7_attention_weight_min": -81.67549133300781,
      "activations/layer8_attention_weight_max": 38.18012237548828,
      "activations/layer8_attention_weight_min": -37.88835906982422,
      "activations/layer9_attention_weight_max": 35.66721725463867,
      "activations/layer9_attention_weight_min": -36.27195739746094,
      "epoch": 5.89,
      "learning_rate": 0.00011312348484848484,
      "loss": 2.8479,
      "step": 101450
    },
    {
      "activations/layer0_attention_weight_max": 18.572160720825195,
      "activations/layer0_attention_weight_min": -13.917837142944336,
      "activations/layer10_attention_weight_max": 34.56963348388672,
      "activations/layer10_attention_weight_min": -30.396116256713867,
      "activations/layer11_attention_weight_max": 31.559040069580078,
      "activations/layer11_attention_weight_min": -27.354833602905273,
      "activations/layer12_attention_weight_max": 34.91884231567383,
      "activations/layer12_attention_weight_min": -28.449872970581055,
      "activations/layer13_attention_weight_max": 43.156272888183594,
      "activations/layer13_attention_weight_min": -32.398712158203125,
      "activations/layer14_attention_weight_max": 52.5462760925293,
      "activations/layer14_attention_weight_min": -41.21110153198242,
      "activations/layer15_attention_weight_max": 37.68705368041992,
      "activations/layer15_attention_weight_min": -28.15389060974121,
      "activations/layer16_attention_weight_max": 35.52210235595703,
      "activations/layer16_attention_weight_min": -28.08719253540039,
      "activations/layer17_attention_weight_max": 37.04892349243164,
      "activations/layer17_attention_weight_min": -25.814834594726562,
      "activations/layer18_attention_weight_max": 33.46735382080078,
      "activations/layer18_attention_weight_min": -21.216663360595703,
      "activations/layer19_attention_weight_max": 38.843196868896484,
      "activations/layer19_attention_weight_min": -30.528461456298828,
      "activations/layer1_attention_weight_max": 16.598737716674805,
      "activations/layer1_attention_weight_min": -14.9346923828125,
      "activations/layer20_attention_weight_max": 33.21455764770508,
      "activations/layer20_attention_weight_min": -26.583251953125,
      "activations/layer21_attention_weight_max": 31.92582130432129,
      "activations/layer21_attention_weight_min": -23.195817947387695,
      "activations/layer22_attention_weight_max": 49.27073287963867,
      "activations/layer22_attention_weight_min": -30.63827133178711,
      "activations/layer23_attention_weight_max": 36.27131652832031,
      "activations/layer23_attention_weight_min": -20.83292579650879,
      "activations/layer2_attention_weight_max": 31.091596603393555,
      "activations/layer2_attention_weight_min": -30.79511833190918,
      "activations/layer3_attention_weight_max": 94.56690216064453,
      "activations/layer3_attention_weight_min": -91.8121566772461,
      "activations/layer4_attention_weight_max": 132.5003204345703,
      "activations/layer4_attention_weight_min": -119.01029205322266,
      "activations/layer5_attention_weight_max": 77.29708099365234,
      "activations/layer5_attention_weight_min": -70.76145935058594,
      "activations/layer6_attention_weight_max": 55.058135986328125,
      "activations/layer6_attention_weight_min": -54.76295471191406,
      "activations/layer7_attention_weight_max": 87.7975845336914,
      "activations/layer7_attention_weight_min": -85.13994598388672,
      "activations/layer8_attention_weight_max": 44.275264739990234,
      "activations/layer8_attention_weight_min": -39.275508880615234,
      "activations/layer9_attention_weight_max": 35.34928512573242,
      "activations/layer9_attention_weight_min": -33.85139083862305,
      "epoch": 5.9,
      "learning_rate": 0.00011310454545454544,
      "loss": 2.8297,
      "step": 101500
    },
    {
      "activations/layer0_attention_weight_max": 16.496456146240234,
      "activations/layer0_attention_weight_min": -11.382819175720215,
      "activations/layer10_attention_weight_max": 34.754478454589844,
      "activations/layer10_attention_weight_min": -33.343414306640625,
      "activations/layer11_attention_weight_max": 35.11632537841797,
      "activations/layer11_attention_weight_min": -30.152145385742188,
      "activations/layer12_attention_weight_max": 28.076082229614258,
      "activations/layer12_attention_weight_min": -23.185144424438477,
      "activations/layer13_attention_weight_max": 49.64086151123047,
      "activations/layer13_attention_weight_min": -33.54804611206055,
      "activations/layer14_attention_weight_max": 66.12651062011719,
      "activations/layer14_attention_weight_min": -47.98408126831055,
      "activations/layer15_attention_weight_max": 40.609989166259766,
      "activations/layer15_attention_weight_min": -32.61565017700195,
      "activations/layer16_attention_weight_max": 41.32074737548828,
      "activations/layer16_attention_weight_min": -28.78166961669922,
      "activations/layer17_attention_weight_max": 40.395416259765625,
      "activations/layer17_attention_weight_min": -27.059797286987305,
      "activations/layer18_attention_weight_max": 43.09349822998047,
      "activations/layer18_attention_weight_min": -23.025331497192383,
      "activations/layer19_attention_weight_max": 46.9854850769043,
      "activations/layer19_attention_weight_min": -31.346460342407227,
      "activations/layer1_attention_weight_max": 17.298748016357422,
      "activations/layer1_attention_weight_min": -16.385568618774414,
      "activations/layer20_attention_weight_max": 37.95425796508789,
      "activations/layer20_attention_weight_min": -24.467374801635742,
      "activations/layer21_attention_weight_max": 41.498085021972656,
      "activations/layer21_attention_weight_min": -21.821069717407227,
      "activations/layer22_attention_weight_max": 61.22936248779297,
      "activations/layer22_attention_weight_min": -29.48283576965332,
      "activations/layer23_attention_weight_max": 42.46135711669922,
      "activations/layer23_attention_weight_min": -21.225067138671875,
      "activations/layer2_attention_weight_max": 31.686992645263672,
      "activations/layer2_attention_weight_min": -30.9598445892334,
      "activations/layer3_attention_weight_max": 95.5923080444336,
      "activations/layer3_attention_weight_min": -93.5791244506836,
      "activations/layer4_attention_weight_max": 130.666015625,
      "activations/layer4_attention_weight_min": -125.88748931884766,
      "activations/layer5_attention_weight_max": 73.47544860839844,
      "activations/layer5_attention_weight_min": -71.98040771484375,
      "activations/layer6_attention_weight_max": 55.84649658203125,
      "activations/layer6_attention_weight_min": -56.395599365234375,
      "activations/layer7_attention_weight_max": 86.66861724853516,
      "activations/layer7_attention_weight_min": -90.47093963623047,
      "activations/layer8_attention_weight_max": 40.25541305541992,
      "activations/layer8_attention_weight_min": -41.60841751098633,
      "activations/layer9_attention_weight_max": 36.96995162963867,
      "activations/layer9_attention_weight_min": -34.84758758544922,
      "epoch": 5.9,
      "learning_rate": 0.00011308560606060604,
      "loss": 2.8381,
      "step": 101550
    },
    {
      "activations/layer0_attention_weight_max": 16.78274917602539,
      "activations/layer0_attention_weight_min": -10.581650733947754,
      "activations/layer10_attention_weight_max": 34.206398010253906,
      "activations/layer10_attention_weight_min": -33.08478546142578,
      "activations/layer11_attention_weight_max": 33.06889724731445,
      "activations/layer11_attention_weight_min": -30.996862411499023,
      "activations/layer12_attention_weight_max": 27.728574752807617,
      "activations/layer12_attention_weight_min": -27.284135818481445,
      "activations/layer13_attention_weight_max": 57.680564880371094,
      "activations/layer13_attention_weight_min": -45.08742141723633,
      "activations/layer14_attention_weight_max": 60.5396842956543,
      "activations/layer14_attention_weight_min": -53.298255920410156,
      "activations/layer15_attention_weight_max": 43.98343276977539,
      "activations/layer15_attention_weight_min": -36.1525993347168,
      "activations/layer16_attention_weight_max": 36.41047668457031,
      "activations/layer16_attention_weight_min": -28.74933433532715,
      "activations/layer17_attention_weight_max": 37.624576568603516,
      "activations/layer17_attention_weight_min": -27.310029983520508,
      "activations/layer18_attention_weight_max": 32.7586669921875,
      "activations/layer18_attention_weight_min": -21.429405212402344,
      "activations/layer19_attention_weight_max": 39.70677185058594,
      "activations/layer19_attention_weight_min": -30.938779830932617,
      "activations/layer1_attention_weight_max": 17.000944137573242,
      "activations/layer1_attention_weight_min": -14.991537094116211,
      "activations/layer20_attention_weight_max": 37.31266403198242,
      "activations/layer20_attention_weight_min": -24.125408172607422,
      "activations/layer21_attention_weight_max": 34.310630798339844,
      "activations/layer21_attention_weight_min": -20.757373809814453,
      "activations/layer22_attention_weight_max": 56.01345443725586,
      "activations/layer22_attention_weight_min": -30.031877517700195,
      "activations/layer23_attention_weight_max": 34.93866729736328,
      "activations/layer23_attention_weight_min": -22.588415145874023,
      "activations/layer2_attention_weight_max": 31.71036148071289,
      "activations/layer2_attention_weight_min": -31.70758056640625,
      "activations/layer3_attention_weight_max": 92.55389404296875,
      "activations/layer3_attention_weight_min": -93.72605895996094,
      "activations/layer4_attention_weight_max": 131.46621704101562,
      "activations/layer4_attention_weight_min": -125.59114837646484,
      "activations/layer5_attention_weight_max": 75.03602600097656,
      "activations/layer5_attention_weight_min": -71.30294799804688,
      "activations/layer6_attention_weight_max": 56.52909851074219,
      "activations/layer6_attention_weight_min": -59.89079666137695,
      "activations/layer7_attention_weight_max": 91.70281982421875,
      "activations/layer7_attention_weight_min": -89.48490142822266,
      "activations/layer8_attention_weight_max": 41.63264846801758,
      "activations/layer8_attention_weight_min": -45.51644515991211,
      "activations/layer9_attention_weight_max": 39.23370361328125,
      "activations/layer9_attention_weight_min": -36.32072067260742,
      "epoch": 5.9,
      "learning_rate": 0.00011306666666666666,
      "loss": 2.8266,
      "step": 101600
    },
    {
      "activations/layer0_attention_weight_max": 15.947531700134277,
      "activations/layer0_attention_weight_min": -11.344598770141602,
      "activations/layer10_attention_weight_max": 33.123802185058594,
      "activations/layer10_attention_weight_min": -30.95332908630371,
      "activations/layer11_attention_weight_max": 34.43739318847656,
      "activations/layer11_attention_weight_min": -27.838115692138672,
      "activations/layer12_attention_weight_max": 51.36301040649414,
      "activations/layer12_attention_weight_min": -28.714561462402344,
      "activations/layer13_attention_weight_max": 53.24771499633789,
      "activations/layer13_attention_weight_min": -32.539100646972656,
      "activations/layer14_attention_weight_max": 64.42459869384766,
      "activations/layer14_attention_weight_min": -45.700538635253906,
      "activations/layer15_attention_weight_max": 43.33137893676758,
      "activations/layer15_attention_weight_min": -28.531190872192383,
      "activations/layer16_attention_weight_max": 36.59596252441406,
      "activations/layer16_attention_weight_min": -27.068038940429688,
      "activations/layer17_attention_weight_max": 37.70368576049805,
      "activations/layer17_attention_weight_min": -25.69281578063965,
      "activations/layer18_attention_weight_max": 41.17808151245117,
      "activations/layer18_attention_weight_min": -19.793136596679688,
      "activations/layer19_attention_weight_max": 43.385047912597656,
      "activations/layer19_attention_weight_min": -29.918792724609375,
      "activations/layer1_attention_weight_max": 16.812883377075195,
      "activations/layer1_attention_weight_min": -13.159233093261719,
      "activations/layer20_attention_weight_max": 34.873043060302734,
      "activations/layer20_attention_weight_min": -23.90666389465332,
      "activations/layer21_attention_weight_max": 39.722564697265625,
      "activations/layer21_attention_weight_min": -21.153905868530273,
      "activations/layer22_attention_weight_max": 52.240909576416016,
      "activations/layer22_attention_weight_min": -27.505300521850586,
      "activations/layer23_attention_weight_max": 36.89676284790039,
      "activations/layer23_attention_weight_min": -20.356727600097656,
      "activations/layer2_attention_weight_max": 31.888166427612305,
      "activations/layer2_attention_weight_min": -31.90191650390625,
      "activations/layer3_attention_weight_max": 92.85870361328125,
      "activations/layer3_attention_weight_min": -90.093017578125,
      "activations/layer4_attention_weight_max": 123.4792251586914,
      "activations/layer4_attention_weight_min": -123.62646484375,
      "activations/layer5_attention_weight_max": 72.72489166259766,
      "activations/layer5_attention_weight_min": -72.4955825805664,
      "activations/layer6_attention_weight_max": 54.426937103271484,
      "activations/layer6_attention_weight_min": -58.1377067565918,
      "activations/layer7_attention_weight_max": 90.2516098022461,
      "activations/layer7_attention_weight_min": -89.78665924072266,
      "activations/layer8_attention_weight_max": 39.02737045288086,
      "activations/layer8_attention_weight_min": -40.07988357543945,
      "activations/layer9_attention_weight_max": 36.02976608276367,
      "activations/layer9_attention_weight_min": -35.163082122802734,
      "epoch": 5.91,
      "learning_rate": 0.00011304772727272727,
      "loss": 2.8377,
      "step": 101650
    },
    {
      "activations/layer0_attention_weight_max": 16.11513328552246,
      "activations/layer0_attention_weight_min": -11.312108039855957,
      "activations/layer10_attention_weight_max": 33.50244140625,
      "activations/layer10_attention_weight_min": -33.51177978515625,
      "activations/layer11_attention_weight_max": 32.677833557128906,
      "activations/layer11_attention_weight_min": -31.002052307128906,
      "activations/layer12_attention_weight_max": 27.314205169677734,
      "activations/layer12_attention_weight_min": -34.73264694213867,
      "activations/layer13_attention_weight_max": 41.71598815917969,
      "activations/layer13_attention_weight_min": -35.13398742675781,
      "activations/layer14_attention_weight_max": 65.56729888916016,
      "activations/layer14_attention_weight_min": -56.02692413330078,
      "activations/layer15_attention_weight_max": 45.505958557128906,
      "activations/layer15_attention_weight_min": -30.750808715820312,
      "activations/layer16_attention_weight_max": 36.96370315551758,
      "activations/layer16_attention_weight_min": -28.034805297851562,
      "activations/layer17_attention_weight_max": 37.36002731323242,
      "activations/layer17_attention_weight_min": -26.02269172668457,
      "activations/layer18_attention_weight_max": 36.7498893737793,
      "activations/layer18_attention_weight_min": -23.08476448059082,
      "activations/layer19_attention_weight_max": 46.287601470947266,
      "activations/layer19_attention_weight_min": -30.823362350463867,
      "activations/layer1_attention_weight_max": 16.697120666503906,
      "activations/layer1_attention_weight_min": -14.540714263916016,
      "activations/layer20_attention_weight_max": 35.61442565917969,
      "activations/layer20_attention_weight_min": -24.161951065063477,
      "activations/layer21_attention_weight_max": 45.842227935791016,
      "activations/layer21_attention_weight_min": -21.571584701538086,
      "activations/layer22_attention_weight_max": 54.80315017700195,
      "activations/layer22_attention_weight_min": -27.961408615112305,
      "activations/layer23_attention_weight_max": 38.42755126953125,
      "activations/layer23_attention_weight_min": -22.460975646972656,
      "activations/layer2_attention_weight_max": 34.03397750854492,
      "activations/layer2_attention_weight_min": -32.32722473144531,
      "activations/layer3_attention_weight_max": 96.17601776123047,
      "activations/layer3_attention_weight_min": -102.11800384521484,
      "activations/layer4_attention_weight_max": 134.0443572998047,
      "activations/layer4_attention_weight_min": -137.86587524414062,
      "activations/layer5_attention_weight_max": 78.40327453613281,
      "activations/layer5_attention_weight_min": -72.96484375,
      "activations/layer6_attention_weight_max": 55.68280792236328,
      "activations/layer6_attention_weight_min": -56.92282485961914,
      "activations/layer7_attention_weight_max": 91.34272003173828,
      "activations/layer7_attention_weight_min": -84.4780044555664,
      "activations/layer8_attention_weight_max": 42.9767951965332,
      "activations/layer8_attention_weight_min": -40.57769775390625,
      "activations/layer9_attention_weight_max": 36.5576057434082,
      "activations/layer9_attention_weight_min": -38.3399543762207,
      "epoch": 5.91,
      "learning_rate": 0.00011302878787878786,
      "loss": 2.8373,
      "step": 101700
    },
    {
      "activations/layer0_attention_weight_max": 15.574691772460938,
      "activations/layer0_attention_weight_min": -10.787461280822754,
      "activations/layer10_attention_weight_max": 36.4634895324707,
      "activations/layer10_attention_weight_min": -33.51137161254883,
      "activations/layer11_attention_weight_max": 34.92047882080078,
      "activations/layer11_attention_weight_min": -29.277379989624023,
      "activations/layer12_attention_weight_max": 59.110931396484375,
      "activations/layer12_attention_weight_min": -34.8972282409668,
      "activations/layer13_attention_weight_max": 69.74134826660156,
      "activations/layer13_attention_weight_min": -41.19643020629883,
      "activations/layer14_attention_weight_max": 55.33671188354492,
      "activations/layer14_attention_weight_min": -40.11937713623047,
      "activations/layer15_attention_weight_max": 60.915428161621094,
      "activations/layer15_attention_weight_min": -33.073326110839844,
      "activations/layer16_attention_weight_max": 31.031049728393555,
      "activations/layer16_attention_weight_min": -25.66339874267578,
      "activations/layer17_attention_weight_max": 33.72515106201172,
      "activations/layer17_attention_weight_min": -22.642213821411133,
      "activations/layer18_attention_weight_max": 27.12553596496582,
      "activations/layer18_attention_weight_min": -19.28910255432129,
      "activations/layer19_attention_weight_max": 33.387332916259766,
      "activations/layer19_attention_weight_min": -31.231592178344727,
      "activations/layer1_attention_weight_max": 17.260805130004883,
      "activations/layer1_attention_weight_min": -13.849981307983398,
      "activations/layer20_attention_weight_max": 28.887672424316406,
      "activations/layer20_attention_weight_min": -21.96438217163086,
      "activations/layer21_attention_weight_max": 26.44937515258789,
      "activations/layer21_attention_weight_min": -19.22344207763672,
      "activations/layer22_attention_weight_max": 41.38419723510742,
      "activations/layer22_attention_weight_min": -27.3242244720459,
      "activations/layer23_attention_weight_max": 27.655574798583984,
      "activations/layer23_attention_weight_min": -20.728561401367188,
      "activations/layer2_attention_weight_max": 32.30992126464844,
      "activations/layer2_attention_weight_min": -30.728126525878906,
      "activations/layer3_attention_weight_max": 92.5699691772461,
      "activations/layer3_attention_weight_min": -94.45604705810547,
      "activations/layer4_attention_weight_max": 130.78025817871094,
      "activations/layer4_attention_weight_min": -127.93273162841797,
      "activations/layer5_attention_weight_max": 73.10295104980469,
      "activations/layer5_attention_weight_min": -72.5329818725586,
      "activations/layer6_attention_weight_max": 56.77207946777344,
      "activations/layer6_attention_weight_min": -55.598941802978516,
      "activations/layer7_attention_weight_max": 98.66761016845703,
      "activations/layer7_attention_weight_min": -89.94860076904297,
      "activations/layer8_attention_weight_max": 40.49663543701172,
      "activations/layer8_attention_weight_min": -39.18729019165039,
      "activations/layer9_attention_weight_max": 36.994285583496094,
      "activations/layer9_attention_weight_min": -38.637630462646484,
      "epoch": 5.91,
      "learning_rate": 0.00011300984848484848,
      "loss": 2.8284,
      "step": 101750
    },
    {
      "activations/layer0_attention_weight_max": 15.781387329101562,
      "activations/layer0_attention_weight_min": -11.936225891113281,
      "activations/layer10_attention_weight_max": 36.186275482177734,
      "activations/layer10_attention_weight_min": -32.329315185546875,
      "activations/layer11_attention_weight_max": 38.14369583129883,
      "activations/layer11_attention_weight_min": -29.99049186706543,
      "activations/layer12_attention_weight_max": 21.93931007385254,
      "activations/layer12_attention_weight_min": -29.313383102416992,
      "activations/layer13_attention_weight_max": 41.03004837036133,
      "activations/layer13_attention_weight_min": -30.805757522583008,
      "activations/layer14_attention_weight_max": 53.04873275756836,
      "activations/layer14_attention_weight_min": -44.09754943847656,
      "activations/layer15_attention_weight_max": 41.52185821533203,
      "activations/layer15_attention_weight_min": -30.243364334106445,
      "activations/layer16_attention_weight_max": 35.370723724365234,
      "activations/layer16_attention_weight_min": -29.88212013244629,
      "activations/layer17_attention_weight_max": 36.239845275878906,
      "activations/layer17_attention_weight_min": -24.34929847717285,
      "activations/layer18_attention_weight_max": 27.87647819519043,
      "activations/layer18_attention_weight_min": -21.82240867614746,
      "activations/layer19_attention_weight_max": 34.540550231933594,
      "activations/layer19_attention_weight_min": -30.257543563842773,
      "activations/layer1_attention_weight_max": 16.98546028137207,
      "activations/layer1_attention_weight_min": -15.205323219299316,
      "activations/layer20_attention_weight_max": 32.7022705078125,
      "activations/layer20_attention_weight_min": -23.27073860168457,
      "activations/layer21_attention_weight_max": 29.232988357543945,
      "activations/layer21_attention_weight_min": -21.42835235595703,
      "activations/layer22_attention_weight_max": 47.97740173339844,
      "activations/layer22_attention_weight_min": -27.184223175048828,
      "activations/layer23_attention_weight_max": 33.5853157043457,
      "activations/layer23_attention_weight_min": -21.057857513427734,
      "activations/layer2_attention_weight_max": 31.205482482910156,
      "activations/layer2_attention_weight_min": -31.81255340576172,
      "activations/layer3_attention_weight_max": 91.2672119140625,
      "activations/layer3_attention_weight_min": -95.01589965820312,
      "activations/layer4_attention_weight_max": 128.0033416748047,
      "activations/layer4_attention_weight_min": -133.989501953125,
      "activations/layer5_attention_weight_max": 73.06900024414062,
      "activations/layer5_attention_weight_min": -76.0383071899414,
      "activations/layer6_attention_weight_max": 54.51481628417969,
      "activations/layer6_attention_weight_min": -57.26762008666992,
      "activations/layer7_attention_weight_max": 87.96864318847656,
      "activations/layer7_attention_weight_min": -95.71710205078125,
      "activations/layer8_attention_weight_max": 43.02641677856445,
      "activations/layer8_attention_weight_min": -42.29523468017578,
      "activations/layer9_attention_weight_max": 37.68049240112305,
      "activations/layer9_attention_weight_min": -35.704200744628906,
      "epoch": 5.92,
      "learning_rate": 0.00011299090909090909,
      "loss": 2.8114,
      "step": 101800
    },
    {
      "activations/layer0_attention_weight_max": 15.459619522094727,
      "activations/layer0_attention_weight_min": -12.19498062133789,
      "activations/layer10_attention_weight_max": 47.35887145996094,
      "activations/layer10_attention_weight_min": -41.17548751831055,
      "activations/layer11_attention_weight_max": 49.268699645996094,
      "activations/layer11_attention_weight_min": -34.60569763183594,
      "activations/layer12_attention_weight_max": 55.440040588378906,
      "activations/layer12_attention_weight_min": -28.632312774658203,
      "activations/layer13_attention_weight_max": 85.16505432128906,
      "activations/layer13_attention_weight_min": -49.018978118896484,
      "activations/layer14_attention_weight_max": 95.86062622070312,
      "activations/layer14_attention_weight_min": -53.008209228515625,
      "activations/layer15_attention_weight_max": 89.82148742675781,
      "activations/layer15_attention_weight_min": -40.44875717163086,
      "activations/layer16_attention_weight_max": 44.53456497192383,
      "activations/layer16_attention_weight_min": -27.577409744262695,
      "activations/layer17_attention_weight_max": 35.24565505981445,
      "activations/layer17_attention_weight_min": -24.518381118774414,
      "activations/layer18_attention_weight_max": 32.59116744995117,
      "activations/layer18_attention_weight_min": -23.291790008544922,
      "activations/layer19_attention_weight_max": 41.32526779174805,
      "activations/layer19_attention_weight_min": -28.527753829956055,
      "activations/layer1_attention_weight_max": 18.618865966796875,
      "activations/layer1_attention_weight_min": -14.292699813842773,
      "activations/layer20_attention_weight_max": 32.14376449584961,
      "activations/layer20_attention_weight_min": -22.13656997680664,
      "activations/layer21_attention_weight_max": 28.377914428710938,
      "activations/layer21_attention_weight_min": -21.241912841796875,
      "activations/layer22_attention_weight_max": 46.5362548828125,
      "activations/layer22_attention_weight_min": -27.615234375,
      "activations/layer23_attention_weight_max": 31.104400634765625,
      "activations/layer23_attention_weight_min": -21.921491622924805,
      "activations/layer2_attention_weight_max": 34.6309814453125,
      "activations/layer2_attention_weight_min": -33.357154846191406,
      "activations/layer3_attention_weight_max": 102.78244018554688,
      "activations/layer3_attention_weight_min": -100.64994812011719,
      "activations/layer4_attention_weight_max": 148.60020446777344,
      "activations/layer4_attention_weight_min": -139.45249938964844,
      "activations/layer5_attention_weight_max": 81.79815673828125,
      "activations/layer5_attention_weight_min": -76.8829574584961,
      "activations/layer6_attention_weight_max": 62.097991943359375,
      "activations/layer6_attention_weight_min": -62.318946838378906,
      "activations/layer7_attention_weight_max": 110.9479751586914,
      "activations/layer7_attention_weight_min": -104.88854217529297,
      "activations/layer8_attention_weight_max": 55.62481689453125,
      "activations/layer8_attention_weight_min": -50.629600524902344,
      "activations/layer9_attention_weight_max": 48.875770568847656,
      "activations/layer9_attention_weight_min": -43.10110092163086,
      "epoch": 5.92,
      "learning_rate": 0.00011297196969696968,
      "loss": 2.8457,
      "step": 101850
    },
    {
      "activations/layer0_attention_weight_max": 16.144336700439453,
      "activations/layer0_attention_weight_min": -12.684592247009277,
      "activations/layer10_attention_weight_max": 34.20432662963867,
      "activations/layer10_attention_weight_min": -28.65073013305664,
      "activations/layer11_attention_weight_max": 32.66370391845703,
      "activations/layer11_attention_weight_min": -27.23269271850586,
      "activations/layer12_attention_weight_max": 26.047208786010742,
      "activations/layer12_attention_weight_min": -25.678058624267578,
      "activations/layer13_attention_weight_max": 45.3365592956543,
      "activations/layer13_attention_weight_min": -30.510793685913086,
      "activations/layer14_attention_weight_max": 57.391422271728516,
      "activations/layer14_attention_weight_min": -42.649497985839844,
      "activations/layer15_attention_weight_max": 38.52366638183594,
      "activations/layer15_attention_weight_min": -27.83551788330078,
      "activations/layer16_attention_weight_max": 34.632144927978516,
      "activations/layer16_attention_weight_min": -26.76096534729004,
      "activations/layer17_attention_weight_max": 38.3712158203125,
      "activations/layer17_attention_weight_min": -25.692203521728516,
      "activations/layer18_attention_weight_max": 33.99840545654297,
      "activations/layer18_attention_weight_min": -19.910985946655273,
      "activations/layer19_attention_weight_max": 40.66867446899414,
      "activations/layer19_attention_weight_min": -28.73225212097168,
      "activations/layer1_attention_weight_max": 17.503894805908203,
      "activations/layer1_attention_weight_min": -14.460081100463867,
      "activations/layer20_attention_weight_max": 33.850990295410156,
      "activations/layer20_attention_weight_min": -22.621793746948242,
      "activations/layer21_attention_weight_max": 32.81962966918945,
      "activations/layer21_attention_weight_min": -19.88857650756836,
      "activations/layer22_attention_weight_max": 47.14563751220703,
      "activations/layer22_attention_weight_min": -26.628036499023438,
      "activations/layer23_attention_weight_max": 33.887447357177734,
      "activations/layer23_attention_weight_min": -21.513418197631836,
      "activations/layer2_attention_weight_max": 31.334808349609375,
      "activations/layer2_attention_weight_min": -31.42732810974121,
      "activations/layer3_attention_weight_max": 88.32369232177734,
      "activations/layer3_attention_weight_min": -93.03607177734375,
      "activations/layer4_attention_weight_max": 121.43904113769531,
      "activations/layer4_attention_weight_min": -122.22464752197266,
      "activations/layer5_attention_weight_max": 72.99066162109375,
      "activations/layer5_attention_weight_min": -68.58737182617188,
      "activations/layer6_attention_weight_max": 56.69363784790039,
      "activations/layer6_attention_weight_min": -52.291358947753906,
      "activations/layer7_attention_weight_max": 83.5984115600586,
      "activations/layer7_attention_weight_min": -86.9857406616211,
      "activations/layer8_attention_weight_max": 40.78608322143555,
      "activations/layer8_attention_weight_min": -39.61781692504883,
      "activations/layer9_attention_weight_max": 35.715553283691406,
      "activations/layer9_attention_weight_min": -34.73185348510742,
      "epoch": 5.92,
      "learning_rate": 0.0001129530303030303,
      "loss": 2.8325,
      "step": 101900
    },
    {
      "activations/layer0_attention_weight_max": 14.44365406036377,
      "activations/layer0_attention_weight_min": -10.603856086730957,
      "activations/layer10_attention_weight_max": 35.13885498046875,
      "activations/layer10_attention_weight_min": -32.25355911254883,
      "activations/layer11_attention_weight_max": 40.74283218383789,
      "activations/layer11_attention_weight_min": -29.62436294555664,
      "activations/layer12_attention_weight_max": 33.44566345214844,
      "activations/layer12_attention_weight_min": -26.734827041625977,
      "activations/layer13_attention_weight_max": 48.391746520996094,
      "activations/layer13_attention_weight_min": -35.91033935546875,
      "activations/layer14_attention_weight_max": 62.308685302734375,
      "activations/layer14_attention_weight_min": -45.20547103881836,
      "activations/layer15_attention_weight_max": 38.757713317871094,
      "activations/layer15_attention_weight_min": -29.479000091552734,
      "activations/layer16_attention_weight_max": 38.93227767944336,
      "activations/layer16_attention_weight_min": -30.304763793945312,
      "activations/layer17_attention_weight_max": 38.71463394165039,
      "activations/layer17_attention_weight_min": -25.75164794921875,
      "activations/layer18_attention_weight_max": 40.062049865722656,
      "activations/layer18_attention_weight_min": -20.144350051879883,
      "activations/layer19_attention_weight_max": 41.95140838623047,
      "activations/layer19_attention_weight_min": -31.251962661743164,
      "activations/layer1_attention_weight_max": 18.875850677490234,
      "activations/layer1_attention_weight_min": -14.378432273864746,
      "activations/layer20_attention_weight_max": 33.88856506347656,
      "activations/layer20_attention_weight_min": -23.0955867767334,
      "activations/layer21_attention_weight_max": 31.96489143371582,
      "activations/layer21_attention_weight_min": -20.92409324645996,
      "activations/layer22_attention_weight_max": 56.5106315612793,
      "activations/layer22_attention_weight_min": -27.08003807067871,
      "activations/layer23_attention_weight_max": 36.58304214477539,
      "activations/layer23_attention_weight_min": -20.47899627685547,
      "activations/layer2_attention_weight_max": 31.33843231201172,
      "activations/layer2_attention_weight_min": -31.218402862548828,
      "activations/layer3_attention_weight_max": 89.22438049316406,
      "activations/layer3_attention_weight_min": -92.7202377319336,
      "activations/layer4_attention_weight_max": 124.41556549072266,
      "activations/layer4_attention_weight_min": -126.0115737915039,
      "activations/layer5_attention_weight_max": 73.32003021240234,
      "activations/layer5_attention_weight_min": -71.42626953125,
      "activations/layer6_attention_weight_max": 52.87619400024414,
      "activations/layer6_attention_weight_min": -54.11239242553711,
      "activations/layer7_attention_weight_max": 86.8608627319336,
      "activations/layer7_attention_weight_min": -86.90036010742188,
      "activations/layer8_attention_weight_max": 42.44865798950195,
      "activations/layer8_attention_weight_min": -41.00069808959961,
      "activations/layer9_attention_weight_max": 39.6184196472168,
      "activations/layer9_attention_weight_min": -44.119834899902344,
      "epoch": 5.92,
      "learning_rate": 0.00011293409090909091,
      "loss": 2.8328,
      "step": 101950
    },
    {
      "activations/layer0_attention_weight_max": 15.076996803283691,
      "activations/layer0_attention_weight_min": -11.421460151672363,
      "activations/layer10_attention_weight_max": 33.5660285949707,
      "activations/layer10_attention_weight_min": -32.300167083740234,
      "activations/layer11_attention_weight_max": 33.6757926940918,
      "activations/layer11_attention_weight_min": -28.309728622436523,
      "activations/layer12_attention_weight_max": 25.473386764526367,
      "activations/layer12_attention_weight_min": -27.486534118652344,
      "activations/layer13_attention_weight_max": 48.03044891357422,
      "activations/layer13_attention_weight_min": -33.35533142089844,
      "activations/layer14_attention_weight_max": 67.02371215820312,
      "activations/layer14_attention_weight_min": -46.443538665771484,
      "activations/layer15_attention_weight_max": 39.65678405761719,
      "activations/layer15_attention_weight_min": -28.381227493286133,
      "activations/layer16_attention_weight_max": 38.22901916503906,
      "activations/layer16_attention_weight_min": -27.782455444335938,
      "activations/layer17_attention_weight_max": 35.97905349731445,
      "activations/layer17_attention_weight_min": -24.727930068969727,
      "activations/layer18_attention_weight_max": 30.9082088470459,
      "activations/layer18_attention_weight_min": -19.628292083740234,
      "activations/layer19_attention_weight_max": 40.52725601196289,
      "activations/layer19_attention_weight_min": -30.689111709594727,
      "activations/layer1_attention_weight_max": 17.846397399902344,
      "activations/layer1_attention_weight_min": -12.950714111328125,
      "activations/layer20_attention_weight_max": 31.779340744018555,
      "activations/layer20_attention_weight_min": -23.11025047302246,
      "activations/layer21_attention_weight_max": 32.54233932495117,
      "activations/layer21_attention_weight_min": -20.881223678588867,
      "activations/layer22_attention_weight_max": 53.35044860839844,
      "activations/layer22_attention_weight_min": -26.70575523376465,
      "activations/layer23_attention_weight_max": 32.452362060546875,
      "activations/layer23_attention_weight_min": -19.83676528930664,
      "activations/layer2_attention_weight_max": 32.18758773803711,
      "activations/layer2_attention_weight_min": -31.440731048583984,
      "activations/layer3_attention_weight_max": 93.56713104248047,
      "activations/layer3_attention_weight_min": -96.8976821899414,
      "activations/layer4_attention_weight_max": 127.158935546875,
      "activations/layer4_attention_weight_min": -131.92837524414062,
      "activations/layer5_attention_weight_max": 72.82478332519531,
      "activations/layer5_attention_weight_min": -72.58369445800781,
      "activations/layer6_attention_weight_max": 55.46208572387695,
      "activations/layer6_attention_weight_min": -55.010650634765625,
      "activations/layer7_attention_weight_max": 87.80256652832031,
      "activations/layer7_attention_weight_min": -87.42969512939453,
      "activations/layer8_attention_weight_max": 41.24287796020508,
      "activations/layer8_attention_weight_min": -44.16461181640625,
      "activations/layer9_attention_weight_max": 37.06269073486328,
      "activations/layer9_attention_weight_min": -33.39909362792969,
      "epoch": 5.93,
      "learning_rate": 0.0001129151515151515,
      "loss": 2.854,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_loss": 2.78515625,
      "eval_runtime": 8.4745,
      "eval_samples_per_second": 506.695,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_openwebtext_loss": 2.78515625,
      "eval_openwebtext_ppl": 16.202349296067883,
      "eval_openwebtext_runtime": 8.4745,
      "eval_openwebtext_samples_per_second": 506.695,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_wikitext_loss": 3.0078125,
      "eval_wikitext_ppl": 20.24306974171993,
      "eval_wikitext_runtime": 2.0101,
      "eval_wikitext_samples_per_second": 226.86,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_lambada_loss": 2.646484375,
      "eval_lambada_ppl": 14.104365720113616,
      "eval_lambada_runtime": 9.533,
      "eval_lambada_samples_per_second": 510.751,
      "step": 102000
    },
    {
      "activations/layer0_attention_weight_max": 16.432077407836914,
      "activations/layer0_attention_weight_min": -12.069313049316406,
      "activations/layer10_attention_weight_max": 32.544795989990234,
      "activations/layer10_attention_weight_min": -31.19586753845215,
      "activations/layer11_attention_weight_max": 32.036231994628906,
      "activations/layer11_attention_weight_min": -28.41318130493164,
      "activations/layer12_attention_weight_max": 23.49891471862793,
      "activations/layer12_attention_weight_min": -24.641910552978516,
      "activations/layer13_attention_weight_max": 40.249351501464844,
      "activations/layer13_attention_weight_min": -31.220882415771484,
      "activations/layer14_attention_weight_max": 58.683433532714844,
      "activations/layer14_attention_weight_min": -46.28724670410156,
      "activations/layer15_attention_weight_max": 40.094261169433594,
      "activations/layer15_attention_weight_min": -27.53290557861328,
      "activations/layer16_attention_weight_max": 37.11244583129883,
      "activations/layer16_attention_weight_min": -28.573623657226562,
      "activations/layer17_attention_weight_max": 37.368465423583984,
      "activations/layer17_attention_weight_min": -25.21123504638672,
      "activations/layer18_attention_weight_max": 33.72633743286133,
      "activations/layer18_attention_weight_min": -23.60124969482422,
      "activations/layer19_attention_weight_max": 36.48499298095703,
      "activations/layer19_attention_weight_min": -29.583250045776367,
      "activations/layer1_attention_weight_max": 17.490156173706055,
      "activations/layer1_attention_weight_min": -14.791374206542969,
      "activations/layer20_attention_weight_max": 30.50616455078125,
      "activations/layer20_attention_weight_min": -23.8193359375,
      "activations/layer21_attention_weight_max": 32.503082275390625,
      "activations/layer21_attention_weight_min": -23.025060653686523,
      "activations/layer22_attention_weight_max": 47.2017936706543,
      "activations/layer22_attention_weight_min": -28.02638053894043,
      "activations/layer23_attention_weight_max": 32.92713928222656,
      "activations/layer23_attention_weight_min": -21.075857162475586,
      "activations/layer2_attention_weight_max": 31.745229721069336,
      "activations/layer2_attention_weight_min": -32.31029510498047,
      "activations/layer3_attention_weight_max": 90.53681945800781,
      "activations/layer3_attention_weight_min": -95.62504577636719,
      "activations/layer4_attention_weight_max": 133.0322723388672,
      "activations/layer4_attention_weight_min": -127.19050598144531,
      "activations/layer5_attention_weight_max": 79.23977661132812,
      "activations/layer5_attention_weight_min": -72.6460952758789,
      "activations/layer6_attention_weight_max": 57.33617401123047,
      "activations/layer6_attention_weight_min": -56.414161682128906,
      "activations/layer7_attention_weight_max": 84.76324462890625,
      "activations/layer7_attention_weight_min": -88.52325439453125,
      "activations/layer8_attention_weight_max": 39.79777908325195,
      "activations/layer8_attention_weight_min": -40.32741165161133,
      "activations/layer9_attention_weight_max": 34.902400970458984,
      "activations/layer9_attention_weight_min": -36.93545913696289,
      "epoch": 5.93,
      "learning_rate": 0.00011289621212121211,
      "loss": 2.8273,
      "step": 102050
    },
    {
      "activations/layer0_attention_weight_max": 14.633712768554688,
      "activations/layer0_attention_weight_min": -11.798102378845215,
      "activations/layer10_attention_weight_max": 31.89828109741211,
      "activations/layer10_attention_weight_min": -33.44765090942383,
      "activations/layer11_attention_weight_max": 32.132652282714844,
      "activations/layer11_attention_weight_min": -29.824796676635742,
      "activations/layer12_attention_weight_max": 24.487178802490234,
      "activations/layer12_attention_weight_min": -31.264833450317383,
      "activations/layer13_attention_weight_max": 42.472625732421875,
      "activations/layer13_attention_weight_min": -32.49435043334961,
      "activations/layer14_attention_weight_max": 51.314537048339844,
      "activations/layer14_attention_weight_min": -40.91814041137695,
      "activations/layer15_attention_weight_max": 36.85874557495117,
      "activations/layer15_attention_weight_min": -31.709978103637695,
      "activations/layer16_attention_weight_max": 35.23564147949219,
      "activations/layer16_attention_weight_min": -27.923723220825195,
      "activations/layer17_attention_weight_max": 34.43220520019531,
      "activations/layer17_attention_weight_min": -25.373504638671875,
      "activations/layer18_attention_weight_max": 27.92742347717285,
      "activations/layer18_attention_weight_min": -21.09259605407715,
      "activations/layer19_attention_weight_max": 36.75429153442383,
      "activations/layer19_attention_weight_min": -28.39713478088379,
      "activations/layer1_attention_weight_max": 16.95274543762207,
      "activations/layer1_attention_weight_min": -15.058586120605469,
      "activations/layer20_attention_weight_max": 29.131574630737305,
      "activations/layer20_attention_weight_min": -23.29500389099121,
      "activations/layer21_attention_weight_max": 29.649276733398438,
      "activations/layer21_attention_weight_min": -20.1766357421875,
      "activations/layer22_attention_weight_max": 41.9924430847168,
      "activations/layer22_attention_weight_min": -27.150056838989258,
      "activations/layer23_attention_weight_max": 31.619220733642578,
      "activations/layer23_attention_weight_min": -19.32342529296875,
      "activations/layer2_attention_weight_max": 35.966758728027344,
      "activations/layer2_attention_weight_min": -34.34894943237305,
      "activations/layer3_attention_weight_max": 91.67385864257812,
      "activations/layer3_attention_weight_min": -96.90067291259766,
      "activations/layer4_attention_weight_max": 130.9381561279297,
      "activations/layer4_attention_weight_min": -134.42828369140625,
      "activations/layer5_attention_weight_max": 71.36478424072266,
      "activations/layer5_attention_weight_min": -74.88768768310547,
      "activations/layer6_attention_weight_max": 52.463993072509766,
      "activations/layer6_attention_weight_min": -58.880584716796875,
      "activations/layer7_attention_weight_max": 84.02076721191406,
      "activations/layer7_attention_weight_min": -91.09902954101562,
      "activations/layer8_attention_weight_max": 39.9596061706543,
      "activations/layer8_attention_weight_min": -40.77046203613281,
      "activations/layer9_attention_weight_max": 35.4323616027832,
      "activations/layer9_attention_weight_min": -35.283023834228516,
      "epoch": 5.93,
      "learning_rate": 0.00011287727272727271,
      "loss": 2.8425,
      "step": 102100
    },
    {
      "activations/layer0_attention_weight_max": 16.635387420654297,
      "activations/layer0_attention_weight_min": -11.961555480957031,
      "activations/layer10_attention_weight_max": 33.20535659790039,
      "activations/layer10_attention_weight_min": -31.799606323242188,
      "activations/layer11_attention_weight_max": 32.82026672363281,
      "activations/layer11_attention_weight_min": -28.24073600769043,
      "activations/layer12_attention_weight_max": 28.659809112548828,
      "activations/layer12_attention_weight_min": -24.391437530517578,
      "activations/layer13_attention_weight_max": 39.90663528442383,
      "activations/layer13_attention_weight_min": -32.747535705566406,
      "activations/layer14_attention_weight_max": 58.68669891357422,
      "activations/layer14_attention_weight_min": -47.025691986083984,
      "activations/layer15_attention_weight_max": 38.24861526489258,
      "activations/layer15_attention_weight_min": -27.408987045288086,
      "activations/layer16_attention_weight_max": 35.87889862060547,
      "activations/layer16_attention_weight_min": -27.8384952545166,
      "activations/layer17_attention_weight_max": 41.4975471496582,
      "activations/layer17_attention_weight_min": -25.91823959350586,
      "activations/layer18_attention_weight_max": 35.896705627441406,
      "activations/layer18_attention_weight_min": -19.820222854614258,
      "activations/layer19_attention_weight_max": 39.998817443847656,
      "activations/layer19_attention_weight_min": -27.327533721923828,
      "activations/layer1_attention_weight_max": 16.837244033813477,
      "activations/layer1_attention_weight_min": -14.616598129272461,
      "activations/layer20_attention_weight_max": 34.5537109375,
      "activations/layer20_attention_weight_min": -23.980175018310547,
      "activations/layer21_attention_weight_max": 36.19001388549805,
      "activations/layer21_attention_weight_min": -20.851097106933594,
      "activations/layer22_attention_weight_max": 50.88043212890625,
      "activations/layer22_attention_weight_min": -26.99338722229004,
      "activations/layer23_attention_weight_max": 33.252159118652344,
      "activations/layer23_attention_weight_min": -19.872684478759766,
      "activations/layer2_attention_weight_max": 29.47454071044922,
      "activations/layer2_attention_weight_min": -29.53511619567871,
      "activations/layer3_attention_weight_max": 85.94131469726562,
      "activations/layer3_attention_weight_min": -87.10529327392578,
      "activations/layer4_attention_weight_max": 123.47017669677734,
      "activations/layer4_attention_weight_min": -120.3556137084961,
      "activations/layer5_attention_weight_max": 70.00643920898438,
      "activations/layer5_attention_weight_min": -71.08322143554688,
      "activations/layer6_attention_weight_max": 51.54303741455078,
      "activations/layer6_attention_weight_min": -53.67194747924805,
      "activations/layer7_attention_weight_max": 83.25232696533203,
      "activations/layer7_attention_weight_min": -86.2845687866211,
      "activations/layer8_attention_weight_max": 40.48036575317383,
      "activations/layer8_attention_weight_min": -47.13936996459961,
      "activations/layer9_attention_weight_max": 36.34773635864258,
      "activations/layer9_attention_weight_min": -36.76564025878906,
      "epoch": 5.94,
      "learning_rate": 0.00011285833333333332,
      "loss": 2.8389,
      "step": 102150
    },
    {
      "activations/layer0_attention_weight_max": 16.127225875854492,
      "activations/layer0_attention_weight_min": -12.353952407836914,
      "activations/layer10_attention_weight_max": 31.433914184570312,
      "activations/layer10_attention_weight_min": -30.343698501586914,
      "activations/layer11_attention_weight_max": 30.861103057861328,
      "activations/layer11_attention_weight_min": -28.669078826904297,
      "activations/layer12_attention_weight_max": 23.670875549316406,
      "activations/layer12_attention_weight_min": -27.07703971862793,
      "activations/layer13_attention_weight_max": 46.15061569213867,
      "activations/layer13_attention_weight_min": -32.755401611328125,
      "activations/layer14_attention_weight_max": 54.54023742675781,
      "activations/layer14_attention_weight_min": -39.68583679199219,
      "activations/layer15_attention_weight_max": 35.277099609375,
      "activations/layer15_attention_weight_min": -27.845237731933594,
      "activations/layer16_attention_weight_max": 31.26808738708496,
      "activations/layer16_attention_weight_min": -26.288400650024414,
      "activations/layer17_attention_weight_max": 35.72077941894531,
      "activations/layer17_attention_weight_min": -23.559812545776367,
      "activations/layer18_attention_weight_max": 30.5540771484375,
      "activations/layer18_attention_weight_min": -22.75725555419922,
      "activations/layer19_attention_weight_max": 35.00834655761719,
      "activations/layer19_attention_weight_min": -27.766117095947266,
      "activations/layer1_attention_weight_max": 16.90497589111328,
      "activations/layer1_attention_weight_min": -12.917774200439453,
      "activations/layer20_attention_weight_max": 31.283977508544922,
      "activations/layer20_attention_weight_min": -23.80115509033203,
      "activations/layer21_attention_weight_max": 29.152666091918945,
      "activations/layer21_attention_weight_min": -22.058874130249023,
      "activations/layer22_attention_weight_max": 49.1380500793457,
      "activations/layer22_attention_weight_min": -29.60297203063965,
      "activations/layer23_attention_weight_max": 33.12543487548828,
      "activations/layer23_attention_weight_min": -24.159482955932617,
      "activations/layer2_attention_weight_max": 30.363265991210938,
      "activations/layer2_attention_weight_min": -30.942218780517578,
      "activations/layer3_attention_weight_max": 87.6266860961914,
      "activations/layer3_attention_weight_min": -90.02958679199219,
      "activations/layer4_attention_weight_max": 128.0289764404297,
      "activations/layer4_attention_weight_min": -124.45027923583984,
      "activations/layer5_attention_weight_max": 71.96522521972656,
      "activations/layer5_attention_weight_min": -70.0390396118164,
      "activations/layer6_attention_weight_max": 51.91771697998047,
      "activations/layer6_attention_weight_min": -54.82917404174805,
      "activations/layer7_attention_weight_max": 83.9523696899414,
      "activations/layer7_attention_weight_min": -85.07223510742188,
      "activations/layer8_attention_weight_max": 38.11176300048828,
      "activations/layer8_attention_weight_min": -38.93638229370117,
      "activations/layer9_attention_weight_max": 33.0428466796875,
      "activations/layer9_attention_weight_min": -39.228206634521484,
      "epoch": 5.94,
      "learning_rate": 0.00011283939393939393,
      "loss": 2.8178,
      "step": 102200
    },
    {
      "activations/layer0_attention_weight_max": 15.567862510681152,
      "activations/layer0_attention_weight_min": -11.662323951721191,
      "activations/layer10_attention_weight_max": 32.37959289550781,
      "activations/layer10_attention_weight_min": -31.567779541015625,
      "activations/layer11_attention_weight_max": 32.84344482421875,
      "activations/layer11_attention_weight_min": -27.96146011352539,
      "activations/layer12_attention_weight_max": 30.823183059692383,
      "activations/layer12_attention_weight_min": -28.02165412902832,
      "activations/layer13_attention_weight_max": 38.139625549316406,
      "activations/layer13_attention_weight_min": -31.47640037536621,
      "activations/layer14_attention_weight_max": 50.196006774902344,
      "activations/layer14_attention_weight_min": -41.24268341064453,
      "activations/layer15_attention_weight_max": 36.339107513427734,
      "activations/layer15_attention_weight_min": -29.515438079833984,
      "activations/layer16_attention_weight_max": 34.98841094970703,
      "activations/layer16_attention_weight_min": -29.113115310668945,
      "activations/layer17_attention_weight_max": 37.69582748413086,
      "activations/layer17_attention_weight_min": -25.54168128967285,
      "activations/layer18_attention_weight_max": 31.260482788085938,
      "activations/layer18_attention_weight_min": -19.457029342651367,
      "activations/layer19_attention_weight_max": 36.898868560791016,
      "activations/layer19_attention_weight_min": -28.046844482421875,
      "activations/layer1_attention_weight_max": 17.60445785522461,
      "activations/layer1_attention_weight_min": -14.501853942871094,
      "activations/layer20_attention_weight_max": 28.941499710083008,
      "activations/layer20_attention_weight_min": -24.2466983795166,
      "activations/layer21_attention_weight_max": 30.788400650024414,
      "activations/layer21_attention_weight_min": -21.37007713317871,
      "activations/layer22_attention_weight_max": 44.31232452392578,
      "activations/layer22_attention_weight_min": -28.7591495513916,
      "activations/layer23_attention_weight_max": 32.953041076660156,
      "activations/layer23_attention_weight_min": -22.865192413330078,
      "activations/layer2_attention_weight_max": 31.772815704345703,
      "activations/layer2_attention_weight_min": -30.09869956970215,
      "activations/layer3_attention_weight_max": 88.28809356689453,
      "activations/layer3_attention_weight_min": -87.40474700927734,
      "activations/layer4_attention_weight_max": 120.9356689453125,
      "activations/layer4_attention_weight_min": -122.21736907958984,
      "activations/layer5_attention_weight_max": 70.92166900634766,
      "activations/layer5_attention_weight_min": -69.68730163574219,
      "activations/layer6_attention_weight_max": 53.0862922668457,
      "activations/layer6_attention_weight_min": -58.14808654785156,
      "activations/layer7_attention_weight_max": 86.0812759399414,
      "activations/layer7_attention_weight_min": -84.1257553100586,
      "activations/layer8_attention_weight_max": 39.382659912109375,
      "activations/layer8_attention_weight_min": -39.58319091796875,
      "activations/layer9_attention_weight_max": 35.24919128417969,
      "activations/layer9_attention_weight_min": -34.1000862121582,
      "epoch": 5.94,
      "learning_rate": 0.00011282045454545453,
      "loss": 2.831,
      "step": 102250
    },
    {
      "activations/layer0_attention_weight_max": 16.697336196899414,
      "activations/layer0_attention_weight_min": -12.440052032470703,
      "activations/layer10_attention_weight_max": 31.56830406188965,
      "activations/layer10_attention_weight_min": -29.909114837646484,
      "activations/layer11_attention_weight_max": 32.738121032714844,
      "activations/layer11_attention_weight_min": -26.584644317626953,
      "activations/layer12_attention_weight_max": 28.939428329467773,
      "activations/layer12_attention_weight_min": -23.99277114868164,
      "activations/layer13_attention_weight_max": 39.81200408935547,
      "activations/layer13_attention_weight_min": -30.658214569091797,
      "activations/layer14_attention_weight_max": 47.892059326171875,
      "activations/layer14_attention_weight_min": -38.579978942871094,
      "activations/layer15_attention_weight_max": 37.16845703125,
      "activations/layer15_attention_weight_min": -28.004411697387695,
      "activations/layer16_attention_weight_max": 33.1225700378418,
      "activations/layer16_attention_weight_min": -26.52574348449707,
      "activations/layer17_attention_weight_max": 36.756744384765625,
      "activations/layer17_attention_weight_min": -23.04783058166504,
      "activations/layer18_attention_weight_max": 33.93110275268555,
      "activations/layer18_attention_weight_min": -21.78839874267578,
      "activations/layer19_attention_weight_max": 39.24502182006836,
      "activations/layer19_attention_weight_min": -29.468399047851562,
      "activations/layer1_attention_weight_max": 16.995378494262695,
      "activations/layer1_attention_weight_min": -17.186403274536133,
      "activations/layer20_attention_weight_max": 33.46562957763672,
      "activations/layer20_attention_weight_min": -25.04225730895996,
      "activations/layer21_attention_weight_max": 30.758867263793945,
      "activations/layer21_attention_weight_min": -24.40921974182129,
      "activations/layer22_attention_weight_max": 49.29331588745117,
      "activations/layer22_attention_weight_min": -31.198421478271484,
      "activations/layer23_attention_weight_max": 34.02582550048828,
      "activations/layer23_attention_weight_min": -22.795242309570312,
      "activations/layer2_attention_weight_max": 33.17401123046875,
      "activations/layer2_attention_weight_min": -31.205299377441406,
      "activations/layer3_attention_weight_max": 91.95855712890625,
      "activations/layer3_attention_weight_min": -92.08917999267578,
      "activations/layer4_attention_weight_max": 123.5226058959961,
      "activations/layer4_attention_weight_min": -119.58647918701172,
      "activations/layer5_attention_weight_max": 70.91533660888672,
      "activations/layer5_attention_weight_min": -69.76731872558594,
      "activations/layer6_attention_weight_max": 53.54989242553711,
      "activations/layer6_attention_weight_min": -54.658077239990234,
      "activations/layer7_attention_weight_max": 85.34642028808594,
      "activations/layer7_attention_weight_min": -82.84343719482422,
      "activations/layer8_attention_weight_max": 39.90372848510742,
      "activations/layer8_attention_weight_min": -39.091888427734375,
      "activations/layer9_attention_weight_max": 35.49011993408203,
      "activations/layer9_attention_weight_min": -33.27672576904297,
      "epoch": 5.94,
      "learning_rate": 0.00011280151515151515,
      "loss": 2.8551,
      "step": 102300
    },
    {
      "activations/layer0_attention_weight_max": 16.38982391357422,
      "activations/layer0_attention_weight_min": -12.478360176086426,
      "activations/layer10_attention_weight_max": 33.713653564453125,
      "activations/layer10_attention_weight_min": -31.094058990478516,
      "activations/layer11_attention_weight_max": 32.1036376953125,
      "activations/layer11_attention_weight_min": -27.826496124267578,
      "activations/layer12_attention_weight_max": 26.34789276123047,
      "activations/layer12_attention_weight_min": -26.672574996948242,
      "activations/layer13_attention_weight_max": 42.50395202636719,
      "activations/layer13_attention_weight_min": -33.2379035949707,
      "activations/layer14_attention_weight_max": 64.40813446044922,
      "activations/layer14_attention_weight_min": -42.34583282470703,
      "activations/layer15_attention_weight_max": 38.2728271484375,
      "activations/layer15_attention_weight_min": -27.790267944335938,
      "activations/layer16_attention_weight_max": 41.075042724609375,
      "activations/layer16_attention_weight_min": -27.499862670898438,
      "activations/layer17_attention_weight_max": 40.77814483642578,
      "activations/layer17_attention_weight_min": -23.512887954711914,
      "activations/layer18_attention_weight_max": 34.76424789428711,
      "activations/layer18_attention_weight_min": -19.962934494018555,
      "activations/layer19_attention_weight_max": 41.6278190612793,
      "activations/layer19_attention_weight_min": -28.364233016967773,
      "activations/layer1_attention_weight_max": 16.99700927734375,
      "activations/layer1_attention_weight_min": -14.613947868347168,
      "activations/layer20_attention_weight_max": 34.415191650390625,
      "activations/layer20_attention_weight_min": -23.09222984313965,
      "activations/layer21_attention_weight_max": 33.13959884643555,
      "activations/layer21_attention_weight_min": -20.086599349975586,
      "activations/layer22_attention_weight_max": 56.58854293823242,
      "activations/layer22_attention_weight_min": -28.403684616088867,
      "activations/layer23_attention_weight_max": 37.79874801635742,
      "activations/layer23_attention_weight_min": -22.21927261352539,
      "activations/layer2_attention_weight_max": 32.149784088134766,
      "activations/layer2_attention_weight_min": -31.2762451171875,
      "activations/layer3_attention_weight_max": 94.30747985839844,
      "activations/layer3_attention_weight_min": -96.31658935546875,
      "activations/layer4_attention_weight_max": 128.0731658935547,
      "activations/layer4_attention_weight_min": -130.2743377685547,
      "activations/layer5_attention_weight_max": 74.62387084960938,
      "activations/layer5_attention_weight_min": -73.69776916503906,
      "activations/layer6_attention_weight_max": 56.081634521484375,
      "activations/layer6_attention_weight_min": -56.49152755737305,
      "activations/layer7_attention_weight_max": 89.07136535644531,
      "activations/layer7_attention_weight_min": -87.0132064819336,
      "activations/layer8_attention_weight_max": 40.469242095947266,
      "activations/layer8_attention_weight_min": -42.9523811340332,
      "activations/layer9_attention_weight_max": 36.891849517822266,
      "activations/layer9_attention_weight_min": -34.82316589355469,
      "epoch": 5.95,
      "learning_rate": 0.00011278257575757575,
      "loss": 2.8431,
      "step": 102350
    },
    {
      "activations/layer0_attention_weight_max": 16.25742530822754,
      "activations/layer0_attention_weight_min": -12.213343620300293,
      "activations/layer10_attention_weight_max": 37.55970001220703,
      "activations/layer10_attention_weight_min": -31.184032440185547,
      "activations/layer11_attention_weight_max": 33.80152893066406,
      "activations/layer11_attention_weight_min": -28.7135009765625,
      "activations/layer12_attention_weight_max": 24.56113624572754,
      "activations/layer12_attention_weight_min": -27.151792526245117,
      "activations/layer13_attention_weight_max": 38.407588958740234,
      "activations/layer13_attention_weight_min": -32.68022537231445,
      "activations/layer14_attention_weight_max": 52.90983200073242,
      "activations/layer14_attention_weight_min": -40.33133316040039,
      "activations/layer15_attention_weight_max": 38.95225524902344,
      "activations/layer15_attention_weight_min": -28.236730575561523,
      "activations/layer16_attention_weight_max": 34.01834487915039,
      "activations/layer16_attention_weight_min": -28.020740509033203,
      "activations/layer17_attention_weight_max": 38.86136245727539,
      "activations/layer17_attention_weight_min": -25.943458557128906,
      "activations/layer18_attention_weight_max": 36.14873504638672,
      "activations/layer18_attention_weight_min": -20.43358039855957,
      "activations/layer19_attention_weight_max": 34.805782318115234,
      "activations/layer19_attention_weight_min": -30.9975528717041,
      "activations/layer1_attention_weight_max": 16.864784240722656,
      "activations/layer1_attention_weight_min": -14.473896026611328,
      "activations/layer20_attention_weight_max": 29.84706687927246,
      "activations/layer20_attention_weight_min": -27.049243927001953,
      "activations/layer21_attention_weight_max": 29.8978328704834,
      "activations/layer21_attention_weight_min": -23.304967880249023,
      "activations/layer22_attention_weight_max": 47.7774543762207,
      "activations/layer22_attention_weight_min": -31.486907958984375,
      "activations/layer23_attention_weight_max": 35.66718673706055,
      "activations/layer23_attention_weight_min": -22.46173858642578,
      "activations/layer2_attention_weight_max": 32.613685607910156,
      "activations/layer2_attention_weight_min": -31.36504364013672,
      "activations/layer3_attention_weight_max": 92.59439086914062,
      "activations/layer3_attention_weight_min": -91.38265991210938,
      "activations/layer4_attention_weight_max": 133.56536865234375,
      "activations/layer4_attention_weight_min": -120.44054412841797,
      "activations/layer5_attention_weight_max": 76.8208999633789,
      "activations/layer5_attention_weight_min": -71.62532043457031,
      "activations/layer6_attention_weight_max": 53.255367279052734,
      "activations/layer6_attention_weight_min": -52.32311248779297,
      "activations/layer7_attention_weight_max": 94.21998596191406,
      "activations/layer7_attention_weight_min": -82.4698257446289,
      "activations/layer8_attention_weight_max": 43.35310745239258,
      "activations/layer8_attention_weight_min": -41.551109313964844,
      "activations/layer9_attention_weight_max": 38.466590881347656,
      "activations/layer9_attention_weight_min": -33.95101547241211,
      "epoch": 5.95,
      "learning_rate": 0.00011276363636363635,
      "loss": 2.8316,
      "step": 102400
    },
    {
      "activations/layer0_attention_weight_max": 15.833099365234375,
      "activations/layer0_attention_weight_min": -12.506728172302246,
      "activations/layer10_attention_weight_max": 35.26363754272461,
      "activations/layer10_attention_weight_min": -34.77796173095703,
      "activations/layer11_attention_weight_max": 35.299957275390625,
      "activations/layer11_attention_weight_min": -29.872299194335938,
      "activations/layer12_attention_weight_max": 28.631349563598633,
      "activations/layer12_attention_weight_min": -25.591352462768555,
      "activations/layer13_attention_weight_max": 47.64277267456055,
      "activations/layer13_attention_weight_min": -33.14190673828125,
      "activations/layer14_attention_weight_max": 84.49668884277344,
      "activations/layer14_attention_weight_min": -56.11515426635742,
      "activations/layer15_attention_weight_max": 45.73064422607422,
      "activations/layer15_attention_weight_min": -29.740814208984375,
      "activations/layer16_attention_weight_max": 41.09868621826172,
      "activations/layer16_attention_weight_min": -26.59568977355957,
      "activations/layer17_attention_weight_max": 41.20901870727539,
      "activations/layer17_attention_weight_min": -24.735597610473633,
      "activations/layer18_attention_weight_max": 31.079561233520508,
      "activations/layer18_attention_weight_min": -21.259971618652344,
      "activations/layer19_attention_weight_max": 45.02927017211914,
      "activations/layer19_attention_weight_min": -30.35209083557129,
      "activations/layer1_attention_weight_max": 16.232742309570312,
      "activations/layer1_attention_weight_min": -15.55764102935791,
      "activations/layer20_attention_weight_max": 37.500179290771484,
      "activations/layer20_attention_weight_min": -25.778575897216797,
      "activations/layer21_attention_weight_max": 36.82583999633789,
      "activations/layer21_attention_weight_min": -22.532470703125,
      "activations/layer22_attention_weight_max": 58.383663177490234,
      "activations/layer22_attention_weight_min": -27.9534912109375,
      "activations/layer23_attention_weight_max": 41.216270446777344,
      "activations/layer23_attention_weight_min": -21.49087142944336,
      "activations/layer2_attention_weight_max": 31.36646842956543,
      "activations/layer2_attention_weight_min": -31.935527801513672,
      "activations/layer3_attention_weight_max": 94.88745880126953,
      "activations/layer3_attention_weight_min": -93.07268524169922,
      "activations/layer4_attention_weight_max": 132.4910125732422,
      "activations/layer4_attention_weight_min": -126.6998291015625,
      "activations/layer5_attention_weight_max": 75.3775634765625,
      "activations/layer5_attention_weight_min": -74.99333190917969,
      "activations/layer6_attention_weight_max": 56.878196716308594,
      "activations/layer6_attention_weight_min": -56.00149917602539,
      "activations/layer7_attention_weight_max": 92.19698333740234,
      "activations/layer7_attention_weight_min": -88.2785415649414,
      "activations/layer8_attention_weight_max": 43.852745056152344,
      "activations/layer8_attention_weight_min": -43.063323974609375,
      "activations/layer9_attention_weight_max": 39.5394172668457,
      "activations/layer9_attention_weight_min": -37.91282653808594,
      "epoch": 5.95,
      "learning_rate": 0.00011274507575757574,
      "loss": 2.8448,
      "step": 102450
    },
    {
      "activations/layer0_attention_weight_max": 15.768486976623535,
      "activations/layer0_attention_weight_min": -10.911638259887695,
      "activations/layer10_attention_weight_max": 35.16441345214844,
      "activations/layer10_attention_weight_min": -31.417930603027344,
      "activations/layer11_attention_weight_max": 38.57902908325195,
      "activations/layer11_attention_weight_min": -33.90778732299805,
      "activations/layer12_attention_weight_max": 32.19316864013672,
      "activations/layer12_attention_weight_min": -28.092370986938477,
      "activations/layer13_attention_weight_max": 47.45444107055664,
      "activations/layer13_attention_weight_min": -32.524784088134766,
      "activations/layer14_attention_weight_max": 69.07855987548828,
      "activations/layer14_attention_weight_min": -49.734413146972656,
      "activations/layer15_attention_weight_max": 37.396827697753906,
      "activations/layer15_attention_weight_min": -29.34776496887207,
      "activations/layer16_attention_weight_max": 38.70506286621094,
      "activations/layer16_attention_weight_min": -27.01058006286621,
      "activations/layer17_attention_weight_max": 36.34785842895508,
      "activations/layer17_attention_weight_min": -24.47044563293457,
      "activations/layer18_attention_weight_max": 33.755775451660156,
      "activations/layer18_attention_weight_min": -23.04403305053711,
      "activations/layer19_attention_weight_max": 40.828514099121094,
      "activations/layer19_attention_weight_min": -30.46283531188965,
      "activations/layer1_attention_weight_max": 18.300363540649414,
      "activations/layer1_attention_weight_min": -16.544023513793945,
      "activations/layer20_attention_weight_max": 36.99285888671875,
      "activations/layer20_attention_weight_min": -24.930004119873047,
      "activations/layer21_attention_weight_max": 35.602203369140625,
      "activations/layer21_attention_weight_min": -23.033428192138672,
      "activations/layer22_attention_weight_max": 51.19953155517578,
      "activations/layer22_attention_weight_min": -28.32041358947754,
      "activations/layer23_attention_weight_max": 33.977745056152344,
      "activations/layer23_attention_weight_min": -23.4262752532959,
      "activations/layer2_attention_weight_max": 30.066387176513672,
      "activations/layer2_attention_weight_min": -30.887006759643555,
      "activations/layer3_attention_weight_max": 87.95125579833984,
      "activations/layer3_attention_weight_min": -88.8121566772461,
      "activations/layer4_attention_weight_max": 124.55152893066406,
      "activations/layer4_attention_weight_min": -121.78775787353516,
      "activations/layer5_attention_weight_max": 70.87680053710938,
      "activations/layer5_attention_weight_min": -69.83795166015625,
      "activations/layer6_attention_weight_max": 53.57756042480469,
      "activations/layer6_attention_weight_min": -53.838584899902344,
      "activations/layer7_attention_weight_max": 88.12789916992188,
      "activations/layer7_attention_weight_min": -88.15850830078125,
      "activations/layer8_attention_weight_max": 42.78859329223633,
      "activations/layer8_attention_weight_min": -42.1761360168457,
      "activations/layer9_attention_weight_max": 41.97582244873047,
      "activations/layer9_attention_weight_min": -36.96772003173828,
      "epoch": 5.96,
      "learning_rate": 0.00011272613636363636,
      "loss": 2.8243,
      "step": 102500
    },
    {
      "activations/layer0_attention_weight_max": 15.598328590393066,
      "activations/layer0_attention_weight_min": -12.349238395690918,
      "activations/layer10_attention_weight_max": 33.573421478271484,
      "activations/layer10_attention_weight_min": -31.56515121459961,
      "activations/layer11_attention_weight_max": 30.99127960205078,
      "activations/layer11_attention_weight_min": -29.818769454956055,
      "activations/layer12_attention_weight_max": 23.25828742980957,
      "activations/layer12_attention_weight_min": -26.465364456176758,
      "activations/layer13_attention_weight_max": 41.13328170776367,
      "activations/layer13_attention_weight_min": -30.492450714111328,
      "activations/layer14_attention_weight_max": 49.642818450927734,
      "activations/layer14_attention_weight_min": -38.614479064941406,
      "activations/layer15_attention_weight_max": 37.03502655029297,
      "activations/layer15_attention_weight_min": -28.670249938964844,
      "activations/layer16_attention_weight_max": 34.17647171020508,
      "activations/layer16_attention_weight_min": -27.743486404418945,
      "activations/layer17_attention_weight_max": 36.025474548339844,
      "activations/layer17_attention_weight_min": -25.199209213256836,
      "activations/layer18_attention_weight_max": 33.710235595703125,
      "activations/layer18_attention_weight_min": -20.158348083496094,
      "activations/layer19_attention_weight_max": 39.573307037353516,
      "activations/layer19_attention_weight_min": -29.703054428100586,
      "activations/layer1_attention_weight_max": 17.1541805267334,
      "activations/layer1_attention_weight_min": -15.716334342956543,
      "activations/layer20_attention_weight_max": 29.36333465576172,
      "activations/layer20_attention_weight_min": -24.221725463867188,
      "activations/layer21_attention_weight_max": 29.90595054626465,
      "activations/layer21_attention_weight_min": -21.462295532226562,
      "activations/layer22_attention_weight_max": 47.638980865478516,
      "activations/layer22_attention_weight_min": -29.525102615356445,
      "activations/layer23_attention_weight_max": 32.526737213134766,
      "activations/layer23_attention_weight_min": -21.072269439697266,
      "activations/layer2_attention_weight_max": 32.450862884521484,
      "activations/layer2_attention_weight_min": -30.024778366088867,
      "activations/layer3_attention_weight_max": 90.15254974365234,
      "activations/layer3_attention_weight_min": -88.01097869873047,
      "activations/layer4_attention_weight_max": 127.28905487060547,
      "activations/layer4_attention_weight_min": -125.9784927368164,
      "activations/layer5_attention_weight_max": 71.74190521240234,
      "activations/layer5_attention_weight_min": -71.57833862304688,
      "activations/layer6_attention_weight_max": 51.522117614746094,
      "activations/layer6_attention_weight_min": -57.512725830078125,
      "activations/layer7_attention_weight_max": 90.04824829101562,
      "activations/layer7_attention_weight_min": -84.33104705810547,
      "activations/layer8_attention_weight_max": 40.278358459472656,
      "activations/layer8_attention_weight_min": -39.14546585083008,
      "activations/layer9_attention_weight_max": 37.66706085205078,
      "activations/layer9_attention_weight_min": -34.46084213256836,
      "epoch": 5.96,
      "learning_rate": 0.00011270719696969697,
      "loss": 2.8295,
      "step": 102550
    },
    {
      "activations/layer0_attention_weight_max": 14.982303619384766,
      "activations/layer0_attention_weight_min": -11.702740669250488,
      "activations/layer10_attention_weight_max": 36.171607971191406,
      "activations/layer10_attention_weight_min": -30.40085220336914,
      "activations/layer11_attention_weight_max": 35.16292953491211,
      "activations/layer11_attention_weight_min": -28.81612777709961,
      "activations/layer12_attention_weight_max": 24.648706436157227,
      "activations/layer12_attention_weight_min": -27.875043869018555,
      "activations/layer13_attention_weight_max": 47.90959548950195,
      "activations/layer13_attention_weight_min": -31.89496612548828,
      "activations/layer14_attention_weight_max": 60.48991775512695,
      "activations/layer14_attention_weight_min": -39.857215881347656,
      "activations/layer15_attention_weight_max": 43.476444244384766,
      "activations/layer15_attention_weight_min": -28.701412200927734,
      "activations/layer16_attention_weight_max": 38.26811599731445,
      "activations/layer16_attention_weight_min": -28.292922973632812,
      "activations/layer17_attention_weight_max": 40.0449104309082,
      "activations/layer17_attention_weight_min": -27.251880645751953,
      "activations/layer18_attention_weight_max": 36.40446853637695,
      "activations/layer18_attention_weight_min": -20.60242462158203,
      "activations/layer19_attention_weight_max": 45.10047912597656,
      "activations/layer19_attention_weight_min": -31.156784057617188,
      "activations/layer1_attention_weight_max": 17.969362258911133,
      "activations/layer1_attention_weight_min": -13.292640686035156,
      "activations/layer20_attention_weight_max": 34.37070846557617,
      "activations/layer20_attention_weight_min": -25.764142990112305,
      "activations/layer21_attention_weight_max": 40.7625846862793,
      "activations/layer21_attention_weight_min": -22.913562774658203,
      "activations/layer22_attention_weight_max": 52.209739685058594,
      "activations/layer22_attention_weight_min": -28.818206787109375,
      "activations/layer23_attention_weight_max": 41.73717498779297,
      "activations/layer23_attention_weight_min": -22.906021118164062,
      "activations/layer2_attention_weight_max": 32.05210876464844,
      "activations/layer2_attention_weight_min": -32.02065658569336,
      "activations/layer3_attention_weight_max": 92.10575103759766,
      "activations/layer3_attention_weight_min": -91.95233154296875,
      "activations/layer4_attention_weight_max": 130.4435577392578,
      "activations/layer4_attention_weight_min": -120.9580078125,
      "activations/layer5_attention_weight_max": 76.32626342773438,
      "activations/layer5_attention_weight_min": -72.88973236083984,
      "activations/layer6_attention_weight_max": 57.73445129394531,
      "activations/layer6_attention_weight_min": -56.05086135864258,
      "activations/layer7_attention_weight_max": 84.7965316772461,
      "activations/layer7_attention_weight_min": -91.57254028320312,
      "activations/layer8_attention_weight_max": 39.1951789855957,
      "activations/layer8_attention_weight_min": -40.824127197265625,
      "activations/layer9_attention_weight_max": 35.80653381347656,
      "activations/layer9_attention_weight_min": -33.13247299194336,
      "epoch": 5.96,
      "learning_rate": 0.00011268825757575756,
      "loss": 2.845,
      "step": 102600
    },
    {
      "activations/layer0_attention_weight_max": 16.129709243774414,
      "activations/layer0_attention_weight_min": -11.220766067504883,
      "activations/layer10_attention_weight_max": 33.66535186767578,
      "activations/layer10_attention_weight_min": -30.39006805419922,
      "activations/layer11_attention_weight_max": 33.894683837890625,
      "activations/layer11_attention_weight_min": -30.207176208496094,
      "activations/layer12_attention_weight_max": 53.22766876220703,
      "activations/layer12_attention_weight_min": -26.84258460998535,
      "activations/layer13_attention_weight_max": 51.6002082824707,
      "activations/layer13_attention_weight_min": -34.52445983886719,
      "activations/layer14_attention_weight_max": 82.4477767944336,
      "activations/layer14_attention_weight_min": -53.34832000732422,
      "activations/layer15_attention_weight_max": 52.71658706665039,
      "activations/layer15_attention_weight_min": -31.502277374267578,
      "activations/layer16_attention_weight_max": 41.425201416015625,
      "activations/layer16_attention_weight_min": -28.715532302856445,
      "activations/layer17_attention_weight_max": 40.486915588378906,
      "activations/layer17_attention_weight_min": -24.965795516967773,
      "activations/layer18_attention_weight_max": 34.172332763671875,
      "activations/layer18_attention_weight_min": -22.200183868408203,
      "activations/layer19_attention_weight_max": 45.00541305541992,
      "activations/layer19_attention_weight_min": -31.33929443359375,
      "activations/layer1_attention_weight_max": 18.411441802978516,
      "activations/layer1_attention_weight_min": -14.484400749206543,
      "activations/layer20_attention_weight_max": 34.095008850097656,
      "activations/layer20_attention_weight_min": -23.941606521606445,
      "activations/layer21_attention_weight_max": 37.67596435546875,
      "activations/layer21_attention_weight_min": -21.52158546447754,
      "activations/layer22_attention_weight_max": 51.82644271850586,
      "activations/layer22_attention_weight_min": -26.30190086364746,
      "activations/layer23_attention_weight_max": 34.33562469482422,
      "activations/layer23_attention_weight_min": -23.701765060424805,
      "activations/layer2_attention_weight_max": 31.42819595336914,
      "activations/layer2_attention_weight_min": -30.220958709716797,
      "activations/layer3_attention_weight_max": 87.38040161132812,
      "activations/layer3_attention_weight_min": -89.4177017211914,
      "activations/layer4_attention_weight_max": 123.75371551513672,
      "activations/layer4_attention_weight_min": -123.75748443603516,
      "activations/layer5_attention_weight_max": 70.59827423095703,
      "activations/layer5_attention_weight_min": -70.73765563964844,
      "activations/layer6_attention_weight_max": 55.10813903808594,
      "activations/layer6_attention_weight_min": -56.50944519042969,
      "activations/layer7_attention_weight_max": 87.47490692138672,
      "activations/layer7_attention_weight_min": -83.34239959716797,
      "activations/layer8_attention_weight_max": 38.2556266784668,
      "activations/layer8_attention_weight_min": -38.86513137817383,
      "activations/layer9_attention_weight_max": 34.92633819580078,
      "activations/layer9_attention_weight_min": -33.42037582397461,
      "epoch": 5.96,
      "learning_rate": 0.00011266931818181817,
      "loss": 2.8341,
      "step": 102650
    },
    {
      "activations/layer0_attention_weight_max": 15.605191230773926,
      "activations/layer0_attention_weight_min": -10.97541332244873,
      "activations/layer10_attention_weight_max": 33.7653694152832,
      "activations/layer10_attention_weight_min": -29.4429931640625,
      "activations/layer11_attention_weight_max": 36.317665100097656,
      "activations/layer11_attention_weight_min": -32.07422637939453,
      "activations/layer12_attention_weight_max": 28.938758850097656,
      "activations/layer12_attention_weight_min": -38.566505432128906,
      "activations/layer13_attention_weight_max": 41.300071716308594,
      "activations/layer13_attention_weight_min": -33.16633224487305,
      "activations/layer14_attention_weight_max": 56.59700393676758,
      "activations/layer14_attention_weight_min": -42.86640167236328,
      "activations/layer15_attention_weight_max": 35.913047790527344,
      "activations/layer15_attention_weight_min": -26.961427688598633,
      "activations/layer16_attention_weight_max": 34.70014953613281,
      "activations/layer16_attention_weight_min": -27.099199295043945,
      "activations/layer17_attention_weight_max": 35.69276809692383,
      "activations/layer17_attention_weight_min": -22.904918670654297,
      "activations/layer18_attention_weight_max": 31.278419494628906,
      "activations/layer18_attention_weight_min": -20.01925277709961,
      "activations/layer19_attention_weight_max": 36.28636932373047,
      "activations/layer19_attention_weight_min": -27.947891235351562,
      "activations/layer1_attention_weight_max": 18.441558837890625,
      "activations/layer1_attention_weight_min": -13.317720413208008,
      "activations/layer20_attention_weight_max": 32.949371337890625,
      "activations/layer20_attention_weight_min": -23.28940773010254,
      "activations/layer21_attention_weight_max": 33.786495208740234,
      "activations/layer21_attention_weight_min": -20.527488708496094,
      "activations/layer22_attention_weight_max": 49.55727767944336,
      "activations/layer22_attention_weight_min": -29.552326202392578,
      "activations/layer23_attention_weight_max": 31.779945373535156,
      "activations/layer23_attention_weight_min": -22.33072280883789,
      "activations/layer2_attention_weight_max": 30.792020797729492,
      "activations/layer2_attention_weight_min": -31.184677124023438,
      "activations/layer3_attention_weight_max": 86.65563201904297,
      "activations/layer3_attention_weight_min": -86.00546264648438,
      "activations/layer4_attention_weight_max": 123.79149627685547,
      "activations/layer4_attention_weight_min": -122.1081771850586,
      "activations/layer5_attention_weight_max": 71.82672119140625,
      "activations/layer5_attention_weight_min": -67.16615295410156,
      "activations/layer6_attention_weight_max": 51.07152557373047,
      "activations/layer6_attention_weight_min": -52.51158905029297,
      "activations/layer7_attention_weight_max": 96.02922821044922,
      "activations/layer7_attention_weight_min": -82.60870361328125,
      "activations/layer8_attention_weight_max": 38.9010124206543,
      "activations/layer8_attention_weight_min": -37.878623962402344,
      "activations/layer9_attention_weight_max": 34.73387908935547,
      "activations/layer9_attention_weight_min": -49.14980697631836,
      "epoch": 5.97,
      "learning_rate": 0.00011265037878787878,
      "loss": 2.8342,
      "step": 102700
    },
    {
      "activations/layer0_attention_weight_max": 16.28759765625,
      "activations/layer0_attention_weight_min": -11.796093940734863,
      "activations/layer10_attention_weight_max": 33.856849670410156,
      "activations/layer10_attention_weight_min": -31.666006088256836,
      "activations/layer11_attention_weight_max": 33.49113464355469,
      "activations/layer11_attention_weight_min": -28.268905639648438,
      "activations/layer12_attention_weight_max": 25.952768325805664,
      "activations/layer12_attention_weight_min": -31.440162658691406,
      "activations/layer13_attention_weight_max": 38.18290328979492,
      "activations/layer13_attention_weight_min": -31.522136688232422,
      "activations/layer14_attention_weight_max": 75.60533905029297,
      "activations/layer14_attention_weight_min": -49.35602569580078,
      "activations/layer15_attention_weight_max": 35.602210998535156,
      "activations/layer15_attention_weight_min": -28.07732582092285,
      "activations/layer16_attention_weight_max": 33.96598815917969,
      "activations/layer16_attention_weight_min": -27.351327896118164,
      "activations/layer17_attention_weight_max": 38.30235290527344,
      "activations/layer17_attention_weight_min": -22.796886444091797,
      "activations/layer18_attention_weight_max": 31.86479377746582,
      "activations/layer18_attention_weight_min": -19.568227767944336,
      "activations/layer19_attention_weight_max": 35.39315414428711,
      "activations/layer19_attention_weight_min": -28.598718643188477,
      "activations/layer1_attention_weight_max": 17.108678817749023,
      "activations/layer1_attention_weight_min": -13.095826148986816,
      "activations/layer20_attention_weight_max": 29.965307235717773,
      "activations/layer20_attention_weight_min": -21.215373992919922,
      "activations/layer21_attention_weight_max": 27.1119441986084,
      "activations/layer21_attention_weight_min": -20.56163215637207,
      "activations/layer22_attention_weight_max": 45.912437438964844,
      "activations/layer22_attention_weight_min": -27.4486141204834,
      "activations/layer23_attention_weight_max": 34.03123474121094,
      "activations/layer23_attention_weight_min": -20.52460479736328,
      "activations/layer2_attention_weight_max": 32.34469223022461,
      "activations/layer2_attention_weight_min": -31.415830612182617,
      "activations/layer3_attention_weight_max": 93.39812469482422,
      "activations/layer3_attention_weight_min": -97.34893035888672,
      "activations/layer4_attention_weight_max": 132.46145629882812,
      "activations/layer4_attention_weight_min": -131.8593292236328,
      "activations/layer5_attention_weight_max": 73.7374496459961,
      "activations/layer5_attention_weight_min": -74.63003540039062,
      "activations/layer6_attention_weight_max": 54.08918380737305,
      "activations/layer6_attention_weight_min": -55.14448165893555,
      "activations/layer7_attention_weight_max": 90.99637603759766,
      "activations/layer7_attention_weight_min": -92.3547134399414,
      "activations/layer8_attention_weight_max": 39.6036262512207,
      "activations/layer8_attention_weight_min": -39.91203308105469,
      "activations/layer9_attention_weight_max": 36.54694366455078,
      "activations/layer9_attention_weight_min": -35.95337677001953,
      "epoch": 5.97,
      "learning_rate": 0.00011263143939393938,
      "loss": 2.8539,
      "step": 102750
    },
    {
      "activations/layer0_attention_weight_max": 16.1509952545166,
      "activations/layer0_attention_weight_min": -12.534279823303223,
      "activations/layer10_attention_weight_max": 31.50441551208496,
      "activations/layer10_attention_weight_min": -31.296615600585938,
      "activations/layer11_attention_weight_max": 30.161766052246094,
      "activations/layer11_attention_weight_min": -30.905746459960938,
      "activations/layer12_attention_weight_max": 31.491132736206055,
      "activations/layer12_attention_weight_min": -25.405452728271484,
      "activations/layer13_attention_weight_max": 37.25054168701172,
      "activations/layer13_attention_weight_min": -34.2149658203125,
      "activations/layer14_attention_weight_max": 53.322113037109375,
      "activations/layer14_attention_weight_min": -43.67391586303711,
      "activations/layer15_attention_weight_max": 32.8497314453125,
      "activations/layer15_attention_weight_min": -28.150272369384766,
      "activations/layer16_attention_weight_max": 31.444673538208008,
      "activations/layer16_attention_weight_min": -29.21953773498535,
      "activations/layer17_attention_weight_max": 33.1265869140625,
      "activations/layer17_attention_weight_min": -24.75918197631836,
      "activations/layer18_attention_weight_max": 34.432281494140625,
      "activations/layer18_attention_weight_min": -21.38863182067871,
      "activations/layer19_attention_weight_max": 39.64863204956055,
      "activations/layer19_attention_weight_min": -32.140525817871094,
      "activations/layer1_attention_weight_max": 18.263761520385742,
      "activations/layer1_attention_weight_min": -14.55500316619873,
      "activations/layer20_attention_weight_max": 33.29080581665039,
      "activations/layer20_attention_weight_min": -25.162864685058594,
      "activations/layer21_attention_weight_max": 33.58028793334961,
      "activations/layer21_attention_weight_min": -23.566869735717773,
      "activations/layer22_attention_weight_max": 51.24288558959961,
      "activations/layer22_attention_weight_min": -29.556625366210938,
      "activations/layer23_attention_weight_max": 34.01646423339844,
      "activations/layer23_attention_weight_min": -19.788908004760742,
      "activations/layer2_attention_weight_max": 31.652629852294922,
      "activations/layer2_attention_weight_min": -30.619123458862305,
      "activations/layer3_attention_weight_max": 93.6966323852539,
      "activations/layer3_attention_weight_min": -94.58988189697266,
      "activations/layer4_attention_weight_max": 134.5900115966797,
      "activations/layer4_attention_weight_min": -131.04930114746094,
      "activations/layer5_attention_weight_max": 73.99569702148438,
      "activations/layer5_attention_weight_min": -73.25874328613281,
      "activations/layer6_attention_weight_max": 54.5590705871582,
      "activations/layer6_attention_weight_min": -55.27312088012695,
      "activations/layer7_attention_weight_max": 82.98987579345703,
      "activations/layer7_attention_weight_min": -86.7916488647461,
      "activations/layer8_attention_weight_max": 41.63264846801758,
      "activations/layer8_attention_weight_min": -41.523651123046875,
      "activations/layer9_attention_weight_max": 35.27095413208008,
      "activations/layer9_attention_weight_min": -36.77763748168945,
      "epoch": 5.97,
      "learning_rate": 0.00011261249999999999,
      "loss": 2.8389,
      "step": 102800
    },
    {
      "activations/layer0_attention_weight_max": 16.371448516845703,
      "activations/layer0_attention_weight_min": -11.960501670837402,
      "activations/layer10_attention_weight_max": 31.625253677368164,
      "activations/layer10_attention_weight_min": -31.09102439880371,
      "activations/layer11_attention_weight_max": 32.037757873535156,
      "activations/layer11_attention_weight_min": -27.896839141845703,
      "activations/layer12_attention_weight_max": 30.579330444335938,
      "activations/layer12_attention_weight_min": -27.3643856048584,
      "activations/layer13_attention_weight_max": 42.239784240722656,
      "activations/layer13_attention_weight_min": -30.862112045288086,
      "activations/layer14_attention_weight_max": 54.01042175292969,
      "activations/layer14_attention_weight_min": -42.30271530151367,
      "activations/layer15_attention_weight_max": 37.47255325317383,
      "activations/layer15_attention_weight_min": -28.002849578857422,
      "activations/layer16_attention_weight_max": 36.126346588134766,
      "activations/layer16_attention_weight_min": -26.560916900634766,
      "activations/layer17_attention_weight_max": 37.93748474121094,
      "activations/layer17_attention_weight_min": -23.432662963867188,
      "activations/layer18_attention_weight_max": 35.15190887451172,
      "activations/layer18_attention_weight_min": -24.871318817138672,
      "activations/layer19_attention_weight_max": 38.3529167175293,
      "activations/layer19_attention_weight_min": -29.017202377319336,
      "activations/layer1_attention_weight_max": 17.481698989868164,
      "activations/layer1_attention_weight_min": -15.726873397827148,
      "activations/layer20_attention_weight_max": 32.63716506958008,
      "activations/layer20_attention_weight_min": -23.395652770996094,
      "activations/layer21_attention_weight_max": 29.497386932373047,
      "activations/layer21_attention_weight_min": -20.525150299072266,
      "activations/layer22_attention_weight_max": 47.7690544128418,
      "activations/layer22_attention_weight_min": -28.6505184173584,
      "activations/layer23_attention_weight_max": 35.16199493408203,
      "activations/layer23_attention_weight_min": -22.372291564941406,
      "activations/layer2_attention_weight_max": 32.56061935424805,
      "activations/layer2_attention_weight_min": -33.70328903198242,
      "activations/layer3_attention_weight_max": 92.35086059570312,
      "activations/layer3_attention_weight_min": -95.59091186523438,
      "activations/layer4_attention_weight_max": 128.16639709472656,
      "activations/layer4_attention_weight_min": -125.4435043334961,
      "activations/layer5_attention_weight_max": 72.78144836425781,
      "activations/layer5_attention_weight_min": -71.90412902832031,
      "activations/layer6_attention_weight_max": 53.94014358520508,
      "activations/layer6_attention_weight_min": -55.46828079223633,
      "activations/layer7_attention_weight_max": 83.4105224609375,
      "activations/layer7_attention_weight_min": -84.05801391601562,
      "activations/layer8_attention_weight_max": 41.89323806762695,
      "activations/layer8_attention_weight_min": -40.85676574707031,
      "activations/layer9_attention_weight_max": 36.70479965209961,
      "activations/layer9_attention_weight_min": -35.416988372802734,
      "epoch": 5.98,
      "learning_rate": 0.0001125935606060606,
      "loss": 2.8383,
      "step": 102850
    },
    {
      "activations/layer0_attention_weight_max": 14.912796974182129,
      "activations/layer0_attention_weight_min": -12.789324760437012,
      "activations/layer10_attention_weight_max": 32.717437744140625,
      "activations/layer10_attention_weight_min": -28.65562629699707,
      "activations/layer11_attention_weight_max": 31.988439559936523,
      "activations/layer11_attention_weight_min": -26.84535026550293,
      "activations/layer12_attention_weight_max": 22.316856384277344,
      "activations/layer12_attention_weight_min": -24.485584259033203,
      "activations/layer13_attention_weight_max": 44.20710754394531,
      "activations/layer13_attention_weight_min": -33.8962516784668,
      "activations/layer14_attention_weight_max": 58.182525634765625,
      "activations/layer14_attention_weight_min": -42.40475082397461,
      "activations/layer15_attention_weight_max": 33.862449645996094,
      "activations/layer15_attention_weight_min": -27.66518211364746,
      "activations/layer16_attention_weight_max": 30.800451278686523,
      "activations/layer16_attention_weight_min": -27.424579620361328,
      "activations/layer17_attention_weight_max": 32.409915924072266,
      "activations/layer17_attention_weight_min": -24.87851905822754,
      "activations/layer18_attention_weight_max": 32.36909103393555,
      "activations/layer18_attention_weight_min": -20.721805572509766,
      "activations/layer19_attention_weight_max": 32.95515060424805,
      "activations/layer19_attention_weight_min": -28.679426193237305,
      "activations/layer1_attention_weight_max": 16.494958877563477,
      "activations/layer1_attention_weight_min": -14.128762245178223,
      "activations/layer20_attention_weight_max": 29.52263641357422,
      "activations/layer20_attention_weight_min": -24.530893325805664,
      "activations/layer21_attention_weight_max": 30.88559913635254,
      "activations/layer21_attention_weight_min": -22.156118392944336,
      "activations/layer22_attention_weight_max": 42.61516571044922,
      "activations/layer22_attention_weight_min": -27.18049430847168,
      "activations/layer23_attention_weight_max": 30.601964950561523,
      "activations/layer23_attention_weight_min": -22.852445602416992,
      "activations/layer2_attention_weight_max": 32.36418151855469,
      "activations/layer2_attention_weight_min": -31.400432586669922,
      "activations/layer3_attention_weight_max": 93.72595977783203,
      "activations/layer3_attention_weight_min": -90.20834350585938,
      "activations/layer4_attention_weight_max": 131.03170776367188,
      "activations/layer4_attention_weight_min": -127.54427337646484,
      "activations/layer5_attention_weight_max": 71.32080078125,
      "activations/layer5_attention_weight_min": -70.56009674072266,
      "activations/layer6_attention_weight_max": 54.92169952392578,
      "activations/layer6_attention_weight_min": -56.2737922668457,
      "activations/layer7_attention_weight_max": 83.53692626953125,
      "activations/layer7_attention_weight_min": -85.82772064208984,
      "activations/layer8_attention_weight_max": 39.76036834716797,
      "activations/layer8_attention_weight_min": -42.58657455444336,
      "activations/layer9_attention_weight_max": 35.600852966308594,
      "activations/layer9_attention_weight_min": -33.14870834350586,
      "epoch": 5.98,
      "learning_rate": 0.0001125746212121212,
      "loss": 2.8412,
      "step": 102900
    },
    {
      "activations/layer0_attention_weight_max": 17.16149139404297,
      "activations/layer0_attention_weight_min": -11.525683403015137,
      "activations/layer10_attention_weight_max": 34.80244827270508,
      "activations/layer10_attention_weight_min": -32.24068069458008,
      "activations/layer11_attention_weight_max": 32.51024627685547,
      "activations/layer11_attention_weight_min": -27.870664596557617,
      "activations/layer12_attention_weight_max": 25.382246017456055,
      "activations/layer12_attention_weight_min": -26.88016128540039,
      "activations/layer13_attention_weight_max": 48.31947326660156,
      "activations/layer13_attention_weight_min": -33.17436981201172,
      "activations/layer14_attention_weight_max": 52.328739166259766,
      "activations/layer14_attention_weight_min": -38.71577453613281,
      "activations/layer15_attention_weight_max": 33.86548614501953,
      "activations/layer15_attention_weight_min": -28.609668731689453,
      "activations/layer16_attention_weight_max": 35.324039459228516,
      "activations/layer16_attention_weight_min": -27.702831268310547,
      "activations/layer17_attention_weight_max": 37.64384841918945,
      "activations/layer17_attention_weight_min": -25.554292678833008,
      "activations/layer18_attention_weight_max": 31.552335739135742,
      "activations/layer18_attention_weight_min": -20.12699317932129,
      "activations/layer19_attention_weight_max": 37.097434997558594,
      "activations/layer19_attention_weight_min": -29.05315589904785,
      "activations/layer1_attention_weight_max": 17.97868537902832,
      "activations/layer1_attention_weight_min": -14.600983619689941,
      "activations/layer20_attention_weight_max": 31.780872344970703,
      "activations/layer20_attention_weight_min": -24.198768615722656,
      "activations/layer21_attention_weight_max": 31.78592872619629,
      "activations/layer21_attention_weight_min": -22.339290618896484,
      "activations/layer22_attention_weight_max": 47.875144958496094,
      "activations/layer22_attention_weight_min": -28.007844924926758,
      "activations/layer23_attention_weight_max": 33.87076187133789,
      "activations/layer23_attention_weight_min": -22.407377243041992,
      "activations/layer2_attention_weight_max": 32.11653137207031,
      "activations/layer2_attention_weight_min": -29.73488426208496,
      "activations/layer3_attention_weight_max": 90.58660888671875,
      "activations/layer3_attention_weight_min": -92.7566909790039,
      "activations/layer4_attention_weight_max": 126.1220703125,
      "activations/layer4_attention_weight_min": -124.8377685546875,
      "activations/layer5_attention_weight_max": 71.52253723144531,
      "activations/layer5_attention_weight_min": -71.49700927734375,
      "activations/layer6_attention_weight_max": 54.831905364990234,
      "activations/layer6_attention_weight_min": -53.44804382324219,
      "activations/layer7_attention_weight_max": 84.78422546386719,
      "activations/layer7_attention_weight_min": -83.73955535888672,
      "activations/layer8_attention_weight_max": 40.578575134277344,
      "activations/layer8_attention_weight_min": -39.44407272338867,
      "activations/layer9_attention_weight_max": 37.5538215637207,
      "activations/layer9_attention_weight_min": -35.490352630615234,
      "epoch": 5.98,
      "learning_rate": 0.00011255568181818181,
      "loss": 2.8326,
      "step": 102950
    },
    {
      "activations/layer0_attention_weight_max": 15.572254180908203,
      "activations/layer0_attention_weight_min": -11.056510925292969,
      "activations/layer10_attention_weight_max": 36.81306838989258,
      "activations/layer10_attention_weight_min": -32.682594299316406,
      "activations/layer11_attention_weight_max": 37.306060791015625,
      "activations/layer11_attention_weight_min": -29.209516525268555,
      "activations/layer12_attention_weight_max": 28.23972511291504,
      "activations/layer12_attention_weight_min": -25.472572326660156,
      "activations/layer13_attention_weight_max": 45.92051696777344,
      "activations/layer13_attention_weight_min": -32.08223342895508,
      "activations/layer14_attention_weight_max": 68.23902893066406,
      "activations/layer14_attention_weight_min": -44.68267822265625,
      "activations/layer15_attention_weight_max": 41.03733444213867,
      "activations/layer15_attention_weight_min": -28.9919490814209,
      "activations/layer16_attention_weight_max": 39.21682357788086,
      "activations/layer16_attention_weight_min": -28.314184188842773,
      "activations/layer17_attention_weight_max": 41.263038635253906,
      "activations/layer17_attention_weight_min": -25.182161331176758,
      "activations/layer18_attention_weight_max": 35.24599838256836,
      "activations/layer18_attention_weight_min": -21.625091552734375,
      "activations/layer19_attention_weight_max": 47.771663665771484,
      "activations/layer19_attention_weight_min": -30.650074005126953,
      "activations/layer1_attention_weight_max": 17.433019638061523,
      "activations/layer1_attention_weight_min": -16.92488670349121,
      "activations/layer20_attention_weight_max": 35.847896575927734,
      "activations/layer20_attention_weight_min": -24.11980628967285,
      "activations/layer21_attention_weight_max": 35.614356994628906,
      "activations/layer21_attention_weight_min": -25.40923500061035,
      "activations/layer22_attention_weight_max": 62.65336990356445,
      "activations/layer22_attention_weight_min": -32.903995513916016,
      "activations/layer23_attention_weight_max": 39.39091110229492,
      "activations/layer23_attention_weight_min": -22.938499450683594,
      "activations/layer2_attention_weight_max": 31.34766387939453,
      "activations/layer2_attention_weight_min": -29.94955062866211,
      "activations/layer3_attention_weight_max": 94.97601318359375,
      "activations/layer3_attention_weight_min": -99.50372314453125,
      "activations/layer4_attention_weight_max": 133.0474395751953,
      "activations/layer4_attention_weight_min": -136.46263122558594,
      "activations/layer5_attention_weight_max": 76.04202270507812,
      "activations/layer5_attention_weight_min": -72.70207214355469,
      "activations/layer6_attention_weight_max": 55.66461181640625,
      "activations/layer6_attention_weight_min": -54.85269546508789,
      "activations/layer7_attention_weight_max": 88.33478546142578,
      "activations/layer7_attention_weight_min": -87.86654663085938,
      "activations/layer8_attention_weight_max": 41.70439147949219,
      "activations/layer8_attention_weight_min": -41.35188293457031,
      "activations/layer9_attention_weight_max": 40.9272575378418,
      "activations/layer9_attention_weight_min": -35.52579116821289,
      "epoch": 5.98,
      "learning_rate": 0.00011253674242424241,
      "loss": 2.8317,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_loss": 2.78515625,
      "eval_runtime": 8.4739,
      "eval_samples_per_second": 506.73,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_openwebtext_loss": 2.78515625,
      "eval_openwebtext_ppl": 16.202349296067883,
      "eval_openwebtext_runtime": 8.4739,
      "eval_openwebtext_samples_per_second": 506.73,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_wikitext_loss": 3.013671875,
      "eval_wikitext_ppl": 20.36202965350754,
      "eval_wikitext_runtime": 1.9457,
      "eval_wikitext_samples_per_second": 234.36,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_lambada_loss": 2.69140625,
      "eval_lambada_ppl": 14.75240691276831,
      "eval_lambada_runtime": 9.5415,
      "eval_lambada_samples_per_second": 510.296,
      "step": 103000
    },
    {
      "activations/layer0_attention_weight_max": 15.052152633666992,
      "activations/layer0_attention_weight_min": -12.60014533996582,
      "activations/layer10_attention_weight_max": 39.195045471191406,
      "activations/layer10_attention_weight_min": -34.29463577270508,
      "activations/layer11_attention_weight_max": 35.56084442138672,
      "activations/layer11_attention_weight_min": -28.63515853881836,
      "activations/layer12_attention_weight_max": 36.6297721862793,
      "activations/layer12_attention_weight_min": -26.72762680053711,
      "activations/layer13_attention_weight_max": 48.90638732910156,
      "activations/layer13_attention_weight_min": -34.18766784667969,
      "activations/layer14_attention_weight_max": 62.9452018737793,
      "activations/layer14_attention_weight_min": -47.7026252746582,
      "activations/layer15_attention_weight_max": 43.149044036865234,
      "activations/layer15_attention_weight_min": -29.108354568481445,
      "activations/layer16_attention_weight_max": 39.906089782714844,
      "activations/layer16_attention_weight_min": -27.86243438720703,
      "activations/layer17_attention_weight_max": 46.31593704223633,
      "activations/layer17_attention_weight_min": -25.003677368164062,
      "activations/layer18_attention_weight_max": 30.805448532104492,
      "activations/layer18_attention_weight_min": -21.15895652770996,
      "activations/layer19_attention_weight_max": 42.861045837402344,
      "activations/layer19_attention_weight_min": -32.132137298583984,
      "activations/layer1_attention_weight_max": 16.659358978271484,
      "activations/layer1_attention_weight_min": -13.131556510925293,
      "activations/layer20_attention_weight_max": 35.63096618652344,
      "activations/layer20_attention_weight_min": -25.756786346435547,
      "activations/layer21_attention_weight_max": 32.18765640258789,
      "activations/layer21_attention_weight_min": -24.74717903137207,
      "activations/layer22_attention_weight_max": 55.29563903808594,
      "activations/layer22_attention_weight_min": -32.49943923950195,
      "activations/layer23_attention_weight_max": 39.186073303222656,
      "activations/layer23_attention_weight_min": -21.01689338684082,
      "activations/layer2_attention_weight_max": 31.501615524291992,
      "activations/layer2_attention_weight_min": -30.18275260925293,
      "activations/layer3_attention_weight_max": 87.85636138916016,
      "activations/layer3_attention_weight_min": -90.88378143310547,
      "activations/layer4_attention_weight_max": 126.88203430175781,
      "activations/layer4_attention_weight_min": -124.8165512084961,
      "activations/layer5_attention_weight_max": 77.0024642944336,
      "activations/layer5_attention_weight_min": -76.3411636352539,
      "activations/layer6_attention_weight_max": 56.19944381713867,
      "activations/layer6_attention_weight_min": -58.321720123291016,
      "activations/layer7_attention_weight_max": 89.26089477539062,
      "activations/layer7_attention_weight_min": -89.5098876953125,
      "activations/layer8_attention_weight_max": 43.80729293823242,
      "activations/layer8_attention_weight_min": -43.96597671508789,
      "activations/layer9_attention_weight_max": 43.14601516723633,
      "activations/layer9_attention_weight_min": -37.364559173583984,
      "epoch": 5.99,
      "learning_rate": 0.00011251780303030303,
      "loss": 2.8144,
      "step": 103050
    },
    {
      "activations/layer0_attention_weight_max": 16.170984268188477,
      "activations/layer0_attention_weight_min": -11.286981582641602,
      "activations/layer10_attention_weight_max": 35.858856201171875,
      "activations/layer10_attention_weight_min": -32.36469650268555,
      "activations/layer11_attention_weight_max": 32.63127136230469,
      "activations/layer11_attention_weight_min": -32.44197082519531,
      "activations/layer12_attention_weight_max": 24.470008850097656,
      "activations/layer12_attention_weight_min": -24.253171920776367,
      "activations/layer13_attention_weight_max": 44.47182846069336,
      "activations/layer13_attention_weight_min": -34.378326416015625,
      "activations/layer14_attention_weight_max": 66.4146957397461,
      "activations/layer14_attention_weight_min": -44.315948486328125,
      "activations/layer15_attention_weight_max": 40.67233657836914,
      "activations/layer15_attention_weight_min": -31.08485221862793,
      "activations/layer16_attention_weight_max": 36.24822998046875,
      "activations/layer16_attention_weight_min": -28.38316535949707,
      "activations/layer17_attention_weight_max": 38.87261962890625,
      "activations/layer17_attention_weight_min": -25.987449645996094,
      "activations/layer18_attention_weight_max": 28.787389755249023,
      "activations/layer18_attention_weight_min": -21.794429779052734,
      "activations/layer19_attention_weight_max": 38.64851379394531,
      "activations/layer19_attention_weight_min": -31.458112716674805,
      "activations/layer1_attention_weight_max": 16.59515380859375,
      "activations/layer1_attention_weight_min": -13.400541305541992,
      "activations/layer20_attention_weight_max": 34.523372650146484,
      "activations/layer20_attention_weight_min": -23.823535919189453,
      "activations/layer21_attention_weight_max": 31.965084075927734,
      "activations/layer21_attention_weight_min": -20.873945236206055,
      "activations/layer22_attention_weight_max": 50.82223892211914,
      "activations/layer22_attention_weight_min": -27.835386276245117,
      "activations/layer23_attention_weight_max": 36.67255401611328,
      "activations/layer23_attention_weight_min": -22.132858276367188,
      "activations/layer2_attention_weight_max": 33.660980224609375,
      "activations/layer2_attention_weight_min": -32.42702102661133,
      "activations/layer3_attention_weight_max": 94.64393615722656,
      "activations/layer3_attention_weight_min": -93.5050277709961,
      "activations/layer4_attention_weight_max": 140.07249450683594,
      "activations/layer4_attention_weight_min": -130.73573303222656,
      "activations/layer5_attention_weight_max": 78.73545837402344,
      "activations/layer5_attention_weight_min": -74.03213500976562,
      "activations/layer6_attention_weight_max": 61.332374572753906,
      "activations/layer6_attention_weight_min": -58.087120056152344,
      "activations/layer7_attention_weight_max": 90.97074890136719,
      "activations/layer7_attention_weight_min": -87.34386444091797,
      "activations/layer8_attention_weight_max": 42.5060920715332,
      "activations/layer8_attention_weight_min": -43.1155891418457,
      "activations/layer9_attention_weight_max": 39.830020904541016,
      "activations/layer9_attention_weight_min": -35.54715347290039,
      "epoch": 5.99,
      "learning_rate": 0.00011249886363636363,
      "loss": 2.8378,
      "step": 103100
    },
    {
      "activations/layer0_attention_weight_max": 16.23564338684082,
      "activations/layer0_attention_weight_min": -11.128557205200195,
      "activations/layer10_attention_weight_max": 33.62884521484375,
      "activations/layer10_attention_weight_min": -32.00248718261719,
      "activations/layer11_attention_weight_max": 33.97417068481445,
      "activations/layer11_attention_weight_min": -30.1036434173584,
      "activations/layer12_attention_weight_max": 28.00468635559082,
      "activations/layer12_attention_weight_min": -25.22945785522461,
      "activations/layer13_attention_weight_max": 45.57785415649414,
      "activations/layer13_attention_weight_min": -33.33827209472656,
      "activations/layer14_attention_weight_max": 63.47043228149414,
      "activations/layer14_attention_weight_min": -51.43015670776367,
      "activations/layer15_attention_weight_max": 36.29737091064453,
      "activations/layer15_attention_weight_min": -27.769136428833008,
      "activations/layer16_attention_weight_max": 39.979347229003906,
      "activations/layer16_attention_weight_min": -27.476089477539062,
      "activations/layer17_attention_weight_max": 35.348777770996094,
      "activations/layer17_attention_weight_min": -23.458181381225586,
      "activations/layer18_attention_weight_max": 31.28857421875,
      "activations/layer18_attention_weight_min": -20.358613967895508,
      "activations/layer19_attention_weight_max": 40.52743148803711,
      "activations/layer19_attention_weight_min": -30.232242584228516,
      "activations/layer1_attention_weight_max": 17.14021110534668,
      "activations/layer1_attention_weight_min": -13.972053527832031,
      "activations/layer20_attention_weight_max": 31.46782875061035,
      "activations/layer20_attention_weight_min": -24.800945281982422,
      "activations/layer21_attention_weight_max": 33.0072135925293,
      "activations/layer21_attention_weight_min": -19.529556274414062,
      "activations/layer22_attention_weight_max": 46.56906509399414,
      "activations/layer22_attention_weight_min": -26.538715362548828,
      "activations/layer23_attention_weight_max": 36.76278305053711,
      "activations/layer23_attention_weight_min": -20.545024871826172,
      "activations/layer2_attention_weight_max": 33.53278732299805,
      "activations/layer2_attention_weight_min": -31.842342376708984,
      "activations/layer3_attention_weight_max": 94.26042175292969,
      "activations/layer3_attention_weight_min": -93.86194610595703,
      "activations/layer4_attention_weight_max": 133.8772430419922,
      "activations/layer4_attention_weight_min": -127.5946273803711,
      "activations/layer5_attention_weight_max": 80.81153869628906,
      "activations/layer5_attention_weight_min": -72.42025756835938,
      "activations/layer6_attention_weight_max": 58.125244140625,
      "activations/layer6_attention_weight_min": -54.328433990478516,
      "activations/layer7_attention_weight_max": 86.89920043945312,
      "activations/layer7_attention_weight_min": -86.99310302734375,
      "activations/layer8_attention_weight_max": 42.587120056152344,
      "activations/layer8_attention_weight_min": -41.810977935791016,
      "activations/layer9_attention_weight_max": 37.65709686279297,
      "activations/layer9_attention_weight_min": -33.84494400024414,
      "epoch": 5.99,
      "learning_rate": 0.00011247992424242423,
      "loss": 2.8218,
      "step": 103150
    },
    {
      "activations/layer0_attention_weight_max": 16.09600257873535,
      "activations/layer0_attention_weight_min": -12.100454330444336,
      "activations/layer10_attention_weight_max": 35.40424346923828,
      "activations/layer10_attention_weight_min": -31.632070541381836,
      "activations/layer11_attention_weight_max": 32.516693115234375,
      "activations/layer11_attention_weight_min": -28.099735260009766,
      "activations/layer12_attention_weight_max": 25.272621154785156,
      "activations/layer12_attention_weight_min": -26.56650733947754,
      "activations/layer13_attention_weight_max": 46.88396072387695,
      "activations/layer13_attention_weight_min": -32.681854248046875,
      "activations/layer14_attention_weight_max": 58.15480041503906,
      "activations/layer14_attention_weight_min": -43.594696044921875,
      "activations/layer15_attention_weight_max": 38.66450881958008,
      "activations/layer15_attention_weight_min": -29.13928985595703,
      "activations/layer16_attention_weight_max": 39.586727142333984,
      "activations/layer16_attention_weight_min": -29.201860427856445,
      "activations/layer17_attention_weight_max": 38.90053176879883,
      "activations/layer17_attention_weight_min": -23.92679786682129,
      "activations/layer18_attention_weight_max": 31.350175857543945,
      "activations/layer18_attention_weight_min": -21.09121322631836,
      "activations/layer19_attention_weight_max": 37.8848876953125,
      "activations/layer19_attention_weight_min": -28.712778091430664,
      "activations/layer1_attention_weight_max": 16.476598739624023,
      "activations/layer1_attention_weight_min": -13.163671493530273,
      "activations/layer20_attention_weight_max": 33.9897575378418,
      "activations/layer20_attention_weight_min": -23.8010196685791,
      "activations/layer21_attention_weight_max": 31.728025436401367,
      "activations/layer21_attention_weight_min": -20.34256362915039,
      "activations/layer22_attention_weight_max": 49.83134841918945,
      "activations/layer22_attention_weight_min": -27.144060134887695,
      "activations/layer23_attention_weight_max": 33.0646858215332,
      "activations/layer23_attention_weight_min": -22.101318359375,
      "activations/layer2_attention_weight_max": 32.18745422363281,
      "activations/layer2_attention_weight_min": -32.61488723754883,
      "activations/layer3_attention_weight_max": 92.67841339111328,
      "activations/layer3_attention_weight_min": -94.76799774169922,
      "activations/layer4_attention_weight_max": 129.6205291748047,
      "activations/layer4_attention_weight_min": -124.84114837646484,
      "activations/layer5_attention_weight_max": 73.0073013305664,
      "activations/layer5_attention_weight_min": -70.83457946777344,
      "activations/layer6_attention_weight_max": 51.36060333251953,
      "activations/layer6_attention_weight_min": -53.24808120727539,
      "activations/layer7_attention_weight_max": 85.41324615478516,
      "activations/layer7_attention_weight_min": -82.28240203857422,
      "activations/layer8_attention_weight_max": 44.11256790161133,
      "activations/layer8_attention_weight_min": -40.67143249511719,
      "activations/layer9_attention_weight_max": 40.30387496948242,
      "activations/layer9_attention_weight_min": -34.21978759765625,
      "epoch": 6.0,
      "learning_rate": 0.00011246098484848485,
      "loss": 2.8274,
      "step": 103200
    },
    {
      "activations/layer0_attention_weight_max": 16.767414093017578,
      "activations/layer0_attention_weight_min": -12.37427043914795,
      "activations/layer10_attention_weight_max": 34.870277404785156,
      "activations/layer10_attention_weight_min": -33.64765548706055,
      "activations/layer11_attention_weight_max": 31.404111862182617,
      "activations/layer11_attention_weight_min": -28.358463287353516,
      "activations/layer12_attention_weight_max": 30.09708023071289,
      "activations/layer12_attention_weight_min": -25.837326049804688,
      "activations/layer13_attention_weight_max": 44.2231559753418,
      "activations/layer13_attention_weight_min": -31.976280212402344,
      "activations/layer14_attention_weight_max": 61.981895446777344,
      "activations/layer14_attention_weight_min": -44.77709197998047,
      "activations/layer15_attention_weight_max": 36.06538391113281,
      "activations/layer15_attention_weight_min": -26.51628303527832,
      "activations/layer16_attention_weight_max": 36.60380172729492,
      "activations/layer16_attention_weight_min": -26.915361404418945,
      "activations/layer17_attention_weight_max": 34.88960647583008,
      "activations/layer17_attention_weight_min": -24.536705017089844,
      "activations/layer18_attention_weight_max": 30.28133773803711,
      "activations/layer18_attention_weight_min": -21.37053108215332,
      "activations/layer19_attention_weight_max": 38.39116668701172,
      "activations/layer19_attention_weight_min": -28.753049850463867,
      "activations/layer1_attention_weight_max": 16.427364349365234,
      "activations/layer1_attention_weight_min": -13.992376327514648,
      "activations/layer20_attention_weight_max": 30.881065368652344,
      "activations/layer20_attention_weight_min": -22.65387535095215,
      "activations/layer21_attention_weight_max": 29.407011032104492,
      "activations/layer21_attention_weight_min": -21.32612419128418,
      "activations/layer22_attention_weight_max": 47.92470932006836,
      "activations/layer22_attention_weight_min": -28.47389030456543,
      "activations/layer23_attention_weight_max": 32.6004524230957,
      "activations/layer23_attention_weight_min": -21.071090698242188,
      "activations/layer2_attention_weight_max": 32.11238098144531,
      "activations/layer2_attention_weight_min": -32.1249885559082,
      "activations/layer3_attention_weight_max": 95.83001708984375,
      "activations/layer3_attention_weight_min": -97.35221099853516,
      "activations/layer4_attention_weight_max": 128.11761474609375,
      "activations/layer4_attention_weight_min": -127.34517669677734,
      "activations/layer5_attention_weight_max": 75.55474853515625,
      "activations/layer5_attention_weight_min": -72.73686218261719,
      "activations/layer6_attention_weight_max": 54.35076141357422,
      "activations/layer6_attention_weight_min": -56.495361328125,
      "activations/layer7_attention_weight_max": 87.8837890625,
      "activations/layer7_attention_weight_min": -88.99505615234375,
      "activations/layer8_attention_weight_max": 44.284423828125,
      "activations/layer8_attention_weight_min": -44.91666030883789,
      "activations/layer9_attention_weight_max": 40.106632232666016,
      "activations/layer9_attention_weight_min": -37.16523361206055,
      "epoch": 6.0,
      "learning_rate": 0.00011244204545454543,
      "loss": 2.8191,
      "step": 103250
    },
    {
      "activations/layer0_attention_weight_max": 15.35439682006836,
      "activations/layer0_attention_weight_min": -10.7449312210083,
      "activations/layer10_attention_weight_max": 34.95863342285156,
      "activations/layer10_attention_weight_min": -32.40660095214844,
      "activations/layer11_attention_weight_max": 37.504554748535156,
      "activations/layer11_attention_weight_min": -27.955541610717773,
      "activations/layer12_attention_weight_max": 36.48502731323242,
      "activations/layer12_attention_weight_min": -32.382415771484375,
      "activations/layer13_attention_weight_max": 43.12511444091797,
      "activations/layer13_attention_weight_min": -32.88611602783203,
      "activations/layer14_attention_weight_max": 54.519989013671875,
      "activations/layer14_attention_weight_min": -42.72759246826172,
      "activations/layer15_attention_weight_max": 46.59773254394531,
      "activations/layer15_attention_weight_min": -30.39165496826172,
      "activations/layer16_attention_weight_max": 37.01659393310547,
      "activations/layer16_attention_weight_min": -26.66925048828125,
      "activations/layer17_attention_weight_max": 37.33428955078125,
      "activations/layer17_attention_weight_min": -24.537569046020508,
      "activations/layer18_attention_weight_max": 40.810699462890625,
      "activations/layer18_attention_weight_min": -20.676559448242188,
      "activations/layer19_attention_weight_max": 43.33547592163086,
      "activations/layer19_attention_weight_min": -31.397953033447266,
      "activations/layer1_attention_weight_max": 18.35262680053711,
      "activations/layer1_attention_weight_min": -14.998678207397461,
      "activations/layer20_attention_weight_max": 34.76564407348633,
      "activations/layer20_attention_weight_min": -24.98720359802246,
      "activations/layer21_attention_weight_max": 36.70158767700195,
      "activations/layer21_attention_weight_min": -20.61217498779297,
      "activations/layer22_attention_weight_max": 57.52092742919922,
      "activations/layer22_attention_weight_min": -27.32459259033203,
      "activations/layer23_attention_weight_max": 38.60888671875,
      "activations/layer23_attention_weight_min": -20.813396453857422,
      "activations/layer2_attention_weight_max": 34.724212646484375,
      "activations/layer2_attention_weight_min": -33.63629913330078,
      "activations/layer3_attention_weight_max": 97.4363021850586,
      "activations/layer3_attention_weight_min": -97.90016174316406,
      "activations/layer4_attention_weight_max": 132.2326202392578,
      "activations/layer4_attention_weight_min": -132.32958984375,
      "activations/layer5_attention_weight_max": 74.8932113647461,
      "activations/layer5_attention_weight_min": -74.6539077758789,
      "activations/layer6_attention_weight_max": 56.894229888916016,
      "activations/layer6_attention_weight_min": -55.832427978515625,
      "activations/layer7_attention_weight_max": 91.78057098388672,
      "activations/layer7_attention_weight_min": -90.02734375,
      "activations/layer8_attention_weight_max": 41.85334014892578,
      "activations/layer8_attention_weight_min": -41.43131637573242,
      "activations/layer9_attention_weight_max": 37.979549407958984,
      "activations/layer9_attention_weight_min": -41.496498107910156,
      "epoch": 6.0,
      "learning_rate": 0.00011242310606060605,
      "loss": 2.8724,
      "step": 103300
    },
    {
      "activations/layer0_attention_weight_max": 15.621244430541992,
      "activations/layer0_attention_weight_min": -11.480456352233887,
      "activations/layer10_attention_weight_max": 34.53437042236328,
      "activations/layer10_attention_weight_min": -31.932479858398438,
      "activations/layer11_attention_weight_max": 33.54982376098633,
      "activations/layer11_attention_weight_min": -28.56110382080078,
      "activations/layer12_attention_weight_max": 26.65325164794922,
      "activations/layer12_attention_weight_min": -25.618473052978516,
      "activations/layer13_attention_weight_max": 44.067203521728516,
      "activations/layer13_attention_weight_min": -33.83324432373047,
      "activations/layer14_attention_weight_max": 74.03307342529297,
      "activations/layer14_attention_weight_min": -50.245750427246094,
      "activations/layer15_attention_weight_max": 40.586387634277344,
      "activations/layer15_attention_weight_min": -27.816911697387695,
      "activations/layer16_attention_weight_max": 37.909393310546875,
      "activations/layer16_attention_weight_min": -28.719636917114258,
      "activations/layer17_attention_weight_max": 41.35271072387695,
      "activations/layer17_attention_weight_min": -23.543643951416016,
      "activations/layer18_attention_weight_max": 32.45360565185547,
      "activations/layer18_attention_weight_min": -20.783912658691406,
      "activations/layer19_attention_weight_max": 38.96589279174805,
      "activations/layer19_attention_weight_min": -28.93499183654785,
      "activations/layer1_attention_weight_max": 18.11391830444336,
      "activations/layer1_attention_weight_min": -15.897406578063965,
      "activations/layer20_attention_weight_max": 32.31129455566406,
      "activations/layer20_attention_weight_min": -24.841957092285156,
      "activations/layer21_attention_weight_max": 33.124114990234375,
      "activations/layer21_attention_weight_min": -21.645118713378906,
      "activations/layer22_attention_weight_max": 53.230953216552734,
      "activations/layer22_attention_weight_min": -28.489595413208008,
      "activations/layer23_attention_weight_max": 36.0788459777832,
      "activations/layer23_attention_weight_min": -20.797834396362305,
      "activations/layer2_attention_weight_max": 31.780563354492188,
      "activations/layer2_attention_weight_min": -30.276504516601562,
      "activations/layer3_attention_weight_max": 92.26866149902344,
      "activations/layer3_attention_weight_min": -90.56002044677734,
      "activations/layer4_attention_weight_max": 127.85267639160156,
      "activations/layer4_attention_weight_min": -123.98371887207031,
      "activations/layer5_attention_weight_max": 74.62117767333984,
      "activations/layer5_attention_weight_min": -70.77677917480469,
      "activations/layer6_attention_weight_max": 53.80596160888672,
      "activations/layer6_attention_weight_min": -54.08481216430664,
      "activations/layer7_attention_weight_max": 84.13591766357422,
      "activations/layer7_attention_weight_min": -83.06751251220703,
      "activations/layer8_attention_weight_max": 39.973697662353516,
      "activations/layer8_attention_weight_min": -39.19050979614258,
      "activations/layer9_attention_weight_max": 37.15802001953125,
      "activations/layer9_attention_weight_min": -35.5955696105957,
      "epoch": 6.01,
      "learning_rate": 0.00011240416666666666,
      "loss": 2.8174,
      "step": 103350
    },
    {
      "activations/layer0_attention_weight_max": 16.798837661743164,
      "activations/layer0_attention_weight_min": -12.329252243041992,
      "activations/layer10_attention_weight_max": 33.08028793334961,
      "activations/layer10_attention_weight_min": -31.40587043762207,
      "activations/layer11_attention_weight_max": 33.34782409667969,
      "activations/layer11_attention_weight_min": -29.210996627807617,
      "activations/layer12_attention_weight_max": 22.82302474975586,
      "activations/layer12_attention_weight_min": -25.79192543029785,
      "activations/layer13_attention_weight_max": 43.841651916503906,
      "activations/layer13_attention_weight_min": -32.4300422668457,
      "activations/layer14_attention_weight_max": 59.76844787597656,
      "activations/layer14_attention_weight_min": -42.68374252319336,
      "activations/layer15_attention_weight_max": 38.11531066894531,
      "activations/layer15_attention_weight_min": -27.893423080444336,
      "activations/layer16_attention_weight_max": 39.99417495727539,
      "activations/layer16_attention_weight_min": -27.28877067565918,
      "activations/layer17_attention_weight_max": 37.1639404296875,
      "activations/layer17_attention_weight_min": -24.762481689453125,
      "activations/layer18_attention_weight_max": 36.41571807861328,
      "activations/layer18_attention_weight_min": -20.346118927001953,
      "activations/layer19_attention_weight_max": 41.674903869628906,
      "activations/layer19_attention_weight_min": -30.258258819580078,
      "activations/layer1_attention_weight_max": 17.177146911621094,
      "activations/layer1_attention_weight_min": -14.943066596984863,
      "activations/layer20_attention_weight_max": 32.22016143798828,
      "activations/layer20_attention_weight_min": -24.714431762695312,
      "activations/layer21_attention_weight_max": 38.305824279785156,
      "activations/layer21_attention_weight_min": -21.02823257446289,
      "activations/layer22_attention_weight_max": 49.38745880126953,
      "activations/layer22_attention_weight_min": -28.155881881713867,
      "activations/layer23_attention_weight_max": 35.92832946777344,
      "activations/layer23_attention_weight_min": -22.732999801635742,
      "activations/layer2_attention_weight_max": 31.898962020874023,
      "activations/layer2_attention_weight_min": -31.12841033935547,
      "activations/layer3_attention_weight_max": 93.6395034790039,
      "activations/layer3_attention_weight_min": -92.22959899902344,
      "activations/layer4_attention_weight_max": 130.84310913085938,
      "activations/layer4_attention_weight_min": -125.17745208740234,
      "activations/layer5_attention_weight_max": 73.73446655273438,
      "activations/layer5_attention_weight_min": -71.83122253417969,
      "activations/layer6_attention_weight_max": 55.21352005004883,
      "activations/layer6_attention_weight_min": -54.93214416503906,
      "activations/layer7_attention_weight_max": 82.62893676757812,
      "activations/layer7_attention_weight_min": -92.53953552246094,
      "activations/layer8_attention_weight_max": 41.67741775512695,
      "activations/layer8_attention_weight_min": -41.51337432861328,
      "activations/layer9_attention_weight_max": 40.02976608276367,
      "activations/layer9_attention_weight_min": -34.05035400390625,
      "epoch": 6.01,
      "learning_rate": 0.00011238522727272725,
      "loss": 2.8191,
      "step": 103400
    },
    {
      "activations/layer0_attention_weight_max": 16.776865005493164,
      "activations/layer0_attention_weight_min": -11.69301700592041,
      "activations/layer10_attention_weight_max": 34.489410400390625,
      "activations/layer10_attention_weight_min": -31.448863983154297,
      "activations/layer11_attention_weight_max": 32.90156936645508,
      "activations/layer11_attention_weight_min": -28.929611206054688,
      "activations/layer12_attention_weight_max": 21.20783805847168,
      "activations/layer12_attention_weight_min": -26.509504318237305,
      "activations/layer13_attention_weight_max": 43.92604064941406,
      "activations/layer13_attention_weight_min": -34.92980194091797,
      "activations/layer14_attention_weight_max": 66.261962890625,
      "activations/layer14_attention_weight_min": -52.77622985839844,
      "activations/layer15_attention_weight_max": 37.61738586425781,
      "activations/layer15_attention_weight_min": -30.323898315429688,
      "activations/layer16_attention_weight_max": 36.64287185668945,
      "activations/layer16_attention_weight_min": -28.337541580200195,
      "activations/layer17_attention_weight_max": 35.376461029052734,
      "activations/layer17_attention_weight_min": -26.5318546295166,
      "activations/layer18_attention_weight_max": 28.48043441772461,
      "activations/layer18_attention_weight_min": -20.426715850830078,
      "activations/layer19_attention_weight_max": 38.81589889526367,
      "activations/layer19_attention_weight_min": -29.78785514831543,
      "activations/layer1_attention_weight_max": 17.752281188964844,
      "activations/layer1_attention_weight_min": -14.02370834350586,
      "activations/layer20_attention_weight_max": 32.75558853149414,
      "activations/layer20_attention_weight_min": -24.043546676635742,
      "activations/layer21_attention_weight_max": 28.549190521240234,
      "activations/layer21_attention_weight_min": -21.990808486938477,
      "activations/layer22_attention_weight_max": 56.065818786621094,
      "activations/layer22_attention_weight_min": -29.33928108215332,
      "activations/layer23_attention_weight_max": 33.32604217529297,
      "activations/layer23_attention_weight_min": -23.61330223083496,
      "activations/layer2_attention_weight_max": 32.19744873046875,
      "activations/layer2_attention_weight_min": -30.351058959960938,
      "activations/layer3_attention_weight_max": 94.30302429199219,
      "activations/layer3_attention_weight_min": -94.17152404785156,
      "activations/layer4_attention_weight_max": 132.22142028808594,
      "activations/layer4_attention_weight_min": -128.49266052246094,
      "activations/layer5_attention_weight_max": 75.8474349975586,
      "activations/layer5_attention_weight_min": -72.2226791381836,
      "activations/layer6_attention_weight_max": 53.60968780517578,
      "activations/layer6_attention_weight_min": -54.85065460205078,
      "activations/layer7_attention_weight_max": 84.52781677246094,
      "activations/layer7_attention_weight_min": -83.57110595703125,
      "activations/layer8_attention_weight_max": 43.579402923583984,
      "activations/layer8_attention_weight_min": -41.85055923461914,
      "activations/layer9_attention_weight_max": 37.622623443603516,
      "activations/layer9_attention_weight_min": -35.33889389038086,
      "epoch": 6.01,
      "learning_rate": 0.00011236628787878787,
      "loss": 2.8435,
      "step": 103450
    },
    {
      "activations/layer0_attention_weight_max": 16.030498504638672,
      "activations/layer0_attention_weight_min": -12.564213752746582,
      "activations/layer10_attention_weight_max": 34.175575256347656,
      "activations/layer10_attention_weight_min": -31.682024002075195,
      "activations/layer11_attention_weight_max": 33.400489807128906,
      "activations/layer11_attention_weight_min": -27.974445343017578,
      "activations/layer12_attention_weight_max": 30.411701202392578,
      "activations/layer12_attention_weight_min": -22.87130355834961,
      "activations/layer13_attention_weight_max": 49.4363899230957,
      "activations/layer13_attention_weight_min": -33.07218551635742,
      "activations/layer14_attention_weight_max": 69.77471923828125,
      "activations/layer14_attention_weight_min": -44.77234649658203,
      "activations/layer15_attention_weight_max": 41.50736618041992,
      "activations/layer15_attention_weight_min": -27.95319366455078,
      "activations/layer16_attention_weight_max": 41.02032470703125,
      "activations/layer16_attention_weight_min": -28.251415252685547,
      "activations/layer17_attention_weight_max": 42.4814338684082,
      "activations/layer17_attention_weight_min": -24.93514060974121,
      "activations/layer18_attention_weight_max": 42.17319107055664,
      "activations/layer18_attention_weight_min": -20.589176177978516,
      "activations/layer19_attention_weight_max": 44.63499069213867,
      "activations/layer19_attention_weight_min": -30.281661987304688,
      "activations/layer1_attention_weight_max": 17.761884689331055,
      "activations/layer1_attention_weight_min": -15.468156814575195,
      "activations/layer20_attention_weight_max": 34.646968841552734,
      "activations/layer20_attention_weight_min": -23.125843048095703,
      "activations/layer21_attention_weight_max": 38.0390510559082,
      "activations/layer21_attention_weight_min": -20.75889015197754,
      "activations/layer22_attention_weight_max": 58.70672607421875,
      "activations/layer22_attention_weight_min": -27.57839584350586,
      "activations/layer23_attention_weight_max": 44.22356414794922,
      "activations/layer23_attention_weight_min": -21.00381088256836,
      "activations/layer2_attention_weight_max": 32.70855712890625,
      "activations/layer2_attention_weight_min": -31.404579162597656,
      "activations/layer3_attention_weight_max": 92.13046264648438,
      "activations/layer3_attention_weight_min": -89.43223571777344,
      "activations/layer4_attention_weight_max": 127.60637664794922,
      "activations/layer4_attention_weight_min": -123.05438232421875,
      "activations/layer5_attention_weight_max": 76.31153869628906,
      "activations/layer5_attention_weight_min": -72.73969268798828,
      "activations/layer6_attention_weight_max": 57.028106689453125,
      "activations/layer6_attention_weight_min": -56.61909484863281,
      "activations/layer7_attention_weight_max": 87.37020874023438,
      "activations/layer7_attention_weight_min": -90.84815979003906,
      "activations/layer8_attention_weight_max": 39.770748138427734,
      "activations/layer8_attention_weight_min": -41.4240837097168,
      "activations/layer9_attention_weight_max": 37.976436614990234,
      "activations/layer9_attention_weight_min": -35.63014221191406,
      "epoch": 6.01,
      "learning_rate": 0.00011234734848484848,
      "loss": 2.82,
      "step": 103500
    },
    {
      "activations/layer0_attention_weight_max": 15.16872787475586,
      "activations/layer0_attention_weight_min": -12.538416862487793,
      "activations/layer10_attention_weight_max": 33.21115493774414,
      "activations/layer10_attention_weight_min": -32.14794921875,
      "activations/layer11_attention_weight_max": 35.319496154785156,
      "activations/layer11_attention_weight_min": -28.954442977905273,
      "activations/layer12_attention_weight_max": 26.948265075683594,
      "activations/layer12_attention_weight_min": -24.35528564453125,
      "activations/layer13_attention_weight_max": 46.50004577636719,
      "activations/layer13_attention_weight_min": -32.25290298461914,
      "activations/layer14_attention_weight_max": 62.569766998291016,
      "activations/layer14_attention_weight_min": -44.53062057495117,
      "activations/layer15_attention_weight_max": 41.90449142456055,
      "activations/layer15_attention_weight_min": -30.151750564575195,
      "activations/layer16_attention_weight_max": 37.52320098876953,
      "activations/layer16_attention_weight_min": -28.705631256103516,
      "activations/layer17_attention_weight_max": 41.11299514770508,
      "activations/layer17_attention_weight_min": -25.08281898498535,
      "activations/layer18_attention_weight_max": 33.86036682128906,
      "activations/layer18_attention_weight_min": -22.808242797851562,
      "activations/layer19_attention_weight_max": 46.7623405456543,
      "activations/layer19_attention_weight_min": -30.827234268188477,
      "activations/layer1_attention_weight_max": 17.546764373779297,
      "activations/layer1_attention_weight_min": -13.608126640319824,
      "activations/layer20_attention_weight_max": 32.89767837524414,
      "activations/layer20_attention_weight_min": -24.772533416748047,
      "activations/layer21_attention_weight_max": 35.106719970703125,
      "activations/layer21_attention_weight_min": -21.72186279296875,
      "activations/layer22_attention_weight_max": 50.1019172668457,
      "activations/layer22_attention_weight_min": -30.502994537353516,
      "activations/layer23_attention_weight_max": 33.713985443115234,
      "activations/layer23_attention_weight_min": -22.55769920349121,
      "activations/layer2_attention_weight_max": 32.777870178222656,
      "activations/layer2_attention_weight_min": -31.234081268310547,
      "activations/layer3_attention_weight_max": 91.98772430419922,
      "activations/layer3_attention_weight_min": -95.42981719970703,
      "activations/layer4_attention_weight_max": 126.48736572265625,
      "activations/layer4_attention_weight_min": -117.69449615478516,
      "activations/layer5_attention_weight_max": 71.09431457519531,
      "activations/layer5_attention_weight_min": -67.86700439453125,
      "activations/layer6_attention_weight_max": 54.062862396240234,
      "activations/layer6_attention_weight_min": -56.22895812988281,
      "activations/layer7_attention_weight_max": 83.99691009521484,
      "activations/layer7_attention_weight_min": -85.8407211303711,
      "activations/layer8_attention_weight_max": 40.28449249267578,
      "activations/layer8_attention_weight_min": -41.89137268066406,
      "activations/layer9_attention_weight_max": 37.0476188659668,
      "activations/layer9_attention_weight_min": -36.259979248046875,
      "epoch": 6.02,
      "learning_rate": 0.00011232840909090908,
      "loss": 2.8404,
      "step": 103550
    },
    {
      "activations/layer0_attention_weight_max": 16.09004020690918,
      "activations/layer0_attention_weight_min": -10.687765121459961,
      "activations/layer10_attention_weight_max": 31.203725814819336,
      "activations/layer10_attention_weight_min": -31.267576217651367,
      "activations/layer11_attention_weight_max": 33.973941802978516,
      "activations/layer11_attention_weight_min": -28.66792106628418,
      "activations/layer12_attention_weight_max": 43.40131759643555,
      "activations/layer12_attention_weight_min": -28.017520904541016,
      "activations/layer13_attention_weight_max": 45.21861267089844,
      "activations/layer13_attention_weight_min": -31.891071319580078,
      "activations/layer14_attention_weight_max": 66.46401977539062,
      "activations/layer14_attention_weight_min": -46.85877990722656,
      "activations/layer15_attention_weight_max": 37.40901184082031,
      "activations/layer15_attention_weight_min": -28.18621063232422,
      "activations/layer16_attention_weight_max": 40.00668716430664,
      "activations/layer16_attention_weight_min": -28.349864959716797,
      "activations/layer17_attention_weight_max": 38.60245132446289,
      "activations/layer17_attention_weight_min": -24.41293716430664,
      "activations/layer18_attention_weight_max": 31.33456039428711,
      "activations/layer18_attention_weight_min": -20.71100425720215,
      "activations/layer19_attention_weight_max": 37.43394088745117,
      "activations/layer19_attention_weight_min": -29.399625778198242,
      "activations/layer1_attention_weight_max": 17.038692474365234,
      "activations/layer1_attention_weight_min": -15.111190795898438,
      "activations/layer20_attention_weight_max": 31.062021255493164,
      "activations/layer20_attention_weight_min": -23.438234329223633,
      "activations/layer21_attention_weight_max": 33.76282501220703,
      "activations/layer21_attention_weight_min": -22.11665916442871,
      "activations/layer22_attention_weight_max": 50.88973617553711,
      "activations/layer22_attention_weight_min": -30.67517852783203,
      "activations/layer23_attention_weight_max": 35.57646179199219,
      "activations/layer23_attention_weight_min": -23.715070724487305,
      "activations/layer2_attention_weight_max": 33.64990997314453,
      "activations/layer2_attention_weight_min": -32.72580337524414,
      "activations/layer3_attention_weight_max": 94.79795837402344,
      "activations/layer3_attention_weight_min": -96.27603912353516,
      "activations/layer4_attention_weight_max": 128.76332092285156,
      "activations/layer4_attention_weight_min": -128.55520629882812,
      "activations/layer5_attention_weight_max": 73.23994445800781,
      "activations/layer5_attention_weight_min": -71.07443237304688,
      "activations/layer6_attention_weight_max": 52.94430160522461,
      "activations/layer6_attention_weight_min": -54.062477111816406,
      "activations/layer7_attention_weight_max": 78.12849426269531,
      "activations/layer7_attention_weight_min": -85.75694274902344,
      "activations/layer8_attention_weight_max": 39.13187789916992,
      "activations/layer8_attention_weight_min": -40.63050079345703,
      "activations/layer9_attention_weight_max": 32.98558044433594,
      "activations/layer9_attention_weight_min": -39.03564453125,
      "epoch": 6.02,
      "learning_rate": 0.00011230946969696968,
      "loss": 2.8209,
      "step": 103600
    },
    {
      "activations/layer0_attention_weight_max": 16.972450256347656,
      "activations/layer0_attention_weight_min": -11.895960807800293,
      "activations/layer10_attention_weight_max": 32.61279296875,
      "activations/layer10_attention_weight_min": -32.77162551879883,
      "activations/layer11_attention_weight_max": 32.457252502441406,
      "activations/layer11_attention_weight_min": -31.168899536132812,
      "activations/layer12_attention_weight_max": 51.10554504394531,
      "activations/layer12_attention_weight_min": -29.828975677490234,
      "activations/layer13_attention_weight_max": 46.3674430847168,
      "activations/layer13_attention_weight_min": -33.36935043334961,
      "activations/layer14_attention_weight_max": 59.12183380126953,
      "activations/layer14_attention_weight_min": -43.26035690307617,
      "activations/layer15_attention_weight_max": 36.429832458496094,
      "activations/layer15_attention_weight_min": -28.996068954467773,
      "activations/layer16_attention_weight_max": 34.52988815307617,
      "activations/layer16_attention_weight_min": -26.85382080078125,
      "activations/layer17_attention_weight_max": 36.741294860839844,
      "activations/layer17_attention_weight_min": -24.16263771057129,
      "activations/layer18_attention_weight_max": 34.04436492919922,
      "activations/layer18_attention_weight_min": -20.900941848754883,
      "activations/layer19_attention_weight_max": 43.25498962402344,
      "activations/layer19_attention_weight_min": -28.792444229125977,
      "activations/layer1_attention_weight_max": 17.51171112060547,
      "activations/layer1_attention_weight_min": -14.43919849395752,
      "activations/layer20_attention_weight_max": 32.25987243652344,
      "activations/layer20_attention_weight_min": -24.408958435058594,
      "activations/layer21_attention_weight_max": 31.14493751525879,
      "activations/layer21_attention_weight_min": -21.692062377929688,
      "activations/layer22_attention_weight_max": 58.632080078125,
      "activations/layer22_attention_weight_min": -31.95500946044922,
      "activations/layer23_attention_weight_max": 35.87565612792969,
      "activations/layer23_attention_weight_min": -22.081459045410156,
      "activations/layer2_attention_weight_max": 30.981266021728516,
      "activations/layer2_attention_weight_min": -31.256088256835938,
      "activations/layer3_attention_weight_max": 91.80159759521484,
      "activations/layer3_attention_weight_min": -93.19567108154297,
      "activations/layer4_attention_weight_max": 128.2349853515625,
      "activations/layer4_attention_weight_min": -125.69612121582031,
      "activations/layer5_attention_weight_max": 75.56719970703125,
      "activations/layer5_attention_weight_min": -72.61416625976562,
      "activations/layer6_attention_weight_max": 58.683143615722656,
      "activations/layer6_attention_weight_min": -55.48872756958008,
      "activations/layer7_attention_weight_max": 88.86680603027344,
      "activations/layer7_attention_weight_min": -86.83419799804688,
      "activations/layer8_attention_weight_max": 39.9108772277832,
      "activations/layer8_attention_weight_min": -41.46632766723633,
      "activations/layer9_attention_weight_max": 35.254695892333984,
      "activations/layer9_attention_weight_min": -34.82308578491211,
      "epoch": 6.02,
      "learning_rate": 0.00011229053030303029,
      "loss": 2.8193,
      "step": 103650
    },
    {
      "activations/layer0_attention_weight_max": 15.773594856262207,
      "activations/layer0_attention_weight_min": -12.364102363586426,
      "activations/layer10_attention_weight_max": 34.47052001953125,
      "activations/layer10_attention_weight_min": -31.25057601928711,
      "activations/layer11_attention_weight_max": 32.098960876464844,
      "activations/layer11_attention_weight_min": -29.187284469604492,
      "activations/layer12_attention_weight_max": 24.84050941467285,
      "activations/layer12_attention_weight_min": -28.26291275024414,
      "activations/layer13_attention_weight_max": 48.49271011352539,
      "activations/layer13_attention_weight_min": -32.82197570800781,
      "activations/layer14_attention_weight_max": 57.202392578125,
      "activations/layer14_attention_weight_min": -45.163543701171875,
      "activations/layer15_attention_weight_max": 38.783756256103516,
      "activations/layer15_attention_weight_min": -31.43495750427246,
      "activations/layer16_attention_weight_max": 35.107582092285156,
      "activations/layer16_attention_weight_min": -26.627206802368164,
      "activations/layer17_attention_weight_max": 35.376319885253906,
      "activations/layer17_attention_weight_min": -24.100522994995117,
      "activations/layer18_attention_weight_max": 29.978500366210938,
      "activations/layer18_attention_weight_min": -22.915233612060547,
      "activations/layer19_attention_weight_max": 35.6932258605957,
      "activations/layer19_attention_weight_min": -28.907669067382812,
      "activations/layer1_attention_weight_max": 17.415695190429688,
      "activations/layer1_attention_weight_min": -13.651805877685547,
      "activations/layer20_attention_weight_max": 29.562419891357422,
      "activations/layer20_attention_weight_min": -23.21526336669922,
      "activations/layer21_attention_weight_max": 30.395614624023438,
      "activations/layer21_attention_weight_min": -21.49503517150879,
      "activations/layer22_attention_weight_max": 46.428409576416016,
      "activations/layer22_attention_weight_min": -26.903982162475586,
      "activations/layer23_attention_weight_max": 32.027069091796875,
      "activations/layer23_attention_weight_min": -21.486927032470703,
      "activations/layer2_attention_weight_max": 32.73638916015625,
      "activations/layer2_attention_weight_min": -31.902463912963867,
      "activations/layer3_attention_weight_max": 93.21112060546875,
      "activations/layer3_attention_weight_min": -95.69403839111328,
      "activations/layer4_attention_weight_max": 133.47262573242188,
      "activations/layer4_attention_weight_min": -126.30899810791016,
      "activations/layer5_attention_weight_max": 73.16729736328125,
      "activations/layer5_attention_weight_min": -72.70845031738281,
      "activations/layer6_attention_weight_max": 58.094390869140625,
      "activations/layer6_attention_weight_min": -56.06415939331055,
      "activations/layer7_attention_weight_max": 91.21910858154297,
      "activations/layer7_attention_weight_min": -87.39120483398438,
      "activations/layer8_attention_weight_max": 38.91665267944336,
      "activations/layer8_attention_weight_min": -41.115196228027344,
      "activations/layer9_attention_weight_max": 36.19417953491211,
      "activations/layer9_attention_weight_min": -35.179595947265625,
      "epoch": 6.03,
      "learning_rate": 0.0001122715909090909,
      "loss": 2.8258,
      "step": 103700
    },
    {
      "activations/layer0_attention_weight_max": 16.909000396728516,
      "activations/layer0_attention_weight_min": -12.263594627380371,
      "activations/layer10_attention_weight_max": 32.836830139160156,
      "activations/layer10_attention_weight_min": -29.788145065307617,
      "activations/layer11_attention_weight_max": 31.10720443725586,
      "activations/layer11_attention_weight_min": -27.05933952331543,
      "activations/layer12_attention_weight_max": 50.04497528076172,
      "activations/layer12_attention_weight_min": -26.887374877929688,
      "activations/layer13_attention_weight_max": 44.67764663696289,
      "activations/layer13_attention_weight_min": -33.80503463745117,
      "activations/layer14_attention_weight_max": 56.600189208984375,
      "activations/layer14_attention_weight_min": -42.56147003173828,
      "activations/layer15_attention_weight_max": 36.29225540161133,
      "activations/layer15_attention_weight_min": -27.3314266204834,
      "activations/layer16_attention_weight_max": 36.690616607666016,
      "activations/layer16_attention_weight_min": -26.800457000732422,
      "activations/layer17_attention_weight_max": 35.99798583984375,
      "activations/layer17_attention_weight_min": -24.59018898010254,
      "activations/layer18_attention_weight_max": 30.398983001708984,
      "activations/layer18_attention_weight_min": -20.05344009399414,
      "activations/layer19_attention_weight_max": 40.87734603881836,
      "activations/layer19_attention_weight_min": -28.857481002807617,
      "activations/layer1_attention_weight_max": 17.096027374267578,
      "activations/layer1_attention_weight_min": -16.03716468811035,
      "activations/layer20_attention_weight_max": 33.16903305053711,
      "activations/layer20_attention_weight_min": -21.737552642822266,
      "activations/layer21_attention_weight_max": 33.79584884643555,
      "activations/layer21_attention_weight_min": -20.933259963989258,
      "activations/layer22_attention_weight_max": 52.54677963256836,
      "activations/layer22_attention_weight_min": -27.508699417114258,
      "activations/layer23_attention_weight_max": 35.648773193359375,
      "activations/layer23_attention_weight_min": -21.310516357421875,
      "activations/layer2_attention_weight_max": 30.799617767333984,
      "activations/layer2_attention_weight_min": -29.805831909179688,
      "activations/layer3_attention_weight_max": 85.13521575927734,
      "activations/layer3_attention_weight_min": -86.89655303955078,
      "activations/layer4_attention_weight_max": 119.63154602050781,
      "activations/layer4_attention_weight_min": -113.98906707763672,
      "activations/layer5_attention_weight_max": 71.23014068603516,
      "activations/layer5_attention_weight_min": -67.24837493896484,
      "activations/layer6_attention_weight_max": 50.636634826660156,
      "activations/layer6_attention_weight_min": -53.087440490722656,
      "activations/layer7_attention_weight_max": 84.18909454345703,
      "activations/layer7_attention_weight_min": -83.42628479003906,
      "activations/layer8_attention_weight_max": 37.83277130126953,
      "activations/layer8_attention_weight_min": -39.04587173461914,
      "activations/layer9_attention_weight_max": 37.79711151123047,
      "activations/layer9_attention_weight_min": -32.22130584716797,
      "epoch": 6.03,
      "learning_rate": 0.0001122526515151515,
      "loss": 2.8122,
      "step": 103750
    },
    {
      "activations/layer0_attention_weight_max": 15.322137832641602,
      "activations/layer0_attention_weight_min": -12.526578903198242,
      "activations/layer10_attention_weight_max": 33.75058364868164,
      "activations/layer10_attention_weight_min": -31.810665130615234,
      "activations/layer11_attention_weight_max": 31.43940544128418,
      "activations/layer11_attention_weight_min": -27.043170928955078,
      "activations/layer12_attention_weight_max": 26.82897186279297,
      "activations/layer12_attention_weight_min": -30.608718872070312,
      "activations/layer13_attention_weight_max": 45.717201232910156,
      "activations/layer13_attention_weight_min": -34.551048278808594,
      "activations/layer14_attention_weight_max": 63.78520202636719,
      "activations/layer14_attention_weight_min": -48.996768951416016,
      "activations/layer15_attention_weight_max": 40.757198333740234,
      "activations/layer15_attention_weight_min": -29.99843978881836,
      "activations/layer16_attention_weight_max": 38.03266143798828,
      "activations/layer16_attention_weight_min": -27.25733184814453,
      "activations/layer17_attention_weight_max": 39.45587158203125,
      "activations/layer17_attention_weight_min": -26.070138931274414,
      "activations/layer18_attention_weight_max": 34.23626708984375,
      "activations/layer18_attention_weight_min": -20.746562957763672,
      "activations/layer19_attention_weight_max": 48.81196975708008,
      "activations/layer19_attention_weight_min": -30.872098922729492,
      "activations/layer1_attention_weight_max": 16.909038543701172,
      "activations/layer1_attention_weight_min": -14.748568534851074,
      "activations/layer20_attention_weight_max": 37.812381744384766,
      "activations/layer20_attention_weight_min": -24.348529815673828,
      "activations/layer21_attention_weight_max": 40.24126052856445,
      "activations/layer21_attention_weight_min": -22.22031593322754,
      "activations/layer22_attention_weight_max": 60.21699905395508,
      "activations/layer22_attention_weight_min": -28.377227783203125,
      "activations/layer23_attention_weight_max": 37.481895446777344,
      "activations/layer23_attention_weight_min": -21.164527893066406,
      "activations/layer2_attention_weight_max": 32.94841003417969,
      "activations/layer2_attention_weight_min": -32.31187438964844,
      "activations/layer3_attention_weight_max": 94.65965270996094,
      "activations/layer3_attention_weight_min": -91.28280639648438,
      "activations/layer4_attention_weight_max": 130.71446228027344,
      "activations/layer4_attention_weight_min": -126.3631362915039,
      "activations/layer5_attention_weight_max": 73.84346771240234,
      "activations/layer5_attention_weight_min": -74.91981506347656,
      "activations/layer6_attention_weight_max": 53.226558685302734,
      "activations/layer6_attention_weight_min": -57.98732376098633,
      "activations/layer7_attention_weight_max": 87.21529388427734,
      "activations/layer7_attention_weight_min": -84.84101104736328,
      "activations/layer8_attention_weight_max": 40.466705322265625,
      "activations/layer8_attention_weight_min": -41.19328689575195,
      "activations/layer9_attention_weight_max": 36.269248962402344,
      "activations/layer9_attention_weight_min": -36.17338943481445,
      "epoch": 6.03,
      "learning_rate": 0.0001122337121212121,
      "loss": 2.8193,
      "step": 103800
    },
    {
      "activations/layer0_attention_weight_max": 15.433135032653809,
      "activations/layer0_attention_weight_min": -12.107701301574707,
      "activations/layer10_attention_weight_max": 32.88176727294922,
      "activations/layer10_attention_weight_min": -34.3480339050293,
      "activations/layer11_attention_weight_max": 32.93679428100586,
      "activations/layer11_attention_weight_min": -28.351158142089844,
      "activations/layer12_attention_weight_max": 28.370590209960938,
      "activations/layer12_attention_weight_min": -27.523496627807617,
      "activations/layer13_attention_weight_max": 47.13166809082031,
      "activations/layer13_attention_weight_min": -31.353500366210938,
      "activations/layer14_attention_weight_max": 66.46607208251953,
      "activations/layer14_attention_weight_min": -41.93338394165039,
      "activations/layer15_attention_weight_max": 40.72904968261719,
      "activations/layer15_attention_weight_min": -29.41409683227539,
      "activations/layer16_attention_weight_max": 39.44558334350586,
      "activations/layer16_attention_weight_min": -28.35297966003418,
      "activations/layer17_attention_weight_max": 41.40055847167969,
      "activations/layer17_attention_weight_min": -24.847558975219727,
      "activations/layer18_attention_weight_max": 32.06089782714844,
      "activations/layer18_attention_weight_min": -21.46881866455078,
      "activations/layer19_attention_weight_max": 44.370662689208984,
      "activations/layer19_attention_weight_min": -29.811548233032227,
      "activations/layer1_attention_weight_max": 17.87373161315918,
      "activations/layer1_attention_weight_min": -14.452835083007812,
      "activations/layer20_attention_weight_max": 33.42954635620117,
      "activations/layer20_attention_weight_min": -23.94915008544922,
      "activations/layer21_attention_weight_max": 30.65169906616211,
      "activations/layer21_attention_weight_min": -21.042890548706055,
      "activations/layer22_attention_weight_max": 51.75864791870117,
      "activations/layer22_attention_weight_min": -28.18440055847168,
      "activations/layer23_attention_weight_max": 39.04524612426758,
      "activations/layer23_attention_weight_min": -21.71053123474121,
      "activations/layer2_attention_weight_max": 33.391021728515625,
      "activations/layer2_attention_weight_min": -32.50279998779297,
      "activations/layer3_attention_weight_max": 91.73452758789062,
      "activations/layer3_attention_weight_min": -95.28337097167969,
      "activations/layer4_attention_weight_max": 129.2979736328125,
      "activations/layer4_attention_weight_min": -131.50523376464844,
      "activations/layer5_attention_weight_max": 71.0703125,
      "activations/layer5_attention_weight_min": -74.54541015625,
      "activations/layer6_attention_weight_max": 53.524864196777344,
      "activations/layer6_attention_weight_min": -53.55155563354492,
      "activations/layer7_attention_weight_max": 86.70116424560547,
      "activations/layer7_attention_weight_min": -85.56929016113281,
      "activations/layer8_attention_weight_max": 42.23555374145508,
      "activations/layer8_attention_weight_min": -42.22663116455078,
      "activations/layer9_attention_weight_max": 36.91075897216797,
      "activations/layer9_attention_weight_min": -34.762916564941406,
      "epoch": 6.03,
      "learning_rate": 0.00011221477272727272,
      "loss": 2.8219,
      "step": 103850
    },
    {
      "activations/layer0_attention_weight_max": 16.295242309570312,
      "activations/layer0_attention_weight_min": -12.600197792053223,
      "activations/layer10_attention_weight_max": 33.98713302612305,
      "activations/layer10_attention_weight_min": -31.88833236694336,
      "activations/layer11_attention_weight_max": 33.396671295166016,
      "activations/layer11_attention_weight_min": -28.43277359008789,
      "activations/layer12_attention_weight_max": 41.59918212890625,
      "activations/layer12_attention_weight_min": -25.737899780273438,
      "activations/layer13_attention_weight_max": 49.3510856628418,
      "activations/layer13_attention_weight_min": -32.245948791503906,
      "activations/layer14_attention_weight_max": 49.8691520690918,
      "activations/layer14_attention_weight_min": -38.098838806152344,
      "activations/layer15_attention_weight_max": 41.264076232910156,
      "activations/layer15_attention_weight_min": -29.584444046020508,
      "activations/layer16_attention_weight_max": 35.51895523071289,
      "activations/layer16_attention_weight_min": -26.901552200317383,
      "activations/layer17_attention_weight_max": 37.18602752685547,
      "activations/layer17_attention_weight_min": -23.631732940673828,
      "activations/layer18_attention_weight_max": 30.852828979492188,
      "activations/layer18_attention_weight_min": -19.951717376708984,
      "activations/layer19_attention_weight_max": 36.73545455932617,
      "activations/layer19_attention_weight_min": -28.37843132019043,
      "activations/layer1_attention_weight_max": 17.342988967895508,
      "activations/layer1_attention_weight_min": -16.281814575195312,
      "activations/layer20_attention_weight_max": 33.18117904663086,
      "activations/layer20_attention_weight_min": -23.97957992553711,
      "activations/layer21_attention_weight_max": 30.334213256835938,
      "activations/layer21_attention_weight_min": -21.263172149658203,
      "activations/layer22_attention_weight_max": 47.7317008972168,
      "activations/layer22_attention_weight_min": -29.234088897705078,
      "activations/layer23_attention_weight_max": 34.156978607177734,
      "activations/layer23_attention_weight_min": -21.602659225463867,
      "activations/layer2_attention_weight_max": 33.19227981567383,
      "activations/layer2_attention_weight_min": -31.624595642089844,
      "activations/layer3_attention_weight_max": 93.81108856201172,
      "activations/layer3_attention_weight_min": -97.45066833496094,
      "activations/layer4_attention_weight_max": 123.21123504638672,
      "activations/layer4_attention_weight_min": -123.46605682373047,
      "activations/layer5_attention_weight_max": 73.38894653320312,
      "activations/layer5_attention_weight_min": -69.423828125,
      "activations/layer6_attention_weight_max": 55.345298767089844,
      "activations/layer6_attention_weight_min": -56.01347351074219,
      "activations/layer7_attention_weight_max": 82.38648223876953,
      "activations/layer7_attention_weight_min": -82.9037857055664,
      "activations/layer8_attention_weight_max": 40.27702713012695,
      "activations/layer8_attention_weight_min": -41.3824348449707,
      "activations/layer9_attention_weight_max": 36.605377197265625,
      "activations/layer9_attention_weight_min": -35.11845016479492,
      "epoch": 6.04,
      "learning_rate": 0.00011219583333333333,
      "loss": 2.8354,
      "step": 103900
    },
    {
      "activations/layer0_attention_weight_max": 16.703027725219727,
      "activations/layer0_attention_weight_min": -12.530092239379883,
      "activations/layer10_attention_weight_max": 35.033607482910156,
      "activations/layer10_attention_weight_min": -32.03960418701172,
      "activations/layer11_attention_weight_max": 33.19443130493164,
      "activations/layer11_attention_weight_min": -29.830108642578125,
      "activations/layer12_attention_weight_max": 30.549392700195312,
      "activations/layer12_attention_weight_min": -27.782352447509766,
      "activations/layer13_attention_weight_max": 54.235267639160156,
      "activations/layer13_attention_weight_min": -38.60291290283203,
      "activations/layer14_attention_weight_max": 65.0162582397461,
      "activations/layer14_attention_weight_min": -44.87089920043945,
      "activations/layer15_attention_weight_max": 42.2113151550293,
      "activations/layer15_attention_weight_min": -28.858091354370117,
      "activations/layer16_attention_weight_max": 38.69228744506836,
      "activations/layer16_attention_weight_min": -27.193065643310547,
      "activations/layer17_attention_weight_max": 45.27195358276367,
      "activations/layer17_attention_weight_min": -24.48594093322754,
      "activations/layer18_attention_weight_max": 35.120277404785156,
      "activations/layer18_attention_weight_min": -20.584028244018555,
      "activations/layer19_attention_weight_max": 48.457088470458984,
      "activations/layer19_attention_weight_min": -30.314111709594727,
      "activations/layer1_attention_weight_max": 16.340206146240234,
      "activations/layer1_attention_weight_min": -15.034981727600098,
      "activations/layer20_attention_weight_max": 33.32155227661133,
      "activations/layer20_attention_weight_min": -24.045778274536133,
      "activations/layer21_attention_weight_max": 36.19588088989258,
      "activations/layer21_attention_weight_min": -21.33437156677246,
      "activations/layer22_attention_weight_max": 55.11762237548828,
      "activations/layer22_attention_weight_min": -28.876556396484375,
      "activations/layer23_attention_weight_max": 37.176639556884766,
      "activations/layer23_attention_weight_min": -22.43655014038086,
      "activations/layer2_attention_weight_max": 32.87058639526367,
      "activations/layer2_attention_weight_min": -32.27709197998047,
      "activations/layer3_attention_weight_max": 94.98450469970703,
      "activations/layer3_attention_weight_min": -97.53926086425781,
      "activations/layer4_attention_weight_max": 132.1658172607422,
      "activations/layer4_attention_weight_min": -127.46905517578125,
      "activations/layer5_attention_weight_max": 77.75611114501953,
      "activations/layer5_attention_weight_min": -72.57347869873047,
      "activations/layer6_attention_weight_max": 58.63177490234375,
      "activations/layer6_attention_weight_min": -59.82794189453125,
      "activations/layer7_attention_weight_max": 86.14541625976562,
      "activations/layer7_attention_weight_min": -86.47140502929688,
      "activations/layer8_attention_weight_max": 44.293113708496094,
      "activations/layer8_attention_weight_min": -40.55255126953125,
      "activations/layer9_attention_weight_max": 38.13197708129883,
      "activations/layer9_attention_weight_min": -35.453704833984375,
      "epoch": 6.04,
      "learning_rate": 0.00011217689393939392,
      "loss": 2.8291,
      "step": 103950
    },
    {
      "activations/layer0_attention_weight_max": 16.358606338500977,
      "activations/layer0_attention_weight_min": -11.767803192138672,
      "activations/layer10_attention_weight_max": 35.3449592590332,
      "activations/layer10_attention_weight_min": -32.67593002319336,
      "activations/layer11_attention_weight_max": 31.4045467376709,
      "activations/layer11_attention_weight_min": -28.527767181396484,
      "activations/layer12_attention_weight_max": 21.896684646606445,
      "activations/layer12_attention_weight_min": -27.249677658081055,
      "activations/layer13_attention_weight_max": 40.30887985229492,
      "activations/layer13_attention_weight_min": -33.26392364501953,
      "activations/layer14_attention_weight_max": 58.11199188232422,
      "activations/layer14_attention_weight_min": -43.24723434448242,
      "activations/layer15_attention_weight_max": 36.80119705200195,
      "activations/layer15_attention_weight_min": -30.035526275634766,
      "activations/layer16_attention_weight_max": 38.6934700012207,
      "activations/layer16_attention_weight_min": -26.984411239624023,
      "activations/layer17_attention_weight_max": 34.94260787963867,
      "activations/layer17_attention_weight_min": -25.08803367614746,
      "activations/layer18_attention_weight_max": 38.22429656982422,
      "activations/layer18_attention_weight_min": -22.283952713012695,
      "activations/layer19_attention_weight_max": 41.5515251159668,
      "activations/layer19_attention_weight_min": -32.40959167480469,
      "activations/layer1_attention_weight_max": 17.451948165893555,
      "activations/layer1_attention_weight_min": -14.20361042022705,
      "activations/layer20_attention_weight_max": 34.46586990356445,
      "activations/layer20_attention_weight_min": -26.665176391601562,
      "activations/layer21_attention_weight_max": 36.92710494995117,
      "activations/layer21_attention_weight_min": -23.013484954833984,
      "activations/layer22_attention_weight_max": 57.98225402832031,
      "activations/layer22_attention_weight_min": -31.8692626953125,
      "activations/layer23_attention_weight_max": 39.98619842529297,
      "activations/layer23_attention_weight_min": -24.09565544128418,
      "activations/layer2_attention_weight_max": 32.25596618652344,
      "activations/layer2_attention_weight_min": -30.90753173828125,
      "activations/layer3_attention_weight_max": 91.18628692626953,
      "activations/layer3_attention_weight_min": -92.80105590820312,
      "activations/layer4_attention_weight_max": 132.48681640625,
      "activations/layer4_attention_weight_min": -127.59869384765625,
      "activations/layer5_attention_weight_max": 71.05804443359375,
      "activations/layer5_attention_weight_min": -70.80442810058594,
      "activations/layer6_attention_weight_max": 54.44544982910156,
      "activations/layer6_attention_weight_min": -54.67599868774414,
      "activations/layer7_attention_weight_max": 87.46420288085938,
      "activations/layer7_attention_weight_min": -84.99906921386719,
      "activations/layer8_attention_weight_max": 41.52096939086914,
      "activations/layer8_attention_weight_min": -44.151947021484375,
      "activations/layer9_attention_weight_max": 36.434932708740234,
      "activations/layer9_attention_weight_min": -35.38827896118164,
      "epoch": 6.04,
      "learning_rate": 0.00011215795454545454,
      "loss": 2.8205,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.5323,
      "eval_samples_per_second": 503.263,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.5323,
      "eval_openwebtext_samples_per_second": 503.263,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_wikitext_loss": 3.021484375,
      "eval_wikitext_ppl": 20.521731031400577,
      "eval_wikitext_runtime": 1.9931,
      "eval_wikitext_samples_per_second": 228.792,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_lambada_loss": 2.703125,
      "eval_lambada_ppl": 14.926303617194517,
      "eval_lambada_runtime": 9.5586,
      "eval_lambada_samples_per_second": 509.383,
      "step": 104000
    },
    {
      "activations/layer0_attention_weight_max": 16.14004898071289,
      "activations/layer0_attention_weight_min": -12.052677154541016,
      "activations/layer10_attention_weight_max": 30.001718521118164,
      "activations/layer10_attention_weight_min": -30.705135345458984,
      "activations/layer11_attention_weight_max": 31.498144149780273,
      "activations/layer11_attention_weight_min": -26.58608055114746,
      "activations/layer12_attention_weight_max": 27.509435653686523,
      "activations/layer12_attention_weight_min": -25.35198402404785,
      "activations/layer13_attention_weight_max": 36.067203521728516,
      "activations/layer13_attention_weight_min": -30.067340850830078,
      "activations/layer14_attention_weight_max": 44.633056640625,
      "activations/layer14_attention_weight_min": -37.99026107788086,
      "activations/layer15_attention_weight_max": 33.72651672363281,
      "activations/layer15_attention_weight_min": -28.69171905517578,
      "activations/layer16_attention_weight_max": 29.857711791992188,
      "activations/layer16_attention_weight_min": -27.7232608795166,
      "activations/layer17_attention_weight_max": 36.62993621826172,
      "activations/layer17_attention_weight_min": -26.27459144592285,
      "activations/layer18_attention_weight_max": 31.643407821655273,
      "activations/layer18_attention_weight_min": -20.791940689086914,
      "activations/layer19_attention_weight_max": 35.237579345703125,
      "activations/layer19_attention_weight_min": -30.861841201782227,
      "activations/layer1_attention_weight_max": 16.760595321655273,
      "activations/layer1_attention_weight_min": -14.340686798095703,
      "activations/layer20_attention_weight_max": 31.305747985839844,
      "activations/layer20_attention_weight_min": -26.165719985961914,
      "activations/layer21_attention_weight_max": 28.442485809326172,
      "activations/layer21_attention_weight_min": -22.79364585876465,
      "activations/layer22_attention_weight_max": 49.96039581298828,
      "activations/layer22_attention_weight_min": -29.06308937072754,
      "activations/layer23_attention_weight_max": 31.99306869506836,
      "activations/layer23_attention_weight_min": -21.219669342041016,
      "activations/layer2_attention_weight_max": 31.4366455078125,
      "activations/layer2_attention_weight_min": -29.326114654541016,
      "activations/layer3_attention_weight_max": 90.0597915649414,
      "activations/layer3_attention_weight_min": -91.11833953857422,
      "activations/layer4_attention_weight_max": 120.65372467041016,
      "activations/layer4_attention_weight_min": -118.8167724609375,
      "activations/layer5_attention_weight_max": 70.05885314941406,
      "activations/layer5_attention_weight_min": -69.07128143310547,
      "activations/layer6_attention_weight_max": 51.89162063598633,
      "activations/layer6_attention_weight_min": -53.097660064697266,
      "activations/layer7_attention_weight_max": 81.12804412841797,
      "activations/layer7_attention_weight_min": -83.5525131225586,
      "activations/layer8_attention_weight_max": 37.486915588378906,
      "activations/layer8_attention_weight_min": -38.123435974121094,
      "activations/layer9_attention_weight_max": 34.166873931884766,
      "activations/layer9_attention_weight_min": -33.951595306396484,
      "epoch": 6.05,
      "learning_rate": 0.00011213901515151515,
      "loss": 2.8327,
      "step": 104050
    },
    {
      "activations/layer0_attention_weight_max": 16.64852523803711,
      "activations/layer0_attention_weight_min": -11.479275703430176,
      "activations/layer10_attention_weight_max": 35.30266189575195,
      "activations/layer10_attention_weight_min": -31.981962203979492,
      "activations/layer11_attention_weight_max": 33.76420974731445,
      "activations/layer11_attention_weight_min": -27.84329605102539,
      "activations/layer12_attention_weight_max": 23.583656311035156,
      "activations/layer12_attention_weight_min": -25.611392974853516,
      "activations/layer13_attention_weight_max": 44.44458770751953,
      "activations/layer13_attention_weight_min": -32.51335906982422,
      "activations/layer14_attention_weight_max": 56.83418273925781,
      "activations/layer14_attention_weight_min": -39.04191970825195,
      "activations/layer15_attention_weight_max": 40.452545166015625,
      "activations/layer15_attention_weight_min": -28.294261932373047,
      "activations/layer16_attention_weight_max": 36.63130187988281,
      "activations/layer16_attention_weight_min": -27.16623306274414,
      "activations/layer17_attention_weight_max": 38.55143356323242,
      "activations/layer17_attention_weight_min": -24.478778839111328,
      "activations/layer18_attention_weight_max": 36.258522033691406,
      "activations/layer18_attention_weight_min": -21.08055877685547,
      "activations/layer19_attention_weight_max": 40.3747673034668,
      "activations/layer19_attention_weight_min": -31.14822769165039,
      "activations/layer1_attention_weight_max": 17.79694366455078,
      "activations/layer1_attention_weight_min": -13.754788398742676,
      "activations/layer20_attention_weight_max": 34.73106384277344,
      "activations/layer20_attention_weight_min": -24.994718551635742,
      "activations/layer21_attention_weight_max": 33.39741897583008,
      "activations/layer21_attention_weight_min": -24.218486785888672,
      "activations/layer22_attention_weight_max": 55.66277313232422,
      "activations/layer22_attention_weight_min": -30.79567527770996,
      "activations/layer23_attention_weight_max": 38.55363464355469,
      "activations/layer23_attention_weight_min": -22.667720794677734,
      "activations/layer2_attention_weight_max": 32.0274658203125,
      "activations/layer2_attention_weight_min": -31.354015350341797,
      "activations/layer3_attention_weight_max": 96.09051513671875,
      "activations/layer3_attention_weight_min": -91.08263397216797,
      "activations/layer4_attention_weight_max": 136.7781219482422,
      "activations/layer4_attention_weight_min": -125.72672271728516,
      "activations/layer5_attention_weight_max": 73.06351470947266,
      "activations/layer5_attention_weight_min": -68.92691040039062,
      "activations/layer6_attention_weight_max": 52.33191680908203,
      "activations/layer6_attention_weight_min": -53.00937271118164,
      "activations/layer7_attention_weight_max": 86.17842864990234,
      "activations/layer7_attention_weight_min": -86.66104125976562,
      "activations/layer8_attention_weight_max": 40.125701904296875,
      "activations/layer8_attention_weight_min": -39.25641632080078,
      "activations/layer9_attention_weight_max": 37.07770919799805,
      "activations/layer9_attention_weight_min": -34.76739501953125,
      "epoch": 6.05,
      "learning_rate": 0.00011212007575757574,
      "loss": 2.8445,
      "step": 104100
    },
    {
      "activations/layer0_attention_weight_max": 15.828742980957031,
      "activations/layer0_attention_weight_min": -11.3464994430542,
      "activations/layer10_attention_weight_max": 34.55311584472656,
      "activations/layer10_attention_weight_min": -30.502593994140625,
      "activations/layer11_attention_weight_max": 34.0189323425293,
      "activations/layer11_attention_weight_min": -29.169157028198242,
      "activations/layer12_attention_weight_max": 28.510513305664062,
      "activations/layer12_attention_weight_min": -27.617408752441406,
      "activations/layer13_attention_weight_max": 42.10885238647461,
      "activations/layer13_attention_weight_min": -33.539695739746094,
      "activations/layer14_attention_weight_max": 55.658287048339844,
      "activations/layer14_attention_weight_min": -40.920841217041016,
      "activations/layer15_attention_weight_max": 41.261959075927734,
      "activations/layer15_attention_weight_min": -29.108156204223633,
      "activations/layer16_attention_weight_max": 33.50059127807617,
      "activations/layer16_attention_weight_min": -26.094850540161133,
      "activations/layer17_attention_weight_max": 35.26979446411133,
      "activations/layer17_attention_weight_min": -24.1240177154541,
      "activations/layer18_attention_weight_max": 29.741281509399414,
      "activations/layer18_attention_weight_min": -19.36666488647461,
      "activations/layer19_attention_weight_max": 35.209285736083984,
      "activations/layer19_attention_weight_min": -27.933765411376953,
      "activations/layer1_attention_weight_max": 17.391517639160156,
      "activations/layer1_attention_weight_min": -16.022390365600586,
      "activations/layer20_attention_weight_max": 28.924427032470703,
      "activations/layer20_attention_weight_min": -21.123497009277344,
      "activations/layer21_attention_weight_max": 27.174596786499023,
      "activations/layer21_attention_weight_min": -20.957185745239258,
      "activations/layer22_attention_weight_max": 44.52985382080078,
      "activations/layer22_attention_weight_min": -26.9733943939209,
      "activations/layer23_attention_weight_max": 33.11994171142578,
      "activations/layer23_attention_weight_min": -21.390560150146484,
      "activations/layer2_attention_weight_max": 33.6001091003418,
      "activations/layer2_attention_weight_min": -31.05681610107422,
      "activations/layer3_attention_weight_max": 91.48976135253906,
      "activations/layer3_attention_weight_min": -89.2927017211914,
      "activations/layer4_attention_weight_max": 126.48748016357422,
      "activations/layer4_attention_weight_min": -125.59667205810547,
      "activations/layer5_attention_weight_max": 73.02374267578125,
      "activations/layer5_attention_weight_min": -69.06840515136719,
      "activations/layer6_attention_weight_max": 56.66755676269531,
      "activations/layer6_attention_weight_min": -56.373573303222656,
      "activations/layer7_attention_weight_max": 89.1925048828125,
      "activations/layer7_attention_weight_min": -84.29242706298828,
      "activations/layer8_attention_weight_max": 42.325496673583984,
      "activations/layer8_attention_weight_min": -41.332557678222656,
      "activations/layer9_attention_weight_max": 38.359249114990234,
      "activations/layer9_attention_weight_min": -38.453975677490234,
      "epoch": 6.05,
      "learning_rate": 0.00011210113636363636,
      "loss": 2.8095,
      "step": 104150
    },
    {
      "activations/layer0_attention_weight_max": 16.369335174560547,
      "activations/layer0_attention_weight_min": -11.992440223693848,
      "activations/layer10_attention_weight_max": 32.95948028564453,
      "activations/layer10_attention_weight_min": -29.22799301147461,
      "activations/layer11_attention_weight_max": 31.663070678710938,
      "activations/layer11_attention_weight_min": -27.740842819213867,
      "activations/layer12_attention_weight_max": 27.057954788208008,
      "activations/layer12_attention_weight_min": -25.22315216064453,
      "activations/layer13_attention_weight_max": 39.89581298828125,
      "activations/layer13_attention_weight_min": -31.833309173583984,
      "activations/layer14_attention_weight_max": 57.99061965942383,
      "activations/layer14_attention_weight_min": -42.909000396728516,
      "activations/layer15_attention_weight_max": 38.60172653198242,
      "activations/layer15_attention_weight_min": -28.311065673828125,
      "activations/layer16_attention_weight_max": 38.587913513183594,
      "activations/layer16_attention_weight_min": -27.78084945678711,
      "activations/layer17_attention_weight_max": 34.337520599365234,
      "activations/layer17_attention_weight_min": -25.179096221923828,
      "activations/layer18_attention_weight_max": 33.50069808959961,
      "activations/layer18_attention_weight_min": -20.817686080932617,
      "activations/layer19_attention_weight_max": 42.4187126159668,
      "activations/layer19_attention_weight_min": -30.81385612487793,
      "activations/layer1_attention_weight_max": 17.360149383544922,
      "activations/layer1_attention_weight_min": -14.141725540161133,
      "activations/layer20_attention_weight_max": 31.93655014038086,
      "activations/layer20_attention_weight_min": -24.636577606201172,
      "activations/layer21_attention_weight_max": 31.871776580810547,
      "activations/layer21_attention_weight_min": -21.60576057434082,
      "activations/layer22_attention_weight_max": 51.89454650878906,
      "activations/layer22_attention_weight_min": -31.32815933227539,
      "activations/layer23_attention_weight_max": 36.79576873779297,
      "activations/layer23_attention_weight_min": -22.8115234375,
      "activations/layer2_attention_weight_max": 30.77872085571289,
      "activations/layer2_attention_weight_min": -30.3262939453125,
      "activations/layer3_attention_weight_max": 86.62537384033203,
      "activations/layer3_attention_weight_min": -91.7211685180664,
      "activations/layer4_attention_weight_max": 125.9725112915039,
      "activations/layer4_attention_weight_min": -124.668212890625,
      "activations/layer5_attention_weight_max": 73.80376434326172,
      "activations/layer5_attention_weight_min": -70.15213775634766,
      "activations/layer6_attention_weight_max": 51.80384826660156,
      "activations/layer6_attention_weight_min": -53.18653106689453,
      "activations/layer7_attention_weight_max": 82.49951171875,
      "activations/layer7_attention_weight_min": -82.55176544189453,
      "activations/layer8_attention_weight_max": 37.81837844848633,
      "activations/layer8_attention_weight_min": -39.22636795043945,
      "activations/layer9_attention_weight_max": 36.34611892700195,
      "activations/layer9_attention_weight_min": -33.30463409423828,
      "epoch": 6.05,
      "learning_rate": 0.00011208219696969696,
      "loss": 2.8156,
      "step": 104200
    },
    {
      "activations/layer0_attention_weight_max": 16.705577850341797,
      "activations/layer0_attention_weight_min": -11.168761253356934,
      "activations/layer10_attention_weight_max": 34.667724609375,
      "activations/layer10_attention_weight_min": -30.39754295349121,
      "activations/layer11_attention_weight_max": 31.534568786621094,
      "activations/layer11_attention_weight_min": -29.513526916503906,
      "activations/layer12_attention_weight_max": 24.305355072021484,
      "activations/layer12_attention_weight_min": -26.052894592285156,
      "activations/layer13_attention_weight_max": 36.52912902832031,
      "activations/layer13_attention_weight_min": -31.511781692504883,
      "activations/layer14_attention_weight_max": 52.04320526123047,
      "activations/layer14_attention_weight_min": -39.54733657836914,
      "activations/layer15_attention_weight_max": 37.465858459472656,
      "activations/layer15_attention_weight_min": -28.2861328125,
      "activations/layer16_attention_weight_max": 34.88629150390625,
      "activations/layer16_attention_weight_min": -26.473814010620117,
      "activations/layer17_attention_weight_max": 35.81344985961914,
      "activations/layer17_attention_weight_min": -25.393543243408203,
      "activations/layer18_attention_weight_max": 30.37295150756836,
      "activations/layer18_attention_weight_min": -24.319169998168945,
      "activations/layer19_attention_weight_max": 36.47189712524414,
      "activations/layer19_attention_weight_min": -29.821842193603516,
      "activations/layer1_attention_weight_max": 17.5732364654541,
      "activations/layer1_attention_weight_min": -15.143092155456543,
      "activations/layer20_attention_weight_max": 31.138572692871094,
      "activations/layer20_attention_weight_min": -24.035594940185547,
      "activations/layer21_attention_weight_max": 31.29287338256836,
      "activations/layer21_attention_weight_min": -20.16578483581543,
      "activations/layer22_attention_weight_max": 51.399845123291016,
      "activations/layer22_attention_weight_min": -29.292463302612305,
      "activations/layer23_attention_weight_max": 32.73223114013672,
      "activations/layer23_attention_weight_min": -26.159452438354492,
      "activations/layer2_attention_weight_max": 30.800939559936523,
      "activations/layer2_attention_weight_min": -29.220014572143555,
      "activations/layer3_attention_weight_max": 91.19198608398438,
      "activations/layer3_attention_weight_min": -87.29302978515625,
      "activations/layer4_attention_weight_max": 132.54920959472656,
      "activations/layer4_attention_weight_min": -116.53480529785156,
      "activations/layer5_attention_weight_max": 75.2777099609375,
      "activations/layer5_attention_weight_min": -66.72120666503906,
      "activations/layer6_attention_weight_max": 53.60222244262695,
      "activations/layer6_attention_weight_min": -55.118377685546875,
      "activations/layer7_attention_weight_max": 82.30912017822266,
      "activations/layer7_attention_weight_min": -81.85264587402344,
      "activations/layer8_attention_weight_max": 40.28564453125,
      "activations/layer8_attention_weight_min": -39.08534622192383,
      "activations/layer9_attention_weight_max": 35.422183990478516,
      "activations/layer9_attention_weight_min": -32.634254455566406,
      "epoch": 6.06,
      "learning_rate": 0.00011206325757575756,
      "loss": 2.8141,
      "step": 104250
    },
    {
      "activations/layer0_attention_weight_max": 15.722371101379395,
      "activations/layer0_attention_weight_min": -12.620945930480957,
      "activations/layer10_attention_weight_max": 36.74903869628906,
      "activations/layer10_attention_weight_min": -33.31173324584961,
      "activations/layer11_attention_weight_max": 35.713279724121094,
      "activations/layer11_attention_weight_min": -28.9271240234375,
      "activations/layer12_attention_weight_max": 35.8243408203125,
      "activations/layer12_attention_weight_min": -25.41370964050293,
      "activations/layer13_attention_weight_max": 44.19519805908203,
      "activations/layer13_attention_weight_min": -32.67512893676758,
      "activations/layer14_attention_weight_max": 64.34386444091797,
      "activations/layer14_attention_weight_min": -41.81422424316406,
      "activations/layer15_attention_weight_max": 40.95515441894531,
      "activations/layer15_attention_weight_min": -30.120187759399414,
      "activations/layer16_attention_weight_max": 35.04501724243164,
      "activations/layer16_attention_weight_min": -29.710031509399414,
      "activations/layer17_attention_weight_max": 39.85747146606445,
      "activations/layer17_attention_weight_min": -25.7286376953125,
      "activations/layer18_attention_weight_max": 34.235595703125,
      "activations/layer18_attention_weight_min": -21.893110275268555,
      "activations/layer19_attention_weight_max": 42.75592041015625,
      "activations/layer19_attention_weight_min": -31.140655517578125,
      "activations/layer1_attention_weight_max": 17.174274444580078,
      "activations/layer1_attention_weight_min": -14.013519287109375,
      "activations/layer20_attention_weight_max": 32.86370849609375,
      "activations/layer20_attention_weight_min": -23.740171432495117,
      "activations/layer21_attention_weight_max": 32.98344421386719,
      "activations/layer21_attention_weight_min": -23.58334732055664,
      "activations/layer22_attention_weight_max": 51.4978141784668,
      "activations/layer22_attention_weight_min": -30.615264892578125,
      "activations/layer23_attention_weight_max": 37.015560150146484,
      "activations/layer23_attention_weight_min": -21.61459732055664,
      "activations/layer2_attention_weight_max": 32.634788513183594,
      "activations/layer2_attention_weight_min": -29.647520065307617,
      "activations/layer3_attention_weight_max": 94.28594207763672,
      "activations/layer3_attention_weight_min": -92.76074981689453,
      "activations/layer4_attention_weight_max": 133.22154235839844,
      "activations/layer4_attention_weight_min": -125.0574722290039,
      "activations/layer5_attention_weight_max": 76.53744506835938,
      "activations/layer5_attention_weight_min": -71.85011291503906,
      "activations/layer6_attention_weight_max": 59.20169448852539,
      "activations/layer6_attention_weight_min": -57.33198928833008,
      "activations/layer7_attention_weight_max": 94.09734344482422,
      "activations/layer7_attention_weight_min": -92.35553741455078,
      "activations/layer8_attention_weight_max": 43.678611755371094,
      "activations/layer8_attention_weight_min": -42.58736038208008,
      "activations/layer9_attention_weight_max": 41.315547943115234,
      "activations/layer9_attention_weight_min": -36.862491607666016,
      "epoch": 6.06,
      "learning_rate": 0.00011204431818181817,
      "loss": 2.8319,
      "step": 104300
    },
    {
      "activations/layer0_attention_weight_max": 15.598169326782227,
      "activations/layer0_attention_weight_min": -12.869836807250977,
      "activations/layer10_attention_weight_max": 31.760251998901367,
      "activations/layer10_attention_weight_min": -33.235897064208984,
      "activations/layer11_attention_weight_max": 29.901187896728516,
      "activations/layer11_attention_weight_min": -30.663000106811523,
      "activations/layer12_attention_weight_max": 31.065401077270508,
      "activations/layer12_attention_weight_min": -27.836549758911133,
      "activations/layer13_attention_weight_max": 44.44102096557617,
      "activations/layer13_attention_weight_min": -33.43925476074219,
      "activations/layer14_attention_weight_max": 58.197547912597656,
      "activations/layer14_attention_weight_min": -44.62244415283203,
      "activations/layer15_attention_weight_max": 44.17198944091797,
      "activations/layer15_attention_weight_min": -31.549564361572266,
      "activations/layer16_attention_weight_max": 39.50564193725586,
      "activations/layer16_attention_weight_min": -29.3365535736084,
      "activations/layer17_attention_weight_max": 36.45392990112305,
      "activations/layer17_attention_weight_min": -25.665599822998047,
      "activations/layer18_attention_weight_max": 30.835500717163086,
      "activations/layer18_attention_weight_min": -19.572261810302734,
      "activations/layer19_attention_weight_max": 48.743804931640625,
      "activations/layer19_attention_weight_min": -33.04551315307617,
      "activations/layer1_attention_weight_max": 16.473051071166992,
      "activations/layer1_attention_weight_min": -15.327754974365234,
      "activations/layer20_attention_weight_max": 32.8538703918457,
      "activations/layer20_attention_weight_min": -23.80379295349121,
      "activations/layer21_attention_weight_max": 29.811452865600586,
      "activations/layer21_attention_weight_min": -21.8680419921875,
      "activations/layer22_attention_weight_max": 48.89844512939453,
      "activations/layer22_attention_weight_min": -26.719886779785156,
      "activations/layer23_attention_weight_max": 35.49610900878906,
      "activations/layer23_attention_weight_min": -22.770198822021484,
      "activations/layer2_attention_weight_max": 30.76435661315918,
      "activations/layer2_attention_weight_min": -30.83071517944336,
      "activations/layer3_attention_weight_max": 89.8702392578125,
      "activations/layer3_attention_weight_min": -88.36099243164062,
      "activations/layer4_attention_weight_max": 125.5696792602539,
      "activations/layer4_attention_weight_min": -126.17256164550781,
      "activations/layer5_attention_weight_max": 71.83612060546875,
      "activations/layer5_attention_weight_min": -69.73159790039062,
      "activations/layer6_attention_weight_max": 54.31325912475586,
      "activations/layer6_attention_weight_min": -52.63570785522461,
      "activations/layer7_attention_weight_max": 89.95690155029297,
      "activations/layer7_attention_weight_min": -85.68403625488281,
      "activations/layer8_attention_weight_max": 41.32788848876953,
      "activations/layer8_attention_weight_min": -44.50481414794922,
      "activations/layer9_attention_weight_max": 36.00495910644531,
      "activations/layer9_attention_weight_min": -34.25022506713867,
      "epoch": 6.06,
      "learning_rate": 0.00011202537878787877,
      "loss": 2.829,
      "step": 104350
    },
    {
      "activations/layer0_attention_weight_max": 16.4064884185791,
      "activations/layer0_attention_weight_min": -12.98759651184082,
      "activations/layer10_attention_weight_max": 35.04917907714844,
      "activations/layer10_attention_weight_min": -31.752456665039062,
      "activations/layer11_attention_weight_max": 32.66182327270508,
      "activations/layer11_attention_weight_min": -29.82977294921875,
      "activations/layer12_attention_weight_max": 24.583337783813477,
      "activations/layer12_attention_weight_min": -30.132261276245117,
      "activations/layer13_attention_weight_max": 40.67822265625,
      "activations/layer13_attention_weight_min": -31.910707473754883,
      "activations/layer14_attention_weight_max": 55.885555267333984,
      "activations/layer14_attention_weight_min": -43.790775299072266,
      "activations/layer15_attention_weight_max": 37.9571647644043,
      "activations/layer15_attention_weight_min": -28.1181697845459,
      "activations/layer16_attention_weight_max": 33.38561248779297,
      "activations/layer16_attention_weight_min": -25.832693099975586,
      "activations/layer17_attention_weight_max": 35.9738883972168,
      "activations/layer17_attention_weight_min": -24.978015899658203,
      "activations/layer18_attention_weight_max": 31.740703582763672,
      "activations/layer18_attention_weight_min": -20.492000579833984,
      "activations/layer19_attention_weight_max": 37.438228607177734,
      "activations/layer19_attention_weight_min": -31.330270767211914,
      "activations/layer1_attention_weight_max": 16.666358947753906,
      "activations/layer1_attention_weight_min": -14.028385162353516,
      "activations/layer20_attention_weight_max": 31.113107681274414,
      "activations/layer20_attention_weight_min": -24.54009246826172,
      "activations/layer21_attention_weight_max": 31.50087547302246,
      "activations/layer21_attention_weight_min": -20.713245391845703,
      "activations/layer22_attention_weight_max": 47.69279098510742,
      "activations/layer22_attention_weight_min": -28.0472469329834,
      "activations/layer23_attention_weight_max": 32.57088851928711,
      "activations/layer23_attention_weight_min": -24.90705108642578,
      "activations/layer2_attention_weight_max": 31.411632537841797,
      "activations/layer2_attention_weight_min": -30.979162216186523,
      "activations/layer3_attention_weight_max": 94.31991577148438,
      "activations/layer3_attention_weight_min": -93.5268783569336,
      "activations/layer4_attention_weight_max": 129.8808135986328,
      "activations/layer4_attention_weight_min": -132.7494354248047,
      "activations/layer5_attention_weight_max": 72.66050720214844,
      "activations/layer5_attention_weight_min": -71.7392349243164,
      "activations/layer6_attention_weight_max": 53.805076599121094,
      "activations/layer6_attention_weight_min": -57.55470657348633,
      "activations/layer7_attention_weight_max": 85.9234848022461,
      "activations/layer7_attention_weight_min": -91.19522094726562,
      "activations/layer8_attention_weight_max": 42.41639709472656,
      "activations/layer8_attention_weight_min": -41.649593353271484,
      "activations/layer9_attention_weight_max": 37.99776077270508,
      "activations/layer9_attention_weight_min": -37.08855056762695,
      "epoch": 6.07,
      "learning_rate": 0.00011200643939393939,
      "loss": 2.8335,
      "step": 104400
    },
    {
      "activations/layer0_attention_weight_max": 15.390898704528809,
      "activations/layer0_attention_weight_min": -13.16321086883545,
      "activations/layer10_attention_weight_max": 34.272308349609375,
      "activations/layer10_attention_weight_min": -30.636531829833984,
      "activations/layer11_attention_weight_max": 31.707914352416992,
      "activations/layer11_attention_weight_min": -27.25507164001465,
      "activations/layer12_attention_weight_max": 26.92551612854004,
      "activations/layer12_attention_weight_min": -26.55046272277832,
      "activations/layer13_attention_weight_max": 42.76353073120117,
      "activations/layer13_attention_weight_min": -32.62537384033203,
      "activations/layer14_attention_weight_max": 54.33712387084961,
      "activations/layer14_attention_weight_min": -40.51913070678711,
      "activations/layer15_attention_weight_max": 36.192604064941406,
      "activations/layer15_attention_weight_min": -28.25318145751953,
      "activations/layer16_attention_weight_max": 35.18793869018555,
      "activations/layer16_attention_weight_min": -28.343711853027344,
      "activations/layer17_attention_weight_max": 36.37302780151367,
      "activations/layer17_attention_weight_min": -24.90171241760254,
      "activations/layer18_attention_weight_max": 31.50604248046875,
      "activations/layer18_attention_weight_min": -21.061437606811523,
      "activations/layer19_attention_weight_max": 41.53753662109375,
      "activations/layer19_attention_weight_min": -30.726470947265625,
      "activations/layer1_attention_weight_max": 18.033119201660156,
      "activations/layer1_attention_weight_min": -13.004640579223633,
      "activations/layer20_attention_weight_max": 33.99884796142578,
      "activations/layer20_attention_weight_min": -26.255569458007812,
      "activations/layer21_attention_weight_max": 29.867990493774414,
      "activations/layer21_attention_weight_min": -22.50613021850586,
      "activations/layer22_attention_weight_max": 49.975677490234375,
      "activations/layer22_attention_weight_min": -27.674348831176758,
      "activations/layer23_attention_weight_max": 35.55760955810547,
      "activations/layer23_attention_weight_min": -22.679161071777344,
      "activations/layer2_attention_weight_max": 29.21218490600586,
      "activations/layer2_attention_weight_min": -28.81369400024414,
      "activations/layer3_attention_weight_max": 83.81275939941406,
      "activations/layer3_attention_weight_min": -86.32369232177734,
      "activations/layer4_attention_weight_max": 118.78224182128906,
      "activations/layer4_attention_weight_min": -120.98087310791016,
      "activations/layer5_attention_weight_max": 70.10189819335938,
      "activations/layer5_attention_weight_min": -67.81361389160156,
      "activations/layer6_attention_weight_max": 50.96830749511719,
      "activations/layer6_attention_weight_min": -54.03669738769531,
      "activations/layer7_attention_weight_max": 80.29263305664062,
      "activations/layer7_attention_weight_min": -79.54647064208984,
      "activations/layer8_attention_weight_max": 37.3673095703125,
      "activations/layer8_attention_weight_min": -39.897117614746094,
      "activations/layer9_attention_weight_max": 33.66329574584961,
      "activations/layer9_attention_weight_min": -32.529598236083984,
      "epoch": 6.07,
      "learning_rate": 0.00011198749999999999,
      "loss": 2.8353,
      "step": 104450
    },
    {
      "activations/layer0_attention_weight_max": 15.604790687561035,
      "activations/layer0_attention_weight_min": -12.79211711883545,
      "activations/layer10_attention_weight_max": 33.636104583740234,
      "activations/layer10_attention_weight_min": -30.88759994506836,
      "activations/layer11_attention_weight_max": 32.452880859375,
      "activations/layer11_attention_weight_min": -29.074119567871094,
      "activations/layer12_attention_weight_max": 47.017669677734375,
      "activations/layer12_attention_weight_min": -26.02124786376953,
      "activations/layer13_attention_weight_max": 57.778404235839844,
      "activations/layer13_attention_weight_min": -34.743751525878906,
      "activations/layer14_attention_weight_max": 69.66938781738281,
      "activations/layer14_attention_weight_min": -49.54000473022461,
      "activations/layer15_attention_weight_max": 42.30681228637695,
      "activations/layer15_attention_weight_min": -28.937053680419922,
      "activations/layer16_attention_weight_max": 43.36692428588867,
      "activations/layer16_attention_weight_min": -28.013675689697266,
      "activations/layer17_attention_weight_max": 44.15410614013672,
      "activations/layer17_attention_weight_min": -24.934011459350586,
      "activations/layer18_attention_weight_max": 38.006256103515625,
      "activations/layer18_attention_weight_min": -21.920331954956055,
      "activations/layer19_attention_weight_max": 43.45570373535156,
      "activations/layer19_attention_weight_min": -29.665668487548828,
      "activations/layer1_attention_weight_max": 16.280134201049805,
      "activations/layer1_attention_weight_min": -13.9834566116333,
      "activations/layer20_attention_weight_max": 35.291282653808594,
      "activations/layer20_attention_weight_min": -23.53078842163086,
      "activations/layer21_attention_weight_max": 36.750579833984375,
      "activations/layer21_attention_weight_min": -20.56719970703125,
      "activations/layer22_attention_weight_max": 59.92334747314453,
      "activations/layer22_attention_weight_min": -27.48831558227539,
      "activations/layer23_attention_weight_max": 37.78471374511719,
      "activations/layer23_attention_weight_min": -22.259246826171875,
      "activations/layer2_attention_weight_max": 31.428335189819336,
      "activations/layer2_attention_weight_min": -30.514297485351562,
      "activations/layer3_attention_weight_max": 90.65211486816406,
      "activations/layer3_attention_weight_min": -95.22010040283203,
      "activations/layer4_attention_weight_max": 126.24471282958984,
      "activations/layer4_attention_weight_min": -122.8249282836914,
      "activations/layer5_attention_weight_max": 68.29222106933594,
      "activations/layer5_attention_weight_min": -67.1199722290039,
      "activations/layer6_attention_weight_max": 50.28473663330078,
      "activations/layer6_attention_weight_min": -54.100582122802734,
      "activations/layer7_attention_weight_max": 81.78923034667969,
      "activations/layer7_attention_weight_min": -85.02299499511719,
      "activations/layer8_attention_weight_max": 39.459720611572266,
      "activations/layer8_attention_weight_min": -40.31850051879883,
      "activations/layer9_attention_weight_max": 37.491336822509766,
      "activations/layer9_attention_weight_min": -33.48317337036133,
      "epoch": 6.07,
      "learning_rate": 0.00011196856060606059,
      "loss": 2.8096,
      "step": 104500
    },
    {
      "activations/layer0_attention_weight_max": 16.125694274902344,
      "activations/layer0_attention_weight_min": -13.116217613220215,
      "activations/layer10_attention_weight_max": 33.899627685546875,
      "activations/layer10_attention_weight_min": -33.52028274536133,
      "activations/layer11_attention_weight_max": 32.285804748535156,
      "activations/layer11_attention_weight_min": -32.07422637939453,
      "activations/layer12_attention_weight_max": 46.41767883300781,
      "activations/layer12_attention_weight_min": -26.956438064575195,
      "activations/layer13_attention_weight_max": 40.0541877746582,
      "activations/layer13_attention_weight_min": -34.3568229675293,
      "activations/layer14_attention_weight_max": 57.304443359375,
      "activations/layer14_attention_weight_min": -45.62510299682617,
      "activations/layer15_attention_weight_max": 35.09520721435547,
      "activations/layer15_attention_weight_min": -29.37553596496582,
      "activations/layer16_attention_weight_max": 33.45301055908203,
      "activations/layer16_attention_weight_min": -26.505836486816406,
      "activations/layer17_attention_weight_max": 34.82735061645508,
      "activations/layer17_attention_weight_min": -24.85117530822754,
      "activations/layer18_attention_weight_max": 32.375431060791016,
      "activations/layer18_attention_weight_min": -21.04151153564453,
      "activations/layer19_attention_weight_max": 35.11698532104492,
      "activations/layer19_attention_weight_min": -28.6680965423584,
      "activations/layer1_attention_weight_max": 16.962169647216797,
      "activations/layer1_attention_weight_min": -13.550661087036133,
      "activations/layer20_attention_weight_max": 29.09569549560547,
      "activations/layer20_attention_weight_min": -22.73304557800293,
      "activations/layer21_attention_weight_max": 30.852386474609375,
      "activations/layer21_attention_weight_min": -20.96954917907715,
      "activations/layer22_attention_weight_max": 50.26374053955078,
      "activations/layer22_attention_weight_min": -29.532527923583984,
      "activations/layer23_attention_weight_max": 39.75178527832031,
      "activations/layer23_attention_weight_min": -22.064048767089844,
      "activations/layer2_attention_weight_max": 30.99726676940918,
      "activations/layer2_attention_weight_min": -28.23030662536621,
      "activations/layer3_attention_weight_max": 86.81950378417969,
      "activations/layer3_attention_weight_min": -86.06742095947266,
      "activations/layer4_attention_weight_max": 120.74785614013672,
      "activations/layer4_attention_weight_min": -118.1792984008789,
      "activations/layer5_attention_weight_max": 71.54004669189453,
      "activations/layer5_attention_weight_min": -67.03665161132812,
      "activations/layer6_attention_weight_max": 54.160953521728516,
      "activations/layer6_attention_weight_min": -54.680782318115234,
      "activations/layer7_attention_weight_max": 81.93720245361328,
      "activations/layer7_attention_weight_min": -88.05277252197266,
      "activations/layer8_attention_weight_max": 42.24677658081055,
      "activations/layer8_attention_weight_min": -42.48310470581055,
      "activations/layer9_attention_weight_max": 36.20595932006836,
      "activations/layer9_attention_weight_min": -36.9050407409668,
      "epoch": 6.07,
      "learning_rate": 0.00011194962121212121,
      "loss": 2.8262,
      "step": 104550
    },
    {
      "activations/layer0_attention_weight_max": 16.236717224121094,
      "activations/layer0_attention_weight_min": -13.692306518554688,
      "activations/layer10_attention_weight_max": 32.82711410522461,
      "activations/layer10_attention_weight_min": -30.16353988647461,
      "activations/layer11_attention_weight_max": 31.206096649169922,
      "activations/layer11_attention_weight_min": -27.45740509033203,
      "activations/layer12_attention_weight_max": 23.38370704650879,
      "activations/layer12_attention_weight_min": -25.591644287109375,
      "activations/layer13_attention_weight_max": 41.52138137817383,
      "activations/layer13_attention_weight_min": -32.18563461303711,
      "activations/layer14_attention_weight_max": 56.33345031738281,
      "activations/layer14_attention_weight_min": -40.38158416748047,
      "activations/layer15_attention_weight_max": 42.30083465576172,
      "activations/layer15_attention_weight_min": -28.700895309448242,
      "activations/layer16_attention_weight_max": 35.54754638671875,
      "activations/layer16_attention_weight_min": -28.47467803955078,
      "activations/layer17_attention_weight_max": 40.20180130004883,
      "activations/layer17_attention_weight_min": -25.07560920715332,
      "activations/layer18_attention_weight_max": 31.941364288330078,
      "activations/layer18_attention_weight_min": -20.596240997314453,
      "activations/layer19_attention_weight_max": 39.500816345214844,
      "activations/layer19_attention_weight_min": -31.188674926757812,
      "activations/layer1_attention_weight_max": 17.001684188842773,
      "activations/layer1_attention_weight_min": -15.717853546142578,
      "activations/layer20_attention_weight_max": 32.41421890258789,
      "activations/layer20_attention_weight_min": -24.343870162963867,
      "activations/layer21_attention_weight_max": 32.729347229003906,
      "activations/layer21_attention_weight_min": -22.766756057739258,
      "activations/layer22_attention_weight_max": 51.13602828979492,
      "activations/layer22_attention_weight_min": -29.97185516357422,
      "activations/layer23_attention_weight_max": 35.601863861083984,
      "activations/layer23_attention_weight_min": -22.00263023376465,
      "activations/layer2_attention_weight_max": 30.35323715209961,
      "activations/layer2_attention_weight_min": -30.13650131225586,
      "activations/layer3_attention_weight_max": 89.30426025390625,
      "activations/layer3_attention_weight_min": -89.33135986328125,
      "activations/layer4_attention_weight_max": 128.17539978027344,
      "activations/layer4_attention_weight_min": -119.10621643066406,
      "activations/layer5_attention_weight_max": 70.37885284423828,
      "activations/layer5_attention_weight_min": -66.84017181396484,
      "activations/layer6_attention_weight_max": 52.404170989990234,
      "activations/layer6_attention_weight_min": -53.50791549682617,
      "activations/layer7_attention_weight_max": 85.51789855957031,
      "activations/layer7_attention_weight_min": -84.42437744140625,
      "activations/layer8_attention_weight_max": 37.82229232788086,
      "activations/layer8_attention_weight_min": -39.87849807739258,
      "activations/layer9_attention_weight_max": 38.41748046875,
      "activations/layer9_attention_weight_min": -33.12937927246094,
      "epoch": 6.08,
      "learning_rate": 0.00011193068181818182,
      "loss": 2.8287,
      "step": 104600
    },
    {
      "activations/layer0_attention_weight_max": 16.513132095336914,
      "activations/layer0_attention_weight_min": -11.956781387329102,
      "activations/layer10_attention_weight_max": 38.5267333984375,
      "activations/layer10_attention_weight_min": -32.16054153442383,
      "activations/layer11_attention_weight_max": 40.70096969604492,
      "activations/layer11_attention_weight_min": -31.617088317871094,
      "activations/layer12_attention_weight_max": 24.717267990112305,
      "activations/layer12_attention_weight_min": -25.113155364990234,
      "activations/layer13_attention_weight_max": 50.24103927612305,
      "activations/layer13_attention_weight_min": -33.13914489746094,
      "activations/layer14_attention_weight_max": 69.20219421386719,
      "activations/layer14_attention_weight_min": -52.44810104370117,
      "activations/layer15_attention_weight_max": 38.83279800415039,
      "activations/layer15_attention_weight_min": -31.288522720336914,
      "activations/layer16_attention_weight_max": 35.48020553588867,
      "activations/layer16_attention_weight_min": -29.803205490112305,
      "activations/layer17_attention_weight_max": 36.9432487487793,
      "activations/layer17_attention_weight_min": -25.32036018371582,
      "activations/layer18_attention_weight_max": 31.335880279541016,
      "activations/layer18_attention_weight_min": -20.683265686035156,
      "activations/layer19_attention_weight_max": 41.81815719604492,
      "activations/layer19_attention_weight_min": -29.782682418823242,
      "activations/layer1_attention_weight_max": 17.448135375976562,
      "activations/layer1_attention_weight_min": -13.191033363342285,
      "activations/layer20_attention_weight_max": 36.12567138671875,
      "activations/layer20_attention_weight_min": -24.525218963623047,
      "activations/layer21_attention_weight_max": 27.86021614074707,
      "activations/layer21_attention_weight_min": -21.21693992614746,
      "activations/layer22_attention_weight_max": 54.02939987182617,
      "activations/layer22_attention_weight_min": -28.467018127441406,
      "activations/layer23_attention_weight_max": 34.19921112060547,
      "activations/layer23_attention_weight_min": -23.22250747680664,
      "activations/layer2_attention_weight_max": 32.74150085449219,
      "activations/layer2_attention_weight_min": -31.47443962097168,
      "activations/layer3_attention_weight_max": 96.97228240966797,
      "activations/layer3_attention_weight_min": -96.91814422607422,
      "activations/layer4_attention_weight_max": 133.5682373046875,
      "activations/layer4_attention_weight_min": -130.366455078125,
      "activations/layer5_attention_weight_max": 75.15031433105469,
      "activations/layer5_attention_weight_min": -72.00324249267578,
      "activations/layer6_attention_weight_max": 54.61414337158203,
      "activations/layer6_attention_weight_min": -56.560829162597656,
      "activations/layer7_attention_weight_max": 90.7855224609375,
      "activations/layer7_attention_weight_min": -89.11234283447266,
      "activations/layer8_attention_weight_max": 42.17607116699219,
      "activations/layer8_attention_weight_min": -40.60511779785156,
      "activations/layer9_attention_weight_max": 38.77855682373047,
      "activations/layer9_attention_weight_min": -40.30867004394531,
      "epoch": 6.08,
      "learning_rate": 0.00011191174242424241,
      "loss": 2.8345,
      "step": 104650
    },
    {
      "activations/layer0_attention_weight_max": 16.73577308654785,
      "activations/layer0_attention_weight_min": -13.171514511108398,
      "activations/layer10_attention_weight_max": 34.588565826416016,
      "activations/layer10_attention_weight_min": -32.050575256347656,
      "activations/layer11_attention_weight_max": 33.03343963623047,
      "activations/layer11_attention_weight_min": -27.667699813842773,
      "activations/layer12_attention_weight_max": 35.7171745300293,
      "activations/layer12_attention_weight_min": -24.988590240478516,
      "activations/layer13_attention_weight_max": 53.190731048583984,
      "activations/layer13_attention_weight_min": -35.588592529296875,
      "activations/layer14_attention_weight_max": 65.56782531738281,
      "activations/layer14_attention_weight_min": -42.21080017089844,
      "activations/layer15_attention_weight_max": 43.22718811035156,
      "activations/layer15_attention_weight_min": -29.64100456237793,
      "activations/layer16_attention_weight_max": 36.30437088012695,
      "activations/layer16_attention_weight_min": -28.50901985168457,
      "activations/layer17_attention_weight_max": 38.99614334106445,
      "activations/layer17_attention_weight_min": -24.984878540039062,
      "activations/layer18_attention_weight_max": 33.77653121948242,
      "activations/layer18_attention_weight_min": -20.53181266784668,
      "activations/layer19_attention_weight_max": 41.853309631347656,
      "activations/layer19_attention_weight_min": -29.130964279174805,
      "activations/layer1_attention_weight_max": 17.29479217529297,
      "activations/layer1_attention_weight_min": -12.692328453063965,
      "activations/layer20_attention_weight_max": 34.71834182739258,
      "activations/layer20_attention_weight_min": -23.991117477416992,
      "activations/layer21_attention_weight_max": 34.35882568359375,
      "activations/layer21_attention_weight_min": -21.30714225769043,
      "activations/layer22_attention_weight_max": 60.94327163696289,
      "activations/layer22_attention_weight_min": -26.759084701538086,
      "activations/layer23_attention_weight_max": 41.96921157836914,
      "activations/layer23_attention_weight_min": -23.31782341003418,
      "activations/layer2_attention_weight_max": 31.099437713623047,
      "activations/layer2_attention_weight_min": -28.966411590576172,
      "activations/layer3_attention_weight_max": 87.9879379272461,
      "activations/layer3_attention_weight_min": -92.601318359375,
      "activations/layer4_attention_weight_max": 127.32939910888672,
      "activations/layer4_attention_weight_min": -124.31536865234375,
      "activations/layer5_attention_weight_max": 71.28724670410156,
      "activations/layer5_attention_weight_min": -69.72393035888672,
      "activations/layer6_attention_weight_max": 59.90782928466797,
      "activations/layer6_attention_weight_min": -53.824825286865234,
      "activations/layer7_attention_weight_max": 85.70021057128906,
      "activations/layer7_attention_weight_min": -88.38130187988281,
      "activations/layer8_attention_weight_max": 39.45880889892578,
      "activations/layer8_attention_weight_min": -39.32160186767578,
      "activations/layer9_attention_weight_max": 36.154441833496094,
      "activations/layer9_attention_weight_min": -34.16611099243164,
      "epoch": 6.08,
      "learning_rate": 0.00011189280303030303,
      "loss": 2.824,
      "step": 104700
    },
    {
      "activations/layer0_attention_weight_max": 16.19345474243164,
      "activations/layer0_attention_weight_min": -12.699578285217285,
      "activations/layer10_attention_weight_max": 36.65673065185547,
      "activations/layer10_attention_weight_min": -31.737157821655273,
      "activations/layer11_attention_weight_max": 36.598236083984375,
      "activations/layer11_attention_weight_min": -32.101524353027344,
      "activations/layer12_attention_weight_max": 33.5394401550293,
      "activations/layer12_attention_weight_min": -28.717247009277344,
      "activations/layer13_attention_weight_max": 42.13645553588867,
      "activations/layer13_attention_weight_min": -30.942773818969727,
      "activations/layer14_attention_weight_max": 68.80179595947266,
      "activations/layer14_attention_weight_min": -44.773216247558594,
      "activations/layer15_attention_weight_max": 37.926475524902344,
      "activations/layer15_attention_weight_min": -28.715341567993164,
      "activations/layer16_attention_weight_max": 40.01612091064453,
      "activations/layer16_attention_weight_min": -29.12652587890625,
      "activations/layer17_attention_weight_max": 38.24787521362305,
      "activations/layer17_attention_weight_min": -24.298254013061523,
      "activations/layer18_attention_weight_max": 28.361181259155273,
      "activations/layer18_attention_weight_min": -20.869932174682617,
      "activations/layer19_attention_weight_max": 41.93291091918945,
      "activations/layer19_attention_weight_min": -31.195669174194336,
      "activations/layer1_attention_weight_max": 17.724851608276367,
      "activations/layer1_attention_weight_min": -15.884800910949707,
      "activations/layer20_attention_weight_max": 30.02300453186035,
      "activations/layer20_attention_weight_min": -24.754188537597656,
      "activations/layer21_attention_weight_max": 26.32491111755371,
      "activations/layer21_attention_weight_min": -20.5037841796875,
      "activations/layer22_attention_weight_max": 52.137474060058594,
      "activations/layer22_attention_weight_min": -27.253673553466797,
      "activations/layer23_attention_weight_max": 39.135623931884766,
      "activations/layer23_attention_weight_min": -21.06589698791504,
      "activations/layer2_attention_weight_max": 28.637248992919922,
      "activations/layer2_attention_weight_min": -29.74509048461914,
      "activations/layer3_attention_weight_max": 87.37226867675781,
      "activations/layer3_attention_weight_min": -89.74510192871094,
      "activations/layer4_attention_weight_max": 127.03873443603516,
      "activations/layer4_attention_weight_min": -123.15742492675781,
      "activations/layer5_attention_weight_max": 72.26670837402344,
      "activations/layer5_attention_weight_min": -67.75845336914062,
      "activations/layer6_attention_weight_max": 54.884037017822266,
      "activations/layer6_attention_weight_min": -53.28884506225586,
      "activations/layer7_attention_weight_max": 87.78515625,
      "activations/layer7_attention_weight_min": -86.97330474853516,
      "activations/layer8_attention_weight_max": 43.298797607421875,
      "activations/layer8_attention_weight_min": -43.63603973388672,
      "activations/layer9_attention_weight_max": 37.56316375732422,
      "activations/layer9_attention_weight_min": -35.78542709350586,
      "epoch": 6.09,
      "learning_rate": 0.00011187386363636361,
      "loss": 2.8282,
      "step": 104750
    },
    {
      "activations/layer0_attention_weight_max": 16.298086166381836,
      "activations/layer0_attention_weight_min": -13.054409980773926,
      "activations/layer10_attention_weight_max": 40.88603973388672,
      "activations/layer10_attention_weight_min": -40.036598205566406,
      "activations/layer11_attention_weight_max": 38.40205001831055,
      "activations/layer11_attention_weight_min": -37.065147399902344,
      "activations/layer12_attention_weight_max": 31.000146865844727,
      "activations/layer12_attention_weight_min": -28.3381290435791,
      "activations/layer13_attention_weight_max": 46.727542877197266,
      "activations/layer13_attention_weight_min": -42.29972839355469,
      "activations/layer14_attention_weight_max": 57.22837448120117,
      "activations/layer14_attention_weight_min": -48.100830078125,
      "activations/layer15_attention_weight_max": 40.574459075927734,
      "activations/layer15_attention_weight_min": -36.00358200073242,
      "activations/layer16_attention_weight_max": 32.23520278930664,
      "activations/layer16_attention_weight_min": -27.207460403442383,
      "activations/layer17_attention_weight_max": 38.199275970458984,
      "activations/layer17_attention_weight_min": -25.106943130493164,
      "activations/layer18_attention_weight_max": 31.21751594543457,
      "activations/layer18_attention_weight_min": -18.852703094482422,
      "activations/layer19_attention_weight_max": 33.9513053894043,
      "activations/layer19_attention_weight_min": -31.15342140197754,
      "activations/layer1_attention_weight_max": 17.027027130126953,
      "activations/layer1_attention_weight_min": -14.60524845123291,
      "activations/layer20_attention_weight_max": 30.32250213623047,
      "activations/layer20_attention_weight_min": -24.892471313476562,
      "activations/layer21_attention_weight_max": 28.072601318359375,
      "activations/layer21_attention_weight_min": -21.58463478088379,
      "activations/layer22_attention_weight_max": 48.38120651245117,
      "activations/layer22_attention_weight_min": -27.147926330566406,
      "activations/layer23_attention_weight_max": 34.168907165527344,
      "activations/layer23_attention_weight_min": -23.21292495727539,
      "activations/layer2_attention_weight_max": 29.41937828063965,
      "activations/layer2_attention_weight_min": -30.803600311279297,
      "activations/layer3_attention_weight_max": 86.29875183105469,
      "activations/layer3_attention_weight_min": -93.58697509765625,
      "activations/layer4_attention_weight_max": 126.79438781738281,
      "activations/layer4_attention_weight_min": -126.05965423583984,
      "activations/layer5_attention_weight_max": 78.6794204711914,
      "activations/layer5_attention_weight_min": -70.71280670166016,
      "activations/layer6_attention_weight_max": 59.24555587768555,
      "activations/layer6_attention_weight_min": -57.5467414855957,
      "activations/layer7_attention_weight_max": 112.24481964111328,
      "activations/layer7_attention_weight_min": -101.5560302734375,
      "activations/layer8_attention_weight_max": 48.632179260253906,
      "activations/layer8_attention_weight_min": -46.85866165161133,
      "activations/layer9_attention_weight_max": 43.24576950073242,
      "activations/layer9_attention_weight_min": -43.926509857177734,
      "epoch": 6.09,
      "learning_rate": 0.00011185530303030303,
      "loss": 2.8339,
      "step": 104800
    },
    {
      "activations/layer0_attention_weight_max": 16.664419174194336,
      "activations/layer0_attention_weight_min": -12.8095121383667,
      "activations/layer10_attention_weight_max": 31.534679412841797,
      "activations/layer10_attention_weight_min": -30.094175338745117,
      "activations/layer11_attention_weight_max": 31.44491195678711,
      "activations/layer11_attention_weight_min": -27.959453582763672,
      "activations/layer12_attention_weight_max": 39.838863372802734,
      "activations/layer12_attention_weight_min": -24.899253845214844,
      "activations/layer13_attention_weight_max": 43.18465805053711,
      "activations/layer13_attention_weight_min": -33.383792877197266,
      "activations/layer14_attention_weight_max": 62.20913314819336,
      "activations/layer14_attention_weight_min": -40.53636932373047,
      "activations/layer15_attention_weight_max": 39.47807312011719,
      "activations/layer15_attention_weight_min": -28.607080459594727,
      "activations/layer16_attention_weight_max": 35.42597198486328,
      "activations/layer16_attention_weight_min": -27.052762985229492,
      "activations/layer17_attention_weight_max": 39.231834411621094,
      "activations/layer17_attention_weight_min": -24.29966163635254,
      "activations/layer18_attention_weight_max": 32.64720153808594,
      "activations/layer18_attention_weight_min": -19.758108139038086,
      "activations/layer19_attention_weight_max": 40.32897186279297,
      "activations/layer19_attention_weight_min": -29.341678619384766,
      "activations/layer1_attention_weight_max": 16.46074676513672,
      "activations/layer1_attention_weight_min": -14.701812744140625,
      "activations/layer20_attention_weight_max": 35.15981674194336,
      "activations/layer20_attention_weight_min": -24.405920028686523,
      "activations/layer21_attention_weight_max": 34.4041862487793,
      "activations/layer21_attention_weight_min": -20.21268653869629,
      "activations/layer22_attention_weight_max": 53.675289154052734,
      "activations/layer22_attention_weight_min": -29.173091888427734,
      "activations/layer23_attention_weight_max": 36.119503021240234,
      "activations/layer23_attention_weight_min": -22.22449493408203,
      "activations/layer2_attention_weight_max": 30.192310333251953,
      "activations/layer2_attention_weight_min": -28.611770629882812,
      "activations/layer3_attention_weight_max": 86.98597717285156,
      "activations/layer3_attention_weight_min": -89.3132553100586,
      "activations/layer4_attention_weight_max": 123.50947570800781,
      "activations/layer4_attention_weight_min": -124.13044738769531,
      "activations/layer5_attention_weight_max": 69.35836791992188,
      "activations/layer5_attention_weight_min": -66.60922241210938,
      "activations/layer6_attention_weight_max": 51.53990936279297,
      "activations/layer6_attention_weight_min": -57.55131149291992,
      "activations/layer7_attention_weight_max": 86.4930191040039,
      "activations/layer7_attention_weight_min": -99.24250030517578,
      "activations/layer8_attention_weight_max": 40.623329162597656,
      "activations/layer8_attention_weight_min": -40.85353088378906,
      "activations/layer9_attention_weight_max": 36.37118148803711,
      "activations/layer9_attention_weight_min": -35.4210319519043,
      "epoch": 6.09,
      "learning_rate": 0.00011183636363636362,
      "loss": 2.826,
      "step": 104850
    },
    {
      "activations/layer0_attention_weight_max": 16.749998092651367,
      "activations/layer0_attention_weight_min": -11.996798515319824,
      "activations/layer10_attention_weight_max": 32.91709899902344,
      "activations/layer10_attention_weight_min": -29.963396072387695,
      "activations/layer11_attention_weight_max": 35.00621795654297,
      "activations/layer11_attention_weight_min": -29.031749725341797,
      "activations/layer12_attention_weight_max": 31.640230178833008,
      "activations/layer12_attention_weight_min": -28.562910079956055,
      "activations/layer13_attention_weight_max": 46.41149139404297,
      "activations/layer13_attention_weight_min": -33.98159408569336,
      "activations/layer14_attention_weight_max": 56.00404739379883,
      "activations/layer14_attention_weight_min": -42.93183898925781,
      "activations/layer15_attention_weight_max": 41.09165954589844,
      "activations/layer15_attention_weight_min": -27.21390151977539,
      "activations/layer16_attention_weight_max": 38.60258102416992,
      "activations/layer16_attention_weight_min": -27.28190040588379,
      "activations/layer17_attention_weight_max": 34.7522087097168,
      "activations/layer17_attention_weight_min": -22.964244842529297,
      "activations/layer18_attention_weight_max": 34.98540496826172,
      "activations/layer18_attention_weight_min": -21.40440559387207,
      "activations/layer19_attention_weight_max": 41.584678649902344,
      "activations/layer19_attention_weight_min": -29.6707706451416,
      "activations/layer1_attention_weight_max": 16.96368980407715,
      "activations/layer1_attention_weight_min": -14.307567596435547,
      "activations/layer20_attention_weight_max": 34.295711517333984,
      "activations/layer20_attention_weight_min": -24.20616340637207,
      "activations/layer21_attention_weight_max": 33.40036392211914,
      "activations/layer21_attention_weight_min": -20.927030563354492,
      "activations/layer22_attention_weight_max": 51.76749801635742,
      "activations/layer22_attention_weight_min": -26.406314849853516,
      "activations/layer23_attention_weight_max": 35.91703414916992,
      "activations/layer23_attention_weight_min": -20.100791931152344,
      "activations/layer2_attention_weight_max": 31.480484008789062,
      "activations/layer2_attention_weight_min": -30.373699188232422,
      "activations/layer3_attention_weight_max": 89.7345962524414,
      "activations/layer3_attention_weight_min": -88.6397476196289,
      "activations/layer4_attention_weight_max": 125.8429183959961,
      "activations/layer4_attention_weight_min": -119.44144439697266,
      "activations/layer5_attention_weight_max": 70.83641052246094,
      "activations/layer5_attention_weight_min": -66.32190704345703,
      "activations/layer6_attention_weight_max": 55.395111083984375,
      "activations/layer6_attention_weight_min": -51.70096969604492,
      "activations/layer7_attention_weight_max": 87.55252838134766,
      "activations/layer7_attention_weight_min": -82.78874969482422,
      "activations/layer8_attention_weight_max": 39.717376708984375,
      "activations/layer8_attention_weight_min": -41.89432144165039,
      "activations/layer9_attention_weight_max": 36.434207916259766,
      "activations/layer9_attention_weight_min": -34.781009674072266,
      "epoch": 6.1,
      "learning_rate": 0.00011181742424242423,
      "loss": 2.8284,
      "step": 104900
    },
    {
      "activations/layer0_attention_weight_max": 16.3969669342041,
      "activations/layer0_attention_weight_min": -12.097994804382324,
      "activations/layer10_attention_weight_max": 33.87028884887695,
      "activations/layer10_attention_weight_min": -30.511669158935547,
      "activations/layer11_attention_weight_max": 32.22692108154297,
      "activations/layer11_attention_weight_min": -28.021812438964844,
      "activations/layer12_attention_weight_max": 25.81932258605957,
      "activations/layer12_attention_weight_min": -25.438032150268555,
      "activations/layer13_attention_weight_max": 44.14521408081055,
      "activations/layer13_attention_weight_min": -30.558259963989258,
      "activations/layer14_attention_weight_max": 62.765743255615234,
      "activations/layer14_attention_weight_min": -42.33803939819336,
      "activations/layer15_attention_weight_max": 51.73346710205078,
      "activations/layer15_attention_weight_min": -29.74099349975586,
      "activations/layer16_attention_weight_max": 47.061744689941406,
      "activations/layer16_attention_weight_min": -26.85470962524414,
      "activations/layer17_attention_weight_max": 53.68501281738281,
      "activations/layer17_attention_weight_min": -24.736534118652344,
      "activations/layer18_attention_weight_max": 39.44808578491211,
      "activations/layer18_attention_weight_min": -25.832006454467773,
      "activations/layer19_attention_weight_max": 43.95248794555664,
      "activations/layer19_attention_weight_min": -29.43267250061035,
      "activations/layer1_attention_weight_max": 16.752779006958008,
      "activations/layer1_attention_weight_min": -16.024778366088867,
      "activations/layer20_attention_weight_max": 34.71076202392578,
      "activations/layer20_attention_weight_min": -26.034568786621094,
      "activations/layer21_attention_weight_max": 39.12004470825195,
      "activations/layer21_attention_weight_min": -22.30225944519043,
      "activations/layer22_attention_weight_max": 57.852413177490234,
      "activations/layer22_attention_weight_min": -29.597492218017578,
      "activations/layer23_attention_weight_max": 37.713645935058594,
      "activations/layer23_attention_weight_min": -20.571189880371094,
      "activations/layer2_attention_weight_max": 29.990161895751953,
      "activations/layer2_attention_weight_min": -28.883304595947266,
      "activations/layer3_attention_weight_max": 83.01390075683594,
      "activations/layer3_attention_weight_min": -86.16824340820312,
      "activations/layer4_attention_weight_max": 118.99488830566406,
      "activations/layer4_attention_weight_min": -116.48046112060547,
      "activations/layer5_attention_weight_max": 67.67578887939453,
      "activations/layer5_attention_weight_min": -67.33848571777344,
      "activations/layer6_attention_weight_max": 49.5431022644043,
      "activations/layer6_attention_weight_min": -52.85763168334961,
      "activations/layer7_attention_weight_max": 87.18177795410156,
      "activations/layer7_attention_weight_min": -80.77078247070312,
      "activations/layer8_attention_weight_max": 39.071163177490234,
      "activations/layer8_attention_weight_min": -41.39078140258789,
      "activations/layer9_attention_weight_max": 36.9403076171875,
      "activations/layer9_attention_weight_min": -33.09308624267578,
      "epoch": 6.1,
      "learning_rate": 0.00011179848484848484,
      "loss": 2.8283,
      "step": 104950
    },
    {
      "activations/layer0_attention_weight_max": 15.089920043945312,
      "activations/layer0_attention_weight_min": -12.608602523803711,
      "activations/layer10_attention_weight_max": 33.0386962890625,
      "activations/layer10_attention_weight_min": -33.49628448486328,
      "activations/layer11_attention_weight_max": 41.63143539428711,
      "activations/layer11_attention_weight_min": -30.397397994995117,
      "activations/layer12_attention_weight_max": 27.857662200927734,
      "activations/layer12_attention_weight_min": -33.348297119140625,
      "activations/layer13_attention_weight_max": 46.5225715637207,
      "activations/layer13_attention_weight_min": -33.00698471069336,
      "activations/layer14_attention_weight_max": 62.0627326965332,
      "activations/layer14_attention_weight_min": -47.70600891113281,
      "activations/layer15_attention_weight_max": 52.39259719848633,
      "activations/layer15_attention_weight_min": -33.03571319580078,
      "activations/layer16_attention_weight_max": 37.61272048950195,
      "activations/layer16_attention_weight_min": -28.558677673339844,
      "activations/layer17_attention_weight_max": 39.850067138671875,
      "activations/layer17_attention_weight_min": -24.8908748626709,
      "activations/layer18_attention_weight_max": 30.48560905456543,
      "activations/layer18_attention_weight_min": -21.017406463623047,
      "activations/layer19_attention_weight_max": 49.17644119262695,
      "activations/layer19_attention_weight_min": -31.040363311767578,
      "activations/layer1_attention_weight_max": 16.87320327758789,
      "activations/layer1_attention_weight_min": -14.82066535949707,
      "activations/layer20_attention_weight_max": 34.95276641845703,
      "activations/layer20_attention_weight_min": -23.79869842529297,
      "activations/layer21_attention_weight_max": 33.192665100097656,
      "activations/layer21_attention_weight_min": -21.53702163696289,
      "activations/layer22_attention_weight_max": 50.29652404785156,
      "activations/layer22_attention_weight_min": -28.98712730407715,
      "activations/layer23_attention_weight_max": 37.9306640625,
      "activations/layer23_attention_weight_min": -18.549102783203125,
      "activations/layer2_attention_weight_max": 32.324195861816406,
      "activations/layer2_attention_weight_min": -31.26019859313965,
      "activations/layer3_attention_weight_max": 99.40106964111328,
      "activations/layer3_attention_weight_min": -101.56594848632812,
      "activations/layer4_attention_weight_max": 139.7390594482422,
      "activations/layer4_attention_weight_min": -136.62457275390625,
      "activations/layer5_attention_weight_max": 72.20611572265625,
      "activations/layer5_attention_weight_min": -69.24849700927734,
      "activations/layer6_attention_weight_max": 55.052955627441406,
      "activations/layer6_attention_weight_min": -57.895301818847656,
      "activations/layer7_attention_weight_max": 91.18388366699219,
      "activations/layer7_attention_weight_min": -89.61402130126953,
      "activations/layer8_attention_weight_max": 41.07744216918945,
      "activations/layer8_attention_weight_min": -41.34253692626953,
      "activations/layer9_attention_weight_max": 35.395076751708984,
      "activations/layer9_attention_weight_min": -40.8206787109375,
      "epoch": 6.1,
      "learning_rate": 0.00011177954545454544,
      "loss": 2.8214,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.4944,
      "eval_samples_per_second": 505.507,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.4944,
      "eval_openwebtext_samples_per_second": 505.507,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_wikitext_loss": 3.01171875,
      "eval_wikitext_ppl": 20.322298876557472,
      "eval_wikitext_runtime": 2.0326,
      "eval_wikitext_samples_per_second": 224.346,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_lambada_loss": 2.67578125,
      "eval_lambada_ppl": 14.523692042874359,
      "eval_lambada_runtime": 9.5538,
      "eval_lambada_samples_per_second": 509.641,
      "step": 105000
    },
    {
      "activations/layer0_attention_weight_max": 15.689809799194336,
      "activations/layer0_attention_weight_min": -12.962636947631836,
      "activations/layer10_attention_weight_max": 42.95907211303711,
      "activations/layer10_attention_weight_min": -38.67055130004883,
      "activations/layer11_attention_weight_max": 41.58198547363281,
      "activations/layer11_attention_weight_min": -35.58449172973633,
      "activations/layer12_attention_weight_max": 24.42937660217285,
      "activations/layer12_attention_weight_min": -27.701566696166992,
      "activations/layer13_attention_weight_max": 45.34885787963867,
      "activations/layer13_attention_weight_min": -34.46955871582031,
      "activations/layer14_attention_weight_max": 61.57570266723633,
      "activations/layer14_attention_weight_min": -44.387943267822266,
      "activations/layer15_attention_weight_max": 39.472129821777344,
      "activations/layer15_attention_weight_min": -28.632062911987305,
      "activations/layer16_attention_weight_max": 35.72970962524414,
      "activations/layer16_attention_weight_min": -27.874412536621094,
      "activations/layer17_attention_weight_max": 36.12444305419922,
      "activations/layer17_attention_weight_min": -23.22836685180664,
      "activations/layer18_attention_weight_max": 32.06159973144531,
      "activations/layer18_attention_weight_min": -20.640029907226562,
      "activations/layer19_attention_weight_max": 37.99897003173828,
      "activations/layer19_attention_weight_min": -28.883222579956055,
      "activations/layer1_attention_weight_max": 16.613317489624023,
      "activations/layer1_attention_weight_min": -14.851658821105957,
      "activations/layer20_attention_weight_max": 31.653579711914062,
      "activations/layer20_attention_weight_min": -22.328115463256836,
      "activations/layer21_attention_weight_max": 33.18545913696289,
      "activations/layer21_attention_weight_min": -19.649524688720703,
      "activations/layer22_attention_weight_max": 53.66535949707031,
      "activations/layer22_attention_weight_min": -24.950458526611328,
      "activations/layer23_attention_weight_max": 34.49107360839844,
      "activations/layer23_attention_weight_min": -20.939525604248047,
      "activations/layer2_attention_weight_max": 34.52248001098633,
      "activations/layer2_attention_weight_min": -31.787071228027344,
      "activations/layer3_attention_weight_max": 98.93771362304688,
      "activations/layer3_attention_weight_min": -98.14373779296875,
      "activations/layer4_attention_weight_max": 139.48416137695312,
      "activations/layer4_attention_weight_min": -128.9381561279297,
      "activations/layer5_attention_weight_max": 83.56964111328125,
      "activations/layer5_attention_weight_min": -71.10877990722656,
      "activations/layer6_attention_weight_max": 62.23338317871094,
      "activations/layer6_attention_weight_min": -54.439083099365234,
      "activations/layer7_attention_weight_max": 108.3185043334961,
      "activations/layer7_attention_weight_min": -94.1630859375,
      "activations/layer8_attention_weight_max": 54.85725784301758,
      "activations/layer8_attention_weight_min": -47.8227653503418,
      "activations/layer9_attention_weight_max": 46.41732406616211,
      "activations/layer9_attention_weight_min": -43.11030197143555,
      "epoch": 6.1,
      "learning_rate": 0.00011176060606060605,
      "loss": 2.83,
      "step": 105050
    },
    {
      "activations/layer0_attention_weight_max": 15.691378593444824,
      "activations/layer0_attention_weight_min": -12.804075241088867,
      "activations/layer10_attention_weight_max": 31.329448699951172,
      "activations/layer10_attention_weight_min": -29.627410888671875,
      "activations/layer11_attention_weight_max": 31.79558563232422,
      "activations/layer11_attention_weight_min": -28.275197982788086,
      "activations/layer12_attention_weight_max": 26.24899673461914,
      "activations/layer12_attention_weight_min": -26.841724395751953,
      "activations/layer13_attention_weight_max": 38.59303665161133,
      "activations/layer13_attention_weight_min": -33.52749252319336,
      "activations/layer14_attention_weight_max": 61.99129104614258,
      "activations/layer14_attention_weight_min": -44.63328170776367,
      "activations/layer15_attention_weight_max": 35.860755920410156,
      "activations/layer15_attention_weight_min": -30.139179229736328,
      "activations/layer16_attention_weight_max": 34.698551177978516,
      "activations/layer16_attention_weight_min": -28.967809677124023,
      "activations/layer17_attention_weight_max": 37.011505126953125,
      "activations/layer17_attention_weight_min": -25.468420028686523,
      "activations/layer18_attention_weight_max": 29.153968811035156,
      "activations/layer18_attention_weight_min": -20.84402847290039,
      "activations/layer19_attention_weight_max": 45.808807373046875,
      "activations/layer19_attention_weight_min": -30.18586540222168,
      "activations/layer1_attention_weight_max": 16.173564910888672,
      "activations/layer1_attention_weight_min": -13.770893096923828,
      "activations/layer20_attention_weight_max": 36.361846923828125,
      "activations/layer20_attention_weight_min": -23.946733474731445,
      "activations/layer21_attention_weight_max": 33.0134162902832,
      "activations/layer21_attention_weight_min": -20.031164169311523,
      "activations/layer22_attention_weight_max": 53.05481719970703,
      "activations/layer22_attention_weight_min": -29.53679656982422,
      "activations/layer23_attention_weight_max": 38.6967658996582,
      "activations/layer23_attention_weight_min": -22.74677276611328,
      "activations/layer2_attention_weight_max": 31.373991012573242,
      "activations/layer2_attention_weight_min": -30.02105712890625,
      "activations/layer3_attention_weight_max": 91.09020233154297,
      "activations/layer3_attention_weight_min": -93.91150665283203,
      "activations/layer4_attention_weight_max": 129.45724487304688,
      "activations/layer4_attention_weight_min": -125.97106170654297,
      "activations/layer5_attention_weight_max": 69.39537811279297,
      "activations/layer5_attention_weight_min": -68.75579071044922,
      "activations/layer6_attention_weight_max": 52.363582611083984,
      "activations/layer6_attention_weight_min": -53.55510711669922,
      "activations/layer7_attention_weight_max": 84.32425689697266,
      "activations/layer7_attention_weight_min": -83.94175720214844,
      "activations/layer8_attention_weight_max": 39.04401779174805,
      "activations/layer8_attention_weight_min": -38.39726638793945,
      "activations/layer9_attention_weight_max": 35.23923873901367,
      "activations/layer9_attention_weight_min": -37.84040451049805,
      "epoch": 6.11,
      "learning_rate": 0.00011174166666666665,
      "loss": 2.8406,
      "step": 105100
    },
    {
      "activations/layer0_attention_weight_max": 16.203218460083008,
      "activations/layer0_attention_weight_min": -12.239972114562988,
      "activations/layer10_attention_weight_max": 34.58418655395508,
      "activations/layer10_attention_weight_min": -35.50244903564453,
      "activations/layer11_attention_weight_max": 31.61305809020996,
      "activations/layer11_attention_weight_min": -29.99769401550293,
      "activations/layer12_attention_weight_max": 28.529739379882812,
      "activations/layer12_attention_weight_min": -27.261066436767578,
      "activations/layer13_attention_weight_max": 45.5259895324707,
      "activations/layer13_attention_weight_min": -32.451602935791016,
      "activations/layer14_attention_weight_max": 76.1526107788086,
      "activations/layer14_attention_weight_min": -47.61427307128906,
      "activations/layer15_attention_weight_max": 38.119667053222656,
      "activations/layer15_attention_weight_min": -27.912506103515625,
      "activations/layer16_attention_weight_max": 38.25222396850586,
      "activations/layer16_attention_weight_min": -29.746641159057617,
      "activations/layer17_attention_weight_max": 37.89089584350586,
      "activations/layer17_attention_weight_min": -25.9939022064209,
      "activations/layer18_attention_weight_max": 35.46786880493164,
      "activations/layer18_attention_weight_min": -20.626909255981445,
      "activations/layer19_attention_weight_max": 40.55616760253906,
      "activations/layer19_attention_weight_min": -29.39620018005371,
      "activations/layer1_attention_weight_max": 17.44750213623047,
      "activations/layer1_attention_weight_min": -13.755745887756348,
      "activations/layer20_attention_weight_max": 35.58012771606445,
      "activations/layer20_attention_weight_min": -23.597301483154297,
      "activations/layer21_attention_weight_max": 33.37725067138672,
      "activations/layer21_attention_weight_min": -21.58616065979004,
      "activations/layer22_attention_weight_max": 52.73209762573242,
      "activations/layer22_attention_weight_min": -28.56821632385254,
      "activations/layer23_attention_weight_max": 40.09606170654297,
      "activations/layer23_attention_weight_min": -22.696670532226562,
      "activations/layer2_attention_weight_max": 32.0363655090332,
      "activations/layer2_attention_weight_min": -32.140811920166016,
      "activations/layer3_attention_weight_max": 93.31808471679688,
      "activations/layer3_attention_weight_min": -96.76596069335938,
      "activations/layer4_attention_weight_max": 134.5345001220703,
      "activations/layer4_attention_weight_min": -127.24458312988281,
      "activations/layer5_attention_weight_max": 73.38188934326172,
      "activations/layer5_attention_weight_min": -72.08796691894531,
      "activations/layer6_attention_weight_max": 55.36343765258789,
      "activations/layer6_attention_weight_min": -59.49216842651367,
      "activations/layer7_attention_weight_max": 93.10430145263672,
      "activations/layer7_attention_weight_min": -96.34089660644531,
      "activations/layer8_attention_weight_max": 41.924530029296875,
      "activations/layer8_attention_weight_min": -42.73622512817383,
      "activations/layer9_attention_weight_max": 37.81173324584961,
      "activations/layer9_attention_weight_min": -37.25393295288086,
      "epoch": 6.11,
      "learning_rate": 0.00011172272727272727,
      "loss": 2.8023,
      "step": 105150
    },
    {
      "activations/layer0_attention_weight_max": 16.49774169921875,
      "activations/layer0_attention_weight_min": -11.206178665161133,
      "activations/layer10_attention_weight_max": 35.453800201416016,
      "activations/layer10_attention_weight_min": -30.980300903320312,
      "activations/layer11_attention_weight_max": 35.718955993652344,
      "activations/layer11_attention_weight_min": -28.703372955322266,
      "activations/layer12_attention_weight_max": 26.65745735168457,
      "activations/layer12_attention_weight_min": -26.72185516357422,
      "activations/layer13_attention_weight_max": 44.88205337524414,
      "activations/layer13_attention_weight_min": -34.033931732177734,
      "activations/layer14_attention_weight_max": 75.75841522216797,
      "activations/layer14_attention_weight_min": -44.596527099609375,
      "activations/layer15_attention_weight_max": 44.4437370300293,
      "activations/layer15_attention_weight_min": -32.632171630859375,
      "activations/layer16_attention_weight_max": 43.11502456665039,
      "activations/layer16_attention_weight_min": -29.886093139648438,
      "activations/layer17_attention_weight_max": 39.42658615112305,
      "activations/layer17_attention_weight_min": -27.702945709228516,
      "activations/layer18_attention_weight_max": 34.10708236694336,
      "activations/layer18_attention_weight_min": -22.10492515563965,
      "activations/layer19_attention_weight_max": 43.41975784301758,
      "activations/layer19_attention_weight_min": -32.143348693847656,
      "activations/layer1_attention_weight_max": 16.358396530151367,
      "activations/layer1_attention_weight_min": -15.517284393310547,
      "activations/layer20_attention_weight_max": 39.02595901489258,
      "activations/layer20_attention_weight_min": -24.285694122314453,
      "activations/layer21_attention_weight_max": 34.335060119628906,
      "activations/layer21_attention_weight_min": -24.262338638305664,
      "activations/layer22_attention_weight_max": 57.22834014892578,
      "activations/layer22_attention_weight_min": -29.12867546081543,
      "activations/layer23_attention_weight_max": 40.06305694580078,
      "activations/layer23_attention_weight_min": -23.35702133178711,
      "activations/layer2_attention_weight_max": 33.159706115722656,
      "activations/layer2_attention_weight_min": -31.421855926513672,
      "activations/layer3_attention_weight_max": 95.97262573242188,
      "activations/layer3_attention_weight_min": -96.55160522460938,
      "activations/layer4_attention_weight_max": 133.5081024169922,
      "activations/layer4_attention_weight_min": -133.2581329345703,
      "activations/layer5_attention_weight_max": 75.28971862792969,
      "activations/layer5_attention_weight_min": -69.99907684326172,
      "activations/layer6_attention_weight_max": 55.24345016479492,
      "activations/layer6_attention_weight_min": -55.89775848388672,
      "activations/layer7_attention_weight_max": 94.78892517089844,
      "activations/layer7_attention_weight_min": -86.07720947265625,
      "activations/layer8_attention_weight_max": 42.19430160522461,
      "activations/layer8_attention_weight_min": -41.84702682495117,
      "activations/layer9_attention_weight_max": 38.45792007446289,
      "activations/layer9_attention_weight_min": -35.28757095336914,
      "epoch": 6.11,
      "learning_rate": 0.00011170378787878787,
      "loss": 2.8327,
      "step": 105200
    },
    {
      "activations/layer0_attention_weight_max": 16.022279739379883,
      "activations/layer0_attention_weight_min": -11.732247352600098,
      "activations/layer10_attention_weight_max": 40.32099533081055,
      "activations/layer10_attention_weight_min": -33.7452392578125,
      "activations/layer11_attention_weight_max": 43.19511032104492,
      "activations/layer11_attention_weight_min": -32.00812911987305,
      "activations/layer12_attention_weight_max": 33.96529769897461,
      "activations/layer12_attention_weight_min": -25.317731857299805,
      "activations/layer13_attention_weight_max": 83.39836883544922,
      "activations/layer13_attention_weight_min": -48.036075592041016,
      "activations/layer14_attention_weight_max": 99.75701141357422,
      "activations/layer14_attention_weight_min": -63.09181594848633,
      "activations/layer15_attention_weight_max": 65.78978729248047,
      "activations/layer15_attention_weight_min": -37.921756744384766,
      "activations/layer16_attention_weight_max": 50.69375991821289,
      "activations/layer16_attention_weight_min": -28.757041931152344,
      "activations/layer17_attention_weight_max": 45.05867385864258,
      "activations/layer17_attention_weight_min": -24.022315979003906,
      "activations/layer18_attention_weight_max": 38.57391357421875,
      "activations/layer18_attention_weight_min": -23.865800857543945,
      "activations/layer19_attention_weight_max": 52.82111358642578,
      "activations/layer19_attention_weight_min": -30.093290328979492,
      "activations/layer1_attention_weight_max": 16.51582908630371,
      "activations/layer1_attention_weight_min": -14.205350875854492,
      "activations/layer20_attention_weight_max": 44.995052337646484,
      "activations/layer20_attention_weight_min": -23.46333885192871,
      "activations/layer21_attention_weight_max": 35.13112258911133,
      "activations/layer21_attention_weight_min": -23.313940048217773,
      "activations/layer22_attention_weight_max": 58.02334976196289,
      "activations/layer22_attention_weight_min": -29.90096092224121,
      "activations/layer23_attention_weight_max": 40.94407653808594,
      "activations/layer23_attention_weight_min": -22.70233726501465,
      "activations/layer2_attention_weight_max": 31.966886520385742,
      "activations/layer2_attention_weight_min": -31.860088348388672,
      "activations/layer3_attention_weight_max": 87.28263092041016,
      "activations/layer3_attention_weight_min": -89.94180297851562,
      "activations/layer4_attention_weight_max": 129.04359436035156,
      "activations/layer4_attention_weight_min": -131.56893920898438,
      "activations/layer5_attention_weight_max": 70.92168426513672,
      "activations/layer5_attention_weight_min": -69.35515594482422,
      "activations/layer6_attention_weight_max": 54.77158737182617,
      "activations/layer6_attention_weight_min": -55.13892364501953,
      "activations/layer7_attention_weight_max": 95.5074691772461,
      "activations/layer7_attention_weight_min": -89.969482421875,
      "activations/layer8_attention_weight_max": 43.712303161621094,
      "activations/layer8_attention_weight_min": -41.393001556396484,
      "activations/layer9_attention_weight_max": 41.951786041259766,
      "activations/layer9_attention_weight_min": -44.95210266113281,
      "epoch": 6.12,
      "learning_rate": 0.00011168484848484847,
      "loss": 2.8226,
      "step": 105250
    },
    {
      "activations/layer0_attention_weight_max": 16.339468002319336,
      "activations/layer0_attention_weight_min": -12.510235786437988,
      "activations/layer10_attention_weight_max": 30.88841438293457,
      "activations/layer10_attention_weight_min": -29.861825942993164,
      "activations/layer11_attention_weight_max": 31.32080841064453,
      "activations/layer11_attention_weight_min": -27.497512817382812,
      "activations/layer12_attention_weight_max": 31.875267028808594,
      "activations/layer12_attention_weight_min": -27.093875885009766,
      "activations/layer13_attention_weight_max": 39.56235885620117,
      "activations/layer13_attention_weight_min": -33.03016662597656,
      "activations/layer14_attention_weight_max": 58.2763671875,
      "activations/layer14_attention_weight_min": -46.24927520751953,
      "activations/layer15_attention_weight_max": 38.23560333251953,
      "activations/layer15_attention_weight_min": -31.0689697265625,
      "activations/layer16_attention_weight_max": 35.83658218383789,
      "activations/layer16_attention_weight_min": -28.338024139404297,
      "activations/layer17_attention_weight_max": 35.53369140625,
      "activations/layer17_attention_weight_min": -25.761743545532227,
      "activations/layer18_attention_weight_max": 34.71943664550781,
      "activations/layer18_attention_weight_min": -22.22221565246582,
      "activations/layer19_attention_weight_max": 38.8505744934082,
      "activations/layer19_attention_weight_min": -30.069486618041992,
      "activations/layer1_attention_weight_max": 16.13030242919922,
      "activations/layer1_attention_weight_min": -18.439315795898438,
      "activations/layer20_attention_weight_max": 31.541791915893555,
      "activations/layer20_attention_weight_min": -23.128440856933594,
      "activations/layer21_attention_weight_max": 29.613601684570312,
      "activations/layer21_attention_weight_min": -21.224851608276367,
      "activations/layer22_attention_weight_max": 53.25188446044922,
      "activations/layer22_attention_weight_min": -28.93977928161621,
      "activations/layer23_attention_weight_max": 38.02281951904297,
      "activations/layer23_attention_weight_min": -22.753164291381836,
      "activations/layer2_attention_weight_max": 31.14189910888672,
      "activations/layer2_attention_weight_min": -29.705121994018555,
      "activations/layer3_attention_weight_max": 91.9035873413086,
      "activations/layer3_attention_weight_min": -90.11224365234375,
      "activations/layer4_attention_weight_max": 127.39263153076172,
      "activations/layer4_attention_weight_min": -118.24571990966797,
      "activations/layer5_attention_weight_max": 74.63606262207031,
      "activations/layer5_attention_weight_min": -69.82518005371094,
      "activations/layer6_attention_weight_max": 52.88950729370117,
      "activations/layer6_attention_weight_min": -54.34581756591797,
      "activations/layer7_attention_weight_max": 80.27655792236328,
      "activations/layer7_attention_weight_min": -82.83157348632812,
      "activations/layer8_attention_weight_max": 36.629432678222656,
      "activations/layer8_attention_weight_min": -38.535011291503906,
      "activations/layer9_attention_weight_max": 33.978553771972656,
      "activations/layer9_attention_weight_min": -33.38819885253906,
      "epoch": 6.12,
      "learning_rate": 0.00011166590909090909,
      "loss": 2.8275,
      "step": 105300
    },
    {
      "activations/layer0_attention_weight_max": 16.650949478149414,
      "activations/layer0_attention_weight_min": -12.796000480651855,
      "activations/layer10_attention_weight_max": 33.18050765991211,
      "activations/layer10_attention_weight_min": -29.940763473510742,
      "activations/layer11_attention_weight_max": 35.916465759277344,
      "activations/layer11_attention_weight_min": -27.554187774658203,
      "activations/layer12_attention_weight_max": 39.2984619140625,
      "activations/layer12_attention_weight_min": -26.037940979003906,
      "activations/layer13_attention_weight_max": 43.25757598876953,
      "activations/layer13_attention_weight_min": -32.12373352050781,
      "activations/layer14_attention_weight_max": 55.32709503173828,
      "activations/layer14_attention_weight_min": -43.13361358642578,
      "activations/layer15_attention_weight_max": 36.92183303833008,
      "activations/layer15_attention_weight_min": -30.264814376831055,
      "activations/layer16_attention_weight_max": 34.46168899536133,
      "activations/layer16_attention_weight_min": -28.315845489501953,
      "activations/layer17_attention_weight_max": 35.10338592529297,
      "activations/layer17_attention_weight_min": -26.445594787597656,
      "activations/layer18_attention_weight_max": 33.12836456298828,
      "activations/layer18_attention_weight_min": -21.317100524902344,
      "activations/layer19_attention_weight_max": 45.76852035522461,
      "activations/layer19_attention_weight_min": -30.356714248657227,
      "activations/layer1_attention_weight_max": 16.355501174926758,
      "activations/layer1_attention_weight_min": -13.1435546875,
      "activations/layer20_attention_weight_max": 34.24338150024414,
      "activations/layer20_attention_weight_min": -25.239665985107422,
      "activations/layer21_attention_weight_max": 34.488746643066406,
      "activations/layer21_attention_weight_min": -23.243118286132812,
      "activations/layer22_attention_weight_max": 49.66709518432617,
      "activations/layer22_attention_weight_min": -32.06675720214844,
      "activations/layer23_attention_weight_max": 38.84440994262695,
      "activations/layer23_attention_weight_min": -25.05770492553711,
      "activations/layer2_attention_weight_max": 33.26451873779297,
      "activations/layer2_attention_weight_min": -30.945348739624023,
      "activations/layer3_attention_weight_max": 90.7228775024414,
      "activations/layer3_attention_weight_min": -90.31100463867188,
      "activations/layer4_attention_weight_max": 124.7938003540039,
      "activations/layer4_attention_weight_min": -120.65349578857422,
      "activations/layer5_attention_weight_max": 69.27101135253906,
      "activations/layer5_attention_weight_min": -67.4212646484375,
      "activations/layer6_attention_weight_max": 51.90317153930664,
      "activations/layer6_attention_weight_min": -54.2364387512207,
      "activations/layer7_attention_weight_max": 81.46612548828125,
      "activations/layer7_attention_weight_min": -82.3565673828125,
      "activations/layer8_attention_weight_max": 38.222572326660156,
      "activations/layer8_attention_weight_min": -39.13139343261719,
      "activations/layer9_attention_weight_max": 35.09343719482422,
      "activations/layer9_attention_weight_min": -34.163230895996094,
      "epoch": 6.12,
      "learning_rate": 0.00011164696969696968,
      "loss": 2.8202,
      "step": 105350
    },
    {
      "activations/layer0_attention_weight_max": 16.671079635620117,
      "activations/layer0_attention_weight_min": -12.483860969543457,
      "activations/layer10_attention_weight_max": 35.37971878051758,
      "activations/layer10_attention_weight_min": -30.66762924194336,
      "activations/layer11_attention_weight_max": 31.72814178466797,
      "activations/layer11_attention_weight_min": -29.51007652282715,
      "activations/layer12_attention_weight_max": 27.776674270629883,
      "activations/layer12_attention_weight_min": -26.943172454833984,
      "activations/layer13_attention_weight_max": 46.53152084350586,
      "activations/layer13_attention_weight_min": -34.721168518066406,
      "activations/layer14_attention_weight_max": 60.28657531738281,
      "activations/layer14_attention_weight_min": -43.411075592041016,
      "activations/layer15_attention_weight_max": 37.86678695678711,
      "activations/layer15_attention_weight_min": -32.47407531738281,
      "activations/layer16_attention_weight_max": 38.64693832397461,
      "activations/layer16_attention_weight_min": -26.627872467041016,
      "activations/layer17_attention_weight_max": 39.52112579345703,
      "activations/layer17_attention_weight_min": -24.823583602905273,
      "activations/layer18_attention_weight_max": 33.85213088989258,
      "activations/layer18_attention_weight_min": -22.548011779785156,
      "activations/layer19_attention_weight_max": 41.58791732788086,
      "activations/layer19_attention_weight_min": -30.592477798461914,
      "activations/layer1_attention_weight_max": 16.971866607666016,
      "activations/layer1_attention_weight_min": -13.95508861541748,
      "activations/layer20_attention_weight_max": 35.79930114746094,
      "activations/layer20_attention_weight_min": -25.641162872314453,
      "activations/layer21_attention_weight_max": 32.29513168334961,
      "activations/layer21_attention_weight_min": -21.489065170288086,
      "activations/layer22_attention_weight_max": 51.38568115234375,
      "activations/layer22_attention_weight_min": -28.792980194091797,
      "activations/layer23_attention_weight_max": 37.03215789794922,
      "activations/layer23_attention_weight_min": -23.192584991455078,
      "activations/layer2_attention_weight_max": 31.835186004638672,
      "activations/layer2_attention_weight_min": -30.03828239440918,
      "activations/layer3_attention_weight_max": 90.23193359375,
      "activations/layer3_attention_weight_min": -90.02168273925781,
      "activations/layer4_attention_weight_max": 123.84883117675781,
      "activations/layer4_attention_weight_min": -122.6037826538086,
      "activations/layer5_attention_weight_max": 71.3314208984375,
      "activations/layer5_attention_weight_min": -68.21829223632812,
      "activations/layer6_attention_weight_max": 53.036434173583984,
      "activations/layer6_attention_weight_min": -53.12507247924805,
      "activations/layer7_attention_weight_max": 84.40056610107422,
      "activations/layer7_attention_weight_min": -87.3180923461914,
      "activations/layer8_attention_weight_max": 39.45497131347656,
      "activations/layer8_attention_weight_min": -41.08751678466797,
      "activations/layer9_attention_weight_max": 36.84340286254883,
      "activations/layer9_attention_weight_min": -34.64042282104492,
      "epoch": 6.12,
      "learning_rate": 0.00011162803030303029,
      "loss": 2.8558,
      "step": 105400
    },
    {
      "activations/layer0_attention_weight_max": 16.00774574279785,
      "activations/layer0_attention_weight_min": -12.529629707336426,
      "activations/layer10_attention_weight_max": 33.694496154785156,
      "activations/layer10_attention_weight_min": -32.85166549682617,
      "activations/layer11_attention_weight_max": 32.06537628173828,
      "activations/layer11_attention_weight_min": -29.151493072509766,
      "activations/layer12_attention_weight_max": 38.770442962646484,
      "activations/layer12_attention_weight_min": -29.055683135986328,
      "activations/layer13_attention_weight_max": 41.55792236328125,
      "activations/layer13_attention_weight_min": -32.10298156738281,
      "activations/layer14_attention_weight_max": 57.725093841552734,
      "activations/layer14_attention_weight_min": -43.38497543334961,
      "activations/layer15_attention_weight_max": 39.08262252807617,
      "activations/layer15_attention_weight_min": -28.56409454345703,
      "activations/layer16_attention_weight_max": 35.20156478881836,
      "activations/layer16_attention_weight_min": -29.9366512298584,
      "activations/layer17_attention_weight_max": 36.674739837646484,
      "activations/layer17_attention_weight_min": -25.840694427490234,
      "activations/layer18_attention_weight_max": 32.40412139892578,
      "activations/layer18_attention_weight_min": -19.723848342895508,
      "activations/layer19_attention_weight_max": 39.23019027709961,
      "activations/layer19_attention_weight_min": -32.8319206237793,
      "activations/layer1_attention_weight_max": 16.193140029907227,
      "activations/layer1_attention_weight_min": -12.62373161315918,
      "activations/layer20_attention_weight_max": 32.95332717895508,
      "activations/layer20_attention_weight_min": -24.629831314086914,
      "activations/layer21_attention_weight_max": 30.820152282714844,
      "activations/layer21_attention_weight_min": -24.153976440429688,
      "activations/layer22_attention_weight_max": 48.7656364440918,
      "activations/layer22_attention_weight_min": -27.55262565612793,
      "activations/layer23_attention_weight_max": 33.954132080078125,
      "activations/layer23_attention_weight_min": -21.34027862548828,
      "activations/layer2_attention_weight_max": 31.030363082885742,
      "activations/layer2_attention_weight_min": -32.1443977355957,
      "activations/layer3_attention_weight_max": 95.06986236572266,
      "activations/layer3_attention_weight_min": -96.01091003417969,
      "activations/layer4_attention_weight_max": 134.15118408203125,
      "activations/layer4_attention_weight_min": -129.2605438232422,
      "activations/layer5_attention_weight_max": 76.45878601074219,
      "activations/layer5_attention_weight_min": -73.17855834960938,
      "activations/layer6_attention_weight_max": 58.506866455078125,
      "activations/layer6_attention_weight_min": -57.832950592041016,
      "activations/layer7_attention_weight_max": 91.39134979248047,
      "activations/layer7_attention_weight_min": -88.4234848022461,
      "activations/layer8_attention_weight_max": 43.80070877075195,
      "activations/layer8_attention_weight_min": -40.24684524536133,
      "activations/layer9_attention_weight_max": 39.39826965332031,
      "activations/layer9_attention_weight_min": -34.88822555541992,
      "epoch": 6.13,
      "learning_rate": 0.0001116090909090909,
      "loss": 2.8297,
      "step": 105450
    },
    {
      "activations/layer0_attention_weight_max": 15.704190254211426,
      "activations/layer0_attention_weight_min": -12.438700675964355,
      "activations/layer10_attention_weight_max": 33.33809280395508,
      "activations/layer10_attention_weight_min": -29.98101043701172,
      "activations/layer11_attention_weight_max": 31.116010665893555,
      "activations/layer11_attention_weight_min": -27.097736358642578,
      "activations/layer12_attention_weight_max": 26.516841888427734,
      "activations/layer12_attention_weight_min": -25.641992568969727,
      "activations/layer13_attention_weight_max": 38.91200256347656,
      "activations/layer13_attention_weight_min": -34.274105072021484,
      "activations/layer14_attention_weight_max": 47.41139221191406,
      "activations/layer14_attention_weight_min": -39.4737663269043,
      "activations/layer15_attention_weight_max": 36.23784637451172,
      "activations/layer15_attention_weight_min": -28.072477340698242,
      "activations/layer16_attention_weight_max": 34.5584602355957,
      "activations/layer16_attention_weight_min": -26.675704956054688,
      "activations/layer17_attention_weight_max": 35.223533630371094,
      "activations/layer17_attention_weight_min": -24.194528579711914,
      "activations/layer18_attention_weight_max": 31.30607795715332,
      "activations/layer18_attention_weight_min": -22.155885696411133,
      "activations/layer19_attention_weight_max": 34.92362594604492,
      "activations/layer19_attention_weight_min": -30.04497718811035,
      "activations/layer1_attention_weight_max": 17.57038116455078,
      "activations/layer1_attention_weight_min": -13.786478996276855,
      "activations/layer20_attention_weight_max": 30.362009048461914,
      "activations/layer20_attention_weight_min": -25.844276428222656,
      "activations/layer21_attention_weight_max": 32.60810470581055,
      "activations/layer21_attention_weight_min": -22.33916664123535,
      "activations/layer22_attention_weight_max": 47.783050537109375,
      "activations/layer22_attention_weight_min": -29.89080047607422,
      "activations/layer23_attention_weight_max": 34.001670837402344,
      "activations/layer23_attention_weight_min": -20.975399017333984,
      "activations/layer2_attention_weight_max": 30.962947845458984,
      "activations/layer2_attention_weight_min": -29.815231323242188,
      "activations/layer3_attention_weight_max": 90.15884399414062,
      "activations/layer3_attention_weight_min": -87.6457290649414,
      "activations/layer4_attention_weight_max": 126.01830291748047,
      "activations/layer4_attention_weight_min": -121.80350494384766,
      "activations/layer5_attention_weight_max": 68.7001953125,
      "activations/layer5_attention_weight_min": -68.78941345214844,
      "activations/layer6_attention_weight_max": 52.1152229309082,
      "activations/layer6_attention_weight_min": -51.281898498535156,
      "activations/layer7_attention_weight_max": 84.59320068359375,
      "activations/layer7_attention_weight_min": -80.8721923828125,
      "activations/layer8_attention_weight_max": 38.37277603149414,
      "activations/layer8_attention_weight_min": -39.25629806518555,
      "activations/layer9_attention_weight_max": 34.98966979980469,
      "activations/layer9_attention_weight_min": -33.36689758300781,
      "epoch": 6.13,
      "learning_rate": 0.0001115901515151515,
      "loss": 2.8353,
      "step": 105500
    },
    {
      "activations/layer0_attention_weight_max": 15.571191787719727,
      "activations/layer0_attention_weight_min": -13.03711223602295,
      "activations/layer10_attention_weight_max": 33.049415588378906,
      "activations/layer10_attention_weight_min": -30.465707778930664,
      "activations/layer11_attention_weight_max": 32.47789764404297,
      "activations/layer11_attention_weight_min": -29.78878402709961,
      "activations/layer12_attention_weight_max": 59.496219635009766,
      "activations/layer12_attention_weight_min": -29.707639694213867,
      "activations/layer13_attention_weight_max": 55.41807174682617,
      "activations/layer13_attention_weight_min": -33.61861038208008,
      "activations/layer14_attention_weight_max": 63.488128662109375,
      "activations/layer14_attention_weight_min": -40.69942092895508,
      "activations/layer15_attention_weight_max": 43.43772506713867,
      "activations/layer15_attention_weight_min": -30.988311767578125,
      "activations/layer16_attention_weight_max": 43.17147445678711,
      "activations/layer16_attention_weight_min": -30.507678985595703,
      "activations/layer17_attention_weight_max": 41.40473175048828,
      "activations/layer17_attention_weight_min": -25.393329620361328,
      "activations/layer18_attention_weight_max": 33.12623977661133,
      "activations/layer18_attention_weight_min": -20.585710525512695,
      "activations/layer19_attention_weight_max": 46.003665924072266,
      "activations/layer19_attention_weight_min": -31.858810424804688,
      "activations/layer1_attention_weight_max": 16.846498489379883,
      "activations/layer1_attention_weight_min": -13.73694896697998,
      "activations/layer20_attention_weight_max": 35.22023010253906,
      "activations/layer20_attention_weight_min": -26.955766677856445,
      "activations/layer21_attention_weight_max": 35.83135986328125,
      "activations/layer21_attention_weight_min": -22.767929077148438,
      "activations/layer22_attention_weight_max": 52.15169143676758,
      "activations/layer22_attention_weight_min": -29.360214233398438,
      "activations/layer23_attention_weight_max": 38.444618225097656,
      "activations/layer23_attention_weight_min": -22.597471237182617,
      "activations/layer2_attention_weight_max": 31.83826446533203,
      "activations/layer2_attention_weight_min": -29.92021942138672,
      "activations/layer3_attention_weight_max": 90.4149398803711,
      "activations/layer3_attention_weight_min": -91.3124008178711,
      "activations/layer4_attention_weight_max": 130.19598388671875,
      "activations/layer4_attention_weight_min": -130.21405029296875,
      "activations/layer5_attention_weight_max": 70.92095184326172,
      "activations/layer5_attention_weight_min": -72.12353515625,
      "activations/layer6_attention_weight_max": 55.11811828613281,
      "activations/layer6_attention_weight_min": -58.08351516723633,
      "activations/layer7_attention_weight_max": 91.52571105957031,
      "activations/layer7_attention_weight_min": -91.8198013305664,
      "activations/layer8_attention_weight_max": 38.85049057006836,
      "activations/layer8_attention_weight_min": -38.68809127807617,
      "activations/layer9_attention_weight_max": 34.94698715209961,
      "activations/layer9_attention_weight_min": -34.01713180541992,
      "epoch": 6.13,
      "learning_rate": 0.00011157121212121211,
      "loss": 2.8174,
      "step": 105550
    },
    {
      "activations/layer0_attention_weight_max": 15.601125717163086,
      "activations/layer0_attention_weight_min": -12.136711120605469,
      "activations/layer10_attention_weight_max": 33.32268142700195,
      "activations/layer10_attention_weight_min": -32.5979118347168,
      "activations/layer11_attention_weight_max": 30.459609985351562,
      "activations/layer11_attention_weight_min": -29.79067039489746,
      "activations/layer12_attention_weight_max": 40.21007537841797,
      "activations/layer12_attention_weight_min": -27.53126335144043,
      "activations/layer13_attention_weight_max": 41.34600067138672,
      "activations/layer13_attention_weight_min": -31.380718231201172,
      "activations/layer14_attention_weight_max": 60.4837532043457,
      "activations/layer14_attention_weight_min": -47.147483825683594,
      "activations/layer15_attention_weight_max": 33.81515121459961,
      "activations/layer15_attention_weight_min": -27.126081466674805,
      "activations/layer16_attention_weight_max": 32.41566848754883,
      "activations/layer16_attention_weight_min": -30.442073822021484,
      "activations/layer17_attention_weight_max": 34.68276596069336,
      "activations/layer17_attention_weight_min": -26.15520668029785,
      "activations/layer18_attention_weight_max": 32.19530487060547,
      "activations/layer18_attention_weight_min": -20.465091705322266,
      "activations/layer19_attention_weight_max": 34.06139373779297,
      "activations/layer19_attention_weight_min": -30.613910675048828,
      "activations/layer1_attention_weight_max": 17.125930786132812,
      "activations/layer1_attention_weight_min": -14.207938194274902,
      "activations/layer20_attention_weight_max": 31.88460350036621,
      "activations/layer20_attention_weight_min": -23.545495986938477,
      "activations/layer21_attention_weight_max": 32.53046417236328,
      "activations/layer21_attention_weight_min": -22.377395629882812,
      "activations/layer22_attention_weight_max": 44.0960807800293,
      "activations/layer22_attention_weight_min": -30.654003143310547,
      "activations/layer23_attention_weight_max": 30.632110595703125,
      "activations/layer23_attention_weight_min": -24.33485221862793,
      "activations/layer2_attention_weight_max": 31.135990142822266,
      "activations/layer2_attention_weight_min": -30.460966110229492,
      "activations/layer3_attention_weight_max": 89.58304595947266,
      "activations/layer3_attention_weight_min": -92.5710678100586,
      "activations/layer4_attention_weight_max": 125.5096206665039,
      "activations/layer4_attention_weight_min": -123.59803771972656,
      "activations/layer5_attention_weight_max": 70.55975341796875,
      "activations/layer5_attention_weight_min": -68.23340606689453,
      "activations/layer6_attention_weight_max": 54.1151237487793,
      "activations/layer6_attention_weight_min": -52.13560104370117,
      "activations/layer7_attention_weight_max": 83.4501724243164,
      "activations/layer7_attention_weight_min": -85.79756164550781,
      "activations/layer8_attention_weight_max": 40.68378829956055,
      "activations/layer8_attention_weight_min": -39.612327575683594,
      "activations/layer9_attention_weight_max": 38.02607345581055,
      "activations/layer9_attention_weight_min": -35.09351348876953,
      "epoch": 6.14,
      "learning_rate": 0.00011155227272727272,
      "loss": 2.8249,
      "step": 105600
    },
    {
      "activations/layer0_attention_weight_max": 16.570112228393555,
      "activations/layer0_attention_weight_min": -12.739463806152344,
      "activations/layer10_attention_weight_max": 33.293731689453125,
      "activations/layer10_attention_weight_min": -32.44346237182617,
      "activations/layer11_attention_weight_max": 31.575231552124023,
      "activations/layer11_attention_weight_min": -30.839374542236328,
      "activations/layer12_attention_weight_max": 25.94567108154297,
      "activations/layer12_attention_weight_min": -28.49335289001465,
      "activations/layer13_attention_weight_max": 43.16674041748047,
      "activations/layer13_attention_weight_min": -32.04256820678711,
      "activations/layer14_attention_weight_max": 65.35701751708984,
      "activations/layer14_attention_weight_min": -49.10284423828125,
      "activations/layer15_attention_weight_max": 38.82792282104492,
      "activations/layer15_attention_weight_min": -30.31444549560547,
      "activations/layer16_attention_weight_max": 34.437198638916016,
      "activations/layer16_attention_weight_min": -28.972301483154297,
      "activations/layer17_attention_weight_max": 35.8537483215332,
      "activations/layer17_attention_weight_min": -24.807077407836914,
      "activations/layer18_attention_weight_max": 31.625215530395508,
      "activations/layer18_attention_weight_min": -20.612913131713867,
      "activations/layer19_attention_weight_max": 41.4819450378418,
      "activations/layer19_attention_weight_min": -29.70393180847168,
      "activations/layer1_attention_weight_max": 17.754728317260742,
      "activations/layer1_attention_weight_min": -13.75435733795166,
      "activations/layer20_attention_weight_max": 35.213783264160156,
      "activations/layer20_attention_weight_min": -24.821937561035156,
      "activations/layer21_attention_weight_max": 33.157867431640625,
      "activations/layer21_attention_weight_min": -20.858715057373047,
      "activations/layer22_attention_weight_max": 52.126529693603516,
      "activations/layer22_attention_weight_min": -27.5589656829834,
      "activations/layer23_attention_weight_max": 35.62042999267578,
      "activations/layer23_attention_weight_min": -22.3042049407959,
      "activations/layer2_attention_weight_max": 32.28031921386719,
      "activations/layer2_attention_weight_min": -31.344383239746094,
      "activations/layer3_attention_weight_max": 87.3819351196289,
      "activations/layer3_attention_weight_min": -89.99120330810547,
      "activations/layer4_attention_weight_max": 123.05118560791016,
      "activations/layer4_attention_weight_min": -127.45305633544922,
      "activations/layer5_attention_weight_max": 70.25668334960938,
      "activations/layer5_attention_weight_min": -69.64452362060547,
      "activations/layer6_attention_weight_max": 54.649288177490234,
      "activations/layer6_attention_weight_min": -54.49805450439453,
      "activations/layer7_attention_weight_max": 83.11150360107422,
      "activations/layer7_attention_weight_min": -91.58328247070312,
      "activations/layer8_attention_weight_max": 38.453895568847656,
      "activations/layer8_attention_weight_min": -42.3253288269043,
      "activations/layer9_attention_weight_max": 35.16395568847656,
      "activations/layer9_attention_weight_min": -41.37589645385742,
      "epoch": 6.14,
      "learning_rate": 0.00011153333333333332,
      "loss": 2.8192,
      "step": 105650
    },
    {
      "activations/layer0_attention_weight_max": 16.173357009887695,
      "activations/layer0_attention_weight_min": -11.447563171386719,
      "activations/layer10_attention_weight_max": 31.48815155029297,
      "activations/layer10_attention_weight_min": -31.69293212890625,
      "activations/layer11_attention_weight_max": 29.979032516479492,
      "activations/layer11_attention_weight_min": -28.10986328125,
      "activations/layer12_attention_weight_max": 34.3453483581543,
      "activations/layer12_attention_weight_min": -39.08506393432617,
      "activations/layer13_attention_weight_max": 50.21553039550781,
      "activations/layer13_attention_weight_min": -33.81572723388672,
      "activations/layer14_attention_weight_max": 59.691707611083984,
      "activations/layer14_attention_weight_min": -41.58441925048828,
      "activations/layer15_attention_weight_max": 40.2838020324707,
      "activations/layer15_attention_weight_min": -28.550853729248047,
      "activations/layer16_attention_weight_max": 34.87004470825195,
      "activations/layer16_attention_weight_min": -26.954593658447266,
      "activations/layer17_attention_weight_max": 40.44116973876953,
      "activations/layer17_attention_weight_min": -24.67471694946289,
      "activations/layer18_attention_weight_max": 33.44583511352539,
      "activations/layer18_attention_weight_min": -24.70207405090332,
      "activations/layer19_attention_weight_max": 42.28133773803711,
      "activations/layer19_attention_weight_min": -27.257709503173828,
      "activations/layer1_attention_weight_max": 17.328100204467773,
      "activations/layer1_attention_weight_min": -14.243913650512695,
      "activations/layer20_attention_weight_max": 36.20573806762695,
      "activations/layer20_attention_weight_min": -23.59613800048828,
      "activations/layer21_attention_weight_max": 31.145353317260742,
      "activations/layer21_attention_weight_min": -22.12862777709961,
      "activations/layer22_attention_weight_max": 53.19027328491211,
      "activations/layer22_attention_weight_min": -27.141695022583008,
      "activations/layer23_attention_weight_max": 33.377174377441406,
      "activations/layer23_attention_weight_min": -20.77128028869629,
      "activations/layer2_attention_weight_max": 31.431425094604492,
      "activations/layer2_attention_weight_min": -29.255123138427734,
      "activations/layer3_attention_weight_max": 85.81194305419922,
      "activations/layer3_attention_weight_min": -91.47787475585938,
      "activations/layer4_attention_weight_max": 123.95549774169922,
      "activations/layer4_attention_weight_min": -119.5217514038086,
      "activations/layer5_attention_weight_max": 66.65306091308594,
      "activations/layer5_attention_weight_min": -64.83804321289062,
      "activations/layer6_attention_weight_max": 51.00313949584961,
      "activations/layer6_attention_weight_min": -51.53481674194336,
      "activations/layer7_attention_weight_max": 87.68109130859375,
      "activations/layer7_attention_weight_min": -81.51981353759766,
      "activations/layer8_attention_weight_max": 39.1598014831543,
      "activations/layer8_attention_weight_min": -38.23537063598633,
      "activations/layer9_attention_weight_max": 33.54174041748047,
      "activations/layer9_attention_weight_min": -44.627254486083984,
      "epoch": 6.14,
      "learning_rate": 0.00011151439393939393,
      "loss": 2.8042,
      "step": 105700
    },
    {
      "activations/layer0_attention_weight_max": 16.222091674804688,
      "activations/layer0_attention_weight_min": -12.877306938171387,
      "activations/layer10_attention_weight_max": 34.26961135864258,
      "activations/layer10_attention_weight_min": -30.831552505493164,
      "activations/layer11_attention_weight_max": 31.15283203125,
      "activations/layer11_attention_weight_min": -28.145200729370117,
      "activations/layer12_attention_weight_max": 25.224449157714844,
      "activations/layer12_attention_weight_min": -26.493444442749023,
      "activations/layer13_attention_weight_max": 41.521507263183594,
      "activations/layer13_attention_weight_min": -31.143774032592773,
      "activations/layer14_attention_weight_max": 54.77217483520508,
      "activations/layer14_attention_weight_min": -41.47639083862305,
      "activations/layer15_attention_weight_max": 36.2585563659668,
      "activations/layer15_attention_weight_min": -28.699621200561523,
      "activations/layer16_attention_weight_max": 35.273197174072266,
      "activations/layer16_attention_weight_min": -28.206012725830078,
      "activations/layer17_attention_weight_max": 38.33414840698242,
      "activations/layer17_attention_weight_min": -24.75330352783203,
      "activations/layer18_attention_weight_max": 33.60688781738281,
      "activations/layer18_attention_weight_min": -19.865510940551758,
      "activations/layer19_attention_weight_max": 41.89768600463867,
      "activations/layer19_attention_weight_min": -31.384597778320312,
      "activations/layer1_attention_weight_max": 16.104520797729492,
      "activations/layer1_attention_weight_min": -14.496577262878418,
      "activations/layer20_attention_weight_max": 33.25569152832031,
      "activations/layer20_attention_weight_min": -24.280315399169922,
      "activations/layer21_attention_weight_max": 33.590919494628906,
      "activations/layer21_attention_weight_min": -23.74485969543457,
      "activations/layer22_attention_weight_max": 49.287349700927734,
      "activations/layer22_attention_weight_min": -28.755538940429688,
      "activations/layer23_attention_weight_max": 39.22588348388672,
      "activations/layer23_attention_weight_min": -23.33687400817871,
      "activations/layer2_attention_weight_max": 30.614871978759766,
      "activations/layer2_attention_weight_min": -32.05821228027344,
      "activations/layer3_attention_weight_max": 89.51343536376953,
      "activations/layer3_attention_weight_min": -89.83768463134766,
      "activations/layer4_attention_weight_max": 123.04686737060547,
      "activations/layer4_attention_weight_min": -121.6270980834961,
      "activations/layer5_attention_weight_max": 69.55400085449219,
      "activations/layer5_attention_weight_min": -65.41783142089844,
      "activations/layer6_attention_weight_max": 50.72114944458008,
      "activations/layer6_attention_weight_min": -54.41813278198242,
      "activations/layer7_attention_weight_max": 80.47065734863281,
      "activations/layer7_attention_weight_min": -83.32096099853516,
      "activations/layer8_attention_weight_max": 41.02030944824219,
      "activations/layer8_attention_weight_min": -39.000205993652344,
      "activations/layer9_attention_weight_max": 36.464027404785156,
      "activations/layer9_attention_weight_min": -33.792388916015625,
      "epoch": 6.14,
      "learning_rate": 0.00011149545454545454,
      "loss": 2.8409,
      "step": 105750
    },
    {
      "activations/layer0_attention_weight_max": 16.280803680419922,
      "activations/layer0_attention_weight_min": -11.517242431640625,
      "activations/layer10_attention_weight_max": 35.70668029785156,
      "activations/layer10_attention_weight_min": -31.21677017211914,
      "activations/layer11_attention_weight_max": 38.38323974609375,
      "activations/layer11_attention_weight_min": -32.15625762939453,
      "activations/layer12_attention_weight_max": 26.38890266418457,
      "activations/layer12_attention_weight_min": -28.63846778869629,
      "activations/layer13_attention_weight_max": 54.0628776550293,
      "activations/layer13_attention_weight_min": -34.87543869018555,
      "activations/layer14_attention_weight_max": 65.40779113769531,
      "activations/layer14_attention_weight_min": -46.88729476928711,
      "activations/layer15_attention_weight_max": 39.701377868652344,
      "activations/layer15_attention_weight_min": -30.61850357055664,
      "activations/layer16_attention_weight_max": 37.971736907958984,
      "activations/layer16_attention_weight_min": -28.032377243041992,
      "activations/layer17_attention_weight_max": 41.04353332519531,
      "activations/layer17_attention_weight_min": -25.133119583129883,
      "activations/layer18_attention_weight_max": 35.70325469970703,
      "activations/layer18_attention_weight_min": -22.052682876586914,
      "activations/layer19_attention_weight_max": 45.03179168701172,
      "activations/layer19_attention_weight_min": -30.157001495361328,
      "activations/layer1_attention_weight_max": 16.356903076171875,
      "activations/layer1_attention_weight_min": -11.497527122497559,
      "activations/layer20_attention_weight_max": 35.358116149902344,
      "activations/layer20_attention_weight_min": -22.736268997192383,
      "activations/layer21_attention_weight_max": 32.250160217285156,
      "activations/layer21_attention_weight_min": -21.441801071166992,
      "activations/layer22_attention_weight_max": 56.253700256347656,
      "activations/layer22_attention_weight_min": -27.27033042907715,
      "activations/layer23_attention_weight_max": 40.70831298828125,
      "activations/layer23_attention_weight_min": -21.93008041381836,
      "activations/layer2_attention_weight_max": 31.374481201171875,
      "activations/layer2_attention_weight_min": -30.83161163330078,
      "activations/layer3_attention_weight_max": 89.02909851074219,
      "activations/layer3_attention_weight_min": -88.78367614746094,
      "activations/layer4_attention_weight_max": 124.5518798828125,
      "activations/layer4_attention_weight_min": -121.96240234375,
      "activations/layer5_attention_weight_max": 70.10398864746094,
      "activations/layer5_attention_weight_min": -68.25355529785156,
      "activations/layer6_attention_weight_max": 53.445613861083984,
      "activations/layer6_attention_weight_min": -57.18515396118164,
      "activations/layer7_attention_weight_max": 88.96530151367188,
      "activations/layer7_attention_weight_min": -86.41796875,
      "activations/layer8_attention_weight_max": 40.45481491088867,
      "activations/layer8_attention_weight_min": -39.776058197021484,
      "activations/layer9_attention_weight_max": 38.2219123840332,
      "activations/layer9_attention_weight_min": -36.31637191772461,
      "epoch": 6.15,
      "learning_rate": 0.00011147651515151514,
      "loss": 2.8313,
      "step": 105800
    },
    {
      "activations/layer0_attention_weight_max": 15.753589630126953,
      "activations/layer0_attention_weight_min": -12.222844123840332,
      "activations/layer10_attention_weight_max": 34.33014678955078,
      "activations/layer10_attention_weight_min": -31.868587493896484,
      "activations/layer11_attention_weight_max": 33.261043548583984,
      "activations/layer11_attention_weight_min": -27.368453979492188,
      "activations/layer12_attention_weight_max": 35.94089126586914,
      "activations/layer12_attention_weight_min": -25.85903549194336,
      "activations/layer13_attention_weight_max": 47.044986724853516,
      "activations/layer13_attention_weight_min": -37.284278869628906,
      "activations/layer14_attention_weight_max": 60.629974365234375,
      "activations/layer14_attention_weight_min": -43.203697204589844,
      "activations/layer15_attention_weight_max": 36.08197021484375,
      "activations/layer15_attention_weight_min": -27.413089752197266,
      "activations/layer16_attention_weight_max": 33.27821731567383,
      "activations/layer16_attention_weight_min": -27.771556854248047,
      "activations/layer17_attention_weight_max": 34.17775344848633,
      "activations/layer17_attention_weight_min": -22.932571411132812,
      "activations/layer18_attention_weight_max": 31.645328521728516,
      "activations/layer18_attention_weight_min": -25.990453720092773,
      "activations/layer19_attention_weight_max": 41.04022216796875,
      "activations/layer19_attention_weight_min": -29.01893424987793,
      "activations/layer1_attention_weight_max": 16.857343673706055,
      "activations/layer1_attention_weight_min": -12.739816665649414,
      "activations/layer20_attention_weight_max": 31.00558090209961,
      "activations/layer20_attention_weight_min": -24.066646575927734,
      "activations/layer21_attention_weight_max": 29.443603515625,
      "activations/layer21_attention_weight_min": -21.47589683532715,
      "activations/layer22_attention_weight_max": 50.857295989990234,
      "activations/layer22_attention_weight_min": -27.438596725463867,
      "activations/layer23_attention_weight_max": 39.957637786865234,
      "activations/layer23_attention_weight_min": -19.79218864440918,
      "activations/layer2_attention_weight_max": 30.88941192626953,
      "activations/layer2_attention_weight_min": -28.762939453125,
      "activations/layer3_attention_weight_max": 88.73580169677734,
      "activations/layer3_attention_weight_min": -91.16847229003906,
      "activations/layer4_attention_weight_max": 122.4349136352539,
      "activations/layer4_attention_weight_min": -121.02650451660156,
      "activations/layer5_attention_weight_max": 70.03494262695312,
      "activations/layer5_attention_weight_min": -67.20879364013672,
      "activations/layer6_attention_weight_max": 54.61663055419922,
      "activations/layer6_attention_weight_min": -55.750308990478516,
      "activations/layer7_attention_weight_max": 85.82984924316406,
      "activations/layer7_attention_weight_min": -89.27494049072266,
      "activations/layer8_attention_weight_max": 42.8137092590332,
      "activations/layer8_attention_weight_min": -41.86686706542969,
      "activations/layer9_attention_weight_max": 36.8148307800293,
      "activations/layer9_attention_weight_min": -35.53550338745117,
      "epoch": 6.15,
      "learning_rate": 0.00011145795454545453,
      "loss": 2.8336,
      "step": 105850
    },
    {
      "activations/layer0_attention_weight_max": 15.05965518951416,
      "activations/layer0_attention_weight_min": -11.831235885620117,
      "activations/layer10_attention_weight_max": 34.393394470214844,
      "activations/layer10_attention_weight_min": -32.69179153442383,
      "activations/layer11_attention_weight_max": 33.59193801879883,
      "activations/layer11_attention_weight_min": -29.91742515563965,
      "activations/layer12_attention_weight_max": 23.587522506713867,
      "activations/layer12_attention_weight_min": -24.284330368041992,
      "activations/layer13_attention_weight_max": 41.06234359741211,
      "activations/layer13_attention_weight_min": -33.714881896972656,
      "activations/layer14_attention_weight_max": 58.58742141723633,
      "activations/layer14_attention_weight_min": -46.300575256347656,
      "activations/layer15_attention_weight_max": 36.676307678222656,
      "activations/layer15_attention_weight_min": -30.60564422607422,
      "activations/layer16_attention_weight_max": 38.499717712402344,
      "activations/layer16_attention_weight_min": -29.846988677978516,
      "activations/layer17_attention_weight_max": 32.862754821777344,
      "activations/layer17_attention_weight_min": -26.11713981628418,
      "activations/layer18_attention_weight_max": 31.59855842590332,
      "activations/layer18_attention_weight_min": -21.43791389465332,
      "activations/layer19_attention_weight_max": 39.1242561340332,
      "activations/layer19_attention_weight_min": -30.56589126586914,
      "activations/layer1_attention_weight_max": 16.89700698852539,
      "activations/layer1_attention_weight_min": -13.94762134552002,
      "activations/layer20_attention_weight_max": 29.480411529541016,
      "activations/layer20_attention_weight_min": -25.64535140991211,
      "activations/layer21_attention_weight_max": 30.482723236083984,
      "activations/layer21_attention_weight_min": -21.05160140991211,
      "activations/layer22_attention_weight_max": 45.421234130859375,
      "activations/layer22_attention_weight_min": -30.054500579833984,
      "activations/layer23_attention_weight_max": 34.990821838378906,
      "activations/layer23_attention_weight_min": -20.903955459594727,
      "activations/layer2_attention_weight_max": 31.101516723632812,
      "activations/layer2_attention_weight_min": -30.804813385009766,
      "activations/layer3_attention_weight_max": 88.32108306884766,
      "activations/layer3_attention_weight_min": -91.17845916748047,
      "activations/layer4_attention_weight_max": 123.92694091796875,
      "activations/layer4_attention_weight_min": -124.85853576660156,
      "activations/layer5_attention_weight_max": 68.62979888916016,
      "activations/layer5_attention_weight_min": -69.55842590332031,
      "activations/layer6_attention_weight_max": 52.931190490722656,
      "activations/layer6_attention_weight_min": -52.90150451660156,
      "activations/layer7_attention_weight_max": 85.48450469970703,
      "activations/layer7_attention_weight_min": -87.99580383300781,
      "activations/layer8_attention_weight_max": 40.275230407714844,
      "activations/layer8_attention_weight_min": -42.089683532714844,
      "activations/layer9_attention_weight_max": 36.8347282409668,
      "activations/layer9_attention_weight_min": -35.69001388549805,
      "epoch": 6.15,
      "learning_rate": 0.00011143901515151515,
      "loss": 2.8224,
      "step": 105900
    },
    {
      "activations/layer0_attention_weight_max": 15.094918251037598,
      "activations/layer0_attention_weight_min": -12.616978645324707,
      "activations/layer10_attention_weight_max": 36.778160095214844,
      "activations/layer10_attention_weight_min": -32.893829345703125,
      "activations/layer11_attention_weight_max": 34.95044708251953,
      "activations/layer11_attention_weight_min": -31.175697326660156,
      "activations/layer12_attention_weight_max": 25.12420654296875,
      "activations/layer12_attention_weight_min": -28.834070205688477,
      "activations/layer13_attention_weight_max": 49.599327087402344,
      "activations/layer13_attention_weight_min": -34.22427749633789,
      "activations/layer14_attention_weight_max": 64.67213439941406,
      "activations/layer14_attention_weight_min": -42.15986251831055,
      "activations/layer15_attention_weight_max": 44.4892463684082,
      "activations/layer15_attention_weight_min": -31.533191680908203,
      "activations/layer16_attention_weight_max": 37.59185028076172,
      "activations/layer16_attention_weight_min": -27.652679443359375,
      "activations/layer17_attention_weight_max": 41.78762435913086,
      "activations/layer17_attention_weight_min": -25.438087463378906,
      "activations/layer18_attention_weight_max": 31.82404899597168,
      "activations/layer18_attention_weight_min": -20.306310653686523,
      "activations/layer19_attention_weight_max": 40.19364929199219,
      "activations/layer19_attention_weight_min": -30.161657333374023,
      "activations/layer1_attention_weight_max": 18.281404495239258,
      "activations/layer1_attention_weight_min": -13.163841247558594,
      "activations/layer20_attention_weight_max": 32.59892654418945,
      "activations/layer20_attention_weight_min": -24.120731353759766,
      "activations/layer21_attention_weight_max": 36.10688018798828,
      "activations/layer21_attention_weight_min": -20.63506507873535,
      "activations/layer22_attention_weight_max": 61.182682037353516,
      "activations/layer22_attention_weight_min": -30.54463768005371,
      "activations/layer23_attention_weight_max": 37.14417266845703,
      "activations/layer23_attention_weight_min": -20.804553985595703,
      "activations/layer2_attention_weight_max": 32.21244812011719,
      "activations/layer2_attention_weight_min": -31.60456085205078,
      "activations/layer3_attention_weight_max": 89.67131805419922,
      "activations/layer3_attention_weight_min": -90.69294738769531,
      "activations/layer4_attention_weight_max": 126.7436294555664,
      "activations/layer4_attention_weight_min": -123.2806167602539,
      "activations/layer5_attention_weight_max": 72.05250549316406,
      "activations/layer5_attention_weight_min": -69.01676940917969,
      "activations/layer6_attention_weight_max": 53.5413932800293,
      "activations/layer6_attention_weight_min": -57.0428466796875,
      "activations/layer7_attention_weight_max": 84.08876037597656,
      "activations/layer7_attention_weight_min": -86.67528533935547,
      "activations/layer8_attention_weight_max": 41.8846549987793,
      "activations/layer8_attention_weight_min": -46.04840850830078,
      "activations/layer9_attention_weight_max": 39.20429992675781,
      "activations/layer9_attention_weight_min": -38.78029251098633,
      "epoch": 6.16,
      "learning_rate": 0.00011142007575757575,
      "loss": 2.8341,
      "step": 105950
    },
    {
      "activations/layer0_attention_weight_max": 15.587098121643066,
      "activations/layer0_attention_weight_min": -11.883702278137207,
      "activations/layer10_attention_weight_max": 35.349029541015625,
      "activations/layer10_attention_weight_min": -31.712383270263672,
      "activations/layer11_attention_weight_max": 33.51475143432617,
      "activations/layer11_attention_weight_min": -28.02609634399414,
      "activations/layer12_attention_weight_max": 22.451353073120117,
      "activations/layer12_attention_weight_min": -24.22295379638672,
      "activations/layer13_attention_weight_max": 40.9219856262207,
      "activations/layer13_attention_weight_min": -31.084192276000977,
      "activations/layer14_attention_weight_max": 65.0961685180664,
      "activations/layer14_attention_weight_min": -44.10251235961914,
      "activations/layer15_attention_weight_max": 37.06247329711914,
      "activations/layer15_attention_weight_min": -28.38968276977539,
      "activations/layer16_attention_weight_max": 35.40230941772461,
      "activations/layer16_attention_weight_min": -29.381925582885742,
      "activations/layer17_attention_weight_max": 34.38374328613281,
      "activations/layer17_attention_weight_min": -24.270771026611328,
      "activations/layer18_attention_weight_max": 31.33928871154785,
      "activations/layer18_attention_weight_min": -19.323043823242188,
      "activations/layer19_attention_weight_max": 38.04536819458008,
      "activations/layer19_attention_weight_min": -30.42302894592285,
      "activations/layer1_attention_weight_max": 18.540002822875977,
      "activations/layer1_attention_weight_min": -14.629902839660645,
      "activations/layer20_attention_weight_max": 34.28139877319336,
      "activations/layer20_attention_weight_min": -23.816625595092773,
      "activations/layer21_attention_weight_max": 30.33380699157715,
      "activations/layer21_attention_weight_min": -21.99808692932129,
      "activations/layer22_attention_weight_max": 50.24932861328125,
      "activations/layer22_attention_weight_min": -28.98218536376953,
      "activations/layer23_attention_weight_max": 35.06598663330078,
      "activations/layer23_attention_weight_min": -21.485519409179688,
      "activations/layer2_attention_weight_max": 31.2343807220459,
      "activations/layer2_attention_weight_min": -30.360998153686523,
      "activations/layer3_attention_weight_max": 87.43659973144531,
      "activations/layer3_attention_weight_min": -90.4864273071289,
      "activations/layer4_attention_weight_max": 123.6756362915039,
      "activations/layer4_attention_weight_min": -126.52381896972656,
      "activations/layer5_attention_weight_max": 72.23113250732422,
      "activations/layer5_attention_weight_min": -68.16921997070312,
      "activations/layer6_attention_weight_max": 54.66399383544922,
      "activations/layer6_attention_weight_min": -56.75496292114258,
      "activations/layer7_attention_weight_max": 90.44490814208984,
      "activations/layer7_attention_weight_min": -89.5942611694336,
      "activations/layer8_attention_weight_max": 41.94469451904297,
      "activations/layer8_attention_weight_min": -41.14751052856445,
      "activations/layer9_attention_weight_max": 37.143287658691406,
      "activations/layer9_attention_weight_min": -35.7993278503418,
      "epoch": 6.16,
      "learning_rate": 0.00011140113636363635,
      "loss": 2.8207,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.5543,
      "eval_samples_per_second": 501.972,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.5543,
      "eval_openwebtext_samples_per_second": 501.972,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_wikitext_loss": 3.01171875,
      "eval_wikitext_ppl": 20.322298876557472,
      "eval_wikitext_runtime": 1.9699,
      "eval_wikitext_samples_per_second": 231.484,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_lambada_loss": 2.5546875,
      "eval_lambada_ppl": 12.867278004795136,
      "eval_lambada_runtime": 9.5776,
      "eval_lambada_samples_per_second": 508.373,
      "step": 106000
    },
    {
      "activations/layer0_attention_weight_max": 16.23831558227539,
      "activations/layer0_attention_weight_min": -12.748514175415039,
      "activations/layer10_attention_weight_max": 40.46470260620117,
      "activations/layer10_attention_weight_min": -34.935157775878906,
      "activations/layer11_attention_weight_max": 42.360565185546875,
      "activations/layer11_attention_weight_min": -35.61743927001953,
      "activations/layer12_attention_weight_max": 49.60254669189453,
      "activations/layer12_attention_weight_min": -33.34792709350586,
      "activations/layer13_attention_weight_max": 59.89596939086914,
      "activations/layer13_attention_weight_min": -34.61336135864258,
      "activations/layer14_attention_weight_max": 57.233821868896484,
      "activations/layer14_attention_weight_min": -42.113746643066406,
      "activations/layer15_attention_weight_max": 51.418941497802734,
      "activations/layer15_attention_weight_min": -28.568954467773438,
      "activations/layer16_attention_weight_max": 35.76516342163086,
      "activations/layer16_attention_weight_min": -28.219905853271484,
      "activations/layer17_attention_weight_max": 33.79350280761719,
      "activations/layer17_attention_weight_min": -26.25406837463379,
      "activations/layer18_attention_weight_max": 32.006412506103516,
      "activations/layer18_attention_weight_min": -21.40912437438965,
      "activations/layer19_attention_weight_max": 36.313629150390625,
      "activations/layer19_attention_weight_min": -29.440841674804688,
      "activations/layer1_attention_weight_max": 18.04554557800293,
      "activations/layer1_attention_weight_min": -15.5413818359375,
      "activations/layer20_attention_weight_max": 30.7353572845459,
      "activations/layer20_attention_weight_min": -25.403919219970703,
      "activations/layer21_attention_weight_max": 29.993083953857422,
      "activations/layer21_attention_weight_min": -22.487751007080078,
      "activations/layer22_attention_weight_max": 45.54153823852539,
      "activations/layer22_attention_weight_min": -26.371335983276367,
      "activations/layer23_attention_weight_max": 32.34111785888672,
      "activations/layer23_attention_weight_min": -20.355228424072266,
      "activations/layer2_attention_weight_max": 33.244712829589844,
      "activations/layer2_attention_weight_min": -31.460067749023438,
      "activations/layer3_attention_weight_max": 90.9312515258789,
      "activations/layer3_attention_weight_min": -92.07179260253906,
      "activations/layer4_attention_weight_max": 128.0542449951172,
      "activations/layer4_attention_weight_min": -127.846923828125,
      "activations/layer5_attention_weight_max": 73.91923522949219,
      "activations/layer5_attention_weight_min": -73.04100036621094,
      "activations/layer6_attention_weight_max": 60.385040283203125,
      "activations/layer6_attention_weight_min": -55.10191345214844,
      "activations/layer7_attention_weight_max": 109.50949096679688,
      "activations/layer7_attention_weight_min": -84.30255126953125,
      "activations/layer8_attention_weight_max": 47.98131561279297,
      "activations/layer8_attention_weight_min": -42.09270095825195,
      "activations/layer9_attention_weight_max": 39.259464263916016,
      "activations/layer9_attention_weight_min": -37.47346115112305,
      "epoch": 6.16,
      "learning_rate": 0.00011138219696969697,
      "loss": 2.8195,
      "step": 106050
    },
    {
      "activations/layer0_attention_weight_max": 15.33700942993164,
      "activations/layer0_attention_weight_min": -12.619065284729004,
      "activations/layer10_attention_weight_max": 36.43207550048828,
      "activations/layer10_attention_weight_min": -34.665916442871094,
      "activations/layer11_attention_weight_max": 32.371299743652344,
      "activations/layer11_attention_weight_min": -28.750898361206055,
      "activations/layer12_attention_weight_max": 31.87392807006836,
      "activations/layer12_attention_weight_min": -24.09069061279297,
      "activations/layer13_attention_weight_max": 40.80141067504883,
      "activations/layer13_attention_weight_min": -33.06450271606445,
      "activations/layer14_attention_weight_max": 53.64057540893555,
      "activations/layer14_attention_weight_min": -42.17164611816406,
      "activations/layer15_attention_weight_max": 43.40510177612305,
      "activations/layer15_attention_weight_min": -29.38691520690918,
      "activations/layer16_attention_weight_max": 42.94174575805664,
      "activations/layer16_attention_weight_min": -27.299970626831055,
      "activations/layer17_attention_weight_max": 40.05000686645508,
      "activations/layer17_attention_weight_min": -25.677453994750977,
      "activations/layer18_attention_weight_max": 31.829586029052734,
      "activations/layer18_attention_weight_min": -19.114356994628906,
      "activations/layer19_attention_weight_max": 40.81411361694336,
      "activations/layer19_attention_weight_min": -28.139835357666016,
      "activations/layer1_attention_weight_max": 18.223176956176758,
      "activations/layer1_attention_weight_min": -14.863959312438965,
      "activations/layer20_attention_weight_max": 30.916412353515625,
      "activations/layer20_attention_weight_min": -22.52660369873047,
      "activations/layer21_attention_weight_max": 29.445907592773438,
      "activations/layer21_attention_weight_min": -19.600236892700195,
      "activations/layer22_attention_weight_max": 48.52299118041992,
      "activations/layer22_attention_weight_min": -26.915878295898438,
      "activations/layer23_attention_weight_max": 35.414588928222656,
      "activations/layer23_attention_weight_min": -20.623292922973633,
      "activations/layer2_attention_weight_max": 29.720823287963867,
      "activations/layer2_attention_weight_min": -29.039764404296875,
      "activations/layer3_attention_weight_max": 90.3516616821289,
      "activations/layer3_attention_weight_min": -91.38050079345703,
      "activations/layer4_attention_weight_max": 128.69175720214844,
      "activations/layer4_attention_weight_min": -126.05902099609375,
      "activations/layer5_attention_weight_max": 75.46200561523438,
      "activations/layer5_attention_weight_min": -71.75607299804688,
      "activations/layer6_attention_weight_max": 54.11143493652344,
      "activations/layer6_attention_weight_min": -56.370018005371094,
      "activations/layer7_attention_weight_max": 89.14500427246094,
      "activations/layer7_attention_weight_min": -90.02525329589844,
      "activations/layer8_attention_weight_max": 45.24738693237305,
      "activations/layer8_attention_weight_min": -43.669525146484375,
      "activations/layer9_attention_weight_max": 37.65834426879883,
      "activations/layer9_attention_weight_min": -36.80586624145508,
      "epoch": 6.17,
      "learning_rate": 0.00011136325757575755,
      "loss": 2.8323,
      "step": 106100
    },
    {
      "activations/layer0_attention_weight_max": 16.091188430786133,
      "activations/layer0_attention_weight_min": -11.392623901367188,
      "activations/layer10_attention_weight_max": 34.60030746459961,
      "activations/layer10_attention_weight_min": -31.75547218322754,
      "activations/layer11_attention_weight_max": 31.765913009643555,
      "activations/layer11_attention_weight_min": -29.562902450561523,
      "activations/layer12_attention_weight_max": 25.104602813720703,
      "activations/layer12_attention_weight_min": -28.640172958374023,
      "activations/layer13_attention_weight_max": 47.708473205566406,
      "activations/layer13_attention_weight_min": -33.969058990478516,
      "activations/layer14_attention_weight_max": 59.16925811767578,
      "activations/layer14_attention_weight_min": -44.97978591918945,
      "activations/layer15_attention_weight_max": 40.884361267089844,
      "activations/layer15_attention_weight_min": -28.739898681640625,
      "activations/layer16_attention_weight_max": 37.106353759765625,
      "activations/layer16_attention_weight_min": -29.321582794189453,
      "activations/layer17_attention_weight_max": 39.45640563964844,
      "activations/layer17_attention_weight_min": -24.727502822875977,
      "activations/layer18_attention_weight_max": 34.62259292602539,
      "activations/layer18_attention_weight_min": -23.537229537963867,
      "activations/layer19_attention_weight_max": 43.27519607543945,
      "activations/layer19_attention_weight_min": -32.0882568359375,
      "activations/layer1_attention_weight_max": 19.11602210998535,
      "activations/layer1_attention_weight_min": -15.236943244934082,
      "activations/layer20_attention_weight_max": 35.06730651855469,
      "activations/layer20_attention_weight_min": -24.514375686645508,
      "activations/layer21_attention_weight_max": 35.100746154785156,
      "activations/layer21_attention_weight_min": -22.145870208740234,
      "activations/layer22_attention_weight_max": 58.14072036743164,
      "activations/layer22_attention_weight_min": -31.03780746459961,
      "activations/layer23_attention_weight_max": 38.03388214111328,
      "activations/layer23_attention_weight_min": -24.3792781829834,
      "activations/layer2_attention_weight_max": 32.213653564453125,
      "activations/layer2_attention_weight_min": -31.407127380371094,
      "activations/layer3_attention_weight_max": 88.06352996826172,
      "activations/layer3_attention_weight_min": -91.05768585205078,
      "activations/layer4_attention_weight_max": 128.880859375,
      "activations/layer4_attention_weight_min": -124.37044525146484,
      "activations/layer5_attention_weight_max": 69.8239974975586,
      "activations/layer5_attention_weight_min": -68.76324462890625,
      "activations/layer6_attention_weight_max": 51.354347229003906,
      "activations/layer6_attention_weight_min": -58.32520294189453,
      "activations/layer7_attention_weight_max": 88.26490020751953,
      "activations/layer7_attention_weight_min": -91.3447036743164,
      "activations/layer8_attention_weight_max": 41.29505157470703,
      "activations/layer8_attention_weight_min": -39.62773895263672,
      "activations/layer9_attention_weight_max": 36.69606399536133,
      "activations/layer9_attention_weight_min": -34.826786041259766,
      "epoch": 6.17,
      "learning_rate": 0.00011134431818181817,
      "loss": 2.8366,
      "step": 106150
    },
    {
      "activations/layer0_attention_weight_max": 14.691388130187988,
      "activations/layer0_attention_weight_min": -12.076436996459961,
      "activations/layer10_attention_weight_max": 35.55482864379883,
      "activations/layer10_attention_weight_min": -29.955078125,
      "activations/layer11_attention_weight_max": 32.28361892700195,
      "activations/layer11_attention_weight_min": -28.50251007080078,
      "activations/layer12_attention_weight_max": 26.4454402923584,
      "activations/layer12_attention_weight_min": -25.218875885009766,
      "activations/layer13_attention_weight_max": 43.19330978393555,
      "activations/layer13_attention_weight_min": -32.92836380004883,
      "activations/layer14_attention_weight_max": 67.20008850097656,
      "activations/layer14_attention_weight_min": -45.83590316772461,
      "activations/layer15_attention_weight_max": 38.73957443237305,
      "activations/layer15_attention_weight_min": -28.66490364074707,
      "activations/layer16_attention_weight_max": 40.332672119140625,
      "activations/layer16_attention_weight_min": -27.858015060424805,
      "activations/layer17_attention_weight_max": 36.63185501098633,
      "activations/layer17_attention_weight_min": -24.50282096862793,
      "activations/layer18_attention_weight_max": 32.34988021850586,
      "activations/layer18_attention_weight_min": -20.826406478881836,
      "activations/layer19_attention_weight_max": 44.46735382080078,
      "activations/layer19_attention_weight_min": -29.6379451751709,
      "activations/layer1_attention_weight_max": 18.86826515197754,
      "activations/layer1_attention_weight_min": -14.92492389678955,
      "activations/layer20_attention_weight_max": 33.178443908691406,
      "activations/layer20_attention_weight_min": -25.7872257232666,
      "activations/layer21_attention_weight_max": 30.73187255859375,
      "activations/layer21_attention_weight_min": -21.04667854309082,
      "activations/layer22_attention_weight_max": 53.982337951660156,
      "activations/layer22_attention_weight_min": -28.069108963012695,
      "activations/layer23_attention_weight_max": 37.486541748046875,
      "activations/layer23_attention_weight_min": -21.672889709472656,
      "activations/layer2_attention_weight_max": 29.007793426513672,
      "activations/layer2_attention_weight_min": -29.06462860107422,
      "activations/layer3_attention_weight_max": 89.13599395751953,
      "activations/layer3_attention_weight_min": -87.10963439941406,
      "activations/layer4_attention_weight_max": 124.39472961425781,
      "activations/layer4_attention_weight_min": -117.34085845947266,
      "activations/layer5_attention_weight_max": 69.11692810058594,
      "activations/layer5_attention_weight_min": -65.293212890625,
      "activations/layer6_attention_weight_max": 51.1788444519043,
      "activations/layer6_attention_weight_min": -53.27639389038086,
      "activations/layer7_attention_weight_max": 86.6012191772461,
      "activations/layer7_attention_weight_min": -83.43363952636719,
      "activations/layer8_attention_weight_max": 38.862953186035156,
      "activations/layer8_attention_weight_min": -41.818416595458984,
      "activations/layer9_attention_weight_max": 35.366233825683594,
      "activations/layer9_attention_weight_min": -32.34543991088867,
      "epoch": 6.17,
      "learning_rate": 0.00011132537878787878,
      "loss": 2.8375,
      "step": 106200
    },
    {
      "activations/layer0_attention_weight_max": 16.11771011352539,
      "activations/layer0_attention_weight_min": -11.474631309509277,
      "activations/layer10_attention_weight_max": 37.502899169921875,
      "activations/layer10_attention_weight_min": -32.283302307128906,
      "activations/layer11_attention_weight_max": 33.93251037597656,
      "activations/layer11_attention_weight_min": -28.774641036987305,
      "activations/layer12_attention_weight_max": 21.549970626831055,
      "activations/layer12_attention_weight_min": -25.19749641418457,
      "activations/layer13_attention_weight_max": 40.31434631347656,
      "activations/layer13_attention_weight_min": -31.53081512451172,
      "activations/layer14_attention_weight_max": 61.89747619628906,
      "activations/layer14_attention_weight_min": -44.080230712890625,
      "activations/layer15_attention_weight_max": 40.35672378540039,
      "activations/layer15_attention_weight_min": -30.777814865112305,
      "activations/layer16_attention_weight_max": 34.710060119628906,
      "activations/layer16_attention_weight_min": -28.005107879638672,
      "activations/layer17_attention_weight_max": 35.99205017089844,
      "activations/layer17_attention_weight_min": -26.102493286132812,
      "activations/layer18_attention_weight_max": 33.30377960205078,
      "activations/layer18_attention_weight_min": -21.040742874145508,
      "activations/layer19_attention_weight_max": 37.72921371459961,
      "activations/layer19_attention_weight_min": -30.000564575195312,
      "activations/layer1_attention_weight_max": 17.350515365600586,
      "activations/layer1_attention_weight_min": -15.702186584472656,
      "activations/layer20_attention_weight_max": 32.830284118652344,
      "activations/layer20_attention_weight_min": -23.161527633666992,
      "activations/layer21_attention_weight_max": 34.14908218383789,
      "activations/layer21_attention_weight_min": -23.571727752685547,
      "activations/layer22_attention_weight_max": 49.03273010253906,
      "activations/layer22_attention_weight_min": -27.65078353881836,
      "activations/layer23_attention_weight_max": 35.948265075683594,
      "activations/layer23_attention_weight_min": -22.157438278198242,
      "activations/layer2_attention_weight_max": 31.00513458251953,
      "activations/layer2_attention_weight_min": -30.984905242919922,
      "activations/layer3_attention_weight_max": 90.49251556396484,
      "activations/layer3_attention_weight_min": -90.86200714111328,
      "activations/layer4_attention_weight_max": 129.3349609375,
      "activations/layer4_attention_weight_min": -127.97265625,
      "activations/layer5_attention_weight_max": 69.59819030761719,
      "activations/layer5_attention_weight_min": -66.776611328125,
      "activations/layer6_attention_weight_max": 52.17002868652344,
      "activations/layer6_attention_weight_min": -56.216644287109375,
      "activations/layer7_attention_weight_max": 85.19650268554688,
      "activations/layer7_attention_weight_min": -88.36679077148438,
      "activations/layer8_attention_weight_max": 39.74519729614258,
      "activations/layer8_attention_weight_min": -40.44325637817383,
      "activations/layer9_attention_weight_max": 36.34772872924805,
      "activations/layer9_attention_weight_min": -34.44505310058594,
      "epoch": 6.17,
      "learning_rate": 0.00011130643939393937,
      "loss": 2.8434,
      "step": 106250
    },
    {
      "activations/layer0_attention_weight_max": 16.571901321411133,
      "activations/layer0_attention_weight_min": -13.088248252868652,
      "activations/layer10_attention_weight_max": 31.7872371673584,
      "activations/layer10_attention_weight_min": -30.266098022460938,
      "activations/layer11_attention_weight_max": 30.195911407470703,
      "activations/layer11_attention_weight_min": -28.768592834472656,
      "activations/layer12_attention_weight_max": 22.002216339111328,
      "activations/layer12_attention_weight_min": -26.607622146606445,
      "activations/layer13_attention_weight_max": 40.5332145690918,
      "activations/layer13_attention_weight_min": -32.054237365722656,
      "activations/layer14_attention_weight_max": 57.65913009643555,
      "activations/layer14_attention_weight_min": -45.061370849609375,
      "activations/layer15_attention_weight_max": 36.4337043762207,
      "activations/layer15_attention_weight_min": -27.346899032592773,
      "activations/layer16_attention_weight_max": 33.373207092285156,
      "activations/layer16_attention_weight_min": -28.57792091369629,
      "activations/layer17_attention_weight_max": 36.35115051269531,
      "activations/layer17_attention_weight_min": -24.309795379638672,
      "activations/layer18_attention_weight_max": 37.62103271484375,
      "activations/layer18_attention_weight_min": -20.755260467529297,
      "activations/layer19_attention_weight_max": 33.261024475097656,
      "activations/layer19_attention_weight_min": -30.005908966064453,
      "activations/layer1_attention_weight_max": 18.187030792236328,
      "activations/layer1_attention_weight_min": -14.847125053405762,
      "activations/layer20_attention_weight_max": 27.19020652770996,
      "activations/layer20_attention_weight_min": -24.667818069458008,
      "activations/layer21_attention_weight_max": 32.78241729736328,
      "activations/layer21_attention_weight_min": -23.214279174804688,
      "activations/layer22_attention_weight_max": 46.23806381225586,
      "activations/layer22_attention_weight_min": -29.557092666625977,
      "activations/layer23_attention_weight_max": 32.70949172973633,
      "activations/layer23_attention_weight_min": -23.14057159423828,
      "activations/layer2_attention_weight_max": 30.943492889404297,
      "activations/layer2_attention_weight_min": -29.360593795776367,
      "activations/layer3_attention_weight_max": 83.0120849609375,
      "activations/layer3_attention_weight_min": -89.551025390625,
      "activations/layer4_attention_weight_max": 113.99342346191406,
      "activations/layer4_attention_weight_min": -119.74798583984375,
      "activations/layer5_attention_weight_max": 63.938697814941406,
      "activations/layer5_attention_weight_min": -67.05721282958984,
      "activations/layer6_attention_weight_max": 50.693790435791016,
      "activations/layer6_attention_weight_min": -54.47966003417969,
      "activations/layer7_attention_weight_max": 82.1515884399414,
      "activations/layer7_attention_weight_min": -84.49447631835938,
      "activations/layer8_attention_weight_max": 38.193511962890625,
      "activations/layer8_attention_weight_min": -41.359474182128906,
      "activations/layer9_attention_weight_max": 33.55173110961914,
      "activations/layer9_attention_weight_min": -34.81577682495117,
      "epoch": 6.18,
      "learning_rate": 0.00011128749999999999,
      "loss": 2.8434,
      "step": 106300
    },
    {
      "activations/layer0_attention_weight_max": 16.521678924560547,
      "activations/layer0_attention_weight_min": -12.189796447753906,
      "activations/layer10_attention_weight_max": 32.9542236328125,
      "activations/layer10_attention_weight_min": -30.8251953125,
      "activations/layer11_attention_weight_max": 40.01228332519531,
      "activations/layer11_attention_weight_min": -30.860044479370117,
      "activations/layer12_attention_weight_max": 28.235897064208984,
      "activations/layer12_attention_weight_min": -28.76630401611328,
      "activations/layer13_attention_weight_max": 47.417179107666016,
      "activations/layer13_attention_weight_min": -37.109127044677734,
      "activations/layer14_attention_weight_max": 55.55757522583008,
      "activations/layer14_attention_weight_min": -44.83073043823242,
      "activations/layer15_attention_weight_max": 49.152069091796875,
      "activations/layer15_attention_weight_min": -35.26791763305664,
      "activations/layer16_attention_weight_max": 37.9156608581543,
      "activations/layer16_attention_weight_min": -28.846967697143555,
      "activations/layer17_attention_weight_max": 36.629154205322266,
      "activations/layer17_attention_weight_min": -25.77247428894043,
      "activations/layer18_attention_weight_max": 29.699480056762695,
      "activations/layer18_attention_weight_min": -21.713163375854492,
      "activations/layer19_attention_weight_max": 37.60681915283203,
      "activations/layer19_attention_weight_min": -33.036293029785156,
      "activations/layer1_attention_weight_max": 16.351516723632812,
      "activations/layer1_attention_weight_min": -13.611222267150879,
      "activations/layer20_attention_weight_max": 35.46998977661133,
      "activations/layer20_attention_weight_min": -24.14289093017578,
      "activations/layer21_attention_weight_max": 31.100921630859375,
      "activations/layer21_attention_weight_min": -24.105920791625977,
      "activations/layer22_attention_weight_max": 47.88603591918945,
      "activations/layer22_attention_weight_min": -29.019420623779297,
      "activations/layer23_attention_weight_max": 37.075042724609375,
      "activations/layer23_attention_weight_min": -25.84441375732422,
      "activations/layer2_attention_weight_max": 31.757919311523438,
      "activations/layer2_attention_weight_min": -28.755598068237305,
      "activations/layer3_attention_weight_max": 89.77429962158203,
      "activations/layer3_attention_weight_min": -88.33404541015625,
      "activations/layer4_attention_weight_max": 123.12867736816406,
      "activations/layer4_attention_weight_min": -122.59864807128906,
      "activations/layer5_attention_weight_max": 68.3876953125,
      "activations/layer5_attention_weight_min": -67.170166015625,
      "activations/layer6_attention_weight_max": 54.65626907348633,
      "activations/layer6_attention_weight_min": -55.77781295776367,
      "activations/layer7_attention_weight_max": 94.38996887207031,
      "activations/layer7_attention_weight_min": -87.78450775146484,
      "activations/layer8_attention_weight_max": 40.44228744506836,
      "activations/layer8_attention_weight_min": -40.40670394897461,
      "activations/layer9_attention_weight_max": 34.07078552246094,
      "activations/layer9_attention_weight_min": -34.81464767456055,
      "epoch": 6.18,
      "learning_rate": 0.0001112685606060606,
      "loss": 2.8343,
      "step": 106350
    },
    {
      "activations/layer0_attention_weight_max": 14.626496315002441,
      "activations/layer0_attention_weight_min": -11.50556468963623,
      "activations/layer10_attention_weight_max": 47.883785247802734,
      "activations/layer10_attention_weight_min": -36.58513259887695,
      "activations/layer11_attention_weight_max": 44.64463424682617,
      "activations/layer11_attention_weight_min": -34.622230529785156,
      "activations/layer12_attention_weight_max": 42.2760124206543,
      "activations/layer12_attention_weight_min": -31.405677795410156,
      "activations/layer13_attention_weight_max": 68.77783203125,
      "activations/layer13_attention_weight_min": -42.80458450317383,
      "activations/layer14_attention_weight_max": 67.93840026855469,
      "activations/layer14_attention_weight_min": -44.88993453979492,
      "activations/layer15_attention_weight_max": 57.96272277832031,
      "activations/layer15_attention_weight_min": -33.257808685302734,
      "activations/layer16_attention_weight_max": 39.15880584716797,
      "activations/layer16_attention_weight_min": -29.279048919677734,
      "activations/layer17_attention_weight_max": 42.64313507080078,
      "activations/layer17_attention_weight_min": -25.3863582611084,
      "activations/layer18_attention_weight_max": 32.770408630371094,
      "activations/layer18_attention_weight_min": -24.568979263305664,
      "activations/layer19_attention_weight_max": 44.659488677978516,
      "activations/layer19_attention_weight_min": -30.95175552368164,
      "activations/layer1_attention_weight_max": 17.921777725219727,
      "activations/layer1_attention_weight_min": -15.716681480407715,
      "activations/layer20_attention_weight_max": 39.489044189453125,
      "activations/layer20_attention_weight_min": -25.902225494384766,
      "activations/layer21_attention_weight_max": 32.958717346191406,
      "activations/layer21_attention_weight_min": -22.508892059326172,
      "activations/layer22_attention_weight_max": 60.386741638183594,
      "activations/layer22_attention_weight_min": -29.192760467529297,
      "activations/layer23_attention_weight_max": 40.077781677246094,
      "activations/layer23_attention_weight_min": -20.617244720458984,
      "activations/layer2_attention_weight_max": 32.161678314208984,
      "activations/layer2_attention_weight_min": -30.432083129882812,
      "activations/layer3_attention_weight_max": 97.06575012207031,
      "activations/layer3_attention_weight_min": -96.59098052978516,
      "activations/layer4_attention_weight_max": 136.41270446777344,
      "activations/layer4_attention_weight_min": -133.90550231933594,
      "activations/layer5_attention_weight_max": 76.41178131103516,
      "activations/layer5_attention_weight_min": -72.83275604248047,
      "activations/layer6_attention_weight_max": 60.3334846496582,
      "activations/layer6_attention_weight_min": -58.91828536987305,
      "activations/layer7_attention_weight_max": 100.5582046508789,
      "activations/layer7_attention_weight_min": -93.82604217529297,
      "activations/layer8_attention_weight_max": 52.39699172973633,
      "activations/layer8_attention_weight_min": -47.68761444091797,
      "activations/layer9_attention_weight_max": 47.65907287597656,
      "activations/layer9_attention_weight_min": -39.707088470458984,
      "epoch": 6.18,
      "learning_rate": 0.0001112496212121212,
      "loss": 2.8237,
      "step": 106400
    },
    {
      "activations/layer0_attention_weight_max": 16.208580017089844,
      "activations/layer0_attention_weight_min": -12.481653213500977,
      "activations/layer10_attention_weight_max": 35.199378967285156,
      "activations/layer10_attention_weight_min": -30.944753646850586,
      "activations/layer11_attention_weight_max": 35.425331115722656,
      "activations/layer11_attention_weight_min": -28.75514030456543,
      "activations/layer12_attention_weight_max": 23.932706832885742,
      "activations/layer12_attention_weight_min": -25.1955623626709,
      "activations/layer13_attention_weight_max": 41.824337005615234,
      "activations/layer13_attention_weight_min": -32.151390075683594,
      "activations/layer14_attention_weight_max": 53.77723693847656,
      "activations/layer14_attention_weight_min": -38.42110061645508,
      "activations/layer15_attention_weight_max": 43.088382720947266,
      "activations/layer15_attention_weight_min": -29.79970359802246,
      "activations/layer16_attention_weight_max": 35.54838180541992,
      "activations/layer16_attention_weight_min": -28.005512237548828,
      "activations/layer17_attention_weight_max": 38.12520217895508,
      "activations/layer17_attention_weight_min": -25.100431442260742,
      "activations/layer18_attention_weight_max": 32.9154052734375,
      "activations/layer18_attention_weight_min": -20.47174644470215,
      "activations/layer19_attention_weight_max": 36.253929138183594,
      "activations/layer19_attention_weight_min": -30.435400009155273,
      "activations/layer1_attention_weight_max": 16.514923095703125,
      "activations/layer1_attention_weight_min": -15.043092727661133,
      "activations/layer20_attention_weight_max": 31.973514556884766,
      "activations/layer20_attention_weight_min": -23.98552131652832,
      "activations/layer21_attention_weight_max": 31.918411254882812,
      "activations/layer21_attention_weight_min": -21.496288299560547,
      "activations/layer22_attention_weight_max": 49.48491668701172,
      "activations/layer22_attention_weight_min": -27.982460021972656,
      "activations/layer23_attention_weight_max": 33.81559753417969,
      "activations/layer23_attention_weight_min": -22.174861907958984,
      "activations/layer2_attention_weight_max": 30.659849166870117,
      "activations/layer2_attention_weight_min": -29.63701629638672,
      "activations/layer3_attention_weight_max": 92.18905639648438,
      "activations/layer3_attention_weight_min": -95.01136016845703,
      "activations/layer4_attention_weight_max": 129.4303741455078,
      "activations/layer4_attention_weight_min": -126.14196014404297,
      "activations/layer5_attention_weight_max": 72.98002624511719,
      "activations/layer5_attention_weight_min": -68.99690246582031,
      "activations/layer6_attention_weight_max": 55.25946807861328,
      "activations/layer6_attention_weight_min": -55.21936798095703,
      "activations/layer7_attention_weight_max": 91.12101745605469,
      "activations/layer7_attention_weight_min": -86.4485855102539,
      "activations/layer8_attention_weight_max": 42.44590759277344,
      "activations/layer8_attention_weight_min": -42.214759826660156,
      "activations/layer9_attention_weight_max": 37.7240104675293,
      "activations/layer9_attention_weight_min": -34.187225341796875,
      "epoch": 6.19,
      "learning_rate": 0.0001112306818181818,
      "loss": 2.8367,
      "step": 106450
    },
    {
      "activations/layer0_attention_weight_max": 16.01750946044922,
      "activations/layer0_attention_weight_min": -12.612020492553711,
      "activations/layer10_attention_weight_max": 33.27830123901367,
      "activations/layer10_attention_weight_min": -29.533639907836914,
      "activations/layer11_attention_weight_max": 30.50680923461914,
      "activations/layer11_attention_weight_min": -29.460050582885742,
      "activations/layer12_attention_weight_max": 26.323261260986328,
      "activations/layer12_attention_weight_min": -28.733200073242188,
      "activations/layer13_attention_weight_max": 39.01896667480469,
      "activations/layer13_attention_weight_min": -34.61055374145508,
      "activations/layer14_attention_weight_max": 53.525630950927734,
      "activations/layer14_attention_weight_min": -40.67112350463867,
      "activations/layer15_attention_weight_max": 34.861839294433594,
      "activations/layer15_attention_weight_min": -28.043556213378906,
      "activations/layer16_attention_weight_max": 32.573280334472656,
      "activations/layer16_attention_weight_min": -28.191686630249023,
      "activations/layer17_attention_weight_max": 35.687347412109375,
      "activations/layer17_attention_weight_min": -23.573781967163086,
      "activations/layer18_attention_weight_max": 32.29135513305664,
      "activations/layer18_attention_weight_min": -20.805988311767578,
      "activations/layer19_attention_weight_max": 38.47085952758789,
      "activations/layer19_attention_weight_min": -28.825407028198242,
      "activations/layer1_attention_weight_max": 16.49347686767578,
      "activations/layer1_attention_weight_min": -14.701327323913574,
      "activations/layer20_attention_weight_max": 30.535676956176758,
      "activations/layer20_attention_weight_min": -22.27789878845215,
      "activations/layer21_attention_weight_max": 31.1645565032959,
      "activations/layer21_attention_weight_min": -22.18413543701172,
      "activations/layer22_attention_weight_max": 48.217594146728516,
      "activations/layer22_attention_weight_min": -26.44710350036621,
      "activations/layer23_attention_weight_max": 33.88482666015625,
      "activations/layer23_attention_weight_min": -22.324697494506836,
      "activations/layer2_attention_weight_max": 30.942901611328125,
      "activations/layer2_attention_weight_min": -29.90608024597168,
      "activations/layer3_attention_weight_max": 88.52315521240234,
      "activations/layer3_attention_weight_min": -89.3338623046875,
      "activations/layer4_attention_weight_max": 124.3578109741211,
      "activations/layer4_attention_weight_min": -126.88858795166016,
      "activations/layer5_attention_weight_max": 69.98509216308594,
      "activations/layer5_attention_weight_min": -66.3744125366211,
      "activations/layer6_attention_weight_max": 51.02519989013672,
      "activations/layer6_attention_weight_min": -51.15756607055664,
      "activations/layer7_attention_weight_max": 85.2863540649414,
      "activations/layer7_attention_weight_min": -83.15264129638672,
      "activations/layer8_attention_weight_max": 39.53520584106445,
      "activations/layer8_attention_weight_min": -36.89192581176758,
      "activations/layer9_attention_weight_max": 35.73616027832031,
      "activations/layer9_attention_weight_min": -37.01222229003906,
      "epoch": 6.19,
      "learning_rate": 0.00011121174242424242,
      "loss": 2.8274,
      "step": 106500
    },
    {
      "activations/layer0_attention_weight_max": 16.216203689575195,
      "activations/layer0_attention_weight_min": -10.981689453125,
      "activations/layer10_attention_weight_max": 31.42435073852539,
      "activations/layer10_attention_weight_min": -32.650535583496094,
      "activations/layer11_attention_weight_max": 31.497854232788086,
      "activations/layer11_attention_weight_min": -29.372421264648438,
      "activations/layer12_attention_weight_max": 25.657129287719727,
      "activations/layer12_attention_weight_min": -27.10127067565918,
      "activations/layer13_attention_weight_max": 42.10308837890625,
      "activations/layer13_attention_weight_min": -33.059654235839844,
      "activations/layer14_attention_weight_max": 51.80858612060547,
      "activations/layer14_attention_weight_min": -40.208805084228516,
      "activations/layer15_attention_weight_max": 37.114864349365234,
      "activations/layer15_attention_weight_min": -27.435592651367188,
      "activations/layer16_attention_weight_max": 35.500091552734375,
      "activations/layer16_attention_weight_min": -28.936491012573242,
      "activations/layer17_attention_weight_max": 34.96733093261719,
      "activations/layer17_attention_weight_min": -24.401470184326172,
      "activations/layer18_attention_weight_max": 30.516801834106445,
      "activations/layer18_attention_weight_min": -19.622121810913086,
      "activations/layer19_attention_weight_max": 39.84731674194336,
      "activations/layer19_attention_weight_min": -30.46522331237793,
      "activations/layer1_attention_weight_max": 17.7985782623291,
      "activations/layer1_attention_weight_min": -14.10147476196289,
      "activations/layer20_attention_weight_max": 30.678781509399414,
      "activations/layer20_attention_weight_min": -24.575603485107422,
      "activations/layer21_attention_weight_max": 29.383071899414062,
      "activations/layer21_attention_weight_min": -21.56076431274414,
      "activations/layer22_attention_weight_max": 49.97623825073242,
      "activations/layer22_attention_weight_min": -28.889856338500977,
      "activations/layer23_attention_weight_max": 35.668270111083984,
      "activations/layer23_attention_weight_min": -23.099464416503906,
      "activations/layer2_attention_weight_max": 32.69309616088867,
      "activations/layer2_attention_weight_min": -31.799423217773438,
      "activations/layer3_attention_weight_max": 95.69916534423828,
      "activations/layer3_attention_weight_min": -96.10790252685547,
      "activations/layer4_attention_weight_max": 133.19595336914062,
      "activations/layer4_attention_weight_min": -125.72985076904297,
      "activations/layer5_attention_weight_max": 71.86709594726562,
      "activations/layer5_attention_weight_min": -68.75234985351562,
      "activations/layer6_attention_weight_max": 55.02622985839844,
      "activations/layer6_attention_weight_min": -56.446197509765625,
      "activations/layer7_attention_weight_max": 85.72868347167969,
      "activations/layer7_attention_weight_min": -87.04461669921875,
      "activations/layer8_attention_weight_max": 39.23037338256836,
      "activations/layer8_attention_weight_min": -40.3430061340332,
      "activations/layer9_attention_weight_max": 37.10404586791992,
      "activations/layer9_attention_weight_min": -34.48850631713867,
      "epoch": 6.19,
      "learning_rate": 0.00011119280303030302,
      "loss": 2.8134,
      "step": 106550
    },
    {
      "activations/layer0_attention_weight_max": 15.952179908752441,
      "activations/layer0_attention_weight_min": -11.759930610656738,
      "activations/layer10_attention_weight_max": 31.94840431213379,
      "activations/layer10_attention_weight_min": -30.068740844726562,
      "activations/layer11_attention_weight_max": 30.962799072265625,
      "activations/layer11_attention_weight_min": -28.160186767578125,
      "activations/layer12_attention_weight_max": 24.855188369750977,
      "activations/layer12_attention_weight_min": -26.79937171936035,
      "activations/layer13_attention_weight_max": 48.16224670410156,
      "activations/layer13_attention_weight_min": -33.49696350097656,
      "activations/layer14_attention_weight_max": 55.3289909362793,
      "activations/layer14_attention_weight_min": -44.76872253417969,
      "activations/layer15_attention_weight_max": 37.118717193603516,
      "activations/layer15_attention_weight_min": -28.033933639526367,
      "activations/layer16_attention_weight_max": 35.24458694458008,
      "activations/layer16_attention_weight_min": -27.498497009277344,
      "activations/layer17_attention_weight_max": 34.72097396850586,
      "activations/layer17_attention_weight_min": -25.3644962310791,
      "activations/layer18_attention_weight_max": 33.75377655029297,
      "activations/layer18_attention_weight_min": -21.39486312866211,
      "activations/layer19_attention_weight_max": 38.43342590332031,
      "activations/layer19_attention_weight_min": -29.086578369140625,
      "activations/layer1_attention_weight_max": 16.506940841674805,
      "activations/layer1_attention_weight_min": -14.549400329589844,
      "activations/layer20_attention_weight_max": 30.861846923828125,
      "activations/layer20_attention_weight_min": -24.478837966918945,
      "activations/layer21_attention_weight_max": 28.839656829833984,
      "activations/layer21_attention_weight_min": -23.173059463500977,
      "activations/layer22_attention_weight_max": 45.86562728881836,
      "activations/layer22_attention_weight_min": -28.925094604492188,
      "activations/layer23_attention_weight_max": 35.918766021728516,
      "activations/layer23_attention_weight_min": -22.205581665039062,
      "activations/layer2_attention_weight_max": 30.31647300720215,
      "activations/layer2_attention_weight_min": -30.29134750366211,
      "activations/layer3_attention_weight_max": 87.84310150146484,
      "activations/layer3_attention_weight_min": -91.05111694335938,
      "activations/layer4_attention_weight_max": 120.45912170410156,
      "activations/layer4_attention_weight_min": -119.80948638916016,
      "activations/layer5_attention_weight_max": 66.31436157226562,
      "activations/layer5_attention_weight_min": -65.38723754882812,
      "activations/layer6_attention_weight_max": 50.93936538696289,
      "activations/layer6_attention_weight_min": -54.6341438293457,
      "activations/layer7_attention_weight_max": 81.19143676757812,
      "activations/layer7_attention_weight_min": -84.47511291503906,
      "activations/layer8_attention_weight_max": 39.671241760253906,
      "activations/layer8_attention_weight_min": -36.366676330566406,
      "activations/layer9_attention_weight_max": 34.25623321533203,
      "activations/layer9_attention_weight_min": -32.058406829833984,
      "epoch": 6.19,
      "learning_rate": 0.00011117386363636362,
      "loss": 2.8326,
      "step": 106600
    },
    {
      "activations/layer0_attention_weight_max": 15.395432472229004,
      "activations/layer0_attention_weight_min": -11.56092643737793,
      "activations/layer10_attention_weight_max": 36.529998779296875,
      "activations/layer10_attention_weight_min": -35.558807373046875,
      "activations/layer11_attention_weight_max": 31.96673583984375,
      "activations/layer11_attention_weight_min": -29.830520629882812,
      "activations/layer12_attention_weight_max": 23.953325271606445,
      "activations/layer12_attention_weight_min": -29.79743766784668,
      "activations/layer13_attention_weight_max": 38.77313232421875,
      "activations/layer13_attention_weight_min": -31.781641006469727,
      "activations/layer14_attention_weight_max": 51.82061767578125,
      "activations/layer14_attention_weight_min": -37.683868408203125,
      "activations/layer15_attention_weight_max": 37.6822509765625,
      "activations/layer15_attention_weight_min": -26.70880699157715,
      "activations/layer16_attention_weight_max": 33.58763885498047,
      "activations/layer16_attention_weight_min": -25.751489639282227,
      "activations/layer17_attention_weight_max": 34.477325439453125,
      "activations/layer17_attention_weight_min": -23.44424057006836,
      "activations/layer18_attention_weight_max": 29.267568588256836,
      "activations/layer18_attention_weight_min": -20.42180824279785,
      "activations/layer19_attention_weight_max": 39.23966598510742,
      "activations/layer19_attention_weight_min": -30.794591903686523,
      "activations/layer1_attention_weight_max": 17.615713119506836,
      "activations/layer1_attention_weight_min": -12.837593078613281,
      "activations/layer20_attention_weight_max": 30.39760398864746,
      "activations/layer20_attention_weight_min": -21.992576599121094,
      "activations/layer21_attention_weight_max": 30.889822006225586,
      "activations/layer21_attention_weight_min": -21.621429443359375,
      "activations/layer22_attention_weight_max": 43.78848648071289,
      "activations/layer22_attention_weight_min": -28.10759735107422,
      "activations/layer23_attention_weight_max": 32.83905029296875,
      "activations/layer23_attention_weight_min": -23.435726165771484,
      "activations/layer2_attention_weight_max": 31.876501083374023,
      "activations/layer2_attention_weight_min": -32.145896911621094,
      "activations/layer3_attention_weight_max": 99.82032775878906,
      "activations/layer3_attention_weight_min": -98.36319732666016,
      "activations/layer4_attention_weight_max": 139.2308349609375,
      "activations/layer4_attention_weight_min": -133.48985290527344,
      "activations/layer5_attention_weight_max": 73.1904525756836,
      "activations/layer5_attention_weight_min": -73.43109130859375,
      "activations/layer6_attention_weight_max": 52.660709381103516,
      "activations/layer6_attention_weight_min": -57.69072341918945,
      "activations/layer7_attention_weight_max": 89.04960632324219,
      "activations/layer7_attention_weight_min": -97.23518371582031,
      "activations/layer8_attention_weight_max": 42.72521209716797,
      "activations/layer8_attention_weight_min": -46.21290588378906,
      "activations/layer9_attention_weight_max": 39.30527114868164,
      "activations/layer9_attention_weight_min": -40.3201789855957,
      "epoch": 6.2,
      "learning_rate": 0.00011115492424242423,
      "loss": 2.8202,
      "step": 106650
    },
    {
      "activations/layer0_attention_weight_max": 15.902111053466797,
      "activations/layer0_attention_weight_min": -11.686197280883789,
      "activations/layer10_attention_weight_max": 33.51344680786133,
      "activations/layer10_attention_weight_min": -30.94169044494629,
      "activations/layer11_attention_weight_max": 33.46694564819336,
      "activations/layer11_attention_weight_min": -27.187833786010742,
      "activations/layer12_attention_weight_max": 23.890602111816406,
      "activations/layer12_attention_weight_min": -23.93143653869629,
      "activations/layer13_attention_weight_max": 42.17144775390625,
      "activations/layer13_attention_weight_min": -34.56770324707031,
      "activations/layer14_attention_weight_max": 56.83833312988281,
      "activations/layer14_attention_weight_min": -44.22468566894531,
      "activations/layer15_attention_weight_max": 40.044715881347656,
      "activations/layer15_attention_weight_min": -27.34402084350586,
      "activations/layer16_attention_weight_max": 38.69801712036133,
      "activations/layer16_attention_weight_min": -27.14889144897461,
      "activations/layer17_attention_weight_max": 42.59856414794922,
      "activations/layer17_attention_weight_min": -24.792152404785156,
      "activations/layer18_attention_weight_max": 36.95423126220703,
      "activations/layer18_attention_weight_min": -20.423688888549805,
      "activations/layer19_attention_weight_max": 38.301002502441406,
      "activations/layer19_attention_weight_min": -30.4626407623291,
      "activations/layer1_attention_weight_max": 16.600360870361328,
      "activations/layer1_attention_weight_min": -14.453338623046875,
      "activations/layer20_attention_weight_max": 40.34623718261719,
      "activations/layer20_attention_weight_min": -25.719205856323242,
      "activations/layer21_attention_weight_max": 37.24541091918945,
      "activations/layer21_attention_weight_min": -22.1417236328125,
      "activations/layer22_attention_weight_max": 48.55137634277344,
      "activations/layer22_attention_weight_min": -29.261613845825195,
      "activations/layer23_attention_weight_max": 35.922584533691406,
      "activations/layer23_attention_weight_min": -20.95029067993164,
      "activations/layer2_attention_weight_max": 32.89656066894531,
      "activations/layer2_attention_weight_min": -31.0718994140625,
      "activations/layer3_attention_weight_max": 91.44715881347656,
      "activations/layer3_attention_weight_min": -94.57398223876953,
      "activations/layer4_attention_weight_max": 125.6915512084961,
      "activations/layer4_attention_weight_min": -124.8239517211914,
      "activations/layer5_attention_weight_max": 70.62348175048828,
      "activations/layer5_attention_weight_min": -67.87576293945312,
      "activations/layer6_attention_weight_max": 53.54523849487305,
      "activations/layer6_attention_weight_min": -57.175418853759766,
      "activations/layer7_attention_weight_max": 86.66171264648438,
      "activations/layer7_attention_weight_min": -90.92753601074219,
      "activations/layer8_attention_weight_max": 45.435523986816406,
      "activations/layer8_attention_weight_min": -40.35047149658203,
      "activations/layer9_attention_weight_max": 35.31492614746094,
      "activations/layer9_attention_weight_min": -33.66747283935547,
      "epoch": 6.2,
      "learning_rate": 0.00011113598484848484,
      "loss": 2.8244,
      "step": 106700
    },
    {
      "activations/layer0_attention_weight_max": 15.116915702819824,
      "activations/layer0_attention_weight_min": -13.150083541870117,
      "activations/layer10_attention_weight_max": 33.399993896484375,
      "activations/layer10_attention_weight_min": -32.10536575317383,
      "activations/layer11_attention_weight_max": 37.193580627441406,
      "activations/layer11_attention_weight_min": -30.434059143066406,
      "activations/layer12_attention_weight_max": 28.375900268554688,
      "activations/layer12_attention_weight_min": -28.944190979003906,
      "activations/layer13_attention_weight_max": 37.2098503112793,
      "activations/layer13_attention_weight_min": -32.39834213256836,
      "activations/layer14_attention_weight_max": 58.25898361206055,
      "activations/layer14_attention_weight_min": -48.224517822265625,
      "activations/layer15_attention_weight_max": 34.935359954833984,
      "activations/layer15_attention_weight_min": -28.712385177612305,
      "activations/layer16_attention_weight_max": 31.495182037353516,
      "activations/layer16_attention_weight_min": -28.1483154296875,
      "activations/layer17_attention_weight_max": 34.65076446533203,
      "activations/layer17_attention_weight_min": -25.61335563659668,
      "activations/layer18_attention_weight_max": 28.344064712524414,
      "activations/layer18_attention_weight_min": -22.766754150390625,
      "activations/layer19_attention_weight_max": 38.87607955932617,
      "activations/layer19_attention_weight_min": -29.723093032836914,
      "activations/layer1_attention_weight_max": 16.06462287902832,
      "activations/layer1_attention_weight_min": -14.81997299194336,
      "activations/layer20_attention_weight_max": 30.356582641601562,
      "activations/layer20_attention_weight_min": -25.0439395904541,
      "activations/layer21_attention_weight_max": 28.959503173828125,
      "activations/layer21_attention_weight_min": -20.4882755279541,
      "activations/layer22_attention_weight_max": 45.228458404541016,
      "activations/layer22_attention_weight_min": -28.576501846313477,
      "activations/layer23_attention_weight_max": 33.30870056152344,
      "activations/layer23_attention_weight_min": -22.952571868896484,
      "activations/layer2_attention_weight_max": 31.27895164489746,
      "activations/layer2_attention_weight_min": -30.30348777770996,
      "activations/layer3_attention_weight_max": 88.96017456054688,
      "activations/layer3_attention_weight_min": -93.3416976928711,
      "activations/layer4_attention_weight_max": 126.92366790771484,
      "activations/layer4_attention_weight_min": -132.66355895996094,
      "activations/layer5_attention_weight_max": 69.11863708496094,
      "activations/layer5_attention_weight_min": -68.52761840820312,
      "activations/layer6_attention_weight_max": 53.24913787841797,
      "activations/layer6_attention_weight_min": -56.06080627441406,
      "activations/layer7_attention_weight_max": 87.15784454345703,
      "activations/layer7_attention_weight_min": -93.15945434570312,
      "activations/layer8_attention_weight_max": 39.7630500793457,
      "activations/layer8_attention_weight_min": -38.79817581176758,
      "activations/layer9_attention_weight_max": 36.380943298339844,
      "activations/layer9_attention_weight_min": -33.614952087402344,
      "epoch": 6.2,
      "learning_rate": 0.00011111704545454546,
      "loss": 2.8177,
      "step": 106750
    },
    {
      "activations/layer0_attention_weight_max": 16.18421173095703,
      "activations/layer0_attention_weight_min": -11.466510772705078,
      "activations/layer10_attention_weight_max": 32.545936584472656,
      "activations/layer10_attention_weight_min": -29.304931640625,
      "activations/layer11_attention_weight_max": 32.633331298828125,
      "activations/layer11_attention_weight_min": -27.714691162109375,
      "activations/layer12_attention_weight_max": 22.26325035095215,
      "activations/layer12_attention_weight_min": -25.00611114501953,
      "activations/layer13_attention_weight_max": 41.50886917114258,
      "activations/layer13_attention_weight_min": -32.05840301513672,
      "activations/layer14_attention_weight_max": 54.658077239990234,
      "activations/layer14_attention_weight_min": -42.50694274902344,
      "activations/layer15_attention_weight_max": 33.69007873535156,
      "activations/layer15_attention_weight_min": -28.563566207885742,
      "activations/layer16_attention_weight_max": 33.88249206542969,
      "activations/layer16_attention_weight_min": -26.936424255371094,
      "activations/layer17_attention_weight_max": 30.44914436340332,
      "activations/layer17_attention_weight_min": -25.002521514892578,
      "activations/layer18_attention_weight_max": 30.137887954711914,
      "activations/layer18_attention_weight_min": -19.457271575927734,
      "activations/layer19_attention_weight_max": 39.860286712646484,
      "activations/layer19_attention_weight_min": -28.640417098999023,
      "activations/layer1_attention_weight_max": 15.952717781066895,
      "activations/layer1_attention_weight_min": -13.910775184631348,
      "activations/layer20_attention_weight_max": 30.43440818786621,
      "activations/layer20_attention_weight_min": -22.73175048828125,
      "activations/layer21_attention_weight_max": 28.09468650817871,
      "activations/layer21_attention_weight_min": -21.53214454650879,
      "activations/layer22_attention_weight_max": 44.6688346862793,
      "activations/layer22_attention_weight_min": -27.326292037963867,
      "activations/layer23_attention_weight_max": 32.048095703125,
      "activations/layer23_attention_weight_min": -21.17472267150879,
      "activations/layer2_attention_weight_max": 31.574771881103516,
      "activations/layer2_attention_weight_min": -30.799034118652344,
      "activations/layer3_attention_weight_max": 87.5383529663086,
      "activations/layer3_attention_weight_min": -92.27196502685547,
      "activations/layer4_attention_weight_max": 120.84773254394531,
      "activations/layer4_attention_weight_min": -127.44597625732422,
      "activations/layer5_attention_weight_max": 67.01094818115234,
      "activations/layer5_attention_weight_min": -68.40290069580078,
      "activations/layer6_attention_weight_max": 50.27743911743164,
      "activations/layer6_attention_weight_min": -55.46735382080078,
      "activations/layer7_attention_weight_max": 80.66680908203125,
      "activations/layer7_attention_weight_min": -85.12467956542969,
      "activations/layer8_attention_weight_max": 37.45094680786133,
      "activations/layer8_attention_weight_min": -37.089637756347656,
      "activations/layer9_attention_weight_max": 33.3553466796875,
      "activations/layer9_attention_weight_min": -32.36017990112305,
      "epoch": 6.21,
      "learning_rate": 0.00011109810606060604,
      "loss": 2.8232,
      "step": 106800
    },
    {
      "activations/layer0_attention_weight_max": 15.500261306762695,
      "activations/layer0_attention_weight_min": -11.169800758361816,
      "activations/layer10_attention_weight_max": 35.470863342285156,
      "activations/layer10_attention_weight_min": -31.759475708007812,
      "activations/layer11_attention_weight_max": 32.28776168823242,
      "activations/layer11_attention_weight_min": -30.136241912841797,
      "activations/layer12_attention_weight_max": 31.4224910736084,
      "activations/layer12_attention_weight_min": -30.05518913269043,
      "activations/layer13_attention_weight_max": 43.50810241699219,
      "activations/layer13_attention_weight_min": -34.04940414428711,
      "activations/layer14_attention_weight_max": 59.85593795776367,
      "activations/layer14_attention_weight_min": -43.263248443603516,
      "activations/layer15_attention_weight_max": 38.89537048339844,
      "activations/layer15_attention_weight_min": -28.872180938720703,
      "activations/layer16_attention_weight_max": 33.51110076904297,
      "activations/layer16_attention_weight_min": -26.982046127319336,
      "activations/layer17_attention_weight_max": 36.7697639465332,
      "activations/layer17_attention_weight_min": -24.663522720336914,
      "activations/layer18_attention_weight_max": 35.747039794921875,
      "activations/layer18_attention_weight_min": -20.4467830657959,
      "activations/layer19_attention_weight_max": 36.14046859741211,
      "activations/layer19_attention_weight_min": -29.991491317749023,
      "activations/layer1_attention_weight_max": 16.66386604309082,
      "activations/layer1_attention_weight_min": -14.115585327148438,
      "activations/layer20_attention_weight_max": 30.062244415283203,
      "activations/layer20_attention_weight_min": -23.77581024169922,
      "activations/layer21_attention_weight_max": 33.700626373291016,
      "activations/layer21_attention_weight_min": -20.874582290649414,
      "activations/layer22_attention_weight_max": 49.82813262939453,
      "activations/layer22_attention_weight_min": -29.418792724609375,
      "activations/layer23_attention_weight_max": 36.17808532714844,
      "activations/layer23_attention_weight_min": -22.01093292236328,
      "activations/layer2_attention_weight_max": 31.074363708496094,
      "activations/layer2_attention_weight_min": -29.62010955810547,
      "activations/layer3_attention_weight_max": 91.70751953125,
      "activations/layer3_attention_weight_min": -90.1484146118164,
      "activations/layer4_attention_weight_max": 125.5099105834961,
      "activations/layer4_attention_weight_min": -124.0135726928711,
      "activations/layer5_attention_weight_max": 70.1673355102539,
      "activations/layer5_attention_weight_min": -70.12100982666016,
      "activations/layer6_attention_weight_max": 54.14939498901367,
      "activations/layer6_attention_weight_min": -56.044044494628906,
      "activations/layer7_attention_weight_max": 85.6390380859375,
      "activations/layer7_attention_weight_min": -84.05667877197266,
      "activations/layer8_attention_weight_max": 39.90938949584961,
      "activations/layer8_attention_weight_min": -41.64961624145508,
      "activations/layer9_attention_weight_max": 39.19438171386719,
      "activations/layer9_attention_weight_min": -34.92998123168945,
      "epoch": 6.21,
      "learning_rate": 0.00011107916666666666,
      "loss": 2.8383,
      "step": 106850
    },
    {
      "activations/layer0_attention_weight_max": 15.327255249023438,
      "activations/layer0_attention_weight_min": -11.367451667785645,
      "activations/layer10_attention_weight_max": 36.66914367675781,
      "activations/layer10_attention_weight_min": -31.069814682006836,
      "activations/layer11_attention_weight_max": 39.136924743652344,
      "activations/layer11_attention_weight_min": -32.112274169921875,
      "activations/layer12_attention_weight_max": 65.26014709472656,
      "activations/layer12_attention_weight_min": -32.73077392578125,
      "activations/layer13_attention_weight_max": 73.80180358886719,
      "activations/layer13_attention_weight_min": -39.40201950073242,
      "activations/layer14_attention_weight_max": 63.20867156982422,
      "activations/layer14_attention_weight_min": -41.72057342529297,
      "activations/layer15_attention_weight_max": 66.30691528320312,
      "activations/layer15_attention_weight_min": -29.98597526550293,
      "activations/layer16_attention_weight_max": 36.283809661865234,
      "activations/layer16_attention_weight_min": -28.188249588012695,
      "activations/layer17_attention_weight_max": 39.1932373046875,
      "activations/layer17_attention_weight_min": -23.631744384765625,
      "activations/layer18_attention_weight_max": 33.438812255859375,
      "activations/layer18_attention_weight_min": -20.333213806152344,
      "activations/layer19_attention_weight_max": 43.125,
      "activations/layer19_attention_weight_min": -32.23341369628906,
      "activations/layer1_attention_weight_max": 18.44777488708496,
      "activations/layer1_attention_weight_min": -15.122030258178711,
      "activations/layer20_attention_weight_max": 37.29298400878906,
      "activations/layer20_attention_weight_min": -23.2475528717041,
      "activations/layer21_attention_weight_max": 34.651466369628906,
      "activations/layer21_attention_weight_min": -21.81097412109375,
      "activations/layer22_attention_weight_max": 52.77908706665039,
      "activations/layer22_attention_weight_min": -25.542301177978516,
      "activations/layer23_attention_weight_max": 35.93056106567383,
      "activations/layer23_attention_weight_min": -19.795452117919922,
      "activations/layer2_attention_weight_max": 34.59454345703125,
      "activations/layer2_attention_weight_min": -29.901290893554688,
      "activations/layer3_attention_weight_max": 101.06774139404297,
      "activations/layer3_attention_weight_min": -93.70758819580078,
      "activations/layer4_attention_weight_max": 141.52488708496094,
      "activations/layer4_attention_weight_min": -127.59207916259766,
      "activations/layer5_attention_weight_max": 73.60147094726562,
      "activations/layer5_attention_weight_min": -70.06574249267578,
      "activations/layer6_attention_weight_max": 59.74633026123047,
      "activations/layer6_attention_weight_min": -54.61601638793945,
      "activations/layer7_attention_weight_max": 98.87422943115234,
      "activations/layer7_attention_weight_min": -88.4016342163086,
      "activations/layer8_attention_weight_max": 42.30441665649414,
      "activations/layer8_attention_weight_min": -39.29676818847656,
      "activations/layer9_attention_weight_max": 38.88752365112305,
      "activations/layer9_attention_weight_min": -39.58316421508789,
      "epoch": 6.21,
      "learning_rate": 0.00011106022727272727,
      "loss": 2.8479,
      "step": 106900
    },
    {
      "activations/layer0_attention_weight_max": 16.142051696777344,
      "activations/layer0_attention_weight_min": -11.799306869506836,
      "activations/layer10_attention_weight_max": 33.82552719116211,
      "activations/layer10_attention_weight_min": -31.213512420654297,
      "activations/layer11_attention_weight_max": 35.84228515625,
      "activations/layer11_attention_weight_min": -28.922985076904297,
      "activations/layer12_attention_weight_max": 24.831296920776367,
      "activations/layer12_attention_weight_min": -26.41666603088379,
      "activations/layer13_attention_weight_max": 46.24867248535156,
      "activations/layer13_attention_weight_min": -35.99774169921875,
      "activations/layer14_attention_weight_max": 60.64100646972656,
      "activations/layer14_attention_weight_min": -47.44837188720703,
      "activations/layer15_attention_weight_max": 41.152061462402344,
      "activations/layer15_attention_weight_min": -28.546340942382812,
      "activations/layer16_attention_weight_max": 33.522308349609375,
      "activations/layer16_attention_weight_min": -28.328990936279297,
      "activations/layer17_attention_weight_max": 33.59660339355469,
      "activations/layer17_attention_weight_min": -25.8426456451416,
      "activations/layer18_attention_weight_max": 31.08474349975586,
      "activations/layer18_attention_weight_min": -23.10886001586914,
      "activations/layer19_attention_weight_max": 41.16915512084961,
      "activations/layer19_attention_weight_min": -29.26657485961914,
      "activations/layer1_attention_weight_max": 18.44565200805664,
      "activations/layer1_attention_weight_min": -15.316423416137695,
      "activations/layer20_attention_weight_max": 30.498809814453125,
      "activations/layer20_attention_weight_min": -21.934614181518555,
      "activations/layer21_attention_weight_max": 28.146324157714844,
      "activations/layer21_attention_weight_min": -22.305818557739258,
      "activations/layer22_attention_weight_max": 45.73920440673828,
      "activations/layer22_attention_weight_min": -27.91824722290039,
      "activations/layer23_attention_weight_max": 31.915061950683594,
      "activations/layer23_attention_weight_min": -21.490177154541016,
      "activations/layer2_attention_weight_max": 32.88695526123047,
      "activations/layer2_attention_weight_min": -31.67374038696289,
      "activations/layer3_attention_weight_max": 94.81629180908203,
      "activations/layer3_attention_weight_min": -96.85630798339844,
      "activations/layer4_attention_weight_max": 139.33709716796875,
      "activations/layer4_attention_weight_min": -132.98056030273438,
      "activations/layer5_attention_weight_max": 77.16874694824219,
      "activations/layer5_attention_weight_min": -73.56159210205078,
      "activations/layer6_attention_weight_max": 56.21501541137695,
      "activations/layer6_attention_weight_min": -53.473304748535156,
      "activations/layer7_attention_weight_max": 89.80941009521484,
      "activations/layer7_attention_weight_min": -88.5291748046875,
      "activations/layer8_attention_weight_max": 40.409568786621094,
      "activations/layer8_attention_weight_min": -40.26483917236328,
      "activations/layer9_attention_weight_max": 37.308807373046875,
      "activations/layer9_attention_weight_min": -35.09370040893555,
      "epoch": 6.21,
      "learning_rate": 0.00011104128787878786,
      "loss": 2.8322,
      "step": 106950
    },
    {
      "activations/layer0_attention_weight_max": 16.18157196044922,
      "activations/layer0_attention_weight_min": -12.696503639221191,
      "activations/layer10_attention_weight_max": 38.955665588378906,
      "activations/layer10_attention_weight_min": -31.745214462280273,
      "activations/layer11_attention_weight_max": 34.516021728515625,
      "activations/layer11_attention_weight_min": -26.884429931640625,
      "activations/layer12_attention_weight_max": 28.36836051940918,
      "activations/layer12_attention_weight_min": -25.98952865600586,
      "activations/layer13_attention_weight_max": 41.532997131347656,
      "activations/layer13_attention_weight_min": -32.24658966064453,
      "activations/layer14_attention_weight_max": 62.765892028808594,
      "activations/layer14_attention_weight_min": -47.571285247802734,
      "activations/layer15_attention_weight_max": 37.029937744140625,
      "activations/layer15_attention_weight_min": -30.274372100830078,
      "activations/layer16_attention_weight_max": 40.6567497253418,
      "activations/layer16_attention_weight_min": -27.009769439697266,
      "activations/layer17_attention_weight_max": 37.71803283691406,
      "activations/layer17_attention_weight_min": -25.046518325805664,
      "activations/layer18_attention_weight_max": 31.997896194458008,
      "activations/layer18_attention_weight_min": -20.531661987304688,
      "activations/layer19_attention_weight_max": 44.59652328491211,
      "activations/layer19_attention_weight_min": -30.238723754882812,
      "activations/layer1_attention_weight_max": 18.083669662475586,
      "activations/layer1_attention_weight_min": -16.068012237548828,
      "activations/layer20_attention_weight_max": 32.668575286865234,
      "activations/layer20_attention_weight_min": -24.086362838745117,
      "activations/layer21_attention_weight_max": 41.02210998535156,
      "activations/layer21_attention_weight_min": -22.046470642089844,
      "activations/layer22_attention_weight_max": 54.448944091796875,
      "activations/layer22_attention_weight_min": -28.672231674194336,
      "activations/layer23_attention_weight_max": 36.577674865722656,
      "activations/layer23_attention_weight_min": -23.217937469482422,
      "activations/layer2_attention_weight_max": 32.0106086730957,
      "activations/layer2_attention_weight_min": -30.510046005249023,
      "activations/layer3_attention_weight_max": 88.09021759033203,
      "activations/layer3_attention_weight_min": -93.3405532836914,
      "activations/layer4_attention_weight_max": 127.82706451416016,
      "activations/layer4_attention_weight_min": -125.8413314819336,
      "activations/layer5_attention_weight_max": 69.87040710449219,
      "activations/layer5_attention_weight_min": -66.64442443847656,
      "activations/layer6_attention_weight_max": 50.37023162841797,
      "activations/layer6_attention_weight_min": -53.96738815307617,
      "activations/layer7_attention_weight_max": 87.81507873535156,
      "activations/layer7_attention_weight_min": -85.20236206054688,
      "activations/layer8_attention_weight_max": 43.3111457824707,
      "activations/layer8_attention_weight_min": -38.1279411315918,
      "activations/layer9_attention_weight_max": 39.062339782714844,
      "activations/layer9_attention_weight_min": -36.379154205322266,
      "epoch": 6.22,
      "learning_rate": 0.00011102234848484848,
      "loss": 2.8355,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.5019,
      "eval_samples_per_second": 505.062,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.5019,
      "eval_openwebtext_samples_per_second": 505.062,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_wikitext_loss": 3.013671875,
      "eval_wikitext_ppl": 20.36202965350754,
      "eval_wikitext_runtime": 1.9723,
      "eval_wikitext_samples_per_second": 231.196,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_lambada_loss": 2.623046875,
      "eval_lambada_ppl": 13.777638440111984,
      "eval_lambada_runtime": 9.5591,
      "eval_lambada_samples_per_second": 509.359,
      "step": 107000
    },
    {
      "activations/layer0_attention_weight_max": 17.053007125854492,
      "activations/layer0_attention_weight_min": -11.575023651123047,
      "activations/layer10_attention_weight_max": 34.66471481323242,
      "activations/layer10_attention_weight_min": -31.06642723083496,
      "activations/layer11_attention_weight_max": 35.58525085449219,
      "activations/layer11_attention_weight_min": -28.366018295288086,
      "activations/layer12_attention_weight_max": 22.928977966308594,
      "activations/layer12_attention_weight_min": -37.740142822265625,
      "activations/layer13_attention_weight_max": 41.091285705566406,
      "activations/layer13_attention_weight_min": -30.275798797607422,
      "activations/layer14_attention_weight_max": 69.22224426269531,
      "activations/layer14_attention_weight_min": -41.914756774902344,
      "activations/layer15_attention_weight_max": 39.255191802978516,
      "activations/layer15_attention_weight_min": -28.413904190063477,
      "activations/layer16_attention_weight_max": 37.52849197387695,
      "activations/layer16_attention_weight_min": -26.437297821044922,
      "activations/layer17_attention_weight_max": 40.56349563598633,
      "activations/layer17_attention_weight_min": -23.09621810913086,
      "activations/layer18_attention_weight_max": 33.26366424560547,
      "activations/layer18_attention_weight_min": -21.701814651489258,
      "activations/layer19_attention_weight_max": 45.047183990478516,
      "activations/layer19_attention_weight_min": -29.628034591674805,
      "activations/layer1_attention_weight_max": 18.172826766967773,
      "activations/layer1_attention_weight_min": -15.266727447509766,
      "activations/layer20_attention_weight_max": 38.123741149902344,
      "activations/layer20_attention_weight_min": -24.842443466186523,
      "activations/layer21_attention_weight_max": 36.68748474121094,
      "activations/layer21_attention_weight_min": -20.729543685913086,
      "activations/layer22_attention_weight_max": 59.73625564575195,
      "activations/layer22_attention_weight_min": -26.854761123657227,
      "activations/layer23_attention_weight_max": 36.449180603027344,
      "activations/layer23_attention_weight_min": -24.643613815307617,
      "activations/layer2_attention_weight_max": 31.41763687133789,
      "activations/layer2_attention_weight_min": -31.181114196777344,
      "activations/layer3_attention_weight_max": 92.91956329345703,
      "activations/layer3_attention_weight_min": -93.68128967285156,
      "activations/layer4_attention_weight_max": 134.20443725585938,
      "activations/layer4_attention_weight_min": -129.1465606689453,
      "activations/layer5_attention_weight_max": 73.69310760498047,
      "activations/layer5_attention_weight_min": -68.98085021972656,
      "activations/layer6_attention_weight_max": 54.655433654785156,
      "activations/layer6_attention_weight_min": -52.48064041137695,
      "activations/layer7_attention_weight_max": 93.20024871826172,
      "activations/layer7_attention_weight_min": -93.84973907470703,
      "activations/layer8_attention_weight_max": 41.94870376586914,
      "activations/layer8_attention_weight_min": -41.276039123535156,
      "activations/layer9_attention_weight_max": 34.981048583984375,
      "activations/layer9_attention_weight_min": -43.225181579589844,
      "epoch": 6.22,
      "learning_rate": 0.00011100340909090908,
      "loss": 2.8158,
      "step": 107050
    },
    {
      "activations/layer0_attention_weight_max": 14.738165855407715,
      "activations/layer0_attention_weight_min": -12.424590110778809,
      "activations/layer10_attention_weight_max": 36.801673889160156,
      "activations/layer10_attention_weight_min": -34.01921463012695,
      "activations/layer11_attention_weight_max": 40.54079055786133,
      "activations/layer11_attention_weight_min": -30.567203521728516,
      "activations/layer12_attention_weight_max": 39.488948822021484,
      "activations/layer12_attention_weight_min": -25.277448654174805,
      "activations/layer13_attention_weight_max": 42.171470642089844,
      "activations/layer13_attention_weight_min": -35.32495880126953,
      "activations/layer14_attention_weight_max": 60.93560028076172,
      "activations/layer14_attention_weight_min": -45.70694351196289,
      "activations/layer15_attention_weight_max": 46.8065185546875,
      "activations/layer15_attention_weight_min": -30.91155433654785,
      "activations/layer16_attention_weight_max": 42.48679733276367,
      "activations/layer16_attention_weight_min": -27.917238235473633,
      "activations/layer17_attention_weight_max": 42.1244010925293,
      "activations/layer17_attention_weight_min": -25.258766174316406,
      "activations/layer18_attention_weight_max": 38.91714096069336,
      "activations/layer18_attention_weight_min": -22.2522029876709,
      "activations/layer19_attention_weight_max": 43.10087203979492,
      "activations/layer19_attention_weight_min": -30.39894676208496,
      "activations/layer1_attention_weight_max": 17.441791534423828,
      "activations/layer1_attention_weight_min": -15.774682998657227,
      "activations/layer20_attention_weight_max": 36.428016662597656,
      "activations/layer20_attention_weight_min": -22.683452606201172,
      "activations/layer21_attention_weight_max": 32.714595794677734,
      "activations/layer21_attention_weight_min": -20.1607666015625,
      "activations/layer22_attention_weight_max": 50.3275146484375,
      "activations/layer22_attention_weight_min": -27.430452346801758,
      "activations/layer23_attention_weight_max": 36.91535186767578,
      "activations/layer23_attention_weight_min": -20.19859504699707,
      "activations/layer2_attention_weight_max": 33.71440887451172,
      "activations/layer2_attention_weight_min": -33.286712646484375,
      "activations/layer3_attention_weight_max": 92.79936218261719,
      "activations/layer3_attention_weight_min": -93.61820220947266,
      "activations/layer4_attention_weight_max": 124.72020721435547,
      "activations/layer4_attention_weight_min": -130.50584411621094,
      "activations/layer5_attention_weight_max": 70.50922393798828,
      "activations/layer5_attention_weight_min": -69.15545654296875,
      "activations/layer6_attention_weight_max": 54.463050842285156,
      "activations/layer6_attention_weight_min": -55.3697395324707,
      "activations/layer7_attention_weight_max": 83.70372009277344,
      "activations/layer7_attention_weight_min": -88.52884674072266,
      "activations/layer8_attention_weight_max": 41.703182220458984,
      "activations/layer8_attention_weight_min": -42.64714431762695,
      "activations/layer9_attention_weight_max": 36.78548812866211,
      "activations/layer9_attention_weight_min": -34.97176742553711,
      "epoch": 6.22,
      "learning_rate": 0.00011098446969696968,
      "loss": 2.8437,
      "step": 107100
    },
    {
      "activations/layer0_attention_weight_max": 15.322065353393555,
      "activations/layer0_attention_weight_min": -11.901641845703125,
      "activations/layer10_attention_weight_max": 35.37382888793945,
      "activations/layer10_attention_weight_min": -32.764373779296875,
      "activations/layer11_attention_weight_max": 33.77113342285156,
      "activations/layer11_attention_weight_min": -28.536712646484375,
      "activations/layer12_attention_weight_max": 32.056785583496094,
      "activations/layer12_attention_weight_min": -23.35289764404297,
      "activations/layer13_attention_weight_max": 46.817893981933594,
      "activations/layer13_attention_weight_min": -36.65277099609375,
      "activations/layer14_attention_weight_max": 64.8260269165039,
      "activations/layer14_attention_weight_min": -50.62265396118164,
      "activations/layer15_attention_weight_max": 36.536094665527344,
      "activations/layer15_attention_weight_min": -31.046199798583984,
      "activations/layer16_attention_weight_max": 35.33298110961914,
      "activations/layer16_attention_weight_min": -29.69014549255371,
      "activations/layer17_attention_weight_max": 35.34815979003906,
      "activations/layer17_attention_weight_min": -24.884126663208008,
      "activations/layer18_attention_weight_max": 32.5987434387207,
      "activations/layer18_attention_weight_min": -20.508825302124023,
      "activations/layer19_attention_weight_max": 43.68154525756836,
      "activations/layer19_attention_weight_min": -30.370981216430664,
      "activations/layer1_attention_weight_max": 16.97638702392578,
      "activations/layer1_attention_weight_min": -14.317262649536133,
      "activations/layer20_attention_weight_max": 31.581329345703125,
      "activations/layer20_attention_weight_min": -24.72972297668457,
      "activations/layer21_attention_weight_max": 30.66434097290039,
      "activations/layer21_attention_weight_min": -22.861499786376953,
      "activations/layer22_attention_weight_max": 48.80213165283203,
      "activations/layer22_attention_weight_min": -32.68286895751953,
      "activations/layer23_attention_weight_max": 36.77608871459961,
      "activations/layer23_attention_weight_min": -20.49156951904297,
      "activations/layer2_attention_weight_max": 33.45673370361328,
      "activations/layer2_attention_weight_min": -30.98935317993164,
      "activations/layer3_attention_weight_max": 92.6567611694336,
      "activations/layer3_attention_weight_min": -94.36009979248047,
      "activations/layer4_attention_weight_max": 128.77969360351562,
      "activations/layer4_attention_weight_min": -132.0463104248047,
      "activations/layer5_attention_weight_max": 74.92924499511719,
      "activations/layer5_attention_weight_min": -73.8648452758789,
      "activations/layer6_attention_weight_max": 55.145538330078125,
      "activations/layer6_attention_weight_min": -59.1860237121582,
      "activations/layer7_attention_weight_max": 87.45684814453125,
      "activations/layer7_attention_weight_min": -92.84922790527344,
      "activations/layer8_attention_weight_max": 40.73524475097656,
      "activations/layer8_attention_weight_min": -44.54654312133789,
      "activations/layer9_attention_weight_max": 36.71346664428711,
      "activations/layer9_attention_weight_min": -36.36112976074219,
      "epoch": 6.23,
      "learning_rate": 0.0001109655303030303,
      "loss": 2.819,
      "step": 107150
    },
    {
      "activations/layer0_attention_weight_max": 16.00632095336914,
      "activations/layer0_attention_weight_min": -12.021306991577148,
      "activations/layer10_attention_weight_max": 29.146703720092773,
      "activations/layer10_attention_weight_min": -29.659452438354492,
      "activations/layer11_attention_weight_max": 33.4096794128418,
      "activations/layer11_attention_weight_min": -29.350727081298828,
      "activations/layer12_attention_weight_max": 24.038257598876953,
      "activations/layer12_attention_weight_min": -26.815004348754883,
      "activations/layer13_attention_weight_max": 36.19929504394531,
      "activations/layer13_attention_weight_min": -31.75018310546875,
      "activations/layer14_attention_weight_max": 47.6855583190918,
      "activations/layer14_attention_weight_min": -41.00044631958008,
      "activations/layer15_attention_weight_max": 40.16451644897461,
      "activations/layer15_attention_weight_min": -27.719539642333984,
      "activations/layer16_attention_weight_max": 33.722740173339844,
      "activations/layer16_attention_weight_min": -28.103784561157227,
      "activations/layer17_attention_weight_max": 35.00392532348633,
      "activations/layer17_attention_weight_min": -25.24047088623047,
      "activations/layer18_attention_weight_max": 31.40023422241211,
      "activations/layer18_attention_weight_min": -19.86604881286621,
      "activations/layer19_attention_weight_max": 38.37089157104492,
      "activations/layer19_attention_weight_min": -31.839557647705078,
      "activations/layer1_attention_weight_max": 16.427593231201172,
      "activations/layer1_attention_weight_min": -15.350131034851074,
      "activations/layer20_attention_weight_max": 30.48828125,
      "activations/layer20_attention_weight_min": -24.12198829650879,
      "activations/layer21_attention_weight_max": 32.33161163330078,
      "activations/layer21_attention_weight_min": -21.264650344848633,
      "activations/layer22_attention_weight_max": 47.85340881347656,
      "activations/layer22_attention_weight_min": -27.046796798706055,
      "activations/layer23_attention_weight_max": 31.73371696472168,
      "activations/layer23_attention_weight_min": -21.236576080322266,
      "activations/layer2_attention_weight_max": 30.26264762878418,
      "activations/layer2_attention_weight_min": -29.8089656829834,
      "activations/layer3_attention_weight_max": 85.23948669433594,
      "activations/layer3_attention_weight_min": -88.02317810058594,
      "activations/layer4_attention_weight_max": 124.29376220703125,
      "activations/layer4_attention_weight_min": -121.03263092041016,
      "activations/layer5_attention_weight_max": 67.35465240478516,
      "activations/layer5_attention_weight_min": -65.40232849121094,
      "activations/layer6_attention_weight_max": 50.37883377075195,
      "activations/layer6_attention_weight_min": -52.819786071777344,
      "activations/layer7_attention_weight_max": 79.39061737060547,
      "activations/layer7_attention_weight_min": -84.869140625,
      "activations/layer8_attention_weight_max": 34.9136848449707,
      "activations/layer8_attention_weight_min": -37.602291107177734,
      "activations/layer9_attention_weight_max": 32.878353118896484,
      "activations/layer9_attention_weight_min": -36.51605224609375,
      "epoch": 6.23,
      "learning_rate": 0.0001109465909090909,
      "loss": 2.8255,
      "step": 107200
    },
    {
      "activations/layer0_attention_weight_max": 16.39461326599121,
      "activations/layer0_attention_weight_min": -12.892900466918945,
      "activations/layer10_attention_weight_max": 32.49293518066406,
      "activations/layer10_attention_weight_min": -31.822784423828125,
      "activations/layer11_attention_weight_max": 31.237594604492188,
      "activations/layer11_attention_weight_min": -27.298660278320312,
      "activations/layer12_attention_weight_max": 23.134689331054688,
      "activations/layer12_attention_weight_min": -25.993310928344727,
      "activations/layer13_attention_weight_max": 36.30516052246094,
      "activations/layer13_attention_weight_min": -30.851425170898438,
      "activations/layer14_attention_weight_max": 44.914146423339844,
      "activations/layer14_attention_weight_min": -38.2213020324707,
      "activations/layer15_attention_weight_max": 31.63296127319336,
      "activations/layer15_attention_weight_min": -28.313737869262695,
      "activations/layer16_attention_weight_max": 32.85441207885742,
      "activations/layer16_attention_weight_min": -29.43730354309082,
      "activations/layer17_attention_weight_max": 35.95158004760742,
      "activations/layer17_attention_weight_min": -26.501750946044922,
      "activations/layer18_attention_weight_max": 26.982563018798828,
      "activations/layer18_attention_weight_min": -20.474781036376953,
      "activations/layer19_attention_weight_max": 36.163902282714844,
      "activations/layer19_attention_weight_min": -30.196561813354492,
      "activations/layer1_attention_weight_max": 17.177934646606445,
      "activations/layer1_attention_weight_min": -12.64040756225586,
      "activations/layer20_attention_weight_max": 26.847118377685547,
      "activations/layer20_attention_weight_min": -24.543973922729492,
      "activations/layer21_attention_weight_max": 28.14097785949707,
      "activations/layer21_attention_weight_min": -21.82825469970703,
      "activations/layer22_attention_weight_max": 44.0543098449707,
      "activations/layer22_attention_weight_min": -27.458953857421875,
      "activations/layer23_attention_weight_max": 33.362030029296875,
      "activations/layer23_attention_weight_min": -24.474578857421875,
      "activations/layer2_attention_weight_max": 32.36590576171875,
      "activations/layer2_attention_weight_min": -30.835315704345703,
      "activations/layer3_attention_weight_max": 85.87907409667969,
      "activations/layer3_attention_weight_min": -91.82734680175781,
      "activations/layer4_attention_weight_max": 123.35017395019531,
      "activations/layer4_attention_weight_min": -122.0335464477539,
      "activations/layer5_attention_weight_max": 66.57252502441406,
      "activations/layer5_attention_weight_min": -65.37965393066406,
      "activations/layer6_attention_weight_max": 51.97373580932617,
      "activations/layer6_attention_weight_min": -51.86092758178711,
      "activations/layer7_attention_weight_max": 84.07907104492188,
      "activations/layer7_attention_weight_min": -84.058837890625,
      "activations/layer8_attention_weight_max": 37.227203369140625,
      "activations/layer8_attention_weight_min": -39.11478042602539,
      "activations/layer9_attention_weight_max": 32.986324310302734,
      "activations/layer9_attention_weight_min": -34.326210021972656,
      "epoch": 6.23,
      "learning_rate": 0.00011092765151515151,
      "loss": 2.8179,
      "step": 107250
    },
    {
      "activations/layer0_attention_weight_max": 16.3570556640625,
      "activations/layer0_attention_weight_min": -12.841012954711914,
      "activations/layer10_attention_weight_max": 31.463003158569336,
      "activations/layer10_attention_weight_min": -29.591176986694336,
      "activations/layer11_attention_weight_max": 30.151748657226562,
      "activations/layer11_attention_weight_min": -26.38345718383789,
      "activations/layer12_attention_weight_max": 22.97309684753418,
      "activations/layer12_attention_weight_min": -27.004819869995117,
      "activations/layer13_attention_weight_max": 36.36180877685547,
      "activations/layer13_attention_weight_min": -30.558496475219727,
      "activations/layer14_attention_weight_max": 53.86027145385742,
      "activations/layer14_attention_weight_min": -41.88488006591797,
      "activations/layer15_attention_weight_max": 36.91093444824219,
      "activations/layer15_attention_weight_min": -27.712791442871094,
      "activations/layer16_attention_weight_max": 33.59398651123047,
      "activations/layer16_attention_weight_min": -27.80137825012207,
      "activations/layer17_attention_weight_max": 36.86159133911133,
      "activations/layer17_attention_weight_min": -23.092601776123047,
      "activations/layer18_attention_weight_max": 34.47541427612305,
      "activations/layer18_attention_weight_min": -19.61506462097168,
      "activations/layer19_attention_weight_max": 38.1700553894043,
      "activations/layer19_attention_weight_min": -29.098779678344727,
      "activations/layer1_attention_weight_max": 17.329294204711914,
      "activations/layer1_attention_weight_min": -16.633363723754883,
      "activations/layer20_attention_weight_max": 34.75511169433594,
      "activations/layer20_attention_weight_min": -22.139238357543945,
      "activations/layer21_attention_weight_max": 32.47153854370117,
      "activations/layer21_attention_weight_min": -21.67987060546875,
      "activations/layer22_attention_weight_max": 43.82557678222656,
      "activations/layer22_attention_weight_min": -27.176666259765625,
      "activations/layer23_attention_weight_max": 33.67729187011719,
      "activations/layer23_attention_weight_min": -19.79987144470215,
      "activations/layer2_attention_weight_max": 31.56532096862793,
      "activations/layer2_attention_weight_min": -29.973142623901367,
      "activations/layer3_attention_weight_max": 92.175048828125,
      "activations/layer3_attention_weight_min": -87.99085235595703,
      "activations/layer4_attention_weight_max": 127.85237884521484,
      "activations/layer4_attention_weight_min": -121.56388092041016,
      "activations/layer5_attention_weight_max": 70.93962097167969,
      "activations/layer5_attention_weight_min": -68.0829086303711,
      "activations/layer6_attention_weight_max": 53.20689392089844,
      "activations/layer6_attention_weight_min": -54.23282241821289,
      "activations/layer7_attention_weight_max": 84.25565338134766,
      "activations/layer7_attention_weight_min": -84.03241729736328,
      "activations/layer8_attention_weight_max": 36.84593963623047,
      "activations/layer8_attention_weight_min": -40.91876220703125,
      "activations/layer9_attention_weight_max": 34.47140884399414,
      "activations/layer9_attention_weight_min": -33.33501434326172,
      "epoch": 6.23,
      "learning_rate": 0.00011090871212121211,
      "loss": 2.8329,
      "step": 107300
    },
    {
      "activations/layer0_attention_weight_max": 15.896428108215332,
      "activations/layer0_attention_weight_min": -13.72605037689209,
      "activations/layer10_attention_weight_max": 33.81863021850586,
      "activations/layer10_attention_weight_min": -33.49617385864258,
      "activations/layer11_attention_weight_max": 32.40894317626953,
      "activations/layer11_attention_weight_min": -31.126148223876953,
      "activations/layer12_attention_weight_max": 25.093841552734375,
      "activations/layer12_attention_weight_min": -26.328006744384766,
      "activations/layer13_attention_weight_max": 42.71806716918945,
      "activations/layer13_attention_weight_min": -34.438926696777344,
      "activations/layer14_attention_weight_max": 61.037498474121094,
      "activations/layer14_attention_weight_min": -43.2529411315918,
      "activations/layer15_attention_weight_max": 36.82744216918945,
      "activations/layer15_attention_weight_min": -28.716758728027344,
      "activations/layer16_attention_weight_max": 40.05194854736328,
      "activations/layer16_attention_weight_min": -30.382112503051758,
      "activations/layer17_attention_weight_max": 39.005828857421875,
      "activations/layer17_attention_weight_min": -27.87661361694336,
      "activations/layer18_attention_weight_max": 36.80133819580078,
      "activations/layer18_attention_weight_min": -23.527427673339844,
      "activations/layer19_attention_weight_max": 46.759151458740234,
      "activations/layer19_attention_weight_min": -31.443689346313477,
      "activations/layer1_attention_weight_max": 16.743406295776367,
      "activations/layer1_attention_weight_min": -13.768525123596191,
      "activations/layer20_attention_weight_max": 33.215484619140625,
      "activations/layer20_attention_weight_min": -23.624746322631836,
      "activations/layer21_attention_weight_max": 31.2255916595459,
      "activations/layer21_attention_weight_min": -21.731611251831055,
      "activations/layer22_attention_weight_max": 51.34890365600586,
      "activations/layer22_attention_weight_min": -28.568805694580078,
      "activations/layer23_attention_weight_max": 35.87427520751953,
      "activations/layer23_attention_weight_min": -19.295940399169922,
      "activations/layer2_attention_weight_max": 33.495567321777344,
      "activations/layer2_attention_weight_min": -32.4417724609375,
      "activations/layer3_attention_weight_max": 97.9815444946289,
      "activations/layer3_attention_weight_min": -98.22754669189453,
      "activations/layer4_attention_weight_max": 140.29486083984375,
      "activations/layer4_attention_weight_min": -138.0355224609375,
      "activations/layer5_attention_weight_max": 75.40748596191406,
      "activations/layer5_attention_weight_min": -74.9083480834961,
      "activations/layer6_attention_weight_max": 56.15314483642578,
      "activations/layer6_attention_weight_min": -59.06403732299805,
      "activations/layer7_attention_weight_max": 90.5802001953125,
      "activations/layer7_attention_weight_min": -93.59686279296875,
      "activations/layer8_attention_weight_max": 42.55475616455078,
      "activations/layer8_attention_weight_min": -43.55882263183594,
      "activations/layer9_attention_weight_max": 36.72246551513672,
      "activations/layer9_attention_weight_min": -37.14999008178711,
      "epoch": 6.24,
      "learning_rate": 0.0001108901515151515,
      "loss": 2.8362,
      "step": 107350
    },
    {
      "activations/layer0_attention_weight_max": 15.935953140258789,
      "activations/layer0_attention_weight_min": -13.32400894165039,
      "activations/layer10_attention_weight_max": 32.898555755615234,
      "activations/layer10_attention_weight_min": -31.61846923828125,
      "activations/layer11_attention_weight_max": 31.477310180664062,
      "activations/layer11_attention_weight_min": -27.265974044799805,
      "activations/layer12_attention_weight_max": 25.249923706054688,
      "activations/layer12_attention_weight_min": -31.72876739501953,
      "activations/layer13_attention_weight_max": 42.0863151550293,
      "activations/layer13_attention_weight_min": -31.315242767333984,
      "activations/layer14_attention_weight_max": 54.48051071166992,
      "activations/layer14_attention_weight_min": -42.70488357543945,
      "activations/layer15_attention_weight_max": 37.381988525390625,
      "activations/layer15_attention_weight_min": -28.080293655395508,
      "activations/layer16_attention_weight_max": 35.85869598388672,
      "activations/layer16_attention_weight_min": -28.898983001708984,
      "activations/layer17_attention_weight_max": 37.75004577636719,
      "activations/layer17_attention_weight_min": -24.8354434967041,
      "activations/layer18_attention_weight_max": 27.82180404663086,
      "activations/layer18_attention_weight_min": -21.032243728637695,
      "activations/layer19_attention_weight_max": 35.48383712768555,
      "activations/layer19_attention_weight_min": -30.428058624267578,
      "activations/layer1_attention_weight_max": 16.814252853393555,
      "activations/layer1_attention_weight_min": -15.659224510192871,
      "activations/layer20_attention_weight_max": 31.85972785949707,
      "activations/layer20_attention_weight_min": -26.701154708862305,
      "activations/layer21_attention_weight_max": 30.563627243041992,
      "activations/layer21_attention_weight_min": -23.75884437561035,
      "activations/layer22_attention_weight_max": 46.4221076965332,
      "activations/layer22_attention_weight_min": -31.063203811645508,
      "activations/layer23_attention_weight_max": 34.50096893310547,
      "activations/layer23_attention_weight_min": -22.891925811767578,
      "activations/layer2_attention_weight_max": 33.00580978393555,
      "activations/layer2_attention_weight_min": -32.5622444152832,
      "activations/layer3_attention_weight_max": 95.66577911376953,
      "activations/layer3_attention_weight_min": -93.33938598632812,
      "activations/layer4_attention_weight_max": 131.31613159179688,
      "activations/layer4_attention_weight_min": -128.10604858398438,
      "activations/layer5_attention_weight_max": 72.80606842041016,
      "activations/layer5_attention_weight_min": -70.77287292480469,
      "activations/layer6_attention_weight_max": 55.58195495605469,
      "activations/layer6_attention_weight_min": -56.497867584228516,
      "activations/layer7_attention_weight_max": 89.49639892578125,
      "activations/layer7_attention_weight_min": -88.13782501220703,
      "activations/layer8_attention_weight_max": 39.99229431152344,
      "activations/layer8_attention_weight_min": -41.434486389160156,
      "activations/layer9_attention_weight_max": 36.5684814453125,
      "activations/layer9_attention_weight_min": -34.24898147583008,
      "epoch": 6.24,
      "learning_rate": 0.0001108712121212121,
      "loss": 2.8283,
      "step": 107400
    },
    {
      "activations/layer0_attention_weight_max": 17.032608032226562,
      "activations/layer0_attention_weight_min": -12.235260009765625,
      "activations/layer10_attention_weight_max": 35.33621597290039,
      "activations/layer10_attention_weight_min": -31.66014862060547,
      "activations/layer11_attention_weight_max": 34.6083984375,
      "activations/layer11_attention_weight_min": -28.156475067138672,
      "activations/layer12_attention_weight_max": 27.57600212097168,
      "activations/layer12_attention_weight_min": -25.662153244018555,
      "activations/layer13_attention_weight_max": 43.25064468383789,
      "activations/layer13_attention_weight_min": -31.612510681152344,
      "activations/layer14_attention_weight_max": 70.64388275146484,
      "activations/layer14_attention_weight_min": -45.661746978759766,
      "activations/layer15_attention_weight_max": 39.77201461791992,
      "activations/layer15_attention_weight_min": -27.507410049438477,
      "activations/layer16_attention_weight_max": 39.500572204589844,
      "activations/layer16_attention_weight_min": -29.359710693359375,
      "activations/layer17_attention_weight_max": 39.2074089050293,
      "activations/layer17_attention_weight_min": -24.837385177612305,
      "activations/layer18_attention_weight_max": 32.65895080566406,
      "activations/layer18_attention_weight_min": -21.480037689208984,
      "activations/layer19_attention_weight_max": 42.451904296875,
      "activations/layer19_attention_weight_min": -30.92831802368164,
      "activations/layer1_attention_weight_max": 16.083803176879883,
      "activations/layer1_attention_weight_min": -18.90744400024414,
      "activations/layer20_attention_weight_max": 33.37409210205078,
      "activations/layer20_attention_weight_min": -25.23240852355957,
      "activations/layer21_attention_weight_max": 33.86844253540039,
      "activations/layer21_attention_weight_min": -23.014102935791016,
      "activations/layer22_attention_weight_max": 52.44388198852539,
      "activations/layer22_attention_weight_min": -29.238075256347656,
      "activations/layer23_attention_weight_max": 34.877044677734375,
      "activations/layer23_attention_weight_min": -23.52602195739746,
      "activations/layer2_attention_weight_max": 30.2944393157959,
      "activations/layer2_attention_weight_min": -31.67409896850586,
      "activations/layer3_attention_weight_max": 89.6187515258789,
      "activations/layer3_attention_weight_min": -95.1230239868164,
      "activations/layer4_attention_weight_max": 122.9339828491211,
      "activations/layer4_attention_weight_min": -123.97936248779297,
      "activations/layer5_attention_weight_max": 70.39285278320312,
      "activations/layer5_attention_weight_min": -67.43777465820312,
      "activations/layer6_attention_weight_max": 52.26876449584961,
      "activations/layer6_attention_weight_min": -51.34330368041992,
      "activations/layer7_attention_weight_max": 91.48755645751953,
      "activations/layer7_attention_weight_min": -81.23251342773438,
      "activations/layer8_attention_weight_max": 39.28889846801758,
      "activations/layer8_attention_weight_min": -40.509918212890625,
      "activations/layer9_attention_weight_max": 38.38431930541992,
      "activations/layer9_attention_weight_min": -35.070289611816406,
      "epoch": 6.24,
      "learning_rate": 0.00011085227272727272,
      "loss": 2.828,
      "step": 107450
    },
    {
      "activations/layer0_attention_weight_max": 16.03876495361328,
      "activations/layer0_attention_weight_min": -11.775924682617188,
      "activations/layer10_attention_weight_max": 33.76487731933594,
      "activations/layer10_attention_weight_min": -30.072214126586914,
      "activations/layer11_attention_weight_max": 31.34918212890625,
      "activations/layer11_attention_weight_min": -26.61966323852539,
      "activations/layer12_attention_weight_max": 21.065650939941406,
      "activations/layer12_attention_weight_min": -30.232654571533203,
      "activations/layer13_attention_weight_max": 41.89461135864258,
      "activations/layer13_attention_weight_min": -30.8017520904541,
      "activations/layer14_attention_weight_max": 53.17185592651367,
      "activations/layer14_attention_weight_min": -40.719139099121094,
      "activations/layer15_attention_weight_max": 35.706214904785156,
      "activations/layer15_attention_weight_min": -28.51630210876465,
      "activations/layer16_attention_weight_max": 29.121679306030273,
      "activations/layer16_attention_weight_min": -27.078157424926758,
      "activations/layer17_attention_weight_max": 36.6280517578125,
      "activations/layer17_attention_weight_min": -24.324689865112305,
      "activations/layer18_attention_weight_max": 28.101417541503906,
      "activations/layer18_attention_weight_min": -20.611112594604492,
      "activations/layer19_attention_weight_max": 37.28983688354492,
      "activations/layer19_attention_weight_min": -27.95346450805664,
      "activations/layer1_attention_weight_max": 16.657522201538086,
      "activations/layer1_attention_weight_min": -16.93206787109375,
      "activations/layer20_attention_weight_max": 28.89069175720215,
      "activations/layer20_attention_weight_min": -23.623943328857422,
      "activations/layer21_attention_weight_max": 28.156864166259766,
      "activations/layer21_attention_weight_min": -20.910776138305664,
      "activations/layer22_attention_weight_max": 44.013710021972656,
      "activations/layer22_attention_weight_min": -27.57320785522461,
      "activations/layer23_attention_weight_max": 33.10956573486328,
      "activations/layer23_attention_weight_min": -20.055925369262695,
      "activations/layer2_attention_weight_max": 32.984947204589844,
      "activations/layer2_attention_weight_min": -31.259855270385742,
      "activations/layer3_attention_weight_max": 90.56137084960938,
      "activations/layer3_attention_weight_min": -92.66885375976562,
      "activations/layer4_attention_weight_max": 127.24212646484375,
      "activations/layer4_attention_weight_min": -124.06316375732422,
      "activations/layer5_attention_weight_max": 70.63554382324219,
      "activations/layer5_attention_weight_min": -67.48292541503906,
      "activations/layer6_attention_weight_max": 56.429569244384766,
      "activations/layer6_attention_weight_min": -54.30531692504883,
      "activations/layer7_attention_weight_max": 86.60792541503906,
      "activations/layer7_attention_weight_min": -84.23214721679688,
      "activations/layer8_attention_weight_max": 39.471736907958984,
      "activations/layer8_attention_weight_min": -38.71152877807617,
      "activations/layer9_attention_weight_max": 38.049285888671875,
      "activations/layer9_attention_weight_min": -34.0255241394043,
      "epoch": 6.25,
      "learning_rate": 0.00011083333333333333,
      "loss": 2.8265,
      "step": 107500
    },
    {
      "activations/layer0_attention_weight_max": 15.867794036865234,
      "activations/layer0_attention_weight_min": -12.832074165344238,
      "activations/layer10_attention_weight_max": 36.04570388793945,
      "activations/layer10_attention_weight_min": -35.26860046386719,
      "activations/layer11_attention_weight_max": 34.95942687988281,
      "activations/layer11_attention_weight_min": -29.00029754638672,
      "activations/layer12_attention_weight_max": 22.311498641967773,
      "activations/layer12_attention_weight_min": -26.631912231445312,
      "activations/layer13_attention_weight_max": 42.20943832397461,
      "activations/layer13_attention_weight_min": -32.947383880615234,
      "activations/layer14_attention_weight_max": 72.6741943359375,
      "activations/layer14_attention_weight_min": -51.25257110595703,
      "activations/layer15_attention_weight_max": 38.17171859741211,
      "activations/layer15_attention_weight_min": -30.111003875732422,
      "activations/layer16_attention_weight_max": 37.16980743408203,
      "activations/layer16_attention_weight_min": -29.0771541595459,
      "activations/layer17_attention_weight_max": 38.95167922973633,
      "activations/layer17_attention_weight_min": -27.345163345336914,
      "activations/layer18_attention_weight_max": 32.673675537109375,
      "activations/layer18_attention_weight_min": -20.012447357177734,
      "activations/layer19_attention_weight_max": 39.683982849121094,
      "activations/layer19_attention_weight_min": -31.351598739624023,
      "activations/layer1_attention_weight_max": 17.859678268432617,
      "activations/layer1_attention_weight_min": -15.106512069702148,
      "activations/layer20_attention_weight_max": 34.91446304321289,
      "activations/layer20_attention_weight_min": -28.02876853942871,
      "activations/layer21_attention_weight_max": 32.310707092285156,
      "activations/layer21_attention_weight_min": -22.843338012695312,
      "activations/layer22_attention_weight_max": 54.3759765625,
      "activations/layer22_attention_weight_min": -33.968257904052734,
      "activations/layer23_attention_weight_max": 39.87347412109375,
      "activations/layer23_attention_weight_min": -22.45428466796875,
      "activations/layer2_attention_weight_max": 33.26258087158203,
      "activations/layer2_attention_weight_min": -32.608489990234375,
      "activations/layer3_attention_weight_max": 95.02410125732422,
      "activations/layer3_attention_weight_min": -95.60230255126953,
      "activations/layer4_attention_weight_max": 134.3606719970703,
      "activations/layer4_attention_weight_min": -133.33189392089844,
      "activations/layer5_attention_weight_max": 77.55780029296875,
      "activations/layer5_attention_weight_min": -72.38712310791016,
      "activations/layer6_attention_weight_max": 61.91781997680664,
      "activations/layer6_attention_weight_min": -55.89494323730469,
      "activations/layer7_attention_weight_max": 91.18467712402344,
      "activations/layer7_attention_weight_min": -93.93256378173828,
      "activations/layer8_attention_weight_max": 44.12076950073242,
      "activations/layer8_attention_weight_min": -40.65723419189453,
      "activations/layer9_attention_weight_max": 39.1036262512207,
      "activations/layer9_attention_weight_min": -35.07028579711914,
      "epoch": 6.25,
      "learning_rate": 0.00011081439393939392,
      "loss": 2.8311,
      "step": 107550
    },
    {
      "activations/layer0_attention_weight_max": 15.840161323547363,
      "activations/layer0_attention_weight_min": -12.25271224975586,
      "activations/layer10_attention_weight_max": 38.54793930053711,
      "activations/layer10_attention_weight_min": -36.0560188293457,
      "activations/layer11_attention_weight_max": 41.0614013671875,
      "activations/layer11_attention_weight_min": -33.43522644042969,
      "activations/layer12_attention_weight_max": 29.035036087036133,
      "activations/layer12_attention_weight_min": -26.258390426635742,
      "activations/layer13_attention_weight_max": 52.835960388183594,
      "activations/layer13_attention_weight_min": -38.128395080566406,
      "activations/layer14_attention_weight_max": 82.50981140136719,
      "activations/layer14_attention_weight_min": -49.0707893371582,
      "activations/layer15_attention_weight_max": 47.190574645996094,
      "activations/layer15_attention_weight_min": -33.11924362182617,
      "activations/layer16_attention_weight_max": 48.17430114746094,
      "activations/layer16_attention_weight_min": -29.96807861328125,
      "activations/layer17_attention_weight_max": 40.68460464477539,
      "activations/layer17_attention_weight_min": -26.688440322875977,
      "activations/layer18_attention_weight_max": 38.40696334838867,
      "activations/layer18_attention_weight_min": -23.863086700439453,
      "activations/layer19_attention_weight_max": 57.26919174194336,
      "activations/layer19_attention_weight_min": -32.96895217895508,
      "activations/layer1_attention_weight_max": 16.816158294677734,
      "activations/layer1_attention_weight_min": -12.462226867675781,
      "activations/layer20_attention_weight_max": 42.506492614746094,
      "activations/layer20_attention_weight_min": -25.056072235107422,
      "activations/layer21_attention_weight_max": 41.289031982421875,
      "activations/layer21_attention_weight_min": -21.795570373535156,
      "activations/layer22_attention_weight_max": 74.5942611694336,
      "activations/layer22_attention_weight_min": -31.532487869262695,
      "activations/layer23_attention_weight_max": 46.911861419677734,
      "activations/layer23_attention_weight_min": -20.412761688232422,
      "activations/layer2_attention_weight_max": 32.91586685180664,
      "activations/layer2_attention_weight_min": -30.793128967285156,
      "activations/layer3_attention_weight_max": 95.02586364746094,
      "activations/layer3_attention_weight_min": -99.66067504882812,
      "activations/layer4_attention_weight_max": 140.65382385253906,
      "activations/layer4_attention_weight_min": -136.2539520263672,
      "activations/layer5_attention_weight_max": 79.10891723632812,
      "activations/layer5_attention_weight_min": -72.80986785888672,
      "activations/layer6_attention_weight_max": 60.510562896728516,
      "activations/layer6_attention_weight_min": -58.04258728027344,
      "activations/layer7_attention_weight_max": 93.34501647949219,
      "activations/layer7_attention_weight_min": -98.70939636230469,
      "activations/layer8_attention_weight_max": 46.26737976074219,
      "activations/layer8_attention_weight_min": -44.90379333496094,
      "activations/layer9_attention_weight_max": 42.0885124206543,
      "activations/layer9_attention_weight_min": -38.77040100097656,
      "epoch": 6.25,
      "learning_rate": 0.00011079545454545454,
      "loss": 2.8303,
      "step": 107600
    },
    {
      "activations/layer0_attention_weight_max": 16.63858985900879,
      "activations/layer0_attention_weight_min": -11.7006254196167,
      "activations/layer10_attention_weight_max": 30.53934669494629,
      "activations/layer10_attention_weight_min": -31.886348724365234,
      "activations/layer11_attention_weight_max": 30.554500579833984,
      "activations/layer11_attention_weight_min": -26.97315788269043,
      "activations/layer12_attention_weight_max": 27.64016342163086,
      "activations/layer12_attention_weight_min": -29.01848602294922,
      "activations/layer13_attention_weight_max": 42.955928802490234,
      "activations/layer13_attention_weight_min": -32.36752700805664,
      "activations/layer14_attention_weight_max": 58.584510803222656,
      "activations/layer14_attention_weight_min": -42.63078689575195,
      "activations/layer15_attention_weight_max": 36.65204620361328,
      "activations/layer15_attention_weight_min": -28.8201847076416,
      "activations/layer16_attention_weight_max": 36.019222259521484,
      "activations/layer16_attention_weight_min": -27.34604835510254,
      "activations/layer17_attention_weight_max": 38.12825393676758,
      "activations/layer17_attention_weight_min": -27.045578002929688,
      "activations/layer18_attention_weight_max": 35.7711181640625,
      "activations/layer18_attention_weight_min": -21.804452896118164,
      "activations/layer19_attention_weight_max": 38.52970504760742,
      "activations/layer19_attention_weight_min": -31.70734977722168,
      "activations/layer1_attention_weight_max": 18.565567016601562,
      "activations/layer1_attention_weight_min": -15.978988647460938,
      "activations/layer20_attention_weight_max": 38.14665222167969,
      "activations/layer20_attention_weight_min": -25.356229782104492,
      "activations/layer21_attention_weight_max": 34.407867431640625,
      "activations/layer21_attention_weight_min": -22.049320220947266,
      "activations/layer22_attention_weight_max": 46.5719108581543,
      "activations/layer22_attention_weight_min": -30.702207565307617,
      "activations/layer23_attention_weight_max": 34.538753509521484,
      "activations/layer23_attention_weight_min": -21.42829704284668,
      "activations/layer2_attention_weight_max": 32.296241760253906,
      "activations/layer2_attention_weight_min": -31.48859405517578,
      "activations/layer3_attention_weight_max": 89.45134735107422,
      "activations/layer3_attention_weight_min": -89.92057037353516,
      "activations/layer4_attention_weight_max": 125.65091705322266,
      "activations/layer4_attention_weight_min": -122.99714660644531,
      "activations/layer5_attention_weight_max": 67.32313537597656,
      "activations/layer5_attention_weight_min": -67.79125213623047,
      "activations/layer6_attention_weight_max": 51.195884704589844,
      "activations/layer6_attention_weight_min": -53.07054901123047,
      "activations/layer7_attention_weight_max": 80.34260559082031,
      "activations/layer7_attention_weight_min": -80.95243072509766,
      "activations/layer8_attention_weight_max": 37.97662353515625,
      "activations/layer8_attention_weight_min": -40.17351531982422,
      "activations/layer9_attention_weight_max": 33.591121673583984,
      "activations/layer9_attention_weight_min": -36.927345275878906,
      "epoch": 6.26,
      "learning_rate": 0.00011077651515151515,
      "loss": 2.8255,
      "step": 107650
    },
    {
      "activations/layer0_attention_weight_max": 15.817097663879395,
      "activations/layer0_attention_weight_min": -13.429296493530273,
      "activations/layer10_attention_weight_max": 35.24789047241211,
      "activations/layer10_attention_weight_min": -32.11912536621094,
      "activations/layer11_attention_weight_max": 35.236968994140625,
      "activations/layer11_attention_weight_min": -29.392333984375,
      "activations/layer12_attention_weight_max": 33.594940185546875,
      "activations/layer12_attention_weight_min": -29.58913803100586,
      "activations/layer13_attention_weight_max": 44.87337112426758,
      "activations/layer13_attention_weight_min": -34.33962631225586,
      "activations/layer14_attention_weight_max": 65.6362533569336,
      "activations/layer14_attention_weight_min": -52.335853576660156,
      "activations/layer15_attention_weight_max": 40.631900787353516,
      "activations/layer15_attention_weight_min": -29.729629516601562,
      "activations/layer16_attention_weight_max": 37.897212982177734,
      "activations/layer16_attention_weight_min": -29.952085494995117,
      "activations/layer17_attention_weight_max": 39.78538131713867,
      "activations/layer17_attention_weight_min": -27.0506591796875,
      "activations/layer18_attention_weight_max": 35.84199142456055,
      "activations/layer18_attention_weight_min": -22.345306396484375,
      "activations/layer19_attention_weight_max": 41.328372955322266,
      "activations/layer19_attention_weight_min": -32.69685363769531,
      "activations/layer1_attention_weight_max": 17.451522827148438,
      "activations/layer1_attention_weight_min": -15.653029441833496,
      "activations/layer20_attention_weight_max": 33.93720626831055,
      "activations/layer20_attention_weight_min": -24.729625701904297,
      "activations/layer21_attention_weight_max": 37.07412338256836,
      "activations/layer21_attention_weight_min": -21.994918823242188,
      "activations/layer22_attention_weight_max": 50.20413589477539,
      "activations/layer22_attention_weight_min": -26.84532928466797,
      "activations/layer23_attention_weight_max": 36.57490921020508,
      "activations/layer23_attention_weight_min": -21.276168823242188,
      "activations/layer2_attention_weight_max": 33.02838897705078,
      "activations/layer2_attention_weight_min": -31.691253662109375,
      "activations/layer3_attention_weight_max": 89.21064758300781,
      "activations/layer3_attention_weight_min": -92.29428100585938,
      "activations/layer4_attention_weight_max": 127.83306121826172,
      "activations/layer4_attention_weight_min": -130.03363037109375,
      "activations/layer5_attention_weight_max": 69.67025756835938,
      "activations/layer5_attention_weight_min": -69.45508575439453,
      "activations/layer6_attention_weight_max": 54.211124420166016,
      "activations/layer6_attention_weight_min": -54.81000518798828,
      "activations/layer7_attention_weight_max": 91.87894439697266,
      "activations/layer7_attention_weight_min": -89.6875991821289,
      "activations/layer8_attention_weight_max": 41.39149856567383,
      "activations/layer8_attention_weight_min": -39.3183479309082,
      "activations/layer9_attention_weight_max": 37.05863952636719,
      "activations/layer9_attention_weight_min": -35.12209701538086,
      "epoch": 6.26,
      "learning_rate": 0.00011075757575757574,
      "loss": 2.8239,
      "step": 107700
    },
    {
      "activations/layer0_attention_weight_max": 16.045801162719727,
      "activations/layer0_attention_weight_min": -13.103381156921387,
      "activations/layer10_attention_weight_max": 33.11590576171875,
      "activations/layer10_attention_weight_min": -30.608060836791992,
      "activations/layer11_attention_weight_max": 30.624561309814453,
      "activations/layer11_attention_weight_min": -28.76557159423828,
      "activations/layer12_attention_weight_max": 22.5175838470459,
      "activations/layer12_attention_weight_min": -27.821855545043945,
      "activations/layer13_attention_weight_max": 42.617488861083984,
      "activations/layer13_attention_weight_min": -31.97538185119629,
      "activations/layer14_attention_weight_max": 59.78889465332031,
      "activations/layer14_attention_weight_min": -46.335594177246094,
      "activations/layer15_attention_weight_max": 37.63462448120117,
      "activations/layer15_attention_weight_min": -28.63496208190918,
      "activations/layer16_attention_weight_max": 35.26368713378906,
      "activations/layer16_attention_weight_min": -29.421405792236328,
      "activations/layer17_attention_weight_max": 36.18074417114258,
      "activations/layer17_attention_weight_min": -24.341392517089844,
      "activations/layer18_attention_weight_max": 34.071258544921875,
      "activations/layer18_attention_weight_min": -22.20595359802246,
      "activations/layer19_attention_weight_max": 39.16909408569336,
      "activations/layer19_attention_weight_min": -29.694107055664062,
      "activations/layer1_attention_weight_max": 17.073902130126953,
      "activations/layer1_attention_weight_min": -14.008522987365723,
      "activations/layer20_attention_weight_max": 29.785789489746094,
      "activations/layer20_attention_weight_min": -24.124345779418945,
      "activations/layer21_attention_weight_max": 29.8662166595459,
      "activations/layer21_attention_weight_min": -23.305889129638672,
      "activations/layer22_attention_weight_max": 47.182682037353516,
      "activations/layer22_attention_weight_min": -29.482656478881836,
      "activations/layer23_attention_weight_max": 37.0614013671875,
      "activations/layer23_attention_weight_min": -24.55303955078125,
      "activations/layer2_attention_weight_max": 30.92928123474121,
      "activations/layer2_attention_weight_min": -29.234111785888672,
      "activations/layer3_attention_weight_max": 85.64104461669922,
      "activations/layer3_attention_weight_min": -87.579833984375,
      "activations/layer4_attention_weight_max": 122.5203857421875,
      "activations/layer4_attention_weight_min": -119.3717041015625,
      "activations/layer5_attention_weight_max": 67.5282974243164,
      "activations/layer5_attention_weight_min": -67.17337036132812,
      "activations/layer6_attention_weight_max": 52.465328216552734,
      "activations/layer6_attention_weight_min": -53.214622497558594,
      "activations/layer7_attention_weight_max": 83.48833465576172,
      "activations/layer7_attention_weight_min": -82.70012664794922,
      "activations/layer8_attention_weight_max": 37.310707092285156,
      "activations/layer8_attention_weight_min": -38.61052322387695,
      "activations/layer9_attention_weight_max": 33.60909652709961,
      "activations/layer9_attention_weight_min": -33.44747543334961,
      "epoch": 6.26,
      "learning_rate": 0.00011073901515151513,
      "loss": 2.8549,
      "step": 107750
    },
    {
      "activations/layer0_attention_weight_max": 15.831568717956543,
      "activations/layer0_attention_weight_min": -13.429317474365234,
      "activations/layer10_attention_weight_max": 33.13844680786133,
      "activations/layer10_attention_weight_min": -29.87680435180664,
      "activations/layer11_attention_weight_max": 30.78106689453125,
      "activations/layer11_attention_weight_min": -28.09042739868164,
      "activations/layer12_attention_weight_max": 21.937896728515625,
      "activations/layer12_attention_weight_min": -27.826997756958008,
      "activations/layer13_attention_weight_max": 41.72666931152344,
      "activations/layer13_attention_weight_min": -33.05199432373047,
      "activations/layer14_attention_weight_max": 51.25550079345703,
      "activations/layer14_attention_weight_min": -40.364559173583984,
      "activations/layer15_attention_weight_max": 36.51401901245117,
      "activations/layer15_attention_weight_min": -28.451725006103516,
      "activations/layer16_attention_weight_max": 35.039886474609375,
      "activations/layer16_attention_weight_min": -28.724546432495117,
      "activations/layer17_attention_weight_max": 34.26997375488281,
      "activations/layer17_attention_weight_min": -25.547571182250977,
      "activations/layer18_attention_weight_max": 30.914091110229492,
      "activations/layer18_attention_weight_min": -21.118730545043945,
      "activations/layer19_attention_weight_max": 38.61797332763672,
      "activations/layer19_attention_weight_min": -28.352313995361328,
      "activations/layer1_attention_weight_max": 19.0794620513916,
      "activations/layer1_attention_weight_min": -13.367228507995605,
      "activations/layer20_attention_weight_max": 30.99058723449707,
      "activations/layer20_attention_weight_min": -22.222219467163086,
      "activations/layer21_attention_weight_max": 32.69548797607422,
      "activations/layer21_attention_weight_min": -21.299535751342773,
      "activations/layer22_attention_weight_max": 53.83810806274414,
      "activations/layer22_attention_weight_min": -29.44998550415039,
      "activations/layer23_attention_weight_max": 36.856651306152344,
      "activations/layer23_attention_weight_min": -21.94904136657715,
      "activations/layer2_attention_weight_max": 31.55780792236328,
      "activations/layer2_attention_weight_min": -29.640975952148438,
      "activations/layer3_attention_weight_max": 86.70690155029297,
      "activations/layer3_attention_weight_min": -86.7453842163086,
      "activations/layer4_attention_weight_max": 118.43873596191406,
      "activations/layer4_attention_weight_min": -116.79603576660156,
      "activations/layer5_attention_weight_max": 68.64694213867188,
      "activations/layer5_attention_weight_min": -66.64909362792969,
      "activations/layer6_attention_weight_max": 51.93206024169922,
      "activations/layer6_attention_weight_min": -53.16284942626953,
      "activations/layer7_attention_weight_max": 82.484619140625,
      "activations/layer7_attention_weight_min": -84.85258483886719,
      "activations/layer8_attention_weight_max": 39.892799377441406,
      "activations/layer8_attention_weight_min": -40.90181350708008,
      "activations/layer9_attention_weight_max": 35.93645095825195,
      "activations/layer9_attention_weight_min": -35.14250564575195,
      "epoch": 6.26,
      "learning_rate": 0.00011072007575757575,
      "loss": 2.8328,
      "step": 107800
    },
    {
      "activations/layer0_attention_weight_max": 16.403169631958008,
      "activations/layer0_attention_weight_min": -11.769143104553223,
      "activations/layer10_attention_weight_max": 35.780941009521484,
      "activations/layer10_attention_weight_min": -33.212703704833984,
      "activations/layer11_attention_weight_max": 40.65681457519531,
      "activations/layer11_attention_weight_min": -30.957996368408203,
      "activations/layer12_attention_weight_max": 23.579015731811523,
      "activations/layer12_attention_weight_min": -26.299827575683594,
      "activations/layer13_attention_weight_max": 43.362300872802734,
      "activations/layer13_attention_weight_min": -31.50140380859375,
      "activations/layer14_attention_weight_max": 65.36717224121094,
      "activations/layer14_attention_weight_min": -42.980438232421875,
      "activations/layer15_attention_weight_max": 39.166439056396484,
      "activations/layer15_attention_weight_min": -27.42600440979004,
      "activations/layer16_attention_weight_max": 37.0412483215332,
      "activations/layer16_attention_weight_min": -26.0137939453125,
      "activations/layer17_attention_weight_max": 35.63288497924805,
      "activations/layer17_attention_weight_min": -23.012760162353516,
      "activations/layer18_attention_weight_max": 30.3603572845459,
      "activations/layer18_attention_weight_min": -20.296653747558594,
      "activations/layer19_attention_weight_max": 34.73247528076172,
      "activations/layer19_attention_weight_min": -28.3670597076416,
      "activations/layer1_attention_weight_max": 17.34943962097168,
      "activations/layer1_attention_weight_min": -13.199899673461914,
      "activations/layer20_attention_weight_max": 27.470096588134766,
      "activations/layer20_attention_weight_min": -24.254497528076172,
      "activations/layer21_attention_weight_max": 27.58567237854004,
      "activations/layer21_attention_weight_min": -20.713714599609375,
      "activations/layer22_attention_weight_max": 47.67349624633789,
      "activations/layer22_attention_weight_min": -27.63578224182129,
      "activations/layer23_attention_weight_max": 33.96331787109375,
      "activations/layer23_attention_weight_min": -20.079727172851562,
      "activations/layer2_attention_weight_max": 31.88738250732422,
      "activations/layer2_attention_weight_min": -30.153087615966797,
      "activations/layer3_attention_weight_max": 92.00668334960938,
      "activations/layer3_attention_weight_min": -93.14835357666016,
      "activations/layer4_attention_weight_max": 123.93572998046875,
      "activations/layer4_attention_weight_min": -121.54424285888672,
      "activations/layer5_attention_weight_max": 67.69361877441406,
      "activations/layer5_attention_weight_min": -68.54472351074219,
      "activations/layer6_attention_weight_max": 53.39535140991211,
      "activations/layer6_attention_weight_min": -57.05472946166992,
      "activations/layer7_attention_weight_max": 94.54437255859375,
      "activations/layer7_attention_weight_min": -94.63506317138672,
      "activations/layer8_attention_weight_max": 43.840965270996094,
      "activations/layer8_attention_weight_min": -42.445594787597656,
      "activations/layer9_attention_weight_max": 38.449806213378906,
      "activations/layer9_attention_weight_min": -34.60989761352539,
      "epoch": 6.27,
      "learning_rate": 0.00011070113636363636,
      "loss": 2.8218,
      "step": 107850
    },
    {
      "activations/layer0_attention_weight_max": 15.236337661743164,
      "activations/layer0_attention_weight_min": -11.908164978027344,
      "activations/layer10_attention_weight_max": 32.384437561035156,
      "activations/layer10_attention_weight_min": -30.176799774169922,
      "activations/layer11_attention_weight_max": 32.926605224609375,
      "activations/layer11_attention_weight_min": -28.85900115966797,
      "activations/layer12_attention_weight_max": 29.986621856689453,
      "activations/layer12_attention_weight_min": -26.3080997467041,
      "activations/layer13_attention_weight_max": 35.853431701660156,
      "activations/layer13_attention_weight_min": -31.366992950439453,
      "activations/layer14_attention_weight_max": 51.84257125854492,
      "activations/layer14_attention_weight_min": -45.378204345703125,
      "activations/layer15_attention_weight_max": 36.05314636230469,
      "activations/layer15_attention_weight_min": -28.875337600708008,
      "activations/layer16_attention_weight_max": 33.48892593383789,
      "activations/layer16_attention_weight_min": -28.120189666748047,
      "activations/layer17_attention_weight_max": 32.73506164550781,
      "activations/layer17_attention_weight_min": -24.999263763427734,
      "activations/layer18_attention_weight_max": 31.246374130249023,
      "activations/layer18_attention_weight_min": -19.262022018432617,
      "activations/layer19_attention_weight_max": 37.66804885864258,
      "activations/layer19_attention_weight_min": -29.014036178588867,
      "activations/layer1_attention_weight_max": 16.851871490478516,
      "activations/layer1_attention_weight_min": -12.723921775817871,
      "activations/layer20_attention_weight_max": 31.41790771484375,
      "activations/layer20_attention_weight_min": -22.824491500854492,
      "activations/layer21_attention_weight_max": 31.69637680053711,
      "activations/layer21_attention_weight_min": -21.587289810180664,
      "activations/layer22_attention_weight_max": 46.59881591796875,
      "activations/layer22_attention_weight_min": -29.750900268554688,
      "activations/layer23_attention_weight_max": 32.00654602050781,
      "activations/layer23_attention_weight_min": -23.708654403686523,
      "activations/layer2_attention_weight_max": 32.79310607910156,
      "activations/layer2_attention_weight_min": -29.58287811279297,
      "activations/layer3_attention_weight_max": 84.38780212402344,
      "activations/layer3_attention_weight_min": -86.6969985961914,
      "activations/layer4_attention_weight_max": 124.63506317138672,
      "activations/layer4_attention_weight_min": -120.3056869506836,
      "activations/layer5_attention_weight_max": 66.40831756591797,
      "activations/layer5_attention_weight_min": -64.26667022705078,
      "activations/layer6_attention_weight_max": 54.698787689208984,
      "activations/layer6_attention_weight_min": -52.73818588256836,
      "activations/layer7_attention_weight_max": 84.17191314697266,
      "activations/layer7_attention_weight_min": -84.72821807861328,
      "activations/layer8_attention_weight_max": 39.431884765625,
      "activations/layer8_attention_weight_min": -42.181583404541016,
      "activations/layer9_attention_weight_max": 33.834266662597656,
      "activations/layer9_attention_weight_min": -35.15461349487305,
      "epoch": 6.27,
      "learning_rate": 0.00011068219696969696,
      "loss": 2.8265,
      "step": 107900
    },
    {
      "activations/layer0_attention_weight_max": 16.12336540222168,
      "activations/layer0_attention_weight_min": -13.31419563293457,
      "activations/layer10_attention_weight_max": 33.9174919128418,
      "activations/layer10_attention_weight_min": -30.477413177490234,
      "activations/layer11_attention_weight_max": 30.761760711669922,
      "activations/layer11_attention_weight_min": -27.969938278198242,
      "activations/layer12_attention_weight_max": 24.411603927612305,
      "activations/layer12_attention_weight_min": -23.0461483001709,
      "activations/layer13_attention_weight_max": 40.41913604736328,
      "activations/layer13_attention_weight_min": -34.89423370361328,
      "activations/layer14_attention_weight_max": 54.98301315307617,
      "activations/layer14_attention_weight_min": -47.855106353759766,
      "activations/layer15_attention_weight_max": 34.21427536010742,
      "activations/layer15_attention_weight_min": -27.645008087158203,
      "activations/layer16_attention_weight_max": 33.32063293457031,
      "activations/layer16_attention_weight_min": -27.87969398498535,
      "activations/layer17_attention_weight_max": 36.35130310058594,
      "activations/layer17_attention_weight_min": -23.939876556396484,
      "activations/layer18_attention_weight_max": 29.12653160095215,
      "activations/layer18_attention_weight_min": -20.626935958862305,
      "activations/layer19_attention_weight_max": 36.01374435424805,
      "activations/layer19_attention_weight_min": -29.598112106323242,
      "activations/layer1_attention_weight_max": 17.228347778320312,
      "activations/layer1_attention_weight_min": -13.963262557983398,
      "activations/layer20_attention_weight_max": 29.687164306640625,
      "activations/layer20_attention_weight_min": -22.637104034423828,
      "activations/layer21_attention_weight_max": 31.990428924560547,
      "activations/layer21_attention_weight_min": -20.736196517944336,
      "activations/layer22_attention_weight_max": 45.36323547363281,
      "activations/layer22_attention_weight_min": -26.253522872924805,
      "activations/layer23_attention_weight_max": 32.32365417480469,
      "activations/layer23_attention_weight_min": -20.911407470703125,
      "activations/layer2_attention_weight_max": 31.967405319213867,
      "activations/layer2_attention_weight_min": -29.390579223632812,
      "activations/layer3_attention_weight_max": 85.50653076171875,
      "activations/layer3_attention_weight_min": -86.33079528808594,
      "activations/layer4_attention_weight_max": 120.7696762084961,
      "activations/layer4_attention_weight_min": -118.97015380859375,
      "activations/layer5_attention_weight_max": 67.03041076660156,
      "activations/layer5_attention_weight_min": -66.66835021972656,
      "activations/layer6_attention_weight_max": 52.38428497314453,
      "activations/layer6_attention_weight_min": -55.413818359375,
      "activations/layer7_attention_weight_max": 87.40619659423828,
      "activations/layer7_attention_weight_min": -87.15473175048828,
      "activations/layer8_attention_weight_max": 39.07540512084961,
      "activations/layer8_attention_weight_min": -41.417083740234375,
      "activations/layer9_attention_weight_max": 35.209075927734375,
      "activations/layer9_attention_weight_min": -34.520164489746094,
      "epoch": 6.27,
      "learning_rate": 0.00011066325757575756,
      "loss": 2.8411,
      "step": 107950
    },
    {
      "activations/layer0_attention_weight_max": 16.334205627441406,
      "activations/layer0_attention_weight_min": -13.216326713562012,
      "activations/layer10_attention_weight_max": 30.785837173461914,
      "activations/layer10_attention_weight_min": -29.94833755493164,
      "activations/layer11_attention_weight_max": 31.84126091003418,
      "activations/layer11_attention_weight_min": -28.525249481201172,
      "activations/layer12_attention_weight_max": 62.39692306518555,
      "activations/layer12_attention_weight_min": -27.67310333251953,
      "activations/layer13_attention_weight_max": 52.22453689575195,
      "activations/layer13_attention_weight_min": -31.356735229492188,
      "activations/layer14_attention_weight_max": 57.6820182800293,
      "activations/layer14_attention_weight_min": -40.82100296020508,
      "activations/layer15_attention_weight_max": 40.63167190551758,
      "activations/layer15_attention_weight_min": -28.525175094604492,
      "activations/layer16_attention_weight_max": 39.83136749267578,
      "activations/layer16_attention_weight_min": -27.94985008239746,
      "activations/layer17_attention_weight_max": 40.49828338623047,
      "activations/layer17_attention_weight_min": -23.846778869628906,
      "activations/layer18_attention_weight_max": 34.04745101928711,
      "activations/layer18_attention_weight_min": -24.768123626708984,
      "activations/layer19_attention_weight_max": 41.893821716308594,
      "activations/layer19_attention_weight_min": -29.34187126159668,
      "activations/layer1_attention_weight_max": 16.317941665649414,
      "activations/layer1_attention_weight_min": -13.188589096069336,
      "activations/layer20_attention_weight_max": 32.29663848876953,
      "activations/layer20_attention_weight_min": -23.620935440063477,
      "activations/layer21_attention_weight_max": 33.9506721496582,
      "activations/layer21_attention_weight_min": -21.587263107299805,
      "activations/layer22_attention_weight_max": 52.63760757446289,
      "activations/layer22_attention_weight_min": -28.92500114440918,
      "activations/layer23_attention_weight_max": 35.443572998046875,
      "activations/layer23_attention_weight_min": -21.970932006835938,
      "activations/layer2_attention_weight_max": 30.770488739013672,
      "activations/layer2_attention_weight_min": -30.113792419433594,
      "activations/layer3_attention_weight_max": 88.39558410644531,
      "activations/layer3_attention_weight_min": -88.66080474853516,
      "activations/layer4_attention_weight_max": 122.98497772216797,
      "activations/layer4_attention_weight_min": -121.4256591796875,
      "activations/layer5_attention_weight_max": 68.54861450195312,
      "activations/layer5_attention_weight_min": -65.51863861083984,
      "activations/layer6_attention_weight_max": 54.547489166259766,
      "activations/layer6_attention_weight_min": -52.73975372314453,
      "activations/layer7_attention_weight_max": 91.8824234008789,
      "activations/layer7_attention_weight_min": -81.13163757324219,
      "activations/layer8_attention_weight_max": 39.41054153442383,
      "activations/layer8_attention_weight_min": -38.422325134277344,
      "activations/layer9_attention_weight_max": 34.151973724365234,
      "activations/layer9_attention_weight_min": -33.24201202392578,
      "epoch": 6.28,
      "learning_rate": 0.00011064431818181817,
      "loss": 2.8339,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_loss": 2.78125,
      "eval_runtime": 8.4338,
      "eval_samples_per_second": 509.139,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_openwebtext_loss": 2.78125,
      "eval_openwebtext_ppl": 16.139182322446413,
      "eval_openwebtext_runtime": 8.4338,
      "eval_openwebtext_samples_per_second": 509.139,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 1.9937,
      "eval_wikitext_samples_per_second": 228.722,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_lambada_loss": 2.59375,
      "eval_lambada_ppl": 13.379852088930456,
      "eval_lambada_runtime": 9.5205,
      "eval_lambada_samples_per_second": 511.422,
      "step": 108000
    },
    {
      "activations/layer0_attention_weight_max": 15.636214256286621,
      "activations/layer0_attention_weight_min": -13.613235473632812,
      "activations/layer10_attention_weight_max": 36.612831115722656,
      "activations/layer10_attention_weight_min": -31.584453582763672,
      "activations/layer11_attention_weight_max": 33.865264892578125,
      "activations/layer11_attention_weight_min": -30.981809616088867,
      "activations/layer12_attention_weight_max": 27.64600944519043,
      "activations/layer12_attention_weight_min": -24.72410774230957,
      "activations/layer13_attention_weight_max": 40.36997604370117,
      "activations/layer13_attention_weight_min": -32.34336471557617,
      "activations/layer14_attention_weight_max": 56.5667610168457,
      "activations/layer14_attention_weight_min": -44.868247985839844,
      "activations/layer15_attention_weight_max": 37.62986755371094,
      "activations/layer15_attention_weight_min": -28.424468994140625,
      "activations/layer16_attention_weight_max": 34.07700729370117,
      "activations/layer16_attention_weight_min": -26.55782127380371,
      "activations/layer17_attention_weight_max": 38.735782623291016,
      "activations/layer17_attention_weight_min": -25.069496154785156,
      "activations/layer18_attention_weight_max": 30.729604721069336,
      "activations/layer18_attention_weight_min": -20.581201553344727,
      "activations/layer19_attention_weight_max": 37.632137298583984,
      "activations/layer19_attention_weight_min": -29.088367462158203,
      "activations/layer1_attention_weight_max": 16.507360458374023,
      "activations/layer1_attention_weight_min": -14.09385871887207,
      "activations/layer20_attention_weight_max": 33.954620361328125,
      "activations/layer20_attention_weight_min": -24.735517501831055,
      "activations/layer21_attention_weight_max": 30.94801902770996,
      "activations/layer21_attention_weight_min": -21.755950927734375,
      "activations/layer22_attention_weight_max": 56.12417984008789,
      "activations/layer22_attention_weight_min": -31.265060424804688,
      "activations/layer23_attention_weight_max": 37.37849044799805,
      "activations/layer23_attention_weight_min": -21.27180290222168,
      "activations/layer2_attention_weight_max": 30.720043182373047,
      "activations/layer2_attention_weight_min": -29.479808807373047,
      "activations/layer3_attention_weight_max": 87.8207778930664,
      "activations/layer3_attention_weight_min": -91.2028579711914,
      "activations/layer4_attention_weight_max": 123.03340148925781,
      "activations/layer4_attention_weight_min": -122.24456024169922,
      "activations/layer5_attention_weight_max": 70.06620788574219,
      "activations/layer5_attention_weight_min": -64.83946990966797,
      "activations/layer6_attention_weight_max": 51.417762756347656,
      "activations/layer6_attention_weight_min": -52.16866683959961,
      "activations/layer7_attention_weight_max": 87.8281021118164,
      "activations/layer7_attention_weight_min": -85.92668151855469,
      "activations/layer8_attention_weight_max": 41.84147262573242,
      "activations/layer8_attention_weight_min": -40.8150520324707,
      "activations/layer9_attention_weight_max": 40.7524299621582,
      "activations/layer9_attention_weight_min": -34.2918815612793,
      "epoch": 6.28,
      "learning_rate": 0.00011062537878787878,
      "loss": 2.8396,
      "step": 108050
    },
    {
      "activations/layer0_attention_weight_max": 15.841111183166504,
      "activations/layer0_attention_weight_min": -13.900301933288574,
      "activations/layer10_attention_weight_max": 34.71416091918945,
      "activations/layer10_attention_weight_min": -32.418968200683594,
      "activations/layer11_attention_weight_max": 34.14528274536133,
      "activations/layer11_attention_weight_min": -28.60443878173828,
      "activations/layer12_attention_weight_max": 30.618946075439453,
      "activations/layer12_attention_weight_min": -29.724912643432617,
      "activations/layer13_attention_weight_max": 40.52937316894531,
      "activations/layer13_attention_weight_min": -32.530391693115234,
      "activations/layer14_attention_weight_max": 61.17807388305664,
      "activations/layer14_attention_weight_min": -43.44638442993164,
      "activations/layer15_attention_weight_max": 38.241600036621094,
      "activations/layer15_attention_weight_min": -29.11012840270996,
      "activations/layer16_attention_weight_max": 38.54472351074219,
      "activations/layer16_attention_weight_min": -27.22439193725586,
      "activations/layer17_attention_weight_max": 41.47966766357422,
      "activations/layer17_attention_weight_min": -24.331323623657227,
      "activations/layer18_attention_weight_max": 34.142784118652344,
      "activations/layer18_attention_weight_min": -21.16619110107422,
      "activations/layer19_attention_weight_max": 39.65401840209961,
      "activations/layer19_attention_weight_min": -30.54490089416504,
      "activations/layer1_attention_weight_max": 16.671829223632812,
      "activations/layer1_attention_weight_min": -14.120615005493164,
      "activations/layer20_attention_weight_max": 32.36248779296875,
      "activations/layer20_attention_weight_min": -23.05489730834961,
      "activations/layer21_attention_weight_max": 35.218482971191406,
      "activations/layer21_attention_weight_min": -23.485515594482422,
      "activations/layer22_attention_weight_max": 54.22409439086914,
      "activations/layer22_attention_weight_min": -31.25861930847168,
      "activations/layer23_attention_weight_max": 37.53572082519531,
      "activations/layer23_attention_weight_min": -22.26751708984375,
      "activations/layer2_attention_weight_max": 32.08760452270508,
      "activations/layer2_attention_weight_min": -31.154714584350586,
      "activations/layer3_attention_weight_max": 91.45751953125,
      "activations/layer3_attention_weight_min": -93.14366912841797,
      "activations/layer4_attention_weight_max": 128.07815551757812,
      "activations/layer4_attention_weight_min": -121.16451263427734,
      "activations/layer5_attention_weight_max": 71.80233764648438,
      "activations/layer5_attention_weight_min": -67.34516143798828,
      "activations/layer6_attention_weight_max": 55.44623947143555,
      "activations/layer6_attention_weight_min": -54.11404800415039,
      "activations/layer7_attention_weight_max": 89.04798126220703,
      "activations/layer7_attention_weight_min": -88.1973648071289,
      "activations/layer8_attention_weight_max": 42.19137191772461,
      "activations/layer8_attention_weight_min": -40.427555084228516,
      "activations/layer9_attention_weight_max": 36.91667938232422,
      "activations/layer9_attention_weight_min": -35.448158264160156,
      "epoch": 6.28,
      "learning_rate": 0.00011060643939393938,
      "loss": 2.8296,
      "step": 108100
    },
    {
      "activations/layer0_attention_weight_max": 14.940606117248535,
      "activations/layer0_attention_weight_min": -11.297462463378906,
      "activations/layer10_attention_weight_max": 33.68722915649414,
      "activations/layer10_attention_weight_min": -32.74883270263672,
      "activations/layer11_attention_weight_max": 32.96457290649414,
      "activations/layer11_attention_weight_min": -31.18794822692871,
      "activations/layer12_attention_weight_max": 33.46901321411133,
      "activations/layer12_attention_weight_min": -32.20448303222656,
      "activations/layer13_attention_weight_max": 56.05583572387695,
      "activations/layer13_attention_weight_min": -38.88496398925781,
      "activations/layer14_attention_weight_max": 51.84928894042969,
      "activations/layer14_attention_weight_min": -44.22007751464844,
      "activations/layer15_attention_weight_max": 34.41081237792969,
      "activations/layer15_attention_weight_min": -29.763086318969727,
      "activations/layer16_attention_weight_max": 34.55252456665039,
      "activations/layer16_attention_weight_min": -26.696504592895508,
      "activations/layer17_attention_weight_max": 32.10492706298828,
      "activations/layer17_attention_weight_min": -25.913917541503906,
      "activations/layer18_attention_weight_max": 33.12282180786133,
      "activations/layer18_attention_weight_min": -20.63218879699707,
      "activations/layer19_attention_weight_max": 40.225948333740234,
      "activations/layer19_attention_weight_min": -27.89798355102539,
      "activations/layer1_attention_weight_max": 18.355161666870117,
      "activations/layer1_attention_weight_min": -14.24216079711914,
      "activations/layer20_attention_weight_max": 29.51862144470215,
      "activations/layer20_attention_weight_min": -22.545127868652344,
      "activations/layer21_attention_weight_max": 29.973312377929688,
      "activations/layer21_attention_weight_min": -21.72893524169922,
      "activations/layer22_attention_weight_max": 44.310550689697266,
      "activations/layer22_attention_weight_min": -27.23839569091797,
      "activations/layer23_attention_weight_max": 36.11781311035156,
      "activations/layer23_attention_weight_min": -21.75829315185547,
      "activations/layer2_attention_weight_max": 33.96154022216797,
      "activations/layer2_attention_weight_min": -31.54159164428711,
      "activations/layer3_attention_weight_max": 96.4455795288086,
      "activations/layer3_attention_weight_min": -94.61442565917969,
      "activations/layer4_attention_weight_max": 130.80938720703125,
      "activations/layer4_attention_weight_min": -125.67317962646484,
      "activations/layer5_attention_weight_max": 71.68374633789062,
      "activations/layer5_attention_weight_min": -68.41925048828125,
      "activations/layer6_attention_weight_max": 51.696781158447266,
      "activations/layer6_attention_weight_min": -54.89396667480469,
      "activations/layer7_attention_weight_max": 93.54724884033203,
      "activations/layer7_attention_weight_min": -85.54524230957031,
      "activations/layer8_attention_weight_max": 41.484764099121094,
      "activations/layer8_attention_weight_min": -42.44282150268555,
      "activations/layer9_attention_weight_max": 35.060707092285156,
      "activations/layer9_attention_weight_min": -38.31624221801758,
      "epoch": 6.28,
      "learning_rate": 0.00011058749999999998,
      "loss": 2.8258,
      "step": 108150
    },
    {
      "activations/layer0_attention_weight_max": 16.81226348876953,
      "activations/layer0_attention_weight_min": -12.333500862121582,
      "activations/layer10_attention_weight_max": 35.022823333740234,
      "activations/layer10_attention_weight_min": -32.902183532714844,
      "activations/layer11_attention_weight_max": 36.931549072265625,
      "activations/layer11_attention_weight_min": -28.695419311523438,
      "activations/layer12_attention_weight_max": 38.3913688659668,
      "activations/layer12_attention_weight_min": -27.19827651977539,
      "activations/layer13_attention_weight_max": 44.79717254638672,
      "activations/layer13_attention_weight_min": -33.90475845336914,
      "activations/layer14_attention_weight_max": 60.99602127075195,
      "activations/layer14_attention_weight_min": -43.71229553222656,
      "activations/layer15_attention_weight_max": 46.32756423950195,
      "activations/layer15_attention_weight_min": -32.00445556640625,
      "activations/layer16_attention_weight_max": 34.14180374145508,
      "activations/layer16_attention_weight_min": -25.664901733398438,
      "activations/layer17_attention_weight_max": 38.238258361816406,
      "activations/layer17_attention_weight_min": -22.935781478881836,
      "activations/layer18_attention_weight_max": 33.10955047607422,
      "activations/layer18_attention_weight_min": -18.819408416748047,
      "activations/layer19_attention_weight_max": 38.96505355834961,
      "activations/layer19_attention_weight_min": -28.783517837524414,
      "activations/layer1_attention_weight_max": 16.668603897094727,
      "activations/layer1_attention_weight_min": -15.71247386932373,
      "activations/layer20_attention_weight_max": 33.177303314208984,
      "activations/layer20_attention_weight_min": -23.645240783691406,
      "activations/layer21_attention_weight_max": 37.411163330078125,
      "activations/layer21_attention_weight_min": -19.89019012451172,
      "activations/layer22_attention_weight_max": 52.21784210205078,
      "activations/layer22_attention_weight_min": -28.171865463256836,
      "activations/layer23_attention_weight_max": 34.94325256347656,
      "activations/layer23_attention_weight_min": -20.275283813476562,
      "activations/layer2_attention_weight_max": 32.831214904785156,
      "activations/layer2_attention_weight_min": -31.906944274902344,
      "activations/layer3_attention_weight_max": 94.234619140625,
      "activations/layer3_attention_weight_min": -92.49829864501953,
      "activations/layer4_attention_weight_max": 136.07858276367188,
      "activations/layer4_attention_weight_min": -124.355224609375,
      "activations/layer5_attention_weight_max": 74.19645690917969,
      "activations/layer5_attention_weight_min": -68.78031158447266,
      "activations/layer6_attention_weight_max": 57.9056396484375,
      "activations/layer6_attention_weight_min": -55.59528350830078,
      "activations/layer7_attention_weight_max": 91.43572998046875,
      "activations/layer7_attention_weight_min": -90.5264663696289,
      "activations/layer8_attention_weight_max": 43.65237808227539,
      "activations/layer8_attention_weight_min": -42.38591384887695,
      "activations/layer9_attention_weight_max": 39.697444915771484,
      "activations/layer9_attention_weight_min": -36.792747497558594,
      "epoch": 6.29,
      "learning_rate": 0.0001105685606060606,
      "loss": 2.8192,
      "step": 108200
    },
    {
      "activations/layer0_attention_weight_max": 15.676491737365723,
      "activations/layer0_attention_weight_min": -12.216313362121582,
      "activations/layer10_attention_weight_max": 30.899465560913086,
      "activations/layer10_attention_weight_min": -32.60883712768555,
      "activations/layer11_attention_weight_max": 31.05583953857422,
      "activations/layer11_attention_weight_min": -32.063629150390625,
      "activations/layer12_attention_weight_max": 38.039398193359375,
      "activations/layer12_attention_weight_min": -31.3411808013916,
      "activations/layer13_attention_weight_max": 44.2452507019043,
      "activations/layer13_attention_weight_min": -31.891468048095703,
      "activations/layer14_attention_weight_max": 59.26107406616211,
      "activations/layer14_attention_weight_min": -47.53916549682617,
      "activations/layer15_attention_weight_max": 35.037845611572266,
      "activations/layer15_attention_weight_min": -29.022672653198242,
      "activations/layer16_attention_weight_max": 32.26935958862305,
      "activations/layer16_attention_weight_min": -26.895998001098633,
      "activations/layer17_attention_weight_max": 32.04788589477539,
      "activations/layer17_attention_weight_min": -24.296018600463867,
      "activations/layer18_attention_weight_max": 28.464757919311523,
      "activations/layer18_attention_weight_min": -22.477458953857422,
      "activations/layer19_attention_weight_max": 35.29304885864258,
      "activations/layer19_attention_weight_min": -27.704687118530273,
      "activations/layer1_attention_weight_max": 17.851255416870117,
      "activations/layer1_attention_weight_min": -14.680534362792969,
      "activations/layer20_attention_weight_max": 27.872936248779297,
      "activations/layer20_attention_weight_min": -23.20553970336914,
      "activations/layer21_attention_weight_max": 28.818021774291992,
      "activations/layer21_attention_weight_min": -21.317834854125977,
      "activations/layer22_attention_weight_max": 43.02124786376953,
      "activations/layer22_attention_weight_min": -29.64282989501953,
      "activations/layer23_attention_weight_max": 34.048561096191406,
      "activations/layer23_attention_weight_min": -24.33530044555664,
      "activations/layer2_attention_weight_max": 34.00762176513672,
      "activations/layer2_attention_weight_min": -32.172027587890625,
      "activations/layer3_attention_weight_max": 89.23204803466797,
      "activations/layer3_attention_weight_min": -93.73326873779297,
      "activations/layer4_attention_weight_max": 129.63327026367188,
      "activations/layer4_attention_weight_min": -125.20833587646484,
      "activations/layer5_attention_weight_max": 72.72636413574219,
      "activations/layer5_attention_weight_min": -66.23251342773438,
      "activations/layer6_attention_weight_max": 50.687522888183594,
      "activations/layer6_attention_weight_min": -54.176536560058594,
      "activations/layer7_attention_weight_max": 83.37336730957031,
      "activations/layer7_attention_weight_min": -81.89962768554688,
      "activations/layer8_attention_weight_max": 38.380001068115234,
      "activations/layer8_attention_weight_min": -41.060855865478516,
      "activations/layer9_attention_weight_max": 35.491722106933594,
      "activations/layer9_attention_weight_min": -33.39053726196289,
      "epoch": 6.29,
      "learning_rate": 0.00011054962121212121,
      "loss": 2.8103,
      "step": 108250
    },
    {
      "activations/layer0_attention_weight_max": 15.476664543151855,
      "activations/layer0_attention_weight_min": -12.601164817810059,
      "activations/layer10_attention_weight_max": 41.5369873046875,
      "activations/layer10_attention_weight_min": -34.58294677734375,
      "activations/layer11_attention_weight_max": 42.56865692138672,
      "activations/layer11_attention_weight_min": -32.18402862548828,
      "activations/layer12_attention_weight_max": 46.26992416381836,
      "activations/layer12_attention_weight_min": -27.15219497680664,
      "activations/layer13_attention_weight_max": 53.784584045410156,
      "activations/layer13_attention_weight_min": -32.799835205078125,
      "activations/layer14_attention_weight_max": 59.176326751708984,
      "activations/layer14_attention_weight_min": -39.416500091552734,
      "activations/layer15_attention_weight_max": 40.17280960083008,
      "activations/layer15_attention_weight_min": -29.473474502563477,
      "activations/layer16_attention_weight_max": 39.55785369873047,
      "activations/layer16_attention_weight_min": -29.174667358398438,
      "activations/layer17_attention_weight_max": 41.19349670410156,
      "activations/layer17_attention_weight_min": -26.84966278076172,
      "activations/layer18_attention_weight_max": 35.81517791748047,
      "activations/layer18_attention_weight_min": -21.37282371520996,
      "activations/layer19_attention_weight_max": 40.13848876953125,
      "activations/layer19_attention_weight_min": -30.17474365234375,
      "activations/layer1_attention_weight_max": 17.143983840942383,
      "activations/layer1_attention_weight_min": -14.512676239013672,
      "activations/layer20_attention_weight_max": 32.865047454833984,
      "activations/layer20_attention_weight_min": -24.172767639160156,
      "activations/layer21_attention_weight_max": 29.376731872558594,
      "activations/layer21_attention_weight_min": -21.070009231567383,
      "activations/layer22_attention_weight_max": 54.287254333496094,
      "activations/layer22_attention_weight_min": -27.778900146484375,
      "activations/layer23_attention_weight_max": 35.63658142089844,
      "activations/layer23_attention_weight_min": -19.783370971679688,
      "activations/layer2_attention_weight_max": 33.84368896484375,
      "activations/layer2_attention_weight_min": -30.969938278198242,
      "activations/layer3_attention_weight_max": 98.6462631225586,
      "activations/layer3_attention_weight_min": -94.86949157714844,
      "activations/layer4_attention_weight_max": 142.45167541503906,
      "activations/layer4_attention_weight_min": -128.58226013183594,
      "activations/layer5_attention_weight_max": 77.0759506225586,
      "activations/layer5_attention_weight_min": -70.39280700683594,
      "activations/layer6_attention_weight_max": 59.655601501464844,
      "activations/layer6_attention_weight_min": -54.493797302246094,
      "activations/layer7_attention_weight_max": 96.80109405517578,
      "activations/layer7_attention_weight_min": -87.8961410522461,
      "activations/layer8_attention_weight_max": 46.93790817260742,
      "activations/layer8_attention_weight_min": -41.18385314941406,
      "activations/layer9_attention_weight_max": 41.34294891357422,
      "activations/layer9_attention_weight_min": -36.012245178222656,
      "epoch": 6.29,
      "learning_rate": 0.0001105306818181818,
      "loss": 2.836,
      "step": 108300
    },
    {
      "activations/layer0_attention_weight_max": 16.818723678588867,
      "activations/layer0_attention_weight_min": -13.477204322814941,
      "activations/layer10_attention_weight_max": 31.763317108154297,
      "activations/layer10_attention_weight_min": -30.75109100341797,
      "activations/layer11_attention_weight_max": 30.732357025146484,
      "activations/layer11_attention_weight_min": -27.539098739624023,
      "activations/layer12_attention_weight_max": 30.59231948852539,
      "activations/layer12_attention_weight_min": -25.948780059814453,
      "activations/layer13_attention_weight_max": 39.59751510620117,
      "activations/layer13_attention_weight_min": -29.864885330200195,
      "activations/layer14_attention_weight_max": 63.24463653564453,
      "activations/layer14_attention_weight_min": -42.18918991088867,
      "activations/layer15_attention_weight_max": 35.10610580444336,
      "activations/layer15_attention_weight_min": -28.112049102783203,
      "activations/layer16_attention_weight_max": 35.128902435302734,
      "activations/layer16_attention_weight_min": -26.955106735229492,
      "activations/layer17_attention_weight_max": 37.528167724609375,
      "activations/layer17_attention_weight_min": -23.973068237304688,
      "activations/layer18_attention_weight_max": 30.0496883392334,
      "activations/layer18_attention_weight_min": -21.34429168701172,
      "activations/layer19_attention_weight_max": 37.59519958496094,
      "activations/layer19_attention_weight_min": -29.927886962890625,
      "activations/layer1_attention_weight_max": 17.290096282958984,
      "activations/layer1_attention_weight_min": -14.557612419128418,
      "activations/layer20_attention_weight_max": 29.76097869873047,
      "activations/layer20_attention_weight_min": -24.08126449584961,
      "activations/layer21_attention_weight_max": 32.171630859375,
      "activations/layer21_attention_weight_min": -21.760360717773438,
      "activations/layer22_attention_weight_max": 45.46622085571289,
      "activations/layer22_attention_weight_min": -28.546995162963867,
      "activations/layer23_attention_weight_max": 32.34537124633789,
      "activations/layer23_attention_weight_min": -22.97676658630371,
      "activations/layer2_attention_weight_max": 33.210567474365234,
      "activations/layer2_attention_weight_min": -31.533170700073242,
      "activations/layer3_attention_weight_max": 87.95159149169922,
      "activations/layer3_attention_weight_min": -90.9743881225586,
      "activations/layer4_attention_weight_max": 124.65950012207031,
      "activations/layer4_attention_weight_min": -124.22674560546875,
      "activations/layer5_attention_weight_max": 68.02716064453125,
      "activations/layer5_attention_weight_min": -68.69697570800781,
      "activations/layer6_attention_weight_max": 51.49045181274414,
      "activations/layer6_attention_weight_min": -53.386192321777344,
      "activations/layer7_attention_weight_max": 85.14493560791016,
      "activations/layer7_attention_weight_min": -85.60067749023438,
      "activations/layer8_attention_weight_max": 40.19461441040039,
      "activations/layer8_attention_weight_min": -39.44050979614258,
      "activations/layer9_attention_weight_max": 35.1244010925293,
      "activations/layer9_attention_weight_min": -35.7200927734375,
      "epoch": 6.3,
      "learning_rate": 0.00011051174242424242,
      "loss": 2.801,
      "step": 108350
    },
    {
      "activations/layer0_attention_weight_max": 16.91446876525879,
      "activations/layer0_attention_weight_min": -12.138919830322266,
      "activations/layer10_attention_weight_max": 32.41957473754883,
      "activations/layer10_attention_weight_min": -30.433536529541016,
      "activations/layer11_attention_weight_max": 33.069305419921875,
      "activations/layer11_attention_weight_min": -28.391935348510742,
      "activations/layer12_attention_weight_max": 55.092777252197266,
      "activations/layer12_attention_weight_min": -30.643529891967773,
      "activations/layer13_attention_weight_max": 48.23246383666992,
      "activations/layer13_attention_weight_min": -33.23550033569336,
      "activations/layer14_attention_weight_max": 59.696937561035156,
      "activations/layer14_attention_weight_min": -42.972171783447266,
      "activations/layer15_attention_weight_max": 37.399784088134766,
      "activations/layer15_attention_weight_min": -28.115066528320312,
      "activations/layer16_attention_weight_max": 35.77322769165039,
      "activations/layer16_attention_weight_min": -28.3071231842041,
      "activations/layer17_attention_weight_max": 35.33885192871094,
      "activations/layer17_attention_weight_min": -25.43379783630371,
      "activations/layer18_attention_weight_max": 32.05500411987305,
      "activations/layer18_attention_weight_min": -20.66486358642578,
      "activations/layer19_attention_weight_max": 37.88584518432617,
      "activations/layer19_attention_weight_min": -29.05859375,
      "activations/layer1_attention_weight_max": 18.012874603271484,
      "activations/layer1_attention_weight_min": -14.0485200881958,
      "activations/layer20_attention_weight_max": 31.305912017822266,
      "activations/layer20_attention_weight_min": -25.577003479003906,
      "activations/layer21_attention_weight_max": 33.40835189819336,
      "activations/layer21_attention_weight_min": -23.074857711791992,
      "activations/layer22_attention_weight_max": 49.38993453979492,
      "activations/layer22_attention_weight_min": -30.01178741455078,
      "activations/layer23_attention_weight_max": 35.940345764160156,
      "activations/layer23_attention_weight_min": -20.670629501342773,
      "activations/layer2_attention_weight_max": 33.906166076660156,
      "activations/layer2_attention_weight_min": -30.487686157226562,
      "activations/layer3_attention_weight_max": 91.09661865234375,
      "activations/layer3_attention_weight_min": -94.17053985595703,
      "activations/layer4_attention_weight_max": 128.54600524902344,
      "activations/layer4_attention_weight_min": -129.29925537109375,
      "activations/layer5_attention_weight_max": 73.60215759277344,
      "activations/layer5_attention_weight_min": -70.70494079589844,
      "activations/layer6_attention_weight_max": 55.70075225830078,
      "activations/layer6_attention_weight_min": -56.00550842285156,
      "activations/layer7_attention_weight_max": 90.34954071044922,
      "activations/layer7_attention_weight_min": -89.42877197265625,
      "activations/layer8_attention_weight_max": 43.02774429321289,
      "activations/layer8_attention_weight_min": -40.32392883300781,
      "activations/layer9_attention_weight_max": 37.32526397705078,
      "activations/layer9_attention_weight_min": -34.6833610534668,
      "epoch": 6.3,
      "learning_rate": 0.00011049280303030303,
      "loss": 2.8399,
      "step": 108400
    },
    {
      "activations/layer0_attention_weight_max": 16.076963424682617,
      "activations/layer0_attention_weight_min": -11.64353084564209,
      "activations/layer10_attention_weight_max": 39.02783966064453,
      "activations/layer10_attention_weight_min": -31.780967712402344,
      "activations/layer11_attention_weight_max": 36.861759185791016,
      "activations/layer11_attention_weight_min": -29.25495147705078,
      "activations/layer12_attention_weight_max": 27.397424697875977,
      "activations/layer12_attention_weight_min": -27.9589786529541,
      "activations/layer13_attention_weight_max": 45.805667877197266,
      "activations/layer13_attention_weight_min": -34.494075775146484,
      "activations/layer14_attention_weight_max": 56.17509841918945,
      "activations/layer14_attention_weight_min": -40.09925842285156,
      "activations/layer15_attention_weight_max": 49.34970474243164,
      "activations/layer15_attention_weight_min": -29.172163009643555,
      "activations/layer16_attention_weight_max": 37.50712966918945,
      "activations/layer16_attention_weight_min": -27.66502571105957,
      "activations/layer17_attention_weight_max": 38.653839111328125,
      "activations/layer17_attention_weight_min": -24.986106872558594,
      "activations/layer18_attention_weight_max": 31.051876068115234,
      "activations/layer18_attention_weight_min": -22.10715675354004,
      "activations/layer19_attention_weight_max": 38.151580810546875,
      "activations/layer19_attention_weight_min": -29.02779197692871,
      "activations/layer1_attention_weight_max": 16.825481414794922,
      "activations/layer1_attention_weight_min": -15.038429260253906,
      "activations/layer20_attention_weight_max": 29.955188751220703,
      "activations/layer20_attention_weight_min": -23.92389678955078,
      "activations/layer21_attention_weight_max": 32.93405532836914,
      "activations/layer21_attention_weight_min": -21.155858993530273,
      "activations/layer22_attention_weight_max": 51.95676803588867,
      "activations/layer22_attention_weight_min": -27.60708236694336,
      "activations/layer23_attention_weight_max": 33.33707809448242,
      "activations/layer23_attention_weight_min": -22.788625717163086,
      "activations/layer2_attention_weight_max": 32.82350158691406,
      "activations/layer2_attention_weight_min": -31.089040756225586,
      "activations/layer3_attention_weight_max": 87.55294799804688,
      "activations/layer3_attention_weight_min": -93.52972412109375,
      "activations/layer4_attention_weight_max": 126.90465545654297,
      "activations/layer4_attention_weight_min": -128.96604919433594,
      "activations/layer5_attention_weight_max": 72.72439575195312,
      "activations/layer5_attention_weight_min": -70.72352600097656,
      "activations/layer6_attention_weight_max": 55.555274963378906,
      "activations/layer6_attention_weight_min": -57.87511444091797,
      "activations/layer7_attention_weight_max": 93.42616271972656,
      "activations/layer7_attention_weight_min": -89.37836456298828,
      "activations/layer8_attention_weight_max": 46.13103103637695,
      "activations/layer8_attention_weight_min": -42.984397888183594,
      "activations/layer9_attention_weight_max": 41.992183685302734,
      "activations/layer9_attention_weight_min": -35.82767868041992,
      "epoch": 6.3,
      "learning_rate": 0.00011047386363636362,
      "loss": 2.8286,
      "step": 108450
    },
    {
      "activations/layer0_attention_weight_max": 16.4696044921875,
      "activations/layer0_attention_weight_min": -11.926679611206055,
      "activations/layer10_attention_weight_max": 34.34309005737305,
      "activations/layer10_attention_weight_min": -31.65525245666504,
      "activations/layer11_attention_weight_max": 32.3829345703125,
      "activations/layer11_attention_weight_min": -29.089862823486328,
      "activations/layer12_attention_weight_max": 24.299922943115234,
      "activations/layer12_attention_weight_min": -37.02398681640625,
      "activations/layer13_attention_weight_max": 42.08179473876953,
      "activations/layer13_attention_weight_min": -33.31669235229492,
      "activations/layer14_attention_weight_max": 59.08334732055664,
      "activations/layer14_attention_weight_min": -42.669219970703125,
      "activations/layer15_attention_weight_max": 37.478145599365234,
      "activations/layer15_attention_weight_min": -28.77460479736328,
      "activations/layer16_attention_weight_max": 34.616371154785156,
      "activations/layer16_attention_weight_min": -26.93462562561035,
      "activations/layer17_attention_weight_max": 37.34031295776367,
      "activations/layer17_attention_weight_min": -24.85857582092285,
      "activations/layer18_attention_weight_max": 30.586246490478516,
      "activations/layer18_attention_weight_min": -18.815038681030273,
      "activations/layer19_attention_weight_max": 42.9133415222168,
      "activations/layer19_attention_weight_min": -28.197629928588867,
      "activations/layer1_attention_weight_max": 16.64655876159668,
      "activations/layer1_attention_weight_min": -16.53653335571289,
      "activations/layer20_attention_weight_max": 33.544898986816406,
      "activations/layer20_attention_weight_min": -23.152843475341797,
      "activations/layer21_attention_weight_max": 34.401397705078125,
      "activations/layer21_attention_weight_min": -21.56329917907715,
      "activations/layer22_attention_weight_max": 53.671653747558594,
      "activations/layer22_attention_weight_min": -28.680009841918945,
      "activations/layer23_attention_weight_max": 38.57878112792969,
      "activations/layer23_attention_weight_min": -21.434171676635742,
      "activations/layer2_attention_weight_max": 33.32973861694336,
      "activations/layer2_attention_weight_min": -31.172338485717773,
      "activations/layer3_attention_weight_max": 95.22332000732422,
      "activations/layer3_attention_weight_min": -98.87679290771484,
      "activations/layer4_attention_weight_max": 128.69146728515625,
      "activations/layer4_attention_weight_min": -133.21852111816406,
      "activations/layer5_attention_weight_max": 69.25688171386719,
      "activations/layer5_attention_weight_min": -68.07135009765625,
      "activations/layer6_attention_weight_max": 51.2422981262207,
      "activations/layer6_attention_weight_min": -59.09675979614258,
      "activations/layer7_attention_weight_max": 87.72178649902344,
      "activations/layer7_attention_weight_min": -87.95860290527344,
      "activations/layer8_attention_weight_max": 38.8492431640625,
      "activations/layer8_attention_weight_min": -41.98336410522461,
      "activations/layer9_attention_weight_max": 35.627811431884766,
      "activations/layer9_attention_weight_min": -33.493446350097656,
      "epoch": 6.3,
      "learning_rate": 0.00011045492424242424,
      "loss": 2.8326,
      "step": 108500
    },
    {
      "activations/layer0_attention_weight_max": 16.1151065826416,
      "activations/layer0_attention_weight_min": -11.5866117477417,
      "activations/layer10_attention_weight_max": 46.76393508911133,
      "activations/layer10_attention_weight_min": -41.61318588256836,
      "activations/layer11_attention_weight_max": 47.373348236083984,
      "activations/layer11_attention_weight_min": -37.21609878540039,
      "activations/layer12_attention_weight_max": 42.29750442504883,
      "activations/layer12_attention_weight_min": -31.784282684326172,
      "activations/layer13_attention_weight_max": 65.1648178100586,
      "activations/layer13_attention_weight_min": -33.27780532836914,
      "activations/layer14_attention_weight_max": 77.91719055175781,
      "activations/layer14_attention_weight_min": -42.575279235839844,
      "activations/layer15_attention_weight_max": 45.217342376708984,
      "activations/layer15_attention_weight_min": -28.522924423217773,
      "activations/layer16_attention_weight_max": 38.085758209228516,
      "activations/layer16_attention_weight_min": -25.434940338134766,
      "activations/layer17_attention_weight_max": 37.555030822753906,
      "activations/layer17_attention_weight_min": -23.23462677001953,
      "activations/layer18_attention_weight_max": 31.544649124145508,
      "activations/layer18_attention_weight_min": -21.87207794189453,
      "activations/layer19_attention_weight_max": 42.94729232788086,
      "activations/layer19_attention_weight_min": -27.739200592041016,
      "activations/layer1_attention_weight_max": 17.93824577331543,
      "activations/layer1_attention_weight_min": -13.540361404418945,
      "activations/layer20_attention_weight_max": 32.50795364379883,
      "activations/layer20_attention_weight_min": -23.810741424560547,
      "activations/layer21_attention_weight_max": 32.43798828125,
      "activations/layer21_attention_weight_min": -21.55047607421875,
      "activations/layer22_attention_weight_max": 51.394229888916016,
      "activations/layer22_attention_weight_min": -28.70964813232422,
      "activations/layer23_attention_weight_max": 38.574493408203125,
      "activations/layer23_attention_weight_min": -20.60207748413086,
      "activations/layer2_attention_weight_max": 36.07285690307617,
      "activations/layer2_attention_weight_min": -32.48099899291992,
      "activations/layer3_attention_weight_max": 99.4478988647461,
      "activations/layer3_attention_weight_min": -97.39017486572266,
      "activations/layer4_attention_weight_max": 140.372314453125,
      "activations/layer4_attention_weight_min": -128.53977966308594,
      "activations/layer5_attention_weight_max": 74.83272552490234,
      "activations/layer5_attention_weight_min": -69.38960266113281,
      "activations/layer6_attention_weight_max": 60.46891784667969,
      "activations/layer6_attention_weight_min": -55.22285842895508,
      "activations/layer7_attention_weight_max": 104.9559555053711,
      "activations/layer7_attention_weight_min": -100.99004364013672,
      "activations/layer8_attention_weight_max": 53.65816116333008,
      "activations/layer8_attention_weight_min": -48.50446319580078,
      "activations/layer9_attention_weight_max": 50.366756439208984,
      "activations/layer9_attention_weight_min": -44.58023452758789,
      "epoch": 6.31,
      "learning_rate": 0.00011043598484848484,
      "loss": 2.8374,
      "step": 108550
    },
    {
      "activations/layer0_attention_weight_max": 16.490901947021484,
      "activations/layer0_attention_weight_min": -11.522568702697754,
      "activations/layer10_attention_weight_max": 30.65585708618164,
      "activations/layer10_attention_weight_min": -28.619482040405273,
      "activations/layer11_attention_weight_max": 28.602598190307617,
      "activations/layer11_attention_weight_min": -26.3074951171875,
      "activations/layer12_attention_weight_max": 20.082433700561523,
      "activations/layer12_attention_weight_min": -27.69114875793457,
      "activations/layer13_attention_weight_max": 37.15907669067383,
      "activations/layer13_attention_weight_min": -29.929298400878906,
      "activations/layer14_attention_weight_max": 47.78984069824219,
      "activations/layer14_attention_weight_min": -44.75304412841797,
      "activations/layer15_attention_weight_max": 35.17132568359375,
      "activations/layer15_attention_weight_min": -28.277664184570312,
      "activations/layer16_attention_weight_max": 32.431522369384766,
      "activations/layer16_attention_weight_min": -25.922075271606445,
      "activations/layer17_attention_weight_max": 31.92738151550293,
      "activations/layer17_attention_weight_min": -23.876487731933594,
      "activations/layer18_attention_weight_max": 33.78990936279297,
      "activations/layer18_attention_weight_min": -19.458343505859375,
      "activations/layer19_attention_weight_max": 38.571067810058594,
      "activations/layer19_attention_weight_min": -27.65983009338379,
      "activations/layer1_attention_weight_max": 17.71877098083496,
      "activations/layer1_attention_weight_min": -13.682579040527344,
      "activations/layer20_attention_weight_max": 31.148900985717773,
      "activations/layer20_attention_weight_min": -22.210790634155273,
      "activations/layer21_attention_weight_max": 31.910160064697266,
      "activations/layer21_attention_weight_min": -19.956218719482422,
      "activations/layer22_attention_weight_max": 43.99463653564453,
      "activations/layer22_attention_weight_min": -27.482358932495117,
      "activations/layer23_attention_weight_max": 35.824920654296875,
      "activations/layer23_attention_weight_min": -22.221607208251953,
      "activations/layer2_attention_weight_max": 32.20777130126953,
      "activations/layer2_attention_weight_min": -30.221553802490234,
      "activations/layer3_attention_weight_max": 90.434326171875,
      "activations/layer3_attention_weight_min": -90.93172454833984,
      "activations/layer4_attention_weight_max": 125.01019287109375,
      "activations/layer4_attention_weight_min": -122.25179290771484,
      "activations/layer5_attention_weight_max": 68.63601684570312,
      "activations/layer5_attention_weight_min": -66.14195251464844,
      "activations/layer6_attention_weight_max": 50.59274673461914,
      "activations/layer6_attention_weight_min": -54.02214813232422,
      "activations/layer7_attention_weight_max": 85.20048522949219,
      "activations/layer7_attention_weight_min": -82.09596252441406,
      "activations/layer8_attention_weight_max": 37.260223388671875,
      "activations/layer8_attention_weight_min": -39.76469421386719,
      "activations/layer9_attention_weight_max": 31.847003936767578,
      "activations/layer9_attention_weight_min": -31.758169174194336,
      "epoch": 6.31,
      "learning_rate": 0.00011041704545454544,
      "loss": 2.8107,
      "step": 108600
    },
    {
      "activations/layer0_attention_weight_max": 16.065837860107422,
      "activations/layer0_attention_weight_min": -13.254952430725098,
      "activations/layer10_attention_weight_max": 33.598121643066406,
      "activations/layer10_attention_weight_min": -30.689611434936523,
      "activations/layer11_attention_weight_max": 34.492958068847656,
      "activations/layer11_attention_weight_min": -27.40431785583496,
      "activations/layer12_attention_weight_max": 22.700040817260742,
      "activations/layer12_attention_weight_min": -27.184648513793945,
      "activations/layer13_attention_weight_max": 42.776546478271484,
      "activations/layer13_attention_weight_min": -33.366798400878906,
      "activations/layer14_attention_weight_max": 57.611812591552734,
      "activations/layer14_attention_weight_min": -46.51823043823242,
      "activations/layer15_attention_weight_max": 35.51615524291992,
      "activations/layer15_attention_weight_min": -28.557828903198242,
      "activations/layer16_attention_weight_max": 35.88437271118164,
      "activations/layer16_attention_weight_min": -26.621965408325195,
      "activations/layer17_attention_weight_max": 34.79300308227539,
      "activations/layer17_attention_weight_min": -25.849828720092773,
      "activations/layer18_attention_weight_max": 33.771766662597656,
      "activations/layer18_attention_weight_min": -19.941946029663086,
      "activations/layer19_attention_weight_max": 39.4172477722168,
      "activations/layer19_attention_weight_min": -30.336978912353516,
      "activations/layer1_attention_weight_max": 17.591156005859375,
      "activations/layer1_attention_weight_min": -15.43559741973877,
      "activations/layer20_attention_weight_max": 32.54154968261719,
      "activations/layer20_attention_weight_min": -23.062467575073242,
      "activations/layer21_attention_weight_max": 32.015472412109375,
      "activations/layer21_attention_weight_min": -22.2241268157959,
      "activations/layer22_attention_weight_max": 47.13215637207031,
      "activations/layer22_attention_weight_min": -28.91660499572754,
      "activations/layer23_attention_weight_max": 35.549537658691406,
      "activations/layer23_attention_weight_min": -23.36512565612793,
      "activations/layer2_attention_weight_max": 33.56220245361328,
      "activations/layer2_attention_weight_min": -33.05609130859375,
      "activations/layer3_attention_weight_max": 94.5406265258789,
      "activations/layer3_attention_weight_min": -97.21906280517578,
      "activations/layer4_attention_weight_max": 128.71205139160156,
      "activations/layer4_attention_weight_min": -130.34051513671875,
      "activations/layer5_attention_weight_max": 71.53077697753906,
      "activations/layer5_attention_weight_min": -69.05599975585938,
      "activations/layer6_attention_weight_max": 57.53840255737305,
      "activations/layer6_attention_weight_min": -56.731224060058594,
      "activations/layer7_attention_weight_max": 88.59758758544922,
      "activations/layer7_attention_weight_min": -89.2312240600586,
      "activations/layer8_attention_weight_max": 39.80103302001953,
      "activations/layer8_attention_weight_min": -39.02538299560547,
      "activations/layer9_attention_weight_max": 35.871639251708984,
      "activations/layer9_attention_weight_min": -32.687110900878906,
      "epoch": 6.31,
      "learning_rate": 0.00011039810606060605,
      "loss": 2.8194,
      "step": 108650
    },
    {
      "activations/layer0_attention_weight_max": 14.927791595458984,
      "activations/layer0_attention_weight_min": -12.384907722473145,
      "activations/layer10_attention_weight_max": 36.600440979003906,
      "activations/layer10_attention_weight_min": -31.729137420654297,
      "activations/layer11_attention_weight_max": 38.982418060302734,
      "activations/layer11_attention_weight_min": -30.22414779663086,
      "activations/layer12_attention_weight_max": 32.71342086791992,
      "activations/layer12_attention_weight_min": -26.15827178955078,
      "activations/layer13_attention_weight_max": 49.39634323120117,
      "activations/layer13_attention_weight_min": -32.47966766357422,
      "activations/layer14_attention_weight_max": 70.53606414794922,
      "activations/layer14_attention_weight_min": -45.54590606689453,
      "activations/layer15_attention_weight_max": 44.42522430419922,
      "activations/layer15_attention_weight_min": -30.331165313720703,
      "activations/layer16_attention_weight_max": 41.28300476074219,
      "activations/layer16_attention_weight_min": -27.419130325317383,
      "activations/layer17_attention_weight_max": 39.4875602722168,
      "activations/layer17_attention_weight_min": -25.16830825805664,
      "activations/layer18_attention_weight_max": 37.8946647644043,
      "activations/layer18_attention_weight_min": -22.614965438842773,
      "activations/layer19_attention_weight_max": 42.98454666137695,
      "activations/layer19_attention_weight_min": -30.69833755493164,
      "activations/layer1_attention_weight_max": 17.292875289916992,
      "activations/layer1_attention_weight_min": -16.07346534729004,
      "activations/layer20_attention_weight_max": 36.728515625,
      "activations/layer20_attention_weight_min": -26.021074295043945,
      "activations/layer21_attention_weight_max": 35.86710739135742,
      "activations/layer21_attention_weight_min": -22.65699577331543,
      "activations/layer22_attention_weight_max": 55.58211135864258,
      "activations/layer22_attention_weight_min": -30.989227294921875,
      "activations/layer23_attention_weight_max": 38.3300895690918,
      "activations/layer23_attention_weight_min": -22.15500259399414,
      "activations/layer2_attention_weight_max": 32.02564239501953,
      "activations/layer2_attention_weight_min": -31.5037899017334,
      "activations/layer3_attention_weight_max": 89.04331970214844,
      "activations/layer3_attention_weight_min": -89.15859985351562,
      "activations/layer4_attention_weight_max": 128.54551696777344,
      "activations/layer4_attention_weight_min": -123.33088684082031,
      "activations/layer5_attention_weight_max": 68.40196228027344,
      "activations/layer5_attention_weight_min": -68.0419692993164,
      "activations/layer6_attention_weight_max": 52.63488006591797,
      "activations/layer6_attention_weight_min": -54.43220901489258,
      "activations/layer7_attention_weight_max": 91.78761291503906,
      "activations/layer7_attention_weight_min": -87.837646484375,
      "activations/layer8_attention_weight_max": 43.77639389038086,
      "activations/layer8_attention_weight_min": -40.94831085205078,
      "activations/layer9_attention_weight_max": 40.43143844604492,
      "activations/layer9_attention_weight_min": -34.61833190917969,
      "epoch": 6.32,
      "learning_rate": 0.00011037916666666665,
      "loss": 2.8411,
      "step": 108700
    },
    {
      "activations/layer0_attention_weight_max": 15.129986763000488,
      "activations/layer0_attention_weight_min": -11.482614517211914,
      "activations/layer10_attention_weight_max": 33.05934143066406,
      "activations/layer10_attention_weight_min": -29.891489028930664,
      "activations/layer11_attention_weight_max": 32.52676773071289,
      "activations/layer11_attention_weight_min": -28.992019653320312,
      "activations/layer12_attention_weight_max": 25.973051071166992,
      "activations/layer12_attention_weight_min": -25.76581573486328,
      "activations/layer13_attention_weight_max": 45.91242980957031,
      "activations/layer13_attention_weight_min": -32.90761947631836,
      "activations/layer14_attention_weight_max": 53.36149597167969,
      "activations/layer14_attention_weight_min": -38.75678253173828,
      "activations/layer15_attention_weight_max": 40.367340087890625,
      "activations/layer15_attention_weight_min": -27.910470962524414,
      "activations/layer16_attention_weight_max": 38.33736801147461,
      "activations/layer16_attention_weight_min": -26.268047332763672,
      "activations/layer17_attention_weight_max": 39.73910903930664,
      "activations/layer17_attention_weight_min": -25.572336196899414,
      "activations/layer18_attention_weight_max": 33.76991271972656,
      "activations/layer18_attention_weight_min": -23.441118240356445,
      "activations/layer19_attention_weight_max": 38.60576248168945,
      "activations/layer19_attention_weight_min": -29.312280654907227,
      "activations/layer1_attention_weight_max": 17.886268615722656,
      "activations/layer1_attention_weight_min": -14.360847473144531,
      "activations/layer20_attention_weight_max": 31.55324363708496,
      "activations/layer20_attention_weight_min": -25.275413513183594,
      "activations/layer21_attention_weight_max": 32.24947738647461,
      "activations/layer21_attention_weight_min": -21.523038864135742,
      "activations/layer22_attention_weight_max": 50.16853332519531,
      "activations/layer22_attention_weight_min": -28.124347686767578,
      "activations/layer23_attention_weight_max": 34.77061462402344,
      "activations/layer23_attention_weight_min": -24.39546775817871,
      "activations/layer2_attention_weight_max": 31.927520751953125,
      "activations/layer2_attention_weight_min": -30.925832748413086,
      "activations/layer3_attention_weight_max": 88.4522933959961,
      "activations/layer3_attention_weight_min": -87.42961120605469,
      "activations/layer4_attention_weight_max": 129.40499877929688,
      "activations/layer4_attention_weight_min": -120.76934051513672,
      "activations/layer5_attention_weight_max": 66.3387451171875,
      "activations/layer5_attention_weight_min": -65.63714599609375,
      "activations/layer6_attention_weight_max": 50.972801208496094,
      "activations/layer6_attention_weight_min": -51.828128814697266,
      "activations/layer7_attention_weight_max": 85.83875274658203,
      "activations/layer7_attention_weight_min": -79.87303924560547,
      "activations/layer8_attention_weight_max": 37.29115676879883,
      "activations/layer8_attention_weight_min": -38.45658874511719,
      "activations/layer9_attention_weight_max": 34.66973876953125,
      "activations/layer9_attention_weight_min": -31.92337417602539,
      "epoch": 6.32,
      "learning_rate": 0.00011036022727272727,
      "loss": 2.8349,
      "step": 108750
    },
    {
      "activations/layer0_attention_weight_max": 15.550230026245117,
      "activations/layer0_attention_weight_min": -12.462843894958496,
      "activations/layer10_attention_weight_max": 37.60955047607422,
      "activations/layer10_attention_weight_min": -32.041072845458984,
      "activations/layer11_attention_weight_max": 42.45326232910156,
      "activations/layer11_attention_weight_min": -33.42586898803711,
      "activations/layer12_attention_weight_max": 29.74882698059082,
      "activations/layer12_attention_weight_min": -28.846899032592773,
      "activations/layer13_attention_weight_max": 51.46137237548828,
      "activations/layer13_attention_weight_min": -33.587303161621094,
      "activations/layer14_attention_weight_max": 59.39431381225586,
      "activations/layer14_attention_weight_min": -46.479217529296875,
      "activations/layer15_attention_weight_max": 39.56175231933594,
      "activations/layer15_attention_weight_min": -30.139633178710938,
      "activations/layer16_attention_weight_max": 32.72923278808594,
      "activations/layer16_attention_weight_min": -26.54964256286621,
      "activations/layer17_attention_weight_max": 36.19328308105469,
      "activations/layer17_attention_weight_min": -24.300676345825195,
      "activations/layer18_attention_weight_max": 37.22633743286133,
      "activations/layer18_attention_weight_min": -20.124603271484375,
      "activations/layer19_attention_weight_max": 37.647216796875,
      "activations/layer19_attention_weight_min": -29.618106842041016,
      "activations/layer1_attention_weight_max": 17.31776237487793,
      "activations/layer1_attention_weight_min": -14.699649810791016,
      "activations/layer20_attention_weight_max": 31.727251052856445,
      "activations/layer20_attention_weight_min": -25.28249168395996,
      "activations/layer21_attention_weight_max": 34.959228515625,
      "activations/layer21_attention_weight_min": -21.1378173828125,
      "activations/layer22_attention_weight_max": 48.78203582763672,
      "activations/layer22_attention_weight_min": -28.998493194580078,
      "activations/layer23_attention_weight_max": 34.633026123046875,
      "activations/layer23_attention_weight_min": -22.904937744140625,
      "activations/layer2_attention_weight_max": 34.564884185791016,
      "activations/layer2_attention_weight_min": -32.51556396484375,
      "activations/layer3_attention_weight_max": 95.09039306640625,
      "activations/layer3_attention_weight_min": -98.21678924560547,
      "activations/layer4_attention_weight_max": 131.7843780517578,
      "activations/layer4_attention_weight_min": -127.4659423828125,
      "activations/layer5_attention_weight_max": 68.01173400878906,
      "activations/layer5_attention_weight_min": -67.24301147460938,
      "activations/layer6_attention_weight_max": 51.91415023803711,
      "activations/layer6_attention_weight_min": -55.8224983215332,
      "activations/layer7_attention_weight_max": 103.77456665039062,
      "activations/layer7_attention_weight_min": -87.22535705566406,
      "activations/layer8_attention_weight_max": 42.34403610229492,
      "activations/layer8_attention_weight_min": -40.055442810058594,
      "activations/layer9_attention_weight_max": 36.2086296081543,
      "activations/layer9_attention_weight_min": -42.79771041870117,
      "epoch": 6.32,
      "learning_rate": 0.00011034128787878787,
      "loss": 2.8126,
      "step": 108800
    },
    {
      "activations/layer0_attention_weight_max": 15.538573265075684,
      "activations/layer0_attention_weight_min": -12.09282398223877,
      "activations/layer10_attention_weight_max": 32.621009826660156,
      "activations/layer10_attention_weight_min": -30.534420013427734,
      "activations/layer11_attention_weight_max": 34.86703109741211,
      "activations/layer11_attention_weight_min": -28.16286849975586,
      "activations/layer12_attention_weight_max": 33.503211975097656,
      "activations/layer12_attention_weight_min": -29.57845687866211,
      "activations/layer13_attention_weight_max": 42.65174865722656,
      "activations/layer13_attention_weight_min": -34.7713737487793,
      "activations/layer14_attention_weight_max": 56.43120193481445,
      "activations/layer14_attention_weight_min": -42.90576171875,
      "activations/layer15_attention_weight_max": 36.56488037109375,
      "activations/layer15_attention_weight_min": -29.306812286376953,
      "activations/layer16_attention_weight_max": 36.62433624267578,
      "activations/layer16_attention_weight_min": -27.294050216674805,
      "activations/layer17_attention_weight_max": 35.06222915649414,
      "activations/layer17_attention_weight_min": -25.232830047607422,
      "activations/layer18_attention_weight_max": 33.181358337402344,
      "activations/layer18_attention_weight_min": -19.72290802001953,
      "activations/layer19_attention_weight_max": 38.60807418823242,
      "activations/layer19_attention_weight_min": -28.708881378173828,
      "activations/layer1_attention_weight_max": 17.7034854888916,
      "activations/layer1_attention_weight_min": -14.596817970275879,
      "activations/layer20_attention_weight_max": 32.75868225097656,
      "activations/layer20_attention_weight_min": -24.518314361572266,
      "activations/layer21_attention_weight_max": 30.009918212890625,
      "activations/layer21_attention_weight_min": -21.288497924804688,
      "activations/layer22_attention_weight_max": 48.47821044921875,
      "activations/layer22_attention_weight_min": -27.797348022460938,
      "activations/layer23_attention_weight_max": 32.64160919189453,
      "activations/layer23_attention_weight_min": -22.010704040527344,
      "activations/layer2_attention_weight_max": 33.554664611816406,
      "activations/layer2_attention_weight_min": -31.673316955566406,
      "activations/layer3_attention_weight_max": 91.79121398925781,
      "activations/layer3_attention_weight_min": -92.98857879638672,
      "activations/layer4_attention_weight_max": 126.4152603149414,
      "activations/layer4_attention_weight_min": -123.47502899169922,
      "activations/layer5_attention_weight_max": 68.77247619628906,
      "activations/layer5_attention_weight_min": -68.27759552001953,
      "activations/layer6_attention_weight_max": 52.46894454956055,
      "activations/layer6_attention_weight_min": -54.75994110107422,
      "activations/layer7_attention_weight_max": 83.26580047607422,
      "activations/layer7_attention_weight_min": -85.0151138305664,
      "activations/layer8_attention_weight_max": 38.626983642578125,
      "activations/layer8_attention_weight_min": -41.314727783203125,
      "activations/layer9_attention_weight_max": 35.00028610229492,
      "activations/layer9_attention_weight_min": -34.182315826416016,
      "epoch": 6.32,
      "learning_rate": 0.00011032234848484847,
      "loss": 2.8293,
      "step": 108850
    },
    {
      "activations/layer0_attention_weight_max": 17.04906463623047,
      "activations/layer0_attention_weight_min": -12.662996292114258,
      "activations/layer10_attention_weight_max": 38.01070022583008,
      "activations/layer10_attention_weight_min": -32.283912658691406,
      "activations/layer11_attention_weight_max": 37.797218322753906,
      "activations/layer11_attention_weight_min": -29.5026912689209,
      "activations/layer12_attention_weight_max": 25.83584213256836,
      "activations/layer12_attention_weight_min": -31.097068786621094,
      "activations/layer13_attention_weight_max": 46.305179595947266,
      "activations/layer13_attention_weight_min": -33.52684020996094,
      "activations/layer14_attention_weight_max": 66.7793197631836,
      "activations/layer14_attention_weight_min": -47.146358489990234,
      "activations/layer15_attention_weight_max": 39.43132400512695,
      "activations/layer15_attention_weight_min": -27.60897445678711,
      "activations/layer16_attention_weight_max": 37.18486785888672,
      "activations/layer16_attention_weight_min": -26.865774154663086,
      "activations/layer17_attention_weight_max": 40.03034210205078,
      "activations/layer17_attention_weight_min": -24.162302017211914,
      "activations/layer18_attention_weight_max": 36.3831901550293,
      "activations/layer18_attention_weight_min": -21.733549118041992,
      "activations/layer19_attention_weight_max": 39.90049362182617,
      "activations/layer19_attention_weight_min": -29.478748321533203,
      "activations/layer1_attention_weight_max": 17.01111602783203,
      "activations/layer1_attention_weight_min": -14.055355072021484,
      "activations/layer20_attention_weight_max": 37.08456802368164,
      "activations/layer20_attention_weight_min": -22.828500747680664,
      "activations/layer21_attention_weight_max": 36.24592208862305,
      "activations/layer21_attention_weight_min": -22.161651611328125,
      "activations/layer22_attention_weight_max": 58.029998779296875,
      "activations/layer22_attention_weight_min": -30.33898162841797,
      "activations/layer23_attention_weight_max": 40.36882400512695,
      "activations/layer23_attention_weight_min": -23.010391235351562,
      "activations/layer2_attention_weight_max": 33.96339416503906,
      "activations/layer2_attention_weight_min": -32.34102249145508,
      "activations/layer3_attention_weight_max": 97.03289031982422,
      "activations/layer3_attention_weight_min": -97.69317626953125,
      "activations/layer4_attention_weight_max": 134.74676513671875,
      "activations/layer4_attention_weight_min": -128.78636169433594,
      "activations/layer5_attention_weight_max": 74.79961395263672,
      "activations/layer5_attention_weight_min": -70.43917083740234,
      "activations/layer6_attention_weight_max": 57.072410583496094,
      "activations/layer6_attention_weight_min": -56.389801025390625,
      "activations/layer7_attention_weight_max": 91.37642669677734,
      "activations/layer7_attention_weight_min": -94.85317993164062,
      "activations/layer8_attention_weight_max": 44.5086784362793,
      "activations/layer8_attention_weight_min": -41.86787414550781,
      "activations/layer9_attention_weight_max": 41.99283981323242,
      "activations/layer9_attention_weight_min": -38.88037872314453,
      "epoch": 6.33,
      "learning_rate": 0.00011030340909090909,
      "loss": 2.8161,
      "step": 108900
    },
    {
      "activations/layer0_attention_weight_max": 16.690547943115234,
      "activations/layer0_attention_weight_min": -10.891907691955566,
      "activations/layer10_attention_weight_max": 31.34918785095215,
      "activations/layer10_attention_weight_min": -31.739957809448242,
      "activations/layer11_attention_weight_max": 35.710304260253906,
      "activations/layer11_attention_weight_min": -29.334455490112305,
      "activations/layer12_attention_weight_max": 23.333662033081055,
      "activations/layer12_attention_weight_min": -26.727218627929688,
      "activations/layer13_attention_weight_max": 42.488853454589844,
      "activations/layer13_attention_weight_min": -32.18107223510742,
      "activations/layer14_attention_weight_max": 54.89973449707031,
      "activations/layer14_attention_weight_min": -44.144073486328125,
      "activations/layer15_attention_weight_max": 46.79658508300781,
      "activations/layer15_attention_weight_min": -30.007213592529297,
      "activations/layer16_attention_weight_max": 43.34450912475586,
      "activations/layer16_attention_weight_min": -32.41733169555664,
      "activations/layer17_attention_weight_max": 34.20530700683594,
      "activations/layer17_attention_weight_min": -25.390975952148438,
      "activations/layer18_attention_weight_max": 31.52059555053711,
      "activations/layer18_attention_weight_min": -23.10226058959961,
      "activations/layer19_attention_weight_max": 44.08218002319336,
      "activations/layer19_attention_weight_min": -30.497053146362305,
      "activations/layer1_attention_weight_max": 17.258014678955078,
      "activations/layer1_attention_weight_min": -14.58553409576416,
      "activations/layer20_attention_weight_max": 35.32375717163086,
      "activations/layer20_attention_weight_min": -24.9435977935791,
      "activations/layer21_attention_weight_max": 36.08771514892578,
      "activations/layer21_attention_weight_min": -21.841867446899414,
      "activations/layer22_attention_weight_max": 54.2046012878418,
      "activations/layer22_attention_weight_min": -29.882253646850586,
      "activations/layer23_attention_weight_max": 37.707515716552734,
      "activations/layer23_attention_weight_min": -24.324649810791016,
      "activations/layer2_attention_weight_max": 32.203521728515625,
      "activations/layer2_attention_weight_min": -31.392013549804688,
      "activations/layer3_attention_weight_max": 90.00245666503906,
      "activations/layer3_attention_weight_min": -91.96456146240234,
      "activations/layer4_attention_weight_max": 122.6308364868164,
      "activations/layer4_attention_weight_min": -121.89000701904297,
      "activations/layer5_attention_weight_max": 65.88578796386719,
      "activations/layer5_attention_weight_min": -69.50094604492188,
      "activations/layer6_attention_weight_max": 52.66941833496094,
      "activations/layer6_attention_weight_min": -55.3844108581543,
      "activations/layer7_attention_weight_max": 85.88916778564453,
      "activations/layer7_attention_weight_min": -83.84841918945312,
      "activations/layer8_attention_weight_max": 38.0998649597168,
      "activations/layer8_attention_weight_min": -38.69392013549805,
      "activations/layer9_attention_weight_max": 36.4261474609375,
      "activations/layer9_attention_weight_min": -34.00544738769531,
      "epoch": 6.33,
      "learning_rate": 0.00011028446969696968,
      "loss": 2.8312,
      "step": 108950
    },
    {
      "activations/layer0_attention_weight_max": 15.724475860595703,
      "activations/layer0_attention_weight_min": -11.96800708770752,
      "activations/layer10_attention_weight_max": 35.11763381958008,
      "activations/layer10_attention_weight_min": -31.336729049682617,
      "activations/layer11_attention_weight_max": 33.778419494628906,
      "activations/layer11_attention_weight_min": -30.498950958251953,
      "activations/layer12_attention_weight_max": 27.79987144470215,
      "activations/layer12_attention_weight_min": -28.100099563598633,
      "activations/layer13_attention_weight_max": 43.99496078491211,
      "activations/layer13_attention_weight_min": -32.908409118652344,
      "activations/layer14_attention_weight_max": 62.760345458984375,
      "activations/layer14_attention_weight_min": -44.71000671386719,
      "activations/layer15_attention_weight_max": 41.098182678222656,
      "activations/layer15_attention_weight_min": -29.012182235717773,
      "activations/layer16_attention_weight_max": 33.69707489013672,
      "activations/layer16_attention_weight_min": -28.187517166137695,
      "activations/layer17_attention_weight_max": 39.934898376464844,
      "activations/layer17_attention_weight_min": -25.863365173339844,
      "activations/layer18_attention_weight_max": 33.810794830322266,
      "activations/layer18_attention_weight_min": -20.142126083374023,
      "activations/layer19_attention_weight_max": 38.33658981323242,
      "activations/layer19_attention_weight_min": -30.6500301361084,
      "activations/layer1_attention_weight_max": 16.673128128051758,
      "activations/layer1_attention_weight_min": -14.444406509399414,
      "activations/layer20_attention_weight_max": 32.78080749511719,
      "activations/layer20_attention_weight_min": -23.709453582763672,
      "activations/layer21_attention_weight_max": 35.937965393066406,
      "activations/layer21_attention_weight_min": -24.039140701293945,
      "activations/layer22_attention_weight_max": 51.26652145385742,
      "activations/layer22_attention_weight_min": -27.91629409790039,
      "activations/layer23_attention_weight_max": 34.61688232421875,
      "activations/layer23_attention_weight_min": -23.787269592285156,
      "activations/layer2_attention_weight_max": 31.04926872253418,
      "activations/layer2_attention_weight_min": -28.568695068359375,
      "activations/layer3_attention_weight_max": 88.48464965820312,
      "activations/layer3_attention_weight_min": -90.02959442138672,
      "activations/layer4_attention_weight_max": 123.5467529296875,
      "activations/layer4_attention_weight_min": -119.6507339477539,
      "activations/layer5_attention_weight_max": 70.03331756591797,
      "activations/layer5_attention_weight_min": -67.14171600341797,
      "activations/layer6_attention_weight_max": 52.110511779785156,
      "activations/layer6_attention_weight_min": -53.43174362182617,
      "activations/layer7_attention_weight_max": 86.26160430908203,
      "activations/layer7_attention_weight_min": -85.27967834472656,
      "activations/layer8_attention_weight_max": 39.71996307373047,
      "activations/layer8_attention_weight_min": -39.29777526855469,
      "activations/layer9_attention_weight_max": 38.60089111328125,
      "activations/layer9_attention_weight_min": -34.03583908081055,
      "epoch": 6.33,
      "learning_rate": 0.00011026553030303029,
      "loss": 2.8363,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5221,
      "eval_samples_per_second": 503.866,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5221,
      "eval_openwebtext_samples_per_second": 503.866,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_wikitext_loss": 3.0,
      "eval_wikitext_ppl": 20.085536923187668,
      "eval_wikitext_runtime": 1.9956,
      "eval_wikitext_samples_per_second": 228.498,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_lambada_loss": 2.728515625,
      "eval_lambada_ppl": 15.310144149387147,
      "eval_lambada_runtime": 9.5784,
      "eval_lambada_samples_per_second": 508.332,
      "step": 109000
    },
    {
      "activations/layer0_attention_weight_max": 16.5616455078125,
      "activations/layer0_attention_weight_min": -12.25913143157959,
      "activations/layer10_attention_weight_max": 32.70991897583008,
      "activations/layer10_attention_weight_min": -31.68816375732422,
      "activations/layer11_attention_weight_max": 34.917171478271484,
      "activations/layer11_attention_weight_min": -29.657318115234375,
      "activations/layer12_attention_weight_max": 28.390634536743164,
      "activations/layer12_attention_weight_min": -26.48533821105957,
      "activations/layer13_attention_weight_max": 39.85689163208008,
      "activations/layer13_attention_weight_min": -35.93424606323242,
      "activations/layer14_attention_weight_max": 55.75349426269531,
      "activations/layer14_attention_weight_min": -41.77397537231445,
      "activations/layer15_attention_weight_max": 37.23799514770508,
      "activations/layer15_attention_weight_min": -29.827192306518555,
      "activations/layer16_attention_weight_max": 35.35192108154297,
      "activations/layer16_attention_weight_min": -28.395551681518555,
      "activations/layer17_attention_weight_max": 38.00653839111328,
      "activations/layer17_attention_weight_min": -24.598276138305664,
      "activations/layer18_attention_weight_max": 32.93507766723633,
      "activations/layer18_attention_weight_min": -21.971445083618164,
      "activations/layer19_attention_weight_max": 44.4151611328125,
      "activations/layer19_attention_weight_min": -30.450239181518555,
      "activations/layer1_attention_weight_max": 16.99728775024414,
      "activations/layer1_attention_weight_min": -14.650237083435059,
      "activations/layer20_attention_weight_max": 32.27713394165039,
      "activations/layer20_attention_weight_min": -23.069101333618164,
      "activations/layer21_attention_weight_max": 31.23546028137207,
      "activations/layer21_attention_weight_min": -22.102575302124023,
      "activations/layer22_attention_weight_max": 48.93310546875,
      "activations/layer22_attention_weight_min": -31.32750129699707,
      "activations/layer23_attention_weight_max": 35.054988861083984,
      "activations/layer23_attention_weight_min": -22.700164794921875,
      "activations/layer2_attention_weight_max": 32.68387985229492,
      "activations/layer2_attention_weight_min": -30.924766540527344,
      "activations/layer3_attention_weight_max": 90.51931762695312,
      "activations/layer3_attention_weight_min": -91.38614654541016,
      "activations/layer4_attention_weight_max": 134.45664978027344,
      "activations/layer4_attention_weight_min": -122.72718811035156,
      "activations/layer5_attention_weight_max": 71.1810073852539,
      "activations/layer5_attention_weight_min": -66.60801696777344,
      "activations/layer6_attention_weight_max": 55.73722839355469,
      "activations/layer6_attention_weight_min": -55.54570007324219,
      "activations/layer7_attention_weight_max": 86.42990112304688,
      "activations/layer7_attention_weight_min": -89.03703308105469,
      "activations/layer8_attention_weight_max": 39.76840591430664,
      "activations/layer8_attention_weight_min": -40.9311408996582,
      "activations/layer9_attention_weight_max": 34.49605941772461,
      "activations/layer9_attention_weight_min": -35.153621673583984,
      "epoch": 6.34,
      "learning_rate": 0.0001102465909090909,
      "loss": 2.8276,
      "step": 109050
    },
    {
      "activations/layer0_attention_weight_max": 17.161685943603516,
      "activations/layer0_attention_weight_min": -12.037236213684082,
      "activations/layer10_attention_weight_max": 33.263179779052734,
      "activations/layer10_attention_weight_min": -32.027320861816406,
      "activations/layer11_attention_weight_max": 33.002098083496094,
      "activations/layer11_attention_weight_min": -28.91498374938965,
      "activations/layer12_attention_weight_max": 48.74250793457031,
      "activations/layer12_attention_weight_min": -27.21762466430664,
      "activations/layer13_attention_weight_max": 44.58926773071289,
      "activations/layer13_attention_weight_min": -32.674068450927734,
      "activations/layer14_attention_weight_max": 71.90148162841797,
      "activations/layer14_attention_weight_min": -50.04671096801758,
      "activations/layer15_attention_weight_max": 37.54642105102539,
      "activations/layer15_attention_weight_min": -30.45831871032715,
      "activations/layer16_attention_weight_max": 36.676509857177734,
      "activations/layer16_attention_weight_min": -28.703426361083984,
      "activations/layer17_attention_weight_max": 34.69536590576172,
      "activations/layer17_attention_weight_min": -27.155946731567383,
      "activations/layer18_attention_weight_max": 38.87139129638672,
      "activations/layer18_attention_weight_min": -21.771413803100586,
      "activations/layer19_attention_weight_max": 40.532691955566406,
      "activations/layer19_attention_weight_min": -29.909353256225586,
      "activations/layer1_attention_weight_max": 16.91687774658203,
      "activations/layer1_attention_weight_min": -14.519584655761719,
      "activations/layer20_attention_weight_max": 33.844451904296875,
      "activations/layer20_attention_weight_min": -23.21410369873047,
      "activations/layer21_attention_weight_max": 32.83360290527344,
      "activations/layer21_attention_weight_min": -21.37619972229004,
      "activations/layer22_attention_weight_max": 47.215171813964844,
      "activations/layer22_attention_weight_min": -28.15287971496582,
      "activations/layer23_attention_weight_max": 34.148048400878906,
      "activations/layer23_attention_weight_min": -23.41554832458496,
      "activations/layer2_attention_weight_max": 31.8957576751709,
      "activations/layer2_attention_weight_min": -31.293964385986328,
      "activations/layer3_attention_weight_max": 91.788818359375,
      "activations/layer3_attention_weight_min": -94.40779113769531,
      "activations/layer4_attention_weight_max": 126.54559326171875,
      "activations/layer4_attention_weight_min": -128.8590850830078,
      "activations/layer5_attention_weight_max": 68.74356079101562,
      "activations/layer5_attention_weight_min": -68.31024932861328,
      "activations/layer6_attention_weight_max": 52.31056594848633,
      "activations/layer6_attention_weight_min": -54.3527717590332,
      "activations/layer7_attention_weight_max": 85.19007110595703,
      "activations/layer7_attention_weight_min": -86.03318786621094,
      "activations/layer8_attention_weight_max": 40.11995315551758,
      "activations/layer8_attention_weight_min": -40.60747528076172,
      "activations/layer9_attention_weight_max": 37.30522155761719,
      "activations/layer9_attention_weight_min": -34.398231506347656,
      "epoch": 6.34,
      "learning_rate": 0.0001102276515151515,
      "loss": 2.8244,
      "step": 109100
    },
    {
      "activations/layer0_attention_weight_max": 16.248428344726562,
      "activations/layer0_attention_weight_min": -10.596370697021484,
      "activations/layer10_attention_weight_max": 32.61525344848633,
      "activations/layer10_attention_weight_min": -30.64369773864746,
      "activations/layer11_attention_weight_max": 33.40570068359375,
      "activations/layer11_attention_weight_min": -27.84710693359375,
      "activations/layer12_attention_weight_max": 26.441972732543945,
      "activations/layer12_attention_weight_min": -32.22328186035156,
      "activations/layer13_attention_weight_max": 43.296138763427734,
      "activations/layer13_attention_weight_min": -32.353858947753906,
      "activations/layer14_attention_weight_max": 54.45717239379883,
      "activations/layer14_attention_weight_min": -38.79274368286133,
      "activations/layer15_attention_weight_max": 35.07927322387695,
      "activations/layer15_attention_weight_min": -29.45782470703125,
      "activations/layer16_attention_weight_max": 32.29580307006836,
      "activations/layer16_attention_weight_min": -27.11495590209961,
      "activations/layer17_attention_weight_max": 34.12582778930664,
      "activations/layer17_attention_weight_min": -24.431514739990234,
      "activations/layer18_attention_weight_max": 30.97838020324707,
      "activations/layer18_attention_weight_min": -22.41033935546875,
      "activations/layer19_attention_weight_max": 36.16936111450195,
      "activations/layer19_attention_weight_min": -30.263486862182617,
      "activations/layer1_attention_weight_max": 18.211360931396484,
      "activations/layer1_attention_weight_min": -15.314821243286133,
      "activations/layer20_attention_weight_max": 31.483293533325195,
      "activations/layer20_attention_weight_min": -25.24211883544922,
      "activations/layer21_attention_weight_max": 31.945432662963867,
      "activations/layer21_attention_weight_min": -23.42095947265625,
      "activations/layer22_attention_weight_max": 51.0995979309082,
      "activations/layer22_attention_weight_min": -29.122501373291016,
      "activations/layer23_attention_weight_max": 34.60009765625,
      "activations/layer23_attention_weight_min": -23.705047607421875,
      "activations/layer2_attention_weight_max": 31.8441162109375,
      "activations/layer2_attention_weight_min": -29.291641235351562,
      "activations/layer3_attention_weight_max": 90.3970947265625,
      "activations/layer3_attention_weight_min": -89.50094604492188,
      "activations/layer4_attention_weight_max": 125.7615966796875,
      "activations/layer4_attention_weight_min": -124.2154769897461,
      "activations/layer5_attention_weight_max": 68.45262145996094,
      "activations/layer5_attention_weight_min": -66.58204650878906,
      "activations/layer6_attention_weight_max": 51.42053985595703,
      "activations/layer6_attention_weight_min": -54.23612976074219,
      "activations/layer7_attention_weight_max": 82.26150512695312,
      "activations/layer7_attention_weight_min": -83.2606430053711,
      "activations/layer8_attention_weight_max": 39.45975875854492,
      "activations/layer8_attention_weight_min": -37.995792388916016,
      "activations/layer9_attention_weight_max": 37.359107971191406,
      "activations/layer9_attention_weight_min": -31.978229522705078,
      "epoch": 6.34,
      "learning_rate": 0.00011020871212121211,
      "loss": 2.8421,
      "step": 109150
    },
    {
      "activations/layer0_attention_weight_max": 16.27789306640625,
      "activations/layer0_attention_weight_min": -11.336482048034668,
      "activations/layer10_attention_weight_max": 34.09974670410156,
      "activations/layer10_attention_weight_min": -34.22901153564453,
      "activations/layer11_attention_weight_max": 31.51160430908203,
      "activations/layer11_attention_weight_min": -30.688003540039062,
      "activations/layer12_attention_weight_max": 21.992706298828125,
      "activations/layer12_attention_weight_min": -27.42420196533203,
      "activations/layer13_attention_weight_max": 38.019996643066406,
      "activations/layer13_attention_weight_min": -32.205665588378906,
      "activations/layer14_attention_weight_max": 55.634925842285156,
      "activations/layer14_attention_weight_min": -44.583702087402344,
      "activations/layer15_attention_weight_max": 36.224510192871094,
      "activations/layer15_attention_weight_min": -29.880290985107422,
      "activations/layer16_attention_weight_max": 36.68035125732422,
      "activations/layer16_attention_weight_min": -27.915729522705078,
      "activations/layer17_attention_weight_max": 39.205806732177734,
      "activations/layer17_attention_weight_min": -24.592905044555664,
      "activations/layer18_attention_weight_max": 32.18041229248047,
      "activations/layer18_attention_weight_min": -19.51581573486328,
      "activations/layer19_attention_weight_max": 40.71670150756836,
      "activations/layer19_attention_weight_min": -32.584442138671875,
      "activations/layer1_attention_weight_max": 18.203136444091797,
      "activations/layer1_attention_weight_min": -13.789478302001953,
      "activations/layer20_attention_weight_max": 33.84330749511719,
      "activations/layer20_attention_weight_min": -25.136272430419922,
      "activations/layer21_attention_weight_max": 32.71913528442383,
      "activations/layer21_attention_weight_min": -21.40906524658203,
      "activations/layer22_attention_weight_max": 46.93754959106445,
      "activations/layer22_attention_weight_min": -27.8950252532959,
      "activations/layer23_attention_weight_max": 33.133602142333984,
      "activations/layer23_attention_weight_min": -22.796947479248047,
      "activations/layer2_attention_weight_max": 32.12980651855469,
      "activations/layer2_attention_weight_min": -29.6448974609375,
      "activations/layer3_attention_weight_max": 89.79528045654297,
      "activations/layer3_attention_weight_min": -89.2459945678711,
      "activations/layer4_attention_weight_max": 126.5494613647461,
      "activations/layer4_attention_weight_min": -125.5418930053711,
      "activations/layer5_attention_weight_max": 68.8240737915039,
      "activations/layer5_attention_weight_min": -67.93997192382812,
      "activations/layer6_attention_weight_max": 54.859561920166016,
      "activations/layer6_attention_weight_min": -52.99922180175781,
      "activations/layer7_attention_weight_max": 90.18753814697266,
      "activations/layer7_attention_weight_min": -84.07921600341797,
      "activations/layer8_attention_weight_max": 40.20292663574219,
      "activations/layer8_attention_weight_min": -41.26434326171875,
      "activations/layer9_attention_weight_max": 38.91324234008789,
      "activations/layer9_attention_weight_min": -34.80641555786133,
      "epoch": 6.35,
      "learning_rate": 0.00011018977272727272,
      "loss": 2.836,
      "step": 109200
    },
    {
      "activations/layer0_attention_weight_max": 15.859739303588867,
      "activations/layer0_attention_weight_min": -11.406669616699219,
      "activations/layer10_attention_weight_max": 33.591407775878906,
      "activations/layer10_attention_weight_min": -31.161996841430664,
      "activations/layer11_attention_weight_max": 32.51363754272461,
      "activations/layer11_attention_weight_min": -30.553974151611328,
      "activations/layer12_attention_weight_max": 26.27677345275879,
      "activations/layer12_attention_weight_min": -28.296178817749023,
      "activations/layer13_attention_weight_max": 46.67888641357422,
      "activations/layer13_attention_weight_min": -33.96355056762695,
      "activations/layer14_attention_weight_max": 61.74982452392578,
      "activations/layer14_attention_weight_min": -45.117576599121094,
      "activations/layer15_attention_weight_max": 37.24515914916992,
      "activations/layer15_attention_weight_min": -29.762672424316406,
      "activations/layer16_attention_weight_max": 35.93104934692383,
      "activations/layer16_attention_weight_min": -30.134973526000977,
      "activations/layer17_attention_weight_max": 40.7974967956543,
      "activations/layer17_attention_weight_min": -26.85258674621582,
      "activations/layer18_attention_weight_max": 32.495887756347656,
      "activations/layer18_attention_weight_min": -22.04151725769043,
      "activations/layer19_attention_weight_max": 41.48231887817383,
      "activations/layer19_attention_weight_min": -30.83625602722168,
      "activations/layer1_attention_weight_max": 17.54965591430664,
      "activations/layer1_attention_weight_min": -15.41904354095459,
      "activations/layer20_attention_weight_max": 34.68966293334961,
      "activations/layer20_attention_weight_min": -23.763059616088867,
      "activations/layer21_attention_weight_max": 41.91102981567383,
      "activations/layer21_attention_weight_min": -22.775856018066406,
      "activations/layer22_attention_weight_max": 53.83026123046875,
      "activations/layer22_attention_weight_min": -28.700193405151367,
      "activations/layer23_attention_weight_max": 41.00730895996094,
      "activations/layer23_attention_weight_min": -20.381690979003906,
      "activations/layer2_attention_weight_max": 31.515583038330078,
      "activations/layer2_attention_weight_min": -31.149343490600586,
      "activations/layer3_attention_weight_max": 89.18119812011719,
      "activations/layer3_attention_weight_min": -91.02609252929688,
      "activations/layer4_attention_weight_max": 126.3009262084961,
      "activations/layer4_attention_weight_min": -123.94237518310547,
      "activations/layer5_attention_weight_max": 70.07398223876953,
      "activations/layer5_attention_weight_min": -65.9451675415039,
      "activations/layer6_attention_weight_max": 54.29948425292969,
      "activations/layer6_attention_weight_min": -57.3878173828125,
      "activations/layer7_attention_weight_max": 86.04430389404297,
      "activations/layer7_attention_weight_min": -87.61139678955078,
      "activations/layer8_attention_weight_max": 40.674110412597656,
      "activations/layer8_attention_weight_min": -39.90030288696289,
      "activations/layer9_attention_weight_max": 37.855953216552734,
      "activations/layer9_attention_weight_min": -35.0922966003418,
      "epoch": 6.35,
      "learning_rate": 0.00011017083333333333,
      "loss": 2.8228,
      "step": 109250
    },
    {
      "activations/layer0_attention_weight_max": 16.28689956665039,
      "activations/layer0_attention_weight_min": -11.537008285522461,
      "activations/layer10_attention_weight_max": 37.58066177368164,
      "activations/layer10_attention_weight_min": -31.406585693359375,
      "activations/layer11_attention_weight_max": 35.67994689941406,
      "activations/layer11_attention_weight_min": -26.703140258789062,
      "activations/layer12_attention_weight_max": 22.970298767089844,
      "activations/layer12_attention_weight_min": -27.05516242980957,
      "activations/layer13_attention_weight_max": 42.079071044921875,
      "activations/layer13_attention_weight_min": -33.03056716918945,
      "activations/layer14_attention_weight_max": 46.71762466430664,
      "activations/layer14_attention_weight_min": -36.49668884277344,
      "activations/layer15_attention_weight_max": 39.14207458496094,
      "activations/layer15_attention_weight_min": -29.4604549407959,
      "activations/layer16_attention_weight_max": 36.42970657348633,
      "activations/layer16_attention_weight_min": -27.178010940551758,
      "activations/layer17_attention_weight_max": 34.6209831237793,
      "activations/layer17_attention_weight_min": -24.505821228027344,
      "activations/layer18_attention_weight_max": 31.515676498413086,
      "activations/layer18_attention_weight_min": -21.055479049682617,
      "activations/layer19_attention_weight_max": 38.13620376586914,
      "activations/layer19_attention_weight_min": -30.167837142944336,
      "activations/layer1_attention_weight_max": 17.448406219482422,
      "activations/layer1_attention_weight_min": -13.760440826416016,
      "activations/layer20_attention_weight_max": 28.8942928314209,
      "activations/layer20_attention_weight_min": -23.967531204223633,
      "activations/layer21_attention_weight_max": 28.960166931152344,
      "activations/layer21_attention_weight_min": -21.317340850830078,
      "activations/layer22_attention_weight_max": 46.78203201293945,
      "activations/layer22_attention_weight_min": -29.164413452148438,
      "activations/layer23_attention_weight_max": 36.58203887939453,
      "activations/layer23_attention_weight_min": -21.925891876220703,
      "activations/layer2_attention_weight_max": 30.026180267333984,
      "activations/layer2_attention_weight_min": -28.91969108581543,
      "activations/layer3_attention_weight_max": 88.9787826538086,
      "activations/layer3_attention_weight_min": -91.88680267333984,
      "activations/layer4_attention_weight_max": 126.08257293701172,
      "activations/layer4_attention_weight_min": -123.3466796875,
      "activations/layer5_attention_weight_max": 71.89625549316406,
      "activations/layer5_attention_weight_min": -67.7623519897461,
      "activations/layer6_attention_weight_max": 53.81636047363281,
      "activations/layer6_attention_weight_min": -56.658836364746094,
      "activations/layer7_attention_weight_max": 83.70491790771484,
      "activations/layer7_attention_weight_min": -86.42327117919922,
      "activations/layer8_attention_weight_max": 37.84880828857422,
      "activations/layer8_attention_weight_min": -40.77364730834961,
      "activations/layer9_attention_weight_max": 35.64470672607422,
      "activations/layer9_attention_weight_min": -33.909759521484375,
      "epoch": 6.35,
      "learning_rate": 0.00011015189393939393,
      "loss": 2.8228,
      "step": 109300
    },
    {
      "activations/layer0_attention_weight_max": 15.836477279663086,
      "activations/layer0_attention_weight_min": -10.972980499267578,
      "activations/layer10_attention_weight_max": 36.73039627075195,
      "activations/layer10_attention_weight_min": -32.73007583618164,
      "activations/layer11_attention_weight_max": 41.40686798095703,
      "activations/layer11_attention_weight_min": -31.10898208618164,
      "activations/layer12_attention_weight_max": 26.096576690673828,
      "activations/layer12_attention_weight_min": -24.94647979736328,
      "activations/layer13_attention_weight_max": 54.024288177490234,
      "activations/layer13_attention_weight_min": -33.3773307800293,
      "activations/layer14_attention_weight_max": 69.02217864990234,
      "activations/layer14_attention_weight_min": -45.82185363769531,
      "activations/layer15_attention_weight_max": 42.36248779296875,
      "activations/layer15_attention_weight_min": -29.712451934814453,
      "activations/layer16_attention_weight_max": 40.294090270996094,
      "activations/layer16_attention_weight_min": -29.327293395996094,
      "activations/layer17_attention_weight_max": 38.92018508911133,
      "activations/layer17_attention_weight_min": -23.455177307128906,
      "activations/layer18_attention_weight_max": 40.038658142089844,
      "activations/layer18_attention_weight_min": -19.066932678222656,
      "activations/layer19_attention_weight_max": 49.335811614990234,
      "activations/layer19_attention_weight_min": -30.026784896850586,
      "activations/layer1_attention_weight_max": 17.427217483520508,
      "activations/layer1_attention_weight_min": -14.130538940429688,
      "activations/layer20_attention_weight_max": 37.894203186035156,
      "activations/layer20_attention_weight_min": -22.602209091186523,
      "activations/layer21_attention_weight_max": 35.64485168457031,
      "activations/layer21_attention_weight_min": -20.615880966186523,
      "activations/layer22_attention_weight_max": 57.908077239990234,
      "activations/layer22_attention_weight_min": -27.2289981842041,
      "activations/layer23_attention_weight_max": 35.17387390136719,
      "activations/layer23_attention_weight_min": -21.98200798034668,
      "activations/layer2_attention_weight_max": 32.004615783691406,
      "activations/layer2_attention_weight_min": -28.88732147216797,
      "activations/layer3_attention_weight_max": 94.53546905517578,
      "activations/layer3_attention_weight_min": -93.71476745605469,
      "activations/layer4_attention_weight_max": 124.2049789428711,
      "activations/layer4_attention_weight_min": -124.18037414550781,
      "activations/layer5_attention_weight_max": 68.64543914794922,
      "activations/layer5_attention_weight_min": -66.47129821777344,
      "activations/layer6_attention_weight_max": 52.40985870361328,
      "activations/layer6_attention_weight_min": -53.23399353027344,
      "activations/layer7_attention_weight_max": 85.5281753540039,
      "activations/layer7_attention_weight_min": -86.41680908203125,
      "activations/layer8_attention_weight_max": 42.49409484863281,
      "activations/layer8_attention_weight_min": -39.25844955444336,
      "activations/layer9_attention_weight_max": 39.15348815917969,
      "activations/layer9_attention_weight_min": -33.93559265136719,
      "epoch": 6.35,
      "learning_rate": 0.00011013295454545454,
      "loss": 2.8285,
      "step": 109350
    },
    {
      "activations/layer0_attention_weight_max": 15.80699634552002,
      "activations/layer0_attention_weight_min": -12.023158073425293,
      "activations/layer10_attention_weight_max": 30.807262420654297,
      "activations/layer10_attention_weight_min": -30.96154022216797,
      "activations/layer11_attention_weight_max": 33.65922546386719,
      "activations/layer11_attention_weight_min": -27.031848907470703,
      "activations/layer12_attention_weight_max": 27.07012367248535,
      "activations/layer12_attention_weight_min": -24.98038673400879,
      "activations/layer13_attention_weight_max": 40.49861526489258,
      "activations/layer13_attention_weight_min": -32.21192169189453,
      "activations/layer14_attention_weight_max": 51.19007873535156,
      "activations/layer14_attention_weight_min": -40.356632232666016,
      "activations/layer15_attention_weight_max": 36.6988525390625,
      "activations/layer15_attention_weight_min": -28.454692840576172,
      "activations/layer16_attention_weight_max": 33.30112838745117,
      "activations/layer16_attention_weight_min": -27.59832000732422,
      "activations/layer17_attention_weight_max": 35.51601791381836,
      "activations/layer17_attention_weight_min": -24.759605407714844,
      "activations/layer18_attention_weight_max": 31.12710189819336,
      "activations/layer18_attention_weight_min": -20.27165985107422,
      "activations/layer19_attention_weight_max": 36.004581451416016,
      "activations/layer19_attention_weight_min": -29.608457565307617,
      "activations/layer1_attention_weight_max": 17.159072875976562,
      "activations/layer1_attention_weight_min": -14.804707527160645,
      "activations/layer20_attention_weight_max": 30.026403427124023,
      "activations/layer20_attention_weight_min": -24.639814376831055,
      "activations/layer21_attention_weight_max": 28.015350341796875,
      "activations/layer21_attention_weight_min": -22.151046752929688,
      "activations/layer22_attention_weight_max": 47.614227294921875,
      "activations/layer22_attention_weight_min": -29.04228401184082,
      "activations/layer23_attention_weight_max": 34.71903991699219,
      "activations/layer23_attention_weight_min": -21.83313751220703,
      "activations/layer2_attention_weight_max": 32.75159454345703,
      "activations/layer2_attention_weight_min": -30.295303344726562,
      "activations/layer3_attention_weight_max": 89.94612121582031,
      "activations/layer3_attention_weight_min": -93.57906341552734,
      "activations/layer4_attention_weight_max": 124.90958404541016,
      "activations/layer4_attention_weight_min": -119.2934341430664,
      "activations/layer5_attention_weight_max": 67.07862854003906,
      "activations/layer5_attention_weight_min": -64.87518310546875,
      "activations/layer6_attention_weight_max": 52.409976959228516,
      "activations/layer6_attention_weight_min": -54.42458724975586,
      "activations/layer7_attention_weight_max": 84.62316131591797,
      "activations/layer7_attention_weight_min": -86.27582550048828,
      "activations/layer8_attention_weight_max": 38.63724899291992,
      "activations/layer8_attention_weight_min": -40.83159255981445,
      "activations/layer9_attention_weight_max": 32.82758712768555,
      "activations/layer9_attention_weight_min": -33.529109954833984,
      "epoch": 6.36,
      "learning_rate": 0.00011011401515151514,
      "loss": 2.8484,
      "step": 109400
    },
    {
      "activations/layer0_attention_weight_max": 16.110994338989258,
      "activations/layer0_attention_weight_min": -12.146181106567383,
      "activations/layer10_attention_weight_max": 36.52170181274414,
      "activations/layer10_attention_weight_min": -32.872718811035156,
      "activations/layer11_attention_weight_max": 33.404544830322266,
      "activations/layer11_attention_weight_min": -30.636951446533203,
      "activations/layer12_attention_weight_max": 23.485801696777344,
      "activations/layer12_attention_weight_min": -24.795562744140625,
      "activations/layer13_attention_weight_max": 44.99101257324219,
      "activations/layer13_attention_weight_min": -33.893951416015625,
      "activations/layer14_attention_weight_max": 68.49314880371094,
      "activations/layer14_attention_weight_min": -46.0966911315918,
      "activations/layer15_attention_weight_max": 39.06663131713867,
      "activations/layer15_attention_weight_min": -28.914636611938477,
      "activations/layer16_attention_weight_max": 34.92825698852539,
      "activations/layer16_attention_weight_min": -28.484193801879883,
      "activations/layer17_attention_weight_max": 39.19044876098633,
      "activations/layer17_attention_weight_min": -25.138681411743164,
      "activations/layer18_attention_weight_max": 35.242610931396484,
      "activations/layer18_attention_weight_min": -19.990249633789062,
      "activations/layer19_attention_weight_max": 41.51166915893555,
      "activations/layer19_attention_weight_min": -32.980316162109375,
      "activations/layer1_attention_weight_max": 16.57169532775879,
      "activations/layer1_attention_weight_min": -14.98323917388916,
      "activations/layer20_attention_weight_max": 36.29596710205078,
      "activations/layer20_attention_weight_min": -23.945465087890625,
      "activations/layer21_attention_weight_max": 33.2179069519043,
      "activations/layer21_attention_weight_min": -24.650930404663086,
      "activations/layer22_attention_weight_max": 56.53655242919922,
      "activations/layer22_attention_weight_min": -30.048662185668945,
      "activations/layer23_attention_weight_max": 40.798248291015625,
      "activations/layer23_attention_weight_min": -21.487770080566406,
      "activations/layer2_attention_weight_max": 32.573753356933594,
      "activations/layer2_attention_weight_min": -30.983524322509766,
      "activations/layer3_attention_weight_max": 90.8849105834961,
      "activations/layer3_attention_weight_min": -93.91897583007812,
      "activations/layer4_attention_weight_max": 131.4273223876953,
      "activations/layer4_attention_weight_min": -126.228515625,
      "activations/layer5_attention_weight_max": 71.29641723632812,
      "activations/layer5_attention_weight_min": -70.0918197631836,
      "activations/layer6_attention_weight_max": 57.86586380004883,
      "activations/layer6_attention_weight_min": -58.174644470214844,
      "activations/layer7_attention_weight_max": 91.77261352539062,
      "activations/layer7_attention_weight_min": -93.28942108154297,
      "activations/layer8_attention_weight_max": 43.724151611328125,
      "activations/layer8_attention_weight_min": -42.27653884887695,
      "activations/layer9_attention_weight_max": 39.67372131347656,
      "activations/layer9_attention_weight_min": -36.415592193603516,
      "epoch": 6.36,
      "learning_rate": 0.00011009507575757575,
      "loss": 2.8407,
      "step": 109450
    },
    {
      "activations/layer0_attention_weight_max": 16.579931259155273,
      "activations/layer0_attention_weight_min": -11.930956840515137,
      "activations/layer10_attention_weight_max": 32.067569732666016,
      "activations/layer10_attention_weight_min": -29.39376449584961,
      "activations/layer11_attention_weight_max": 27.78188133239746,
      "activations/layer11_attention_weight_min": -28.27679443359375,
      "activations/layer12_attention_weight_max": 20.669677734375,
      "activations/layer12_attention_weight_min": -26.837486267089844,
      "activations/layer13_attention_weight_max": 37.57674789428711,
      "activations/layer13_attention_weight_min": -31.993213653564453,
      "activations/layer14_attention_weight_max": 49.9959602355957,
      "activations/layer14_attention_weight_min": -43.008522033691406,
      "activations/layer15_attention_weight_max": 34.27613067626953,
      "activations/layer15_attention_weight_min": -29.127296447753906,
      "activations/layer16_attention_weight_max": 31.22804832458496,
      "activations/layer16_attention_weight_min": -27.393754959106445,
      "activations/layer17_attention_weight_max": 37.90785217285156,
      "activations/layer17_attention_weight_min": -24.471832275390625,
      "activations/layer18_attention_weight_max": 34.95454025268555,
      "activations/layer18_attention_weight_min": -20.479373931884766,
      "activations/layer19_attention_weight_max": 35.76993942260742,
      "activations/layer19_attention_weight_min": -29.5100040435791,
      "activations/layer1_attention_weight_max": 16.75840187072754,
      "activations/layer1_attention_weight_min": -12.549843788146973,
      "activations/layer20_attention_weight_max": 29.676252365112305,
      "activations/layer20_attention_weight_min": -25.79523277282715,
      "activations/layer21_attention_weight_max": 29.815013885498047,
      "activations/layer21_attention_weight_min": -22.565601348876953,
      "activations/layer22_attention_weight_max": 44.286033630371094,
      "activations/layer22_attention_weight_min": -29.45973014831543,
      "activations/layer23_attention_weight_max": 30.740291595458984,
      "activations/layer23_attention_weight_min": -21.83255386352539,
      "activations/layer2_attention_weight_max": 31.170162200927734,
      "activations/layer2_attention_weight_min": -29.059059143066406,
      "activations/layer3_attention_weight_max": 88.22694396972656,
      "activations/layer3_attention_weight_min": -87.13208770751953,
      "activations/layer4_attention_weight_max": 122.55974578857422,
      "activations/layer4_attention_weight_min": -116.22442626953125,
      "activations/layer5_attention_weight_max": 70.12753295898438,
      "activations/layer5_attention_weight_min": -64.37885284423828,
      "activations/layer6_attention_weight_max": 55.51671600341797,
      "activations/layer6_attention_weight_min": -53.36921310424805,
      "activations/layer7_attention_weight_max": 86.31932067871094,
      "activations/layer7_attention_weight_min": -82.11595916748047,
      "activations/layer8_attention_weight_max": 38.59242248535156,
      "activations/layer8_attention_weight_min": -42.66954040527344,
      "activations/layer9_attention_weight_max": 33.20689392089844,
      "activations/layer9_attention_weight_min": -31.460342407226562,
      "epoch": 6.36,
      "learning_rate": 0.00011007613636363635,
      "loss": 2.8378,
      "step": 109500
    },
    {
      "activations/layer0_attention_weight_max": 16.710336685180664,
      "activations/layer0_attention_weight_min": -11.167340278625488,
      "activations/layer10_attention_weight_max": 37.650821685791016,
      "activations/layer10_attention_weight_min": -31.64439582824707,
      "activations/layer11_attention_weight_max": 37.30381393432617,
      "activations/layer11_attention_weight_min": -29.711896896362305,
      "activations/layer12_attention_weight_max": 25.517059326171875,
      "activations/layer12_attention_weight_min": -26.36405372619629,
      "activations/layer13_attention_weight_max": 46.558658599853516,
      "activations/layer13_attention_weight_min": -33.227256774902344,
      "activations/layer14_attention_weight_max": 61.07243728637695,
      "activations/layer14_attention_weight_min": -47.23564529418945,
      "activations/layer15_attention_weight_max": 36.36106872558594,
      "activations/layer15_attention_weight_min": -28.480392456054688,
      "activations/layer16_attention_weight_max": 36.868988037109375,
      "activations/layer16_attention_weight_min": -31.08472442626953,
      "activations/layer17_attention_weight_max": 38.485599517822266,
      "activations/layer17_attention_weight_min": -24.663755416870117,
      "activations/layer18_attention_weight_max": 34.05860137939453,
      "activations/layer18_attention_weight_min": -19.210186004638672,
      "activations/layer19_attention_weight_max": 39.31581115722656,
      "activations/layer19_attention_weight_min": -29.33376121520996,
      "activations/layer1_attention_weight_max": 16.888240814208984,
      "activations/layer1_attention_weight_min": -14.317475318908691,
      "activations/layer20_attention_weight_max": 30.783218383789062,
      "activations/layer20_attention_weight_min": -23.100418090820312,
      "activations/layer21_attention_weight_max": 37.30481719970703,
      "activations/layer21_attention_weight_min": -21.97850799560547,
      "activations/layer22_attention_weight_max": 51.337669372558594,
      "activations/layer22_attention_weight_min": -30.29972267150879,
      "activations/layer23_attention_weight_max": 36.85003662109375,
      "activations/layer23_attention_weight_min": -22.0974178314209,
      "activations/layer2_attention_weight_max": 34.57905578613281,
      "activations/layer2_attention_weight_min": -32.161827087402344,
      "activations/layer3_attention_weight_max": 93.0765151977539,
      "activations/layer3_attention_weight_min": -92.45873260498047,
      "activations/layer4_attention_weight_max": 129.76976013183594,
      "activations/layer4_attention_weight_min": -123.77140045166016,
      "activations/layer5_attention_weight_max": 75.41517639160156,
      "activations/layer5_attention_weight_min": -68.81352996826172,
      "activations/layer6_attention_weight_max": 53.265098571777344,
      "activations/layer6_attention_weight_min": -56.29892349243164,
      "activations/layer7_attention_weight_max": 89.47358703613281,
      "activations/layer7_attention_weight_min": -85.95232391357422,
      "activations/layer8_attention_weight_max": 42.27041244506836,
      "activations/layer8_attention_weight_min": -41.614261627197266,
      "activations/layer9_attention_weight_max": 39.30552291870117,
      "activations/layer9_attention_weight_min": -34.11045455932617,
      "epoch": 6.37,
      "learning_rate": 0.00011005719696969696,
      "loss": 2.8403,
      "step": 109550
    },
    {
      "activations/layer0_attention_weight_max": 16.242090225219727,
      "activations/layer0_attention_weight_min": -12.46554946899414,
      "activations/layer10_attention_weight_max": 34.62687301635742,
      "activations/layer10_attention_weight_min": -34.412681579589844,
      "activations/layer11_attention_weight_max": 38.46638488769531,
      "activations/layer11_attention_weight_min": -30.862289428710938,
      "activations/layer12_attention_weight_max": 38.893943786621094,
      "activations/layer12_attention_weight_min": -26.596134185791016,
      "activations/layer13_attention_weight_max": 44.59650802612305,
      "activations/layer13_attention_weight_min": -32.42267990112305,
      "activations/layer14_attention_weight_max": 61.48289489746094,
      "activations/layer14_attention_weight_min": -42.2908935546875,
      "activations/layer15_attention_weight_max": 40.734130859375,
      "activations/layer15_attention_weight_min": -31.171438217163086,
      "activations/layer16_attention_weight_max": 38.740814208984375,
      "activations/layer16_attention_weight_min": -29.19194793701172,
      "activations/layer17_attention_weight_max": 38.19743728637695,
      "activations/layer17_attention_weight_min": -26.014089584350586,
      "activations/layer18_attention_weight_max": 32.84874725341797,
      "activations/layer18_attention_weight_min": -23.537817001342773,
      "activations/layer19_attention_weight_max": 38.91709518432617,
      "activations/layer19_attention_weight_min": -30.41864013671875,
      "activations/layer1_attention_weight_max": 18.00333595275879,
      "activations/layer1_attention_weight_min": -13.77758502960205,
      "activations/layer20_attention_weight_max": 32.16498565673828,
      "activations/layer20_attention_weight_min": -23.130924224853516,
      "activations/layer21_attention_weight_max": 32.40391159057617,
      "activations/layer21_attention_weight_min": -22.114856719970703,
      "activations/layer22_attention_weight_max": 50.65439987182617,
      "activations/layer22_attention_weight_min": -32.384376525878906,
      "activations/layer23_attention_weight_max": 32.73780059814453,
      "activations/layer23_attention_weight_min": -24.7398681640625,
      "activations/layer2_attention_weight_max": 33.2828369140625,
      "activations/layer2_attention_weight_min": -31.028846740722656,
      "activations/layer3_attention_weight_max": 90.21331787109375,
      "activations/layer3_attention_weight_min": -89.13301849365234,
      "activations/layer4_attention_weight_max": 135.92726135253906,
      "activations/layer4_attention_weight_min": -121.89623260498047,
      "activations/layer5_attention_weight_max": 72.60166931152344,
      "activations/layer5_attention_weight_min": -65.37228393554688,
      "activations/layer6_attention_weight_max": 58.688720703125,
      "activations/layer6_attention_weight_min": -51.614662170410156,
      "activations/layer7_attention_weight_max": 90.08983612060547,
      "activations/layer7_attention_weight_min": -87.10128784179688,
      "activations/layer8_attention_weight_max": 41.6394157409668,
      "activations/layer8_attention_weight_min": -40.52153778076172,
      "activations/layer9_attention_weight_max": 37.79384231567383,
      "activations/layer9_attention_weight_min": -34.71458053588867,
      "epoch": 6.37,
      "learning_rate": 0.00011003825757575758,
      "loss": 2.8163,
      "step": 109600
    },
    {
      "activations/layer0_attention_weight_max": 15.667858123779297,
      "activations/layer0_attention_weight_min": -11.30219841003418,
      "activations/layer10_attention_weight_max": 35.51172637939453,
      "activations/layer10_attention_weight_min": -30.384416580200195,
      "activations/layer11_attention_weight_max": 37.89012908935547,
      "activations/layer11_attention_weight_min": -29.74036407470703,
      "activations/layer12_attention_weight_max": 30.74944305419922,
      "activations/layer12_attention_weight_min": -32.26367950439453,
      "activations/layer13_attention_weight_max": 51.24427032470703,
      "activations/layer13_attention_weight_min": -35.97545623779297,
      "activations/layer14_attention_weight_max": 56.09932327270508,
      "activations/layer14_attention_weight_min": -44.08831787109375,
      "activations/layer15_attention_weight_max": 40.95497131347656,
      "activations/layer15_attention_weight_min": -29.291309356689453,
      "activations/layer16_attention_weight_max": 35.58842086791992,
      "activations/layer16_attention_weight_min": -28.195594787597656,
      "activations/layer17_attention_weight_max": 34.48257064819336,
      "activations/layer17_attention_weight_min": -24.017837524414062,
      "activations/layer18_attention_weight_max": 31.6257381439209,
      "activations/layer18_attention_weight_min": -22.739849090576172,
      "activations/layer19_attention_weight_max": 36.907108306884766,
      "activations/layer19_attention_weight_min": -28.450342178344727,
      "activations/layer1_attention_weight_max": 16.729595184326172,
      "activations/layer1_attention_weight_min": -14.08264446258545,
      "activations/layer20_attention_weight_max": 29.41884422302246,
      "activations/layer20_attention_weight_min": -23.093809127807617,
      "activations/layer21_attention_weight_max": 29.72842025756836,
      "activations/layer21_attention_weight_min": -20.727357864379883,
      "activations/layer22_attention_weight_max": 47.10528564453125,
      "activations/layer22_attention_weight_min": -29.000255584716797,
      "activations/layer23_attention_weight_max": 34.44540023803711,
      "activations/layer23_attention_weight_min": -22.702457427978516,
      "activations/layer2_attention_weight_max": 34.9325065612793,
      "activations/layer2_attention_weight_min": -31.79702377319336,
      "activations/layer3_attention_weight_max": 96.96273040771484,
      "activations/layer3_attention_weight_min": -94.41486358642578,
      "activations/layer4_attention_weight_max": 125.68701171875,
      "activations/layer4_attention_weight_min": -120.2361068725586,
      "activations/layer5_attention_weight_max": 70.14723205566406,
      "activations/layer5_attention_weight_min": -67.03324890136719,
      "activations/layer6_attention_weight_max": 54.94844436645508,
      "activations/layer6_attention_weight_min": -54.59292221069336,
      "activations/layer7_attention_weight_max": 95.60315704345703,
      "activations/layer7_attention_weight_min": -86.3039321899414,
      "activations/layer8_attention_weight_max": 42.138267517089844,
      "activations/layer8_attention_weight_min": -41.714988708496094,
      "activations/layer9_attention_weight_max": 35.72380447387695,
      "activations/layer9_attention_weight_min": -36.68408966064453,
      "epoch": 6.37,
      "learning_rate": 0.00011001931818181816,
      "loss": 2.824,
      "step": 109650
    },
    {
      "activations/layer0_attention_weight_max": 15.152822494506836,
      "activations/layer0_attention_weight_min": -13.023266792297363,
      "activations/layer10_attention_weight_max": 32.06526565551758,
      "activations/layer10_attention_weight_min": -30.81642723083496,
      "activations/layer11_attention_weight_max": 29.766704559326172,
      "activations/layer11_attention_weight_min": -28.437049865722656,
      "activations/layer12_attention_weight_max": 32.256996154785156,
      "activations/layer12_attention_weight_min": -27.9228572845459,
      "activations/layer13_attention_weight_max": 39.733489990234375,
      "activations/layer13_attention_weight_min": -32.185157775878906,
      "activations/layer14_attention_weight_max": 54.652278900146484,
      "activations/layer14_attention_weight_min": -45.26023483276367,
      "activations/layer15_attention_weight_max": 34.96785354614258,
      "activations/layer15_attention_weight_min": -29.02791976928711,
      "activations/layer16_attention_weight_max": 32.80012893676758,
      "activations/layer16_attention_weight_min": -28.63652229309082,
      "activations/layer17_attention_weight_max": 37.49974060058594,
      "activations/layer17_attention_weight_min": -24.437898635864258,
      "activations/layer18_attention_weight_max": 31.840389251708984,
      "activations/layer18_attention_weight_min": -20.44375228881836,
      "activations/layer19_attention_weight_max": 36.31895446777344,
      "activations/layer19_attention_weight_min": -29.050825119018555,
      "activations/layer1_attention_weight_max": 19.040857315063477,
      "activations/layer1_attention_weight_min": -14.603714942932129,
      "activations/layer20_attention_weight_max": 33.95543670654297,
      "activations/layer20_attention_weight_min": -25.551918029785156,
      "activations/layer21_attention_weight_max": 29.723388671875,
      "activations/layer21_attention_weight_min": -22.445396423339844,
      "activations/layer22_attention_weight_max": 45.26817321777344,
      "activations/layer22_attention_weight_min": -30.337841033935547,
      "activations/layer23_attention_weight_max": 34.189231872558594,
      "activations/layer23_attention_weight_min": -22.685592651367188,
      "activations/layer2_attention_weight_max": 32.071754455566406,
      "activations/layer2_attention_weight_min": -30.856643676757812,
      "activations/layer3_attention_weight_max": 90.96206665039062,
      "activations/layer3_attention_weight_min": -92.29098510742188,
      "activations/layer4_attention_weight_max": 125.26712799072266,
      "activations/layer4_attention_weight_min": -120.74925994873047,
      "activations/layer5_attention_weight_max": 65.3971176147461,
      "activations/layer5_attention_weight_min": -64.54439544677734,
      "activations/layer6_attention_weight_max": 50.83926010131836,
      "activations/layer6_attention_weight_min": -53.56869888305664,
      "activations/layer7_attention_weight_max": 80.4575424194336,
      "activations/layer7_attention_weight_min": -84.4716567993164,
      "activations/layer8_attention_weight_max": 37.63178253173828,
      "activations/layer8_attention_weight_min": -38.19892120361328,
      "activations/layer9_attention_weight_max": 35.35169982910156,
      "activations/layer9_attention_weight_min": -33.66611099243164,
      "epoch": 6.37,
      "learning_rate": 0.00011000037878787878,
      "loss": 2.8416,
      "step": 109700
    },
    {
      "activations/layer0_attention_weight_max": 16.81276512145996,
      "activations/layer0_attention_weight_min": -12.398831367492676,
      "activations/layer10_attention_weight_max": 37.12662887573242,
      "activations/layer10_attention_weight_min": -32.906822204589844,
      "activations/layer11_attention_weight_max": 38.305076599121094,
      "activations/layer11_attention_weight_min": -29.983200073242188,
      "activations/layer12_attention_weight_max": 26.56897735595703,
      "activations/layer12_attention_weight_min": -25.11128044128418,
      "activations/layer13_attention_weight_max": 38.08277130126953,
      "activations/layer13_attention_weight_min": -32.64072799682617,
      "activations/layer14_attention_weight_max": 53.4595947265625,
      "activations/layer14_attention_weight_min": -42.77035140991211,
      "activations/layer15_attention_weight_max": 35.97575759887695,
      "activations/layer15_attention_weight_min": -27.354549407958984,
      "activations/layer16_attention_weight_max": 36.343666076660156,
      "activations/layer16_attention_weight_min": -29.805150985717773,
      "activations/layer17_attention_weight_max": 34.465850830078125,
      "activations/layer17_attention_weight_min": -24.37872314453125,
      "activations/layer18_attention_weight_max": 32.664459228515625,
      "activations/layer18_attention_weight_min": -21.213167190551758,
      "activations/layer19_attention_weight_max": 37.99262237548828,
      "activations/layer19_attention_weight_min": -28.72802734375,
      "activations/layer1_attention_weight_max": 17.853830337524414,
      "activations/layer1_attention_weight_min": -14.145417213439941,
      "activations/layer20_attention_weight_max": 36.048057556152344,
      "activations/layer20_attention_weight_min": -22.136911392211914,
      "activations/layer21_attention_weight_max": 37.39860534667969,
      "activations/layer21_attention_weight_min": -22.28995704650879,
      "activations/layer22_attention_weight_max": 48.019935607910156,
      "activations/layer22_attention_weight_min": -26.54973602294922,
      "activations/layer23_attention_weight_max": 34.75840759277344,
      "activations/layer23_attention_weight_min": -21.022380828857422,
      "activations/layer2_attention_weight_max": 34.513397216796875,
      "activations/layer2_attention_weight_min": -32.79204559326172,
      "activations/layer3_attention_weight_max": 98.64359283447266,
      "activations/layer3_attention_weight_min": -94.53311920166016,
      "activations/layer4_attention_weight_max": 136.86448669433594,
      "activations/layer4_attention_weight_min": -125.5108642578125,
      "activations/layer5_attention_weight_max": 68.45259857177734,
      "activations/layer5_attention_weight_min": -65.89192199707031,
      "activations/layer6_attention_weight_max": 52.74733352661133,
      "activations/layer6_attention_weight_min": -53.59144592285156,
      "activations/layer7_attention_weight_max": 85.54731750488281,
      "activations/layer7_attention_weight_min": -86.0927505493164,
      "activations/layer8_attention_weight_max": 42.13136672973633,
      "activations/layer8_attention_weight_min": -42.93246841430664,
      "activations/layer9_attention_weight_max": 36.75149154663086,
      "activations/layer9_attention_weight_min": -35.899696350097656,
      "epoch": 6.38,
      "learning_rate": 0.0001099814393939394,
      "loss": 2.8306,
      "step": 109750
    },
    {
      "activations/layer0_attention_weight_max": 16.467119216918945,
      "activations/layer0_attention_weight_min": -12.732917785644531,
      "activations/layer10_attention_weight_max": 36.88637924194336,
      "activations/layer10_attention_weight_min": -37.83106994628906,
      "activations/layer11_attention_weight_max": 36.350833892822266,
      "activations/layer11_attention_weight_min": -35.176231384277344,
      "activations/layer12_attention_weight_max": 22.82468605041504,
      "activations/layer12_attention_weight_min": -29.08067512512207,
      "activations/layer13_attention_weight_max": 42.120479583740234,
      "activations/layer13_attention_weight_min": -32.49300765991211,
      "activations/layer14_attention_weight_max": 56.62282943725586,
      "activations/layer14_attention_weight_min": -45.756141662597656,
      "activations/layer15_attention_weight_max": 39.11875534057617,
      "activations/layer15_attention_weight_min": -29.109926223754883,
      "activations/layer16_attention_weight_max": 33.75535583496094,
      "activations/layer16_attention_weight_min": -27.587751388549805,
      "activations/layer17_attention_weight_max": 35.5643196105957,
      "activations/layer17_attention_weight_min": -25.082704544067383,
      "activations/layer18_attention_weight_max": 36.8564453125,
      "activations/layer18_attention_weight_min": -20.554025650024414,
      "activations/layer19_attention_weight_max": 38.85877990722656,
      "activations/layer19_attention_weight_min": -29.04656410217285,
      "activations/layer1_attention_weight_max": 17.87827491760254,
      "activations/layer1_attention_weight_min": -17.1203556060791,
      "activations/layer20_attention_weight_max": 35.62856674194336,
      "activations/layer20_attention_weight_min": -23.963537216186523,
      "activations/layer21_attention_weight_max": 33.54085922241211,
      "activations/layer21_attention_weight_min": -22.08135986328125,
      "activations/layer22_attention_weight_max": 53.01173400878906,
      "activations/layer22_attention_weight_min": -27.91219711303711,
      "activations/layer23_attention_weight_max": 37.230987548828125,
      "activations/layer23_attention_weight_min": -21.270496368408203,
      "activations/layer2_attention_weight_max": 31.940250396728516,
      "activations/layer2_attention_weight_min": -30.673015594482422,
      "activations/layer3_attention_weight_max": 94.5867919921875,
      "activations/layer3_attention_weight_min": -94.12491607666016,
      "activations/layer4_attention_weight_max": 128.7726287841797,
      "activations/layer4_attention_weight_min": -124.25172424316406,
      "activations/layer5_attention_weight_max": 70.37690734863281,
      "activations/layer5_attention_weight_min": -71.53038024902344,
      "activations/layer6_attention_weight_max": 53.358333587646484,
      "activations/layer6_attention_weight_min": -55.9214973449707,
      "activations/layer7_attention_weight_max": 91.11073303222656,
      "activations/layer7_attention_weight_min": -98.6712875366211,
      "activations/layer8_attention_weight_max": 45.04096603393555,
      "activations/layer8_attention_weight_min": -46.45751190185547,
      "activations/layer9_attention_weight_max": 38.87446212768555,
      "activations/layer9_attention_weight_min": -37.553314208984375,
      "epoch": 6.38,
      "learning_rate": 0.00010996249999999998,
      "loss": 2.8358,
      "step": 109800
    },
    {
      "activations/layer0_attention_weight_max": 15.986319541931152,
      "activations/layer0_attention_weight_min": -11.397224426269531,
      "activations/layer10_attention_weight_max": 32.918304443359375,
      "activations/layer10_attention_weight_min": -33.42767333984375,
      "activations/layer11_attention_weight_max": 32.91096496582031,
      "activations/layer11_attention_weight_min": -29.36073875427246,
      "activations/layer12_attention_weight_max": 24.312721252441406,
      "activations/layer12_attention_weight_min": -26.60609245300293,
      "activations/layer13_attention_weight_max": 39.96344757080078,
      "activations/layer13_attention_weight_min": -33.18613052368164,
      "activations/layer14_attention_weight_max": 52.23934555053711,
      "activations/layer14_attention_weight_min": -42.065128326416016,
      "activations/layer15_attention_weight_max": 37.32624435424805,
      "activations/layer15_attention_weight_min": -31.462291717529297,
      "activations/layer16_attention_weight_max": 34.758872985839844,
      "activations/layer16_attention_weight_min": -30.513647079467773,
      "activations/layer17_attention_weight_max": 36.82076644897461,
      "activations/layer17_attention_weight_min": -27.682310104370117,
      "activations/layer18_attention_weight_max": 28.717037200927734,
      "activations/layer18_attention_weight_min": -19.852386474609375,
      "activations/layer19_attention_weight_max": 38.17600631713867,
      "activations/layer19_attention_weight_min": -34.280364990234375,
      "activations/layer1_attention_weight_max": 16.995410919189453,
      "activations/layer1_attention_weight_min": -14.566320419311523,
      "activations/layer20_attention_weight_max": 35.81068801879883,
      "activations/layer20_attention_weight_min": -26.861114501953125,
      "activations/layer21_attention_weight_max": 31.973730087280273,
      "activations/layer21_attention_weight_min": -23.127304077148438,
      "activations/layer22_attention_weight_max": 50.36881637573242,
      "activations/layer22_attention_weight_min": -28.034748077392578,
      "activations/layer23_attention_weight_max": 32.97681427001953,
      "activations/layer23_attention_weight_min": -21.07598304748535,
      "activations/layer2_attention_weight_max": 33.25725173950195,
      "activations/layer2_attention_weight_min": -30.715116500854492,
      "activations/layer3_attention_weight_max": 88.48882293701172,
      "activations/layer3_attention_weight_min": -90.19111633300781,
      "activations/layer4_attention_weight_max": 124.20191955566406,
      "activations/layer4_attention_weight_min": -117.4240951538086,
      "activations/layer5_attention_weight_max": 69.63388061523438,
      "activations/layer5_attention_weight_min": -66.57795715332031,
      "activations/layer6_attention_weight_max": 53.08071517944336,
      "activations/layer6_attention_weight_min": -55.85363006591797,
      "activations/layer7_attention_weight_max": 86.65282440185547,
      "activations/layer7_attention_weight_min": -87.39218139648438,
      "activations/layer8_attention_weight_max": 40.754661560058594,
      "activations/layer8_attention_weight_min": -41.171836853027344,
      "activations/layer9_attention_weight_max": 37.94976043701172,
      "activations/layer9_attention_weight_min": -34.06707000732422,
      "epoch": 6.38,
      "learning_rate": 0.0001099435606060606,
      "loss": 2.8266,
      "step": 109850
    },
    {
      "activations/layer0_attention_weight_max": 15.063454627990723,
      "activations/layer0_attention_weight_min": -13.346660614013672,
      "activations/layer10_attention_weight_max": 36.63428497314453,
      "activations/layer10_attention_weight_min": -32.634368896484375,
      "activations/layer11_attention_weight_max": 33.564666748046875,
      "activations/layer11_attention_weight_min": -29.3430233001709,
      "activations/layer12_attention_weight_max": 25.247779846191406,
      "activations/layer12_attention_weight_min": -25.96815299987793,
      "activations/layer13_attention_weight_max": 44.1041145324707,
      "activations/layer13_attention_weight_min": -33.61936569213867,
      "activations/layer14_attention_weight_max": 64.19882202148438,
      "activations/layer14_attention_weight_min": -44.93338394165039,
      "activations/layer15_attention_weight_max": 37.10814666748047,
      "activations/layer15_attention_weight_min": -27.49824333190918,
      "activations/layer16_attention_weight_max": 41.24607467651367,
      "activations/layer16_attention_weight_min": -29.86177635192871,
      "activations/layer17_attention_weight_max": 38.10905456542969,
      "activations/layer17_attention_weight_min": -25.849550247192383,
      "activations/layer18_attention_weight_max": 31.670211791992188,
      "activations/layer18_attention_weight_min": -19.74217414855957,
      "activations/layer19_attention_weight_max": 42.239471435546875,
      "activations/layer19_attention_weight_min": -32.39934158325195,
      "activations/layer1_attention_weight_max": 16.477697372436523,
      "activations/layer1_attention_weight_min": -12.971490859985352,
      "activations/layer20_attention_weight_max": 35.62804412841797,
      "activations/layer20_attention_weight_min": -24.349811553955078,
      "activations/layer21_attention_weight_max": 31.954227447509766,
      "activations/layer21_attention_weight_min": -20.301464080810547,
      "activations/layer22_attention_weight_max": 55.409461975097656,
      "activations/layer22_attention_weight_min": -27.19884490966797,
      "activations/layer23_attention_weight_max": 37.9394416809082,
      "activations/layer23_attention_weight_min": -21.715192794799805,
      "activations/layer2_attention_weight_max": 32.18303680419922,
      "activations/layer2_attention_weight_min": -31.902565002441406,
      "activations/layer3_attention_weight_max": 91.83637237548828,
      "activations/layer3_attention_weight_min": -92.36573028564453,
      "activations/layer4_attention_weight_max": 129.69210815429688,
      "activations/layer4_attention_weight_min": -125.12039947509766,
      "activations/layer5_attention_weight_max": 71.31524658203125,
      "activations/layer5_attention_weight_min": -66.5086898803711,
      "activations/layer6_attention_weight_max": 51.85710144042969,
      "activations/layer6_attention_weight_min": -56.59245300292969,
      "activations/layer7_attention_weight_max": 88.75798034667969,
      "activations/layer7_attention_weight_min": -85.46240997314453,
      "activations/layer8_attention_weight_max": 44.16803741455078,
      "activations/layer8_attention_weight_min": -42.16859436035156,
      "activations/layer9_attention_weight_max": 39.547916412353516,
      "activations/layer9_attention_weight_min": -37.81480026245117,
      "epoch": 6.39,
      "learning_rate": 0.0001099246212121212,
      "loss": 2.8395,
      "step": 109900
    },
    {
      "activations/layer0_attention_weight_max": 14.988451957702637,
      "activations/layer0_attention_weight_min": -11.792519569396973,
      "activations/layer10_attention_weight_max": 32.99980545043945,
      "activations/layer10_attention_weight_min": -30.856882095336914,
      "activations/layer11_attention_weight_max": 35.954345703125,
      "activations/layer11_attention_weight_min": -29.452556610107422,
      "activations/layer12_attention_weight_max": 26.833641052246094,
      "activations/layer12_attention_weight_min": -35.60184860229492,
      "activations/layer13_attention_weight_max": 41.2203369140625,
      "activations/layer13_attention_weight_min": -32.85268783569336,
      "activations/layer14_attention_weight_max": 47.784000396728516,
      "activations/layer14_attention_weight_min": -39.75059127807617,
      "activations/layer15_attention_weight_max": 46.60266876220703,
      "activations/layer15_attention_weight_min": -30.322555541992188,
      "activations/layer16_attention_weight_max": 37.08811569213867,
      "activations/layer16_attention_weight_min": -27.226097106933594,
      "activations/layer17_attention_weight_max": 34.19625473022461,
      "activations/layer17_attention_weight_min": -25.476083755493164,
      "activations/layer18_attention_weight_max": 33.7493896484375,
      "activations/layer18_attention_weight_min": -21.424156188964844,
      "activations/layer19_attention_weight_max": 38.38044357299805,
      "activations/layer19_attention_weight_min": -33.27806854248047,
      "activations/layer1_attention_weight_max": 19.277963638305664,
      "activations/layer1_attention_weight_min": -13.222532272338867,
      "activations/layer20_attention_weight_max": 30.099597930908203,
      "activations/layer20_attention_weight_min": -23.331708908081055,
      "activations/layer21_attention_weight_max": 30.683860778808594,
      "activations/layer21_attention_weight_min": -21.528940200805664,
      "activations/layer22_attention_weight_max": 52.23579788208008,
      "activations/layer22_attention_weight_min": -31.815746307373047,
      "activations/layer23_attention_weight_max": 37.65496826171875,
      "activations/layer23_attention_weight_min": -24.554792404174805,
      "activations/layer2_attention_weight_max": 31.976802825927734,
      "activations/layer2_attention_weight_min": -30.56488800048828,
      "activations/layer3_attention_weight_max": 89.74388885498047,
      "activations/layer3_attention_weight_min": -89.46915435791016,
      "activations/layer4_attention_weight_max": 126.44170379638672,
      "activations/layer4_attention_weight_min": -126.6343994140625,
      "activations/layer5_attention_weight_max": 69.5673828125,
      "activations/layer5_attention_weight_min": -66.47171020507812,
      "activations/layer6_attention_weight_max": 51.52924346923828,
      "activations/layer6_attention_weight_min": -56.39751434326172,
      "activations/layer7_attention_weight_max": 89.85476684570312,
      "activations/layer7_attention_weight_min": -86.30091857910156,
      "activations/layer8_attention_weight_max": 40.05659484863281,
      "activations/layer8_attention_weight_min": -38.46968460083008,
      "activations/layer9_attention_weight_max": 33.98386764526367,
      "activations/layer9_attention_weight_min": -43.87120056152344,
      "epoch": 6.39,
      "learning_rate": 0.0001099056818181818,
      "loss": 2.8321,
      "step": 109950
    },
    {
      "activations/layer0_attention_weight_max": 15.960637092590332,
      "activations/layer0_attention_weight_min": -12.157766342163086,
      "activations/layer10_attention_weight_max": 32.50326919555664,
      "activations/layer10_attention_weight_min": -29.758026123046875,
      "activations/layer11_attention_weight_max": 34.70057678222656,
      "activations/layer11_attention_weight_min": -27.75414276123047,
      "activations/layer12_attention_weight_max": 21.82695198059082,
      "activations/layer12_attention_weight_min": -26.220096588134766,
      "activations/layer13_attention_weight_max": 41.23862838745117,
      "activations/layer13_attention_weight_min": -33.2098503112793,
      "activations/layer14_attention_weight_max": 64.75395965576172,
      "activations/layer14_attention_weight_min": -51.264400482177734,
      "activations/layer15_attention_weight_max": 36.28993606567383,
      "activations/layer15_attention_weight_min": -31.02099609375,
      "activations/layer16_attention_weight_max": 37.57996368408203,
      "activations/layer16_attention_weight_min": -30.286897659301758,
      "activations/layer17_attention_weight_max": 33.532752990722656,
      "activations/layer17_attention_weight_min": -27.21636390686035,
      "activations/layer18_attention_weight_max": 34.26728057861328,
      "activations/layer18_attention_weight_min": -24.114622116088867,
      "activations/layer19_attention_weight_max": 39.22332000732422,
      "activations/layer19_attention_weight_min": -30.922704696655273,
      "activations/layer1_attention_weight_max": 16.666513442993164,
      "activations/layer1_attention_weight_min": -14.552412986755371,
      "activations/layer20_attention_weight_max": 30.833133697509766,
      "activations/layer20_attention_weight_min": -25.639259338378906,
      "activations/layer21_attention_weight_max": 31.062597274780273,
      "activations/layer21_attention_weight_min": -24.71409797668457,
      "activations/layer22_attention_weight_max": 49.89804458618164,
      "activations/layer22_attention_weight_min": -34.204524993896484,
      "activations/layer23_attention_weight_max": 39.277191162109375,
      "activations/layer23_attention_weight_min": -22.43996238708496,
      "activations/layer2_attention_weight_max": 35.269615173339844,
      "activations/layer2_attention_weight_min": -29.993322372436523,
      "activations/layer3_attention_weight_max": 95.10696411132812,
      "activations/layer3_attention_weight_min": -90.65660095214844,
      "activations/layer4_attention_weight_max": 133.23291015625,
      "activations/layer4_attention_weight_min": -122.76947021484375,
      "activations/layer5_attention_weight_max": 72.27236938476562,
      "activations/layer5_attention_weight_min": -65.78509521484375,
      "activations/layer6_attention_weight_max": 54.624271392822266,
      "activations/layer6_attention_weight_min": -54.92698287963867,
      "activations/layer7_attention_weight_max": 88.55728149414062,
      "activations/layer7_attention_weight_min": -84.4521713256836,
      "activations/layer8_attention_weight_max": 40.1609992980957,
      "activations/layer8_attention_weight_min": -38.609012603759766,
      "activations/layer9_attention_weight_max": 36.021522521972656,
      "activations/layer9_attention_weight_min": -33.19734573364258,
      "epoch": 6.39,
      "learning_rate": 0.00010988674242424242,
      "loss": 2.8414,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5082,
      "eval_samples_per_second": 504.691,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5082,
      "eval_openwebtext_samples_per_second": 504.691,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 1.9539,
      "eval_wikitext_samples_per_second": 233.374,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_lambada_loss": 2.72265625,
      "eval_lambada_ppl": 15.220698576992893,
      "eval_lambada_runtime": 9.5524,
      "eval_lambada_samples_per_second": 509.714,
      "step": 110000
    },
    {
      "activations/layer0_attention_weight_max": 16.12516212463379,
      "activations/layer0_attention_weight_min": -10.856735229492188,
      "activations/layer10_attention_weight_max": 33.44105529785156,
      "activations/layer10_attention_weight_min": -33.85413360595703,
      "activations/layer11_attention_weight_max": 32.14125442504883,
      "activations/layer11_attention_weight_min": -28.224010467529297,
      "activations/layer12_attention_weight_max": 27.955337524414062,
      "activations/layer12_attention_weight_min": -28.13176918029785,
      "activations/layer13_attention_weight_max": 42.271446228027344,
      "activations/layer13_attention_weight_min": -30.70888328552246,
      "activations/layer14_attention_weight_max": 65.237060546875,
      "activations/layer14_attention_weight_min": -49.51788330078125,
      "activations/layer15_attention_weight_max": 40.04731750488281,
      "activations/layer15_attention_weight_min": -30.20036506652832,
      "activations/layer16_attention_weight_max": 34.26021957397461,
      "activations/layer16_attention_weight_min": -28.57880401611328,
      "activations/layer17_attention_weight_max": 36.40217971801758,
      "activations/layer17_attention_weight_min": -25.08611297607422,
      "activations/layer18_attention_weight_max": 30.016347885131836,
      "activations/layer18_attention_weight_min": -20.80249786376953,
      "activations/layer19_attention_weight_max": 37.09568405151367,
      "activations/layer19_attention_weight_min": -30.569395065307617,
      "activations/layer1_attention_weight_max": 17.46480369567871,
      "activations/layer1_attention_weight_min": -14.87307357788086,
      "activations/layer20_attention_weight_max": 29.829811096191406,
      "activations/layer20_attention_weight_min": -23.024564743041992,
      "activations/layer21_attention_weight_max": 32.0152587890625,
      "activations/layer21_attention_weight_min": -21.365320205688477,
      "activations/layer22_attention_weight_max": 48.60086441040039,
      "activations/layer22_attention_weight_min": -28.97257423400879,
      "activations/layer23_attention_weight_max": 35.262969970703125,
      "activations/layer23_attention_weight_min": -22.500831604003906,
      "activations/layer2_attention_weight_max": 35.384315490722656,
      "activations/layer2_attention_weight_min": -34.591148376464844,
      "activations/layer3_attention_weight_max": 98.76534271240234,
      "activations/layer3_attention_weight_min": -97.84965515136719,
      "activations/layer4_attention_weight_max": 129.44668579101562,
      "activations/layer4_attention_weight_min": -127.93790435791016,
      "activations/layer5_attention_weight_max": 72.00402069091797,
      "activations/layer5_attention_weight_min": -70.21170043945312,
      "activations/layer6_attention_weight_max": 57.97492599487305,
      "activations/layer6_attention_weight_min": -54.220523834228516,
      "activations/layer7_attention_weight_max": 96.54899597167969,
      "activations/layer7_attention_weight_min": -87.18018341064453,
      "activations/layer8_attention_weight_max": 40.28499221801758,
      "activations/layer8_attention_weight_min": -39.89126968383789,
      "activations/layer9_attention_weight_max": 37.5966911315918,
      "activations/layer9_attention_weight_min": -36.128807067871094,
      "epoch": 6.39,
      "learning_rate": 0.0001098681818181818,
      "loss": 2.8173,
      "step": 110050
    },
    {
      "activations/layer0_attention_weight_max": 15.192723274230957,
      "activations/layer0_attention_weight_min": -12.92817497253418,
      "activations/layer10_attention_weight_max": 32.2711181640625,
      "activations/layer10_attention_weight_min": -30.88694953918457,
      "activations/layer11_attention_weight_max": 32.756553649902344,
      "activations/layer11_attention_weight_min": -28.31939697265625,
      "activations/layer12_attention_weight_max": 47.91426086425781,
      "activations/layer12_attention_weight_min": -24.717506408691406,
      "activations/layer13_attention_weight_max": 47.77878952026367,
      "activations/layer13_attention_weight_min": -31.473403930664062,
      "activations/layer14_attention_weight_max": 65.12408447265625,
      "activations/layer14_attention_weight_min": -46.73006820678711,
      "activations/layer15_attention_weight_max": 42.192893981933594,
      "activations/layer15_attention_weight_min": -28.011428833007812,
      "activations/layer16_attention_weight_max": 34.25025177001953,
      "activations/layer16_attention_weight_min": -28.660255432128906,
      "activations/layer17_attention_weight_max": 41.47173309326172,
      "activations/layer17_attention_weight_min": -24.556238174438477,
      "activations/layer18_attention_weight_max": 33.122833251953125,
      "activations/layer18_attention_weight_min": -20.345149993896484,
      "activations/layer19_attention_weight_max": 38.47896957397461,
      "activations/layer19_attention_weight_min": -28.62871742248535,
      "activations/layer1_attention_weight_max": 18.202213287353516,
      "activations/layer1_attention_weight_min": -13.800263404846191,
      "activations/layer20_attention_weight_max": 34.780391693115234,
      "activations/layer20_attention_weight_min": -22.571081161499023,
      "activations/layer21_attention_weight_max": 33.51198196411133,
      "activations/layer21_attention_weight_min": -20.679630279541016,
      "activations/layer22_attention_weight_max": 59.70432662963867,
      "activations/layer22_attention_weight_min": -28.7130184173584,
      "activations/layer23_attention_weight_max": 35.43659973144531,
      "activations/layer23_attention_weight_min": -20.390378952026367,
      "activations/layer2_attention_weight_max": 30.874576568603516,
      "activations/layer2_attention_weight_min": -30.556568145751953,
      "activations/layer3_attention_weight_max": 88.90058135986328,
      "activations/layer3_attention_weight_min": -88.34272766113281,
      "activations/layer4_attention_weight_max": 120.11614227294922,
      "activations/layer4_attention_weight_min": -120.87462615966797,
      "activations/layer5_attention_weight_max": 65.3052749633789,
      "activations/layer5_attention_weight_min": -63.64303207397461,
      "activations/layer6_attention_weight_max": 51.15256118774414,
      "activations/layer6_attention_weight_min": -56.823028564453125,
      "activations/layer7_attention_weight_max": 90.4093017578125,
      "activations/layer7_attention_weight_min": -83.8061752319336,
      "activations/layer8_attention_weight_max": 43.3873176574707,
      "activations/layer8_attention_weight_min": -41.93465805053711,
      "activations/layer9_attention_weight_max": 37.64128494262695,
      "activations/layer9_attention_weight_min": -34.143959045410156,
      "epoch": 6.4,
      "learning_rate": 0.00010984924242424241,
      "loss": 2.8271,
      "step": 110100
    },
    {
      "activations/layer0_attention_weight_max": 14.478583335876465,
      "activations/layer0_attention_weight_min": -13.912432670593262,
      "activations/layer10_attention_weight_max": 34.61699295043945,
      "activations/layer10_attention_weight_min": -34.259254455566406,
      "activations/layer11_attention_weight_max": 34.12560272216797,
      "activations/layer11_attention_weight_min": -30.69293212890625,
      "activations/layer12_attention_weight_max": 34.37358474731445,
      "activations/layer12_attention_weight_min": -28.846271514892578,
      "activations/layer13_attention_weight_max": 46.607357025146484,
      "activations/layer13_attention_weight_min": -32.68839645385742,
      "activations/layer14_attention_weight_max": 78.03117370605469,
      "activations/layer14_attention_weight_min": -48.404605865478516,
      "activations/layer15_attention_weight_max": 41.20658874511719,
      "activations/layer15_attention_weight_min": -28.475343704223633,
      "activations/layer16_attention_weight_max": 45.741493225097656,
      "activations/layer16_attention_weight_min": -29.794736862182617,
      "activations/layer17_attention_weight_max": 38.76860427856445,
      "activations/layer17_attention_weight_min": -26.09609603881836,
      "activations/layer18_attention_weight_max": 34.27578353881836,
      "activations/layer18_attention_weight_min": -20.858152389526367,
      "activations/layer19_attention_weight_max": 46.070289611816406,
      "activations/layer19_attention_weight_min": -31.343530654907227,
      "activations/layer1_attention_weight_max": 17.65000343322754,
      "activations/layer1_attention_weight_min": -15.643024444580078,
      "activations/layer20_attention_weight_max": 34.68365478515625,
      "activations/layer20_attention_weight_min": -23.487098693847656,
      "activations/layer21_attention_weight_max": 32.421390533447266,
      "activations/layer21_attention_weight_min": -22.675241470336914,
      "activations/layer22_attention_weight_max": 56.96921157836914,
      "activations/layer22_attention_weight_min": -29.158159255981445,
      "activations/layer23_attention_weight_max": 40.915138244628906,
      "activations/layer23_attention_weight_min": -26.544116973876953,
      "activations/layer2_attention_weight_max": 32.215415954589844,
      "activations/layer2_attention_weight_min": -30.428449630737305,
      "activations/layer3_attention_weight_max": 99.90499877929688,
      "activations/layer3_attention_weight_min": -97.2247314453125,
      "activations/layer4_attention_weight_max": 136.0289306640625,
      "activations/layer4_attention_weight_min": -129.40478515625,
      "activations/layer5_attention_weight_max": 71.55366516113281,
      "activations/layer5_attention_weight_min": -68.04993438720703,
      "activations/layer6_attention_weight_max": 55.49395751953125,
      "activations/layer6_attention_weight_min": -57.3222770690918,
      "activations/layer7_attention_weight_max": 93.21997833251953,
      "activations/layer7_attention_weight_min": -93.93512725830078,
      "activations/layer8_attention_weight_max": 41.70827102661133,
      "activations/layer8_attention_weight_min": -42.44974136352539,
      "activations/layer9_attention_weight_max": 39.53034210205078,
      "activations/layer9_attention_weight_min": -35.81116485595703,
      "epoch": 6.4,
      "learning_rate": 0.00010983030303030302,
      "loss": 2.8479,
      "step": 110150
    },
    {
      "activations/layer0_attention_weight_max": 15.17789363861084,
      "activations/layer0_attention_weight_min": -12.298327445983887,
      "activations/layer10_attention_weight_max": 34.63203048706055,
      "activations/layer10_attention_weight_min": -34.135231018066406,
      "activations/layer11_attention_weight_max": 32.316349029541016,
      "activations/layer11_attention_weight_min": -28.334178924560547,
      "activations/layer12_attention_weight_max": 30.013708114624023,
      "activations/layer12_attention_weight_min": -27.17608070373535,
      "activations/layer13_attention_weight_max": 44.73641586303711,
      "activations/layer13_attention_weight_min": -32.14268493652344,
      "activations/layer14_attention_weight_max": 70.44293975830078,
      "activations/layer14_attention_weight_min": -47.97971725463867,
      "activations/layer15_attention_weight_max": 39.03217697143555,
      "activations/layer15_attention_weight_min": -28.90009307861328,
      "activations/layer16_attention_weight_max": 34.97093200683594,
      "activations/layer16_attention_weight_min": -30.137964248657227,
      "activations/layer17_attention_weight_max": 37.883399963378906,
      "activations/layer17_attention_weight_min": -24.908554077148438,
      "activations/layer18_attention_weight_max": 32.77458190917969,
      "activations/layer18_attention_weight_min": -20.31939697265625,
      "activations/layer19_attention_weight_max": 42.15606689453125,
      "activations/layer19_attention_weight_min": -30.85914421081543,
      "activations/layer1_attention_weight_max": 17.30088233947754,
      "activations/layer1_attention_weight_min": -14.985893249511719,
      "activations/layer20_attention_weight_max": 34.24217224121094,
      "activations/layer20_attention_weight_min": -25.522348403930664,
      "activations/layer21_attention_weight_max": 33.20184326171875,
      "activations/layer21_attention_weight_min": -24.732486724853516,
      "activations/layer22_attention_weight_max": 54.120330810546875,
      "activations/layer22_attention_weight_min": -29.90274429321289,
      "activations/layer23_attention_weight_max": 37.874610900878906,
      "activations/layer23_attention_weight_min": -23.996509552001953,
      "activations/layer2_attention_weight_max": 31.55670166015625,
      "activations/layer2_attention_weight_min": -30.735332489013672,
      "activations/layer3_attention_weight_max": 93.9243392944336,
      "activations/layer3_attention_weight_min": -93.1792984008789,
      "activations/layer4_attention_weight_max": 129.1334686279297,
      "activations/layer4_attention_weight_min": -124.42667388916016,
      "activations/layer5_attention_weight_max": 72.16523742675781,
      "activations/layer5_attention_weight_min": -66.05751037597656,
      "activations/layer6_attention_weight_max": 52.94630813598633,
      "activations/layer6_attention_weight_min": -55.90863800048828,
      "activations/layer7_attention_weight_max": 87.53993225097656,
      "activations/layer7_attention_weight_min": -87.49002838134766,
      "activations/layer8_attention_weight_max": 40.661865234375,
      "activations/layer8_attention_weight_min": -40.88778305053711,
      "activations/layer9_attention_weight_max": 37.601985931396484,
      "activations/layer9_attention_weight_min": -35.99104690551758,
      "epoch": 6.4,
      "learning_rate": 0.00010981136363636362,
      "loss": 2.8351,
      "step": 110200
    },
    {
      "activations/layer0_attention_weight_max": 16.55382537841797,
      "activations/layer0_attention_weight_min": -12.244940757751465,
      "activations/layer10_attention_weight_max": 38.676151275634766,
      "activations/layer10_attention_weight_min": -32.367652893066406,
      "activations/layer11_attention_weight_max": 37.44960021972656,
      "activations/layer11_attention_weight_min": -31.528470993041992,
      "activations/layer12_attention_weight_max": 28.31010627746582,
      "activations/layer12_attention_weight_min": -29.03400993347168,
      "activations/layer13_attention_weight_max": 50.80107879638672,
      "activations/layer13_attention_weight_min": -31.41262435913086,
      "activations/layer14_attention_weight_max": 62.29791259765625,
      "activations/layer14_attention_weight_min": -41.17755889892578,
      "activations/layer15_attention_weight_max": 43.775230407714844,
      "activations/layer15_attention_weight_min": -30.147695541381836,
      "activations/layer16_attention_weight_max": 40.67171859741211,
      "activations/layer16_attention_weight_min": -27.21746063232422,
      "activations/layer17_attention_weight_max": 38.7704963684082,
      "activations/layer17_attention_weight_min": -26.642444610595703,
      "activations/layer18_attention_weight_max": 31.7728271484375,
      "activations/layer18_attention_weight_min": -20.90265464782715,
      "activations/layer19_attention_weight_max": 47.01445388793945,
      "activations/layer19_attention_weight_min": -28.896408081054688,
      "activations/layer1_attention_weight_max": 18.220441818237305,
      "activations/layer1_attention_weight_min": -15.47005844116211,
      "activations/layer20_attention_weight_max": 38.34522247314453,
      "activations/layer20_attention_weight_min": -23.54649543762207,
      "activations/layer21_attention_weight_max": 41.039344787597656,
      "activations/layer21_attention_weight_min": -19.949092864990234,
      "activations/layer22_attention_weight_max": 51.570674896240234,
      "activations/layer22_attention_weight_min": -29.309988021850586,
      "activations/layer23_attention_weight_max": 37.52330017089844,
      "activations/layer23_attention_weight_min": -24.012420654296875,
      "activations/layer2_attention_weight_max": 32.56315612792969,
      "activations/layer2_attention_weight_min": -29.18722152709961,
      "activations/layer3_attention_weight_max": 91.01199340820312,
      "activations/layer3_attention_weight_min": -90.37545013427734,
      "activations/layer4_attention_weight_max": 133.1019287109375,
      "activations/layer4_attention_weight_min": -134.263427734375,
      "activations/layer5_attention_weight_max": 72.36923217773438,
      "activations/layer5_attention_weight_min": -71.42265319824219,
      "activations/layer6_attention_weight_max": 54.74089813232422,
      "activations/layer6_attention_weight_min": -55.813995361328125,
      "activations/layer7_attention_weight_max": 92.02446746826172,
      "activations/layer7_attention_weight_min": -88.41997528076172,
      "activations/layer8_attention_weight_max": 41.57258605957031,
      "activations/layer8_attention_weight_min": -41.560577392578125,
      "activations/layer9_attention_weight_max": 37.76194763183594,
      "activations/layer9_attention_weight_min": -33.73111343383789,
      "epoch": 6.41,
      "learning_rate": 0.00010979242424242423,
      "loss": 2.815,
      "step": 110250
    },
    {
      "activations/layer0_attention_weight_max": 16.02416229248047,
      "activations/layer0_attention_weight_min": -12.83318042755127,
      "activations/layer10_attention_weight_max": 33.81513214111328,
      "activations/layer10_attention_weight_min": -31.543560028076172,
      "activations/layer11_attention_weight_max": 32.26604461669922,
      "activations/layer11_attention_weight_min": -28.188749313354492,
      "activations/layer12_attention_weight_max": 20.936559677124023,
      "activations/layer12_attention_weight_min": -24.717695236206055,
      "activations/layer13_attention_weight_max": 38.808650970458984,
      "activations/layer13_attention_weight_min": -32.850711822509766,
      "activations/layer14_attention_weight_max": 56.330108642578125,
      "activations/layer14_attention_weight_min": -40.585201263427734,
      "activations/layer15_attention_weight_max": 34.32762908935547,
      "activations/layer15_attention_weight_min": -28.79364776611328,
      "activations/layer16_attention_weight_max": 36.414913177490234,
      "activations/layer16_attention_weight_min": -27.10779571533203,
      "activations/layer17_attention_weight_max": 36.49785614013672,
      "activations/layer17_attention_weight_min": -24.025182723999023,
      "activations/layer18_attention_weight_max": 33.229488372802734,
      "activations/layer18_attention_weight_min": -19.545236587524414,
      "activations/layer19_attention_weight_max": 43.75386047363281,
      "activations/layer19_attention_weight_min": -30.909982681274414,
      "activations/layer1_attention_weight_max": 18.990690231323242,
      "activations/layer1_attention_weight_min": -20.0455322265625,
      "activations/layer20_attention_weight_max": 30.833389282226562,
      "activations/layer20_attention_weight_min": -24.07573127746582,
      "activations/layer21_attention_weight_max": 31.280075073242188,
      "activations/layer21_attention_weight_min": -20.523771286010742,
      "activations/layer22_attention_weight_max": 49.47078323364258,
      "activations/layer22_attention_weight_min": -30.14215660095215,
      "activations/layer23_attention_weight_max": 38.05722427368164,
      "activations/layer23_attention_weight_min": -21.09156036376953,
      "activations/layer2_attention_weight_max": 33.225486755371094,
      "activations/layer2_attention_weight_min": -30.86414909362793,
      "activations/layer3_attention_weight_max": 99.34547424316406,
      "activations/layer3_attention_weight_min": -98.58686828613281,
      "activations/layer4_attention_weight_max": 131.56134033203125,
      "activations/layer4_attention_weight_min": -131.6698455810547,
      "activations/layer5_attention_weight_max": 71.54165649414062,
      "activations/layer5_attention_weight_min": -69.66926574707031,
      "activations/layer6_attention_weight_max": 55.6827507019043,
      "activations/layer6_attention_weight_min": -56.761375427246094,
      "activations/layer7_attention_weight_max": 89.46199035644531,
      "activations/layer7_attention_weight_min": -92.3838882446289,
      "activations/layer8_attention_weight_max": 40.18750762939453,
      "activations/layer8_attention_weight_min": -41.25819778442383,
      "activations/layer9_attention_weight_max": 38.13256072998047,
      "activations/layer9_attention_weight_min": -34.621429443359375,
      "epoch": 6.41,
      "learning_rate": 0.00010977348484848484,
      "loss": 2.8375,
      "step": 110300
    },
    {
      "activations/layer0_attention_weight_max": 15.742301940917969,
      "activations/layer0_attention_weight_min": -12.38235092163086,
      "activations/layer10_attention_weight_max": 36.727176666259766,
      "activations/layer10_attention_weight_min": -32.28325271606445,
      "activations/layer11_attention_weight_max": 39.53408432006836,
      "activations/layer11_attention_weight_min": -33.59809875488281,
      "activations/layer12_attention_weight_max": 27.570066452026367,
      "activations/layer12_attention_weight_min": -27.507272720336914,
      "activations/layer13_attention_weight_max": 45.62477111816406,
      "activations/layer13_attention_weight_min": -31.76202392578125,
      "activations/layer14_attention_weight_max": 70.27762603759766,
      "activations/layer14_attention_weight_min": -45.86092758178711,
      "activations/layer15_attention_weight_max": 45.33332061767578,
      "activations/layer15_attention_weight_min": -28.701051712036133,
      "activations/layer16_attention_weight_max": 43.438262939453125,
      "activations/layer16_attention_weight_min": -28.440393447875977,
      "activations/layer17_attention_weight_max": 40.05043029785156,
      "activations/layer17_attention_weight_min": -23.43059730529785,
      "activations/layer18_attention_weight_max": 31.685894012451172,
      "activations/layer18_attention_weight_min": -21.615934371948242,
      "activations/layer19_attention_weight_max": 46.99180221557617,
      "activations/layer19_attention_weight_min": -28.224960327148438,
      "activations/layer1_attention_weight_max": 17.11446762084961,
      "activations/layer1_attention_weight_min": -15.973758697509766,
      "activations/layer20_attention_weight_max": 36.2987060546875,
      "activations/layer20_attention_weight_min": -20.908411026000977,
      "activations/layer21_attention_weight_max": 33.299354553222656,
      "activations/layer21_attention_weight_min": -20.717918395996094,
      "activations/layer22_attention_weight_max": 53.97938537597656,
      "activations/layer22_attention_weight_min": -27.542478561401367,
      "activations/layer23_attention_weight_max": 37.846534729003906,
      "activations/layer23_attention_weight_min": -20.654071807861328,
      "activations/layer2_attention_weight_max": 36.40760040283203,
      "activations/layer2_attention_weight_min": -32.12700271606445,
      "activations/layer3_attention_weight_max": 90.56106567382812,
      "activations/layer3_attention_weight_min": -95.44989776611328,
      "activations/layer4_attention_weight_max": 128.4740753173828,
      "activations/layer4_attention_weight_min": -131.26344299316406,
      "activations/layer5_attention_weight_max": 72.22655487060547,
      "activations/layer5_attention_weight_min": -70.35929107666016,
      "activations/layer6_attention_weight_max": 54.94321060180664,
      "activations/layer6_attention_weight_min": -58.682029724121094,
      "activations/layer7_attention_weight_max": 91.30106353759766,
      "activations/layer7_attention_weight_min": -96.75080871582031,
      "activations/layer8_attention_weight_max": 43.656742095947266,
      "activations/layer8_attention_weight_min": -42.21639633178711,
      "activations/layer9_attention_weight_max": 41.28782272338867,
      "activations/layer9_attention_weight_min": -33.9354362487793,
      "epoch": 6.41,
      "learning_rate": 0.00010975454545454546,
      "loss": 2.8195,
      "step": 110350
    },
    {
      "activations/layer0_attention_weight_max": 16.308820724487305,
      "activations/layer0_attention_weight_min": -12.501102447509766,
      "activations/layer10_attention_weight_max": 31.671171188354492,
      "activations/layer10_attention_weight_min": -30.889892578125,
      "activations/layer11_attention_weight_max": 31.595182418823242,
      "activations/layer11_attention_weight_min": -29.1566162109375,
      "activations/layer12_attention_weight_max": 26.377153396606445,
      "activations/layer12_attention_weight_min": -24.160587310791016,
      "activations/layer13_attention_weight_max": 38.04248046875,
      "activations/layer13_attention_weight_min": -33.01399612426758,
      "activations/layer14_attention_weight_max": 58.133567810058594,
      "activations/layer14_attention_weight_min": -46.18447494506836,
      "activations/layer15_attention_weight_max": 37.48551559448242,
      "activations/layer15_attention_weight_min": -27.975906372070312,
      "activations/layer16_attention_weight_max": 33.90616226196289,
      "activations/layer16_attention_weight_min": -28.675785064697266,
      "activations/layer17_attention_weight_max": 32.00249481201172,
      "activations/layer17_attention_weight_min": -24.16179656982422,
      "activations/layer18_attention_weight_max": 30.062273025512695,
      "activations/layer18_attention_weight_min": -20.855409622192383,
      "activations/layer19_attention_weight_max": 40.65678024291992,
      "activations/layer19_attention_weight_min": -31.69632911682129,
      "activations/layer1_attention_weight_max": 17.592369079589844,
      "activations/layer1_attention_weight_min": -13.457172393798828,
      "activations/layer20_attention_weight_max": 33.97604751586914,
      "activations/layer20_attention_weight_min": -24.264820098876953,
      "activations/layer21_attention_weight_max": 29.78701400756836,
      "activations/layer21_attention_weight_min": -22.49324607849121,
      "activations/layer22_attention_weight_max": 49.70001220703125,
      "activations/layer22_attention_weight_min": -31.03163719177246,
      "activations/layer23_attention_weight_max": 37.53387451171875,
      "activations/layer23_attention_weight_min": -22.648990631103516,
      "activations/layer2_attention_weight_max": 31.5140380859375,
      "activations/layer2_attention_weight_min": -29.523786544799805,
      "activations/layer3_attention_weight_max": 84.78209686279297,
      "activations/layer3_attention_weight_min": -87.76189422607422,
      "activations/layer4_attention_weight_max": 117.7490234375,
      "activations/layer4_attention_weight_min": -117.9571762084961,
      "activations/layer5_attention_weight_max": 63.800750732421875,
      "activations/layer5_attention_weight_min": -62.62361145019531,
      "activations/layer6_attention_weight_max": 50.06150436401367,
      "activations/layer6_attention_weight_min": -50.16215896606445,
      "activations/layer7_attention_weight_max": 80.45303344726562,
      "activations/layer7_attention_weight_min": -78.20492553710938,
      "activations/layer8_attention_weight_max": 36.631004333496094,
      "activations/layer8_attention_weight_min": -39.87651443481445,
      "activations/layer9_attention_weight_max": 32.70610427856445,
      "activations/layer9_attention_weight_min": -32.478233337402344,
      "epoch": 6.41,
      "learning_rate": 0.00010973560606060604,
      "loss": 2.8278,
      "step": 110400
    },
    {
      "activations/layer0_attention_weight_max": 15.953584671020508,
      "activations/layer0_attention_weight_min": -11.54505443572998,
      "activations/layer10_attention_weight_max": 32.48237228393555,
      "activations/layer10_attention_weight_min": -30.35331153869629,
      "activations/layer11_attention_weight_max": 32.00067138671875,
      "activations/layer11_attention_weight_min": -29.891597747802734,
      "activations/layer12_attention_weight_max": 34.61478042602539,
      "activations/layer12_attention_weight_min": -26.527952194213867,
      "activations/layer13_attention_weight_max": 44.95016098022461,
      "activations/layer13_attention_weight_min": -38.150230407714844,
      "activations/layer14_attention_weight_max": 49.82704544067383,
      "activations/layer14_attention_weight_min": -42.879188537597656,
      "activations/layer15_attention_weight_max": 40.39291763305664,
      "activations/layer15_attention_weight_min": -30.43861961364746,
      "activations/layer16_attention_weight_max": 32.904666900634766,
      "activations/layer16_attention_weight_min": -28.65457534790039,
      "activations/layer17_attention_weight_max": 35.24658966064453,
      "activations/layer17_attention_weight_min": -24.267597198486328,
      "activations/layer18_attention_weight_max": 32.54412841796875,
      "activations/layer18_attention_weight_min": -22.22887420654297,
      "activations/layer19_attention_weight_max": 36.99081802368164,
      "activations/layer19_attention_weight_min": -28.962263107299805,
      "activations/layer1_attention_weight_max": 16.31151008605957,
      "activations/layer1_attention_weight_min": -13.33314323425293,
      "activations/layer20_attention_weight_max": 29.69033432006836,
      "activations/layer20_attention_weight_min": -23.6575984954834,
      "activations/layer21_attention_weight_max": 29.476531982421875,
      "activations/layer21_attention_weight_min": -21.890501022338867,
      "activations/layer22_attention_weight_max": 49.837711334228516,
      "activations/layer22_attention_weight_min": -27.732255935668945,
      "activations/layer23_attention_weight_max": 35.96194076538086,
      "activations/layer23_attention_weight_min": -21.784744262695312,
      "activations/layer2_attention_weight_max": 30.620071411132812,
      "activations/layer2_attention_weight_min": -29.762901306152344,
      "activations/layer3_attention_weight_max": 89.69361114501953,
      "activations/layer3_attention_weight_min": -92.74105072021484,
      "activations/layer4_attention_weight_max": 125.42675018310547,
      "activations/layer4_attention_weight_min": -124.79186248779297,
      "activations/layer5_attention_weight_max": 69.20719909667969,
      "activations/layer5_attention_weight_min": -68.89836883544922,
      "activations/layer6_attention_weight_max": 51.424190521240234,
      "activations/layer6_attention_weight_min": -53.82909393310547,
      "activations/layer7_attention_weight_max": 90.10292053222656,
      "activations/layer7_attention_weight_min": -88.49961853027344,
      "activations/layer8_attention_weight_max": 39.58056640625,
      "activations/layer8_attention_weight_min": -40.47695541381836,
      "activations/layer9_attention_weight_max": 35.35676193237305,
      "activations/layer9_attention_weight_min": -33.64359664916992,
      "epoch": 6.42,
      "learning_rate": 0.00010971666666666666,
      "loss": 2.8348,
      "step": 110450
    },
    {
      "activations/layer0_attention_weight_max": 15.623187065124512,
      "activations/layer0_attention_weight_min": -12.649761199951172,
      "activations/layer10_attention_weight_max": 33.41096496582031,
      "activations/layer10_attention_weight_min": -31.03290367126465,
      "activations/layer11_attention_weight_max": 31.376136779785156,
      "activations/layer11_attention_weight_min": -27.685943603515625,
      "activations/layer12_attention_weight_max": 35.84133529663086,
      "activations/layer12_attention_weight_min": -26.964969635009766,
      "activations/layer13_attention_weight_max": 47.3542366027832,
      "activations/layer13_attention_weight_min": -34.787696838378906,
      "activations/layer14_attention_weight_max": 59.182533264160156,
      "activations/layer14_attention_weight_min": -44.99264144897461,
      "activations/layer15_attention_weight_max": 39.23265838623047,
      "activations/layer15_attention_weight_min": -31.36538314819336,
      "activations/layer16_attention_weight_max": 37.0109748840332,
      "activations/layer16_attention_weight_min": -27.48268699645996,
      "activations/layer17_attention_weight_max": 38.461299896240234,
      "activations/layer17_attention_weight_min": -25.33265495300293,
      "activations/layer18_attention_weight_max": 35.29692840576172,
      "activations/layer18_attention_weight_min": -21.645524978637695,
      "activations/layer19_attention_weight_max": 41.565155029296875,
      "activations/layer19_attention_weight_min": -31.002685546875,
      "activations/layer1_attention_weight_max": 16.287267684936523,
      "activations/layer1_attention_weight_min": -13.501182556152344,
      "activations/layer20_attention_weight_max": 32.80298614501953,
      "activations/layer20_attention_weight_min": -24.05266571044922,
      "activations/layer21_attention_weight_max": 35.55617904663086,
      "activations/layer21_attention_weight_min": -21.47456169128418,
      "activations/layer22_attention_weight_max": 56.70085906982422,
      "activations/layer22_attention_weight_min": -28.82520294189453,
      "activations/layer23_attention_weight_max": 39.810935974121094,
      "activations/layer23_attention_weight_min": -22.29725456237793,
      "activations/layer2_attention_weight_max": 32.816490173339844,
      "activations/layer2_attention_weight_min": -31.914188385009766,
      "activations/layer3_attention_weight_max": 94.06937408447266,
      "activations/layer3_attention_weight_min": -96.37633514404297,
      "activations/layer4_attention_weight_max": 127.5116958618164,
      "activations/layer4_attention_weight_min": -132.40219116210938,
      "activations/layer5_attention_weight_max": 68.4847640991211,
      "activations/layer5_attention_weight_min": -67.53663635253906,
      "activations/layer6_attention_weight_max": 53.565486907958984,
      "activations/layer6_attention_weight_min": -56.33055114746094,
      "activations/layer7_attention_weight_max": 88.04315185546875,
      "activations/layer7_attention_weight_min": -92.88063049316406,
      "activations/layer8_attention_weight_max": 42.47065353393555,
      "activations/layer8_attention_weight_min": -39.484676361083984,
      "activations/layer9_attention_weight_max": 40.190399169921875,
      "activations/layer9_attention_weight_min": -35.425052642822266,
      "epoch": 6.42,
      "learning_rate": 0.00010969772727272727,
      "loss": 2.8286,
      "step": 110500
    },
    {
      "activations/layer0_attention_weight_max": 16.6254825592041,
      "activations/layer0_attention_weight_min": -11.536700248718262,
      "activations/layer10_attention_weight_max": 32.200408935546875,
      "activations/layer10_attention_weight_min": -29.821720123291016,
      "activations/layer11_attention_weight_max": 33.71460723876953,
      "activations/layer11_attention_weight_min": -26.941734313964844,
      "activations/layer12_attention_weight_max": 26.848005294799805,
      "activations/layer12_attention_weight_min": -25.6827335357666,
      "activations/layer13_attention_weight_max": 43.16494369506836,
      "activations/layer13_attention_weight_min": -35.771053314208984,
      "activations/layer14_attention_weight_max": 58.27077102661133,
      "activations/layer14_attention_weight_min": -42.05867385864258,
      "activations/layer15_attention_weight_max": 37.79096984863281,
      "activations/layer15_attention_weight_min": -29.228586196899414,
      "activations/layer16_attention_weight_max": 34.711341857910156,
      "activations/layer16_attention_weight_min": -26.60785484313965,
      "activations/layer17_attention_weight_max": 37.42927932739258,
      "activations/layer17_attention_weight_min": -24.5762882232666,
      "activations/layer18_attention_weight_max": 37.858741760253906,
      "activations/layer18_attention_weight_min": -19.842994689941406,
      "activations/layer19_attention_weight_max": 42.15604019165039,
      "activations/layer19_attention_weight_min": -29.462461471557617,
      "activations/layer1_attention_weight_max": 18.62409782409668,
      "activations/layer1_attention_weight_min": -14.749357223510742,
      "activations/layer20_attention_weight_max": 32.74030303955078,
      "activations/layer20_attention_weight_min": -24.01740837097168,
      "activations/layer21_attention_weight_max": 33.95945358276367,
      "activations/layer21_attention_weight_min": -21.7395076751709,
      "activations/layer22_attention_weight_max": 54.82377243041992,
      "activations/layer22_attention_weight_min": -28.420143127441406,
      "activations/layer23_attention_weight_max": 35.92688751220703,
      "activations/layer23_attention_weight_min": -21.680633544921875,
      "activations/layer2_attention_weight_max": 32.47996520996094,
      "activations/layer2_attention_weight_min": -31.061857223510742,
      "activations/layer3_attention_weight_max": 86.8111572265625,
      "activations/layer3_attention_weight_min": -89.0016860961914,
      "activations/layer4_attention_weight_max": 121.14617919921875,
      "activations/layer4_attention_weight_min": -115.99820709228516,
      "activations/layer5_attention_weight_max": 62.78425216674805,
      "activations/layer5_attention_weight_min": -61.891693115234375,
      "activations/layer6_attention_weight_max": 49.38047409057617,
      "activations/layer6_attention_weight_min": -50.90184783935547,
      "activations/layer7_attention_weight_max": 83.80193328857422,
      "activations/layer7_attention_weight_min": -85.60499572753906,
      "activations/layer8_attention_weight_max": 37.739967346191406,
      "activations/layer8_attention_weight_min": -38.706459045410156,
      "activations/layer9_attention_weight_max": 35.40126037597656,
      "activations/layer9_attention_weight_min": -31.136404037475586,
      "epoch": 6.42,
      "learning_rate": 0.00010967878787878786,
      "loss": 2.835,
      "step": 110550
    },
    {
      "activations/layer0_attention_weight_max": 16.153156280517578,
      "activations/layer0_attention_weight_min": -13.129064559936523,
      "activations/layer10_attention_weight_max": 34.359962463378906,
      "activations/layer10_attention_weight_min": -30.831010818481445,
      "activations/layer11_attention_weight_max": 32.784217834472656,
      "activations/layer11_attention_weight_min": -29.539167404174805,
      "activations/layer12_attention_weight_max": 33.0220947265625,
      "activations/layer12_attention_weight_min": -27.489946365356445,
      "activations/layer13_attention_weight_max": 46.04916000366211,
      "activations/layer13_attention_weight_min": -33.23771286010742,
      "activations/layer14_attention_weight_max": 57.689788818359375,
      "activations/layer14_attention_weight_min": -40.1471061706543,
      "activations/layer15_attention_weight_max": 46.70850372314453,
      "activations/layer15_attention_weight_min": -30.875118255615234,
      "activations/layer16_attention_weight_max": 41.86283874511719,
      "activations/layer16_attention_weight_min": -29.018367767333984,
      "activations/layer17_attention_weight_max": 38.637393951416016,
      "activations/layer17_attention_weight_min": -24.944807052612305,
      "activations/layer18_attention_weight_max": 37.30678939819336,
      "activations/layer18_attention_weight_min": -21.683752059936523,
      "activations/layer19_attention_weight_max": 42.9814567565918,
      "activations/layer19_attention_weight_min": -30.267438888549805,
      "activations/layer1_attention_weight_max": 17.366973876953125,
      "activations/layer1_attention_weight_min": -14.362215995788574,
      "activations/layer20_attention_weight_max": 34.42210006713867,
      "activations/layer20_attention_weight_min": -24.14904022216797,
      "activations/layer21_attention_weight_max": 33.826316833496094,
      "activations/layer21_attention_weight_min": -21.907102584838867,
      "activations/layer22_attention_weight_max": 54.45513916015625,
      "activations/layer22_attention_weight_min": -29.36551856994629,
      "activations/layer23_attention_weight_max": 35.601314544677734,
      "activations/layer23_attention_weight_min": -23.65747833251953,
      "activations/layer2_attention_weight_max": 31.700435638427734,
      "activations/layer2_attention_weight_min": -31.62384796142578,
      "activations/layer3_attention_weight_max": 87.41867065429688,
      "activations/layer3_attention_weight_min": -90.0676040649414,
      "activations/layer4_attention_weight_max": 124.24954986572266,
      "activations/layer4_attention_weight_min": -120.28932189941406,
      "activations/layer5_attention_weight_max": 66.6412353515625,
      "activations/layer5_attention_weight_min": -63.230979919433594,
      "activations/layer6_attention_weight_max": 52.240657806396484,
      "activations/layer6_attention_weight_min": -52.89778137207031,
      "activations/layer7_attention_weight_max": 87.55221557617188,
      "activations/layer7_attention_weight_min": -87.8423080444336,
      "activations/layer8_attention_weight_max": 39.64211654663086,
      "activations/layer8_attention_weight_min": -40.59674072265625,
      "activations/layer9_attention_weight_max": 37.69136047363281,
      "activations/layer9_attention_weight_min": -34.338863372802734,
      "epoch": 6.43,
      "learning_rate": 0.00010965984848484848,
      "loss": 2.8228,
      "step": 110600
    },
    {
      "activations/layer0_attention_weight_max": 15.57396411895752,
      "activations/layer0_attention_weight_min": -13.34288501739502,
      "activations/layer10_attention_weight_max": 36.86096954345703,
      "activations/layer10_attention_weight_min": -31.065927505493164,
      "activations/layer11_attention_weight_max": 35.34741973876953,
      "activations/layer11_attention_weight_min": -29.10253143310547,
      "activations/layer12_attention_weight_max": 24.900997161865234,
      "activations/layer12_attention_weight_min": -30.099672317504883,
      "activations/layer13_attention_weight_max": 44.450157165527344,
      "activations/layer13_attention_weight_min": -32.140193939208984,
      "activations/layer14_attention_weight_max": 64.44522857666016,
      "activations/layer14_attention_weight_min": -44.11367416381836,
      "activations/layer15_attention_weight_max": 40.926902770996094,
      "activations/layer15_attention_weight_min": -28.231508255004883,
      "activations/layer16_attention_weight_max": 41.99674987792969,
      "activations/layer16_attention_weight_min": -26.65979766845703,
      "activations/layer17_attention_weight_max": 52.244972229003906,
      "activations/layer17_attention_weight_min": -24.795312881469727,
      "activations/layer18_attention_weight_max": 36.345703125,
      "activations/layer18_attention_weight_min": -22.82620620727539,
      "activations/layer19_attention_weight_max": 50.353187561035156,
      "activations/layer19_attention_weight_min": -30.05427360534668,
      "activations/layer1_attention_weight_max": 16.868194580078125,
      "activations/layer1_attention_weight_min": -14.051305770874023,
      "activations/layer20_attention_weight_max": 41.19414520263672,
      "activations/layer20_attention_weight_min": -25.194629669189453,
      "activations/layer21_attention_weight_max": 41.5457878112793,
      "activations/layer21_attention_weight_min": -22.49721908569336,
      "activations/layer22_attention_weight_max": 63.77890396118164,
      "activations/layer22_attention_weight_min": -28.4349365234375,
      "activations/layer23_attention_weight_max": 45.6081657409668,
      "activations/layer23_attention_weight_min": -24.43227195739746,
      "activations/layer2_attention_weight_max": 33.37091064453125,
      "activations/layer2_attention_weight_min": -31.913131713867188,
      "activations/layer3_attention_weight_max": 92.68920135498047,
      "activations/layer3_attention_weight_min": -95.67132568359375,
      "activations/layer4_attention_weight_max": 131.07310485839844,
      "activations/layer4_attention_weight_min": -127.75724029541016,
      "activations/layer5_attention_weight_max": 71.12723541259766,
      "activations/layer5_attention_weight_min": -66.23966979980469,
      "activations/layer6_attention_weight_max": 54.69578170776367,
      "activations/layer6_attention_weight_min": -54.99872589111328,
      "activations/layer7_attention_weight_max": 88.85639953613281,
      "activations/layer7_attention_weight_min": -89.39449310302734,
      "activations/layer8_attention_weight_max": 43.19850540161133,
      "activations/layer8_attention_weight_min": -40.72329330444336,
      "activations/layer9_attention_weight_max": 38.7248420715332,
      "activations/layer9_attention_weight_min": -35.130767822265625,
      "epoch": 6.43,
      "learning_rate": 0.00010964090909090908,
      "loss": 2.826,
      "step": 110650
    },
    {
      "activations/layer0_attention_weight_max": 15.872024536132812,
      "activations/layer0_attention_weight_min": -11.382518768310547,
      "activations/layer10_attention_weight_max": 36.33936309814453,
      "activations/layer10_attention_weight_min": -34.63165283203125,
      "activations/layer11_attention_weight_max": 35.87485885620117,
      "activations/layer11_attention_weight_min": -30.632991790771484,
      "activations/layer12_attention_weight_max": 28.9212703704834,
      "activations/layer12_attention_weight_min": -26.798498153686523,
      "activations/layer13_attention_weight_max": 50.18440246582031,
      "activations/layer13_attention_weight_min": -35.97149658203125,
      "activations/layer14_attention_weight_max": 66.1324462890625,
      "activations/layer14_attention_weight_min": -54.185691833496094,
      "activations/layer15_attention_weight_max": 48.02233123779297,
      "activations/layer15_attention_weight_min": -31.633081436157227,
      "activations/layer16_attention_weight_max": 35.518978118896484,
      "activations/layer16_attention_weight_min": -30.65616798400879,
      "activations/layer17_attention_weight_max": 38.02338790893555,
      "activations/layer17_attention_weight_min": -24.357675552368164,
      "activations/layer18_attention_weight_max": 30.198740005493164,
      "activations/layer18_attention_weight_min": -20.957971572875977,
      "activations/layer19_attention_weight_max": 44.4737548828125,
      "activations/layer19_attention_weight_min": -29.63429832458496,
      "activations/layer1_attention_weight_max": 17.1683292388916,
      "activations/layer1_attention_weight_min": -16.690460205078125,
      "activations/layer20_attention_weight_max": 32.01959228515625,
      "activations/layer20_attention_weight_min": -24.63988494873047,
      "activations/layer21_attention_weight_max": 30.351524353027344,
      "activations/layer21_attention_weight_min": -20.776769638061523,
      "activations/layer22_attention_weight_max": 55.34174346923828,
      "activations/layer22_attention_weight_min": -29.936508178710938,
      "activations/layer23_attention_weight_max": 40.60160446166992,
      "activations/layer23_attention_weight_min": -19.921558380126953,
      "activations/layer2_attention_weight_max": 32.58431625366211,
      "activations/layer2_attention_weight_min": -31.487537384033203,
      "activations/layer3_attention_weight_max": 94.28015899658203,
      "activations/layer3_attention_weight_min": -94.8944091796875,
      "activations/layer4_attention_weight_max": 129.63490295410156,
      "activations/layer4_attention_weight_min": -128.61422729492188,
      "activations/layer5_attention_weight_max": 72.07147979736328,
      "activations/layer5_attention_weight_min": -70.42324829101562,
      "activations/layer6_attention_weight_max": 52.72780227661133,
      "activations/layer6_attention_weight_min": -58.91538619995117,
      "activations/layer7_attention_weight_max": 88.8318099975586,
      "activations/layer7_attention_weight_min": -100.80390167236328,
      "activations/layer8_attention_weight_max": 42.19962692260742,
      "activations/layer8_attention_weight_min": -43.6983528137207,
      "activations/layer9_attention_weight_max": 38.943241119384766,
      "activations/layer9_attention_weight_min": -37.41827392578125,
      "epoch": 6.43,
      "learning_rate": 0.00010962196969696968,
      "loss": 2.8231,
      "step": 110700
    },
    {
      "activations/layer0_attention_weight_max": 15.312142372131348,
      "activations/layer0_attention_weight_min": -11.205599784851074,
      "activations/layer10_attention_weight_max": 35.13843536376953,
      "activations/layer10_attention_weight_min": -33.168888092041016,
      "activations/layer11_attention_weight_max": 39.532161712646484,
      "activations/layer11_attention_weight_min": -34.992828369140625,
      "activations/layer12_attention_weight_max": 49.093017578125,
      "activations/layer12_attention_weight_min": -27.438997268676758,
      "activations/layer13_attention_weight_max": 61.22718811035156,
      "activations/layer13_attention_weight_min": -44.78602600097656,
      "activations/layer14_attention_weight_max": 64.7278060913086,
      "activations/layer14_attention_weight_min": -50.52481460571289,
      "activations/layer15_attention_weight_max": 56.148983001708984,
      "activations/layer15_attention_weight_min": -37.065826416015625,
      "activations/layer16_attention_weight_max": 33.5602912902832,
      "activations/layer16_attention_weight_min": -28.026535034179688,
      "activations/layer17_attention_weight_max": 35.37553787231445,
      "activations/layer17_attention_weight_min": -25.31201934814453,
      "activations/layer18_attention_weight_max": 28.26280403137207,
      "activations/layer18_attention_weight_min": -21.37717628479004,
      "activations/layer19_attention_weight_max": 34.5897102355957,
      "activations/layer19_attention_weight_min": -29.583356857299805,
      "activations/layer1_attention_weight_max": 17.877361297607422,
      "activations/layer1_attention_weight_min": -13.76294994354248,
      "activations/layer20_attention_weight_max": 28.969987869262695,
      "activations/layer20_attention_weight_min": -25.209766387939453,
      "activations/layer21_attention_weight_max": 28.27741050720215,
      "activations/layer21_attention_weight_min": -21.45108413696289,
      "activations/layer22_attention_weight_max": 45.1547966003418,
      "activations/layer22_attention_weight_min": -31.414005279541016,
      "activations/layer23_attention_weight_max": 34.3945426940918,
      "activations/layer23_attention_weight_min": -21.85048484802246,
      "activations/layer2_attention_weight_max": 32.48651123046875,
      "activations/layer2_attention_weight_min": -31.177642822265625,
      "activations/layer3_attention_weight_max": 94.80180358886719,
      "activations/layer3_attention_weight_min": -95.38217163085938,
      "activations/layer4_attention_weight_max": 127.00748443603516,
      "activations/layer4_attention_weight_min": -130.9227752685547,
      "activations/layer5_attention_weight_max": 68.87275695800781,
      "activations/layer5_attention_weight_min": -69.98583984375,
      "activations/layer6_attention_weight_max": 52.838157653808594,
      "activations/layer6_attention_weight_min": -56.66911315917969,
      "activations/layer7_attention_weight_max": 96.1406478881836,
      "activations/layer7_attention_weight_min": -87.48204803466797,
      "activations/layer8_attention_weight_max": 41.82339859008789,
      "activations/layer8_attention_weight_min": -40.73373794555664,
      "activations/layer9_attention_weight_max": 36.65913391113281,
      "activations/layer9_attention_weight_min": -35.82830810546875,
      "epoch": 6.44,
      "learning_rate": 0.0001096030303030303,
      "loss": 2.8289,
      "step": 110750
    },
    {
      "activations/layer0_attention_weight_max": 16.844669342041016,
      "activations/layer0_attention_weight_min": -12.414661407470703,
      "activations/layer10_attention_weight_max": 41.93056106567383,
      "activations/layer10_attention_weight_min": -35.40603256225586,
      "activations/layer11_attention_weight_max": 42.66676330566406,
      "activations/layer11_attention_weight_min": -35.7342529296875,
      "activations/layer12_attention_weight_max": 30.940593719482422,
      "activations/layer12_attention_weight_min": -27.27220916748047,
      "activations/layer13_attention_weight_max": 65.15116882324219,
      "activations/layer13_attention_weight_min": -40.82334899902344,
      "activations/layer14_attention_weight_max": 72.60142517089844,
      "activations/layer14_attention_weight_min": -47.74740219116211,
      "activations/layer15_attention_weight_max": 52.21022033691406,
      "activations/layer15_attention_weight_min": -34.11727523803711,
      "activations/layer16_attention_weight_max": 36.108978271484375,
      "activations/layer16_attention_weight_min": -29.0119686126709,
      "activations/layer17_attention_weight_max": 38.83793640136719,
      "activations/layer17_attention_weight_min": -25.48387336730957,
      "activations/layer18_attention_weight_max": 37.397369384765625,
      "activations/layer18_attention_weight_min": -22.601701736450195,
      "activations/layer19_attention_weight_max": 41.12708282470703,
      "activations/layer19_attention_weight_min": -30.02825927734375,
      "activations/layer1_attention_weight_max": 17.061450958251953,
      "activations/layer1_attention_weight_min": -13.774489402770996,
      "activations/layer20_attention_weight_max": 32.83985900878906,
      "activations/layer20_attention_weight_min": -23.247947692871094,
      "activations/layer21_attention_weight_max": 31.72618293762207,
      "activations/layer21_attention_weight_min": -20.03068733215332,
      "activations/layer22_attention_weight_max": 55.10006332397461,
      "activations/layer22_attention_weight_min": -27.00738525390625,
      "activations/layer23_attention_weight_max": 35.95402145385742,
      "activations/layer23_attention_weight_min": -21.614439010620117,
      "activations/layer2_attention_weight_max": 31.62013816833496,
      "activations/layer2_attention_weight_min": -31.140113830566406,
      "activations/layer3_attention_weight_max": 95.27764892578125,
      "activations/layer3_attention_weight_min": -98.14512634277344,
      "activations/layer4_attention_weight_max": 134.14027404785156,
      "activations/layer4_attention_weight_min": -133.55270385742188,
      "activations/layer5_attention_weight_max": 71.44157409667969,
      "activations/layer5_attention_weight_min": -69.70570373535156,
      "activations/layer6_attention_weight_max": 54.592857360839844,
      "activations/layer6_attention_weight_min": -56.336700439453125,
      "activations/layer7_attention_weight_max": 101.84383392333984,
      "activations/layer7_attention_weight_min": -93.29289245605469,
      "activations/layer8_attention_weight_max": 46.22734451293945,
      "activations/layer8_attention_weight_min": -43.858245849609375,
      "activations/layer9_attention_weight_max": 41.45181655883789,
      "activations/layer9_attention_weight_min": -37.5853385925293,
      "epoch": 6.44,
      "learning_rate": 0.0001095840909090909,
      "loss": 2.82,
      "step": 110800
    },
    {
      "activations/layer0_attention_weight_max": 15.906171798706055,
      "activations/layer0_attention_weight_min": -12.696203231811523,
      "activations/layer10_attention_weight_max": 32.922523498535156,
      "activations/layer10_attention_weight_min": -34.20619583129883,
      "activations/layer11_attention_weight_max": 31.887554168701172,
      "activations/layer11_attention_weight_min": -30.25227165222168,
      "activations/layer12_attention_weight_max": 25.14048957824707,
      "activations/layer12_attention_weight_min": -25.000646591186523,
      "activations/layer13_attention_weight_max": 39.70250701904297,
      "activations/layer13_attention_weight_min": -33.35451889038086,
      "activations/layer14_attention_weight_max": 57.20256805419922,
      "activations/layer14_attention_weight_min": -43.20742416381836,
      "activations/layer15_attention_weight_max": 35.92182922363281,
      "activations/layer15_attention_weight_min": -29.80546760559082,
      "activations/layer16_attention_weight_max": 39.53620147705078,
      "activations/layer16_attention_weight_min": -28.231523513793945,
      "activations/layer17_attention_weight_max": 38.39067077636719,
      "activations/layer17_attention_weight_min": -25.3404483795166,
      "activations/layer18_attention_weight_max": 31.131305694580078,
      "activations/layer18_attention_weight_min": -20.57076644897461,
      "activations/layer19_attention_weight_max": 40.121559143066406,
      "activations/layer19_attention_weight_min": -30.161773681640625,
      "activations/layer1_attention_weight_max": 17.2415714263916,
      "activations/layer1_attention_weight_min": -12.15381145477295,
      "activations/layer20_attention_weight_max": 32.017974853515625,
      "activations/layer20_attention_weight_min": -23.684589385986328,
      "activations/layer21_attention_weight_max": 29.248159408569336,
      "activations/layer21_attention_weight_min": -22.145259857177734,
      "activations/layer22_attention_weight_max": 47.08152770996094,
      "activations/layer22_attention_weight_min": -29.479103088378906,
      "activations/layer23_attention_weight_max": 34.535614013671875,
      "activations/layer23_attention_weight_min": -22.219863891601562,
      "activations/layer2_attention_weight_max": 29.728885650634766,
      "activations/layer2_attention_weight_min": -30.22492790222168,
      "activations/layer3_attention_weight_max": 86.31758117675781,
      "activations/layer3_attention_weight_min": -87.09793853759766,
      "activations/layer4_attention_weight_max": 119.67032623291016,
      "activations/layer4_attention_weight_min": -121.82097625732422,
      "activations/layer5_attention_weight_max": 64.73790740966797,
      "activations/layer5_attention_weight_min": -68.5649185180664,
      "activations/layer6_attention_weight_max": 54.881736755371094,
      "activations/layer6_attention_weight_min": -54.35308074951172,
      "activations/layer7_attention_weight_max": 86.66583251953125,
      "activations/layer7_attention_weight_min": -88.28563690185547,
      "activations/layer8_attention_weight_max": 40.59999465942383,
      "activations/layer8_attention_weight_min": -41.69114685058594,
      "activations/layer9_attention_weight_max": 35.462642669677734,
      "activations/layer9_attention_weight_min": -35.419090270996094,
      "epoch": 6.44,
      "learning_rate": 0.00010956515151515151,
      "loss": 2.8171,
      "step": 110850
    },
    {
      "activations/layer0_attention_weight_max": 15.232593536376953,
      "activations/layer0_attention_weight_min": -12.366745948791504,
      "activations/layer10_attention_weight_max": 33.28266143798828,
      "activations/layer10_attention_weight_min": -30.074880599975586,
      "activations/layer11_attention_weight_max": 30.965408325195312,
      "activations/layer11_attention_weight_min": -29.06133270263672,
      "activations/layer12_attention_weight_max": 24.102134704589844,
      "activations/layer12_attention_weight_min": -31.056055068969727,
      "activations/layer13_attention_weight_max": 42.812767028808594,
      "activations/layer13_attention_weight_min": -33.22718811035156,
      "activations/layer14_attention_weight_max": 50.0604248046875,
      "activations/layer14_attention_weight_min": -40.228492736816406,
      "activations/layer15_attention_weight_max": 36.250038146972656,
      "activations/layer15_attention_weight_min": -30.238237380981445,
      "activations/layer16_attention_weight_max": 35.937828063964844,
      "activations/layer16_attention_weight_min": -27.3167781829834,
      "activations/layer17_attention_weight_max": 36.54819107055664,
      "activations/layer17_attention_weight_min": -24.423524856567383,
      "activations/layer18_attention_weight_max": 34.29684829711914,
      "activations/layer18_attention_weight_min": -20.056243896484375,
      "activations/layer19_attention_weight_max": 42.56350326538086,
      "activations/layer19_attention_weight_min": -29.665536880493164,
      "activations/layer1_attention_weight_max": 17.68317413330078,
      "activations/layer1_attention_weight_min": -13.45238208770752,
      "activations/layer20_attention_weight_max": 31.98370933532715,
      "activations/layer20_attention_weight_min": -22.07220458984375,
      "activations/layer21_attention_weight_max": 35.18024444580078,
      "activations/layer21_attention_weight_min": -22.200048446655273,
      "activations/layer22_attention_weight_max": 48.9450798034668,
      "activations/layer22_attention_weight_min": -30.66884994506836,
      "activations/layer23_attention_weight_max": 39.01646423339844,
      "activations/layer23_attention_weight_min": -22.933250427246094,
      "activations/layer2_attention_weight_max": 30.74262809753418,
      "activations/layer2_attention_weight_min": -30.077510833740234,
      "activations/layer3_attention_weight_max": 87.14909362792969,
      "activations/layer3_attention_weight_min": -87.1844711303711,
      "activations/layer4_attention_weight_max": 123.42585754394531,
      "activations/layer4_attention_weight_min": -119.71331024169922,
      "activations/layer5_attention_weight_max": 68.98251342773438,
      "activations/layer5_attention_weight_min": -63.534324645996094,
      "activations/layer6_attention_weight_max": 53.70534133911133,
      "activations/layer6_attention_weight_min": -56.09636688232422,
      "activations/layer7_attention_weight_max": 85.50421142578125,
      "activations/layer7_attention_weight_min": -86.28026580810547,
      "activations/layer8_attention_weight_max": 42.278865814208984,
      "activations/layer8_attention_weight_min": -38.73390579223633,
      "activations/layer9_attention_weight_max": 39.33852767944336,
      "activations/layer9_attention_weight_min": -34.247806549072266,
      "epoch": 6.44,
      "learning_rate": 0.0001095465909090909,
      "loss": 2.8262,
      "step": 110900
    },
    {
      "activations/layer0_attention_weight_max": 15.70576286315918,
      "activations/layer0_attention_weight_min": -11.739420890808105,
      "activations/layer10_attention_weight_max": 34.102394104003906,
      "activations/layer10_attention_weight_min": -31.739994049072266,
      "activations/layer11_attention_weight_max": 31.848678588867188,
      "activations/layer11_attention_weight_min": -29.58462905883789,
      "activations/layer12_attention_weight_max": 21.894222259521484,
      "activations/layer12_attention_weight_min": -28.15488624572754,
      "activations/layer13_attention_weight_max": 42.84043884277344,
      "activations/layer13_attention_weight_min": -31.9327449798584,
      "activations/layer14_attention_weight_max": 46.37546920776367,
      "activations/layer14_attention_weight_min": -34.92095184326172,
      "activations/layer15_attention_weight_max": 35.17424774169922,
      "activations/layer15_attention_weight_min": -28.279600143432617,
      "activations/layer16_attention_weight_max": 34.67380142211914,
      "activations/layer16_attention_weight_min": -26.67890167236328,
      "activations/layer17_attention_weight_max": 34.964237213134766,
      "activations/layer17_attention_weight_min": -24.344322204589844,
      "activations/layer18_attention_weight_max": 31.546953201293945,
      "activations/layer18_attention_weight_min": -19.498310089111328,
      "activations/layer19_attention_weight_max": 36.43561935424805,
      "activations/layer19_attention_weight_min": -30.340042114257812,
      "activations/layer1_attention_weight_max": 19.124849319458008,
      "activations/layer1_attention_weight_min": -17.951566696166992,
      "activations/layer20_attention_weight_max": 30.128305435180664,
      "activations/layer20_attention_weight_min": -24.254924774169922,
      "activations/layer21_attention_weight_max": 28.79390525817871,
      "activations/layer21_attention_weight_min": -23.09766960144043,
      "activations/layer22_attention_weight_max": 48.28358459472656,
      "activations/layer22_attention_weight_min": -28.899873733520508,
      "activations/layer23_attention_weight_max": 38.92196273803711,
      "activations/layer23_attention_weight_min": -23.457244873046875,
      "activations/layer2_attention_weight_max": 30.259756088256836,
      "activations/layer2_attention_weight_min": -29.32844352722168,
      "activations/layer3_attention_weight_max": 83.13357543945312,
      "activations/layer3_attention_weight_min": -84.623046875,
      "activations/layer4_attention_weight_max": 121.7747802734375,
      "activations/layer4_attention_weight_min": -115.2836685180664,
      "activations/layer5_attention_weight_max": 65.81895446777344,
      "activations/layer5_attention_weight_min": -63.22212219238281,
      "activations/layer6_attention_weight_max": 50.173160552978516,
      "activations/layer6_attention_weight_min": -53.56562805175781,
      "activations/layer7_attention_weight_max": 85.64684295654297,
      "activations/layer7_attention_weight_min": -83.28227233886719,
      "activations/layer8_attention_weight_max": 38.25819778442383,
      "activations/layer8_attention_weight_min": -40.63145065307617,
      "activations/layer9_attention_weight_max": 33.84148025512695,
      "activations/layer9_attention_weight_min": -33.78168869018555,
      "epoch": 6.45,
      "learning_rate": 0.0001095276515151515,
      "loss": 2.8202,
      "step": 110950
    },
    {
      "activations/layer0_attention_weight_max": 15.835872650146484,
      "activations/layer0_attention_weight_min": -12.888949394226074,
      "activations/layer10_attention_weight_max": 33.885536193847656,
      "activations/layer10_attention_weight_min": -33.52911376953125,
      "activations/layer11_attention_weight_max": 31.93988037109375,
      "activations/layer11_attention_weight_min": -29.45070457458496,
      "activations/layer12_attention_weight_max": 35.32915115356445,
      "activations/layer12_attention_weight_min": -27.18470001220703,
      "activations/layer13_attention_weight_max": 44.17510223388672,
      "activations/layer13_attention_weight_min": -32.62482452392578,
      "activations/layer14_attention_weight_max": 58.616294860839844,
      "activations/layer14_attention_weight_min": -42.21430206298828,
      "activations/layer15_attention_weight_max": 38.85625076293945,
      "activations/layer15_attention_weight_min": -28.005571365356445,
      "activations/layer16_attention_weight_max": 35.1850471496582,
      "activations/layer16_attention_weight_min": -28.09117317199707,
      "activations/layer17_attention_weight_max": 37.09982681274414,
      "activations/layer17_attention_weight_min": -24.166471481323242,
      "activations/layer18_attention_weight_max": 32.51799774169922,
      "activations/layer18_attention_weight_min": -21.436710357666016,
      "activations/layer19_attention_weight_max": 39.6579704284668,
      "activations/layer19_attention_weight_min": -30.006698608398438,
      "activations/layer1_attention_weight_max": 17.83863067626953,
      "activations/layer1_attention_weight_min": -17.3727970123291,
      "activations/layer20_attention_weight_max": 33.25390625,
      "activations/layer20_attention_weight_min": -26.623933792114258,
      "activations/layer21_attention_weight_max": 31.996688842773438,
      "activations/layer21_attention_weight_min": -24.493423461914062,
      "activations/layer22_attention_weight_max": 51.60102081298828,
      "activations/layer22_attention_weight_min": -29.132402420043945,
      "activations/layer23_attention_weight_max": 32.96243667602539,
      "activations/layer23_attention_weight_min": -23.631160736083984,
      "activations/layer2_attention_weight_max": 31.366424560546875,
      "activations/layer2_attention_weight_min": -30.626514434814453,
      "activations/layer3_attention_weight_max": 88.97344207763672,
      "activations/layer3_attention_weight_min": -89.40988159179688,
      "activations/layer4_attention_weight_max": 124.14424896240234,
      "activations/layer4_attention_weight_min": -120.47175598144531,
      "activations/layer5_attention_weight_max": 66.37686920166016,
      "activations/layer5_attention_weight_min": -66.13336944580078,
      "activations/layer6_attention_weight_max": 53.35112762451172,
      "activations/layer6_attention_weight_min": -52.75383758544922,
      "activations/layer7_attention_weight_max": 87.48737335205078,
      "activations/layer7_attention_weight_min": -86.37586975097656,
      "activations/layer8_attention_weight_max": 39.673492431640625,
      "activations/layer8_attention_weight_min": -40.97402572631836,
      "activations/layer9_attention_weight_max": 35.6353874206543,
      "activations/layer9_attention_weight_min": -35.84352111816406,
      "epoch": 6.45,
      "learning_rate": 0.0001095087121212121,
      "loss": 2.8151,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5163,
      "eval_samples_per_second": 504.212,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5163,
      "eval_openwebtext_samples_per_second": 504.212,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_wikitext_loss": 2.99609375,
      "eval_wikitext_ppl": 20.007230835729846,
      "eval_wikitext_runtime": 1.9647,
      "eval_wikitext_samples_per_second": 232.102,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_lambada_loss": 2.62109375,
      "eval_lambada_ppl": 13.750755251693265,
      "eval_lambada_runtime": 9.5607,
      "eval_lambada_samples_per_second": 509.274,
      "step": 111000
    },
    {
      "activations/layer0_attention_weight_max": 16.25923728942871,
      "activations/layer0_attention_weight_min": -11.776546478271484,
      "activations/layer10_attention_weight_max": 36.97948455810547,
      "activations/layer10_attention_weight_min": -35.1975212097168,
      "activations/layer11_attention_weight_max": 34.5700569152832,
      "activations/layer11_attention_weight_min": -32.12934875488281,
      "activations/layer12_attention_weight_max": 25.03751564025879,
      "activations/layer12_attention_weight_min": -28.6505069732666,
      "activations/layer13_attention_weight_max": 45.81977462768555,
      "activations/layer13_attention_weight_min": -33.31378936767578,
      "activations/layer14_attention_weight_max": 67.07669067382812,
      "activations/layer14_attention_weight_min": -48.826656341552734,
      "activations/layer15_attention_weight_max": 43.00605010986328,
      "activations/layer15_attention_weight_min": -27.276391983032227,
      "activations/layer16_attention_weight_max": 38.53116989135742,
      "activations/layer16_attention_weight_min": -28.12886619567871,
      "activations/layer17_attention_weight_max": 41.147422790527344,
      "activations/layer17_attention_weight_min": -24.983945846557617,
      "activations/layer18_attention_weight_max": 34.89555740356445,
      "activations/layer18_attention_weight_min": -22.111669540405273,
      "activations/layer19_attention_weight_max": 40.59763717651367,
      "activations/layer19_attention_weight_min": -31.206327438354492,
      "activations/layer1_attention_weight_max": 18.760894775390625,
      "activations/layer1_attention_weight_min": -16.79423713684082,
      "activations/layer20_attention_weight_max": 32.696041107177734,
      "activations/layer20_attention_weight_min": -23.12069320678711,
      "activations/layer21_attention_weight_max": 35.24361801147461,
      "activations/layer21_attention_weight_min": -22.33180809020996,
      "activations/layer22_attention_weight_max": 54.890281677246094,
      "activations/layer22_attention_weight_min": -33.88449478149414,
      "activations/layer23_attention_weight_max": 42.21257019042969,
      "activations/layer23_attention_weight_min": -25.61267852783203,
      "activations/layer2_attention_weight_max": 31.57794952392578,
      "activations/layer2_attention_weight_min": -30.25381088256836,
      "activations/layer3_attention_weight_max": 86.97368621826172,
      "activations/layer3_attention_weight_min": -90.24726104736328,
      "activations/layer4_attention_weight_max": 124.23766326904297,
      "activations/layer4_attention_weight_min": -127.97618865966797,
      "activations/layer5_attention_weight_max": 68.84214782714844,
      "activations/layer5_attention_weight_min": -65.515869140625,
      "activations/layer6_attention_weight_max": 50.31645202636719,
      "activations/layer6_attention_weight_min": -53.7602424621582,
      "activations/layer7_attention_weight_max": 86.06532287597656,
      "activations/layer7_attention_weight_min": -87.46251678466797,
      "activations/layer8_attention_weight_max": 40.24180603027344,
      "activations/layer8_attention_weight_min": -42.39442443847656,
      "activations/layer9_attention_weight_max": 41.5305290222168,
      "activations/layer9_attention_weight_min": -36.0959587097168,
      "epoch": 6.45,
      "learning_rate": 0.00010948977272727272,
      "loss": 2.813,
      "step": 111050
    },
    {
      "activations/layer0_attention_weight_max": 15.98669147491455,
      "activations/layer0_attention_weight_min": -11.820359230041504,
      "activations/layer10_attention_weight_max": 35.2067756652832,
      "activations/layer10_attention_weight_min": -31.65777587890625,
      "activations/layer11_attention_weight_max": 36.38206481933594,
      "activations/layer11_attention_weight_min": -30.053199768066406,
      "activations/layer12_attention_weight_max": 40.1221923828125,
      "activations/layer12_attention_weight_min": -24.604124069213867,
      "activations/layer13_attention_weight_max": 45.94963073730469,
      "activations/layer13_attention_weight_min": -33.924713134765625,
      "activations/layer14_attention_weight_max": 64.54442596435547,
      "activations/layer14_attention_weight_min": -45.84834289550781,
      "activations/layer15_attention_weight_max": 39.1092529296875,
      "activations/layer15_attention_weight_min": -28.891422271728516,
      "activations/layer16_attention_weight_max": 37.75981521606445,
      "activations/layer16_attention_weight_min": -27.965007781982422,
      "activations/layer17_attention_weight_max": 39.14144515991211,
      "activations/layer17_attention_weight_min": -26.251522064208984,
      "activations/layer18_attention_weight_max": 34.74598693847656,
      "activations/layer18_attention_weight_min": -20.767894744873047,
      "activations/layer19_attention_weight_max": 40.871604919433594,
      "activations/layer19_attention_weight_min": -29.204437255859375,
      "activations/layer1_attention_weight_max": 17.052350997924805,
      "activations/layer1_attention_weight_min": -16.26146125793457,
      "activations/layer20_attention_weight_max": 37.11757278442383,
      "activations/layer20_attention_weight_min": -23.749988555908203,
      "activations/layer21_attention_weight_max": 35.99802780151367,
      "activations/layer21_attention_weight_min": -21.53229522705078,
      "activations/layer22_attention_weight_max": 52.381534576416016,
      "activations/layer22_attention_weight_min": -27.874738693237305,
      "activations/layer23_attention_weight_max": 38.00263977050781,
      "activations/layer23_attention_weight_min": -21.18653106689453,
      "activations/layer2_attention_weight_max": 32.53175735473633,
      "activations/layer2_attention_weight_min": -31.252399444580078,
      "activations/layer3_attention_weight_max": 90.01237487792969,
      "activations/layer3_attention_weight_min": -89.6356430053711,
      "activations/layer4_attention_weight_max": 125.1229476928711,
      "activations/layer4_attention_weight_min": -125.85770416259766,
      "activations/layer5_attention_weight_max": 66.25342559814453,
      "activations/layer5_attention_weight_min": -69.08358764648438,
      "activations/layer6_attention_weight_max": 51.678871154785156,
      "activations/layer6_attention_weight_min": -55.43250274658203,
      "activations/layer7_attention_weight_max": 89.88127899169922,
      "activations/layer7_attention_weight_min": -87.05315399169922,
      "activations/layer8_attention_weight_max": 40.556793212890625,
      "activations/layer8_attention_weight_min": -40.783668518066406,
      "activations/layer9_attention_weight_max": 37.26430892944336,
      "activations/layer9_attention_weight_min": -35.38078308105469,
      "epoch": 6.46,
      "learning_rate": 0.00010947083333333334,
      "loss": 2.8312,
      "step": 111100
    },
    {
      "activations/layer0_attention_weight_max": 15.978211402893066,
      "activations/layer0_attention_weight_min": -12.33161735534668,
      "activations/layer10_attention_weight_max": 31.91143798828125,
      "activations/layer10_attention_weight_min": -29.419979095458984,
      "activations/layer11_attention_weight_max": 31.317424774169922,
      "activations/layer11_attention_weight_min": -26.66421127319336,
      "activations/layer12_attention_weight_max": 25.4121036529541,
      "activations/layer12_attention_weight_min": -26.770301818847656,
      "activations/layer13_attention_weight_max": 42.546138763427734,
      "activations/layer13_attention_weight_min": -31.912343978881836,
      "activations/layer14_attention_weight_max": 61.48737716674805,
      "activations/layer14_attention_weight_min": -45.009239196777344,
      "activations/layer15_attention_weight_max": 37.88388442993164,
      "activations/layer15_attention_weight_min": -28.324827194213867,
      "activations/layer16_attention_weight_max": 38.42466354370117,
      "activations/layer16_attention_weight_min": -30.09349250793457,
      "activations/layer17_attention_weight_max": 37.51945877075195,
      "activations/layer17_attention_weight_min": -26.296419143676758,
      "activations/layer18_attention_weight_max": 31.10576820373535,
      "activations/layer18_attention_weight_min": -20.128639221191406,
      "activations/layer19_attention_weight_max": 47.8956413269043,
      "activations/layer19_attention_weight_min": -29.451841354370117,
      "activations/layer1_attention_weight_max": 18.873882293701172,
      "activations/layer1_attention_weight_min": -13.516494750976562,
      "activations/layer20_attention_weight_max": 38.637760162353516,
      "activations/layer20_attention_weight_min": -22.87632179260254,
      "activations/layer21_attention_weight_max": 34.70332336425781,
      "activations/layer21_attention_weight_min": -21.627796173095703,
      "activations/layer22_attention_weight_max": 53.68964385986328,
      "activations/layer22_attention_weight_min": -27.64240264892578,
      "activations/layer23_attention_weight_max": 34.53929901123047,
      "activations/layer23_attention_weight_min": -21.48770523071289,
      "activations/layer2_attention_weight_max": 32.21168899536133,
      "activations/layer2_attention_weight_min": -29.36077880859375,
      "activations/layer3_attention_weight_max": 90.4033432006836,
      "activations/layer3_attention_weight_min": -91.33490753173828,
      "activations/layer4_attention_weight_max": 122.2375717163086,
      "activations/layer4_attention_weight_min": -122.37764739990234,
      "activations/layer5_attention_weight_max": 62.90256881713867,
      "activations/layer5_attention_weight_min": -65.21174621582031,
      "activations/layer6_attention_weight_max": 51.56367492675781,
      "activations/layer6_attention_weight_min": -56.292728424072266,
      "activations/layer7_attention_weight_max": 87.04337310791016,
      "activations/layer7_attention_weight_min": -88.5537338256836,
      "activations/layer8_attention_weight_max": 38.631160736083984,
      "activations/layer8_attention_weight_min": -37.33462905883789,
      "activations/layer9_attention_weight_max": 35.74022674560547,
      "activations/layer9_attention_weight_min": -32.83982467651367,
      "epoch": 6.46,
      "learning_rate": 0.00010945189393939392,
      "loss": 2.8361,
      "step": 111150
    },
    {
      "activations/layer0_attention_weight_max": 17.389102935791016,
      "activations/layer0_attention_weight_min": -12.333086013793945,
      "activations/layer10_attention_weight_max": 36.65703582763672,
      "activations/layer10_attention_weight_min": -33.948158264160156,
      "activations/layer11_attention_weight_max": 34.67942810058594,
      "activations/layer11_attention_weight_min": -32.14363479614258,
      "activations/layer12_attention_weight_max": 24.368362426757812,
      "activations/layer12_attention_weight_min": -24.89144515991211,
      "activations/layer13_attention_weight_max": 47.57016372680664,
      "activations/layer13_attention_weight_min": -34.17347717285156,
      "activations/layer14_attention_weight_max": 69.88695526123047,
      "activations/layer14_attention_weight_min": -44.70824432373047,
      "activations/layer15_attention_weight_max": 42.68342208862305,
      "activations/layer15_attention_weight_min": -29.384254455566406,
      "activations/layer16_attention_weight_max": 42.259765625,
      "activations/layer16_attention_weight_min": -28.555622100830078,
      "activations/layer17_attention_weight_max": 42.065711975097656,
      "activations/layer17_attention_weight_min": -25.226730346679688,
      "activations/layer18_attention_weight_max": 37.4160041809082,
      "activations/layer18_attention_weight_min": -22.047103881835938,
      "activations/layer19_attention_weight_max": 48.50087356567383,
      "activations/layer19_attention_weight_min": -30.437332153320312,
      "activations/layer1_attention_weight_max": 16.86634063720703,
      "activations/layer1_attention_weight_min": -15.976149559020996,
      "activations/layer20_attention_weight_max": 41.26681900024414,
      "activations/layer20_attention_weight_min": -24.66243553161621,
      "activations/layer21_attention_weight_max": 41.00775909423828,
      "activations/layer21_attention_weight_min": -22.48407554626465,
      "activations/layer22_attention_weight_max": 62.24673843383789,
      "activations/layer22_attention_weight_min": -29.007844924926758,
      "activations/layer23_attention_weight_max": 43.110469818115234,
      "activations/layer23_attention_weight_min": -23.097427368164062,
      "activations/layer2_attention_weight_max": 34.48482131958008,
      "activations/layer2_attention_weight_min": -30.841306686401367,
      "activations/layer3_attention_weight_max": 92.36107635498047,
      "activations/layer3_attention_weight_min": -93.83480072021484,
      "activations/layer4_attention_weight_max": 128.49163818359375,
      "activations/layer4_attention_weight_min": -124.51856994628906,
      "activations/layer5_attention_weight_max": 69.5247802734375,
      "activations/layer5_attention_weight_min": -66.87910461425781,
      "activations/layer6_attention_weight_max": 54.84424591064453,
      "activations/layer6_attention_weight_min": -59.16452407836914,
      "activations/layer7_attention_weight_max": 93.71886444091797,
      "activations/layer7_attention_weight_min": -86.3571548461914,
      "activations/layer8_attention_weight_max": 42.01289749145508,
      "activations/layer8_attention_weight_min": -41.3164176940918,
      "activations/layer9_attention_weight_max": 39.17030715942383,
      "activations/layer9_attention_weight_min": -38.24635696411133,
      "epoch": 6.46,
      "learning_rate": 0.00010943295454545454,
      "loss": 2.8207,
      "step": 111200
    },
    {
      "activations/layer0_attention_weight_max": 16.402973175048828,
      "activations/layer0_attention_weight_min": -12.174066543579102,
      "activations/layer10_attention_weight_max": 31.40887451171875,
      "activations/layer10_attention_weight_min": -31.572582244873047,
      "activations/layer11_attention_weight_max": 30.43136215209961,
      "activations/layer11_attention_weight_min": -27.612079620361328,
      "activations/layer12_attention_weight_max": 23.227521896362305,
      "activations/layer12_attention_weight_min": -26.601686477661133,
      "activations/layer13_attention_weight_max": 46.662723541259766,
      "activations/layer13_attention_weight_min": -33.00086212158203,
      "activations/layer14_attention_weight_max": 49.72583770751953,
      "activations/layer14_attention_weight_min": -40.14395523071289,
      "activations/layer15_attention_weight_max": 35.88844299316406,
      "activations/layer15_attention_weight_min": -28.257553100585938,
      "activations/layer16_attention_weight_max": 33.060630798339844,
      "activations/layer16_attention_weight_min": -30.11016845703125,
      "activations/layer17_attention_weight_max": 34.23786163330078,
      "activations/layer17_attention_weight_min": -25.26372718811035,
      "activations/layer18_attention_weight_max": 32.188114166259766,
      "activations/layer18_attention_weight_min": -20.9303035736084,
      "activations/layer19_attention_weight_max": 35.1480712890625,
      "activations/layer19_attention_weight_min": -30.128732681274414,
      "activations/layer1_attention_weight_max": 17.16010284423828,
      "activations/layer1_attention_weight_min": -12.957817077636719,
      "activations/layer20_attention_weight_max": 27.264848709106445,
      "activations/layer20_attention_weight_min": -24.716384887695312,
      "activations/layer21_attention_weight_max": 31.367685317993164,
      "activations/layer21_attention_weight_min": -23.02289581298828,
      "activations/layer22_attention_weight_max": 44.17292785644531,
      "activations/layer22_attention_weight_min": -29.26809310913086,
      "activations/layer23_attention_weight_max": 32.41019821166992,
      "activations/layer23_attention_weight_min": -22.563133239746094,
      "activations/layer2_attention_weight_max": 31.058734893798828,
      "activations/layer2_attention_weight_min": -30.368932723999023,
      "activations/layer3_attention_weight_max": 88.2744369506836,
      "activations/layer3_attention_weight_min": -91.44025421142578,
      "activations/layer4_attention_weight_max": 124.94013977050781,
      "activations/layer4_attention_weight_min": -129.09544372558594,
      "activations/layer5_attention_weight_max": 70.88899993896484,
      "activations/layer5_attention_weight_min": -69.60538482666016,
      "activations/layer6_attention_weight_max": 53.70390701293945,
      "activations/layer6_attention_weight_min": -55.33203887939453,
      "activations/layer7_attention_weight_max": 85.49870300292969,
      "activations/layer7_attention_weight_min": -94.61372375488281,
      "activations/layer8_attention_weight_max": 39.7382698059082,
      "activations/layer8_attention_weight_min": -41.00839614868164,
      "activations/layer9_attention_weight_max": 34.99794387817383,
      "activations/layer9_attention_weight_min": -36.06924057006836,
      "epoch": 6.46,
      "learning_rate": 0.00010941401515151515,
      "loss": 2.819,
      "step": 111250
    },
    {
      "activations/layer0_attention_weight_max": 14.786581039428711,
      "activations/layer0_attention_weight_min": -12.150672912597656,
      "activations/layer10_attention_weight_max": 34.55222702026367,
      "activations/layer10_attention_weight_min": -31.72348976135254,
      "activations/layer11_attention_weight_max": 36.858238220214844,
      "activations/layer11_attention_weight_min": -31.70771026611328,
      "activations/layer12_attention_weight_max": 33.42751693725586,
      "activations/layer12_attention_weight_min": -25.082841873168945,
      "activations/layer13_attention_weight_max": 43.859230041503906,
      "activations/layer13_attention_weight_min": -33.302154541015625,
      "activations/layer14_attention_weight_max": 57.69916534423828,
      "activations/layer14_attention_weight_min": -44.38257598876953,
      "activations/layer15_attention_weight_max": 35.61274337768555,
      "activations/layer15_attention_weight_min": -28.574922561645508,
      "activations/layer16_attention_weight_max": 34.804264068603516,
      "activations/layer16_attention_weight_min": -28.28487205505371,
      "activations/layer17_attention_weight_max": 37.37165069580078,
      "activations/layer17_attention_weight_min": -24.32741355895996,
      "activations/layer18_attention_weight_max": 32.16549301147461,
      "activations/layer18_attention_weight_min": -20.99985122680664,
      "activations/layer19_attention_weight_max": 37.98948287963867,
      "activations/layer19_attention_weight_min": -29.136884689331055,
      "activations/layer1_attention_weight_max": 18.06570053100586,
      "activations/layer1_attention_weight_min": -13.579883575439453,
      "activations/layer20_attention_weight_max": 30.808088302612305,
      "activations/layer20_attention_weight_min": -25.3375186920166,
      "activations/layer21_attention_weight_max": 29.526342391967773,
      "activations/layer21_attention_weight_min": -21.652013778686523,
      "activations/layer22_attention_weight_max": 54.5313720703125,
      "activations/layer22_attention_weight_min": -31.17036247253418,
      "activations/layer23_attention_weight_max": 36.938621520996094,
      "activations/layer23_attention_weight_min": -21.397520065307617,
      "activations/layer2_attention_weight_max": 30.93142318725586,
      "activations/layer2_attention_weight_min": -31.21160888671875,
      "activations/layer3_attention_weight_max": 94.32356262207031,
      "activations/layer3_attention_weight_min": -94.86102294921875,
      "activations/layer4_attention_weight_max": 123.70404052734375,
      "activations/layer4_attention_weight_min": -127.35393524169922,
      "activations/layer5_attention_weight_max": 68.1981201171875,
      "activations/layer5_attention_weight_min": -69.8630142211914,
      "activations/layer6_attention_weight_max": 53.73591613769531,
      "activations/layer6_attention_weight_min": -54.723323822021484,
      "activations/layer7_attention_weight_max": 89.05625915527344,
      "activations/layer7_attention_weight_min": -88.08002471923828,
      "activations/layer8_attention_weight_max": 43.669044494628906,
      "activations/layer8_attention_weight_min": -40.21668243408203,
      "activations/layer9_attention_weight_max": 37.62709045410156,
      "activations/layer9_attention_weight_min": -37.10586166381836,
      "epoch": 6.47,
      "learning_rate": 0.00010939507575757574,
      "loss": 2.8359,
      "step": 111300
    },
    {
      "activations/layer0_attention_weight_max": 15.466876983642578,
      "activations/layer0_attention_weight_min": -12.734164237976074,
      "activations/layer10_attention_weight_max": 34.835052490234375,
      "activations/layer10_attention_weight_min": -32.8282356262207,
      "activations/layer11_attention_weight_max": 34.79765319824219,
      "activations/layer11_attention_weight_min": -32.89988708496094,
      "activations/layer12_attention_weight_max": 27.286483764648438,
      "activations/layer12_attention_weight_min": -23.065322875976562,
      "activations/layer13_attention_weight_max": 35.481422424316406,
      "activations/layer13_attention_weight_min": -33.46776580810547,
      "activations/layer14_attention_weight_max": 56.6212158203125,
      "activations/layer14_attention_weight_min": -47.22377395629883,
      "activations/layer15_attention_weight_max": 36.59654998779297,
      "activations/layer15_attention_weight_min": -28.362239837646484,
      "activations/layer16_attention_weight_max": 34.02655792236328,
      "activations/layer16_attention_weight_min": -29.713897705078125,
      "activations/layer17_attention_weight_max": 36.65495681762695,
      "activations/layer17_attention_weight_min": -25.514760971069336,
      "activations/layer18_attention_weight_max": 31.751182556152344,
      "activations/layer18_attention_weight_min": -19.607589721679688,
      "activations/layer19_attention_weight_max": 37.42942428588867,
      "activations/layer19_attention_weight_min": -30.21388816833496,
      "activations/layer1_attention_weight_max": 16.626998901367188,
      "activations/layer1_attention_weight_min": -14.169060707092285,
      "activations/layer20_attention_weight_max": 28.877410888671875,
      "activations/layer20_attention_weight_min": -23.40672492980957,
      "activations/layer21_attention_weight_max": 33.50566482543945,
      "activations/layer21_attention_weight_min": -21.26038932800293,
      "activations/layer22_attention_weight_max": 48.306419372558594,
      "activations/layer22_attention_weight_min": -29.64130973815918,
      "activations/layer23_attention_weight_max": 32.560916900634766,
      "activations/layer23_attention_weight_min": -21.523841857910156,
      "activations/layer2_attention_weight_max": 31.44056510925293,
      "activations/layer2_attention_weight_min": -30.525495529174805,
      "activations/layer3_attention_weight_max": 91.13736724853516,
      "activations/layer3_attention_weight_min": -89.79680633544922,
      "activations/layer4_attention_weight_max": 132.12510681152344,
      "activations/layer4_attention_weight_min": -121.74181365966797,
      "activations/layer5_attention_weight_max": 68.26350402832031,
      "activations/layer5_attention_weight_min": -65.86565399169922,
      "activations/layer6_attention_weight_max": 56.53470230102539,
      "activations/layer6_attention_weight_min": -54.82709884643555,
      "activations/layer7_attention_weight_max": 86.77346801757812,
      "activations/layer7_attention_weight_min": -88.8857650756836,
      "activations/layer8_attention_weight_max": 43.415565490722656,
      "activations/layer8_attention_weight_min": -42.39854431152344,
      "activations/layer9_attention_weight_max": 39.52786636352539,
      "activations/layer9_attention_weight_min": -36.0335807800293,
      "epoch": 6.47,
      "learning_rate": 0.00010937613636363636,
      "loss": 2.8277,
      "step": 111350
    },
    {
      "activations/layer0_attention_weight_max": 16.984291076660156,
      "activations/layer0_attention_weight_min": -12.336889266967773,
      "activations/layer10_attention_weight_max": 34.53572082519531,
      "activations/layer10_attention_weight_min": -32.303653717041016,
      "activations/layer11_attention_weight_max": 31.448270797729492,
      "activations/layer11_attention_weight_min": -28.78437042236328,
      "activations/layer12_attention_weight_max": 31.11371421813965,
      "activations/layer12_attention_weight_min": -30.311479568481445,
      "activations/layer13_attention_weight_max": 38.0653190612793,
      "activations/layer13_attention_weight_min": -30.074981689453125,
      "activations/layer14_attention_weight_max": 53.35029220581055,
      "activations/layer14_attention_weight_min": -38.657501220703125,
      "activations/layer15_attention_weight_max": 35.26799392700195,
      "activations/layer15_attention_weight_min": -29.047510147094727,
      "activations/layer16_attention_weight_max": 33.823089599609375,
      "activations/layer16_attention_weight_min": -29.826740264892578,
      "activations/layer17_attention_weight_max": 34.25934600830078,
      "activations/layer17_attention_weight_min": -24.1125545501709,
      "activations/layer18_attention_weight_max": 30.579885482788086,
      "activations/layer18_attention_weight_min": -21.74917221069336,
      "activations/layer19_attention_weight_max": 35.748294830322266,
      "activations/layer19_attention_weight_min": -28.37700843811035,
      "activations/layer1_attention_weight_max": 17.86577606201172,
      "activations/layer1_attention_weight_min": -12.858933448791504,
      "activations/layer20_attention_weight_max": 29.982376098632812,
      "activations/layer20_attention_weight_min": -22.898197174072266,
      "activations/layer21_attention_weight_max": 31.43305015563965,
      "activations/layer21_attention_weight_min": -20.723731994628906,
      "activations/layer22_attention_weight_max": 45.88831329345703,
      "activations/layer22_attention_weight_min": -26.8438777923584,
      "activations/layer23_attention_weight_max": 34.272865295410156,
      "activations/layer23_attention_weight_min": -22.11449432373047,
      "activations/layer2_attention_weight_max": 32.78195571899414,
      "activations/layer2_attention_weight_min": -31.903594970703125,
      "activations/layer3_attention_weight_max": 91.7752914428711,
      "activations/layer3_attention_weight_min": -92.2929916381836,
      "activations/layer4_attention_weight_max": 124.43621063232422,
      "activations/layer4_attention_weight_min": -126.94425201416016,
      "activations/layer5_attention_weight_max": 67.99878692626953,
      "activations/layer5_attention_weight_min": -66.67160034179688,
      "activations/layer6_attention_weight_max": 54.20145034790039,
      "activations/layer6_attention_weight_min": -55.610687255859375,
      "activations/layer7_attention_weight_max": 90.4081039428711,
      "activations/layer7_attention_weight_min": -84.70289611816406,
      "activations/layer8_attention_weight_max": 42.00426483154297,
      "activations/layer8_attention_weight_min": -41.10383224487305,
      "activations/layer9_attention_weight_max": 35.20977783203125,
      "activations/layer9_attention_weight_min": -34.66472625732422,
      "epoch": 6.47,
      "learning_rate": 0.00010935719696969696,
      "loss": 2.8288,
      "step": 111400
    },
    {
      "activations/layer0_attention_weight_max": 16.4716796875,
      "activations/layer0_attention_weight_min": -11.318037986755371,
      "activations/layer10_attention_weight_max": 34.86246871948242,
      "activations/layer10_attention_weight_min": -33.36996841430664,
      "activations/layer11_attention_weight_max": 40.44197463989258,
      "activations/layer11_attention_weight_min": -31.681259155273438,
      "activations/layer12_attention_weight_max": 39.141117095947266,
      "activations/layer12_attention_weight_min": -29.625425338745117,
      "activations/layer13_attention_weight_max": 57.73925018310547,
      "activations/layer13_attention_weight_min": -33.60987091064453,
      "activations/layer14_attention_weight_max": 61.015655517578125,
      "activations/layer14_attention_weight_min": -41.68498611450195,
      "activations/layer15_attention_weight_max": 55.32943344116211,
      "activations/layer15_attention_weight_min": -28.70320701599121,
      "activations/layer16_attention_weight_max": 38.3178596496582,
      "activations/layer16_attention_weight_min": -26.49082374572754,
      "activations/layer17_attention_weight_max": 34.13410568237305,
      "activations/layer17_attention_weight_min": -24.200763702392578,
      "activations/layer18_attention_weight_max": 29.218101501464844,
      "activations/layer18_attention_weight_min": -20.596324920654297,
      "activations/layer19_attention_weight_max": 34.87827682495117,
      "activations/layer19_attention_weight_min": -30.242538452148438,
      "activations/layer1_attention_weight_max": 16.35199546813965,
      "activations/layer1_attention_weight_min": -13.135892868041992,
      "activations/layer20_attention_weight_max": 31.271284103393555,
      "activations/layer20_attention_weight_min": -20.705549240112305,
      "activations/layer21_attention_weight_max": 28.06216812133789,
      "activations/layer21_attention_weight_min": -18.958431243896484,
      "activations/layer22_attention_weight_max": 45.00274658203125,
      "activations/layer22_attention_weight_min": -27.037609100341797,
      "activations/layer23_attention_weight_max": 29.015663146972656,
      "activations/layer23_attention_weight_min": -20.681396484375,
      "activations/layer2_attention_weight_max": 29.339567184448242,
      "activations/layer2_attention_weight_min": -28.976913452148438,
      "activations/layer3_attention_weight_max": 83.7049331665039,
      "activations/layer3_attention_weight_min": -88.59468841552734,
      "activations/layer4_attention_weight_max": 121.74295806884766,
      "activations/layer4_attention_weight_min": -126.10298919677734,
      "activations/layer5_attention_weight_max": 67.44673156738281,
      "activations/layer5_attention_weight_min": -67.73089599609375,
      "activations/layer6_attention_weight_max": 51.71296691894531,
      "activations/layer6_attention_weight_min": -57.99361038208008,
      "activations/layer7_attention_weight_max": 88.14998626708984,
      "activations/layer7_attention_weight_min": -93.87854766845703,
      "activations/layer8_attention_weight_max": 43.50115966796875,
      "activations/layer8_attention_weight_min": -42.879676818847656,
      "activations/layer9_attention_weight_max": 34.060306549072266,
      "activations/layer9_attention_weight_min": -37.136329650878906,
      "epoch": 6.48,
      "learning_rate": 0.00010933825757575756,
      "loss": 2.8255,
      "step": 111450
    },
    {
      "activations/layer0_attention_weight_max": 15.874831199645996,
      "activations/layer0_attention_weight_min": -12.03624439239502,
      "activations/layer10_attention_weight_max": 42.28488540649414,
      "activations/layer10_attention_weight_min": -39.5509033203125,
      "activations/layer11_attention_weight_max": 45.94765853881836,
      "activations/layer11_attention_weight_min": -36.682647705078125,
      "activations/layer12_attention_weight_max": 30.236791610717773,
      "activations/layer12_attention_weight_min": -27.93034553527832,
      "activations/layer13_attention_weight_max": 63.12737274169922,
      "activations/layer13_attention_weight_min": -36.28548049926758,
      "activations/layer14_attention_weight_max": 78.8045883178711,
      "activations/layer14_attention_weight_min": -48.04569625854492,
      "activations/layer15_attention_weight_max": 66.4246597290039,
      "activations/layer15_attention_weight_min": -32.05244445800781,
      "activations/layer16_attention_weight_max": 44.588134765625,
      "activations/layer16_attention_weight_min": -26.888607025146484,
      "activations/layer17_attention_weight_max": 38.123878479003906,
      "activations/layer17_attention_weight_min": -24.448772430419922,
      "activations/layer18_attention_weight_max": 41.67427062988281,
      "activations/layer18_attention_weight_min": -23.63298797607422,
      "activations/layer19_attention_weight_max": 51.60532760620117,
      "activations/layer19_attention_weight_min": -31.155643463134766,
      "activations/layer1_attention_weight_max": 17.82851791381836,
      "activations/layer1_attention_weight_min": -15.723556518554688,
      "activations/layer20_attention_weight_max": 37.10983657836914,
      "activations/layer20_attention_weight_min": -23.02940559387207,
      "activations/layer21_attention_weight_max": 37.725372314453125,
      "activations/layer21_attention_weight_min": -20.391437530517578,
      "activations/layer22_attention_weight_max": 60.84255599975586,
      "activations/layer22_attention_weight_min": -27.65557098388672,
      "activations/layer23_attention_weight_max": 40.2789421081543,
      "activations/layer23_attention_weight_min": -21.905399322509766,
      "activations/layer2_attention_weight_max": 32.860809326171875,
      "activations/layer2_attention_weight_min": -32.166282653808594,
      "activations/layer3_attention_weight_max": 91.80014038085938,
      "activations/layer3_attention_weight_min": -99.60301971435547,
      "activations/layer4_attention_weight_max": 132.60484313964844,
      "activations/layer4_attention_weight_min": -132.8907928466797,
      "activations/layer5_attention_weight_max": 70.16812133789062,
      "activations/layer5_attention_weight_min": -67.67268371582031,
      "activations/layer6_attention_weight_max": 53.074806213378906,
      "activations/layer6_attention_weight_min": -55.01716995239258,
      "activations/layer7_attention_weight_max": 94.60094451904297,
      "activations/layer7_attention_weight_min": -91.04170989990234,
      "activations/layer8_attention_weight_max": 45.054954528808594,
      "activations/layer8_attention_weight_min": -43.39112854003906,
      "activations/layer9_attention_weight_max": 40.851097106933594,
      "activations/layer9_attention_weight_min": -38.501773834228516,
      "epoch": 6.48,
      "learning_rate": 0.00010931931818181817,
      "loss": 2.8399,
      "step": 111500
    },
    {
      "activations/layer0_attention_weight_max": 15.020333290100098,
      "activations/layer0_attention_weight_min": -12.721333503723145,
      "activations/layer10_attention_weight_max": 35.02044677734375,
      "activations/layer10_attention_weight_min": -31.868545532226562,
      "activations/layer11_attention_weight_max": 32.00299072265625,
      "activations/layer11_attention_weight_min": -28.650739669799805,
      "activations/layer12_attention_weight_max": 22.995956420898438,
      "activations/layer12_attention_weight_min": -29.927595138549805,
      "activations/layer13_attention_weight_max": 39.97166061401367,
      "activations/layer13_attention_weight_min": -33.827266693115234,
      "activations/layer14_attention_weight_max": 62.10091018676758,
      "activations/layer14_attention_weight_min": -45.582435607910156,
      "activations/layer15_attention_weight_max": 36.03403091430664,
      "activations/layer15_attention_weight_min": -28.600629806518555,
      "activations/layer16_attention_weight_max": 37.08982467651367,
      "activations/layer16_attention_weight_min": -29.102251052856445,
      "activations/layer17_attention_weight_max": 35.689971923828125,
      "activations/layer17_attention_weight_min": -24.789827346801758,
      "activations/layer18_attention_weight_max": 32.551631927490234,
      "activations/layer18_attention_weight_min": -21.390111923217773,
      "activations/layer19_attention_weight_max": 42.59528732299805,
      "activations/layer19_attention_weight_min": -31.8270206451416,
      "activations/layer1_attention_weight_max": 16.754247665405273,
      "activations/layer1_attention_weight_min": -14.846433639526367,
      "activations/layer20_attention_weight_max": 32.68295669555664,
      "activations/layer20_attention_weight_min": -24.900283813476562,
      "activations/layer21_attention_weight_max": 32.0787467956543,
      "activations/layer21_attention_weight_min": -22.206205368041992,
      "activations/layer22_attention_weight_max": 48.985172271728516,
      "activations/layer22_attention_weight_min": -29.575435638427734,
      "activations/layer23_attention_weight_max": 39.14057159423828,
      "activations/layer23_attention_weight_min": -22.265827178955078,
      "activations/layer2_attention_weight_max": 33.18768310546875,
      "activations/layer2_attention_weight_min": -31.727386474609375,
      "activations/layer3_attention_weight_max": 92.57110595703125,
      "activations/layer3_attention_weight_min": -94.32275390625,
      "activations/layer4_attention_weight_max": 126.72084045410156,
      "activations/layer4_attention_weight_min": -124.33683013916016,
      "activations/layer5_attention_weight_max": 68.54168701171875,
      "activations/layer5_attention_weight_min": -66.40987396240234,
      "activations/layer6_attention_weight_max": 53.23255157470703,
      "activations/layer6_attention_weight_min": -53.62520217895508,
      "activations/layer7_attention_weight_max": 85.17214965820312,
      "activations/layer7_attention_weight_min": -88.09395599365234,
      "activations/layer8_attention_weight_max": 42.47682571411133,
      "activations/layer8_attention_weight_min": -40.86482620239258,
      "activations/layer9_attention_weight_max": 36.38167953491211,
      "activations/layer9_attention_weight_min": -33.85021209716797,
      "epoch": 6.48,
      "learning_rate": 0.00010930037878787878,
      "loss": 2.8518,
      "step": 111550
    },
    {
      "activations/layer0_attention_weight_max": 16.41440200805664,
      "activations/layer0_attention_weight_min": -12.930130004882812,
      "activations/layer10_attention_weight_max": 36.18785095214844,
      "activations/layer10_attention_weight_min": -32.74579620361328,
      "activations/layer11_attention_weight_max": 38.38175964355469,
      "activations/layer11_attention_weight_min": -31.112659454345703,
      "activations/layer12_attention_weight_max": 44.01926040649414,
      "activations/layer12_attention_weight_min": -26.35035514831543,
      "activations/layer13_attention_weight_max": 68.13604736328125,
      "activations/layer13_attention_weight_min": -40.36213684082031,
      "activations/layer14_attention_weight_max": 64.97319793701172,
      "activations/layer14_attention_weight_min": -43.01901626586914,
      "activations/layer15_attention_weight_max": 65.71170806884766,
      "activations/layer15_attention_weight_min": -32.21774673461914,
      "activations/layer16_attention_weight_max": 38.51716613769531,
      "activations/layer16_attention_weight_min": -27.878278732299805,
      "activations/layer17_attention_weight_max": 32.11677169799805,
      "activations/layer17_attention_weight_min": -25.014923095703125,
      "activations/layer18_attention_weight_max": 34.2689323425293,
      "activations/layer18_attention_weight_min": -21.185161590576172,
      "activations/layer19_attention_weight_max": 35.837520599365234,
      "activations/layer19_attention_weight_min": -27.95533561706543,
      "activations/layer1_attention_weight_max": 17.329240798950195,
      "activations/layer1_attention_weight_min": -12.939374923706055,
      "activations/layer20_attention_weight_max": 28.276165008544922,
      "activations/layer20_attention_weight_min": -23.707073211669922,
      "activations/layer21_attention_weight_max": 30.931718826293945,
      "activations/layer21_attention_weight_min": -22.23581314086914,
      "activations/layer22_attention_weight_max": 45.3427619934082,
      "activations/layer22_attention_weight_min": -28.26090431213379,
      "activations/layer23_attention_weight_max": 34.46207809448242,
      "activations/layer23_attention_weight_min": -22.425033569335938,
      "activations/layer2_attention_weight_max": 33.410011291503906,
      "activations/layer2_attention_weight_min": -32.5770263671875,
      "activations/layer3_attention_weight_max": 92.76359558105469,
      "activations/layer3_attention_weight_min": -93.92890167236328,
      "activations/layer4_attention_weight_max": 124.61248779296875,
      "activations/layer4_attention_weight_min": -122.4540786743164,
      "activations/layer5_attention_weight_max": 64.97337341308594,
      "activations/layer5_attention_weight_min": -64.05804443359375,
      "activations/layer6_attention_weight_max": 53.084590911865234,
      "activations/layer6_attention_weight_min": -52.51346206665039,
      "activations/layer7_attention_weight_max": 93.75530242919922,
      "activations/layer7_attention_weight_min": -92.44482421875,
      "activations/layer8_attention_weight_max": 41.38794708251953,
      "activations/layer8_attention_weight_min": -39.07258224487305,
      "activations/layer9_attention_weight_max": 38.052093505859375,
      "activations/layer9_attention_weight_min": -34.12693405151367,
      "epoch": 6.48,
      "learning_rate": 0.00010928143939393939,
      "loss": 2.8273,
      "step": 111600
    },
    {
      "activations/layer0_attention_weight_max": 16.477874755859375,
      "activations/layer0_attention_weight_min": -12.329031944274902,
      "activations/layer10_attention_weight_max": 38.31745147705078,
      "activations/layer10_attention_weight_min": -33.88913345336914,
      "activations/layer11_attention_weight_max": 37.139320373535156,
      "activations/layer11_attention_weight_min": -29.745731353759766,
      "activations/layer12_attention_weight_max": 24.852081298828125,
      "activations/layer12_attention_weight_min": -31.104324340820312,
      "activations/layer13_attention_weight_max": 48.098304748535156,
      "activations/layer13_attention_weight_min": -32.40230941772461,
      "activations/layer14_attention_weight_max": 60.348976135253906,
      "activations/layer14_attention_weight_min": -41.7371826171875,
      "activations/layer15_attention_weight_max": 43.20581817626953,
      "activations/layer15_attention_weight_min": -28.1088924407959,
      "activations/layer16_attention_weight_max": 39.08279800415039,
      "activations/layer16_attention_weight_min": -27.155677795410156,
      "activations/layer17_attention_weight_max": 40.091224670410156,
      "activations/layer17_attention_weight_min": -24.68284034729004,
      "activations/layer18_attention_weight_max": 30.794795989990234,
      "activations/layer18_attention_weight_min": -21.47756004333496,
      "activations/layer19_attention_weight_max": 36.45849609375,
      "activations/layer19_attention_weight_min": -28.394153594970703,
      "activations/layer1_attention_weight_max": 16.343114852905273,
      "activations/layer1_attention_weight_min": -13.894242286682129,
      "activations/layer20_attention_weight_max": 34.55679702758789,
      "activations/layer20_attention_weight_min": -23.202503204345703,
      "activations/layer21_attention_weight_max": 29.91568946838379,
      "activations/layer21_attention_weight_min": -20.753618240356445,
      "activations/layer22_attention_weight_max": 48.32732009887695,
      "activations/layer22_attention_weight_min": -27.51356315612793,
      "activations/layer23_attention_weight_max": 32.960166931152344,
      "activations/layer23_attention_weight_min": -21.24637794494629,
      "activations/layer2_attention_weight_max": 31.54568099975586,
      "activations/layer2_attention_weight_min": -29.74554443359375,
      "activations/layer3_attention_weight_max": 90.53510284423828,
      "activations/layer3_attention_weight_min": -91.1653060913086,
      "activations/layer4_attention_weight_max": 124.20458221435547,
      "activations/layer4_attention_weight_min": -121.36461639404297,
      "activations/layer5_attention_weight_max": 68.403564453125,
      "activations/layer5_attention_weight_min": -66.22409057617188,
      "activations/layer6_attention_weight_max": 54.931053161621094,
      "activations/layer6_attention_weight_min": -54.96543502807617,
      "activations/layer7_attention_weight_max": 89.8051986694336,
      "activations/layer7_attention_weight_min": -91.10990905761719,
      "activations/layer8_attention_weight_max": 41.82122039794922,
      "activations/layer8_attention_weight_min": -42.701622009277344,
      "activations/layer9_attention_weight_max": 38.91746139526367,
      "activations/layer9_attention_weight_min": -36.51826858520508,
      "epoch": 6.49,
      "learning_rate": 0.00010926249999999999,
      "loss": 2.8149,
      "step": 111650
    },
    {
      "activations/layer0_attention_weight_max": 14.941839218139648,
      "activations/layer0_attention_weight_min": -11.594117164611816,
      "activations/layer10_attention_weight_max": 38.894630432128906,
      "activations/layer10_attention_weight_min": -36.198341369628906,
      "activations/layer11_attention_weight_max": 41.29608917236328,
      "activations/layer11_attention_weight_min": -32.77770233154297,
      "activations/layer12_attention_weight_max": 28.471237182617188,
      "activations/layer12_attention_weight_min": -27.12982940673828,
      "activations/layer13_attention_weight_max": 53.4096794128418,
      "activations/layer13_attention_weight_min": -33.02070617675781,
      "activations/layer14_attention_weight_max": 79.40926361083984,
      "activations/layer14_attention_weight_min": -50.11482620239258,
      "activations/layer15_attention_weight_max": 43.029109954833984,
      "activations/layer15_attention_weight_min": -29.638254165649414,
      "activations/layer16_attention_weight_max": 41.01392364501953,
      "activations/layer16_attention_weight_min": -26.427385330200195,
      "activations/layer17_attention_weight_max": 45.574886322021484,
      "activations/layer17_attention_weight_min": -22.78070640563965,
      "activations/layer18_attention_weight_max": 37.760833740234375,
      "activations/layer18_attention_weight_min": -23.59933853149414,
      "activations/layer19_attention_weight_max": 51.263607025146484,
      "activations/layer19_attention_weight_min": -29.314971923828125,
      "activations/layer1_attention_weight_max": 17.625789642333984,
      "activations/layer1_attention_weight_min": -14.433426856994629,
      "activations/layer20_attention_weight_max": 43.406890869140625,
      "activations/layer20_attention_weight_min": -24.246213912963867,
      "activations/layer21_attention_weight_max": 38.87628936767578,
      "activations/layer21_attention_weight_min": -21.03032684326172,
      "activations/layer22_attention_weight_max": 69.60582733154297,
      "activations/layer22_attention_weight_min": -28.457448959350586,
      "activations/layer23_attention_weight_max": 46.74027633666992,
      "activations/layer23_attention_weight_min": -21.690841674804688,
      "activations/layer2_attention_weight_max": 32.43424987792969,
      "activations/layer2_attention_weight_min": -30.2718505859375,
      "activations/layer3_attention_weight_max": 91.48419189453125,
      "activations/layer3_attention_weight_min": -96.17440795898438,
      "activations/layer4_attention_weight_max": 131.99266052246094,
      "activations/layer4_attention_weight_min": -128.83132934570312,
      "activations/layer5_attention_weight_max": 71.57197570800781,
      "activations/layer5_attention_weight_min": -68.08008575439453,
      "activations/layer6_attention_weight_max": 54.18238830566406,
      "activations/layer6_attention_weight_min": -58.326358795166016,
      "activations/layer7_attention_weight_max": 91.64281463623047,
      "activations/layer7_attention_weight_min": -93.64103698730469,
      "activations/layer8_attention_weight_max": 42.322792053222656,
      "activations/layer8_attention_weight_min": -43.567909240722656,
      "activations/layer9_attention_weight_max": 41.43156814575195,
      "activations/layer9_attention_weight_min": -37.981319427490234,
      "epoch": 6.49,
      "learning_rate": 0.0001092435606060606,
      "loss": 2.8302,
      "step": 111700
    },
    {
      "activations/layer0_attention_weight_max": 17.055904388427734,
      "activations/layer0_attention_weight_min": -10.995779991149902,
      "activations/layer10_attention_weight_max": 30.912504196166992,
      "activations/layer10_attention_weight_min": -28.22995948791504,
      "activations/layer11_attention_weight_max": 30.103614807128906,
      "activations/layer11_attention_weight_min": -26.631053924560547,
      "activations/layer12_attention_weight_max": 21.023109436035156,
      "activations/layer12_attention_weight_min": -24.541114807128906,
      "activations/layer13_attention_weight_max": 37.337310791015625,
      "activations/layer13_attention_weight_min": -30.875436782836914,
      "activations/layer14_attention_weight_max": 46.247581481933594,
      "activations/layer14_attention_weight_min": -39.15461349487305,
      "activations/layer15_attention_weight_max": 34.8671989440918,
      "activations/layer15_attention_weight_min": -27.922624588012695,
      "activations/layer16_attention_weight_max": 32.23257064819336,
      "activations/layer16_attention_weight_min": -26.746177673339844,
      "activations/layer17_attention_weight_max": 36.248130798339844,
      "activations/layer17_attention_weight_min": -24.394956588745117,
      "activations/layer18_attention_weight_max": 32.23719024658203,
      "activations/layer18_attention_weight_min": -20.957090377807617,
      "activations/layer19_attention_weight_max": 35.79462432861328,
      "activations/layer19_attention_weight_min": -28.28301429748535,
      "activations/layer1_attention_weight_max": 16.65578269958496,
      "activations/layer1_attention_weight_min": -13.86748218536377,
      "activations/layer20_attention_weight_max": 32.30699157714844,
      "activations/layer20_attention_weight_min": -22.916362762451172,
      "activations/layer21_attention_weight_max": 28.503887176513672,
      "activations/layer21_attention_weight_min": -21.979881286621094,
      "activations/layer22_attention_weight_max": 45.885738372802734,
      "activations/layer22_attention_weight_min": -29.00356674194336,
      "activations/layer23_attention_weight_max": 33.50499725341797,
      "activations/layer23_attention_weight_min": -21.25455665588379,
      "activations/layer2_attention_weight_max": 33.61424255371094,
      "activations/layer2_attention_weight_min": -30.028682708740234,
      "activations/layer3_attention_weight_max": 89.7558822631836,
      "activations/layer3_attention_weight_min": -90.51629638671875,
      "activations/layer4_attention_weight_max": 125.33477783203125,
      "activations/layer4_attention_weight_min": -126.66609191894531,
      "activations/layer5_attention_weight_max": 69.02842712402344,
      "activations/layer5_attention_weight_min": -63.075355529785156,
      "activations/layer6_attention_weight_max": 54.82693099975586,
      "activations/layer6_attention_weight_min": -52.172019958496094,
      "activations/layer7_attention_weight_max": 85.72958374023438,
      "activations/layer7_attention_weight_min": -81.14537048339844,
      "activations/layer8_attention_weight_max": 38.89155578613281,
      "activations/layer8_attention_weight_min": -36.89005661010742,
      "activations/layer9_attention_weight_max": 32.007930755615234,
      "activations/layer9_attention_weight_min": -29.78590965270996,
      "epoch": 6.49,
      "learning_rate": 0.00010922462121212121,
      "loss": 2.8501,
      "step": 111750
    },
    {
      "activations/layer0_attention_weight_max": 16.392379760742188,
      "activations/layer0_attention_weight_min": -12.009596824645996,
      "activations/layer10_attention_weight_max": 36.073734283447266,
      "activations/layer10_attention_weight_min": -33.67141342163086,
      "activations/layer11_attention_weight_max": 35.2701416015625,
      "activations/layer11_attention_weight_min": -32.153297424316406,
      "activations/layer12_attention_weight_max": 22.035608291625977,
      "activations/layer12_attention_weight_min": -26.56049346923828,
      "activations/layer13_attention_weight_max": 48.37526321411133,
      "activations/layer13_attention_weight_min": -34.32829284667969,
      "activations/layer14_attention_weight_max": 64.7349853515625,
      "activations/layer14_attention_weight_min": -45.94122314453125,
      "activations/layer15_attention_weight_max": 37.12952423095703,
      "activations/layer15_attention_weight_min": -28.858572006225586,
      "activations/layer16_attention_weight_max": 35.88068389892578,
      "activations/layer16_attention_weight_min": -29.1094970703125,
      "activations/layer17_attention_weight_max": 36.26134490966797,
      "activations/layer17_attention_weight_min": -25.0267391204834,
      "activations/layer18_attention_weight_max": 31.163591384887695,
      "activations/layer18_attention_weight_min": -21.34724998474121,
      "activations/layer19_attention_weight_max": 39.9942741394043,
      "activations/layer19_attention_weight_min": -30.186765670776367,
      "activations/layer1_attention_weight_max": 16.174020767211914,
      "activations/layer1_attention_weight_min": -13.811075210571289,
      "activations/layer20_attention_weight_max": 32.9458122253418,
      "activations/layer20_attention_weight_min": -21.962268829345703,
      "activations/layer21_attention_weight_max": 30.698095321655273,
      "activations/layer21_attention_weight_min": -20.8565731048584,
      "activations/layer22_attention_weight_max": 45.58316421508789,
      "activations/layer22_attention_weight_min": -29.847070693969727,
      "activations/layer23_attention_weight_max": 35.275856018066406,
      "activations/layer23_attention_weight_min": -24.439456939697266,
      "activations/layer2_attention_weight_max": 32.450767517089844,
      "activations/layer2_attention_weight_min": -33.185691833496094,
      "activations/layer3_attention_weight_max": 89.49662017822266,
      "activations/layer3_attention_weight_min": -94.68289184570312,
      "activations/layer4_attention_weight_max": 129.53488159179688,
      "activations/layer4_attention_weight_min": -128.5039825439453,
      "activations/layer5_attention_weight_max": 68.45454406738281,
      "activations/layer5_attention_weight_min": -68.0840072631836,
      "activations/layer6_attention_weight_max": 51.90629577636719,
      "activations/layer6_attention_weight_min": -55.2664794921875,
      "activations/layer7_attention_weight_max": 90.65457916259766,
      "activations/layer7_attention_weight_min": -86.32245635986328,
      "activations/layer8_attention_weight_max": 43.331138610839844,
      "activations/layer8_attention_weight_min": -42.927696228027344,
      "activations/layer9_attention_weight_max": 39.05006790161133,
      "activations/layer9_attention_weight_min": -35.81438064575195,
      "epoch": 6.5,
      "learning_rate": 0.0001092056818181818,
      "loss": 2.8234,
      "step": 111800
    },
    {
      "activations/layer0_attention_weight_max": 16.664709091186523,
      "activations/layer0_attention_weight_min": -12.225008964538574,
      "activations/layer10_attention_weight_max": 30.78603172302246,
      "activations/layer10_attention_weight_min": -30.180517196655273,
      "activations/layer11_attention_weight_max": 31.79385757446289,
      "activations/layer11_attention_weight_min": -29.30615997314453,
      "activations/layer12_attention_weight_max": 24.496536254882812,
      "activations/layer12_attention_weight_min": -29.085193634033203,
      "activations/layer13_attention_weight_max": 41.588539123535156,
      "activations/layer13_attention_weight_min": -32.430931091308594,
      "activations/layer14_attention_weight_max": 55.88512420654297,
      "activations/layer14_attention_weight_min": -42.86643981933594,
      "activations/layer15_attention_weight_max": 35.763038635253906,
      "activations/layer15_attention_weight_min": -31.206233978271484,
      "activations/layer16_attention_weight_max": 30.883333206176758,
      "activations/layer16_attention_weight_min": -27.416683197021484,
      "activations/layer17_attention_weight_max": 36.54584503173828,
      "activations/layer17_attention_weight_min": -25.306079864501953,
      "activations/layer18_attention_weight_max": 35.921661376953125,
      "activations/layer18_attention_weight_min": -19.135929107666016,
      "activations/layer19_attention_weight_max": 34.98965072631836,
      "activations/layer19_attention_weight_min": -29.4171085357666,
      "activations/layer1_attention_weight_max": 16.799623489379883,
      "activations/layer1_attention_weight_min": -12.950422286987305,
      "activations/layer20_attention_weight_max": 29.140148162841797,
      "activations/layer20_attention_weight_min": -21.990325927734375,
      "activations/layer21_attention_weight_max": 28.928691864013672,
      "activations/layer21_attention_weight_min": -20.662540435791016,
      "activations/layer22_attention_weight_max": 43.98300552368164,
      "activations/layer22_attention_weight_min": -27.29082679748535,
      "activations/layer23_attention_weight_max": 32.19624710083008,
      "activations/layer23_attention_weight_min": -20.11032485961914,
      "activations/layer2_attention_weight_max": 31.988874435424805,
      "activations/layer2_attention_weight_min": -29.389286041259766,
      "activations/layer3_attention_weight_max": 87.64282989501953,
      "activations/layer3_attention_weight_min": -86.78172302246094,
      "activations/layer4_attention_weight_max": 121.57618713378906,
      "activations/layer4_attention_weight_min": -120.30269622802734,
      "activations/layer5_attention_weight_max": 64.19904327392578,
      "activations/layer5_attention_weight_min": -62.07301330566406,
      "activations/layer6_attention_weight_max": 51.11615753173828,
      "activations/layer6_attention_weight_min": -48.370155334472656,
      "activations/layer7_attention_weight_max": 83.98406982421875,
      "activations/layer7_attention_weight_min": -81.63099670410156,
      "activations/layer8_attention_weight_max": 38.88309860229492,
      "activations/layer8_attention_weight_min": -37.04942321777344,
      "activations/layer9_attention_weight_max": 33.77798080444336,
      "activations/layer9_attention_weight_min": -33.664913177490234,
      "epoch": 6.5,
      "learning_rate": 0.00010918674242424241,
      "loss": 2.8407,
      "step": 111850
    },
    {
      "activations/layer0_attention_weight_max": 16.660066604614258,
      "activations/layer0_attention_weight_min": -13.000663757324219,
      "activations/layer10_attention_weight_max": 32.44523620605469,
      "activations/layer10_attention_weight_min": -31.301225662231445,
      "activations/layer11_attention_weight_max": 31.63601303100586,
      "activations/layer11_attention_weight_min": -29.428070068359375,
      "activations/layer12_attention_weight_max": 22.088499069213867,
      "activations/layer12_attention_weight_min": -23.997766494750977,
      "activations/layer13_attention_weight_max": 39.618709564208984,
      "activations/layer13_attention_weight_min": -33.92271423339844,
      "activations/layer14_attention_weight_max": 53.615238189697266,
      "activations/layer14_attention_weight_min": -44.813907623291016,
      "activations/layer15_attention_weight_max": 33.72771072387695,
      "activations/layer15_attention_weight_min": -29.381208419799805,
      "activations/layer16_attention_weight_max": 33.61453628540039,
      "activations/layer16_attention_weight_min": -30.398038864135742,
      "activations/layer17_attention_weight_max": 33.92867660522461,
      "activations/layer17_attention_weight_min": -25.74003028869629,
      "activations/layer18_attention_weight_max": 30.23175048828125,
      "activations/layer18_attention_weight_min": -22.28839874267578,
      "activations/layer19_attention_weight_max": 37.84774398803711,
      "activations/layer19_attention_weight_min": -31.435531616210938,
      "activations/layer1_attention_weight_max": 16.913223266601562,
      "activations/layer1_attention_weight_min": -14.03472900390625,
      "activations/layer20_attention_weight_max": 27.895755767822266,
      "activations/layer20_attention_weight_min": -22.10204315185547,
      "activations/layer21_attention_weight_max": 29.322282791137695,
      "activations/layer21_attention_weight_min": -21.81812858581543,
      "activations/layer22_attention_weight_max": 44.59375,
      "activations/layer22_attention_weight_min": -28.864946365356445,
      "activations/layer23_attention_weight_max": 30.86566734313965,
      "activations/layer23_attention_weight_min": -22.777061462402344,
      "activations/layer2_attention_weight_max": 33.44732666015625,
      "activations/layer2_attention_weight_min": -31.259456634521484,
      "activations/layer3_attention_weight_max": 94.55292510986328,
      "activations/layer3_attention_weight_min": -93.6323013305664,
      "activations/layer4_attention_weight_max": 130.41268920898438,
      "activations/layer4_attention_weight_min": -126.57191467285156,
      "activations/layer5_attention_weight_max": 67.53500366210938,
      "activations/layer5_attention_weight_min": -67.8812255859375,
      "activations/layer6_attention_weight_max": 56.02638244628906,
      "activations/layer6_attention_weight_min": -57.49403762817383,
      "activations/layer7_attention_weight_max": 87.35159301757812,
      "activations/layer7_attention_weight_min": -90.18099212646484,
      "activations/layer8_attention_weight_max": 41.09043884277344,
      "activations/layer8_attention_weight_min": -46.2811279296875,
      "activations/layer9_attention_weight_max": 37.846736907958984,
      "activations/layer9_attention_weight_min": -35.549766540527344,
      "epoch": 6.5,
      "learning_rate": 0.00010916780303030303,
      "loss": 2.8213,
      "step": 111900
    },
    {
      "activations/layer0_attention_weight_max": 15.643077850341797,
      "activations/layer0_attention_weight_min": -11.97193431854248,
      "activations/layer10_attention_weight_max": 40.752540588378906,
      "activations/layer10_attention_weight_min": -36.17476272583008,
      "activations/layer11_attention_weight_max": 41.986061096191406,
      "activations/layer11_attention_weight_min": -32.37409973144531,
      "activations/layer12_attention_weight_max": 35.47231674194336,
      "activations/layer12_attention_weight_min": -30.017942428588867,
      "activations/layer13_attention_weight_max": 65.47357177734375,
      "activations/layer13_attention_weight_min": -42.948509216308594,
      "activations/layer14_attention_weight_max": 104.30331420898438,
      "activations/layer14_attention_weight_min": -61.58586883544922,
      "activations/layer15_attention_weight_max": 63.32838439941406,
      "activations/layer15_attention_weight_min": -32.52240753173828,
      "activations/layer16_attention_weight_max": 48.75979995727539,
      "activations/layer16_attention_weight_min": -29.50084114074707,
      "activations/layer17_attention_weight_max": 46.31618118286133,
      "activations/layer17_attention_weight_min": -25.815364837646484,
      "activations/layer18_attention_weight_max": 41.24984359741211,
      "activations/layer18_attention_weight_min": -24.0092830657959,
      "activations/layer19_attention_weight_max": 57.43499755859375,
      "activations/layer19_attention_weight_min": -31.163816452026367,
      "activations/layer1_attention_weight_max": 17.051424026489258,
      "activations/layer1_attention_weight_min": -14.463933944702148,
      "activations/layer20_attention_weight_max": 43.29066467285156,
      "activations/layer20_attention_weight_min": -25.183263778686523,
      "activations/layer21_attention_weight_max": 41.40614318847656,
      "activations/layer21_attention_weight_min": -21.20531463623047,
      "activations/layer22_attention_weight_max": 82.45289611816406,
      "activations/layer22_attention_weight_min": -30.470943450927734,
      "activations/layer23_attention_weight_max": 54.68927764892578,
      "activations/layer23_attention_weight_min": -22.008560180664062,
      "activations/layer2_attention_weight_max": 34.529300689697266,
      "activations/layer2_attention_weight_min": -32.466888427734375,
      "activations/layer3_attention_weight_max": 94.73776245117188,
      "activations/layer3_attention_weight_min": -94.37398529052734,
      "activations/layer4_attention_weight_max": 135.2017364501953,
      "activations/layer4_attention_weight_min": -130.85618591308594,
      "activations/layer5_attention_weight_max": 74.0194320678711,
      "activations/layer5_attention_weight_min": -70.01394653320312,
      "activations/layer6_attention_weight_max": 55.86976623535156,
      "activations/layer6_attention_weight_min": -56.25064468383789,
      "activations/layer7_attention_weight_max": 97.86115264892578,
      "activations/layer7_attention_weight_min": -91.85714721679688,
      "activations/layer8_attention_weight_max": 47.51522445678711,
      "activations/layer8_attention_weight_min": -43.82695388793945,
      "activations/layer9_attention_weight_max": 43.51435470581055,
      "activations/layer9_attention_weight_min": -37.83111572265625,
      "epoch": 6.5,
      "learning_rate": 0.00010914886363636362,
      "loss": 2.8375,
      "step": 111950
    },
    {
      "activations/layer0_attention_weight_max": 16.209739685058594,
      "activations/layer0_attention_weight_min": -12.676122665405273,
      "activations/layer10_attention_weight_max": 32.074527740478516,
      "activations/layer10_attention_weight_min": -33.254268646240234,
      "activations/layer11_attention_weight_max": 31.330856323242188,
      "activations/layer11_attention_weight_min": -29.892452239990234,
      "activations/layer12_attention_weight_max": 22.15077018737793,
      "activations/layer12_attention_weight_min": -29.772624969482422,
      "activations/layer13_attention_weight_max": 40.82355499267578,
      "activations/layer13_attention_weight_min": -34.80004119873047,
      "activations/layer14_attention_weight_max": 52.962276458740234,
      "activations/layer14_attention_weight_min": -43.96456527709961,
      "activations/layer15_attention_weight_max": 35.93913269042969,
      "activations/layer15_attention_weight_min": -29.879013061523438,
      "activations/layer16_attention_weight_max": 36.783809661865234,
      "activations/layer16_attention_weight_min": -30.024410247802734,
      "activations/layer17_attention_weight_max": 34.26589584350586,
      "activations/layer17_attention_weight_min": -26.396535873413086,
      "activations/layer18_attention_weight_max": 29.845260620117188,
      "activations/layer18_attention_weight_min": -20.313838958740234,
      "activations/layer19_attention_weight_max": 38.12986373901367,
      "activations/layer19_attention_weight_min": -30.880781173706055,
      "activations/layer1_attention_weight_max": 16.269397735595703,
      "activations/layer1_attention_weight_min": -14.176894187927246,
      "activations/layer20_attention_weight_max": 31.006736755371094,
      "activations/layer20_attention_weight_min": -25.974214553833008,
      "activations/layer21_attention_weight_max": 29.034099578857422,
      "activations/layer21_attention_weight_min": -23.479063034057617,
      "activations/layer22_attention_weight_max": 47.4920768737793,
      "activations/layer22_attention_weight_min": -29.74800682067871,
      "activations/layer23_attention_weight_max": 34.85311508178711,
      "activations/layer23_attention_weight_min": -24.43029022216797,
      "activations/layer2_attention_weight_max": 34.994468688964844,
      "activations/layer2_attention_weight_min": -32.84996032714844,
      "activations/layer3_attention_weight_max": 93.6893081665039,
      "activations/layer3_attention_weight_min": -96.67469024658203,
      "activations/layer4_attention_weight_max": 133.72354125976562,
      "activations/layer4_attention_weight_min": -124.58524322509766,
      "activations/layer5_attention_weight_max": 72.07008361816406,
      "activations/layer5_attention_weight_min": -69.6120376586914,
      "activations/layer6_attention_weight_max": 56.22819519042969,
      "activations/layer6_attention_weight_min": -54.52631378173828,
      "activations/layer7_attention_weight_max": 86.33556365966797,
      "activations/layer7_attention_weight_min": -85.75066375732422,
      "activations/layer8_attention_weight_max": 38.72603225708008,
      "activations/layer8_attention_weight_min": -42.27031326293945,
      "activations/layer9_attention_weight_max": 35.47631072998047,
      "activations/layer9_attention_weight_min": -36.655582427978516,
      "epoch": 6.51,
      "learning_rate": 0.00010912992424242423,
      "loss": 2.8216,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5036,
      "eval_samples_per_second": 504.965,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5036,
      "eval_openwebtext_samples_per_second": 504.965,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_wikitext_loss": 3.00390625,
      "eval_wikitext_ppl": 20.16414949200632,
      "eval_wikitext_runtime": 1.9713,
      "eval_wikitext_samples_per_second": 231.323,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_lambada_loss": 2.6484375,
      "eval_lambada_ppl": 14.131940228876227,
      "eval_lambada_runtime": 9.6098,
      "eval_lambada_samples_per_second": 506.668,
      "step": 112000
    },
    {
      "activations/layer0_attention_weight_max": 15.332428932189941,
      "activations/layer0_attention_weight_min": -11.436347961425781,
      "activations/layer10_attention_weight_max": 32.80889892578125,
      "activations/layer10_attention_weight_min": -32.18524932861328,
      "activations/layer11_attention_weight_max": 31.744144439697266,
      "activations/layer11_attention_weight_min": -30.211593627929688,
      "activations/layer12_attention_weight_max": 25.773746490478516,
      "activations/layer12_attention_weight_min": -28.840604782104492,
      "activations/layer13_attention_weight_max": 40.69908142089844,
      "activations/layer13_attention_weight_min": -32.544708251953125,
      "activations/layer14_attention_weight_max": 54.244407653808594,
      "activations/layer14_attention_weight_min": -41.184017181396484,
      "activations/layer15_attention_weight_max": 35.975528717041016,
      "activations/layer15_attention_weight_min": -27.67196273803711,
      "activations/layer16_attention_weight_max": 34.81033706665039,
      "activations/layer16_attention_weight_min": -27.85546112060547,
      "activations/layer17_attention_weight_max": 35.40739822387695,
      "activations/layer17_attention_weight_min": -23.147382736206055,
      "activations/layer18_attention_weight_max": 30.383371353149414,
      "activations/layer18_attention_weight_min": -21.065242767333984,
      "activations/layer19_attention_weight_max": 35.49809646606445,
      "activations/layer19_attention_weight_min": -28.980627059936523,
      "activations/layer1_attention_weight_max": 16.949378967285156,
      "activations/layer1_attention_weight_min": -14.330048561096191,
      "activations/layer20_attention_weight_max": 33.53955841064453,
      "activations/layer20_attention_weight_min": -24.47030258178711,
      "activations/layer21_attention_weight_max": 30.995901107788086,
      "activations/layer21_attention_weight_min": -22.232559204101562,
      "activations/layer22_attention_weight_max": 50.76811599731445,
      "activations/layer22_attention_weight_min": -27.978933334350586,
      "activations/layer23_attention_weight_max": 32.99114990234375,
      "activations/layer23_attention_weight_min": -22.985397338867188,
      "activations/layer2_attention_weight_max": 32.14875030517578,
      "activations/layer2_attention_weight_min": -32.175682067871094,
      "activations/layer3_attention_weight_max": 90.6414794921875,
      "activations/layer3_attention_weight_min": -94.93478393554688,
      "activations/layer4_attention_weight_max": 126.36085510253906,
      "activations/layer4_attention_weight_min": -121.91704559326172,
      "activations/layer5_attention_weight_max": 68.21316528320312,
      "activations/layer5_attention_weight_min": -64.26792907714844,
      "activations/layer6_attention_weight_max": 50.93446350097656,
      "activations/layer6_attention_weight_min": -53.02058792114258,
      "activations/layer7_attention_weight_max": 84.18840026855469,
      "activations/layer7_attention_weight_min": -89.93163299560547,
      "activations/layer8_attention_weight_max": 39.69902420043945,
      "activations/layer8_attention_weight_min": -40.407161712646484,
      "activations/layer9_attention_weight_max": 35.38408660888672,
      "activations/layer9_attention_weight_min": -34.26102066040039,
      "epoch": 6.51,
      "learning_rate": 0.00010911098484848485,
      "loss": 2.822,
      "step": 112050
    },
    {
      "activations/layer0_attention_weight_max": 16.12388038635254,
      "activations/layer0_attention_weight_min": -12.361794471740723,
      "activations/layer10_attention_weight_max": 31.40655517578125,
      "activations/layer10_attention_weight_min": -32.64783477783203,
      "activations/layer11_attention_weight_max": 32.228607177734375,
      "activations/layer11_attention_weight_min": -31.24148178100586,
      "activations/layer12_attention_weight_max": 27.65737533569336,
      "activations/layer12_attention_weight_min": -24.940214157104492,
      "activations/layer13_attention_weight_max": 40.901344299316406,
      "activations/layer13_attention_weight_min": -32.109012603759766,
      "activations/layer14_attention_weight_max": 53.28934860229492,
      "activations/layer14_attention_weight_min": -39.666255950927734,
      "activations/layer15_attention_weight_max": 36.247066497802734,
      "activations/layer15_attention_weight_min": -28.86543083190918,
      "activations/layer16_attention_weight_max": 38.270694732666016,
      "activations/layer16_attention_weight_min": -26.804222106933594,
      "activations/layer17_attention_weight_max": 38.53137969970703,
      "activations/layer17_attention_weight_min": -24.254323959350586,
      "activations/layer18_attention_weight_max": 35.80105209350586,
      "activations/layer18_attention_weight_min": -21.800912857055664,
      "activations/layer19_attention_weight_max": 38.506771087646484,
      "activations/layer19_attention_weight_min": -29.326078414916992,
      "activations/layer1_attention_weight_max": 16.63419532775879,
      "activations/layer1_attention_weight_min": -13.534404754638672,
      "activations/layer20_attention_weight_max": 37.51048278808594,
      "activations/layer20_attention_weight_min": -24.039932250976562,
      "activations/layer21_attention_weight_max": 33.586334228515625,
      "activations/layer21_attention_weight_min": -21.979299545288086,
      "activations/layer22_attention_weight_max": 50.82307434082031,
      "activations/layer22_attention_weight_min": -29.970455169677734,
      "activations/layer23_attention_weight_max": 35.31950759887695,
      "activations/layer23_attention_weight_min": -23.279987335205078,
      "activations/layer2_attention_weight_max": 29.213214874267578,
      "activations/layer2_attention_weight_min": -28.199443817138672,
      "activations/layer3_attention_weight_max": 86.87340545654297,
      "activations/layer3_attention_weight_min": -85.053466796875,
      "activations/layer4_attention_weight_max": 118.63142395019531,
      "activations/layer4_attention_weight_min": -118.839599609375,
      "activations/layer5_attention_weight_max": 65.48306274414062,
      "activations/layer5_attention_weight_min": -62.95665740966797,
      "activations/layer6_attention_weight_max": 49.25849914550781,
      "activations/layer6_attention_weight_min": -52.604331970214844,
      "activations/layer7_attention_weight_max": 79.78128051757812,
      "activations/layer7_attention_weight_min": -79.47409057617188,
      "activations/layer8_attention_weight_max": 37.259403228759766,
      "activations/layer8_attention_weight_min": -39.20884323120117,
      "activations/layer9_attention_weight_max": 33.263065338134766,
      "activations/layer9_attention_weight_min": -32.98413848876953,
      "epoch": 6.51,
      "learning_rate": 0.00010909204545454545,
      "loss": 2.8225,
      "step": 112100
    },
    {
      "activations/layer0_attention_weight_max": 16.273513793945312,
      "activations/layer0_attention_weight_min": -12.242545127868652,
      "activations/layer10_attention_weight_max": 35.373199462890625,
      "activations/layer10_attention_weight_min": -30.280895233154297,
      "activations/layer11_attention_weight_max": 32.8698844909668,
      "activations/layer11_attention_weight_min": -28.788352966308594,
      "activations/layer12_attention_weight_max": 29.473913192749023,
      "activations/layer12_attention_weight_min": -28.31987762451172,
      "activations/layer13_attention_weight_max": 40.75108337402344,
      "activations/layer13_attention_weight_min": -33.28062438964844,
      "activations/layer14_attention_weight_max": 49.31391906738281,
      "activations/layer14_attention_weight_min": -40.114898681640625,
      "activations/layer15_attention_weight_max": 35.40546417236328,
      "activations/layer15_attention_weight_min": -27.515350341796875,
      "activations/layer16_attention_weight_max": 33.11931228637695,
      "activations/layer16_attention_weight_min": -27.251468658447266,
      "activations/layer17_attention_weight_max": 34.1862678527832,
      "activations/layer17_attention_weight_min": -24.596681594848633,
      "activations/layer18_attention_weight_max": 34.294918060302734,
      "activations/layer18_attention_weight_min": -22.597036361694336,
      "activations/layer19_attention_weight_max": 37.17034149169922,
      "activations/layer19_attention_weight_min": -30.22525405883789,
      "activations/layer1_attention_weight_max": 18.229990005493164,
      "activations/layer1_attention_weight_min": -14.0250825881958,
      "activations/layer20_attention_weight_max": 29.539039611816406,
      "activations/layer20_attention_weight_min": -22.778364181518555,
      "activations/layer21_attention_weight_max": 32.544960021972656,
      "activations/layer21_attention_weight_min": -21.538000106811523,
      "activations/layer22_attention_weight_max": 47.978240966796875,
      "activations/layer22_attention_weight_min": -29.841718673706055,
      "activations/layer23_attention_weight_max": 35.48362731933594,
      "activations/layer23_attention_weight_min": -23.28325653076172,
      "activations/layer2_attention_weight_max": 31.29199981689453,
      "activations/layer2_attention_weight_min": -30.02904510498047,
      "activations/layer3_attention_weight_max": 92.25122833251953,
      "activations/layer3_attention_weight_min": -92.93687438964844,
      "activations/layer4_attention_weight_max": 127.3995132446289,
      "activations/layer4_attention_weight_min": -119.23674774169922,
      "activations/layer5_attention_weight_max": 68.94197082519531,
      "activations/layer5_attention_weight_min": -66.29765319824219,
      "activations/layer6_attention_weight_max": 53.48488235473633,
      "activations/layer6_attention_weight_min": -55.65993118286133,
      "activations/layer7_attention_weight_max": 88.34336853027344,
      "activations/layer7_attention_weight_min": -88.41886901855469,
      "activations/layer8_attention_weight_max": 41.829925537109375,
      "activations/layer8_attention_weight_min": -41.995086669921875,
      "activations/layer9_attention_weight_max": 37.52396011352539,
      "activations/layer9_attention_weight_min": -34.636661529541016,
      "epoch": 6.52,
      "learning_rate": 0.00010907310606060605,
      "loss": 2.8419,
      "step": 112150
    },
    {
      "activations/layer0_attention_weight_max": 16.08382225036621,
      "activations/layer0_attention_weight_min": -12.596994400024414,
      "activations/layer10_attention_weight_max": 30.190250396728516,
      "activations/layer10_attention_weight_min": -30.855186462402344,
      "activations/layer11_attention_weight_max": 29.612625122070312,
      "activations/layer11_attention_weight_min": -27.632986068725586,
      "activations/layer12_attention_weight_max": 22.834924697875977,
      "activations/layer12_attention_weight_min": -28.595518112182617,
      "activations/layer13_attention_weight_max": 40.814231872558594,
      "activations/layer13_attention_weight_min": -31.74913787841797,
      "activations/layer14_attention_weight_max": 55.25868606567383,
      "activations/layer14_attention_weight_min": -43.65562057495117,
      "activations/layer15_attention_weight_max": 34.10489273071289,
      "activations/layer15_attention_weight_min": -28.12068748474121,
      "activations/layer16_attention_weight_max": 35.951663970947266,
      "activations/layer16_attention_weight_min": -29.43819236755371,
      "activations/layer17_attention_weight_max": 34.00157165527344,
      "activations/layer17_attention_weight_min": -26.767961502075195,
      "activations/layer18_attention_weight_max": 31.041166305541992,
      "activations/layer18_attention_weight_min": -22.689956665039062,
      "activations/layer19_attention_weight_max": 38.92811584472656,
      "activations/layer19_attention_weight_min": -30.07989501953125,
      "activations/layer1_attention_weight_max": 16.26270866394043,
      "activations/layer1_attention_weight_min": -15.147878646850586,
      "activations/layer20_attention_weight_max": 31.22637176513672,
      "activations/layer20_attention_weight_min": -24.36916732788086,
      "activations/layer21_attention_weight_max": 35.870765686035156,
      "activations/layer21_attention_weight_min": -24.17430305480957,
      "activations/layer22_attention_weight_max": 50.20930862426758,
      "activations/layer22_attention_weight_min": -30.009525299072266,
      "activations/layer23_attention_weight_max": 35.610008239746094,
      "activations/layer23_attention_weight_min": -23.466899871826172,
      "activations/layer2_attention_weight_max": 31.63599967956543,
      "activations/layer2_attention_weight_min": -31.45180320739746,
      "activations/layer3_attention_weight_max": 92.56590270996094,
      "activations/layer3_attention_weight_min": -98.00405883789062,
      "activations/layer4_attention_weight_max": 129.5847930908203,
      "activations/layer4_attention_weight_min": -133.9349822998047,
      "activations/layer5_attention_weight_max": 67.86616516113281,
      "activations/layer5_attention_weight_min": -68.64659118652344,
      "activations/layer6_attention_weight_max": 52.0135612487793,
      "activations/layer6_attention_weight_min": -55.57052993774414,
      "activations/layer7_attention_weight_max": 86.59642791748047,
      "activations/layer7_attention_weight_min": -85.64632415771484,
      "activations/layer8_attention_weight_max": 39.36355972290039,
      "activations/layer8_attention_weight_min": -38.391151428222656,
      "activations/layer9_attention_weight_max": 33.4720344543457,
      "activations/layer9_attention_weight_min": -33.79356002807617,
      "epoch": 6.52,
      "learning_rate": 0.00010905416666666666,
      "loss": 2.824,
      "step": 112200
    },
    {
      "activations/layer0_attention_weight_max": 16.175474166870117,
      "activations/layer0_attention_weight_min": -11.018352508544922,
      "activations/layer10_attention_weight_max": 33.03839874267578,
      "activations/layer10_attention_weight_min": -32.152130126953125,
      "activations/layer11_attention_weight_max": 31.986732482910156,
      "activations/layer11_attention_weight_min": -29.437137603759766,
      "activations/layer12_attention_weight_max": 23.396825790405273,
      "activations/layer12_attention_weight_min": -25.527177810668945,
      "activations/layer13_attention_weight_max": 43.60820007324219,
      "activations/layer13_attention_weight_min": -34.998207092285156,
      "activations/layer14_attention_weight_max": 57.23956298828125,
      "activations/layer14_attention_weight_min": -46.10160446166992,
      "activations/layer15_attention_weight_max": 37.26858139038086,
      "activations/layer15_attention_weight_min": -29.334150314331055,
      "activations/layer16_attention_weight_max": 38.574554443359375,
      "activations/layer16_attention_weight_min": -29.3904972076416,
      "activations/layer17_attention_weight_max": 39.08030319213867,
      "activations/layer17_attention_weight_min": -26.948829650878906,
      "activations/layer18_attention_weight_max": 30.270071029663086,
      "activations/layer18_attention_weight_min": -22.082609176635742,
      "activations/layer19_attention_weight_max": 38.651058197021484,
      "activations/layer19_attention_weight_min": -28.924901962280273,
      "activations/layer1_attention_weight_max": 16.262449264526367,
      "activations/layer1_attention_weight_min": -14.161543846130371,
      "activations/layer20_attention_weight_max": 30.872146606445312,
      "activations/layer20_attention_weight_min": -23.33258628845215,
      "activations/layer21_attention_weight_max": 31.086776733398438,
      "activations/layer21_attention_weight_min": -21.130468368530273,
      "activations/layer22_attention_weight_max": 51.4402961730957,
      "activations/layer22_attention_weight_min": -27.734617233276367,
      "activations/layer23_attention_weight_max": 38.164791107177734,
      "activations/layer23_attention_weight_min": -22.428722381591797,
      "activations/layer2_attention_weight_max": 31.81549072265625,
      "activations/layer2_attention_weight_min": -31.44363021850586,
      "activations/layer3_attention_weight_max": 89.0207290649414,
      "activations/layer3_attention_weight_min": -91.14522552490234,
      "activations/layer4_attention_weight_max": 122.52520751953125,
      "activations/layer4_attention_weight_min": -127.12657928466797,
      "activations/layer5_attention_weight_max": 67.29811096191406,
      "activations/layer5_attention_weight_min": -65.30398559570312,
      "activations/layer6_attention_weight_max": 51.43657684326172,
      "activations/layer6_attention_weight_min": -55.491302490234375,
      "activations/layer7_attention_weight_max": 84.81852722167969,
      "activations/layer7_attention_weight_min": -89.51253509521484,
      "activations/layer8_attention_weight_max": 38.62288284301758,
      "activations/layer8_attention_weight_min": -43.250823974609375,
      "activations/layer9_attention_weight_max": 35.6661262512207,
      "activations/layer9_attention_weight_min": -34.58308029174805,
      "epoch": 6.52,
      "learning_rate": 0.00010903522727272726,
      "loss": 2.8489,
      "step": 112250
    },
    {
      "activations/layer0_attention_weight_max": 16.015056610107422,
      "activations/layer0_attention_weight_min": -12.234139442443848,
      "activations/layer10_attention_weight_max": 35.79735565185547,
      "activations/layer10_attention_weight_min": -32.037391662597656,
      "activations/layer11_attention_weight_max": 37.79597473144531,
      "activations/layer11_attention_weight_min": -31.226652145385742,
      "activations/layer12_attention_weight_max": 27.2755069732666,
      "activations/layer12_attention_weight_min": -26.447410583496094,
      "activations/layer13_attention_weight_max": 57.82450866699219,
      "activations/layer13_attention_weight_min": -36.4989128112793,
      "activations/layer14_attention_weight_max": 58.203487396240234,
      "activations/layer14_attention_weight_min": -45.421165466308594,
      "activations/layer15_attention_weight_max": 51.72036361694336,
      "activations/layer15_attention_weight_min": -33.04726028442383,
      "activations/layer16_attention_weight_max": 37.53089904785156,
      "activations/layer16_attention_weight_min": -28.678485870361328,
      "activations/layer17_attention_weight_max": 36.52825927734375,
      "activations/layer17_attention_weight_min": -26.635578155517578,
      "activations/layer18_attention_weight_max": 35.05376052856445,
      "activations/layer18_attention_weight_min": -22.0854434967041,
      "activations/layer19_attention_weight_max": 39.16871643066406,
      "activations/layer19_attention_weight_min": -30.035358428955078,
      "activations/layer1_attention_weight_max": 17.047698974609375,
      "activations/layer1_attention_weight_min": -14.304062843322754,
      "activations/layer20_attention_weight_max": 34.43244934082031,
      "activations/layer20_attention_weight_min": -24.349855422973633,
      "activations/layer21_attention_weight_max": 34.16828536987305,
      "activations/layer21_attention_weight_min": -22.861412048339844,
      "activations/layer22_attention_weight_max": 57.033905029296875,
      "activations/layer22_attention_weight_min": -28.997785568237305,
      "activations/layer23_attention_weight_max": 35.10416793823242,
      "activations/layer23_attention_weight_min": -19.640859603881836,
      "activations/layer2_attention_weight_max": 33.36640930175781,
      "activations/layer2_attention_weight_min": -30.456905364990234,
      "activations/layer3_attention_weight_max": 96.70574188232422,
      "activations/layer3_attention_weight_min": -95.29806518554688,
      "activations/layer4_attention_weight_max": 129.0665740966797,
      "activations/layer4_attention_weight_min": -125.30763244628906,
      "activations/layer5_attention_weight_max": 69.87382507324219,
      "activations/layer5_attention_weight_min": -66.93724060058594,
      "activations/layer6_attention_weight_max": 54.405277252197266,
      "activations/layer6_attention_weight_min": -55.148841857910156,
      "activations/layer7_attention_weight_max": 88.16632080078125,
      "activations/layer7_attention_weight_min": -89.86983489990234,
      "activations/layer8_attention_weight_max": 42.78740310668945,
      "activations/layer8_attention_weight_min": -39.72772979736328,
      "activations/layer9_attention_weight_max": 38.401920318603516,
      "activations/layer9_attention_weight_min": -38.097713470458984,
      "epoch": 6.53,
      "learning_rate": 0.00010901628787878787,
      "loss": 2.8346,
      "step": 112300
    },
    {
      "activations/layer0_attention_weight_max": 16.574695587158203,
      "activations/layer0_attention_weight_min": -12.203069686889648,
      "activations/layer10_attention_weight_max": 34.27594757080078,
      "activations/layer10_attention_weight_min": -32.51909255981445,
      "activations/layer11_attention_weight_max": 35.28813171386719,
      "activations/layer11_attention_weight_min": -31.92843246459961,
      "activations/layer12_attention_weight_max": 25.3162899017334,
      "activations/layer12_attention_weight_min": -24.305538177490234,
      "activations/layer13_attention_weight_max": 45.689735412597656,
      "activations/layer13_attention_weight_min": -34.49583053588867,
      "activations/layer14_attention_weight_max": 53.50740051269531,
      "activations/layer14_attention_weight_min": -39.96004104614258,
      "activations/layer15_attention_weight_max": 36.04237365722656,
      "activations/layer15_attention_weight_min": -28.57857322692871,
      "activations/layer16_attention_weight_max": 35.82981872558594,
      "activations/layer16_attention_weight_min": -27.31869888305664,
      "activations/layer17_attention_weight_max": 35.088401794433594,
      "activations/layer17_attention_weight_min": -23.937904357910156,
      "activations/layer18_attention_weight_max": 31.743608474731445,
      "activations/layer18_attention_weight_min": -20.448898315429688,
      "activations/layer19_attention_weight_max": 41.0987663269043,
      "activations/layer19_attention_weight_min": -29.762022018432617,
      "activations/layer1_attention_weight_max": 16.465757369995117,
      "activations/layer1_attention_weight_min": -14.5735502243042,
      "activations/layer20_attention_weight_max": 31.593233108520508,
      "activations/layer20_attention_weight_min": -23.0380916595459,
      "activations/layer21_attention_weight_max": 31.396512985229492,
      "activations/layer21_attention_weight_min": -21.363309860229492,
      "activations/layer22_attention_weight_max": 58.78507614135742,
      "activations/layer22_attention_weight_min": -29.406341552734375,
      "activations/layer23_attention_weight_max": 40.62664031982422,
      "activations/layer23_attention_weight_min": -22.875905990600586,
      "activations/layer2_attention_weight_max": 32.06171417236328,
      "activations/layer2_attention_weight_min": -32.29445266723633,
      "activations/layer3_attention_weight_max": 95.02412414550781,
      "activations/layer3_attention_weight_min": -96.79510498046875,
      "activations/layer4_attention_weight_max": 137.38902282714844,
      "activations/layer4_attention_weight_min": -130.02883911132812,
      "activations/layer5_attention_weight_max": 71.20687866210938,
      "activations/layer5_attention_weight_min": -69.03954315185547,
      "activations/layer6_attention_weight_max": 56.03197479248047,
      "activations/layer6_attention_weight_min": -56.29445266723633,
      "activations/layer7_attention_weight_max": 91.73851013183594,
      "activations/layer7_attention_weight_min": -93.93666076660156,
      "activations/layer8_attention_weight_max": 41.01294708251953,
      "activations/layer8_attention_weight_min": -42.71295166015625,
      "activations/layer9_attention_weight_max": 37.983360290527344,
      "activations/layer9_attention_weight_min": -35.880489349365234,
      "epoch": 6.53,
      "learning_rate": 0.00010899734848484847,
      "loss": 2.8181,
      "step": 112350
    },
    {
      "activations/layer0_attention_weight_max": 14.074042320251465,
      "activations/layer0_attention_weight_min": -11.824191093444824,
      "activations/layer10_attention_weight_max": 41.37176513671875,
      "activations/layer10_attention_weight_min": -34.61796188354492,
      "activations/layer11_attention_weight_max": 46.26399230957031,
      "activations/layer11_attention_weight_min": -31.509153366088867,
      "activations/layer12_attention_weight_max": 39.0312614440918,
      "activations/layer12_attention_weight_min": -27.197324752807617,
      "activations/layer13_attention_weight_max": 46.98938751220703,
      "activations/layer13_attention_weight_min": -31.087759017944336,
      "activations/layer14_attention_weight_max": 65.21531677246094,
      "activations/layer14_attention_weight_min": -38.954811096191406,
      "activations/layer15_attention_weight_max": 41.33620071411133,
      "activations/layer15_attention_weight_min": -30.20655059814453,
      "activations/layer16_attention_weight_max": 42.30984878540039,
      "activations/layer16_attention_weight_min": -27.32862663269043,
      "activations/layer17_attention_weight_max": 34.336021423339844,
      "activations/layer17_attention_weight_min": -24.788982391357422,
      "activations/layer18_attention_weight_max": 30.370502471923828,
      "activations/layer18_attention_weight_min": -21.97305679321289,
      "activations/layer19_attention_weight_max": 44.761348724365234,
      "activations/layer19_attention_weight_min": -30.167348861694336,
      "activations/layer1_attention_weight_max": 18.421646118164062,
      "activations/layer1_attention_weight_min": -14.987378120422363,
      "activations/layer20_attention_weight_max": 33.92142105102539,
      "activations/layer20_attention_weight_min": -25.656593322753906,
      "activations/layer21_attention_weight_max": 31.873760223388672,
      "activations/layer21_attention_weight_min": -21.763221740722656,
      "activations/layer22_attention_weight_max": 49.207271575927734,
      "activations/layer22_attention_weight_min": -27.539440155029297,
      "activations/layer23_attention_weight_max": 34.053680419921875,
      "activations/layer23_attention_weight_min": -22.068805694580078,
      "activations/layer2_attention_weight_max": 32.49156188964844,
      "activations/layer2_attention_weight_min": -31.270198822021484,
      "activations/layer3_attention_weight_max": 88.07049560546875,
      "activations/layer3_attention_weight_min": -85.88233947753906,
      "activations/layer4_attention_weight_max": 124.15032196044922,
      "activations/layer4_attention_weight_min": -121.18436431884766,
      "activations/layer5_attention_weight_max": 68.50993347167969,
      "activations/layer5_attention_weight_min": -66.48731231689453,
      "activations/layer6_attention_weight_max": 52.4342155456543,
      "activations/layer6_attention_weight_min": -52.21084976196289,
      "activations/layer7_attention_weight_max": 95.49321746826172,
      "activations/layer7_attention_weight_min": -87.17969512939453,
      "activations/layer8_attention_weight_max": 43.697540283203125,
      "activations/layer8_attention_weight_min": -43.94105911254883,
      "activations/layer9_attention_weight_max": 40.39951705932617,
      "activations/layer9_attention_weight_min": -37.59202194213867,
      "epoch": 6.53,
      "learning_rate": 0.00010897840909090908,
      "loss": 2.8123,
      "step": 112400
    },
    {
      "activations/layer0_attention_weight_max": 15.653164863586426,
      "activations/layer0_attention_weight_min": -11.67347240447998,
      "activations/layer10_attention_weight_max": 37.3304328918457,
      "activations/layer10_attention_weight_min": -30.238304138183594,
      "activations/layer11_attention_weight_max": 38.8456916809082,
      "activations/layer11_attention_weight_min": -30.243253707885742,
      "activations/layer12_attention_weight_max": 25.080081939697266,
      "activations/layer12_attention_weight_min": -28.099489212036133,
      "activations/layer13_attention_weight_max": 45.259212493896484,
      "activations/layer13_attention_weight_min": -33.777400970458984,
      "activations/layer14_attention_weight_max": 64.79754638671875,
      "activations/layer14_attention_weight_min": -44.46586227416992,
      "activations/layer15_attention_weight_max": 38.281681060791016,
      "activations/layer15_attention_weight_min": -30.168787002563477,
      "activations/layer16_attention_weight_max": 36.49855422973633,
      "activations/layer16_attention_weight_min": -27.960391998291016,
      "activations/layer17_attention_weight_max": 36.90547180175781,
      "activations/layer17_attention_weight_min": -26.004486083984375,
      "activations/layer18_attention_weight_max": 33.42387771606445,
      "activations/layer18_attention_weight_min": -23.449825286865234,
      "activations/layer19_attention_weight_max": 45.32229995727539,
      "activations/layer19_attention_weight_min": -34.548484802246094,
      "activations/layer1_attention_weight_max": 15.785512924194336,
      "activations/layer1_attention_weight_min": -13.541624069213867,
      "activations/layer20_attention_weight_max": 33.38374710083008,
      "activations/layer20_attention_weight_min": -24.29530906677246,
      "activations/layer21_attention_weight_max": 37.19199752807617,
      "activations/layer21_attention_weight_min": -23.498600006103516,
      "activations/layer22_attention_weight_max": 64.81201171875,
      "activations/layer22_attention_weight_min": -30.83149528503418,
      "activations/layer23_attention_weight_max": 39.457740783691406,
      "activations/layer23_attention_weight_min": -22.909488677978516,
      "activations/layer2_attention_weight_max": 31.348867416381836,
      "activations/layer2_attention_weight_min": -30.451663970947266,
      "activations/layer3_attention_weight_max": 85.72173309326172,
      "activations/layer3_attention_weight_min": -88.46650695800781,
      "activations/layer4_attention_weight_max": 122.14886474609375,
      "activations/layer4_attention_weight_min": -127.7093734741211,
      "activations/layer5_attention_weight_max": 63.064918518066406,
      "activations/layer5_attention_weight_min": -66.449951171875,
      "activations/layer6_attention_weight_max": 48.20640563964844,
      "activations/layer6_attention_weight_min": -53.788856506347656,
      "activations/layer7_attention_weight_max": 86.8069076538086,
      "activations/layer7_attention_weight_min": -89.48612976074219,
      "activations/layer8_attention_weight_max": 38.17304992675781,
      "activations/layer8_attention_weight_min": -37.909000396728516,
      "activations/layer9_attention_weight_max": 37.383575439453125,
      "activations/layer9_attention_weight_min": -34.18733215332031,
      "epoch": 6.53,
      "learning_rate": 0.0001089594696969697,
      "loss": 2.8375,
      "step": 112450
    },
    {
      "activations/layer0_attention_weight_max": 16.394699096679688,
      "activations/layer0_attention_weight_min": -11.227546691894531,
      "activations/layer10_attention_weight_max": 38.874813079833984,
      "activations/layer10_attention_weight_min": -34.351627349853516,
      "activations/layer11_attention_weight_max": 40.02492141723633,
      "activations/layer11_attention_weight_min": -29.953842163085938,
      "activations/layer12_attention_weight_max": 42.59218215942383,
      "activations/layer12_attention_weight_min": -29.857873916625977,
      "activations/layer13_attention_weight_max": 45.197635650634766,
      "activations/layer13_attention_weight_min": -32.81446075439453,
      "activations/layer14_attention_weight_max": 68.99246215820312,
      "activations/layer14_attention_weight_min": -44.07664108276367,
      "activations/layer15_attention_weight_max": 45.99937438964844,
      "activations/layer15_attention_weight_min": -31.78946304321289,
      "activations/layer16_attention_weight_max": 43.254051208496094,
      "activations/layer16_attention_weight_min": -28.621665954589844,
      "activations/layer17_attention_weight_max": 36.242435455322266,
      "activations/layer17_attention_weight_min": -25.444913864135742,
      "activations/layer18_attention_weight_max": 29.530696868896484,
      "activations/layer18_attention_weight_min": -21.070363998413086,
      "activations/layer19_attention_weight_max": 46.330745697021484,
      "activations/layer19_attention_weight_min": -29.728458404541016,
      "activations/layer1_attention_weight_max": 17.846647262573242,
      "activations/layer1_attention_weight_min": -14.171770095825195,
      "activations/layer20_attention_weight_max": 35.53677749633789,
      "activations/layer20_attention_weight_min": -24.051271438598633,
      "activations/layer21_attention_weight_max": 31.64927101135254,
      "activations/layer21_attention_weight_min": -21.119375228881836,
      "activations/layer22_attention_weight_max": 51.76815414428711,
      "activations/layer22_attention_weight_min": -28.322269439697266,
      "activations/layer23_attention_weight_max": 37.53394317626953,
      "activations/layer23_attention_weight_min": -21.222827911376953,
      "activations/layer2_attention_weight_max": 35.42213821411133,
      "activations/layer2_attention_weight_min": -33.122859954833984,
      "activations/layer3_attention_weight_max": 95.63035583496094,
      "activations/layer3_attention_weight_min": -97.63163757324219,
      "activations/layer4_attention_weight_max": 143.35133361816406,
      "activations/layer4_attention_weight_min": -132.8982391357422,
      "activations/layer5_attention_weight_max": 77.57563018798828,
      "activations/layer5_attention_weight_min": -69.27058410644531,
      "activations/layer6_attention_weight_max": 56.98117446899414,
      "activations/layer6_attention_weight_min": -56.64119338989258,
      "activations/layer7_attention_weight_max": 107.11270141601562,
      "activations/layer7_attention_weight_min": -92.9129409790039,
      "activations/layer8_attention_weight_max": 47.72041702270508,
      "activations/layer8_attention_weight_min": -44.671512603759766,
      "activations/layer9_attention_weight_max": 42.171199798583984,
      "activations/layer9_attention_weight_min": -36.201717376708984,
      "epoch": 6.54,
      "learning_rate": 0.00010894053030303029,
      "loss": 2.8239,
      "step": 112500
    },
    {
      "activations/layer0_attention_weight_max": 15.052743911743164,
      "activations/layer0_attention_weight_min": -12.180156707763672,
      "activations/layer10_attention_weight_max": 39.695037841796875,
      "activations/layer10_attention_weight_min": -36.063480377197266,
      "activations/layer11_attention_weight_max": 42.578426361083984,
      "activations/layer11_attention_weight_min": -31.900297164916992,
      "activations/layer12_attention_weight_max": 26.673179626464844,
      "activations/layer12_attention_weight_min": -24.567766189575195,
      "activations/layer13_attention_weight_max": 44.79740905761719,
      "activations/layer13_attention_weight_min": -32.88146209716797,
      "activations/layer14_attention_weight_max": 72.89213562011719,
      "activations/layer14_attention_weight_min": -45.42897415161133,
      "activations/layer15_attention_weight_max": 43.85795974731445,
      "activations/layer15_attention_weight_min": -29.15049171447754,
      "activations/layer16_attention_weight_max": 43.044036865234375,
      "activations/layer16_attention_weight_min": -27.818294525146484,
      "activations/layer17_attention_weight_max": 43.98652648925781,
      "activations/layer17_attention_weight_min": -24.38232421875,
      "activations/layer18_attention_weight_max": 38.45635223388672,
      "activations/layer18_attention_weight_min": -20.713180541992188,
      "activations/layer19_attention_weight_max": 48.32465744018555,
      "activations/layer19_attention_weight_min": -30.366092681884766,
      "activations/layer1_attention_weight_max": 17.571197509765625,
      "activations/layer1_attention_weight_min": -14.958498001098633,
      "activations/layer20_attention_weight_max": 39.64118576049805,
      "activations/layer20_attention_weight_min": -24.850446701049805,
      "activations/layer21_attention_weight_max": 35.37383270263672,
      "activations/layer21_attention_weight_min": -21.404876708984375,
      "activations/layer22_attention_weight_max": 64.26895141601562,
      "activations/layer22_attention_weight_min": -29.998058319091797,
      "activations/layer23_attention_weight_max": 45.7879638671875,
      "activations/layer23_attention_weight_min": -23.065685272216797,
      "activations/layer2_attention_weight_max": 35.494598388671875,
      "activations/layer2_attention_weight_min": -32.904327392578125,
      "activations/layer3_attention_weight_max": 89.48340606689453,
      "activations/layer3_attention_weight_min": -92.368408203125,
      "activations/layer4_attention_weight_max": 128.51339721679688,
      "activations/layer4_attention_weight_min": -126.21284484863281,
      "activations/layer5_attention_weight_max": 73.06167602539062,
      "activations/layer5_attention_weight_min": -66.28496551513672,
      "activations/layer6_attention_weight_max": 57.90858840942383,
      "activations/layer6_attention_weight_min": -53.918663024902344,
      "activations/layer7_attention_weight_max": 93.42225646972656,
      "activations/layer7_attention_weight_min": -91.86994171142578,
      "activations/layer8_attention_weight_max": 46.48441696166992,
      "activations/layer8_attention_weight_min": -43.35677719116211,
      "activations/layer9_attention_weight_max": 39.069095611572266,
      "activations/layer9_attention_weight_min": -36.5349235534668,
      "epoch": 6.54,
      "learning_rate": 0.0001089215909090909,
      "loss": 2.823,
      "step": 112550
    },
    {
      "activations/layer0_attention_weight_max": 17.31414222717285,
      "activations/layer0_attention_weight_min": -12.312455177307129,
      "activations/layer10_attention_weight_max": 34.559486389160156,
      "activations/layer10_attention_weight_min": -32.58049392700195,
      "activations/layer11_attention_weight_max": 37.35697937011719,
      "activations/layer11_attention_weight_min": -30.475833892822266,
      "activations/layer12_attention_weight_max": 32.503211975097656,
      "activations/layer12_attention_weight_min": -34.70330810546875,
      "activations/layer13_attention_weight_max": 58.678226470947266,
      "activations/layer13_attention_weight_min": -47.22844696044922,
      "activations/layer14_attention_weight_max": 64.4545669555664,
      "activations/layer14_attention_weight_min": -47.9597053527832,
      "activations/layer15_attention_weight_max": 47.42876052856445,
      "activations/layer15_attention_weight_min": -37.55244064331055,
      "activations/layer16_attention_weight_max": 35.228111267089844,
      "activations/layer16_attention_weight_min": -28.04908561706543,
      "activations/layer17_attention_weight_max": 36.288299560546875,
      "activations/layer17_attention_weight_min": -27.091394424438477,
      "activations/layer18_attention_weight_max": 32.374908447265625,
      "activations/layer18_attention_weight_min": -23.090116500854492,
      "activations/layer19_attention_weight_max": 35.59333419799805,
      "activations/layer19_attention_weight_min": -28.52862548828125,
      "activations/layer1_attention_weight_max": 18.78952407836914,
      "activations/layer1_attention_weight_min": -13.26661491394043,
      "activations/layer20_attention_weight_max": 30.665264129638672,
      "activations/layer20_attention_weight_min": -22.28369140625,
      "activations/layer21_attention_weight_max": 28.409122467041016,
      "activations/layer21_attention_weight_min": -20.86336326599121,
      "activations/layer22_attention_weight_max": 48.43177032470703,
      "activations/layer22_attention_weight_min": -27.99847412109375,
      "activations/layer23_attention_weight_max": 33.16545486450195,
      "activations/layer23_attention_weight_min": -21.53420639038086,
      "activations/layer2_attention_weight_max": 33.4857063293457,
      "activations/layer2_attention_weight_min": -31.180850982666016,
      "activations/layer3_attention_weight_max": 90.90130615234375,
      "activations/layer3_attention_weight_min": -91.47449493408203,
      "activations/layer4_attention_weight_max": 124.06520080566406,
      "activations/layer4_attention_weight_min": -115.98121643066406,
      "activations/layer5_attention_weight_max": 68.20620727539062,
      "activations/layer5_attention_weight_min": -61.31553649902344,
      "activations/layer6_attention_weight_max": 51.55024719238281,
      "activations/layer6_attention_weight_min": -52.562049865722656,
      "activations/layer7_attention_weight_max": 96.22752380371094,
      "activations/layer7_attention_weight_min": -89.71250915527344,
      "activations/layer8_attention_weight_max": 43.068321228027344,
      "activations/layer8_attention_weight_min": -40.79583740234375,
      "activations/layer9_attention_weight_max": 36.096588134765625,
      "activations/layer9_attention_weight_min": -42.73625946044922,
      "epoch": 6.54,
      "learning_rate": 0.00010890265151515152,
      "loss": 2.8234,
      "step": 112600
    },
    {
      "activations/layer0_attention_weight_max": 16.008930206298828,
      "activations/layer0_attention_weight_min": -12.79164981842041,
      "activations/layer10_attention_weight_max": 34.03697204589844,
      "activations/layer10_attention_weight_min": -31.96482276916504,
      "activations/layer11_attention_weight_max": 33.45139694213867,
      "activations/layer11_attention_weight_min": -30.057477951049805,
      "activations/layer12_attention_weight_max": 28.139583587646484,
      "activations/layer12_attention_weight_min": -26.333017349243164,
      "activations/layer13_attention_weight_max": 48.350032806396484,
      "activations/layer13_attention_weight_min": -35.56399154663086,
      "activations/layer14_attention_weight_max": 63.70101547241211,
      "activations/layer14_attention_weight_min": -44.06440734863281,
      "activations/layer15_attention_weight_max": 43.08060073852539,
      "activations/layer15_attention_weight_min": -28.17154312133789,
      "activations/layer16_attention_weight_max": 40.546791076660156,
      "activations/layer16_attention_weight_min": -28.665063858032227,
      "activations/layer17_attention_weight_max": 39.34422302246094,
      "activations/layer17_attention_weight_min": -23.81346321105957,
      "activations/layer18_attention_weight_max": 33.133758544921875,
      "activations/layer18_attention_weight_min": -21.05199432373047,
      "activations/layer19_attention_weight_max": 44.65390396118164,
      "activations/layer19_attention_weight_min": -30.027694702148438,
      "activations/layer1_attention_weight_max": 17.21621322631836,
      "activations/layer1_attention_weight_min": -14.616830825805664,
      "activations/layer20_attention_weight_max": 33.23198318481445,
      "activations/layer20_attention_weight_min": -24.7094669342041,
      "activations/layer21_attention_weight_max": 30.581315994262695,
      "activations/layer21_attention_weight_min": -22.49614715576172,
      "activations/layer22_attention_weight_max": 51.39485549926758,
      "activations/layer22_attention_weight_min": -27.81070899963379,
      "activations/layer23_attention_weight_max": 40.30766296386719,
      "activations/layer23_attention_weight_min": -23.418556213378906,
      "activations/layer2_attention_weight_max": 31.78253746032715,
      "activations/layer2_attention_weight_min": -29.778671264648438,
      "activations/layer3_attention_weight_max": 89.02656555175781,
      "activations/layer3_attention_weight_min": -92.5559310913086,
      "activations/layer4_attention_weight_max": 127.08956146240234,
      "activations/layer4_attention_weight_min": -123.2508316040039,
      "activations/layer5_attention_weight_max": 67.45539855957031,
      "activations/layer5_attention_weight_min": -65.26285552978516,
      "activations/layer6_attention_weight_max": 54.200992584228516,
      "activations/layer6_attention_weight_min": -52.270694732666016,
      "activations/layer7_attention_weight_max": 93.03103637695312,
      "activations/layer7_attention_weight_min": -84.23255920410156,
      "activations/layer8_attention_weight_max": 41.60600662231445,
      "activations/layer8_attention_weight_min": -41.390872955322266,
      "activations/layer9_attention_weight_max": 36.7974853515625,
      "activations/layer9_attention_weight_min": -33.90375518798828,
      "epoch": 6.55,
      "learning_rate": 0.0001088837121212121,
      "loss": 2.83,
      "step": 112650
    },
    {
      "activations/layer0_attention_weight_max": 16.862751007080078,
      "activations/layer0_attention_weight_min": -10.283452987670898,
      "activations/layer10_attention_weight_max": 37.35944366455078,
      "activations/layer10_attention_weight_min": -36.52013397216797,
      "activations/layer11_attention_weight_max": 37.505615234375,
      "activations/layer11_attention_weight_min": -31.93856430053711,
      "activations/layer12_attention_weight_max": 34.6518440246582,
      "activations/layer12_attention_weight_min": -27.620441436767578,
      "activations/layer13_attention_weight_max": 52.90583419799805,
      "activations/layer13_attention_weight_min": -34.891021728515625,
      "activations/layer14_attention_weight_max": 76.77296447753906,
      "activations/layer14_attention_weight_min": -48.47245788574219,
      "activations/layer15_attention_weight_max": 46.26106262207031,
      "activations/layer15_attention_weight_min": -30.001516342163086,
      "activations/layer16_attention_weight_max": 44.11025619506836,
      "activations/layer16_attention_weight_min": -28.47307586669922,
      "activations/layer17_attention_weight_max": 41.30815505981445,
      "activations/layer17_attention_weight_min": -25.6359806060791,
      "activations/layer18_attention_weight_max": 34.74189758300781,
      "activations/layer18_attention_weight_min": -22.131311416625977,
      "activations/layer19_attention_weight_max": 45.908966064453125,
      "activations/layer19_attention_weight_min": -29.121686935424805,
      "activations/layer1_attention_weight_max": 18.779809951782227,
      "activations/layer1_attention_weight_min": -14.756503105163574,
      "activations/layer20_attention_weight_max": 36.90833282470703,
      "activations/layer20_attention_weight_min": -22.969785690307617,
      "activations/layer21_attention_weight_max": 34.39768600463867,
      "activations/layer21_attention_weight_min": -21.024948120117188,
      "activations/layer22_attention_weight_max": 68.04346466064453,
      "activations/layer22_attention_weight_min": -28.215797424316406,
      "activations/layer23_attention_weight_max": 41.20298385620117,
      "activations/layer23_attention_weight_min": -20.31371307373047,
      "activations/layer2_attention_weight_max": 35.43902587890625,
      "activations/layer2_attention_weight_min": -32.05778503417969,
      "activations/layer3_attention_weight_max": 96.38924407958984,
      "activations/layer3_attention_weight_min": -95.36265563964844,
      "activations/layer4_attention_weight_max": 137.69488525390625,
      "activations/layer4_attention_weight_min": -134.1774139404297,
      "activations/layer5_attention_weight_max": 72.07103729248047,
      "activations/layer5_attention_weight_min": -69.20114135742188,
      "activations/layer6_attention_weight_max": 53.66716766357422,
      "activations/layer6_attention_weight_min": -54.826107025146484,
      "activations/layer7_attention_weight_max": 93.9139175415039,
      "activations/layer7_attention_weight_min": -93.36631774902344,
      "activations/layer8_attention_weight_max": 43.5654296875,
      "activations/layer8_attention_weight_min": -45.06319046020508,
      "activations/layer9_attention_weight_max": 39.8385124206543,
      "activations/layer9_attention_weight_min": -40.12293243408203,
      "epoch": 6.55,
      "learning_rate": 0.00010886477272727272,
      "loss": 2.8227,
      "step": 112700
    },
    {
      "activations/layer0_attention_weight_max": 16.025310516357422,
      "activations/layer0_attention_weight_min": -11.986677169799805,
      "activations/layer10_attention_weight_max": 33.97774887084961,
      "activations/layer10_attention_weight_min": -30.00180435180664,
      "activations/layer11_attention_weight_max": 29.31667709350586,
      "activations/layer11_attention_weight_min": -28.50143814086914,
      "activations/layer12_attention_weight_max": 25.40166664123535,
      "activations/layer12_attention_weight_min": -38.700740814208984,
      "activations/layer13_attention_weight_max": 38.924373626708984,
      "activations/layer13_attention_weight_min": -31.363475799560547,
      "activations/layer14_attention_weight_max": 47.39886474609375,
      "activations/layer14_attention_weight_min": -41.72251892089844,
      "activations/layer15_attention_weight_max": 38.339847564697266,
      "activations/layer15_attention_weight_min": -28.94679069519043,
      "activations/layer16_attention_weight_max": 34.60615158081055,
      "activations/layer16_attention_weight_min": -29.380783081054688,
      "activations/layer17_attention_weight_max": 38.494422912597656,
      "activations/layer17_attention_weight_min": -24.542865753173828,
      "activations/layer18_attention_weight_max": 33.67890548706055,
      "activations/layer18_attention_weight_min": -20.833948135375977,
      "activations/layer19_attention_weight_max": 42.230674743652344,
      "activations/layer19_attention_weight_min": -31.226886749267578,
      "activations/layer1_attention_weight_max": 17.897138595581055,
      "activations/layer1_attention_weight_min": -13.528329849243164,
      "activations/layer20_attention_weight_max": 31.223482131958008,
      "activations/layer20_attention_weight_min": -23.767915725708008,
      "activations/layer21_attention_weight_max": 33.7788200378418,
      "activations/layer21_attention_weight_min": -21.340429306030273,
      "activations/layer22_attention_weight_max": 52.127967834472656,
      "activations/layer22_attention_weight_min": -30.26224708557129,
      "activations/layer23_attention_weight_max": 40.8466682434082,
      "activations/layer23_attention_weight_min": -22.5942440032959,
      "activations/layer2_attention_weight_max": 32.770469665527344,
      "activations/layer2_attention_weight_min": -30.973365783691406,
      "activations/layer3_attention_weight_max": 94.44561767578125,
      "activations/layer3_attention_weight_min": -94.15969848632812,
      "activations/layer4_attention_weight_max": 125.26297760009766,
      "activations/layer4_attention_weight_min": -124.38304901123047,
      "activations/layer5_attention_weight_max": 66.14076232910156,
      "activations/layer5_attention_weight_min": -66.45339965820312,
      "activations/layer6_attention_weight_max": 55.54249572753906,
      "activations/layer6_attention_weight_min": -54.79979705810547,
      "activations/layer7_attention_weight_max": 87.1819076538086,
      "activations/layer7_attention_weight_min": -87.07249450683594,
      "activations/layer8_attention_weight_max": 39.0401611328125,
      "activations/layer8_attention_weight_min": -39.88682174682617,
      "activations/layer9_attention_weight_max": 36.562660217285156,
      "activations/layer9_attention_weight_min": -41.55440902709961,
      "epoch": 6.55,
      "learning_rate": 0.00010884583333333333,
      "loss": 2.8364,
      "step": 112750
    },
    {
      "activations/layer0_attention_weight_max": 15.68057632446289,
      "activations/layer0_attention_weight_min": -11.737775802612305,
      "activations/layer10_attention_weight_max": 38.292701721191406,
      "activations/layer10_attention_weight_min": -33.34790802001953,
      "activations/layer11_attention_weight_max": 34.62037658691406,
      "activations/layer11_attention_weight_min": -28.431982040405273,
      "activations/layer12_attention_weight_max": 29.160829544067383,
      "activations/layer12_attention_weight_min": -28.428804397583008,
      "activations/layer13_attention_weight_max": 47.20762634277344,
      "activations/layer13_attention_weight_min": -32.55061340332031,
      "activations/layer14_attention_weight_max": 61.5399284362793,
      "activations/layer14_attention_weight_min": -45.200523376464844,
      "activations/layer15_attention_weight_max": 41.08871841430664,
      "activations/layer15_attention_weight_min": -31.271461486816406,
      "activations/layer16_attention_weight_max": 39.11726379394531,
      "activations/layer16_attention_weight_min": -27.676719665527344,
      "activations/layer17_attention_weight_max": 40.374942779541016,
      "activations/layer17_attention_weight_min": -24.40692138671875,
      "activations/layer18_attention_weight_max": 36.86252975463867,
      "activations/layer18_attention_weight_min": -20.846349716186523,
      "activations/layer19_attention_weight_max": 45.45585632324219,
      "activations/layer19_attention_weight_min": -31.149953842163086,
      "activations/layer1_attention_weight_max": 17.685461044311523,
      "activations/layer1_attention_weight_min": -15.32170295715332,
      "activations/layer20_attention_weight_max": 37.10962677001953,
      "activations/layer20_attention_weight_min": -24.322385787963867,
      "activations/layer21_attention_weight_max": 37.77645492553711,
      "activations/layer21_attention_weight_min": -22.758747100830078,
      "activations/layer22_attention_weight_max": 57.32008361816406,
      "activations/layer22_attention_weight_min": -33.14107894897461,
      "activations/layer23_attention_weight_max": 38.6428337097168,
      "activations/layer23_attention_weight_min": -25.33238983154297,
      "activations/layer2_attention_weight_max": 32.858158111572266,
      "activations/layer2_attention_weight_min": -31.533010482788086,
      "activations/layer3_attention_weight_max": 90.541015625,
      "activations/layer3_attention_weight_min": -92.75507354736328,
      "activations/layer4_attention_weight_max": 123.96760559082031,
      "activations/layer4_attention_weight_min": -123.7535171508789,
      "activations/layer5_attention_weight_max": 70.05738830566406,
      "activations/layer5_attention_weight_min": -64.86381530761719,
      "activations/layer6_attention_weight_max": 52.62763595581055,
      "activations/layer6_attention_weight_min": -58.268760681152344,
      "activations/layer7_attention_weight_max": 88.15330505371094,
      "activations/layer7_attention_weight_min": -90.6857681274414,
      "activations/layer8_attention_weight_max": 41.85718536376953,
      "activations/layer8_attention_weight_min": -43.20976257324219,
      "activations/layer9_attention_weight_max": 37.109710693359375,
      "activations/layer9_attention_weight_min": -36.585693359375,
      "epoch": 6.55,
      "learning_rate": 0.00010882689393939392,
      "loss": 2.8316,
      "step": 112800
    },
    {
      "activations/layer0_attention_weight_max": 15.366352081298828,
      "activations/layer0_attention_weight_min": -11.089235305786133,
      "activations/layer10_attention_weight_max": 32.77597427368164,
      "activations/layer10_attention_weight_min": -32.460182189941406,
      "activations/layer11_attention_weight_max": 33.071136474609375,
      "activations/layer11_attention_weight_min": -29.76678466796875,
      "activations/layer12_attention_weight_max": 25.08160972595215,
      "activations/layer12_attention_weight_min": -25.210628509521484,
      "activations/layer13_attention_weight_max": 44.71426773071289,
      "activations/layer13_attention_weight_min": -31.893712997436523,
      "activations/layer14_attention_weight_max": 49.399723052978516,
      "activations/layer14_attention_weight_min": -40.88777160644531,
      "activations/layer15_attention_weight_max": 38.10673141479492,
      "activations/layer15_attention_weight_min": -29.52069091796875,
      "activations/layer16_attention_weight_max": 35.50182342529297,
      "activations/layer16_attention_weight_min": -30.327543258666992,
      "activations/layer17_attention_weight_max": 36.42044448852539,
      "activations/layer17_attention_weight_min": -26.249807357788086,
      "activations/layer18_attention_weight_max": 31.34516716003418,
      "activations/layer18_attention_weight_min": -20.20118522644043,
      "activations/layer19_attention_weight_max": 37.86367416381836,
      "activations/layer19_attention_weight_min": -32.2118034362793,
      "activations/layer1_attention_weight_max": 18.051877975463867,
      "activations/layer1_attention_weight_min": -14.96047592163086,
      "activations/layer20_attention_weight_max": 30.539249420166016,
      "activations/layer20_attention_weight_min": -25.236711502075195,
      "activations/layer21_attention_weight_max": 32.0114860534668,
      "activations/layer21_attention_weight_min": -23.973705291748047,
      "activations/layer22_attention_weight_max": 50.42950439453125,
      "activations/layer22_attention_weight_min": -28.71909523010254,
      "activations/layer23_attention_weight_max": 37.89318084716797,
      "activations/layer23_attention_weight_min": -23.406795501708984,
      "activations/layer2_attention_weight_max": 31.95766830444336,
      "activations/layer2_attention_weight_min": -31.197378158569336,
      "activations/layer3_attention_weight_max": 94.25516510009766,
      "activations/layer3_attention_weight_min": -90.037109375,
      "activations/layer4_attention_weight_max": 132.12229919433594,
      "activations/layer4_attention_weight_min": -125.05606842041016,
      "activations/layer5_attention_weight_max": 70.54051971435547,
      "activations/layer5_attention_weight_min": -66.84420776367188,
      "activations/layer6_attention_weight_max": 52.081825256347656,
      "activations/layer6_attention_weight_min": -53.28501892089844,
      "activations/layer7_attention_weight_max": 89.74047088623047,
      "activations/layer7_attention_weight_min": -84.01322937011719,
      "activations/layer8_attention_weight_max": 39.21133804321289,
      "activations/layer8_attention_weight_min": -39.42453384399414,
      "activations/layer9_attention_weight_max": 35.848388671875,
      "activations/layer9_attention_weight_min": -33.85236358642578,
      "epoch": 6.56,
      "learning_rate": 0.00010880795454545454,
      "loss": 2.8502,
      "step": 112850
    },
    {
      "activations/layer0_attention_weight_max": 15.15868091583252,
      "activations/layer0_attention_weight_min": -12.257607460021973,
      "activations/layer10_attention_weight_max": 33.2620849609375,
      "activations/layer10_attention_weight_min": -31.635873794555664,
      "activations/layer11_attention_weight_max": 34.513084411621094,
      "activations/layer11_attention_weight_min": -30.267601013183594,
      "activations/layer12_attention_weight_max": 24.571598052978516,
      "activations/layer12_attention_weight_min": -27.36390495300293,
      "activations/layer13_attention_weight_max": 44.68083572387695,
      "activations/layer13_attention_weight_min": -33.442527770996094,
      "activations/layer14_attention_weight_max": 58.51129150390625,
      "activations/layer14_attention_weight_min": -44.37473678588867,
      "activations/layer15_attention_weight_max": 39.739498138427734,
      "activations/layer15_attention_weight_min": -29.07111358642578,
      "activations/layer16_attention_weight_max": 35.58805465698242,
      "activations/layer16_attention_weight_min": -27.073997497558594,
      "activations/layer17_attention_weight_max": 40.16520309448242,
      "activations/layer17_attention_weight_min": -23.98314666748047,
      "activations/layer18_attention_weight_max": 30.12081527709961,
      "activations/layer18_attention_weight_min": -20.0479793548584,
      "activations/layer19_attention_weight_max": 36.32722473144531,
      "activations/layer19_attention_weight_min": -29.564565658569336,
      "activations/layer1_attention_weight_max": 16.598604202270508,
      "activations/layer1_attention_weight_min": -13.539580345153809,
      "activations/layer20_attention_weight_max": 30.595468521118164,
      "activations/layer20_attention_weight_min": -23.084611892700195,
      "activations/layer21_attention_weight_max": 31.41874122619629,
      "activations/layer21_attention_weight_min": -23.21987533569336,
      "activations/layer22_attention_weight_max": 54.78040313720703,
      "activations/layer22_attention_weight_min": -28.117347717285156,
      "activations/layer23_attention_weight_max": 37.498878479003906,
      "activations/layer23_attention_weight_min": -23.816360473632812,
      "activations/layer2_attention_weight_max": 34.58009338378906,
      "activations/layer2_attention_weight_min": -30.796741485595703,
      "activations/layer3_attention_weight_max": 90.7411880493164,
      "activations/layer3_attention_weight_min": -93.29784393310547,
      "activations/layer4_attention_weight_max": 127.86144256591797,
      "activations/layer4_attention_weight_min": -135.38380432128906,
      "activations/layer5_attention_weight_max": 70.8124771118164,
      "activations/layer5_attention_weight_min": -67.93882751464844,
      "activations/layer6_attention_weight_max": 55.59539794921875,
      "activations/layer6_attention_weight_min": -57.61345672607422,
      "activations/layer7_attention_weight_max": 90.25653076171875,
      "activations/layer7_attention_weight_min": -88.73140716552734,
      "activations/layer8_attention_weight_max": 40.014991760253906,
      "activations/layer8_attention_weight_min": -42.04692459106445,
      "activations/layer9_attention_weight_max": 35.60322570800781,
      "activations/layer9_attention_weight_min": -35.206329345703125,
      "epoch": 6.56,
      "learning_rate": 0.00010878901515151514,
      "loss": 2.8435,
      "step": 112900
    },
    {
      "activations/layer0_attention_weight_max": 16.584230422973633,
      "activations/layer0_attention_weight_min": -11.88919734954834,
      "activations/layer10_attention_weight_max": 32.7563362121582,
      "activations/layer10_attention_weight_min": -32.81147766113281,
      "activations/layer11_attention_weight_max": 33.30963134765625,
      "activations/layer11_attention_weight_min": -32.8528938293457,
      "activations/layer12_attention_weight_max": 25.839929580688477,
      "activations/layer12_attention_weight_min": -29.441360473632812,
      "activations/layer13_attention_weight_max": 46.4562873840332,
      "activations/layer13_attention_weight_min": -34.212181091308594,
      "activations/layer14_attention_weight_max": 61.01591873168945,
      "activations/layer14_attention_weight_min": -53.90869140625,
      "activations/layer15_attention_weight_max": 37.599056243896484,
      "activations/layer15_attention_weight_min": -29.624774932861328,
      "activations/layer16_attention_weight_max": 34.927001953125,
      "activations/layer16_attention_weight_min": -31.782665252685547,
      "activations/layer17_attention_weight_max": 37.095916748046875,
      "activations/layer17_attention_weight_min": -25.25591468811035,
      "activations/layer18_attention_weight_max": 32.421775817871094,
      "activations/layer18_attention_weight_min": -22.090599060058594,
      "activations/layer19_attention_weight_max": 42.281551361083984,
      "activations/layer19_attention_weight_min": -33.23263168334961,
      "activations/layer1_attention_weight_max": 17.67642593383789,
      "activations/layer1_attention_weight_min": -14.968684196472168,
      "activations/layer20_attention_weight_max": 34.934452056884766,
      "activations/layer20_attention_weight_min": -25.52065658569336,
      "activations/layer21_attention_weight_max": 30.105972290039062,
      "activations/layer21_attention_weight_min": -23.89866828918457,
      "activations/layer22_attention_weight_max": 54.51051330566406,
      "activations/layer22_attention_weight_min": -32.20824432373047,
      "activations/layer23_attention_weight_max": 35.5373649597168,
      "activations/layer23_attention_weight_min": -21.82971954345703,
      "activations/layer2_attention_weight_max": 32.58148193359375,
      "activations/layer2_attention_weight_min": -31.805492401123047,
      "activations/layer3_attention_weight_max": 91.36055755615234,
      "activations/layer3_attention_weight_min": -97.3274917602539,
      "activations/layer4_attention_weight_max": 127.64092254638672,
      "activations/layer4_attention_weight_min": -135.15306091308594,
      "activations/layer5_attention_weight_max": 71.16732788085938,
      "activations/layer5_attention_weight_min": -72.48204040527344,
      "activations/layer6_attention_weight_max": 51.76737976074219,
      "activations/layer6_attention_weight_min": -57.16309356689453,
      "activations/layer7_attention_weight_max": 87.72020721435547,
      "activations/layer7_attention_weight_min": -92.87645721435547,
      "activations/layer8_attention_weight_max": 40.837730407714844,
      "activations/layer8_attention_weight_min": -42.63565444946289,
      "activations/layer9_attention_weight_max": 38.50790023803711,
      "activations/layer9_attention_weight_min": -37.82616424560547,
      "epoch": 6.56,
      "learning_rate": 0.00010877007575757575,
      "loss": 2.8363,
      "step": 112950
    },
    {
      "activations/layer0_attention_weight_max": 15.546725273132324,
      "activations/layer0_attention_weight_min": -12.259312629699707,
      "activations/layer10_attention_weight_max": 33.093971252441406,
      "activations/layer10_attention_weight_min": -31.924583435058594,
      "activations/layer11_attention_weight_max": 34.94636535644531,
      "activations/layer11_attention_weight_min": -29.538482666015625,
      "activations/layer12_attention_weight_max": 24.97314453125,
      "activations/layer12_attention_weight_min": -25.38215446472168,
      "activations/layer13_attention_weight_max": 45.6939582824707,
      "activations/layer13_attention_weight_min": -34.12421798706055,
      "activations/layer14_attention_weight_max": 60.1146125793457,
      "activations/layer14_attention_weight_min": -41.162601470947266,
      "activations/layer15_attention_weight_max": 41.402732849121094,
      "activations/layer15_attention_weight_min": -29.849811553955078,
      "activations/layer16_attention_weight_max": 39.20514678955078,
      "activations/layer16_attention_weight_min": -29.384389877319336,
      "activations/layer17_attention_weight_max": 41.780059814453125,
      "activations/layer17_attention_weight_min": -26.76018714904785,
      "activations/layer18_attention_weight_max": 35.974021911621094,
      "activations/layer18_attention_weight_min": -22.54567527770996,
      "activations/layer19_attention_weight_max": 48.865631103515625,
      "activations/layer19_attention_weight_min": -29.222274780273438,
      "activations/layer1_attention_weight_max": 17.958148956298828,
      "activations/layer1_attention_weight_min": -14.276432991027832,
      "activations/layer20_attention_weight_max": 37.61799240112305,
      "activations/layer20_attention_weight_min": -24.169477462768555,
      "activations/layer21_attention_weight_max": 35.142181396484375,
      "activations/layer21_attention_weight_min": -20.831764221191406,
      "activations/layer22_attention_weight_max": 59.37369918823242,
      "activations/layer22_attention_weight_min": -28.262563705444336,
      "activations/layer23_attention_weight_max": 41.48509979248047,
      "activations/layer23_attention_weight_min": -22.0701904296875,
      "activations/layer2_attention_weight_max": 31.563678741455078,
      "activations/layer2_attention_weight_min": -30.651824951171875,
      "activations/layer3_attention_weight_max": 89.24494934082031,
      "activations/layer3_attention_weight_min": -91.35942077636719,
      "activations/layer4_attention_weight_max": 123.3031234741211,
      "activations/layer4_attention_weight_min": -124.1157455444336,
      "activations/layer5_attention_weight_max": 66.05294036865234,
      "activations/layer5_attention_weight_min": -64.02057647705078,
      "activations/layer6_attention_weight_max": 51.58284378051758,
      "activations/layer6_attention_weight_min": -53.92478942871094,
      "activations/layer7_attention_weight_max": 87.47674560546875,
      "activations/layer7_attention_weight_min": -86.81282043457031,
      "activations/layer8_attention_weight_max": 38.35273742675781,
      "activations/layer8_attention_weight_min": -42.49748611450195,
      "activations/layer9_attention_weight_max": 36.39549255371094,
      "activations/layer9_attention_weight_min": -34.96059799194336,
      "epoch": 6.57,
      "learning_rate": 0.00010875113636363636,
      "loss": 2.834,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_loss": 2.77734375,
      "eval_runtime": 8.475,
      "eval_samples_per_second": 506.669,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_openwebtext_loss": 2.77734375,
      "eval_openwebtext_ppl": 16.076261613516785,
      "eval_openwebtext_runtime": 8.475,
      "eval_openwebtext_samples_per_second": 506.669,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_wikitext_loss": 3.005859375,
      "eval_wikitext_ppl": 20.203571081597193,
      "eval_wikitext_runtime": 1.99,
      "eval_wikitext_samples_per_second": 229.148,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_lambada_loss": 2.62109375,
      "eval_lambada_ppl": 13.750755251693265,
      "eval_lambada_runtime": 9.5743,
      "eval_lambada_samples_per_second": 508.551,
      "step": 113000
    },
    {
      "activations/layer0_attention_weight_max": 16.674196243286133,
      "activations/layer0_attention_weight_min": -12.12452220916748,
      "activations/layer10_attention_weight_max": 31.541648864746094,
      "activations/layer10_attention_weight_min": -32.56789016723633,
      "activations/layer11_attention_weight_max": 32.44701385498047,
      "activations/layer11_attention_weight_min": -30.124309539794922,
      "activations/layer12_attention_weight_max": 22.74606704711914,
      "activations/layer12_attention_weight_min": -30.173355102539062,
      "activations/layer13_attention_weight_max": 36.958740234375,
      "activations/layer13_attention_weight_min": -31.930681228637695,
      "activations/layer14_attention_weight_max": 52.7293815612793,
      "activations/layer14_attention_weight_min": -42.371707916259766,
      "activations/layer15_attention_weight_max": 36.21748352050781,
      "activations/layer15_attention_weight_min": -28.294971466064453,
      "activations/layer16_attention_weight_max": 34.05851364135742,
      "activations/layer16_attention_weight_min": -28.214067459106445,
      "activations/layer17_attention_weight_max": 37.274803161621094,
      "activations/layer17_attention_weight_min": -26.943235397338867,
      "activations/layer18_attention_weight_max": 30.574796676635742,
      "activations/layer18_attention_weight_min": -21.88075828552246,
      "activations/layer19_attention_weight_max": 40.53025436401367,
      "activations/layer19_attention_weight_min": -31.203664779663086,
      "activations/layer1_attention_weight_max": 16.53329849243164,
      "activations/layer1_attention_weight_min": -14.976569175720215,
      "activations/layer20_attention_weight_max": 32.64319610595703,
      "activations/layer20_attention_weight_min": -24.87865447998047,
      "activations/layer21_attention_weight_max": 29.905174255371094,
      "activations/layer21_attention_weight_min": -23.113683700561523,
      "activations/layer22_attention_weight_max": 50.859073638916016,
      "activations/layer22_attention_weight_min": -29.09469985961914,
      "activations/layer23_attention_weight_max": 38.048065185546875,
      "activations/layer23_attention_weight_min": -23.909788131713867,
      "activations/layer2_attention_weight_max": 31.765789031982422,
      "activations/layer2_attention_weight_min": -31.60419464111328,
      "activations/layer3_attention_weight_max": 91.63737487792969,
      "activations/layer3_attention_weight_min": -99.41495513916016,
      "activations/layer4_attention_weight_max": 126.2529067993164,
      "activations/layer4_attention_weight_min": -135.6583251953125,
      "activations/layer5_attention_weight_max": 64.82568359375,
      "activations/layer5_attention_weight_min": -66.56716918945312,
      "activations/layer6_attention_weight_max": 51.33684158325195,
      "activations/layer6_attention_weight_min": -52.6625862121582,
      "activations/layer7_attention_weight_max": 83.34716796875,
      "activations/layer7_attention_weight_min": -82.20318603515625,
      "activations/layer8_attention_weight_max": 38.231082916259766,
      "activations/layer8_attention_weight_min": -42.78840637207031,
      "activations/layer9_attention_weight_max": 34.26029968261719,
      "activations/layer9_attention_weight_min": -34.68703079223633,
      "epoch": 6.57,
      "learning_rate": 0.00010873257575757575,
      "loss": 2.8198,
      "step": 113050
    },
    {
      "activations/layer0_attention_weight_max": 16.385530471801758,
      "activations/layer0_attention_weight_min": -11.94178581237793,
      "activations/layer10_attention_weight_max": 40.55613708496094,
      "activations/layer10_attention_weight_min": -36.909828186035156,
      "activations/layer11_attention_weight_max": 42.52565002441406,
      "activations/layer11_attention_weight_min": -35.39529800415039,
      "activations/layer12_attention_weight_max": 70.3799819946289,
      "activations/layer12_attention_weight_min": -31.253313064575195,
      "activations/layer13_attention_weight_max": 75.41078186035156,
      "activations/layer13_attention_weight_min": -36.22618103027344,
      "activations/layer14_attention_weight_max": 83.0006103515625,
      "activations/layer14_attention_weight_min": -48.424407958984375,
      "activations/layer15_attention_weight_max": 64.52738952636719,
      "activations/layer15_attention_weight_min": -31.41029930114746,
      "activations/layer16_attention_weight_max": 43.20362854003906,
      "activations/layer16_attention_weight_min": -30.0062255859375,
      "activations/layer17_attention_weight_max": 45.04815673828125,
      "activations/layer17_attention_weight_min": -25.35023307800293,
      "activations/layer18_attention_weight_max": 39.386688232421875,
      "activations/layer18_attention_weight_min": -21.771007537841797,
      "activations/layer19_attention_weight_max": 55.45415115356445,
      "activations/layer19_attention_weight_min": -28.61777687072754,
      "activations/layer1_attention_weight_max": 16.584848403930664,
      "activations/layer1_attention_weight_min": -13.772249221801758,
      "activations/layer20_attention_weight_max": 42.75996017456055,
      "activations/layer20_attention_weight_min": -22.80180549621582,
      "activations/layer21_attention_weight_max": 41.0498161315918,
      "activations/layer21_attention_weight_min": -20.61794662475586,
      "activations/layer22_attention_weight_max": 69.87760925292969,
      "activations/layer22_attention_weight_min": -28.984928131103516,
      "activations/layer23_attention_weight_max": 45.228302001953125,
      "activations/layer23_attention_weight_min": -23.958600997924805,
      "activations/layer2_attention_weight_max": 33.398956298828125,
      "activations/layer2_attention_weight_min": -33.111412048339844,
      "activations/layer3_attention_weight_max": 94.7999267578125,
      "activations/layer3_attention_weight_min": -95.94869995117188,
      "activations/layer4_attention_weight_max": 131.2047119140625,
      "activations/layer4_attention_weight_min": -131.91749572753906,
      "activations/layer5_attention_weight_max": 72.94261169433594,
      "activations/layer5_attention_weight_min": -69.51077270507812,
      "activations/layer6_attention_weight_max": 56.34245681762695,
      "activations/layer6_attention_weight_min": -57.8765983581543,
      "activations/layer7_attention_weight_max": 104.60184478759766,
      "activations/layer7_attention_weight_min": -99.30673217773438,
      "activations/layer8_attention_weight_max": 46.650386810302734,
      "activations/layer8_attention_weight_min": -43.21113204956055,
      "activations/layer9_attention_weight_max": 41.71088790893555,
      "activations/layer9_attention_weight_min": -37.37271499633789,
      "epoch": 6.57,
      "learning_rate": 0.00010871363636363635,
      "loss": 2.8201,
      "step": 113100
    },
    {
      "activations/layer0_attention_weight_max": 16.44796371459961,
      "activations/layer0_attention_weight_min": -11.114850044250488,
      "activations/layer10_attention_weight_max": 30.348276138305664,
      "activations/layer10_attention_weight_min": -31.156810760498047,
      "activations/layer11_attention_weight_max": 30.48597526550293,
      "activations/layer11_attention_weight_min": -29.04070281982422,
      "activations/layer12_attention_weight_max": 22.083728790283203,
      "activations/layer12_attention_weight_min": -24.97597885131836,
      "activations/layer13_attention_weight_max": 38.290767669677734,
      "activations/layer13_attention_weight_min": -32.00230026245117,
      "activations/layer14_attention_weight_max": 52.37466812133789,
      "activations/layer14_attention_weight_min": -39.54596710205078,
      "activations/layer15_attention_weight_max": 34.47713851928711,
      "activations/layer15_attention_weight_min": -27.53509521484375,
      "activations/layer16_attention_weight_max": 31.781156539916992,
      "activations/layer16_attention_weight_min": -26.799760818481445,
      "activations/layer17_attention_weight_max": 35.50096893310547,
      "activations/layer17_attention_weight_min": -24.662216186523438,
      "activations/layer18_attention_weight_max": 34.77263641357422,
      "activations/layer18_attention_weight_min": -22.247760772705078,
      "activations/layer19_attention_weight_max": 35.40378952026367,
      "activations/layer19_attention_weight_min": -29.09025001525879,
      "activations/layer1_attention_weight_max": 17.26015281677246,
      "activations/layer1_attention_weight_min": -14.264981269836426,
      "activations/layer20_attention_weight_max": 31.553627014160156,
      "activations/layer20_attention_weight_min": -23.882627487182617,
      "activations/layer21_attention_weight_max": 30.048049926757812,
      "activations/layer21_attention_weight_min": -22.449064254760742,
      "activations/layer22_attention_weight_max": 44.22589111328125,
      "activations/layer22_attention_weight_min": -29.62471580505371,
      "activations/layer23_attention_weight_max": 33.63653564453125,
      "activations/layer23_attention_weight_min": -22.008934020996094,
      "activations/layer2_attention_weight_max": 31.92194938659668,
      "activations/layer2_attention_weight_min": -31.50045394897461,
      "activations/layer3_attention_weight_max": 92.63307189941406,
      "activations/layer3_attention_weight_min": -96.15447235107422,
      "activations/layer4_attention_weight_max": 128.96473693847656,
      "activations/layer4_attention_weight_min": -126.27935791015625,
      "activations/layer5_attention_weight_max": 64.83811950683594,
      "activations/layer5_attention_weight_min": -63.460426330566406,
      "activations/layer6_attention_weight_max": 49.724205017089844,
      "activations/layer6_attention_weight_min": -52.80863952636719,
      "activations/layer7_attention_weight_max": 87.08077239990234,
      "activations/layer7_attention_weight_min": -86.31195831298828,
      "activations/layer8_attention_weight_max": 38.164188385009766,
      "activations/layer8_attention_weight_min": -40.6444091796875,
      "activations/layer9_attention_weight_max": 33.45112609863281,
      "activations/layer9_attention_weight_min": -33.20419692993164,
      "epoch": 6.57,
      "learning_rate": 0.00010869469696969696,
      "loss": 2.8285,
      "step": 113150
    },
    {
      "activations/layer0_attention_weight_max": 15.36101245880127,
      "activations/layer0_attention_weight_min": -12.698302268981934,
      "activations/layer10_attention_weight_max": 35.731483459472656,
      "activations/layer10_attention_weight_min": -32.508209228515625,
      "activations/layer11_attention_weight_max": 35.076995849609375,
      "activations/layer11_attention_weight_min": -29.085628509521484,
      "activations/layer12_attention_weight_max": 22.069543838500977,
      "activations/layer12_attention_weight_min": -26.016843795776367,
      "activations/layer13_attention_weight_max": 39.05158233642578,
      "activations/layer13_attention_weight_min": -32.38527297973633,
      "activations/layer14_attention_weight_max": 49.0373420715332,
      "activations/layer14_attention_weight_min": -39.28428268432617,
      "activations/layer15_attention_weight_max": 35.93621063232422,
      "activations/layer15_attention_weight_min": -28.63172149658203,
      "activations/layer16_attention_weight_max": 32.36199188232422,
      "activations/layer16_attention_weight_min": -28.31180191040039,
      "activations/layer17_attention_weight_max": 33.734981536865234,
      "activations/layer17_attention_weight_min": -23.41031837463379,
      "activations/layer18_attention_weight_max": 30.634380340576172,
      "activations/layer18_attention_weight_min": -20.07184410095215,
      "activations/layer19_attention_weight_max": 34.58237838745117,
      "activations/layer19_attention_weight_min": -30.624069213867188,
      "activations/layer1_attention_weight_max": 16.847261428833008,
      "activations/layer1_attention_weight_min": -14.545736312866211,
      "activations/layer20_attention_weight_max": 29.448583602905273,
      "activations/layer20_attention_weight_min": -23.3424129486084,
      "activations/layer21_attention_weight_max": 30.87900161743164,
      "activations/layer21_attention_weight_min": -21.101778030395508,
      "activations/layer22_attention_weight_max": 46.195674896240234,
      "activations/layer22_attention_weight_min": -28.577377319335938,
      "activations/layer23_attention_weight_max": 32.46470642089844,
      "activations/layer23_attention_weight_min": -20.986215591430664,
      "activations/layer2_attention_weight_max": 32.5347900390625,
      "activations/layer2_attention_weight_min": -33.191898345947266,
      "activations/layer3_attention_weight_max": 87.41047668457031,
      "activations/layer3_attention_weight_min": -90.86791229248047,
      "activations/layer4_attention_weight_max": 126.05085754394531,
      "activations/layer4_attention_weight_min": -126.00711822509766,
      "activations/layer5_attention_weight_max": 68.70802307128906,
      "activations/layer5_attention_weight_min": -67.93253326416016,
      "activations/layer6_attention_weight_max": 52.55668640136719,
      "activations/layer6_attention_weight_min": -56.39484786987305,
      "activations/layer7_attention_weight_max": 87.67552185058594,
      "activations/layer7_attention_weight_min": -91.42645263671875,
      "activations/layer8_attention_weight_max": 41.66362380981445,
      "activations/layer8_attention_weight_min": -41.653411865234375,
      "activations/layer9_attention_weight_max": 38.48052215576172,
      "activations/layer9_attention_weight_min": -36.3987922668457,
      "epoch": 6.58,
      "learning_rate": 0.00010867575757575758,
      "loss": 2.8255,
      "step": 113200
    },
    {
      "activations/layer0_attention_weight_max": 16.033288955688477,
      "activations/layer0_attention_weight_min": -12.212786674499512,
      "activations/layer10_attention_weight_max": 42.02028274536133,
      "activations/layer10_attention_weight_min": -39.413665771484375,
      "activations/layer11_attention_weight_max": 45.475460052490234,
      "activations/layer11_attention_weight_min": -36.33985900878906,
      "activations/layer12_attention_weight_max": 37.34489822387695,
      "activations/layer12_attention_weight_min": -27.30840301513672,
      "activations/layer13_attention_weight_max": 64.0492935180664,
      "activations/layer13_attention_weight_min": -39.15047836303711,
      "activations/layer14_attention_weight_max": 68.50253295898438,
      "activations/layer14_attention_weight_min": -46.320987701416016,
      "activations/layer15_attention_weight_max": 66.86196899414062,
      "activations/layer15_attention_weight_min": -33.761539459228516,
      "activations/layer16_attention_weight_max": 45.99732971191406,
      "activations/layer16_attention_weight_min": -31.35884666442871,
      "activations/layer17_attention_weight_max": 38.53549575805664,
      "activations/layer17_attention_weight_min": -26.100624084472656,
      "activations/layer18_attention_weight_max": 32.875858306884766,
      "activations/layer18_attention_weight_min": -21.762401580810547,
      "activations/layer19_attention_weight_max": 40.59116744995117,
      "activations/layer19_attention_weight_min": -30.57264518737793,
      "activations/layer1_attention_weight_max": 17.304367065429688,
      "activations/layer1_attention_weight_min": -14.459943771362305,
      "activations/layer20_attention_weight_max": 36.39167022705078,
      "activations/layer20_attention_weight_min": -25.313596725463867,
      "activations/layer21_attention_weight_max": 33.450721740722656,
      "activations/layer21_attention_weight_min": -22.674514770507812,
      "activations/layer22_attention_weight_max": 51.280296325683594,
      "activations/layer22_attention_weight_min": -30.348440170288086,
      "activations/layer23_attention_weight_max": 35.28518295288086,
      "activations/layer23_attention_weight_min": -22.614093780517578,
      "activations/layer2_attention_weight_max": 33.411903381347656,
      "activations/layer2_attention_weight_min": -34.06288528442383,
      "activations/layer3_attention_weight_max": 96.9808120727539,
      "activations/layer3_attention_weight_min": -99.00639343261719,
      "activations/layer4_attention_weight_max": 138.45420837402344,
      "activations/layer4_attention_weight_min": -128.1912841796875,
      "activations/layer5_attention_weight_max": 74.58125305175781,
      "activations/layer5_attention_weight_min": -67.45108795166016,
      "activations/layer6_attention_weight_max": 61.44580841064453,
      "activations/layer6_attention_weight_min": -53.402000427246094,
      "activations/layer7_attention_weight_max": 110.26130676269531,
      "activations/layer7_attention_weight_min": -95.71327209472656,
      "activations/layer8_attention_weight_max": 52.14662551879883,
      "activations/layer8_attention_weight_min": -45.59822463989258,
      "activations/layer9_attention_weight_max": 44.66523742675781,
      "activations/layer9_attention_weight_min": -42.2261962890625,
      "epoch": 6.58,
      "learning_rate": 0.00010865681818181817,
      "loss": 2.8201,
      "step": 113250
    },
    {
      "activations/layer0_attention_weight_max": 16.763425827026367,
      "activations/layer0_attention_weight_min": -13.428268432617188,
      "activations/layer10_attention_weight_max": 31.315574645996094,
      "activations/layer10_attention_weight_min": -30.59244155883789,
      "activations/layer11_attention_weight_max": 30.52911949157715,
      "activations/layer11_attention_weight_min": -27.910655975341797,
      "activations/layer12_attention_weight_max": 21.488521575927734,
      "activations/layer12_attention_weight_min": -24.742956161499023,
      "activations/layer13_attention_weight_max": 38.35836410522461,
      "activations/layer13_attention_weight_min": -30.85542869567871,
      "activations/layer14_attention_weight_max": 47.4986457824707,
      "activations/layer14_attention_weight_min": -41.127662658691406,
      "activations/layer15_attention_weight_max": 35.477806091308594,
      "activations/layer15_attention_weight_min": -28.23681640625,
      "activations/layer16_attention_weight_max": 33.04491424560547,
      "activations/layer16_attention_weight_min": -26.69135856628418,
      "activations/layer17_attention_weight_max": 33.44108200073242,
      "activations/layer17_attention_weight_min": -24.354442596435547,
      "activations/layer18_attention_weight_max": 32.42499923706055,
      "activations/layer18_attention_weight_min": -20.425294876098633,
      "activations/layer19_attention_weight_max": 37.23012924194336,
      "activations/layer19_attention_weight_min": -31.704029083251953,
      "activations/layer1_attention_weight_max": 17.167823791503906,
      "activations/layer1_attention_weight_min": -14.563934326171875,
      "activations/layer20_attention_weight_max": 28.97054672241211,
      "activations/layer20_attention_weight_min": -24.02461814880371,
      "activations/layer21_attention_weight_max": 29.650888442993164,
      "activations/layer21_attention_weight_min": -24.079641342163086,
      "activations/layer22_attention_weight_max": 44.01279067993164,
      "activations/layer22_attention_weight_min": -29.947019577026367,
      "activations/layer23_attention_weight_max": 33.165000915527344,
      "activations/layer23_attention_weight_min": -23.207014083862305,
      "activations/layer2_attention_weight_max": 32.32786560058594,
      "activations/layer2_attention_weight_min": -31.30073356628418,
      "activations/layer3_attention_weight_max": 89.29696655273438,
      "activations/layer3_attention_weight_min": -90.29649353027344,
      "activations/layer4_attention_weight_max": 123.34162902832031,
      "activations/layer4_attention_weight_min": -121.80140686035156,
      "activations/layer5_attention_weight_max": 65.96380615234375,
      "activations/layer5_attention_weight_min": -64.82914733886719,
      "activations/layer6_attention_weight_max": 58.630218505859375,
      "activations/layer6_attention_weight_min": -55.26775360107422,
      "activations/layer7_attention_weight_max": 94.93920135498047,
      "activations/layer7_attention_weight_min": -88.58047485351562,
      "activations/layer8_attention_weight_max": 39.366214752197266,
      "activations/layer8_attention_weight_min": -39.8262939453125,
      "activations/layer9_attention_weight_max": 34.75400161743164,
      "activations/layer9_attention_weight_min": -34.25898742675781,
      "epoch": 6.58,
      "learning_rate": 0.00010863787878787878,
      "loss": 2.8043,
      "step": 113300
    },
    {
      "activations/layer0_attention_weight_max": 17.01715660095215,
      "activations/layer0_attention_weight_min": -13.030461311340332,
      "activations/layer10_attention_weight_max": 32.847679138183594,
      "activations/layer10_attention_weight_min": -31.004955291748047,
      "activations/layer11_attention_weight_max": 32.61519241333008,
      "activations/layer11_attention_weight_min": -28.421367645263672,
      "activations/layer12_attention_weight_max": 21.637420654296875,
      "activations/layer12_attention_weight_min": -24.922245025634766,
      "activations/layer13_attention_weight_max": 41.43905258178711,
      "activations/layer13_attention_weight_min": -34.203468322753906,
      "activations/layer14_attention_weight_max": 53.40803909301758,
      "activations/layer14_attention_weight_min": -40.69224548339844,
      "activations/layer15_attention_weight_max": 35.39411544799805,
      "activations/layer15_attention_weight_min": -29.968137741088867,
      "activations/layer16_attention_weight_max": 33.81181335449219,
      "activations/layer16_attention_weight_min": -27.16861343383789,
      "activations/layer17_attention_weight_max": 34.674156188964844,
      "activations/layer17_attention_weight_min": -24.338335037231445,
      "activations/layer18_attention_weight_max": 34.39236831665039,
      "activations/layer18_attention_weight_min": -20.353416442871094,
      "activations/layer19_attention_weight_max": 38.86050796508789,
      "activations/layer19_attention_weight_min": -28.767704010009766,
      "activations/layer1_attention_weight_max": 16.699663162231445,
      "activations/layer1_attention_weight_min": -14.389204025268555,
      "activations/layer20_attention_weight_max": 29.96465492248535,
      "activations/layer20_attention_weight_min": -23.57506561279297,
      "activations/layer21_attention_weight_max": 31.938810348510742,
      "activations/layer21_attention_weight_min": -20.790878295898438,
      "activations/layer22_attention_weight_max": 45.267616271972656,
      "activations/layer22_attention_weight_min": -28.203231811523438,
      "activations/layer23_attention_weight_max": 34.05675506591797,
      "activations/layer23_attention_weight_min": -23.556987762451172,
      "activations/layer2_attention_weight_max": 34.26734161376953,
      "activations/layer2_attention_weight_min": -30.830894470214844,
      "activations/layer3_attention_weight_max": 85.27810668945312,
      "activations/layer3_attention_weight_min": -86.48843383789062,
      "activations/layer4_attention_weight_max": 121.22159576416016,
      "activations/layer4_attention_weight_min": -118.58312225341797,
      "activations/layer5_attention_weight_max": 66.6764144897461,
      "activations/layer5_attention_weight_min": -65.34766387939453,
      "activations/layer6_attention_weight_max": 51.66372299194336,
      "activations/layer6_attention_weight_min": -56.3377571105957,
      "activations/layer7_attention_weight_max": 86.61463165283203,
      "activations/layer7_attention_weight_min": -87.62714385986328,
      "activations/layer8_attention_weight_max": 42.82423782348633,
      "activations/layer8_attention_weight_min": -43.35466003417969,
      "activations/layer9_attention_weight_max": 37.267730712890625,
      "activations/layer9_attention_weight_min": -34.5147590637207,
      "epoch": 6.59,
      "learning_rate": 0.0001086189393939394,
      "loss": 2.8319,
      "step": 113350
    },
    {
      "activations/layer0_attention_weight_max": 16.994674682617188,
      "activations/layer0_attention_weight_min": -12.040814399719238,
      "activations/layer10_attention_weight_max": 41.669097900390625,
      "activations/layer10_attention_weight_min": -40.275882720947266,
      "activations/layer11_attention_weight_max": 44.5728759765625,
      "activations/layer11_attention_weight_min": -38.66371536254883,
      "activations/layer12_attention_weight_max": 51.9180793762207,
      "activations/layer12_attention_weight_min": -26.995906829833984,
      "activations/layer13_attention_weight_max": 62.968162536621094,
      "activations/layer13_attention_weight_min": -39.77385711669922,
      "activations/layer14_attention_weight_max": 58.93562316894531,
      "activations/layer14_attention_weight_min": -45.69181442260742,
      "activations/layer15_attention_weight_max": 41.58976745605469,
      "activations/layer15_attention_weight_min": -28.79799461364746,
      "activations/layer16_attention_weight_max": 34.412078857421875,
      "activations/layer16_attention_weight_min": -25.44634437561035,
      "activations/layer17_attention_weight_max": 33.719722747802734,
      "activations/layer17_attention_weight_min": -24.491973876953125,
      "activations/layer18_attention_weight_max": 28.482168197631836,
      "activations/layer18_attention_weight_min": -20.314266204833984,
      "activations/layer19_attention_weight_max": 37.006500244140625,
      "activations/layer19_attention_weight_min": -29.31244468688965,
      "activations/layer1_attention_weight_max": 17.44371795654297,
      "activations/layer1_attention_weight_min": -15.500237464904785,
      "activations/layer20_attention_weight_max": 28.699745178222656,
      "activations/layer20_attention_weight_min": -23.82364273071289,
      "activations/layer21_attention_weight_max": 26.930419921875,
      "activations/layer21_attention_weight_min": -21.321500778198242,
      "activations/layer22_attention_weight_max": 45.61166763305664,
      "activations/layer22_attention_weight_min": -31.300312042236328,
      "activations/layer23_attention_weight_max": 34.82208251953125,
      "activations/layer23_attention_weight_min": -24.93557357788086,
      "activations/layer2_attention_weight_max": 34.527591705322266,
      "activations/layer2_attention_weight_min": -34.16194152832031,
      "activations/layer3_attention_weight_max": 93.0709457397461,
      "activations/layer3_attention_weight_min": -95.27484893798828,
      "activations/layer4_attention_weight_max": 143.7982635498047,
      "activations/layer4_attention_weight_min": -129.26182556152344,
      "activations/layer5_attention_weight_max": 72.61811065673828,
      "activations/layer5_attention_weight_min": -72.86920166015625,
      "activations/layer6_attention_weight_max": 57.50798034667969,
      "activations/layer6_attention_weight_min": -54.04331970214844,
      "activations/layer7_attention_weight_max": 105.8636474609375,
      "activations/layer7_attention_weight_min": -103.08648681640625,
      "activations/layer8_attention_weight_max": 50.21674346923828,
      "activations/layer8_attention_weight_min": -48.6538200378418,
      "activations/layer9_attention_weight_max": 44.91468811035156,
      "activations/layer9_attention_weight_min": -44.790565490722656,
      "epoch": 6.59,
      "learning_rate": 0.00010859999999999998,
      "loss": 2.8242,
      "step": 113400
    },
    {
      "activations/layer0_attention_weight_max": 14.990462303161621,
      "activations/layer0_attention_weight_min": -13.407822608947754,
      "activations/layer10_attention_weight_max": 32.68225860595703,
      "activations/layer10_attention_weight_min": -30.310007095336914,
      "activations/layer11_attention_weight_max": 30.99994659423828,
      "activations/layer11_attention_weight_min": -28.191640853881836,
      "activations/layer12_attention_weight_max": 25.434940338134766,
      "activations/layer12_attention_weight_min": -28.21999740600586,
      "activations/layer13_attention_weight_max": 44.37034606933594,
      "activations/layer13_attention_weight_min": -30.099918365478516,
      "activations/layer14_attention_weight_max": 53.779964447021484,
      "activations/layer14_attention_weight_min": -39.52669143676758,
      "activations/layer15_attention_weight_max": 34.26274108886719,
      "activations/layer15_attention_weight_min": -28.86702537536621,
      "activations/layer16_attention_weight_max": 34.721195220947266,
      "activations/layer16_attention_weight_min": -25.826555252075195,
      "activations/layer17_attention_weight_max": 36.48189926147461,
      "activations/layer17_attention_weight_min": -24.579469680786133,
      "activations/layer18_attention_weight_max": 31.533845901489258,
      "activations/layer18_attention_weight_min": -22.206235885620117,
      "activations/layer19_attention_weight_max": 34.83232879638672,
      "activations/layer19_attention_weight_min": -29.442365646362305,
      "activations/layer1_attention_weight_max": 16.868383407592773,
      "activations/layer1_attention_weight_min": -13.723907470703125,
      "activations/layer20_attention_weight_max": 29.555063247680664,
      "activations/layer20_attention_weight_min": -23.840465545654297,
      "activations/layer21_attention_weight_max": 28.537242889404297,
      "activations/layer21_attention_weight_min": -21.721895217895508,
      "activations/layer22_attention_weight_max": 47.76772689819336,
      "activations/layer22_attention_weight_min": -28.138858795166016,
      "activations/layer23_attention_weight_max": 35.36844253540039,
      "activations/layer23_attention_weight_min": -20.759078979492188,
      "activations/layer2_attention_weight_max": 31.193714141845703,
      "activations/layer2_attention_weight_min": -31.916173934936523,
      "activations/layer3_attention_weight_max": 84.35417175292969,
      "activations/layer3_attention_weight_min": -90.89502716064453,
      "activations/layer4_attention_weight_max": 123.15985107421875,
      "activations/layer4_attention_weight_min": -121.14347076416016,
      "activations/layer5_attention_weight_max": 68.5860595703125,
      "activations/layer5_attention_weight_min": -68.44775390625,
      "activations/layer6_attention_weight_max": 52.96901321411133,
      "activations/layer6_attention_weight_min": -53.66460418701172,
      "activations/layer7_attention_weight_max": 80.21256256103516,
      "activations/layer7_attention_weight_min": -84.68905639648438,
      "activations/layer8_attention_weight_max": 38.4793701171875,
      "activations/layer8_attention_weight_min": -41.00917434692383,
      "activations/layer9_attention_weight_max": 35.390525817871094,
      "activations/layer9_attention_weight_min": -32.30913543701172,
      "epoch": 6.59,
      "learning_rate": 0.0001085810606060606,
      "loss": 2.8407,
      "step": 113450
    },
    {
      "activations/layer0_attention_weight_max": 15.836000442504883,
      "activations/layer0_attention_weight_min": -12.75383186340332,
      "activations/layer10_attention_weight_max": 34.79182815551758,
      "activations/layer10_attention_weight_min": -32.23307800292969,
      "activations/layer11_attention_weight_max": 36.785743713378906,
      "activations/layer11_attention_weight_min": -30.081266403198242,
      "activations/layer12_attention_weight_max": 27.906553268432617,
      "activations/layer12_attention_weight_min": -26.215476989746094,
      "activations/layer13_attention_weight_max": 46.0255241394043,
      "activations/layer13_attention_weight_min": -31.876815795898438,
      "activations/layer14_attention_weight_max": 62.517822265625,
      "activations/layer14_attention_weight_min": -39.44174575805664,
      "activations/layer15_attention_weight_max": 38.291019439697266,
      "activations/layer15_attention_weight_min": -30.11935806274414,
      "activations/layer16_attention_weight_max": 35.729373931884766,
      "activations/layer16_attention_weight_min": -27.426118850708008,
      "activations/layer17_attention_weight_max": 37.70610046386719,
      "activations/layer17_attention_weight_min": -23.499523162841797,
      "activations/layer18_attention_weight_max": 38.188446044921875,
      "activations/layer18_attention_weight_min": -20.298038482666016,
      "activations/layer19_attention_weight_max": 40.42860794067383,
      "activations/layer19_attention_weight_min": -29.197240829467773,
      "activations/layer1_attention_weight_max": 17.62000274658203,
      "activations/layer1_attention_weight_min": -17.411819458007812,
      "activations/layer20_attention_weight_max": 32.12958526611328,
      "activations/layer20_attention_weight_min": -22.77303123474121,
      "activations/layer21_attention_weight_max": 32.4742431640625,
      "activations/layer21_attention_weight_min": -19.854084014892578,
      "activations/layer22_attention_weight_max": 50.24368667602539,
      "activations/layer22_attention_weight_min": -26.994478225708008,
      "activations/layer23_attention_weight_max": 38.325626373291016,
      "activations/layer23_attention_weight_min": -21.7620849609375,
      "activations/layer2_attention_weight_max": 30.07620620727539,
      "activations/layer2_attention_weight_min": -30.339141845703125,
      "activations/layer3_attention_weight_max": 85.26021575927734,
      "activations/layer3_attention_weight_min": -91.79264831542969,
      "activations/layer4_attention_weight_max": 122.30498504638672,
      "activations/layer4_attention_weight_min": -128.6580810546875,
      "activations/layer5_attention_weight_max": 68.01637268066406,
      "activations/layer5_attention_weight_min": -67.22293090820312,
      "activations/layer6_attention_weight_max": 55.69346237182617,
      "activations/layer6_attention_weight_min": -55.38034439086914,
      "activations/layer7_attention_weight_max": 87.20193481445312,
      "activations/layer7_attention_weight_min": -93.12297821044922,
      "activations/layer8_attention_weight_max": 44.25701904296875,
      "activations/layer8_attention_weight_min": -41.29969024658203,
      "activations/layer9_attention_weight_max": 36.960567474365234,
      "activations/layer9_attention_weight_min": -35.21256637573242,
      "epoch": 6.59,
      "learning_rate": 0.0001085621212121212,
      "loss": 2.8288,
      "step": 113500
    },
    {
      "activations/layer0_attention_weight_max": 16.808427810668945,
      "activations/layer0_attention_weight_min": -12.96956729888916,
      "activations/layer10_attention_weight_max": 33.598148345947266,
      "activations/layer10_attention_weight_min": -31.057540893554688,
      "activations/layer11_attention_weight_max": 32.882362365722656,
      "activations/layer11_attention_weight_min": -29.411052703857422,
      "activations/layer12_attention_weight_max": 22.05596923828125,
      "activations/layer12_attention_weight_min": -27.901243209838867,
      "activations/layer13_attention_weight_max": 38.037715911865234,
      "activations/layer13_attention_weight_min": -31.527027130126953,
      "activations/layer14_attention_weight_max": 49.07603073120117,
      "activations/layer14_attention_weight_min": -40.187400817871094,
      "activations/layer15_attention_weight_max": 39.2882194519043,
      "activations/layer15_attention_weight_min": -28.57678985595703,
      "activations/layer16_attention_weight_max": 32.44062805175781,
      "activations/layer16_attention_weight_min": -28.297428131103516,
      "activations/layer17_attention_weight_max": 37.02725601196289,
      "activations/layer17_attention_weight_min": -25.845901489257812,
      "activations/layer18_attention_weight_max": 37.357933044433594,
      "activations/layer18_attention_weight_min": -23.419897079467773,
      "activations/layer19_attention_weight_max": 37.835418701171875,
      "activations/layer19_attention_weight_min": -28.236661911010742,
      "activations/layer1_attention_weight_max": 17.100818634033203,
      "activations/layer1_attention_weight_min": -14.0567626953125,
      "activations/layer20_attention_weight_max": 35.17302703857422,
      "activations/layer20_attention_weight_min": -25.193740844726562,
      "activations/layer21_attention_weight_max": 33.93914794921875,
      "activations/layer21_attention_weight_min": -21.767099380493164,
      "activations/layer22_attention_weight_max": 52.267086029052734,
      "activations/layer22_attention_weight_min": -29.211734771728516,
      "activations/layer23_attention_weight_max": 38.550323486328125,
      "activations/layer23_attention_weight_min": -20.703330993652344,
      "activations/layer2_attention_weight_max": 30.853878021240234,
      "activations/layer2_attention_weight_min": -31.515018463134766,
      "activations/layer3_attention_weight_max": 86.5202865600586,
      "activations/layer3_attention_weight_min": -89.01880645751953,
      "activations/layer4_attention_weight_max": 124.72843170166016,
      "activations/layer4_attention_weight_min": -127.4012680053711,
      "activations/layer5_attention_weight_max": 65.13945770263672,
      "activations/layer5_attention_weight_min": -64.88531494140625,
      "activations/layer6_attention_weight_max": 49.70246505737305,
      "activations/layer6_attention_weight_min": -52.999759674072266,
      "activations/layer7_attention_weight_max": 81.49637603759766,
      "activations/layer7_attention_weight_min": -87.41560363769531,
      "activations/layer8_attention_weight_max": 41.15302276611328,
      "activations/layer8_attention_weight_min": -38.641639709472656,
      "activations/layer9_attention_weight_max": 36.09469223022461,
      "activations/layer9_attention_weight_min": -36.622764587402344,
      "epoch": 6.6,
      "learning_rate": 0.0001085431818181818,
      "loss": 2.8198,
      "step": 113550
    },
    {
      "activations/layer0_attention_weight_max": 17.38522720336914,
      "activations/layer0_attention_weight_min": -11.780570983886719,
      "activations/layer10_attention_weight_max": 35.80645751953125,
      "activations/layer10_attention_weight_min": -32.02156066894531,
      "activations/layer11_attention_weight_max": 35.23414993286133,
      "activations/layer11_attention_weight_min": -32.123924255371094,
      "activations/layer12_attention_weight_max": 25.24551010131836,
      "activations/layer12_attention_weight_min": -25.40185546875,
      "activations/layer13_attention_weight_max": 43.058021545410156,
      "activations/layer13_attention_weight_min": -32.745689392089844,
      "activations/layer14_attention_weight_max": 54.15879440307617,
      "activations/layer14_attention_weight_min": -42.953758239746094,
      "activations/layer15_attention_weight_max": 37.16264724731445,
      "activations/layer15_attention_weight_min": -32.5322265625,
      "activations/layer16_attention_weight_max": 35.52433776855469,
      "activations/layer16_attention_weight_min": -27.451494216918945,
      "activations/layer17_attention_weight_max": 39.78508377075195,
      "activations/layer17_attention_weight_min": -24.310205459594727,
      "activations/layer18_attention_weight_max": 32.0126953125,
      "activations/layer18_attention_weight_min": -21.810617446899414,
      "activations/layer19_attention_weight_max": 36.00785827636719,
      "activations/layer19_attention_weight_min": -30.784826278686523,
      "activations/layer1_attention_weight_max": 18.407470703125,
      "activations/layer1_attention_weight_min": -13.710041046142578,
      "activations/layer20_attention_weight_max": 34.21720886230469,
      "activations/layer20_attention_weight_min": -24.942419052124023,
      "activations/layer21_attention_weight_max": 32.369834899902344,
      "activations/layer21_attention_weight_min": -22.71268653869629,
      "activations/layer22_attention_weight_max": 53.56233215332031,
      "activations/layer22_attention_weight_min": -31.766557693481445,
      "activations/layer23_attention_weight_max": 36.5597038269043,
      "activations/layer23_attention_weight_min": -23.479801177978516,
      "activations/layer2_attention_weight_max": 31.46809959411621,
      "activations/layer2_attention_weight_min": -31.112289428710938,
      "activations/layer3_attention_weight_max": 86.5203857421875,
      "activations/layer3_attention_weight_min": -89.31818389892578,
      "activations/layer4_attention_weight_max": 124.06471252441406,
      "activations/layer4_attention_weight_min": -119.45735168457031,
      "activations/layer5_attention_weight_max": 66.38882446289062,
      "activations/layer5_attention_weight_min": -62.99231719970703,
      "activations/layer6_attention_weight_max": 52.86648941040039,
      "activations/layer6_attention_weight_min": -52.74834060668945,
      "activations/layer7_attention_weight_max": 85.61830139160156,
      "activations/layer7_attention_weight_min": -85.62635803222656,
      "activations/layer8_attention_weight_max": 42.52019119262695,
      "activations/layer8_attention_weight_min": -40.919281005859375,
      "activations/layer9_attention_weight_max": 38.329402923583984,
      "activations/layer9_attention_weight_min": -33.751583099365234,
      "epoch": 6.6,
      "learning_rate": 0.00010852424242424242,
      "loss": 2.8469,
      "step": 113600
    },
    {
      "activations/layer0_attention_weight_max": 16.485116958618164,
      "activations/layer0_attention_weight_min": -12.906781196594238,
      "activations/layer10_attention_weight_max": 32.68607711791992,
      "activations/layer10_attention_weight_min": -30.99419403076172,
      "activations/layer11_attention_weight_max": 32.00640869140625,
      "activations/layer11_attention_weight_min": -28.070430755615234,
      "activations/layer12_attention_weight_max": 23.41117286682129,
      "activations/layer12_attention_weight_min": -25.65607452392578,
      "activations/layer13_attention_weight_max": 41.817047119140625,
      "activations/layer13_attention_weight_min": -31.529136657714844,
      "activations/layer14_attention_weight_max": 43.895042419433594,
      "activations/layer14_attention_weight_min": -38.0698356628418,
      "activations/layer15_attention_weight_max": 35.98052978515625,
      "activations/layer15_attention_weight_min": -29.555734634399414,
      "activations/layer16_attention_weight_max": 31.47662353515625,
      "activations/layer16_attention_weight_min": -26.57685661315918,
      "activations/layer17_attention_weight_max": 35.356502532958984,
      "activations/layer17_attention_weight_min": -24.32568359375,
      "activations/layer18_attention_weight_max": 32.84822463989258,
      "activations/layer18_attention_weight_min": -20.099397659301758,
      "activations/layer19_attention_weight_max": 36.94830322265625,
      "activations/layer19_attention_weight_min": -30.27327537536621,
      "activations/layer1_attention_weight_max": 16.31839370727539,
      "activations/layer1_attention_weight_min": -14.528005599975586,
      "activations/layer20_attention_weight_max": 29.492353439331055,
      "activations/layer20_attention_weight_min": -25.635974884033203,
      "activations/layer21_attention_weight_max": 31.945669174194336,
      "activations/layer21_attention_weight_min": -22.122865676879883,
      "activations/layer22_attention_weight_max": 45.84809112548828,
      "activations/layer22_attention_weight_min": -28.860273361206055,
      "activations/layer23_attention_weight_max": 38.6036262512207,
      "activations/layer23_attention_weight_min": -21.542232513427734,
      "activations/layer2_attention_weight_max": 30.471221923828125,
      "activations/layer2_attention_weight_min": -30.88397789001465,
      "activations/layer3_attention_weight_max": 89.13473510742188,
      "activations/layer3_attention_weight_min": -94.43888854980469,
      "activations/layer4_attention_weight_max": 127.13514709472656,
      "activations/layer4_attention_weight_min": -126.92939758300781,
      "activations/layer5_attention_weight_max": 71.30099487304688,
      "activations/layer5_attention_weight_min": -66.179931640625,
      "activations/layer6_attention_weight_max": 56.66312789916992,
      "activations/layer6_attention_weight_min": -53.67573165893555,
      "activations/layer7_attention_weight_max": 85.43701934814453,
      "activations/layer7_attention_weight_min": -85.80642700195312,
      "activations/layer8_attention_weight_max": 40.27181625366211,
      "activations/layer8_attention_weight_min": -38.566566467285156,
      "activations/layer9_attention_weight_max": 34.4169921875,
      "activations/layer9_attention_weight_min": -32.275386810302734,
      "epoch": 6.6,
      "learning_rate": 0.00010850530303030302,
      "loss": 2.8358,
      "step": 113650
    },
    {
      "activations/layer0_attention_weight_max": 17.053110122680664,
      "activations/layer0_attention_weight_min": -12.289289474487305,
      "activations/layer10_attention_weight_max": 34.92436218261719,
      "activations/layer10_attention_weight_min": -30.13222885131836,
      "activations/layer11_attention_weight_max": 33.201847076416016,
      "activations/layer11_attention_weight_min": -28.708904266357422,
      "activations/layer12_attention_weight_max": 45.24555587768555,
      "activations/layer12_attention_weight_min": -28.3176326751709,
      "activations/layer13_attention_weight_max": 39.00885772705078,
      "activations/layer13_attention_weight_min": -31.84817886352539,
      "activations/layer14_attention_weight_max": 55.557533264160156,
      "activations/layer14_attention_weight_min": -38.48143005371094,
      "activations/layer15_attention_weight_max": 38.819366455078125,
      "activations/layer15_attention_weight_min": -29.84441375732422,
      "activations/layer16_attention_weight_max": 37.93467330932617,
      "activations/layer16_attention_weight_min": -28.382570266723633,
      "activations/layer17_attention_weight_max": 38.218711853027344,
      "activations/layer17_attention_weight_min": -24.15702247619629,
      "activations/layer18_attention_weight_max": 40.20038986206055,
      "activations/layer18_attention_weight_min": -22.335716247558594,
      "activations/layer19_attention_weight_max": 43.53604507446289,
      "activations/layer19_attention_weight_min": -32.69170379638672,
      "activations/layer1_attention_weight_max": 17.10065269470215,
      "activations/layer1_attention_weight_min": -13.022984504699707,
      "activations/layer20_attention_weight_max": 37.71208190917969,
      "activations/layer20_attention_weight_min": -25.054441452026367,
      "activations/layer21_attention_weight_max": 44.84724044799805,
      "activations/layer21_attention_weight_min": -22.731409072875977,
      "activations/layer22_attention_weight_max": 58.40633010864258,
      "activations/layer22_attention_weight_min": -28.43290138244629,
      "activations/layer23_attention_weight_max": 38.972251892089844,
      "activations/layer23_attention_weight_min": -22.026365280151367,
      "activations/layer2_attention_weight_max": 30.77448081970215,
      "activations/layer2_attention_weight_min": -30.245718002319336,
      "activations/layer3_attention_weight_max": 84.45397186279297,
      "activations/layer3_attention_weight_min": -89.59544372558594,
      "activations/layer4_attention_weight_max": 116.2618179321289,
      "activations/layer4_attention_weight_min": -126.67774963378906,
      "activations/layer5_attention_weight_max": 63.258628845214844,
      "activations/layer5_attention_weight_min": -63.76285934448242,
      "activations/layer6_attention_weight_max": 50.54697036743164,
      "activations/layer6_attention_weight_min": -52.01408386230469,
      "activations/layer7_attention_weight_max": 86.28765106201172,
      "activations/layer7_attention_weight_min": -84.96736145019531,
      "activations/layer8_attention_weight_max": 40.70341873168945,
      "activations/layer8_attention_weight_min": -39.3096809387207,
      "activations/layer9_attention_weight_max": 35.4462776184082,
      "activations/layer9_attention_weight_min": -33.64840316772461,
      "epoch": 6.61,
      "learning_rate": 0.00010848636363636363,
      "loss": 2.8066,
      "step": 113700
    },
    {
      "activations/layer0_attention_weight_max": 16.094552993774414,
      "activations/layer0_attention_weight_min": -12.088619232177734,
      "activations/layer10_attention_weight_max": 34.698429107666016,
      "activations/layer10_attention_weight_min": -33.39851379394531,
      "activations/layer11_attention_weight_max": 32.5295295715332,
      "activations/layer11_attention_weight_min": -28.68861198425293,
      "activations/layer12_attention_weight_max": 22.33379364013672,
      "activations/layer12_attention_weight_min": -25.706546783447266,
      "activations/layer13_attention_weight_max": 39.25483322143555,
      "activations/layer13_attention_weight_min": -31.508594512939453,
      "activations/layer14_attention_weight_max": 51.33489227294922,
      "activations/layer14_attention_weight_min": -46.10602569580078,
      "activations/layer15_attention_weight_max": 41.42319107055664,
      "activations/layer15_attention_weight_min": -29.44736671447754,
      "activations/layer16_attention_weight_max": 30.948184967041016,
      "activations/layer16_attention_weight_min": -26.406936645507812,
      "activations/layer17_attention_weight_max": 37.91648483276367,
      "activations/layer17_attention_weight_min": -25.443838119506836,
      "activations/layer18_attention_weight_max": 30.695592880249023,
      "activations/layer18_attention_weight_min": -20.046354293823242,
      "activations/layer19_attention_weight_max": 35.33295440673828,
      "activations/layer19_attention_weight_min": -29.551183700561523,
      "activations/layer1_attention_weight_max": 16.927684783935547,
      "activations/layer1_attention_weight_min": -15.766596794128418,
      "activations/layer20_attention_weight_max": 27.024206161499023,
      "activations/layer20_attention_weight_min": -23.811771392822266,
      "activations/layer21_attention_weight_max": 24.872142791748047,
      "activations/layer21_attention_weight_min": -22.023113250732422,
      "activations/layer22_attention_weight_max": 47.268306732177734,
      "activations/layer22_attention_weight_min": -28.01959800720215,
      "activations/layer23_attention_weight_max": 30.363143920898438,
      "activations/layer23_attention_weight_min": -21.867359161376953,
      "activations/layer2_attention_weight_max": 31.20759391784668,
      "activations/layer2_attention_weight_min": -31.877731323242188,
      "activations/layer3_attention_weight_max": 88.67181396484375,
      "activations/layer3_attention_weight_min": -90.8949966430664,
      "activations/layer4_attention_weight_max": 132.34356689453125,
      "activations/layer4_attention_weight_min": -128.1304931640625,
      "activations/layer5_attention_weight_max": 73.0239486694336,
      "activations/layer5_attention_weight_min": -68.17630004882812,
      "activations/layer6_attention_weight_max": 55.33586502075195,
      "activations/layer6_attention_weight_min": -55.82225036621094,
      "activations/layer7_attention_weight_max": 88.36173248291016,
      "activations/layer7_attention_weight_min": -90.63582611083984,
      "activations/layer8_attention_weight_max": 40.09651565551758,
      "activations/layer8_attention_weight_min": -43.910301208496094,
      "activations/layer9_attention_weight_max": 38.11421203613281,
      "activations/layer9_attention_weight_min": -38.16303634643555,
      "epoch": 6.61,
      "learning_rate": 0.00010846742424242423,
      "loss": 2.8234,
      "step": 113750
    },
    {
      "activations/layer0_attention_weight_max": 15.764052391052246,
      "activations/layer0_attention_weight_min": -12.410341262817383,
      "activations/layer10_attention_weight_max": 33.12969970703125,
      "activations/layer10_attention_weight_min": -31.395891189575195,
      "activations/layer11_attention_weight_max": 31.518585205078125,
      "activations/layer11_attention_weight_min": -28.089954376220703,
      "activations/layer12_attention_weight_max": 33.3134765625,
      "activations/layer12_attention_weight_min": -24.55075454711914,
      "activations/layer13_attention_weight_max": 41.32976531982422,
      "activations/layer13_attention_weight_min": -31.77260398864746,
      "activations/layer14_attention_weight_max": 51.43246078491211,
      "activations/layer14_attention_weight_min": -41.07070541381836,
      "activations/layer15_attention_weight_max": 34.059940338134766,
      "activations/layer15_attention_weight_min": -28.71621322631836,
      "activations/layer16_attention_weight_max": 36.01519775390625,
      "activations/layer16_attention_weight_min": -27.663902282714844,
      "activations/layer17_attention_weight_max": 36.91604995727539,
      "activations/layer17_attention_weight_min": -27.22569465637207,
      "activations/layer18_attention_weight_max": 28.390365600585938,
      "activations/layer18_attention_weight_min": -20.30010986328125,
      "activations/layer19_attention_weight_max": 42.484375,
      "activations/layer19_attention_weight_min": -28.15358543395996,
      "activations/layer1_attention_weight_max": 16.846050262451172,
      "activations/layer1_attention_weight_min": -14.422737121582031,
      "activations/layer20_attention_weight_max": 32.19116973876953,
      "activations/layer20_attention_weight_min": -25.023420333862305,
      "activations/layer21_attention_weight_max": 30.69244384765625,
      "activations/layer21_attention_weight_min": -21.6370906829834,
      "activations/layer22_attention_weight_max": 48.65557861328125,
      "activations/layer22_attention_weight_min": -31.2678279876709,
      "activations/layer23_attention_weight_max": 36.71947479248047,
      "activations/layer23_attention_weight_min": -20.781383514404297,
      "activations/layer2_attention_weight_max": 32.000186920166016,
      "activations/layer2_attention_weight_min": -30.05294418334961,
      "activations/layer3_attention_weight_max": 90.34908294677734,
      "activations/layer3_attention_weight_min": -88.8811264038086,
      "activations/layer4_attention_weight_max": 130.245361328125,
      "activations/layer4_attention_weight_min": -120.2842788696289,
      "activations/layer5_attention_weight_max": 67.16543579101562,
      "activations/layer5_attention_weight_min": -64.35123443603516,
      "activations/layer6_attention_weight_max": 51.700233459472656,
      "activations/layer6_attention_weight_min": -52.63849639892578,
      "activations/layer7_attention_weight_max": 87.84266662597656,
      "activations/layer7_attention_weight_min": -82.59844207763672,
      "activations/layer8_attention_weight_max": 40.66804504394531,
      "activations/layer8_attention_weight_min": -39.48504638671875,
      "activations/layer9_attention_weight_max": 37.79360580444336,
      "activations/layer9_attention_weight_min": -33.83219909667969,
      "epoch": 6.61,
      "learning_rate": 0.00010844848484848484,
      "loss": 2.8315,
      "step": 113800
    },
    {
      "activations/layer0_attention_weight_max": 14.852331161499023,
      "activations/layer0_attention_weight_min": -11.46302318572998,
      "activations/layer10_attention_weight_max": 42.482967376708984,
      "activations/layer10_attention_weight_min": -33.53618240356445,
      "activations/layer11_attention_weight_max": 45.18505859375,
      "activations/layer11_attention_weight_min": -35.831443786621094,
      "activations/layer12_attention_weight_max": 32.636207580566406,
      "activations/layer12_attention_weight_min": -27.41293716430664,
      "activations/layer13_attention_weight_max": 45.55532455444336,
      "activations/layer13_attention_weight_min": -30.621482849121094,
      "activations/layer14_attention_weight_max": 69.45487213134766,
      "activations/layer14_attention_weight_min": -46.20737838745117,
      "activations/layer15_attention_weight_max": 42.76177215576172,
      "activations/layer15_attention_weight_min": -29.239242553710938,
      "activations/layer16_attention_weight_max": 41.315059661865234,
      "activations/layer16_attention_weight_min": -28.24643325805664,
      "activations/layer17_attention_weight_max": 36.83835983276367,
      "activations/layer17_attention_weight_min": -24.674560546875,
      "activations/layer18_attention_weight_max": 32.67782974243164,
      "activations/layer18_attention_weight_min": -20.936397552490234,
      "activations/layer19_attention_weight_max": 49.70745849609375,
      "activations/layer19_attention_weight_min": -31.23836326599121,
      "activations/layer1_attention_weight_max": 17.111600875854492,
      "activations/layer1_attention_weight_min": -13.62450122833252,
      "activations/layer20_attention_weight_max": 41.32412338256836,
      "activations/layer20_attention_weight_min": -23.74165153503418,
      "activations/layer21_attention_weight_max": 36.90876388549805,
      "activations/layer21_attention_weight_min": -20.67668342590332,
      "activations/layer22_attention_weight_max": 59.15391159057617,
      "activations/layer22_attention_weight_min": -28.08087158203125,
      "activations/layer23_attention_weight_max": 39.76435089111328,
      "activations/layer23_attention_weight_min": -23.604598999023438,
      "activations/layer2_attention_weight_max": 34.20989990234375,
      "activations/layer2_attention_weight_min": -31.025524139404297,
      "activations/layer3_attention_weight_max": 90.17426300048828,
      "activations/layer3_attention_weight_min": -90.51620483398438,
      "activations/layer4_attention_weight_max": 126.4158935546875,
      "activations/layer4_attention_weight_min": -124.9540023803711,
      "activations/layer5_attention_weight_max": 70.08627319335938,
      "activations/layer5_attention_weight_min": -65.35440063476562,
      "activations/layer6_attention_weight_max": 54.730873107910156,
      "activations/layer6_attention_weight_min": -53.5361213684082,
      "activations/layer7_attention_weight_max": 105.59253692626953,
      "activations/layer7_attention_weight_min": -86.83321380615234,
      "activations/layer8_attention_weight_max": 47.95232009887695,
      "activations/layer8_attention_weight_min": -42.503196716308594,
      "activations/layer9_attention_weight_max": 44.82990646362305,
      "activations/layer9_attention_weight_min": -36.69662094116211,
      "epoch": 6.62,
      "learning_rate": 0.00010842954545454545,
      "loss": 2.8341,
      "step": 113850
    },
    {
      "activations/layer0_attention_weight_max": 17.03430938720703,
      "activations/layer0_attention_weight_min": -12.60572338104248,
      "activations/layer10_attention_weight_max": 34.639469146728516,
      "activations/layer10_attention_weight_min": -31.08687973022461,
      "activations/layer11_attention_weight_max": 34.508758544921875,
      "activations/layer11_attention_weight_min": -30.379478454589844,
      "activations/layer12_attention_weight_max": 30.111860275268555,
      "activations/layer12_attention_weight_min": -31.161067962646484,
      "activations/layer13_attention_weight_max": 42.43183517456055,
      "activations/layer13_attention_weight_min": -33.59406661987305,
      "activations/layer14_attention_weight_max": 56.17914581298828,
      "activations/layer14_attention_weight_min": -39.102813720703125,
      "activations/layer15_attention_weight_max": 36.85112762451172,
      "activations/layer15_attention_weight_min": -29.781627655029297,
      "activations/layer16_attention_weight_max": 36.25431823730469,
      "activations/layer16_attention_weight_min": -27.813413619995117,
      "activations/layer17_attention_weight_max": 37.99077606201172,
      "activations/layer17_attention_weight_min": -25.910568237304688,
      "activations/layer18_attention_weight_max": 30.545333862304688,
      "activations/layer18_attention_weight_min": -21.47715950012207,
      "activations/layer19_attention_weight_max": 39.01005172729492,
      "activations/layer19_attention_weight_min": -32.10819625854492,
      "activations/layer1_attention_weight_max": 17.650850296020508,
      "activations/layer1_attention_weight_min": -14.211406707763672,
      "activations/layer20_attention_weight_max": 33.30777359008789,
      "activations/layer20_attention_weight_min": -26.055879592895508,
      "activations/layer21_attention_weight_max": 36.64026641845703,
      "activations/layer21_attention_weight_min": -23.41710662841797,
      "activations/layer22_attention_weight_max": 52.919944763183594,
      "activations/layer22_attention_weight_min": -28.90302085876465,
      "activations/layer23_attention_weight_max": 36.89775085449219,
      "activations/layer23_attention_weight_min": -23.477291107177734,
      "activations/layer2_attention_weight_max": 31.346607208251953,
      "activations/layer2_attention_weight_min": -30.03232192993164,
      "activations/layer3_attention_weight_max": 90.68767547607422,
      "activations/layer3_attention_weight_min": -91.68931579589844,
      "activations/layer4_attention_weight_max": 126.72599029541016,
      "activations/layer4_attention_weight_min": -126.7286148071289,
      "activations/layer5_attention_weight_max": 67.33871459960938,
      "activations/layer5_attention_weight_min": -65.08589935302734,
      "activations/layer6_attention_weight_max": 51.20383834838867,
      "activations/layer6_attention_weight_min": -54.31621170043945,
      "activations/layer7_attention_weight_max": 86.91896057128906,
      "activations/layer7_attention_weight_min": -92.58168029785156,
      "activations/layer8_attention_weight_max": 38.94234085083008,
      "activations/layer8_attention_weight_min": -40.082183837890625,
      "activations/layer9_attention_weight_max": 35.75502395629883,
      "activations/layer9_attention_weight_min": -35.37010955810547,
      "epoch": 6.62,
      "learning_rate": 0.00010841060606060605,
      "loss": 2.8319,
      "step": 113900
    },
    {
      "activations/layer0_attention_weight_max": 16.215421676635742,
      "activations/layer0_attention_weight_min": -12.743219375610352,
      "activations/layer10_attention_weight_max": 36.80064392089844,
      "activations/layer10_attention_weight_min": -32.9371223449707,
      "activations/layer11_attention_weight_max": 33.047515869140625,
      "activations/layer11_attention_weight_min": -28.548795700073242,
      "activations/layer12_attention_weight_max": 22.065324783325195,
      "activations/layer12_attention_weight_min": -28.05799674987793,
      "activations/layer13_attention_weight_max": 47.361289978027344,
      "activations/layer13_attention_weight_min": -32.55089569091797,
      "activations/layer14_attention_weight_max": 62.73320770263672,
      "activations/layer14_attention_weight_min": -46.17197036743164,
      "activations/layer15_attention_weight_max": 41.33793258666992,
      "activations/layer15_attention_weight_min": -26.881446838378906,
      "activations/layer16_attention_weight_max": 33.814979553222656,
      "activations/layer16_attention_weight_min": -25.250972747802734,
      "activations/layer17_attention_weight_max": 35.544944763183594,
      "activations/layer17_attention_weight_min": -23.081472396850586,
      "activations/layer18_attention_weight_max": 28.063636779785156,
      "activations/layer18_attention_weight_min": -20.053651809692383,
      "activations/layer19_attention_weight_max": 35.05952072143555,
      "activations/layer19_attention_weight_min": -28.06195068359375,
      "activations/layer1_attention_weight_max": 18.654277801513672,
      "activations/layer1_attention_weight_min": -17.102632522583008,
      "activations/layer20_attention_weight_max": 28.764442443847656,
      "activations/layer20_attention_weight_min": -25.217716217041016,
      "activations/layer21_attention_weight_max": 26.918813705444336,
      "activations/layer21_attention_weight_min": -21.364471435546875,
      "activations/layer22_attention_weight_max": 47.94812774658203,
      "activations/layer22_attention_weight_min": -28.601959228515625,
      "activations/layer23_attention_weight_max": 33.808563232421875,
      "activations/layer23_attention_weight_min": -23.070697784423828,
      "activations/layer2_attention_weight_max": 32.85986328125,
      "activations/layer2_attention_weight_min": -32.13710403442383,
      "activations/layer3_attention_weight_max": 93.71438598632812,
      "activations/layer3_attention_weight_min": -93.99249267578125,
      "activations/layer4_attention_weight_max": 138.45687866210938,
      "activations/layer4_attention_weight_min": -134.13426208496094,
      "activations/layer5_attention_weight_max": 78.03843688964844,
      "activations/layer5_attention_weight_min": -70.19883728027344,
      "activations/layer6_attention_weight_max": 57.60322189331055,
      "activations/layer6_attention_weight_min": -58.71308517456055,
      "activations/layer7_attention_weight_max": 95.73023986816406,
      "activations/layer7_attention_weight_min": -96.6849594116211,
      "activations/layer8_attention_weight_max": 44.96748733520508,
      "activations/layer8_attention_weight_min": -42.49968719482422,
      "activations/layer9_attention_weight_max": 39.96882247924805,
      "activations/layer9_attention_weight_min": -37.70295333862305,
      "epoch": 6.62,
      "learning_rate": 0.00010839166666666665,
      "loss": 2.8415,
      "step": 113950
    },
    {
      "activations/layer0_attention_weight_max": 17.066211700439453,
      "activations/layer0_attention_weight_min": -11.85865306854248,
      "activations/layer10_attention_weight_max": 35.62757873535156,
      "activations/layer10_attention_weight_min": -31.10527229309082,
      "activations/layer11_attention_weight_max": 35.46442413330078,
      "activations/layer11_attention_weight_min": -28.371509552001953,
      "activations/layer12_attention_weight_max": 22.76420783996582,
      "activations/layer12_attention_weight_min": -24.56827735900879,
      "activations/layer13_attention_weight_max": 43.193302154541016,
      "activations/layer13_attention_weight_min": -31.013599395751953,
      "activations/layer14_attention_weight_max": 59.14289474487305,
      "activations/layer14_attention_weight_min": -39.9040641784668,
      "activations/layer15_attention_weight_max": 39.607669830322266,
      "activations/layer15_attention_weight_min": -28.019826889038086,
      "activations/layer16_attention_weight_max": 35.65008544921875,
      "activations/layer16_attention_weight_min": -28.07256507873535,
      "activations/layer17_attention_weight_max": 39.1007080078125,
      "activations/layer17_attention_weight_min": -24.650619506835938,
      "activations/layer18_attention_weight_max": 35.45713424682617,
      "activations/layer18_attention_weight_min": -21.077171325683594,
      "activations/layer19_attention_weight_max": 38.74868392944336,
      "activations/layer19_attention_weight_min": -29.95025062561035,
      "activations/layer1_attention_weight_max": 16.667057037353516,
      "activations/layer1_attention_weight_min": -14.260689735412598,
      "activations/layer20_attention_weight_max": 35.87897872924805,
      "activations/layer20_attention_weight_min": -24.253074645996094,
      "activations/layer21_attention_weight_max": 37.548458099365234,
      "activations/layer21_attention_weight_min": -21.9345760345459,
      "activations/layer22_attention_weight_max": 55.58396530151367,
      "activations/layer22_attention_weight_min": -28.959009170532227,
      "activations/layer23_attention_weight_max": 37.21546173095703,
      "activations/layer23_attention_weight_min": -21.289794921875,
      "activations/layer2_attention_weight_max": 29.73982048034668,
      "activations/layer2_attention_weight_min": -29.300453186035156,
      "activations/layer3_attention_weight_max": 82.91031646728516,
      "activations/layer3_attention_weight_min": -83.498779296875,
      "activations/layer4_attention_weight_max": 121.00142669677734,
      "activations/layer4_attention_weight_min": -123.86309051513672,
      "activations/layer5_attention_weight_max": 65.21891021728516,
      "activations/layer5_attention_weight_min": -63.19029998779297,
      "activations/layer6_attention_weight_max": 52.864742279052734,
      "activations/layer6_attention_weight_min": -52.48598098754883,
      "activations/layer7_attention_weight_max": 88.14167022705078,
      "activations/layer7_attention_weight_min": -87.40866088867188,
      "activations/layer8_attention_weight_max": 43.647621154785156,
      "activations/layer8_attention_weight_min": -40.066749572753906,
      "activations/layer9_attention_weight_max": 36.332763671875,
      "activations/layer9_attention_weight_min": -34.85346984863281,
      "epoch": 6.62,
      "learning_rate": 0.00010837272727272727,
      "loss": 2.8328,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_loss": 2.77734375,
      "eval_runtime": 8.5452,
      "eval_samples_per_second": 502.504,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_openwebtext_loss": 2.77734375,
      "eval_openwebtext_ppl": 16.076261613516785,
      "eval_openwebtext_runtime": 8.5452,
      "eval_openwebtext_samples_per_second": 502.504,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 1.9903,
      "eval_wikitext_samples_per_second": 229.11,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_lambada_loss": 2.69921875,
      "eval_lambada_ppl": 14.868111474214228,
      "eval_lambada_runtime": 9.5904,
      "eval_lambada_samples_per_second": 507.695,
      "step": 114000
    },
    {
      "activations/layer0_attention_weight_max": 16.14271354675293,
      "activations/layer0_attention_weight_min": -12.169014930725098,
      "activations/layer10_attention_weight_max": 33.2756233215332,
      "activations/layer10_attention_weight_min": -30.126644134521484,
      "activations/layer11_attention_weight_max": 33.193477630615234,
      "activations/layer11_attention_weight_min": -29.5531063079834,
      "activations/layer12_attention_weight_max": 26.84286880493164,
      "activations/layer12_attention_weight_min": -31.089879989624023,
      "activations/layer13_attention_weight_max": 42.810890197753906,
      "activations/layer13_attention_weight_min": -32.40012741088867,
      "activations/layer14_attention_weight_max": 59.459407806396484,
      "activations/layer14_attention_weight_min": -41.438987731933594,
      "activations/layer15_attention_weight_max": 38.663509368896484,
      "activations/layer15_attention_weight_min": -28.462907791137695,
      "activations/layer16_attention_weight_max": 36.111419677734375,
      "activations/layer16_attention_weight_min": -28.011789321899414,
      "activations/layer17_attention_weight_max": 39.87703323364258,
      "activations/layer17_attention_weight_min": -26.36455726623535,
      "activations/layer18_attention_weight_max": 30.382732391357422,
      "activations/layer18_attention_weight_min": -24.213090896606445,
      "activations/layer19_attention_weight_max": 40.08203125,
      "activations/layer19_attention_weight_min": -29.888227462768555,
      "activations/layer1_attention_weight_max": 17.189023971557617,
      "activations/layer1_attention_weight_min": -15.017815589904785,
      "activations/layer20_attention_weight_max": 37.33501434326172,
      "activations/layer20_attention_weight_min": -26.547313690185547,
      "activations/layer21_attention_weight_max": 34.774879455566406,
      "activations/layer21_attention_weight_min": -23.28997802734375,
      "activations/layer22_attention_weight_max": 51.44850158691406,
      "activations/layer22_attention_weight_min": -30.368799209594727,
      "activations/layer23_attention_weight_max": 37.30772399902344,
      "activations/layer23_attention_weight_min": -22.2103328704834,
      "activations/layer2_attention_weight_max": 31.627302169799805,
      "activations/layer2_attention_weight_min": -28.886810302734375,
      "activations/layer3_attention_weight_max": 88.56670379638672,
      "activations/layer3_attention_weight_min": -88.28192138671875,
      "activations/layer4_attention_weight_max": 133.8076171875,
      "activations/layer4_attention_weight_min": -120.71044921875,
      "activations/layer5_attention_weight_max": 66.50135040283203,
      "activations/layer5_attention_weight_min": -65.1138916015625,
      "activations/layer6_attention_weight_max": 52.59324264526367,
      "activations/layer6_attention_weight_min": -53.16035079956055,
      "activations/layer7_attention_weight_max": 86.56939697265625,
      "activations/layer7_attention_weight_min": -92.02566528320312,
      "activations/layer8_attention_weight_max": 41.596778869628906,
      "activations/layer8_attention_weight_min": -40.78620147705078,
      "activations/layer9_attention_weight_max": 37.671119689941406,
      "activations/layer9_attention_weight_min": -34.80721664428711,
      "epoch": 6.63,
      "learning_rate": 0.00010835378787878786,
      "loss": 2.8233,
      "step": 114050
    },
    {
      "activations/layer0_attention_weight_max": 16.306049346923828,
      "activations/layer0_attention_weight_min": -11.651576042175293,
      "activations/layer10_attention_weight_max": 35.9775276184082,
      "activations/layer10_attention_weight_min": -31.880962371826172,
      "activations/layer11_attention_weight_max": 34.1343879699707,
      "activations/layer11_attention_weight_min": -29.637378692626953,
      "activations/layer12_attention_weight_max": 22.77436065673828,
      "activations/layer12_attention_weight_min": -26.872058868408203,
      "activations/layer13_attention_weight_max": 42.70781707763672,
      "activations/layer13_attention_weight_min": -32.44028854370117,
      "activations/layer14_attention_weight_max": 55.342586517333984,
      "activations/layer14_attention_weight_min": -39.65585708618164,
      "activations/layer15_attention_weight_max": 38.02024459838867,
      "activations/layer15_attention_weight_min": -29.680830001831055,
      "activations/layer16_attention_weight_max": 34.65439224243164,
      "activations/layer16_attention_weight_min": -27.265424728393555,
      "activations/layer17_attention_weight_max": 38.92026138305664,
      "activations/layer17_attention_weight_min": -24.485116958618164,
      "activations/layer18_attention_weight_max": 30.995759963989258,
      "activations/layer18_attention_weight_min": -21.714046478271484,
      "activations/layer19_attention_weight_max": 39.317840576171875,
      "activations/layer19_attention_weight_min": -29.207406997680664,
      "activations/layer1_attention_weight_max": 15.899833679199219,
      "activations/layer1_attention_weight_min": -15.063647270202637,
      "activations/layer20_attention_weight_max": 30.912635803222656,
      "activations/layer20_attention_weight_min": -23.951047897338867,
      "activations/layer21_attention_weight_max": 29.364187240600586,
      "activations/layer21_attention_weight_min": -22.531108856201172,
      "activations/layer22_attention_weight_max": 49.55022430419922,
      "activations/layer22_attention_weight_min": -28.33257484436035,
      "activations/layer23_attention_weight_max": 38.39927291870117,
      "activations/layer23_attention_weight_min": -21.654415130615234,
      "activations/layer2_attention_weight_max": 30.424549102783203,
      "activations/layer2_attention_weight_min": -30.86860466003418,
      "activations/layer3_attention_weight_max": 85.26567077636719,
      "activations/layer3_attention_weight_min": -87.772216796875,
      "activations/layer4_attention_weight_max": 121.49029541015625,
      "activations/layer4_attention_weight_min": -123.55301666259766,
      "activations/layer5_attention_weight_max": 65.44696807861328,
      "activations/layer5_attention_weight_min": -68.85427856445312,
      "activations/layer6_attention_weight_max": 51.75631332397461,
      "activations/layer6_attention_weight_min": -52.61881637573242,
      "activations/layer7_attention_weight_max": 84.95677185058594,
      "activations/layer7_attention_weight_min": -89.3447265625,
      "activations/layer8_attention_weight_max": 39.32188034057617,
      "activations/layer8_attention_weight_min": -39.28267288208008,
      "activations/layer9_attention_weight_max": 39.70690155029297,
      "activations/layer9_attention_weight_min": -35.1805534362793,
      "epoch": 6.63,
      "learning_rate": 0.00010833484848484847,
      "loss": 2.8217,
      "step": 114100
    },
    {
      "activations/layer0_attention_weight_max": 14.779328346252441,
      "activations/layer0_attention_weight_min": -11.484200477600098,
      "activations/layer10_attention_weight_max": 33.5030517578125,
      "activations/layer10_attention_weight_min": -30.11925506591797,
      "activations/layer11_attention_weight_max": 33.09321212768555,
      "activations/layer11_attention_weight_min": -28.914546966552734,
      "activations/layer12_attention_weight_max": 25.902616500854492,
      "activations/layer12_attention_weight_min": -29.82750129699707,
      "activations/layer13_attention_weight_max": 38.94994354248047,
      "activations/layer13_attention_weight_min": -31.936410903930664,
      "activations/layer14_attention_weight_max": 62.084991455078125,
      "activations/layer14_attention_weight_min": -43.36073684692383,
      "activations/layer15_attention_weight_max": 39.11360549926758,
      "activations/layer15_attention_weight_min": -29.63631820678711,
      "activations/layer16_attention_weight_max": 34.151485443115234,
      "activations/layer16_attention_weight_min": -26.68893814086914,
      "activations/layer17_attention_weight_max": 35.51359939575195,
      "activations/layer17_attention_weight_min": -25.05594825744629,
      "activations/layer18_attention_weight_max": 33.34321212768555,
      "activations/layer18_attention_weight_min": -21.572179794311523,
      "activations/layer19_attention_weight_max": 36.52007293701172,
      "activations/layer19_attention_weight_min": -31.591846466064453,
      "activations/layer1_attention_weight_max": 15.902359008789062,
      "activations/layer1_attention_weight_min": -15.007285118103027,
      "activations/layer20_attention_weight_max": 28.499082565307617,
      "activations/layer20_attention_weight_min": -24.6039981842041,
      "activations/layer21_attention_weight_max": 28.607040405273438,
      "activations/layer21_attention_weight_min": -21.808958053588867,
      "activations/layer22_attention_weight_max": 46.34248733520508,
      "activations/layer22_attention_weight_min": -29.395477294921875,
      "activations/layer23_attention_weight_max": 34.437171936035156,
      "activations/layer23_attention_weight_min": -22.84107780456543,
      "activations/layer2_attention_weight_max": 30.9299373626709,
      "activations/layer2_attention_weight_min": -28.334583282470703,
      "activations/layer3_attention_weight_max": 88.6131820678711,
      "activations/layer3_attention_weight_min": -84.87837982177734,
      "activations/layer4_attention_weight_max": 123.03026580810547,
      "activations/layer4_attention_weight_min": -124.3980941772461,
      "activations/layer5_attention_weight_max": 67.16903686523438,
      "activations/layer5_attention_weight_min": -64.40646362304688,
      "activations/layer6_attention_weight_max": 52.20853805541992,
      "activations/layer6_attention_weight_min": -51.5671272277832,
      "activations/layer7_attention_weight_max": 89.6572265625,
      "activations/layer7_attention_weight_min": -86.15552520751953,
      "activations/layer8_attention_weight_max": 40.21333694458008,
      "activations/layer8_attention_weight_min": -38.97146224975586,
      "activations/layer9_attention_weight_max": 35.67864227294922,
      "activations/layer9_attention_weight_min": -35.22433090209961,
      "epoch": 6.63,
      "learning_rate": 0.00010831590909090909,
      "loss": 2.8238,
      "step": 114150
    },
    {
      "activations/layer0_attention_weight_max": 17.105863571166992,
      "activations/layer0_attention_weight_min": -12.237689971923828,
      "activations/layer10_attention_weight_max": 33.54011917114258,
      "activations/layer10_attention_weight_min": -31.438894271850586,
      "activations/layer11_attention_weight_max": 33.80810546875,
      "activations/layer11_attention_weight_min": -29.757719039916992,
      "activations/layer12_attention_weight_max": 22.8143253326416,
      "activations/layer12_attention_weight_min": -31.56241226196289,
      "activations/layer13_attention_weight_max": 41.03300857543945,
      "activations/layer13_attention_weight_min": -30.513803482055664,
      "activations/layer14_attention_weight_max": 52.23363494873047,
      "activations/layer14_attention_weight_min": -42.69706726074219,
      "activations/layer15_attention_weight_max": 33.65043258666992,
      "activations/layer15_attention_weight_min": -29.379295349121094,
      "activations/layer16_attention_weight_max": 30.667407989501953,
      "activations/layer16_attention_weight_min": -26.930028915405273,
      "activations/layer17_attention_weight_max": 34.378700256347656,
      "activations/layer17_attention_weight_min": -24.728466033935547,
      "activations/layer18_attention_weight_max": 31.631126403808594,
      "activations/layer18_attention_weight_min": -20.205549240112305,
      "activations/layer19_attention_weight_max": 38.03337860107422,
      "activations/layer19_attention_weight_min": -29.77326774597168,
      "activations/layer1_attention_weight_max": 17.24019432067871,
      "activations/layer1_attention_weight_min": -13.815836906433105,
      "activations/layer20_attention_weight_max": 31.62999153137207,
      "activations/layer20_attention_weight_min": -25.43728256225586,
      "activations/layer21_attention_weight_max": 32.39776611328125,
      "activations/layer21_attention_weight_min": -22.42621421813965,
      "activations/layer22_attention_weight_max": 52.056148529052734,
      "activations/layer22_attention_weight_min": -29.31527328491211,
      "activations/layer23_attention_weight_max": 36.17519760131836,
      "activations/layer23_attention_weight_min": -22.932308197021484,
      "activations/layer2_attention_weight_max": 34.59591293334961,
      "activations/layer2_attention_weight_min": -32.82884216308594,
      "activations/layer3_attention_weight_max": 99.34288024902344,
      "activations/layer3_attention_weight_min": -96.73039245605469,
      "activations/layer4_attention_weight_max": 152.76194763183594,
      "activations/layer4_attention_weight_min": -134.72900390625,
      "activations/layer5_attention_weight_max": 69.396240234375,
      "activations/layer5_attention_weight_min": -68.64723205566406,
      "activations/layer6_attention_weight_max": 55.82848358154297,
      "activations/layer6_attention_weight_min": -55.5026741027832,
      "activations/layer7_attention_weight_max": 94.10075378417969,
      "activations/layer7_attention_weight_min": -91.97437286376953,
      "activations/layer8_attention_weight_max": 43.90518569946289,
      "activations/layer8_attention_weight_min": -43.60367965698242,
      "activations/layer9_attention_weight_max": 35.2055778503418,
      "activations/layer9_attention_weight_min": -43.12445068359375,
      "epoch": 6.64,
      "learning_rate": 0.00010829696969696969,
      "loss": 2.8222,
      "step": 114200
    },
    {
      "activations/layer0_attention_weight_max": 14.277116775512695,
      "activations/layer0_attention_weight_min": -12.705121040344238,
      "activations/layer10_attention_weight_max": 36.75802230834961,
      "activations/layer10_attention_weight_min": -33.17774200439453,
      "activations/layer11_attention_weight_max": 35.264930725097656,
      "activations/layer11_attention_weight_min": -30.02179527282715,
      "activations/layer12_attention_weight_max": 43.35373306274414,
      "activations/layer12_attention_weight_min": -25.902301788330078,
      "activations/layer13_attention_weight_max": 50.57358169555664,
      "activations/layer13_attention_weight_min": -31.86912727355957,
      "activations/layer14_attention_weight_max": 67.74205780029297,
      "activations/layer14_attention_weight_min": -43.78138732910156,
      "activations/layer15_attention_weight_max": 40.41973114013672,
      "activations/layer15_attention_weight_min": -31.22184181213379,
      "activations/layer16_attention_weight_max": 41.16824722290039,
      "activations/layer16_attention_weight_min": -28.187175750732422,
      "activations/layer17_attention_weight_max": 39.37749481201172,
      "activations/layer17_attention_weight_min": -25.334497451782227,
      "activations/layer18_attention_weight_max": 34.947269439697266,
      "activations/layer18_attention_weight_min": -22.128021240234375,
      "activations/layer19_attention_weight_max": 44.627017974853516,
      "activations/layer19_attention_weight_min": -29.62122917175293,
      "activations/layer1_attention_weight_max": 17.735416412353516,
      "activations/layer1_attention_weight_min": -13.13160228729248,
      "activations/layer20_attention_weight_max": 37.98687744140625,
      "activations/layer20_attention_weight_min": -24.069440841674805,
      "activations/layer21_attention_weight_max": 33.85294723510742,
      "activations/layer21_attention_weight_min": -21.747446060180664,
      "activations/layer22_attention_weight_max": 60.13523483276367,
      "activations/layer22_attention_weight_min": -30.964061737060547,
      "activations/layer23_attention_weight_max": 38.97654724121094,
      "activations/layer23_attention_weight_min": -24.194631576538086,
      "activations/layer2_attention_weight_max": 29.867252349853516,
      "activations/layer2_attention_weight_min": -29.70928955078125,
      "activations/layer3_attention_weight_max": 86.2061996459961,
      "activations/layer3_attention_weight_min": -90.45886993408203,
      "activations/layer4_attention_weight_max": 120.22164154052734,
      "activations/layer4_attention_weight_min": -119.72149658203125,
      "activations/layer5_attention_weight_max": 65.88462829589844,
      "activations/layer5_attention_weight_min": -65.53971862792969,
      "activations/layer6_attention_weight_max": 51.97157669067383,
      "activations/layer6_attention_weight_min": -53.159385681152344,
      "activations/layer7_attention_weight_max": 88.89522552490234,
      "activations/layer7_attention_weight_min": -85.90087127685547,
      "activations/layer8_attention_weight_max": 44.21199417114258,
      "activations/layer8_attention_weight_min": -43.2163200378418,
      "activations/layer9_attention_weight_max": 38.36778259277344,
      "activations/layer9_attention_weight_min": -35.448116302490234,
      "epoch": 6.64,
      "learning_rate": 0.00010827840909090908,
      "loss": 2.812,
      "step": 114250
    },
    {
      "activations/layer0_attention_weight_max": 16.48714256286621,
      "activations/layer0_attention_weight_min": -12.183920860290527,
      "activations/layer10_attention_weight_max": 31.959529876708984,
      "activations/layer10_attention_weight_min": -32.359535217285156,
      "activations/layer11_attention_weight_max": 31.84784507751465,
      "activations/layer11_attention_weight_min": -29.115774154663086,
      "activations/layer12_attention_weight_max": 24.517898559570312,
      "activations/layer12_attention_weight_min": -25.768306732177734,
      "activations/layer13_attention_weight_max": 37.40711212158203,
      "activations/layer13_attention_weight_min": -35.4095458984375,
      "activations/layer14_attention_weight_max": 56.050559997558594,
      "activations/layer14_attention_weight_min": -43.877445220947266,
      "activations/layer15_attention_weight_max": 34.455387115478516,
      "activations/layer15_attention_weight_min": -32.178218841552734,
      "activations/layer16_attention_weight_max": 33.744022369384766,
      "activations/layer16_attention_weight_min": -29.09453582763672,
      "activations/layer17_attention_weight_max": 33.104576110839844,
      "activations/layer17_attention_weight_min": -24.308513641357422,
      "activations/layer18_attention_weight_max": 30.255939483642578,
      "activations/layer18_attention_weight_min": -20.852996826171875,
      "activations/layer19_attention_weight_max": 36.443077087402344,
      "activations/layer19_attention_weight_min": -32.01863098144531,
      "activations/layer1_attention_weight_max": 16.58866310119629,
      "activations/layer1_attention_weight_min": -14.011614799499512,
      "activations/layer20_attention_weight_max": 29.578075408935547,
      "activations/layer20_attention_weight_min": -25.75938606262207,
      "activations/layer21_attention_weight_max": 28.12651252746582,
      "activations/layer21_attention_weight_min": -22.08402442932129,
      "activations/layer22_attention_weight_max": 44.607093811035156,
      "activations/layer22_attention_weight_min": -27.91563606262207,
      "activations/layer23_attention_weight_max": 32.278526306152344,
      "activations/layer23_attention_weight_min": -21.176454544067383,
      "activations/layer2_attention_weight_max": 30.261661529541016,
      "activations/layer2_attention_weight_min": -29.94868278503418,
      "activations/layer3_attention_weight_max": 90.72026824951172,
      "activations/layer3_attention_weight_min": -93.44727325439453,
      "activations/layer4_attention_weight_max": 129.75318908691406,
      "activations/layer4_attention_weight_min": -125.81971740722656,
      "activations/layer5_attention_weight_max": 65.31177520751953,
      "activations/layer5_attention_weight_min": -65.70901489257812,
      "activations/layer6_attention_weight_max": 51.868282318115234,
      "activations/layer6_attention_weight_min": -54.333946228027344,
      "activations/layer7_attention_weight_max": 88.0645751953125,
      "activations/layer7_attention_weight_min": -86.22083282470703,
      "activations/layer8_attention_weight_max": 38.42288589477539,
      "activations/layer8_attention_weight_min": -41.48020935058594,
      "activations/layer9_attention_weight_max": 34.0682487487793,
      "activations/layer9_attention_weight_min": -32.80006408691406,
      "epoch": 6.64,
      "learning_rate": 0.00010825946969696968,
      "loss": 2.8255,
      "step": 114300
    },
    {
      "activations/layer0_attention_weight_max": 16.315378189086914,
      "activations/layer0_attention_weight_min": -11.9915132522583,
      "activations/layer10_attention_weight_max": 33.6632194519043,
      "activations/layer10_attention_weight_min": -29.459075927734375,
      "activations/layer11_attention_weight_max": 32.92811584472656,
      "activations/layer11_attention_weight_min": -28.183338165283203,
      "activations/layer12_attention_weight_max": 22.685871124267578,
      "activations/layer12_attention_weight_min": -26.85404396057129,
      "activations/layer13_attention_weight_max": 41.99293899536133,
      "activations/layer13_attention_weight_min": -32.64316940307617,
      "activations/layer14_attention_weight_max": 54.192527770996094,
      "activations/layer14_attention_weight_min": -39.78456497192383,
      "activations/layer15_attention_weight_max": 36.152713775634766,
      "activations/layer15_attention_weight_min": -28.508275985717773,
      "activations/layer16_attention_weight_max": 34.12709426879883,
      "activations/layer16_attention_weight_min": -27.422771453857422,
      "activations/layer17_attention_weight_max": 34.545475006103516,
      "activations/layer17_attention_weight_min": -24.4835262298584,
      "activations/layer18_attention_weight_max": 30.27779769897461,
      "activations/layer18_attention_weight_min": -19.562400817871094,
      "activations/layer19_attention_weight_max": 37.78825759887695,
      "activations/layer19_attention_weight_min": -28.082843780517578,
      "activations/layer1_attention_weight_max": 16.420021057128906,
      "activations/layer1_attention_weight_min": -12.341906547546387,
      "activations/layer20_attention_weight_max": 29.492992401123047,
      "activations/layer20_attention_weight_min": -22.36742401123047,
      "activations/layer21_attention_weight_max": 26.247220993041992,
      "activations/layer21_attention_weight_min": -22.098529815673828,
      "activations/layer22_attention_weight_max": 45.26133728027344,
      "activations/layer22_attention_weight_min": -27.43723487854004,
      "activations/layer23_attention_weight_max": 33.89173889160156,
      "activations/layer23_attention_weight_min": -22.04011344909668,
      "activations/layer2_attention_weight_max": 31.281719207763672,
      "activations/layer2_attention_weight_min": -29.801631927490234,
      "activations/layer3_attention_weight_max": 87.82792663574219,
      "activations/layer3_attention_weight_min": -88.7821044921875,
      "activations/layer4_attention_weight_max": 123.8552474975586,
      "activations/layer4_attention_weight_min": -125.07381439208984,
      "activations/layer5_attention_weight_max": 64.93438720703125,
      "activations/layer5_attention_weight_min": -64.04610443115234,
      "activations/layer6_attention_weight_max": 52.0450325012207,
      "activations/layer6_attention_weight_min": -52.117977142333984,
      "activations/layer7_attention_weight_max": 88.0599594116211,
      "activations/layer7_attention_weight_min": -79.9368667602539,
      "activations/layer8_attention_weight_max": 39.566959381103516,
      "activations/layer8_attention_weight_min": -38.731346130371094,
      "activations/layer9_attention_weight_max": 34.561676025390625,
      "activations/layer9_attention_weight_min": -31.254016876220703,
      "epoch": 6.64,
      "learning_rate": 0.0001082405303030303,
      "loss": 2.838,
      "step": 114350
    },
    {
      "activations/layer0_attention_weight_max": 16.44989776611328,
      "activations/layer0_attention_weight_min": -12.89980411529541,
      "activations/layer10_attention_weight_max": 33.52569580078125,
      "activations/layer10_attention_weight_min": -31.048507690429688,
      "activations/layer11_attention_weight_max": 36.43561553955078,
      "activations/layer11_attention_weight_min": -28.278553009033203,
      "activations/layer12_attention_weight_max": 23.06000328063965,
      "activations/layer12_attention_weight_min": -29.77431869506836,
      "activations/layer13_attention_weight_max": 42.759098052978516,
      "activations/layer13_attention_weight_min": -32.83258819580078,
      "activations/layer14_attention_weight_max": 63.69619369506836,
      "activations/layer14_attention_weight_min": -46.63694763183594,
      "activations/layer15_attention_weight_max": 38.431541442871094,
      "activations/layer15_attention_weight_min": -30.32221031188965,
      "activations/layer16_attention_weight_max": 34.127471923828125,
      "activations/layer16_attention_weight_min": -28.360979080200195,
      "activations/layer17_attention_weight_max": 37.812782287597656,
      "activations/layer17_attention_weight_min": -25.364055633544922,
      "activations/layer18_attention_weight_max": 33.6718635559082,
      "activations/layer18_attention_weight_min": -21.11532211303711,
      "activations/layer19_attention_weight_max": 39.495704650878906,
      "activations/layer19_attention_weight_min": -35.28377914428711,
      "activations/layer1_attention_weight_max": 16.66307830810547,
      "activations/layer1_attention_weight_min": -13.389228820800781,
      "activations/layer20_attention_weight_max": 33.598079681396484,
      "activations/layer20_attention_weight_min": -25.63507652282715,
      "activations/layer21_attention_weight_max": 32.29344940185547,
      "activations/layer21_attention_weight_min": -23.1591796875,
      "activations/layer22_attention_weight_max": 51.99750518798828,
      "activations/layer22_attention_weight_min": -27.800033569335938,
      "activations/layer23_attention_weight_max": 36.14771270751953,
      "activations/layer23_attention_weight_min": -22.09846305847168,
      "activations/layer2_attention_weight_max": 31.492368698120117,
      "activations/layer2_attention_weight_min": -31.42955780029297,
      "activations/layer3_attention_weight_max": 87.02189636230469,
      "activations/layer3_attention_weight_min": -90.30815124511719,
      "activations/layer4_attention_weight_max": 124.01207733154297,
      "activations/layer4_attention_weight_min": -120.01861572265625,
      "activations/layer5_attention_weight_max": 68.0242919921875,
      "activations/layer5_attention_weight_min": -63.30973815917969,
      "activations/layer6_attention_weight_max": 52.327945709228516,
      "activations/layer6_attention_weight_min": -53.318721771240234,
      "activations/layer7_attention_weight_max": 90.25916290283203,
      "activations/layer7_attention_weight_min": -86.91265869140625,
      "activations/layer8_attention_weight_max": 41.76294708251953,
      "activations/layer8_attention_weight_min": -42.26005554199219,
      "activations/layer9_attention_weight_max": 36.67911148071289,
      "activations/layer9_attention_weight_min": -37.48468780517578,
      "epoch": 6.65,
      "learning_rate": 0.0001082215909090909,
      "loss": 2.8482,
      "step": 114400
    },
    {
      "activations/layer0_attention_weight_max": 15.132387161254883,
      "activations/layer0_attention_weight_min": -12.314689636230469,
      "activations/layer10_attention_weight_max": 35.57632064819336,
      "activations/layer10_attention_weight_min": -34.33799743652344,
      "activations/layer11_attention_weight_max": 35.44419860839844,
      "activations/layer11_attention_weight_min": -30.54830551147461,
      "activations/layer12_attention_weight_max": 31.465906143188477,
      "activations/layer12_attention_weight_min": -27.989566802978516,
      "activations/layer13_attention_weight_max": 47.01757049560547,
      "activations/layer13_attention_weight_min": -35.26282501220703,
      "activations/layer14_attention_weight_max": 59.564918518066406,
      "activations/layer14_attention_weight_min": -45.06403732299805,
      "activations/layer15_attention_weight_max": 44.618003845214844,
      "activations/layer15_attention_weight_min": -31.084728240966797,
      "activations/layer16_attention_weight_max": 34.98868179321289,
      "activations/layer16_attention_weight_min": -29.265531539916992,
      "activations/layer17_attention_weight_max": 42.189823150634766,
      "activations/layer17_attention_weight_min": -26.24747085571289,
      "activations/layer18_attention_weight_max": 43.49102020263672,
      "activations/layer18_attention_weight_min": -21.252965927124023,
      "activations/layer19_attention_weight_max": 41.073482513427734,
      "activations/layer19_attention_weight_min": -31.348318099975586,
      "activations/layer1_attention_weight_max": 17.986286163330078,
      "activations/layer1_attention_weight_min": -14.89444637298584,
      "activations/layer20_attention_weight_max": 33.20467758178711,
      "activations/layer20_attention_weight_min": -26.534757614135742,
      "activations/layer21_attention_weight_max": 31.501834869384766,
      "activations/layer21_attention_weight_min": -22.717449188232422,
      "activations/layer22_attention_weight_max": 55.71548843383789,
      "activations/layer22_attention_weight_min": -30.39457130432129,
      "activations/layer23_attention_weight_max": 40.548316955566406,
      "activations/layer23_attention_weight_min": -22.239591598510742,
      "activations/layer2_attention_weight_max": 32.85669708251953,
      "activations/layer2_attention_weight_min": -31.54810905456543,
      "activations/layer3_attention_weight_max": 93.9752426147461,
      "activations/layer3_attention_weight_min": -96.33099365234375,
      "activations/layer4_attention_weight_max": 132.1334228515625,
      "activations/layer4_attention_weight_min": -132.79373168945312,
      "activations/layer5_attention_weight_max": 70.89968872070312,
      "activations/layer5_attention_weight_min": -69.48181915283203,
      "activations/layer6_attention_weight_max": 56.14011001586914,
      "activations/layer6_attention_weight_min": -56.98357391357422,
      "activations/layer7_attention_weight_max": 92.21229553222656,
      "activations/layer7_attention_weight_min": -95.01473236083984,
      "activations/layer8_attention_weight_max": 42.72235107421875,
      "activations/layer8_attention_weight_min": -44.82277297973633,
      "activations/layer9_attention_weight_max": 38.3822135925293,
      "activations/layer9_attention_weight_min": -38.982688903808594,
      "epoch": 6.65,
      "learning_rate": 0.00010820265151515151,
      "loss": 2.8375,
      "step": 114450
    },
    {
      "activations/layer0_attention_weight_max": 16.188724517822266,
      "activations/layer0_attention_weight_min": -12.767692565917969,
      "activations/layer10_attention_weight_max": 33.0794563293457,
      "activations/layer10_attention_weight_min": -33.046146392822266,
      "activations/layer11_attention_weight_max": 33.3013801574707,
      "activations/layer11_attention_weight_min": -30.27503204345703,
      "activations/layer12_attention_weight_max": 23.15691375732422,
      "activations/layer12_attention_weight_min": -26.104310989379883,
      "activations/layer13_attention_weight_max": 42.5694694519043,
      "activations/layer13_attention_weight_min": -32.72797393798828,
      "activations/layer14_attention_weight_max": 62.32985305786133,
      "activations/layer14_attention_weight_min": -42.64883041381836,
      "activations/layer15_attention_weight_max": 38.55442428588867,
      "activations/layer15_attention_weight_min": -31.512685775756836,
      "activations/layer16_attention_weight_max": 35.651893615722656,
      "activations/layer16_attention_weight_min": -26.986114501953125,
      "activations/layer17_attention_weight_max": 34.84897232055664,
      "activations/layer17_attention_weight_min": -27.39752960205078,
      "activations/layer18_attention_weight_max": 30.407089233398438,
      "activations/layer18_attention_weight_min": -21.464290618896484,
      "activations/layer19_attention_weight_max": 39.25937271118164,
      "activations/layer19_attention_weight_min": -31.824697494506836,
      "activations/layer1_attention_weight_max": 16.00153160095215,
      "activations/layer1_attention_weight_min": -14.073378562927246,
      "activations/layer20_attention_weight_max": 31.779394149780273,
      "activations/layer20_attention_weight_min": -26.114439010620117,
      "activations/layer21_attention_weight_max": 28.349843978881836,
      "activations/layer21_attention_weight_min": -22.646148681640625,
      "activations/layer22_attention_weight_max": 51.307559967041016,
      "activations/layer22_attention_weight_min": -30.320486068725586,
      "activations/layer23_attention_weight_max": 40.808204650878906,
      "activations/layer23_attention_weight_min": -22.80975914001465,
      "activations/layer2_attention_weight_max": 31.736967086791992,
      "activations/layer2_attention_weight_min": -30.19391441345215,
      "activations/layer3_attention_weight_max": 86.84085083007812,
      "activations/layer3_attention_weight_min": -88.45763397216797,
      "activations/layer4_attention_weight_max": 122.92179107666016,
      "activations/layer4_attention_weight_min": -122.93025207519531,
      "activations/layer5_attention_weight_max": 65.97195434570312,
      "activations/layer5_attention_weight_min": -69.76610565185547,
      "activations/layer6_attention_weight_max": 51.321083068847656,
      "activations/layer6_attention_weight_min": -53.722389221191406,
      "activations/layer7_attention_weight_max": 89.67028045654297,
      "activations/layer7_attention_weight_min": -88.2601318359375,
      "activations/layer8_attention_weight_max": 38.52101516723633,
      "activations/layer8_attention_weight_min": -41.46171951293945,
      "activations/layer9_attention_weight_max": 36.61335372924805,
      "activations/layer9_attention_weight_min": -35.487693786621094,
      "epoch": 6.65,
      "learning_rate": 0.00010818371212121211,
      "loss": 2.8217,
      "step": 114500
    },
    {
      "activations/layer0_attention_weight_max": 15.767810821533203,
      "activations/layer0_attention_weight_min": -12.302515983581543,
      "activations/layer10_attention_weight_max": 31.763647079467773,
      "activations/layer10_attention_weight_min": -29.867855072021484,
      "activations/layer11_attention_weight_max": 30.94275665283203,
      "activations/layer11_attention_weight_min": -27.66130828857422,
      "activations/layer12_attention_weight_max": 21.863311767578125,
      "activations/layer12_attention_weight_min": -26.8537654876709,
      "activations/layer13_attention_weight_max": 40.73973846435547,
      "activations/layer13_attention_weight_min": -33.813201904296875,
      "activations/layer14_attention_weight_max": 54.94540786743164,
      "activations/layer14_attention_weight_min": -41.63041687011719,
      "activations/layer15_attention_weight_max": 36.42421340942383,
      "activations/layer15_attention_weight_min": -29.329185485839844,
      "activations/layer16_attention_weight_max": 36.39332962036133,
      "activations/layer16_attention_weight_min": -28.29758071899414,
      "activations/layer17_attention_weight_max": 34.670677185058594,
      "activations/layer17_attention_weight_min": -25.99700927734375,
      "activations/layer18_attention_weight_max": 33.81017303466797,
      "activations/layer18_attention_weight_min": -21.64598846435547,
      "activations/layer19_attention_weight_max": 40.58444595336914,
      "activations/layer19_attention_weight_min": -31.68796730041504,
      "activations/layer1_attention_weight_max": 17.083616256713867,
      "activations/layer1_attention_weight_min": -15.147128105163574,
      "activations/layer20_attention_weight_max": 32.1405029296875,
      "activations/layer20_attention_weight_min": -25.60061264038086,
      "activations/layer21_attention_weight_max": 31.735551834106445,
      "activations/layer21_attention_weight_min": -23.408123016357422,
      "activations/layer22_attention_weight_max": 52.02531051635742,
      "activations/layer22_attention_weight_min": -30.42330551147461,
      "activations/layer23_attention_weight_max": 39.69081497192383,
      "activations/layer23_attention_weight_min": -22.925796508789062,
      "activations/layer2_attention_weight_max": 32.700843811035156,
      "activations/layer2_attention_weight_min": -29.758298873901367,
      "activations/layer3_attention_weight_max": 92.2015380859375,
      "activations/layer3_attention_weight_min": -91.35633850097656,
      "activations/layer4_attention_weight_max": 129.04527282714844,
      "activations/layer4_attention_weight_min": -125.42950439453125,
      "activations/layer5_attention_weight_max": 68.46070861816406,
      "activations/layer5_attention_weight_min": -65.55754852294922,
      "activations/layer6_attention_weight_max": 53.210426330566406,
      "activations/layer6_attention_weight_min": -52.70868682861328,
      "activations/layer7_attention_weight_max": 87.47373962402344,
      "activations/layer7_attention_weight_min": -84.17161560058594,
      "activations/layer8_attention_weight_max": 40.594276428222656,
      "activations/layer8_attention_weight_min": -40.45513153076172,
      "activations/layer9_attention_weight_max": 34.765777587890625,
      "activations/layer9_attention_weight_min": -33.04831314086914,
      "epoch": 6.66,
      "learning_rate": 0.00010816477272727272,
      "loss": 2.8232,
      "step": 114550
    },
    {
      "activations/layer0_attention_weight_max": 16.132732391357422,
      "activations/layer0_attention_weight_min": -12.242258071899414,
      "activations/layer10_attention_weight_max": 28.25847625732422,
      "activations/layer10_attention_weight_min": -28.309823989868164,
      "activations/layer11_attention_weight_max": 28.242277145385742,
      "activations/layer11_attention_weight_min": -28.318836212158203,
      "activations/layer12_attention_weight_max": 25.469776153564453,
      "activations/layer12_attention_weight_min": -30.63485336303711,
      "activations/layer13_attention_weight_max": 39.85380935668945,
      "activations/layer13_attention_weight_min": -31.70345687866211,
      "activations/layer14_attention_weight_max": 51.15983200073242,
      "activations/layer14_attention_weight_min": -40.77556228637695,
      "activations/layer15_attention_weight_max": 37.695587158203125,
      "activations/layer15_attention_weight_min": -29.751684188842773,
      "activations/layer16_attention_weight_max": 35.43789291381836,
      "activations/layer16_attention_weight_min": -26.28024673461914,
      "activations/layer17_attention_weight_max": 36.182498931884766,
      "activations/layer17_attention_weight_min": -25.08211326599121,
      "activations/layer18_attention_weight_max": 32.7563362121582,
      "activations/layer18_attention_weight_min": -23.29132080078125,
      "activations/layer19_attention_weight_max": 38.12926483154297,
      "activations/layer19_attention_weight_min": -28.82846450805664,
      "activations/layer1_attention_weight_max": 18.3419246673584,
      "activations/layer1_attention_weight_min": -13.476868629455566,
      "activations/layer20_attention_weight_max": 31.957286834716797,
      "activations/layer20_attention_weight_min": -24.109130859375,
      "activations/layer21_attention_weight_max": 33.59798049926758,
      "activations/layer21_attention_weight_min": -19.786911010742188,
      "activations/layer22_attention_weight_max": 48.71758270263672,
      "activations/layer22_attention_weight_min": -27.40509033203125,
      "activations/layer23_attention_weight_max": 33.96415710449219,
      "activations/layer23_attention_weight_min": -20.497549057006836,
      "activations/layer2_attention_weight_max": 31.889432907104492,
      "activations/layer2_attention_weight_min": -30.094593048095703,
      "activations/layer3_attention_weight_max": 88.4675064086914,
      "activations/layer3_attention_weight_min": -86.750732421875,
      "activations/layer4_attention_weight_max": 120.88623046875,
      "activations/layer4_attention_weight_min": -122.29402160644531,
      "activations/layer5_attention_weight_max": 63.68134689331055,
      "activations/layer5_attention_weight_min": -61.93749237060547,
      "activations/layer6_attention_weight_max": 51.900569915771484,
      "activations/layer6_attention_weight_min": -51.39638137817383,
      "activations/layer7_attention_weight_max": 88.73705291748047,
      "activations/layer7_attention_weight_min": -77.65593719482422,
      "activations/layer8_attention_weight_max": 38.22421646118164,
      "activations/layer8_attention_weight_min": -37.02882385253906,
      "activations/layer9_attention_weight_max": 32.6246337890625,
      "activations/layer9_attention_weight_min": -42.279727935791016,
      "epoch": 6.66,
      "learning_rate": 0.00010814583333333333,
      "loss": 2.8224,
      "step": 114600
    },
    {
      "activations/layer0_attention_weight_max": 16.580551147460938,
      "activations/layer0_attention_weight_min": -12.13390064239502,
      "activations/layer10_attention_weight_max": 30.453115463256836,
      "activations/layer10_attention_weight_min": -32.106101989746094,
      "activations/layer11_attention_weight_max": 31.034770965576172,
      "activations/layer11_attention_weight_min": -28.998394012451172,
      "activations/layer12_attention_weight_max": 25.173702239990234,
      "activations/layer12_attention_weight_min": -29.307514190673828,
      "activations/layer13_attention_weight_max": 45.05890655517578,
      "activations/layer13_attention_weight_min": -32.670204162597656,
      "activations/layer14_attention_weight_max": 53.85641860961914,
      "activations/layer14_attention_weight_min": -38.34238052368164,
      "activations/layer15_attention_weight_max": 35.12811279296875,
      "activations/layer15_attention_weight_min": -29.206586837768555,
      "activations/layer16_attention_weight_max": 33.87867736816406,
      "activations/layer16_attention_weight_min": -26.398330688476562,
      "activations/layer17_attention_weight_max": 38.72810745239258,
      "activations/layer17_attention_weight_min": -24.884307861328125,
      "activations/layer18_attention_weight_max": 34.50984573364258,
      "activations/layer18_attention_weight_min": -21.251094818115234,
      "activations/layer19_attention_weight_max": 43.3857536315918,
      "activations/layer19_attention_weight_min": -30.050128936767578,
      "activations/layer1_attention_weight_max": 16.82792854309082,
      "activations/layer1_attention_weight_min": -14.038666725158691,
      "activations/layer20_attention_weight_max": 38.051170349121094,
      "activations/layer20_attention_weight_min": -25.278642654418945,
      "activations/layer21_attention_weight_max": 35.55557632446289,
      "activations/layer21_attention_weight_min": -23.309398651123047,
      "activations/layer22_attention_weight_max": 52.73523712158203,
      "activations/layer22_attention_weight_min": -28.15986442565918,
      "activations/layer23_attention_weight_max": 40.025108337402344,
      "activations/layer23_attention_weight_min": -22.467906951904297,
      "activations/layer2_attention_weight_max": 30.660324096679688,
      "activations/layer2_attention_weight_min": -29.205360412597656,
      "activations/layer3_attention_weight_max": 87.4109878540039,
      "activations/layer3_attention_weight_min": -89.86482238769531,
      "activations/layer4_attention_weight_max": 120.7719497680664,
      "activations/layer4_attention_weight_min": -123.72639465332031,
      "activations/layer5_attention_weight_max": 64.54324340820312,
      "activations/layer5_attention_weight_min": -60.31771469116211,
      "activations/layer6_attention_weight_max": 50.79941177368164,
      "activations/layer6_attention_weight_min": -51.372379302978516,
      "activations/layer7_attention_weight_max": 85.62165832519531,
      "activations/layer7_attention_weight_min": -85.01507568359375,
      "activations/layer8_attention_weight_max": 39.96629333496094,
      "activations/layer8_attention_weight_min": -39.862064361572266,
      "activations/layer9_attention_weight_max": 35.33530044555664,
      "activations/layer9_attention_weight_min": -33.323524475097656,
      "epoch": 6.66,
      "learning_rate": 0.00010812689393939393,
      "loss": 2.8164,
      "step": 114650
    },
    {
      "activations/layer0_attention_weight_max": 16.843076705932617,
      "activations/layer0_attention_weight_min": -12.691347122192383,
      "activations/layer10_attention_weight_max": 32.00999450683594,
      "activations/layer10_attention_weight_min": -33.230979919433594,
      "activations/layer11_attention_weight_max": 32.304176330566406,
      "activations/layer11_attention_weight_min": -29.303218841552734,
      "activations/layer12_attention_weight_max": 25.858173370361328,
      "activations/layer12_attention_weight_min": -24.998626708984375,
      "activations/layer13_attention_weight_max": 38.59117889404297,
      "activations/layer13_attention_weight_min": -34.05801010131836,
      "activations/layer14_attention_weight_max": 54.21967315673828,
      "activations/layer14_attention_weight_min": -41.15314483642578,
      "activations/layer15_attention_weight_max": 36.74587631225586,
      "activations/layer15_attention_weight_min": -30.22220230102539,
      "activations/layer16_attention_weight_max": 33.141502380371094,
      "activations/layer16_attention_weight_min": -28.936542510986328,
      "activations/layer17_attention_weight_max": 34.27095413208008,
      "activations/layer17_attention_weight_min": -25.491392135620117,
      "activations/layer18_attention_weight_max": 34.906551361083984,
      "activations/layer18_attention_weight_min": -21.60150146484375,
      "activations/layer19_attention_weight_max": 38.309322357177734,
      "activations/layer19_attention_weight_min": -30.864912033081055,
      "activations/layer1_attention_weight_max": 20.921586990356445,
      "activations/layer1_attention_weight_min": -18.406890869140625,
      "activations/layer20_attention_weight_max": 30.547189712524414,
      "activations/layer20_attention_weight_min": -26.3299617767334,
      "activations/layer21_attention_weight_max": 30.51207160949707,
      "activations/layer21_attention_weight_min": -22.26974105834961,
      "activations/layer22_attention_weight_max": 49.7271614074707,
      "activations/layer22_attention_weight_min": -30.02200698852539,
      "activations/layer23_attention_weight_max": 36.4649772644043,
      "activations/layer23_attention_weight_min": -24.31218719482422,
      "activations/layer2_attention_weight_max": 30.24925994873047,
      "activations/layer2_attention_weight_min": -30.251371383666992,
      "activations/layer3_attention_weight_max": 87.76637268066406,
      "activations/layer3_attention_weight_min": -86.70391845703125,
      "activations/layer4_attention_weight_max": 123.93495178222656,
      "activations/layer4_attention_weight_min": -124.29280090332031,
      "activations/layer5_attention_weight_max": 70.24050903320312,
      "activations/layer5_attention_weight_min": -64.95503997802734,
      "activations/layer6_attention_weight_max": 50.59486389160156,
      "activations/layer6_attention_weight_min": -52.78367614746094,
      "activations/layer7_attention_weight_max": 85.0464096069336,
      "activations/layer7_attention_weight_min": -85.72796630859375,
      "activations/layer8_attention_weight_max": 39.922874450683594,
      "activations/layer8_attention_weight_min": -39.21680450439453,
      "activations/layer9_attention_weight_max": 34.5230598449707,
      "activations/layer9_attention_weight_min": -32.77512741088867,
      "epoch": 6.66,
      "learning_rate": 0.00010810795454545453,
      "loss": 2.8244,
      "step": 114700
    },
    {
      "activations/layer0_attention_weight_max": 16.010860443115234,
      "activations/layer0_attention_weight_min": -13.401885032653809,
      "activations/layer10_attention_weight_max": 32.03607177734375,
      "activations/layer10_attention_weight_min": -32.94231033325195,
      "activations/layer11_attention_weight_max": 30.900714874267578,
      "activations/layer11_attention_weight_min": -31.459516525268555,
      "activations/layer12_attention_weight_max": 23.95250701904297,
      "activations/layer12_attention_weight_min": -29.14814567565918,
      "activations/layer13_attention_weight_max": 46.371212005615234,
      "activations/layer13_attention_weight_min": -34.46331024169922,
      "activations/layer14_attention_weight_max": 57.96232223510742,
      "activations/layer14_attention_weight_min": -44.555625915527344,
      "activations/layer15_attention_weight_max": 40.41099548339844,
      "activations/layer15_attention_weight_min": -31.26200294494629,
      "activations/layer16_attention_weight_max": 35.21330261230469,
      "activations/layer16_attention_weight_min": -27.892730712890625,
      "activations/layer17_attention_weight_max": 38.1908073425293,
      "activations/layer17_attention_weight_min": -25.671390533447266,
      "activations/layer18_attention_weight_max": 38.62590789794922,
      "activations/layer18_attention_weight_min": -20.506128311157227,
      "activations/layer19_attention_weight_max": 40.849300384521484,
      "activations/layer19_attention_weight_min": -30.634092330932617,
      "activations/layer1_attention_weight_max": 17.45888328552246,
      "activations/layer1_attention_weight_min": -14.137616157531738,
      "activations/layer20_attention_weight_max": 32.73055648803711,
      "activations/layer20_attention_weight_min": -24.40334701538086,
      "activations/layer21_attention_weight_max": 35.76885223388672,
      "activations/layer21_attention_weight_min": -22.38727569580078,
      "activations/layer22_attention_weight_max": 52.76765441894531,
      "activations/layer22_attention_weight_min": -30.179271697998047,
      "activations/layer23_attention_weight_max": 36.7971076965332,
      "activations/layer23_attention_weight_min": -23.313190460205078,
      "activations/layer2_attention_weight_max": 30.455778121948242,
      "activations/layer2_attention_weight_min": -29.500240325927734,
      "activations/layer3_attention_weight_max": 87.63086700439453,
      "activations/layer3_attention_weight_min": -86.26492309570312,
      "activations/layer4_attention_weight_max": 122.69696044921875,
      "activations/layer4_attention_weight_min": -122.71278381347656,
      "activations/layer5_attention_weight_max": 67.06336975097656,
      "activations/layer5_attention_weight_min": -66.05616760253906,
      "activations/layer6_attention_weight_max": 54.69105911254883,
      "activations/layer6_attention_weight_min": -55.374820709228516,
      "activations/layer7_attention_weight_max": 86.24093627929688,
      "activations/layer7_attention_weight_min": -87.56045532226562,
      "activations/layer8_attention_weight_max": 39.506919860839844,
      "activations/layer8_attention_weight_min": -39.79426574707031,
      "activations/layer9_attention_weight_max": 35.376792907714844,
      "activations/layer9_attention_weight_min": -33.642520904541016,
      "epoch": 6.67,
      "learning_rate": 0.00010808901515151515,
      "loss": 2.8156,
      "step": 114750
    },
    {
      "activations/layer0_attention_weight_max": 16.683164596557617,
      "activations/layer0_attention_weight_min": -12.889634132385254,
      "activations/layer10_attention_weight_max": 31.352977752685547,
      "activations/layer10_attention_weight_min": -29.191158294677734,
      "activations/layer11_attention_weight_max": 30.61620330810547,
      "activations/layer11_attention_weight_min": -29.51305389404297,
      "activations/layer12_attention_weight_max": 20.826797485351562,
      "activations/layer12_attention_weight_min": -26.835363388061523,
      "activations/layer13_attention_weight_max": 38.54509353637695,
      "activations/layer13_attention_weight_min": -31.742938995361328,
      "activations/layer14_attention_weight_max": 54.823917388916016,
      "activations/layer14_attention_weight_min": -40.079158782958984,
      "activations/layer15_attention_weight_max": 40.312034606933594,
      "activations/layer15_attention_weight_min": -28.473400115966797,
      "activations/layer16_attention_weight_max": 32.90242385864258,
      "activations/layer16_attention_weight_min": -30.22538185119629,
      "activations/layer17_attention_weight_max": 33.82190704345703,
      "activations/layer17_attention_weight_min": -25.097726821899414,
      "activations/layer18_attention_weight_max": 30.133563995361328,
      "activations/layer18_attention_weight_min": -21.558305740356445,
      "activations/layer19_attention_weight_max": 34.06311798095703,
      "activations/layer19_attention_weight_min": -30.36215591430664,
      "activations/layer1_attention_weight_max": 17.085004806518555,
      "activations/layer1_attention_weight_min": -13.516324996948242,
      "activations/layer20_attention_weight_max": 30.1776123046875,
      "activations/layer20_attention_weight_min": -23.620351791381836,
      "activations/layer21_attention_weight_max": 29.572628021240234,
      "activations/layer21_attention_weight_min": -21.0755558013916,
      "activations/layer22_attention_weight_max": 44.54551315307617,
      "activations/layer22_attention_weight_min": -29.544546127319336,
      "activations/layer23_attention_weight_max": 33.10772705078125,
      "activations/layer23_attention_weight_min": -25.548246383666992,
      "activations/layer2_attention_weight_max": 30.569366455078125,
      "activations/layer2_attention_weight_min": -30.67577362060547,
      "activations/layer3_attention_weight_max": 85.16537475585938,
      "activations/layer3_attention_weight_min": -86.64761352539062,
      "activations/layer4_attention_weight_max": 118.87923431396484,
      "activations/layer4_attention_weight_min": -116.99787902832031,
      "activations/layer5_attention_weight_max": 62.92510223388672,
      "activations/layer5_attention_weight_min": -64.14633178710938,
      "activations/layer6_attention_weight_max": 55.19349670410156,
      "activations/layer6_attention_weight_min": -51.964298248291016,
      "activations/layer7_attention_weight_max": 86.5599594116211,
      "activations/layer7_attention_weight_min": -84.46015930175781,
      "activations/layer8_attention_weight_max": 37.946510314941406,
      "activations/layer8_attention_weight_min": -37.55671691894531,
      "activations/layer9_attention_weight_max": 34.473480224609375,
      "activations/layer9_attention_weight_min": -32.558868408203125,
      "epoch": 6.67,
      "learning_rate": 0.00010807045454545454,
      "loss": 2.8141,
      "step": 114800
    },
    {
      "activations/layer0_attention_weight_max": 16.196325302124023,
      "activations/layer0_attention_weight_min": -12.272658348083496,
      "activations/layer10_attention_weight_max": 34.88526916503906,
      "activations/layer10_attention_weight_min": -33.62180709838867,
      "activations/layer11_attention_weight_max": 34.26402282714844,
      "activations/layer11_attention_weight_min": -29.4082088470459,
      "activations/layer12_attention_weight_max": 39.61065673828125,
      "activations/layer12_attention_weight_min": -28.732933044433594,
      "activations/layer13_attention_weight_max": 48.913082122802734,
      "activations/layer13_attention_weight_min": -34.76173400878906,
      "activations/layer14_attention_weight_max": 52.090694427490234,
      "activations/layer14_attention_weight_min": -47.899314880371094,
      "activations/layer15_attention_weight_max": 36.1109504699707,
      "activations/layer15_attention_weight_min": -27.706880569458008,
      "activations/layer16_attention_weight_max": 36.90217590332031,
      "activations/layer16_attention_weight_min": -25.946590423583984,
      "activations/layer17_attention_weight_max": 36.632266998291016,
      "activations/layer17_attention_weight_min": -24.919189453125,
      "activations/layer18_attention_weight_max": 31.237028121948242,
      "activations/layer18_attention_weight_min": -19.867713928222656,
      "activations/layer19_attention_weight_max": 38.068153381347656,
      "activations/layer19_attention_weight_min": -28.55990982055664,
      "activations/layer1_attention_weight_max": 16.611597061157227,
      "activations/layer1_attention_weight_min": -13.531454086303711,
      "activations/layer20_attention_weight_max": 32.41432571411133,
      "activations/layer20_attention_weight_min": -23.16303253173828,
      "activations/layer21_attention_weight_max": 32.328372955322266,
      "activations/layer21_attention_weight_min": -20.822797775268555,
      "activations/layer22_attention_weight_max": 52.55610656738281,
      "activations/layer22_attention_weight_min": -26.74477195739746,
      "activations/layer23_attention_weight_max": 35.08372497558594,
      "activations/layer23_attention_weight_min": -23.44955062866211,
      "activations/layer2_attention_weight_max": 29.728750228881836,
      "activations/layer2_attention_weight_min": -29.11773681640625,
      "activations/layer3_attention_weight_max": 90.194580078125,
      "activations/layer3_attention_weight_min": -89.89259338378906,
      "activations/layer4_attention_weight_max": 125.68448638916016,
      "activations/layer4_attention_weight_min": -121.01954650878906,
      "activations/layer5_attention_weight_max": 67.03166198730469,
      "activations/layer5_attention_weight_min": -63.3548583984375,
      "activations/layer6_attention_weight_max": 52.53486633300781,
      "activations/layer6_attention_weight_min": -52.40308380126953,
      "activations/layer7_attention_weight_max": 88.56473541259766,
      "activations/layer7_attention_weight_min": -87.54010772705078,
      "activations/layer8_attention_weight_max": 41.08089065551758,
      "activations/layer8_attention_weight_min": -43.15108108520508,
      "activations/layer9_attention_weight_max": 40.765995025634766,
      "activations/layer9_attention_weight_min": -38.294334411621094,
      "epoch": 6.67,
      "learning_rate": 0.00010805151515151515,
      "loss": 2.8183,
      "step": 114850
    },
    {
      "activations/layer0_attention_weight_max": 15.584811210632324,
      "activations/layer0_attention_weight_min": -11.865005493164062,
      "activations/layer10_attention_weight_max": 33.1590461730957,
      "activations/layer10_attention_weight_min": -30.29111671447754,
      "activations/layer11_attention_weight_max": 33.83919906616211,
      "activations/layer11_attention_weight_min": -30.10733413696289,
      "activations/layer12_attention_weight_max": 22.859891891479492,
      "activations/layer12_attention_weight_min": -25.91753387451172,
      "activations/layer13_attention_weight_max": 42.66346740722656,
      "activations/layer13_attention_weight_min": -33.63520812988281,
      "activations/layer14_attention_weight_max": 57.105010986328125,
      "activations/layer14_attention_weight_min": -43.78219985961914,
      "activations/layer15_attention_weight_max": 36.18195724487305,
      "activations/layer15_attention_weight_min": -28.452056884765625,
      "activations/layer16_attention_weight_max": 33.50068664550781,
      "activations/layer16_attention_weight_min": -28.408390045166016,
      "activations/layer17_attention_weight_max": 38.09930419921875,
      "activations/layer17_attention_weight_min": -25.397315979003906,
      "activations/layer18_attention_weight_max": 27.64793586730957,
      "activations/layer18_attention_weight_min": -19.695755004882812,
      "activations/layer19_attention_weight_max": 36.868717193603516,
      "activations/layer19_attention_weight_min": -29.320093154907227,
      "activations/layer1_attention_weight_max": 16.830669403076172,
      "activations/layer1_attention_weight_min": -13.845866203308105,
      "activations/layer20_attention_weight_max": 33.703704833984375,
      "activations/layer20_attention_weight_min": -23.814725875854492,
      "activations/layer21_attention_weight_max": 31.353097915649414,
      "activations/layer21_attention_weight_min": -19.910306930541992,
      "activations/layer22_attention_weight_max": 48.24378204345703,
      "activations/layer22_attention_weight_min": -27.001131057739258,
      "activations/layer23_attention_weight_max": 34.745521545410156,
      "activations/layer23_attention_weight_min": -20.341724395751953,
      "activations/layer2_attention_weight_max": 31.912681579589844,
      "activations/layer2_attention_weight_min": -29.018417358398438,
      "activations/layer3_attention_weight_max": 94.05438232421875,
      "activations/layer3_attention_weight_min": -92.80931091308594,
      "activations/layer4_attention_weight_max": 129.5780487060547,
      "activations/layer4_attention_weight_min": -124.0185546875,
      "activations/layer5_attention_weight_max": 66.82140350341797,
      "activations/layer5_attention_weight_min": -63.93479919433594,
      "activations/layer6_attention_weight_max": 54.78443908691406,
      "activations/layer6_attention_weight_min": -54.092342376708984,
      "activations/layer7_attention_weight_max": 88.6444320678711,
      "activations/layer7_attention_weight_min": -82.96723175048828,
      "activations/layer8_attention_weight_max": 39.2696533203125,
      "activations/layer8_attention_weight_min": -40.037322998046875,
      "activations/layer9_attention_weight_max": 36.878963470458984,
      "activations/layer9_attention_weight_min": -35.847145080566406,
      "epoch": 6.68,
      "learning_rate": 0.00010803257575757574,
      "loss": 2.8155,
      "step": 114900
    },
    {
      "activations/layer0_attention_weight_max": 16.502561569213867,
      "activations/layer0_attention_weight_min": -13.111629486083984,
      "activations/layer10_attention_weight_max": 34.09835433959961,
      "activations/layer10_attention_weight_min": -32.99440002441406,
      "activations/layer11_attention_weight_max": 32.93818664550781,
      "activations/layer11_attention_weight_min": -29.177406311035156,
      "activations/layer12_attention_weight_max": 24.356021881103516,
      "activations/layer12_attention_weight_min": -25.89985466003418,
      "activations/layer13_attention_weight_max": 41.572357177734375,
      "activations/layer13_attention_weight_min": -32.063968658447266,
      "activations/layer14_attention_weight_max": 54.46969985961914,
      "activations/layer14_attention_weight_min": -40.456478118896484,
      "activations/layer15_attention_weight_max": 35.053916931152344,
      "activations/layer15_attention_weight_min": -29.413972854614258,
      "activations/layer16_attention_weight_max": 31.834386825561523,
      "activations/layer16_attention_weight_min": -26.205463409423828,
      "activations/layer17_attention_weight_max": 34.08148193359375,
      "activations/layer17_attention_weight_min": -23.397994995117188,
      "activations/layer18_attention_weight_max": 31.6013126373291,
      "activations/layer18_attention_weight_min": -20.783353805541992,
      "activations/layer19_attention_weight_max": 36.815467834472656,
      "activations/layer19_attention_weight_min": -29.638662338256836,
      "activations/layer1_attention_weight_max": 18.548236846923828,
      "activations/layer1_attention_weight_min": -15.676534652709961,
      "activations/layer20_attention_weight_max": 31.7247314453125,
      "activations/layer20_attention_weight_min": -25.090044021606445,
      "activations/layer21_attention_weight_max": 32.27469253540039,
      "activations/layer21_attention_weight_min": -22.4266300201416,
      "activations/layer22_attention_weight_max": 49.63276672363281,
      "activations/layer22_attention_weight_min": -31.29738998413086,
      "activations/layer23_attention_weight_max": 37.132835388183594,
      "activations/layer23_attention_weight_min": -23.850378036499023,
      "activations/layer2_attention_weight_max": 32.26561737060547,
      "activations/layer2_attention_weight_min": -30.395282745361328,
      "activations/layer3_attention_weight_max": 91.6414566040039,
      "activations/layer3_attention_weight_min": -89.51768493652344,
      "activations/layer4_attention_weight_max": 123.20845794677734,
      "activations/layer4_attention_weight_min": -121.5626449584961,
      "activations/layer5_attention_weight_max": 66.42681884765625,
      "activations/layer5_attention_weight_min": -65.49839782714844,
      "activations/layer6_attention_weight_max": 52.31084060668945,
      "activations/layer6_attention_weight_min": -58.7628173828125,
      "activations/layer7_attention_weight_max": 90.68575286865234,
      "activations/layer7_attention_weight_min": -89.61505889892578,
      "activations/layer8_attention_weight_max": 45.24946975708008,
      "activations/layer8_attention_weight_min": -43.62483215332031,
      "activations/layer9_attention_weight_max": 39.544822692871094,
      "activations/layer9_attention_weight_min": -36.62386703491211,
      "epoch": 6.68,
      "learning_rate": 0.00010801363636363636,
      "loss": 2.8281,
      "step": 114950
    },
    {
      "activations/layer0_attention_weight_max": 17.087722778320312,
      "activations/layer0_attention_weight_min": -12.81939697265625,
      "activations/layer10_attention_weight_max": 33.555419921875,
      "activations/layer10_attention_weight_min": -31.782602310180664,
      "activations/layer11_attention_weight_max": 30.60214614868164,
      "activations/layer11_attention_weight_min": -28.401371002197266,
      "activations/layer12_attention_weight_max": 22.613605499267578,
      "activations/layer12_attention_weight_min": -25.401153564453125,
      "activations/layer13_attention_weight_max": 41.74895095825195,
      "activations/layer13_attention_weight_min": -31.43966293334961,
      "activations/layer14_attention_weight_max": 52.97136688232422,
      "activations/layer14_attention_weight_min": -37.715755462646484,
      "activations/layer15_attention_weight_max": 42.03947067260742,
      "activations/layer15_attention_weight_min": -28.698116302490234,
      "activations/layer16_attention_weight_max": 34.461822509765625,
      "activations/layer16_attention_weight_min": -27.586872100830078,
      "activations/layer17_attention_weight_max": 38.081520080566406,
      "activations/layer17_attention_weight_min": -24.64423370361328,
      "activations/layer18_attention_weight_max": 32.04041290283203,
      "activations/layer18_attention_weight_min": -23.669282913208008,
      "activations/layer19_attention_weight_max": 37.270259857177734,
      "activations/layer19_attention_weight_min": -29.923660278320312,
      "activations/layer1_attention_weight_max": 16.572006225585938,
      "activations/layer1_attention_weight_min": -14.676136016845703,
      "activations/layer20_attention_weight_max": 30.850006103515625,
      "activations/layer20_attention_weight_min": -23.097145080566406,
      "activations/layer21_attention_weight_max": 28.22586441040039,
      "activations/layer21_attention_weight_min": -22.239980697631836,
      "activations/layer22_attention_weight_max": 51.7919921875,
      "activations/layer22_attention_weight_min": -30.02952766418457,
      "activations/layer23_attention_weight_max": 33.08136749267578,
      "activations/layer23_attention_weight_min": -22.793106079101562,
      "activations/layer2_attention_weight_max": 33.23480224609375,
      "activations/layer2_attention_weight_min": -29.623699188232422,
      "activations/layer3_attention_weight_max": 85.18279266357422,
      "activations/layer3_attention_weight_min": -84.15480041503906,
      "activations/layer4_attention_weight_max": 121.290771484375,
      "activations/layer4_attention_weight_min": -116.48332977294922,
      "activations/layer5_attention_weight_max": 67.06244659423828,
      "activations/layer5_attention_weight_min": -64.07953643798828,
      "activations/layer6_attention_weight_max": 52.087039947509766,
      "activations/layer6_attention_weight_min": -55.33402633666992,
      "activations/layer7_attention_weight_max": 86.24475860595703,
      "activations/layer7_attention_weight_min": -85.32964324951172,
      "activations/layer8_attention_weight_max": 38.81243896484375,
      "activations/layer8_attention_weight_min": -40.865333557128906,
      "activations/layer9_attention_weight_max": 35.38628005981445,
      "activations/layer9_attention_weight_min": -32.54658889770508,
      "epoch": 6.68,
      "learning_rate": 0.00010799469696969696,
      "loss": 2.8291,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_loss": 2.775390625,
      "eval_runtime": 8.49,
      "eval_samples_per_second": 505.77,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_openwebtext_loss": 2.775390625,
      "eval_openwebtext_ppl": 16.04489330813532,
      "eval_openwebtext_runtime": 8.49,
      "eval_openwebtext_samples_per_second": 505.77,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_wikitext_loss": 3.0,
      "eval_wikitext_ppl": 20.085536923187668,
      "eval_wikitext_runtime": 2.0004,
      "eval_wikitext_samples_per_second": 227.957,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_lambada_loss": 2.765625,
      "eval_lambada_ppl": 15.888967492629165,
      "eval_lambada_runtime": 9.5826,
      "eval_lambada_samples_per_second": 508.108,
      "step": 115000
    },
    {
      "activations/layer0_attention_weight_max": 16.555391311645508,
      "activations/layer0_attention_weight_min": -12.44687557220459,
      "activations/layer10_attention_weight_max": 36.47563171386719,
      "activations/layer10_attention_weight_min": -34.50177764892578,
      "activations/layer11_attention_weight_max": 39.08207702636719,
      "activations/layer11_attention_weight_min": -32.25590896606445,
      "activations/layer12_attention_weight_max": 39.224647521972656,
      "activations/layer12_attention_weight_min": -25.14470100402832,
      "activations/layer13_attention_weight_max": 56.33625411987305,
      "activations/layer13_attention_weight_min": -33.65986251831055,
      "activations/layer14_attention_weight_max": 62.57758331298828,
      "activations/layer14_attention_weight_min": -47.50083923339844,
      "activations/layer15_attention_weight_max": 49.66992950439453,
      "activations/layer15_attention_weight_min": -30.97239875793457,
      "activations/layer16_attention_weight_max": 38.86851119995117,
      "activations/layer16_attention_weight_min": -26.988895416259766,
      "activations/layer17_attention_weight_max": 34.100608825683594,
      "activations/layer17_attention_weight_min": -25.23612403869629,
      "activations/layer18_attention_weight_max": 35.06472396850586,
      "activations/layer18_attention_weight_min": -22.367706298828125,
      "activations/layer19_attention_weight_max": 51.37274169921875,
      "activations/layer19_attention_weight_min": -29.664569854736328,
      "activations/layer1_attention_weight_max": 16.285247802734375,
      "activations/layer1_attention_weight_min": -14.200672149658203,
      "activations/layer20_attention_weight_max": 35.414730072021484,
      "activations/layer20_attention_weight_min": -22.102439880371094,
      "activations/layer21_attention_weight_max": 37.3736457824707,
      "activations/layer21_attention_weight_min": -20.604127883911133,
      "activations/layer22_attention_weight_max": 51.055145263671875,
      "activations/layer22_attention_weight_min": -27.901447296142578,
      "activations/layer23_attention_weight_max": 39.42138671875,
      "activations/layer23_attention_weight_min": -21.07772445678711,
      "activations/layer2_attention_weight_max": 30.87477684020996,
      "activations/layer2_attention_weight_min": -29.372028350830078,
      "activations/layer3_attention_weight_max": 92.48681640625,
      "activations/layer3_attention_weight_min": -89.83634948730469,
      "activations/layer4_attention_weight_max": 127.0368881225586,
      "activations/layer4_attention_weight_min": -124.24761962890625,
      "activations/layer5_attention_weight_max": 71.32344055175781,
      "activations/layer5_attention_weight_min": -66.18496704101562,
      "activations/layer6_attention_weight_max": 54.32096862792969,
      "activations/layer6_attention_weight_min": -53.41426467895508,
      "activations/layer7_attention_weight_max": 91.16090393066406,
      "activations/layer7_attention_weight_min": -86.85709381103516,
      "activations/layer8_attention_weight_max": 42.26869583129883,
      "activations/layer8_attention_weight_min": -42.29597854614258,
      "activations/layer9_attention_weight_max": 37.9704475402832,
      "activations/layer9_attention_weight_min": -36.2013053894043,
      "epoch": 6.69,
      "learning_rate": 0.00010797575757575756,
      "loss": 2.8359,
      "step": 115050
    },
    {
      "activations/layer0_attention_weight_max": 15.493992805480957,
      "activations/layer0_attention_weight_min": -12.458513259887695,
      "activations/layer10_attention_weight_max": 36.41167068481445,
      "activations/layer10_attention_weight_min": -32.35676193237305,
      "activations/layer11_attention_weight_max": 37.78988265991211,
      "activations/layer11_attention_weight_min": -30.31097984313965,
      "activations/layer12_attention_weight_max": 36.101741790771484,
      "activations/layer12_attention_weight_min": -25.567453384399414,
      "activations/layer13_attention_weight_max": 52.33263397216797,
      "activations/layer13_attention_weight_min": -30.510122299194336,
      "activations/layer14_attention_weight_max": 71.8559799194336,
      "activations/layer14_attention_weight_min": -39.13029479980469,
      "activations/layer15_attention_weight_max": 44.00739288330078,
      "activations/layer15_attention_weight_min": -31.71549415588379,
      "activations/layer16_attention_weight_max": 41.73128890991211,
      "activations/layer16_attention_weight_min": -27.050596237182617,
      "activations/layer17_attention_weight_max": 44.477691650390625,
      "activations/layer17_attention_weight_min": -29.243677139282227,
      "activations/layer18_attention_weight_max": 36.6954231262207,
      "activations/layer18_attention_weight_min": -23.106142044067383,
      "activations/layer19_attention_weight_max": 46.35651779174805,
      "activations/layer19_attention_weight_min": -33.56351852416992,
      "activations/layer1_attention_weight_max": 17.31451988220215,
      "activations/layer1_attention_weight_min": -16.26567268371582,
      "activations/layer20_attention_weight_max": 38.52352523803711,
      "activations/layer20_attention_weight_min": -23.692729949951172,
      "activations/layer21_attention_weight_max": 36.68971633911133,
      "activations/layer21_attention_weight_min": -21.17591094970703,
      "activations/layer22_attention_weight_max": 60.71257781982422,
      "activations/layer22_attention_weight_min": -28.499408721923828,
      "activations/layer23_attention_weight_max": 42.95403289794922,
      "activations/layer23_attention_weight_min": -23.584270477294922,
      "activations/layer2_attention_weight_max": 32.21875,
      "activations/layer2_attention_weight_min": -31.098058700561523,
      "activations/layer3_attention_weight_max": 93.5761947631836,
      "activations/layer3_attention_weight_min": -95.42369842529297,
      "activations/layer4_attention_weight_max": 130.33079528808594,
      "activations/layer4_attention_weight_min": -133.7435302734375,
      "activations/layer5_attention_weight_max": 70.33509826660156,
      "activations/layer5_attention_weight_min": -66.93521881103516,
      "activations/layer6_attention_weight_max": 52.68283462524414,
      "activations/layer6_attention_weight_min": -54.83483123779297,
      "activations/layer7_attention_weight_max": 88.67803955078125,
      "activations/layer7_attention_weight_min": -84.39329528808594,
      "activations/layer8_attention_weight_max": 41.361629486083984,
      "activations/layer8_attention_weight_min": -40.15310287475586,
      "activations/layer9_attention_weight_max": 38.74516677856445,
      "activations/layer9_attention_weight_min": -35.346797943115234,
      "epoch": 6.69,
      "learning_rate": 0.00010795681818181818,
      "loss": 2.8145,
      "step": 115100
    },
    {
      "activations/layer0_attention_weight_max": 15.862692832946777,
      "activations/layer0_attention_weight_min": -12.352229118347168,
      "activations/layer10_attention_weight_max": 33.56599044799805,
      "activations/layer10_attention_weight_min": -31.656951904296875,
      "activations/layer11_attention_weight_max": 33.281246185302734,
      "activations/layer11_attention_weight_min": -28.425426483154297,
      "activations/layer12_attention_weight_max": 25.586606979370117,
      "activations/layer12_attention_weight_min": -26.105592727661133,
      "activations/layer13_attention_weight_max": 41.26244354248047,
      "activations/layer13_attention_weight_min": -32.02545928955078,
      "activations/layer14_attention_weight_max": 57.37504959106445,
      "activations/layer14_attention_weight_min": -42.52643585205078,
      "activations/layer15_attention_weight_max": 37.624855041503906,
      "activations/layer15_attention_weight_min": -28.815887451171875,
      "activations/layer16_attention_weight_max": 37.860626220703125,
      "activations/layer16_attention_weight_min": -26.442188262939453,
      "activations/layer17_attention_weight_max": 38.1314697265625,
      "activations/layer17_attention_weight_min": -23.42722511291504,
      "activations/layer18_attention_weight_max": 34.23819351196289,
      "activations/layer18_attention_weight_min": -20.248334884643555,
      "activations/layer19_attention_weight_max": 44.18238067626953,
      "activations/layer19_attention_weight_min": -28.188678741455078,
      "activations/layer1_attention_weight_max": 16.67976951599121,
      "activations/layer1_attention_weight_min": -14.94588851928711,
      "activations/layer20_attention_weight_max": 35.572662353515625,
      "activations/layer20_attention_weight_min": -21.62033462524414,
      "activations/layer21_attention_weight_max": 33.473045349121094,
      "activations/layer21_attention_weight_min": -20.436235427856445,
      "activations/layer22_attention_weight_max": 53.66812515258789,
      "activations/layer22_attention_weight_min": -29.544715881347656,
      "activations/layer23_attention_weight_max": 43.85845947265625,
      "activations/layer23_attention_weight_min": -23.44857406616211,
      "activations/layer2_attention_weight_max": 32.95425796508789,
      "activations/layer2_attention_weight_min": -32.61603546142578,
      "activations/layer3_attention_weight_max": 91.15270233154297,
      "activations/layer3_attention_weight_min": -90.87012481689453,
      "activations/layer4_attention_weight_max": 127.6371841430664,
      "activations/layer4_attention_weight_min": -128.690673828125,
      "activations/layer5_attention_weight_max": 67.40681457519531,
      "activations/layer5_attention_weight_min": -65.24473571777344,
      "activations/layer6_attention_weight_max": 52.307010650634766,
      "activations/layer6_attention_weight_min": -55.130043029785156,
      "activations/layer7_attention_weight_max": 87.65286254882812,
      "activations/layer7_attention_weight_min": -86.45457458496094,
      "activations/layer8_attention_weight_max": 41.01044464111328,
      "activations/layer8_attention_weight_min": -40.924964904785156,
      "activations/layer9_attention_weight_max": 36.9334602355957,
      "activations/layer9_attention_weight_min": -34.87202835083008,
      "epoch": 6.69,
      "learning_rate": 0.00010793787878787878,
      "loss": 2.8195,
      "step": 115150
    },
    {
      "activations/layer0_attention_weight_max": 15.869763374328613,
      "activations/layer0_attention_weight_min": -12.457135200500488,
      "activations/layer10_attention_weight_max": 31.994102478027344,
      "activations/layer10_attention_weight_min": -34.97127914428711,
      "activations/layer11_attention_weight_max": 31.78797721862793,
      "activations/layer11_attention_weight_min": -30.90028953552246,
      "activations/layer12_attention_weight_max": 25.914047241210938,
      "activations/layer12_attention_weight_min": -26.662736892700195,
      "activations/layer13_attention_weight_max": 41.280364990234375,
      "activations/layer13_attention_weight_min": -32.37411117553711,
      "activations/layer14_attention_weight_max": 65.66780090332031,
      "activations/layer14_attention_weight_min": -47.50880432128906,
      "activations/layer15_attention_weight_max": 39.13875961303711,
      "activations/layer15_attention_weight_min": -29.39414405822754,
      "activations/layer16_attention_weight_max": 33.212039947509766,
      "activations/layer16_attention_weight_min": -29.112255096435547,
      "activations/layer17_attention_weight_max": 38.153751373291016,
      "activations/layer17_attention_weight_min": -24.859636306762695,
      "activations/layer18_attention_weight_max": 32.776702880859375,
      "activations/layer18_attention_weight_min": -22.941665649414062,
      "activations/layer19_attention_weight_max": 39.5756950378418,
      "activations/layer19_attention_weight_min": -30.716323852539062,
      "activations/layer1_attention_weight_max": 17.794992446899414,
      "activations/layer1_attention_weight_min": -15.044196128845215,
      "activations/layer20_attention_weight_max": 32.07960891723633,
      "activations/layer20_attention_weight_min": -24.95092010498047,
      "activations/layer21_attention_weight_max": 31.809207916259766,
      "activations/layer21_attention_weight_min": -22.142393112182617,
      "activations/layer22_attention_weight_max": 46.53489685058594,
      "activations/layer22_attention_weight_min": -29.640087127685547,
      "activations/layer23_attention_weight_max": 40.48706817626953,
      "activations/layer23_attention_weight_min": -22.45122528076172,
      "activations/layer2_attention_weight_max": 31.36827850341797,
      "activations/layer2_attention_weight_min": -31.080211639404297,
      "activations/layer3_attention_weight_max": 90.9997329711914,
      "activations/layer3_attention_weight_min": -89.91907501220703,
      "activations/layer4_attention_weight_max": 125.651123046875,
      "activations/layer4_attention_weight_min": -119.20501708984375,
      "activations/layer5_attention_weight_max": 67.95242309570312,
      "activations/layer5_attention_weight_min": -65.0329818725586,
      "activations/layer6_attention_weight_max": 51.619327545166016,
      "activations/layer6_attention_weight_min": -54.00733947753906,
      "activations/layer7_attention_weight_max": 82.49876403808594,
      "activations/layer7_attention_weight_min": -82.27355194091797,
      "activations/layer8_attention_weight_max": 38.49088668823242,
      "activations/layer8_attention_weight_min": -40.7746696472168,
      "activations/layer9_attention_weight_max": 34.105751037597656,
      "activations/layer9_attention_weight_min": -35.72152328491211,
      "epoch": 6.69,
      "learning_rate": 0.00010791893939393939,
      "loss": 2.8279,
      "step": 115200
    },
    {
      "activations/layer0_attention_weight_max": 16.181015014648438,
      "activations/layer0_attention_weight_min": -11.33103084564209,
      "activations/layer10_attention_weight_max": 37.026153564453125,
      "activations/layer10_attention_weight_min": -32.66702651977539,
      "activations/layer11_attention_weight_max": 40.95030212402344,
      "activations/layer11_attention_weight_min": -34.453338623046875,
      "activations/layer12_attention_weight_max": 23.546859741210938,
      "activations/layer12_attention_weight_min": -26.61091423034668,
      "activations/layer13_attention_weight_max": 46.16447830200195,
      "activations/layer13_attention_weight_min": -34.75043869018555,
      "activations/layer14_attention_weight_max": 51.38031005859375,
      "activations/layer14_attention_weight_min": -41.53221130371094,
      "activations/layer15_attention_weight_max": 34.308292388916016,
      "activations/layer15_attention_weight_min": -27.9205265045166,
      "activations/layer16_attention_weight_max": 33.13162612915039,
      "activations/layer16_attention_weight_min": -27.523752212524414,
      "activations/layer17_attention_weight_max": 38.8340950012207,
      "activations/layer17_attention_weight_min": -25.053895950317383,
      "activations/layer18_attention_weight_max": 31.6204833984375,
      "activations/layer18_attention_weight_min": -20.196792602539062,
      "activations/layer19_attention_weight_max": 38.899192810058594,
      "activations/layer19_attention_weight_min": -29.684402465820312,
      "activations/layer1_attention_weight_max": 18.03424644470215,
      "activations/layer1_attention_weight_min": -13.944639205932617,
      "activations/layer20_attention_weight_max": 33.146366119384766,
      "activations/layer20_attention_weight_min": -24.74123191833496,
      "activations/layer21_attention_weight_max": 31.37027931213379,
      "activations/layer21_attention_weight_min": -21.618896484375,
      "activations/layer22_attention_weight_max": 48.94912338256836,
      "activations/layer22_attention_weight_min": -28.79462242126465,
      "activations/layer23_attention_weight_max": 35.17253112792969,
      "activations/layer23_attention_weight_min": -21.92951011657715,
      "activations/layer2_attention_weight_max": 33.20134735107422,
      "activations/layer2_attention_weight_min": -31.01624298095703,
      "activations/layer3_attention_weight_max": 95.03543090820312,
      "activations/layer3_attention_weight_min": -95.84777069091797,
      "activations/layer4_attention_weight_max": 131.51673889160156,
      "activations/layer4_attention_weight_min": -129.36233520507812,
      "activations/layer5_attention_weight_max": 67.29154968261719,
      "activations/layer5_attention_weight_min": -68.8170394897461,
      "activations/layer6_attention_weight_max": 52.53517532348633,
      "activations/layer6_attention_weight_min": -54.00633239746094,
      "activations/layer7_attention_weight_max": 98.98237609863281,
      "activations/layer7_attention_weight_min": -87.3834228515625,
      "activations/layer8_attention_weight_max": 43.532432556152344,
      "activations/layer8_attention_weight_min": -41.6823844909668,
      "activations/layer9_attention_weight_max": 39.69035339355469,
      "activations/layer9_attention_weight_min": -35.54458236694336,
      "epoch": 6.7,
      "learning_rate": 0.0001079,
      "loss": 2.8237,
      "step": 115250
    },
    {
      "activations/layer0_attention_weight_max": 15.360817909240723,
      "activations/layer0_attention_weight_min": -12.743424415588379,
      "activations/layer10_attention_weight_max": 38.28499984741211,
      "activations/layer10_attention_weight_min": -32.86635208129883,
      "activations/layer11_attention_weight_max": 38.26429748535156,
      "activations/layer11_attention_weight_min": -30.512557983398438,
      "activations/layer12_attention_weight_max": 23.910526275634766,
      "activations/layer12_attention_weight_min": -25.092586517333984,
      "activations/layer13_attention_weight_max": 40.17545700073242,
      "activations/layer13_attention_weight_min": -31.394580841064453,
      "activations/layer14_attention_weight_max": 57.30500793457031,
      "activations/layer14_attention_weight_min": -46.81012725830078,
      "activations/layer15_attention_weight_max": 37.19267654418945,
      "activations/layer15_attention_weight_min": -30.05036735534668,
      "activations/layer16_attention_weight_max": 31.62215232849121,
      "activations/layer16_attention_weight_min": -30.25908660888672,
      "activations/layer17_attention_weight_max": 39.68916702270508,
      "activations/layer17_attention_weight_min": -24.780351638793945,
      "activations/layer18_attention_weight_max": 28.599483489990234,
      "activations/layer18_attention_weight_min": -20.665315628051758,
      "activations/layer19_attention_weight_max": 35.41102981567383,
      "activations/layer19_attention_weight_min": -31.595205307006836,
      "activations/layer1_attention_weight_max": 17.13872528076172,
      "activations/layer1_attention_weight_min": -14.779167175292969,
      "activations/layer20_attention_weight_max": 30.59029197692871,
      "activations/layer20_attention_weight_min": -22.377124786376953,
      "activations/layer21_attention_weight_max": 29.50865936279297,
      "activations/layer21_attention_weight_min": -22.59797477722168,
      "activations/layer22_attention_weight_max": 44.3062629699707,
      "activations/layer22_attention_weight_min": -28.144758224487305,
      "activations/layer23_attention_weight_max": 34.44013214111328,
      "activations/layer23_attention_weight_min": -20.020265579223633,
      "activations/layer2_attention_weight_max": 33.214088439941406,
      "activations/layer2_attention_weight_min": -30.53255844116211,
      "activations/layer3_attention_weight_max": 96.05220031738281,
      "activations/layer3_attention_weight_min": -93.97150421142578,
      "activations/layer4_attention_weight_max": 136.968017578125,
      "activations/layer4_attention_weight_min": -123.8379135131836,
      "activations/layer5_attention_weight_max": 73.0007553100586,
      "activations/layer5_attention_weight_min": -64.36036682128906,
      "activations/layer6_attention_weight_max": 54.7774543762207,
      "activations/layer6_attention_weight_min": -52.45322036743164,
      "activations/layer7_attention_weight_max": 100.1346206665039,
      "activations/layer7_attention_weight_min": -85.20597839355469,
      "activations/layer8_attention_weight_max": 44.931026458740234,
      "activations/layer8_attention_weight_min": -41.779232025146484,
      "activations/layer9_attention_weight_max": 39.16587448120117,
      "activations/layer9_attention_weight_min": -36.779415130615234,
      "epoch": 6.7,
      "learning_rate": 0.0001078810606060606,
      "loss": 2.8337,
      "step": 115300
    },
    {
      "activations/layer0_attention_weight_max": 15.979373931884766,
      "activations/layer0_attention_weight_min": -11.808783531188965,
      "activations/layer10_attention_weight_max": 31.851213455200195,
      "activations/layer10_attention_weight_min": -30.47789764404297,
      "activations/layer11_attention_weight_max": 32.609134674072266,
      "activations/layer11_attention_weight_min": -30.159414291381836,
      "activations/layer12_attention_weight_max": 22.962989807128906,
      "activations/layer12_attention_weight_min": -26.725330352783203,
      "activations/layer13_attention_weight_max": 43.06812286376953,
      "activations/layer13_attention_weight_min": -32.709716796875,
      "activations/layer14_attention_weight_max": 58.168487548828125,
      "activations/layer14_attention_weight_min": -42.608917236328125,
      "activations/layer15_attention_weight_max": 38.504920959472656,
      "activations/layer15_attention_weight_min": -28.965572357177734,
      "activations/layer16_attention_weight_max": 37.44522476196289,
      "activations/layer16_attention_weight_min": -29.149568557739258,
      "activations/layer17_attention_weight_max": 41.613773345947266,
      "activations/layer17_attention_weight_min": -24.7789363861084,
      "activations/layer18_attention_weight_max": 34.39942932128906,
      "activations/layer18_attention_weight_min": -19.72211265563965,
      "activations/layer19_attention_weight_max": 38.56260299682617,
      "activations/layer19_attention_weight_min": -30.210744857788086,
      "activations/layer1_attention_weight_max": 17.76181983947754,
      "activations/layer1_attention_weight_min": -14.842958450317383,
      "activations/layer20_attention_weight_max": 30.78633689880371,
      "activations/layer20_attention_weight_min": -24.852537155151367,
      "activations/layer21_attention_weight_max": 31.942455291748047,
      "activations/layer21_attention_weight_min": -23.538949966430664,
      "activations/layer22_attention_weight_max": 51.828487396240234,
      "activations/layer22_attention_weight_min": -29.033031463623047,
      "activations/layer23_attention_weight_max": 38.608699798583984,
      "activations/layer23_attention_weight_min": -22.07277488708496,
      "activations/layer2_attention_weight_max": 32.51292419433594,
      "activations/layer2_attention_weight_min": -30.449186325073242,
      "activations/layer3_attention_weight_max": 91.24873352050781,
      "activations/layer3_attention_weight_min": -92.88300323486328,
      "activations/layer4_attention_weight_max": 124.03450012207031,
      "activations/layer4_attention_weight_min": -120.13091278076172,
      "activations/layer5_attention_weight_max": 65.5845947265625,
      "activations/layer5_attention_weight_min": -65.31932830810547,
      "activations/layer6_attention_weight_max": 51.53838348388672,
      "activations/layer6_attention_weight_min": -51.20964431762695,
      "activations/layer7_attention_weight_max": 87.42689514160156,
      "activations/layer7_attention_weight_min": -81.13327026367188,
      "activations/layer8_attention_weight_max": 38.20663833618164,
      "activations/layer8_attention_weight_min": -37.876625061035156,
      "activations/layer9_attention_weight_max": 33.876220703125,
      "activations/layer9_attention_weight_min": -32.923072814941406,
      "epoch": 6.7,
      "learning_rate": 0.00010786212121212121,
      "loss": 2.8109,
      "step": 115350
    },
    {
      "activations/layer0_attention_weight_max": 15.8121337890625,
      "activations/layer0_attention_weight_min": -12.015144348144531,
      "activations/layer10_attention_weight_max": 33.984161376953125,
      "activations/layer10_attention_weight_min": -30.911508560180664,
      "activations/layer11_attention_weight_max": 33.22422790527344,
      "activations/layer11_attention_weight_min": -28.0568904876709,
      "activations/layer12_attention_weight_max": 28.47296905517578,
      "activations/layer12_attention_weight_min": -27.513046264648438,
      "activations/layer13_attention_weight_max": 46.29147720336914,
      "activations/layer13_attention_weight_min": -34.24565124511719,
      "activations/layer14_attention_weight_max": 53.47599792480469,
      "activations/layer14_attention_weight_min": -40.56899642944336,
      "activations/layer15_attention_weight_max": 40.1096305847168,
      "activations/layer15_attention_weight_min": -30.921554565429688,
      "activations/layer16_attention_weight_max": 34.539485931396484,
      "activations/layer16_attention_weight_min": -26.06614875793457,
      "activations/layer17_attention_weight_max": 36.4554328918457,
      "activations/layer17_attention_weight_min": -24.343399047851562,
      "activations/layer18_attention_weight_max": 30.096397399902344,
      "activations/layer18_attention_weight_min": -21.548934936523438,
      "activations/layer19_attention_weight_max": 39.910518646240234,
      "activations/layer19_attention_weight_min": -29.46052360534668,
      "activations/layer1_attention_weight_max": 16.02834701538086,
      "activations/layer1_attention_weight_min": -15.797196388244629,
      "activations/layer20_attention_weight_max": 34.531410217285156,
      "activations/layer20_attention_weight_min": -23.3693904876709,
      "activations/layer21_attention_weight_max": 34.16757583618164,
      "activations/layer21_attention_weight_min": -22.825231552124023,
      "activations/layer22_attention_weight_max": 53.3442497253418,
      "activations/layer22_attention_weight_min": -29.430891036987305,
      "activations/layer23_attention_weight_max": 37.43916320800781,
      "activations/layer23_attention_weight_min": -22.08704948425293,
      "activations/layer2_attention_weight_max": 32.776649475097656,
      "activations/layer2_attention_weight_min": -30.404888153076172,
      "activations/layer3_attention_weight_max": 93.42841339111328,
      "activations/layer3_attention_weight_min": -95.01300048828125,
      "activations/layer4_attention_weight_max": 128.93276977539062,
      "activations/layer4_attention_weight_min": -127.29461669921875,
      "activations/layer5_attention_weight_max": 68.86896514892578,
      "activations/layer5_attention_weight_min": -66.81121826171875,
      "activations/layer6_attention_weight_max": 52.07836151123047,
      "activations/layer6_attention_weight_min": -54.253578186035156,
      "activations/layer7_attention_weight_max": 83.72694396972656,
      "activations/layer7_attention_weight_min": -83.5603256225586,
      "activations/layer8_attention_weight_max": 40.36254119873047,
      "activations/layer8_attention_weight_min": -38.974666595458984,
      "activations/layer9_attention_weight_max": 35.857723236083984,
      "activations/layer9_attention_weight_min": -34.020294189453125,
      "epoch": 6.71,
      "learning_rate": 0.0001078431818181818,
      "loss": 2.8248,
      "step": 115400
    },
    {
      "activations/layer0_attention_weight_max": 15.824413299560547,
      "activations/layer0_attention_weight_min": -11.821632385253906,
      "activations/layer10_attention_weight_max": 33.48769760131836,
      "activations/layer10_attention_weight_min": -33.60993194580078,
      "activations/layer11_attention_weight_max": 34.911293029785156,
      "activations/layer11_attention_weight_min": -30.525930404663086,
      "activations/layer12_attention_weight_max": 43.4115104675293,
      "activations/layer12_attention_weight_min": -25.977563858032227,
      "activations/layer13_attention_weight_max": 40.1901969909668,
      "activations/layer13_attention_weight_min": -33.669029235839844,
      "activations/layer14_attention_weight_max": 59.57115936279297,
      "activations/layer14_attention_weight_min": -44.390419006347656,
      "activations/layer15_attention_weight_max": 37.4463996887207,
      "activations/layer15_attention_weight_min": -31.368581771850586,
      "activations/layer16_attention_weight_max": 37.46443557739258,
      "activations/layer16_attention_weight_min": -27.979137420654297,
      "activations/layer17_attention_weight_max": 35.95713424682617,
      "activations/layer17_attention_weight_min": -25.083410263061523,
      "activations/layer18_attention_weight_max": 33.23750305175781,
      "activations/layer18_attention_weight_min": -21.492544174194336,
      "activations/layer19_attention_weight_max": 38.09272384643555,
      "activations/layer19_attention_weight_min": -29.85562515258789,
      "activations/layer1_attention_weight_max": 16.78154182434082,
      "activations/layer1_attention_weight_min": -15.471115112304688,
      "activations/layer20_attention_weight_max": 30.981420516967773,
      "activations/layer20_attention_weight_min": -26.167373657226562,
      "activations/layer21_attention_weight_max": 34.212223052978516,
      "activations/layer21_attention_weight_min": -22.485164642333984,
      "activations/layer22_attention_weight_max": 54.052894592285156,
      "activations/layer22_attention_weight_min": -29.99704933166504,
      "activations/layer23_attention_weight_max": 40.10305404663086,
      "activations/layer23_attention_weight_min": -21.902877807617188,
      "activations/layer2_attention_weight_max": 31.395259857177734,
      "activations/layer2_attention_weight_min": -30.270620346069336,
      "activations/layer3_attention_weight_max": 90.18083953857422,
      "activations/layer3_attention_weight_min": -88.78246307373047,
      "activations/layer4_attention_weight_max": 122.3496322631836,
      "activations/layer4_attention_weight_min": -121.5300521850586,
      "activations/layer5_attention_weight_max": 65.05538940429688,
      "activations/layer5_attention_weight_min": -62.636940002441406,
      "activations/layer6_attention_weight_max": 50.3421516418457,
      "activations/layer6_attention_weight_min": -53.15673065185547,
      "activations/layer7_attention_weight_max": 83.12708282470703,
      "activations/layer7_attention_weight_min": -84.39352416992188,
      "activations/layer8_attention_weight_max": 42.790767669677734,
      "activations/layer8_attention_weight_min": -38.27385330200195,
      "activations/layer9_attention_weight_max": 37.38316345214844,
      "activations/layer9_attention_weight_min": -33.250511169433594,
      "epoch": 6.71,
      "learning_rate": 0.00010782424242424241,
      "loss": 2.8263,
      "step": 115450
    },
    {
      "activations/layer0_attention_weight_max": 15.894268989562988,
      "activations/layer0_attention_weight_min": -11.783451080322266,
      "activations/layer10_attention_weight_max": 35.533973693847656,
      "activations/layer10_attention_weight_min": -31.892562866210938,
      "activations/layer11_attention_weight_max": 35.76301574707031,
      "activations/layer11_attention_weight_min": -30.180870056152344,
      "activations/layer12_attention_weight_max": 29.059972763061523,
      "activations/layer12_attention_weight_min": -26.38792610168457,
      "activations/layer13_attention_weight_max": 39.36705780029297,
      "activations/layer13_attention_weight_min": -31.44647979736328,
      "activations/layer14_attention_weight_max": 56.803436279296875,
      "activations/layer14_attention_weight_min": -43.28327560424805,
      "activations/layer15_attention_weight_max": 34.982322692871094,
      "activations/layer15_attention_weight_min": -29.713050842285156,
      "activations/layer16_attention_weight_max": 35.71165084838867,
      "activations/layer16_attention_weight_min": -28.5148868560791,
      "activations/layer17_attention_weight_max": 38.929866790771484,
      "activations/layer17_attention_weight_min": -24.725820541381836,
      "activations/layer18_attention_weight_max": 30.25657844543457,
      "activations/layer18_attention_weight_min": -20.14723777770996,
      "activations/layer19_attention_weight_max": 38.536861419677734,
      "activations/layer19_attention_weight_min": -28.54496192932129,
      "activations/layer1_attention_weight_max": 16.539731979370117,
      "activations/layer1_attention_weight_min": -13.600198745727539,
      "activations/layer20_attention_weight_max": 30.5463809967041,
      "activations/layer20_attention_weight_min": -22.657472610473633,
      "activations/layer21_attention_weight_max": 28.438405990600586,
      "activations/layer21_attention_weight_min": -21.260967254638672,
      "activations/layer22_attention_weight_max": 43.17094039916992,
      "activations/layer22_attention_weight_min": -30.379255294799805,
      "activations/layer23_attention_weight_max": 31.898021697998047,
      "activations/layer23_attention_weight_min": -21.539844512939453,
      "activations/layer2_attention_weight_max": 30.947927474975586,
      "activations/layer2_attention_weight_min": -31.301712036132812,
      "activations/layer3_attention_weight_max": 90.64607238769531,
      "activations/layer3_attention_weight_min": -90.09686279296875,
      "activations/layer4_attention_weight_max": 130.00820922851562,
      "activations/layer4_attention_weight_min": -126.02294921875,
      "activations/layer5_attention_weight_max": 68.9805908203125,
      "activations/layer5_attention_weight_min": -64.15836334228516,
      "activations/layer6_attention_weight_max": 50.246849060058594,
      "activations/layer6_attention_weight_min": -51.76023483276367,
      "activations/layer7_attention_weight_max": 86.21043395996094,
      "activations/layer7_attention_weight_min": -84.09930419921875,
      "activations/layer8_attention_weight_max": 40.72233200073242,
      "activations/layer8_attention_weight_min": -41.09333038330078,
      "activations/layer9_attention_weight_max": 37.627079010009766,
      "activations/layer9_attention_weight_min": -33.438968658447266,
      "epoch": 6.71,
      "learning_rate": 0.00010780530303030303,
      "loss": 2.8076,
      "step": 115500
    },
    {
      "activations/layer0_attention_weight_max": 16.03258514404297,
      "activations/layer0_attention_weight_min": -11.778167724609375,
      "activations/layer10_attention_weight_max": 34.65353775024414,
      "activations/layer10_attention_weight_min": -32.705902099609375,
      "activations/layer11_attention_weight_max": 31.567230224609375,
      "activations/layer11_attention_weight_min": -27.59451675415039,
      "activations/layer12_attention_weight_max": 24.38805389404297,
      "activations/layer12_attention_weight_min": -23.89431381225586,
      "activations/layer13_attention_weight_max": 43.60139465332031,
      "activations/layer13_attention_weight_min": -32.999542236328125,
      "activations/layer14_attention_weight_max": 55.38232421875,
      "activations/layer14_attention_weight_min": -41.185699462890625,
      "activations/layer15_attention_weight_max": 38.72002029418945,
      "activations/layer15_attention_weight_min": -29.307174682617188,
      "activations/layer16_attention_weight_max": 35.003116607666016,
      "activations/layer16_attention_weight_min": -28.01018714904785,
      "activations/layer17_attention_weight_max": 35.3509521484375,
      "activations/layer17_attention_weight_min": -25.922529220581055,
      "activations/layer18_attention_weight_max": 31.568941116333008,
      "activations/layer18_attention_weight_min": -23.068580627441406,
      "activations/layer19_attention_weight_max": 37.54134750366211,
      "activations/layer19_attention_weight_min": -29.38791275024414,
      "activations/layer1_attention_weight_max": 16.700796127319336,
      "activations/layer1_attention_weight_min": -16.37378692626953,
      "activations/layer20_attention_weight_max": 31.756195068359375,
      "activations/layer20_attention_weight_min": -23.67546844482422,
      "activations/layer21_attention_weight_max": 30.6229305267334,
      "activations/layer21_attention_weight_min": -22.84288215637207,
      "activations/layer22_attention_weight_max": 55.99406051635742,
      "activations/layer22_attention_weight_min": -32.08195114135742,
      "activations/layer23_attention_weight_max": 39.01371765136719,
      "activations/layer23_attention_weight_min": -28.365888595581055,
      "activations/layer2_attention_weight_max": 33.103912353515625,
      "activations/layer2_attention_weight_min": -30.807270050048828,
      "activations/layer3_attention_weight_max": 91.24321746826172,
      "activations/layer3_attention_weight_min": -90.20603942871094,
      "activations/layer4_attention_weight_max": 124.3694839477539,
      "activations/layer4_attention_weight_min": -120.67396545410156,
      "activations/layer5_attention_weight_max": 65.04222106933594,
      "activations/layer5_attention_weight_min": -62.16856384277344,
      "activations/layer6_attention_weight_max": 52.19004821777344,
      "activations/layer6_attention_weight_min": -55.42430877685547,
      "activations/layer7_attention_weight_max": 82.65196228027344,
      "activations/layer7_attention_weight_min": -85.25012969970703,
      "activations/layer8_attention_weight_max": 39.23931884765625,
      "activations/layer8_attention_weight_min": -38.211997985839844,
      "activations/layer9_attention_weight_max": 36.135711669921875,
      "activations/layer9_attention_weight_min": -34.2216682434082,
      "epoch": 6.71,
      "learning_rate": 0.00010778636363636362,
      "loss": 2.8151,
      "step": 115550
    },
    {
      "activations/layer0_attention_weight_max": 15.19680404663086,
      "activations/layer0_attention_weight_min": -12.189688682556152,
      "activations/layer10_attention_weight_max": 37.833778381347656,
      "activations/layer10_attention_weight_min": -35.56995391845703,
      "activations/layer11_attention_weight_max": 36.340274810791016,
      "activations/layer11_attention_weight_min": -33.598182678222656,
      "activations/layer12_attention_weight_max": 33.96062088012695,
      "activations/layer12_attention_weight_min": -25.094711303710938,
      "activations/layer13_attention_weight_max": 49.420475006103516,
      "activations/layer13_attention_weight_min": -38.80078887939453,
      "activations/layer14_attention_weight_max": 58.016998291015625,
      "activations/layer14_attention_weight_min": -45.714988708496094,
      "activations/layer15_attention_weight_max": 39.84360885620117,
      "activations/layer15_attention_weight_min": -33.18738555908203,
      "activations/layer16_attention_weight_max": 38.173431396484375,
      "activations/layer16_attention_weight_min": -27.453548431396484,
      "activations/layer17_attention_weight_max": 36.33198928833008,
      "activations/layer17_attention_weight_min": -25.077531814575195,
      "activations/layer18_attention_weight_max": 32.23640441894531,
      "activations/layer18_attention_weight_min": -20.845582962036133,
      "activations/layer19_attention_weight_max": 36.941749572753906,
      "activations/layer19_attention_weight_min": -30.353153228759766,
      "activations/layer1_attention_weight_max": 18.036205291748047,
      "activations/layer1_attention_weight_min": -13.911495208740234,
      "activations/layer20_attention_weight_max": 32.886619567871094,
      "activations/layer20_attention_weight_min": -24.03959846496582,
      "activations/layer21_attention_weight_max": 28.220272064208984,
      "activations/layer21_attention_weight_min": -22.027122497558594,
      "activations/layer22_attention_weight_max": 45.005489349365234,
      "activations/layer22_attention_weight_min": -27.86090087890625,
      "activations/layer23_attention_weight_max": 36.29961395263672,
      "activations/layer23_attention_weight_min": -25.440488815307617,
      "activations/layer2_attention_weight_max": 30.906042098999023,
      "activations/layer2_attention_weight_min": -30.739395141601562,
      "activations/layer3_attention_weight_max": 89.77802276611328,
      "activations/layer3_attention_weight_min": -91.61643981933594,
      "activations/layer4_attention_weight_max": 124.29693603515625,
      "activations/layer4_attention_weight_min": -126.42420959472656,
      "activations/layer5_attention_weight_max": 67.36611938476562,
      "activations/layer5_attention_weight_min": -66.10601043701172,
      "activations/layer6_attention_weight_max": 51.76859664916992,
      "activations/layer6_attention_weight_min": -56.189971923828125,
      "activations/layer7_attention_weight_max": 92.35944366455078,
      "activations/layer7_attention_weight_min": -91.37816619873047,
      "activations/layer8_attention_weight_max": 48.54442596435547,
      "activations/layer8_attention_weight_min": -45.56880569458008,
      "activations/layer9_attention_weight_max": 44.08314895629883,
      "activations/layer9_attention_weight_min": -39.75515365600586,
      "epoch": 6.72,
      "learning_rate": 0.00010776742424242423,
      "loss": 2.8406,
      "step": 115600
    },
    {
      "activations/layer0_attention_weight_max": 16.636743545532227,
      "activations/layer0_attention_weight_min": -11.843302726745605,
      "activations/layer10_attention_weight_max": 35.57371520996094,
      "activations/layer10_attention_weight_min": -32.21327590942383,
      "activations/layer11_attention_weight_max": 33.293304443359375,
      "activations/layer11_attention_weight_min": -29.501493453979492,
      "activations/layer12_attention_weight_max": 22.32147789001465,
      "activations/layer12_attention_weight_min": -26.1591739654541,
      "activations/layer13_attention_weight_max": 44.61460876464844,
      "activations/layer13_attention_weight_min": -33.707374572753906,
      "activations/layer14_attention_weight_max": 50.33738327026367,
      "activations/layer14_attention_weight_min": -40.644779205322266,
      "activations/layer15_attention_weight_max": 37.06134033203125,
      "activations/layer15_attention_weight_min": -29.557233810424805,
      "activations/layer16_attention_weight_max": 32.42716979980469,
      "activations/layer16_attention_weight_min": -28.64851188659668,
      "activations/layer17_attention_weight_max": 34.81415557861328,
      "activations/layer17_attention_weight_min": -25.480653762817383,
      "activations/layer18_attention_weight_max": 32.07191467285156,
      "activations/layer18_attention_weight_min": -23.853269577026367,
      "activations/layer19_attention_weight_max": 37.57284927368164,
      "activations/layer19_attention_weight_min": -30.803577423095703,
      "activations/layer1_attention_weight_max": 17.7707462310791,
      "activations/layer1_attention_weight_min": -14.325399398803711,
      "activations/layer20_attention_weight_max": 30.758726119995117,
      "activations/layer20_attention_weight_min": -23.850786209106445,
      "activations/layer21_attention_weight_max": 29.941293716430664,
      "activations/layer21_attention_weight_min": -22.57822036743164,
      "activations/layer22_attention_weight_max": 49.880271911621094,
      "activations/layer22_attention_weight_min": -28.32978057861328,
      "activations/layer23_attention_weight_max": 36.524139404296875,
      "activations/layer23_attention_weight_min": -22.095287322998047,
      "activations/layer2_attention_weight_max": 32.410369873046875,
      "activations/layer2_attention_weight_min": -33.15705108642578,
      "activations/layer3_attention_weight_max": 93.88899230957031,
      "activations/layer3_attention_weight_min": -95.94475555419922,
      "activations/layer4_attention_weight_max": 128.4871368408203,
      "activations/layer4_attention_weight_min": -123.18462371826172,
      "activations/layer5_attention_weight_max": 69.41473388671875,
      "activations/layer5_attention_weight_min": -69.56294250488281,
      "activations/layer6_attention_weight_max": 53.39908218383789,
      "activations/layer6_attention_weight_min": -57.514381408691406,
      "activations/layer7_attention_weight_max": 90.82818603515625,
      "activations/layer7_attention_weight_min": -94.52154541015625,
      "activations/layer8_attention_weight_max": 41.24220657348633,
      "activations/layer8_attention_weight_min": -43.431114196777344,
      "activations/layer9_attention_weight_max": 35.98142623901367,
      "activations/layer9_attention_weight_min": -36.02800369262695,
      "epoch": 6.72,
      "learning_rate": 0.00010774848484848485,
      "loss": 2.8108,
      "step": 115650
    },
    {
      "activations/layer0_attention_weight_max": 16.923194885253906,
      "activations/layer0_attention_weight_min": -12.692994117736816,
      "activations/layer10_attention_weight_max": 40.34391403198242,
      "activations/layer10_attention_weight_min": -35.32405471801758,
      "activations/layer11_attention_weight_max": 40.93813705444336,
      "activations/layer11_attention_weight_min": -31.189123153686523,
      "activations/layer12_attention_weight_max": 32.767486572265625,
      "activations/layer12_attention_weight_min": -31.031436920166016,
      "activations/layer13_attention_weight_max": 55.874351501464844,
      "activations/layer13_attention_weight_min": -35.68169403076172,
      "activations/layer14_attention_weight_max": 67.7854995727539,
      "activations/layer14_attention_weight_min": -43.056217193603516,
      "activations/layer15_attention_weight_max": 47.5658073425293,
      "activations/layer15_attention_weight_min": -29.072538375854492,
      "activations/layer16_attention_weight_max": 37.71127700805664,
      "activations/layer16_attention_weight_min": -28.155418395996094,
      "activations/layer17_attention_weight_max": 42.836769104003906,
      "activations/layer17_attention_weight_min": -23.446508407592773,
      "activations/layer18_attention_weight_max": 32.13300704956055,
      "activations/layer18_attention_weight_min": -20.96143913269043,
      "activations/layer19_attention_weight_max": 45.78001022338867,
      "activations/layer19_attention_weight_min": -28.5626220703125,
      "activations/layer1_attention_weight_max": 16.2518253326416,
      "activations/layer1_attention_weight_min": -15.104546546936035,
      "activations/layer20_attention_weight_max": 37.42877960205078,
      "activations/layer20_attention_weight_min": -21.9786319732666,
      "activations/layer21_attention_weight_max": 35.22379684448242,
      "activations/layer21_attention_weight_min": -20.264026641845703,
      "activations/layer22_attention_weight_max": 61.56745147705078,
      "activations/layer22_attention_weight_min": -28.31085777282715,
      "activations/layer23_attention_weight_max": 44.57535934448242,
      "activations/layer23_attention_weight_min": -21.66252899169922,
      "activations/layer2_attention_weight_max": 32.64959716796875,
      "activations/layer2_attention_weight_min": -31.509937286376953,
      "activations/layer3_attention_weight_max": 92.40568542480469,
      "activations/layer3_attention_weight_min": -94.44808959960938,
      "activations/layer4_attention_weight_max": 131.1629638671875,
      "activations/layer4_attention_weight_min": -129.5844268798828,
      "activations/layer5_attention_weight_max": 70.92868041992188,
      "activations/layer5_attention_weight_min": -70.592529296875,
      "activations/layer6_attention_weight_max": 51.38579559326172,
      "activations/layer6_attention_weight_min": -55.21869659423828,
      "activations/layer7_attention_weight_max": 92.10112762451172,
      "activations/layer7_attention_weight_min": -90.3587417602539,
      "activations/layer8_attention_weight_max": 45.06306457519531,
      "activations/layer8_attention_weight_min": -43.26995849609375,
      "activations/layer9_attention_weight_max": 41.55690383911133,
      "activations/layer9_attention_weight_min": -36.92619323730469,
      "epoch": 6.72,
      "learning_rate": 0.00010772954545454543,
      "loss": 2.8146,
      "step": 115700
    },
    {
      "activations/layer0_attention_weight_max": 16.282268524169922,
      "activations/layer0_attention_weight_min": -12.08386516571045,
      "activations/layer10_attention_weight_max": 33.89482498168945,
      "activations/layer10_attention_weight_min": -32.76729202270508,
      "activations/layer11_attention_weight_max": 35.394309997558594,
      "activations/layer11_attention_weight_min": -32.18788146972656,
      "activations/layer12_attention_weight_max": 31.20602035522461,
      "activations/layer12_attention_weight_min": -29.33201789855957,
      "activations/layer13_attention_weight_max": 40.839622497558594,
      "activations/layer13_attention_weight_min": -31.8460693359375,
      "activations/layer14_attention_weight_max": 55.24686050415039,
      "activations/layer14_attention_weight_min": -43.424476623535156,
      "activations/layer15_attention_weight_max": 34.94418716430664,
      "activations/layer15_attention_weight_min": -27.656110763549805,
      "activations/layer16_attention_weight_max": 31.256038665771484,
      "activations/layer16_attention_weight_min": -27.239456176757812,
      "activations/layer17_attention_weight_max": 34.667999267578125,
      "activations/layer17_attention_weight_min": -24.206329345703125,
      "activations/layer18_attention_weight_max": 30.15709114074707,
      "activations/layer18_attention_weight_min": -20.962663650512695,
      "activations/layer19_attention_weight_max": 36.5894775390625,
      "activations/layer19_attention_weight_min": -29.19502067565918,
      "activations/layer1_attention_weight_max": 17.202585220336914,
      "activations/layer1_attention_weight_min": -14.725637435913086,
      "activations/layer20_attention_weight_max": 32.240657806396484,
      "activations/layer20_attention_weight_min": -25.9390926361084,
      "activations/layer21_attention_weight_max": 33.306884765625,
      "activations/layer21_attention_weight_min": -23.13762855529785,
      "activations/layer22_attention_weight_max": 46.97352600097656,
      "activations/layer22_attention_weight_min": -28.91884994506836,
      "activations/layer23_attention_weight_max": 32.86182403564453,
      "activations/layer23_attention_weight_min": -20.924163818359375,
      "activations/layer2_attention_weight_max": 32.04621124267578,
      "activations/layer2_attention_weight_min": -29.443952560424805,
      "activations/layer3_attention_weight_max": 88.83882904052734,
      "activations/layer3_attention_weight_min": -89.71648406982422,
      "activations/layer4_attention_weight_max": 122.5916519165039,
      "activations/layer4_attention_weight_min": -120.44750213623047,
      "activations/layer5_attention_weight_max": 66.81927490234375,
      "activations/layer5_attention_weight_min": -63.67557907104492,
      "activations/layer6_attention_weight_max": 51.80824279785156,
      "activations/layer6_attention_weight_min": -55.60198974609375,
      "activations/layer7_attention_weight_max": 89.65042114257812,
      "activations/layer7_attention_weight_min": -85.10746765136719,
      "activations/layer8_attention_weight_max": 41.57296371459961,
      "activations/layer8_attention_weight_min": -40.550777435302734,
      "activations/layer9_attention_weight_max": 38.954280853271484,
      "activations/layer9_attention_weight_min": -33.879241943359375,
      "epoch": 6.73,
      "learning_rate": 0.00010771060606060605,
      "loss": 2.7994,
      "step": 115750
    },
    {
      "activations/layer0_attention_weight_max": 16.578853607177734,
      "activations/layer0_attention_weight_min": -12.395187377929688,
      "activations/layer10_attention_weight_max": 34.618263244628906,
      "activations/layer10_attention_weight_min": -31.40291976928711,
      "activations/layer11_attention_weight_max": 33.4005241394043,
      "activations/layer11_attention_weight_min": -29.278430938720703,
      "activations/layer12_attention_weight_max": 41.58208465576172,
      "activations/layer12_attention_weight_min": -30.462413787841797,
      "activations/layer13_attention_weight_max": 46.469242095947266,
      "activations/layer13_attention_weight_min": -33.06114196777344,
      "activations/layer14_attention_weight_max": 60.13361358642578,
      "activations/layer14_attention_weight_min": -41.975128173828125,
      "activations/layer15_attention_weight_max": 36.087772369384766,
      "activations/layer15_attention_weight_min": -30.438583374023438,
      "activations/layer16_attention_weight_max": 35.67194366455078,
      "activations/layer16_attention_weight_min": -28.325145721435547,
      "activations/layer17_attention_weight_max": 37.92012405395508,
      "activations/layer17_attention_weight_min": -24.85264778137207,
      "activations/layer18_attention_weight_max": 33.13151168823242,
      "activations/layer18_attention_weight_min": -19.980224609375,
      "activations/layer19_attention_weight_max": 34.933170318603516,
      "activations/layer19_attention_weight_min": -30.503408432006836,
      "activations/layer1_attention_weight_max": 17.453493118286133,
      "activations/layer1_attention_weight_min": -17.553144454956055,
      "activations/layer20_attention_weight_max": 35.382259368896484,
      "activations/layer20_attention_weight_min": -23.83877182006836,
      "activations/layer21_attention_weight_max": 32.230712890625,
      "activations/layer21_attention_weight_min": -20.874723434448242,
      "activations/layer22_attention_weight_max": 48.95977020263672,
      "activations/layer22_attention_weight_min": -27.50699806213379,
      "activations/layer23_attention_weight_max": 34.57526397705078,
      "activations/layer23_attention_weight_min": -22.314483642578125,
      "activations/layer2_attention_weight_max": 33.33787536621094,
      "activations/layer2_attention_weight_min": -30.5704345703125,
      "activations/layer3_attention_weight_max": 94.73816680908203,
      "activations/layer3_attention_weight_min": -95.26148223876953,
      "activations/layer4_attention_weight_max": 123.04357147216797,
      "activations/layer4_attention_weight_min": -133.0190887451172,
      "activations/layer5_attention_weight_max": 65.91873168945312,
      "activations/layer5_attention_weight_min": -68.00961303710938,
      "activations/layer6_attention_weight_max": 53.591129302978516,
      "activations/layer6_attention_weight_min": -55.75448989868164,
      "activations/layer7_attention_weight_max": 89.49964141845703,
      "activations/layer7_attention_weight_min": -86.69725036621094,
      "activations/layer8_attention_weight_max": 40.53703689575195,
      "activations/layer8_attention_weight_min": -39.46453857421875,
      "activations/layer9_attention_weight_max": 38.108795166015625,
      "activations/layer9_attention_weight_min": -33.892051696777344,
      "epoch": 6.73,
      "learning_rate": 0.00010769166666666666,
      "loss": 2.841,
      "step": 115800
    },
    {
      "activations/layer0_attention_weight_max": 15.311934471130371,
      "activations/layer0_attention_weight_min": -11.467179298400879,
      "activations/layer10_attention_weight_max": 35.17837905883789,
      "activations/layer10_attention_weight_min": -33.97959899902344,
      "activations/layer11_attention_weight_max": 34.45796585083008,
      "activations/layer11_attention_weight_min": -31.531539916992188,
      "activations/layer12_attention_weight_max": 44.789894104003906,
      "activations/layer12_attention_weight_min": -25.746395111083984,
      "activations/layer13_attention_weight_max": 43.2590217590332,
      "activations/layer13_attention_weight_min": -32.63092803955078,
      "activations/layer14_attention_weight_max": 54.38829803466797,
      "activations/layer14_attention_weight_min": -44.000282287597656,
      "activations/layer15_attention_weight_max": 41.57888412475586,
      "activations/layer15_attention_weight_min": -31.564802169799805,
      "activations/layer16_attention_weight_max": 37.48728942871094,
      "activations/layer16_attention_weight_min": -27.296249389648438,
      "activations/layer17_attention_weight_max": 37.55007553100586,
      "activations/layer17_attention_weight_min": -27.516855239868164,
      "activations/layer18_attention_weight_max": 33.494686126708984,
      "activations/layer18_attention_weight_min": -20.92559242248535,
      "activations/layer19_attention_weight_max": 40.888893127441406,
      "activations/layer19_attention_weight_min": -30.372283935546875,
      "activations/layer1_attention_weight_max": 17.101293563842773,
      "activations/layer1_attention_weight_min": -14.852971076965332,
      "activations/layer20_attention_weight_max": 34.40232849121094,
      "activations/layer20_attention_weight_min": -25.738384246826172,
      "activations/layer21_attention_weight_max": 33.86777114868164,
      "activations/layer21_attention_weight_min": -23.53990936279297,
      "activations/layer22_attention_weight_max": 55.18619155883789,
      "activations/layer22_attention_weight_min": -30.09944725036621,
      "activations/layer23_attention_weight_max": 37.59039306640625,
      "activations/layer23_attention_weight_min": -21.176441192626953,
      "activations/layer2_attention_weight_max": 33.24369812011719,
      "activations/layer2_attention_weight_min": -29.69571304321289,
      "activations/layer3_attention_weight_max": 92.4056167602539,
      "activations/layer3_attention_weight_min": -92.38065338134766,
      "activations/layer4_attention_weight_max": 130.94383239746094,
      "activations/layer4_attention_weight_min": -126.57250213623047,
      "activations/layer5_attention_weight_max": 68.9975814819336,
      "activations/layer5_attention_weight_min": -64.93299865722656,
      "activations/layer6_attention_weight_max": 55.207176208496094,
      "activations/layer6_attention_weight_min": -55.48027420043945,
      "activations/layer7_attention_weight_max": 91.04676818847656,
      "activations/layer7_attention_weight_min": -93.91323852539062,
      "activations/layer8_attention_weight_max": 42.019874572753906,
      "activations/layer8_attention_weight_min": -43.4096565246582,
      "activations/layer9_attention_weight_max": 37.29383850097656,
      "activations/layer9_attention_weight_min": -36.934303283691406,
      "epoch": 6.73,
      "learning_rate": 0.00010767272727272727,
      "loss": 2.8271,
      "step": 115850
    },
    {
      "activations/layer0_attention_weight_max": 16.367584228515625,
      "activations/layer0_attention_weight_min": -12.502305030822754,
      "activations/layer10_attention_weight_max": 41.113975524902344,
      "activations/layer10_attention_weight_min": -36.70811080932617,
      "activations/layer11_attention_weight_max": 39.38835144042969,
      "activations/layer11_attention_weight_min": -32.91828155517578,
      "activations/layer12_attention_weight_max": 28.365615844726562,
      "activations/layer12_attention_weight_min": -38.912540435791016,
      "activations/layer13_attention_weight_max": 53.26192092895508,
      "activations/layer13_attention_weight_min": -36.449485778808594,
      "activations/layer14_attention_weight_max": 71.89578247070312,
      "activations/layer14_attention_weight_min": -49.165199279785156,
      "activations/layer15_attention_weight_max": 41.23430633544922,
      "activations/layer15_attention_weight_min": -32.68532180786133,
      "activations/layer16_attention_weight_max": 39.13369369506836,
      "activations/layer16_attention_weight_min": -26.76503562927246,
      "activations/layer17_attention_weight_max": 40.57828140258789,
      "activations/layer17_attention_weight_min": -25.42777442932129,
      "activations/layer18_attention_weight_max": 36.746360778808594,
      "activations/layer18_attention_weight_min": -23.593507766723633,
      "activations/layer19_attention_weight_max": 46.03049087524414,
      "activations/layer19_attention_weight_min": -30.353235244750977,
      "activations/layer1_attention_weight_max": 17.099287033081055,
      "activations/layer1_attention_weight_min": -15.379103660583496,
      "activations/layer20_attention_weight_max": 37.922149658203125,
      "activations/layer20_attention_weight_min": -25.214059829711914,
      "activations/layer21_attention_weight_max": 34.239967346191406,
      "activations/layer21_attention_weight_min": -22.115163803100586,
      "activations/layer22_attention_weight_max": 59.27891540527344,
      "activations/layer22_attention_weight_min": -29.885915756225586,
      "activations/layer23_attention_weight_max": 44.17681884765625,
      "activations/layer23_attention_weight_min": -22.99873924255371,
      "activations/layer2_attention_weight_max": 33.27799606323242,
      "activations/layer2_attention_weight_min": -31.931289672851562,
      "activations/layer3_attention_weight_max": 91.7210464477539,
      "activations/layer3_attention_weight_min": -93.71247863769531,
      "activations/layer4_attention_weight_max": 132.68067932128906,
      "activations/layer4_attention_weight_min": -131.8419189453125,
      "activations/layer5_attention_weight_max": 73.70207214355469,
      "activations/layer5_attention_weight_min": -69.62872314453125,
      "activations/layer6_attention_weight_max": 57.63942337036133,
      "activations/layer6_attention_weight_min": -58.875431060791016,
      "activations/layer7_attention_weight_max": 101.72651672363281,
      "activations/layer7_attention_weight_min": -99.70413970947266,
      "activations/layer8_attention_weight_max": 48.81313705444336,
      "activations/layer8_attention_weight_min": -46.08448791503906,
      "activations/layer9_attention_weight_max": 43.727012634277344,
      "activations/layer9_attention_weight_min": -39.72212600708008,
      "epoch": 6.73,
      "learning_rate": 0.00010765378787878787,
      "loss": 2.8242,
      "step": 115900
    },
    {
      "activations/layer0_attention_weight_max": 15.463424682617188,
      "activations/layer0_attention_weight_min": -12.32894515991211,
      "activations/layer10_attention_weight_max": 31.8997802734375,
      "activations/layer10_attention_weight_min": -29.564388275146484,
      "activations/layer11_attention_weight_max": 33.42438888549805,
      "activations/layer11_attention_weight_min": -28.61794090270996,
      "activations/layer12_attention_weight_max": 26.197904586791992,
      "activations/layer12_attention_weight_min": -27.771808624267578,
      "activations/layer13_attention_weight_max": 42.07267379760742,
      "activations/layer13_attention_weight_min": -32.187923431396484,
      "activations/layer14_attention_weight_max": 54.189231872558594,
      "activations/layer14_attention_weight_min": -41.56013870239258,
      "activations/layer15_attention_weight_max": 38.0239372253418,
      "activations/layer15_attention_weight_min": -28.49202537536621,
      "activations/layer16_attention_weight_max": 36.41291809082031,
      "activations/layer16_attention_weight_min": -28.11623191833496,
      "activations/layer17_attention_weight_max": 35.99481964111328,
      "activations/layer17_attention_weight_min": -25.776561737060547,
      "activations/layer18_attention_weight_max": 30.514041900634766,
      "activations/layer18_attention_weight_min": -21.943470001220703,
      "activations/layer19_attention_weight_max": 39.5157356262207,
      "activations/layer19_attention_weight_min": -31.261213302612305,
      "activations/layer1_attention_weight_max": 16.05799102783203,
      "activations/layer1_attention_weight_min": -13.992963790893555,
      "activations/layer20_attention_weight_max": 34.0103874206543,
      "activations/layer20_attention_weight_min": -25.90615463256836,
      "activations/layer21_attention_weight_max": 33.04363250732422,
      "activations/layer21_attention_weight_min": -22.8136043548584,
      "activations/layer22_attention_weight_max": 49.50568389892578,
      "activations/layer22_attention_weight_min": -32.55295181274414,
      "activations/layer23_attention_weight_max": 36.06382369995117,
      "activations/layer23_attention_weight_min": -22.950881958007812,
      "activations/layer2_attention_weight_max": 29.454139709472656,
      "activations/layer2_attention_weight_min": -29.57240104675293,
      "activations/layer3_attention_weight_max": 83.89970397949219,
      "activations/layer3_attention_weight_min": -84.48544311523438,
      "activations/layer4_attention_weight_max": 115.41334533691406,
      "activations/layer4_attention_weight_min": -112.84354400634766,
      "activations/layer5_attention_weight_max": 59.21903610229492,
      "activations/layer5_attention_weight_min": -59.2166862487793,
      "activations/layer6_attention_weight_max": 46.98503875732422,
      "activations/layer6_attention_weight_min": -48.04747772216797,
      "activations/layer7_attention_weight_max": 79.48573303222656,
      "activations/layer7_attention_weight_min": -80.8865737915039,
      "activations/layer8_attention_weight_max": 37.1318359375,
      "activations/layer8_attention_weight_min": -38.30165481567383,
      "activations/layer9_attention_weight_max": 37.13301086425781,
      "activations/layer9_attention_weight_min": -36.30965805053711,
      "epoch": 6.74,
      "learning_rate": 0.00010763484848484847,
      "loss": 2.8169,
      "step": 115950
    },
    {
      "activations/layer0_attention_weight_max": 16.119525909423828,
      "activations/layer0_attention_weight_min": -11.855161666870117,
      "activations/layer10_attention_weight_max": 43.9066162109375,
      "activations/layer10_attention_weight_min": -37.663936614990234,
      "activations/layer11_attention_weight_max": 48.318206787109375,
      "activations/layer11_attention_weight_min": -35.22678756713867,
      "activations/layer12_attention_weight_max": 46.702693939208984,
      "activations/layer12_attention_weight_min": -30.894071578979492,
      "activations/layer13_attention_weight_max": 74.5608139038086,
      "activations/layer13_attention_weight_min": -39.90499496459961,
      "activations/layer14_attention_weight_max": 58.630828857421875,
      "activations/layer14_attention_weight_min": -38.603424072265625,
      "activations/layer15_attention_weight_max": 70.72421264648438,
      "activations/layer15_attention_weight_min": -39.064517974853516,
      "activations/layer16_attention_weight_max": 50.69189453125,
      "activations/layer16_attention_weight_min": -26.65463638305664,
      "activations/layer17_attention_weight_max": 35.99543762207031,
      "activations/layer17_attention_weight_min": -23.09918975830078,
      "activations/layer18_attention_weight_max": 31.821701049804688,
      "activations/layer18_attention_weight_min": -21.901840209960938,
      "activations/layer19_attention_weight_max": 37.05624008178711,
      "activations/layer19_attention_weight_min": -30.655689239501953,
      "activations/layer1_attention_weight_max": 17.563756942749023,
      "activations/layer1_attention_weight_min": -16.511472702026367,
      "activations/layer20_attention_weight_max": 29.794282913208008,
      "activations/layer20_attention_weight_min": -25.804649353027344,
      "activations/layer21_attention_weight_max": 31.877309799194336,
      "activations/layer21_attention_weight_min": -22.044530868530273,
      "activations/layer22_attention_weight_max": 43.90933609008789,
      "activations/layer22_attention_weight_min": -27.740081787109375,
      "activations/layer23_attention_weight_max": 33.194740295410156,
      "activations/layer23_attention_weight_min": -22.153076171875,
      "activations/layer2_attention_weight_max": 33.891395568847656,
      "activations/layer2_attention_weight_min": -31.96753692626953,
      "activations/layer3_attention_weight_max": 87.65548706054688,
      "activations/layer3_attention_weight_min": -90.26866149902344,
      "activations/layer4_attention_weight_max": 126.5690689086914,
      "activations/layer4_attention_weight_min": -116.75308990478516,
      "activations/layer5_attention_weight_max": 68.01589965820312,
      "activations/layer5_attention_weight_min": -63.415016174316406,
      "activations/layer6_attention_weight_max": 54.78342056274414,
      "activations/layer6_attention_weight_min": -50.26137924194336,
      "activations/layer7_attention_weight_max": 104.65058135986328,
      "activations/layer7_attention_weight_min": -88.49005889892578,
      "activations/layer8_attention_weight_max": 47.72037124633789,
      "activations/layer8_attention_weight_min": -43.68440246582031,
      "activations/layer9_attention_weight_max": 46.264766693115234,
      "activations/layer9_attention_weight_min": -40.644649505615234,
      "epoch": 6.74,
      "learning_rate": 0.00010761590909090908,
      "loss": 2.8208,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_loss": 2.775390625,
      "eval_runtime": 8.5487,
      "eval_samples_per_second": 502.298,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_openwebtext_loss": 2.775390625,
      "eval_openwebtext_ppl": 16.04489330813532,
      "eval_openwebtext_runtime": 8.5487,
      "eval_openwebtext_samples_per_second": 502.298,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_wikitext_loss": 3.00390625,
      "eval_wikitext_ppl": 20.16414949200632,
      "eval_wikitext_runtime": 2.0387,
      "eval_wikitext_samples_per_second": 223.67,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_lambada_loss": 2.638671875,
      "eval_lambada_ppl": 13.994604675279588,
      "eval_lambada_runtime": 9.581,
      "eval_lambada_samples_per_second": 508.194,
      "step": 116000
    },
    {
      "activations/layer0_attention_weight_max": 17.083829879760742,
      "activations/layer0_attention_weight_min": -11.108901977539062,
      "activations/layer10_attention_weight_max": 34.52505111694336,
      "activations/layer10_attention_weight_min": -32.05339431762695,
      "activations/layer11_attention_weight_max": 34.099937438964844,
      "activations/layer11_attention_weight_min": -33.25388717651367,
      "activations/layer12_attention_weight_max": 22.784482955932617,
      "activations/layer12_attention_weight_min": -26.1092472076416,
      "activations/layer13_attention_weight_max": 40.3638916015625,
      "activations/layer13_attention_weight_min": -33.47785568237305,
      "activations/layer14_attention_weight_max": 54.173728942871094,
      "activations/layer14_attention_weight_min": -43.93010711669922,
      "activations/layer15_attention_weight_max": 38.1697883605957,
      "activations/layer15_attention_weight_min": -33.042232513427734,
      "activations/layer16_attention_weight_max": 33.61832809448242,
      "activations/layer16_attention_weight_min": -28.57439422607422,
      "activations/layer17_attention_weight_max": 35.073089599609375,
      "activations/layer17_attention_weight_min": -25.887866973876953,
      "activations/layer18_attention_weight_max": 34.35973358154297,
      "activations/layer18_attention_weight_min": -19.74813461303711,
      "activations/layer19_attention_weight_max": 36.94749069213867,
      "activations/layer19_attention_weight_min": -30.180234909057617,
      "activations/layer1_attention_weight_max": 17.04568099975586,
      "activations/layer1_attention_weight_min": -14.005106925964355,
      "activations/layer20_attention_weight_max": 30.446685791015625,
      "activations/layer20_attention_weight_min": -22.337230682373047,
      "activations/layer21_attention_weight_max": 33.029937744140625,
      "activations/layer21_attention_weight_min": -22.083629608154297,
      "activations/layer22_attention_weight_max": 45.88895797729492,
      "activations/layer22_attention_weight_min": -27.788625717163086,
      "activations/layer23_attention_weight_max": 35.151466369628906,
      "activations/layer23_attention_weight_min": -20.985641479492188,
      "activations/layer2_attention_weight_max": 32.049503326416016,
      "activations/layer2_attention_weight_min": -31.401958465576172,
      "activations/layer3_attention_weight_max": 91.6268310546875,
      "activations/layer3_attention_weight_min": -93.95787048339844,
      "activations/layer4_attention_weight_max": 131.16229248046875,
      "activations/layer4_attention_weight_min": -134.9481201171875,
      "activations/layer5_attention_weight_max": 68.5953369140625,
      "activations/layer5_attention_weight_min": -69.4963607788086,
      "activations/layer6_attention_weight_max": 54.94337844848633,
      "activations/layer6_attention_weight_min": -56.70407485961914,
      "activations/layer7_attention_weight_max": 90.3945083618164,
      "activations/layer7_attention_weight_min": -93.31778717041016,
      "activations/layer8_attention_weight_max": 42.52254104614258,
      "activations/layer8_attention_weight_min": -42.156951904296875,
      "activations/layer9_attention_weight_max": 38.01456832885742,
      "activations/layer9_attention_weight_min": -35.75790786743164,
      "epoch": 6.74,
      "learning_rate": 0.0001075969696969697,
      "loss": 2.8227,
      "step": 116050
    },
    {
      "activations/layer0_attention_weight_max": 16.915241241455078,
      "activations/layer0_attention_weight_min": -11.921221733093262,
      "activations/layer10_attention_weight_max": 34.29734802246094,
      "activations/layer10_attention_weight_min": -33.4356575012207,
      "activations/layer11_attention_weight_max": 31.567916870117188,
      "activations/layer11_attention_weight_min": -28.865386962890625,
      "activations/layer12_attention_weight_max": 23.949995040893555,
      "activations/layer12_attention_weight_min": -28.034818649291992,
      "activations/layer13_attention_weight_max": 37.300811767578125,
      "activations/layer13_attention_weight_min": -32.7506217956543,
      "activations/layer14_attention_weight_max": 54.62339782714844,
      "activations/layer14_attention_weight_min": -44.104637145996094,
      "activations/layer15_attention_weight_max": 36.89676284790039,
      "activations/layer15_attention_weight_min": -28.521989822387695,
      "activations/layer16_attention_weight_max": 35.913108825683594,
      "activations/layer16_attention_weight_min": -28.012189865112305,
      "activations/layer17_attention_weight_max": 36.517845153808594,
      "activations/layer17_attention_weight_min": -24.602088928222656,
      "activations/layer18_attention_weight_max": 29.278018951416016,
      "activations/layer18_attention_weight_min": -20.75289535522461,
      "activations/layer19_attention_weight_max": 38.90656661987305,
      "activations/layer19_attention_weight_min": -28.935453414916992,
      "activations/layer1_attention_weight_max": 16.519929885864258,
      "activations/layer1_attention_weight_min": -15.37239933013916,
      "activations/layer20_attention_weight_max": 29.791929244995117,
      "activations/layer20_attention_weight_min": -23.341175079345703,
      "activations/layer21_attention_weight_max": 29.52145767211914,
      "activations/layer21_attention_weight_min": -22.61273956298828,
      "activations/layer22_attention_weight_max": 48.89040756225586,
      "activations/layer22_attention_weight_min": -28.197391510009766,
      "activations/layer23_attention_weight_max": 37.261199951171875,
      "activations/layer23_attention_weight_min": -21.868118286132812,
      "activations/layer2_attention_weight_max": 31.79925537109375,
      "activations/layer2_attention_weight_min": -30.51051139831543,
      "activations/layer3_attention_weight_max": 89.34390258789062,
      "activations/layer3_attention_weight_min": -91.03790283203125,
      "activations/layer4_attention_weight_max": 121.93123626708984,
      "activations/layer4_attention_weight_min": -121.7695083618164,
      "activations/layer5_attention_weight_max": 63.34859848022461,
      "activations/layer5_attention_weight_min": -61.8226432800293,
      "activations/layer6_attention_weight_max": 49.472511291503906,
      "activations/layer6_attention_weight_min": -49.96589279174805,
      "activations/layer7_attention_weight_max": 89.98180389404297,
      "activations/layer7_attention_weight_min": -85.19483184814453,
      "activations/layer8_attention_weight_max": 43.82004165649414,
      "activations/layer8_attention_weight_min": -42.54053497314453,
      "activations/layer9_attention_weight_max": 37.45062255859375,
      "activations/layer9_attention_weight_min": -35.851600646972656,
      "epoch": 6.75,
      "learning_rate": 0.00010757803030303029,
      "loss": 2.8282,
      "step": 116100
    },
    {
      "activations/layer0_attention_weight_max": 16.78365707397461,
      "activations/layer0_attention_weight_min": -12.35703182220459,
      "activations/layer10_attention_weight_max": 49.7545051574707,
      "activations/layer10_attention_weight_min": -42.586673736572266,
      "activations/layer11_attention_weight_max": 60.20588684082031,
      "activations/layer11_attention_weight_min": -41.99676513671875,
      "activations/layer12_attention_weight_max": 28.10587501525879,
      "activations/layer12_attention_weight_min": -25.7239933013916,
      "activations/layer13_attention_weight_max": 39.75524139404297,
      "activations/layer13_attention_weight_min": -32.13987350463867,
      "activations/layer14_attention_weight_max": 58.599143981933594,
      "activations/layer14_attention_weight_min": -51.58452606201172,
      "activations/layer15_attention_weight_max": 36.48009490966797,
      "activations/layer15_attention_weight_min": -29.7680721282959,
      "activations/layer16_attention_weight_max": 32.67599868774414,
      "activations/layer16_attention_weight_min": -29.268360137939453,
      "activations/layer17_attention_weight_max": 32.46070861816406,
      "activations/layer17_attention_weight_min": -26.077505111694336,
      "activations/layer18_attention_weight_max": 31.418794631958008,
      "activations/layer18_attention_weight_min": -22.843244552612305,
      "activations/layer19_attention_weight_max": 32.19797897338867,
      "activations/layer19_attention_weight_min": -30.04046630859375,
      "activations/layer1_attention_weight_max": 17.028249740600586,
      "activations/layer1_attention_weight_min": -14.030645370483398,
      "activations/layer20_attention_weight_max": 28.328189849853516,
      "activations/layer20_attention_weight_min": -23.569211959838867,
      "activations/layer21_attention_weight_max": 29.58592414855957,
      "activations/layer21_attention_weight_min": -22.919876098632812,
      "activations/layer22_attention_weight_max": 50.49333953857422,
      "activations/layer22_attention_weight_min": -30.85124397277832,
      "activations/layer23_attention_weight_max": 37.460670471191406,
      "activations/layer23_attention_weight_min": -21.469640731811523,
      "activations/layer2_attention_weight_max": 31.635162353515625,
      "activations/layer2_attention_weight_min": -30.556432723999023,
      "activations/layer3_attention_weight_max": 92.83613586425781,
      "activations/layer3_attention_weight_min": -90.20850372314453,
      "activations/layer4_attention_weight_max": 133.38902282714844,
      "activations/layer4_attention_weight_min": -121.36408996582031,
      "activations/layer5_attention_weight_max": 68.6754150390625,
      "activations/layer5_attention_weight_min": -63.434471130371094,
      "activations/layer6_attention_weight_max": 53.751243591308594,
      "activations/layer6_attention_weight_min": -53.14468002319336,
      "activations/layer7_attention_weight_max": 106.46501922607422,
      "activations/layer7_attention_weight_min": -88.97925567626953,
      "activations/layer8_attention_weight_max": 50.17005920410156,
      "activations/layer8_attention_weight_min": -46.45927810668945,
      "activations/layer9_attention_weight_max": 47.951988220214844,
      "activations/layer9_attention_weight_min": -43.86785888671875,
      "epoch": 6.75,
      "learning_rate": 0.0001075590909090909,
      "loss": 2.8176,
      "step": 116150
    },
    {
      "activations/layer0_attention_weight_max": 16.1090030670166,
      "activations/layer0_attention_weight_min": -11.931982040405273,
      "activations/layer10_attention_weight_max": 35.75410079956055,
      "activations/layer10_attention_weight_min": -31.717575073242188,
      "activations/layer11_attention_weight_max": 34.631202697753906,
      "activations/layer11_attention_weight_min": -30.210479736328125,
      "activations/layer12_attention_weight_max": 24.319786071777344,
      "activations/layer12_attention_weight_min": -28.078886032104492,
      "activations/layer13_attention_weight_max": 43.382930755615234,
      "activations/layer13_attention_weight_min": -35.205482482910156,
      "activations/layer14_attention_weight_max": 69.53721618652344,
      "activations/layer14_attention_weight_min": -50.30863571166992,
      "activations/layer15_attention_weight_max": 38.898902893066406,
      "activations/layer15_attention_weight_min": -31.627620697021484,
      "activations/layer16_attention_weight_max": 35.973899841308594,
      "activations/layer16_attention_weight_min": -28.00200653076172,
      "activations/layer17_attention_weight_max": 40.34589767456055,
      "activations/layer17_attention_weight_min": -25.32504653930664,
      "activations/layer18_attention_weight_max": 41.24906921386719,
      "activations/layer18_attention_weight_min": -22.6944637298584,
      "activations/layer19_attention_weight_max": 40.439918518066406,
      "activations/layer19_attention_weight_min": -30.077848434448242,
      "activations/layer1_attention_weight_max": 17.00458335876465,
      "activations/layer1_attention_weight_min": -13.595062255859375,
      "activations/layer20_attention_weight_max": 33.63072967529297,
      "activations/layer20_attention_weight_min": -25.84874725341797,
      "activations/layer21_attention_weight_max": 35.36506271362305,
      "activations/layer21_attention_weight_min": -22.469118118286133,
      "activations/layer22_attention_weight_max": 55.29063034057617,
      "activations/layer22_attention_weight_min": -29.100324630737305,
      "activations/layer23_attention_weight_max": 39.868682861328125,
      "activations/layer23_attention_weight_min": -23.775272369384766,
      "activations/layer2_attention_weight_max": 32.394203186035156,
      "activations/layer2_attention_weight_min": -30.86224937438965,
      "activations/layer3_attention_weight_max": 88.72708892822266,
      "activations/layer3_attention_weight_min": -87.96058654785156,
      "activations/layer4_attention_weight_max": 121.49364471435547,
      "activations/layer4_attention_weight_min": -123.1796875,
      "activations/layer5_attention_weight_max": 64.23919677734375,
      "activations/layer5_attention_weight_min": -62.973548889160156,
      "activations/layer6_attention_weight_max": 52.58939743041992,
      "activations/layer6_attention_weight_min": -53.14414978027344,
      "activations/layer7_attention_weight_max": 86.93425750732422,
      "activations/layer7_attention_weight_min": -86.24261474609375,
      "activations/layer8_attention_weight_max": 42.64244079589844,
      "activations/layer8_attention_weight_min": -40.28440856933594,
      "activations/layer9_attention_weight_max": 38.74624252319336,
      "activations/layer9_attention_weight_min": -34.098201751708984,
      "epoch": 6.75,
      "learning_rate": 0.00010754015151515152,
      "loss": 2.847,
      "step": 116200
    },
    {
      "activations/layer0_attention_weight_max": 14.935708999633789,
      "activations/layer0_attention_weight_min": -11.390437126159668,
      "activations/layer10_attention_weight_max": 34.906402587890625,
      "activations/layer10_attention_weight_min": -32.48925018310547,
      "activations/layer11_attention_weight_max": 32.8900146484375,
      "activations/layer11_attention_weight_min": -29.399137496948242,
      "activations/layer12_attention_weight_max": 25.660449981689453,
      "activations/layer12_attention_weight_min": -23.973617553710938,
      "activations/layer13_attention_weight_max": 42.958309173583984,
      "activations/layer13_attention_weight_min": -34.70409393310547,
      "activations/layer14_attention_weight_max": 70.98448944091797,
      "activations/layer14_attention_weight_min": -51.933963775634766,
      "activations/layer15_attention_weight_max": 39.36137771606445,
      "activations/layer15_attention_weight_min": -29.185209274291992,
      "activations/layer16_attention_weight_max": 37.60208511352539,
      "activations/layer16_attention_weight_min": -29.42787742614746,
      "activations/layer17_attention_weight_max": 41.51921463012695,
      "activations/layer17_attention_weight_min": -25.574411392211914,
      "activations/layer18_attention_weight_max": 31.906574249267578,
      "activations/layer18_attention_weight_min": -20.913631439208984,
      "activations/layer19_attention_weight_max": 42.46650695800781,
      "activations/layer19_attention_weight_min": -32.04793167114258,
      "activations/layer1_attention_weight_max": 17.168136596679688,
      "activations/layer1_attention_weight_min": -12.910749435424805,
      "activations/layer20_attention_weight_max": 37.6285400390625,
      "activations/layer20_attention_weight_min": -22.926849365234375,
      "activations/layer21_attention_weight_max": 36.57538604736328,
      "activations/layer21_attention_weight_min": -22.184484481811523,
      "activations/layer22_attention_weight_max": 55.468963623046875,
      "activations/layer22_attention_weight_min": -25.833480834960938,
      "activations/layer23_attention_weight_max": 38.767982482910156,
      "activations/layer23_attention_weight_min": -21.07000160217285,
      "activations/layer2_attention_weight_max": 32.25273132324219,
      "activations/layer2_attention_weight_min": -31.18634033203125,
      "activations/layer3_attention_weight_max": 92.20087432861328,
      "activations/layer3_attention_weight_min": -90.71569061279297,
      "activations/layer4_attention_weight_max": 127.0855941772461,
      "activations/layer4_attention_weight_min": -123.2887954711914,
      "activations/layer5_attention_weight_max": 65.77533721923828,
      "activations/layer5_attention_weight_min": -66.52169799804688,
      "activations/layer6_attention_weight_max": 53.82111740112305,
      "activations/layer6_attention_weight_min": -54.635704040527344,
      "activations/layer7_attention_weight_max": 90.53488159179688,
      "activations/layer7_attention_weight_min": -87.39981079101562,
      "activations/layer8_attention_weight_max": 41.17470932006836,
      "activations/layer8_attention_weight_min": -40.68232727050781,
      "activations/layer9_attention_weight_max": 38.3250846862793,
      "activations/layer9_attention_weight_min": -36.730220794677734,
      "epoch": 6.75,
      "learning_rate": 0.0001075212121212121,
      "loss": 2.8203,
      "step": 116250
    },
    {
      "activations/layer0_attention_weight_max": 16.544973373413086,
      "activations/layer0_attention_weight_min": -12.817007064819336,
      "activations/layer10_attention_weight_max": 35.30061340332031,
      "activations/layer10_attention_weight_min": -32.1539306640625,
      "activations/layer11_attention_weight_max": 37.364768981933594,
      "activations/layer11_attention_weight_min": -32.07722091674805,
      "activations/layer12_attention_weight_max": 41.676448822021484,
      "activations/layer12_attention_weight_min": -30.099773406982422,
      "activations/layer13_attention_weight_max": 43.45390701293945,
      "activations/layer13_attention_weight_min": -34.30820083618164,
      "activations/layer14_attention_weight_max": 59.1146354675293,
      "activations/layer14_attention_weight_min": -43.13686752319336,
      "activations/layer15_attention_weight_max": 37.0301513671875,
      "activations/layer15_attention_weight_min": -29.644927978515625,
      "activations/layer16_attention_weight_max": 33.6257438659668,
      "activations/layer16_attention_weight_min": -26.980640411376953,
      "activations/layer17_attention_weight_max": 36.22838592529297,
      "activations/layer17_attention_weight_min": -23.965715408325195,
      "activations/layer18_attention_weight_max": 32.19525146484375,
      "activations/layer18_attention_weight_min": -18.7845516204834,
      "activations/layer19_attention_weight_max": 36.427494049072266,
      "activations/layer19_attention_weight_min": -28.284137725830078,
      "activations/layer1_attention_weight_max": 17.62364959716797,
      "activations/layer1_attention_weight_min": -12.88884449005127,
      "activations/layer20_attention_weight_max": 29.905492782592773,
      "activations/layer20_attention_weight_min": -22.612382888793945,
      "activations/layer21_attention_weight_max": 28.647228240966797,
      "activations/layer21_attention_weight_min": -19.905323028564453,
      "activations/layer22_attention_weight_max": 45.85051345825195,
      "activations/layer22_attention_weight_min": -27.306795120239258,
      "activations/layer23_attention_weight_max": 32.591941833496094,
      "activations/layer23_attention_weight_min": -22.487380981445312,
      "activations/layer2_attention_weight_max": 33.532630920410156,
      "activations/layer2_attention_weight_min": -31.359031677246094,
      "activations/layer3_attention_weight_max": 92.25544738769531,
      "activations/layer3_attention_weight_min": -95.84286499023438,
      "activations/layer4_attention_weight_max": 134.80081176757812,
      "activations/layer4_attention_weight_min": -125.1147689819336,
      "activations/layer5_attention_weight_max": 67.57533264160156,
      "activations/layer5_attention_weight_min": -64.99884033203125,
      "activations/layer6_attention_weight_max": 55.917205810546875,
      "activations/layer6_attention_weight_min": -56.5766716003418,
      "activations/layer7_attention_weight_max": 96.34281921386719,
      "activations/layer7_attention_weight_min": -93.80538177490234,
      "activations/layer8_attention_weight_max": 40.50215530395508,
      "activations/layer8_attention_weight_min": -42.10782241821289,
      "activations/layer9_attention_weight_max": 34.821102142333984,
      "activations/layer9_attention_weight_min": -35.71671676635742,
      "epoch": 6.76,
      "learning_rate": 0.00010750227272727272,
      "loss": 2.8187,
      "step": 116300
    },
    {
      "activations/layer0_attention_weight_max": 15.860416412353516,
      "activations/layer0_attention_weight_min": -12.271178245544434,
      "activations/layer10_attention_weight_max": 34.80134963989258,
      "activations/layer10_attention_weight_min": -32.622314453125,
      "activations/layer11_attention_weight_max": 33.97468566894531,
      "activations/layer11_attention_weight_min": -29.567121505737305,
      "activations/layer12_attention_weight_max": 24.081735610961914,
      "activations/layer12_attention_weight_min": -26.367881774902344,
      "activations/layer13_attention_weight_max": 41.728553771972656,
      "activations/layer13_attention_weight_min": -33.00463104248047,
      "activations/layer14_attention_weight_max": 62.5048713684082,
      "activations/layer14_attention_weight_min": -46.9748420715332,
      "activations/layer15_attention_weight_max": 38.13151931762695,
      "activations/layer15_attention_weight_min": -30.200410842895508,
      "activations/layer16_attention_weight_max": 36.16448211669922,
      "activations/layer16_attention_weight_min": -27.240821838378906,
      "activations/layer17_attention_weight_max": 33.83854293823242,
      "activations/layer17_attention_weight_min": -23.95444107055664,
      "activations/layer18_attention_weight_max": 33.62660217285156,
      "activations/layer18_attention_weight_min": -21.760738372802734,
      "activations/layer19_attention_weight_max": 38.15159225463867,
      "activations/layer19_attention_weight_min": -30.570852279663086,
      "activations/layer1_attention_weight_max": 16.233591079711914,
      "activations/layer1_attention_weight_min": -15.397418975830078,
      "activations/layer20_attention_weight_max": 31.65200424194336,
      "activations/layer20_attention_weight_min": -25.208749771118164,
      "activations/layer21_attention_weight_max": 28.734817504882812,
      "activations/layer21_attention_weight_min": -22.26755714416504,
      "activations/layer22_attention_weight_max": 49.37765121459961,
      "activations/layer22_attention_weight_min": -27.079174041748047,
      "activations/layer23_attention_weight_max": 35.796993255615234,
      "activations/layer23_attention_weight_min": -21.72200584411621,
      "activations/layer2_attention_weight_max": 32.686641693115234,
      "activations/layer2_attention_weight_min": -30.23781967163086,
      "activations/layer3_attention_weight_max": 96.28144836425781,
      "activations/layer3_attention_weight_min": -92.9568862915039,
      "activations/layer4_attention_weight_max": 124.31786346435547,
      "activations/layer4_attention_weight_min": -124.78380584716797,
      "activations/layer5_attention_weight_max": 66.98876190185547,
      "activations/layer5_attention_weight_min": -65.30181121826172,
      "activations/layer6_attention_weight_max": 53.819881439208984,
      "activations/layer6_attention_weight_min": -53.977256774902344,
      "activations/layer7_attention_weight_max": 90.2862319946289,
      "activations/layer7_attention_weight_min": -89.45187377929688,
      "activations/layer8_attention_weight_max": 44.6411018371582,
      "activations/layer8_attention_weight_min": -42.44532012939453,
      "activations/layer9_attention_weight_max": 38.75151443481445,
      "activations/layer9_attention_weight_min": -36.29682540893555,
      "epoch": 6.76,
      "learning_rate": 0.00010748333333333332,
      "loss": 2.8101,
      "step": 116350
    },
    {
      "activations/layer0_attention_weight_max": 16.532604217529297,
      "activations/layer0_attention_weight_min": -11.830207824707031,
      "activations/layer10_attention_weight_max": 33.54689407348633,
      "activations/layer10_attention_weight_min": -31.545536041259766,
      "activations/layer11_attention_weight_max": 31.798999786376953,
      "activations/layer11_attention_weight_min": -28.974918365478516,
      "activations/layer12_attention_weight_max": 28.66865348815918,
      "activations/layer12_attention_weight_min": -24.904447555541992,
      "activations/layer13_attention_weight_max": 39.96403884887695,
      "activations/layer13_attention_weight_min": -31.881519317626953,
      "activations/layer14_attention_weight_max": 56.076744079589844,
      "activations/layer14_attention_weight_min": -45.35163116455078,
      "activations/layer15_attention_weight_max": 36.28129196166992,
      "activations/layer15_attention_weight_min": -27.72136878967285,
      "activations/layer16_attention_weight_max": 36.993064880371094,
      "activations/layer16_attention_weight_min": -29.38945960998535,
      "activations/layer17_attention_weight_max": 33.14756393432617,
      "activations/layer17_attention_weight_min": -24.444927215576172,
      "activations/layer18_attention_weight_max": 29.736087799072266,
      "activations/layer18_attention_weight_min": -20.47596549987793,
      "activations/layer19_attention_weight_max": 38.21213150024414,
      "activations/layer19_attention_weight_min": -31.52964210510254,
      "activations/layer1_attention_weight_max": 16.931350708007812,
      "activations/layer1_attention_weight_min": -16.271677017211914,
      "activations/layer20_attention_weight_max": 30.039073944091797,
      "activations/layer20_attention_weight_min": -24.35112762451172,
      "activations/layer21_attention_weight_max": 32.08457565307617,
      "activations/layer21_attention_weight_min": -22.824630737304688,
      "activations/layer22_attention_weight_max": 50.1259765625,
      "activations/layer22_attention_weight_min": -27.64065170288086,
      "activations/layer23_attention_weight_max": 32.2867431640625,
      "activations/layer23_attention_weight_min": -22.382190704345703,
      "activations/layer2_attention_weight_max": 31.79364585876465,
      "activations/layer2_attention_weight_min": -29.759809494018555,
      "activations/layer3_attention_weight_max": 90.28377532958984,
      "activations/layer3_attention_weight_min": -91.99270629882812,
      "activations/layer4_attention_weight_max": 122.523681640625,
      "activations/layer4_attention_weight_min": -119.77278900146484,
      "activations/layer5_attention_weight_max": 65.98138427734375,
      "activations/layer5_attention_weight_min": -62.930931091308594,
      "activations/layer6_attention_weight_max": 53.74027633666992,
      "activations/layer6_attention_weight_min": -52.20497512817383,
      "activations/layer7_attention_weight_max": 90.75778198242188,
      "activations/layer7_attention_weight_min": -86.0239028930664,
      "activations/layer8_attention_weight_max": 40.92097473144531,
      "activations/layer8_attention_weight_min": -39.315185546875,
      "activations/layer9_attention_weight_max": 34.22722625732422,
      "activations/layer9_attention_weight_min": -35.08418273925781,
      "epoch": 6.76,
      "learning_rate": 0.00010746439393939392,
      "loss": 2.8299,
      "step": 116400
    },
    {
      "activations/layer0_attention_weight_max": 16.74882698059082,
      "activations/layer0_attention_weight_min": -11.895071029663086,
      "activations/layer10_attention_weight_max": 36.199405670166016,
      "activations/layer10_attention_weight_min": -33.079505920410156,
      "activations/layer11_attention_weight_max": 40.971561431884766,
      "activations/layer11_attention_weight_min": -33.760414123535156,
      "activations/layer12_attention_weight_max": 24.79328155517578,
      "activations/layer12_attention_weight_min": -30.947404861450195,
      "activations/layer13_attention_weight_max": 49.8776741027832,
      "activations/layer13_attention_weight_min": -32.75068664550781,
      "activations/layer14_attention_weight_max": 85.43042755126953,
      "activations/layer14_attention_weight_min": -50.47658157348633,
      "activations/layer15_attention_weight_max": 43.9996223449707,
      "activations/layer15_attention_weight_min": -28.9476318359375,
      "activations/layer16_attention_weight_max": 39.801673889160156,
      "activations/layer16_attention_weight_min": -27.68161964416504,
      "activations/layer17_attention_weight_max": 40.95956802368164,
      "activations/layer17_attention_weight_min": -24.655797958374023,
      "activations/layer18_attention_weight_max": 32.285438537597656,
      "activations/layer18_attention_weight_min": -22.34148597717285,
      "activations/layer19_attention_weight_max": 48.290836334228516,
      "activations/layer19_attention_weight_min": -28.981735229492188,
      "activations/layer1_attention_weight_max": 17.032583236694336,
      "activations/layer1_attention_weight_min": -13.583181381225586,
      "activations/layer20_attention_weight_max": 38.26084899902344,
      "activations/layer20_attention_weight_min": -23.579605102539062,
      "activations/layer21_attention_weight_max": 33.61690139770508,
      "activations/layer21_attention_weight_min": -21.469430923461914,
      "activations/layer22_attention_weight_max": 61.57280731201172,
      "activations/layer22_attention_weight_min": -28.06985855102539,
      "activations/layer23_attention_weight_max": 39.790855407714844,
      "activations/layer23_attention_weight_min": -21.695274353027344,
      "activations/layer2_attention_weight_max": 32.342384338378906,
      "activations/layer2_attention_weight_min": -31.324628829956055,
      "activations/layer3_attention_weight_max": 93.10610961914062,
      "activations/layer3_attention_weight_min": -95.14112854003906,
      "activations/layer4_attention_weight_max": 129.23040771484375,
      "activations/layer4_attention_weight_min": -127.47505950927734,
      "activations/layer5_attention_weight_max": 69.71563720703125,
      "activations/layer5_attention_weight_min": -66.20186614990234,
      "activations/layer6_attention_weight_max": 53.04231643676758,
      "activations/layer6_attention_weight_min": -52.82712936401367,
      "activations/layer7_attention_weight_max": 95.59799194335938,
      "activations/layer7_attention_weight_min": -89.22140502929688,
      "activations/layer8_attention_weight_max": 40.58637237548828,
      "activations/layer8_attention_weight_min": -42.11920166015625,
      "activations/layer9_attention_weight_max": 34.47115707397461,
      "activations/layer9_attention_weight_min": -36.71412658691406,
      "epoch": 6.77,
      "learning_rate": 0.00010744545454545454,
      "loss": 2.8181,
      "step": 116450
    },
    {
      "activations/layer0_attention_weight_max": 16.37130355834961,
      "activations/layer0_attention_weight_min": -12.911188125610352,
      "activations/layer10_attention_weight_max": 36.976173400878906,
      "activations/layer10_attention_weight_min": -33.28157424926758,
      "activations/layer11_attention_weight_max": 42.284019470214844,
      "activations/layer11_attention_weight_min": -33.7696533203125,
      "activations/layer12_attention_weight_max": 24.65966796875,
      "activations/layer12_attention_weight_min": -25.544992446899414,
      "activations/layer13_attention_weight_max": 42.93796920776367,
      "activations/layer13_attention_weight_min": -32.14472198486328,
      "activations/layer14_attention_weight_max": 64.5909652709961,
      "activations/layer14_attention_weight_min": -40.83247756958008,
      "activations/layer15_attention_weight_max": 43.91606521606445,
      "activations/layer15_attention_weight_min": -31.45302391052246,
      "activations/layer16_attention_weight_max": 37.816619873046875,
      "activations/layer16_attention_weight_min": -27.81463623046875,
      "activations/layer17_attention_weight_max": 41.10883712768555,
      "activations/layer17_attention_weight_min": -25.731164932250977,
      "activations/layer18_attention_weight_max": 29.88385581970215,
      "activations/layer18_attention_weight_min": -19.564842224121094,
      "activations/layer19_attention_weight_max": 40.310970306396484,
      "activations/layer19_attention_weight_min": -28.251379013061523,
      "activations/layer1_attention_weight_max": 16.673254013061523,
      "activations/layer1_attention_weight_min": -13.981613159179688,
      "activations/layer20_attention_weight_max": 30.237369537353516,
      "activations/layer20_attention_weight_min": -23.742815017700195,
      "activations/layer21_attention_weight_max": 27.609445571899414,
      "activations/layer21_attention_weight_min": -21.0274715423584,
      "activations/layer22_attention_weight_max": 46.98820877075195,
      "activations/layer22_attention_weight_min": -29.12458038330078,
      "activations/layer23_attention_weight_max": 31.01191520690918,
      "activations/layer23_attention_weight_min": -22.998624801635742,
      "activations/layer2_attention_weight_max": 33.69764709472656,
      "activations/layer2_attention_weight_min": -31.905052185058594,
      "activations/layer3_attention_weight_max": 96.0439224243164,
      "activations/layer3_attention_weight_min": -94.70848846435547,
      "activations/layer4_attention_weight_max": 135.69180297851562,
      "activations/layer4_attention_weight_min": -125.11170959472656,
      "activations/layer5_attention_weight_max": 66.41799926757812,
      "activations/layer5_attention_weight_min": -65.23625183105469,
      "activations/layer6_attention_weight_max": 54.732852935791016,
      "activations/layer6_attention_weight_min": -55.67618179321289,
      "activations/layer7_attention_weight_max": 100.5213394165039,
      "activations/layer7_attention_weight_min": -92.49393463134766,
      "activations/layer8_attention_weight_max": 44.930419921875,
      "activations/layer8_attention_weight_min": -41.42739486694336,
      "activations/layer9_attention_weight_max": 39.96857833862305,
      "activations/layer9_attention_weight_min": -37.583099365234375,
      "epoch": 6.77,
      "learning_rate": 0.00010742651515151514,
      "loss": 2.8296,
      "step": 116500
    },
    {
      "activations/layer0_attention_weight_max": 16.23643684387207,
      "activations/layer0_attention_weight_min": -10.97226619720459,
      "activations/layer10_attention_weight_max": 33.87312316894531,
      "activations/layer10_attention_weight_min": -31.304019927978516,
      "activations/layer11_attention_weight_max": 32.64476776123047,
      "activations/layer11_attention_weight_min": -30.669189453125,
      "activations/layer12_attention_weight_max": 25.006488800048828,
      "activations/layer12_attention_weight_min": -24.07764434814453,
      "activations/layer13_attention_weight_max": 48.96388244628906,
      "activations/layer13_attention_weight_min": -35.57447052001953,
      "activations/layer14_attention_weight_max": 57.26311492919922,
      "activations/layer14_attention_weight_min": -40.132259368896484,
      "activations/layer15_attention_weight_max": 38.69679260253906,
      "activations/layer15_attention_weight_min": -29.643064498901367,
      "activations/layer16_attention_weight_max": 35.646244049072266,
      "activations/layer16_attention_weight_min": -28.00853729248047,
      "activations/layer17_attention_weight_max": 37.86076354980469,
      "activations/layer17_attention_weight_min": -24.62602424621582,
      "activations/layer18_attention_weight_max": 30.987812042236328,
      "activations/layer18_attention_weight_min": -20.199338912963867,
      "activations/layer19_attention_weight_max": 39.6568489074707,
      "activations/layer19_attention_weight_min": -29.504379272460938,
      "activations/layer1_attention_weight_max": 17.960262298583984,
      "activations/layer1_attention_weight_min": -13.915212631225586,
      "activations/layer20_attention_weight_max": 31.750648498535156,
      "activations/layer20_attention_weight_min": -22.33112907409668,
      "activations/layer21_attention_weight_max": 34.85418701171875,
      "activations/layer21_attention_weight_min": -20.070011138916016,
      "activations/layer22_attention_weight_max": 54.111446380615234,
      "activations/layer22_attention_weight_min": -28.260374069213867,
      "activations/layer23_attention_weight_max": 34.41667938232422,
      "activations/layer23_attention_weight_min": -23.223533630371094,
      "activations/layer2_attention_weight_max": 32.11146926879883,
      "activations/layer2_attention_weight_min": -32.01420593261719,
      "activations/layer3_attention_weight_max": 88.55316925048828,
      "activations/layer3_attention_weight_min": -90.49701690673828,
      "activations/layer4_attention_weight_max": 127.2902603149414,
      "activations/layer4_attention_weight_min": -122.60454559326172,
      "activations/layer5_attention_weight_max": 66.51763916015625,
      "activations/layer5_attention_weight_min": -64.3044204711914,
      "activations/layer6_attention_weight_max": 51.195133209228516,
      "activations/layer6_attention_weight_min": -52.399662017822266,
      "activations/layer7_attention_weight_max": 89.57304382324219,
      "activations/layer7_attention_weight_min": -85.52361297607422,
      "activations/layer8_attention_weight_max": 38.56332778930664,
      "activations/layer8_attention_weight_min": -40.61317825317383,
      "activations/layer9_attention_weight_max": 36.45265197753906,
      "activations/layer9_attention_weight_min": -34.65754318237305,
      "epoch": 6.77,
      "learning_rate": 0.00010740757575757575,
      "loss": 2.8162,
      "step": 116550
    },
    {
      "activations/layer0_attention_weight_max": 16.1578426361084,
      "activations/layer0_attention_weight_min": -12.159329414367676,
      "activations/layer10_attention_weight_max": 38.595481872558594,
      "activations/layer10_attention_weight_min": -32.89654541015625,
      "activations/layer11_attention_weight_max": 37.516632080078125,
      "activations/layer11_attention_weight_min": -30.730438232421875,
      "activations/layer12_attention_weight_max": 25.439834594726562,
      "activations/layer12_attention_weight_min": -25.977548599243164,
      "activations/layer13_attention_weight_max": 47.54087829589844,
      "activations/layer13_attention_weight_min": -37.474090576171875,
      "activations/layer14_attention_weight_max": 56.52157974243164,
      "activations/layer14_attention_weight_min": -40.14505386352539,
      "activations/layer15_attention_weight_max": 43.8466682434082,
      "activations/layer15_attention_weight_min": -30.4050350189209,
      "activations/layer16_attention_weight_max": 36.06658935546875,
      "activations/layer16_attention_weight_min": -26.27777099609375,
      "activations/layer17_attention_weight_max": 41.193729400634766,
      "activations/layer17_attention_weight_min": -24.520776748657227,
      "activations/layer18_attention_weight_max": 34.36712646484375,
      "activations/layer18_attention_weight_min": -20.537572860717773,
      "activations/layer19_attention_weight_max": 40.1281623840332,
      "activations/layer19_attention_weight_min": -29.171588897705078,
      "activations/layer1_attention_weight_max": 17.741662979125977,
      "activations/layer1_attention_weight_min": -13.408088684082031,
      "activations/layer20_attention_weight_max": 34.48520278930664,
      "activations/layer20_attention_weight_min": -23.112041473388672,
      "activations/layer21_attention_weight_max": 31.28135871887207,
      "activations/layer21_attention_weight_min": -20.55223846435547,
      "activations/layer22_attention_weight_max": 53.61756134033203,
      "activations/layer22_attention_weight_min": -26.889293670654297,
      "activations/layer23_attention_weight_max": 35.886985778808594,
      "activations/layer23_attention_weight_min": -21.252811431884766,
      "activations/layer2_attention_weight_max": 32.01813507080078,
      "activations/layer2_attention_weight_min": -31.080204010009766,
      "activations/layer3_attention_weight_max": 90.88230895996094,
      "activations/layer3_attention_weight_min": -92.65269470214844,
      "activations/layer4_attention_weight_max": 126.22059631347656,
      "activations/layer4_attention_weight_min": -129.16526794433594,
      "activations/layer5_attention_weight_max": 67.8454360961914,
      "activations/layer5_attention_weight_min": -68.5302505493164,
      "activations/layer6_attention_weight_max": 54.17906951904297,
      "activations/layer6_attention_weight_min": -54.21033477783203,
      "activations/layer7_attention_weight_max": 95.27777862548828,
      "activations/layer7_attention_weight_min": -92.25340270996094,
      "activations/layer8_attention_weight_max": 44.25355529785156,
      "activations/layer8_attention_weight_min": -45.50126266479492,
      "activations/layer9_attention_weight_max": 42.12801742553711,
      "activations/layer9_attention_weight_min": -36.29977035522461,
      "epoch": 6.78,
      "learning_rate": 0.00010738863636363636,
      "loss": 2.8235,
      "step": 116600
    },
    {
      "activations/layer0_attention_weight_max": 16.00037384033203,
      "activations/layer0_attention_weight_min": -11.932347297668457,
      "activations/layer10_attention_weight_max": 36.86771774291992,
      "activations/layer10_attention_weight_min": -36.62978744506836,
      "activations/layer11_attention_weight_max": 38.05085754394531,
      "activations/layer11_attention_weight_min": -30.431180953979492,
      "activations/layer12_attention_weight_max": 50.11905288696289,
      "activations/layer12_attention_weight_min": -26.868684768676758,
      "activations/layer13_attention_weight_max": 55.713768005371094,
      "activations/layer13_attention_weight_min": -31.440561294555664,
      "activations/layer14_attention_weight_max": 52.91315460205078,
      "activations/layer14_attention_weight_min": -39.40614700317383,
      "activations/layer15_attention_weight_max": 34.290260314941406,
      "activations/layer15_attention_weight_min": -28.173480987548828,
      "activations/layer16_attention_weight_max": 33.809288024902344,
      "activations/layer16_attention_weight_min": -30.265153884887695,
      "activations/layer17_attention_weight_max": 35.089054107666016,
      "activations/layer17_attention_weight_min": -24.646934509277344,
      "activations/layer18_attention_weight_max": 39.14277267456055,
      "activations/layer18_attention_weight_min": -22.196027755737305,
      "activations/layer19_attention_weight_max": 36.848655700683594,
      "activations/layer19_attention_weight_min": -29.84977149963379,
      "activations/layer1_attention_weight_max": 17.142131805419922,
      "activations/layer1_attention_weight_min": -15.418800354003906,
      "activations/layer20_attention_weight_max": 29.449661254882812,
      "activations/layer20_attention_weight_min": -23.2465877532959,
      "activations/layer21_attention_weight_max": 39.393428802490234,
      "activations/layer21_attention_weight_min": -21.08022117614746,
      "activations/layer22_attention_weight_max": 49.22787094116211,
      "activations/layer22_attention_weight_min": -29.041364669799805,
      "activations/layer23_attention_weight_max": 33.42986297607422,
      "activations/layer23_attention_weight_min": -22.062246322631836,
      "activations/layer2_attention_weight_max": 36.219810485839844,
      "activations/layer2_attention_weight_min": -33.05214309692383,
      "activations/layer3_attention_weight_max": 96.93600463867188,
      "activations/layer3_attention_weight_min": -96.15480041503906,
      "activations/layer4_attention_weight_max": 136.73577880859375,
      "activations/layer4_attention_weight_min": -123.64350891113281,
      "activations/layer5_attention_weight_max": 69.87017822265625,
      "activations/layer5_attention_weight_min": -67.53893280029297,
      "activations/layer6_attention_weight_max": 54.57590866088867,
      "activations/layer6_attention_weight_min": -54.33017349243164,
      "activations/layer7_attention_weight_max": 91.54956817626953,
      "activations/layer7_attention_weight_min": -90.5859146118164,
      "activations/layer8_attention_weight_max": 45.66779708862305,
      "activations/layer8_attention_weight_min": -43.41836929321289,
      "activations/layer9_attention_weight_max": 41.553585052490234,
      "activations/layer9_attention_weight_min": -42.8983268737793,
      "epoch": 6.78,
      "learning_rate": 0.00010736969696969696,
      "loss": 2.8275,
      "step": 116650
    },
    {
      "activations/layer0_attention_weight_max": 15.813084602355957,
      "activations/layer0_attention_weight_min": -11.894844055175781,
      "activations/layer10_attention_weight_max": 31.521484375,
      "activations/layer10_attention_weight_min": -31.296348571777344,
      "activations/layer11_attention_weight_max": 30.93630027770996,
      "activations/layer11_attention_weight_min": -29.73244857788086,
      "activations/layer12_attention_weight_max": 29.293128967285156,
      "activations/layer12_attention_weight_min": -27.5616397857666,
      "activations/layer13_attention_weight_max": 45.39530944824219,
      "activations/layer13_attention_weight_min": -34.544090270996094,
      "activations/layer14_attention_weight_max": 63.91218185424805,
      "activations/layer14_attention_weight_min": -49.49617004394531,
      "activations/layer15_attention_weight_max": 38.512088775634766,
      "activations/layer15_attention_weight_min": -27.91538429260254,
      "activations/layer16_attention_weight_max": 36.212581634521484,
      "activations/layer16_attention_weight_min": -28.808277130126953,
      "activations/layer17_attention_weight_max": 36.971900939941406,
      "activations/layer17_attention_weight_min": -25.106096267700195,
      "activations/layer18_attention_weight_max": 31.974063873291016,
      "activations/layer18_attention_weight_min": -22.246217727661133,
      "activations/layer19_attention_weight_max": 42.10512924194336,
      "activations/layer19_attention_weight_min": -32.47180938720703,
      "activations/layer1_attention_weight_max": 17.47909164428711,
      "activations/layer1_attention_weight_min": -15.34826374053955,
      "activations/layer20_attention_weight_max": 34.02238845825195,
      "activations/layer20_attention_weight_min": -23.453994750976562,
      "activations/layer21_attention_weight_max": 30.31455421447754,
      "activations/layer21_attention_weight_min": -23.034067153930664,
      "activations/layer22_attention_weight_max": 55.46327209472656,
      "activations/layer22_attention_weight_min": -28.269041061401367,
      "activations/layer23_attention_weight_max": 38.94469451904297,
      "activations/layer23_attention_weight_min": -23.250652313232422,
      "activations/layer2_attention_weight_max": 32.46501922607422,
      "activations/layer2_attention_weight_min": -30.228900909423828,
      "activations/layer3_attention_weight_max": 95.92225646972656,
      "activations/layer3_attention_weight_min": -94.60350036621094,
      "activations/layer4_attention_weight_max": 127.67156219482422,
      "activations/layer4_attention_weight_min": -126.6541748046875,
      "activations/layer5_attention_weight_max": 64.86143493652344,
      "activations/layer5_attention_weight_min": -63.871482849121094,
      "activations/layer6_attention_weight_max": 55.116249084472656,
      "activations/layer6_attention_weight_min": -54.09242248535156,
      "activations/layer7_attention_weight_max": 91.37937927246094,
      "activations/layer7_attention_weight_min": -90.59436798095703,
      "activations/layer8_attention_weight_max": 40.7065315246582,
      "activations/layer8_attention_weight_min": -40.98341751098633,
      "activations/layer9_attention_weight_max": 34.425682067871094,
      "activations/layer9_attention_weight_min": -34.10459899902344,
      "epoch": 6.78,
      "learning_rate": 0.00010735075757575757,
      "loss": 2.8275,
      "step": 116700
    },
    {
      "activations/layer0_attention_weight_max": 14.949237823486328,
      "activations/layer0_attention_weight_min": -12.575948715209961,
      "activations/layer10_attention_weight_max": 36.04482650756836,
      "activations/layer10_attention_weight_min": -31.148210525512695,
      "activations/layer11_attention_weight_max": 35.099449157714844,
      "activations/layer11_attention_weight_min": -30.0283203125,
      "activations/layer12_attention_weight_max": 25.97127914428711,
      "activations/layer12_attention_weight_min": -25.907527923583984,
      "activations/layer13_attention_weight_max": 47.423622131347656,
      "activations/layer13_attention_weight_min": -33.71609878540039,
      "activations/layer14_attention_weight_max": 55.962337493896484,
      "activations/layer14_attention_weight_min": -40.995540618896484,
      "activations/layer15_attention_weight_max": 38.754478454589844,
      "activations/layer15_attention_weight_min": -29.647796630859375,
      "activations/layer16_attention_weight_max": 36.325496673583984,
      "activations/layer16_attention_weight_min": -29.433958053588867,
      "activations/layer17_attention_weight_max": 38.06330490112305,
      "activations/layer17_attention_weight_min": -25.735965728759766,
      "activations/layer18_attention_weight_max": 32.431427001953125,
      "activations/layer18_attention_weight_min": -20.844274520874023,
      "activations/layer19_attention_weight_max": 41.9095573425293,
      "activations/layer19_attention_weight_min": -30.27707862854004,
      "activations/layer1_attention_weight_max": 17.122173309326172,
      "activations/layer1_attention_weight_min": -14.521743774414062,
      "activations/layer20_attention_weight_max": 38.11991500854492,
      "activations/layer20_attention_weight_min": -24.074010848999023,
      "activations/layer21_attention_weight_max": 30.67442512512207,
      "activations/layer21_attention_weight_min": -20.799455642700195,
      "activations/layer22_attention_weight_max": 52.568519592285156,
      "activations/layer22_attention_weight_min": -28.258543014526367,
      "activations/layer23_attention_weight_max": 39.05756378173828,
      "activations/layer23_attention_weight_min": -22.101375579833984,
      "activations/layer2_attention_weight_max": 32.68556594848633,
      "activations/layer2_attention_weight_min": -32.375579833984375,
      "activations/layer3_attention_weight_max": 92.46995544433594,
      "activations/layer3_attention_weight_min": -94.63447570800781,
      "activations/layer4_attention_weight_max": 130.5642852783203,
      "activations/layer4_attention_weight_min": -121.83302307128906,
      "activations/layer5_attention_weight_max": 70.67168426513672,
      "activations/layer5_attention_weight_min": -64.67616271972656,
      "activations/layer6_attention_weight_max": 54.29347229003906,
      "activations/layer6_attention_weight_min": -55.79672622680664,
      "activations/layer7_attention_weight_max": 91.96984100341797,
      "activations/layer7_attention_weight_min": -89.16099548339844,
      "activations/layer8_attention_weight_max": 42.10820388793945,
      "activations/layer8_attention_weight_min": -44.53501510620117,
      "activations/layer9_attention_weight_max": 39.265995025634766,
      "activations/layer9_attention_weight_min": -36.59563064575195,
      "epoch": 6.78,
      "learning_rate": 0.00010733181818181817,
      "loss": 2.8247,
      "step": 116750
    },
    {
      "activations/layer0_attention_weight_max": 15.71249008178711,
      "activations/layer0_attention_weight_min": -12.397234916687012,
      "activations/layer10_attention_weight_max": 37.41077423095703,
      "activations/layer10_attention_weight_min": -36.20697784423828,
      "activations/layer11_attention_weight_max": 38.50967788696289,
      "activations/layer11_attention_weight_min": -32.6528205871582,
      "activations/layer12_attention_weight_max": 25.66335678100586,
      "activations/layer12_attention_weight_min": -27.11307716369629,
      "activations/layer13_attention_weight_max": 51.98611068725586,
      "activations/layer13_attention_weight_min": -36.017635345458984,
      "activations/layer14_attention_weight_max": 53.793514251708984,
      "activations/layer14_attention_weight_min": -41.82956314086914,
      "activations/layer15_attention_weight_max": 39.340614318847656,
      "activations/layer15_attention_weight_min": -29.848575592041016,
      "activations/layer16_attention_weight_max": 34.168121337890625,
      "activations/layer16_attention_weight_min": -28.17585563659668,
      "activations/layer17_attention_weight_max": 37.38172912597656,
      "activations/layer17_attention_weight_min": -26.880237579345703,
      "activations/layer18_attention_weight_max": 33.51481246948242,
      "activations/layer18_attention_weight_min": -22.15252685546875,
      "activations/layer19_attention_weight_max": 39.106773376464844,
      "activations/layer19_attention_weight_min": -31.135175704956055,
      "activations/layer1_attention_weight_max": 17.237262725830078,
      "activations/layer1_attention_weight_min": -13.568123817443848,
      "activations/layer20_attention_weight_max": 30.81364631652832,
      "activations/layer20_attention_weight_min": -26.893482208251953,
      "activations/layer21_attention_weight_max": 32.11152648925781,
      "activations/layer21_attention_weight_min": -25.071752548217773,
      "activations/layer22_attention_weight_max": 46.34274673461914,
      "activations/layer22_attention_weight_min": -28.430688858032227,
      "activations/layer23_attention_weight_max": 34.409515380859375,
      "activations/layer23_attention_weight_min": -23.508590698242188,
      "activations/layer2_attention_weight_max": 36.375694274902344,
      "activations/layer2_attention_weight_min": -32.28483581542969,
      "activations/layer3_attention_weight_max": 96.1652603149414,
      "activations/layer3_attention_weight_min": -97.3695297241211,
      "activations/layer4_attention_weight_max": 136.2132568359375,
      "activations/layer4_attention_weight_min": -126.51509094238281,
      "activations/layer5_attention_weight_max": 70.66848754882812,
      "activations/layer5_attention_weight_min": -65.41397094726562,
      "activations/layer6_attention_weight_max": 52.56741714477539,
      "activations/layer6_attention_weight_min": -52.67124557495117,
      "activations/layer7_attention_weight_max": 101.4668197631836,
      "activations/layer7_attention_weight_min": -94.13585662841797,
      "activations/layer8_attention_weight_max": 46.62162399291992,
      "activations/layer8_attention_weight_min": -44.76259231567383,
      "activations/layer9_attention_weight_max": 39.754024505615234,
      "activations/layer9_attention_weight_min": -37.74068832397461,
      "epoch": 6.79,
      "learning_rate": 0.00010731287878787878,
      "loss": 2.8353,
      "step": 116800
    },
    {
      "activations/layer0_attention_weight_max": 16.09750747680664,
      "activations/layer0_attention_weight_min": -12.360929489135742,
      "activations/layer10_attention_weight_max": 32.5369987487793,
      "activations/layer10_attention_weight_min": -30.104248046875,
      "activations/layer11_attention_weight_max": 30.808162689208984,
      "activations/layer11_attention_weight_min": -28.141599655151367,
      "activations/layer12_attention_weight_max": 26.666929244995117,
      "activations/layer12_attention_weight_min": -27.4459285736084,
      "activations/layer13_attention_weight_max": 38.45613479614258,
      "activations/layer13_attention_weight_min": -30.82489585876465,
      "activations/layer14_attention_weight_max": 51.57897186279297,
      "activations/layer14_attention_weight_min": -38.38449478149414,
      "activations/layer15_attention_weight_max": 36.755821228027344,
      "activations/layer15_attention_weight_min": -29.837265014648438,
      "activations/layer16_attention_weight_max": 31.954532623291016,
      "activations/layer16_attention_weight_min": -26.911972045898438,
      "activations/layer17_attention_weight_max": 35.516326904296875,
      "activations/layer17_attention_weight_min": -24.357818603515625,
      "activations/layer18_attention_weight_max": 32.882999420166016,
      "activations/layer18_attention_weight_min": -20.691875457763672,
      "activations/layer19_attention_weight_max": 37.5388069152832,
      "activations/layer19_attention_weight_min": -28.952682495117188,
      "activations/layer1_attention_weight_max": 17.93416976928711,
      "activations/layer1_attention_weight_min": -18.496498107910156,
      "activations/layer20_attention_weight_max": 29.330230712890625,
      "activations/layer20_attention_weight_min": -22.93303108215332,
      "activations/layer21_attention_weight_max": 30.625167846679688,
      "activations/layer21_attention_weight_min": -22.531539916992188,
      "activations/layer22_attention_weight_max": 45.78013610839844,
      "activations/layer22_attention_weight_min": -29.133930206298828,
      "activations/layer23_attention_weight_max": 35.914215087890625,
      "activations/layer23_attention_weight_min": -23.457595825195312,
      "activations/layer2_attention_weight_max": 34.54010772705078,
      "activations/layer2_attention_weight_min": -31.879924774169922,
      "activations/layer3_attention_weight_max": 94.9005126953125,
      "activations/layer3_attention_weight_min": -95.53765106201172,
      "activations/layer4_attention_weight_max": 128.26817321777344,
      "activations/layer4_attention_weight_min": -130.58226013183594,
      "activations/layer5_attention_weight_max": 65.45097351074219,
      "activations/layer5_attention_weight_min": -62.47019577026367,
      "activations/layer6_attention_weight_max": 50.58659744262695,
      "activations/layer6_attention_weight_min": -51.962100982666016,
      "activations/layer7_attention_weight_max": 88.13854217529297,
      "activations/layer7_attention_weight_min": -86.74260711669922,
      "activations/layer8_attention_weight_max": 37.4656982421875,
      "activations/layer8_attention_weight_min": -38.237525939941406,
      "activations/layer9_attention_weight_max": 35.37372970581055,
      "activations/layer9_attention_weight_min": -32.26996994018555,
      "epoch": 6.79,
      "learning_rate": 0.00010729393939393939,
      "loss": 2.8192,
      "step": 116850
    },
    {
      "activations/layer0_attention_weight_max": 16.434932708740234,
      "activations/layer0_attention_weight_min": -11.55910873413086,
      "activations/layer10_attention_weight_max": 29.25662612915039,
      "activations/layer10_attention_weight_min": -30.523357391357422,
      "activations/layer11_attention_weight_max": 31.286388397216797,
      "activations/layer11_attention_weight_min": -28.992698669433594,
      "activations/layer12_attention_weight_max": 24.630115509033203,
      "activations/layer12_attention_weight_min": -26.506200790405273,
      "activations/layer13_attention_weight_max": 37.98945617675781,
      "activations/layer13_attention_weight_min": -32.4088020324707,
      "activations/layer14_attention_weight_max": 55.120155334472656,
      "activations/layer14_attention_weight_min": -41.31950759887695,
      "activations/layer15_attention_weight_max": 34.663841247558594,
      "activations/layer15_attention_weight_min": -31.643474578857422,
      "activations/layer16_attention_weight_max": 29.96932029724121,
      "activations/layer16_attention_weight_min": -29.165302276611328,
      "activations/layer17_attention_weight_max": 33.541343688964844,
      "activations/layer17_attention_weight_min": -26.83818817138672,
      "activations/layer18_attention_weight_max": 30.45405387878418,
      "activations/layer18_attention_weight_min": -19.99312400817871,
      "activations/layer19_attention_weight_max": 33.398895263671875,
      "activations/layer19_attention_weight_min": -30.515146255493164,
      "activations/layer1_attention_weight_max": 16.638477325439453,
      "activations/layer1_attention_weight_min": -13.497939109802246,
      "activations/layer20_attention_weight_max": 29.066144943237305,
      "activations/layer20_attention_weight_min": -23.256797790527344,
      "activations/layer21_attention_weight_max": 30.7128963470459,
      "activations/layer21_attention_weight_min": -21.00503921508789,
      "activations/layer22_attention_weight_max": 48.39191436767578,
      "activations/layer22_attention_weight_min": -26.65250015258789,
      "activations/layer23_attention_weight_max": 33.212371826171875,
      "activations/layer23_attention_weight_min": -23.10311508178711,
      "activations/layer2_attention_weight_max": 32.34891128540039,
      "activations/layer2_attention_weight_min": -31.846113204956055,
      "activations/layer3_attention_weight_max": 94.09913635253906,
      "activations/layer3_attention_weight_min": -94.18872833251953,
      "activations/layer4_attention_weight_max": 127.24039459228516,
      "activations/layer4_attention_weight_min": -127.15882873535156,
      "activations/layer5_attention_weight_max": 65.40533447265625,
      "activations/layer5_attention_weight_min": -66.09103393554688,
      "activations/layer6_attention_weight_max": 52.523902893066406,
      "activations/layer6_attention_weight_min": -55.558555603027344,
      "activations/layer7_attention_weight_max": 89.95713806152344,
      "activations/layer7_attention_weight_min": -86.71195983886719,
      "activations/layer8_attention_weight_max": 40.807369232177734,
      "activations/layer8_attention_weight_min": -39.22893142700195,
      "activations/layer9_attention_weight_max": 32.44417953491211,
      "activations/layer9_attention_weight_min": -33.04278564453125,
      "epoch": 6.79,
      "learning_rate": 0.00010727499999999998,
      "loss": 2.8268,
      "step": 116900
    },
    {
      "activations/layer0_attention_weight_max": 15.514632225036621,
      "activations/layer0_attention_weight_min": -13.41793441772461,
      "activations/layer10_attention_weight_max": 39.83938217163086,
      "activations/layer10_attention_weight_min": -32.72982406616211,
      "activations/layer11_attention_weight_max": 42.97565460205078,
      "activations/layer11_attention_weight_min": -30.72250747680664,
      "activations/layer12_attention_weight_max": 25.51930046081543,
      "activations/layer12_attention_weight_min": -29.75860023498535,
      "activations/layer13_attention_weight_max": 50.444541931152344,
      "activations/layer13_attention_weight_min": -32.69159698486328,
      "activations/layer14_attention_weight_max": 63.38414764404297,
      "activations/layer14_attention_weight_min": -42.92424011230469,
      "activations/layer15_attention_weight_max": 38.026065826416016,
      "activations/layer15_attention_weight_min": -30.109189987182617,
      "activations/layer16_attention_weight_max": 34.677696228027344,
      "activations/layer16_attention_weight_min": -29.0457820892334,
      "activations/layer17_attention_weight_max": 36.57802200317383,
      "activations/layer17_attention_weight_min": -25.85248374938965,
      "activations/layer18_attention_weight_max": 32.58147430419922,
      "activations/layer18_attention_weight_min": -21.657608032226562,
      "activations/layer19_attention_weight_max": 40.33475112915039,
      "activations/layer19_attention_weight_min": -30.8294620513916,
      "activations/layer1_attention_weight_max": 16.695175170898438,
      "activations/layer1_attention_weight_min": -13.181063652038574,
      "activations/layer20_attention_weight_max": 32.82160186767578,
      "activations/layer20_attention_weight_min": -24.21171760559082,
      "activations/layer21_attention_weight_max": 34.72462844848633,
      "activations/layer21_attention_weight_min": -22.494701385498047,
      "activations/layer22_attention_weight_max": 51.37136459350586,
      "activations/layer22_attention_weight_min": -29.71756362915039,
      "activations/layer23_attention_weight_max": 37.98561096191406,
      "activations/layer23_attention_weight_min": -23.883941650390625,
      "activations/layer2_attention_weight_max": 33.69197082519531,
      "activations/layer2_attention_weight_min": -32.08708190917969,
      "activations/layer3_attention_weight_max": 92.14604187011719,
      "activations/layer3_attention_weight_min": -98.77194213867188,
      "activations/layer4_attention_weight_max": 127.96602630615234,
      "activations/layer4_attention_weight_min": -136.45193481445312,
      "activations/layer5_attention_weight_max": 65.7331314086914,
      "activations/layer5_attention_weight_min": -62.648284912109375,
      "activations/layer6_attention_weight_max": 54.78556442260742,
      "activations/layer6_attention_weight_min": -54.90745544433594,
      "activations/layer7_attention_weight_max": 100.75959777832031,
      "activations/layer7_attention_weight_min": -90.5666732788086,
      "activations/layer8_attention_weight_max": 46.49430465698242,
      "activations/layer8_attention_weight_min": -46.10855484008789,
      "activations/layer9_attention_weight_max": 39.92214584350586,
      "activations/layer9_attention_weight_min": -38.22740936279297,
      "epoch": 6.8,
      "learning_rate": 0.0001072560606060606,
      "loss": 2.8323,
      "step": 116950
    },
    {
      "activations/layer0_attention_weight_max": 16.644237518310547,
      "activations/layer0_attention_weight_min": -12.794051170349121,
      "activations/layer10_attention_weight_max": 35.075992584228516,
      "activations/layer10_attention_weight_min": -34.774078369140625,
      "activations/layer11_attention_weight_max": 34.41034698486328,
      "activations/layer11_attention_weight_min": -29.217018127441406,
      "activations/layer12_attention_weight_max": 30.13035774230957,
      "activations/layer12_attention_weight_min": -24.554012298583984,
      "activations/layer13_attention_weight_max": 41.95659255981445,
      "activations/layer13_attention_weight_min": -32.560577392578125,
      "activations/layer14_attention_weight_max": 60.492591857910156,
      "activations/layer14_attention_weight_min": -44.485015869140625,
      "activations/layer15_attention_weight_max": 38.2061767578125,
      "activations/layer15_attention_weight_min": -29.85237693786621,
      "activations/layer16_attention_weight_max": 35.138065338134766,
      "activations/layer16_attention_weight_min": -27.410114288330078,
      "activations/layer17_attention_weight_max": 37.441810607910156,
      "activations/layer17_attention_weight_min": -24.1293888092041,
      "activations/layer18_attention_weight_max": 32.472137451171875,
      "activations/layer18_attention_weight_min": -22.014055252075195,
      "activations/layer19_attention_weight_max": 42.90315628051758,
      "activations/layer19_attention_weight_min": -30.256078720092773,
      "activations/layer1_attention_weight_max": 17.623807907104492,
      "activations/layer1_attention_weight_min": -14.240800857543945,
      "activations/layer20_attention_weight_max": 34.83469009399414,
      "activations/layer20_attention_weight_min": -24.115690231323242,
      "activations/layer21_attention_weight_max": 29.868267059326172,
      "activations/layer21_attention_weight_min": -23.790897369384766,
      "activations/layer22_attention_weight_max": 51.890419006347656,
      "activations/layer22_attention_weight_min": -28.701448440551758,
      "activations/layer23_attention_weight_max": 38.23878860473633,
      "activations/layer23_attention_weight_min": -22.928787231445312,
      "activations/layer2_attention_weight_max": 34.42354202270508,
      "activations/layer2_attention_weight_min": -32.86435317993164,
      "activations/layer3_attention_weight_max": 94.203369140625,
      "activations/layer3_attention_weight_min": -93.56852722167969,
      "activations/layer4_attention_weight_max": 128.96217346191406,
      "activations/layer4_attention_weight_min": -124.37513732910156,
      "activations/layer5_attention_weight_max": 69.27151489257812,
      "activations/layer5_attention_weight_min": -67.52082824707031,
      "activations/layer6_attention_weight_max": 54.810550689697266,
      "activations/layer6_attention_weight_min": -55.85879898071289,
      "activations/layer7_attention_weight_max": 92.39360809326172,
      "activations/layer7_attention_weight_min": -90.97135162353516,
      "activations/layer8_attention_weight_max": 42.464111328125,
      "activations/layer8_attention_weight_min": -44.310604095458984,
      "activations/layer9_attention_weight_max": 36.21381378173828,
      "activations/layer9_attention_weight_min": -35.17765426635742,
      "epoch": 6.8,
      "learning_rate": 0.00010723712121212121,
      "loss": 2.8185,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_loss": 2.775390625,
      "eval_runtime": 8.4957,
      "eval_samples_per_second": 505.431,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_openwebtext_loss": 2.775390625,
      "eval_openwebtext_ppl": 16.04489330813532,
      "eval_openwebtext_runtime": 8.4957,
      "eval_openwebtext_samples_per_second": 505.431,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 1.9904,
      "eval_wikitext_samples_per_second": 229.1,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_lambada_loss": 2.7109375,
      "eval_lambada_ppl": 15.04337206739713,
      "eval_lambada_runtime": 9.5769,
      "eval_lambada_samples_per_second": 508.41,
      "step": 117000
    },
    {
      "activations/layer0_attention_weight_max": 16.063396453857422,
      "activations/layer0_attention_weight_min": -11.36158561706543,
      "activations/layer10_attention_weight_max": 33.24431610107422,
      "activations/layer10_attention_weight_min": -31.611448287963867,
      "activations/layer11_attention_weight_max": 34.16564178466797,
      "activations/layer11_attention_weight_min": -29.974178314208984,
      "activations/layer12_attention_weight_max": 22.856348037719727,
      "activations/layer12_attention_weight_min": -26.151689529418945,
      "activations/layer13_attention_weight_max": 42.24638748168945,
      "activations/layer13_attention_weight_min": -33.22689437866211,
      "activations/layer14_attention_weight_max": 51.880950927734375,
      "activations/layer14_attention_weight_min": -40.456661224365234,
      "activations/layer15_attention_weight_max": 38.58268356323242,
      "activations/layer15_attention_weight_min": -28.01470375061035,
      "activations/layer16_attention_weight_max": 35.667091369628906,
      "activations/layer16_attention_weight_min": -28.173200607299805,
      "activations/layer17_attention_weight_max": 40.58845520019531,
      "activations/layer17_attention_weight_min": -23.87258529663086,
      "activations/layer18_attention_weight_max": 34.038902282714844,
      "activations/layer18_attention_weight_min": -20.82943344116211,
      "activations/layer19_attention_weight_max": 40.1612434387207,
      "activations/layer19_attention_weight_min": -32.37928771972656,
      "activations/layer1_attention_weight_max": 17.859634399414062,
      "activations/layer1_attention_weight_min": -14.010260581970215,
      "activations/layer20_attention_weight_max": 36.608001708984375,
      "activations/layer20_attention_weight_min": -24.87248992919922,
      "activations/layer21_attention_weight_max": 35.34019470214844,
      "activations/layer21_attention_weight_min": -22.0167179107666,
      "activations/layer22_attention_weight_max": 48.39813995361328,
      "activations/layer22_attention_weight_min": -27.67317771911621,
      "activations/layer23_attention_weight_max": 34.7628173828125,
      "activations/layer23_attention_weight_min": -23.383533477783203,
      "activations/layer2_attention_weight_max": 32.12965774536133,
      "activations/layer2_attention_weight_min": -30.736289978027344,
      "activations/layer3_attention_weight_max": 90.8520736694336,
      "activations/layer3_attention_weight_min": -89.09648132324219,
      "activations/layer4_attention_weight_max": 124.46380615234375,
      "activations/layer4_attention_weight_min": -119.5412368774414,
      "activations/layer5_attention_weight_max": 65.41514587402344,
      "activations/layer5_attention_weight_min": -65.1785888671875,
      "activations/layer6_attention_weight_max": 52.80949783325195,
      "activations/layer6_attention_weight_min": -54.53852462768555,
      "activations/layer7_attention_weight_max": 88.77368927001953,
      "activations/layer7_attention_weight_min": -87.81260681152344,
      "activations/layer8_attention_weight_max": 40.89468002319336,
      "activations/layer8_attention_weight_min": -40.127525329589844,
      "activations/layer9_attention_weight_max": 36.734825134277344,
      "activations/layer9_attention_weight_min": -36.61416244506836,
      "epoch": 6.8,
      "learning_rate": 0.0001072181818181818,
      "loss": 2.8309,
      "step": 117050
    },
    {
      "activations/layer0_attention_weight_max": 16.605327606201172,
      "activations/layer0_attention_weight_min": -10.886924743652344,
      "activations/layer10_attention_weight_max": 33.59962844848633,
      "activations/layer10_attention_weight_min": -32.13915252685547,
      "activations/layer11_attention_weight_max": 33.779014587402344,
      "activations/layer11_attention_weight_min": -30.421485900878906,
      "activations/layer12_attention_weight_max": 25.626846313476562,
      "activations/layer12_attention_weight_min": -27.559553146362305,
      "activations/layer13_attention_weight_max": 41.1158447265625,
      "activations/layer13_attention_weight_min": -32.79486083984375,
      "activations/layer14_attention_weight_max": 49.04732894897461,
      "activations/layer14_attention_weight_min": -41.25192642211914,
      "activations/layer15_attention_weight_max": 37.66015625,
      "activations/layer15_attention_weight_min": -30.14402961730957,
      "activations/layer16_attention_weight_max": 33.88288497924805,
      "activations/layer16_attention_weight_min": -28.504533767700195,
      "activations/layer17_attention_weight_max": 37.57883071899414,
      "activations/layer17_attention_weight_min": -25.400571823120117,
      "activations/layer18_attention_weight_max": 33.41436767578125,
      "activations/layer18_attention_weight_min": -19.84086036682129,
      "activations/layer19_attention_weight_max": 36.6682243347168,
      "activations/layer19_attention_weight_min": -29.41806411743164,
      "activations/layer1_attention_weight_max": 17.5333194732666,
      "activations/layer1_attention_weight_min": -13.815655708312988,
      "activations/layer20_attention_weight_max": 31.170820236206055,
      "activations/layer20_attention_weight_min": -23.083995819091797,
      "activations/layer21_attention_weight_max": 31.340129852294922,
      "activations/layer21_attention_weight_min": -21.863306045532227,
      "activations/layer22_attention_weight_max": 51.11967849731445,
      "activations/layer22_attention_weight_min": -28.7607364654541,
      "activations/layer23_attention_weight_max": 35.14540100097656,
      "activations/layer23_attention_weight_min": -23.246273040771484,
      "activations/layer2_attention_weight_max": 32.09945297241211,
      "activations/layer2_attention_weight_min": -33.17178726196289,
      "activations/layer3_attention_weight_max": 92.74554443359375,
      "activations/layer3_attention_weight_min": -95.5904769897461,
      "activations/layer4_attention_weight_max": 127.75349426269531,
      "activations/layer4_attention_weight_min": -130.6153106689453,
      "activations/layer5_attention_weight_max": 65.80337524414062,
      "activations/layer5_attention_weight_min": -66.94142150878906,
      "activations/layer6_attention_weight_max": 50.01362991333008,
      "activations/layer6_attention_weight_min": -54.09786605834961,
      "activations/layer7_attention_weight_max": 86.17279815673828,
      "activations/layer7_attention_weight_min": -90.21604919433594,
      "activations/layer8_attention_weight_max": 41.14580154418945,
      "activations/layer8_attention_weight_min": -43.11118698120117,
      "activations/layer9_attention_weight_max": 37.100929260253906,
      "activations/layer9_attention_weight_min": -36.28506088256836,
      "epoch": 6.8,
      "learning_rate": 0.00010719924242424241,
      "loss": 2.8227,
      "step": 117100
    },
    {
      "activations/layer0_attention_weight_max": 16.552209854125977,
      "activations/layer0_attention_weight_min": -11.702159881591797,
      "activations/layer10_attention_weight_max": 35.7774543762207,
      "activations/layer10_attention_weight_min": -34.590728759765625,
      "activations/layer11_attention_weight_max": 35.875274658203125,
      "activations/layer11_attention_weight_min": -32.74531936645508,
      "activations/layer12_attention_weight_max": 24.947540283203125,
      "activations/layer12_attention_weight_min": -26.75742530822754,
      "activations/layer13_attention_weight_max": 47.95438766479492,
      "activations/layer13_attention_weight_min": -33.113922119140625,
      "activations/layer14_attention_weight_max": 69.57228088378906,
      "activations/layer14_attention_weight_min": -47.70851135253906,
      "activations/layer15_attention_weight_max": 40.66664123535156,
      "activations/layer15_attention_weight_min": -29.797359466552734,
      "activations/layer16_attention_weight_max": 43.562374114990234,
      "activations/layer16_attention_weight_min": -28.48869514465332,
      "activations/layer17_attention_weight_max": 41.25375747680664,
      "activations/layer17_attention_weight_min": -25.67875099182129,
      "activations/layer18_attention_weight_max": 37.80445861816406,
      "activations/layer18_attention_weight_min": -20.82545280456543,
      "activations/layer19_attention_weight_max": 47.698360443115234,
      "activations/layer19_attention_weight_min": -30.76444435119629,
      "activations/layer1_attention_weight_max": 16.539226531982422,
      "activations/layer1_attention_weight_min": -16.383596420288086,
      "activations/layer20_attention_weight_max": 39.75852584838867,
      "activations/layer20_attention_weight_min": -24.473796844482422,
      "activations/layer21_attention_weight_max": 38.4842529296875,
      "activations/layer21_attention_weight_min": -21.836132049560547,
      "activations/layer22_attention_weight_max": 70.10073852539062,
      "activations/layer22_attention_weight_min": -27.701271057128906,
      "activations/layer23_attention_weight_max": 46.991146087646484,
      "activations/layer23_attention_weight_min": -23.191123962402344,
      "activations/layer2_attention_weight_max": 34.862281799316406,
      "activations/layer2_attention_weight_min": -32.158756256103516,
      "activations/layer3_attention_weight_max": 100.09326934814453,
      "activations/layer3_attention_weight_min": -99.56648254394531,
      "activations/layer4_attention_weight_max": 140.77685546875,
      "activations/layer4_attention_weight_min": -130.544921875,
      "activations/layer5_attention_weight_max": 71.18309783935547,
      "activations/layer5_attention_weight_min": -70.57332611083984,
      "activations/layer6_attention_weight_max": 59.85869216918945,
      "activations/layer6_attention_weight_min": -59.782352447509766,
      "activations/layer7_attention_weight_max": 96.01654052734375,
      "activations/layer7_attention_weight_min": -107.08895111083984,
      "activations/layer8_attention_weight_max": 45.710933685302734,
      "activations/layer8_attention_weight_min": -45.81924057006836,
      "activations/layer9_attention_weight_max": 39.796730041503906,
      "activations/layer9_attention_weight_min": -39.611602783203125,
      "epoch": 6.81,
      "learning_rate": 0.00010718030303030303,
      "loss": 2.825,
      "step": 117150
    },
    {
      "activations/layer0_attention_weight_max": 16.243850708007812,
      "activations/layer0_attention_weight_min": -11.950850486755371,
      "activations/layer10_attention_weight_max": 37.8358039855957,
      "activations/layer10_attention_weight_min": -33.53609085083008,
      "activations/layer11_attention_weight_max": 35.03730773925781,
      "activations/layer11_attention_weight_min": -30.803037643432617,
      "activations/layer12_attention_weight_max": 29.97404670715332,
      "activations/layer12_attention_weight_min": -28.290342330932617,
      "activations/layer13_attention_weight_max": 43.04186248779297,
      "activations/layer13_attention_weight_min": -34.59977722167969,
      "activations/layer14_attention_weight_max": 57.91826248168945,
      "activations/layer14_attention_weight_min": -44.794158935546875,
      "activations/layer15_attention_weight_max": 37.80558776855469,
      "activations/layer15_attention_weight_min": -30.291404724121094,
      "activations/layer16_attention_weight_max": 34.31210708618164,
      "activations/layer16_attention_weight_min": -28.740495681762695,
      "activations/layer17_attention_weight_max": 35.637939453125,
      "activations/layer17_attention_weight_min": -27.74736976623535,
      "activations/layer18_attention_weight_max": 31.6445255279541,
      "activations/layer18_attention_weight_min": -22.56875228881836,
      "activations/layer19_attention_weight_max": 39.21575164794922,
      "activations/layer19_attention_weight_min": -30.469837188720703,
      "activations/layer1_attention_weight_max": 17.17709732055664,
      "activations/layer1_attention_weight_min": -13.302998542785645,
      "activations/layer20_attention_weight_max": 33.20551300048828,
      "activations/layer20_attention_weight_min": -24.696393966674805,
      "activations/layer21_attention_weight_max": 32.051841735839844,
      "activations/layer21_attention_weight_min": -23.728147506713867,
      "activations/layer22_attention_weight_max": 48.264469146728516,
      "activations/layer22_attention_weight_min": -30.411252975463867,
      "activations/layer23_attention_weight_max": 36.19233703613281,
      "activations/layer23_attention_weight_min": -24.2939510345459,
      "activations/layer2_attention_weight_max": 31.674659729003906,
      "activations/layer2_attention_weight_min": -32.70470428466797,
      "activations/layer3_attention_weight_max": 91.22228240966797,
      "activations/layer3_attention_weight_min": -99.23737335205078,
      "activations/layer4_attention_weight_max": 122.82486724853516,
      "activations/layer4_attention_weight_min": -125.85662841796875,
      "activations/layer5_attention_weight_max": 66.91826629638672,
      "activations/layer5_attention_weight_min": -65.9798355102539,
      "activations/layer6_attention_weight_max": 51.24150466918945,
      "activations/layer6_attention_weight_min": -55.17078399658203,
      "activations/layer7_attention_weight_max": 86.30973052978516,
      "activations/layer7_attention_weight_min": -96.04789733886719,
      "activations/layer8_attention_weight_max": 43.011131286621094,
      "activations/layer8_attention_weight_min": -44.08784484863281,
      "activations/layer9_attention_weight_max": 38.84055709838867,
      "activations/layer9_attention_weight_min": -36.43891906738281,
      "epoch": 6.81,
      "learning_rate": 0.00010716136363636363,
      "loss": 2.8236,
      "step": 117200
    },
    {
      "activations/layer0_attention_weight_max": 16.118694305419922,
      "activations/layer0_attention_weight_min": -12.796738624572754,
      "activations/layer10_attention_weight_max": 35.76633834838867,
      "activations/layer10_attention_weight_min": -33.53471374511719,
      "activations/layer11_attention_weight_max": 34.53376007080078,
      "activations/layer11_attention_weight_min": -31.92302703857422,
      "activations/layer12_attention_weight_max": 23.741437911987305,
      "activations/layer12_attention_weight_min": -31.246212005615234,
      "activations/layer13_attention_weight_max": 43.839717864990234,
      "activations/layer13_attention_weight_min": -34.34727096557617,
      "activations/layer14_attention_weight_max": 56.40594482421875,
      "activations/layer14_attention_weight_min": -45.31550979614258,
      "activations/layer15_attention_weight_max": 37.855533599853516,
      "activations/layer15_attention_weight_min": -30.42923355102539,
      "activations/layer16_attention_weight_max": 33.98831558227539,
      "activations/layer16_attention_weight_min": -28.21202278137207,
      "activations/layer17_attention_weight_max": 38.089420318603516,
      "activations/layer17_attention_weight_min": -24.988784790039062,
      "activations/layer18_attention_weight_max": 30.135717391967773,
      "activations/layer18_attention_weight_min": -20.29669952392578,
      "activations/layer19_attention_weight_max": 36.87963104248047,
      "activations/layer19_attention_weight_min": -29.269174575805664,
      "activations/layer1_attention_weight_max": 16.257850646972656,
      "activations/layer1_attention_weight_min": -14.487072944641113,
      "activations/layer20_attention_weight_max": 33.22740173339844,
      "activations/layer20_attention_weight_min": -23.584047317504883,
      "activations/layer21_attention_weight_max": 30.7071533203125,
      "activations/layer21_attention_weight_min": -21.8304386138916,
      "activations/layer22_attention_weight_max": 46.27500534057617,
      "activations/layer22_attention_weight_min": -27.135719299316406,
      "activations/layer23_attention_weight_max": 36.03804397583008,
      "activations/layer23_attention_weight_min": -20.310258865356445,
      "activations/layer2_attention_weight_max": 31.763349533081055,
      "activations/layer2_attention_weight_min": -30.816417694091797,
      "activations/layer3_attention_weight_max": 91.00236511230469,
      "activations/layer3_attention_weight_min": -96.16414642333984,
      "activations/layer4_attention_weight_max": 127.20111083984375,
      "activations/layer4_attention_weight_min": -126.0111312866211,
      "activations/layer5_attention_weight_max": 69.01639556884766,
      "activations/layer5_attention_weight_min": -69.04487609863281,
      "activations/layer6_attention_weight_max": 52.07908248901367,
      "activations/layer6_attention_weight_min": -55.0409049987793,
      "activations/layer7_attention_weight_max": 91.11589813232422,
      "activations/layer7_attention_weight_min": -93.7682113647461,
      "activations/layer8_attention_weight_max": 42.99325180053711,
      "activations/layer8_attention_weight_min": -45.36613845825195,
      "activations/layer9_attention_weight_max": 37.3912467956543,
      "activations/layer9_attention_weight_min": -37.21353530883789,
      "epoch": 6.81,
      "learning_rate": 0.00010714280303030302,
      "loss": 2.8359,
      "step": 117250
    },
    {
      "activations/layer0_attention_weight_max": 16.72834587097168,
      "activations/layer0_attention_weight_min": -12.73099422454834,
      "activations/layer10_attention_weight_max": 33.884803771972656,
      "activations/layer10_attention_weight_min": -30.791854858398438,
      "activations/layer11_attention_weight_max": 34.37373352050781,
      "activations/layer11_attention_weight_min": -29.6466064453125,
      "activations/layer12_attention_weight_max": 53.63584899902344,
      "activations/layer12_attention_weight_min": -25.5176944732666,
      "activations/layer13_attention_weight_max": 55.887611389160156,
      "activations/layer13_attention_weight_min": -31.40996551513672,
      "activations/layer14_attention_weight_max": 55.6256103515625,
      "activations/layer14_attention_weight_min": -42.81764602661133,
      "activations/layer15_attention_weight_max": 36.05441665649414,
      "activations/layer15_attention_weight_min": -28.534936904907227,
      "activations/layer16_attention_weight_max": 31.28510284423828,
      "activations/layer16_attention_weight_min": -28.698442459106445,
      "activations/layer17_attention_weight_max": 33.56443405151367,
      "activations/layer17_attention_weight_min": -24.918041229248047,
      "activations/layer18_attention_weight_max": 32.256717681884766,
      "activations/layer18_attention_weight_min": -23.20203971862793,
      "activations/layer19_attention_weight_max": 38.35798263549805,
      "activations/layer19_attention_weight_min": -29.334775924682617,
      "activations/layer1_attention_weight_max": 17.333105087280273,
      "activations/layer1_attention_weight_min": -15.59349536895752,
      "activations/layer20_attention_weight_max": 32.65376281738281,
      "activations/layer20_attention_weight_min": -23.895551681518555,
      "activations/layer21_attention_weight_max": 31.111143112182617,
      "activations/layer21_attention_weight_min": -22.34130859375,
      "activations/layer22_attention_weight_max": 48.99168014526367,
      "activations/layer22_attention_weight_min": -26.158721923828125,
      "activations/layer23_attention_weight_max": 37.15574264526367,
      "activations/layer23_attention_weight_min": -22.335357666015625,
      "activations/layer2_attention_weight_max": 33.128273010253906,
      "activations/layer2_attention_weight_min": -30.96985626220703,
      "activations/layer3_attention_weight_max": 89.59904479980469,
      "activations/layer3_attention_weight_min": -90.54276275634766,
      "activations/layer4_attention_weight_max": 127.69792938232422,
      "activations/layer4_attention_weight_min": -124.5215072631836,
      "activations/layer5_attention_weight_max": 67.30711364746094,
      "activations/layer5_attention_weight_min": -71.86216735839844,
      "activations/layer6_attention_weight_max": 55.89882278442383,
      "activations/layer6_attention_weight_min": -53.81647491455078,
      "activations/layer7_attention_weight_max": 89.69338989257812,
      "activations/layer7_attention_weight_min": -92.76080322265625,
      "activations/layer8_attention_weight_max": 39.907981872558594,
      "activations/layer8_attention_weight_min": -39.7034797668457,
      "activations/layer9_attention_weight_max": 36.43510818481445,
      "activations/layer9_attention_weight_min": -33.850486755371094,
      "epoch": 6.82,
      "learning_rate": 0.00010712386363636363,
      "loss": 2.8261,
      "step": 117300
    },
    {
      "activations/layer0_attention_weight_max": 15.781614303588867,
      "activations/layer0_attention_weight_min": -12.957049369812012,
      "activations/layer10_attention_weight_max": 31.608211517333984,
      "activations/layer10_attention_weight_min": -32.55833435058594,
      "activations/layer11_attention_weight_max": 32.32622528076172,
      "activations/layer11_attention_weight_min": -30.52408218383789,
      "activations/layer12_attention_weight_max": 21.57696533203125,
      "activations/layer12_attention_weight_min": -25.130807876586914,
      "activations/layer13_attention_weight_max": 41.8548583984375,
      "activations/layer13_attention_weight_min": -32.446537017822266,
      "activations/layer14_attention_weight_max": 57.60087966918945,
      "activations/layer14_attention_weight_min": -41.882545471191406,
      "activations/layer15_attention_weight_max": 35.8693733215332,
      "activations/layer15_attention_weight_min": -27.263608932495117,
      "activations/layer16_attention_weight_max": 34.534969329833984,
      "activations/layer16_attention_weight_min": -26.673568725585938,
      "activations/layer17_attention_weight_max": 35.2232551574707,
      "activations/layer17_attention_weight_min": -25.176044464111328,
      "activations/layer18_attention_weight_max": 30.417903900146484,
      "activations/layer18_attention_weight_min": -20.921241760253906,
      "activations/layer19_attention_weight_max": 35.77847671508789,
      "activations/layer19_attention_weight_min": -29.964099884033203,
      "activations/layer1_attention_weight_max": 19.912887573242188,
      "activations/layer1_attention_weight_min": -15.387134552001953,
      "activations/layer20_attention_weight_max": 30.412168502807617,
      "activations/layer20_attention_weight_min": -22.983842849731445,
      "activations/layer21_attention_weight_max": 34.14874267578125,
      "activations/layer21_attention_weight_min": -20.98384666442871,
      "activations/layer22_attention_weight_max": 47.45198059082031,
      "activations/layer22_attention_weight_min": -27.745698928833008,
      "activations/layer23_attention_weight_max": 34.1854248046875,
      "activations/layer23_attention_weight_min": -22.721858978271484,
      "activations/layer2_attention_weight_max": 29.378692626953125,
      "activations/layer2_attention_weight_min": -31.565099716186523,
      "activations/layer3_attention_weight_max": 85.1305923461914,
      "activations/layer3_attention_weight_min": -90.09722900390625,
      "activations/layer4_attention_weight_max": 122.12239074707031,
      "activations/layer4_attention_weight_min": -126.00975799560547,
      "activations/layer5_attention_weight_max": 67.91404724121094,
      "activations/layer5_attention_weight_min": -64.50100708007812,
      "activations/layer6_attention_weight_max": 50.01444625854492,
      "activations/layer6_attention_weight_min": -51.89883041381836,
      "activations/layer7_attention_weight_max": 84.81138610839844,
      "activations/layer7_attention_weight_min": -84.1229476928711,
      "activations/layer8_attention_weight_max": 39.46652603149414,
      "activations/layer8_attention_weight_min": -40.16334915161133,
      "activations/layer9_attention_weight_max": 34.58967208862305,
      "activations/layer9_attention_weight_min": -35.04469299316406,
      "epoch": 6.82,
      "learning_rate": 0.00010710492424242424,
      "loss": 2.823,
      "step": 117350
    },
    {
      "activations/layer0_attention_weight_max": 15.548463821411133,
      "activations/layer0_attention_weight_min": -11.57321548461914,
      "activations/layer10_attention_weight_max": 32.674869537353516,
      "activations/layer10_attention_weight_min": -30.61007308959961,
      "activations/layer11_attention_weight_max": 32.36763000488281,
      "activations/layer11_attention_weight_min": -28.46913719177246,
      "activations/layer12_attention_weight_max": 24.77337074279785,
      "activations/layer12_attention_weight_min": -27.723066329956055,
      "activations/layer13_attention_weight_max": 42.615074157714844,
      "activations/layer13_attention_weight_min": -31.781768798828125,
      "activations/layer14_attention_weight_max": 47.82301712036133,
      "activations/layer14_attention_weight_min": -38.17726516723633,
      "activations/layer15_attention_weight_max": 38.29140090942383,
      "activations/layer15_attention_weight_min": -28.552825927734375,
      "activations/layer16_attention_weight_max": 33.04536819458008,
      "activations/layer16_attention_weight_min": -27.72962188720703,
      "activations/layer17_attention_weight_max": 37.17979049682617,
      "activations/layer17_attention_weight_min": -25.355693817138672,
      "activations/layer18_attention_weight_max": 37.10245895385742,
      "activations/layer18_attention_weight_min": -21.495834350585938,
      "activations/layer19_attention_weight_max": 37.9326286315918,
      "activations/layer19_attention_weight_min": -29.08379554748535,
      "activations/layer1_attention_weight_max": 17.146930694580078,
      "activations/layer1_attention_weight_min": -13.720880508422852,
      "activations/layer20_attention_weight_max": 33.79610061645508,
      "activations/layer20_attention_weight_min": -23.454486846923828,
      "activations/layer21_attention_weight_max": 33.198482513427734,
      "activations/layer21_attention_weight_min": -22.53399658203125,
      "activations/layer22_attention_weight_max": 50.93667221069336,
      "activations/layer22_attention_weight_min": -30.200090408325195,
      "activations/layer23_attention_weight_max": 38.26576232910156,
      "activations/layer23_attention_weight_min": -22.250228881835938,
      "activations/layer2_attention_weight_max": 32.95400619506836,
      "activations/layer2_attention_weight_min": -31.83086395263672,
      "activations/layer3_attention_weight_max": 92.7064208984375,
      "activations/layer3_attention_weight_min": -92.94302368164062,
      "activations/layer4_attention_weight_max": 121.92623138427734,
      "activations/layer4_attention_weight_min": -124.0408706665039,
      "activations/layer5_attention_weight_max": 66.71205139160156,
      "activations/layer5_attention_weight_min": -66.74647521972656,
      "activations/layer6_attention_weight_max": 52.30077362060547,
      "activations/layer6_attention_weight_min": -53.23407745361328,
      "activations/layer7_attention_weight_max": 89.35663604736328,
      "activations/layer7_attention_weight_min": -86.95930480957031,
      "activations/layer8_attention_weight_max": 42.62724685668945,
      "activations/layer8_attention_weight_min": -42.17444610595703,
      "activations/layer9_attention_weight_max": 37.9025993347168,
      "activations/layer9_attention_weight_min": -34.06547164916992,
      "epoch": 6.82,
      "learning_rate": 0.00010708636363636363,
      "loss": 2.8195,
      "step": 117400
    },
    {
      "activations/layer0_attention_weight_max": 16.029550552368164,
      "activations/layer0_attention_weight_min": -11.758275032043457,
      "activations/layer10_attention_weight_max": 34.046939849853516,
      "activations/layer10_attention_weight_min": -29.756425857543945,
      "activations/layer11_attention_weight_max": 31.69792938232422,
      "activations/layer11_attention_weight_min": -27.708404541015625,
      "activations/layer12_attention_weight_max": 21.899898529052734,
      "activations/layer12_attention_weight_min": -24.921005249023438,
      "activations/layer13_attention_weight_max": 38.06559753417969,
      "activations/layer13_attention_weight_min": -33.196990966796875,
      "activations/layer14_attention_weight_max": 50.78884506225586,
      "activations/layer14_attention_weight_min": -46.76222229003906,
      "activations/layer15_attention_weight_max": 34.31631088256836,
      "activations/layer15_attention_weight_min": -30.510774612426758,
      "activations/layer16_attention_weight_max": 31.692981719970703,
      "activations/layer16_attention_weight_min": -27.038806915283203,
      "activations/layer17_attention_weight_max": 36.66791534423828,
      "activations/layer17_attention_weight_min": -24.856624603271484,
      "activations/layer18_attention_weight_max": 34.205318450927734,
      "activations/layer18_attention_weight_min": -22.8663330078125,
      "activations/layer19_attention_weight_max": 38.34221649169922,
      "activations/layer19_attention_weight_min": -30.403905868530273,
      "activations/layer1_attention_weight_max": 17.607791900634766,
      "activations/layer1_attention_weight_min": -15.166157722473145,
      "activations/layer20_attention_weight_max": 32.03278350830078,
      "activations/layer20_attention_weight_min": -24.113134384155273,
      "activations/layer21_attention_weight_max": 35.74892044067383,
      "activations/layer21_attention_weight_min": -23.811355590820312,
      "activations/layer22_attention_weight_max": 53.13191604614258,
      "activations/layer22_attention_weight_min": -28.76799964904785,
      "activations/layer23_attention_weight_max": 36.270790100097656,
      "activations/layer23_attention_weight_min": -22.957502365112305,
      "activations/layer2_attention_weight_max": 30.606380462646484,
      "activations/layer2_attention_weight_min": -30.33877944946289,
      "activations/layer3_attention_weight_max": 87.63956451416016,
      "activations/layer3_attention_weight_min": -87.20464324951172,
      "activations/layer4_attention_weight_max": 119.34093475341797,
      "activations/layer4_attention_weight_min": -117.6696548461914,
      "activations/layer5_attention_weight_max": 65.32229614257812,
      "activations/layer5_attention_weight_min": -63.81766891479492,
      "activations/layer6_attention_weight_max": 51.30051040649414,
      "activations/layer6_attention_weight_min": -52.93162155151367,
      "activations/layer7_attention_weight_max": 84.11988067626953,
      "activations/layer7_attention_weight_min": -83.60528564453125,
      "activations/layer8_attention_weight_max": 40.569305419921875,
      "activations/layer8_attention_weight_min": -38.796348571777344,
      "activations/layer9_attention_weight_max": 35.10552215576172,
      "activations/layer9_attention_weight_min": -33.90494918823242,
      "epoch": 6.82,
      "learning_rate": 0.00010706742424242423,
      "loss": 2.8168,
      "step": 117450
    },
    {
      "activations/layer0_attention_weight_max": 16.28597640991211,
      "activations/layer0_attention_weight_min": -12.266439437866211,
      "activations/layer10_attention_weight_max": 41.5764045715332,
      "activations/layer10_attention_weight_min": -37.906490325927734,
      "activations/layer11_attention_weight_max": 43.77532196044922,
      "activations/layer11_attention_weight_min": -35.12049102783203,
      "activations/layer12_attention_weight_max": 68.50382995605469,
      "activations/layer12_attention_weight_min": -28.661354064941406,
      "activations/layer13_attention_weight_max": 69.88717651367188,
      "activations/layer13_attention_weight_min": -36.38014221191406,
      "activations/layer14_attention_weight_max": 86.87467956542969,
      "activations/layer14_attention_weight_min": -51.19523620605469,
      "activations/layer15_attention_weight_max": 51.78437423706055,
      "activations/layer15_attention_weight_min": -30.645286560058594,
      "activations/layer16_attention_weight_max": 45.09123992919922,
      "activations/layer16_attention_weight_min": -26.28204345703125,
      "activations/layer17_attention_weight_max": 61.5156135559082,
      "activations/layer17_attention_weight_min": -26.013349533081055,
      "activations/layer18_attention_weight_max": 46.361209869384766,
      "activations/layer18_attention_weight_min": -25.377946853637695,
      "activations/layer19_attention_weight_max": 43.559356689453125,
      "activations/layer19_attention_weight_min": -29.329547882080078,
      "activations/layer1_attention_weight_max": 16.854278564453125,
      "activations/layer1_attention_weight_min": -15.199966430664062,
      "activations/layer20_attention_weight_max": 33.72583770751953,
      "activations/layer20_attention_weight_min": -24.606800079345703,
      "activations/layer21_attention_weight_max": 33.86357116699219,
      "activations/layer21_attention_weight_min": -21.83588981628418,
      "activations/layer22_attention_weight_max": 48.90593338012695,
      "activations/layer22_attention_weight_min": -29.06101417541504,
      "activations/layer23_attention_weight_max": 33.83209991455078,
      "activations/layer23_attention_weight_min": -22.590375900268555,
      "activations/layer2_attention_weight_max": 32.292667388916016,
      "activations/layer2_attention_weight_min": -30.6633358001709,
      "activations/layer3_attention_weight_max": 86.88445281982422,
      "activations/layer3_attention_weight_min": -89.19379425048828,
      "activations/layer4_attention_weight_max": 120.9165267944336,
      "activations/layer4_attention_weight_min": -118.918212890625,
      "activations/layer5_attention_weight_max": 66.21383666992188,
      "activations/layer5_attention_weight_min": -64.028076171875,
      "activations/layer6_attention_weight_max": 57.58403778076172,
      "activations/layer6_attention_weight_min": -55.43578338623047,
      "activations/layer7_attention_weight_max": 96.4478759765625,
      "activations/layer7_attention_weight_min": -89.68992614746094,
      "activations/layer8_attention_weight_max": 47.67734146118164,
      "activations/layer8_attention_weight_min": -42.783119201660156,
      "activations/layer9_attention_weight_max": 45.871395111083984,
      "activations/layer9_attention_weight_min": -41.65219497680664,
      "epoch": 6.83,
      "learning_rate": 0.00010704848484848484,
      "loss": 2.8304,
      "step": 117500
    },
    {
      "activations/layer0_attention_weight_max": 16.928129196166992,
      "activations/layer0_attention_weight_min": -11.495601654052734,
      "activations/layer10_attention_weight_max": 33.26991653442383,
      "activations/layer10_attention_weight_min": -32.771461486816406,
      "activations/layer11_attention_weight_max": 34.486576080322266,
      "activations/layer11_attention_weight_min": -30.937397003173828,
      "activations/layer12_attention_weight_max": 24.438369750976562,
      "activations/layer12_attention_weight_min": -29.167438507080078,
      "activations/layer13_attention_weight_max": 37.53480911254883,
      "activations/layer13_attention_weight_min": -32.258628845214844,
      "activations/layer14_attention_weight_max": 54.019432067871094,
      "activations/layer14_attention_weight_min": -44.820579528808594,
      "activations/layer15_attention_weight_max": 35.32490158081055,
      "activations/layer15_attention_weight_min": -30.013484954833984,
      "activations/layer16_attention_weight_max": 31.487157821655273,
      "activations/layer16_attention_weight_min": -29.767250061035156,
      "activations/layer17_attention_weight_max": 32.72373962402344,
      "activations/layer17_attention_weight_min": -25.60040283203125,
      "activations/layer18_attention_weight_max": 27.898826599121094,
      "activations/layer18_attention_weight_min": -23.156848907470703,
      "activations/layer19_attention_weight_max": 36.520164489746094,
      "activations/layer19_attention_weight_min": -31.49383544921875,
      "activations/layer1_attention_weight_max": 17.434444427490234,
      "activations/layer1_attention_weight_min": -12.284537315368652,
      "activations/layer20_attention_weight_max": 30.51913070678711,
      "activations/layer20_attention_weight_min": -24.75054931640625,
      "activations/layer21_attention_weight_max": 26.826061248779297,
      "activations/layer21_attention_weight_min": -22.96731185913086,
      "activations/layer22_attention_weight_max": 50.98292541503906,
      "activations/layer22_attention_weight_min": -31.186927795410156,
      "activations/layer23_attention_weight_max": 32.26898956298828,
      "activations/layer23_attention_weight_min": -23.930145263671875,
      "activations/layer2_attention_weight_max": 30.649303436279297,
      "activations/layer2_attention_weight_min": -29.473243713378906,
      "activations/layer3_attention_weight_max": 87.71327209472656,
      "activations/layer3_attention_weight_min": -88.04509735107422,
      "activations/layer4_attention_weight_max": 125.70345306396484,
      "activations/layer4_attention_weight_min": -125.83522033691406,
      "activations/layer5_attention_weight_max": 66.61194610595703,
      "activations/layer5_attention_weight_min": -66.91328430175781,
      "activations/layer6_attention_weight_max": 53.08174514770508,
      "activations/layer6_attention_weight_min": -54.892799377441406,
      "activations/layer7_attention_weight_max": 89.5324935913086,
      "activations/layer7_attention_weight_min": -91.42414855957031,
      "activations/layer8_attention_weight_max": 42.34440994262695,
      "activations/layer8_attention_weight_min": -41.81941223144531,
      "activations/layer9_attention_weight_max": 37.36054229736328,
      "activations/layer9_attention_weight_min": -35.17981719970703,
      "epoch": 6.83,
      "learning_rate": 0.00010702954545454546,
      "loss": 2.8129,
      "step": 117550
    },
    {
      "activations/layer0_attention_weight_max": 15.904350280761719,
      "activations/layer0_attention_weight_min": -12.707505226135254,
      "activations/layer10_attention_weight_max": 36.39986038208008,
      "activations/layer10_attention_weight_min": -33.176456451416016,
      "activations/layer11_attention_weight_max": 34.117088317871094,
      "activations/layer11_attention_weight_min": -31.45069122314453,
      "activations/layer12_attention_weight_max": 21.819374084472656,
      "activations/layer12_attention_weight_min": -28.58972930908203,
      "activations/layer13_attention_weight_max": 42.001243591308594,
      "activations/layer13_attention_weight_min": -32.750431060791016,
      "activations/layer14_attention_weight_max": 46.27740478515625,
      "activations/layer14_attention_weight_min": -40.69502639770508,
      "activations/layer15_attention_weight_max": 37.316749572753906,
      "activations/layer15_attention_weight_min": -28.315364837646484,
      "activations/layer16_attention_weight_max": 32.954986572265625,
      "activations/layer16_attention_weight_min": -26.646846771240234,
      "activations/layer17_attention_weight_max": 35.063899993896484,
      "activations/layer17_attention_weight_min": -24.430736541748047,
      "activations/layer18_attention_weight_max": 31.099525451660156,
      "activations/layer18_attention_weight_min": -19.691171646118164,
      "activations/layer19_attention_weight_max": 40.71820831298828,
      "activations/layer19_attention_weight_min": -29.180662155151367,
      "activations/layer1_attention_weight_max": 17.069494247436523,
      "activations/layer1_attention_weight_min": -14.364293098449707,
      "activations/layer20_attention_weight_max": 33.338706970214844,
      "activations/layer20_attention_weight_min": -24.232847213745117,
      "activations/layer21_attention_weight_max": 31.70726203918457,
      "activations/layer21_attention_weight_min": -21.582056045532227,
      "activations/layer22_attention_weight_max": 51.290443420410156,
      "activations/layer22_attention_weight_min": -27.362430572509766,
      "activations/layer23_attention_weight_max": 34.212711334228516,
      "activations/layer23_attention_weight_min": -21.493816375732422,
      "activations/layer2_attention_weight_max": 31.457786560058594,
      "activations/layer2_attention_weight_min": -30.450117111206055,
      "activations/layer3_attention_weight_max": 88.40441131591797,
      "activations/layer3_attention_weight_min": -86.3889389038086,
      "activations/layer4_attention_weight_max": 127.7950439453125,
      "activations/layer4_attention_weight_min": -115.59881591796875,
      "activations/layer5_attention_weight_max": 64.08187866210938,
      "activations/layer5_attention_weight_min": -64.69024658203125,
      "activations/layer6_attention_weight_max": 53.5930061340332,
      "activations/layer6_attention_weight_min": -53.50259780883789,
      "activations/layer7_attention_weight_max": 87.25959777832031,
      "activations/layer7_attention_weight_min": -84.77251434326172,
      "activations/layer8_attention_weight_max": 39.5994758605957,
      "activations/layer8_attention_weight_min": -42.15153884887695,
      "activations/layer9_attention_weight_max": 38.85695266723633,
      "activations/layer9_attention_weight_min": -35.195919036865234,
      "epoch": 6.83,
      "learning_rate": 0.00010701060606060605,
      "loss": 2.8156,
      "step": 117600
    },
    {
      "activations/layer0_attention_weight_max": 15.601801872253418,
      "activations/layer0_attention_weight_min": -12.978281021118164,
      "activations/layer10_attention_weight_max": 35.711734771728516,
      "activations/layer10_attention_weight_min": -32.79655075073242,
      "activations/layer11_attention_weight_max": 40.1092529296875,
      "activations/layer11_attention_weight_min": -29.343090057373047,
      "activations/layer12_attention_weight_max": 24.167943954467773,
      "activations/layer12_attention_weight_min": -26.50829315185547,
      "activations/layer13_attention_weight_max": 49.61376190185547,
      "activations/layer13_attention_weight_min": -39.792327880859375,
      "activations/layer14_attention_weight_max": 55.70004653930664,
      "activations/layer14_attention_weight_min": -44.53813552856445,
      "activations/layer15_attention_weight_max": 45.27470397949219,
      "activations/layer15_attention_weight_min": -37.75017547607422,
      "activations/layer16_attention_weight_max": 32.27711486816406,
      "activations/layer16_attention_weight_min": -28.452117919921875,
      "activations/layer17_attention_weight_max": 36.118568420410156,
      "activations/layer17_attention_weight_min": -24.49460792541504,
      "activations/layer18_attention_weight_max": 30.8165225982666,
      "activations/layer18_attention_weight_min": -22.81709098815918,
      "activations/layer19_attention_weight_max": 34.45396041870117,
      "activations/layer19_attention_weight_min": -31.52107810974121,
      "activations/layer1_attention_weight_max": 17.32965660095215,
      "activations/layer1_attention_weight_min": -12.584354400634766,
      "activations/layer20_attention_weight_max": 28.708953857421875,
      "activations/layer20_attention_weight_min": -23.953641891479492,
      "activations/layer21_attention_weight_max": 27.812108993530273,
      "activations/layer21_attention_weight_min": -22.109413146972656,
      "activations/layer22_attention_weight_max": 46.8487663269043,
      "activations/layer22_attention_weight_min": -30.445268630981445,
      "activations/layer23_attention_weight_max": 34.96037292480469,
      "activations/layer23_attention_weight_min": -22.32038688659668,
      "activations/layer2_attention_weight_max": 32.65723419189453,
      "activations/layer2_attention_weight_min": -32.07759094238281,
      "activations/layer3_attention_weight_max": 96.06684112548828,
      "activations/layer3_attention_weight_min": -93.51812744140625,
      "activations/layer4_attention_weight_max": 131.7825927734375,
      "activations/layer4_attention_weight_min": -126.48209381103516,
      "activations/layer5_attention_weight_max": 70.42130279541016,
      "activations/layer5_attention_weight_min": -66.57628631591797,
      "activations/layer6_attention_weight_max": 55.49828338623047,
      "activations/layer6_attention_weight_min": -59.90205764770508,
      "activations/layer7_attention_weight_max": 92.56221771240234,
      "activations/layer7_attention_weight_min": -94.80741119384766,
      "activations/layer8_attention_weight_max": 43.11079788208008,
      "activations/layer8_attention_weight_min": -43.89064025878906,
      "activations/layer9_attention_weight_max": 38.91501998901367,
      "activations/layer9_attention_weight_min": -36.06707763671875,
      "epoch": 6.84,
      "learning_rate": 0.00010699166666666666,
      "loss": 2.8176,
      "step": 117650
    },
    {
      "activations/layer0_attention_weight_max": 16.420196533203125,
      "activations/layer0_attention_weight_min": -11.666707992553711,
      "activations/layer10_attention_weight_max": 34.242149353027344,
      "activations/layer10_attention_weight_min": -34.297576904296875,
      "activations/layer11_attention_weight_max": 33.079368591308594,
      "activations/layer11_attention_weight_min": -31.245433807373047,
      "activations/layer12_attention_weight_max": 24.888803482055664,
      "activations/layer12_attention_weight_min": -24.912403106689453,
      "activations/layer13_attention_weight_max": 47.26232147216797,
      "activations/layer13_attention_weight_min": -35.070037841796875,
      "activations/layer14_attention_weight_max": 56.62147903442383,
      "activations/layer14_attention_weight_min": -40.92306900024414,
      "activations/layer15_attention_weight_max": 41.23118591308594,
      "activations/layer15_attention_weight_min": -31.781879425048828,
      "activations/layer16_attention_weight_max": 38.63805389404297,
      "activations/layer16_attention_weight_min": -28.55451202392578,
      "activations/layer17_attention_weight_max": 36.25934982299805,
      "activations/layer17_attention_weight_min": -26.178815841674805,
      "activations/layer18_attention_weight_max": 31.8880615234375,
      "activations/layer18_attention_weight_min": -23.287311553955078,
      "activations/layer19_attention_weight_max": 42.513092041015625,
      "activations/layer19_attention_weight_min": -30.875524520874023,
      "activations/layer1_attention_weight_max": 17.99321937561035,
      "activations/layer1_attention_weight_min": -13.618180274963379,
      "activations/layer20_attention_weight_max": 35.04967498779297,
      "activations/layer20_attention_weight_min": -24.140430450439453,
      "activations/layer21_attention_weight_max": 33.943180084228516,
      "activations/layer21_attention_weight_min": -21.423763275146484,
      "activations/layer22_attention_weight_max": 48.63953399658203,
      "activations/layer22_attention_weight_min": -29.48783302307129,
      "activations/layer23_attention_weight_max": 38.248260498046875,
      "activations/layer23_attention_weight_min": -22.977188110351562,
      "activations/layer2_attention_weight_max": 32.6962890625,
      "activations/layer2_attention_weight_min": -30.98537826538086,
      "activations/layer3_attention_weight_max": 89.53564453125,
      "activations/layer3_attention_weight_min": -88.2676010131836,
      "activations/layer4_attention_weight_max": 122.12993621826172,
      "activations/layer4_attention_weight_min": -122.10839080810547,
      "activations/layer5_attention_weight_max": 65.86111450195312,
      "activations/layer5_attention_weight_min": -64.65559387207031,
      "activations/layer6_attention_weight_max": 51.13108444213867,
      "activations/layer6_attention_weight_min": -51.95246887207031,
      "activations/layer7_attention_weight_max": 87.82439422607422,
      "activations/layer7_attention_weight_min": -90.89862060546875,
      "activations/layer8_attention_weight_max": 40.94419479370117,
      "activations/layer8_attention_weight_min": -39.52080154418945,
      "activations/layer9_attention_weight_max": 39.371910095214844,
      "activations/layer9_attention_weight_min": -34.613216400146484,
      "epoch": 6.84,
      "learning_rate": 0.00010697272727272728,
      "loss": 2.8296,
      "step": 117700
    },
    {
      "activations/layer0_attention_weight_max": 15.983800888061523,
      "activations/layer0_attention_weight_min": -12.721617698669434,
      "activations/layer10_attention_weight_max": 31.83976936340332,
      "activations/layer10_attention_weight_min": -31.201805114746094,
      "activations/layer11_attention_weight_max": 32.21272277832031,
      "activations/layer11_attention_weight_min": -30.231321334838867,
      "activations/layer12_attention_weight_max": 21.40897560119629,
      "activations/layer12_attention_weight_min": -30.342512130737305,
      "activations/layer13_attention_weight_max": 38.95192337036133,
      "activations/layer13_attention_weight_min": -33.51919174194336,
      "activations/layer14_attention_weight_max": 53.24571990966797,
      "activations/layer14_attention_weight_min": -35.65636444091797,
      "activations/layer15_attention_weight_max": 41.330543518066406,
      "activations/layer15_attention_weight_min": -29.062543869018555,
      "activations/layer16_attention_weight_max": 34.00691223144531,
      "activations/layer16_attention_weight_min": -26.397342681884766,
      "activations/layer17_attention_weight_max": 36.23439407348633,
      "activations/layer17_attention_weight_min": -24.356708526611328,
      "activations/layer18_attention_weight_max": 32.773223876953125,
      "activations/layer18_attention_weight_min": -22.57149887084961,
      "activations/layer19_attention_weight_max": 37.60752487182617,
      "activations/layer19_attention_weight_min": -29.013935089111328,
      "activations/layer1_attention_weight_max": 17.678972244262695,
      "activations/layer1_attention_weight_min": -14.825878143310547,
      "activations/layer20_attention_weight_max": 32.21187210083008,
      "activations/layer20_attention_weight_min": -23.376585006713867,
      "activations/layer21_attention_weight_max": 30.95775032043457,
      "activations/layer21_attention_weight_min": -22.956192016601562,
      "activations/layer22_attention_weight_max": 47.47514724731445,
      "activations/layer22_attention_weight_min": -27.579689025878906,
      "activations/layer23_attention_weight_max": 34.711822509765625,
      "activations/layer23_attention_weight_min": -23.353065490722656,
      "activations/layer2_attention_weight_max": 31.794109344482422,
      "activations/layer2_attention_weight_min": -34.071990966796875,
      "activations/layer3_attention_weight_max": 88.22859954833984,
      "activations/layer3_attention_weight_min": -90.72381591796875,
      "activations/layer4_attention_weight_max": 123.394287109375,
      "activations/layer4_attention_weight_min": -119.6196060180664,
      "activations/layer5_attention_weight_max": 69.0329818725586,
      "activations/layer5_attention_weight_min": -65.524658203125,
      "activations/layer6_attention_weight_max": 53.02037811279297,
      "activations/layer6_attention_weight_min": -51.93742752075195,
      "activations/layer7_attention_weight_max": 87.1365966796875,
      "activations/layer7_attention_weight_min": -85.53419494628906,
      "activations/layer8_attention_weight_max": 42.35768508911133,
      "activations/layer8_attention_weight_min": -41.99043655395508,
      "activations/layer9_attention_weight_max": 36.8699951171875,
      "activations/layer9_attention_weight_min": -35.46536636352539,
      "epoch": 6.84,
      "learning_rate": 0.00010695378787878786,
      "loss": 2.8169,
      "step": 117750
    },
    {
      "activations/layer0_attention_weight_max": 16.983936309814453,
      "activations/layer0_attention_weight_min": -12.719625473022461,
      "activations/layer10_attention_weight_max": 32.587894439697266,
      "activations/layer10_attention_weight_min": -29.18657112121582,
      "activations/layer11_attention_weight_max": 32.923484802246094,
      "activations/layer11_attention_weight_min": -27.962783813476562,
      "activations/layer12_attention_weight_max": 24.021728515625,
      "activations/layer12_attention_weight_min": -27.673633575439453,
      "activations/layer13_attention_weight_max": 38.04506301879883,
      "activations/layer13_attention_weight_min": -31.734371185302734,
      "activations/layer14_attention_weight_max": 49.17458724975586,
      "activations/layer14_attention_weight_min": -38.46638488769531,
      "activations/layer15_attention_weight_max": 35.8482551574707,
      "activations/layer15_attention_weight_min": -29.15068817138672,
      "activations/layer16_attention_weight_max": 32.479713439941406,
      "activations/layer16_attention_weight_min": -29.955183029174805,
      "activations/layer17_attention_weight_max": 36.507564544677734,
      "activations/layer17_attention_weight_min": -26.943872451782227,
      "activations/layer18_attention_weight_max": 31.027385711669922,
      "activations/layer18_attention_weight_min": -26.14166831970215,
      "activations/layer19_attention_weight_max": 36.650814056396484,
      "activations/layer19_attention_weight_min": -30.763696670532227,
      "activations/layer1_attention_weight_max": 17.079181671142578,
      "activations/layer1_attention_weight_min": -12.07851791381836,
      "activations/layer20_attention_weight_max": 31.227760314941406,
      "activations/layer20_attention_weight_min": -24.142839431762695,
      "activations/layer21_attention_weight_max": 28.872344970703125,
      "activations/layer21_attention_weight_min": -22.446311950683594,
      "activations/layer22_attention_weight_max": 50.04173278808594,
      "activations/layer22_attention_weight_min": -29.331111907958984,
      "activations/layer23_attention_weight_max": 39.074127197265625,
      "activations/layer23_attention_weight_min": -24.654170989990234,
      "activations/layer2_attention_weight_max": 31.916160583496094,
      "activations/layer2_attention_weight_min": -30.15425682067871,
      "activations/layer3_attention_weight_max": 86.0555419921875,
      "activations/layer3_attention_weight_min": -87.30780029296875,
      "activations/layer4_attention_weight_max": 121.92940521240234,
      "activations/layer4_attention_weight_min": -115.56510162353516,
      "activations/layer5_attention_weight_max": 65.6673583984375,
      "activations/layer5_attention_weight_min": -62.09710693359375,
      "activations/layer6_attention_weight_max": 49.954750061035156,
      "activations/layer6_attention_weight_min": -52.41933059692383,
      "activations/layer7_attention_weight_max": 82.81935119628906,
      "activations/layer7_attention_weight_min": -83.66332244873047,
      "activations/layer8_attention_weight_max": 41.088680267333984,
      "activations/layer8_attention_weight_min": -38.179447174072266,
      "activations/layer9_attention_weight_max": 34.659950256347656,
      "activations/layer9_attention_weight_min": -31.66535758972168,
      "epoch": 6.84,
      "learning_rate": 0.00010693484848484848,
      "loss": 2.8371,
      "step": 117800
    },
    {
      "activations/layer0_attention_weight_max": 16.024011611938477,
      "activations/layer0_attention_weight_min": -12.084492683410645,
      "activations/layer10_attention_weight_max": 33.22029113769531,
      "activations/layer10_attention_weight_min": -32.42926788330078,
      "activations/layer11_attention_weight_max": 32.92788314819336,
      "activations/layer11_attention_weight_min": -29.857101440429688,
      "activations/layer12_attention_weight_max": 22.804731369018555,
      "activations/layer12_attention_weight_min": -24.326921463012695,
      "activations/layer13_attention_weight_max": 41.99622344970703,
      "activations/layer13_attention_weight_min": -31.71051025390625,
      "activations/layer14_attention_weight_max": 52.94947814941406,
      "activations/layer14_attention_weight_min": -39.57646560668945,
      "activations/layer15_attention_weight_max": 39.04117202758789,
      "activations/layer15_attention_weight_min": -28.75071907043457,
      "activations/layer16_attention_weight_max": 35.6621208190918,
      "activations/layer16_attention_weight_min": -28.600061416625977,
      "activations/layer17_attention_weight_max": 34.4802131652832,
      "activations/layer17_attention_weight_min": -26.435747146606445,
      "activations/layer18_attention_weight_max": 28.930654525756836,
      "activations/layer18_attention_weight_min": -20.748239517211914,
      "activations/layer19_attention_weight_max": 38.07944869995117,
      "activations/layer19_attention_weight_min": -31.05423355102539,
      "activations/layer1_attention_weight_max": 15.964993476867676,
      "activations/layer1_attention_weight_min": -15.035527229309082,
      "activations/layer20_attention_weight_max": 31.051164627075195,
      "activations/layer20_attention_weight_min": -24.065292358398438,
      "activations/layer21_attention_weight_max": 27.62801170349121,
      "activations/layer21_attention_weight_min": -22.707454681396484,
      "activations/layer22_attention_weight_max": 47.04656982421875,
      "activations/layer22_attention_weight_min": -28.59724235534668,
      "activations/layer23_attention_weight_max": 33.190677642822266,
      "activations/layer23_attention_weight_min": -21.498878479003906,
      "activations/layer2_attention_weight_max": 32.4263801574707,
      "activations/layer2_attention_weight_min": -31.105009078979492,
      "activations/layer3_attention_weight_max": 91.13099670410156,
      "activations/layer3_attention_weight_min": -91.17510986328125,
      "activations/layer4_attention_weight_max": 123.85851287841797,
      "activations/layer4_attention_weight_min": -123.3019790649414,
      "activations/layer5_attention_weight_max": 67.41899871826172,
      "activations/layer5_attention_weight_min": -61.80903625488281,
      "activations/layer6_attention_weight_max": 52.899234771728516,
      "activations/layer6_attention_weight_min": -55.08184814453125,
      "activations/layer7_attention_weight_max": 89.15626525878906,
      "activations/layer7_attention_weight_min": -84.9613037109375,
      "activations/layer8_attention_weight_max": 39.913352966308594,
      "activations/layer8_attention_weight_min": -40.22842788696289,
      "activations/layer9_attention_weight_max": 34.94599533081055,
      "activations/layer9_attention_weight_min": -34.098899841308594,
      "epoch": 6.85,
      "learning_rate": 0.00010691590909090908,
      "loss": 2.8293,
      "step": 117850
    },
    {
      "activations/layer0_attention_weight_max": 16.04724884033203,
      "activations/layer0_attention_weight_min": -11.627798080444336,
      "activations/layer10_attention_weight_max": 33.48855209350586,
      "activations/layer10_attention_weight_min": -35.014503479003906,
      "activations/layer11_attention_weight_max": 33.075172424316406,
      "activations/layer11_attention_weight_min": -29.77699851989746,
      "activations/layer12_attention_weight_max": 22.5603084564209,
      "activations/layer12_attention_weight_min": -25.381784439086914,
      "activations/layer13_attention_weight_max": 41.87837219238281,
      "activations/layer13_attention_weight_min": -34.3831901550293,
      "activations/layer14_attention_weight_max": 58.707889556884766,
      "activations/layer14_attention_weight_min": -45.989959716796875,
      "activations/layer15_attention_weight_max": 36.3534049987793,
      "activations/layer15_attention_weight_min": -29.868539810180664,
      "activations/layer16_attention_weight_max": 33.08977508544922,
      "activations/layer16_attention_weight_min": -28.110790252685547,
      "activations/layer17_attention_weight_max": 39.735958099365234,
      "activations/layer17_attention_weight_min": -24.868009567260742,
      "activations/layer18_attention_weight_max": 33.795284271240234,
      "activations/layer18_attention_weight_min": -21.932331085205078,
      "activations/layer19_attention_weight_max": 36.82204818725586,
      "activations/layer19_attention_weight_min": -30.3629150390625,
      "activations/layer1_attention_weight_max": 16.676069259643555,
      "activations/layer1_attention_weight_min": -14.983261108398438,
      "activations/layer20_attention_weight_max": 33.11949157714844,
      "activations/layer20_attention_weight_min": -26.25417137145996,
      "activations/layer21_attention_weight_max": 29.650320053100586,
      "activations/layer21_attention_weight_min": -22.43109703063965,
      "activations/layer22_attention_weight_max": 51.34308624267578,
      "activations/layer22_attention_weight_min": -28.3533878326416,
      "activations/layer23_attention_weight_max": 36.80509567260742,
      "activations/layer23_attention_weight_min": -23.773094177246094,
      "activations/layer2_attention_weight_max": 33.478981018066406,
      "activations/layer2_attention_weight_min": -31.364980697631836,
      "activations/layer3_attention_weight_max": 89.36992645263672,
      "activations/layer3_attention_weight_min": -89.5821762084961,
      "activations/layer4_attention_weight_max": 124.49761199951172,
      "activations/layer4_attention_weight_min": -126.59310150146484,
      "activations/layer5_attention_weight_max": 65.57435607910156,
      "activations/layer5_attention_weight_min": -67.87748718261719,
      "activations/layer6_attention_weight_max": 52.38512420654297,
      "activations/layer6_attention_weight_min": -53.515987396240234,
      "activations/layer7_attention_weight_max": 88.754638671875,
      "activations/layer7_attention_weight_min": -89.67528533935547,
      "activations/layer8_attention_weight_max": 41.30524826049805,
      "activations/layer8_attention_weight_min": -40.69041442871094,
      "activations/layer9_attention_weight_max": 35.5529670715332,
      "activations/layer9_attention_weight_min": -36.02565383911133,
      "epoch": 6.85,
      "learning_rate": 0.00010689696969696968,
      "loss": 2.827,
      "step": 117900
    },
    {
      "activations/layer0_attention_weight_max": 16.70452117919922,
      "activations/layer0_attention_weight_min": -11.386271476745605,
      "activations/layer10_attention_weight_max": 34.8568000793457,
      "activations/layer10_attention_weight_min": -31.945119857788086,
      "activations/layer11_attention_weight_max": 40.52394104003906,
      "activations/layer11_attention_weight_min": -30.192882537841797,
      "activations/layer12_attention_weight_max": 26.703527450561523,
      "activations/layer12_attention_weight_min": -26.812599182128906,
      "activations/layer13_attention_weight_max": 44.90260314941406,
      "activations/layer13_attention_weight_min": -33.35822296142578,
      "activations/layer14_attention_weight_max": 74.37297821044922,
      "activations/layer14_attention_weight_min": -47.10538101196289,
      "activations/layer15_attention_weight_max": 52.88670349121094,
      "activations/layer15_attention_weight_min": -30.011245727539062,
      "activations/layer16_attention_weight_max": 40.80854415893555,
      "activations/layer16_attention_weight_min": -26.681425094604492,
      "activations/layer17_attention_weight_max": 41.25059127807617,
      "activations/layer17_attention_weight_min": -25.14124298095703,
      "activations/layer18_attention_weight_max": 36.55821990966797,
      "activations/layer18_attention_weight_min": -21.801549911499023,
      "activations/layer19_attention_weight_max": 44.028202056884766,
      "activations/layer19_attention_weight_min": -28.971023559570312,
      "activations/layer1_attention_weight_max": 17.75471305847168,
      "activations/layer1_attention_weight_min": -14.07423210144043,
      "activations/layer20_attention_weight_max": 36.17383575439453,
      "activations/layer20_attention_weight_min": -21.580425262451172,
      "activations/layer21_attention_weight_max": 33.392730712890625,
      "activations/layer21_attention_weight_min": -20.141586303710938,
      "activations/layer22_attention_weight_max": 54.08273696899414,
      "activations/layer22_attention_weight_min": -26.20840072631836,
      "activations/layer23_attention_weight_max": 40.481422424316406,
      "activations/layer23_attention_weight_min": -21.780033111572266,
      "activations/layer2_attention_weight_max": 35.17709732055664,
      "activations/layer2_attention_weight_min": -30.77218246459961,
      "activations/layer3_attention_weight_max": 92.38832092285156,
      "activations/layer3_attention_weight_min": -87.64834594726562,
      "activations/layer4_attention_weight_max": 124.41883850097656,
      "activations/layer4_attention_weight_min": -124.09288787841797,
      "activations/layer5_attention_weight_max": 67.21525573730469,
      "activations/layer5_attention_weight_min": -64.67977905273438,
      "activations/layer6_attention_weight_max": 54.49667739868164,
      "activations/layer6_attention_weight_min": -53.79759216308594,
      "activations/layer7_attention_weight_max": 90.24740600585938,
      "activations/layer7_attention_weight_min": -86.74813079833984,
      "activations/layer8_attention_weight_max": 40.04036331176758,
      "activations/layer8_attention_weight_min": -40.06199264526367,
      "activations/layer9_attention_weight_max": 36.204891204833984,
      "activations/layer9_attention_weight_min": -33.74880599975586,
      "epoch": 6.85,
      "learning_rate": 0.0001068780303030303,
      "loss": 2.8312,
      "step": 117950
    },
    {
      "activations/layer0_attention_weight_max": 16.67267608642578,
      "activations/layer0_attention_weight_min": -12.39096736907959,
      "activations/layer10_attention_weight_max": 33.58244705200195,
      "activations/layer10_attention_weight_min": -31.917057037353516,
      "activations/layer11_attention_weight_max": 37.63528823852539,
      "activations/layer11_attention_weight_min": -29.574848175048828,
      "activations/layer12_attention_weight_max": 31.375186920166016,
      "activations/layer12_attention_weight_min": -27.681745529174805,
      "activations/layer13_attention_weight_max": 44.49720001220703,
      "activations/layer13_attention_weight_min": -32.68842315673828,
      "activations/layer14_attention_weight_max": 55.793033599853516,
      "activations/layer14_attention_weight_min": -43.517276763916016,
      "activations/layer15_attention_weight_max": 42.4151496887207,
      "activations/layer15_attention_weight_min": -30.255512237548828,
      "activations/layer16_attention_weight_max": 37.98764419555664,
      "activations/layer16_attention_weight_min": -28.265743255615234,
      "activations/layer17_attention_weight_max": 39.93803405761719,
      "activations/layer17_attention_weight_min": -27.00052833557129,
      "activations/layer18_attention_weight_max": 31.85833168029785,
      "activations/layer18_attention_weight_min": -21.418357849121094,
      "activations/layer19_attention_weight_max": 46.17047119140625,
      "activations/layer19_attention_weight_min": -32.76091003417969,
      "activations/layer1_attention_weight_max": 16.166072845458984,
      "activations/layer1_attention_weight_min": -13.398454666137695,
      "activations/layer20_attention_weight_max": 39.79293441772461,
      "activations/layer20_attention_weight_min": -26.38556480407715,
      "activations/layer21_attention_weight_max": 32.99861526489258,
      "activations/layer21_attention_weight_min": -23.757854461669922,
      "activations/layer22_attention_weight_max": 51.93196105957031,
      "activations/layer22_attention_weight_min": -29.559961318969727,
      "activations/layer23_attention_weight_max": 37.01573181152344,
      "activations/layer23_attention_weight_min": -23.94049835205078,
      "activations/layer2_attention_weight_max": 31.302045822143555,
      "activations/layer2_attention_weight_min": -31.663631439208984,
      "activations/layer3_attention_weight_max": 88.43378448486328,
      "activations/layer3_attention_weight_min": -89.91118621826172,
      "activations/layer4_attention_weight_max": 123.85204315185547,
      "activations/layer4_attention_weight_min": -128.74171447753906,
      "activations/layer5_attention_weight_max": 68.1810073852539,
      "activations/layer5_attention_weight_min": -67.85995483398438,
      "activations/layer6_attention_weight_max": 51.74922180175781,
      "activations/layer6_attention_weight_min": -54.30543518066406,
      "activations/layer7_attention_weight_max": 86.75111389160156,
      "activations/layer7_attention_weight_min": -87.66234588623047,
      "activations/layer8_attention_weight_max": 40.86168670654297,
      "activations/layer8_attention_weight_min": -37.66693878173828,
      "activations/layer9_attention_weight_max": 32.9232177734375,
      "activations/layer9_attention_weight_min": -31.907316207885742,
      "epoch": 6.86,
      "learning_rate": 0.0001068590909090909,
      "loss": 2.8288,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.5365,
      "eval_samples_per_second": 503.019,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.5365,
      "eval_openwebtext_samples_per_second": 503.019,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_wikitext_loss": 3.005859375,
      "eval_wikitext_ppl": 20.203571081597193,
      "eval_wikitext_runtime": 1.9945,
      "eval_wikitext_samples_per_second": 228.634,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_lambada_loss": 2.681640625,
      "eval_lambada_ppl": 14.609041604294383,
      "eval_lambada_runtime": 9.6425,
      "eval_lambada_samples_per_second": 504.95,
      "step": 118000
    },
    {
      "activations/layer0_attention_weight_max": 15.359734535217285,
      "activations/layer0_attention_weight_min": -11.829652786254883,
      "activations/layer10_attention_weight_max": 31.10626220703125,
      "activations/layer10_attention_weight_min": -30.3077392578125,
      "activations/layer11_attention_weight_max": 34.90193176269531,
      "activations/layer11_attention_weight_min": -29.341941833496094,
      "activations/layer12_attention_weight_max": 23.70548439025879,
      "activations/layer12_attention_weight_min": -23.768903732299805,
      "activations/layer13_attention_weight_max": 40.284339904785156,
      "activations/layer13_attention_weight_min": -31.831890106201172,
      "activations/layer14_attention_weight_max": 49.250423431396484,
      "activations/layer14_attention_weight_min": -37.47621154785156,
      "activations/layer15_attention_weight_max": 37.089500427246094,
      "activations/layer15_attention_weight_min": -28.094022750854492,
      "activations/layer16_attention_weight_max": 34.07682800292969,
      "activations/layer16_attention_weight_min": -29.113052368164062,
      "activations/layer17_attention_weight_max": 36.023887634277344,
      "activations/layer17_attention_weight_min": -26.35102653503418,
      "activations/layer18_attention_weight_max": 30.15603256225586,
      "activations/layer18_attention_weight_min": -20.59886360168457,
      "activations/layer19_attention_weight_max": 38.29973220825195,
      "activations/layer19_attention_weight_min": -33.6753044128418,
      "activations/layer1_attention_weight_max": 17.796424865722656,
      "activations/layer1_attention_weight_min": -13.452739715576172,
      "activations/layer20_attention_weight_max": 31.694961547851562,
      "activations/layer20_attention_weight_min": -24.740673065185547,
      "activations/layer21_attention_weight_max": 31.688817977905273,
      "activations/layer21_attention_weight_min": -20.920284271240234,
      "activations/layer22_attention_weight_max": 45.86201477050781,
      "activations/layer22_attention_weight_min": -27.823137283325195,
      "activations/layer23_attention_weight_max": 36.129600524902344,
      "activations/layer23_attention_weight_min": -23.172056198120117,
      "activations/layer2_attention_weight_max": 31.44257164001465,
      "activations/layer2_attention_weight_min": -31.08063507080078,
      "activations/layer3_attention_weight_max": 94.06776428222656,
      "activations/layer3_attention_weight_min": -92.3778076171875,
      "activations/layer4_attention_weight_max": 132.37937927246094,
      "activations/layer4_attention_weight_min": -123.4010238647461,
      "activations/layer5_attention_weight_max": 69.92950439453125,
      "activations/layer5_attention_weight_min": -68.24100494384766,
      "activations/layer6_attention_weight_max": 52.418678283691406,
      "activations/layer6_attention_weight_min": -54.344093322753906,
      "activations/layer7_attention_weight_max": 89.28679656982422,
      "activations/layer7_attention_weight_min": -90.64016723632812,
      "activations/layer8_attention_weight_max": 38.75502014160156,
      "activations/layer8_attention_weight_min": -41.717979431152344,
      "activations/layer9_attention_weight_max": 35.51797866821289,
      "activations/layer9_attention_weight_min": -34.38840103149414,
      "epoch": 6.86,
      "learning_rate": 0.0001068401515151515,
      "loss": 2.8047,
      "step": 118050
    },
    {
      "activations/layer0_attention_weight_max": 16.126693725585938,
      "activations/layer0_attention_weight_min": -11.884613037109375,
      "activations/layer10_attention_weight_max": 36.912567138671875,
      "activations/layer10_attention_weight_min": -31.19590950012207,
      "activations/layer11_attention_weight_max": 37.53174591064453,
      "activations/layer11_attention_weight_min": -30.193893432617188,
      "activations/layer12_attention_weight_max": 24.924747467041016,
      "activations/layer12_attention_weight_min": -26.68979835510254,
      "activations/layer13_attention_weight_max": 41.863685607910156,
      "activations/layer13_attention_weight_min": -31.601852416992188,
      "activations/layer14_attention_weight_max": 57.36258316040039,
      "activations/layer14_attention_weight_min": -43.074371337890625,
      "activations/layer15_attention_weight_max": 38.939666748046875,
      "activations/layer15_attention_weight_min": -30.234283447265625,
      "activations/layer16_attention_weight_max": 39.98102569580078,
      "activations/layer16_attention_weight_min": -29.329004287719727,
      "activations/layer17_attention_weight_max": 42.56281280517578,
      "activations/layer17_attention_weight_min": -23.984060287475586,
      "activations/layer18_attention_weight_max": 36.3819465637207,
      "activations/layer18_attention_weight_min": -21.13705062866211,
      "activations/layer19_attention_weight_max": 43.058536529541016,
      "activations/layer19_attention_weight_min": -30.037586212158203,
      "activations/layer1_attention_weight_max": 17.061742782592773,
      "activations/layer1_attention_weight_min": -14.715548515319824,
      "activations/layer20_attention_weight_max": 31.669143676757812,
      "activations/layer20_attention_weight_min": -24.39103889465332,
      "activations/layer21_attention_weight_max": 29.29490089416504,
      "activations/layer21_attention_weight_min": -22.710973739624023,
      "activations/layer22_attention_weight_max": 45.97856521606445,
      "activations/layer22_attention_weight_min": -28.81842613220215,
      "activations/layer23_attention_weight_max": 36.86800003051758,
      "activations/layer23_attention_weight_min": -22.757774353027344,
      "activations/layer2_attention_weight_max": 33.04254913330078,
      "activations/layer2_attention_weight_min": -34.070316314697266,
      "activations/layer3_attention_weight_max": 94.10489654541016,
      "activations/layer3_attention_weight_min": -95.46780395507812,
      "activations/layer4_attention_weight_max": 125.51234436035156,
      "activations/layer4_attention_weight_min": -126.638671875,
      "activations/layer5_attention_weight_max": 64.44216918945312,
      "activations/layer5_attention_weight_min": -64.88713073730469,
      "activations/layer6_attention_weight_max": 52.22307205200195,
      "activations/layer6_attention_weight_min": -54.13960266113281,
      "activations/layer7_attention_weight_max": 88.32630920410156,
      "activations/layer7_attention_weight_min": -92.78008270263672,
      "activations/layer8_attention_weight_max": 39.879051208496094,
      "activations/layer8_attention_weight_min": -40.292293548583984,
      "activations/layer9_attention_weight_max": 36.08768844604492,
      "activations/layer9_attention_weight_min": -34.71233367919922,
      "epoch": 6.86,
      "learning_rate": 0.00010682121212121211,
      "loss": 2.8103,
      "step": 118100
    },
    {
      "activations/layer0_attention_weight_max": 16.2901611328125,
      "activations/layer0_attention_weight_min": -11.418405532836914,
      "activations/layer10_attention_weight_max": 31.96122932434082,
      "activations/layer10_attention_weight_min": -29.865060806274414,
      "activations/layer11_attention_weight_max": 31.15212059020996,
      "activations/layer11_attention_weight_min": -28.23802947998047,
      "activations/layer12_attention_weight_max": 19.82084083557129,
      "activations/layer12_attention_weight_min": -29.738719940185547,
      "activations/layer13_attention_weight_max": 37.93185043334961,
      "activations/layer13_attention_weight_min": -31.004222869873047,
      "activations/layer14_attention_weight_max": 49.91207504272461,
      "activations/layer14_attention_weight_min": -41.71382141113281,
      "activations/layer15_attention_weight_max": 33.998565673828125,
      "activations/layer15_attention_weight_min": -27.56643295288086,
      "activations/layer16_attention_weight_max": 32.01127243041992,
      "activations/layer16_attention_weight_min": -27.098039627075195,
      "activations/layer17_attention_weight_max": 33.61967086791992,
      "activations/layer17_attention_weight_min": -23.600845336914062,
      "activations/layer18_attention_weight_max": 29.305814743041992,
      "activations/layer18_attention_weight_min": -19.92595863342285,
      "activations/layer19_attention_weight_max": 40.161556243896484,
      "activations/layer19_attention_weight_min": -32.829891204833984,
      "activations/layer1_attention_weight_max": 17.106090545654297,
      "activations/layer1_attention_weight_min": -14.70589542388916,
      "activations/layer20_attention_weight_max": 30.31390953063965,
      "activations/layer20_attention_weight_min": -22.75194549560547,
      "activations/layer21_attention_weight_max": 27.89495086669922,
      "activations/layer21_attention_weight_min": -20.70469093322754,
      "activations/layer22_attention_weight_max": 44.52033233642578,
      "activations/layer22_attention_weight_min": -29.086931228637695,
      "activations/layer23_attention_weight_max": 34.91651153564453,
      "activations/layer23_attention_weight_min": -21.656177520751953,
      "activations/layer2_attention_weight_max": 32.73426818847656,
      "activations/layer2_attention_weight_min": -32.20925521850586,
      "activations/layer3_attention_weight_max": 95.2696533203125,
      "activations/layer3_attention_weight_min": -95.29351806640625,
      "activations/layer4_attention_weight_max": 126.93395233154297,
      "activations/layer4_attention_weight_min": -123.0144271850586,
      "activations/layer5_attention_weight_max": 65.85689544677734,
      "activations/layer5_attention_weight_min": -63.30129623413086,
      "activations/layer6_attention_weight_max": 50.74216842651367,
      "activations/layer6_attention_weight_min": -52.04033660888672,
      "activations/layer7_attention_weight_max": 83.37025451660156,
      "activations/layer7_attention_weight_min": -90.72450256347656,
      "activations/layer8_attention_weight_max": 40.62114334106445,
      "activations/layer8_attention_weight_min": -36.785282135009766,
      "activations/layer9_attention_weight_max": 33.3112678527832,
      "activations/layer9_attention_weight_min": -32.772151947021484,
      "epoch": 6.87,
      "learning_rate": 0.00010680227272727272,
      "loss": 2.8296,
      "step": 118150
    },
    {
      "activations/layer0_attention_weight_max": 15.827958106994629,
      "activations/layer0_attention_weight_min": -12.596207618713379,
      "activations/layer10_attention_weight_max": 31.38792610168457,
      "activations/layer10_attention_weight_min": -31.000701904296875,
      "activations/layer11_attention_weight_max": 30.35638427734375,
      "activations/layer11_attention_weight_min": -29.41432762145996,
      "activations/layer12_attention_weight_max": 24.372379302978516,
      "activations/layer12_attention_weight_min": -30.00210189819336,
      "activations/layer13_attention_weight_max": 39.367225646972656,
      "activations/layer13_attention_weight_min": -31.20391273498535,
      "activations/layer14_attention_weight_max": 53.768672943115234,
      "activations/layer14_attention_weight_min": -36.89948654174805,
      "activations/layer15_attention_weight_max": 34.59397506713867,
      "activations/layer15_attention_weight_min": -28.693742752075195,
      "activations/layer16_attention_weight_max": 34.02695083618164,
      "activations/layer16_attention_weight_min": -29.690786361694336,
      "activations/layer17_attention_weight_max": 34.31604766845703,
      "activations/layer17_attention_weight_min": -24.333036422729492,
      "activations/layer18_attention_weight_max": 36.402748107910156,
      "activations/layer18_attention_weight_min": -21.23758316040039,
      "activations/layer19_attention_weight_max": 36.97883224487305,
      "activations/layer19_attention_weight_min": -30.598785400390625,
      "activations/layer1_attention_weight_max": 17.35918617248535,
      "activations/layer1_attention_weight_min": -13.993584632873535,
      "activations/layer20_attention_weight_max": 30.13139533996582,
      "activations/layer20_attention_weight_min": -24.312467575073242,
      "activations/layer21_attention_weight_max": 28.1462459564209,
      "activations/layer21_attention_weight_min": -20.997339248657227,
      "activations/layer22_attention_weight_max": 46.05833435058594,
      "activations/layer22_attention_weight_min": -27.571516036987305,
      "activations/layer23_attention_weight_max": 37.39020538330078,
      "activations/layer23_attention_weight_min": -23.25752067565918,
      "activations/layer2_attention_weight_max": 32.1287841796875,
      "activations/layer2_attention_weight_min": -31.414844512939453,
      "activations/layer3_attention_weight_max": 90.3309097290039,
      "activations/layer3_attention_weight_min": -94.50518798828125,
      "activations/layer4_attention_weight_max": 122.0645523071289,
      "activations/layer4_attention_weight_min": -121.60208892822266,
      "activations/layer5_attention_weight_max": 67.62887573242188,
      "activations/layer5_attention_weight_min": -64.84989166259766,
      "activations/layer6_attention_weight_max": 50.762596130371094,
      "activations/layer6_attention_weight_min": -53.00942611694336,
      "activations/layer7_attention_weight_max": 89.47832489013672,
      "activations/layer7_attention_weight_min": -81.84001159667969,
      "activations/layer8_attention_weight_max": 40.60377883911133,
      "activations/layer8_attention_weight_min": -40.60391616821289,
      "activations/layer9_attention_weight_max": 35.383033752441406,
      "activations/layer9_attention_weight_min": -36.30276870727539,
      "epoch": 6.87,
      "learning_rate": 0.00010678333333333333,
      "loss": 2.8216,
      "step": 118200
    },
    {
      "activations/layer0_attention_weight_max": 15.464263916015625,
      "activations/layer0_attention_weight_min": -11.25308609008789,
      "activations/layer10_attention_weight_max": 33.13254928588867,
      "activations/layer10_attention_weight_min": -31.83054542541504,
      "activations/layer11_attention_weight_max": 32.34282684326172,
      "activations/layer11_attention_weight_min": -29.287023544311523,
      "activations/layer12_attention_weight_max": 24.150583267211914,
      "activations/layer12_attention_weight_min": -32.63618850708008,
      "activations/layer13_attention_weight_max": 41.80784225463867,
      "activations/layer13_attention_weight_min": -30.583415985107422,
      "activations/layer14_attention_weight_max": 53.357749938964844,
      "activations/layer14_attention_weight_min": -39.080326080322266,
      "activations/layer15_attention_weight_max": 38.131431579589844,
      "activations/layer15_attention_weight_min": -27.80658721923828,
      "activations/layer16_attention_weight_max": 35.62358856201172,
      "activations/layer16_attention_weight_min": -26.927461624145508,
      "activations/layer17_attention_weight_max": 37.45954895019531,
      "activations/layer17_attention_weight_min": -25.31949234008789,
      "activations/layer18_attention_weight_max": 31.925607681274414,
      "activations/layer18_attention_weight_min": -20.89507484436035,
      "activations/layer19_attention_weight_max": 40.93021011352539,
      "activations/layer19_attention_weight_min": -29.5408992767334,
      "activations/layer1_attention_weight_max": 17.444265365600586,
      "activations/layer1_attention_weight_min": -13.866378784179688,
      "activations/layer20_attention_weight_max": 33.4518928527832,
      "activations/layer20_attention_weight_min": -24.11919403076172,
      "activations/layer21_attention_weight_max": 34.91261291503906,
      "activations/layer21_attention_weight_min": -20.86179542541504,
      "activations/layer22_attention_weight_max": 51.912742614746094,
      "activations/layer22_attention_weight_min": -29.122522354125977,
      "activations/layer23_attention_weight_max": 41.470333099365234,
      "activations/layer23_attention_weight_min": -21.693464279174805,
      "activations/layer2_attention_weight_max": 32.137935638427734,
      "activations/layer2_attention_weight_min": -31.690959930419922,
      "activations/layer3_attention_weight_max": 90.76243591308594,
      "activations/layer3_attention_weight_min": -93.22830963134766,
      "activations/layer4_attention_weight_max": 125.73994445800781,
      "activations/layer4_attention_weight_min": -118.84889221191406,
      "activations/layer5_attention_weight_max": 67.43572235107422,
      "activations/layer5_attention_weight_min": -62.744895935058594,
      "activations/layer6_attention_weight_max": 54.74241256713867,
      "activations/layer6_attention_weight_min": -51.68103790283203,
      "activations/layer7_attention_weight_max": 92.5337905883789,
      "activations/layer7_attention_weight_min": -85.61756896972656,
      "activations/layer8_attention_weight_max": 39.95949935913086,
      "activations/layer8_attention_weight_min": -40.525367736816406,
      "activations/layer9_attention_weight_max": 37.43568801879883,
      "activations/layer9_attention_weight_min": -36.67551040649414,
      "epoch": 6.87,
      "learning_rate": 0.00010676439393939392,
      "loss": 2.8416,
      "step": 118250
    },
    {
      "activations/layer0_attention_weight_max": 15.862028121948242,
      "activations/layer0_attention_weight_min": -12.025907516479492,
      "activations/layer10_attention_weight_max": 31.627422332763672,
      "activations/layer10_attention_weight_min": -29.523632049560547,
      "activations/layer11_attention_weight_max": 30.19033432006836,
      "activations/layer11_attention_weight_min": -29.091026306152344,
      "activations/layer12_attention_weight_max": 21.613309860229492,
      "activations/layer12_attention_weight_min": -25.041824340820312,
      "activations/layer13_attention_weight_max": 38.528953552246094,
      "activations/layer13_attention_weight_min": -31.836687088012695,
      "activations/layer14_attention_weight_max": 53.905513763427734,
      "activations/layer14_attention_weight_min": -45.25156021118164,
      "activations/layer15_attention_weight_max": 36.75153732299805,
      "activations/layer15_attention_weight_min": -29.50379180908203,
      "activations/layer16_attention_weight_max": 32.033775329589844,
      "activations/layer16_attention_weight_min": -28.697797775268555,
      "activations/layer17_attention_weight_max": 37.20085906982422,
      "activations/layer17_attention_weight_min": -25.030309677124023,
      "activations/layer18_attention_weight_max": 34.06243133544922,
      "activations/layer18_attention_weight_min": -20.75309944152832,
      "activations/layer19_attention_weight_max": 37.32780075073242,
      "activations/layer19_attention_weight_min": -32.5909538269043,
      "activations/layer1_attention_weight_max": 17.08919334411621,
      "activations/layer1_attention_weight_min": -15.398029327392578,
      "activations/layer20_attention_weight_max": 33.450191497802734,
      "activations/layer20_attention_weight_min": -26.59914207458496,
      "activations/layer21_attention_weight_max": 37.15486145019531,
      "activations/layer21_attention_weight_min": -23.70352554321289,
      "activations/layer22_attention_weight_max": 49.1117057800293,
      "activations/layer22_attention_weight_min": -26.1231632232666,
      "activations/layer23_attention_weight_max": 35.85935974121094,
      "activations/layer23_attention_weight_min": -22.541744232177734,
      "activations/layer2_attention_weight_max": 32.04528045654297,
      "activations/layer2_attention_weight_min": -29.891202926635742,
      "activations/layer3_attention_weight_max": 88.86691284179688,
      "activations/layer3_attention_weight_min": -89.29508209228516,
      "activations/layer4_attention_weight_max": 120.69923400878906,
      "activations/layer4_attention_weight_min": -116.5744400024414,
      "activations/layer5_attention_weight_max": 64.22463989257812,
      "activations/layer5_attention_weight_min": -61.82311248779297,
      "activations/layer6_attention_weight_max": 53.740299224853516,
      "activations/layer6_attention_weight_min": -50.99917984008789,
      "activations/layer7_attention_weight_max": 86.95491790771484,
      "activations/layer7_attention_weight_min": -85.46820831298828,
      "activations/layer8_attention_weight_max": 38.236244201660156,
      "activations/layer8_attention_weight_min": -38.770084381103516,
      "activations/layer9_attention_weight_max": 34.292564392089844,
      "activations/layer9_attention_weight_min": -33.337013244628906,
      "epoch": 6.87,
      "learning_rate": 0.00010674545454545453,
      "loss": 2.8325,
      "step": 118300
    },
    {
      "activations/layer0_attention_weight_max": 15.584089279174805,
      "activations/layer0_attention_weight_min": -10.81161880493164,
      "activations/layer10_attention_weight_max": 34.30778121948242,
      "activations/layer10_attention_weight_min": -32.290618896484375,
      "activations/layer11_attention_weight_max": 33.21617889404297,
      "activations/layer11_attention_weight_min": -31.004926681518555,
      "activations/layer12_attention_weight_max": 36.02749252319336,
      "activations/layer12_attention_weight_min": -26.552486419677734,
      "activations/layer13_attention_weight_max": 43.55487823486328,
      "activations/layer13_attention_weight_min": -34.27936935424805,
      "activations/layer14_attention_weight_max": 58.54274368286133,
      "activations/layer14_attention_weight_min": -40.631412506103516,
      "activations/layer15_attention_weight_max": 38.89614486694336,
      "activations/layer15_attention_weight_min": -29.457622528076172,
      "activations/layer16_attention_weight_max": 37.497074127197266,
      "activations/layer16_attention_weight_min": -26.87533950805664,
      "activations/layer17_attention_weight_max": 37.214012145996094,
      "activations/layer17_attention_weight_min": -24.978776931762695,
      "activations/layer18_attention_weight_max": 30.187589645385742,
      "activations/layer18_attention_weight_min": -22.27153205871582,
      "activations/layer19_attention_weight_max": 38.983177185058594,
      "activations/layer19_attention_weight_min": -32.535701751708984,
      "activations/layer1_attention_weight_max": 16.61969566345215,
      "activations/layer1_attention_weight_min": -14.055794715881348,
      "activations/layer20_attention_weight_max": 33.551109313964844,
      "activations/layer20_attention_weight_min": -26.27907943725586,
      "activations/layer21_attention_weight_max": 32.0749397277832,
      "activations/layer21_attention_weight_min": -25.017112731933594,
      "activations/layer22_attention_weight_max": 47.4510383605957,
      "activations/layer22_attention_weight_min": -30.30773162841797,
      "activations/layer23_attention_weight_max": 34.23768997192383,
      "activations/layer23_attention_weight_min": -24.32440948486328,
      "activations/layer2_attention_weight_max": 32.62784957885742,
      "activations/layer2_attention_weight_min": -29.93067741394043,
      "activations/layer3_attention_weight_max": 88.6533203125,
      "activations/layer3_attention_weight_min": -93.00597381591797,
      "activations/layer4_attention_weight_max": 123.72759246826172,
      "activations/layer4_attention_weight_min": -121.03633117675781,
      "activations/layer5_attention_weight_max": 68.17158508300781,
      "activations/layer5_attention_weight_min": -63.133445739746094,
      "activations/layer6_attention_weight_max": 50.7414436340332,
      "activations/layer6_attention_weight_min": -51.740840911865234,
      "activations/layer7_attention_weight_max": 88.86500549316406,
      "activations/layer7_attention_weight_min": -87.61864471435547,
      "activations/layer8_attention_weight_max": 40.0245246887207,
      "activations/layer8_attention_weight_min": -38.56158447265625,
      "activations/layer9_attention_weight_max": 36.08176040649414,
      "activations/layer9_attention_weight_min": -34.59351348876953,
      "epoch": 6.88,
      "learning_rate": 0.00010672651515151515,
      "loss": 2.8294,
      "step": 118350
    },
    {
      "activations/layer0_attention_weight_max": 16.19314956665039,
      "activations/layer0_attention_weight_min": -12.157683372497559,
      "activations/layer10_attention_weight_max": 30.021038055419922,
      "activations/layer10_attention_weight_min": -31.02144432067871,
      "activations/layer11_attention_weight_max": 31.028385162353516,
      "activations/layer11_attention_weight_min": -27.79543113708496,
      "activations/layer12_attention_weight_max": 26.86944007873535,
      "activations/layer12_attention_weight_min": -25.613101959228516,
      "activations/layer13_attention_weight_max": 39.97700119018555,
      "activations/layer13_attention_weight_min": -33.30094528198242,
      "activations/layer14_attention_weight_max": 66.29147338867188,
      "activations/layer14_attention_weight_min": -47.325828552246094,
      "activations/layer15_attention_weight_max": 33.49890899658203,
      "activations/layer15_attention_weight_min": -28.692279815673828,
      "activations/layer16_attention_weight_max": 31.619983673095703,
      "activations/layer16_attention_weight_min": -27.34371566772461,
      "activations/layer17_attention_weight_max": 33.26378631591797,
      "activations/layer17_attention_weight_min": -23.796279907226562,
      "activations/layer18_attention_weight_max": 27.307443618774414,
      "activations/layer18_attention_weight_min": -21.030553817749023,
      "activations/layer19_attention_weight_max": 36.9137077331543,
      "activations/layer19_attention_weight_min": -30.146100997924805,
      "activations/layer1_attention_weight_max": 17.300880432128906,
      "activations/layer1_attention_weight_min": -12.26075553894043,
      "activations/layer20_attention_weight_max": 31.601818084716797,
      "activations/layer20_attention_weight_min": -24.92975425720215,
      "activations/layer21_attention_weight_max": 31.424583435058594,
      "activations/layer21_attention_weight_min": -23.68549156188965,
      "activations/layer22_attention_weight_max": 47.178680419921875,
      "activations/layer22_attention_weight_min": -29.715377807617188,
      "activations/layer23_attention_weight_max": 33.55870056152344,
      "activations/layer23_attention_weight_min": -22.161487579345703,
      "activations/layer2_attention_weight_max": 31.826215744018555,
      "activations/layer2_attention_weight_min": -29.275869369506836,
      "activations/layer3_attention_weight_max": 85.9646987915039,
      "activations/layer3_attention_weight_min": -87.38772583007812,
      "activations/layer4_attention_weight_max": 123.62554168701172,
      "activations/layer4_attention_weight_min": -121.35621643066406,
      "activations/layer5_attention_weight_max": 64.33169555664062,
      "activations/layer5_attention_weight_min": -65.64320373535156,
      "activations/layer6_attention_weight_max": 53.653438568115234,
      "activations/layer6_attention_weight_min": -50.604305267333984,
      "activations/layer7_attention_weight_max": 90.0821304321289,
      "activations/layer7_attention_weight_min": -86.24032592773438,
      "activations/layer8_attention_weight_max": 39.188865661621094,
      "activations/layer8_attention_weight_min": -39.66290283203125,
      "activations/layer9_attention_weight_max": 33.817970275878906,
      "activations/layer9_attention_weight_min": -33.227970123291016,
      "epoch": 6.88,
      "learning_rate": 0.00010670757575757574,
      "loss": 2.8265,
      "step": 118400
    },
    {
      "activations/layer0_attention_weight_max": 16.539453506469727,
      "activations/layer0_attention_weight_min": -12.40725040435791,
      "activations/layer10_attention_weight_max": 35.315956115722656,
      "activations/layer10_attention_weight_min": -31.403072357177734,
      "activations/layer11_attention_weight_max": 36.24412536621094,
      "activations/layer11_attention_weight_min": -29.827125549316406,
      "activations/layer12_attention_weight_max": 41.413509368896484,
      "activations/layer12_attention_weight_min": -26.05681800842285,
      "activations/layer13_attention_weight_max": 42.81409454345703,
      "activations/layer13_attention_weight_min": -32.46894073486328,
      "activations/layer14_attention_weight_max": 56.41462326049805,
      "activations/layer14_attention_weight_min": -39.08700180053711,
      "activations/layer15_attention_weight_max": 36.952396392822266,
      "activations/layer15_attention_weight_min": -28.820873260498047,
      "activations/layer16_attention_weight_max": 34.60685729980469,
      "activations/layer16_attention_weight_min": -27.07236671447754,
      "activations/layer17_attention_weight_max": 44.987178802490234,
      "activations/layer17_attention_weight_min": -24.34219741821289,
      "activations/layer18_attention_weight_max": 37.732269287109375,
      "activations/layer18_attention_weight_min": -21.652544021606445,
      "activations/layer19_attention_weight_max": 43.77045822143555,
      "activations/layer19_attention_weight_min": -30.721378326416016,
      "activations/layer1_attention_weight_max": 17.398696899414062,
      "activations/layer1_attention_weight_min": -13.360055923461914,
      "activations/layer20_attention_weight_max": 35.26425552368164,
      "activations/layer20_attention_weight_min": -25.600744247436523,
      "activations/layer21_attention_weight_max": 35.024818420410156,
      "activations/layer21_attention_weight_min": -22.64850616455078,
      "activations/layer22_attention_weight_max": 57.44376754760742,
      "activations/layer22_attention_weight_min": -32.2142448425293,
      "activations/layer23_attention_weight_max": 44.19738006591797,
      "activations/layer23_attention_weight_min": -22.182369232177734,
      "activations/layer2_attention_weight_max": 28.222883224487305,
      "activations/layer2_attention_weight_min": -29.309528350830078,
      "activations/layer3_attention_weight_max": 83.22445678710938,
      "activations/layer3_attention_weight_min": -83.0002670288086,
      "activations/layer4_attention_weight_max": 122.69664764404297,
      "activations/layer4_attention_weight_min": -113.07498931884766,
      "activations/layer5_attention_weight_max": 61.633445739746094,
      "activations/layer5_attention_weight_min": -60.600929260253906,
      "activations/layer6_attention_weight_max": 49.967227935791016,
      "activations/layer6_attention_weight_min": -50.22739028930664,
      "activations/layer7_attention_weight_max": 87.14929962158203,
      "activations/layer7_attention_weight_min": -87.74199676513672,
      "activations/layer8_attention_weight_max": 43.35493850708008,
      "activations/layer8_attention_weight_min": -45.04530715942383,
      "activations/layer9_attention_weight_max": 40.39509963989258,
      "activations/layer9_attention_weight_min": -36.55654525756836,
      "epoch": 6.88,
      "learning_rate": 0.00010668863636363635,
      "loss": 2.8226,
      "step": 118450
    },
    {
      "activations/layer0_attention_weight_max": 15.03672981262207,
      "activations/layer0_attention_weight_min": -12.964064598083496,
      "activations/layer10_attention_weight_max": 30.595409393310547,
      "activations/layer10_attention_weight_min": -30.992355346679688,
      "activations/layer11_attention_weight_max": 31.840803146362305,
      "activations/layer11_attention_weight_min": -27.493261337280273,
      "activations/layer12_attention_weight_max": 28.35654067993164,
      "activations/layer12_attention_weight_min": -30.01002311706543,
      "activations/layer13_attention_weight_max": 37.67717361450195,
      "activations/layer13_attention_weight_min": -33.41938781738281,
      "activations/layer14_attention_weight_max": 50.10684585571289,
      "activations/layer14_attention_weight_min": -38.69649887084961,
      "activations/layer15_attention_weight_max": 36.22268295288086,
      "activations/layer15_attention_weight_min": -28.373027801513672,
      "activations/layer16_attention_weight_max": 34.39834213256836,
      "activations/layer16_attention_weight_min": -28.939661026000977,
      "activations/layer17_attention_weight_max": 38.04926681518555,
      "activations/layer17_attention_weight_min": -25.769433975219727,
      "activations/layer18_attention_weight_max": 33.998443603515625,
      "activations/layer18_attention_weight_min": -21.782032012939453,
      "activations/layer19_attention_weight_max": 38.686370849609375,
      "activations/layer19_attention_weight_min": -31.498498916625977,
      "activations/layer1_attention_weight_max": 17.34658432006836,
      "activations/layer1_attention_weight_min": -12.906329154968262,
      "activations/layer20_attention_weight_max": 33.70452117919922,
      "activations/layer20_attention_weight_min": -24.25468635559082,
      "activations/layer21_attention_weight_max": 34.7103271484375,
      "activations/layer21_attention_weight_min": -24.508214950561523,
      "activations/layer22_attention_weight_max": 58.162254333496094,
      "activations/layer22_attention_weight_min": -28.902002334594727,
      "activations/layer23_attention_weight_max": 38.196102142333984,
      "activations/layer23_attention_weight_min": -22.787687301635742,
      "activations/layer2_attention_weight_max": 31.69650650024414,
      "activations/layer2_attention_weight_min": -31.3095703125,
      "activations/layer3_attention_weight_max": 90.70550537109375,
      "activations/layer3_attention_weight_min": -94.43287658691406,
      "activations/layer4_attention_weight_max": 120.3134765625,
      "activations/layer4_attention_weight_min": -126.31549072265625,
      "activations/layer5_attention_weight_max": 64.43850708007812,
      "activations/layer5_attention_weight_min": -62.28630065917969,
      "activations/layer6_attention_weight_max": 50.87215042114258,
      "activations/layer6_attention_weight_min": -51.59209060668945,
      "activations/layer7_attention_weight_max": 88.60050964355469,
      "activations/layer7_attention_weight_min": -83.78874206542969,
      "activations/layer8_attention_weight_max": 39.357582092285156,
      "activations/layer8_attention_weight_min": -39.071746826171875,
      "activations/layer9_attention_weight_max": 33.960487365722656,
      "activations/layer9_attention_weight_min": -33.02421188354492,
      "epoch": 6.89,
      "learning_rate": 0.00010667007575757574,
      "loss": 2.8157,
      "step": 118500
    },
    {
      "activations/layer0_attention_weight_max": 16.43558692932129,
      "activations/layer0_attention_weight_min": -11.4390869140625,
      "activations/layer10_attention_weight_max": 34.34828186035156,
      "activations/layer10_attention_weight_min": -31.419902801513672,
      "activations/layer11_attention_weight_max": 36.59892272949219,
      "activations/layer11_attention_weight_min": -29.81714630126953,
      "activations/layer12_attention_weight_max": 28.274782180786133,
      "activations/layer12_attention_weight_min": -29.564897537231445,
      "activations/layer13_attention_weight_max": 41.05287551879883,
      "activations/layer13_attention_weight_min": -33.467411041259766,
      "activations/layer14_attention_weight_max": 64.61631774902344,
      "activations/layer14_attention_weight_min": -40.842750549316406,
      "activations/layer15_attention_weight_max": 41.91078567504883,
      "activations/layer15_attention_weight_min": -29.565391540527344,
      "activations/layer16_attention_weight_max": 40.48594284057617,
      "activations/layer16_attention_weight_min": -29.407285690307617,
      "activations/layer17_attention_weight_max": 37.54512405395508,
      "activations/layer17_attention_weight_min": -24.67547035217285,
      "activations/layer18_attention_weight_max": 33.926475524902344,
      "activations/layer18_attention_weight_min": -21.520986557006836,
      "activations/layer19_attention_weight_max": 38.0183219909668,
      "activations/layer19_attention_weight_min": -29.7303524017334,
      "activations/layer1_attention_weight_max": 16.57622528076172,
      "activations/layer1_attention_weight_min": -13.024637222290039,
      "activations/layer20_attention_weight_max": 31.56523895263672,
      "activations/layer20_attention_weight_min": -22.925308227539062,
      "activations/layer21_attention_weight_max": 31.67140769958496,
      "activations/layer21_attention_weight_min": -22.238983154296875,
      "activations/layer22_attention_weight_max": 56.99452209472656,
      "activations/layer22_attention_weight_min": -27.805845260620117,
      "activations/layer23_attention_weight_max": 39.95774841308594,
      "activations/layer23_attention_weight_min": -21.94354820251465,
      "activations/layer2_attention_weight_max": 31.38089370727539,
      "activations/layer2_attention_weight_min": -31.137815475463867,
      "activations/layer3_attention_weight_max": 89.73612213134766,
      "activations/layer3_attention_weight_min": -91.70035552978516,
      "activations/layer4_attention_weight_max": 125.7698745727539,
      "activations/layer4_attention_weight_min": -129.26658630371094,
      "activations/layer5_attention_weight_max": 64.72992706298828,
      "activations/layer5_attention_weight_min": -65.60557556152344,
      "activations/layer6_attention_weight_max": 54.54109191894531,
      "activations/layer6_attention_weight_min": -53.72529602050781,
      "activations/layer7_attention_weight_max": 85.58804321289062,
      "activations/layer7_attention_weight_min": -88.17024230957031,
      "activations/layer8_attention_weight_max": 40.41344451904297,
      "activations/layer8_attention_weight_min": -41.97282409667969,
      "activations/layer9_attention_weight_max": 37.122371673583984,
      "activations/layer9_attention_weight_min": -34.78156661987305,
      "epoch": 6.89,
      "learning_rate": 0.00010665113636363636,
      "loss": 2.8187,
      "step": 118550
    },
    {
      "activations/layer0_attention_weight_max": 16.28643035888672,
      "activations/layer0_attention_weight_min": -11.731512069702148,
      "activations/layer10_attention_weight_max": 37.52056884765625,
      "activations/layer10_attention_weight_min": -33.4281005859375,
      "activations/layer11_attention_weight_max": 44.81981658935547,
      "activations/layer11_attention_weight_min": -34.77971649169922,
      "activations/layer12_attention_weight_max": 49.241920471191406,
      "activations/layer12_attention_weight_min": -28.999387741088867,
      "activations/layer13_attention_weight_max": 76.76480102539062,
      "activations/layer13_attention_weight_min": -39.04600524902344,
      "activations/layer14_attention_weight_max": 59.667327880859375,
      "activations/layer14_attention_weight_min": -42.79506301879883,
      "activations/layer15_attention_weight_max": 67.8447265625,
      "activations/layer15_attention_weight_min": -33.732791900634766,
      "activations/layer16_attention_weight_max": 34.23346710205078,
      "activations/layer16_attention_weight_min": -26.654523849487305,
      "activations/layer17_attention_weight_max": 37.237510681152344,
      "activations/layer17_attention_weight_min": -25.058435440063477,
      "activations/layer18_attention_weight_max": 31.383544921875,
      "activations/layer18_attention_weight_min": -20.719547271728516,
      "activations/layer19_attention_weight_max": 36.56879806518555,
      "activations/layer19_attention_weight_min": -30.804489135742188,
      "activations/layer1_attention_weight_max": 17.195663452148438,
      "activations/layer1_attention_weight_min": -13.446340560913086,
      "activations/layer20_attention_weight_max": 29.987459182739258,
      "activations/layer20_attention_weight_min": -24.4095401763916,
      "activations/layer21_attention_weight_max": 30.16661834716797,
      "activations/layer21_attention_weight_min": -21.133066177368164,
      "activations/layer22_attention_weight_max": 45.61370849609375,
      "activations/layer22_attention_weight_min": -29.194225311279297,
      "activations/layer23_attention_weight_max": 35.081756591796875,
      "activations/layer23_attention_weight_min": -21.373350143432617,
      "activations/layer2_attention_weight_max": 32.433349609375,
      "activations/layer2_attention_weight_min": -31.799819946289062,
      "activations/layer3_attention_weight_max": 87.91674041748047,
      "activations/layer3_attention_weight_min": -88.07266998291016,
      "activations/layer4_attention_weight_max": 125.2160415649414,
      "activations/layer4_attention_weight_min": -119.80500793457031,
      "activations/layer5_attention_weight_max": 68.79522705078125,
      "activations/layer5_attention_weight_min": -60.66872787475586,
      "activations/layer6_attention_weight_max": 52.958614349365234,
      "activations/layer6_attention_weight_min": -50.11482238769531,
      "activations/layer7_attention_weight_max": 91.73294830322266,
      "activations/layer7_attention_weight_min": -87.2598876953125,
      "activations/layer8_attention_weight_max": 42.920997619628906,
      "activations/layer8_attention_weight_min": -39.884910583496094,
      "activations/layer9_attention_weight_max": 38.28855895996094,
      "activations/layer9_attention_weight_min": -43.68296432495117,
      "epoch": 6.89,
      "learning_rate": 0.00010663219696969696,
      "loss": 2.8403,
      "step": 118600
    },
    {
      "activations/layer0_attention_weight_max": 16.790321350097656,
      "activations/layer0_attention_weight_min": -11.124473571777344,
      "activations/layer10_attention_weight_max": 34.2110595703125,
      "activations/layer10_attention_weight_min": -33.223106384277344,
      "activations/layer11_attention_weight_max": 34.20286178588867,
      "activations/layer11_attention_weight_min": -31.008560180664062,
      "activations/layer12_attention_weight_max": 40.23921203613281,
      "activations/layer12_attention_weight_min": -28.185483932495117,
      "activations/layer13_attention_weight_max": 44.08794403076172,
      "activations/layer13_attention_weight_min": -33.58056640625,
      "activations/layer14_attention_weight_max": 56.27813720703125,
      "activations/layer14_attention_weight_min": -44.35459518432617,
      "activations/layer15_attention_weight_max": 39.671409606933594,
      "activations/layer15_attention_weight_min": -31.391525268554688,
      "activations/layer16_attention_weight_max": 36.47819137573242,
      "activations/layer16_attention_weight_min": -29.0078125,
      "activations/layer17_attention_weight_max": 40.61069107055664,
      "activations/layer17_attention_weight_min": -27.720672607421875,
      "activations/layer18_attention_weight_max": 33.58953094482422,
      "activations/layer18_attention_weight_min": -23.778888702392578,
      "activations/layer19_attention_weight_max": 37.84355163574219,
      "activations/layer19_attention_weight_min": -33.31118392944336,
      "activations/layer1_attention_weight_max": 17.934003829956055,
      "activations/layer1_attention_weight_min": -14.359124183654785,
      "activations/layer20_attention_weight_max": 30.857391357421875,
      "activations/layer20_attention_weight_min": -25.057334899902344,
      "activations/layer21_attention_weight_max": 33.33830261230469,
      "activations/layer21_attention_weight_min": -23.69132423400879,
      "activations/layer22_attention_weight_max": 48.55291748046875,
      "activations/layer22_attention_weight_min": -30.14720916748047,
      "activations/layer23_attention_weight_max": 37.65553283691406,
      "activations/layer23_attention_weight_min": -22.960586547851562,
      "activations/layer2_attention_weight_max": 31.736692428588867,
      "activations/layer2_attention_weight_min": -31.697593688964844,
      "activations/layer3_attention_weight_max": 93.4578857421875,
      "activations/layer3_attention_weight_min": -94.09357452392578,
      "activations/layer4_attention_weight_max": 130.16024780273438,
      "activations/layer4_attention_weight_min": -123.7264404296875,
      "activations/layer5_attention_weight_max": 66.82168579101562,
      "activations/layer5_attention_weight_min": -63.11114501953125,
      "activations/layer6_attention_weight_max": 53.77570724487305,
      "activations/layer6_attention_weight_min": -52.77750778198242,
      "activations/layer7_attention_weight_max": 88.67684936523438,
      "activations/layer7_attention_weight_min": -90.96924591064453,
      "activations/layer8_attention_weight_max": 43.983097076416016,
      "activations/layer8_attention_weight_min": -39.56232833862305,
      "activations/layer9_attention_weight_max": 36.54987716674805,
      "activations/layer9_attention_weight_min": -34.948421478271484,
      "epoch": 6.89,
      "learning_rate": 0.00010661325757575756,
      "loss": 2.8343,
      "step": 118650
    },
    {
      "activations/layer0_attention_weight_max": 16.014129638671875,
      "activations/layer0_attention_weight_min": -10.832887649536133,
      "activations/layer10_attention_weight_max": 35.171173095703125,
      "activations/layer10_attention_weight_min": -32.74454879760742,
      "activations/layer11_attention_weight_max": 33.51968765258789,
      "activations/layer11_attention_weight_min": -29.37447166442871,
      "activations/layer12_attention_weight_max": 21.891569137573242,
      "activations/layer12_attention_weight_min": -25.79901695251465,
      "activations/layer13_attention_weight_max": 38.59961700439453,
      "activations/layer13_attention_weight_min": -31.724660873413086,
      "activations/layer14_attention_weight_max": 52.514400482177734,
      "activations/layer14_attention_weight_min": -39.77360916137695,
      "activations/layer15_attention_weight_max": 36.814796447753906,
      "activations/layer15_attention_weight_min": -29.02493667602539,
      "activations/layer16_attention_weight_max": 32.84046936035156,
      "activations/layer16_attention_weight_min": -29.732297897338867,
      "activations/layer17_attention_weight_max": 36.199703216552734,
      "activations/layer17_attention_weight_min": -26.074430465698242,
      "activations/layer18_attention_weight_max": 32.2682991027832,
      "activations/layer18_attention_weight_min": -21.096628189086914,
      "activations/layer19_attention_weight_max": 36.9772834777832,
      "activations/layer19_attention_weight_min": -30.5797176361084,
      "activations/layer1_attention_weight_max": 17.519887924194336,
      "activations/layer1_attention_weight_min": -13.249207496643066,
      "activations/layer20_attention_weight_max": 29.010448455810547,
      "activations/layer20_attention_weight_min": -26.02383041381836,
      "activations/layer21_attention_weight_max": 29.349533081054688,
      "activations/layer21_attention_weight_min": -22.50930404663086,
      "activations/layer22_attention_weight_max": 48.53007888793945,
      "activations/layer22_attention_weight_min": -29.03158187866211,
      "activations/layer23_attention_weight_max": 42.258155822753906,
      "activations/layer23_attention_weight_min": -23.67842674255371,
      "activations/layer2_attention_weight_max": 34.235984802246094,
      "activations/layer2_attention_weight_min": -31.622180938720703,
      "activations/layer3_attention_weight_max": 92.68968200683594,
      "activations/layer3_attention_weight_min": -93.64899444580078,
      "activations/layer4_attention_weight_max": 136.1355438232422,
      "activations/layer4_attention_weight_min": -126.7362289428711,
      "activations/layer5_attention_weight_max": 70.29667663574219,
      "activations/layer5_attention_weight_min": -66.12596893310547,
      "activations/layer6_attention_weight_max": 54.53931427001953,
      "activations/layer6_attention_weight_min": -53.064109802246094,
      "activations/layer7_attention_weight_max": 91.94641876220703,
      "activations/layer7_attention_weight_min": -86.30224609375,
      "activations/layer8_attention_weight_max": 43.43259048461914,
      "activations/layer8_attention_weight_min": -43.02677536010742,
      "activations/layer9_attention_weight_max": 38.246944427490234,
      "activations/layer9_attention_weight_min": -33.7399787902832,
      "epoch": 6.9,
      "learning_rate": 0.00010659431818181818,
      "loss": 2.8395,
      "step": 118700
    },
    {
      "activations/layer0_attention_weight_max": 15.894209861755371,
      "activations/layer0_attention_weight_min": -12.074581146240234,
      "activations/layer10_attention_weight_max": 34.3753776550293,
      "activations/layer10_attention_weight_min": -32.495357513427734,
      "activations/layer11_attention_weight_max": 38.535457611083984,
      "activations/layer11_attention_weight_min": -30.293935775756836,
      "activations/layer12_attention_weight_max": 24.98583984375,
      "activations/layer12_attention_weight_min": -26.829435348510742,
      "activations/layer13_attention_weight_max": 38.29780960083008,
      "activations/layer13_attention_weight_min": -34.062957763671875,
      "activations/layer14_attention_weight_max": 55.165313720703125,
      "activations/layer14_attention_weight_min": -41.380184173583984,
      "activations/layer15_attention_weight_max": 38.19346618652344,
      "activations/layer15_attention_weight_min": -31.97541618347168,
      "activations/layer16_attention_weight_max": 36.61384582519531,
      "activations/layer16_attention_weight_min": -28.07577133178711,
      "activations/layer17_attention_weight_max": 37.278682708740234,
      "activations/layer17_attention_weight_min": -26.55808448791504,
      "activations/layer18_attention_weight_max": 32.33041763305664,
      "activations/layer18_attention_weight_min": -21.310346603393555,
      "activations/layer19_attention_weight_max": 39.06577682495117,
      "activations/layer19_attention_weight_min": -31.019763946533203,
      "activations/layer1_attention_weight_max": 17.435516357421875,
      "activations/layer1_attention_weight_min": -14.466814041137695,
      "activations/layer20_attention_weight_max": 31.070371627807617,
      "activations/layer20_attention_weight_min": -23.71535301208496,
      "activations/layer21_attention_weight_max": 30.388957977294922,
      "activations/layer21_attention_weight_min": -21.364425659179688,
      "activations/layer22_attention_weight_max": 45.542816162109375,
      "activations/layer22_attention_weight_min": -28.71919059753418,
      "activations/layer23_attention_weight_max": 35.1654052734375,
      "activations/layer23_attention_weight_min": -22.821853637695312,
      "activations/layer2_attention_weight_max": 34.59050369262695,
      "activations/layer2_attention_weight_min": -33.0421142578125,
      "activations/layer3_attention_weight_max": 96.28194427490234,
      "activations/layer3_attention_weight_min": -94.13257598876953,
      "activations/layer4_attention_weight_max": 129.43429565429688,
      "activations/layer4_attention_weight_min": -122.88846588134766,
      "activations/layer5_attention_weight_max": 67.71176147460938,
      "activations/layer5_attention_weight_min": -64.13152313232422,
      "activations/layer6_attention_weight_max": 53.44242858886719,
      "activations/layer6_attention_weight_min": -55.237709045410156,
      "activations/layer7_attention_weight_max": 88.3904800415039,
      "activations/layer7_attention_weight_min": -92.09760284423828,
      "activations/layer8_attention_weight_max": 41.37247085571289,
      "activations/layer8_attention_weight_min": -40.29775619506836,
      "activations/layer9_attention_weight_max": 34.63459014892578,
      "activations/layer9_attention_weight_min": -35.587158203125,
      "epoch": 6.9,
      "learning_rate": 0.00010657537878787878,
      "loss": 2.8221,
      "step": 118750
    },
    {
      "activations/layer0_attention_weight_max": 16.008773803710938,
      "activations/layer0_attention_weight_min": -11.590023040771484,
      "activations/layer10_attention_weight_max": 35.99871063232422,
      "activations/layer10_attention_weight_min": -32.148189544677734,
      "activations/layer11_attention_weight_max": 34.92811584472656,
      "activations/layer11_attention_weight_min": -29.461891174316406,
      "activations/layer12_attention_weight_max": 23.251155853271484,
      "activations/layer12_attention_weight_min": -31.832887649536133,
      "activations/layer13_attention_weight_max": 41.817771911621094,
      "activations/layer13_attention_weight_min": -31.875102996826172,
      "activations/layer14_attention_weight_max": 53.77133560180664,
      "activations/layer14_attention_weight_min": -45.346885681152344,
      "activations/layer15_attention_weight_max": 38.293941497802734,
      "activations/layer15_attention_weight_min": -27.933792114257812,
      "activations/layer16_attention_weight_max": 35.39668273925781,
      "activations/layer16_attention_weight_min": -26.847970962524414,
      "activations/layer17_attention_weight_max": 34.438838958740234,
      "activations/layer17_attention_weight_min": -24.732690811157227,
      "activations/layer18_attention_weight_max": 32.15948486328125,
      "activations/layer18_attention_weight_min": -20.345916748046875,
      "activations/layer19_attention_weight_max": 39.189369201660156,
      "activations/layer19_attention_weight_min": -28.16412353515625,
      "activations/layer1_attention_weight_max": 16.443031311035156,
      "activations/layer1_attention_weight_min": -14.0452880859375,
      "activations/layer20_attention_weight_max": 32.29909133911133,
      "activations/layer20_attention_weight_min": -23.866287231445312,
      "activations/layer21_attention_weight_max": 30.727861404418945,
      "activations/layer21_attention_weight_min": -20.11037254333496,
      "activations/layer22_attention_weight_max": 51.63818359375,
      "activations/layer22_attention_weight_min": -29.54360008239746,
      "activations/layer23_attention_weight_max": 37.37032699584961,
      "activations/layer23_attention_weight_min": -24.280567169189453,
      "activations/layer2_attention_weight_max": 32.442684173583984,
      "activations/layer2_attention_weight_min": -31.526077270507812,
      "activations/layer3_attention_weight_max": 93.1047592163086,
      "activations/layer3_attention_weight_min": -99.70801544189453,
      "activations/layer4_attention_weight_max": 132.23464965820312,
      "activations/layer4_attention_weight_min": -134.59617614746094,
      "activations/layer5_attention_weight_max": 66.74339294433594,
      "activations/layer5_attention_weight_min": -65.25605773925781,
      "activations/layer6_attention_weight_max": 54.507408142089844,
      "activations/layer6_attention_weight_min": -54.26042938232422,
      "activations/layer7_attention_weight_max": 100.72599792480469,
      "activations/layer7_attention_weight_min": -91.9730453491211,
      "activations/layer8_attention_weight_max": 42.38919448852539,
      "activations/layer8_attention_weight_min": -42.439476013183594,
      "activations/layer9_attention_weight_max": 41.44429397583008,
      "activations/layer9_attention_weight_min": -38.88861846923828,
      "epoch": 6.9,
      "learning_rate": 0.00010655643939393938,
      "loss": 2.8037,
      "step": 118800
    },
    {
      "activations/layer0_attention_weight_max": 14.907828330993652,
      "activations/layer0_attention_weight_min": -12.491629600524902,
      "activations/layer10_attention_weight_max": 32.78630065917969,
      "activations/layer10_attention_weight_min": -31.708995819091797,
      "activations/layer11_attention_weight_max": 31.34687042236328,
      "activations/layer11_attention_weight_min": -28.013778686523438,
      "activations/layer12_attention_weight_max": 46.2315673828125,
      "activations/layer12_attention_weight_min": -28.92861557006836,
      "activations/layer13_attention_weight_max": 47.60157775878906,
      "activations/layer13_attention_weight_min": -32.57988357543945,
      "activations/layer14_attention_weight_max": 51.80161666870117,
      "activations/layer14_attention_weight_min": -42.21515655517578,
      "activations/layer15_attention_weight_max": 38.5542106628418,
      "activations/layer15_attention_weight_min": -28.898683547973633,
      "activations/layer16_attention_weight_max": 33.81196212768555,
      "activations/layer16_attention_weight_min": -28.80784034729004,
      "activations/layer17_attention_weight_max": 32.53007888793945,
      "activations/layer17_attention_weight_min": -24.904375076293945,
      "activations/layer18_attention_weight_max": 32.94350051879883,
      "activations/layer18_attention_weight_min": -18.873937606811523,
      "activations/layer19_attention_weight_max": 38.87882995605469,
      "activations/layer19_attention_weight_min": -30.431690216064453,
      "activations/layer1_attention_weight_max": 17.721389770507812,
      "activations/layer1_attention_weight_min": -13.915821075439453,
      "activations/layer20_attention_weight_max": 30.914770126342773,
      "activations/layer20_attention_weight_min": -23.781211853027344,
      "activations/layer21_attention_weight_max": 30.0616512298584,
      "activations/layer21_attention_weight_min": -20.392629623413086,
      "activations/layer22_attention_weight_max": 44.71816635131836,
      "activations/layer22_attention_weight_min": -29.241872787475586,
      "activations/layer23_attention_weight_max": 32.70477294921875,
      "activations/layer23_attention_weight_min": -22.504344940185547,
      "activations/layer2_attention_weight_max": 32.009117126464844,
      "activations/layer2_attention_weight_min": -31.135419845581055,
      "activations/layer3_attention_weight_max": 91.49181365966797,
      "activations/layer3_attention_weight_min": -90.21063995361328,
      "activations/layer4_attention_weight_max": 121.35675048828125,
      "activations/layer4_attention_weight_min": -120.2149887084961,
      "activations/layer5_attention_weight_max": 63.89181137084961,
      "activations/layer5_attention_weight_min": -63.941951751708984,
      "activations/layer6_attention_weight_max": 49.869773864746094,
      "activations/layer6_attention_weight_min": -51.70126724243164,
      "activations/layer7_attention_weight_max": 85.14804077148438,
      "activations/layer7_attention_weight_min": -85.53121185302734,
      "activations/layer8_attention_weight_max": 35.419281005859375,
      "activations/layer8_attention_weight_min": -36.959693908691406,
      "activations/layer9_attention_weight_max": 31.778244018554688,
      "activations/layer9_attention_weight_min": -38.198997497558594,
      "epoch": 6.91,
      "learning_rate": 0.0001065375,
      "loss": 2.8028,
      "step": 118850
    },
    {
      "activations/layer0_attention_weight_max": 16.10852813720703,
      "activations/layer0_attention_weight_min": -12.438488006591797,
      "activations/layer10_attention_weight_max": 31.086715698242188,
      "activations/layer10_attention_weight_min": -30.561620712280273,
      "activations/layer11_attention_weight_max": 31.56378936767578,
      "activations/layer11_attention_weight_min": -29.562734603881836,
      "activations/layer12_attention_weight_max": 64.02589416503906,
      "activations/layer12_attention_weight_min": -25.19771957397461,
      "activations/layer13_attention_weight_max": 67.43351745605469,
      "activations/layer13_attention_weight_min": -35.08894729614258,
      "activations/layer14_attention_weight_max": 70.37950134277344,
      "activations/layer14_attention_weight_min": -45.73810577392578,
      "activations/layer15_attention_weight_max": 49.90464401245117,
      "activations/layer15_attention_weight_min": -30.940570831298828,
      "activations/layer16_attention_weight_max": 34.55105972290039,
      "activations/layer16_attention_weight_min": -28.518892288208008,
      "activations/layer17_attention_weight_max": 35.278953552246094,
      "activations/layer17_attention_weight_min": -24.490442276000977,
      "activations/layer18_attention_weight_max": 32.998687744140625,
      "activations/layer18_attention_weight_min": -21.682796478271484,
      "activations/layer19_attention_weight_max": 38.135257720947266,
      "activations/layer19_attention_weight_min": -31.39206886291504,
      "activations/layer1_attention_weight_max": 17.12027931213379,
      "activations/layer1_attention_weight_min": -13.155892372131348,
      "activations/layer20_attention_weight_max": 34.33119201660156,
      "activations/layer20_attention_weight_min": -23.147489547729492,
      "activations/layer21_attention_weight_max": 35.20380401611328,
      "activations/layer21_attention_weight_min": -21.776193618774414,
      "activations/layer22_attention_weight_max": 50.5177116394043,
      "activations/layer22_attention_weight_min": -28.131258010864258,
      "activations/layer23_attention_weight_max": 36.17747497558594,
      "activations/layer23_attention_weight_min": -22.687416076660156,
      "activations/layer2_attention_weight_max": 30.87548828125,
      "activations/layer2_attention_weight_min": -30.301006317138672,
      "activations/layer3_attention_weight_max": 90.66797637939453,
      "activations/layer3_attention_weight_min": -87.3737564086914,
      "activations/layer4_attention_weight_max": 115.80266571044922,
      "activations/layer4_attention_weight_min": -118.45936584472656,
      "activations/layer5_attention_weight_max": 64.80113220214844,
      "activations/layer5_attention_weight_min": -64.88660430908203,
      "activations/layer6_attention_weight_max": 50.983192443847656,
      "activations/layer6_attention_weight_min": -53.03268051147461,
      "activations/layer7_attention_weight_max": 82.50140380859375,
      "activations/layer7_attention_weight_min": -86.4518051147461,
      "activations/layer8_attention_weight_max": 39.42573928833008,
      "activations/layer8_attention_weight_min": -41.50666046142578,
      "activations/layer9_attention_weight_max": 32.90236282348633,
      "activations/layer9_attention_weight_min": -31.661767959594727,
      "epoch": 6.91,
      "learning_rate": 0.0001065185606060606,
      "loss": 2.8403,
      "step": 118900
    },
    {
      "activations/layer0_attention_weight_max": 15.964059829711914,
      "activations/layer0_attention_weight_min": -11.376887321472168,
      "activations/layer10_attention_weight_max": 36.60786819458008,
      "activations/layer10_attention_weight_min": -32.51905822753906,
      "activations/layer11_attention_weight_max": 36.695762634277344,
      "activations/layer11_attention_weight_min": -31.31963348388672,
      "activations/layer12_attention_weight_max": 31.345415115356445,
      "activations/layer12_attention_weight_min": -23.306421279907227,
      "activations/layer13_attention_weight_max": 39.662132263183594,
      "activations/layer13_attention_weight_min": -31.657594680786133,
      "activations/layer14_attention_weight_max": 68.52594757080078,
      "activations/layer14_attention_weight_min": -41.20553970336914,
      "activations/layer15_attention_weight_max": 37.17184066772461,
      "activations/layer15_attention_weight_min": -28.53473663330078,
      "activations/layer16_attention_weight_max": 36.41556930541992,
      "activations/layer16_attention_weight_min": -27.25742530822754,
      "activations/layer17_attention_weight_max": 33.74959182739258,
      "activations/layer17_attention_weight_min": -25.2027587890625,
      "activations/layer18_attention_weight_max": 28.53921127319336,
      "activations/layer18_attention_weight_min": -20.717321395874023,
      "activations/layer19_attention_weight_max": 38.784725189208984,
      "activations/layer19_attention_weight_min": -29.623870849609375,
      "activations/layer1_attention_weight_max": 16.725364685058594,
      "activations/layer1_attention_weight_min": -14.249686241149902,
      "activations/layer20_attention_weight_max": 35.94664764404297,
      "activations/layer20_attention_weight_min": -24.04167366027832,
      "activations/layer21_attention_weight_max": 29.19174575805664,
      "activations/layer21_attention_weight_min": -21.527090072631836,
      "activations/layer22_attention_weight_max": 47.77296447753906,
      "activations/layer22_attention_weight_min": -27.78721809387207,
      "activations/layer23_attention_weight_max": 40.13771057128906,
      "activations/layer23_attention_weight_min": -21.498058319091797,
      "activations/layer2_attention_weight_max": 30.2100830078125,
      "activations/layer2_attention_weight_min": -29.90594482421875,
      "activations/layer3_attention_weight_max": 93.36526489257812,
      "activations/layer3_attention_weight_min": -92.97774505615234,
      "activations/layer4_attention_weight_max": 127.30687713623047,
      "activations/layer4_attention_weight_min": -124.28557586669922,
      "activations/layer5_attention_weight_max": 68.68069458007812,
      "activations/layer5_attention_weight_min": -69.4622573852539,
      "activations/layer6_attention_weight_max": 54.60677719116211,
      "activations/layer6_attention_weight_min": -56.872894287109375,
      "activations/layer7_attention_weight_max": 97.94646453857422,
      "activations/layer7_attention_weight_min": -97.2380142211914,
      "activations/layer8_attention_weight_max": 43.52922439575195,
      "activations/layer8_attention_weight_min": -45.477909088134766,
      "activations/layer9_attention_weight_max": 36.83991241455078,
      "activations/layer9_attention_weight_min": -37.34431076049805,
      "epoch": 6.91,
      "learning_rate": 0.00010649962121212121,
      "loss": 2.845,
      "step": 118950
    },
    {
      "activations/layer0_attention_weight_max": 15.097555160522461,
      "activations/layer0_attention_weight_min": -11.862722396850586,
      "activations/layer10_attention_weight_max": 34.28156280517578,
      "activations/layer10_attention_weight_min": -32.1953125,
      "activations/layer11_attention_weight_max": 32.28520202636719,
      "activations/layer11_attention_weight_min": -31.577816009521484,
      "activations/layer12_attention_weight_max": 23.248010635375977,
      "activations/layer12_attention_weight_min": -26.912803649902344,
      "activations/layer13_attention_weight_max": 41.549312591552734,
      "activations/layer13_attention_weight_min": -32.80713653564453,
      "activations/layer14_attention_weight_max": 49.97359085083008,
      "activations/layer14_attention_weight_min": -37.60356903076172,
      "activations/layer15_attention_weight_max": 37.341487884521484,
      "activations/layer15_attention_weight_min": -30.843626022338867,
      "activations/layer16_attention_weight_max": 35.64200973510742,
      "activations/layer16_attention_weight_min": -27.989009857177734,
      "activations/layer17_attention_weight_max": 38.8907585144043,
      "activations/layer17_attention_weight_min": -24.802640914916992,
      "activations/layer18_attention_weight_max": 37.68540573120117,
      "activations/layer18_attention_weight_min": -20.22549057006836,
      "activations/layer19_attention_weight_max": 42.75053405761719,
      "activations/layer19_attention_weight_min": -30.815017700195312,
      "activations/layer1_attention_weight_max": 16.903623580932617,
      "activations/layer1_attention_weight_min": -15.096698760986328,
      "activations/layer20_attention_weight_max": 32.43577575683594,
      "activations/layer20_attention_weight_min": -24.922744750976562,
      "activations/layer21_attention_weight_max": 36.388885498046875,
      "activations/layer21_attention_weight_min": -23.271516799926758,
      "activations/layer22_attention_weight_max": 53.74802017211914,
      "activations/layer22_attention_weight_min": -28.662919998168945,
      "activations/layer23_attention_weight_max": 43.899330139160156,
      "activations/layer23_attention_weight_min": -23.390270233154297,
      "activations/layer2_attention_weight_max": 32.26627731323242,
      "activations/layer2_attention_weight_min": -31.78827476501465,
      "activations/layer3_attention_weight_max": 91.2417221069336,
      "activations/layer3_attention_weight_min": -92.62435913085938,
      "activations/layer4_attention_weight_max": 129.25115966796875,
      "activations/layer4_attention_weight_min": -130.50674438476562,
      "activations/layer5_attention_weight_max": 68.71109771728516,
      "activations/layer5_attention_weight_min": -64.27644348144531,
      "activations/layer6_attention_weight_max": 53.07762145996094,
      "activations/layer6_attention_weight_min": -54.54611587524414,
      "activations/layer7_attention_weight_max": 89.59623718261719,
      "activations/layer7_attention_weight_min": -91.37194061279297,
      "activations/layer8_attention_weight_max": 42.45115661621094,
      "activations/layer8_attention_weight_min": -41.316551208496094,
      "activations/layer9_attention_weight_max": 38.219783782958984,
      "activations/layer9_attention_weight_min": -34.175724029541016,
      "epoch": 6.91,
      "learning_rate": 0.0001064806818181818,
      "loss": 2.8389,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.557,
      "eval_samples_per_second": 501.814,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.557,
      "eval_openwebtext_samples_per_second": 501.814,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_wikitext_loss": 2.99609375,
      "eval_wikitext_ppl": 20.007230835729846,
      "eval_wikitext_runtime": 2.0053,
      "eval_wikitext_samples_per_second": 227.4,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_lambada_loss": 2.80078125,
      "eval_lambada_ppl": 16.457499171202052,
      "eval_lambada_runtime": 9.6024,
      "eval_lambada_samples_per_second": 507.059,
      "step": 119000
    },
    {
      "activations/layer0_attention_weight_max": 15.766571044921875,
      "activations/layer0_attention_weight_min": -12.336764335632324,
      "activations/layer10_attention_weight_max": 35.42108917236328,
      "activations/layer10_attention_weight_min": -32.14860916137695,
      "activations/layer11_attention_weight_max": 32.089359283447266,
      "activations/layer11_attention_weight_min": -29.146818161010742,
      "activations/layer12_attention_weight_max": 40.15176773071289,
      "activations/layer12_attention_weight_min": -26.334260940551758,
      "activations/layer13_attention_weight_max": 44.96660614013672,
      "activations/layer13_attention_weight_min": -36.06099319458008,
      "activations/layer14_attention_weight_max": 62.246490478515625,
      "activations/layer14_attention_weight_min": -52.93689727783203,
      "activations/layer15_attention_weight_max": 37.37434768676758,
      "activations/layer15_attention_weight_min": -31.478927612304688,
      "activations/layer16_attention_weight_max": 36.89702606201172,
      "activations/layer16_attention_weight_min": -30.172245025634766,
      "activations/layer17_attention_weight_max": 39.51512908935547,
      "activations/layer17_attention_weight_min": -25.571353912353516,
      "activations/layer18_attention_weight_max": 33.778717041015625,
      "activations/layer18_attention_weight_min": -21.40924644470215,
      "activations/layer19_attention_weight_max": 40.228492736816406,
      "activations/layer19_attention_weight_min": -33.29070281982422,
      "activations/layer1_attention_weight_max": 17.305025100708008,
      "activations/layer1_attention_weight_min": -12.854290962219238,
      "activations/layer20_attention_weight_max": 35.08171463012695,
      "activations/layer20_attention_weight_min": -26.75689697265625,
      "activations/layer21_attention_weight_max": 33.601783752441406,
      "activations/layer21_attention_weight_min": -24.206857681274414,
      "activations/layer22_attention_weight_max": 51.32763671875,
      "activations/layer22_attention_weight_min": -30.481870651245117,
      "activations/layer23_attention_weight_max": 35.96941375732422,
      "activations/layer23_attention_weight_min": -24.333295822143555,
      "activations/layer2_attention_weight_max": 29.923831939697266,
      "activations/layer2_attention_weight_min": -29.55988121032715,
      "activations/layer3_attention_weight_max": 89.2754135131836,
      "activations/layer3_attention_weight_min": -87.099853515625,
      "activations/layer4_attention_weight_max": 126.5536880493164,
      "activations/layer4_attention_weight_min": -119.05653381347656,
      "activations/layer5_attention_weight_max": 68.04440307617188,
      "activations/layer5_attention_weight_min": -66.14464569091797,
      "activations/layer6_attention_weight_max": 53.64406204223633,
      "activations/layer6_attention_weight_min": -53.31629943847656,
      "activations/layer7_attention_weight_max": 92.97760772705078,
      "activations/layer7_attention_weight_min": -91.13502502441406,
      "activations/layer8_attention_weight_max": 40.66217041015625,
      "activations/layer8_attention_weight_min": -38.80501937866211,
      "activations/layer9_attention_weight_max": 38.326969146728516,
      "activations/layer9_attention_weight_min": -33.81170654296875,
      "epoch": 6.92,
      "learning_rate": 0.0001064621212121212,
      "loss": 2.8172,
      "step": 119050
    },
    {
      "activations/layer0_attention_weight_max": 15.37993335723877,
      "activations/layer0_attention_weight_min": -11.303869247436523,
      "activations/layer10_attention_weight_max": 35.14079666137695,
      "activations/layer10_attention_weight_min": -32.23152542114258,
      "activations/layer11_attention_weight_max": 33.17625427246094,
      "activations/layer11_attention_weight_min": -30.772695541381836,
      "activations/layer12_attention_weight_max": 45.38260269165039,
      "activations/layer12_attention_weight_min": -27.67601203918457,
      "activations/layer13_attention_weight_max": 41.248435974121094,
      "activations/layer13_attention_weight_min": -32.43952560424805,
      "activations/layer14_attention_weight_max": 53.510231018066406,
      "activations/layer14_attention_weight_min": -38.06591033935547,
      "activations/layer15_attention_weight_max": 37.54707336425781,
      "activations/layer15_attention_weight_min": -28.672563552856445,
      "activations/layer16_attention_weight_max": 35.55535888671875,
      "activations/layer16_attention_weight_min": -26.68897819519043,
      "activations/layer17_attention_weight_max": 33.040828704833984,
      "activations/layer17_attention_weight_min": -25.42806053161621,
      "activations/layer18_attention_weight_max": 29.736061096191406,
      "activations/layer18_attention_weight_min": -20.676843643188477,
      "activations/layer19_attention_weight_max": 39.96759796142578,
      "activations/layer19_attention_weight_min": -29.911197662353516,
      "activations/layer1_attention_weight_max": 17.02475357055664,
      "activations/layer1_attention_weight_min": -13.518540382385254,
      "activations/layer20_attention_weight_max": 29.87709617614746,
      "activations/layer20_attention_weight_min": -23.80963706970215,
      "activations/layer21_attention_weight_max": 31.407257080078125,
      "activations/layer21_attention_weight_min": -20.883007049560547,
      "activations/layer22_attention_weight_max": 49.83829116821289,
      "activations/layer22_attention_weight_min": -27.4157657623291,
      "activations/layer23_attention_weight_max": 36.55883026123047,
      "activations/layer23_attention_weight_min": -21.588085174560547,
      "activations/layer2_attention_weight_max": 29.38055419921875,
      "activations/layer2_attention_weight_min": -28.25652313232422,
      "activations/layer3_attention_weight_max": 87.6834945678711,
      "activations/layer3_attention_weight_min": -86.0063247680664,
      "activations/layer4_attention_weight_max": 119.81197357177734,
      "activations/layer4_attention_weight_min": -117.4139404296875,
      "activations/layer5_attention_weight_max": 64.65914916992188,
      "activations/layer5_attention_weight_min": -63.6026725769043,
      "activations/layer6_attention_weight_max": 49.23699188232422,
      "activations/layer6_attention_weight_min": -52.61622619628906,
      "activations/layer7_attention_weight_max": 85.3727035522461,
      "activations/layer7_attention_weight_min": -84.09915161132812,
      "activations/layer8_attention_weight_max": 39.28009796142578,
      "activations/layer8_attention_weight_min": -40.05210876464844,
      "activations/layer9_attention_weight_max": 37.19054412841797,
      "activations/layer9_attention_weight_min": -34.129268646240234,
      "epoch": 6.92,
      "learning_rate": 0.0001064431818181818,
      "loss": 2.8307,
      "step": 119100
    },
    {
      "activations/layer0_attention_weight_max": 16.139108657836914,
      "activations/layer0_attention_weight_min": -11.248367309570312,
      "activations/layer10_attention_weight_max": 31.96377944946289,
      "activations/layer10_attention_weight_min": -31.449655532836914,
      "activations/layer11_attention_weight_max": 32.331600189208984,
      "activations/layer11_attention_weight_min": -28.422195434570312,
      "activations/layer12_attention_weight_max": 24.06577491760254,
      "activations/layer12_attention_weight_min": -24.58856773376465,
      "activations/layer13_attention_weight_max": 40.908851623535156,
      "activations/layer13_attention_weight_min": -30.77916145324707,
      "activations/layer14_attention_weight_max": 56.837867736816406,
      "activations/layer14_attention_weight_min": -40.32540512084961,
      "activations/layer15_attention_weight_max": 37.31768035888672,
      "activations/layer15_attention_weight_min": -27.907405853271484,
      "activations/layer16_attention_weight_max": 35.337799072265625,
      "activations/layer16_attention_weight_min": -27.384933471679688,
      "activations/layer17_attention_weight_max": 33.342655181884766,
      "activations/layer17_attention_weight_min": -24.038602828979492,
      "activations/layer18_attention_weight_max": 33.347740173339844,
      "activations/layer18_attention_weight_min": -24.306396484375,
      "activations/layer19_attention_weight_max": 37.70858383178711,
      "activations/layer19_attention_weight_min": -29.53339195251465,
      "activations/layer1_attention_weight_max": 16.799819946289062,
      "activations/layer1_attention_weight_min": -12.175053596496582,
      "activations/layer20_attention_weight_max": 33.88193893432617,
      "activations/layer20_attention_weight_min": -24.136625289916992,
      "activations/layer21_attention_weight_max": 30.011455535888672,
      "activations/layer21_attention_weight_min": -22.10532569885254,
      "activations/layer22_attention_weight_max": 52.1618537902832,
      "activations/layer22_attention_weight_min": -26.715744018554688,
      "activations/layer23_attention_weight_max": 37.904747009277344,
      "activations/layer23_attention_weight_min": -22.611495971679688,
      "activations/layer2_attention_weight_max": 31.407079696655273,
      "activations/layer2_attention_weight_min": -32.3177604675293,
      "activations/layer3_attention_weight_max": 93.1700439453125,
      "activations/layer3_attention_weight_min": -92.46551513671875,
      "activations/layer4_attention_weight_max": 122.3572769165039,
      "activations/layer4_attention_weight_min": -122.07024383544922,
      "activations/layer5_attention_weight_max": 64.52580261230469,
      "activations/layer5_attention_weight_min": -63.99982452392578,
      "activations/layer6_attention_weight_max": 50.02670669555664,
      "activations/layer6_attention_weight_min": -55.01020050048828,
      "activations/layer7_attention_weight_max": 89.48091888427734,
      "activations/layer7_attention_weight_min": -86.81864929199219,
      "activations/layer8_attention_weight_max": 39.05930709838867,
      "activations/layer8_attention_weight_min": -40.54524230957031,
      "activations/layer9_attention_weight_max": 36.4246940612793,
      "activations/layer9_attention_weight_min": -35.560611724853516,
      "epoch": 6.92,
      "learning_rate": 0.00010642424242424242,
      "loss": 2.8106,
      "step": 119150
    },
    {
      "activations/layer0_attention_weight_max": 15.25031566619873,
      "activations/layer0_attention_weight_min": -11.419075012207031,
      "activations/layer10_attention_weight_max": 36.0981559753418,
      "activations/layer10_attention_weight_min": -31.451847076416016,
      "activations/layer11_attention_weight_max": 33.86222457885742,
      "activations/layer11_attention_weight_min": -30.053709030151367,
      "activations/layer12_attention_weight_max": 26.50604820251465,
      "activations/layer12_attention_weight_min": -26.163793563842773,
      "activations/layer13_attention_weight_max": 39.207244873046875,
      "activations/layer13_attention_weight_min": -32.33547592163086,
      "activations/layer14_attention_weight_max": 52.46855926513672,
      "activations/layer14_attention_weight_min": -41.776187896728516,
      "activations/layer15_attention_weight_max": 35.98421096801758,
      "activations/layer15_attention_weight_min": -30.714719772338867,
      "activations/layer16_attention_weight_max": 33.03110122680664,
      "activations/layer16_attention_weight_min": -28.111833572387695,
      "activations/layer17_attention_weight_max": 35.38502883911133,
      "activations/layer17_attention_weight_min": -27.901527404785156,
      "activations/layer18_attention_weight_max": 34.019710540771484,
      "activations/layer18_attention_weight_min": -21.637792587280273,
      "activations/layer19_attention_weight_max": 40.77539825439453,
      "activations/layer19_attention_weight_min": -30.909622192382812,
      "activations/layer1_attention_weight_max": 17.23208236694336,
      "activations/layer1_attention_weight_min": -11.727389335632324,
      "activations/layer20_attention_weight_max": 30.131689071655273,
      "activations/layer20_attention_weight_min": -27.21984100341797,
      "activations/layer21_attention_weight_max": 32.61498260498047,
      "activations/layer21_attention_weight_min": -24.30776023864746,
      "activations/layer22_attention_weight_max": 53.97264099121094,
      "activations/layer22_attention_weight_min": -30.83112907409668,
      "activations/layer23_attention_weight_max": 37.02745819091797,
      "activations/layer23_attention_weight_min": -25.437129974365234,
      "activations/layer2_attention_weight_max": 28.392135620117188,
      "activations/layer2_attention_weight_min": -27.11833953857422,
      "activations/layer3_attention_weight_max": 87.18199920654297,
      "activations/layer3_attention_weight_min": -85.12654113769531,
      "activations/layer4_attention_weight_max": 119.06160736083984,
      "activations/layer4_attention_weight_min": -125.9649429321289,
      "activations/layer5_attention_weight_max": 64.64328002929688,
      "activations/layer5_attention_weight_min": -64.02901458740234,
      "activations/layer6_attention_weight_max": 52.65148162841797,
      "activations/layer6_attention_weight_min": -54.573341369628906,
      "activations/layer7_attention_weight_max": 85.98929595947266,
      "activations/layer7_attention_weight_min": -89.91152954101562,
      "activations/layer8_attention_weight_max": 39.52214431762695,
      "activations/layer8_attention_weight_min": -43.637123107910156,
      "activations/layer9_attention_weight_max": 37.39570617675781,
      "activations/layer9_attention_weight_min": -34.209007263183594,
      "epoch": 6.93,
      "learning_rate": 0.00010640530303030301,
      "loss": 2.8179,
      "step": 119200
    },
    {
      "activations/layer0_attention_weight_max": 15.798234939575195,
      "activations/layer0_attention_weight_min": -11.952251434326172,
      "activations/layer10_attention_weight_max": 36.983882904052734,
      "activations/layer10_attention_weight_min": -33.8163948059082,
      "activations/layer11_attention_weight_max": 38.73704147338867,
      "activations/layer11_attention_weight_min": -32.525569915771484,
      "activations/layer12_attention_weight_max": 31.090375900268555,
      "activations/layer12_attention_weight_min": -27.987455368041992,
      "activations/layer13_attention_weight_max": 53.70317459106445,
      "activations/layer13_attention_weight_min": -33.842918395996094,
      "activations/layer14_attention_weight_max": 58.16679763793945,
      "activations/layer14_attention_weight_min": -38.85664749145508,
      "activations/layer15_attention_weight_max": 55.17877960205078,
      "activations/layer15_attention_weight_min": -32.67615509033203,
      "activations/layer16_attention_weight_max": 41.800418853759766,
      "activations/layer16_attention_weight_min": -27.484512329101562,
      "activations/layer17_attention_weight_max": 39.70361328125,
      "activations/layer17_attention_weight_min": -23.951475143432617,
      "activations/layer18_attention_weight_max": 31.017667770385742,
      "activations/layer18_attention_weight_min": -21.662418365478516,
      "activations/layer19_attention_weight_max": 42.927791595458984,
      "activations/layer19_attention_weight_min": -28.927297592163086,
      "activations/layer1_attention_weight_max": 17.815467834472656,
      "activations/layer1_attention_weight_min": -14.181998252868652,
      "activations/layer20_attention_weight_max": 34.40713882446289,
      "activations/layer20_attention_weight_min": -23.41619300842285,
      "activations/layer21_attention_weight_max": 31.443960189819336,
      "activations/layer21_attention_weight_min": -21.636255264282227,
      "activations/layer22_attention_weight_max": 53.435489654541016,
      "activations/layer22_attention_weight_min": -28.331806182861328,
      "activations/layer23_attention_weight_max": 37.81861877441406,
      "activations/layer23_attention_weight_min": -23.718952178955078,
      "activations/layer2_attention_weight_max": 29.336563110351562,
      "activations/layer2_attention_weight_min": -27.7450008392334,
      "activations/layer3_attention_weight_max": 79.9950942993164,
      "activations/layer3_attention_weight_min": -79.68563842773438,
      "activations/layer4_attention_weight_max": 117.02325439453125,
      "activations/layer4_attention_weight_min": -113.8757553100586,
      "activations/layer5_attention_weight_max": 63.908546447753906,
      "activations/layer5_attention_weight_min": -62.48093795776367,
      "activations/layer6_attention_weight_max": 54.98288345336914,
      "activations/layer6_attention_weight_min": -52.96292495727539,
      "activations/layer7_attention_weight_max": 89.86575317382812,
      "activations/layer7_attention_weight_min": -88.818115234375,
      "activations/layer8_attention_weight_max": 42.015106201171875,
      "activations/layer8_attention_weight_min": -41.43721008300781,
      "activations/layer9_attention_weight_max": 40.17828369140625,
      "activations/layer9_attention_weight_min": -35.23191833496094,
      "epoch": 6.93,
      "learning_rate": 0.00010638636363636362,
      "loss": 2.8326,
      "step": 119250
    },
    {
      "activations/layer0_attention_weight_max": 16.575544357299805,
      "activations/layer0_attention_weight_min": -11.929583549499512,
      "activations/layer10_attention_weight_max": 32.65695571899414,
      "activations/layer10_attention_weight_min": -31.97648811340332,
      "activations/layer11_attention_weight_max": 31.919904708862305,
      "activations/layer11_attention_weight_min": -29.22292709350586,
      "activations/layer12_attention_weight_max": 59.004756927490234,
      "activations/layer12_attention_weight_min": -31.002235412597656,
      "activations/layer13_attention_weight_max": 53.446537017822266,
      "activations/layer13_attention_weight_min": -32.277950286865234,
      "activations/layer14_attention_weight_max": 55.81043243408203,
      "activations/layer14_attention_weight_min": -45.000492095947266,
      "activations/layer15_attention_weight_max": 38.84185028076172,
      "activations/layer15_attention_weight_min": -28.571475982666016,
      "activations/layer16_attention_weight_max": 40.913387298583984,
      "activations/layer16_attention_weight_min": -29.540624618530273,
      "activations/layer17_attention_weight_max": 36.428470611572266,
      "activations/layer17_attention_weight_min": -25.41937828063965,
      "activations/layer18_attention_weight_max": 34.916500091552734,
      "activations/layer18_attention_weight_min": -21.51639747619629,
      "activations/layer19_attention_weight_max": 44.6588249206543,
      "activations/layer19_attention_weight_min": -32.84675979614258,
      "activations/layer1_attention_weight_max": 16.62030029296875,
      "activations/layer1_attention_weight_min": -12.356013298034668,
      "activations/layer20_attention_weight_max": 37.91279220581055,
      "activations/layer20_attention_weight_min": -24.03797149658203,
      "activations/layer21_attention_weight_max": 35.824440002441406,
      "activations/layer21_attention_weight_min": -24.134328842163086,
      "activations/layer22_attention_weight_max": 64.73258972167969,
      "activations/layer22_attention_weight_min": -28.1580867767334,
      "activations/layer23_attention_weight_max": 43.87494659423828,
      "activations/layer23_attention_weight_min": -23.103179931640625,
      "activations/layer2_attention_weight_max": 28.896350860595703,
      "activations/layer2_attention_weight_min": -27.27596092224121,
      "activations/layer3_attention_weight_max": 86.61832427978516,
      "activations/layer3_attention_weight_min": -88.49107360839844,
      "activations/layer4_attention_weight_max": 122.14420318603516,
      "activations/layer4_attention_weight_min": -124.48339080810547,
      "activations/layer5_attention_weight_max": 63.571632385253906,
      "activations/layer5_attention_weight_min": -63.5402946472168,
      "activations/layer6_attention_weight_max": 51.44601821899414,
      "activations/layer6_attention_weight_min": -51.328208923339844,
      "activations/layer7_attention_weight_max": 85.23587799072266,
      "activations/layer7_attention_weight_min": -87.38860321044922,
      "activations/layer8_attention_weight_max": 38.78843307495117,
      "activations/layer8_attention_weight_min": -42.28436279296875,
      "activations/layer9_attention_weight_max": 35.27912521362305,
      "activations/layer9_attention_weight_min": -33.296146392822266,
      "epoch": 6.93,
      "learning_rate": 0.00010636742424242424,
      "loss": 2.8244,
      "step": 119300
    },
    {
      "activations/layer0_attention_weight_max": 16.427762985229492,
      "activations/layer0_attention_weight_min": -11.883062362670898,
      "activations/layer10_attention_weight_max": 36.8826789855957,
      "activations/layer10_attention_weight_min": -33.371971130371094,
      "activations/layer11_attention_weight_max": 39.45039367675781,
      "activations/layer11_attention_weight_min": -32.36643981933594,
      "activations/layer12_attention_weight_max": 24.66874885559082,
      "activations/layer12_attention_weight_min": -26.80448341369629,
      "activations/layer13_attention_weight_max": 51.08200454711914,
      "activations/layer13_attention_weight_min": -32.73654556274414,
      "activations/layer14_attention_weight_max": 67.0503158569336,
      "activations/layer14_attention_weight_min": -46.944705963134766,
      "activations/layer15_attention_weight_max": 46.10839080810547,
      "activations/layer15_attention_weight_min": -30.78601837158203,
      "activations/layer16_attention_weight_max": 38.89226531982422,
      "activations/layer16_attention_weight_min": -28.08934783935547,
      "activations/layer17_attention_weight_max": 39.338958740234375,
      "activations/layer17_attention_weight_min": -25.854042053222656,
      "activations/layer18_attention_weight_max": 32.8330078125,
      "activations/layer18_attention_weight_min": -21.73900032043457,
      "activations/layer19_attention_weight_max": 46.75120544433594,
      "activations/layer19_attention_weight_min": -29.53608512878418,
      "activations/layer1_attention_weight_max": 17.873926162719727,
      "activations/layer1_attention_weight_min": -13.853768348693848,
      "activations/layer20_attention_weight_max": 40.37250518798828,
      "activations/layer20_attention_weight_min": -24.305631637573242,
      "activations/layer21_attention_weight_max": 35.67731857299805,
      "activations/layer21_attention_weight_min": -21.60642433166504,
      "activations/layer22_attention_weight_max": 55.08053207397461,
      "activations/layer22_attention_weight_min": -30.322433471679688,
      "activations/layer23_attention_weight_max": 39.73965072631836,
      "activations/layer23_attention_weight_min": -21.90487289428711,
      "activations/layer2_attention_weight_max": 29.346982955932617,
      "activations/layer2_attention_weight_min": -30.05663299560547,
      "activations/layer3_attention_weight_max": 84.87427520751953,
      "activations/layer3_attention_weight_min": -89.27603149414062,
      "activations/layer4_attention_weight_max": 122.470458984375,
      "activations/layer4_attention_weight_min": -117.76427459716797,
      "activations/layer5_attention_weight_max": 64.99919128417969,
      "activations/layer5_attention_weight_min": -61.121864318847656,
      "activations/layer6_attention_weight_max": 53.69760513305664,
      "activations/layer6_attention_weight_min": -52.01126480102539,
      "activations/layer7_attention_weight_max": 103.49165344238281,
      "activations/layer7_attention_weight_min": -94.1422348022461,
      "activations/layer8_attention_weight_max": 47.67062759399414,
      "activations/layer8_attention_weight_min": -44.553958892822266,
      "activations/layer9_attention_weight_max": 40.69344711303711,
      "activations/layer9_attention_weight_min": -38.71952438354492,
      "epoch": 6.93,
      "learning_rate": 0.00010634848484848484,
      "loss": 2.8194,
      "step": 119350
    },
    {
      "activations/layer0_attention_weight_max": 16.580522537231445,
      "activations/layer0_attention_weight_min": -12.68713092803955,
      "activations/layer10_attention_weight_max": 29.737455368041992,
      "activations/layer10_attention_weight_min": -30.933902740478516,
      "activations/layer11_attention_weight_max": 31.718612670898438,
      "activations/layer11_attention_weight_min": -28.209800720214844,
      "activations/layer12_attention_weight_max": 23.26565933227539,
      "activations/layer12_attention_weight_min": -28.681272506713867,
      "activations/layer13_attention_weight_max": 41.67649841308594,
      "activations/layer13_attention_weight_min": -31.354347229003906,
      "activations/layer14_attention_weight_max": 49.14936828613281,
      "activations/layer14_attention_weight_min": -39.654449462890625,
      "activations/layer15_attention_weight_max": 39.17209243774414,
      "activations/layer15_attention_weight_min": -28.788564682006836,
      "activations/layer16_attention_weight_max": 33.294490814208984,
      "activations/layer16_attention_weight_min": -27.665239334106445,
      "activations/layer17_attention_weight_max": 37.661739349365234,
      "activations/layer17_attention_weight_min": -24.65848731994629,
      "activations/layer18_attention_weight_max": 38.06463623046875,
      "activations/layer18_attention_weight_min": -21.994779586791992,
      "activations/layer19_attention_weight_max": 36.228179931640625,
      "activations/layer19_attention_weight_min": -30.974924087524414,
      "activations/layer1_attention_weight_max": 17.12235450744629,
      "activations/layer1_attention_weight_min": -12.977446556091309,
      "activations/layer20_attention_weight_max": 29.976539611816406,
      "activations/layer20_attention_weight_min": -23.99644660949707,
      "activations/layer21_attention_weight_max": 32.50594711303711,
      "activations/layer21_attention_weight_min": -23.03329849243164,
      "activations/layer22_attention_weight_max": 50.438568115234375,
      "activations/layer22_attention_weight_min": -28.525049209594727,
      "activations/layer23_attention_weight_max": 35.20509719848633,
      "activations/layer23_attention_weight_min": -23.862125396728516,
      "activations/layer2_attention_weight_max": 31.170745849609375,
      "activations/layer2_attention_weight_min": -30.868200302124023,
      "activations/layer3_attention_weight_max": 87.55093383789062,
      "activations/layer3_attention_weight_min": -92.67810821533203,
      "activations/layer4_attention_weight_max": 122.14421844482422,
      "activations/layer4_attention_weight_min": -122.0372314453125,
      "activations/layer5_attention_weight_max": 66.83284759521484,
      "activations/layer5_attention_weight_min": -63.72898483276367,
      "activations/layer6_attention_weight_max": 50.80559539794922,
      "activations/layer6_attention_weight_min": -51.565956115722656,
      "activations/layer7_attention_weight_max": 87.10877990722656,
      "activations/layer7_attention_weight_min": -88.98236846923828,
      "activations/layer8_attention_weight_max": 38.89216232299805,
      "activations/layer8_attention_weight_min": -39.114261627197266,
      "activations/layer9_attention_weight_max": 33.506195068359375,
      "activations/layer9_attention_weight_min": -32.23505783081055,
      "epoch": 6.94,
      "learning_rate": 0.00010632954545454544,
      "loss": 2.8242,
      "step": 119400
    },
    {
      "activations/layer0_attention_weight_max": 15.643253326416016,
      "activations/layer0_attention_weight_min": -11.393824577331543,
      "activations/layer10_attention_weight_max": 31.052709579467773,
      "activations/layer10_attention_weight_min": -31.147747039794922,
      "activations/layer11_attention_weight_max": 30.38227081298828,
      "activations/layer11_attention_weight_min": -27.947368621826172,
      "activations/layer12_attention_weight_max": 35.57944107055664,
      "activations/layer12_attention_weight_min": -26.224340438842773,
      "activations/layer13_attention_weight_max": 38.330604553222656,
      "activations/layer13_attention_weight_min": -32.09529113769531,
      "activations/layer14_attention_weight_max": 55.42566680908203,
      "activations/layer14_attention_weight_min": -40.94263458251953,
      "activations/layer15_attention_weight_max": 41.06351089477539,
      "activations/layer15_attention_weight_min": -29.294740676879883,
      "activations/layer16_attention_weight_max": 34.185935974121094,
      "activations/layer16_attention_weight_min": -27.63033103942871,
      "activations/layer17_attention_weight_max": 38.64873504638672,
      "activations/layer17_attention_weight_min": -23.864591598510742,
      "activations/layer18_attention_weight_max": 35.302120208740234,
      "activations/layer18_attention_weight_min": -21.659582138061523,
      "activations/layer19_attention_weight_max": 42.07756042480469,
      "activations/layer19_attention_weight_min": -29.085500717163086,
      "activations/layer1_attention_weight_max": 16.720304489135742,
      "activations/layer1_attention_weight_min": -13.482686042785645,
      "activations/layer20_attention_weight_max": 37.49611282348633,
      "activations/layer20_attention_weight_min": -23.064516067504883,
      "activations/layer21_attention_weight_max": 38.86441421508789,
      "activations/layer21_attention_weight_min": -23.068269729614258,
      "activations/layer22_attention_weight_max": 51.76338577270508,
      "activations/layer22_attention_weight_min": -29.100801467895508,
      "activations/layer23_attention_weight_max": 38.937923431396484,
      "activations/layer23_attention_weight_min": -21.84161376953125,
      "activations/layer2_attention_weight_max": 30.99588394165039,
      "activations/layer2_attention_weight_min": -28.881088256835938,
      "activations/layer3_attention_weight_max": 87.07960510253906,
      "activations/layer3_attention_weight_min": -87.22298431396484,
      "activations/layer4_attention_weight_max": 120.1713638305664,
      "activations/layer4_attention_weight_min": -118.9649887084961,
      "activations/layer5_attention_weight_max": 62.24738311767578,
      "activations/layer5_attention_weight_min": -61.1486930847168,
      "activations/layer6_attention_weight_max": 53.1088981628418,
      "activations/layer6_attention_weight_min": -51.33285903930664,
      "activations/layer7_attention_weight_max": 87.20681762695312,
      "activations/layer7_attention_weight_min": -85.85037231445312,
      "activations/layer8_attention_weight_max": 38.86029052734375,
      "activations/layer8_attention_weight_min": -37.78317642211914,
      "activations/layer9_attention_weight_max": 36.89243698120117,
      "activations/layer9_attention_weight_min": -37.25307083129883,
      "epoch": 6.94,
      "learning_rate": 0.00010631060606060606,
      "loss": 2.8221,
      "step": 119450
    },
    {
      "activations/layer0_attention_weight_max": 16.7677059173584,
      "activations/layer0_attention_weight_min": -11.634359359741211,
      "activations/layer10_attention_weight_max": 32.93989944458008,
      "activations/layer10_attention_weight_min": -30.338863372802734,
      "activations/layer11_attention_weight_max": 30.38672637939453,
      "activations/layer11_attention_weight_min": -29.309146881103516,
      "activations/layer12_attention_weight_max": 35.653560638427734,
      "activations/layer12_attention_weight_min": -32.312828063964844,
      "activations/layer13_attention_weight_max": 42.366004943847656,
      "activations/layer13_attention_weight_min": -32.19472122192383,
      "activations/layer14_attention_weight_max": 51.686546325683594,
      "activations/layer14_attention_weight_min": -38.69062423706055,
      "activations/layer15_attention_weight_max": 37.84648895263672,
      "activations/layer15_attention_weight_min": -29.677709579467773,
      "activations/layer16_attention_weight_max": 34.3948974609375,
      "activations/layer16_attention_weight_min": -26.511436462402344,
      "activations/layer17_attention_weight_max": 35.5555305480957,
      "activations/layer17_attention_weight_min": -23.33490753173828,
      "activations/layer18_attention_weight_max": 33.84905242919922,
      "activations/layer18_attention_weight_min": -23.206993103027344,
      "activations/layer19_attention_weight_max": 39.66872787475586,
      "activations/layer19_attention_weight_min": -30.358686447143555,
      "activations/layer1_attention_weight_max": 18.49829864501953,
      "activations/layer1_attention_weight_min": -13.025031089782715,
      "activations/layer20_attention_weight_max": 33.329837799072266,
      "activations/layer20_attention_weight_min": -22.76980209350586,
      "activations/layer21_attention_weight_max": 32.809295654296875,
      "activations/layer21_attention_weight_min": -21.75783348083496,
      "activations/layer22_attention_weight_max": 54.074378967285156,
      "activations/layer22_attention_weight_min": -28.24362564086914,
      "activations/layer23_attention_weight_max": 43.85060119628906,
      "activations/layer23_attention_weight_min": -24.095714569091797,
      "activations/layer2_attention_weight_max": 30.337133407592773,
      "activations/layer2_attention_weight_min": -30.660839080810547,
      "activations/layer3_attention_weight_max": 86.03071594238281,
      "activations/layer3_attention_weight_min": -88.35289001464844,
      "activations/layer4_attention_weight_max": 116.80045318603516,
      "activations/layer4_attention_weight_min": -119.07564544677734,
      "activations/layer5_attention_weight_max": 60.650970458984375,
      "activations/layer5_attention_weight_min": -60.22037887573242,
      "activations/layer6_attention_weight_max": 52.183406829833984,
      "activations/layer6_attention_weight_min": -52.02750778198242,
      "activations/layer7_attention_weight_max": 88.72046661376953,
      "activations/layer7_attention_weight_min": -81.80562591552734,
      "activations/layer8_attention_weight_max": 40.41081619262695,
      "activations/layer8_attention_weight_min": -39.55426788330078,
      "activations/layer9_attention_weight_max": 34.20675277709961,
      "activations/layer9_attention_weight_min": -32.81521224975586,
      "epoch": 6.94,
      "learning_rate": 0.00010629166666666666,
      "loss": 2.8149,
      "step": 119500
    },
    {
      "activations/layer0_attention_weight_max": 17.01004981994629,
      "activations/layer0_attention_weight_min": -12.263389587402344,
      "activations/layer10_attention_weight_max": 34.89466857910156,
      "activations/layer10_attention_weight_min": -31.319772720336914,
      "activations/layer11_attention_weight_max": 32.83482360839844,
      "activations/layer11_attention_weight_min": -29.318157196044922,
      "activations/layer12_attention_weight_max": 24.46483039855957,
      "activations/layer12_attention_weight_min": -25.817907333374023,
      "activations/layer13_attention_weight_max": 42.39781951904297,
      "activations/layer13_attention_weight_min": -32.609012603759766,
      "activations/layer14_attention_weight_max": 49.542747497558594,
      "activations/layer14_attention_weight_min": -38.06395721435547,
      "activations/layer15_attention_weight_max": 36.79766082763672,
      "activations/layer15_attention_weight_min": -30.102319717407227,
      "activations/layer16_attention_weight_max": 33.50679016113281,
      "activations/layer16_attention_weight_min": -28.19367218017578,
      "activations/layer17_attention_weight_max": 35.38705825805664,
      "activations/layer17_attention_weight_min": -25.917898178100586,
      "activations/layer18_attention_weight_max": 31.943626403808594,
      "activations/layer18_attention_weight_min": -21.940019607543945,
      "activations/layer19_attention_weight_max": 34.918025970458984,
      "activations/layer19_attention_weight_min": -30.729825973510742,
      "activations/layer1_attention_weight_max": 16.58391571044922,
      "activations/layer1_attention_weight_min": -13.589057922363281,
      "activations/layer20_attention_weight_max": 29.615650177001953,
      "activations/layer20_attention_weight_min": -24.080039978027344,
      "activations/layer21_attention_weight_max": 31.393712997436523,
      "activations/layer21_attention_weight_min": -21.96199607849121,
      "activations/layer22_attention_weight_max": 45.56928253173828,
      "activations/layer22_attention_weight_min": -28.493867874145508,
      "activations/layer23_attention_weight_max": 39.76188659667969,
      "activations/layer23_attention_weight_min": -24.446392059326172,
      "activations/layer2_attention_weight_max": 31.39609146118164,
      "activations/layer2_attention_weight_min": -29.526456832885742,
      "activations/layer3_attention_weight_max": 90.17902374267578,
      "activations/layer3_attention_weight_min": -89.4615249633789,
      "activations/layer4_attention_weight_max": 126.76476287841797,
      "activations/layer4_attention_weight_min": -119.2340087890625,
      "activations/layer5_attention_weight_max": 64.9020004272461,
      "activations/layer5_attention_weight_min": -66.83504486083984,
      "activations/layer6_attention_weight_max": 54.843441009521484,
      "activations/layer6_attention_weight_min": -53.317222595214844,
      "activations/layer7_attention_weight_max": 87.57940673828125,
      "activations/layer7_attention_weight_min": -87.82384490966797,
      "activations/layer8_attention_weight_max": 39.77650833129883,
      "activations/layer8_attention_weight_min": -38.68622970581055,
      "activations/layer9_attention_weight_max": 34.53756332397461,
      "activations/layer9_attention_weight_min": -33.408748626708984,
      "epoch": 6.95,
      "learning_rate": 0.00010627272727272726,
      "loss": 2.8164,
      "step": 119550
    },
    {
      "activations/layer0_attention_weight_max": 15.869056701660156,
      "activations/layer0_attention_weight_min": -10.318159103393555,
      "activations/layer10_attention_weight_max": 36.710350036621094,
      "activations/layer10_attention_weight_min": -31.50946044921875,
      "activations/layer11_attention_weight_max": 35.81939697265625,
      "activations/layer11_attention_weight_min": -31.233779907226562,
      "activations/layer12_attention_weight_max": 23.91022491455078,
      "activations/layer12_attention_weight_min": -24.915069580078125,
      "activations/layer13_attention_weight_max": 41.339019775390625,
      "activations/layer13_attention_weight_min": -32.43666458129883,
      "activations/layer14_attention_weight_max": 51.634212493896484,
      "activations/layer14_attention_weight_min": -40.81430435180664,
      "activations/layer15_attention_weight_max": 36.12705993652344,
      "activations/layer15_attention_weight_min": -29.938037872314453,
      "activations/layer16_attention_weight_max": 32.54991912841797,
      "activations/layer16_attention_weight_min": -27.110431671142578,
      "activations/layer17_attention_weight_max": 35.15448760986328,
      "activations/layer17_attention_weight_min": -23.88010597229004,
      "activations/layer18_attention_weight_max": 30.998849868774414,
      "activations/layer18_attention_weight_min": -20.237638473510742,
      "activations/layer19_attention_weight_max": 37.17031478881836,
      "activations/layer19_attention_weight_min": -28.463247299194336,
      "activations/layer1_attention_weight_max": 16.696855545043945,
      "activations/layer1_attention_weight_min": -13.085607528686523,
      "activations/layer20_attention_weight_max": 30.925718307495117,
      "activations/layer20_attention_weight_min": -22.4805850982666,
      "activations/layer21_attention_weight_max": 28.44169807434082,
      "activations/layer21_attention_weight_min": -20.03226661682129,
      "activations/layer22_attention_weight_max": 48.7420768737793,
      "activations/layer22_attention_weight_min": -27.967405319213867,
      "activations/layer23_attention_weight_max": 39.26042175292969,
      "activations/layer23_attention_weight_min": -21.183910369873047,
      "activations/layer2_attention_weight_max": 30.218101501464844,
      "activations/layer2_attention_weight_min": -29.744258880615234,
      "activations/layer3_attention_weight_max": 86.1511001586914,
      "activations/layer3_attention_weight_min": -88.95417785644531,
      "activations/layer4_attention_weight_max": 121.7084732055664,
      "activations/layer4_attention_weight_min": -120.9632797241211,
      "activations/layer5_attention_weight_max": 65.71232604980469,
      "activations/layer5_attention_weight_min": -63.77848434448242,
      "activations/layer6_attention_weight_max": 52.559993743896484,
      "activations/layer6_attention_weight_min": -54.73221969604492,
      "activations/layer7_attention_weight_max": 88.07415771484375,
      "activations/layer7_attention_weight_min": -92.12360382080078,
      "activations/layer8_attention_weight_max": 41.72563552856445,
      "activations/layer8_attention_weight_min": -42.92277145385742,
      "activations/layer9_attention_weight_max": 36.6419563293457,
      "activations/layer9_attention_weight_min": -36.66035842895508,
      "epoch": 6.95,
      "learning_rate": 0.00010625378787878787,
      "loss": 2.8255,
      "step": 119600
    },
    {
      "activations/layer0_attention_weight_max": 16.047998428344727,
      "activations/layer0_attention_weight_min": -11.813539505004883,
      "activations/layer10_attention_weight_max": 33.45274353027344,
      "activations/layer10_attention_weight_min": -35.035701751708984,
      "activations/layer11_attention_weight_max": 32.698036193847656,
      "activations/layer11_attention_weight_min": -30.423450469970703,
      "activations/layer12_attention_weight_max": 22.922931671142578,
      "activations/layer12_attention_weight_min": -24.90047836303711,
      "activations/layer13_attention_weight_max": 38.87067794799805,
      "activations/layer13_attention_weight_min": -31.747962951660156,
      "activations/layer14_attention_weight_max": 53.36368179321289,
      "activations/layer14_attention_weight_min": -41.10386276245117,
      "activations/layer15_attention_weight_max": 36.25006103515625,
      "activations/layer15_attention_weight_min": -30.491809844970703,
      "activations/layer16_attention_weight_max": 33.690914154052734,
      "activations/layer16_attention_weight_min": -26.753902435302734,
      "activations/layer17_attention_weight_max": 33.69595718383789,
      "activations/layer17_attention_weight_min": -23.743255615234375,
      "activations/layer18_attention_weight_max": 31.04732894897461,
      "activations/layer18_attention_weight_min": -20.151287078857422,
      "activations/layer19_attention_weight_max": 38.38572311401367,
      "activations/layer19_attention_weight_min": -31.452392578125,
      "activations/layer1_attention_weight_max": 17.08951759338379,
      "activations/layer1_attention_weight_min": -13.102707862854004,
      "activations/layer20_attention_weight_max": 30.834400177001953,
      "activations/layer20_attention_weight_min": -24.955467224121094,
      "activations/layer21_attention_weight_max": 30.056785583496094,
      "activations/layer21_attention_weight_min": -21.569704055786133,
      "activations/layer22_attention_weight_max": 50.55976104736328,
      "activations/layer22_attention_weight_min": -27.95116424560547,
      "activations/layer23_attention_weight_max": 42.89703369140625,
      "activations/layer23_attention_weight_min": -22.07207489013672,
      "activations/layer2_attention_weight_max": 32.97179412841797,
      "activations/layer2_attention_weight_min": -30.96047592163086,
      "activations/layer3_attention_weight_max": 89.0951156616211,
      "activations/layer3_attention_weight_min": -87.67926025390625,
      "activations/layer4_attention_weight_max": 122.79788970947266,
      "activations/layer4_attention_weight_min": -119.30425262451172,
      "activations/layer5_attention_weight_max": 64.20140838623047,
      "activations/layer5_attention_weight_min": -62.393150329589844,
      "activations/layer6_attention_weight_max": 54.10137176513672,
      "activations/layer6_attention_weight_min": -54.57102966308594,
      "activations/layer7_attention_weight_max": 86.21295166015625,
      "activations/layer7_attention_weight_min": -85.39865112304688,
      "activations/layer8_attention_weight_max": 41.72572708129883,
      "activations/layer8_attention_weight_min": -42.859615325927734,
      "activations/layer9_attention_weight_max": 37.71110916137695,
      "activations/layer9_attention_weight_min": -34.9347038269043,
      "epoch": 6.95,
      "learning_rate": 0.00010623484848484848,
      "loss": 2.8055,
      "step": 119650
    },
    {
      "activations/layer0_attention_weight_max": 16.271808624267578,
      "activations/layer0_attention_weight_min": -11.74572467803955,
      "activations/layer10_attention_weight_max": 35.2414436340332,
      "activations/layer10_attention_weight_min": -33.40266799926758,
      "activations/layer11_attention_weight_max": 36.17934799194336,
      "activations/layer11_attention_weight_min": -29.855026245117188,
      "activations/layer12_attention_weight_max": 25.963802337646484,
      "activations/layer12_attention_weight_min": -24.932706832885742,
      "activations/layer13_attention_weight_max": 49.311824798583984,
      "activations/layer13_attention_weight_min": -33.46751022338867,
      "activations/layer14_attention_weight_max": 63.81232452392578,
      "activations/layer14_attention_weight_min": -43.21072769165039,
      "activations/layer15_attention_weight_max": 39.800926208496094,
      "activations/layer15_attention_weight_min": -28.809629440307617,
      "activations/layer16_attention_weight_max": 38.49816131591797,
      "activations/layer16_attention_weight_min": -28.436113357543945,
      "activations/layer17_attention_weight_max": 42.052459716796875,
      "activations/layer17_attention_weight_min": -25.289695739746094,
      "activations/layer18_attention_weight_max": 34.51976013183594,
      "activations/layer18_attention_weight_min": -20.517309188842773,
      "activations/layer19_attention_weight_max": 40.24388122558594,
      "activations/layer19_attention_weight_min": -30.535573959350586,
      "activations/layer1_attention_weight_max": 16.88132095336914,
      "activations/layer1_attention_weight_min": -14.350693702697754,
      "activations/layer20_attention_weight_max": 39.54389572143555,
      "activations/layer20_attention_weight_min": -24.577098846435547,
      "activations/layer21_attention_weight_max": 35.34409713745117,
      "activations/layer21_attention_weight_min": -23.04784393310547,
      "activations/layer22_attention_weight_max": 53.345985412597656,
      "activations/layer22_attention_weight_min": -27.73093032836914,
      "activations/layer23_attention_weight_max": 35.62229919433594,
      "activations/layer23_attention_weight_min": -24.837568283081055,
      "activations/layer2_attention_weight_max": 31.97602081298828,
      "activations/layer2_attention_weight_min": -30.4931640625,
      "activations/layer3_attention_weight_max": 90.21685791015625,
      "activations/layer3_attention_weight_min": -90.24977111816406,
      "activations/layer4_attention_weight_max": 128.2269744873047,
      "activations/layer4_attention_weight_min": -121.84034729003906,
      "activations/layer5_attention_weight_max": 67.85983276367188,
      "activations/layer5_attention_weight_min": -63.71919250488281,
      "activations/layer6_attention_weight_max": 55.12914276123047,
      "activations/layer6_attention_weight_min": -56.329010009765625,
      "activations/layer7_attention_weight_max": 94.72346496582031,
      "activations/layer7_attention_weight_min": -92.18920135498047,
      "activations/layer8_attention_weight_max": 46.094207763671875,
      "activations/layer8_attention_weight_min": -44.978233337402344,
      "activations/layer9_attention_weight_max": 38.977046966552734,
      "activations/layer9_attention_weight_min": -35.481327056884766,
      "epoch": 6.96,
      "learning_rate": 0.00010621590909090909,
      "loss": 2.8086,
      "step": 119700
    },
    {
      "activations/layer0_attention_weight_max": 16.29297637939453,
      "activations/layer0_attention_weight_min": -12.226800918579102,
      "activations/layer10_attention_weight_max": 32.84939193725586,
      "activations/layer10_attention_weight_min": -33.29628372192383,
      "activations/layer11_attention_weight_max": 31.697418212890625,
      "activations/layer11_attention_weight_min": -32.51081848144531,
      "activations/layer12_attention_weight_max": 26.670841217041016,
      "activations/layer12_attention_weight_min": -26.094602584838867,
      "activations/layer13_attention_weight_max": 52.06086349487305,
      "activations/layer13_attention_weight_min": -38.182777404785156,
      "activations/layer14_attention_weight_max": 60.91713333129883,
      "activations/layer14_attention_weight_min": -43.92912292480469,
      "activations/layer15_attention_weight_max": 39.121131896972656,
      "activations/layer15_attention_weight_min": -29.565723419189453,
      "activations/layer16_attention_weight_max": 35.81451416015625,
      "activations/layer16_attention_weight_min": -27.990482330322266,
      "activations/layer17_attention_weight_max": 34.259342193603516,
      "activations/layer17_attention_weight_min": -26.82626724243164,
      "activations/layer18_attention_weight_max": 34.162017822265625,
      "activations/layer18_attention_weight_min": -21.4350643157959,
      "activations/layer19_attention_weight_max": 38.29840850830078,
      "activations/layer19_attention_weight_min": -28.11617088317871,
      "activations/layer1_attention_weight_max": 17.142698287963867,
      "activations/layer1_attention_weight_min": -13.157352447509766,
      "activations/layer20_attention_weight_max": 34.37883377075195,
      "activations/layer20_attention_weight_min": -23.047027587890625,
      "activations/layer21_attention_weight_max": 35.12648391723633,
      "activations/layer21_attention_weight_min": -22.307493209838867,
      "activations/layer22_attention_weight_max": 50.41817092895508,
      "activations/layer22_attention_weight_min": -28.315237045288086,
      "activations/layer23_attention_weight_max": 35.865264892578125,
      "activations/layer23_attention_weight_min": -25.81625747680664,
      "activations/layer2_attention_weight_max": 29.283573150634766,
      "activations/layer2_attention_weight_min": -29.343725204467773,
      "activations/layer3_attention_weight_max": 87.10027313232422,
      "activations/layer3_attention_weight_min": -88.81437683105469,
      "activations/layer4_attention_weight_max": 117.50239562988281,
      "activations/layer4_attention_weight_min": -117.16234588623047,
      "activations/layer5_attention_weight_max": 60.180538177490234,
      "activations/layer5_attention_weight_min": -60.48406219482422,
      "activations/layer6_attention_weight_max": 47.9450569152832,
      "activations/layer6_attention_weight_min": -51.74338150024414,
      "activations/layer7_attention_weight_max": 83.0500717163086,
      "activations/layer7_attention_weight_min": -84.65019226074219,
      "activations/layer8_attention_weight_max": 38.97712707519531,
      "activations/layer8_attention_weight_min": -40.45689010620117,
      "activations/layer9_attention_weight_max": 34.50390625,
      "activations/layer9_attention_weight_min": -35.307132720947266,
      "epoch": 6.96,
      "learning_rate": 0.00010619696969696968,
      "loss": 2.8322,
      "step": 119750
    },
    {
      "activations/layer0_attention_weight_max": 16.109786987304688,
      "activations/layer0_attention_weight_min": -12.19670295715332,
      "activations/layer10_attention_weight_max": 37.529136657714844,
      "activations/layer10_attention_weight_min": -33.8597297668457,
      "activations/layer11_attention_weight_max": 37.881378173828125,
      "activations/layer11_attention_weight_min": -34.79576873779297,
      "activations/layer12_attention_weight_max": 26.85194206237793,
      "activations/layer12_attention_weight_min": -31.208131790161133,
      "activations/layer13_attention_weight_max": 37.660865783691406,
      "activations/layer13_attention_weight_min": -33.43708801269531,
      "activations/layer14_attention_weight_max": 52.727577209472656,
      "activations/layer14_attention_weight_min": -43.24192810058594,
      "activations/layer15_attention_weight_max": 34.79975509643555,
      "activations/layer15_attention_weight_min": -29.20876693725586,
      "activations/layer16_attention_weight_max": 33.2426872253418,
      "activations/layer16_attention_weight_min": -27.36224365234375,
      "activations/layer17_attention_weight_max": 32.815040588378906,
      "activations/layer17_attention_weight_min": -24.461008071899414,
      "activations/layer18_attention_weight_max": 28.719486236572266,
      "activations/layer18_attention_weight_min": -23.401363372802734,
      "activations/layer19_attention_weight_max": 34.80012893676758,
      "activations/layer19_attention_weight_min": -30.42852783203125,
      "activations/layer1_attention_weight_max": 17.448488235473633,
      "activations/layer1_attention_weight_min": -13.814330101013184,
      "activations/layer20_attention_weight_max": 28.174304962158203,
      "activations/layer20_attention_weight_min": -24.488996505737305,
      "activations/layer21_attention_weight_max": 28.779541015625,
      "activations/layer21_attention_weight_min": -23.140886306762695,
      "activations/layer22_attention_weight_max": 44.90751266479492,
      "activations/layer22_attention_weight_min": -31.303417205810547,
      "activations/layer23_attention_weight_max": 33.60927200317383,
      "activations/layer23_attention_weight_min": -22.564603805541992,
      "activations/layer2_attention_weight_max": 30.207263946533203,
      "activations/layer2_attention_weight_min": -30.181447982788086,
      "activations/layer3_attention_weight_max": 90.35185241699219,
      "activations/layer3_attention_weight_min": -89.31696319580078,
      "activations/layer4_attention_weight_max": 122.26619720458984,
      "activations/layer4_attention_weight_min": -118.0481185913086,
      "activations/layer5_attention_weight_max": 67.45591735839844,
      "activations/layer5_attention_weight_min": -61.518367767333984,
      "activations/layer6_attention_weight_max": 57.77452087402344,
      "activations/layer6_attention_weight_min": -51.238792419433594,
      "activations/layer7_attention_weight_max": 106.5743408203125,
      "activations/layer7_attention_weight_min": -96.19116973876953,
      "activations/layer8_attention_weight_max": 48.239383697509766,
      "activations/layer8_attention_weight_min": -44.302188873291016,
      "activations/layer9_attention_weight_max": 43.932464599609375,
      "activations/layer9_attention_weight_min": -38.49617385864258,
      "epoch": 6.96,
      "learning_rate": 0.0001061780303030303,
      "loss": 2.798,
      "step": 119800
    },
    {
      "activations/layer0_attention_weight_max": 16.281713485717773,
      "activations/layer0_attention_weight_min": -11.254148483276367,
      "activations/layer10_attention_weight_max": 32.9298210144043,
      "activations/layer10_attention_weight_min": -30.778606414794922,
      "activations/layer11_attention_weight_max": 33.599178314208984,
      "activations/layer11_attention_weight_min": -28.625850677490234,
      "activations/layer12_attention_weight_max": 23.152725219726562,
      "activations/layer12_attention_weight_min": -23.947189331054688,
      "activations/layer13_attention_weight_max": 39.15919876098633,
      "activations/layer13_attention_weight_min": -33.63486862182617,
      "activations/layer14_attention_weight_max": 53.42776870727539,
      "activations/layer14_attention_weight_min": -39.659671783447266,
      "activations/layer15_attention_weight_max": 35.028934478759766,
      "activations/layer15_attention_weight_min": -30.59248924255371,
      "activations/layer16_attention_weight_max": 35.73224639892578,
      "activations/layer16_attention_weight_min": -28.64990997314453,
      "activations/layer17_attention_weight_max": 37.62746047973633,
      "activations/layer17_attention_weight_min": -25.711273193359375,
      "activations/layer18_attention_weight_max": 31.700008392333984,
      "activations/layer18_attention_weight_min": -22.898412704467773,
      "activations/layer19_attention_weight_max": 38.785179138183594,
      "activations/layer19_attention_weight_min": -29.03132438659668,
      "activations/layer1_attention_weight_max": 17.010072708129883,
      "activations/layer1_attention_weight_min": -13.705452919006348,
      "activations/layer20_attention_weight_max": 31.954883575439453,
      "activations/layer20_attention_weight_min": -25.706192016601562,
      "activations/layer21_attention_weight_max": 30.92094612121582,
      "activations/layer21_attention_weight_min": -23.053781509399414,
      "activations/layer22_attention_weight_max": 49.21867752075195,
      "activations/layer22_attention_weight_min": -29.21314239501953,
      "activations/layer23_attention_weight_max": 36.969154357910156,
      "activations/layer23_attention_weight_min": -22.00465965270996,
      "activations/layer2_attention_weight_max": 31.633642196655273,
      "activations/layer2_attention_weight_min": -31.711759567260742,
      "activations/layer3_attention_weight_max": 91.35828399658203,
      "activations/layer3_attention_weight_min": -91.14691162109375,
      "activations/layer4_attention_weight_max": 124.94807434082031,
      "activations/layer4_attention_weight_min": -119.3418197631836,
      "activations/layer5_attention_weight_max": 65.13343048095703,
      "activations/layer5_attention_weight_min": -63.115455627441406,
      "activations/layer6_attention_weight_max": 52.947322845458984,
      "activations/layer6_attention_weight_min": -53.786170959472656,
      "activations/layer7_attention_weight_max": 89.17069244384766,
      "activations/layer7_attention_weight_min": -89.80136108398438,
      "activations/layer8_attention_weight_max": 40.55641555786133,
      "activations/layer8_attention_weight_min": -42.27073669433594,
      "activations/layer9_attention_weight_max": 35.082576751708984,
      "activations/layer9_attention_weight_min": -34.816688537597656,
      "epoch": 6.96,
      "learning_rate": 0.00010615909090909091,
      "loss": 2.8219,
      "step": 119850
    },
    {
      "activations/layer0_attention_weight_max": 16.895185470581055,
      "activations/layer0_attention_weight_min": -11.557694435119629,
      "activations/layer10_attention_weight_max": 31.600215911865234,
      "activations/layer10_attention_weight_min": -32.66542434692383,
      "activations/layer11_attention_weight_max": 29.48245620727539,
      "activations/layer11_attention_weight_min": -29.27461814880371,
      "activations/layer12_attention_weight_max": 26.252843856811523,
      "activations/layer12_attention_weight_min": -26.720857620239258,
      "activations/layer13_attention_weight_max": 42.03643035888672,
      "activations/layer13_attention_weight_min": -33.82383728027344,
      "activations/layer14_attention_weight_max": 51.74991226196289,
      "activations/layer14_attention_weight_min": -42.82439041137695,
      "activations/layer15_attention_weight_max": 36.88590621948242,
      "activations/layer15_attention_weight_min": -28.566213607788086,
      "activations/layer16_attention_weight_max": 31.932775497436523,
      "activations/layer16_attention_weight_min": -27.25889015197754,
      "activations/layer17_attention_weight_max": 34.08047103881836,
      "activations/layer17_attention_weight_min": -25.479806900024414,
      "activations/layer18_attention_weight_max": 30.012834548950195,
      "activations/layer18_attention_weight_min": -21.17814826965332,
      "activations/layer19_attention_weight_max": 36.38018798828125,
      "activations/layer19_attention_weight_min": -28.852506637573242,
      "activations/layer1_attention_weight_max": 17.58087921142578,
      "activations/layer1_attention_weight_min": -14.571383476257324,
      "activations/layer20_attention_weight_max": 32.49659729003906,
      "activations/layer20_attention_weight_min": -23.974943161010742,
      "activations/layer21_attention_weight_max": 30.714683532714844,
      "activations/layer21_attention_weight_min": -22.75263023376465,
      "activations/layer22_attention_weight_max": 49.35599899291992,
      "activations/layer22_attention_weight_min": -28.1865177154541,
      "activations/layer23_attention_weight_max": 34.90370178222656,
      "activations/layer23_attention_weight_min": -21.907958984375,
      "activations/layer2_attention_weight_max": 32.599578857421875,
      "activations/layer2_attention_weight_min": -30.869468688964844,
      "activations/layer3_attention_weight_max": 86.42566680908203,
      "activations/layer3_attention_weight_min": -88.47029876708984,
      "activations/layer4_attention_weight_max": 116.461181640625,
      "activations/layer4_attention_weight_min": -120.53018951416016,
      "activations/layer5_attention_weight_max": 63.66438293457031,
      "activations/layer5_attention_weight_min": -68.3228759765625,
      "activations/layer6_attention_weight_max": 50.88323211669922,
      "activations/layer6_attention_weight_min": -53.91633605957031,
      "activations/layer7_attention_weight_max": 84.24212646484375,
      "activations/layer7_attention_weight_min": -88.53559875488281,
      "activations/layer8_attention_weight_max": 37.78315734863281,
      "activations/layer8_attention_weight_min": -41.2845344543457,
      "activations/layer9_attention_weight_max": 34.78170394897461,
      "activations/layer9_attention_weight_min": -36.505218505859375,
      "epoch": 6.97,
      "learning_rate": 0.0001061401515151515,
      "loss": 2.8364,
      "step": 119900
    },
    {
      "activations/layer0_attention_weight_max": 16.01876449584961,
      "activations/layer0_attention_weight_min": -12.384248733520508,
      "activations/layer10_attention_weight_max": 30.792198181152344,
      "activations/layer10_attention_weight_min": -31.942968368530273,
      "activations/layer11_attention_weight_max": 32.331764221191406,
      "activations/layer11_attention_weight_min": -31.510047912597656,
      "activations/layer12_attention_weight_max": 22.021543502807617,
      "activations/layer12_attention_weight_min": -27.278926849365234,
      "activations/layer13_attention_weight_max": 37.827247619628906,
      "activations/layer13_attention_weight_min": -32.81699752807617,
      "activations/layer14_attention_weight_max": 52.55647659301758,
      "activations/layer14_attention_weight_min": -44.86708450317383,
      "activations/layer15_attention_weight_max": 36.024532318115234,
      "activations/layer15_attention_weight_min": -29.54958152770996,
      "activations/layer16_attention_weight_max": 34.24053192138672,
      "activations/layer16_attention_weight_min": -27.298803329467773,
      "activations/layer17_attention_weight_max": 33.32392883300781,
      "activations/layer17_attention_weight_min": -24.069217681884766,
      "activations/layer18_attention_weight_max": 30.482511520385742,
      "activations/layer18_attention_weight_min": -21.399667739868164,
      "activations/layer19_attention_weight_max": 38.485904693603516,
      "activations/layer19_attention_weight_min": -30.714094161987305,
      "activations/layer1_attention_weight_max": 16.96649932861328,
      "activations/layer1_attention_weight_min": -13.85328197479248,
      "activations/layer20_attention_weight_max": 31.93327522277832,
      "activations/layer20_attention_weight_min": -22.342754364013672,
      "activations/layer21_attention_weight_max": 29.736644744873047,
      "activations/layer21_attention_weight_min": -20.510393142700195,
      "activations/layer22_attention_weight_max": 46.96994400024414,
      "activations/layer22_attention_weight_min": -28.69866180419922,
      "activations/layer23_attention_weight_max": 32.528289794921875,
      "activations/layer23_attention_weight_min": -22.962299346923828,
      "activations/layer2_attention_weight_max": 31.561206817626953,
      "activations/layer2_attention_weight_min": -32.320613861083984,
      "activations/layer3_attention_weight_max": 93.5227279663086,
      "activations/layer3_attention_weight_min": -96.66925811767578,
      "activations/layer4_attention_weight_max": 123.75410461425781,
      "activations/layer4_attention_weight_min": -122.45863342285156,
      "activations/layer5_attention_weight_max": 60.82583236694336,
      "activations/layer5_attention_weight_min": -62.31707763671875,
      "activations/layer6_attention_weight_max": 50.074058532714844,
      "activations/layer6_attention_weight_min": -52.007198333740234,
      "activations/layer7_attention_weight_max": 88.10906982421875,
      "activations/layer7_attention_weight_min": -91.0647964477539,
      "activations/layer8_attention_weight_max": 38.845664978027344,
      "activations/layer8_attention_weight_min": -43.25383377075195,
      "activations/layer9_attention_weight_max": 32.170780181884766,
      "activations/layer9_attention_weight_min": -37.498836517333984,
      "epoch": 6.97,
      "learning_rate": 0.00010612121212121211,
      "loss": 2.8242,
      "step": 119950
    },
    {
      "activations/layer0_attention_weight_max": 15.435181617736816,
      "activations/layer0_attention_weight_min": -12.685418128967285,
      "activations/layer10_attention_weight_max": 46.2726936340332,
      "activations/layer10_attention_weight_min": -39.782569885253906,
      "activations/layer11_attention_weight_max": 50.44355392456055,
      "activations/layer11_attention_weight_min": -41.109073638916016,
      "activations/layer12_attention_weight_max": 51.54002380371094,
      "activations/layer12_attention_weight_min": -30.652183532714844,
      "activations/layer13_attention_weight_max": 89.54554748535156,
      "activations/layer13_attention_weight_min": -38.51332092285156,
      "activations/layer14_attention_weight_max": 79.79013061523438,
      "activations/layer14_attention_weight_min": -42.51860427856445,
      "activations/layer15_attention_weight_max": 76.08292388916016,
      "activations/layer15_attention_weight_min": -35.05607986450195,
      "activations/layer16_attention_weight_max": 39.3650016784668,
      "activations/layer16_attention_weight_min": -30.789112091064453,
      "activations/layer17_attention_weight_max": 34.27192687988281,
      "activations/layer17_attention_weight_min": -27.948246002197266,
      "activations/layer18_attention_weight_max": 31.907644271850586,
      "activations/layer18_attention_weight_min": -21.893329620361328,
      "activations/layer19_attention_weight_max": 39.14271545410156,
      "activations/layer19_attention_weight_min": -29.33782958984375,
      "activations/layer1_attention_weight_max": 16.745779037475586,
      "activations/layer1_attention_weight_min": -14.074799537658691,
      "activations/layer20_attention_weight_max": 31.057016372680664,
      "activations/layer20_attention_weight_min": -23.615137100219727,
      "activations/layer21_attention_weight_max": 30.322174072265625,
      "activations/layer21_attention_weight_min": -21.442073822021484,
      "activations/layer22_attention_weight_max": 56.2693977355957,
      "activations/layer22_attention_weight_min": -28.205341339111328,
      "activations/layer23_attention_weight_max": 37.46361541748047,
      "activations/layer23_attention_weight_min": -20.192306518554688,
      "activations/layer2_attention_weight_max": 32.920806884765625,
      "activations/layer2_attention_weight_min": -32.550392150878906,
      "activations/layer3_attention_weight_max": 93.99454498291016,
      "activations/layer3_attention_weight_min": -97.33529663085938,
      "activations/layer4_attention_weight_max": 135.20945739746094,
      "activations/layer4_attention_weight_min": -133.01559448242188,
      "activations/layer5_attention_weight_max": 72.09228515625,
      "activations/layer5_attention_weight_min": -69.62342071533203,
      "activations/layer6_attention_weight_max": 57.94075012207031,
      "activations/layer6_attention_weight_min": -54.82832336425781,
      "activations/layer7_attention_weight_max": 110.14086151123047,
      "activations/layer7_attention_weight_min": -104.23342895507812,
      "activations/layer8_attention_weight_max": 51.09511184692383,
      "activations/layer8_attention_weight_min": -46.51372146606445,
      "activations/layer9_attention_weight_max": 47.28248596191406,
      "activations/layer9_attention_weight_min": -42.073028564453125,
      "epoch": 6.97,
      "learning_rate": 0.00010610227272727273,
      "loss": 2.8197,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.4962,
      "eval_samples_per_second": 505.403,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.4962,
      "eval_openwebtext_samples_per_second": 505.403,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 1.9659,
      "eval_wikitext_samples_per_second": 231.952,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_lambada_loss": 2.669921875,
      "eval_lambada_ppl": 14.438841114275835,
      "eval_lambada_runtime": 9.6334,
      "eval_lambada_samples_per_second": 505.426,
      "step": 120000
    },
    {
      "activations/layer0_attention_weight_max": 16.259445190429688,
      "activations/layer0_attention_weight_min": -11.208871841430664,
      "activations/layer10_attention_weight_max": 34.5670166015625,
      "activations/layer10_attention_weight_min": -31.20928192138672,
      "activations/layer11_attention_weight_max": 35.46454620361328,
      "activations/layer11_attention_weight_min": -30.043380737304688,
      "activations/layer12_attention_weight_max": 37.07573318481445,
      "activations/layer12_attention_weight_min": -28.380571365356445,
      "activations/layer13_attention_weight_max": 49.37156295776367,
      "activations/layer13_attention_weight_min": -33.70933151245117,
      "activations/layer14_attention_weight_max": 56.334590911865234,
      "activations/layer14_attention_weight_min": -40.65052032470703,
      "activations/layer15_attention_weight_max": 47.01080322265625,
      "activations/layer15_attention_weight_min": -29.76594352722168,
      "activations/layer16_attention_weight_max": 39.36928939819336,
      "activations/layer16_attention_weight_min": -28.26852035522461,
      "activations/layer17_attention_weight_max": 37.723289489746094,
      "activations/layer17_attention_weight_min": -25.8837833404541,
      "activations/layer18_attention_weight_max": 34.309661865234375,
      "activations/layer18_attention_weight_min": -20.416622161865234,
      "activations/layer19_attention_weight_max": 41.60954284667969,
      "activations/layer19_attention_weight_min": -28.939992904663086,
      "activations/layer1_attention_weight_max": 16.635814666748047,
      "activations/layer1_attention_weight_min": -15.036375999450684,
      "activations/layer20_attention_weight_max": 35.33806610107422,
      "activations/layer20_attention_weight_min": -23.671987533569336,
      "activations/layer21_attention_weight_max": 32.96851348876953,
      "activations/layer21_attention_weight_min": -22.311683654785156,
      "activations/layer22_attention_weight_max": 48.07671356201172,
      "activations/layer22_attention_weight_min": -27.951950073242188,
      "activations/layer23_attention_weight_max": 36.7995719909668,
      "activations/layer23_attention_weight_min": -21.208755493164062,
      "activations/layer2_attention_weight_max": 28.867279052734375,
      "activations/layer2_attention_weight_min": -28.50624656677246,
      "activations/layer3_attention_weight_max": 92.66976928710938,
      "activations/layer3_attention_weight_min": -92.07080078125,
      "activations/layer4_attention_weight_max": 127.48347473144531,
      "activations/layer4_attention_weight_min": -118.08367156982422,
      "activations/layer5_attention_weight_max": 64.33953857421875,
      "activations/layer5_attention_weight_min": -63.463775634765625,
      "activations/layer6_attention_weight_max": 52.26746368408203,
      "activations/layer6_attention_weight_min": -53.002933502197266,
      "activations/layer7_attention_weight_max": 87.10670471191406,
      "activations/layer7_attention_weight_min": -87.812744140625,
      "activations/layer8_attention_weight_max": 42.46541213989258,
      "activations/layer8_attention_weight_min": -39.40913009643555,
      "activations/layer9_attention_weight_max": 36.20765686035156,
      "activations/layer9_attention_weight_min": -34.13758087158203,
      "epoch": 6.98,
      "learning_rate": 0.00010608333333333331,
      "loss": 2.8328,
      "step": 120050
    },
    {
      "activations/layer0_attention_weight_max": 16.39659309387207,
      "activations/layer0_attention_weight_min": -11.818535804748535,
      "activations/layer10_attention_weight_max": 32.27965545654297,
      "activations/layer10_attention_weight_min": -33.27899932861328,
      "activations/layer11_attention_weight_max": 31.76067352294922,
      "activations/layer11_attention_weight_min": -28.844282150268555,
      "activations/layer12_attention_weight_max": 21.718021392822266,
      "activations/layer12_attention_weight_min": -27.38758659362793,
      "activations/layer13_attention_weight_max": 35.9778938293457,
      "activations/layer13_attention_weight_min": -32.59245300292969,
      "activations/layer14_attention_weight_max": 45.81491470336914,
      "activations/layer14_attention_weight_min": -39.85436248779297,
      "activations/layer15_attention_weight_max": 32.73352813720703,
      "activations/layer15_attention_weight_min": -30.008058547973633,
      "activations/layer16_attention_weight_max": 30.72015380859375,
      "activations/layer16_attention_weight_min": -26.14101219177246,
      "activations/layer17_attention_weight_max": 33.257781982421875,
      "activations/layer17_attention_weight_min": -24.07052230834961,
      "activations/layer18_attention_weight_max": 33.0382194519043,
      "activations/layer18_attention_weight_min": -20.602155685424805,
      "activations/layer19_attention_weight_max": 36.42716598510742,
      "activations/layer19_attention_weight_min": -30.41742515563965,
      "activations/layer1_attention_weight_max": 17.65755844116211,
      "activations/layer1_attention_weight_min": -15.035503387451172,
      "activations/layer20_attention_weight_max": 28.450088500976562,
      "activations/layer20_attention_weight_min": -23.96351432800293,
      "activations/layer21_attention_weight_max": 26.773210525512695,
      "activations/layer21_attention_weight_min": -23.770584106445312,
      "activations/layer22_attention_weight_max": 45.86289978027344,
      "activations/layer22_attention_weight_min": -29.45594024658203,
      "activations/layer23_attention_weight_max": 32.72412872314453,
      "activations/layer23_attention_weight_min": -22.79425621032715,
      "activations/layer2_attention_weight_max": 31.559242248535156,
      "activations/layer2_attention_weight_min": -30.705915451049805,
      "activations/layer3_attention_weight_max": 92.62155151367188,
      "activations/layer3_attention_weight_min": -94.69506072998047,
      "activations/layer4_attention_weight_max": 129.07073974609375,
      "activations/layer4_attention_weight_min": -123.05077362060547,
      "activations/layer5_attention_weight_max": 63.95948791503906,
      "activations/layer5_attention_weight_min": -66.04976654052734,
      "activations/layer6_attention_weight_max": 52.01992416381836,
      "activations/layer6_attention_weight_min": -53.18251037597656,
      "activations/layer7_attention_weight_max": 87.86688995361328,
      "activations/layer7_attention_weight_min": -88.38587951660156,
      "activations/layer8_attention_weight_max": 39.9856071472168,
      "activations/layer8_attention_weight_min": -39.39902877807617,
      "activations/layer9_attention_weight_max": 33.571353912353516,
      "activations/layer9_attention_weight_min": -36.45935821533203,
      "epoch": 6.98,
      "learning_rate": 0.00010606439393939393,
      "loss": 2.8345,
      "step": 120100
    },
    {
      "activations/layer0_attention_weight_max": 15.89494800567627,
      "activations/layer0_attention_weight_min": -11.497589111328125,
      "activations/layer10_attention_weight_max": 34.38039016723633,
      "activations/layer10_attention_weight_min": -29.928762435913086,
      "activations/layer11_attention_weight_max": 32.00444793701172,
      "activations/layer11_attention_weight_min": -29.52880859375,
      "activations/layer12_attention_weight_max": 21.780155181884766,
      "activations/layer12_attention_weight_min": -26.38700294494629,
      "activations/layer13_attention_weight_max": 42.90074157714844,
      "activations/layer13_attention_weight_min": -33.41275405883789,
      "activations/layer14_attention_weight_max": 50.03961181640625,
      "activations/layer14_attention_weight_min": -40.552642822265625,
      "activations/layer15_attention_weight_max": 35.67348098754883,
      "activations/layer15_attention_weight_min": -30.389345169067383,
      "activations/layer16_attention_weight_max": 31.169918060302734,
      "activations/layer16_attention_weight_min": -28.774763107299805,
      "activations/layer17_attention_weight_max": 35.93223571777344,
      "activations/layer17_attention_weight_min": -25.326854705810547,
      "activations/layer18_attention_weight_max": 28.693397521972656,
      "activations/layer18_attention_weight_min": -21.320167541503906,
      "activations/layer19_attention_weight_max": 36.904273986816406,
      "activations/layer19_attention_weight_min": -29.15911865234375,
      "activations/layer1_attention_weight_max": 16.818471908569336,
      "activations/layer1_attention_weight_min": -14.085744857788086,
      "activations/layer20_attention_weight_max": 29.19322395324707,
      "activations/layer20_attention_weight_min": -24.287654876708984,
      "activations/layer21_attention_weight_max": 29.14067840576172,
      "activations/layer21_attention_weight_min": -21.64192771911621,
      "activations/layer22_attention_weight_max": 46.58902359008789,
      "activations/layer22_attention_weight_min": -26.84189796447754,
      "activations/layer23_attention_weight_max": 36.14116287231445,
      "activations/layer23_attention_weight_min": -21.939044952392578,
      "activations/layer2_attention_weight_max": 30.71070671081543,
      "activations/layer2_attention_weight_min": -30.576574325561523,
      "activations/layer3_attention_weight_max": 91.89804077148438,
      "activations/layer3_attention_weight_min": -92.75760650634766,
      "activations/layer4_attention_weight_max": 124.14701080322266,
      "activations/layer4_attention_weight_min": -120.611083984375,
      "activations/layer5_attention_weight_max": 63.925758361816406,
      "activations/layer5_attention_weight_min": -62.91759490966797,
      "activations/layer6_attention_weight_max": 51.65689468383789,
      "activations/layer6_attention_weight_min": -53.445457458496094,
      "activations/layer7_attention_weight_max": 83.20301055908203,
      "activations/layer7_attention_weight_min": -84.89894104003906,
      "activations/layer8_attention_weight_max": 39.27569580078125,
      "activations/layer8_attention_weight_min": -39.01411819458008,
      "activations/layer9_attention_weight_max": 35.78306579589844,
      "activations/layer9_attention_weight_min": -33.308837890625,
      "epoch": 6.98,
      "learning_rate": 0.00010604545454545453,
      "loss": 2.8348,
      "step": 120150
    },
    {
      "activations/layer0_attention_weight_max": 16.364421844482422,
      "activations/layer0_attention_weight_min": -10.776941299438477,
      "activations/layer10_attention_weight_max": 33.61012649536133,
      "activations/layer10_attention_weight_min": -29.652111053466797,
      "activations/layer11_attention_weight_max": 36.73524475097656,
      "activations/layer11_attention_weight_min": -28.22372817993164,
      "activations/layer12_attention_weight_max": 28.784921646118164,
      "activations/layer12_attention_weight_min": -28.737646102905273,
      "activations/layer13_attention_weight_max": 37.78387451171875,
      "activations/layer13_attention_weight_min": -32.213539123535156,
      "activations/layer14_attention_weight_max": 53.06835174560547,
      "activations/layer14_attention_weight_min": -41.09988784790039,
      "activations/layer15_attention_weight_max": 36.665252685546875,
      "activations/layer15_attention_weight_min": -30.717012405395508,
      "activations/layer16_attention_weight_max": 34.43536376953125,
      "activations/layer16_attention_weight_min": -30.64360237121582,
      "activations/layer17_attention_weight_max": 36.56110382080078,
      "activations/layer17_attention_weight_min": -26.038955688476562,
      "activations/layer18_attention_weight_max": 32.43060302734375,
      "activations/layer18_attention_weight_min": -21.047639846801758,
      "activations/layer19_attention_weight_max": 35.49113845825195,
      "activations/layer19_attention_weight_min": -30.195348739624023,
      "activations/layer1_attention_weight_max": 16.62083625793457,
      "activations/layer1_attention_weight_min": -14.216663360595703,
      "activations/layer20_attention_weight_max": 31.110483169555664,
      "activations/layer20_attention_weight_min": -24.03843879699707,
      "activations/layer21_attention_weight_max": 31.057205200195312,
      "activations/layer21_attention_weight_min": -22.586645126342773,
      "activations/layer22_attention_weight_max": 46.863525390625,
      "activations/layer22_attention_weight_min": -27.381454467773438,
      "activations/layer23_attention_weight_max": 36.08191680908203,
      "activations/layer23_attention_weight_min": -23.330554962158203,
      "activations/layer2_attention_weight_max": 31.882450103759766,
      "activations/layer2_attention_weight_min": -31.11067771911621,
      "activations/layer3_attention_weight_max": 88.14864349365234,
      "activations/layer3_attention_weight_min": -90.78765869140625,
      "activations/layer4_attention_weight_max": 121.15925598144531,
      "activations/layer4_attention_weight_min": -120.1717529296875,
      "activations/layer5_attention_weight_max": 63.142303466796875,
      "activations/layer5_attention_weight_min": -61.926414489746094,
      "activations/layer6_attention_weight_max": 51.80537033081055,
      "activations/layer6_attention_weight_min": -52.808021545410156,
      "activations/layer7_attention_weight_max": 88.48924255371094,
      "activations/layer7_attention_weight_min": -89.67131042480469,
      "activations/layer8_attention_weight_max": 41.31691360473633,
      "activations/layer8_attention_weight_min": -37.80803680419922,
      "activations/layer9_attention_weight_max": 34.88932418823242,
      "activations/layer9_attention_weight_min": -32.900516510009766,
      "epoch": 6.98,
      "learning_rate": 0.00010602651515151515,
      "loss": 2.8087,
      "step": 120200
    },
    {
      "activations/layer0_attention_weight_max": 15.214761734008789,
      "activations/layer0_attention_weight_min": -12.110336303710938,
      "activations/layer10_attention_weight_max": 31.985942840576172,
      "activations/layer10_attention_weight_min": -30.251867294311523,
      "activations/layer11_attention_weight_max": 30.476268768310547,
      "activations/layer11_attention_weight_min": -28.480436325073242,
      "activations/layer12_attention_weight_max": 31.001174926757812,
      "activations/layer12_attention_weight_min": -25.443408966064453,
      "activations/layer13_attention_weight_max": 38.5804443359375,
      "activations/layer13_attention_weight_min": -32.15209197998047,
      "activations/layer14_attention_weight_max": 52.329368591308594,
      "activations/layer14_attention_weight_min": -43.32218551635742,
      "activations/layer15_attention_weight_max": 38.77141189575195,
      "activations/layer15_attention_weight_min": -28.503461837768555,
      "activations/layer16_attention_weight_max": 36.210289001464844,
      "activations/layer16_attention_weight_min": -28.794363021850586,
      "activations/layer17_attention_weight_max": 36.62474822998047,
      "activations/layer17_attention_weight_min": -25.95256233215332,
      "activations/layer18_attention_weight_max": 32.32419967651367,
      "activations/layer18_attention_weight_min": -21.8299503326416,
      "activations/layer19_attention_weight_max": 38.47977828979492,
      "activations/layer19_attention_weight_min": -30.661954879760742,
      "activations/layer1_attention_weight_max": 17.266794204711914,
      "activations/layer1_attention_weight_min": -14.370050430297852,
      "activations/layer20_attention_weight_max": 32.665069580078125,
      "activations/layer20_attention_weight_min": -24.451526641845703,
      "activations/layer21_attention_weight_max": 31.193449020385742,
      "activations/layer21_attention_weight_min": -23.163522720336914,
      "activations/layer22_attention_weight_max": 50.383785247802734,
      "activations/layer22_attention_weight_min": -27.672218322753906,
      "activations/layer23_attention_weight_max": 36.89557647705078,
      "activations/layer23_attention_weight_min": -21.79553985595703,
      "activations/layer2_attention_weight_max": 30.86353302001953,
      "activations/layer2_attention_weight_min": -30.76723289489746,
      "activations/layer3_attention_weight_max": 90.7059097290039,
      "activations/layer3_attention_weight_min": -88.0896987915039,
      "activations/layer4_attention_weight_max": 122.69718170166016,
      "activations/layer4_attention_weight_min": -116.27922821044922,
      "activations/layer5_attention_weight_max": 64.6798324584961,
      "activations/layer5_attention_weight_min": -61.93868637084961,
      "activations/layer6_attention_weight_max": 49.449092864990234,
      "activations/layer6_attention_weight_min": -52.5753288269043,
      "activations/layer7_attention_weight_max": 83.21308898925781,
      "activations/layer7_attention_weight_min": -88.154296875,
      "activations/layer8_attention_weight_max": 38.37489318847656,
      "activations/layer8_attention_weight_min": -39.99707794189453,
      "activations/layer9_attention_weight_max": 32.98490524291992,
      "activations/layer9_attention_weight_min": -33.590755462646484,
      "epoch": 6.99,
      "learning_rate": 0.00010600757575757575,
      "loss": 2.8209,
      "step": 120250
    },
    {
      "activations/layer0_attention_weight_max": 15.653302192687988,
      "activations/layer0_attention_weight_min": -11.433067321777344,
      "activations/layer10_attention_weight_max": 33.12611389160156,
      "activations/layer10_attention_weight_min": -30.957242965698242,
      "activations/layer11_attention_weight_max": 32.66309356689453,
      "activations/layer11_attention_weight_min": -28.543094635009766,
      "activations/layer12_attention_weight_max": 32.660911560058594,
      "activations/layer12_attention_weight_min": -29.929927825927734,
      "activations/layer13_attention_weight_max": 41.1155891418457,
      "activations/layer13_attention_weight_min": -32.172752380371094,
      "activations/layer14_attention_weight_max": 61.6265754699707,
      "activations/layer14_attention_weight_min": -45.195091247558594,
      "activations/layer15_attention_weight_max": 37.59694290161133,
      "activations/layer15_attention_weight_min": -34.13695526123047,
      "activations/layer16_attention_weight_max": 34.18632888793945,
      "activations/layer16_attention_weight_min": -29.36301612854004,
      "activations/layer17_attention_weight_max": 32.205387115478516,
      "activations/layer17_attention_weight_min": -27.071420669555664,
      "activations/layer18_attention_weight_max": 30.932653427124023,
      "activations/layer18_attention_weight_min": -20.538217544555664,
      "activations/layer19_attention_weight_max": 38.23564529418945,
      "activations/layer19_attention_weight_min": -30.828516006469727,
      "activations/layer1_attention_weight_max": 17.103431701660156,
      "activations/layer1_attention_weight_min": -14.968937873840332,
      "activations/layer20_attention_weight_max": 32.22687530517578,
      "activations/layer20_attention_weight_min": -25.610639572143555,
      "activations/layer21_attention_weight_max": 29.938804626464844,
      "activations/layer21_attention_weight_min": -22.739158630371094,
      "activations/layer22_attention_weight_max": 45.756004333496094,
      "activations/layer22_attention_weight_min": -31.37438201904297,
      "activations/layer23_attention_weight_max": 34.65396499633789,
      "activations/layer23_attention_weight_min": -22.955562591552734,
      "activations/layer2_attention_weight_max": 32.017189025878906,
      "activations/layer2_attention_weight_min": -31.458349227905273,
      "activations/layer3_attention_weight_max": 96.55233764648438,
      "activations/layer3_attention_weight_min": -94.11864471435547,
      "activations/layer4_attention_weight_max": 130.00140380859375,
      "activations/layer4_attention_weight_min": -127.73870086669922,
      "activations/layer5_attention_weight_max": 67.91450500488281,
      "activations/layer5_attention_weight_min": -65.30523681640625,
      "activations/layer6_attention_weight_max": 53.98850631713867,
      "activations/layer6_attention_weight_min": -56.72758483886719,
      "activations/layer7_attention_weight_max": 94.97579956054688,
      "activations/layer7_attention_weight_min": -89.20948028564453,
      "activations/layer8_attention_weight_max": 41.17176818847656,
      "activations/layer8_attention_weight_min": -40.446044921875,
      "activations/layer9_attention_weight_max": 36.95437240600586,
      "activations/layer9_attention_weight_min": -33.70048904418945,
      "epoch": 6.99,
      "learning_rate": 0.00010598863636363635,
      "loss": 2.8217,
      "step": 120300
    },
    {
      "activations/layer0_attention_weight_max": 15.457547187805176,
      "activations/layer0_attention_weight_min": -12.172464370727539,
      "activations/layer10_attention_weight_max": 31.74124526977539,
      "activations/layer10_attention_weight_min": -30.748929977416992,
      "activations/layer11_attention_weight_max": 32.392086029052734,
      "activations/layer11_attention_weight_min": -30.524600982666016,
      "activations/layer12_attention_weight_max": 21.66433334350586,
      "activations/layer12_attention_weight_min": -31.688526153564453,
      "activations/layer13_attention_weight_max": 36.988887786865234,
      "activations/layer13_attention_weight_min": -31.017629623413086,
      "activations/layer14_attention_weight_max": 48.7026252746582,
      "activations/layer14_attention_weight_min": -39.793663024902344,
      "activations/layer15_attention_weight_max": 36.25369644165039,
      "activations/layer15_attention_weight_min": -28.191837310791016,
      "activations/layer16_attention_weight_max": 32.642974853515625,
      "activations/layer16_attention_weight_min": -26.881803512573242,
      "activations/layer17_attention_weight_max": 34.20570755004883,
      "activations/layer17_attention_weight_min": -23.51056480407715,
      "activations/layer18_attention_weight_max": 31.956335067749023,
      "activations/layer18_attention_weight_min": -19.29424476623535,
      "activations/layer19_attention_weight_max": 34.216949462890625,
      "activations/layer19_attention_weight_min": -29.181812286376953,
      "activations/layer1_attention_weight_max": 17.543102264404297,
      "activations/layer1_attention_weight_min": -14.75808334350586,
      "activations/layer20_attention_weight_max": 31.264341354370117,
      "activations/layer20_attention_weight_min": -23.23073959350586,
      "activations/layer21_attention_weight_max": 27.74014663696289,
      "activations/layer21_attention_weight_min": -20.82677459716797,
      "activations/layer22_attention_weight_max": 45.50815200805664,
      "activations/layer22_attention_weight_min": -26.614065170288086,
      "activations/layer23_attention_weight_max": 35.79123306274414,
      "activations/layer23_attention_weight_min": -21.579313278198242,
      "activations/layer2_attention_weight_max": 31.65846061706543,
      "activations/layer2_attention_weight_min": -31.298612594604492,
      "activations/layer3_attention_weight_max": 91.07989501953125,
      "activations/layer3_attention_weight_min": -93.48845672607422,
      "activations/layer4_attention_weight_max": 133.84349060058594,
      "activations/layer4_attention_weight_min": -125.73592376708984,
      "activations/layer5_attention_weight_max": 64.51009368896484,
      "activations/layer5_attention_weight_min": -65.06483459472656,
      "activations/layer6_attention_weight_max": 49.89271926879883,
      "activations/layer6_attention_weight_min": -53.46236038208008,
      "activations/layer7_attention_weight_max": 89.72174835205078,
      "activations/layer7_attention_weight_min": -88.72856140136719,
      "activations/layer8_attention_weight_max": 40.077327728271484,
      "activations/layer8_attention_weight_min": -39.7648811340332,
      "activations/layer9_attention_weight_max": 35.602481842041016,
      "activations/layer9_attention_weight_min": -36.243499755859375,
      "epoch": 6.99,
      "learning_rate": 0.00010596969696969696,
      "loss": 2.8111,
      "step": 120350
    },
    {
      "activations/layer0_attention_weight_max": 16.322710037231445,
      "activations/layer0_attention_weight_min": -12.040342330932617,
      "activations/layer10_attention_weight_max": 31.66259002685547,
      "activations/layer10_attention_weight_min": -30.24650001525879,
      "activations/layer11_attention_weight_max": 31.440093994140625,
      "activations/layer11_attention_weight_min": -29.68019676208496,
      "activations/layer12_attention_weight_max": 25.98862648010254,
      "activations/layer12_attention_weight_min": -24.774459838867188,
      "activations/layer13_attention_weight_max": 40.40116882324219,
      "activations/layer13_attention_weight_min": -31.96595001220703,
      "activations/layer14_attention_weight_max": 48.07869338989258,
      "activations/layer14_attention_weight_min": -37.80586242675781,
      "activations/layer15_attention_weight_max": 34.99275588989258,
      "activations/layer15_attention_weight_min": -29.756696701049805,
      "activations/layer16_attention_weight_max": 38.40610885620117,
      "activations/layer16_attention_weight_min": -26.983280181884766,
      "activations/layer17_attention_weight_max": 35.494300842285156,
      "activations/layer17_attention_weight_min": -27.225143432617188,
      "activations/layer18_attention_weight_max": 31.522855758666992,
      "activations/layer18_attention_weight_min": -20.565553665161133,
      "activations/layer19_attention_weight_max": 40.752933502197266,
      "activations/layer19_attention_weight_min": -31.262521743774414,
      "activations/layer1_attention_weight_max": 17.15091896057129,
      "activations/layer1_attention_weight_min": -15.145478248596191,
      "activations/layer20_attention_weight_max": 32.65043640136719,
      "activations/layer20_attention_weight_min": -24.313825607299805,
      "activations/layer21_attention_weight_max": 33.74074172973633,
      "activations/layer21_attention_weight_min": -22.73766326904297,
      "activations/layer22_attention_weight_max": 49.52620315551758,
      "activations/layer22_attention_weight_min": -30.468231201171875,
      "activations/layer23_attention_weight_max": 35.06031036376953,
      "activations/layer23_attention_weight_min": -25.753158569335938,
      "activations/layer2_attention_weight_max": 30.96947479248047,
      "activations/layer2_attention_weight_min": -29.593334197998047,
      "activations/layer3_attention_weight_max": 89.13970184326172,
      "activations/layer3_attention_weight_min": -90.01167297363281,
      "activations/layer4_attention_weight_max": 121.4085693359375,
      "activations/layer4_attention_weight_min": -120.1037368774414,
      "activations/layer5_attention_weight_max": 63.239540100097656,
      "activations/layer5_attention_weight_min": -63.188873291015625,
      "activations/layer6_attention_weight_max": 50.019962310791016,
      "activations/layer6_attention_weight_min": -51.40711212158203,
      "activations/layer7_attention_weight_max": 84.73529052734375,
      "activations/layer7_attention_weight_min": -89.12333679199219,
      "activations/layer8_attention_weight_max": 37.17299270629883,
      "activations/layer8_attention_weight_min": -40.613502502441406,
      "activations/layer9_attention_weight_max": 34.704627990722656,
      "activations/layer9_attention_weight_min": -33.556060791015625,
      "epoch": 7.0,
      "learning_rate": 0.00010595075757575757,
      "loss": 2.8249,
      "step": 120400
    },
    {
      "activations/layer0_attention_weight_max": 15.552010536193848,
      "activations/layer0_attention_weight_min": -10.779994010925293,
      "activations/layer10_attention_weight_max": 30.94551658630371,
      "activations/layer10_attention_weight_min": -30.815547943115234,
      "activations/layer11_attention_weight_max": 30.132755279541016,
      "activations/layer11_attention_weight_min": -28.087574005126953,
      "activations/layer12_attention_weight_max": 27.391828536987305,
      "activations/layer12_attention_weight_min": -26.608943939208984,
      "activations/layer13_attention_weight_max": 40.361629486083984,
      "activations/layer13_attention_weight_min": -34.352596282958984,
      "activations/layer14_attention_weight_max": 51.71931457519531,
      "activations/layer14_attention_weight_min": -39.810218811035156,
      "activations/layer15_attention_weight_max": 38.33134460449219,
      "activations/layer15_attention_weight_min": -28.677295684814453,
      "activations/layer16_attention_weight_max": 35.83171463012695,
      "activations/layer16_attention_weight_min": -27.48543930053711,
      "activations/layer17_attention_weight_max": 33.79931640625,
      "activations/layer17_attention_weight_min": -24.908870697021484,
      "activations/layer18_attention_weight_max": 30.448280334472656,
      "activations/layer18_attention_weight_min": -20.265966415405273,
      "activations/layer19_attention_weight_max": 43.53562545776367,
      "activations/layer19_attention_weight_min": -30.331287384033203,
      "activations/layer1_attention_weight_max": 17.205020904541016,
      "activations/layer1_attention_weight_min": -13.909977912902832,
      "activations/layer20_attention_weight_max": 34.6837043762207,
      "activations/layer20_attention_weight_min": -22.65684700012207,
      "activations/layer21_attention_weight_max": 34.698081970214844,
      "activations/layer21_attention_weight_min": -20.869430541992188,
      "activations/layer22_attention_weight_max": 46.958370208740234,
      "activations/layer22_attention_weight_min": -28.13836669921875,
      "activations/layer23_attention_weight_max": 34.269187927246094,
      "activations/layer23_attention_weight_min": -22.79663848876953,
      "activations/layer2_attention_weight_max": 31.101669311523438,
      "activations/layer2_attention_weight_min": -29.548137664794922,
      "activations/layer3_attention_weight_max": 91.97930908203125,
      "activations/layer3_attention_weight_min": -91.67822265625,
      "activations/layer4_attention_weight_max": 124.45892333984375,
      "activations/layer4_attention_weight_min": -121.171875,
      "activations/layer5_attention_weight_max": 66.47451782226562,
      "activations/layer5_attention_weight_min": -62.23304748535156,
      "activations/layer6_attention_weight_max": 52.16432189941406,
      "activations/layer6_attention_weight_min": -53.19601058959961,
      "activations/layer7_attention_weight_max": 82.05927276611328,
      "activations/layer7_attention_weight_min": -88.39298248291016,
      "activations/layer8_attention_weight_max": 40.49839401245117,
      "activations/layer8_attention_weight_min": -39.14590072631836,
      "activations/layer9_attention_weight_max": 33.84035873413086,
      "activations/layer9_attention_weight_min": -32.4064826965332,
      "epoch": 7.0,
      "learning_rate": 0.00010593181818181817,
      "loss": 2.8266,
      "step": 120450
    },
    {
      "activations/layer0_attention_weight_max": 16.624807357788086,
      "activations/layer0_attention_weight_min": -11.30959415435791,
      "activations/layer10_attention_weight_max": 30.529478073120117,
      "activations/layer10_attention_weight_min": -33.07731246948242,
      "activations/layer11_attention_weight_max": 30.158859252929688,
      "activations/layer11_attention_weight_min": -29.880531311035156,
      "activations/layer12_attention_weight_max": 26.001876831054688,
      "activations/layer12_attention_weight_min": -27.388809204101562,
      "activations/layer13_attention_weight_max": 37.358619689941406,
      "activations/layer13_attention_weight_min": -33.542015075683594,
      "activations/layer14_attention_weight_max": 53.59571075439453,
      "activations/layer14_attention_weight_min": -41.35708999633789,
      "activations/layer15_attention_weight_max": 35.49479293823242,
      "activations/layer15_attention_weight_min": -28.583847045898438,
      "activations/layer16_attention_weight_max": 31.66119384765625,
      "activations/layer16_attention_weight_min": -29.036903381347656,
      "activations/layer17_attention_weight_max": 36.326358795166016,
      "activations/layer17_attention_weight_min": -24.526870727539062,
      "activations/layer18_attention_weight_max": 30.534048080444336,
      "activations/layer18_attention_weight_min": -21.884672164916992,
      "activations/layer19_attention_weight_max": 36.24264144897461,
      "activations/layer19_attention_weight_min": -30.07269859313965,
      "activations/layer1_attention_weight_max": 18.28824234008789,
      "activations/layer1_attention_weight_min": -14.990450859069824,
      "activations/layer20_attention_weight_max": 31.017953872680664,
      "activations/layer20_attention_weight_min": -24.177165985107422,
      "activations/layer21_attention_weight_max": 32.80471420288086,
      "activations/layer21_attention_weight_min": -22.03557777404785,
      "activations/layer22_attention_weight_max": 47.957298278808594,
      "activations/layer22_attention_weight_min": -28.737621307373047,
      "activations/layer23_attention_weight_max": 33.432395935058594,
      "activations/layer23_attention_weight_min": -21.911056518554688,
      "activations/layer2_attention_weight_max": 31.535268783569336,
      "activations/layer2_attention_weight_min": -32.611572265625,
      "activations/layer3_attention_weight_max": 90.85415649414062,
      "activations/layer3_attention_weight_min": -95.27384185791016,
      "activations/layer4_attention_weight_max": 124.39411926269531,
      "activations/layer4_attention_weight_min": -123.8250961303711,
      "activations/layer5_attention_weight_max": 65.35579681396484,
      "activations/layer5_attention_weight_min": -67.06745147705078,
      "activations/layer6_attention_weight_max": 50.23618698120117,
      "activations/layer6_attention_weight_min": -55.50596618652344,
      "activations/layer7_attention_weight_max": 84.73697662353516,
      "activations/layer7_attention_weight_min": -87.61759185791016,
      "activations/layer8_attention_weight_max": 39.09450149536133,
      "activations/layer8_attention_weight_min": -40.29005813598633,
      "activations/layer9_attention_weight_max": 32.94606399536133,
      "activations/layer9_attention_weight_min": -34.764381408691406,
      "epoch": 7.0,
      "learning_rate": 0.00010591249999999998,
      "loss": 2.8704,
      "step": 120500
    },
    {
      "activations/layer0_attention_weight_max": 16.316564559936523,
      "activations/layer0_attention_weight_min": -11.589715003967285,
      "activations/layer10_attention_weight_max": 30.59059715270996,
      "activations/layer10_attention_weight_min": -29.538284301757812,
      "activations/layer11_attention_weight_max": 31.276912689208984,
      "activations/layer11_attention_weight_min": -28.754108428955078,
      "activations/layer12_attention_weight_max": 30.452516555786133,
      "activations/layer12_attention_weight_min": -26.668607711791992,
      "activations/layer13_attention_weight_max": 39.5355224609375,
      "activations/layer13_attention_weight_min": -32.57057189941406,
      "activations/layer14_attention_weight_max": 50.47391128540039,
      "activations/layer14_attention_weight_min": -41.11234664916992,
      "activations/layer15_attention_weight_max": 37.256526947021484,
      "activations/layer15_attention_weight_min": -28.523284912109375,
      "activations/layer16_attention_weight_max": 31.634700775146484,
      "activations/layer16_attention_weight_min": -27.399478912353516,
      "activations/layer17_attention_weight_max": 34.088035583496094,
      "activations/layer17_attention_weight_min": -24.098072052001953,
      "activations/layer18_attention_weight_max": 28.483781814575195,
      "activations/layer18_attention_weight_min": -20.31966781616211,
      "activations/layer19_attention_weight_max": 33.905452728271484,
      "activations/layer19_attention_weight_min": -28.506988525390625,
      "activations/layer1_attention_weight_max": 16.477893829345703,
      "activations/layer1_attention_weight_min": -13.88975715637207,
      "activations/layer20_attention_weight_max": 29.428863525390625,
      "activations/layer20_attention_weight_min": -21.245582580566406,
      "activations/layer21_attention_weight_max": 30.174095153808594,
      "activations/layer21_attention_weight_min": -21.934288024902344,
      "activations/layer22_attention_weight_max": 45.50244140625,
      "activations/layer22_attention_weight_min": -29.389490127563477,
      "activations/layer23_attention_weight_max": 36.101043701171875,
      "activations/layer23_attention_weight_min": -25.15801239013672,
      "activations/layer2_attention_weight_max": 31.92901611328125,
      "activations/layer2_attention_weight_min": -31.260440826416016,
      "activations/layer3_attention_weight_max": 91.17835998535156,
      "activations/layer3_attention_weight_min": -90.89649963378906,
      "activations/layer4_attention_weight_max": 122.80238342285156,
      "activations/layer4_attention_weight_min": -118.9447021484375,
      "activations/layer5_attention_weight_max": 64.77679443359375,
      "activations/layer5_attention_weight_min": -64.87478637695312,
      "activations/layer6_attention_weight_max": 54.326255798339844,
      "activations/layer6_attention_weight_min": -53.63323211669922,
      "activations/layer7_attention_weight_max": 83.46312713623047,
      "activations/layer7_attention_weight_min": -88.32611083984375,
      "activations/layer8_attention_weight_max": 39.274662017822266,
      "activations/layer8_attention_weight_min": -39.442989349365234,
      "activations/layer9_attention_weight_max": 34.59428024291992,
      "activations/layer9_attention_weight_min": -33.74668502807617,
      "epoch": 7.0,
      "learning_rate": 0.0001058935606060606,
      "loss": 2.8138,
      "step": 120550
    },
    {
      "activations/layer0_attention_weight_max": 16.09929847717285,
      "activations/layer0_attention_weight_min": -10.841891288757324,
      "activations/layer10_attention_weight_max": 35.28928756713867,
      "activations/layer10_attention_weight_min": -33.216766357421875,
      "activations/layer11_attention_weight_max": 34.843814849853516,
      "activations/layer11_attention_weight_min": -30.63581085205078,
      "activations/layer12_attention_weight_max": 24.560155868530273,
      "activations/layer12_attention_weight_min": -30.6846866607666,
      "activations/layer13_attention_weight_max": 42.88459396362305,
      "activations/layer13_attention_weight_min": -36.192134857177734,
      "activations/layer14_attention_weight_max": 53.61756896972656,
      "activations/layer14_attention_weight_min": -42.11220932006836,
      "activations/layer15_attention_weight_max": 37.6002311706543,
      "activations/layer15_attention_weight_min": -30.80538558959961,
      "activations/layer16_attention_weight_max": 34.69094467163086,
      "activations/layer16_attention_weight_min": -27.966005325317383,
      "activations/layer17_attention_weight_max": 36.67629623413086,
      "activations/layer17_attention_weight_min": -25.074913024902344,
      "activations/layer18_attention_weight_max": 34.5486946105957,
      "activations/layer18_attention_weight_min": -20.592714309692383,
      "activations/layer19_attention_weight_max": 40.95351791381836,
      "activations/layer19_attention_weight_min": -29.56916618347168,
      "activations/layer1_attention_weight_max": 17.81500244140625,
      "activations/layer1_attention_weight_min": -13.545507431030273,
      "activations/layer20_attention_weight_max": 31.853975296020508,
      "activations/layer20_attention_weight_min": -23.224971771240234,
      "activations/layer21_attention_weight_max": 33.296504974365234,
      "activations/layer21_attention_weight_min": -22.01137351989746,
      "activations/layer22_attention_weight_max": 54.203758239746094,
      "activations/layer22_attention_weight_min": -29.506927490234375,
      "activations/layer23_attention_weight_max": 40.453948974609375,
      "activations/layer23_attention_weight_min": -22.165569305419922,
      "activations/layer2_attention_weight_max": 31.12641143798828,
      "activations/layer2_attention_weight_min": -31.910526275634766,
      "activations/layer3_attention_weight_max": 89.47074890136719,
      "activations/layer3_attention_weight_min": -94.8228988647461,
      "activations/layer4_attention_weight_max": 120.84261322021484,
      "activations/layer4_attention_weight_min": -123.40373992919922,
      "activations/layer5_attention_weight_max": 64.42153930664062,
      "activations/layer5_attention_weight_min": -64.66316223144531,
      "activations/layer6_attention_weight_max": 51.83952331542969,
      "activations/layer6_attention_weight_min": -54.84839630126953,
      "activations/layer7_attention_weight_max": 91.12802124023438,
      "activations/layer7_attention_weight_min": -88.92561340332031,
      "activations/layer8_attention_weight_max": 42.75221633911133,
      "activations/layer8_attention_weight_min": -44.051212310791016,
      "activations/layer9_attention_weight_max": 39.42638397216797,
      "activations/layer9_attention_weight_min": -37.04636764526367,
      "epoch": 7.01,
      "learning_rate": 0.00010587462121212121,
      "loss": 2.8224,
      "step": 120600
    },
    {
      "activations/layer0_attention_weight_max": 15.83450698852539,
      "activations/layer0_attention_weight_min": -11.796544075012207,
      "activations/layer10_attention_weight_max": 31.946361541748047,
      "activations/layer10_attention_weight_min": -29.862972259521484,
      "activations/layer11_attention_weight_max": 29.655380249023438,
      "activations/layer11_attention_weight_min": -27.865467071533203,
      "activations/layer12_attention_weight_max": 32.656715393066406,
      "activations/layer12_attention_weight_min": -25.318145751953125,
      "activations/layer13_attention_weight_max": 37.58647155761719,
      "activations/layer13_attention_weight_min": -32.13966369628906,
      "activations/layer14_attention_weight_max": 55.13605499267578,
      "activations/layer14_attention_weight_min": -42.58686065673828,
      "activations/layer15_attention_weight_max": 36.34812545776367,
      "activations/layer15_attention_weight_min": -30.594314575195312,
      "activations/layer16_attention_weight_max": 35.87745666503906,
      "activations/layer16_attention_weight_min": -27.0219783782959,
      "activations/layer17_attention_weight_max": 36.56546401977539,
      "activations/layer17_attention_weight_min": -26.479202270507812,
      "activations/layer18_attention_weight_max": 39.2901725769043,
      "activations/layer18_attention_weight_min": -22.299240112304688,
      "activations/layer19_attention_weight_max": 40.154869079589844,
      "activations/layer19_attention_weight_min": -28.743167877197266,
      "activations/layer1_attention_weight_max": 17.351163864135742,
      "activations/layer1_attention_weight_min": -15.29537582397461,
      "activations/layer20_attention_weight_max": 33.39540481567383,
      "activations/layer20_attention_weight_min": -22.299211502075195,
      "activations/layer21_attention_weight_max": 32.08306884765625,
      "activations/layer21_attention_weight_min": -21.866695404052734,
      "activations/layer22_attention_weight_max": 47.66419982910156,
      "activations/layer22_attention_weight_min": -29.58217430114746,
      "activations/layer23_attention_weight_max": 37.487308502197266,
      "activations/layer23_attention_weight_min": -23.57007598876953,
      "activations/layer2_attention_weight_max": 31.318708419799805,
      "activations/layer2_attention_weight_min": -31.305593490600586,
      "activations/layer3_attention_weight_max": 89.55017852783203,
      "activations/layer3_attention_weight_min": -87.7896957397461,
      "activations/layer4_attention_weight_max": 121.10839080810547,
      "activations/layer4_attention_weight_min": -115.74202728271484,
      "activations/layer5_attention_weight_max": 61.906883239746094,
      "activations/layer5_attention_weight_min": -61.7100830078125,
      "activations/layer6_attention_weight_max": 49.16371154785156,
      "activations/layer6_attention_weight_min": -49.44497299194336,
      "activations/layer7_attention_weight_max": 82.64151000976562,
      "activations/layer7_attention_weight_min": -81.57069396972656,
      "activations/layer8_attention_weight_max": 38.758541107177734,
      "activations/layer8_attention_weight_min": -38.19272232055664,
      "activations/layer9_attention_weight_max": 33.740718841552734,
      "activations/layer9_attention_weight_min": -33.324398040771484,
      "epoch": 7.01,
      "learning_rate": 0.0001058556818181818,
      "loss": 2.817,
      "step": 120650
    },
    {
      "activations/layer0_attention_weight_max": 15.449569702148438,
      "activations/layer0_attention_weight_min": -12.314949989318848,
      "activations/layer10_attention_weight_max": 36.54212188720703,
      "activations/layer10_attention_weight_min": -30.69207000732422,
      "activations/layer11_attention_weight_max": 35.15787887573242,
      "activations/layer11_attention_weight_min": -31.065967559814453,
      "activations/layer12_attention_weight_max": 28.061059951782227,
      "activations/layer12_attention_weight_min": -26.676050186157227,
      "activations/layer13_attention_weight_max": 42.44838333129883,
      "activations/layer13_attention_weight_min": -32.96692657470703,
      "activations/layer14_attention_weight_max": 57.62248611450195,
      "activations/layer14_attention_weight_min": -42.37721633911133,
      "activations/layer15_attention_weight_max": 36.862247467041016,
      "activations/layer15_attention_weight_min": -28.934789657592773,
      "activations/layer16_attention_weight_max": 36.345149993896484,
      "activations/layer16_attention_weight_min": -27.509994506835938,
      "activations/layer17_attention_weight_max": 35.44819259643555,
      "activations/layer17_attention_weight_min": -25.53740119934082,
      "activations/layer18_attention_weight_max": 37.22713088989258,
      "activations/layer18_attention_weight_min": -20.540557861328125,
      "activations/layer19_attention_weight_max": 38.18166732788086,
      "activations/layer19_attention_weight_min": -29.90504264831543,
      "activations/layer1_attention_weight_max": 16.560705184936523,
      "activations/layer1_attention_weight_min": -15.99381160736084,
      "activations/layer20_attention_weight_max": 32.41312789916992,
      "activations/layer20_attention_weight_min": -23.923200607299805,
      "activations/layer21_attention_weight_max": 30.783475875854492,
      "activations/layer21_attention_weight_min": -22.943910598754883,
      "activations/layer22_attention_weight_max": 47.1181755065918,
      "activations/layer22_attention_weight_min": -28.34080696105957,
      "activations/layer23_attention_weight_max": 36.148433685302734,
      "activations/layer23_attention_weight_min": -24.17740249633789,
      "activations/layer2_attention_weight_max": 30.467161178588867,
      "activations/layer2_attention_weight_min": -29.486312866210938,
      "activations/layer3_attention_weight_max": 90.58992004394531,
      "activations/layer3_attention_weight_min": -88.20182800292969,
      "activations/layer4_attention_weight_max": 124.63507080078125,
      "activations/layer4_attention_weight_min": -123.1170654296875,
      "activations/layer5_attention_weight_max": 64.72404479980469,
      "activations/layer5_attention_weight_min": -65.81210327148438,
      "activations/layer6_attention_weight_max": 50.697574615478516,
      "activations/layer6_attention_weight_min": -55.25424575805664,
      "activations/layer7_attention_weight_max": 87.64959716796875,
      "activations/layer7_attention_weight_min": -85.56478118896484,
      "activations/layer8_attention_weight_max": 41.943878173828125,
      "activations/layer8_attention_weight_min": -38.2872200012207,
      "activations/layer9_attention_weight_max": 36.13465118408203,
      "activations/layer9_attention_weight_min": -33.51616287231445,
      "epoch": 7.01,
      "learning_rate": 0.00010583674242424241,
      "loss": 2.8227,
      "step": 120700
    },
    {
      "activations/layer0_attention_weight_max": 15.925960540771484,
      "activations/layer0_attention_weight_min": -11.541733741760254,
      "activations/layer10_attention_weight_max": 40.029624938964844,
      "activations/layer10_attention_weight_min": -36.63006591796875,
      "activations/layer11_attention_weight_max": 39.194828033447266,
      "activations/layer11_attention_weight_min": -37.72905731201172,
      "activations/layer12_attention_weight_max": 36.534339904785156,
      "activations/layer12_attention_weight_min": -30.052824020385742,
      "activations/layer13_attention_weight_max": 54.83741760253906,
      "activations/layer13_attention_weight_min": -40.85698318481445,
      "activations/layer14_attention_weight_max": 72.6718978881836,
      "activations/layer14_attention_weight_min": -55.04680633544922,
      "activations/layer15_attention_weight_max": 46.92087173461914,
      "activations/layer15_attention_weight_min": -34.37605667114258,
      "activations/layer16_attention_weight_max": 40.48996353149414,
      "activations/layer16_attention_weight_min": -28.379411697387695,
      "activations/layer17_attention_weight_max": 42.38542175292969,
      "activations/layer17_attention_weight_min": -28.562795639038086,
      "activations/layer18_attention_weight_max": 36.524070739746094,
      "activations/layer18_attention_weight_min": -23.32514762878418,
      "activations/layer19_attention_weight_max": 47.40482711791992,
      "activations/layer19_attention_weight_min": -29.824365615844727,
      "activations/layer1_attention_weight_max": 17.183141708374023,
      "activations/layer1_attention_weight_min": -14.620430946350098,
      "activations/layer20_attention_weight_max": 45.47990417480469,
      "activations/layer20_attention_weight_min": -24.73415756225586,
      "activations/layer21_attention_weight_max": 42.63311767578125,
      "activations/layer21_attention_weight_min": -23.184953689575195,
      "activations/layer22_attention_weight_max": 57.5434455871582,
      "activations/layer22_attention_weight_min": -31.843219757080078,
      "activations/layer23_attention_weight_max": 45.78833770751953,
      "activations/layer23_attention_weight_min": -24.538328170776367,
      "activations/layer2_attention_weight_max": 31.350936889648438,
      "activations/layer2_attention_weight_min": -30.231731414794922,
      "activations/layer3_attention_weight_max": 95.08262634277344,
      "activations/layer3_attention_weight_min": -94.0060043334961,
      "activations/layer4_attention_weight_max": 130.17689514160156,
      "activations/layer4_attention_weight_min": -122.57989501953125,
      "activations/layer5_attention_weight_max": 66.8815689086914,
      "activations/layer5_attention_weight_min": -62.123497009277344,
      "activations/layer6_attention_weight_max": 51.32344055175781,
      "activations/layer6_attention_weight_min": -51.17255401611328,
      "activations/layer7_attention_weight_max": 95.22187042236328,
      "activations/layer7_attention_weight_min": -90.5937271118164,
      "activations/layer8_attention_weight_max": 48.60195541381836,
      "activations/layer8_attention_weight_min": -43.62858581542969,
      "activations/layer9_attention_weight_max": 44.45858383178711,
      "activations/layer9_attention_weight_min": -37.89982604980469,
      "epoch": 7.02,
      "learning_rate": 0.00010581780303030303,
      "loss": 2.8252,
      "step": 120750
    },
    {
      "activations/layer0_attention_weight_max": 15.593085289001465,
      "activations/layer0_attention_weight_min": -12.92931842803955,
      "activations/layer10_attention_weight_max": 31.32754898071289,
      "activations/layer10_attention_weight_min": -30.40079116821289,
      "activations/layer11_attention_weight_max": 30.049545288085938,
      "activations/layer11_attention_weight_min": -28.656028747558594,
      "activations/layer12_attention_weight_max": 36.36735534667969,
      "activations/layer12_attention_weight_min": -27.65389633178711,
      "activations/layer13_attention_weight_max": 40.146636962890625,
      "activations/layer13_attention_weight_min": -31.968990325927734,
      "activations/layer14_attention_weight_max": 53.00076675415039,
      "activations/layer14_attention_weight_min": -40.91740798950195,
      "activations/layer15_attention_weight_max": 42.78972625732422,
      "activations/layer15_attention_weight_min": -31.16750144958496,
      "activations/layer16_attention_weight_max": 35.75525665283203,
      "activations/layer16_attention_weight_min": -27.301753997802734,
      "activations/layer17_attention_weight_max": 38.05463409423828,
      "activations/layer17_attention_weight_min": -26.577856063842773,
      "activations/layer18_attention_weight_max": 35.3802604675293,
      "activations/layer18_attention_weight_min": -22.014949798583984,
      "activations/layer19_attention_weight_max": 38.66967010498047,
      "activations/layer19_attention_weight_min": -29.777740478515625,
      "activations/layer1_attention_weight_max": 16.064908981323242,
      "activations/layer1_attention_weight_min": -14.576915740966797,
      "activations/layer20_attention_weight_max": 32.89011001586914,
      "activations/layer20_attention_weight_min": -23.68213653564453,
      "activations/layer21_attention_weight_max": 39.001068115234375,
      "activations/layer21_attention_weight_min": -22.802289962768555,
      "activations/layer22_attention_weight_max": 51.64242935180664,
      "activations/layer22_attention_weight_min": -27.835895538330078,
      "activations/layer23_attention_weight_max": 36.70215606689453,
      "activations/layer23_attention_weight_min": -23.30567741394043,
      "activations/layer2_attention_weight_max": 30.467674255371094,
      "activations/layer2_attention_weight_min": -29.20545768737793,
      "activations/layer3_attention_weight_max": 87.71134185791016,
      "activations/layer3_attention_weight_min": -85.02674865722656,
      "activations/layer4_attention_weight_max": 119.2172622680664,
      "activations/layer4_attention_weight_min": -115.12152099609375,
      "activations/layer5_attention_weight_max": 62.13581848144531,
      "activations/layer5_attention_weight_min": -59.53910446166992,
      "activations/layer6_attention_weight_max": 51.8620491027832,
      "activations/layer6_attention_weight_min": -50.28501510620117,
      "activations/layer7_attention_weight_max": 85.56307220458984,
      "activations/layer7_attention_weight_min": -82.7258529663086,
      "activations/layer8_attention_weight_max": 39.083187103271484,
      "activations/layer8_attention_weight_min": -39.07601547241211,
      "activations/layer9_attention_weight_max": 34.912288665771484,
      "activations/layer9_attention_weight_min": -33.231224060058594,
      "epoch": 7.02,
      "learning_rate": 0.00010579886363636363,
      "loss": 2.8048,
      "step": 120800
    },
    {
      "activations/layer0_attention_weight_max": 15.33016300201416,
      "activations/layer0_attention_weight_min": -12.252022743225098,
      "activations/layer10_attention_weight_max": 34.6283073425293,
      "activations/layer10_attention_weight_min": -35.17491912841797,
      "activations/layer11_attention_weight_max": 34.24678039550781,
      "activations/layer11_attention_weight_min": -31.627309799194336,
      "activations/layer12_attention_weight_max": 22.214887619018555,
      "activations/layer12_attention_weight_min": -25.221532821655273,
      "activations/layer13_attention_weight_max": 39.79927062988281,
      "activations/layer13_attention_weight_min": -32.96369552612305,
      "activations/layer14_attention_weight_max": 54.51789474487305,
      "activations/layer14_attention_weight_min": -40.15955352783203,
      "activations/layer15_attention_weight_max": 42.504207611083984,
      "activations/layer15_attention_weight_min": -27.898338317871094,
      "activations/layer16_attention_weight_max": 40.1238899230957,
      "activations/layer16_attention_weight_min": -27.891611099243164,
      "activations/layer17_attention_weight_max": 43.05415344238281,
      "activations/layer17_attention_weight_min": -24.105388641357422,
      "activations/layer18_attention_weight_max": 40.077754974365234,
      "activations/layer18_attention_weight_min": -21.59143829345703,
      "activations/layer19_attention_weight_max": 43.076717376708984,
      "activations/layer19_attention_weight_min": -31.134626388549805,
      "activations/layer1_attention_weight_max": 16.392921447753906,
      "activations/layer1_attention_weight_min": -14.458625793457031,
      "activations/layer20_attention_weight_max": 33.39102554321289,
      "activations/layer20_attention_weight_min": -23.803449630737305,
      "activations/layer21_attention_weight_max": 35.64667892456055,
      "activations/layer21_attention_weight_min": -23.182842254638672,
      "activations/layer22_attention_weight_max": 54.31947326660156,
      "activations/layer22_attention_weight_min": -30.108951568603516,
      "activations/layer23_attention_weight_max": 38.47373962402344,
      "activations/layer23_attention_weight_min": -23.929000854492188,
      "activations/layer2_attention_weight_max": 31.133880615234375,
      "activations/layer2_attention_weight_min": -30.583694458007812,
      "activations/layer3_attention_weight_max": 88.46875,
      "activations/layer3_attention_weight_min": -90.38461303710938,
      "activations/layer4_attention_weight_max": 122.7109146118164,
      "activations/layer4_attention_weight_min": -120.7204818725586,
      "activations/layer5_attention_weight_max": 66.51742553710938,
      "activations/layer5_attention_weight_min": -64.17749786376953,
      "activations/layer6_attention_weight_max": 52.234622955322266,
      "activations/layer6_attention_weight_min": -53.56991195678711,
      "activations/layer7_attention_weight_max": 90.03915405273438,
      "activations/layer7_attention_weight_min": -92.59125518798828,
      "activations/layer8_attention_weight_max": 42.02992630004883,
      "activations/layer8_attention_weight_min": -42.47353744506836,
      "activations/layer9_attention_weight_max": 38.36259841918945,
      "activations/layer9_attention_weight_min": -35.29293441772461,
      "epoch": 7.02,
      "learning_rate": 0.00010577992424242423,
      "loss": 2.8125,
      "step": 120850
    },
    {
      "activations/layer0_attention_weight_max": 16.185428619384766,
      "activations/layer0_attention_weight_min": -11.538663864135742,
      "activations/layer10_attention_weight_max": 40.33015823364258,
      "activations/layer10_attention_weight_min": -40.20534133911133,
      "activations/layer11_attention_weight_max": 37.61170959472656,
      "activations/layer11_attention_weight_min": -36.8079719543457,
      "activations/layer12_attention_weight_max": 45.00175476074219,
      "activations/layer12_attention_weight_min": -25.48155975341797,
      "activations/layer13_attention_weight_max": 54.00992202758789,
      "activations/layer13_attention_weight_min": -37.575531005859375,
      "activations/layer14_attention_weight_max": 53.03038787841797,
      "activations/layer14_attention_weight_min": -40.607051849365234,
      "activations/layer15_attention_weight_max": 46.214717864990234,
      "activations/layer15_attention_weight_min": -30.51008415222168,
      "activations/layer16_attention_weight_max": 32.130897521972656,
      "activations/layer16_attention_weight_min": -28.52953338623047,
      "activations/layer17_attention_weight_max": 35.873023986816406,
      "activations/layer17_attention_weight_min": -24.32786750793457,
      "activations/layer18_attention_weight_max": 30.227779388427734,
      "activations/layer18_attention_weight_min": -21.57703399658203,
      "activations/layer19_attention_weight_max": 37.20984649658203,
      "activations/layer19_attention_weight_min": -31.313108444213867,
      "activations/layer1_attention_weight_max": 17.292085647583008,
      "activations/layer1_attention_weight_min": -14.463603973388672,
      "activations/layer20_attention_weight_max": 30.860759735107422,
      "activations/layer20_attention_weight_min": -23.48752784729004,
      "activations/layer21_attention_weight_max": 31.69037437438965,
      "activations/layer21_attention_weight_min": -23.88896942138672,
      "activations/layer22_attention_weight_max": 49.73160171508789,
      "activations/layer22_attention_weight_min": -30.019495010375977,
      "activations/layer23_attention_weight_max": 36.21051025390625,
      "activations/layer23_attention_weight_min": -21.509258270263672,
      "activations/layer2_attention_weight_max": 31.70691680908203,
      "activations/layer2_attention_weight_min": -30.492109298706055,
      "activations/layer3_attention_weight_max": 90.9708251953125,
      "activations/layer3_attention_weight_min": -93.42866516113281,
      "activations/layer4_attention_weight_max": 120.6479263305664,
      "activations/layer4_attention_weight_min": -117.66942596435547,
      "activations/layer5_attention_weight_max": 65.9985580444336,
      "activations/layer5_attention_weight_min": -63.35935974121094,
      "activations/layer6_attention_weight_max": 49.45906066894531,
      "activations/layer6_attention_weight_min": -53.43758010864258,
      "activations/layer7_attention_weight_max": 102.27361297607422,
      "activations/layer7_attention_weight_min": -91.58152770996094,
      "activations/layer8_attention_weight_max": 44.9219856262207,
      "activations/layer8_attention_weight_min": -44.00307083129883,
      "activations/layer9_attention_weight_max": 40.86896896362305,
      "activations/layer9_attention_weight_min": -38.85480880737305,
      "epoch": 7.02,
      "learning_rate": 0.00010576098484848483,
      "loss": 2.8168,
      "step": 120900
    },
    {
      "activations/layer0_attention_weight_max": 16.561283111572266,
      "activations/layer0_attention_weight_min": -11.519664764404297,
      "activations/layer10_attention_weight_max": 35.312965393066406,
      "activations/layer10_attention_weight_min": -30.063705444335938,
      "activations/layer11_attention_weight_max": 34.44534683227539,
      "activations/layer11_attention_weight_min": -30.62051773071289,
      "activations/layer12_attention_weight_max": 27.372623443603516,
      "activations/layer12_attention_weight_min": -25.443965911865234,
      "activations/layer13_attention_weight_max": 45.37731170654297,
      "activations/layer13_attention_weight_min": -36.22893142700195,
      "activations/layer14_attention_weight_max": 57.25840377807617,
      "activations/layer14_attention_weight_min": -41.528358459472656,
      "activations/layer15_attention_weight_max": 42.3270149230957,
      "activations/layer15_attention_weight_min": -31.665021896362305,
      "activations/layer16_attention_weight_max": 35.48553466796875,
      "activations/layer16_attention_weight_min": -27.240917205810547,
      "activations/layer17_attention_weight_max": 40.217552185058594,
      "activations/layer17_attention_weight_min": -23.410072326660156,
      "activations/layer18_attention_weight_max": 32.7674560546875,
      "activations/layer18_attention_weight_min": -20.749717712402344,
      "activations/layer19_attention_weight_max": 42.11918258666992,
      "activations/layer19_attention_weight_min": -30.571882247924805,
      "activations/layer1_attention_weight_max": 16.147930145263672,
      "activations/layer1_attention_weight_min": -13.889013290405273,
      "activations/layer20_attention_weight_max": 34.255210876464844,
      "activations/layer20_attention_weight_min": -23.90705108642578,
      "activations/layer21_attention_weight_max": 34.58440017700195,
      "activations/layer21_attention_weight_min": -22.385833740234375,
      "activations/layer22_attention_weight_max": 48.3621940612793,
      "activations/layer22_attention_weight_min": -27.92804527282715,
      "activations/layer23_attention_weight_max": 35.02876663208008,
      "activations/layer23_attention_weight_min": -22.812101364135742,
      "activations/layer2_attention_weight_max": 30.82790184020996,
      "activations/layer2_attention_weight_min": -30.989086151123047,
      "activations/layer3_attention_weight_max": 88.89730072021484,
      "activations/layer3_attention_weight_min": -90.74064636230469,
      "activations/layer4_attention_weight_max": 121.81500244140625,
      "activations/layer4_attention_weight_min": -117.85357666015625,
      "activations/layer5_attention_weight_max": 64.03433990478516,
      "activations/layer5_attention_weight_min": -61.734580993652344,
      "activations/layer6_attention_weight_max": 51.1134147644043,
      "activations/layer6_attention_weight_min": -51.0753059387207,
      "activations/layer7_attention_weight_max": 90.03826141357422,
      "activations/layer7_attention_weight_min": -84.0338134765625,
      "activations/layer8_attention_weight_max": 40.93669509887695,
      "activations/layer8_attention_weight_min": -38.8226203918457,
      "activations/layer9_attention_weight_max": 35.63337707519531,
      "activations/layer9_attention_weight_min": -34.19990158081055,
      "epoch": 7.03,
      "learning_rate": 0.00010574204545454545,
      "loss": 2.8285,
      "step": 120950
    },
    {
      "activations/layer0_attention_weight_max": 16.547117233276367,
      "activations/layer0_attention_weight_min": -11.585920333862305,
      "activations/layer10_attention_weight_max": 31.14333724975586,
      "activations/layer10_attention_weight_min": -33.09281539916992,
      "activations/layer11_attention_weight_max": 33.49474334716797,
      "activations/layer11_attention_weight_min": -30.496597290039062,
      "activations/layer12_attention_weight_max": 59.41081237792969,
      "activations/layer12_attention_weight_min": -25.640302658081055,
      "activations/layer13_attention_weight_max": 58.90061569213867,
      "activations/layer13_attention_weight_min": -33.93144989013672,
      "activations/layer14_attention_weight_max": 55.936397552490234,
      "activations/layer14_attention_weight_min": -41.5971565246582,
      "activations/layer15_attention_weight_max": 41.672969818115234,
      "activations/layer15_attention_weight_min": -32.434566497802734,
      "activations/layer16_attention_weight_max": 39.048736572265625,
      "activations/layer16_attention_weight_min": -29.602699279785156,
      "activations/layer17_attention_weight_max": 40.725318908691406,
      "activations/layer17_attention_weight_min": -24.568628311157227,
      "activations/layer18_attention_weight_max": 34.777557373046875,
      "activations/layer18_attention_weight_min": -20.723817825317383,
      "activations/layer19_attention_weight_max": 41.06739807128906,
      "activations/layer19_attention_weight_min": -28.512292861938477,
      "activations/layer1_attention_weight_max": 16.897655487060547,
      "activations/layer1_attention_weight_min": -13.279216766357422,
      "activations/layer20_attention_weight_max": 35.07011413574219,
      "activations/layer20_attention_weight_min": -24.229690551757812,
      "activations/layer21_attention_weight_max": 33.93300247192383,
      "activations/layer21_attention_weight_min": -23.23956871032715,
      "activations/layer22_attention_weight_max": 55.23475646972656,
      "activations/layer22_attention_weight_min": -28.872787475585938,
      "activations/layer23_attention_weight_max": 44.999267578125,
      "activations/layer23_attention_weight_min": -22.39450454711914,
      "activations/layer2_attention_weight_max": 32.1048469543457,
      "activations/layer2_attention_weight_min": -29.567096710205078,
      "activations/layer3_attention_weight_max": 93.08891296386719,
      "activations/layer3_attention_weight_min": -92.52720642089844,
      "activations/layer4_attention_weight_max": 122.3879623413086,
      "activations/layer4_attention_weight_min": -118.09380340576172,
      "activations/layer5_attention_weight_max": 62.87968444824219,
      "activations/layer5_attention_weight_min": -60.78066635131836,
      "activations/layer6_attention_weight_max": 52.03534698486328,
      "activations/layer6_attention_weight_min": -49.65828323364258,
      "activations/layer7_attention_weight_max": 83.2149887084961,
      "activations/layer7_attention_weight_min": -85.44306182861328,
      "activations/layer8_attention_weight_max": 37.656280517578125,
      "activations/layer8_attention_weight_min": -37.904109954833984,
      "activations/layer9_attention_weight_max": 33.45742416381836,
      "activations/layer9_attention_weight_min": -33.416873931884766,
      "epoch": 7.03,
      "learning_rate": 0.00010572310606060606,
      "loss": 2.8032,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.527,
      "eval_samples_per_second": 503.578,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.527,
      "eval_openwebtext_samples_per_second": 503.578,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_wikitext_loss": 3.0,
      "eval_wikitext_ppl": 20.085536923187668,
      "eval_wikitext_runtime": 2.0165,
      "eval_wikitext_samples_per_second": 226.133,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_lambada_loss": 2.705078125,
      "eval_lambada_ppl": 14.95548504215564,
      "eval_lambada_runtime": 9.6209,
      "eval_lambada_samples_per_second": 506.083,
      "step": 121000
    },
    {
      "activations/layer0_attention_weight_max": 16.790952682495117,
      "activations/layer0_attention_weight_min": -11.988420486450195,
      "activations/layer10_attention_weight_max": 29.535680770874023,
      "activations/layer10_attention_weight_min": -28.610916137695312,
      "activations/layer11_attention_weight_max": 29.262052536010742,
      "activations/layer11_attention_weight_min": -27.363508224487305,
      "activations/layer12_attention_weight_max": 22.43113136291504,
      "activations/layer12_attention_weight_min": -25.703493118286133,
      "activations/layer13_attention_weight_max": 40.200008392333984,
      "activations/layer13_attention_weight_min": -31.256153106689453,
      "activations/layer14_attention_weight_max": 44.735191345214844,
      "activations/layer14_attention_weight_min": -36.21004867553711,
      "activations/layer15_attention_weight_max": 35.75389862060547,
      "activations/layer15_attention_weight_min": -28.85482406616211,
      "activations/layer16_attention_weight_max": 32.69899368286133,
      "activations/layer16_attention_weight_min": -26.48640251159668,
      "activations/layer17_attention_weight_max": 32.07584762573242,
      "activations/layer17_attention_weight_min": -24.018922805786133,
      "activations/layer18_attention_weight_max": 35.91217041015625,
      "activations/layer18_attention_weight_min": -20.99308967590332,
      "activations/layer19_attention_weight_max": 35.69315719604492,
      "activations/layer19_attention_weight_min": -28.235517501831055,
      "activations/layer1_attention_weight_max": 17.175134658813477,
      "activations/layer1_attention_weight_min": -13.448503494262695,
      "activations/layer20_attention_weight_max": 29.805030822753906,
      "activations/layer20_attention_weight_min": -25.20330810546875,
      "activations/layer21_attention_weight_max": 31.81930923461914,
      "activations/layer21_attention_weight_min": -21.841838836669922,
      "activations/layer22_attention_weight_max": 47.06077194213867,
      "activations/layer22_attention_weight_min": -29.524093627929688,
      "activations/layer23_attention_weight_max": 40.548179626464844,
      "activations/layer23_attention_weight_min": -23.614608764648438,
      "activations/layer2_attention_weight_max": 33.04202651977539,
      "activations/layer2_attention_weight_min": -30.45734977722168,
      "activations/layer3_attention_weight_max": 92.12933349609375,
      "activations/layer3_attention_weight_min": -93.35030364990234,
      "activations/layer4_attention_weight_max": 117.9455337524414,
      "activations/layer4_attention_weight_min": -118.30146789550781,
      "activations/layer5_attention_weight_max": 61.6590690612793,
      "activations/layer5_attention_weight_min": -60.819610595703125,
      "activations/layer6_attention_weight_max": 50.63944625854492,
      "activations/layer6_attention_weight_min": -53.42361068725586,
      "activations/layer7_attention_weight_max": 84.88603210449219,
      "activations/layer7_attention_weight_min": -82.1451187133789,
      "activations/layer8_attention_weight_max": 37.019004821777344,
      "activations/layer8_attention_weight_min": -39.59374237060547,
      "activations/layer9_attention_weight_max": 32.4962043762207,
      "activations/layer9_attention_weight_min": -31.861215591430664,
      "epoch": 7.03,
      "learning_rate": 0.00010570416666666665,
      "loss": 2.8104,
      "step": 121050
    },
    {
      "activations/layer0_attention_weight_max": 14.88394546508789,
      "activations/layer0_attention_weight_min": -12.412018775939941,
      "activations/layer10_attention_weight_max": 35.93418884277344,
      "activations/layer10_attention_weight_min": -32.10992431640625,
      "activations/layer11_attention_weight_max": 34.54991912841797,
      "activations/layer11_attention_weight_min": -30.73605728149414,
      "activations/layer12_attention_weight_max": 24.433927536010742,
      "activations/layer12_attention_weight_min": -25.775169372558594,
      "activations/layer13_attention_weight_max": 45.450679779052734,
      "activations/layer13_attention_weight_min": -32.5683479309082,
      "activations/layer14_attention_weight_max": 50.907257080078125,
      "activations/layer14_attention_weight_min": -37.83088684082031,
      "activations/layer15_attention_weight_max": 39.19855499267578,
      "activations/layer15_attention_weight_min": -31.041202545166016,
      "activations/layer16_attention_weight_max": 37.4863395690918,
      "activations/layer16_attention_weight_min": -26.23223304748535,
      "activations/layer17_attention_weight_max": 40.25191116333008,
      "activations/layer17_attention_weight_min": -24.07171058654785,
      "activations/layer18_attention_weight_max": 34.876434326171875,
      "activations/layer18_attention_weight_min": -22.12187385559082,
      "activations/layer19_attention_weight_max": 38.81077194213867,
      "activations/layer19_attention_weight_min": -30.131383895874023,
      "activations/layer1_attention_weight_max": 16.59173583984375,
      "activations/layer1_attention_weight_min": -13.656521797180176,
      "activations/layer20_attention_weight_max": 33.581329345703125,
      "activations/layer20_attention_weight_min": -23.516496658325195,
      "activations/layer21_attention_weight_max": 32.65053176879883,
      "activations/layer21_attention_weight_min": -23.28912925720215,
      "activations/layer22_attention_weight_max": 49.55314254760742,
      "activations/layer22_attention_weight_min": -29.391292572021484,
      "activations/layer23_attention_weight_max": 39.08413314819336,
      "activations/layer23_attention_weight_min": -23.24520492553711,
      "activations/layer2_attention_weight_max": 30.95362091064453,
      "activations/layer2_attention_weight_min": -29.973447799682617,
      "activations/layer3_attention_weight_max": 89.9013442993164,
      "activations/layer3_attention_weight_min": -93.8272476196289,
      "activations/layer4_attention_weight_max": 121.78680419921875,
      "activations/layer4_attention_weight_min": -117.19781494140625,
      "activations/layer5_attention_weight_max": 65.51687622070312,
      "activations/layer5_attention_weight_min": -62.595638275146484,
      "activations/layer6_attention_weight_max": 51.40902328491211,
      "activations/layer6_attention_weight_min": -52.995750427246094,
      "activations/layer7_attention_weight_max": 87.71341705322266,
      "activations/layer7_attention_weight_min": -87.38166046142578,
      "activations/layer8_attention_weight_max": 39.96370315551758,
      "activations/layer8_attention_weight_min": -41.67774200439453,
      "activations/layer9_attention_weight_max": 37.90117645263672,
      "activations/layer9_attention_weight_min": -35.12519454956055,
      "epoch": 7.04,
      "learning_rate": 0.00010568522727272726,
      "loss": 2.81,
      "step": 121100
    },
    {
      "activations/layer0_attention_weight_max": 14.851800918579102,
      "activations/layer0_attention_weight_min": -11.26362133026123,
      "activations/layer10_attention_weight_max": 36.07219314575195,
      "activations/layer10_attention_weight_min": -31.488676071166992,
      "activations/layer11_attention_weight_max": 39.42237854003906,
      "activations/layer11_attention_weight_min": -30.194374084472656,
      "activations/layer12_attention_weight_max": 21.95244789123535,
      "activations/layer12_attention_weight_min": -31.78084945678711,
      "activations/layer13_attention_weight_max": 44.01591491699219,
      "activations/layer13_attention_weight_min": -33.46894454956055,
      "activations/layer14_attention_weight_max": 56.77376174926758,
      "activations/layer14_attention_weight_min": -43.760284423828125,
      "activations/layer15_attention_weight_max": 38.325321197509766,
      "activations/layer15_attention_weight_min": -30.992843627929688,
      "activations/layer16_attention_weight_max": 36.8834114074707,
      "activations/layer16_attention_weight_min": -30.32932472229004,
      "activations/layer17_attention_weight_max": 38.48883819580078,
      "activations/layer17_attention_weight_min": -26.197473526000977,
      "activations/layer18_attention_weight_max": 40.14590072631836,
      "activations/layer18_attention_weight_min": -20.587621688842773,
      "activations/layer19_attention_weight_max": 38.90434646606445,
      "activations/layer19_attention_weight_min": -30.33172035217285,
      "activations/layer1_attention_weight_max": 16.386281967163086,
      "activations/layer1_attention_weight_min": -14.78724479675293,
      "activations/layer20_attention_weight_max": 34.50852584838867,
      "activations/layer20_attention_weight_min": -24.56102180480957,
      "activations/layer21_attention_weight_max": 38.3619499206543,
      "activations/layer21_attention_weight_min": -24.70497703552246,
      "activations/layer22_attention_weight_max": 57.65467071533203,
      "activations/layer22_attention_weight_min": -29.671085357666016,
      "activations/layer23_attention_weight_max": 40.41412353515625,
      "activations/layer23_attention_weight_min": -22.529781341552734,
      "activations/layer2_attention_weight_max": 35.23628616333008,
      "activations/layer2_attention_weight_min": -31.40877342224121,
      "activations/layer3_attention_weight_max": 99.13068389892578,
      "activations/layer3_attention_weight_min": -96.22428131103516,
      "activations/layer4_attention_weight_max": 138.51931762695312,
      "activations/layer4_attention_weight_min": -126.2197494506836,
      "activations/layer5_attention_weight_max": 70.10387420654297,
      "activations/layer5_attention_weight_min": -64.18486785888672,
      "activations/layer6_attention_weight_max": 53.09956359863281,
      "activations/layer6_attention_weight_min": -53.03217697143555,
      "activations/layer7_attention_weight_max": 95.40137481689453,
      "activations/layer7_attention_weight_min": -88.62916564941406,
      "activations/layer8_attention_weight_max": 41.450836181640625,
      "activations/layer8_attention_weight_min": -38.86000442504883,
      "activations/layer9_attention_weight_max": 39.3442497253418,
      "activations/layer9_attention_weight_min": -33.47040939331055,
      "epoch": 7.04,
      "learning_rate": 0.00010566628787878788,
      "loss": 2.8223,
      "step": 121150
    },
    {
      "activations/layer0_attention_weight_max": 15.971505165100098,
      "activations/layer0_attention_weight_min": -12.226097106933594,
      "activations/layer10_attention_weight_max": 32.24755096435547,
      "activations/layer10_attention_weight_min": -32.02601623535156,
      "activations/layer11_attention_weight_max": 34.76681900024414,
      "activations/layer11_attention_weight_min": -30.100454330444336,
      "activations/layer12_attention_weight_max": 23.62869644165039,
      "activations/layer12_attention_weight_min": -26.665855407714844,
      "activations/layer13_attention_weight_max": 41.69955062866211,
      "activations/layer13_attention_weight_min": -31.776098251342773,
      "activations/layer14_attention_weight_max": 58.55133819580078,
      "activations/layer14_attention_weight_min": -46.9540901184082,
      "activations/layer15_attention_weight_max": 35.98944091796875,
      "activations/layer15_attention_weight_min": -30.492475509643555,
      "activations/layer16_attention_weight_max": 30.115007400512695,
      "activations/layer16_attention_weight_min": -26.819555282592773,
      "activations/layer17_attention_weight_max": 33.923492431640625,
      "activations/layer17_attention_weight_min": -26.564546585083008,
      "activations/layer18_attention_weight_max": 27.942842483520508,
      "activations/layer18_attention_weight_min": -20.85072135925293,
      "activations/layer19_attention_weight_max": 33.264060974121094,
      "activations/layer19_attention_weight_min": -31.736310958862305,
      "activations/layer1_attention_weight_max": 18.23248863220215,
      "activations/layer1_attention_weight_min": -13.938508987426758,
      "activations/layer20_attention_weight_max": 28.51534080505371,
      "activations/layer20_attention_weight_min": -23.909597396850586,
      "activations/layer21_attention_weight_max": 28.183162689208984,
      "activations/layer21_attention_weight_min": -22.164337158203125,
      "activations/layer22_attention_weight_max": 43.73831558227539,
      "activations/layer22_attention_weight_min": -28.920610427856445,
      "activations/layer23_attention_weight_max": 31.8363094329834,
      "activations/layer23_attention_weight_min": -22.57080078125,
      "activations/layer2_attention_weight_max": 30.35834503173828,
      "activations/layer2_attention_weight_min": -30.596290588378906,
      "activations/layer3_attention_weight_max": 89.12203216552734,
      "activations/layer3_attention_weight_min": -88.86634826660156,
      "activations/layer4_attention_weight_max": 118.5739517211914,
      "activations/layer4_attention_weight_min": -117.27789306640625,
      "activations/layer5_attention_weight_max": 61.66059494018555,
      "activations/layer5_attention_weight_min": -61.40538024902344,
      "activations/layer6_attention_weight_max": 48.099422454833984,
      "activations/layer6_attention_weight_min": -53.21512985229492,
      "activations/layer7_attention_weight_max": 82.3966293334961,
      "activations/layer7_attention_weight_min": -83.65858459472656,
      "activations/layer8_attention_weight_max": 39.53465270996094,
      "activations/layer8_attention_weight_min": -39.804222106933594,
      "activations/layer9_attention_weight_max": 37.773277282714844,
      "activations/layer9_attention_weight_min": -33.809059143066406,
      "epoch": 7.04,
      "learning_rate": 0.00010564734848484847,
      "loss": 2.8194,
      "step": 121200
    },
    {
      "activations/layer0_attention_weight_max": 16.305816650390625,
      "activations/layer0_attention_weight_min": -12.278738021850586,
      "activations/layer10_attention_weight_max": 29.9259033203125,
      "activations/layer10_attention_weight_min": -29.45450210571289,
      "activations/layer11_attention_weight_max": 28.888906478881836,
      "activations/layer11_attention_weight_min": -27.782814025878906,
      "activations/layer12_attention_weight_max": 20.142475128173828,
      "activations/layer12_attention_weight_min": -27.899316787719727,
      "activations/layer13_attention_weight_max": 38.30732727050781,
      "activations/layer13_attention_weight_min": -31.965450286865234,
      "activations/layer14_attention_weight_max": 46.05298614501953,
      "activations/layer14_attention_weight_min": -38.947349548339844,
      "activations/layer15_attention_weight_max": 33.1190185546875,
      "activations/layer15_attention_weight_min": -28.68865966796875,
      "activations/layer16_attention_weight_max": 33.000526428222656,
      "activations/layer16_attention_weight_min": -27.688644409179688,
      "activations/layer17_attention_weight_max": 30.521575927734375,
      "activations/layer17_attention_weight_min": -25.344438552856445,
      "activations/layer18_attention_weight_max": 29.681968688964844,
      "activations/layer18_attention_weight_min": -21.131929397583008,
      "activations/layer19_attention_weight_max": 34.03757858276367,
      "activations/layer19_attention_weight_min": -28.798002243041992,
      "activations/layer1_attention_weight_max": 16.994609832763672,
      "activations/layer1_attention_weight_min": -14.327113151550293,
      "activations/layer20_attention_weight_max": 28.567176818847656,
      "activations/layer20_attention_weight_min": -23.090795516967773,
      "activations/layer21_attention_weight_max": 28.275157928466797,
      "activations/layer21_attention_weight_min": -22.026424407958984,
      "activations/layer22_attention_weight_max": 42.30878448486328,
      "activations/layer22_attention_weight_min": -28.49574089050293,
      "activations/layer23_attention_weight_max": 33.50935363769531,
      "activations/layer23_attention_weight_min": -23.80677604675293,
      "activations/layer2_attention_weight_max": 31.487529754638672,
      "activations/layer2_attention_weight_min": -30.408401489257812,
      "activations/layer3_attention_weight_max": 88.40760803222656,
      "activations/layer3_attention_weight_min": -88.93074798583984,
      "activations/layer4_attention_weight_max": 121.2237319946289,
      "activations/layer4_attention_weight_min": -120.14263153076172,
      "activations/layer5_attention_weight_max": 68.12115478515625,
      "activations/layer5_attention_weight_min": -62.26909637451172,
      "activations/layer6_attention_weight_max": 50.310707092285156,
      "activations/layer6_attention_weight_min": -52.79401397705078,
      "activations/layer7_attention_weight_max": 82.15261840820312,
      "activations/layer7_attention_weight_min": -82.6617660522461,
      "activations/layer8_attention_weight_max": 38.12434005737305,
      "activations/layer8_attention_weight_min": -37.68123245239258,
      "activations/layer9_attention_weight_max": 34.010894775390625,
      "activations/layer9_attention_weight_min": -33.035194396972656,
      "epoch": 7.05,
      "learning_rate": 0.00010562840909090908,
      "loss": 2.7951,
      "step": 121250
    },
    {
      "activations/layer0_attention_weight_max": 16.268070220947266,
      "activations/layer0_attention_weight_min": -12.474365234375,
      "activations/layer10_attention_weight_max": 32.26124954223633,
      "activations/layer10_attention_weight_min": -31.52604103088379,
      "activations/layer11_attention_weight_max": 31.996875762939453,
      "activations/layer11_attention_weight_min": -28.845670700073242,
      "activations/layer12_attention_weight_max": 27.219806671142578,
      "activations/layer12_attention_weight_min": -26.54892921447754,
      "activations/layer13_attention_weight_max": 41.48952865600586,
      "activations/layer13_attention_weight_min": -32.43074035644531,
      "activations/layer14_attention_weight_max": 52.60735321044922,
      "activations/layer14_attention_weight_min": -44.63996505737305,
      "activations/layer15_attention_weight_max": 34.19517135620117,
      "activations/layer15_attention_weight_min": -27.81243896484375,
      "activations/layer16_attention_weight_max": 32.16025161743164,
      "activations/layer16_attention_weight_min": -29.198875427246094,
      "activations/layer17_attention_weight_max": 36.407806396484375,
      "activations/layer17_attention_weight_min": -25.000139236450195,
      "activations/layer18_attention_weight_max": 32.90805435180664,
      "activations/layer18_attention_weight_min": -22.767986297607422,
      "activations/layer19_attention_weight_max": 38.2481803894043,
      "activations/layer19_attention_weight_min": -28.596826553344727,
      "activations/layer1_attention_weight_max": 17.1567440032959,
      "activations/layer1_attention_weight_min": -15.47406005859375,
      "activations/layer20_attention_weight_max": 32.67081069946289,
      "activations/layer20_attention_weight_min": -23.253740310668945,
      "activations/layer21_attention_weight_max": 30.572532653808594,
      "activations/layer21_attention_weight_min": -21.960622787475586,
      "activations/layer22_attention_weight_max": 45.52482223510742,
      "activations/layer22_attention_weight_min": -27.781829833984375,
      "activations/layer23_attention_weight_max": 35.04957580566406,
      "activations/layer23_attention_weight_min": -23.037017822265625,
      "activations/layer2_attention_weight_max": 33.69123077392578,
      "activations/layer2_attention_weight_min": -32.99506378173828,
      "activations/layer3_attention_weight_max": 90.30258178710938,
      "activations/layer3_attention_weight_min": -92.46436309814453,
      "activations/layer4_attention_weight_max": 126.00628662109375,
      "activations/layer4_attention_weight_min": -123.1485595703125,
      "activations/layer5_attention_weight_max": 64.34226989746094,
      "activations/layer5_attention_weight_min": -65.20538330078125,
      "activations/layer6_attention_weight_max": 50.56816482543945,
      "activations/layer6_attention_weight_min": -52.09376907348633,
      "activations/layer7_attention_weight_max": 85.154296875,
      "activations/layer7_attention_weight_min": -83.17924499511719,
      "activations/layer8_attention_weight_max": 41.34941101074219,
      "activations/layer8_attention_weight_min": -39.18172073364258,
      "activations/layer9_attention_weight_max": 35.98625183105469,
      "activations/layer9_attention_weight_min": -33.73405456542969,
      "epoch": 7.05,
      "learning_rate": 0.0001056094696969697,
      "loss": 2.8252,
      "step": 121300
    },
    {
      "activations/layer0_attention_weight_max": 16.118785858154297,
      "activations/layer0_attention_weight_min": -11.900829315185547,
      "activations/layer10_attention_weight_max": 31.87954330444336,
      "activations/layer10_attention_weight_min": -31.379711151123047,
      "activations/layer11_attention_weight_max": 30.09839630126953,
      "activations/layer11_attention_weight_min": -31.038414001464844,
      "activations/layer12_attention_weight_max": 25.997163772583008,
      "activations/layer12_attention_weight_min": -22.78939437866211,
      "activations/layer13_attention_weight_max": 51.05352020263672,
      "activations/layer13_attention_weight_min": -41.51704025268555,
      "activations/layer14_attention_weight_max": 56.139034271240234,
      "activations/layer14_attention_weight_min": -48.579627990722656,
      "activations/layer15_attention_weight_max": 38.58544158935547,
      "activations/layer15_attention_weight_min": -29.547231674194336,
      "activations/layer16_attention_weight_max": 37.9738655090332,
      "activations/layer16_attention_weight_min": -30.503515243530273,
      "activations/layer17_attention_weight_max": 33.81351089477539,
      "activations/layer17_attention_weight_min": -25.57847023010254,
      "activations/layer18_attention_weight_max": 28.73675537109375,
      "activations/layer18_attention_weight_min": -20.137252807617188,
      "activations/layer19_attention_weight_max": 40.1294059753418,
      "activations/layer19_attention_weight_min": -29.844663619995117,
      "activations/layer1_attention_weight_max": 17.396814346313477,
      "activations/layer1_attention_weight_min": -12.567851066589355,
      "activations/layer20_attention_weight_max": 33.23139190673828,
      "activations/layer20_attention_weight_min": -23.335020065307617,
      "activations/layer21_attention_weight_max": 31.729156494140625,
      "activations/layer21_attention_weight_min": -21.677278518676758,
      "activations/layer22_attention_weight_max": 51.53247833251953,
      "activations/layer22_attention_weight_min": -29.860885620117188,
      "activations/layer23_attention_weight_max": 37.53594207763672,
      "activations/layer23_attention_weight_min": -25.032176971435547,
      "activations/layer2_attention_weight_max": 31.24036407470703,
      "activations/layer2_attention_weight_min": -29.13015365600586,
      "activations/layer3_attention_weight_max": 87.98097229003906,
      "activations/layer3_attention_weight_min": -87.47373962402344,
      "activations/layer4_attention_weight_max": 124.28765106201172,
      "activations/layer4_attention_weight_min": -116.50154113769531,
      "activations/layer5_attention_weight_max": 65.3821029663086,
      "activations/layer5_attention_weight_min": -62.42059326171875,
      "activations/layer6_attention_weight_max": 50.01633071899414,
      "activations/layer6_attention_weight_min": -52.06007766723633,
      "activations/layer7_attention_weight_max": 89.48214721679688,
      "activations/layer7_attention_weight_min": -84.63980865478516,
      "activations/layer8_attention_weight_max": 38.614768981933594,
      "activations/layer8_attention_weight_min": -39.01847839355469,
      "activations/layer9_attention_weight_max": 34.14084243774414,
      "activations/layer9_attention_weight_min": -32.10456848144531,
      "epoch": 7.05,
      "learning_rate": 0.00010559053030303029,
      "loss": 2.8157,
      "step": 121350
    },
    {
      "activations/layer0_attention_weight_max": 15.643220901489258,
      "activations/layer0_attention_weight_min": -11.303996086120605,
      "activations/layer10_attention_weight_max": 33.598724365234375,
      "activations/layer10_attention_weight_min": -31.28435516357422,
      "activations/layer11_attention_weight_max": 32.188873291015625,
      "activations/layer11_attention_weight_min": -29.15547752380371,
      "activations/layer12_attention_weight_max": 22.884761810302734,
      "activations/layer12_attention_weight_min": -24.092010498046875,
      "activations/layer13_attention_weight_max": 45.59428787231445,
      "activations/layer13_attention_weight_min": -34.78199768066406,
      "activations/layer14_attention_weight_max": 57.385841369628906,
      "activations/layer14_attention_weight_min": -40.46351623535156,
      "activations/layer15_attention_weight_max": 38.32949447631836,
      "activations/layer15_attention_weight_min": -28.76934814453125,
      "activations/layer16_attention_weight_max": 33.9686164855957,
      "activations/layer16_attention_weight_min": -29.340681076049805,
      "activations/layer17_attention_weight_max": 37.08247375488281,
      "activations/layer17_attention_weight_min": -28.15949058532715,
      "activations/layer18_attention_weight_max": 32.23695373535156,
      "activations/layer18_attention_weight_min": -21.534997940063477,
      "activations/layer19_attention_weight_max": 44.05842208862305,
      "activations/layer19_attention_weight_min": -31.375699996948242,
      "activations/layer1_attention_weight_max": 17.49266815185547,
      "activations/layer1_attention_weight_min": -13.134740829467773,
      "activations/layer20_attention_weight_max": 29.317277908325195,
      "activations/layer20_attention_weight_min": -24.235157012939453,
      "activations/layer21_attention_weight_max": 28.98326301574707,
      "activations/layer21_attention_weight_min": -24.09783363342285,
      "activations/layer22_attention_weight_max": 45.361053466796875,
      "activations/layer22_attention_weight_min": -30.565820693969727,
      "activations/layer23_attention_weight_max": 33.42844772338867,
      "activations/layer23_attention_weight_min": -22.48153305053711,
      "activations/layer2_attention_weight_max": 31.211994171142578,
      "activations/layer2_attention_weight_min": -30.078786849975586,
      "activations/layer3_attention_weight_max": 90.9860610961914,
      "activations/layer3_attention_weight_min": -89.05845642089844,
      "activations/layer4_attention_weight_max": 125.09905242919922,
      "activations/layer4_attention_weight_min": -118.65235900878906,
      "activations/layer5_attention_weight_max": 62.94712829589844,
      "activations/layer5_attention_weight_min": -61.37386703491211,
      "activations/layer6_attention_weight_max": 52.097415924072266,
      "activations/layer6_attention_weight_min": -55.304603576660156,
      "activations/layer7_attention_weight_max": 94.16191864013672,
      "activations/layer7_attention_weight_min": -88.4924545288086,
      "activations/layer8_attention_weight_max": 42.12017822265625,
      "activations/layer8_attention_weight_min": -40.90193557739258,
      "activations/layer9_attention_weight_max": 36.09287643432617,
      "activations/layer9_attention_weight_min": -34.331298828125,
      "epoch": 7.05,
      "learning_rate": 0.0001055715909090909,
      "loss": 2.8017,
      "step": 121400
    },
    {
      "activations/layer0_attention_weight_max": 14.826577186584473,
      "activations/layer0_attention_weight_min": -11.423660278320312,
      "activations/layer10_attention_weight_max": 33.33906173706055,
      "activations/layer10_attention_weight_min": -30.57890510559082,
      "activations/layer11_attention_weight_max": 31.884601593017578,
      "activations/layer11_attention_weight_min": -28.050939559936523,
      "activations/layer12_attention_weight_max": 24.000885009765625,
      "activations/layer12_attention_weight_min": -25.127975463867188,
      "activations/layer13_attention_weight_max": 38.256561279296875,
      "activations/layer13_attention_weight_min": -31.405263900756836,
      "activations/layer14_attention_weight_max": 54.32351303100586,
      "activations/layer14_attention_weight_min": -41.094608306884766,
      "activations/layer15_attention_weight_max": 36.250762939453125,
      "activations/layer15_attention_weight_min": -28.814292907714844,
      "activations/layer16_attention_weight_max": 33.22248458862305,
      "activations/layer16_attention_weight_min": -26.060482025146484,
      "activations/layer17_attention_weight_max": 37.95359802246094,
      "activations/layer17_attention_weight_min": -23.85692596435547,
      "activations/layer18_attention_weight_max": 31.182451248168945,
      "activations/layer18_attention_weight_min": -21.370515823364258,
      "activations/layer19_attention_weight_max": 35.00433349609375,
      "activations/layer19_attention_weight_min": -29.23419761657715,
      "activations/layer1_attention_weight_max": 17.22684097290039,
      "activations/layer1_attention_weight_min": -13.742305755615234,
      "activations/layer20_attention_weight_max": 29.45427894592285,
      "activations/layer20_attention_weight_min": -24.1319580078125,
      "activations/layer21_attention_weight_max": 31.67770004272461,
      "activations/layer21_attention_weight_min": -22.26883888244629,
      "activations/layer22_attention_weight_max": 46.611881256103516,
      "activations/layer22_attention_weight_min": -29.358930587768555,
      "activations/layer23_attention_weight_max": 36.05901336669922,
      "activations/layer23_attention_weight_min": -22.02259635925293,
      "activations/layer2_attention_weight_max": 29.53059959411621,
      "activations/layer2_attention_weight_min": -30.475013732910156,
      "activations/layer3_attention_weight_max": 90.40428924560547,
      "activations/layer3_attention_weight_min": -90.39625549316406,
      "activations/layer4_attention_weight_max": 118.31966400146484,
      "activations/layer4_attention_weight_min": -113.58919525146484,
      "activations/layer5_attention_weight_max": 61.804718017578125,
      "activations/layer5_attention_weight_min": -62.142372131347656,
      "activations/layer6_attention_weight_max": 48.535865783691406,
      "activations/layer6_attention_weight_min": -52.36240768432617,
      "activations/layer7_attention_weight_max": 84.71590423583984,
      "activations/layer7_attention_weight_min": -85.16484832763672,
      "activations/layer8_attention_weight_max": 38.33312225341797,
      "activations/layer8_attention_weight_min": -38.10578155517578,
      "activations/layer9_attention_weight_max": 35.275657653808594,
      "activations/layer9_attention_weight_min": -32.43123245239258,
      "epoch": 7.06,
      "learning_rate": 0.0001055526515151515,
      "loss": 2.81,
      "step": 121450
    },
    {
      "activations/layer0_attention_weight_max": 16.618059158325195,
      "activations/layer0_attention_weight_min": -11.025654792785645,
      "activations/layer10_attention_weight_max": 39.18746566772461,
      "activations/layer10_attention_weight_min": -32.85243606567383,
      "activations/layer11_attention_weight_max": 41.79822540283203,
      "activations/layer11_attention_weight_min": -33.21692657470703,
      "activations/layer12_attention_weight_max": 32.29318618774414,
      "activations/layer12_attention_weight_min": -23.9599666595459,
      "activations/layer13_attention_weight_max": 44.15614318847656,
      "activations/layer13_attention_weight_min": -36.63395690917969,
      "activations/layer14_attention_weight_max": 57.359439849853516,
      "activations/layer14_attention_weight_min": -42.07926940917969,
      "activations/layer15_attention_weight_max": 45.45719528198242,
      "activations/layer15_attention_weight_min": -31.550142288208008,
      "activations/layer16_attention_weight_max": 39.32597732543945,
      "activations/layer16_attention_weight_min": -31.76192283630371,
      "activations/layer17_attention_weight_max": 40.46855926513672,
      "activations/layer17_attention_weight_min": -27.225561141967773,
      "activations/layer18_attention_weight_max": 31.455543518066406,
      "activations/layer18_attention_weight_min": -23.295001983642578,
      "activations/layer19_attention_weight_max": 49.15592575073242,
      "activations/layer19_attention_weight_min": -32.49176788330078,
      "activations/layer1_attention_weight_max": 16.834218978881836,
      "activations/layer1_attention_weight_min": -13.254651069641113,
      "activations/layer20_attention_weight_max": 39.416343688964844,
      "activations/layer20_attention_weight_min": -25.52507781982422,
      "activations/layer21_attention_weight_max": 34.006324768066406,
      "activations/layer21_attention_weight_min": -22.832204818725586,
      "activations/layer22_attention_weight_max": 57.41434860229492,
      "activations/layer22_attention_weight_min": -29.13089942932129,
      "activations/layer23_attention_weight_max": 37.78110122680664,
      "activations/layer23_attention_weight_min": -21.465246200561523,
      "activations/layer2_attention_weight_max": 31.447935104370117,
      "activations/layer2_attention_weight_min": -30.573102951049805,
      "activations/layer3_attention_weight_max": 91.51406860351562,
      "activations/layer3_attention_weight_min": -91.74259948730469,
      "activations/layer4_attention_weight_max": 123.0846176147461,
      "activations/layer4_attention_weight_min": -120.9013671875,
      "activations/layer5_attention_weight_max": 65.41732788085938,
      "activations/layer5_attention_weight_min": -63.06591033935547,
      "activations/layer6_attention_weight_max": 54.91298294067383,
      "activations/layer6_attention_weight_min": -53.75044631958008,
      "activations/layer7_attention_weight_max": 95.53311920166016,
      "activations/layer7_attention_weight_min": -92.11253356933594,
      "activations/layer8_attention_weight_max": 42.43404006958008,
      "activations/layer8_attention_weight_min": -41.79135513305664,
      "activations/layer9_attention_weight_max": 38.75532531738281,
      "activations/layer9_attention_weight_min": -36.21702575683594,
      "epoch": 7.06,
      "learning_rate": 0.0001055340909090909,
      "loss": 2.8012,
      "step": 121500
    },
    {
      "activations/layer0_attention_weight_max": 16.144556045532227,
      "activations/layer0_attention_weight_min": -11.300448417663574,
      "activations/layer10_attention_weight_max": 35.88134765625,
      "activations/layer10_attention_weight_min": -33.80268096923828,
      "activations/layer11_attention_weight_max": 33.58419418334961,
      "activations/layer11_attention_weight_min": -30.403987884521484,
      "activations/layer12_attention_weight_max": 23.655466079711914,
      "activations/layer12_attention_weight_min": -27.433534622192383,
      "activations/layer13_attention_weight_max": 44.098854064941406,
      "activations/layer13_attention_weight_min": -35.125789642333984,
      "activations/layer14_attention_weight_max": 48.59967803955078,
      "activations/layer14_attention_weight_min": -37.400054931640625,
      "activations/layer15_attention_weight_max": 37.79032897949219,
      "activations/layer15_attention_weight_min": -29.833499908447266,
      "activations/layer16_attention_weight_max": 38.25509262084961,
      "activations/layer16_attention_weight_min": -28.441791534423828,
      "activations/layer17_attention_weight_max": 36.14120864868164,
      "activations/layer17_attention_weight_min": -23.39482879638672,
      "activations/layer18_attention_weight_max": 31.635215759277344,
      "activations/layer18_attention_weight_min": -20.74201011657715,
      "activations/layer19_attention_weight_max": 42.03519821166992,
      "activations/layer19_attention_weight_min": -29.75126075744629,
      "activations/layer1_attention_weight_max": 17.0742244720459,
      "activations/layer1_attention_weight_min": -14.426961898803711,
      "activations/layer20_attention_weight_max": 33.635894775390625,
      "activations/layer20_attention_weight_min": -23.099699020385742,
      "activations/layer21_attention_weight_max": 30.493778228759766,
      "activations/layer21_attention_weight_min": -21.431678771972656,
      "activations/layer22_attention_weight_max": 47.218017578125,
      "activations/layer22_attention_weight_min": -28.217384338378906,
      "activations/layer23_attention_weight_max": 35.60071563720703,
      "activations/layer23_attention_weight_min": -21.472885131835938,
      "activations/layer2_attention_weight_max": 31.0496826171875,
      "activations/layer2_attention_weight_min": -30.632488250732422,
      "activations/layer3_attention_weight_max": 91.4856948852539,
      "activations/layer3_attention_weight_min": -91.42487335205078,
      "activations/layer4_attention_weight_max": 123.34754943847656,
      "activations/layer4_attention_weight_min": -121.21382141113281,
      "activations/layer5_attention_weight_max": 64.52288818359375,
      "activations/layer5_attention_weight_min": -61.52848815917969,
      "activations/layer6_attention_weight_max": 54.57278823852539,
      "activations/layer6_attention_weight_min": -54.13610076904297,
      "activations/layer7_attention_weight_max": 92.41502380371094,
      "activations/layer7_attention_weight_min": -90.79295349121094,
      "activations/layer8_attention_weight_max": 42.76057052612305,
      "activations/layer8_attention_weight_min": -43.320404052734375,
      "activations/layer9_attention_weight_max": 39.38509750366211,
      "activations/layer9_attention_weight_min": -36.603878021240234,
      "epoch": 7.06,
      "learning_rate": 0.00010551515151515151,
      "loss": 2.8242,
      "step": 121550
    },
    {
      "activations/layer0_attention_weight_max": 16.492406845092773,
      "activations/layer0_attention_weight_min": -11.718807220458984,
      "activations/layer10_attention_weight_max": 39.53988265991211,
      "activations/layer10_attention_weight_min": -34.75715637207031,
      "activations/layer11_attention_weight_max": 38.240928649902344,
      "activations/layer11_attention_weight_min": -32.313846588134766,
      "activations/layer12_attention_weight_max": 25.429351806640625,
      "activations/layer12_attention_weight_min": -27.37577247619629,
      "activations/layer13_attention_weight_max": 47.12095642089844,
      "activations/layer13_attention_weight_min": -33.80082321166992,
      "activations/layer14_attention_weight_max": 62.72340774536133,
      "activations/layer14_attention_weight_min": -40.16486358642578,
      "activations/layer15_attention_weight_max": 38.02529525756836,
      "activations/layer15_attention_weight_min": -29.872488021850586,
      "activations/layer16_attention_weight_max": 33.40632629394531,
      "activations/layer16_attention_weight_min": -27.411176681518555,
      "activations/layer17_attention_weight_max": 38.7990837097168,
      "activations/layer17_attention_weight_min": -24.919130325317383,
      "activations/layer18_attention_weight_max": 35.429229736328125,
      "activations/layer18_attention_weight_min": -21.51323127746582,
      "activations/layer19_attention_weight_max": 38.028053283691406,
      "activations/layer19_attention_weight_min": -32.1713752746582,
      "activations/layer1_attention_weight_max": 16.642990112304688,
      "activations/layer1_attention_weight_min": -14.78152084350586,
      "activations/layer20_attention_weight_max": 30.19866943359375,
      "activations/layer20_attention_weight_min": -24.132648468017578,
      "activations/layer21_attention_weight_max": 32.1220817565918,
      "activations/layer21_attention_weight_min": -23.440744400024414,
      "activations/layer22_attention_weight_max": 54.37200164794922,
      "activations/layer22_attention_weight_min": -30.21172332763672,
      "activations/layer23_attention_weight_max": 38.85344696044922,
      "activations/layer23_attention_weight_min": -23.01388931274414,
      "activations/layer2_attention_weight_max": 33.35043716430664,
      "activations/layer2_attention_weight_min": -32.410186767578125,
      "activations/layer3_attention_weight_max": 93.94032287597656,
      "activations/layer3_attention_weight_min": -94.24942016601562,
      "activations/layer4_attention_weight_max": 131.73263549804688,
      "activations/layer4_attention_weight_min": -126.97521209716797,
      "activations/layer5_attention_weight_max": 71.15504455566406,
      "activations/layer5_attention_weight_min": -64.94889068603516,
      "activations/layer6_attention_weight_max": 56.272464752197266,
      "activations/layer6_attention_weight_min": -56.364784240722656,
      "activations/layer7_attention_weight_max": 94.34927368164062,
      "activations/layer7_attention_weight_min": -93.05946350097656,
      "activations/layer8_attention_weight_max": 48.61162185668945,
      "activations/layer8_attention_weight_min": -45.5592155456543,
      "activations/layer9_attention_weight_max": 43.36042404174805,
      "activations/layer9_attention_weight_min": -37.91776657104492,
      "epoch": 7.07,
      "learning_rate": 0.00010549621212121211,
      "loss": 2.8289,
      "step": 121600
    },
    {
      "activations/layer0_attention_weight_max": 16.16585922241211,
      "activations/layer0_attention_weight_min": -12.819759368896484,
      "activations/layer10_attention_weight_max": 35.54696273803711,
      "activations/layer10_attention_weight_min": -31.315532684326172,
      "activations/layer11_attention_weight_max": 34.8064079284668,
      "activations/layer11_attention_weight_min": -30.121885299682617,
      "activations/layer12_attention_weight_max": 23.723783493041992,
      "activations/layer12_attention_weight_min": -26.855051040649414,
      "activations/layer13_attention_weight_max": 46.89881896972656,
      "activations/layer13_attention_weight_min": -33.38869857788086,
      "activations/layer14_attention_weight_max": 52.67457580566406,
      "activations/layer14_attention_weight_min": -40.38138961791992,
      "activations/layer15_attention_weight_max": 38.92287826538086,
      "activations/layer15_attention_weight_min": -29.072463989257812,
      "activations/layer16_attention_weight_max": 36.06263732910156,
      "activations/layer16_attention_weight_min": -29.816165924072266,
      "activations/layer17_attention_weight_max": 34.62458419799805,
      "activations/layer17_attention_weight_min": -25.759716033935547,
      "activations/layer18_attention_weight_max": 36.430023193359375,
      "activations/layer18_attention_weight_min": -21.679292678833008,
      "activations/layer19_attention_weight_max": 35.20840072631836,
      "activations/layer19_attention_weight_min": -31.498062133789062,
      "activations/layer1_attention_weight_max": 16.772197723388672,
      "activations/layer1_attention_weight_min": -14.807929992675781,
      "activations/layer20_attention_weight_max": 30.133981704711914,
      "activations/layer20_attention_weight_min": -24.10428237915039,
      "activations/layer21_attention_weight_max": 31.909446716308594,
      "activations/layer21_attention_weight_min": -22.30061149597168,
      "activations/layer22_attention_weight_max": 49.67279052734375,
      "activations/layer22_attention_weight_min": -31.00902557373047,
      "activations/layer23_attention_weight_max": 37.215152740478516,
      "activations/layer23_attention_weight_min": -23.947711944580078,
      "activations/layer2_attention_weight_max": 32.68185806274414,
      "activations/layer2_attention_weight_min": -30.55196762084961,
      "activations/layer3_attention_weight_max": 90.31021881103516,
      "activations/layer3_attention_weight_min": -91.3589859008789,
      "activations/layer4_attention_weight_max": 123.39176177978516,
      "activations/layer4_attention_weight_min": -121.87174224853516,
      "activations/layer5_attention_weight_max": 66.87844848632812,
      "activations/layer5_attention_weight_min": -62.831687927246094,
      "activations/layer6_attention_weight_max": 51.83235168457031,
      "activations/layer6_attention_weight_min": -54.04961395263672,
      "activations/layer7_attention_weight_max": 86.59671783447266,
      "activations/layer7_attention_weight_min": -90.20672607421875,
      "activations/layer8_attention_weight_max": 40.40640640258789,
      "activations/layer8_attention_weight_min": -41.630985260009766,
      "activations/layer9_attention_weight_max": 35.62567901611328,
      "activations/layer9_attention_weight_min": -34.40680694580078,
      "epoch": 7.07,
      "learning_rate": 0.00010547727272727271,
      "loss": 2.8192,
      "step": 121650
    },
    {
      "activations/layer0_attention_weight_max": 16.158559799194336,
      "activations/layer0_attention_weight_min": -11.580374717712402,
      "activations/layer10_attention_weight_max": 33.47614288330078,
      "activations/layer10_attention_weight_min": -33.30864334106445,
      "activations/layer11_attention_weight_max": 35.915321350097656,
      "activations/layer11_attention_weight_min": -28.924789428710938,
      "activations/layer12_attention_weight_max": 27.5828800201416,
      "activations/layer12_attention_weight_min": -28.055334091186523,
      "activations/layer13_attention_weight_max": 42.74271011352539,
      "activations/layer13_attention_weight_min": -32.70726013183594,
      "activations/layer14_attention_weight_max": 59.430355072021484,
      "activations/layer14_attention_weight_min": -38.76504898071289,
      "activations/layer15_attention_weight_max": 37.71498489379883,
      "activations/layer15_attention_weight_min": -29.36597442626953,
      "activations/layer16_attention_weight_max": 35.32524871826172,
      "activations/layer16_attention_weight_min": -27.024688720703125,
      "activations/layer17_attention_weight_max": 38.37314224243164,
      "activations/layer17_attention_weight_min": -24.78232192993164,
      "activations/layer18_attention_weight_max": 39.8629264831543,
      "activations/layer18_attention_weight_min": -21.450035095214844,
      "activations/layer19_attention_weight_max": 39.62858200073242,
      "activations/layer19_attention_weight_min": -31.658597946166992,
      "activations/layer1_attention_weight_max": 16.671783447265625,
      "activations/layer1_attention_weight_min": -14.078631401062012,
      "activations/layer20_attention_weight_max": 33.27036666870117,
      "activations/layer20_attention_weight_min": -24.40544319152832,
      "activations/layer21_attention_weight_max": 39.290367126464844,
      "activations/layer21_attention_weight_min": -23.262842178344727,
      "activations/layer22_attention_weight_max": 47.06662368774414,
      "activations/layer22_attention_weight_min": -29.262075424194336,
      "activations/layer23_attention_weight_max": 39.363582611083984,
      "activations/layer23_attention_weight_min": -24.527448654174805,
      "activations/layer2_attention_weight_max": 31.03528594970703,
      "activations/layer2_attention_weight_min": -30.446388244628906,
      "activations/layer3_attention_weight_max": 87.85684204101562,
      "activations/layer3_attention_weight_min": -90.70277404785156,
      "activations/layer4_attention_weight_max": 118.93651580810547,
      "activations/layer4_attention_weight_min": -126.06670379638672,
      "activations/layer5_attention_weight_max": 64.27603149414062,
      "activations/layer5_attention_weight_min": -68.2183837890625,
      "activations/layer6_attention_weight_max": 54.576683044433594,
      "activations/layer6_attention_weight_min": -53.78946304321289,
      "activations/layer7_attention_weight_max": 88.00154876708984,
      "activations/layer7_attention_weight_min": -88.09638977050781,
      "activations/layer8_attention_weight_max": 41.02119827270508,
      "activations/layer8_attention_weight_min": -41.50418472290039,
      "activations/layer9_attention_weight_max": 36.79233932495117,
      "activations/layer9_attention_weight_min": -35.15219497680664,
      "epoch": 7.07,
      "learning_rate": 0.00010545833333333333,
      "loss": 2.8129,
      "step": 121700
    },
    {
      "activations/layer0_attention_weight_max": 16.2741756439209,
      "activations/layer0_attention_weight_min": -11.426190376281738,
      "activations/layer10_attention_weight_max": 32.69905471801758,
      "activations/layer10_attention_weight_min": -38.18178939819336,
      "activations/layer11_attention_weight_max": 34.961570739746094,
      "activations/layer11_attention_weight_min": -33.012290954589844,
      "activations/layer12_attention_weight_max": 51.58198547363281,
      "activations/layer12_attention_weight_min": -28.858396530151367,
      "activations/layer13_attention_weight_max": 55.093082427978516,
      "activations/layer13_attention_weight_min": -34.8592643737793,
      "activations/layer14_attention_weight_max": 82.16326904296875,
      "activations/layer14_attention_weight_min": -48.07023239135742,
      "activations/layer15_attention_weight_max": 43.44807815551758,
      "activations/layer15_attention_weight_min": -30.43097496032715,
      "activations/layer16_attention_weight_max": 35.958126068115234,
      "activations/layer16_attention_weight_min": -27.708744049072266,
      "activations/layer17_attention_weight_max": 42.740482330322266,
      "activations/layer17_attention_weight_min": -25.897396087646484,
      "activations/layer18_attention_weight_max": 37.69554901123047,
      "activations/layer18_attention_weight_min": -21.664091110229492,
      "activations/layer19_attention_weight_max": 38.531898498535156,
      "activations/layer19_attention_weight_min": -28.595096588134766,
      "activations/layer1_attention_weight_max": 17.869768142700195,
      "activations/layer1_attention_weight_min": -14.768057823181152,
      "activations/layer20_attention_weight_max": 33.473079681396484,
      "activations/layer20_attention_weight_min": -23.596643447875977,
      "activations/layer21_attention_weight_max": 30.190732955932617,
      "activations/layer21_attention_weight_min": -23.46742820739746,
      "activations/layer22_attention_weight_max": 44.305702209472656,
      "activations/layer22_attention_weight_min": -27.449615478515625,
      "activations/layer23_attention_weight_max": 34.50554656982422,
      "activations/layer23_attention_weight_min": -21.708364486694336,
      "activations/layer2_attention_weight_max": 32.09680938720703,
      "activations/layer2_attention_weight_min": -33.775550842285156,
      "activations/layer3_attention_weight_max": 94.12751007080078,
      "activations/layer3_attention_weight_min": -103.33749389648438,
      "activations/layer4_attention_weight_max": 128.68203735351562,
      "activations/layer4_attention_weight_min": -132.29061889648438,
      "activations/layer5_attention_weight_max": 62.717281341552734,
      "activations/layer5_attention_weight_min": -64.81074523925781,
      "activations/layer6_attention_weight_max": 50.046932220458984,
      "activations/layer6_attention_weight_min": -52.90617752075195,
      "activations/layer7_attention_weight_max": 90.92524719238281,
      "activations/layer7_attention_weight_min": -100.36925506591797,
      "activations/layer8_attention_weight_max": 42.588050842285156,
      "activations/layer8_attention_weight_min": -43.58097457885742,
      "activations/layer9_attention_weight_max": 37.66999816894531,
      "activations/layer9_attention_weight_min": -37.69889831542969,
      "epoch": 7.07,
      "learning_rate": 0.00010543939393939394,
      "loss": 2.8106,
      "step": 121750
    },
    {
      "activations/layer0_attention_weight_max": 15.35495662689209,
      "activations/layer0_attention_weight_min": -11.885478973388672,
      "activations/layer10_attention_weight_max": 32.713314056396484,
      "activations/layer10_attention_weight_min": -31.515865325927734,
      "activations/layer11_attention_weight_max": 31.417203903198242,
      "activations/layer11_attention_weight_min": -28.859708786010742,
      "activations/layer12_attention_weight_max": 26.537755966186523,
      "activations/layer12_attention_weight_min": -25.586963653564453,
      "activations/layer13_attention_weight_max": 39.967430114746094,
      "activations/layer13_attention_weight_min": -33.33723068237305,
      "activations/layer14_attention_weight_max": 57.666751861572266,
      "activations/layer14_attention_weight_min": -44.67677688598633,
      "activations/layer15_attention_weight_max": 37.72584915161133,
      "activations/layer15_attention_weight_min": -30.181684494018555,
      "activations/layer16_attention_weight_max": 34.075469970703125,
      "activations/layer16_attention_weight_min": -28.351076126098633,
      "activations/layer17_attention_weight_max": 34.92219161987305,
      "activations/layer17_attention_weight_min": -25.65250015258789,
      "activations/layer18_attention_weight_max": 36.035945892333984,
      "activations/layer18_attention_weight_min": -21.351577758789062,
      "activations/layer19_attention_weight_max": 39.545265197753906,
      "activations/layer19_attention_weight_min": -29.742258071899414,
      "activations/layer1_attention_weight_max": 17.657304763793945,
      "activations/layer1_attention_weight_min": -14.082357406616211,
      "activations/layer20_attention_weight_max": 33.25269317626953,
      "activations/layer20_attention_weight_min": -24.388309478759766,
      "activations/layer21_attention_weight_max": 34.16156005859375,
      "activations/layer21_attention_weight_min": -23.245691299438477,
      "activations/layer22_attention_weight_max": 55.28147888183594,
      "activations/layer22_attention_weight_min": -31.361509323120117,
      "activations/layer23_attention_weight_max": 37.47460174560547,
      "activations/layer23_attention_weight_min": -23.661151885986328,
      "activations/layer2_attention_weight_max": 32.17212677001953,
      "activations/layer2_attention_weight_min": -31.46527862548828,
      "activations/layer3_attention_weight_max": 91.56047058105469,
      "activations/layer3_attention_weight_min": -91.2809829711914,
      "activations/layer4_attention_weight_max": 126.61299896240234,
      "activations/layer4_attention_weight_min": -128.87200927734375,
      "activations/layer5_attention_weight_max": 66.22685241699219,
      "activations/layer5_attention_weight_min": -65.92338562011719,
      "activations/layer6_attention_weight_max": 52.53107452392578,
      "activations/layer6_attention_weight_min": -55.58094787597656,
      "activations/layer7_attention_weight_max": 89.45169067382812,
      "activations/layer7_attention_weight_min": -91.28124237060547,
      "activations/layer8_attention_weight_max": 42.096351623535156,
      "activations/layer8_attention_weight_min": -41.978755950927734,
      "activations/layer9_attention_weight_max": 37.14875411987305,
      "activations/layer9_attention_weight_min": -35.281314849853516,
      "epoch": 7.08,
      "learning_rate": 0.00010542045454545453,
      "loss": 2.8184,
      "step": 121800
    },
    {
      "activations/layer0_attention_weight_max": 16.902929306030273,
      "activations/layer0_attention_weight_min": -11.696581840515137,
      "activations/layer10_attention_weight_max": 31.214927673339844,
      "activations/layer10_attention_weight_min": -30.470340728759766,
      "activations/layer11_attention_weight_max": 30.894149780273438,
      "activations/layer11_attention_weight_min": -28.38166618347168,
      "activations/layer12_attention_weight_max": 47.4356803894043,
      "activations/layer12_attention_weight_min": -24.668048858642578,
      "activations/layer13_attention_weight_max": 49.972679138183594,
      "activations/layer13_attention_weight_min": -33.01774215698242,
      "activations/layer14_attention_weight_max": 65.80423736572266,
      "activations/layer14_attention_weight_min": -43.912601470947266,
      "activations/layer15_attention_weight_max": 39.939247131347656,
      "activations/layer15_attention_weight_min": -30.315324783325195,
      "activations/layer16_attention_weight_max": 39.69752883911133,
      "activations/layer16_attention_weight_min": -27.64246368408203,
      "activations/layer17_attention_weight_max": 42.20780563354492,
      "activations/layer17_attention_weight_min": -25.585657119750977,
      "activations/layer18_attention_weight_max": 33.10704803466797,
      "activations/layer18_attention_weight_min": -21.828535079956055,
      "activations/layer19_attention_weight_max": 42.24154281616211,
      "activations/layer19_attention_weight_min": -30.945276260375977,
      "activations/layer1_attention_weight_max": 17.592470169067383,
      "activations/layer1_attention_weight_min": -13.042757034301758,
      "activations/layer20_attention_weight_max": 35.516605377197266,
      "activations/layer20_attention_weight_min": -24.408784866333008,
      "activations/layer21_attention_weight_max": 30.739620208740234,
      "activations/layer21_attention_weight_min": -22.847360610961914,
      "activations/layer22_attention_weight_max": 49.1533203125,
      "activations/layer22_attention_weight_min": -28.86798858642578,
      "activations/layer23_attention_weight_max": 37.709903717041016,
      "activations/layer23_attention_weight_min": -22.30800437927246,
      "activations/layer2_attention_weight_max": 30.604244232177734,
      "activations/layer2_attention_weight_min": -29.457061767578125,
      "activations/layer3_attention_weight_max": 87.30845642089844,
      "activations/layer3_attention_weight_min": -89.5801773071289,
      "activations/layer4_attention_weight_max": 124.53314208984375,
      "activations/layer4_attention_weight_min": -122.63433837890625,
      "activations/layer5_attention_weight_max": 62.170249938964844,
      "activations/layer5_attention_weight_min": -64.3143081665039,
      "activations/layer6_attention_weight_max": 47.79894256591797,
      "activations/layer6_attention_weight_min": -49.98994064331055,
      "activations/layer7_attention_weight_max": 85.07520294189453,
      "activations/layer7_attention_weight_min": -84.45132446289062,
      "activations/layer8_attention_weight_max": 37.64659118652344,
      "activations/layer8_attention_weight_min": -38.821964263916016,
      "activations/layer9_attention_weight_max": 34.72672653198242,
      "activations/layer9_attention_weight_min": -32.787574768066406,
      "epoch": 7.08,
      "learning_rate": 0.00010540151515151514,
      "loss": 2.8352,
      "step": 121850
    },
    {
      "activations/layer0_attention_weight_max": 14.848493576049805,
      "activations/layer0_attention_weight_min": -11.822619438171387,
      "activations/layer10_attention_weight_max": 33.91131591796875,
      "activations/layer10_attention_weight_min": -32.08374786376953,
      "activations/layer11_attention_weight_max": 33.28864288330078,
      "activations/layer11_attention_weight_min": -31.661361694335938,
      "activations/layer12_attention_weight_max": 27.75959014892578,
      "activations/layer12_attention_weight_min": -23.058500289916992,
      "activations/layer13_attention_weight_max": 41.33220291137695,
      "activations/layer13_attention_weight_min": -33.240840911865234,
      "activations/layer14_attention_weight_max": 64.64519500732422,
      "activations/layer14_attention_weight_min": -42.81761932373047,
      "activations/layer15_attention_weight_max": 37.14231872558594,
      "activations/layer15_attention_weight_min": -32.44219207763672,
      "activations/layer16_attention_weight_max": 37.5543212890625,
      "activations/layer16_attention_weight_min": -28.46360969543457,
      "activations/layer17_attention_weight_max": 36.13637161254883,
      "activations/layer17_attention_weight_min": -24.748876571655273,
      "activations/layer18_attention_weight_max": 34.51857376098633,
      "activations/layer18_attention_weight_min": -20.925647735595703,
      "activations/layer19_attention_weight_max": 39.600006103515625,
      "activations/layer19_attention_weight_min": -28.458372116088867,
      "activations/layer1_attention_weight_max": 16.71876335144043,
      "activations/layer1_attention_weight_min": -15.316166877746582,
      "activations/layer20_attention_weight_max": 35.36984634399414,
      "activations/layer20_attention_weight_min": -24.07632827758789,
      "activations/layer21_attention_weight_max": 32.5970573425293,
      "activations/layer21_attention_weight_min": -22.52994728088379,
      "activations/layer22_attention_weight_max": 52.21311950683594,
      "activations/layer22_attention_weight_min": -28.15662956237793,
      "activations/layer23_attention_weight_max": 39.98355484008789,
      "activations/layer23_attention_weight_min": -23.004669189453125,
      "activations/layer2_attention_weight_max": 31.52914810180664,
      "activations/layer2_attention_weight_min": -30.64680290222168,
      "activations/layer3_attention_weight_max": 89.83081817626953,
      "activations/layer3_attention_weight_min": -88.49665832519531,
      "activations/layer4_attention_weight_max": 125.69146728515625,
      "activations/layer4_attention_weight_min": -122.69978332519531,
      "activations/layer5_attention_weight_max": 65.5157699584961,
      "activations/layer5_attention_weight_min": -63.02680969238281,
      "activations/layer6_attention_weight_max": 51.5040397644043,
      "activations/layer6_attention_weight_min": -53.0584831237793,
      "activations/layer7_attention_weight_max": 94.15614318847656,
      "activations/layer7_attention_weight_min": -88.47280883789062,
      "activations/layer8_attention_weight_max": 41.76909637451172,
      "activations/layer8_attention_weight_min": -42.08623504638672,
      "activations/layer9_attention_weight_max": 36.791011810302734,
      "activations/layer9_attention_weight_min": -35.408809661865234,
      "epoch": 7.08,
      "learning_rate": 0.00010538257575757576,
      "loss": 2.8186,
      "step": 121900
    },
    {
      "activations/layer0_attention_weight_max": 15.869497299194336,
      "activations/layer0_attention_weight_min": -11.216785430908203,
      "activations/layer10_attention_weight_max": 32.6431770324707,
      "activations/layer10_attention_weight_min": -29.840805053710938,
      "activations/layer11_attention_weight_max": 30.841262817382812,
      "activations/layer11_attention_weight_min": -29.22779655456543,
      "activations/layer12_attention_weight_max": 23.9849853515625,
      "activations/layer12_attention_weight_min": -28.0418643951416,
      "activations/layer13_attention_weight_max": 37.983333587646484,
      "activations/layer13_attention_weight_min": -32.7061767578125,
      "activations/layer14_attention_weight_max": 50.12240982055664,
      "activations/layer14_attention_weight_min": -43.63025665283203,
      "activations/layer15_attention_weight_max": 33.51887893676758,
      "activations/layer15_attention_weight_min": -27.2347354888916,
      "activations/layer16_attention_weight_max": 30.809200286865234,
      "activations/layer16_attention_weight_min": -28.074918746948242,
      "activations/layer17_attention_weight_max": 33.9013786315918,
      "activations/layer17_attention_weight_min": -24.32200050354004,
      "activations/layer18_attention_weight_max": 33.37455368041992,
      "activations/layer18_attention_weight_min": -22.160112380981445,
      "activations/layer19_attention_weight_max": 35.824398040771484,
      "activations/layer19_attention_weight_min": -30.137845993041992,
      "activations/layer1_attention_weight_max": 17.40435028076172,
      "activations/layer1_attention_weight_min": -14.383697509765625,
      "activations/layer20_attention_weight_max": 28.081907272338867,
      "activations/layer20_attention_weight_min": -23.654966354370117,
      "activations/layer21_attention_weight_max": 31.973636627197266,
      "activations/layer21_attention_weight_min": -23.221033096313477,
      "activations/layer22_attention_weight_max": 44.784732818603516,
      "activations/layer22_attention_weight_min": -28.482894897460938,
      "activations/layer23_attention_weight_max": 36.60783767700195,
      "activations/layer23_attention_weight_min": -22.01772117614746,
      "activations/layer2_attention_weight_max": 31.767911911010742,
      "activations/layer2_attention_weight_min": -31.394432067871094,
      "activations/layer3_attention_weight_max": 87.81443786621094,
      "activations/layer3_attention_weight_min": -92.74224853515625,
      "activations/layer4_attention_weight_max": 121.72342681884766,
      "activations/layer4_attention_weight_min": -128.29293823242188,
      "activations/layer5_attention_weight_max": 64.32283020019531,
      "activations/layer5_attention_weight_min": -68.96913146972656,
      "activations/layer6_attention_weight_max": 53.24272918701172,
      "activations/layer6_attention_weight_min": -53.95131301879883,
      "activations/layer7_attention_weight_max": 87.40792083740234,
      "activations/layer7_attention_weight_min": -90.3952865600586,
      "activations/layer8_attention_weight_max": 39.34050369262695,
      "activations/layer8_attention_weight_min": -40.42877197265625,
      "activations/layer9_attention_weight_max": 33.765342712402344,
      "activations/layer9_attention_weight_min": -34.146244049072266,
      "epoch": 7.09,
      "learning_rate": 0.00010536363636363635,
      "loss": 2.8112,
      "step": 121950
    },
    {
      "activations/layer0_attention_weight_max": 15.887840270996094,
      "activations/layer0_attention_weight_min": -11.333723068237305,
      "activations/layer10_attention_weight_max": 34.62599563598633,
      "activations/layer10_attention_weight_min": -32.006072998046875,
      "activations/layer11_attention_weight_max": 32.04512023925781,
      "activations/layer11_attention_weight_min": -29.551528930664062,
      "activations/layer12_attention_weight_max": 26.279253005981445,
      "activations/layer12_attention_weight_min": -25.097681045532227,
      "activations/layer13_attention_weight_max": 42.972190856933594,
      "activations/layer13_attention_weight_min": -33.51592254638672,
      "activations/layer14_attention_weight_max": 51.506591796875,
      "activations/layer14_attention_weight_min": -39.06664276123047,
      "activations/layer15_attention_weight_max": 35.557003021240234,
      "activations/layer15_attention_weight_min": -29.093103408813477,
      "activations/layer16_attention_weight_max": 35.70314407348633,
      "activations/layer16_attention_weight_min": -28.659584045410156,
      "activations/layer17_attention_weight_max": 34.73906326293945,
      "activations/layer17_attention_weight_min": -24.39979362487793,
      "activations/layer18_attention_weight_max": 32.72074890136719,
      "activations/layer18_attention_weight_min": -22.840673446655273,
      "activations/layer19_attention_weight_max": 39.94913101196289,
      "activations/layer19_attention_weight_min": -31.46699333190918,
      "activations/layer1_attention_weight_max": 17.722854614257812,
      "activations/layer1_attention_weight_min": -13.512988090515137,
      "activations/layer20_attention_weight_max": 31.22245979309082,
      "activations/layer20_attention_weight_min": -23.5444393157959,
      "activations/layer21_attention_weight_max": 31.900859832763672,
      "activations/layer21_attention_weight_min": -22.035280227661133,
      "activations/layer22_attention_weight_max": 50.92134094238281,
      "activations/layer22_attention_weight_min": -29.561683654785156,
      "activations/layer23_attention_weight_max": 37.132118225097656,
      "activations/layer23_attention_weight_min": -24.173795700073242,
      "activations/layer2_attention_weight_max": 31.953100204467773,
      "activations/layer2_attention_weight_min": -29.97681427001953,
      "activations/layer3_attention_weight_max": 88.50040435791016,
      "activations/layer3_attention_weight_min": -89.20416259765625,
      "activations/layer4_attention_weight_max": 117.530029296875,
      "activations/layer4_attention_weight_min": -115.05963134765625,
      "activations/layer5_attention_weight_max": 61.916587829589844,
      "activations/layer5_attention_weight_min": -60.973426818847656,
      "activations/layer6_attention_weight_max": 48.73380661010742,
      "activations/layer6_attention_weight_min": -51.61738967895508,
      "activations/layer7_attention_weight_max": 85.36678314208984,
      "activations/layer7_attention_weight_min": -89.46382904052734,
      "activations/layer8_attention_weight_max": 40.81227493286133,
      "activations/layer8_attention_weight_min": -39.57667922973633,
      "activations/layer9_attention_weight_max": 35.728519439697266,
      "activations/layer9_attention_weight_min": -32.78111267089844,
      "epoch": 7.09,
      "learning_rate": 0.00010534469696969696,
      "loss": 2.8067,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5072,
      "eval_samples_per_second": 504.75,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5072,
      "eval_openwebtext_samples_per_second": 504.75,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_wikitext_loss": 2.99609375,
      "eval_wikitext_ppl": 20.007230835729846,
      "eval_wikitext_runtime": 1.9837,
      "eval_wikitext_samples_per_second": 229.876,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.5845,
      "eval_lambada_samples_per_second": 508.007,
      "step": 122000
    },
    {
      "activations/layer0_attention_weight_max": 14.959535598754883,
      "activations/layer0_attention_weight_min": -11.747703552246094,
      "activations/layer10_attention_weight_max": 30.58670997619629,
      "activations/layer10_attention_weight_min": -30.937870025634766,
      "activations/layer11_attention_weight_max": 30.621551513671875,
      "activations/layer11_attention_weight_min": -29.660175323486328,
      "activations/layer12_attention_weight_max": 22.975725173950195,
      "activations/layer12_attention_weight_min": -25.825315475463867,
      "activations/layer13_attention_weight_max": 42.728187561035156,
      "activations/layer13_attention_weight_min": -32.50048828125,
      "activations/layer14_attention_weight_max": 57.104286193847656,
      "activations/layer14_attention_weight_min": -44.35341262817383,
      "activations/layer15_attention_weight_max": 42.50000762939453,
      "activations/layer15_attention_weight_min": -29.189611434936523,
      "activations/layer16_attention_weight_max": 35.733673095703125,
      "activations/layer16_attention_weight_min": -28.694067001342773,
      "activations/layer17_attention_weight_max": 42.14063262939453,
      "activations/layer17_attention_weight_min": -25.800378799438477,
      "activations/layer18_attention_weight_max": 35.19557189941406,
      "activations/layer18_attention_weight_min": -22.00188636779785,
      "activations/layer19_attention_weight_max": 39.143314361572266,
      "activations/layer19_attention_weight_min": -30.65960693359375,
      "activations/layer1_attention_weight_max": 16.60095977783203,
      "activations/layer1_attention_weight_min": -15.45893383026123,
      "activations/layer20_attention_weight_max": 36.22749328613281,
      "activations/layer20_attention_weight_min": -23.090288162231445,
      "activations/layer21_attention_weight_max": 33.10557556152344,
      "activations/layer21_attention_weight_min": -24.561460494995117,
      "activations/layer22_attention_weight_max": 55.66313934326172,
      "activations/layer22_attention_weight_min": -27.74125099182129,
      "activations/layer23_attention_weight_max": 38.37974166870117,
      "activations/layer23_attention_weight_min": -24.285446166992188,
      "activations/layer2_attention_weight_max": 31.60628318786621,
      "activations/layer2_attention_weight_min": -31.281476974487305,
      "activations/layer3_attention_weight_max": 92.25212097167969,
      "activations/layer3_attention_weight_min": -90.213134765625,
      "activations/layer4_attention_weight_max": 125.24297332763672,
      "activations/layer4_attention_weight_min": -118.37686920166016,
      "activations/layer5_attention_weight_max": 62.21019744873047,
      "activations/layer5_attention_weight_min": -61.11140060424805,
      "activations/layer6_attention_weight_max": 49.499961853027344,
      "activations/layer6_attention_weight_min": -53.416419982910156,
      "activations/layer7_attention_weight_max": 83.22513580322266,
      "activations/layer7_attention_weight_min": -88.39152526855469,
      "activations/layer8_attention_weight_max": 37.727073669433594,
      "activations/layer8_attention_weight_min": -39.174617767333984,
      "activations/layer9_attention_weight_max": 31.35256004333496,
      "activations/layer9_attention_weight_min": -32.97239685058594,
      "epoch": 7.09,
      "learning_rate": 0.00010532575757575758,
      "loss": 2.8027,
      "step": 122050
    },
    {
      "activations/layer0_attention_weight_max": 15.250687599182129,
      "activations/layer0_attention_weight_min": -11.886640548706055,
      "activations/layer10_attention_weight_max": 31.396160125732422,
      "activations/layer10_attention_weight_min": -30.76228904724121,
      "activations/layer11_attention_weight_max": 34.68597412109375,
      "activations/layer11_attention_weight_min": -28.076948165893555,
      "activations/layer12_attention_weight_max": 29.350234985351562,
      "activations/layer12_attention_weight_min": -24.16000747680664,
      "activations/layer13_attention_weight_max": 42.30137252807617,
      "activations/layer13_attention_weight_min": -31.172395706176758,
      "activations/layer14_attention_weight_max": 71.38883972167969,
      "activations/layer14_attention_weight_min": -45.107154846191406,
      "activations/layer15_attention_weight_max": 47.03884506225586,
      "activations/layer15_attention_weight_min": -29.148452758789062,
      "activations/layer16_attention_weight_max": 43.42335891723633,
      "activations/layer16_attention_weight_min": -28.698307037353516,
      "activations/layer17_attention_weight_max": 40.379600524902344,
      "activations/layer17_attention_weight_min": -24.044721603393555,
      "activations/layer18_attention_weight_max": 41.12686538696289,
      "activations/layer18_attention_weight_min": -22.58570671081543,
      "activations/layer19_attention_weight_max": 45.84450149536133,
      "activations/layer19_attention_weight_min": -29.478681564331055,
      "activations/layer1_attention_weight_max": 18.27292823791504,
      "activations/layer1_attention_weight_min": -14.014922142028809,
      "activations/layer20_attention_weight_max": 38.991695404052734,
      "activations/layer20_attention_weight_min": -24.229032516479492,
      "activations/layer21_attention_weight_max": 41.22507095336914,
      "activations/layer21_attention_weight_min": -22.069169998168945,
      "activations/layer22_attention_weight_max": 62.0068359375,
      "activations/layer22_attention_weight_min": -27.29121208190918,
      "activations/layer23_attention_weight_max": 43.02128219604492,
      "activations/layer23_attention_weight_min": -22.296405792236328,
      "activations/layer2_attention_weight_max": 31.272483825683594,
      "activations/layer2_attention_weight_min": -30.617656707763672,
      "activations/layer3_attention_weight_max": 91.32858276367188,
      "activations/layer3_attention_weight_min": -92.52458190917969,
      "activations/layer4_attention_weight_max": 123.28983306884766,
      "activations/layer4_attention_weight_min": -118.49623107910156,
      "activations/layer5_attention_weight_max": 64.4649429321289,
      "activations/layer5_attention_weight_min": -62.54032897949219,
      "activations/layer6_attention_weight_max": 48.184043884277344,
      "activations/layer6_attention_weight_min": -52.72663116455078,
      "activations/layer7_attention_weight_max": 84.83634948730469,
      "activations/layer7_attention_weight_min": -83.48748779296875,
      "activations/layer8_attention_weight_max": 39.68776321411133,
      "activations/layer8_attention_weight_min": -37.80929183959961,
      "activations/layer9_attention_weight_max": 36.37761688232422,
      "activations/layer9_attention_weight_min": -32.256771087646484,
      "epoch": 7.09,
      "learning_rate": 0.00010530681818181817,
      "loss": 2.8175,
      "step": 122100
    },
    {
      "activations/layer0_attention_weight_max": 18.219844818115234,
      "activations/layer0_attention_weight_min": -11.308492660522461,
      "activations/layer10_attention_weight_max": 36.67134475708008,
      "activations/layer10_attention_weight_min": -30.67876434326172,
      "activations/layer11_attention_weight_max": 38.34941101074219,
      "activations/layer11_attention_weight_min": -31.92444610595703,
      "activations/layer12_attention_weight_max": 55.33658218383789,
      "activations/layer12_attention_weight_min": -29.613374710083008,
      "activations/layer13_attention_weight_max": 56.79663848876953,
      "activations/layer13_attention_weight_min": -34.73978805541992,
      "activations/layer14_attention_weight_max": 57.437686920166016,
      "activations/layer14_attention_weight_min": -45.07825469970703,
      "activations/layer15_attention_weight_max": 53.38002014160156,
      "activations/layer15_attention_weight_min": -30.954919815063477,
      "activations/layer16_attention_weight_max": 35.35682678222656,
      "activations/layer16_attention_weight_min": -26.434093475341797,
      "activations/layer17_attention_weight_max": 33.7425651550293,
      "activations/layer17_attention_weight_min": -24.377552032470703,
      "activations/layer18_attention_weight_max": 30.535776138305664,
      "activations/layer18_attention_weight_min": -20.27007293701172,
      "activations/layer19_attention_weight_max": 34.632266998291016,
      "activations/layer19_attention_weight_min": -32.72294998168945,
      "activations/layer1_attention_weight_max": 16.825149536132812,
      "activations/layer1_attention_weight_min": -17.230005264282227,
      "activations/layer20_attention_weight_max": 30.50038719177246,
      "activations/layer20_attention_weight_min": -23.543973922729492,
      "activations/layer21_attention_weight_max": 30.759855270385742,
      "activations/layer21_attention_weight_min": -21.12793731689453,
      "activations/layer22_attention_weight_max": 48.3633918762207,
      "activations/layer22_attention_weight_min": -28.39537811279297,
      "activations/layer23_attention_weight_max": 34.375282287597656,
      "activations/layer23_attention_weight_min": -22.392316818237305,
      "activations/layer2_attention_weight_max": 31.49439811706543,
      "activations/layer2_attention_weight_min": -32.334999084472656,
      "activations/layer3_attention_weight_max": 91.61170196533203,
      "activations/layer3_attention_weight_min": -98.56693267822266,
      "activations/layer4_attention_weight_max": 124.76476287841797,
      "activations/layer4_attention_weight_min": -121.28656005859375,
      "activations/layer5_attention_weight_max": 64.8740234375,
      "activations/layer5_attention_weight_min": -61.83518981933594,
      "activations/layer6_attention_weight_max": 53.16999053955078,
      "activations/layer6_attention_weight_min": -51.69728088378906,
      "activations/layer7_attention_weight_max": 99.35588836669922,
      "activations/layer7_attention_weight_min": -87.59703826904297,
      "activations/layer8_attention_weight_max": 42.98705291748047,
      "activations/layer8_attention_weight_min": -39.81525802612305,
      "activations/layer9_attention_weight_max": 36.663692474365234,
      "activations/layer9_attention_weight_min": -37.186363220214844,
      "epoch": 7.1,
      "learning_rate": 0.00010528787878787878,
      "loss": 2.8161,
      "step": 122150
    },
    {
      "activations/layer0_attention_weight_max": 14.796119689941406,
      "activations/layer0_attention_weight_min": -11.59355640411377,
      "activations/layer10_attention_weight_max": 36.7878303527832,
      "activations/layer10_attention_weight_min": -32.71141052246094,
      "activations/layer11_attention_weight_max": 35.26642990112305,
      "activations/layer11_attention_weight_min": -30.197721481323242,
      "activations/layer12_attention_weight_max": 25.1949462890625,
      "activations/layer12_attention_weight_min": -28.264713287353516,
      "activations/layer13_attention_weight_max": 43.862483978271484,
      "activations/layer13_attention_weight_min": -33.12950134277344,
      "activations/layer14_attention_weight_max": 52.34640884399414,
      "activations/layer14_attention_weight_min": -37.56929397583008,
      "activations/layer15_attention_weight_max": 40.98088455200195,
      "activations/layer15_attention_weight_min": -28.10245704650879,
      "activations/layer16_attention_weight_max": 37.98072052001953,
      "activations/layer16_attention_weight_min": -30.084829330444336,
      "activations/layer17_attention_weight_max": 41.8597297668457,
      "activations/layer17_attention_weight_min": -24.474895477294922,
      "activations/layer18_attention_weight_max": 39.90911102294922,
      "activations/layer18_attention_weight_min": -21.844594955444336,
      "activations/layer19_attention_weight_max": 43.04159164428711,
      "activations/layer19_attention_weight_min": -31.496957778930664,
      "activations/layer1_attention_weight_max": 17.186193466186523,
      "activations/layer1_attention_weight_min": -15.215588569641113,
      "activations/layer20_attention_weight_max": 33.33598327636719,
      "activations/layer20_attention_weight_min": -24.161266326904297,
      "activations/layer21_attention_weight_max": 33.00503921508789,
      "activations/layer21_attention_weight_min": -23.55643653869629,
      "activations/layer22_attention_weight_max": 50.20928955078125,
      "activations/layer22_attention_weight_min": -29.007078170776367,
      "activations/layer23_attention_weight_max": 40.804664611816406,
      "activations/layer23_attention_weight_min": -23.17416763305664,
      "activations/layer2_attention_weight_max": 32.03940963745117,
      "activations/layer2_attention_weight_min": -31.091957092285156,
      "activations/layer3_attention_weight_max": 93.88946533203125,
      "activations/layer3_attention_weight_min": -92.53105926513672,
      "activations/layer4_attention_weight_max": 133.61546325683594,
      "activations/layer4_attention_weight_min": -125.79338836669922,
      "activations/layer5_attention_weight_max": 67.8092269897461,
      "activations/layer5_attention_weight_min": -65.88228607177734,
      "activations/layer6_attention_weight_max": 51.77695846557617,
      "activations/layer6_attention_weight_min": -53.382205963134766,
      "activations/layer7_attention_weight_max": 89.73763275146484,
      "activations/layer7_attention_weight_min": -92.44941711425781,
      "activations/layer8_attention_weight_max": 44.16019058227539,
      "activations/layer8_attention_weight_min": -40.681819915771484,
      "activations/layer9_attention_weight_max": 38.36067581176758,
      "activations/layer9_attention_weight_min": -34.705867767333984,
      "epoch": 7.1,
      "learning_rate": 0.00010526893939393938,
      "loss": 2.8331,
      "step": 122200
    },
    {
      "activations/layer0_attention_weight_max": 15.665788650512695,
      "activations/layer0_attention_weight_min": -11.795042037963867,
      "activations/layer10_attention_weight_max": 31.773496627807617,
      "activations/layer10_attention_weight_min": -30.08612632751465,
      "activations/layer11_attention_weight_max": 32.45417785644531,
      "activations/layer11_attention_weight_min": -29.777286529541016,
      "activations/layer12_attention_weight_max": 25.957773208618164,
      "activations/layer12_attention_weight_min": -27.80405616760254,
      "activations/layer13_attention_weight_max": 39.111083984375,
      "activations/layer13_attention_weight_min": -30.777681350708008,
      "activations/layer14_attention_weight_max": 51.445865631103516,
      "activations/layer14_attention_weight_min": -36.72483444213867,
      "activations/layer15_attention_weight_max": 33.738059997558594,
      "activations/layer15_attention_weight_min": -27.344446182250977,
      "activations/layer16_attention_weight_max": 31.25229835510254,
      "activations/layer16_attention_weight_min": -24.933837890625,
      "activations/layer17_attention_weight_max": 34.34038543701172,
      "activations/layer17_attention_weight_min": -23.845735549926758,
      "activations/layer18_attention_weight_max": 31.966049194335938,
      "activations/layer18_attention_weight_min": -21.750938415527344,
      "activations/layer19_attention_weight_max": 36.56438446044922,
      "activations/layer19_attention_weight_min": -27.152692794799805,
      "activations/layer1_attention_weight_max": 16.133577346801758,
      "activations/layer1_attention_weight_min": -14.233576774597168,
      "activations/layer20_attention_weight_max": 29.025388717651367,
      "activations/layer20_attention_weight_min": -23.010515213012695,
      "activations/layer21_attention_weight_max": 30.84294319152832,
      "activations/layer21_attention_weight_min": -21.308788299560547,
      "activations/layer22_attention_weight_max": 44.02580642700195,
      "activations/layer22_attention_weight_min": -27.74222183227539,
      "activations/layer23_attention_weight_max": 36.5379524230957,
      "activations/layer23_attention_weight_min": -20.686847686767578,
      "activations/layer2_attention_weight_max": 30.73756980895996,
      "activations/layer2_attention_weight_min": -30.51921844482422,
      "activations/layer3_attention_weight_max": 89.5820541381836,
      "activations/layer3_attention_weight_min": -92.17273712158203,
      "activations/layer4_attention_weight_max": 128.03233337402344,
      "activations/layer4_attention_weight_min": -120.46562957763672,
      "activations/layer5_attention_weight_max": 68.3568115234375,
      "activations/layer5_attention_weight_min": -63.99909591674805,
      "activations/layer6_attention_weight_max": 49.676979064941406,
      "activations/layer6_attention_weight_min": -51.3318977355957,
      "activations/layer7_attention_weight_max": 83.66034698486328,
      "activations/layer7_attention_weight_min": -85.5533447265625,
      "activations/layer8_attention_weight_max": 39.967594146728516,
      "activations/layer8_attention_weight_min": -42.301055908203125,
      "activations/layer9_attention_weight_max": 35.19948959350586,
      "activations/layer9_attention_weight_min": -33.622310638427734,
      "epoch": 7.1,
      "learning_rate": 0.00010524999999999998,
      "loss": 2.8017,
      "step": 122250
    },
    {
      "activations/layer0_attention_weight_max": 14.702703475952148,
      "activations/layer0_attention_weight_min": -11.095268249511719,
      "activations/layer10_attention_weight_max": 32.52809143066406,
      "activations/layer10_attention_weight_min": -33.7335205078125,
      "activations/layer11_attention_weight_max": 32.356605529785156,
      "activations/layer11_attention_weight_min": -29.685352325439453,
      "activations/layer12_attention_weight_max": 27.51780891418457,
      "activations/layer12_attention_weight_min": -26.51205825805664,
      "activations/layer13_attention_weight_max": 37.53045654296875,
      "activations/layer13_attention_weight_min": -32.174278259277344,
      "activations/layer14_attention_weight_max": 42.862491607666016,
      "activations/layer14_attention_weight_min": -37.298458099365234,
      "activations/layer15_attention_weight_max": 32.13093948364258,
      "activations/layer15_attention_weight_min": -29.969266891479492,
      "activations/layer16_attention_weight_max": 30.622264862060547,
      "activations/layer16_attention_weight_min": -27.50697135925293,
      "activations/layer17_attention_weight_max": 33.653072357177734,
      "activations/layer17_attention_weight_min": -23.465770721435547,
      "activations/layer18_attention_weight_max": 33.30412292480469,
      "activations/layer18_attention_weight_min": -20.57270622253418,
      "activations/layer19_attention_weight_max": 32.95192337036133,
      "activations/layer19_attention_weight_min": -29.791797637939453,
      "activations/layer1_attention_weight_max": 18.347856521606445,
      "activations/layer1_attention_weight_min": -14.30991268157959,
      "activations/layer20_attention_weight_max": 29.79738426208496,
      "activations/layer20_attention_weight_min": -23.515871047973633,
      "activations/layer21_attention_weight_max": 30.579076766967773,
      "activations/layer21_attention_weight_min": -21.798349380493164,
      "activations/layer22_attention_weight_max": 46.74728775024414,
      "activations/layer22_attention_weight_min": -27.864471435546875,
      "activations/layer23_attention_weight_max": 33.515708923339844,
      "activations/layer23_attention_weight_min": -26.08209800720215,
      "activations/layer2_attention_weight_max": 31.168949127197266,
      "activations/layer2_attention_weight_min": -32.838134765625,
      "activations/layer3_attention_weight_max": 89.65593719482422,
      "activations/layer3_attention_weight_min": -93.33235168457031,
      "activations/layer4_attention_weight_max": 123.0406723022461,
      "activations/layer4_attention_weight_min": -128.26329040527344,
      "activations/layer5_attention_weight_max": 63.65581512451172,
      "activations/layer5_attention_weight_min": -65.86534881591797,
      "activations/layer6_attention_weight_max": 49.93805694580078,
      "activations/layer6_attention_weight_min": -52.749759674072266,
      "activations/layer7_attention_weight_max": 94.033935546875,
      "activations/layer7_attention_weight_min": -91.76941680908203,
      "activations/layer8_attention_weight_max": 40.65327835083008,
      "activations/layer8_attention_weight_min": -43.96779251098633,
      "activations/layer9_attention_weight_max": 35.783267974853516,
      "activations/layer9_attention_weight_min": -34.95109176635742,
      "epoch": 7.11,
      "learning_rate": 0.0001052310606060606,
      "loss": 2.8006,
      "step": 122300
    },
    {
      "activations/layer0_attention_weight_max": 15.212142944335938,
      "activations/layer0_attention_weight_min": -10.779064178466797,
      "activations/layer10_attention_weight_max": 37.19847869873047,
      "activations/layer10_attention_weight_min": -36.502174377441406,
      "activations/layer11_attention_weight_max": 37.941993713378906,
      "activations/layer11_attention_weight_min": -32.83523941040039,
      "activations/layer12_attention_weight_max": 29.657100677490234,
      "activations/layer12_attention_weight_min": -28.369503021240234,
      "activations/layer13_attention_weight_max": 44.2054328918457,
      "activations/layer13_attention_weight_min": -36.280540466308594,
      "activations/layer14_attention_weight_max": 73.37495422363281,
      "activations/layer14_attention_weight_min": -50.50300216674805,
      "activations/layer15_attention_weight_max": 38.310062408447266,
      "activations/layer15_attention_weight_min": -30.091331481933594,
      "activations/layer16_attention_weight_max": 35.13642120361328,
      "activations/layer16_attention_weight_min": -28.107501983642578,
      "activations/layer17_attention_weight_max": 39.86553192138672,
      "activations/layer17_attention_weight_min": -26.07706642150879,
      "activations/layer18_attention_weight_max": 34.910308837890625,
      "activations/layer18_attention_weight_min": -21.78580665588379,
      "activations/layer19_attention_weight_max": 39.4614372253418,
      "activations/layer19_attention_weight_min": -29.100452423095703,
      "activations/layer1_attention_weight_max": 17.513355255126953,
      "activations/layer1_attention_weight_min": -14.906841278076172,
      "activations/layer20_attention_weight_max": 32.81013107299805,
      "activations/layer20_attention_weight_min": -22.976335525512695,
      "activations/layer21_attention_weight_max": 35.436580657958984,
      "activations/layer21_attention_weight_min": -22.569820404052734,
      "activations/layer22_attention_weight_max": 53.1282958984375,
      "activations/layer22_attention_weight_min": -31.492998123168945,
      "activations/layer23_attention_weight_max": 37.84019470214844,
      "activations/layer23_attention_weight_min": -23.107803344726562,
      "activations/layer2_attention_weight_max": 29.58945083618164,
      "activations/layer2_attention_weight_min": -31.018192291259766,
      "activations/layer3_attention_weight_max": 91.21311950683594,
      "activations/layer3_attention_weight_min": -91.88809204101562,
      "activations/layer4_attention_weight_max": 127.85616302490234,
      "activations/layer4_attention_weight_min": -127.05738830566406,
      "activations/layer5_attention_weight_max": 69.53300476074219,
      "activations/layer5_attention_weight_min": -70.41287994384766,
      "activations/layer6_attention_weight_max": 53.336219787597656,
      "activations/layer6_attention_weight_min": -59.672401428222656,
      "activations/layer7_attention_weight_max": 99.29859924316406,
      "activations/layer7_attention_weight_min": -97.51544952392578,
      "activations/layer8_attention_weight_max": 46.32582473754883,
      "activations/layer8_attention_weight_min": -46.6019287109375,
      "activations/layer9_attention_weight_max": 39.62361526489258,
      "activations/layer9_attention_weight_min": -39.005210876464844,
      "epoch": 7.11,
      "learning_rate": 0.0001052121212121212,
      "loss": 2.8139,
      "step": 122350
    },
    {
      "activations/layer0_attention_weight_max": 14.94837760925293,
      "activations/layer0_attention_weight_min": -11.20257568359375,
      "activations/layer10_attention_weight_max": 31.200578689575195,
      "activations/layer10_attention_weight_min": -33.21062088012695,
      "activations/layer11_attention_weight_max": 33.0811882019043,
      "activations/layer11_attention_weight_min": -32.01803207397461,
      "activations/layer12_attention_weight_max": 23.152095794677734,
      "activations/layer12_attention_weight_min": -26.029760360717773,
      "activations/layer13_attention_weight_max": 38.54932403564453,
      "activations/layer13_attention_weight_min": -35.42866134643555,
      "activations/layer14_attention_weight_max": 75.3450698852539,
      "activations/layer14_attention_weight_min": -44.461647033691406,
      "activations/layer15_attention_weight_max": 39.71158981323242,
      "activations/layer15_attention_weight_min": -29.404848098754883,
      "activations/layer16_attention_weight_max": 44.296512603759766,
      "activations/layer16_attention_weight_min": -29.301591873168945,
      "activations/layer17_attention_weight_max": 35.8648681640625,
      "activations/layer17_attention_weight_min": -24.757638931274414,
      "activations/layer18_attention_weight_max": 33.12443542480469,
      "activations/layer18_attention_weight_min": -20.870359420776367,
      "activations/layer19_attention_weight_max": 44.73745346069336,
      "activations/layer19_attention_weight_min": -29.614545822143555,
      "activations/layer1_attention_weight_max": 17.305957794189453,
      "activations/layer1_attention_weight_min": -17.99936866760254,
      "activations/layer20_attention_weight_max": 33.29804992675781,
      "activations/layer20_attention_weight_min": -23.665409088134766,
      "activations/layer21_attention_weight_max": 32.851409912109375,
      "activations/layer21_attention_weight_min": -22.22686767578125,
      "activations/layer22_attention_weight_max": 52.88066482543945,
      "activations/layer22_attention_weight_min": -31.90259552001953,
      "activations/layer23_attention_weight_max": 38.1910285949707,
      "activations/layer23_attention_weight_min": -23.715055465698242,
      "activations/layer2_attention_weight_max": 29.649044036865234,
      "activations/layer2_attention_weight_min": -29.677574157714844,
      "activations/layer3_attention_weight_max": 86.69366455078125,
      "activations/layer3_attention_weight_min": -91.4858169555664,
      "activations/layer4_attention_weight_max": 119.96666717529297,
      "activations/layer4_attention_weight_min": -118.20738983154297,
      "activations/layer5_attention_weight_max": 62.31927490234375,
      "activations/layer5_attention_weight_min": -65.19242858886719,
      "activations/layer6_attention_weight_max": 49.78342819213867,
      "activations/layer6_attention_weight_min": -53.23972702026367,
      "activations/layer7_attention_weight_max": 85.15628814697266,
      "activations/layer7_attention_weight_min": -90.95429229736328,
      "activations/layer8_attention_weight_max": 40.98497772216797,
      "activations/layer8_attention_weight_min": -45.968814849853516,
      "activations/layer9_attention_weight_max": 35.1526985168457,
      "activations/layer9_attention_weight_min": -36.449832916259766,
      "epoch": 7.11,
      "learning_rate": 0.00010519318181818181,
      "loss": 2.8062,
      "step": 122400
    },
    {
      "activations/layer0_attention_weight_max": 14.934263229370117,
      "activations/layer0_attention_weight_min": -10.909492492675781,
      "activations/layer10_attention_weight_max": 38.192264556884766,
      "activations/layer10_attention_weight_min": -32.919700622558594,
      "activations/layer11_attention_weight_max": 38.472129821777344,
      "activations/layer11_attention_weight_min": -31.472335815429688,
      "activations/layer12_attention_weight_max": 31.48086166381836,
      "activations/layer12_attention_weight_min": -29.379606246948242,
      "activations/layer13_attention_weight_max": 49.52925109863281,
      "activations/layer13_attention_weight_min": -34.296905517578125,
      "activations/layer14_attention_weight_max": 65.61605834960938,
      "activations/layer14_attention_weight_min": -47.37571334838867,
      "activations/layer15_attention_weight_max": 42.674659729003906,
      "activations/layer15_attention_weight_min": -31.001924514770508,
      "activations/layer16_attention_weight_max": 34.858455657958984,
      "activations/layer16_attention_weight_min": -27.178117752075195,
      "activations/layer17_attention_weight_max": 33.9005126953125,
      "activations/layer17_attention_weight_min": -24.94240379333496,
      "activations/layer18_attention_weight_max": 29.88949203491211,
      "activations/layer18_attention_weight_min": -21.262758255004883,
      "activations/layer19_attention_weight_max": 40.45469284057617,
      "activations/layer19_attention_weight_min": -28.8504695892334,
      "activations/layer1_attention_weight_max": 17.337509155273438,
      "activations/layer1_attention_weight_min": -14.03172492980957,
      "activations/layer20_attention_weight_max": 33.7641487121582,
      "activations/layer20_attention_weight_min": -23.438926696777344,
      "activations/layer21_attention_weight_max": 30.98765754699707,
      "activations/layer21_attention_weight_min": -22.293169021606445,
      "activations/layer22_attention_weight_max": 48.08311462402344,
      "activations/layer22_attention_weight_min": -31.536088943481445,
      "activations/layer23_attention_weight_max": 38.422752380371094,
      "activations/layer23_attention_weight_min": -22.190519332885742,
      "activations/layer2_attention_weight_max": 30.848281860351562,
      "activations/layer2_attention_weight_min": -30.85373306274414,
      "activations/layer3_attention_weight_max": 88.77507781982422,
      "activations/layer3_attention_weight_min": -91.81568908691406,
      "activations/layer4_attention_weight_max": 123.3256607055664,
      "activations/layer4_attention_weight_min": -125.21428680419922,
      "activations/layer5_attention_weight_max": 62.61243438720703,
      "activations/layer5_attention_weight_min": -62.85460662841797,
      "activations/layer6_attention_weight_max": 50.52650833129883,
      "activations/layer6_attention_weight_min": -54.10301971435547,
      "activations/layer7_attention_weight_max": 94.74398040771484,
      "activations/layer7_attention_weight_min": -95.09756469726562,
      "activations/layer8_attention_weight_max": 42.24045181274414,
      "activations/layer8_attention_weight_min": -44.34355926513672,
      "activations/layer9_attention_weight_max": 39.65140914916992,
      "activations/layer9_attention_weight_min": -35.78314208984375,
      "epoch": 7.12,
      "learning_rate": 0.00010517424242424242,
      "loss": 2.8175,
      "step": 122450
    },
    {
      "activations/layer0_attention_weight_max": 14.269475936889648,
      "activations/layer0_attention_weight_min": -11.043436050415039,
      "activations/layer10_attention_weight_max": 41.91604995727539,
      "activations/layer10_attention_weight_min": -37.006988525390625,
      "activations/layer11_attention_weight_max": 43.18092346191406,
      "activations/layer11_attention_weight_min": -35.69317626953125,
      "activations/layer12_attention_weight_max": 29.121362686157227,
      "activations/layer12_attention_weight_min": -27.722328186035156,
      "activations/layer13_attention_weight_max": 51.098453521728516,
      "activations/layer13_attention_weight_min": -35.949806213378906,
      "activations/layer14_attention_weight_max": 70.09774017333984,
      "activations/layer14_attention_weight_min": -45.133277893066406,
      "activations/layer15_attention_weight_max": 41.496368408203125,
      "activations/layer15_attention_weight_min": -29.84834098815918,
      "activations/layer16_attention_weight_max": 44.365386962890625,
      "activations/layer16_attention_weight_min": -27.947490692138672,
      "activations/layer17_attention_weight_max": 37.62468338012695,
      "activations/layer17_attention_weight_min": -24.04839515686035,
      "activations/layer18_attention_weight_max": 33.981868743896484,
      "activations/layer18_attention_weight_min": -22.022581100463867,
      "activations/layer19_attention_weight_max": 50.256656646728516,
      "activations/layer19_attention_weight_min": -30.039243698120117,
      "activations/layer1_attention_weight_max": 18.38796615600586,
      "activations/layer1_attention_weight_min": -17.16094970703125,
      "activations/layer20_attention_weight_max": 34.08304977416992,
      "activations/layer20_attention_weight_min": -24.30772590637207,
      "activations/layer21_attention_weight_max": 29.235605239868164,
      "activations/layer21_attention_weight_min": -22.07456398010254,
      "activations/layer22_attention_weight_max": 54.51483154296875,
      "activations/layer22_attention_weight_min": -30.102962493896484,
      "activations/layer23_attention_weight_max": 43.12218475341797,
      "activations/layer23_attention_weight_min": -22.935598373413086,
      "activations/layer2_attention_weight_max": 30.808364868164062,
      "activations/layer2_attention_weight_min": -31.8383846282959,
      "activations/layer3_attention_weight_max": 87.73849487304688,
      "activations/layer3_attention_weight_min": -89.45551300048828,
      "activations/layer4_attention_weight_max": 122.7780532836914,
      "activations/layer4_attention_weight_min": -122.22501373291016,
      "activations/layer5_attention_weight_max": 64.12588500976562,
      "activations/layer5_attention_weight_min": -63.793174743652344,
      "activations/layer6_attention_weight_max": 51.83107376098633,
      "activations/layer6_attention_weight_min": -53.49980926513672,
      "activations/layer7_attention_weight_max": 99.30997467041016,
      "activations/layer7_attention_weight_min": -101.19538116455078,
      "activations/layer8_attention_weight_max": 48.60517883300781,
      "activations/layer8_attention_weight_min": -44.92685317993164,
      "activations/layer9_attention_weight_max": 45.974639892578125,
      "activations/layer9_attention_weight_min": -40.20050811767578,
      "epoch": 7.12,
      "learning_rate": 0.00010515530303030302,
      "loss": 2.8109,
      "step": 122500
    },
    {
      "activations/layer0_attention_weight_max": 15.089362144470215,
      "activations/layer0_attention_weight_min": -12.286944389343262,
      "activations/layer10_attention_weight_max": 34.66448974609375,
      "activations/layer10_attention_weight_min": -32.667396545410156,
      "activations/layer11_attention_weight_max": 35.04054260253906,
      "activations/layer11_attention_weight_min": -31.292781829833984,
      "activations/layer12_attention_weight_max": 23.77230453491211,
      "activations/layer12_attention_weight_min": -22.889244079589844,
      "activations/layer13_attention_weight_max": 44.53314971923828,
      "activations/layer13_attention_weight_min": -34.07102584838867,
      "activations/layer14_attention_weight_max": 60.21947479248047,
      "activations/layer14_attention_weight_min": -42.24192810058594,
      "activations/layer15_attention_weight_max": 37.09729766845703,
      "activations/layer15_attention_weight_min": -30.483692169189453,
      "activations/layer16_attention_weight_max": 33.38819122314453,
      "activations/layer16_attention_weight_min": -28.508588790893555,
      "activations/layer17_attention_weight_max": 36.4832878112793,
      "activations/layer17_attention_weight_min": -25.61443519592285,
      "activations/layer18_attention_weight_max": 32.903106689453125,
      "activations/layer18_attention_weight_min": -20.4675350189209,
      "activations/layer19_attention_weight_max": 36.71218490600586,
      "activations/layer19_attention_weight_min": -30.602985382080078,
      "activations/layer1_attention_weight_max": 17.376060485839844,
      "activations/layer1_attention_weight_min": -15.808857917785645,
      "activations/layer20_attention_weight_max": 30.641923904418945,
      "activations/layer20_attention_weight_min": -23.65850257873535,
      "activations/layer21_attention_weight_max": 32.965736389160156,
      "activations/layer21_attention_weight_min": -22.147693634033203,
      "activations/layer22_attention_weight_max": 51.73521423339844,
      "activations/layer22_attention_weight_min": -29.659809112548828,
      "activations/layer23_attention_weight_max": 37.048316955566406,
      "activations/layer23_attention_weight_min": -21.60017204284668,
      "activations/layer2_attention_weight_max": 31.62203598022461,
      "activations/layer2_attention_weight_min": -31.689476013183594,
      "activations/layer3_attention_weight_max": 91.33467864990234,
      "activations/layer3_attention_weight_min": -93.18377685546875,
      "activations/layer4_attention_weight_max": 121.57568359375,
      "activations/layer4_attention_weight_min": -124.03948211669922,
      "activations/layer5_attention_weight_max": 64.84762573242188,
      "activations/layer5_attention_weight_min": -63.8662109375,
      "activations/layer6_attention_weight_max": 52.74778366088867,
      "activations/layer6_attention_weight_min": -54.01618576049805,
      "activations/layer7_attention_weight_max": 89.09252166748047,
      "activations/layer7_attention_weight_min": -89.76422882080078,
      "activations/layer8_attention_weight_max": 43.06110763549805,
      "activations/layer8_attention_weight_min": -41.85139846801758,
      "activations/layer9_attention_weight_max": 39.02198028564453,
      "activations/layer9_attention_weight_min": -36.34999465942383,
      "epoch": 7.12,
      "learning_rate": 0.00010513636363636363,
      "loss": 2.8331,
      "step": 122550
    },
    {
      "activations/layer0_attention_weight_max": 14.229104995727539,
      "activations/layer0_attention_weight_min": -10.654974937438965,
      "activations/layer10_attention_weight_max": 33.93083572387695,
      "activations/layer10_attention_weight_min": -34.44606399536133,
      "activations/layer11_attention_weight_max": 32.822662353515625,
      "activations/layer11_attention_weight_min": -31.443700790405273,
      "activations/layer12_attention_weight_max": 23.686328887939453,
      "activations/layer12_attention_weight_min": -27.36806869506836,
      "activations/layer13_attention_weight_max": 40.40856170654297,
      "activations/layer13_attention_weight_min": -33.063819885253906,
      "activations/layer14_attention_weight_max": 48.66042709350586,
      "activations/layer14_attention_weight_min": -43.48219680786133,
      "activations/layer15_attention_weight_max": 36.58948516845703,
      "activations/layer15_attention_weight_min": -30.184385299682617,
      "activations/layer16_attention_weight_max": 37.264957427978516,
      "activations/layer16_attention_weight_min": -30.53520393371582,
      "activations/layer17_attention_weight_max": 34.40110778808594,
      "activations/layer17_attention_weight_min": -25.999540328979492,
      "activations/layer18_attention_weight_max": 30.8314266204834,
      "activations/layer18_attention_weight_min": -21.3768367767334,
      "activations/layer19_attention_weight_max": 39.9515495300293,
      "activations/layer19_attention_weight_min": -30.757238388061523,
      "activations/layer1_attention_weight_max": 16.464073181152344,
      "activations/layer1_attention_weight_min": -14.10812759399414,
      "activations/layer20_attention_weight_max": 29.110822677612305,
      "activations/layer20_attention_weight_min": -23.275222778320312,
      "activations/layer21_attention_weight_max": 32.236061096191406,
      "activations/layer21_attention_weight_min": -22.001708984375,
      "activations/layer22_attention_weight_max": 51.69363021850586,
      "activations/layer22_attention_weight_min": -26.98274803161621,
      "activations/layer23_attention_weight_max": 34.906036376953125,
      "activations/layer23_attention_weight_min": -22.329322814941406,
      "activations/layer2_attention_weight_max": 31.530136108398438,
      "activations/layer2_attention_weight_min": -31.41650390625,
      "activations/layer3_attention_weight_max": 91.4292221069336,
      "activations/layer3_attention_weight_min": -95.45172119140625,
      "activations/layer4_attention_weight_max": 122.4448013305664,
      "activations/layer4_attention_weight_min": -123.6322021484375,
      "activations/layer5_attention_weight_max": 64.21134948730469,
      "activations/layer5_attention_weight_min": -65.16204071044922,
      "activations/layer6_attention_weight_max": 54.845890045166016,
      "activations/layer6_attention_weight_min": -55.648197174072266,
      "activations/layer7_attention_weight_max": 90.3214340209961,
      "activations/layer7_attention_weight_min": -89.14270782470703,
      "activations/layer8_attention_weight_max": 42.93227005004883,
      "activations/layer8_attention_weight_min": -43.84685134887695,
      "activations/layer9_attention_weight_max": 36.642086029052734,
      "activations/layer9_attention_weight_min": -35.178951263427734,
      "epoch": 7.12,
      "learning_rate": 0.00010511742424242422,
      "loss": 2.8158,
      "step": 122600
    },
    {
      "activations/layer0_attention_weight_max": 14.998190879821777,
      "activations/layer0_attention_weight_min": -10.817302703857422,
      "activations/layer10_attention_weight_max": 32.971839904785156,
      "activations/layer10_attention_weight_min": -31.912399291992188,
      "activations/layer11_attention_weight_max": 33.74452209472656,
      "activations/layer11_attention_weight_min": -27.656959533691406,
      "activations/layer12_attention_weight_max": 23.32451629638672,
      "activations/layer12_attention_weight_min": -27.010089874267578,
      "activations/layer13_attention_weight_max": 47.78129196166992,
      "activations/layer13_attention_weight_min": -33.59346008300781,
      "activations/layer14_attention_weight_max": 54.033409118652344,
      "activations/layer14_attention_weight_min": -39.94172286987305,
      "activations/layer15_attention_weight_max": 37.0833740234375,
      "activations/layer15_attention_weight_min": -30.92829132080078,
      "activations/layer16_attention_weight_max": 33.00285720825195,
      "activations/layer16_attention_weight_min": -26.67367172241211,
      "activations/layer17_attention_weight_max": 34.632835388183594,
      "activations/layer17_attention_weight_min": -25.96558952331543,
      "activations/layer18_attention_weight_max": 32.87889862060547,
      "activations/layer18_attention_weight_min": -20.589637756347656,
      "activations/layer19_attention_weight_max": 38.754730224609375,
      "activations/layer19_attention_weight_min": -30.17886734008789,
      "activations/layer1_attention_weight_max": 17.169830322265625,
      "activations/layer1_attention_weight_min": -13.660262107849121,
      "activations/layer20_attention_weight_max": 30.03911590576172,
      "activations/layer20_attention_weight_min": -25.531408309936523,
      "activations/layer21_attention_weight_max": 34.00732421875,
      "activations/layer21_attention_weight_min": -22.763235092163086,
      "activations/layer22_attention_weight_max": 46.021183013916016,
      "activations/layer22_attention_weight_min": -29.58714485168457,
      "activations/layer23_attention_weight_max": 36.217010498046875,
      "activations/layer23_attention_weight_min": -26.197179794311523,
      "activations/layer2_attention_weight_max": 30.622447967529297,
      "activations/layer2_attention_weight_min": -31.600561141967773,
      "activations/layer3_attention_weight_max": 96.52324676513672,
      "activations/layer3_attention_weight_min": -98.50635528564453,
      "activations/layer4_attention_weight_max": 126.23307037353516,
      "activations/layer4_attention_weight_min": -127.34539794921875,
      "activations/layer5_attention_weight_max": 64.57740783691406,
      "activations/layer5_attention_weight_min": -65.77838134765625,
      "activations/layer6_attention_weight_max": 49.40760803222656,
      "activations/layer6_attention_weight_min": -53.35236358642578,
      "activations/layer7_attention_weight_max": 91.87390899658203,
      "activations/layer7_attention_weight_min": -86.94123077392578,
      "activations/layer8_attention_weight_max": 40.86378479003906,
      "activations/layer8_attention_weight_min": -40.18960189819336,
      "activations/layer9_attention_weight_max": 35.49711227416992,
      "activations/layer9_attention_weight_min": -35.18502426147461,
      "epoch": 7.13,
      "learning_rate": 0.00010509848484848484,
      "loss": 2.7912,
      "step": 122650
    },
    {
      "activations/layer0_attention_weight_max": 15.362200736999512,
      "activations/layer0_attention_weight_min": -10.075494766235352,
      "activations/layer10_attention_weight_max": 39.28496170043945,
      "activations/layer10_attention_weight_min": -37.28743362426758,
      "activations/layer11_attention_weight_max": 37.815792083740234,
      "activations/layer11_attention_weight_min": -33.304725646972656,
      "activations/layer12_attention_weight_max": 28.770713806152344,
      "activations/layer12_attention_weight_min": -29.79198455810547,
      "activations/layer13_attention_weight_max": 45.54951477050781,
      "activations/layer13_attention_weight_min": -35.93916702270508,
      "activations/layer14_attention_weight_max": 61.27143096923828,
      "activations/layer14_attention_weight_min": -55.298316955566406,
      "activations/layer15_attention_weight_max": 40.36723709106445,
      "activations/layer15_attention_weight_min": -31.485116958618164,
      "activations/layer16_attention_weight_max": 34.79768371582031,
      "activations/layer16_attention_weight_min": -29.744613647460938,
      "activations/layer17_attention_weight_max": 41.462120056152344,
      "activations/layer17_attention_weight_min": -26.344970703125,
      "activations/layer18_attention_weight_max": 29.476327896118164,
      "activations/layer18_attention_weight_min": -22.011091232299805,
      "activations/layer19_attention_weight_max": 36.943355560302734,
      "activations/layer19_attention_weight_min": -32.77094650268555,
      "activations/layer1_attention_weight_max": 16.83841896057129,
      "activations/layer1_attention_weight_min": -14.017693519592285,
      "activations/layer20_attention_weight_max": 33.737064361572266,
      "activations/layer20_attention_weight_min": -25.897769927978516,
      "activations/layer21_attention_weight_max": 31.840896606445312,
      "activations/layer21_attention_weight_min": -24.049022674560547,
      "activations/layer22_attention_weight_max": 50.96446228027344,
      "activations/layer22_attention_weight_min": -28.97959327697754,
      "activations/layer23_attention_weight_max": 33.42878723144531,
      "activations/layer23_attention_weight_min": -22.850997924804688,
      "activations/layer2_attention_weight_max": 33.598663330078125,
      "activations/layer2_attention_weight_min": -35.1080207824707,
      "activations/layer3_attention_weight_max": 94.75067901611328,
      "activations/layer3_attention_weight_min": -97.11122131347656,
      "activations/layer4_attention_weight_max": 122.5899887084961,
      "activations/layer4_attention_weight_min": -120.35201263427734,
      "activations/layer5_attention_weight_max": 64.58109283447266,
      "activations/layer5_attention_weight_min": -63.734615325927734,
      "activations/layer6_attention_weight_max": 53.78010559082031,
      "activations/layer6_attention_weight_min": -53.279422760009766,
      "activations/layer7_attention_weight_max": 92.98834228515625,
      "activations/layer7_attention_weight_min": -100.55142974853516,
      "activations/layer8_attention_weight_max": 44.29951858520508,
      "activations/layer8_attention_weight_min": -45.967288970947266,
      "activations/layer9_attention_weight_max": 43.28634262084961,
      "activations/layer9_attention_weight_min": -39.12250900268555,
      "epoch": 7.13,
      "learning_rate": 0.00010507954545454545,
      "loss": 2.8045,
      "step": 122700
    },
    {
      "activations/layer0_attention_weight_max": 15.067421913146973,
      "activations/layer0_attention_weight_min": -10.748796463012695,
      "activations/layer10_attention_weight_max": 35.30219650268555,
      "activations/layer10_attention_weight_min": -33.59772491455078,
      "activations/layer11_attention_weight_max": 40.657371520996094,
      "activations/layer11_attention_weight_min": -30.025630950927734,
      "activations/layer12_attention_weight_max": 36.16932678222656,
      "activations/layer12_attention_weight_min": -24.651721954345703,
      "activations/layer13_attention_weight_max": 40.16457748413086,
      "activations/layer13_attention_weight_min": -32.23561096191406,
      "activations/layer14_attention_weight_max": 52.61854934692383,
      "activations/layer14_attention_weight_min": -38.80062484741211,
      "activations/layer15_attention_weight_max": 40.02626419067383,
      "activations/layer15_attention_weight_min": -28.64447593688965,
      "activations/layer16_attention_weight_max": 37.505550384521484,
      "activations/layer16_attention_weight_min": -29.320913314819336,
      "activations/layer17_attention_weight_max": 35.78943634033203,
      "activations/layer17_attention_weight_min": -24.77764320373535,
      "activations/layer18_attention_weight_max": 30.69228744506836,
      "activations/layer18_attention_weight_min": -23.190059661865234,
      "activations/layer19_attention_weight_max": 38.20652389526367,
      "activations/layer19_attention_weight_min": -28.47015953063965,
      "activations/layer1_attention_weight_max": 16.454978942871094,
      "activations/layer1_attention_weight_min": -14.152077674865723,
      "activations/layer20_attention_weight_max": 31.198246002197266,
      "activations/layer20_attention_weight_min": -23.443880081176758,
      "activations/layer21_attention_weight_max": 29.697113037109375,
      "activations/layer21_attention_weight_min": -22.330015182495117,
      "activations/layer22_attention_weight_max": 47.36839294433594,
      "activations/layer22_attention_weight_min": -28.920164108276367,
      "activations/layer23_attention_weight_max": 36.880245208740234,
      "activations/layer23_attention_weight_min": -21.79995346069336,
      "activations/layer2_attention_weight_max": 31.472618103027344,
      "activations/layer2_attention_weight_min": -31.989622116088867,
      "activations/layer3_attention_weight_max": 93.01551055908203,
      "activations/layer3_attention_weight_min": -94.1093978881836,
      "activations/layer4_attention_weight_max": 133.51097106933594,
      "activations/layer4_attention_weight_min": -128.1151580810547,
      "activations/layer5_attention_weight_max": 69.80386352539062,
      "activations/layer5_attention_weight_min": -68.33583068847656,
      "activations/layer6_attention_weight_max": 52.81045150756836,
      "activations/layer6_attention_weight_min": -54.49787521362305,
      "activations/layer7_attention_weight_max": 95.0158920288086,
      "activations/layer7_attention_weight_min": -88.41304779052734,
      "activations/layer8_attention_weight_max": 41.4360237121582,
      "activations/layer8_attention_weight_min": -40.84733581542969,
      "activations/layer9_attention_weight_max": 36.76853942871094,
      "activations/layer9_attention_weight_min": -34.943450927734375,
      "epoch": 7.13,
      "learning_rate": 0.00010506060606060604,
      "loss": 2.8282,
      "step": 122750
    },
    {
      "activations/layer0_attention_weight_max": 14.44473934173584,
      "activations/layer0_attention_weight_min": -10.884607315063477,
      "activations/layer10_attention_weight_max": 31.988121032714844,
      "activations/layer10_attention_weight_min": -31.799108505249023,
      "activations/layer11_attention_weight_max": 31.3043212890625,
      "activations/layer11_attention_weight_min": -28.25020408630371,
      "activations/layer12_attention_weight_max": 23.696081161499023,
      "activations/layer12_attention_weight_min": -26.589725494384766,
      "activations/layer13_attention_weight_max": 44.29667282104492,
      "activations/layer13_attention_weight_min": -33.68483352661133,
      "activations/layer14_attention_weight_max": 57.07661819458008,
      "activations/layer14_attention_weight_min": -44.546634674072266,
      "activations/layer15_attention_weight_max": 35.407955169677734,
      "activations/layer15_attention_weight_min": -29.074962615966797,
      "activations/layer16_attention_weight_max": 34.277130126953125,
      "activations/layer16_attention_weight_min": -28.445125579833984,
      "activations/layer17_attention_weight_max": 39.16861343383789,
      "activations/layer17_attention_weight_min": -26.3493595123291,
      "activations/layer18_attention_weight_max": 33.39256286621094,
      "activations/layer18_attention_weight_min": -21.514095306396484,
      "activations/layer19_attention_weight_max": 37.96813201904297,
      "activations/layer19_attention_weight_min": -31.399194717407227,
      "activations/layer1_attention_weight_max": 16.871477127075195,
      "activations/layer1_attention_weight_min": -15.395644187927246,
      "activations/layer20_attention_weight_max": 32.24129104614258,
      "activations/layer20_attention_weight_min": -23.617046356201172,
      "activations/layer21_attention_weight_max": 31.112396240234375,
      "activations/layer21_attention_weight_min": -23.615842819213867,
      "activations/layer22_attention_weight_max": 52.54973602294922,
      "activations/layer22_attention_weight_min": -27.92566680908203,
      "activations/layer23_attention_weight_max": 36.29301071166992,
      "activations/layer23_attention_weight_min": -23.767454147338867,
      "activations/layer2_attention_weight_max": 32.297698974609375,
      "activations/layer2_attention_weight_min": -32.079856872558594,
      "activations/layer3_attention_weight_max": 90.14320373535156,
      "activations/layer3_attention_weight_min": -91.21820068359375,
      "activations/layer4_attention_weight_max": 121.48946380615234,
      "activations/layer4_attention_weight_min": -121.8656005859375,
      "activations/layer5_attention_weight_max": 65.37232971191406,
      "activations/layer5_attention_weight_min": -63.28837203979492,
      "activations/layer6_attention_weight_max": 51.30629348754883,
      "activations/layer6_attention_weight_min": -52.9930534362793,
      "activations/layer7_attention_weight_max": 86.05001831054688,
      "activations/layer7_attention_weight_min": -87.2989501953125,
      "activations/layer8_attention_weight_max": 43.959415435791016,
      "activations/layer8_attention_weight_min": -43.776893615722656,
      "activations/layer9_attention_weight_max": 36.011016845703125,
      "activations/layer9_attention_weight_min": -34.7883415222168,
      "epoch": 7.14,
      "learning_rate": 0.00010504166666666665,
      "loss": 2.8236,
      "step": 122800
    },
    {
      "activations/layer0_attention_weight_max": 14.303386688232422,
      "activations/layer0_attention_weight_min": -11.58070182800293,
      "activations/layer10_attention_weight_max": 32.418617248535156,
      "activations/layer10_attention_weight_min": -31.116079330444336,
      "activations/layer11_attention_weight_max": 32.12189483642578,
      "activations/layer11_attention_weight_min": -28.167205810546875,
      "activations/layer12_attention_weight_max": 21.58125114440918,
      "activations/layer12_attention_weight_min": -25.04429054260254,
      "activations/layer13_attention_weight_max": 38.487300872802734,
      "activations/layer13_attention_weight_min": -32.50349044799805,
      "activations/layer14_attention_weight_max": 49.12309265136719,
      "activations/layer14_attention_weight_min": -40.67627716064453,
      "activations/layer15_attention_weight_max": 33.864524841308594,
      "activations/layer15_attention_weight_min": -28.728248596191406,
      "activations/layer16_attention_weight_max": 32.16117858886719,
      "activations/layer16_attention_weight_min": -28.1534481048584,
      "activations/layer17_attention_weight_max": 38.03524398803711,
      "activations/layer17_attention_weight_min": -23.69820213317871,
      "activations/layer18_attention_weight_max": 31.589754104614258,
      "activations/layer18_attention_weight_min": -20.34684944152832,
      "activations/layer19_attention_weight_max": 35.92068862915039,
      "activations/layer19_attention_weight_min": -31.480712890625,
      "activations/layer1_attention_weight_max": 16.4833984375,
      "activations/layer1_attention_weight_min": -13.944993019104004,
      "activations/layer20_attention_weight_max": 34.52592086791992,
      "activations/layer20_attention_weight_min": -24.949485778808594,
      "activations/layer21_attention_weight_max": 32.56857681274414,
      "activations/layer21_attention_weight_min": -24.869779586791992,
      "activations/layer22_attention_weight_max": 51.073089599609375,
      "activations/layer22_attention_weight_min": -31.313732147216797,
      "activations/layer23_attention_weight_max": 35.881858825683594,
      "activations/layer23_attention_weight_min": -26.070602416992188,
      "activations/layer2_attention_weight_max": 33.3759765625,
      "activations/layer2_attention_weight_min": -30.94556999206543,
      "activations/layer3_attention_weight_max": 89.68109130859375,
      "activations/layer3_attention_weight_min": -90.98822021484375,
      "activations/layer4_attention_weight_max": 122.40032958984375,
      "activations/layer4_attention_weight_min": -118.1793441772461,
      "activations/layer5_attention_weight_max": 62.8275146484375,
      "activations/layer5_attention_weight_min": -63.14471435546875,
      "activations/layer6_attention_weight_max": 53.89742660522461,
      "activations/layer6_attention_weight_min": -52.99387741088867,
      "activations/layer7_attention_weight_max": 87.95085144042969,
      "activations/layer7_attention_weight_min": -86.21928405761719,
      "activations/layer8_attention_weight_max": 38.52474594116211,
      "activations/layer8_attention_weight_min": -40.62345504760742,
      "activations/layer9_attention_weight_max": 35.15690231323242,
      "activations/layer9_attention_weight_min": -34.482627868652344,
      "epoch": 7.14,
      "learning_rate": 0.00010502272727272727,
      "loss": 2.8106,
      "step": 122850
    },
    {
      "activations/layer0_attention_weight_max": 14.42548656463623,
      "activations/layer0_attention_weight_min": -11.74937629699707,
      "activations/layer10_attention_weight_max": 32.879051208496094,
      "activations/layer10_attention_weight_min": -31.56728744506836,
      "activations/layer11_attention_weight_max": 31.762386322021484,
      "activations/layer11_attention_weight_min": -29.628890991210938,
      "activations/layer12_attention_weight_max": 24.255001068115234,
      "activations/layer12_attention_weight_min": -23.98240089416504,
      "activations/layer13_attention_weight_max": 42.313011169433594,
      "activations/layer13_attention_weight_min": -31.244644165039062,
      "activations/layer14_attention_weight_max": 68.08976745605469,
      "activations/layer14_attention_weight_min": -40.63286590576172,
      "activations/layer15_attention_weight_max": 40.59708786010742,
      "activations/layer15_attention_weight_min": -30.15277862548828,
      "activations/layer16_attention_weight_max": 38.492496490478516,
      "activations/layer16_attention_weight_min": -26.5000057220459,
      "activations/layer17_attention_weight_max": 35.297203063964844,
      "activations/layer17_attention_weight_min": -24.94472885131836,
      "activations/layer18_attention_weight_max": 31.158634185791016,
      "activations/layer18_attention_weight_min": -21.85335922241211,
      "activations/layer19_attention_weight_max": 43.024234771728516,
      "activations/layer19_attention_weight_min": -29.602079391479492,
      "activations/layer1_attention_weight_max": 17.47731590270996,
      "activations/layer1_attention_weight_min": -14.570520401000977,
      "activations/layer20_attention_weight_max": 36.47787857055664,
      "activations/layer20_attention_weight_min": -26.25799560546875,
      "activations/layer21_attention_weight_max": 35.84394073486328,
      "activations/layer21_attention_weight_min": -23.83184051513672,
      "activations/layer22_attention_weight_max": 53.988304138183594,
      "activations/layer22_attention_weight_min": -29.099931716918945,
      "activations/layer23_attention_weight_max": 38.154029846191406,
      "activations/layer23_attention_weight_min": -22.76924705505371,
      "activations/layer2_attention_weight_max": 31.262603759765625,
      "activations/layer2_attention_weight_min": -30.44438934326172,
      "activations/layer3_attention_weight_max": 91.49227142333984,
      "activations/layer3_attention_weight_min": -89.2854995727539,
      "activations/layer4_attention_weight_max": 126.80879211425781,
      "activations/layer4_attention_weight_min": -119.84493255615234,
      "activations/layer5_attention_weight_max": 67.23667907714844,
      "activations/layer5_attention_weight_min": -64.06334686279297,
      "activations/layer6_attention_weight_max": 52.757511138916016,
      "activations/layer6_attention_weight_min": -52.217857360839844,
      "activations/layer7_attention_weight_max": 91.93485260009766,
      "activations/layer7_attention_weight_min": -85.76653289794922,
      "activations/layer8_attention_weight_max": 42.036930084228516,
      "activations/layer8_attention_weight_min": -41.447811126708984,
      "activations/layer9_attention_weight_max": 35.75722885131836,
      "activations/layer9_attention_weight_min": -33.610103607177734,
      "epoch": 7.14,
      "learning_rate": 0.00010500378787878787,
      "loss": 2.8192,
      "step": 122900
    },
    {
      "activations/layer0_attention_weight_max": 15.817397117614746,
      "activations/layer0_attention_weight_min": -10.46069049835205,
      "activations/layer10_attention_weight_max": 32.01265335083008,
      "activations/layer10_attention_weight_min": -31.277889251708984,
      "activations/layer11_attention_weight_max": 31.787086486816406,
      "activations/layer11_attention_weight_min": -28.69094467163086,
      "activations/layer12_attention_weight_max": 35.1093864440918,
      "activations/layer12_attention_weight_min": -35.943546295166016,
      "activations/layer13_attention_weight_max": 38.32109832763672,
      "activations/layer13_attention_weight_min": -32.773292541503906,
      "activations/layer14_attention_weight_max": 54.72910690307617,
      "activations/layer14_attention_weight_min": -45.09291076660156,
      "activations/layer15_attention_weight_max": 36.706905364990234,
      "activations/layer15_attention_weight_min": -28.150991439819336,
      "activations/layer16_attention_weight_max": 36.00297927856445,
      "activations/layer16_attention_weight_min": -28.082653045654297,
      "activations/layer17_attention_weight_max": 30.822662353515625,
      "activations/layer17_attention_weight_min": -23.990665435791016,
      "activations/layer18_attention_weight_max": 29.15424156188965,
      "activations/layer18_attention_weight_min": -19.573287963867188,
      "activations/layer19_attention_weight_max": 37.885128021240234,
      "activations/layer19_attention_weight_min": -28.47185707092285,
      "activations/layer1_attention_weight_max": 17.970848083496094,
      "activations/layer1_attention_weight_min": -14.045170783996582,
      "activations/layer20_attention_weight_max": 27.43999481201172,
      "activations/layer20_attention_weight_min": -22.594053268432617,
      "activations/layer21_attention_weight_max": 27.150968551635742,
      "activations/layer21_attention_weight_min": -20.53084373474121,
      "activations/layer22_attention_weight_max": 44.09529113769531,
      "activations/layer22_attention_weight_min": -26.488645553588867,
      "activations/layer23_attention_weight_max": 37.55168151855469,
      "activations/layer23_attention_weight_min": -21.98922348022461,
      "activations/layer2_attention_weight_max": 30.592185974121094,
      "activations/layer2_attention_weight_min": -34.72304916381836,
      "activations/layer3_attention_weight_max": 91.86305236816406,
      "activations/layer3_attention_weight_min": -100.6082763671875,
      "activations/layer4_attention_weight_max": 126.05159759521484,
      "activations/layer4_attention_weight_min": -125.03905487060547,
      "activations/layer5_attention_weight_max": 63.5538330078125,
      "activations/layer5_attention_weight_min": -61.41498947143555,
      "activations/layer6_attention_weight_max": 51.020565032958984,
      "activations/layer6_attention_weight_min": -52.08037567138672,
      "activations/layer7_attention_weight_max": 95.33160400390625,
      "activations/layer7_attention_weight_min": -82.95195007324219,
      "activations/layer8_attention_weight_max": 39.31953811645508,
      "activations/layer8_attention_weight_min": -40.387237548828125,
      "activations/layer9_attention_weight_max": 34.162071228027344,
      "activations/layer9_attention_weight_min": -47.17876434326172,
      "epoch": 7.14,
      "learning_rate": 0.00010498484848484847,
      "loss": 2.8071,
      "step": 122950
    },
    {
      "activations/layer0_attention_weight_max": 14.537694931030273,
      "activations/layer0_attention_weight_min": -11.339561462402344,
      "activations/layer10_attention_weight_max": 36.13697052001953,
      "activations/layer10_attention_weight_min": -32.259674072265625,
      "activations/layer11_attention_weight_max": 32.70023727416992,
      "activations/layer11_attention_weight_min": -30.60016632080078,
      "activations/layer12_attention_weight_max": 29.75164222717285,
      "activations/layer12_attention_weight_min": -28.346759796142578,
      "activations/layer13_attention_weight_max": 47.19725799560547,
      "activations/layer13_attention_weight_min": -34.72554016113281,
      "activations/layer14_attention_weight_max": 66.76529693603516,
      "activations/layer14_attention_weight_min": -46.03449630737305,
      "activations/layer15_attention_weight_max": 39.52820587158203,
      "activations/layer15_attention_weight_min": -28.340579986572266,
      "activations/layer16_attention_weight_max": 32.80338668823242,
      "activations/layer16_attention_weight_min": -28.660457611083984,
      "activations/layer17_attention_weight_max": 37.21034240722656,
      "activations/layer17_attention_weight_min": -25.5819034576416,
      "activations/layer18_attention_weight_max": 33.988441467285156,
      "activations/layer18_attention_weight_min": -22.17131805419922,
      "activations/layer19_attention_weight_max": 38.308631896972656,
      "activations/layer19_attention_weight_min": -29.903329849243164,
      "activations/layer1_attention_weight_max": 16.773984909057617,
      "activations/layer1_attention_weight_min": -14.209952354431152,
      "activations/layer20_attention_weight_max": 31.3080997467041,
      "activations/layer20_attention_weight_min": -24.929643630981445,
      "activations/layer21_attention_weight_max": 30.902360916137695,
      "activations/layer21_attention_weight_min": -22.72297477722168,
      "activations/layer22_attention_weight_max": 51.45891571044922,
      "activations/layer22_attention_weight_min": -28.75509262084961,
      "activations/layer23_attention_weight_max": 37.982337951660156,
      "activations/layer23_attention_weight_min": -24.796079635620117,
      "activations/layer2_attention_weight_max": 33.40052032470703,
      "activations/layer2_attention_weight_min": -31.38190460205078,
      "activations/layer3_attention_weight_max": 92.80522155761719,
      "activations/layer3_attention_weight_min": -95.8672866821289,
      "activations/layer4_attention_weight_max": 126.74269104003906,
      "activations/layer4_attention_weight_min": -128.7825469970703,
      "activations/layer5_attention_weight_max": 64.36742401123047,
      "activations/layer5_attention_weight_min": -69.46253204345703,
      "activations/layer6_attention_weight_max": 51.50406265258789,
      "activations/layer6_attention_weight_min": -53.58333969116211,
      "activations/layer7_attention_weight_max": 100.90979766845703,
      "activations/layer7_attention_weight_min": -94.61579895019531,
      "activations/layer8_attention_weight_max": 45.17238235473633,
      "activations/layer8_attention_weight_min": -45.298095703125,
      "activations/layer9_attention_weight_max": 38.76121139526367,
      "activations/layer9_attention_weight_min": -36.45554733276367,
      "epoch": 7.15,
      "learning_rate": 0.00010496590909090909,
      "loss": 2.8007,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5229,
      "eval_samples_per_second": 503.819,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5229,
      "eval_openwebtext_samples_per_second": 503.819,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_wikitext_loss": 3.009765625,
      "eval_wikitext_ppl": 20.282645623050005,
      "eval_wikitext_runtime": 2.0023,
      "eval_wikitext_samples_per_second": 227.74,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_lambada_loss": 2.6953125,
      "eval_lambada_ppl": 14.81014620089916,
      "eval_lambada_runtime": 9.6225,
      "eval_lambada_samples_per_second": 506.0,
      "step": 123000
    },
    {
      "activations/layer0_attention_weight_max": 15.348812103271484,
      "activations/layer0_attention_weight_min": -10.7245454788208,
      "activations/layer10_attention_weight_max": 35.32497024536133,
      "activations/layer10_attention_weight_min": -34.061279296875,
      "activations/layer11_attention_weight_max": 34.382965087890625,
      "activations/layer11_attention_weight_min": -30.819250106811523,
      "activations/layer12_attention_weight_max": 32.74646759033203,
      "activations/layer12_attention_weight_min": -26.980506896972656,
      "activations/layer13_attention_weight_max": 51.505924224853516,
      "activations/layer13_attention_weight_min": -36.94057846069336,
      "activations/layer14_attention_weight_max": 59.323421478271484,
      "activations/layer14_attention_weight_min": -40.59941101074219,
      "activations/layer15_attention_weight_max": 38.003841400146484,
      "activations/layer15_attention_weight_min": -28.695016860961914,
      "activations/layer16_attention_weight_max": 31.896488189697266,
      "activations/layer16_attention_weight_min": -26.16837501525879,
      "activations/layer17_attention_weight_max": 35.45378112792969,
      "activations/layer17_attention_weight_min": -24.32874298095703,
      "activations/layer18_attention_weight_max": 31.811006546020508,
      "activations/layer18_attention_weight_min": -22.06390953063965,
      "activations/layer19_attention_weight_max": 34.442996978759766,
      "activations/layer19_attention_weight_min": -29.10031509399414,
      "activations/layer1_attention_weight_max": 16.875581741333008,
      "activations/layer1_attention_weight_min": -16.330596923828125,
      "activations/layer20_attention_weight_max": 31.03228759765625,
      "activations/layer20_attention_weight_min": -24.40038299560547,
      "activations/layer21_attention_weight_max": 31.580230712890625,
      "activations/layer21_attention_weight_min": -22.396411895751953,
      "activations/layer22_attention_weight_max": 45.6083869934082,
      "activations/layer22_attention_weight_min": -29.811771392822266,
      "activations/layer23_attention_weight_max": 37.859127044677734,
      "activations/layer23_attention_weight_min": -22.513641357421875,
      "activations/layer2_attention_weight_max": 32.857521057128906,
      "activations/layer2_attention_weight_min": -30.594581604003906,
      "activations/layer3_attention_weight_max": 90.80284881591797,
      "activations/layer3_attention_weight_min": -91.1514663696289,
      "activations/layer4_attention_weight_max": 123.40001678466797,
      "activations/layer4_attention_weight_min": -118.70731353759766,
      "activations/layer5_attention_weight_max": 65.82638549804688,
      "activations/layer5_attention_weight_min": -63.42144775390625,
      "activations/layer6_attention_weight_max": 53.15317153930664,
      "activations/layer6_attention_weight_min": -56.158409118652344,
      "activations/layer7_attention_weight_max": 87.89570617675781,
      "activations/layer7_attention_weight_min": -91.68073272705078,
      "activations/layer8_attention_weight_max": 42.56318664550781,
      "activations/layer8_attention_weight_min": -42.2720832824707,
      "activations/layer9_attention_weight_max": 37.77770233154297,
      "activations/layer9_attention_weight_min": -37.19989013671875,
      "epoch": 7.15,
      "learning_rate": 0.00010494696969696969,
      "loss": 2.8078,
      "step": 123050
    },
    {
      "activations/layer0_attention_weight_max": 15.375567436218262,
      "activations/layer0_attention_weight_min": -11.862663269042969,
      "activations/layer10_attention_weight_max": 33.20145797729492,
      "activations/layer10_attention_weight_min": -30.500885009765625,
      "activations/layer11_attention_weight_max": 31.621849060058594,
      "activations/layer11_attention_weight_min": -28.914588928222656,
      "activations/layer12_attention_weight_max": 23.213260650634766,
      "activations/layer12_attention_weight_min": -25.506454467773438,
      "activations/layer13_attention_weight_max": 37.85747146606445,
      "activations/layer13_attention_weight_min": -32.20573425292969,
      "activations/layer14_attention_weight_max": 51.473392486572266,
      "activations/layer14_attention_weight_min": -38.70793914794922,
      "activations/layer15_attention_weight_max": 36.57716369628906,
      "activations/layer15_attention_weight_min": -31.749704360961914,
      "activations/layer16_attention_weight_max": 33.91948318481445,
      "activations/layer16_attention_weight_min": -29.039731979370117,
      "activations/layer17_attention_weight_max": 34.044681549072266,
      "activations/layer17_attention_weight_min": -24.7973575592041,
      "activations/layer18_attention_weight_max": 35.890350341796875,
      "activations/layer18_attention_weight_min": -20.955509185791016,
      "activations/layer19_attention_weight_max": 35.89704132080078,
      "activations/layer19_attention_weight_min": -29.57785987854004,
      "activations/layer1_attention_weight_max": 16.362464904785156,
      "activations/layer1_attention_weight_min": -14.468365669250488,
      "activations/layer20_attention_weight_max": 30.31002426147461,
      "activations/layer20_attention_weight_min": -23.25639533996582,
      "activations/layer21_attention_weight_max": 27.817861557006836,
      "activations/layer21_attention_weight_min": -22.35358428955078,
      "activations/layer22_attention_weight_max": 48.79033660888672,
      "activations/layer22_attention_weight_min": -30.459484100341797,
      "activations/layer23_attention_weight_max": 34.54340362548828,
      "activations/layer23_attention_weight_min": -23.640663146972656,
      "activations/layer2_attention_weight_max": 33.53253173828125,
      "activations/layer2_attention_weight_min": -31.666074752807617,
      "activations/layer3_attention_weight_max": 93.66801452636719,
      "activations/layer3_attention_weight_min": -93.88627624511719,
      "activations/layer4_attention_weight_max": 125.60375213623047,
      "activations/layer4_attention_weight_min": -119.83106994628906,
      "activations/layer5_attention_weight_max": 63.2891960144043,
      "activations/layer5_attention_weight_min": -61.33448028564453,
      "activations/layer6_attention_weight_max": 48.80970001220703,
      "activations/layer6_attention_weight_min": -51.87206268310547,
      "activations/layer7_attention_weight_max": 84.4531021118164,
      "activations/layer7_attention_weight_min": -85.3167953491211,
      "activations/layer8_attention_weight_max": 38.030426025390625,
      "activations/layer8_attention_weight_min": -38.3748893737793,
      "activations/layer9_attention_weight_max": 35.018131256103516,
      "activations/layer9_attention_weight_min": -32.123565673828125,
      "epoch": 7.15,
      "learning_rate": 0.0001049280303030303,
      "loss": 2.8024,
      "step": 123100
    },
    {
      "activations/layer0_attention_weight_max": 15.077120780944824,
      "activations/layer0_attention_weight_min": -11.635787963867188,
      "activations/layer10_attention_weight_max": 40.549598693847656,
      "activations/layer10_attention_weight_min": -33.14675521850586,
      "activations/layer11_attention_weight_max": 36.459232330322266,
      "activations/layer11_attention_weight_min": -29.78533172607422,
      "activations/layer12_attention_weight_max": 23.78268814086914,
      "activations/layer12_attention_weight_min": -24.848407745361328,
      "activations/layer13_attention_weight_max": 50.96702194213867,
      "activations/layer13_attention_weight_min": -35.78569793701172,
      "activations/layer14_attention_weight_max": 56.3682975769043,
      "activations/layer14_attention_weight_min": -39.76206970214844,
      "activations/layer15_attention_weight_max": 40.179073333740234,
      "activations/layer15_attention_weight_min": -30.42911148071289,
      "activations/layer16_attention_weight_max": 33.44010543823242,
      "activations/layer16_attention_weight_min": -26.9271240234375,
      "activations/layer17_attention_weight_max": 34.32969284057617,
      "activations/layer17_attention_weight_min": -25.288070678710938,
      "activations/layer18_attention_weight_max": 31.3685302734375,
      "activations/layer18_attention_weight_min": -21.939289093017578,
      "activations/layer19_attention_weight_max": 40.15790939331055,
      "activations/layer19_attention_weight_min": -30.68536376953125,
      "activations/layer1_attention_weight_max": 16.663808822631836,
      "activations/layer1_attention_weight_min": -13.789945602416992,
      "activations/layer20_attention_weight_max": 31.387840270996094,
      "activations/layer20_attention_weight_min": -23.37082290649414,
      "activations/layer21_attention_weight_max": 31.371780395507812,
      "activations/layer21_attention_weight_min": -21.668611526489258,
      "activations/layer22_attention_weight_max": 52.5268669128418,
      "activations/layer22_attention_weight_min": -27.324787139892578,
      "activations/layer23_attention_weight_max": 36.5313720703125,
      "activations/layer23_attention_weight_min": -20.456398010253906,
      "activations/layer2_attention_weight_max": 32.95764923095703,
      "activations/layer2_attention_weight_min": -33.05467224121094,
      "activations/layer3_attention_weight_max": 93.72172546386719,
      "activations/layer3_attention_weight_min": -100.85743713378906,
      "activations/layer4_attention_weight_max": 126.82857513427734,
      "activations/layer4_attention_weight_min": -127.53136444091797,
      "activations/layer5_attention_weight_max": 68.06669616699219,
      "activations/layer5_attention_weight_min": -65.36624145507812,
      "activations/layer6_attention_weight_max": 55.55656433105469,
      "activations/layer6_attention_weight_min": -55.49150466918945,
      "activations/layer7_attention_weight_max": 97.34951782226562,
      "activations/layer7_attention_weight_min": -97.02464294433594,
      "activations/layer8_attention_weight_max": 45.28202438354492,
      "activations/layer8_attention_weight_min": -44.4747200012207,
      "activations/layer9_attention_weight_max": 41.62324142456055,
      "activations/layer9_attention_weight_min": -36.66830062866211,
      "epoch": 7.16,
      "learning_rate": 0.00010490909090909089,
      "loss": 2.807,
      "step": 123150
    },
    {
      "activations/layer0_attention_weight_max": 16.166221618652344,
      "activations/layer0_attention_weight_min": -11.740216255187988,
      "activations/layer10_attention_weight_max": 33.7239990234375,
      "activations/layer10_attention_weight_min": -30.974388122558594,
      "activations/layer11_attention_weight_max": 33.96027755737305,
      "activations/layer11_attention_weight_min": -30.04755401611328,
      "activations/layer12_attention_weight_max": 31.113121032714844,
      "activations/layer12_attention_weight_min": -25.16729736328125,
      "activations/layer13_attention_weight_max": 47.074344635009766,
      "activations/layer13_attention_weight_min": -32.60154342651367,
      "activations/layer14_attention_weight_max": 50.755279541015625,
      "activations/layer14_attention_weight_min": -40.44990158081055,
      "activations/layer15_attention_weight_max": 39.78166198730469,
      "activations/layer15_attention_weight_min": -28.321731567382812,
      "activations/layer16_attention_weight_max": 34.78913879394531,
      "activations/layer16_attention_weight_min": -28.275129318237305,
      "activations/layer17_attention_weight_max": 37.48277282714844,
      "activations/layer17_attention_weight_min": -25.551599502563477,
      "activations/layer18_attention_weight_max": 31.245439529418945,
      "activations/layer18_attention_weight_min": -20.976964950561523,
      "activations/layer19_attention_weight_max": 37.205806732177734,
      "activations/layer19_attention_weight_min": -30.519882202148438,
      "activations/layer1_attention_weight_max": 15.872081756591797,
      "activations/layer1_attention_weight_min": -14.780071258544922,
      "activations/layer20_attention_weight_max": 30.860261917114258,
      "activations/layer20_attention_weight_min": -25.613401412963867,
      "activations/layer21_attention_weight_max": 32.082916259765625,
      "activations/layer21_attention_weight_min": -22.6778621673584,
      "activations/layer22_attention_weight_max": 54.26625061035156,
      "activations/layer22_attention_weight_min": -29.27960968017578,
      "activations/layer23_attention_weight_max": 37.72400665283203,
      "activations/layer23_attention_weight_min": -23.58999252319336,
      "activations/layer2_attention_weight_max": 31.702314376831055,
      "activations/layer2_attention_weight_min": -30.95467185974121,
      "activations/layer3_attention_weight_max": 92.43634033203125,
      "activations/layer3_attention_weight_min": -95.49442291259766,
      "activations/layer4_attention_weight_max": 129.10507202148438,
      "activations/layer4_attention_weight_min": -125.95775604248047,
      "activations/layer5_attention_weight_max": 66.68447875976562,
      "activations/layer5_attention_weight_min": -66.59609985351562,
      "activations/layer6_attention_weight_max": 51.448097229003906,
      "activations/layer6_attention_weight_min": -57.34794235229492,
      "activations/layer7_attention_weight_max": 88.26473999023438,
      "activations/layer7_attention_weight_min": -92.20983123779297,
      "activations/layer8_attention_weight_max": 40.62529373168945,
      "activations/layer8_attention_weight_min": -39.85145568847656,
      "activations/layer9_attention_weight_max": 38.99775314331055,
      "activations/layer9_attention_weight_min": -35.35001754760742,
      "epoch": 7.16,
      "learning_rate": 0.0001048901515151515,
      "loss": 2.7955,
      "step": 123200
    },
    {
      "activations/layer0_attention_weight_max": 14.582430839538574,
      "activations/layer0_attention_weight_min": -11.5944185256958,
      "activations/layer10_attention_weight_max": 35.51729202270508,
      "activations/layer10_attention_weight_min": -34.192909240722656,
      "activations/layer11_attention_weight_max": 33.55199432373047,
      "activations/layer11_attention_weight_min": -30.788288116455078,
      "activations/layer12_attention_weight_max": 24.34048080444336,
      "activations/layer12_attention_weight_min": -25.533334732055664,
      "activations/layer13_attention_weight_max": 42.130775451660156,
      "activations/layer13_attention_weight_min": -35.7979850769043,
      "activations/layer14_attention_weight_max": 56.325618743896484,
      "activations/layer14_attention_weight_min": -43.86907958984375,
      "activations/layer15_attention_weight_max": 40.55088806152344,
      "activations/layer15_attention_weight_min": -30.771390914916992,
      "activations/layer16_attention_weight_max": 34.183448791503906,
      "activations/layer16_attention_weight_min": -28.80354881286621,
      "activations/layer17_attention_weight_max": 39.27182388305664,
      "activations/layer17_attention_weight_min": -26.027074813842773,
      "activations/layer18_attention_weight_max": 37.429683685302734,
      "activations/layer18_attention_weight_min": -21.794694900512695,
      "activations/layer19_attention_weight_max": 42.894874572753906,
      "activations/layer19_attention_weight_min": -30.72271728515625,
      "activations/layer1_attention_weight_max": 17.970890045166016,
      "activations/layer1_attention_weight_min": -14.658137321472168,
      "activations/layer20_attention_weight_max": 35.837345123291016,
      "activations/layer20_attention_weight_min": -24.619773864746094,
      "activations/layer21_attention_weight_max": 33.425987243652344,
      "activations/layer21_attention_weight_min": -22.40542984008789,
      "activations/layer22_attention_weight_max": 52.85871124267578,
      "activations/layer22_attention_weight_min": -31.018356323242188,
      "activations/layer23_attention_weight_max": 38.08626937866211,
      "activations/layer23_attention_weight_min": -23.126203536987305,
      "activations/layer2_attention_weight_max": 32.73603820800781,
      "activations/layer2_attention_weight_min": -32.12370681762695,
      "activations/layer3_attention_weight_max": 92.48594665527344,
      "activations/layer3_attention_weight_min": -93.09069061279297,
      "activations/layer4_attention_weight_max": 125.93309783935547,
      "activations/layer4_attention_weight_min": -126.9418716430664,
      "activations/layer5_attention_weight_max": 65.94793701171875,
      "activations/layer5_attention_weight_min": -65.80525207519531,
      "activations/layer6_attention_weight_max": 52.589778900146484,
      "activations/layer6_attention_weight_min": -55.40259552001953,
      "activations/layer7_attention_weight_max": 89.68721008300781,
      "activations/layer7_attention_weight_min": -88.48768615722656,
      "activations/layer8_attention_weight_max": 40.93922805786133,
      "activations/layer8_attention_weight_min": -43.88014221191406,
      "activations/layer9_attention_weight_max": 37.05678176879883,
      "activations/layer9_attention_weight_min": -36.13360595703125,
      "epoch": 7.16,
      "learning_rate": 0.00010487121212121212,
      "loss": 2.7916,
      "step": 123250
    },
    {
      "activations/layer0_attention_weight_max": 15.542810440063477,
      "activations/layer0_attention_weight_min": -11.465978622436523,
      "activations/layer10_attention_weight_max": 33.68647003173828,
      "activations/layer10_attention_weight_min": -30.934215545654297,
      "activations/layer11_attention_weight_max": 33.24623107910156,
      "activations/layer11_attention_weight_min": -28.539960861206055,
      "activations/layer12_attention_weight_max": 39.44980239868164,
      "activations/layer12_attention_weight_min": -24.82585906982422,
      "activations/layer13_attention_weight_max": 44.19747543334961,
      "activations/layer13_attention_weight_min": -34.086185455322266,
      "activations/layer14_attention_weight_max": 47.608604431152344,
      "activations/layer14_attention_weight_min": -38.853233337402344,
      "activations/layer15_attention_weight_max": 37.92656707763672,
      "activations/layer15_attention_weight_min": -30.963186264038086,
      "activations/layer16_attention_weight_max": 32.82547378540039,
      "activations/layer16_attention_weight_min": -27.272768020629883,
      "activations/layer17_attention_weight_max": 34.45872497558594,
      "activations/layer17_attention_weight_min": -25.106781005859375,
      "activations/layer18_attention_weight_max": 32.492855072021484,
      "activations/layer18_attention_weight_min": -20.42031478881836,
      "activations/layer19_attention_weight_max": 35.08625411987305,
      "activations/layer19_attention_weight_min": -32.40869140625,
      "activations/layer1_attention_weight_max": 16.232088088989258,
      "activations/layer1_attention_weight_min": -14.729883193969727,
      "activations/layer20_attention_weight_max": 29.241371154785156,
      "activations/layer20_attention_weight_min": -22.679264068603516,
      "activations/layer21_attention_weight_max": 28.979101181030273,
      "activations/layer21_attention_weight_min": -23.154386520385742,
      "activations/layer22_attention_weight_max": 46.76249313354492,
      "activations/layer22_attention_weight_min": -28.078460693359375,
      "activations/layer23_attention_weight_max": 33.59716796875,
      "activations/layer23_attention_weight_min": -24.424957275390625,
      "activations/layer2_attention_weight_max": 31.896595001220703,
      "activations/layer2_attention_weight_min": -31.6287841796875,
      "activations/layer3_attention_weight_max": 89.74160766601562,
      "activations/layer3_attention_weight_min": -92.1585922241211,
      "activations/layer4_attention_weight_max": 121.71671295166016,
      "activations/layer4_attention_weight_min": -120.69542694091797,
      "activations/layer5_attention_weight_max": 62.8023681640625,
      "activations/layer5_attention_weight_min": -62.18761444091797,
      "activations/layer6_attention_weight_max": 50.21342468261719,
      "activations/layer6_attention_weight_min": -52.38184356689453,
      "activations/layer7_attention_weight_max": 84.73428344726562,
      "activations/layer7_attention_weight_min": -88.7311782836914,
      "activations/layer8_attention_weight_max": 37.686744689941406,
      "activations/layer8_attention_weight_min": -40.21147155761719,
      "activations/layer9_attention_weight_max": 34.46981430053711,
      "activations/layer9_attention_weight_min": -32.6255989074707,
      "epoch": 7.16,
      "learning_rate": 0.00010485227272727271,
      "loss": 2.7852,
      "step": 123300
    },
    {
      "activations/layer0_attention_weight_max": 15.259611129760742,
      "activations/layer0_attention_weight_min": -10.92354965209961,
      "activations/layer10_attention_weight_max": 35.09536361694336,
      "activations/layer10_attention_weight_min": -33.0074462890625,
      "activations/layer11_attention_weight_max": 33.244510650634766,
      "activations/layer11_attention_weight_min": -30.201740264892578,
      "activations/layer12_attention_weight_max": 25.53350067138672,
      "activations/layer12_attention_weight_min": -29.337512969970703,
      "activations/layer13_attention_weight_max": 42.80781936645508,
      "activations/layer13_attention_weight_min": -34.885311126708984,
      "activations/layer14_attention_weight_max": 54.82379150390625,
      "activations/layer14_attention_weight_min": -43.453922271728516,
      "activations/layer15_attention_weight_max": 37.49464416503906,
      "activations/layer15_attention_weight_min": -29.61494255065918,
      "activations/layer16_attention_weight_max": 35.32536315917969,
      "activations/layer16_attention_weight_min": -28.352537155151367,
      "activations/layer17_attention_weight_max": 38.8618278503418,
      "activations/layer17_attention_weight_min": -26.073007583618164,
      "activations/layer18_attention_weight_max": 34.19404220581055,
      "activations/layer18_attention_weight_min": -22.107370376586914,
      "activations/layer19_attention_weight_max": 41.89105987548828,
      "activations/layer19_attention_weight_min": -29.728952407836914,
      "activations/layer1_attention_weight_max": 16.638450622558594,
      "activations/layer1_attention_weight_min": -15.921651840209961,
      "activations/layer20_attention_weight_max": 32.5750732421875,
      "activations/layer20_attention_weight_min": -23.842845916748047,
      "activations/layer21_attention_weight_max": 31.21979331970215,
      "activations/layer21_attention_weight_min": -23.76091766357422,
      "activations/layer22_attention_weight_max": 51.486812591552734,
      "activations/layer22_attention_weight_min": -29.10794448852539,
      "activations/layer23_attention_weight_max": 40.831146240234375,
      "activations/layer23_attention_weight_min": -22.77251434326172,
      "activations/layer2_attention_weight_max": 31.628978729248047,
      "activations/layer2_attention_weight_min": -30.984586715698242,
      "activations/layer3_attention_weight_max": 91.49473571777344,
      "activations/layer3_attention_weight_min": -94.8189926147461,
      "activations/layer4_attention_weight_max": 126.81871795654297,
      "activations/layer4_attention_weight_min": -126.49505615234375,
      "activations/layer5_attention_weight_max": 65.84030151367188,
      "activations/layer5_attention_weight_min": -62.819252014160156,
      "activations/layer6_attention_weight_max": 53.35587692260742,
      "activations/layer6_attention_weight_min": -53.64279556274414,
      "activations/layer7_attention_weight_max": 94.628173828125,
      "activations/layer7_attention_weight_min": -94.43980407714844,
      "activations/layer8_attention_weight_max": 41.86921310424805,
      "activations/layer8_attention_weight_min": -42.000335693359375,
      "activations/layer9_attention_weight_max": 38.3011474609375,
      "activations/layer9_attention_weight_min": -35.25896453857422,
      "epoch": 7.17,
      "learning_rate": 0.00010483333333333332,
      "loss": 2.8204,
      "step": 123350
    },
    {
      "activations/layer0_attention_weight_max": 14.25961971282959,
      "activations/layer0_attention_weight_min": -11.4182710647583,
      "activations/layer10_attention_weight_max": 32.86940383911133,
      "activations/layer10_attention_weight_min": -31.5371150970459,
      "activations/layer11_attention_weight_max": 34.68464279174805,
      "activations/layer11_attention_weight_min": -30.350765228271484,
      "activations/layer12_attention_weight_max": 31.749141693115234,
      "activations/layer12_attention_weight_min": -28.0455379486084,
      "activations/layer13_attention_weight_max": 39.111568450927734,
      "activations/layer13_attention_weight_min": -31.85985565185547,
      "activations/layer14_attention_weight_max": 52.12512969970703,
      "activations/layer14_attention_weight_min": -40.58829879760742,
      "activations/layer15_attention_weight_max": 36.1297721862793,
      "activations/layer15_attention_weight_min": -30.922443389892578,
      "activations/layer16_attention_weight_max": 32.84739303588867,
      "activations/layer16_attention_weight_min": -26.143043518066406,
      "activations/layer17_attention_weight_max": 34.88735580444336,
      "activations/layer17_attention_weight_min": -24.10479736328125,
      "activations/layer18_attention_weight_max": 36.16399383544922,
      "activations/layer18_attention_weight_min": -20.587688446044922,
      "activations/layer19_attention_weight_max": 34.824337005615234,
      "activations/layer19_attention_weight_min": -29.494001388549805,
      "activations/layer1_attention_weight_max": 16.66864585876465,
      "activations/layer1_attention_weight_min": -16.272253036499023,
      "activations/layer20_attention_weight_max": 30.710655212402344,
      "activations/layer20_attention_weight_min": -23.783733367919922,
      "activations/layer21_attention_weight_max": 29.30364990234375,
      "activations/layer21_attention_weight_min": -21.873369216918945,
      "activations/layer22_attention_weight_max": 46.1759033203125,
      "activations/layer22_attention_weight_min": -28.231483459472656,
      "activations/layer23_attention_weight_max": 36.457366943359375,
      "activations/layer23_attention_weight_min": -23.137052536010742,
      "activations/layer2_attention_weight_max": 31.937253952026367,
      "activations/layer2_attention_weight_min": -32.404754638671875,
      "activations/layer3_attention_weight_max": 90.7529067993164,
      "activations/layer3_attention_weight_min": -97.10594940185547,
      "activations/layer4_attention_weight_max": 126.79029846191406,
      "activations/layer4_attention_weight_min": -124.31078338623047,
      "activations/layer5_attention_weight_max": 67.45655059814453,
      "activations/layer5_attention_weight_min": -66.98065185546875,
      "activations/layer6_attention_weight_max": 53.49809646606445,
      "activations/layer6_attention_weight_min": -54.440731048583984,
      "activations/layer7_attention_weight_max": 91.24018859863281,
      "activations/layer7_attention_weight_min": -88.42949676513672,
      "activations/layer8_attention_weight_max": 40.507686614990234,
      "activations/layer8_attention_weight_min": -42.012657165527344,
      "activations/layer9_attention_weight_max": 33.874385833740234,
      "activations/layer9_attention_weight_min": -34.28470230102539,
      "epoch": 7.17,
      "learning_rate": 0.00010481439393939394,
      "loss": 2.7815,
      "step": 123400
    },
    {
      "activations/layer0_attention_weight_max": 15.75340461730957,
      "activations/layer0_attention_weight_min": -11.455816268920898,
      "activations/layer10_attention_weight_max": 36.983211517333984,
      "activations/layer10_attention_weight_min": -35.68372344970703,
      "activations/layer11_attention_weight_max": 36.02423858642578,
      "activations/layer11_attention_weight_min": -32.62650680541992,
      "activations/layer12_attention_weight_max": 41.512611389160156,
      "activations/layer12_attention_weight_min": -23.913022994995117,
      "activations/layer13_attention_weight_max": 42.53224182128906,
      "activations/layer13_attention_weight_min": -31.74626350402832,
      "activations/layer14_attention_weight_max": 58.1844596862793,
      "activations/layer14_attention_weight_min": -41.01918411254883,
      "activations/layer15_attention_weight_max": 42.575016021728516,
      "activations/layer15_attention_weight_min": -30.32894515991211,
      "activations/layer16_attention_weight_max": 37.632999420166016,
      "activations/layer16_attention_weight_min": -26.486648559570312,
      "activations/layer17_attention_weight_max": 37.21407699584961,
      "activations/layer17_attention_weight_min": -23.78875160217285,
      "activations/layer18_attention_weight_max": 35.29574203491211,
      "activations/layer18_attention_weight_min": -21.268951416015625,
      "activations/layer19_attention_weight_max": 40.518253326416016,
      "activations/layer19_attention_weight_min": -30.008779525756836,
      "activations/layer1_attention_weight_max": 16.460771560668945,
      "activations/layer1_attention_weight_min": -14.075681686401367,
      "activations/layer20_attention_weight_max": 34.76227569580078,
      "activations/layer20_attention_weight_min": -23.005634307861328,
      "activations/layer21_attention_weight_max": 37.05141830444336,
      "activations/layer21_attention_weight_min": -22.024150848388672,
      "activations/layer22_attention_weight_max": 53.156005859375,
      "activations/layer22_attention_weight_min": -26.1461181640625,
      "activations/layer23_attention_weight_max": 35.90024948120117,
      "activations/layer23_attention_weight_min": -22.859878540039062,
      "activations/layer2_attention_weight_max": 34.674400329589844,
      "activations/layer2_attention_weight_min": -31.222118377685547,
      "activations/layer3_attention_weight_max": 91.81990051269531,
      "activations/layer3_attention_weight_min": -92.07010650634766,
      "activations/layer4_attention_weight_max": 122.20503234863281,
      "activations/layer4_attention_weight_min": -120.57315826416016,
      "activations/layer5_attention_weight_max": 65.04000091552734,
      "activations/layer5_attention_weight_min": -67.08251953125,
      "activations/layer6_attention_weight_max": 51.68220138549805,
      "activations/layer6_attention_weight_min": -55.54094314575195,
      "activations/layer7_attention_weight_max": 86.70475006103516,
      "activations/layer7_attention_weight_min": -88.5147476196289,
      "activations/layer8_attention_weight_max": 40.74464416503906,
      "activations/layer8_attention_weight_min": -43.551612854003906,
      "activations/layer9_attention_weight_max": 37.49018478393555,
      "activations/layer9_attention_weight_min": -37.63608932495117,
      "epoch": 7.17,
      "learning_rate": 0.00010479545454545453,
      "loss": 2.7885,
      "step": 123450
    },
    {
      "activations/layer0_attention_weight_max": 15.008699417114258,
      "activations/layer0_attention_weight_min": -10.820282936096191,
      "activations/layer10_attention_weight_max": 33.0576286315918,
      "activations/layer10_attention_weight_min": -33.67998123168945,
      "activations/layer11_attention_weight_max": 32.78814697265625,
      "activations/layer11_attention_weight_min": -30.55769157409668,
      "activations/layer12_attention_weight_max": 26.154376983642578,
      "activations/layer12_attention_weight_min": -26.155256271362305,
      "activations/layer13_attention_weight_max": 37.724769592285156,
      "activations/layer13_attention_weight_min": -33.440025329589844,
      "activations/layer14_attention_weight_max": 53.10826873779297,
      "activations/layer14_attention_weight_min": -42.62520217895508,
      "activations/layer15_attention_weight_max": 36.64928436279297,
      "activations/layer15_attention_weight_min": -29.360258102416992,
      "activations/layer16_attention_weight_max": 33.1345100402832,
      "activations/layer16_attention_weight_min": -27.203887939453125,
      "activations/layer17_attention_weight_max": 35.45252990722656,
      "activations/layer17_attention_weight_min": -24.574777603149414,
      "activations/layer18_attention_weight_max": 31.606496810913086,
      "activations/layer18_attention_weight_min": -22.128042221069336,
      "activations/layer19_attention_weight_max": 36.943782806396484,
      "activations/layer19_attention_weight_min": -29.234960556030273,
      "activations/layer1_attention_weight_max": 16.28575325012207,
      "activations/layer1_attention_weight_min": -13.68896770477295,
      "activations/layer20_attention_weight_max": 31.43825912475586,
      "activations/layer20_attention_weight_min": -23.60154151916504,
      "activations/layer21_attention_weight_max": 28.976858139038086,
      "activations/layer21_attention_weight_min": -21.329330444335938,
      "activations/layer22_attention_weight_max": 54.165225982666016,
      "activations/layer22_attention_weight_min": -28.262638092041016,
      "activations/layer23_attention_weight_max": 36.047821044921875,
      "activations/layer23_attention_weight_min": -23.628944396972656,
      "activations/layer2_attention_weight_max": 33.792964935302734,
      "activations/layer2_attention_weight_min": -34.14066696166992,
      "activations/layer3_attention_weight_max": 93.30824279785156,
      "activations/layer3_attention_weight_min": -102.30282592773438,
      "activations/layer4_attention_weight_max": 124.82537841796875,
      "activations/layer4_attention_weight_min": -131.828857421875,
      "activations/layer5_attention_weight_max": 66.3129653930664,
      "activations/layer5_attention_weight_min": -66.74494171142578,
      "activations/layer6_attention_weight_max": 52.68600845336914,
      "activations/layer6_attention_weight_min": -57.17055130004883,
      "activations/layer7_attention_weight_max": 90.28716278076172,
      "activations/layer7_attention_weight_min": -90.92921447753906,
      "activations/layer8_attention_weight_max": 40.66053771972656,
      "activations/layer8_attention_weight_min": -41.64664840698242,
      "activations/layer9_attention_weight_max": 36.56112289428711,
      "activations/layer9_attention_weight_min": -35.6959342956543,
      "epoch": 7.18,
      "learning_rate": 0.00010477651515151514,
      "loss": 2.8077,
      "step": 123500
    },
    {
      "activations/layer0_attention_weight_max": 15.702549934387207,
      "activations/layer0_attention_weight_min": -11.032610893249512,
      "activations/layer10_attention_weight_max": 34.91374206542969,
      "activations/layer10_attention_weight_min": -33.3895149230957,
      "activations/layer11_attention_weight_max": 33.623504638671875,
      "activations/layer11_attention_weight_min": -30.158802032470703,
      "activations/layer12_attention_weight_max": 23.976375579833984,
      "activations/layer12_attention_weight_min": -27.56801986694336,
      "activations/layer13_attention_weight_max": 41.12551498413086,
      "activations/layer13_attention_weight_min": -34.84498596191406,
      "activations/layer14_attention_weight_max": 59.65922927856445,
      "activations/layer14_attention_weight_min": -40.87245178222656,
      "activations/layer15_attention_weight_max": 39.26700210571289,
      "activations/layer15_attention_weight_min": -30.5505313873291,
      "activations/layer16_attention_weight_max": 39.776798248291016,
      "activations/layer16_attention_weight_min": -27.587282180786133,
      "activations/layer17_attention_weight_max": 37.0578498840332,
      "activations/layer17_attention_weight_min": -25.609779357910156,
      "activations/layer18_attention_weight_max": 29.416898727416992,
      "activations/layer18_attention_weight_min": -20.589929580688477,
      "activations/layer19_attention_weight_max": 42.2611198425293,
      "activations/layer19_attention_weight_min": -30.0739803314209,
      "activations/layer1_attention_weight_max": 17.83048439025879,
      "activations/layer1_attention_weight_min": -17.006122589111328,
      "activations/layer20_attention_weight_max": 33.702178955078125,
      "activations/layer20_attention_weight_min": -24.799297332763672,
      "activations/layer21_attention_weight_max": 30.06572723388672,
      "activations/layer21_attention_weight_min": -22.28340721130371,
      "activations/layer22_attention_weight_max": 50.62200164794922,
      "activations/layer22_attention_weight_min": -27.63884162902832,
      "activations/layer23_attention_weight_max": 37.05793762207031,
      "activations/layer23_attention_weight_min": -21.46521759033203,
      "activations/layer2_attention_weight_max": 33.410362243652344,
      "activations/layer2_attention_weight_min": -31.373645782470703,
      "activations/layer3_attention_weight_max": 96.99345397949219,
      "activations/layer3_attention_weight_min": -96.94789123535156,
      "activations/layer4_attention_weight_max": 127.87934875488281,
      "activations/layer4_attention_weight_min": -124.14151763916016,
      "activations/layer5_attention_weight_max": 67.4742660522461,
      "activations/layer5_attention_weight_min": -64.11857604980469,
      "activations/layer6_attention_weight_max": 53.14924240112305,
      "activations/layer6_attention_weight_min": -51.374305725097656,
      "activations/layer7_attention_weight_max": 88.22978210449219,
      "activations/layer7_attention_weight_min": -90.06681060791016,
      "activations/layer8_attention_weight_max": 41.37800216674805,
      "activations/layer8_attention_weight_min": -41.2386589050293,
      "activations/layer9_attention_weight_max": 38.8077507019043,
      "activations/layer9_attention_weight_min": -34.979976654052734,
      "epoch": 7.18,
      "learning_rate": 0.00010475757575757574,
      "loss": 2.7966,
      "step": 123550
    },
    {
      "activations/layer0_attention_weight_max": 15.459975242614746,
      "activations/layer0_attention_weight_min": -11.077508926391602,
      "activations/layer10_attention_weight_max": 37.473304748535156,
      "activations/layer10_attention_weight_min": -35.641502380371094,
      "activations/layer11_attention_weight_max": 38.357582092285156,
      "activations/layer11_attention_weight_min": -33.00713348388672,
      "activations/layer12_attention_weight_max": 35.24577331542969,
      "activations/layer12_attention_weight_min": -26.85103988647461,
      "activations/layer13_attention_weight_max": 45.65231704711914,
      "activations/layer13_attention_weight_min": -34.299476623535156,
      "activations/layer14_attention_weight_max": 58.65581130981445,
      "activations/layer14_attention_weight_min": -45.06700134277344,
      "activations/layer15_attention_weight_max": 40.958091735839844,
      "activations/layer15_attention_weight_min": -30.209171295166016,
      "activations/layer16_attention_weight_max": 38.00415802001953,
      "activations/layer16_attention_weight_min": -26.16594123840332,
      "activations/layer17_attention_weight_max": 40.134132385253906,
      "activations/layer17_attention_weight_min": -25.592588424682617,
      "activations/layer18_attention_weight_max": 35.88018035888672,
      "activations/layer18_attention_weight_min": -20.96858787536621,
      "activations/layer19_attention_weight_max": 41.05263900756836,
      "activations/layer19_attention_weight_min": -29.15950584411621,
      "activations/layer1_attention_weight_max": 17.755775451660156,
      "activations/layer1_attention_weight_min": -13.787391662597656,
      "activations/layer20_attention_weight_max": 36.07704544067383,
      "activations/layer20_attention_weight_min": -24.566238403320312,
      "activations/layer21_attention_weight_max": 35.00520706176758,
      "activations/layer21_attention_weight_min": -21.418710708618164,
      "activations/layer22_attention_weight_max": 54.56060028076172,
      "activations/layer22_attention_weight_min": -28.40387725830078,
      "activations/layer23_attention_weight_max": 37.32843780517578,
      "activations/layer23_attention_weight_min": -21.538311004638672,
      "activations/layer2_attention_weight_max": 32.143775939941406,
      "activations/layer2_attention_weight_min": -32.17163848876953,
      "activations/layer3_attention_weight_max": 96.85200500488281,
      "activations/layer3_attention_weight_min": -97.54436492919922,
      "activations/layer4_attention_weight_max": 132.5644989013672,
      "activations/layer4_attention_weight_min": -126.56123352050781,
      "activations/layer5_attention_weight_max": 65.59905242919922,
      "activations/layer5_attention_weight_min": -65.59602355957031,
      "activations/layer6_attention_weight_max": 51.18042755126953,
      "activations/layer6_attention_weight_min": -54.37326431274414,
      "activations/layer7_attention_weight_max": 90.54988098144531,
      "activations/layer7_attention_weight_min": -89.96102905273438,
      "activations/layer8_attention_weight_max": 42.851539611816406,
      "activations/layer8_attention_weight_min": -42.14826583862305,
      "activations/layer9_attention_weight_max": 41.308780670166016,
      "activations/layer9_attention_weight_min": -36.1781005859375,
      "epoch": 7.18,
      "learning_rate": 0.00010473863636363635,
      "loss": 2.7971,
      "step": 123600
    },
    {
      "activations/layer0_attention_weight_max": 15.769994735717773,
      "activations/layer0_attention_weight_min": -11.263267517089844,
      "activations/layer10_attention_weight_max": 31.444982528686523,
      "activations/layer10_attention_weight_min": -31.96194839477539,
      "activations/layer11_attention_weight_max": 33.274505615234375,
      "activations/layer11_attention_weight_min": -30.22744369506836,
      "activations/layer12_attention_weight_max": 24.31344223022461,
      "activations/layer12_attention_weight_min": -29.638490676879883,
      "activations/layer13_attention_weight_max": 38.44450759887695,
      "activations/layer13_attention_weight_min": -31.28105354309082,
      "activations/layer14_attention_weight_max": 60.16410446166992,
      "activations/layer14_attention_weight_min": -51.06081008911133,
      "activations/layer15_attention_weight_max": 36.130245208740234,
      "activations/layer15_attention_weight_min": -29.05668067932129,
      "activations/layer16_attention_weight_max": 33.00485610961914,
      "activations/layer16_attention_weight_min": -27.814071655273438,
      "activations/layer17_attention_weight_max": 35.353721618652344,
      "activations/layer17_attention_weight_min": -25.334461212158203,
      "activations/layer18_attention_weight_max": 30.743961334228516,
      "activations/layer18_attention_weight_min": -21.5153865814209,
      "activations/layer19_attention_weight_max": 38.843807220458984,
      "activations/layer19_attention_weight_min": -31.003671646118164,
      "activations/layer1_attention_weight_max": 16.447851181030273,
      "activations/layer1_attention_weight_min": -12.970394134521484,
      "activations/layer20_attention_weight_max": 32.13174057006836,
      "activations/layer20_attention_weight_min": -23.27978515625,
      "activations/layer21_attention_weight_max": 31.641815185546875,
      "activations/layer21_attention_weight_min": -23.072851181030273,
      "activations/layer22_attention_weight_max": 53.748477935791016,
      "activations/layer22_attention_weight_min": -27.514537811279297,
      "activations/layer23_attention_weight_max": 39.24448776245117,
      "activations/layer23_attention_weight_min": -22.43842887878418,
      "activations/layer2_attention_weight_max": 31.234439849853516,
      "activations/layer2_attention_weight_min": -32.334205627441406,
      "activations/layer3_attention_weight_max": 90.93608856201172,
      "activations/layer3_attention_weight_min": -96.70176696777344,
      "activations/layer4_attention_weight_max": 123.27603912353516,
      "activations/layer4_attention_weight_min": -130.11915588378906,
      "activations/layer5_attention_weight_max": 62.80660629272461,
      "activations/layer5_attention_weight_min": -63.60538864135742,
      "activations/layer6_attention_weight_max": 47.954078674316406,
      "activations/layer6_attention_weight_min": -54.58536148071289,
      "activations/layer7_attention_weight_max": 90.14662170410156,
      "activations/layer7_attention_weight_min": -99.60009765625,
      "activations/layer8_attention_weight_max": 39.31452941894531,
      "activations/layer8_attention_weight_min": -40.84889602661133,
      "activations/layer9_attention_weight_max": 33.81769561767578,
      "activations/layer9_attention_weight_min": -33.412696838378906,
      "epoch": 7.18,
      "learning_rate": 0.00010471969696969696,
      "loss": 2.8027,
      "step": 123650
    },
    {
      "activations/layer0_attention_weight_max": 15.865466117858887,
      "activations/layer0_attention_weight_min": -10.617526054382324,
      "activations/layer10_attention_weight_max": 35.0528564453125,
      "activations/layer10_attention_weight_min": -33.105892181396484,
      "activations/layer11_attention_weight_max": 31.61806297302246,
      "activations/layer11_attention_weight_min": -30.774078369140625,
      "activations/layer12_attention_weight_max": 21.647720336914062,
      "activations/layer12_attention_weight_min": -23.491092681884766,
      "activations/layer13_attention_weight_max": 40.08425521850586,
      "activations/layer13_attention_weight_min": -32.902549743652344,
      "activations/layer14_attention_weight_max": 50.40531539916992,
      "activations/layer14_attention_weight_min": -40.56093215942383,
      "activations/layer15_attention_weight_max": 38.4533805847168,
      "activations/layer15_attention_weight_min": -30.27971649169922,
      "activations/layer16_attention_weight_max": 33.8210334777832,
      "activations/layer16_attention_weight_min": -28.719528198242188,
      "activations/layer17_attention_weight_max": 36.375267028808594,
      "activations/layer17_attention_weight_min": -23.949241638183594,
      "activations/layer18_attention_weight_max": 34.82160186767578,
      "activations/layer18_attention_weight_min": -21.566499710083008,
      "activations/layer19_attention_weight_max": 39.25722885131836,
      "activations/layer19_attention_weight_min": -29.160146713256836,
      "activations/layer1_attention_weight_max": 17.028703689575195,
      "activations/layer1_attention_weight_min": -13.199170112609863,
      "activations/layer20_attention_weight_max": 31.646656036376953,
      "activations/layer20_attention_weight_min": -24.564884185791016,
      "activations/layer21_attention_weight_max": 32.03786087036133,
      "activations/layer21_attention_weight_min": -22.281585693359375,
      "activations/layer22_attention_weight_max": 47.64268493652344,
      "activations/layer22_attention_weight_min": -26.58250617980957,
      "activations/layer23_attention_weight_max": 36.89701461791992,
      "activations/layer23_attention_weight_min": -22.497398376464844,
      "activations/layer2_attention_weight_max": 30.12073516845703,
      "activations/layer2_attention_weight_min": -31.502708435058594,
      "activations/layer3_attention_weight_max": 91.41899108886719,
      "activations/layer3_attention_weight_min": -93.44456481933594,
      "activations/layer4_attention_weight_max": 125.87413787841797,
      "activations/layer4_attention_weight_min": -121.05909729003906,
      "activations/layer5_attention_weight_max": 65.74620819091797,
      "activations/layer5_attention_weight_min": -64.65055084228516,
      "activations/layer6_attention_weight_max": 52.85930633544922,
      "activations/layer6_attention_weight_min": -51.826133728027344,
      "activations/layer7_attention_weight_max": 86.44330596923828,
      "activations/layer7_attention_weight_min": -87.85289764404297,
      "activations/layer8_attention_weight_max": 41.30366897583008,
      "activations/layer8_attention_weight_min": -39.89420700073242,
      "activations/layer9_attention_weight_max": 35.68111801147461,
      "activations/layer9_attention_weight_min": -34.72536849975586,
      "epoch": 7.19,
      "learning_rate": 0.00010470075757575756,
      "loss": 2.8091,
      "step": 123700
    },
    {
      "activations/layer0_attention_weight_max": 15.069350242614746,
      "activations/layer0_attention_weight_min": -11.12306022644043,
      "activations/layer10_attention_weight_max": 39.93544387817383,
      "activations/layer10_attention_weight_min": -33.52338409423828,
      "activations/layer11_attention_weight_max": 44.2082633972168,
      "activations/layer11_attention_weight_min": -33.73151397705078,
      "activations/layer12_attention_weight_max": 27.332374572753906,
      "activations/layer12_attention_weight_min": -27.04535484313965,
      "activations/layer13_attention_weight_max": 41.59815216064453,
      "activations/layer13_attention_weight_min": -34.1535758972168,
      "activations/layer14_attention_weight_max": 57.15949630737305,
      "activations/layer14_attention_weight_min": -40.92167663574219,
      "activations/layer15_attention_weight_max": 40.11130142211914,
      "activations/layer15_attention_weight_min": -30.05451202392578,
      "activations/layer16_attention_weight_max": 38.53334045410156,
      "activations/layer16_attention_weight_min": -28.291337966918945,
      "activations/layer17_attention_weight_max": 35.42717742919922,
      "activations/layer17_attention_weight_min": -25.51154136657715,
      "activations/layer18_attention_weight_max": 34.78731918334961,
      "activations/layer18_attention_weight_min": -21.448450088500977,
      "activations/layer19_attention_weight_max": 42.087913513183594,
      "activations/layer19_attention_weight_min": -33.56486129760742,
      "activations/layer1_attention_weight_max": 16.410106658935547,
      "activations/layer1_attention_weight_min": -14.10913372039795,
      "activations/layer20_attention_weight_max": 37.84306716918945,
      "activations/layer20_attention_weight_min": -24.504154205322266,
      "activations/layer21_attention_weight_max": 34.5674934387207,
      "activations/layer21_attention_weight_min": -22.735849380493164,
      "activations/layer22_attention_weight_max": 50.97922897338867,
      "activations/layer22_attention_weight_min": -27.71619987487793,
      "activations/layer23_attention_weight_max": 38.737972259521484,
      "activations/layer23_attention_weight_min": -21.061832427978516,
      "activations/layer2_attention_weight_max": 32.31354904174805,
      "activations/layer2_attention_weight_min": -32.95833969116211,
      "activations/layer3_attention_weight_max": 97.52581024169922,
      "activations/layer3_attention_weight_min": -100.65068817138672,
      "activations/layer4_attention_weight_max": 128.90444946289062,
      "activations/layer4_attention_weight_min": -125.4796371459961,
      "activations/layer5_attention_weight_max": 67.05535888671875,
      "activations/layer5_attention_weight_min": -64.71321105957031,
      "activations/layer6_attention_weight_max": 52.941139221191406,
      "activations/layer6_attention_weight_min": -54.50520324707031,
      "activations/layer7_attention_weight_max": 93.32147979736328,
      "activations/layer7_attention_weight_min": -93.55949401855469,
      "activations/layer8_attention_weight_max": 42.187408447265625,
      "activations/layer8_attention_weight_min": -42.70903778076172,
      "activations/layer9_attention_weight_max": 37.71537780761719,
      "activations/layer9_attention_weight_min": -36.945533752441406,
      "epoch": 7.19,
      "learning_rate": 0.00010468181818181818,
      "loss": 2.7972,
      "step": 123750
    },
    {
      "activations/layer0_attention_weight_max": 15.394330978393555,
      "activations/layer0_attention_weight_min": -11.132258415222168,
      "activations/layer10_attention_weight_max": 34.83168411254883,
      "activations/layer10_attention_weight_min": -31.348209381103516,
      "activations/layer11_attention_weight_max": 34.1141357421875,
      "activations/layer11_attention_weight_min": -30.951435089111328,
      "activations/layer12_attention_weight_max": 24.20459747314453,
      "activations/layer12_attention_weight_min": -24.103389739990234,
      "activations/layer13_attention_weight_max": 42.633453369140625,
      "activations/layer13_attention_weight_min": -34.37622833251953,
      "activations/layer14_attention_weight_max": 54.01755142211914,
      "activations/layer14_attention_weight_min": -40.311641693115234,
      "activations/layer15_attention_weight_max": 39.0523567199707,
      "activations/layer15_attention_weight_min": -29.467060089111328,
      "activations/layer16_attention_weight_max": 35.60104751586914,
      "activations/layer16_attention_weight_min": -28.071043014526367,
      "activations/layer17_attention_weight_max": 36.0804557800293,
      "activations/layer17_attention_weight_min": -25.311473846435547,
      "activations/layer18_attention_weight_max": 33.95831298828125,
      "activations/layer18_attention_weight_min": -22.372684478759766,
      "activations/layer19_attention_weight_max": 41.772212982177734,
      "activations/layer19_attention_weight_min": -28.00806999206543,
      "activations/layer1_attention_weight_max": 16.512752532958984,
      "activations/layer1_attention_weight_min": -14.613414764404297,
      "activations/layer20_attention_weight_max": 29.796178817749023,
      "activations/layer20_attention_weight_min": -24.101152420043945,
      "activations/layer21_attention_weight_max": 32.23601150512695,
      "activations/layer21_attention_weight_min": -21.14060401916504,
      "activations/layer22_attention_weight_max": 51.979583740234375,
      "activations/layer22_attention_weight_min": -27.77239990234375,
      "activations/layer23_attention_weight_max": 40.38806915283203,
      "activations/layer23_attention_weight_min": -22.907623291015625,
      "activations/layer2_attention_weight_max": 32.6171989440918,
      "activations/layer2_attention_weight_min": -34.18022537231445,
      "activations/layer3_attention_weight_max": 96.24828338623047,
      "activations/layer3_attention_weight_min": -99.85581970214844,
      "activations/layer4_attention_weight_max": 130.49876403808594,
      "activations/layer4_attention_weight_min": -125.31806945800781,
      "activations/layer5_attention_weight_max": 64.0638427734375,
      "activations/layer5_attention_weight_min": -63.0006103515625,
      "activations/layer6_attention_weight_max": 52.886024475097656,
      "activations/layer6_attention_weight_min": -52.104854583740234,
      "activations/layer7_attention_weight_max": 87.5055160522461,
      "activations/layer7_attention_weight_min": -85.6136474609375,
      "activations/layer8_attention_weight_max": 41.34005355834961,
      "activations/layer8_attention_weight_min": -41.9807243347168,
      "activations/layer9_attention_weight_max": 36.19175720214844,
      "activations/layer9_attention_weight_min": -34.30195617675781,
      "epoch": 7.19,
      "learning_rate": 0.00010466287878787878,
      "loss": 2.8064,
      "step": 123800
    },
    {
      "activations/layer0_attention_weight_max": 15.88195514678955,
      "activations/layer0_attention_weight_min": -10.956130027770996,
      "activations/layer10_attention_weight_max": 31.59153938293457,
      "activations/layer10_attention_weight_min": -29.48354721069336,
      "activations/layer11_attention_weight_max": 31.19801139831543,
      "activations/layer11_attention_weight_min": -27.02920913696289,
      "activations/layer12_attention_weight_max": 22.605220794677734,
      "activations/layer12_attention_weight_min": -28.684654235839844,
      "activations/layer13_attention_weight_max": 40.248104095458984,
      "activations/layer13_attention_weight_min": -33.629669189453125,
      "activations/layer14_attention_weight_max": 47.84173583984375,
      "activations/layer14_attention_weight_min": -41.062461853027344,
      "activations/layer15_attention_weight_max": 34.44249725341797,
      "activations/layer15_attention_weight_min": -27.89522361755371,
      "activations/layer16_attention_weight_max": 31.345653533935547,
      "activations/layer16_attention_weight_min": -27.322938919067383,
      "activations/layer17_attention_weight_max": 30.42319107055664,
      "activations/layer17_attention_weight_min": -23.394596099853516,
      "activations/layer18_attention_weight_max": 33.53605651855469,
      "activations/layer18_attention_weight_min": -20.666473388671875,
      "activations/layer19_attention_weight_max": 34.806182861328125,
      "activations/layer19_attention_weight_min": -29.026676177978516,
      "activations/layer1_attention_weight_max": 16.163225173950195,
      "activations/layer1_attention_weight_min": -13.875025749206543,
      "activations/layer20_attention_weight_max": 26.9440860748291,
      "activations/layer20_attention_weight_min": -21.927579879760742,
      "activations/layer21_attention_weight_max": 36.77552032470703,
      "activations/layer21_attention_weight_min": -21.088245391845703,
      "activations/layer22_attention_weight_max": 46.88868713378906,
      "activations/layer22_attention_weight_min": -28.607311248779297,
      "activations/layer23_attention_weight_max": 32.385589599609375,
      "activations/layer23_attention_weight_min": -23.6899471282959,
      "activations/layer2_attention_weight_max": 31.81025505065918,
      "activations/layer2_attention_weight_min": -31.14057159423828,
      "activations/layer3_attention_weight_max": 95.18477630615234,
      "activations/layer3_attention_weight_min": -92.3657455444336,
      "activations/layer4_attention_weight_max": 130.20982360839844,
      "activations/layer4_attention_weight_min": -121.78606414794922,
      "activations/layer5_attention_weight_max": 66.43817138671875,
      "activations/layer5_attention_weight_min": -61.44338607788086,
      "activations/layer6_attention_weight_max": 52.79367446899414,
      "activations/layer6_attention_weight_min": -52.04594802856445,
      "activations/layer7_attention_weight_max": 88.06126403808594,
      "activations/layer7_attention_weight_min": -84.77875518798828,
      "activations/layer8_attention_weight_max": 39.529273986816406,
      "activations/layer8_attention_weight_min": -41.47078323364258,
      "activations/layer9_attention_weight_max": 34.4753532409668,
      "activations/layer9_attention_weight_min": -33.443824768066406,
      "epoch": 7.2,
      "learning_rate": 0.00010464393939393938,
      "loss": 2.8155,
      "step": 123850
    },
    {
      "activations/layer0_attention_weight_max": 15.129316329956055,
      "activations/layer0_attention_weight_min": -11.18056583404541,
      "activations/layer10_attention_weight_max": 33.03419876098633,
      "activations/layer10_attention_weight_min": -31.6328125,
      "activations/layer11_attention_weight_max": 34.73158264160156,
      "activations/layer11_attention_weight_min": -28.9041748046875,
      "activations/layer12_attention_weight_max": 22.29877281188965,
      "activations/layer12_attention_weight_min": -24.085060119628906,
      "activations/layer13_attention_weight_max": 39.897315979003906,
      "activations/layer13_attention_weight_min": -32.39216613769531,
      "activations/layer14_attention_weight_max": 50.69755554199219,
      "activations/layer14_attention_weight_min": -38.32393264770508,
      "activations/layer15_attention_weight_max": 38.45222091674805,
      "activations/layer15_attention_weight_min": -29.665979385375977,
      "activations/layer16_attention_weight_max": 33.66023635864258,
      "activations/layer16_attention_weight_min": -27.85849380493164,
      "activations/layer17_attention_weight_max": 35.19255447387695,
      "activations/layer17_attention_weight_min": -25.59324073791504,
      "activations/layer18_attention_weight_max": 36.395050048828125,
      "activations/layer18_attention_weight_min": -24.433420181274414,
      "activations/layer19_attention_weight_max": 35.85514450073242,
      "activations/layer19_attention_weight_min": -30.188129425048828,
      "activations/layer1_attention_weight_max": 16.162113189697266,
      "activations/layer1_attention_weight_min": -13.950860977172852,
      "activations/layer20_attention_weight_max": 29.27141571044922,
      "activations/layer20_attention_weight_min": -23.743059158325195,
      "activations/layer21_attention_weight_max": 32.10081481933594,
      "activations/layer21_attention_weight_min": -22.422264099121094,
      "activations/layer22_attention_weight_max": 46.28675842285156,
      "activations/layer22_attention_weight_min": -27.82059097290039,
      "activations/layer23_attention_weight_max": 36.5665168762207,
      "activations/layer23_attention_weight_min": -23.113506317138672,
      "activations/layer2_attention_weight_max": 32.60017395019531,
      "activations/layer2_attention_weight_min": -31.629552841186523,
      "activations/layer3_attention_weight_max": 96.3814926147461,
      "activations/layer3_attention_weight_min": -94.72393035888672,
      "activations/layer4_attention_weight_max": 127.22148895263672,
      "activations/layer4_attention_weight_min": -125.83646392822266,
      "activations/layer5_attention_weight_max": 65.71451568603516,
      "activations/layer5_attention_weight_min": -64.3485107421875,
      "activations/layer6_attention_weight_max": 52.094703674316406,
      "activations/layer6_attention_weight_min": -57.279991149902344,
      "activations/layer7_attention_weight_max": 88.9751968383789,
      "activations/layer7_attention_weight_min": -89.14605712890625,
      "activations/layer8_attention_weight_max": 41.41623306274414,
      "activations/layer8_attention_weight_min": -41.085880279541016,
      "activations/layer9_attention_weight_max": 35.40493392944336,
      "activations/layer9_attention_weight_min": -34.88238525390625,
      "epoch": 7.2,
      "learning_rate": 0.000104625,
      "loss": 2.8055,
      "step": 123900
    },
    {
      "activations/layer0_attention_weight_max": 15.912898063659668,
      "activations/layer0_attention_weight_min": -11.404149055480957,
      "activations/layer10_attention_weight_max": 34.627506256103516,
      "activations/layer10_attention_weight_min": -34.91718673706055,
      "activations/layer11_attention_weight_max": 32.37259292602539,
      "activations/layer11_attention_weight_min": -31.81991195678711,
      "activations/layer12_attention_weight_max": 30.411212921142578,
      "activations/layer12_attention_weight_min": -28.492895126342773,
      "activations/layer13_attention_weight_max": 51.38913345336914,
      "activations/layer13_attention_weight_min": -41.85845947265625,
      "activations/layer14_attention_weight_max": 62.93036651611328,
      "activations/layer14_attention_weight_min": -46.73005294799805,
      "activations/layer15_attention_weight_max": 46.88350296020508,
      "activations/layer15_attention_weight_min": -36.65118408203125,
      "activations/layer16_attention_weight_max": 37.28600311279297,
      "activations/layer16_attention_weight_min": -30.19995880126953,
      "activations/layer17_attention_weight_max": 34.81822204589844,
      "activations/layer17_attention_weight_min": -27.670289993286133,
      "activations/layer18_attention_weight_max": 31.606298446655273,
      "activations/layer18_attention_weight_min": -21.49744987487793,
      "activations/layer19_attention_weight_max": 41.57950973510742,
      "activations/layer19_attention_weight_min": -30.870901107788086,
      "activations/layer1_attention_weight_max": 17.852924346923828,
      "activations/layer1_attention_weight_min": -16.79120445251465,
      "activations/layer20_attention_weight_max": 31.108226776123047,
      "activations/layer20_attention_weight_min": -25.096660614013672,
      "activations/layer21_attention_weight_max": 31.36469078063965,
      "activations/layer21_attention_weight_min": -21.08785629272461,
      "activations/layer22_attention_weight_max": 52.00309371948242,
      "activations/layer22_attention_weight_min": -28.63273048400879,
      "activations/layer23_attention_weight_max": 35.19446563720703,
      "activations/layer23_attention_weight_min": -24.612911224365234,
      "activations/layer2_attention_weight_max": 32.704322814941406,
      "activations/layer2_attention_weight_min": -32.590457916259766,
      "activations/layer3_attention_weight_max": 100.77839660644531,
      "activations/layer3_attention_weight_min": -97.60716247558594,
      "activations/layer4_attention_weight_max": 128.3037567138672,
      "activations/layer4_attention_weight_min": -124.6451416015625,
      "activations/layer5_attention_weight_max": 64.10916137695312,
      "activations/layer5_attention_weight_min": -64.10330963134766,
      "activations/layer6_attention_weight_max": 51.77012252807617,
      "activations/layer6_attention_weight_min": -56.559696197509766,
      "activations/layer7_attention_weight_max": 87.91847229003906,
      "activations/layer7_attention_weight_min": -95.63893127441406,
      "activations/layer8_attention_weight_max": 41.837135314941406,
      "activations/layer8_attention_weight_min": -43.05275344848633,
      "activations/layer9_attention_weight_max": 37.144020080566406,
      "activations/layer9_attention_weight_min": -35.928489685058594,
      "epoch": 7.2,
      "learning_rate": 0.00010460606060606061,
      "loss": 2.8095,
      "step": 123950
    },
    {
      "activations/layer0_attention_weight_max": 15.499341011047363,
      "activations/layer0_attention_weight_min": -11.047043800354004,
      "activations/layer10_attention_weight_max": 33.56965255737305,
      "activations/layer10_attention_weight_min": -31.556325912475586,
      "activations/layer11_attention_weight_max": 32.98847961425781,
      "activations/layer11_attention_weight_min": -30.681503295898438,
      "activations/layer12_attention_weight_max": 34.18781280517578,
      "activations/layer12_attention_weight_min": -24.54694366455078,
      "activations/layer13_attention_weight_max": 36.46293258666992,
      "activations/layer13_attention_weight_min": -32.304237365722656,
      "activations/layer14_attention_weight_max": 52.396934509277344,
      "activations/layer14_attention_weight_min": -42.316078186035156,
      "activations/layer15_attention_weight_max": 34.93742370605469,
      "activations/layer15_attention_weight_min": -29.390287399291992,
      "activations/layer16_attention_weight_max": 32.89271926879883,
      "activations/layer16_attention_weight_min": -25.673072814941406,
      "activations/layer17_attention_weight_max": 37.0843391418457,
      "activations/layer17_attention_weight_min": -24.494640350341797,
      "activations/layer18_attention_weight_max": 31.200031280517578,
      "activations/layer18_attention_weight_min": -20.778865814208984,
      "activations/layer19_attention_weight_max": 37.909095764160156,
      "activations/layer19_attention_weight_min": -31.213178634643555,
      "activations/layer1_attention_weight_max": 18.31234359741211,
      "activations/layer1_attention_weight_min": -13.852323532104492,
      "activations/layer20_attention_weight_max": 28.9228458404541,
      "activations/layer20_attention_weight_min": -24.672849655151367,
      "activations/layer21_attention_weight_max": 27.96602439880371,
      "activations/layer21_attention_weight_min": -22.479055404663086,
      "activations/layer22_attention_weight_max": 43.76644515991211,
      "activations/layer22_attention_weight_min": -29.17958641052246,
      "activations/layer23_attention_weight_max": 33.14827346801758,
      "activations/layer23_attention_weight_min": -23.062236785888672,
      "activations/layer2_attention_weight_max": 35.04111862182617,
      "activations/layer2_attention_weight_min": -32.91230392456055,
      "activations/layer3_attention_weight_max": 93.67109680175781,
      "activations/layer3_attention_weight_min": -93.01213073730469,
      "activations/layer4_attention_weight_max": 127.82362365722656,
      "activations/layer4_attention_weight_min": -122.7748794555664,
      "activations/layer5_attention_weight_max": 68.9734115600586,
      "activations/layer5_attention_weight_min": -64.25117492675781,
      "activations/layer6_attention_weight_max": 52.46377944946289,
      "activations/layer6_attention_weight_min": -56.59185028076172,
      "activations/layer7_attention_weight_max": 86.47437286376953,
      "activations/layer7_attention_weight_min": -89.03646850585938,
      "activations/layer8_attention_weight_max": 39.48532485961914,
      "activations/layer8_attention_weight_min": -40.15207290649414,
      "activations/layer9_attention_weight_max": 37.137874603271484,
      "activations/layer9_attention_weight_min": -34.39084243774414,
      "epoch": 7.21,
      "learning_rate": 0.00010458787878787878,
      "loss": 2.8029,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.8761,
      "eval_samples_per_second": 483.771,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.8761,
      "eval_openwebtext_samples_per_second": 483.771,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 1.9997,
      "eval_wikitext_samples_per_second": 228.03,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_lambada_loss": 2.5859375,
      "eval_lambada_ppl": 13.275729253905588,
      "eval_lambada_runtime": 9.6195,
      "eval_lambada_samples_per_second": 506.159,
      "step": 124000
    },
    {
      "activations/layer0_attention_weight_max": 15.546565055847168,
      "activations/layer0_attention_weight_min": -11.042384147644043,
      "activations/layer10_attention_weight_max": 35.10692596435547,
      "activations/layer10_attention_weight_min": -33.5263786315918,
      "activations/layer11_attention_weight_max": 34.091922760009766,
      "activations/layer11_attention_weight_min": -29.23967742919922,
      "activations/layer12_attention_weight_max": 24.52281379699707,
      "activations/layer12_attention_weight_min": -26.053598403930664,
      "activations/layer13_attention_weight_max": 41.12491989135742,
      "activations/layer13_attention_weight_min": -30.761844635009766,
      "activations/layer14_attention_weight_max": 54.242618560791016,
      "activations/layer14_attention_weight_min": -38.3257942199707,
      "activations/layer15_attention_weight_max": 36.19873809814453,
      "activations/layer15_attention_weight_min": -29.938127517700195,
      "activations/layer16_attention_weight_max": 34.94585418701172,
      "activations/layer16_attention_weight_min": -27.707958221435547,
      "activations/layer17_attention_weight_max": 36.14221954345703,
      "activations/layer17_attention_weight_min": -24.822174072265625,
      "activations/layer18_attention_weight_max": 32.73005294799805,
      "activations/layer18_attention_weight_min": -21.982126235961914,
      "activations/layer19_attention_weight_max": 35.85913848876953,
      "activations/layer19_attention_weight_min": -29.54486083984375,
      "activations/layer1_attention_weight_max": 17.04045867919922,
      "activations/layer1_attention_weight_min": -13.630807876586914,
      "activations/layer20_attention_weight_max": 31.284311294555664,
      "activations/layer20_attention_weight_min": -23.998138427734375,
      "activations/layer21_attention_weight_max": 29.00325584411621,
      "activations/layer21_attention_weight_min": -21.492467880249023,
      "activations/layer22_attention_weight_max": 50.58805465698242,
      "activations/layer22_attention_weight_min": -30.23992919921875,
      "activations/layer23_attention_weight_max": 34.85796356201172,
      "activations/layer23_attention_weight_min": -22.02752685546875,
      "activations/layer2_attention_weight_max": 33.10057067871094,
      "activations/layer2_attention_weight_min": -31.311870574951172,
      "activations/layer3_attention_weight_max": 94.04454803466797,
      "activations/layer3_attention_weight_min": -94.85660552978516,
      "activations/layer4_attention_weight_max": 125.49056243896484,
      "activations/layer4_attention_weight_min": -121.78731536865234,
      "activations/layer5_attention_weight_max": 65.99757385253906,
      "activations/layer5_attention_weight_min": -63.0372314453125,
      "activations/layer6_attention_weight_max": 51.37839126586914,
      "activations/layer6_attention_weight_min": -51.85585021972656,
      "activations/layer7_attention_weight_max": 82.22262573242188,
      "activations/layer7_attention_weight_min": -86.1313705444336,
      "activations/layer8_attention_weight_max": 39.07659912109375,
      "activations/layer8_attention_weight_min": -41.71598815917969,
      "activations/layer9_attention_weight_max": 37.45328140258789,
      "activations/layer9_attention_weight_min": -36.0859260559082,
      "epoch": 7.21,
      "learning_rate": 0.00010456893939393939,
      "loss": 2.8132,
      "step": 124050
    },
    {
      "activations/layer0_attention_weight_max": 16.615873336791992,
      "activations/layer0_attention_weight_min": -10.718400955200195,
      "activations/layer10_attention_weight_max": 30.33879280090332,
      "activations/layer10_attention_weight_min": -31.164398193359375,
      "activations/layer11_attention_weight_max": 29.09803009033203,
      "activations/layer11_attention_weight_min": -27.866527557373047,
      "activations/layer12_attention_weight_max": 25.122453689575195,
      "activations/layer12_attention_weight_min": -26.83408546447754,
      "activations/layer13_attention_weight_max": 47.29845428466797,
      "activations/layer13_attention_weight_min": -33.46842575073242,
      "activations/layer14_attention_weight_max": 56.708763122558594,
      "activations/layer14_attention_weight_min": -43.21449661254883,
      "activations/layer15_attention_weight_max": 41.220455169677734,
      "activations/layer15_attention_weight_min": -29.12763023376465,
      "activations/layer16_attention_weight_max": 33.23380661010742,
      "activations/layer16_attention_weight_min": -27.85164451599121,
      "activations/layer17_attention_weight_max": 42.664859771728516,
      "activations/layer17_attention_weight_min": -24.741090774536133,
      "activations/layer18_attention_weight_max": 31.605709075927734,
      "activations/layer18_attention_weight_min": -20.0029354095459,
      "activations/layer19_attention_weight_max": 40.4326286315918,
      "activations/layer19_attention_weight_min": -31.2289981842041,
      "activations/layer1_attention_weight_max": 16.682788848876953,
      "activations/layer1_attention_weight_min": -12.918825149536133,
      "activations/layer20_attention_weight_max": 34.20500564575195,
      "activations/layer20_attention_weight_min": -24.297327041625977,
      "activations/layer21_attention_weight_max": 36.57617950439453,
      "activations/layer21_attention_weight_min": -21.429372787475586,
      "activations/layer22_attention_weight_max": 51.444278717041016,
      "activations/layer22_attention_weight_min": -28.13343620300293,
      "activations/layer23_attention_weight_max": 33.187747955322266,
      "activations/layer23_attention_weight_min": -23.08319091796875,
      "activations/layer2_attention_weight_max": 31.20827293395996,
      "activations/layer2_attention_weight_min": -31.279659271240234,
      "activations/layer3_attention_weight_max": 91.029052734375,
      "activations/layer3_attention_weight_min": -93.28467559814453,
      "activations/layer4_attention_weight_max": 121.10359191894531,
      "activations/layer4_attention_weight_min": -116.0185546875,
      "activations/layer5_attention_weight_max": 62.9260368347168,
      "activations/layer5_attention_weight_min": -61.44799041748047,
      "activations/layer6_attention_weight_max": 51.465065002441406,
      "activations/layer6_attention_weight_min": -52.45820617675781,
      "activations/layer7_attention_weight_max": 88.56275177001953,
      "activations/layer7_attention_weight_min": -84.77857208251953,
      "activations/layer8_attention_weight_max": 37.729251861572266,
      "activations/layer8_attention_weight_min": -40.98965072631836,
      "activations/layer9_attention_weight_max": 33.5843620300293,
      "activations/layer9_attention_weight_min": -32.59128952026367,
      "epoch": 7.21,
      "learning_rate": 0.00010454999999999998,
      "loss": 2.8003,
      "step": 124100
    },
    {
      "activations/layer0_attention_weight_max": 16.197021484375,
      "activations/layer0_attention_weight_min": -11.383275032043457,
      "activations/layer10_attention_weight_max": 34.32466506958008,
      "activations/layer10_attention_weight_min": -30.154050827026367,
      "activations/layer11_attention_weight_max": 32.285400390625,
      "activations/layer11_attention_weight_min": -27.756305694580078,
      "activations/layer12_attention_weight_max": 23.12312126159668,
      "activations/layer12_attention_weight_min": -29.01957130432129,
      "activations/layer13_attention_weight_max": 38.31997299194336,
      "activations/layer13_attention_weight_min": -30.751333236694336,
      "activations/layer14_attention_weight_max": 54.899818420410156,
      "activations/layer14_attention_weight_min": -40.82248306274414,
      "activations/layer15_attention_weight_max": 36.304901123046875,
      "activations/layer15_attention_weight_min": -29.74012565612793,
      "activations/layer16_attention_weight_max": 33.90757751464844,
      "activations/layer16_attention_weight_min": -26.980749130249023,
      "activations/layer17_attention_weight_max": 34.787811279296875,
      "activations/layer17_attention_weight_min": -24.804529190063477,
      "activations/layer18_attention_weight_max": 32.5697135925293,
      "activations/layer18_attention_weight_min": -22.184621810913086,
      "activations/layer19_attention_weight_max": 38.874874114990234,
      "activations/layer19_attention_weight_min": -32.80630111694336,
      "activations/layer1_attention_weight_max": 16.495960235595703,
      "activations/layer1_attention_weight_min": -13.603384971618652,
      "activations/layer20_attention_weight_max": 33.665828704833984,
      "activations/layer20_attention_weight_min": -24.946353912353516,
      "activations/layer21_attention_weight_max": 30.7054443359375,
      "activations/layer21_attention_weight_min": -22.26138687133789,
      "activations/layer22_attention_weight_max": 49.3150749206543,
      "activations/layer22_attention_weight_min": -27.2549991607666,
      "activations/layer23_attention_weight_max": 38.412071228027344,
      "activations/layer23_attention_weight_min": -23.233779907226562,
      "activations/layer2_attention_weight_max": 32.12321853637695,
      "activations/layer2_attention_weight_min": -31.304157257080078,
      "activations/layer3_attention_weight_max": 93.13299560546875,
      "activations/layer3_attention_weight_min": -91.77848052978516,
      "activations/layer4_attention_weight_max": 123.90831756591797,
      "activations/layer4_attention_weight_min": -115.6679916381836,
      "activations/layer5_attention_weight_max": 60.69851303100586,
      "activations/layer5_attention_weight_min": -61.79551696777344,
      "activations/layer6_attention_weight_max": 51.943214416503906,
      "activations/layer6_attention_weight_min": -50.70063400268555,
      "activations/layer7_attention_weight_max": 82.4750747680664,
      "activations/layer7_attention_weight_min": -87.6337661743164,
      "activations/layer8_attention_weight_max": 41.113338470458984,
      "activations/layer8_attention_weight_min": -38.12654495239258,
      "activations/layer9_attention_weight_max": 36.215087890625,
      "activations/layer9_attention_weight_min": -32.75847625732422,
      "epoch": 7.21,
      "learning_rate": 0.0001045310606060606,
      "loss": 2.8071,
      "step": 124150
    },
    {
      "activations/layer0_attention_weight_max": 15.72634220123291,
      "activations/layer0_attention_weight_min": -11.016427993774414,
      "activations/layer10_attention_weight_max": 36.62495803833008,
      "activations/layer10_attention_weight_min": -34.207279205322266,
      "activations/layer11_attention_weight_max": 35.26896286010742,
      "activations/layer11_attention_weight_min": -31.45323944091797,
      "activations/layer12_attention_weight_max": 30.207138061523438,
      "activations/layer12_attention_weight_min": -30.266063690185547,
      "activations/layer13_attention_weight_max": 41.233612060546875,
      "activations/layer13_attention_weight_min": -32.189903259277344,
      "activations/layer14_attention_weight_max": 53.40781784057617,
      "activations/layer14_attention_weight_min": -37.47362518310547,
      "activations/layer15_attention_weight_max": 38.692291259765625,
      "activations/layer15_attention_weight_min": -29.076860427856445,
      "activations/layer16_attention_weight_max": 34.55217361450195,
      "activations/layer16_attention_weight_min": -27.572179794311523,
      "activations/layer17_attention_weight_max": 35.24991989135742,
      "activations/layer17_attention_weight_min": -24.669815063476562,
      "activations/layer18_attention_weight_max": 32.34745407104492,
      "activations/layer18_attention_weight_min": -20.60076141357422,
      "activations/layer19_attention_weight_max": 39.88381576538086,
      "activations/layer19_attention_weight_min": -28.965606689453125,
      "activations/layer1_attention_weight_max": 17.67573356628418,
      "activations/layer1_attention_weight_min": -13.347046852111816,
      "activations/layer20_attention_weight_max": 34.06070327758789,
      "activations/layer20_attention_weight_min": -24.476362228393555,
      "activations/layer21_attention_weight_max": 34.7726936340332,
      "activations/layer21_attention_weight_min": -21.45800018310547,
      "activations/layer22_attention_weight_max": 53.56052780151367,
      "activations/layer22_attention_weight_min": -29.058835983276367,
      "activations/layer23_attention_weight_max": 42.778865814208984,
      "activations/layer23_attention_weight_min": -23.704837799072266,
      "activations/layer2_attention_weight_max": 32.58819580078125,
      "activations/layer2_attention_weight_min": -32.08413314819336,
      "activations/layer3_attention_weight_max": 90.23109436035156,
      "activations/layer3_attention_weight_min": -92.73689270019531,
      "activations/layer4_attention_weight_max": 118.7385482788086,
      "activations/layer4_attention_weight_min": -117.9923324584961,
      "activations/layer5_attention_weight_max": 63.77363586425781,
      "activations/layer5_attention_weight_min": -61.595703125,
      "activations/layer6_attention_weight_max": 50.64964294433594,
      "activations/layer6_attention_weight_min": -52.33491897583008,
      "activations/layer7_attention_weight_max": 89.33998107910156,
      "activations/layer7_attention_weight_min": -84.45765686035156,
      "activations/layer8_attention_weight_max": 40.621429443359375,
      "activations/layer8_attention_weight_min": -39.302894592285156,
      "activations/layer9_attention_weight_max": 39.7894172668457,
      "activations/layer9_attention_weight_min": -35.22281265258789,
      "epoch": 7.22,
      "learning_rate": 0.00010451212121212121,
      "loss": 2.8042,
      "step": 124200
    },
    {
      "activations/layer0_attention_weight_max": 16.12479019165039,
      "activations/layer0_attention_weight_min": -11.164785385131836,
      "activations/layer10_attention_weight_max": 30.890310287475586,
      "activations/layer10_attention_weight_min": -31.237930297851562,
      "activations/layer11_attention_weight_max": 29.017688751220703,
      "activations/layer11_attention_weight_min": -29.04755401611328,
      "activations/layer12_attention_weight_max": 29.69416046142578,
      "activations/layer12_attention_weight_min": -25.799047470092773,
      "activations/layer13_attention_weight_max": 36.603736877441406,
      "activations/layer13_attention_weight_min": -33.11790084838867,
      "activations/layer14_attention_weight_max": 50.02299118041992,
      "activations/layer14_attention_weight_min": -40.711917877197266,
      "activations/layer15_attention_weight_max": 34.639007568359375,
      "activations/layer15_attention_weight_min": -31.340961456298828,
      "activations/layer16_attention_weight_max": 33.85686111450195,
      "activations/layer16_attention_weight_min": -27.43629264831543,
      "activations/layer17_attention_weight_max": 35.63616180419922,
      "activations/layer17_attention_weight_min": -24.020166397094727,
      "activations/layer18_attention_weight_max": 29.305076599121094,
      "activations/layer18_attention_weight_min": -20.383907318115234,
      "activations/layer19_attention_weight_max": 39.11671829223633,
      "activations/layer19_attention_weight_min": -29.921249389648438,
      "activations/layer1_attention_weight_max": 16.40105438232422,
      "activations/layer1_attention_weight_min": -13.45522689819336,
      "activations/layer20_attention_weight_max": 30.852081298828125,
      "activations/layer20_attention_weight_min": -25.82357406616211,
      "activations/layer21_attention_weight_max": 31.7863826751709,
      "activations/layer21_attention_weight_min": -21.433807373046875,
      "activations/layer22_attention_weight_max": 47.40737533569336,
      "activations/layer22_attention_weight_min": -26.879602432250977,
      "activations/layer23_attention_weight_max": 36.26757049560547,
      "activations/layer23_attention_weight_min": -21.061161041259766,
      "activations/layer2_attention_weight_max": 32.29472732543945,
      "activations/layer2_attention_weight_min": -31.579071044921875,
      "activations/layer3_attention_weight_max": 87.52796936035156,
      "activations/layer3_attention_weight_min": -88.55856323242188,
      "activations/layer4_attention_weight_max": 118.3915023803711,
      "activations/layer4_attention_weight_min": -116.9522933959961,
      "activations/layer5_attention_weight_max": 59.96773910522461,
      "activations/layer5_attention_weight_min": -59.82526779174805,
      "activations/layer6_attention_weight_max": 47.925235748291016,
      "activations/layer6_attention_weight_min": -51.028419494628906,
      "activations/layer7_attention_weight_max": 87.87981414794922,
      "activations/layer7_attention_weight_min": -83.53534698486328,
      "activations/layer8_attention_weight_max": 38.3443489074707,
      "activations/layer8_attention_weight_min": -39.71445846557617,
      "activations/layer9_attention_weight_max": 32.565555572509766,
      "activations/layer9_attention_weight_min": -33.57230758666992,
      "epoch": 7.22,
      "learning_rate": 0.0001044931818181818,
      "loss": 2.8199,
      "step": 124250
    },
    {
      "activations/layer0_attention_weight_max": 15.001762390136719,
      "activations/layer0_attention_weight_min": -10.98939323425293,
      "activations/layer10_attention_weight_max": 33.43672180175781,
      "activations/layer10_attention_weight_min": -32.31007385253906,
      "activations/layer11_attention_weight_max": 33.63508224487305,
      "activations/layer11_attention_weight_min": -29.007179260253906,
      "activations/layer12_attention_weight_max": 43.936195373535156,
      "activations/layer12_attention_weight_min": -28.784318923950195,
      "activations/layer13_attention_weight_max": 62.037010192871094,
      "activations/layer13_attention_weight_min": -39.19368362426758,
      "activations/layer14_attention_weight_max": 53.44596862792969,
      "activations/layer14_attention_weight_min": -39.033817291259766,
      "activations/layer15_attention_weight_max": 42.6391716003418,
      "activations/layer15_attention_weight_min": -33.99102783203125,
      "activations/layer16_attention_weight_max": 36.25275421142578,
      "activations/layer16_attention_weight_min": -28.57802391052246,
      "activations/layer17_attention_weight_max": 35.059322357177734,
      "activations/layer17_attention_weight_min": -25.64505958557129,
      "activations/layer18_attention_weight_max": 36.68572235107422,
      "activations/layer18_attention_weight_min": -22.079925537109375,
      "activations/layer19_attention_weight_max": 39.54026412963867,
      "activations/layer19_attention_weight_min": -32.625709533691406,
      "activations/layer1_attention_weight_max": 16.70771598815918,
      "activations/layer1_attention_weight_min": -12.819258689880371,
      "activations/layer20_attention_weight_max": 32.26272201538086,
      "activations/layer20_attention_weight_min": -25.89128303527832,
      "activations/layer21_attention_weight_max": 35.09877395629883,
      "activations/layer21_attention_weight_min": -22.931034088134766,
      "activations/layer22_attention_weight_max": 48.27973937988281,
      "activations/layer22_attention_weight_min": -29.128095626831055,
      "activations/layer23_attention_weight_max": 37.1802864074707,
      "activations/layer23_attention_weight_min": -23.78243637084961,
      "activations/layer2_attention_weight_max": 31.941030502319336,
      "activations/layer2_attention_weight_min": -30.03547477722168,
      "activations/layer3_attention_weight_max": 95.05831146240234,
      "activations/layer3_attention_weight_min": -94.3194580078125,
      "activations/layer4_attention_weight_max": 125.89994049072266,
      "activations/layer4_attention_weight_min": -126.18547821044922,
      "activations/layer5_attention_weight_max": 66.046142578125,
      "activations/layer5_attention_weight_min": -64.5022201538086,
      "activations/layer6_attention_weight_max": 52.44645690917969,
      "activations/layer6_attention_weight_min": -57.73059844970703,
      "activations/layer7_attention_weight_max": 94.97562408447266,
      "activations/layer7_attention_weight_min": -95.25520324707031,
      "activations/layer8_attention_weight_max": 44.87432098388672,
      "activations/layer8_attention_weight_min": -43.38153839111328,
      "activations/layer9_attention_weight_max": 37.91718292236328,
      "activations/layer9_attention_weight_min": -35.772605895996094,
      "epoch": 7.22,
      "learning_rate": 0.00010447424242424241,
      "loss": 2.7971,
      "step": 124300
    },
    {
      "activations/layer0_attention_weight_max": 15.997111320495605,
      "activations/layer0_attention_weight_min": -11.309892654418945,
      "activations/layer10_attention_weight_max": 35.9116096496582,
      "activations/layer10_attention_weight_min": -33.86099624633789,
      "activations/layer11_attention_weight_max": 33.42984390258789,
      "activations/layer11_attention_weight_min": -29.719778060913086,
      "activations/layer12_attention_weight_max": 24.282297134399414,
      "activations/layer12_attention_weight_min": -23.521764755249023,
      "activations/layer13_attention_weight_max": 40.004188537597656,
      "activations/layer13_attention_weight_min": -33.1605339050293,
      "activations/layer14_attention_weight_max": 55.64750289916992,
      "activations/layer14_attention_weight_min": -42.2574462890625,
      "activations/layer15_attention_weight_max": 34.85689163208008,
      "activations/layer15_attention_weight_min": -28.517995834350586,
      "activations/layer16_attention_weight_max": 32.59775161743164,
      "activations/layer16_attention_weight_min": -26.69019317626953,
      "activations/layer17_attention_weight_max": 34.30078125,
      "activations/layer17_attention_weight_min": -26.009092330932617,
      "activations/layer18_attention_weight_max": 37.68292236328125,
      "activations/layer18_attention_weight_min": -22.253761291503906,
      "activations/layer19_attention_weight_max": 38.30735778808594,
      "activations/layer19_attention_weight_min": -31.6396541595459,
      "activations/layer1_attention_weight_max": 16.473512649536133,
      "activations/layer1_attention_weight_min": -14.095176696777344,
      "activations/layer20_attention_weight_max": 31.0548152923584,
      "activations/layer20_attention_weight_min": -25.754150390625,
      "activations/layer21_attention_weight_max": 32.83574295043945,
      "activations/layer21_attention_weight_min": -22.773590087890625,
      "activations/layer22_attention_weight_max": 48.54566955566406,
      "activations/layer22_attention_weight_min": -29.611135482788086,
      "activations/layer23_attention_weight_max": 38.35874938964844,
      "activations/layer23_attention_weight_min": -21.100509643554688,
      "activations/layer2_attention_weight_max": 32.21647262573242,
      "activations/layer2_attention_weight_min": -31.460357666015625,
      "activations/layer3_attention_weight_max": 89.25386047363281,
      "activations/layer3_attention_weight_min": -94.03606414794922,
      "activations/layer4_attention_weight_max": 123.3164291381836,
      "activations/layer4_attention_weight_min": -122.37525939941406,
      "activations/layer5_attention_weight_max": 64.5341796875,
      "activations/layer5_attention_weight_min": -61.39447784423828,
      "activations/layer6_attention_weight_max": 51.382591247558594,
      "activations/layer6_attention_weight_min": -52.1257438659668,
      "activations/layer7_attention_weight_max": 89.7204360961914,
      "activations/layer7_attention_weight_min": -88.19043731689453,
      "activations/layer8_attention_weight_max": 42.071651458740234,
      "activations/layer8_attention_weight_min": -41.15673065185547,
      "activations/layer9_attention_weight_max": 36.81532287597656,
      "activations/layer9_attention_weight_min": -34.13630676269531,
      "epoch": 7.23,
      "learning_rate": 0.00010445530303030303,
      "loss": 2.7863,
      "step": 124350
    },
    {
      "activations/layer0_attention_weight_max": 16.26368522644043,
      "activations/layer0_attention_weight_min": -11.383838653564453,
      "activations/layer10_attention_weight_max": 33.541717529296875,
      "activations/layer10_attention_weight_min": -31.347631454467773,
      "activations/layer11_attention_weight_max": 32.574668884277344,
      "activations/layer11_attention_weight_min": -29.58401107788086,
      "activations/layer12_attention_weight_max": 39.524471282958984,
      "activations/layer12_attention_weight_min": -27.791751861572266,
      "activations/layer13_attention_weight_max": 40.48308181762695,
      "activations/layer13_attention_weight_min": -33.207603454589844,
      "activations/layer14_attention_weight_max": 48.909217834472656,
      "activations/layer14_attention_weight_min": -39.52415084838867,
      "activations/layer15_attention_weight_max": 36.28190994262695,
      "activations/layer15_attention_weight_min": -32.259033203125,
      "activations/layer16_attention_weight_max": 34.29911422729492,
      "activations/layer16_attention_weight_min": -28.02100372314453,
      "activations/layer17_attention_weight_max": 33.642433166503906,
      "activations/layer17_attention_weight_min": -25.348857879638672,
      "activations/layer18_attention_weight_max": 30.970739364624023,
      "activations/layer18_attention_weight_min": -20.864906311035156,
      "activations/layer19_attention_weight_max": 39.1198844909668,
      "activations/layer19_attention_weight_min": -30.424909591674805,
      "activations/layer1_attention_weight_max": 16.072996139526367,
      "activations/layer1_attention_weight_min": -14.695844650268555,
      "activations/layer20_attention_weight_max": 30.082414627075195,
      "activations/layer20_attention_weight_min": -24.21722984313965,
      "activations/layer21_attention_weight_max": 29.300228118896484,
      "activations/layer21_attention_weight_min": -22.228271484375,
      "activations/layer22_attention_weight_max": 47.59624099731445,
      "activations/layer22_attention_weight_min": -29.50194549560547,
      "activations/layer23_attention_weight_max": 34.956642150878906,
      "activations/layer23_attention_weight_min": -25.875865936279297,
      "activations/layer2_attention_weight_max": 32.96337127685547,
      "activations/layer2_attention_weight_min": -31.594594955444336,
      "activations/layer3_attention_weight_max": 92.2672119140625,
      "activations/layer3_attention_weight_min": -89.48717498779297,
      "activations/layer4_attention_weight_max": 123.0090103149414,
      "activations/layer4_attention_weight_min": -118.11258697509766,
      "activations/layer5_attention_weight_max": 63.29620361328125,
      "activations/layer5_attention_weight_min": -61.398284912109375,
      "activations/layer6_attention_weight_max": 51.973697662353516,
      "activations/layer6_attention_weight_min": -53.1016731262207,
      "activations/layer7_attention_weight_max": 90.06666564941406,
      "activations/layer7_attention_weight_min": -88.95758819580078,
      "activations/layer8_attention_weight_max": 40.9647102355957,
      "activations/layer8_attention_weight_min": -38.465511322021484,
      "activations/layer9_attention_weight_max": 35.79219055175781,
      "activations/layer9_attention_weight_min": -33.66691589355469,
      "epoch": 7.23,
      "learning_rate": 0.00010443636363636363,
      "loss": 2.8233,
      "step": 124400
    },
    {
      "activations/layer0_attention_weight_max": 15.308786392211914,
      "activations/layer0_attention_weight_min": -10.866764068603516,
      "activations/layer10_attention_weight_max": 31.59103012084961,
      "activations/layer10_attention_weight_min": -30.734092712402344,
      "activations/layer11_attention_weight_max": 31.088516235351562,
      "activations/layer11_attention_weight_min": -27.826231002807617,
      "activations/layer12_attention_weight_max": 36.49394989013672,
      "activations/layer12_attention_weight_min": -32.68056869506836,
      "activations/layer13_attention_weight_max": 36.436588287353516,
      "activations/layer13_attention_weight_min": -31.483129501342773,
      "activations/layer14_attention_weight_max": 47.957366943359375,
      "activations/layer14_attention_weight_min": -38.4306755065918,
      "activations/layer15_attention_weight_max": 34.16414260864258,
      "activations/layer15_attention_weight_min": -29.262197494506836,
      "activations/layer16_attention_weight_max": 31.04459571838379,
      "activations/layer16_attention_weight_min": -26.774721145629883,
      "activations/layer17_attention_weight_max": 31.67890739440918,
      "activations/layer17_attention_weight_min": -24.078210830688477,
      "activations/layer18_attention_weight_max": 30.959630966186523,
      "activations/layer18_attention_weight_min": -21.43355369567871,
      "activations/layer19_attention_weight_max": 32.73005294799805,
      "activations/layer19_attention_weight_min": -28.903913497924805,
      "activations/layer1_attention_weight_max": 16.886959075927734,
      "activations/layer1_attention_weight_min": -13.795230865478516,
      "activations/layer20_attention_weight_max": 28.027219772338867,
      "activations/layer20_attention_weight_min": -23.828275680541992,
      "activations/layer21_attention_weight_max": 28.13345718383789,
      "activations/layer21_attention_weight_min": -20.707881927490234,
      "activations/layer22_attention_weight_max": 46.368408203125,
      "activations/layer22_attention_weight_min": -30.049362182617188,
      "activations/layer23_attention_weight_max": 33.86973571777344,
      "activations/layer23_attention_weight_min": -23.156452178955078,
      "activations/layer2_attention_weight_max": 31.543548583984375,
      "activations/layer2_attention_weight_min": -33.083526611328125,
      "activations/layer3_attention_weight_max": 92.83313751220703,
      "activations/layer3_attention_weight_min": -94.26589965820312,
      "activations/layer4_attention_weight_max": 129.04100036621094,
      "activations/layer4_attention_weight_min": -122.88211822509766,
      "activations/layer5_attention_weight_max": 63.876380920410156,
      "activations/layer5_attention_weight_min": -63.956329345703125,
      "activations/layer6_attention_weight_max": 52.27444839477539,
      "activations/layer6_attention_weight_min": -54.14626693725586,
      "activations/layer7_attention_weight_max": 88.11503601074219,
      "activations/layer7_attention_weight_min": -89.3044204711914,
      "activations/layer8_attention_weight_max": 39.55817413330078,
      "activations/layer8_attention_weight_min": -40.226531982421875,
      "activations/layer9_attention_weight_max": 32.845577239990234,
      "activations/layer9_attention_weight_min": -34.87583923339844,
      "epoch": 7.23,
      "learning_rate": 0.00010441742424242423,
      "loss": 2.8046,
      "step": 124450
    },
    {
      "activations/layer0_attention_weight_max": 15.959686279296875,
      "activations/layer0_attention_weight_min": -11.908145904541016,
      "activations/layer10_attention_weight_max": 31.92313003540039,
      "activations/layer10_attention_weight_min": -32.85221481323242,
      "activations/layer11_attention_weight_max": 33.93231964111328,
      "activations/layer11_attention_weight_min": -27.528350830078125,
      "activations/layer12_attention_weight_max": 23.63628578186035,
      "activations/layer12_attention_weight_min": -28.29233741760254,
      "activations/layer13_attention_weight_max": 43.07148361206055,
      "activations/layer13_attention_weight_min": -31.259174346923828,
      "activations/layer14_attention_weight_max": 54.18130874633789,
      "activations/layer14_attention_weight_min": -39.202354431152344,
      "activations/layer15_attention_weight_max": 34.45828628540039,
      "activations/layer15_attention_weight_min": -29.075159072875977,
      "activations/layer16_attention_weight_max": 36.20940017700195,
      "activations/layer16_attention_weight_min": -26.70820426940918,
      "activations/layer17_attention_weight_max": 35.47273635864258,
      "activations/layer17_attention_weight_min": -23.696212768554688,
      "activations/layer18_attention_weight_max": 32.893310546875,
      "activations/layer18_attention_weight_min": -21.998716354370117,
      "activations/layer19_attention_weight_max": 37.59874725341797,
      "activations/layer19_attention_weight_min": -30.72015380859375,
      "activations/layer1_attention_weight_max": 17.352981567382812,
      "activations/layer1_attention_weight_min": -14.796964645385742,
      "activations/layer20_attention_weight_max": 31.97014045715332,
      "activations/layer20_attention_weight_min": -23.065818786621094,
      "activations/layer21_attention_weight_max": 29.31682014465332,
      "activations/layer21_attention_weight_min": -21.691574096679688,
      "activations/layer22_attention_weight_max": 47.23539352416992,
      "activations/layer22_attention_weight_min": -28.645729064941406,
      "activations/layer23_attention_weight_max": 34.721778869628906,
      "activations/layer23_attention_weight_min": -24.46926498413086,
      "activations/layer2_attention_weight_max": 30.52252769470215,
      "activations/layer2_attention_weight_min": -31.547229766845703,
      "activations/layer3_attention_weight_max": 89.0369873046875,
      "activations/layer3_attention_weight_min": -94.0199203491211,
      "activations/layer4_attention_weight_max": 120.95818328857422,
      "activations/layer4_attention_weight_min": -122.95894622802734,
      "activations/layer5_attention_weight_max": 63.3202018737793,
      "activations/layer5_attention_weight_min": -64.52654266357422,
      "activations/layer6_attention_weight_max": 50.728519439697266,
      "activations/layer6_attention_weight_min": -51.440895080566406,
      "activations/layer7_attention_weight_max": 87.58118438720703,
      "activations/layer7_attention_weight_min": -86.7693862915039,
      "activations/layer8_attention_weight_max": 41.090824127197266,
      "activations/layer8_attention_weight_min": -40.810489654541016,
      "activations/layer9_attention_weight_max": 36.106014251708984,
      "activations/layer9_attention_weight_min": -35.29059982299805,
      "epoch": 7.23,
      "learning_rate": 0.00010439848484848483,
      "loss": 2.8086,
      "step": 124500
    },
    {
      "activations/layer0_attention_weight_max": 15.713854789733887,
      "activations/layer0_attention_weight_min": -11.279454231262207,
      "activations/layer10_attention_weight_max": 32.27009201049805,
      "activations/layer10_attention_weight_min": -30.940025329589844,
      "activations/layer11_attention_weight_max": 31.929128646850586,
      "activations/layer11_attention_weight_min": -29.420211791992188,
      "activations/layer12_attention_weight_max": 26.614992141723633,
      "activations/layer12_attention_weight_min": -24.252031326293945,
      "activations/layer13_attention_weight_max": 41.93196105957031,
      "activations/layer13_attention_weight_min": -33.531185150146484,
      "activations/layer14_attention_weight_max": 56.204402923583984,
      "activations/layer14_attention_weight_min": -42.426822662353516,
      "activations/layer15_attention_weight_max": 40.62778854370117,
      "activations/layer15_attention_weight_min": -31.32177734375,
      "activations/layer16_attention_weight_max": 39.08183670043945,
      "activations/layer16_attention_weight_min": -29.342939376831055,
      "activations/layer17_attention_weight_max": 42.24336624145508,
      "activations/layer17_attention_weight_min": -25.711572647094727,
      "activations/layer18_attention_weight_max": 32.207313537597656,
      "activations/layer18_attention_weight_min": -22.012290954589844,
      "activations/layer19_attention_weight_max": 42.75613784790039,
      "activations/layer19_attention_weight_min": -29.992521286010742,
      "activations/layer1_attention_weight_max": 17.18012237548828,
      "activations/layer1_attention_weight_min": -12.335919380187988,
      "activations/layer20_attention_weight_max": 39.28940200805664,
      "activations/layer20_attention_weight_min": -24.38676643371582,
      "activations/layer21_attention_weight_max": 35.28310012817383,
      "activations/layer21_attention_weight_min": -21.83963966369629,
      "activations/layer22_attention_weight_max": 54.25998306274414,
      "activations/layer22_attention_weight_min": -30.587697982788086,
      "activations/layer23_attention_weight_max": 39.06803512573242,
      "activations/layer23_attention_weight_min": -23.150920867919922,
      "activations/layer2_attention_weight_max": 32.003623962402344,
      "activations/layer2_attention_weight_min": -32.94053268432617,
      "activations/layer3_attention_weight_max": 91.44036865234375,
      "activations/layer3_attention_weight_min": -89.01294708251953,
      "activations/layer4_attention_weight_max": 123.83831787109375,
      "activations/layer4_attention_weight_min": -117.95748138427734,
      "activations/layer5_attention_weight_max": 67.5289306640625,
      "activations/layer5_attention_weight_min": -63.1783447265625,
      "activations/layer6_attention_weight_max": 54.82200241088867,
      "activations/layer6_attention_weight_min": -53.10157775878906,
      "activations/layer7_attention_weight_max": 92.42488861083984,
      "activations/layer7_attention_weight_min": -85.91120147705078,
      "activations/layer8_attention_weight_max": 41.78321838378906,
      "activations/layer8_attention_weight_min": -41.79515075683594,
      "activations/layer9_attention_weight_max": 35.895599365234375,
      "activations/layer9_attention_weight_min": -34.85270309448242,
      "epoch": 7.24,
      "learning_rate": 0.00010437954545454545,
      "loss": 2.8282,
      "step": 124550
    },
    {
      "activations/layer0_attention_weight_max": 16.419443130493164,
      "activations/layer0_attention_weight_min": -11.369841575622559,
      "activations/layer10_attention_weight_max": 33.360618591308594,
      "activations/layer10_attention_weight_min": -32.84391784667969,
      "activations/layer11_attention_weight_max": 32.0497932434082,
      "activations/layer11_attention_weight_min": -29.823532104492188,
      "activations/layer12_attention_weight_max": 22.64407730102539,
      "activations/layer12_attention_weight_min": -24.553211212158203,
      "activations/layer13_attention_weight_max": 40.375892639160156,
      "activations/layer13_attention_weight_min": -30.50043487548828,
      "activations/layer14_attention_weight_max": 48.880245208740234,
      "activations/layer14_attention_weight_min": -39.99463653564453,
      "activations/layer15_attention_weight_max": 36.26582717895508,
      "activations/layer15_attention_weight_min": -28.205427169799805,
      "activations/layer16_attention_weight_max": 30.911409378051758,
      "activations/layer16_attention_weight_min": -25.996931076049805,
      "activations/layer17_attention_weight_max": 35.721065521240234,
      "activations/layer17_attention_weight_min": -23.72797966003418,
      "activations/layer18_attention_weight_max": 31.844154357910156,
      "activations/layer18_attention_weight_min": -20.300121307373047,
      "activations/layer19_attention_weight_max": 41.77326965332031,
      "activations/layer19_attention_weight_min": -29.306808471679688,
      "activations/layer1_attention_weight_max": 16.318260192871094,
      "activations/layer1_attention_weight_min": -15.203102111816406,
      "activations/layer20_attention_weight_max": 34.34190368652344,
      "activations/layer20_attention_weight_min": -23.738481521606445,
      "activations/layer21_attention_weight_max": 31.61051368713379,
      "activations/layer21_attention_weight_min": -20.6445255279541,
      "activations/layer22_attention_weight_max": 50.68743133544922,
      "activations/layer22_attention_weight_min": -27.074800491333008,
      "activations/layer23_attention_weight_max": 36.62095642089844,
      "activations/layer23_attention_weight_min": -23.51199722290039,
      "activations/layer2_attention_weight_max": 32.09954833984375,
      "activations/layer2_attention_weight_min": -32.44917678833008,
      "activations/layer3_attention_weight_max": 95.24116516113281,
      "activations/layer3_attention_weight_min": -94.68434143066406,
      "activations/layer4_attention_weight_max": 121.90736389160156,
      "activations/layer4_attention_weight_min": -120.6849136352539,
      "activations/layer5_attention_weight_max": 63.73078155517578,
      "activations/layer5_attention_weight_min": -63.73012924194336,
      "activations/layer6_attention_weight_max": 50.93497848510742,
      "activations/layer6_attention_weight_min": -52.3000373840332,
      "activations/layer7_attention_weight_max": 93.3145523071289,
      "activations/layer7_attention_weight_min": -91.728271484375,
      "activations/layer8_attention_weight_max": 43.70152282714844,
      "activations/layer8_attention_weight_min": -39.51283645629883,
      "activations/layer9_attention_weight_max": 36.35511779785156,
      "activations/layer9_attention_weight_min": -34.457340240478516,
      "epoch": 7.24,
      "learning_rate": 0.00010436060606060605,
      "loss": 2.8111,
      "step": 124600
    },
    {
      "activations/layer0_attention_weight_max": 16.021120071411133,
      "activations/layer0_attention_weight_min": -11.262784004211426,
      "activations/layer10_attention_weight_max": 31.078001022338867,
      "activations/layer10_attention_weight_min": -31.137451171875,
      "activations/layer11_attention_weight_max": 30.229795455932617,
      "activations/layer11_attention_weight_min": -31.19666290283203,
      "activations/layer12_attention_weight_max": 44.3140869140625,
      "activations/layer12_attention_weight_min": -26.943347930908203,
      "activations/layer13_attention_weight_max": 42.860050201416016,
      "activations/layer13_attention_weight_min": -33.098175048828125,
      "activations/layer14_attention_weight_max": 59.3369255065918,
      "activations/layer14_attention_weight_min": -45.44289779663086,
      "activations/layer15_attention_weight_max": 34.15117645263672,
      "activations/layer15_attention_weight_min": -32.096214294433594,
      "activations/layer16_attention_weight_max": 33.296424865722656,
      "activations/layer16_attention_weight_min": -28.277843475341797,
      "activations/layer17_attention_weight_max": 34.64052200317383,
      "activations/layer17_attention_weight_min": -25.27637481689453,
      "activations/layer18_attention_weight_max": 28.786046981811523,
      "activations/layer18_attention_weight_min": -20.847736358642578,
      "activations/layer19_attention_weight_max": 37.21882247924805,
      "activations/layer19_attention_weight_min": -31.606735229492188,
      "activations/layer1_attention_weight_max": 17.252683639526367,
      "activations/layer1_attention_weight_min": -13.895061492919922,
      "activations/layer20_attention_weight_max": 32.44208908081055,
      "activations/layer20_attention_weight_min": -25.694921493530273,
      "activations/layer21_attention_weight_max": 30.823623657226562,
      "activations/layer21_attention_weight_min": -22.52290153503418,
      "activations/layer22_attention_weight_max": 48.246604919433594,
      "activations/layer22_attention_weight_min": -28.69537925720215,
      "activations/layer23_attention_weight_max": 34.32205581665039,
      "activations/layer23_attention_weight_min": -21.5535888671875,
      "activations/layer2_attention_weight_max": 32.44325637817383,
      "activations/layer2_attention_weight_min": -32.250877380371094,
      "activations/layer3_attention_weight_max": 91.07572174072266,
      "activations/layer3_attention_weight_min": -89.67994689941406,
      "activations/layer4_attention_weight_max": 121.1528549194336,
      "activations/layer4_attention_weight_min": -123.1348648071289,
      "activations/layer5_attention_weight_max": 61.94622802734375,
      "activations/layer5_attention_weight_min": -66.03489685058594,
      "activations/layer6_attention_weight_max": 50.25113296508789,
      "activations/layer6_attention_weight_min": -54.11812210083008,
      "activations/layer7_attention_weight_max": 86.8456039428711,
      "activations/layer7_attention_weight_min": -89.30604553222656,
      "activations/layer8_attention_weight_max": 38.959259033203125,
      "activations/layer8_attention_weight_min": -39.75547409057617,
      "activations/layer9_attention_weight_max": 33.4300651550293,
      "activations/layer9_attention_weight_min": -32.57717514038086,
      "epoch": 7.24,
      "learning_rate": 0.00010434166666666665,
      "loss": 2.8075,
      "step": 124650
    },
    {
      "activations/layer0_attention_weight_max": 16.070222854614258,
      "activations/layer0_attention_weight_min": -11.445144653320312,
      "activations/layer10_attention_weight_max": 33.514686584472656,
      "activations/layer10_attention_weight_min": -32.128456115722656,
      "activations/layer11_attention_weight_max": 33.086490631103516,
      "activations/layer11_attention_weight_min": -29.06869888305664,
      "activations/layer12_attention_weight_max": 42.290767669677734,
      "activations/layer12_attention_weight_min": -22.52629280090332,
      "activations/layer13_attention_weight_max": 46.131248474121094,
      "activations/layer13_attention_weight_min": -33.70006561279297,
      "activations/layer14_attention_weight_max": 55.15157699584961,
      "activations/layer14_attention_weight_min": -43.009891510009766,
      "activations/layer15_attention_weight_max": 39.71211242675781,
      "activations/layer15_attention_weight_min": -31.553979873657227,
      "activations/layer16_attention_weight_max": 36.249202728271484,
      "activations/layer16_attention_weight_min": -27.434526443481445,
      "activations/layer17_attention_weight_max": 38.187255859375,
      "activations/layer17_attention_weight_min": -26.337432861328125,
      "activations/layer18_attention_weight_max": 31.717802047729492,
      "activations/layer18_attention_weight_min": -21.216096878051758,
      "activations/layer19_attention_weight_max": 41.71830749511719,
      "activations/layer19_attention_weight_min": -30.20380973815918,
      "activations/layer1_attention_weight_max": 15.852774620056152,
      "activations/layer1_attention_weight_min": -13.88515853881836,
      "activations/layer20_attention_weight_max": 39.28101348876953,
      "activations/layer20_attention_weight_min": -26.378244400024414,
      "activations/layer21_attention_weight_max": 38.95093536376953,
      "activations/layer21_attention_weight_min": -24.009958267211914,
      "activations/layer22_attention_weight_max": 50.64581298828125,
      "activations/layer22_attention_weight_min": -30.396162033081055,
      "activations/layer23_attention_weight_max": 38.588951110839844,
      "activations/layer23_attention_weight_min": -21.337783813476562,
      "activations/layer2_attention_weight_max": 32.510772705078125,
      "activations/layer2_attention_weight_min": -30.403858184814453,
      "activations/layer3_attention_weight_max": 90.78254699707031,
      "activations/layer3_attention_weight_min": -89.27530670166016,
      "activations/layer4_attention_weight_max": 121.4268798828125,
      "activations/layer4_attention_weight_min": -119.89810943603516,
      "activations/layer5_attention_weight_max": 63.17047882080078,
      "activations/layer5_attention_weight_min": -64.88775634765625,
      "activations/layer6_attention_weight_max": 49.35083770751953,
      "activations/layer6_attention_weight_min": -54.03133010864258,
      "activations/layer7_attention_weight_max": 89.0673828125,
      "activations/layer7_attention_weight_min": -87.1657943725586,
      "activations/layer8_attention_weight_max": 38.498294830322266,
      "activations/layer8_attention_weight_min": -38.339874267578125,
      "activations/layer9_attention_weight_max": 35.249576568603516,
      "activations/layer9_attention_weight_min": -34.77783203125,
      "epoch": 7.25,
      "learning_rate": 0.00010432272727272727,
      "loss": 2.8084,
      "step": 124700
    },
    {
      "activations/layer0_attention_weight_max": 15.542337417602539,
      "activations/layer0_attention_weight_min": -11.833138465881348,
      "activations/layer10_attention_weight_max": 33.13446044921875,
      "activations/layer10_attention_weight_min": -33.571754455566406,
      "activations/layer11_attention_weight_max": 30.12953758239746,
      "activations/layer11_attention_weight_min": -29.658878326416016,
      "activations/layer12_attention_weight_max": 21.30696678161621,
      "activations/layer12_attention_weight_min": -24.537872314453125,
      "activations/layer13_attention_weight_max": 38.31355285644531,
      "activations/layer13_attention_weight_min": -30.997241973876953,
      "activations/layer14_attention_weight_max": 43.45610427856445,
      "activations/layer14_attention_weight_min": -39.267860412597656,
      "activations/layer15_attention_weight_max": 33.543148040771484,
      "activations/layer15_attention_weight_min": -31.717824935913086,
      "activations/layer16_attention_weight_max": 31.235652923583984,
      "activations/layer16_attention_weight_min": -28.233856201171875,
      "activations/layer17_attention_weight_max": 33.783775329589844,
      "activations/layer17_attention_weight_min": -23.921825408935547,
      "activations/layer18_attention_weight_max": 28.704195022583008,
      "activations/layer18_attention_weight_min": -22.854358673095703,
      "activations/layer19_attention_weight_max": 35.41485595703125,
      "activations/layer19_attention_weight_min": -29.338525772094727,
      "activations/layer1_attention_weight_max": 17.06801414489746,
      "activations/layer1_attention_weight_min": -14.54939079284668,
      "activations/layer20_attention_weight_max": 29.843528747558594,
      "activations/layer20_attention_weight_min": -24.623849868774414,
      "activations/layer21_attention_weight_max": 28.16445541381836,
      "activations/layer21_attention_weight_min": -23.1143798828125,
      "activations/layer22_attention_weight_max": 40.62074279785156,
      "activations/layer22_attention_weight_min": -26.980358123779297,
      "activations/layer23_attention_weight_max": 33.48745346069336,
      "activations/layer23_attention_weight_min": -23.465774536132812,
      "activations/layer2_attention_weight_max": 32.7982292175293,
      "activations/layer2_attention_weight_min": -32.56346893310547,
      "activations/layer3_attention_weight_max": 92.98277282714844,
      "activations/layer3_attention_weight_min": -99.48711395263672,
      "activations/layer4_attention_weight_max": 126.89189910888672,
      "activations/layer4_attention_weight_min": -123.7907485961914,
      "activations/layer5_attention_weight_max": 66.013427734375,
      "activations/layer5_attention_weight_min": -63.90853500366211,
      "activations/layer6_attention_weight_max": 53.250999450683594,
      "activations/layer6_attention_weight_min": -53.56608200073242,
      "activations/layer7_attention_weight_max": 88.35574340820312,
      "activations/layer7_attention_weight_min": -92.44290924072266,
      "activations/layer8_attention_weight_max": 41.8209228515625,
      "activations/layer8_attention_weight_min": -44.55575942993164,
      "activations/layer9_attention_weight_max": 35.082462310791016,
      "activations/layer9_attention_weight_min": -35.20061492919922,
      "epoch": 7.25,
      "learning_rate": 0.00010430378787878788,
      "loss": 2.8078,
      "step": 124750
    },
    {
      "activations/layer0_attention_weight_max": 15.893548965454102,
      "activations/layer0_attention_weight_min": -11.336766242980957,
      "activations/layer10_attention_weight_max": 34.125919342041016,
      "activations/layer10_attention_weight_min": -30.121477127075195,
      "activations/layer11_attention_weight_max": 31.87727165222168,
      "activations/layer11_attention_weight_min": -28.860151290893555,
      "activations/layer12_attention_weight_max": 23.271100997924805,
      "activations/layer12_attention_weight_min": -24.863019943237305,
      "activations/layer13_attention_weight_max": 41.53325271606445,
      "activations/layer13_attention_weight_min": -31.204317092895508,
      "activations/layer14_attention_weight_max": 48.001678466796875,
      "activations/layer14_attention_weight_min": -35.751441955566406,
      "activations/layer15_attention_weight_max": 36.06519317626953,
      "activations/layer15_attention_weight_min": -29.03375244140625,
      "activations/layer16_attention_weight_max": 33.025943756103516,
      "activations/layer16_attention_weight_min": -27.462474822998047,
      "activations/layer17_attention_weight_max": 36.81526565551758,
      "activations/layer17_attention_weight_min": -24.159278869628906,
      "activations/layer18_attention_weight_max": 33.88890838623047,
      "activations/layer18_attention_weight_min": -20.941654205322266,
      "activations/layer19_attention_weight_max": 39.20729064941406,
      "activations/layer19_attention_weight_min": -31.84706687927246,
      "activations/layer1_attention_weight_max": 16.76861572265625,
      "activations/layer1_attention_weight_min": -14.816329956054688,
      "activations/layer20_attention_weight_max": 31.29287338256836,
      "activations/layer20_attention_weight_min": -24.278270721435547,
      "activations/layer21_attention_weight_max": 32.72578811645508,
      "activations/layer21_attention_weight_min": -22.42055320739746,
      "activations/layer22_attention_weight_max": 48.91887664794922,
      "activations/layer22_attention_weight_min": -30.26019859313965,
      "activations/layer23_attention_weight_max": 40.55796432495117,
      "activations/layer23_attention_weight_min": -21.759231567382812,
      "activations/layer2_attention_weight_max": 32.1617317199707,
      "activations/layer2_attention_weight_min": -31.967803955078125,
      "activations/layer3_attention_weight_max": 92.31419372558594,
      "activations/layer3_attention_weight_min": -95.19779968261719,
      "activations/layer4_attention_weight_max": 123.20903778076172,
      "activations/layer4_attention_weight_min": -122.980712890625,
      "activations/layer5_attention_weight_max": 64.33367919921875,
      "activations/layer5_attention_weight_min": -65.64334106445312,
      "activations/layer6_attention_weight_max": 52.54692077636719,
      "activations/layer6_attention_weight_min": -52.8607063293457,
      "activations/layer7_attention_weight_max": 88.72850036621094,
      "activations/layer7_attention_weight_min": -89.63302612304688,
      "activations/layer8_attention_weight_max": 40.35004425048828,
      "activations/layer8_attention_weight_min": -37.6573600769043,
      "activations/layer9_attention_weight_max": 36.65595245361328,
      "activations/layer9_attention_weight_min": -32.18935775756836,
      "epoch": 7.25,
      "learning_rate": 0.00010428484848484847,
      "loss": 2.8065,
      "step": 124800
    },
    {
      "activations/layer0_attention_weight_max": 14.80522632598877,
      "activations/layer0_attention_weight_min": -10.492685317993164,
      "activations/layer10_attention_weight_max": 32.01390838623047,
      "activations/layer10_attention_weight_min": -29.828227996826172,
      "activations/layer11_attention_weight_max": 30.943756103515625,
      "activations/layer11_attention_weight_min": -29.263675689697266,
      "activations/layer12_attention_weight_max": 23.337480545043945,
      "activations/layer12_attention_weight_min": -34.392723083496094,
      "activations/layer13_attention_weight_max": 42.056007385253906,
      "activations/layer13_attention_weight_min": -31.54617691040039,
      "activations/layer14_attention_weight_max": 58.248104095458984,
      "activations/layer14_attention_weight_min": -42.207550048828125,
      "activations/layer15_attention_weight_max": 40.044189453125,
      "activations/layer15_attention_weight_min": -30.454517364501953,
      "activations/layer16_attention_weight_max": 37.58952331542969,
      "activations/layer16_attention_weight_min": -28.329456329345703,
      "activations/layer17_attention_weight_max": 38.42839431762695,
      "activations/layer17_attention_weight_min": -26.38283920288086,
      "activations/layer18_attention_weight_max": 32.512229919433594,
      "activations/layer18_attention_weight_min": -21.4885311126709,
      "activations/layer19_attention_weight_max": 37.508670806884766,
      "activations/layer19_attention_weight_min": -29.199811935424805,
      "activations/layer1_attention_weight_max": 17.724645614624023,
      "activations/layer1_attention_weight_min": -13.253015518188477,
      "activations/layer20_attention_weight_max": 31.154224395751953,
      "activations/layer20_attention_weight_min": -25.694766998291016,
      "activations/layer21_attention_weight_max": 26.0950927734375,
      "activations/layer21_attention_weight_min": -22.636722564697266,
      "activations/layer22_attention_weight_max": 47.43482971191406,
      "activations/layer22_attention_weight_min": -28.757640838623047,
      "activations/layer23_attention_weight_max": 33.67544937133789,
      "activations/layer23_attention_weight_min": -23.01007080078125,
      "activations/layer2_attention_weight_max": 32.380958557128906,
      "activations/layer2_attention_weight_min": -32.30906677246094,
      "activations/layer3_attention_weight_max": 96.34069061279297,
      "activations/layer3_attention_weight_min": -100.27832794189453,
      "activations/layer4_attention_weight_max": 126.72811126708984,
      "activations/layer4_attention_weight_min": -121.76102447509766,
      "activations/layer5_attention_weight_max": 63.04842758178711,
      "activations/layer5_attention_weight_min": -63.38465881347656,
      "activations/layer6_attention_weight_max": 51.46183776855469,
      "activations/layer6_attention_weight_min": -53.64435577392578,
      "activations/layer7_attention_weight_max": 83.20353698730469,
      "activations/layer7_attention_weight_min": -90.64019012451172,
      "activations/layer8_attention_weight_max": 41.85707473754883,
      "activations/layer8_attention_weight_min": -39.88542556762695,
      "activations/layer9_attention_weight_max": 35.64268112182617,
      "activations/layer9_attention_weight_min": -42.83226776123047,
      "epoch": 7.25,
      "learning_rate": 0.00010426590909090908,
      "loss": 2.8244,
      "step": 124850
    },
    {
      "activations/layer0_attention_weight_max": 15.224153518676758,
      "activations/layer0_attention_weight_min": -11.884906768798828,
      "activations/layer10_attention_weight_max": 35.280555725097656,
      "activations/layer10_attention_weight_min": -31.96896743774414,
      "activations/layer11_attention_weight_max": 35.67363739013672,
      "activations/layer11_attention_weight_min": -28.66141128540039,
      "activations/layer12_attention_weight_max": 26.01233673095703,
      "activations/layer12_attention_weight_min": -24.795825958251953,
      "activations/layer13_attention_weight_max": 42.29020309448242,
      "activations/layer13_attention_weight_min": -31.96514320373535,
      "activations/layer14_attention_weight_max": 49.833412170410156,
      "activations/layer14_attention_weight_min": -38.98711395263672,
      "activations/layer15_attention_weight_max": 34.7357292175293,
      "activations/layer15_attention_weight_min": -27.38465690612793,
      "activations/layer16_attention_weight_max": 33.28555679321289,
      "activations/layer16_attention_weight_min": -27.97527313232422,
      "activations/layer17_attention_weight_max": 33.97941970825195,
      "activations/layer17_attention_weight_min": -24.761579513549805,
      "activations/layer18_attention_weight_max": 38.19761276245117,
      "activations/layer18_attention_weight_min": -20.19419288635254,
      "activations/layer19_attention_weight_max": 41.028133392333984,
      "activations/layer19_attention_weight_min": -28.643957138061523,
      "activations/layer1_attention_weight_max": 17.950958251953125,
      "activations/layer1_attention_weight_min": -15.169422149658203,
      "activations/layer20_attention_weight_max": 31.855539321899414,
      "activations/layer20_attention_weight_min": -23.483224868774414,
      "activations/layer21_attention_weight_max": 34.85842514038086,
      "activations/layer21_attention_weight_min": -22.334991455078125,
      "activations/layer22_attention_weight_max": 56.7959098815918,
      "activations/layer22_attention_weight_min": -28.642047882080078,
      "activations/layer23_attention_weight_max": 44.42835998535156,
      "activations/layer23_attention_weight_min": -23.746089935302734,
      "activations/layer2_attention_weight_max": 34.285133361816406,
      "activations/layer2_attention_weight_min": -32.144004821777344,
      "activations/layer3_attention_weight_max": 94.5698471069336,
      "activations/layer3_attention_weight_min": -93.4330062866211,
      "activations/layer4_attention_weight_max": 125.48551177978516,
      "activations/layer4_attention_weight_min": -120.18021392822266,
      "activations/layer5_attention_weight_max": 63.879371643066406,
      "activations/layer5_attention_weight_min": -63.82465744018555,
      "activations/layer6_attention_weight_max": 52.29264450073242,
      "activations/layer6_attention_weight_min": -51.17222213745117,
      "activations/layer7_attention_weight_max": 91.09153747558594,
      "activations/layer7_attention_weight_min": -86.05137634277344,
      "activations/layer8_attention_weight_max": 41.818870544433594,
      "activations/layer8_attention_weight_min": -38.99931716918945,
      "activations/layer9_attention_weight_max": 36.69794845581055,
      "activations/layer9_attention_weight_min": -35.39236068725586,
      "epoch": 7.26,
      "learning_rate": 0.0001042469696969697,
      "loss": 2.8043,
      "step": 124900
    },
    {
      "activations/layer0_attention_weight_max": 15.526922225952148,
      "activations/layer0_attention_weight_min": -11.831686973571777,
      "activations/layer10_attention_weight_max": 32.16282653808594,
      "activations/layer10_attention_weight_min": -32.71070861816406,
      "activations/layer11_attention_weight_max": 33.12419509887695,
      "activations/layer11_attention_weight_min": -29.446273803710938,
      "activations/layer12_attention_weight_max": 27.860307693481445,
      "activations/layer12_attention_weight_min": -24.9680233001709,
      "activations/layer13_attention_weight_max": 41.53776931762695,
      "activations/layer13_attention_weight_min": -34.62620162963867,
      "activations/layer14_attention_weight_max": 51.33854293823242,
      "activations/layer14_attention_weight_min": -42.67951965332031,
      "activations/layer15_attention_weight_max": 35.098140716552734,
      "activations/layer15_attention_weight_min": -31.474925994873047,
      "activations/layer16_attention_weight_max": 41.857147216796875,
      "activations/layer16_attention_weight_min": -29.680953979492188,
      "activations/layer17_attention_weight_max": 35.59888458251953,
      "activations/layer17_attention_weight_min": -25.638147354125977,
      "activations/layer18_attention_weight_max": 32.314605712890625,
      "activations/layer18_attention_weight_min": -22.23634910583496,
      "activations/layer19_attention_weight_max": 41.947837829589844,
      "activations/layer19_attention_weight_min": -29.274938583374023,
      "activations/layer1_attention_weight_max": 16.980127334594727,
      "activations/layer1_attention_weight_min": -13.4989013671875,
      "activations/layer20_attention_weight_max": 30.575456619262695,
      "activations/layer20_attention_weight_min": -24.271862030029297,
      "activations/layer21_attention_weight_max": 35.2392692565918,
      "activations/layer21_attention_weight_min": -22.078222274780273,
      "activations/layer22_attention_weight_max": 45.67051315307617,
      "activations/layer22_attention_weight_min": -26.7247257232666,
      "activations/layer23_attention_weight_max": 35.72634506225586,
      "activations/layer23_attention_weight_min": -21.49599266052246,
      "activations/layer2_attention_weight_max": 30.729597091674805,
      "activations/layer2_attention_weight_min": -30.924583435058594,
      "activations/layer3_attention_weight_max": 91.11603546142578,
      "activations/layer3_attention_weight_min": -91.5937271118164,
      "activations/layer4_attention_weight_max": 117.91753387451172,
      "activations/layer4_attention_weight_min": -120.7881851196289,
      "activations/layer5_attention_weight_max": 60.90668869018555,
      "activations/layer5_attention_weight_min": -63.43266296386719,
      "activations/layer6_attention_weight_max": 50.033653259277344,
      "activations/layer6_attention_weight_min": -53.010013580322266,
      "activations/layer7_attention_weight_max": 90.4317855834961,
      "activations/layer7_attention_weight_min": -85.71218872070312,
      "activations/layer8_attention_weight_max": 37.43446350097656,
      "activations/layer8_attention_weight_min": -38.84663772583008,
      "activations/layer9_attention_weight_max": 33.84591293334961,
      "activations/layer9_attention_weight_min": -37.65361022949219,
      "epoch": 7.26,
      "learning_rate": 0.00010422803030303029,
      "loss": 2.8135,
      "step": 124950
    },
    {
      "activations/layer0_attention_weight_max": 16.378324508666992,
      "activations/layer0_attention_weight_min": -10.76315975189209,
      "activations/layer10_attention_weight_max": 33.59587478637695,
      "activations/layer10_attention_weight_min": -29.600826263427734,
      "activations/layer11_attention_weight_max": 33.02571105957031,
      "activations/layer11_attention_weight_min": -28.00475311279297,
      "activations/layer12_attention_weight_max": 23.11208724975586,
      "activations/layer12_attention_weight_min": -28.402219772338867,
      "activations/layer13_attention_weight_max": 38.876731872558594,
      "activations/layer13_attention_weight_min": -33.35651779174805,
      "activations/layer14_attention_weight_max": 49.00918197631836,
      "activations/layer14_attention_weight_min": -39.00393295288086,
      "activations/layer15_attention_weight_max": 35.932220458984375,
      "activations/layer15_attention_weight_min": -29.413469314575195,
      "activations/layer16_attention_weight_max": 37.224510192871094,
      "activations/layer16_attention_weight_min": -28.242576599121094,
      "activations/layer17_attention_weight_max": 33.28264617919922,
      "activations/layer17_attention_weight_min": -24.501550674438477,
      "activations/layer18_attention_weight_max": 32.337440490722656,
      "activations/layer18_attention_weight_min": -21.258882522583008,
      "activations/layer19_attention_weight_max": 38.773406982421875,
      "activations/layer19_attention_weight_min": -30.122848510742188,
      "activations/layer1_attention_weight_max": 17.27503776550293,
      "activations/layer1_attention_weight_min": -13.715550422668457,
      "activations/layer20_attention_weight_max": 28.923229217529297,
      "activations/layer20_attention_weight_min": -21.826017379760742,
      "activations/layer21_attention_weight_max": 30.07265853881836,
      "activations/layer21_attention_weight_min": -22.561737060546875,
      "activations/layer22_attention_weight_max": 47.75054168701172,
      "activations/layer22_attention_weight_min": -27.354007720947266,
      "activations/layer23_attention_weight_max": 41.340492248535156,
      "activations/layer23_attention_weight_min": -20.890342712402344,
      "activations/layer2_attention_weight_max": 31.970722198486328,
      "activations/layer2_attention_weight_min": -32.807472229003906,
      "activations/layer3_attention_weight_max": 91.9668960571289,
      "activations/layer3_attention_weight_min": -94.70197296142578,
      "activations/layer4_attention_weight_max": 123.51383209228516,
      "activations/layer4_attention_weight_min": -129.4141082763672,
      "activations/layer5_attention_weight_max": 62.659523010253906,
      "activations/layer5_attention_weight_min": -67.9358901977539,
      "activations/layer6_attention_weight_max": 49.997074127197266,
      "activations/layer6_attention_weight_min": -51.31293487548828,
      "activations/layer7_attention_weight_max": 87.6982650756836,
      "activations/layer7_attention_weight_min": -88.78887939453125,
      "activations/layer8_attention_weight_max": 40.37898635864258,
      "activations/layer8_attention_weight_min": -41.90178680419922,
      "activations/layer9_attention_weight_max": 37.070011138916016,
      "activations/layer9_attention_weight_min": -33.829402923583984,
      "epoch": 7.26,
      "learning_rate": 0.0001042090909090909,
      "loss": 2.8203,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_loss": 2.763671875,
      "eval_runtime": 8.4755,
      "eval_samples_per_second": 506.637,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_openwebtext_loss": 2.763671875,
      "eval_openwebtext_ppl": 15.85796463907484,
      "eval_openwebtext_runtime": 8.4755,
      "eval_openwebtext_samples_per_second": 506.637,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 2.0003,
      "eval_wikitext_samples_per_second": 227.967,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_lambada_loss": 2.57421875,
      "eval_lambada_ppl": 13.121062323276538,
      "eval_lambada_runtime": 9.6153,
      "eval_lambada_samples_per_second": 506.378,
      "step": 125000
    },
    {
      "activations/layer0_attention_weight_max": 15.414226531982422,
      "activations/layer0_attention_weight_min": -11.35372257232666,
      "activations/layer10_attention_weight_max": 35.822608947753906,
      "activations/layer10_attention_weight_min": -31.938127517700195,
      "activations/layer11_attention_weight_max": 32.40937423706055,
      "activations/layer11_attention_weight_min": -29.64453887939453,
      "activations/layer12_attention_weight_max": 24.204790115356445,
      "activations/layer12_attention_weight_min": -26.826444625854492,
      "activations/layer13_attention_weight_max": 46.80821228027344,
      "activations/layer13_attention_weight_min": -33.33013916015625,
      "activations/layer14_attention_weight_max": 52.85970687866211,
      "activations/layer14_attention_weight_min": -41.677833557128906,
      "activations/layer15_attention_weight_max": 40.714481353759766,
      "activations/layer15_attention_weight_min": -31.10249137878418,
      "activations/layer16_attention_weight_max": 34.88601303100586,
      "activations/layer16_attention_weight_min": -26.179840087890625,
      "activations/layer17_attention_weight_max": 39.90327835083008,
      "activations/layer17_attention_weight_min": -25.12436294555664,
      "activations/layer18_attention_weight_max": 31.24934196472168,
      "activations/layer18_attention_weight_min": -20.654191970825195,
      "activations/layer19_attention_weight_max": 34.36344528198242,
      "activations/layer19_attention_weight_min": -27.662952423095703,
      "activations/layer1_attention_weight_max": 16.36757469177246,
      "activations/layer1_attention_weight_min": -13.89153003692627,
      "activations/layer20_attention_weight_max": 29.70293617248535,
      "activations/layer20_attention_weight_min": -22.651649475097656,
      "activations/layer21_attention_weight_max": 29.637968063354492,
      "activations/layer21_attention_weight_min": -21.074037551879883,
      "activations/layer22_attention_weight_max": 46.541099548339844,
      "activations/layer22_attention_weight_min": -32.44169998168945,
      "activations/layer23_attention_weight_max": 34.072261810302734,
      "activations/layer23_attention_weight_min": -23.80510711669922,
      "activations/layer2_attention_weight_max": 32.216468811035156,
      "activations/layer2_attention_weight_min": -30.773046493530273,
      "activations/layer3_attention_weight_max": 89.58977508544922,
      "activations/layer3_attention_weight_min": -90.84943389892578,
      "activations/layer4_attention_weight_max": 121.07037353515625,
      "activations/layer4_attention_weight_min": -116.49281311035156,
      "activations/layer5_attention_weight_max": 60.87211227416992,
      "activations/layer5_attention_weight_min": -62.365840911865234,
      "activations/layer6_attention_weight_max": 50.11660385131836,
      "activations/layer6_attention_weight_min": -54.05519485473633,
      "activations/layer7_attention_weight_max": 88.64818572998047,
      "activations/layer7_attention_weight_min": -85.15937042236328,
      "activations/layer8_attention_weight_max": 40.495182037353516,
      "activations/layer8_attention_weight_min": -39.1553840637207,
      "activations/layer9_attention_weight_max": 35.559532165527344,
      "activations/layer9_attention_weight_min": -33.4365348815918,
      "epoch": 7.27,
      "learning_rate": 0.0001041901515151515,
      "loss": 2.8137,
      "step": 125050
    },
    {
      "activations/layer0_attention_weight_max": 16.244184494018555,
      "activations/layer0_attention_weight_min": -10.31190013885498,
      "activations/layer10_attention_weight_max": 34.13275146484375,
      "activations/layer10_attention_weight_min": -31.180173873901367,
      "activations/layer11_attention_weight_max": 32.09001922607422,
      "activations/layer11_attention_weight_min": -27.945545196533203,
      "activations/layer12_attention_weight_max": 21.498952865600586,
      "activations/layer12_attention_weight_min": -27.763198852539062,
      "activations/layer13_attention_weight_max": 38.59910583496094,
      "activations/layer13_attention_weight_min": -31.302616119384766,
      "activations/layer14_attention_weight_max": 47.92871856689453,
      "activations/layer14_attention_weight_min": -38.85977554321289,
      "activations/layer15_attention_weight_max": 35.504310607910156,
      "activations/layer15_attention_weight_min": -29.452848434448242,
      "activations/layer16_attention_weight_max": 31.123205184936523,
      "activations/layer16_attention_weight_min": -27.326364517211914,
      "activations/layer17_attention_weight_max": 34.63604736328125,
      "activations/layer17_attention_weight_min": -25.605554580688477,
      "activations/layer18_attention_weight_max": 30.515066146850586,
      "activations/layer18_attention_weight_min": -20.669946670532227,
      "activations/layer19_attention_weight_max": 36.607574462890625,
      "activations/layer19_attention_weight_min": -28.094770431518555,
      "activations/layer1_attention_weight_max": 16.690237045288086,
      "activations/layer1_attention_weight_min": -14.988801002502441,
      "activations/layer20_attention_weight_max": 32.80485534667969,
      "activations/layer20_attention_weight_min": -22.28729820251465,
      "activations/layer21_attention_weight_max": 31.817962646484375,
      "activations/layer21_attention_weight_min": -21.29187774658203,
      "activations/layer22_attention_weight_max": 49.042991638183594,
      "activations/layer22_attention_weight_min": -27.409618377685547,
      "activations/layer23_attention_weight_max": 37.65919494628906,
      "activations/layer23_attention_weight_min": -22.224475860595703,
      "activations/layer2_attention_weight_max": 31.77581787109375,
      "activations/layer2_attention_weight_min": -31.549274444580078,
      "activations/layer3_attention_weight_max": 89.8293685913086,
      "activations/layer3_attention_weight_min": -92.02875518798828,
      "activations/layer4_attention_weight_max": 119.35204315185547,
      "activations/layer4_attention_weight_min": -117.8858642578125,
      "activations/layer5_attention_weight_max": 62.144493103027344,
      "activations/layer5_attention_weight_min": -62.42023849487305,
      "activations/layer6_attention_weight_max": 48.324302673339844,
      "activations/layer6_attention_weight_min": -53.19517135620117,
      "activations/layer7_attention_weight_max": 84.38379669189453,
      "activations/layer7_attention_weight_min": -89.71652221679688,
      "activations/layer8_attention_weight_max": 38.94809341430664,
      "activations/layer8_attention_weight_min": -39.43430709838867,
      "activations/layer9_attention_weight_max": 34.932918548583984,
      "activations/layer9_attention_weight_min": -33.540889739990234,
      "epoch": 7.27,
      "learning_rate": 0.0001041712121212121,
      "loss": 2.817,
      "step": 125100
    },
    {
      "activations/layer0_attention_weight_max": 15.829071998596191,
      "activations/layer0_attention_weight_min": -10.256545066833496,
      "activations/layer10_attention_weight_max": 33.16280746459961,
      "activations/layer10_attention_weight_min": -31.78395652770996,
      "activations/layer11_attention_weight_max": 34.516029357910156,
      "activations/layer11_attention_weight_min": -30.54305648803711,
      "activations/layer12_attention_weight_max": 26.83927345275879,
      "activations/layer12_attention_weight_min": -24.15434455871582,
      "activations/layer13_attention_weight_max": 37.36415100097656,
      "activations/layer13_attention_weight_min": -31.303340911865234,
      "activations/layer14_attention_weight_max": 51.873905181884766,
      "activations/layer14_attention_weight_min": -40.34200668334961,
      "activations/layer15_attention_weight_max": 34.35520553588867,
      "activations/layer15_attention_weight_min": -27.965635299682617,
      "activations/layer16_attention_weight_max": 35.3763427734375,
      "activations/layer16_attention_weight_min": -25.198631286621094,
      "activations/layer17_attention_weight_max": 35.30704116821289,
      "activations/layer17_attention_weight_min": -25.199726104736328,
      "activations/layer18_attention_weight_max": 30.153953552246094,
      "activations/layer18_attention_weight_min": -20.86377716064453,
      "activations/layer19_attention_weight_max": 40.42124557495117,
      "activations/layer19_attention_weight_min": -32.792694091796875,
      "activations/layer1_attention_weight_max": 16.046184539794922,
      "activations/layer1_attention_weight_min": -13.408947944641113,
      "activations/layer20_attention_weight_max": 36.03886795043945,
      "activations/layer20_attention_weight_min": -22.548566818237305,
      "activations/layer21_attention_weight_max": 32.80572509765625,
      "activations/layer21_attention_weight_min": -21.538005828857422,
      "activations/layer22_attention_weight_max": 49.47632598876953,
      "activations/layer22_attention_weight_min": -26.78295135498047,
      "activations/layer23_attention_weight_max": 37.14537811279297,
      "activations/layer23_attention_weight_min": -21.85483169555664,
      "activations/layer2_attention_weight_max": 32.305694580078125,
      "activations/layer2_attention_weight_min": -32.91371154785156,
      "activations/layer3_attention_weight_max": 93.88888549804688,
      "activations/layer3_attention_weight_min": -97.26901245117188,
      "activations/layer4_attention_weight_max": 125.92565155029297,
      "activations/layer4_attention_weight_min": -126.62727355957031,
      "activations/layer5_attention_weight_max": 63.685035705566406,
      "activations/layer5_attention_weight_min": -65.32643127441406,
      "activations/layer6_attention_weight_max": 53.071189880371094,
      "activations/layer6_attention_weight_min": -54.89283752441406,
      "activations/layer7_attention_weight_max": 98.8313980102539,
      "activations/layer7_attention_weight_min": -93.32728576660156,
      "activations/layer8_attention_weight_max": 43.62382507324219,
      "activations/layer8_attention_weight_min": -41.81220626831055,
      "activations/layer9_attention_weight_max": 36.38473129272461,
      "activations/layer9_attention_weight_min": -40.0388298034668,
      "epoch": 7.27,
      "learning_rate": 0.00010415227272727272,
      "loss": 2.8064,
      "step": 125150
    },
    {
      "activations/layer0_attention_weight_max": 15.76009750366211,
      "activations/layer0_attention_weight_min": -11.571635246276855,
      "activations/layer10_attention_weight_max": 32.44700622558594,
      "activations/layer10_attention_weight_min": -31.516286849975586,
      "activations/layer11_attention_weight_max": 30.89818572998047,
      "activations/layer11_attention_weight_min": -29.22661590576172,
      "activations/layer12_attention_weight_max": 21.42122459411621,
      "activations/layer12_attention_weight_min": -27.814815521240234,
      "activations/layer13_attention_weight_max": 36.665313720703125,
      "activations/layer13_attention_weight_min": -32.67575454711914,
      "activations/layer14_attention_weight_max": 54.71105194091797,
      "activations/layer14_attention_weight_min": -40.61384201049805,
      "activations/layer15_attention_weight_max": 33.99687576293945,
      "activations/layer15_attention_weight_min": -30.17190170288086,
      "activations/layer16_attention_weight_max": 32.722381591796875,
      "activations/layer16_attention_weight_min": -29.39423942565918,
      "activations/layer17_attention_weight_max": 35.60603713989258,
      "activations/layer17_attention_weight_min": -24.862598419189453,
      "activations/layer18_attention_weight_max": 30.36448860168457,
      "activations/layer18_attention_weight_min": -22.370874404907227,
      "activations/layer19_attention_weight_max": 36.71514129638672,
      "activations/layer19_attention_weight_min": -30.24187660217285,
      "activations/layer1_attention_weight_max": 16.043922424316406,
      "activations/layer1_attention_weight_min": -15.430120468139648,
      "activations/layer20_attention_weight_max": 30.971288681030273,
      "activations/layer20_attention_weight_min": -22.95732307434082,
      "activations/layer21_attention_weight_max": 31.255149841308594,
      "activations/layer21_attention_weight_min": -21.778545379638672,
      "activations/layer22_attention_weight_max": 48.28117752075195,
      "activations/layer22_attention_weight_min": -25.672468185424805,
      "activations/layer23_attention_weight_max": 39.25822067260742,
      "activations/layer23_attention_weight_min": -22.61159896850586,
      "activations/layer2_attention_weight_max": 33.38397979736328,
      "activations/layer2_attention_weight_min": -32.080078125,
      "activations/layer3_attention_weight_max": 90.90251922607422,
      "activations/layer3_attention_weight_min": -95.96797943115234,
      "activations/layer4_attention_weight_max": 119.75127410888672,
      "activations/layer4_attention_weight_min": -121.59862518310547,
      "activations/layer5_attention_weight_max": 61.42116928100586,
      "activations/layer5_attention_weight_min": -65.79119873046875,
      "activations/layer6_attention_weight_max": 52.94603729248047,
      "activations/layer6_attention_weight_min": -52.45723342895508,
      "activations/layer7_attention_weight_max": 89.25320434570312,
      "activations/layer7_attention_weight_min": -86.21279907226562,
      "activations/layer8_attention_weight_max": 39.00271987915039,
      "activations/layer8_attention_weight_min": -44.03595733642578,
      "activations/layer9_attention_weight_max": 36.2203254699707,
      "activations/layer9_attention_weight_min": -34.7865104675293,
      "epoch": 7.27,
      "learning_rate": 0.00010413333333333332,
      "loss": 2.8192,
      "step": 125200
    },
    {
      "activations/layer0_attention_weight_max": 15.448831558227539,
      "activations/layer0_attention_weight_min": -11.573137283325195,
      "activations/layer10_attention_weight_max": 31.12433433532715,
      "activations/layer10_attention_weight_min": -34.67987060546875,
      "activations/layer11_attention_weight_max": 31.652019500732422,
      "activations/layer11_attention_weight_min": -30.54243278503418,
      "activations/layer12_attention_weight_max": 25.270301818847656,
      "activations/layer12_attention_weight_min": -28.975317001342773,
      "activations/layer13_attention_weight_max": 39.58901596069336,
      "activations/layer13_attention_weight_min": -32.79570007324219,
      "activations/layer14_attention_weight_max": 48.373046875,
      "activations/layer14_attention_weight_min": -41.6143798828125,
      "activations/layer15_attention_weight_max": 36.231346130371094,
      "activations/layer15_attention_weight_min": -28.853748321533203,
      "activations/layer16_attention_weight_max": 34.463314056396484,
      "activations/layer16_attention_weight_min": -26.722721099853516,
      "activations/layer17_attention_weight_max": 38.895755767822266,
      "activations/layer17_attention_weight_min": -23.90639305114746,
      "activations/layer18_attention_weight_max": 29.876558303833008,
      "activations/layer18_attention_weight_min": -21.214885711669922,
      "activations/layer19_attention_weight_max": 38.587589263916016,
      "activations/layer19_attention_weight_min": -28.685766220092773,
      "activations/layer1_attention_weight_max": 17.97789192199707,
      "activations/layer1_attention_weight_min": -18.982736587524414,
      "activations/layer20_attention_weight_max": 30.124122619628906,
      "activations/layer20_attention_weight_min": -21.814790725708008,
      "activations/layer21_attention_weight_max": 28.205730438232422,
      "activations/layer21_attention_weight_min": -21.103086471557617,
      "activations/layer22_attention_weight_max": 46.42566680908203,
      "activations/layer22_attention_weight_min": -28.77972984313965,
      "activations/layer23_attention_weight_max": 32.61227798461914,
      "activations/layer23_attention_weight_min": -22.839345932006836,
      "activations/layer2_attention_weight_max": 34.6680908203125,
      "activations/layer2_attention_weight_min": -33.29796600341797,
      "activations/layer3_attention_weight_max": 90.62800598144531,
      "activations/layer3_attention_weight_min": -99.86949920654297,
      "activations/layer4_attention_weight_max": 123.10340881347656,
      "activations/layer4_attention_weight_min": -130.5487823486328,
      "activations/layer5_attention_weight_max": 65.35527801513672,
      "activations/layer5_attention_weight_min": -68.72145080566406,
      "activations/layer6_attention_weight_max": 50.89301300048828,
      "activations/layer6_attention_weight_min": -54.041812896728516,
      "activations/layer7_attention_weight_max": 89.8934326171875,
      "activations/layer7_attention_weight_min": -90.05802154541016,
      "activations/layer8_attention_weight_max": 41.226470947265625,
      "activations/layer8_attention_weight_min": -41.550445556640625,
      "activations/layer9_attention_weight_max": 35.54548263549805,
      "activations/layer9_attention_weight_min": -35.68022918701172,
      "epoch": 7.28,
      "learning_rate": 0.00010411439393939394,
      "loss": 2.8132,
      "step": 125250
    },
    {
      "activations/layer0_attention_weight_max": 16.026002883911133,
      "activations/layer0_attention_weight_min": -12.435965538024902,
      "activations/layer10_attention_weight_max": 35.843833923339844,
      "activations/layer10_attention_weight_min": -32.48952865600586,
      "activations/layer11_attention_weight_max": 35.23381042480469,
      "activations/layer11_attention_weight_min": -30.426618576049805,
      "activations/layer12_attention_weight_max": 31.699691772460938,
      "activations/layer12_attention_weight_min": -23.299335479736328,
      "activations/layer13_attention_weight_max": 48.9634895324707,
      "activations/layer13_attention_weight_min": -34.46935272216797,
      "activations/layer14_attention_weight_max": 55.821014404296875,
      "activations/layer14_attention_weight_min": -40.395687103271484,
      "activations/layer15_attention_weight_max": 42.208351135253906,
      "activations/layer15_attention_weight_min": -31.94597625732422,
      "activations/layer16_attention_weight_max": 35.155372619628906,
      "activations/layer16_attention_weight_min": -26.78457260131836,
      "activations/layer17_attention_weight_max": 34.617042541503906,
      "activations/layer17_attention_weight_min": -24.811498641967773,
      "activations/layer18_attention_weight_max": 33.564796447753906,
      "activations/layer18_attention_weight_min": -20.51461410522461,
      "activations/layer19_attention_weight_max": 34.92887878417969,
      "activations/layer19_attention_weight_min": -29.42122459411621,
      "activations/layer1_attention_weight_max": 17.051189422607422,
      "activations/layer1_attention_weight_min": -15.063032150268555,
      "activations/layer20_attention_weight_max": 29.228187561035156,
      "activations/layer20_attention_weight_min": -21.20197105407715,
      "activations/layer21_attention_weight_max": 27.9719295501709,
      "activations/layer21_attention_weight_min": -21.502424240112305,
      "activations/layer22_attention_weight_max": 47.477943420410156,
      "activations/layer22_attention_weight_min": -27.473562240600586,
      "activations/layer23_attention_weight_max": 34.88686752319336,
      "activations/layer23_attention_weight_min": -21.764423370361328,
      "activations/layer2_attention_weight_max": 33.718143463134766,
      "activations/layer2_attention_weight_min": -33.199928283691406,
      "activations/layer3_attention_weight_max": 96.48673248291016,
      "activations/layer3_attention_weight_min": -98.31023406982422,
      "activations/layer4_attention_weight_max": 129.4852752685547,
      "activations/layer4_attention_weight_min": -131.31764221191406,
      "activations/layer5_attention_weight_max": 63.517723083496094,
      "activations/layer5_attention_weight_min": -66.68754577636719,
      "activations/layer6_attention_weight_max": 53.02577209472656,
      "activations/layer6_attention_weight_min": -54.28018569946289,
      "activations/layer7_attention_weight_max": 91.96742248535156,
      "activations/layer7_attention_weight_min": -92.14540100097656,
      "activations/layer8_attention_weight_max": 43.69710922241211,
      "activations/layer8_attention_weight_min": -41.53187561035156,
      "activations/layer9_attention_weight_max": 38.87150955200195,
      "activations/layer9_attention_weight_min": -36.74092483520508,
      "epoch": 7.28,
      "learning_rate": 0.00010409545454545454,
      "loss": 2.8059,
      "step": 125300
    },
    {
      "activations/layer0_attention_weight_max": 16.684415817260742,
      "activations/layer0_attention_weight_min": -11.367056846618652,
      "activations/layer10_attention_weight_max": 34.68714904785156,
      "activations/layer10_attention_weight_min": -30.563268661499023,
      "activations/layer11_attention_weight_max": 32.547386169433594,
      "activations/layer11_attention_weight_min": -30.491474151611328,
      "activations/layer12_attention_weight_max": 25.794235229492188,
      "activations/layer12_attention_weight_min": -32.902191162109375,
      "activations/layer13_attention_weight_max": 35.90877914428711,
      "activations/layer13_attention_weight_min": -31.843759536743164,
      "activations/layer14_attention_weight_max": 51.85712814331055,
      "activations/layer14_attention_weight_min": -39.51222610473633,
      "activations/layer15_attention_weight_max": 34.92268753051758,
      "activations/layer15_attention_weight_min": -28.332895278930664,
      "activations/layer16_attention_weight_max": 35.15182876586914,
      "activations/layer16_attention_weight_min": -26.939477920532227,
      "activations/layer17_attention_weight_max": 38.823760986328125,
      "activations/layer17_attention_weight_min": -24.635080337524414,
      "activations/layer18_attention_weight_max": 33.93989562988281,
      "activations/layer18_attention_weight_min": -21.977115631103516,
      "activations/layer19_attention_weight_max": 34.80838394165039,
      "activations/layer19_attention_weight_min": -28.350406646728516,
      "activations/layer1_attention_weight_max": 17.683034896850586,
      "activations/layer1_attention_weight_min": -14.918207168579102,
      "activations/layer20_attention_weight_max": 31.147043228149414,
      "activations/layer20_attention_weight_min": -26.71343994140625,
      "activations/layer21_attention_weight_max": 27.194456100463867,
      "activations/layer21_attention_weight_min": -24.140954971313477,
      "activations/layer22_attention_weight_max": 45.33843994140625,
      "activations/layer22_attention_weight_min": -30.62324333190918,
      "activations/layer23_attention_weight_max": 32.97761917114258,
      "activations/layer23_attention_weight_min": -25.659942626953125,
      "activations/layer2_attention_weight_max": 35.60418701171875,
      "activations/layer2_attention_weight_min": -32.269134521484375,
      "activations/layer3_attention_weight_max": 100.2303237915039,
      "activations/layer3_attention_weight_min": -99.6515121459961,
      "activations/layer4_attention_weight_max": 127.04158020019531,
      "activations/layer4_attention_weight_min": -121.62911224365234,
      "activations/layer5_attention_weight_max": 62.7169189453125,
      "activations/layer5_attention_weight_min": -62.448463439941406,
      "activations/layer6_attention_weight_max": 48.72026062011719,
      "activations/layer6_attention_weight_min": -52.91120147705078,
      "activations/layer7_attention_weight_max": 90.43992614746094,
      "activations/layer7_attention_weight_min": -87.799072265625,
      "activations/layer8_attention_weight_max": 41.280906677246094,
      "activations/layer8_attention_weight_min": -39.63486099243164,
      "activations/layer9_attention_weight_max": 33.487850189208984,
      "activations/layer9_attention_weight_min": -38.916934967041016,
      "epoch": 7.28,
      "learning_rate": 0.00010407651515151514,
      "loss": 2.8058,
      "step": 125350
    },
    {
      "activations/layer0_attention_weight_max": 16.581645965576172,
      "activations/layer0_attention_weight_min": -10.901172637939453,
      "activations/layer10_attention_weight_max": 31.18235206604004,
      "activations/layer10_attention_weight_min": -29.543935775756836,
      "activations/layer11_attention_weight_max": 30.46999740600586,
      "activations/layer11_attention_weight_min": -27.374744415283203,
      "activations/layer12_attention_weight_max": 24.643766403198242,
      "activations/layer12_attention_weight_min": -24.76190185546875,
      "activations/layer13_attention_weight_max": 36.560298919677734,
      "activations/layer13_attention_weight_min": -30.86259651184082,
      "activations/layer14_attention_weight_max": 48.42545700073242,
      "activations/layer14_attention_weight_min": -40.663536071777344,
      "activations/layer15_attention_weight_max": 34.186256408691406,
      "activations/layer15_attention_weight_min": -27.588716506958008,
      "activations/layer16_attention_weight_max": 32.386653900146484,
      "activations/layer16_attention_weight_min": -28.805828094482422,
      "activations/layer17_attention_weight_max": 34.0076904296875,
      "activations/layer17_attention_weight_min": -23.6010684967041,
      "activations/layer18_attention_weight_max": 29.976675033569336,
      "activations/layer18_attention_weight_min": -20.699600219726562,
      "activations/layer19_attention_weight_max": 36.55946350097656,
      "activations/layer19_attention_weight_min": -29.38472557067871,
      "activations/layer1_attention_weight_max": 16.535425186157227,
      "activations/layer1_attention_weight_min": -13.145089149475098,
      "activations/layer20_attention_weight_max": 31.454694747924805,
      "activations/layer20_attention_weight_min": -22.79730796813965,
      "activations/layer21_attention_weight_max": 28.918752670288086,
      "activations/layer21_attention_weight_min": -21.413270950317383,
      "activations/layer22_attention_weight_max": 48.701072692871094,
      "activations/layer22_attention_weight_min": -30.59770393371582,
      "activations/layer23_attention_weight_max": 36.25743865966797,
      "activations/layer23_attention_weight_min": -23.21969985961914,
      "activations/layer2_attention_weight_max": 30.692434310913086,
      "activations/layer2_attention_weight_min": -30.435558319091797,
      "activations/layer3_attention_weight_max": 88.62236785888672,
      "activations/layer3_attention_weight_min": -88.02600860595703,
      "activations/layer4_attention_weight_max": 124.33795166015625,
      "activations/layer4_attention_weight_min": -118.39662170410156,
      "activations/layer5_attention_weight_max": 65.23169708251953,
      "activations/layer5_attention_weight_min": -62.436187744140625,
      "activations/layer6_attention_weight_max": 52.511173248291016,
      "activations/layer6_attention_weight_min": -53.14841079711914,
      "activations/layer7_attention_weight_max": 91.71055603027344,
      "activations/layer7_attention_weight_min": -84.87608337402344,
      "activations/layer8_attention_weight_max": 40.15898132324219,
      "activations/layer8_attention_weight_min": -39.60197830200195,
      "activations/layer9_attention_weight_max": 33.40839767456055,
      "activations/layer9_attention_weight_min": -32.94224166870117,
      "epoch": 7.29,
      "learning_rate": 0.00010405757575757575,
      "loss": 2.8214,
      "step": 125400
    },
    {
      "activations/layer0_attention_weight_max": 16.13686752319336,
      "activations/layer0_attention_weight_min": -11.473491668701172,
      "activations/layer10_attention_weight_max": 36.14917755126953,
      "activations/layer10_attention_weight_min": -36.727630615234375,
      "activations/layer11_attention_weight_max": 34.78139877319336,
      "activations/layer11_attention_weight_min": -32.67487716674805,
      "activations/layer12_attention_weight_max": 23.941707611083984,
      "activations/layer12_attention_weight_min": -28.2321720123291,
      "activations/layer13_attention_weight_max": 41.77757263183594,
      "activations/layer13_attention_weight_min": -32.59889221191406,
      "activations/layer14_attention_weight_max": 56.59882354736328,
      "activations/layer14_attention_weight_min": -43.3746223449707,
      "activations/layer15_attention_weight_max": 36.8657341003418,
      "activations/layer15_attention_weight_min": -30.199234008789062,
      "activations/layer16_attention_weight_max": 33.82380676269531,
      "activations/layer16_attention_weight_min": -30.306655883789062,
      "activations/layer17_attention_weight_max": 36.140350341796875,
      "activations/layer17_attention_weight_min": -27.960786819458008,
      "activations/layer18_attention_weight_max": 32.137027740478516,
      "activations/layer18_attention_weight_min": -20.020132064819336,
      "activations/layer19_attention_weight_max": 39.32057189941406,
      "activations/layer19_attention_weight_min": -31.17893409729004,
      "activations/layer1_attention_weight_max": 17.224699020385742,
      "activations/layer1_attention_weight_min": -13.9035005569458,
      "activations/layer20_attention_weight_max": 34.05662536621094,
      "activations/layer20_attention_weight_min": -27.044315338134766,
      "activations/layer21_attention_weight_max": 31.714330673217773,
      "activations/layer21_attention_weight_min": -22.36496925354004,
      "activations/layer22_attention_weight_max": 46.83745574951172,
      "activations/layer22_attention_weight_min": -28.20490837097168,
      "activations/layer23_attention_weight_max": 37.16086196899414,
      "activations/layer23_attention_weight_min": -25.555049896240234,
      "activations/layer2_attention_weight_max": 31.72701072692871,
      "activations/layer2_attention_weight_min": -34.831443786621094,
      "activations/layer3_attention_weight_max": 97.28933715820312,
      "activations/layer3_attention_weight_min": -98.4606704711914,
      "activations/layer4_attention_weight_max": 133.91957092285156,
      "activations/layer4_attention_weight_min": -137.3939208984375,
      "activations/layer5_attention_weight_max": 66.26432800292969,
      "activations/layer5_attention_weight_min": -71.76148986816406,
      "activations/layer6_attention_weight_max": 54.877418518066406,
      "activations/layer6_attention_weight_min": -58.02235412597656,
      "activations/layer7_attention_weight_max": 95.05862426757812,
      "activations/layer7_attention_weight_min": -105.77286529541016,
      "activations/layer8_attention_weight_max": 44.119686126708984,
      "activations/layer8_attention_weight_min": -45.67798614501953,
      "activations/layer9_attention_weight_max": 39.402915954589844,
      "activations/layer9_attention_weight_min": -37.93463897705078,
      "epoch": 7.29,
      "learning_rate": 0.00010403863636363634,
      "loss": 2.8207,
      "step": 125450
    },
    {
      "activations/layer0_attention_weight_max": 15.266325950622559,
      "activations/layer0_attention_weight_min": -11.908014297485352,
      "activations/layer10_attention_weight_max": 35.19639587402344,
      "activations/layer10_attention_weight_min": -30.381975173950195,
      "activations/layer11_attention_weight_max": 36.248016357421875,
      "activations/layer11_attention_weight_min": -29.53841209411621,
      "activations/layer12_attention_weight_max": 26.72594451904297,
      "activations/layer12_attention_weight_min": -26.202714920043945,
      "activations/layer13_attention_weight_max": 42.964717864990234,
      "activations/layer13_attention_weight_min": -31.998659133911133,
      "activations/layer14_attention_weight_max": 49.73893356323242,
      "activations/layer14_attention_weight_min": -39.913291931152344,
      "activations/layer15_attention_weight_max": 35.459449768066406,
      "activations/layer15_attention_weight_min": -31.359878540039062,
      "activations/layer16_attention_weight_max": 34.048583984375,
      "activations/layer16_attention_weight_min": -26.18299674987793,
      "activations/layer17_attention_weight_max": 36.739803314208984,
      "activations/layer17_attention_weight_min": -23.603900909423828,
      "activations/layer18_attention_weight_max": 30.084909439086914,
      "activations/layer18_attention_weight_min": -20.58956527709961,
      "activations/layer19_attention_weight_max": 34.679412841796875,
      "activations/layer19_attention_weight_min": -29.48044204711914,
      "activations/layer1_attention_weight_max": 17.00164222717285,
      "activations/layer1_attention_weight_min": -14.38757038116455,
      "activations/layer20_attention_weight_max": 30.32738494873047,
      "activations/layer20_attention_weight_min": -23.055458068847656,
      "activations/layer21_attention_weight_max": 30.74274253845215,
      "activations/layer21_attention_weight_min": -21.2441349029541,
      "activations/layer22_attention_weight_max": 45.608055114746094,
      "activations/layer22_attention_weight_min": -27.56279945373535,
      "activations/layer23_attention_weight_max": 33.92456817626953,
      "activations/layer23_attention_weight_min": -22.401920318603516,
      "activations/layer2_attention_weight_max": 34.33073043823242,
      "activations/layer2_attention_weight_min": -33.79120635986328,
      "activations/layer3_attention_weight_max": 96.37248229980469,
      "activations/layer3_attention_weight_min": -97.65143585205078,
      "activations/layer4_attention_weight_max": 123.93096923828125,
      "activations/layer4_attention_weight_min": -122.73943328857422,
      "activations/layer5_attention_weight_max": 66.23779296875,
      "activations/layer5_attention_weight_min": -65.79773712158203,
      "activations/layer6_attention_weight_max": 51.59981155395508,
      "activations/layer6_attention_weight_min": -52.73893737792969,
      "activations/layer7_attention_weight_max": 85.67584991455078,
      "activations/layer7_attention_weight_min": -88.50379943847656,
      "activations/layer8_attention_weight_max": 40.10834503173828,
      "activations/layer8_attention_weight_min": -39.99242401123047,
      "activations/layer9_attention_weight_max": 35.54378890991211,
      "activations/layer9_attention_weight_min": -34.9705924987793,
      "epoch": 7.29,
      "learning_rate": 0.00010401969696969696,
      "loss": 2.7951,
      "step": 125500
    },
    {
      "activations/layer0_attention_weight_max": 16.442092895507812,
      "activations/layer0_attention_weight_min": -11.036606788635254,
      "activations/layer10_attention_weight_max": 36.0078010559082,
      "activations/layer10_attention_weight_min": -32.82380294799805,
      "activations/layer11_attention_weight_max": 38.289833068847656,
      "activations/layer11_attention_weight_min": -30.119342803955078,
      "activations/layer12_attention_weight_max": 22.99077796936035,
      "activations/layer12_attention_weight_min": -24.85068702697754,
      "activations/layer13_attention_weight_max": 38.88088607788086,
      "activations/layer13_attention_weight_min": -31.2114200592041,
      "activations/layer14_attention_weight_max": 54.701507568359375,
      "activations/layer14_attention_weight_min": -40.03121566772461,
      "activations/layer15_attention_weight_max": 40.61268615722656,
      "activations/layer15_attention_weight_min": -29.014957427978516,
      "activations/layer16_attention_weight_max": 34.154109954833984,
      "activations/layer16_attention_weight_min": -28.894392013549805,
      "activations/layer17_attention_weight_max": 32.68924331665039,
      "activations/layer17_attention_weight_min": -24.37354278564453,
      "activations/layer18_attention_weight_max": 30.670406341552734,
      "activations/layer18_attention_weight_min": -21.73597526550293,
      "activations/layer19_attention_weight_max": 38.16182327270508,
      "activations/layer19_attention_weight_min": -30.249176025390625,
      "activations/layer1_attention_weight_max": 17.119260787963867,
      "activations/layer1_attention_weight_min": -14.709303855895996,
      "activations/layer20_attention_weight_max": 29.100801467895508,
      "activations/layer20_attention_weight_min": -23.700124740600586,
      "activations/layer21_attention_weight_max": 29.397783279418945,
      "activations/layer21_attention_weight_min": -22.355335235595703,
      "activations/layer22_attention_weight_max": 50.169429779052734,
      "activations/layer22_attention_weight_min": -28.69526481628418,
      "activations/layer23_attention_weight_max": 34.921043395996094,
      "activations/layer23_attention_weight_min": -22.638099670410156,
      "activations/layer2_attention_weight_max": 32.686763763427734,
      "activations/layer2_attention_weight_min": -32.2515754699707,
      "activations/layer3_attention_weight_max": 89.28902435302734,
      "activations/layer3_attention_weight_min": -90.74082946777344,
      "activations/layer4_attention_weight_max": 119.45853424072266,
      "activations/layer4_attention_weight_min": -121.3598403930664,
      "activations/layer5_attention_weight_max": 60.92708969116211,
      "activations/layer5_attention_weight_min": -63.13576889038086,
      "activations/layer6_attention_weight_max": 49.07588577270508,
      "activations/layer6_attention_weight_min": -53.01511764526367,
      "activations/layer7_attention_weight_max": 87.52239227294922,
      "activations/layer7_attention_weight_min": -89.9833755493164,
      "activations/layer8_attention_weight_max": 40.104759216308594,
      "activations/layer8_attention_weight_min": -41.420196533203125,
      "activations/layer9_attention_weight_max": 37.004913330078125,
      "activations/layer9_attention_weight_min": -36.37123489379883,
      "epoch": 7.3,
      "learning_rate": 0.00010400075757575757,
      "loss": 2.8123,
      "step": 125550
    },
    {
      "activations/layer0_attention_weight_max": 15.654829025268555,
      "activations/layer0_attention_weight_min": -11.902826309204102,
      "activations/layer10_attention_weight_max": 29.44150733947754,
      "activations/layer10_attention_weight_min": -30.450613021850586,
      "activations/layer11_attention_weight_max": 30.661436080932617,
      "activations/layer11_attention_weight_min": -29.788482666015625,
      "activations/layer12_attention_weight_max": 21.83719253540039,
      "activations/layer12_attention_weight_min": -26.28937339782715,
      "activations/layer13_attention_weight_max": 36.518829345703125,
      "activations/layer13_attention_weight_min": -31.075733184814453,
      "activations/layer14_attention_weight_max": 46.51920700073242,
      "activations/layer14_attention_weight_min": -37.6887321472168,
      "activations/layer15_attention_weight_max": 37.56855773925781,
      "activations/layer15_attention_weight_min": -28.35910415649414,
      "activations/layer16_attention_weight_max": 30.690645217895508,
      "activations/layer16_attention_weight_min": -27.53199577331543,
      "activations/layer17_attention_weight_max": 36.12215805053711,
      "activations/layer17_attention_weight_min": -25.198976516723633,
      "activations/layer18_attention_weight_max": 32.018497467041016,
      "activations/layer18_attention_weight_min": -19.932693481445312,
      "activations/layer19_attention_weight_max": 35.8708610534668,
      "activations/layer19_attention_weight_min": -29.4542179107666,
      "activations/layer1_attention_weight_max": 16.51468276977539,
      "activations/layer1_attention_weight_min": -15.313796997070312,
      "activations/layer20_attention_weight_max": 28.16215705871582,
      "activations/layer20_attention_weight_min": -24.000640869140625,
      "activations/layer21_attention_weight_max": 29.017826080322266,
      "activations/layer21_attention_weight_min": -21.458663940429688,
      "activations/layer22_attention_weight_max": 46.69619369506836,
      "activations/layer22_attention_weight_min": -27.074527740478516,
      "activations/layer23_attention_weight_max": 33.678279876708984,
      "activations/layer23_attention_weight_min": -23.034503936767578,
      "activations/layer2_attention_weight_max": 30.69941520690918,
      "activations/layer2_attention_weight_min": -31.66875457763672,
      "activations/layer3_attention_weight_max": 89.0440673828125,
      "activations/layer3_attention_weight_min": -93.10888671875,
      "activations/layer4_attention_weight_max": 115.70819091796875,
      "activations/layer4_attention_weight_min": -114.97308349609375,
      "activations/layer5_attention_weight_max": 60.02340316772461,
      "activations/layer5_attention_weight_min": -64.57376098632812,
      "activations/layer6_attention_weight_max": 48.35154342651367,
      "activations/layer6_attention_weight_min": -52.0638313293457,
      "activations/layer7_attention_weight_max": 86.75157165527344,
      "activations/layer7_attention_weight_min": -82.49592590332031,
      "activations/layer8_attention_weight_max": 37.29904556274414,
      "activations/layer8_attention_weight_min": -39.338462829589844,
      "activations/layer9_attention_weight_max": 32.766151428222656,
      "activations/layer9_attention_weight_min": -35.25489807128906,
      "epoch": 7.3,
      "learning_rate": 0.00010398181818181816,
      "loss": 2.8024,
      "step": 125600
    },
    {
      "activations/layer0_attention_weight_max": 16.337604522705078,
      "activations/layer0_attention_weight_min": -11.710627555847168,
      "activations/layer10_attention_weight_max": 33.57801055908203,
      "activations/layer10_attention_weight_min": -31.35199546813965,
      "activations/layer11_attention_weight_max": 38.271156311035156,
      "activations/layer11_attention_weight_min": -29.19385528564453,
      "activations/layer12_attention_weight_max": 21.60340118408203,
      "activations/layer12_attention_weight_min": -26.262903213500977,
      "activations/layer13_attention_weight_max": 44.4827995300293,
      "activations/layer13_attention_weight_min": -31.42289924621582,
      "activations/layer14_attention_weight_max": 61.88374710083008,
      "activations/layer14_attention_weight_min": -42.239349365234375,
      "activations/layer15_attention_weight_max": 36.1187744140625,
      "activations/layer15_attention_weight_min": -27.953527450561523,
      "activations/layer16_attention_weight_max": 33.21489334106445,
      "activations/layer16_attention_weight_min": -27.648239135742188,
      "activations/layer17_attention_weight_max": 33.59657669067383,
      "activations/layer17_attention_weight_min": -24.149585723876953,
      "activations/layer18_attention_weight_max": 29.979881286621094,
      "activations/layer18_attention_weight_min": -20.878957748413086,
      "activations/layer19_attention_weight_max": 33.35474395751953,
      "activations/layer19_attention_weight_min": -29.517248153686523,
      "activations/layer1_attention_weight_max": 17.793399810791016,
      "activations/layer1_attention_weight_min": -14.424623489379883,
      "activations/layer20_attention_weight_max": 30.068092346191406,
      "activations/layer20_attention_weight_min": -21.479507446289062,
      "activations/layer21_attention_weight_max": 32.191436767578125,
      "activations/layer21_attention_weight_min": -20.68511390686035,
      "activations/layer22_attention_weight_max": 45.674072265625,
      "activations/layer22_attention_weight_min": -28.54587173461914,
      "activations/layer23_attention_weight_max": 34.38854217529297,
      "activations/layer23_attention_weight_min": -21.84166717529297,
      "activations/layer2_attention_weight_max": 34.1484260559082,
      "activations/layer2_attention_weight_min": -31.325164794921875,
      "activations/layer3_attention_weight_max": 89.35786437988281,
      "activations/layer3_attention_weight_min": -90.08116912841797,
      "activations/layer4_attention_weight_max": 119.4816665649414,
      "activations/layer4_attention_weight_min": -116.55098724365234,
      "activations/layer5_attention_weight_max": 60.18313217163086,
      "activations/layer5_attention_weight_min": -64.00930786132812,
      "activations/layer6_attention_weight_max": 49.3062629699707,
      "activations/layer6_attention_weight_min": -54.61531066894531,
      "activations/layer7_attention_weight_max": 86.79105377197266,
      "activations/layer7_attention_weight_min": -88.23358154296875,
      "activations/layer8_attention_weight_max": 40.93398666381836,
      "activations/layer8_attention_weight_min": -38.97364807128906,
      "activations/layer9_attention_weight_max": 33.300289154052734,
      "activations/layer9_attention_weight_min": -33.08474349975586,
      "epoch": 7.3,
      "learning_rate": 0.00010396287878787878,
      "loss": 2.8139,
      "step": 125650
    },
    {
      "activations/layer0_attention_weight_max": 16.650299072265625,
      "activations/layer0_attention_weight_min": -11.700258255004883,
      "activations/layer10_attention_weight_max": 32.87925338745117,
      "activations/layer10_attention_weight_min": -32.899169921875,
      "activations/layer11_attention_weight_max": 31.528060913085938,
      "activations/layer11_attention_weight_min": -31.250141143798828,
      "activations/layer12_attention_weight_max": 23.01181411743164,
      "activations/layer12_attention_weight_min": -24.957901000976562,
      "activations/layer13_attention_weight_max": 43.831886291503906,
      "activations/layer13_attention_weight_min": -34.719749450683594,
      "activations/layer14_attention_weight_max": 51.739356994628906,
      "activations/layer14_attention_weight_min": -43.518836975097656,
      "activations/layer15_attention_weight_max": 36.47721481323242,
      "activations/layer15_attention_weight_min": -29.638185501098633,
      "activations/layer16_attention_weight_max": 30.91490936279297,
      "activations/layer16_attention_weight_min": -28.467029571533203,
      "activations/layer17_attention_weight_max": 35.03240966796875,
      "activations/layer17_attention_weight_min": -25.749156951904297,
      "activations/layer18_attention_weight_max": 34.80978012084961,
      "activations/layer18_attention_weight_min": -20.29498863220215,
      "activations/layer19_attention_weight_max": 37.68989944458008,
      "activations/layer19_attention_weight_min": -28.6319637298584,
      "activations/layer1_attention_weight_max": 17.43606948852539,
      "activations/layer1_attention_weight_min": -13.871262550354004,
      "activations/layer20_attention_weight_max": 31.919282913208008,
      "activations/layer20_attention_weight_min": -23.840478897094727,
      "activations/layer21_attention_weight_max": 34.677490234375,
      "activations/layer21_attention_weight_min": -21.071611404418945,
      "activations/layer22_attention_weight_max": 52.01885986328125,
      "activations/layer22_attention_weight_min": -25.657238006591797,
      "activations/layer23_attention_weight_max": 36.50767135620117,
      "activations/layer23_attention_weight_min": -23.217527389526367,
      "activations/layer2_attention_weight_max": 32.15032958984375,
      "activations/layer2_attention_weight_min": -32.079742431640625,
      "activations/layer3_attention_weight_max": 93.59503936767578,
      "activations/layer3_attention_weight_min": -94.25008392333984,
      "activations/layer4_attention_weight_max": 119.49061584472656,
      "activations/layer4_attention_weight_min": -117.40885162353516,
      "activations/layer5_attention_weight_max": 60.690406799316406,
      "activations/layer5_attention_weight_min": -61.886024475097656,
      "activations/layer6_attention_weight_max": 50.45024490356445,
      "activations/layer6_attention_weight_min": -52.01474380493164,
      "activations/layer7_attention_weight_max": 82.72071075439453,
      "activations/layer7_attention_weight_min": -84.50102996826172,
      "activations/layer8_attention_weight_max": 39.3067626953125,
      "activations/layer8_attention_weight_min": -38.848915100097656,
      "activations/layer9_attention_weight_max": 37.5583610534668,
      "activations/layer9_attention_weight_min": -33.92985153198242,
      "epoch": 7.3,
      "learning_rate": 0.00010394393939393939,
      "loss": 2.8007,
      "step": 125700
    },
    {
      "activations/layer0_attention_weight_max": 16.41054916381836,
      "activations/layer0_attention_weight_min": -12.446331024169922,
      "activations/layer10_attention_weight_max": 34.31214141845703,
      "activations/layer10_attention_weight_min": -31.64534568786621,
      "activations/layer11_attention_weight_max": 34.868282318115234,
      "activations/layer11_attention_weight_min": -30.260066986083984,
      "activations/layer12_attention_weight_max": 25.080032348632812,
      "activations/layer12_attention_weight_min": -26.80262565612793,
      "activations/layer13_attention_weight_max": 47.53071975708008,
      "activations/layer13_attention_weight_min": -32.95765686035156,
      "activations/layer14_attention_weight_max": 59.240135192871094,
      "activations/layer14_attention_weight_min": -46.179386138916016,
      "activations/layer15_attention_weight_max": 39.10265350341797,
      "activations/layer15_attention_weight_min": -31.599084854125977,
      "activations/layer16_attention_weight_max": 35.2967643737793,
      "activations/layer16_attention_weight_min": -29.006383895874023,
      "activations/layer17_attention_weight_max": 37.816192626953125,
      "activations/layer17_attention_weight_min": -24.269935607910156,
      "activations/layer18_attention_weight_max": 35.15858840942383,
      "activations/layer18_attention_weight_min": -20.932443618774414,
      "activations/layer19_attention_weight_max": 37.7495231628418,
      "activations/layer19_attention_weight_min": -31.929462432861328,
      "activations/layer1_attention_weight_max": 16.727039337158203,
      "activations/layer1_attention_weight_min": -14.09390640258789,
      "activations/layer20_attention_weight_max": 33.96502685546875,
      "activations/layer20_attention_weight_min": -25.16585922241211,
      "activations/layer21_attention_weight_max": 32.71575927734375,
      "activations/layer21_attention_weight_min": -23.49474334716797,
      "activations/layer22_attention_weight_max": 53.86558532714844,
      "activations/layer22_attention_weight_min": -28.479475021362305,
      "activations/layer23_attention_weight_max": 42.415706634521484,
      "activations/layer23_attention_weight_min": -23.748035430908203,
      "activations/layer2_attention_weight_max": 32.7525520324707,
      "activations/layer2_attention_weight_min": -33.51042175292969,
      "activations/layer3_attention_weight_max": 91.51805877685547,
      "activations/layer3_attention_weight_min": -95.15606689453125,
      "activations/layer4_attention_weight_max": 122.06591033935547,
      "activations/layer4_attention_weight_min": -120.0907211303711,
      "activations/layer5_attention_weight_max": 65.65179443359375,
      "activations/layer5_attention_weight_min": -63.08332443237305,
      "activations/layer6_attention_weight_max": 53.599952697753906,
      "activations/layer6_attention_weight_min": -53.905174255371094,
      "activations/layer7_attention_weight_max": 90.90288543701172,
      "activations/layer7_attention_weight_min": -91.22527313232422,
      "activations/layer8_attention_weight_max": 41.126625061035156,
      "activations/layer8_attention_weight_min": -40.29318618774414,
      "activations/layer9_attention_weight_max": 35.69630813598633,
      "activations/layer9_attention_weight_min": -34.38591766357422,
      "epoch": 7.31,
      "learning_rate": 0.00010392499999999999,
      "loss": 2.801,
      "step": 125750
    },
    {
      "activations/layer0_attention_weight_max": 15.906279563903809,
      "activations/layer0_attention_weight_min": -10.664212226867676,
      "activations/layer10_attention_weight_max": 34.00273895263672,
      "activations/layer10_attention_weight_min": -32.308475494384766,
      "activations/layer11_attention_weight_max": 34.25242614746094,
      "activations/layer11_attention_weight_min": -32.51008224487305,
      "activations/layer12_attention_weight_max": 23.6341609954834,
      "activations/layer12_attention_weight_min": -26.13957405090332,
      "activations/layer13_attention_weight_max": 42.105224609375,
      "activations/layer13_attention_weight_min": -32.81437683105469,
      "activations/layer14_attention_weight_max": 57.93748092651367,
      "activations/layer14_attention_weight_min": -43.39976119995117,
      "activations/layer15_attention_weight_max": 35.15538024902344,
      "activations/layer15_attention_weight_min": -30.06739616394043,
      "activations/layer16_attention_weight_max": 32.50200271606445,
      "activations/layer16_attention_weight_min": -28.003923416137695,
      "activations/layer17_attention_weight_max": 34.324790954589844,
      "activations/layer17_attention_weight_min": -25.351470947265625,
      "activations/layer18_attention_weight_max": 33.001708984375,
      "activations/layer18_attention_weight_min": -22.493194580078125,
      "activations/layer19_attention_weight_max": 38.39202880859375,
      "activations/layer19_attention_weight_min": -29.37775230407715,
      "activations/layer1_attention_weight_max": 18.572715759277344,
      "activations/layer1_attention_weight_min": -16.89338493347168,
      "activations/layer20_attention_weight_max": 30.5814266204834,
      "activations/layer20_attention_weight_min": -24.23661231994629,
      "activations/layer21_attention_weight_max": 29.84600067138672,
      "activations/layer21_attention_weight_min": -22.141836166381836,
      "activations/layer22_attention_weight_max": 46.0912971496582,
      "activations/layer22_attention_weight_min": -29.353193283081055,
      "activations/layer23_attention_weight_max": 33.85052490234375,
      "activations/layer23_attention_weight_min": -24.126155853271484,
      "activations/layer2_attention_weight_max": 31.817554473876953,
      "activations/layer2_attention_weight_min": -33.10160827636719,
      "activations/layer3_attention_weight_max": 97.16717529296875,
      "activations/layer3_attention_weight_min": -96.2767333984375,
      "activations/layer4_attention_weight_max": 126.3918685913086,
      "activations/layer4_attention_weight_min": -121.51395416259766,
      "activations/layer5_attention_weight_max": 63.22361373901367,
      "activations/layer5_attention_weight_min": -62.19612121582031,
      "activations/layer6_attention_weight_max": 49.83063507080078,
      "activations/layer6_attention_weight_min": -52.93059539794922,
      "activations/layer7_attention_weight_max": 92.48372650146484,
      "activations/layer7_attention_weight_min": -94.2472152709961,
      "activations/layer8_attention_weight_max": 40.63957977294922,
      "activations/layer8_attention_weight_min": -40.905086517333984,
      "activations/layer9_attention_weight_max": 37.04965591430664,
      "activations/layer9_attention_weight_min": -35.987606048583984,
      "epoch": 7.31,
      "learning_rate": 0.0001039060606060606,
      "loss": 2.8179,
      "step": 125800
    },
    {
      "activations/layer0_attention_weight_max": 15.260494232177734,
      "activations/layer0_attention_weight_min": -10.994324684143066,
      "activations/layer10_attention_weight_max": 33.773353576660156,
      "activations/layer10_attention_weight_min": -30.298593521118164,
      "activations/layer11_attention_weight_max": 33.275333404541016,
      "activations/layer11_attention_weight_min": -27.59811019897461,
      "activations/layer12_attention_weight_max": 23.38326644897461,
      "activations/layer12_attention_weight_min": -24.29192543029785,
      "activations/layer13_attention_weight_max": 39.75630569458008,
      "activations/layer13_attention_weight_min": -32.91682052612305,
      "activations/layer14_attention_weight_max": 49.258731842041016,
      "activations/layer14_attention_weight_min": -40.9096565246582,
      "activations/layer15_attention_weight_max": 36.16948318481445,
      "activations/layer15_attention_weight_min": -29.273365020751953,
      "activations/layer16_attention_weight_max": 33.900943756103516,
      "activations/layer16_attention_weight_min": -27.563247680664062,
      "activations/layer17_attention_weight_max": 34.61159896850586,
      "activations/layer17_attention_weight_min": -25.2731990814209,
      "activations/layer18_attention_weight_max": 32.946693420410156,
      "activations/layer18_attention_weight_min": -21.484525680541992,
      "activations/layer19_attention_weight_max": 36.86182403564453,
      "activations/layer19_attention_weight_min": -29.089645385742188,
      "activations/layer1_attention_weight_max": 18.91292381286621,
      "activations/layer1_attention_weight_min": -15.07551097869873,
      "activations/layer20_attention_weight_max": 31.748973846435547,
      "activations/layer20_attention_weight_min": -23.21912384033203,
      "activations/layer21_attention_weight_max": 30.840898513793945,
      "activations/layer21_attention_weight_min": -23.00027084350586,
      "activations/layer22_attention_weight_max": 50.32554626464844,
      "activations/layer22_attention_weight_min": -27.93777084350586,
      "activations/layer23_attention_weight_max": 38.337982177734375,
      "activations/layer23_attention_weight_min": -23.049068450927734,
      "activations/layer2_attention_weight_max": 31.502212524414062,
      "activations/layer2_attention_weight_min": -31.99993896484375,
      "activations/layer3_attention_weight_max": 88.3667221069336,
      "activations/layer3_attention_weight_min": -91.38309478759766,
      "activations/layer4_attention_weight_max": 113.44420623779297,
      "activations/layer4_attention_weight_min": -113.34647369384766,
      "activations/layer5_attention_weight_max": 59.283111572265625,
      "activations/layer5_attention_weight_min": -59.9058837890625,
      "activations/layer6_attention_weight_max": 49.41374588012695,
      "activations/layer6_attention_weight_min": -49.62277603149414,
      "activations/layer7_attention_weight_max": 86.86311340332031,
      "activations/layer7_attention_weight_min": -84.49667358398438,
      "activations/layer8_attention_weight_max": 38.71001434326172,
      "activations/layer8_attention_weight_min": -39.758480072021484,
      "activations/layer9_attention_weight_max": 35.62610626220703,
      "activations/layer9_attention_weight_min": -33.503902435302734,
      "epoch": 7.31,
      "learning_rate": 0.00010388712121212121,
      "loss": 2.8107,
      "step": 125850
    },
    {
      "activations/layer0_attention_weight_max": 16.01629066467285,
      "activations/layer0_attention_weight_min": -11.371670722961426,
      "activations/layer10_attention_weight_max": 32.26097869873047,
      "activations/layer10_attention_weight_min": -34.092159271240234,
      "activations/layer11_attention_weight_max": 37.2744140625,
      "activations/layer11_attention_weight_min": -34.67097091674805,
      "activations/layer12_attention_weight_max": 27.233646392822266,
      "activations/layer12_attention_weight_min": -24.654813766479492,
      "activations/layer13_attention_weight_max": 49.44124221801758,
      "activations/layer13_attention_weight_min": -33.15291213989258,
      "activations/layer14_attention_weight_max": 57.9439697265625,
      "activations/layer14_attention_weight_min": -41.8475456237793,
      "activations/layer15_attention_weight_max": 49.005977630615234,
      "activations/layer15_attention_weight_min": -29.176929473876953,
      "activations/layer16_attention_weight_max": 36.1348991394043,
      "activations/layer16_attention_weight_min": -26.720518112182617,
      "activations/layer17_attention_weight_max": 42.47918701171875,
      "activations/layer17_attention_weight_min": -23.313669204711914,
      "activations/layer18_attention_weight_max": 35.89204025268555,
      "activations/layer18_attention_weight_min": -21.845922470092773,
      "activations/layer19_attention_weight_max": 42.60068130493164,
      "activations/layer19_attention_weight_min": -28.820302963256836,
      "activations/layer1_attention_weight_max": 18.73613929748535,
      "activations/layer1_attention_weight_min": -14.559979438781738,
      "activations/layer20_attention_weight_max": 37.14680862426758,
      "activations/layer20_attention_weight_min": -22.60135269165039,
      "activations/layer21_attention_weight_max": 35.91968536376953,
      "activations/layer21_attention_weight_min": -20.69886589050293,
      "activations/layer22_attention_weight_max": 50.784366607666016,
      "activations/layer22_attention_weight_min": -26.280345916748047,
      "activations/layer23_attention_weight_max": 37.24128723144531,
      "activations/layer23_attention_weight_min": -20.017227172851562,
      "activations/layer2_attention_weight_max": 31.431324005126953,
      "activations/layer2_attention_weight_min": -30.864017486572266,
      "activations/layer3_attention_weight_max": 93.98587799072266,
      "activations/layer3_attention_weight_min": -92.89604187011719,
      "activations/layer4_attention_weight_max": 120.93087005615234,
      "activations/layer4_attention_weight_min": -123.1888656616211,
      "activations/layer5_attention_weight_max": 62.783836364746094,
      "activations/layer5_attention_weight_min": -62.272125244140625,
      "activations/layer6_attention_weight_max": 53.74058151245117,
      "activations/layer6_attention_weight_min": -53.90319061279297,
      "activations/layer7_attention_weight_max": 88.07984161376953,
      "activations/layer7_attention_weight_min": -87.30645751953125,
      "activations/layer8_attention_weight_max": 39.51308059692383,
      "activations/layer8_attention_weight_min": -39.9560546875,
      "activations/layer9_attention_weight_max": 34.61820602416992,
      "activations/layer9_attention_weight_min": -32.59398651123047,
      "epoch": 7.32,
      "learning_rate": 0.00010386818181818181,
      "loss": 2.8256,
      "step": 125900
    },
    {
      "activations/layer0_attention_weight_max": 15.342916488647461,
      "activations/layer0_attention_weight_min": -11.951384544372559,
      "activations/layer10_attention_weight_max": 33.138671875,
      "activations/layer10_attention_weight_min": -30.99437713623047,
      "activations/layer11_attention_weight_max": 34.80122756958008,
      "activations/layer11_attention_weight_min": -30.047285079956055,
      "activations/layer12_attention_weight_max": 22.965253829956055,
      "activations/layer12_attention_weight_min": -24.41607666015625,
      "activations/layer13_attention_weight_max": 36.72274398803711,
      "activations/layer13_attention_weight_min": -34.27482604980469,
      "activations/layer14_attention_weight_max": 49.3182487487793,
      "activations/layer14_attention_weight_min": -45.83338928222656,
      "activations/layer15_attention_weight_max": 36.1421012878418,
      "activations/layer15_attention_weight_min": -29.318635940551758,
      "activations/layer16_attention_weight_max": 32.5965461730957,
      "activations/layer16_attention_weight_min": -27.902982711791992,
      "activations/layer17_attention_weight_max": 31.562211990356445,
      "activations/layer17_attention_weight_min": -25.551067352294922,
      "activations/layer18_attention_weight_max": 30.291112899780273,
      "activations/layer18_attention_weight_min": -21.985939025878906,
      "activations/layer19_attention_weight_max": 37.55447769165039,
      "activations/layer19_attention_weight_min": -32.91404724121094,
      "activations/layer1_attention_weight_max": 18.687013626098633,
      "activations/layer1_attention_weight_min": -15.53186321258545,
      "activations/layer20_attention_weight_max": 31.118635177612305,
      "activations/layer20_attention_weight_min": -27.703079223632812,
      "activations/layer21_attention_weight_max": 28.21117401123047,
      "activations/layer21_attention_weight_min": -24.223087310791016,
      "activations/layer22_attention_weight_max": 45.47685623168945,
      "activations/layer22_attention_weight_min": -31.16854476928711,
      "activations/layer23_attention_weight_max": 34.41365051269531,
      "activations/layer23_attention_weight_min": -25.763952255249023,
      "activations/layer2_attention_weight_max": 31.654178619384766,
      "activations/layer2_attention_weight_min": -32.07542419433594,
      "activations/layer3_attention_weight_max": 95.83744049072266,
      "activations/layer3_attention_weight_min": -98.39236450195312,
      "activations/layer4_attention_weight_max": 121.6875,
      "activations/layer4_attention_weight_min": -124.86175537109375,
      "activations/layer5_attention_weight_max": 64.86614227294922,
      "activations/layer5_attention_weight_min": -65.13491821289062,
      "activations/layer6_attention_weight_max": 48.49335479736328,
      "activations/layer6_attention_weight_min": -53.964698791503906,
      "activations/layer7_attention_weight_max": 96.3038101196289,
      "activations/layer7_attention_weight_min": -90.4373779296875,
      "activations/layer8_attention_weight_max": 39.37773513793945,
      "activations/layer8_attention_weight_min": -39.26041030883789,
      "activations/layer9_attention_weight_max": 35.42958068847656,
      "activations/layer9_attention_weight_min": -36.61252975463867,
      "epoch": 7.32,
      "learning_rate": 0.00010384924242424242,
      "loss": 2.8082,
      "step": 125950
    },
    {
      "activations/layer0_attention_weight_max": 15.36202621459961,
      "activations/layer0_attention_weight_min": -12.020319938659668,
      "activations/layer10_attention_weight_max": 31.769895553588867,
      "activations/layer10_attention_weight_min": -31.09187889099121,
      "activations/layer11_attention_weight_max": 31.98406219482422,
      "activations/layer11_attention_weight_min": -31.314315795898438,
      "activations/layer12_attention_weight_max": 25.600862503051758,
      "activations/layer12_attention_weight_min": -26.073348999023438,
      "activations/layer13_attention_weight_max": 36.741310119628906,
      "activations/layer13_attention_weight_min": -31.619937896728516,
      "activations/layer14_attention_weight_max": 45.8160400390625,
      "activations/layer14_attention_weight_min": -40.31346893310547,
      "activations/layer15_attention_weight_max": 43.65633010864258,
      "activations/layer15_attention_weight_min": -32.8832893371582,
      "activations/layer16_attention_weight_max": 34.67787551879883,
      "activations/layer16_attention_weight_min": -27.449260711669922,
      "activations/layer17_attention_weight_max": 34.82722854614258,
      "activations/layer17_attention_weight_min": -24.695098876953125,
      "activations/layer18_attention_weight_max": 33.92356491088867,
      "activations/layer18_attention_weight_min": -22.15031623840332,
      "activations/layer19_attention_weight_max": 38.30736541748047,
      "activations/layer19_attention_weight_min": -30.4355525970459,
      "activations/layer1_attention_weight_max": 16.802936553955078,
      "activations/layer1_attention_weight_min": -15.811406135559082,
      "activations/layer20_attention_weight_max": 34.705875396728516,
      "activations/layer20_attention_weight_min": -23.5997257232666,
      "activations/layer21_attention_weight_max": 34.18841552734375,
      "activations/layer21_attention_weight_min": -21.93060302734375,
      "activations/layer22_attention_weight_max": 53.435855865478516,
      "activations/layer22_attention_weight_min": -28.94412612915039,
      "activations/layer23_attention_weight_max": 38.372886657714844,
      "activations/layer23_attention_weight_min": -25.495071411132812,
      "activations/layer2_attention_weight_max": 33.31770706176758,
      "activations/layer2_attention_weight_min": -30.40081024169922,
      "activations/layer3_attention_weight_max": 89.46195983886719,
      "activations/layer3_attention_weight_min": -89.19508361816406,
      "activations/layer4_attention_weight_max": 119.8967056274414,
      "activations/layer4_attention_weight_min": -118.7806167602539,
      "activations/layer5_attention_weight_max": 61.95066833496094,
      "activations/layer5_attention_weight_min": -64.10111999511719,
      "activations/layer6_attention_weight_max": 50.183109283447266,
      "activations/layer6_attention_weight_min": -52.42178726196289,
      "activations/layer7_attention_weight_max": 84.60398864746094,
      "activations/layer7_attention_weight_min": -84.54832458496094,
      "activations/layer8_attention_weight_max": 39.56947708129883,
      "activations/layer8_attention_weight_min": -39.37385940551758,
      "activations/layer9_attention_weight_max": 32.18794631958008,
      "activations/layer9_attention_weight_min": -31.75105857849121,
      "epoch": 7.32,
      "learning_rate": 0.00010383068181818182,
      "loss": 2.8109,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5022,
      "eval_samples_per_second": 505.043,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5022,
      "eval_openwebtext_samples_per_second": 505.043,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_wikitext_loss": 2.99609375,
      "eval_wikitext_ppl": 20.007230835729846,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.293,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_lambada_loss": 2.56640625,
      "eval_lambada_ppl": 13.018953406185608,
      "eval_lambada_runtime": 9.5939,
      "eval_lambada_samples_per_second": 507.508,
      "step": 126000
    },
    {
      "activations/layer0_attention_weight_max": 16.30091667175293,
      "activations/layer0_attention_weight_min": -11.240214347839355,
      "activations/layer10_attention_weight_max": 33.943603515625,
      "activations/layer10_attention_weight_min": -31.91924285888672,
      "activations/layer11_attention_weight_max": 31.973737716674805,
      "activations/layer11_attention_weight_min": -30.013187408447266,
      "activations/layer12_attention_weight_max": 28.06965446472168,
      "activations/layer12_attention_weight_min": -30.143970489501953,
      "activations/layer13_attention_weight_max": 38.34734344482422,
      "activations/layer13_attention_weight_min": -35.05788040161133,
      "activations/layer14_attention_weight_max": 49.9753532409668,
      "activations/layer14_attention_weight_min": -43.38212585449219,
      "activations/layer15_attention_weight_max": 36.96149444580078,
      "activations/layer15_attention_weight_min": -32.25572967529297,
      "activations/layer16_attention_weight_max": 32.45114517211914,
      "activations/layer16_attention_weight_min": -28.791797637939453,
      "activations/layer17_attention_weight_max": 35.23554611206055,
      "activations/layer17_attention_weight_min": -27.48413848876953,
      "activations/layer18_attention_weight_max": 32.753902435302734,
      "activations/layer18_attention_weight_min": -21.53520393371582,
      "activations/layer19_attention_weight_max": 36.135494232177734,
      "activations/layer19_attention_weight_min": -33.43871307373047,
      "activations/layer1_attention_weight_max": 16.61547088623047,
      "activations/layer1_attention_weight_min": -14.513270378112793,
      "activations/layer20_attention_weight_max": 31.898176193237305,
      "activations/layer20_attention_weight_min": -27.117557525634766,
      "activations/layer21_attention_weight_max": 31.372814178466797,
      "activations/layer21_attention_weight_min": -24.075590133666992,
      "activations/layer22_attention_weight_max": 47.399288177490234,
      "activations/layer22_attention_weight_min": -29.88004493713379,
      "activations/layer23_attention_weight_max": 36.00310516357422,
      "activations/layer23_attention_weight_min": -22.91314697265625,
      "activations/layer2_attention_weight_max": 32.16304016113281,
      "activations/layer2_attention_weight_min": -30.919912338256836,
      "activations/layer3_attention_weight_max": 91.5466079711914,
      "activations/layer3_attention_weight_min": -89.77978515625,
      "activations/layer4_attention_weight_max": 124.90312957763672,
      "activations/layer4_attention_weight_min": -124.08155822753906,
      "activations/layer5_attention_weight_max": 62.741546630859375,
      "activations/layer5_attention_weight_min": -62.08131408691406,
      "activations/layer6_attention_weight_max": 53.53504180908203,
      "activations/layer6_attention_weight_min": -52.838199615478516,
      "activations/layer7_attention_weight_max": 88.41920471191406,
      "activations/layer7_attention_weight_min": -86.94815826416016,
      "activations/layer8_attention_weight_max": 39.66560745239258,
      "activations/layer8_attention_weight_min": -40.7567253112793,
      "activations/layer9_attention_weight_max": 33.50651168823242,
      "activations/layer9_attention_weight_min": -32.89260482788086,
      "epoch": 7.32,
      "learning_rate": 0.00010381174242424242,
      "loss": 2.8036,
      "step": 126050
    },
    {
      "activations/layer0_attention_weight_max": 16.22806739807129,
      "activations/layer0_attention_weight_min": -12.119711875915527,
      "activations/layer10_attention_weight_max": 29.83907127380371,
      "activations/layer10_attention_weight_min": -32.041748046875,
      "activations/layer11_attention_weight_max": 31.652236938476562,
      "activations/layer11_attention_weight_min": -29.963764190673828,
      "activations/layer12_attention_weight_max": 21.4636287689209,
      "activations/layer12_attention_weight_min": -27.47327423095703,
      "activations/layer13_attention_weight_max": 38.85813522338867,
      "activations/layer13_attention_weight_min": -31.246191024780273,
      "activations/layer14_attention_weight_max": 46.70516586303711,
      "activations/layer14_attention_weight_min": -39.363059997558594,
      "activations/layer15_attention_weight_max": 37.529380798339844,
      "activations/layer15_attention_weight_min": -30.93353271484375,
      "activations/layer16_attention_weight_max": 31.006973266601562,
      "activations/layer16_attention_weight_min": -26.63344955444336,
      "activations/layer17_attention_weight_max": 35.52024841308594,
      "activations/layer17_attention_weight_min": -26.46282958984375,
      "activations/layer18_attention_weight_max": 32.442440032958984,
      "activations/layer18_attention_weight_min": -22.382280349731445,
      "activations/layer19_attention_weight_max": 34.954349517822266,
      "activations/layer19_attention_weight_min": -28.838825225830078,
      "activations/layer1_attention_weight_max": 16.71955108642578,
      "activations/layer1_attention_weight_min": -15.171048164367676,
      "activations/layer20_attention_weight_max": 30.306396484375,
      "activations/layer20_attention_weight_min": -24.131582260131836,
      "activations/layer21_attention_weight_max": 30.84423828125,
      "activations/layer21_attention_weight_min": -22.182668685913086,
      "activations/layer22_attention_weight_max": 45.69617462158203,
      "activations/layer22_attention_weight_min": -27.705184936523438,
      "activations/layer23_attention_weight_max": 32.3980598449707,
      "activations/layer23_attention_weight_min": -23.488277435302734,
      "activations/layer2_attention_weight_max": 31.093242645263672,
      "activations/layer2_attention_weight_min": -32.53564453125,
      "activations/layer3_attention_weight_max": 92.12591552734375,
      "activations/layer3_attention_weight_min": -95.41532897949219,
      "activations/layer4_attention_weight_max": 119.85262298583984,
      "activations/layer4_attention_weight_min": -120.8824462890625,
      "activations/layer5_attention_weight_max": 60.0556755065918,
      "activations/layer5_attention_weight_min": -62.44261932373047,
      "activations/layer6_attention_weight_max": 50.61781692504883,
      "activations/layer6_attention_weight_min": -52.271278381347656,
      "activations/layer7_attention_weight_max": 82.91106414794922,
      "activations/layer7_attention_weight_min": -87.47795867919922,
      "activations/layer8_attention_weight_max": 37.15217208862305,
      "activations/layer8_attention_weight_min": -39.682960510253906,
      "activations/layer9_attention_weight_max": 34.8707389831543,
      "activations/layer9_attention_weight_min": -32.695831298828125,
      "epoch": 7.33,
      "learning_rate": 0.00010379280303030302,
      "loss": 2.8202,
      "step": 126100
    },
    {
      "activations/layer0_attention_weight_max": 16.008699417114258,
      "activations/layer0_attention_weight_min": -11.408251762390137,
      "activations/layer10_attention_weight_max": 31.17329978942871,
      "activations/layer10_attention_weight_min": -29.101036071777344,
      "activations/layer11_attention_weight_max": 32.98488235473633,
      "activations/layer11_attention_weight_min": -28.597036361694336,
      "activations/layer12_attention_weight_max": 27.187885284423828,
      "activations/layer12_attention_weight_min": -27.36271858215332,
      "activations/layer13_attention_weight_max": 39.64384841918945,
      "activations/layer13_attention_weight_min": -33.66858673095703,
      "activations/layer14_attention_weight_max": 44.512176513671875,
      "activations/layer14_attention_weight_min": -36.65125274658203,
      "activations/layer15_attention_weight_max": 38.429813385009766,
      "activations/layer15_attention_weight_min": -29.91421127319336,
      "activations/layer16_attention_weight_max": 35.83293914794922,
      "activations/layer16_attention_weight_min": -28.53135871887207,
      "activations/layer17_attention_weight_max": 33.75542449951172,
      "activations/layer17_attention_weight_min": -24.453655242919922,
      "activations/layer18_attention_weight_max": 31.986614227294922,
      "activations/layer18_attention_weight_min": -21.632970809936523,
      "activations/layer19_attention_weight_max": 37.63217544555664,
      "activations/layer19_attention_weight_min": -29.814374923706055,
      "activations/layer1_attention_weight_max": 16.626495361328125,
      "activations/layer1_attention_weight_min": -13.94811725616455,
      "activations/layer20_attention_weight_max": 30.35736656188965,
      "activations/layer20_attention_weight_min": -23.420909881591797,
      "activations/layer21_attention_weight_max": 31.927518844604492,
      "activations/layer21_attention_weight_min": -21.64967918395996,
      "activations/layer22_attention_weight_max": 44.940250396728516,
      "activations/layer22_attention_weight_min": -28.307289123535156,
      "activations/layer23_attention_weight_max": 35.36601257324219,
      "activations/layer23_attention_weight_min": -22.49787139892578,
      "activations/layer2_attention_weight_max": 31.36830711364746,
      "activations/layer2_attention_weight_min": -31.484996795654297,
      "activations/layer3_attention_weight_max": 86.76063537597656,
      "activations/layer3_attention_weight_min": -92.91922760009766,
      "activations/layer4_attention_weight_max": 116.0786361694336,
      "activations/layer4_attention_weight_min": -117.50634002685547,
      "activations/layer5_attention_weight_max": 57.82049560546875,
      "activations/layer5_attention_weight_min": -61.933326721191406,
      "activations/layer6_attention_weight_max": 46.981056213378906,
      "activations/layer6_attention_weight_min": -51.35615539550781,
      "activations/layer7_attention_weight_max": 87.41624450683594,
      "activations/layer7_attention_weight_min": -86.02316284179688,
      "activations/layer8_attention_weight_max": 34.91901779174805,
      "activations/layer8_attention_weight_min": -37.706268310546875,
      "activations/layer9_attention_weight_max": 32.927425384521484,
      "activations/layer9_attention_weight_min": -31.615116119384766,
      "epoch": 7.33,
      "learning_rate": 0.00010377386363636363,
      "loss": 2.8092,
      "step": 126150
    },
    {
      "activations/layer0_attention_weight_max": 15.103763580322266,
      "activations/layer0_attention_weight_min": -11.208868026733398,
      "activations/layer10_attention_weight_max": 33.3764533996582,
      "activations/layer10_attention_weight_min": -31.407363891601562,
      "activations/layer11_attention_weight_max": 33.64936447143555,
      "activations/layer11_attention_weight_min": -29.21408462524414,
      "activations/layer12_attention_weight_max": 23.914140701293945,
      "activations/layer12_attention_weight_min": -27.772296905517578,
      "activations/layer13_attention_weight_max": 39.30006790161133,
      "activations/layer13_attention_weight_min": -33.19234085083008,
      "activations/layer14_attention_weight_max": 54.438411712646484,
      "activations/layer14_attention_weight_min": -38.13945388793945,
      "activations/layer15_attention_weight_max": 39.28356170654297,
      "activations/layer15_attention_weight_min": -29.293354034423828,
      "activations/layer16_attention_weight_max": 33.11771774291992,
      "activations/layer16_attention_weight_min": -28.851728439331055,
      "activations/layer17_attention_weight_max": 35.348899841308594,
      "activations/layer17_attention_weight_min": -24.980751037597656,
      "activations/layer18_attention_weight_max": 35.872161865234375,
      "activations/layer18_attention_weight_min": -21.702192306518555,
      "activations/layer19_attention_weight_max": 38.59576416015625,
      "activations/layer19_attention_weight_min": -31.592443466186523,
      "activations/layer1_attention_weight_max": 16.813159942626953,
      "activations/layer1_attention_weight_min": -14.37442684173584,
      "activations/layer20_attention_weight_max": 29.15788459777832,
      "activations/layer20_attention_weight_min": -27.459287643432617,
      "activations/layer21_attention_weight_max": 30.10941505432129,
      "activations/layer21_attention_weight_min": -23.247909545898438,
      "activations/layer22_attention_weight_max": 46.27096939086914,
      "activations/layer22_attention_weight_min": -28.765554428100586,
      "activations/layer23_attention_weight_max": 37.29762268066406,
      "activations/layer23_attention_weight_min": -21.65028953552246,
      "activations/layer2_attention_weight_max": 33.447547912597656,
      "activations/layer2_attention_weight_min": -34.255889892578125,
      "activations/layer3_attention_weight_max": 95.2132797241211,
      "activations/layer3_attention_weight_min": -97.66924285888672,
      "activations/layer4_attention_weight_max": 124.17337799072266,
      "activations/layer4_attention_weight_min": -120.75011444091797,
      "activations/layer5_attention_weight_max": 63.10609436035156,
      "activations/layer5_attention_weight_min": -63.896026611328125,
      "activations/layer6_attention_weight_max": 51.06877899169922,
      "activations/layer6_attention_weight_min": -53.697750091552734,
      "activations/layer7_attention_weight_max": 87.9864273071289,
      "activations/layer7_attention_weight_min": -90.15792083740234,
      "activations/layer8_attention_weight_max": 41.25795364379883,
      "activations/layer8_attention_weight_min": -39.404747009277344,
      "activations/layer9_attention_weight_max": 35.672855377197266,
      "activations/layer9_attention_weight_min": -32.98469924926758,
      "epoch": 7.33,
      "learning_rate": 0.00010375492424242422,
      "loss": 2.7946,
      "step": 126200
    },
    {
      "activations/layer0_attention_weight_max": 15.56305980682373,
      "activations/layer0_attention_weight_min": -11.552542686462402,
      "activations/layer10_attention_weight_max": 38.50237274169922,
      "activations/layer10_attention_weight_min": -34.10375213623047,
      "activations/layer11_attention_weight_max": 36.89239501953125,
      "activations/layer11_attention_weight_min": -32.21388626098633,
      "activations/layer12_attention_weight_max": 27.958219528198242,
      "activations/layer12_attention_weight_min": -24.918249130249023,
      "activations/layer13_attention_weight_max": 50.06467056274414,
      "activations/layer13_attention_weight_min": -34.29116439819336,
      "activations/layer14_attention_weight_max": 75.2500228881836,
      "activations/layer14_attention_weight_min": -49.20286178588867,
      "activations/layer15_attention_weight_max": 41.42058181762695,
      "activations/layer15_attention_weight_min": -29.485750198364258,
      "activations/layer16_attention_weight_max": 34.62920379638672,
      "activations/layer16_attention_weight_min": -25.589296340942383,
      "activations/layer17_attention_weight_max": 36.3726692199707,
      "activations/layer17_attention_weight_min": -25.558130264282227,
      "activations/layer18_attention_weight_max": 32.343780517578125,
      "activations/layer18_attention_weight_min": -21.314109802246094,
      "activations/layer19_attention_weight_max": 40.27134323120117,
      "activations/layer19_attention_weight_min": -28.922164916992188,
      "activations/layer1_attention_weight_max": 17.38286781311035,
      "activations/layer1_attention_weight_min": -14.08959674835205,
      "activations/layer20_attention_weight_max": 34.66898727416992,
      "activations/layer20_attention_weight_min": -23.662817001342773,
      "activations/layer21_attention_weight_max": 32.037235260009766,
      "activations/layer21_attention_weight_min": -23.187801361083984,
      "activations/layer22_attention_weight_max": 54.89808654785156,
      "activations/layer22_attention_weight_min": -28.223623275756836,
      "activations/layer23_attention_weight_max": 37.83162307739258,
      "activations/layer23_attention_weight_min": -21.907028198242188,
      "activations/layer2_attention_weight_max": 32.94598388671875,
      "activations/layer2_attention_weight_min": -32.32670211791992,
      "activations/layer3_attention_weight_max": 93.16773223876953,
      "activations/layer3_attention_weight_min": -91.88812255859375,
      "activations/layer4_attention_weight_max": 120.59166717529297,
      "activations/layer4_attention_weight_min": -119.52550506591797,
      "activations/layer5_attention_weight_max": 62.529502868652344,
      "activations/layer5_attention_weight_min": -63.72148132324219,
      "activations/layer6_attention_weight_max": 49.77935028076172,
      "activations/layer6_attention_weight_min": -53.081119537353516,
      "activations/layer7_attention_weight_max": 91.23090362548828,
      "activations/layer7_attention_weight_min": -89.23634338378906,
      "activations/layer8_attention_weight_max": 42.925899505615234,
      "activations/layer8_attention_weight_min": -40.56801986694336,
      "activations/layer9_attention_weight_max": 39.46692657470703,
      "activations/layer9_attention_weight_min": -35.52993392944336,
      "epoch": 7.34,
      "learning_rate": 0.00010373598484848484,
      "loss": 2.8079,
      "step": 126250
    },
    {
      "activations/layer0_attention_weight_max": 15.133695602416992,
      "activations/layer0_attention_weight_min": -11.960113525390625,
      "activations/layer10_attention_weight_max": 31.2760009765625,
      "activations/layer10_attention_weight_min": -29.804473876953125,
      "activations/layer11_attention_weight_max": 30.763687133789062,
      "activations/layer11_attention_weight_min": -28.79095458984375,
      "activations/layer12_attention_weight_max": 21.99063491821289,
      "activations/layer12_attention_weight_min": -23.298112869262695,
      "activations/layer13_attention_weight_max": 38.99884033203125,
      "activations/layer13_attention_weight_min": -31.400100708007812,
      "activations/layer14_attention_weight_max": 50.81808853149414,
      "activations/layer14_attention_weight_min": -36.63093948364258,
      "activations/layer15_attention_weight_max": 37.343597412109375,
      "activations/layer15_attention_weight_min": -27.212039947509766,
      "activations/layer16_attention_weight_max": 34.56365966796875,
      "activations/layer16_attention_weight_min": -27.46525001525879,
      "activations/layer17_attention_weight_max": 38.55476379394531,
      "activations/layer17_attention_weight_min": -24.69004249572754,
      "activations/layer18_attention_weight_max": 31.59144401550293,
      "activations/layer18_attention_weight_min": -23.100290298461914,
      "activations/layer19_attention_weight_max": 39.01232147216797,
      "activations/layer19_attention_weight_min": -28.14383888244629,
      "activations/layer1_attention_weight_max": 17.229066848754883,
      "activations/layer1_attention_weight_min": -14.844948768615723,
      "activations/layer20_attention_weight_max": 34.11206817626953,
      "activations/layer20_attention_weight_min": -23.335403442382812,
      "activations/layer21_attention_weight_max": 33.20013427734375,
      "activations/layer21_attention_weight_min": -23.5591983795166,
      "activations/layer22_attention_weight_max": 53.41501235961914,
      "activations/layer22_attention_weight_min": -28.5520076751709,
      "activations/layer23_attention_weight_max": 37.53455352783203,
      "activations/layer23_attention_weight_min": -22.22256851196289,
      "activations/layer2_attention_weight_max": 31.771841049194336,
      "activations/layer2_attention_weight_min": -31.23621368408203,
      "activations/layer3_attention_weight_max": 89.6857681274414,
      "activations/layer3_attention_weight_min": -91.39765930175781,
      "activations/layer4_attention_weight_max": 120.14652252197266,
      "activations/layer4_attention_weight_min": -118.5374526977539,
      "activations/layer5_attention_weight_max": 61.02423858642578,
      "activations/layer5_attention_weight_min": -60.952728271484375,
      "activations/layer6_attention_weight_max": 49.86231994628906,
      "activations/layer6_attention_weight_min": -51.26825714111328,
      "activations/layer7_attention_weight_max": 84.52140808105469,
      "activations/layer7_attention_weight_min": -84.57168579101562,
      "activations/layer8_attention_weight_max": 39.49603271484375,
      "activations/layer8_attention_weight_min": -38.81605911254883,
      "activations/layer9_attention_weight_max": 34.28644943237305,
      "activations/layer9_attention_weight_min": -32.67533493041992,
      "epoch": 7.34,
      "learning_rate": 0.00010371704545454545,
      "loss": 2.8128,
      "step": 126300
    },
    {
      "activations/layer0_attention_weight_max": 15.16843318939209,
      "activations/layer0_attention_weight_min": -11.7203369140625,
      "activations/layer10_attention_weight_max": 30.62405776977539,
      "activations/layer10_attention_weight_min": -30.422178268432617,
      "activations/layer11_attention_weight_max": 32.071205139160156,
      "activations/layer11_attention_weight_min": -30.13311767578125,
      "activations/layer12_attention_weight_max": 31.219783782958984,
      "activations/layer12_attention_weight_min": -26.389585494995117,
      "activations/layer13_attention_weight_max": 45.35649108886719,
      "activations/layer13_attention_weight_min": -32.244293212890625,
      "activations/layer14_attention_weight_max": 49.73451232910156,
      "activations/layer14_attention_weight_min": -40.20976257324219,
      "activations/layer15_attention_weight_max": 43.096824645996094,
      "activations/layer15_attention_weight_min": -29.260875701904297,
      "activations/layer16_attention_weight_max": 31.81409454345703,
      "activations/layer16_attention_weight_min": -29.061294555664062,
      "activations/layer17_attention_weight_max": 34.781150817871094,
      "activations/layer17_attention_weight_min": -24.809476852416992,
      "activations/layer18_attention_weight_max": 28.017250061035156,
      "activations/layer18_attention_weight_min": -20.587804794311523,
      "activations/layer19_attention_weight_max": 35.845420837402344,
      "activations/layer19_attention_weight_min": -28.757801055908203,
      "activations/layer1_attention_weight_max": 17.388830184936523,
      "activations/layer1_attention_weight_min": -14.383480072021484,
      "activations/layer20_attention_weight_max": 32.08478927612305,
      "activations/layer20_attention_weight_min": -24.7600040435791,
      "activations/layer21_attention_weight_max": 28.87160301208496,
      "activations/layer21_attention_weight_min": -22.829391479492188,
      "activations/layer22_attention_weight_max": 45.82870864868164,
      "activations/layer22_attention_weight_min": -26.35625648498535,
      "activations/layer23_attention_weight_max": 36.59465026855469,
      "activations/layer23_attention_weight_min": -22.836761474609375,
      "activations/layer2_attention_weight_max": 31.205265045166016,
      "activations/layer2_attention_weight_min": -31.01287078857422,
      "activations/layer3_attention_weight_max": 90.11019897460938,
      "activations/layer3_attention_weight_min": -88.82125091552734,
      "activations/layer4_attention_weight_max": 117.97728729248047,
      "activations/layer4_attention_weight_min": -122.72552490234375,
      "activations/layer5_attention_weight_max": 60.58976364135742,
      "activations/layer5_attention_weight_min": -61.87748718261719,
      "activations/layer6_attention_weight_max": 49.214317321777344,
      "activations/layer6_attention_weight_min": -52.23297882080078,
      "activations/layer7_attention_weight_max": 88.64644622802734,
      "activations/layer7_attention_weight_min": -85.84131622314453,
      "activations/layer8_attention_weight_max": 38.97857666015625,
      "activations/layer8_attention_weight_min": -38.942718505859375,
      "activations/layer9_attention_weight_max": 33.758216857910156,
      "activations/layer9_attention_weight_min": -32.75081253051758,
      "epoch": 7.34,
      "learning_rate": 0.00010369810606060604,
      "loss": 2.8118,
      "step": 126350
    },
    {
      "activations/layer0_attention_weight_max": 15.445104598999023,
      "activations/layer0_attention_weight_min": -10.972249984741211,
      "activations/layer10_attention_weight_max": 32.97964859008789,
      "activations/layer10_attention_weight_min": -31.044279098510742,
      "activations/layer11_attention_weight_max": 31.994373321533203,
      "activations/layer11_attention_weight_min": -28.00171661376953,
      "activations/layer12_attention_weight_max": 24.239349365234375,
      "activations/layer12_attention_weight_min": -29.5869083404541,
      "activations/layer13_attention_weight_max": 42.70399856567383,
      "activations/layer13_attention_weight_min": -33.75221633911133,
      "activations/layer14_attention_weight_max": 52.04104232788086,
      "activations/layer14_attention_weight_min": -38.647491455078125,
      "activations/layer15_attention_weight_max": 39.49821090698242,
      "activations/layer15_attention_weight_min": -28.504924774169922,
      "activations/layer16_attention_weight_max": 36.36593246459961,
      "activations/layer16_attention_weight_min": -27.01111602783203,
      "activations/layer17_attention_weight_max": 40.46056365966797,
      "activations/layer17_attention_weight_min": -25.257240295410156,
      "activations/layer18_attention_weight_max": 31.414159774780273,
      "activations/layer18_attention_weight_min": -21.599897384643555,
      "activations/layer19_attention_weight_max": 40.11677169799805,
      "activations/layer19_attention_weight_min": -33.411869049072266,
      "activations/layer1_attention_weight_max": 17.534513473510742,
      "activations/layer1_attention_weight_min": -15.606368064880371,
      "activations/layer20_attention_weight_max": 33.82728576660156,
      "activations/layer20_attention_weight_min": -23.857723236083984,
      "activations/layer21_attention_weight_max": 31.669729232788086,
      "activations/layer21_attention_weight_min": -23.060302734375,
      "activations/layer22_attention_weight_max": 49.18852615356445,
      "activations/layer22_attention_weight_min": -29.54462432861328,
      "activations/layer23_attention_weight_max": 35.53009796142578,
      "activations/layer23_attention_weight_min": -22.81610870361328,
      "activations/layer2_attention_weight_max": 30.474632263183594,
      "activations/layer2_attention_weight_min": -31.588436126708984,
      "activations/layer3_attention_weight_max": 85.67890930175781,
      "activations/layer3_attention_weight_min": -87.47916412353516,
      "activations/layer4_attention_weight_max": 119.37883758544922,
      "activations/layer4_attention_weight_min": -113.6740951538086,
      "activations/layer5_attention_weight_max": 61.141639709472656,
      "activations/layer5_attention_weight_min": -59.472633361816406,
      "activations/layer6_attention_weight_max": 48.48966979980469,
      "activations/layer6_attention_weight_min": -53.78293991088867,
      "activations/layer7_attention_weight_max": 90.62449645996094,
      "activations/layer7_attention_weight_min": -88.59632110595703,
      "activations/layer8_attention_weight_max": 40.060890197753906,
      "activations/layer8_attention_weight_min": -39.44257736206055,
      "activations/layer9_attention_weight_max": 37.82213592529297,
      "activations/layer9_attention_weight_min": -33.669647216796875,
      "epoch": 7.34,
      "learning_rate": 0.00010367916666666665,
      "loss": 2.8184,
      "step": 126400
    },
    {
      "activations/layer0_attention_weight_max": 15.9346923828125,
      "activations/layer0_attention_weight_min": -11.288895606994629,
      "activations/layer10_attention_weight_max": 33.8782844543457,
      "activations/layer10_attention_weight_min": -31.899932861328125,
      "activations/layer11_attention_weight_max": 33.2857666015625,
      "activations/layer11_attention_weight_min": -29.45770263671875,
      "activations/layer12_attention_weight_max": 21.985286712646484,
      "activations/layer12_attention_weight_min": -25.870088577270508,
      "activations/layer13_attention_weight_max": 41.80780029296875,
      "activations/layer13_attention_weight_min": -33.41920852661133,
      "activations/layer14_attention_weight_max": 52.29997253417969,
      "activations/layer14_attention_weight_min": -42.05412673950195,
      "activations/layer15_attention_weight_max": 36.767059326171875,
      "activations/layer15_attention_weight_min": -29.78937530517578,
      "activations/layer16_attention_weight_max": 29.864564895629883,
      "activations/layer16_attention_weight_min": -28.892549514770508,
      "activations/layer17_attention_weight_max": 34.41874694824219,
      "activations/layer17_attention_weight_min": -26.878459930419922,
      "activations/layer18_attention_weight_max": 29.05232048034668,
      "activations/layer18_attention_weight_min": -22.118377685546875,
      "activations/layer19_attention_weight_max": 34.25678634643555,
      "activations/layer19_attention_weight_min": -32.3536376953125,
      "activations/layer1_attention_weight_max": 16.758953094482422,
      "activations/layer1_attention_weight_min": -14.317245483398438,
      "activations/layer20_attention_weight_max": 27.428592681884766,
      "activations/layer20_attention_weight_min": -24.16256332397461,
      "activations/layer21_attention_weight_max": 24.914934158325195,
      "activations/layer21_attention_weight_min": -23.636016845703125,
      "activations/layer22_attention_weight_max": 44.22105026245117,
      "activations/layer22_attention_weight_min": -30.229320526123047,
      "activations/layer23_attention_weight_max": 33.904541015625,
      "activations/layer23_attention_weight_min": -24.55470848083496,
      "activations/layer2_attention_weight_max": 30.38935661315918,
      "activations/layer2_attention_weight_min": -31.33584976196289,
      "activations/layer3_attention_weight_max": 89.18912506103516,
      "activations/layer3_attention_weight_min": -90.48836517333984,
      "activations/layer4_attention_weight_max": 119.27680969238281,
      "activations/layer4_attention_weight_min": -117.43743896484375,
      "activations/layer5_attention_weight_max": 62.11366271972656,
      "activations/layer5_attention_weight_min": -63.53422164916992,
      "activations/layer6_attention_weight_max": 50.47718811035156,
      "activations/layer6_attention_weight_min": -56.10108947753906,
      "activations/layer7_attention_weight_max": 88.67489624023438,
      "activations/layer7_attention_weight_min": -89.36174774169922,
      "activations/layer8_attention_weight_max": 39.3945198059082,
      "activations/layer8_attention_weight_min": -41.43116760253906,
      "activations/layer9_attention_weight_max": 38.4516716003418,
      "activations/layer9_attention_weight_min": -34.48905944824219,
      "epoch": 7.35,
      "learning_rate": 0.00010366022727272727,
      "loss": 2.8129,
      "step": 126450
    },
    {
      "activations/layer0_attention_weight_max": 15.239093780517578,
      "activations/layer0_attention_weight_min": -10.764062881469727,
      "activations/layer10_attention_weight_max": 31.695999145507812,
      "activations/layer10_attention_weight_min": -31.60228157043457,
      "activations/layer11_attention_weight_max": 31.402326583862305,
      "activations/layer11_attention_weight_min": -28.68914794921875,
      "activations/layer12_attention_weight_max": 20.47476577758789,
      "activations/layer12_attention_weight_min": -29.734617233276367,
      "activations/layer13_attention_weight_max": 34.69538116455078,
      "activations/layer13_attention_weight_min": -33.91504669189453,
      "activations/layer14_attention_weight_max": 44.75442123413086,
      "activations/layer14_attention_weight_min": -37.29518508911133,
      "activations/layer15_attention_weight_max": 34.33770751953125,
      "activations/layer15_attention_weight_min": -27.373939514160156,
      "activations/layer16_attention_weight_max": 28.83759880065918,
      "activations/layer16_attention_weight_min": -27.667259216308594,
      "activations/layer17_attention_weight_max": 32.83640670776367,
      "activations/layer17_attention_weight_min": -24.63054847717285,
      "activations/layer18_attention_weight_max": 29.4758358001709,
      "activations/layer18_attention_weight_min": -19.68454933166504,
      "activations/layer19_attention_weight_max": 31.808378219604492,
      "activations/layer19_attention_weight_min": -29.606781005859375,
      "activations/layer1_attention_weight_max": 16.881221771240234,
      "activations/layer1_attention_weight_min": -15.495668411254883,
      "activations/layer20_attention_weight_max": 27.86003303527832,
      "activations/layer20_attention_weight_min": -22.256669998168945,
      "activations/layer21_attention_weight_max": 27.171409606933594,
      "activations/layer21_attention_weight_min": -22.05271339416504,
      "activations/layer22_attention_weight_max": 43.06217575073242,
      "activations/layer22_attention_weight_min": -31.10002326965332,
      "activations/layer23_attention_weight_max": 32.47582244873047,
      "activations/layer23_attention_weight_min": -24.579984664916992,
      "activations/layer2_attention_weight_max": 29.52467918395996,
      "activations/layer2_attention_weight_min": -30.283613204956055,
      "activations/layer3_attention_weight_max": 88.27647399902344,
      "activations/layer3_attention_weight_min": -89.71248626708984,
      "activations/layer4_attention_weight_max": 119.21711730957031,
      "activations/layer4_attention_weight_min": -115.5439682006836,
      "activations/layer5_attention_weight_max": 61.680511474609375,
      "activations/layer5_attention_weight_min": -64.81194305419922,
      "activations/layer6_attention_weight_max": 49.11140823364258,
      "activations/layer6_attention_weight_min": -51.641353607177734,
      "activations/layer7_attention_weight_max": 83.9129867553711,
      "activations/layer7_attention_weight_min": -86.08909606933594,
      "activations/layer8_attention_weight_max": 39.0063362121582,
      "activations/layer8_attention_weight_min": -39.28727340698242,
      "activations/layer9_attention_weight_max": 34.11020278930664,
      "activations/layer9_attention_weight_min": -33.60051727294922,
      "epoch": 7.35,
      "learning_rate": 0.00010364128787878787,
      "loss": 2.818,
      "step": 126500
    },
    {
      "activations/layer0_attention_weight_max": 15.895590782165527,
      "activations/layer0_attention_weight_min": -11.3524169921875,
      "activations/layer10_attention_weight_max": 33.56018829345703,
      "activations/layer10_attention_weight_min": -32.20977020263672,
      "activations/layer11_attention_weight_max": 35.90033721923828,
      "activations/layer11_attention_weight_min": -30.453590393066406,
      "activations/layer12_attention_weight_max": 27.573932647705078,
      "activations/layer12_attention_weight_min": -24.53667640686035,
      "activations/layer13_attention_weight_max": 44.68312454223633,
      "activations/layer13_attention_weight_min": -34.81053924560547,
      "activations/layer14_attention_weight_max": 56.939144134521484,
      "activations/layer14_attention_weight_min": -41.848045349121094,
      "activations/layer15_attention_weight_max": 48.87779998779297,
      "activations/layer15_attention_weight_min": -31.07785987854004,
      "activations/layer16_attention_weight_max": 36.373172760009766,
      "activations/layer16_attention_weight_min": -29.36378288269043,
      "activations/layer17_attention_weight_max": 37.44477462768555,
      "activations/layer17_attention_weight_min": -24.64171600341797,
      "activations/layer18_attention_weight_max": 39.601585388183594,
      "activations/layer18_attention_weight_min": -21.846399307250977,
      "activations/layer19_attention_weight_max": 40.590721130371094,
      "activations/layer19_attention_weight_min": -32.699974060058594,
      "activations/layer1_attention_weight_max": 16.492460250854492,
      "activations/layer1_attention_weight_min": -15.066125869750977,
      "activations/layer20_attention_weight_max": 35.139312744140625,
      "activations/layer20_attention_weight_min": -25.03285789489746,
      "activations/layer21_attention_weight_max": 35.65908432006836,
      "activations/layer21_attention_weight_min": -22.848604202270508,
      "activations/layer22_attention_weight_max": 54.48419952392578,
      "activations/layer22_attention_weight_min": -29.09330940246582,
      "activations/layer23_attention_weight_max": 37.01360321044922,
      "activations/layer23_attention_weight_min": -23.166175842285156,
      "activations/layer2_attention_weight_max": 33.38712692260742,
      "activations/layer2_attention_weight_min": -30.732357025146484,
      "activations/layer3_attention_weight_max": 92.7899169921875,
      "activations/layer3_attention_weight_min": -93.70673370361328,
      "activations/layer4_attention_weight_max": 120.6831283569336,
      "activations/layer4_attention_weight_min": -118.60595703125,
      "activations/layer5_attention_weight_max": 61.361724853515625,
      "activations/layer5_attention_weight_min": -60.68775177001953,
      "activations/layer6_attention_weight_max": 51.96250915527344,
      "activations/layer6_attention_weight_min": -53.78945541381836,
      "activations/layer7_attention_weight_max": 88.42391967773438,
      "activations/layer7_attention_weight_min": -93.86097717285156,
      "activations/layer8_attention_weight_max": 42.29246520996094,
      "activations/layer8_attention_weight_min": -41.485740661621094,
      "activations/layer9_attention_weight_max": 37.53965377807617,
      "activations/layer9_attention_weight_min": -33.8873291015625,
      "epoch": 7.35,
      "learning_rate": 0.00010362234848484847,
      "loss": 2.8145,
      "step": 126550
    },
    {
      "activations/layer0_attention_weight_max": 16.17974090576172,
      "activations/layer0_attention_weight_min": -11.605663299560547,
      "activations/layer10_attention_weight_max": 35.37614059448242,
      "activations/layer10_attention_weight_min": -30.317522048950195,
      "activations/layer11_attention_weight_max": 33.12932586669922,
      "activations/layer11_attention_weight_min": -29.741718292236328,
      "activations/layer12_attention_weight_max": 25.96923065185547,
      "activations/layer12_attention_weight_min": -25.377479553222656,
      "activations/layer13_attention_weight_max": 42.306724548339844,
      "activations/layer13_attention_weight_min": -35.27574920654297,
      "activations/layer14_attention_weight_max": 52.91134262084961,
      "activations/layer14_attention_weight_min": -39.781620025634766,
      "activations/layer15_attention_weight_max": 35.83420181274414,
      "activations/layer15_attention_weight_min": -31.648351669311523,
      "activations/layer16_attention_weight_max": 31.854080200195312,
      "activations/layer16_attention_weight_min": -28.56986427307129,
      "activations/layer17_attention_weight_max": 34.55837631225586,
      "activations/layer17_attention_weight_min": -28.13804054260254,
      "activations/layer18_attention_weight_max": 27.92454719543457,
      "activations/layer18_attention_weight_min": -22.896015167236328,
      "activations/layer19_attention_weight_max": 37.7081184387207,
      "activations/layer19_attention_weight_min": -28.753799438476562,
      "activations/layer1_attention_weight_max": 16.64877700805664,
      "activations/layer1_attention_weight_min": -14.009479522705078,
      "activations/layer20_attention_weight_max": 31.26479721069336,
      "activations/layer20_attention_weight_min": -22.873859405517578,
      "activations/layer21_attention_weight_max": 34.76643753051758,
      "activations/layer21_attention_weight_min": -21.62665367126465,
      "activations/layer22_attention_weight_max": 45.03092575073242,
      "activations/layer22_attention_weight_min": -32.250648498535156,
      "activations/layer23_attention_weight_max": 31.84375762939453,
      "activations/layer23_attention_weight_min": -25.50467300415039,
      "activations/layer2_attention_weight_max": 30.893178939819336,
      "activations/layer2_attention_weight_min": -30.87390899658203,
      "activations/layer3_attention_weight_max": 86.61983489990234,
      "activations/layer3_attention_weight_min": -90.78900146484375,
      "activations/layer4_attention_weight_max": 119.94742584228516,
      "activations/layer4_attention_weight_min": -115.44024658203125,
      "activations/layer5_attention_weight_max": 59.20783615112305,
      "activations/layer5_attention_weight_min": -61.51375961303711,
      "activations/layer6_attention_weight_max": 49.003578186035156,
      "activations/layer6_attention_weight_min": -55.89910125732422,
      "activations/layer7_attention_weight_max": 85.31148529052734,
      "activations/layer7_attention_weight_min": -87.42398834228516,
      "activations/layer8_attention_weight_max": 40.353153228759766,
      "activations/layer8_attention_weight_min": -39.6143684387207,
      "activations/layer9_attention_weight_max": 38.55217742919922,
      "activations/layer9_attention_weight_min": -33.10779571533203,
      "epoch": 7.36,
      "learning_rate": 0.00010360340909090909,
      "loss": 2.7993,
      "step": 126600
    },
    {
      "activations/layer0_attention_weight_max": 15.750515937805176,
      "activations/layer0_attention_weight_min": -11.33028507232666,
      "activations/layer10_attention_weight_max": 34.03931427001953,
      "activations/layer10_attention_weight_min": -30.49173927307129,
      "activations/layer11_attention_weight_max": 34.70235824584961,
      "activations/layer11_attention_weight_min": -27.534879684448242,
      "activations/layer12_attention_weight_max": 22.8067684173584,
      "activations/layer12_attention_weight_min": -24.94168472290039,
      "activations/layer13_attention_weight_max": 39.48821258544922,
      "activations/layer13_attention_weight_min": -30.806650161743164,
      "activations/layer14_attention_weight_max": 51.93921661376953,
      "activations/layer14_attention_weight_min": -41.9967041015625,
      "activations/layer15_attention_weight_max": 35.811805725097656,
      "activations/layer15_attention_weight_min": -29.41788101196289,
      "activations/layer16_attention_weight_max": 35.257843017578125,
      "activations/layer16_attention_weight_min": -28.385356903076172,
      "activations/layer17_attention_weight_max": 37.96501159667969,
      "activations/layer17_attention_weight_min": -25.887226104736328,
      "activations/layer18_attention_weight_max": 31.08110237121582,
      "activations/layer18_attention_weight_min": -21.106157302856445,
      "activations/layer19_attention_weight_max": 37.3015022277832,
      "activations/layer19_attention_weight_min": -30.073923110961914,
      "activations/layer1_attention_weight_max": 16.979299545288086,
      "activations/layer1_attention_weight_min": -14.260164260864258,
      "activations/layer20_attention_weight_max": 35.32835388183594,
      "activations/layer20_attention_weight_min": -23.572948455810547,
      "activations/layer21_attention_weight_max": 31.709716796875,
      "activations/layer21_attention_weight_min": -21.728240966796875,
      "activations/layer22_attention_weight_max": 48.779415130615234,
      "activations/layer22_attention_weight_min": -27.39972496032715,
      "activations/layer23_attention_weight_max": 35.66703796386719,
      "activations/layer23_attention_weight_min": -22.6671142578125,
      "activations/layer2_attention_weight_max": 30.99425506591797,
      "activations/layer2_attention_weight_min": -32.25666046142578,
      "activations/layer3_attention_weight_max": 91.23150634765625,
      "activations/layer3_attention_weight_min": -93.87779235839844,
      "activations/layer4_attention_weight_max": 116.95008850097656,
      "activations/layer4_attention_weight_min": -114.8981704711914,
      "activations/layer5_attention_weight_max": 60.39056396484375,
      "activations/layer5_attention_weight_min": -64.46675109863281,
      "activations/layer6_attention_weight_max": 49.41832733154297,
      "activations/layer6_attention_weight_min": -51.66841125488281,
      "activations/layer7_attention_weight_max": 84.71841430664062,
      "activations/layer7_attention_weight_min": -87.9229736328125,
      "activations/layer8_attention_weight_max": 38.87400817871094,
      "activations/layer8_attention_weight_min": -38.740604400634766,
      "activations/layer9_attention_weight_max": 36.15572738647461,
      "activations/layer9_attention_weight_min": -32.30618667602539,
      "epoch": 7.36,
      "learning_rate": 0.00010358446969696969,
      "loss": 2.8113,
      "step": 126650
    },
    {
      "activations/layer0_attention_weight_max": 16.514251708984375,
      "activations/layer0_attention_weight_min": -11.699790954589844,
      "activations/layer10_attention_weight_max": 31.500186920166016,
      "activations/layer10_attention_weight_min": -29.32370376586914,
      "activations/layer11_attention_weight_max": 33.00891876220703,
      "activations/layer11_attention_weight_min": -28.339643478393555,
      "activations/layer12_attention_weight_max": 21.2458553314209,
      "activations/layer12_attention_weight_min": -24.641611099243164,
      "activations/layer13_attention_weight_max": 37.748680114746094,
      "activations/layer13_attention_weight_min": -33.81700134277344,
      "activations/layer14_attention_weight_max": 48.95139694213867,
      "activations/layer14_attention_weight_min": -40.16636276245117,
      "activations/layer15_attention_weight_max": 33.87382507324219,
      "activations/layer15_attention_weight_min": -30.72667121887207,
      "activations/layer16_attention_weight_max": 29.666656494140625,
      "activations/layer16_attention_weight_min": -28.374223709106445,
      "activations/layer17_attention_weight_max": 31.63633155822754,
      "activations/layer17_attention_weight_min": -24.859222412109375,
      "activations/layer18_attention_weight_max": 25.294408798217773,
      "activations/layer18_attention_weight_min": -19.918874740600586,
      "activations/layer19_attention_weight_max": 32.105037689208984,
      "activations/layer19_attention_weight_min": -30.248477935791016,
      "activations/layer1_attention_weight_max": 16.652416229248047,
      "activations/layer1_attention_weight_min": -17.695051193237305,
      "activations/layer20_attention_weight_max": 29.814319610595703,
      "activations/layer20_attention_weight_min": -24.304222106933594,
      "activations/layer21_attention_weight_max": 24.532604217529297,
      "activations/layer21_attention_weight_min": -21.47760009765625,
      "activations/layer22_attention_weight_max": 40.65092086791992,
      "activations/layer22_attention_weight_min": -27.680343627929688,
      "activations/layer23_attention_weight_max": 29.94856071472168,
      "activations/layer23_attention_weight_min": -21.322296142578125,
      "activations/layer2_attention_weight_max": 32.571224212646484,
      "activations/layer2_attention_weight_min": -30.073081970214844,
      "activations/layer3_attention_weight_max": 91.91004180908203,
      "activations/layer3_attention_weight_min": -91.55987548828125,
      "activations/layer4_attention_weight_max": 118.66558074951172,
      "activations/layer4_attention_weight_min": -113.165283203125,
      "activations/layer5_attention_weight_max": 59.2016716003418,
      "activations/layer5_attention_weight_min": -58.579952239990234,
      "activations/layer6_attention_weight_max": 47.95005416870117,
      "activations/layer6_attention_weight_min": -50.627464294433594,
      "activations/layer7_attention_weight_max": 96.63365173339844,
      "activations/layer7_attention_weight_min": -84.45408630371094,
      "activations/layer8_attention_weight_max": 39.34209060668945,
      "activations/layer8_attention_weight_min": -37.794979095458984,
      "activations/layer9_attention_weight_max": 33.13877487182617,
      "activations/layer9_attention_weight_min": -32.17955017089844,
      "epoch": 7.36,
      "learning_rate": 0.00010356553030303029,
      "loss": 2.8148,
      "step": 126700
    },
    {
      "activations/layer0_attention_weight_max": 15.419626235961914,
      "activations/layer0_attention_weight_min": -11.4216890335083,
      "activations/layer10_attention_weight_max": 37.83293151855469,
      "activations/layer10_attention_weight_min": -32.70458221435547,
      "activations/layer11_attention_weight_max": 34.230377197265625,
      "activations/layer11_attention_weight_min": -30.619037628173828,
      "activations/layer12_attention_weight_max": 24.978485107421875,
      "activations/layer12_attention_weight_min": -23.91802406311035,
      "activations/layer13_attention_weight_max": 38.8104362487793,
      "activations/layer13_attention_weight_min": -33.17791748046875,
      "activations/layer14_attention_weight_max": 53.84451675415039,
      "activations/layer14_attention_weight_min": -44.2920036315918,
      "activations/layer15_attention_weight_max": 37.40288543701172,
      "activations/layer15_attention_weight_min": -30.64768409729004,
      "activations/layer16_attention_weight_max": 34.074745178222656,
      "activations/layer16_attention_weight_min": -28.05699348449707,
      "activations/layer17_attention_weight_max": 35.28380584716797,
      "activations/layer17_attention_weight_min": -25.3563175201416,
      "activations/layer18_attention_weight_max": 35.596553802490234,
      "activations/layer18_attention_weight_min": -22.553430557250977,
      "activations/layer19_attention_weight_max": 40.04736328125,
      "activations/layer19_attention_weight_min": -27.96650505065918,
      "activations/layer1_attention_weight_max": 16.14639663696289,
      "activations/layer1_attention_weight_min": -13.304732322692871,
      "activations/layer20_attention_weight_max": 31.39641571044922,
      "activations/layer20_attention_weight_min": -23.357194900512695,
      "activations/layer21_attention_weight_max": 30.280393600463867,
      "activations/layer21_attention_weight_min": -22.074695587158203,
      "activations/layer22_attention_weight_max": 51.6843147277832,
      "activations/layer22_attention_weight_min": -28.558002471923828,
      "activations/layer23_attention_weight_max": 37.16473388671875,
      "activations/layer23_attention_weight_min": -23.94073486328125,
      "activations/layer2_attention_weight_max": 31.917118072509766,
      "activations/layer2_attention_weight_min": -32.56428909301758,
      "activations/layer3_attention_weight_max": 89.87772369384766,
      "activations/layer3_attention_weight_min": -91.40409088134766,
      "activations/layer4_attention_weight_max": 120.76094818115234,
      "activations/layer4_attention_weight_min": -117.99884033203125,
      "activations/layer5_attention_weight_max": 62.44805908203125,
      "activations/layer5_attention_weight_min": -64.33073425292969,
      "activations/layer6_attention_weight_max": 51.0446891784668,
      "activations/layer6_attention_weight_min": -54.38220977783203,
      "activations/layer7_attention_weight_max": 92.3048095703125,
      "activations/layer7_attention_weight_min": -90.44355010986328,
      "activations/layer8_attention_weight_max": 41.057273864746094,
      "activations/layer8_attention_weight_min": -41.26688003540039,
      "activations/layer9_attention_weight_max": 37.52608871459961,
      "activations/layer9_attention_weight_min": -33.81931686401367,
      "epoch": 7.36,
      "learning_rate": 0.00010354659090909089,
      "loss": 2.8192,
      "step": 126750
    },
    {
      "activations/layer0_attention_weight_max": 15.235891342163086,
      "activations/layer0_attention_weight_min": -10.868560791015625,
      "activations/layer10_attention_weight_max": 31.901689529418945,
      "activations/layer10_attention_weight_min": -29.196044921875,
      "activations/layer11_attention_weight_max": 31.37049674987793,
      "activations/layer11_attention_weight_min": -27.934133529663086,
      "activations/layer12_attention_weight_max": 21.52543830871582,
      "activations/layer12_attention_weight_min": -27.444379806518555,
      "activations/layer13_attention_weight_max": 33.899452209472656,
      "activations/layer13_attention_weight_min": -31.897140502929688,
      "activations/layer14_attention_weight_max": 42.63751220703125,
      "activations/layer14_attention_weight_min": -39.50044250488281,
      "activations/layer15_attention_weight_max": 36.101016998291016,
      "activations/layer15_attention_weight_min": -27.34674644470215,
      "activations/layer16_attention_weight_max": 34.18521499633789,
      "activations/layer16_attention_weight_min": -26.900068283081055,
      "activations/layer17_attention_weight_max": 33.22016525268555,
      "activations/layer17_attention_weight_min": -23.644149780273438,
      "activations/layer18_attention_weight_max": 30.662677764892578,
      "activations/layer18_attention_weight_min": -19.99075698852539,
      "activations/layer19_attention_weight_max": 40.96198272705078,
      "activations/layer19_attention_weight_min": -28.72883415222168,
      "activations/layer1_attention_weight_max": 16.417272567749023,
      "activations/layer1_attention_weight_min": -14.969534873962402,
      "activations/layer20_attention_weight_max": 30.040071487426758,
      "activations/layer20_attention_weight_min": -24.014684677124023,
      "activations/layer21_attention_weight_max": 28.792436599731445,
      "activations/layer21_attention_weight_min": -21.460365295410156,
      "activations/layer22_attention_weight_max": 43.88317108154297,
      "activations/layer22_attention_weight_min": -26.829145431518555,
      "activations/layer23_attention_weight_max": 31.549680709838867,
      "activations/layer23_attention_weight_min": -22.713109970092773,
      "activations/layer2_attention_weight_max": 33.465797424316406,
      "activations/layer2_attention_weight_min": -33.09789276123047,
      "activations/layer3_attention_weight_max": 89.82714080810547,
      "activations/layer3_attention_weight_min": -97.46781158447266,
      "activations/layer4_attention_weight_max": 118.38581848144531,
      "activations/layer4_attention_weight_min": -121.93187713623047,
      "activations/layer5_attention_weight_max": 63.15708541870117,
      "activations/layer5_attention_weight_min": -62.29061508178711,
      "activations/layer6_attention_weight_max": 49.58340072631836,
      "activations/layer6_attention_weight_min": -52.76704788208008,
      "activations/layer7_attention_weight_max": 85.6047592163086,
      "activations/layer7_attention_weight_min": -87.78271484375,
      "activations/layer8_attention_weight_max": 41.16194534301758,
      "activations/layer8_attention_weight_min": -37.75809860229492,
      "activations/layer9_attention_weight_max": 35.099727630615234,
      "activations/layer9_attention_weight_min": -33.710391998291016,
      "epoch": 7.37,
      "learning_rate": 0.00010352765151515151,
      "loss": 2.8073,
      "step": 126800
    },
    {
      "activations/layer0_attention_weight_max": 17.098270416259766,
      "activations/layer0_attention_weight_min": -10.740029335021973,
      "activations/layer10_attention_weight_max": 31.070682525634766,
      "activations/layer10_attention_weight_min": -29.95995330810547,
      "activations/layer11_attention_weight_max": 30.57186508178711,
      "activations/layer11_attention_weight_min": -28.164398193359375,
      "activations/layer12_attention_weight_max": 22.04401397705078,
      "activations/layer12_attention_weight_min": -38.26952362060547,
      "activations/layer13_attention_weight_max": 37.79698181152344,
      "activations/layer13_attention_weight_min": -30.435199737548828,
      "activations/layer14_attention_weight_max": 52.016448974609375,
      "activations/layer14_attention_weight_min": -44.70600891113281,
      "activations/layer15_attention_weight_max": 32.872779846191406,
      "activations/layer15_attention_weight_min": -28.936655044555664,
      "activations/layer16_attention_weight_max": 33.652748107910156,
      "activations/layer16_attention_weight_min": -27.920873641967773,
      "activations/layer17_attention_weight_max": 32.50839614868164,
      "activations/layer17_attention_weight_min": -26.09787940979004,
      "activations/layer18_attention_weight_max": 29.812095642089844,
      "activations/layer18_attention_weight_min": -21.521259307861328,
      "activations/layer19_attention_weight_max": 38.73876953125,
      "activations/layer19_attention_weight_min": -34.19684982299805,
      "activations/layer1_attention_weight_max": 17.083620071411133,
      "activations/layer1_attention_weight_min": -14.987029075622559,
      "activations/layer20_attention_weight_max": 29.443132400512695,
      "activations/layer20_attention_weight_min": -26.037878036499023,
      "activations/layer21_attention_weight_max": 30.29582405090332,
      "activations/layer21_attention_weight_min": -23.049564361572266,
      "activations/layer22_attention_weight_max": 42.76473617553711,
      "activations/layer22_attention_weight_min": -29.018253326416016,
      "activations/layer23_attention_weight_max": 30.82999038696289,
      "activations/layer23_attention_weight_min": -23.40735626220703,
      "activations/layer2_attention_weight_max": 33.363059997558594,
      "activations/layer2_attention_weight_min": -31.04686737060547,
      "activations/layer3_attention_weight_max": 95.21725463867188,
      "activations/layer3_attention_weight_min": -94.46502685546875,
      "activations/layer4_attention_weight_max": 124.66405487060547,
      "activations/layer4_attention_weight_min": -122.20281219482422,
      "activations/layer5_attention_weight_max": 61.42375946044922,
      "activations/layer5_attention_weight_min": -62.12125778198242,
      "activations/layer6_attention_weight_max": 50.16938400268555,
      "activations/layer6_attention_weight_min": -51.971595764160156,
      "activations/layer7_attention_weight_max": 91.61675262451172,
      "activations/layer7_attention_weight_min": -82.01728057861328,
      "activations/layer8_attention_weight_max": 40.9094352722168,
      "activations/layer8_attention_weight_min": -38.34892272949219,
      "activations/layer9_attention_weight_max": 34.59914779663086,
      "activations/layer9_attention_weight_min": -40.112579345703125,
      "epoch": 7.37,
      "learning_rate": 0.00010350871212121212,
      "loss": 2.7946,
      "step": 126850
    },
    {
      "activations/layer0_attention_weight_max": 15.935097694396973,
      "activations/layer0_attention_weight_min": -10.995402336120605,
      "activations/layer10_attention_weight_max": 35.232765197753906,
      "activations/layer10_attention_weight_min": -31.367076873779297,
      "activations/layer11_attention_weight_max": 35.67966842651367,
      "activations/layer11_attention_weight_min": -30.93840789794922,
      "activations/layer12_attention_weight_max": 34.47710037231445,
      "activations/layer12_attention_weight_min": -30.543718338012695,
      "activations/layer13_attention_weight_max": 43.92116928100586,
      "activations/layer13_attention_weight_min": -37.48512649536133,
      "activations/layer14_attention_weight_max": 61.16097640991211,
      "activations/layer14_attention_weight_min": -42.1800537109375,
      "activations/layer15_attention_weight_max": 39.39223098754883,
      "activations/layer15_attention_weight_min": -32.73994827270508,
      "activations/layer16_attention_weight_max": 36.865108489990234,
      "activations/layer16_attention_weight_min": -27.589494705200195,
      "activations/layer17_attention_weight_max": 38.20719528198242,
      "activations/layer17_attention_weight_min": -26.59054183959961,
      "activations/layer18_attention_weight_max": 38.43463134765625,
      "activations/layer18_attention_weight_min": -22.700084686279297,
      "activations/layer19_attention_weight_max": 42.05302429199219,
      "activations/layer19_attention_weight_min": -33.22944259643555,
      "activations/layer1_attention_weight_max": 16.242191314697266,
      "activations/layer1_attention_weight_min": -14.427153587341309,
      "activations/layer20_attention_weight_max": 39.624305725097656,
      "activations/layer20_attention_weight_min": -27.040687561035156,
      "activations/layer21_attention_weight_max": 37.656898498535156,
      "activations/layer21_attention_weight_min": -23.8824405670166,
      "activations/layer22_attention_weight_max": 55.376949310302734,
      "activations/layer22_attention_weight_min": -33.229576110839844,
      "activations/layer23_attention_weight_max": 37.879974365234375,
      "activations/layer23_attention_weight_min": -26.711212158203125,
      "activations/layer2_attention_weight_max": 32.33722686767578,
      "activations/layer2_attention_weight_min": -31.893178939819336,
      "activations/layer3_attention_weight_max": 91.91751098632812,
      "activations/layer3_attention_weight_min": -93.42308807373047,
      "activations/layer4_attention_weight_max": 117.9791488647461,
      "activations/layer4_attention_weight_min": -118.62171936035156,
      "activations/layer5_attention_weight_max": 60.926414489746094,
      "activations/layer5_attention_weight_min": -60.56932830810547,
      "activations/layer6_attention_weight_max": 48.35316848754883,
      "activations/layer6_attention_weight_min": -52.62531661987305,
      "activations/layer7_attention_weight_max": 95.05538940429688,
      "activations/layer7_attention_weight_min": -89.3433837890625,
      "activations/layer8_attention_weight_max": 40.8414306640625,
      "activations/layer8_attention_weight_min": -40.031986236572266,
      "activations/layer9_attention_weight_max": 36.38084030151367,
      "activations/layer9_attention_weight_min": -35.005462646484375,
      "epoch": 7.37,
      "learning_rate": 0.00010348977272727271,
      "loss": 2.7948,
      "step": 126900
    },
    {
      "activations/layer0_attention_weight_max": 15.731761932373047,
      "activations/layer0_attention_weight_min": -10.965888023376465,
      "activations/layer10_attention_weight_max": 31.931148529052734,
      "activations/layer10_attention_weight_min": -30.310270309448242,
      "activations/layer11_attention_weight_max": 33.84805679321289,
      "activations/layer11_attention_weight_min": -29.377086639404297,
      "activations/layer12_attention_weight_max": 27.92383575439453,
      "activations/layer12_attention_weight_min": -26.022356033325195,
      "activations/layer13_attention_weight_max": 42.86823272705078,
      "activations/layer13_attention_weight_min": -32.82160568237305,
      "activations/layer14_attention_weight_max": 49.90850830078125,
      "activations/layer14_attention_weight_min": -42.15744400024414,
      "activations/layer15_attention_weight_max": 38.224334716796875,
      "activations/layer15_attention_weight_min": -30.337554931640625,
      "activations/layer16_attention_weight_max": 35.002197265625,
      "activations/layer16_attention_weight_min": -27.070119857788086,
      "activations/layer17_attention_weight_max": 33.76179885864258,
      "activations/layer17_attention_weight_min": -27.1778507232666,
      "activations/layer18_attention_weight_max": 32.69106674194336,
      "activations/layer18_attention_weight_min": -24.400997161865234,
      "activations/layer19_attention_weight_max": 38.69053268432617,
      "activations/layer19_attention_weight_min": -34.599464416503906,
      "activations/layer1_attention_weight_max": 15.947595596313477,
      "activations/layer1_attention_weight_min": -14.582282066345215,
      "activations/layer20_attention_weight_max": 31.578983306884766,
      "activations/layer20_attention_weight_min": -27.667922973632812,
      "activations/layer21_attention_weight_max": 30.98177146911621,
      "activations/layer21_attention_weight_min": -25.133264541625977,
      "activations/layer22_attention_weight_max": 49.17539596557617,
      "activations/layer22_attention_weight_min": -32.331443786621094,
      "activations/layer23_attention_weight_max": 38.132991790771484,
      "activations/layer23_attention_weight_min": -25.652061462402344,
      "activations/layer2_attention_weight_max": 30.778366088867188,
      "activations/layer2_attention_weight_min": -30.538999557495117,
      "activations/layer3_attention_weight_max": 89.46798706054688,
      "activations/layer3_attention_weight_min": -89.44429016113281,
      "activations/layer4_attention_weight_max": 116.87420654296875,
      "activations/layer4_attention_weight_min": -120.0804214477539,
      "activations/layer5_attention_weight_max": 57.792720794677734,
      "activations/layer5_attention_weight_min": -60.42534637451172,
      "activations/layer6_attention_weight_max": 50.0883674621582,
      "activations/layer6_attention_weight_min": -50.25533676147461,
      "activations/layer7_attention_weight_max": 87.269287109375,
      "activations/layer7_attention_weight_min": -85.47406005859375,
      "activations/layer8_attention_weight_max": 38.707000732421875,
      "activations/layer8_attention_weight_min": -37.80337142944336,
      "activations/layer9_attention_weight_max": 36.0583381652832,
      "activations/layer9_attention_weight_min": -32.44352340698242,
      "epoch": 7.38,
      "learning_rate": 0.00010347083333333333,
      "loss": 2.818,
      "step": 126950
    },
    {
      "activations/layer0_attention_weight_max": 15.579072952270508,
      "activations/layer0_attention_weight_min": -11.603941917419434,
      "activations/layer10_attention_weight_max": 41.745452880859375,
      "activations/layer10_attention_weight_min": -38.17689514160156,
      "activations/layer11_attention_weight_max": 43.07842254638672,
      "activations/layer11_attention_weight_min": -38.06032180786133,
      "activations/layer12_attention_weight_max": 22.73027992248535,
      "activations/layer12_attention_weight_min": -28.33293342590332,
      "activations/layer13_attention_weight_max": 37.84123611450195,
      "activations/layer13_attention_weight_min": -31.349327087402344,
      "activations/layer14_attention_weight_max": 68.92814636230469,
      "activations/layer14_attention_weight_min": -48.1978645324707,
      "activations/layer15_attention_weight_max": 36.95413589477539,
      "activations/layer15_attention_weight_min": -29.931230545043945,
      "activations/layer16_attention_weight_max": 35.033103942871094,
      "activations/layer16_attention_weight_min": -27.790693283081055,
      "activations/layer17_attention_weight_max": 31.868812561035156,
      "activations/layer17_attention_weight_min": -26.49464225769043,
      "activations/layer18_attention_weight_max": 29.504745483398438,
      "activations/layer18_attention_weight_min": -21.8064022064209,
      "activations/layer19_attention_weight_max": 41.44047546386719,
      "activations/layer19_attention_weight_min": -31.64796257019043,
      "activations/layer1_attention_weight_max": 16.845918655395508,
      "activations/layer1_attention_weight_min": -15.149659156799316,
      "activations/layer20_attention_weight_max": 35.32499694824219,
      "activations/layer20_attention_weight_min": -25.20824432373047,
      "activations/layer21_attention_weight_max": 31.60552215576172,
      "activations/layer21_attention_weight_min": -23.214458465576172,
      "activations/layer22_attention_weight_max": 51.84978103637695,
      "activations/layer22_attention_weight_min": -29.032711029052734,
      "activations/layer23_attention_weight_max": 36.85369110107422,
      "activations/layer23_attention_weight_min": -23.70296287536621,
      "activations/layer2_attention_weight_max": 33.281829833984375,
      "activations/layer2_attention_weight_min": -33.612857818603516,
      "activations/layer3_attention_weight_max": 96.24628448486328,
      "activations/layer3_attention_weight_min": -95.66082763671875,
      "activations/layer4_attention_weight_max": 136.50350952148438,
      "activations/layer4_attention_weight_min": -127.88419342041016,
      "activations/layer5_attention_weight_max": 66.68704223632812,
      "activations/layer5_attention_weight_min": -63.558162689208984,
      "activations/layer6_attention_weight_max": 54.184574127197266,
      "activations/layer6_attention_weight_min": -55.11905288696289,
      "activations/layer7_attention_weight_max": 107.53595733642578,
      "activations/layer7_attention_weight_min": -100.08538818359375,
      "activations/layer8_attention_weight_max": 48.65272903442383,
      "activations/layer8_attention_weight_min": -46.84282684326172,
      "activations/layer9_attention_weight_max": 42.78999328613281,
      "activations/layer9_attention_weight_min": -39.521907806396484,
      "epoch": 7.38,
      "learning_rate": 0.00010345227272727272,
      "loss": 2.8272,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5328,
      "eval_samples_per_second": 503.233,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5328,
      "eval_openwebtext_samples_per_second": 503.233,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 2.0057,
      "eval_wikitext_samples_per_second": 227.349,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_lambada_loss": 2.5078125,
      "eval_lambada_ppl": 12.278042445054236,
      "eval_lambada_runtime": 9.5869,
      "eval_lambada_samples_per_second": 507.882,
      "step": 127000
    },
    {
      "activations/layer0_attention_weight_max": 15.146723747253418,
      "activations/layer0_attention_weight_min": -11.777212142944336,
      "activations/layer10_attention_weight_max": 32.134857177734375,
      "activations/layer10_attention_weight_min": -30.855749130249023,
      "activations/layer11_attention_weight_max": 31.90595817565918,
      "activations/layer11_attention_weight_min": -28.896543502807617,
      "activations/layer12_attention_weight_max": 23.53351593017578,
      "activations/layer12_attention_weight_min": -27.661197662353516,
      "activations/layer13_attention_weight_max": 39.3261833190918,
      "activations/layer13_attention_weight_min": -33.51074981689453,
      "activations/layer14_attention_weight_max": 63.28346633911133,
      "activations/layer14_attention_weight_min": -47.104530334472656,
      "activations/layer15_attention_weight_max": 38.15886688232422,
      "activations/layer15_attention_weight_min": -28.8084659576416,
      "activations/layer16_attention_weight_max": 36.88493728637695,
      "activations/layer16_attention_weight_min": -29.680587768554688,
      "activations/layer17_attention_weight_max": 33.71501159667969,
      "activations/layer17_attention_weight_min": -26.175369262695312,
      "activations/layer18_attention_weight_max": 32.914424896240234,
      "activations/layer18_attention_weight_min": -24.225805282592773,
      "activations/layer19_attention_weight_max": 36.26481628417969,
      "activations/layer19_attention_weight_min": -31.525297164916992,
      "activations/layer1_attention_weight_max": 16.308237075805664,
      "activations/layer1_attention_weight_min": -14.217768669128418,
      "activations/layer20_attention_weight_max": 32.26216506958008,
      "activations/layer20_attention_weight_min": -26.0553035736084,
      "activations/layer21_attention_weight_max": 32.07716751098633,
      "activations/layer21_attention_weight_min": -24.30191421508789,
      "activations/layer22_attention_weight_max": 48.51013946533203,
      "activations/layer22_attention_weight_min": -28.926145553588867,
      "activations/layer23_attention_weight_max": 36.75482940673828,
      "activations/layer23_attention_weight_min": -24.62346649169922,
      "activations/layer2_attention_weight_max": 31.159496307373047,
      "activations/layer2_attention_weight_min": -29.808876037597656,
      "activations/layer3_attention_weight_max": 91.65391540527344,
      "activations/layer3_attention_weight_min": -90.36170959472656,
      "activations/layer4_attention_weight_max": 119.82994842529297,
      "activations/layer4_attention_weight_min": -117.767333984375,
      "activations/layer5_attention_weight_max": 63.78506088256836,
      "activations/layer5_attention_weight_min": -63.170806884765625,
      "activations/layer6_attention_weight_max": 53.396568298339844,
      "activations/layer6_attention_weight_min": -53.66020965576172,
      "activations/layer7_attention_weight_max": 94.27530670166016,
      "activations/layer7_attention_weight_min": -90.00442504882812,
      "activations/layer8_attention_weight_max": 41.753257751464844,
      "activations/layer8_attention_weight_min": -41.41583251953125,
      "activations/layer9_attention_weight_max": 38.84379196166992,
      "activations/layer9_attention_weight_min": -33.89716339111328,
      "epoch": 7.38,
      "learning_rate": 0.00010343333333333333,
      "loss": 2.81,
      "step": 127050
    },
    {
      "activations/layer0_attention_weight_max": 15.737081527709961,
      "activations/layer0_attention_weight_min": -11.124576568603516,
      "activations/layer10_attention_weight_max": 33.415618896484375,
      "activations/layer10_attention_weight_min": -35.151039123535156,
      "activations/layer11_attention_weight_max": 34.769622802734375,
      "activations/layer11_attention_weight_min": -33.45459747314453,
      "activations/layer12_attention_weight_max": 32.0734977722168,
      "activations/layer12_attention_weight_min": -27.740413665771484,
      "activations/layer13_attention_weight_max": 44.38349914550781,
      "activations/layer13_attention_weight_min": -33.7058219909668,
      "activations/layer14_attention_weight_max": 57.60731506347656,
      "activations/layer14_attention_weight_min": -41.83692169189453,
      "activations/layer15_attention_weight_max": 43.855690002441406,
      "activations/layer15_attention_weight_min": -29.705169677734375,
      "activations/layer16_attention_weight_max": 34.87464141845703,
      "activations/layer16_attention_weight_min": -28.53803253173828,
      "activations/layer17_attention_weight_max": 35.05808639526367,
      "activations/layer17_attention_weight_min": -25.167612075805664,
      "activations/layer18_attention_weight_max": 36.47124099731445,
      "activations/layer18_attention_weight_min": -21.478551864624023,
      "activations/layer19_attention_weight_max": 41.01165008544922,
      "activations/layer19_attention_weight_min": -32.0982780456543,
      "activations/layer1_attention_weight_max": 16.27473258972168,
      "activations/layer1_attention_weight_min": -16.630380630493164,
      "activations/layer20_attention_weight_max": 35.627845764160156,
      "activations/layer20_attention_weight_min": -25.82439422607422,
      "activations/layer21_attention_weight_max": 34.60903549194336,
      "activations/layer21_attention_weight_min": -26.75065040588379,
      "activations/layer22_attention_weight_max": 47.33872985839844,
      "activations/layer22_attention_weight_min": -28.423513412475586,
      "activations/layer23_attention_weight_max": 34.99790954589844,
      "activations/layer23_attention_weight_min": -23.989212036132812,
      "activations/layer2_attention_weight_max": 32.30121612548828,
      "activations/layer2_attention_weight_min": -31.867687225341797,
      "activations/layer3_attention_weight_max": 91.64745330810547,
      "activations/layer3_attention_weight_min": -95.1434097290039,
      "activations/layer4_attention_weight_max": 121.93303680419922,
      "activations/layer4_attention_weight_min": -125.2856674194336,
      "activations/layer5_attention_weight_max": 64.59727478027344,
      "activations/layer5_attention_weight_min": -63.20878601074219,
      "activations/layer6_attention_weight_max": 53.531166076660156,
      "activations/layer6_attention_weight_min": -55.66645431518555,
      "activations/layer7_attention_weight_max": 91.54983520507812,
      "activations/layer7_attention_weight_min": -98.24299621582031,
      "activations/layer8_attention_weight_max": 42.671630859375,
      "activations/layer8_attention_weight_min": -43.710350036621094,
      "activations/layer9_attention_weight_max": 38.59541320800781,
      "activations/layer9_attention_weight_min": -34.827762603759766,
      "epoch": 7.39,
      "learning_rate": 0.00010341439393939392,
      "loss": 2.7909,
      "step": 127100
    },
    {
      "activations/layer0_attention_weight_max": 15.227304458618164,
      "activations/layer0_attention_weight_min": -12.386820793151855,
      "activations/layer10_attention_weight_max": 33.43487548828125,
      "activations/layer10_attention_weight_min": -28.96305274963379,
      "activations/layer11_attention_weight_max": 30.879318237304688,
      "activations/layer11_attention_weight_min": -27.80245018005371,
      "activations/layer12_attention_weight_max": 25.736530303955078,
      "activations/layer12_attention_weight_min": -26.51565170288086,
      "activations/layer13_attention_weight_max": 39.10552215576172,
      "activations/layer13_attention_weight_min": -31.68988037109375,
      "activations/layer14_attention_weight_max": 60.172767639160156,
      "activations/layer14_attention_weight_min": -46.07720947265625,
      "activations/layer15_attention_weight_max": 38.85475540161133,
      "activations/layer15_attention_weight_min": -28.89443016052246,
      "activations/layer16_attention_weight_max": 33.554508209228516,
      "activations/layer16_attention_weight_min": -26.32727813720703,
      "activations/layer17_attention_weight_max": 38.41325759887695,
      "activations/layer17_attention_weight_min": -24.86476707458496,
      "activations/layer18_attention_weight_max": 40.49882507324219,
      "activations/layer18_attention_weight_min": -22.59642791748047,
      "activations/layer19_attention_weight_max": 39.26054382324219,
      "activations/layer19_attention_weight_min": -30.587324142456055,
      "activations/layer1_attention_weight_max": 16.169069290161133,
      "activations/layer1_attention_weight_min": -13.014457702636719,
      "activations/layer20_attention_weight_max": 29.509897232055664,
      "activations/layer20_attention_weight_min": -23.137245178222656,
      "activations/layer21_attention_weight_max": 34.396484375,
      "activations/layer21_attention_weight_min": -23.16465950012207,
      "activations/layer22_attention_weight_max": 50.35775375366211,
      "activations/layer22_attention_weight_min": -28.0855712890625,
      "activations/layer23_attention_weight_max": 36.473358154296875,
      "activations/layer23_attention_weight_min": -23.899524688720703,
      "activations/layer2_attention_weight_max": 29.591224670410156,
      "activations/layer2_attention_weight_min": -29.496740341186523,
      "activations/layer3_attention_weight_max": 83.87439727783203,
      "activations/layer3_attention_weight_min": -84.05339813232422,
      "activations/layer4_attention_weight_max": 119.26175689697266,
      "activations/layer4_attention_weight_min": -112.0316390991211,
      "activations/layer5_attention_weight_max": 58.53501510620117,
      "activations/layer5_attention_weight_min": -63.485382080078125,
      "activations/layer6_attention_weight_max": 50.21813201904297,
      "activations/layer6_attention_weight_min": -51.464054107666016,
      "activations/layer7_attention_weight_max": 88.22223663330078,
      "activations/layer7_attention_weight_min": -88.53712463378906,
      "activations/layer8_attention_weight_max": 37.87232971191406,
      "activations/layer8_attention_weight_min": -39.37374496459961,
      "activations/layer9_attention_weight_max": 33.96103286743164,
      "activations/layer9_attention_weight_min": -32.28960037231445,
      "epoch": 7.39,
      "learning_rate": 0.00010339545454545453,
      "loss": 2.8237,
      "step": 127150
    },
    {
      "activations/layer0_attention_weight_max": 15.490802764892578,
      "activations/layer0_attention_weight_min": -12.11003303527832,
      "activations/layer10_attention_weight_max": 30.974546432495117,
      "activations/layer10_attention_weight_min": -30.537492752075195,
      "activations/layer11_attention_weight_max": 32.123504638671875,
      "activations/layer11_attention_weight_min": -30.459026336669922,
      "activations/layer12_attention_weight_max": 23.265378952026367,
      "activations/layer12_attention_weight_min": -26.52151870727539,
      "activations/layer13_attention_weight_max": 36.56013107299805,
      "activations/layer13_attention_weight_min": -33.20555877685547,
      "activations/layer14_attention_weight_max": 52.26559066772461,
      "activations/layer14_attention_weight_min": -42.253997802734375,
      "activations/layer15_attention_weight_max": 33.447532653808594,
      "activations/layer15_attention_weight_min": -30.141544342041016,
      "activations/layer16_attention_weight_max": 32.78543472290039,
      "activations/layer16_attention_weight_min": -26.7381534576416,
      "activations/layer17_attention_weight_max": 34.59101486206055,
      "activations/layer17_attention_weight_min": -24.510814666748047,
      "activations/layer18_attention_weight_max": 31.372282028198242,
      "activations/layer18_attention_weight_min": -21.853612899780273,
      "activations/layer19_attention_weight_max": 36.66135025024414,
      "activations/layer19_attention_weight_min": -30.803232192993164,
      "activations/layer1_attention_weight_max": 16.04458236694336,
      "activations/layer1_attention_weight_min": -15.058123588562012,
      "activations/layer20_attention_weight_max": 28.870323181152344,
      "activations/layer20_attention_weight_min": -26.994760513305664,
      "activations/layer21_attention_weight_max": 29.59184455871582,
      "activations/layer21_attention_weight_min": -23.387205123901367,
      "activations/layer22_attention_weight_max": 46.72829818725586,
      "activations/layer22_attention_weight_min": -29.84447479248047,
      "activations/layer23_attention_weight_max": 33.47758102416992,
      "activations/layer23_attention_weight_min": -23.909948348999023,
      "activations/layer2_attention_weight_max": 30.349960327148438,
      "activations/layer2_attention_weight_min": -32.52692413330078,
      "activations/layer3_attention_weight_max": 88.26392364501953,
      "activations/layer3_attention_weight_min": -91.61175537109375,
      "activations/layer4_attention_weight_max": 122.3521499633789,
      "activations/layer4_attention_weight_min": -119.72540283203125,
      "activations/layer5_attention_weight_max": 61.02699279785156,
      "activations/layer5_attention_weight_min": -61.21009063720703,
      "activations/layer6_attention_weight_max": 48.87415313720703,
      "activations/layer6_attention_weight_min": -50.77007293701172,
      "activations/layer7_attention_weight_max": 82.37583923339844,
      "activations/layer7_attention_weight_min": -84.4477767944336,
      "activations/layer8_attention_weight_max": 37.230010986328125,
      "activations/layer8_attention_weight_min": -37.939605712890625,
      "activations/layer9_attention_weight_max": 32.5061149597168,
      "activations/layer9_attention_weight_min": -32.836944580078125,
      "epoch": 7.39,
      "learning_rate": 0.00010337651515151515,
      "loss": 2.811,
      "step": 127200
    },
    {
      "activations/layer0_attention_weight_max": 15.91723918914795,
      "activations/layer0_attention_weight_min": -12.245757102966309,
      "activations/layer10_attention_weight_max": 34.28074645996094,
      "activations/layer10_attention_weight_min": -32.91621398925781,
      "activations/layer11_attention_weight_max": 34.48948669433594,
      "activations/layer11_attention_weight_min": -33.251773834228516,
      "activations/layer12_attention_weight_max": 24.792850494384766,
      "activations/layer12_attention_weight_min": -23.419248580932617,
      "activations/layer13_attention_weight_max": 47.76580047607422,
      "activations/layer13_attention_weight_min": -31.828876495361328,
      "activations/layer14_attention_weight_max": 54.769412994384766,
      "activations/layer14_attention_weight_min": -42.95763397216797,
      "activations/layer15_attention_weight_max": 38.673004150390625,
      "activations/layer15_attention_weight_min": -30.13768768310547,
      "activations/layer16_attention_weight_max": 34.4770393371582,
      "activations/layer16_attention_weight_min": -26.948198318481445,
      "activations/layer17_attention_weight_max": 37.38237380981445,
      "activations/layer17_attention_weight_min": -23.749446868896484,
      "activations/layer18_attention_weight_max": 40.70977783203125,
      "activations/layer18_attention_weight_min": -21.6557559967041,
      "activations/layer19_attention_weight_max": 40.60714340209961,
      "activations/layer19_attention_weight_min": -28.89111328125,
      "activations/layer1_attention_weight_max": 17.079116821289062,
      "activations/layer1_attention_weight_min": -15.15468978881836,
      "activations/layer20_attention_weight_max": 40.40280532836914,
      "activations/layer20_attention_weight_min": -23.230972290039062,
      "activations/layer21_attention_weight_max": 38.64982223510742,
      "activations/layer21_attention_weight_min": -21.592504501342773,
      "activations/layer22_attention_weight_max": 56.12350845336914,
      "activations/layer22_attention_weight_min": -28.3524112701416,
      "activations/layer23_attention_weight_max": 42.4610710144043,
      "activations/layer23_attention_weight_min": -22.55939483642578,
      "activations/layer2_attention_weight_max": 31.927282333374023,
      "activations/layer2_attention_weight_min": -32.463863372802734,
      "activations/layer3_attention_weight_max": 93.36701202392578,
      "activations/layer3_attention_weight_min": -94.37419128417969,
      "activations/layer4_attention_weight_max": 126.70002746582031,
      "activations/layer4_attention_weight_min": -127.56317138671875,
      "activations/layer5_attention_weight_max": 64.27400207519531,
      "activations/layer5_attention_weight_min": -66.00205993652344,
      "activations/layer6_attention_weight_max": 53.81509017944336,
      "activations/layer6_attention_weight_min": -55.93702697753906,
      "activations/layer7_attention_weight_max": 91.77678680419922,
      "activations/layer7_attention_weight_min": -89.296142578125,
      "activations/layer8_attention_weight_max": 42.55483627319336,
      "activations/layer8_attention_weight_min": -40.2984504699707,
      "activations/layer9_attention_weight_max": 37.92717742919922,
      "activations/layer9_attention_weight_min": -38.47708511352539,
      "epoch": 7.39,
      "learning_rate": 0.00010335757575757575,
      "loss": 2.8299,
      "step": 127250
    },
    {
      "activations/layer0_attention_weight_max": 16.0616512298584,
      "activations/layer0_attention_weight_min": -11.719681739807129,
      "activations/layer10_attention_weight_max": 30.875131607055664,
      "activations/layer10_attention_weight_min": -30.77449607849121,
      "activations/layer11_attention_weight_max": 30.510047912597656,
      "activations/layer11_attention_weight_min": -29.961803436279297,
      "activations/layer12_attention_weight_max": 31.910051345825195,
      "activations/layer12_attention_weight_min": -29.53596305847168,
      "activations/layer13_attention_weight_max": 40.65224838256836,
      "activations/layer13_attention_weight_min": -32.55729293823242,
      "activations/layer14_attention_weight_max": 46.94236373901367,
      "activations/layer14_attention_weight_min": -37.579620361328125,
      "activations/layer15_attention_weight_max": 35.03412628173828,
      "activations/layer15_attention_weight_min": -29.217092514038086,
      "activations/layer16_attention_weight_max": 29.959075927734375,
      "activations/layer16_attention_weight_min": -26.89882469177246,
      "activations/layer17_attention_weight_max": 33.389217376708984,
      "activations/layer17_attention_weight_min": -24.64041519165039,
      "activations/layer18_attention_weight_max": 30.224763870239258,
      "activations/layer18_attention_weight_min": -18.5081844329834,
      "activations/layer19_attention_weight_max": 33.08323287963867,
      "activations/layer19_attention_weight_min": -30.616907119750977,
      "activations/layer1_attention_weight_max": 17.355894088745117,
      "activations/layer1_attention_weight_min": -16.17551040649414,
      "activations/layer20_attention_weight_max": 27.106138229370117,
      "activations/layer20_attention_weight_min": -23.515317916870117,
      "activations/layer21_attention_weight_max": 26.368728637695312,
      "activations/layer21_attention_weight_min": -22.367029190063477,
      "activations/layer22_attention_weight_max": 43.43959426879883,
      "activations/layer22_attention_weight_min": -26.770946502685547,
      "activations/layer23_attention_weight_max": 33.186492919921875,
      "activations/layer23_attention_weight_min": -22.180204391479492,
      "activations/layer2_attention_weight_max": 31.77877426147461,
      "activations/layer2_attention_weight_min": -31.686613082885742,
      "activations/layer3_attention_weight_max": 94.902099609375,
      "activations/layer3_attention_weight_min": -96.39778900146484,
      "activations/layer4_attention_weight_max": 119.83849334716797,
      "activations/layer4_attention_weight_min": -122.89508056640625,
      "activations/layer5_attention_weight_max": 60.576324462890625,
      "activations/layer5_attention_weight_min": -65.65784454345703,
      "activations/layer6_attention_weight_max": 49.49843215942383,
      "activations/layer6_attention_weight_min": -53.73036193847656,
      "activations/layer7_attention_weight_max": 85.99837493896484,
      "activations/layer7_attention_weight_min": -92.26629638671875,
      "activations/layer8_attention_weight_max": 39.66698455810547,
      "activations/layer8_attention_weight_min": -40.497467041015625,
      "activations/layer9_attention_weight_max": 34.87260055541992,
      "activations/layer9_attention_weight_min": -33.15171432495117,
      "epoch": 7.4,
      "learning_rate": 0.00010333863636363635,
      "loss": 2.7955,
      "step": 127300
    },
    {
      "activations/layer0_attention_weight_max": 15.888246536254883,
      "activations/layer0_attention_weight_min": -11.236702919006348,
      "activations/layer10_attention_weight_max": 35.006683349609375,
      "activations/layer10_attention_weight_min": -30.843229293823242,
      "activations/layer11_attention_weight_max": 34.24421310424805,
      "activations/layer11_attention_weight_min": -29.87415885925293,
      "activations/layer12_attention_weight_max": 41.15751266479492,
      "activations/layer12_attention_weight_min": -23.609079360961914,
      "activations/layer13_attention_weight_max": 42.026023864746094,
      "activations/layer13_attention_weight_min": -34.43513488769531,
      "activations/layer14_attention_weight_max": 55.887855529785156,
      "activations/layer14_attention_weight_min": -44.54622268676758,
      "activations/layer15_attention_weight_max": 36.907005310058594,
      "activations/layer15_attention_weight_min": -30.451562881469727,
      "activations/layer16_attention_weight_max": 33.13937759399414,
      "activations/layer16_attention_weight_min": -28.024316787719727,
      "activations/layer17_attention_weight_max": 34.072566986083984,
      "activations/layer17_attention_weight_min": -24.780797958374023,
      "activations/layer18_attention_weight_max": 33.678199768066406,
      "activations/layer18_attention_weight_min": -21.0761661529541,
      "activations/layer19_attention_weight_max": 37.25321578979492,
      "activations/layer19_attention_weight_min": -32.0859375,
      "activations/layer1_attention_weight_max": 17.750627517700195,
      "activations/layer1_attention_weight_min": -14.88198184967041,
      "activations/layer20_attention_weight_max": 32.555599212646484,
      "activations/layer20_attention_weight_min": -26.55449867248535,
      "activations/layer21_attention_weight_max": 31.625986099243164,
      "activations/layer21_attention_weight_min": -23.197416305541992,
      "activations/layer22_attention_weight_max": 49.666229248046875,
      "activations/layer22_attention_weight_min": -28.620988845825195,
      "activations/layer23_attention_weight_max": 38.72991180419922,
      "activations/layer23_attention_weight_min": -23.261564254760742,
      "activations/layer2_attention_weight_max": 33.001102447509766,
      "activations/layer2_attention_weight_min": -32.489017486572266,
      "activations/layer3_attention_weight_max": 94.80377960205078,
      "activations/layer3_attention_weight_min": -95.17903900146484,
      "activations/layer4_attention_weight_max": 129.65524291992188,
      "activations/layer4_attention_weight_min": -123.78517150878906,
      "activations/layer5_attention_weight_max": 66.8819580078125,
      "activations/layer5_attention_weight_min": -66.76946258544922,
      "activations/layer6_attention_weight_max": 51.29300308227539,
      "activations/layer6_attention_weight_min": -54.69923782348633,
      "activations/layer7_attention_weight_max": 88.34468841552734,
      "activations/layer7_attention_weight_min": -86.89215850830078,
      "activations/layer8_attention_weight_max": 42.6744384765625,
      "activations/layer8_attention_weight_min": -41.68048095703125,
      "activations/layer9_attention_weight_max": 37.444183349609375,
      "activations/layer9_attention_weight_min": -34.91865158081055,
      "epoch": 7.4,
      "learning_rate": 0.00010331969696969695,
      "loss": 2.8198,
      "step": 127350
    },
    {
      "activations/layer0_attention_weight_max": 15.926712036132812,
      "activations/layer0_attention_weight_min": -10.812206268310547,
      "activations/layer10_attention_weight_max": 31.975236892700195,
      "activations/layer10_attention_weight_min": -30.636734008789062,
      "activations/layer11_attention_weight_max": 32.188377380371094,
      "activations/layer11_attention_weight_min": -28.8548641204834,
      "activations/layer12_attention_weight_max": 25.277849197387695,
      "activations/layer12_attention_weight_min": -27.034835815429688,
      "activations/layer13_attention_weight_max": 40.335662841796875,
      "activations/layer13_attention_weight_min": -32.745697021484375,
      "activations/layer14_attention_weight_max": 51.98577880859375,
      "activations/layer14_attention_weight_min": -38.73408508300781,
      "activations/layer15_attention_weight_max": 37.185028076171875,
      "activations/layer15_attention_weight_min": -29.766056060791016,
      "activations/layer16_attention_weight_max": 32.12873840332031,
      "activations/layer16_attention_weight_min": -28.2989501953125,
      "activations/layer17_attention_weight_max": 34.21870803833008,
      "activations/layer17_attention_weight_min": -24.512805938720703,
      "activations/layer18_attention_weight_max": 35.585025787353516,
      "activations/layer18_attention_weight_min": -21.637609481811523,
      "activations/layer19_attention_weight_max": 39.520389556884766,
      "activations/layer19_attention_weight_min": -29.649099349975586,
      "activations/layer1_attention_weight_max": 17.499805450439453,
      "activations/layer1_attention_weight_min": -16.675785064697266,
      "activations/layer20_attention_weight_max": 35.18648910522461,
      "activations/layer20_attention_weight_min": -24.439863204956055,
      "activations/layer21_attention_weight_max": 32.72963333129883,
      "activations/layer21_attention_weight_min": -23.203462600708008,
      "activations/layer22_attention_weight_max": 52.09513473510742,
      "activations/layer22_attention_weight_min": -29.79615020751953,
      "activations/layer23_attention_weight_max": 34.139320373535156,
      "activations/layer23_attention_weight_min": -22.10871124267578,
      "activations/layer2_attention_weight_max": 31.15636444091797,
      "activations/layer2_attention_weight_min": -31.359655380249023,
      "activations/layer3_attention_weight_max": 88.98783111572266,
      "activations/layer3_attention_weight_min": -92.74383544921875,
      "activations/layer4_attention_weight_max": 120.26287078857422,
      "activations/layer4_attention_weight_min": -120.30492401123047,
      "activations/layer5_attention_weight_max": 60.714500427246094,
      "activations/layer5_attention_weight_min": -60.58669662475586,
      "activations/layer6_attention_weight_max": 50.07307052612305,
      "activations/layer6_attention_weight_min": -50.22319793701172,
      "activations/layer7_attention_weight_max": 84.27081298828125,
      "activations/layer7_attention_weight_min": -86.3595199584961,
      "activations/layer8_attention_weight_max": 39.57947540283203,
      "activations/layer8_attention_weight_min": -39.163330078125,
      "activations/layer9_attention_weight_max": 35.68199920654297,
      "activations/layer9_attention_weight_min": -32.46645736694336,
      "epoch": 7.4,
      "learning_rate": 0.00010330075757575757,
      "loss": 2.8052,
      "step": 127400
    },
    {
      "activations/layer0_attention_weight_max": 15.673815727233887,
      "activations/layer0_attention_weight_min": -11.53448486328125,
      "activations/layer10_attention_weight_max": 32.75326919555664,
      "activations/layer10_attention_weight_min": -35.752838134765625,
      "activations/layer11_attention_weight_max": 34.25514602661133,
      "activations/layer11_attention_weight_min": -29.45360565185547,
      "activations/layer12_attention_weight_max": 23.258359909057617,
      "activations/layer12_attention_weight_min": -27.343585968017578,
      "activations/layer13_attention_weight_max": 43.532752990722656,
      "activations/layer13_attention_weight_min": -33.540138244628906,
      "activations/layer14_attention_weight_max": 50.876590728759766,
      "activations/layer14_attention_weight_min": -39.72689437866211,
      "activations/layer15_attention_weight_max": 38.092140197753906,
      "activations/layer15_attention_weight_min": -29.683488845825195,
      "activations/layer16_attention_weight_max": 35.559444427490234,
      "activations/layer16_attention_weight_min": -29.966833114624023,
      "activations/layer17_attention_weight_max": 37.388179779052734,
      "activations/layer17_attention_weight_min": -25.932191848754883,
      "activations/layer18_attention_weight_max": 34.04548263549805,
      "activations/layer18_attention_weight_min": -21.421220779418945,
      "activations/layer19_attention_weight_max": 39.93839645385742,
      "activations/layer19_attention_weight_min": -30.63702964782715,
      "activations/layer1_attention_weight_max": 16.99497413635254,
      "activations/layer1_attention_weight_min": -14.08576488494873,
      "activations/layer20_attention_weight_max": 32.30092239379883,
      "activations/layer20_attention_weight_min": -25.066316604614258,
      "activations/layer21_attention_weight_max": 31.948537826538086,
      "activations/layer21_attention_weight_min": -22.489126205444336,
      "activations/layer22_attention_weight_max": 50.23771286010742,
      "activations/layer22_attention_weight_min": -31.042932510375977,
      "activations/layer23_attention_weight_max": 39.40007781982422,
      "activations/layer23_attention_weight_min": -23.974166870117188,
      "activations/layer2_attention_weight_max": 32.129486083984375,
      "activations/layer2_attention_weight_min": -32.195255279541016,
      "activations/layer3_attention_weight_max": 90.42304229736328,
      "activations/layer3_attention_weight_min": -93.1800308227539,
      "activations/layer4_attention_weight_max": 124.39851379394531,
      "activations/layer4_attention_weight_min": -120.23695373535156,
      "activations/layer5_attention_weight_max": 61.102516174316406,
      "activations/layer5_attention_weight_min": -64.32005310058594,
      "activations/layer6_attention_weight_max": 49.502235412597656,
      "activations/layer6_attention_weight_min": -50.92525863647461,
      "activations/layer7_attention_weight_max": 87.4516830444336,
      "activations/layer7_attention_weight_min": -87.34449005126953,
      "activations/layer8_attention_weight_max": 38.26356887817383,
      "activations/layer8_attention_weight_min": -43.811119079589844,
      "activations/layer9_attention_weight_max": 34.54367446899414,
      "activations/layer9_attention_weight_min": -36.53974533081055,
      "epoch": 7.41,
      "learning_rate": 0.00010328181818181817,
      "loss": 2.8083,
      "step": 127450
    },
    {
      "activations/layer0_attention_weight_max": 16.381166458129883,
      "activations/layer0_attention_weight_min": -11.74907398223877,
      "activations/layer10_attention_weight_max": 37.3353157043457,
      "activations/layer10_attention_weight_min": -34.60342788696289,
      "activations/layer11_attention_weight_max": 36.14643478393555,
      "activations/layer11_attention_weight_min": -31.135719299316406,
      "activations/layer12_attention_weight_max": 23.36200714111328,
      "activations/layer12_attention_weight_min": -35.21486282348633,
      "activations/layer13_attention_weight_max": 42.443946838378906,
      "activations/layer13_attention_weight_min": -35.6510009765625,
      "activations/layer14_attention_weight_max": 58.47169494628906,
      "activations/layer14_attention_weight_min": -42.5046501159668,
      "activations/layer15_attention_weight_max": 44.612491607666016,
      "activations/layer15_attention_weight_min": -30.546953201293945,
      "activations/layer16_attention_weight_max": 33.294639587402344,
      "activations/layer16_attention_weight_min": -29.66981315612793,
      "activations/layer17_attention_weight_max": 34.40898513793945,
      "activations/layer17_attention_weight_min": -24.702028274536133,
      "activations/layer18_attention_weight_max": 31.510713577270508,
      "activations/layer18_attention_weight_min": -21.364656448364258,
      "activations/layer19_attention_weight_max": 39.019344329833984,
      "activations/layer19_attention_weight_min": -29.6560115814209,
      "activations/layer1_attention_weight_max": 16.063966751098633,
      "activations/layer1_attention_weight_min": -15.250955581665039,
      "activations/layer20_attention_weight_max": 30.485212326049805,
      "activations/layer20_attention_weight_min": -24.16550636291504,
      "activations/layer21_attention_weight_max": 27.71883773803711,
      "activations/layer21_attention_weight_min": -20.367233276367188,
      "activations/layer22_attention_weight_max": 50.705997467041016,
      "activations/layer22_attention_weight_min": -29.641225814819336,
      "activations/layer23_attention_weight_max": 37.83388137817383,
      "activations/layer23_attention_weight_min": -22.023733139038086,
      "activations/layer2_attention_weight_max": 31.585647583007812,
      "activations/layer2_attention_weight_min": -30.494197845458984,
      "activations/layer3_attention_weight_max": 92.00454711914062,
      "activations/layer3_attention_weight_min": -92.40901184082031,
      "activations/layer4_attention_weight_max": 127.439208984375,
      "activations/layer4_attention_weight_min": -118.02168273925781,
      "activations/layer5_attention_weight_max": 65.53895568847656,
      "activations/layer5_attention_weight_min": -62.10764694213867,
      "activations/layer6_attention_weight_max": 51.867088317871094,
      "activations/layer6_attention_weight_min": -53.447731018066406,
      "activations/layer7_attention_weight_max": 99.66017150878906,
      "activations/layer7_attention_weight_min": -91.80313110351562,
      "activations/layer8_attention_weight_max": 43.30866622924805,
      "activations/layer8_attention_weight_min": -43.25157165527344,
      "activations/layer9_attention_weight_max": 40.03986740112305,
      "activations/layer9_attention_weight_min": -35.36532211303711,
      "epoch": 7.41,
      "learning_rate": 0.00010326287878787877,
      "loss": 2.8073,
      "step": 127500
    },
    {
      "activations/layer0_attention_weight_max": 15.773653030395508,
      "activations/layer0_attention_weight_min": -11.145231246948242,
      "activations/layer10_attention_weight_max": 34.23457336425781,
      "activations/layer10_attention_weight_min": -35.88859558105469,
      "activations/layer11_attention_weight_max": 31.921842575073242,
      "activations/layer11_attention_weight_min": -30.900907516479492,
      "activations/layer12_attention_weight_max": 23.0645751953125,
      "activations/layer12_attention_weight_min": -29.197505950927734,
      "activations/layer13_attention_weight_max": 42.70314025878906,
      "activations/layer13_attention_weight_min": -34.12384796142578,
      "activations/layer14_attention_weight_max": 65.1147689819336,
      "activations/layer14_attention_weight_min": -49.084781646728516,
      "activations/layer15_attention_weight_max": 38.92415237426758,
      "activations/layer15_attention_weight_min": -32.24037170410156,
      "activations/layer16_attention_weight_max": 36.38689422607422,
      "activations/layer16_attention_weight_min": -29.559551239013672,
      "activations/layer17_attention_weight_max": 38.992897033691406,
      "activations/layer17_attention_weight_min": -26.62009620666504,
      "activations/layer18_attention_weight_max": 32.28413772583008,
      "activations/layer18_attention_weight_min": -23.518417358398438,
      "activations/layer19_attention_weight_max": 40.121192932128906,
      "activations/layer19_attention_weight_min": -33.5790901184082,
      "activations/layer1_attention_weight_max": 17.29176139831543,
      "activations/layer1_attention_weight_min": -13.943378448486328,
      "activations/layer20_attention_weight_max": 35.85387420654297,
      "activations/layer20_attention_weight_min": -27.26095199584961,
      "activations/layer21_attention_weight_max": 37.01537322998047,
      "activations/layer21_attention_weight_min": -24.740568161010742,
      "activations/layer22_attention_weight_max": 55.44416809082031,
      "activations/layer22_attention_weight_min": -32.09856033325195,
      "activations/layer23_attention_weight_max": 37.02264404296875,
      "activations/layer23_attention_weight_min": -24.6805477142334,
      "activations/layer2_attention_weight_max": 30.636032104492188,
      "activations/layer2_attention_weight_min": -31.233755111694336,
      "activations/layer3_attention_weight_max": 89.9629898071289,
      "activations/layer3_attention_weight_min": -89.60501098632812,
      "activations/layer4_attention_weight_max": 127.03771209716797,
      "activations/layer4_attention_weight_min": -127.3624038696289,
      "activations/layer5_attention_weight_max": 67.16020965576172,
      "activations/layer5_attention_weight_min": -63.98629379272461,
      "activations/layer6_attention_weight_max": 50.84614562988281,
      "activations/layer6_attention_weight_min": -51.49137878417969,
      "activations/layer7_attention_weight_max": 93.04940032958984,
      "activations/layer7_attention_weight_min": -94.8766098022461,
      "activations/layer8_attention_weight_max": 39.524112701416016,
      "activations/layer8_attention_weight_min": -42.28255081176758,
      "activations/layer9_attention_weight_max": 35.05573272705078,
      "activations/layer9_attention_weight_min": -36.19807052612305,
      "epoch": 7.41,
      "learning_rate": 0.00010324393939393939,
      "loss": 2.8226,
      "step": 127550
    },
    {
      "activations/layer0_attention_weight_max": 15.736297607421875,
      "activations/layer0_attention_weight_min": -12.250465393066406,
      "activations/layer10_attention_weight_max": 32.766845703125,
      "activations/layer10_attention_weight_min": -31.777950286865234,
      "activations/layer11_attention_weight_max": 31.990386962890625,
      "activations/layer11_attention_weight_min": -28.557933807373047,
      "activations/layer12_attention_weight_max": 44.599395751953125,
      "activations/layer12_attention_weight_min": -27.14276885986328,
      "activations/layer13_attention_weight_max": 51.584964752197266,
      "activations/layer13_attention_weight_min": -32.42005157470703,
      "activations/layer14_attention_weight_max": 47.25263214111328,
      "activations/layer14_attention_weight_min": -41.750755310058594,
      "activations/layer15_attention_weight_max": 35.77674102783203,
      "activations/layer15_attention_weight_min": -27.811323165893555,
      "activations/layer16_attention_weight_max": 31.807666778564453,
      "activations/layer16_attention_weight_min": -27.244495391845703,
      "activations/layer17_attention_weight_max": 36.559165954589844,
      "activations/layer17_attention_weight_min": -24.913192749023438,
      "activations/layer18_attention_weight_max": 35.913177490234375,
      "activations/layer18_attention_weight_min": -24.411090850830078,
      "activations/layer19_attention_weight_max": 32.48483657836914,
      "activations/layer19_attention_weight_min": -29.09549903869629,
      "activations/layer1_attention_weight_max": 16.677364349365234,
      "activations/layer1_attention_weight_min": -14.281220436096191,
      "activations/layer20_attention_weight_max": 27.513689041137695,
      "activations/layer20_attention_weight_min": -23.378673553466797,
      "activations/layer21_attention_weight_max": 28.796213150024414,
      "activations/layer21_attention_weight_min": -20.856313705444336,
      "activations/layer22_attention_weight_max": 45.246944427490234,
      "activations/layer22_attention_weight_min": -28.972715377807617,
      "activations/layer23_attention_weight_max": 35.16967010498047,
      "activations/layer23_attention_weight_min": -21.811246871948242,
      "activations/layer2_attention_weight_max": 33.65401840209961,
      "activations/layer2_attention_weight_min": -31.887609481811523,
      "activations/layer3_attention_weight_max": 95.79097747802734,
      "activations/layer3_attention_weight_min": -95.0538330078125,
      "activations/layer4_attention_weight_max": 121.04878997802734,
      "activations/layer4_attention_weight_min": -118.46843719482422,
      "activations/layer5_attention_weight_max": 63.911407470703125,
      "activations/layer5_attention_weight_min": -60.99821090698242,
      "activations/layer6_attention_weight_max": 50.96018981933594,
      "activations/layer6_attention_weight_min": -51.610267639160156,
      "activations/layer7_attention_weight_max": 85.30744934082031,
      "activations/layer7_attention_weight_min": -84.59423065185547,
      "activations/layer8_attention_weight_max": 37.86007308959961,
      "activations/layer8_attention_weight_min": -39.68629455566406,
      "activations/layer9_attention_weight_max": 33.97039794921875,
      "activations/layer9_attention_weight_min": -32.688270568847656,
      "epoch": 7.41,
      "learning_rate": 0.000103225,
      "loss": 2.8338,
      "step": 127600
    },
    {
      "activations/layer0_attention_weight_max": 15.443562507629395,
      "activations/layer0_attention_weight_min": -12.314271926879883,
      "activations/layer10_attention_weight_max": 34.78285598754883,
      "activations/layer10_attention_weight_min": -31.9909725189209,
      "activations/layer11_attention_weight_max": 37.19929122924805,
      "activations/layer11_attention_weight_min": -29.360036849975586,
      "activations/layer12_attention_weight_max": 33.797183990478516,
      "activations/layer12_attention_weight_min": -36.21089553833008,
      "activations/layer13_attention_weight_max": 46.75315475463867,
      "activations/layer13_attention_weight_min": -33.64688491821289,
      "activations/layer14_attention_weight_max": 58.54676818847656,
      "activations/layer14_attention_weight_min": -37.484718322753906,
      "activations/layer15_attention_weight_max": 46.239601135253906,
      "activations/layer15_attention_weight_min": -31.056184768676758,
      "activations/layer16_attention_weight_max": 36.18056106567383,
      "activations/layer16_attention_weight_min": -30.078994750976562,
      "activations/layer17_attention_weight_max": 40.788082122802734,
      "activations/layer17_attention_weight_min": -25.738304138183594,
      "activations/layer18_attention_weight_max": 33.87043380737305,
      "activations/layer18_attention_weight_min": -21.952884674072266,
      "activations/layer19_attention_weight_max": 38.49601364135742,
      "activations/layer19_attention_weight_min": -29.141355514526367,
      "activations/layer1_attention_weight_max": 18.260570526123047,
      "activations/layer1_attention_weight_min": -13.857138633728027,
      "activations/layer20_attention_weight_max": 35.527687072753906,
      "activations/layer20_attention_weight_min": -22.711593627929688,
      "activations/layer21_attention_weight_max": 33.84550857543945,
      "activations/layer21_attention_weight_min": -21.828651428222656,
      "activations/layer22_attention_weight_max": 51.42582321166992,
      "activations/layer22_attention_weight_min": -30.944292068481445,
      "activations/layer23_attention_weight_max": 38.73397445678711,
      "activations/layer23_attention_weight_min": -22.80397605895996,
      "activations/layer2_attention_weight_max": 34.14142608642578,
      "activations/layer2_attention_weight_min": -32.199241638183594,
      "activations/layer3_attention_weight_max": 95.1761245727539,
      "activations/layer3_attention_weight_min": -95.56976318359375,
      "activations/layer4_attention_weight_max": 128.25796508789062,
      "activations/layer4_attention_weight_min": -120.75746154785156,
      "activations/layer5_attention_weight_max": 63.025054931640625,
      "activations/layer5_attention_weight_min": -59.22001647949219,
      "activations/layer6_attention_weight_max": 52.39472198486328,
      "activations/layer6_attention_weight_min": -52.84027862548828,
      "activations/layer7_attention_weight_max": 91.93508911132812,
      "activations/layer7_attention_weight_min": -86.81412506103516,
      "activations/layer8_attention_weight_max": 41.746402740478516,
      "activations/layer8_attention_weight_min": -39.70026779174805,
      "activations/layer9_attention_weight_max": 36.666160583496094,
      "activations/layer9_attention_weight_min": -33.72365951538086,
      "epoch": 7.42,
      "learning_rate": 0.00010320606060606059,
      "loss": 2.797,
      "step": 127650
    },
    {
      "activations/layer0_attention_weight_max": 15.851798057556152,
      "activations/layer0_attention_weight_min": -11.385749816894531,
      "activations/layer10_attention_weight_max": 31.521738052368164,
      "activations/layer10_attention_weight_min": -30.339336395263672,
      "activations/layer11_attention_weight_max": 32.05809020996094,
      "activations/layer11_attention_weight_min": -29.367374420166016,
      "activations/layer12_attention_weight_max": 22.604312896728516,
      "activations/layer12_attention_weight_min": -26.02715301513672,
      "activations/layer13_attention_weight_max": 37.65019226074219,
      "activations/layer13_attention_weight_min": -32.94797897338867,
      "activations/layer14_attention_weight_max": 47.018699645996094,
      "activations/layer14_attention_weight_min": -40.2397346496582,
      "activations/layer15_attention_weight_max": 35.25478744506836,
      "activations/layer15_attention_weight_min": -29.6667423248291,
      "activations/layer16_attention_weight_max": 33.18338394165039,
      "activations/layer16_attention_weight_min": -28.889575958251953,
      "activations/layer17_attention_weight_max": 36.553585052490234,
      "activations/layer17_attention_weight_min": -25.446916580200195,
      "activations/layer18_attention_weight_max": 37.94940185546875,
      "activations/layer18_attention_weight_min": -21.845491409301758,
      "activations/layer19_attention_weight_max": 39.22268295288086,
      "activations/layer19_attention_weight_min": -34.999568939208984,
      "activations/layer1_attention_weight_max": 16.822202682495117,
      "activations/layer1_attention_weight_min": -15.861698150634766,
      "activations/layer20_attention_weight_max": 30.8739070892334,
      "activations/layer20_attention_weight_min": -25.742897033691406,
      "activations/layer21_attention_weight_max": 29.569622039794922,
      "activations/layer21_attention_weight_min": -22.343109130859375,
      "activations/layer22_attention_weight_max": 46.16299057006836,
      "activations/layer22_attention_weight_min": -30.353607177734375,
      "activations/layer23_attention_weight_max": 35.83527374267578,
      "activations/layer23_attention_weight_min": -26.62221336364746,
      "activations/layer2_attention_weight_max": 31.234617233276367,
      "activations/layer2_attention_weight_min": -30.018301010131836,
      "activations/layer3_attention_weight_max": 90.83027648925781,
      "activations/layer3_attention_weight_min": -89.7424545288086,
      "activations/layer4_attention_weight_max": 120.65199279785156,
      "activations/layer4_attention_weight_min": -116.1150894165039,
      "activations/layer5_attention_weight_max": 60.738670349121094,
      "activations/layer5_attention_weight_min": -58.13325119018555,
      "activations/layer6_attention_weight_max": 50.96540832519531,
      "activations/layer6_attention_weight_min": -49.61251449584961,
      "activations/layer7_attention_weight_max": 84.76943969726562,
      "activations/layer7_attention_weight_min": -88.46586608886719,
      "activations/layer8_attention_weight_max": 39.50523376464844,
      "activations/layer8_attention_weight_min": -39.27690124511719,
      "activations/layer9_attention_weight_max": 33.29621124267578,
      "activations/layer9_attention_weight_min": -32.49972915649414,
      "epoch": 7.42,
      "learning_rate": 0.0001031871212121212,
      "loss": 2.8284,
      "step": 127700
    },
    {
      "activations/layer0_attention_weight_max": 17.21213150024414,
      "activations/layer0_attention_weight_min": -10.649568557739258,
      "activations/layer10_attention_weight_max": 35.154823303222656,
      "activations/layer10_attention_weight_min": -33.763736724853516,
      "activations/layer11_attention_weight_max": 36.4379997253418,
      "activations/layer11_attention_weight_min": -31.33393669128418,
      "activations/layer12_attention_weight_max": 34.48389434814453,
      "activations/layer12_attention_weight_min": -30.8203182220459,
      "activations/layer13_attention_weight_max": 47.120906829833984,
      "activations/layer13_attention_weight_min": -34.835880279541016,
      "activations/layer14_attention_weight_max": 57.37068557739258,
      "activations/layer14_attention_weight_min": -43.51449966430664,
      "activations/layer15_attention_weight_max": 39.46857452392578,
      "activations/layer15_attention_weight_min": -28.840404510498047,
      "activations/layer16_attention_weight_max": 32.99034881591797,
      "activations/layer16_attention_weight_min": -29.41135025024414,
      "activations/layer17_attention_weight_max": 44.83055114746094,
      "activations/layer17_attention_weight_min": -26.954444885253906,
      "activations/layer18_attention_weight_max": 38.59048080444336,
      "activations/layer18_attention_weight_min": -23.412960052490234,
      "activations/layer19_attention_weight_max": 39.60064697265625,
      "activations/layer19_attention_weight_min": -30.46611976623535,
      "activations/layer1_attention_weight_max": 18.066158294677734,
      "activations/layer1_attention_weight_min": -13.662266731262207,
      "activations/layer20_attention_weight_max": 36.965110778808594,
      "activations/layer20_attention_weight_min": -22.888059616088867,
      "activations/layer21_attention_weight_max": 36.11289596557617,
      "activations/layer21_attention_weight_min": -22.238269805908203,
      "activations/layer22_attention_weight_max": 52.14158248901367,
      "activations/layer22_attention_weight_min": -28.7625789642334,
      "activations/layer23_attention_weight_max": 44.033958435058594,
      "activations/layer23_attention_weight_min": -22.537006378173828,
      "activations/layer2_attention_weight_max": 37.704925537109375,
      "activations/layer2_attention_weight_min": -34.02406311035156,
      "activations/layer3_attention_weight_max": 97.5482406616211,
      "activations/layer3_attention_weight_min": -98.77143859863281,
      "activations/layer4_attention_weight_max": 126.03314208984375,
      "activations/layer4_attention_weight_min": -124.04781341552734,
      "activations/layer5_attention_weight_max": 64.4593734741211,
      "activations/layer5_attention_weight_min": -60.63929748535156,
      "activations/layer6_attention_weight_max": 52.715023040771484,
      "activations/layer6_attention_weight_min": -53.82925033569336,
      "activations/layer7_attention_weight_max": 102.22888946533203,
      "activations/layer7_attention_weight_min": -94.85327911376953,
      "activations/layer8_attention_weight_max": 43.16261291503906,
      "activations/layer8_attention_weight_min": -43.563446044921875,
      "activations/layer9_attention_weight_max": 38.74485397338867,
      "activations/layer9_attention_weight_min": -40.59108352661133,
      "epoch": 7.42,
      "learning_rate": 0.00010316818181818182,
      "loss": 2.8009,
      "step": 127750
    },
    {
      "activations/layer0_attention_weight_max": 15.698518753051758,
      "activations/layer0_attention_weight_min": -10.87907600402832,
      "activations/layer10_attention_weight_max": 33.00305938720703,
      "activations/layer10_attention_weight_min": -30.84856414794922,
      "activations/layer11_attention_weight_max": 30.698286056518555,
      "activations/layer11_attention_weight_min": -29.937679290771484,
      "activations/layer12_attention_weight_max": 35.12837600708008,
      "activations/layer12_attention_weight_min": -28.678495407104492,
      "activations/layer13_attention_weight_max": 40.419246673583984,
      "activations/layer13_attention_weight_min": -33.10781478881836,
      "activations/layer14_attention_weight_max": 53.07244110107422,
      "activations/layer14_attention_weight_min": -43.91041946411133,
      "activations/layer15_attention_weight_max": 35.22300338745117,
      "activations/layer15_attention_weight_min": -31.688106536865234,
      "activations/layer16_attention_weight_max": 33.26333236694336,
      "activations/layer16_attention_weight_min": -28.894819259643555,
      "activations/layer17_attention_weight_max": 35.20454025268555,
      "activations/layer17_attention_weight_min": -26.836956024169922,
      "activations/layer18_attention_weight_max": 30.02519416809082,
      "activations/layer18_attention_weight_min": -22.45867347717285,
      "activations/layer19_attention_weight_max": 37.823692321777344,
      "activations/layer19_attention_weight_min": -30.89918327331543,
      "activations/layer1_attention_weight_max": 17.450124740600586,
      "activations/layer1_attention_weight_min": -13.800350189208984,
      "activations/layer20_attention_weight_max": 28.952659606933594,
      "activations/layer20_attention_weight_min": -25.375347137451172,
      "activations/layer21_attention_weight_max": 30.875516891479492,
      "activations/layer21_attention_weight_min": -24.871423721313477,
      "activations/layer22_attention_weight_max": 45.931640625,
      "activations/layer22_attention_weight_min": -34.36208724975586,
      "activations/layer23_attention_weight_max": 38.328125,
      "activations/layer23_attention_weight_min": -26.59832000732422,
      "activations/layer2_attention_weight_max": 33.9786376953125,
      "activations/layer2_attention_weight_min": -32.079219818115234,
      "activations/layer3_attention_weight_max": 91.65380096435547,
      "activations/layer3_attention_weight_min": -95.36502075195312,
      "activations/layer4_attention_weight_max": 121.69083404541016,
      "activations/layer4_attention_weight_min": -122.14554595947266,
      "activations/layer5_attention_weight_max": 61.422054290771484,
      "activations/layer5_attention_weight_min": -63.24746322631836,
      "activations/layer6_attention_weight_max": 50.17087173461914,
      "activations/layer6_attention_weight_min": -51.79839324951172,
      "activations/layer7_attention_weight_max": 84.89663696289062,
      "activations/layer7_attention_weight_min": -86.19091796875,
      "activations/layer8_attention_weight_max": 37.15078353881836,
      "activations/layer8_attention_weight_min": -39.93418884277344,
      "activations/layer9_attention_weight_max": 35.4910888671875,
      "activations/layer9_attention_weight_min": -33.663326263427734,
      "epoch": 7.43,
      "learning_rate": 0.00010314924242424241,
      "loss": 2.8319,
      "step": 127800
    },
    {
      "activations/layer0_attention_weight_max": 16.543237686157227,
      "activations/layer0_attention_weight_min": -10.905560493469238,
      "activations/layer10_attention_weight_max": 32.872520446777344,
      "activations/layer10_attention_weight_min": -33.75384521484375,
      "activations/layer11_attention_weight_max": 33.67727279663086,
      "activations/layer11_attention_weight_min": -30.601369857788086,
      "activations/layer12_attention_weight_max": 23.48906707763672,
      "activations/layer12_attention_weight_min": -28.92059898376465,
      "activations/layer13_attention_weight_max": 42.184967041015625,
      "activations/layer13_attention_weight_min": -35.09868240356445,
      "activations/layer14_attention_weight_max": 50.344539642333984,
      "activations/layer14_attention_weight_min": -43.87666320800781,
      "activations/layer15_attention_weight_max": 38.59920120239258,
      "activations/layer15_attention_weight_min": -30.120018005371094,
      "activations/layer16_attention_weight_max": 33.35336685180664,
      "activations/layer16_attention_weight_min": -27.803089141845703,
      "activations/layer17_attention_weight_max": 32.920013427734375,
      "activations/layer17_attention_weight_min": -26.5013370513916,
      "activations/layer18_attention_weight_max": 34.4476432800293,
      "activations/layer18_attention_weight_min": -21.15068244934082,
      "activations/layer19_attention_weight_max": 45.040157318115234,
      "activations/layer19_attention_weight_min": -32.68244934082031,
      "activations/layer1_attention_weight_max": 17.92254638671875,
      "activations/layer1_attention_weight_min": -14.356118202209473,
      "activations/layer20_attention_weight_max": 34.697654724121094,
      "activations/layer20_attention_weight_min": -23.763469696044922,
      "activations/layer21_attention_weight_max": 35.0102424621582,
      "activations/layer21_attention_weight_min": -22.86970329284668,
      "activations/layer22_attention_weight_max": 50.24507522583008,
      "activations/layer22_attention_weight_min": -30.030420303344727,
      "activations/layer23_attention_weight_max": 39.592315673828125,
      "activations/layer23_attention_weight_min": -24.000926971435547,
      "activations/layer2_attention_weight_max": 32.28893280029297,
      "activations/layer2_attention_weight_min": -30.615406036376953,
      "activations/layer3_attention_weight_max": 93.20867156982422,
      "activations/layer3_attention_weight_min": -90.7478256225586,
      "activations/layer4_attention_weight_max": 124.17496490478516,
      "activations/layer4_attention_weight_min": -116.91155242919922,
      "activations/layer5_attention_weight_max": 64.1462173461914,
      "activations/layer5_attention_weight_min": -61.901344299316406,
      "activations/layer6_attention_weight_max": 49.418827056884766,
      "activations/layer6_attention_weight_min": -51.258480072021484,
      "activations/layer7_attention_weight_max": 87.68357849121094,
      "activations/layer7_attention_weight_min": -87.48323059082031,
      "activations/layer8_attention_weight_max": 40.29457473754883,
      "activations/layer8_attention_weight_min": -39.49896240234375,
      "activations/layer9_attention_weight_max": 37.213382720947266,
      "activations/layer9_attention_weight_min": -35.62263488769531,
      "epoch": 7.43,
      "learning_rate": 0.00010313030303030302,
      "loss": 2.8118,
      "step": 127850
    },
    {
      "activations/layer0_attention_weight_max": 16.117460250854492,
      "activations/layer0_attention_weight_min": -10.725092887878418,
      "activations/layer10_attention_weight_max": 37.487579345703125,
      "activations/layer10_attention_weight_min": -32.9556999206543,
      "activations/layer11_attention_weight_max": 40.12571334838867,
      "activations/layer11_attention_weight_min": -30.979337692260742,
      "activations/layer12_attention_weight_max": 28.773160934448242,
      "activations/layer12_attention_weight_min": -25.439620971679688,
      "activations/layer13_attention_weight_max": 41.38087463378906,
      "activations/layer13_attention_weight_min": -32.25447082519531,
      "activations/layer14_attention_weight_max": 54.059017181396484,
      "activations/layer14_attention_weight_min": -39.40267562866211,
      "activations/layer15_attention_weight_max": 44.203304290771484,
      "activations/layer15_attention_weight_min": -29.24018669128418,
      "activations/layer16_attention_weight_max": 33.84275817871094,
      "activations/layer16_attention_weight_min": -27.419742584228516,
      "activations/layer17_attention_weight_max": 38.13971710205078,
      "activations/layer17_attention_weight_min": -23.96345329284668,
      "activations/layer18_attention_weight_max": 35.90153503417969,
      "activations/layer18_attention_weight_min": -21.399620056152344,
      "activations/layer19_attention_weight_max": 40.19747543334961,
      "activations/layer19_attention_weight_min": -30.218408584594727,
      "activations/layer1_attention_weight_max": 18.35755157470703,
      "activations/layer1_attention_weight_min": -14.301456451416016,
      "activations/layer20_attention_weight_max": 36.86576843261719,
      "activations/layer20_attention_weight_min": -24.24992561340332,
      "activations/layer21_attention_weight_max": 34.95140838623047,
      "activations/layer21_attention_weight_min": -22.739015579223633,
      "activations/layer22_attention_weight_max": 62.05796432495117,
      "activations/layer22_attention_weight_min": -28.336078643798828,
      "activations/layer23_attention_weight_max": 41.27893829345703,
      "activations/layer23_attention_weight_min": -21.706729888916016,
      "activations/layer2_attention_weight_max": 31.217557907104492,
      "activations/layer2_attention_weight_min": -29.217191696166992,
      "activations/layer3_attention_weight_max": 94.00048828125,
      "activations/layer3_attention_weight_min": -93.62246704101562,
      "activations/layer4_attention_weight_max": 125.0982666015625,
      "activations/layer4_attention_weight_min": -118.8498764038086,
      "activations/layer5_attention_weight_max": 64.51287841796875,
      "activations/layer5_attention_weight_min": -62.77116394042969,
      "activations/layer6_attention_weight_max": 53.56333923339844,
      "activations/layer6_attention_weight_min": -53.968143463134766,
      "activations/layer7_attention_weight_max": 97.21965026855469,
      "activations/layer7_attention_weight_min": -88.2623291015625,
      "activations/layer8_attention_weight_max": 42.76140594482422,
      "activations/layer8_attention_weight_min": -42.00140380859375,
      "activations/layer9_attention_weight_max": 38.207096099853516,
      "activations/layer9_attention_weight_min": -34.544925689697266,
      "epoch": 7.43,
      "learning_rate": 0.00010311136363636362,
      "loss": 2.8275,
      "step": 127900
    },
    {
      "activations/layer0_attention_weight_max": 16.501461029052734,
      "activations/layer0_attention_weight_min": -11.750493049621582,
      "activations/layer10_attention_weight_max": 32.83208084106445,
      "activations/layer10_attention_weight_min": -31.1527099609375,
      "activations/layer11_attention_weight_max": 31.670398712158203,
      "activations/layer11_attention_weight_min": -27.510814666748047,
      "activations/layer12_attention_weight_max": 40.6378173828125,
      "activations/layer12_attention_weight_min": -26.565486907958984,
      "activations/layer13_attention_weight_max": 42.56046676635742,
      "activations/layer13_attention_weight_min": -31.30252456665039,
      "activations/layer14_attention_weight_max": 59.56455612182617,
      "activations/layer14_attention_weight_min": -43.72917938232422,
      "activations/layer15_attention_weight_max": 41.47336196899414,
      "activations/layer15_attention_weight_min": -29.04960823059082,
      "activations/layer16_attention_weight_max": 35.79754638671875,
      "activations/layer16_attention_weight_min": -26.533586502075195,
      "activations/layer17_attention_weight_max": 39.70997619628906,
      "activations/layer17_attention_weight_min": -25.030805587768555,
      "activations/layer18_attention_weight_max": 35.93464279174805,
      "activations/layer18_attention_weight_min": -20.156028747558594,
      "activations/layer19_attention_weight_max": 40.37532424926758,
      "activations/layer19_attention_weight_min": -28.7424373626709,
      "activations/layer1_attention_weight_max": 17.27482795715332,
      "activations/layer1_attention_weight_min": -15.408610343933105,
      "activations/layer20_attention_weight_max": 34.61616516113281,
      "activations/layer20_attention_weight_min": -23.045684814453125,
      "activations/layer21_attention_weight_max": 31.246109008789062,
      "activations/layer21_attention_weight_min": -20.00030517578125,
      "activations/layer22_attention_weight_max": 49.10967254638672,
      "activations/layer22_attention_weight_min": -27.871437072753906,
      "activations/layer23_attention_weight_max": 38.80341339111328,
      "activations/layer23_attention_weight_min": -23.640472412109375,
      "activations/layer2_attention_weight_max": 32.050636291503906,
      "activations/layer2_attention_weight_min": -31.15479278564453,
      "activations/layer3_attention_weight_max": 93.38742065429688,
      "activations/layer3_attention_weight_min": -94.94730377197266,
      "activations/layer4_attention_weight_max": 122.2569580078125,
      "activations/layer4_attention_weight_min": -116.20157623291016,
      "activations/layer5_attention_weight_max": 62.15348434448242,
      "activations/layer5_attention_weight_min": -62.26586151123047,
      "activations/layer6_attention_weight_max": 52.03010559082031,
      "activations/layer6_attention_weight_min": -52.90398025512695,
      "activations/layer7_attention_weight_max": 88.70408630371094,
      "activations/layer7_attention_weight_min": -88.85848999023438,
      "activations/layer8_attention_weight_max": 41.50788879394531,
      "activations/layer8_attention_weight_min": -39.915164947509766,
      "activations/layer9_attention_weight_max": 34.38929748535156,
      "activations/layer9_attention_weight_min": -33.662479400634766,
      "epoch": 7.43,
      "learning_rate": 0.00010309242424242423,
      "loss": 2.8236,
      "step": 127950
    },
    {
      "activations/layer0_attention_weight_max": 15.193984985351562,
      "activations/layer0_attention_weight_min": -12.52139663696289,
      "activations/layer10_attention_weight_max": 36.4114990234375,
      "activations/layer10_attention_weight_min": -34.09955978393555,
      "activations/layer11_attention_weight_max": 38.43418884277344,
      "activations/layer11_attention_weight_min": -31.838590621948242,
      "activations/layer12_attention_weight_max": 33.831703186035156,
      "activations/layer12_attention_weight_min": -25.44005012512207,
      "activations/layer13_attention_weight_max": 44.45673751831055,
      "activations/layer13_attention_weight_min": -32.362449645996094,
      "activations/layer14_attention_weight_max": 67.63731384277344,
      "activations/layer14_attention_weight_min": -46.2910270690918,
      "activations/layer15_attention_weight_max": 41.52613830566406,
      "activations/layer15_attention_weight_min": -29.6912841796875,
      "activations/layer16_attention_weight_max": 39.26481628417969,
      "activations/layer16_attention_weight_min": -27.29458236694336,
      "activations/layer17_attention_weight_max": 34.70833969116211,
      "activations/layer17_attention_weight_min": -23.805837631225586,
      "activations/layer18_attention_weight_max": 29.33544921875,
      "activations/layer18_attention_weight_min": -21.449390411376953,
      "activations/layer19_attention_weight_max": 42.35209655761719,
      "activations/layer19_attention_weight_min": -32.33818054199219,
      "activations/layer1_attention_weight_max": 17.264299392700195,
      "activations/layer1_attention_weight_min": -14.134825706481934,
      "activations/layer20_attention_weight_max": 33.558692932128906,
      "activations/layer20_attention_weight_min": -24.691633224487305,
      "activations/layer21_attention_weight_max": 33.292518615722656,
      "activations/layer21_attention_weight_min": -22.51495361328125,
      "activations/layer22_attention_weight_max": 51.39139175415039,
      "activations/layer22_attention_weight_min": -28.803926467895508,
      "activations/layer23_attention_weight_max": 34.82460403442383,
      "activations/layer23_attention_weight_min": -20.809362411499023,
      "activations/layer2_attention_weight_max": 32.644920349121094,
      "activations/layer2_attention_weight_min": -32.32048797607422,
      "activations/layer3_attention_weight_max": 93.01148223876953,
      "activations/layer3_attention_weight_min": -93.62886047363281,
      "activations/layer4_attention_weight_max": 117.06529998779297,
      "activations/layer4_attention_weight_min": -120.35253143310547,
      "activations/layer5_attention_weight_max": 61.383888244628906,
      "activations/layer5_attention_weight_min": -63.93607711791992,
      "activations/layer6_attention_weight_max": 52.59013748168945,
      "activations/layer6_attention_weight_min": -51.33586120605469,
      "activations/layer7_attention_weight_max": 96.52949523925781,
      "activations/layer7_attention_weight_min": -97.1383056640625,
      "activations/layer8_attention_weight_max": 44.08693313598633,
      "activations/layer8_attention_weight_min": -45.01830291748047,
      "activations/layer9_attention_weight_max": 38.15352249145508,
      "activations/layer9_attention_weight_min": -36.83271408081055,
      "epoch": 7.44,
      "learning_rate": 0.00010307348484848484,
      "loss": 2.7997,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5209,
      "eval_samples_per_second": 503.937,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5209,
      "eval_openwebtext_samples_per_second": 503.937,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_wikitext_loss": 2.9921875,
      "eval_wikitext_ppl": 19.929230034775262,
      "eval_wikitext_runtime": 2.0197,
      "eval_wikitext_samples_per_second": 225.781,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_lambada_loss": 2.685546875,
      "eval_lambada_ppl": 14.666219776472907,
      "eval_lambada_runtime": 9.5973,
      "eval_lambada_samples_per_second": 507.331,
      "step": 128000
    },
    {
      "activations/layer0_attention_weight_max": 15.526142120361328,
      "activations/layer0_attention_weight_min": -12.075102806091309,
      "activations/layer10_attention_weight_max": 34.85358428955078,
      "activations/layer10_attention_weight_min": -31.356822967529297,
      "activations/layer11_attention_weight_max": 31.708890914916992,
      "activations/layer11_attention_weight_min": -29.44023895263672,
      "activations/layer12_attention_weight_max": 23.282997131347656,
      "activations/layer12_attention_weight_min": -26.76896095275879,
      "activations/layer13_attention_weight_max": 43.60654067993164,
      "activations/layer13_attention_weight_min": -32.15482711791992,
      "activations/layer14_attention_weight_max": 54.73664855957031,
      "activations/layer14_attention_weight_min": -37.90708541870117,
      "activations/layer15_attention_weight_max": 37.467529296875,
      "activations/layer15_attention_weight_min": -27.3807430267334,
      "activations/layer16_attention_weight_max": 33.41405487060547,
      "activations/layer16_attention_weight_min": -26.01585578918457,
      "activations/layer17_attention_weight_max": 36.82212448120117,
      "activations/layer17_attention_weight_min": -24.721776962280273,
      "activations/layer18_attention_weight_max": 32.300018310546875,
      "activations/layer18_attention_weight_min": -19.6123104095459,
      "activations/layer19_attention_weight_max": 35.41904067993164,
      "activations/layer19_attention_weight_min": -27.832855224609375,
      "activations/layer1_attention_weight_max": 16.63154411315918,
      "activations/layer1_attention_weight_min": -14.688779830932617,
      "activations/layer20_attention_weight_max": 32.78758239746094,
      "activations/layer20_attention_weight_min": -22.821748733520508,
      "activations/layer21_attention_weight_max": 29.56362533569336,
      "activations/layer21_attention_weight_min": -21.012605667114258,
      "activations/layer22_attention_weight_max": 46.48380661010742,
      "activations/layer22_attention_weight_min": -25.78658103942871,
      "activations/layer23_attention_weight_max": 32.75865936279297,
      "activations/layer23_attention_weight_min": -22.621356964111328,
      "activations/layer2_attention_weight_max": 32.655357360839844,
      "activations/layer2_attention_weight_min": -32.40105438232422,
      "activations/layer3_attention_weight_max": 93.64913177490234,
      "activations/layer3_attention_weight_min": -93.33788299560547,
      "activations/layer4_attention_weight_max": 122.3210678100586,
      "activations/layer4_attention_weight_min": -120.67571258544922,
      "activations/layer5_attention_weight_max": 63.39582061767578,
      "activations/layer5_attention_weight_min": -62.57487106323242,
      "activations/layer6_attention_weight_max": 51.18698501586914,
      "activations/layer6_attention_weight_min": -52.58314895629883,
      "activations/layer7_attention_weight_max": 94.56887817382812,
      "activations/layer7_attention_weight_min": -85.966552734375,
      "activations/layer8_attention_weight_max": 41.30015182495117,
      "activations/layer8_attention_weight_min": -39.47279739379883,
      "activations/layer9_attention_weight_max": 37.85158920288086,
      "activations/layer9_attention_weight_min": -33.268863677978516,
      "epoch": 7.44,
      "learning_rate": 0.00010305454545454544,
      "loss": 2.7996,
      "step": 128050
    },
    {
      "activations/layer0_attention_weight_max": 15.81919002532959,
      "activations/layer0_attention_weight_min": -12.292194366455078,
      "activations/layer10_attention_weight_max": 34.69461441040039,
      "activations/layer10_attention_weight_min": -33.74005889892578,
      "activations/layer11_attention_weight_max": 33.19190979003906,
      "activations/layer11_attention_weight_min": -31.485347747802734,
      "activations/layer12_attention_weight_max": 25.46430206298828,
      "activations/layer12_attention_weight_min": -28.63463592529297,
      "activations/layer13_attention_weight_max": 44.02191925048828,
      "activations/layer13_attention_weight_min": -33.64371871948242,
      "activations/layer14_attention_weight_max": 58.1351432800293,
      "activations/layer14_attention_weight_min": -41.569664001464844,
      "activations/layer15_attention_weight_max": 37.91817092895508,
      "activations/layer15_attention_weight_min": -30.612821578979492,
      "activations/layer16_attention_weight_max": 39.62599563598633,
      "activations/layer16_attention_weight_min": -29.400747299194336,
      "activations/layer17_attention_weight_max": 39.016815185546875,
      "activations/layer17_attention_weight_min": -24.9541015625,
      "activations/layer18_attention_weight_max": 36.70876693725586,
      "activations/layer18_attention_weight_min": -20.19710922241211,
      "activations/layer19_attention_weight_max": 41.065006256103516,
      "activations/layer19_attention_weight_min": -30.371475219726562,
      "activations/layer1_attention_weight_max": 16.455211639404297,
      "activations/layer1_attention_weight_min": -14.608165740966797,
      "activations/layer20_attention_weight_max": 36.16846466064453,
      "activations/layer20_attention_weight_min": -23.74315071105957,
      "activations/layer21_attention_weight_max": 35.69354248046875,
      "activations/layer21_attention_weight_min": -21.564510345458984,
      "activations/layer22_attention_weight_max": 54.76630401611328,
      "activations/layer22_attention_weight_min": -30.02427864074707,
      "activations/layer23_attention_weight_max": 36.35116958618164,
      "activations/layer23_attention_weight_min": -22.250974655151367,
      "activations/layer2_attention_weight_max": 33.82263946533203,
      "activations/layer2_attention_weight_min": -31.930618286132812,
      "activations/layer3_attention_weight_max": 97.60096740722656,
      "activations/layer3_attention_weight_min": -94.07283020019531,
      "activations/layer4_attention_weight_max": 126.5895004272461,
      "activations/layer4_attention_weight_min": -117.92037200927734,
      "activations/layer5_attention_weight_max": 62.18309020996094,
      "activations/layer5_attention_weight_min": -60.16311264038086,
      "activations/layer6_attention_weight_max": 52.5971565246582,
      "activations/layer6_attention_weight_min": -52.70094680786133,
      "activations/layer7_attention_weight_max": 92.19554901123047,
      "activations/layer7_attention_weight_min": -90.6797866821289,
      "activations/layer8_attention_weight_max": 43.32421875,
      "activations/layer8_attention_weight_min": -42.15028762817383,
      "activations/layer9_attention_weight_max": 36.80515670776367,
      "activations/layer9_attention_weight_min": -36.09574508666992,
      "epoch": 7.44,
      "learning_rate": 0.00010303560606060606,
      "loss": 2.8007,
      "step": 128100
    },
    {
      "activations/layer0_attention_weight_max": 15.80277156829834,
      "activations/layer0_attention_weight_min": -11.398003578186035,
      "activations/layer10_attention_weight_max": 33.95093536376953,
      "activations/layer10_attention_weight_min": -33.28936004638672,
      "activations/layer11_attention_weight_max": 30.990585327148438,
      "activations/layer11_attention_weight_min": -30.252864837646484,
      "activations/layer12_attention_weight_max": 54.62964630126953,
      "activations/layer12_attention_weight_min": -24.61224365234375,
      "activations/layer13_attention_weight_max": 55.4897575378418,
      "activations/layer13_attention_weight_min": -33.30529022216797,
      "activations/layer14_attention_weight_max": 50.09303283691406,
      "activations/layer14_attention_weight_min": -41.38665008544922,
      "activations/layer15_attention_weight_max": 41.566261291503906,
      "activations/layer15_attention_weight_min": -30.0302734375,
      "activations/layer16_attention_weight_max": 34.7943229675293,
      "activations/layer16_attention_weight_min": -26.989727020263672,
      "activations/layer17_attention_weight_max": 32.78830337524414,
      "activations/layer17_attention_weight_min": -27.3903865814209,
      "activations/layer18_attention_weight_max": 30.7626953125,
      "activations/layer18_attention_weight_min": -19.928754806518555,
      "activations/layer19_attention_weight_max": 41.72197341918945,
      "activations/layer19_attention_weight_min": -30.54829216003418,
      "activations/layer1_attention_weight_max": 16.951194763183594,
      "activations/layer1_attention_weight_min": -14.156280517578125,
      "activations/layer20_attention_weight_max": 32.812469482421875,
      "activations/layer20_attention_weight_min": -24.28745460510254,
      "activations/layer21_attention_weight_max": 32.01799011230469,
      "activations/layer21_attention_weight_min": -22.292743682861328,
      "activations/layer22_attention_weight_max": 44.314247131347656,
      "activations/layer22_attention_weight_min": -26.36941909790039,
      "activations/layer23_attention_weight_max": 35.63506317138672,
      "activations/layer23_attention_weight_min": -21.27381134033203,
      "activations/layer2_attention_weight_max": 32.928306579589844,
      "activations/layer2_attention_weight_min": -32.17109680175781,
      "activations/layer3_attention_weight_max": 95.44356536865234,
      "activations/layer3_attention_weight_min": -99.44973754882812,
      "activations/layer4_attention_weight_max": 118.62896728515625,
      "activations/layer4_attention_weight_min": -123.80908203125,
      "activations/layer5_attention_weight_max": 60.77839660644531,
      "activations/layer5_attention_weight_min": -63.99181365966797,
      "activations/layer6_attention_weight_max": 49.06602096557617,
      "activations/layer6_attention_weight_min": -52.6342658996582,
      "activations/layer7_attention_weight_max": 86.37960815429688,
      "activations/layer7_attention_weight_min": -88.53392791748047,
      "activations/layer8_attention_weight_max": 39.431480407714844,
      "activations/layer8_attention_weight_min": -40.26783752441406,
      "activations/layer9_attention_weight_max": 34.148441314697266,
      "activations/layer9_attention_weight_min": -35.576045989990234,
      "epoch": 7.45,
      "learning_rate": 0.00010301704545454545,
      "loss": 2.8079,
      "step": 128150
    },
    {
      "activations/layer0_attention_weight_max": 15.968637466430664,
      "activations/layer0_attention_weight_min": -11.038080215454102,
      "activations/layer10_attention_weight_max": 38.447853088378906,
      "activations/layer10_attention_weight_min": -32.61030960083008,
      "activations/layer11_attention_weight_max": 40.38315963745117,
      "activations/layer11_attention_weight_min": -32.61919021606445,
      "activations/layer12_attention_weight_max": 32.17775344848633,
      "activations/layer12_attention_weight_min": -26.4180965423584,
      "activations/layer13_attention_weight_max": 54.275394439697266,
      "activations/layer13_attention_weight_min": -32.9589729309082,
      "activations/layer14_attention_weight_max": 63.110862731933594,
      "activations/layer14_attention_weight_min": -42.1556510925293,
      "activations/layer15_attention_weight_max": 54.92597961425781,
      "activations/layer15_attention_weight_min": -29.285327911376953,
      "activations/layer16_attention_weight_max": 37.86051559448242,
      "activations/layer16_attention_weight_min": -27.00682830810547,
      "activations/layer17_attention_weight_max": 39.99774932861328,
      "activations/layer17_attention_weight_min": -25.897851943969727,
      "activations/layer18_attention_weight_max": 37.48405456542969,
      "activations/layer18_attention_weight_min": -22.97323226928711,
      "activations/layer19_attention_weight_max": 44.322208404541016,
      "activations/layer19_attention_weight_min": -31.690399169921875,
      "activations/layer1_attention_weight_max": 17.044218063354492,
      "activations/layer1_attention_weight_min": -18.16208267211914,
      "activations/layer20_attention_weight_max": 40.8531608581543,
      "activations/layer20_attention_weight_min": -23.4201717376709,
      "activations/layer21_attention_weight_max": 40.133663177490234,
      "activations/layer21_attention_weight_min": -20.91019058227539,
      "activations/layer22_attention_weight_max": 56.41635513305664,
      "activations/layer22_attention_weight_min": -27.802453994750977,
      "activations/layer23_attention_weight_max": 40.33140563964844,
      "activations/layer23_attention_weight_min": -22.756698608398438,
      "activations/layer2_attention_weight_max": 33.975425720214844,
      "activations/layer2_attention_weight_min": -32.752254486083984,
      "activations/layer3_attention_weight_max": 95.32929229736328,
      "activations/layer3_attention_weight_min": -97.0423583984375,
      "activations/layer4_attention_weight_max": 125.67919158935547,
      "activations/layer4_attention_weight_min": -122.0412368774414,
      "activations/layer5_attention_weight_max": 61.04148864746094,
      "activations/layer5_attention_weight_min": -62.200225830078125,
      "activations/layer6_attention_weight_max": 50.55710983276367,
      "activations/layer6_attention_weight_min": -54.59842300415039,
      "activations/layer7_attention_weight_max": 90.61300659179688,
      "activations/layer7_attention_weight_min": -87.22074127197266,
      "activations/layer8_attention_weight_max": 40.40339279174805,
      "activations/layer8_attention_weight_min": -42.21748733520508,
      "activations/layer9_attention_weight_max": 36.66908264160156,
      "activations/layer9_attention_weight_min": -32.84117889404297,
      "epoch": 7.45,
      "learning_rate": 0.00010299810606060605,
      "loss": 2.8102,
      "step": 128200
    },
    {
      "activations/layer0_attention_weight_max": 16.21211814880371,
      "activations/layer0_attention_weight_min": -11.10355281829834,
      "activations/layer10_attention_weight_max": 33.506988525390625,
      "activations/layer10_attention_weight_min": -31.309444427490234,
      "activations/layer11_attention_weight_max": 33.152000427246094,
      "activations/layer11_attention_weight_min": -30.365886688232422,
      "activations/layer12_attention_weight_max": 27.5793514251709,
      "activations/layer12_attention_weight_min": -26.603412628173828,
      "activations/layer13_attention_weight_max": 39.02788162231445,
      "activations/layer13_attention_weight_min": -32.04791259765625,
      "activations/layer14_attention_weight_max": 45.78596878051758,
      "activations/layer14_attention_weight_min": -37.17634201049805,
      "activations/layer15_attention_weight_max": 34.73869705200195,
      "activations/layer15_attention_weight_min": -29.822277069091797,
      "activations/layer16_attention_weight_max": 33.072322845458984,
      "activations/layer16_attention_weight_min": -29.966760635375977,
      "activations/layer17_attention_weight_max": 37.31012725830078,
      "activations/layer17_attention_weight_min": -27.02823257446289,
      "activations/layer18_attention_weight_max": 31.868825912475586,
      "activations/layer18_attention_weight_min": -22.109792709350586,
      "activations/layer19_attention_weight_max": 38.15095138549805,
      "activations/layer19_attention_weight_min": -30.670812606811523,
      "activations/layer1_attention_weight_max": 16.666719436645508,
      "activations/layer1_attention_weight_min": -15.13366413116455,
      "activations/layer20_attention_weight_max": 29.09916877746582,
      "activations/layer20_attention_weight_min": -23.883729934692383,
      "activations/layer21_attention_weight_max": 27.019577026367188,
      "activations/layer21_attention_weight_min": -21.955183029174805,
      "activations/layer22_attention_weight_max": 48.24729537963867,
      "activations/layer22_attention_weight_min": -27.500791549682617,
      "activations/layer23_attention_weight_max": 36.26622009277344,
      "activations/layer23_attention_weight_min": -22.101194381713867,
      "activations/layer2_attention_weight_max": 33.6157341003418,
      "activations/layer2_attention_weight_min": -33.37156677246094,
      "activations/layer3_attention_weight_max": 96.78270721435547,
      "activations/layer3_attention_weight_min": -95.74028015136719,
      "activations/layer4_attention_weight_max": 132.6004638671875,
      "activations/layer4_attention_weight_min": -123.5875015258789,
      "activations/layer5_attention_weight_max": 66.4713134765625,
      "activations/layer5_attention_weight_min": -65.03425598144531,
      "activations/layer6_attention_weight_max": 53.28001022338867,
      "activations/layer6_attention_weight_min": -55.37841033935547,
      "activations/layer7_attention_weight_max": 92.62078857421875,
      "activations/layer7_attention_weight_min": -90.15835571289062,
      "activations/layer8_attention_weight_max": 43.94026565551758,
      "activations/layer8_attention_weight_min": -43.40449905395508,
      "activations/layer9_attention_weight_max": 37.29782485961914,
      "activations/layer9_attention_weight_min": -35.81719970703125,
      "epoch": 7.45,
      "learning_rate": 0.00010297916666666665,
      "loss": 2.8059,
      "step": 128250
    },
    {
      "activations/layer0_attention_weight_max": 15.774856567382812,
      "activations/layer0_attention_weight_min": -11.211310386657715,
      "activations/layer10_attention_weight_max": 32.572914123535156,
      "activations/layer10_attention_weight_min": -33.568702697753906,
      "activations/layer11_attention_weight_max": 31.613697052001953,
      "activations/layer11_attention_weight_min": -30.532814025878906,
      "activations/layer12_attention_weight_max": 21.048206329345703,
      "activations/layer12_attention_weight_min": -27.173324584960938,
      "activations/layer13_attention_weight_max": 37.59600830078125,
      "activations/layer13_attention_weight_min": -35.91608810424805,
      "activations/layer14_attention_weight_max": 48.7044792175293,
      "activations/layer14_attention_weight_min": -40.552398681640625,
      "activations/layer15_attention_weight_max": 36.193702697753906,
      "activations/layer15_attention_weight_min": -30.9820613861084,
      "activations/layer16_attention_weight_max": 35.28650665283203,
      "activations/layer16_attention_weight_min": -27.96346664428711,
      "activations/layer17_attention_weight_max": 31.96628761291504,
      "activations/layer17_attention_weight_min": -25.4883975982666,
      "activations/layer18_attention_weight_max": 31.262187957763672,
      "activations/layer18_attention_weight_min": -21.95224952697754,
      "activations/layer19_attention_weight_max": 33.82945251464844,
      "activations/layer19_attention_weight_min": -31.544034957885742,
      "activations/layer1_attention_weight_max": 18.357406616210938,
      "activations/layer1_attention_weight_min": -14.344042778015137,
      "activations/layer20_attention_weight_max": 28.85498046875,
      "activations/layer20_attention_weight_min": -24.723400115966797,
      "activations/layer21_attention_weight_max": 27.310216903686523,
      "activations/layer21_attention_weight_min": -22.768056869506836,
      "activations/layer22_attention_weight_max": 42.22007369995117,
      "activations/layer22_attention_weight_min": -30.286848068237305,
      "activations/layer23_attention_weight_max": 32.2635383605957,
      "activations/layer23_attention_weight_min": -21.53753662109375,
      "activations/layer2_attention_weight_max": 32.565277099609375,
      "activations/layer2_attention_weight_min": -31.57329559326172,
      "activations/layer3_attention_weight_max": 90.56230163574219,
      "activations/layer3_attention_weight_min": -90.60060119628906,
      "activations/layer4_attention_weight_max": 126.87454986572266,
      "activations/layer4_attention_weight_min": -118.49315643310547,
      "activations/layer5_attention_weight_max": 63.28033447265625,
      "activations/layer5_attention_weight_min": -63.7309684753418,
      "activations/layer6_attention_weight_max": 53.35411071777344,
      "activations/layer6_attention_weight_min": -53.1721076965332,
      "activations/layer7_attention_weight_max": 93.7158203125,
      "activations/layer7_attention_weight_min": -91.46430969238281,
      "activations/layer8_attention_weight_max": 39.76287078857422,
      "activations/layer8_attention_weight_min": -43.7225227355957,
      "activations/layer9_attention_weight_max": 34.90983200073242,
      "activations/layer9_attention_weight_min": -34.131744384765625,
      "epoch": 7.45,
      "learning_rate": 0.00010296022727272727,
      "loss": 2.8205,
      "step": 128300
    },
    {
      "activations/layer0_attention_weight_max": 16.404460906982422,
      "activations/layer0_attention_weight_min": -10.979682922363281,
      "activations/layer10_attention_weight_max": 35.1561393737793,
      "activations/layer10_attention_weight_min": -32.63570022583008,
      "activations/layer11_attention_weight_max": 32.911720275878906,
      "activations/layer11_attention_weight_min": -30.81731414794922,
      "activations/layer12_attention_weight_max": 23.909692764282227,
      "activations/layer12_attention_weight_min": -25.225662231445312,
      "activations/layer13_attention_weight_max": 42.53010940551758,
      "activations/layer13_attention_weight_min": -31.844697952270508,
      "activations/layer14_attention_weight_max": 53.136470794677734,
      "activations/layer14_attention_weight_min": -39.817440032958984,
      "activations/layer15_attention_weight_max": 38.8853645324707,
      "activations/layer15_attention_weight_min": -29.27317237854004,
      "activations/layer16_attention_weight_max": 35.44499588012695,
      "activations/layer16_attention_weight_min": -28.552366256713867,
      "activations/layer17_attention_weight_max": 35.602909088134766,
      "activations/layer17_attention_weight_min": -25.79104995727539,
      "activations/layer18_attention_weight_max": 31.333961486816406,
      "activations/layer18_attention_weight_min": -21.1884822845459,
      "activations/layer19_attention_weight_max": 37.9116325378418,
      "activations/layer19_attention_weight_min": -29.805021286010742,
      "activations/layer1_attention_weight_max": 17.368669509887695,
      "activations/layer1_attention_weight_min": -14.334288597106934,
      "activations/layer20_attention_weight_max": 34.07686233520508,
      "activations/layer20_attention_weight_min": -24.5126895904541,
      "activations/layer21_attention_weight_max": 35.69960403442383,
      "activations/layer21_attention_weight_min": -22.2475643157959,
      "activations/layer22_attention_weight_max": 51.31226348876953,
      "activations/layer22_attention_weight_min": -28.69989013671875,
      "activations/layer23_attention_weight_max": 36.17148971557617,
      "activations/layer23_attention_weight_min": -20.92416763305664,
      "activations/layer2_attention_weight_max": 32.462806701660156,
      "activations/layer2_attention_weight_min": -31.25110626220703,
      "activations/layer3_attention_weight_max": 97.60171508789062,
      "activations/layer3_attention_weight_min": -96.65283966064453,
      "activations/layer4_attention_weight_max": 124.80742645263672,
      "activations/layer4_attention_weight_min": -125.01415252685547,
      "activations/layer5_attention_weight_max": 64.24111938476562,
      "activations/layer5_attention_weight_min": -62.566673278808594,
      "activations/layer6_attention_weight_max": 52.29530715942383,
      "activations/layer6_attention_weight_min": -54.44118881225586,
      "activations/layer7_attention_weight_max": 87.02925109863281,
      "activations/layer7_attention_weight_min": -89.66648864746094,
      "activations/layer8_attention_weight_max": 42.63023376464844,
      "activations/layer8_attention_weight_min": -43.03541946411133,
      "activations/layer9_attention_weight_max": 36.96708297729492,
      "activations/layer9_attention_weight_min": -35.97940444946289,
      "epoch": 7.46,
      "learning_rate": 0.00010294128787878788,
      "loss": 2.8139,
      "step": 128350
    },
    {
      "activations/layer0_attention_weight_max": 16.02669334411621,
      "activations/layer0_attention_weight_min": -11.411919593811035,
      "activations/layer10_attention_weight_max": 31.204875946044922,
      "activations/layer10_attention_weight_min": -32.35914611816406,
      "activations/layer11_attention_weight_max": 31.83302116394043,
      "activations/layer11_attention_weight_min": -28.83069610595703,
      "activations/layer12_attention_weight_max": 24.933563232421875,
      "activations/layer12_attention_weight_min": -25.558101654052734,
      "activations/layer13_attention_weight_max": 39.4462776184082,
      "activations/layer13_attention_weight_min": -31.6076717376709,
      "activations/layer14_attention_weight_max": 52.690189361572266,
      "activations/layer14_attention_weight_min": -46.06917190551758,
      "activations/layer15_attention_weight_max": 36.95309829711914,
      "activations/layer15_attention_weight_min": -28.620786666870117,
      "activations/layer16_attention_weight_max": 35.132667541503906,
      "activations/layer16_attention_weight_min": -28.226669311523438,
      "activations/layer17_attention_weight_max": 33.05085372924805,
      "activations/layer17_attention_weight_min": -23.819393157958984,
      "activations/layer18_attention_weight_max": 31.5069522857666,
      "activations/layer18_attention_weight_min": -19.87796401977539,
      "activations/layer19_attention_weight_max": 40.24415588378906,
      "activations/layer19_attention_weight_min": -30.92608642578125,
      "activations/layer1_attention_weight_max": 16.277257919311523,
      "activations/layer1_attention_weight_min": -14.947759628295898,
      "activations/layer20_attention_weight_max": 32.812355041503906,
      "activations/layer20_attention_weight_min": -23.98686981201172,
      "activations/layer21_attention_weight_max": 34.582637786865234,
      "activations/layer21_attention_weight_min": -22.541980743408203,
      "activations/layer22_attention_weight_max": 48.020450592041016,
      "activations/layer22_attention_weight_min": -28.11321449279785,
      "activations/layer23_attention_weight_max": 36.45396423339844,
      "activations/layer23_attention_weight_min": -23.527435302734375,
      "activations/layer2_attention_weight_max": 32.02790832519531,
      "activations/layer2_attention_weight_min": -31.343250274658203,
      "activations/layer3_attention_weight_max": 89.89154815673828,
      "activations/layer3_attention_weight_min": -92.05999755859375,
      "activations/layer4_attention_weight_max": 119.72431945800781,
      "activations/layer4_attention_weight_min": -117.3423080444336,
      "activations/layer5_attention_weight_max": 60.495967864990234,
      "activations/layer5_attention_weight_min": -62.130062103271484,
      "activations/layer6_attention_weight_max": 48.97355270385742,
      "activations/layer6_attention_weight_min": -52.334938049316406,
      "activations/layer7_attention_weight_max": 87.41754913330078,
      "activations/layer7_attention_weight_min": -90.03520202636719,
      "activations/layer8_attention_weight_max": 38.44151306152344,
      "activations/layer8_attention_weight_min": -39.8631591796875,
      "activations/layer9_attention_weight_max": 34.9806022644043,
      "activations/layer9_attention_weight_min": -34.25516891479492,
      "epoch": 7.46,
      "learning_rate": 0.00010292234848484847,
      "loss": 2.8123,
      "step": 128400
    },
    {
      "activations/layer0_attention_weight_max": 15.628227233886719,
      "activations/layer0_attention_weight_min": -11.527402877807617,
      "activations/layer10_attention_weight_max": 34.7656364440918,
      "activations/layer10_attention_weight_min": -32.74754333496094,
      "activations/layer11_attention_weight_max": 32.979698181152344,
      "activations/layer11_attention_weight_min": -30.279937744140625,
      "activations/layer12_attention_weight_max": 22.86760902404785,
      "activations/layer12_attention_weight_min": -30.775487899780273,
      "activations/layer13_attention_weight_max": 40.069419860839844,
      "activations/layer13_attention_weight_min": -31.90074348449707,
      "activations/layer14_attention_weight_max": 48.4566650390625,
      "activations/layer14_attention_weight_min": -38.10871505737305,
      "activations/layer15_attention_weight_max": 38.520503997802734,
      "activations/layer15_attention_weight_min": -28.435590744018555,
      "activations/layer16_attention_weight_max": 31.794523239135742,
      "activations/layer16_attention_weight_min": -27.979808807373047,
      "activations/layer17_attention_weight_max": 34.95064163208008,
      "activations/layer17_attention_weight_min": -25.534109115600586,
      "activations/layer18_attention_weight_max": 30.801305770874023,
      "activations/layer18_attention_weight_min": -23.506277084350586,
      "activations/layer19_attention_weight_max": 35.02399444580078,
      "activations/layer19_attention_weight_min": -29.467382431030273,
      "activations/layer1_attention_weight_max": 17.12429428100586,
      "activations/layer1_attention_weight_min": -12.867836952209473,
      "activations/layer20_attention_weight_max": 29.556982040405273,
      "activations/layer20_attention_weight_min": -23.795269012451172,
      "activations/layer21_attention_weight_max": 27.683349609375,
      "activations/layer21_attention_weight_min": -21.57257843017578,
      "activations/layer22_attention_weight_max": 46.6995964050293,
      "activations/layer22_attention_weight_min": -29.03288459777832,
      "activations/layer23_attention_weight_max": 37.01892852783203,
      "activations/layer23_attention_weight_min": -25.346296310424805,
      "activations/layer2_attention_weight_max": 31.92420196533203,
      "activations/layer2_attention_weight_min": -32.0304069519043,
      "activations/layer3_attention_weight_max": 91.09544372558594,
      "activations/layer3_attention_weight_min": -94.37088775634766,
      "activations/layer4_attention_weight_max": 123.79095458984375,
      "activations/layer4_attention_weight_min": -121.16548156738281,
      "activations/layer5_attention_weight_max": 59.554893493652344,
      "activations/layer5_attention_weight_min": -63.1905632019043,
      "activations/layer6_attention_weight_max": 50.71908187866211,
      "activations/layer6_attention_weight_min": -51.98189163208008,
      "activations/layer7_attention_weight_max": 93.50770568847656,
      "activations/layer7_attention_weight_min": -86.39982604980469,
      "activations/layer8_attention_weight_max": 39.856990814208984,
      "activations/layer8_attention_weight_min": -38.421417236328125,
      "activations/layer9_attention_weight_max": 37.239967346191406,
      "activations/layer9_attention_weight_min": -36.98674774169922,
      "epoch": 7.46,
      "learning_rate": 0.00010290340909090908,
      "loss": 2.8169,
      "step": 128450
    },
    {
      "activations/layer0_attention_weight_max": 15.866137504577637,
      "activations/layer0_attention_weight_min": -12.425980567932129,
      "activations/layer10_attention_weight_max": 32.11507797241211,
      "activations/layer10_attention_weight_min": -30.973262786865234,
      "activations/layer11_attention_weight_max": 31.500835418701172,
      "activations/layer11_attention_weight_min": -27.843839645385742,
      "activations/layer12_attention_weight_max": 22.205429077148438,
      "activations/layer12_attention_weight_min": -28.022443771362305,
      "activations/layer13_attention_weight_max": 38.55559539794922,
      "activations/layer13_attention_weight_min": -32.42189025878906,
      "activations/layer14_attention_weight_max": 52.942630767822266,
      "activations/layer14_attention_weight_min": -38.58308792114258,
      "activations/layer15_attention_weight_max": 34.45457458496094,
      "activations/layer15_attention_weight_min": -30.089494705200195,
      "activations/layer16_attention_weight_max": 34.99884796142578,
      "activations/layer16_attention_weight_min": -28.473243713378906,
      "activations/layer17_attention_weight_max": 32.58492660522461,
      "activations/layer17_attention_weight_min": -25.737695693969727,
      "activations/layer18_attention_weight_max": 33.13174057006836,
      "activations/layer18_attention_weight_min": -21.529674530029297,
      "activations/layer19_attention_weight_max": 34.57267379760742,
      "activations/layer19_attention_weight_min": -30.1961669921875,
      "activations/layer1_attention_weight_max": 16.180471420288086,
      "activations/layer1_attention_weight_min": -13.644171714782715,
      "activations/layer20_attention_weight_max": 32.28561019897461,
      "activations/layer20_attention_weight_min": -25.832273483276367,
      "activations/layer21_attention_weight_max": 30.356292724609375,
      "activations/layer21_attention_weight_min": -22.794565200805664,
      "activations/layer22_attention_weight_max": 48.11284637451172,
      "activations/layer22_attention_weight_min": -29.862634658813477,
      "activations/layer23_attention_weight_max": 32.61408233642578,
      "activations/layer23_attention_weight_min": -25.232158660888672,
      "activations/layer2_attention_weight_max": 31.90709686279297,
      "activations/layer2_attention_weight_min": -33.010440826416016,
      "activations/layer3_attention_weight_max": 91.3877182006836,
      "activations/layer3_attention_weight_min": -96.2343521118164,
      "activations/layer4_attention_weight_max": 120.35916900634766,
      "activations/layer4_attention_weight_min": -123.44488525390625,
      "activations/layer5_attention_weight_max": 59.79508590698242,
      "activations/layer5_attention_weight_min": -62.00910568237305,
      "activations/layer6_attention_weight_max": 50.40048599243164,
      "activations/layer6_attention_weight_min": -53.8712272644043,
      "activations/layer7_attention_weight_max": 85.76950073242188,
      "activations/layer7_attention_weight_min": -93.50993347167969,
      "activations/layer8_attention_weight_max": 38.20705795288086,
      "activations/layer8_attention_weight_min": -39.56145095825195,
      "activations/layer9_attention_weight_max": 32.980899810791016,
      "activations/layer9_attention_weight_min": -34.54656982421875,
      "epoch": 7.47,
      "learning_rate": 0.0001028844696969697,
      "loss": 2.794,
      "step": 128500
    },
    {
      "activations/layer0_attention_weight_max": 15.162957191467285,
      "activations/layer0_attention_weight_min": -12.000677108764648,
      "activations/layer10_attention_weight_max": 33.37947463989258,
      "activations/layer10_attention_weight_min": -30.672983169555664,
      "activations/layer11_attention_weight_max": 32.64434051513672,
      "activations/layer11_attention_weight_min": -30.991270065307617,
      "activations/layer12_attention_weight_max": 24.436370849609375,
      "activations/layer12_attention_weight_min": -26.05008316040039,
      "activations/layer13_attention_weight_max": 42.98881149291992,
      "activations/layer13_attention_weight_min": -33.309844970703125,
      "activations/layer14_attention_weight_max": 49.69078063964844,
      "activations/layer14_attention_weight_min": -38.26168441772461,
      "activations/layer15_attention_weight_max": 36.615787506103516,
      "activations/layer15_attention_weight_min": -29.10009765625,
      "activations/layer16_attention_weight_max": 35.77627944946289,
      "activations/layer16_attention_weight_min": -28.53245735168457,
      "activations/layer17_attention_weight_max": 36.40853500366211,
      "activations/layer17_attention_weight_min": -24.992542266845703,
      "activations/layer18_attention_weight_max": 41.290565490722656,
      "activations/layer18_attention_weight_min": -21.076656341552734,
      "activations/layer19_attention_weight_max": 38.51174545288086,
      "activations/layer19_attention_weight_min": -28.875669479370117,
      "activations/layer1_attention_weight_max": 16.292509078979492,
      "activations/layer1_attention_weight_min": -14.642488479614258,
      "activations/layer20_attention_weight_max": 33.61931228637695,
      "activations/layer20_attention_weight_min": -23.744718551635742,
      "activations/layer21_attention_weight_max": 36.77745819091797,
      "activations/layer21_attention_weight_min": -21.965505599975586,
      "activations/layer22_attention_weight_max": 52.074405670166016,
      "activations/layer22_attention_weight_min": -29.657062530517578,
      "activations/layer23_attention_weight_max": 39.40697479248047,
      "activations/layer23_attention_weight_min": -22.300945281982422,
      "activations/layer2_attention_weight_max": 33.36782455444336,
      "activations/layer2_attention_weight_min": -33.22669982910156,
      "activations/layer3_attention_weight_max": 93.16279602050781,
      "activations/layer3_attention_weight_min": -96.60611724853516,
      "activations/layer4_attention_weight_max": 120.0457763671875,
      "activations/layer4_attention_weight_min": -119.49525451660156,
      "activations/layer5_attention_weight_max": 62.82263946533203,
      "activations/layer5_attention_weight_min": -63.00687789916992,
      "activations/layer6_attention_weight_max": 50.057594299316406,
      "activations/layer6_attention_weight_min": -52.584083557128906,
      "activations/layer7_attention_weight_max": 89.84281921386719,
      "activations/layer7_attention_weight_min": -89.9356689453125,
      "activations/layer8_attention_weight_max": 40.292449951171875,
      "activations/layer8_attention_weight_min": -40.62382888793945,
      "activations/layer9_attention_weight_max": 35.23862838745117,
      "activations/layer9_attention_weight_min": -34.31478500366211,
      "epoch": 7.47,
      "learning_rate": 0.00010286553030303029,
      "loss": 2.8088,
      "step": 128550
    },
    {
      "activations/layer0_attention_weight_max": 16.292512893676758,
      "activations/layer0_attention_weight_min": -11.850753784179688,
      "activations/layer10_attention_weight_max": 31.82400131225586,
      "activations/layer10_attention_weight_min": -30.007726669311523,
      "activations/layer11_attention_weight_max": 30.311397552490234,
      "activations/layer11_attention_weight_min": -27.57501983642578,
      "activations/layer12_attention_weight_max": 21.624692916870117,
      "activations/layer12_attention_weight_min": -26.096704483032227,
      "activations/layer13_attention_weight_max": 38.19343948364258,
      "activations/layer13_attention_weight_min": -32.426788330078125,
      "activations/layer14_attention_weight_max": 46.544334411621094,
      "activations/layer14_attention_weight_min": -40.93716812133789,
      "activations/layer15_attention_weight_max": 37.068275451660156,
      "activations/layer15_attention_weight_min": -30.599655151367188,
      "activations/layer16_attention_weight_max": 33.10075378417969,
      "activations/layer16_attention_weight_min": -26.922203063964844,
      "activations/layer17_attention_weight_max": 30.92271614074707,
      "activations/layer17_attention_weight_min": -25.87708282470703,
      "activations/layer18_attention_weight_max": 33.8112678527832,
      "activations/layer18_attention_weight_min": -21.07184600830078,
      "activations/layer19_attention_weight_max": 39.52021789550781,
      "activations/layer19_attention_weight_min": -31.29081153869629,
      "activations/layer1_attention_weight_max": 16.813373565673828,
      "activations/layer1_attention_weight_min": -14.963475227355957,
      "activations/layer20_attention_weight_max": 30.622079849243164,
      "activations/layer20_attention_weight_min": -22.766210556030273,
      "activations/layer21_attention_weight_max": 33.31697463989258,
      "activations/layer21_attention_weight_min": -24.60088539123535,
      "activations/layer22_attention_weight_max": 47.95440673828125,
      "activations/layer22_attention_weight_min": -29.3934326171875,
      "activations/layer23_attention_weight_max": 39.0699462890625,
      "activations/layer23_attention_weight_min": -22.899349212646484,
      "activations/layer2_attention_weight_max": 33.53544998168945,
      "activations/layer2_attention_weight_min": -33.21997833251953,
      "activations/layer3_attention_weight_max": 94.15272521972656,
      "activations/layer3_attention_weight_min": -95.00923919677734,
      "activations/layer4_attention_weight_max": 124.5460205078125,
      "activations/layer4_attention_weight_min": -122.90560150146484,
      "activations/layer5_attention_weight_max": 62.7365837097168,
      "activations/layer5_attention_weight_min": -61.96720886230469,
      "activations/layer6_attention_weight_max": 50.924217224121094,
      "activations/layer6_attention_weight_min": -51.806556701660156,
      "activations/layer7_attention_weight_max": 92.17321014404297,
      "activations/layer7_attention_weight_min": -87.10323333740234,
      "activations/layer8_attention_weight_max": 41.16081619262695,
      "activations/layer8_attention_weight_min": -40.15105056762695,
      "activations/layer9_attention_weight_max": 35.625244140625,
      "activations/layer9_attention_weight_min": -32.97903060913086,
      "epoch": 7.47,
      "learning_rate": 0.0001028465909090909,
      "loss": 2.8164,
      "step": 128600
    },
    {
      "activations/layer0_attention_weight_max": 15.799447059631348,
      "activations/layer0_attention_weight_min": -12.06455135345459,
      "activations/layer10_attention_weight_max": 34.839813232421875,
      "activations/layer10_attention_weight_min": -33.42181396484375,
      "activations/layer11_attention_weight_max": 33.35198211669922,
      "activations/layer11_attention_weight_min": -31.788921356201172,
      "activations/layer12_attention_weight_max": 27.872312545776367,
      "activations/layer12_attention_weight_min": -28.68073272705078,
      "activations/layer13_attention_weight_max": 41.40932846069336,
      "activations/layer13_attention_weight_min": -32.53190231323242,
      "activations/layer14_attention_weight_max": 52.401859283447266,
      "activations/layer14_attention_weight_min": -40.007781982421875,
      "activations/layer15_attention_weight_max": 37.99938201904297,
      "activations/layer15_attention_weight_min": -30.812828063964844,
      "activations/layer16_attention_weight_max": 34.699501037597656,
      "activations/layer16_attention_weight_min": -27.252046585083008,
      "activations/layer17_attention_weight_max": 38.626853942871094,
      "activations/layer17_attention_weight_min": -26.192625045776367,
      "activations/layer18_attention_weight_max": 38.586273193359375,
      "activations/layer18_attention_weight_min": -22.238544464111328,
      "activations/layer19_attention_weight_max": 43.90554428100586,
      "activations/layer19_attention_weight_min": -30.72995948791504,
      "activations/layer1_attention_weight_max": 17.131275177001953,
      "activations/layer1_attention_weight_min": -16.156248092651367,
      "activations/layer20_attention_weight_max": 36.81398010253906,
      "activations/layer20_attention_weight_min": -25.07768440246582,
      "activations/layer21_attention_weight_max": 41.10658645629883,
      "activations/layer21_attention_weight_min": -24.42089080810547,
      "activations/layer22_attention_weight_max": 50.141136169433594,
      "activations/layer22_attention_weight_min": -30.583126068115234,
      "activations/layer23_attention_weight_max": 39.59120178222656,
      "activations/layer23_attention_weight_min": -23.544591903686523,
      "activations/layer2_attention_weight_max": 34.24415588378906,
      "activations/layer2_attention_weight_min": -33.22528076171875,
      "activations/layer3_attention_weight_max": 98.76852416992188,
      "activations/layer3_attention_weight_min": -99.18087768554688,
      "activations/layer4_attention_weight_max": 128.28135681152344,
      "activations/layer4_attention_weight_min": -126.67166900634766,
      "activations/layer5_attention_weight_max": 64.30291748046875,
      "activations/layer5_attention_weight_min": -64.5445556640625,
      "activations/layer6_attention_weight_max": 54.62360382080078,
      "activations/layer6_attention_weight_min": -54.37920379638672,
      "activations/layer7_attention_weight_max": 92.97027587890625,
      "activations/layer7_attention_weight_min": -91.63592529296875,
      "activations/layer8_attention_weight_max": 41.77562713623047,
      "activations/layer8_attention_weight_min": -41.446563720703125,
      "activations/layer9_attention_weight_max": 35.7085075378418,
      "activations/layer9_attention_weight_min": -36.924659729003906,
      "epoch": 7.48,
      "learning_rate": 0.0001028276515151515,
      "loss": 2.8014,
      "step": 128650
    },
    {
      "activations/layer0_attention_weight_max": 16.44483757019043,
      "activations/layer0_attention_weight_min": -11.922283172607422,
      "activations/layer10_attention_weight_max": 34.17848205566406,
      "activations/layer10_attention_weight_min": -31.670207977294922,
      "activations/layer11_attention_weight_max": 33.05236053466797,
      "activations/layer11_attention_weight_min": -29.10589599609375,
      "activations/layer12_attention_weight_max": 24.154903411865234,
      "activations/layer12_attention_weight_min": -25.607038497924805,
      "activations/layer13_attention_weight_max": 40.030845642089844,
      "activations/layer13_attention_weight_min": -30.535940170288086,
      "activations/layer14_attention_weight_max": 52.14220428466797,
      "activations/layer14_attention_weight_min": -39.75791549682617,
      "activations/layer15_attention_weight_max": 37.22500228881836,
      "activations/layer15_attention_weight_min": -29.99171257019043,
      "activations/layer16_attention_weight_max": 34.662689208984375,
      "activations/layer16_attention_weight_min": -29.237384796142578,
      "activations/layer17_attention_weight_max": 35.91781997680664,
      "activations/layer17_attention_weight_min": -25.03667449951172,
      "activations/layer18_attention_weight_max": 32.000877380371094,
      "activations/layer18_attention_weight_min": -21.960092544555664,
      "activations/layer19_attention_weight_max": 38.26667785644531,
      "activations/layer19_attention_weight_min": -29.655441284179688,
      "activations/layer1_attention_weight_max": 16.89983367919922,
      "activations/layer1_attention_weight_min": -14.859198570251465,
      "activations/layer20_attention_weight_max": 30.556718826293945,
      "activations/layer20_attention_weight_min": -24.376981735229492,
      "activations/layer21_attention_weight_max": 28.0797119140625,
      "activations/layer21_attention_weight_min": -22.947031021118164,
      "activations/layer22_attention_weight_max": 45.62759780883789,
      "activations/layer22_attention_weight_min": -30.73015594482422,
      "activations/layer23_attention_weight_max": 37.061241149902344,
      "activations/layer23_attention_weight_min": -22.52920913696289,
      "activations/layer2_attention_weight_max": 32.053924560546875,
      "activations/layer2_attention_weight_min": -30.27695083618164,
      "activations/layer3_attention_weight_max": 91.08141326904297,
      "activations/layer3_attention_weight_min": -89.5813980102539,
      "activations/layer4_attention_weight_max": 118.6670913696289,
      "activations/layer4_attention_weight_min": -121.07083892822266,
      "activations/layer5_attention_weight_max": 60.13890838623047,
      "activations/layer5_attention_weight_min": -60.38773727416992,
      "activations/layer6_attention_weight_max": 49.423744201660156,
      "activations/layer6_attention_weight_min": -50.85368728637695,
      "activations/layer7_attention_weight_max": 87.48212432861328,
      "activations/layer7_attention_weight_min": -82.7647933959961,
      "activations/layer8_attention_weight_max": 38.69446563720703,
      "activations/layer8_attention_weight_min": -38.19081497192383,
      "activations/layer9_attention_weight_max": 34.993038177490234,
      "activations/layer9_attention_weight_min": -31.98651123046875,
      "epoch": 7.48,
      "learning_rate": 0.0001028087121212121,
      "loss": 2.8177,
      "step": 128700
    },
    {
      "activations/layer0_attention_weight_max": 16.08950424194336,
      "activations/layer0_attention_weight_min": -11.788116455078125,
      "activations/layer10_attention_weight_max": 40.35295104980469,
      "activations/layer10_attention_weight_min": -35.881813049316406,
      "activations/layer11_attention_weight_max": 40.2381477355957,
      "activations/layer11_attention_weight_min": -34.86526870727539,
      "activations/layer12_attention_weight_max": 27.16697120666504,
      "activations/layer12_attention_weight_min": -25.92448616027832,
      "activations/layer13_attention_weight_max": 42.816837310791016,
      "activations/layer13_attention_weight_min": -34.7383918762207,
      "activations/layer14_attention_weight_max": 53.15755844116211,
      "activations/layer14_attention_weight_min": -41.30189514160156,
      "activations/layer15_attention_weight_max": 36.3565673828125,
      "activations/layer15_attention_weight_min": -30.866350173950195,
      "activations/layer16_attention_weight_max": 32.58334732055664,
      "activations/layer16_attention_weight_min": -29.024343490600586,
      "activations/layer17_attention_weight_max": 34.7611198425293,
      "activations/layer17_attention_weight_min": -25.583839416503906,
      "activations/layer18_attention_weight_max": 31.659709930419922,
      "activations/layer18_attention_weight_min": -20.727989196777344,
      "activations/layer19_attention_weight_max": 34.49363327026367,
      "activations/layer19_attention_weight_min": -31.798221588134766,
      "activations/layer1_attention_weight_max": 17.508834838867188,
      "activations/layer1_attention_weight_min": -14.010662078857422,
      "activations/layer20_attention_weight_max": 31.828258514404297,
      "activations/layer20_attention_weight_min": -24.254539489746094,
      "activations/layer21_attention_weight_max": 34.36791229248047,
      "activations/layer21_attention_weight_min": -23.232561111450195,
      "activations/layer22_attention_weight_max": 45.45650100708008,
      "activations/layer22_attention_weight_min": -29.57117462158203,
      "activations/layer23_attention_weight_max": 37.18763732910156,
      "activations/layer23_attention_weight_min": -24.566604614257812,
      "activations/layer2_attention_weight_max": 35.84417724609375,
      "activations/layer2_attention_weight_min": -34.3721809387207,
      "activations/layer3_attention_weight_max": 100.91248321533203,
      "activations/layer3_attention_weight_min": -102.39134979248047,
      "activations/layer4_attention_weight_max": 129.8607177734375,
      "activations/layer4_attention_weight_min": -131.12266540527344,
      "activations/layer5_attention_weight_max": 64.19000244140625,
      "activations/layer5_attention_weight_min": -62.73481369018555,
      "activations/layer6_attention_weight_max": 53.881587982177734,
      "activations/layer6_attention_weight_min": -54.683937072753906,
      "activations/layer7_attention_weight_max": 100.58338928222656,
      "activations/layer7_attention_weight_min": -91.02799224853516,
      "activations/layer8_attention_weight_max": 45.82533264160156,
      "activations/layer8_attention_weight_min": -41.97865676879883,
      "activations/layer9_attention_weight_max": 40.67060852050781,
      "activations/layer9_attention_weight_min": -36.320125579833984,
      "epoch": 7.48,
      "learning_rate": 0.00010278977272727272,
      "loss": 2.8268,
      "step": 128750
    },
    {
      "activations/layer0_attention_weight_max": 16.517446517944336,
      "activations/layer0_attention_weight_min": -11.441965103149414,
      "activations/layer10_attention_weight_max": 40.15724563598633,
      "activations/layer10_attention_weight_min": -36.49348449707031,
      "activations/layer11_attention_weight_max": 37.28358459472656,
      "activations/layer11_attention_weight_min": -32.66291046142578,
      "activations/layer12_attention_weight_max": 25.025352478027344,
      "activations/layer12_attention_weight_min": -26.070289611816406,
      "activations/layer13_attention_weight_max": 47.53834533691406,
      "activations/layer13_attention_weight_min": -33.36906051635742,
      "activations/layer14_attention_weight_max": 60.632755279541016,
      "activations/layer14_attention_weight_min": -43.831459045410156,
      "activations/layer15_attention_weight_max": 38.71600341796875,
      "activations/layer15_attention_weight_min": -29.94387435913086,
      "activations/layer16_attention_weight_max": 37.34458541870117,
      "activations/layer16_attention_weight_min": -28.0035457611084,
      "activations/layer17_attention_weight_max": 37.61293029785156,
      "activations/layer17_attention_weight_min": -25.002185821533203,
      "activations/layer18_attention_weight_max": 31.146272659301758,
      "activations/layer18_attention_weight_min": -21.475170135498047,
      "activations/layer19_attention_weight_max": 42.40463638305664,
      "activations/layer19_attention_weight_min": -28.48089599609375,
      "activations/layer1_attention_weight_max": 17.42011260986328,
      "activations/layer1_attention_weight_min": -14.406230926513672,
      "activations/layer20_attention_weight_max": 34.710723876953125,
      "activations/layer20_attention_weight_min": -22.6280460357666,
      "activations/layer21_attention_weight_max": 32.330989837646484,
      "activations/layer21_attention_weight_min": -20.848756790161133,
      "activations/layer22_attention_weight_max": 53.695674896240234,
      "activations/layer22_attention_weight_min": -27.7557315826416,
      "activations/layer23_attention_weight_max": 38.70597457885742,
      "activations/layer23_attention_weight_min": -23.170143127441406,
      "activations/layer2_attention_weight_max": 33.290523529052734,
      "activations/layer2_attention_weight_min": -32.672943115234375,
      "activations/layer3_attention_weight_max": 94.5274429321289,
      "activations/layer3_attention_weight_min": -93.39244842529297,
      "activations/layer4_attention_weight_max": 127.17439270019531,
      "activations/layer4_attention_weight_min": -125.77992248535156,
      "activations/layer5_attention_weight_max": 66.86186981201172,
      "activations/layer5_attention_weight_min": -65.36480712890625,
      "activations/layer6_attention_weight_max": 55.7230339050293,
      "activations/layer6_attention_weight_min": -58.37425231933594,
      "activations/layer7_attention_weight_max": 101.47531127929688,
      "activations/layer7_attention_weight_min": -94.33930206298828,
      "activations/layer8_attention_weight_max": 47.47116470336914,
      "activations/layer8_attention_weight_min": -44.746986389160156,
      "activations/layer9_attention_weight_max": 42.06715393066406,
      "activations/layer9_attention_weight_min": -37.92831039428711,
      "epoch": 7.48,
      "learning_rate": 0.00010277083333333332,
      "loss": 2.8191,
      "step": 128800
    },
    {
      "activations/layer0_attention_weight_max": 16.09062385559082,
      "activations/layer0_attention_weight_min": -11.417177200317383,
      "activations/layer10_attention_weight_max": 37.01934814453125,
      "activations/layer10_attention_weight_min": -35.3718147277832,
      "activations/layer11_attention_weight_max": 41.6370849609375,
      "activations/layer11_attention_weight_min": -33.872230529785156,
      "activations/layer12_attention_weight_max": 43.631954193115234,
      "activations/layer12_attention_weight_min": -26.19365692138672,
      "activations/layer13_attention_weight_max": 64.75164031982422,
      "activations/layer13_attention_weight_min": -34.780677795410156,
      "activations/layer14_attention_weight_max": 68.08624267578125,
      "activations/layer14_attention_weight_min": -43.928619384765625,
      "activations/layer15_attention_weight_max": 50.21147918701172,
      "activations/layer15_attention_weight_min": -30.619897842407227,
      "activations/layer16_attention_weight_max": 37.346893310546875,
      "activations/layer16_attention_weight_min": -28.45720100402832,
      "activations/layer17_attention_weight_max": 37.32713317871094,
      "activations/layer17_attention_weight_min": -25.692331314086914,
      "activations/layer18_attention_weight_max": 34.32300567626953,
      "activations/layer18_attention_weight_min": -22.437578201293945,
      "activations/layer19_attention_weight_max": 42.833091735839844,
      "activations/layer19_attention_weight_min": -29.227787017822266,
      "activations/layer1_attention_weight_max": 18.234067916870117,
      "activations/layer1_attention_weight_min": -14.573614120483398,
      "activations/layer20_attention_weight_max": 35.562583923339844,
      "activations/layer20_attention_weight_min": -24.252979278564453,
      "activations/layer21_attention_weight_max": 33.868534088134766,
      "activations/layer21_attention_weight_min": -22.2899227142334,
      "activations/layer22_attention_weight_max": 65.8946304321289,
      "activations/layer22_attention_weight_min": -29.586444854736328,
      "activations/layer23_attention_weight_max": 42.87340545654297,
      "activations/layer23_attention_weight_min": -25.6854305267334,
      "activations/layer2_attention_weight_max": 31.65579605102539,
      "activations/layer2_attention_weight_min": -31.842893600463867,
      "activations/layer3_attention_weight_max": 91.77670288085938,
      "activations/layer3_attention_weight_min": -94.70757293701172,
      "activations/layer4_attention_weight_max": 119.95899963378906,
      "activations/layer4_attention_weight_min": -124.80156707763672,
      "activations/layer5_attention_weight_max": 63.35285568237305,
      "activations/layer5_attention_weight_min": -62.61170959472656,
      "activations/layer6_attention_weight_max": 51.62214660644531,
      "activations/layer6_attention_weight_min": -52.839839935302734,
      "activations/layer7_attention_weight_max": 93.59365844726562,
      "activations/layer7_attention_weight_min": -91.1459732055664,
      "activations/layer8_attention_weight_max": 46.761199951171875,
      "activations/layer8_attention_weight_min": -40.163970947265625,
      "activations/layer9_attention_weight_max": 40.003597259521484,
      "activations/layer9_attention_weight_min": -36.10862731933594,
      "epoch": 7.49,
      "learning_rate": 0.00010275189393939394,
      "loss": 2.8114,
      "step": 128850
    },
    {
      "activations/layer0_attention_weight_max": 16.38762092590332,
      "activations/layer0_attention_weight_min": -11.804532051086426,
      "activations/layer10_attention_weight_max": 33.30615234375,
      "activations/layer10_attention_weight_min": -30.858642578125,
      "activations/layer11_attention_weight_max": 31.455968856811523,
      "activations/layer11_attention_weight_min": -28.771154403686523,
      "activations/layer12_attention_weight_max": 22.831928253173828,
      "activations/layer12_attention_weight_min": -23.373443603515625,
      "activations/layer13_attention_weight_max": 40.12491989135742,
      "activations/layer13_attention_weight_min": -32.90309143066406,
      "activations/layer14_attention_weight_max": 54.11038589477539,
      "activations/layer14_attention_weight_min": -45.28627395629883,
      "activations/layer15_attention_weight_max": 37.695255279541016,
      "activations/layer15_attention_weight_min": -29.61900520324707,
      "activations/layer16_attention_weight_max": 35.04161834716797,
      "activations/layer16_attention_weight_min": -29.33359718322754,
      "activations/layer17_attention_weight_max": 40.20808792114258,
      "activations/layer17_attention_weight_min": -26.14969825744629,
      "activations/layer18_attention_weight_max": 35.14057540893555,
      "activations/layer18_attention_weight_min": -22.264951705932617,
      "activations/layer19_attention_weight_max": 35.94814682006836,
      "activations/layer19_attention_weight_min": -28.222930908203125,
      "activations/layer1_attention_weight_max": 16.052080154418945,
      "activations/layer1_attention_weight_min": -14.35622787475586,
      "activations/layer20_attention_weight_max": 32.51176452636719,
      "activations/layer20_attention_weight_min": -24.094358444213867,
      "activations/layer21_attention_weight_max": 30.48209571838379,
      "activations/layer21_attention_weight_min": -22.88266372680664,
      "activations/layer22_attention_weight_max": 45.256980895996094,
      "activations/layer22_attention_weight_min": -30.00977897644043,
      "activations/layer23_attention_weight_max": 38.801029205322266,
      "activations/layer23_attention_weight_min": -24.319799423217773,
      "activations/layer2_attention_weight_max": 32.00936508178711,
      "activations/layer2_attention_weight_min": -31.760976791381836,
      "activations/layer3_attention_weight_max": 91.81483459472656,
      "activations/layer3_attention_weight_min": -95.32951354980469,
      "activations/layer4_attention_weight_max": 125.2205581665039,
      "activations/layer4_attention_weight_min": -118.06201171875,
      "activations/layer5_attention_weight_max": 60.27095031738281,
      "activations/layer5_attention_weight_min": -63.06871032714844,
      "activations/layer6_attention_weight_max": 50.75896453857422,
      "activations/layer6_attention_weight_min": -54.28144073486328,
      "activations/layer7_attention_weight_max": 84.8141860961914,
      "activations/layer7_attention_weight_min": -86.30094909667969,
      "activations/layer8_attention_weight_max": 41.32815170288086,
      "activations/layer8_attention_weight_min": -44.00877380371094,
      "activations/layer9_attention_weight_max": 36.8036003112793,
      "activations/layer9_attention_weight_min": -32.77702713012695,
      "epoch": 7.49,
      "learning_rate": 0.00010273295454545454,
      "loss": 2.8031,
      "step": 128900
    },
    {
      "activations/layer0_attention_weight_max": 15.308124542236328,
      "activations/layer0_attention_weight_min": -12.176664352416992,
      "activations/layer10_attention_weight_max": 30.313720703125,
      "activations/layer10_attention_weight_min": -29.974266052246094,
      "activations/layer11_attention_weight_max": 29.004684448242188,
      "activations/layer11_attention_weight_min": -28.767074584960938,
      "activations/layer12_attention_weight_max": 21.417789459228516,
      "activations/layer12_attention_weight_min": -24.51726722717285,
      "activations/layer13_attention_weight_max": 37.71361541748047,
      "activations/layer13_attention_weight_min": -31.978540420532227,
      "activations/layer14_attention_weight_max": 47.63111877441406,
      "activations/layer14_attention_weight_min": -40.49871063232422,
      "activations/layer15_attention_weight_max": 35.886051177978516,
      "activations/layer15_attention_weight_min": -27.813486099243164,
      "activations/layer16_attention_weight_max": 31.102176666259766,
      "activations/layer16_attention_weight_min": -26.569416046142578,
      "activations/layer17_attention_weight_max": 35.92435836791992,
      "activations/layer17_attention_weight_min": -25.762914657592773,
      "activations/layer18_attention_weight_max": 33.27776336669922,
      "activations/layer18_attention_weight_min": -22.260379791259766,
      "activations/layer19_attention_weight_max": 36.74995040893555,
      "activations/layer19_attention_weight_min": -30.092084884643555,
      "activations/layer1_attention_weight_max": 17.490022659301758,
      "activations/layer1_attention_weight_min": -13.840094566345215,
      "activations/layer20_attention_weight_max": 29.41399383544922,
      "activations/layer20_attention_weight_min": -24.31365966796875,
      "activations/layer21_attention_weight_max": 31.786741256713867,
      "activations/layer21_attention_weight_min": -23.279977798461914,
      "activations/layer22_attention_weight_max": 48.3975715637207,
      "activations/layer22_attention_weight_min": -29.493772506713867,
      "activations/layer23_attention_weight_max": 38.87982177734375,
      "activations/layer23_attention_weight_min": -26.097652435302734,
      "activations/layer2_attention_weight_max": 32.31377410888672,
      "activations/layer2_attention_weight_min": -33.07194137573242,
      "activations/layer3_attention_weight_max": 89.50469207763672,
      "activations/layer3_attention_weight_min": -91.45443725585938,
      "activations/layer4_attention_weight_max": 119.9927978515625,
      "activations/layer4_attention_weight_min": -114.60347747802734,
      "activations/layer5_attention_weight_max": 58.782325744628906,
      "activations/layer5_attention_weight_min": -60.93055725097656,
      "activations/layer6_attention_weight_max": 47.1497688293457,
      "activations/layer6_attention_weight_min": -50.24392318725586,
      "activations/layer7_attention_weight_max": 81.61414337158203,
      "activations/layer7_attention_weight_min": -81.74209594726562,
      "activations/layer8_attention_weight_max": 38.50010299682617,
      "activations/layer8_attention_weight_min": -38.395050048828125,
      "activations/layer9_attention_weight_max": 33.07194137573242,
      "activations/layer9_attention_weight_min": -33.98401641845703,
      "epoch": 7.49,
      "learning_rate": 0.00010271401515151514,
      "loss": 2.8091,
      "step": 128950
    },
    {
      "activations/layer0_attention_weight_max": 15.452112197875977,
      "activations/layer0_attention_weight_min": -12.06190299987793,
      "activations/layer10_attention_weight_max": 34.147247314453125,
      "activations/layer10_attention_weight_min": -32.4797248840332,
      "activations/layer11_attention_weight_max": 31.99080467224121,
      "activations/layer11_attention_weight_min": -30.983678817749023,
      "activations/layer12_attention_weight_max": 24.718393325805664,
      "activations/layer12_attention_weight_min": -24.685482025146484,
      "activations/layer13_attention_weight_max": 39.75700759887695,
      "activations/layer13_attention_weight_min": -31.630847930908203,
      "activations/layer14_attention_weight_max": 51.98305892944336,
      "activations/layer14_attention_weight_min": -40.13544845581055,
      "activations/layer15_attention_weight_max": 38.73744583129883,
      "activations/layer15_attention_weight_min": -29.265533447265625,
      "activations/layer16_attention_weight_max": 38.090694427490234,
      "activations/layer16_attention_weight_min": -28.6026554107666,
      "activations/layer17_attention_weight_max": 34.888668060302734,
      "activations/layer17_attention_weight_min": -24.8560791015625,
      "activations/layer18_attention_weight_max": 30.769084930419922,
      "activations/layer18_attention_weight_min": -20.820974349975586,
      "activations/layer19_attention_weight_max": 38.69677734375,
      "activations/layer19_attention_weight_min": -29.843027114868164,
      "activations/layer1_attention_weight_max": 17.799556732177734,
      "activations/layer1_attention_weight_min": -16.33527374267578,
      "activations/layer20_attention_weight_max": 36.30043029785156,
      "activations/layer20_attention_weight_min": -24.116195678710938,
      "activations/layer21_attention_weight_max": 33.067447662353516,
      "activations/layer21_attention_weight_min": -23.82920265197754,
      "activations/layer22_attention_weight_max": 47.77207946777344,
      "activations/layer22_attention_weight_min": -28.51097297668457,
      "activations/layer23_attention_weight_max": 36.21360778808594,
      "activations/layer23_attention_weight_min": -23.357730865478516,
      "activations/layer2_attention_weight_max": 31.851774215698242,
      "activations/layer2_attention_weight_min": -32.56303405761719,
      "activations/layer3_attention_weight_max": 90.27669525146484,
      "activations/layer3_attention_weight_min": -91.03314971923828,
      "activations/layer4_attention_weight_max": 119.38301086425781,
      "activations/layer4_attention_weight_min": -115.68310546875,
      "activations/layer5_attention_weight_max": 59.221473693847656,
      "activations/layer5_attention_weight_min": -61.81766891479492,
      "activations/layer6_attention_weight_max": 48.77915954589844,
      "activations/layer6_attention_weight_min": -51.062747955322266,
      "activations/layer7_attention_weight_max": 89.77942657470703,
      "activations/layer7_attention_weight_min": -84.63899230957031,
      "activations/layer8_attention_weight_max": 40.90028762817383,
      "activations/layer8_attention_weight_min": -39.86664962768555,
      "activations/layer9_attention_weight_max": 36.62416076660156,
      "activations/layer9_attention_weight_min": -34.00674057006836,
      "epoch": 7.5,
      "learning_rate": 0.00010269507575757575,
      "loss": 2.829,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5067,
      "eval_samples_per_second": 504.78,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5067,
      "eval_openwebtext_samples_per_second": 504.78,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 2.01,
      "eval_wikitext_samples_per_second": 226.866,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.6444,
      "eval_lambada_samples_per_second": 504.852,
      "step": 129000
    },
    {
      "activations/layer0_attention_weight_max": 16.532777786254883,
      "activations/layer0_attention_weight_min": -12.698481559753418,
      "activations/layer10_attention_weight_max": 39.744606018066406,
      "activations/layer10_attention_weight_min": -36.2000846862793,
      "activations/layer11_attention_weight_max": 38.908180236816406,
      "activations/layer11_attention_weight_min": -34.731040954589844,
      "activations/layer12_attention_weight_max": 26.99006462097168,
      "activations/layer12_attention_weight_min": -25.212772369384766,
      "activations/layer13_attention_weight_max": 48.7819938659668,
      "activations/layer13_attention_weight_min": -33.66351318359375,
      "activations/layer14_attention_weight_max": 58.003273010253906,
      "activations/layer14_attention_weight_min": -41.917972564697266,
      "activations/layer15_attention_weight_max": 47.388065338134766,
      "activations/layer15_attention_weight_min": -31.649024963378906,
      "activations/layer16_attention_weight_max": 37.23727798461914,
      "activations/layer16_attention_weight_min": -27.350786209106445,
      "activations/layer17_attention_weight_max": 39.23247146606445,
      "activations/layer17_attention_weight_min": -24.415164947509766,
      "activations/layer18_attention_weight_max": 33.9141731262207,
      "activations/layer18_attention_weight_min": -20.63202476501465,
      "activations/layer19_attention_weight_max": 38.3864860534668,
      "activations/layer19_attention_weight_min": -30.203216552734375,
      "activations/layer1_attention_weight_max": 17.684858322143555,
      "activations/layer1_attention_weight_min": -15.378402709960938,
      "activations/layer20_attention_weight_max": 31.548370361328125,
      "activations/layer20_attention_weight_min": -23.35268211364746,
      "activations/layer21_attention_weight_max": 30.75804328918457,
      "activations/layer21_attention_weight_min": -23.10309410095215,
      "activations/layer22_attention_weight_max": 52.380409240722656,
      "activations/layer22_attention_weight_min": -28.131563186645508,
      "activations/layer23_attention_weight_max": 39.49769592285156,
      "activations/layer23_attention_weight_min": -22.233388900756836,
      "activations/layer2_attention_weight_max": 31.723661422729492,
      "activations/layer2_attention_weight_min": -33.2963752746582,
      "activations/layer3_attention_weight_max": 97.37274932861328,
      "activations/layer3_attention_weight_min": -94.55077362060547,
      "activations/layer4_attention_weight_max": 128.0067596435547,
      "activations/layer4_attention_weight_min": -119.1619644165039,
      "activations/layer5_attention_weight_max": 64.45681762695312,
      "activations/layer5_attention_weight_min": -60.98405456542969,
      "activations/layer6_attention_weight_max": 52.377559661865234,
      "activations/layer6_attention_weight_min": -52.83481979370117,
      "activations/layer7_attention_weight_max": 95.10215759277344,
      "activations/layer7_attention_weight_min": -94.18994903564453,
      "activations/layer8_attention_weight_max": 44.75474548339844,
      "activations/layer8_attention_weight_min": -44.06891632080078,
      "activations/layer9_attention_weight_max": 43.818607330322266,
      "activations/layer9_attention_weight_min": -38.17500305175781,
      "epoch": 7.5,
      "learning_rate": 0.00010267651515151515,
      "loss": 2.8258,
      "step": 129050
    },
    {
      "activations/layer0_attention_weight_max": 15.56321907043457,
      "activations/layer0_attention_weight_min": -10.376657485961914,
      "activations/layer10_attention_weight_max": 31.858854293823242,
      "activations/layer10_attention_weight_min": -29.379653930664062,
      "activations/layer11_attention_weight_max": 33.70988464355469,
      "activations/layer11_attention_weight_min": -29.67847442626953,
      "activations/layer12_attention_weight_max": 22.921274185180664,
      "activations/layer12_attention_weight_min": -24.84489631652832,
      "activations/layer13_attention_weight_max": 40.04021453857422,
      "activations/layer13_attention_weight_min": -33.43003845214844,
      "activations/layer14_attention_weight_max": 56.10577392578125,
      "activations/layer14_attention_weight_min": -43.15938949584961,
      "activations/layer15_attention_weight_max": 35.427696228027344,
      "activations/layer15_attention_weight_min": -30.840959548950195,
      "activations/layer16_attention_weight_max": 31.307035446166992,
      "activations/layer16_attention_weight_min": -27.619171142578125,
      "activations/layer17_attention_weight_max": 32.13594055175781,
      "activations/layer17_attention_weight_min": -24.378206253051758,
      "activations/layer18_attention_weight_max": 29.12433433532715,
      "activations/layer18_attention_weight_min": -20.3618221282959,
      "activations/layer19_attention_weight_max": 35.5988655090332,
      "activations/layer19_attention_weight_min": -29.280155181884766,
      "activations/layer1_attention_weight_max": 17.052265167236328,
      "activations/layer1_attention_weight_min": -16.20112419128418,
      "activations/layer20_attention_weight_max": 29.063343048095703,
      "activations/layer20_attention_weight_min": -24.444284439086914,
      "activations/layer21_attention_weight_max": 28.765396118164062,
      "activations/layer21_attention_weight_min": -23.581680297851562,
      "activations/layer22_attention_weight_max": 41.276119232177734,
      "activations/layer22_attention_weight_min": -27.167980194091797,
      "activations/layer23_attention_weight_max": 31.699607849121094,
      "activations/layer23_attention_weight_min": -22.0450496673584,
      "activations/layer2_attention_weight_max": 32.89230728149414,
      "activations/layer2_attention_weight_min": -32.64968490600586,
      "activations/layer3_attention_weight_max": 89.72999572753906,
      "activations/layer3_attention_weight_min": -92.48153686523438,
      "activations/layer4_attention_weight_max": 119.3863525390625,
      "activations/layer4_attention_weight_min": -113.79754638671875,
      "activations/layer5_attention_weight_max": 61.118831634521484,
      "activations/layer5_attention_weight_min": -61.31034851074219,
      "activations/layer6_attention_weight_max": 49.954402923583984,
      "activations/layer6_attention_weight_min": -49.00841522216797,
      "activations/layer7_attention_weight_max": 86.27123260498047,
      "activations/layer7_attention_weight_min": -83.41669464111328,
      "activations/layer8_attention_weight_max": 40.00644302368164,
      "activations/layer8_attention_weight_min": -39.44951248168945,
      "activations/layer9_attention_weight_max": 36.88351821899414,
      "activations/layer9_attention_weight_min": -33.99483108520508,
      "epoch": 7.5,
      "learning_rate": 0.00010265757575757576,
      "loss": 2.8111,
      "step": 129100
    },
    {
      "activations/layer0_attention_weight_max": 15.55335521697998,
      "activations/layer0_attention_weight_min": -12.29719352722168,
      "activations/layer10_attention_weight_max": 31.062124252319336,
      "activations/layer10_attention_weight_min": -30.275833129882812,
      "activations/layer11_attention_weight_max": 30.783782958984375,
      "activations/layer11_attention_weight_min": -29.256202697753906,
      "activations/layer12_attention_weight_max": 22.82012939453125,
      "activations/layer12_attention_weight_min": -29.287691116333008,
      "activations/layer13_attention_weight_max": 36.26359176635742,
      "activations/layer13_attention_weight_min": -33.71156311035156,
      "activations/layer14_attention_weight_max": 43.63715744018555,
      "activations/layer14_attention_weight_min": -37.5203742980957,
      "activations/layer15_attention_weight_max": 39.13789749145508,
      "activations/layer15_attention_weight_min": -28.982635498046875,
      "activations/layer16_attention_weight_max": 30.197839736938477,
      "activations/layer16_attention_weight_min": -26.62250328063965,
      "activations/layer17_attention_weight_max": 32.82412338256836,
      "activations/layer17_attention_weight_min": -25.12201690673828,
      "activations/layer18_attention_weight_max": 29.092872619628906,
      "activations/layer18_attention_weight_min": -20.58220863342285,
      "activations/layer19_attention_weight_max": 32.38865661621094,
      "activations/layer19_attention_weight_min": -27.937219619750977,
      "activations/layer1_attention_weight_max": 17.291234970092773,
      "activations/layer1_attention_weight_min": -14.78239631652832,
      "activations/layer20_attention_weight_max": 27.962034225463867,
      "activations/layer20_attention_weight_min": -23.625703811645508,
      "activations/layer21_attention_weight_max": 30.755474090576172,
      "activations/layer21_attention_weight_min": -21.641918182373047,
      "activations/layer22_attention_weight_max": 42.39407730102539,
      "activations/layer22_attention_weight_min": -29.00753402709961,
      "activations/layer23_attention_weight_max": 30.74628448486328,
      "activations/layer23_attention_weight_min": -23.411954879760742,
      "activations/layer2_attention_weight_max": 33.2857666015625,
      "activations/layer2_attention_weight_min": -32.939239501953125,
      "activations/layer3_attention_weight_max": 94.94111633300781,
      "activations/layer3_attention_weight_min": -97.28419494628906,
      "activations/layer4_attention_weight_max": 125.76966857910156,
      "activations/layer4_attention_weight_min": -122.72258758544922,
      "activations/layer5_attention_weight_max": 59.45610046386719,
      "activations/layer5_attention_weight_min": -62.47595977783203,
      "activations/layer6_attention_weight_max": 50.03109359741211,
      "activations/layer6_attention_weight_min": -51.55620193481445,
      "activations/layer7_attention_weight_max": 91.74909973144531,
      "activations/layer7_attention_weight_min": -92.91001892089844,
      "activations/layer8_attention_weight_max": 37.954830169677734,
      "activations/layer8_attention_weight_min": -39.06221389770508,
      "activations/layer9_attention_weight_max": 33.88313674926758,
      "activations/layer9_attention_weight_min": -32.7286376953125,
      "epoch": 7.5,
      "learning_rate": 0.00010263863636363635,
      "loss": 2.8116,
      "step": 129150
    },
    {
      "activations/layer0_attention_weight_max": 15.630352020263672,
      "activations/layer0_attention_weight_min": -11.955049514770508,
      "activations/layer10_attention_weight_max": 33.00919723510742,
      "activations/layer10_attention_weight_min": -31.355863571166992,
      "activations/layer11_attention_weight_max": 33.478416442871094,
      "activations/layer11_attention_weight_min": -29.39131736755371,
      "activations/layer12_attention_weight_max": 27.026586532592773,
      "activations/layer12_attention_weight_min": -25.661762237548828,
      "activations/layer13_attention_weight_max": 40.616363525390625,
      "activations/layer13_attention_weight_min": -32.65693283081055,
      "activations/layer14_attention_weight_max": 51.008914947509766,
      "activations/layer14_attention_weight_min": -43.48409652709961,
      "activations/layer15_attention_weight_max": 37.20655822753906,
      "activations/layer15_attention_weight_min": -30.011863708496094,
      "activations/layer16_attention_weight_max": 35.29026412963867,
      "activations/layer16_attention_weight_min": -26.890172958374023,
      "activations/layer17_attention_weight_max": 34.56795883178711,
      "activations/layer17_attention_weight_min": -24.1307373046875,
      "activations/layer18_attention_weight_max": 29.67839241027832,
      "activations/layer18_attention_weight_min": -20.061996459960938,
      "activations/layer19_attention_weight_max": 37.650630950927734,
      "activations/layer19_attention_weight_min": -31.10296630859375,
      "activations/layer1_attention_weight_max": 17.32842254638672,
      "activations/layer1_attention_weight_min": -13.207937240600586,
      "activations/layer20_attention_weight_max": 29.475872039794922,
      "activations/layer20_attention_weight_min": -23.06399917602539,
      "activations/layer21_attention_weight_max": 29.133867263793945,
      "activations/layer21_attention_weight_min": -21.224145889282227,
      "activations/layer22_attention_weight_max": 44.57486343383789,
      "activations/layer22_attention_weight_min": -28.87560272216797,
      "activations/layer23_attention_weight_max": 33.76127624511719,
      "activations/layer23_attention_weight_min": -23.040386199951172,
      "activations/layer2_attention_weight_max": 30.725330352783203,
      "activations/layer2_attention_weight_min": -29.75390625,
      "activations/layer3_attention_weight_max": 85.98186492919922,
      "activations/layer3_attention_weight_min": -89.29832458496094,
      "activations/layer4_attention_weight_max": 113.89324188232422,
      "activations/layer4_attention_weight_min": -113.03112030029297,
      "activations/layer5_attention_weight_max": 60.23196029663086,
      "activations/layer5_attention_weight_min": -59.103981018066406,
      "activations/layer6_attention_weight_max": 49.262874603271484,
      "activations/layer6_attention_weight_min": -56.79093933105469,
      "activations/layer7_attention_weight_max": 84.99717712402344,
      "activations/layer7_attention_weight_min": -87.08915710449219,
      "activations/layer8_attention_weight_max": 38.14335632324219,
      "activations/layer8_attention_weight_min": -39.64186477661133,
      "activations/layer9_attention_weight_max": 33.936092376708984,
      "activations/layer9_attention_weight_min": -33.84484100341797,
      "epoch": 7.51,
      "learning_rate": 0.00010261969696969696,
      "loss": 2.8045,
      "step": 129200
    },
    {
      "activations/layer0_attention_weight_max": 15.783653259277344,
      "activations/layer0_attention_weight_min": -10.85749340057373,
      "activations/layer10_attention_weight_max": 32.593387603759766,
      "activations/layer10_attention_weight_min": -33.526161193847656,
      "activations/layer11_attention_weight_max": 36.73023223876953,
      "activations/layer11_attention_weight_min": -30.59455108642578,
      "activations/layer12_attention_weight_max": 26.833341598510742,
      "activations/layer12_attention_weight_min": -27.57842254638672,
      "activations/layer13_attention_weight_max": 39.140113830566406,
      "activations/layer13_attention_weight_min": -32.17729949951172,
      "activations/layer14_attention_weight_max": 54.97060012817383,
      "activations/layer14_attention_weight_min": -44.76591110229492,
      "activations/layer15_attention_weight_max": 35.781375885009766,
      "activations/layer15_attention_weight_min": -29.750160217285156,
      "activations/layer16_attention_weight_max": 32.058876037597656,
      "activations/layer16_attention_weight_min": -28.474002838134766,
      "activations/layer17_attention_weight_max": 35.36219024658203,
      "activations/layer17_attention_weight_min": -25.78691864013672,
      "activations/layer18_attention_weight_max": 30.791725158691406,
      "activations/layer18_attention_weight_min": -20.347469329833984,
      "activations/layer19_attention_weight_max": 35.2658805847168,
      "activations/layer19_attention_weight_min": -30.682226181030273,
      "activations/layer1_attention_weight_max": 17.312776565551758,
      "activations/layer1_attention_weight_min": -13.760025978088379,
      "activations/layer20_attention_weight_max": 29.26150131225586,
      "activations/layer20_attention_weight_min": -26.024158477783203,
      "activations/layer21_attention_weight_max": 32.11847686767578,
      "activations/layer21_attention_weight_min": -23.84535789489746,
      "activations/layer22_attention_weight_max": 45.99958038330078,
      "activations/layer22_attention_weight_min": -27.98600959777832,
      "activations/layer23_attention_weight_max": 34.10081100463867,
      "activations/layer23_attention_weight_min": -22.2463436126709,
      "activations/layer2_attention_weight_max": 32.04364776611328,
      "activations/layer2_attention_weight_min": -32.527137756347656,
      "activations/layer3_attention_weight_max": 86.15206909179688,
      "activations/layer3_attention_weight_min": -90.03353118896484,
      "activations/layer4_attention_weight_max": 114.81017303466797,
      "activations/layer4_attention_weight_min": -119.02977752685547,
      "activations/layer5_attention_weight_max": 57.94628143310547,
      "activations/layer5_attention_weight_min": -59.9749755859375,
      "activations/layer6_attention_weight_max": 47.86220932006836,
      "activations/layer6_attention_weight_min": -52.03017807006836,
      "activations/layer7_attention_weight_max": 88.96687316894531,
      "activations/layer7_attention_weight_min": -93.61939239501953,
      "activations/layer8_attention_weight_max": 38.202327728271484,
      "activations/layer8_attention_weight_min": -40.6446418762207,
      "activations/layer9_attention_weight_max": 33.011478424072266,
      "activations/layer9_attention_weight_min": -33.870018005371094,
      "epoch": 7.51,
      "learning_rate": 0.00010260075757575758,
      "loss": 2.8271,
      "step": 129250
    },
    {
      "activations/layer0_attention_weight_max": 15.807765007019043,
      "activations/layer0_attention_weight_min": -11.917667388916016,
      "activations/layer10_attention_weight_max": 31.263639450073242,
      "activations/layer10_attention_weight_min": -29.820425033569336,
      "activations/layer11_attention_weight_max": 28.526790618896484,
      "activations/layer11_attention_weight_min": -29.309955596923828,
      "activations/layer12_attention_weight_max": 34.174415588378906,
      "activations/layer12_attention_weight_min": -28.197961807250977,
      "activations/layer13_attention_weight_max": 36.00419616699219,
      "activations/layer13_attention_weight_min": -32.633750915527344,
      "activations/layer14_attention_weight_max": 49.125797271728516,
      "activations/layer14_attention_weight_min": -36.1400146484375,
      "activations/layer15_attention_weight_max": 35.616092681884766,
      "activations/layer15_attention_weight_min": -29.7686710357666,
      "activations/layer16_attention_weight_max": 34.81928253173828,
      "activations/layer16_attention_weight_min": -25.983196258544922,
      "activations/layer17_attention_weight_max": 35.22193908691406,
      "activations/layer17_attention_weight_min": -26.091800689697266,
      "activations/layer18_attention_weight_max": 30.981618881225586,
      "activations/layer18_attention_weight_min": -19.740137100219727,
      "activations/layer19_attention_weight_max": 41.88780212402344,
      "activations/layer19_attention_weight_min": -29.301742553710938,
      "activations/layer1_attention_weight_max": 16.360742568969727,
      "activations/layer1_attention_weight_min": -13.41653823852539,
      "activations/layer20_attention_weight_max": 30.554397583007812,
      "activations/layer20_attention_weight_min": -25.23656463623047,
      "activations/layer21_attention_weight_max": 30.213943481445312,
      "activations/layer21_attention_weight_min": -22.932138442993164,
      "activations/layer22_attention_weight_max": 48.37678527832031,
      "activations/layer22_attention_weight_min": -29.737289428710938,
      "activations/layer23_attention_weight_max": 41.910545349121094,
      "activations/layer23_attention_weight_min": -23.569942474365234,
      "activations/layer2_attention_weight_max": 31.292125701904297,
      "activations/layer2_attention_weight_min": -31.645000457763672,
      "activations/layer3_attention_weight_max": 89.77711486816406,
      "activations/layer3_attention_weight_min": -88.9364013671875,
      "activations/layer4_attention_weight_max": 113.40534973144531,
      "activations/layer4_attention_weight_min": -115.3023452758789,
      "activations/layer5_attention_weight_max": 56.62040328979492,
      "activations/layer5_attention_weight_min": -62.34444808959961,
      "activations/layer6_attention_weight_max": 53.03898239135742,
      "activations/layer6_attention_weight_min": -51.536720275878906,
      "activations/layer7_attention_weight_max": 82.16343688964844,
      "activations/layer7_attention_weight_min": -84.20134735107422,
      "activations/layer8_attention_weight_max": 37.14604568481445,
      "activations/layer8_attention_weight_min": -39.29984664916992,
      "activations/layer9_attention_weight_max": 31.98573875427246,
      "activations/layer9_attention_weight_min": -32.0503044128418,
      "epoch": 7.51,
      "learning_rate": 0.00010258181818181817,
      "loss": 2.8292,
      "step": 129300
    },
    {
      "activations/layer0_attention_weight_max": 15.967884063720703,
      "activations/layer0_attention_weight_min": -11.554239273071289,
      "activations/layer10_attention_weight_max": 33.29189682006836,
      "activations/layer10_attention_weight_min": -32.75897979736328,
      "activations/layer11_attention_weight_max": 34.79662322998047,
      "activations/layer11_attention_weight_min": -29.662784576416016,
      "activations/layer12_attention_weight_max": 33.41672134399414,
      "activations/layer12_attention_weight_min": -24.74797248840332,
      "activations/layer13_attention_weight_max": 45.44057846069336,
      "activations/layer13_attention_weight_min": -31.22072982788086,
      "activations/layer14_attention_weight_max": 63.470767974853516,
      "activations/layer14_attention_weight_min": -39.51504898071289,
      "activations/layer15_attention_weight_max": 43.29186248779297,
      "activations/layer15_attention_weight_min": -30.729557037353516,
      "activations/layer16_attention_weight_max": 36.54045104980469,
      "activations/layer16_attention_weight_min": -25.089275360107422,
      "activations/layer17_attention_weight_max": 40.44731521606445,
      "activations/layer17_attention_weight_min": -25.298593521118164,
      "activations/layer18_attention_weight_max": 35.97088623046875,
      "activations/layer18_attention_weight_min": -21.022924423217773,
      "activations/layer19_attention_weight_max": 41.435733795166016,
      "activations/layer19_attention_weight_min": -29.190061569213867,
      "activations/layer1_attention_weight_max": 16.29631996154785,
      "activations/layer1_attention_weight_min": -15.588520050048828,
      "activations/layer20_attention_weight_max": 37.463741302490234,
      "activations/layer20_attention_weight_min": -24.54583168029785,
      "activations/layer21_attention_weight_max": 36.81104278564453,
      "activations/layer21_attention_weight_min": -22.094463348388672,
      "activations/layer22_attention_weight_max": 56.30668258666992,
      "activations/layer22_attention_weight_min": -26.613950729370117,
      "activations/layer23_attention_weight_max": 40.38675308227539,
      "activations/layer23_attention_weight_min": -23.12842559814453,
      "activations/layer2_attention_weight_max": 33.00800323486328,
      "activations/layer2_attention_weight_min": -32.66312789916992,
      "activations/layer3_attention_weight_max": 95.08729553222656,
      "activations/layer3_attention_weight_min": -96.98619079589844,
      "activations/layer4_attention_weight_max": 124.30775451660156,
      "activations/layer4_attention_weight_min": -121.44669342041016,
      "activations/layer5_attention_weight_max": 58.957252502441406,
      "activations/layer5_attention_weight_min": -63.546051025390625,
      "activations/layer6_attention_weight_max": 49.47612762451172,
      "activations/layer6_attention_weight_min": -51.8921012878418,
      "activations/layer7_attention_weight_max": 93.66864776611328,
      "activations/layer7_attention_weight_min": -90.12845611572266,
      "activations/layer8_attention_weight_max": 41.4843635559082,
      "activations/layer8_attention_weight_min": -40.45730972290039,
      "activations/layer9_attention_weight_max": 37.27280044555664,
      "activations/layer9_attention_weight_min": -35.426124572753906,
      "epoch": 7.52,
      "learning_rate": 0.00010256287878787878,
      "loss": 2.8016,
      "step": 129350
    },
    {
      "activations/layer0_attention_weight_max": 16.028797149658203,
      "activations/layer0_attention_weight_min": -10.827350616455078,
      "activations/layer10_attention_weight_max": 32.953125,
      "activations/layer10_attention_weight_min": -31.774517059326172,
      "activations/layer11_attention_weight_max": 34.08412551879883,
      "activations/layer11_attention_weight_min": -29.866432189941406,
      "activations/layer12_attention_weight_max": 27.24827766418457,
      "activations/layer12_attention_weight_min": -24.222408294677734,
      "activations/layer13_attention_weight_max": 36.88877868652344,
      "activations/layer13_attention_weight_min": -33.08333969116211,
      "activations/layer14_attention_weight_max": 59.41965866088867,
      "activations/layer14_attention_weight_min": -43.06782913208008,
      "activations/layer15_attention_weight_max": 34.29606628417969,
      "activations/layer15_attention_weight_min": -29.06656837463379,
      "activations/layer16_attention_weight_max": 33.23430633544922,
      "activations/layer16_attention_weight_min": -26.493701934814453,
      "activations/layer17_attention_weight_max": 35.388877868652344,
      "activations/layer17_attention_weight_min": -25.459943771362305,
      "activations/layer18_attention_weight_max": 32.624786376953125,
      "activations/layer18_attention_weight_min": -20.512128829956055,
      "activations/layer19_attention_weight_max": 37.589202880859375,
      "activations/layer19_attention_weight_min": -31.08890151977539,
      "activations/layer1_attention_weight_max": 17.051156997680664,
      "activations/layer1_attention_weight_min": -16.22647476196289,
      "activations/layer20_attention_weight_max": 33.16035461425781,
      "activations/layer20_attention_weight_min": -23.778249740600586,
      "activations/layer21_attention_weight_max": 32.179141998291016,
      "activations/layer21_attention_weight_min": -23.928598403930664,
      "activations/layer22_attention_weight_max": 48.02973175048828,
      "activations/layer22_attention_weight_min": -27.498403549194336,
      "activations/layer23_attention_weight_max": 35.228023529052734,
      "activations/layer23_attention_weight_min": -23.59090805053711,
      "activations/layer2_attention_weight_max": 31.5492000579834,
      "activations/layer2_attention_weight_min": -30.422788619995117,
      "activations/layer3_attention_weight_max": 86.7035140991211,
      "activations/layer3_attention_weight_min": -88.43154907226562,
      "activations/layer4_attention_weight_max": 115.0083236694336,
      "activations/layer4_attention_weight_min": -110.09852600097656,
      "activations/layer5_attention_weight_max": 58.039730072021484,
      "activations/layer5_attention_weight_min": -60.73582077026367,
      "activations/layer6_attention_weight_max": 51.17375183105469,
      "activations/layer6_attention_weight_min": -53.363426208496094,
      "activations/layer7_attention_weight_max": 85.48258972167969,
      "activations/layer7_attention_weight_min": -86.75299072265625,
      "activations/layer8_attention_weight_max": 40.63447952270508,
      "activations/layer8_attention_weight_min": -39.717350006103516,
      "activations/layer9_attention_weight_max": 33.62529373168945,
      "activations/layer9_attention_weight_min": -34.167415618896484,
      "epoch": 7.52,
      "learning_rate": 0.00010254393939393938,
      "loss": 2.8189,
      "step": 129400
    },
    {
      "activations/layer0_attention_weight_max": 15.37564468383789,
      "activations/layer0_attention_weight_min": -12.043270111083984,
      "activations/layer10_attention_weight_max": 33.07695770263672,
      "activations/layer10_attention_weight_min": -30.641887664794922,
      "activations/layer11_attention_weight_max": 31.948333740234375,
      "activations/layer11_attention_weight_min": -29.833913803100586,
      "activations/layer12_attention_weight_max": 22.240463256835938,
      "activations/layer12_attention_weight_min": -28.74378776550293,
      "activations/layer13_attention_weight_max": 40.10315704345703,
      "activations/layer13_attention_weight_min": -34.696659088134766,
      "activations/layer14_attention_weight_max": 50.97663497924805,
      "activations/layer14_attention_weight_min": -42.28369903564453,
      "activations/layer15_attention_weight_max": 36.16086959838867,
      "activations/layer15_attention_weight_min": -32.463871002197266,
      "activations/layer16_attention_weight_max": 32.86201477050781,
      "activations/layer16_attention_weight_min": -29.109954833984375,
      "activations/layer17_attention_weight_max": 37.591217041015625,
      "activations/layer17_attention_weight_min": -26.349206924438477,
      "activations/layer18_attention_weight_max": 31.1082706451416,
      "activations/layer18_attention_weight_min": -23.043193817138672,
      "activations/layer19_attention_weight_max": 36.37228775024414,
      "activations/layer19_attention_weight_min": -31.982501983642578,
      "activations/layer1_attention_weight_max": 15.747490882873535,
      "activations/layer1_attention_weight_min": -13.262453079223633,
      "activations/layer20_attention_weight_max": 30.713701248168945,
      "activations/layer20_attention_weight_min": -26.283491134643555,
      "activations/layer21_attention_weight_max": 27.390560150146484,
      "activations/layer21_attention_weight_min": -22.702550888061523,
      "activations/layer22_attention_weight_max": 49.20492935180664,
      "activations/layer22_attention_weight_min": -31.77626609802246,
      "activations/layer23_attention_weight_max": 38.55217742919922,
      "activations/layer23_attention_weight_min": -26.22243881225586,
      "activations/layer2_attention_weight_max": 32.18986511230469,
      "activations/layer2_attention_weight_min": -32.710086822509766,
      "activations/layer3_attention_weight_max": 93.2010498046875,
      "activations/layer3_attention_weight_min": -94.42597961425781,
      "activations/layer4_attention_weight_max": 121.73957824707031,
      "activations/layer4_attention_weight_min": -122.0084457397461,
      "activations/layer5_attention_weight_max": 61.642967224121094,
      "activations/layer5_attention_weight_min": -63.39948654174805,
      "activations/layer6_attention_weight_max": 50.75678634643555,
      "activations/layer6_attention_weight_min": -55.22539520263672,
      "activations/layer7_attention_weight_max": 88.07207489013672,
      "activations/layer7_attention_weight_min": -98.62464904785156,
      "activations/layer8_attention_weight_max": 42.21147155761719,
      "activations/layer8_attention_weight_min": -39.33367919921875,
      "activations/layer9_attention_weight_max": 37.01737594604492,
      "activations/layer9_attention_weight_min": -32.78683853149414,
      "epoch": 7.52,
      "learning_rate": 0.00010252499999999998,
      "loss": 2.8184,
      "step": 129450
    },
    {
      "activations/layer0_attention_weight_max": 16.39759063720703,
      "activations/layer0_attention_weight_min": -11.412649154663086,
      "activations/layer10_attention_weight_max": 34.7001953125,
      "activations/layer10_attention_weight_min": -29.72439193725586,
      "activations/layer11_attention_weight_max": 33.324310302734375,
      "activations/layer11_attention_weight_min": -29.59552574157715,
      "activations/layer12_attention_weight_max": 24.394275665283203,
      "activations/layer12_attention_weight_min": -25.561080932617188,
      "activations/layer13_attention_weight_max": 42.75202560424805,
      "activations/layer13_attention_weight_min": -32.10828399658203,
      "activations/layer14_attention_weight_max": 59.834407806396484,
      "activations/layer14_attention_weight_min": -40.73212432861328,
      "activations/layer15_attention_weight_max": 39.09320831298828,
      "activations/layer15_attention_weight_min": -27.59242057800293,
      "activations/layer16_attention_weight_max": 35.77037048339844,
      "activations/layer16_attention_weight_min": -26.35024642944336,
      "activations/layer17_attention_weight_max": 40.26649856567383,
      "activations/layer17_attention_weight_min": -24.995847702026367,
      "activations/layer18_attention_weight_max": 34.942596435546875,
      "activations/layer18_attention_weight_min": -23.21762466430664,
      "activations/layer19_attention_weight_max": 42.578590393066406,
      "activations/layer19_attention_weight_min": -27.879322052001953,
      "activations/layer1_attention_weight_max": 17.4577579498291,
      "activations/layer1_attention_weight_min": -13.780978202819824,
      "activations/layer20_attention_weight_max": 33.98282241821289,
      "activations/layer20_attention_weight_min": -24.553733825683594,
      "activations/layer21_attention_weight_max": 35.73143768310547,
      "activations/layer21_attention_weight_min": -21.225553512573242,
      "activations/layer22_attention_weight_max": 53.89512252807617,
      "activations/layer22_attention_weight_min": -27.39043617248535,
      "activations/layer23_attention_weight_max": 38.24598693847656,
      "activations/layer23_attention_weight_min": -23.240116119384766,
      "activations/layer2_attention_weight_max": 31.549907684326172,
      "activations/layer2_attention_weight_min": -30.41877555847168,
      "activations/layer3_attention_weight_max": 88.91947174072266,
      "activations/layer3_attention_weight_min": -87.0859146118164,
      "activations/layer4_attention_weight_max": 116.79911041259766,
      "activations/layer4_attention_weight_min": -109.82452392578125,
      "activations/layer5_attention_weight_max": 58.715267181396484,
      "activations/layer5_attention_weight_min": -57.96138000488281,
      "activations/layer6_attention_weight_max": 49.905479431152344,
      "activations/layer6_attention_weight_min": -53.855552673339844,
      "activations/layer7_attention_weight_max": 89.62127685546875,
      "activations/layer7_attention_weight_min": -84.90052795410156,
      "activations/layer8_attention_weight_max": 42.08169174194336,
      "activations/layer8_attention_weight_min": -39.4183235168457,
      "activations/layer9_attention_weight_max": 36.08544158935547,
      "activations/layer9_attention_weight_min": -31.972644805908203,
      "epoch": 7.52,
      "learning_rate": 0.0001025060606060606,
      "loss": 2.8308,
      "step": 129500
    },
    {
      "activations/layer0_attention_weight_max": 16.281841278076172,
      "activations/layer0_attention_weight_min": -11.480888366699219,
      "activations/layer10_attention_weight_max": 30.027013778686523,
      "activations/layer10_attention_weight_min": -28.655038833618164,
      "activations/layer11_attention_weight_max": 28.39552116394043,
      "activations/layer11_attention_weight_min": -26.881305694580078,
      "activations/layer12_attention_weight_max": 21.381200790405273,
      "activations/layer12_attention_weight_min": -27.677148818969727,
      "activations/layer13_attention_weight_max": 37.360984802246094,
      "activations/layer13_attention_weight_min": -30.906654357910156,
      "activations/layer14_attention_weight_max": 47.18021011352539,
      "activations/layer14_attention_weight_min": -38.961769104003906,
      "activations/layer15_attention_weight_max": 34.5771598815918,
      "activations/layer15_attention_weight_min": -27.53439712524414,
      "activations/layer16_attention_weight_max": 36.183353424072266,
      "activations/layer16_attention_weight_min": -28.433029174804688,
      "activations/layer17_attention_weight_max": 32.28832244873047,
      "activations/layer17_attention_weight_min": -24.285070419311523,
      "activations/layer18_attention_weight_max": 29.412029266357422,
      "activations/layer18_attention_weight_min": -23.892847061157227,
      "activations/layer19_attention_weight_max": 36.60818862915039,
      "activations/layer19_attention_weight_min": -29.765762329101562,
      "activations/layer1_attention_weight_max": 17.04824447631836,
      "activations/layer1_attention_weight_min": -14.147913932800293,
      "activations/layer20_attention_weight_max": 31.801651000976562,
      "activations/layer20_attention_weight_min": -24.115222930908203,
      "activations/layer21_attention_weight_max": 30.307525634765625,
      "activations/layer21_attention_weight_min": -22.39621353149414,
      "activations/layer22_attention_weight_max": 45.6290168762207,
      "activations/layer22_attention_weight_min": -27.999061584472656,
      "activations/layer23_attention_weight_max": 33.434722900390625,
      "activations/layer23_attention_weight_min": -22.981101989746094,
      "activations/layer2_attention_weight_max": 31.55677032470703,
      "activations/layer2_attention_weight_min": -30.658287048339844,
      "activations/layer3_attention_weight_max": 87.48014068603516,
      "activations/layer3_attention_weight_min": -90.86540985107422,
      "activations/layer4_attention_weight_max": 116.94941711425781,
      "activations/layer4_attention_weight_min": -114.6246566772461,
      "activations/layer5_attention_weight_max": 58.016944885253906,
      "activations/layer5_attention_weight_min": -65.12647247314453,
      "activations/layer6_attention_weight_max": 47.95265579223633,
      "activations/layer6_attention_weight_min": -50.322227478027344,
      "activations/layer7_attention_weight_max": 84.60808563232422,
      "activations/layer7_attention_weight_min": -81.98369598388672,
      "activations/layer8_attention_weight_max": 39.29988479614258,
      "activations/layer8_attention_weight_min": -38.844886779785156,
      "activations/layer9_attention_weight_max": 30.7622013092041,
      "activations/layer9_attention_weight_min": -33.17576217651367,
      "epoch": 7.53,
      "learning_rate": 0.0001024871212121212,
      "loss": 2.7989,
      "step": 129550
    },
    {
      "activations/layer0_attention_weight_max": 15.66987133026123,
      "activations/layer0_attention_weight_min": -11.867621421813965,
      "activations/layer10_attention_weight_max": 36.43193054199219,
      "activations/layer10_attention_weight_min": -31.43346405029297,
      "activations/layer11_attention_weight_max": 35.04412078857422,
      "activations/layer11_attention_weight_min": -30.849653244018555,
      "activations/layer12_attention_weight_max": 58.77162551879883,
      "activations/layer12_attention_weight_min": -25.524471282958984,
      "activations/layer13_attention_weight_max": 57.19694137573242,
      "activations/layer13_attention_weight_min": -36.132530212402344,
      "activations/layer14_attention_weight_max": 50.6661491394043,
      "activations/layer14_attention_weight_min": -43.40412521362305,
      "activations/layer15_attention_weight_max": 42.17152404785156,
      "activations/layer15_attention_weight_min": -31.47679328918457,
      "activations/layer16_attention_weight_max": 34.825477600097656,
      "activations/layer16_attention_weight_min": -28.777294158935547,
      "activations/layer17_attention_weight_max": 33.75230407714844,
      "activations/layer17_attention_weight_min": -28.822261810302734,
      "activations/layer18_attention_weight_max": 29.502849578857422,
      "activations/layer18_attention_weight_min": -22.998260498046875,
      "activations/layer19_attention_weight_max": 40.409114837646484,
      "activations/layer19_attention_weight_min": -32.81822967529297,
      "activations/layer1_attention_weight_max": 17.039169311523438,
      "activations/layer1_attention_weight_min": -14.386170387268066,
      "activations/layer20_attention_weight_max": 31.754375457763672,
      "activations/layer20_attention_weight_min": -25.99410057067871,
      "activations/layer21_attention_weight_max": 32.185672760009766,
      "activations/layer21_attention_weight_min": -23.163511276245117,
      "activations/layer22_attention_weight_max": 49.9646110534668,
      "activations/layer22_attention_weight_min": -30.605710983276367,
      "activations/layer23_attention_weight_max": 37.63584899902344,
      "activations/layer23_attention_weight_min": -24.166044235229492,
      "activations/layer2_attention_weight_max": 33.565826416015625,
      "activations/layer2_attention_weight_min": -31.741134643554688,
      "activations/layer3_attention_weight_max": 97.27704620361328,
      "activations/layer3_attention_weight_min": -95.86662292480469,
      "activations/layer4_attention_weight_max": 126.22430419921875,
      "activations/layer4_attention_weight_min": -120.50936126708984,
      "activations/layer5_attention_weight_max": 63.42988586425781,
      "activations/layer5_attention_weight_min": -60.77232360839844,
      "activations/layer6_attention_weight_max": 52.93149948120117,
      "activations/layer6_attention_weight_min": -53.62833023071289,
      "activations/layer7_attention_weight_max": 92.9349365234375,
      "activations/layer7_attention_weight_min": -86.70925903320312,
      "activations/layer8_attention_weight_max": 42.34616470336914,
      "activations/layer8_attention_weight_min": -41.00630187988281,
      "activations/layer9_attention_weight_max": 38.753326416015625,
      "activations/layer9_attention_weight_min": -35.92412185668945,
      "epoch": 7.53,
      "learning_rate": 0.00010246818181818182,
      "loss": 2.8183,
      "step": 129600
    },
    {
      "activations/layer0_attention_weight_max": 17.121828079223633,
      "activations/layer0_attention_weight_min": -12.079217910766602,
      "activations/layer10_attention_weight_max": 34.69972610473633,
      "activations/layer10_attention_weight_min": -31.881038665771484,
      "activations/layer11_attention_weight_max": 35.75518035888672,
      "activations/layer11_attention_weight_min": -30.3883056640625,
      "activations/layer12_attention_weight_max": 24.992755889892578,
      "activations/layer12_attention_weight_min": -23.88352394104004,
      "activations/layer13_attention_weight_max": 47.22899627685547,
      "activations/layer13_attention_weight_min": -31.840641021728516,
      "activations/layer14_attention_weight_max": 64.06739807128906,
      "activations/layer14_attention_weight_min": -41.13468551635742,
      "activations/layer15_attention_weight_max": 40.81652069091797,
      "activations/layer15_attention_weight_min": -29.386850357055664,
      "activations/layer16_attention_weight_max": 35.7647819519043,
      "activations/layer16_attention_weight_min": -28.155168533325195,
      "activations/layer17_attention_weight_max": 34.76454544067383,
      "activations/layer17_attention_weight_min": -24.134254455566406,
      "activations/layer18_attention_weight_max": 33.316287994384766,
      "activations/layer18_attention_weight_min": -21.410409927368164,
      "activations/layer19_attention_weight_max": 40.98330307006836,
      "activations/layer19_attention_weight_min": -28.53541374206543,
      "activations/layer1_attention_weight_max": 17.339075088500977,
      "activations/layer1_attention_weight_min": -14.56138801574707,
      "activations/layer20_attention_weight_max": 34.448543548583984,
      "activations/layer20_attention_weight_min": -23.0380916595459,
      "activations/layer21_attention_weight_max": 31.650272369384766,
      "activations/layer21_attention_weight_min": -20.083450317382812,
      "activations/layer22_attention_weight_max": 48.160709381103516,
      "activations/layer22_attention_weight_min": -26.853757858276367,
      "activations/layer23_attention_weight_max": 39.52495574951172,
      "activations/layer23_attention_weight_min": -24.96930694580078,
      "activations/layer2_attention_weight_max": 31.706443786621094,
      "activations/layer2_attention_weight_min": -30.423009872436523,
      "activations/layer3_attention_weight_max": 88.43758392333984,
      "activations/layer3_attention_weight_min": -90.04937744140625,
      "activations/layer4_attention_weight_max": 114.4645767211914,
      "activations/layer4_attention_weight_min": -110.76820373535156,
      "activations/layer5_attention_weight_max": 59.98102569580078,
      "activations/layer5_attention_weight_min": -60.064186096191406,
      "activations/layer6_attention_weight_max": 49.39626693725586,
      "activations/layer6_attention_weight_min": -50.997520446777344,
      "activations/layer7_attention_weight_max": 87.9827880859375,
      "activations/layer7_attention_weight_min": -87.14778900146484,
      "activations/layer8_attention_weight_max": 40.3632698059082,
      "activations/layer8_attention_weight_min": -40.151756286621094,
      "activations/layer9_attention_weight_max": 36.43095016479492,
      "activations/layer9_attention_weight_min": -35.29895782470703,
      "epoch": 7.53,
      "learning_rate": 0.00010244924242424242,
      "loss": 2.8179,
      "step": 129650
    },
    {
      "activations/layer0_attention_weight_max": 15.600454330444336,
      "activations/layer0_attention_weight_min": -11.614304542541504,
      "activations/layer10_attention_weight_max": 33.9991569519043,
      "activations/layer10_attention_weight_min": -36.822452545166016,
      "activations/layer11_attention_weight_max": 32.68112564086914,
      "activations/layer11_attention_weight_min": -33.08536911010742,
      "activations/layer12_attention_weight_max": 26.182832717895508,
      "activations/layer12_attention_weight_min": -27.68583869934082,
      "activations/layer13_attention_weight_max": 45.62950897216797,
      "activations/layer13_attention_weight_min": -33.190982818603516,
      "activations/layer14_attention_weight_max": 59.527137756347656,
      "activations/layer14_attention_weight_min": -38.95032501220703,
      "activations/layer15_attention_weight_max": 39.20646667480469,
      "activations/layer15_attention_weight_min": -29.62928581237793,
      "activations/layer16_attention_weight_max": 36.097015380859375,
      "activations/layer16_attention_weight_min": -26.919696807861328,
      "activations/layer17_attention_weight_max": 37.9017219543457,
      "activations/layer17_attention_weight_min": -24.509296417236328,
      "activations/layer18_attention_weight_max": 33.852054595947266,
      "activations/layer18_attention_weight_min": -20.716262817382812,
      "activations/layer19_attention_weight_max": 42.29199981689453,
      "activations/layer19_attention_weight_min": -28.67142677307129,
      "activations/layer1_attention_weight_max": 16.406179428100586,
      "activations/layer1_attention_weight_min": -14.525632858276367,
      "activations/layer20_attention_weight_max": 35.780250549316406,
      "activations/layer20_attention_weight_min": -24.270374298095703,
      "activations/layer21_attention_weight_max": 33.4457893371582,
      "activations/layer21_attention_weight_min": -23.733505249023438,
      "activations/layer22_attention_weight_max": 65.74623107910156,
      "activations/layer22_attention_weight_min": -28.853601455688477,
      "activations/layer23_attention_weight_max": 40.728675842285156,
      "activations/layer23_attention_weight_min": -22.750843048095703,
      "activations/layer2_attention_weight_max": 31.245473861694336,
      "activations/layer2_attention_weight_min": -30.38896942138672,
      "activations/layer3_attention_weight_max": 88.87279510498047,
      "activations/layer3_attention_weight_min": -89.96759796142578,
      "activations/layer4_attention_weight_max": 121.0206069946289,
      "activations/layer4_attention_weight_min": -117.30989074707031,
      "activations/layer5_attention_weight_max": 61.9491081237793,
      "activations/layer5_attention_weight_min": -63.160125732421875,
      "activations/layer6_attention_weight_max": 53.74319076538086,
      "activations/layer6_attention_weight_min": -54.50514602661133,
      "activations/layer7_attention_weight_max": 93.82023620605469,
      "activations/layer7_attention_weight_min": -92.07393646240234,
      "activations/layer8_attention_weight_max": 44.07378005981445,
      "activations/layer8_attention_weight_min": -43.5384407043457,
      "activations/layer9_attention_weight_max": 37.5605583190918,
      "activations/layer9_attention_weight_min": -38.57859802246094,
      "epoch": 7.54,
      "learning_rate": 0.00010243030303030302,
      "loss": 2.7973,
      "step": 129700
    },
    {
      "activations/layer0_attention_weight_max": 16.18931007385254,
      "activations/layer0_attention_weight_min": -11.549026489257812,
      "activations/layer10_attention_weight_max": 31.609060287475586,
      "activations/layer10_attention_weight_min": -30.51513671875,
      "activations/layer11_attention_weight_max": 31.778934478759766,
      "activations/layer11_attention_weight_min": -28.984642028808594,
      "activations/layer12_attention_weight_max": 26.2678165435791,
      "activations/layer12_attention_weight_min": -24.012704849243164,
      "activations/layer13_attention_weight_max": 41.050811767578125,
      "activations/layer13_attention_weight_min": -30.3293514251709,
      "activations/layer14_attention_weight_max": 48.769615173339844,
      "activations/layer14_attention_weight_min": -40.11159896850586,
      "activations/layer15_attention_weight_max": 35.24807357788086,
      "activations/layer15_attention_weight_min": -28.20123291015625,
      "activations/layer16_attention_weight_max": 30.104970932006836,
      "activations/layer16_attention_weight_min": -25.762653350830078,
      "activations/layer17_attention_weight_max": 31.367794036865234,
      "activations/layer17_attention_weight_min": -23.850444793701172,
      "activations/layer18_attention_weight_max": 29.8543758392334,
      "activations/layer18_attention_weight_min": -20.992979049682617,
      "activations/layer19_attention_weight_max": 34.372188568115234,
      "activations/layer19_attention_weight_min": -27.88623046875,
      "activations/layer1_attention_weight_max": 18.16790199279785,
      "activations/layer1_attention_weight_min": -13.903460502624512,
      "activations/layer20_attention_weight_max": 30.8574161529541,
      "activations/layer20_attention_weight_min": -23.08689308166504,
      "activations/layer21_attention_weight_max": 32.21009063720703,
      "activations/layer21_attention_weight_min": -20.725854873657227,
      "activations/layer22_attention_weight_max": 47.50105667114258,
      "activations/layer22_attention_weight_min": -26.55612564086914,
      "activations/layer23_attention_weight_max": 33.437904357910156,
      "activations/layer23_attention_weight_min": -20.71133041381836,
      "activations/layer2_attention_weight_max": 31.844039916992188,
      "activations/layer2_attention_weight_min": -32.69147491455078,
      "activations/layer3_attention_weight_max": 89.01629638671875,
      "activations/layer3_attention_weight_min": -95.49797821044922,
      "activations/layer4_attention_weight_max": 116.20269012451172,
      "activations/layer4_attention_weight_min": -121.80035400390625,
      "activations/layer5_attention_weight_max": 58.12081527709961,
      "activations/layer5_attention_weight_min": -60.68379211425781,
      "activations/layer6_attention_weight_max": 47.033668518066406,
      "activations/layer6_attention_weight_min": -48.82603073120117,
      "activations/layer7_attention_weight_max": 84.66828918457031,
      "activations/layer7_attention_weight_min": -80.9489974975586,
      "activations/layer8_attention_weight_max": 37.77931213378906,
      "activations/layer8_attention_weight_min": -38.2797966003418,
      "activations/layer9_attention_weight_max": 33.685237884521484,
      "activations/layer9_attention_weight_min": -32.53474044799805,
      "epoch": 7.54,
      "learning_rate": 0.00010241136363636363,
      "loss": 2.81,
      "step": 129750
    },
    {
      "activations/layer0_attention_weight_max": 16.01130485534668,
      "activations/layer0_attention_weight_min": -11.474786758422852,
      "activations/layer10_attention_weight_max": 31.171428680419922,
      "activations/layer10_attention_weight_min": -31.580106735229492,
      "activations/layer11_attention_weight_max": 30.742965698242188,
      "activations/layer11_attention_weight_min": -28.69180679321289,
      "activations/layer12_attention_weight_max": 21.47822380065918,
      "activations/layer12_attention_weight_min": -24.964792251586914,
      "activations/layer13_attention_weight_max": 37.624794006347656,
      "activations/layer13_attention_weight_min": -31.79521369934082,
      "activations/layer14_attention_weight_max": 56.003177642822266,
      "activations/layer14_attention_weight_min": -42.10951232910156,
      "activations/layer15_attention_weight_max": 35.32284927368164,
      "activations/layer15_attention_weight_min": -31.406455993652344,
      "activations/layer16_attention_weight_max": 34.076255798339844,
      "activations/layer16_attention_weight_min": -26.717937469482422,
      "activations/layer17_attention_weight_max": 35.130245208740234,
      "activations/layer17_attention_weight_min": -26.244550704956055,
      "activations/layer18_attention_weight_max": 38.9128303527832,
      "activations/layer18_attention_weight_min": -22.982240676879883,
      "activations/layer19_attention_weight_max": 39.673988342285156,
      "activations/layer19_attention_weight_min": -30.9739990234375,
      "activations/layer1_attention_weight_max": 17.535140991210938,
      "activations/layer1_attention_weight_min": -14.617656707763672,
      "activations/layer20_attention_weight_max": 33.78449249267578,
      "activations/layer20_attention_weight_min": -25.03276824951172,
      "activations/layer21_attention_weight_max": 32.36795425415039,
      "activations/layer21_attention_weight_min": -23.368362426757812,
      "activations/layer22_attention_weight_max": 51.304656982421875,
      "activations/layer22_attention_weight_min": -32.11063766479492,
      "activations/layer23_attention_weight_max": 36.535545349121094,
      "activations/layer23_attention_weight_min": -23.8204345703125,
      "activations/layer2_attention_weight_max": 31.871089935302734,
      "activations/layer2_attention_weight_min": -30.262746810913086,
      "activations/layer3_attention_weight_max": 89.59832000732422,
      "activations/layer3_attention_weight_min": -88.142822265625,
      "activations/layer4_attention_weight_max": 120.12239837646484,
      "activations/layer4_attention_weight_min": -116.27046203613281,
      "activations/layer5_attention_weight_max": 63.60001754760742,
      "activations/layer5_attention_weight_min": -63.72221374511719,
      "activations/layer6_attention_weight_max": 52.693721771240234,
      "activations/layer6_attention_weight_min": -57.52372741699219,
      "activations/layer7_attention_weight_max": 87.74834442138672,
      "activations/layer7_attention_weight_min": -89.64570617675781,
      "activations/layer8_attention_weight_max": 39.73396682739258,
      "activations/layer8_attention_weight_min": -40.95762634277344,
      "activations/layer9_attention_weight_max": 36.02735137939453,
      "activations/layer9_attention_weight_min": -37.49274444580078,
      "epoch": 7.54,
      "learning_rate": 0.00010239242424242422,
      "loss": 2.8071,
      "step": 129800
    },
    {
      "activations/layer0_attention_weight_max": 15.827122688293457,
      "activations/layer0_attention_weight_min": -12.114763259887695,
      "activations/layer10_attention_weight_max": 34.07562255859375,
      "activations/layer10_attention_weight_min": -33.133644104003906,
      "activations/layer11_attention_weight_max": 32.811256408691406,
      "activations/layer11_attention_weight_min": -32.568485260009766,
      "activations/layer12_attention_weight_max": 24.313720703125,
      "activations/layer12_attention_weight_min": -27.378202438354492,
      "activations/layer13_attention_weight_max": 40.945701599121094,
      "activations/layer13_attention_weight_min": -33.39895248413086,
      "activations/layer14_attention_weight_max": 57.43035125732422,
      "activations/layer14_attention_weight_min": -46.596134185791016,
      "activations/layer15_attention_weight_max": 36.55511474609375,
      "activations/layer15_attention_weight_min": -31.012319564819336,
      "activations/layer16_attention_weight_max": 34.13661575317383,
      "activations/layer16_attention_weight_min": -27.237226486206055,
      "activations/layer17_attention_weight_max": 36.06444549560547,
      "activations/layer17_attention_weight_min": -24.094148635864258,
      "activations/layer18_attention_weight_max": 33.33009719848633,
      "activations/layer18_attention_weight_min": -21.429325103759766,
      "activations/layer19_attention_weight_max": 38.45976257324219,
      "activations/layer19_attention_weight_min": -28.5071964263916,
      "activations/layer1_attention_weight_max": 16.75152587890625,
      "activations/layer1_attention_weight_min": -13.205236434936523,
      "activations/layer20_attention_weight_max": 33.09767150878906,
      "activations/layer20_attention_weight_min": -23.179513931274414,
      "activations/layer21_attention_weight_max": 33.420448303222656,
      "activations/layer21_attention_weight_min": -21.262462615966797,
      "activations/layer22_attention_weight_max": 48.359657287597656,
      "activations/layer22_attention_weight_min": -28.84171485900879,
      "activations/layer23_attention_weight_max": 34.9822883605957,
      "activations/layer23_attention_weight_min": -23.335758209228516,
      "activations/layer2_attention_weight_max": 31.293964385986328,
      "activations/layer2_attention_weight_min": -29.760848999023438,
      "activations/layer3_attention_weight_max": 88.38642883300781,
      "activations/layer3_attention_weight_min": -89.77359008789062,
      "activations/layer4_attention_weight_max": 120.16060638427734,
      "activations/layer4_attention_weight_min": -119.05902099609375,
      "activations/layer5_attention_weight_max": 62.6729736328125,
      "activations/layer5_attention_weight_min": -62.95513153076172,
      "activations/layer6_attention_weight_max": 53.649723052978516,
      "activations/layer6_attention_weight_min": -56.933799743652344,
      "activations/layer7_attention_weight_max": 94.46732330322266,
      "activations/layer7_attention_weight_min": -103.42769622802734,
      "activations/layer8_attention_weight_max": 43.05017852783203,
      "activations/layer8_attention_weight_min": -44.66261672973633,
      "activations/layer9_attention_weight_max": 36.83552932739258,
      "activations/layer9_attention_weight_min": -35.54072570800781,
      "epoch": 7.55,
      "learning_rate": 0.00010237348484848484,
      "loss": 2.8055,
      "step": 129850
    },
    {
      "activations/layer0_attention_weight_max": 17.49549102783203,
      "activations/layer0_attention_weight_min": -11.447361946105957,
      "activations/layer10_attention_weight_max": 36.27396774291992,
      "activations/layer10_attention_weight_min": -36.66287612915039,
      "activations/layer11_attention_weight_max": 36.343414306640625,
      "activations/layer11_attention_weight_min": -31.354734420776367,
      "activations/layer12_attention_weight_max": 24.39190101623535,
      "activations/layer12_attention_weight_min": -25.11099624633789,
      "activations/layer13_attention_weight_max": 41.5889778137207,
      "activations/layer13_attention_weight_min": -31.903545379638672,
      "activations/layer14_attention_weight_max": 52.52854537963867,
      "activations/layer14_attention_weight_min": -37.78486633300781,
      "activations/layer15_attention_weight_max": 35.480003356933594,
      "activations/layer15_attention_weight_min": -30.51714324951172,
      "activations/layer16_attention_weight_max": 36.61612319946289,
      "activations/layer16_attention_weight_min": -27.39909553527832,
      "activations/layer17_attention_weight_max": 36.92830276489258,
      "activations/layer17_attention_weight_min": -25.816564559936523,
      "activations/layer18_attention_weight_max": 32.80662536621094,
      "activations/layer18_attention_weight_min": -22.093769073486328,
      "activations/layer19_attention_weight_max": 37.757843017578125,
      "activations/layer19_attention_weight_min": -30.939395904541016,
      "activations/layer1_attention_weight_max": 17.18520164489746,
      "activations/layer1_attention_weight_min": -13.928894996643066,
      "activations/layer20_attention_weight_max": 32.2745475769043,
      "activations/layer20_attention_weight_min": -24.118375778198242,
      "activations/layer21_attention_weight_max": 32.15966033935547,
      "activations/layer21_attention_weight_min": -22.73870086669922,
      "activations/layer22_attention_weight_max": 50.165931701660156,
      "activations/layer22_attention_weight_min": -29.536176681518555,
      "activations/layer23_attention_weight_max": 37.487300872802734,
      "activations/layer23_attention_weight_min": -23.788890838623047,
      "activations/layer2_attention_weight_max": 32.03911590576172,
      "activations/layer2_attention_weight_min": -30.643293380737305,
      "activations/layer3_attention_weight_max": 92.61024475097656,
      "activations/layer3_attention_weight_min": -93.28192138671875,
      "activations/layer4_attention_weight_max": 122.40462493896484,
      "activations/layer4_attention_weight_min": -117.2531967163086,
      "activations/layer5_attention_weight_max": 60.70602035522461,
      "activations/layer5_attention_weight_min": -64.2588119506836,
      "activations/layer6_attention_weight_max": 52.79707717895508,
      "activations/layer6_attention_weight_min": -53.93532180786133,
      "activations/layer7_attention_weight_max": 90.14498901367188,
      "activations/layer7_attention_weight_min": -92.49903869628906,
      "activations/layer8_attention_weight_max": 41.73459243774414,
      "activations/layer8_attention_weight_min": -43.9002571105957,
      "activations/layer9_attention_weight_max": 38.13282775878906,
      "activations/layer9_attention_weight_min": -37.291114807128906,
      "epoch": 7.55,
      "learning_rate": 0.00010235454545454545,
      "loss": 2.7967,
      "step": 129900
    },
    {
      "activations/layer0_attention_weight_max": 15.919194221496582,
      "activations/layer0_attention_weight_min": -12.357144355773926,
      "activations/layer10_attention_weight_max": 33.2250862121582,
      "activations/layer10_attention_weight_min": -31.69940757751465,
      "activations/layer11_attention_weight_max": 30.502256393432617,
      "activations/layer11_attention_weight_min": -30.648557662963867,
      "activations/layer12_attention_weight_max": 22.330524444580078,
      "activations/layer12_attention_weight_min": -23.845590591430664,
      "activations/layer13_attention_weight_max": 36.29823684692383,
      "activations/layer13_attention_weight_min": -30.079660415649414,
      "activations/layer14_attention_weight_max": 45.9960823059082,
      "activations/layer14_attention_weight_min": -36.39480972290039,
      "activations/layer15_attention_weight_max": 34.192535400390625,
      "activations/layer15_attention_weight_min": -30.018604278564453,
      "activations/layer16_attention_weight_max": 31.636669158935547,
      "activations/layer16_attention_weight_min": -28.350845336914062,
      "activations/layer17_attention_weight_max": 32.24320602416992,
      "activations/layer17_attention_weight_min": -24.238754272460938,
      "activations/layer18_attention_weight_max": 30.026201248168945,
      "activations/layer18_attention_weight_min": -22.147743225097656,
      "activations/layer19_attention_weight_max": 34.638179779052734,
      "activations/layer19_attention_weight_min": -30.923343658447266,
      "activations/layer1_attention_weight_max": 16.714067459106445,
      "activations/layer1_attention_weight_min": -19.157846450805664,
      "activations/layer20_attention_weight_max": 31.541717529296875,
      "activations/layer20_attention_weight_min": -24.27557945251465,
      "activations/layer21_attention_weight_max": 28.241355895996094,
      "activations/layer21_attention_weight_min": -21.0021915435791,
      "activations/layer22_attention_weight_max": 42.565338134765625,
      "activations/layer22_attention_weight_min": -28.691776275634766,
      "activations/layer23_attention_weight_max": 32.3267822265625,
      "activations/layer23_attention_weight_min": -22.786170959472656,
      "activations/layer2_attention_weight_max": 30.90410614013672,
      "activations/layer2_attention_weight_min": -30.32027816772461,
      "activations/layer3_attention_weight_max": 89.3870849609375,
      "activations/layer3_attention_weight_min": -92.09678649902344,
      "activations/layer4_attention_weight_max": 116.48854064941406,
      "activations/layer4_attention_weight_min": -116.14913177490234,
      "activations/layer5_attention_weight_max": 58.84259033203125,
      "activations/layer5_attention_weight_min": -57.64265060424805,
      "activations/layer6_attention_weight_max": 49.28114700317383,
      "activations/layer6_attention_weight_min": -49.97343826293945,
      "activations/layer7_attention_weight_max": 85.42792510986328,
      "activations/layer7_attention_weight_min": -78.93159484863281,
      "activations/layer8_attention_weight_max": 36.25233459472656,
      "activations/layer8_attention_weight_min": -39.83682632446289,
      "activations/layer9_attention_weight_max": 34.72561264038086,
      "activations/layer9_attention_weight_min": -32.556461334228516,
      "epoch": 7.55,
      "learning_rate": 0.00010233560606060604,
      "loss": 2.7989,
      "step": 129950
    },
    {
      "activations/layer0_attention_weight_max": 16.60506248474121,
      "activations/layer0_attention_weight_min": -11.943391799926758,
      "activations/layer10_attention_weight_max": 33.99374771118164,
      "activations/layer10_attention_weight_min": -30.708999633789062,
      "activations/layer11_attention_weight_max": 36.107460021972656,
      "activations/layer11_attention_weight_min": -30.447202682495117,
      "activations/layer12_attention_weight_max": 41.16530227661133,
      "activations/layer12_attention_weight_min": -27.55182456970215,
      "activations/layer13_attention_weight_max": 42.31679153442383,
      "activations/layer13_attention_weight_min": -32.065303802490234,
      "activations/layer14_attention_weight_max": 62.48344421386719,
      "activations/layer14_attention_weight_min": -40.520477294921875,
      "activations/layer15_attention_weight_max": 40.070743560791016,
      "activations/layer15_attention_weight_min": -31.36481285095215,
      "activations/layer16_attention_weight_max": 35.96955490112305,
      "activations/layer16_attention_weight_min": -29.29667854309082,
      "activations/layer17_attention_weight_max": 37.159847259521484,
      "activations/layer17_attention_weight_min": -26.527496337890625,
      "activations/layer18_attention_weight_max": 38.92943572998047,
      "activations/layer18_attention_weight_min": -23.446088790893555,
      "activations/layer19_attention_weight_max": 42.2169303894043,
      "activations/layer19_attention_weight_min": -31.125967025756836,
      "activations/layer1_attention_weight_max": 19.96547508239746,
      "activations/layer1_attention_weight_min": -16.309120178222656,
      "activations/layer20_attention_weight_max": 35.35300827026367,
      "activations/layer20_attention_weight_min": -23.94663429260254,
      "activations/layer21_attention_weight_max": 37.258731842041016,
      "activations/layer21_attention_weight_min": -23.245901107788086,
      "activations/layer22_attention_weight_max": 53.77506637573242,
      "activations/layer22_attention_weight_min": -29.614826202392578,
      "activations/layer23_attention_weight_max": 39.26531982421875,
      "activations/layer23_attention_weight_min": -22.0330867767334,
      "activations/layer2_attention_weight_max": 34.31029510498047,
      "activations/layer2_attention_weight_min": -33.569976806640625,
      "activations/layer3_attention_weight_max": 92.99453735351562,
      "activations/layer3_attention_weight_min": -97.22356414794922,
      "activations/layer4_attention_weight_max": 120.99078369140625,
      "activations/layer4_attention_weight_min": -112.86286926269531,
      "activations/layer5_attention_weight_max": 59.26402282714844,
      "activations/layer5_attention_weight_min": -60.25120162963867,
      "activations/layer6_attention_weight_max": 48.61091613769531,
      "activations/layer6_attention_weight_min": -50.794979095458984,
      "activations/layer7_attention_weight_max": 84.80683135986328,
      "activations/layer7_attention_weight_min": -92.89232635498047,
      "activations/layer8_attention_weight_max": 40.3798942565918,
      "activations/layer8_attention_weight_min": -39.25333786010742,
      "activations/layer9_attention_weight_max": 36.04585266113281,
      "activations/layer9_attention_weight_min": -32.75469970703125,
      "epoch": 7.55,
      "learning_rate": 0.00010231666666666666,
      "loss": 2.8264,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_loss": 2.763671875,
      "eval_runtime": 8.5039,
      "eval_samples_per_second": 504.942,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_openwebtext_loss": 2.763671875,
      "eval_openwebtext_ppl": 15.85796463907484,
      "eval_openwebtext_runtime": 8.5039,
      "eval_openwebtext_samples_per_second": 504.942,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 2.022,
      "eval_wikitext_samples_per_second": 225.519,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_lambada_loss": 2.591796875,
      "eval_lambada_ppl": 13.35374506875535,
      "eval_lambada_runtime": 9.5919,
      "eval_lambada_samples_per_second": 507.616,
      "step": 130000
    },
    {
      "activations/layer0_attention_weight_max": 15.563244819641113,
      "activations/layer0_attention_weight_min": -11.555426597595215,
      "activations/layer10_attention_weight_max": 35.46780776977539,
      "activations/layer10_attention_weight_min": -32.17664337158203,
      "activations/layer11_attention_weight_max": 38.80287551879883,
      "activations/layer11_attention_weight_min": -29.976776123046875,
      "activations/layer12_attention_weight_max": 24.271570205688477,
      "activations/layer12_attention_weight_min": -26.977624893188477,
      "activations/layer13_attention_weight_max": 45.700050354003906,
      "activations/layer13_attention_weight_min": -32.95349884033203,
      "activations/layer14_attention_weight_max": 54.723915100097656,
      "activations/layer14_attention_weight_min": -41.144676208496094,
      "activations/layer15_attention_weight_max": 44.62028884887695,
      "activations/layer15_attention_weight_min": -30.698850631713867,
      "activations/layer16_attention_weight_max": 37.14084243774414,
      "activations/layer16_attention_weight_min": -27.052183151245117,
      "activations/layer17_attention_weight_max": 36.66493225097656,
      "activations/layer17_attention_weight_min": -25.739809036254883,
      "activations/layer18_attention_weight_max": 35.346370697021484,
      "activations/layer18_attention_weight_min": -20.628870010375977,
      "activations/layer19_attention_weight_max": 38.54744338989258,
      "activations/layer19_attention_weight_min": -30.73752212524414,
      "activations/layer1_attention_weight_max": 16.054927825927734,
      "activations/layer1_attention_weight_min": -13.566158294677734,
      "activations/layer20_attention_weight_max": 31.943634033203125,
      "activations/layer20_attention_weight_min": -23.44249725341797,
      "activations/layer21_attention_weight_max": 32.48974609375,
      "activations/layer21_attention_weight_min": -23.952728271484375,
      "activations/layer22_attention_weight_max": 54.11988830566406,
      "activations/layer22_attention_weight_min": -31.444278717041016,
      "activations/layer23_attention_weight_max": 42.264461517333984,
      "activations/layer23_attention_weight_min": -25.505704879760742,
      "activations/layer2_attention_weight_max": 31.79586410522461,
      "activations/layer2_attention_weight_min": -30.147916793823242,
      "activations/layer3_attention_weight_max": 91.72355651855469,
      "activations/layer3_attention_weight_min": -92.97511291503906,
      "activations/layer4_attention_weight_max": 123.52093505859375,
      "activations/layer4_attention_weight_min": -119.03580474853516,
      "activations/layer5_attention_weight_max": 61.34260177612305,
      "activations/layer5_attention_weight_min": -61.7728385925293,
      "activations/layer6_attention_weight_max": 51.29424285888672,
      "activations/layer6_attention_weight_min": -53.0175666809082,
      "activations/layer7_attention_weight_max": 88.76061248779297,
      "activations/layer7_attention_weight_min": -92.11434936523438,
      "activations/layer8_attention_weight_max": 40.27424240112305,
      "activations/layer8_attention_weight_min": -40.67403793334961,
      "activations/layer9_attention_weight_max": 36.015052795410156,
      "activations/layer9_attention_weight_min": -34.692562103271484,
      "epoch": 7.56,
      "learning_rate": 0.00010229772727272727,
      "loss": 2.8335,
      "step": 130050
    },
    {
      "activations/layer0_attention_weight_max": 15.569316864013672,
      "activations/layer0_attention_weight_min": -10.824664115905762,
      "activations/layer10_attention_weight_max": 33.356014251708984,
      "activations/layer10_attention_weight_min": -33.192237854003906,
      "activations/layer11_attention_weight_max": 33.17669677734375,
      "activations/layer11_attention_weight_min": -33.021568298339844,
      "activations/layer12_attention_weight_max": 26.640396118164062,
      "activations/layer12_attention_weight_min": -24.743391036987305,
      "activations/layer13_attention_weight_max": 37.89991760253906,
      "activations/layer13_attention_weight_min": -33.02968978881836,
      "activations/layer14_attention_weight_max": 50.582679748535156,
      "activations/layer14_attention_weight_min": -44.5732536315918,
      "activations/layer15_attention_weight_max": 38.477718353271484,
      "activations/layer15_attention_weight_min": -30.550189971923828,
      "activations/layer16_attention_weight_max": 33.54425811767578,
      "activations/layer16_attention_weight_min": -28.722366333007812,
      "activations/layer17_attention_weight_max": 36.2773323059082,
      "activations/layer17_attention_weight_min": -25.58590316772461,
      "activations/layer18_attention_weight_max": 29.675931930541992,
      "activations/layer18_attention_weight_min": -21.397220611572266,
      "activations/layer19_attention_weight_max": 38.99123001098633,
      "activations/layer19_attention_weight_min": -31.85367774963379,
      "activations/layer1_attention_weight_max": 16.966657638549805,
      "activations/layer1_attention_weight_min": -20.31155014038086,
      "activations/layer20_attention_weight_max": 31.177343368530273,
      "activations/layer20_attention_weight_min": -24.78148651123047,
      "activations/layer21_attention_weight_max": 31.557432174682617,
      "activations/layer21_attention_weight_min": -22.51603126525879,
      "activations/layer22_attention_weight_max": 51.557411193847656,
      "activations/layer22_attention_weight_min": -27.51483917236328,
      "activations/layer23_attention_weight_max": 35.56037139892578,
      "activations/layer23_attention_weight_min": -23.670150756835938,
      "activations/layer2_attention_weight_max": 33.52576446533203,
      "activations/layer2_attention_weight_min": -32.426307678222656,
      "activations/layer3_attention_weight_max": 89.83305358886719,
      "activations/layer3_attention_weight_min": -93.07107543945312,
      "activations/layer4_attention_weight_max": 123.2841567993164,
      "activations/layer4_attention_weight_min": -120.35892486572266,
      "activations/layer5_attention_weight_max": 61.22987365722656,
      "activations/layer5_attention_weight_min": -62.08595275878906,
      "activations/layer6_attention_weight_max": 49.42811584472656,
      "activations/layer6_attention_weight_min": -56.341636657714844,
      "activations/layer7_attention_weight_max": 91.87017822265625,
      "activations/layer7_attention_weight_min": -92.40064239501953,
      "activations/layer8_attention_weight_max": 40.717952728271484,
      "activations/layer8_attention_weight_min": -42.31669998168945,
      "activations/layer9_attention_weight_max": 36.02175521850586,
      "activations/layer9_attention_weight_min": -34.98539352416992,
      "epoch": 7.56,
      "learning_rate": 0.00010227878787878787,
      "loss": 2.8079,
      "step": 130100
    },
    {
      "activations/layer0_attention_weight_max": 15.81276798248291,
      "activations/layer0_attention_weight_min": -12.306900024414062,
      "activations/layer10_attention_weight_max": 35.27717971801758,
      "activations/layer10_attention_weight_min": -34.62428665161133,
      "activations/layer11_attention_weight_max": 35.17173767089844,
      "activations/layer11_attention_weight_min": -31.82625389099121,
      "activations/layer12_attention_weight_max": 29.693571090698242,
      "activations/layer12_attention_weight_min": -24.23849868774414,
      "activations/layer13_attention_weight_max": 43.142066955566406,
      "activations/layer13_attention_weight_min": -34.27265167236328,
      "activations/layer14_attention_weight_max": 54.71062469482422,
      "activations/layer14_attention_weight_min": -41.314414978027344,
      "activations/layer15_attention_weight_max": 42.484710693359375,
      "activations/layer15_attention_weight_min": -29.044076919555664,
      "activations/layer16_attention_weight_max": 34.1163215637207,
      "activations/layer16_attention_weight_min": -28.156593322753906,
      "activations/layer17_attention_weight_max": 36.26549530029297,
      "activations/layer17_attention_weight_min": -25.254886627197266,
      "activations/layer18_attention_weight_max": 32.668453216552734,
      "activations/layer18_attention_weight_min": -21.974597930908203,
      "activations/layer19_attention_weight_max": 37.791046142578125,
      "activations/layer19_attention_weight_min": -29.784658432006836,
      "activations/layer1_attention_weight_max": 17.19681739807129,
      "activations/layer1_attention_weight_min": -14.699970245361328,
      "activations/layer20_attention_weight_max": 31.97919273376465,
      "activations/layer20_attention_weight_min": -25.275039672851562,
      "activations/layer21_attention_weight_max": 30.459884643554688,
      "activations/layer21_attention_weight_min": -23.28643035888672,
      "activations/layer22_attention_weight_max": 50.84149169921875,
      "activations/layer22_attention_weight_min": -29.30345916748047,
      "activations/layer23_attention_weight_max": 36.74446105957031,
      "activations/layer23_attention_weight_min": -22.475448608398438,
      "activations/layer2_attention_weight_max": 32.127193450927734,
      "activations/layer2_attention_weight_min": -31.14754867553711,
      "activations/layer3_attention_weight_max": 94.99181365966797,
      "activations/layer3_attention_weight_min": -95.22793579101562,
      "activations/layer4_attention_weight_max": 126.46369934082031,
      "activations/layer4_attention_weight_min": -118.2401351928711,
      "activations/layer5_attention_weight_max": 64.71880340576172,
      "activations/layer5_attention_weight_min": -62.535888671875,
      "activations/layer6_attention_weight_max": 50.5792121887207,
      "activations/layer6_attention_weight_min": -54.57468795776367,
      "activations/layer7_attention_weight_max": 87.58842468261719,
      "activations/layer7_attention_weight_min": -90.73944091796875,
      "activations/layer8_attention_weight_max": 41.7402229309082,
      "activations/layer8_attention_weight_min": -40.0059814453125,
      "activations/layer9_attention_weight_max": 37.768157958984375,
      "activations/layer9_attention_weight_min": -36.289798736572266,
      "epoch": 7.56,
      "learning_rate": 0.00010225984848484847,
      "loss": 2.8147,
      "step": 130150
    },
    {
      "activations/layer0_attention_weight_max": 16.839122772216797,
      "activations/layer0_attention_weight_min": -11.019590377807617,
      "activations/layer10_attention_weight_max": 34.030147552490234,
      "activations/layer10_attention_weight_min": -30.704540252685547,
      "activations/layer11_attention_weight_max": 35.57099151611328,
      "activations/layer11_attention_weight_min": -30.711185455322266,
      "activations/layer12_attention_weight_max": 25.10631561279297,
      "activations/layer12_attention_weight_min": -30.641124725341797,
      "activations/layer13_attention_weight_max": 39.62081527709961,
      "activations/layer13_attention_weight_min": -31.87200927734375,
      "activations/layer14_attention_weight_max": 54.86426544189453,
      "activations/layer14_attention_weight_min": -38.455631256103516,
      "activations/layer15_attention_weight_max": 37.731807708740234,
      "activations/layer15_attention_weight_min": -31.739933013916016,
      "activations/layer16_attention_weight_max": 34.680274963378906,
      "activations/layer16_attention_weight_min": -30.30016326904297,
      "activations/layer17_attention_weight_max": 34.23343276977539,
      "activations/layer17_attention_weight_min": -23.712739944458008,
      "activations/layer18_attention_weight_max": 31.038305282592773,
      "activations/layer18_attention_weight_min": -20.692224502563477,
      "activations/layer19_attention_weight_max": 36.82884216308594,
      "activations/layer19_attention_weight_min": -29.894826889038086,
      "activations/layer1_attention_weight_max": 17.225296020507812,
      "activations/layer1_attention_weight_min": -14.261571884155273,
      "activations/layer20_attention_weight_max": 32.67403030395508,
      "activations/layer20_attention_weight_min": -24.201208114624023,
      "activations/layer21_attention_weight_max": 31.389249801635742,
      "activations/layer21_attention_weight_min": -21.92196273803711,
      "activations/layer22_attention_weight_max": 53.989715576171875,
      "activations/layer22_attention_weight_min": -27.782848358154297,
      "activations/layer23_attention_weight_max": 36.90449142456055,
      "activations/layer23_attention_weight_min": -22.59355926513672,
      "activations/layer2_attention_weight_max": 31.445310592651367,
      "activations/layer2_attention_weight_min": -33.169708251953125,
      "activations/layer3_attention_weight_max": 92.39338684082031,
      "activations/layer3_attention_weight_min": -94.16665649414062,
      "activations/layer4_attention_weight_max": 119.8954849243164,
      "activations/layer4_attention_weight_min": -119.82279205322266,
      "activations/layer5_attention_weight_max": 60.69834518432617,
      "activations/layer5_attention_weight_min": -59.59342956542969,
      "activations/layer6_attention_weight_max": 50.55528259277344,
      "activations/layer6_attention_weight_min": -52.15983963012695,
      "activations/layer7_attention_weight_max": 90.54873657226562,
      "activations/layer7_attention_weight_min": -89.29302215576172,
      "activations/layer8_attention_weight_max": 41.76689910888672,
      "activations/layer8_attention_weight_min": -39.692142486572266,
      "activations/layer9_attention_weight_max": 38.31920623779297,
      "activations/layer9_attention_weight_min": -35.89825439453125,
      "epoch": 7.57,
      "learning_rate": 0.00010224090909090909,
      "loss": 2.8244,
      "step": 130200
    },
    {
      "activations/layer0_attention_weight_max": 16.6297550201416,
      "activations/layer0_attention_weight_min": -12.172916412353516,
      "activations/layer10_attention_weight_max": 30.722795486450195,
      "activations/layer10_attention_weight_min": -31.40534019470215,
      "activations/layer11_attention_weight_max": 30.96263885498047,
      "activations/layer11_attention_weight_min": -28.66193199157715,
      "activations/layer12_attention_weight_max": 26.466537475585938,
      "activations/layer12_attention_weight_min": -26.823219299316406,
      "activations/layer13_attention_weight_max": 37.04343795776367,
      "activations/layer13_attention_weight_min": -35.4976806640625,
      "activations/layer14_attention_weight_max": 50.612937927246094,
      "activations/layer14_attention_weight_min": -42.3346061706543,
      "activations/layer15_attention_weight_max": 32.40638732910156,
      "activations/layer15_attention_weight_min": -28.633047103881836,
      "activations/layer16_attention_weight_max": 32.55122756958008,
      "activations/layer16_attention_weight_min": -26.756237030029297,
      "activations/layer17_attention_weight_max": 30.84717559814453,
      "activations/layer17_attention_weight_min": -24.23782730102539,
      "activations/layer18_attention_weight_max": 28.06688117980957,
      "activations/layer18_attention_weight_min": -20.448423385620117,
      "activations/layer19_attention_weight_max": 34.64085388183594,
      "activations/layer19_attention_weight_min": -28.759580612182617,
      "activations/layer1_attention_weight_max": 16.517024993896484,
      "activations/layer1_attention_weight_min": -15.990449905395508,
      "activations/layer20_attention_weight_max": 29.33110237121582,
      "activations/layer20_attention_weight_min": -23.47956085205078,
      "activations/layer21_attention_weight_max": 27.49807357788086,
      "activations/layer21_attention_weight_min": -22.04256248474121,
      "activations/layer22_attention_weight_max": 41.646820068359375,
      "activations/layer22_attention_weight_min": -27.971038818359375,
      "activations/layer23_attention_weight_max": 32.121089935302734,
      "activations/layer23_attention_weight_min": -23.457439422607422,
      "activations/layer2_attention_weight_max": 32.05831527709961,
      "activations/layer2_attention_weight_min": -31.0760555267334,
      "activations/layer3_attention_weight_max": 88.25286102294922,
      "activations/layer3_attention_weight_min": -91.33841705322266,
      "activations/layer4_attention_weight_max": 121.72740936279297,
      "activations/layer4_attention_weight_min": -113.8221664428711,
      "activations/layer5_attention_weight_max": 59.7550048828125,
      "activations/layer5_attention_weight_min": -60.852073669433594,
      "activations/layer6_attention_weight_max": 48.001216888427734,
      "activations/layer6_attention_weight_min": -50.26737976074219,
      "activations/layer7_attention_weight_max": 80.42881774902344,
      "activations/layer7_attention_weight_min": -81.29095458984375,
      "activations/layer8_attention_weight_max": 37.08346176147461,
      "activations/layer8_attention_weight_min": -38.885555267333984,
      "activations/layer9_attention_weight_max": 33.7789306640625,
      "activations/layer9_attention_weight_min": -33.537353515625,
      "epoch": 7.57,
      "learning_rate": 0.00010222196969696969,
      "loss": 2.8033,
      "step": 130250
    },
    {
      "activations/layer0_attention_weight_max": 16.555418014526367,
      "activations/layer0_attention_weight_min": -11.161869049072266,
      "activations/layer10_attention_weight_max": 32.11471176147461,
      "activations/layer10_attention_weight_min": -31.80769920349121,
      "activations/layer11_attention_weight_max": 29.557262420654297,
      "activations/layer11_attention_weight_min": -28.825313568115234,
      "activations/layer12_attention_weight_max": 27.827009201049805,
      "activations/layer12_attention_weight_min": -27.37034797668457,
      "activations/layer13_attention_weight_max": 35.76044464111328,
      "activations/layer13_attention_weight_min": -32.22773742675781,
      "activations/layer14_attention_weight_max": 42.720008850097656,
      "activations/layer14_attention_weight_min": -38.49318313598633,
      "activations/layer15_attention_weight_max": 33.3281135559082,
      "activations/layer15_attention_weight_min": -28.8843936920166,
      "activations/layer16_attention_weight_max": 31.75738525390625,
      "activations/layer16_attention_weight_min": -26.275054931640625,
      "activations/layer17_attention_weight_max": 31.264596939086914,
      "activations/layer17_attention_weight_min": -23.747657775878906,
      "activations/layer18_attention_weight_max": 30.29216957092285,
      "activations/layer18_attention_weight_min": -20.99485969543457,
      "activations/layer19_attention_weight_max": 34.690330505371094,
      "activations/layer19_attention_weight_min": -30.27878761291504,
      "activations/layer1_attention_weight_max": 16.8424072265625,
      "activations/layer1_attention_weight_min": -16.50324249267578,
      "activations/layer20_attention_weight_max": 29.82436752319336,
      "activations/layer20_attention_weight_min": -24.057247161865234,
      "activations/layer21_attention_weight_max": 29.865562438964844,
      "activations/layer21_attention_weight_min": -21.856155395507812,
      "activations/layer22_attention_weight_max": 43.01300811767578,
      "activations/layer22_attention_weight_min": -29.96871566772461,
      "activations/layer23_attention_weight_max": 33.88151550292969,
      "activations/layer23_attention_weight_min": -24.59564781188965,
      "activations/layer2_attention_weight_max": 31.253108978271484,
      "activations/layer2_attention_weight_min": -30.733495712280273,
      "activations/layer3_attention_weight_max": 87.26850891113281,
      "activations/layer3_attention_weight_min": -87.1186294555664,
      "activations/layer4_attention_weight_max": 118.07320404052734,
      "activations/layer4_attention_weight_min": -114.2061996459961,
      "activations/layer5_attention_weight_max": 61.89984130859375,
      "activations/layer5_attention_weight_min": -64.30290985107422,
      "activations/layer6_attention_weight_max": 51.96399688720703,
      "activations/layer6_attention_weight_min": -54.26696014404297,
      "activations/layer7_attention_weight_max": 89.69577026367188,
      "activations/layer7_attention_weight_min": -92.28797912597656,
      "activations/layer8_attention_weight_max": 37.74259567260742,
      "activations/layer8_attention_weight_min": -40.14079666137695,
      "activations/layer9_attention_weight_max": 33.43943405151367,
      "activations/layer9_attention_weight_min": -32.0560417175293,
      "epoch": 7.57,
      "learning_rate": 0.00010220303030303029,
      "loss": 2.8426,
      "step": 130300
    },
    {
      "activations/layer0_attention_weight_max": 16.189281463623047,
      "activations/layer0_attention_weight_min": -12.059803009033203,
      "activations/layer10_attention_weight_max": 34.2949104309082,
      "activations/layer10_attention_weight_min": -37.75840377807617,
      "activations/layer11_attention_weight_max": 37.03935241699219,
      "activations/layer11_attention_weight_min": -34.82545852661133,
      "activations/layer12_attention_weight_max": 24.126859664916992,
      "activations/layer12_attention_weight_min": -30.139429092407227,
      "activations/layer13_attention_weight_max": 49.17903518676758,
      "activations/layer13_attention_weight_min": -36.36031723022461,
      "activations/layer14_attention_weight_max": 62.4951286315918,
      "activations/layer14_attention_weight_min": -46.58100891113281,
      "activations/layer15_attention_weight_max": 39.566795349121094,
      "activations/layer15_attention_weight_min": -31.351490020751953,
      "activations/layer16_attention_weight_max": 40.8483772277832,
      "activations/layer16_attention_weight_min": -27.1785831451416,
      "activations/layer17_attention_weight_max": 41.73774719238281,
      "activations/layer17_attention_weight_min": -25.161237716674805,
      "activations/layer18_attention_weight_max": 32.09630584716797,
      "activations/layer18_attention_weight_min": -22.626585006713867,
      "activations/layer19_attention_weight_max": 43.69144821166992,
      "activations/layer19_attention_weight_min": -31.27382469177246,
      "activations/layer1_attention_weight_max": 16.412860870361328,
      "activations/layer1_attention_weight_min": -11.5408296585083,
      "activations/layer20_attention_weight_max": 38.8472785949707,
      "activations/layer20_attention_weight_min": -23.841550827026367,
      "activations/layer21_attention_weight_max": 33.83788299560547,
      "activations/layer21_attention_weight_min": -23.60446548461914,
      "activations/layer22_attention_weight_max": 52.11967849731445,
      "activations/layer22_attention_weight_min": -29.4928035736084,
      "activations/layer23_attention_weight_max": 40.4625358581543,
      "activations/layer23_attention_weight_min": -24.529953002929688,
      "activations/layer2_attention_weight_max": 29.30118179321289,
      "activations/layer2_attention_weight_min": -29.323223114013672,
      "activations/layer3_attention_weight_max": 95.2806625366211,
      "activations/layer3_attention_weight_min": -92.06372833251953,
      "activations/layer4_attention_weight_max": 135.47702026367188,
      "activations/layer4_attention_weight_min": -119.276123046875,
      "activations/layer5_attention_weight_max": 63.149410247802734,
      "activations/layer5_attention_weight_min": -60.5305290222168,
      "activations/layer6_attention_weight_max": 49.346736907958984,
      "activations/layer6_attention_weight_min": -51.18324661254883,
      "activations/layer7_attention_weight_max": 87.84673309326172,
      "activations/layer7_attention_weight_min": -85.80154418945312,
      "activations/layer8_attention_weight_max": 41.76993942260742,
      "activations/layer8_attention_weight_min": -45.33687973022461,
      "activations/layer9_attention_weight_max": 35.663082122802734,
      "activations/layer9_attention_weight_min": -37.76278305053711,
      "epoch": 7.57,
      "learning_rate": 0.00010218409090909089,
      "loss": 2.8143,
      "step": 130350
    },
    {
      "activations/layer0_attention_weight_max": 16.331159591674805,
      "activations/layer0_attention_weight_min": -12.7597017288208,
      "activations/layer10_attention_weight_max": 33.508724212646484,
      "activations/layer10_attention_weight_min": -34.286041259765625,
      "activations/layer11_attention_weight_max": 35.108543395996094,
      "activations/layer11_attention_weight_min": -30.895721435546875,
      "activations/layer12_attention_weight_max": 25.70006561279297,
      "activations/layer12_attention_weight_min": -24.971879959106445,
      "activations/layer13_attention_weight_max": 42.14221954345703,
      "activations/layer13_attention_weight_min": -35.02932357788086,
      "activations/layer14_attention_weight_max": 56.89333724975586,
      "activations/layer14_attention_weight_min": -42.03302764892578,
      "activations/layer15_attention_weight_max": 39.53955841064453,
      "activations/layer15_attention_weight_min": -30.47062110900879,
      "activations/layer16_attention_weight_max": 36.98514175415039,
      "activations/layer16_attention_weight_min": -29.222158432006836,
      "activations/layer17_attention_weight_max": 36.619693756103516,
      "activations/layer17_attention_weight_min": -26.109405517578125,
      "activations/layer18_attention_weight_max": 31.313316345214844,
      "activations/layer18_attention_weight_min": -21.42057228088379,
      "activations/layer19_attention_weight_max": 38.957637786865234,
      "activations/layer19_attention_weight_min": -32.18781280517578,
      "activations/layer1_attention_weight_max": 18.675708770751953,
      "activations/layer1_attention_weight_min": -12.724089622497559,
      "activations/layer20_attention_weight_max": 32.16950988769531,
      "activations/layer20_attention_weight_min": -27.637224197387695,
      "activations/layer21_attention_weight_max": 31.971799850463867,
      "activations/layer21_attention_weight_min": -23.411985397338867,
      "activations/layer22_attention_weight_max": 48.6202392578125,
      "activations/layer22_attention_weight_min": -30.47414207458496,
      "activations/layer23_attention_weight_max": 38.673072814941406,
      "activations/layer23_attention_weight_min": -25.352699279785156,
      "activations/layer2_attention_weight_max": 31.101181030273438,
      "activations/layer2_attention_weight_min": -30.684301376342773,
      "activations/layer3_attention_weight_max": 88.74918365478516,
      "activations/layer3_attention_weight_min": -89.9927978515625,
      "activations/layer4_attention_weight_max": 121.93904876708984,
      "activations/layer4_attention_weight_min": -118.53131103515625,
      "activations/layer5_attention_weight_max": 61.05681610107422,
      "activations/layer5_attention_weight_min": -64.78794860839844,
      "activations/layer6_attention_weight_max": 52.07847595214844,
      "activations/layer6_attention_weight_min": -55.75236511230469,
      "activations/layer7_attention_weight_max": 90.86341857910156,
      "activations/layer7_attention_weight_min": -94.53610229492188,
      "activations/layer8_attention_weight_max": 41.39506912231445,
      "activations/layer8_attention_weight_min": -46.63178634643555,
      "activations/layer9_attention_weight_max": 36.88300323486328,
      "activations/layer9_attention_weight_min": -35.582740783691406,
      "epoch": 7.58,
      "learning_rate": 0.00010216515151515151,
      "loss": 2.8339,
      "step": 130400
    },
    {
      "activations/layer0_attention_weight_max": 15.62928295135498,
      "activations/layer0_attention_weight_min": -11.335728645324707,
      "activations/layer10_attention_weight_max": 32.865966796875,
      "activations/layer10_attention_weight_min": -32.95749282836914,
      "activations/layer11_attention_weight_max": 30.928775787353516,
      "activations/layer11_attention_weight_min": -28.857290267944336,
      "activations/layer12_attention_weight_max": 22.0695858001709,
      "activations/layer12_attention_weight_min": -24.285520553588867,
      "activations/layer13_attention_weight_max": 38.03292465209961,
      "activations/layer13_attention_weight_min": -32.854774475097656,
      "activations/layer14_attention_weight_max": 46.63724136352539,
      "activations/layer14_attention_weight_min": -38.371002197265625,
      "activations/layer15_attention_weight_max": 36.510772705078125,
      "activations/layer15_attention_weight_min": -31.726764678955078,
      "activations/layer16_attention_weight_max": 30.69814109802246,
      "activations/layer16_attention_weight_min": -26.7849178314209,
      "activations/layer17_attention_weight_max": 36.083927154541016,
      "activations/layer17_attention_weight_min": -24.81174659729004,
      "activations/layer18_attention_weight_max": 31.0206241607666,
      "activations/layer18_attention_weight_min": -22.33006477355957,
      "activations/layer19_attention_weight_max": 35.0416145324707,
      "activations/layer19_attention_weight_min": -32.54594421386719,
      "activations/layer1_attention_weight_max": 20.125009536743164,
      "activations/layer1_attention_weight_min": -15.490344047546387,
      "activations/layer20_attention_weight_max": 29.8949031829834,
      "activations/layer20_attention_weight_min": -27.070226669311523,
      "activations/layer21_attention_weight_max": 28.117692947387695,
      "activations/layer21_attention_weight_min": -23.55661964416504,
      "activations/layer22_attention_weight_max": 46.18437957763672,
      "activations/layer22_attention_weight_min": -31.004438400268555,
      "activations/layer23_attention_weight_max": 35.443397521972656,
      "activations/layer23_attention_weight_min": -24.870548248291016,
      "activations/layer2_attention_weight_max": 31.031082153320312,
      "activations/layer2_attention_weight_min": -30.981281280517578,
      "activations/layer3_attention_weight_max": 96.43721008300781,
      "activations/layer3_attention_weight_min": -95.40902709960938,
      "activations/layer4_attention_weight_max": 122.12220764160156,
      "activations/layer4_attention_weight_min": -115.74012756347656,
      "activations/layer5_attention_weight_max": 60.596946716308594,
      "activations/layer5_attention_weight_min": -63.232933044433594,
      "activations/layer6_attention_weight_max": 49.93574142456055,
      "activations/layer6_attention_weight_min": -51.541324615478516,
      "activations/layer7_attention_weight_max": 86.38844299316406,
      "activations/layer7_attention_weight_min": -87.10872650146484,
      "activations/layer8_attention_weight_max": 38.06798553466797,
      "activations/layer8_attention_weight_min": -40.067474365234375,
      "activations/layer9_attention_weight_max": 33.66545486450195,
      "activations/layer9_attention_weight_min": -32.61688232421875,
      "epoch": 7.58,
      "learning_rate": 0.00010214621212121212,
      "loss": 2.8111,
      "step": 130450
    },
    {
      "activations/layer0_attention_weight_max": 16.46860122680664,
      "activations/layer0_attention_weight_min": -10.17442512512207,
      "activations/layer10_attention_weight_max": 30.955228805541992,
      "activations/layer10_attention_weight_min": -31.953392028808594,
      "activations/layer11_attention_weight_max": 30.825332641601562,
      "activations/layer11_attention_weight_min": -29.66748046875,
      "activations/layer12_attention_weight_max": 21.594560623168945,
      "activations/layer12_attention_weight_min": -24.907875061035156,
      "activations/layer13_attention_weight_max": 36.463783264160156,
      "activations/layer13_attention_weight_min": -32.07262420654297,
      "activations/layer14_attention_weight_max": 51.39637756347656,
      "activations/layer14_attention_weight_min": -40.64994812011719,
      "activations/layer15_attention_weight_max": 35.73566818237305,
      "activations/layer15_attention_weight_min": -29.46565818786621,
      "activations/layer16_attention_weight_max": 30.74513816833496,
      "activations/layer16_attention_weight_min": -27.2441463470459,
      "activations/layer17_attention_weight_max": 33.56618881225586,
      "activations/layer17_attention_weight_min": -25.147001266479492,
      "activations/layer18_attention_weight_max": 31.559640884399414,
      "activations/layer18_attention_weight_min": -21.08869743347168,
      "activations/layer19_attention_weight_max": 35.36985397338867,
      "activations/layer19_attention_weight_min": -28.84425163269043,
      "activations/layer1_attention_weight_max": 16.485904693603516,
      "activations/layer1_attention_weight_min": -13.842555046081543,
      "activations/layer20_attention_weight_max": 33.73451614379883,
      "activations/layer20_attention_weight_min": -23.866649627685547,
      "activations/layer21_attention_weight_max": 31.021724700927734,
      "activations/layer21_attention_weight_min": -22.316890716552734,
      "activations/layer22_attention_weight_max": 43.80270767211914,
      "activations/layer22_attention_weight_min": -28.797653198242188,
      "activations/layer23_attention_weight_max": 30.56243896484375,
      "activations/layer23_attention_weight_min": -23.985820770263672,
      "activations/layer2_attention_weight_max": 30.85304069519043,
      "activations/layer2_attention_weight_min": -30.017059326171875,
      "activations/layer3_attention_weight_max": 93.51273345947266,
      "activations/layer3_attention_weight_min": -92.9956283569336,
      "activations/layer4_attention_weight_max": 122.46661376953125,
      "activations/layer4_attention_weight_min": -119.84867095947266,
      "activations/layer5_attention_weight_max": 62.53514862060547,
      "activations/layer5_attention_weight_min": -63.34998321533203,
      "activations/layer6_attention_weight_max": 53.030765533447266,
      "activations/layer6_attention_weight_min": -54.2209587097168,
      "activations/layer7_attention_weight_max": 89.15058898925781,
      "activations/layer7_attention_weight_min": -90.70206451416016,
      "activations/layer8_attention_weight_max": 38.809940338134766,
      "activations/layer8_attention_weight_min": -38.69821548461914,
      "activations/layer9_attention_weight_max": 31.29827880859375,
      "activations/layer9_attention_weight_min": -34.23502731323242,
      "epoch": 7.58,
      "learning_rate": 0.00010212727272727271,
      "loss": 2.8129,
      "step": 130500
    },
    {
      "activations/layer0_attention_weight_max": 16.051218032836914,
      "activations/layer0_attention_weight_min": -12.368818283081055,
      "activations/layer10_attention_weight_max": 32.28581237792969,
      "activations/layer10_attention_weight_min": -31.53990364074707,
      "activations/layer11_attention_weight_max": 31.12891387939453,
      "activations/layer11_attention_weight_min": -28.213993072509766,
      "activations/layer12_attention_weight_max": 25.657520294189453,
      "activations/layer12_attention_weight_min": -26.334077835083008,
      "activations/layer13_attention_weight_max": 39.61677169799805,
      "activations/layer13_attention_weight_min": -32.328182220458984,
      "activations/layer14_attention_weight_max": 49.352901458740234,
      "activations/layer14_attention_weight_min": -39.28750991821289,
      "activations/layer15_attention_weight_max": 37.016475677490234,
      "activations/layer15_attention_weight_min": -29.39918327331543,
      "activations/layer16_attention_weight_max": 34.84566116333008,
      "activations/layer16_attention_weight_min": -27.73295021057129,
      "activations/layer17_attention_weight_max": 35.5142707824707,
      "activations/layer17_attention_weight_min": -24.417911529541016,
      "activations/layer18_attention_weight_max": 36.064876556396484,
      "activations/layer18_attention_weight_min": -21.12604522705078,
      "activations/layer19_attention_weight_max": 36.715946197509766,
      "activations/layer19_attention_weight_min": -29.716583251953125,
      "activations/layer1_attention_weight_max": 16.506940841674805,
      "activations/layer1_attention_weight_min": -14.516667366027832,
      "activations/layer20_attention_weight_max": 33.10331344604492,
      "activations/layer20_attention_weight_min": -25.824018478393555,
      "activations/layer21_attention_weight_max": 32.17457962036133,
      "activations/layer21_attention_weight_min": -21.99432945251465,
      "activations/layer22_attention_weight_max": 46.93782043457031,
      "activations/layer22_attention_weight_min": -27.86986541748047,
      "activations/layer23_attention_weight_max": 34.67188262939453,
      "activations/layer23_attention_weight_min": -22.103979110717773,
      "activations/layer2_attention_weight_max": 31.266559600830078,
      "activations/layer2_attention_weight_min": -30.913612365722656,
      "activations/layer3_attention_weight_max": 89.10870361328125,
      "activations/layer3_attention_weight_min": -89.90999603271484,
      "activations/layer4_attention_weight_max": 118.32714080810547,
      "activations/layer4_attention_weight_min": -114.06819915771484,
      "activations/layer5_attention_weight_max": 58.03124237060547,
      "activations/layer5_attention_weight_min": -61.901275634765625,
      "activations/layer6_attention_weight_max": 47.86099624633789,
      "activations/layer6_attention_weight_min": -51.11909103393555,
      "activations/layer7_attention_weight_max": 81.54657745361328,
      "activations/layer7_attention_weight_min": -84.08418273925781,
      "activations/layer8_attention_weight_max": 40.1845817565918,
      "activations/layer8_attention_weight_min": -36.81709671020508,
      "activations/layer9_attention_weight_max": 32.86243438720703,
      "activations/layer9_attention_weight_min": -32.40009307861328,
      "epoch": 7.59,
      "learning_rate": 0.00010210833333333333,
      "loss": 2.8189,
      "step": 130550
    },
    {
      "activations/layer0_attention_weight_max": 17.09469985961914,
      "activations/layer0_attention_weight_min": -11.484745025634766,
      "activations/layer10_attention_weight_max": 33.164642333984375,
      "activations/layer10_attention_weight_min": -32.4344482421875,
      "activations/layer11_attention_weight_max": 30.609481811523438,
      "activations/layer11_attention_weight_min": -27.548954010009766,
      "activations/layer12_attention_weight_max": 21.944580078125,
      "activations/layer12_attention_weight_min": -37.27614974975586,
      "activations/layer13_attention_weight_max": 38.1217155456543,
      "activations/layer13_attention_weight_min": -30.674327850341797,
      "activations/layer14_attention_weight_max": 44.70187759399414,
      "activations/layer14_attention_weight_min": -38.66329574584961,
      "activations/layer15_attention_weight_max": 31.65570068359375,
      "activations/layer15_attention_weight_min": -30.683910369873047,
      "activations/layer16_attention_weight_max": 30.968530654907227,
      "activations/layer16_attention_weight_min": -30.60591697692871,
      "activations/layer17_attention_weight_max": 32.431732177734375,
      "activations/layer17_attention_weight_min": -26.478059768676758,
      "activations/layer18_attention_weight_max": 31.407527923583984,
      "activations/layer18_attention_weight_min": -21.288898468017578,
      "activations/layer19_attention_weight_max": 34.6395263671875,
      "activations/layer19_attention_weight_min": -37.76775360107422,
      "activations/layer1_attention_weight_max": 16.0773983001709,
      "activations/layer1_attention_weight_min": -13.5977201461792,
      "activations/layer20_attention_weight_max": 30.833045959472656,
      "activations/layer20_attention_weight_min": -29.872285842895508,
      "activations/layer21_attention_weight_max": 34.32896423339844,
      "activations/layer21_attention_weight_min": -28.58498191833496,
      "activations/layer22_attention_weight_max": 44.26740264892578,
      "activations/layer22_attention_weight_min": -29.44864273071289,
      "activations/layer23_attention_weight_max": 33.98018264770508,
      "activations/layer23_attention_weight_min": -22.565399169921875,
      "activations/layer2_attention_weight_max": 32.56952667236328,
      "activations/layer2_attention_weight_min": -30.571691513061523,
      "activations/layer3_attention_weight_max": 92.76156616210938,
      "activations/layer3_attention_weight_min": -93.55270385742188,
      "activations/layer4_attention_weight_max": 126.81737518310547,
      "activations/layer4_attention_weight_min": -116.94561004638672,
      "activations/layer5_attention_weight_max": 60.47665023803711,
      "activations/layer5_attention_weight_min": -60.835716247558594,
      "activations/layer6_attention_weight_max": 54.26874923706055,
      "activations/layer6_attention_weight_min": -51.91151809692383,
      "activations/layer7_attention_weight_max": 92.8278579711914,
      "activations/layer7_attention_weight_min": -87.5933609008789,
      "activations/layer8_attention_weight_max": 41.6917724609375,
      "activations/layer8_attention_weight_min": -40.253170013427734,
      "activations/layer9_attention_weight_max": 37.32371139526367,
      "activations/layer9_attention_weight_min": -34.84767532348633,
      "epoch": 7.59,
      "learning_rate": 0.00010208939393939394,
      "loss": 2.8138,
      "step": 130600
    },
    {
      "activations/layer0_attention_weight_max": 16.02447509765625,
      "activations/layer0_attention_weight_min": -11.5567045211792,
      "activations/layer10_attention_weight_max": 34.14338684082031,
      "activations/layer10_attention_weight_min": -32.671546936035156,
      "activations/layer11_attention_weight_max": 32.80561447143555,
      "activations/layer11_attention_weight_min": -30.537450790405273,
      "activations/layer12_attention_weight_max": 29.610897064208984,
      "activations/layer12_attention_weight_min": -24.526365280151367,
      "activations/layer13_attention_weight_max": 38.41816711425781,
      "activations/layer13_attention_weight_min": -31.979951858520508,
      "activations/layer14_attention_weight_max": 50.3800163269043,
      "activations/layer14_attention_weight_min": -39.22792434692383,
      "activations/layer15_attention_weight_max": 35.84251022338867,
      "activations/layer15_attention_weight_min": -29.937129974365234,
      "activations/layer16_attention_weight_max": 29.778493881225586,
      "activations/layer16_attention_weight_min": -27.04917335510254,
      "activations/layer17_attention_weight_max": 32.99717712402344,
      "activations/layer17_attention_weight_min": -26.07819175720215,
      "activations/layer18_attention_weight_max": 31.013582229614258,
      "activations/layer18_attention_weight_min": -22.82196617126465,
      "activations/layer19_attention_weight_max": 40.53571701049805,
      "activations/layer19_attention_weight_min": -29.758869171142578,
      "activations/layer1_attention_weight_max": 17.645475387573242,
      "activations/layer1_attention_weight_min": -14.125109672546387,
      "activations/layer20_attention_weight_max": 29.945415496826172,
      "activations/layer20_attention_weight_min": -24.618669509887695,
      "activations/layer21_attention_weight_max": 30.039676666259766,
      "activations/layer21_attention_weight_min": -23.68085289001465,
      "activations/layer22_attention_weight_max": 50.29862594604492,
      "activations/layer22_attention_weight_min": -29.20168685913086,
      "activations/layer23_attention_weight_max": 38.744346618652344,
      "activations/layer23_attention_weight_min": -22.68891143798828,
      "activations/layer2_attention_weight_max": 31.253042221069336,
      "activations/layer2_attention_weight_min": -30.294082641601562,
      "activations/layer3_attention_weight_max": 92.85737609863281,
      "activations/layer3_attention_weight_min": -91.1265640258789,
      "activations/layer4_attention_weight_max": 127.3542709350586,
      "activations/layer4_attention_weight_min": -117.60192108154297,
      "activations/layer5_attention_weight_max": 65.5206527709961,
      "activations/layer5_attention_weight_min": -58.81690216064453,
      "activations/layer6_attention_weight_max": 50.94395065307617,
      "activations/layer6_attention_weight_min": -52.33224105834961,
      "activations/layer7_attention_weight_max": 88.09841918945312,
      "activations/layer7_attention_weight_min": -83.79949951171875,
      "activations/layer8_attention_weight_max": 41.415443420410156,
      "activations/layer8_attention_weight_min": -41.0629768371582,
      "activations/layer9_attention_weight_max": 38.241798400878906,
      "activations/layer9_attention_weight_min": -35.571571350097656,
      "epoch": 7.59,
      "learning_rate": 0.00010207083333333333,
      "loss": 2.7995,
      "step": 130650
    },
    {
      "activations/layer0_attention_weight_max": 17.12135124206543,
      "activations/layer0_attention_weight_min": -12.405997276306152,
      "activations/layer10_attention_weight_max": 37.13270568847656,
      "activations/layer10_attention_weight_min": -31.95004653930664,
      "activations/layer11_attention_weight_max": 40.1350212097168,
      "activations/layer11_attention_weight_min": -30.90559196472168,
      "activations/layer12_attention_weight_max": 23.633371353149414,
      "activations/layer12_attention_weight_min": -27.59429359436035,
      "activations/layer13_attention_weight_max": 41.949729919433594,
      "activations/layer13_attention_weight_min": -34.674285888671875,
      "activations/layer14_attention_weight_max": 49.07209014892578,
      "activations/layer14_attention_weight_min": -40.59409713745117,
      "activations/layer15_attention_weight_max": 39.92741775512695,
      "activations/layer15_attention_weight_min": -32.83449172973633,
      "activations/layer16_attention_weight_max": 33.26376724243164,
      "activations/layer16_attention_weight_min": -29.118024826049805,
      "activations/layer17_attention_weight_max": 33.80311584472656,
      "activations/layer17_attention_weight_min": -27.239084243774414,
      "activations/layer18_attention_weight_max": 30.863937377929688,
      "activations/layer18_attention_weight_min": -21.779964447021484,
      "activations/layer19_attention_weight_max": 34.0994987487793,
      "activations/layer19_attention_weight_min": -31.322540283203125,
      "activations/layer1_attention_weight_max": 17.241769790649414,
      "activations/layer1_attention_weight_min": -14.061013221740723,
      "activations/layer20_attention_weight_max": 30.154129028320312,
      "activations/layer20_attention_weight_min": -25.39431381225586,
      "activations/layer21_attention_weight_max": 31.42015838623047,
      "activations/layer21_attention_weight_min": -21.732372283935547,
      "activations/layer22_attention_weight_max": 43.84449005126953,
      "activations/layer22_attention_weight_min": -32.01519775390625,
      "activations/layer23_attention_weight_max": 32.17020034790039,
      "activations/layer23_attention_weight_min": -25.630722045898438,
      "activations/layer2_attention_weight_max": 30.00881004333496,
      "activations/layer2_attention_weight_min": -29.456510543823242,
      "activations/layer3_attention_weight_max": 89.302734375,
      "activations/layer3_attention_weight_min": -89.88787841796875,
      "activations/layer4_attention_weight_max": 122.69876861572266,
      "activations/layer4_attention_weight_min": -120.38829803466797,
      "activations/layer5_attention_weight_max": 62.556427001953125,
      "activations/layer5_attention_weight_min": -68.17437744140625,
      "activations/layer6_attention_weight_max": 47.307594299316406,
      "activations/layer6_attention_weight_min": -54.63723373413086,
      "activations/layer7_attention_weight_max": 90.18498992919922,
      "activations/layer7_attention_weight_min": -86.67303466796875,
      "activations/layer8_attention_weight_max": 43.38901138305664,
      "activations/layer8_attention_weight_min": -40.23605728149414,
      "activations/layer9_attention_weight_max": 37.9558219909668,
      "activations/layer9_attention_weight_min": -33.87131118774414,
      "epoch": 7.59,
      "learning_rate": 0.00010205189393939392,
      "loss": 2.8214,
      "step": 130700
    },
    {
      "activations/layer0_attention_weight_max": 16.606903076171875,
      "activations/layer0_attention_weight_min": -11.30337142944336,
      "activations/layer10_attention_weight_max": 31.60260009765625,
      "activations/layer10_attention_weight_min": -30.610355377197266,
      "activations/layer11_attention_weight_max": 34.50785446166992,
      "activations/layer11_attention_weight_min": -29.102170944213867,
      "activations/layer12_attention_weight_max": 20.979785919189453,
      "activations/layer12_attention_weight_min": -27.613162994384766,
      "activations/layer13_attention_weight_max": 44.04600143432617,
      "activations/layer13_attention_weight_min": -33.382652282714844,
      "activations/layer14_attention_weight_max": 47.55433654785156,
      "activations/layer14_attention_weight_min": -44.02854537963867,
      "activations/layer15_attention_weight_max": 35.68506622314453,
      "activations/layer15_attention_weight_min": -29.704490661621094,
      "activations/layer16_attention_weight_max": 30.025360107421875,
      "activations/layer16_attention_weight_min": -28.08411979675293,
      "activations/layer17_attention_weight_max": 36.42399597167969,
      "activations/layer17_attention_weight_min": -26.39037322998047,
      "activations/layer18_attention_weight_max": 31.082304000854492,
      "activations/layer18_attention_weight_min": -20.39020347595215,
      "activations/layer19_attention_weight_max": 36.76818084716797,
      "activations/layer19_attention_weight_min": -31.745697021484375,
      "activations/layer1_attention_weight_max": 16.947002410888672,
      "activations/layer1_attention_weight_min": -12.613268852233887,
      "activations/layer20_attention_weight_max": 30.707706451416016,
      "activations/layer20_attention_weight_min": -26.18914794921875,
      "activations/layer21_attention_weight_max": 27.520017623901367,
      "activations/layer21_attention_weight_min": -22.671466827392578,
      "activations/layer22_attention_weight_max": 45.0194091796875,
      "activations/layer22_attention_weight_min": -28.19999122619629,
      "activations/layer23_attention_weight_max": 34.45232391357422,
      "activations/layer23_attention_weight_min": -23.927719116210938,
      "activations/layer2_attention_weight_max": 30.462352752685547,
      "activations/layer2_attention_weight_min": -29.129356384277344,
      "activations/layer3_attention_weight_max": 87.32186126708984,
      "activations/layer3_attention_weight_min": -90.37850189208984,
      "activations/layer4_attention_weight_max": 117.60733795166016,
      "activations/layer4_attention_weight_min": -115.4967041015625,
      "activations/layer5_attention_weight_max": 59.07647705078125,
      "activations/layer5_attention_weight_min": -62.604705810546875,
      "activations/layer6_attention_weight_max": 48.231040954589844,
      "activations/layer6_attention_weight_min": -49.56313705444336,
      "activations/layer7_attention_weight_max": 84.77093505859375,
      "activations/layer7_attention_weight_min": -83.66156768798828,
      "activations/layer8_attention_weight_max": 41.631439208984375,
      "activations/layer8_attention_weight_min": -37.010494232177734,
      "activations/layer9_attention_weight_max": 34.346012115478516,
      "activations/layer9_attention_weight_min": -32.24696350097656,
      "epoch": 7.6,
      "learning_rate": 0.00010203295454545453,
      "loss": 2.8229,
      "step": 130750
    },
    {
      "activations/layer0_attention_weight_max": 15.638593673706055,
      "activations/layer0_attention_weight_min": -12.309036254882812,
      "activations/layer10_attention_weight_max": 36.28011703491211,
      "activations/layer10_attention_weight_min": -32.92658233642578,
      "activations/layer11_attention_weight_max": 35.088829040527344,
      "activations/layer11_attention_weight_min": -30.50910186767578,
      "activations/layer12_attention_weight_max": 24.80057144165039,
      "activations/layer12_attention_weight_min": -24.890729904174805,
      "activations/layer13_attention_weight_max": 42.5657844543457,
      "activations/layer13_attention_weight_min": -32.40058517456055,
      "activations/layer14_attention_weight_max": 44.69135665893555,
      "activations/layer14_attention_weight_min": -36.08196258544922,
      "activations/layer15_attention_weight_max": 38.3109245300293,
      "activations/layer15_attention_weight_min": -29.652345657348633,
      "activations/layer16_attention_weight_max": 35.06875228881836,
      "activations/layer16_attention_weight_min": -28.0580997467041,
      "activations/layer17_attention_weight_max": 39.98374938964844,
      "activations/layer17_attention_weight_min": -25.5762882232666,
      "activations/layer18_attention_weight_max": 36.76277542114258,
      "activations/layer18_attention_weight_min": -22.062162399291992,
      "activations/layer19_attention_weight_max": 39.42643356323242,
      "activations/layer19_attention_weight_min": -29.197494506835938,
      "activations/layer1_attention_weight_max": 16.278980255126953,
      "activations/layer1_attention_weight_min": -15.055425643920898,
      "activations/layer20_attention_weight_max": 33.38654708862305,
      "activations/layer20_attention_weight_min": -22.56099510192871,
      "activations/layer21_attention_weight_max": 30.07076072692871,
      "activations/layer21_attention_weight_min": -21.03370475769043,
      "activations/layer22_attention_weight_max": 49.90958786010742,
      "activations/layer22_attention_weight_min": -27.481090545654297,
      "activations/layer23_attention_weight_max": 40.665252685546875,
      "activations/layer23_attention_weight_min": -22.472152709960938,
      "activations/layer2_attention_weight_max": 31.257041931152344,
      "activations/layer2_attention_weight_min": -32.3992919921875,
      "activations/layer3_attention_weight_max": 89.49488830566406,
      "activations/layer3_attention_weight_min": -92.49298095703125,
      "activations/layer4_attention_weight_max": 117.01063537597656,
      "activations/layer4_attention_weight_min": -117.94377899169922,
      "activations/layer5_attention_weight_max": 60.452083587646484,
      "activations/layer5_attention_weight_min": -62.51650619506836,
      "activations/layer6_attention_weight_max": 50.05002212524414,
      "activations/layer6_attention_weight_min": -53.98176956176758,
      "activations/layer7_attention_weight_max": 84.90116119384766,
      "activations/layer7_attention_weight_min": -92.15365600585938,
      "activations/layer8_attention_weight_max": 41.284873962402344,
      "activations/layer8_attention_weight_min": -42.42341613769531,
      "activations/layer9_attention_weight_max": 36.13251495361328,
      "activations/layer9_attention_weight_min": -36.317256927490234,
      "epoch": 7.6,
      "learning_rate": 0.00010201401515151515,
      "loss": 2.815,
      "step": 130800
    },
    {
      "activations/layer0_attention_weight_max": 16.611854553222656,
      "activations/layer0_attention_weight_min": -12.195241928100586,
      "activations/layer10_attention_weight_max": 33.49625778198242,
      "activations/layer10_attention_weight_min": -31.71830940246582,
      "activations/layer11_attention_weight_max": 30.014022827148438,
      "activations/layer11_attention_weight_min": -29.13875389099121,
      "activations/layer12_attention_weight_max": 23.634923934936523,
      "activations/layer12_attention_weight_min": -24.219560623168945,
      "activations/layer13_attention_weight_max": 39.473663330078125,
      "activations/layer13_attention_weight_min": -33.24162292480469,
      "activations/layer14_attention_weight_max": 47.547969818115234,
      "activations/layer14_attention_weight_min": -37.968833923339844,
      "activations/layer15_attention_weight_max": 39.40913772583008,
      "activations/layer15_attention_weight_min": -30.0275936126709,
      "activations/layer16_attention_weight_max": 32.23965835571289,
      "activations/layer16_attention_weight_min": -26.948863983154297,
      "activations/layer17_attention_weight_max": 35.17952346801758,
      "activations/layer17_attention_weight_min": -25.578533172607422,
      "activations/layer18_attention_weight_max": 35.192256927490234,
      "activations/layer18_attention_weight_min": -21.758533477783203,
      "activations/layer19_attention_weight_max": 36.981197357177734,
      "activations/layer19_attention_weight_min": -30.353403091430664,
      "activations/layer1_attention_weight_max": 20.818012237548828,
      "activations/layer1_attention_weight_min": -16.105390548706055,
      "activations/layer20_attention_weight_max": 31.697551727294922,
      "activations/layer20_attention_weight_min": -26.750343322753906,
      "activations/layer21_attention_weight_max": 33.70876693725586,
      "activations/layer21_attention_weight_min": -24.132848739624023,
      "activations/layer22_attention_weight_max": 50.3468132019043,
      "activations/layer22_attention_weight_min": -32.47233200073242,
      "activations/layer23_attention_weight_max": 39.00361633300781,
      "activations/layer23_attention_weight_min": -26.24525260925293,
      "activations/layer2_attention_weight_max": 31.106870651245117,
      "activations/layer2_attention_weight_min": -30.54163360595703,
      "activations/layer3_attention_weight_max": 92.20572662353516,
      "activations/layer3_attention_weight_min": -93.4078369140625,
      "activations/layer4_attention_weight_max": 124.1144027709961,
      "activations/layer4_attention_weight_min": -115.41810607910156,
      "activations/layer5_attention_weight_max": 61.99674606323242,
      "activations/layer5_attention_weight_min": -61.88658142089844,
      "activations/layer6_attention_weight_max": 53.33088684082031,
      "activations/layer6_attention_weight_min": -52.877403259277344,
      "activations/layer7_attention_weight_max": 91.69078063964844,
      "activations/layer7_attention_weight_min": -90.39923095703125,
      "activations/layer8_attention_weight_max": 40.481536865234375,
      "activations/layer8_attention_weight_min": -39.10048294067383,
      "activations/layer9_attention_weight_max": 34.96882247924805,
      "activations/layer9_attention_weight_min": -35.073421478271484,
      "epoch": 7.6,
      "learning_rate": 0.00010199507575757575,
      "loss": 2.8308,
      "step": 130850
    },
    {
      "activations/layer0_attention_weight_max": 16.004323959350586,
      "activations/layer0_attention_weight_min": -11.606764793395996,
      "activations/layer10_attention_weight_max": 34.64997100830078,
      "activations/layer10_attention_weight_min": -32.0640869140625,
      "activations/layer11_attention_weight_max": 32.36045455932617,
      "activations/layer11_attention_weight_min": -29.852689743041992,
      "activations/layer12_attention_weight_max": 23.5728702545166,
      "activations/layer12_attention_weight_min": -26.88488006591797,
      "activations/layer13_attention_weight_max": 46.27975845336914,
      "activations/layer13_attention_weight_min": -32.83354187011719,
      "activations/layer14_attention_weight_max": 48.54165267944336,
      "activations/layer14_attention_weight_min": -37.287452697753906,
      "activations/layer15_attention_weight_max": 39.46235275268555,
      "activations/layer15_attention_weight_min": -29.02095603942871,
      "activations/layer16_attention_weight_max": 34.376216888427734,
      "activations/layer16_attention_weight_min": -27.997440338134766,
      "activations/layer17_attention_weight_max": 33.52983474731445,
      "activations/layer17_attention_weight_min": -24.581296920776367,
      "activations/layer18_attention_weight_max": 30.135555267333984,
      "activations/layer18_attention_weight_min": -22.40325355529785,
      "activations/layer19_attention_weight_max": 37.165584564208984,
      "activations/layer19_attention_weight_min": -32.12514877319336,
      "activations/layer1_attention_weight_max": 17.208301544189453,
      "activations/layer1_attention_weight_min": -14.200477600097656,
      "activations/layer20_attention_weight_max": 31.423084259033203,
      "activations/layer20_attention_weight_min": -25.324663162231445,
      "activations/layer21_attention_weight_max": 26.81361961364746,
      "activations/layer21_attention_weight_min": -22.641477584838867,
      "activations/layer22_attention_weight_max": 49.611122131347656,
      "activations/layer22_attention_weight_min": -30.318296432495117,
      "activations/layer23_attention_weight_max": 34.101741790771484,
      "activations/layer23_attention_weight_min": -23.387432098388672,
      "activations/layer2_attention_weight_max": 33.54267883300781,
      "activations/layer2_attention_weight_min": -32.21583557128906,
      "activations/layer3_attention_weight_max": 95.13520050048828,
      "activations/layer3_attention_weight_min": -96.82785034179688,
      "activations/layer4_attention_weight_max": 116.46305084228516,
      "activations/layer4_attention_weight_min": -118.91267395019531,
      "activations/layer5_attention_weight_max": 58.394508361816406,
      "activations/layer5_attention_weight_min": -60.47419357299805,
      "activations/layer6_attention_weight_max": 49.92207336425781,
      "activations/layer6_attention_weight_min": -49.90372848510742,
      "activations/layer7_attention_weight_max": 83.78773498535156,
      "activations/layer7_attention_weight_min": -86.06088256835938,
      "activations/layer8_attention_weight_max": 37.539302825927734,
      "activations/layer8_attention_weight_min": -39.08797073364258,
      "activations/layer9_attention_weight_max": 36.7116584777832,
      "activations/layer9_attention_weight_min": -33.92184066772461,
      "epoch": 7.61,
      "learning_rate": 0.00010197613636363635,
      "loss": 2.8072,
      "step": 130900
    },
    {
      "activations/layer0_attention_weight_max": 16.07969093322754,
      "activations/layer0_attention_weight_min": -11.700854301452637,
      "activations/layer10_attention_weight_max": 29.505111694335938,
      "activations/layer10_attention_weight_min": -29.20351791381836,
      "activations/layer11_attention_weight_max": 28.546031951904297,
      "activations/layer11_attention_weight_min": -28.153202056884766,
      "activations/layer12_attention_weight_max": 22.15791130065918,
      "activations/layer12_attention_weight_min": -30.83656120300293,
      "activations/layer13_attention_weight_max": 36.21110153198242,
      "activations/layer13_attention_weight_min": -32.531620025634766,
      "activations/layer14_attention_weight_max": 48.5428352355957,
      "activations/layer14_attention_weight_min": -46.40738296508789,
      "activations/layer15_attention_weight_max": 35.55902099609375,
      "activations/layer15_attention_weight_min": -31.214754104614258,
      "activations/layer16_attention_weight_max": 30.334529876708984,
      "activations/layer16_attention_weight_min": -32.102638244628906,
      "activations/layer17_attention_weight_max": 33.54657745361328,
      "activations/layer17_attention_weight_min": -27.158315658569336,
      "activations/layer18_attention_weight_max": 32.63938903808594,
      "activations/layer18_attention_weight_min": -20.948772430419922,
      "activations/layer19_attention_weight_max": 36.783267974853516,
      "activations/layer19_attention_weight_min": -33.58597946166992,
      "activations/layer1_attention_weight_max": 17.9744873046875,
      "activations/layer1_attention_weight_min": -13.472075462341309,
      "activations/layer20_attention_weight_max": 30.068767547607422,
      "activations/layer20_attention_weight_min": -25.112882614135742,
      "activations/layer21_attention_weight_max": 32.430633544921875,
      "activations/layer21_attention_weight_min": -24.879411697387695,
      "activations/layer22_attention_weight_max": 49.53474807739258,
      "activations/layer22_attention_weight_min": -29.106355667114258,
      "activations/layer23_attention_weight_max": 36.55194854736328,
      "activations/layer23_attention_weight_min": -24.623268127441406,
      "activations/layer2_attention_weight_max": 31.209585189819336,
      "activations/layer2_attention_weight_min": -32.225765228271484,
      "activations/layer3_attention_weight_max": 89.5309829711914,
      "activations/layer3_attention_weight_min": -95.0334701538086,
      "activations/layer4_attention_weight_max": 117.02043914794922,
      "activations/layer4_attention_weight_min": -119.9471435546875,
      "activations/layer5_attention_weight_max": 56.70161437988281,
      "activations/layer5_attention_weight_min": -61.84332275390625,
      "activations/layer6_attention_weight_max": 46.58009719848633,
      "activations/layer6_attention_weight_min": -51.00160598754883,
      "activations/layer7_attention_weight_max": 90.2365493774414,
      "activations/layer7_attention_weight_min": -84.82311248779297,
      "activations/layer8_attention_weight_max": 36.66402053833008,
      "activations/layer8_attention_weight_min": -38.78712844848633,
      "activations/layer9_attention_weight_max": 32.61859130859375,
      "activations/layer9_attention_weight_min": -32.59053421020508,
      "epoch": 7.61,
      "learning_rate": 0.00010195719696969695,
      "loss": 2.8141,
      "step": 130950
    },
    {
      "activations/layer0_attention_weight_max": 16.571508407592773,
      "activations/layer0_attention_weight_min": -11.808045387268066,
      "activations/layer10_attention_weight_max": 32.579776763916016,
      "activations/layer10_attention_weight_min": -31.690954208374023,
      "activations/layer11_attention_weight_max": 32.15565490722656,
      "activations/layer11_attention_weight_min": -29.630897521972656,
      "activations/layer12_attention_weight_max": 23.131366729736328,
      "activations/layer12_attention_weight_min": -24.767982482910156,
      "activations/layer13_attention_weight_max": 39.71615219116211,
      "activations/layer13_attention_weight_min": -32.63090133666992,
      "activations/layer14_attention_weight_max": 55.46767807006836,
      "activations/layer14_attention_weight_min": -41.17605972290039,
      "activations/layer15_attention_weight_max": 36.915550231933594,
      "activations/layer15_attention_weight_min": -29.67650604248047,
      "activations/layer16_attention_weight_max": 32.865848541259766,
      "activations/layer16_attention_weight_min": -27.721839904785156,
      "activations/layer17_attention_weight_max": 39.15421676635742,
      "activations/layer17_attention_weight_min": -26.03723907470703,
      "activations/layer18_attention_weight_max": 34.22072219848633,
      "activations/layer18_attention_weight_min": -21.513280868530273,
      "activations/layer19_attention_weight_max": 36.16653823852539,
      "activations/layer19_attention_weight_min": -29.399953842163086,
      "activations/layer1_attention_weight_max": 17.357942581176758,
      "activations/layer1_attention_weight_min": -14.46398639678955,
      "activations/layer20_attention_weight_max": 30.755435943603516,
      "activations/layer20_attention_weight_min": -23.05118751525879,
      "activations/layer21_attention_weight_max": 29.616567611694336,
      "activations/layer21_attention_weight_min": -22.48748779296875,
      "activations/layer22_attention_weight_max": 47.498260498046875,
      "activations/layer22_attention_weight_min": -27.903732299804688,
      "activations/layer23_attention_weight_max": 35.70459747314453,
      "activations/layer23_attention_weight_min": -22.05336570739746,
      "activations/layer2_attention_weight_max": 32.09815979003906,
      "activations/layer2_attention_weight_min": -31.212142944335938,
      "activations/layer3_attention_weight_max": 93.58056640625,
      "activations/layer3_attention_weight_min": -96.22508239746094,
      "activations/layer4_attention_weight_max": 122.9635009765625,
      "activations/layer4_attention_weight_min": -124.74359130859375,
      "activations/layer5_attention_weight_max": 61.84850311279297,
      "activations/layer5_attention_weight_min": -64.39485168457031,
      "activations/layer6_attention_weight_max": 49.82148361206055,
      "activations/layer6_attention_weight_min": -55.69017028808594,
      "activations/layer7_attention_weight_max": 85.36038208007812,
      "activations/layer7_attention_weight_min": -89.12374114990234,
      "activations/layer8_attention_weight_max": 39.47909927368164,
      "activations/layer8_attention_weight_min": -39.21307373046875,
      "activations/layer9_attention_weight_max": 33.95372009277344,
      "activations/layer9_attention_weight_min": -33.511024475097656,
      "epoch": 7.61,
      "learning_rate": 0.00010193825757575757,
      "loss": 2.8068,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5169,
      "eval_samples_per_second": 504.176,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5169,
      "eval_openwebtext_samples_per_second": 504.176,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 2.0194,
      "eval_wikitext_samples_per_second": 225.809,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5833,
      "eval_lambada_samples_per_second": 508.072,
      "step": 131000
    },
    {
      "activations/layer0_attention_weight_max": 15.830495834350586,
      "activations/layer0_attention_weight_min": -11.474998474121094,
      "activations/layer10_attention_weight_max": 36.99929428100586,
      "activations/layer10_attention_weight_min": -31.31332778930664,
      "activations/layer11_attention_weight_max": 35.169132232666016,
      "activations/layer11_attention_weight_min": -31.610912322998047,
      "activations/layer12_attention_weight_max": 24.54252815246582,
      "activations/layer12_attention_weight_min": -24.3403263092041,
      "activations/layer13_attention_weight_max": 48.2110481262207,
      "activations/layer13_attention_weight_min": -34.477752685546875,
      "activations/layer14_attention_weight_max": 58.42374038696289,
      "activations/layer14_attention_weight_min": -42.77989959716797,
      "activations/layer15_attention_weight_max": 40.61820983886719,
      "activations/layer15_attention_weight_min": -31.296384811401367,
      "activations/layer16_attention_weight_max": 33.9903564453125,
      "activations/layer16_attention_weight_min": -30.02289581298828,
      "activations/layer17_attention_weight_max": 38.15746307373047,
      "activations/layer17_attention_weight_min": -25.80845069885254,
      "activations/layer18_attention_weight_max": 32.51937484741211,
      "activations/layer18_attention_weight_min": -21.507709503173828,
      "activations/layer19_attention_weight_max": 39.70009994506836,
      "activations/layer19_attention_weight_min": -31.895069122314453,
      "activations/layer1_attention_weight_max": 17.35068130493164,
      "activations/layer1_attention_weight_min": -13.67536449432373,
      "activations/layer20_attention_weight_max": 36.7337646484375,
      "activations/layer20_attention_weight_min": -25.059545516967773,
      "activations/layer21_attention_weight_max": 32.83393478393555,
      "activations/layer21_attention_weight_min": -23.542715072631836,
      "activations/layer22_attention_weight_max": 51.99309158325195,
      "activations/layer22_attention_weight_min": -29.29319190979004,
      "activations/layer23_attention_weight_max": 35.874755859375,
      "activations/layer23_attention_weight_min": -22.373228073120117,
      "activations/layer2_attention_weight_max": 32.5111083984375,
      "activations/layer2_attention_weight_min": -30.636619567871094,
      "activations/layer3_attention_weight_max": 89.15994262695312,
      "activations/layer3_attention_weight_min": -90.86469268798828,
      "activations/layer4_attention_weight_max": 118.42878723144531,
      "activations/layer4_attention_weight_min": -118.80157470703125,
      "activations/layer5_attention_weight_max": 62.22980880737305,
      "activations/layer5_attention_weight_min": -59.029869079589844,
      "activations/layer6_attention_weight_max": 52.38823318481445,
      "activations/layer6_attention_weight_min": -57.147300720214844,
      "activations/layer7_attention_weight_max": 93.11640167236328,
      "activations/layer7_attention_weight_min": -89.99330139160156,
      "activations/layer8_attention_weight_max": 42.45618438720703,
      "activations/layer8_attention_weight_min": -41.03679275512695,
      "activations/layer9_attention_weight_max": 38.22561264038086,
      "activations/layer9_attention_weight_min": -34.25211715698242,
      "epoch": 7.61,
      "learning_rate": 0.00010191931818181817,
      "loss": 2.8327,
      "step": 131050
    },
    {
      "activations/layer0_attention_weight_max": 15.03271484375,
      "activations/layer0_attention_weight_min": -11.863747596740723,
      "activations/layer10_attention_weight_max": 35.61442947387695,
      "activations/layer10_attention_weight_min": -32.91895294189453,
      "activations/layer11_attention_weight_max": 35.41498565673828,
      "activations/layer11_attention_weight_min": -31.244853973388672,
      "activations/layer12_attention_weight_max": 26.779821395874023,
      "activations/layer12_attention_weight_min": -30.25889778137207,
      "activations/layer13_attention_weight_max": 43.767799377441406,
      "activations/layer13_attention_weight_min": -34.20297622680664,
      "activations/layer14_attention_weight_max": 53.36083221435547,
      "activations/layer14_attention_weight_min": -44.564674377441406,
      "activations/layer15_attention_weight_max": 44.28898239135742,
      "activations/layer15_attention_weight_min": -30.93148422241211,
      "activations/layer16_attention_weight_max": 34.74092483520508,
      "activations/layer16_attention_weight_min": -27.0125732421875,
      "activations/layer17_attention_weight_max": 39.22560501098633,
      "activations/layer17_attention_weight_min": -24.94951820373535,
      "activations/layer18_attention_weight_max": 33.33552932739258,
      "activations/layer18_attention_weight_min": -21.70850372314453,
      "activations/layer19_attention_weight_max": 45.32291793823242,
      "activations/layer19_attention_weight_min": -30.329999923706055,
      "activations/layer1_attention_weight_max": 16.45184898376465,
      "activations/layer1_attention_weight_min": -13.865839004516602,
      "activations/layer20_attention_weight_max": 40.07643508911133,
      "activations/layer20_attention_weight_min": -25.959428787231445,
      "activations/layer21_attention_weight_max": 35.21636962890625,
      "activations/layer21_attention_weight_min": -23.06039810180664,
      "activations/layer22_attention_weight_max": 59.82163619995117,
      "activations/layer22_attention_weight_min": -29.887834548950195,
      "activations/layer23_attention_weight_max": 43.35289764404297,
      "activations/layer23_attention_weight_min": -22.580638885498047,
      "activations/layer2_attention_weight_max": 32.47078323364258,
      "activations/layer2_attention_weight_min": -31.962787628173828,
      "activations/layer3_attention_weight_max": 91.52491760253906,
      "activations/layer3_attention_weight_min": -90.36238098144531,
      "activations/layer4_attention_weight_max": 122.4013671875,
      "activations/layer4_attention_weight_min": -121.634765625,
      "activations/layer5_attention_weight_max": 61.60410690307617,
      "activations/layer5_attention_weight_min": -61.47910690307617,
      "activations/layer6_attention_weight_max": 49.76558303833008,
      "activations/layer6_attention_weight_min": -53.66350173950195,
      "activations/layer7_attention_weight_max": 84.53014373779297,
      "activations/layer7_attention_weight_min": -91.9536361694336,
      "activations/layer8_attention_weight_max": 41.549537658691406,
      "activations/layer8_attention_weight_min": -41.98919677734375,
      "activations/layer9_attention_weight_max": 36.89493179321289,
      "activations/layer9_attention_weight_min": -34.06942367553711,
      "epoch": 7.62,
      "learning_rate": 0.00010190037878787877,
      "loss": 2.8216,
      "step": 131100
    },
    {
      "activations/layer0_attention_weight_max": 16.538610458374023,
      "activations/layer0_attention_weight_min": -12.593367576599121,
      "activations/layer10_attention_weight_max": 33.82769775390625,
      "activations/layer10_attention_weight_min": -32.60603713989258,
      "activations/layer11_attention_weight_max": 33.579978942871094,
      "activations/layer11_attention_weight_min": -31.306930541992188,
      "activations/layer12_attention_weight_max": 24.14859962463379,
      "activations/layer12_attention_weight_min": -24.277400970458984,
      "activations/layer13_attention_weight_max": 42.4825325012207,
      "activations/layer13_attention_weight_min": -33.7462043762207,
      "activations/layer14_attention_weight_max": 52.80866622924805,
      "activations/layer14_attention_weight_min": -43.29328155517578,
      "activations/layer15_attention_weight_max": 37.16462707519531,
      "activations/layer15_attention_weight_min": -33.15214157104492,
      "activations/layer16_attention_weight_max": 32.33747482299805,
      "activations/layer16_attention_weight_min": -27.809526443481445,
      "activations/layer17_attention_weight_max": 34.862571716308594,
      "activations/layer17_attention_weight_min": -26.486652374267578,
      "activations/layer18_attention_weight_max": 36.38396453857422,
      "activations/layer18_attention_weight_min": -22.56418228149414,
      "activations/layer19_attention_weight_max": 38.64744186401367,
      "activations/layer19_attention_weight_min": -31.139066696166992,
      "activations/layer1_attention_weight_max": 17.76375961303711,
      "activations/layer1_attention_weight_min": -13.144279479980469,
      "activations/layer20_attention_weight_max": 32.999813079833984,
      "activations/layer20_attention_weight_min": -24.452268600463867,
      "activations/layer21_attention_weight_max": 33.37950897216797,
      "activations/layer21_attention_weight_min": -23.13347625732422,
      "activations/layer22_attention_weight_max": 52.84952163696289,
      "activations/layer22_attention_weight_min": -29.638370513916016,
      "activations/layer23_attention_weight_max": 38.77726364135742,
      "activations/layer23_attention_weight_min": -22.10073471069336,
      "activations/layer2_attention_weight_max": 32.55955505371094,
      "activations/layer2_attention_weight_min": -31.947452545166016,
      "activations/layer3_attention_weight_max": 93.87789916992188,
      "activations/layer3_attention_weight_min": -93.63024139404297,
      "activations/layer4_attention_weight_max": 127.8402328491211,
      "activations/layer4_attention_weight_min": -123.39627838134766,
      "activations/layer5_attention_weight_max": 64.86907196044922,
      "activations/layer5_attention_weight_min": -66.90263366699219,
      "activations/layer6_attention_weight_max": 52.51282501220703,
      "activations/layer6_attention_weight_min": -56.071109771728516,
      "activations/layer7_attention_weight_max": 90.11293029785156,
      "activations/layer7_attention_weight_min": -88.44486999511719,
      "activations/layer8_attention_weight_max": 40.791683197021484,
      "activations/layer8_attention_weight_min": -41.54036331176758,
      "activations/layer9_attention_weight_max": 35.6011848449707,
      "activations/layer9_attention_weight_min": -34.108238220214844,
      "epoch": 7.62,
      "learning_rate": 0.00010188143939393939,
      "loss": 2.7988,
      "step": 131150
    },
    {
      "activations/layer0_attention_weight_max": 16.626312255859375,
      "activations/layer0_attention_weight_min": -11.760384559631348,
      "activations/layer10_attention_weight_max": 35.127994537353516,
      "activations/layer10_attention_weight_min": -31.249202728271484,
      "activations/layer11_attention_weight_max": 37.91077423095703,
      "activations/layer11_attention_weight_min": -31.080554962158203,
      "activations/layer12_attention_weight_max": 42.6336555480957,
      "activations/layer12_attention_weight_min": -27.509334564208984,
      "activations/layer13_attention_weight_max": 57.68524169921875,
      "activations/layer13_attention_weight_min": -33.80609893798828,
      "activations/layer14_attention_weight_max": 58.77294158935547,
      "activations/layer14_attention_weight_min": -41.492759704589844,
      "activations/layer15_attention_weight_max": 41.29280471801758,
      "activations/layer15_attention_weight_min": -29.3197078704834,
      "activations/layer16_attention_weight_max": 35.77870178222656,
      "activations/layer16_attention_weight_min": -28.001806259155273,
      "activations/layer17_attention_weight_max": 34.440425872802734,
      "activations/layer17_attention_weight_min": -25.840120315551758,
      "activations/layer18_attention_weight_max": 33.78907775878906,
      "activations/layer18_attention_weight_min": -20.520658493041992,
      "activations/layer19_attention_weight_max": 39.70092010498047,
      "activations/layer19_attention_weight_min": -36.31154251098633,
      "activations/layer1_attention_weight_max": 17.750905990600586,
      "activations/layer1_attention_weight_min": -14.130707740783691,
      "activations/layer20_attention_weight_max": 29.7559757232666,
      "activations/layer20_attention_weight_min": -27.21478271484375,
      "activations/layer21_attention_weight_max": 29.207931518554688,
      "activations/layer21_attention_weight_min": -25.008432388305664,
      "activations/layer22_attention_weight_max": 43.90169906616211,
      "activations/layer22_attention_weight_min": -33.461734771728516,
      "activations/layer23_attention_weight_max": 32.08137512207031,
      "activations/layer23_attention_weight_min": -25.006134033203125,
      "activations/layer2_attention_weight_max": 31.807348251342773,
      "activations/layer2_attention_weight_min": -31.393096923828125,
      "activations/layer3_attention_weight_max": 94.07746124267578,
      "activations/layer3_attention_weight_min": -93.46187591552734,
      "activations/layer4_attention_weight_max": 125.3384017944336,
      "activations/layer4_attention_weight_min": -119.22847747802734,
      "activations/layer5_attention_weight_max": 62.11045837402344,
      "activations/layer5_attention_weight_min": -60.88249969482422,
      "activations/layer6_attention_weight_max": 52.158714294433594,
      "activations/layer6_attention_weight_min": -54.54357147216797,
      "activations/layer7_attention_weight_max": 90.17249298095703,
      "activations/layer7_attention_weight_min": -94.76750946044922,
      "activations/layer8_attention_weight_max": 44.1854362487793,
      "activations/layer8_attention_weight_min": -42.518165588378906,
      "activations/layer9_attention_weight_max": 36.939361572265625,
      "activations/layer9_attention_weight_min": -35.2335205078125,
      "epoch": 7.62,
      "learning_rate": 0.0001018625,
      "loss": 2.8091,
      "step": 131200
    },
    {
      "activations/layer0_attention_weight_max": 16.604711532592773,
      "activations/layer0_attention_weight_min": -12.292201042175293,
      "activations/layer10_attention_weight_max": 32.2655143737793,
      "activations/layer10_attention_weight_min": -31.26873016357422,
      "activations/layer11_attention_weight_max": 36.177337646484375,
      "activations/layer11_attention_weight_min": -28.737842559814453,
      "activations/layer12_attention_weight_max": 34.35516357421875,
      "activations/layer12_attention_weight_min": -28.539674758911133,
      "activations/layer13_attention_weight_max": 42.45307159423828,
      "activations/layer13_attention_weight_min": -30.331096649169922,
      "activations/layer14_attention_weight_max": 68.4835205078125,
      "activations/layer14_attention_weight_min": -41.75526428222656,
      "activations/layer15_attention_weight_max": 40.542503356933594,
      "activations/layer15_attention_weight_min": -29.03718376159668,
      "activations/layer16_attention_weight_max": 41.88030242919922,
      "activations/layer16_attention_weight_min": -26.815614700317383,
      "activations/layer17_attention_weight_max": 37.73428726196289,
      "activations/layer17_attention_weight_min": -24.99043846130371,
      "activations/layer18_attention_weight_max": 37.19596481323242,
      "activations/layer18_attention_weight_min": -20.390968322753906,
      "activations/layer19_attention_weight_max": 42.205955505371094,
      "activations/layer19_attention_weight_min": -30.34153175354004,
      "activations/layer1_attention_weight_max": 17.45608139038086,
      "activations/layer1_attention_weight_min": -13.911632537841797,
      "activations/layer20_attention_weight_max": 37.4022216796875,
      "activations/layer20_attention_weight_min": -22.654874801635742,
      "activations/layer21_attention_weight_max": 36.19291305541992,
      "activations/layer21_attention_weight_min": -23.587539672851562,
      "activations/layer22_attention_weight_max": 44.4051628112793,
      "activations/layer22_attention_weight_min": -28.359989166259766,
      "activations/layer23_attention_weight_max": 32.56988525390625,
      "activations/layer23_attention_weight_min": -23.426830291748047,
      "activations/layer2_attention_weight_max": 32.879844665527344,
      "activations/layer2_attention_weight_min": -31.320552825927734,
      "activations/layer3_attention_weight_max": 90.71524047851562,
      "activations/layer3_attention_weight_min": -90.86380767822266,
      "activations/layer4_attention_weight_max": 113.56678771972656,
      "activations/layer4_attention_weight_min": -113.58418273925781,
      "activations/layer5_attention_weight_max": 57.47190856933594,
      "activations/layer5_attention_weight_min": -62.74872589111328,
      "activations/layer6_attention_weight_max": 51.526363372802734,
      "activations/layer6_attention_weight_min": -50.09109115600586,
      "activations/layer7_attention_weight_max": 85.45318603515625,
      "activations/layer7_attention_weight_min": -84.3985595703125,
      "activations/layer8_attention_weight_max": 38.12932586669922,
      "activations/layer8_attention_weight_min": -37.90559387207031,
      "activations/layer9_attention_weight_max": 33.68923568725586,
      "activations/layer9_attention_weight_min": -32.543434143066406,
      "epoch": 7.63,
      "learning_rate": 0.00010184356060606059,
      "loss": 2.817,
      "step": 131250
    },
    {
      "activations/layer0_attention_weight_max": 15.78210163116455,
      "activations/layer0_attention_weight_min": -11.417470932006836,
      "activations/layer10_attention_weight_max": 38.71200180053711,
      "activations/layer10_attention_weight_min": -32.365928649902344,
      "activations/layer11_attention_weight_max": 37.65877914428711,
      "activations/layer11_attention_weight_min": -34.72178649902344,
      "activations/layer12_attention_weight_max": 45.279052734375,
      "activations/layer12_attention_weight_min": -24.799339294433594,
      "activations/layer13_attention_weight_max": 54.339962005615234,
      "activations/layer13_attention_weight_min": -38.779396057128906,
      "activations/layer14_attention_weight_max": 64.13553619384766,
      "activations/layer14_attention_weight_min": -48.93013381958008,
      "activations/layer15_attention_weight_max": 48.23574447631836,
      "activations/layer15_attention_weight_min": -37.0671501159668,
      "activations/layer16_attention_weight_max": 34.40652084350586,
      "activations/layer16_attention_weight_min": -27.394010543823242,
      "activations/layer17_attention_weight_max": 32.0604133605957,
      "activations/layer17_attention_weight_min": -24.009992599487305,
      "activations/layer18_attention_weight_max": 31.081295013427734,
      "activations/layer18_attention_weight_min": -24.704315185546875,
      "activations/layer19_attention_weight_max": 38.51424789428711,
      "activations/layer19_attention_weight_min": -29.774372100830078,
      "activations/layer1_attention_weight_max": 17.46459197998047,
      "activations/layer1_attention_weight_min": -14.326227188110352,
      "activations/layer20_attention_weight_max": 30.088937759399414,
      "activations/layer20_attention_weight_min": -23.5058650970459,
      "activations/layer21_attention_weight_max": 29.907577514648438,
      "activations/layer21_attention_weight_min": -23.059160232543945,
      "activations/layer22_attention_weight_max": 55.456298828125,
      "activations/layer22_attention_weight_min": -29.36467170715332,
      "activations/layer23_attention_weight_max": 34.07012939453125,
      "activations/layer23_attention_weight_min": -21.97780990600586,
      "activations/layer2_attention_weight_max": 32.21034622192383,
      "activations/layer2_attention_weight_min": -32.26660919189453,
      "activations/layer3_attention_weight_max": 93.95079803466797,
      "activations/layer3_attention_weight_min": -95.84608459472656,
      "activations/layer4_attention_weight_max": 121.65239715576172,
      "activations/layer4_attention_weight_min": -121.84381103515625,
      "activations/layer5_attention_weight_max": 62.224609375,
      "activations/layer5_attention_weight_min": -64.40946960449219,
      "activations/layer6_attention_weight_max": 54.39353942871094,
      "activations/layer6_attention_weight_min": -55.05162811279297,
      "activations/layer7_attention_weight_max": 102.03411865234375,
      "activations/layer7_attention_weight_min": -97.44377899169922,
      "activations/layer8_attention_weight_max": 44.951168060302734,
      "activations/layer8_attention_weight_min": -42.87223815917969,
      "activations/layer9_attention_weight_max": 41.12400436401367,
      "activations/layer9_attention_weight_min": -36.373390197753906,
      "epoch": 7.63,
      "learning_rate": 0.0001018246212121212,
      "loss": 2.8148,
      "step": 131300
    },
    {
      "activations/layer0_attention_weight_max": 16.70439910888672,
      "activations/layer0_attention_weight_min": -12.588275909423828,
      "activations/layer10_attention_weight_max": 36.43132019042969,
      "activations/layer10_attention_weight_min": -36.08517074584961,
      "activations/layer11_attention_weight_max": 39.52446365356445,
      "activations/layer11_attention_weight_min": -34.424560546875,
      "activations/layer12_attention_weight_max": 22.165699005126953,
      "activations/layer12_attention_weight_min": -27.508113861083984,
      "activations/layer13_attention_weight_max": 39.162288665771484,
      "activations/layer13_attention_weight_min": -34.66313552856445,
      "activations/layer14_attention_weight_max": 55.18339538574219,
      "activations/layer14_attention_weight_min": -48.15339660644531,
      "activations/layer15_attention_weight_max": 33.00603485107422,
      "activations/layer15_attention_weight_min": -30.59676170349121,
      "activations/layer16_attention_weight_max": 31.648588180541992,
      "activations/layer16_attention_weight_min": -27.953529357910156,
      "activations/layer17_attention_weight_max": 34.45249938964844,
      "activations/layer17_attention_weight_min": -24.38985252380371,
      "activations/layer18_attention_weight_max": 29.777074813842773,
      "activations/layer18_attention_weight_min": -20.49252700805664,
      "activations/layer19_attention_weight_max": 34.071205139160156,
      "activations/layer19_attention_weight_min": -30.515005111694336,
      "activations/layer1_attention_weight_max": 18.24256134033203,
      "activations/layer1_attention_weight_min": -15.17851734161377,
      "activations/layer20_attention_weight_max": 28.616941452026367,
      "activations/layer20_attention_weight_min": -25.072349548339844,
      "activations/layer21_attention_weight_max": 28.301132202148438,
      "activations/layer21_attention_weight_min": -23.485519409179688,
      "activations/layer22_attention_weight_max": 44.85745620727539,
      "activations/layer22_attention_weight_min": -27.8952579498291,
      "activations/layer23_attention_weight_max": 36.73123550415039,
      "activations/layer23_attention_weight_min": -23.085201263427734,
      "activations/layer2_attention_weight_max": 32.30992889404297,
      "activations/layer2_attention_weight_min": -31.177650451660156,
      "activations/layer3_attention_weight_max": 94.38008880615234,
      "activations/layer3_attention_weight_min": -94.49560546875,
      "activations/layer4_attention_weight_max": 123.10887908935547,
      "activations/layer4_attention_weight_min": -123.45922088623047,
      "activations/layer5_attention_weight_max": 63.73554229736328,
      "activations/layer5_attention_weight_min": -63.37184143066406,
      "activations/layer6_attention_weight_max": 50.363101959228516,
      "activations/layer6_attention_weight_min": -54.75157928466797,
      "activations/layer7_attention_weight_max": 95.05030822753906,
      "activations/layer7_attention_weight_min": -94.41608428955078,
      "activations/layer8_attention_weight_max": 42.099308013916016,
      "activations/layer8_attention_weight_min": -45.61687088012695,
      "activations/layer9_attention_weight_max": 38.26537322998047,
      "activations/layer9_attention_weight_min": -41.56087112426758,
      "epoch": 7.63,
      "learning_rate": 0.00010180568181818182,
      "loss": 2.8105,
      "step": 131350
    },
    {
      "activations/layer0_attention_weight_max": 16.42013931274414,
      "activations/layer0_attention_weight_min": -11.618194580078125,
      "activations/layer10_attention_weight_max": 33.14027786254883,
      "activations/layer10_attention_weight_min": -31.83194923400879,
      "activations/layer11_attention_weight_max": 33.6327018737793,
      "activations/layer11_attention_weight_min": -29.882701873779297,
      "activations/layer12_attention_weight_max": 22.893386840820312,
      "activations/layer12_attention_weight_min": -26.430984497070312,
      "activations/layer13_attention_weight_max": 39.929908752441406,
      "activations/layer13_attention_weight_min": -33.22336196899414,
      "activations/layer14_attention_weight_max": 51.496238708496094,
      "activations/layer14_attention_weight_min": -39.20356369018555,
      "activations/layer15_attention_weight_max": 35.267311096191406,
      "activations/layer15_attention_weight_min": -30.34779930114746,
      "activations/layer16_attention_weight_max": 35.31266784667969,
      "activations/layer16_attention_weight_min": -27.781293869018555,
      "activations/layer17_attention_weight_max": 37.11642837524414,
      "activations/layer17_attention_weight_min": -24.798927307128906,
      "activations/layer18_attention_weight_max": 33.77815628051758,
      "activations/layer18_attention_weight_min": -20.760332107543945,
      "activations/layer19_attention_weight_max": 36.30580139160156,
      "activations/layer19_attention_weight_min": -29.947580337524414,
      "activations/layer1_attention_weight_max": 17.288578033447266,
      "activations/layer1_attention_weight_min": -14.495059967041016,
      "activations/layer20_attention_weight_max": 32.35539245605469,
      "activations/layer20_attention_weight_min": -25.113615036010742,
      "activations/layer21_attention_weight_max": 31.7557373046875,
      "activations/layer21_attention_weight_min": -23.463333129882812,
      "activations/layer22_attention_weight_max": 51.11416244506836,
      "activations/layer22_attention_weight_min": -28.574079513549805,
      "activations/layer23_attention_weight_max": 35.239864349365234,
      "activations/layer23_attention_weight_min": -24.586111068725586,
      "activations/layer2_attention_weight_max": 34.376834869384766,
      "activations/layer2_attention_weight_min": -33.1906852722168,
      "activations/layer3_attention_weight_max": 94.2476806640625,
      "activations/layer3_attention_weight_min": -96.47313690185547,
      "activations/layer4_attention_weight_max": 122.86329650878906,
      "activations/layer4_attention_weight_min": -119.22193908691406,
      "activations/layer5_attention_weight_max": 60.86573791503906,
      "activations/layer5_attention_weight_min": -63.226558685302734,
      "activations/layer6_attention_weight_max": 51.79494094848633,
      "activations/layer6_attention_weight_min": -55.037452697753906,
      "activations/layer7_attention_weight_max": 85.5175552368164,
      "activations/layer7_attention_weight_min": -90.6246109008789,
      "activations/layer8_attention_weight_max": 39.98360061645508,
      "activations/layer8_attention_weight_min": -41.27992248535156,
      "activations/layer9_attention_weight_max": 34.32997131347656,
      "activations/layer9_attention_weight_min": -34.7458610534668,
      "epoch": 7.64,
      "learning_rate": 0.00010178674242424241,
      "loss": 2.812,
      "step": 131400
    },
    {
      "activations/layer0_attention_weight_max": 16.457887649536133,
      "activations/layer0_attention_weight_min": -11.555872917175293,
      "activations/layer10_attention_weight_max": 35.67168045043945,
      "activations/layer10_attention_weight_min": -33.555885314941406,
      "activations/layer11_attention_weight_max": 33.50401306152344,
      "activations/layer11_attention_weight_min": -32.27936553955078,
      "activations/layer12_attention_weight_max": 25.710498809814453,
      "activations/layer12_attention_weight_min": -24.205116271972656,
      "activations/layer13_attention_weight_max": 44.075294494628906,
      "activations/layer13_attention_weight_min": -33.34335708618164,
      "activations/layer14_attention_weight_max": 59.41623306274414,
      "activations/layer14_attention_weight_min": -40.23357009887695,
      "activations/layer15_attention_weight_max": 35.56932830810547,
      "activations/layer15_attention_weight_min": -30.012529373168945,
      "activations/layer16_attention_weight_max": 34.91575622558594,
      "activations/layer16_attention_weight_min": -26.659337997436523,
      "activations/layer17_attention_weight_max": 35.07895278930664,
      "activations/layer17_attention_weight_min": -24.76848793029785,
      "activations/layer18_attention_weight_max": 31.693904876708984,
      "activations/layer18_attention_weight_min": -22.700145721435547,
      "activations/layer19_attention_weight_max": 37.769466400146484,
      "activations/layer19_attention_weight_min": -30.684860229492188,
      "activations/layer1_attention_weight_max": 17.052515029907227,
      "activations/layer1_attention_weight_min": -13.877850532531738,
      "activations/layer20_attention_weight_max": 31.43986701965332,
      "activations/layer20_attention_weight_min": -26.226743698120117,
      "activations/layer21_attention_weight_max": 30.20187759399414,
      "activations/layer21_attention_weight_min": -24.215038299560547,
      "activations/layer22_attention_weight_max": 48.46584701538086,
      "activations/layer22_attention_weight_min": -28.603437423706055,
      "activations/layer23_attention_weight_max": 37.52931594848633,
      "activations/layer23_attention_weight_min": -24.07720947265625,
      "activations/layer2_attention_weight_max": 33.17142105102539,
      "activations/layer2_attention_weight_min": -31.962570190429688,
      "activations/layer3_attention_weight_max": 91.73808288574219,
      "activations/layer3_attention_weight_min": -97.11577606201172,
      "activations/layer4_attention_weight_max": 121.63822174072266,
      "activations/layer4_attention_weight_min": -120.7615966796875,
      "activations/layer5_attention_weight_max": 65.34584045410156,
      "activations/layer5_attention_weight_min": -61.967037200927734,
      "activations/layer6_attention_weight_max": 52.4818000793457,
      "activations/layer6_attention_weight_min": -56.381072998046875,
      "activations/layer7_attention_weight_max": 90.7813720703125,
      "activations/layer7_attention_weight_min": -86.63310241699219,
      "activations/layer8_attention_weight_max": 41.83668899536133,
      "activations/layer8_attention_weight_min": -43.609764099121094,
      "activations/layer9_attention_weight_max": 37.13237762451172,
      "activations/layer9_attention_weight_min": -36.60787582397461,
      "epoch": 7.64,
      "learning_rate": 0.00010176780303030302,
      "loss": 2.814,
      "step": 131450
    },
    {
      "activations/layer0_attention_weight_max": 15.515339851379395,
      "activations/layer0_attention_weight_min": -11.43429183959961,
      "activations/layer10_attention_weight_max": 37.203041076660156,
      "activations/layer10_attention_weight_min": -32.97051239013672,
      "activations/layer11_attention_weight_max": 35.572120666503906,
      "activations/layer11_attention_weight_min": -31.815616607666016,
      "activations/layer12_attention_weight_max": 24.223222732543945,
      "activations/layer12_attention_weight_min": -23.521778106689453,
      "activations/layer13_attention_weight_max": 41.316741943359375,
      "activations/layer13_attention_weight_min": -33.56822204589844,
      "activations/layer14_attention_weight_max": 51.105186462402344,
      "activations/layer14_attention_weight_min": -41.154361724853516,
      "activations/layer15_attention_weight_max": 40.1251220703125,
      "activations/layer15_attention_weight_min": -30.076730728149414,
      "activations/layer16_attention_weight_max": 35.77665328979492,
      "activations/layer16_attention_weight_min": -27.627582550048828,
      "activations/layer17_attention_weight_max": 39.92775344848633,
      "activations/layer17_attention_weight_min": -27.11066436767578,
      "activations/layer18_attention_weight_max": 34.96782302856445,
      "activations/layer18_attention_weight_min": -22.51767921447754,
      "activations/layer19_attention_weight_max": 38.567073822021484,
      "activations/layer19_attention_weight_min": -33.365150451660156,
      "activations/layer1_attention_weight_max": 16.838184356689453,
      "activations/layer1_attention_weight_min": -13.937349319458008,
      "activations/layer20_attention_weight_max": 33.79323196411133,
      "activations/layer20_attention_weight_min": -24.51337242126465,
      "activations/layer21_attention_weight_max": 32.98636245727539,
      "activations/layer21_attention_weight_min": -22.407501220703125,
      "activations/layer22_attention_weight_max": 50.797481536865234,
      "activations/layer22_attention_weight_min": -29.13086700439453,
      "activations/layer23_attention_weight_max": 40.070438385009766,
      "activations/layer23_attention_weight_min": -24.098140716552734,
      "activations/layer2_attention_weight_max": 33.53633117675781,
      "activations/layer2_attention_weight_min": -33.28665542602539,
      "activations/layer3_attention_weight_max": 98.56381225585938,
      "activations/layer3_attention_weight_min": -94.48633575439453,
      "activations/layer4_attention_weight_max": 127.51297760009766,
      "activations/layer4_attention_weight_min": -121.90185546875,
      "activations/layer5_attention_weight_max": 63.30512237548828,
      "activations/layer5_attention_weight_min": -62.21771240234375,
      "activations/layer6_attention_weight_max": 52.6524543762207,
      "activations/layer6_attention_weight_min": -55.76664733886719,
      "activations/layer7_attention_weight_max": 95.87616729736328,
      "activations/layer7_attention_weight_min": -94.69715881347656,
      "activations/layer8_attention_weight_max": 42.3781623840332,
      "activations/layer8_attention_weight_min": -42.956295013427734,
      "activations/layer9_attention_weight_max": 39.55825424194336,
      "activations/layer9_attention_weight_min": -36.52629089355469,
      "epoch": 7.64,
      "learning_rate": 0.00010174886363636363,
      "loss": 2.8001,
      "step": 131500
    },
    {
      "activations/layer0_attention_weight_max": 15.639137268066406,
      "activations/layer0_attention_weight_min": -12.061929702758789,
      "activations/layer10_attention_weight_max": 33.01938247680664,
      "activations/layer10_attention_weight_min": -31.954788208007812,
      "activations/layer11_attention_weight_max": 32.07025146484375,
      "activations/layer11_attention_weight_min": -30.144611358642578,
      "activations/layer12_attention_weight_max": 22.873687744140625,
      "activations/layer12_attention_weight_min": -24.18876075744629,
      "activations/layer13_attention_weight_max": 37.55620193481445,
      "activations/layer13_attention_weight_min": -34.84001159667969,
      "activations/layer14_attention_weight_max": 49.34331130981445,
      "activations/layer14_attention_weight_min": -39.31193161010742,
      "activations/layer15_attention_weight_max": 36.58576202392578,
      "activations/layer15_attention_weight_min": -30.76738166809082,
      "activations/layer16_attention_weight_max": 34.35759735107422,
      "activations/layer16_attention_weight_min": -28.631994247436523,
      "activations/layer17_attention_weight_max": 36.35480880737305,
      "activations/layer17_attention_weight_min": -26.455894470214844,
      "activations/layer18_attention_weight_max": 33.6911735534668,
      "activations/layer18_attention_weight_min": -24.262269973754883,
      "activations/layer19_attention_weight_max": 34.33842849731445,
      "activations/layer19_attention_weight_min": -30.152099609375,
      "activations/layer1_attention_weight_max": 19.3642520904541,
      "activations/layer1_attention_weight_min": -17.25819969177246,
      "activations/layer20_attention_weight_max": 31.419076919555664,
      "activations/layer20_attention_weight_min": -26.400468826293945,
      "activations/layer21_attention_weight_max": 29.022750854492188,
      "activations/layer21_attention_weight_min": -26.767141342163086,
      "activations/layer22_attention_weight_max": 46.540523529052734,
      "activations/layer22_attention_weight_min": -31.51643943786621,
      "activations/layer23_attention_weight_max": 34.53574752807617,
      "activations/layer23_attention_weight_min": -26.143043518066406,
      "activations/layer2_attention_weight_max": 31.51791000366211,
      "activations/layer2_attention_weight_min": -32.065269470214844,
      "activations/layer3_attention_weight_max": 94.74029541015625,
      "activations/layer3_attention_weight_min": -94.28218078613281,
      "activations/layer4_attention_weight_max": 121.61323547363281,
      "activations/layer4_attention_weight_min": -122.6212387084961,
      "activations/layer5_attention_weight_max": 60.956390380859375,
      "activations/layer5_attention_weight_min": -63.07398223876953,
      "activations/layer6_attention_weight_max": 50.28392028808594,
      "activations/layer6_attention_weight_min": -50.660423278808594,
      "activations/layer7_attention_weight_max": 86.32015991210938,
      "activations/layer7_attention_weight_min": -85.15876770019531,
      "activations/layer8_attention_weight_max": 38.222537994384766,
      "activations/layer8_attention_weight_min": -39.168331146240234,
      "activations/layer9_attention_weight_max": 34.619258880615234,
      "activations/layer9_attention_weight_min": -34.20344924926758,
      "epoch": 7.64,
      "learning_rate": 0.00010172992424242423,
      "loss": 2.8132,
      "step": 131550
    },
    {
      "activations/layer0_attention_weight_max": 15.33005428314209,
      "activations/layer0_attention_weight_min": -11.685948371887207,
      "activations/layer10_attention_weight_max": 35.95086669921875,
      "activations/layer10_attention_weight_min": -32.20195388793945,
      "activations/layer11_attention_weight_max": 36.95244598388672,
      "activations/layer11_attention_weight_min": -31.518125534057617,
      "activations/layer12_attention_weight_max": 40.90016174316406,
      "activations/layer12_attention_weight_min": -31.785995483398438,
      "activations/layer13_attention_weight_max": 49.12828063964844,
      "activations/layer13_attention_weight_min": -34.279212951660156,
      "activations/layer14_attention_weight_max": 60.92652130126953,
      "activations/layer14_attention_weight_min": -41.84192657470703,
      "activations/layer15_attention_weight_max": 41.30306625366211,
      "activations/layer15_attention_weight_min": -31.717622756958008,
      "activations/layer16_attention_weight_max": 43.60530471801758,
      "activations/layer16_attention_weight_min": -29.21476173400879,
      "activations/layer17_attention_weight_max": 37.836971282958984,
      "activations/layer17_attention_weight_min": -24.429533004760742,
      "activations/layer18_attention_weight_max": 35.709712982177734,
      "activations/layer18_attention_weight_min": -21.462818145751953,
      "activations/layer19_attention_weight_max": 43.8147087097168,
      "activations/layer19_attention_weight_min": -31.74659538269043,
      "activations/layer1_attention_weight_max": 17.05699920654297,
      "activations/layer1_attention_weight_min": -14.389925003051758,
      "activations/layer20_attention_weight_max": 36.081939697265625,
      "activations/layer20_attention_weight_min": -22.9458065032959,
      "activations/layer21_attention_weight_max": 32.983619689941406,
      "activations/layer21_attention_weight_min": -21.930723190307617,
      "activations/layer22_attention_weight_max": 55.2351188659668,
      "activations/layer22_attention_weight_min": -30.05194091796875,
      "activations/layer23_attention_weight_max": 39.971885681152344,
      "activations/layer23_attention_weight_min": -22.905719757080078,
      "activations/layer2_attention_weight_max": 32.020748138427734,
      "activations/layer2_attention_weight_min": -30.796958923339844,
      "activations/layer3_attention_weight_max": 93.02957916259766,
      "activations/layer3_attention_weight_min": -94.34452819824219,
      "activations/layer4_attention_weight_max": 119.2402572631836,
      "activations/layer4_attention_weight_min": -116.83109283447266,
      "activations/layer5_attention_weight_max": 60.75580978393555,
      "activations/layer5_attention_weight_min": -60.763206481933594,
      "activations/layer6_attention_weight_max": 48.40251541137695,
      "activations/layer6_attention_weight_min": -51.743003845214844,
      "activations/layer7_attention_weight_max": 88.51254272460938,
      "activations/layer7_attention_weight_min": -85.74881744384766,
      "activations/layer8_attention_weight_max": 42.28347396850586,
      "activations/layer8_attention_weight_min": -41.118675231933594,
      "activations/layer9_attention_weight_max": 38.49663162231445,
      "activations/layer9_attention_weight_min": -35.370033264160156,
      "epoch": 7.65,
      "learning_rate": 0.00010171098484848484,
      "loss": 2.8012,
      "step": 131600
    },
    {
      "activations/layer0_attention_weight_max": 14.686904907226562,
      "activations/layer0_attention_weight_min": -11.756518363952637,
      "activations/layer10_attention_weight_max": 41.82960891723633,
      "activations/layer10_attention_weight_min": -36.75984573364258,
      "activations/layer11_attention_weight_max": 45.714027404785156,
      "activations/layer11_attention_weight_min": -36.375099182128906,
      "activations/layer12_attention_weight_max": 29.324264526367188,
      "activations/layer12_attention_weight_min": -30.925228118896484,
      "activations/layer13_attention_weight_max": 42.670169830322266,
      "activations/layer13_attention_weight_min": -32.76032638549805,
      "activations/layer14_attention_weight_max": 55.702999114990234,
      "activations/layer14_attention_weight_min": -42.397377014160156,
      "activations/layer15_attention_weight_max": 47.69070053100586,
      "activations/layer15_attention_weight_min": -30.206026077270508,
      "activations/layer16_attention_weight_max": 35.09469985961914,
      "activations/layer16_attention_weight_min": -28.096546173095703,
      "activations/layer17_attention_weight_max": 38.65430450439453,
      "activations/layer17_attention_weight_min": -27.11432456970215,
      "activations/layer18_attention_weight_max": 35.7263298034668,
      "activations/layer18_attention_weight_min": -23.8768310546875,
      "activations/layer19_attention_weight_max": 37.58663558959961,
      "activations/layer19_attention_weight_min": -34.735164642333984,
      "activations/layer1_attention_weight_max": 16.801315307617188,
      "activations/layer1_attention_weight_min": -15.831148147583008,
      "activations/layer20_attention_weight_max": 34.06642532348633,
      "activations/layer20_attention_weight_min": -29.426490783691406,
      "activations/layer21_attention_weight_max": 30.587905883789062,
      "activations/layer21_attention_weight_min": -26.05975341796875,
      "activations/layer22_attention_weight_max": 53.77959442138672,
      "activations/layer22_attention_weight_min": -34.28127670288086,
      "activations/layer23_attention_weight_max": 39.24987030029297,
      "activations/layer23_attention_weight_min": -24.57843017578125,
      "activations/layer2_attention_weight_max": 34.882957458496094,
      "activations/layer2_attention_weight_min": -34.299869537353516,
      "activations/layer3_attention_weight_max": 100.04235076904297,
      "activations/layer3_attention_weight_min": -105.67747497558594,
      "activations/layer4_attention_weight_max": 128.63189697265625,
      "activations/layer4_attention_weight_min": -121.36866760253906,
      "activations/layer5_attention_weight_max": 64.70964813232422,
      "activations/layer5_attention_weight_min": -64.58966064453125,
      "activations/layer6_attention_weight_max": 57.3642692565918,
      "activations/layer6_attention_weight_min": -53.802040100097656,
      "activations/layer7_attention_weight_max": 113.67386627197266,
      "activations/layer7_attention_weight_min": -102.00946044921875,
      "activations/layer8_attention_weight_max": 54.42545700073242,
      "activations/layer8_attention_weight_min": -48.28013229370117,
      "activations/layer9_attention_weight_max": 45.55449676513672,
      "activations/layer9_attention_weight_min": -39.13014221191406,
      "epoch": 7.65,
      "learning_rate": 0.00010169204545454544,
      "loss": 2.8176,
      "step": 131650
    },
    {
      "activations/layer0_attention_weight_max": 16.85000991821289,
      "activations/layer0_attention_weight_min": -11.700672149658203,
      "activations/layer10_attention_weight_max": 36.532691955566406,
      "activations/layer10_attention_weight_min": -33.00941467285156,
      "activations/layer11_attention_weight_max": 34.6234130859375,
      "activations/layer11_attention_weight_min": -30.90198516845703,
      "activations/layer12_attention_weight_max": 26.821083068847656,
      "activations/layer12_attention_weight_min": -23.964111328125,
      "activations/layer13_attention_weight_max": 41.54964065551758,
      "activations/layer13_attention_weight_min": -33.10944366455078,
      "activations/layer14_attention_weight_max": 47.31819534301758,
      "activations/layer14_attention_weight_min": -40.28825759887695,
      "activations/layer15_attention_weight_max": 36.76263427734375,
      "activations/layer15_attention_weight_min": -29.43821907043457,
      "activations/layer16_attention_weight_max": 34.33438491821289,
      "activations/layer16_attention_weight_min": -26.82819175720215,
      "activations/layer17_attention_weight_max": 37.80827331542969,
      "activations/layer17_attention_weight_min": -24.683176040649414,
      "activations/layer18_attention_weight_max": 35.36494827270508,
      "activations/layer18_attention_weight_min": -22.540203094482422,
      "activations/layer19_attention_weight_max": 40.98452377319336,
      "activations/layer19_attention_weight_min": -30.363733291625977,
      "activations/layer1_attention_weight_max": 16.598058700561523,
      "activations/layer1_attention_weight_min": -14.636758804321289,
      "activations/layer20_attention_weight_max": 35.4147834777832,
      "activations/layer20_attention_weight_min": -25.849956512451172,
      "activations/layer21_attention_weight_max": 32.848548889160156,
      "activations/layer21_attention_weight_min": -23.1036319732666,
      "activations/layer22_attention_weight_max": 52.383689880371094,
      "activations/layer22_attention_weight_min": -28.949005126953125,
      "activations/layer23_attention_weight_max": 40.19371032714844,
      "activations/layer23_attention_weight_min": -23.83541488647461,
      "activations/layer2_attention_weight_max": 32.03767395019531,
      "activations/layer2_attention_weight_min": -31.84572410583496,
      "activations/layer3_attention_weight_max": 93.05957794189453,
      "activations/layer3_attention_weight_min": -94.63447570800781,
      "activations/layer4_attention_weight_max": 126.8911361694336,
      "activations/layer4_attention_weight_min": -123.135498046875,
      "activations/layer5_attention_weight_max": 62.876216888427734,
      "activations/layer5_attention_weight_min": -63.8098258972168,
      "activations/layer6_attention_weight_max": 50.50151062011719,
      "activations/layer6_attention_weight_min": -52.66166687011719,
      "activations/layer7_attention_weight_max": 86.57547760009766,
      "activations/layer7_attention_weight_min": -91.66858673095703,
      "activations/layer8_attention_weight_max": 39.712589263916016,
      "activations/layer8_attention_weight_min": -40.42316436767578,
      "activations/layer9_attention_weight_max": 36.37519836425781,
      "activations/layer9_attention_weight_min": -34.914119720458984,
      "epoch": 7.65,
      "learning_rate": 0.00010167310606060606,
      "loss": 2.8249,
      "step": 131700
    },
    {
      "activations/layer0_attention_weight_max": 15.947569847106934,
      "activations/layer0_attention_weight_min": -12.068283081054688,
      "activations/layer10_attention_weight_max": 33.390708923339844,
      "activations/layer10_attention_weight_min": -31.237442016601562,
      "activations/layer11_attention_weight_max": 34.68564987182617,
      "activations/layer11_attention_weight_min": -31.013423919677734,
      "activations/layer12_attention_weight_max": 23.913753509521484,
      "activations/layer12_attention_weight_min": -23.545589447021484,
      "activations/layer13_attention_weight_max": 44.91200637817383,
      "activations/layer13_attention_weight_min": -32.16910934448242,
      "activations/layer14_attention_weight_max": 49.04581832885742,
      "activations/layer14_attention_weight_min": -43.18821334838867,
      "activations/layer15_attention_weight_max": 38.345848083496094,
      "activations/layer15_attention_weight_min": -31.328420639038086,
      "activations/layer16_attention_weight_max": 36.57201385498047,
      "activations/layer16_attention_weight_min": -29.704301834106445,
      "activations/layer17_attention_weight_max": 37.06789779663086,
      "activations/layer17_attention_weight_min": -25.079988479614258,
      "activations/layer18_attention_weight_max": 34.57030487060547,
      "activations/layer18_attention_weight_min": -21.83306884765625,
      "activations/layer19_attention_weight_max": 43.50864028930664,
      "activations/layer19_attention_weight_min": -32.26023483276367,
      "activations/layer1_attention_weight_max": 17.605422973632812,
      "activations/layer1_attention_weight_min": -13.389799118041992,
      "activations/layer20_attention_weight_max": 36.390357971191406,
      "activations/layer20_attention_weight_min": -23.97548484802246,
      "activations/layer21_attention_weight_max": 33.249549865722656,
      "activations/layer21_attention_weight_min": -21.767332077026367,
      "activations/layer22_attention_weight_max": 51.49051284790039,
      "activations/layer22_attention_weight_min": -28.5035400390625,
      "activations/layer23_attention_weight_max": 39.65755844116211,
      "activations/layer23_attention_weight_min": -23.768537521362305,
      "activations/layer2_attention_weight_max": 33.30712127685547,
      "activations/layer2_attention_weight_min": -33.59593963623047,
      "activations/layer3_attention_weight_max": 99.52820587158203,
      "activations/layer3_attention_weight_min": -102.01959991455078,
      "activations/layer4_attention_weight_max": 126.2117919921875,
      "activations/layer4_attention_weight_min": -126.7837142944336,
      "activations/layer5_attention_weight_max": 61.545562744140625,
      "activations/layer5_attention_weight_min": -60.89420700073242,
      "activations/layer6_attention_weight_max": 53.22281265258789,
      "activations/layer6_attention_weight_min": -51.910255432128906,
      "activations/layer7_attention_weight_max": 96.84783935546875,
      "activations/layer7_attention_weight_min": -85.7083740234375,
      "activations/layer8_attention_weight_max": 41.447608947753906,
      "activations/layer8_attention_weight_min": -39.93518829345703,
      "activations/layer9_attention_weight_max": 36.080928802490234,
      "activations/layer9_attention_weight_min": -32.54465866088867,
      "epoch": 7.66,
      "learning_rate": 0.00010165416666666666,
      "loss": 2.8056,
      "step": 131750
    },
    {
      "activations/layer0_attention_weight_max": 16.341020584106445,
      "activations/layer0_attention_weight_min": -11.595481872558594,
      "activations/layer10_attention_weight_max": 34.09603500366211,
      "activations/layer10_attention_weight_min": -33.22161102294922,
      "activations/layer11_attention_weight_max": 32.785057067871094,
      "activations/layer11_attention_weight_min": -29.72675323486328,
      "activations/layer12_attention_weight_max": 23.059064865112305,
      "activations/layer12_attention_weight_min": -28.030664443969727,
      "activations/layer13_attention_weight_max": 38.589744567871094,
      "activations/layer13_attention_weight_min": -34.50525665283203,
      "activations/layer14_attention_weight_max": 52.73668670654297,
      "activations/layer14_attention_weight_min": -42.63858413696289,
      "activations/layer15_attention_weight_max": 34.847938537597656,
      "activations/layer15_attention_weight_min": -30.435401916503906,
      "activations/layer16_attention_weight_max": 31.94944953918457,
      "activations/layer16_attention_weight_min": -29.809343338012695,
      "activations/layer17_attention_weight_max": 35.684993743896484,
      "activations/layer17_attention_weight_min": -26.634422302246094,
      "activations/layer18_attention_weight_max": 31.193105697631836,
      "activations/layer18_attention_weight_min": -20.34581184387207,
      "activations/layer19_attention_weight_max": 37.66064453125,
      "activations/layer19_attention_weight_min": -28.334531784057617,
      "activations/layer1_attention_weight_max": 16.300466537475586,
      "activations/layer1_attention_weight_min": -13.567441940307617,
      "activations/layer20_attention_weight_max": 34.59972381591797,
      "activations/layer20_attention_weight_min": -24.176279067993164,
      "activations/layer21_attention_weight_max": 32.22840118408203,
      "activations/layer21_attention_weight_min": -21.71703338623047,
      "activations/layer22_attention_weight_max": 52.70503616333008,
      "activations/layer22_attention_weight_min": -28.84499740600586,
      "activations/layer23_attention_weight_max": 41.452842712402344,
      "activations/layer23_attention_weight_min": -21.661060333251953,
      "activations/layer2_attention_weight_max": 32.1168212890625,
      "activations/layer2_attention_weight_min": -32.47503662109375,
      "activations/layer3_attention_weight_max": 90.726318359375,
      "activations/layer3_attention_weight_min": -95.51174926757812,
      "activations/layer4_attention_weight_max": 119.8229751586914,
      "activations/layer4_attention_weight_min": -115.92866516113281,
      "activations/layer5_attention_weight_max": 59.248741149902344,
      "activations/layer5_attention_weight_min": -61.339881896972656,
      "activations/layer6_attention_weight_max": 49.26683044433594,
      "activations/layer6_attention_weight_min": -50.904659271240234,
      "activations/layer7_attention_weight_max": 90.75989532470703,
      "activations/layer7_attention_weight_min": -90.97221374511719,
      "activations/layer8_attention_weight_max": 40.22111892700195,
      "activations/layer8_attention_weight_min": -39.4608268737793,
      "activations/layer9_attention_weight_max": 37.114707946777344,
      "activations/layer9_attention_weight_min": -36.50536346435547,
      "epoch": 7.66,
      "learning_rate": 0.00010163522727272726,
      "loss": 2.8124,
      "step": 131800
    },
    {
      "activations/layer0_attention_weight_max": 14.8970308303833,
      "activations/layer0_attention_weight_min": -11.477499961853027,
      "activations/layer10_attention_weight_max": 32.43667221069336,
      "activations/layer10_attention_weight_min": -32.42470169067383,
      "activations/layer11_attention_weight_max": 31.16071891784668,
      "activations/layer11_attention_weight_min": -29.49116325378418,
      "activations/layer12_attention_weight_max": 26.068933486938477,
      "activations/layer12_attention_weight_min": -25.907445907592773,
      "activations/layer13_attention_weight_max": 37.58802032470703,
      "activations/layer13_attention_weight_min": -32.02009963989258,
      "activations/layer14_attention_weight_max": 47.52656936645508,
      "activations/layer14_attention_weight_min": -39.55976486206055,
      "activations/layer15_attention_weight_max": 34.7775764465332,
      "activations/layer15_attention_weight_min": -29.099090576171875,
      "activations/layer16_attention_weight_max": 34.18724060058594,
      "activations/layer16_attention_weight_min": -28.97397804260254,
      "activations/layer17_attention_weight_max": 32.41984939575195,
      "activations/layer17_attention_weight_min": -23.686803817749023,
      "activations/layer18_attention_weight_max": 30.858516693115234,
      "activations/layer18_attention_weight_min": -20.54407501220703,
      "activations/layer19_attention_weight_max": 37.62371826171875,
      "activations/layer19_attention_weight_min": -29.513916015625,
      "activations/layer1_attention_weight_max": 17.843366622924805,
      "activations/layer1_attention_weight_min": -14.86716365814209,
      "activations/layer20_attention_weight_max": 29.058738708496094,
      "activations/layer20_attention_weight_min": -25.092479705810547,
      "activations/layer21_attention_weight_max": 27.997093200683594,
      "activations/layer21_attention_weight_min": -22.52587127685547,
      "activations/layer22_attention_weight_max": 41.495235443115234,
      "activations/layer22_attention_weight_min": -28.14727783203125,
      "activations/layer23_attention_weight_max": 34.5192985534668,
      "activations/layer23_attention_weight_min": -22.691556930541992,
      "activations/layer2_attention_weight_max": 32.39799499511719,
      "activations/layer2_attention_weight_min": -32.20947265625,
      "activations/layer3_attention_weight_max": 93.29753875732422,
      "activations/layer3_attention_weight_min": -93.42112731933594,
      "activations/layer4_attention_weight_max": 121.19891357421875,
      "activations/layer4_attention_weight_min": -119.5767822265625,
      "activations/layer5_attention_weight_max": 59.66253662109375,
      "activations/layer5_attention_weight_min": -62.67092514038086,
      "activations/layer6_attention_weight_max": 49.27848815917969,
      "activations/layer6_attention_weight_min": -52.99158477783203,
      "activations/layer7_attention_weight_max": 88.30995178222656,
      "activations/layer7_attention_weight_min": -88.5208969116211,
      "activations/layer8_attention_weight_max": 38.0363655090332,
      "activations/layer8_attention_weight_min": -42.02732849121094,
      "activations/layer9_attention_weight_max": 32.568302154541016,
      "activations/layer9_attention_weight_min": -34.88444519042969,
      "epoch": 7.66,
      "learning_rate": 0.00010161628787878788,
      "loss": 2.8056,
      "step": 131850
    },
    {
      "activations/layer0_attention_weight_max": 16.343746185302734,
      "activations/layer0_attention_weight_min": -10.907179832458496,
      "activations/layer10_attention_weight_max": 31.816049575805664,
      "activations/layer10_attention_weight_min": -33.51800537109375,
      "activations/layer11_attention_weight_max": 31.75560760498047,
      "activations/layer11_attention_weight_min": -29.452970504760742,
      "activations/layer12_attention_weight_max": 53.24338150024414,
      "activations/layer12_attention_weight_min": -24.66448402404785,
      "activations/layer13_attention_weight_max": 52.44062423706055,
      "activations/layer13_attention_weight_min": -30.497142791748047,
      "activations/layer14_attention_weight_max": 50.343223571777344,
      "activations/layer14_attention_weight_min": -37.9984016418457,
      "activations/layer15_attention_weight_max": 35.995853424072266,
      "activations/layer15_attention_weight_min": -30.017427444458008,
      "activations/layer16_attention_weight_max": 33.710670471191406,
      "activations/layer16_attention_weight_min": -26.84528350830078,
      "activations/layer17_attention_weight_max": 35.193214416503906,
      "activations/layer17_attention_weight_min": -24.5521297454834,
      "activations/layer18_attention_weight_max": 34.068824768066406,
      "activations/layer18_attention_weight_min": -21.920564651489258,
      "activations/layer19_attention_weight_max": 37.52428436279297,
      "activations/layer19_attention_weight_min": -31.38142967224121,
      "activations/layer1_attention_weight_max": 16.781293869018555,
      "activations/layer1_attention_weight_min": -14.840396881103516,
      "activations/layer20_attention_weight_max": 31.564924240112305,
      "activations/layer20_attention_weight_min": -26.5935115814209,
      "activations/layer21_attention_weight_max": 30.68634605407715,
      "activations/layer21_attention_weight_min": -23.194578170776367,
      "activations/layer22_attention_weight_max": 46.26398468017578,
      "activations/layer22_attention_weight_min": -29.956912994384766,
      "activations/layer23_attention_weight_max": 34.56029510498047,
      "activations/layer23_attention_weight_min": -23.36353874206543,
      "activations/layer2_attention_weight_max": 32.85102844238281,
      "activations/layer2_attention_weight_min": -31.7113037109375,
      "activations/layer3_attention_weight_max": 94.37936401367188,
      "activations/layer3_attention_weight_min": -94.21149444580078,
      "activations/layer4_attention_weight_max": 123.44625091552734,
      "activations/layer4_attention_weight_min": -114.7938003540039,
      "activations/layer5_attention_weight_max": 61.29328918457031,
      "activations/layer5_attention_weight_min": -62.51445770263672,
      "activations/layer6_attention_weight_max": 50.46195983886719,
      "activations/layer6_attention_weight_min": -50.409420013427734,
      "activations/layer7_attention_weight_max": 89.5117416381836,
      "activations/layer7_attention_weight_min": -87.6095962524414,
      "activations/layer8_attention_weight_max": 40.5538215637207,
      "activations/layer8_attention_weight_min": -39.63380813598633,
      "activations/layer9_attention_weight_max": 34.30958938598633,
      "activations/layer9_attention_weight_min": -34.42918014526367,
      "epoch": 7.66,
      "learning_rate": 0.00010159734848484848,
      "loss": 2.8284,
      "step": 131900
    },
    {
      "activations/layer0_attention_weight_max": 15.116952896118164,
      "activations/layer0_attention_weight_min": -12.795968055725098,
      "activations/layer10_attention_weight_max": 32.916290283203125,
      "activations/layer10_attention_weight_min": -32.29383850097656,
      "activations/layer11_attention_weight_max": 34.017391204833984,
      "activations/layer11_attention_weight_min": -30.03783416748047,
      "activations/layer12_attention_weight_max": 21.786115646362305,
      "activations/layer12_attention_weight_min": -25.196496963500977,
      "activations/layer13_attention_weight_max": 44.025630950927734,
      "activations/layer13_attention_weight_min": -32.93561935424805,
      "activations/layer14_attention_weight_max": 48.8017578125,
      "activations/layer14_attention_weight_min": -39.20632553100586,
      "activations/layer15_attention_weight_max": 35.252777099609375,
      "activations/layer15_attention_weight_min": -28.76839256286621,
      "activations/layer16_attention_weight_max": 32.32126235961914,
      "activations/layer16_attention_weight_min": -26.5933780670166,
      "activations/layer17_attention_weight_max": 34.22478103637695,
      "activations/layer17_attention_weight_min": -23.697389602661133,
      "activations/layer18_attention_weight_max": 30.014713287353516,
      "activations/layer18_attention_weight_min": -20.735660552978516,
      "activations/layer19_attention_weight_max": 35.41117477416992,
      "activations/layer19_attention_weight_min": -28.807889938354492,
      "activations/layer1_attention_weight_max": 18.6469669342041,
      "activations/layer1_attention_weight_min": -14.104293823242188,
      "activations/layer20_attention_weight_max": 28.33995819091797,
      "activations/layer20_attention_weight_min": -23.092693328857422,
      "activations/layer21_attention_weight_max": 27.488616943359375,
      "activations/layer21_attention_weight_min": -21.14642906188965,
      "activations/layer22_attention_weight_max": 45.52299118041992,
      "activations/layer22_attention_weight_min": -27.551321029663086,
      "activations/layer23_attention_weight_max": 38.78491973876953,
      "activations/layer23_attention_weight_min": -22.325382232666016,
      "activations/layer2_attention_weight_max": 33.809059143066406,
      "activations/layer2_attention_weight_min": -32.79487609863281,
      "activations/layer3_attention_weight_max": 95.0303955078125,
      "activations/layer3_attention_weight_min": -98.6995620727539,
      "activations/layer4_attention_weight_max": 120.3724365234375,
      "activations/layer4_attention_weight_min": -119.95014953613281,
      "activations/layer5_attention_weight_max": 60.14971923828125,
      "activations/layer5_attention_weight_min": -61.57964324951172,
      "activations/layer6_attention_weight_max": 52.20136260986328,
      "activations/layer6_attention_weight_min": -51.60313415527344,
      "activations/layer7_attention_weight_max": 95.24188995361328,
      "activations/layer7_attention_weight_min": -83.90226745605469,
      "activations/layer8_attention_weight_max": 41.7359504699707,
      "activations/layer8_attention_weight_min": -39.7020149230957,
      "activations/layer9_attention_weight_max": 34.44284439086914,
      "activations/layer9_attention_weight_min": -35.039390563964844,
      "epoch": 7.67,
      "learning_rate": 0.00010157840909090908,
      "loss": 2.8075,
      "step": 131950
    },
    {
      "activations/layer0_attention_weight_max": 15.966381072998047,
      "activations/layer0_attention_weight_min": -12.064614295959473,
      "activations/layer10_attention_weight_max": 38.345130920410156,
      "activations/layer10_attention_weight_min": -33.23065948486328,
      "activations/layer11_attention_weight_max": 38.692779541015625,
      "activations/layer11_attention_weight_min": -31.32697296142578,
      "activations/layer12_attention_weight_max": 24.859464645385742,
      "activations/layer12_attention_weight_min": -25.986291885375977,
      "activations/layer13_attention_weight_max": 44.24457550048828,
      "activations/layer13_attention_weight_min": -33.70696258544922,
      "activations/layer14_attention_weight_max": 54.817527770996094,
      "activations/layer14_attention_weight_min": -38.145164489746094,
      "activations/layer15_attention_weight_max": 36.18876266479492,
      "activations/layer15_attention_weight_min": -29.812026977539062,
      "activations/layer16_attention_weight_max": 33.08401870727539,
      "activations/layer16_attention_weight_min": -26.627283096313477,
      "activations/layer17_attention_weight_max": 35.69656753540039,
      "activations/layer17_attention_weight_min": -25.023231506347656,
      "activations/layer18_attention_weight_max": 33.13613510131836,
      "activations/layer18_attention_weight_min": -22.27950096130371,
      "activations/layer19_attention_weight_max": 39.64939880371094,
      "activations/layer19_attention_weight_min": -30.080575942993164,
      "activations/layer1_attention_weight_max": 17.554990768432617,
      "activations/layer1_attention_weight_min": -13.6003999710083,
      "activations/layer20_attention_weight_max": 34.435546875,
      "activations/layer20_attention_weight_min": -24.56954002380371,
      "activations/layer21_attention_weight_max": 30.985191345214844,
      "activations/layer21_attention_weight_min": -21.534088134765625,
      "activations/layer22_attention_weight_max": 45.492210388183594,
      "activations/layer22_attention_weight_min": -28.413097381591797,
      "activations/layer23_attention_weight_max": 34.62103271484375,
      "activations/layer23_attention_weight_min": -23.374832153320312,
      "activations/layer2_attention_weight_max": 32.582115173339844,
      "activations/layer2_attention_weight_min": -32.785133361816406,
      "activations/layer3_attention_weight_max": 92.24726867675781,
      "activations/layer3_attention_weight_min": -95.67530822753906,
      "activations/layer4_attention_weight_max": 126.95101165771484,
      "activations/layer4_attention_weight_min": -122.46382141113281,
      "activations/layer5_attention_weight_max": 67.31172943115234,
      "activations/layer5_attention_weight_min": -65.81524658203125,
      "activations/layer6_attention_weight_max": 57.96425247192383,
      "activations/layer6_attention_weight_min": -58.7364387512207,
      "activations/layer7_attention_weight_max": 97.92804718017578,
      "activations/layer7_attention_weight_min": -98.82858276367188,
      "activations/layer8_attention_weight_max": 46.694091796875,
      "activations/layer8_attention_weight_min": -44.091285705566406,
      "activations/layer9_attention_weight_max": 40.20127487182617,
      "activations/layer9_attention_weight_min": -35.32468032836914,
      "epoch": 7.67,
      "learning_rate": 0.0001015594696969697,
      "loss": 2.8051,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_loss": 2.763671875,
      "eval_runtime": 8.4852,
      "eval_samples_per_second": 506.061,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_openwebtext_loss": 2.763671875,
      "eval_openwebtext_ppl": 15.85796463907484,
      "eval_openwebtext_runtime": 8.4852,
      "eval_openwebtext_samples_per_second": 506.061,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 1.991,
      "eval_wikitext_samples_per_second": 229.027,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_lambada_loss": 2.681640625,
      "eval_lambada_ppl": 14.609041604294383,
      "eval_lambada_runtime": 9.6036,
      "eval_lambada_samples_per_second": 506.997,
      "step": 132000
    },
    {
      "activations/layer0_attention_weight_max": 16.337570190429688,
      "activations/layer0_attention_weight_min": -12.583518028259277,
      "activations/layer10_attention_weight_max": 31.4931640625,
      "activations/layer10_attention_weight_min": -32.06949234008789,
      "activations/layer11_attention_weight_max": 29.141742706298828,
      "activations/layer11_attention_weight_min": -28.88043212890625,
      "activations/layer12_attention_weight_max": 26.71013069152832,
      "activations/layer12_attention_weight_min": -26.671100616455078,
      "activations/layer13_attention_weight_max": 43.767555236816406,
      "activations/layer13_attention_weight_min": -33.52927780151367,
      "activations/layer14_attention_weight_max": 50.0734977722168,
      "activations/layer14_attention_weight_min": -38.46602249145508,
      "activations/layer15_attention_weight_max": 38.30631637573242,
      "activations/layer15_attention_weight_min": -30.032241821289062,
      "activations/layer16_attention_weight_max": 32.31192398071289,
      "activations/layer16_attention_weight_min": -27.20564842224121,
      "activations/layer17_attention_weight_max": 34.264259338378906,
      "activations/layer17_attention_weight_min": -23.95847511291504,
      "activations/layer18_attention_weight_max": 31.368816375732422,
      "activations/layer18_attention_weight_min": -25.049861907958984,
      "activations/layer19_attention_weight_max": 35.41276168823242,
      "activations/layer19_attention_weight_min": -33.238651275634766,
      "activations/layer1_attention_weight_max": 16.806659698486328,
      "activations/layer1_attention_weight_min": -15.386567115783691,
      "activations/layer20_attention_weight_max": 28.0120792388916,
      "activations/layer20_attention_weight_min": -24.7193546295166,
      "activations/layer21_attention_weight_max": 29.58224105834961,
      "activations/layer21_attention_weight_min": -22.94365692138672,
      "activations/layer22_attention_weight_max": 43.91226577758789,
      "activations/layer22_attention_weight_min": -28.550399780273438,
      "activations/layer23_attention_weight_max": 33.17522430419922,
      "activations/layer23_attention_weight_min": -21.813411712646484,
      "activations/layer2_attention_weight_max": 33.160308837890625,
      "activations/layer2_attention_weight_min": -31.511112213134766,
      "activations/layer3_attention_weight_max": 92.0255126953125,
      "activations/layer3_attention_weight_min": -96.02732849121094,
      "activations/layer4_attention_weight_max": 116.86905670166016,
      "activations/layer4_attention_weight_min": -112.62749481201172,
      "activations/layer5_attention_weight_max": 56.61764144897461,
      "activations/layer5_attention_weight_min": -58.61662292480469,
      "activations/layer6_attention_weight_max": 48.64122772216797,
      "activations/layer6_attention_weight_min": -51.49806213378906,
      "activations/layer7_attention_weight_max": 85.43230438232422,
      "activations/layer7_attention_weight_min": -82.84204864501953,
      "activations/layer8_attention_weight_max": 39.35047149658203,
      "activations/layer8_attention_weight_min": -38.22652816772461,
      "activations/layer9_attention_weight_max": 35.588714599609375,
      "activations/layer9_attention_weight_min": -35.347652435302734,
      "epoch": 7.67,
      "learning_rate": 0.00010154053030303028,
      "loss": 2.8215,
      "step": 132050
    },
    {
      "activations/layer0_attention_weight_max": 14.758094787597656,
      "activations/layer0_attention_weight_min": -12.236540794372559,
      "activations/layer10_attention_weight_max": 35.19778823852539,
      "activations/layer10_attention_weight_min": -34.33955764770508,
      "activations/layer11_attention_weight_max": 34.374900817871094,
      "activations/layer11_attention_weight_min": -33.09654235839844,
      "activations/layer12_attention_weight_max": 31.541418075561523,
      "activations/layer12_attention_weight_min": -31.67034339904785,
      "activations/layer13_attention_weight_max": 47.5965690612793,
      "activations/layer13_attention_weight_min": -33.16988754272461,
      "activations/layer14_attention_weight_max": 58.81755828857422,
      "activations/layer14_attention_weight_min": -41.42841720581055,
      "activations/layer15_attention_weight_max": 45.167484283447266,
      "activations/layer15_attention_weight_min": -32.39267349243164,
      "activations/layer16_attention_weight_max": 42.84855651855469,
      "activations/layer16_attention_weight_min": -28.299564361572266,
      "activations/layer17_attention_weight_max": 40.47441864013672,
      "activations/layer17_attention_weight_min": -27.300369262695312,
      "activations/layer18_attention_weight_max": 33.413028717041016,
      "activations/layer18_attention_weight_min": -22.504562377929688,
      "activations/layer19_attention_weight_max": 45.383094787597656,
      "activations/layer19_attention_weight_min": -28.828632354736328,
      "activations/layer1_attention_weight_max": 17.002826690673828,
      "activations/layer1_attention_weight_min": -15.14266586303711,
      "activations/layer20_attention_weight_max": 37.13750076293945,
      "activations/layer20_attention_weight_min": -21.556474685668945,
      "activations/layer21_attention_weight_max": 38.252079010009766,
      "activations/layer21_attention_weight_min": -20.349533081054688,
      "activations/layer22_attention_weight_max": 56.09004592895508,
      "activations/layer22_attention_weight_min": -26.43185043334961,
      "activations/layer23_attention_weight_max": 39.39228057861328,
      "activations/layer23_attention_weight_min": -20.667240142822266,
      "activations/layer2_attention_weight_max": 33.61998748779297,
      "activations/layer2_attention_weight_min": -32.64354705810547,
      "activations/layer3_attention_weight_max": 90.94682312011719,
      "activations/layer3_attention_weight_min": -96.71533203125,
      "activations/layer4_attention_weight_max": 121.57836151123047,
      "activations/layer4_attention_weight_min": -120.1579360961914,
      "activations/layer5_attention_weight_max": 61.2672233581543,
      "activations/layer5_attention_weight_min": -62.000335693359375,
      "activations/layer6_attention_weight_max": 50.56587600708008,
      "activations/layer6_attention_weight_min": -54.2331657409668,
      "activations/layer7_attention_weight_max": 97.95259094238281,
      "activations/layer7_attention_weight_min": -96.66614532470703,
      "activations/layer8_attention_weight_max": 42.28022766113281,
      "activations/layer8_attention_weight_min": -44.58610534667969,
      "activations/layer9_attention_weight_max": 39.08767318725586,
      "activations/layer9_attention_weight_min": -36.83700180053711,
      "epoch": 7.68,
      "learning_rate": 0.0001015215909090909,
      "loss": 2.8007,
      "step": 132100
    },
    {
      "activations/layer0_attention_weight_max": 17.18876075744629,
      "activations/layer0_attention_weight_min": -12.27009391784668,
      "activations/layer10_attention_weight_max": 30.43282127380371,
      "activations/layer10_attention_weight_min": -31.60333824157715,
      "activations/layer11_attention_weight_max": 31.5056209564209,
      "activations/layer11_attention_weight_min": -29.109943389892578,
      "activations/layer12_attention_weight_max": 22.660863876342773,
      "activations/layer12_attention_weight_min": -29.578336715698242,
      "activations/layer13_attention_weight_max": 36.599449157714844,
      "activations/layer13_attention_weight_min": -32.81114959716797,
      "activations/layer14_attention_weight_max": 54.908546447753906,
      "activations/layer14_attention_weight_min": -46.126285552978516,
      "activations/layer15_attention_weight_max": 33.32434844970703,
      "activations/layer15_attention_weight_min": -28.914077758789062,
      "activations/layer16_attention_weight_max": 30.867870330810547,
      "activations/layer16_attention_weight_min": -26.777210235595703,
      "activations/layer17_attention_weight_max": 30.21164894104004,
      "activations/layer17_attention_weight_min": -24.179080963134766,
      "activations/layer18_attention_weight_max": 31.586288452148438,
      "activations/layer18_attention_weight_min": -21.3338680267334,
      "activations/layer19_attention_weight_max": 36.202880859375,
      "activations/layer19_attention_weight_min": -28.810144424438477,
      "activations/layer1_attention_weight_max": 17.263896942138672,
      "activations/layer1_attention_weight_min": -15.986370086669922,
      "activations/layer20_attention_weight_max": 29.480894088745117,
      "activations/layer20_attention_weight_min": -23.52513313293457,
      "activations/layer21_attention_weight_max": 29.443571090698242,
      "activations/layer21_attention_weight_min": -21.56182289123535,
      "activations/layer22_attention_weight_max": 47.41362762451172,
      "activations/layer22_attention_weight_min": -30.427942276000977,
      "activations/layer23_attention_weight_max": 34.86642074584961,
      "activations/layer23_attention_weight_min": -24.379615783691406,
      "activations/layer2_attention_weight_max": 30.546573638916016,
      "activations/layer2_attention_weight_min": -29.20772361755371,
      "activations/layer3_attention_weight_max": 82.32064056396484,
      "activations/layer3_attention_weight_min": -88.68518829345703,
      "activations/layer4_attention_weight_max": 113.41703033447266,
      "activations/layer4_attention_weight_min": -115.64347076416016,
      "activations/layer5_attention_weight_max": 56.81208038330078,
      "activations/layer5_attention_weight_min": -63.45585632324219,
      "activations/layer6_attention_weight_max": 46.164852142333984,
      "activations/layer6_attention_weight_min": -51.0390739440918,
      "activations/layer7_attention_weight_max": 88.1142578125,
      "activations/layer7_attention_weight_min": -87.5634765625,
      "activations/layer8_attention_weight_max": 39.41592025756836,
      "activations/layer8_attention_weight_min": -40.42040252685547,
      "activations/layer9_attention_weight_max": 33.37119674682617,
      "activations/layer9_attention_weight_min": -34.164363861083984,
      "epoch": 7.68,
      "learning_rate": 0.00010150303030303029,
      "loss": 2.793,
      "step": 132150
    },
    {
      "activations/layer0_attention_weight_max": 16.845783233642578,
      "activations/layer0_attention_weight_min": -13.040478706359863,
      "activations/layer10_attention_weight_max": 33.00660705566406,
      "activations/layer10_attention_weight_min": -30.259422302246094,
      "activations/layer11_attention_weight_max": 30.73497772216797,
      "activations/layer11_attention_weight_min": -30.860427856445312,
      "activations/layer12_attention_weight_max": 22.71832275390625,
      "activations/layer12_attention_weight_min": -25.233306884765625,
      "activations/layer13_attention_weight_max": 36.82853698730469,
      "activations/layer13_attention_weight_min": -31.146486282348633,
      "activations/layer14_attention_weight_max": 49.10103225708008,
      "activations/layer14_attention_weight_min": -36.94929504394531,
      "activations/layer15_attention_weight_max": 35.67647933959961,
      "activations/layer15_attention_weight_min": -27.476411819458008,
      "activations/layer16_attention_weight_max": 32.298828125,
      "activations/layer16_attention_weight_min": -26.86878204345703,
      "activations/layer17_attention_weight_max": 31.80770492553711,
      "activations/layer17_attention_weight_min": -25.207548141479492,
      "activations/layer18_attention_weight_max": 33.296600341796875,
      "activations/layer18_attention_weight_min": -23.160879135131836,
      "activations/layer19_attention_weight_max": 36.72039794921875,
      "activations/layer19_attention_weight_min": -31.4298095703125,
      "activations/layer1_attention_weight_max": 16.14981460571289,
      "activations/layer1_attention_weight_min": -14.920663833618164,
      "activations/layer20_attention_weight_max": 30.143535614013672,
      "activations/layer20_attention_weight_min": -23.696489334106445,
      "activations/layer21_attention_weight_max": 28.977224349975586,
      "activations/layer21_attention_weight_min": -22.07539939880371,
      "activations/layer22_attention_weight_max": 50.34695816040039,
      "activations/layer22_attention_weight_min": -29.243019104003906,
      "activations/layer23_attention_weight_max": 35.636329650878906,
      "activations/layer23_attention_weight_min": -22.919803619384766,
      "activations/layer2_attention_weight_max": 32.292076110839844,
      "activations/layer2_attention_weight_min": -30.551143646240234,
      "activations/layer3_attention_weight_max": 89.68733978271484,
      "activations/layer3_attention_weight_min": -91.97752380371094,
      "activations/layer4_attention_weight_max": 120.60786437988281,
      "activations/layer4_attention_weight_min": -117.77469635009766,
      "activations/layer5_attention_weight_max": 57.839393615722656,
      "activations/layer5_attention_weight_min": -61.43561935424805,
      "activations/layer6_attention_weight_max": 53.635196685791016,
      "activations/layer6_attention_weight_min": -51.024192810058594,
      "activations/layer7_attention_weight_max": 88.85643005371094,
      "activations/layer7_attention_weight_min": -91.52993774414062,
      "activations/layer8_attention_weight_max": 40.49751663208008,
      "activations/layer8_attention_weight_min": -42.43452835083008,
      "activations/layer9_attention_weight_max": 34.8924446105957,
      "activations/layer9_attention_weight_min": -34.28774642944336,
      "epoch": 7.68,
      "learning_rate": 0.0001014840909090909,
      "loss": 2.8142,
      "step": 132200
    },
    {
      "activations/layer0_attention_weight_max": 16.593109130859375,
      "activations/layer0_attention_weight_min": -12.33997917175293,
      "activations/layer10_attention_weight_max": 35.073333740234375,
      "activations/layer10_attention_weight_min": -34.10868835449219,
      "activations/layer11_attention_weight_max": 36.910423278808594,
      "activations/layer11_attention_weight_min": -32.691688537597656,
      "activations/layer12_attention_weight_max": 22.94266128540039,
      "activations/layer12_attention_weight_min": -28.17466163635254,
      "activations/layer13_attention_weight_max": 42.955814361572266,
      "activations/layer13_attention_weight_min": -33.47425079345703,
      "activations/layer14_attention_weight_max": 56.8287353515625,
      "activations/layer14_attention_weight_min": -44.560428619384766,
      "activations/layer15_attention_weight_max": 35.833457946777344,
      "activations/layer15_attention_weight_min": -30.024925231933594,
      "activations/layer16_attention_weight_max": 34.416648864746094,
      "activations/layer16_attention_weight_min": -29.593740463256836,
      "activations/layer17_attention_weight_max": 35.59681701660156,
      "activations/layer17_attention_weight_min": -26.252269744873047,
      "activations/layer18_attention_weight_max": 30.507272720336914,
      "activations/layer18_attention_weight_min": -20.49829864501953,
      "activations/layer19_attention_weight_max": 43.73881149291992,
      "activations/layer19_attention_weight_min": -29.091983795166016,
      "activations/layer1_attention_weight_max": 17.486858367919922,
      "activations/layer1_attention_weight_min": -15.897522926330566,
      "activations/layer20_attention_weight_max": 30.030017852783203,
      "activations/layer20_attention_weight_min": -22.655807495117188,
      "activations/layer21_attention_weight_max": 27.777055740356445,
      "activations/layer21_attention_weight_min": -20.563369750976562,
      "activations/layer22_attention_weight_max": 47.93646240234375,
      "activations/layer22_attention_weight_min": -28.09099769592285,
      "activations/layer23_attention_weight_max": 35.20156478881836,
      "activations/layer23_attention_weight_min": -22.907438278198242,
      "activations/layer2_attention_weight_max": 32.30313491821289,
      "activations/layer2_attention_weight_min": -32.29833984375,
      "activations/layer3_attention_weight_max": 90.89308166503906,
      "activations/layer3_attention_weight_min": -96.38172912597656,
      "activations/layer4_attention_weight_max": 120.0965576171875,
      "activations/layer4_attention_weight_min": -123.81957244873047,
      "activations/layer5_attention_weight_max": 58.63983917236328,
      "activations/layer5_attention_weight_min": -64.51213073730469,
      "activations/layer6_attention_weight_max": 50.29780960083008,
      "activations/layer6_attention_weight_min": -53.33211898803711,
      "activations/layer7_attention_weight_max": 103.91868591308594,
      "activations/layer7_attention_weight_min": -98.71289825439453,
      "activations/layer8_attention_weight_max": 42.64302444458008,
      "activations/layer8_attention_weight_min": -42.17597961425781,
      "activations/layer9_attention_weight_max": 36.078346252441406,
      "activations/layer9_attention_weight_min": -35.39664077758789,
      "epoch": 7.68,
      "learning_rate": 0.0001014651515151515,
      "loss": 2.8058,
      "step": 132250
    },
    {
      "activations/layer0_attention_weight_max": 16.198928833007812,
      "activations/layer0_attention_weight_min": -12.840239524841309,
      "activations/layer10_attention_weight_max": 33.25290298461914,
      "activations/layer10_attention_weight_min": -34.88594055175781,
      "activations/layer11_attention_weight_max": 32.081199645996094,
      "activations/layer11_attention_weight_min": -31.3914794921875,
      "activations/layer12_attention_weight_max": 23.114891052246094,
      "activations/layer12_attention_weight_min": -24.187334060668945,
      "activations/layer13_attention_weight_max": 41.60441207885742,
      "activations/layer13_attention_weight_min": -32.07643127441406,
      "activations/layer14_attention_weight_max": 52.56685256958008,
      "activations/layer14_attention_weight_min": -42.591556549072266,
      "activations/layer15_attention_weight_max": 36.53651809692383,
      "activations/layer15_attention_weight_min": -31.2490177154541,
      "activations/layer16_attention_weight_max": 34.76967239379883,
      "activations/layer16_attention_weight_min": -25.89521026611328,
      "activations/layer17_attention_weight_max": 35.84185028076172,
      "activations/layer17_attention_weight_min": -25.180631637573242,
      "activations/layer18_attention_weight_max": 32.669342041015625,
      "activations/layer18_attention_weight_min": -20.123125076293945,
      "activations/layer19_attention_weight_max": 36.38547897338867,
      "activations/layer19_attention_weight_min": -29.379840850830078,
      "activations/layer1_attention_weight_max": 16.45210838317871,
      "activations/layer1_attention_weight_min": -14.303756713867188,
      "activations/layer20_attention_weight_max": 33.020442962646484,
      "activations/layer20_attention_weight_min": -24.94610023498535,
      "activations/layer21_attention_weight_max": 31.660606384277344,
      "activations/layer21_attention_weight_min": -22.1602840423584,
      "activations/layer22_attention_weight_max": 49.725502014160156,
      "activations/layer22_attention_weight_min": -29.86348533630371,
      "activations/layer23_attention_weight_max": 38.30574035644531,
      "activations/layer23_attention_weight_min": -25.331907272338867,
      "activations/layer2_attention_weight_max": 30.64851188659668,
      "activations/layer2_attention_weight_min": -29.814815521240234,
      "activations/layer3_attention_weight_max": 88.94510650634766,
      "activations/layer3_attention_weight_min": -96.0565414428711,
      "activations/layer4_attention_weight_max": 118.00244140625,
      "activations/layer4_attention_weight_min": -116.92559814453125,
      "activations/layer5_attention_weight_max": 59.366844177246094,
      "activations/layer5_attention_weight_min": -62.94770431518555,
      "activations/layer6_attention_weight_max": 50.80900573730469,
      "activations/layer6_attention_weight_min": -52.56980514526367,
      "activations/layer7_attention_weight_max": 86.4403305053711,
      "activations/layer7_attention_weight_min": -86.91060638427734,
      "activations/layer8_attention_weight_max": 39.12236404418945,
      "activations/layer8_attention_weight_min": -39.92454147338867,
      "activations/layer9_attention_weight_max": 36.39046859741211,
      "activations/layer9_attention_weight_min": -36.0184440612793,
      "epoch": 7.69,
      "learning_rate": 0.0001014462121212121,
      "loss": 2.8126,
      "step": 132300
    },
    {
      "activations/layer0_attention_weight_max": 16.366823196411133,
      "activations/layer0_attention_weight_min": -11.485332489013672,
      "activations/layer10_attention_weight_max": 42.11647415161133,
      "activations/layer10_attention_weight_min": -32.950439453125,
      "activations/layer11_attention_weight_max": 39.96110916137695,
      "activations/layer11_attention_weight_min": -33.92543411254883,
      "activations/layer12_attention_weight_max": 26.468725204467773,
      "activations/layer12_attention_weight_min": -29.719057083129883,
      "activations/layer13_attention_weight_max": 42.734840393066406,
      "activations/layer13_attention_weight_min": -33.2479133605957,
      "activations/layer14_attention_weight_max": 60.5135383605957,
      "activations/layer14_attention_weight_min": -47.63640213012695,
      "activations/layer15_attention_weight_max": 45.9146614074707,
      "activations/layer15_attention_weight_min": -31.66410255432129,
      "activations/layer16_attention_weight_max": 31.89134979248047,
      "activations/layer16_attention_weight_min": -27.179092407226562,
      "activations/layer17_attention_weight_max": 32.742340087890625,
      "activations/layer17_attention_weight_min": -25.02004051208496,
      "activations/layer18_attention_weight_max": 34.09130096435547,
      "activations/layer18_attention_weight_min": -22.34612274169922,
      "activations/layer19_attention_weight_max": 33.52001190185547,
      "activations/layer19_attention_weight_min": -31.62312126159668,
      "activations/layer1_attention_weight_max": 16.896106719970703,
      "activations/layer1_attention_weight_min": -14.267687797546387,
      "activations/layer20_attention_weight_max": 29.302560806274414,
      "activations/layer20_attention_weight_min": -25.05999755859375,
      "activations/layer21_attention_weight_max": 30.95419692993164,
      "activations/layer21_attention_weight_min": -23.116863250732422,
      "activations/layer22_attention_weight_max": 46.870845794677734,
      "activations/layer22_attention_weight_min": -31.084484100341797,
      "activations/layer23_attention_weight_max": 33.98157501220703,
      "activations/layer23_attention_weight_min": -23.44161033630371,
      "activations/layer2_attention_weight_max": 32.99950408935547,
      "activations/layer2_attention_weight_min": -32.59687042236328,
      "activations/layer3_attention_weight_max": 94.05953979492188,
      "activations/layer3_attention_weight_min": -96.08797454833984,
      "activations/layer4_attention_weight_max": 125.26325988769531,
      "activations/layer4_attention_weight_min": -116.53185272216797,
      "activations/layer5_attention_weight_max": 59.47815704345703,
      "activations/layer5_attention_weight_min": -62.44956588745117,
      "activations/layer6_attention_weight_max": 51.65708541870117,
      "activations/layer6_attention_weight_min": -50.78364944458008,
      "activations/layer7_attention_weight_max": 98.17724609375,
      "activations/layer7_attention_weight_min": -87.05435943603516,
      "activations/layer8_attention_weight_max": 44.057796478271484,
      "activations/layer8_attention_weight_min": -40.24671936035156,
      "activations/layer9_attention_weight_max": 40.649818420410156,
      "activations/layer9_attention_weight_min": -36.86531448364258,
      "epoch": 7.69,
      "learning_rate": 0.00010142727272727272,
      "loss": 2.7952,
      "step": 132350
    },
    {
      "activations/layer0_attention_weight_max": 15.501016616821289,
      "activations/layer0_attention_weight_min": -11.61101245880127,
      "activations/layer10_attention_weight_max": 31.484167098999023,
      "activations/layer10_attention_weight_min": -31.514890670776367,
      "activations/layer11_attention_weight_max": 30.93600082397461,
      "activations/layer11_attention_weight_min": -28.982688903808594,
      "activations/layer12_attention_weight_max": 20.853591918945312,
      "activations/layer12_attention_weight_min": -29.023380279541016,
      "activations/layer13_attention_weight_max": 39.025634765625,
      "activations/layer13_attention_weight_min": -31.732728958129883,
      "activations/layer14_attention_weight_max": 54.27405548095703,
      "activations/layer14_attention_weight_min": -40.77785873413086,
      "activations/layer15_attention_weight_max": 34.834407806396484,
      "activations/layer15_attention_weight_min": -28.823442459106445,
      "activations/layer16_attention_weight_max": 32.195987701416016,
      "activations/layer16_attention_weight_min": -29.001161575317383,
      "activations/layer17_attention_weight_max": 32.702606201171875,
      "activations/layer17_attention_weight_min": -26.12158966064453,
      "activations/layer18_attention_weight_max": 30.400915145874023,
      "activations/layer18_attention_weight_min": -21.09243392944336,
      "activations/layer19_attention_weight_max": 34.476234436035156,
      "activations/layer19_attention_weight_min": -30.570966720581055,
      "activations/layer1_attention_weight_max": 16.526569366455078,
      "activations/layer1_attention_weight_min": -15.795722007751465,
      "activations/layer20_attention_weight_max": 31.938989639282227,
      "activations/layer20_attention_weight_min": -22.83140754699707,
      "activations/layer21_attention_weight_max": 29.82623291015625,
      "activations/layer21_attention_weight_min": -22.07909393310547,
      "activations/layer22_attention_weight_max": 49.89262008666992,
      "activations/layer22_attention_weight_min": -28.276439666748047,
      "activations/layer23_attention_weight_max": 36.484100341796875,
      "activations/layer23_attention_weight_min": -23.140830993652344,
      "activations/layer2_attention_weight_max": 32.588279724121094,
      "activations/layer2_attention_weight_min": -32.6569709777832,
      "activations/layer3_attention_weight_max": 95.40705108642578,
      "activations/layer3_attention_weight_min": -98.84652709960938,
      "activations/layer4_attention_weight_max": 120.06671142578125,
      "activations/layer4_attention_weight_min": -121.19693756103516,
      "activations/layer5_attention_weight_max": 60.03926086425781,
      "activations/layer5_attention_weight_min": -60.98528289794922,
      "activations/layer6_attention_weight_max": 49.90764236450195,
      "activations/layer6_attention_weight_min": -53.78779983520508,
      "activations/layer7_attention_weight_max": 91.96322631835938,
      "activations/layer7_attention_weight_min": -94.87545013427734,
      "activations/layer8_attention_weight_max": 40.66716384887695,
      "activations/layer8_attention_weight_min": -42.20907211303711,
      "activations/layer9_attention_weight_max": 34.559871673583984,
      "activations/layer9_attention_weight_min": -34.45912551879883,
      "epoch": 7.69,
      "learning_rate": 0.00010140833333333332,
      "loss": 2.818,
      "step": 132400
    },
    {
      "activations/layer0_attention_weight_max": 15.872541427612305,
      "activations/layer0_attention_weight_min": -11.729547500610352,
      "activations/layer10_attention_weight_max": 33.78577423095703,
      "activations/layer10_attention_weight_min": -29.84025001525879,
      "activations/layer11_attention_weight_max": 34.06069564819336,
      "activations/layer11_attention_weight_min": -27.874374389648438,
      "activations/layer12_attention_weight_max": 24.900074005126953,
      "activations/layer12_attention_weight_min": -27.379016876220703,
      "activations/layer13_attention_weight_max": 38.67076110839844,
      "activations/layer13_attention_weight_min": -33.40095520019531,
      "activations/layer14_attention_weight_max": 48.74295425415039,
      "activations/layer14_attention_weight_min": -41.25089645385742,
      "activations/layer15_attention_weight_max": 37.766265869140625,
      "activations/layer15_attention_weight_min": -28.91283416748047,
      "activations/layer16_attention_weight_max": 33.96807861328125,
      "activations/layer16_attention_weight_min": -29.299713134765625,
      "activations/layer17_attention_weight_max": 31.571598052978516,
      "activations/layer17_attention_weight_min": -26.293027877807617,
      "activations/layer18_attention_weight_max": 31.122093200683594,
      "activations/layer18_attention_weight_min": -22.6502685546875,
      "activations/layer19_attention_weight_max": 36.98516845703125,
      "activations/layer19_attention_weight_min": -29.982303619384766,
      "activations/layer1_attention_weight_max": 16.294063568115234,
      "activations/layer1_attention_weight_min": -13.684429168701172,
      "activations/layer20_attention_weight_max": 33.0707893371582,
      "activations/layer20_attention_weight_min": -26.000505447387695,
      "activations/layer21_attention_weight_max": 31.89366912841797,
      "activations/layer21_attention_weight_min": -23.36079978942871,
      "activations/layer22_attention_weight_max": 44.63162612915039,
      "activations/layer22_attention_weight_min": -31.017873764038086,
      "activations/layer23_attention_weight_max": 34.39896011352539,
      "activations/layer23_attention_weight_min": -25.593914031982422,
      "activations/layer2_attention_weight_max": 30.687551498413086,
      "activations/layer2_attention_weight_min": -31.01010513305664,
      "activations/layer3_attention_weight_max": 90.09420776367188,
      "activations/layer3_attention_weight_min": -91.52830505371094,
      "activations/layer4_attention_weight_max": 115.94661712646484,
      "activations/layer4_attention_weight_min": -120.3289566040039,
      "activations/layer5_attention_weight_max": 57.06544494628906,
      "activations/layer5_attention_weight_min": -63.138580322265625,
      "activations/layer6_attention_weight_max": 48.06951141357422,
      "activations/layer6_attention_weight_min": -53.219139099121094,
      "activations/layer7_attention_weight_max": 90.20084381103516,
      "activations/layer7_attention_weight_min": -93.85360717773438,
      "activations/layer8_attention_weight_max": 38.6152229309082,
      "activations/layer8_attention_weight_min": -39.0938606262207,
      "activations/layer9_attention_weight_max": 33.480323791503906,
      "activations/layer9_attention_weight_min": -33.054969787597656,
      "epoch": 7.7,
      "learning_rate": 0.00010138939393939394,
      "loss": 2.8176,
      "step": 132450
    },
    {
      "activations/layer0_attention_weight_max": 14.924721717834473,
      "activations/layer0_attention_weight_min": -12.694356918334961,
      "activations/layer10_attention_weight_max": 36.452125549316406,
      "activations/layer10_attention_weight_min": -32.45231628417969,
      "activations/layer11_attention_weight_max": 38.707359313964844,
      "activations/layer11_attention_weight_min": -31.717716217041016,
      "activations/layer12_attention_weight_max": 40.03682327270508,
      "activations/layer12_attention_weight_min": -36.5984001159668,
      "activations/layer13_attention_weight_max": 46.94486618041992,
      "activations/layer13_attention_weight_min": -33.45121765136719,
      "activations/layer14_attention_weight_max": 56.37131881713867,
      "activations/layer14_attention_weight_min": -41.86067581176758,
      "activations/layer15_attention_weight_max": 38.14179611206055,
      "activations/layer15_attention_weight_min": -31.13373565673828,
      "activations/layer16_attention_weight_max": 34.97427749633789,
      "activations/layer16_attention_weight_min": -27.0405216217041,
      "activations/layer17_attention_weight_max": 41.30373001098633,
      "activations/layer17_attention_weight_min": -24.010879516601562,
      "activations/layer18_attention_weight_max": 33.39800262451172,
      "activations/layer18_attention_weight_min": -20.545387268066406,
      "activations/layer19_attention_weight_max": 41.29537582397461,
      "activations/layer19_attention_weight_min": -29.201038360595703,
      "activations/layer1_attention_weight_max": 17.271469116210938,
      "activations/layer1_attention_weight_min": -15.033525466918945,
      "activations/layer20_attention_weight_max": 34.902000427246094,
      "activations/layer20_attention_weight_min": -23.135957717895508,
      "activations/layer21_attention_weight_max": 30.094924926757812,
      "activations/layer21_attention_weight_min": -23.205352783203125,
      "activations/layer22_attention_weight_max": 48.808616638183594,
      "activations/layer22_attention_weight_min": -27.590396881103516,
      "activations/layer23_attention_weight_max": 35.931190490722656,
      "activations/layer23_attention_weight_min": -22.66267204284668,
      "activations/layer2_attention_weight_max": 35.120140075683594,
      "activations/layer2_attention_weight_min": -32.98511505126953,
      "activations/layer3_attention_weight_max": 100.61639404296875,
      "activations/layer3_attention_weight_min": -102.37464141845703,
      "activations/layer4_attention_weight_max": 126.85847473144531,
      "activations/layer4_attention_weight_min": -132.08946228027344,
      "activations/layer5_attention_weight_max": 63.70306396484375,
      "activations/layer5_attention_weight_min": -61.63808822631836,
      "activations/layer6_attention_weight_max": 52.59992218017578,
      "activations/layer6_attention_weight_min": -55.1381721496582,
      "activations/layer7_attention_weight_max": 94.7702865600586,
      "activations/layer7_attention_weight_min": -93.82177734375,
      "activations/layer8_attention_weight_max": 43.28308868408203,
      "activations/layer8_attention_weight_min": -43.47648239135742,
      "activations/layer9_attention_weight_max": 39.28866958618164,
      "activations/layer9_attention_weight_min": -37.46883010864258,
      "epoch": 7.7,
      "learning_rate": 0.00010137045454545454,
      "loss": 2.8031,
      "step": 132500
    },
    {
      "activations/layer0_attention_weight_max": 16.580894470214844,
      "activations/layer0_attention_weight_min": -12.210246086120605,
      "activations/layer10_attention_weight_max": 34.911773681640625,
      "activations/layer10_attention_weight_min": -33.6589241027832,
      "activations/layer11_attention_weight_max": 33.46157455444336,
      "activations/layer11_attention_weight_min": -31.459501266479492,
      "activations/layer12_attention_weight_max": 25.701557159423828,
      "activations/layer12_attention_weight_min": -27.213668823242188,
      "activations/layer13_attention_weight_max": 38.53945541381836,
      "activations/layer13_attention_weight_min": -30.936466217041016,
      "activations/layer14_attention_weight_max": 56.17348098754883,
      "activations/layer14_attention_weight_min": -39.071353912353516,
      "activations/layer15_attention_weight_max": 37.20899963378906,
      "activations/layer15_attention_weight_min": -29.555679321289062,
      "activations/layer16_attention_weight_max": 33.40082931518555,
      "activations/layer16_attention_weight_min": -27.068065643310547,
      "activations/layer17_attention_weight_max": 34.057315826416016,
      "activations/layer17_attention_weight_min": -24.195636749267578,
      "activations/layer18_attention_weight_max": 36.434226989746094,
      "activations/layer18_attention_weight_min": -22.00004768371582,
      "activations/layer19_attention_weight_max": 48.66777038574219,
      "activations/layer19_attention_weight_min": -35.52714538574219,
      "activations/layer1_attention_weight_max": 17.30643081665039,
      "activations/layer1_attention_weight_min": -14.201136589050293,
      "activations/layer20_attention_weight_max": 32.53931427001953,
      "activations/layer20_attention_weight_min": -24.152732849121094,
      "activations/layer21_attention_weight_max": 29.745641708374023,
      "activations/layer21_attention_weight_min": -22.25965690612793,
      "activations/layer22_attention_weight_max": 52.29436111450195,
      "activations/layer22_attention_weight_min": -29.9599666595459,
      "activations/layer23_attention_weight_max": 44.25692367553711,
      "activations/layer23_attention_weight_min": -25.198820114135742,
      "activations/layer2_attention_weight_max": 33.89757537841797,
      "activations/layer2_attention_weight_min": -31.773984909057617,
      "activations/layer3_attention_weight_max": 92.18110656738281,
      "activations/layer3_attention_weight_min": -92.69184875488281,
      "activations/layer4_attention_weight_max": 122.73011779785156,
      "activations/layer4_attention_weight_min": -118.00923919677734,
      "activations/layer5_attention_weight_max": 62.424861907958984,
      "activations/layer5_attention_weight_min": -63.284637451171875,
      "activations/layer6_attention_weight_max": 50.794124603271484,
      "activations/layer6_attention_weight_min": -53.94272232055664,
      "activations/layer7_attention_weight_max": 87.08649444580078,
      "activations/layer7_attention_weight_min": -93.24884033203125,
      "activations/layer8_attention_weight_max": 41.62962341308594,
      "activations/layer8_attention_weight_min": -40.408729553222656,
      "activations/layer9_attention_weight_max": 37.411102294921875,
      "activations/layer9_attention_weight_min": -34.35616683959961,
      "epoch": 7.7,
      "learning_rate": 0.00010135151515151514,
      "loss": 2.8235,
      "step": 132550
    },
    {
      "activations/layer0_attention_weight_max": 16.03441619873047,
      "activations/layer0_attention_weight_min": -12.986173629760742,
      "activations/layer10_attention_weight_max": 33.110145568847656,
      "activations/layer10_attention_weight_min": -29.88698387145996,
      "activations/layer11_attention_weight_max": 33.22915267944336,
      "activations/layer11_attention_weight_min": -29.527610778808594,
      "activations/layer12_attention_weight_max": 24.144577026367188,
      "activations/layer12_attention_weight_min": -24.93352699279785,
      "activations/layer13_attention_weight_max": 37.71183395385742,
      "activations/layer13_attention_weight_min": -31.649690628051758,
      "activations/layer14_attention_weight_max": 43.54448699951172,
      "activations/layer14_attention_weight_min": -37.96228790283203,
      "activations/layer15_attention_weight_max": 41.191593170166016,
      "activations/layer15_attention_weight_min": -31.409685134887695,
      "activations/layer16_attention_weight_max": 31.361160278320312,
      "activations/layer16_attention_weight_min": -28.446701049804688,
      "activations/layer17_attention_weight_max": 30.531490325927734,
      "activations/layer17_attention_weight_min": -24.796188354492188,
      "activations/layer18_attention_weight_max": 32.21318054199219,
      "activations/layer18_attention_weight_min": -21.058164596557617,
      "activations/layer19_attention_weight_max": 35.1500244140625,
      "activations/layer19_attention_weight_min": -31.11368751525879,
      "activations/layer1_attention_weight_max": 16.57082176208496,
      "activations/layer1_attention_weight_min": -15.355551719665527,
      "activations/layer20_attention_weight_max": 30.19734764099121,
      "activations/layer20_attention_weight_min": -23.123449325561523,
      "activations/layer21_attention_weight_max": 28.578521728515625,
      "activations/layer21_attention_weight_min": -22.31281280517578,
      "activations/layer22_attention_weight_max": 45.89691925048828,
      "activations/layer22_attention_weight_min": -28.523136138916016,
      "activations/layer23_attention_weight_max": 35.019508361816406,
      "activations/layer23_attention_weight_min": -24.130664825439453,
      "activations/layer2_attention_weight_max": 32.465049743652344,
      "activations/layer2_attention_weight_min": -31.2396297454834,
      "activations/layer3_attention_weight_max": 95.18363189697266,
      "activations/layer3_attention_weight_min": -97.84510803222656,
      "activations/layer4_attention_weight_max": 119.41597747802734,
      "activations/layer4_attention_weight_min": -118.58863830566406,
      "activations/layer5_attention_weight_max": 61.1622314453125,
      "activations/layer5_attention_weight_min": -62.71440124511719,
      "activations/layer6_attention_weight_max": 49.17433166503906,
      "activations/layer6_attention_weight_min": -52.23740005493164,
      "activations/layer7_attention_weight_max": 85.12944793701172,
      "activations/layer7_attention_weight_min": -86.828369140625,
      "activations/layer8_attention_weight_max": 38.75811767578125,
      "activations/layer8_attention_weight_min": -38.36588668823242,
      "activations/layer9_attention_weight_max": 35.12239074707031,
      "activations/layer9_attention_weight_min": -33.115230560302734,
      "epoch": 7.7,
      "learning_rate": 0.00010133257575757576,
      "loss": 2.8178,
      "step": 132600
    },
    {
      "activations/layer0_attention_weight_max": 17.07172966003418,
      "activations/layer0_attention_weight_min": -12.466740608215332,
      "activations/layer10_attention_weight_max": 38.904762268066406,
      "activations/layer10_attention_weight_min": -33.64775848388672,
      "activations/layer11_attention_weight_max": 41.62736892700195,
      "activations/layer11_attention_weight_min": -31.99386215209961,
      "activations/layer12_attention_weight_max": 35.65897750854492,
      "activations/layer12_attention_weight_min": -26.6047420501709,
      "activations/layer13_attention_weight_max": 54.48612976074219,
      "activations/layer13_attention_weight_min": -33.67157745361328,
      "activations/layer14_attention_weight_max": 54.20685958862305,
      "activations/layer14_attention_weight_min": -39.326148986816406,
      "activations/layer15_attention_weight_max": 50.820133209228516,
      "activations/layer15_attention_weight_min": -31.44835090637207,
      "activations/layer16_attention_weight_max": 39.402034759521484,
      "activations/layer16_attention_weight_min": -28.169727325439453,
      "activations/layer17_attention_weight_max": 33.99079132080078,
      "activations/layer17_attention_weight_min": -25.45991325378418,
      "activations/layer18_attention_weight_max": 34.91138458251953,
      "activations/layer18_attention_weight_min": -22.699722290039062,
      "activations/layer19_attention_weight_max": 40.11288070678711,
      "activations/layer19_attention_weight_min": -29.66071128845215,
      "activations/layer1_attention_weight_max": 18.68773078918457,
      "activations/layer1_attention_weight_min": -13.64689826965332,
      "activations/layer20_attention_weight_max": 33.81147003173828,
      "activations/layer20_attention_weight_min": -23.9078426361084,
      "activations/layer21_attention_weight_max": 36.00611114501953,
      "activations/layer21_attention_weight_min": -23.8529109954834,
      "activations/layer22_attention_weight_max": 51.910858154296875,
      "activations/layer22_attention_weight_min": -29.89125633239746,
      "activations/layer23_attention_weight_max": 40.30031204223633,
      "activations/layer23_attention_weight_min": -24.757740020751953,
      "activations/layer2_attention_weight_max": 31.871326446533203,
      "activations/layer2_attention_weight_min": -30.607898712158203,
      "activations/layer3_attention_weight_max": 88.72163391113281,
      "activations/layer3_attention_weight_min": -90.94446563720703,
      "activations/layer4_attention_weight_max": 115.77166748046875,
      "activations/layer4_attention_weight_min": -109.55681610107422,
      "activations/layer5_attention_weight_max": 57.97930908203125,
      "activations/layer5_attention_weight_min": -60.64297103881836,
      "activations/layer6_attention_weight_max": 49.02943801879883,
      "activations/layer6_attention_weight_min": -49.56809616088867,
      "activations/layer7_attention_weight_max": 90.15277862548828,
      "activations/layer7_attention_weight_min": -83.9537353515625,
      "activations/layer8_attention_weight_max": 42.314491271972656,
      "activations/layer8_attention_weight_min": -40.63591384887695,
      "activations/layer9_attention_weight_max": 41.15402603149414,
      "activations/layer9_attention_weight_min": -35.894752502441406,
      "epoch": 7.71,
      "learning_rate": 0.00010131363636363634,
      "loss": 2.8017,
      "step": 132650
    },
    {
      "activations/layer0_attention_weight_max": 16.211336135864258,
      "activations/layer0_attention_weight_min": -10.858123779296875,
      "activations/layer10_attention_weight_max": 34.97322463989258,
      "activations/layer10_attention_weight_min": -31.403966903686523,
      "activations/layer11_attention_weight_max": 32.076026916503906,
      "activations/layer11_attention_weight_min": -30.187541961669922,
      "activations/layer12_attention_weight_max": 23.28091049194336,
      "activations/layer12_attention_weight_min": -24.59897804260254,
      "activations/layer13_attention_weight_max": 43.45365524291992,
      "activations/layer13_attention_weight_min": -31.6511173248291,
      "activations/layer14_attention_weight_max": 49.9575309753418,
      "activations/layer14_attention_weight_min": -39.59812927246094,
      "activations/layer15_attention_weight_max": 45.80820083618164,
      "activations/layer15_attention_weight_min": -32.20321273803711,
      "activations/layer16_attention_weight_max": 33.53778839111328,
      "activations/layer16_attention_weight_min": -28.601457595825195,
      "activations/layer17_attention_weight_max": 34.309669494628906,
      "activations/layer17_attention_weight_min": -27.644546508789062,
      "activations/layer18_attention_weight_max": 31.646526336669922,
      "activations/layer18_attention_weight_min": -21.52069664001465,
      "activations/layer19_attention_weight_max": 36.23771286010742,
      "activations/layer19_attention_weight_min": -30.934955596923828,
      "activations/layer1_attention_weight_max": 16.756847381591797,
      "activations/layer1_attention_weight_min": -15.047959327697754,
      "activations/layer20_attention_weight_max": 34.154415130615234,
      "activations/layer20_attention_weight_min": -23.223236083984375,
      "activations/layer21_attention_weight_max": 33.98411560058594,
      "activations/layer21_attention_weight_min": -21.171283721923828,
      "activations/layer22_attention_weight_max": 48.9172477722168,
      "activations/layer22_attention_weight_min": -27.65050506591797,
      "activations/layer23_attention_weight_max": 40.24253463745117,
      "activations/layer23_attention_weight_min": -22.85814666748047,
      "activations/layer2_attention_weight_max": 32.401336669921875,
      "activations/layer2_attention_weight_min": -31.338016510009766,
      "activations/layer3_attention_weight_max": 93.89153289794922,
      "activations/layer3_attention_weight_min": -93.53772735595703,
      "activations/layer4_attention_weight_max": 122.06257629394531,
      "activations/layer4_attention_weight_min": -113.82441711425781,
      "activations/layer5_attention_weight_max": 60.536739349365234,
      "activations/layer5_attention_weight_min": -63.258358001708984,
      "activations/layer6_attention_weight_max": 52.41767501831055,
      "activations/layer6_attention_weight_min": -51.926029205322266,
      "activations/layer7_attention_weight_max": 86.78500366210938,
      "activations/layer7_attention_weight_min": -90.4560317993164,
      "activations/layer8_attention_weight_max": 40.44689178466797,
      "activations/layer8_attention_weight_min": -41.525455474853516,
      "activations/layer9_attention_weight_max": 35.03824234008789,
      "activations/layer9_attention_weight_min": -33.4669075012207,
      "epoch": 7.71,
      "learning_rate": 0.00010129469696969696,
      "loss": 2.8193,
      "step": 132700
    },
    {
      "activations/layer0_attention_weight_max": 17.20468521118164,
      "activations/layer0_attention_weight_min": -12.157185554504395,
      "activations/layer10_attention_weight_max": 35.07228088378906,
      "activations/layer10_attention_weight_min": -31.168149948120117,
      "activations/layer11_attention_weight_max": 31.261688232421875,
      "activations/layer11_attention_weight_min": -28.688335418701172,
      "activations/layer12_attention_weight_max": 23.632261276245117,
      "activations/layer12_attention_weight_min": -23.682268142700195,
      "activations/layer13_attention_weight_max": 40.14338302612305,
      "activations/layer13_attention_weight_min": -31.035449981689453,
      "activations/layer14_attention_weight_max": 50.28327560424805,
      "activations/layer14_attention_weight_min": -37.772377014160156,
      "activations/layer15_attention_weight_max": 37.649200439453125,
      "activations/layer15_attention_weight_min": -28.788267135620117,
      "activations/layer16_attention_weight_max": 34.74016571044922,
      "activations/layer16_attention_weight_min": -30.161405563354492,
      "activations/layer17_attention_weight_max": 39.572265625,
      "activations/layer17_attention_weight_min": -25.441373825073242,
      "activations/layer18_attention_weight_max": 32.869964599609375,
      "activations/layer18_attention_weight_min": -23.24982261657715,
      "activations/layer19_attention_weight_max": 39.8231201171875,
      "activations/layer19_attention_weight_min": -30.875024795532227,
      "activations/layer1_attention_weight_max": 16.48219871520996,
      "activations/layer1_attention_weight_min": -15.415121078491211,
      "activations/layer20_attention_weight_max": 34.37638473510742,
      "activations/layer20_attention_weight_min": -25.1278018951416,
      "activations/layer21_attention_weight_max": 31.17045783996582,
      "activations/layer21_attention_weight_min": -23.08078956604004,
      "activations/layer22_attention_weight_max": 52.03804397583008,
      "activations/layer22_attention_weight_min": -28.79124641418457,
      "activations/layer23_attention_weight_max": 44.18815612792969,
      "activations/layer23_attention_weight_min": -22.673908233642578,
      "activations/layer2_attention_weight_max": 32.78151321411133,
      "activations/layer2_attention_weight_min": -31.8459529876709,
      "activations/layer3_attention_weight_max": 92.82162475585938,
      "activations/layer3_attention_weight_min": -95.30072021484375,
      "activations/layer4_attention_weight_max": 122.4674072265625,
      "activations/layer4_attention_weight_min": -116.88960266113281,
      "activations/layer5_attention_weight_max": 60.52640914916992,
      "activations/layer5_attention_weight_min": -60.76069641113281,
      "activations/layer6_attention_weight_max": 52.309974670410156,
      "activations/layer6_attention_weight_min": -53.81454849243164,
      "activations/layer7_attention_weight_max": 90.8184585571289,
      "activations/layer7_attention_weight_min": -84.82740020751953,
      "activations/layer8_attention_weight_max": 39.04154968261719,
      "activations/layer8_attention_weight_min": -39.84389877319336,
      "activations/layer9_attention_weight_max": 35.50725555419922,
      "activations/layer9_attention_weight_min": -34.50373458862305,
      "epoch": 7.71,
      "learning_rate": 0.00010127575757575757,
      "loss": 2.8138,
      "step": 132750
    },
    {
      "activations/layer0_attention_weight_max": 16.369770050048828,
      "activations/layer0_attention_weight_min": -13.262980461120605,
      "activations/layer10_attention_weight_max": 33.189300537109375,
      "activations/layer10_attention_weight_min": -29.80414390563965,
      "activations/layer11_attention_weight_max": 32.075096130371094,
      "activations/layer11_attention_weight_min": -28.32154655456543,
      "activations/layer12_attention_weight_max": 23.78400230407715,
      "activations/layer12_attention_weight_min": -26.35704231262207,
      "activations/layer13_attention_weight_max": 43.758548736572266,
      "activations/layer13_attention_weight_min": -34.45072937011719,
      "activations/layer14_attention_weight_max": 54.69921875,
      "activations/layer14_attention_weight_min": -42.80296325683594,
      "activations/layer15_attention_weight_max": 39.610557556152344,
      "activations/layer15_attention_weight_min": -29.960491180419922,
      "activations/layer16_attention_weight_max": 37.833290100097656,
      "activations/layer16_attention_weight_min": -27.76250648498535,
      "activations/layer17_attention_weight_max": 36.75312805175781,
      "activations/layer17_attention_weight_min": -24.665903091430664,
      "activations/layer18_attention_weight_max": 33.11529541015625,
      "activations/layer18_attention_weight_min": -21.027578353881836,
      "activations/layer19_attention_weight_max": 42.15751266479492,
      "activations/layer19_attention_weight_min": -31.886795043945312,
      "activations/layer1_attention_weight_max": 16.397207260131836,
      "activations/layer1_attention_weight_min": -14.446823120117188,
      "activations/layer20_attention_weight_max": 37.087127685546875,
      "activations/layer20_attention_weight_min": -23.893808364868164,
      "activations/layer21_attention_weight_max": 33.98580551147461,
      "activations/layer21_attention_weight_min": -21.91228675842285,
      "activations/layer22_attention_weight_max": 59.84412384033203,
      "activations/layer22_attention_weight_min": -28.7677001953125,
      "activations/layer23_attention_weight_max": 42.06045150756836,
      "activations/layer23_attention_weight_min": -23.631454467773438,
      "activations/layer2_attention_weight_max": 32.74034881591797,
      "activations/layer2_attention_weight_min": -33.08043670654297,
      "activations/layer3_attention_weight_max": 90.42452239990234,
      "activations/layer3_attention_weight_min": -99.85107421875,
      "activations/layer4_attention_weight_max": 117.36229705810547,
      "activations/layer4_attention_weight_min": -124.90715789794922,
      "activations/layer5_attention_weight_max": 56.79946517944336,
      "activations/layer5_attention_weight_min": -60.86045455932617,
      "activations/layer6_attention_weight_max": 48.391048431396484,
      "activations/layer6_attention_weight_min": -54.31452941894531,
      "activations/layer7_attention_weight_max": 85.17833709716797,
      "activations/layer7_attention_weight_min": -88.54539489746094,
      "activations/layer8_attention_weight_max": 38.932037353515625,
      "activations/layer8_attention_weight_min": -38.98772048950195,
      "activations/layer9_attention_weight_max": 35.293914794921875,
      "activations/layer9_attention_weight_min": -32.264060974121094,
      "epoch": 7.72,
      "learning_rate": 0.00010125681818181816,
      "loss": 2.7987,
      "step": 132800
    },
    {
      "activations/layer0_attention_weight_max": 14.778481483459473,
      "activations/layer0_attention_weight_min": -12.231231689453125,
      "activations/layer10_attention_weight_max": 31.725067138671875,
      "activations/layer10_attention_weight_min": -30.887939453125,
      "activations/layer11_attention_weight_max": 30.887989044189453,
      "activations/layer11_attention_weight_min": -28.803184509277344,
      "activations/layer12_attention_weight_max": 25.505876541137695,
      "activations/layer12_attention_weight_min": -26.577346801757812,
      "activations/layer13_attention_weight_max": 38.0634651184082,
      "activations/layer13_attention_weight_min": -34.232452392578125,
      "activations/layer14_attention_weight_max": 54.82415008544922,
      "activations/layer14_attention_weight_min": -45.633399963378906,
      "activations/layer15_attention_weight_max": 35.47812271118164,
      "activations/layer15_attention_weight_min": -32.37177276611328,
      "activations/layer16_attention_weight_max": 31.643177032470703,
      "activations/layer16_attention_weight_min": -27.191274642944336,
      "activations/layer17_attention_weight_max": 33.71074676513672,
      "activations/layer17_attention_weight_min": -25.344112396240234,
      "activations/layer18_attention_weight_max": 29.738317489624023,
      "activations/layer18_attention_weight_min": -21.096179962158203,
      "activations/layer19_attention_weight_max": 34.59316635131836,
      "activations/layer19_attention_weight_min": -29.701797485351562,
      "activations/layer1_attention_weight_max": 16.979494094848633,
      "activations/layer1_attention_weight_min": -15.52208137512207,
      "activations/layer20_attention_weight_max": 28.642988204956055,
      "activations/layer20_attention_weight_min": -25.33022117614746,
      "activations/layer21_attention_weight_max": 30.710397720336914,
      "activations/layer21_attention_weight_min": -22.401880264282227,
      "activations/layer22_attention_weight_max": 49.350921630859375,
      "activations/layer22_attention_weight_min": -27.99863052368164,
      "activations/layer23_attention_weight_max": 36.16654968261719,
      "activations/layer23_attention_weight_min": -21.709808349609375,
      "activations/layer2_attention_weight_max": 32.67285919189453,
      "activations/layer2_attention_weight_min": -31.611652374267578,
      "activations/layer3_attention_weight_max": 94.6185073852539,
      "activations/layer3_attention_weight_min": -93.73371124267578,
      "activations/layer4_attention_weight_max": 118.4051284790039,
      "activations/layer4_attention_weight_min": -117.7735595703125,
      "activations/layer5_attention_weight_max": 57.3944206237793,
      "activations/layer5_attention_weight_min": -61.181640625,
      "activations/layer6_attention_weight_max": 49.62013244628906,
      "activations/layer6_attention_weight_min": -52.47351837158203,
      "activations/layer7_attention_weight_max": 93.80660247802734,
      "activations/layer7_attention_weight_min": -89.50091552734375,
      "activations/layer8_attention_weight_max": 38.951255798339844,
      "activations/layer8_attention_weight_min": -39.59925079345703,
      "activations/layer9_attention_weight_max": 33.624053955078125,
      "activations/layer9_attention_weight_min": -32.48827362060547,
      "epoch": 7.72,
      "learning_rate": 0.00010123787878787878,
      "loss": 2.8144,
      "step": 132850
    },
    {
      "activations/layer0_attention_weight_max": 16.35154914855957,
      "activations/layer0_attention_weight_min": -11.74543285369873,
      "activations/layer10_attention_weight_max": 35.94446563720703,
      "activations/layer10_attention_weight_min": -34.18730545043945,
      "activations/layer11_attention_weight_max": 36.321441650390625,
      "activations/layer11_attention_weight_min": -34.35615539550781,
      "activations/layer12_attention_weight_max": 28.346439361572266,
      "activations/layer12_attention_weight_min": -25.834505081176758,
      "activations/layer13_attention_weight_max": 46.808597564697266,
      "activations/layer13_attention_weight_min": -36.213592529296875,
      "activations/layer14_attention_weight_max": 51.95478057861328,
      "activations/layer14_attention_weight_min": -41.717018127441406,
      "activations/layer15_attention_weight_max": 40.39423370361328,
      "activations/layer15_attention_weight_min": -34.158599853515625,
      "activations/layer16_attention_weight_max": 34.74710464477539,
      "activations/layer16_attention_weight_min": -27.746780395507812,
      "activations/layer17_attention_weight_max": 36.02370071411133,
      "activations/layer17_attention_weight_min": -29.69116973876953,
      "activations/layer18_attention_weight_max": 32.151588439941406,
      "activations/layer18_attention_weight_min": -25.455625534057617,
      "activations/layer19_attention_weight_max": 38.357147216796875,
      "activations/layer19_attention_weight_min": -30.194143295288086,
      "activations/layer1_attention_weight_max": 16.179487228393555,
      "activations/layer1_attention_weight_min": -14.49779224395752,
      "activations/layer20_attention_weight_max": 31.0734920501709,
      "activations/layer20_attention_weight_min": -24.12862205505371,
      "activations/layer21_attention_weight_max": 30.43712615966797,
      "activations/layer21_attention_weight_min": -21.57975196838379,
      "activations/layer22_attention_weight_max": 46.8939323425293,
      "activations/layer22_attention_weight_min": -30.06770133972168,
      "activations/layer23_attention_weight_max": 38.18795394897461,
      "activations/layer23_attention_weight_min": -23.317602157592773,
      "activations/layer2_attention_weight_max": 32.4419059753418,
      "activations/layer2_attention_weight_min": -31.71808624267578,
      "activations/layer3_attention_weight_max": 90.78301239013672,
      "activations/layer3_attention_weight_min": -94.15209197998047,
      "activations/layer4_attention_weight_max": 116.46239471435547,
      "activations/layer4_attention_weight_min": -114.05043029785156,
      "activations/layer5_attention_weight_max": 59.18518829345703,
      "activations/layer5_attention_weight_min": -62.505672454833984,
      "activations/layer6_attention_weight_max": 49.655357360839844,
      "activations/layer6_attention_weight_min": -52.46308517456055,
      "activations/layer7_attention_weight_max": 92.13801574707031,
      "activations/layer7_attention_weight_min": -92.59385681152344,
      "activations/layer8_attention_weight_max": 42.330570220947266,
      "activations/layer8_attention_weight_min": -40.28535461425781,
      "activations/layer9_attention_weight_max": 36.46220016479492,
      "activations/layer9_attention_weight_min": -35.676902770996094,
      "epoch": 7.72,
      "learning_rate": 0.00010121893939393939,
      "loss": 2.8104,
      "step": 132900
    },
    {
      "activations/layer0_attention_weight_max": 16.83343505859375,
      "activations/layer0_attention_weight_min": -12.563384056091309,
      "activations/layer10_attention_weight_max": 32.630226135253906,
      "activations/layer10_attention_weight_min": -31.148284912109375,
      "activations/layer11_attention_weight_max": 35.431556701660156,
      "activations/layer11_attention_weight_min": -29.00716209411621,
      "activations/layer12_attention_weight_max": 24.852649688720703,
      "activations/layer12_attention_weight_min": -25.73897933959961,
      "activations/layer13_attention_weight_max": 46.369789123535156,
      "activations/layer13_attention_weight_min": -35.897422790527344,
      "activations/layer14_attention_weight_max": 47.0813102722168,
      "activations/layer14_attention_weight_min": -38.515377044677734,
      "activations/layer15_attention_weight_max": 34.172916412353516,
      "activations/layer15_attention_weight_min": -28.31231689453125,
      "activations/layer16_attention_weight_max": 33.801231384277344,
      "activations/layer16_attention_weight_min": -27.644973754882812,
      "activations/layer17_attention_weight_max": 34.87041091918945,
      "activations/layer17_attention_weight_min": -25.3605899810791,
      "activations/layer18_attention_weight_max": 36.46311950683594,
      "activations/layer18_attention_weight_min": -22.46112823486328,
      "activations/layer19_attention_weight_max": 38.341583251953125,
      "activations/layer19_attention_weight_min": -30.436710357666016,
      "activations/layer1_attention_weight_max": 17.044979095458984,
      "activations/layer1_attention_weight_min": -14.7748441696167,
      "activations/layer20_attention_weight_max": 31.46261215209961,
      "activations/layer20_attention_weight_min": -24.865428924560547,
      "activations/layer21_attention_weight_max": 32.59363555908203,
      "activations/layer21_attention_weight_min": -23.7894287109375,
      "activations/layer22_attention_weight_max": 50.46841812133789,
      "activations/layer22_attention_weight_min": -29.97772789001465,
      "activations/layer23_attention_weight_max": 38.91037368774414,
      "activations/layer23_attention_weight_min": -23.309818267822266,
      "activations/layer2_attention_weight_max": 33.77278137207031,
      "activations/layer2_attention_weight_min": -33.29720687866211,
      "activations/layer3_attention_weight_max": 93.37645721435547,
      "activations/layer3_attention_weight_min": -96.76744842529297,
      "activations/layer4_attention_weight_max": 121.26000213623047,
      "activations/layer4_attention_weight_min": -116.61089324951172,
      "activations/layer5_attention_weight_max": 58.677955627441406,
      "activations/layer5_attention_weight_min": -59.9654655456543,
      "activations/layer6_attention_weight_max": 50.0817985534668,
      "activations/layer6_attention_weight_min": -51.46992111206055,
      "activations/layer7_attention_weight_max": 87.96300506591797,
      "activations/layer7_attention_weight_min": -87.88326263427734,
      "activations/layer8_attention_weight_max": 39.43223190307617,
      "activations/layer8_attention_weight_min": -38.45183181762695,
      "activations/layer9_attention_weight_max": 34.183109283447266,
      "activations/layer9_attention_weight_min": -34.4113883972168,
      "epoch": 7.73,
      "learning_rate": 0.0001012,
      "loss": 2.8109,
      "step": 132950
    },
    {
      "activations/layer0_attention_weight_max": 15.170537948608398,
      "activations/layer0_attention_weight_min": -11.707005500793457,
      "activations/layer10_attention_weight_max": 33.31224060058594,
      "activations/layer10_attention_weight_min": -31.38986587524414,
      "activations/layer11_attention_weight_max": 33.57773208618164,
      "activations/layer11_attention_weight_min": -28.142810821533203,
      "activations/layer12_attention_weight_max": 25.108264923095703,
      "activations/layer12_attention_weight_min": -25.155405044555664,
      "activations/layer13_attention_weight_max": 38.424827575683594,
      "activations/layer13_attention_weight_min": -32.7066535949707,
      "activations/layer14_attention_weight_max": 55.14146041870117,
      "activations/layer14_attention_weight_min": -40.5982551574707,
      "activations/layer15_attention_weight_max": 41.261077880859375,
      "activations/layer15_attention_weight_min": -30.19863510131836,
      "activations/layer16_attention_weight_max": 34.63871383666992,
      "activations/layer16_attention_weight_min": -28.70596694946289,
      "activations/layer17_attention_weight_max": 34.35014724731445,
      "activations/layer17_attention_weight_min": -24.96088409423828,
      "activations/layer18_attention_weight_max": 31.319782257080078,
      "activations/layer18_attention_weight_min": -21.8652400970459,
      "activations/layer19_attention_weight_max": 36.639007568359375,
      "activations/layer19_attention_weight_min": -30.517654418945312,
      "activations/layer1_attention_weight_max": 18.094161987304688,
      "activations/layer1_attention_weight_min": -15.95416259765625,
      "activations/layer20_attention_weight_max": 31.40118408203125,
      "activations/layer20_attention_weight_min": -24.649152755737305,
      "activations/layer21_attention_weight_max": 28.18764877319336,
      "activations/layer21_attention_weight_min": -23.5264892578125,
      "activations/layer22_attention_weight_max": 45.76093292236328,
      "activations/layer22_attention_weight_min": -28.003313064575195,
      "activations/layer23_attention_weight_max": 34.15533447265625,
      "activations/layer23_attention_weight_min": -23.70807456970215,
      "activations/layer2_attention_weight_max": 32.05942916870117,
      "activations/layer2_attention_weight_min": -30.273900985717773,
      "activations/layer3_attention_weight_max": 87.97515106201172,
      "activations/layer3_attention_weight_min": -88.57490539550781,
      "activations/layer4_attention_weight_max": 113.5422134399414,
      "activations/layer4_attention_weight_min": -113.66761779785156,
      "activations/layer5_attention_weight_max": 57.058834075927734,
      "activations/layer5_attention_weight_min": -62.69743347167969,
      "activations/layer6_attention_weight_max": 48.149879455566406,
      "activations/layer6_attention_weight_min": -52.34067916870117,
      "activations/layer7_attention_weight_max": 87.13792419433594,
      "activations/layer7_attention_weight_min": -88.98957061767578,
      "activations/layer8_attention_weight_max": 38.285457611083984,
      "activations/layer8_attention_weight_min": -40.442989349365234,
      "activations/layer9_attention_weight_max": 34.24052429199219,
      "activations/layer9_attention_weight_min": -38.03153991699219,
      "epoch": 7.73,
      "learning_rate": 0.0001011810606060606,
      "loss": 2.844,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5098,
      "eval_samples_per_second": 504.592,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5098,
      "eval_openwebtext_samples_per_second": 504.592,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_wikitext_loss": 2.99609375,
      "eval_wikitext_ppl": 20.007230835729846,
      "eval_wikitext_runtime": 2.0212,
      "eval_wikitext_samples_per_second": 225.604,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_lambada_loss": 2.689453125,
      "eval_lambada_ppl": 14.723621737689907,
      "eval_lambada_runtime": 9.6015,
      "eval_lambada_samples_per_second": 507.107,
      "step": 133000
    },
    {
      "activations/layer0_attention_weight_max": 16.642065048217773,
      "activations/layer0_attention_weight_min": -13.090555191040039,
      "activations/layer10_attention_weight_max": 38.47232437133789,
      "activations/layer10_attention_weight_min": -32.454654693603516,
      "activations/layer11_attention_weight_max": 36.79119110107422,
      "activations/layer11_attention_weight_min": -32.32770538330078,
      "activations/layer12_attention_weight_max": 24.63262367248535,
      "activations/layer12_attention_weight_min": -23.762662887573242,
      "activations/layer13_attention_weight_max": 39.40228271484375,
      "activations/layer13_attention_weight_min": -33.955543518066406,
      "activations/layer14_attention_weight_max": 45.95745849609375,
      "activations/layer14_attention_weight_min": -38.17601013183594,
      "activations/layer15_attention_weight_max": 34.16737747192383,
      "activations/layer15_attention_weight_min": -29.76119613647461,
      "activations/layer16_attention_weight_max": 29.574249267578125,
      "activations/layer16_attention_weight_min": -28.78290557861328,
      "activations/layer17_attention_weight_max": 35.30085372924805,
      "activations/layer17_attention_weight_min": -25.5271053314209,
      "activations/layer18_attention_weight_max": 32.328521728515625,
      "activations/layer18_attention_weight_min": -21.922346115112305,
      "activations/layer19_attention_weight_max": 34.751705169677734,
      "activations/layer19_attention_weight_min": -31.130849838256836,
      "activations/layer1_attention_weight_max": 17.67770767211914,
      "activations/layer1_attention_weight_min": -16.122377395629883,
      "activations/layer20_attention_weight_max": 31.5810489654541,
      "activations/layer20_attention_weight_min": -24.031322479248047,
      "activations/layer21_attention_weight_max": 31.162649154663086,
      "activations/layer21_attention_weight_min": -22.706741333007812,
      "activations/layer22_attention_weight_max": 51.02389144897461,
      "activations/layer22_attention_weight_min": -29.149229049682617,
      "activations/layer23_attention_weight_max": 35.76990509033203,
      "activations/layer23_attention_weight_min": -24.371845245361328,
      "activations/layer2_attention_weight_max": 31.91373062133789,
      "activations/layer2_attention_weight_min": -32.26689147949219,
      "activations/layer3_attention_weight_max": 86.19515991210938,
      "activations/layer3_attention_weight_min": -87.64763641357422,
      "activations/layer4_attention_weight_max": 115.36659240722656,
      "activations/layer4_attention_weight_min": -118.37458801269531,
      "activations/layer5_attention_weight_max": 59.28575134277344,
      "activations/layer5_attention_weight_min": -61.851097106933594,
      "activations/layer6_attention_weight_max": 49.89917755126953,
      "activations/layer6_attention_weight_min": -53.50972366333008,
      "activations/layer7_attention_weight_max": 88.50688934326172,
      "activations/layer7_attention_weight_min": -87.66526794433594,
      "activations/layer8_attention_weight_max": 42.82355499267578,
      "activations/layer8_attention_weight_min": -39.29159164428711,
      "activations/layer9_attention_weight_max": 38.32316589355469,
      "activations/layer9_attention_weight_min": -34.2254638671875,
      "epoch": 7.73,
      "learning_rate": 0.00010116212121212121,
      "loss": 2.814,
      "step": 133050
    },
    {
      "activations/layer0_attention_weight_max": 16.61579132080078,
      "activations/layer0_attention_weight_min": -12.197896957397461,
      "activations/layer10_attention_weight_max": 32.27162551879883,
      "activations/layer10_attention_weight_min": -31.486955642700195,
      "activations/layer11_attention_weight_max": 32.146480560302734,
      "activations/layer11_attention_weight_min": -30.638341903686523,
      "activations/layer12_attention_weight_max": 48.278194427490234,
      "activations/layer12_attention_weight_min": -27.93442726135254,
      "activations/layer13_attention_weight_max": 48.22970962524414,
      "activations/layer13_attention_weight_min": -33.15566635131836,
      "activations/layer14_attention_weight_max": 58.66789627075195,
      "activations/layer14_attention_weight_min": -41.063255310058594,
      "activations/layer15_attention_weight_max": 40.6182746887207,
      "activations/layer15_attention_weight_min": -29.88397789001465,
      "activations/layer16_attention_weight_max": 36.62199783325195,
      "activations/layer16_attention_weight_min": -26.98542022705078,
      "activations/layer17_attention_weight_max": 36.21922302246094,
      "activations/layer17_attention_weight_min": -24.9965763092041,
      "activations/layer18_attention_weight_max": 36.50675964355469,
      "activations/layer18_attention_weight_min": -21.93156623840332,
      "activations/layer19_attention_weight_max": 38.332252502441406,
      "activations/layer19_attention_weight_min": -30.787199020385742,
      "activations/layer1_attention_weight_max": 16.905292510986328,
      "activations/layer1_attention_weight_min": -14.665081024169922,
      "activations/layer20_attention_weight_max": 34.6212043762207,
      "activations/layer20_attention_weight_min": -24.391942977905273,
      "activations/layer21_attention_weight_max": 35.21042251586914,
      "activations/layer21_attention_weight_min": -23.316984176635742,
      "activations/layer22_attention_weight_max": 52.69952392578125,
      "activations/layer22_attention_weight_min": -29.032564163208008,
      "activations/layer23_attention_weight_max": 35.13005447387695,
      "activations/layer23_attention_weight_min": -21.93178939819336,
      "activations/layer2_attention_weight_max": 31.066848754882812,
      "activations/layer2_attention_weight_min": -30.9127197265625,
      "activations/layer3_attention_weight_max": 85.2609634399414,
      "activations/layer3_attention_weight_min": -92.76270294189453,
      "activations/layer4_attention_weight_max": 109.41289520263672,
      "activations/layer4_attention_weight_min": -111.81644439697266,
      "activations/layer5_attention_weight_max": 53.55210876464844,
      "activations/layer5_attention_weight_min": -61.93000030517578,
      "activations/layer6_attention_weight_max": 46.58677291870117,
      "activations/layer6_attention_weight_min": -50.039344787597656,
      "activations/layer7_attention_weight_max": 83.63790130615234,
      "activations/layer7_attention_weight_min": -88.96623992919922,
      "activations/layer8_attention_weight_max": 38.37495803833008,
      "activations/layer8_attention_weight_min": -38.521732330322266,
      "activations/layer9_attention_weight_max": 34.4136962890625,
      "activations/layer9_attention_weight_min": -32.38186264038086,
      "epoch": 7.73,
      "learning_rate": 0.00010114318181818181,
      "loss": 2.8211,
      "step": 133100
    },
    {
      "activations/layer0_attention_weight_max": 16.339139938354492,
      "activations/layer0_attention_weight_min": -12.321860313415527,
      "activations/layer10_attention_weight_max": 29.979013442993164,
      "activations/layer10_attention_weight_min": -35.04642868041992,
      "activations/layer11_attention_weight_max": 29.693899154663086,
      "activations/layer11_attention_weight_min": -30.391708374023438,
      "activations/layer12_attention_weight_max": 22.99665641784668,
      "activations/layer12_attention_weight_min": -25.932954788208008,
      "activations/layer13_attention_weight_max": 37.167076110839844,
      "activations/layer13_attention_weight_min": -32.21109390258789,
      "activations/layer14_attention_weight_max": 45.811988830566406,
      "activations/layer14_attention_weight_min": -37.57392120361328,
      "activations/layer15_attention_weight_max": 34.414791107177734,
      "activations/layer15_attention_weight_min": -31.11495018005371,
      "activations/layer16_attention_weight_max": 32.465641021728516,
      "activations/layer16_attention_weight_min": -28.20198631286621,
      "activations/layer17_attention_weight_max": 34.49142074584961,
      "activations/layer17_attention_weight_min": -27.607585906982422,
      "activations/layer18_attention_weight_max": 29.32036781311035,
      "activations/layer18_attention_weight_min": -23.64072608947754,
      "activations/layer19_attention_weight_max": 33.701900482177734,
      "activations/layer19_attention_weight_min": -30.04193115234375,
      "activations/layer1_attention_weight_max": 16.22767448425293,
      "activations/layer1_attention_weight_min": -14.740425109863281,
      "activations/layer20_attention_weight_max": 29.701570510864258,
      "activations/layer20_attention_weight_min": -25.812828063964844,
      "activations/layer21_attention_weight_max": 28.919851303100586,
      "activations/layer21_attention_weight_min": -25.252525329589844,
      "activations/layer22_attention_weight_max": 42.0445442199707,
      "activations/layer22_attention_weight_min": -28.870651245117188,
      "activations/layer23_attention_weight_max": 34.52339172363281,
      "activations/layer23_attention_weight_min": -23.975299835205078,
      "activations/layer2_attention_weight_max": 32.34136962890625,
      "activations/layer2_attention_weight_min": -32.390281677246094,
      "activations/layer3_attention_weight_max": 91.69737243652344,
      "activations/layer3_attention_weight_min": -92.9259033203125,
      "activations/layer4_attention_weight_max": 117.61577606201172,
      "activations/layer4_attention_weight_min": -118.44640350341797,
      "activations/layer5_attention_weight_max": 59.05970764160156,
      "activations/layer5_attention_weight_min": -60.277618408203125,
      "activations/layer6_attention_weight_max": 50.2476921081543,
      "activations/layer6_attention_weight_min": -50.704490661621094,
      "activations/layer7_attention_weight_max": 85.91900634765625,
      "activations/layer7_attention_weight_min": -94.27377319335938,
      "activations/layer8_attention_weight_max": 37.83293914794922,
      "activations/layer8_attention_weight_min": -44.56671905517578,
      "activations/layer9_attention_weight_max": 33.69763946533203,
      "activations/layer9_attention_weight_min": -36.444881439208984,
      "epoch": 7.74,
      "learning_rate": 0.00010112424242424241,
      "loss": 2.825,
      "step": 133150
    },
    {
      "activations/layer0_attention_weight_max": 16.46209144592285,
      "activations/layer0_attention_weight_min": -12.873103141784668,
      "activations/layer10_attention_weight_max": 33.89018249511719,
      "activations/layer10_attention_weight_min": -33.10676193237305,
      "activations/layer11_attention_weight_max": 32.05400085449219,
      "activations/layer11_attention_weight_min": -30.54009246826172,
      "activations/layer12_attention_weight_max": 22.672849655151367,
      "activations/layer12_attention_weight_min": -24.93412208557129,
      "activations/layer13_attention_weight_max": 40.30288314819336,
      "activations/layer13_attention_weight_min": -34.93670654296875,
      "activations/layer14_attention_weight_max": 49.1088981628418,
      "activations/layer14_attention_weight_min": -40.04490661621094,
      "activations/layer15_attention_weight_max": 37.17379379272461,
      "activations/layer15_attention_weight_min": -30.65465545654297,
      "activations/layer16_attention_weight_max": 33.069427490234375,
      "activations/layer16_attention_weight_min": -26.97543716430664,
      "activations/layer17_attention_weight_max": 33.3500862121582,
      "activations/layer17_attention_weight_min": -24.57366371154785,
      "activations/layer18_attention_weight_max": 34.17494583129883,
      "activations/layer18_attention_weight_min": -23.14939308166504,
      "activations/layer19_attention_weight_max": 39.97535705566406,
      "activations/layer19_attention_weight_min": -29.371265411376953,
      "activations/layer1_attention_weight_max": 15.857793807983398,
      "activations/layer1_attention_weight_min": -14.542356491088867,
      "activations/layer20_attention_weight_max": 31.251462936401367,
      "activations/layer20_attention_weight_min": -23.816143035888672,
      "activations/layer21_attention_weight_max": 33.37614059448242,
      "activations/layer21_attention_weight_min": -21.279630661010742,
      "activations/layer22_attention_weight_max": 53.47660446166992,
      "activations/layer22_attention_weight_min": -28.17386817932129,
      "activations/layer23_attention_weight_max": 37.41728973388672,
      "activations/layer23_attention_weight_min": -23.03521728515625,
      "activations/layer2_attention_weight_max": 33.26582717895508,
      "activations/layer2_attention_weight_min": -31.291454315185547,
      "activations/layer3_attention_weight_max": 93.14336395263672,
      "activations/layer3_attention_weight_min": -90.61197662353516,
      "activations/layer4_attention_weight_max": 120.8767318725586,
      "activations/layer4_attention_weight_min": -117.58595275878906,
      "activations/layer5_attention_weight_max": 60.293296813964844,
      "activations/layer5_attention_weight_min": -61.87044906616211,
      "activations/layer6_attention_weight_max": 52.493289947509766,
      "activations/layer6_attention_weight_min": -52.8262825012207,
      "activations/layer7_attention_weight_max": 95.31427001953125,
      "activations/layer7_attention_weight_min": -90.92069244384766,
      "activations/layer8_attention_weight_max": 43.91189193725586,
      "activations/layer8_attention_weight_min": -43.2513542175293,
      "activations/layer9_attention_weight_max": 36.56142807006836,
      "activations/layer9_attention_weight_min": -35.652069091796875,
      "epoch": 7.74,
      "learning_rate": 0.00010110530303030301,
      "loss": 2.8213,
      "step": 133200
    },
    {
      "activations/layer0_attention_weight_max": 16.98896598815918,
      "activations/layer0_attention_weight_min": -11.59457778930664,
      "activations/layer10_attention_weight_max": 33.051212310791016,
      "activations/layer10_attention_weight_min": -32.46253204345703,
      "activations/layer11_attention_weight_max": 31.941139221191406,
      "activations/layer11_attention_weight_min": -30.180469512939453,
      "activations/layer12_attention_weight_max": 30.681604385375977,
      "activations/layer12_attention_weight_min": -27.752456665039062,
      "activations/layer13_attention_weight_max": 39.93998718261719,
      "activations/layer13_attention_weight_min": -33.3984489440918,
      "activations/layer14_attention_weight_max": 56.75249099731445,
      "activations/layer14_attention_weight_min": -42.851959228515625,
      "activations/layer15_attention_weight_max": 36.458709716796875,
      "activations/layer15_attention_weight_min": -30.812685012817383,
      "activations/layer16_attention_weight_max": 35.47701644897461,
      "activations/layer16_attention_weight_min": -28.982271194458008,
      "activations/layer17_attention_weight_max": 36.417572021484375,
      "activations/layer17_attention_weight_min": -25.562305450439453,
      "activations/layer18_attention_weight_max": 32.05903244018555,
      "activations/layer18_attention_weight_min": -22.293474197387695,
      "activations/layer19_attention_weight_max": 41.74409484863281,
      "activations/layer19_attention_weight_min": -30.1114559173584,
      "activations/layer1_attention_weight_max": 17.0943603515625,
      "activations/layer1_attention_weight_min": -15.121343612670898,
      "activations/layer20_attention_weight_max": 33.34967041015625,
      "activations/layer20_attention_weight_min": -23.578264236450195,
      "activations/layer21_attention_weight_max": 32.26799011230469,
      "activations/layer21_attention_weight_min": -23.13995361328125,
      "activations/layer22_attention_weight_max": 48.960758209228516,
      "activations/layer22_attention_weight_min": -27.205373764038086,
      "activations/layer23_attention_weight_max": 35.33721160888672,
      "activations/layer23_attention_weight_min": -24.2651424407959,
      "activations/layer2_attention_weight_max": 32.61103820800781,
      "activations/layer2_attention_weight_min": -32.396705627441406,
      "activations/layer3_attention_weight_max": 91.63243865966797,
      "activations/layer3_attention_weight_min": -93.68096923828125,
      "activations/layer4_attention_weight_max": 123.00711822509766,
      "activations/layer4_attention_weight_min": -120.52978515625,
      "activations/layer5_attention_weight_max": 63.40479278564453,
      "activations/layer5_attention_weight_min": -60.59627914428711,
      "activations/layer6_attention_weight_max": 54.49873352050781,
      "activations/layer6_attention_weight_min": -53.211483001708984,
      "activations/layer7_attention_weight_max": 89.27842712402344,
      "activations/layer7_attention_weight_min": -91.67851257324219,
      "activations/layer8_attention_weight_max": 40.82245635986328,
      "activations/layer8_attention_weight_min": -41.99140930175781,
      "activations/layer9_attention_weight_max": 35.53950500488281,
      "activations/layer9_attention_weight_min": -34.33525085449219,
      "epoch": 7.74,
      "learning_rate": 0.00010108636363636363,
      "loss": 2.8121,
      "step": 133250
    },
    {
      "activations/layer0_attention_weight_max": 16.21222686767578,
      "activations/layer0_attention_weight_min": -12.75679874420166,
      "activations/layer10_attention_weight_max": 36.244598388671875,
      "activations/layer10_attention_weight_min": -33.90315628051758,
      "activations/layer11_attention_weight_max": 34.93114471435547,
      "activations/layer11_attention_weight_min": -30.722553253173828,
      "activations/layer12_attention_weight_max": 24.137170791625977,
      "activations/layer12_attention_weight_min": -27.171554565429688,
      "activations/layer13_attention_weight_max": 40.352691650390625,
      "activations/layer13_attention_weight_min": -33.203529357910156,
      "activations/layer14_attention_weight_max": 50.420589447021484,
      "activations/layer14_attention_weight_min": -36.838600158691406,
      "activations/layer15_attention_weight_max": 36.270416259765625,
      "activations/layer15_attention_weight_min": -29.427988052368164,
      "activations/layer16_attention_weight_max": 32.99452209472656,
      "activations/layer16_attention_weight_min": -28.811067581176758,
      "activations/layer17_attention_weight_max": 34.83561325073242,
      "activations/layer17_attention_weight_min": -25.000808715820312,
      "activations/layer18_attention_weight_max": 30.179738998413086,
      "activations/layer18_attention_weight_min": -21.642250061035156,
      "activations/layer19_attention_weight_max": 33.02766799926758,
      "activations/layer19_attention_weight_min": -29.652484893798828,
      "activations/layer1_attention_weight_max": 16.611066818237305,
      "activations/layer1_attention_weight_min": -16.4887752532959,
      "activations/layer20_attention_weight_max": 28.437294006347656,
      "activations/layer20_attention_weight_min": -26.50078010559082,
      "activations/layer21_attention_weight_max": 28.343151092529297,
      "activations/layer21_attention_weight_min": -22.427444458007812,
      "activations/layer22_attention_weight_max": 42.70885467529297,
      "activations/layer22_attention_weight_min": -30.387632369995117,
      "activations/layer23_attention_weight_max": 32.22303771972656,
      "activations/layer23_attention_weight_min": -23.835708618164062,
      "activations/layer2_attention_weight_max": 33.87377166748047,
      "activations/layer2_attention_weight_min": -32.7672233581543,
      "activations/layer3_attention_weight_max": 87.960693359375,
      "activations/layer3_attention_weight_min": -90.6861801147461,
      "activations/layer4_attention_weight_max": 116.85491180419922,
      "activations/layer4_attention_weight_min": -115.87847900390625,
      "activations/layer5_attention_weight_max": 59.81615447998047,
      "activations/layer5_attention_weight_min": -61.01625061035156,
      "activations/layer6_attention_weight_max": 50.22237777709961,
      "activations/layer6_attention_weight_min": -52.921382904052734,
      "activations/layer7_attention_weight_max": 86.89879608154297,
      "activations/layer7_attention_weight_min": -86.14913940429688,
      "activations/layer8_attention_weight_max": 40.57167053222656,
      "activations/layer8_attention_weight_min": -40.21721267700195,
      "activations/layer9_attention_weight_max": 37.339385986328125,
      "activations/layer9_attention_weight_min": -34.479305267333984,
      "epoch": 7.75,
      "learning_rate": 0.00010106742424242424,
      "loss": 2.8317,
      "step": 133300
    },
    {
      "activations/layer0_attention_weight_max": 16.07847023010254,
      "activations/layer0_attention_weight_min": -12.016727447509766,
      "activations/layer10_attention_weight_max": 37.819862365722656,
      "activations/layer10_attention_weight_min": -37.6041259765625,
      "activations/layer11_attention_weight_max": 38.56371307373047,
      "activations/layer11_attention_weight_min": -38.76099395751953,
      "activations/layer12_attention_weight_max": 34.089229583740234,
      "activations/layer12_attention_weight_min": -29.32544708251953,
      "activations/layer13_attention_weight_max": 42.31451416015625,
      "activations/layer13_attention_weight_min": -37.943145751953125,
      "activations/layer14_attention_weight_max": 56.86357498168945,
      "activations/layer14_attention_weight_min": -41.16606903076172,
      "activations/layer15_attention_weight_max": 45.74015426635742,
      "activations/layer15_attention_weight_min": -34.76959991455078,
      "activations/layer16_attention_weight_max": 34.59583282470703,
      "activations/layer16_attention_weight_min": -27.6644287109375,
      "activations/layer17_attention_weight_max": 38.73710250854492,
      "activations/layer17_attention_weight_min": -24.45355987548828,
      "activations/layer18_attention_weight_max": 30.343175888061523,
      "activations/layer18_attention_weight_min": -21.119869232177734,
      "activations/layer19_attention_weight_max": 36.792884826660156,
      "activations/layer19_attention_weight_min": -31.11954116821289,
      "activations/layer1_attention_weight_max": 18.619539260864258,
      "activations/layer1_attention_weight_min": -16.083328247070312,
      "activations/layer20_attention_weight_max": 31.19448471069336,
      "activations/layer20_attention_weight_min": -24.35653305053711,
      "activations/layer21_attention_weight_max": 30.016826629638672,
      "activations/layer21_attention_weight_min": -22.24065589904785,
      "activations/layer22_attention_weight_max": 51.28993225097656,
      "activations/layer22_attention_weight_min": -28.99500846862793,
      "activations/layer23_attention_weight_max": 38.098846435546875,
      "activations/layer23_attention_weight_min": -23.474376678466797,
      "activations/layer2_attention_weight_max": 33.40739822387695,
      "activations/layer2_attention_weight_min": -32.34379577636719,
      "activations/layer3_attention_weight_max": 93.55247497558594,
      "activations/layer3_attention_weight_min": -97.5410385131836,
      "activations/layer4_attention_weight_max": 122.682373046875,
      "activations/layer4_attention_weight_min": -118.1507797241211,
      "activations/layer5_attention_weight_max": 57.22904586791992,
      "activations/layer5_attention_weight_min": -57.468605041503906,
      "activations/layer6_attention_weight_max": 47.01652145385742,
      "activations/layer6_attention_weight_min": -51.471370697021484,
      "activations/layer7_attention_weight_max": 102.70043182373047,
      "activations/layer7_attention_weight_min": -93.56555938720703,
      "activations/layer8_attention_weight_max": 46.1711311340332,
      "activations/layer8_attention_weight_min": -43.459774017333984,
      "activations/layer9_attention_weight_max": 39.6491813659668,
      "activations/layer9_attention_weight_min": -38.573631286621094,
      "epoch": 7.75,
      "learning_rate": 0.00010104848484848483,
      "loss": 2.8057,
      "step": 133350
    },
    {
      "activations/layer0_attention_weight_max": 16.270349502563477,
      "activations/layer0_attention_weight_min": -11.463818550109863,
      "activations/layer10_attention_weight_max": 40.600074768066406,
      "activations/layer10_attention_weight_min": -34.86423110961914,
      "activations/layer11_attention_weight_max": 44.04277801513672,
      "activations/layer11_attention_weight_min": -37.54924392700195,
      "activations/layer12_attention_weight_max": 25.53156089782715,
      "activations/layer12_attention_weight_min": -26.953842163085938,
      "activations/layer13_attention_weight_max": 39.17988586425781,
      "activations/layer13_attention_weight_min": -32.05715560913086,
      "activations/layer14_attention_weight_max": 54.059165954589844,
      "activations/layer14_attention_weight_min": -37.80264663696289,
      "activations/layer15_attention_weight_max": 37.574668884277344,
      "activations/layer15_attention_weight_min": -29.528207778930664,
      "activations/layer16_attention_weight_max": 31.63590431213379,
      "activations/layer16_attention_weight_min": -27.01194190979004,
      "activations/layer17_attention_weight_max": 36.3277587890625,
      "activations/layer17_attention_weight_min": -24.630130767822266,
      "activations/layer18_attention_weight_max": 33.469322204589844,
      "activations/layer18_attention_weight_min": -24.77051544189453,
      "activations/layer19_attention_weight_max": 35.6208381652832,
      "activations/layer19_attention_weight_min": -31.18848991394043,
      "activations/layer1_attention_weight_max": 17.917699813842773,
      "activations/layer1_attention_weight_min": -15.59899616241455,
      "activations/layer20_attention_weight_max": 31.280654907226562,
      "activations/layer20_attention_weight_min": -25.12706756591797,
      "activations/layer21_attention_weight_max": 30.79344367980957,
      "activations/layer21_attention_weight_min": -24.466848373413086,
      "activations/layer22_attention_weight_max": 50.65536880493164,
      "activations/layer22_attention_weight_min": -30.42749786376953,
      "activations/layer23_attention_weight_max": 37.57708740234375,
      "activations/layer23_attention_weight_min": -23.83942222595215,
      "activations/layer2_attention_weight_max": 32.02888488769531,
      "activations/layer2_attention_weight_min": -30.71515655517578,
      "activations/layer3_attention_weight_max": 91.5533218383789,
      "activations/layer3_attention_weight_min": -90.68072509765625,
      "activations/layer4_attention_weight_max": 122.3613052368164,
      "activations/layer4_attention_weight_min": -111.12257385253906,
      "activations/layer5_attention_weight_max": 58.47233200073242,
      "activations/layer5_attention_weight_min": -61.018795013427734,
      "activations/layer6_attention_weight_max": 51.54771423339844,
      "activations/layer6_attention_weight_min": -50.24805450439453,
      "activations/layer7_attention_weight_max": 103.12867736816406,
      "activations/layer7_attention_weight_min": -86.77883911132812,
      "activations/layer8_attention_weight_max": 45.63420486450195,
      "activations/layer8_attention_weight_min": -40.70098114013672,
      "activations/layer9_attention_weight_max": 36.858272552490234,
      "activations/layer9_attention_weight_min": -34.989830017089844,
      "epoch": 7.75,
      "learning_rate": 0.00010102954545454545,
      "loss": 2.8183,
      "step": 133400
    },
    {
      "activations/layer0_attention_weight_max": 15.211087226867676,
      "activations/layer0_attention_weight_min": -12.34957504272461,
      "activations/layer10_attention_weight_max": 38.19631576538086,
      "activations/layer10_attention_weight_min": -31.595701217651367,
      "activations/layer11_attention_weight_max": 34.7858772277832,
      "activations/layer11_attention_weight_min": -30.689964294433594,
      "activations/layer12_attention_weight_max": 23.527420043945312,
      "activations/layer12_attention_weight_min": -24.708091735839844,
      "activations/layer13_attention_weight_max": 38.585227966308594,
      "activations/layer13_attention_weight_min": -31.72702980041504,
      "activations/layer14_attention_weight_max": 58.49358367919922,
      "activations/layer14_attention_weight_min": -45.86763000488281,
      "activations/layer15_attention_weight_max": 36.893577575683594,
      "activations/layer15_attention_weight_min": -30.093658447265625,
      "activations/layer16_attention_weight_max": 34.52517318725586,
      "activations/layer16_attention_weight_min": -25.205724716186523,
      "activations/layer17_attention_weight_max": 35.85038375854492,
      "activations/layer17_attention_weight_min": -24.740976333618164,
      "activations/layer18_attention_weight_max": 36.515445709228516,
      "activations/layer18_attention_weight_min": -23.16398811340332,
      "activations/layer19_attention_weight_max": 38.751670837402344,
      "activations/layer19_attention_weight_min": -30.163711547851562,
      "activations/layer1_attention_weight_max": 18.234773635864258,
      "activations/layer1_attention_weight_min": -14.229630470275879,
      "activations/layer20_attention_weight_max": 33.54782485961914,
      "activations/layer20_attention_weight_min": -23.840343475341797,
      "activations/layer21_attention_weight_max": 34.963104248046875,
      "activations/layer21_attention_weight_min": -22.878915786743164,
      "activations/layer22_attention_weight_max": 52.26521682739258,
      "activations/layer22_attention_weight_min": -29.81011962890625,
      "activations/layer23_attention_weight_max": 42.52085494995117,
      "activations/layer23_attention_weight_min": -22.05913543701172,
      "activations/layer2_attention_weight_max": 33.78408432006836,
      "activations/layer2_attention_weight_min": -32.42626190185547,
      "activations/layer3_attention_weight_max": 91.9336929321289,
      "activations/layer3_attention_weight_min": -92.139404296875,
      "activations/layer4_attention_weight_max": 120.95464324951172,
      "activations/layer4_attention_weight_min": -122.61991119384766,
      "activations/layer5_attention_weight_max": 63.481468200683594,
      "activations/layer5_attention_weight_min": -64.28289031982422,
      "activations/layer6_attention_weight_max": 51.697486877441406,
      "activations/layer6_attention_weight_min": -55.49116134643555,
      "activations/layer7_attention_weight_max": 92.62994384765625,
      "activations/layer7_attention_weight_min": -87.5293197631836,
      "activations/layer8_attention_weight_max": 41.08297348022461,
      "activations/layer8_attention_weight_min": -41.29900360107422,
      "activations/layer9_attention_weight_max": 35.5008659362793,
      "activations/layer9_attention_weight_min": -33.4448127746582,
      "epoch": 7.75,
      "learning_rate": 0.00010101060606060606,
      "loss": 2.8094,
      "step": 133450
    },
    {
      "activations/layer0_attention_weight_max": 16.245820999145508,
      "activations/layer0_attention_weight_min": -12.710611343383789,
      "activations/layer10_attention_weight_max": 33.054691314697266,
      "activations/layer10_attention_weight_min": -31.240270614624023,
      "activations/layer11_attention_weight_max": 33.837135314941406,
      "activations/layer11_attention_weight_min": -28.946125030517578,
      "activations/layer12_attention_weight_max": 25.85370445251465,
      "activations/layer12_attention_weight_min": -26.53466033935547,
      "activations/layer13_attention_weight_max": 41.64144515991211,
      "activations/layer13_attention_weight_min": -33.042484283447266,
      "activations/layer14_attention_weight_max": 55.123111724853516,
      "activations/layer14_attention_weight_min": -40.90035629272461,
      "activations/layer15_attention_weight_max": 38.744441986083984,
      "activations/layer15_attention_weight_min": -30.96915626525879,
      "activations/layer16_attention_weight_max": 34.91366958618164,
      "activations/layer16_attention_weight_min": -28.165992736816406,
      "activations/layer17_attention_weight_max": 34.97062683105469,
      "activations/layer17_attention_weight_min": -25.677614212036133,
      "activations/layer18_attention_weight_max": 33.2181510925293,
      "activations/layer18_attention_weight_min": -21.52964973449707,
      "activations/layer19_attention_weight_max": 39.2855110168457,
      "activations/layer19_attention_weight_min": -29.125513076782227,
      "activations/layer1_attention_weight_max": 17.206674575805664,
      "activations/layer1_attention_weight_min": -14.352804183959961,
      "activations/layer20_attention_weight_max": 32.80241394042969,
      "activations/layer20_attention_weight_min": -25.665205001831055,
      "activations/layer21_attention_weight_max": 31.241275787353516,
      "activations/layer21_attention_weight_min": -23.932153701782227,
      "activations/layer22_attention_weight_max": 50.4935302734375,
      "activations/layer22_attention_weight_min": -29.469295501708984,
      "activations/layer23_attention_weight_max": 37.343509674072266,
      "activations/layer23_attention_weight_min": -25.686193466186523,
      "activations/layer2_attention_weight_max": 31.491832733154297,
      "activations/layer2_attention_weight_min": -31.302427291870117,
      "activations/layer3_attention_weight_max": 90.05209350585938,
      "activations/layer3_attention_weight_min": -90.24984741210938,
      "activations/layer4_attention_weight_max": 116.99635314941406,
      "activations/layer4_attention_weight_min": -112.90284729003906,
      "activations/layer5_attention_weight_max": 58.62831115722656,
      "activations/layer5_attention_weight_min": -60.16133117675781,
      "activations/layer6_attention_weight_max": 52.14101028442383,
      "activations/layer6_attention_weight_min": -49.959495544433594,
      "activations/layer7_attention_weight_max": 88.0931625366211,
      "activations/layer7_attention_weight_min": -89.07572174072266,
      "activations/layer8_attention_weight_max": 39.89433670043945,
      "activations/layer8_attention_weight_min": -39.636260986328125,
      "activations/layer9_attention_weight_max": 35.86571502685547,
      "activations/layer9_attention_weight_min": -33.455387115478516,
      "epoch": 7.76,
      "learning_rate": 0.00010099166666666665,
      "loss": 2.8166,
      "step": 133500
    },
    {
      "activations/layer0_attention_weight_max": 15.279802322387695,
      "activations/layer0_attention_weight_min": -12.848450660705566,
      "activations/layer10_attention_weight_max": 34.26240539550781,
      "activations/layer10_attention_weight_min": -33.33253860473633,
      "activations/layer11_attention_weight_max": 35.604774475097656,
      "activations/layer11_attention_weight_min": -31.037443161010742,
      "activations/layer12_attention_weight_max": 24.000274658203125,
      "activations/layer12_attention_weight_min": -25.175329208374023,
      "activations/layer13_attention_weight_max": 41.367347717285156,
      "activations/layer13_attention_weight_min": -35.351566314697266,
      "activations/layer14_attention_weight_max": 54.11944580078125,
      "activations/layer14_attention_weight_min": -43.568756103515625,
      "activations/layer15_attention_weight_max": 34.63817596435547,
      "activations/layer15_attention_weight_min": -31.24269676208496,
      "activations/layer16_attention_weight_max": 30.753862380981445,
      "activations/layer16_attention_weight_min": -29.96201515197754,
      "activations/layer17_attention_weight_max": 32.9824104309082,
      "activations/layer17_attention_weight_min": -26.86736488342285,
      "activations/layer18_attention_weight_max": 30.48845100402832,
      "activations/layer18_attention_weight_min": -22.35258674621582,
      "activations/layer19_attention_weight_max": 40.87838363647461,
      "activations/layer19_attention_weight_min": -34.34793472290039,
      "activations/layer1_attention_weight_max": 16.997074127197266,
      "activations/layer1_attention_weight_min": -13.393440246582031,
      "activations/layer20_attention_weight_max": 31.66756820678711,
      "activations/layer20_attention_weight_min": -25.99074363708496,
      "activations/layer21_attention_weight_max": 35.1740608215332,
      "activations/layer21_attention_weight_min": -23.25588607788086,
      "activations/layer22_attention_weight_max": 46.82036590576172,
      "activations/layer22_attention_weight_min": -29.816993713378906,
      "activations/layer23_attention_weight_max": 35.50916290283203,
      "activations/layer23_attention_weight_min": -25.59412384033203,
      "activations/layer2_attention_weight_max": 28.60647964477539,
      "activations/layer2_attention_weight_min": -29.914691925048828,
      "activations/layer3_attention_weight_max": 85.9866714477539,
      "activations/layer3_attention_weight_min": -91.16876220703125,
      "activations/layer4_attention_weight_max": 119.1539077758789,
      "activations/layer4_attention_weight_min": -114.2811050415039,
      "activations/layer5_attention_weight_max": 59.673187255859375,
      "activations/layer5_attention_weight_min": -65.24918365478516,
      "activations/layer6_attention_weight_max": 49.3076286315918,
      "activations/layer6_attention_weight_min": -53.44302749633789,
      "activations/layer7_attention_weight_max": 86.77507781982422,
      "activations/layer7_attention_weight_min": -95.3346939086914,
      "activations/layer8_attention_weight_max": 38.781986236572266,
      "activations/layer8_attention_weight_min": -42.85052490234375,
      "activations/layer9_attention_weight_max": 33.919517517089844,
      "activations/layer9_attention_weight_min": -34.6191520690918,
      "epoch": 7.76,
      "learning_rate": 0.00010097272727272727,
      "loss": 2.8006,
      "step": 133550
    },
    {
      "activations/layer0_attention_weight_max": 16.318544387817383,
      "activations/layer0_attention_weight_min": -12.232287406921387,
      "activations/layer10_attention_weight_max": 32.496944427490234,
      "activations/layer10_attention_weight_min": -31.00609016418457,
      "activations/layer11_attention_weight_max": 31.846668243408203,
      "activations/layer11_attention_weight_min": -28.6142578125,
      "activations/layer12_attention_weight_max": 23.327293395996094,
      "activations/layer12_attention_weight_min": -25.888587951660156,
      "activations/layer13_attention_weight_max": 43.105777740478516,
      "activations/layer13_attention_weight_min": -31.924943923950195,
      "activations/layer14_attention_weight_max": 48.98492431640625,
      "activations/layer14_attention_weight_min": -42.91783142089844,
      "activations/layer15_attention_weight_max": 35.34149169921875,
      "activations/layer15_attention_weight_min": -29.992361068725586,
      "activations/layer16_attention_weight_max": 34.23232650756836,
      "activations/layer16_attention_weight_min": -27.315513610839844,
      "activations/layer17_attention_weight_max": 33.19452667236328,
      "activations/layer17_attention_weight_min": -27.895292282104492,
      "activations/layer18_attention_weight_max": 31.48969078063965,
      "activations/layer18_attention_weight_min": -21.614667892456055,
      "activations/layer19_attention_weight_max": 37.775508880615234,
      "activations/layer19_attention_weight_min": -31.65431785583496,
      "activations/layer1_attention_weight_max": 17.167675018310547,
      "activations/layer1_attention_weight_min": -16.20049476623535,
      "activations/layer20_attention_weight_max": 31.84410285949707,
      "activations/layer20_attention_weight_min": -25.430286407470703,
      "activations/layer21_attention_weight_max": 33.112159729003906,
      "activations/layer21_attention_weight_min": -23.818485260009766,
      "activations/layer22_attention_weight_max": 45.63113784790039,
      "activations/layer22_attention_weight_min": -27.445772171020508,
      "activations/layer23_attention_weight_max": 32.247108459472656,
      "activations/layer23_attention_weight_min": -24.466732025146484,
      "activations/layer2_attention_weight_max": 33.634159088134766,
      "activations/layer2_attention_weight_min": -33.264713287353516,
      "activations/layer3_attention_weight_max": 88.7635726928711,
      "activations/layer3_attention_weight_min": -91.5140380859375,
      "activations/layer4_attention_weight_max": 118.82464599609375,
      "activations/layer4_attention_weight_min": -117.9679183959961,
      "activations/layer5_attention_weight_max": 60.410125732421875,
      "activations/layer5_attention_weight_min": -61.558631896972656,
      "activations/layer6_attention_weight_max": 51.24211883544922,
      "activations/layer6_attention_weight_min": -55.438819885253906,
      "activations/layer7_attention_weight_max": 88.82218933105469,
      "activations/layer7_attention_weight_min": -87.63040161132812,
      "activations/layer8_attention_weight_max": 40.00081253051758,
      "activations/layer8_attention_weight_min": -41.64380645751953,
      "activations/layer9_attention_weight_max": 35.64042282104492,
      "activations/layer9_attention_weight_min": -33.29783248901367,
      "epoch": 7.76,
      "learning_rate": 0.00010095378787878787,
      "loss": 2.7909,
      "step": 133600
    },
    {
      "activations/layer0_attention_weight_max": 15.931130409240723,
      "activations/layer0_attention_weight_min": -12.14169979095459,
      "activations/layer10_attention_weight_max": 34.61056900024414,
      "activations/layer10_attention_weight_min": -30.279033660888672,
      "activations/layer11_attention_weight_max": 31.40670394897461,
      "activations/layer11_attention_weight_min": -29.240690231323242,
      "activations/layer12_attention_weight_max": 21.736570358276367,
      "activations/layer12_attention_weight_min": -23.879098892211914,
      "activations/layer13_attention_weight_max": 36.15510177612305,
      "activations/layer13_attention_weight_min": -32.07666778564453,
      "activations/layer14_attention_weight_max": 43.76932907104492,
      "activations/layer14_attention_weight_min": -38.80598449707031,
      "activations/layer15_attention_weight_max": 35.31226348876953,
      "activations/layer15_attention_weight_min": -30.660964965820312,
      "activations/layer16_attention_weight_max": 31.246509552001953,
      "activations/layer16_attention_weight_min": -29.27684211730957,
      "activations/layer17_attention_weight_max": 34.56222915649414,
      "activations/layer17_attention_weight_min": -25.191650390625,
      "activations/layer18_attention_weight_max": 30.687593460083008,
      "activations/layer18_attention_weight_min": -21.407791137695312,
      "activations/layer19_attention_weight_max": 35.40024185180664,
      "activations/layer19_attention_weight_min": -30.414539337158203,
      "activations/layer1_attention_weight_max": 17.378559112548828,
      "activations/layer1_attention_weight_min": -15.310868263244629,
      "activations/layer20_attention_weight_max": 29.567903518676758,
      "activations/layer20_attention_weight_min": -25.01894187927246,
      "activations/layer21_attention_weight_max": 28.49539566040039,
      "activations/layer21_attention_weight_min": -21.775314331054688,
      "activations/layer22_attention_weight_max": 45.933860778808594,
      "activations/layer22_attention_weight_min": -29.230825424194336,
      "activations/layer23_attention_weight_max": 37.23353576660156,
      "activations/layer23_attention_weight_min": -26.31081771850586,
      "activations/layer2_attention_weight_max": 31.628276824951172,
      "activations/layer2_attention_weight_min": -31.061628341674805,
      "activations/layer3_attention_weight_max": 91.68070220947266,
      "activations/layer3_attention_weight_min": -94.20638275146484,
      "activations/layer4_attention_weight_max": 120.10400390625,
      "activations/layer4_attention_weight_min": -121.53560638427734,
      "activations/layer5_attention_weight_max": 61.36792755126953,
      "activations/layer5_attention_weight_min": -62.155357360839844,
      "activations/layer6_attention_weight_max": 48.07283020019531,
      "activations/layer6_attention_weight_min": -53.08839797973633,
      "activations/layer7_attention_weight_max": 86.93930053710938,
      "activations/layer7_attention_weight_min": -91.8908462524414,
      "activations/layer8_attention_weight_max": 39.90806198120117,
      "activations/layer8_attention_weight_min": -40.93993377685547,
      "activations/layer9_attention_weight_max": 35.86802673339844,
      "activations/layer9_attention_weight_min": -34.49913787841797,
      "epoch": 7.77,
      "learning_rate": 0.00010093484848484847,
      "loss": 2.8194,
      "step": 133650
    },
    {
      "activations/layer0_attention_weight_max": 15.312531471252441,
      "activations/layer0_attention_weight_min": -11.861254692077637,
      "activations/layer10_attention_weight_max": 34.237274169921875,
      "activations/layer10_attention_weight_min": -35.24237823486328,
      "activations/layer11_attention_weight_max": 34.43659210205078,
      "activations/layer11_attention_weight_min": -31.918556213378906,
      "activations/layer12_attention_weight_max": 23.472015380859375,
      "activations/layer12_attention_weight_min": -24.435667037963867,
      "activations/layer13_attention_weight_max": 40.91900634765625,
      "activations/layer13_attention_weight_min": -33.194828033447266,
      "activations/layer14_attention_weight_max": 53.030574798583984,
      "activations/layer14_attention_weight_min": -39.814735412597656,
      "activations/layer15_attention_weight_max": 38.80044174194336,
      "activations/layer15_attention_weight_min": -29.57663345336914,
      "activations/layer16_attention_weight_max": 35.93113327026367,
      "activations/layer16_attention_weight_min": -28.00657844543457,
      "activations/layer17_attention_weight_max": 35.83943557739258,
      "activations/layer17_attention_weight_min": -26.229049682617188,
      "activations/layer18_attention_weight_max": 30.80450439453125,
      "activations/layer18_attention_weight_min": -21.631772994995117,
      "activations/layer19_attention_weight_max": 35.49931335449219,
      "activations/layer19_attention_weight_min": -29.984769821166992,
      "activations/layer1_attention_weight_max": 17.05504035949707,
      "activations/layer1_attention_weight_min": -15.991961479187012,
      "activations/layer20_attention_weight_max": 31.417774200439453,
      "activations/layer20_attention_weight_min": -23.989011764526367,
      "activations/layer21_attention_weight_max": 30.732736587524414,
      "activations/layer21_attention_weight_min": -21.473608016967773,
      "activations/layer22_attention_weight_max": 44.9778938293457,
      "activations/layer22_attention_weight_min": -30.130550384521484,
      "activations/layer23_attention_weight_max": 37.420082092285156,
      "activations/layer23_attention_weight_min": -22.422256469726562,
      "activations/layer2_attention_weight_max": 34.65675354003906,
      "activations/layer2_attention_weight_min": -32.40752029418945,
      "activations/layer3_attention_weight_max": 94.32373809814453,
      "activations/layer3_attention_weight_min": -94.09458923339844,
      "activations/layer4_attention_weight_max": 120.1747817993164,
      "activations/layer4_attention_weight_min": -121.57330322265625,
      "activations/layer5_attention_weight_max": 59.988006591796875,
      "activations/layer5_attention_weight_min": -62.79358673095703,
      "activations/layer6_attention_weight_max": 51.73432159423828,
      "activations/layer6_attention_weight_min": -53.65495681762695,
      "activations/layer7_attention_weight_max": 92.46084594726562,
      "activations/layer7_attention_weight_min": -94.064453125,
      "activations/layer8_attention_weight_max": 43.88284683227539,
      "activations/layer8_attention_weight_min": -46.70567321777344,
      "activations/layer9_attention_weight_max": 38.26119613647461,
      "activations/layer9_attention_weight_min": -38.648250579833984,
      "epoch": 7.77,
      "learning_rate": 0.00010091590909090908,
      "loss": 2.8038,
      "step": 133700
    },
    {
      "activations/layer0_attention_weight_max": 16.504980087280273,
      "activations/layer0_attention_weight_min": -11.324810981750488,
      "activations/layer10_attention_weight_max": 34.51164245605469,
      "activations/layer10_attention_weight_min": -30.98659896850586,
      "activations/layer11_attention_weight_max": 31.627782821655273,
      "activations/layer11_attention_weight_min": -31.148679733276367,
      "activations/layer12_attention_weight_max": 26.532594680786133,
      "activations/layer12_attention_weight_min": -24.279306411743164,
      "activations/layer13_attention_weight_max": 40.94762420654297,
      "activations/layer13_attention_weight_min": -33.26089859008789,
      "activations/layer14_attention_weight_max": 50.93138122558594,
      "activations/layer14_attention_weight_min": -40.31235122680664,
      "activations/layer15_attention_weight_max": 35.045528411865234,
      "activations/layer15_attention_weight_min": -29.57573699951172,
      "activations/layer16_attention_weight_max": 36.86852264404297,
      "activations/layer16_attention_weight_min": -27.43419075012207,
      "activations/layer17_attention_weight_max": 35.75381088256836,
      "activations/layer17_attention_weight_min": -25.157699584960938,
      "activations/layer18_attention_weight_max": 31.538606643676758,
      "activations/layer18_attention_weight_min": -23.086660385131836,
      "activations/layer19_attention_weight_max": 41.78824234008789,
      "activations/layer19_attention_weight_min": -32.68218231201172,
      "activations/layer1_attention_weight_max": 16.970706939697266,
      "activations/layer1_attention_weight_min": -15.038724899291992,
      "activations/layer20_attention_weight_max": 32.992618560791016,
      "activations/layer20_attention_weight_min": -25.501697540283203,
      "activations/layer21_attention_weight_max": 29.81991958618164,
      "activations/layer21_attention_weight_min": -24.297622680664062,
      "activations/layer22_attention_weight_max": 49.878536224365234,
      "activations/layer22_attention_weight_min": -33.0461311340332,
      "activations/layer23_attention_weight_max": 37.73592758178711,
      "activations/layer23_attention_weight_min": -22.47493553161621,
      "activations/layer2_attention_weight_max": 34.30449676513672,
      "activations/layer2_attention_weight_min": -30.996091842651367,
      "activations/layer3_attention_weight_max": 90.54464721679688,
      "activations/layer3_attention_weight_min": -91.17623901367188,
      "activations/layer4_attention_weight_max": 122.0031509399414,
      "activations/layer4_attention_weight_min": -119.86002349853516,
      "activations/layer5_attention_weight_max": 59.69391632080078,
      "activations/layer5_attention_weight_min": -61.748504638671875,
      "activations/layer6_attention_weight_max": 50.47012710571289,
      "activations/layer6_attention_weight_min": -53.20869445800781,
      "activations/layer7_attention_weight_max": 92.9959945678711,
      "activations/layer7_attention_weight_min": -91.81340026855469,
      "activations/layer8_attention_weight_max": 42.32682418823242,
      "activations/layer8_attention_weight_min": -41.307838439941406,
      "activations/layer9_attention_weight_max": 38.06771469116211,
      "activations/layer9_attention_weight_min": -34.44432067871094,
      "epoch": 7.77,
      "learning_rate": 0.00010089696969696968,
      "loss": 2.8027,
      "step": 133750
    },
    {
      "activations/layer0_attention_weight_max": 15.917885780334473,
      "activations/layer0_attention_weight_min": -13.2553071975708,
      "activations/layer10_attention_weight_max": 32.95614242553711,
      "activations/layer10_attention_weight_min": -32.546627044677734,
      "activations/layer11_attention_weight_max": 31.106151580810547,
      "activations/layer11_attention_weight_min": -30.23324966430664,
      "activations/layer12_attention_weight_max": 22.86724090576172,
      "activations/layer12_attention_weight_min": -28.35890007019043,
      "activations/layer13_attention_weight_max": 40.25130844116211,
      "activations/layer13_attention_weight_min": -32.970787048339844,
      "activations/layer14_attention_weight_max": 50.92092514038086,
      "activations/layer14_attention_weight_min": -43.37702560424805,
      "activations/layer15_attention_weight_max": 37.36868667602539,
      "activations/layer15_attention_weight_min": -30.96595573425293,
      "activations/layer16_attention_weight_max": 36.142234802246094,
      "activations/layer16_attention_weight_min": -27.050052642822266,
      "activations/layer17_attention_weight_max": 35.91933059692383,
      "activations/layer17_attention_weight_min": -26.913942337036133,
      "activations/layer18_attention_weight_max": 33.12534713745117,
      "activations/layer18_attention_weight_min": -20.908191680908203,
      "activations/layer19_attention_weight_max": 38.02865219116211,
      "activations/layer19_attention_weight_min": -29.922250747680664,
      "activations/layer1_attention_weight_max": 16.782312393188477,
      "activations/layer1_attention_weight_min": -15.111432075500488,
      "activations/layer20_attention_weight_max": 30.43886375427246,
      "activations/layer20_attention_weight_min": -25.187192916870117,
      "activations/layer21_attention_weight_max": 30.9835262298584,
      "activations/layer21_attention_weight_min": -21.919551849365234,
      "activations/layer22_attention_weight_max": 48.748287200927734,
      "activations/layer22_attention_weight_min": -27.93033218383789,
      "activations/layer23_attention_weight_max": 39.34790802001953,
      "activations/layer23_attention_weight_min": -23.114036560058594,
      "activations/layer2_attention_weight_max": 32.81346893310547,
      "activations/layer2_attention_weight_min": -32.86804962158203,
      "activations/layer3_attention_weight_max": 91.66559600830078,
      "activations/layer3_attention_weight_min": -94.98193359375,
      "activations/layer4_attention_weight_max": 118.96952056884766,
      "activations/layer4_attention_weight_min": -118.65962982177734,
      "activations/layer5_attention_weight_max": 58.0052604675293,
      "activations/layer5_attention_weight_min": -59.832763671875,
      "activations/layer6_attention_weight_max": 50.29325866699219,
      "activations/layer6_attention_weight_min": -53.55353546142578,
      "activations/layer7_attention_weight_max": 86.73411560058594,
      "activations/layer7_attention_weight_min": -87.53787231445312,
      "activations/layer8_attention_weight_max": 43.51203536987305,
      "activations/layer8_attention_weight_min": -40.045066833496094,
      "activations/layer9_attention_weight_max": 36.315731048583984,
      "activations/layer9_attention_weight_min": -34.7392692565918,
      "epoch": 7.77,
      "learning_rate": 0.0001008780303030303,
      "loss": 2.8064,
      "step": 133800
    },
    {
      "activations/layer0_attention_weight_max": 14.060154914855957,
      "activations/layer0_attention_weight_min": -12.688756942749023,
      "activations/layer10_attention_weight_max": 31.96782112121582,
      "activations/layer10_attention_weight_min": -31.351593017578125,
      "activations/layer11_attention_weight_max": 32.047611236572266,
      "activations/layer11_attention_weight_min": -28.37562370300293,
      "activations/layer12_attention_weight_max": 25.3308048248291,
      "activations/layer12_attention_weight_min": -24.96453285217285,
      "activations/layer13_attention_weight_max": 40.641624450683594,
      "activations/layer13_attention_weight_min": -32.81105422973633,
      "activations/layer14_attention_weight_max": 49.541690826416016,
      "activations/layer14_attention_weight_min": -39.606483459472656,
      "activations/layer15_attention_weight_max": 35.79281997680664,
      "activations/layer15_attention_weight_min": -28.878694534301758,
      "activations/layer16_attention_weight_max": 37.86946487426758,
      "activations/layer16_attention_weight_min": -28.393360137939453,
      "activations/layer17_attention_weight_max": 33.77587127685547,
      "activations/layer17_attention_weight_min": -24.12169075012207,
      "activations/layer18_attention_weight_max": 32.96293258666992,
      "activations/layer18_attention_weight_min": -21.06001091003418,
      "activations/layer19_attention_weight_max": 46.36003875732422,
      "activations/layer19_attention_weight_min": -29.72686767578125,
      "activations/layer1_attention_weight_max": 18.649192810058594,
      "activations/layer1_attention_weight_min": -15.16180419921875,
      "activations/layer20_attention_weight_max": 36.29814147949219,
      "activations/layer20_attention_weight_min": -24.27351951599121,
      "activations/layer21_attention_weight_max": 31.78636932373047,
      "activations/layer21_attention_weight_min": -23.3931884765625,
      "activations/layer22_attention_weight_max": 47.5044059753418,
      "activations/layer22_attention_weight_min": -29.769393920898438,
      "activations/layer23_attention_weight_max": 43.060302734375,
      "activations/layer23_attention_weight_min": -22.398038864135742,
      "activations/layer2_attention_weight_max": 31.027767181396484,
      "activations/layer2_attention_weight_min": -31.82223129272461,
      "activations/layer3_attention_weight_max": 88.1449203491211,
      "activations/layer3_attention_weight_min": -89.57683563232422,
      "activations/layer4_attention_weight_max": 116.162353515625,
      "activations/layer4_attention_weight_min": -113.43184661865234,
      "activations/layer5_attention_weight_max": 59.096290588378906,
      "activations/layer5_attention_weight_min": -61.11173629760742,
      "activations/layer6_attention_weight_max": 48.863468170166016,
      "activations/layer6_attention_weight_min": -49.284603118896484,
      "activations/layer7_attention_weight_max": 87.73050689697266,
      "activations/layer7_attention_weight_min": -84.2971420288086,
      "activations/layer8_attention_weight_max": 39.9532585144043,
      "activations/layer8_attention_weight_min": -41.71287155151367,
      "activations/layer9_attention_weight_max": 32.799713134765625,
      "activations/layer9_attention_weight_min": -32.08530044555664,
      "epoch": 7.78,
      "learning_rate": 0.0001008590909090909,
      "loss": 2.8112,
      "step": 133850
    },
    {
      "activations/layer0_attention_weight_max": 16.171419143676758,
      "activations/layer0_attention_weight_min": -12.099363327026367,
      "activations/layer10_attention_weight_max": 33.4061164855957,
      "activations/layer10_attention_weight_min": -32.9442024230957,
      "activations/layer11_attention_weight_max": 34.346046447753906,
      "activations/layer11_attention_weight_min": -29.376615524291992,
      "activations/layer12_attention_weight_max": 21.460683822631836,
      "activations/layer12_attention_weight_min": -38.17445755004883,
      "activations/layer13_attention_weight_max": 36.60472106933594,
      "activations/layer13_attention_weight_min": -32.136192321777344,
      "activations/layer14_attention_weight_max": 43.33769989013672,
      "activations/layer14_attention_weight_min": -34.471126556396484,
      "activations/layer15_attention_weight_max": 33.734642028808594,
      "activations/layer15_attention_weight_min": -30.619617462158203,
      "activations/layer16_attention_weight_max": 33.91537094116211,
      "activations/layer16_attention_weight_min": -29.017440795898438,
      "activations/layer17_attention_weight_max": 31.25473403930664,
      "activations/layer17_attention_weight_min": -24.20302963256836,
      "activations/layer18_attention_weight_max": 29.177310943603516,
      "activations/layer18_attention_weight_min": -21.996915817260742,
      "activations/layer19_attention_weight_max": 32.83335494995117,
      "activations/layer19_attention_weight_min": -30.438352584838867,
      "activations/layer1_attention_weight_max": 17.359865188598633,
      "activations/layer1_attention_weight_min": -14.513753890991211,
      "activations/layer20_attention_weight_max": 28.226287841796875,
      "activations/layer20_attention_weight_min": -23.075258255004883,
      "activations/layer21_attention_weight_max": 27.251461029052734,
      "activations/layer21_attention_weight_min": -20.91961669921875,
      "activations/layer22_attention_weight_max": 44.77170944213867,
      "activations/layer22_attention_weight_min": -26.77363395690918,
      "activations/layer23_attention_weight_max": 33.20220184326172,
      "activations/layer23_attention_weight_min": -24.1095027923584,
      "activations/layer2_attention_weight_max": 32.77564239501953,
      "activations/layer2_attention_weight_min": -31.933101654052734,
      "activations/layer3_attention_weight_max": 94.39837646484375,
      "activations/layer3_attention_weight_min": -95.21562957763672,
      "activations/layer4_attention_weight_max": 119.9607925415039,
      "activations/layer4_attention_weight_min": -119.12259674072266,
      "activations/layer5_attention_weight_max": 58.1705436706543,
      "activations/layer5_attention_weight_min": -62.151763916015625,
      "activations/layer6_attention_weight_max": 51.15955352783203,
      "activations/layer6_attention_weight_min": -51.57535934448242,
      "activations/layer7_attention_weight_max": 93.71652221679688,
      "activations/layer7_attention_weight_min": -90.21115112304688,
      "activations/layer8_attention_weight_max": 40.601951599121094,
      "activations/layer8_attention_weight_min": -40.55454635620117,
      "activations/layer9_attention_weight_max": 34.76575469970703,
      "activations/layer9_attention_weight_min": -40.64942932128906,
      "epoch": 7.78,
      "learning_rate": 0.0001008401515151515,
      "loss": 2.8174,
      "step": 133900
    },
    {
      "activations/layer0_attention_weight_max": 16.353017807006836,
      "activations/layer0_attention_weight_min": -12.94467544555664,
      "activations/layer10_attention_weight_max": 30.340068817138672,
      "activations/layer10_attention_weight_min": -31.11234474182129,
      "activations/layer11_attention_weight_max": 32.285400390625,
      "activations/layer11_attention_weight_min": -28.618240356445312,
      "activations/layer12_attention_weight_max": 22.044878005981445,
      "activations/layer12_attention_weight_min": -29.419239044189453,
      "activations/layer13_attention_weight_max": 36.95918655395508,
      "activations/layer13_attention_weight_min": -31.160715103149414,
      "activations/layer14_attention_weight_max": 51.722267150878906,
      "activations/layer14_attention_weight_min": -42.595733642578125,
      "activations/layer15_attention_weight_max": 33.24363327026367,
      "activations/layer15_attention_weight_min": -29.952590942382812,
      "activations/layer16_attention_weight_max": 31.266279220581055,
      "activations/layer16_attention_weight_min": -26.572538375854492,
      "activations/layer17_attention_weight_max": 32.05796432495117,
      "activations/layer17_attention_weight_min": -25.290624618530273,
      "activations/layer18_attention_weight_max": 31.940143585205078,
      "activations/layer18_attention_weight_min": -22.570505142211914,
      "activations/layer19_attention_weight_max": 38.16544723510742,
      "activations/layer19_attention_weight_min": -29.67532730102539,
      "activations/layer1_attention_weight_max": 17.131837844848633,
      "activations/layer1_attention_weight_min": -14.600834846496582,
      "activations/layer20_attention_weight_max": 31.7579345703125,
      "activations/layer20_attention_weight_min": -24.10317611694336,
      "activations/layer21_attention_weight_max": 29.976659774780273,
      "activations/layer21_attention_weight_min": -22.228137969970703,
      "activations/layer22_attention_weight_max": 46.26187515258789,
      "activations/layer22_attention_weight_min": -27.26109504699707,
      "activations/layer23_attention_weight_max": 32.92433166503906,
      "activations/layer23_attention_weight_min": -24.472511291503906,
      "activations/layer2_attention_weight_max": 32.56438064575195,
      "activations/layer2_attention_weight_min": -32.329307556152344,
      "activations/layer3_attention_weight_max": 95.13568115234375,
      "activations/layer3_attention_weight_min": -98.505615234375,
      "activations/layer4_attention_weight_max": 119.61894989013672,
      "activations/layer4_attention_weight_min": -120.81267547607422,
      "activations/layer5_attention_weight_max": 58.72520065307617,
      "activations/layer5_attention_weight_min": -66.53849792480469,
      "activations/layer6_attention_weight_max": 52.70486831665039,
      "activations/layer6_attention_weight_min": -52.01238250732422,
      "activations/layer7_attention_weight_max": 90.99250793457031,
      "activations/layer7_attention_weight_min": -86.25175476074219,
      "activations/layer8_attention_weight_max": 41.06498718261719,
      "activations/layer8_attention_weight_min": -40.715049743652344,
      "activations/layer9_attention_weight_max": 32.71783447265625,
      "activations/layer9_attention_weight_min": -33.42720031738281,
      "epoch": 7.78,
      "learning_rate": 0.00010082121212121212,
      "loss": 2.8108,
      "step": 133950
    },
    {
      "activations/layer0_attention_weight_max": 16.089344024658203,
      "activations/layer0_attention_weight_min": -12.020991325378418,
      "activations/layer10_attention_weight_max": 35.2694091796875,
      "activations/layer10_attention_weight_min": -32.62220001220703,
      "activations/layer11_attention_weight_max": 35.79154968261719,
      "activations/layer11_attention_weight_min": -31.346342086791992,
      "activations/layer12_attention_weight_max": 21.21351432800293,
      "activations/layer12_attention_weight_min": -28.247297286987305,
      "activations/layer13_attention_weight_max": 38.843135833740234,
      "activations/layer13_attention_weight_min": -33.18701171875,
      "activations/layer14_attention_weight_max": 43.63083267211914,
      "activations/layer14_attention_weight_min": -37.162506103515625,
      "activations/layer15_attention_weight_max": 38.153011322021484,
      "activations/layer15_attention_weight_min": -27.864896774291992,
      "activations/layer16_attention_weight_max": 33.628631591796875,
      "activations/layer16_attention_weight_min": -27.19508171081543,
      "activations/layer17_attention_weight_max": 33.11711502075195,
      "activations/layer17_attention_weight_min": -24.404579162597656,
      "activations/layer18_attention_weight_max": 28.324357986450195,
      "activations/layer18_attention_weight_min": -22.129091262817383,
      "activations/layer19_attention_weight_max": 32.109683990478516,
      "activations/layer19_attention_weight_min": -29.946399688720703,
      "activations/layer1_attention_weight_max": 17.562685012817383,
      "activations/layer1_attention_weight_min": -14.097016334533691,
      "activations/layer20_attention_weight_max": 28.109373092651367,
      "activations/layer20_attention_weight_min": -24.134965896606445,
      "activations/layer21_attention_weight_max": 29.38393783569336,
      "activations/layer21_attention_weight_min": -21.969484329223633,
      "activations/layer22_attention_weight_max": 45.86113739013672,
      "activations/layer22_attention_weight_min": -28.491365432739258,
      "activations/layer23_attention_weight_max": 33.65813446044922,
      "activations/layer23_attention_weight_min": -22.345760345458984,
      "activations/layer2_attention_weight_max": 32.529964447021484,
      "activations/layer2_attention_weight_min": -31.798213958740234,
      "activations/layer3_attention_weight_max": 95.47117614746094,
      "activations/layer3_attention_weight_min": -94.2182388305664,
      "activations/layer4_attention_weight_max": 123.54902648925781,
      "activations/layer4_attention_weight_min": -119.2494125366211,
      "activations/layer5_attention_weight_max": 60.895668029785156,
      "activations/layer5_attention_weight_min": -64.08778381347656,
      "activations/layer6_attention_weight_max": 51.09852600097656,
      "activations/layer6_attention_weight_min": -53.875186920166016,
      "activations/layer7_attention_weight_max": 91.86270904541016,
      "activations/layer7_attention_weight_min": -88.22781372070312,
      "activations/layer8_attention_weight_max": 42.88725662231445,
      "activations/layer8_attention_weight_min": -42.741371154785156,
      "activations/layer9_attention_weight_max": 37.83454132080078,
      "activations/layer9_attention_weight_min": -35.566410064697266,
      "epoch": 7.79,
      "learning_rate": 0.00010080227272727272,
      "loss": 2.8072,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_loss": 2.763671875,
      "eval_runtime": 8.521,
      "eval_samples_per_second": 503.931,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_openwebtext_loss": 2.763671875,
      "eval_openwebtext_ppl": 15.85796463907484,
      "eval_openwebtext_runtime": 8.521,
      "eval_openwebtext_samples_per_second": 503.931,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 2.0368,
      "eval_wikitext_samples_per_second": 223.88,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_lambada_loss": 2.669921875,
      "eval_lambada_ppl": 14.438841114275835,
      "eval_lambada_runtime": 9.5942,
      "eval_lambada_samples_per_second": 507.492,
      "step": 134000
    },
    {
      "activations/layer0_attention_weight_max": 14.503241539001465,
      "activations/layer0_attention_weight_min": -12.687862396240234,
      "activations/layer10_attention_weight_max": 43.88185501098633,
      "activations/layer10_attention_weight_min": -37.97352981567383,
      "activations/layer11_attention_weight_max": 49.50889205932617,
      "activations/layer11_attention_weight_min": -38.988990783691406,
      "activations/layer12_attention_weight_max": 53.05390930175781,
      "activations/layer12_attention_weight_min": -26.28006362915039,
      "activations/layer13_attention_weight_max": 73.39599609375,
      "activations/layer13_attention_weight_min": -42.08252716064453,
      "activations/layer14_attention_weight_max": 71.28373718261719,
      "activations/layer14_attention_weight_min": -45.68128204345703,
      "activations/layer15_attention_weight_max": 74.83802795410156,
      "activations/layer15_attention_weight_min": -37.40822219848633,
      "activations/layer16_attention_weight_max": 49.00468444824219,
      "activations/layer16_attention_weight_min": -29.073143005371094,
      "activations/layer17_attention_weight_max": 33.675575256347656,
      "activations/layer17_attention_weight_min": -28.446048736572266,
      "activations/layer18_attention_weight_max": 31.655231475830078,
      "activations/layer18_attention_weight_min": -25.40328598022461,
      "activations/layer19_attention_weight_max": 34.843353271484375,
      "activations/layer19_attention_weight_min": -32.4752082824707,
      "activations/layer1_attention_weight_max": 18.64544105529785,
      "activations/layer1_attention_weight_min": -14.930045127868652,
      "activations/layer20_attention_weight_max": 29.189334869384766,
      "activations/layer20_attention_weight_min": -24.8493709564209,
      "activations/layer21_attention_weight_max": 26.453125,
      "activations/layer21_attention_weight_min": -22.305540084838867,
      "activations/layer22_attention_weight_max": 45.02063751220703,
      "activations/layer22_attention_weight_min": -28.894989013671875,
      "activations/layer23_attention_weight_max": 36.043212890625,
      "activations/layer23_attention_weight_min": -23.254491806030273,
      "activations/layer2_attention_weight_max": 33.31277084350586,
      "activations/layer2_attention_weight_min": -32.53144836425781,
      "activations/layer3_attention_weight_max": 88.30201721191406,
      "activations/layer3_attention_weight_min": -92.06593322753906,
      "activations/layer4_attention_weight_max": 117.1950912475586,
      "activations/layer4_attention_weight_min": -113.55362701416016,
      "activations/layer5_attention_weight_max": 59.159786224365234,
      "activations/layer5_attention_weight_min": -60.152645111083984,
      "activations/layer6_attention_weight_max": 56.05850601196289,
      "activations/layer6_attention_weight_min": -51.44755172729492,
      "activations/layer7_attention_weight_max": 109.62516784667969,
      "activations/layer7_attention_weight_min": -89.93913269042969,
      "activations/layer8_attention_weight_max": 49.9769287109375,
      "activations/layer8_attention_weight_min": -46.49528884887695,
      "activations/layer9_attention_weight_max": 44.08676528930664,
      "activations/layer9_attention_weight_min": -41.017486572265625,
      "epoch": 7.79,
      "learning_rate": 0.00010078333333333332,
      "loss": 2.8221,
      "step": 134050
    },
    {
      "activations/layer0_attention_weight_max": 14.889971733093262,
      "activations/layer0_attention_weight_min": -13.286611557006836,
      "activations/layer10_attention_weight_max": 36.50049591064453,
      "activations/layer10_attention_weight_min": -33.10954666137695,
      "activations/layer11_attention_weight_max": 41.303672790527344,
      "activations/layer11_attention_weight_min": -33.13889694213867,
      "activations/layer12_attention_weight_max": 24.112009048461914,
      "activations/layer12_attention_weight_min": -23.983444213867188,
      "activations/layer13_attention_weight_max": 41.826961517333984,
      "activations/layer13_attention_weight_min": -34.4337043762207,
      "activations/layer14_attention_weight_max": 63.07968521118164,
      "activations/layer14_attention_weight_min": -42.80207061767578,
      "activations/layer15_attention_weight_max": 39.63615417480469,
      "activations/layer15_attention_weight_min": -33.52460861206055,
      "activations/layer16_attention_weight_max": 37.08259963989258,
      "activations/layer16_attention_weight_min": -29.557266235351562,
      "activations/layer17_attention_weight_max": 39.07503128051758,
      "activations/layer17_attention_weight_min": -26.896650314331055,
      "activations/layer18_attention_weight_max": 32.525901794433594,
      "activations/layer18_attention_weight_min": -21.94106674194336,
      "activations/layer19_attention_weight_max": 38.933982849121094,
      "activations/layer19_attention_weight_min": -31.67404556274414,
      "activations/layer1_attention_weight_max": 16.84974479675293,
      "activations/layer1_attention_weight_min": -15.440969467163086,
      "activations/layer20_attention_weight_max": 35.60690689086914,
      "activations/layer20_attention_weight_min": -24.929637908935547,
      "activations/layer21_attention_weight_max": 31.42256736755371,
      "activations/layer21_attention_weight_min": -21.80742835998535,
      "activations/layer22_attention_weight_max": 53.3973503112793,
      "activations/layer22_attention_weight_min": -32.109432220458984,
      "activations/layer23_attention_weight_max": 37.97993469238281,
      "activations/layer23_attention_weight_min": -23.022235870361328,
      "activations/layer2_attention_weight_max": 33.9267578125,
      "activations/layer2_attention_weight_min": -33.63719177246094,
      "activations/layer3_attention_weight_max": 95.92694091796875,
      "activations/layer3_attention_weight_min": -94.34650421142578,
      "activations/layer4_attention_weight_max": 122.99525451660156,
      "activations/layer4_attention_weight_min": -121.6201171875,
      "activations/layer5_attention_weight_max": 63.32167053222656,
      "activations/layer5_attention_weight_min": -66.22006225585938,
      "activations/layer6_attention_weight_max": 54.30088806152344,
      "activations/layer6_attention_weight_min": -54.89542007446289,
      "activations/layer7_attention_weight_max": 98.95262908935547,
      "activations/layer7_attention_weight_min": -94.46229553222656,
      "activations/layer8_attention_weight_max": 44.418800354003906,
      "activations/layer8_attention_weight_min": -43.20210647583008,
      "activations/layer9_attention_weight_max": 38.88631820678711,
      "activations/layer9_attention_weight_min": -37.109737396240234,
      "epoch": 7.79,
      "learning_rate": 0.00010076439393939394,
      "loss": 2.8151,
      "step": 134100
    },
    {
      "activations/layer0_attention_weight_max": 15.274510383605957,
      "activations/layer0_attention_weight_min": -12.87769889831543,
      "activations/layer10_attention_weight_max": 40.78860092163086,
      "activations/layer10_attention_weight_min": -39.220890045166016,
      "activations/layer11_attention_weight_max": 37.07560729980469,
      "activations/layer11_attention_weight_min": -33.534576416015625,
      "activations/layer12_attention_weight_max": 47.13887023925781,
      "activations/layer12_attention_weight_min": -25.150192260742188,
      "activations/layer13_attention_weight_max": 48.8585090637207,
      "activations/layer13_attention_weight_min": -35.123329162597656,
      "activations/layer14_attention_weight_max": 68.93814086914062,
      "activations/layer14_attention_weight_min": -44.4141960144043,
      "activations/layer15_attention_weight_max": 52.216007232666016,
      "activations/layer15_attention_weight_min": -32.09857177734375,
      "activations/layer16_attention_weight_max": 36.869380950927734,
      "activations/layer16_attention_weight_min": -28.323503494262695,
      "activations/layer17_attention_weight_max": 36.79166793823242,
      "activations/layer17_attention_weight_min": -25.16837501525879,
      "activations/layer18_attention_weight_max": 32.531375885009766,
      "activations/layer18_attention_weight_min": -22.253658294677734,
      "activations/layer19_attention_weight_max": 37.022499084472656,
      "activations/layer19_attention_weight_min": -31.049619674682617,
      "activations/layer1_attention_weight_max": 17.64686393737793,
      "activations/layer1_attention_weight_min": -15.893060684204102,
      "activations/layer20_attention_weight_max": 30.904029846191406,
      "activations/layer20_attention_weight_min": -25.985084533691406,
      "activations/layer21_attention_weight_max": 30.556865692138672,
      "activations/layer21_attention_weight_min": -22.354650497436523,
      "activations/layer22_attention_weight_max": 49.23218536376953,
      "activations/layer22_attention_weight_min": -29.233823776245117,
      "activations/layer23_attention_weight_max": 41.03325653076172,
      "activations/layer23_attention_weight_min": -24.082813262939453,
      "activations/layer2_attention_weight_max": 33.614013671875,
      "activations/layer2_attention_weight_min": -32.261940002441406,
      "activations/layer3_attention_weight_max": 95.93402099609375,
      "activations/layer3_attention_weight_min": -93.97175598144531,
      "activations/layer4_attention_weight_max": 122.4737319946289,
      "activations/layer4_attention_weight_min": -120.5466079711914,
      "activations/layer5_attention_weight_max": 60.29095458984375,
      "activations/layer5_attention_weight_min": -63.04937744140625,
      "activations/layer6_attention_weight_max": 53.034828186035156,
      "activations/layer6_attention_weight_min": -53.33332824707031,
      "activations/layer7_attention_weight_max": 93.92610931396484,
      "activations/layer7_attention_weight_min": -93.18991088867188,
      "activations/layer8_attention_weight_max": 42.755680084228516,
      "activations/layer8_attention_weight_min": -46.03276824951172,
      "activations/layer9_attention_weight_max": 39.80732345581055,
      "activations/layer9_attention_weight_min": -38.411739349365234,
      "epoch": 7.79,
      "learning_rate": 0.00010074545454545452,
      "loss": 2.8136,
      "step": 134150
    },
    {
      "activations/layer0_attention_weight_max": 14.928523063659668,
      "activations/layer0_attention_weight_min": -11.522865295410156,
      "activations/layer10_attention_weight_max": 33.401817321777344,
      "activations/layer10_attention_weight_min": -34.3801155090332,
      "activations/layer11_attention_weight_max": 34.907562255859375,
      "activations/layer11_attention_weight_min": -30.496105194091797,
      "activations/layer12_attention_weight_max": 32.83454132080078,
      "activations/layer12_attention_weight_min": -24.388031005859375,
      "activations/layer13_attention_weight_max": 43.76596450805664,
      "activations/layer13_attention_weight_min": -32.87930679321289,
      "activations/layer14_attention_weight_max": 65.8385238647461,
      "activations/layer14_attention_weight_min": -45.06023025512695,
      "activations/layer15_attention_weight_max": 37.53635787963867,
      "activations/layer15_attention_weight_min": -30.814882278442383,
      "activations/layer16_attention_weight_max": 34.977542877197266,
      "activations/layer16_attention_weight_min": -30.319623947143555,
      "activations/layer17_attention_weight_max": 37.950584411621094,
      "activations/layer17_attention_weight_min": -25.650714874267578,
      "activations/layer18_attention_weight_max": 33.55579376220703,
      "activations/layer18_attention_weight_min": -22.5196533203125,
      "activations/layer19_attention_weight_max": 35.19331741333008,
      "activations/layer19_attention_weight_min": -31.79833984375,
      "activations/layer1_attention_weight_max": 17.8547420501709,
      "activations/layer1_attention_weight_min": -16.046117782592773,
      "activations/layer20_attention_weight_max": 30.875463485717773,
      "activations/layer20_attention_weight_min": -24.420475006103516,
      "activations/layer21_attention_weight_max": 31.334402084350586,
      "activations/layer21_attention_weight_min": -23.314550399780273,
      "activations/layer22_attention_weight_max": 48.89114761352539,
      "activations/layer22_attention_weight_min": -29.94861602783203,
      "activations/layer23_attention_weight_max": 36.9671630859375,
      "activations/layer23_attention_weight_min": -23.817157745361328,
      "activations/layer2_attention_weight_max": 33.16895294189453,
      "activations/layer2_attention_weight_min": -31.840652465820312,
      "activations/layer3_attention_weight_max": 91.87629699707031,
      "activations/layer3_attention_weight_min": -94.69635772705078,
      "activations/layer4_attention_weight_max": 121.3071060180664,
      "activations/layer4_attention_weight_min": -117.4915771484375,
      "activations/layer5_attention_weight_max": 61.558265686035156,
      "activations/layer5_attention_weight_min": -63.30705261230469,
      "activations/layer6_attention_weight_max": 51.86014175415039,
      "activations/layer6_attention_weight_min": -54.14594268798828,
      "activations/layer7_attention_weight_max": 90.94190979003906,
      "activations/layer7_attention_weight_min": -96.41358184814453,
      "activations/layer8_attention_weight_max": 40.796897888183594,
      "activations/layer8_attention_weight_min": -44.09281921386719,
      "activations/layer9_attention_weight_max": 35.641563415527344,
      "activations/layer9_attention_weight_min": -35.5356559753418,
      "epoch": 7.8,
      "learning_rate": 0.00010072651515151514,
      "loss": 2.8168,
      "step": 134200
    },
    {
      "activations/layer0_attention_weight_max": 16.128562927246094,
      "activations/layer0_attention_weight_min": -13.115604400634766,
      "activations/layer10_attention_weight_max": 35.484527587890625,
      "activations/layer10_attention_weight_min": -33.86015701293945,
      "activations/layer11_attention_weight_max": 33.040367126464844,
      "activations/layer11_attention_weight_min": -30.306232452392578,
      "activations/layer12_attention_weight_max": 23.18998146057129,
      "activations/layer12_attention_weight_min": -24.561279296875,
      "activations/layer13_attention_weight_max": 38.11881637573242,
      "activations/layer13_attention_weight_min": -32.7869987487793,
      "activations/layer14_attention_weight_max": 47.21277618408203,
      "activations/layer14_attention_weight_min": -37.806514739990234,
      "activations/layer15_attention_weight_max": 35.24651336669922,
      "activations/layer15_attention_weight_min": -30.162473678588867,
      "activations/layer16_attention_weight_max": 34.220584869384766,
      "activations/layer16_attention_weight_min": -26.512651443481445,
      "activations/layer17_attention_weight_max": 31.92702865600586,
      "activations/layer17_attention_weight_min": -25.25151824951172,
      "activations/layer18_attention_weight_max": 31.39018440246582,
      "activations/layer18_attention_weight_min": -20.927593231201172,
      "activations/layer19_attention_weight_max": 34.72371292114258,
      "activations/layer19_attention_weight_min": -29.678171157836914,
      "activations/layer1_attention_weight_max": 16.018959045410156,
      "activations/layer1_attention_weight_min": -15.056791305541992,
      "activations/layer20_attention_weight_max": 31.214513778686523,
      "activations/layer20_attention_weight_min": -25.105762481689453,
      "activations/layer21_attention_weight_max": 29.223094940185547,
      "activations/layer21_attention_weight_min": -22.21199607849121,
      "activations/layer22_attention_weight_max": 42.65147018432617,
      "activations/layer22_attention_weight_min": -27.703149795532227,
      "activations/layer23_attention_weight_max": 35.49560546875,
      "activations/layer23_attention_weight_min": -25.087034225463867,
      "activations/layer2_attention_weight_max": 33.16077423095703,
      "activations/layer2_attention_weight_min": -32.87909698486328,
      "activations/layer3_attention_weight_max": 94.2295913696289,
      "activations/layer3_attention_weight_min": -95.57797241210938,
      "activations/layer4_attention_weight_max": 123.1912841796875,
      "activations/layer4_attention_weight_min": -124.90919494628906,
      "activations/layer5_attention_weight_max": 64.78648376464844,
      "activations/layer5_attention_weight_min": -64.76564025878906,
      "activations/layer6_attention_weight_max": 53.84134292602539,
      "activations/layer6_attention_weight_min": -57.05814743041992,
      "activations/layer7_attention_weight_max": 92.46009063720703,
      "activations/layer7_attention_weight_min": -96.59133911132812,
      "activations/layer8_attention_weight_max": 42.05733871459961,
      "activations/layer8_attention_weight_min": -44.48967742919922,
      "activations/layer9_attention_weight_max": 37.89603805541992,
      "activations/layer9_attention_weight_min": -36.27831268310547,
      "epoch": 7.8,
      "learning_rate": 0.00010070757575757575,
      "loss": 2.8072,
      "step": 134250
    },
    {
      "activations/layer0_attention_weight_max": 15.687912940979004,
      "activations/layer0_attention_weight_min": -13.094759941101074,
      "activations/layer10_attention_weight_max": 33.708805084228516,
      "activations/layer10_attention_weight_min": -30.672061920166016,
      "activations/layer11_attention_weight_max": 31.477062225341797,
      "activations/layer11_attention_weight_min": -29.54474449157715,
      "activations/layer12_attention_weight_max": 23.7540225982666,
      "activations/layer12_attention_weight_min": -22.793054580688477,
      "activations/layer13_attention_weight_max": 40.611419677734375,
      "activations/layer13_attention_weight_min": -33.18488693237305,
      "activations/layer14_attention_weight_max": 51.81535339355469,
      "activations/layer14_attention_weight_min": -44.31401062011719,
      "activations/layer15_attention_weight_max": 36.23300552368164,
      "activations/layer15_attention_weight_min": -29.363771438598633,
      "activations/layer16_attention_weight_max": 31.877946853637695,
      "activations/layer16_attention_weight_min": -27.652746200561523,
      "activations/layer17_attention_weight_max": 34.6120491027832,
      "activations/layer17_attention_weight_min": -27.43216896057129,
      "activations/layer18_attention_weight_max": 31.99517250061035,
      "activations/layer18_attention_weight_min": -21.71845817565918,
      "activations/layer19_attention_weight_max": 36.12364196777344,
      "activations/layer19_attention_weight_min": -30.233701705932617,
      "activations/layer1_attention_weight_max": 17.813838958740234,
      "activations/layer1_attention_weight_min": -15.143876075744629,
      "activations/layer20_attention_weight_max": 32.47890090942383,
      "activations/layer20_attention_weight_min": -24.48615074157715,
      "activations/layer21_attention_weight_max": 32.29607009887695,
      "activations/layer21_attention_weight_min": -22.389734268188477,
      "activations/layer22_attention_weight_max": 52.91054916381836,
      "activations/layer22_attention_weight_min": -28.26792335510254,
      "activations/layer23_attention_weight_max": 37.447532653808594,
      "activations/layer23_attention_weight_min": -23.438587188720703,
      "activations/layer2_attention_weight_max": 33.844261169433594,
      "activations/layer2_attention_weight_min": -31.090042114257812,
      "activations/layer3_attention_weight_max": 92.34674835205078,
      "activations/layer3_attention_weight_min": -89.37059783935547,
      "activations/layer4_attention_weight_max": 121.3087158203125,
      "activations/layer4_attention_weight_min": -115.35576629638672,
      "activations/layer5_attention_weight_max": 62.83019256591797,
      "activations/layer5_attention_weight_min": -59.75334930419922,
      "activations/layer6_attention_weight_max": 50.21937561035156,
      "activations/layer6_attention_weight_min": -50.21843338012695,
      "activations/layer7_attention_weight_max": 89.0145034790039,
      "activations/layer7_attention_weight_min": -89.33609771728516,
      "activations/layer8_attention_weight_max": 42.72846221923828,
      "activations/layer8_attention_weight_min": -40.59279251098633,
      "activations/layer9_attention_weight_max": 39.310359954833984,
      "activations/layer9_attention_weight_min": -36.25749206542969,
      "epoch": 7.8,
      "learning_rate": 0.00010068901515151514,
      "loss": 2.8228,
      "step": 134300
    },
    {
      "activations/layer0_attention_weight_max": 16.070016860961914,
      "activations/layer0_attention_weight_min": -11.496063232421875,
      "activations/layer10_attention_weight_max": 30.55320930480957,
      "activations/layer10_attention_weight_min": -29.319808959960938,
      "activations/layer11_attention_weight_max": 32.72013473510742,
      "activations/layer11_attention_weight_min": -28.596237182617188,
      "activations/layer12_attention_weight_max": 24.362157821655273,
      "activations/layer12_attention_weight_min": -24.01694107055664,
      "activations/layer13_attention_weight_max": 43.99057388305664,
      "activations/layer13_attention_weight_min": -31.762325286865234,
      "activations/layer14_attention_weight_max": 46.47941207885742,
      "activations/layer14_attention_weight_min": -39.16826629638672,
      "activations/layer15_attention_weight_max": 38.84901428222656,
      "activations/layer15_attention_weight_min": -28.800264358520508,
      "activations/layer16_attention_weight_max": 37.679908752441406,
      "activations/layer16_attention_weight_min": -30.67561912536621,
      "activations/layer17_attention_weight_max": 34.53166580200195,
      "activations/layer17_attention_weight_min": -23.571901321411133,
      "activations/layer18_attention_weight_max": 32.25141525268555,
      "activations/layer18_attention_weight_min": -20.687786102294922,
      "activations/layer19_attention_weight_max": 39.795536041259766,
      "activations/layer19_attention_weight_min": -28.384647369384766,
      "activations/layer1_attention_weight_max": 16.887659072875977,
      "activations/layer1_attention_weight_min": -14.878175735473633,
      "activations/layer20_attention_weight_max": 36.08396530151367,
      "activations/layer20_attention_weight_min": -23.714696884155273,
      "activations/layer21_attention_weight_max": 31.67624855041504,
      "activations/layer21_attention_weight_min": -21.163665771484375,
      "activations/layer22_attention_weight_max": 49.188011169433594,
      "activations/layer22_attention_weight_min": -27.58086395263672,
      "activations/layer23_attention_weight_max": 32.543243408203125,
      "activations/layer23_attention_weight_min": -23.259733200073242,
      "activations/layer2_attention_weight_max": 32.61102294921875,
      "activations/layer2_attention_weight_min": -31.27385711669922,
      "activations/layer3_attention_weight_max": 88.87923431396484,
      "activations/layer3_attention_weight_min": -90.99333190917969,
      "activations/layer4_attention_weight_max": 120.69403076171875,
      "activations/layer4_attention_weight_min": -121.23176574707031,
      "activations/layer5_attention_weight_max": 60.37518310546875,
      "activations/layer5_attention_weight_min": -61.038795471191406,
      "activations/layer6_attention_weight_max": 50.92076110839844,
      "activations/layer6_attention_weight_min": -50.7706413269043,
      "activations/layer7_attention_weight_max": 82.19732666015625,
      "activations/layer7_attention_weight_min": -87.98821258544922,
      "activations/layer8_attention_weight_max": 38.00431823730469,
      "activations/layer8_attention_weight_min": -39.3226203918457,
      "activations/layer9_attention_weight_max": 32.47623825073242,
      "activations/layer9_attention_weight_min": -33.39850997924805,
      "epoch": 7.81,
      "learning_rate": 0.00010067007575757575,
      "loss": 2.7845,
      "step": 134350
    },
    {
      "activations/layer0_attention_weight_max": 14.978792190551758,
      "activations/layer0_attention_weight_min": -11.751681327819824,
      "activations/layer10_attention_weight_max": 34.992652893066406,
      "activations/layer10_attention_weight_min": -34.56706619262695,
      "activations/layer11_attention_weight_max": 35.991580963134766,
      "activations/layer11_attention_weight_min": -31.908002853393555,
      "activations/layer12_attention_weight_max": 31.763652801513672,
      "activations/layer12_attention_weight_min": -25.88585662841797,
      "activations/layer13_attention_weight_max": 42.01658248901367,
      "activations/layer13_attention_weight_min": -38.95781707763672,
      "activations/layer14_attention_weight_max": 60.53990936279297,
      "activations/layer14_attention_weight_min": -48.71171569824219,
      "activations/layer15_attention_weight_max": 36.72502517700195,
      "activations/layer15_attention_weight_min": -31.38880729675293,
      "activations/layer16_attention_weight_max": 34.377159118652344,
      "activations/layer16_attention_weight_min": -30.783052444458008,
      "activations/layer17_attention_weight_max": 34.92040252685547,
      "activations/layer17_attention_weight_min": -25.349693298339844,
      "activations/layer18_attention_weight_max": 41.566368103027344,
      "activations/layer18_attention_weight_min": -23.403968811035156,
      "activations/layer19_attention_weight_max": 38.30607604980469,
      "activations/layer19_attention_weight_min": -30.569721221923828,
      "activations/layer1_attention_weight_max": 15.786723136901855,
      "activations/layer1_attention_weight_min": -14.724157333374023,
      "activations/layer20_attention_weight_max": 36.18492126464844,
      "activations/layer20_attention_weight_min": -25.3842830657959,
      "activations/layer21_attention_weight_max": 33.31159210205078,
      "activations/layer21_attention_weight_min": -23.164703369140625,
      "activations/layer22_attention_weight_max": 52.58964920043945,
      "activations/layer22_attention_weight_min": -28.745882034301758,
      "activations/layer23_attention_weight_max": 35.75838088989258,
      "activations/layer23_attention_weight_min": -25.101669311523438,
      "activations/layer2_attention_weight_max": 32.443702697753906,
      "activations/layer2_attention_weight_min": -32.706764221191406,
      "activations/layer3_attention_weight_max": 91.71063232421875,
      "activations/layer3_attention_weight_min": -97.1413803100586,
      "activations/layer4_attention_weight_max": 122.37386322021484,
      "activations/layer4_attention_weight_min": -123.91707611083984,
      "activations/layer5_attention_weight_max": 61.692405700683594,
      "activations/layer5_attention_weight_min": -65.31556701660156,
      "activations/layer6_attention_weight_max": 51.77578353881836,
      "activations/layer6_attention_weight_min": -57.31047439575195,
      "activations/layer7_attention_weight_max": 93.47296142578125,
      "activations/layer7_attention_weight_min": -96.46932983398438,
      "activations/layer8_attention_weight_max": 41.1850471496582,
      "activations/layer8_attention_weight_min": -43.305179595947266,
      "activations/layer9_attention_weight_max": 36.193275451660156,
      "activations/layer9_attention_weight_min": -35.845909118652344,
      "epoch": 7.81,
      "learning_rate": 0.00010065113636363635,
      "loss": 2.8161,
      "step": 134400
    },
    {
      "activations/layer0_attention_weight_max": 15.187996864318848,
      "activations/layer0_attention_weight_min": -12.577760696411133,
      "activations/layer10_attention_weight_max": 32.07077407836914,
      "activations/layer10_attention_weight_min": -31.167924880981445,
      "activations/layer11_attention_weight_max": 32.4705696105957,
      "activations/layer11_attention_weight_min": -30.280960083007812,
      "activations/layer12_attention_weight_max": 38.33991622924805,
      "activations/layer12_attention_weight_min": -24.38123893737793,
      "activations/layer13_attention_weight_max": 47.079463958740234,
      "activations/layer13_attention_weight_min": -36.1185302734375,
      "activations/layer14_attention_weight_max": 61.167137145996094,
      "activations/layer14_attention_weight_min": -47.6264533996582,
      "activations/layer15_attention_weight_max": 35.52196502685547,
      "activations/layer15_attention_weight_min": -28.418397903442383,
      "activations/layer16_attention_weight_max": 31.458751678466797,
      "activations/layer16_attention_weight_min": -27.325145721435547,
      "activations/layer17_attention_weight_max": 36.16166305541992,
      "activations/layer17_attention_weight_min": -27.144697189331055,
      "activations/layer18_attention_weight_max": 33.25481414794922,
      "activations/layer18_attention_weight_min": -23.741535186767578,
      "activations/layer19_attention_weight_max": 33.764278411865234,
      "activations/layer19_attention_weight_min": -29.14101219177246,
      "activations/layer1_attention_weight_max": 16.727128982543945,
      "activations/layer1_attention_weight_min": -16.468978881835938,
      "activations/layer20_attention_weight_max": 31.0511417388916,
      "activations/layer20_attention_weight_min": -23.034502029418945,
      "activations/layer21_attention_weight_max": 29.447158813476562,
      "activations/layer21_attention_weight_min": -22.136680603027344,
      "activations/layer22_attention_weight_max": 46.515953063964844,
      "activations/layer22_attention_weight_min": -30.15338134765625,
      "activations/layer23_attention_weight_max": 35.73927688598633,
      "activations/layer23_attention_weight_min": -24.89112091064453,
      "activations/layer2_attention_weight_max": 33.35369873046875,
      "activations/layer2_attention_weight_min": -31.987529754638672,
      "activations/layer3_attention_weight_max": 93.17838287353516,
      "activations/layer3_attention_weight_min": -90.36195373535156,
      "activations/layer4_attention_weight_max": 122.09539794921875,
      "activations/layer4_attention_weight_min": -117.7238540649414,
      "activations/layer5_attention_weight_max": 60.67134094238281,
      "activations/layer5_attention_weight_min": -61.698394775390625,
      "activations/layer6_attention_weight_max": 51.44100570678711,
      "activations/layer6_attention_weight_min": -52.16794204711914,
      "activations/layer7_attention_weight_max": 86.34473419189453,
      "activations/layer7_attention_weight_min": -84.64879608154297,
      "activations/layer8_attention_weight_max": 38.65418243408203,
      "activations/layer8_attention_weight_min": -42.88225555419922,
      "activations/layer9_attention_weight_max": 33.9532470703125,
      "activations/layer9_attention_weight_min": -32.40470504760742,
      "epoch": 7.81,
      "learning_rate": 0.00010063219696969696,
      "loss": 2.8167,
      "step": 134450
    },
    {
      "activations/layer0_attention_weight_max": 16.019989013671875,
      "activations/layer0_attention_weight_min": -12.50975227355957,
      "activations/layer10_attention_weight_max": 41.05043029785156,
      "activations/layer10_attention_weight_min": -36.205162048339844,
      "activations/layer11_attention_weight_max": 40.756099700927734,
      "activations/layer11_attention_weight_min": -33.73527526855469,
      "activations/layer12_attention_weight_max": 23.787227630615234,
      "activations/layer12_attention_weight_min": -32.68599319458008,
      "activations/layer13_attention_weight_max": 40.73076629638672,
      "activations/layer13_attention_weight_min": -31.718032836914062,
      "activations/layer14_attention_weight_max": 50.699737548828125,
      "activations/layer14_attention_weight_min": -42.2177619934082,
      "activations/layer15_attention_weight_max": 39.5081672668457,
      "activations/layer15_attention_weight_min": -31.377683639526367,
      "activations/layer16_attention_weight_max": 36.64823532104492,
      "activations/layer16_attention_weight_min": -28.766826629638672,
      "activations/layer17_attention_weight_max": 36.66823196411133,
      "activations/layer17_attention_weight_min": -26.229394912719727,
      "activations/layer18_attention_weight_max": 32.92266082763672,
      "activations/layer18_attention_weight_min": -21.992162704467773,
      "activations/layer19_attention_weight_max": 38.998779296875,
      "activations/layer19_attention_weight_min": -30.592878341674805,
      "activations/layer1_attention_weight_max": 16.77235984802246,
      "activations/layer1_attention_weight_min": -15.01106071472168,
      "activations/layer20_attention_weight_max": 30.775676727294922,
      "activations/layer20_attention_weight_min": -25.3964786529541,
      "activations/layer21_attention_weight_max": 31.98482322692871,
      "activations/layer21_attention_weight_min": -23.937833786010742,
      "activations/layer22_attention_weight_max": 53.75040817260742,
      "activations/layer22_attention_weight_min": -29.83380889892578,
      "activations/layer23_attention_weight_max": 36.97173309326172,
      "activations/layer23_attention_weight_min": -21.19736671447754,
      "activations/layer2_attention_weight_max": 33.9295654296875,
      "activations/layer2_attention_weight_min": -34.01277160644531,
      "activations/layer3_attention_weight_max": 96.04637145996094,
      "activations/layer3_attention_weight_min": -98.10655212402344,
      "activations/layer4_attention_weight_max": 122.28382873535156,
      "activations/layer4_attention_weight_min": -121.11783599853516,
      "activations/layer5_attention_weight_max": 58.64982223510742,
      "activations/layer5_attention_weight_min": -64.95912170410156,
      "activations/layer6_attention_weight_max": 51.37810516357422,
      "activations/layer6_attention_weight_min": -57.3048210144043,
      "activations/layer7_attention_weight_max": 107.84901428222656,
      "activations/layer7_attention_weight_min": -94.32830047607422,
      "activations/layer8_attention_weight_max": 46.386600494384766,
      "activations/layer8_attention_weight_min": -40.85179901123047,
      "activations/layer9_attention_weight_max": 39.75590133666992,
      "activations/layer9_attention_weight_min": -37.03572463989258,
      "epoch": 7.82,
      "learning_rate": 0.00010061325757575756,
      "loss": 2.8033,
      "step": 134500
    },
    {
      "activations/layer0_attention_weight_max": 15.640419006347656,
      "activations/layer0_attention_weight_min": -11.942103385925293,
      "activations/layer10_attention_weight_max": 33.95081329345703,
      "activations/layer10_attention_weight_min": -31.18485450744629,
      "activations/layer11_attention_weight_max": 32.09557342529297,
      "activations/layer11_attention_weight_min": -29.618541717529297,
      "activations/layer12_attention_weight_max": 22.032520294189453,
      "activations/layer12_attention_weight_min": -26.853195190429688,
      "activations/layer13_attention_weight_max": 41.749813079833984,
      "activations/layer13_attention_weight_min": -32.028865814208984,
      "activations/layer14_attention_weight_max": 49.066505432128906,
      "activations/layer14_attention_weight_min": -42.87430191040039,
      "activations/layer15_attention_weight_max": 37.40168762207031,
      "activations/layer15_attention_weight_min": -28.469764709472656,
      "activations/layer16_attention_weight_max": 30.06711196899414,
      "activations/layer16_attention_weight_min": -26.988298416137695,
      "activations/layer17_attention_weight_max": 34.020145416259766,
      "activations/layer17_attention_weight_min": -26.710887908935547,
      "activations/layer18_attention_weight_max": 31.93536376953125,
      "activations/layer18_attention_weight_min": -21.04251480102539,
      "activations/layer19_attention_weight_max": 33.538883209228516,
      "activations/layer19_attention_weight_min": -28.7548828125,
      "activations/layer1_attention_weight_max": 18.174898147583008,
      "activations/layer1_attention_weight_min": -14.53142261505127,
      "activations/layer20_attention_weight_max": 29.337322235107422,
      "activations/layer20_attention_weight_min": -24.585081100463867,
      "activations/layer21_attention_weight_max": 28.775632858276367,
      "activations/layer21_attention_weight_min": -23.054792404174805,
      "activations/layer22_attention_weight_max": 44.16112518310547,
      "activations/layer22_attention_weight_min": -29.33547592163086,
      "activations/layer23_attention_weight_max": 36.67255401611328,
      "activations/layer23_attention_weight_min": -24.220060348510742,
      "activations/layer2_attention_weight_max": 32.95433807373047,
      "activations/layer2_attention_weight_min": -31.307891845703125,
      "activations/layer3_attention_weight_max": 94.68558502197266,
      "activations/layer3_attention_weight_min": -91.73127746582031,
      "activations/layer4_attention_weight_max": 121.08855438232422,
      "activations/layer4_attention_weight_min": -115.78717041015625,
      "activations/layer5_attention_weight_max": 62.447479248046875,
      "activations/layer5_attention_weight_min": -60.86074447631836,
      "activations/layer6_attention_weight_max": 51.00823211669922,
      "activations/layer6_attention_weight_min": -52.86418914794922,
      "activations/layer7_attention_weight_max": 89.19800567626953,
      "activations/layer7_attention_weight_min": -92.70438385009766,
      "activations/layer8_attention_weight_max": 41.638023376464844,
      "activations/layer8_attention_weight_min": -39.50257873535156,
      "activations/layer9_attention_weight_max": 36.15068817138672,
      "activations/layer9_attention_weight_min": -32.93117904663086,
      "epoch": 7.82,
      "learning_rate": 0.00010059431818181818,
      "loss": 2.8007,
      "step": 134550
    },
    {
      "activations/layer0_attention_weight_max": 15.25040054321289,
      "activations/layer0_attention_weight_min": -12.406542778015137,
      "activations/layer10_attention_weight_max": 34.991878509521484,
      "activations/layer10_attention_weight_min": -33.36826705932617,
      "activations/layer11_attention_weight_max": 36.64238357543945,
      "activations/layer11_attention_weight_min": -32.088661193847656,
      "activations/layer12_attention_weight_max": 27.14297866821289,
      "activations/layer12_attention_weight_min": -28.1372127532959,
      "activations/layer13_attention_weight_max": 45.00066375732422,
      "activations/layer13_attention_weight_min": -35.710853576660156,
      "activations/layer14_attention_weight_max": 59.261474609375,
      "activations/layer14_attention_weight_min": -43.028812408447266,
      "activations/layer15_attention_weight_max": 44.06485366821289,
      "activations/layer15_attention_weight_min": -31.0528621673584,
      "activations/layer16_attention_weight_max": 36.70347213745117,
      "activations/layer16_attention_weight_min": -27.75670623779297,
      "activations/layer17_attention_weight_max": 38.194766998291016,
      "activations/layer17_attention_weight_min": -26.662118911743164,
      "activations/layer18_attention_weight_max": 37.35117721557617,
      "activations/layer18_attention_weight_min": -20.79975128173828,
      "activations/layer19_attention_weight_max": 40.49872970581055,
      "activations/layer19_attention_weight_min": -32.441162109375,
      "activations/layer1_attention_weight_max": 18.803653717041016,
      "activations/layer1_attention_weight_min": -14.00080680847168,
      "activations/layer20_attention_weight_max": 34.99335479736328,
      "activations/layer20_attention_weight_min": -25.060895919799805,
      "activations/layer21_attention_weight_max": 34.92793273925781,
      "activations/layer21_attention_weight_min": -23.208951950073242,
      "activations/layer22_attention_weight_max": 58.81888198852539,
      "activations/layer22_attention_weight_min": -29.896041870117188,
      "activations/layer23_attention_weight_max": 39.36888885498047,
      "activations/layer23_attention_weight_min": -25.907651901245117,
      "activations/layer2_attention_weight_max": 32.61466598510742,
      "activations/layer2_attention_weight_min": -31.545209884643555,
      "activations/layer3_attention_weight_max": 93.18093872070312,
      "activations/layer3_attention_weight_min": -96.82711791992188,
      "activations/layer4_attention_weight_max": 123.04119110107422,
      "activations/layer4_attention_weight_min": -119.89562225341797,
      "activations/layer5_attention_weight_max": 61.70909118652344,
      "activations/layer5_attention_weight_min": -62.991722106933594,
      "activations/layer6_attention_weight_max": 51.18033218383789,
      "activations/layer6_attention_weight_min": -52.944217681884766,
      "activations/layer7_attention_weight_max": 100.24429321289062,
      "activations/layer7_attention_weight_min": -91.22235870361328,
      "activations/layer8_attention_weight_max": 44.1104621887207,
      "activations/layer8_attention_weight_min": -42.66892623901367,
      "activations/layer9_attention_weight_max": 38.05155944824219,
      "activations/layer9_attention_weight_min": -34.3075065612793,
      "epoch": 7.82,
      "learning_rate": 0.00010057575757575757,
      "loss": 2.8226,
      "step": 134600
    },
    {
      "activations/layer0_attention_weight_max": 16.795074462890625,
      "activations/layer0_attention_weight_min": -12.040434837341309,
      "activations/layer10_attention_weight_max": 31.166606903076172,
      "activations/layer10_attention_weight_min": -29.748416900634766,
      "activations/layer11_attention_weight_max": 31.59756088256836,
      "activations/layer11_attention_weight_min": -30.13104248046875,
      "activations/layer12_attention_weight_max": 26.44767951965332,
      "activations/layer12_attention_weight_min": -25.579647064208984,
      "activations/layer13_attention_weight_max": 44.26835632324219,
      "activations/layer13_attention_weight_min": -30.858016967773438,
      "activations/layer14_attention_weight_max": 46.09050750732422,
      "activations/layer14_attention_weight_min": -37.535362243652344,
      "activations/layer15_attention_weight_max": 36.89683532714844,
      "activations/layer15_attention_weight_min": -29.26199722290039,
      "activations/layer16_attention_weight_max": 35.1211051940918,
      "activations/layer16_attention_weight_min": -27.38437843322754,
      "activations/layer17_attention_weight_max": 36.49623489379883,
      "activations/layer17_attention_weight_min": -24.27516746520996,
      "activations/layer18_attention_weight_max": 37.882720947265625,
      "activations/layer18_attention_weight_min": -22.149782180786133,
      "activations/layer19_attention_weight_max": 36.30416488647461,
      "activations/layer19_attention_weight_min": -29.518972396850586,
      "activations/layer1_attention_weight_max": 17.07217788696289,
      "activations/layer1_attention_weight_min": -13.926253318786621,
      "activations/layer20_attention_weight_max": 32.35675811767578,
      "activations/layer20_attention_weight_min": -23.706819534301758,
      "activations/layer21_attention_weight_max": 33.17832946777344,
      "activations/layer21_attention_weight_min": -21.112518310546875,
      "activations/layer22_attention_weight_max": 49.06930160522461,
      "activations/layer22_attention_weight_min": -28.147172927856445,
      "activations/layer23_attention_weight_max": 37.67153549194336,
      "activations/layer23_attention_weight_min": -22.56471061706543,
      "activations/layer2_attention_weight_max": 33.339725494384766,
      "activations/layer2_attention_weight_min": -32.29533767700195,
      "activations/layer3_attention_weight_max": 90.83421325683594,
      "activations/layer3_attention_weight_min": -92.64501953125,
      "activations/layer4_attention_weight_max": 119.71114349365234,
      "activations/layer4_attention_weight_min": -116.4916000366211,
      "activations/layer5_attention_weight_max": 59.03030014038086,
      "activations/layer5_attention_weight_min": -61.691619873046875,
      "activations/layer6_attention_weight_max": 48.90111541748047,
      "activations/layer6_attention_weight_min": -50.47534942626953,
      "activations/layer7_attention_weight_max": 83.04806518554688,
      "activations/layer7_attention_weight_min": -86.01338195800781,
      "activations/layer8_attention_weight_max": 39.78091812133789,
      "activations/layer8_attention_weight_min": -38.66964340209961,
      "activations/layer9_attention_weight_max": 34.10893249511719,
      "activations/layer9_attention_weight_min": -33.720829010009766,
      "epoch": 7.82,
      "learning_rate": 0.00010055681818181817,
      "loss": 2.8161,
      "step": 134650
    },
    {
      "activations/layer0_attention_weight_max": 15.690339088439941,
      "activations/layer0_attention_weight_min": -11.704850196838379,
      "activations/layer10_attention_weight_max": 33.460079193115234,
      "activations/layer10_attention_weight_min": -31.96490478515625,
      "activations/layer11_attention_weight_max": 35.72922897338867,
      "activations/layer11_attention_weight_min": -31.157466888427734,
      "activations/layer12_attention_weight_max": 23.793107986450195,
      "activations/layer12_attention_weight_min": -26.161481857299805,
      "activations/layer13_attention_weight_max": 51.78574752807617,
      "activations/layer13_attention_weight_min": -35.0429801940918,
      "activations/layer14_attention_weight_max": 64.5685806274414,
      "activations/layer14_attention_weight_min": -45.591461181640625,
      "activations/layer15_attention_weight_max": 37.65452194213867,
      "activations/layer15_attention_weight_min": -28.970972061157227,
      "activations/layer16_attention_weight_max": 42.64863967895508,
      "activations/layer16_attention_weight_min": -27.19904136657715,
      "activations/layer17_attention_weight_max": 34.610260009765625,
      "activations/layer17_attention_weight_min": -25.672677993774414,
      "activations/layer18_attention_weight_max": 32.0673942565918,
      "activations/layer18_attention_weight_min": -21.103431701660156,
      "activations/layer19_attention_weight_max": 47.46611022949219,
      "activations/layer19_attention_weight_min": -29.715682983398438,
      "activations/layer1_attention_weight_max": 16.197153091430664,
      "activations/layer1_attention_weight_min": -13.627784729003906,
      "activations/layer20_attention_weight_max": 36.0653076171875,
      "activations/layer20_attention_weight_min": -25.2734432220459,
      "activations/layer21_attention_weight_max": 31.003684997558594,
      "activations/layer21_attention_weight_min": -22.484546661376953,
      "activations/layer22_attention_weight_max": 52.85142135620117,
      "activations/layer22_attention_weight_min": -28.247108459472656,
      "activations/layer23_attention_weight_max": 43.003631591796875,
      "activations/layer23_attention_weight_min": -22.919017791748047,
      "activations/layer2_attention_weight_max": 32.7706298828125,
      "activations/layer2_attention_weight_min": -32.359920501708984,
      "activations/layer3_attention_weight_max": 89.2412338256836,
      "activations/layer3_attention_weight_min": -91.73560333251953,
      "activations/layer4_attention_weight_max": 119.06280517578125,
      "activations/layer4_attention_weight_min": -119.20355224609375,
      "activations/layer5_attention_weight_max": 57.459136962890625,
      "activations/layer5_attention_weight_min": -61.68077087402344,
      "activations/layer6_attention_weight_max": 49.31598663330078,
      "activations/layer6_attention_weight_min": -51.03593444824219,
      "activations/layer7_attention_weight_max": 91.11421966552734,
      "activations/layer7_attention_weight_min": -84.1849365234375,
      "activations/layer8_attention_weight_max": 42.55351638793945,
      "activations/layer8_attention_weight_min": -42.23923110961914,
      "activations/layer9_attention_weight_max": 38.25994873046875,
      "activations/layer9_attention_weight_min": -33.81724166870117,
      "epoch": 7.83,
      "learning_rate": 0.00010053787878787877,
      "loss": 2.8275,
      "step": 134700
    },
    {
      "activations/layer0_attention_weight_max": 16.722091674804688,
      "activations/layer0_attention_weight_min": -12.306367874145508,
      "activations/layer10_attention_weight_max": 31.324283599853516,
      "activations/layer10_attention_weight_min": -31.593204498291016,
      "activations/layer11_attention_weight_max": 30.181198120117188,
      "activations/layer11_attention_weight_min": -29.291248321533203,
      "activations/layer12_attention_weight_max": 22.231857299804688,
      "activations/layer12_attention_weight_min": -27.646835327148438,
      "activations/layer13_attention_weight_max": 39.69213104248047,
      "activations/layer13_attention_weight_min": -31.538921356201172,
      "activations/layer14_attention_weight_max": 49.921730041503906,
      "activations/layer14_attention_weight_min": -41.6360969543457,
      "activations/layer15_attention_weight_max": 37.234195709228516,
      "activations/layer15_attention_weight_min": -29.734251022338867,
      "activations/layer16_attention_weight_max": 34.43362808227539,
      "activations/layer16_attention_weight_min": -28.11150360107422,
      "activations/layer17_attention_weight_max": 36.171077728271484,
      "activations/layer17_attention_weight_min": -25.194499969482422,
      "activations/layer18_attention_weight_max": 33.574893951416016,
      "activations/layer18_attention_weight_min": -22.2713680267334,
      "activations/layer19_attention_weight_max": 36.52622604370117,
      "activations/layer19_attention_weight_min": -30.640634536743164,
      "activations/layer1_attention_weight_max": 17.20134162902832,
      "activations/layer1_attention_weight_min": -14.637150764465332,
      "activations/layer20_attention_weight_max": 33.04513168334961,
      "activations/layer20_attention_weight_min": -24.190011978149414,
      "activations/layer21_attention_weight_max": 29.83467674255371,
      "activations/layer21_attention_weight_min": -22.0255069732666,
      "activations/layer22_attention_weight_max": 47.77412414550781,
      "activations/layer22_attention_weight_min": -27.977802276611328,
      "activations/layer23_attention_weight_max": 34.41606140136719,
      "activations/layer23_attention_weight_min": -21.355093002319336,
      "activations/layer2_attention_weight_max": 31.29524803161621,
      "activations/layer2_attention_weight_min": -31.21274185180664,
      "activations/layer3_attention_weight_max": 92.21722412109375,
      "activations/layer3_attention_weight_min": -96.9054946899414,
      "activations/layer4_attention_weight_max": 125.95643615722656,
      "activations/layer4_attention_weight_min": -120.44002532958984,
      "activations/layer5_attention_weight_max": 61.672298431396484,
      "activations/layer5_attention_weight_min": -61.73724365234375,
      "activations/layer6_attention_weight_max": 48.9345588684082,
      "activations/layer6_attention_weight_min": -50.776302337646484,
      "activations/layer7_attention_weight_max": 86.20589447021484,
      "activations/layer7_attention_weight_min": -84.39717864990234,
      "activations/layer8_attention_weight_max": 38.10810089111328,
      "activations/layer8_attention_weight_min": -39.5523567199707,
      "activations/layer9_attention_weight_max": 32.832088470458984,
      "activations/layer9_attention_weight_min": -31.643329620361328,
      "epoch": 7.83,
      "learning_rate": 0.00010051893939393939,
      "loss": 2.8141,
      "step": 134750
    },
    {
      "activations/layer0_attention_weight_max": 16.593236923217773,
      "activations/layer0_attention_weight_min": -11.84269905090332,
      "activations/layer10_attention_weight_max": 33.336578369140625,
      "activations/layer10_attention_weight_min": -31.89743995666504,
      "activations/layer11_attention_weight_max": 33.8359489440918,
      "activations/layer11_attention_weight_min": -32.0765380859375,
      "activations/layer12_attention_weight_max": 24.458980560302734,
      "activations/layer12_attention_weight_min": -26.782320022583008,
      "activations/layer13_attention_weight_max": 42.67079544067383,
      "activations/layer13_attention_weight_min": -35.15117263793945,
      "activations/layer14_attention_weight_max": 57.11103057861328,
      "activations/layer14_attention_weight_min": -44.32826614379883,
      "activations/layer15_attention_weight_max": 42.94480895996094,
      "activations/layer15_attention_weight_min": -29.79552459716797,
      "activations/layer16_attention_weight_max": 34.76078414916992,
      "activations/layer16_attention_weight_min": -29.40013313293457,
      "activations/layer17_attention_weight_max": 34.87742233276367,
      "activations/layer17_attention_weight_min": -25.239784240722656,
      "activations/layer18_attention_weight_max": 32.22679901123047,
      "activations/layer18_attention_weight_min": -23.83355712890625,
      "activations/layer19_attention_weight_max": 40.234798431396484,
      "activations/layer19_attention_weight_min": -31.8867244720459,
      "activations/layer1_attention_weight_max": 16.876890182495117,
      "activations/layer1_attention_weight_min": -14.704278945922852,
      "activations/layer20_attention_weight_max": 36.659324645996094,
      "activations/layer20_attention_weight_min": -24.898887634277344,
      "activations/layer21_attention_weight_max": 33.413116455078125,
      "activations/layer21_attention_weight_min": -24.000818252563477,
      "activations/layer22_attention_weight_max": 52.92863845825195,
      "activations/layer22_attention_weight_min": -29.47000503540039,
      "activations/layer23_attention_weight_max": 38.55457305908203,
      "activations/layer23_attention_weight_min": -21.313655853271484,
      "activations/layer2_attention_weight_max": 31.875568389892578,
      "activations/layer2_attention_weight_min": -32.01103973388672,
      "activations/layer3_attention_weight_max": 92.18800354003906,
      "activations/layer3_attention_weight_min": -94.30076599121094,
      "activations/layer4_attention_weight_max": 118.27950286865234,
      "activations/layer4_attention_weight_min": -120.68340301513672,
      "activations/layer5_attention_weight_max": 57.237571716308594,
      "activations/layer5_attention_weight_min": -63.26289367675781,
      "activations/layer6_attention_weight_max": 50.22895812988281,
      "activations/layer6_attention_weight_min": -54.35033416748047,
      "activations/layer7_attention_weight_max": 90.27765655517578,
      "activations/layer7_attention_weight_min": -86.44044494628906,
      "activations/layer8_attention_weight_max": 37.67937469482422,
      "activations/layer8_attention_weight_min": -41.037513732910156,
      "activations/layer9_attention_weight_max": 34.84915542602539,
      "activations/layer9_attention_weight_min": -33.99142074584961,
      "epoch": 7.83,
      "learning_rate": 0.0001005,
      "loss": 2.7981,
      "step": 134800
    },
    {
      "activations/layer0_attention_weight_max": 17.11128044128418,
      "activations/layer0_attention_weight_min": -11.747629165649414,
      "activations/layer10_attention_weight_max": 35.07742691040039,
      "activations/layer10_attention_weight_min": -34.851844787597656,
      "activations/layer11_attention_weight_max": 35.062992095947266,
      "activations/layer11_attention_weight_min": -31.09871482849121,
      "activations/layer12_attention_weight_max": 25.492042541503906,
      "activations/layer12_attention_weight_min": -26.167362213134766,
      "activations/layer13_attention_weight_max": 42.039512634277344,
      "activations/layer13_attention_weight_min": -30.841896057128906,
      "activations/layer14_attention_weight_max": 48.90433883666992,
      "activations/layer14_attention_weight_min": -39.131168365478516,
      "activations/layer15_attention_weight_max": 38.65282440185547,
      "activations/layer15_attention_weight_min": -29.95281982421875,
      "activations/layer16_attention_weight_max": 35.619117736816406,
      "activations/layer16_attention_weight_min": -29.738786697387695,
      "activations/layer17_attention_weight_max": 41.345211029052734,
      "activations/layer17_attention_weight_min": -25.35467529296875,
      "activations/layer18_attention_weight_max": 35.49829864501953,
      "activations/layer18_attention_weight_min": -22.26488494873047,
      "activations/layer19_attention_weight_max": 38.57363510131836,
      "activations/layer19_attention_weight_min": -28.547714233398438,
      "activations/layer1_attention_weight_max": 17.359539031982422,
      "activations/layer1_attention_weight_min": -12.525358200073242,
      "activations/layer20_attention_weight_max": 36.35703659057617,
      "activations/layer20_attention_weight_min": -24.52330207824707,
      "activations/layer21_attention_weight_max": 39.05995178222656,
      "activations/layer21_attention_weight_min": -22.16161346435547,
      "activations/layer22_attention_weight_max": 52.38677215576172,
      "activations/layer22_attention_weight_min": -29.42485809326172,
      "activations/layer23_attention_weight_max": 36.483985900878906,
      "activations/layer23_attention_weight_min": -24.518346786499023,
      "activations/layer2_attention_weight_max": 31.37755584716797,
      "activations/layer2_attention_weight_min": -30.025671005249023,
      "activations/layer3_attention_weight_max": 86.84334564208984,
      "activations/layer3_attention_weight_min": -85.43000793457031,
      "activations/layer4_attention_weight_max": 130.86990356445312,
      "activations/layer4_attention_weight_min": -122.95511627197266,
      "activations/layer5_attention_weight_max": 65.38085174560547,
      "activations/layer5_attention_weight_min": -67.01144409179688,
      "activations/layer6_attention_weight_max": 53.766456604003906,
      "activations/layer6_attention_weight_min": -50.41436004638672,
      "activations/layer7_attention_weight_max": 86.26297760009766,
      "activations/layer7_attention_weight_min": -88.40786743164062,
      "activations/layer8_attention_weight_max": 41.60721969604492,
      "activations/layer8_attention_weight_min": -41.46485137939453,
      "activations/layer9_attention_weight_max": 37.59503936767578,
      "activations/layer9_attention_weight_min": -36.659889221191406,
      "epoch": 7.84,
      "learning_rate": 0.00010048106060606059,
      "loss": 2.8289,
      "step": 134850
    },
    {
      "activations/layer0_attention_weight_max": 15.120194435119629,
      "activations/layer0_attention_weight_min": -12.522972106933594,
      "activations/layer10_attention_weight_max": 32.01005935668945,
      "activations/layer10_attention_weight_min": -32.045562744140625,
      "activations/layer11_attention_weight_max": 34.2387809753418,
      "activations/layer11_attention_weight_min": -30.62348175048828,
      "activations/layer12_attention_weight_max": 28.724292755126953,
      "activations/layer12_attention_weight_min": -31.456588745117188,
      "activations/layer13_attention_weight_max": 37.153656005859375,
      "activations/layer13_attention_weight_min": -33.71003341674805,
      "activations/layer14_attention_weight_max": 54.517704010009766,
      "activations/layer14_attention_weight_min": -45.101200103759766,
      "activations/layer15_attention_weight_max": 38.21234130859375,
      "activations/layer15_attention_weight_min": -30.2060546875,
      "activations/layer16_attention_weight_max": 33.30689239501953,
      "activations/layer16_attention_weight_min": -28.507999420166016,
      "activations/layer17_attention_weight_max": 37.74226379394531,
      "activations/layer17_attention_weight_min": -24.743837356567383,
      "activations/layer18_attention_weight_max": 35.16830825805664,
      "activations/layer18_attention_weight_min": -20.73661994934082,
      "activations/layer19_attention_weight_max": 35.80287551879883,
      "activations/layer19_attention_weight_min": -28.715805053710938,
      "activations/layer1_attention_weight_max": 17.24055290222168,
      "activations/layer1_attention_weight_min": -16.726675033569336,
      "activations/layer20_attention_weight_max": 30.00002670288086,
      "activations/layer20_attention_weight_min": -24.323915481567383,
      "activations/layer21_attention_weight_max": 30.42333221435547,
      "activations/layer21_attention_weight_min": -21.438594818115234,
      "activations/layer22_attention_weight_max": 48.40390396118164,
      "activations/layer22_attention_weight_min": -28.51129722595215,
      "activations/layer23_attention_weight_max": 37.97689437866211,
      "activations/layer23_attention_weight_min": -23.267078399658203,
      "activations/layer2_attention_weight_max": 32.20265579223633,
      "activations/layer2_attention_weight_min": -30.958694458007812,
      "activations/layer3_attention_weight_max": 91.40323638916016,
      "activations/layer3_attention_weight_min": -93.25936889648438,
      "activations/layer4_attention_weight_max": 123.27032470703125,
      "activations/layer4_attention_weight_min": -120.25273895263672,
      "activations/layer5_attention_weight_max": 63.01598358154297,
      "activations/layer5_attention_weight_min": -61.598968505859375,
      "activations/layer6_attention_weight_max": 49.23048400878906,
      "activations/layer6_attention_weight_min": -53.245155334472656,
      "activations/layer7_attention_weight_max": 89.08901977539062,
      "activations/layer7_attention_weight_min": -92.19397735595703,
      "activations/layer8_attention_weight_max": 40.951900482177734,
      "activations/layer8_attention_weight_min": -42.49101638793945,
      "activations/layer9_attention_weight_max": 33.83893966674805,
      "activations/layer9_attention_weight_min": -35.01909255981445,
      "epoch": 7.84,
      "learning_rate": 0.0001004621212121212,
      "loss": 2.8221,
      "step": 134900
    },
    {
      "activations/layer0_attention_weight_max": 15.094531059265137,
      "activations/layer0_attention_weight_min": -13.303196907043457,
      "activations/layer10_attention_weight_max": 35.38029098510742,
      "activations/layer10_attention_weight_min": -31.833200454711914,
      "activations/layer11_attention_weight_max": 35.24497985839844,
      "activations/layer11_attention_weight_min": -30.423030853271484,
      "activations/layer12_attention_weight_max": 22.13971519470215,
      "activations/layer12_attention_weight_min": -26.95759391784668,
      "activations/layer13_attention_weight_max": 40.30983352661133,
      "activations/layer13_attention_weight_min": -32.52296829223633,
      "activations/layer14_attention_weight_max": 43.096317291259766,
      "activations/layer14_attention_weight_min": -36.83340072631836,
      "activations/layer15_attention_weight_max": 38.59033203125,
      "activations/layer15_attention_weight_min": -30.72059440612793,
      "activations/layer16_attention_weight_max": 31.66853904724121,
      "activations/layer16_attention_weight_min": -27.364660263061523,
      "activations/layer17_attention_weight_max": 33.5015869140625,
      "activations/layer17_attention_weight_min": -25.119070053100586,
      "activations/layer18_attention_weight_max": 33.92300033569336,
      "activations/layer18_attention_weight_min": -24.120357513427734,
      "activations/layer19_attention_weight_max": 33.44377136230469,
      "activations/layer19_attention_weight_min": -28.885465621948242,
      "activations/layer1_attention_weight_max": 17.140518188476562,
      "activations/layer1_attention_weight_min": -13.89810562133789,
      "activations/layer20_attention_weight_max": 28.71722984313965,
      "activations/layer20_attention_weight_min": -23.454618453979492,
      "activations/layer21_attention_weight_max": 28.042051315307617,
      "activations/layer21_attention_weight_min": -21.480636596679688,
      "activations/layer22_attention_weight_max": 45.78938674926758,
      "activations/layer22_attention_weight_min": -29.24571990966797,
      "activations/layer23_attention_weight_max": 32.49774932861328,
      "activations/layer23_attention_weight_min": -22.940589904785156,
      "activations/layer2_attention_weight_max": 33.803314208984375,
      "activations/layer2_attention_weight_min": -32.024269104003906,
      "activations/layer3_attention_weight_max": 91.74580383300781,
      "activations/layer3_attention_weight_min": -93.4937515258789,
      "activations/layer4_attention_weight_max": 122.54669189453125,
      "activations/layer4_attention_weight_min": -117.8718032836914,
      "activations/layer5_attention_weight_max": 59.70747375488281,
      "activations/layer5_attention_weight_min": -61.41428756713867,
      "activations/layer6_attention_weight_max": 53.09843444824219,
      "activations/layer6_attention_weight_min": -49.789329528808594,
      "activations/layer7_attention_weight_max": 93.25839233398438,
      "activations/layer7_attention_weight_min": -93.44969940185547,
      "activations/layer8_attention_weight_max": 41.37556076049805,
      "activations/layer8_attention_weight_min": -41.08821487426758,
      "activations/layer9_attention_weight_max": 35.001644134521484,
      "activations/layer9_attention_weight_min": -34.970157623291016,
      "epoch": 7.84,
      "learning_rate": 0.00010044318181818182,
      "loss": 2.8145,
      "step": 134950
    },
    {
      "activations/layer0_attention_weight_max": 15.300381660461426,
      "activations/layer0_attention_weight_min": -12.998167037963867,
      "activations/layer10_attention_weight_max": 41.98788833618164,
      "activations/layer10_attention_weight_min": -36.87715530395508,
      "activations/layer11_attention_weight_max": 43.242591857910156,
      "activations/layer11_attention_weight_min": -37.180809020996094,
      "activations/layer12_attention_weight_max": 53.04473114013672,
      "activations/layer12_attention_weight_min": -26.486330032348633,
      "activations/layer13_attention_weight_max": 76.33133697509766,
      "activations/layer13_attention_weight_min": -37.600128173828125,
      "activations/layer14_attention_weight_max": 64.20197296142578,
      "activations/layer14_attention_weight_min": -39.81734085083008,
      "activations/layer15_attention_weight_max": 64.20462799072266,
      "activations/layer15_attention_weight_min": -33.61591720581055,
      "activations/layer16_attention_weight_max": 34.53165054321289,
      "activations/layer16_attention_weight_min": -26.87866973876953,
      "activations/layer17_attention_weight_max": 38.866065979003906,
      "activations/layer17_attention_weight_min": -24.736915588378906,
      "activations/layer18_attention_weight_max": 31.644445419311523,
      "activations/layer18_attention_weight_min": -23.031341552734375,
      "activations/layer19_attention_weight_max": 45.00870895385742,
      "activations/layer19_attention_weight_min": -29.828092575073242,
      "activations/layer1_attention_weight_max": 16.181394577026367,
      "activations/layer1_attention_weight_min": -16.90492057800293,
      "activations/layer20_attention_weight_max": 38.827972412109375,
      "activations/layer20_attention_weight_min": -25.087322235107422,
      "activations/layer21_attention_weight_max": 37.82430648803711,
      "activations/layer21_attention_weight_min": -23.00876808166504,
      "activations/layer22_attention_weight_max": 56.20634841918945,
      "activations/layer22_attention_weight_min": -29.6593074798584,
      "activations/layer23_attention_weight_max": 44.62472152709961,
      "activations/layer23_attention_weight_min": -20.959705352783203,
      "activations/layer2_attention_weight_max": 33.02369689941406,
      "activations/layer2_attention_weight_min": -33.83436965942383,
      "activations/layer3_attention_weight_max": 91.4752197265625,
      "activations/layer3_attention_weight_min": -96.50707244873047,
      "activations/layer4_attention_weight_max": 118.82667541503906,
      "activations/layer4_attention_weight_min": -117.59773254394531,
      "activations/layer5_attention_weight_max": 63.658973693847656,
      "activations/layer5_attention_weight_min": -60.50746154785156,
      "activations/layer6_attention_weight_max": 54.733638763427734,
      "activations/layer6_attention_weight_min": -48.558536529541016,
      "activations/layer7_attention_weight_max": 97.85848236083984,
      "activations/layer7_attention_weight_min": -91.80068969726562,
      "activations/layer8_attention_weight_max": 48.720436096191406,
      "activations/layer8_attention_weight_min": -43.88591384887695,
      "activations/layer9_attention_weight_max": 42.36859130859375,
      "activations/layer9_attention_weight_min": -38.54320526123047,
      "epoch": 7.84,
      "learning_rate": 0.00010042424242424241,
      "loss": 2.7946,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.531,
      "eval_samples_per_second": 503.343,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.531,
      "eval_openwebtext_samples_per_second": 503.343,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 2.0231,
      "eval_wikitext_samples_per_second": 225.401,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_lambada_loss": 2.52734375,
      "eval_lambada_ppl": 12.520205133467638,
      "eval_lambada_runtime": 9.5745,
      "eval_lambada_samples_per_second": 508.539,
      "step": 135000
    },
    {
      "activations/layer0_attention_weight_max": 15.135438919067383,
      "activations/layer0_attention_weight_min": -12.117125511169434,
      "activations/layer10_attention_weight_max": 33.03679275512695,
      "activations/layer10_attention_weight_min": -33.84831619262695,
      "activations/layer11_attention_weight_max": 32.21892547607422,
      "activations/layer11_attention_weight_min": -33.45402908325195,
      "activations/layer12_attention_weight_max": 24.38949203491211,
      "activations/layer12_attention_weight_min": -26.672330856323242,
      "activations/layer13_attention_weight_max": 42.24589538574219,
      "activations/layer13_attention_weight_min": -34.16862487792969,
      "activations/layer14_attention_weight_max": 51.52606201171875,
      "activations/layer14_attention_weight_min": -43.5417594909668,
      "activations/layer15_attention_weight_max": 36.69236755371094,
      "activations/layer15_attention_weight_min": -32.00262451171875,
      "activations/layer16_attention_weight_max": 31.972867965698242,
      "activations/layer16_attention_weight_min": -27.79861068725586,
      "activations/layer17_attention_weight_max": 36.616737365722656,
      "activations/layer17_attention_weight_min": -25.04306411743164,
      "activations/layer18_attention_weight_max": 32.69724655151367,
      "activations/layer18_attention_weight_min": -20.80885124206543,
      "activations/layer19_attention_weight_max": 36.37641525268555,
      "activations/layer19_attention_weight_min": -29.7908935546875,
      "activations/layer1_attention_weight_max": 16.876434326171875,
      "activations/layer1_attention_weight_min": -15.463876724243164,
      "activations/layer20_attention_weight_max": 32.811466217041016,
      "activations/layer20_attention_weight_min": -23.329906463623047,
      "activations/layer21_attention_weight_max": 30.61379051208496,
      "activations/layer21_attention_weight_min": -21.739646911621094,
      "activations/layer22_attention_weight_max": 48.89409255981445,
      "activations/layer22_attention_weight_min": -27.772260665893555,
      "activations/layer23_attention_weight_max": 35.900535583496094,
      "activations/layer23_attention_weight_min": -23.354909896850586,
      "activations/layer2_attention_weight_max": 31.24103546142578,
      "activations/layer2_attention_weight_min": -30.887441635131836,
      "activations/layer3_attention_weight_max": 90.78614807128906,
      "activations/layer3_attention_weight_min": -93.84540557861328,
      "activations/layer4_attention_weight_max": 120.96475982666016,
      "activations/layer4_attention_weight_min": -115.77281951904297,
      "activations/layer5_attention_weight_max": 60.02168273925781,
      "activations/layer5_attention_weight_min": -61.72578048706055,
      "activations/layer6_attention_weight_max": 50.07876205444336,
      "activations/layer6_attention_weight_min": -53.59764099121094,
      "activations/layer7_attention_weight_max": 88.21711730957031,
      "activations/layer7_attention_weight_min": -90.84635925292969,
      "activations/layer8_attention_weight_max": 38.72072982788086,
      "activations/layer8_attention_weight_min": -39.55289840698242,
      "activations/layer9_attention_weight_max": 37.39107894897461,
      "activations/layer9_attention_weight_min": -33.313087463378906,
      "epoch": 7.85,
      "learning_rate": 0.00010040530303030302,
      "loss": 2.8176,
      "step": 135050
    },
    {
      "activations/layer0_attention_weight_max": 14.839258193969727,
      "activations/layer0_attention_weight_min": -12.724867820739746,
      "activations/layer10_attention_weight_max": 34.602508544921875,
      "activations/layer10_attention_weight_min": -32.24910354614258,
      "activations/layer11_attention_weight_max": 34.32573318481445,
      "activations/layer11_attention_weight_min": -30.7627010345459,
      "activations/layer12_attention_weight_max": 29.015348434448242,
      "activations/layer12_attention_weight_min": -28.08313751220703,
      "activations/layer13_attention_weight_max": 43.4215202331543,
      "activations/layer13_attention_weight_min": -34.18115997314453,
      "activations/layer14_attention_weight_max": 62.346466064453125,
      "activations/layer14_attention_weight_min": -43.85850524902344,
      "activations/layer15_attention_weight_max": 40.88410949707031,
      "activations/layer15_attention_weight_min": -30.305212020874023,
      "activations/layer16_attention_weight_max": 38.94617462158203,
      "activations/layer16_attention_weight_min": -27.781803131103516,
      "activations/layer17_attention_weight_max": 37.160465240478516,
      "activations/layer17_attention_weight_min": -26.824386596679688,
      "activations/layer18_attention_weight_max": 37.9864501953125,
      "activations/layer18_attention_weight_min": -23.217544555664062,
      "activations/layer19_attention_weight_max": 42.42391586303711,
      "activations/layer19_attention_weight_min": -35.12539291381836,
      "activations/layer1_attention_weight_max": 16.73335075378418,
      "activations/layer1_attention_weight_min": -17.164630889892578,
      "activations/layer20_attention_weight_max": 34.05250549316406,
      "activations/layer20_attention_weight_min": -24.515689849853516,
      "activations/layer21_attention_weight_max": 34.44966506958008,
      "activations/layer21_attention_weight_min": -24.6737117767334,
      "activations/layer22_attention_weight_max": 53.98989486694336,
      "activations/layer22_attention_weight_min": -30.799428939819336,
      "activations/layer23_attention_weight_max": 40.73853302001953,
      "activations/layer23_attention_weight_min": -23.225849151611328,
      "activations/layer2_attention_weight_max": 33.266326904296875,
      "activations/layer2_attention_weight_min": -32.510135650634766,
      "activations/layer3_attention_weight_max": 91.77465057373047,
      "activations/layer3_attention_weight_min": -93.58146667480469,
      "activations/layer4_attention_weight_max": 117.48375701904297,
      "activations/layer4_attention_weight_min": -118.34590911865234,
      "activations/layer5_attention_weight_max": 60.1019401550293,
      "activations/layer5_attention_weight_min": -63.79587936401367,
      "activations/layer6_attention_weight_max": 49.11771774291992,
      "activations/layer6_attention_weight_min": -53.38386154174805,
      "activations/layer7_attention_weight_max": 83.6988525390625,
      "activations/layer7_attention_weight_min": -89.40228271484375,
      "activations/layer8_attention_weight_max": 39.08049392700195,
      "activations/layer8_attention_weight_min": -41.006988525390625,
      "activations/layer9_attention_weight_max": 34.82317352294922,
      "activations/layer9_attention_weight_min": -33.558345794677734,
      "epoch": 7.85,
      "learning_rate": 0.00010038636363636363,
      "loss": 2.8188,
      "step": 135100
    },
    {
      "activations/layer0_attention_weight_max": 15.728818893432617,
      "activations/layer0_attention_weight_min": -12.694791793823242,
      "activations/layer10_attention_weight_max": 34.00009536743164,
      "activations/layer10_attention_weight_min": -32.688697814941406,
      "activations/layer11_attention_weight_max": 32.7738037109375,
      "activations/layer11_attention_weight_min": -30.694190979003906,
      "activations/layer12_attention_weight_max": 23.212989807128906,
      "activations/layer12_attention_weight_min": -29.187152862548828,
      "activations/layer13_attention_weight_max": 41.76660919189453,
      "activations/layer13_attention_weight_min": -33.57078170776367,
      "activations/layer14_attention_weight_max": 49.18564987182617,
      "activations/layer14_attention_weight_min": -40.79188537597656,
      "activations/layer15_attention_weight_max": 35.695655822753906,
      "activations/layer15_attention_weight_min": -29.490558624267578,
      "activations/layer16_attention_weight_max": 31.346004486083984,
      "activations/layer16_attention_weight_min": -27.132038116455078,
      "activations/layer17_attention_weight_max": 35.43947219848633,
      "activations/layer17_attention_weight_min": -25.28325843811035,
      "activations/layer18_attention_weight_max": 31.758872985839844,
      "activations/layer18_attention_weight_min": -21.15693473815918,
      "activations/layer19_attention_weight_max": 35.463138580322266,
      "activations/layer19_attention_weight_min": -30.308523178100586,
      "activations/layer1_attention_weight_max": 16.171789169311523,
      "activations/layer1_attention_weight_min": -14.731865882873535,
      "activations/layer20_attention_weight_max": 31.654905319213867,
      "activations/layer20_attention_weight_min": -26.148820877075195,
      "activations/layer21_attention_weight_max": 30.51671600341797,
      "activations/layer21_attention_weight_min": -24.23419189453125,
      "activations/layer22_attention_weight_max": 50.525386810302734,
      "activations/layer22_attention_weight_min": -27.59915542602539,
      "activations/layer23_attention_weight_max": 35.947208404541016,
      "activations/layer23_attention_weight_min": -23.317169189453125,
      "activations/layer2_attention_weight_max": 33.056541442871094,
      "activations/layer2_attention_weight_min": -32.16291809082031,
      "activations/layer3_attention_weight_max": 97.6199722290039,
      "activations/layer3_attention_weight_min": -99.48246002197266,
      "activations/layer4_attention_weight_max": 120.7169189453125,
      "activations/layer4_attention_weight_min": -119.3187026977539,
      "activations/layer5_attention_weight_max": 58.2258415222168,
      "activations/layer5_attention_weight_min": -60.74030685424805,
      "activations/layer6_attention_weight_max": 48.338802337646484,
      "activations/layer6_attention_weight_min": -50.287010192871094,
      "activations/layer7_attention_weight_max": 87.77713012695312,
      "activations/layer7_attention_weight_min": -85.51978302001953,
      "activations/layer8_attention_weight_max": 39.76973342895508,
      "activations/layer8_attention_weight_min": -41.63887023925781,
      "activations/layer9_attention_weight_max": 34.88929748535156,
      "activations/layer9_attention_weight_min": -35.033973693847656,
      "epoch": 7.85,
      "learning_rate": 0.00010036742424242423,
      "loss": 2.8166,
      "step": 135150
    },
    {
      "activations/layer0_attention_weight_max": 15.285250663757324,
      "activations/layer0_attention_weight_min": -12.17383098602295,
      "activations/layer10_attention_weight_max": 42.78512954711914,
      "activations/layer10_attention_weight_min": -37.425540924072266,
      "activations/layer11_attention_weight_max": 41.20928955078125,
      "activations/layer11_attention_weight_min": -36.253662109375,
      "activations/layer12_attention_weight_max": 33.18497848510742,
      "activations/layer12_attention_weight_min": -28.1906681060791,
      "activations/layer13_attention_weight_max": 55.91032409667969,
      "activations/layer13_attention_weight_min": -33.2495002746582,
      "activations/layer14_attention_weight_max": 73.71723175048828,
      "activations/layer14_attention_weight_min": -47.30507278442383,
      "activations/layer15_attention_weight_max": 49.12716293334961,
      "activations/layer15_attention_weight_min": -32.158626556396484,
      "activations/layer16_attention_weight_max": 43.331298828125,
      "activations/layer16_attention_weight_min": -30.452449798583984,
      "activations/layer17_attention_weight_max": 46.408348083496094,
      "activations/layer17_attention_weight_min": -27.813112258911133,
      "activations/layer18_attention_weight_max": 37.79525375366211,
      "activations/layer18_attention_weight_min": -23.761539459228516,
      "activations/layer19_attention_weight_max": 51.132083892822266,
      "activations/layer19_attention_weight_min": -28.875333786010742,
      "activations/layer1_attention_weight_max": 18.820659637451172,
      "activations/layer1_attention_weight_min": -14.242843627929688,
      "activations/layer20_attention_weight_max": 40.941410064697266,
      "activations/layer20_attention_weight_min": -23.670015335083008,
      "activations/layer21_attention_weight_max": 35.964805603027344,
      "activations/layer21_attention_weight_min": -22.05029296875,
      "activations/layer22_attention_weight_max": 63.24325942993164,
      "activations/layer22_attention_weight_min": -29.210256576538086,
      "activations/layer23_attention_weight_max": 49.096336364746094,
      "activations/layer23_attention_weight_min": -23.495885848999023,
      "activations/layer2_attention_weight_max": 32.454681396484375,
      "activations/layer2_attention_weight_min": -31.686901092529297,
      "activations/layer3_attention_weight_max": 91.8131332397461,
      "activations/layer3_attention_weight_min": -89.595947265625,
      "activations/layer4_attention_weight_max": 122.6464614868164,
      "activations/layer4_attention_weight_min": -117.08324432373047,
      "activations/layer5_attention_weight_max": 61.3400993347168,
      "activations/layer5_attention_weight_min": -62.57505416870117,
      "activations/layer6_attention_weight_max": 52.65723419189453,
      "activations/layer6_attention_weight_min": -56.398590087890625,
      "activations/layer7_attention_weight_max": 97.69514465332031,
      "activations/layer7_attention_weight_min": -95.26325988769531,
      "activations/layer8_attention_weight_max": 46.88074493408203,
      "activations/layer8_attention_weight_min": -48.459800720214844,
      "activations/layer9_attention_weight_max": 42.69044876098633,
      "activations/layer9_attention_weight_min": -37.10172653198242,
      "epoch": 7.86,
      "learning_rate": 0.00010034848484848484,
      "loss": 2.8072,
      "step": 135200
    },
    {
      "activations/layer0_attention_weight_max": 15.3969087600708,
      "activations/layer0_attention_weight_min": -11.972536087036133,
      "activations/layer10_attention_weight_max": 32.744056701660156,
      "activations/layer10_attention_weight_min": -31.969823837280273,
      "activations/layer11_attention_weight_max": 31.379714965820312,
      "activations/layer11_attention_weight_min": -29.879486083984375,
      "activations/layer12_attention_weight_max": 21.79051971435547,
      "activations/layer12_attention_weight_min": -28.056917190551758,
      "activations/layer13_attention_weight_max": 41.1379508972168,
      "activations/layer13_attention_weight_min": -33.507896423339844,
      "activations/layer14_attention_weight_max": 48.7349739074707,
      "activations/layer14_attention_weight_min": -42.68187713623047,
      "activations/layer15_attention_weight_max": 40.251060485839844,
      "activations/layer15_attention_weight_min": -31.86308479309082,
      "activations/layer16_attention_weight_max": 38.289432525634766,
      "activations/layer16_attention_weight_min": -27.53264045715332,
      "activations/layer17_attention_weight_max": 37.16160583496094,
      "activations/layer17_attention_weight_min": -25.935115814208984,
      "activations/layer18_attention_weight_max": 33.70463562011719,
      "activations/layer18_attention_weight_min": -23.350650787353516,
      "activations/layer19_attention_weight_max": 41.07894515991211,
      "activations/layer19_attention_weight_min": -32.876747131347656,
      "activations/layer1_attention_weight_max": 16.86441421508789,
      "activations/layer1_attention_weight_min": -14.514703750610352,
      "activations/layer20_attention_weight_max": 37.88798522949219,
      "activations/layer20_attention_weight_min": -24.6950740814209,
      "activations/layer21_attention_weight_max": 32.483482360839844,
      "activations/layer21_attention_weight_min": -22.9886417388916,
      "activations/layer22_attention_weight_max": 54.14451217651367,
      "activations/layer22_attention_weight_min": -29.24031639099121,
      "activations/layer23_attention_weight_max": 40.82550048828125,
      "activations/layer23_attention_weight_min": -21.986921310424805,
      "activations/layer2_attention_weight_max": 31.26631736755371,
      "activations/layer2_attention_weight_min": -31.836410522460938,
      "activations/layer3_attention_weight_max": 90.14698028564453,
      "activations/layer3_attention_weight_min": -95.06697845458984,
      "activations/layer4_attention_weight_max": 120.60957336425781,
      "activations/layer4_attention_weight_min": -122.11461639404297,
      "activations/layer5_attention_weight_max": 58.68601608276367,
      "activations/layer5_attention_weight_min": -62.09228515625,
      "activations/layer6_attention_weight_max": 47.154136657714844,
      "activations/layer6_attention_weight_min": -54.44155502319336,
      "activations/layer7_attention_weight_max": 88.71094512939453,
      "activations/layer7_attention_weight_min": -91.7928237915039,
      "activations/layer8_attention_weight_max": 42.75422668457031,
      "activations/layer8_attention_weight_min": -43.610416412353516,
      "activations/layer9_attention_weight_max": 35.634761810302734,
      "activations/layer9_attention_weight_min": -34.7446403503418,
      "epoch": 7.86,
      "learning_rate": 0.00010032954545454544,
      "loss": 2.8263,
      "step": 135250
    },
    {
      "activations/layer0_attention_weight_max": 15.793208122253418,
      "activations/layer0_attention_weight_min": -13.509190559387207,
      "activations/layer10_attention_weight_max": 33.70768356323242,
      "activations/layer10_attention_weight_min": -31.029325485229492,
      "activations/layer11_attention_weight_max": 31.528350830078125,
      "activations/layer11_attention_weight_min": -30.266048431396484,
      "activations/layer12_attention_weight_max": 27.021886825561523,
      "activations/layer12_attention_weight_min": -27.366043090820312,
      "activations/layer13_attention_weight_max": 39.30156707763672,
      "activations/layer13_attention_weight_min": -33.22532653808594,
      "activations/layer14_attention_weight_max": 52.4123649597168,
      "activations/layer14_attention_weight_min": -45.013916015625,
      "activations/layer15_attention_weight_max": 36.14181137084961,
      "activations/layer15_attention_weight_min": -29.630300521850586,
      "activations/layer16_attention_weight_max": 36.93257522583008,
      "activations/layer16_attention_weight_min": -28.84967613220215,
      "activations/layer17_attention_weight_max": 36.89120864868164,
      "activations/layer17_attention_weight_min": -24.719585418701172,
      "activations/layer18_attention_weight_max": 32.09759521484375,
      "activations/layer18_attention_weight_min": -20.383298873901367,
      "activations/layer19_attention_weight_max": 37.71614456176758,
      "activations/layer19_attention_weight_min": -30.622262954711914,
      "activations/layer1_attention_weight_max": 17.23324966430664,
      "activations/layer1_attention_weight_min": -13.37603759765625,
      "activations/layer20_attention_weight_max": 34.72554016113281,
      "activations/layer20_attention_weight_min": -23.896631240844727,
      "activations/layer21_attention_weight_max": 32.90241241455078,
      "activations/layer21_attention_weight_min": -21.870716094970703,
      "activations/layer22_attention_weight_max": 50.12234878540039,
      "activations/layer22_attention_weight_min": -29.0362606048584,
      "activations/layer23_attention_weight_max": 38.83050537109375,
      "activations/layer23_attention_weight_min": -23.399381637573242,
      "activations/layer2_attention_weight_max": 32.5014533996582,
      "activations/layer2_attention_weight_min": -32.39136505126953,
      "activations/layer3_attention_weight_max": 97.5560073852539,
      "activations/layer3_attention_weight_min": -97.73367309570312,
      "activations/layer4_attention_weight_max": 124.4325180053711,
      "activations/layer4_attention_weight_min": -117.67168426513672,
      "activations/layer5_attention_weight_max": 59.64107894897461,
      "activations/layer5_attention_weight_min": -65.81684875488281,
      "activations/layer6_attention_weight_max": 52.18875503540039,
      "activations/layer6_attention_weight_min": -52.969139099121094,
      "activations/layer7_attention_weight_max": 87.13518524169922,
      "activations/layer7_attention_weight_min": -90.87329864501953,
      "activations/layer8_attention_weight_max": 40.73721694946289,
      "activations/layer8_attention_weight_min": -39.01295471191406,
      "activations/layer9_attention_weight_max": 33.022823333740234,
      "activations/layer9_attention_weight_min": -32.51957321166992,
      "epoch": 7.86,
      "learning_rate": 0.00010031060606060606,
      "loss": 2.8107,
      "step": 135300
    },
    {
      "activations/layer0_attention_weight_max": 14.867480278015137,
      "activations/layer0_attention_weight_min": -12.863714218139648,
      "activations/layer10_attention_weight_max": 32.59965896606445,
      "activations/layer10_attention_weight_min": -31.512454986572266,
      "activations/layer11_attention_weight_max": 34.883392333984375,
      "activations/layer11_attention_weight_min": -28.73400115966797,
      "activations/layer12_attention_weight_max": 26.183969497680664,
      "activations/layer12_attention_weight_min": -24.06658172607422,
      "activations/layer13_attention_weight_max": 35.83651351928711,
      "activations/layer13_attention_weight_min": -31.994064331054688,
      "activations/layer14_attention_weight_max": 52.34449768066406,
      "activations/layer14_attention_weight_min": -39.60377502441406,
      "activations/layer15_attention_weight_max": 36.278106689453125,
      "activations/layer15_attention_weight_min": -28.7176513671875,
      "activations/layer16_attention_weight_max": 36.7170295715332,
      "activations/layer16_attention_weight_min": -28.43821907043457,
      "activations/layer17_attention_weight_max": 33.763267517089844,
      "activations/layer17_attention_weight_min": -26.70292854309082,
      "activations/layer18_attention_weight_max": 34.375885009765625,
      "activations/layer18_attention_weight_min": -21.951261520385742,
      "activations/layer19_attention_weight_max": 36.31925582885742,
      "activations/layer19_attention_weight_min": -33.285274505615234,
      "activations/layer1_attention_weight_max": 16.75872230529785,
      "activations/layer1_attention_weight_min": -15.266036033630371,
      "activations/layer20_attention_weight_max": 29.177324295043945,
      "activations/layer20_attention_weight_min": -24.36585235595703,
      "activations/layer21_attention_weight_max": 33.63554382324219,
      "activations/layer21_attention_weight_min": -23.1475887298584,
      "activations/layer22_attention_weight_max": 49.27574157714844,
      "activations/layer22_attention_weight_min": -31.25102424621582,
      "activations/layer23_attention_weight_max": 36.39764404296875,
      "activations/layer23_attention_weight_min": -25.804180145263672,
      "activations/layer2_attention_weight_max": 34.92556381225586,
      "activations/layer2_attention_weight_min": -33.89947509765625,
      "activations/layer3_attention_weight_max": 96.96880340576172,
      "activations/layer3_attention_weight_min": -100.0970687866211,
      "activations/layer4_attention_weight_max": 124.40534973144531,
      "activations/layer4_attention_weight_min": -120.51775360107422,
      "activations/layer5_attention_weight_max": 59.90334701538086,
      "activations/layer5_attention_weight_min": -61.228187561035156,
      "activations/layer6_attention_weight_max": 49.29692840576172,
      "activations/layer6_attention_weight_min": -49.41495132446289,
      "activations/layer7_attention_weight_max": 86.48380279541016,
      "activations/layer7_attention_weight_min": -89.12737274169922,
      "activations/layer8_attention_weight_max": 39.044490814208984,
      "activations/layer8_attention_weight_min": -40.907981872558594,
      "activations/layer9_attention_weight_max": 34.546478271484375,
      "activations/layer9_attention_weight_min": -35.109737396240234,
      "epoch": 7.86,
      "learning_rate": 0.00010029166666666666,
      "loss": 2.806,
      "step": 135350
    },
    {
      "activations/layer0_attention_weight_max": 16.019731521606445,
      "activations/layer0_attention_weight_min": -12.433767318725586,
      "activations/layer10_attention_weight_max": 33.4165153503418,
      "activations/layer10_attention_weight_min": -32.96458435058594,
      "activations/layer11_attention_weight_max": 33.34682846069336,
      "activations/layer11_attention_weight_min": -31.540851593017578,
      "activations/layer12_attention_weight_max": 22.28648567199707,
      "activations/layer12_attention_weight_min": -29.028141021728516,
      "activations/layer13_attention_weight_max": 39.176231384277344,
      "activations/layer13_attention_weight_min": -33.50755310058594,
      "activations/layer14_attention_weight_max": 49.56828689575195,
      "activations/layer14_attention_weight_min": -37.68441390991211,
      "activations/layer15_attention_weight_max": 39.034603118896484,
      "activations/layer15_attention_weight_min": -31.82929801940918,
      "activations/layer16_attention_weight_max": 34.08358383178711,
      "activations/layer16_attention_weight_min": -27.493227005004883,
      "activations/layer17_attention_weight_max": 38.01542282104492,
      "activations/layer17_attention_weight_min": -24.896076202392578,
      "activations/layer18_attention_weight_max": 34.851688385009766,
      "activations/layer18_attention_weight_min": -23.149871826171875,
      "activations/layer19_attention_weight_max": 42.74713134765625,
      "activations/layer19_attention_weight_min": -31.113574981689453,
      "activations/layer1_attention_weight_max": 18.550386428833008,
      "activations/layer1_attention_weight_min": -14.638189315795898,
      "activations/layer20_attention_weight_max": 31.12896728515625,
      "activations/layer20_attention_weight_min": -27.091880798339844,
      "activations/layer21_attention_weight_max": 30.075572967529297,
      "activations/layer21_attention_weight_min": -25.9966983795166,
      "activations/layer22_attention_weight_max": 49.52766799926758,
      "activations/layer22_attention_weight_min": -32.25107955932617,
      "activations/layer23_attention_weight_max": 35.831321716308594,
      "activations/layer23_attention_weight_min": -24.350263595581055,
      "activations/layer2_attention_weight_max": 33.753868103027344,
      "activations/layer2_attention_weight_min": -34.446868896484375,
      "activations/layer3_attention_weight_max": 93.192138671875,
      "activations/layer3_attention_weight_min": -95.1415786743164,
      "activations/layer4_attention_weight_max": 120.0975341796875,
      "activations/layer4_attention_weight_min": -122.84282684326172,
      "activations/layer5_attention_weight_max": 58.85993957519531,
      "activations/layer5_attention_weight_min": -61.696617126464844,
      "activations/layer6_attention_weight_max": 50.946231842041016,
      "activations/layer6_attention_weight_min": -52.48622512817383,
      "activations/layer7_attention_weight_max": 89.06822204589844,
      "activations/layer7_attention_weight_min": -88.04855346679688,
      "activations/layer8_attention_weight_max": 40.266475677490234,
      "activations/layer8_attention_weight_min": -39.74079513549805,
      "activations/layer9_attention_weight_max": 37.025245666503906,
      "activations/layer9_attention_weight_min": -34.48818588256836,
      "epoch": 7.87,
      "learning_rate": 0.00010027272727272726,
      "loss": 2.813,
      "step": 135400
    },
    {
      "activations/layer0_attention_weight_max": 15.249837875366211,
      "activations/layer0_attention_weight_min": -11.811984062194824,
      "activations/layer10_attention_weight_max": 33.93217849731445,
      "activations/layer10_attention_weight_min": -31.228899002075195,
      "activations/layer11_attention_weight_max": 31.460567474365234,
      "activations/layer11_attention_weight_min": -29.756099700927734,
      "activations/layer12_attention_weight_max": 34.35240936279297,
      "activations/layer12_attention_weight_min": -27.27163314819336,
      "activations/layer13_attention_weight_max": 37.9310188293457,
      "activations/layer13_attention_weight_min": -34.041107177734375,
      "activations/layer14_attention_weight_max": 52.069915771484375,
      "activations/layer14_attention_weight_min": -44.33233642578125,
      "activations/layer15_attention_weight_max": 36.320411682128906,
      "activations/layer15_attention_weight_min": -32.45585632324219,
      "activations/layer16_attention_weight_max": 37.41344451904297,
      "activations/layer16_attention_weight_min": -28.473979949951172,
      "activations/layer17_attention_weight_max": 37.86312484741211,
      "activations/layer17_attention_weight_min": -28.398082733154297,
      "activations/layer18_attention_weight_max": 32.84418869018555,
      "activations/layer18_attention_weight_min": -24.05369758605957,
      "activations/layer19_attention_weight_max": 39.04105758666992,
      "activations/layer19_attention_weight_min": -32.22125244140625,
      "activations/layer1_attention_weight_max": 17.802127838134766,
      "activations/layer1_attention_weight_min": -14.963499069213867,
      "activations/layer20_attention_weight_max": 33.03071975708008,
      "activations/layer20_attention_weight_min": -26.617870330810547,
      "activations/layer21_attention_weight_max": 32.44123077392578,
      "activations/layer21_attention_weight_min": -23.405349731445312,
      "activations/layer22_attention_weight_max": 45.545127868652344,
      "activations/layer22_attention_weight_min": -30.130130767822266,
      "activations/layer23_attention_weight_max": 35.384613037109375,
      "activations/layer23_attention_weight_min": -24.736351013183594,
      "activations/layer2_attention_weight_max": 32.07537078857422,
      "activations/layer2_attention_weight_min": -32.230384826660156,
      "activations/layer3_attention_weight_max": 90.85604095458984,
      "activations/layer3_attention_weight_min": -90.33501434326172,
      "activations/layer4_attention_weight_max": 121.60005950927734,
      "activations/layer4_attention_weight_min": -116.26019287109375,
      "activations/layer5_attention_weight_max": 57.3028564453125,
      "activations/layer5_attention_weight_min": -64.86845397949219,
      "activations/layer6_attention_weight_max": 48.244075775146484,
      "activations/layer6_attention_weight_min": -53.999427795410156,
      "activations/layer7_attention_weight_max": 89.19676971435547,
      "activations/layer7_attention_weight_min": -88.82324981689453,
      "activations/layer8_attention_weight_max": 39.28287887573242,
      "activations/layer8_attention_weight_min": -39.473934173583984,
      "activations/layer9_attention_weight_max": 33.62698745727539,
      "activations/layer9_attention_weight_min": -32.918983459472656,
      "epoch": 7.87,
      "learning_rate": 0.00010025378787878788,
      "loss": 2.8097,
      "step": 135450
    },
    {
      "activations/layer0_attention_weight_max": 15.781966209411621,
      "activations/layer0_attention_weight_min": -12.624343872070312,
      "activations/layer10_attention_weight_max": 32.268157958984375,
      "activations/layer10_attention_weight_min": -33.069366455078125,
      "activations/layer11_attention_weight_max": 32.995361328125,
      "activations/layer11_attention_weight_min": -31.09478759765625,
      "activations/layer12_attention_weight_max": 25.987119674682617,
      "activations/layer12_attention_weight_min": -27.201526641845703,
      "activations/layer13_attention_weight_max": 45.161991119384766,
      "activations/layer13_attention_weight_min": -35.0506591796875,
      "activations/layer14_attention_weight_max": 52.87450408935547,
      "activations/layer14_attention_weight_min": -44.33377456665039,
      "activations/layer15_attention_weight_max": 38.36827087402344,
      "activations/layer15_attention_weight_min": -29.51984405517578,
      "activations/layer16_attention_weight_max": 35.426025390625,
      "activations/layer16_attention_weight_min": -26.561080932617188,
      "activations/layer17_attention_weight_max": 33.781673431396484,
      "activations/layer17_attention_weight_min": -24.70635414123535,
      "activations/layer18_attention_weight_max": 27.989906311035156,
      "activations/layer18_attention_weight_min": -20.634456634521484,
      "activations/layer19_attention_weight_max": 38.032440185546875,
      "activations/layer19_attention_weight_min": -30.462188720703125,
      "activations/layer1_attention_weight_max": 17.15080451965332,
      "activations/layer1_attention_weight_min": -16.892175674438477,
      "activations/layer20_attention_weight_max": 32.614356994628906,
      "activations/layer20_attention_weight_min": -23.079875946044922,
      "activations/layer21_attention_weight_max": 30.942296981811523,
      "activations/layer21_attention_weight_min": -21.69981575012207,
      "activations/layer22_attention_weight_max": 47.25885772705078,
      "activations/layer22_attention_weight_min": -27.01605796813965,
      "activations/layer23_attention_weight_max": 35.94358444213867,
      "activations/layer23_attention_weight_min": -23.43560028076172,
      "activations/layer2_attention_weight_max": 34.01710891723633,
      "activations/layer2_attention_weight_min": -32.65174102783203,
      "activations/layer3_attention_weight_max": 96.44135284423828,
      "activations/layer3_attention_weight_min": -93.3078842163086,
      "activations/layer4_attention_weight_max": 126.33878326416016,
      "activations/layer4_attention_weight_min": -122.33837127685547,
      "activations/layer5_attention_weight_max": 59.558040618896484,
      "activations/layer5_attention_weight_min": -60.272884368896484,
      "activations/layer6_attention_weight_max": 50.11427307128906,
      "activations/layer6_attention_weight_min": -49.94395446777344,
      "activations/layer7_attention_weight_max": 90.68112182617188,
      "activations/layer7_attention_weight_min": -88.29212188720703,
      "activations/layer8_attention_weight_max": 40.24556350708008,
      "activations/layer8_attention_weight_min": -40.72433853149414,
      "activations/layer9_attention_weight_max": 34.7741813659668,
      "activations/layer9_attention_weight_min": -34.47551727294922,
      "epoch": 7.87,
      "learning_rate": 0.00010023484848484846,
      "loss": 2.8143,
      "step": 135500
    },
    {
      "activations/layer0_attention_weight_max": 15.823263168334961,
      "activations/layer0_attention_weight_min": -12.293514251708984,
      "activations/layer10_attention_weight_max": 34.19255447387695,
      "activations/layer10_attention_weight_min": -33.944881439208984,
      "activations/layer11_attention_weight_max": 35.72233581542969,
      "activations/layer11_attention_weight_min": -32.50917434692383,
      "activations/layer12_attention_weight_max": 25.42022705078125,
      "activations/layer12_attention_weight_min": -27.790340423583984,
      "activations/layer13_attention_weight_max": 42.91484451293945,
      "activations/layer13_attention_weight_min": -34.4603271484375,
      "activations/layer14_attention_weight_max": 56.55857467651367,
      "activations/layer14_attention_weight_min": -45.94180679321289,
      "activations/layer15_attention_weight_max": 35.881839752197266,
      "activations/layer15_attention_weight_min": -29.4737548828125,
      "activations/layer16_attention_weight_max": 30.589799880981445,
      "activations/layer16_attention_weight_min": -28.48960304260254,
      "activations/layer17_attention_weight_max": 33.34957504272461,
      "activations/layer17_attention_weight_min": -25.383508682250977,
      "activations/layer18_attention_weight_max": 30.643268585205078,
      "activations/layer18_attention_weight_min": -20.3460636138916,
      "activations/layer19_attention_weight_max": 34.91913986206055,
      "activations/layer19_attention_weight_min": -29.605453491210938,
      "activations/layer1_attention_weight_max": 16.6932315826416,
      "activations/layer1_attention_weight_min": -14.459288597106934,
      "activations/layer20_attention_weight_max": 31.096113204956055,
      "activations/layer20_attention_weight_min": -24.740373611450195,
      "activations/layer21_attention_weight_max": 28.871700286865234,
      "activations/layer21_attention_weight_min": -23.218469619750977,
      "activations/layer22_attention_weight_max": 45.21055221557617,
      "activations/layer22_attention_weight_min": -31.885284423828125,
      "activations/layer23_attention_weight_max": 35.32868194580078,
      "activations/layer23_attention_weight_min": -25.27483367919922,
      "activations/layer2_attention_weight_max": 34.066383361816406,
      "activations/layer2_attention_weight_min": -33.067848205566406,
      "activations/layer3_attention_weight_max": 93.45118713378906,
      "activations/layer3_attention_weight_min": -95.0615234375,
      "activations/layer4_attention_weight_max": 124.23943328857422,
      "activations/layer4_attention_weight_min": -119.87957000732422,
      "activations/layer5_attention_weight_max": 62.53368377685547,
      "activations/layer5_attention_weight_min": -61.72434997558594,
      "activations/layer6_attention_weight_max": 53.30659484863281,
      "activations/layer6_attention_weight_min": -53.84849548339844,
      "activations/layer7_attention_weight_max": 90.060302734375,
      "activations/layer7_attention_weight_min": -90.38106536865234,
      "activations/layer8_attention_weight_max": 42.184326171875,
      "activations/layer8_attention_weight_min": -43.32967758178711,
      "activations/layer9_attention_weight_max": 37.296630859375,
      "activations/layer9_attention_weight_min": -37.28816604614258,
      "epoch": 7.88,
      "learning_rate": 0.00010021590909090908,
      "loss": 2.8071,
      "step": 135550
    },
    {
      "activations/layer0_attention_weight_max": 14.728744506835938,
      "activations/layer0_attention_weight_min": -11.649678230285645,
      "activations/layer10_attention_weight_max": 32.45878982543945,
      "activations/layer10_attention_weight_min": -33.10871124267578,
      "activations/layer11_attention_weight_max": 30.46234893798828,
      "activations/layer11_attention_weight_min": -31.211721420288086,
      "activations/layer12_attention_weight_max": 22.351842880249023,
      "activations/layer12_attention_weight_min": -27.196969985961914,
      "activations/layer13_attention_weight_max": 39.70924758911133,
      "activations/layer13_attention_weight_min": -34.98320770263672,
      "activations/layer14_attention_weight_max": 46.82024002075195,
      "activations/layer14_attention_weight_min": -39.83390426635742,
      "activations/layer15_attention_weight_max": 34.70719528198242,
      "activations/layer15_attention_weight_min": -30.41935157775879,
      "activations/layer16_attention_weight_max": 35.077518463134766,
      "activations/layer16_attention_weight_min": -27.956632614135742,
      "activations/layer17_attention_weight_max": 31.988651275634766,
      "activations/layer17_attention_weight_min": -26.483482360839844,
      "activations/layer18_attention_weight_max": 29.43647575378418,
      "activations/layer18_attention_weight_min": -22.322744369506836,
      "activations/layer19_attention_weight_max": 36.473506927490234,
      "activations/layer19_attention_weight_min": -29.689361572265625,
      "activations/layer1_attention_weight_max": 17.66779136657715,
      "activations/layer1_attention_weight_min": -13.846929550170898,
      "activations/layer20_attention_weight_max": 30.760520935058594,
      "activations/layer20_attention_weight_min": -24.51879119873047,
      "activations/layer21_attention_weight_max": 28.97704315185547,
      "activations/layer21_attention_weight_min": -22.34368324279785,
      "activations/layer22_attention_weight_max": 43.85898208618164,
      "activations/layer22_attention_weight_min": -33.79314422607422,
      "activations/layer23_attention_weight_max": 34.79820251464844,
      "activations/layer23_attention_weight_min": -28.063297271728516,
      "activations/layer2_attention_weight_max": 31.599227905273438,
      "activations/layer2_attention_weight_min": -29.812767028808594,
      "activations/layer3_attention_weight_max": 90.59373474121094,
      "activations/layer3_attention_weight_min": -90.8548355102539,
      "activations/layer4_attention_weight_max": 118.853271484375,
      "activations/layer4_attention_weight_min": -116.67974853515625,
      "activations/layer5_attention_weight_max": 60.165313720703125,
      "activations/layer5_attention_weight_min": -62.07200622558594,
      "activations/layer6_attention_weight_max": 49.65474319458008,
      "activations/layer6_attention_weight_min": -50.86952590942383,
      "activations/layer7_attention_weight_max": 87.12012481689453,
      "activations/layer7_attention_weight_min": -86.5585708618164,
      "activations/layer8_attention_weight_max": 40.198219299316406,
      "activations/layer8_attention_weight_min": -41.52338409423828,
      "activations/layer9_attention_weight_max": 35.35547637939453,
      "activations/layer9_attention_weight_min": -40.49680709838867,
      "epoch": 7.88,
      "learning_rate": 0.0001001969696969697,
      "loss": 2.8342,
      "step": 135600
    },
    {
      "activations/layer0_attention_weight_max": 16.10280990600586,
      "activations/layer0_attention_weight_min": -12.568990707397461,
      "activations/layer10_attention_weight_max": 32.588783264160156,
      "activations/layer10_attention_weight_min": -29.96170425415039,
      "activations/layer11_attention_weight_max": 32.72211837768555,
      "activations/layer11_attention_weight_min": -29.884979248046875,
      "activations/layer12_attention_weight_max": 24.799455642700195,
      "activations/layer12_attention_weight_min": -30.020389556884766,
      "activations/layer13_attention_weight_max": 45.14165115356445,
      "activations/layer13_attention_weight_min": -32.65653610229492,
      "activations/layer14_attention_weight_max": 48.943660736083984,
      "activations/layer14_attention_weight_min": -39.333587646484375,
      "activations/layer15_attention_weight_max": 37.218017578125,
      "activations/layer15_attention_weight_min": -29.575246810913086,
      "activations/layer16_attention_weight_max": 32.83391189575195,
      "activations/layer16_attention_weight_min": -27.512319564819336,
      "activations/layer17_attention_weight_max": 37.22426986694336,
      "activations/layer17_attention_weight_min": -25.407114028930664,
      "activations/layer18_attention_weight_max": 32.46526336669922,
      "activations/layer18_attention_weight_min": -21.738771438598633,
      "activations/layer19_attention_weight_max": 37.48298263549805,
      "activations/layer19_attention_weight_min": -29.163156509399414,
      "activations/layer1_attention_weight_max": 17.58065414428711,
      "activations/layer1_attention_weight_min": -15.163804054260254,
      "activations/layer20_attention_weight_max": 31.502593994140625,
      "activations/layer20_attention_weight_min": -23.189311981201172,
      "activations/layer21_attention_weight_max": 33.81440353393555,
      "activations/layer21_attention_weight_min": -22.976057052612305,
      "activations/layer22_attention_weight_max": 45.75482940673828,
      "activations/layer22_attention_weight_min": -26.912508010864258,
      "activations/layer23_attention_weight_max": 34.24962615966797,
      "activations/layer23_attention_weight_min": -22.261940002441406,
      "activations/layer2_attention_weight_max": 35.39229965209961,
      "activations/layer2_attention_weight_min": -33.33741760253906,
      "activations/layer3_attention_weight_max": 94.2174301147461,
      "activations/layer3_attention_weight_min": -95.1401596069336,
      "activations/layer4_attention_weight_max": 118.54664611816406,
      "activations/layer4_attention_weight_min": -115.66676330566406,
      "activations/layer5_attention_weight_max": 60.71567916870117,
      "activations/layer5_attention_weight_min": -57.802330017089844,
      "activations/layer6_attention_weight_max": 50.01358413696289,
      "activations/layer6_attention_weight_min": -51.550086975097656,
      "activations/layer7_attention_weight_max": 88.04585266113281,
      "activations/layer7_attention_weight_min": -89.49285125732422,
      "activations/layer8_attention_weight_max": 40.081722259521484,
      "activations/layer8_attention_weight_min": -39.72417068481445,
      "activations/layer9_attention_weight_max": 34.167903900146484,
      "activations/layer9_attention_weight_min": -31.763654708862305,
      "epoch": 7.88,
      "learning_rate": 0.00010017803030303028,
      "loss": 2.8251,
      "step": 135650
    },
    {
      "activations/layer0_attention_weight_max": 16.509727478027344,
      "activations/layer0_attention_weight_min": -13.17483139038086,
      "activations/layer10_attention_weight_max": 36.284461975097656,
      "activations/layer10_attention_weight_min": -33.6251335144043,
      "activations/layer11_attention_weight_max": 33.516300201416016,
      "activations/layer11_attention_weight_min": -31.223766326904297,
      "activations/layer12_attention_weight_max": 23.721784591674805,
      "activations/layer12_attention_weight_min": -23.631641387939453,
      "activations/layer13_attention_weight_max": 38.90410614013672,
      "activations/layer13_attention_weight_min": -33.799537658691406,
      "activations/layer14_attention_weight_max": 41.673065185546875,
      "activations/layer14_attention_weight_min": -39.183990478515625,
      "activations/layer15_attention_weight_max": 38.45558547973633,
      "activations/layer15_attention_weight_min": -32.221370697021484,
      "activations/layer16_attention_weight_max": 29.730056762695312,
      "activations/layer16_attention_weight_min": -27.68781852722168,
      "activations/layer17_attention_weight_max": 34.32945251464844,
      "activations/layer17_attention_weight_min": -23.708383560180664,
      "activations/layer18_attention_weight_max": 34.72705841064453,
      "activations/layer18_attention_weight_min": -22.94428062438965,
      "activations/layer19_attention_weight_max": 34.87502670288086,
      "activations/layer19_attention_weight_min": -31.941904067993164,
      "activations/layer1_attention_weight_max": 16.477943420410156,
      "activations/layer1_attention_weight_min": -14.499748229980469,
      "activations/layer20_attention_weight_max": 32.08237075805664,
      "activations/layer20_attention_weight_min": -23.484512329101562,
      "activations/layer21_attention_weight_max": 31.238210678100586,
      "activations/layer21_attention_weight_min": -23.420719146728516,
      "activations/layer22_attention_weight_max": 42.278114318847656,
      "activations/layer22_attention_weight_min": -29.58108139038086,
      "activations/layer23_attention_weight_max": 33.86402893066406,
      "activations/layer23_attention_weight_min": -24.410072326660156,
      "activations/layer2_attention_weight_max": 32.768680572509766,
      "activations/layer2_attention_weight_min": -32.89326477050781,
      "activations/layer3_attention_weight_max": 94.50813293457031,
      "activations/layer3_attention_weight_min": -95.50725555419922,
      "activations/layer4_attention_weight_max": 121.03902435302734,
      "activations/layer4_attention_weight_min": -120.99407196044922,
      "activations/layer5_attention_weight_max": 56.676509857177734,
      "activations/layer5_attention_weight_min": -62.22361755371094,
      "activations/layer6_attention_weight_max": 47.455421447753906,
      "activations/layer6_attention_weight_min": -53.697452545166016,
      "activations/layer7_attention_weight_max": 86.0460205078125,
      "activations/layer7_attention_weight_min": -88.48250579833984,
      "activations/layer8_attention_weight_max": 41.01533889770508,
      "activations/layer8_attention_weight_min": -42.42985916137695,
      "activations/layer9_attention_weight_max": 37.70559310913086,
      "activations/layer9_attention_weight_min": -33.53694152832031,
      "epoch": 7.88,
      "learning_rate": 0.0001001590909090909,
      "loss": 2.8078,
      "step": 135700
    },
    {
      "activations/layer0_attention_weight_max": 15.506370544433594,
      "activations/layer0_attention_weight_min": -11.377066612243652,
      "activations/layer10_attention_weight_max": 34.6431770324707,
      "activations/layer10_attention_weight_min": -32.83024597167969,
      "activations/layer11_attention_weight_max": 33.06345748901367,
      "activations/layer11_attention_weight_min": -29.022106170654297,
      "activations/layer12_attention_weight_max": 23.69611167907715,
      "activations/layer12_attention_weight_min": -25.9947452545166,
      "activations/layer13_attention_weight_max": 37.03583908081055,
      "activations/layer13_attention_weight_min": -32.14331817626953,
      "activations/layer14_attention_weight_max": 46.41462707519531,
      "activations/layer14_attention_weight_min": -40.20222473144531,
      "activations/layer15_attention_weight_max": 36.17717361450195,
      "activations/layer15_attention_weight_min": -29.28446388244629,
      "activations/layer16_attention_weight_max": 32.108001708984375,
      "activations/layer16_attention_weight_min": -26.80866813659668,
      "activations/layer17_attention_weight_max": 33.30303955078125,
      "activations/layer17_attention_weight_min": -24.35967445373535,
      "activations/layer18_attention_weight_max": 33.41279983520508,
      "activations/layer18_attention_weight_min": -21.398372650146484,
      "activations/layer19_attention_weight_max": 35.87421798706055,
      "activations/layer19_attention_weight_min": -31.662357330322266,
      "activations/layer1_attention_weight_max": 16.479936599731445,
      "activations/layer1_attention_weight_min": -14.865107536315918,
      "activations/layer20_attention_weight_max": 31.09991455078125,
      "activations/layer20_attention_weight_min": -25.807828903198242,
      "activations/layer21_attention_weight_max": 29.758865356445312,
      "activations/layer21_attention_weight_min": -22.763477325439453,
      "activations/layer22_attention_weight_max": 46.606773376464844,
      "activations/layer22_attention_weight_min": -31.798067092895508,
      "activations/layer23_attention_weight_max": 34.1153564453125,
      "activations/layer23_attention_weight_min": -25.154834747314453,
      "activations/layer2_attention_weight_max": 34.49015808105469,
      "activations/layer2_attention_weight_min": -33.420326232910156,
      "activations/layer3_attention_weight_max": 93.74773406982422,
      "activations/layer3_attention_weight_min": -99.93685150146484,
      "activations/layer4_attention_weight_max": 117.76167297363281,
      "activations/layer4_attention_weight_min": -117.26637268066406,
      "activations/layer5_attention_weight_max": 59.44824981689453,
      "activations/layer5_attention_weight_min": -60.55825424194336,
      "activations/layer6_attention_weight_max": 48.93916702270508,
      "activations/layer6_attention_weight_min": -50.61882781982422,
      "activations/layer7_attention_weight_max": 84.30172729492188,
      "activations/layer7_attention_weight_min": -88.0358657836914,
      "activations/layer8_attention_weight_max": 41.42926788330078,
      "activations/layer8_attention_weight_min": -39.6102180480957,
      "activations/layer9_attention_weight_max": 35.20307922363281,
      "activations/layer9_attention_weight_min": -32.28367614746094,
      "epoch": 7.89,
      "learning_rate": 0.00010014015151515151,
      "loss": 2.8179,
      "step": 135750
    },
    {
      "activations/layer0_attention_weight_max": 15.118093490600586,
      "activations/layer0_attention_weight_min": -11.772802352905273,
      "activations/layer10_attention_weight_max": 30.567245483398438,
      "activations/layer10_attention_weight_min": -30.52477264404297,
      "activations/layer11_attention_weight_max": 29.79848289489746,
      "activations/layer11_attention_weight_min": -30.38991928100586,
      "activations/layer12_attention_weight_max": 23.70414924621582,
      "activations/layer12_attention_weight_min": -28.201190948486328,
      "activations/layer13_attention_weight_max": 35.21561050415039,
      "activations/layer13_attention_weight_min": -31.934864044189453,
      "activations/layer14_attention_weight_max": 47.700801849365234,
      "activations/layer14_attention_weight_min": -38.506595611572266,
      "activations/layer15_attention_weight_max": 34.88634490966797,
      "activations/layer15_attention_weight_min": -29.877490997314453,
      "activations/layer16_attention_weight_max": 31.215282440185547,
      "activations/layer16_attention_weight_min": -28.232210159301758,
      "activations/layer17_attention_weight_max": 34.00009536743164,
      "activations/layer17_attention_weight_min": -25.0194091796875,
      "activations/layer18_attention_weight_max": 32.4868278503418,
      "activations/layer18_attention_weight_min": -22.580577850341797,
      "activations/layer19_attention_weight_max": 36.8543586730957,
      "activations/layer19_attention_weight_min": -30.90203857421875,
      "activations/layer1_attention_weight_max": 16.38446807861328,
      "activations/layer1_attention_weight_min": -13.950094223022461,
      "activations/layer20_attention_weight_max": 30.32448387145996,
      "activations/layer20_attention_weight_min": -22.094615936279297,
      "activations/layer21_attention_weight_max": 29.413877487182617,
      "activations/layer21_attention_weight_min": -22.300973892211914,
      "activations/layer22_attention_weight_max": 43.354705810546875,
      "activations/layer22_attention_weight_min": -27.8857479095459,
      "activations/layer23_attention_weight_max": 34.590972900390625,
      "activations/layer23_attention_weight_min": -24.128585815429688,
      "activations/layer2_attention_weight_max": 33.40440368652344,
      "activations/layer2_attention_weight_min": -30.37735366821289,
      "activations/layer3_attention_weight_max": 93.26847076416016,
      "activations/layer3_attention_weight_min": -93.61727142333984,
      "activations/layer4_attention_weight_max": 120.01834869384766,
      "activations/layer4_attention_weight_min": -114.66182708740234,
      "activations/layer5_attention_weight_max": 58.13153076171875,
      "activations/layer5_attention_weight_min": -64.99463653564453,
      "activations/layer6_attention_weight_max": 48.30775833129883,
      "activations/layer6_attention_weight_min": -51.47185516357422,
      "activations/layer7_attention_weight_max": 89.30233001708984,
      "activations/layer7_attention_weight_min": -88.81843566894531,
      "activations/layer8_attention_weight_max": 38.44926071166992,
      "activations/layer8_attention_weight_min": -39.179447174072266,
      "activations/layer9_attention_weight_max": 32.3055305480957,
      "activations/layer9_attention_weight_min": -33.11748504638672,
      "epoch": 7.89,
      "learning_rate": 0.0001001212121212121,
      "loss": 2.7999,
      "step": 135800
    },
    {
      "activations/layer0_attention_weight_max": 15.290979385375977,
      "activations/layer0_attention_weight_min": -12.166946411132812,
      "activations/layer10_attention_weight_max": 34.17303466796875,
      "activations/layer10_attention_weight_min": -30.67252540588379,
      "activations/layer11_attention_weight_max": 30.792003631591797,
      "activations/layer11_attention_weight_min": -31.576370239257812,
      "activations/layer12_attention_weight_max": 22.311328887939453,
      "activations/layer12_attention_weight_min": -27.011341094970703,
      "activations/layer13_attention_weight_max": 38.0062255859375,
      "activations/layer13_attention_weight_min": -33.91975784301758,
      "activations/layer14_attention_weight_max": 50.57475662231445,
      "activations/layer14_attention_weight_min": -43.42736053466797,
      "activations/layer15_attention_weight_max": 34.271873474121094,
      "activations/layer15_attention_weight_min": -30.968639373779297,
      "activations/layer16_attention_weight_max": 34.9619026184082,
      "activations/layer16_attention_weight_min": -27.273263931274414,
      "activations/layer17_attention_weight_max": 35.99325942993164,
      "activations/layer17_attention_weight_min": -25.601688385009766,
      "activations/layer18_attention_weight_max": 31.98466682434082,
      "activations/layer18_attention_weight_min": -22.46047592163086,
      "activations/layer19_attention_weight_max": 36.077877044677734,
      "activations/layer19_attention_weight_min": -30.96881675720215,
      "activations/layer1_attention_weight_max": 17.125783920288086,
      "activations/layer1_attention_weight_min": -14.681501388549805,
      "activations/layer20_attention_weight_max": 28.767244338989258,
      "activations/layer20_attention_weight_min": -24.41844940185547,
      "activations/layer21_attention_weight_max": 28.908742904663086,
      "activations/layer21_attention_weight_min": -24.01898193359375,
      "activations/layer22_attention_weight_max": 50.42170715332031,
      "activations/layer22_attention_weight_min": -29.087251663208008,
      "activations/layer23_attention_weight_max": 35.55817413330078,
      "activations/layer23_attention_weight_min": -24.200071334838867,
      "activations/layer2_attention_weight_max": 32.827789306640625,
      "activations/layer2_attention_weight_min": -32.176536560058594,
      "activations/layer3_attention_weight_max": 92.51567077636719,
      "activations/layer3_attention_weight_min": -94.30430603027344,
      "activations/layer4_attention_weight_max": 118.90630340576172,
      "activations/layer4_attention_weight_min": -117.38016510009766,
      "activations/layer5_attention_weight_max": 58.35280990600586,
      "activations/layer5_attention_weight_min": -63.18766403198242,
      "activations/layer6_attention_weight_max": 50.79753875732422,
      "activations/layer6_attention_weight_min": -52.17130661010742,
      "activations/layer7_attention_weight_max": 87.40061950683594,
      "activations/layer7_attention_weight_min": -86.37464141845703,
      "activations/layer8_attention_weight_max": 39.79025650024414,
      "activations/layer8_attention_weight_min": -38.39250946044922,
      "activations/layer9_attention_weight_max": 33.57344055175781,
      "activations/layer9_attention_weight_min": -33.09243392944336,
      "epoch": 7.89,
      "learning_rate": 0.00010010227272727272,
      "loss": 2.8225,
      "step": 135850
    },
    {
      "activations/layer0_attention_weight_max": 15.731761932373047,
      "activations/layer0_attention_weight_min": -12.450613021850586,
      "activations/layer10_attention_weight_max": 38.931819915771484,
      "activations/layer10_attention_weight_min": -34.356536865234375,
      "activations/layer11_attention_weight_max": 37.50312042236328,
      "activations/layer11_attention_weight_min": -31.79974365234375,
      "activations/layer12_attention_weight_max": 25.36644172668457,
      "activations/layer12_attention_weight_min": -28.385059356689453,
      "activations/layer13_attention_weight_max": 41.401790618896484,
      "activations/layer13_attention_weight_min": -33.59650421142578,
      "activations/layer14_attention_weight_max": 51.92280578613281,
      "activations/layer14_attention_weight_min": -39.37484359741211,
      "activations/layer15_attention_weight_max": 36.65715408325195,
      "activations/layer15_attention_weight_min": -29.429298400878906,
      "activations/layer16_attention_weight_max": 35.27369689941406,
      "activations/layer16_attention_weight_min": -28.73523712158203,
      "activations/layer17_attention_weight_max": 34.678802490234375,
      "activations/layer17_attention_weight_min": -26.439889907836914,
      "activations/layer18_attention_weight_max": 32.14911651611328,
      "activations/layer18_attention_weight_min": -20.4554500579834,
      "activations/layer19_attention_weight_max": 38.79930877685547,
      "activations/layer19_attention_weight_min": -31.750263214111328,
      "activations/layer1_attention_weight_max": 16.630369186401367,
      "activations/layer1_attention_weight_min": -15.342514038085938,
      "activations/layer20_attention_weight_max": 34.10993576049805,
      "activations/layer20_attention_weight_min": -24.333860397338867,
      "activations/layer21_attention_weight_max": 28.571813583374023,
      "activations/layer21_attention_weight_min": -23.261791229248047,
      "activations/layer22_attention_weight_max": 50.667110443115234,
      "activations/layer22_attention_weight_min": -26.632083892822266,
      "activations/layer23_attention_weight_max": 38.78572082519531,
      "activations/layer23_attention_weight_min": -22.089797973632812,
      "activations/layer2_attention_weight_max": 33.91083526611328,
      "activations/layer2_attention_weight_min": -32.61127853393555,
      "activations/layer3_attention_weight_max": 95.89259338378906,
      "activations/layer3_attention_weight_min": -92.4951171875,
      "activations/layer4_attention_weight_max": 120.84368896484375,
      "activations/layer4_attention_weight_min": -117.02388763427734,
      "activations/layer5_attention_weight_max": 58.525543212890625,
      "activations/layer5_attention_weight_min": -63.08120346069336,
      "activations/layer6_attention_weight_max": 49.711143493652344,
      "activations/layer6_attention_weight_min": -50.10417556762695,
      "activations/layer7_attention_weight_max": 93.56683349609375,
      "activations/layer7_attention_weight_min": -90.49676513671875,
      "activations/layer8_attention_weight_max": 42.99771499633789,
      "activations/layer8_attention_weight_min": -39.887996673583984,
      "activations/layer9_attention_weight_max": 39.909393310546875,
      "activations/layer9_attention_weight_min": -36.75088119506836,
      "epoch": 7.9,
      "learning_rate": 0.00010008333333333333,
      "loss": 2.8117,
      "step": 135900
    },
    {
      "activations/layer0_attention_weight_max": 15.74687671661377,
      "activations/layer0_attention_weight_min": -12.177424430847168,
      "activations/layer10_attention_weight_max": 34.13116455078125,
      "activations/layer10_attention_weight_min": -32.84157180786133,
      "activations/layer11_attention_weight_max": 33.77469253540039,
      "activations/layer11_attention_weight_min": -32.957157135009766,
      "activations/layer12_attention_weight_max": 30.941606521606445,
      "activations/layer12_attention_weight_min": -24.808738708496094,
      "activations/layer13_attention_weight_max": 42.85626983642578,
      "activations/layer13_attention_weight_min": -35.81321334838867,
      "activations/layer14_attention_weight_max": 49.71402359008789,
      "activations/layer14_attention_weight_min": -41.6595458984375,
      "activations/layer15_attention_weight_max": 39.89508819580078,
      "activations/layer15_attention_weight_min": -30.76984977722168,
      "activations/layer16_attention_weight_max": 33.09262466430664,
      "activations/layer16_attention_weight_min": -28.03877067565918,
      "activations/layer17_attention_weight_max": 36.37520980834961,
      "activations/layer17_attention_weight_min": -24.895021438598633,
      "activations/layer18_attention_weight_max": 31.477209091186523,
      "activations/layer18_attention_weight_min": -21.298229217529297,
      "activations/layer19_attention_weight_max": 39.348716735839844,
      "activations/layer19_attention_weight_min": -28.659509658813477,
      "activations/layer1_attention_weight_max": 16.813060760498047,
      "activations/layer1_attention_weight_min": -14.999632835388184,
      "activations/layer20_attention_weight_max": 31.42894172668457,
      "activations/layer20_attention_weight_min": -24.418210983276367,
      "activations/layer21_attention_weight_max": 30.486690521240234,
      "activations/layer21_attention_weight_min": -22.026132583618164,
      "activations/layer22_attention_weight_max": 47.95048141479492,
      "activations/layer22_attention_weight_min": -27.249937057495117,
      "activations/layer23_attention_weight_max": 38.5349235534668,
      "activations/layer23_attention_weight_min": -23.71481704711914,
      "activations/layer2_attention_weight_max": 32.47578430175781,
      "activations/layer2_attention_weight_min": -32.871826171875,
      "activations/layer3_attention_weight_max": 97.56537628173828,
      "activations/layer3_attention_weight_min": -97.35941314697266,
      "activations/layer4_attention_weight_max": 118.9499740600586,
      "activations/layer4_attention_weight_min": -120.1009750366211,
      "activations/layer5_attention_weight_max": 57.10674285888672,
      "activations/layer5_attention_weight_min": -62.50487518310547,
      "activations/layer6_attention_weight_max": 48.465843200683594,
      "activations/layer6_attention_weight_min": -54.36368179321289,
      "activations/layer7_attention_weight_max": 88.30117797851562,
      "activations/layer7_attention_weight_min": -95.8807601928711,
      "activations/layer8_attention_weight_max": 39.46479415893555,
      "activations/layer8_attention_weight_min": -42.63639450073242,
      "activations/layer9_attention_weight_max": 34.81437301635742,
      "activations/layer9_attention_weight_min": -33.71361541748047,
      "epoch": 7.9,
      "learning_rate": 0.00010006477272727272,
      "loss": 2.8081,
      "step": 135950
    },
    {
      "activations/layer0_attention_weight_max": 15.305407524108887,
      "activations/layer0_attention_weight_min": -12.943920135498047,
      "activations/layer10_attention_weight_max": 31.622482299804688,
      "activations/layer10_attention_weight_min": -32.522056579589844,
      "activations/layer11_attention_weight_max": 32.240638732910156,
      "activations/layer11_attention_weight_min": -31.05763816833496,
      "activations/layer12_attention_weight_max": 26.011146545410156,
      "activations/layer12_attention_weight_min": -24.925504684448242,
      "activations/layer13_attention_weight_max": 42.172569274902344,
      "activations/layer13_attention_weight_min": -34.045082092285156,
      "activations/layer14_attention_weight_max": 48.201541900634766,
      "activations/layer14_attention_weight_min": -38.7617073059082,
      "activations/layer15_attention_weight_max": 38.33122634887695,
      "activations/layer15_attention_weight_min": -34.15656661987305,
      "activations/layer16_attention_weight_max": 34.66096496582031,
      "activations/layer16_attention_weight_min": -27.96950912475586,
      "activations/layer17_attention_weight_max": 37.4642219543457,
      "activations/layer17_attention_weight_min": -26.035505294799805,
      "activations/layer18_attention_weight_max": 30.513057708740234,
      "activations/layer18_attention_weight_min": -20.80417251586914,
      "activations/layer19_attention_weight_max": 39.588294982910156,
      "activations/layer19_attention_weight_min": -33.12510681152344,
      "activations/layer1_attention_weight_max": 16.755290985107422,
      "activations/layer1_attention_weight_min": -14.530839920043945,
      "activations/layer20_attention_weight_max": 30.438459396362305,
      "activations/layer20_attention_weight_min": -24.313440322875977,
      "activations/layer21_attention_weight_max": 28.415874481201172,
      "activations/layer21_attention_weight_min": -21.820730209350586,
      "activations/layer22_attention_weight_max": 44.930580139160156,
      "activations/layer22_attention_weight_min": -28.195215225219727,
      "activations/layer23_attention_weight_max": 34.479095458984375,
      "activations/layer23_attention_weight_min": -22.790687561035156,
      "activations/layer2_attention_weight_max": 32.28968048095703,
      "activations/layer2_attention_weight_min": -32.58179473876953,
      "activations/layer3_attention_weight_max": 92.41791534423828,
      "activations/layer3_attention_weight_min": -96.40462493896484,
      "activations/layer4_attention_weight_max": 122.24239349365234,
      "activations/layer4_attention_weight_min": -121.87853240966797,
      "activations/layer5_attention_weight_max": 58.60600280761719,
      "activations/layer5_attention_weight_min": -62.14106369018555,
      "activations/layer6_attention_weight_max": 52.18595504760742,
      "activations/layer6_attention_weight_min": -54.94239807128906,
      "activations/layer7_attention_weight_max": 90.38529968261719,
      "activations/layer7_attention_weight_min": -89.2215347290039,
      "activations/layer8_attention_weight_max": 41.7032356262207,
      "activations/layer8_attention_weight_min": -40.460227966308594,
      "activations/layer9_attention_weight_max": 34.55634689331055,
      "activations/layer9_attention_weight_min": -34.11794662475586,
      "epoch": 7.9,
      "learning_rate": 0.00010004583333333332,
      "loss": 2.8129,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.4736,
      "eval_samples_per_second": 506.752,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.4736,
      "eval_openwebtext_samples_per_second": 506.752,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_wikitext_loss": 2.970703125,
      "eval_wikitext_ppl": 19.505629671380603,
      "eval_wikitext_runtime": 2.0293,
      "eval_wikitext_samples_per_second": 224.707,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_lambada_loss": 2.52734375,
      "eval_lambada_ppl": 12.520205133467638,
      "eval_lambada_runtime": 9.5595,
      "eval_lambada_samples_per_second": 509.336,
      "step": 136000
    },
    {
      "activations/layer0_attention_weight_max": 15.884866714477539,
      "activations/layer0_attention_weight_min": -12.322294235229492,
      "activations/layer10_attention_weight_max": 39.004554748535156,
      "activations/layer10_attention_weight_min": -40.798213958740234,
      "activations/layer11_attention_weight_max": 36.85015106201172,
      "activations/layer11_attention_weight_min": -34.64073944091797,
      "activations/layer12_attention_weight_max": 31.081159591674805,
      "activations/layer12_attention_weight_min": -23.934986114501953,
      "activations/layer13_attention_weight_max": 51.86132049560547,
      "activations/layer13_attention_weight_min": -42.20292663574219,
      "activations/layer14_attention_weight_max": 70.49845123291016,
      "activations/layer14_attention_weight_min": -54.96105194091797,
      "activations/layer15_attention_weight_max": 38.67076873779297,
      "activations/layer15_attention_weight_min": -30.546266555786133,
      "activations/layer16_attention_weight_max": 33.60948181152344,
      "activations/layer16_attention_weight_min": -27.278966903686523,
      "activations/layer17_attention_weight_max": 35.75511169433594,
      "activations/layer17_attention_weight_min": -26.030290603637695,
      "activations/layer18_attention_weight_max": 33.333335876464844,
      "activations/layer18_attention_weight_min": -22.052888870239258,
      "activations/layer19_attention_weight_max": 36.01361083984375,
      "activations/layer19_attention_weight_min": -33.3597526550293,
      "activations/layer1_attention_weight_max": 19.059589385986328,
      "activations/layer1_attention_weight_min": -17.826007843017578,
      "activations/layer20_attention_weight_max": 31.742475509643555,
      "activations/layer20_attention_weight_min": -29.212879180908203,
      "activations/layer21_attention_weight_max": 31.563901901245117,
      "activations/layer21_attention_weight_min": -25.011754989624023,
      "activations/layer22_attention_weight_max": 55.71652603149414,
      "activations/layer22_attention_weight_min": -33.34065246582031,
      "activations/layer23_attention_weight_max": 35.81391906738281,
      "activations/layer23_attention_weight_min": -24.499462127685547,
      "activations/layer2_attention_weight_max": 35.62920379638672,
      "activations/layer2_attention_weight_min": -32.88880920410156,
      "activations/layer3_attention_weight_max": 96.26654815673828,
      "activations/layer3_attention_weight_min": -92.38427734375,
      "activations/layer4_attention_weight_max": 124.47596740722656,
      "activations/layer4_attention_weight_min": -112.29052734375,
      "activations/layer5_attention_weight_max": 57.95756149291992,
      "activations/layer5_attention_weight_min": -59.24412155151367,
      "activations/layer6_attention_weight_max": 50.60177993774414,
      "activations/layer6_attention_weight_min": -51.55149841308594,
      "activations/layer7_attention_weight_max": 91.22248077392578,
      "activations/layer7_attention_weight_min": -90.7725830078125,
      "activations/layer8_attention_weight_max": 48.285465240478516,
      "activations/layer8_attention_weight_min": -44.375972747802734,
      "activations/layer9_attention_weight_max": 39.9017448425293,
      "activations/layer9_attention_weight_min": -38.3171272277832,
      "epoch": 7.91,
      "learning_rate": 0.00010002689393939394,
      "loss": 2.8176,
      "step": 136050
    },
    {
      "activations/layer0_attention_weight_max": 16.129131317138672,
      "activations/layer0_attention_weight_min": -12.106171607971191,
      "activations/layer10_attention_weight_max": 35.197364807128906,
      "activations/layer10_attention_weight_min": -32.59761428833008,
      "activations/layer11_attention_weight_max": 33.52130889892578,
      "activations/layer11_attention_weight_min": -29.484256744384766,
      "activations/layer12_attention_weight_max": 26.85242462158203,
      "activations/layer12_attention_weight_min": -29.0355167388916,
      "activations/layer13_attention_weight_max": 45.73208999633789,
      "activations/layer13_attention_weight_min": -33.12724685668945,
      "activations/layer14_attention_weight_max": 49.0946044921875,
      "activations/layer14_attention_weight_min": -38.52811050415039,
      "activations/layer15_attention_weight_max": 38.436859130859375,
      "activations/layer15_attention_weight_min": -29.56143569946289,
      "activations/layer16_attention_weight_max": 32.325565338134766,
      "activations/layer16_attention_weight_min": -27.827899932861328,
      "activations/layer17_attention_weight_max": 36.105499267578125,
      "activations/layer17_attention_weight_min": -24.04140281677246,
      "activations/layer18_attention_weight_max": 34.3443717956543,
      "activations/layer18_attention_weight_min": -21.050640106201172,
      "activations/layer19_attention_weight_max": 37.858028411865234,
      "activations/layer19_attention_weight_min": -28.562524795532227,
      "activations/layer1_attention_weight_max": 17.06136703491211,
      "activations/layer1_attention_weight_min": -15.45991325378418,
      "activations/layer20_attention_weight_max": 32.35014343261719,
      "activations/layer20_attention_weight_min": -24.432817459106445,
      "activations/layer21_attention_weight_max": 33.26017379760742,
      "activations/layer21_attention_weight_min": -22.720617294311523,
      "activations/layer22_attention_weight_max": 49.5364875793457,
      "activations/layer22_attention_weight_min": -27.969085693359375,
      "activations/layer23_attention_weight_max": 36.926124572753906,
      "activations/layer23_attention_weight_min": -24.88032341003418,
      "activations/layer2_attention_weight_max": 32.51209259033203,
      "activations/layer2_attention_weight_min": -33.635093688964844,
      "activations/layer3_attention_weight_max": 92.58626556396484,
      "activations/layer3_attention_weight_min": -97.70425415039062,
      "activations/layer4_attention_weight_max": 121.52953338623047,
      "activations/layer4_attention_weight_min": -122.5953140258789,
      "activations/layer5_attention_weight_max": 58.8485221862793,
      "activations/layer5_attention_weight_min": -63.80029296875,
      "activations/layer6_attention_weight_max": 52.445804595947266,
      "activations/layer6_attention_weight_min": -53.57767868041992,
      "activations/layer7_attention_weight_max": 89.38334655761719,
      "activations/layer7_attention_weight_min": -90.21353912353516,
      "activations/layer8_attention_weight_max": 43.758522033691406,
      "activations/layer8_attention_weight_min": -44.21417236328125,
      "activations/layer9_attention_weight_max": 37.8551139831543,
      "activations/layer9_attention_weight_min": -35.24613952636719,
      "epoch": 7.91,
      "learning_rate": 0.00010000795454545454,
      "loss": 2.8103,
      "step": 136100
    },
    {
      "activations/layer0_attention_weight_max": 15.257214546203613,
      "activations/layer0_attention_weight_min": -11.442084312438965,
      "activations/layer10_attention_weight_max": 33.9425163269043,
      "activations/layer10_attention_weight_min": -30.855979919433594,
      "activations/layer11_attention_weight_max": 30.608139038085938,
      "activations/layer11_attention_weight_min": -28.622282028198242,
      "activations/layer12_attention_weight_max": 27.332260131835938,
      "activations/layer12_attention_weight_min": -26.042245864868164,
      "activations/layer13_attention_weight_max": 36.02758026123047,
      "activations/layer13_attention_weight_min": -29.925769805908203,
      "activations/layer14_attention_weight_max": 45.404151916503906,
      "activations/layer14_attention_weight_min": -34.694725036621094,
      "activations/layer15_attention_weight_max": 33.00136947631836,
      "activations/layer15_attention_weight_min": -28.49006462097168,
      "activations/layer16_attention_weight_max": 30.493738174438477,
      "activations/layer16_attention_weight_min": -26.954862594604492,
      "activations/layer17_attention_weight_max": 31.770944595336914,
      "activations/layer17_attention_weight_min": -23.168672561645508,
      "activations/layer18_attention_weight_max": 31.325944900512695,
      "activations/layer18_attention_weight_min": -20.49275779724121,
      "activations/layer19_attention_weight_max": 35.34536361694336,
      "activations/layer19_attention_weight_min": -28.113492965698242,
      "activations/layer1_attention_weight_max": 16.105070114135742,
      "activations/layer1_attention_weight_min": -14.949877738952637,
      "activations/layer20_attention_weight_max": 30.53643035888672,
      "activations/layer20_attention_weight_min": -23.69652557373047,
      "activations/layer21_attention_weight_max": 27.143173217773438,
      "activations/layer21_attention_weight_min": -22.663772583007812,
      "activations/layer22_attention_weight_max": 45.305660247802734,
      "activations/layer22_attention_weight_min": -27.29850196838379,
      "activations/layer23_attention_weight_max": 33.64549255371094,
      "activations/layer23_attention_weight_min": -23.703723907470703,
      "activations/layer2_attention_weight_max": 31.730175018310547,
      "activations/layer2_attention_weight_min": -32.103050231933594,
      "activations/layer3_attention_weight_max": 89.74340057373047,
      "activations/layer3_attention_weight_min": -95.34895324707031,
      "activations/layer4_attention_weight_max": 113.04666900634766,
      "activations/layer4_attention_weight_min": -112.06707000732422,
      "activations/layer5_attention_weight_max": 56.347412109375,
      "activations/layer5_attention_weight_min": -63.36260986328125,
      "activations/layer6_attention_weight_max": 51.79784393310547,
      "activations/layer6_attention_weight_min": -51.840938568115234,
      "activations/layer7_attention_weight_max": 85.77967071533203,
      "activations/layer7_attention_weight_min": -83.05130004882812,
      "activations/layer8_attention_weight_max": 37.82202911376953,
      "activations/layer8_attention_weight_min": -40.01942443847656,
      "activations/layer9_attention_weight_max": 32.958106994628906,
      "activations/layer9_attention_weight_min": -32.42217254638672,
      "epoch": 7.91,
      "learning_rate": 9.998901515151514e-05,
      "loss": 2.8009,
      "step": 136150
    },
    {
      "activations/layer0_attention_weight_max": 15.619339942932129,
      "activations/layer0_attention_weight_min": -11.683856964111328,
      "activations/layer10_attention_weight_max": 32.40171432495117,
      "activations/layer10_attention_weight_min": -30.89070701599121,
      "activations/layer11_attention_weight_max": 31.547603607177734,
      "activations/layer11_attention_weight_min": -28.38311004638672,
      "activations/layer12_attention_weight_max": 36.133766174316406,
      "activations/layer12_attention_weight_min": -25.099811553955078,
      "activations/layer13_attention_weight_max": 47.18312454223633,
      "activations/layer13_attention_weight_min": -32.790077209472656,
      "activations/layer14_attention_weight_max": 55.15208435058594,
      "activations/layer14_attention_weight_min": -40.2889518737793,
      "activations/layer15_attention_weight_max": 37.94097137451172,
      "activations/layer15_attention_weight_min": -30.54659080505371,
      "activations/layer16_attention_weight_max": 31.790027618408203,
      "activations/layer16_attention_weight_min": -27.863142013549805,
      "activations/layer17_attention_weight_max": 45.03179931640625,
      "activations/layer17_attention_weight_min": -25.974348068237305,
      "activations/layer18_attention_weight_max": 33.4567985534668,
      "activations/layer18_attention_weight_min": -21.71926498413086,
      "activations/layer19_attention_weight_max": 41.61099624633789,
      "activations/layer19_attention_weight_min": -31.060964584350586,
      "activations/layer1_attention_weight_max": 17.303190231323242,
      "activations/layer1_attention_weight_min": -15.775654792785645,
      "activations/layer20_attention_weight_max": 30.689680099487305,
      "activations/layer20_attention_weight_min": -25.004770278930664,
      "activations/layer21_attention_weight_max": 33.3827018737793,
      "activations/layer21_attention_weight_min": -21.798669815063477,
      "activations/layer22_attention_weight_max": 48.48685836791992,
      "activations/layer22_attention_weight_min": -29.09503173828125,
      "activations/layer23_attention_weight_max": 41.074432373046875,
      "activations/layer23_attention_weight_min": -23.4757080078125,
      "activations/layer2_attention_weight_max": 33.83796691894531,
      "activations/layer2_attention_weight_min": -33.94316864013672,
      "activations/layer3_attention_weight_max": 94.12651824951172,
      "activations/layer3_attention_weight_min": -93.65684509277344,
      "activations/layer4_attention_weight_max": 115.87623596191406,
      "activations/layer4_attention_weight_min": -114.36605072021484,
      "activations/layer5_attention_weight_max": 55.60942077636719,
      "activations/layer5_attention_weight_min": -60.81135177612305,
      "activations/layer6_attention_weight_max": 49.288856506347656,
      "activations/layer6_attention_weight_min": -51.1068000793457,
      "activations/layer7_attention_weight_max": 86.28722381591797,
      "activations/layer7_attention_weight_min": -85.70805358886719,
      "activations/layer8_attention_weight_max": 39.56593704223633,
      "activations/layer8_attention_weight_min": -39.2176513671875,
      "activations/layer9_attention_weight_max": 37.608394622802734,
      "activations/layer9_attention_weight_min": -35.15572738647461,
      "epoch": 7.91,
      "learning_rate": 9.997007575757576e-05,
      "loss": 2.8125,
      "step": 136200
    },
    {
      "activations/layer0_attention_weight_max": 15.937398910522461,
      "activations/layer0_attention_weight_min": -11.590621948242188,
      "activations/layer10_attention_weight_max": 32.169227600097656,
      "activations/layer10_attention_weight_min": -28.734737396240234,
      "activations/layer11_attention_weight_max": 29.327678680419922,
      "activations/layer11_attention_weight_min": -28.867399215698242,
      "activations/layer12_attention_weight_max": 24.38227653503418,
      "activations/layer12_attention_weight_min": -27.30634117126465,
      "activations/layer13_attention_weight_max": 36.00836181640625,
      "activations/layer13_attention_weight_min": -31.288375854492188,
      "activations/layer14_attention_weight_max": 49.54792022705078,
      "activations/layer14_attention_weight_min": -37.12394714355469,
      "activations/layer15_attention_weight_max": 35.11847686767578,
      "activations/layer15_attention_weight_min": -31.08135986328125,
      "activations/layer16_attention_weight_max": 34.692501068115234,
      "activations/layer16_attention_weight_min": -27.354536056518555,
      "activations/layer17_attention_weight_max": 32.853721618652344,
      "activations/layer17_attention_weight_min": -25.25594139099121,
      "activations/layer18_attention_weight_max": 32.9233512878418,
      "activations/layer18_attention_weight_min": -21.073190689086914,
      "activations/layer19_attention_weight_max": 37.711483001708984,
      "activations/layer19_attention_weight_min": -29.7729434967041,
      "activations/layer1_attention_weight_max": 17.004993438720703,
      "activations/layer1_attention_weight_min": -13.923184394836426,
      "activations/layer20_attention_weight_max": 31.219160079956055,
      "activations/layer20_attention_weight_min": -22.554201126098633,
      "activations/layer21_attention_weight_max": 34.58782958984375,
      "activations/layer21_attention_weight_min": -21.6507568359375,
      "activations/layer22_attention_weight_max": 45.774497985839844,
      "activations/layer22_attention_weight_min": -28.55516242980957,
      "activations/layer23_attention_weight_max": 36.641117095947266,
      "activations/layer23_attention_weight_min": -23.002056121826172,
      "activations/layer2_attention_weight_max": 32.284671783447266,
      "activations/layer2_attention_weight_min": -31.07857322692871,
      "activations/layer3_attention_weight_max": 88.89014434814453,
      "activations/layer3_attention_weight_min": -91.12798309326172,
      "activations/layer4_attention_weight_max": 115.72489166259766,
      "activations/layer4_attention_weight_min": -110.47877502441406,
      "activations/layer5_attention_weight_max": 57.55596923828125,
      "activations/layer5_attention_weight_min": -59.17951583862305,
      "activations/layer6_attention_weight_max": 45.862083435058594,
      "activations/layer6_attention_weight_min": -50.39287185668945,
      "activations/layer7_attention_weight_max": 85.8042221069336,
      "activations/layer7_attention_weight_min": -81.35169219970703,
      "activations/layer8_attention_weight_max": 35.69996643066406,
      "activations/layer8_attention_weight_min": -37.889366149902344,
      "activations/layer9_attention_weight_max": 31.664274215698242,
      "activations/layer9_attention_weight_min": -31.990524291992188,
      "epoch": 7.92,
      "learning_rate": 9.995113636363634e-05,
      "loss": 2.8127,
      "step": 136250
    },
    {
      "activations/layer0_attention_weight_max": 15.398723602294922,
      "activations/layer0_attention_weight_min": -11.9339017868042,
      "activations/layer10_attention_weight_max": 34.42584991455078,
      "activations/layer10_attention_weight_min": -31.826231002807617,
      "activations/layer11_attention_weight_max": 32.24896240234375,
      "activations/layer11_attention_weight_min": -31.00177001953125,
      "activations/layer12_attention_weight_max": 30.865047454833984,
      "activations/layer12_attention_weight_min": -29.96013069152832,
      "activations/layer13_attention_weight_max": 37.0029182434082,
      "activations/layer13_attention_weight_min": -32.912322998046875,
      "activations/layer14_attention_weight_max": 48.15711212158203,
      "activations/layer14_attention_weight_min": -39.56584548950195,
      "activations/layer15_attention_weight_max": 34.8539924621582,
      "activations/layer15_attention_weight_min": -29.222475051879883,
      "activations/layer16_attention_weight_max": 32.75422286987305,
      "activations/layer16_attention_weight_min": -27.664487838745117,
      "activations/layer17_attention_weight_max": 33.52187728881836,
      "activations/layer17_attention_weight_min": -24.99674415588379,
      "activations/layer18_attention_weight_max": 32.296539306640625,
      "activations/layer18_attention_weight_min": -21.937374114990234,
      "activations/layer19_attention_weight_max": 35.172019958496094,
      "activations/layer19_attention_weight_min": -34.282283782958984,
      "activations/layer1_attention_weight_max": 16.44224739074707,
      "activations/layer1_attention_weight_min": -15.871712684631348,
      "activations/layer20_attention_weight_max": 28.351715087890625,
      "activations/layer20_attention_weight_min": -24.213848114013672,
      "activations/layer21_attention_weight_max": 29.28302001953125,
      "activations/layer21_attention_weight_min": -22.580411911010742,
      "activations/layer22_attention_weight_max": 45.31638717651367,
      "activations/layer22_attention_weight_min": -27.800825119018555,
      "activations/layer23_attention_weight_max": 36.958038330078125,
      "activations/layer23_attention_weight_min": -22.55133056640625,
      "activations/layer2_attention_weight_max": 32.55718994140625,
      "activations/layer2_attention_weight_min": -32.333621978759766,
      "activations/layer3_attention_weight_max": 90.72340393066406,
      "activations/layer3_attention_weight_min": -92.00484466552734,
      "activations/layer4_attention_weight_max": 122.33465576171875,
      "activations/layer4_attention_weight_min": -119.759521484375,
      "activations/layer5_attention_weight_max": 58.61639404296875,
      "activations/layer5_attention_weight_min": -63.25091552734375,
      "activations/layer6_attention_weight_max": 52.18314743041992,
      "activations/layer6_attention_weight_min": -52.11823654174805,
      "activations/layer7_attention_weight_max": 91.65151977539062,
      "activations/layer7_attention_weight_min": -84.10855865478516,
      "activations/layer8_attention_weight_max": 39.30826187133789,
      "activations/layer8_attention_weight_min": -39.80949401855469,
      "activations/layer9_attention_weight_max": 37.00156021118164,
      "activations/layer9_attention_weight_min": -32.95954513549805,
      "epoch": 7.92,
      "learning_rate": 9.993219696969696e-05,
      "loss": 2.8175,
      "step": 136300
    },
    {
      "activations/layer0_attention_weight_max": 15.385019302368164,
      "activations/layer0_attention_weight_min": -12.161203384399414,
      "activations/layer10_attention_weight_max": 34.75407028198242,
      "activations/layer10_attention_weight_min": -34.77630615234375,
      "activations/layer11_attention_weight_max": 31.403076171875,
      "activations/layer11_attention_weight_min": -29.454113006591797,
      "activations/layer12_attention_weight_max": 26.898433685302734,
      "activations/layer12_attention_weight_min": -26.12485694885254,
      "activations/layer13_attention_weight_max": 39.9306526184082,
      "activations/layer13_attention_weight_min": -32.128692626953125,
      "activations/layer14_attention_weight_max": 54.48057556152344,
      "activations/layer14_attention_weight_min": -44.979827880859375,
      "activations/layer15_attention_weight_max": 41.870811462402344,
      "activations/layer15_attention_weight_min": -29.273344039916992,
      "activations/layer16_attention_weight_max": 31.948745727539062,
      "activations/layer16_attention_weight_min": -28.18093490600586,
      "activations/layer17_attention_weight_max": 38.003536224365234,
      "activations/layer17_attention_weight_min": -25.74949073791504,
      "activations/layer18_attention_weight_max": 30.297334671020508,
      "activations/layer18_attention_weight_min": -20.383636474609375,
      "activations/layer19_attention_weight_max": 42.381019592285156,
      "activations/layer19_attention_weight_min": -28.943872451782227,
      "activations/layer1_attention_weight_max": 16.87660789489746,
      "activations/layer1_attention_weight_min": -14.707308769226074,
      "activations/layer20_attention_weight_max": 37.1713981628418,
      "activations/layer20_attention_weight_min": -23.770280838012695,
      "activations/layer21_attention_weight_max": 35.9316291809082,
      "activations/layer21_attention_weight_min": -22.65203285217285,
      "activations/layer22_attention_weight_max": 56.16782760620117,
      "activations/layer22_attention_weight_min": -28.111787796020508,
      "activations/layer23_attention_weight_max": 41.297080993652344,
      "activations/layer23_attention_weight_min": -23.648086547851562,
      "activations/layer2_attention_weight_max": 32.18632507324219,
      "activations/layer2_attention_weight_min": -31.888553619384766,
      "activations/layer3_attention_weight_max": 96.44691467285156,
      "activations/layer3_attention_weight_min": -93.23162841796875,
      "activations/layer4_attention_weight_max": 130.2488250732422,
      "activations/layer4_attention_weight_min": -119.28275299072266,
      "activations/layer5_attention_weight_max": 60.145347595214844,
      "activations/layer5_attention_weight_min": -65.33570861816406,
      "activations/layer6_attention_weight_max": 50.48330307006836,
      "activations/layer6_attention_weight_min": -50.50400924682617,
      "activations/layer7_attention_weight_max": 92.22176361083984,
      "activations/layer7_attention_weight_min": -87.88727569580078,
      "activations/layer8_attention_weight_max": 39.31911087036133,
      "activations/layer8_attention_weight_min": -40.180484771728516,
      "activations/layer9_attention_weight_max": 34.92025375366211,
      "activations/layer9_attention_weight_min": -34.68050765991211,
      "epoch": 7.92,
      "learning_rate": 9.991325757575757e-05,
      "loss": 2.8242,
      "step": 136350
    },
    {
      "activations/layer0_attention_weight_max": 17.076641082763672,
      "activations/layer0_attention_weight_min": -12.267416954040527,
      "activations/layer10_attention_weight_max": 34.643577575683594,
      "activations/layer10_attention_weight_min": -31.080162048339844,
      "activations/layer11_attention_weight_max": 34.819435119628906,
      "activations/layer11_attention_weight_min": -28.658336639404297,
      "activations/layer12_attention_weight_max": 30.82246208190918,
      "activations/layer12_attention_weight_min": -27.861467361450195,
      "activations/layer13_attention_weight_max": 48.943111419677734,
      "activations/layer13_attention_weight_min": -34.186153411865234,
      "activations/layer14_attention_weight_max": 64.69078063964844,
      "activations/layer14_attention_weight_min": -42.20126724243164,
      "activations/layer15_attention_weight_max": 40.84175109863281,
      "activations/layer15_attention_weight_min": -33.565303802490234,
      "activations/layer16_attention_weight_max": 35.23423385620117,
      "activations/layer16_attention_weight_min": -27.441177368164062,
      "activations/layer17_attention_weight_max": 35.78004837036133,
      "activations/layer17_attention_weight_min": -24.36773109436035,
      "activations/layer18_attention_weight_max": 40.20869064331055,
      "activations/layer18_attention_weight_min": -21.991098403930664,
      "activations/layer19_attention_weight_max": 44.03404235839844,
      "activations/layer19_attention_weight_min": -30.576353073120117,
      "activations/layer1_attention_weight_max": 19.02865982055664,
      "activations/layer1_attention_weight_min": -13.330679893493652,
      "activations/layer20_attention_weight_max": 36.83169937133789,
      "activations/layer20_attention_weight_min": -23.44054412841797,
      "activations/layer21_attention_weight_max": 35.44892501831055,
      "activations/layer21_attention_weight_min": -22.814590454101562,
      "activations/layer22_attention_weight_max": 53.58880615234375,
      "activations/layer22_attention_weight_min": -29.16172218322754,
      "activations/layer23_attention_weight_max": 38.42435073852539,
      "activations/layer23_attention_weight_min": -23.03542137145996,
      "activations/layer2_attention_weight_max": 32.16374588012695,
      "activations/layer2_attention_weight_min": -30.98044204711914,
      "activations/layer3_attention_weight_max": 91.79444885253906,
      "activations/layer3_attention_weight_min": -94.73787689208984,
      "activations/layer4_attention_weight_max": 122.50701141357422,
      "activations/layer4_attention_weight_min": -126.1352310180664,
      "activations/layer5_attention_weight_max": 59.84934616088867,
      "activations/layer5_attention_weight_min": -65.80205535888672,
      "activations/layer6_attention_weight_max": 51.47959518432617,
      "activations/layer6_attention_weight_min": -54.48446273803711,
      "activations/layer7_attention_weight_max": 87.96680450439453,
      "activations/layer7_attention_weight_min": -89.82206726074219,
      "activations/layer8_attention_weight_max": 40.955692291259766,
      "activations/layer8_attention_weight_min": -40.78112030029297,
      "activations/layer9_attention_weight_max": 36.96780014038086,
      "activations/layer9_attention_weight_min": -33.93378829956055,
      "epoch": 7.93,
      "learning_rate": 9.989431818181816e-05,
      "loss": 2.8037,
      "step": 136400
    },
    {
      "activations/layer0_attention_weight_max": 15.59191608428955,
      "activations/layer0_attention_weight_min": -12.522360801696777,
      "activations/layer10_attention_weight_max": 32.64573669433594,
      "activations/layer10_attention_weight_min": -31.380260467529297,
      "activations/layer11_attention_weight_max": 35.55133056640625,
      "activations/layer11_attention_weight_min": -29.569129943847656,
      "activations/layer12_attention_weight_max": 46.75003433227539,
      "activations/layer12_attention_weight_min": -26.559085845947266,
      "activations/layer13_attention_weight_max": 64.42478942871094,
      "activations/layer13_attention_weight_min": -38.85451126098633,
      "activations/layer14_attention_weight_max": 57.111270904541016,
      "activations/layer14_attention_weight_min": -40.341278076171875,
      "activations/layer15_attention_weight_max": 70.18646240234375,
      "activations/layer15_attention_weight_min": -35.7537841796875,
      "activations/layer16_attention_weight_max": 43.46613311767578,
      "activations/layer16_attention_weight_min": -29.460691452026367,
      "activations/layer17_attention_weight_max": 36.62623977661133,
      "activations/layer17_attention_weight_min": -25.019454956054688,
      "activations/layer18_attention_weight_max": 33.36185073852539,
      "activations/layer18_attention_weight_min": -22.440509796142578,
      "activations/layer19_attention_weight_max": 43.152244567871094,
      "activations/layer19_attention_weight_min": -28.817495346069336,
      "activations/layer1_attention_weight_max": 17.74903106689453,
      "activations/layer1_attention_weight_min": -14.838046073913574,
      "activations/layer20_attention_weight_max": 37.24095153808594,
      "activations/layer20_attention_weight_min": -23.68610954284668,
      "activations/layer21_attention_weight_max": 35.98588943481445,
      "activations/layer21_attention_weight_min": -22.66059684753418,
      "activations/layer22_attention_weight_max": 49.443424224853516,
      "activations/layer22_attention_weight_min": -29.61569595336914,
      "activations/layer23_attention_weight_max": 43.277191162109375,
      "activations/layer23_attention_weight_min": -24.35455322265625,
      "activations/layer2_attention_weight_max": 31.253334045410156,
      "activations/layer2_attention_weight_min": -30.658370971679688,
      "activations/layer3_attention_weight_max": 87.68002319335938,
      "activations/layer3_attention_weight_min": -89.03776550292969,
      "activations/layer4_attention_weight_max": 113.42479705810547,
      "activations/layer4_attention_weight_min": -114.5160140991211,
      "activations/layer5_attention_weight_max": 54.8150749206543,
      "activations/layer5_attention_weight_min": -59.8515739440918,
      "activations/layer6_attention_weight_max": 48.19443130493164,
      "activations/layer6_attention_weight_min": -52.95838928222656,
      "activations/layer7_attention_weight_max": 89.34228515625,
      "activations/layer7_attention_weight_min": -90.02767181396484,
      "activations/layer8_attention_weight_max": 39.89552688598633,
      "activations/layer8_attention_weight_min": -40.55029296875,
      "activations/layer9_attention_weight_max": 37.22405242919922,
      "activations/layer9_attention_weight_min": -33.44521713256836,
      "epoch": 7.93,
      "learning_rate": 9.987537878787878e-05,
      "loss": 2.8314,
      "step": 136450
    },
    {
      "activations/layer0_attention_weight_max": 16.35968780517578,
      "activations/layer0_attention_weight_min": -12.840577125549316,
      "activations/layer10_attention_weight_max": 35.553077697753906,
      "activations/layer10_attention_weight_min": -33.082183837890625,
      "activations/layer11_attention_weight_max": 38.26810073852539,
      "activations/layer11_attention_weight_min": -30.947019577026367,
      "activations/layer12_attention_weight_max": 24.525060653686523,
      "activations/layer12_attention_weight_min": -26.230588912963867,
      "activations/layer13_attention_weight_max": 50.96471405029297,
      "activations/layer13_attention_weight_min": -35.999916076660156,
      "activations/layer14_attention_weight_max": 58.80818557739258,
      "activations/layer14_attention_weight_min": -44.155845642089844,
      "activations/layer15_attention_weight_max": 42.39719772338867,
      "activations/layer15_attention_weight_min": -32.700401306152344,
      "activations/layer16_attention_weight_max": 34.60960388183594,
      "activations/layer16_attention_weight_min": -29.318443298339844,
      "activations/layer17_attention_weight_max": 34.584014892578125,
      "activations/layer17_attention_weight_min": -25.41997528076172,
      "activations/layer18_attention_weight_max": 32.70856857299805,
      "activations/layer18_attention_weight_min": -24.15507698059082,
      "activations/layer19_attention_weight_max": 37.9649772644043,
      "activations/layer19_attention_weight_min": -31.310382843017578,
      "activations/layer1_attention_weight_max": 17.091657638549805,
      "activations/layer1_attention_weight_min": -13.880838394165039,
      "activations/layer20_attention_weight_max": 34.70546340942383,
      "activations/layer20_attention_weight_min": -25.944292068481445,
      "activations/layer21_attention_weight_max": 29.694448471069336,
      "activations/layer21_attention_weight_min": -23.121437072753906,
      "activations/layer22_attention_weight_max": 49.68748474121094,
      "activations/layer22_attention_weight_min": -31.074514389038086,
      "activations/layer23_attention_weight_max": 38.11125183105469,
      "activations/layer23_attention_weight_min": -27.617305755615234,
      "activations/layer2_attention_weight_max": 32.323150634765625,
      "activations/layer2_attention_weight_min": -32.34961700439453,
      "activations/layer3_attention_weight_max": 90.44116973876953,
      "activations/layer3_attention_weight_min": -93.40262603759766,
      "activations/layer4_attention_weight_max": 116.86528778076172,
      "activations/layer4_attention_weight_min": -122.97320556640625,
      "activations/layer5_attention_weight_max": 54.868892669677734,
      "activations/layer5_attention_weight_min": -63.08635711669922,
      "activations/layer6_attention_weight_max": 49.59260559082031,
      "activations/layer6_attention_weight_min": -49.29767990112305,
      "activations/layer7_attention_weight_max": 90.44824981689453,
      "activations/layer7_attention_weight_min": -89.40586853027344,
      "activations/layer8_attention_weight_max": 42.27786636352539,
      "activations/layer8_attention_weight_min": -40.91304397583008,
      "activations/layer9_attention_weight_max": 36.38203048706055,
      "activations/layer9_attention_weight_min": -34.51161193847656,
      "epoch": 7.93,
      "learning_rate": 9.985643939393939e-05,
      "loss": 2.8207,
      "step": 136500
    },
    {
      "activations/layer0_attention_weight_max": 14.61681079864502,
      "activations/layer0_attention_weight_min": -11.635058403015137,
      "activations/layer10_attention_weight_max": 34.36745071411133,
      "activations/layer10_attention_weight_min": -30.82107162475586,
      "activations/layer11_attention_weight_max": 31.457599639892578,
      "activations/layer11_attention_weight_min": -29.832693099975586,
      "activations/layer12_attention_weight_max": 25.09979248046875,
      "activations/layer12_attention_weight_min": -23.80669593811035,
      "activations/layer13_attention_weight_max": 37.408958435058594,
      "activations/layer13_attention_weight_min": -32.88805389404297,
      "activations/layer14_attention_weight_max": 46.47080993652344,
      "activations/layer14_attention_weight_min": -36.98704528808594,
      "activations/layer15_attention_weight_max": 36.83820343017578,
      "activations/layer15_attention_weight_min": -30.744598388671875,
      "activations/layer16_attention_weight_max": 30.660999298095703,
      "activations/layer16_attention_weight_min": -29.324872970581055,
      "activations/layer17_attention_weight_max": 35.191802978515625,
      "activations/layer17_attention_weight_min": -25.581493377685547,
      "activations/layer18_attention_weight_max": 34.99905014038086,
      "activations/layer18_attention_weight_min": -22.3857421875,
      "activations/layer19_attention_weight_max": 36.46218490600586,
      "activations/layer19_attention_weight_min": -30.481037139892578,
      "activations/layer1_attention_weight_max": 16.242172241210938,
      "activations/layer1_attention_weight_min": -14.677630424499512,
      "activations/layer20_attention_weight_max": 31.191316604614258,
      "activations/layer20_attention_weight_min": -23.890329360961914,
      "activations/layer21_attention_weight_max": 33.01433563232422,
      "activations/layer21_attention_weight_min": -22.743057250976562,
      "activations/layer22_attention_weight_max": 46.7573127746582,
      "activations/layer22_attention_weight_min": -27.258211135864258,
      "activations/layer23_attention_weight_max": 36.538230895996094,
      "activations/layer23_attention_weight_min": -23.20545768737793,
      "activations/layer2_attention_weight_max": 33.19328689575195,
      "activations/layer2_attention_weight_min": -32.54780578613281,
      "activations/layer3_attention_weight_max": 95.72190856933594,
      "activations/layer3_attention_weight_min": -95.36444091796875,
      "activations/layer4_attention_weight_max": 123.28556060791016,
      "activations/layer4_attention_weight_min": -118.1205825805664,
      "activations/layer5_attention_weight_max": 56.975372314453125,
      "activations/layer5_attention_weight_min": -60.898521423339844,
      "activations/layer6_attention_weight_max": 48.571556091308594,
      "activations/layer6_attention_weight_min": -50.06619644165039,
      "activations/layer7_attention_weight_max": 89.78252410888672,
      "activations/layer7_attention_weight_min": -84.09027099609375,
      "activations/layer8_attention_weight_max": 39.49885559082031,
      "activations/layer8_attention_weight_min": -40.57442855834961,
      "activations/layer9_attention_weight_max": 33.632530212402344,
      "activations/layer9_attention_weight_min": -33.300289154052734,
      "epoch": 7.93,
      "learning_rate": 9.983749999999998e-05,
      "loss": 2.8313,
      "step": 136550
    },
    {
      "activations/layer0_attention_weight_max": 15.85715389251709,
      "activations/layer0_attention_weight_min": -12.791168212890625,
      "activations/layer10_attention_weight_max": 34.506317138671875,
      "activations/layer10_attention_weight_min": -34.04867935180664,
      "activations/layer11_attention_weight_max": 33.25107192993164,
      "activations/layer11_attention_weight_min": -30.208744049072266,
      "activations/layer12_attention_weight_max": 37.67782211303711,
      "activations/layer12_attention_weight_min": -23.55930519104004,
      "activations/layer13_attention_weight_max": 48.5737419128418,
      "activations/layer13_attention_weight_min": -34.55522918701172,
      "activations/layer14_attention_weight_max": 59.07612609863281,
      "activations/layer14_attention_weight_min": -43.52954864501953,
      "activations/layer15_attention_weight_max": 37.76935577392578,
      "activations/layer15_attention_weight_min": -29.97305679321289,
      "activations/layer16_attention_weight_max": 34.80561065673828,
      "activations/layer16_attention_weight_min": -28.668458938598633,
      "activations/layer17_attention_weight_max": 36.823570251464844,
      "activations/layer17_attention_weight_min": -24.61907958984375,
      "activations/layer18_attention_weight_max": 33.64408493041992,
      "activations/layer18_attention_weight_min": -21.184532165527344,
      "activations/layer19_attention_weight_max": 38.94072341918945,
      "activations/layer19_attention_weight_min": -31.09646987915039,
      "activations/layer1_attention_weight_max": 16.44381332397461,
      "activations/layer1_attention_weight_min": -15.577152252197266,
      "activations/layer20_attention_weight_max": 33.59856414794922,
      "activations/layer20_attention_weight_min": -23.953758239746094,
      "activations/layer21_attention_weight_max": 29.61748504638672,
      "activations/layer21_attention_weight_min": -21.765100479125977,
      "activations/layer22_attention_weight_max": 61.67650604248047,
      "activations/layer22_attention_weight_min": -28.50584602355957,
      "activations/layer23_attention_weight_max": 36.97284698486328,
      "activations/layer23_attention_weight_min": -21.95796012878418,
      "activations/layer2_attention_weight_max": 32.65559387207031,
      "activations/layer2_attention_weight_min": -32.81005859375,
      "activations/layer3_attention_weight_max": 97.4583969116211,
      "activations/layer3_attention_weight_min": -97.17964172363281,
      "activations/layer4_attention_weight_max": 130.8335418701172,
      "activations/layer4_attention_weight_min": -120.576416015625,
      "activations/layer5_attention_weight_max": 61.20722198486328,
      "activations/layer5_attention_weight_min": -64.21196746826172,
      "activations/layer6_attention_weight_max": 50.69184875488281,
      "activations/layer6_attention_weight_min": -53.63473892211914,
      "activations/layer7_attention_weight_max": 94.60812377929688,
      "activations/layer7_attention_weight_min": -93.0099105834961,
      "activations/layer8_attention_weight_max": 41.409156799316406,
      "activations/layer8_attention_weight_min": -43.831382751464844,
      "activations/layer9_attention_weight_max": 37.58810043334961,
      "activations/layer9_attention_weight_min": -36.35614776611328,
      "epoch": 7.94,
      "learning_rate": 9.98185606060606e-05,
      "loss": 2.7974,
      "step": 136600
    },
    {
      "activations/layer0_attention_weight_max": 15.419708251953125,
      "activations/layer0_attention_weight_min": -11.160762786865234,
      "activations/layer10_attention_weight_max": 32.36979293823242,
      "activations/layer10_attention_weight_min": -31.43605613708496,
      "activations/layer11_attention_weight_max": 32.81410217285156,
      "activations/layer11_attention_weight_min": -29.36628532409668,
      "activations/layer12_attention_weight_max": 22.656909942626953,
      "activations/layer12_attention_weight_min": -27.027307510375977,
      "activations/layer13_attention_weight_max": 43.6485481262207,
      "activations/layer13_attention_weight_min": -32.68573760986328,
      "activations/layer14_attention_weight_max": 53.7092399597168,
      "activations/layer14_attention_weight_min": -39.72119903564453,
      "activations/layer15_attention_weight_max": 37.78453826904297,
      "activations/layer15_attention_weight_min": -29.933021545410156,
      "activations/layer16_attention_weight_max": 31.77986717224121,
      "activations/layer16_attention_weight_min": -27.43318748474121,
      "activations/layer17_attention_weight_max": 37.44917678833008,
      "activations/layer17_attention_weight_min": -24.178607940673828,
      "activations/layer18_attention_weight_max": 31.806316375732422,
      "activations/layer18_attention_weight_min": -22.64889144897461,
      "activations/layer19_attention_weight_max": 34.181880950927734,
      "activations/layer19_attention_weight_min": -29.302412033081055,
      "activations/layer1_attention_weight_max": 16.704954147338867,
      "activations/layer1_attention_weight_min": -16.119462966918945,
      "activations/layer20_attention_weight_max": 30.91774559020996,
      "activations/layer20_attention_weight_min": -25.72984504699707,
      "activations/layer21_attention_weight_max": 29.589170455932617,
      "activations/layer21_attention_weight_min": -24.39128303527832,
      "activations/layer22_attention_weight_max": 43.84939956665039,
      "activations/layer22_attention_weight_min": -28.122278213500977,
      "activations/layer23_attention_weight_max": 35.833133697509766,
      "activations/layer23_attention_weight_min": -22.75611686706543,
      "activations/layer2_attention_weight_max": 31.673519134521484,
      "activations/layer2_attention_weight_min": -30.668989181518555,
      "activations/layer3_attention_weight_max": 89.95291137695312,
      "activations/layer3_attention_weight_min": -89.33958435058594,
      "activations/layer4_attention_weight_max": 117.0381088256836,
      "activations/layer4_attention_weight_min": -112.4859619140625,
      "activations/layer5_attention_weight_max": 56.289039611816406,
      "activations/layer5_attention_weight_min": -58.13726043701172,
      "activations/layer6_attention_weight_max": 46.60051727294922,
      "activations/layer6_attention_weight_min": -50.70791244506836,
      "activations/layer7_attention_weight_max": 82.75782012939453,
      "activations/layer7_attention_weight_min": -86.69723510742188,
      "activations/layer8_attention_weight_max": 38.90637969970703,
      "activations/layer8_attention_weight_min": -38.65306854248047,
      "activations/layer9_attention_weight_max": 34.10822677612305,
      "activations/layer9_attention_weight_min": -33.285308837890625,
      "epoch": 7.94,
      "learning_rate": 9.979962121212121e-05,
      "loss": 2.8115,
      "step": 136650
    },
    {
      "activations/layer0_attention_weight_max": 14.949986457824707,
      "activations/layer0_attention_weight_min": -11.709953308105469,
      "activations/layer10_attention_weight_max": 35.99358367919922,
      "activations/layer10_attention_weight_min": -32.20139694213867,
      "activations/layer11_attention_weight_max": 36.63671112060547,
      "activations/layer11_attention_weight_min": -30.84429168701172,
      "activations/layer12_attention_weight_max": 24.333179473876953,
      "activations/layer12_attention_weight_min": -23.68150520324707,
      "activations/layer13_attention_weight_max": 41.106056213378906,
      "activations/layer13_attention_weight_min": -34.81149673461914,
      "activations/layer14_attention_weight_max": 49.46601867675781,
      "activations/layer14_attention_weight_min": -39.85451126098633,
      "activations/layer15_attention_weight_max": 35.89179611206055,
      "activations/layer15_attention_weight_min": -30.03827667236328,
      "activations/layer16_attention_weight_max": 33.210418701171875,
      "activations/layer16_attention_weight_min": -27.663684844970703,
      "activations/layer17_attention_weight_max": 37.088863372802734,
      "activations/layer17_attention_weight_min": -26.401674270629883,
      "activations/layer18_attention_weight_max": 34.791629791259766,
      "activations/layer18_attention_weight_min": -22.44398307800293,
      "activations/layer19_attention_weight_max": 37.961116790771484,
      "activations/layer19_attention_weight_min": -31.81186294555664,
      "activations/layer1_attention_weight_max": 16.286853790283203,
      "activations/layer1_attention_weight_min": -15.421648979187012,
      "activations/layer20_attention_weight_max": 33.131629943847656,
      "activations/layer20_attention_weight_min": -27.195144653320312,
      "activations/layer21_attention_weight_max": 31.248807907104492,
      "activations/layer21_attention_weight_min": -25.29315185546875,
      "activations/layer22_attention_weight_max": 57.88277816772461,
      "activations/layer22_attention_weight_min": -32.24078369140625,
      "activations/layer23_attention_weight_max": 39.74941635131836,
      "activations/layer23_attention_weight_min": -25.19453239440918,
      "activations/layer2_attention_weight_max": 33.231109619140625,
      "activations/layer2_attention_weight_min": -32.62956237792969,
      "activations/layer3_attention_weight_max": 91.47538757324219,
      "activations/layer3_attention_weight_min": -95.24689483642578,
      "activations/layer4_attention_weight_max": 120.34126281738281,
      "activations/layer4_attention_weight_min": -121.40923309326172,
      "activations/layer5_attention_weight_max": 60.16874313354492,
      "activations/layer5_attention_weight_min": -63.70548629760742,
      "activations/layer6_attention_weight_max": 49.075538635253906,
      "activations/layer6_attention_weight_min": -56.092933654785156,
      "activations/layer7_attention_weight_max": 84.89425659179688,
      "activations/layer7_attention_weight_min": -92.79080200195312,
      "activations/layer8_attention_weight_max": 44.1995735168457,
      "activations/layer8_attention_weight_min": -45.90039825439453,
      "activations/layer9_attention_weight_max": 36.78745651245117,
      "activations/layer9_attention_weight_min": -35.02951431274414,
      "epoch": 7.94,
      "learning_rate": 9.978068181818181e-05,
      "loss": 2.8029,
      "step": 136700
    },
    {
      "activations/layer0_attention_weight_max": 15.363058090209961,
      "activations/layer0_attention_weight_min": -13.062232971191406,
      "activations/layer10_attention_weight_max": 32.305091857910156,
      "activations/layer10_attention_weight_min": -33.6595458984375,
      "activations/layer11_attention_weight_max": 32.511451721191406,
      "activations/layer11_attention_weight_min": -29.409414291381836,
      "activations/layer12_attention_weight_max": 21.749582290649414,
      "activations/layer12_attention_weight_min": -25.6401309967041,
      "activations/layer13_attention_weight_max": 36.69029235839844,
      "activations/layer13_attention_weight_min": -34.071266174316406,
      "activations/layer14_attention_weight_max": 48.36631393432617,
      "activations/layer14_attention_weight_min": -40.624412536621094,
      "activations/layer15_attention_weight_max": 32.52503967285156,
      "activations/layer15_attention_weight_min": -30.729230880737305,
      "activations/layer16_attention_weight_max": 33.839996337890625,
      "activations/layer16_attention_weight_min": -26.781007766723633,
      "activations/layer17_attention_weight_max": 30.71647071838379,
      "activations/layer17_attention_weight_min": -25.542316436767578,
      "activations/layer18_attention_weight_max": 29.82615852355957,
      "activations/layer18_attention_weight_min": -23.696374893188477,
      "activations/layer19_attention_weight_max": 38.96633529663086,
      "activations/layer19_attention_weight_min": -32.38987350463867,
      "activations/layer1_attention_weight_max": 17.05539321899414,
      "activations/layer1_attention_weight_min": -15.87131118774414,
      "activations/layer20_attention_weight_max": 29.715469360351562,
      "activations/layer20_attention_weight_min": -23.76690101623535,
      "activations/layer21_attention_weight_max": 29.10708999633789,
      "activations/layer21_attention_weight_min": -22.54031753540039,
      "activations/layer22_attention_weight_max": 50.67282485961914,
      "activations/layer22_attention_weight_min": -28.148483276367188,
      "activations/layer23_attention_weight_max": 36.40143585205078,
      "activations/layer23_attention_weight_min": -26.229595184326172,
      "activations/layer2_attention_weight_max": 32.51219940185547,
      "activations/layer2_attention_weight_min": -32.01592254638672,
      "activations/layer3_attention_weight_max": 93.51314544677734,
      "activations/layer3_attention_weight_min": -94.23414611816406,
      "activations/layer4_attention_weight_max": 118.5006103515625,
      "activations/layer4_attention_weight_min": -120.14949798583984,
      "activations/layer5_attention_weight_max": 57.39630889892578,
      "activations/layer5_attention_weight_min": -61.53469467163086,
      "activations/layer6_attention_weight_max": 47.91474914550781,
      "activations/layer6_attention_weight_min": -52.76912307739258,
      "activations/layer7_attention_weight_max": 88.45929718017578,
      "activations/layer7_attention_weight_min": -86.9087905883789,
      "activations/layer8_attention_weight_max": 40.47001266479492,
      "activations/layer8_attention_weight_min": -41.22258758544922,
      "activations/layer9_attention_weight_max": 33.69602584838867,
      "activations/layer9_attention_weight_min": -34.457088470458984,
      "epoch": 7.95,
      "learning_rate": 9.976174242424241e-05,
      "loss": 2.8039,
      "step": 136750
    },
    {
      "activations/layer0_attention_weight_max": 15.499778747558594,
      "activations/layer0_attention_weight_min": -10.672639846801758,
      "activations/layer10_attention_weight_max": 30.602108001708984,
      "activations/layer10_attention_weight_min": -29.360807418823242,
      "activations/layer11_attention_weight_max": 33.38549041748047,
      "activations/layer11_attention_weight_min": -31.096736907958984,
      "activations/layer12_attention_weight_max": 20.532438278198242,
      "activations/layer12_attention_weight_min": -25.82347869873047,
      "activations/layer13_attention_weight_max": 35.037811279296875,
      "activations/layer13_attention_weight_min": -33.401893615722656,
      "activations/layer14_attention_weight_max": 44.85921096801758,
      "activations/layer14_attention_weight_min": -42.18954086303711,
      "activations/layer15_attention_weight_max": 32.8448371887207,
      "activations/layer15_attention_weight_min": -31.283674240112305,
      "activations/layer16_attention_weight_max": 32.489532470703125,
      "activations/layer16_attention_weight_min": -26.908084869384766,
      "activations/layer17_attention_weight_max": 30.83761215209961,
      "activations/layer17_attention_weight_min": -25.674081802368164,
      "activations/layer18_attention_weight_max": 28.206340789794922,
      "activations/layer18_attention_weight_min": -21.815853118896484,
      "activations/layer19_attention_weight_max": 35.96525955200195,
      "activations/layer19_attention_weight_min": -29.498632431030273,
      "activations/layer1_attention_weight_max": 16.959026336669922,
      "activations/layer1_attention_weight_min": -14.0880708694458,
      "activations/layer20_attention_weight_max": 30.8211669921875,
      "activations/layer20_attention_weight_min": -24.08098030090332,
      "activations/layer21_attention_weight_max": 30.008331298828125,
      "activations/layer21_attention_weight_min": -21.91991424560547,
      "activations/layer22_attention_weight_max": 44.52959060668945,
      "activations/layer22_attention_weight_min": -26.828536987304688,
      "activations/layer23_attention_weight_max": 35.74986267089844,
      "activations/layer23_attention_weight_min": -22.09810447692871,
      "activations/layer2_attention_weight_max": 30.787555694580078,
      "activations/layer2_attention_weight_min": -31.526813507080078,
      "activations/layer3_attention_weight_max": 87.14840698242188,
      "activations/layer3_attention_weight_min": -88.47756958007812,
      "activations/layer4_attention_weight_max": 113.15669250488281,
      "activations/layer4_attention_weight_min": -114.39217376708984,
      "activations/layer5_attention_weight_max": 56.376197814941406,
      "activations/layer5_attention_weight_min": -64.34785461425781,
      "activations/layer6_attention_weight_max": 48.633541107177734,
      "activations/layer6_attention_weight_min": -49.81596374511719,
      "activations/layer7_attention_weight_max": 86.36608123779297,
      "activations/layer7_attention_weight_min": -87.57432556152344,
      "activations/layer8_attention_weight_max": 39.32170867919922,
      "activations/layer8_attention_weight_min": -37.370670318603516,
      "activations/layer9_attention_weight_max": 32.4268913269043,
      "activations/layer9_attention_weight_min": -31.183103561401367,
      "epoch": 7.95,
      "learning_rate": 9.974280303030302e-05,
      "loss": 2.8057,
      "step": 136800
    },
    {
      "activations/layer0_attention_weight_max": 14.964125633239746,
      "activations/layer0_attention_weight_min": -11.085887908935547,
      "activations/layer10_attention_weight_max": 33.236305236816406,
      "activations/layer10_attention_weight_min": -30.311758041381836,
      "activations/layer11_attention_weight_max": 32.34219741821289,
      "activations/layer11_attention_weight_min": -28.67329216003418,
      "activations/layer12_attention_weight_max": 23.956281661987305,
      "activations/layer12_attention_weight_min": -30.785964965820312,
      "activations/layer13_attention_weight_max": 41.34703063964844,
      "activations/layer13_attention_weight_min": -33.503353118896484,
      "activations/layer14_attention_weight_max": 45.79988479614258,
      "activations/layer14_attention_weight_min": -36.73252487182617,
      "activations/layer15_attention_weight_max": 34.13497543334961,
      "activations/layer15_attention_weight_min": -29.06162452697754,
      "activations/layer16_attention_weight_max": 31.140169143676758,
      "activations/layer16_attention_weight_min": -26.323362350463867,
      "activations/layer17_attention_weight_max": 33.04520797729492,
      "activations/layer17_attention_weight_min": -23.592004776000977,
      "activations/layer18_attention_weight_max": 30.865520477294922,
      "activations/layer18_attention_weight_min": -19.418581008911133,
      "activations/layer19_attention_weight_max": 35.77672576904297,
      "activations/layer19_attention_weight_min": -28.089920043945312,
      "activations/layer1_attention_weight_max": 17.93490982055664,
      "activations/layer1_attention_weight_min": -14.956727027893066,
      "activations/layer20_attention_weight_max": 31.614990234375,
      "activations/layer20_attention_weight_min": -23.764808654785156,
      "activations/layer21_attention_weight_max": 32.96688461303711,
      "activations/layer21_attention_weight_min": -21.894643783569336,
      "activations/layer22_attention_weight_max": 48.88895797729492,
      "activations/layer22_attention_weight_min": -29.877580642700195,
      "activations/layer23_attention_weight_max": 36.358741760253906,
      "activations/layer23_attention_weight_min": -25.987831115722656,
      "activations/layer2_attention_weight_max": 31.21841049194336,
      "activations/layer2_attention_weight_min": -30.84603500366211,
      "activations/layer3_attention_weight_max": 93.31791687011719,
      "activations/layer3_attention_weight_min": -91.69324493408203,
      "activations/layer4_attention_weight_max": 119.50394439697266,
      "activations/layer4_attention_weight_min": -117.08231353759766,
      "activations/layer5_attention_weight_max": 57.567955017089844,
      "activations/layer5_attention_weight_min": -61.95641326904297,
      "activations/layer6_attention_weight_max": 49.73762893676758,
      "activations/layer6_attention_weight_min": -51.790138244628906,
      "activations/layer7_attention_weight_max": 89.73129272460938,
      "activations/layer7_attention_weight_min": -83.88969421386719,
      "activations/layer8_attention_weight_max": 42.70064163208008,
      "activations/layer8_attention_weight_min": -42.062747955322266,
      "activations/layer9_attention_weight_max": 35.784786224365234,
      "activations/layer9_attention_weight_min": -34.59159469604492,
      "epoch": 7.95,
      "learning_rate": 9.972386363636363e-05,
      "loss": 2.8124,
      "step": 136850
    },
    {
      "activations/layer0_attention_weight_max": 15.171525955200195,
      "activations/layer0_attention_weight_min": -12.569778442382812,
      "activations/layer10_attention_weight_max": 33.40305709838867,
      "activations/layer10_attention_weight_min": -30.79500961303711,
      "activations/layer11_attention_weight_max": 29.230321884155273,
      "activations/layer11_attention_weight_min": -29.681974411010742,
      "activations/layer12_attention_weight_max": 21.914276123046875,
      "activations/layer12_attention_weight_min": -26.390419006347656,
      "activations/layer13_attention_weight_max": 52.007606506347656,
      "activations/layer13_attention_weight_min": -33.78016662597656,
      "activations/layer14_attention_weight_max": 51.817420959472656,
      "activations/layer14_attention_weight_min": -39.097564697265625,
      "activations/layer15_attention_weight_max": 39.88467025756836,
      "activations/layer15_attention_weight_min": -29.400056838989258,
      "activations/layer16_attention_weight_max": 40.82326126098633,
      "activations/layer16_attention_weight_min": -29.12601661682129,
      "activations/layer17_attention_weight_max": 35.232242584228516,
      "activations/layer17_attention_weight_min": -24.391326904296875,
      "activations/layer18_attention_weight_max": 29.801889419555664,
      "activations/layer18_attention_weight_min": -19.948699951171875,
      "activations/layer19_attention_weight_max": 49.04659652709961,
      "activations/layer19_attention_weight_min": -27.979516983032227,
      "activations/layer1_attention_weight_max": 16.72691535949707,
      "activations/layer1_attention_weight_min": -13.731738090515137,
      "activations/layer20_attention_weight_max": 35.94207000732422,
      "activations/layer20_attention_weight_min": -23.97566795349121,
      "activations/layer21_attention_weight_max": 33.523128509521484,
      "activations/layer21_attention_weight_min": -20.842741012573242,
      "activations/layer22_attention_weight_max": 52.97677993774414,
      "activations/layer22_attention_weight_min": -27.417057037353516,
      "activations/layer23_attention_weight_max": 39.32395935058594,
      "activations/layer23_attention_weight_min": -20.984270095825195,
      "activations/layer2_attention_weight_max": 31.444974899291992,
      "activations/layer2_attention_weight_min": -32.79531478881836,
      "activations/layer3_attention_weight_max": 91.98663330078125,
      "activations/layer3_attention_weight_min": -99.88014221191406,
      "activations/layer4_attention_weight_max": 119.6513442993164,
      "activations/layer4_attention_weight_min": -120.114501953125,
      "activations/layer5_attention_weight_max": 59.451316833496094,
      "activations/layer5_attention_weight_min": -63.5752067565918,
      "activations/layer6_attention_weight_max": 48.62112808227539,
      "activations/layer6_attention_weight_min": -52.519508361816406,
      "activations/layer7_attention_weight_max": 91.63672637939453,
      "activations/layer7_attention_weight_min": -90.0559310913086,
      "activations/layer8_attention_weight_max": 41.536468505859375,
      "activations/layer8_attention_weight_min": -41.760772705078125,
      "activations/layer9_attention_weight_max": 37.338626861572266,
      "activations/layer9_attention_weight_min": -36.34149932861328,
      "epoch": 7.95,
      "learning_rate": 9.970492424242425e-05,
      "loss": 2.8119,
      "step": 136900
    },
    {
      "activations/layer0_attention_weight_max": 14.984708786010742,
      "activations/layer0_attention_weight_min": -12.99941635131836,
      "activations/layer10_attention_weight_max": 34.97917938232422,
      "activations/layer10_attention_weight_min": -31.558691024780273,
      "activations/layer11_attention_weight_max": 35.81047058105469,
      "activations/layer11_attention_weight_min": -30.30211639404297,
      "activations/layer12_attention_weight_max": 26.093280792236328,
      "activations/layer12_attention_weight_min": -25.744050979614258,
      "activations/layer13_attention_weight_max": 42.228084564208984,
      "activations/layer13_attention_weight_min": -32.71783447265625,
      "activations/layer14_attention_weight_max": 50.84896469116211,
      "activations/layer14_attention_weight_min": -39.585628509521484,
      "activations/layer15_attention_weight_max": 35.634605407714844,
      "activations/layer15_attention_weight_min": -29.26618766784668,
      "activations/layer16_attention_weight_max": 33.775875091552734,
      "activations/layer16_attention_weight_min": -27.404556274414062,
      "activations/layer17_attention_weight_max": 35.57459259033203,
      "activations/layer17_attention_weight_min": -24.28340721130371,
      "activations/layer18_attention_weight_max": 31.3714599609375,
      "activations/layer18_attention_weight_min": -22.126882553100586,
      "activations/layer19_attention_weight_max": 36.0425910949707,
      "activations/layer19_attention_weight_min": -29.58601951599121,
      "activations/layer1_attention_weight_max": 17.158580780029297,
      "activations/layer1_attention_weight_min": -15.300697326660156,
      "activations/layer20_attention_weight_max": 33.74152374267578,
      "activations/layer20_attention_weight_min": -23.806814193725586,
      "activations/layer21_attention_weight_max": 32.775447845458984,
      "activations/layer21_attention_weight_min": -23.628568649291992,
      "activations/layer22_attention_weight_max": 50.09552764892578,
      "activations/layer22_attention_weight_min": -29.70153045654297,
      "activations/layer23_attention_weight_max": 34.38201141357422,
      "activations/layer23_attention_weight_min": -26.693565368652344,
      "activations/layer2_attention_weight_max": 33.478965759277344,
      "activations/layer2_attention_weight_min": -32.47563934326172,
      "activations/layer3_attention_weight_max": 93.60931396484375,
      "activations/layer3_attention_weight_min": -97.32604217529297,
      "activations/layer4_attention_weight_max": 125.94770812988281,
      "activations/layer4_attention_weight_min": -121.5659408569336,
      "activations/layer5_attention_weight_max": 58.604087829589844,
      "activations/layer5_attention_weight_min": -64.51094055175781,
      "activations/layer6_attention_weight_max": 49.292293548583984,
      "activations/layer6_attention_weight_min": -50.550376892089844,
      "activations/layer7_attention_weight_max": 89.0857925415039,
      "activations/layer7_attention_weight_min": -87.23281860351562,
      "activations/layer8_attention_weight_max": 41.39384841918945,
      "activations/layer8_attention_weight_min": -41.771724700927734,
      "activations/layer9_attention_weight_max": 36.69535446166992,
      "activations/layer9_attention_weight_min": -36.614715576171875,
      "epoch": 7.96,
      "learning_rate": 9.968598484848483e-05,
      "loss": 2.8179,
      "step": 136950
    },
    {
      "activations/layer0_attention_weight_max": 15.644859313964844,
      "activations/layer0_attention_weight_min": -12.881864547729492,
      "activations/layer10_attention_weight_max": 38.527374267578125,
      "activations/layer10_attention_weight_min": -35.2427978515625,
      "activations/layer11_attention_weight_max": 38.2900390625,
      "activations/layer11_attention_weight_min": -33.293331146240234,
      "activations/layer12_attention_weight_max": 28.34261703491211,
      "activations/layer12_attention_weight_min": -33.98405838012695,
      "activations/layer13_attention_weight_max": 46.22879409790039,
      "activations/layer13_attention_weight_min": -34.881446838378906,
      "activations/layer14_attention_weight_max": 47.23505783081055,
      "activations/layer14_attention_weight_min": -39.297828674316406,
      "activations/layer15_attention_weight_max": 42.039337158203125,
      "activations/layer15_attention_weight_min": -30.857223510742188,
      "activations/layer16_attention_weight_max": 32.93423843383789,
      "activations/layer16_attention_weight_min": -28.874773025512695,
      "activations/layer17_attention_weight_max": 32.18149185180664,
      "activations/layer17_attention_weight_min": -25.0268611907959,
      "activations/layer18_attention_weight_max": 31.96484375,
      "activations/layer18_attention_weight_min": -23.491535186767578,
      "activations/layer19_attention_weight_max": 33.79378128051758,
      "activations/layer19_attention_weight_min": -31.999942779541016,
      "activations/layer1_attention_weight_max": 16.381887435913086,
      "activations/layer1_attention_weight_min": -15.446779251098633,
      "activations/layer20_attention_weight_max": 28.332378387451172,
      "activations/layer20_attention_weight_min": -24.2269287109375,
      "activations/layer21_attention_weight_max": 28.693105697631836,
      "activations/layer21_attention_weight_min": -22.0632381439209,
      "activations/layer22_attention_weight_max": 47.00151824951172,
      "activations/layer22_attention_weight_min": -30.987058639526367,
      "activations/layer23_attention_weight_max": 36.732696533203125,
      "activations/layer23_attention_weight_min": -26.00600814819336,
      "activations/layer2_attention_weight_max": 32.70976257324219,
      "activations/layer2_attention_weight_min": -33.03984451293945,
      "activations/layer3_attention_weight_max": 99.04139709472656,
      "activations/layer3_attention_weight_min": -96.05249786376953,
      "activations/layer4_attention_weight_max": 129.20956420898438,
      "activations/layer4_attention_weight_min": -124.54957580566406,
      "activations/layer5_attention_weight_max": 60.542938232421875,
      "activations/layer5_attention_weight_min": -58.77029037475586,
      "activations/layer6_attention_weight_max": 51.2594108581543,
      "activations/layer6_attention_weight_min": -50.75619888305664,
      "activations/layer7_attention_weight_max": 89.18975067138672,
      "activations/layer7_attention_weight_min": -90.94146728515625,
      "activations/layer8_attention_weight_max": 42.38376235961914,
      "activations/layer8_attention_weight_min": -39.957061767578125,
      "activations/layer9_attention_weight_max": 38.27534484863281,
      "activations/layer9_attention_weight_min": -36.08000183105469,
      "epoch": 7.96,
      "learning_rate": 9.966704545454545e-05,
      "loss": 2.817,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5072,
      "eval_samples_per_second": 504.751,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5072,
      "eval_openwebtext_samples_per_second": 504.751,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 2.0178,
      "eval_wikitext_samples_per_second": 225.984,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_lambada_loss": 2.51171875,
      "eval_lambada_ppl": 12.326097344475713,
      "eval_lambada_runtime": 9.6009,
      "eval_lambada_samples_per_second": 507.14,
      "step": 137000
    },
    {
      "activations/layer0_attention_weight_max": 15.622153282165527,
      "activations/layer0_attention_weight_min": -12.410501480102539,
      "activations/layer10_attention_weight_max": 31.812780380249023,
      "activations/layer10_attention_weight_min": -33.0272216796875,
      "activations/layer11_attention_weight_max": 30.407245635986328,
      "activations/layer11_attention_weight_min": -31.20619773864746,
      "activations/layer12_attention_weight_max": 30.61972427368164,
      "activations/layer12_attention_weight_min": -29.751859664916992,
      "activations/layer13_attention_weight_max": 39.890846252441406,
      "activations/layer13_attention_weight_min": -33.628055572509766,
      "activations/layer14_attention_weight_max": 53.34803009033203,
      "activations/layer14_attention_weight_min": -39.048152923583984,
      "activations/layer15_attention_weight_max": 36.551361083984375,
      "activations/layer15_attention_weight_min": -29.82826805114746,
      "activations/layer16_attention_weight_max": 36.01370620727539,
      "activations/layer16_attention_weight_min": -26.44414520263672,
      "activations/layer17_attention_weight_max": 32.31562423706055,
      "activations/layer17_attention_weight_min": -24.526330947875977,
      "activations/layer18_attention_weight_max": 29.43804359436035,
      "activations/layer18_attention_weight_min": -21.612459182739258,
      "activations/layer19_attention_weight_max": 35.576698303222656,
      "activations/layer19_attention_weight_min": -29.585391998291016,
      "activations/layer1_attention_weight_max": 16.397859573364258,
      "activations/layer1_attention_weight_min": -14.714329719543457,
      "activations/layer20_attention_weight_max": 29.641895294189453,
      "activations/layer20_attention_weight_min": -24.26519012451172,
      "activations/layer21_attention_weight_max": 29.126699447631836,
      "activations/layer21_attention_weight_min": -22.33590316772461,
      "activations/layer22_attention_weight_max": 51.28879165649414,
      "activations/layer22_attention_weight_min": -30.628469467163086,
      "activations/layer23_attention_weight_max": 35.85022735595703,
      "activations/layer23_attention_weight_min": -24.325353622436523,
      "activations/layer2_attention_weight_max": 35.031009674072266,
      "activations/layer2_attention_weight_min": -33.88654327392578,
      "activations/layer3_attention_weight_max": 95.31495666503906,
      "activations/layer3_attention_weight_min": -96.55048370361328,
      "activations/layer4_attention_weight_max": 122.9772720336914,
      "activations/layer4_attention_weight_min": -116.8549575805664,
      "activations/layer5_attention_weight_max": 61.433197021484375,
      "activations/layer5_attention_weight_min": -61.885520935058594,
      "activations/layer6_attention_weight_max": 50.24611282348633,
      "activations/layer6_attention_weight_min": -49.88019943237305,
      "activations/layer7_attention_weight_max": 85.57810974121094,
      "activations/layer7_attention_weight_min": -84.76126861572266,
      "activations/layer8_attention_weight_max": 39.28636932373047,
      "activations/layer8_attention_weight_min": -39.33021926879883,
      "activations/layer9_attention_weight_max": 33.69626998901367,
      "activations/layer9_attention_weight_min": -33.32711410522461,
      "epoch": 7.96,
      "learning_rate": 9.964810606060606e-05,
      "loss": 2.8136,
      "step": 137050
    },
    {
      "activations/layer0_attention_weight_max": 16.50439453125,
      "activations/layer0_attention_weight_min": -12.492923736572266,
      "activations/layer10_attention_weight_max": 31.256040573120117,
      "activations/layer10_attention_weight_min": -32.21141052246094,
      "activations/layer11_attention_weight_max": 34.35570526123047,
      "activations/layer11_attention_weight_min": -29.983137130737305,
      "activations/layer12_attention_weight_max": 32.08247375488281,
      "activations/layer12_attention_weight_min": -24.762290954589844,
      "activations/layer13_attention_weight_max": 39.850704193115234,
      "activations/layer13_attention_weight_min": -32.06734085083008,
      "activations/layer14_attention_weight_max": 47.42667770385742,
      "activations/layer14_attention_weight_min": -38.30002212524414,
      "activations/layer15_attention_weight_max": 34.89439010620117,
      "activations/layer15_attention_weight_min": -28.725004196166992,
      "activations/layer16_attention_weight_max": 36.69697952270508,
      "activations/layer16_attention_weight_min": -27.93299674987793,
      "activations/layer17_attention_weight_max": 32.68648147583008,
      "activations/layer17_attention_weight_min": -25.054325103759766,
      "activations/layer18_attention_weight_max": 32.63026428222656,
      "activations/layer18_attention_weight_min": -21.96201515197754,
      "activations/layer19_attention_weight_max": 37.865665435791016,
      "activations/layer19_attention_weight_min": -30.825407028198242,
      "activations/layer1_attention_weight_max": 17.710254669189453,
      "activations/layer1_attention_weight_min": -15.26728630065918,
      "activations/layer20_attention_weight_max": 34.84272003173828,
      "activations/layer20_attention_weight_min": -25.689470291137695,
      "activations/layer21_attention_weight_max": 33.50029373168945,
      "activations/layer21_attention_weight_min": -23.25240135192871,
      "activations/layer22_attention_weight_max": 48.5860710144043,
      "activations/layer22_attention_weight_min": -26.653751373291016,
      "activations/layer23_attention_weight_max": 37.650508880615234,
      "activations/layer23_attention_weight_min": -22.40236473083496,
      "activations/layer2_attention_weight_max": 31.32553482055664,
      "activations/layer2_attention_weight_min": -32.3579216003418,
      "activations/layer3_attention_weight_max": 94.20650482177734,
      "activations/layer3_attention_weight_min": -95.44322204589844,
      "activations/layer4_attention_weight_max": 113.6279067993164,
      "activations/layer4_attention_weight_min": -112.54945373535156,
      "activations/layer5_attention_weight_max": 55.8635368347168,
      "activations/layer5_attention_weight_min": -59.591121673583984,
      "activations/layer6_attention_weight_max": 47.74779510498047,
      "activations/layer6_attention_weight_min": -52.01633834838867,
      "activations/layer7_attention_weight_max": 89.05860137939453,
      "activations/layer7_attention_weight_min": -85.248779296875,
      "activations/layer8_attention_weight_max": 38.98109817504883,
      "activations/layer8_attention_weight_min": -38.24519348144531,
      "activations/layer9_attention_weight_max": 35.507164001464844,
      "activations/layer9_attention_weight_min": -34.45497512817383,
      "epoch": 7.97,
      "learning_rate": 9.962916666666665e-05,
      "loss": 2.8122,
      "step": 137100
    },
    {
      "activations/layer0_attention_weight_max": 16.34893226623535,
      "activations/layer0_attention_weight_min": -12.545767784118652,
      "activations/layer10_attention_weight_max": 32.4963264465332,
      "activations/layer10_attention_weight_min": -29.923568725585938,
      "activations/layer11_attention_weight_max": 31.747106552124023,
      "activations/layer11_attention_weight_min": -28.61353874206543,
      "activations/layer12_attention_weight_max": 22.37425422668457,
      "activations/layer12_attention_weight_min": -25.45742416381836,
      "activations/layer13_attention_weight_max": 38.020992279052734,
      "activations/layer13_attention_weight_min": -35.51251983642578,
      "activations/layer14_attention_weight_max": 45.1292610168457,
      "activations/layer14_attention_weight_min": -38.41543960571289,
      "activations/layer15_attention_weight_max": 36.447261810302734,
      "activations/layer15_attention_weight_min": -30.646865844726562,
      "activations/layer16_attention_weight_max": 32.34553146362305,
      "activations/layer16_attention_weight_min": -28.845481872558594,
      "activations/layer17_attention_weight_max": 34.120235443115234,
      "activations/layer17_attention_weight_min": -26.17120361328125,
      "activations/layer18_attention_weight_max": 33.814544677734375,
      "activations/layer18_attention_weight_min": -20.76858139038086,
      "activations/layer19_attention_weight_max": 36.23347091674805,
      "activations/layer19_attention_weight_min": -31.01984977722168,
      "activations/layer1_attention_weight_max": 15.52186393737793,
      "activations/layer1_attention_weight_min": -15.142210960388184,
      "activations/layer20_attention_weight_max": 30.87044906616211,
      "activations/layer20_attention_weight_min": -23.247661590576172,
      "activations/layer21_attention_weight_max": 30.734952926635742,
      "activations/layer21_attention_weight_min": -22.11012840270996,
      "activations/layer22_attention_weight_max": 43.853389739990234,
      "activations/layer22_attention_weight_min": -28.515975952148438,
      "activations/layer23_attention_weight_max": 35.147857666015625,
      "activations/layer23_attention_weight_min": -24.139646530151367,
      "activations/layer2_attention_weight_max": 33.201290130615234,
      "activations/layer2_attention_weight_min": -33.013671875,
      "activations/layer3_attention_weight_max": 93.9922866821289,
      "activations/layer3_attention_weight_min": -92.30577850341797,
      "activations/layer4_attention_weight_max": 118.11732482910156,
      "activations/layer4_attention_weight_min": -114.5373306274414,
      "activations/layer5_attention_weight_max": 56.972225189208984,
      "activations/layer5_attention_weight_min": -62.081016540527344,
      "activations/layer6_attention_weight_max": 49.37246322631836,
      "activations/layer6_attention_weight_min": -52.139503479003906,
      "activations/layer7_attention_weight_max": 85.96815490722656,
      "activations/layer7_attention_weight_min": -86.53916931152344,
      "activations/layer8_attention_weight_max": 39.22023391723633,
      "activations/layer8_attention_weight_min": -39.81879806518555,
      "activations/layer9_attention_weight_max": 34.60319137573242,
      "activations/layer9_attention_weight_min": -34.13358688354492,
      "epoch": 7.97,
      "learning_rate": 9.961022727272727e-05,
      "loss": 2.8012,
      "step": 137150
    },
    {
      "activations/layer0_attention_weight_max": 15.143739700317383,
      "activations/layer0_attention_weight_min": -12.082626342773438,
      "activations/layer10_attention_weight_max": 39.35735321044922,
      "activations/layer10_attention_weight_min": -36.09064483642578,
      "activations/layer11_attention_weight_max": 39.565635681152344,
      "activations/layer11_attention_weight_min": -31.94943618774414,
      "activations/layer12_attention_weight_max": 42.69514846801758,
      "activations/layer12_attention_weight_min": -27.40806007385254,
      "activations/layer13_attention_weight_max": 55.01155471801758,
      "activations/layer13_attention_weight_min": -35.00746154785156,
      "activations/layer14_attention_weight_max": 50.80304718017578,
      "activations/layer14_attention_weight_min": -39.92491912841797,
      "activations/layer15_attention_weight_max": 44.671409606933594,
      "activations/layer15_attention_weight_min": -33.5262336730957,
      "activations/layer16_attention_weight_max": 33.73617935180664,
      "activations/layer16_attention_weight_min": -28.424713134765625,
      "activations/layer17_attention_weight_max": 32.136722564697266,
      "activations/layer17_attention_weight_min": -25.568195343017578,
      "activations/layer18_attention_weight_max": 31.600053787231445,
      "activations/layer18_attention_weight_min": -23.01948356628418,
      "activations/layer19_attention_weight_max": 34.62821578979492,
      "activations/layer19_attention_weight_min": -29.531320571899414,
      "activations/layer1_attention_weight_max": 17.13089942932129,
      "activations/layer1_attention_weight_min": -13.520845413208008,
      "activations/layer20_attention_weight_max": 29.6728458404541,
      "activations/layer20_attention_weight_min": -24.651456832885742,
      "activations/layer21_attention_weight_max": 26.66813850402832,
      "activations/layer21_attention_weight_min": -22.515548706054688,
      "activations/layer22_attention_weight_max": 50.634639739990234,
      "activations/layer22_attention_weight_min": -29.4053955078125,
      "activations/layer23_attention_weight_max": 33.279449462890625,
      "activations/layer23_attention_weight_min": -25.503185272216797,
      "activations/layer2_attention_weight_max": 31.534385681152344,
      "activations/layer2_attention_weight_min": -31.416471481323242,
      "activations/layer3_attention_weight_max": 93.00897216796875,
      "activations/layer3_attention_weight_min": -93.64171600341797,
      "activations/layer4_attention_weight_max": 119.88330078125,
      "activations/layer4_attention_weight_min": -117.83746337890625,
      "activations/layer5_attention_weight_max": 60.188682556152344,
      "activations/layer5_attention_weight_min": -61.1795654296875,
      "activations/layer6_attention_weight_max": 54.11738204956055,
      "activations/layer6_attention_weight_min": -52.05536651611328,
      "activations/layer7_attention_weight_max": 98.97128295898438,
      "activations/layer7_attention_weight_min": -88.29243469238281,
      "activations/layer8_attention_weight_max": 46.30937576293945,
      "activations/layer8_attention_weight_min": -41.12358093261719,
      "activations/layer9_attention_weight_max": 38.24043655395508,
      "activations/layer9_attention_weight_min": -35.9921989440918,
      "epoch": 7.97,
      "learning_rate": 9.959128787878787e-05,
      "loss": 2.8077,
      "step": 137200
    },
    {
      "activations/layer0_attention_weight_max": 15.871464729309082,
      "activations/layer0_attention_weight_min": -13.169416427612305,
      "activations/layer10_attention_weight_max": 38.242244720458984,
      "activations/layer10_attention_weight_min": -33.48944091796875,
      "activations/layer11_attention_weight_max": 38.989234924316406,
      "activations/layer11_attention_weight_min": -32.582923889160156,
      "activations/layer12_attention_weight_max": 33.93385314941406,
      "activations/layer12_attention_weight_min": -30.303831100463867,
      "activations/layer13_attention_weight_max": 38.11336135864258,
      "activations/layer13_attention_weight_min": -32.51860809326172,
      "activations/layer14_attention_weight_max": 47.7524528503418,
      "activations/layer14_attention_weight_min": -38.5947151184082,
      "activations/layer15_attention_weight_max": 36.68364334106445,
      "activations/layer15_attention_weight_min": -27.987215042114258,
      "activations/layer16_attention_weight_max": 35.511802673339844,
      "activations/layer16_attention_weight_min": -27.779865264892578,
      "activations/layer17_attention_weight_max": 32.404640197753906,
      "activations/layer17_attention_weight_min": -25.444490432739258,
      "activations/layer18_attention_weight_max": 34.53640365600586,
      "activations/layer18_attention_weight_min": -21.61815643310547,
      "activations/layer19_attention_weight_max": 41.78907012939453,
      "activations/layer19_attention_weight_min": -30.6104679107666,
      "activations/layer1_attention_weight_max": 18.02307891845703,
      "activations/layer1_attention_weight_min": -15.3453950881958,
      "activations/layer20_attention_weight_max": 33.43775177001953,
      "activations/layer20_attention_weight_min": -24.56093978881836,
      "activations/layer21_attention_weight_max": 30.180692672729492,
      "activations/layer21_attention_weight_min": -22.91630744934082,
      "activations/layer22_attention_weight_max": 46.85803985595703,
      "activations/layer22_attention_weight_min": -29.23568344116211,
      "activations/layer23_attention_weight_max": 35.69499969482422,
      "activations/layer23_attention_weight_min": -24.195592880249023,
      "activations/layer2_attention_weight_max": 35.12187576293945,
      "activations/layer2_attention_weight_min": -34.342628479003906,
      "activations/layer3_attention_weight_max": 104.9112777709961,
      "activations/layer3_attention_weight_min": -104.15795135498047,
      "activations/layer4_attention_weight_max": 127.89896392822266,
      "activations/layer4_attention_weight_min": -125.7269287109375,
      "activations/layer5_attention_weight_max": 62.392818450927734,
      "activations/layer5_attention_weight_min": -59.74726104736328,
      "activations/layer6_attention_weight_max": 48.765480041503906,
      "activations/layer6_attention_weight_min": -51.43510818481445,
      "activations/layer7_attention_weight_max": 97.61370849609375,
      "activations/layer7_attention_weight_min": -96.00424194335938,
      "activations/layer8_attention_weight_max": 44.8239860534668,
      "activations/layer8_attention_weight_min": -43.79783248901367,
      "activations/layer9_attention_weight_max": 40.761741638183594,
      "activations/layer9_attention_weight_min": -37.93696975708008,
      "epoch": 7.97,
      "learning_rate": 9.957234848484847e-05,
      "loss": 2.8001,
      "step": 137250
    },
    {
      "activations/layer0_attention_weight_max": 15.114363670349121,
      "activations/layer0_attention_weight_min": -11.910004615783691,
      "activations/layer10_attention_weight_max": 36.645355224609375,
      "activations/layer10_attention_weight_min": -30.80187225341797,
      "activations/layer11_attention_weight_max": 33.97964859008789,
      "activations/layer11_attention_weight_min": -29.06917953491211,
      "activations/layer12_attention_weight_max": 21.89338493347168,
      "activations/layer12_attention_weight_min": -25.369916915893555,
      "activations/layer13_attention_weight_max": 39.79217529296875,
      "activations/layer13_attention_weight_min": -32.92424774169922,
      "activations/layer14_attention_weight_max": 48.186275482177734,
      "activations/layer14_attention_weight_min": -39.89326858520508,
      "activations/layer15_attention_weight_max": 39.761932373046875,
      "activations/layer15_attention_weight_min": -29.10740089416504,
      "activations/layer16_attention_weight_max": 34.9224853515625,
      "activations/layer16_attention_weight_min": -28.989974975585938,
      "activations/layer17_attention_weight_max": 33.38146209716797,
      "activations/layer17_attention_weight_min": -26.238250732421875,
      "activations/layer18_attention_weight_max": 32.365108489990234,
      "activations/layer18_attention_weight_min": -21.69364356994629,
      "activations/layer19_attention_weight_max": 34.90858459472656,
      "activations/layer19_attention_weight_min": -30.646163940429688,
      "activations/layer1_attention_weight_max": 16.718786239624023,
      "activations/layer1_attention_weight_min": -15.367518424987793,
      "activations/layer20_attention_weight_max": 29.740184783935547,
      "activations/layer20_attention_weight_min": -24.21739387512207,
      "activations/layer21_attention_weight_max": 33.83592987060547,
      "activations/layer21_attention_weight_min": -22.536279678344727,
      "activations/layer22_attention_weight_max": 56.18816375732422,
      "activations/layer22_attention_weight_min": -30.950342178344727,
      "activations/layer23_attention_weight_max": 34.03363800048828,
      "activations/layer23_attention_weight_min": -25.565471649169922,
      "activations/layer2_attention_weight_max": 33.039344787597656,
      "activations/layer2_attention_weight_min": -31.447723388671875,
      "activations/layer3_attention_weight_max": 94.86302947998047,
      "activations/layer3_attention_weight_min": -93.19828033447266,
      "activations/layer4_attention_weight_max": 122.46192932128906,
      "activations/layer4_attention_weight_min": -115.87544250488281,
      "activations/layer5_attention_weight_max": 60.74066925048828,
      "activations/layer5_attention_weight_min": -64.12472534179688,
      "activations/layer6_attention_weight_max": 50.589176177978516,
      "activations/layer6_attention_weight_min": -50.215553283691406,
      "activations/layer7_attention_weight_max": 90.64494323730469,
      "activations/layer7_attention_weight_min": -86.68981170654297,
      "activations/layer8_attention_weight_max": 42.19327926635742,
      "activations/layer8_attention_weight_min": -42.159610748291016,
      "activations/layer9_attention_weight_max": 37.42588424682617,
      "activations/layer9_attention_weight_min": -35.026123046875,
      "epoch": 7.98,
      "learning_rate": 9.955340909090908e-05,
      "loss": 2.8095,
      "step": 137300
    },
    {
      "activations/layer0_attention_weight_max": 16.311752319335938,
      "activations/layer0_attention_weight_min": -12.45211410522461,
      "activations/layer10_attention_weight_max": 31.551921844482422,
      "activations/layer10_attention_weight_min": -30.37174415588379,
      "activations/layer11_attention_weight_max": 33.69017028808594,
      "activations/layer11_attention_weight_min": -29.88697624206543,
      "activations/layer12_attention_weight_max": 24.583999633789062,
      "activations/layer12_attention_weight_min": -28.11724281311035,
      "activations/layer13_attention_weight_max": 39.02513885498047,
      "activations/layer13_attention_weight_min": -32.48292541503906,
      "activations/layer14_attention_weight_max": 44.160804748535156,
      "activations/layer14_attention_weight_min": -36.18918991088867,
      "activations/layer15_attention_weight_max": 39.968196868896484,
      "activations/layer15_attention_weight_min": -29.934844970703125,
      "activations/layer16_attention_weight_max": 37.589942932128906,
      "activations/layer16_attention_weight_min": -26.59181785583496,
      "activations/layer17_attention_weight_max": 35.11394119262695,
      "activations/layer17_attention_weight_min": -28.102725982666016,
      "activations/layer18_attention_weight_max": 32.5341682434082,
      "activations/layer18_attention_weight_min": -22.317781448364258,
      "activations/layer19_attention_weight_max": 40.15420150756836,
      "activations/layer19_attention_weight_min": -32.7147102355957,
      "activations/layer1_attention_weight_max": 17.47228240966797,
      "activations/layer1_attention_weight_min": -14.376255989074707,
      "activations/layer20_attention_weight_max": 33.2874641418457,
      "activations/layer20_attention_weight_min": -27.510459899902344,
      "activations/layer21_attention_weight_max": 33.5565299987793,
      "activations/layer21_attention_weight_min": -25.61009979248047,
      "activations/layer22_attention_weight_max": 49.64834976196289,
      "activations/layer22_attention_weight_min": -30.136825561523438,
      "activations/layer23_attention_weight_max": 37.7678337097168,
      "activations/layer23_attention_weight_min": -24.436826705932617,
      "activations/layer2_attention_weight_max": 34.029300689697266,
      "activations/layer2_attention_weight_min": -31.693607330322266,
      "activations/layer3_attention_weight_max": 94.75923919677734,
      "activations/layer3_attention_weight_min": -94.84403991699219,
      "activations/layer4_attention_weight_max": 119.24996185302734,
      "activations/layer4_attention_weight_min": -117.4498062133789,
      "activations/layer5_attention_weight_max": 61.39844512939453,
      "activations/layer5_attention_weight_min": -61.189083099365234,
      "activations/layer6_attention_weight_max": 52.688716888427734,
      "activations/layer6_attention_weight_min": -53.71971893310547,
      "activations/layer7_attention_weight_max": 89.27777099609375,
      "activations/layer7_attention_weight_min": -88.02906036376953,
      "activations/layer8_attention_weight_max": 40.87904739379883,
      "activations/layer8_attention_weight_min": -40.16851806640625,
      "activations/layer9_attention_weight_max": 33.68659591674805,
      "activations/layer9_attention_weight_min": -34.363704681396484,
      "epoch": 7.98,
      "learning_rate": 9.953446969696969e-05,
      "loss": 2.818,
      "step": 137350
    },
    {
      "activations/layer0_attention_weight_max": 15.922286033630371,
      "activations/layer0_attention_weight_min": -13.370189666748047,
      "activations/layer10_attention_weight_max": 31.50792694091797,
      "activations/layer10_attention_weight_min": -30.79629135131836,
      "activations/layer11_attention_weight_max": 30.90445327758789,
      "activations/layer11_attention_weight_min": -29.344160079956055,
      "activations/layer12_attention_weight_max": 21.728565216064453,
      "activations/layer12_attention_weight_min": -27.052045822143555,
      "activations/layer13_attention_weight_max": 37.53251266479492,
      "activations/layer13_attention_weight_min": -32.37366485595703,
      "activations/layer14_attention_weight_max": 53.32014465332031,
      "activations/layer14_attention_weight_min": -43.35399627685547,
      "activations/layer15_attention_weight_max": 36.52751541137695,
      "activations/layer15_attention_weight_min": -30.81419563293457,
      "activations/layer16_attention_weight_max": 32.97372055053711,
      "activations/layer16_attention_weight_min": -28.400514602661133,
      "activations/layer17_attention_weight_max": 34.17874526977539,
      "activations/layer17_attention_weight_min": -24.474197387695312,
      "activations/layer18_attention_weight_max": 34.83521270751953,
      "activations/layer18_attention_weight_min": -24.147003173828125,
      "activations/layer19_attention_weight_max": 36.674495697021484,
      "activations/layer19_attention_weight_min": -29.29129409790039,
      "activations/layer1_attention_weight_max": 16.286888122558594,
      "activations/layer1_attention_weight_min": -14.579426765441895,
      "activations/layer20_attention_weight_max": 33.35895538330078,
      "activations/layer20_attention_weight_min": -24.453285217285156,
      "activations/layer21_attention_weight_max": 31.28286361694336,
      "activations/layer21_attention_weight_min": -23.120891571044922,
      "activations/layer22_attention_weight_max": 46.569549560546875,
      "activations/layer22_attention_weight_min": -30.379907608032227,
      "activations/layer23_attention_weight_max": 35.69533920288086,
      "activations/layer23_attention_weight_min": -24.172338485717773,
      "activations/layer2_attention_weight_max": 32.72803497314453,
      "activations/layer2_attention_weight_min": -32.457763671875,
      "activations/layer3_attention_weight_max": 92.6946792602539,
      "activations/layer3_attention_weight_min": -94.34956359863281,
      "activations/layer4_attention_weight_max": 121.9018325805664,
      "activations/layer4_attention_weight_min": -118.76759338378906,
      "activations/layer5_attention_weight_max": 59.97971725463867,
      "activations/layer5_attention_weight_min": -59.71307373046875,
      "activations/layer6_attention_weight_max": 51.40388107299805,
      "activations/layer6_attention_weight_min": -51.07063674926758,
      "activations/layer7_attention_weight_max": 91.04322052001953,
      "activations/layer7_attention_weight_min": -87.3475570678711,
      "activations/layer8_attention_weight_max": 38.09687042236328,
      "activations/layer8_attention_weight_min": -39.138675689697266,
      "activations/layer9_attention_weight_max": 33.999080657958984,
      "activations/layer9_attention_weight_min": -33.2470588684082,
      "epoch": 7.98,
      "learning_rate": 9.95155303030303e-05,
      "loss": 2.8004,
      "step": 137400
    },
    {
      "activations/layer0_attention_weight_max": 15.789401054382324,
      "activations/layer0_attention_weight_min": -12.5997896194458,
      "activations/layer10_attention_weight_max": 34.35612487792969,
      "activations/layer10_attention_weight_min": -34.1044807434082,
      "activations/layer11_attention_weight_max": 33.275753021240234,
      "activations/layer11_attention_weight_min": -32.4279899597168,
      "activations/layer12_attention_weight_max": 22.34323501586914,
      "activations/layer12_attention_weight_min": -27.705228805541992,
      "activations/layer13_attention_weight_max": 50.93724822998047,
      "activations/layer13_attention_weight_min": -38.49082946777344,
      "activations/layer14_attention_weight_max": 51.964027404785156,
      "activations/layer14_attention_weight_min": -44.731292724609375,
      "activations/layer15_attention_weight_max": 37.86949920654297,
      "activations/layer15_attention_weight_min": -30.370161056518555,
      "activations/layer16_attention_weight_max": 32.77035903930664,
      "activations/layer16_attention_weight_min": -27.720212936401367,
      "activations/layer17_attention_weight_max": 37.00528335571289,
      "activations/layer17_attention_weight_min": -25.044944763183594,
      "activations/layer18_attention_weight_max": 33.1771240234375,
      "activations/layer18_attention_weight_min": -21.15430450439453,
      "activations/layer19_attention_weight_max": 35.56954574584961,
      "activations/layer19_attention_weight_min": -30.355411529541016,
      "activations/layer1_attention_weight_max": 17.906023025512695,
      "activations/layer1_attention_weight_min": -19.246213912963867,
      "activations/layer20_attention_weight_max": 30.707656860351562,
      "activations/layer20_attention_weight_min": -28.009387969970703,
      "activations/layer21_attention_weight_max": 29.527860641479492,
      "activations/layer21_attention_weight_min": -24.13814353942871,
      "activations/layer22_attention_weight_max": 47.884464263916016,
      "activations/layer22_attention_weight_min": -29.735965728759766,
      "activations/layer23_attention_weight_max": 38.014923095703125,
      "activations/layer23_attention_weight_min": -24.63288116455078,
      "activations/layer2_attention_weight_max": 33.18849182128906,
      "activations/layer2_attention_weight_min": -32.74632263183594,
      "activations/layer3_attention_weight_max": 93.69779205322266,
      "activations/layer3_attention_weight_min": -96.38629913330078,
      "activations/layer4_attention_weight_max": 123.44222259521484,
      "activations/layer4_attention_weight_min": -128.09068298339844,
      "activations/layer5_attention_weight_max": 59.928199768066406,
      "activations/layer5_attention_weight_min": -63.55028533935547,
      "activations/layer6_attention_weight_max": 49.61952209472656,
      "activations/layer6_attention_weight_min": -51.7450065612793,
      "activations/layer7_attention_weight_max": 99.49022674560547,
      "activations/layer7_attention_weight_min": -92.88907623291016,
      "activations/layer8_attention_weight_max": 44.18749237060547,
      "activations/layer8_attention_weight_min": -44.37887954711914,
      "activations/layer9_attention_weight_max": 36.29467010498047,
      "activations/layer9_attention_weight_min": -36.125972747802734,
      "epoch": 7.99,
      "learning_rate": 9.94965909090909e-05,
      "loss": 2.8253,
      "step": 137450
    },
    {
      "activations/layer0_attention_weight_max": 16.113479614257812,
      "activations/layer0_attention_weight_min": -12.619987487792969,
      "activations/layer10_attention_weight_max": 34.167869567871094,
      "activations/layer10_attention_weight_min": -35.12647247314453,
      "activations/layer11_attention_weight_max": 32.47572326660156,
      "activations/layer11_attention_weight_min": -32.93171691894531,
      "activations/layer12_attention_weight_max": 25.053253173828125,
      "activations/layer12_attention_weight_min": -26.377286911010742,
      "activations/layer13_attention_weight_max": 36.16801452636719,
      "activations/layer13_attention_weight_min": -33.11206817626953,
      "activations/layer14_attention_weight_max": 50.476280212402344,
      "activations/layer14_attention_weight_min": -44.83790588378906,
      "activations/layer15_attention_weight_max": 34.92650604248047,
      "activations/layer15_attention_weight_min": -31.90852165222168,
      "activations/layer16_attention_weight_max": 29.339685440063477,
      "activations/layer16_attention_weight_min": -28.56479835510254,
      "activations/layer17_attention_weight_max": 33.246543884277344,
      "activations/layer17_attention_weight_min": -26.160198211669922,
      "activations/layer18_attention_weight_max": 32.016483306884766,
      "activations/layer18_attention_weight_min": -22.9891357421875,
      "activations/layer19_attention_weight_max": 35.01146697998047,
      "activations/layer19_attention_weight_min": -32.14592361450195,
      "activations/layer1_attention_weight_max": 16.452421188354492,
      "activations/layer1_attention_weight_min": -15.853867530822754,
      "activations/layer20_attention_weight_max": 30.121746063232422,
      "activations/layer20_attention_weight_min": -22.74536895751953,
      "activations/layer21_attention_weight_max": 30.246084213256836,
      "activations/layer21_attention_weight_min": -22.954336166381836,
      "activations/layer22_attention_weight_max": 45.17266845703125,
      "activations/layer22_attention_weight_min": -29.409740447998047,
      "activations/layer23_attention_weight_max": 33.10734176635742,
      "activations/layer23_attention_weight_min": -22.875402450561523,
      "activations/layer2_attention_weight_max": 33.176551818847656,
      "activations/layer2_attention_weight_min": -32.51408386230469,
      "activations/layer3_attention_weight_max": 94.90206146240234,
      "activations/layer3_attention_weight_min": -96.17237854003906,
      "activations/layer4_attention_weight_max": 121.81351470947266,
      "activations/layer4_attention_weight_min": -122.5248031616211,
      "activations/layer5_attention_weight_max": 60.127410888671875,
      "activations/layer5_attention_weight_min": -62.46575164794922,
      "activations/layer6_attention_weight_max": 50.714683532714844,
      "activations/layer6_attention_weight_min": -53.4233512878418,
      "activations/layer7_attention_weight_max": 92.29972076416016,
      "activations/layer7_attention_weight_min": -88.24937438964844,
      "activations/layer8_attention_weight_max": 40.14278030395508,
      "activations/layer8_attention_weight_min": -41.41098403930664,
      "activations/layer9_attention_weight_max": 35.45826721191406,
      "activations/layer9_attention_weight_min": -35.165443420410156,
      "epoch": 7.99,
      "learning_rate": 9.94776515151515e-05,
      "loss": 2.7973,
      "step": 137500
    },
    {
      "activations/layer0_attention_weight_max": 16.525785446166992,
      "activations/layer0_attention_weight_min": -13.183876991271973,
      "activations/layer10_attention_weight_max": 32.20271301269531,
      "activations/layer10_attention_weight_min": -32.143978118896484,
      "activations/layer11_attention_weight_max": 31.026371002197266,
      "activations/layer11_attention_weight_min": -30.287948608398438,
      "activations/layer12_attention_weight_max": 23.38086700439453,
      "activations/layer12_attention_weight_min": -25.119144439697266,
      "activations/layer13_attention_weight_max": 40.54322814941406,
      "activations/layer13_attention_weight_min": -33.69816207885742,
      "activations/layer14_attention_weight_max": 53.70509719848633,
      "activations/layer14_attention_weight_min": -37.69001007080078,
      "activations/layer15_attention_weight_max": 39.556480407714844,
      "activations/layer15_attention_weight_min": -29.617563247680664,
      "activations/layer16_attention_weight_max": 34.765899658203125,
      "activations/layer16_attention_weight_min": -28.708934783935547,
      "activations/layer17_attention_weight_max": 37.720001220703125,
      "activations/layer17_attention_weight_min": -27.005779266357422,
      "activations/layer18_attention_weight_max": 38.9200553894043,
      "activations/layer18_attention_weight_min": -22.698619842529297,
      "activations/layer19_attention_weight_max": 40.294795989990234,
      "activations/layer19_attention_weight_min": -31.878162384033203,
      "activations/layer1_attention_weight_max": 16.560792922973633,
      "activations/layer1_attention_weight_min": -14.193687438964844,
      "activations/layer20_attention_weight_max": 33.385929107666016,
      "activations/layer20_attention_weight_min": -25.674413681030273,
      "activations/layer21_attention_weight_max": 35.427520751953125,
      "activations/layer21_attention_weight_min": -23.057153701782227,
      "activations/layer22_attention_weight_max": 50.74916076660156,
      "activations/layer22_attention_weight_min": -28.869293212890625,
      "activations/layer23_attention_weight_max": 38.750389099121094,
      "activations/layer23_attention_weight_min": -21.579341888427734,
      "activations/layer2_attention_weight_max": 33.254432678222656,
      "activations/layer2_attention_weight_min": -33.51988220214844,
      "activations/layer3_attention_weight_max": 95.68972778320312,
      "activations/layer3_attention_weight_min": -101.41706085205078,
      "activations/layer4_attention_weight_max": 122.1451187133789,
      "activations/layer4_attention_weight_min": -125.05763244628906,
      "activations/layer5_attention_weight_max": 56.191741943359375,
      "activations/layer5_attention_weight_min": -66.22223663330078,
      "activations/layer6_attention_weight_max": 50.98751449584961,
      "activations/layer6_attention_weight_min": -53.3253288269043,
      "activations/layer7_attention_weight_max": 90.13233947753906,
      "activations/layer7_attention_weight_min": -91.28729248046875,
      "activations/layer8_attention_weight_max": 39.76507568359375,
      "activations/layer8_attention_weight_min": -43.18144607543945,
      "activations/layer9_attention_weight_max": 33.14618682861328,
      "activations/layer9_attention_weight_min": -35.21323776245117,
      "epoch": 7.99,
      "learning_rate": 9.945871212121212e-05,
      "loss": 2.8241,
      "step": 137550
    },
    {
      "activations/layer0_attention_weight_max": 15.584081649780273,
      "activations/layer0_attention_weight_min": -12.244430541992188,
      "activations/layer10_attention_weight_max": 34.95116424560547,
      "activations/layer10_attention_weight_min": -32.1878776550293,
      "activations/layer11_attention_weight_max": 32.12670135498047,
      "activations/layer11_attention_weight_min": -32.1965217590332,
      "activations/layer12_attention_weight_max": 27.13086700439453,
      "activations/layer12_attention_weight_min": -28.50861167907715,
      "activations/layer13_attention_weight_max": 39.110328674316406,
      "activations/layer13_attention_weight_min": -31.770179748535156,
      "activations/layer14_attention_weight_max": 48.008663177490234,
      "activations/layer14_attention_weight_min": -36.14808654785156,
      "activations/layer15_attention_weight_max": 37.23371887207031,
      "activations/layer15_attention_weight_min": -31.447254180908203,
      "activations/layer16_attention_weight_max": 33.4515266418457,
      "activations/layer16_attention_weight_min": -30.009593963623047,
      "activations/layer17_attention_weight_max": 33.65389633178711,
      "activations/layer17_attention_weight_min": -25.402406692504883,
      "activations/layer18_attention_weight_max": 29.071205139160156,
      "activations/layer18_attention_weight_min": -22.12057113647461,
      "activations/layer19_attention_weight_max": 38.66140365600586,
      "activations/layer19_attention_weight_min": -30.446868896484375,
      "activations/layer1_attention_weight_max": 18.26742172241211,
      "activations/layer1_attention_weight_min": -16.00328254699707,
      "activations/layer20_attention_weight_max": 28.965015411376953,
      "activations/layer20_attention_weight_min": -25.364904403686523,
      "activations/layer21_attention_weight_max": 27.698551177978516,
      "activations/layer21_attention_weight_min": -21.763072967529297,
      "activations/layer22_attention_weight_max": 42.59976577758789,
      "activations/layer22_attention_weight_min": -30.066364288330078,
      "activations/layer23_attention_weight_max": 37.442195892333984,
      "activations/layer23_attention_weight_min": -24.64889907836914,
      "activations/layer2_attention_weight_max": 33.67857360839844,
      "activations/layer2_attention_weight_min": -34.60848617553711,
      "activations/layer3_attention_weight_max": 93.70359802246094,
      "activations/layer3_attention_weight_min": -99.63722229003906,
      "activations/layer4_attention_weight_max": 117.89900207519531,
      "activations/layer4_attention_weight_min": -119.56620025634766,
      "activations/layer5_attention_weight_max": 59.26225280761719,
      "activations/layer5_attention_weight_min": -61.10677719116211,
      "activations/layer6_attention_weight_max": 48.899635314941406,
      "activations/layer6_attention_weight_min": -52.72853088378906,
      "activations/layer7_attention_weight_max": 89.66123962402344,
      "activations/layer7_attention_weight_min": -90.87203979492188,
      "activations/layer8_attention_weight_max": 40.426666259765625,
      "activations/layer8_attention_weight_min": -39.8904914855957,
      "activations/layer9_attention_weight_max": 34.695716857910156,
      "activations/layer9_attention_weight_min": -34.52018356323242,
      "epoch": 8.0,
      "learning_rate": 9.943977272727272e-05,
      "loss": 2.8145,
      "step": 137600
    },
    {
      "activations/layer0_attention_weight_max": 15.078130722045898,
      "activations/layer0_attention_weight_min": -12.83639144897461,
      "activations/layer10_attention_weight_max": 36.947479248046875,
      "activations/layer10_attention_weight_min": -33.33319091796875,
      "activations/layer11_attention_weight_max": 38.961219787597656,
      "activations/layer11_attention_weight_min": -32.35054016113281,
      "activations/layer12_attention_weight_max": 26.663515090942383,
      "activations/layer12_attention_weight_min": -24.977291107177734,
      "activations/layer13_attention_weight_max": 47.623722076416016,
      "activations/layer13_attention_weight_min": -33.49024963378906,
      "activations/layer14_attention_weight_max": 58.80012893676758,
      "activations/layer14_attention_weight_min": -37.92605209350586,
      "activations/layer15_attention_weight_max": 43.955013275146484,
      "activations/layer15_attention_weight_min": -33.6512565612793,
      "activations/layer16_attention_weight_max": 34.81148910522461,
      "activations/layer16_attention_weight_min": -28.85761833190918,
      "activations/layer17_attention_weight_max": 41.690696716308594,
      "activations/layer17_attention_weight_min": -27.471759796142578,
      "activations/layer18_attention_weight_max": 36.83411407470703,
      "activations/layer18_attention_weight_min": -22.051889419555664,
      "activations/layer19_attention_weight_max": 37.58658981323242,
      "activations/layer19_attention_weight_min": -30.404111862182617,
      "activations/layer1_attention_weight_max": 17.05121612548828,
      "activations/layer1_attention_weight_min": -15.38780689239502,
      "activations/layer20_attention_weight_max": 38.357940673828125,
      "activations/layer20_attention_weight_min": -25.173072814941406,
      "activations/layer21_attention_weight_max": 31.756710052490234,
      "activations/layer21_attention_weight_min": -23.079282760620117,
      "activations/layer22_attention_weight_max": 53.87683868408203,
      "activations/layer22_attention_weight_min": -30.58860206604004,
      "activations/layer23_attention_weight_max": 39.77259826660156,
      "activations/layer23_attention_weight_min": -24.9826717376709,
      "activations/layer2_attention_weight_max": 32.67055130004883,
      "activations/layer2_attention_weight_min": -32.65867614746094,
      "activations/layer3_attention_weight_max": 97.08657836914062,
      "activations/layer3_attention_weight_min": -101.47277069091797,
      "activations/layer4_attention_weight_max": 125.708251953125,
      "activations/layer4_attention_weight_min": -128.49517822265625,
      "activations/layer5_attention_weight_max": 61.63568878173828,
      "activations/layer5_attention_weight_min": -66.15939331054688,
      "activations/layer6_attention_weight_max": 50.4367790222168,
      "activations/layer6_attention_weight_min": -55.546043395996094,
      "activations/layer7_attention_weight_max": 92.09557342529297,
      "activations/layer7_attention_weight_min": -93.59432983398438,
      "activations/layer8_attention_weight_max": 42.136253356933594,
      "activations/layer8_attention_weight_min": -43.031063079833984,
      "activations/layer9_attention_weight_max": 37.64963150024414,
      "activations/layer9_attention_weight_min": -35.85593795776367,
      "epoch": 8.0,
      "learning_rate": 9.942083333333332e-05,
      "loss": 2.8098,
      "step": 137650
    },
    {
      "activations/layer0_attention_weight_max": 15.37578296661377,
      "activations/layer0_attention_weight_min": -11.85401439666748,
      "activations/layer10_attention_weight_max": 29.880163192749023,
      "activations/layer10_attention_weight_min": -29.513647079467773,
      "activations/layer11_attention_weight_max": 29.774627685546875,
      "activations/layer11_attention_weight_min": -27.34088897705078,
      "activations/layer12_attention_weight_max": 26.675230026245117,
      "activations/layer12_attention_weight_min": -26.94141387939453,
      "activations/layer13_attention_weight_max": 36.08491134643555,
      "activations/layer13_attention_weight_min": -31.169330596923828,
      "activations/layer14_attention_weight_max": 50.26515579223633,
      "activations/layer14_attention_weight_min": -43.1532096862793,
      "activations/layer15_attention_weight_max": 33.48504638671875,
      "activations/layer15_attention_weight_min": -28.713598251342773,
      "activations/layer16_attention_weight_max": 32.38969421386719,
      "activations/layer16_attention_weight_min": -28.489824295043945,
      "activations/layer17_attention_weight_max": 30.882368087768555,
      "activations/layer17_attention_weight_min": -24.8330135345459,
      "activations/layer18_attention_weight_max": 29.560775756835938,
      "activations/layer18_attention_weight_min": -21.15727424621582,
      "activations/layer19_attention_weight_max": 40.39811706542969,
      "activations/layer19_attention_weight_min": -28.984268188476562,
      "activations/layer1_attention_weight_max": 17.3251953125,
      "activations/layer1_attention_weight_min": -15.493776321411133,
      "activations/layer20_attention_weight_max": 28.330780029296875,
      "activations/layer20_attention_weight_min": -23.569438934326172,
      "activations/layer21_attention_weight_max": 26.333694458007812,
      "activations/layer21_attention_weight_min": -21.344406127929688,
      "activations/layer22_attention_weight_max": 48.98016357421875,
      "activations/layer22_attention_weight_min": -27.123748779296875,
      "activations/layer23_attention_weight_max": 35.2806396484375,
      "activations/layer23_attention_weight_min": -23.251426696777344,
      "activations/layer2_attention_weight_max": 31.372955322265625,
      "activations/layer2_attention_weight_min": -33.10102462768555,
      "activations/layer3_attention_weight_max": 88.87797546386719,
      "activations/layer3_attention_weight_min": -92.39418029785156,
      "activations/layer4_attention_weight_max": 117.63679504394531,
      "activations/layer4_attention_weight_min": -114.938720703125,
      "activations/layer5_attention_weight_max": 56.32339096069336,
      "activations/layer5_attention_weight_min": -61.803627014160156,
      "activations/layer6_attention_weight_max": 45.5885124206543,
      "activations/layer6_attention_weight_min": -52.67155075073242,
      "activations/layer7_attention_weight_max": 81.28451538085938,
      "activations/layer7_attention_weight_min": -85.99952697753906,
      "activations/layer8_attention_weight_max": 37.114532470703125,
      "activations/layer8_attention_weight_min": -40.109981536865234,
      "activations/layer9_attention_weight_max": 32.792396545410156,
      "activations/layer9_attention_weight_min": -33.00815963745117,
      "epoch": 8.0,
      "learning_rate": 9.940189393939394e-05,
      "loss": 2.8594,
      "step": 137700
    },
    {
      "activations/layer0_attention_weight_max": 14.885379791259766,
      "activations/layer0_attention_weight_min": -12.249434471130371,
      "activations/layer10_attention_weight_max": 35.927127838134766,
      "activations/layer10_attention_weight_min": -32.769859313964844,
      "activations/layer11_attention_weight_max": 34.45134353637695,
      "activations/layer11_attention_weight_min": -30.893409729003906,
      "activations/layer12_attention_weight_max": 24.048030853271484,
      "activations/layer12_attention_weight_min": -26.749975204467773,
      "activations/layer13_attention_weight_max": 45.00737762451172,
      "activations/layer13_attention_weight_min": -35.30370330810547,
      "activations/layer14_attention_weight_max": 59.04591369628906,
      "activations/layer14_attention_weight_min": -42.481201171875,
      "activations/layer15_attention_weight_max": 39.67898178100586,
      "activations/layer15_attention_weight_min": -29.14670753479004,
      "activations/layer16_attention_weight_max": 34.97219467163086,
      "activations/layer16_attention_weight_min": -29.42414665222168,
      "activations/layer17_attention_weight_max": 35.550025939941406,
      "activations/layer17_attention_weight_min": -24.79731559753418,
      "activations/layer18_attention_weight_max": 36.06595230102539,
      "activations/layer18_attention_weight_min": -23.785888671875,
      "activations/layer19_attention_weight_max": 39.445735931396484,
      "activations/layer19_attention_weight_min": -31.372915267944336,
      "activations/layer1_attention_weight_max": 17.204177856445312,
      "activations/layer1_attention_weight_min": -14.165168762207031,
      "activations/layer20_attention_weight_max": 35.89665603637695,
      "activations/layer20_attention_weight_min": -25.403675079345703,
      "activations/layer21_attention_weight_max": 37.05967712402344,
      "activations/layer21_attention_weight_min": -22.69765853881836,
      "activations/layer22_attention_weight_max": 54.74180221557617,
      "activations/layer22_attention_weight_min": -29.520856857299805,
      "activations/layer23_attention_weight_max": 43.20823287963867,
      "activations/layer23_attention_weight_min": -22.881134033203125,
      "activations/layer2_attention_weight_max": 30.8231201171875,
      "activations/layer2_attention_weight_min": -32.029788970947266,
      "activations/layer3_attention_weight_max": 94.55599212646484,
      "activations/layer3_attention_weight_min": -97.82565307617188,
      "activations/layer4_attention_weight_max": 120.67711639404297,
      "activations/layer4_attention_weight_min": -120.13325500488281,
      "activations/layer5_attention_weight_max": 56.95262908935547,
      "activations/layer5_attention_weight_min": -61.93555450439453,
      "activations/layer6_attention_weight_max": 48.73849868774414,
      "activations/layer6_attention_weight_min": -51.425254821777344,
      "activations/layer7_attention_weight_max": 89.43997192382812,
      "activations/layer7_attention_weight_min": -90.77759552001953,
      "activations/layer8_attention_weight_max": 40.78278732299805,
      "activations/layer8_attention_weight_min": -43.3740119934082,
      "activations/layer9_attention_weight_max": 35.733829498291016,
      "activations/layer9_attention_weight_min": -33.95024108886719,
      "epoch": 8.0,
      "learning_rate": 9.938295454545452e-05,
      "loss": 2.8027,
      "step": 137750
    },
    {
      "activations/layer0_attention_weight_max": 16.760942459106445,
      "activations/layer0_attention_weight_min": -12.568231582641602,
      "activations/layer10_attention_weight_max": 31.245359420776367,
      "activations/layer10_attention_weight_min": -31.11161994934082,
      "activations/layer11_attention_weight_max": 31.506284713745117,
      "activations/layer11_attention_weight_min": -29.533687591552734,
      "activations/layer12_attention_weight_max": 22.8918514251709,
      "activations/layer12_attention_weight_min": -27.698020935058594,
      "activations/layer13_attention_weight_max": 36.80341720581055,
      "activations/layer13_attention_weight_min": -31.333309173583984,
      "activations/layer14_attention_weight_max": 48.09292221069336,
      "activations/layer14_attention_weight_min": -37.195743560791016,
      "activations/layer15_attention_weight_max": 35.18004608154297,
      "activations/layer15_attention_weight_min": -32.12752151489258,
      "activations/layer16_attention_weight_max": 32.63972091674805,
      "activations/layer16_attention_weight_min": -27.97325325012207,
      "activations/layer17_attention_weight_max": 35.36971664428711,
      "activations/layer17_attention_weight_min": -24.83609962463379,
      "activations/layer18_attention_weight_max": 30.50494956970215,
      "activations/layer18_attention_weight_min": -21.409738540649414,
      "activations/layer19_attention_weight_max": 39.216064453125,
      "activations/layer19_attention_weight_min": -28.73345947265625,
      "activations/layer1_attention_weight_max": 16.766273498535156,
      "activations/layer1_attention_weight_min": -14.980545997619629,
      "activations/layer20_attention_weight_max": 29.21807861328125,
      "activations/layer20_attention_weight_min": -23.85720443725586,
      "activations/layer21_attention_weight_max": 30.80095100402832,
      "activations/layer21_attention_weight_min": -22.043453216552734,
      "activations/layer22_attention_weight_max": 45.606414794921875,
      "activations/layer22_attention_weight_min": -26.820425033569336,
      "activations/layer23_attention_weight_max": 36.383995056152344,
      "activations/layer23_attention_weight_min": -23.568363189697266,
      "activations/layer2_attention_weight_max": 31.772098541259766,
      "activations/layer2_attention_weight_min": -30.60605239868164,
      "activations/layer3_attention_weight_max": 91.89674377441406,
      "activations/layer3_attention_weight_min": -89.9619140625,
      "activations/layer4_attention_weight_max": 118.8997802734375,
      "activations/layer4_attention_weight_min": -115.52388000488281,
      "activations/layer5_attention_weight_max": 56.43546676635742,
      "activations/layer5_attention_weight_min": -62.80030059814453,
      "activations/layer6_attention_weight_max": 48.600276947021484,
      "activations/layer6_attention_weight_min": -51.85652160644531,
      "activations/layer7_attention_weight_max": 86.13131713867188,
      "activations/layer7_attention_weight_min": -84.45048522949219,
      "activations/layer8_attention_weight_max": 36.6679573059082,
      "activations/layer8_attention_weight_min": -39.10114669799805,
      "activations/layer9_attention_weight_max": 33.1836051940918,
      "activations/layer9_attention_weight_min": -33.3907470703125,
      "epoch": 8.01,
      "learning_rate": 9.936401515151514e-05,
      "loss": 2.7885,
      "step": 137800
    },
    {
      "activations/layer0_attention_weight_max": 15.414617538452148,
      "activations/layer0_attention_weight_min": -11.975722312927246,
      "activations/layer10_attention_weight_max": 31.86610984802246,
      "activations/layer10_attention_weight_min": -31.909013748168945,
      "activations/layer11_attention_weight_max": 31.541728973388672,
      "activations/layer11_attention_weight_min": -30.457866668701172,
      "activations/layer12_attention_weight_max": 23.27268409729004,
      "activations/layer12_attention_weight_min": -33.80778121948242,
      "activations/layer13_attention_weight_max": 40.560829162597656,
      "activations/layer13_attention_weight_min": -35.94036865234375,
      "activations/layer14_attention_weight_max": 52.36166000366211,
      "activations/layer14_attention_weight_min": -41.6583366394043,
      "activations/layer15_attention_weight_max": 40.828277587890625,
      "activations/layer15_attention_weight_min": -28.73983383178711,
      "activations/layer16_attention_weight_max": 35.5507698059082,
      "activations/layer16_attention_weight_min": -27.462541580200195,
      "activations/layer17_attention_weight_max": 35.6938591003418,
      "activations/layer17_attention_weight_min": -24.419832229614258,
      "activations/layer18_attention_weight_max": 35.23789596557617,
      "activations/layer18_attention_weight_min": -21.295766830444336,
      "activations/layer19_attention_weight_max": 38.07732009887695,
      "activations/layer19_attention_weight_min": -29.048572540283203,
      "activations/layer1_attention_weight_max": 17.192440032958984,
      "activations/layer1_attention_weight_min": -14.601415634155273,
      "activations/layer20_attention_weight_max": 34.71160888671875,
      "activations/layer20_attention_weight_min": -25.151870727539062,
      "activations/layer21_attention_weight_max": 31.48215103149414,
      "activations/layer21_attention_weight_min": -23.71329116821289,
      "activations/layer22_attention_weight_max": 51.32089614868164,
      "activations/layer22_attention_weight_min": -28.921953201293945,
      "activations/layer23_attention_weight_max": 39.01519775390625,
      "activations/layer23_attention_weight_min": -19.812511444091797,
      "activations/layer2_attention_weight_max": 31.231443405151367,
      "activations/layer2_attention_weight_min": -33.6876106262207,
      "activations/layer3_attention_weight_max": 91.89024353027344,
      "activations/layer3_attention_weight_min": -98.86114501953125,
      "activations/layer4_attention_weight_max": 117.98805236816406,
      "activations/layer4_attention_weight_min": -119.710205078125,
      "activations/layer5_attention_weight_max": 55.242366790771484,
      "activations/layer5_attention_weight_min": -64.06986999511719,
      "activations/layer6_attention_weight_max": 49.443382263183594,
      "activations/layer6_attention_weight_min": -50.00870132446289,
      "activations/layer7_attention_weight_max": 87.28014373779297,
      "activations/layer7_attention_weight_min": -86.14537811279297,
      "activations/layer8_attention_weight_max": 38.594886779785156,
      "activations/layer8_attention_weight_min": -38.997596740722656,
      "activations/layer9_attention_weight_max": 32.204654693603516,
      "activations/layer9_attention_weight_min": -34.3076286315918,
      "epoch": 8.01,
      "learning_rate": 9.934507575757575e-05,
      "loss": 2.7979,
      "step": 137850
    },
    {
      "activations/layer0_attention_weight_max": 15.624698638916016,
      "activations/layer0_attention_weight_min": -12.468621253967285,
      "activations/layer10_attention_weight_max": 33.96051788330078,
      "activations/layer10_attention_weight_min": -33.1084098815918,
      "activations/layer11_attention_weight_max": 37.07283020019531,
      "activations/layer11_attention_weight_min": -31.706710815429688,
      "activations/layer12_attention_weight_max": 24.1908016204834,
      "activations/layer12_attention_weight_min": -23.359432220458984,
      "activations/layer13_attention_weight_max": 43.440773010253906,
      "activations/layer13_attention_weight_min": -34.054927825927734,
      "activations/layer14_attention_weight_max": 49.83918380737305,
      "activations/layer14_attention_weight_min": -45.938358306884766,
      "activations/layer15_attention_weight_max": 46.47052001953125,
      "activations/layer15_attention_weight_min": -31.941341400146484,
      "activations/layer16_attention_weight_max": 32.52507400512695,
      "activations/layer16_attention_weight_min": -30.106664657592773,
      "activations/layer17_attention_weight_max": 33.53919982910156,
      "activations/layer17_attention_weight_min": -24.557437896728516,
      "activations/layer18_attention_weight_max": 30.740697860717773,
      "activations/layer18_attention_weight_min": -21.550321578979492,
      "activations/layer19_attention_weight_max": 34.889163970947266,
      "activations/layer19_attention_weight_min": -34.29071044921875,
      "activations/layer1_attention_weight_max": 16.923280715942383,
      "activations/layer1_attention_weight_min": -15.63532829284668,
      "activations/layer20_attention_weight_max": 32.084388732910156,
      "activations/layer20_attention_weight_min": -24.40849494934082,
      "activations/layer21_attention_weight_max": 31.20651626586914,
      "activations/layer21_attention_weight_min": -22.675622940063477,
      "activations/layer22_attention_weight_max": 46.96989059448242,
      "activations/layer22_attention_weight_min": -28.25861358642578,
      "activations/layer23_attention_weight_max": 33.147613525390625,
      "activations/layer23_attention_weight_min": -23.84062957763672,
      "activations/layer2_attention_weight_max": 33.562034606933594,
      "activations/layer2_attention_weight_min": -35.37635040283203,
      "activations/layer3_attention_weight_max": 95.15575408935547,
      "activations/layer3_attention_weight_min": -98.859619140625,
      "activations/layer4_attention_weight_max": 122.9892807006836,
      "activations/layer4_attention_weight_min": -121.377197265625,
      "activations/layer5_attention_weight_max": 57.8665771484375,
      "activations/layer5_attention_weight_min": -62.31789779663086,
      "activations/layer6_attention_weight_max": 51.140419006347656,
      "activations/layer6_attention_weight_min": -53.80107116699219,
      "activations/layer7_attention_weight_max": 89.75957489013672,
      "activations/layer7_attention_weight_min": -91.07785034179688,
      "activations/layer8_attention_weight_max": 42.13344192504883,
      "activations/layer8_attention_weight_min": -42.77885818481445,
      "activations/layer9_attention_weight_max": 34.42244338989258,
      "activations/layer9_attention_weight_min": -35.70022201538086,
      "epoch": 8.01,
      "learning_rate": 9.932613636363634e-05,
      "loss": 2.8033,
      "step": 137900
    },
    {
      "activations/layer0_attention_weight_max": 16.444002151489258,
      "activations/layer0_attention_weight_min": -12.756089210510254,
      "activations/layer10_attention_weight_max": 31.12450408935547,
      "activations/layer10_attention_weight_min": -31.48457908630371,
      "activations/layer11_attention_weight_max": 30.46136474609375,
      "activations/layer11_attention_weight_min": -29.92864227294922,
      "activations/layer12_attention_weight_max": 21.842844009399414,
      "activations/layer12_attention_weight_min": -26.576614379882812,
      "activations/layer13_attention_weight_max": 37.631103515625,
      "activations/layer13_attention_weight_min": -30.96926498413086,
      "activations/layer14_attention_weight_max": 46.33868408203125,
      "activations/layer14_attention_weight_min": -37.73698806762695,
      "activations/layer15_attention_weight_max": 36.45323944091797,
      "activations/layer15_attention_weight_min": -30.34157943725586,
      "activations/layer16_attention_weight_max": 32.841609954833984,
      "activations/layer16_attention_weight_min": -27.421415328979492,
      "activations/layer17_attention_weight_max": 32.65218734741211,
      "activations/layer17_attention_weight_min": -24.430551528930664,
      "activations/layer18_attention_weight_max": 32.918365478515625,
      "activations/layer18_attention_weight_min": -22.885271072387695,
      "activations/layer19_attention_weight_max": 35.408748626708984,
      "activations/layer19_attention_weight_min": -30.99668312072754,
      "activations/layer1_attention_weight_max": 16.636503219604492,
      "activations/layer1_attention_weight_min": -14.615082740783691,
      "activations/layer20_attention_weight_max": 32.469757080078125,
      "activations/layer20_attention_weight_min": -23.64478874206543,
      "activations/layer21_attention_weight_max": 31.56413459777832,
      "activations/layer21_attention_weight_min": -23.830265045166016,
      "activations/layer22_attention_weight_max": 44.62139892578125,
      "activations/layer22_attention_weight_min": -30.328060150146484,
      "activations/layer23_attention_weight_max": 33.24688720703125,
      "activations/layer23_attention_weight_min": -24.02236557006836,
      "activations/layer2_attention_weight_max": 31.953445434570312,
      "activations/layer2_attention_weight_min": -32.659515380859375,
      "activations/layer3_attention_weight_max": 90.9327621459961,
      "activations/layer3_attention_weight_min": -91.87797546386719,
      "activations/layer4_attention_weight_max": 118.24552154541016,
      "activations/layer4_attention_weight_min": -113.13117980957031,
      "activations/layer5_attention_weight_max": 58.563720703125,
      "activations/layer5_attention_weight_min": -61.114501953125,
      "activations/layer6_attention_weight_max": 49.06254959106445,
      "activations/layer6_attention_weight_min": -48.96207809448242,
      "activations/layer7_attention_weight_max": 83.34420013427734,
      "activations/layer7_attention_weight_min": -85.67143249511719,
      "activations/layer8_attention_weight_max": 38.830543518066406,
      "activations/layer8_attention_weight_min": -38.52046585083008,
      "activations/layer9_attention_weight_max": 34.65983963012695,
      "activations/layer9_attention_weight_min": -32.97231674194336,
      "epoch": 8.02,
      "learning_rate": 9.930719696969696e-05,
      "loss": 2.8061,
      "step": 137950
    },
    {
      "activations/layer0_attention_weight_max": 15.90402603149414,
      "activations/layer0_attention_weight_min": -12.322442054748535,
      "activations/layer10_attention_weight_max": 43.82106399536133,
      "activations/layer10_attention_weight_min": -37.40990447998047,
      "activations/layer11_attention_weight_max": 43.12123107910156,
      "activations/layer11_attention_weight_min": -37.297576904296875,
      "activations/layer12_attention_weight_max": 43.48809051513672,
      "activations/layer12_attention_weight_min": -30.93391227722168,
      "activations/layer13_attention_weight_max": 52.81704330444336,
      "activations/layer13_attention_weight_min": -33.488563537597656,
      "activations/layer14_attention_weight_max": 64.36341857910156,
      "activations/layer14_attention_weight_min": -44.15288162231445,
      "activations/layer15_attention_weight_max": 49.88941192626953,
      "activations/layer15_attention_weight_min": -31.268070220947266,
      "activations/layer16_attention_weight_max": 39.595069885253906,
      "activations/layer16_attention_weight_min": -27.880611419677734,
      "activations/layer17_attention_weight_max": 40.740699768066406,
      "activations/layer17_attention_weight_min": -27.544771194458008,
      "activations/layer18_attention_weight_max": 34.49532699584961,
      "activations/layer18_attention_weight_min": -24.26873207092285,
      "activations/layer19_attention_weight_max": 41.60025405883789,
      "activations/layer19_attention_weight_min": -32.863861083984375,
      "activations/layer1_attention_weight_max": 18.00583839416504,
      "activations/layer1_attention_weight_min": -13.800389289855957,
      "activations/layer20_attention_weight_max": 32.94173049926758,
      "activations/layer20_attention_weight_min": -25.165794372558594,
      "activations/layer21_attention_weight_max": 34.69847106933594,
      "activations/layer21_attention_weight_min": -24.12701988220215,
      "activations/layer22_attention_weight_max": 48.0062370300293,
      "activations/layer22_attention_weight_min": -28.36599349975586,
      "activations/layer23_attention_weight_max": 36.25365447998047,
      "activations/layer23_attention_weight_min": -24.43075942993164,
      "activations/layer2_attention_weight_max": 33.19074630737305,
      "activations/layer2_attention_weight_min": -33.292755126953125,
      "activations/layer3_attention_weight_max": 97.12637329101562,
      "activations/layer3_attention_weight_min": -99.98776245117188,
      "activations/layer4_attention_weight_max": 123.8989486694336,
      "activations/layer4_attention_weight_min": -123.20137023925781,
      "activations/layer5_attention_weight_max": 59.38124084472656,
      "activations/layer5_attention_weight_min": -60.07569885253906,
      "activations/layer6_attention_weight_max": 51.04167938232422,
      "activations/layer6_attention_weight_min": -50.89510726928711,
      "activations/layer7_attention_weight_max": 93.6163330078125,
      "activations/layer7_attention_weight_min": -98.7561264038086,
      "activations/layer8_attention_weight_max": 45.68460464477539,
      "activations/layer8_attention_weight_min": -42.99989700317383,
      "activations/layer9_attention_weight_max": 43.78144454956055,
      "activations/layer9_attention_weight_min": -37.67551040649414,
      "epoch": 8.02,
      "learning_rate": 9.928863636363635e-05,
      "loss": 2.8138,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.5183,
      "eval_samples_per_second": 504.093,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.5183,
      "eval_openwebtext_samples_per_second": 504.093,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 2.0334,
      "eval_wikitext_samples_per_second": 224.251,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_lambada_loss": 2.623046875,
      "eval_lambada_ppl": 13.777638440111984,
      "eval_lambada_runtime": 9.6665,
      "eval_lambada_samples_per_second": 503.699,
      "step": 138000
    },
    {
      "activations/layer0_attention_weight_max": 15.939244270324707,
      "activations/layer0_attention_weight_min": -12.031224250793457,
      "activations/layer10_attention_weight_max": 33.32924270629883,
      "activations/layer10_attention_weight_min": -32.26461410522461,
      "activations/layer11_attention_weight_max": 32.060585021972656,
      "activations/layer11_attention_weight_min": -29.278820037841797,
      "activations/layer12_attention_weight_max": 26.16222381591797,
      "activations/layer12_attention_weight_min": -27.88620948791504,
      "activations/layer13_attention_weight_max": 41.67246627807617,
      "activations/layer13_attention_weight_min": -32.18137741088867,
      "activations/layer14_attention_weight_max": 50.60782241821289,
      "activations/layer14_attention_weight_min": -41.71120834350586,
      "activations/layer15_attention_weight_max": 40.0455207824707,
      "activations/layer15_attention_weight_min": -29.22539520263672,
      "activations/layer16_attention_weight_max": 35.980159759521484,
      "activations/layer16_attention_weight_min": -27.323238372802734,
      "activations/layer17_attention_weight_max": 37.76795196533203,
      "activations/layer17_attention_weight_min": -26.423847198486328,
      "activations/layer18_attention_weight_max": 34.983360290527344,
      "activations/layer18_attention_weight_min": -21.66261100769043,
      "activations/layer19_attention_weight_max": 42.7113151550293,
      "activations/layer19_attention_weight_min": -31.02559471130371,
      "activations/layer1_attention_weight_max": 17.056358337402344,
      "activations/layer1_attention_weight_min": -13.038025856018066,
      "activations/layer20_attention_weight_max": 36.028194427490234,
      "activations/layer20_attention_weight_min": -24.7979679107666,
      "activations/layer21_attention_weight_max": 34.6043586730957,
      "activations/layer21_attention_weight_min": -23.212745666503906,
      "activations/layer22_attention_weight_max": 55.34889221191406,
      "activations/layer22_attention_weight_min": -29.517114639282227,
      "activations/layer23_attention_weight_max": 44.213985443115234,
      "activations/layer23_attention_weight_min": -24.641353607177734,
      "activations/layer2_attention_weight_max": 33.51723098754883,
      "activations/layer2_attention_weight_min": -32.4732551574707,
      "activations/layer3_attention_weight_max": 93.00508117675781,
      "activations/layer3_attention_weight_min": -98.96849060058594,
      "activations/layer4_attention_weight_max": 120.3786392211914,
      "activations/layer4_attention_weight_min": -121.5667495727539,
      "activations/layer5_attention_weight_max": 59.396297454833984,
      "activations/layer5_attention_weight_min": -64.96586608886719,
      "activations/layer6_attention_weight_max": 51.34618377685547,
      "activations/layer6_attention_weight_min": -53.720462799072266,
      "activations/layer7_attention_weight_max": 93.29430389404297,
      "activations/layer7_attention_weight_min": -89.9712142944336,
      "activations/layer8_attention_weight_max": 41.017601013183594,
      "activations/layer8_attention_weight_min": -44.42018127441406,
      "activations/layer9_attention_weight_max": 35.6341667175293,
      "activations/layer9_attention_weight_min": -35.904884338378906,
      "epoch": 8.02,
      "learning_rate": 9.926969696969696e-05,
      "loss": 2.8077,
      "step": 138050
    },
    {
      "activations/layer0_attention_weight_max": 15.978351593017578,
      "activations/layer0_attention_weight_min": -12.862858772277832,
      "activations/layer10_attention_weight_max": 32.9939079284668,
      "activations/layer10_attention_weight_min": -33.68764114379883,
      "activations/layer11_attention_weight_max": 32.0509033203125,
      "activations/layer11_attention_weight_min": -31.71719741821289,
      "activations/layer12_attention_weight_max": 24.655147552490234,
      "activations/layer12_attention_weight_min": -28.92245101928711,
      "activations/layer13_attention_weight_max": 41.68177795410156,
      "activations/layer13_attention_weight_min": -31.41598129272461,
      "activations/layer14_attention_weight_max": 49.397186279296875,
      "activations/layer14_attention_weight_min": -38.73689270019531,
      "activations/layer15_attention_weight_max": 35.26675796508789,
      "activations/layer15_attention_weight_min": -29.02039337158203,
      "activations/layer16_attention_weight_max": 34.42388153076172,
      "activations/layer16_attention_weight_min": -26.342453002929688,
      "activations/layer17_attention_weight_max": 32.57475280761719,
      "activations/layer17_attention_weight_min": -25.69199562072754,
      "activations/layer18_attention_weight_max": 28.296566009521484,
      "activations/layer18_attention_weight_min": -21.885942459106445,
      "activations/layer19_attention_weight_max": 36.193870544433594,
      "activations/layer19_attention_weight_min": -31.6370906829834,
      "activations/layer1_attention_weight_max": 16.478824615478516,
      "activations/layer1_attention_weight_min": -13.988507270812988,
      "activations/layer20_attention_weight_max": 29.528806686401367,
      "activations/layer20_attention_weight_min": -25.326147079467773,
      "activations/layer21_attention_weight_max": 29.196807861328125,
      "activations/layer21_attention_weight_min": -23.94968032836914,
      "activations/layer22_attention_weight_max": 53.175167083740234,
      "activations/layer22_attention_weight_min": -29.990522384643555,
      "activations/layer23_attention_weight_max": 35.2736701965332,
      "activations/layer23_attention_weight_min": -26.83399772644043,
      "activations/layer2_attention_weight_max": 31.9613037109375,
      "activations/layer2_attention_weight_min": -32.567352294921875,
      "activations/layer3_attention_weight_max": 92.10470581054688,
      "activations/layer3_attention_weight_min": -98.15421295166016,
      "activations/layer4_attention_weight_max": 117.7928695678711,
      "activations/layer4_attention_weight_min": -122.95210266113281,
      "activations/layer5_attention_weight_max": 55.732765197753906,
      "activations/layer5_attention_weight_min": -61.72237777709961,
      "activations/layer6_attention_weight_max": 45.7214241027832,
      "activations/layer6_attention_weight_min": -51.10451889038086,
      "activations/layer7_attention_weight_max": 87.12006378173828,
      "activations/layer7_attention_weight_min": -88.92233276367188,
      "activations/layer8_attention_weight_max": 41.14598083496094,
      "activations/layer8_attention_weight_min": -41.26042556762695,
      "activations/layer9_attention_weight_max": 36.63866424560547,
      "activations/layer9_attention_weight_min": -34.57373809814453,
      "epoch": 8.02,
      "learning_rate": 9.925075757575757e-05,
      "loss": 2.7953,
      "step": 138100
    },
    {
      "activations/layer0_attention_weight_max": 15.774059295654297,
      "activations/layer0_attention_weight_min": -11.625368118286133,
      "activations/layer10_attention_weight_max": 31.470041275024414,
      "activations/layer10_attention_weight_min": -31.715829849243164,
      "activations/layer11_attention_weight_max": 30.42153549194336,
      "activations/layer11_attention_weight_min": -31.2938232421875,
      "activations/layer12_attention_weight_max": 20.246082305908203,
      "activations/layer12_attention_weight_min": -26.538509368896484,
      "activations/layer13_attention_weight_max": 40.49778366088867,
      "activations/layer13_attention_weight_min": -34.07395935058594,
      "activations/layer14_attention_weight_max": 45.694358825683594,
      "activations/layer14_attention_weight_min": -41.16545486450195,
      "activations/layer15_attention_weight_max": 32.02023696899414,
      "activations/layer15_attention_weight_min": -30.17557144165039,
      "activations/layer16_attention_weight_max": 28.529552459716797,
      "activations/layer16_attention_weight_min": -27.215017318725586,
      "activations/layer17_attention_weight_max": 29.78211784362793,
      "activations/layer17_attention_weight_min": -24.493013381958008,
      "activations/layer18_attention_weight_max": 27.694189071655273,
      "activations/layer18_attention_weight_min": -20.04120445251465,
      "activations/layer19_attention_weight_max": 31.500308990478516,
      "activations/layer19_attention_weight_min": -31.992040634155273,
      "activations/layer1_attention_weight_max": 15.740666389465332,
      "activations/layer1_attention_weight_min": -14.123495101928711,
      "activations/layer20_attention_weight_max": 28.42729949951172,
      "activations/layer20_attention_weight_min": -23.267980575561523,
      "activations/layer21_attention_weight_max": 26.571935653686523,
      "activations/layer21_attention_weight_min": -23.823974609375,
      "activations/layer22_attention_weight_max": 40.688453674316406,
      "activations/layer22_attention_weight_min": -27.37718391418457,
      "activations/layer23_attention_weight_max": 37.01353454589844,
      "activations/layer23_attention_weight_min": -21.727842330932617,
      "activations/layer2_attention_weight_max": 32.17837142944336,
      "activations/layer2_attention_weight_min": -31.866771697998047,
      "activations/layer3_attention_weight_max": 88.41624450683594,
      "activations/layer3_attention_weight_min": -91.24488067626953,
      "activations/layer4_attention_weight_max": 111.67070770263672,
      "activations/layer4_attention_weight_min": -113.2127685546875,
      "activations/layer5_attention_weight_max": 54.260005950927734,
      "activations/layer5_attention_weight_min": -59.9255485534668,
      "activations/layer6_attention_weight_max": 48.01178741455078,
      "activations/layer6_attention_weight_min": -50.64646911621094,
      "activations/layer7_attention_weight_max": 85.14360046386719,
      "activations/layer7_attention_weight_min": -89.74555969238281,
      "activations/layer8_attention_weight_max": 36.26560592651367,
      "activations/layer8_attention_weight_min": -38.6367073059082,
      "activations/layer9_attention_weight_max": 31.885770797729492,
      "activations/layer9_attention_weight_min": -33.5858268737793,
      "epoch": 8.03,
      "learning_rate": 9.923181818181817e-05,
      "loss": 2.8001,
      "step": 138150
    },
    {
      "activations/layer0_attention_weight_max": 15.917202949523926,
      "activations/layer0_attention_weight_min": -12.22724723815918,
      "activations/layer10_attention_weight_max": 37.89136505126953,
      "activations/layer10_attention_weight_min": -34.41668701171875,
      "activations/layer11_attention_weight_max": 36.403568267822266,
      "activations/layer11_attention_weight_min": -33.760231018066406,
      "activations/layer12_attention_weight_max": 43.219635009765625,
      "activations/layer12_attention_weight_min": -26.80014419555664,
      "activations/layer13_attention_weight_max": 50.072147369384766,
      "activations/layer13_attention_weight_min": -35.14159393310547,
      "activations/layer14_attention_weight_max": 48.94619369506836,
      "activations/layer14_attention_weight_min": -42.511566162109375,
      "activations/layer15_attention_weight_max": 38.597137451171875,
      "activations/layer15_attention_weight_min": -31.089405059814453,
      "activations/layer16_attention_weight_max": 31.415515899658203,
      "activations/layer16_attention_weight_min": -27.91776466369629,
      "activations/layer17_attention_weight_max": 38.17161560058594,
      "activations/layer17_attention_weight_min": -24.757482528686523,
      "activations/layer18_attention_weight_max": 33.11728286743164,
      "activations/layer18_attention_weight_min": -22.530614852905273,
      "activations/layer19_attention_weight_max": 36.666831970214844,
      "activations/layer19_attention_weight_min": -29.569229125976562,
      "activations/layer1_attention_weight_max": 16.626699447631836,
      "activations/layer1_attention_weight_min": -14.959091186523438,
      "activations/layer20_attention_weight_max": 29.895355224609375,
      "activations/layer20_attention_weight_min": -25.070280075073242,
      "activations/layer21_attention_weight_max": 32.7408561706543,
      "activations/layer21_attention_weight_min": -22.993684768676758,
      "activations/layer22_attention_weight_max": 51.484073638916016,
      "activations/layer22_attention_weight_min": -28.266069412231445,
      "activations/layer23_attention_weight_max": 37.44581604003906,
      "activations/layer23_attention_weight_min": -24.51442527770996,
      "activations/layer2_attention_weight_max": 33.054054260253906,
      "activations/layer2_attention_weight_min": -32.086708068847656,
      "activations/layer3_attention_weight_max": 91.111328125,
      "activations/layer3_attention_weight_min": -91.70252227783203,
      "activations/layer4_attention_weight_max": 118.5578842163086,
      "activations/layer4_attention_weight_min": -117.99210357666016,
      "activations/layer5_attention_weight_max": 59.16154098510742,
      "activations/layer5_attention_weight_min": -61.520050048828125,
      "activations/layer6_attention_weight_max": 50.95376968383789,
      "activations/layer6_attention_weight_min": -51.43593978881836,
      "activations/layer7_attention_weight_max": 89.95668029785156,
      "activations/layer7_attention_weight_min": -88.80390167236328,
      "activations/layer8_attention_weight_max": 42.3941535949707,
      "activations/layer8_attention_weight_min": -41.11211013793945,
      "activations/layer9_attention_weight_max": 41.03385925292969,
      "activations/layer9_attention_weight_min": -34.98200607299805,
      "epoch": 8.03,
      "learning_rate": 9.921287878787878e-05,
      "loss": 2.8021,
      "step": 138200
    },
    {
      "activations/layer0_attention_weight_max": 17.90062713623047,
      "activations/layer0_attention_weight_min": -12.350381851196289,
      "activations/layer10_attention_weight_max": 32.123291015625,
      "activations/layer10_attention_weight_min": -30.50213623046875,
      "activations/layer11_attention_weight_max": 32.572959899902344,
      "activations/layer11_attention_weight_min": -28.989490509033203,
      "activations/layer12_attention_weight_max": 24.181068420410156,
      "activations/layer12_attention_weight_min": -30.25189781188965,
      "activations/layer13_attention_weight_max": 43.55953598022461,
      "activations/layer13_attention_weight_min": -33.201446533203125,
      "activations/layer14_attention_weight_max": 50.057315826416016,
      "activations/layer14_attention_weight_min": -41.53353500366211,
      "activations/layer15_attention_weight_max": 37.32369613647461,
      "activations/layer15_attention_weight_min": -31.57440185546875,
      "activations/layer16_attention_weight_max": 35.20035171508789,
      "activations/layer16_attention_weight_min": -29.37648582458496,
      "activations/layer17_attention_weight_max": 35.827999114990234,
      "activations/layer17_attention_weight_min": -25.156824111938477,
      "activations/layer18_attention_weight_max": 30.230880737304688,
      "activations/layer18_attention_weight_min": -22.00720977783203,
      "activations/layer19_attention_weight_max": 43.23332214355469,
      "activations/layer19_attention_weight_min": -32.555213928222656,
      "activations/layer1_attention_weight_max": 17.938125610351562,
      "activations/layer1_attention_weight_min": -13.15183162689209,
      "activations/layer20_attention_weight_max": 32.363101959228516,
      "activations/layer20_attention_weight_min": -24.462940216064453,
      "activations/layer21_attention_weight_max": 31.31467056274414,
      "activations/layer21_attention_weight_min": -22.999065399169922,
      "activations/layer22_attention_weight_max": 51.63034439086914,
      "activations/layer22_attention_weight_min": -29.009260177612305,
      "activations/layer23_attention_weight_max": 37.49522399902344,
      "activations/layer23_attention_weight_min": -24.140033721923828,
      "activations/layer2_attention_weight_max": 33.055999755859375,
      "activations/layer2_attention_weight_min": -33.981056213378906,
      "activations/layer3_attention_weight_max": 97.6697769165039,
      "activations/layer3_attention_weight_min": -98.33123779296875,
      "activations/layer4_attention_weight_max": 124.212890625,
      "activations/layer4_attention_weight_min": -119.95458221435547,
      "activations/layer5_attention_weight_max": 58.35362243652344,
      "activations/layer5_attention_weight_min": -65.82337951660156,
      "activations/layer6_attention_weight_max": 48.98651885986328,
      "activations/layer6_attention_weight_min": -51.005184173583984,
      "activations/layer7_attention_weight_max": 89.45051574707031,
      "activations/layer7_attention_weight_min": -86.07386016845703,
      "activations/layer8_attention_weight_max": 39.30110168457031,
      "activations/layer8_attention_weight_min": -38.84680938720703,
      "activations/layer9_attention_weight_max": 33.52301788330078,
      "activations/layer9_attention_weight_min": -34.420616149902344,
      "epoch": 8.03,
      "learning_rate": 9.919393939393938e-05,
      "loss": 2.8129,
      "step": 138250
    },
    {
      "activations/layer0_attention_weight_max": 15.737068176269531,
      "activations/layer0_attention_weight_min": -12.121395111083984,
      "activations/layer10_attention_weight_max": 30.325725555419922,
      "activations/layer10_attention_weight_min": -29.743253707885742,
      "activations/layer11_attention_weight_max": 33.088829040527344,
      "activations/layer11_attention_weight_min": -30.564228057861328,
      "activations/layer12_attention_weight_max": 21.38442039489746,
      "activations/layer12_attention_weight_min": -25.97324562072754,
      "activations/layer13_attention_weight_max": 37.91722106933594,
      "activations/layer13_attention_weight_min": -32.107418060302734,
      "activations/layer14_attention_weight_max": 46.14938735961914,
      "activations/layer14_attention_weight_min": -38.2269172668457,
      "activations/layer15_attention_weight_max": 34.62067413330078,
      "activations/layer15_attention_weight_min": -29.046409606933594,
      "activations/layer16_attention_weight_max": 29.912403106689453,
      "activations/layer16_attention_weight_min": -28.2158145904541,
      "activations/layer17_attention_weight_max": 34.59638595581055,
      "activations/layer17_attention_weight_min": -24.295408248901367,
      "activations/layer18_attention_weight_max": 31.074790954589844,
      "activations/layer18_attention_weight_min": -21.232908248901367,
      "activations/layer19_attention_weight_max": 36.05763626098633,
      "activations/layer19_attention_weight_min": -30.686447143554688,
      "activations/layer1_attention_weight_max": 17.35587501525879,
      "activations/layer1_attention_weight_min": -13.196441650390625,
      "activations/layer20_attention_weight_max": 29.838787078857422,
      "activations/layer20_attention_weight_min": -25.329822540283203,
      "activations/layer21_attention_weight_max": 27.638017654418945,
      "activations/layer21_attention_weight_min": -23.444854736328125,
      "activations/layer22_attention_weight_max": 44.95853042602539,
      "activations/layer22_attention_weight_min": -29.896739959716797,
      "activations/layer23_attention_weight_max": 35.95011520385742,
      "activations/layer23_attention_weight_min": -24.895034790039062,
      "activations/layer2_attention_weight_max": 31.970558166503906,
      "activations/layer2_attention_weight_min": -32.01499938964844,
      "activations/layer3_attention_weight_max": 92.66316223144531,
      "activations/layer3_attention_weight_min": -91.20315551757812,
      "activations/layer4_attention_weight_max": 118.84178924560547,
      "activations/layer4_attention_weight_min": -119.7534408569336,
      "activations/layer5_attention_weight_max": 58.633766174316406,
      "activations/layer5_attention_weight_min": -63.780601501464844,
      "activations/layer6_attention_weight_max": 51.25227355957031,
      "activations/layer6_attention_weight_min": -51.34273147583008,
      "activations/layer7_attention_weight_max": 90.41438293457031,
      "activations/layer7_attention_weight_min": -89.74003601074219,
      "activations/layer8_attention_weight_max": 40.450565338134766,
      "activations/layer8_attention_weight_min": -40.81560516357422,
      "activations/layer9_attention_weight_max": 35.13510513305664,
      "activations/layer9_attention_weight_min": -32.1657600402832,
      "epoch": 8.04,
      "learning_rate": 9.9175e-05,
      "loss": 2.7975,
      "step": 138300
    },
    {
      "activations/layer0_attention_weight_max": 14.935401916503906,
      "activations/layer0_attention_weight_min": -11.608850479125977,
      "activations/layer10_attention_weight_max": 33.49514389038086,
      "activations/layer10_attention_weight_min": -31.175338745117188,
      "activations/layer11_attention_weight_max": 33.00141525268555,
      "activations/layer11_attention_weight_min": -29.289047241210938,
      "activations/layer12_attention_weight_max": 22.63062286376953,
      "activations/layer12_attention_weight_min": -23.724811553955078,
      "activations/layer13_attention_weight_max": 37.67997360229492,
      "activations/layer13_attention_weight_min": -33.15729522705078,
      "activations/layer14_attention_weight_max": 46.627742767333984,
      "activations/layer14_attention_weight_min": -42.71843719482422,
      "activations/layer15_attention_weight_max": 34.23152542114258,
      "activations/layer15_attention_weight_min": -31.24669647216797,
      "activations/layer16_attention_weight_max": 32.54418182373047,
      "activations/layer16_attention_weight_min": -27.879892349243164,
      "activations/layer17_attention_weight_max": 32.56489562988281,
      "activations/layer17_attention_weight_min": -25.8013973236084,
      "activations/layer18_attention_weight_max": 33.91056442260742,
      "activations/layer18_attention_weight_min": -22.20215606689453,
      "activations/layer19_attention_weight_max": 34.18484878540039,
      "activations/layer19_attention_weight_min": -31.752704620361328,
      "activations/layer1_attention_weight_max": 16.404769897460938,
      "activations/layer1_attention_weight_min": -14.883926391601562,
      "activations/layer20_attention_weight_max": 29.79307746887207,
      "activations/layer20_attention_weight_min": -26.430156707763672,
      "activations/layer21_attention_weight_max": 35.000423431396484,
      "activations/layer21_attention_weight_min": -25.252117156982422,
      "activations/layer22_attention_weight_max": 49.044227600097656,
      "activations/layer22_attention_weight_min": -36.00410461425781,
      "activations/layer23_attention_weight_max": 34.88127136230469,
      "activations/layer23_attention_weight_min": -29.38916015625,
      "activations/layer2_attention_weight_max": 31.382375717163086,
      "activations/layer2_attention_weight_min": -31.894855499267578,
      "activations/layer3_attention_weight_max": 90.4229507446289,
      "activations/layer3_attention_weight_min": -91.5709228515625,
      "activations/layer4_attention_weight_max": 121.47069549560547,
      "activations/layer4_attention_weight_min": -113.68218994140625,
      "activations/layer5_attention_weight_max": 57.163360595703125,
      "activations/layer5_attention_weight_min": -59.91736602783203,
      "activations/layer6_attention_weight_max": 48.092872619628906,
      "activations/layer6_attention_weight_min": -51.01715850830078,
      "activations/layer7_attention_weight_max": 87.37328338623047,
      "activations/layer7_attention_weight_min": -87.52916717529297,
      "activations/layer8_attention_weight_max": 38.809967041015625,
      "activations/layer8_attention_weight_min": -40.310848236083984,
      "activations/layer9_attention_weight_max": 34.377220153808594,
      "activations/layer9_attention_weight_min": -33.36702346801758,
      "epoch": 8.04,
      "learning_rate": 9.915643939393939e-05,
      "loss": 2.8114,
      "step": 138350
    },
    {
      "activations/layer0_attention_weight_max": 15.890759468078613,
      "activations/layer0_attention_weight_min": -12.72263240814209,
      "activations/layer10_attention_weight_max": 38.73456573486328,
      "activations/layer10_attention_weight_min": -33.04206848144531,
      "activations/layer11_attention_weight_max": 41.29009246826172,
      "activations/layer11_attention_weight_min": -32.32794189453125,
      "activations/layer12_attention_weight_max": 22.48078155517578,
      "activations/layer12_attention_weight_min": -30.554723739624023,
      "activations/layer13_attention_weight_max": 40.62965774536133,
      "activations/layer13_attention_weight_min": -32.23823547363281,
      "activations/layer14_attention_weight_max": 51.915283203125,
      "activations/layer14_attention_weight_min": -42.2347412109375,
      "activations/layer15_attention_weight_max": 39.351688385009766,
      "activations/layer15_attention_weight_min": -29.13088035583496,
      "activations/layer16_attention_weight_max": 33.578670501708984,
      "activations/layer16_attention_weight_min": -27.17598533630371,
      "activations/layer17_attention_weight_max": 36.8169059753418,
      "activations/layer17_attention_weight_min": -25.41468620300293,
      "activations/layer18_attention_weight_max": 31.067630767822266,
      "activations/layer18_attention_weight_min": -20.832059860229492,
      "activations/layer19_attention_weight_max": 39.24628829956055,
      "activations/layer19_attention_weight_min": -29.313791275024414,
      "activations/layer1_attention_weight_max": 16.238311767578125,
      "activations/layer1_attention_weight_min": -14.517592430114746,
      "activations/layer20_attention_weight_max": 34.94414520263672,
      "activations/layer20_attention_weight_min": -23.317089080810547,
      "activations/layer21_attention_weight_max": 29.966079711914062,
      "activations/layer21_attention_weight_min": -21.969507217407227,
      "activations/layer22_attention_weight_max": 55.24256134033203,
      "activations/layer22_attention_weight_min": -29.384601593017578,
      "activations/layer23_attention_weight_max": 34.65578842163086,
      "activations/layer23_attention_weight_min": -23.25629997253418,
      "activations/layer2_attention_weight_max": 32.13468933105469,
      "activations/layer2_attention_weight_min": -30.806198120117188,
      "activations/layer3_attention_weight_max": 89.09239196777344,
      "activations/layer3_attention_weight_min": -91.1398696899414,
      "activations/layer4_attention_weight_max": 118.84529876708984,
      "activations/layer4_attention_weight_min": -113.26393127441406,
      "activations/layer5_attention_weight_max": 57.49836349487305,
      "activations/layer5_attention_weight_min": -61.460121154785156,
      "activations/layer6_attention_weight_max": 48.60288619995117,
      "activations/layer6_attention_weight_min": -51.54301071166992,
      "activations/layer7_attention_weight_max": 98.91529846191406,
      "activations/layer7_attention_weight_min": -88.95346069335938,
      "activations/layer8_attention_weight_max": 46.798866271972656,
      "activations/layer8_attention_weight_min": -44.41066360473633,
      "activations/layer9_attention_weight_max": 39.75468826293945,
      "activations/layer9_attention_weight_min": -34.52645492553711,
      "epoch": 8.04,
      "learning_rate": 9.91375e-05,
      "loss": 2.8167,
      "step": 138400
    },
    {
      "activations/layer0_attention_weight_max": 15.126957893371582,
      "activations/layer0_attention_weight_min": -11.796272277832031,
      "activations/layer10_attention_weight_max": 36.29179000854492,
      "activations/layer10_attention_weight_min": -31.91575050354004,
      "activations/layer11_attention_weight_max": 33.01921081542969,
      "activations/layer11_attention_weight_min": -29.622512817382812,
      "activations/layer12_attention_weight_max": 39.46358108520508,
      "activations/layer12_attention_weight_min": -30.075908660888672,
      "activations/layer13_attention_weight_max": 45.075233459472656,
      "activations/layer13_attention_weight_min": -33.72889709472656,
      "activations/layer14_attention_weight_max": 49.59540939331055,
      "activations/layer14_attention_weight_min": -38.868141174316406,
      "activations/layer15_attention_weight_max": 38.805728912353516,
      "activations/layer15_attention_weight_min": -30.573802947998047,
      "activations/layer16_attention_weight_max": 32.53394317626953,
      "activations/layer16_attention_weight_min": -28.289119720458984,
      "activations/layer17_attention_weight_max": 33.34493637084961,
      "activations/layer17_attention_weight_min": -25.56947135925293,
      "activations/layer18_attention_weight_max": 30.841327667236328,
      "activations/layer18_attention_weight_min": -21.22972297668457,
      "activations/layer19_attention_weight_max": 35.45363998413086,
      "activations/layer19_attention_weight_min": -32.56541442871094,
      "activations/layer1_attention_weight_max": 16.355377197265625,
      "activations/layer1_attention_weight_min": -14.481173515319824,
      "activations/layer20_attention_weight_max": 30.167190551757812,
      "activations/layer20_attention_weight_min": -25.930330276489258,
      "activations/layer21_attention_weight_max": 28.052030563354492,
      "activations/layer21_attention_weight_min": -21.264663696289062,
      "activations/layer22_attention_weight_max": 43.445899963378906,
      "activations/layer22_attention_weight_min": -26.76557731628418,
      "activations/layer23_attention_weight_max": 33.28632736206055,
      "activations/layer23_attention_weight_min": -21.64178466796875,
      "activations/layer2_attention_weight_max": 31.594266891479492,
      "activations/layer2_attention_weight_min": -31.13671875,
      "activations/layer3_attention_weight_max": 90.13245391845703,
      "activations/layer3_attention_weight_min": -87.6357192993164,
      "activations/layer4_attention_weight_max": 123.64723205566406,
      "activations/layer4_attention_weight_min": -115.65827941894531,
      "activations/layer5_attention_weight_max": 58.332908630371094,
      "activations/layer5_attention_weight_min": -62.31333541870117,
      "activations/layer6_attention_weight_max": 51.002132415771484,
      "activations/layer6_attention_weight_min": -52.689414978027344,
      "activations/layer7_attention_weight_max": 87.78775787353516,
      "activations/layer7_attention_weight_min": -89.18385314941406,
      "activations/layer8_attention_weight_max": 39.51687240600586,
      "activations/layer8_attention_weight_min": -41.2369270324707,
      "activations/layer9_attention_weight_max": 40.43541717529297,
      "activations/layer9_attention_weight_min": -34.7578239440918,
      "epoch": 8.04,
      "learning_rate": 9.911856060606059e-05,
      "loss": 2.8084,
      "step": 138450
    },
    {
      "activations/layer0_attention_weight_max": 16.270645141601562,
      "activations/layer0_attention_weight_min": -11.77026653289795,
      "activations/layer10_attention_weight_max": 32.244384765625,
      "activations/layer10_attention_weight_min": -37.60383605957031,
      "activations/layer11_attention_weight_max": 31.340055465698242,
      "activations/layer11_attention_weight_min": -33.71239471435547,
      "activations/layer12_attention_weight_max": 36.08522415161133,
      "activations/layer12_attention_weight_min": -26.067153930664062,
      "activations/layer13_attention_weight_max": 38.28525924682617,
      "activations/layer13_attention_weight_min": -32.33794021606445,
      "activations/layer14_attention_weight_max": 48.25384521484375,
      "activations/layer14_attention_weight_min": -40.66838836669922,
      "activations/layer15_attention_weight_max": 35.57126998901367,
      "activations/layer15_attention_weight_min": -29.669448852539062,
      "activations/layer16_attention_weight_max": 33.45293045043945,
      "activations/layer16_attention_weight_min": -29.247737884521484,
      "activations/layer17_attention_weight_max": 32.967002868652344,
      "activations/layer17_attention_weight_min": -26.332155227661133,
      "activations/layer18_attention_weight_max": 34.57089614868164,
      "activations/layer18_attention_weight_min": -22.36279296875,
      "activations/layer19_attention_weight_max": 34.47086715698242,
      "activations/layer19_attention_weight_min": -31.704883575439453,
      "activations/layer1_attention_weight_max": 16.424346923828125,
      "activations/layer1_attention_weight_min": -14.767807960510254,
      "activations/layer20_attention_weight_max": 30.0259952545166,
      "activations/layer20_attention_weight_min": -25.208850860595703,
      "activations/layer21_attention_weight_max": 27.94830322265625,
      "activations/layer21_attention_weight_min": -23.74163055419922,
      "activations/layer22_attention_weight_max": 45.8558235168457,
      "activations/layer22_attention_weight_min": -29.42332649230957,
      "activations/layer23_attention_weight_max": 37.7421760559082,
      "activations/layer23_attention_weight_min": -22.920202255249023,
      "activations/layer2_attention_weight_max": 31.865402221679688,
      "activations/layer2_attention_weight_min": -32.38822937011719,
      "activations/layer3_attention_weight_max": 91.38615417480469,
      "activations/layer3_attention_weight_min": -93.61527252197266,
      "activations/layer4_attention_weight_max": 115.10665130615234,
      "activations/layer4_attention_weight_min": -112.0051498413086,
      "activations/layer5_attention_weight_max": 56.46877670288086,
      "activations/layer5_attention_weight_min": -61.38429260253906,
      "activations/layer6_attention_weight_max": 48.03245162963867,
      "activations/layer6_attention_weight_min": -48.37686538696289,
      "activations/layer7_attention_weight_max": 87.64869689941406,
      "activations/layer7_attention_weight_min": -92.73377227783203,
      "activations/layer8_attention_weight_max": 40.66437530517578,
      "activations/layer8_attention_weight_min": -42.77928924560547,
      "activations/layer9_attention_weight_max": 33.85053634643555,
      "activations/layer9_attention_weight_min": -37.35822677612305,
      "epoch": 8.05,
      "learning_rate": 9.909962121212121e-05,
      "loss": 2.8083,
      "step": 138500
    },
    {
      "activations/layer0_attention_weight_max": 15.330365180969238,
      "activations/layer0_attention_weight_min": -12.852202415466309,
      "activations/layer10_attention_weight_max": 31.192283630371094,
      "activations/layer10_attention_weight_min": -30.694107055664062,
      "activations/layer11_attention_weight_max": 31.52454948425293,
      "activations/layer11_attention_weight_min": -29.360485076904297,
      "activations/layer12_attention_weight_max": 34.8377799987793,
      "activations/layer12_attention_weight_min": -27.041318893432617,
      "activations/layer13_attention_weight_max": 39.53230667114258,
      "activations/layer13_attention_weight_min": -33.61342239379883,
      "activations/layer14_attention_weight_max": 63.56151580810547,
      "activations/layer14_attention_weight_min": -44.904300689697266,
      "activations/layer15_attention_weight_max": 38.37684631347656,
      "activations/layer15_attention_weight_min": -29.57670021057129,
      "activations/layer16_attention_weight_max": 35.941078186035156,
      "activations/layer16_attention_weight_min": -28.212928771972656,
      "activations/layer17_attention_weight_max": 43.41318130493164,
      "activations/layer17_attention_weight_min": -25.202173233032227,
      "activations/layer18_attention_weight_max": 35.449275970458984,
      "activations/layer18_attention_weight_min": -21.709857940673828,
      "activations/layer19_attention_weight_max": 38.15470886230469,
      "activations/layer19_attention_weight_min": -29.568246841430664,
      "activations/layer1_attention_weight_max": 16.549104690551758,
      "activations/layer1_attention_weight_min": -13.126200675964355,
      "activations/layer20_attention_weight_max": 37.11624526977539,
      "activations/layer20_attention_weight_min": -23.796903610229492,
      "activations/layer21_attention_weight_max": 35.34777069091797,
      "activations/layer21_attention_weight_min": -21.931747436523438,
      "activations/layer22_attention_weight_max": 53.58112335205078,
      "activations/layer22_attention_weight_min": -28.196565628051758,
      "activations/layer23_attention_weight_max": 35.51467514038086,
      "activations/layer23_attention_weight_min": -23.59724998474121,
      "activations/layer2_attention_weight_max": 30.12782096862793,
      "activations/layer2_attention_weight_min": -30.686826705932617,
      "activations/layer3_attention_weight_max": 87.22859954833984,
      "activations/layer3_attention_weight_min": -94.47644805908203,
      "activations/layer4_attention_weight_max": 118.30191802978516,
      "activations/layer4_attention_weight_min": -117.61345672607422,
      "activations/layer5_attention_weight_max": 54.13794708251953,
      "activations/layer5_attention_weight_min": -60.92189025878906,
      "activations/layer6_attention_weight_max": 46.75181579589844,
      "activations/layer6_attention_weight_min": -52.924739837646484,
      "activations/layer7_attention_weight_max": 83.48457336425781,
      "activations/layer7_attention_weight_min": -89.07801055908203,
      "activations/layer8_attention_weight_max": 37.44404220581055,
      "activations/layer8_attention_weight_min": -40.98784255981445,
      "activations/layer9_attention_weight_max": 33.82352828979492,
      "activations/layer9_attention_weight_min": -31.5192813873291,
      "epoch": 8.05,
      "learning_rate": 9.908068181818182e-05,
      "loss": 2.8186,
      "step": 138550
    },
    {
      "activations/layer0_attention_weight_max": 16.092119216918945,
      "activations/layer0_attention_weight_min": -12.760031700134277,
      "activations/layer10_attention_weight_max": 36.51700210571289,
      "activations/layer10_attention_weight_min": -33.2638053894043,
      "activations/layer11_attention_weight_max": 36.32000732421875,
      "activations/layer11_attention_weight_min": -31.175045013427734,
      "activations/layer12_attention_weight_max": 24.32661247253418,
      "activations/layer12_attention_weight_min": -26.106416702270508,
      "activations/layer13_attention_weight_max": 41.076324462890625,
      "activations/layer13_attention_weight_min": -32.063873291015625,
      "activations/layer14_attention_weight_max": 44.0021858215332,
      "activations/layer14_attention_weight_min": -36.30609130859375,
      "activations/layer15_attention_weight_max": 35.730140686035156,
      "activations/layer15_attention_weight_min": -29.2723331451416,
      "activations/layer16_attention_weight_max": 31.659826278686523,
      "activations/layer16_attention_weight_min": -27.016939163208008,
      "activations/layer17_attention_weight_max": 31.942922592163086,
      "activations/layer17_attention_weight_min": -24.477766036987305,
      "activations/layer18_attention_weight_max": 28.981834411621094,
      "activations/layer18_attention_weight_min": -21.58323097229004,
      "activations/layer19_attention_weight_max": 32.67165756225586,
      "activations/layer19_attention_weight_min": -29.08582878112793,
      "activations/layer1_attention_weight_max": 17.201473236083984,
      "activations/layer1_attention_weight_min": -15.250226020812988,
      "activations/layer20_attention_weight_max": 27.086376190185547,
      "activations/layer20_attention_weight_min": -23.22453498840332,
      "activations/layer21_attention_weight_max": 27.506107330322266,
      "activations/layer21_attention_weight_min": -22.624624252319336,
      "activations/layer22_attention_weight_max": 42.38604736328125,
      "activations/layer22_attention_weight_min": -33.652931213378906,
      "activations/layer23_attention_weight_max": 33.49939727783203,
      "activations/layer23_attention_weight_min": -24.47931480407715,
      "activations/layer2_attention_weight_max": 33.33123779296875,
      "activations/layer2_attention_weight_min": -33.1466178894043,
      "activations/layer3_attention_weight_max": 91.80105590820312,
      "activations/layer3_attention_weight_min": -96.64576721191406,
      "activations/layer4_attention_weight_max": 115.9112777709961,
      "activations/layer4_attention_weight_min": -115.44771575927734,
      "activations/layer5_attention_weight_max": 55.60049057006836,
      "activations/layer5_attention_weight_min": -62.09474182128906,
      "activations/layer6_attention_weight_max": 46.89683532714844,
      "activations/layer6_attention_weight_min": -54.26749801635742,
      "activations/layer7_attention_weight_max": 90.96625518798828,
      "activations/layer7_attention_weight_min": -90.58517456054688,
      "activations/layer8_attention_weight_max": 41.87965774536133,
      "activations/layer8_attention_weight_min": -40.92378616333008,
      "activations/layer9_attention_weight_max": 38.31926727294922,
      "activations/layer9_attention_weight_min": -34.60232162475586,
      "epoch": 8.05,
      "learning_rate": 9.906174242424241e-05,
      "loss": 2.8175,
      "step": 138600
    },
    {
      "activations/layer0_attention_weight_max": 17.416305541992188,
      "activations/layer0_attention_weight_min": -12.5352201461792,
      "activations/layer10_attention_weight_max": 37.0433235168457,
      "activations/layer10_attention_weight_min": -34.71038055419922,
      "activations/layer11_attention_weight_max": 34.744483947753906,
      "activations/layer11_attention_weight_min": -33.0419807434082,
      "activations/layer12_attention_weight_max": 24.79464340209961,
      "activations/layer12_attention_weight_min": -25.233707427978516,
      "activations/layer13_attention_weight_max": 41.93617248535156,
      "activations/layer13_attention_weight_min": -32.99117660522461,
      "activations/layer14_attention_weight_max": 51.97749328613281,
      "activations/layer14_attention_weight_min": -39.079769134521484,
      "activations/layer15_attention_weight_max": 38.240562438964844,
      "activations/layer15_attention_weight_min": -28.912595748901367,
      "activations/layer16_attention_weight_max": 35.7881965637207,
      "activations/layer16_attention_weight_min": -26.680776596069336,
      "activations/layer17_attention_weight_max": 35.51033020019531,
      "activations/layer17_attention_weight_min": -24.797151565551758,
      "activations/layer18_attention_weight_max": 30.875036239624023,
      "activations/layer18_attention_weight_min": -23.960901260375977,
      "activations/layer19_attention_weight_max": 38.64421844482422,
      "activations/layer19_attention_weight_min": -31.90532875061035,
      "activations/layer1_attention_weight_max": 16.214439392089844,
      "activations/layer1_attention_weight_min": -14.910533905029297,
      "activations/layer20_attention_weight_max": 33.88365173339844,
      "activations/layer20_attention_weight_min": -24.907730102539062,
      "activations/layer21_attention_weight_max": 32.86058044433594,
      "activations/layer21_attention_weight_min": -23.400609970092773,
      "activations/layer22_attention_weight_max": 50.88271713256836,
      "activations/layer22_attention_weight_min": -27.006542205810547,
      "activations/layer23_attention_weight_max": 38.39928436279297,
      "activations/layer23_attention_weight_min": -23.22254753112793,
      "activations/layer2_attention_weight_max": 30.978404998779297,
      "activations/layer2_attention_weight_min": -31.249900817871094,
      "activations/layer3_attention_weight_max": 95.9786376953125,
      "activations/layer3_attention_weight_min": -93.6430435180664,
      "activations/layer4_attention_weight_max": 124.88079833984375,
      "activations/layer4_attention_weight_min": -124.64034271240234,
      "activations/layer5_attention_weight_max": 62.19114685058594,
      "activations/layer5_attention_weight_min": -64.1969985961914,
      "activations/layer6_attention_weight_max": 51.464149475097656,
      "activations/layer6_attention_weight_min": -55.097137451171875,
      "activations/layer7_attention_weight_max": 97.02214813232422,
      "activations/layer7_attention_weight_min": -96.55686950683594,
      "activations/layer8_attention_weight_max": 42.92609786987305,
      "activations/layer8_attention_weight_min": -42.96457290649414,
      "activations/layer9_attention_weight_max": 37.85100173950195,
      "activations/layer9_attention_weight_min": -35.163509368896484,
      "epoch": 8.06,
      "learning_rate": 9.904280303030303e-05,
      "loss": 2.8016,
      "step": 138650
    },
    {
      "activations/layer0_attention_weight_max": 16.513805389404297,
      "activations/layer0_attention_weight_min": -12.714164733886719,
      "activations/layer10_attention_weight_max": 32.833370208740234,
      "activations/layer10_attention_weight_min": -31.31534767150879,
      "activations/layer11_attention_weight_max": 32.829795837402344,
      "activations/layer11_attention_weight_min": -30.77028465270996,
      "activations/layer12_attention_weight_max": 23.504600524902344,
      "activations/layer12_attention_weight_min": -28.004030227661133,
      "activations/layer13_attention_weight_max": 42.6423454284668,
      "activations/layer13_attention_weight_min": -34.09700012207031,
      "activations/layer14_attention_weight_max": 50.65407943725586,
      "activations/layer14_attention_weight_min": -38.720542907714844,
      "activations/layer15_attention_weight_max": 37.60891342163086,
      "activations/layer15_attention_weight_min": -28.257766723632812,
      "activations/layer16_attention_weight_max": 32.78855514526367,
      "activations/layer16_attention_weight_min": -26.03057098388672,
      "activations/layer17_attention_weight_max": 33.06619644165039,
      "activations/layer17_attention_weight_min": -22.559534072875977,
      "activations/layer18_attention_weight_max": 27.873722076416016,
      "activations/layer18_attention_weight_min": -18.998226165771484,
      "activations/layer19_attention_weight_max": 38.54875183105469,
      "activations/layer19_attention_weight_min": -28.232913970947266,
      "activations/layer1_attention_weight_max": 16.762361526489258,
      "activations/layer1_attention_weight_min": -14.795764923095703,
      "activations/layer20_attention_weight_max": 28.096107482910156,
      "activations/layer20_attention_weight_min": -22.93921661376953,
      "activations/layer21_attention_weight_max": 30.7852840423584,
      "activations/layer21_attention_weight_min": -21.45305633544922,
      "activations/layer22_attention_weight_max": 46.2381706237793,
      "activations/layer22_attention_weight_min": -26.679468154907227,
      "activations/layer23_attention_weight_max": 35.06591796875,
      "activations/layer23_attention_weight_min": -21.538116455078125,
      "activations/layer2_attention_weight_max": 32.69990539550781,
      "activations/layer2_attention_weight_min": -31.856182098388672,
      "activations/layer3_attention_weight_max": 97.20352172851562,
      "activations/layer3_attention_weight_min": -92.54573822021484,
      "activations/layer4_attention_weight_max": 127.32054901123047,
      "activations/layer4_attention_weight_min": -119.70111083984375,
      "activations/layer5_attention_weight_max": 60.21378707885742,
      "activations/layer5_attention_weight_min": -61.33488082885742,
      "activations/layer6_attention_weight_max": 52.76929473876953,
      "activations/layer6_attention_weight_min": -52.520076751708984,
      "activations/layer7_attention_weight_max": 89.17564392089844,
      "activations/layer7_attention_weight_min": -89.10560607910156,
      "activations/layer8_attention_weight_max": 39.818946838378906,
      "activations/layer8_attention_weight_min": -40.50675582885742,
      "activations/layer9_attention_weight_max": 34.49345397949219,
      "activations/layer9_attention_weight_min": -32.32041549682617,
      "epoch": 8.06,
      "learning_rate": 9.902386363636363e-05,
      "loss": 2.8011,
      "step": 138700
    },
    {
      "activations/layer0_attention_weight_max": 15.391846656799316,
      "activations/layer0_attention_weight_min": -11.906744956970215,
      "activations/layer10_attention_weight_max": 35.73878860473633,
      "activations/layer10_attention_weight_min": -33.19691467285156,
      "activations/layer11_attention_weight_max": 37.873130798339844,
      "activations/layer11_attention_weight_min": -31.1065731048584,
      "activations/layer12_attention_weight_max": 24.492481231689453,
      "activations/layer12_attention_weight_min": -31.99175262451172,
      "activations/layer13_attention_weight_max": 44.216495513916016,
      "activations/layer13_attention_weight_min": -32.542274475097656,
      "activations/layer14_attention_weight_max": 61.868011474609375,
      "activations/layer14_attention_weight_min": -43.77731704711914,
      "activations/layer15_attention_weight_max": 40.388065338134766,
      "activations/layer15_attention_weight_min": -31.90778350830078,
      "activations/layer16_attention_weight_max": 35.446895599365234,
      "activations/layer16_attention_weight_min": -29.7067813873291,
      "activations/layer17_attention_weight_max": 38.850242614746094,
      "activations/layer17_attention_weight_min": -27.411495208740234,
      "activations/layer18_attention_weight_max": 32.80910873413086,
      "activations/layer18_attention_weight_min": -22.458959579467773,
      "activations/layer19_attention_weight_max": 44.149288177490234,
      "activations/layer19_attention_weight_min": -32.22214889526367,
      "activations/layer1_attention_weight_max": 17.165891647338867,
      "activations/layer1_attention_weight_min": -13.147713661193848,
      "activations/layer20_attention_weight_max": 35.416725158691406,
      "activations/layer20_attention_weight_min": -27.795989990234375,
      "activations/layer21_attention_weight_max": 34.13600158691406,
      "activations/layer21_attention_weight_min": -23.8546085357666,
      "activations/layer22_attention_weight_max": 54.02301025390625,
      "activations/layer22_attention_weight_min": -29.652233123779297,
      "activations/layer23_attention_weight_max": 44.822547912597656,
      "activations/layer23_attention_weight_min": -24.34633445739746,
      "activations/layer2_attention_weight_max": 33.11328125,
      "activations/layer2_attention_weight_min": -33.852561950683594,
      "activations/layer3_attention_weight_max": 96.46202850341797,
      "activations/layer3_attention_weight_min": -95.3067855834961,
      "activations/layer4_attention_weight_max": 124.18578338623047,
      "activations/layer4_attention_weight_min": -123.43373107910156,
      "activations/layer5_attention_weight_max": 59.399871826171875,
      "activations/layer5_attention_weight_min": -64.11538696289062,
      "activations/layer6_attention_weight_max": 49.567359924316406,
      "activations/layer6_attention_weight_min": -54.827125549316406,
      "activations/layer7_attention_weight_max": 89.46407318115234,
      "activations/layer7_attention_weight_min": -90.47017669677734,
      "activations/layer8_attention_weight_max": 41.91516876220703,
      "activations/layer8_attention_weight_min": -43.25092315673828,
      "activations/layer9_attention_weight_max": 37.16923141479492,
      "activations/layer9_attention_weight_min": -36.31749725341797,
      "epoch": 8.06,
      "learning_rate": 9.900492424242423e-05,
      "loss": 2.8097,
      "step": 138750
    },
    {
      "activations/layer0_attention_weight_max": 14.505614280700684,
      "activations/layer0_attention_weight_min": -13.113529205322266,
      "activations/layer10_attention_weight_max": 31.758108139038086,
      "activations/layer10_attention_weight_min": -31.068695068359375,
      "activations/layer11_attention_weight_max": 31.391063690185547,
      "activations/layer11_attention_weight_min": -29.539127349853516,
      "activations/layer12_attention_weight_max": 24.104265213012695,
      "activations/layer12_attention_weight_min": -29.911510467529297,
      "activations/layer13_attention_weight_max": 39.940757751464844,
      "activations/layer13_attention_weight_min": -33.630001068115234,
      "activations/layer14_attention_weight_max": 55.75606155395508,
      "activations/layer14_attention_weight_min": -39.64479064941406,
      "activations/layer15_attention_weight_max": 35.930686950683594,
      "activations/layer15_attention_weight_min": -29.13270378112793,
      "activations/layer16_attention_weight_max": 34.09566116333008,
      "activations/layer16_attention_weight_min": -26.442142486572266,
      "activations/layer17_attention_weight_max": 35.33786392211914,
      "activations/layer17_attention_weight_min": -25.875288009643555,
      "activations/layer18_attention_weight_max": 30.786029815673828,
      "activations/layer18_attention_weight_min": -20.634963989257812,
      "activations/layer19_attention_weight_max": 32.50339889526367,
      "activations/layer19_attention_weight_min": -28.26913833618164,
      "activations/layer1_attention_weight_max": 18.49650764465332,
      "activations/layer1_attention_weight_min": -13.100634574890137,
      "activations/layer20_attention_weight_max": 29.36277961730957,
      "activations/layer20_attention_weight_min": -24.1221923828125,
      "activations/layer21_attention_weight_max": 25.64413833618164,
      "activations/layer21_attention_weight_min": -21.897018432617188,
      "activations/layer22_attention_weight_max": 49.86474609375,
      "activations/layer22_attention_weight_min": -28.710309982299805,
      "activations/layer23_attention_weight_max": 35.596534729003906,
      "activations/layer23_attention_weight_min": -22.89337158203125,
      "activations/layer2_attention_weight_max": 32.09575653076172,
      "activations/layer2_attention_weight_min": -32.82019805908203,
      "activations/layer3_attention_weight_max": 93.97694396972656,
      "activations/layer3_attention_weight_min": -95.93325805664062,
      "activations/layer4_attention_weight_max": 120.40992736816406,
      "activations/layer4_attention_weight_min": -120.52742767333984,
      "activations/layer5_attention_weight_max": 57.83470153808594,
      "activations/layer5_attention_weight_min": -62.86503219604492,
      "activations/layer6_attention_weight_max": 47.90404510498047,
      "activations/layer6_attention_weight_min": -53.3250617980957,
      "activations/layer7_attention_weight_max": 85.6543197631836,
      "activations/layer7_attention_weight_min": -92.35369873046875,
      "activations/layer8_attention_weight_max": 38.69355392456055,
      "activations/layer8_attention_weight_min": -39.32280349731445,
      "activations/layer9_attention_weight_max": 33.458465576171875,
      "activations/layer9_attention_weight_min": -33.88029098510742,
      "epoch": 8.07,
      "learning_rate": 9.898598484848484e-05,
      "loss": 2.809,
      "step": 138800
    },
    {
      "activations/layer0_attention_weight_max": 15.895242691040039,
      "activations/layer0_attention_weight_min": -12.362464904785156,
      "activations/layer10_attention_weight_max": 31.73970603942871,
      "activations/layer10_attention_weight_min": -30.56967544555664,
      "activations/layer11_attention_weight_max": 31.790725708007812,
      "activations/layer11_attention_weight_min": -30.371091842651367,
      "activations/layer12_attention_weight_max": 24.636436462402344,
      "activations/layer12_attention_weight_min": -24.814849853515625,
      "activations/layer13_attention_weight_max": 38.82514953613281,
      "activations/layer13_attention_weight_min": -31.97332000732422,
      "activations/layer14_attention_weight_max": 44.30217361450195,
      "activations/layer14_attention_weight_min": -39.080684661865234,
      "activations/layer15_attention_weight_max": 37.36622619628906,
      "activations/layer15_attention_weight_min": -29.284656524658203,
      "activations/layer16_attention_weight_max": 34.7890510559082,
      "activations/layer16_attention_weight_min": -27.461938858032227,
      "activations/layer17_attention_weight_max": 36.45549392700195,
      "activations/layer17_attention_weight_min": -24.358396530151367,
      "activations/layer18_attention_weight_max": 32.50634765625,
      "activations/layer18_attention_weight_min": -22.9370174407959,
      "activations/layer19_attention_weight_max": 40.28669738769531,
      "activations/layer19_attention_weight_min": -31.982709884643555,
      "activations/layer1_attention_weight_max": 17.113908767700195,
      "activations/layer1_attention_weight_min": -14.537149429321289,
      "activations/layer20_attention_weight_max": 36.61715316772461,
      "activations/layer20_attention_weight_min": -23.318389892578125,
      "activations/layer21_attention_weight_max": 32.43619155883789,
      "activations/layer21_attention_weight_min": -23.27382469177246,
      "activations/layer22_attention_weight_max": 52.77947998046875,
      "activations/layer22_attention_weight_min": -29.50409698486328,
      "activations/layer23_attention_weight_max": 36.398529052734375,
      "activations/layer23_attention_weight_min": -23.342063903808594,
      "activations/layer2_attention_weight_max": 32.090965270996094,
      "activations/layer2_attention_weight_min": -32.13325500488281,
      "activations/layer3_attention_weight_max": 88.92938995361328,
      "activations/layer3_attention_weight_min": -96.24779510498047,
      "activations/layer4_attention_weight_max": 113.3868179321289,
      "activations/layer4_attention_weight_min": -115.0940170288086,
      "activations/layer5_attention_weight_max": 55.31627655029297,
      "activations/layer5_attention_weight_min": -62.84466552734375,
      "activations/layer6_attention_weight_max": 46.749366760253906,
      "activations/layer6_attention_weight_min": -49.81971740722656,
      "activations/layer7_attention_weight_max": 86.00104522705078,
      "activations/layer7_attention_weight_min": -86.52538299560547,
      "activations/layer8_attention_weight_max": 38.570735931396484,
      "activations/layer8_attention_weight_min": -38.07186508178711,
      "activations/layer9_attention_weight_max": 32.61360549926758,
      "activations/layer9_attention_weight_min": -32.202632904052734,
      "epoch": 8.07,
      "learning_rate": 9.896704545454544e-05,
      "loss": 2.8201,
      "step": 138850
    },
    {
      "activations/layer0_attention_weight_max": 14.903329849243164,
      "activations/layer0_attention_weight_min": -11.7134428024292,
      "activations/layer10_attention_weight_max": 35.295082092285156,
      "activations/layer10_attention_weight_min": -33.73991775512695,
      "activations/layer11_attention_weight_max": 33.6766471862793,
      "activations/layer11_attention_weight_min": -30.018798828125,
      "activations/layer12_attention_weight_max": 24.048553466796875,
      "activations/layer12_attention_weight_min": -23.302425384521484,
      "activations/layer13_attention_weight_max": 39.877845764160156,
      "activations/layer13_attention_weight_min": -33.74803161621094,
      "activations/layer14_attention_weight_max": 48.991512298583984,
      "activations/layer14_attention_weight_min": -40.62592697143555,
      "activations/layer15_attention_weight_max": 39.74025344848633,
      "activations/layer15_attention_weight_min": -31.925209045410156,
      "activations/layer16_attention_weight_max": 32.79395294189453,
      "activations/layer16_attention_weight_min": -27.73322105407715,
      "activations/layer17_attention_weight_max": 41.258026123046875,
      "activations/layer17_attention_weight_min": -25.5383243560791,
      "activations/layer18_attention_weight_max": 36.169647216796875,
      "activations/layer18_attention_weight_min": -21.751108169555664,
      "activations/layer19_attention_weight_max": 38.38200759887695,
      "activations/layer19_attention_weight_min": -30.8309383392334,
      "activations/layer1_attention_weight_max": 17.49810791015625,
      "activations/layer1_attention_weight_min": -13.824652671813965,
      "activations/layer20_attention_weight_max": 36.70915603637695,
      "activations/layer20_attention_weight_min": -25.05977439880371,
      "activations/layer21_attention_weight_max": 33.03547286987305,
      "activations/layer21_attention_weight_min": -24.08247947692871,
      "activations/layer22_attention_weight_max": 53.851654052734375,
      "activations/layer22_attention_weight_min": -29.554182052612305,
      "activations/layer23_attention_weight_max": 37.77347946166992,
      "activations/layer23_attention_weight_min": -22.937519073486328,
      "activations/layer2_attention_weight_max": 30.591413497924805,
      "activations/layer2_attention_weight_min": -29.688682556152344,
      "activations/layer3_attention_weight_max": 92.48052978515625,
      "activations/layer3_attention_weight_min": -89.82787322998047,
      "activations/layer4_attention_weight_max": 123.13618469238281,
      "activations/layer4_attention_weight_min": -122.8123550415039,
      "activations/layer5_attention_weight_max": 58.70579147338867,
      "activations/layer5_attention_weight_min": -64.45205688476562,
      "activations/layer6_attention_weight_max": 49.88679504394531,
      "activations/layer6_attention_weight_min": -54.31911849975586,
      "activations/layer7_attention_weight_max": 88.61358642578125,
      "activations/layer7_attention_weight_min": -90.8402328491211,
      "activations/layer8_attention_weight_max": 39.632080078125,
      "activations/layer8_attention_weight_min": -40.97359085083008,
      "activations/layer9_attention_weight_max": 35.31986618041992,
      "activations/layer9_attention_weight_min": -33.134098052978516,
      "epoch": 8.07,
      "learning_rate": 9.894810606060605e-05,
      "loss": 2.7874,
      "step": 138900
    },
    {
      "activations/layer0_attention_weight_max": 15.372501373291016,
      "activations/layer0_attention_weight_min": -11.926569938659668,
      "activations/layer10_attention_weight_max": 30.88983726501465,
      "activations/layer10_attention_weight_min": -30.234079360961914,
      "activations/layer11_attention_weight_max": 30.303075790405273,
      "activations/layer11_attention_weight_min": -28.29952621459961,
      "activations/layer12_attention_weight_max": 21.712650299072266,
      "activations/layer12_attention_weight_min": -24.64551544189453,
      "activations/layer13_attention_weight_max": 38.48616409301758,
      "activations/layer13_attention_weight_min": -30.81427574157715,
      "activations/layer14_attention_weight_max": 44.77876281738281,
      "activations/layer14_attention_weight_min": -39.53089141845703,
      "activations/layer15_attention_weight_max": 34.63846969604492,
      "activations/layer15_attention_weight_min": -28.127927780151367,
      "activations/layer16_attention_weight_max": 31.847110748291016,
      "activations/layer16_attention_weight_min": -27.76486587524414,
      "activations/layer17_attention_weight_max": 31.30963897705078,
      "activations/layer17_attention_weight_min": -24.872112274169922,
      "activations/layer18_attention_weight_max": 28.39323616027832,
      "activations/layer18_attention_weight_min": -20.320276260375977,
      "activations/layer19_attention_weight_max": 34.636802673339844,
      "activations/layer19_attention_weight_min": -30.36468505859375,
      "activations/layer1_attention_weight_max": 16.631202697753906,
      "activations/layer1_attention_weight_min": -15.246805191040039,
      "activations/layer20_attention_weight_max": 30.176395416259766,
      "activations/layer20_attention_weight_min": -25.877775192260742,
      "activations/layer21_attention_weight_max": 25.84008026123047,
      "activations/layer21_attention_weight_min": -21.495662689208984,
      "activations/layer22_attention_weight_max": 41.81690979003906,
      "activations/layer22_attention_weight_min": -29.97381591796875,
      "activations/layer23_attention_weight_max": 32.76630783081055,
      "activations/layer23_attention_weight_min": -23.341705322265625,
      "activations/layer2_attention_weight_max": 30.31447982788086,
      "activations/layer2_attention_weight_min": -31.140653610229492,
      "activations/layer3_attention_weight_max": 87.12446594238281,
      "activations/layer3_attention_weight_min": -88.53764343261719,
      "activations/layer4_attention_weight_max": 118.96527099609375,
      "activations/layer4_attention_weight_min": -119.28047180175781,
      "activations/layer5_attention_weight_max": 56.4063835144043,
      "activations/layer5_attention_weight_min": -62.82598876953125,
      "activations/layer6_attention_weight_max": 46.34817123413086,
      "activations/layer6_attention_weight_min": -52.43889236450195,
      "activations/layer7_attention_weight_max": 92.0202865600586,
      "activations/layer7_attention_weight_min": -86.9590835571289,
      "activations/layer8_attention_weight_max": 40.33491516113281,
      "activations/layer8_attention_weight_min": -38.551490783691406,
      "activations/layer9_attention_weight_max": 33.71760940551758,
      "activations/layer9_attention_weight_min": -33.05106735229492,
      "epoch": 8.07,
      "learning_rate": 9.892916666666666e-05,
      "loss": 2.8025,
      "step": 138950
    },
    {
      "activations/layer0_attention_weight_max": 16.275606155395508,
      "activations/layer0_attention_weight_min": -11.901392936706543,
      "activations/layer10_attention_weight_max": 32.82319259643555,
      "activations/layer10_attention_weight_min": -34.98247146606445,
      "activations/layer11_attention_weight_max": 34.612754821777344,
      "activations/layer11_attention_weight_min": -31.497173309326172,
      "activations/layer12_attention_weight_max": 24.91378402709961,
      "activations/layer12_attention_weight_min": -25.79062271118164,
      "activations/layer13_attention_weight_max": 45.063480377197266,
      "activations/layer13_attention_weight_min": -35.763038635253906,
      "activations/layer14_attention_weight_max": 59.96125411987305,
      "activations/layer14_attention_weight_min": -44.713645935058594,
      "activations/layer15_attention_weight_max": 40.14431381225586,
      "activations/layer15_attention_weight_min": -29.902620315551758,
      "activations/layer16_attention_weight_max": 35.396583557128906,
      "activations/layer16_attention_weight_min": -28.279340744018555,
      "activations/layer17_attention_weight_max": 35.6084098815918,
      "activations/layer17_attention_weight_min": -25.59344482421875,
      "activations/layer18_attention_weight_max": 30.923194885253906,
      "activations/layer18_attention_weight_min": -21.395954132080078,
      "activations/layer19_attention_weight_max": 38.7788200378418,
      "activations/layer19_attention_weight_min": -29.58637046813965,
      "activations/layer1_attention_weight_max": 17.036357879638672,
      "activations/layer1_attention_weight_min": -13.987308502197266,
      "activations/layer20_attention_weight_max": 32.361412048339844,
      "activations/layer20_attention_weight_min": -25.929920196533203,
      "activations/layer21_attention_weight_max": 31.28011131286621,
      "activations/layer21_attention_weight_min": -24.356224060058594,
      "activations/layer22_attention_weight_max": 52.57941436767578,
      "activations/layer22_attention_weight_min": -29.335168838500977,
      "activations/layer23_attention_weight_max": 40.377037048339844,
      "activations/layer23_attention_weight_min": -23.056560516357422,
      "activations/layer2_attention_weight_max": 31.734619140625,
      "activations/layer2_attention_weight_min": -29.794919967651367,
      "activations/layer3_attention_weight_max": 89.36166381835938,
      "activations/layer3_attention_weight_min": -93.19007873535156,
      "activations/layer4_attention_weight_max": 122.9150390625,
      "activations/layer4_attention_weight_min": -118.57746887207031,
      "activations/layer5_attention_weight_max": 58.66231918334961,
      "activations/layer5_attention_weight_min": -63.701560974121094,
      "activations/layer6_attention_weight_max": 51.56165313720703,
      "activations/layer6_attention_weight_min": -52.51256561279297,
      "activations/layer7_attention_weight_max": 87.71463012695312,
      "activations/layer7_attention_weight_min": -90.24996948242188,
      "activations/layer8_attention_weight_max": 39.634796142578125,
      "activations/layer8_attention_weight_min": -43.3360481262207,
      "activations/layer9_attention_weight_max": 33.899051666259766,
      "activations/layer9_attention_weight_min": -34.14202117919922,
      "epoch": 8.08,
      "learning_rate": 9.891022727272726e-05,
      "loss": 2.7937,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5144,
      "eval_samples_per_second": 504.319,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5144,
      "eval_openwebtext_samples_per_second": 504.319,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 2.0098,
      "eval_wikitext_samples_per_second": 226.891,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_lambada_loss": 2.646484375,
      "eval_lambada_ppl": 14.104365720113616,
      "eval_lambada_runtime": 9.5862,
      "eval_lambada_samples_per_second": 507.918,
      "step": 139000
    },
    {
      "activations/layer0_attention_weight_max": 15.369172096252441,
      "activations/layer0_attention_weight_min": -12.48743724822998,
      "activations/layer10_attention_weight_max": 33.687408447265625,
      "activations/layer10_attention_weight_min": -32.14240646362305,
      "activations/layer11_attention_weight_max": 33.59868240356445,
      "activations/layer11_attention_weight_min": -31.568126678466797,
      "activations/layer12_attention_weight_max": 25.871400833129883,
      "activations/layer12_attention_weight_min": -24.942073822021484,
      "activations/layer13_attention_weight_max": 39.23173904418945,
      "activations/layer13_attention_weight_min": -34.35722732543945,
      "activations/layer14_attention_weight_max": 52.89270782470703,
      "activations/layer14_attention_weight_min": -40.836692810058594,
      "activations/layer15_attention_weight_max": 37.84443664550781,
      "activations/layer15_attention_weight_min": -31.331159591674805,
      "activations/layer16_attention_weight_max": 35.198326110839844,
      "activations/layer16_attention_weight_min": -28.619853973388672,
      "activations/layer17_attention_weight_max": 36.47450637817383,
      "activations/layer17_attention_weight_min": -24.018314361572266,
      "activations/layer18_attention_weight_max": 31.27729606628418,
      "activations/layer18_attention_weight_min": -20.515811920166016,
      "activations/layer19_attention_weight_max": 35.9123649597168,
      "activations/layer19_attention_weight_min": -29.62637710571289,
      "activations/layer1_attention_weight_max": 16.301132202148438,
      "activations/layer1_attention_weight_min": -15.291348457336426,
      "activations/layer20_attention_weight_max": 31.15563201904297,
      "activations/layer20_attention_weight_min": -24.00067710876465,
      "activations/layer21_attention_weight_max": 31.124534606933594,
      "activations/layer21_attention_weight_min": -23.767105102539062,
      "activations/layer22_attention_weight_max": 45.10651397705078,
      "activations/layer22_attention_weight_min": -28.67513656616211,
      "activations/layer23_attention_weight_max": 37.46917724609375,
      "activations/layer23_attention_weight_min": -24.519933700561523,
      "activations/layer2_attention_weight_max": 33.243438720703125,
      "activations/layer2_attention_weight_min": -32.57799530029297,
      "activations/layer3_attention_weight_max": 90.89840698242188,
      "activations/layer3_attention_weight_min": -94.27017211914062,
      "activations/layer4_attention_weight_max": 124.30675506591797,
      "activations/layer4_attention_weight_min": -121.83312225341797,
      "activations/layer5_attention_weight_max": 57.257423400878906,
      "activations/layer5_attention_weight_min": -64.51656341552734,
      "activations/layer6_attention_weight_max": 50.083805084228516,
      "activations/layer6_attention_weight_min": -54.633304595947266,
      "activations/layer7_attention_weight_max": 91.01615905761719,
      "activations/layer7_attention_weight_min": -89.38093566894531,
      "activations/layer8_attention_weight_max": 42.341583251953125,
      "activations/layer8_attention_weight_min": -42.11330032348633,
      "activations/layer9_attention_weight_max": 36.317474365234375,
      "activations/layer9_attention_weight_min": -35.97801208496094,
      "epoch": 8.08,
      "learning_rate": 9.889128787878788e-05,
      "loss": 2.7911,
      "step": 139050
    },
    {
      "activations/layer0_attention_weight_max": 15.95155143737793,
      "activations/layer0_attention_weight_min": -11.891189575195312,
      "activations/layer10_attention_weight_max": 36.2257080078125,
      "activations/layer10_attention_weight_min": -30.9176082611084,
      "activations/layer11_attention_weight_max": 37.89670944213867,
      "activations/layer11_attention_weight_min": -30.945167541503906,
      "activations/layer12_attention_weight_max": 42.91983413696289,
      "activations/layer12_attention_weight_min": -25.681140899658203,
      "activations/layer13_attention_weight_max": 52.274044036865234,
      "activations/layer13_attention_weight_min": -30.655542373657227,
      "activations/layer14_attention_weight_max": 54.31233215332031,
      "activations/layer14_attention_weight_min": -38.02071762084961,
      "activations/layer15_attention_weight_max": 44.14596176147461,
      "activations/layer15_attention_weight_min": -29.478492736816406,
      "activations/layer16_attention_weight_max": 38.17734146118164,
      "activations/layer16_attention_weight_min": -27.619800567626953,
      "activations/layer17_attention_weight_max": 47.31051254272461,
      "activations/layer17_attention_weight_min": -26.06760597229004,
      "activations/layer18_attention_weight_max": 40.29896926879883,
      "activations/layer18_attention_weight_min": -21.550085067749023,
      "activations/layer19_attention_weight_max": 39.508358001708984,
      "activations/layer19_attention_weight_min": -32.40165328979492,
      "activations/layer1_attention_weight_max": 16.132951736450195,
      "activations/layer1_attention_weight_min": -15.430577278137207,
      "activations/layer20_attention_weight_max": 32.68779754638672,
      "activations/layer20_attention_weight_min": -27.639873504638672,
      "activations/layer21_attention_weight_max": 32.8390998840332,
      "activations/layer21_attention_weight_min": -24.267332077026367,
      "activations/layer22_attention_weight_max": 55.36369705200195,
      "activations/layer22_attention_weight_min": -32.523441314697266,
      "activations/layer23_attention_weight_max": 40.486141204833984,
      "activations/layer23_attention_weight_min": -24.820903778076172,
      "activations/layer2_attention_weight_max": 31.75455093383789,
      "activations/layer2_attention_weight_min": -32.055179595947266,
      "activations/layer3_attention_weight_max": 90.92621612548828,
      "activations/layer3_attention_weight_min": -92.2496337890625,
      "activations/layer4_attention_weight_max": 122.66759490966797,
      "activations/layer4_attention_weight_min": -118.5052490234375,
      "activations/layer5_attention_weight_max": 56.38636016845703,
      "activations/layer5_attention_weight_min": -62.075355529785156,
      "activations/layer6_attention_weight_max": 48.28963851928711,
      "activations/layer6_attention_weight_min": -50.454566955566406,
      "activations/layer7_attention_weight_max": 87.00958251953125,
      "activations/layer7_attention_weight_min": -88.94336700439453,
      "activations/layer8_attention_weight_max": 39.06096649169922,
      "activations/layer8_attention_weight_min": -38.77155303955078,
      "activations/layer9_attention_weight_max": 34.377037048339844,
      "activations/layer9_attention_weight_min": -32.091033935546875,
      "epoch": 8.08,
      "learning_rate": 9.887234848484847e-05,
      "loss": 2.7993,
      "step": 139100
    },
    {
      "activations/layer0_attention_weight_max": 15.942887306213379,
      "activations/layer0_attention_weight_min": -12.072861671447754,
      "activations/layer10_attention_weight_max": 32.58671951293945,
      "activations/layer10_attention_weight_min": -32.32160186767578,
      "activations/layer11_attention_weight_max": 33.32579803466797,
      "activations/layer11_attention_weight_min": -29.431859970092773,
      "activations/layer12_attention_weight_max": 35.507080078125,
      "activations/layer12_attention_weight_min": -25.765335083007812,
      "activations/layer13_attention_weight_max": 37.926692962646484,
      "activations/layer13_attention_weight_min": -33.041202545166016,
      "activations/layer14_attention_weight_max": 49.300628662109375,
      "activations/layer14_attention_weight_min": -37.70127868652344,
      "activations/layer15_attention_weight_max": 37.44839859008789,
      "activations/layer15_attention_weight_min": -29.1612548828125,
      "activations/layer16_attention_weight_max": 33.44035720825195,
      "activations/layer16_attention_weight_min": -27.906164169311523,
      "activations/layer17_attention_weight_max": 37.52299880981445,
      "activations/layer17_attention_weight_min": -24.001407623291016,
      "activations/layer18_attention_weight_max": 29.194583892822266,
      "activations/layer18_attention_weight_min": -20.51654624938965,
      "activations/layer19_attention_weight_max": 37.86149978637695,
      "activations/layer19_attention_weight_min": -27.501251220703125,
      "activations/layer1_attention_weight_max": 16.4052791595459,
      "activations/layer1_attention_weight_min": -14.760331153869629,
      "activations/layer20_attention_weight_max": 31.995088577270508,
      "activations/layer20_attention_weight_min": -22.114408493041992,
      "activations/layer21_attention_weight_max": 31.13040542602539,
      "activations/layer21_attention_weight_min": -22.027429580688477,
      "activations/layer22_attention_weight_max": 47.16436767578125,
      "activations/layer22_attention_weight_min": -26.472497940063477,
      "activations/layer23_attention_weight_max": 36.0617561340332,
      "activations/layer23_attention_weight_min": -22.320547103881836,
      "activations/layer2_attention_weight_max": 32.41212463378906,
      "activations/layer2_attention_weight_min": -31.490652084350586,
      "activations/layer3_attention_weight_max": 92.5138168334961,
      "activations/layer3_attention_weight_min": -93.26972198486328,
      "activations/layer4_attention_weight_max": 120.4880142211914,
      "activations/layer4_attention_weight_min": -116.1910629272461,
      "activations/layer5_attention_weight_max": 61.4051628112793,
      "activations/layer5_attention_weight_min": -60.79069519042969,
      "activations/layer6_attention_weight_max": 51.30070114135742,
      "activations/layer6_attention_weight_min": -51.505027770996094,
      "activations/layer7_attention_weight_max": 91.35413360595703,
      "activations/layer7_attention_weight_min": -86.66126251220703,
      "activations/layer8_attention_weight_max": 39.77537155151367,
      "activations/layer8_attention_weight_min": -42.34141540527344,
      "activations/layer9_attention_weight_max": 33.841766357421875,
      "activations/layer9_attention_weight_min": -34.396175384521484,
      "epoch": 8.09,
      "learning_rate": 9.885340909090908e-05,
      "loss": 2.8108,
      "step": 139150
    },
    {
      "activations/layer0_attention_weight_max": 16.82616424560547,
      "activations/layer0_attention_weight_min": -12.056288719177246,
      "activations/layer10_attention_weight_max": 33.00377655029297,
      "activations/layer10_attention_weight_min": -31.79613494873047,
      "activations/layer11_attention_weight_max": 35.24079132080078,
      "activations/layer11_attention_weight_min": -29.539548873901367,
      "activations/layer12_attention_weight_max": 24.457557678222656,
      "activations/layer12_attention_weight_min": -24.374046325683594,
      "activations/layer13_attention_weight_max": 41.87559127807617,
      "activations/layer13_attention_weight_min": -32.30134963989258,
      "activations/layer14_attention_weight_max": 52.18086242675781,
      "activations/layer14_attention_weight_min": -41.6746711730957,
      "activations/layer15_attention_weight_max": 38.44074630737305,
      "activations/layer15_attention_weight_min": -29.521940231323242,
      "activations/layer16_attention_weight_max": 39.379173278808594,
      "activations/layer16_attention_weight_min": -27.60268211364746,
      "activations/layer17_attention_weight_max": 34.81353759765625,
      "activations/layer17_attention_weight_min": -26.225961685180664,
      "activations/layer18_attention_weight_max": 30.58140754699707,
      "activations/layer18_attention_weight_min": -21.38774871826172,
      "activations/layer19_attention_weight_max": 44.437355041503906,
      "activations/layer19_attention_weight_min": -29.207448959350586,
      "activations/layer1_attention_weight_max": 16.650819778442383,
      "activations/layer1_attention_weight_min": -15.987639427185059,
      "activations/layer20_attention_weight_max": 31.846405029296875,
      "activations/layer20_attention_weight_min": -23.052026748657227,
      "activations/layer21_attention_weight_max": 30.588451385498047,
      "activations/layer21_attention_weight_min": -22.180816650390625,
      "activations/layer22_attention_weight_max": 54.30026626586914,
      "activations/layer22_attention_weight_min": -29.74985122680664,
      "activations/layer23_attention_weight_max": 39.70682144165039,
      "activations/layer23_attention_weight_min": -24.902759552001953,
      "activations/layer2_attention_weight_max": 31.48927879333496,
      "activations/layer2_attention_weight_min": -32.38877868652344,
      "activations/layer3_attention_weight_max": 90.49349975585938,
      "activations/layer3_attention_weight_min": -97.27046203613281,
      "activations/layer4_attention_weight_max": 118.9839096069336,
      "activations/layer4_attention_weight_min": -119.279541015625,
      "activations/layer5_attention_weight_max": 57.3149528503418,
      "activations/layer5_attention_weight_min": -63.20861053466797,
      "activations/layer6_attention_weight_max": 49.32679748535156,
      "activations/layer6_attention_weight_min": -55.021888732910156,
      "activations/layer7_attention_weight_max": 87.25218963623047,
      "activations/layer7_attention_weight_min": -97.38867950439453,
      "activations/layer8_attention_weight_max": 39.79123306274414,
      "activations/layer8_attention_weight_min": -41.36467742919922,
      "activations/layer9_attention_weight_max": 34.140743255615234,
      "activations/layer9_attention_weight_min": -34.91115951538086,
      "epoch": 8.09,
      "learning_rate": 9.88344696969697e-05,
      "loss": 2.7939,
      "step": 139200
    },
    {
      "activations/layer0_attention_weight_max": 16.19915008544922,
      "activations/layer0_attention_weight_min": -11.010937690734863,
      "activations/layer10_attention_weight_max": 33.53536605834961,
      "activations/layer10_attention_weight_min": -31.2113094329834,
      "activations/layer11_attention_weight_max": 32.17702102661133,
      "activations/layer11_attention_weight_min": -29.576457977294922,
      "activations/layer12_attention_weight_max": 23.3270320892334,
      "activations/layer12_attention_weight_min": -24.221725463867188,
      "activations/layer13_attention_weight_max": 43.54093551635742,
      "activations/layer13_attention_weight_min": -32.9653434753418,
      "activations/layer14_attention_weight_max": 45.58147048950195,
      "activations/layer14_attention_weight_min": -42.20844650268555,
      "activations/layer15_attention_weight_max": 37.92238998413086,
      "activations/layer15_attention_weight_min": -29.036434173583984,
      "activations/layer16_attention_weight_max": 33.5052604675293,
      "activations/layer16_attention_weight_min": -27.802446365356445,
      "activations/layer17_attention_weight_max": 33.3101921081543,
      "activations/layer17_attention_weight_min": -24.47605323791504,
      "activations/layer18_attention_weight_max": 32.30729293823242,
      "activations/layer18_attention_weight_min": -23.30580711364746,
      "activations/layer19_attention_weight_max": 32.8795166015625,
      "activations/layer19_attention_weight_min": -32.02572250366211,
      "activations/layer1_attention_weight_max": 17.783153533935547,
      "activations/layer1_attention_weight_min": -16.317550659179688,
      "activations/layer20_attention_weight_max": 27.942522048950195,
      "activations/layer20_attention_weight_min": -24.176132202148438,
      "activations/layer21_attention_weight_max": 29.082183837890625,
      "activations/layer21_attention_weight_min": -22.849353790283203,
      "activations/layer22_attention_weight_max": 43.90519332885742,
      "activations/layer22_attention_weight_min": -29.393369674682617,
      "activations/layer23_attention_weight_max": 35.467185974121094,
      "activations/layer23_attention_weight_min": -26.29167938232422,
      "activations/layer2_attention_weight_max": 33.199825286865234,
      "activations/layer2_attention_weight_min": -31.653099060058594,
      "activations/layer3_attention_weight_max": 97.70651245117188,
      "activations/layer3_attention_weight_min": -97.99910736083984,
      "activations/layer4_attention_weight_max": 123.1212158203125,
      "activations/layer4_attention_weight_min": -119.51885986328125,
      "activations/layer5_attention_weight_max": 62.311885833740234,
      "activations/layer5_attention_weight_min": -64.89297485351562,
      "activations/layer6_attention_weight_max": 51.02338790893555,
      "activations/layer6_attention_weight_min": -52.40669631958008,
      "activations/layer7_attention_weight_max": 91.63033294677734,
      "activations/layer7_attention_weight_min": -90.88079071044922,
      "activations/layer8_attention_weight_max": 42.17253494262695,
      "activations/layer8_attention_weight_min": -42.41179275512695,
      "activations/layer9_attention_weight_max": 36.96430587768555,
      "activations/layer9_attention_weight_min": -35.315513610839844,
      "epoch": 8.09,
      "learning_rate": 9.881553030303028e-05,
      "loss": 2.8133,
      "step": 139250
    },
    {
      "activations/layer0_attention_weight_max": 16.852025985717773,
      "activations/layer0_attention_weight_min": -13.327003479003906,
      "activations/layer10_attention_weight_max": 34.681861877441406,
      "activations/layer10_attention_weight_min": -33.906158447265625,
      "activations/layer11_attention_weight_max": 35.050750732421875,
      "activations/layer11_attention_weight_min": -31.930952072143555,
      "activations/layer12_attention_weight_max": 25.44769287109375,
      "activations/layer12_attention_weight_min": -24.870092391967773,
      "activations/layer13_attention_weight_max": 45.593753814697266,
      "activations/layer13_attention_weight_min": -35.32322692871094,
      "activations/layer14_attention_weight_max": 66.02184295654297,
      "activations/layer14_attention_weight_min": -43.84754943847656,
      "activations/layer15_attention_weight_max": 43.534950256347656,
      "activations/layer15_attention_weight_min": -33.81318283081055,
      "activations/layer16_attention_weight_max": 40.82645034790039,
      "activations/layer16_attention_weight_min": -31.763072967529297,
      "activations/layer17_attention_weight_max": 36.19597244262695,
      "activations/layer17_attention_weight_min": -25.498741149902344,
      "activations/layer18_attention_weight_max": 34.52648162841797,
      "activations/layer18_attention_weight_min": -21.874452590942383,
      "activations/layer19_attention_weight_max": 49.6058235168457,
      "activations/layer19_attention_weight_min": -30.14474868774414,
      "activations/layer1_attention_weight_max": 16.22430419921875,
      "activations/layer1_attention_weight_min": -13.663820266723633,
      "activations/layer20_attention_weight_max": 38.973052978515625,
      "activations/layer20_attention_weight_min": -23.63602066040039,
      "activations/layer21_attention_weight_max": 35.4445686340332,
      "activations/layer21_attention_weight_min": -21.509151458740234,
      "activations/layer22_attention_weight_max": 58.23073959350586,
      "activations/layer22_attention_weight_min": -28.15614891052246,
      "activations/layer23_attention_weight_max": 42.04212188720703,
      "activations/layer23_attention_weight_min": -25.162364959716797,
      "activations/layer2_attention_weight_max": 32.315513610839844,
      "activations/layer2_attention_weight_min": -31.912925720214844,
      "activations/layer3_attention_weight_max": 95.00686645507812,
      "activations/layer3_attention_weight_min": -93.77485656738281,
      "activations/layer4_attention_weight_max": 120.67750549316406,
      "activations/layer4_attention_weight_min": -118.2499008178711,
      "activations/layer5_attention_weight_max": 59.399410247802734,
      "activations/layer5_attention_weight_min": -61.321372985839844,
      "activations/layer6_attention_weight_max": 50.981300354003906,
      "activations/layer6_attention_weight_min": -53.202659606933594,
      "activations/layer7_attention_weight_max": 96.3910140991211,
      "activations/layer7_attention_weight_min": -88.22850799560547,
      "activations/layer8_attention_weight_max": 43.70889663696289,
      "activations/layer8_attention_weight_min": -41.238826751708984,
      "activations/layer9_attention_weight_max": 37.50888442993164,
      "activations/layer9_attention_weight_min": -34.76271438598633,
      "epoch": 8.09,
      "learning_rate": 9.87965909090909e-05,
      "loss": 2.801,
      "step": 139300
    },
    {
      "activations/layer0_attention_weight_max": 16.358795166015625,
      "activations/layer0_attention_weight_min": -11.842272758483887,
      "activations/layer10_attention_weight_max": 33.91365051269531,
      "activations/layer10_attention_weight_min": -35.244598388671875,
      "activations/layer11_attention_weight_max": 33.4634895324707,
      "activations/layer11_attention_weight_min": -32.008331298828125,
      "activations/layer12_attention_weight_max": 23.97218132019043,
      "activations/layer12_attention_weight_min": -30.523700714111328,
      "activations/layer13_attention_weight_max": 40.97892379760742,
      "activations/layer13_attention_weight_min": -31.875497817993164,
      "activations/layer14_attention_weight_max": 51.93683624267578,
      "activations/layer14_attention_weight_min": -47.059444427490234,
      "activations/layer15_attention_weight_max": 39.179500579833984,
      "activations/layer15_attention_weight_min": -29.466812133789062,
      "activations/layer16_attention_weight_max": 35.97334289550781,
      "activations/layer16_attention_weight_min": -28.65889549255371,
      "activations/layer17_attention_weight_max": 35.2794075012207,
      "activations/layer17_attention_weight_min": -26.036449432373047,
      "activations/layer18_attention_weight_max": 31.4814510345459,
      "activations/layer18_attention_weight_min": -19.989978790283203,
      "activations/layer19_attention_weight_max": 36.593379974365234,
      "activations/layer19_attention_weight_min": -28.651182174682617,
      "activations/layer1_attention_weight_max": 16.17560386657715,
      "activations/layer1_attention_weight_min": -13.733804702758789,
      "activations/layer20_attention_weight_max": 30.064563751220703,
      "activations/layer20_attention_weight_min": -23.553699493408203,
      "activations/layer21_attention_weight_max": 30.2502498626709,
      "activations/layer21_attention_weight_min": -22.061054229736328,
      "activations/layer22_attention_weight_max": 47.45808792114258,
      "activations/layer22_attention_weight_min": -30.063825607299805,
      "activations/layer23_attention_weight_max": 34.3850212097168,
      "activations/layer23_attention_weight_min": -24.052030563354492,
      "activations/layer2_attention_weight_max": 31.819618225097656,
      "activations/layer2_attention_weight_min": -31.327983856201172,
      "activations/layer3_attention_weight_max": 93.84656524658203,
      "activations/layer3_attention_weight_min": -95.67182922363281,
      "activations/layer4_attention_weight_max": 118.17061614990234,
      "activations/layer4_attention_weight_min": -118.47962951660156,
      "activations/layer5_attention_weight_max": 58.077125549316406,
      "activations/layer5_attention_weight_min": -61.49053955078125,
      "activations/layer6_attention_weight_max": 47.89704513549805,
      "activations/layer6_attention_weight_min": -51.055301666259766,
      "activations/layer7_attention_weight_max": 88.56478881835938,
      "activations/layer7_attention_weight_min": -91.58073425292969,
      "activations/layer8_attention_weight_max": 39.34392166137695,
      "activations/layer8_attention_weight_min": -39.509117126464844,
      "activations/layer9_attention_weight_max": 35.003509521484375,
      "activations/layer9_attention_weight_min": -33.9625358581543,
      "epoch": 8.1,
      "learning_rate": 9.877803030303029e-05,
      "loss": 2.7893,
      "step": 139350
    },
    {
      "activations/layer0_attention_weight_max": 15.99656867980957,
      "activations/layer0_attention_weight_min": -11.360589981079102,
      "activations/layer10_attention_weight_max": 30.78160858154297,
      "activations/layer10_attention_weight_min": -32.494808197021484,
      "activations/layer11_attention_weight_max": 32.10862350463867,
      "activations/layer11_attention_weight_min": -30.34345054626465,
      "activations/layer12_attention_weight_max": 25.70047378540039,
      "activations/layer12_attention_weight_min": -25.70933723449707,
      "activations/layer13_attention_weight_max": 36.438232421875,
      "activations/layer13_attention_weight_min": -33.79701614379883,
      "activations/layer14_attention_weight_max": 47.77427291870117,
      "activations/layer14_attention_weight_min": -41.174652099609375,
      "activations/layer15_attention_weight_max": 34.69662094116211,
      "activations/layer15_attention_weight_min": -31.122968673706055,
      "activations/layer16_attention_weight_max": 29.9211483001709,
      "activations/layer16_attention_weight_min": -27.157005310058594,
      "activations/layer17_attention_weight_max": 30.751413345336914,
      "activations/layer17_attention_weight_min": -24.53157615661621,
      "activations/layer18_attention_weight_max": 28.994604110717773,
      "activations/layer18_attention_weight_min": -22.60472297668457,
      "activations/layer19_attention_weight_max": 32.311397552490234,
      "activations/layer19_attention_weight_min": -31.126646041870117,
      "activations/layer1_attention_weight_max": 16.634862899780273,
      "activations/layer1_attention_weight_min": -14.047930717468262,
      "activations/layer20_attention_weight_max": 29.142337799072266,
      "activations/layer20_attention_weight_min": -23.868375778198242,
      "activations/layer21_attention_weight_max": 25.725004196166992,
      "activations/layer21_attention_weight_min": -22.22507095336914,
      "activations/layer22_attention_weight_max": 42.69811248779297,
      "activations/layer22_attention_weight_min": -27.60624885559082,
      "activations/layer23_attention_weight_max": 31.306676864624023,
      "activations/layer23_attention_weight_min": -24.781599044799805,
      "activations/layer2_attention_weight_max": 31.332916259765625,
      "activations/layer2_attention_weight_min": -32.45542907714844,
      "activations/layer3_attention_weight_max": 90.2685317993164,
      "activations/layer3_attention_weight_min": -95.65164947509766,
      "activations/layer4_attention_weight_max": 120.69740295410156,
      "activations/layer4_attention_weight_min": -122.57567596435547,
      "activations/layer5_attention_weight_max": 60.77219772338867,
      "activations/layer5_attention_weight_min": -64.01691436767578,
      "activations/layer6_attention_weight_max": 51.20743179321289,
      "activations/layer6_attention_weight_min": -53.92470932006836,
      "activations/layer7_attention_weight_max": 90.2306137084961,
      "activations/layer7_attention_weight_min": -88.45137786865234,
      "activations/layer8_attention_weight_max": 40.652217864990234,
      "activations/layer8_attention_weight_min": -39.32307815551758,
      "activations/layer9_attention_weight_max": 33.6585578918457,
      "activations/layer9_attention_weight_min": -33.08149337768555,
      "epoch": 8.1,
      "learning_rate": 9.87590909090909e-05,
      "loss": 2.802,
      "step": 139400
    },
    {
      "activations/layer0_attention_weight_max": 16.15291404724121,
      "activations/layer0_attention_weight_min": -12.000683784484863,
      "activations/layer10_attention_weight_max": 35.64017868041992,
      "activations/layer10_attention_weight_min": -32.210968017578125,
      "activations/layer11_attention_weight_max": 32.42561340332031,
      "activations/layer11_attention_weight_min": -29.90917205810547,
      "activations/layer12_attention_weight_max": 23.073270797729492,
      "activations/layer12_attention_weight_min": -25.401721954345703,
      "activations/layer13_attention_weight_max": 38.37701416015625,
      "activations/layer13_attention_weight_min": -32.2412109375,
      "activations/layer14_attention_weight_max": 47.217281341552734,
      "activations/layer14_attention_weight_min": -39.03252410888672,
      "activations/layer15_attention_weight_max": 35.962181091308594,
      "activations/layer15_attention_weight_min": -29.819826126098633,
      "activations/layer16_attention_weight_max": 31.307239532470703,
      "activations/layer16_attention_weight_min": -26.689245223999023,
      "activations/layer17_attention_weight_max": 34.50386428833008,
      "activations/layer17_attention_weight_min": -24.493751525878906,
      "activations/layer18_attention_weight_max": 29.68779182434082,
      "activations/layer18_attention_weight_min": -21.20087242126465,
      "activations/layer19_attention_weight_max": 35.74346160888672,
      "activations/layer19_attention_weight_min": -28.000640869140625,
      "activations/layer1_attention_weight_max": 16.91689109802246,
      "activations/layer1_attention_weight_min": -14.820131301879883,
      "activations/layer20_attention_weight_max": 29.415544509887695,
      "activations/layer20_attention_weight_min": -24.697973251342773,
      "activations/layer21_attention_weight_max": 29.385679244995117,
      "activations/layer21_attention_weight_min": -22.620384216308594,
      "activations/layer22_attention_weight_max": 45.4880485534668,
      "activations/layer22_attention_weight_min": -29.281797409057617,
      "activations/layer23_attention_weight_max": 33.936729431152344,
      "activations/layer23_attention_weight_min": -24.13118553161621,
      "activations/layer2_attention_weight_max": 31.88892936706543,
      "activations/layer2_attention_weight_min": -33.64607620239258,
      "activations/layer3_attention_weight_max": 87.19336700439453,
      "activations/layer3_attention_weight_min": -91.52083587646484,
      "activations/layer4_attention_weight_max": 118.9591293334961,
      "activations/layer4_attention_weight_min": -120.6224365234375,
      "activations/layer5_attention_weight_max": 59.576881408691406,
      "activations/layer5_attention_weight_min": -61.244407653808594,
      "activations/layer6_attention_weight_max": 49.92814254760742,
      "activations/layer6_attention_weight_min": -50.805641174316406,
      "activations/layer7_attention_weight_max": 87.34968566894531,
      "activations/layer7_attention_weight_min": -85.28974914550781,
      "activations/layer8_attention_weight_max": 42.435909271240234,
      "activations/layer8_attention_weight_min": -41.27751541137695,
      "activations/layer9_attention_weight_max": 38.36916732788086,
      "activations/layer9_attention_weight_min": -34.06216049194336,
      "epoch": 8.1,
      "learning_rate": 9.87401515151515e-05,
      "loss": 2.8042,
      "step": 139450
    },
    {
      "activations/layer0_attention_weight_max": 15.7051420211792,
      "activations/layer0_attention_weight_min": -11.522846221923828,
      "activations/layer10_attention_weight_max": 31.454891204833984,
      "activations/layer10_attention_weight_min": -30.67265510559082,
      "activations/layer11_attention_weight_max": 29.59897232055664,
      "activations/layer11_attention_weight_min": -28.969785690307617,
      "activations/layer12_attention_weight_max": 41.984954833984375,
      "activations/layer12_attention_weight_min": -24.913475036621094,
      "activations/layer13_attention_weight_max": 44.38064193725586,
      "activations/layer13_attention_weight_min": -32.55067443847656,
      "activations/layer14_attention_weight_max": 45.1778450012207,
      "activations/layer14_attention_weight_min": -35.53995132446289,
      "activations/layer15_attention_weight_max": 36.243743896484375,
      "activations/layer15_attention_weight_min": -30.528085708618164,
      "activations/layer16_attention_weight_max": 30.327795028686523,
      "activations/layer16_attention_weight_min": -26.398622512817383,
      "activations/layer17_attention_weight_max": 31.799617767333984,
      "activations/layer17_attention_weight_min": -25.508543014526367,
      "activations/layer18_attention_weight_max": 30.367082595825195,
      "activations/layer18_attention_weight_min": -20.866384506225586,
      "activations/layer19_attention_weight_max": 34.85996627807617,
      "activations/layer19_attention_weight_min": -31.17152214050293,
      "activations/layer1_attention_weight_max": 17.090343475341797,
      "activations/layer1_attention_weight_min": -12.313788414001465,
      "activations/layer20_attention_weight_max": 28.318767547607422,
      "activations/layer20_attention_weight_min": -25.778818130493164,
      "activations/layer21_attention_weight_max": 28.18459701538086,
      "activations/layer21_attention_weight_min": -22.032304763793945,
      "activations/layer22_attention_weight_max": 46.47504425048828,
      "activations/layer22_attention_weight_min": -27.091352462768555,
      "activations/layer23_attention_weight_max": 36.9971923828125,
      "activations/layer23_attention_weight_min": -23.919950485229492,
      "activations/layer2_attention_weight_max": 31.831764221191406,
      "activations/layer2_attention_weight_min": -30.57539939880371,
      "activations/layer3_attention_weight_max": 86.94847106933594,
      "activations/layer3_attention_weight_min": -90.62000274658203,
      "activations/layer4_attention_weight_max": 122.37060546875,
      "activations/layer4_attention_weight_min": -125.71495819091797,
      "activations/layer5_attention_weight_max": 59.961151123046875,
      "activations/layer5_attention_weight_min": -64.80516052246094,
      "activations/layer6_attention_weight_max": 47.84396743774414,
      "activations/layer6_attention_weight_min": -52.91813278198242,
      "activations/layer7_attention_weight_max": 89.4504165649414,
      "activations/layer7_attention_weight_min": -90.04935455322266,
      "activations/layer8_attention_weight_max": 37.08702850341797,
      "activations/layer8_attention_weight_min": -36.87690353393555,
      "activations/layer9_attention_weight_max": 32.9321403503418,
      "activations/layer9_attention_weight_min": -31.956838607788086,
      "epoch": 8.11,
      "learning_rate": 9.872121212121211e-05,
      "loss": 2.8244,
      "step": 139500
    },
    {
      "activations/layer0_attention_weight_max": 15.116095542907715,
      "activations/layer0_attention_weight_min": -11.896353721618652,
      "activations/layer10_attention_weight_max": 43.88886642456055,
      "activations/layer10_attention_weight_min": -40.54301834106445,
      "activations/layer11_attention_weight_max": 46.030574798583984,
      "activations/layer11_attention_weight_min": -40.21111297607422,
      "activations/layer12_attention_weight_max": 28.435697555541992,
      "activations/layer12_attention_weight_min": -25.44715690612793,
      "activations/layer13_attention_weight_max": 59.46266555786133,
      "activations/layer13_attention_weight_min": -38.044857025146484,
      "activations/layer14_attention_weight_max": 63.62909698486328,
      "activations/layer14_attention_weight_min": -49.220458984375,
      "activations/layer15_attention_weight_max": 54.02140808105469,
      "activations/layer15_attention_weight_min": -33.843929290771484,
      "activations/layer16_attention_weight_max": 37.985782623291016,
      "activations/layer16_attention_weight_min": -29.944002151489258,
      "activations/layer17_attention_weight_max": 44.939266204833984,
      "activations/layer17_attention_weight_min": -28.127639770507812,
      "activations/layer18_attention_weight_max": 28.581026077270508,
      "activations/layer18_attention_weight_min": -22.188234329223633,
      "activations/layer19_attention_weight_max": 37.26919174194336,
      "activations/layer19_attention_weight_min": -35.16505432128906,
      "activations/layer1_attention_weight_max": 16.894451141357422,
      "activations/layer1_attention_weight_min": -13.937042236328125,
      "activations/layer20_attention_weight_max": 32.63911056518555,
      "activations/layer20_attention_weight_min": -26.54738998413086,
      "activations/layer21_attention_weight_max": 30.53095054626465,
      "activations/layer21_attention_weight_min": -22.876304626464844,
      "activations/layer22_attention_weight_max": 47.532615661621094,
      "activations/layer22_attention_weight_min": -28.496051788330078,
      "activations/layer23_attention_weight_max": 35.928558349609375,
      "activations/layer23_attention_weight_min": -23.74783706665039,
      "activations/layer2_attention_weight_max": 34.00596618652344,
      "activations/layer2_attention_weight_min": -33.601470947265625,
      "activations/layer3_attention_weight_max": 97.42129516601562,
      "activations/layer3_attention_weight_min": -97.55840301513672,
      "activations/layer4_attention_weight_max": 129.8539276123047,
      "activations/layer4_attention_weight_min": -123.7164077758789,
      "activations/layer5_attention_weight_max": 61.079498291015625,
      "activations/layer5_attention_weight_min": -57.721458435058594,
      "activations/layer6_attention_weight_max": 53.51659393310547,
      "activations/layer6_attention_weight_min": -51.84823226928711,
      "activations/layer7_attention_weight_max": 114.68376159667969,
      "activations/layer7_attention_weight_min": -98.99617004394531,
      "activations/layer8_attention_weight_max": 52.54353332519531,
      "activations/layer8_attention_weight_min": -48.41606140136719,
      "activations/layer9_attention_weight_max": 46.29094314575195,
      "activations/layer9_attention_weight_min": -44.39490509033203,
      "epoch": 8.11,
      "learning_rate": 9.870227272727272e-05,
      "loss": 2.7987,
      "step": 139550
    },
    {
      "activations/layer0_attention_weight_max": 14.979458808898926,
      "activations/layer0_attention_weight_min": -11.364116668701172,
      "activations/layer10_attention_weight_max": 29.702016830444336,
      "activations/layer10_attention_weight_min": -30.452573776245117,
      "activations/layer11_attention_weight_max": 32.055335998535156,
      "activations/layer11_attention_weight_min": -29.467529296875,
      "activations/layer12_attention_weight_max": 34.50343704223633,
      "activations/layer12_attention_weight_min": -23.795211791992188,
      "activations/layer13_attention_weight_max": 40.979820251464844,
      "activations/layer13_attention_weight_min": -33.86634826660156,
      "activations/layer14_attention_weight_max": 50.4465217590332,
      "activations/layer14_attention_weight_min": -38.50868606567383,
      "activations/layer15_attention_weight_max": 37.88226318359375,
      "activations/layer15_attention_weight_min": -30.477140426635742,
      "activations/layer16_attention_weight_max": 32.412715911865234,
      "activations/layer16_attention_weight_min": -27.69095802307129,
      "activations/layer17_attention_weight_max": 34.28509521484375,
      "activations/layer17_attention_weight_min": -25.020769119262695,
      "activations/layer18_attention_weight_max": 32.660186767578125,
      "activations/layer18_attention_weight_min": -23.358165740966797,
      "activations/layer19_attention_weight_max": 36.644466400146484,
      "activations/layer19_attention_weight_min": -29.988977432250977,
      "activations/layer1_attention_weight_max": 16.24009132385254,
      "activations/layer1_attention_weight_min": -14.009809494018555,
      "activations/layer20_attention_weight_max": 31.37194061279297,
      "activations/layer20_attention_weight_min": -25.501625061035156,
      "activations/layer21_attention_weight_max": 31.101110458374023,
      "activations/layer21_attention_weight_min": -22.828521728515625,
      "activations/layer22_attention_weight_max": 50.6422119140625,
      "activations/layer22_attention_weight_min": -28.67697525024414,
      "activations/layer23_attention_weight_max": 37.68157958984375,
      "activations/layer23_attention_weight_min": -23.971542358398438,
      "activations/layer2_attention_weight_max": 31.61128044128418,
      "activations/layer2_attention_weight_min": -32.450897216796875,
      "activations/layer3_attention_weight_max": 84.89225006103516,
      "activations/layer3_attention_weight_min": -89.49060821533203,
      "activations/layer4_attention_weight_max": 114.420166015625,
      "activations/layer4_attention_weight_min": -122.49498748779297,
      "activations/layer5_attention_weight_max": 55.27448272705078,
      "activations/layer5_attention_weight_min": -61.871681213378906,
      "activations/layer6_attention_weight_max": 47.282039642333984,
      "activations/layer6_attention_weight_min": -51.02549743652344,
      "activations/layer7_attention_weight_max": 86.6324691772461,
      "activations/layer7_attention_weight_min": -88.21337127685547,
      "activations/layer8_attention_weight_max": 37.76700973510742,
      "activations/layer8_attention_weight_min": -39.7210578918457,
      "activations/layer9_attention_weight_max": 32.1130485534668,
      "activations/layer9_attention_weight_min": -32.40873336791992,
      "epoch": 8.11,
      "learning_rate": 9.868333333333332e-05,
      "loss": 2.8007,
      "step": 139600
    },
    {
      "activations/layer0_attention_weight_max": 15.724542617797852,
      "activations/layer0_attention_weight_min": -11.101810455322266,
      "activations/layer10_attention_weight_max": 31.881181716918945,
      "activations/layer10_attention_weight_min": -31.251632690429688,
      "activations/layer11_attention_weight_max": 30.956573486328125,
      "activations/layer11_attention_weight_min": -29.75758171081543,
      "activations/layer12_attention_weight_max": 21.473894119262695,
      "activations/layer12_attention_weight_min": -32.58740234375,
      "activations/layer13_attention_weight_max": 37.66527557373047,
      "activations/layer13_attention_weight_min": -32.96343231201172,
      "activations/layer14_attention_weight_max": 43.655235290527344,
      "activations/layer14_attention_weight_min": -36.62837600708008,
      "activations/layer15_attention_weight_max": 31.913381576538086,
      "activations/layer15_attention_weight_min": -28.407567977905273,
      "activations/layer16_attention_weight_max": 29.81934928894043,
      "activations/layer16_attention_weight_min": -26.66474151611328,
      "activations/layer17_attention_weight_max": 29.839149475097656,
      "activations/layer17_attention_weight_min": -26.081707000732422,
      "activations/layer18_attention_weight_max": 33.675907135009766,
      "activations/layer18_attention_weight_min": -20.303686141967773,
      "activations/layer19_attention_weight_max": 33.67037582397461,
      "activations/layer19_attention_weight_min": -31.5709285736084,
      "activations/layer1_attention_weight_max": 16.693687438964844,
      "activations/layer1_attention_weight_min": -14.769472122192383,
      "activations/layer20_attention_weight_max": 28.001140594482422,
      "activations/layer20_attention_weight_min": -24.193525314331055,
      "activations/layer21_attention_weight_max": 29.0472412109375,
      "activations/layer21_attention_weight_min": -22.953359603881836,
      "activations/layer22_attention_weight_max": 41.97248840332031,
      "activations/layer22_attention_weight_min": -27.95549201965332,
      "activations/layer23_attention_weight_max": 34.60187530517578,
      "activations/layer23_attention_weight_min": -21.997314453125,
      "activations/layer2_attention_weight_max": 31.871562957763672,
      "activations/layer2_attention_weight_min": -31.619739532470703,
      "activations/layer3_attention_weight_max": 96.49702453613281,
      "activations/layer3_attention_weight_min": -97.80296325683594,
      "activations/layer4_attention_weight_max": 129.01145935058594,
      "activations/layer4_attention_weight_min": -122.93509674072266,
      "activations/layer5_attention_weight_max": 58.24374008178711,
      "activations/layer5_attention_weight_min": -66.1593017578125,
      "activations/layer6_attention_weight_max": 47.005760192871094,
      "activations/layer6_attention_weight_min": -51.30789566040039,
      "activations/layer7_attention_weight_max": 89.69275665283203,
      "activations/layer7_attention_weight_min": -91.92295837402344,
      "activations/layer8_attention_weight_max": 41.60017776489258,
      "activations/layer8_attention_weight_min": -40.78025817871094,
      "activations/layer9_attention_weight_max": 32.4613037109375,
      "activations/layer9_attention_weight_min": -37.321929931640625,
      "epoch": 8.11,
      "learning_rate": 9.866439393939393e-05,
      "loss": 2.8018,
      "step": 139650
    },
    {
      "activations/layer0_attention_weight_max": 15.24741268157959,
      "activations/layer0_attention_weight_min": -11.34642219543457,
      "activations/layer10_attention_weight_max": 35.179134368896484,
      "activations/layer10_attention_weight_min": -35.09459686279297,
      "activations/layer11_attention_weight_max": 36.118019104003906,
      "activations/layer11_attention_weight_min": -32.35353088378906,
      "activations/layer12_attention_weight_max": 25.90941047668457,
      "activations/layer12_attention_weight_min": -42.66946029663086,
      "activations/layer13_attention_weight_max": 40.3057975769043,
      "activations/layer13_attention_weight_min": -34.66483688354492,
      "activations/layer14_attention_weight_max": 51.019805908203125,
      "activations/layer14_attention_weight_min": -38.69295883178711,
      "activations/layer15_attention_weight_max": 35.45627975463867,
      "activations/layer15_attention_weight_min": -29.888097763061523,
      "activations/layer16_attention_weight_max": 32.954566955566406,
      "activations/layer16_attention_weight_min": -27.90006446838379,
      "activations/layer17_attention_weight_max": 32.64702606201172,
      "activations/layer17_attention_weight_min": -25.22507095336914,
      "activations/layer18_attention_weight_max": 36.653873443603516,
      "activations/layer18_attention_weight_min": -20.313810348510742,
      "activations/layer19_attention_weight_max": 37.2186393737793,
      "activations/layer19_attention_weight_min": -28.174375534057617,
      "activations/layer1_attention_weight_max": 18.179582595825195,
      "activations/layer1_attention_weight_min": -14.62917423248291,
      "activations/layer20_attention_weight_max": 29.506141662597656,
      "activations/layer20_attention_weight_min": -22.570255279541016,
      "activations/layer21_attention_weight_max": 33.4838981628418,
      "activations/layer21_attention_weight_min": -22.844823837280273,
      "activations/layer22_attention_weight_max": 49.283878326416016,
      "activations/layer22_attention_weight_min": -27.72528648376465,
      "activations/layer23_attention_weight_max": 35.595848083496094,
      "activations/layer23_attention_weight_min": -23.52534294128418,
      "activations/layer2_attention_weight_max": 32.38492965698242,
      "activations/layer2_attention_weight_min": -31.237125396728516,
      "activations/layer3_attention_weight_max": 90.28137969970703,
      "activations/layer3_attention_weight_min": -93.39799499511719,
      "activations/layer4_attention_weight_max": 123.57491302490234,
      "activations/layer4_attention_weight_min": -116.45686340332031,
      "activations/layer5_attention_weight_max": 57.03075408935547,
      "activations/layer5_attention_weight_min": -63.53107452392578,
      "activations/layer6_attention_weight_max": 49.76696014404297,
      "activations/layer6_attention_weight_min": -50.0920524597168,
      "activations/layer7_attention_weight_max": 96.645751953125,
      "activations/layer7_attention_weight_min": -92.7016372680664,
      "activations/layer8_attention_weight_max": 42.86491394042969,
      "activations/layer8_attention_weight_min": -40.42995834350586,
      "activations/layer9_attention_weight_max": 35.901145935058594,
      "activations/layer9_attention_weight_min": -36.21919250488281,
      "epoch": 8.12,
      "learning_rate": 9.864545454545454e-05,
      "loss": 2.7782,
      "step": 139700
    },
    {
      "activations/layer0_attention_weight_max": 15.067590713500977,
      "activations/layer0_attention_weight_min": -10.89836311340332,
      "activations/layer10_attention_weight_max": 38.7281494140625,
      "activations/layer10_attention_weight_min": -35.07956314086914,
      "activations/layer11_attention_weight_max": 41.25165557861328,
      "activations/layer11_attention_weight_min": -33.22995376586914,
      "activations/layer12_attention_weight_max": 26.828571319580078,
      "activations/layer12_attention_weight_min": -36.78425216674805,
      "activations/layer13_attention_weight_max": 41.82535934448242,
      "activations/layer13_attention_weight_min": -33.182376861572266,
      "activations/layer14_attention_weight_max": 77.84818267822266,
      "activations/layer14_attention_weight_min": -45.17543029785156,
      "activations/layer15_attention_weight_max": 38.518184661865234,
      "activations/layer15_attention_weight_min": -30.696657180786133,
      "activations/layer16_attention_weight_max": 37.107425689697266,
      "activations/layer16_attention_weight_min": -29.870397567749023,
      "activations/layer17_attention_weight_max": 31.144948959350586,
      "activations/layer17_attention_weight_min": -25.94671058654785,
      "activations/layer18_attention_weight_max": 30.494136810302734,
      "activations/layer18_attention_weight_min": -22.100452423095703,
      "activations/layer19_attention_weight_max": 40.2132568359375,
      "activations/layer19_attention_weight_min": -30.572370529174805,
      "activations/layer1_attention_weight_max": 16.851648330688477,
      "activations/layer1_attention_weight_min": -14.287837982177734,
      "activations/layer20_attention_weight_max": 31.41133689880371,
      "activations/layer20_attention_weight_min": -27.189468383789062,
      "activations/layer21_attention_weight_max": 29.1134090423584,
      "activations/layer21_attention_weight_min": -23.99022674560547,
      "activations/layer22_attention_weight_max": 47.79104232788086,
      "activations/layer22_attention_weight_min": -33.56182098388672,
      "activations/layer23_attention_weight_max": 36.090553283691406,
      "activations/layer23_attention_weight_min": -24.003829956054688,
      "activations/layer2_attention_weight_max": 31.435956954956055,
      "activations/layer2_attention_weight_min": -31.96990966796875,
      "activations/layer3_attention_weight_max": 89.23784637451172,
      "activations/layer3_attention_weight_min": -95.0022201538086,
      "activations/layer4_attention_weight_max": 117.80879211425781,
      "activations/layer4_attention_weight_min": -121.34239959716797,
      "activations/layer5_attention_weight_max": 57.06218719482422,
      "activations/layer5_attention_weight_min": -62.435726165771484,
      "activations/layer6_attention_weight_max": 47.19281768798828,
      "activations/layer6_attention_weight_min": -52.9477653503418,
      "activations/layer7_attention_weight_max": 100.97042083740234,
      "activations/layer7_attention_weight_min": -97.46159362792969,
      "activations/layer8_attention_weight_max": 42.31694793701172,
      "activations/layer8_attention_weight_min": -43.16014862060547,
      "activations/layer9_attention_weight_max": 36.593101501464844,
      "activations/layer9_attention_weight_min": -35.63003158569336,
      "epoch": 8.12,
      "learning_rate": 9.862651515151514e-05,
      "loss": 2.8069,
      "step": 139750
    },
    {
      "activations/layer0_attention_weight_max": 15.735548973083496,
      "activations/layer0_attention_weight_min": -11.950185775756836,
      "activations/layer10_attention_weight_max": 34.69296646118164,
      "activations/layer10_attention_weight_min": -31.447256088256836,
      "activations/layer11_attention_weight_max": 36.8111686706543,
      "activations/layer11_attention_weight_min": -31.08984375,
      "activations/layer12_attention_weight_max": 30.816068649291992,
      "activations/layer12_attention_weight_min": -23.807249069213867,
      "activations/layer13_attention_weight_max": 37.28782272338867,
      "activations/layer13_attention_weight_min": -33.40806579589844,
      "activations/layer14_attention_weight_max": 50.69142150878906,
      "activations/layer14_attention_weight_min": -39.23434829711914,
      "activations/layer15_attention_weight_max": 34.24343490600586,
      "activations/layer15_attention_weight_min": -31.89075469970703,
      "activations/layer16_attention_weight_max": 31.71845245361328,
      "activations/layer16_attention_weight_min": -26.628154754638672,
      "activations/layer17_attention_weight_max": 32.167640686035156,
      "activations/layer17_attention_weight_min": -25.72208595275879,
      "activations/layer18_attention_weight_max": 32.111244201660156,
      "activations/layer18_attention_weight_min": -20.649097442626953,
      "activations/layer19_attention_weight_max": 34.778141021728516,
      "activations/layer19_attention_weight_min": -29.931425094604492,
      "activations/layer1_attention_weight_max": 16.879255294799805,
      "activations/layer1_attention_weight_min": -14.276446342468262,
      "activations/layer20_attention_weight_max": 29.32936668395996,
      "activations/layer20_attention_weight_min": -23.383296966552734,
      "activations/layer21_attention_weight_max": 28.889183044433594,
      "activations/layer21_attention_weight_min": -22.110519409179688,
      "activations/layer22_attention_weight_max": 45.20347595214844,
      "activations/layer22_attention_weight_min": -30.594486236572266,
      "activations/layer23_attention_weight_max": 31.81831169128418,
      "activations/layer23_attention_weight_min": -23.55200958251953,
      "activations/layer2_attention_weight_max": 33.99018096923828,
      "activations/layer2_attention_weight_min": -32.23540115356445,
      "activations/layer3_attention_weight_max": 92.4749755859375,
      "activations/layer3_attention_weight_min": -92.1671142578125,
      "activations/layer4_attention_weight_max": 125.53582000732422,
      "activations/layer4_attention_weight_min": -115.73490142822266,
      "activations/layer5_attention_weight_max": 58.80157470703125,
      "activations/layer5_attention_weight_min": -59.41219711303711,
      "activations/layer6_attention_weight_max": 49.0996208190918,
      "activations/layer6_attention_weight_min": -52.8449821472168,
      "activations/layer7_attention_weight_max": 91.71183013916016,
      "activations/layer7_attention_weight_min": -90.16521453857422,
      "activations/layer8_attention_weight_max": 41.71614074707031,
      "activations/layer8_attention_weight_min": -42.298095703125,
      "activations/layer9_attention_weight_max": 34.92988586425781,
      "activations/layer9_attention_weight_min": -34.10715866088867,
      "epoch": 8.12,
      "learning_rate": 9.860757575757576e-05,
      "loss": 2.805,
      "step": 139800
    },
    {
      "activations/layer0_attention_weight_max": 15.77952766418457,
      "activations/layer0_attention_weight_min": -12.462176322937012,
      "activations/layer10_attention_weight_max": 32.164241790771484,
      "activations/layer10_attention_weight_min": -30.538896560668945,
      "activations/layer11_attention_weight_max": 34.39054870605469,
      "activations/layer11_attention_weight_min": -29.299652099609375,
      "activations/layer12_attention_weight_max": 21.9753360748291,
      "activations/layer12_attention_weight_min": -23.62639808654785,
      "activations/layer13_attention_weight_max": 38.5606689453125,
      "activations/layer13_attention_weight_min": -31.820045471191406,
      "activations/layer14_attention_weight_max": 46.700374603271484,
      "activations/layer14_attention_weight_min": -37.42272186279297,
      "activations/layer15_attention_weight_max": 33.45386505126953,
      "activations/layer15_attention_weight_min": -28.258501052856445,
      "activations/layer16_attention_weight_max": 32.01201248168945,
      "activations/layer16_attention_weight_min": -26.09012794494629,
      "activations/layer17_attention_weight_max": 33.1005744934082,
      "activations/layer17_attention_weight_min": -23.53759002685547,
      "activations/layer18_attention_weight_max": 31.361103057861328,
      "activations/layer18_attention_weight_min": -21.27060317993164,
      "activations/layer19_attention_weight_max": 41.867671966552734,
      "activations/layer19_attention_weight_min": -30.689010620117188,
      "activations/layer1_attention_weight_max": 16.416345596313477,
      "activations/layer1_attention_weight_min": -14.04277229309082,
      "activations/layer20_attention_weight_max": 32.65418243408203,
      "activations/layer20_attention_weight_min": -23.197294235229492,
      "activations/layer21_attention_weight_max": 31.146209716796875,
      "activations/layer21_attention_weight_min": -21.6109676361084,
      "activations/layer22_attention_weight_max": 51.708763122558594,
      "activations/layer22_attention_weight_min": -28.598249435424805,
      "activations/layer23_attention_weight_max": 41.63970184326172,
      "activations/layer23_attention_weight_min": -23.323591232299805,
      "activations/layer2_attention_weight_max": 31.64669418334961,
      "activations/layer2_attention_weight_min": -32.242942810058594,
      "activations/layer3_attention_weight_max": 88.51785278320312,
      "activations/layer3_attention_weight_min": -91.65363311767578,
      "activations/layer4_attention_weight_max": 117.82061767578125,
      "activations/layer4_attention_weight_min": -116.93544006347656,
      "activations/layer5_attention_weight_max": 56.298492431640625,
      "activations/layer5_attention_weight_min": -61.586036682128906,
      "activations/layer6_attention_weight_max": 47.076168060302734,
      "activations/layer6_attention_weight_min": -50.08351516723633,
      "activations/layer7_attention_weight_max": 84.54375457763672,
      "activations/layer7_attention_weight_min": -87.77550506591797,
      "activations/layer8_attention_weight_max": 39.130455017089844,
      "activations/layer8_attention_weight_min": -40.65126419067383,
      "activations/layer9_attention_weight_max": 34.278804779052734,
      "activations/layer9_attention_weight_min": -33.19934844970703,
      "epoch": 8.13,
      "learning_rate": 9.858863636363635e-05,
      "loss": 2.79,
      "step": 139850
    },
    {
      "activations/layer0_attention_weight_max": 15.74875545501709,
      "activations/layer0_attention_weight_min": -11.79113483428955,
      "activations/layer10_attention_weight_max": 32.64516067504883,
      "activations/layer10_attention_weight_min": -32.91497039794922,
      "activations/layer11_attention_weight_max": 30.945266723632812,
      "activations/layer11_attention_weight_min": -31.72658348083496,
      "activations/layer12_attention_weight_max": 22.28143310546875,
      "activations/layer12_attention_weight_min": -25.80988883972168,
      "activations/layer13_attention_weight_max": 40.048583984375,
      "activations/layer13_attention_weight_min": -31.320703506469727,
      "activations/layer14_attention_weight_max": 50.09290313720703,
      "activations/layer14_attention_weight_min": -43.511016845703125,
      "activations/layer15_attention_weight_max": 33.705753326416016,
      "activations/layer15_attention_weight_min": -29.375146865844727,
      "activations/layer16_attention_weight_max": 32.26085662841797,
      "activations/layer16_attention_weight_min": -28.129335403442383,
      "activations/layer17_attention_weight_max": 35.551815032958984,
      "activations/layer17_attention_weight_min": -25.211992263793945,
      "activations/layer18_attention_weight_max": 29.96527671813965,
      "activations/layer18_attention_weight_min": -20.886144638061523,
      "activations/layer19_attention_weight_max": 36.46574783325195,
      "activations/layer19_attention_weight_min": -29.876428604125977,
      "activations/layer1_attention_weight_max": 16.739103317260742,
      "activations/layer1_attention_weight_min": -13.398125648498535,
      "activations/layer20_attention_weight_max": 30.512378692626953,
      "activations/layer20_attention_weight_min": -23.470870971679688,
      "activations/layer21_attention_weight_max": 28.453428268432617,
      "activations/layer21_attention_weight_min": -21.694595336914062,
      "activations/layer22_attention_weight_max": 45.092403411865234,
      "activations/layer22_attention_weight_min": -31.794872283935547,
      "activations/layer23_attention_weight_max": 34.21270751953125,
      "activations/layer23_attention_weight_min": -22.034759521484375,
      "activations/layer2_attention_weight_max": 31.201040267944336,
      "activations/layer2_attention_weight_min": -30.04890251159668,
      "activations/layer3_attention_weight_max": 92.60057067871094,
      "activations/layer3_attention_weight_min": -93.71620178222656,
      "activations/layer4_attention_weight_max": 117.7781753540039,
      "activations/layer4_attention_weight_min": -115.08988189697266,
      "activations/layer5_attention_weight_max": 56.12748336791992,
      "activations/layer5_attention_weight_min": -64.73095703125,
      "activations/layer6_attention_weight_max": 50.050697326660156,
      "activations/layer6_attention_weight_min": -50.51717758178711,
      "activations/layer7_attention_weight_max": 86.81015014648438,
      "activations/layer7_attention_weight_min": -90.17383575439453,
      "activations/layer8_attention_weight_max": 37.517520904541016,
      "activations/layer8_attention_weight_min": -38.63578796386719,
      "activations/layer9_attention_weight_max": 34.65815353393555,
      "activations/layer9_attention_weight_min": -31.953750610351562,
      "epoch": 8.13,
      "learning_rate": 9.856969696969696e-05,
      "loss": 2.8085,
      "step": 139900
    },
    {
      "activations/layer0_attention_weight_max": 15.145710945129395,
      "activations/layer0_attention_weight_min": -11.484770774841309,
      "activations/layer10_attention_weight_max": 34.08411407470703,
      "activations/layer10_attention_weight_min": -32.116546630859375,
      "activations/layer11_attention_weight_max": 33.803245544433594,
      "activations/layer11_attention_weight_min": -30.224973678588867,
      "activations/layer12_attention_weight_max": 22.99256706237793,
      "activations/layer12_attention_weight_min": -24.974899291992188,
      "activations/layer13_attention_weight_max": 40.3721809387207,
      "activations/layer13_attention_weight_min": -33.02667236328125,
      "activations/layer14_attention_weight_max": 49.205081939697266,
      "activations/layer14_attention_weight_min": -39.24385452270508,
      "activations/layer15_attention_weight_max": 35.55775833129883,
      "activations/layer15_attention_weight_min": -30.693050384521484,
      "activations/layer16_attention_weight_max": 35.99717330932617,
      "activations/layer16_attention_weight_min": -27.37982749938965,
      "activations/layer17_attention_weight_max": 38.53435134887695,
      "activations/layer17_attention_weight_min": -25.982120513916016,
      "activations/layer18_attention_weight_max": 34.14179611206055,
      "activations/layer18_attention_weight_min": -21.70106315612793,
      "activations/layer19_attention_weight_max": 38.6176872253418,
      "activations/layer19_attention_weight_min": -31.502206802368164,
      "activations/layer1_attention_weight_max": 17.107440948486328,
      "activations/layer1_attention_weight_min": -16.559080123901367,
      "activations/layer20_attention_weight_max": 32.962581634521484,
      "activations/layer20_attention_weight_min": -24.592267990112305,
      "activations/layer21_attention_weight_max": 30.924545288085938,
      "activations/layer21_attention_weight_min": -23.18073081970215,
      "activations/layer22_attention_weight_max": 45.018028259277344,
      "activations/layer22_attention_weight_min": -28.21709442138672,
      "activations/layer23_attention_weight_max": 34.63990020751953,
      "activations/layer23_attention_weight_min": -23.212621688842773,
      "activations/layer2_attention_weight_max": 30.530200958251953,
      "activations/layer2_attention_weight_min": -31.135900497436523,
      "activations/layer3_attention_weight_max": 92.48440551757812,
      "activations/layer3_attention_weight_min": -92.68217468261719,
      "activations/layer4_attention_weight_max": 123.15003967285156,
      "activations/layer4_attention_weight_min": -117.7020034790039,
      "activations/layer5_attention_weight_max": 59.21218490600586,
      "activations/layer5_attention_weight_min": -62.336421966552734,
      "activations/layer6_attention_weight_max": 49.65256881713867,
      "activations/layer6_attention_weight_min": -51.33222198486328,
      "activations/layer7_attention_weight_max": 89.0702896118164,
      "activations/layer7_attention_weight_min": -87.9878158569336,
      "activations/layer8_attention_weight_max": 40.75834655761719,
      "activations/layer8_attention_weight_min": -42.88100051879883,
      "activations/layer9_attention_weight_max": 36.47549057006836,
      "activations/layer9_attention_weight_min": -34.79857635498047,
      "epoch": 8.13,
      "learning_rate": 9.855075757575758e-05,
      "loss": 2.8065,
      "step": 139950
    },
    {
      "activations/layer0_attention_weight_max": 13.867667198181152,
      "activations/layer0_attention_weight_min": -11.847295761108398,
      "activations/layer10_attention_weight_max": 36.017311096191406,
      "activations/layer10_attention_weight_min": -33.85706329345703,
      "activations/layer11_attention_weight_max": 34.628578186035156,
      "activations/layer11_attention_weight_min": -32.73735427856445,
      "activations/layer12_attention_weight_max": 29.674474716186523,
      "activations/layer12_attention_weight_min": -25.096607208251953,
      "activations/layer13_attention_weight_max": 39.2412109375,
      "activations/layer13_attention_weight_min": -35.629154205322266,
      "activations/layer14_attention_weight_max": 53.477684020996094,
      "activations/layer14_attention_weight_min": -41.855552673339844,
      "activations/layer15_attention_weight_max": 35.208980560302734,
      "activations/layer15_attention_weight_min": -30.61339569091797,
      "activations/layer16_attention_weight_max": 33.83930206298828,
      "activations/layer16_attention_weight_min": -29.40154266357422,
      "activations/layer17_attention_weight_max": 35.63394546508789,
      "activations/layer17_attention_weight_min": -25.159687042236328,
      "activations/layer18_attention_weight_max": 30.405248641967773,
      "activations/layer18_attention_weight_min": -21.547889709472656,
      "activations/layer19_attention_weight_max": 35.74044418334961,
      "activations/layer19_attention_weight_min": -29.876245498657227,
      "activations/layer1_attention_weight_max": 16.50394058227539,
      "activations/layer1_attention_weight_min": -14.397035598754883,
      "activations/layer20_attention_weight_max": 29.535045623779297,
      "activations/layer20_attention_weight_min": -22.107463836669922,
      "activations/layer21_attention_weight_max": 29.2633113861084,
      "activations/layer21_attention_weight_min": -21.46201515197754,
      "activations/layer22_attention_weight_max": 47.62016677856445,
      "activations/layer22_attention_weight_min": -26.93893051147461,
      "activations/layer23_attention_weight_max": 35.85838317871094,
      "activations/layer23_attention_weight_min": -22.72383689880371,
      "activations/layer2_attention_weight_max": 33.949031829833984,
      "activations/layer2_attention_weight_min": -33.37537384033203,
      "activations/layer3_attention_weight_max": 98.7431869506836,
      "activations/layer3_attention_weight_min": -100.42955780029297,
      "activations/layer4_attention_weight_max": 127.81349182128906,
      "activations/layer4_attention_weight_min": -130.50132751464844,
      "activations/layer5_attention_weight_max": 60.58635711669922,
      "activations/layer5_attention_weight_min": -65.62552642822266,
      "activations/layer6_attention_weight_max": 51.85182571411133,
      "activations/layer6_attention_weight_min": -55.21504211425781,
      "activations/layer7_attention_weight_max": 91.57749938964844,
      "activations/layer7_attention_weight_min": -97.50596618652344,
      "activations/layer8_attention_weight_max": 42.95330047607422,
      "activations/layer8_attention_weight_min": -45.35236358642578,
      "activations/layer9_attention_weight_max": 36.93337631225586,
      "activations/layer9_attention_weight_min": -35.70309066772461,
      "epoch": 8.13,
      "learning_rate": 9.853181818181816e-05,
      "loss": 2.8052,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5436,
      "eval_samples_per_second": 502.597,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5436,
      "eval_openwebtext_samples_per_second": 502.597,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 2.0491,
      "eval_wikitext_samples_per_second": 222.54,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_lambada_loss": 2.603515625,
      "eval_lambada_ppl": 13.511154789957821,
      "eval_lambada_runtime": 9.5928,
      "eval_lambada_samples_per_second": 507.57,
      "step": 140000
    },
    {
      "activations/layer0_attention_weight_max": 15.385457038879395,
      "activations/layer0_attention_weight_min": -12.4093017578125,
      "activations/layer10_attention_weight_max": 36.376373291015625,
      "activations/layer10_attention_weight_min": -37.03292465209961,
      "activations/layer11_attention_weight_max": 36.33631134033203,
      "activations/layer11_attention_weight_min": -35.05696105957031,
      "activations/layer12_attention_weight_max": 23.33527183532715,
      "activations/layer12_attention_weight_min": -25.656599044799805,
      "activations/layer13_attention_weight_max": 35.014461517333984,
      "activations/layer13_attention_weight_min": -33.298160552978516,
      "activations/layer14_attention_weight_max": 49.50312805175781,
      "activations/layer14_attention_weight_min": -40.990745544433594,
      "activations/layer15_attention_weight_max": 36.17667007446289,
      "activations/layer15_attention_weight_min": -29.35880470275879,
      "activations/layer16_attention_weight_max": 33.83072280883789,
      "activations/layer16_attention_weight_min": -27.634769439697266,
      "activations/layer17_attention_weight_max": 32.147762298583984,
      "activations/layer17_attention_weight_min": -25.783329010009766,
      "activations/layer18_attention_weight_max": 28.4652156829834,
      "activations/layer18_attention_weight_min": -21.066158294677734,
      "activations/layer19_attention_weight_max": 36.28408432006836,
      "activations/layer19_attention_weight_min": -31.297266006469727,
      "activations/layer1_attention_weight_max": 16.87192726135254,
      "activations/layer1_attention_weight_min": -14.524046897888184,
      "activations/layer20_attention_weight_max": 29.050525665283203,
      "activations/layer20_attention_weight_min": -25.165475845336914,
      "activations/layer21_attention_weight_max": 26.162107467651367,
      "activations/layer21_attention_weight_min": -22.62965202331543,
      "activations/layer22_attention_weight_max": 41.1198616027832,
      "activations/layer22_attention_weight_min": -29.371593475341797,
      "activations/layer23_attention_weight_max": 35.63898468017578,
      "activations/layer23_attention_weight_min": -23.186500549316406,
      "activations/layer2_attention_weight_max": 33.71073532104492,
      "activations/layer2_attention_weight_min": -34.09935760498047,
      "activations/layer3_attention_weight_max": 98.55155181884766,
      "activations/layer3_attention_weight_min": -100.79148864746094,
      "activations/layer4_attention_weight_max": 131.5780487060547,
      "activations/layer4_attention_weight_min": -125.36275482177734,
      "activations/layer5_attention_weight_max": 62.66196823120117,
      "activations/layer5_attention_weight_min": -67.23419189453125,
      "activations/layer6_attention_weight_max": 55.216796875,
      "activations/layer6_attention_weight_min": -54.42708206176758,
      "activations/layer7_attention_weight_max": 99.61665344238281,
      "activations/layer7_attention_weight_min": -95.18014526367188,
      "activations/layer8_attention_weight_max": 47.61894989013672,
      "activations/layer8_attention_weight_min": -44.980892181396484,
      "activations/layer9_attention_weight_max": 37.93916702270508,
      "activations/layer9_attention_weight_min": -38.804840087890625,
      "epoch": 8.14,
      "learning_rate": 9.851287878787878e-05,
      "loss": 2.8081,
      "step": 140050
    },
    {
      "activations/layer0_attention_weight_max": 15.839427947998047,
      "activations/layer0_attention_weight_min": -11.804579734802246,
      "activations/layer10_attention_weight_max": 35.6419563293457,
      "activations/layer10_attention_weight_min": -33.61376953125,
      "activations/layer11_attention_weight_max": 37.386024475097656,
      "activations/layer11_attention_weight_min": -32.17127227783203,
      "activations/layer12_attention_weight_max": 24.06109046936035,
      "activations/layer12_attention_weight_min": -25.268762588500977,
      "activations/layer13_attention_weight_max": 41.63469696044922,
      "activations/layer13_attention_weight_min": -33.95362091064453,
      "activations/layer14_attention_weight_max": 55.6998405456543,
      "activations/layer14_attention_weight_min": -41.69805145263672,
      "activations/layer15_attention_weight_max": 36.521644592285156,
      "activations/layer15_attention_weight_min": -28.493724822998047,
      "activations/layer16_attention_weight_max": 32.10947036743164,
      "activations/layer16_attention_weight_min": -28.953630447387695,
      "activations/layer17_attention_weight_max": 37.24528121948242,
      "activations/layer17_attention_weight_min": -28.639657974243164,
      "activations/layer18_attention_weight_max": 34.88447570800781,
      "activations/layer18_attention_weight_min": -21.41668701171875,
      "activations/layer19_attention_weight_max": 38.2520866394043,
      "activations/layer19_attention_weight_min": -31.743253707885742,
      "activations/layer1_attention_weight_max": 16.23180389404297,
      "activations/layer1_attention_weight_min": -14.600800514221191,
      "activations/layer20_attention_weight_max": 31.637556076049805,
      "activations/layer20_attention_weight_min": -28.168210983276367,
      "activations/layer21_attention_weight_max": 34.79488754272461,
      "activations/layer21_attention_weight_min": -24.341886520385742,
      "activations/layer22_attention_weight_max": 48.15903091430664,
      "activations/layer22_attention_weight_min": -30.21078872680664,
      "activations/layer23_attention_weight_max": 37.48763656616211,
      "activations/layer23_attention_weight_min": -24.294538497924805,
      "activations/layer2_attention_weight_max": 30.710559844970703,
      "activations/layer2_attention_weight_min": -30.557109832763672,
      "activations/layer3_attention_weight_max": 93.1502685546875,
      "activations/layer3_attention_weight_min": -94.72259521484375,
      "activations/layer4_attention_weight_max": 121.56307220458984,
      "activations/layer4_attention_weight_min": -118.34474182128906,
      "activations/layer5_attention_weight_max": 58.73152160644531,
      "activations/layer5_attention_weight_min": -61.80515670776367,
      "activations/layer6_attention_weight_max": 49.51735305786133,
      "activations/layer6_attention_weight_min": -51.90747833251953,
      "activations/layer7_attention_weight_max": 88.68490600585938,
      "activations/layer7_attention_weight_min": -88.2755126953125,
      "activations/layer8_attention_weight_max": 42.12093734741211,
      "activations/layer8_attention_weight_min": -42.20216369628906,
      "activations/layer9_attention_weight_max": 37.8533821105957,
      "activations/layer9_attention_weight_min": -34.44082260131836,
      "epoch": 8.14,
      "learning_rate": 9.84939393939394e-05,
      "loss": 2.8022,
      "step": 140100
    },
    {
      "activations/layer0_attention_weight_max": 15.963878631591797,
      "activations/layer0_attention_weight_min": -11.43562126159668,
      "activations/layer10_attention_weight_max": 32.58453369140625,
      "activations/layer10_attention_weight_min": -32.75868606567383,
      "activations/layer11_attention_weight_max": 32.816993713378906,
      "activations/layer11_attention_weight_min": -29.967700958251953,
      "activations/layer12_attention_weight_max": 28.942052841186523,
      "activations/layer12_attention_weight_min": -26.08161735534668,
      "activations/layer13_attention_weight_max": 39.463802337646484,
      "activations/layer13_attention_weight_min": -32.92158508300781,
      "activations/layer14_attention_weight_max": 64.84221649169922,
      "activations/layer14_attention_weight_min": -45.27469253540039,
      "activations/layer15_attention_weight_max": 41.38801956176758,
      "activations/layer15_attention_weight_min": -30.1695499420166,
      "activations/layer16_attention_weight_max": 35.52159881591797,
      "activations/layer16_attention_weight_min": -27.702497482299805,
      "activations/layer17_attention_weight_max": 35.30690383911133,
      "activations/layer17_attention_weight_min": -26.373804092407227,
      "activations/layer18_attention_weight_max": 35.92986297607422,
      "activations/layer18_attention_weight_min": -21.817800521850586,
      "activations/layer19_attention_weight_max": 39.34605407714844,
      "activations/layer19_attention_weight_min": -33.3921012878418,
      "activations/layer1_attention_weight_max": 15.989235877990723,
      "activations/layer1_attention_weight_min": -15.054444313049316,
      "activations/layer20_attention_weight_max": 36.39749526977539,
      "activations/layer20_attention_weight_min": -26.248289108276367,
      "activations/layer21_attention_weight_max": 38.65913772583008,
      "activations/layer21_attention_weight_min": -26.337223052978516,
      "activations/layer22_attention_weight_max": 58.2633056640625,
      "activations/layer22_attention_weight_min": -31.03631591796875,
      "activations/layer23_attention_weight_max": 40.90806579589844,
      "activations/layer23_attention_weight_min": -25.594274520874023,
      "activations/layer2_attention_weight_max": 31.513364791870117,
      "activations/layer2_attention_weight_min": -31.415807723999023,
      "activations/layer3_attention_weight_max": 90.31024169921875,
      "activations/layer3_attention_weight_min": -92.31989288330078,
      "activations/layer4_attention_weight_max": 119.47480773925781,
      "activations/layer4_attention_weight_min": -118.13209533691406,
      "activations/layer5_attention_weight_max": 58.384029388427734,
      "activations/layer5_attention_weight_min": -63.68955993652344,
      "activations/layer6_attention_weight_max": 49.733036041259766,
      "activations/layer6_attention_weight_min": -52.5595588684082,
      "activations/layer7_attention_weight_max": 89.58479309082031,
      "activations/layer7_attention_weight_min": -94.27411651611328,
      "activations/layer8_attention_weight_max": 42.90953826904297,
      "activations/layer8_attention_weight_min": -40.65386962890625,
      "activations/layer9_attention_weight_max": 34.35380172729492,
      "activations/layer9_attention_weight_min": -34.69937515258789,
      "epoch": 8.14,
      "learning_rate": 9.847499999999998e-05,
      "loss": 2.8165,
      "step": 140150
    },
    {
      "activations/layer0_attention_weight_max": 16.402307510375977,
      "activations/layer0_attention_weight_min": -12.271381378173828,
      "activations/layer10_attention_weight_max": 36.296630859375,
      "activations/layer10_attention_weight_min": -36.17338943481445,
      "activations/layer11_attention_weight_max": 37.97358322143555,
      "activations/layer11_attention_weight_min": -33.02857208251953,
      "activations/layer12_attention_weight_max": 25.695117950439453,
      "activations/layer12_attention_weight_min": -28.092439651489258,
      "activations/layer13_attention_weight_max": 42.74903869628906,
      "activations/layer13_attention_weight_min": -33.93486404418945,
      "activations/layer14_attention_weight_max": 50.31557083129883,
      "activations/layer14_attention_weight_min": -38.90750503540039,
      "activations/layer15_attention_weight_max": 38.6429328918457,
      "activations/layer15_attention_weight_min": -32.92534637451172,
      "activations/layer16_attention_weight_max": 32.48350524902344,
      "activations/layer16_attention_weight_min": -27.76413917541504,
      "activations/layer17_attention_weight_max": 39.81110382080078,
      "activations/layer17_attention_weight_min": -25.574445724487305,
      "activations/layer18_attention_weight_max": 33.85851287841797,
      "activations/layer18_attention_weight_min": -19.74611473083496,
      "activations/layer19_attention_weight_max": 36.15031051635742,
      "activations/layer19_attention_weight_min": -29.167516708374023,
      "activations/layer1_attention_weight_max": 16.81131935119629,
      "activations/layer1_attention_weight_min": -14.775736808776855,
      "activations/layer20_attention_weight_max": 29.8769474029541,
      "activations/layer20_attention_weight_min": -23.547346115112305,
      "activations/layer21_attention_weight_max": 30.88644790649414,
      "activations/layer21_attention_weight_min": -23.946006774902344,
      "activations/layer22_attention_weight_max": 48.69329833984375,
      "activations/layer22_attention_weight_min": -26.10611915588379,
      "activations/layer23_attention_weight_max": 35.16046905517578,
      "activations/layer23_attention_weight_min": -23.54705810546875,
      "activations/layer2_attention_weight_max": 32.8901481628418,
      "activations/layer2_attention_weight_min": -34.40229034423828,
      "activations/layer3_attention_weight_max": 103.94969177246094,
      "activations/layer3_attention_weight_min": -111.23664093017578,
      "activations/layer4_attention_weight_max": 134.41014099121094,
      "activations/layer4_attention_weight_min": -129.8120880126953,
      "activations/layer5_attention_weight_max": 61.44963836669922,
      "activations/layer5_attention_weight_min": -64.82440948486328,
      "activations/layer6_attention_weight_max": 52.95298767089844,
      "activations/layer6_attention_weight_min": -56.76262664794922,
      "activations/layer7_attention_weight_max": 93.09314727783203,
      "activations/layer7_attention_weight_min": -100.63662719726562,
      "activations/layer8_attention_weight_max": 43.401424407958984,
      "activations/layer8_attention_weight_min": -45.9134521484375,
      "activations/layer9_attention_weight_max": 38.29936599731445,
      "activations/layer9_attention_weight_min": -39.084468841552734,
      "epoch": 8.15,
      "learning_rate": 9.84560606060606e-05,
      "loss": 2.8084,
      "step": 140200
    },
    {
      "activations/layer0_attention_weight_max": 15.961036682128906,
      "activations/layer0_attention_weight_min": -12.053152084350586,
      "activations/layer10_attention_weight_max": 30.793777465820312,
      "activations/layer10_attention_weight_min": -31.23193359375,
      "activations/layer11_attention_weight_max": 29.508087158203125,
      "activations/layer11_attention_weight_min": -30.463903427124023,
      "activations/layer12_attention_weight_max": 22.708677291870117,
      "activations/layer12_attention_weight_min": -27.670368194580078,
      "activations/layer13_attention_weight_max": 37.09701156616211,
      "activations/layer13_attention_weight_min": -35.70935821533203,
      "activations/layer14_attention_weight_max": 47.727256774902344,
      "activations/layer14_attention_weight_min": -38.4031867980957,
      "activations/layer15_attention_weight_max": 37.07654571533203,
      "activations/layer15_attention_weight_min": -29.740646362304688,
      "activations/layer16_attention_weight_max": 32.05666732788086,
      "activations/layer16_attention_weight_min": -28.821245193481445,
      "activations/layer17_attention_weight_max": 33.300758361816406,
      "activations/layer17_attention_weight_min": -24.99806022644043,
      "activations/layer18_attention_weight_max": 29.981380462646484,
      "activations/layer18_attention_weight_min": -21.24126434326172,
      "activations/layer19_attention_weight_max": 36.4693717956543,
      "activations/layer19_attention_weight_min": -30.312885284423828,
      "activations/layer1_attention_weight_max": 16.865163803100586,
      "activations/layer1_attention_weight_min": -13.740143775939941,
      "activations/layer20_attention_weight_max": 30.927379608154297,
      "activations/layer20_attention_weight_min": -24.962797164916992,
      "activations/layer21_attention_weight_max": 28.085861206054688,
      "activations/layer21_attention_weight_min": -23.018802642822266,
      "activations/layer22_attention_weight_max": 43.960723876953125,
      "activations/layer22_attention_weight_min": -29.721040725708008,
      "activations/layer23_attention_weight_max": 36.99673080444336,
      "activations/layer23_attention_weight_min": -25.368453979492188,
      "activations/layer2_attention_weight_max": 32.521644592285156,
      "activations/layer2_attention_weight_min": -30.304426193237305,
      "activations/layer3_attention_weight_max": 92.7519760131836,
      "activations/layer3_attention_weight_min": -91.17485046386719,
      "activations/layer4_attention_weight_max": 120.15699768066406,
      "activations/layer4_attention_weight_min": -113.97318267822266,
      "activations/layer5_attention_weight_max": 59.20622253417969,
      "activations/layer5_attention_weight_min": -62.07537841796875,
      "activations/layer6_attention_weight_max": 48.732975006103516,
      "activations/layer6_attention_weight_min": -50.87548065185547,
      "activations/layer7_attention_weight_max": 84.45773315429688,
      "activations/layer7_attention_weight_min": -84.92554473876953,
      "activations/layer8_attention_weight_max": 37.773460388183594,
      "activations/layer8_attention_weight_min": -38.58224105834961,
      "activations/layer9_attention_weight_max": 33.047157287597656,
      "activations/layer9_attention_weight_min": -32.15439987182617,
      "epoch": 8.15,
      "learning_rate": 9.843712121212121e-05,
      "loss": 2.7935,
      "step": 140250
    },
    {
      "activations/layer0_attention_weight_max": 15.597129821777344,
      "activations/layer0_attention_weight_min": -12.201485633850098,
      "activations/layer10_attention_weight_max": 32.994564056396484,
      "activations/layer10_attention_weight_min": -31.548320770263672,
      "activations/layer11_attention_weight_max": 31.53567123413086,
      "activations/layer11_attention_weight_min": -29.368114471435547,
      "activations/layer12_attention_weight_max": 23.045276641845703,
      "activations/layer12_attention_weight_min": -25.29581642150879,
      "activations/layer13_attention_weight_max": 35.87049102783203,
      "activations/layer13_attention_weight_min": -30.67943572998047,
      "activations/layer14_attention_weight_max": 50.1578369140625,
      "activations/layer14_attention_weight_min": -37.35893249511719,
      "activations/layer15_attention_weight_max": 35.5025520324707,
      "activations/layer15_attention_weight_min": -28.03821563720703,
      "activations/layer16_attention_weight_max": 33.48206329345703,
      "activations/layer16_attention_weight_min": -25.846298217773438,
      "activations/layer17_attention_weight_max": 31.857648849487305,
      "activations/layer17_attention_weight_min": -24.60100555419922,
      "activations/layer18_attention_weight_max": 30.442096710205078,
      "activations/layer18_attention_weight_min": -19.973217010498047,
      "activations/layer19_attention_weight_max": 36.39690017700195,
      "activations/layer19_attention_weight_min": -29.174823760986328,
      "activations/layer1_attention_weight_max": 16.6461238861084,
      "activations/layer1_attention_weight_min": -13.489013671875,
      "activations/layer20_attention_weight_max": 28.944679260253906,
      "activations/layer20_attention_weight_min": -23.67458152770996,
      "activations/layer21_attention_weight_max": 27.98134422302246,
      "activations/layer21_attention_weight_min": -22.223278045654297,
      "activations/layer22_attention_weight_max": 45.29020309448242,
      "activations/layer22_attention_weight_min": -29.38020133972168,
      "activations/layer23_attention_weight_max": 33.472938537597656,
      "activations/layer23_attention_weight_min": -22.091285705566406,
      "activations/layer2_attention_weight_max": 32.33330154418945,
      "activations/layer2_attention_weight_min": -32.15587615966797,
      "activations/layer3_attention_weight_max": 92.47821807861328,
      "activations/layer3_attention_weight_min": -95.68821716308594,
      "activations/layer4_attention_weight_max": 120.24433898925781,
      "activations/layer4_attention_weight_min": -116.07069396972656,
      "activations/layer5_attention_weight_max": 58.393211364746094,
      "activations/layer5_attention_weight_min": -61.66290283203125,
      "activations/layer6_attention_weight_max": 51.81986618041992,
      "activations/layer6_attention_weight_min": -49.07109451293945,
      "activations/layer7_attention_weight_max": 82.17036437988281,
      "activations/layer7_attention_weight_min": -82.6048355102539,
      "activations/layer8_attention_weight_max": 38.02073669433594,
      "activations/layer8_attention_weight_min": -39.28923034667969,
      "activations/layer9_attention_weight_max": 34.293304443359375,
      "activations/layer9_attention_weight_min": -33.54716491699219,
      "epoch": 8.15,
      "learning_rate": 9.841818181818181e-05,
      "loss": 2.7943,
      "step": 140300
    },
    {
      "activations/layer0_attention_weight_max": 16.008989334106445,
      "activations/layer0_attention_weight_min": -12.272805213928223,
      "activations/layer10_attention_weight_max": 34.91106033325195,
      "activations/layer10_attention_weight_min": -30.338584899902344,
      "activations/layer11_attention_weight_max": 36.49699783325195,
      "activations/layer11_attention_weight_min": -28.865623474121094,
      "activations/layer12_attention_weight_max": 21.83745574951172,
      "activations/layer12_attention_weight_min": -25.83979606628418,
      "activations/layer13_attention_weight_max": 45.49738693237305,
      "activations/layer13_attention_weight_min": -33.65601348876953,
      "activations/layer14_attention_weight_max": 57.755165100097656,
      "activations/layer14_attention_weight_min": -43.841224670410156,
      "activations/layer15_attention_weight_max": 40.84754180908203,
      "activations/layer15_attention_weight_min": -29.604938507080078,
      "activations/layer16_attention_weight_max": 33.843833923339844,
      "activations/layer16_attention_weight_min": -27.205421447753906,
      "activations/layer17_attention_weight_max": 34.6738166809082,
      "activations/layer17_attention_weight_min": -24.19940185546875,
      "activations/layer18_attention_weight_max": 30.731664657592773,
      "activations/layer18_attention_weight_min": -20.695220947265625,
      "activations/layer19_attention_weight_max": 37.85439682006836,
      "activations/layer19_attention_weight_min": -30.04593276977539,
      "activations/layer1_attention_weight_max": 16.950273513793945,
      "activations/layer1_attention_weight_min": -14.846293449401855,
      "activations/layer20_attention_weight_max": 33.804962158203125,
      "activations/layer20_attention_weight_min": -22.596145629882812,
      "activations/layer21_attention_weight_max": 29.697622299194336,
      "activations/layer21_attention_weight_min": -21.91847038269043,
      "activations/layer22_attention_weight_max": 51.31924819946289,
      "activations/layer22_attention_weight_min": -27.171703338623047,
      "activations/layer23_attention_weight_max": 35.3626708984375,
      "activations/layer23_attention_weight_min": -21.57271385192871,
      "activations/layer2_attention_weight_max": 32.526214599609375,
      "activations/layer2_attention_weight_min": -33.17291259765625,
      "activations/layer3_attention_weight_max": 94.98168182373047,
      "activations/layer3_attention_weight_min": -97.13310241699219,
      "activations/layer4_attention_weight_max": 124.33177947998047,
      "activations/layer4_attention_weight_min": -130.5050506591797,
      "activations/layer5_attention_weight_max": 57.083839416503906,
      "activations/layer5_attention_weight_min": -66.04464721679688,
      "activations/layer6_attention_weight_max": 48.909812927246094,
      "activations/layer6_attention_weight_min": -56.823020935058594,
      "activations/layer7_attention_weight_max": 87.697265625,
      "activations/layer7_attention_weight_min": -93.70939636230469,
      "activations/layer8_attention_weight_max": 39.59959411621094,
      "activations/layer8_attention_weight_min": -40.00844192504883,
      "activations/layer9_attention_weight_max": 34.447418212890625,
      "activations/layer9_attention_weight_min": -33.72462463378906,
      "epoch": 8.16,
      "learning_rate": 9.839924242424241e-05,
      "loss": 2.8134,
      "step": 140350
    },
    {
      "activations/layer0_attention_weight_max": 15.553875923156738,
      "activations/layer0_attention_weight_min": -12.522660255432129,
      "activations/layer10_attention_weight_max": 34.51327896118164,
      "activations/layer10_attention_weight_min": -35.53192138671875,
      "activations/layer11_attention_weight_max": 34.00598907470703,
      "activations/layer11_attention_weight_min": -30.57087516784668,
      "activations/layer12_attention_weight_max": 33.149391174316406,
      "activations/layer12_attention_weight_min": -23.997312545776367,
      "activations/layer13_attention_weight_max": 41.03786849975586,
      "activations/layer13_attention_weight_min": -33.561012268066406,
      "activations/layer14_attention_weight_max": 51.47956466674805,
      "activations/layer14_attention_weight_min": -39.24667739868164,
      "activations/layer15_attention_weight_max": 38.259986877441406,
      "activations/layer15_attention_weight_min": -30.51622772216797,
      "activations/layer16_attention_weight_max": 32.209022521972656,
      "activations/layer16_attention_weight_min": -29.028518676757812,
      "activations/layer17_attention_weight_max": 36.67097854614258,
      "activations/layer17_attention_weight_min": -25.781442642211914,
      "activations/layer18_attention_weight_max": 31.347248077392578,
      "activations/layer18_attention_weight_min": -20.944887161254883,
      "activations/layer19_attention_weight_max": 36.09170913696289,
      "activations/layer19_attention_weight_min": -32.66407775878906,
      "activations/layer1_attention_weight_max": 16.041643142700195,
      "activations/layer1_attention_weight_min": -15.275851249694824,
      "activations/layer20_attention_weight_max": 30.559995651245117,
      "activations/layer20_attention_weight_min": -24.52898597717285,
      "activations/layer21_attention_weight_max": 31.91156005859375,
      "activations/layer21_attention_weight_min": -22.644784927368164,
      "activations/layer22_attention_weight_max": 47.406246185302734,
      "activations/layer22_attention_weight_min": -29.717866897583008,
      "activations/layer23_attention_weight_max": 39.56949234008789,
      "activations/layer23_attention_weight_min": -23.217552185058594,
      "activations/layer2_attention_weight_max": 32.130645751953125,
      "activations/layer2_attention_weight_min": -32.02486038208008,
      "activations/layer3_attention_weight_max": 93.74703216552734,
      "activations/layer3_attention_weight_min": -91.50041961669922,
      "activations/layer4_attention_weight_max": 124.533447265625,
      "activations/layer4_attention_weight_min": -123.51179504394531,
      "activations/layer5_attention_weight_max": 62.24265670776367,
      "activations/layer5_attention_weight_min": -64.18538665771484,
      "activations/layer6_attention_weight_max": 51.961673736572266,
      "activations/layer6_attention_weight_min": -52.26564025878906,
      "activations/layer7_attention_weight_max": 97.3918228149414,
      "activations/layer7_attention_weight_min": -89.69426727294922,
      "activations/layer8_attention_weight_max": 42.59442901611328,
      "activations/layer8_attention_weight_min": -42.69899368286133,
      "activations/layer9_attention_weight_max": 35.85635757446289,
      "activations/layer9_attention_weight_min": -35.91435623168945,
      "epoch": 8.16,
      "learning_rate": 9.838030303030302e-05,
      "loss": 2.8144,
      "step": 140400
    },
    {
      "activations/layer0_attention_weight_max": 14.671202659606934,
      "activations/layer0_attention_weight_min": -11.668877601623535,
      "activations/layer10_attention_weight_max": 32.7359733581543,
      "activations/layer10_attention_weight_min": -31.65212059020996,
      "activations/layer11_attention_weight_max": 30.556827545166016,
      "activations/layer11_attention_weight_min": -29.358562469482422,
      "activations/layer12_attention_weight_max": 21.7421932220459,
      "activations/layer12_attention_weight_min": -25.3543701171875,
      "activations/layer13_attention_weight_max": 36.78804016113281,
      "activations/layer13_attention_weight_min": -33.63807678222656,
      "activations/layer14_attention_weight_max": 45.88404846191406,
      "activations/layer14_attention_weight_min": -37.72199249267578,
      "activations/layer15_attention_weight_max": 34.866939544677734,
      "activations/layer15_attention_weight_min": -28.38158416748047,
      "activations/layer16_attention_weight_max": 32.12648010253906,
      "activations/layer16_attention_weight_min": -29.57451057434082,
      "activations/layer17_attention_weight_max": 33.48396301269531,
      "activations/layer17_attention_weight_min": -24.137779235839844,
      "activations/layer18_attention_weight_max": 29.138534545898438,
      "activations/layer18_attention_weight_min": -19.844282150268555,
      "activations/layer19_attention_weight_max": 34.89339065551758,
      "activations/layer19_attention_weight_min": -28.946252822875977,
      "activations/layer1_attention_weight_max": 17.36017417907715,
      "activations/layer1_attention_weight_min": -15.995528221130371,
      "activations/layer20_attention_weight_max": 32.13461685180664,
      "activations/layer20_attention_weight_min": -23.62331771850586,
      "activations/layer21_attention_weight_max": 28.17040252685547,
      "activations/layer21_attention_weight_min": -21.772912979125977,
      "activations/layer22_attention_weight_max": 44.89956283569336,
      "activations/layer22_attention_weight_min": -26.26445960998535,
      "activations/layer23_attention_weight_max": 33.96281051635742,
      "activations/layer23_attention_weight_min": -23.764036178588867,
      "activations/layer2_attention_weight_max": 33.43994903564453,
      "activations/layer2_attention_weight_min": -32.0764045715332,
      "activations/layer3_attention_weight_max": 94.61117553710938,
      "activations/layer3_attention_weight_min": -92.0147476196289,
      "activations/layer4_attention_weight_max": 123.78157043457031,
      "activations/layer4_attention_weight_min": -120.81974029541016,
      "activations/layer5_attention_weight_max": 59.83538055419922,
      "activations/layer5_attention_weight_min": -64.05038452148438,
      "activations/layer6_attention_weight_max": 49.23208999633789,
      "activations/layer6_attention_weight_min": -52.62205505371094,
      "activations/layer7_attention_weight_max": 97.12471008300781,
      "activations/layer7_attention_weight_min": -95.37330627441406,
      "activations/layer8_attention_weight_max": 40.82499313354492,
      "activations/layer8_attention_weight_min": -42.91670608520508,
      "activations/layer9_attention_weight_max": 35.12897872924805,
      "activations/layer9_attention_weight_min": -33.334205627441406,
      "epoch": 8.16,
      "learning_rate": 9.836136363636363e-05,
      "loss": 2.8139,
      "step": 140450
    },
    {
      "activations/layer0_attention_weight_max": 15.124555587768555,
      "activations/layer0_attention_weight_min": -11.394426345825195,
      "activations/layer10_attention_weight_max": 42.187992095947266,
      "activations/layer10_attention_weight_min": -36.09340286254883,
      "activations/layer11_attention_weight_max": 40.44825744628906,
      "activations/layer11_attention_weight_min": -31.43411636352539,
      "activations/layer12_attention_weight_max": 29.620967864990234,
      "activations/layer12_attention_weight_min": -32.4788818359375,
      "activations/layer13_attention_weight_max": 45.63545227050781,
      "activations/layer13_attention_weight_min": -31.28299331665039,
      "activations/layer14_attention_weight_max": 65.45498657226562,
      "activations/layer14_attention_weight_min": -37.92850875854492,
      "activations/layer15_attention_weight_max": 42.801109313964844,
      "activations/layer15_attention_weight_min": -28.791963577270508,
      "activations/layer16_attention_weight_max": 43.8226432800293,
      "activations/layer16_attention_weight_min": -26.435155868530273,
      "activations/layer17_attention_weight_max": 38.14003372192383,
      "activations/layer17_attention_weight_min": -22.882280349731445,
      "activations/layer18_attention_weight_max": 31.2799015045166,
      "activations/layer18_attention_weight_min": -22.51280403137207,
      "activations/layer19_attention_weight_max": 40.71312713623047,
      "activations/layer19_attention_weight_min": -27.445362091064453,
      "activations/layer1_attention_weight_max": 17.249540328979492,
      "activations/layer1_attention_weight_min": -15.158369064331055,
      "activations/layer20_attention_weight_max": 32.07883071899414,
      "activations/layer20_attention_weight_min": -22.75735855102539,
      "activations/layer21_attention_weight_max": 29.791311264038086,
      "activations/layer21_attention_weight_min": -21.183523178100586,
      "activations/layer22_attention_weight_max": 44.80510711669922,
      "activations/layer22_attention_weight_min": -30.74588966369629,
      "activations/layer23_attention_weight_max": 35.95295715332031,
      "activations/layer23_attention_weight_min": -25.21674919128418,
      "activations/layer2_attention_weight_max": 36.87437057495117,
      "activations/layer2_attention_weight_min": -34.5514030456543,
      "activations/layer3_attention_weight_max": 102.30447387695312,
      "activations/layer3_attention_weight_min": -102.89993286132812,
      "activations/layer4_attention_weight_max": 128.92367553710938,
      "activations/layer4_attention_weight_min": -124.6962890625,
      "activations/layer5_attention_weight_max": 59.60992431640625,
      "activations/layer5_attention_weight_min": -62.306758880615234,
      "activations/layer6_attention_weight_max": 53.53940200805664,
      "activations/layer6_attention_weight_min": -51.31350326538086,
      "activations/layer7_attention_weight_max": 104.74983978271484,
      "activations/layer7_attention_weight_min": -95.84413146972656,
      "activations/layer8_attention_weight_max": 47.99101257324219,
      "activations/layer8_attention_weight_min": -43.864349365234375,
      "activations/layer9_attention_weight_max": 42.50589370727539,
      "activations/layer9_attention_weight_min": -35.81459045410156,
      "epoch": 8.16,
      "learning_rate": 9.834242424242425e-05,
      "loss": 2.7887,
      "step": 140500
    },
    {
      "activations/layer0_attention_weight_max": 16.257604598999023,
      "activations/layer0_attention_weight_min": -12.955269813537598,
      "activations/layer10_attention_weight_max": 35.42692947387695,
      "activations/layer10_attention_weight_min": -36.81645584106445,
      "activations/layer11_attention_weight_max": 33.2493782043457,
      "activations/layer11_attention_weight_min": -34.70978546142578,
      "activations/layer12_attention_weight_max": 28.544076919555664,
      "activations/layer12_attention_weight_min": -26.376461029052734,
      "activations/layer13_attention_weight_max": 41.43205642700195,
      "activations/layer13_attention_weight_min": -35.93840026855469,
      "activations/layer14_attention_weight_max": 53.78157043457031,
      "activations/layer14_attention_weight_min": -41.710758209228516,
      "activations/layer15_attention_weight_max": 40.0444221496582,
      "activations/layer15_attention_weight_min": -32.50700759887695,
      "activations/layer16_attention_weight_max": 36.69086837768555,
      "activations/layer16_attention_weight_min": -28.16591453552246,
      "activations/layer17_attention_weight_max": 35.159812927246094,
      "activations/layer17_attention_weight_min": -24.4185791015625,
      "activations/layer18_attention_weight_max": 37.18932342529297,
      "activations/layer18_attention_weight_min": -22.944847106933594,
      "activations/layer19_attention_weight_max": 41.66717529296875,
      "activations/layer19_attention_weight_min": -30.141183853149414,
      "activations/layer1_attention_weight_max": 16.622325897216797,
      "activations/layer1_attention_weight_min": -13.494484901428223,
      "activations/layer20_attention_weight_max": 34.36250686645508,
      "activations/layer20_attention_weight_min": -24.055395126342773,
      "activations/layer21_attention_weight_max": 37.96421813964844,
      "activations/layer21_attention_weight_min": -22.97767448425293,
      "activations/layer22_attention_weight_max": 52.47563552856445,
      "activations/layer22_attention_weight_min": -29.61430549621582,
      "activations/layer23_attention_weight_max": 38.5950813293457,
      "activations/layer23_attention_weight_min": -23.55398941040039,
      "activations/layer2_attention_weight_max": 33.208824157714844,
      "activations/layer2_attention_weight_min": -34.126319885253906,
      "activations/layer3_attention_weight_max": 101.13449096679688,
      "activations/layer3_attention_weight_min": -103.40608978271484,
      "activations/layer4_attention_weight_max": 124.34178924560547,
      "activations/layer4_attention_weight_min": -125.71871185302734,
      "activations/layer5_attention_weight_max": 59.09553146362305,
      "activations/layer5_attention_weight_min": -65.42312622070312,
      "activations/layer6_attention_weight_max": 50.341129302978516,
      "activations/layer6_attention_weight_min": -53.212162017822266,
      "activations/layer7_attention_weight_max": 93.91432189941406,
      "activations/layer7_attention_weight_min": -96.50968933105469,
      "activations/layer8_attention_weight_max": 44.536800384521484,
      "activations/layer8_attention_weight_min": -45.77336502075195,
      "activations/layer9_attention_weight_max": 39.494014739990234,
      "activations/layer9_attention_weight_min": -38.519081115722656,
      "epoch": 8.17,
      "learning_rate": 9.832348484848483e-05,
      "loss": 2.8051,
      "step": 140550
    },
    {
      "activations/layer0_attention_weight_max": 16.467754364013672,
      "activations/layer0_attention_weight_min": -12.716934204101562,
      "activations/layer10_attention_weight_max": 32.51005935668945,
      "activations/layer10_attention_weight_min": -32.41563415527344,
      "activations/layer11_attention_weight_max": 32.93632507324219,
      "activations/layer11_attention_weight_min": -30.564361572265625,
      "activations/layer12_attention_weight_max": 22.870834350585938,
      "activations/layer12_attention_weight_min": -24.32248306274414,
      "activations/layer13_attention_weight_max": 36.788902282714844,
      "activations/layer13_attention_weight_min": -30.75522804260254,
      "activations/layer14_attention_weight_max": 49.105857849121094,
      "activations/layer14_attention_weight_min": -40.23994445800781,
      "activations/layer15_attention_weight_max": 34.49624252319336,
      "activations/layer15_attention_weight_min": -27.987504959106445,
      "activations/layer16_attention_weight_max": 34.328773498535156,
      "activations/layer16_attention_weight_min": -27.833234786987305,
      "activations/layer17_attention_weight_max": 32.922019958496094,
      "activations/layer17_attention_weight_min": -25.904821395874023,
      "activations/layer18_attention_weight_max": 31.28607177734375,
      "activations/layer18_attention_weight_min": -20.620012283325195,
      "activations/layer19_attention_weight_max": 38.39108657836914,
      "activations/layer19_attention_weight_min": -31.43194580078125,
      "activations/layer1_attention_weight_max": 16.49484634399414,
      "activations/layer1_attention_weight_min": -14.640769004821777,
      "activations/layer20_attention_weight_max": 30.05898094177246,
      "activations/layer20_attention_weight_min": -24.28794288635254,
      "activations/layer21_attention_weight_max": 29.999523162841797,
      "activations/layer21_attention_weight_min": -23.725114822387695,
      "activations/layer22_attention_weight_max": 42.93915939331055,
      "activations/layer22_attention_weight_min": -29.73441505432129,
      "activations/layer23_attention_weight_max": 32.73377227783203,
      "activations/layer23_attention_weight_min": -26.024709701538086,
      "activations/layer2_attention_weight_max": 33.665077209472656,
      "activations/layer2_attention_weight_min": -31.678688049316406,
      "activations/layer3_attention_weight_max": 95.70475006103516,
      "activations/layer3_attention_weight_min": -93.83570098876953,
      "activations/layer4_attention_weight_max": 122.35350799560547,
      "activations/layer4_attention_weight_min": -118.89385223388672,
      "activations/layer5_attention_weight_max": 58.35914611816406,
      "activations/layer5_attention_weight_min": -64.29229736328125,
      "activations/layer6_attention_weight_max": 49.393043518066406,
      "activations/layer6_attention_weight_min": -52.9106330871582,
      "activations/layer7_attention_weight_max": 89.09915161132812,
      "activations/layer7_attention_weight_min": -89.79229736328125,
      "activations/layer8_attention_weight_max": 41.57400131225586,
      "activations/layer8_attention_weight_min": -44.40169143676758,
      "activations/layer9_attention_weight_max": 35.896419525146484,
      "activations/layer9_attention_weight_min": -34.98873519897461,
      "epoch": 8.17,
      "learning_rate": 9.830454545454545e-05,
      "loss": 2.8103,
      "step": 140600
    },
    {
      "activations/layer0_attention_weight_max": 15.966492652893066,
      "activations/layer0_attention_weight_min": -11.902231216430664,
      "activations/layer10_attention_weight_max": 31.59286117553711,
      "activations/layer10_attention_weight_min": -31.18755340576172,
      "activations/layer11_attention_weight_max": 31.046878814697266,
      "activations/layer11_attention_weight_min": -29.026622772216797,
      "activations/layer12_attention_weight_max": 22.965469360351562,
      "activations/layer12_attention_weight_min": -26.225685119628906,
      "activations/layer13_attention_weight_max": 43.14157485961914,
      "activations/layer13_attention_weight_min": -32.40240478515625,
      "activations/layer14_attention_weight_max": 50.41580581665039,
      "activations/layer14_attention_weight_min": -38.81359100341797,
      "activations/layer15_attention_weight_max": 34.045921325683594,
      "activations/layer15_attention_weight_min": -28.995323181152344,
      "activations/layer16_attention_weight_max": 32.01538848876953,
      "activations/layer16_attention_weight_min": -30.810869216918945,
      "activations/layer17_attention_weight_max": 31.91229248046875,
      "activations/layer17_attention_weight_min": -24.602792739868164,
      "activations/layer18_attention_weight_max": 29.15507698059082,
      "activations/layer18_attention_weight_min": -20.44173812866211,
      "activations/layer19_attention_weight_max": 42.622535705566406,
      "activations/layer19_attention_weight_min": -32.432308197021484,
      "activations/layer1_attention_weight_max": 17.14217758178711,
      "activations/layer1_attention_weight_min": -14.92025375366211,
      "activations/layer20_attention_weight_max": 28.174047470092773,
      "activations/layer20_attention_weight_min": -27.402114868164062,
      "activations/layer21_attention_weight_max": 29.726011276245117,
      "activations/layer21_attention_weight_min": -24.982730865478516,
      "activations/layer22_attention_weight_max": 47.25950622558594,
      "activations/layer22_attention_weight_min": -29.655561447143555,
      "activations/layer23_attention_weight_max": 36.65427017211914,
      "activations/layer23_attention_weight_min": -25.259923934936523,
      "activations/layer2_attention_weight_max": 31.988933563232422,
      "activations/layer2_attention_weight_min": -30.556577682495117,
      "activations/layer3_attention_weight_max": 91.63660430908203,
      "activations/layer3_attention_weight_min": -90.46784210205078,
      "activations/layer4_attention_weight_max": 115.42164611816406,
      "activations/layer4_attention_weight_min": -117.1501693725586,
      "activations/layer5_attention_weight_max": 54.980499267578125,
      "activations/layer5_attention_weight_min": -60.266632080078125,
      "activations/layer6_attention_weight_max": 47.65995788574219,
      "activations/layer6_attention_weight_min": -50.58395004272461,
      "activations/layer7_attention_weight_max": 86.054443359375,
      "activations/layer7_attention_weight_min": -89.12974548339844,
      "activations/layer8_attention_weight_max": 38.17535400390625,
      "activations/layer8_attention_weight_min": -42.528099060058594,
      "activations/layer9_attention_weight_max": 31.74598503112793,
      "activations/layer9_attention_weight_min": -33.47594451904297,
      "epoch": 8.17,
      "learning_rate": 9.828560606060606e-05,
      "loss": 2.8104,
      "step": 140650
    },
    {
      "activations/layer0_attention_weight_max": 14.958045959472656,
      "activations/layer0_attention_weight_min": -11.369418144226074,
      "activations/layer10_attention_weight_max": 30.227245330810547,
      "activations/layer10_attention_weight_min": -30.304519653320312,
      "activations/layer11_attention_weight_max": 30.437772750854492,
      "activations/layer11_attention_weight_min": -28.90078353881836,
      "activations/layer12_attention_weight_max": 30.751340866088867,
      "activations/layer12_attention_weight_min": -29.47022247314453,
      "activations/layer13_attention_weight_max": 42.7918586730957,
      "activations/layer13_attention_weight_min": -33.310752868652344,
      "activations/layer14_attention_weight_max": 52.896419525146484,
      "activations/layer14_attention_weight_min": -37.80820083618164,
      "activations/layer15_attention_weight_max": 42.90779495239258,
      "activations/layer15_attention_weight_min": -29.840221405029297,
      "activations/layer16_attention_weight_max": 35.0022087097168,
      "activations/layer16_attention_weight_min": -25.969629287719727,
      "activations/layer17_attention_weight_max": 37.9131965637207,
      "activations/layer17_attention_weight_min": -24.438867568969727,
      "activations/layer18_attention_weight_max": 37.54255676269531,
      "activations/layer18_attention_weight_min": -20.74781036376953,
      "activations/layer19_attention_weight_max": 39.1863899230957,
      "activations/layer19_attention_weight_min": -29.460296630859375,
      "activations/layer1_attention_weight_max": 17.190025329589844,
      "activations/layer1_attention_weight_min": -13.885451316833496,
      "activations/layer20_attention_weight_max": 34.82636260986328,
      "activations/layer20_attention_weight_min": -23.343013763427734,
      "activations/layer21_attention_weight_max": 34.255271911621094,
      "activations/layer21_attention_weight_min": -21.54389190673828,
      "activations/layer22_attention_weight_max": 51.774383544921875,
      "activations/layer22_attention_weight_min": -26.760848999023438,
      "activations/layer23_attention_weight_max": 42.31040954589844,
      "activations/layer23_attention_weight_min": -23.7526912689209,
      "activations/layer2_attention_weight_max": 31.631391525268555,
      "activations/layer2_attention_weight_min": -31.891523361206055,
      "activations/layer3_attention_weight_max": 90.24371337890625,
      "activations/layer3_attention_weight_min": -94.85076141357422,
      "activations/layer4_attention_weight_max": 117.6958236694336,
      "activations/layer4_attention_weight_min": -116.75182342529297,
      "activations/layer5_attention_weight_max": 55.557411193847656,
      "activations/layer5_attention_weight_min": -63.339805603027344,
      "activations/layer6_attention_weight_max": 48.292049407958984,
      "activations/layer6_attention_weight_min": -53.61025619506836,
      "activations/layer7_attention_weight_max": 88.27422332763672,
      "activations/layer7_attention_weight_min": -86.72527313232422,
      "activations/layer8_attention_weight_max": 39.917694091796875,
      "activations/layer8_attention_weight_min": -38.40260696411133,
      "activations/layer9_attention_weight_max": 35.26924133300781,
      "activations/layer9_attention_weight_min": -33.343421936035156,
      "epoch": 8.18,
      "learning_rate": 9.826666666666665e-05,
      "loss": 2.8199,
      "step": 140700
    },
    {
      "activations/layer0_attention_weight_max": 16.245040893554688,
      "activations/layer0_attention_weight_min": -12.337870597839355,
      "activations/layer10_attention_weight_max": 31.450273513793945,
      "activations/layer10_attention_weight_min": -30.83123779296875,
      "activations/layer11_attention_weight_max": 29.478147506713867,
      "activations/layer11_attention_weight_min": -27.635169982910156,
      "activations/layer12_attention_weight_max": 20.981658935546875,
      "activations/layer12_attention_weight_min": -26.16367530822754,
      "activations/layer13_attention_weight_max": 36.479183197021484,
      "activations/layer13_attention_weight_min": -35.067596435546875,
      "activations/layer14_attention_weight_max": 42.544376373291016,
      "activations/layer14_attention_weight_min": -38.24614334106445,
      "activations/layer15_attention_weight_max": 35.7424430847168,
      "activations/layer15_attention_weight_min": -32.183135986328125,
      "activations/layer16_attention_weight_max": 29.98754119873047,
      "activations/layer16_attention_weight_min": -28.04107666015625,
      "activations/layer17_attention_weight_max": 32.47346878051758,
      "activations/layer17_attention_weight_min": -24.755977630615234,
      "activations/layer18_attention_weight_max": 29.09686279296875,
      "activations/layer18_attention_weight_min": -22.45382308959961,
      "activations/layer19_attention_weight_max": 35.850337982177734,
      "activations/layer19_attention_weight_min": -29.348840713500977,
      "activations/layer1_attention_weight_max": 16.059635162353516,
      "activations/layer1_attention_weight_min": -17.22150421142578,
      "activations/layer20_attention_weight_max": 29.430667877197266,
      "activations/layer20_attention_weight_min": -24.6415958404541,
      "activations/layer21_attention_weight_max": 32.3661003112793,
      "activations/layer21_attention_weight_min": -24.795923233032227,
      "activations/layer22_attention_weight_max": 42.126914978027344,
      "activations/layer22_attention_weight_min": -27.79476547241211,
      "activations/layer23_attention_weight_max": 34.35347366333008,
      "activations/layer23_attention_weight_min": -23.93202018737793,
      "activations/layer2_attention_weight_max": 32.14353942871094,
      "activations/layer2_attention_weight_min": -31.250289916992188,
      "activations/layer3_attention_weight_max": 90.63874816894531,
      "activations/layer3_attention_weight_min": -90.11824035644531,
      "activations/layer4_attention_weight_max": 114.89155578613281,
      "activations/layer4_attention_weight_min": -119.3521728515625,
      "activations/layer5_attention_weight_max": 59.087623596191406,
      "activations/layer5_attention_weight_min": -65.74324035644531,
      "activations/layer6_attention_weight_max": 46.55666732788086,
      "activations/layer6_attention_weight_min": -50.04481887817383,
      "activations/layer7_attention_weight_max": 87.13297271728516,
      "activations/layer7_attention_weight_min": -91.39573669433594,
      "activations/layer8_attention_weight_max": 39.711788177490234,
      "activations/layer8_attention_weight_min": -39.245948791503906,
      "activations/layer9_attention_weight_max": 33.099891662597656,
      "activations/layer9_attention_weight_min": -32.89933776855469,
      "epoch": 8.18,
      "learning_rate": 9.824772727272727e-05,
      "loss": 2.8038,
      "step": 140750
    },
    {
      "activations/layer0_attention_weight_max": 14.957854270935059,
      "activations/layer0_attention_weight_min": -11.764276504516602,
      "activations/layer10_attention_weight_max": 33.748451232910156,
      "activations/layer10_attention_weight_min": -35.21770095825195,
      "activations/layer11_attention_weight_max": 33.32279968261719,
      "activations/layer11_attention_weight_min": -31.050376892089844,
      "activations/layer12_attention_weight_max": 23.92057991027832,
      "activations/layer12_attention_weight_min": -27.78044891357422,
      "activations/layer13_attention_weight_max": 40.3414192199707,
      "activations/layer13_attention_weight_min": -35.35749435424805,
      "activations/layer14_attention_weight_max": 45.54254913330078,
      "activations/layer14_attention_weight_min": -36.725685119628906,
      "activations/layer15_attention_weight_max": 36.59198760986328,
      "activations/layer15_attention_weight_min": -31.937973022460938,
      "activations/layer16_attention_weight_max": 30.768444061279297,
      "activations/layer16_attention_weight_min": -27.280824661254883,
      "activations/layer17_attention_weight_max": 31.694318771362305,
      "activations/layer17_attention_weight_min": -23.81925392150879,
      "activations/layer18_attention_weight_max": 32.385860443115234,
      "activations/layer18_attention_weight_min": -21.159162521362305,
      "activations/layer19_attention_weight_max": 35.16304397583008,
      "activations/layer19_attention_weight_min": -29.87396812438965,
      "activations/layer1_attention_weight_max": 16.671695709228516,
      "activations/layer1_attention_weight_min": -13.702265739440918,
      "activations/layer20_attention_weight_max": 29.610795974731445,
      "activations/layer20_attention_weight_min": -23.296213150024414,
      "activations/layer21_attention_weight_max": 28.86733627319336,
      "activations/layer21_attention_weight_min": -21.89682960510254,
      "activations/layer22_attention_weight_max": 43.5222282409668,
      "activations/layer22_attention_weight_min": -30.982316970825195,
      "activations/layer23_attention_weight_max": 34.14726638793945,
      "activations/layer23_attention_weight_min": -24.260501861572266,
      "activations/layer2_attention_weight_max": 30.612056732177734,
      "activations/layer2_attention_weight_min": -30.27375602722168,
      "activations/layer3_attention_weight_max": 92.42948913574219,
      "activations/layer3_attention_weight_min": -90.42213439941406,
      "activations/layer4_attention_weight_max": 119.10704803466797,
      "activations/layer4_attention_weight_min": -117.2706298828125,
      "activations/layer5_attention_weight_max": 56.787750244140625,
      "activations/layer5_attention_weight_min": -62.74742126464844,
      "activations/layer6_attention_weight_max": 47.13637924194336,
      "activations/layer6_attention_weight_min": -52.86616516113281,
      "activations/layer7_attention_weight_max": 89.78690338134766,
      "activations/layer7_attention_weight_min": -91.48888397216797,
      "activations/layer8_attention_weight_max": 41.495269775390625,
      "activations/layer8_attention_weight_min": -45.5233268737793,
      "activations/layer9_attention_weight_max": 35.19649887084961,
      "activations/layer9_attention_weight_min": -36.986595153808594,
      "epoch": 8.18,
      "learning_rate": 9.822878787878787e-05,
      "loss": 2.8169,
      "step": 140800
    },
    {
      "activations/layer0_attention_weight_max": 16.212263107299805,
      "activations/layer0_attention_weight_min": -12.375005722045898,
      "activations/layer10_attention_weight_max": 31.000205993652344,
      "activations/layer10_attention_weight_min": -31.73931884765625,
      "activations/layer11_attention_weight_max": 30.3704891204834,
      "activations/layer11_attention_weight_min": -30.510395050048828,
      "activations/layer12_attention_weight_max": 27.71658706665039,
      "activations/layer12_attention_weight_min": -26.264772415161133,
      "activations/layer13_attention_weight_max": 43.158531188964844,
      "activations/layer13_attention_weight_min": -35.878807067871094,
      "activations/layer14_attention_weight_max": 56.87177276611328,
      "activations/layer14_attention_weight_min": -43.07674789428711,
      "activations/layer15_attention_weight_max": 38.68326187133789,
      "activations/layer15_attention_weight_min": -32.84035110473633,
      "activations/layer16_attention_weight_max": 36.96572494506836,
      "activations/layer16_attention_weight_min": -28.586254119873047,
      "activations/layer17_attention_weight_max": 37.73442459106445,
      "activations/layer17_attention_weight_min": -26.5096378326416,
      "activations/layer18_attention_weight_max": 30.970272064208984,
      "activations/layer18_attention_weight_min": -21.40614891052246,
      "activations/layer19_attention_weight_max": 40.53594207763672,
      "activations/layer19_attention_weight_min": -31.37646484375,
      "activations/layer1_attention_weight_max": 17.56683921813965,
      "activations/layer1_attention_weight_min": -14.062485694885254,
      "activations/layer20_attention_weight_max": 36.425750732421875,
      "activations/layer20_attention_weight_min": -24.652637481689453,
      "activations/layer21_attention_weight_max": 32.77595901489258,
      "activations/layer21_attention_weight_min": -23.942480087280273,
      "activations/layer22_attention_weight_max": 51.1807746887207,
      "activations/layer22_attention_weight_min": -29.79522705078125,
      "activations/layer23_attention_weight_max": 40.17462921142578,
      "activations/layer23_attention_weight_min": -24.329374313354492,
      "activations/layer2_attention_weight_max": 33.5331916809082,
      "activations/layer2_attention_weight_min": -32.399253845214844,
      "activations/layer3_attention_weight_max": 93.65370178222656,
      "activations/layer3_attention_weight_min": -92.42791748046875,
      "activations/layer4_attention_weight_max": 122.50093841552734,
      "activations/layer4_attention_weight_min": -120.012939453125,
      "activations/layer5_attention_weight_max": 59.721946716308594,
      "activations/layer5_attention_weight_min": -64.99819946289062,
      "activations/layer6_attention_weight_max": 47.83598709106445,
      "activations/layer6_attention_weight_min": -52.147457122802734,
      "activations/layer7_attention_weight_max": 86.18182373046875,
      "activations/layer7_attention_weight_min": -86.73687744140625,
      "activations/layer8_attention_weight_max": 40.13722610473633,
      "activations/layer8_attention_weight_min": -40.661964416503906,
      "activations/layer9_attention_weight_max": 33.69851303100586,
      "activations/layer9_attention_weight_min": -34.42483139038086,
      "epoch": 8.18,
      "learning_rate": 9.820984848484847e-05,
      "loss": 2.8118,
      "step": 140850
    },
    {
      "activations/layer0_attention_weight_max": 16.21585464477539,
      "activations/layer0_attention_weight_min": -12.342497825622559,
      "activations/layer10_attention_weight_max": 31.74307632446289,
      "activations/layer10_attention_weight_min": -34.20573806762695,
      "activations/layer11_attention_weight_max": 31.57857322692871,
      "activations/layer11_attention_weight_min": -33.104923248291016,
      "activations/layer12_attention_weight_max": 22.53274154663086,
      "activations/layer12_attention_weight_min": -26.020095825195312,
      "activations/layer13_attention_weight_max": 38.33843994140625,
      "activations/layer13_attention_weight_min": -32.9161376953125,
      "activations/layer14_attention_weight_max": 45.842796325683594,
      "activations/layer14_attention_weight_min": -37.50812530517578,
      "activations/layer15_attention_weight_max": 36.521385192871094,
      "activations/layer15_attention_weight_min": -32.35588073730469,
      "activations/layer16_attention_weight_max": 35.11968994140625,
      "activations/layer16_attention_weight_min": -27.858354568481445,
      "activations/layer17_attention_weight_max": 34.2286262512207,
      "activations/layer17_attention_weight_min": -25.982807159423828,
      "activations/layer18_attention_weight_max": 32.048160552978516,
      "activations/layer18_attention_weight_min": -23.276416778564453,
      "activations/layer19_attention_weight_max": 37.215728759765625,
      "activations/layer19_attention_weight_min": -30.6047306060791,
      "activations/layer1_attention_weight_max": 17.651845932006836,
      "activations/layer1_attention_weight_min": -13.96484661102295,
      "activations/layer20_attention_weight_max": 30.88582992553711,
      "activations/layer20_attention_weight_min": -24.256261825561523,
      "activations/layer21_attention_weight_max": 30.199705123901367,
      "activations/layer21_attention_weight_min": -23.8013973236084,
      "activations/layer22_attention_weight_max": 46.09906005859375,
      "activations/layer22_attention_weight_min": -32.110740661621094,
      "activations/layer23_attention_weight_max": 35.82440185546875,
      "activations/layer23_attention_weight_min": -24.956493377685547,
      "activations/layer2_attention_weight_max": 32.74114227294922,
      "activations/layer2_attention_weight_min": -32.237850189208984,
      "activations/layer3_attention_weight_max": 95.83863830566406,
      "activations/layer3_attention_weight_min": -92.0784683227539,
      "activations/layer4_attention_weight_max": 126.90185546875,
      "activations/layer4_attention_weight_min": -122.57366180419922,
      "activations/layer5_attention_weight_max": 61.99287033081055,
      "activations/layer5_attention_weight_min": -65.52359771728516,
      "activations/layer6_attention_weight_max": 52.308963775634766,
      "activations/layer6_attention_weight_min": -52.09092330932617,
      "activations/layer7_attention_weight_max": 89.88739013671875,
      "activations/layer7_attention_weight_min": -90.22425079345703,
      "activations/layer8_attention_weight_max": 42.35471725463867,
      "activations/layer8_attention_weight_min": -42.050750732421875,
      "activations/layer9_attention_weight_max": 35.2766227722168,
      "activations/layer9_attention_weight_min": -34.317970275878906,
      "epoch": 8.19,
      "learning_rate": 9.819090909090908e-05,
      "loss": 2.8176,
      "step": 140900
    },
    {
      "activations/layer0_attention_weight_max": 16.841304779052734,
      "activations/layer0_attention_weight_min": -11.949728012084961,
      "activations/layer10_attention_weight_max": 36.62398147583008,
      "activations/layer10_attention_weight_min": -34.12278366088867,
      "activations/layer11_attention_weight_max": 40.70857238769531,
      "activations/layer11_attention_weight_min": -30.63996124267578,
      "activations/layer12_attention_weight_max": 33.03159713745117,
      "activations/layer12_attention_weight_min": -29.022676467895508,
      "activations/layer13_attention_weight_max": 42.50636672973633,
      "activations/layer13_attention_weight_min": -32.88044738769531,
      "activations/layer14_attention_weight_max": 64.33736419677734,
      "activations/layer14_attention_weight_min": -46.71698760986328,
      "activations/layer15_attention_weight_max": 37.1209602355957,
      "activations/layer15_attention_weight_min": -31.02215576171875,
      "activations/layer16_attention_weight_max": 35.712013244628906,
      "activations/layer16_attention_weight_min": -27.812671661376953,
      "activations/layer17_attention_weight_max": 34.22632598876953,
      "activations/layer17_attention_weight_min": -24.7204532623291,
      "activations/layer18_attention_weight_max": 30.881614685058594,
      "activations/layer18_attention_weight_min": -21.03742027282715,
      "activations/layer19_attention_weight_max": 39.515602111816406,
      "activations/layer19_attention_weight_min": -35.23499298095703,
      "activations/layer1_attention_weight_max": 17.436386108398438,
      "activations/layer1_attention_weight_min": -13.580333709716797,
      "activations/layer20_attention_weight_max": 32.34403991699219,
      "activations/layer20_attention_weight_min": -23.644704818725586,
      "activations/layer21_attention_weight_max": 29.8573055267334,
      "activations/layer21_attention_weight_min": -22.992294311523438,
      "activations/layer22_attention_weight_max": 49.46584701538086,
      "activations/layer22_attention_weight_min": -28.864973068237305,
      "activations/layer23_attention_weight_max": 40.09736633300781,
      "activations/layer23_attention_weight_min": -25.720104217529297,
      "activations/layer2_attention_weight_max": 32.16755676269531,
      "activations/layer2_attention_weight_min": -32.21846389770508,
      "activations/layer3_attention_weight_max": 89.79695129394531,
      "activations/layer3_attention_weight_min": -93.0207748413086,
      "activations/layer4_attention_weight_max": 115.84991455078125,
      "activations/layer4_attention_weight_min": -116.01201629638672,
      "activations/layer5_attention_weight_max": 54.166778564453125,
      "activations/layer5_attention_weight_min": -62.353126525878906,
      "activations/layer6_attention_weight_max": 47.06499481201172,
      "activations/layer6_attention_weight_min": -52.511573791503906,
      "activations/layer7_attention_weight_max": 87.13904571533203,
      "activations/layer7_attention_weight_min": -97.31729125976562,
      "activations/layer8_attention_weight_max": 41.680015563964844,
      "activations/layer8_attention_weight_min": -40.74298095703125,
      "activations/layer9_attention_weight_max": 36.692935943603516,
      "activations/layer9_attention_weight_min": -34.87845230102539,
      "epoch": 8.19,
      "learning_rate": 9.817196969696969e-05,
      "loss": 2.8138,
      "step": 140950
    },
    {
      "activations/layer0_attention_weight_max": 15.717646598815918,
      "activations/layer0_attention_weight_min": -12.743292808532715,
      "activations/layer10_attention_weight_max": 33.90910720825195,
      "activations/layer10_attention_weight_min": -32.270118713378906,
      "activations/layer11_attention_weight_max": 34.023529052734375,
      "activations/layer11_attention_weight_min": -29.727384567260742,
      "activations/layer12_attention_weight_max": 30.632890701293945,
      "activations/layer12_attention_weight_min": -26.05980682373047,
      "activations/layer13_attention_weight_max": 40.666748046875,
      "activations/layer13_attention_weight_min": -32.21359634399414,
      "activations/layer14_attention_weight_max": 56.09905242919922,
      "activations/layer14_attention_weight_min": -42.03731155395508,
      "activations/layer15_attention_weight_max": 38.34089660644531,
      "activations/layer15_attention_weight_min": -31.679048538208008,
      "activations/layer16_attention_weight_max": 34.890098571777344,
      "activations/layer16_attention_weight_min": -27.17898941040039,
      "activations/layer17_attention_weight_max": 34.14134979248047,
      "activations/layer17_attention_weight_min": -25.355674743652344,
      "activations/layer18_attention_weight_max": 30.39781379699707,
      "activations/layer18_attention_weight_min": -20.942472457885742,
      "activations/layer19_attention_weight_max": 35.711185455322266,
      "activations/layer19_attention_weight_min": -31.113834381103516,
      "activations/layer1_attention_weight_max": 17.531578063964844,
      "activations/layer1_attention_weight_min": -13.543978691101074,
      "activations/layer20_attention_weight_max": 32.00184631347656,
      "activations/layer20_attention_weight_min": -24.364208221435547,
      "activations/layer21_attention_weight_max": 29.833402633666992,
      "activations/layer21_attention_weight_min": -24.26054573059082,
      "activations/layer22_attention_weight_max": 45.09519958496094,
      "activations/layer22_attention_weight_min": -27.916643142700195,
      "activations/layer23_attention_weight_max": 36.61531066894531,
      "activations/layer23_attention_weight_min": -22.15317153930664,
      "activations/layer2_attention_weight_max": 32.53178405761719,
      "activations/layer2_attention_weight_min": -31.310964584350586,
      "activations/layer3_attention_weight_max": 90.9952621459961,
      "activations/layer3_attention_weight_min": -92.95308685302734,
      "activations/layer4_attention_weight_max": 119.10145568847656,
      "activations/layer4_attention_weight_min": -117.02191162109375,
      "activations/layer5_attention_weight_max": 55.93145751953125,
      "activations/layer5_attention_weight_min": -61.466705322265625,
      "activations/layer6_attention_weight_max": 49.15874099731445,
      "activations/layer6_attention_weight_min": -50.96379470825195,
      "activations/layer7_attention_weight_max": 91.91574096679688,
      "activations/layer7_attention_weight_min": -92.88455200195312,
      "activations/layer8_attention_weight_max": 42.38544464111328,
      "activations/layer8_attention_weight_min": -41.06450271606445,
      "activations/layer9_attention_weight_max": 35.833614349365234,
      "activations/layer9_attention_weight_min": -35.431888580322266,
      "epoch": 8.19,
      "learning_rate": 9.815303030303029e-05,
      "loss": 2.8015,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.505,
      "eval_samples_per_second": 504.878,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.505,
      "eval_openwebtext_samples_per_second": 504.878,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 1.9929,
      "eval_wikitext_samples_per_second": 228.814,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_lambada_loss": 2.615234375,
      "eval_lambada_ppl": 13.67042000714299,
      "eval_lambada_runtime": 9.628,
      "eval_lambada_samples_per_second": 505.715,
      "step": 141000
    },
    {
      "activations/layer0_attention_weight_max": 15.994624137878418,
      "activations/layer0_attention_weight_min": -12.022552490234375,
      "activations/layer10_attention_weight_max": 31.026031494140625,
      "activations/layer10_attention_weight_min": -32.22913360595703,
      "activations/layer11_attention_weight_max": 30.899635314941406,
      "activations/layer11_attention_weight_min": -29.18895721435547,
      "activations/layer12_attention_weight_max": 22.935325622558594,
      "activations/layer12_attention_weight_min": -23.777099609375,
      "activations/layer13_attention_weight_max": 38.64784622192383,
      "activations/layer13_attention_weight_min": -32.149288177490234,
      "activations/layer14_attention_weight_max": 46.81636047363281,
      "activations/layer14_attention_weight_min": -37.926815032958984,
      "activations/layer15_attention_weight_max": 34.58406066894531,
      "activations/layer15_attention_weight_min": -30.53313636779785,
      "activations/layer16_attention_weight_max": 30.89866065979004,
      "activations/layer16_attention_weight_min": -26.60662078857422,
      "activations/layer17_attention_weight_max": 32.07973098754883,
      "activations/layer17_attention_weight_min": -23.98809242248535,
      "activations/layer18_attention_weight_max": 30.52077865600586,
      "activations/layer18_attention_weight_min": -21.612812042236328,
      "activations/layer19_attention_weight_max": 33.980079650878906,
      "activations/layer19_attention_weight_min": -29.07236671447754,
      "activations/layer1_attention_weight_max": 16.55400276184082,
      "activations/layer1_attention_weight_min": -14.787223815917969,
      "activations/layer20_attention_weight_max": 27.371274948120117,
      "activations/layer20_attention_weight_min": -23.20823860168457,
      "activations/layer21_attention_weight_max": 29.812870025634766,
      "activations/layer21_attention_weight_min": -21.678800582885742,
      "activations/layer22_attention_weight_max": 44.1956787109375,
      "activations/layer22_attention_weight_min": -29.881120681762695,
      "activations/layer23_attention_weight_max": 36.21861267089844,
      "activations/layer23_attention_weight_min": -23.27668571472168,
      "activations/layer2_attention_weight_max": 32.738746643066406,
      "activations/layer2_attention_weight_min": -31.469755172729492,
      "activations/layer3_attention_weight_max": 92.84796905517578,
      "activations/layer3_attention_weight_min": -91.36119079589844,
      "activations/layer4_attention_weight_max": 116.77137756347656,
      "activations/layer4_attention_weight_min": -116.0653305053711,
      "activations/layer5_attention_weight_max": 55.479366302490234,
      "activations/layer5_attention_weight_min": -61.49540328979492,
      "activations/layer6_attention_weight_max": 46.58169937133789,
      "activations/layer6_attention_weight_min": -50.46730422973633,
      "activations/layer7_attention_weight_max": 83.32890319824219,
      "activations/layer7_attention_weight_min": -86.85051727294922,
      "activations/layer8_attention_weight_max": 39.55469512939453,
      "activations/layer8_attention_weight_min": -40.468772888183594,
      "activations/layer9_attention_weight_max": 33.98566818237305,
      "activations/layer9_attention_weight_min": -32.030662536621094,
      "epoch": 8.2,
      "learning_rate": 9.81340909090909e-05,
      "loss": 2.8035,
      "step": 141050
    },
    {
      "activations/layer0_attention_weight_max": 15.723738670349121,
      "activations/layer0_attention_weight_min": -12.448391914367676,
      "activations/layer10_attention_weight_max": 32.39828109741211,
      "activations/layer10_attention_weight_min": -32.55180740356445,
      "activations/layer11_attention_weight_max": 30.393674850463867,
      "activations/layer11_attention_weight_min": -30.291893005371094,
      "activations/layer12_attention_weight_max": 22.773313522338867,
      "activations/layer12_attention_weight_min": -27.89128303527832,
      "activations/layer13_attention_weight_max": 45.80268478393555,
      "activations/layer13_attention_weight_min": -33.897727966308594,
      "activations/layer14_attention_weight_max": 57.00923156738281,
      "activations/layer14_attention_weight_min": -44.067626953125,
      "activations/layer15_attention_weight_max": 38.047882080078125,
      "activations/layer15_attention_weight_min": -28.4294490814209,
      "activations/layer16_attention_weight_max": 35.27579879760742,
      "activations/layer16_attention_weight_min": -30.55069923400879,
      "activations/layer17_attention_weight_max": 34.809391021728516,
      "activations/layer17_attention_weight_min": -24.748239517211914,
      "activations/layer18_attention_weight_max": 31.40448570251465,
      "activations/layer18_attention_weight_min": -21.026884078979492,
      "activations/layer19_attention_weight_max": 38.44559097290039,
      "activations/layer19_attention_weight_min": -30.0639705657959,
      "activations/layer1_attention_weight_max": 16.702072143554688,
      "activations/layer1_attention_weight_min": -15.67052936553955,
      "activations/layer20_attention_weight_max": 33.61670684814453,
      "activations/layer20_attention_weight_min": -22.81722068786621,
      "activations/layer21_attention_weight_max": 31.002248764038086,
      "activations/layer21_attention_weight_min": -23.642606735229492,
      "activations/layer22_attention_weight_max": 47.99857711791992,
      "activations/layer22_attention_weight_min": -27.384424209594727,
      "activations/layer23_attention_weight_max": 37.87160110473633,
      "activations/layer23_attention_weight_min": -23.52627182006836,
      "activations/layer2_attention_weight_max": 33.86686706542969,
      "activations/layer2_attention_weight_min": -33.3865966796875,
      "activations/layer3_attention_weight_max": 96.83808135986328,
      "activations/layer3_attention_weight_min": -97.86571502685547,
      "activations/layer4_attention_weight_max": 124.27192687988281,
      "activations/layer4_attention_weight_min": -127.71117401123047,
      "activations/layer5_attention_weight_max": 57.21564865112305,
      "activations/layer5_attention_weight_min": -62.892173767089844,
      "activations/layer6_attention_weight_max": 50.361759185791016,
      "activations/layer6_attention_weight_min": -50.13888931274414,
      "activations/layer7_attention_weight_max": 90.21685028076172,
      "activations/layer7_attention_weight_min": -85.11626434326172,
      "activations/layer8_attention_weight_max": 39.88037872314453,
      "activations/layer8_attention_weight_min": -39.687686920166016,
      "activations/layer9_attention_weight_max": 32.84132766723633,
      "activations/layer9_attention_weight_min": -34.64361572265625,
      "epoch": 8.2,
      "learning_rate": 9.81151515151515e-05,
      "loss": 2.8013,
      "step": 141100
    },
    {
      "activations/layer0_attention_weight_max": 15.384099960327148,
      "activations/layer0_attention_weight_min": -12.47765064239502,
      "activations/layer10_attention_weight_max": 30.920785903930664,
      "activations/layer10_attention_weight_min": -31.522436141967773,
      "activations/layer11_attention_weight_max": 31.217987060546875,
      "activations/layer11_attention_weight_min": -29.866619110107422,
      "activations/layer12_attention_weight_max": 23.92983627319336,
      "activations/layer12_attention_weight_min": -25.610702514648438,
      "activations/layer13_attention_weight_max": 36.60813522338867,
      "activations/layer13_attention_weight_min": -32.08637237548828,
      "activations/layer14_attention_weight_max": 41.67593765258789,
      "activations/layer14_attention_weight_min": -34.437103271484375,
      "activations/layer15_attention_weight_max": 36.79071044921875,
      "activations/layer15_attention_weight_min": -28.335433959960938,
      "activations/layer16_attention_weight_max": 31.608264923095703,
      "activations/layer16_attention_weight_min": -27.72623634338379,
      "activations/layer17_attention_weight_max": 33.25070571899414,
      "activations/layer17_attention_weight_min": -24.918729782104492,
      "activations/layer18_attention_weight_max": 30.27468490600586,
      "activations/layer18_attention_weight_min": -20.989368438720703,
      "activations/layer19_attention_weight_max": 37.69050216674805,
      "activations/layer19_attention_weight_min": -29.48516273498535,
      "activations/layer1_attention_weight_max": 16.207910537719727,
      "activations/layer1_attention_weight_min": -13.580419540405273,
      "activations/layer20_attention_weight_max": 29.348308563232422,
      "activations/layer20_attention_weight_min": -22.911670684814453,
      "activations/layer21_attention_weight_max": 28.684234619140625,
      "activations/layer21_attention_weight_min": -21.168628692626953,
      "activations/layer22_attention_weight_max": 44.44550323486328,
      "activations/layer22_attention_weight_min": -28.206098556518555,
      "activations/layer23_attention_weight_max": 38.53390884399414,
      "activations/layer23_attention_weight_min": -24.46013641357422,
      "activations/layer2_attention_weight_max": 31.031919479370117,
      "activations/layer2_attention_weight_min": -30.95569610595703,
      "activations/layer3_attention_weight_max": 92.73970031738281,
      "activations/layer3_attention_weight_min": -91.71627807617188,
      "activations/layer4_attention_weight_max": 121.66705322265625,
      "activations/layer4_attention_weight_min": -117.96565246582031,
      "activations/layer5_attention_weight_max": 58.31620407104492,
      "activations/layer5_attention_weight_min": -60.221649169921875,
      "activations/layer6_attention_weight_max": 47.926734924316406,
      "activations/layer6_attention_weight_min": -52.9837532043457,
      "activations/layer7_attention_weight_max": 84.67462158203125,
      "activations/layer7_attention_weight_min": -91.14895629882812,
      "activations/layer8_attention_weight_max": 37.776268005371094,
      "activations/layer8_attention_weight_min": -39.5443115234375,
      "activations/layer9_attention_weight_max": 32.0934944152832,
      "activations/layer9_attention_weight_min": -32.75297927856445,
      "epoch": 8.2,
      "learning_rate": 9.809621212121212e-05,
      "loss": 2.7815,
      "step": 141150
    },
    {
      "activations/layer0_attention_weight_max": 15.745409965515137,
      "activations/layer0_attention_weight_min": -12.038646697998047,
      "activations/layer10_attention_weight_max": 35.17626953125,
      "activations/layer10_attention_weight_min": -32.5737190246582,
      "activations/layer11_attention_weight_max": 33.43095016479492,
      "activations/layer11_attention_weight_min": -29.664321899414062,
      "activations/layer12_attention_weight_max": 35.59659957885742,
      "activations/layer12_attention_weight_min": -29.14141273498535,
      "activations/layer13_attention_weight_max": 39.86574935913086,
      "activations/layer13_attention_weight_min": -34.039390563964844,
      "activations/layer14_attention_weight_max": 51.63786315917969,
      "activations/layer14_attention_weight_min": -41.60573196411133,
      "activations/layer15_attention_weight_max": 36.64084243774414,
      "activations/layer15_attention_weight_min": -28.995113372802734,
      "activations/layer16_attention_weight_max": 33.74663162231445,
      "activations/layer16_attention_weight_min": -26.711383819580078,
      "activations/layer17_attention_weight_max": 33.222747802734375,
      "activations/layer17_attention_weight_min": -25.121530532836914,
      "activations/layer18_attention_weight_max": 27.568344116210938,
      "activations/layer18_attention_weight_min": -20.57678985595703,
      "activations/layer19_attention_weight_max": 34.649200439453125,
      "activations/layer19_attention_weight_min": -32.22848129272461,
      "activations/layer1_attention_weight_max": 16.120710372924805,
      "activations/layer1_attention_weight_min": -13.561966896057129,
      "activations/layer20_attention_weight_max": 28.561779022216797,
      "activations/layer20_attention_weight_min": -24.866966247558594,
      "activations/layer21_attention_weight_max": 28.88693618774414,
      "activations/layer21_attention_weight_min": -22.722286224365234,
      "activations/layer22_attention_weight_max": 47.2137336730957,
      "activations/layer22_attention_weight_min": -32.03883361816406,
      "activations/layer23_attention_weight_max": 35.66250228881836,
      "activations/layer23_attention_weight_min": -23.930105209350586,
      "activations/layer2_attention_weight_max": 30.80489730834961,
      "activations/layer2_attention_weight_min": -29.82696533203125,
      "activations/layer3_attention_weight_max": 89.8116455078125,
      "activations/layer3_attention_weight_min": -90.89772033691406,
      "activations/layer4_attention_weight_max": 119.19188690185547,
      "activations/layer4_attention_weight_min": -115.13201904296875,
      "activations/layer5_attention_weight_max": 59.60930633544922,
      "activations/layer5_attention_weight_min": -63.9791374206543,
      "activations/layer6_attention_weight_max": 48.7835807800293,
      "activations/layer6_attention_weight_min": -51.49827194213867,
      "activations/layer7_attention_weight_max": 95.3553695678711,
      "activations/layer7_attention_weight_min": -90.45601654052734,
      "activations/layer8_attention_weight_max": 42.17120361328125,
      "activations/layer8_attention_weight_min": -39.26578903198242,
      "activations/layer9_attention_weight_max": 35.055686950683594,
      "activations/layer9_attention_weight_min": -32.49509811401367,
      "epoch": 8.2,
      "learning_rate": 9.807727272727272e-05,
      "loss": 2.802,
      "step": 141200
    },
    {
      "activations/layer0_attention_weight_max": 16.136009216308594,
      "activations/layer0_attention_weight_min": -12.054847717285156,
      "activations/layer10_attention_weight_max": 31.84654426574707,
      "activations/layer10_attention_weight_min": -30.50870704650879,
      "activations/layer11_attention_weight_max": 30.829038619995117,
      "activations/layer11_attention_weight_min": -28.980466842651367,
      "activations/layer12_attention_weight_max": 23.365819931030273,
      "activations/layer12_attention_weight_min": -25.767610549926758,
      "activations/layer13_attention_weight_max": 46.35270309448242,
      "activations/layer13_attention_weight_min": -32.26628112792969,
      "activations/layer14_attention_weight_max": 51.21561813354492,
      "activations/layer14_attention_weight_min": -40.178348541259766,
      "activations/layer15_attention_weight_max": 37.71011734008789,
      "activations/layer15_attention_weight_min": -30.315065383911133,
      "activations/layer16_attention_weight_max": 34.81275177001953,
      "activations/layer16_attention_weight_min": -29.089223861694336,
      "activations/layer17_attention_weight_max": 33.887107849121094,
      "activations/layer17_attention_weight_min": -26.972721099853516,
      "activations/layer18_attention_weight_max": 29.608661651611328,
      "activations/layer18_attention_weight_min": -24.901687622070312,
      "activations/layer19_attention_weight_max": 37.639488220214844,
      "activations/layer19_attention_weight_min": -32.89857864379883,
      "activations/layer1_attention_weight_max": 17.149202346801758,
      "activations/layer1_attention_weight_min": -15.64389419555664,
      "activations/layer20_attention_weight_max": 30.955459594726562,
      "activations/layer20_attention_weight_min": -24.385374069213867,
      "activations/layer21_attention_weight_max": 30.581918716430664,
      "activations/layer21_attention_weight_min": -23.1922550201416,
      "activations/layer22_attention_weight_max": 51.77131652832031,
      "activations/layer22_attention_weight_min": -29.195480346679688,
      "activations/layer23_attention_weight_max": 35.33690643310547,
      "activations/layer23_attention_weight_min": -24.28573989868164,
      "activations/layer2_attention_weight_max": 33.32701110839844,
      "activations/layer2_attention_weight_min": -33.59394073486328,
      "activations/layer3_attention_weight_max": 99.62812042236328,
      "activations/layer3_attention_weight_min": -100.56597900390625,
      "activations/layer4_attention_weight_max": 127.7291030883789,
      "activations/layer4_attention_weight_min": -132.17752075195312,
      "activations/layer5_attention_weight_max": 60.46255111694336,
      "activations/layer5_attention_weight_min": -64.13878631591797,
      "activations/layer6_attention_weight_max": 50.18015670776367,
      "activations/layer6_attention_weight_min": -54.730106353759766,
      "activations/layer7_attention_weight_max": 90.31521606445312,
      "activations/layer7_attention_weight_min": -86.8783187866211,
      "activations/layer8_attention_weight_max": 39.296329498291016,
      "activations/layer8_attention_weight_min": -41.150474548339844,
      "activations/layer9_attention_weight_max": 33.1700325012207,
      "activations/layer9_attention_weight_min": -33.609779357910156,
      "epoch": 8.21,
      "learning_rate": 9.805833333333332e-05,
      "loss": 2.7959,
      "step": 141250
    },
    {
      "activations/layer0_attention_weight_max": 16.149520874023438,
      "activations/layer0_attention_weight_min": -12.572381973266602,
      "activations/layer10_attention_weight_max": 34.86993408203125,
      "activations/layer10_attention_weight_min": -32.193626403808594,
      "activations/layer11_attention_weight_max": 32.80054473876953,
      "activations/layer11_attention_weight_min": -31.55632781982422,
      "activations/layer12_attention_weight_max": 24.099878311157227,
      "activations/layer12_attention_weight_min": -23.727354049682617,
      "activations/layer13_attention_weight_max": 43.97336959838867,
      "activations/layer13_attention_weight_min": -32.51849365234375,
      "activations/layer14_attention_weight_max": 46.961585998535156,
      "activations/layer14_attention_weight_min": -40.98184585571289,
      "activations/layer15_attention_weight_max": 37.106834411621094,
      "activations/layer15_attention_weight_min": -29.901641845703125,
      "activations/layer16_attention_weight_max": 33.81463623046875,
      "activations/layer16_attention_weight_min": -27.357913970947266,
      "activations/layer17_attention_weight_max": 34.950538635253906,
      "activations/layer17_attention_weight_min": -24.99721908569336,
      "activations/layer18_attention_weight_max": 38.927188873291016,
      "activations/layer18_attention_weight_min": -21.917051315307617,
      "activations/layer19_attention_weight_max": 33.459617614746094,
      "activations/layer19_attention_weight_min": -29.064655303955078,
      "activations/layer1_attention_weight_max": 16.14997100830078,
      "activations/layer1_attention_weight_min": -14.917820930480957,
      "activations/layer20_attention_weight_max": 32.99687194824219,
      "activations/layer20_attention_weight_min": -23.260087966918945,
      "activations/layer21_attention_weight_max": 31.4741268157959,
      "activations/layer21_attention_weight_min": -20.58289337158203,
      "activations/layer22_attention_weight_max": 48.19004440307617,
      "activations/layer22_attention_weight_min": -28.398040771484375,
      "activations/layer23_attention_weight_max": 34.063385009765625,
      "activations/layer23_attention_weight_min": -22.988012313842773,
      "activations/layer2_attention_weight_max": 32.56104278564453,
      "activations/layer2_attention_weight_min": -31.066909790039062,
      "activations/layer3_attention_weight_max": 94.42931365966797,
      "activations/layer3_attention_weight_min": -96.7441635131836,
      "activations/layer4_attention_weight_max": 123.83280181884766,
      "activations/layer4_attention_weight_min": -118.2807846069336,
      "activations/layer5_attention_weight_max": 59.368408203125,
      "activations/layer5_attention_weight_min": -60.234107971191406,
      "activations/layer6_attention_weight_max": 50.44460678100586,
      "activations/layer6_attention_weight_min": -52.873538970947266,
      "activations/layer7_attention_weight_max": 96.87142181396484,
      "activations/layer7_attention_weight_min": -90.25923919677734,
      "activations/layer8_attention_weight_max": 40.538631439208984,
      "activations/layer8_attention_weight_min": -40.561500549316406,
      "activations/layer9_attention_weight_max": 35.26494598388672,
      "activations/layer9_attention_weight_min": -33.50101089477539,
      "epoch": 8.21,
      "learning_rate": 9.803939393939394e-05,
      "loss": 2.808,
      "step": 141300
    },
    {
      "activations/layer0_attention_weight_max": 16.464839935302734,
      "activations/layer0_attention_weight_min": -12.545735359191895,
      "activations/layer10_attention_weight_max": 33.65386199951172,
      "activations/layer10_attention_weight_min": -31.06707000732422,
      "activations/layer11_attention_weight_max": 33.377685546875,
      "activations/layer11_attention_weight_min": -29.238697052001953,
      "activations/layer12_attention_weight_max": 21.91655921936035,
      "activations/layer12_attention_weight_min": -25.262367248535156,
      "activations/layer13_attention_weight_max": 41.397003173828125,
      "activations/layer13_attention_weight_min": -31.545013427734375,
      "activations/layer14_attention_weight_max": 48.23038101196289,
      "activations/layer14_attention_weight_min": -39.007118225097656,
      "activations/layer15_attention_weight_max": 33.55619430541992,
      "activations/layer15_attention_weight_min": -27.758527755737305,
      "activations/layer16_attention_weight_max": 30.36726951599121,
      "activations/layer16_attention_weight_min": -26.346820831298828,
      "activations/layer17_attention_weight_max": 34.82980728149414,
      "activations/layer17_attention_weight_min": -24.074617385864258,
      "activations/layer18_attention_weight_max": 31.648759841918945,
      "activations/layer18_attention_weight_min": -23.14077377319336,
      "activations/layer19_attention_weight_max": 36.34040832519531,
      "activations/layer19_attention_weight_min": -28.366840362548828,
      "activations/layer1_attention_weight_max": 17.514427185058594,
      "activations/layer1_attention_weight_min": -14.999734878540039,
      "activations/layer20_attention_weight_max": 31.313480377197266,
      "activations/layer20_attention_weight_min": -24.765947341918945,
      "activations/layer21_attention_weight_max": 29.896671295166016,
      "activations/layer21_attention_weight_min": -21.68465232849121,
      "activations/layer22_attention_weight_max": 49.95430374145508,
      "activations/layer22_attention_weight_min": -32.50497817993164,
      "activations/layer23_attention_weight_max": 37.9703254699707,
      "activations/layer23_attention_weight_min": -26.089704513549805,
      "activations/layer2_attention_weight_max": 32.3433837890625,
      "activations/layer2_attention_weight_min": -33.43470764160156,
      "activations/layer3_attention_weight_max": 91.46277618408203,
      "activations/layer3_attention_weight_min": -92.21189880371094,
      "activations/layer4_attention_weight_max": 119.61791229248047,
      "activations/layer4_attention_weight_min": -114.01625061035156,
      "activations/layer5_attention_weight_max": 56.02433776855469,
      "activations/layer5_attention_weight_min": -61.974510192871094,
      "activations/layer6_attention_weight_max": 49.724571228027344,
      "activations/layer6_attention_weight_min": -49.56407928466797,
      "activations/layer7_attention_weight_max": 89.56905364990234,
      "activations/layer7_attention_weight_min": -83.63590240478516,
      "activations/layer8_attention_weight_max": 39.00249099731445,
      "activations/layer8_attention_weight_min": -39.97575759887695,
      "activations/layer9_attention_weight_max": 37.789146423339844,
      "activations/layer9_attention_weight_min": -33.903106689453125,
      "epoch": 8.21,
      "learning_rate": 9.802045454545453e-05,
      "loss": 2.7934,
      "step": 141350
    },
    {
      "activations/layer0_attention_weight_max": 16.079124450683594,
      "activations/layer0_attention_weight_min": -12.217768669128418,
      "activations/layer10_attention_weight_max": 30.771270751953125,
      "activations/layer10_attention_weight_min": -30.684452056884766,
      "activations/layer11_attention_weight_max": 32.07069396972656,
      "activations/layer11_attention_weight_min": -28.341611862182617,
      "activations/layer12_attention_weight_max": 30.936832427978516,
      "activations/layer12_attention_weight_min": -26.22785758972168,
      "activations/layer13_attention_weight_max": 36.067230224609375,
      "activations/layer13_attention_weight_min": -31.046974182128906,
      "activations/layer14_attention_weight_max": 46.23046875,
      "activations/layer14_attention_weight_min": -38.35630798339844,
      "activations/layer15_attention_weight_max": 34.49618911743164,
      "activations/layer15_attention_weight_min": -28.904823303222656,
      "activations/layer16_attention_weight_max": 34.88504409790039,
      "activations/layer16_attention_weight_min": -26.615434646606445,
      "activations/layer17_attention_weight_max": 37.929832458496094,
      "activations/layer17_attention_weight_min": -24.89934539794922,
      "activations/layer18_attention_weight_max": 30.650165557861328,
      "activations/layer18_attention_weight_min": -24.240726470947266,
      "activations/layer19_attention_weight_max": 34.83944320678711,
      "activations/layer19_attention_weight_min": -31.355655670166016,
      "activations/layer1_attention_weight_max": 16.322906494140625,
      "activations/layer1_attention_weight_min": -13.831404685974121,
      "activations/layer20_attention_weight_max": 29.893896102905273,
      "activations/layer20_attention_weight_min": -22.915014266967773,
      "activations/layer21_attention_weight_max": 29.726652145385742,
      "activations/layer21_attention_weight_min": -21.64089584350586,
      "activations/layer22_attention_weight_max": 42.289451599121094,
      "activations/layer22_attention_weight_min": -28.51931381225586,
      "activations/layer23_attention_weight_max": 36.83911895751953,
      "activations/layer23_attention_weight_min": -25.3898983001709,
      "activations/layer2_attention_weight_max": 30.355026245117188,
      "activations/layer2_attention_weight_min": -30.25908660888672,
      "activations/layer3_attention_weight_max": 92.50881958007812,
      "activations/layer3_attention_weight_min": -95.32275390625,
      "activations/layer4_attention_weight_max": 116.01155090332031,
      "activations/layer4_attention_weight_min": -116.5150146484375,
      "activations/layer5_attention_weight_max": 55.24493408203125,
      "activations/layer5_attention_weight_min": -58.95348358154297,
      "activations/layer6_attention_weight_max": 47.111270904541016,
      "activations/layer6_attention_weight_min": -49.553829193115234,
      "activations/layer7_attention_weight_max": 83.7206802368164,
      "activations/layer7_attention_weight_min": -85.1069107055664,
      "activations/layer8_attention_weight_max": 39.959232330322266,
      "activations/layer8_attention_weight_min": -40.309757232666016,
      "activations/layer9_attention_weight_max": 32.8880729675293,
      "activations/layer9_attention_weight_min": -31.462507247924805,
      "epoch": 8.22,
      "learning_rate": 9.800189393939394e-05,
      "loss": 2.7971,
      "step": 141400
    },
    {
      "activations/layer0_attention_weight_max": 14.890567779541016,
      "activations/layer0_attention_weight_min": -12.854453086853027,
      "activations/layer10_attention_weight_max": 36.49928665161133,
      "activations/layer10_attention_weight_min": -32.31005096435547,
      "activations/layer11_attention_weight_max": 34.140071868896484,
      "activations/layer11_attention_weight_min": -31.699087142944336,
      "activations/layer12_attention_weight_max": 25.80340003967285,
      "activations/layer12_attention_weight_min": -29.29009437561035,
      "activations/layer13_attention_weight_max": 44.71118927001953,
      "activations/layer13_attention_weight_min": -32.21421813964844,
      "activations/layer14_attention_weight_max": 60.79807662963867,
      "activations/layer14_attention_weight_min": -42.14425277709961,
      "activations/layer15_attention_weight_max": 40.02638244628906,
      "activations/layer15_attention_weight_min": -29.583229064941406,
      "activations/layer16_attention_weight_max": 34.582645416259766,
      "activations/layer16_attention_weight_min": -26.661935806274414,
      "activations/layer17_attention_weight_max": 36.18656921386719,
      "activations/layer17_attention_weight_min": -25.51858901977539,
      "activations/layer18_attention_weight_max": 32.661590576171875,
      "activations/layer18_attention_weight_min": -22.364849090576172,
      "activations/layer19_attention_weight_max": 37.64234924316406,
      "activations/layer19_attention_weight_min": -31.83551597595215,
      "activations/layer1_attention_weight_max": 17.24535369873047,
      "activations/layer1_attention_weight_min": -13.919565200805664,
      "activations/layer20_attention_weight_max": 30.909927368164062,
      "activations/layer20_attention_weight_min": -26.354347229003906,
      "activations/layer21_attention_weight_max": 29.207401275634766,
      "activations/layer21_attention_weight_min": -22.76573371887207,
      "activations/layer22_attention_weight_max": 53.0013313293457,
      "activations/layer22_attention_weight_min": -30.658641815185547,
      "activations/layer23_attention_weight_max": 38.59983825683594,
      "activations/layer23_attention_weight_min": -24.033483505249023,
      "activations/layer2_attention_weight_max": 32.55175018310547,
      "activations/layer2_attention_weight_min": -31.69471549987793,
      "activations/layer3_attention_weight_max": 93.310302734375,
      "activations/layer3_attention_weight_min": -95.42506408691406,
      "activations/layer4_attention_weight_max": 122.3032455444336,
      "activations/layer4_attention_weight_min": -118.98844146728516,
      "activations/layer5_attention_weight_max": 59.50360107421875,
      "activations/layer5_attention_weight_min": -61.094947814941406,
      "activations/layer6_attention_weight_max": 52.11212921142578,
      "activations/layer6_attention_weight_min": -53.44031524658203,
      "activations/layer7_attention_weight_max": 96.60095977783203,
      "activations/layer7_attention_weight_min": -89.33802795410156,
      "activations/layer8_attention_weight_max": 44.89445114135742,
      "activations/layer8_attention_weight_min": -42.09230422973633,
      "activations/layer9_attention_weight_max": 38.653724670410156,
      "activations/layer9_attention_weight_min": -34.78603744506836,
      "epoch": 8.22,
      "learning_rate": 9.798295454545453e-05,
      "loss": 2.8066,
      "step": 141450
    },
    {
      "activations/layer0_attention_weight_max": 15.015246391296387,
      "activations/layer0_attention_weight_min": -12.532100677490234,
      "activations/layer10_attention_weight_max": 36.856689453125,
      "activations/layer10_attention_weight_min": -36.436012268066406,
      "activations/layer11_attention_weight_max": 41.42835998535156,
      "activations/layer11_attention_weight_min": -36.592002868652344,
      "activations/layer12_attention_weight_max": 48.3564567565918,
      "activations/layer12_attention_weight_min": -27.667957305908203,
      "activations/layer13_attention_weight_max": 53.21491622924805,
      "activations/layer13_attention_weight_min": -36.920860290527344,
      "activations/layer14_attention_weight_max": 49.972068786621094,
      "activations/layer14_attention_weight_min": -45.522315979003906,
      "activations/layer15_attention_weight_max": 41.77920150756836,
      "activations/layer15_attention_weight_min": -40.26115417480469,
      "activations/layer16_attention_weight_max": 28.844179153442383,
      "activations/layer16_attention_weight_min": -28.13292694091797,
      "activations/layer17_attention_weight_max": 31.10648536682129,
      "activations/layer17_attention_weight_min": -25.380502700805664,
      "activations/layer18_attention_weight_max": 29.460060119628906,
      "activations/layer18_attention_weight_min": -21.198354721069336,
      "activations/layer19_attention_weight_max": 31.90816307067871,
      "activations/layer19_attention_weight_min": -29.36444091796875,
      "activations/layer1_attention_weight_max": 17.355098724365234,
      "activations/layer1_attention_weight_min": -15.311344146728516,
      "activations/layer20_attention_weight_max": 29.361421585083008,
      "activations/layer20_attention_weight_min": -25.433149337768555,
      "activations/layer21_attention_weight_max": 29.31442642211914,
      "activations/layer21_attention_weight_min": -24.235837936401367,
      "activations/layer22_attention_weight_max": 45.007164001464844,
      "activations/layer22_attention_weight_min": -28.98585319519043,
      "activations/layer23_attention_weight_max": 31.798879623413086,
      "activations/layer23_attention_weight_min": -25.34575653076172,
      "activations/layer2_attention_weight_max": 34.62554168701172,
      "activations/layer2_attention_weight_min": -33.41503143310547,
      "activations/layer3_attention_weight_max": 99.54071044921875,
      "activations/layer3_attention_weight_min": -100.68576049804688,
      "activations/layer4_attention_weight_max": 120.1225814819336,
      "activations/layer4_attention_weight_min": -118.7479019165039,
      "activations/layer5_attention_weight_max": 56.707847595214844,
      "activations/layer5_attention_weight_min": -60.23830795288086,
      "activations/layer6_attention_weight_max": 49.671119689941406,
      "activations/layer6_attention_weight_min": -49.887393951416016,
      "activations/layer7_attention_weight_max": 99.31431579589844,
      "activations/layer7_attention_weight_min": -90.67547607421875,
      "activations/layer8_attention_weight_max": 45.17912292480469,
      "activations/layer8_attention_weight_min": -43.7115592956543,
      "activations/layer9_attention_weight_max": 37.46442413330078,
      "activations/layer9_attention_weight_min": -36.40809631347656,
      "epoch": 8.22,
      "learning_rate": 9.796401515151515e-05,
      "loss": 2.7977,
      "step": 141500
    },
    {
      "activations/layer0_attention_weight_max": 15.739482879638672,
      "activations/layer0_attention_weight_min": -11.863238334655762,
      "activations/layer10_attention_weight_max": 30.757890701293945,
      "activations/layer10_attention_weight_min": -32.07994842529297,
      "activations/layer11_attention_weight_max": 31.609121322631836,
      "activations/layer11_attention_weight_min": -30.507862091064453,
      "activations/layer12_attention_weight_max": 31.80541229248047,
      "activations/layer12_attention_weight_min": -29.921653747558594,
      "activations/layer13_attention_weight_max": 43.86778259277344,
      "activations/layer13_attention_weight_min": -33.52729415893555,
      "activations/layer14_attention_weight_max": 53.540218353271484,
      "activations/layer14_attention_weight_min": -41.700225830078125,
      "activations/layer15_attention_weight_max": 43.93507385253906,
      "activations/layer15_attention_weight_min": -30.23958969116211,
      "activations/layer16_attention_weight_max": 35.274837493896484,
      "activations/layer16_attention_weight_min": -27.99767303466797,
      "activations/layer17_attention_weight_max": 34.653507232666016,
      "activations/layer17_attention_weight_min": -25.13092803955078,
      "activations/layer18_attention_weight_max": 35.82544708251953,
      "activations/layer18_attention_weight_min": -21.153047561645508,
      "activations/layer19_attention_weight_max": 40.65205001831055,
      "activations/layer19_attention_weight_min": -31.39593505859375,
      "activations/layer1_attention_weight_max": 16.376996994018555,
      "activations/layer1_attention_weight_min": -14.484273910522461,
      "activations/layer20_attention_weight_max": 33.65181350708008,
      "activations/layer20_attention_weight_min": -25.183488845825195,
      "activations/layer21_attention_weight_max": 31.96770477294922,
      "activations/layer21_attention_weight_min": -22.929729461669922,
      "activations/layer22_attention_weight_max": 44.003170013427734,
      "activations/layer22_attention_weight_min": -29.350337982177734,
      "activations/layer23_attention_weight_max": 34.84066390991211,
      "activations/layer23_attention_weight_min": -23.90120506286621,
      "activations/layer2_attention_weight_max": 32.38970184326172,
      "activations/layer2_attention_weight_min": -33.23353576660156,
      "activations/layer3_attention_weight_max": 91.32655334472656,
      "activations/layer3_attention_weight_min": -93.6351318359375,
      "activations/layer4_attention_weight_max": 123.8601303100586,
      "activations/layer4_attention_weight_min": -117.0016098022461,
      "activations/layer5_attention_weight_max": 56.589447021484375,
      "activations/layer5_attention_weight_min": -62.734405517578125,
      "activations/layer6_attention_weight_max": 49.36555099487305,
      "activations/layer6_attention_weight_min": -52.0182991027832,
      "activations/layer7_attention_weight_max": 91.4061508178711,
      "activations/layer7_attention_weight_min": -86.8935546875,
      "activations/layer8_attention_weight_max": 39.33956527709961,
      "activations/layer8_attention_weight_min": -39.33904266357422,
      "activations/layer9_attention_weight_max": 33.2452278137207,
      "activations/layer9_attention_weight_min": -34.994178771972656,
      "epoch": 8.22,
      "learning_rate": 9.794507575757575e-05,
      "loss": 2.8127,
      "step": 141550
    },
    {
      "activations/layer0_attention_weight_max": 16.45656967163086,
      "activations/layer0_attention_weight_min": -12.10996150970459,
      "activations/layer10_attention_weight_max": 32.979522705078125,
      "activations/layer10_attention_weight_min": -30.743125915527344,
      "activations/layer11_attention_weight_max": 32.47686767578125,
      "activations/layer11_attention_weight_min": -29.478004455566406,
      "activations/layer12_attention_weight_max": 21.21307373046875,
      "activations/layer12_attention_weight_min": -29.5347843170166,
      "activations/layer13_attention_weight_max": 36.39939498901367,
      "activations/layer13_attention_weight_min": -30.84328842163086,
      "activations/layer14_attention_weight_max": 44.48977279663086,
      "activations/layer14_attention_weight_min": -36.101898193359375,
      "activations/layer15_attention_weight_max": 34.08938980102539,
      "activations/layer15_attention_weight_min": -29.395416259765625,
      "activations/layer16_attention_weight_max": 28.83863639831543,
      "activations/layer16_attention_weight_min": -27.87727165222168,
      "activations/layer17_attention_weight_max": 31.518524169921875,
      "activations/layer17_attention_weight_min": -25.062232971191406,
      "activations/layer18_attention_weight_max": 31.83421516418457,
      "activations/layer18_attention_weight_min": -20.63457679748535,
      "activations/layer19_attention_weight_max": 33.69460678100586,
      "activations/layer19_attention_weight_min": -29.904449462890625,
      "activations/layer1_attention_weight_max": 16.181806564331055,
      "activations/layer1_attention_weight_min": -15.441119194030762,
      "activations/layer20_attention_weight_max": 30.372325897216797,
      "activations/layer20_attention_weight_min": -24.1199893951416,
      "activations/layer21_attention_weight_max": 31.395477294921875,
      "activations/layer21_attention_weight_min": -23.039152145385742,
      "activations/layer22_attention_weight_max": 44.18601608276367,
      "activations/layer22_attention_weight_min": -30.128923416137695,
      "activations/layer23_attention_weight_max": 34.93378829956055,
      "activations/layer23_attention_weight_min": -25.332630157470703,
      "activations/layer2_attention_weight_max": 33.370849609375,
      "activations/layer2_attention_weight_min": -31.692487716674805,
      "activations/layer3_attention_weight_max": 92.88670349121094,
      "activations/layer3_attention_weight_min": -92.38796997070312,
      "activations/layer4_attention_weight_max": 121.01253509521484,
      "activations/layer4_attention_weight_min": -118.0811767578125,
      "activations/layer5_attention_weight_max": 58.769935607910156,
      "activations/layer5_attention_weight_min": -62.5376091003418,
      "activations/layer6_attention_weight_max": 49.45384979248047,
      "activations/layer6_attention_weight_min": -52.84422302246094,
      "activations/layer7_attention_weight_max": 86.20027923583984,
      "activations/layer7_attention_weight_min": -84.78678894042969,
      "activations/layer8_attention_weight_max": 41.3867073059082,
      "activations/layer8_attention_weight_min": -39.410675048828125,
      "activations/layer9_attention_weight_max": 34.92333984375,
      "activations/layer9_attention_weight_min": -33.41262435913086,
      "epoch": 8.23,
      "learning_rate": 9.792613636363635e-05,
      "loss": 2.8037,
      "step": 141600
    },
    {
      "activations/layer0_attention_weight_max": 16.23037338256836,
      "activations/layer0_attention_weight_min": -11.92424201965332,
      "activations/layer10_attention_weight_max": 35.12424850463867,
      "activations/layer10_attention_weight_min": -32.58066177368164,
      "activations/layer11_attention_weight_max": 35.549224853515625,
      "activations/layer11_attention_weight_min": -31.52132797241211,
      "activations/layer12_attention_weight_max": 27.116418838500977,
      "activations/layer12_attention_weight_min": -24.780975341796875,
      "activations/layer13_attention_weight_max": 35.983158111572266,
      "activations/layer13_attention_weight_min": -36.40667724609375,
      "activations/layer14_attention_weight_max": 44.294036865234375,
      "activations/layer14_attention_weight_min": -39.16841506958008,
      "activations/layer15_attention_weight_max": 31.863147735595703,
      "activations/layer15_attention_weight_min": -31.235958099365234,
      "activations/layer16_attention_weight_max": 27.869342803955078,
      "activations/layer16_attention_weight_min": -27.369903564453125,
      "activations/layer17_attention_weight_max": 30.858036041259766,
      "activations/layer17_attention_weight_min": -24.514406204223633,
      "activations/layer18_attention_weight_max": 29.33587074279785,
      "activations/layer18_attention_weight_min": -21.12839126586914,
      "activations/layer19_attention_weight_max": 34.014564514160156,
      "activations/layer19_attention_weight_min": -29.555566787719727,
      "activations/layer1_attention_weight_max": 16.53850746154785,
      "activations/layer1_attention_weight_min": -13.801441192626953,
      "activations/layer20_attention_weight_max": 29.106210708618164,
      "activations/layer20_attention_weight_min": -24.93243408203125,
      "activations/layer21_attention_weight_max": 26.65672492980957,
      "activations/layer21_attention_weight_min": -21.733570098876953,
      "activations/layer22_attention_weight_max": 45.616493225097656,
      "activations/layer22_attention_weight_min": -29.993684768676758,
      "activations/layer23_attention_weight_max": 32.54098129272461,
      "activations/layer23_attention_weight_min": -23.91347885131836,
      "activations/layer2_attention_weight_max": 33.22492980957031,
      "activations/layer2_attention_weight_min": -31.76984214782715,
      "activations/layer3_attention_weight_max": 94.58204650878906,
      "activations/layer3_attention_weight_min": -99.96170806884766,
      "activations/layer4_attention_weight_max": 127.31086730957031,
      "activations/layer4_attention_weight_min": -122.06014251708984,
      "activations/layer5_attention_weight_max": 61.125732421875,
      "activations/layer5_attention_weight_min": -64.39407348632812,
      "activations/layer6_attention_weight_max": 51.5090217590332,
      "activations/layer6_attention_weight_min": -52.66248321533203,
      "activations/layer7_attention_weight_max": 91.74093627929688,
      "activations/layer7_attention_weight_min": -86.59361267089844,
      "activations/layer8_attention_weight_max": 41.69860076904297,
      "activations/layer8_attention_weight_min": -43.71337127685547,
      "activations/layer9_attention_weight_max": 37.68952941894531,
      "activations/layer9_attention_weight_min": -34.47895050048828,
      "epoch": 8.23,
      "learning_rate": 9.790719696969696e-05,
      "loss": 2.8142,
      "step": 141650
    },
    {
      "activations/layer0_attention_weight_max": 15.553189277648926,
      "activations/layer0_attention_weight_min": -12.460329055786133,
      "activations/layer10_attention_weight_max": 32.803184509277344,
      "activations/layer10_attention_weight_min": -31.192094802856445,
      "activations/layer11_attention_weight_max": 32.105247497558594,
      "activations/layer11_attention_weight_min": -31.167205810546875,
      "activations/layer12_attention_weight_max": 22.246152877807617,
      "activations/layer12_attention_weight_min": -23.80529022216797,
      "activations/layer13_attention_weight_max": 34.411651611328125,
      "activations/layer13_attention_weight_min": -32.431884765625,
      "activations/layer14_attention_weight_max": 56.519866943359375,
      "activations/layer14_attention_weight_min": -41.46531677246094,
      "activations/layer15_attention_weight_max": 33.26268768310547,
      "activations/layer15_attention_weight_min": -33.69502258300781,
      "activations/layer16_attention_weight_max": 32.338436126708984,
      "activations/layer16_attention_weight_min": -29.233577728271484,
      "activations/layer17_attention_weight_max": 37.23870086669922,
      "activations/layer17_attention_weight_min": -27.831676483154297,
      "activations/layer18_attention_weight_max": 31.38267707824707,
      "activations/layer18_attention_weight_min": -21.56475257873535,
      "activations/layer19_attention_weight_max": 37.301063537597656,
      "activations/layer19_attention_weight_min": -32.6706657409668,
      "activations/layer1_attention_weight_max": 17.62550926208496,
      "activations/layer1_attention_weight_min": -15.168550491333008,
      "activations/layer20_attention_weight_max": 35.06792449951172,
      "activations/layer20_attention_weight_min": -26.9904727935791,
      "activations/layer21_attention_weight_max": 38.48038864135742,
      "activations/layer21_attention_weight_min": -24.85516357421875,
      "activations/layer22_attention_weight_max": 47.93826675415039,
      "activations/layer22_attention_weight_min": -29.69286346435547,
      "activations/layer23_attention_weight_max": 32.95159149169922,
      "activations/layer23_attention_weight_min": -23.684398651123047,
      "activations/layer2_attention_weight_max": 33.60243225097656,
      "activations/layer2_attention_weight_min": -34.798927307128906,
      "activations/layer3_attention_weight_max": 94.8298568725586,
      "activations/layer3_attention_weight_min": -102.05329132080078,
      "activations/layer4_attention_weight_max": 123.99564361572266,
      "activations/layer4_attention_weight_min": -124.62054443359375,
      "activations/layer5_attention_weight_max": 59.39716720581055,
      "activations/layer5_attention_weight_min": -63.56208801269531,
      "activations/layer6_attention_weight_max": 51.358848571777344,
      "activations/layer6_attention_weight_min": -51.57854080200195,
      "activations/layer7_attention_weight_max": 91.37013244628906,
      "activations/layer7_attention_weight_min": -84.48380279541016,
      "activations/layer8_attention_weight_max": 42.18258285522461,
      "activations/layer8_attention_weight_min": -38.57686233520508,
      "activations/layer9_attention_weight_max": 34.8260383605957,
      "activations/layer9_attention_weight_min": -34.01137924194336,
      "epoch": 8.23,
      "learning_rate": 9.788825757575757e-05,
      "loss": 2.8,
      "step": 141700
    },
    {
      "activations/layer0_attention_weight_max": 14.778190612792969,
      "activations/layer0_attention_weight_min": -12.073369979858398,
      "activations/layer10_attention_weight_max": 33.66292190551758,
      "activations/layer10_attention_weight_min": -29.536407470703125,
      "activations/layer11_attention_weight_max": 34.9700927734375,
      "activations/layer11_attention_weight_min": -29.013439178466797,
      "activations/layer12_attention_weight_max": 24.82351303100586,
      "activations/layer12_attention_weight_min": -29.183691024780273,
      "activations/layer13_attention_weight_max": 37.078365325927734,
      "activations/layer13_attention_weight_min": -32.68608093261719,
      "activations/layer14_attention_weight_max": 43.94477462768555,
      "activations/layer14_attention_weight_min": -41.9559326171875,
      "activations/layer15_attention_weight_max": 40.734317779541016,
      "activations/layer15_attention_weight_min": -29.313230514526367,
      "activations/layer16_attention_weight_max": 31.7330322265625,
      "activations/layer16_attention_weight_min": -26.57660484313965,
      "activations/layer17_attention_weight_max": 32.05182647705078,
      "activations/layer17_attention_weight_min": -24.147254943847656,
      "activations/layer18_attention_weight_max": 30.481369018554688,
      "activations/layer18_attention_weight_min": -19.855159759521484,
      "activations/layer19_attention_weight_max": 31.34397315979004,
      "activations/layer19_attention_weight_min": -29.462255477905273,
      "activations/layer1_attention_weight_max": 17.61842918395996,
      "activations/layer1_attention_weight_min": -14.550249099731445,
      "activations/layer20_attention_weight_max": 24.458311080932617,
      "activations/layer20_attention_weight_min": -24.309106826782227,
      "activations/layer21_attention_weight_max": 27.172990798950195,
      "activations/layer21_attention_weight_min": -24.169248580932617,
      "activations/layer22_attention_weight_max": 44.00118637084961,
      "activations/layer22_attention_weight_min": -29.192298889160156,
      "activations/layer23_attention_weight_max": 32.99811553955078,
      "activations/layer23_attention_weight_min": -22.260730743408203,
      "activations/layer2_attention_weight_max": 36.25335693359375,
      "activations/layer2_attention_weight_min": -36.30497360229492,
      "activations/layer3_attention_weight_max": 100.24226379394531,
      "activations/layer3_attention_weight_min": -101.08610534667969,
      "activations/layer4_attention_weight_max": 126.12786865234375,
      "activations/layer4_attention_weight_min": -120.87191009521484,
      "activations/layer5_attention_weight_max": 62.000083923339844,
      "activations/layer5_attention_weight_min": -66.48574829101562,
      "activations/layer6_attention_weight_max": 50.03835678100586,
      "activations/layer6_attention_weight_min": -52.51262664794922,
      "activations/layer7_attention_weight_max": 91.80966186523438,
      "activations/layer7_attention_weight_min": -90.68482208251953,
      "activations/layer8_attention_weight_max": 39.7404670715332,
      "activations/layer8_attention_weight_min": -40.11048889160156,
      "activations/layer9_attention_weight_max": 33.20013427734375,
      "activations/layer9_attention_weight_min": -33.56499099731445,
      "epoch": 8.24,
      "learning_rate": 9.786931818181817e-05,
      "loss": 2.8158,
      "step": 141750
    },
    {
      "activations/layer0_attention_weight_max": 16.36482810974121,
      "activations/layer0_attention_weight_min": -12.115219116210938,
      "activations/layer10_attention_weight_max": 39.84314727783203,
      "activations/layer10_attention_weight_min": -34.0855598449707,
      "activations/layer11_attention_weight_max": 45.37369918823242,
      "activations/layer11_attention_weight_min": -37.271690368652344,
      "activations/layer12_attention_weight_max": 52.0851936340332,
      "activations/layer12_attention_weight_min": -25.333513259887695,
      "activations/layer13_attention_weight_max": 60.33207321166992,
      "activations/layer13_attention_weight_min": -36.55937194824219,
      "activations/layer14_attention_weight_max": 65.4927749633789,
      "activations/layer14_attention_weight_min": -43.669158935546875,
      "activations/layer15_attention_weight_max": 50.10256576538086,
      "activations/layer15_attention_weight_min": -32.963558197021484,
      "activations/layer16_attention_weight_max": 38.4072151184082,
      "activations/layer16_attention_weight_min": -27.785873413085938,
      "activations/layer17_attention_weight_max": 48.97431945800781,
      "activations/layer17_attention_weight_min": -25.52231788635254,
      "activations/layer18_attention_weight_max": 40.91742706298828,
      "activations/layer18_attention_weight_min": -24.392959594726562,
      "activations/layer19_attention_weight_max": 42.75513458251953,
      "activations/layer19_attention_weight_min": -31.432281494140625,
      "activations/layer1_attention_weight_max": 17.810754776000977,
      "activations/layer1_attention_weight_min": -17.623533248901367,
      "activations/layer20_attention_weight_max": 39.59867858886719,
      "activations/layer20_attention_weight_min": -25.870399475097656,
      "activations/layer21_attention_weight_max": 34.47429275512695,
      "activations/layer21_attention_weight_min": -23.59378433227539,
      "activations/layer22_attention_weight_max": 59.38909149169922,
      "activations/layer22_attention_weight_min": -29.53266143798828,
      "activations/layer23_attention_weight_max": 49.68315124511719,
      "activations/layer23_attention_weight_min": -23.915428161621094,
      "activations/layer2_attention_weight_max": 32.22267150878906,
      "activations/layer2_attention_weight_min": -32.5583610534668,
      "activations/layer3_attention_weight_max": 95.04215240478516,
      "activations/layer3_attention_weight_min": -92.82267761230469,
      "activations/layer4_attention_weight_max": 125.11846923828125,
      "activations/layer4_attention_weight_min": -117.9500732421875,
      "activations/layer5_attention_weight_max": 59.34258270263672,
      "activations/layer5_attention_weight_min": -59.91683578491211,
      "activations/layer6_attention_weight_max": 52.99039077758789,
      "activations/layer6_attention_weight_min": -51.150516510009766,
      "activations/layer7_attention_weight_max": 94.01557159423828,
      "activations/layer7_attention_weight_min": -92.90975189208984,
      "activations/layer8_attention_weight_max": 46.83255386352539,
      "activations/layer8_attention_weight_min": -43.347835540771484,
      "activations/layer9_attention_weight_max": 42.41474914550781,
      "activations/layer9_attention_weight_min": -38.620880126953125,
      "epoch": 8.24,
      "learning_rate": 9.785037878787878e-05,
      "loss": 2.797,
      "step": 141800
    },
    {
      "activations/layer0_attention_weight_max": 15.974242210388184,
      "activations/layer0_attention_weight_min": -10.840550422668457,
      "activations/layer10_attention_weight_max": 35.82455062866211,
      "activations/layer10_attention_weight_min": -32.679779052734375,
      "activations/layer11_attention_weight_max": 36.29315185546875,
      "activations/layer11_attention_weight_min": -33.471839904785156,
      "activations/layer12_attention_weight_max": 32.98950958251953,
      "activations/layer12_attention_weight_min": -28.684444427490234,
      "activations/layer13_attention_weight_max": 49.42118453979492,
      "activations/layer13_attention_weight_min": -35.353843688964844,
      "activations/layer14_attention_weight_max": 48.81039810180664,
      "activations/layer14_attention_weight_min": -40.49527359008789,
      "activations/layer15_attention_weight_max": 46.02413558959961,
      "activations/layer15_attention_weight_min": -29.84501838684082,
      "activations/layer16_attention_weight_max": 31.990978240966797,
      "activations/layer16_attention_weight_min": -29.238866806030273,
      "activations/layer17_attention_weight_max": 36.22012710571289,
      "activations/layer17_attention_weight_min": -25.210805892944336,
      "activations/layer18_attention_weight_max": 32.55720901489258,
      "activations/layer18_attention_weight_min": -22.941320419311523,
      "activations/layer19_attention_weight_max": 32.435157775878906,
      "activations/layer19_attention_weight_min": -30.122007369995117,
      "activations/layer1_attention_weight_max": 17.917482376098633,
      "activations/layer1_attention_weight_min": -14.7441987991333,
      "activations/layer20_attention_weight_max": 27.845840454101562,
      "activations/layer20_attention_weight_min": -26.198719024658203,
      "activations/layer21_attention_weight_max": 30.34621238708496,
      "activations/layer21_attention_weight_min": -23.607126235961914,
      "activations/layer22_attention_weight_max": 46.60334014892578,
      "activations/layer22_attention_weight_min": -28.85248565673828,
      "activations/layer23_attention_weight_max": 35.669273376464844,
      "activations/layer23_attention_weight_min": -24.99011993408203,
      "activations/layer2_attention_weight_max": 31.056812286376953,
      "activations/layer2_attention_weight_min": -30.526355743408203,
      "activations/layer3_attention_weight_max": 90.22347259521484,
      "activations/layer3_attention_weight_min": -91.74125671386719,
      "activations/layer4_attention_weight_max": 120.0134506225586,
      "activations/layer4_attention_weight_min": -120.744140625,
      "activations/layer5_attention_weight_max": 58.63227081298828,
      "activations/layer5_attention_weight_min": -61.01044845581055,
      "activations/layer6_attention_weight_max": 49.02771759033203,
      "activations/layer6_attention_weight_min": -54.69988250732422,
      "activations/layer7_attention_weight_max": 93.51750946044922,
      "activations/layer7_attention_weight_min": -93.56297302246094,
      "activations/layer8_attention_weight_max": 41.60740661621094,
      "activations/layer8_attention_weight_min": -40.53042221069336,
      "activations/layer9_attention_weight_max": 38.15189743041992,
      "activations/layer9_attention_weight_min": -34.19062805175781,
      "epoch": 8.24,
      "learning_rate": 9.783143939393938e-05,
      "loss": 2.7951,
      "step": 141850
    },
    {
      "activations/layer0_attention_weight_max": 16.654394149780273,
      "activations/layer0_attention_weight_min": -11.969466209411621,
      "activations/layer10_attention_weight_max": 31.898069381713867,
      "activations/layer10_attention_weight_min": -31.767065048217773,
      "activations/layer11_attention_weight_max": 32.79990768432617,
      "activations/layer11_attention_weight_min": -29.82184600830078,
      "activations/layer12_attention_weight_max": 29.810535430908203,
      "activations/layer12_attention_weight_min": -23.90766143798828,
      "activations/layer13_attention_weight_max": 38.324790954589844,
      "activations/layer13_attention_weight_min": -36.440921783447266,
      "activations/layer14_attention_weight_max": 50.44455337524414,
      "activations/layer14_attention_weight_min": -40.75479507446289,
      "activations/layer15_attention_weight_max": 38.59528732299805,
      "activations/layer15_attention_weight_min": -32.109596252441406,
      "activations/layer16_attention_weight_max": 35.59519958496094,
      "activations/layer16_attention_weight_min": -28.415836334228516,
      "activations/layer17_attention_weight_max": 37.53769302368164,
      "activations/layer17_attention_weight_min": -25.717927932739258,
      "activations/layer18_attention_weight_max": 32.4912109375,
      "activations/layer18_attention_weight_min": -22.346025466918945,
      "activations/layer19_attention_weight_max": 34.38182067871094,
      "activations/layer19_attention_weight_min": -28.1427059173584,
      "activations/layer1_attention_weight_max": 16.558713912963867,
      "activations/layer1_attention_weight_min": -13.795958518981934,
      "activations/layer20_attention_weight_max": 30.71727752685547,
      "activations/layer20_attention_weight_min": -23.484651565551758,
      "activations/layer21_attention_weight_max": 32.96796417236328,
      "activations/layer21_attention_weight_min": -21.72034454345703,
      "activations/layer22_attention_weight_max": 45.814517974853516,
      "activations/layer22_attention_weight_min": -28.245426177978516,
      "activations/layer23_attention_weight_max": 33.44817352294922,
      "activations/layer23_attention_weight_min": -21.47687339782715,
      "activations/layer2_attention_weight_max": 32.47464370727539,
      "activations/layer2_attention_weight_min": -33.24523162841797,
      "activations/layer3_attention_weight_max": 97.27369689941406,
      "activations/layer3_attention_weight_min": -98.07613372802734,
      "activations/layer4_attention_weight_max": 119.3267822265625,
      "activations/layer4_attention_weight_min": -127.56013488769531,
      "activations/layer5_attention_weight_max": 57.92809295654297,
      "activations/layer5_attention_weight_min": -62.76017379760742,
      "activations/layer6_attention_weight_max": 48.44562530517578,
      "activations/layer6_attention_weight_min": -53.449974060058594,
      "activations/layer7_attention_weight_max": 89.94917297363281,
      "activations/layer7_attention_weight_min": -90.623291015625,
      "activations/layer8_attention_weight_max": 39.42143249511719,
      "activations/layer8_attention_weight_min": -43.16569900512695,
      "activations/layer9_attention_weight_max": 34.795413970947266,
      "activations/layer9_attention_weight_min": -33.74765396118164,
      "epoch": 8.25,
      "learning_rate": 9.78125e-05,
      "loss": 2.8201,
      "step": 141900
    },
    {
      "activations/layer0_attention_weight_max": 15.313918113708496,
      "activations/layer0_attention_weight_min": -11.84380054473877,
      "activations/layer10_attention_weight_max": 34.678165435791016,
      "activations/layer10_attention_weight_min": -31.012779235839844,
      "activations/layer11_attention_weight_max": 35.09137725830078,
      "activations/layer11_attention_weight_min": -29.216949462890625,
      "activations/layer12_attention_weight_max": 22.87104606628418,
      "activations/layer12_attention_weight_min": -30.351097106933594,
      "activations/layer13_attention_weight_max": 38.6945686340332,
      "activations/layer13_attention_weight_min": -31.500411987304688,
      "activations/layer14_attention_weight_max": 48.70663833618164,
      "activations/layer14_attention_weight_min": -38.65773010253906,
      "activations/layer15_attention_weight_max": 34.388824462890625,
      "activations/layer15_attention_weight_min": -29.515668869018555,
      "activations/layer16_attention_weight_max": 32.14474868774414,
      "activations/layer16_attention_weight_min": -26.789758682250977,
      "activations/layer17_attention_weight_max": 32.649681091308594,
      "activations/layer17_attention_weight_min": -25.40068244934082,
      "activations/layer18_attention_weight_max": 32.13277053833008,
      "activations/layer18_attention_weight_min": -21.164018630981445,
      "activations/layer19_attention_weight_max": 33.82444381713867,
      "activations/layer19_attention_weight_min": -28.560077667236328,
      "activations/layer1_attention_weight_max": 17.095516204833984,
      "activations/layer1_attention_weight_min": -14.986892700195312,
      "activations/layer20_attention_weight_max": 27.8582763671875,
      "activations/layer20_attention_weight_min": -22.498746871948242,
      "activations/layer21_attention_weight_max": 29.182010650634766,
      "activations/layer21_attention_weight_min": -23.53902244567871,
      "activations/layer22_attention_weight_max": 43.02561569213867,
      "activations/layer22_attention_weight_min": -28.813575744628906,
      "activations/layer23_attention_weight_max": 34.42779541015625,
      "activations/layer23_attention_weight_min": -23.62417984008789,
      "activations/layer2_attention_weight_max": 34.748565673828125,
      "activations/layer2_attention_weight_min": -31.600601196289062,
      "activations/layer3_attention_weight_max": 92.43616485595703,
      "activations/layer3_attention_weight_min": -90.8822021484375,
      "activations/layer4_attention_weight_max": 120.34559631347656,
      "activations/layer4_attention_weight_min": -110.33296966552734,
      "activations/layer5_attention_weight_max": 58.267921447753906,
      "activations/layer5_attention_weight_min": -59.966766357421875,
      "activations/layer6_attention_weight_max": 50.539302825927734,
      "activations/layer6_attention_weight_min": -51.32422637939453,
      "activations/layer7_attention_weight_max": 93.83451843261719,
      "activations/layer7_attention_weight_min": -89.26016998291016,
      "activations/layer8_attention_weight_max": 42.64727783203125,
      "activations/layer8_attention_weight_min": -40.152706146240234,
      "activations/layer9_attention_weight_max": 35.126399993896484,
      "activations/layer9_attention_weight_min": -34.69614791870117,
      "epoch": 8.25,
      "learning_rate": 9.779356060606059e-05,
      "loss": 2.8032,
      "step": 141950
    },
    {
      "activations/layer0_attention_weight_max": 15.593199729919434,
      "activations/layer0_attention_weight_min": -11.959808349609375,
      "activations/layer10_attention_weight_max": 35.89458084106445,
      "activations/layer10_attention_weight_min": -32.00183868408203,
      "activations/layer11_attention_weight_max": 36.75891876220703,
      "activations/layer11_attention_weight_min": -31.30849266052246,
      "activations/layer12_attention_weight_max": 25.09568214416504,
      "activations/layer12_attention_weight_min": -28.95146369934082,
      "activations/layer13_attention_weight_max": 42.605369567871094,
      "activations/layer13_attention_weight_min": -35.844600677490234,
      "activations/layer14_attention_weight_max": 59.28798294067383,
      "activations/layer14_attention_weight_min": -40.76822280883789,
      "activations/layer15_attention_weight_max": 36.41676330566406,
      "activations/layer15_attention_weight_min": -29.739883422851562,
      "activations/layer16_attention_weight_max": 33.28044128417969,
      "activations/layer16_attention_weight_min": -29.73173713684082,
      "activations/layer17_attention_weight_max": 33.139461517333984,
      "activations/layer17_attention_weight_min": -24.50996971130371,
      "activations/layer18_attention_weight_max": 30.426517486572266,
      "activations/layer18_attention_weight_min": -21.547626495361328,
      "activations/layer19_attention_weight_max": 34.59564208984375,
      "activations/layer19_attention_weight_min": -29.493640899658203,
      "activations/layer1_attention_weight_max": 17.812389373779297,
      "activations/layer1_attention_weight_min": -15.960780143737793,
      "activations/layer20_attention_weight_max": 33.14076614379883,
      "activations/layer20_attention_weight_min": -24.994604110717773,
      "activations/layer21_attention_weight_max": 28.315832138061523,
      "activations/layer21_attention_weight_min": -22.561838150024414,
      "activations/layer22_attention_weight_max": 50.15335464477539,
      "activations/layer22_attention_weight_min": -29.250064849853516,
      "activations/layer23_attention_weight_max": 33.66484069824219,
      "activations/layer23_attention_weight_min": -21.555591583251953,
      "activations/layer2_attention_weight_max": 34.27950668334961,
      "activations/layer2_attention_weight_min": -32.982933044433594,
      "activations/layer3_attention_weight_max": 101.75627136230469,
      "activations/layer3_attention_weight_min": -102.07304382324219,
      "activations/layer4_attention_weight_max": 131.0770721435547,
      "activations/layer4_attention_weight_min": -119.93995666503906,
      "activations/layer5_attention_weight_max": 61.58806610107422,
      "activations/layer5_attention_weight_min": -62.32124328613281,
      "activations/layer6_attention_weight_max": 53.72134017944336,
      "activations/layer6_attention_weight_min": -50.75849151611328,
      "activations/layer7_attention_weight_max": 94.85696411132812,
      "activations/layer7_attention_weight_min": -91.49166870117188,
      "activations/layer8_attention_weight_max": 41.88238525390625,
      "activations/layer8_attention_weight_min": -40.56023025512695,
      "activations/layer9_attention_weight_max": 38.18126678466797,
      "activations/layer9_attention_weight_min": -34.818721771240234,
      "epoch": 8.25,
      "learning_rate": 9.77746212121212e-05,
      "loss": 2.8077,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_loss": 2.7578125,
      "eval_runtime": 8.5152,
      "eval_samples_per_second": 504.277,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_openwebtext_loss": 2.7578125,
      "eval_openwebtext_ppl": 15.765318566620541,
      "eval_openwebtext_runtime": 8.5152,
      "eval_openwebtext_samples_per_second": 504.277,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_wikitext_loss": 2.978515625,
      "eval_wikitext_ppl": 19.658614220966218,
      "eval_wikitext_runtime": 2.0148,
      "eval_wikitext_samples_per_second": 226.328,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_lambada_loss": 2.6796875,
      "eval_lambada_ppl": 14.580536166314408,
      "eval_lambada_runtime": 9.6176,
      "eval_lambada_samples_per_second": 506.261,
      "step": 142000
    },
    {
      "activations/layer0_attention_weight_max": 15.513041496276855,
      "activations/layer0_attention_weight_min": -11.626723289489746,
      "activations/layer10_attention_weight_max": 38.553897857666016,
      "activations/layer10_attention_weight_min": -33.4380989074707,
      "activations/layer11_attention_weight_max": 38.114524841308594,
      "activations/layer11_attention_weight_min": -32.305023193359375,
      "activations/layer12_attention_weight_max": 22.797630310058594,
      "activations/layer12_attention_weight_min": -24.870933532714844,
      "activations/layer13_attention_weight_max": 41.248924255371094,
      "activations/layer13_attention_weight_min": -33.694190979003906,
      "activations/layer14_attention_weight_max": 44.96529769897461,
      "activations/layer14_attention_weight_min": -41.333412170410156,
      "activations/layer15_attention_weight_max": 37.82612609863281,
      "activations/layer15_attention_weight_min": -30.206764221191406,
      "activations/layer16_attention_weight_max": 31.983362197875977,
      "activations/layer16_attention_weight_min": -26.376976013183594,
      "activations/layer17_attention_weight_max": 34.002113342285156,
      "activations/layer17_attention_weight_min": -24.840898513793945,
      "activations/layer18_attention_weight_max": 31.338802337646484,
      "activations/layer18_attention_weight_min": -22.282567977905273,
      "activations/layer19_attention_weight_max": 35.62936782836914,
      "activations/layer19_attention_weight_min": -29.533201217651367,
      "activations/layer1_attention_weight_max": 16.896297454833984,
      "activations/layer1_attention_weight_min": -14.550873756408691,
      "activations/layer20_attention_weight_max": 31.32064437866211,
      "activations/layer20_attention_weight_min": -24.846689224243164,
      "activations/layer21_attention_weight_max": 29.087478637695312,
      "activations/layer21_attention_weight_min": -25.450035095214844,
      "activations/layer22_attention_weight_max": 44.81425857543945,
      "activations/layer22_attention_weight_min": -31.388227462768555,
      "activations/layer23_attention_weight_max": 39.57186508178711,
      "activations/layer23_attention_weight_min": -24.293811798095703,
      "activations/layer2_attention_weight_max": 33.02424621582031,
      "activations/layer2_attention_weight_min": -31.82361602783203,
      "activations/layer3_attention_weight_max": 93.03927612304688,
      "activations/layer3_attention_weight_min": -92.0902099609375,
      "activations/layer4_attention_weight_max": 123.74578094482422,
      "activations/layer4_attention_weight_min": -114.96233367919922,
      "activations/layer5_attention_weight_max": 59.93662643432617,
      "activations/layer5_attention_weight_min": -58.52952194213867,
      "activations/layer6_attention_weight_max": 47.65894317626953,
      "activations/layer6_attention_weight_min": -49.95364761352539,
      "activations/layer7_attention_weight_max": 86.75111389160156,
      "activations/layer7_attention_weight_min": -87.18895721435547,
      "activations/layer8_attention_weight_max": 41.63459777832031,
      "activations/layer8_attention_weight_min": -40.43290710449219,
      "activations/layer9_attention_weight_max": 37.28376388549805,
      "activations/layer9_attention_weight_min": -34.21488571166992,
      "epoch": 8.25,
      "learning_rate": 9.775568181818182e-05,
      "loss": 2.8005,
      "step": 142050
    },
    {
      "activations/layer0_attention_weight_max": 16.50475311279297,
      "activations/layer0_attention_weight_min": -11.761302947998047,
      "activations/layer10_attention_weight_max": 33.89229202270508,
      "activations/layer10_attention_weight_min": -32.240230560302734,
      "activations/layer11_attention_weight_max": 35.00538635253906,
      "activations/layer11_attention_weight_min": -30.38127899169922,
      "activations/layer12_attention_weight_max": 23.52208709716797,
      "activations/layer12_attention_weight_min": -24.854877471923828,
      "activations/layer13_attention_weight_max": 40.0937385559082,
      "activations/layer13_attention_weight_min": -32.579246520996094,
      "activations/layer14_attention_weight_max": 48.889793395996094,
      "activations/layer14_attention_weight_min": -38.17919158935547,
      "activations/layer15_attention_weight_max": 36.87900924682617,
      "activations/layer15_attention_weight_min": -30.012065887451172,
      "activations/layer16_attention_weight_max": 34.50947952270508,
      "activations/layer16_attention_weight_min": -26.16625213623047,
      "activations/layer17_attention_weight_max": 33.497615814208984,
      "activations/layer17_attention_weight_min": -25.0454158782959,
      "activations/layer18_attention_weight_max": 27.97294807434082,
      "activations/layer18_attention_weight_min": -21.4144344329834,
      "activations/layer19_attention_weight_max": 37.90237808227539,
      "activations/layer19_attention_weight_min": -29.987829208374023,
      "activations/layer1_attention_weight_max": 17.422338485717773,
      "activations/layer1_attention_weight_min": -15.774735450744629,
      "activations/layer20_attention_weight_max": 28.030410766601562,
      "activations/layer20_attention_weight_min": -22.926834106445312,
      "activations/layer21_attention_weight_max": 32.012969970703125,
      "activations/layer21_attention_weight_min": -21.75309181213379,
      "activations/layer22_attention_weight_max": 44.4459228515625,
      "activations/layer22_attention_weight_min": -28.49189567565918,
      "activations/layer23_attention_weight_max": 34.0294189453125,
      "activations/layer23_attention_weight_min": -23.191890716552734,
      "activations/layer2_attention_weight_max": 33.736427307128906,
      "activations/layer2_attention_weight_min": -33.97774124145508,
      "activations/layer3_attention_weight_max": 96.14824676513672,
      "activations/layer3_attention_weight_min": -96.44475555419922,
      "activations/layer4_attention_weight_max": 121.1655044555664,
      "activations/layer4_attention_weight_min": -119.9620132446289,
      "activations/layer5_attention_weight_max": 57.060054779052734,
      "activations/layer5_attention_weight_min": -63.359214782714844,
      "activations/layer6_attention_weight_max": 48.224979400634766,
      "activations/layer6_attention_weight_min": -52.243507385253906,
      "activations/layer7_attention_weight_max": 95.53887176513672,
      "activations/layer7_attention_weight_min": -91.19496154785156,
      "activations/layer8_attention_weight_max": 41.908084869384766,
      "activations/layer8_attention_weight_min": -41.624290466308594,
      "activations/layer9_attention_weight_max": 38.26268768310547,
      "activations/layer9_attention_weight_min": -36.1148796081543,
      "epoch": 8.26,
      "learning_rate": 9.77367424242424e-05,
      "loss": 2.798,
      "step": 142100
    },
    {
      "activations/layer0_attention_weight_max": 15.45176887512207,
      "activations/layer0_attention_weight_min": -11.594804763793945,
      "activations/layer10_attention_weight_max": 38.84941482543945,
      "activations/layer10_attention_weight_min": -34.95486831665039,
      "activations/layer11_attention_weight_max": 38.456016540527344,
      "activations/layer11_attention_weight_min": -33.217525482177734,
      "activations/layer12_attention_weight_max": 24.595264434814453,
      "activations/layer12_attention_weight_min": -28.50482940673828,
      "activations/layer13_attention_weight_max": 34.273075103759766,
      "activations/layer13_attention_weight_min": -32.80799865722656,
      "activations/layer14_attention_weight_max": 47.786468505859375,
      "activations/layer14_attention_weight_min": -39.81453323364258,
      "activations/layer15_attention_weight_max": 39.037349700927734,
      "activations/layer15_attention_weight_min": -30.942394256591797,
      "activations/layer16_attention_weight_max": 30.94938850402832,
      "activations/layer16_attention_weight_min": -27.18141746520996,
      "activations/layer17_attention_weight_max": 31.849931716918945,
      "activations/layer17_attention_weight_min": -24.465051651000977,
      "activations/layer18_attention_weight_max": 28.783540725708008,
      "activations/layer18_attention_weight_min": -20.32431983947754,
      "activations/layer19_attention_weight_max": 35.80062484741211,
      "activations/layer19_attention_weight_min": -30.330007553100586,
      "activations/layer1_attention_weight_max": 17.369186401367188,
      "activations/layer1_attention_weight_min": -16.055461883544922,
      "activations/layer20_attention_weight_max": 30.04155731201172,
      "activations/layer20_attention_weight_min": -24.5924015045166,
      "activations/layer21_attention_weight_max": 29.428922653198242,
      "activations/layer21_attention_weight_min": -22.8635311126709,
      "activations/layer22_attention_weight_max": 43.75455093383789,
      "activations/layer22_attention_weight_min": -29.513731002807617,
      "activations/layer23_attention_weight_max": 38.291603088378906,
      "activations/layer23_attention_weight_min": -23.741973876953125,
      "activations/layer2_attention_weight_max": 34.664756774902344,
      "activations/layer2_attention_weight_min": -33.530426025390625,
      "activations/layer3_attention_weight_max": 104.05557250976562,
      "activations/layer3_attention_weight_min": -106.20928192138672,
      "activations/layer4_attention_weight_max": 134.6275177001953,
      "activations/layer4_attention_weight_min": -123.76252746582031,
      "activations/layer5_attention_weight_max": 65.513427734375,
      "activations/layer5_attention_weight_min": -62.891632080078125,
      "activations/layer6_attention_weight_max": 53.270347595214844,
      "activations/layer6_attention_weight_min": -51.99031066894531,
      "activations/layer7_attention_weight_max": 102.30622100830078,
      "activations/layer7_attention_weight_min": -96.169189453125,
      "activations/layer8_attention_weight_max": 48.4164924621582,
      "activations/layer8_attention_weight_min": -44.25917434692383,
      "activations/layer9_attention_weight_max": 37.616512298583984,
      "activations/layer9_attention_weight_min": -37.67871856689453,
      "epoch": 8.26,
      "learning_rate": 9.771780303030302e-05,
      "loss": 2.8084,
      "step": 142150
    },
    {
      "activations/layer0_attention_weight_max": 15.68303108215332,
      "activations/layer0_attention_weight_min": -12.613667488098145,
      "activations/layer10_attention_weight_max": 35.39332962036133,
      "activations/layer10_attention_weight_min": -31.98002052307129,
      "activations/layer11_attention_weight_max": 33.904449462890625,
      "activations/layer11_attention_weight_min": -31.267333984375,
      "activations/layer12_attention_weight_max": 24.054086685180664,
      "activations/layer12_attention_weight_min": -23.506940841674805,
      "activations/layer13_attention_weight_max": 40.221946716308594,
      "activations/layer13_attention_weight_min": -33.36970901489258,
      "activations/layer14_attention_weight_max": 46.13645553588867,
      "activations/layer14_attention_weight_min": -37.67285919189453,
      "activations/layer15_attention_weight_max": 36.903343200683594,
      "activations/layer15_attention_weight_min": -30.20642852783203,
      "activations/layer16_attention_weight_max": 31.680753707885742,
      "activations/layer16_attention_weight_min": -31.273637771606445,
      "activations/layer17_attention_weight_max": 36.3275260925293,
      "activations/layer17_attention_weight_min": -26.100191116333008,
      "activations/layer18_attention_weight_max": 34.40586853027344,
      "activations/layer18_attention_weight_min": -22.808452606201172,
      "activations/layer19_attention_weight_max": 38.7008056640625,
      "activations/layer19_attention_weight_min": -29.369892120361328,
      "activations/layer1_attention_weight_max": 16.484533309936523,
      "activations/layer1_attention_weight_min": -15.406831741333008,
      "activations/layer20_attention_weight_max": 35.672264099121094,
      "activations/layer20_attention_weight_min": -26.91552734375,
      "activations/layer21_attention_weight_max": 33.48966598510742,
      "activations/layer21_attention_weight_min": -22.390304565429688,
      "activations/layer22_attention_weight_max": 53.47810363769531,
      "activations/layer22_attention_weight_min": -31.217092514038086,
      "activations/layer23_attention_weight_max": 38.39555358886719,
      "activations/layer23_attention_weight_min": -21.41079330444336,
      "activations/layer2_attention_weight_max": 34.72857666015625,
      "activations/layer2_attention_weight_min": -33.620079040527344,
      "activations/layer3_attention_weight_max": 97.65489196777344,
      "activations/layer3_attention_weight_min": -96.46693420410156,
      "activations/layer4_attention_weight_max": 128.27931213378906,
      "activations/layer4_attention_weight_min": -121.9729232788086,
      "activations/layer5_attention_weight_max": 59.63689422607422,
      "activations/layer5_attention_weight_min": -64.7596206665039,
      "activations/layer6_attention_weight_max": 50.606815338134766,
      "activations/layer6_attention_weight_min": -53.49930191040039,
      "activations/layer7_attention_weight_max": 89.7397689819336,
      "activations/layer7_attention_weight_min": -92.86808776855469,
      "activations/layer8_attention_weight_max": 43.019432067871094,
      "activations/layer8_attention_weight_min": -43.30598831176758,
      "activations/layer9_attention_weight_max": 39.560302734375,
      "activations/layer9_attention_weight_min": -33.815425872802734,
      "epoch": 8.26,
      "learning_rate": 9.769886363636364e-05,
      "loss": 2.8284,
      "step": 142200
    },
    {
      "activations/layer0_attention_weight_max": 15.976099967956543,
      "activations/layer0_attention_weight_min": -12.056303024291992,
      "activations/layer10_attention_weight_max": 33.30532455444336,
      "activations/layer10_attention_weight_min": -30.757408142089844,
      "activations/layer11_attention_weight_max": 32.402610778808594,
      "activations/layer11_attention_weight_min": -28.64989471435547,
      "activations/layer12_attention_weight_max": 24.01287841796875,
      "activations/layer12_attention_weight_min": -24.355697631835938,
      "activations/layer13_attention_weight_max": 42.02302169799805,
      "activations/layer13_attention_weight_min": -34.37306594848633,
      "activations/layer14_attention_weight_max": 48.40689468383789,
      "activations/layer14_attention_weight_min": -42.41192626953125,
      "activations/layer15_attention_weight_max": 36.7955436706543,
      "activations/layer15_attention_weight_min": -28.299835205078125,
      "activations/layer16_attention_weight_max": 34.48318862915039,
      "activations/layer16_attention_weight_min": -27.765775680541992,
      "activations/layer17_attention_weight_max": 36.412845611572266,
      "activations/layer17_attention_weight_min": -24.90701675415039,
      "activations/layer18_attention_weight_max": 34.321014404296875,
      "activations/layer18_attention_weight_min": -21.611125946044922,
      "activations/layer19_attention_weight_max": 35.2703971862793,
      "activations/layer19_attention_weight_min": -28.135482788085938,
      "activations/layer1_attention_weight_max": 16.771543502807617,
      "activations/layer1_attention_weight_min": -15.608409881591797,
      "activations/layer20_attention_weight_max": 30.699695587158203,
      "activations/layer20_attention_weight_min": -22.34503936767578,
      "activations/layer21_attention_weight_max": 27.304351806640625,
      "activations/layer21_attention_weight_min": -21.787939071655273,
      "activations/layer22_attention_weight_max": 41.92826843261719,
      "activations/layer22_attention_weight_min": -28.019121170043945,
      "activations/layer23_attention_weight_max": 34.095176696777344,
      "activations/layer23_attention_weight_min": -26.56216049194336,
      "activations/layer2_attention_weight_max": 34.734642028808594,
      "activations/layer2_attention_weight_min": -34.28026580810547,
      "activations/layer3_attention_weight_max": 91.11727905273438,
      "activations/layer3_attention_weight_min": -94.15628814697266,
      "activations/layer4_attention_weight_max": 117.89046478271484,
      "activations/layer4_attention_weight_min": -118.3561019897461,
      "activations/layer5_attention_weight_max": 56.68036651611328,
      "activations/layer5_attention_weight_min": -65.44254302978516,
      "activations/layer6_attention_weight_max": 48.78430938720703,
      "activations/layer6_attention_weight_min": -52.403953552246094,
      "activations/layer7_attention_weight_max": 94.05958557128906,
      "activations/layer7_attention_weight_min": -88.46916961669922,
      "activations/layer8_attention_weight_max": 45.444419860839844,
      "activations/layer8_attention_weight_min": -44.945186614990234,
      "activations/layer9_attention_weight_max": 38.55351638793945,
      "activations/layer9_attention_weight_min": -36.74388885498047,
      "epoch": 8.27,
      "learning_rate": 9.767992424242422e-05,
      "loss": 2.8262,
      "step": 142250
    },
    {
      "activations/layer0_attention_weight_max": 15.492082595825195,
      "activations/layer0_attention_weight_min": -12.736698150634766,
      "activations/layer10_attention_weight_max": 33.49257278442383,
      "activations/layer10_attention_weight_min": -33.69880294799805,
      "activations/layer11_attention_weight_max": 33.46311569213867,
      "activations/layer11_attention_weight_min": -30.89675521850586,
      "activations/layer12_attention_weight_max": 38.78874206542969,
      "activations/layer12_attention_weight_min": -25.486507415771484,
      "activations/layer13_attention_weight_max": 44.11259841918945,
      "activations/layer13_attention_weight_min": -38.24494552612305,
      "activations/layer14_attention_weight_max": 64.65738677978516,
      "activations/layer14_attention_weight_min": -47.34024429321289,
      "activations/layer15_attention_weight_max": 40.22898864746094,
      "activations/layer15_attention_weight_min": -33.55144119262695,
      "activations/layer16_attention_weight_max": 39.185569763183594,
      "activations/layer16_attention_weight_min": -28.70368003845215,
      "activations/layer17_attention_weight_max": 40.0128059387207,
      "activations/layer17_attention_weight_min": -25.674835205078125,
      "activations/layer18_attention_weight_max": 36.983055114746094,
      "activations/layer18_attention_weight_min": -24.404041290283203,
      "activations/layer19_attention_weight_max": 45.11446762084961,
      "activations/layer19_attention_weight_min": -32.11892318725586,
      "activations/layer1_attention_weight_max": 19.364717483520508,
      "activations/layer1_attention_weight_min": -16.015600204467773,
      "activations/layer20_attention_weight_max": 35.21165466308594,
      "activations/layer20_attention_weight_min": -25.594785690307617,
      "activations/layer21_attention_weight_max": 38.3847770690918,
      "activations/layer21_attention_weight_min": -23.885025024414062,
      "activations/layer22_attention_weight_max": 60.888031005859375,
      "activations/layer22_attention_weight_min": -29.44671630859375,
      "activations/layer23_attention_weight_max": 40.79656219482422,
      "activations/layer23_attention_weight_min": -23.683883666992188,
      "activations/layer2_attention_weight_max": 33.25532913208008,
      "activations/layer2_attention_weight_min": -31.12482452392578,
      "activations/layer3_attention_weight_max": 96.62278747558594,
      "activations/layer3_attention_weight_min": -93.33879089355469,
      "activations/layer4_attention_weight_max": 123.34651184082031,
      "activations/layer4_attention_weight_min": -113.93675994873047,
      "activations/layer5_attention_weight_max": 64.0215072631836,
      "activations/layer5_attention_weight_min": -62.72400665283203,
      "activations/layer6_attention_weight_max": 50.734413146972656,
      "activations/layer6_attention_weight_min": -51.08496856689453,
      "activations/layer7_attention_weight_max": 90.6019058227539,
      "activations/layer7_attention_weight_min": -93.20465087890625,
      "activations/layer8_attention_weight_max": 44.80698776245117,
      "activations/layer8_attention_weight_min": -44.51774597167969,
      "activations/layer9_attention_weight_max": 38.231990814208984,
      "activations/layer9_attention_weight_min": -36.8380126953125,
      "epoch": 8.27,
      "learning_rate": 9.766098484848484e-05,
      "loss": 2.7991,
      "step": 142300
    },
    {
      "activations/layer0_attention_weight_max": 15.812908172607422,
      "activations/layer0_attention_weight_min": -11.697009086608887,
      "activations/layer10_attention_weight_max": 36.46023178100586,
      "activations/layer10_attention_weight_min": -33.99014663696289,
      "activations/layer11_attention_weight_max": 39.921451568603516,
      "activations/layer11_attention_weight_min": -33.15980529785156,
      "activations/layer12_attention_weight_max": 39.08235549926758,
      "activations/layer12_attention_weight_min": -28.367685317993164,
      "activations/layer13_attention_weight_max": 58.991825103759766,
      "activations/layer13_attention_weight_min": -34.84550094604492,
      "activations/layer14_attention_weight_max": 60.885868072509766,
      "activations/layer14_attention_weight_min": -42.67155075073242,
      "activations/layer15_attention_weight_max": 48.065895080566406,
      "activations/layer15_attention_weight_min": -29.64885711669922,
      "activations/layer16_attention_weight_max": 35.86793518066406,
      "activations/layer16_attention_weight_min": -25.91370391845703,
      "activations/layer17_attention_weight_max": 35.81397247314453,
      "activations/layer17_attention_weight_min": -24.509387969970703,
      "activations/layer18_attention_weight_max": 30.112932205200195,
      "activations/layer18_attention_weight_min": -21.200075149536133,
      "activations/layer19_attention_weight_max": 39.65443420410156,
      "activations/layer19_attention_weight_min": -31.101444244384766,
      "activations/layer1_attention_weight_max": 16.738447189331055,
      "activations/layer1_attention_weight_min": -15.76242446899414,
      "activations/layer20_attention_weight_max": 33.09303665161133,
      "activations/layer20_attention_weight_min": -24.01198959350586,
      "activations/layer21_attention_weight_max": 30.27396011352539,
      "activations/layer21_attention_weight_min": -23.090402603149414,
      "activations/layer22_attention_weight_max": 48.6401252746582,
      "activations/layer22_attention_weight_min": -29.128849029541016,
      "activations/layer23_attention_weight_max": 38.030792236328125,
      "activations/layer23_attention_weight_min": -24.799331665039062,
      "activations/layer2_attention_weight_max": 33.942626953125,
      "activations/layer2_attention_weight_min": -34.041831970214844,
      "activations/layer3_attention_weight_max": 97.78235626220703,
      "activations/layer3_attention_weight_min": -100.5158462524414,
      "activations/layer4_attention_weight_max": 124.14983367919922,
      "activations/layer4_attention_weight_min": -120.98945617675781,
      "activations/layer5_attention_weight_max": 60.51029968261719,
      "activations/layer5_attention_weight_min": -59.79348373413086,
      "activations/layer6_attention_weight_max": 48.62944412231445,
      "activations/layer6_attention_weight_min": -55.43318557739258,
      "activations/layer7_attention_weight_max": 93.80130767822266,
      "activations/layer7_attention_weight_min": -99.76790618896484,
      "activations/layer8_attention_weight_max": 41.993003845214844,
      "activations/layer8_attention_weight_min": -44.549808502197266,
      "activations/layer9_attention_weight_max": 36.66112518310547,
      "activations/layer9_attention_weight_min": -37.08939743041992,
      "epoch": 8.27,
      "learning_rate": 9.764204545454545e-05,
      "loss": 2.7969,
      "step": 142350
    },
    {
      "activations/layer0_attention_weight_max": 15.571407318115234,
      "activations/layer0_attention_weight_min": -12.130682945251465,
      "activations/layer10_attention_weight_max": 34.601715087890625,
      "activations/layer10_attention_weight_min": -36.889957427978516,
      "activations/layer11_attention_weight_max": 34.28934860229492,
      "activations/layer11_attention_weight_min": -32.471229553222656,
      "activations/layer12_attention_weight_max": 26.997318267822266,
      "activations/layer12_attention_weight_min": -24.843585968017578,
      "activations/layer13_attention_weight_max": 46.54824447631836,
      "activations/layer13_attention_weight_min": -34.79542922973633,
      "activations/layer14_attention_weight_max": 53.9210319519043,
      "activations/layer14_attention_weight_min": -41.611209869384766,
      "activations/layer15_attention_weight_max": 41.294097900390625,
      "activations/layer15_attention_weight_min": -32.379371643066406,
      "activations/layer16_attention_weight_max": 36.31572341918945,
      "activations/layer16_attention_weight_min": -27.42711639404297,
      "activations/layer17_attention_weight_max": 39.2178955078125,
      "activations/layer17_attention_weight_min": -26.0865421295166,
      "activations/layer18_attention_weight_max": 34.55840301513672,
      "activations/layer18_attention_weight_min": -22.600711822509766,
      "activations/layer19_attention_weight_max": 41.482139587402344,
      "activations/layer19_attention_weight_min": -31.47654151916504,
      "activations/layer1_attention_weight_max": 16.976181030273438,
      "activations/layer1_attention_weight_min": -14.25908374786377,
      "activations/layer20_attention_weight_max": 39.14585876464844,
      "activations/layer20_attention_weight_min": -23.924335479736328,
      "activations/layer21_attention_weight_max": 35.53725814819336,
      "activations/layer21_attention_weight_min": -21.79292869567871,
      "activations/layer22_attention_weight_max": 62.36502456665039,
      "activations/layer22_attention_weight_min": -32.44602966308594,
      "activations/layer23_attention_weight_max": 45.570648193359375,
      "activations/layer23_attention_weight_min": -25.59883689880371,
      "activations/layer2_attention_weight_max": 31.251569747924805,
      "activations/layer2_attention_weight_min": -32.37866973876953,
      "activations/layer3_attention_weight_max": 94.14903259277344,
      "activations/layer3_attention_weight_min": -96.11045837402344,
      "activations/layer4_attention_weight_max": 125.65311431884766,
      "activations/layer4_attention_weight_min": -125.00361633300781,
      "activations/layer5_attention_weight_max": 59.210235595703125,
      "activations/layer5_attention_weight_min": -62.57130432128906,
      "activations/layer6_attention_weight_max": 51.110042572021484,
      "activations/layer6_attention_weight_min": -51.79296112060547,
      "activations/layer7_attention_weight_max": 92.98670196533203,
      "activations/layer7_attention_weight_min": -88.52474212646484,
      "activations/layer8_attention_weight_max": 40.46982955932617,
      "activations/layer8_attention_weight_min": -42.70581817626953,
      "activations/layer9_attention_weight_max": 38.030296325683594,
      "activations/layer9_attention_weight_min": -38.60399627685547,
      "epoch": 8.27,
      "learning_rate": 9.762310606060605e-05,
      "loss": 2.8039,
      "step": 142400
    },
    {
      "activations/layer0_attention_weight_max": 15.972679138183594,
      "activations/layer0_attention_weight_min": -10.680469512939453,
      "activations/layer10_attention_weight_max": 35.030029296875,
      "activations/layer10_attention_weight_min": -34.837684631347656,
      "activations/layer11_attention_weight_max": 35.781612396240234,
      "activations/layer11_attention_weight_min": -32.2307014465332,
      "activations/layer12_attention_weight_max": 36.385215759277344,
      "activations/layer12_attention_weight_min": -30.069618225097656,
      "activations/layer13_attention_weight_max": 45.01872634887695,
      "activations/layer13_attention_weight_min": -35.943885803222656,
      "activations/layer14_attention_weight_max": 60.09050750732422,
      "activations/layer14_attention_weight_min": -41.93796920776367,
      "activations/layer15_attention_weight_max": 36.0743522644043,
      "activations/layer15_attention_weight_min": -30.236865997314453,
      "activations/layer16_attention_weight_max": 33.013343811035156,
      "activations/layer16_attention_weight_min": -26.19020652770996,
      "activations/layer17_attention_weight_max": 33.564083099365234,
      "activations/layer17_attention_weight_min": -24.650259017944336,
      "activations/layer18_attention_weight_max": 30.1805362701416,
      "activations/layer18_attention_weight_min": -22.679344177246094,
      "activations/layer19_attention_weight_max": 34.13045120239258,
      "activations/layer19_attention_weight_min": -29.96409034729004,
      "activations/layer1_attention_weight_max": 15.973093032836914,
      "activations/layer1_attention_weight_min": -13.597941398620605,
      "activations/layer20_attention_weight_max": 29.79191780090332,
      "activations/layer20_attention_weight_min": -22.895723342895508,
      "activations/layer21_attention_weight_max": 29.521785736083984,
      "activations/layer21_attention_weight_min": -25.036205291748047,
      "activations/layer22_attention_weight_max": 44.94038772583008,
      "activations/layer22_attention_weight_min": -27.779647827148438,
      "activations/layer23_attention_weight_max": 36.537986755371094,
      "activations/layer23_attention_weight_min": -23.927284240722656,
      "activations/layer2_attention_weight_max": 31.452619552612305,
      "activations/layer2_attention_weight_min": -31.660465240478516,
      "activations/layer3_attention_weight_max": 97.40948486328125,
      "activations/layer3_attention_weight_min": -94.7191390991211,
      "activations/layer4_attention_weight_max": 127.26507568359375,
      "activations/layer4_attention_weight_min": -121.92765045166016,
      "activations/layer5_attention_weight_max": 59.006126403808594,
      "activations/layer5_attention_weight_min": -63.60610580444336,
      "activations/layer6_attention_weight_max": 51.467445373535156,
      "activations/layer6_attention_weight_min": -55.325950622558594,
      "activations/layer7_attention_weight_max": 91.32427978515625,
      "activations/layer7_attention_weight_min": -94.571533203125,
      "activations/layer8_attention_weight_max": 41.531410217285156,
      "activations/layer8_attention_weight_min": -46.37956619262695,
      "activations/layer9_attention_weight_max": 34.702510833740234,
      "activations/layer9_attention_weight_min": -33.702022552490234,
      "epoch": 8.28,
      "learning_rate": 9.760416666666666e-05,
      "loss": 2.8267,
      "step": 142450
    },
    {
      "activations/layer0_attention_weight_max": 15.717981338500977,
      "activations/layer0_attention_weight_min": -11.822151184082031,
      "activations/layer10_attention_weight_max": 35.137245178222656,
      "activations/layer10_attention_weight_min": -33.11968994140625,
      "activations/layer11_attention_weight_max": 33.425086975097656,
      "activations/layer11_attention_weight_min": -29.83301544189453,
      "activations/layer12_attention_weight_max": 30.47824478149414,
      "activations/layer12_attention_weight_min": -26.572555541992188,
      "activations/layer13_attention_weight_max": 41.80887985229492,
      "activations/layer13_attention_weight_min": -33.03663635253906,
      "activations/layer14_attention_weight_max": 49.853336334228516,
      "activations/layer14_attention_weight_min": -43.862579345703125,
      "activations/layer15_attention_weight_max": 37.29151153564453,
      "activations/layer15_attention_weight_min": -30.759992599487305,
      "activations/layer16_attention_weight_max": 36.54328918457031,
      "activations/layer16_attention_weight_min": -28.0510311126709,
      "activations/layer17_attention_weight_max": 37.47539520263672,
      "activations/layer17_attention_weight_min": -25.944406509399414,
      "activations/layer18_attention_weight_max": 35.99854278564453,
      "activations/layer18_attention_weight_min": -23.852596282958984,
      "activations/layer19_attention_weight_max": 36.86273193359375,
      "activations/layer19_attention_weight_min": -34.367584228515625,
      "activations/layer1_attention_weight_max": 16.756038665771484,
      "activations/layer1_attention_weight_min": -15.883475303649902,
      "activations/layer20_attention_weight_max": 37.0078125,
      "activations/layer20_attention_weight_min": -25.26567840576172,
      "activations/layer21_attention_weight_max": 34.0558967590332,
      "activations/layer21_attention_weight_min": -23.96599578857422,
      "activations/layer22_attention_weight_max": 49.85771942138672,
      "activations/layer22_attention_weight_min": -31.242910385131836,
      "activations/layer23_attention_weight_max": 35.51392364501953,
      "activations/layer23_attention_weight_min": -24.536468505859375,
      "activations/layer2_attention_weight_max": 33.87348175048828,
      "activations/layer2_attention_weight_min": -32.42768478393555,
      "activations/layer3_attention_weight_max": 100.37005615234375,
      "activations/layer3_attention_weight_min": -94.7389144897461,
      "activations/layer4_attention_weight_max": 130.46847534179688,
      "activations/layer4_attention_weight_min": -118.95550537109375,
      "activations/layer5_attention_weight_max": 60.46952819824219,
      "activations/layer5_attention_weight_min": -61.816097259521484,
      "activations/layer6_attention_weight_max": 50.528194427490234,
      "activations/layer6_attention_weight_min": -54.30411911010742,
      "activations/layer7_attention_weight_max": 99.3371353149414,
      "activations/layer7_attention_weight_min": -91.08512878417969,
      "activations/layer8_attention_weight_max": 42.66553497314453,
      "activations/layer8_attention_weight_min": -42.09703826904297,
      "activations/layer9_attention_weight_max": 37.600154876708984,
      "activations/layer9_attention_weight_min": -34.5510368347168,
      "epoch": 8.28,
      "learning_rate": 9.758522727272726e-05,
      "loss": 2.8034,
      "step": 142500
    },
    {
      "activations/layer0_attention_weight_max": 15.363812446594238,
      "activations/layer0_attention_weight_min": -11.908905982971191,
      "activations/layer10_attention_weight_max": 32.15343475341797,
      "activations/layer10_attention_weight_min": -29.606639862060547,
      "activations/layer11_attention_weight_max": 31.726980209350586,
      "activations/layer11_attention_weight_min": -28.66921043395996,
      "activations/layer12_attention_weight_max": 51.595619201660156,
      "activations/layer12_attention_weight_min": -25.31203842163086,
      "activations/layer13_attention_weight_max": 46.65412521362305,
      "activations/layer13_attention_weight_min": -32.00288772583008,
      "activations/layer14_attention_weight_max": 49.91817092895508,
      "activations/layer14_attention_weight_min": -40.636348724365234,
      "activations/layer15_attention_weight_max": 39.33613204956055,
      "activations/layer15_attention_weight_min": -32.7891960144043,
      "activations/layer16_attention_weight_max": 33.50687789916992,
      "activations/layer16_attention_weight_min": -28.05663299560547,
      "activations/layer17_attention_weight_max": 35.91435623168945,
      "activations/layer17_attention_weight_min": -24.74969482421875,
      "activations/layer18_attention_weight_max": 32.582305908203125,
      "activations/layer18_attention_weight_min": -22.36735725402832,
      "activations/layer19_attention_weight_max": 36.18142318725586,
      "activations/layer19_attention_weight_min": -31.144744873046875,
      "activations/layer1_attention_weight_max": 16.765216827392578,
      "activations/layer1_attention_weight_min": -13.511689186096191,
      "activations/layer20_attention_weight_max": 32.62391662597656,
      "activations/layer20_attention_weight_min": -26.9924259185791,
      "activations/layer21_attention_weight_max": 32.872840881347656,
      "activations/layer21_attention_weight_min": -24.536991119384766,
      "activations/layer22_attention_weight_max": 46.74370193481445,
      "activations/layer22_attention_weight_min": -30.23346519470215,
      "activations/layer23_attention_weight_max": 36.34996795654297,
      "activations/layer23_attention_weight_min": -25.287277221679688,
      "activations/layer2_attention_weight_max": 31.862125396728516,
      "activations/layer2_attention_weight_min": -33.164772033691406,
      "activations/layer3_attention_weight_max": 91.16864776611328,
      "activations/layer3_attention_weight_min": -91.85202026367188,
      "activations/layer4_attention_weight_max": 118.74415588378906,
      "activations/layer4_attention_weight_min": -113.66584777832031,
      "activations/layer5_attention_weight_max": 53.870277404785156,
      "activations/layer5_attention_weight_min": -58.69921875,
      "activations/layer6_attention_weight_max": 46.609901428222656,
      "activations/layer6_attention_weight_min": -49.389957427978516,
      "activations/layer7_attention_weight_max": 87.83174896240234,
      "activations/layer7_attention_weight_min": -86.68130493164062,
      "activations/layer8_attention_weight_max": 37.61650466918945,
      "activations/layer8_attention_weight_min": -39.488826751708984,
      "activations/layer9_attention_weight_max": 33.36849594116211,
      "activations/layer9_attention_weight_min": -31.432981491088867,
      "epoch": 8.28,
      "learning_rate": 9.756666666666666e-05,
      "loss": 2.7863,
      "step": 142550
    },
    {
      "activations/layer0_attention_weight_max": 16.741147994995117,
      "activations/layer0_attention_weight_min": -12.40175724029541,
      "activations/layer10_attention_weight_max": 36.729583740234375,
      "activations/layer10_attention_weight_min": -35.39726638793945,
      "activations/layer11_attention_weight_max": 33.95814895629883,
      "activations/layer11_attention_weight_min": -31.481460571289062,
      "activations/layer12_attention_weight_max": 41.59436798095703,
      "activations/layer12_attention_weight_min": -26.230051040649414,
      "activations/layer13_attention_weight_max": 44.75321578979492,
      "activations/layer13_attention_weight_min": -34.3642463684082,
      "activations/layer14_attention_weight_max": 58.05222702026367,
      "activations/layer14_attention_weight_min": -42.80611801147461,
      "activations/layer15_attention_weight_max": 39.62903594970703,
      "activations/layer15_attention_weight_min": -32.51263427734375,
      "activations/layer16_attention_weight_max": 36.16390609741211,
      "activations/layer16_attention_weight_min": -28.659931182861328,
      "activations/layer17_attention_weight_max": 37.94709014892578,
      "activations/layer17_attention_weight_min": -27.96522331237793,
      "activations/layer18_attention_weight_max": 31.460556030273438,
      "activations/layer18_attention_weight_min": -23.401996612548828,
      "activations/layer19_attention_weight_max": 41.309078216552734,
      "activations/layer19_attention_weight_min": -29.769254684448242,
      "activations/layer1_attention_weight_max": 16.499677658081055,
      "activations/layer1_attention_weight_min": -13.485382080078125,
      "activations/layer20_attention_weight_max": 32.015201568603516,
      "activations/layer20_attention_weight_min": -24.50612449645996,
      "activations/layer21_attention_weight_max": 32.54846954345703,
      "activations/layer21_attention_weight_min": -23.11357879638672,
      "activations/layer22_attention_weight_max": 58.23942947387695,
      "activations/layer22_attention_weight_min": -29.072927474975586,
      "activations/layer23_attention_weight_max": 43.92393493652344,
      "activations/layer23_attention_weight_min": -24.32540512084961,
      "activations/layer2_attention_weight_max": 31.7161922454834,
      "activations/layer2_attention_weight_min": -31.78693389892578,
      "activations/layer3_attention_weight_max": 92.03482055664062,
      "activations/layer3_attention_weight_min": -96.46015930175781,
      "activations/layer4_attention_weight_max": 121.10721588134766,
      "activations/layer4_attention_weight_min": -120.18019104003906,
      "activations/layer5_attention_weight_max": 59.155029296875,
      "activations/layer5_attention_weight_min": -62.18032455444336,
      "activations/layer6_attention_weight_max": 50.42255401611328,
      "activations/layer6_attention_weight_min": -52.84815216064453,
      "activations/layer7_attention_weight_max": 93.16217041015625,
      "activations/layer7_attention_weight_min": -91.43618774414062,
      "activations/layer8_attention_weight_max": 42.558319091796875,
      "activations/layer8_attention_weight_min": -41.151493072509766,
      "activations/layer9_attention_weight_max": 37.175479888916016,
      "activations/layer9_attention_weight_min": -34.912532806396484,
      "epoch": 8.29,
      "learning_rate": 9.754810606060605e-05,
      "loss": 2.8104,
      "step": 142600
    },
    {
      "activations/layer0_attention_weight_max": 15.89367389678955,
      "activations/layer0_attention_weight_min": -12.167831420898438,
      "activations/layer10_attention_weight_max": 38.06643295288086,
      "activations/layer10_attention_weight_min": -38.711997985839844,
      "activations/layer11_attention_weight_max": 35.906044006347656,
      "activations/layer11_attention_weight_min": -34.597747802734375,
      "activations/layer12_attention_weight_max": 27.796236038208008,
      "activations/layer12_attention_weight_min": -26.098224639892578,
      "activations/layer13_attention_weight_max": 53.55205535888672,
      "activations/layer13_attention_weight_min": -36.096492767333984,
      "activations/layer14_attention_weight_max": 62.48182678222656,
      "activations/layer14_attention_weight_min": -45.93917465209961,
      "activations/layer15_attention_weight_max": 42.645687103271484,
      "activations/layer15_attention_weight_min": -31.92155647277832,
      "activations/layer16_attention_weight_max": 44.3597412109375,
      "activations/layer16_attention_weight_min": -31.14532470703125,
      "activations/layer17_attention_weight_max": 40.77897262573242,
      "activations/layer17_attention_weight_min": -27.87973976135254,
      "activations/layer18_attention_weight_max": 36.571022033691406,
      "activations/layer18_attention_weight_min": -25.275907516479492,
      "activations/layer19_attention_weight_max": 47.8420295715332,
      "activations/layer19_attention_weight_min": -35.01296615600586,
      "activations/layer1_attention_weight_max": 17.452274322509766,
      "activations/layer1_attention_weight_min": -14.842745780944824,
      "activations/layer20_attention_weight_max": 36.992340087890625,
      "activations/layer20_attention_weight_min": -28.218242645263672,
      "activations/layer21_attention_weight_max": 32.624202728271484,
      "activations/layer21_attention_weight_min": -25.556005477905273,
      "activations/layer22_attention_weight_max": 52.39107131958008,
      "activations/layer22_attention_weight_min": -33.08728790283203,
      "activations/layer23_attention_weight_max": 42.632328033447266,
      "activations/layer23_attention_weight_min": -24.383651733398438,
      "activations/layer2_attention_weight_max": 36.75611114501953,
      "activations/layer2_attention_weight_min": -32.666175842285156,
      "activations/layer3_attention_weight_max": 100.88446807861328,
      "activations/layer3_attention_weight_min": -96.81977844238281,
      "activations/layer4_attention_weight_max": 129.32778930664062,
      "activations/layer4_attention_weight_min": -125.87147521972656,
      "activations/layer5_attention_weight_max": 61.539398193359375,
      "activations/layer5_attention_weight_min": -61.37822723388672,
      "activations/layer6_attention_weight_max": 53.60482406616211,
      "activations/layer6_attention_weight_min": -52.65846633911133,
      "activations/layer7_attention_weight_max": 101.20887756347656,
      "activations/layer7_attention_weight_min": -100.37267303466797,
      "activations/layer8_attention_weight_max": 46.27499771118164,
      "activations/layer8_attention_weight_min": -46.32923889160156,
      "activations/layer9_attention_weight_max": 42.764766693115234,
      "activations/layer9_attention_weight_min": -39.901832580566406,
      "epoch": 8.29,
      "learning_rate": 9.752916666666665e-05,
      "loss": 2.795,
      "step": 142650
    },
    {
      "activations/layer0_attention_weight_max": 16.5186767578125,
      "activations/layer0_attention_weight_min": -12.529409408569336,
      "activations/layer10_attention_weight_max": 30.75156593322754,
      "activations/layer10_attention_weight_min": -29.028366088867188,
      "activations/layer11_attention_weight_max": 29.409650802612305,
      "activations/layer11_attention_weight_min": -27.457822799682617,
      "activations/layer12_attention_weight_max": 22.222562789916992,
      "activations/layer12_attention_weight_min": -25.31008529663086,
      "activations/layer13_attention_weight_max": 39.79480743408203,
      "activations/layer13_attention_weight_min": -31.03091049194336,
      "activations/layer14_attention_weight_max": 53.835933685302734,
      "activations/layer14_attention_weight_min": -43.8416862487793,
      "activations/layer15_attention_weight_max": 31.958463668823242,
      "activations/layer15_attention_weight_min": -29.279531478881836,
      "activations/layer16_attention_weight_max": 31.60894775390625,
      "activations/layer16_attention_weight_min": -26.82387924194336,
      "activations/layer17_attention_weight_max": 31.378870010375977,
      "activations/layer17_attention_weight_min": -25.198266983032227,
      "activations/layer18_attention_weight_max": 33.125701904296875,
      "activations/layer18_attention_weight_min": -23.253604888916016,
      "activations/layer19_attention_weight_max": 40.473045349121094,
      "activations/layer19_attention_weight_min": -32.38117980957031,
      "activations/layer1_attention_weight_max": 16.709924697875977,
      "activations/layer1_attention_weight_min": -14.706201553344727,
      "activations/layer20_attention_weight_max": 27.772430419921875,
      "activations/layer20_attention_weight_min": -23.80170440673828,
      "activations/layer21_attention_weight_max": 28.68984603881836,
      "activations/layer21_attention_weight_min": -23.164775848388672,
      "activations/layer22_attention_weight_max": 42.9012336730957,
      "activations/layer22_attention_weight_min": -32.20033264160156,
      "activations/layer23_attention_weight_max": 35.690242767333984,
      "activations/layer23_attention_weight_min": -23.334754943847656,
      "activations/layer2_attention_weight_max": 31.630096435546875,
      "activations/layer2_attention_weight_min": -31.516637802124023,
      "activations/layer3_attention_weight_max": 90.5704574584961,
      "activations/layer3_attention_weight_min": -96.1899642944336,
      "activations/layer4_attention_weight_max": 114.89327239990234,
      "activations/layer4_attention_weight_min": -114.38387298583984,
      "activations/layer5_attention_weight_max": 55.46845626831055,
      "activations/layer5_attention_weight_min": -61.59636688232422,
      "activations/layer6_attention_weight_max": 49.07679748535156,
      "activations/layer6_attention_weight_min": -50.06839370727539,
      "activations/layer7_attention_weight_max": 90.23423767089844,
      "activations/layer7_attention_weight_min": -89.19981384277344,
      "activations/layer8_attention_weight_max": 39.44377517700195,
      "activations/layer8_attention_weight_min": -39.85175704956055,
      "activations/layer9_attention_weight_max": 32.7763671875,
      "activations/layer9_attention_weight_min": -32.033016204833984,
      "epoch": 8.29,
      "learning_rate": 9.751022727272727e-05,
      "loss": 2.8022,
      "step": 142700
    },
    {
      "activations/layer0_attention_weight_max": 15.80752944946289,
      "activations/layer0_attention_weight_min": -12.209482192993164,
      "activations/layer10_attention_weight_max": 30.759658813476562,
      "activations/layer10_attention_weight_min": -31.283769607543945,
      "activations/layer11_attention_weight_max": 30.694303512573242,
      "activations/layer11_attention_weight_min": -29.3886775970459,
      "activations/layer12_attention_weight_max": 22.835803985595703,
      "activations/layer12_attention_weight_min": -25.641681671142578,
      "activations/layer13_attention_weight_max": 42.06977462768555,
      "activations/layer13_attention_weight_min": -36.870567321777344,
      "activations/layer14_attention_weight_max": 50.93580627441406,
      "activations/layer14_attention_weight_min": -43.384307861328125,
      "activations/layer15_attention_weight_max": 34.06932830810547,
      "activations/layer15_attention_weight_min": -32.03120422363281,
      "activations/layer16_attention_weight_max": 30.752742767333984,
      "activations/layer16_attention_weight_min": -30.767711639404297,
      "activations/layer17_attention_weight_max": 34.764835357666016,
      "activations/layer17_attention_weight_min": -24.89693832397461,
      "activations/layer18_attention_weight_max": 30.894515991210938,
      "activations/layer18_attention_weight_min": -21.12299156188965,
      "activations/layer19_attention_weight_max": 43.81391525268555,
      "activations/layer19_attention_weight_min": -31.296295166015625,
      "activations/layer1_attention_weight_max": 16.85649299621582,
      "activations/layer1_attention_weight_min": -14.687602043151855,
      "activations/layer20_attention_weight_max": 29.892166137695312,
      "activations/layer20_attention_weight_min": -24.263853073120117,
      "activations/layer21_attention_weight_max": 29.052932739257812,
      "activations/layer21_attention_weight_min": -22.588516235351562,
      "activations/layer22_attention_weight_max": 47.269161224365234,
      "activations/layer22_attention_weight_min": -28.46023178100586,
      "activations/layer23_attention_weight_max": 36.30498504638672,
      "activations/layer23_attention_weight_min": -25.532621383666992,
      "activations/layer2_attention_weight_max": 30.79445457458496,
      "activations/layer2_attention_weight_min": -32.453033447265625,
      "activations/layer3_attention_weight_max": 91.79154968261719,
      "activations/layer3_attention_weight_min": -93.34541320800781,
      "activations/layer4_attention_weight_max": 121.3095703125,
      "activations/layer4_attention_weight_min": -118.9394302368164,
      "activations/layer5_attention_weight_max": 59.5146484375,
      "activations/layer5_attention_weight_min": -64.1687240600586,
      "activations/layer6_attention_weight_max": 49.17961883544922,
      "activations/layer6_attention_weight_min": -50.03461837768555,
      "activations/layer7_attention_weight_max": 86.79623413085938,
      "activations/layer7_attention_weight_min": -88.64327239990234,
      "activations/layer8_attention_weight_max": 37.9125862121582,
      "activations/layer8_attention_weight_min": -40.11635971069336,
      "activations/layer9_attention_weight_max": 33.26665496826172,
      "activations/layer9_attention_weight_min": -33.31735610961914,
      "epoch": 8.29,
      "learning_rate": 9.749128787878787e-05,
      "loss": 2.8086,
      "step": 142750
    },
    {
      "activations/layer0_attention_weight_max": 15.291491508483887,
      "activations/layer0_attention_weight_min": -12.925287246704102,
      "activations/layer10_attention_weight_max": 34.10629653930664,
      "activations/layer10_attention_weight_min": -32.02815628051758,
      "activations/layer11_attention_weight_max": 34.06367111206055,
      "activations/layer11_attention_weight_min": -29.862995147705078,
      "activations/layer12_attention_weight_max": 25.09217071533203,
      "activations/layer12_attention_weight_min": -31.83064079284668,
      "activations/layer13_attention_weight_max": 45.83735275268555,
      "activations/layer13_attention_weight_min": -34.159114837646484,
      "activations/layer14_attention_weight_max": 54.41421127319336,
      "activations/layer14_attention_weight_min": -42.9391975402832,
      "activations/layer15_attention_weight_max": 36.67805480957031,
      "activations/layer15_attention_weight_min": -30.34836769104004,
      "activations/layer16_attention_weight_max": 34.10382080078125,
      "activations/layer16_attention_weight_min": -27.300222396850586,
      "activations/layer17_attention_weight_max": 35.81917190551758,
      "activations/layer17_attention_weight_min": -25.31046485900879,
      "activations/layer18_attention_weight_max": 32.224430084228516,
      "activations/layer18_attention_weight_min": -22.299596786499023,
      "activations/layer19_attention_weight_max": 40.631675720214844,
      "activations/layer19_attention_weight_min": -29.923765182495117,
      "activations/layer1_attention_weight_max": 16.758779525756836,
      "activations/layer1_attention_weight_min": -14.490106582641602,
      "activations/layer20_attention_weight_max": 33.49822998046875,
      "activations/layer20_attention_weight_min": -26.232276916503906,
      "activations/layer21_attention_weight_max": 32.98887252807617,
      "activations/layer21_attention_weight_min": -23.464237213134766,
      "activations/layer22_attention_weight_max": 51.0119514465332,
      "activations/layer22_attention_weight_min": -28.89649200439453,
      "activations/layer23_attention_weight_max": 39.14275360107422,
      "activations/layer23_attention_weight_min": -23.202550888061523,
      "activations/layer2_attention_weight_max": 33.26872253417969,
      "activations/layer2_attention_weight_min": -34.92188262939453,
      "activations/layer3_attention_weight_max": 95.96126556396484,
      "activations/layer3_attention_weight_min": -96.84627532958984,
      "activations/layer4_attention_weight_max": 123.91948699951172,
      "activations/layer4_attention_weight_min": -126.18153381347656,
      "activations/layer5_attention_weight_max": 57.76877975463867,
      "activations/layer5_attention_weight_min": -63.836463928222656,
      "activations/layer6_attention_weight_max": 49.63389587402344,
      "activations/layer6_attention_weight_min": -53.399288177490234,
      "activations/layer7_attention_weight_max": 90.13553619384766,
      "activations/layer7_attention_weight_min": -90.98854064941406,
      "activations/layer8_attention_weight_max": 41.9398078918457,
      "activations/layer8_attention_weight_min": -42.08977127075195,
      "activations/layer9_attention_weight_max": 36.010009765625,
      "activations/layer9_attention_weight_min": -33.812232971191406,
      "epoch": 8.3,
      "learning_rate": 9.747234848484847e-05,
      "loss": 2.803,
      "step": 142800
    },
    {
      "activations/layer0_attention_weight_max": 16.203147888183594,
      "activations/layer0_attention_weight_min": -12.633830070495605,
      "activations/layer10_attention_weight_max": 32.568363189697266,
      "activations/layer10_attention_weight_min": -33.32394790649414,
      "activations/layer11_attention_weight_max": 32.58118438720703,
      "activations/layer11_attention_weight_min": -30.836273193359375,
      "activations/layer12_attention_weight_max": 40.90827178955078,
      "activations/layer12_attention_weight_min": -25.292245864868164,
      "activations/layer13_attention_weight_max": 44.640560150146484,
      "activations/layer13_attention_weight_min": -35.62321090698242,
      "activations/layer14_attention_weight_max": 53.32147979736328,
      "activations/layer14_attention_weight_min": -41.29378890991211,
      "activations/layer15_attention_weight_max": 40.03065490722656,
      "activations/layer15_attention_weight_min": -32.61638259887695,
      "activations/layer16_attention_weight_max": 37.493125915527344,
      "activations/layer16_attention_weight_min": -28.488340377807617,
      "activations/layer17_attention_weight_max": 35.79844284057617,
      "activations/layer17_attention_weight_min": -24.830684661865234,
      "activations/layer18_attention_weight_max": 32.85834503173828,
      "activations/layer18_attention_weight_min": -24.424285888671875,
      "activations/layer19_attention_weight_max": 42.07411193847656,
      "activations/layer19_attention_weight_min": -30.36785316467285,
      "activations/layer1_attention_weight_max": 17.720827102661133,
      "activations/layer1_attention_weight_min": -15.782200813293457,
      "activations/layer20_attention_weight_max": 35.410545349121094,
      "activations/layer20_attention_weight_min": -25.419700622558594,
      "activations/layer21_attention_weight_max": 32.30378341674805,
      "activations/layer21_attention_weight_min": -23.6529541015625,
      "activations/layer22_attention_weight_max": 54.89734649658203,
      "activations/layer22_attention_weight_min": -26.722043991088867,
      "activations/layer23_attention_weight_max": 39.731082916259766,
      "activations/layer23_attention_weight_min": -23.113040924072266,
      "activations/layer2_attention_weight_max": 35.19385528564453,
      "activations/layer2_attention_weight_min": -34.98497009277344,
      "activations/layer3_attention_weight_max": 97.90379333496094,
      "activations/layer3_attention_weight_min": -100.32874298095703,
      "activations/layer4_attention_weight_max": 125.39833068847656,
      "activations/layer4_attention_weight_min": -123.46624755859375,
      "activations/layer5_attention_weight_max": 57.45915985107422,
      "activations/layer5_attention_weight_min": -64.41108703613281,
      "activations/layer6_attention_weight_max": 50.42267990112305,
      "activations/layer6_attention_weight_min": -55.67677307128906,
      "activations/layer7_attention_weight_max": 92.11333465576172,
      "activations/layer7_attention_weight_min": -93.66952514648438,
      "activations/layer8_attention_weight_max": 39.56796646118164,
      "activations/layer8_attention_weight_min": -41.39360427856445,
      "activations/layer9_attention_weight_max": 36.177425384521484,
      "activations/layer9_attention_weight_min": -34.436946868896484,
      "epoch": 8.3,
      "learning_rate": 9.745340909090909e-05,
      "loss": 2.8003,
      "step": 142850
    },
    {
      "activations/layer0_attention_weight_max": 16.379716873168945,
      "activations/layer0_attention_weight_min": -12.437024116516113,
      "activations/layer10_attention_weight_max": 37.06361770629883,
      "activations/layer10_attention_weight_min": -32.272308349609375,
      "activations/layer11_attention_weight_max": 34.80079650878906,
      "activations/layer11_attention_weight_min": -30.02875518798828,
      "activations/layer12_attention_weight_max": 24.943479537963867,
      "activations/layer12_attention_weight_min": -23.915800094604492,
      "activations/layer13_attention_weight_max": 43.4764518737793,
      "activations/layer13_attention_weight_min": -32.3181266784668,
      "activations/layer14_attention_weight_max": 47.450401306152344,
      "activations/layer14_attention_weight_min": -41.21784210205078,
      "activations/layer15_attention_weight_max": 39.30750274658203,
      "activations/layer15_attention_weight_min": -29.49778175354004,
      "activations/layer16_attention_weight_max": 32.05315017700195,
      "activations/layer16_attention_weight_min": -27.725717544555664,
      "activations/layer17_attention_weight_max": 35.691219329833984,
      "activations/layer17_attention_weight_min": -25.551673889160156,
      "activations/layer18_attention_weight_max": 33.40623474121094,
      "activations/layer18_attention_weight_min": -24.868087768554688,
      "activations/layer19_attention_weight_max": 34.77828598022461,
      "activations/layer19_attention_weight_min": -30.994338989257812,
      "activations/layer1_attention_weight_max": 17.796466827392578,
      "activations/layer1_attention_weight_min": -15.692241668701172,
      "activations/layer20_attention_weight_max": 33.13737869262695,
      "activations/layer20_attention_weight_min": -26.21630859375,
      "activations/layer21_attention_weight_max": 28.81627082824707,
      "activations/layer21_attention_weight_min": -23.405441284179688,
      "activations/layer22_attention_weight_max": 59.77219009399414,
      "activations/layer22_attention_weight_min": -31.26850700378418,
      "activations/layer23_attention_weight_max": 35.867706298828125,
      "activations/layer23_attention_weight_min": -22.65057373046875,
      "activations/layer2_attention_weight_max": 32.721839904785156,
      "activations/layer2_attention_weight_min": -31.774879455566406,
      "activations/layer3_attention_weight_max": 91.24646759033203,
      "activations/layer3_attention_weight_min": -92.97892761230469,
      "activations/layer4_attention_weight_max": 121.53508758544922,
      "activations/layer4_attention_weight_min": -117.17986297607422,
      "activations/layer5_attention_weight_max": 57.90064239501953,
      "activations/layer5_attention_weight_min": -63.94053268432617,
      "activations/layer6_attention_weight_max": 50.74443054199219,
      "activations/layer6_attention_weight_min": -54.7507209777832,
      "activations/layer7_attention_weight_max": 90.1404037475586,
      "activations/layer7_attention_weight_min": -91.6222915649414,
      "activations/layer8_attention_weight_max": 43.68757629394531,
      "activations/layer8_attention_weight_min": -43.19514846801758,
      "activations/layer9_attention_weight_max": 36.04608154296875,
      "activations/layer9_attention_weight_min": -33.53676986694336,
      "epoch": 8.3,
      "learning_rate": 9.743446969696968e-05,
      "loss": 2.79,
      "step": 142900
    },
    {
      "activations/layer0_attention_weight_max": 16.62745475769043,
      "activations/layer0_attention_weight_min": -12.301806449890137,
      "activations/layer10_attention_weight_max": 30.91069793701172,
      "activations/layer10_attention_weight_min": -29.459367752075195,
      "activations/layer11_attention_weight_max": 30.63216209411621,
      "activations/layer11_attention_weight_min": -28.095304489135742,
      "activations/layer12_attention_weight_max": 22.890419006347656,
      "activations/layer12_attention_weight_min": -22.759750366210938,
      "activations/layer13_attention_weight_max": 35.66310501098633,
      "activations/layer13_attention_weight_min": -31.15182876586914,
      "activations/layer14_attention_weight_max": 46.585548400878906,
      "activations/layer14_attention_weight_min": -36.951473236083984,
      "activations/layer15_attention_weight_max": 35.27928924560547,
      "activations/layer15_attention_weight_min": -28.916471481323242,
      "activations/layer16_attention_weight_max": 32.487342834472656,
      "activations/layer16_attention_weight_min": -26.789888381958008,
      "activations/layer17_attention_weight_max": 34.9531135559082,
      "activations/layer17_attention_weight_min": -25.517065048217773,
      "activations/layer18_attention_weight_max": 32.939666748046875,
      "activations/layer18_attention_weight_min": -21.801305770874023,
      "activations/layer19_attention_weight_max": 34.986915588378906,
      "activations/layer19_attention_weight_min": -30.303924560546875,
      "activations/layer1_attention_weight_max": 15.94119930267334,
      "activations/layer1_attention_weight_min": -14.369543075561523,
      "activations/layer20_attention_weight_max": 27.894445419311523,
      "activations/layer20_attention_weight_min": -23.274127960205078,
      "activations/layer21_attention_weight_max": 31.390165328979492,
      "activations/layer21_attention_weight_min": -23.99314308166504,
      "activations/layer22_attention_weight_max": 44.19065856933594,
      "activations/layer22_attention_weight_min": -30.355819702148438,
      "activations/layer23_attention_weight_max": 35.647159576416016,
      "activations/layer23_attention_weight_min": -25.22170066833496,
      "activations/layer2_attention_weight_max": 32.623077392578125,
      "activations/layer2_attention_weight_min": -31.776813507080078,
      "activations/layer3_attention_weight_max": 94.4432601928711,
      "activations/layer3_attention_weight_min": -94.11293029785156,
      "activations/layer4_attention_weight_max": 120.22918701171875,
      "activations/layer4_attention_weight_min": -118.00200653076172,
      "activations/layer5_attention_weight_max": 56.91817855834961,
      "activations/layer5_attention_weight_min": -60.20982360839844,
      "activations/layer6_attention_weight_max": 48.02095413208008,
      "activations/layer6_attention_weight_min": -50.56180953979492,
      "activations/layer7_attention_weight_max": 88.0351791381836,
      "activations/layer7_attention_weight_min": -85.22067260742188,
      "activations/layer8_attention_weight_max": 37.41246795654297,
      "activations/layer8_attention_weight_min": -40.63691711425781,
      "activations/layer9_attention_weight_max": 31.679920196533203,
      "activations/layer9_attention_weight_min": -30.819522857666016,
      "epoch": 8.31,
      "learning_rate": 9.741553030303029e-05,
      "loss": 2.8103,
      "step": 142950
    },
    {
      "activations/layer0_attention_weight_max": 16.069644927978516,
      "activations/layer0_attention_weight_min": -11.704277038574219,
      "activations/layer10_attention_weight_max": 41.491973876953125,
      "activations/layer10_attention_weight_min": -38.936363220214844,
      "activations/layer11_attention_weight_max": 44.12705612182617,
      "activations/layer11_attention_weight_min": -40.13887023925781,
      "activations/layer12_attention_weight_max": 26.439279556274414,
      "activations/layer12_attention_weight_min": -28.209680557250977,
      "activations/layer13_attention_weight_max": 42.020050048828125,
      "activations/layer13_attention_weight_min": -36.96644592285156,
      "activations/layer14_attention_weight_max": 52.899044036865234,
      "activations/layer14_attention_weight_min": -44.624656677246094,
      "activations/layer15_attention_weight_max": 37.243194580078125,
      "activations/layer15_attention_weight_min": -30.668848037719727,
      "activations/layer16_attention_weight_max": 32.59450912475586,
      "activations/layer16_attention_weight_min": -27.270030975341797,
      "activations/layer17_attention_weight_max": 37.22658157348633,
      "activations/layer17_attention_weight_min": -27.2908992767334,
      "activations/layer18_attention_weight_max": 35.38470458984375,
      "activations/layer18_attention_weight_min": -22.326353073120117,
      "activations/layer19_attention_weight_max": 37.76447296142578,
      "activations/layer19_attention_weight_min": -29.524860382080078,
      "activations/layer1_attention_weight_max": 16.84718894958496,
      "activations/layer1_attention_weight_min": -14.009811401367188,
      "activations/layer20_attention_weight_max": 35.82042694091797,
      "activations/layer20_attention_weight_min": -24.91622543334961,
      "activations/layer21_attention_weight_max": 35.235496520996094,
      "activations/layer21_attention_weight_min": -23.276470184326172,
      "activations/layer22_attention_weight_max": 46.70909118652344,
      "activations/layer22_attention_weight_min": -28.182188034057617,
      "activations/layer23_attention_weight_max": 36.64114761352539,
      "activations/layer23_attention_weight_min": -21.561750411987305,
      "activations/layer2_attention_weight_max": 32.165374755859375,
      "activations/layer2_attention_weight_min": -32.30268859863281,
      "activations/layer3_attention_weight_max": 96.0590591430664,
      "activations/layer3_attention_weight_min": -95.85043334960938,
      "activations/layer4_attention_weight_max": 120.67597961425781,
      "activations/layer4_attention_weight_min": -119.9749526977539,
      "activations/layer5_attention_weight_max": 59.38589096069336,
      "activations/layer5_attention_weight_min": -63.4109001159668,
      "activations/layer6_attention_weight_max": 51.81381607055664,
      "activations/layer6_attention_weight_min": -50.00930404663086,
      "activations/layer7_attention_weight_max": 107.33267974853516,
      "activations/layer7_attention_weight_min": -90.78044891357422,
      "activations/layer8_attention_weight_max": 48.239715576171875,
      "activations/layer8_attention_weight_min": -44.702972412109375,
      "activations/layer9_attention_weight_max": 43.038089752197266,
      "activations/layer9_attention_weight_min": -41.5168342590332,
      "epoch": 8.31,
      "learning_rate": 9.73965909090909e-05,
      "loss": 2.8076,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_loss": 2.7578125,
      "eval_runtime": 8.5577,
      "eval_samples_per_second": 501.772,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_openwebtext_loss": 2.7578125,
      "eval_openwebtext_ppl": 15.765318566620541,
      "eval_openwebtext_runtime": 8.5577,
      "eval_openwebtext_samples_per_second": 501.772,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0084,
      "eval_wikitext_samples_per_second": 227.046,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_lambada_loss": 2.525390625,
      "eval_lambada_ppl": 12.495775472672934,
      "eval_lambada_runtime": 9.6228,
      "eval_lambada_samples_per_second": 505.986,
      "step": 143000
    },
    {
      "activations/layer0_attention_weight_max": 15.42225456237793,
      "activations/layer0_attention_weight_min": -12.435895919799805,
      "activations/layer10_attention_weight_max": 41.33636474609375,
      "activations/layer10_attention_weight_min": -33.97998046875,
      "activations/layer11_attention_weight_max": 41.3323860168457,
      "activations/layer11_attention_weight_min": -33.213504791259766,
      "activations/layer12_attention_weight_max": 29.8767032623291,
      "activations/layer12_attention_weight_min": -31.82953453063965,
      "activations/layer13_attention_weight_max": 53.2066535949707,
      "activations/layer13_attention_weight_min": -33.98197555541992,
      "activations/layer14_attention_weight_max": 64.89808654785156,
      "activations/layer14_attention_weight_min": -40.857765197753906,
      "activations/layer15_attention_weight_max": 50.3912467956543,
      "activations/layer15_attention_weight_min": -30.23001480102539,
      "activations/layer16_attention_weight_max": 44.79179000854492,
      "activations/layer16_attention_weight_min": -27.400146484375,
      "activations/layer17_attention_weight_max": 34.49758529663086,
      "activations/layer17_attention_weight_min": -24.693918228149414,
      "activations/layer18_attention_weight_max": 33.225521087646484,
      "activations/layer18_attention_weight_min": -22.67889404296875,
      "activations/layer19_attention_weight_max": 51.4961051940918,
      "activations/layer19_attention_weight_min": -29.950170516967773,
      "activations/layer1_attention_weight_max": 16.89780044555664,
      "activations/layer1_attention_weight_min": -14.245723724365234,
      "activations/layer20_attention_weight_max": 33.25790023803711,
      "activations/layer20_attention_weight_min": -23.14887046813965,
      "activations/layer21_attention_weight_max": 28.528194427490234,
      "activations/layer21_attention_weight_min": -22.745410919189453,
      "activations/layer22_attention_weight_max": 47.27708435058594,
      "activations/layer22_attention_weight_min": -29.42981719970703,
      "activations/layer23_attention_weight_max": 43.82457733154297,
      "activations/layer23_attention_weight_min": -24.22270965576172,
      "activations/layer2_attention_weight_max": 35.47321319580078,
      "activations/layer2_attention_weight_min": -33.646820068359375,
      "activations/layer3_attention_weight_max": 99.73139190673828,
      "activations/layer3_attention_weight_min": -102.64545440673828,
      "activations/layer4_attention_weight_max": 129.5065460205078,
      "activations/layer4_attention_weight_min": -125.98506927490234,
      "activations/layer5_attention_weight_max": 60.85366439819336,
      "activations/layer5_attention_weight_min": -62.27924346923828,
      "activations/layer6_attention_weight_max": 50.26063919067383,
      "activations/layer6_attention_weight_min": -52.6430549621582,
      "activations/layer7_attention_weight_max": 91.88960266113281,
      "activations/layer7_attention_weight_min": -89.25110626220703,
      "activations/layer8_attention_weight_max": 44.68056106567383,
      "activations/layer8_attention_weight_min": -42.275733947753906,
      "activations/layer9_attention_weight_max": 38.8146858215332,
      "activations/layer9_attention_weight_min": -34.94418716430664,
      "epoch": 8.31,
      "learning_rate": 9.737765151515151e-05,
      "loss": 2.7952,
      "step": 143050
    },
    {
      "activations/layer0_attention_weight_max": 16.05579948425293,
      "activations/layer0_attention_weight_min": -12.392865180969238,
      "activations/layer10_attention_weight_max": 36.843292236328125,
      "activations/layer10_attention_weight_min": -33.066951751708984,
      "activations/layer11_attention_weight_max": 36.94438171386719,
      "activations/layer11_attention_weight_min": -34.13822937011719,
      "activations/layer12_attention_weight_max": 27.280746459960938,
      "activations/layer12_attention_weight_min": -28.388212203979492,
      "activations/layer13_attention_weight_max": 51.163047790527344,
      "activations/layer13_attention_weight_min": -36.0140266418457,
      "activations/layer14_attention_weight_max": 50.640174865722656,
      "activations/layer14_attention_weight_min": -42.842384338378906,
      "activations/layer15_attention_weight_max": 48.55572509765625,
      "activations/layer15_attention_weight_min": -37.906768798828125,
      "activations/layer16_attention_weight_max": 33.65763473510742,
      "activations/layer16_attention_weight_min": -28.174890518188477,
      "activations/layer17_attention_weight_max": 37.29264831542969,
      "activations/layer17_attention_weight_min": -26.97062110900879,
      "activations/layer18_attention_weight_max": 33.50014114379883,
      "activations/layer18_attention_weight_min": -22.793411254882812,
      "activations/layer19_attention_weight_max": 37.19533920288086,
      "activations/layer19_attention_weight_min": -29.252988815307617,
      "activations/layer1_attention_weight_max": 17.557466506958008,
      "activations/layer1_attention_weight_min": -14.057859420776367,
      "activations/layer20_attention_weight_max": 31.48308563232422,
      "activations/layer20_attention_weight_min": -22.95532989501953,
      "activations/layer21_attention_weight_max": 30.74385643005371,
      "activations/layer21_attention_weight_min": -23.1156063079834,
      "activations/layer22_attention_weight_max": 52.1916618347168,
      "activations/layer22_attention_weight_min": -28.34568214416504,
      "activations/layer23_attention_weight_max": 38.28468322753906,
      "activations/layer23_attention_weight_min": -25.945335388183594,
      "activations/layer2_attention_weight_max": 33.065162658691406,
      "activations/layer2_attention_weight_min": -33.194602966308594,
      "activations/layer3_attention_weight_max": 95.76710510253906,
      "activations/layer3_attention_weight_min": -97.89900207519531,
      "activations/layer4_attention_weight_max": 124.96046447753906,
      "activations/layer4_attention_weight_min": -119.58524322509766,
      "activations/layer5_attention_weight_max": 58.967247009277344,
      "activations/layer5_attention_weight_min": -61.65388107299805,
      "activations/layer6_attention_weight_max": 52.825416564941406,
      "activations/layer6_attention_weight_min": -52.62936782836914,
      "activations/layer7_attention_weight_max": 96.52738189697266,
      "activations/layer7_attention_weight_min": -97.25696563720703,
      "activations/layer8_attention_weight_max": 45.64799880981445,
      "activations/layer8_attention_weight_min": -42.98085403442383,
      "activations/layer9_attention_weight_max": 37.67214584350586,
      "activations/layer9_attention_weight_min": -37.956539154052734,
      "epoch": 8.31,
      "learning_rate": 9.735871212121211e-05,
      "loss": 2.8009,
      "step": 143100
    },
    {
      "activations/layer0_attention_weight_max": 16.556106567382812,
      "activations/layer0_attention_weight_min": -11.61602783203125,
      "activations/layer10_attention_weight_max": 33.619258880615234,
      "activations/layer10_attention_weight_min": -32.80164337158203,
      "activations/layer11_attention_weight_max": 33.024356842041016,
      "activations/layer11_attention_weight_min": -29.87928581237793,
      "activations/layer12_attention_weight_max": 29.442401885986328,
      "activations/layer12_attention_weight_min": -25.680261611938477,
      "activations/layer13_attention_weight_max": 39.70805358886719,
      "activations/layer13_attention_weight_min": -33.11750411987305,
      "activations/layer14_attention_weight_max": 56.647865295410156,
      "activations/layer14_attention_weight_min": -42.420597076416016,
      "activations/layer15_attention_weight_max": 39.74174118041992,
      "activations/layer15_attention_weight_min": -33.36198425292969,
      "activations/layer16_attention_weight_max": 34.752471923828125,
      "activations/layer16_attention_weight_min": -29.290983200073242,
      "activations/layer17_attention_weight_max": 37.992034912109375,
      "activations/layer17_attention_weight_min": -26.696889877319336,
      "activations/layer18_attention_weight_max": 42.04098892211914,
      "activations/layer18_attention_weight_min": -25.266193389892578,
      "activations/layer19_attention_weight_max": 41.30805587768555,
      "activations/layer19_attention_weight_min": -30.476316452026367,
      "activations/layer1_attention_weight_max": 16.424232482910156,
      "activations/layer1_attention_weight_min": -14.069706916809082,
      "activations/layer20_attention_weight_max": 34.15926742553711,
      "activations/layer20_attention_weight_min": -24.3995418548584,
      "activations/layer21_attention_weight_max": 40.0257453918457,
      "activations/layer21_attention_weight_min": -22.78318214416504,
      "activations/layer22_attention_weight_max": 52.126949310302734,
      "activations/layer22_attention_weight_min": -29.867795944213867,
      "activations/layer23_attention_weight_max": 35.92076873779297,
      "activations/layer23_attention_weight_min": -25.22334098815918,
      "activations/layer2_attention_weight_max": 33.396263122558594,
      "activations/layer2_attention_weight_min": -33.49040222167969,
      "activations/layer3_attention_weight_max": 92.493408203125,
      "activations/layer3_attention_weight_min": -95.01118469238281,
      "activations/layer4_attention_weight_max": 125.80525207519531,
      "activations/layer4_attention_weight_min": -121.0501937866211,
      "activations/layer5_attention_weight_max": 58.7713737487793,
      "activations/layer5_attention_weight_min": -62.96647644042969,
      "activations/layer6_attention_weight_max": 50.533058166503906,
      "activations/layer6_attention_weight_min": -52.50832748413086,
      "activations/layer7_attention_weight_max": 88.47176361083984,
      "activations/layer7_attention_weight_min": -86.66157531738281,
      "activations/layer8_attention_weight_max": 40.16748809814453,
      "activations/layer8_attention_weight_min": -40.034759521484375,
      "activations/layer9_attention_weight_max": 34.36438751220703,
      "activations/layer9_attention_weight_min": -33.769657135009766,
      "epoch": 8.32,
      "learning_rate": 9.733977272727272e-05,
      "loss": 2.8288,
      "step": 143150
    },
    {
      "activations/layer0_attention_weight_max": 15.875602722167969,
      "activations/layer0_attention_weight_min": -12.392951965332031,
      "activations/layer10_attention_weight_max": 32.95832061767578,
      "activations/layer10_attention_weight_min": -33.64162063598633,
      "activations/layer11_attention_weight_max": 34.02828598022461,
      "activations/layer11_attention_weight_min": -32.150943756103516,
      "activations/layer12_attention_weight_max": 35.75321578979492,
      "activations/layer12_attention_weight_min": -26.420690536499023,
      "activations/layer13_attention_weight_max": 40.793609619140625,
      "activations/layer13_attention_weight_min": -33.04585266113281,
      "activations/layer14_attention_weight_max": 49.857818603515625,
      "activations/layer14_attention_weight_min": -40.68631362915039,
      "activations/layer15_attention_weight_max": 39.014686584472656,
      "activations/layer15_attention_weight_min": -30.956674575805664,
      "activations/layer16_attention_weight_max": 39.439300537109375,
      "activations/layer16_attention_weight_min": -26.86430549621582,
      "activations/layer17_attention_weight_max": 34.368412017822266,
      "activations/layer17_attention_weight_min": -25.54615592956543,
      "activations/layer18_attention_weight_max": 33.54414367675781,
      "activations/layer18_attention_weight_min": -21.726144790649414,
      "activations/layer19_attention_weight_max": 36.41445541381836,
      "activations/layer19_attention_weight_min": -31.45083999633789,
      "activations/layer1_attention_weight_max": 16.18299102783203,
      "activations/layer1_attention_weight_min": -13.25031566619873,
      "activations/layer20_attention_weight_max": 31.0816707611084,
      "activations/layer20_attention_weight_min": -24.26420021057129,
      "activations/layer21_attention_weight_max": 29.74294662475586,
      "activations/layer21_attention_weight_min": -23.246318817138672,
      "activations/layer22_attention_weight_max": 53.81966018676758,
      "activations/layer22_attention_weight_min": -29.71296501159668,
      "activations/layer23_attention_weight_max": 38.448272705078125,
      "activations/layer23_attention_weight_min": -26.614688873291016,
      "activations/layer2_attention_weight_max": 32.18549346923828,
      "activations/layer2_attention_weight_min": -32.18415069580078,
      "activations/layer3_attention_weight_max": 92.6670150756836,
      "activations/layer3_attention_weight_min": -89.49784088134766,
      "activations/layer4_attention_weight_max": 119.09596252441406,
      "activations/layer4_attention_weight_min": -116.8426284790039,
      "activations/layer5_attention_weight_max": 57.89360809326172,
      "activations/layer5_attention_weight_min": -60.539344787597656,
      "activations/layer6_attention_weight_max": 46.78243637084961,
      "activations/layer6_attention_weight_min": -50.78585433959961,
      "activations/layer7_attention_weight_max": 91.68807983398438,
      "activations/layer7_attention_weight_min": -90.9783706665039,
      "activations/layer8_attention_weight_max": 40.464935302734375,
      "activations/layer8_attention_weight_min": -40.9677734375,
      "activations/layer9_attention_weight_max": 36.54875183105469,
      "activations/layer9_attention_weight_min": -34.45884323120117,
      "epoch": 8.32,
      "learning_rate": 9.732083333333332e-05,
      "loss": 2.7985,
      "step": 143200
    },
    {
      "activations/layer0_attention_weight_max": 16.577856063842773,
      "activations/layer0_attention_weight_min": -12.443086624145508,
      "activations/layer10_attention_weight_max": 34.71078109741211,
      "activations/layer10_attention_weight_min": -31.41947364807129,
      "activations/layer11_attention_weight_max": 31.63131332397461,
      "activations/layer11_attention_weight_min": -31.698728561401367,
      "activations/layer12_attention_weight_max": 22.42414665222168,
      "activations/layer12_attention_weight_min": -30.495820999145508,
      "activations/layer13_attention_weight_max": 42.285308837890625,
      "activations/layer13_attention_weight_min": -35.22566223144531,
      "activations/layer14_attention_weight_max": 55.11658477783203,
      "activations/layer14_attention_weight_min": -44.24559020996094,
      "activations/layer15_attention_weight_max": 37.49556350708008,
      "activations/layer15_attention_weight_min": -31.635709762573242,
      "activations/layer16_attention_weight_max": 31.768648147583008,
      "activations/layer16_attention_weight_min": -28.125873565673828,
      "activations/layer17_attention_weight_max": 36.15424346923828,
      "activations/layer17_attention_weight_min": -25.74212074279785,
      "activations/layer18_attention_weight_max": 32.37437057495117,
      "activations/layer18_attention_weight_min": -22.797161102294922,
      "activations/layer19_attention_weight_max": 35.86312484741211,
      "activations/layer19_attention_weight_min": -31.452926635742188,
      "activations/layer1_attention_weight_max": 17.099864959716797,
      "activations/layer1_attention_weight_min": -14.90205192565918,
      "activations/layer20_attention_weight_max": 31.470420837402344,
      "activations/layer20_attention_weight_min": -26.689746856689453,
      "activations/layer21_attention_weight_max": 30.239553451538086,
      "activations/layer21_attention_weight_min": -22.90595054626465,
      "activations/layer22_attention_weight_max": 47.55374526977539,
      "activations/layer22_attention_weight_min": -30.97528839111328,
      "activations/layer23_attention_weight_max": 40.24870300292969,
      "activations/layer23_attention_weight_min": -25.851806640625,
      "activations/layer2_attention_weight_max": 32.593406677246094,
      "activations/layer2_attention_weight_min": -32.6261100769043,
      "activations/layer3_attention_weight_max": 91.8786392211914,
      "activations/layer3_attention_weight_min": -96.5655288696289,
      "activations/layer4_attention_weight_max": 119.75323486328125,
      "activations/layer4_attention_weight_min": -120.3028793334961,
      "activations/layer5_attention_weight_max": 61.010772705078125,
      "activations/layer5_attention_weight_min": -66.25723266601562,
      "activations/layer6_attention_weight_max": 55.259178161621094,
      "activations/layer6_attention_weight_min": -52.55249786376953,
      "activations/layer7_attention_weight_max": 90.72799682617188,
      "activations/layer7_attention_weight_min": -90.72638702392578,
      "activations/layer8_attention_weight_max": 41.96101379394531,
      "activations/layer8_attention_weight_min": -42.76034927368164,
      "activations/layer9_attention_weight_max": 36.678321838378906,
      "activations/layer9_attention_weight_min": -34.357887268066406,
      "epoch": 8.32,
      "learning_rate": 9.730189393939393e-05,
      "loss": 2.7957,
      "step": 143250
    },
    {
      "activations/layer0_attention_weight_max": 16.02652931213379,
      "activations/layer0_attention_weight_min": -12.634544372558594,
      "activations/layer10_attention_weight_max": 31.779691696166992,
      "activations/layer10_attention_weight_min": -30.309133529663086,
      "activations/layer11_attention_weight_max": 32.49016571044922,
      "activations/layer11_attention_weight_min": -30.620643615722656,
      "activations/layer12_attention_weight_max": 22.952545166015625,
      "activations/layer12_attention_weight_min": -25.89185905456543,
      "activations/layer13_attention_weight_max": 36.868377685546875,
      "activations/layer13_attention_weight_min": -31.330602645874023,
      "activations/layer14_attention_weight_max": 41.329341888427734,
      "activations/layer14_attention_weight_min": -36.49641799926758,
      "activations/layer15_attention_weight_max": 32.882652282714844,
      "activations/layer15_attention_weight_min": -28.57000160217285,
      "activations/layer16_attention_weight_max": 31.70949363708496,
      "activations/layer16_attention_weight_min": -26.437763214111328,
      "activations/layer17_attention_weight_max": 32.4887580871582,
      "activations/layer17_attention_weight_min": -24.21662712097168,
      "activations/layer18_attention_weight_max": 31.45869255065918,
      "activations/layer18_attention_weight_min": -21.05799674987793,
      "activations/layer19_attention_weight_max": 36.269981384277344,
      "activations/layer19_attention_weight_min": -30.3933162689209,
      "activations/layer1_attention_weight_max": 16.86176109313965,
      "activations/layer1_attention_weight_min": -13.718522071838379,
      "activations/layer20_attention_weight_max": 30.313194274902344,
      "activations/layer20_attention_weight_min": -23.689634323120117,
      "activations/layer21_attention_weight_max": 30.068620681762695,
      "activations/layer21_attention_weight_min": -21.80101203918457,
      "activations/layer22_attention_weight_max": 47.264644622802734,
      "activations/layer22_attention_weight_min": -28.576881408691406,
      "activations/layer23_attention_weight_max": 35.53634262084961,
      "activations/layer23_attention_weight_min": -25.401844024658203,
      "activations/layer2_attention_weight_max": 29.864635467529297,
      "activations/layer2_attention_weight_min": -30.498703002929688,
      "activations/layer3_attention_weight_max": 89.49767303466797,
      "activations/layer3_attention_weight_min": -94.0186538696289,
      "activations/layer4_attention_weight_max": 114.63836669921875,
      "activations/layer4_attention_weight_min": -116.35107421875,
      "activations/layer5_attention_weight_max": 53.68566131591797,
      "activations/layer5_attention_weight_min": -63.45439910888672,
      "activations/layer6_attention_weight_max": 45.78029251098633,
      "activations/layer6_attention_weight_min": -50.11957550048828,
      "activations/layer7_attention_weight_max": 81.8464126586914,
      "activations/layer7_attention_weight_min": -88.43072509765625,
      "activations/layer8_attention_weight_max": 38.14012908935547,
      "activations/layer8_attention_weight_min": -40.50394821166992,
      "activations/layer9_attention_weight_max": 32.210506439208984,
      "activations/layer9_attention_weight_min": -31.351743698120117,
      "epoch": 8.33,
      "learning_rate": 9.728295454545454e-05,
      "loss": 2.8164,
      "step": 143300
    },
    {
      "activations/layer0_attention_weight_max": 15.82414436340332,
      "activations/layer0_attention_weight_min": -11.614471435546875,
      "activations/layer10_attention_weight_max": 29.989364624023438,
      "activations/layer10_attention_weight_min": -28.305477142333984,
      "activations/layer11_attention_weight_max": 29.9423885345459,
      "activations/layer11_attention_weight_min": -28.164100646972656,
      "activations/layer12_attention_weight_max": 27.823091506958008,
      "activations/layer12_attention_weight_min": -26.17290687561035,
      "activations/layer13_attention_weight_max": 40.58211135864258,
      "activations/layer13_attention_weight_min": -33.054832458496094,
      "activations/layer14_attention_weight_max": 49.29764938354492,
      "activations/layer14_attention_weight_min": -42.75443649291992,
      "activations/layer15_attention_weight_max": 41.0379524230957,
      "activations/layer15_attention_weight_min": -31.115690231323242,
      "activations/layer16_attention_weight_max": 33.561119079589844,
      "activations/layer16_attention_weight_min": -29.320402145385742,
      "activations/layer17_attention_weight_max": 36.23593521118164,
      "activations/layer17_attention_weight_min": -24.71470069885254,
      "activations/layer18_attention_weight_max": 32.15625762939453,
      "activations/layer18_attention_weight_min": -23.19228172302246,
      "activations/layer19_attention_weight_max": 38.609405517578125,
      "activations/layer19_attention_weight_min": -29.3759708404541,
      "activations/layer1_attention_weight_max": 17.42593002319336,
      "activations/layer1_attention_weight_min": -12.670831680297852,
      "activations/layer20_attention_weight_max": 28.804174423217773,
      "activations/layer20_attention_weight_min": -24.538732528686523,
      "activations/layer21_attention_weight_max": 29.681854248046875,
      "activations/layer21_attention_weight_min": -22.254077911376953,
      "activations/layer22_attention_weight_max": 46.514976501464844,
      "activations/layer22_attention_weight_min": -29.8338680267334,
      "activations/layer23_attention_weight_max": 33.52536392211914,
      "activations/layer23_attention_weight_min": -22.817089080810547,
      "activations/layer2_attention_weight_max": 35.67382049560547,
      "activations/layer2_attention_weight_min": -31.46950912475586,
      "activations/layer3_attention_weight_max": 83.40762329101562,
      "activations/layer3_attention_weight_min": -86.64375305175781,
      "activations/layer4_attention_weight_max": 107.69647216796875,
      "activations/layer4_attention_weight_min": -97.9625015258789,
      "activations/layer5_attention_weight_max": 51.173797607421875,
      "activations/layer5_attention_weight_min": -56.463653564453125,
      "activations/layer6_attention_weight_max": 45.2562141418457,
      "activations/layer6_attention_weight_min": -44.92831039428711,
      "activations/layer7_attention_weight_max": 79.99866485595703,
      "activations/layer7_attention_weight_min": -80.29086303710938,
      "activations/layer8_attention_weight_max": 37.27808380126953,
      "activations/layer8_attention_weight_min": -36.93632888793945,
      "activations/layer9_attention_weight_max": 32.02567672729492,
      "activations/layer9_attention_weight_min": -30.231103897094727,
      "epoch": 8.33,
      "learning_rate": 9.726401515151514e-05,
      "loss": 2.8105,
      "step": 143350
    },
    {
      "activations/layer0_attention_weight_max": 16.340343475341797,
      "activations/layer0_attention_weight_min": -12.276067733764648,
      "activations/layer10_attention_weight_max": 32.246578216552734,
      "activations/layer10_attention_weight_min": -31.976604461669922,
      "activations/layer11_attention_weight_max": 31.468557357788086,
      "activations/layer11_attention_weight_min": -29.54682159423828,
      "activations/layer12_attention_weight_max": 24.19220733642578,
      "activations/layer12_attention_weight_min": -28.04849624633789,
      "activations/layer13_attention_weight_max": 37.154937744140625,
      "activations/layer13_attention_weight_min": -32.39739227294922,
      "activations/layer14_attention_weight_max": 41.53783416748047,
      "activations/layer14_attention_weight_min": -36.18035888671875,
      "activations/layer15_attention_weight_max": 33.443294525146484,
      "activations/layer15_attention_weight_min": -30.025510787963867,
      "activations/layer16_attention_weight_max": 32.4150390625,
      "activations/layer16_attention_weight_min": -27.021251678466797,
      "activations/layer17_attention_weight_max": 32.643463134765625,
      "activations/layer17_attention_weight_min": -24.656421661376953,
      "activations/layer18_attention_weight_max": 30.76222038269043,
      "activations/layer18_attention_weight_min": -23.47921371459961,
      "activations/layer19_attention_weight_max": 33.88767623901367,
      "activations/layer19_attention_weight_min": -30.157108306884766,
      "activations/layer1_attention_weight_max": 18.910449981689453,
      "activations/layer1_attention_weight_min": -13.863751411437988,
      "activations/layer20_attention_weight_max": 31.04734992980957,
      "activations/layer20_attention_weight_min": -26.165849685668945,
      "activations/layer21_attention_weight_max": 27.661983489990234,
      "activations/layer21_attention_weight_min": -24.947595596313477,
      "activations/layer22_attention_weight_max": 47.55009841918945,
      "activations/layer22_attention_weight_min": -31.917430877685547,
      "activations/layer23_attention_weight_max": 31.633010864257812,
      "activations/layer23_attention_weight_min": -25.197059631347656,
      "activations/layer2_attention_weight_max": 28.292943954467773,
      "activations/layer2_attention_weight_min": -32.319313049316406,
      "activations/layer3_attention_weight_max": 84.72760772705078,
      "activations/layer3_attention_weight_min": -87.83368682861328,
      "activations/layer4_attention_weight_max": 110.27203369140625,
      "activations/layer4_attention_weight_min": -113.86067962646484,
      "activations/layer5_attention_weight_max": 53.53425598144531,
      "activations/layer5_attention_weight_min": -61.80048370361328,
      "activations/layer6_attention_weight_max": 44.79701614379883,
      "activations/layer6_attention_weight_min": -49.41315841674805,
      "activations/layer7_attention_weight_max": 88.17010498046875,
      "activations/layer7_attention_weight_min": -85.73517608642578,
      "activations/layer8_attention_weight_max": 38.95145034790039,
      "activations/layer8_attention_weight_min": -41.62527084350586,
      "activations/layer9_attention_weight_max": 32.71638488769531,
      "activations/layer9_attention_weight_min": -32.23468780517578,
      "epoch": 8.33,
      "learning_rate": 9.724507575757576e-05,
      "loss": 2.7972,
      "step": 143400
    },
    {
      "activations/layer0_attention_weight_max": 16.357149124145508,
      "activations/layer0_attention_weight_min": -12.911492347717285,
      "activations/layer10_attention_weight_max": 28.605783462524414,
      "activations/layer10_attention_weight_min": -29.491125106811523,
      "activations/layer11_attention_weight_max": 28.666168212890625,
      "activations/layer11_attention_weight_min": -29.685688018798828,
      "activations/layer12_attention_weight_max": 20.237573623657227,
      "activations/layer12_attention_weight_min": -27.868574142456055,
      "activations/layer13_attention_weight_max": 33.05166244506836,
      "activations/layer13_attention_weight_min": -33.08430480957031,
      "activations/layer14_attention_weight_max": 46.22819900512695,
      "activations/layer14_attention_weight_min": -43.676109313964844,
      "activations/layer15_attention_weight_max": 34.73744583129883,
      "activations/layer15_attention_weight_min": -31.833293914794922,
      "activations/layer16_attention_weight_max": 32.88364028930664,
      "activations/layer16_attention_weight_min": -28.756427764892578,
      "activations/layer17_attention_weight_max": 30.408411026000977,
      "activations/layer17_attention_weight_min": -25.92937469482422,
      "activations/layer18_attention_weight_max": 28.721364974975586,
      "activations/layer18_attention_weight_min": -22.710607528686523,
      "activations/layer19_attention_weight_max": 36.751766204833984,
      "activations/layer19_attention_weight_min": -30.139942169189453,
      "activations/layer1_attention_weight_max": 16.451099395751953,
      "activations/layer1_attention_weight_min": -13.368613243103027,
      "activations/layer20_attention_weight_max": 28.572370529174805,
      "activations/layer20_attention_weight_min": -24.54277992248535,
      "activations/layer21_attention_weight_max": 30.102745056152344,
      "activations/layer21_attention_weight_min": -22.0629940032959,
      "activations/layer22_attention_weight_max": 42.2358283996582,
      "activations/layer22_attention_weight_min": -28.524662017822266,
      "activations/layer23_attention_weight_max": 34.596981048583984,
      "activations/layer23_attention_weight_min": -25.5697078704834,
      "activations/layer2_attention_weight_max": 30.21994400024414,
      "activations/layer2_attention_weight_min": -30.371397018432617,
      "activations/layer3_attention_weight_max": 87.81649780273438,
      "activations/layer3_attention_weight_min": -87.64679718017578,
      "activations/layer4_attention_weight_max": 114.16571044921875,
      "activations/layer4_attention_weight_min": -111.71416473388672,
      "activations/layer5_attention_weight_max": 52.447540283203125,
      "activations/layer5_attention_weight_min": -60.15666198730469,
      "activations/layer6_attention_weight_max": 47.061214447021484,
      "activations/layer6_attention_weight_min": -48.736568450927734,
      "activations/layer7_attention_weight_max": 85.78597259521484,
      "activations/layer7_attention_weight_min": -81.40096282958984,
      "activations/layer8_attention_weight_max": 36.94241714477539,
      "activations/layer8_attention_weight_min": -37.97542190551758,
      "activations/layer9_attention_weight_max": 30.02907371520996,
      "activations/layer9_attention_weight_min": -30.973907470703125,
      "epoch": 8.34,
      "learning_rate": 9.722613636363635e-05,
      "loss": 2.8245,
      "step": 143450
    },
    {
      "activations/layer0_attention_weight_max": 15.4146728515625,
      "activations/layer0_attention_weight_min": -13.149311065673828,
      "activations/layer10_attention_weight_max": 32.494815826416016,
      "activations/layer10_attention_weight_min": -30.71728515625,
      "activations/layer11_attention_weight_max": 30.119735717773438,
      "activations/layer11_attention_weight_min": -30.519338607788086,
      "activations/layer12_attention_weight_max": 24.07052993774414,
      "activations/layer12_attention_weight_min": -28.075576782226562,
      "activations/layer13_attention_weight_max": 38.394859313964844,
      "activations/layer13_attention_weight_min": -35.32329559326172,
      "activations/layer14_attention_weight_max": 58.586631774902344,
      "activations/layer14_attention_weight_min": -46.1923942565918,
      "activations/layer15_attention_weight_max": 40.16992950439453,
      "activations/layer15_attention_weight_min": -30.566997528076172,
      "activations/layer16_attention_weight_max": 35.60857009887695,
      "activations/layer16_attention_weight_min": -28.507232666015625,
      "activations/layer17_attention_weight_max": 34.12582778930664,
      "activations/layer17_attention_weight_min": -27.602642059326172,
      "activations/layer18_attention_weight_max": 33.07759094238281,
      "activations/layer18_attention_weight_min": -21.95265007019043,
      "activations/layer19_attention_weight_max": 35.467708587646484,
      "activations/layer19_attention_weight_min": -29.825143814086914,
      "activations/layer1_attention_weight_max": 17.230546951293945,
      "activations/layer1_attention_weight_min": -16.04195785522461,
      "activations/layer20_attention_weight_max": 31.511445999145508,
      "activations/layer20_attention_weight_min": -24.811790466308594,
      "activations/layer21_attention_weight_max": 31.637020111083984,
      "activations/layer21_attention_weight_min": -22.92161750793457,
      "activations/layer22_attention_weight_max": 42.43328094482422,
      "activations/layer22_attention_weight_min": -29.198171615600586,
      "activations/layer23_attention_weight_max": 35.49604797363281,
      "activations/layer23_attention_weight_min": -26.281341552734375,
      "activations/layer2_attention_weight_max": 30.7313232421875,
      "activations/layer2_attention_weight_min": -31.69318962097168,
      "activations/layer3_attention_weight_max": 88.03649139404297,
      "activations/layer3_attention_weight_min": -91.24823760986328,
      "activations/layer4_attention_weight_max": 115.2576675415039,
      "activations/layer4_attention_weight_min": -113.60980224609375,
      "activations/layer5_attention_weight_max": 56.723724365234375,
      "activations/layer5_attention_weight_min": -61.94831848144531,
      "activations/layer6_attention_weight_max": 48.42621612548828,
      "activations/layer6_attention_weight_min": -51.083717346191406,
      "activations/layer7_attention_weight_max": 84.85797119140625,
      "activations/layer7_attention_weight_min": -92.89439392089844,
      "activations/layer8_attention_weight_max": 40.19590377807617,
      "activations/layer8_attention_weight_min": -40.626220703125,
      "activations/layer9_attention_weight_max": 35.22315216064453,
      "activations/layer9_attention_weight_min": -34.937721252441406,
      "epoch": 8.34,
      "learning_rate": 9.720719696969696e-05,
      "loss": 2.8115,
      "step": 143500
    },
    {
      "activations/layer0_attention_weight_max": 16.178653717041016,
      "activations/layer0_attention_weight_min": -13.778385162353516,
      "activations/layer10_attention_weight_max": 33.113624572753906,
      "activations/layer10_attention_weight_min": -31.628774642944336,
      "activations/layer11_attention_weight_max": 32.307979583740234,
      "activations/layer11_attention_weight_min": -29.616836547851562,
      "activations/layer12_attention_weight_max": 28.262313842773438,
      "activations/layer12_attention_weight_min": -24.596403121948242,
      "activations/layer13_attention_weight_max": 41.75959014892578,
      "activations/layer13_attention_weight_min": -32.02832794189453,
      "activations/layer14_attention_weight_max": 50.30750274658203,
      "activations/layer14_attention_weight_min": -40.421443939208984,
      "activations/layer15_attention_weight_max": 38.163246154785156,
      "activations/layer15_attention_weight_min": -29.25055694580078,
      "activations/layer16_attention_weight_max": 33.700782775878906,
      "activations/layer16_attention_weight_min": -26.7950382232666,
      "activations/layer17_attention_weight_max": 33.42323303222656,
      "activations/layer17_attention_weight_min": -24.637205123901367,
      "activations/layer18_attention_weight_max": 33.81940841674805,
      "activations/layer18_attention_weight_min": -22.83705711364746,
      "activations/layer19_attention_weight_max": 44.37934112548828,
      "activations/layer19_attention_weight_min": -30.77345848083496,
      "activations/layer1_attention_weight_max": 17.031291961669922,
      "activations/layer1_attention_weight_min": -14.196462631225586,
      "activations/layer20_attention_weight_max": 35.74113845825195,
      "activations/layer20_attention_weight_min": -25.915388107299805,
      "activations/layer21_attention_weight_max": 35.60968017578125,
      "activations/layer21_attention_weight_min": -23.436813354492188,
      "activations/layer22_attention_weight_max": 49.29301834106445,
      "activations/layer22_attention_weight_min": -28.992509841918945,
      "activations/layer23_attention_weight_max": 36.00454330444336,
      "activations/layer23_attention_weight_min": -25.146825790405273,
      "activations/layer2_attention_weight_max": 31.275836944580078,
      "activations/layer2_attention_weight_min": -30.107528686523438,
      "activations/layer3_attention_weight_max": 88.76184844970703,
      "activations/layer3_attention_weight_min": -87.59212493896484,
      "activations/layer4_attention_weight_max": 118.43663787841797,
      "activations/layer4_attention_weight_min": -115.48756408691406,
      "activations/layer5_attention_weight_max": 56.07552719116211,
      "activations/layer5_attention_weight_min": -62.41987609863281,
      "activations/layer6_attention_weight_max": 48.16732406616211,
      "activations/layer6_attention_weight_min": -52.82460021972656,
      "activations/layer7_attention_weight_max": 86.98471069335938,
      "activations/layer7_attention_weight_min": -93.83112335205078,
      "activations/layer8_attention_weight_max": 37.65610122680664,
      "activations/layer8_attention_weight_min": -39.29550552368164,
      "activations/layer9_attention_weight_max": 34.01940155029297,
      "activations/layer9_attention_weight_min": -33.29258346557617,
      "epoch": 8.34,
      "learning_rate": 9.718825757575758e-05,
      "loss": 2.8154,
      "step": 143550
    },
    {
      "activations/layer0_attention_weight_max": 15.655692100524902,
      "activations/layer0_attention_weight_min": -13.646151542663574,
      "activations/layer10_attention_weight_max": 32.55646514892578,
      "activations/layer10_attention_weight_min": -31.222476959228516,
      "activations/layer11_attention_weight_max": 32.36980056762695,
      "activations/layer11_attention_weight_min": -31.429962158203125,
      "activations/layer12_attention_weight_max": 23.98379135131836,
      "activations/layer12_attention_weight_min": -23.824649810791016,
      "activations/layer13_attention_weight_max": 39.127159118652344,
      "activations/layer13_attention_weight_min": -33.80656433105469,
      "activations/layer14_attention_weight_max": 48.66871643066406,
      "activations/layer14_attention_weight_min": -40.153907775878906,
      "activations/layer15_attention_weight_max": 36.224849700927734,
      "activations/layer15_attention_weight_min": -31.46501350402832,
      "activations/layer16_attention_weight_max": 35.44392776489258,
      "activations/layer16_attention_weight_min": -27.631208419799805,
      "activations/layer17_attention_weight_max": 33.1434211730957,
      "activations/layer17_attention_weight_min": -25.0417423248291,
      "activations/layer18_attention_weight_max": 31.839513778686523,
      "activations/layer18_attention_weight_min": -21.684520721435547,
      "activations/layer19_attention_weight_max": 39.83412170410156,
      "activations/layer19_attention_weight_min": -35.5359992980957,
      "activations/layer1_attention_weight_max": 16.970043182373047,
      "activations/layer1_attention_weight_min": -14.044157028198242,
      "activations/layer20_attention_weight_max": 34.819087982177734,
      "activations/layer20_attention_weight_min": -26.58077621459961,
      "activations/layer21_attention_weight_max": 31.446243286132812,
      "activations/layer21_attention_weight_min": -23.891427993774414,
      "activations/layer22_attention_weight_max": 46.40821075439453,
      "activations/layer22_attention_weight_min": -31.783811569213867,
      "activations/layer23_attention_weight_max": 37.778541564941406,
      "activations/layer23_attention_weight_min": -23.744613647460938,
      "activations/layer2_attention_weight_max": 30.566579818725586,
      "activations/layer2_attention_weight_min": -31.72894859313965,
      "activations/layer3_attention_weight_max": 90.59188079833984,
      "activations/layer3_attention_weight_min": -90.48892211914062,
      "activations/layer4_attention_weight_max": 117.7175521850586,
      "activations/layer4_attention_weight_min": -118.89556884765625,
      "activations/layer5_attention_weight_max": 54.9366569519043,
      "activations/layer5_attention_weight_min": -62.474884033203125,
      "activations/layer6_attention_weight_max": 49.10873031616211,
      "activations/layer6_attention_weight_min": -53.966732025146484,
      "activations/layer7_attention_weight_max": 85.66812133789062,
      "activations/layer7_attention_weight_min": -91.24256896972656,
      "activations/layer8_attention_weight_max": 38.216129302978516,
      "activations/layer8_attention_weight_min": -40.44623565673828,
      "activations/layer9_attention_weight_max": 34.81352233886719,
      "activations/layer9_attention_weight_min": -32.29326248168945,
      "epoch": 8.34,
      "learning_rate": 9.716931818181816e-05,
      "loss": 2.8042,
      "step": 143600
    },
    {
      "activations/layer0_attention_weight_max": 16.126012802124023,
      "activations/layer0_attention_weight_min": -12.27409553527832,
      "activations/layer10_attention_weight_max": 33.57000732421875,
      "activations/layer10_attention_weight_min": -32.05727005004883,
      "activations/layer11_attention_weight_max": 34.718994140625,
      "activations/layer11_attention_weight_min": -30.13454818725586,
      "activations/layer12_attention_weight_max": 22.61142349243164,
      "activations/layer12_attention_weight_min": -27.384912490844727,
      "activations/layer13_attention_weight_max": 38.79413604736328,
      "activations/layer13_attention_weight_min": -34.867088317871094,
      "activations/layer14_attention_weight_max": 47.33388137817383,
      "activations/layer14_attention_weight_min": -40.21269607543945,
      "activations/layer15_attention_weight_max": 35.90985870361328,
      "activations/layer15_attention_weight_min": -28.89128303527832,
      "activations/layer16_attention_weight_max": 32.526187896728516,
      "activations/layer16_attention_weight_min": -27.36614418029785,
      "activations/layer17_attention_weight_max": 31.88189697265625,
      "activations/layer17_attention_weight_min": -24.111125946044922,
      "activations/layer18_attention_weight_max": 30.298973083496094,
      "activations/layer18_attention_weight_min": -20.016952514648438,
      "activations/layer19_attention_weight_max": 38.61800765991211,
      "activations/layer19_attention_weight_min": -28.615406036376953,
      "activations/layer1_attention_weight_max": 16.683486938476562,
      "activations/layer1_attention_weight_min": -14.033281326293945,
      "activations/layer20_attention_weight_max": 32.525150299072266,
      "activations/layer20_attention_weight_min": -23.217443466186523,
      "activations/layer21_attention_weight_max": 30.536571502685547,
      "activations/layer21_attention_weight_min": -22.098901748657227,
      "activations/layer22_attention_weight_max": 44.64971160888672,
      "activations/layer22_attention_weight_min": -27.70326805114746,
      "activations/layer23_attention_weight_max": 37.05815887451172,
      "activations/layer23_attention_weight_min": -24.97333526611328,
      "activations/layer2_attention_weight_max": 30.430667877197266,
      "activations/layer2_attention_weight_min": -30.89478874206543,
      "activations/layer3_attention_weight_max": 87.546142578125,
      "activations/layer3_attention_weight_min": -91.48572540283203,
      "activations/layer4_attention_weight_max": 115.35736083984375,
      "activations/layer4_attention_weight_min": -120.6493148803711,
      "activations/layer5_attention_weight_max": 53.018489837646484,
      "activations/layer5_attention_weight_min": -59.966888427734375,
      "activations/layer6_attention_weight_max": 48.0339469909668,
      "activations/layer6_attention_weight_min": -49.052188873291016,
      "activations/layer7_attention_weight_max": 89.32231903076172,
      "activations/layer7_attention_weight_min": -86.80387115478516,
      "activations/layer8_attention_weight_max": 38.885345458984375,
      "activations/layer8_attention_weight_min": -40.26681137084961,
      "activations/layer9_attention_weight_max": 34.14728927612305,
      "activations/layer9_attention_weight_min": -33.29043197631836,
      "epoch": 8.35,
      "learning_rate": 9.715037878787878e-05,
      "loss": 2.8213,
      "step": 143650
    },
    {
      "activations/layer0_attention_weight_max": 15.505749702453613,
      "activations/layer0_attention_weight_min": -11.851807594299316,
      "activations/layer10_attention_weight_max": 31.875072479248047,
      "activations/layer10_attention_weight_min": -30.865299224853516,
      "activations/layer11_attention_weight_max": 30.54938316345215,
      "activations/layer11_attention_weight_min": -30.84790802001953,
      "activations/layer12_attention_weight_max": 23.548538208007812,
      "activations/layer12_attention_weight_min": -24.613908767700195,
      "activations/layer13_attention_weight_max": 38.052974700927734,
      "activations/layer13_attention_weight_min": -30.884035110473633,
      "activations/layer14_attention_weight_max": 42.71900177001953,
      "activations/layer14_attention_weight_min": -39.23713684082031,
      "activations/layer15_attention_weight_max": 35.803810119628906,
      "activations/layer15_attention_weight_min": -29.339107513427734,
      "activations/layer16_attention_weight_max": 31.1646728515625,
      "activations/layer16_attention_weight_min": -27.345937728881836,
      "activations/layer17_attention_weight_max": 34.526546478271484,
      "activations/layer17_attention_weight_min": -25.465639114379883,
      "activations/layer18_attention_weight_max": 32.094825744628906,
      "activations/layer18_attention_weight_min": -23.057140350341797,
      "activations/layer19_attention_weight_max": 36.42644119262695,
      "activations/layer19_attention_weight_min": -30.93363380432129,
      "activations/layer1_attention_weight_max": 18.934724807739258,
      "activations/layer1_attention_weight_min": -14.267179489135742,
      "activations/layer20_attention_weight_max": 28.018043518066406,
      "activations/layer20_attention_weight_min": -25.211231231689453,
      "activations/layer21_attention_weight_max": 28.559953689575195,
      "activations/layer21_attention_weight_min": -24.578857421875,
      "activations/layer22_attention_weight_max": 43.04582214355469,
      "activations/layer22_attention_weight_min": -29.11147689819336,
      "activations/layer23_attention_weight_max": 37.72898864746094,
      "activations/layer23_attention_weight_min": -24.542760848999023,
      "activations/layer2_attention_weight_max": 32.22950744628906,
      "activations/layer2_attention_weight_min": -32.20098876953125,
      "activations/layer3_attention_weight_max": 91.37053680419922,
      "activations/layer3_attention_weight_min": -96.02790832519531,
      "activations/layer4_attention_weight_max": 120.13448333740234,
      "activations/layer4_attention_weight_min": -117.91131591796875,
      "activations/layer5_attention_weight_max": 55.941650390625,
      "activations/layer5_attention_weight_min": -61.449371337890625,
      "activations/layer6_attention_weight_max": 49.83612823486328,
      "activations/layer6_attention_weight_min": -54.22557067871094,
      "activations/layer7_attention_weight_max": 84.52359008789062,
      "activations/layer7_attention_weight_min": -87.30619812011719,
      "activations/layer8_attention_weight_max": 40.9282112121582,
      "activations/layer8_attention_weight_min": -42.033592224121094,
      "activations/layer9_attention_weight_max": 35.41525650024414,
      "activations/layer9_attention_weight_min": -33.6523551940918,
      "epoch": 8.35,
      "learning_rate": 9.71314393939394e-05,
      "loss": 2.7978,
      "step": 143700
    },
    {
      "activations/layer0_attention_weight_max": 16.322481155395508,
      "activations/layer0_attention_weight_min": -12.441218376159668,
      "activations/layer10_attention_weight_max": 31.138469696044922,
      "activations/layer10_attention_weight_min": -30.219650268554688,
      "activations/layer11_attention_weight_max": 31.573760986328125,
      "activations/layer11_attention_weight_min": -29.28549575805664,
      "activations/layer12_attention_weight_max": 22.53972625732422,
      "activations/layer12_attention_weight_min": -24.202198028564453,
      "activations/layer13_attention_weight_max": 41.06199264526367,
      "activations/layer13_attention_weight_min": -31.52443504333496,
      "activations/layer14_attention_weight_max": 47.72245788574219,
      "activations/layer14_attention_weight_min": -38.5378303527832,
      "activations/layer15_attention_weight_max": 37.32593536376953,
      "activations/layer15_attention_weight_min": -30.115604400634766,
      "activations/layer16_attention_weight_max": 31.485427856445312,
      "activations/layer16_attention_weight_min": -27.32859992980957,
      "activations/layer17_attention_weight_max": 38.908660888671875,
      "activations/layer17_attention_weight_min": -24.974365234375,
      "activations/layer18_attention_weight_max": 30.440519332885742,
      "activations/layer18_attention_weight_min": -20.405319213867188,
      "activations/layer19_attention_weight_max": 36.33613967895508,
      "activations/layer19_attention_weight_min": -32.52951431274414,
      "activations/layer1_attention_weight_max": 17.051414489746094,
      "activations/layer1_attention_weight_min": -14.123324394226074,
      "activations/layer20_attention_weight_max": 35.60031509399414,
      "activations/layer20_attention_weight_min": -23.7188663482666,
      "activations/layer21_attention_weight_max": 36.556766510009766,
      "activations/layer21_attention_weight_min": -22.345727920532227,
      "activations/layer22_attention_weight_max": 52.02570343017578,
      "activations/layer22_attention_weight_min": -26.76249122619629,
      "activations/layer23_attention_weight_max": 36.958431243896484,
      "activations/layer23_attention_weight_min": -22.517391204833984,
      "activations/layer2_attention_weight_max": 30.48944091796875,
      "activations/layer2_attention_weight_min": -30.649822235107422,
      "activations/layer3_attention_weight_max": 92.17378997802734,
      "activations/layer3_attention_weight_min": -92.42323303222656,
      "activations/layer4_attention_weight_max": 113.14717102050781,
      "activations/layer4_attention_weight_min": -119.97403717041016,
      "activations/layer5_attention_weight_max": 55.38001251220703,
      "activations/layer5_attention_weight_min": -61.82929229736328,
      "activations/layer6_attention_weight_max": 47.752017974853516,
      "activations/layer6_attention_weight_min": -52.086605072021484,
      "activations/layer7_attention_weight_max": 87.82366180419922,
      "activations/layer7_attention_weight_min": -89.93634033203125,
      "activations/layer8_attention_weight_max": 38.68136978149414,
      "activations/layer8_attention_weight_min": -39.1070442199707,
      "activations/layer9_attention_weight_max": 35.36741256713867,
      "activations/layer9_attention_weight_min": -32.35378646850586,
      "epoch": 8.35,
      "learning_rate": 9.711287878787878e-05,
      "loss": 2.7996,
      "step": 143750
    },
    {
      "activations/layer0_attention_weight_max": 16.133541107177734,
      "activations/layer0_attention_weight_min": -12.499622344970703,
      "activations/layer10_attention_weight_max": 33.55440139770508,
      "activations/layer10_attention_weight_min": -31.491647720336914,
      "activations/layer11_attention_weight_max": 31.805639266967773,
      "activations/layer11_attention_weight_min": -29.759849548339844,
      "activations/layer12_attention_weight_max": 23.286771774291992,
      "activations/layer12_attention_weight_min": -24.643836975097656,
      "activations/layer13_attention_weight_max": 38.964027404785156,
      "activations/layer13_attention_weight_min": -32.790489196777344,
      "activations/layer14_attention_weight_max": 47.16984939575195,
      "activations/layer14_attention_weight_min": -39.8798713684082,
      "activations/layer15_attention_weight_max": 36.8747673034668,
      "activations/layer15_attention_weight_min": -30.578561782836914,
      "activations/layer16_attention_weight_max": 35.42658615112305,
      "activations/layer16_attention_weight_min": -27.659587860107422,
      "activations/layer17_attention_weight_max": 34.62778091430664,
      "activations/layer17_attention_weight_min": -27.88813018798828,
      "activations/layer18_attention_weight_max": 32.06512451171875,
      "activations/layer18_attention_weight_min": -24.059057235717773,
      "activations/layer19_attention_weight_max": 39.496822357177734,
      "activations/layer19_attention_weight_min": -29.96356773376465,
      "activations/layer1_attention_weight_max": 17.010787963867188,
      "activations/layer1_attention_weight_min": -14.945328712463379,
      "activations/layer20_attention_weight_max": 33.53201675415039,
      "activations/layer20_attention_weight_min": -23.951528549194336,
      "activations/layer21_attention_weight_max": 30.927501678466797,
      "activations/layer21_attention_weight_min": -23.128253936767578,
      "activations/layer22_attention_weight_max": 48.6807746887207,
      "activations/layer22_attention_weight_min": -28.934221267700195,
      "activations/layer23_attention_weight_max": 37.134796142578125,
      "activations/layer23_attention_weight_min": -23.798633575439453,
      "activations/layer2_attention_weight_max": 31.432905197143555,
      "activations/layer2_attention_weight_min": -31.74853515625,
      "activations/layer3_attention_weight_max": 90.22522735595703,
      "activations/layer3_attention_weight_min": -92.93600463867188,
      "activations/layer4_attention_weight_max": 117.14566802978516,
      "activations/layer4_attention_weight_min": -119.31233215332031,
      "activations/layer5_attention_weight_max": 57.10248565673828,
      "activations/layer5_attention_weight_min": -67.49701690673828,
      "activations/layer6_attention_weight_max": 47.672340393066406,
      "activations/layer6_attention_weight_min": -52.402305603027344,
      "activations/layer7_attention_weight_max": 87.35658264160156,
      "activations/layer7_attention_weight_min": -86.7120590209961,
      "activations/layer8_attention_weight_max": 38.57832336425781,
      "activations/layer8_attention_weight_min": -40.74677658081055,
      "activations/layer9_attention_weight_max": 34.79866027832031,
      "activations/layer9_attention_weight_min": -33.311988830566406,
      "epoch": 8.36,
      "learning_rate": 9.709393939393937e-05,
      "loss": 2.795,
      "step": 143800
    },
    {
      "activations/layer0_attention_weight_max": 14.83353328704834,
      "activations/layer0_attention_weight_min": -11.967323303222656,
      "activations/layer10_attention_weight_max": 33.49433135986328,
      "activations/layer10_attention_weight_min": -33.47003173828125,
      "activations/layer11_attention_weight_max": 33.57136535644531,
      "activations/layer11_attention_weight_min": -31.409101486206055,
      "activations/layer12_attention_weight_max": 25.53559112548828,
      "activations/layer12_attention_weight_min": -23.807287216186523,
      "activations/layer13_attention_weight_max": 40.79594039916992,
      "activations/layer13_attention_weight_min": -34.85045623779297,
      "activations/layer14_attention_weight_max": 48.753475189208984,
      "activations/layer14_attention_weight_min": -36.994834899902344,
      "activations/layer15_attention_weight_max": 38.84396743774414,
      "activations/layer15_attention_weight_min": -31.634807586669922,
      "activations/layer16_attention_weight_max": 40.04918670654297,
      "activations/layer16_attention_weight_min": -27.504953384399414,
      "activations/layer17_attention_weight_max": 33.48583221435547,
      "activations/layer17_attention_weight_min": -24.27545928955078,
      "activations/layer18_attention_weight_max": 31.450565338134766,
      "activations/layer18_attention_weight_min": -21.10240936279297,
      "activations/layer19_attention_weight_max": 40.30650329589844,
      "activations/layer19_attention_weight_min": -29.69284439086914,
      "activations/layer1_attention_weight_max": 15.926419258117676,
      "activations/layer1_attention_weight_min": -12.135419845581055,
      "activations/layer20_attention_weight_max": 34.492225646972656,
      "activations/layer20_attention_weight_min": -23.277446746826172,
      "activations/layer21_attention_weight_max": 34.02913284301758,
      "activations/layer21_attention_weight_min": -22.786222457885742,
      "activations/layer22_attention_weight_max": 48.963470458984375,
      "activations/layer22_attention_weight_min": -28.54486656188965,
      "activations/layer23_attention_weight_max": 37.20322799682617,
      "activations/layer23_attention_weight_min": -22.260770797729492,
      "activations/layer2_attention_weight_max": 28.915847778320312,
      "activations/layer2_attention_weight_min": -29.74100112915039,
      "activations/layer3_attention_weight_max": 88.18222045898438,
      "activations/layer3_attention_weight_min": -90.20563507080078,
      "activations/layer4_attention_weight_max": 115.555419921875,
      "activations/layer4_attention_weight_min": -123.43863677978516,
      "activations/layer5_attention_weight_max": 55.51363754272461,
      "activations/layer5_attention_weight_min": -62.82878875732422,
      "activations/layer6_attention_weight_max": 48.70292282104492,
      "activations/layer6_attention_weight_min": -52.40907669067383,
      "activations/layer7_attention_weight_max": 88.30191040039062,
      "activations/layer7_attention_weight_min": -91.01626586914062,
      "activations/layer8_attention_weight_max": 40.616085052490234,
      "activations/layer8_attention_weight_min": -42.51054382324219,
      "activations/layer9_attention_weight_max": 34.13922882080078,
      "activations/layer9_attention_weight_min": -37.20553207397461,
      "epoch": 8.36,
      "learning_rate": 9.707499999999999e-05,
      "loss": 2.8049,
      "step": 143850
    },
    {
      "activations/layer0_attention_weight_max": 16.08683204650879,
      "activations/layer0_attention_weight_min": -12.454631805419922,
      "activations/layer10_attention_weight_max": 30.55788230895996,
      "activations/layer10_attention_weight_min": -29.883358001708984,
      "activations/layer11_attention_weight_max": 29.68672752380371,
      "activations/layer11_attention_weight_min": -30.46413803100586,
      "activations/layer12_attention_weight_max": 31.400737762451172,
      "activations/layer12_attention_weight_min": -23.207956314086914,
      "activations/layer13_attention_weight_max": 37.879146575927734,
      "activations/layer13_attention_weight_min": -33.491539001464844,
      "activations/layer14_attention_weight_max": 54.8154296875,
      "activations/layer14_attention_weight_min": -43.76936721801758,
      "activations/layer15_attention_weight_max": 34.862579345703125,
      "activations/layer15_attention_weight_min": -29.315019607543945,
      "activations/layer16_attention_weight_max": 33.33848571777344,
      "activations/layer16_attention_weight_min": -27.405193328857422,
      "activations/layer17_attention_weight_max": 34.043731689453125,
      "activations/layer17_attention_weight_min": -26.588096618652344,
      "activations/layer18_attention_weight_max": 39.304962158203125,
      "activations/layer18_attention_weight_min": -23.86638641357422,
      "activations/layer19_attention_weight_max": 35.72806167602539,
      "activations/layer19_attention_weight_min": -29.772397994995117,
      "activations/layer1_attention_weight_max": 16.963058471679688,
      "activations/layer1_attention_weight_min": -14.519700050354004,
      "activations/layer20_attention_weight_max": 29.232698440551758,
      "activations/layer20_attention_weight_min": -23.16327667236328,
      "activations/layer21_attention_weight_max": 32.391910552978516,
      "activations/layer21_attention_weight_min": -22.394845962524414,
      "activations/layer22_attention_weight_max": 46.72158432006836,
      "activations/layer22_attention_weight_min": -28.944862365722656,
      "activations/layer23_attention_weight_max": 32.889530181884766,
      "activations/layer23_attention_weight_min": -23.034343719482422,
      "activations/layer2_attention_weight_max": 30.88595962524414,
      "activations/layer2_attention_weight_min": -31.030553817749023,
      "activations/layer3_attention_weight_max": 86.80225372314453,
      "activations/layer3_attention_weight_min": -89.74895477294922,
      "activations/layer4_attention_weight_max": 113.74859619140625,
      "activations/layer4_attention_weight_min": -112.84696197509766,
      "activations/layer5_attention_weight_max": 55.739051818847656,
      "activations/layer5_attention_weight_min": -60.4722900390625,
      "activations/layer6_attention_weight_max": 48.28834915161133,
      "activations/layer6_attention_weight_min": -53.10554504394531,
      "activations/layer7_attention_weight_max": 85.81661224365234,
      "activations/layer7_attention_weight_min": -88.82947540283203,
      "activations/layer8_attention_weight_max": 37.08641815185547,
      "activations/layer8_attention_weight_min": -38.366615295410156,
      "activations/layer9_attention_weight_max": 32.87417221069336,
      "activations/layer9_attention_weight_min": -32.36833190917969,
      "epoch": 8.36,
      "learning_rate": 9.70560606060606e-05,
      "loss": 2.7985,
      "step": 143900
    },
    {
      "activations/layer0_attention_weight_max": 15.80639362335205,
      "activations/layer0_attention_weight_min": -12.816229820251465,
      "activations/layer10_attention_weight_max": 32.95172882080078,
      "activations/layer10_attention_weight_min": -31.796268463134766,
      "activations/layer11_attention_weight_max": 31.27283477783203,
      "activations/layer11_attention_weight_min": -31.028217315673828,
      "activations/layer12_attention_weight_max": 28.352846145629883,
      "activations/layer12_attention_weight_min": -29.56315803527832,
      "activations/layer13_attention_weight_max": 37.027442932128906,
      "activations/layer13_attention_weight_min": -34.432254791259766,
      "activations/layer14_attention_weight_max": 47.029197692871094,
      "activations/layer14_attention_weight_min": -40.20043182373047,
      "activations/layer15_attention_weight_max": 35.861934661865234,
      "activations/layer15_attention_weight_min": -30.722042083740234,
      "activations/layer16_attention_weight_max": 32.02339172363281,
      "activations/layer16_attention_weight_min": -28.4479923248291,
      "activations/layer17_attention_weight_max": 34.69380569458008,
      "activations/layer17_attention_weight_min": -26.270578384399414,
      "activations/layer18_attention_weight_max": 31.912080764770508,
      "activations/layer18_attention_weight_min": -21.830766677856445,
      "activations/layer19_attention_weight_max": 39.65237045288086,
      "activations/layer19_attention_weight_min": -33.10103988647461,
      "activations/layer1_attention_weight_max": 16.21111297607422,
      "activations/layer1_attention_weight_min": -13.988651275634766,
      "activations/layer20_attention_weight_max": 36.51232147216797,
      "activations/layer20_attention_weight_min": -27.022830963134766,
      "activations/layer21_attention_weight_max": 33.27696990966797,
      "activations/layer21_attention_weight_min": -26.322568893432617,
      "activations/layer22_attention_weight_max": 46.16411590576172,
      "activations/layer22_attention_weight_min": -28.74100685119629,
      "activations/layer23_attention_weight_max": 36.599185943603516,
      "activations/layer23_attention_weight_min": -24.621444702148438,
      "activations/layer2_attention_weight_max": 33.78412628173828,
      "activations/layer2_attention_weight_min": -31.88837242126465,
      "activations/layer3_attention_weight_max": 92.37142944335938,
      "activations/layer3_attention_weight_min": -92.31522369384766,
      "activations/layer4_attention_weight_max": 116.93355560302734,
      "activations/layer4_attention_weight_min": -115.94050598144531,
      "activations/layer5_attention_weight_max": 57.134300231933594,
      "activations/layer5_attention_weight_min": -62.916648864746094,
      "activations/layer6_attention_weight_max": 49.648902893066406,
      "activations/layer6_attention_weight_min": -50.86485290527344,
      "activations/layer7_attention_weight_max": 91.47466278076172,
      "activations/layer7_attention_weight_min": -85.42395782470703,
      "activations/layer8_attention_weight_max": 41.88115310668945,
      "activations/layer8_attention_weight_min": -40.31160354614258,
      "activations/layer9_attention_weight_max": 37.55121612548828,
      "activations/layer9_attention_weight_min": -33.726165771484375,
      "epoch": 8.36,
      "learning_rate": 9.70371212121212e-05,
      "loss": 2.8198,
      "step": 143950
    },
    {
      "activations/layer0_attention_weight_max": 15.201384544372559,
      "activations/layer0_attention_weight_min": -11.697416305541992,
      "activations/layer10_attention_weight_max": 35.0574951171875,
      "activations/layer10_attention_weight_min": -33.63206100463867,
      "activations/layer11_attention_weight_max": 36.38530731201172,
      "activations/layer11_attention_weight_min": -33.60248565673828,
      "activations/layer12_attention_weight_max": 29.982654571533203,
      "activations/layer12_attention_weight_min": -27.97871208190918,
      "activations/layer13_attention_weight_max": 41.17891311645508,
      "activations/layer13_attention_weight_min": -34.753902435302734,
      "activations/layer14_attention_weight_max": 46.421634674072266,
      "activations/layer14_attention_weight_min": -41.88083267211914,
      "activations/layer15_attention_weight_max": 36.11349868774414,
      "activations/layer15_attention_weight_min": -31.64750099182129,
      "activations/layer16_attention_weight_max": 35.34099197387695,
      "activations/layer16_attention_weight_min": -27.557315826416016,
      "activations/layer17_attention_weight_max": 34.06086730957031,
      "activations/layer17_attention_weight_min": -24.276199340820312,
      "activations/layer18_attention_weight_max": 30.559293746948242,
      "activations/layer18_attention_weight_min": -21.300296783447266,
      "activations/layer19_attention_weight_max": 40.1964225769043,
      "activations/layer19_attention_weight_min": -29.256668090820312,
      "activations/layer1_attention_weight_max": 16.66652488708496,
      "activations/layer1_attention_weight_min": -15.084554672241211,
      "activations/layer20_attention_weight_max": 34.98698425292969,
      "activations/layer20_attention_weight_min": -23.44710922241211,
      "activations/layer21_attention_weight_max": 34.909088134765625,
      "activations/layer21_attention_weight_min": -21.8795108795166,
      "activations/layer22_attention_weight_max": 49.837562561035156,
      "activations/layer22_attention_weight_min": -26.47551918029785,
      "activations/layer23_attention_weight_max": 40.362159729003906,
      "activations/layer23_attention_weight_min": -23.57979965209961,
      "activations/layer2_attention_weight_max": 35.13789749145508,
      "activations/layer2_attention_weight_min": -30.790302276611328,
      "activations/layer3_attention_weight_max": 95.7988052368164,
      "activations/layer3_attention_weight_min": -90.97539520263672,
      "activations/layer4_attention_weight_max": 134.69781494140625,
      "activations/layer4_attention_weight_min": -119.0537338256836,
      "activations/layer5_attention_weight_max": 64.1822738647461,
      "activations/layer5_attention_weight_min": -62.61079406738281,
      "activations/layer6_attention_weight_max": 55.27805709838867,
      "activations/layer6_attention_weight_min": -53.39301300048828,
      "activations/layer7_attention_weight_max": 101.9354248046875,
      "activations/layer7_attention_weight_min": -91.55878448486328,
      "activations/layer8_attention_weight_max": 43.5379524230957,
      "activations/layer8_attention_weight_min": -41.2348518371582,
      "activations/layer9_attention_weight_max": 37.14085006713867,
      "activations/layer9_attention_weight_min": -36.31075668334961,
      "epoch": 8.37,
      "learning_rate": 9.70181818181818e-05,
      "loss": 2.8011,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5502,
      "eval_samples_per_second": 502.212,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5502,
      "eval_openwebtext_samples_per_second": 502.212,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 2.0619,
      "eval_wikitext_samples_per_second": 221.152,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_lambada_loss": 2.603515625,
      "eval_lambada_ppl": 13.511154789957821,
      "eval_lambada_runtime": 9.6038,
      "eval_lambada_samples_per_second": 506.986,
      "step": 144000
    },
    {
      "activations/layer0_attention_weight_max": 14.549858093261719,
      "activations/layer0_attention_weight_min": -12.184931755065918,
      "activations/layer10_attention_weight_max": 33.981605529785156,
      "activations/layer10_attention_weight_min": -31.927743911743164,
      "activations/layer11_attention_weight_max": 32.6292839050293,
      "activations/layer11_attention_weight_min": -31.804718017578125,
      "activations/layer12_attention_weight_max": 23.76334571838379,
      "activations/layer12_attention_weight_min": -27.770042419433594,
      "activations/layer13_attention_weight_max": 38.684669494628906,
      "activations/layer13_attention_weight_min": -34.11112976074219,
      "activations/layer14_attention_weight_max": 49.37077713012695,
      "activations/layer14_attention_weight_min": -37.06502151489258,
      "activations/layer15_attention_weight_max": 35.31254959106445,
      "activations/layer15_attention_weight_min": -30.809349060058594,
      "activations/layer16_attention_weight_max": 33.84575271606445,
      "activations/layer16_attention_weight_min": -27.14825439453125,
      "activations/layer17_attention_weight_max": 37.122772216796875,
      "activations/layer17_attention_weight_min": -25.95974349975586,
      "activations/layer18_attention_weight_max": 34.334659576416016,
      "activations/layer18_attention_weight_min": -21.727468490600586,
      "activations/layer19_attention_weight_max": 39.36281204223633,
      "activations/layer19_attention_weight_min": -29.673032760620117,
      "activations/layer1_attention_weight_max": 18.4001407623291,
      "activations/layer1_attention_weight_min": -17.170652389526367,
      "activations/layer20_attention_weight_max": 31.65629768371582,
      "activations/layer20_attention_weight_min": -25.11202049255371,
      "activations/layer21_attention_weight_max": 31.247207641601562,
      "activations/layer21_attention_weight_min": -22.792097091674805,
      "activations/layer22_attention_weight_max": 47.41485595703125,
      "activations/layer22_attention_weight_min": -29.174436569213867,
      "activations/layer23_attention_weight_max": 38.475135803222656,
      "activations/layer23_attention_weight_min": -21.84606170654297,
      "activations/layer2_attention_weight_max": 32.28395080566406,
      "activations/layer2_attention_weight_min": -31.302608489990234,
      "activations/layer3_attention_weight_max": 92.36785888671875,
      "activations/layer3_attention_weight_min": -89.74117279052734,
      "activations/layer4_attention_weight_max": 120.29390716552734,
      "activations/layer4_attention_weight_min": -121.47428131103516,
      "activations/layer5_attention_weight_max": 56.28976058959961,
      "activations/layer5_attention_weight_min": -64.01129150390625,
      "activations/layer6_attention_weight_max": 53.73727798461914,
      "activations/layer6_attention_weight_min": -52.4265251159668,
      "activations/layer7_attention_weight_max": 89.14100646972656,
      "activations/layer7_attention_weight_min": -88.49900817871094,
      "activations/layer8_attention_weight_max": 39.6692008972168,
      "activations/layer8_attention_weight_min": -39.98565673828125,
      "activations/layer9_attention_weight_max": 35.34821701049805,
      "activations/layer9_attention_weight_min": -35.04857635498047,
      "epoch": 8.37,
      "learning_rate": 9.699924242424242e-05,
      "loss": 2.8054,
      "step": 144050
    },
    {
      "activations/layer0_attention_weight_max": 15.907403945922852,
      "activations/layer0_attention_weight_min": -12.480926513671875,
      "activations/layer10_attention_weight_max": 37.47786331176758,
      "activations/layer10_attention_weight_min": -33.165061950683594,
      "activations/layer11_attention_weight_max": 37.4775505065918,
      "activations/layer11_attention_weight_min": -32.80264663696289,
      "activations/layer12_attention_weight_max": 25.230091094970703,
      "activations/layer12_attention_weight_min": -25.1614933013916,
      "activations/layer13_attention_weight_max": 42.300315856933594,
      "activations/layer13_attention_weight_min": -33.989723205566406,
      "activations/layer14_attention_weight_max": 48.580718994140625,
      "activations/layer14_attention_weight_min": -37.66666030883789,
      "activations/layer15_attention_weight_max": 38.85995101928711,
      "activations/layer15_attention_weight_min": -28.076478958129883,
      "activations/layer16_attention_weight_max": 32.539695739746094,
      "activations/layer16_attention_weight_min": -26.314849853515625,
      "activations/layer17_attention_weight_max": 36.482933044433594,
      "activations/layer17_attention_weight_min": -23.58213233947754,
      "activations/layer18_attention_weight_max": 31.84353256225586,
      "activations/layer18_attention_weight_min": -23.737398147583008,
      "activations/layer19_attention_weight_max": 37.30838394165039,
      "activations/layer19_attention_weight_min": -29.701278686523438,
      "activations/layer1_attention_weight_max": 17.35957145690918,
      "activations/layer1_attention_weight_min": -15.40461254119873,
      "activations/layer20_attention_weight_max": 33.054744720458984,
      "activations/layer20_attention_weight_min": -24.860607147216797,
      "activations/layer21_attention_weight_max": 31.80791473388672,
      "activations/layer21_attention_weight_min": -23.71915054321289,
      "activations/layer22_attention_weight_max": 49.359375,
      "activations/layer22_attention_weight_min": -28.73208999633789,
      "activations/layer23_attention_weight_max": 36.64543151855469,
      "activations/layer23_attention_weight_min": -24.93187713623047,
      "activations/layer2_attention_weight_max": 33.177574157714844,
      "activations/layer2_attention_weight_min": -32.062618255615234,
      "activations/layer3_attention_weight_max": 91.3512954711914,
      "activations/layer3_attention_weight_min": -95.15628814697266,
      "activations/layer4_attention_weight_max": 123.72417449951172,
      "activations/layer4_attention_weight_min": -122.34132385253906,
      "activations/layer5_attention_weight_max": 60.978111267089844,
      "activations/layer5_attention_weight_min": -63.596527099609375,
      "activations/layer6_attention_weight_max": 52.962947845458984,
      "activations/layer6_attention_weight_min": -52.39301681518555,
      "activations/layer7_attention_weight_max": 97.42898559570312,
      "activations/layer7_attention_weight_min": -97.76175689697266,
      "activations/layer8_attention_weight_max": 46.3079833984375,
      "activations/layer8_attention_weight_min": -45.34866714477539,
      "activations/layer9_attention_weight_max": 39.52608871459961,
      "activations/layer9_attention_weight_min": -35.046661376953125,
      "epoch": 8.37,
      "learning_rate": 9.698030303030302e-05,
      "loss": 2.8151,
      "step": 144100
    },
    {
      "activations/layer0_attention_weight_max": 15.453754425048828,
      "activations/layer0_attention_weight_min": -11.588767051696777,
      "activations/layer10_attention_weight_max": 32.183677673339844,
      "activations/layer10_attention_weight_min": -31.40365982055664,
      "activations/layer11_attention_weight_max": 33.05335235595703,
      "activations/layer11_attention_weight_min": -30.632986068725586,
      "activations/layer12_attention_weight_max": 22.883493423461914,
      "activations/layer12_attention_weight_min": -27.282211303710938,
      "activations/layer13_attention_weight_max": 39.03807830810547,
      "activations/layer13_attention_weight_min": -33.680015563964844,
      "activations/layer14_attention_weight_max": 54.32709884643555,
      "activations/layer14_attention_weight_min": -39.43876647949219,
      "activations/layer15_attention_weight_max": 37.31904220581055,
      "activations/layer15_attention_weight_min": -31.34498405456543,
      "activations/layer16_attention_weight_max": 34.491943359375,
      "activations/layer16_attention_weight_min": -27.256628036499023,
      "activations/layer17_attention_weight_max": 34.19966125488281,
      "activations/layer17_attention_weight_min": -26.524805068969727,
      "activations/layer18_attention_weight_max": 33.566043853759766,
      "activations/layer18_attention_weight_min": -23.1568603515625,
      "activations/layer19_attention_weight_max": 40.40975570678711,
      "activations/layer19_attention_weight_min": -32.415321350097656,
      "activations/layer1_attention_weight_max": 16.22605323791504,
      "activations/layer1_attention_weight_min": -16.13324737548828,
      "activations/layer20_attention_weight_max": 32.71299362182617,
      "activations/layer20_attention_weight_min": -24.188737869262695,
      "activations/layer21_attention_weight_max": 33.09247970581055,
      "activations/layer21_attention_weight_min": -24.687387466430664,
      "activations/layer22_attention_weight_max": 47.85084533691406,
      "activations/layer22_attention_weight_min": -29.226009368896484,
      "activations/layer23_attention_weight_max": 36.24185562133789,
      "activations/layer23_attention_weight_min": -25.410152435302734,
      "activations/layer2_attention_weight_max": 32.900146484375,
      "activations/layer2_attention_weight_min": -34.55685806274414,
      "activations/layer3_attention_weight_max": 92.46281433105469,
      "activations/layer3_attention_weight_min": -97.06181335449219,
      "activations/layer4_attention_weight_max": 119.02388763427734,
      "activations/layer4_attention_weight_min": -121.05207824707031,
      "activations/layer5_attention_weight_max": 56.08730697631836,
      "activations/layer5_attention_weight_min": -64.37950897216797,
      "activations/layer6_attention_weight_max": 48.82074737548828,
      "activations/layer6_attention_weight_min": -53.739891052246094,
      "activations/layer7_attention_weight_max": 91.7193603515625,
      "activations/layer7_attention_weight_min": -92.1860580444336,
      "activations/layer8_attention_weight_max": 40.700374603271484,
      "activations/layer8_attention_weight_min": -41.84246063232422,
      "activations/layer9_attention_weight_max": 35.991966247558594,
      "activations/layer9_attention_weight_min": -35.26338577270508,
      "epoch": 8.38,
      "learning_rate": 9.696136363636364e-05,
      "loss": 2.8073,
      "step": 144150
    },
    {
      "activations/layer0_attention_weight_max": 15.670866966247559,
      "activations/layer0_attention_weight_min": -12.229451179504395,
      "activations/layer10_attention_weight_max": 35.637939453125,
      "activations/layer10_attention_weight_min": -32.10525131225586,
      "activations/layer11_attention_weight_max": 34.338890075683594,
      "activations/layer11_attention_weight_min": -29.903480529785156,
      "activations/layer12_attention_weight_max": 23.810998916625977,
      "activations/layer12_attention_weight_min": -24.883167266845703,
      "activations/layer13_attention_weight_max": 41.66608428955078,
      "activations/layer13_attention_weight_min": -31.75110626220703,
      "activations/layer14_attention_weight_max": 51.35164260864258,
      "activations/layer14_attention_weight_min": -38.152862548828125,
      "activations/layer15_attention_weight_max": 37.556880950927734,
      "activations/layer15_attention_weight_min": -29.34402084350586,
      "activations/layer16_attention_weight_max": 39.7095947265625,
      "activations/layer16_attention_weight_min": -27.96005630493164,
      "activations/layer17_attention_weight_max": 36.748046875,
      "activations/layer17_attention_weight_min": -25.145235061645508,
      "activations/layer18_attention_weight_max": 38.7630729675293,
      "activations/layer18_attention_weight_min": -21.41806411743164,
      "activations/layer19_attention_weight_max": 43.88471603393555,
      "activations/layer19_attention_weight_min": -30.159826278686523,
      "activations/layer1_attention_weight_max": 16.55650520324707,
      "activations/layer1_attention_weight_min": -14.376951217651367,
      "activations/layer20_attention_weight_max": 36.059417724609375,
      "activations/layer20_attention_weight_min": -23.38259506225586,
      "activations/layer21_attention_weight_max": 35.325199127197266,
      "activations/layer21_attention_weight_min": -22.940120697021484,
      "activations/layer22_attention_weight_max": 51.80831527709961,
      "activations/layer22_attention_weight_min": -27.63995933532715,
      "activations/layer23_attention_weight_max": 41.57571029663086,
      "activations/layer23_attention_weight_min": -24.015480041503906,
      "activations/layer2_attention_weight_max": 32.31983947753906,
      "activations/layer2_attention_weight_min": -31.248432159423828,
      "activations/layer3_attention_weight_max": 92.5046157836914,
      "activations/layer3_attention_weight_min": -92.24236297607422,
      "activations/layer4_attention_weight_max": 117.37340545654297,
      "activations/layer4_attention_weight_min": -111.5400390625,
      "activations/layer5_attention_weight_max": 57.84124755859375,
      "activations/layer5_attention_weight_min": -60.50909423828125,
      "activations/layer6_attention_weight_max": 49.22004318237305,
      "activations/layer6_attention_weight_min": -50.99530792236328,
      "activations/layer7_attention_weight_max": 90.66578674316406,
      "activations/layer7_attention_weight_min": -87.83596801757812,
      "activations/layer8_attention_weight_max": 40.454795837402344,
      "activations/layer8_attention_weight_min": -40.2577018737793,
      "activations/layer9_attention_weight_max": 35.90574264526367,
      "activations/layer9_attention_weight_min": -34.41254806518555,
      "epoch": 8.38,
      "learning_rate": 9.694242424242423e-05,
      "loss": 2.816,
      "step": 144200
    },
    {
      "activations/layer0_attention_weight_max": 15.689945220947266,
      "activations/layer0_attention_weight_min": -11.74680233001709,
      "activations/layer10_attention_weight_max": 33.892974853515625,
      "activations/layer10_attention_weight_min": -29.148576736450195,
      "activations/layer11_attention_weight_max": 33.001373291015625,
      "activations/layer11_attention_weight_min": -28.45966148376465,
      "activations/layer12_attention_weight_max": 22.463550567626953,
      "activations/layer12_attention_weight_min": -33.898075103759766,
      "activations/layer13_attention_weight_max": 40.51442337036133,
      "activations/layer13_attention_weight_min": -31.13572120666504,
      "activations/layer14_attention_weight_max": 48.633522033691406,
      "activations/layer14_attention_weight_min": -38.55191421508789,
      "activations/layer15_attention_weight_max": 35.62089157104492,
      "activations/layer15_attention_weight_min": -29.79535675048828,
      "activations/layer16_attention_weight_max": 33.262306213378906,
      "activations/layer16_attention_weight_min": -26.928251266479492,
      "activations/layer17_attention_weight_max": 31.66578483581543,
      "activations/layer17_attention_weight_min": -25.803348541259766,
      "activations/layer18_attention_weight_max": 30.750463485717773,
      "activations/layer18_attention_weight_min": -21.51158332824707,
      "activations/layer19_attention_weight_max": 38.75599670410156,
      "activations/layer19_attention_weight_min": -31.364469528198242,
      "activations/layer1_attention_weight_max": 16.909900665283203,
      "activations/layer1_attention_weight_min": -14.091028213500977,
      "activations/layer20_attention_weight_max": 30.311988830566406,
      "activations/layer20_attention_weight_min": -25.798168182373047,
      "activations/layer21_attention_weight_max": 30.330656051635742,
      "activations/layer21_attention_weight_min": -23.5194034576416,
      "activations/layer22_attention_weight_max": 45.67691421508789,
      "activations/layer22_attention_weight_min": -28.743316650390625,
      "activations/layer23_attention_weight_max": 41.44269561767578,
      "activations/layer23_attention_weight_min": -23.71810531616211,
      "activations/layer2_attention_weight_max": 32.38018798828125,
      "activations/layer2_attention_weight_min": -30.675918579101562,
      "activations/layer3_attention_weight_max": 88.58049774169922,
      "activations/layer3_attention_weight_min": -95.25304412841797,
      "activations/layer4_attention_weight_max": 115.5898666381836,
      "activations/layer4_attention_weight_min": -122.33756256103516,
      "activations/layer5_attention_weight_max": 53.47224426269531,
      "activations/layer5_attention_weight_min": -62.43458557128906,
      "activations/layer6_attention_weight_max": 46.66883087158203,
      "activations/layer6_attention_weight_min": -50.30934143066406,
      "activations/layer7_attention_weight_max": 86.19683837890625,
      "activations/layer7_attention_weight_min": -85.63995361328125,
      "activations/layer8_attention_weight_max": 37.663917541503906,
      "activations/layer8_attention_weight_min": -40.35186767578125,
      "activations/layer9_attention_weight_max": 33.429744720458984,
      "activations/layer9_attention_weight_min": -36.61204147338867,
      "epoch": 8.38,
      "learning_rate": 9.692348484848484e-05,
      "loss": 2.8031,
      "step": 144250
    },
    {
      "activations/layer0_attention_weight_max": 15.256359100341797,
      "activations/layer0_attention_weight_min": -11.443340301513672,
      "activations/layer10_attention_weight_max": 33.189369201660156,
      "activations/layer10_attention_weight_min": -33.33643341064453,
      "activations/layer11_attention_weight_max": 34.287349700927734,
      "activations/layer11_attention_weight_min": -31.60389518737793,
      "activations/layer12_attention_weight_max": 36.80017852783203,
      "activations/layer12_attention_weight_min": -24.99323081970215,
      "activations/layer13_attention_weight_max": 39.86616897583008,
      "activations/layer13_attention_weight_min": -34.3173942565918,
      "activations/layer14_attention_weight_max": 52.26502990722656,
      "activations/layer14_attention_weight_min": -42.53647232055664,
      "activations/layer15_attention_weight_max": 37.57435989379883,
      "activations/layer15_attention_weight_min": -31.007213592529297,
      "activations/layer16_attention_weight_max": 33.49518966674805,
      "activations/layer16_attention_weight_min": -27.387968063354492,
      "activations/layer17_attention_weight_max": 34.27367401123047,
      "activations/layer17_attention_weight_min": -25.784313201904297,
      "activations/layer18_attention_weight_max": 30.6563777923584,
      "activations/layer18_attention_weight_min": -21.038150787353516,
      "activations/layer19_attention_weight_max": 37.660491943359375,
      "activations/layer19_attention_weight_min": -31.436315536499023,
      "activations/layer1_attention_weight_max": 16.44342613220215,
      "activations/layer1_attention_weight_min": -13.976491928100586,
      "activations/layer20_attention_weight_max": 32.90169143676758,
      "activations/layer20_attention_weight_min": -23.765453338623047,
      "activations/layer21_attention_weight_max": 29.719297409057617,
      "activations/layer21_attention_weight_min": -23.15628433227539,
      "activations/layer22_attention_weight_max": 46.533348083496094,
      "activations/layer22_attention_weight_min": -28.4415225982666,
      "activations/layer23_attention_weight_max": 35.643157958984375,
      "activations/layer23_attention_weight_min": -26.11540985107422,
      "activations/layer2_attention_weight_max": 32.25374221801758,
      "activations/layer2_attention_weight_min": -32.52132034301758,
      "activations/layer3_attention_weight_max": 86.65385437011719,
      "activations/layer3_attention_weight_min": -92.56119537353516,
      "activations/layer4_attention_weight_max": 111.91632080078125,
      "activations/layer4_attention_weight_min": -116.04341888427734,
      "activations/layer5_attention_weight_max": 52.56074523925781,
      "activations/layer5_attention_weight_min": -59.09550094604492,
      "activations/layer6_attention_weight_max": 49.82485580444336,
      "activations/layer6_attention_weight_min": -53.69591522216797,
      "activations/layer7_attention_weight_max": 87.03861999511719,
      "activations/layer7_attention_weight_min": -90.35096740722656,
      "activations/layer8_attention_weight_max": 41.50109100341797,
      "activations/layer8_attention_weight_min": -42.563655853271484,
      "activations/layer9_attention_weight_max": 39.11084747314453,
      "activations/layer9_attention_weight_min": -35.23809051513672,
      "epoch": 8.38,
      "learning_rate": 9.690454545454546e-05,
      "loss": 2.8068,
      "step": 144300
    },
    {
      "activations/layer0_attention_weight_max": 16.617189407348633,
      "activations/layer0_attention_weight_min": -11.558242797851562,
      "activations/layer10_attention_weight_max": 30.986417770385742,
      "activations/layer10_attention_weight_min": -31.66166114807129,
      "activations/layer11_attention_weight_max": 30.353710174560547,
      "activations/layer11_attention_weight_min": -29.238616943359375,
      "activations/layer12_attention_weight_max": 22.733036041259766,
      "activations/layer12_attention_weight_min": -25.98097038269043,
      "activations/layer13_attention_weight_max": 37.55477523803711,
      "activations/layer13_attention_weight_min": -31.935705184936523,
      "activations/layer14_attention_weight_max": 48.383872985839844,
      "activations/layer14_attention_weight_min": -38.99610900878906,
      "activations/layer15_attention_weight_max": 35.71160125732422,
      "activations/layer15_attention_weight_min": -28.457889556884766,
      "activations/layer16_attention_weight_max": 29.775514602661133,
      "activations/layer16_attention_weight_min": -28.161270141601562,
      "activations/layer17_attention_weight_max": 33.993255615234375,
      "activations/layer17_attention_weight_min": -25.26995277404785,
      "activations/layer18_attention_weight_max": 30.2901554107666,
      "activations/layer18_attention_weight_min": -20.742712020874023,
      "activations/layer19_attention_weight_max": 36.10438537597656,
      "activations/layer19_attention_weight_min": -29.603229522705078,
      "activations/layer1_attention_weight_max": 18.673030853271484,
      "activations/layer1_attention_weight_min": -15.80240535736084,
      "activations/layer20_attention_weight_max": 32.344261169433594,
      "activations/layer20_attention_weight_min": -24.159648895263672,
      "activations/layer21_attention_weight_max": 30.505680084228516,
      "activations/layer21_attention_weight_min": -22.32318115234375,
      "activations/layer22_attention_weight_max": 47.290931701660156,
      "activations/layer22_attention_weight_min": -29.694259643554688,
      "activations/layer23_attention_weight_max": 33.44636535644531,
      "activations/layer23_attention_weight_min": -25.395315170288086,
      "activations/layer2_attention_weight_max": 30.89704704284668,
      "activations/layer2_attention_weight_min": -32.71096420288086,
      "activations/layer3_attention_weight_max": 91.96661376953125,
      "activations/layer3_attention_weight_min": -89.17198181152344,
      "activations/layer4_attention_weight_max": 118.29115295410156,
      "activations/layer4_attention_weight_min": -116.05509185791016,
      "activations/layer5_attention_weight_max": 54.7406120300293,
      "activations/layer5_attention_weight_min": -63.220977783203125,
      "activations/layer6_attention_weight_max": 47.37444305419922,
      "activations/layer6_attention_weight_min": -49.50328826904297,
      "activations/layer7_attention_weight_max": 90.09150695800781,
      "activations/layer7_attention_weight_min": -84.03292083740234,
      "activations/layer8_attention_weight_max": 39.85626983642578,
      "activations/layer8_attention_weight_min": -39.99536895751953,
      "activations/layer9_attention_weight_max": 35.55126190185547,
      "activations/layer9_attention_weight_min": -33.21552276611328,
      "epoch": 8.39,
      "learning_rate": 9.688560606060604e-05,
      "loss": 2.8095,
      "step": 144350
    },
    {
      "activations/layer0_attention_weight_max": 15.247020721435547,
      "activations/layer0_attention_weight_min": -11.376991271972656,
      "activations/layer10_attention_weight_max": 31.21147918701172,
      "activations/layer10_attention_weight_min": -32.876014709472656,
      "activations/layer11_attention_weight_max": 31.853729248046875,
      "activations/layer11_attention_weight_min": -30.424680709838867,
      "activations/layer12_attention_weight_max": 24.974761962890625,
      "activations/layer12_attention_weight_min": -24.66135025024414,
      "activations/layer13_attention_weight_max": 40.16339874267578,
      "activations/layer13_attention_weight_min": -32.614967346191406,
      "activations/layer14_attention_weight_max": 53.414581298828125,
      "activations/layer14_attention_weight_min": -39.83842468261719,
      "activations/layer15_attention_weight_max": 38.10276412963867,
      "activations/layer15_attention_weight_min": -30.626646041870117,
      "activations/layer16_attention_weight_max": 38.98343276977539,
      "activations/layer16_attention_weight_min": -26.68107795715332,
      "activations/layer17_attention_weight_max": 34.0746955871582,
      "activations/layer17_attention_weight_min": -23.690828323364258,
      "activations/layer18_attention_weight_max": 28.2491512298584,
      "activations/layer18_attention_weight_min": -21.055994033813477,
      "activations/layer19_attention_weight_max": 40.9246940612793,
      "activations/layer19_attention_weight_min": -30.097299575805664,
      "activations/layer1_attention_weight_max": 16.99150848388672,
      "activations/layer1_attention_weight_min": -15.674927711486816,
      "activations/layer20_attention_weight_max": 32.523841857910156,
      "activations/layer20_attention_weight_min": -23.481712341308594,
      "activations/layer21_attention_weight_max": 32.21604537963867,
      "activations/layer21_attention_weight_min": -21.071504592895508,
      "activations/layer22_attention_weight_max": 52.42949676513672,
      "activations/layer22_attention_weight_min": -25.862051010131836,
      "activations/layer23_attention_weight_max": 39.793663024902344,
      "activations/layer23_attention_weight_min": -22.079307556152344,
      "activations/layer2_attention_weight_max": 32.667396545410156,
      "activations/layer2_attention_weight_min": -31.651683807373047,
      "activations/layer3_attention_weight_max": 89.12005615234375,
      "activations/layer3_attention_weight_min": -91.32422637939453,
      "activations/layer4_attention_weight_max": 121.1027603149414,
      "activations/layer4_attention_weight_min": -118.96465301513672,
      "activations/layer5_attention_weight_max": 59.983734130859375,
      "activations/layer5_attention_weight_min": -62.528038024902344,
      "activations/layer6_attention_weight_max": 50.30089569091797,
      "activations/layer6_attention_weight_min": -54.502586364746094,
      "activations/layer7_attention_weight_max": 90.4358901977539,
      "activations/layer7_attention_weight_min": -96.83265686035156,
      "activations/layer8_attention_weight_max": 41.160133361816406,
      "activations/layer8_attention_weight_min": -41.52525329589844,
      "activations/layer9_attention_weight_max": 34.02685546875,
      "activations/layer9_attention_weight_min": -34.772735595703125,
      "epoch": 8.39,
      "learning_rate": 9.686666666666666e-05,
      "loss": 2.8155,
      "step": 144400
    },
    {
      "activations/layer0_attention_weight_max": 16.02802276611328,
      "activations/layer0_attention_weight_min": -12.338388442993164,
      "activations/layer10_attention_weight_max": 31.7828426361084,
      "activations/layer10_attention_weight_min": -33.13770294189453,
      "activations/layer11_attention_weight_max": 31.57370376586914,
      "activations/layer11_attention_weight_min": -30.746288299560547,
      "activations/layer12_attention_weight_max": 22.552608489990234,
      "activations/layer12_attention_weight_min": -23.971899032592773,
      "activations/layer13_attention_weight_max": 37.28479766845703,
      "activations/layer13_attention_weight_min": -32.62952423095703,
      "activations/layer14_attention_weight_max": 49.093055725097656,
      "activations/layer14_attention_weight_min": -40.71150207519531,
      "activations/layer15_attention_weight_max": 33.90510559082031,
      "activations/layer15_attention_weight_min": -30.179929733276367,
      "activations/layer16_attention_weight_max": 30.629470825195312,
      "activations/layer16_attention_weight_min": -26.105525970458984,
      "activations/layer17_attention_weight_max": 34.18132019042969,
      "activations/layer17_attention_weight_min": -25.01081085205078,
      "activations/layer18_attention_weight_max": 31.32613754272461,
      "activations/layer18_attention_weight_min": -21.150287628173828,
      "activations/layer19_attention_weight_max": 34.804359436035156,
      "activations/layer19_attention_weight_min": -31.37601089477539,
      "activations/layer1_attention_weight_max": 16.480560302734375,
      "activations/layer1_attention_weight_min": -15.266402244567871,
      "activations/layer20_attention_weight_max": 28.585615158081055,
      "activations/layer20_attention_weight_min": -23.510435104370117,
      "activations/layer21_attention_weight_max": 31.373489379882812,
      "activations/layer21_attention_weight_min": -23.776662826538086,
      "activations/layer22_attention_weight_max": 49.03388214111328,
      "activations/layer22_attention_weight_min": -29.510406494140625,
      "activations/layer23_attention_weight_max": 34.15012741088867,
      "activations/layer23_attention_weight_min": -23.642520904541016,
      "activations/layer2_attention_weight_max": 34.51890182495117,
      "activations/layer2_attention_weight_min": -33.24095153808594,
      "activations/layer3_attention_weight_max": 99.86695861816406,
      "activations/layer3_attention_weight_min": -97.69603729248047,
      "activations/layer4_attention_weight_max": 119.2542495727539,
      "activations/layer4_attention_weight_min": -120.9637680053711,
      "activations/layer5_attention_weight_max": 55.12184143066406,
      "activations/layer5_attention_weight_min": -60.38529586791992,
      "activations/layer6_attention_weight_max": 49.96147918701172,
      "activations/layer6_attention_weight_min": -52.293880462646484,
      "activations/layer7_attention_weight_max": 91.95609283447266,
      "activations/layer7_attention_weight_min": -89.73837280273438,
      "activations/layer8_attention_weight_max": 41.349159240722656,
      "activations/layer8_attention_weight_min": -41.45893096923828,
      "activations/layer9_attention_weight_max": 36.32028579711914,
      "activations/layer9_attention_weight_min": -36.94857406616211,
      "epoch": 8.39,
      "learning_rate": 9.684772727272727e-05,
      "loss": 2.8173,
      "step": 144450
    },
    {
      "activations/layer0_attention_weight_max": 16.31822967529297,
      "activations/layer0_attention_weight_min": -12.663681983947754,
      "activations/layer10_attention_weight_max": 32.84695816040039,
      "activations/layer10_attention_weight_min": -32.426578521728516,
      "activations/layer11_attention_weight_max": 29.61260986328125,
      "activations/layer11_attention_weight_min": -31.059547424316406,
      "activations/layer12_attention_weight_max": 22.343795776367188,
      "activations/layer12_attention_weight_min": -24.705564498901367,
      "activations/layer13_attention_weight_max": 35.84524154663086,
      "activations/layer13_attention_weight_min": -31.664567947387695,
      "activations/layer14_attention_weight_max": 46.290836334228516,
      "activations/layer14_attention_weight_min": -38.24009323120117,
      "activations/layer15_attention_weight_max": 34.713218688964844,
      "activations/layer15_attention_weight_min": -30.580846786499023,
      "activations/layer16_attention_weight_max": 31.654569625854492,
      "activations/layer16_attention_weight_min": -26.284080505371094,
      "activations/layer17_attention_weight_max": 30.76778221130371,
      "activations/layer17_attention_weight_min": -24.010663986206055,
      "activations/layer18_attention_weight_max": 30.5645751953125,
      "activations/layer18_attention_weight_min": -21.719087600708008,
      "activations/layer19_attention_weight_max": 34.623348236083984,
      "activations/layer19_attention_weight_min": -32.09355163574219,
      "activations/layer1_attention_weight_max": 15.982821464538574,
      "activations/layer1_attention_weight_min": -15.575460433959961,
      "activations/layer20_attention_weight_max": 28.699371337890625,
      "activations/layer20_attention_weight_min": -24.709787368774414,
      "activations/layer21_attention_weight_max": 27.830026626586914,
      "activations/layer21_attention_weight_min": -23.55013084411621,
      "activations/layer22_attention_weight_max": 43.13298797607422,
      "activations/layer22_attention_weight_min": -29.524417877197266,
      "activations/layer23_attention_weight_max": 33.36897277832031,
      "activations/layer23_attention_weight_min": -24.534297943115234,
      "activations/layer2_attention_weight_max": 33.356422424316406,
      "activations/layer2_attention_weight_min": -31.07062530517578,
      "activations/layer3_attention_weight_max": 92.11276245117188,
      "activations/layer3_attention_weight_min": -90.42707824707031,
      "activations/layer4_attention_weight_max": 119.83614349365234,
      "activations/layer4_attention_weight_min": -117.05992889404297,
      "activations/layer5_attention_weight_max": 54.242881774902344,
      "activations/layer5_attention_weight_min": -63.044532775878906,
      "activations/layer6_attention_weight_max": 45.270484924316406,
      "activations/layer6_attention_weight_min": -50.53245544433594,
      "activations/layer7_attention_weight_max": 85.61234283447266,
      "activations/layer7_attention_weight_min": -86.24946594238281,
      "activations/layer8_attention_weight_max": 38.631690979003906,
      "activations/layer8_attention_weight_min": -42.39437484741211,
      "activations/layer9_attention_weight_max": 34.55747604370117,
      "activations/layer9_attention_weight_min": -34.0304069519043,
      "epoch": 8.4,
      "learning_rate": 9.682878787878786e-05,
      "loss": 2.8107,
      "step": 144500
    },
    {
      "activations/layer0_attention_weight_max": 16.05787467956543,
      "activations/layer0_attention_weight_min": -12.465204238891602,
      "activations/layer10_attention_weight_max": 33.34255599975586,
      "activations/layer10_attention_weight_min": -31.288951873779297,
      "activations/layer11_attention_weight_max": 33.798583984375,
      "activations/layer11_attention_weight_min": -30.259769439697266,
      "activations/layer12_attention_weight_max": 24.58819007873535,
      "activations/layer12_attention_weight_min": -25.01972198486328,
      "activations/layer13_attention_weight_max": 41.87849426269531,
      "activations/layer13_attention_weight_min": -32.568260192871094,
      "activations/layer14_attention_weight_max": 63.20374298095703,
      "activations/layer14_attention_weight_min": -42.23655319213867,
      "activations/layer15_attention_weight_max": 37.278167724609375,
      "activations/layer15_attention_weight_min": -31.117719650268555,
      "activations/layer16_attention_weight_max": 33.06257247924805,
      "activations/layer16_attention_weight_min": -26.566150665283203,
      "activations/layer17_attention_weight_max": 33.50408172607422,
      "activations/layer17_attention_weight_min": -24.790647506713867,
      "activations/layer18_attention_weight_max": 29.998836517333984,
      "activations/layer18_attention_weight_min": -22.062915802001953,
      "activations/layer19_attention_weight_max": 41.77366256713867,
      "activations/layer19_attention_weight_min": -30.37005615234375,
      "activations/layer1_attention_weight_max": 16.287574768066406,
      "activations/layer1_attention_weight_min": -13.961966514587402,
      "activations/layer20_attention_weight_max": 33.919002532958984,
      "activations/layer20_attention_weight_min": -24.282310485839844,
      "activations/layer21_attention_weight_max": 32.41991424560547,
      "activations/layer21_attention_weight_min": -22.759464263916016,
      "activations/layer22_attention_weight_max": 46.26792907714844,
      "activations/layer22_attention_weight_min": -29.3980770111084,
      "activations/layer23_attention_weight_max": 39.76347351074219,
      "activations/layer23_attention_weight_min": -22.261037826538086,
      "activations/layer2_attention_weight_max": 33.44166946411133,
      "activations/layer2_attention_weight_min": -31.824073791503906,
      "activations/layer3_attention_weight_max": 92.88521575927734,
      "activations/layer3_attention_weight_min": -91.5020751953125,
      "activations/layer4_attention_weight_max": 117.53456115722656,
      "activations/layer4_attention_weight_min": -114.10079193115234,
      "activations/layer5_attention_weight_max": 56.640830993652344,
      "activations/layer5_attention_weight_min": -63.0966796875,
      "activations/layer6_attention_weight_max": 48.00077819824219,
      "activations/layer6_attention_weight_min": -48.94637680053711,
      "activations/layer7_attention_weight_max": 91.75288391113281,
      "activations/layer7_attention_weight_min": -92.6254653930664,
      "activations/layer8_attention_weight_max": 42.731082916259766,
      "activations/layer8_attention_weight_min": -42.878395080566406,
      "activations/layer9_attention_weight_max": 35.89039611816406,
      "activations/layer9_attention_weight_min": -33.5629768371582,
      "epoch": 8.4,
      "learning_rate": 9.680984848484848e-05,
      "loss": 2.8157,
      "step": 144550
    },
    {
      "activations/layer0_attention_weight_max": 16.452911376953125,
      "activations/layer0_attention_weight_min": -12.164666175842285,
      "activations/layer10_attention_weight_max": 29.86431121826172,
      "activations/layer10_attention_weight_min": -29.897050857543945,
      "activations/layer11_attention_weight_max": 32.18484115600586,
      "activations/layer11_attention_weight_min": -28.83857536315918,
      "activations/layer12_attention_weight_max": 23.568796157836914,
      "activations/layer12_attention_weight_min": -30.529788970947266,
      "activations/layer13_attention_weight_max": 36.103206634521484,
      "activations/layer13_attention_weight_min": -32.185150146484375,
      "activations/layer14_attention_weight_max": 45.25909423828125,
      "activations/layer14_attention_weight_min": -35.852901458740234,
      "activations/layer15_attention_weight_max": 32.13109588623047,
      "activations/layer15_attention_weight_min": -30.851160049438477,
      "activations/layer16_attention_weight_max": 29.680082321166992,
      "activations/layer16_attention_weight_min": -27.285781860351562,
      "activations/layer17_attention_weight_max": 32.79853439331055,
      "activations/layer17_attention_weight_min": -24.4155216217041,
      "activations/layer18_attention_weight_max": 29.667699813842773,
      "activations/layer18_attention_weight_min": -23.099180221557617,
      "activations/layer19_attention_weight_max": 32.75594711303711,
      "activations/layer19_attention_weight_min": -29.763708114624023,
      "activations/layer1_attention_weight_max": 16.11921501159668,
      "activations/layer1_attention_weight_min": -14.170347213745117,
      "activations/layer20_attention_weight_max": 27.594276428222656,
      "activations/layer20_attention_weight_min": -23.62630271911621,
      "activations/layer21_attention_weight_max": 27.734832763671875,
      "activations/layer21_attention_weight_min": -21.74811363220215,
      "activations/layer22_attention_weight_max": 43.63216781616211,
      "activations/layer22_attention_weight_min": -29.230052947998047,
      "activations/layer23_attention_weight_max": 33.75950241088867,
      "activations/layer23_attention_weight_min": -24.406721115112305,
      "activations/layer2_attention_weight_max": 33.81602478027344,
      "activations/layer2_attention_weight_min": -31.343677520751953,
      "activations/layer3_attention_weight_max": 90.76111602783203,
      "activations/layer3_attention_weight_min": -90.3482437133789,
      "activations/layer4_attention_weight_max": 116.4741439819336,
      "activations/layer4_attention_weight_min": -115.22869110107422,
      "activations/layer5_attention_weight_max": 52.126930236816406,
      "activations/layer5_attention_weight_min": -62.20159149169922,
      "activations/layer6_attention_weight_max": 44.45793914794922,
      "activations/layer6_attention_weight_min": -49.72160339355469,
      "activations/layer7_attention_weight_max": 85.79925537109375,
      "activations/layer7_attention_weight_min": -86.06962585449219,
      "activations/layer8_attention_weight_max": 37.61430740356445,
      "activations/layer8_attention_weight_min": -37.20523452758789,
      "activations/layer9_attention_weight_max": 32.16074752807617,
      "activations/layer9_attention_weight_min": -32.14671325683594,
      "epoch": 8.4,
      "learning_rate": 9.679090909090908e-05,
      "loss": 2.8119,
      "step": 144600
    },
    {
      "activations/layer0_attention_weight_max": 15.096120834350586,
      "activations/layer0_attention_weight_min": -12.20816421508789,
      "activations/layer10_attention_weight_max": 35.81913375854492,
      "activations/layer10_attention_weight_min": -32.725746154785156,
      "activations/layer11_attention_weight_max": 33.98880386352539,
      "activations/layer11_attention_weight_min": -31.12875747680664,
      "activations/layer12_attention_weight_max": 24.404863357543945,
      "activations/layer12_attention_weight_min": -24.471744537353516,
      "activations/layer13_attention_weight_max": 41.97770309448242,
      "activations/layer13_attention_weight_min": -34.202632904052734,
      "activations/layer14_attention_weight_max": 50.75784683227539,
      "activations/layer14_attention_weight_min": -41.20985412597656,
      "activations/layer15_attention_weight_max": 39.19229507446289,
      "activations/layer15_attention_weight_min": -29.750993728637695,
      "activations/layer16_attention_weight_max": 35.282264709472656,
      "activations/layer16_attention_weight_min": -28.252092361450195,
      "activations/layer17_attention_weight_max": 36.06599807739258,
      "activations/layer17_attention_weight_min": -24.38003158569336,
      "activations/layer18_attention_weight_max": 29.316072463989258,
      "activations/layer18_attention_weight_min": -20.613544464111328,
      "activations/layer19_attention_weight_max": 37.17646408081055,
      "activations/layer19_attention_weight_min": -29.042484283447266,
      "activations/layer1_attention_weight_max": 16.061912536621094,
      "activations/layer1_attention_weight_min": -15.730360984802246,
      "activations/layer20_attention_weight_max": 34.27133560180664,
      "activations/layer20_attention_weight_min": -25.048044204711914,
      "activations/layer21_attention_weight_max": 30.739639282226562,
      "activations/layer21_attention_weight_min": -23.691869735717773,
      "activations/layer22_attention_weight_max": 48.6602668762207,
      "activations/layer22_attention_weight_min": -28.096372604370117,
      "activations/layer23_attention_weight_max": 38.6794548034668,
      "activations/layer23_attention_weight_min": -22.324918746948242,
      "activations/layer2_attention_weight_max": 32.41228485107422,
      "activations/layer2_attention_weight_min": -32.16242218017578,
      "activations/layer3_attention_weight_max": 92.56376647949219,
      "activations/layer3_attention_weight_min": -100.82180786132812,
      "activations/layer4_attention_weight_max": 127.24810791015625,
      "activations/layer4_attention_weight_min": -119.2349853515625,
      "activations/layer5_attention_weight_max": 58.69300842285156,
      "activations/layer5_attention_weight_min": -61.95534896850586,
      "activations/layer6_attention_weight_max": 50.7329216003418,
      "activations/layer6_attention_weight_min": -51.64938735961914,
      "activations/layer7_attention_weight_max": 90.85993194580078,
      "activations/layer7_attention_weight_min": -92.454345703125,
      "activations/layer8_attention_weight_max": 41.9334716796875,
      "activations/layer8_attention_weight_min": -41.27449417114258,
      "activations/layer9_attention_weight_max": 38.951900482177734,
      "activations/layer9_attention_weight_min": -34.7696647644043,
      "epoch": 8.4,
      "learning_rate": 9.677196969696969e-05,
      "loss": 2.805,
      "step": 144650
    },
    {
      "activations/layer0_attention_weight_max": 15.851298332214355,
      "activations/layer0_attention_weight_min": -12.536022186279297,
      "activations/layer10_attention_weight_max": 32.2424430847168,
      "activations/layer10_attention_weight_min": -30.270889282226562,
      "activations/layer11_attention_weight_max": 31.140188217163086,
      "activations/layer11_attention_weight_min": -29.004545211791992,
      "activations/layer12_attention_weight_max": 21.744516372680664,
      "activations/layer12_attention_weight_min": -24.938791275024414,
      "activations/layer13_attention_weight_max": 34.75308609008789,
      "activations/layer13_attention_weight_min": -32.235191345214844,
      "activations/layer14_attention_weight_max": 45.92464065551758,
      "activations/layer14_attention_weight_min": -37.91719436645508,
      "activations/layer15_attention_weight_max": 34.90477752685547,
      "activations/layer15_attention_weight_min": -29.17338752746582,
      "activations/layer16_attention_weight_max": 28.918121337890625,
      "activations/layer16_attention_weight_min": -30.07697296142578,
      "activations/layer17_attention_weight_max": 35.10795974731445,
      "activations/layer17_attention_weight_min": -26.09745216369629,
      "activations/layer18_attention_weight_max": 31.531978607177734,
      "activations/layer18_attention_weight_min": -22.290369033813477,
      "activations/layer19_attention_weight_max": 36.08213424682617,
      "activations/layer19_attention_weight_min": -28.829526901245117,
      "activations/layer1_attention_weight_max": 16.778657913208008,
      "activations/layer1_attention_weight_min": -12.64765453338623,
      "activations/layer20_attention_weight_max": 33.414306640625,
      "activations/layer20_attention_weight_min": -23.951753616333008,
      "activations/layer21_attention_weight_max": 32.89525604248047,
      "activations/layer21_attention_weight_min": -23.30558204650879,
      "activations/layer22_attention_weight_max": 52.125335693359375,
      "activations/layer22_attention_weight_min": -29.60616683959961,
      "activations/layer23_attention_weight_max": 42.00537109375,
      "activations/layer23_attention_weight_min": -22.459579467773438,
      "activations/layer2_attention_weight_max": 30.190513610839844,
      "activations/layer2_attention_weight_min": -30.255855560302734,
      "activations/layer3_attention_weight_max": 90.15428924560547,
      "activations/layer3_attention_weight_min": -89.33927917480469,
      "activations/layer4_attention_weight_max": 114.7000961303711,
      "activations/layer4_attention_weight_min": -112.61246490478516,
      "activations/layer5_attention_weight_max": 53.76780319213867,
      "activations/layer5_attention_weight_min": -60.245361328125,
      "activations/layer6_attention_weight_max": 47.18193817138672,
      "activations/layer6_attention_weight_min": -48.846797943115234,
      "activations/layer7_attention_weight_max": 89.84834289550781,
      "activations/layer7_attention_weight_min": -83.96232604980469,
      "activations/layer8_attention_weight_max": 40.7606086730957,
      "activations/layer8_attention_weight_min": -38.38363265991211,
      "activations/layer9_attention_weight_max": 32.48485565185547,
      "activations/layer9_attention_weight_min": -31.53602409362793,
      "epoch": 8.41,
      "learning_rate": 9.67530303030303e-05,
      "loss": 2.8091,
      "step": 144700
    },
    {
      "activations/layer0_attention_weight_max": 15.91023063659668,
      "activations/layer0_attention_weight_min": -12.506970405578613,
      "activations/layer10_attention_weight_max": 31.877899169921875,
      "activations/layer10_attention_weight_min": -32.1048469543457,
      "activations/layer11_attention_weight_max": 31.3382625579834,
      "activations/layer11_attention_weight_min": -30.512569427490234,
      "activations/layer12_attention_weight_max": 22.46477508544922,
      "activations/layer12_attention_weight_min": -25.282896041870117,
      "activations/layer13_attention_weight_max": 36.722747802734375,
      "activations/layer13_attention_weight_min": -31.24087905883789,
      "activations/layer14_attention_weight_max": 45.183475494384766,
      "activations/layer14_attention_weight_min": -39.245853424072266,
      "activations/layer15_attention_weight_max": 35.11026382446289,
      "activations/layer15_attention_weight_min": -29.252832412719727,
      "activations/layer16_attention_weight_max": 30.612991333007812,
      "activations/layer16_attention_weight_min": -26.038042068481445,
      "activations/layer17_attention_weight_max": 35.94813919067383,
      "activations/layer17_attention_weight_min": -23.84263038635254,
      "activations/layer18_attention_weight_max": 32.78200149536133,
      "activations/layer18_attention_weight_min": -20.807783126831055,
      "activations/layer19_attention_weight_max": 35.93759536743164,
      "activations/layer19_attention_weight_min": -29.756534576416016,
      "activations/layer1_attention_weight_max": 15.86009407043457,
      "activations/layer1_attention_weight_min": -13.998324394226074,
      "activations/layer20_attention_weight_max": 33.232303619384766,
      "activations/layer20_attention_weight_min": -24.46294593811035,
      "activations/layer21_attention_weight_max": 33.019710540771484,
      "activations/layer21_attention_weight_min": -23.317102432250977,
      "activations/layer22_attention_weight_max": 47.176361083984375,
      "activations/layer22_attention_weight_min": -28.50489616394043,
      "activations/layer23_attention_weight_max": 34.912940979003906,
      "activations/layer23_attention_weight_min": -24.159988403320312,
      "activations/layer2_attention_weight_max": 30.70821189880371,
      "activations/layer2_attention_weight_min": -30.89417266845703,
      "activations/layer3_attention_weight_max": 88.12967681884766,
      "activations/layer3_attention_weight_min": -92.83316040039062,
      "activations/layer4_attention_weight_max": 115.66524505615234,
      "activations/layer4_attention_weight_min": -117.8015365600586,
      "activations/layer5_attention_weight_max": 53.62833786010742,
      "activations/layer5_attention_weight_min": -64.05192565917969,
      "activations/layer6_attention_weight_max": 45.61886978149414,
      "activations/layer6_attention_weight_min": -51.95285415649414,
      "activations/layer7_attention_weight_max": 87.55196380615234,
      "activations/layer7_attention_weight_min": -87.92288970947266,
      "activations/layer8_attention_weight_max": 37.893775939941406,
      "activations/layer8_attention_weight_min": -39.04964065551758,
      "activations/layer9_attention_weight_max": 32.181034088134766,
      "activations/layer9_attention_weight_min": -31.97845458984375,
      "epoch": 8.41,
      "learning_rate": 9.67340909090909e-05,
      "loss": 2.8186,
      "step": 144750
    },
    {
      "activations/layer0_attention_weight_max": 15.796822547912598,
      "activations/layer0_attention_weight_min": -11.552827835083008,
      "activations/layer10_attention_weight_max": 34.595855712890625,
      "activations/layer10_attention_weight_min": -31.49880599975586,
      "activations/layer11_attention_weight_max": 32.43663787841797,
      "activations/layer11_attention_weight_min": -29.514251708984375,
      "activations/layer12_attention_weight_max": 23.99355697631836,
      "activations/layer12_attention_weight_min": -34.09504699707031,
      "activations/layer13_attention_weight_max": 41.33076858520508,
      "activations/layer13_attention_weight_min": -31.79276466369629,
      "activations/layer14_attention_weight_max": 45.23785400390625,
      "activations/layer14_attention_weight_min": -35.97660827636719,
      "activations/layer15_attention_weight_max": 35.961585998535156,
      "activations/layer15_attention_weight_min": -28.137815475463867,
      "activations/layer16_attention_weight_max": 32.97829055786133,
      "activations/layer16_attention_weight_min": -27.378555297851562,
      "activations/layer17_attention_weight_max": 33.656402587890625,
      "activations/layer17_attention_weight_min": -23.742158889770508,
      "activations/layer18_attention_weight_max": 30.85564613342285,
      "activations/layer18_attention_weight_min": -24.2288818359375,
      "activations/layer19_attention_weight_max": 34.17164993286133,
      "activations/layer19_attention_weight_min": -27.437366485595703,
      "activations/layer1_attention_weight_max": 17.818044662475586,
      "activations/layer1_attention_weight_min": -16.63787841796875,
      "activations/layer20_attention_weight_max": 31.57573699951172,
      "activations/layer20_attention_weight_min": -24.565950393676758,
      "activations/layer21_attention_weight_max": 30.27904510498047,
      "activations/layer21_attention_weight_min": -22.462215423583984,
      "activations/layer22_attention_weight_max": 44.22331237792969,
      "activations/layer22_attention_weight_min": -26.709821701049805,
      "activations/layer23_attention_weight_max": 35.029449462890625,
      "activations/layer23_attention_weight_min": -22.4727840423584,
      "activations/layer2_attention_weight_max": 35.19169616699219,
      "activations/layer2_attention_weight_min": -34.25679016113281,
      "activations/layer3_attention_weight_max": 96.25850677490234,
      "activations/layer3_attention_weight_min": -95.50166320800781,
      "activations/layer4_attention_weight_max": 115.3360366821289,
      "activations/layer4_attention_weight_min": -115.32915496826172,
      "activations/layer5_attention_weight_max": 55.37641906738281,
      "activations/layer5_attention_weight_min": -60.812129974365234,
      "activations/layer6_attention_weight_max": 47.99711608886719,
      "activations/layer6_attention_weight_min": -50.29843521118164,
      "activations/layer7_attention_weight_max": 86.83772277832031,
      "activations/layer7_attention_weight_min": -87.98933410644531,
      "activations/layer8_attention_weight_max": 40.9684944152832,
      "activations/layer8_attention_weight_min": -38.276268005371094,
      "activations/layer9_attention_weight_max": 34.63606262207031,
      "activations/layer9_attention_weight_min": -39.02927780151367,
      "epoch": 8.41,
      "learning_rate": 9.671515151515151e-05,
      "loss": 2.8073,
      "step": 144800
    },
    {
      "activations/layer0_attention_weight_max": 14.710392951965332,
      "activations/layer0_attention_weight_min": -11.826010704040527,
      "activations/layer10_attention_weight_max": 34.069664001464844,
      "activations/layer10_attention_weight_min": -32.52464294433594,
      "activations/layer11_attention_weight_max": 36.225746154785156,
      "activations/layer11_attention_weight_min": -31.83834457397461,
      "activations/layer12_attention_weight_max": 28.939241409301758,
      "activations/layer12_attention_weight_min": -25.982276916503906,
      "activations/layer13_attention_weight_max": 43.344600677490234,
      "activations/layer13_attention_weight_min": -35.140647888183594,
      "activations/layer14_attention_weight_max": 60.02289962768555,
      "activations/layer14_attention_weight_min": -42.534423828125,
      "activations/layer15_attention_weight_max": 48.83918762207031,
      "activations/layer15_attention_weight_min": -32.15837478637695,
      "activations/layer16_attention_weight_max": 34.13820266723633,
      "activations/layer16_attention_weight_min": -28.65835189819336,
      "activations/layer17_attention_weight_max": 36.33264923095703,
      "activations/layer17_attention_weight_min": -25.44087028503418,
      "activations/layer18_attention_weight_max": 30.577007293701172,
      "activations/layer18_attention_weight_min": -23.552900314331055,
      "activations/layer19_attention_weight_max": 36.879093170166016,
      "activations/layer19_attention_weight_min": -31.738250732421875,
      "activations/layer1_attention_weight_max": 17.479942321777344,
      "activations/layer1_attention_weight_min": -14.693938255310059,
      "activations/layer20_attention_weight_max": 30.978986740112305,
      "activations/layer20_attention_weight_min": -25.128822326660156,
      "activations/layer21_attention_weight_max": 30.379911422729492,
      "activations/layer21_attention_weight_min": -22.28005027770996,
      "activations/layer22_attention_weight_max": 57.57796096801758,
      "activations/layer22_attention_weight_min": -29.678815841674805,
      "activations/layer23_attention_weight_max": 37.10464096069336,
      "activations/layer23_attention_weight_min": -22.373037338256836,
      "activations/layer2_attention_weight_max": 33.26014709472656,
      "activations/layer2_attention_weight_min": -32.91985321044922,
      "activations/layer3_attention_weight_max": 91.44544219970703,
      "activations/layer3_attention_weight_min": -96.46028900146484,
      "activations/layer4_attention_weight_max": 118.90592193603516,
      "activations/layer4_attention_weight_min": -117.96739959716797,
      "activations/layer5_attention_weight_max": 56.36843490600586,
      "activations/layer5_attention_weight_min": -61.92716598510742,
      "activations/layer6_attention_weight_max": 51.77985763549805,
      "activations/layer6_attention_weight_min": -50.3540153503418,
      "activations/layer7_attention_weight_max": 91.28762817382812,
      "activations/layer7_attention_weight_min": -85.25260925292969,
      "activations/layer8_attention_weight_max": 41.8343620300293,
      "activations/layer8_attention_weight_min": -42.02885055541992,
      "activations/layer9_attention_weight_max": 33.973751068115234,
      "activations/layer9_attention_weight_min": -33.00102615356445,
      "epoch": 8.42,
      "learning_rate": 9.669621212121211e-05,
      "loss": 2.7909,
      "step": 144850
    },
    {
      "activations/layer0_attention_weight_max": 15.995366096496582,
      "activations/layer0_attention_weight_min": -13.091424942016602,
      "activations/layer10_attention_weight_max": 32.37897872924805,
      "activations/layer10_attention_weight_min": -30.62999725341797,
      "activations/layer11_attention_weight_max": 33.16486740112305,
      "activations/layer11_attention_weight_min": -29.854671478271484,
      "activations/layer12_attention_weight_max": 23.1803035736084,
      "activations/layer12_attention_weight_min": -24.987964630126953,
      "activations/layer13_attention_weight_max": 38.864463806152344,
      "activations/layer13_attention_weight_min": -34.013877868652344,
      "activations/layer14_attention_weight_max": 47.26383972167969,
      "activations/layer14_attention_weight_min": -41.094200134277344,
      "activations/layer15_attention_weight_max": 37.99251174926758,
      "activations/layer15_attention_weight_min": -30.996322631835938,
      "activations/layer16_attention_weight_max": 31.72998046875,
      "activations/layer16_attention_weight_min": -26.83046531677246,
      "activations/layer17_attention_weight_max": 33.15485763549805,
      "activations/layer17_attention_weight_min": -25.954586029052734,
      "activations/layer18_attention_weight_max": 33.208213806152344,
      "activations/layer18_attention_weight_min": -21.93082618713379,
      "activations/layer19_attention_weight_max": 35.54872512817383,
      "activations/layer19_attention_weight_min": -29.95538330078125,
      "activations/layer1_attention_weight_max": 16.18181610107422,
      "activations/layer1_attention_weight_min": -14.575372695922852,
      "activations/layer20_attention_weight_max": 28.78776741027832,
      "activations/layer20_attention_weight_min": -24.540250778198242,
      "activations/layer21_attention_weight_max": 29.39438247680664,
      "activations/layer21_attention_weight_min": -24.944053649902344,
      "activations/layer22_attention_weight_max": 45.43833541870117,
      "activations/layer22_attention_weight_min": -29.223373413085938,
      "activations/layer23_attention_weight_max": 40.24138641357422,
      "activations/layer23_attention_weight_min": -24.13071060180664,
      "activations/layer2_attention_weight_max": 32.40182113647461,
      "activations/layer2_attention_weight_min": -32.02729034423828,
      "activations/layer3_attention_weight_max": 88.0982666015625,
      "activations/layer3_attention_weight_min": -88.74663543701172,
      "activations/layer4_attention_weight_max": 115.18289947509766,
      "activations/layer4_attention_weight_min": -116.90233612060547,
      "activations/layer5_attention_weight_max": 53.86763000488281,
      "activations/layer5_attention_weight_min": -61.321617126464844,
      "activations/layer6_attention_weight_max": 47.10920715332031,
      "activations/layer6_attention_weight_min": -51.73276138305664,
      "activations/layer7_attention_weight_max": 83.23583984375,
      "activations/layer7_attention_weight_min": -88.97756958007812,
      "activations/layer8_attention_weight_max": 40.245357513427734,
      "activations/layer8_attention_weight_min": -40.94767761230469,
      "activations/layer9_attention_weight_max": 36.511383056640625,
      "activations/layer9_attention_weight_min": -33.78740692138672,
      "epoch": 8.42,
      "learning_rate": 9.667727272727271e-05,
      "loss": 2.8054,
      "step": 144900
    },
    {
      "activations/layer0_attention_weight_max": 15.939281463623047,
      "activations/layer0_attention_weight_min": -12.328198432922363,
      "activations/layer10_attention_weight_max": 34.69560623168945,
      "activations/layer10_attention_weight_min": -31.868844985961914,
      "activations/layer11_attention_weight_max": 33.38829803466797,
      "activations/layer11_attention_weight_min": -31.60029411315918,
      "activations/layer12_attention_weight_max": 24.14642906188965,
      "activations/layer12_attention_weight_min": -24.27751350402832,
      "activations/layer13_attention_weight_max": 45.45795822143555,
      "activations/layer13_attention_weight_min": -34.335243225097656,
      "activations/layer14_attention_weight_max": 55.452362060546875,
      "activations/layer14_attention_weight_min": -40.9020881652832,
      "activations/layer15_attention_weight_max": 35.1142692565918,
      "activations/layer15_attention_weight_min": -30.01874351501465,
      "activations/layer16_attention_weight_max": 33.69074630737305,
      "activations/layer16_attention_weight_min": -28.816335678100586,
      "activations/layer17_attention_weight_max": 34.97433090209961,
      "activations/layer17_attention_weight_min": -24.951553344726562,
      "activations/layer18_attention_weight_max": 33.00324630737305,
      "activations/layer18_attention_weight_min": -21.5637264251709,
      "activations/layer19_attention_weight_max": 38.856632232666016,
      "activations/layer19_attention_weight_min": -31.16278648376465,
      "activations/layer1_attention_weight_max": 16.20326805114746,
      "activations/layer1_attention_weight_min": -16.01032066345215,
      "activations/layer20_attention_weight_max": 33.95347595214844,
      "activations/layer20_attention_weight_min": -23.5341739654541,
      "activations/layer21_attention_weight_max": 30.450510025024414,
      "activations/layer21_attention_weight_min": -21.855043411254883,
      "activations/layer22_attention_weight_max": 46.60725402832031,
      "activations/layer22_attention_weight_min": -27.830333709716797,
      "activations/layer23_attention_weight_max": 38.11137390136719,
      "activations/layer23_attention_weight_min": -25.609699249267578,
      "activations/layer2_attention_weight_max": 32.85778045654297,
      "activations/layer2_attention_weight_min": -33.18333435058594,
      "activations/layer3_attention_weight_max": 95.9304428100586,
      "activations/layer3_attention_weight_min": -95.9451675415039,
      "activations/layer4_attention_weight_max": 125.2074966430664,
      "activations/layer4_attention_weight_min": -120.53282928466797,
      "activations/layer5_attention_weight_max": 57.68707275390625,
      "activations/layer5_attention_weight_min": -62.54288864135742,
      "activations/layer6_attention_weight_max": 52.05982208251953,
      "activations/layer6_attention_weight_min": -53.41030502319336,
      "activations/layer7_attention_weight_max": 93.12947845458984,
      "activations/layer7_attention_weight_min": -96.6551513671875,
      "activations/layer8_attention_weight_max": 39.822120666503906,
      "activations/layer8_attention_weight_min": -42.56877136230469,
      "activations/layer9_attention_weight_max": 38.674930572509766,
      "activations/layer9_attention_weight_min": -33.337547302246094,
      "epoch": 8.42,
      "learning_rate": 9.66587121212121e-05,
      "loss": 2.8125,
      "step": 144950
    },
    {
      "activations/layer0_attention_weight_max": 15.104825019836426,
      "activations/layer0_attention_weight_min": -12.183122634887695,
      "activations/layer10_attention_weight_max": 36.76935958862305,
      "activations/layer10_attention_weight_min": -36.96137237548828,
      "activations/layer11_attention_weight_max": 37.300437927246094,
      "activations/layer11_attention_weight_min": -35.440345764160156,
      "activations/layer12_attention_weight_max": 35.55950164794922,
      "activations/layer12_attention_weight_min": -25.857831954956055,
      "activations/layer13_attention_weight_max": 46.59354019165039,
      "activations/layer13_attention_weight_min": -34.895694732666016,
      "activations/layer14_attention_weight_max": 58.61310958862305,
      "activations/layer14_attention_weight_min": -40.558284759521484,
      "activations/layer15_attention_weight_max": 43.47604751586914,
      "activations/layer15_attention_weight_min": -33.91572189331055,
      "activations/layer16_attention_weight_max": 39.425086975097656,
      "activations/layer16_attention_weight_min": -28.56227684020996,
      "activations/layer17_attention_weight_max": 42.466007232666016,
      "activations/layer17_attention_weight_min": -26.740259170532227,
      "activations/layer18_attention_weight_max": 37.38804626464844,
      "activations/layer18_attention_weight_min": -22.745811462402344,
      "activations/layer19_attention_weight_max": 43.64264678955078,
      "activations/layer19_attention_weight_min": -30.70001220703125,
      "activations/layer1_attention_weight_max": 16.54484748840332,
      "activations/layer1_attention_weight_min": -15.036493301391602,
      "activations/layer20_attention_weight_max": 37.41596603393555,
      "activations/layer20_attention_weight_min": -26.150728225708008,
      "activations/layer21_attention_weight_max": 35.6137580871582,
      "activations/layer21_attention_weight_min": -23.1650390625,
      "activations/layer22_attention_weight_max": 55.0822639465332,
      "activations/layer22_attention_weight_min": -31.137882232666016,
      "activations/layer23_attention_weight_max": 47.19733810424805,
      "activations/layer23_attention_weight_min": -25.678129196166992,
      "activations/layer2_attention_weight_max": 30.6873836517334,
      "activations/layer2_attention_weight_min": -31.108150482177734,
      "activations/layer3_attention_weight_max": 93.06608581542969,
      "activations/layer3_attention_weight_min": -95.31550598144531,
      "activations/layer4_attention_weight_max": 123.20616912841797,
      "activations/layer4_attention_weight_min": -118.8438720703125,
      "activations/layer5_attention_weight_max": 58.32273483276367,
      "activations/layer5_attention_weight_min": -63.46607208251953,
      "activations/layer6_attention_weight_max": 49.20625305175781,
      "activations/layer6_attention_weight_min": -53.12434387207031,
      "activations/layer7_attention_weight_max": 92.30902099609375,
      "activations/layer7_attention_weight_min": -92.32410430908203,
      "activations/layer8_attention_weight_max": 43.56214904785156,
      "activations/layer8_attention_weight_min": -46.101768493652344,
      "activations/layer9_attention_weight_max": 39.8018913269043,
      "activations/layer9_attention_weight_min": -36.52042770385742,
      "epoch": 8.43,
      "learning_rate": 9.663977272727272e-05,
      "loss": 2.814,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.529,
      "eval_samples_per_second": 503.462,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.529,
      "eval_openwebtext_samples_per_second": 503.462,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0395,
      "eval_wikitext_samples_per_second": 223.583,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_lambada_loss": 2.6484375,
      "eval_lambada_ppl": 14.131940228876227,
      "eval_lambada_runtime": 9.6214,
      "eval_lambada_samples_per_second": 506.059,
      "step": 145000
    },
    {
      "activations/layer0_attention_weight_max": 15.344168663024902,
      "activations/layer0_attention_weight_min": -11.889317512512207,
      "activations/layer10_attention_weight_max": 32.76371765136719,
      "activations/layer10_attention_weight_min": -31.616390228271484,
      "activations/layer11_attention_weight_max": 31.785985946655273,
      "activations/layer11_attention_weight_min": -29.4263973236084,
      "activations/layer12_attention_weight_max": 23.683292388916016,
      "activations/layer12_attention_weight_min": -26.29829216003418,
      "activations/layer13_attention_weight_max": 41.55154037475586,
      "activations/layer13_attention_weight_min": -34.46999740600586,
      "activations/layer14_attention_weight_max": 48.120426177978516,
      "activations/layer14_attention_weight_min": -40.780574798583984,
      "activations/layer15_attention_weight_max": 37.15155029296875,
      "activations/layer15_attention_weight_min": -31.18610191345215,
      "activations/layer16_attention_weight_max": 33.037601470947266,
      "activations/layer16_attention_weight_min": -26.936824798583984,
      "activations/layer17_attention_weight_max": 32.24461364746094,
      "activations/layer17_attention_weight_min": -24.709802627563477,
      "activations/layer18_attention_weight_max": 29.70167350769043,
      "activations/layer18_attention_weight_min": -20.58582878112793,
      "activations/layer19_attention_weight_max": 37.049869537353516,
      "activations/layer19_attention_weight_min": -29.102710723876953,
      "activations/layer1_attention_weight_max": 16.965131759643555,
      "activations/layer1_attention_weight_min": -15.842405319213867,
      "activations/layer20_attention_weight_max": 30.29341697692871,
      "activations/layer20_attention_weight_min": -23.61406707763672,
      "activations/layer21_attention_weight_max": 29.729812622070312,
      "activations/layer21_attention_weight_min": -22.199270248413086,
      "activations/layer22_attention_weight_max": 45.662620544433594,
      "activations/layer22_attention_weight_min": -28.487627029418945,
      "activations/layer23_attention_weight_max": 35.193450927734375,
      "activations/layer23_attention_weight_min": -22.800390243530273,
      "activations/layer2_attention_weight_max": 31.348430633544922,
      "activations/layer2_attention_weight_min": -32.07352066040039,
      "activations/layer3_attention_weight_max": 88.24337005615234,
      "activations/layer3_attention_weight_min": -91.05677795410156,
      "activations/layer4_attention_weight_max": 111.0971908569336,
      "activations/layer4_attention_weight_min": -113.14066314697266,
      "activations/layer5_attention_weight_max": 50.94395065307617,
      "activations/layer5_attention_weight_min": -62.208518981933594,
      "activations/layer6_attention_weight_max": 46.95592498779297,
      "activations/layer6_attention_weight_min": -52.3504753112793,
      "activations/layer7_attention_weight_max": 87.8523941040039,
      "activations/layer7_attention_weight_min": -89.4689712524414,
      "activations/layer8_attention_weight_max": 40.48076248168945,
      "activations/layer8_attention_weight_min": -40.999488830566406,
      "activations/layer9_attention_weight_max": 37.676239013671875,
      "activations/layer9_attention_weight_min": -33.15334701538086,
      "epoch": 8.43,
      "learning_rate": 9.662083333333334e-05,
      "loss": 2.8092,
      "step": 145050
    },
    {
      "activations/layer0_attention_weight_max": 16.383512496948242,
      "activations/layer0_attention_weight_min": -12.867392539978027,
      "activations/layer10_attention_weight_max": 33.05200958251953,
      "activations/layer10_attention_weight_min": -32.677764892578125,
      "activations/layer11_attention_weight_max": 34.89109802246094,
      "activations/layer11_attention_weight_min": -30.36163330078125,
      "activations/layer12_attention_weight_max": 23.742610931396484,
      "activations/layer12_attention_weight_min": -25.540857315063477,
      "activations/layer13_attention_weight_max": 44.12694549560547,
      "activations/layer13_attention_weight_min": -34.99522399902344,
      "activations/layer14_attention_weight_max": 44.2878303527832,
      "activations/layer14_attention_weight_min": -38.8787727355957,
      "activations/layer15_attention_weight_max": 35.292903900146484,
      "activations/layer15_attention_weight_min": -31.290443420410156,
      "activations/layer16_attention_weight_max": 31.09740447998047,
      "activations/layer16_attention_weight_min": -28.23631477355957,
      "activations/layer17_attention_weight_max": 32.61008834838867,
      "activations/layer17_attention_weight_min": -25.33497428894043,
      "activations/layer18_attention_weight_max": 33.12220764160156,
      "activations/layer18_attention_weight_min": -22.178422927856445,
      "activations/layer19_attention_weight_max": 33.530418395996094,
      "activations/layer19_attention_weight_min": -30.581396102905273,
      "activations/layer1_attention_weight_max": 16.649967193603516,
      "activations/layer1_attention_weight_min": -16.09079933166504,
      "activations/layer20_attention_weight_max": 28.019624710083008,
      "activations/layer20_attention_weight_min": -24.021373748779297,
      "activations/layer21_attention_weight_max": 32.80320739746094,
      "activations/layer21_attention_weight_min": -24.146759033203125,
      "activations/layer22_attention_weight_max": 42.06587219238281,
      "activations/layer22_attention_weight_min": -28.443790435791016,
      "activations/layer23_attention_weight_max": 34.369747161865234,
      "activations/layer23_attention_weight_min": -23.41450309753418,
      "activations/layer2_attention_weight_max": 30.924976348876953,
      "activations/layer2_attention_weight_min": -32.15440368652344,
      "activations/layer3_attention_weight_max": 89.88863372802734,
      "activations/layer3_attention_weight_min": -89.8788833618164,
      "activations/layer4_attention_weight_max": 117.2413330078125,
      "activations/layer4_attention_weight_min": -112.6689453125,
      "activations/layer5_attention_weight_max": 55.509273529052734,
      "activations/layer5_attention_weight_min": -62.00350570678711,
      "activations/layer6_attention_weight_max": 47.54471206665039,
      "activations/layer6_attention_weight_min": -51.810203552246094,
      "activations/layer7_attention_weight_max": 88.48069763183594,
      "activations/layer7_attention_weight_min": -86.70740509033203,
      "activations/layer8_attention_weight_max": 40.31522750854492,
      "activations/layer8_attention_weight_min": -41.22043228149414,
      "activations/layer9_attention_weight_max": 36.48841094970703,
      "activations/layer9_attention_weight_min": -34.34623718261719,
      "epoch": 8.43,
      "learning_rate": 9.660189393939392e-05,
      "loss": 2.8035,
      "step": 145100
    },
    {
      "activations/layer0_attention_weight_max": 15.717057228088379,
      "activations/layer0_attention_weight_min": -11.24114990234375,
      "activations/layer10_attention_weight_max": 34.77262496948242,
      "activations/layer10_attention_weight_min": -31.079160690307617,
      "activations/layer11_attention_weight_max": 33.89344024658203,
      "activations/layer11_attention_weight_min": -29.698688507080078,
      "activations/layer12_attention_weight_max": 24.51700210571289,
      "activations/layer12_attention_weight_min": -24.068721771240234,
      "activations/layer13_attention_weight_max": 43.72758483886719,
      "activations/layer13_attention_weight_min": -33.67232131958008,
      "activations/layer14_attention_weight_max": 48.54295349121094,
      "activations/layer14_attention_weight_min": -36.51712417602539,
      "activations/layer15_attention_weight_max": 39.054344177246094,
      "activations/layer15_attention_weight_min": -30.843280792236328,
      "activations/layer16_attention_weight_max": 36.0854606628418,
      "activations/layer16_attention_weight_min": -28.272220611572266,
      "activations/layer17_attention_weight_max": 37.6573371887207,
      "activations/layer17_attention_weight_min": -27.298175811767578,
      "activations/layer18_attention_weight_max": 34.60512924194336,
      "activations/layer18_attention_weight_min": -23.009544372558594,
      "activations/layer19_attention_weight_max": 44.43971633911133,
      "activations/layer19_attention_weight_min": -33.03644943237305,
      "activations/layer1_attention_weight_max": 16.018083572387695,
      "activations/layer1_attention_weight_min": -14.7654390335083,
      "activations/layer20_attention_weight_max": 34.95132827758789,
      "activations/layer20_attention_weight_min": -24.357133865356445,
      "activations/layer21_attention_weight_max": 34.34872055053711,
      "activations/layer21_attention_weight_min": -21.497051239013672,
      "activations/layer22_attention_weight_max": 61.20625305175781,
      "activations/layer22_attention_weight_min": -29.031511306762695,
      "activations/layer23_attention_weight_max": 45.33256530761719,
      "activations/layer23_attention_weight_min": -24.41537857055664,
      "activations/layer2_attention_weight_max": 31.682310104370117,
      "activations/layer2_attention_weight_min": -31.511940002441406,
      "activations/layer3_attention_weight_max": 90.44567108154297,
      "activations/layer3_attention_weight_min": -89.74857330322266,
      "activations/layer4_attention_weight_max": 116.43033599853516,
      "activations/layer4_attention_weight_min": -117.71160888671875,
      "activations/layer5_attention_weight_max": 56.283363342285156,
      "activations/layer5_attention_weight_min": -64.29579162597656,
      "activations/layer6_attention_weight_max": 49.30541229248047,
      "activations/layer6_attention_weight_min": -51.76628494262695,
      "activations/layer7_attention_weight_max": 89.66934967041016,
      "activations/layer7_attention_weight_min": -86.75696563720703,
      "activations/layer8_attention_weight_max": 40.85425567626953,
      "activations/layer8_attention_weight_min": -40.24654769897461,
      "activations/layer9_attention_weight_max": 35.697078704833984,
      "activations/layer9_attention_weight_min": -33.874488830566406,
      "epoch": 8.43,
      "learning_rate": 9.658295454545454e-05,
      "loss": 2.8072,
      "step": 145150
    },
    {
      "activations/layer0_attention_weight_max": 15.681035995483398,
      "activations/layer0_attention_weight_min": -11.156943321228027,
      "activations/layer10_attention_weight_max": 32.64613723754883,
      "activations/layer10_attention_weight_min": -33.02656173706055,
      "activations/layer11_attention_weight_max": 31.29153060913086,
      "activations/layer11_attention_weight_min": -31.065811157226562,
      "activations/layer12_attention_weight_max": 31.24858856201172,
      "activations/layer12_attention_weight_min": -25.502140045166016,
      "activations/layer13_attention_weight_max": 36.80924606323242,
      "activations/layer13_attention_weight_min": -34.09315872192383,
      "activations/layer14_attention_weight_max": 46.21828842163086,
      "activations/layer14_attention_weight_min": -39.60932159423828,
      "activations/layer15_attention_weight_max": 34.68436050415039,
      "activations/layer15_attention_weight_min": -32.77376937866211,
      "activations/layer16_attention_weight_max": 31.232776641845703,
      "activations/layer16_attention_weight_min": -26.997079849243164,
      "activations/layer17_attention_weight_max": 31.15986442565918,
      "activations/layer17_attention_weight_min": -28.205507278442383,
      "activations/layer18_attention_weight_max": 34.53057098388672,
      "activations/layer18_attention_weight_min": -21.50358009338379,
      "activations/layer19_attention_weight_max": 34.148624420166016,
      "activations/layer19_attention_weight_min": -33.30618667602539,
      "activations/layer1_attention_weight_max": 16.828779220581055,
      "activations/layer1_attention_weight_min": -14.602733612060547,
      "activations/layer20_attention_weight_max": 29.10906219482422,
      "activations/layer20_attention_weight_min": -24.368772506713867,
      "activations/layer21_attention_weight_max": 28.892784118652344,
      "activations/layer21_attention_weight_min": -23.511642456054688,
      "activations/layer22_attention_weight_max": 42.8348388671875,
      "activations/layer22_attention_weight_min": -28.799530029296875,
      "activations/layer23_attention_weight_max": 34.03041076660156,
      "activations/layer23_attention_weight_min": -22.553916931152344,
      "activations/layer2_attention_weight_max": 32.02062225341797,
      "activations/layer2_attention_weight_min": -29.857769012451172,
      "activations/layer3_attention_weight_max": 90.23985290527344,
      "activations/layer3_attention_weight_min": -90.187255859375,
      "activations/layer4_attention_weight_max": 115.8448486328125,
      "activations/layer4_attention_weight_min": -114.38069152832031,
      "activations/layer5_attention_weight_max": 56.30633544921875,
      "activations/layer5_attention_weight_min": -59.43476867675781,
      "activations/layer6_attention_weight_max": 46.318424224853516,
      "activations/layer6_attention_weight_min": -50.283836364746094,
      "activations/layer7_attention_weight_max": 83.13294219970703,
      "activations/layer7_attention_weight_min": -87.45970153808594,
      "activations/layer8_attention_weight_max": 38.83926773071289,
      "activations/layer8_attention_weight_min": -39.9378547668457,
      "activations/layer9_attention_weight_max": 32.237770080566406,
      "activations/layer9_attention_weight_min": -34.401371002197266,
      "epoch": 8.44,
      "learning_rate": 9.656401515151515e-05,
      "loss": 2.7842,
      "step": 145200
    },
    {
      "activations/layer0_attention_weight_max": 16.584665298461914,
      "activations/layer0_attention_weight_min": -12.367655754089355,
      "activations/layer10_attention_weight_max": 29.62894630432129,
      "activations/layer10_attention_weight_min": -30.527629852294922,
      "activations/layer11_attention_weight_max": 29.696630477905273,
      "activations/layer11_attention_weight_min": -29.252838134765625,
      "activations/layer12_attention_weight_max": 23.794090270996094,
      "activations/layer12_attention_weight_min": -24.35200309753418,
      "activations/layer13_attention_weight_max": 36.38262176513672,
      "activations/layer13_attention_weight_min": -31.697154998779297,
      "activations/layer14_attention_weight_max": 43.492103576660156,
      "activations/layer14_attention_weight_min": -38.453670501708984,
      "activations/layer15_attention_weight_max": 32.5194206237793,
      "activations/layer15_attention_weight_min": -30.15313720703125,
      "activations/layer16_attention_weight_max": 32.22593688964844,
      "activations/layer16_attention_weight_min": -27.668283462524414,
      "activations/layer17_attention_weight_max": 30.885772705078125,
      "activations/layer17_attention_weight_min": -26.017709732055664,
      "activations/layer18_attention_weight_max": 32.75886154174805,
      "activations/layer18_attention_weight_min": -24.221477508544922,
      "activations/layer19_attention_weight_max": 33.82867431640625,
      "activations/layer19_attention_weight_min": -30.195240020751953,
      "activations/layer1_attention_weight_max": 16.831846237182617,
      "activations/layer1_attention_weight_min": -15.723106384277344,
      "activations/layer20_attention_weight_max": 27.026792526245117,
      "activations/layer20_attention_weight_min": -25.166791915893555,
      "activations/layer21_attention_weight_max": 27.57393455505371,
      "activations/layer21_attention_weight_min": -23.1168212890625,
      "activations/layer22_attention_weight_max": 44.84364318847656,
      "activations/layer22_attention_weight_min": -30.495685577392578,
      "activations/layer23_attention_weight_max": 35.37603759765625,
      "activations/layer23_attention_weight_min": -24.457176208496094,
      "activations/layer2_attention_weight_max": 31.224302291870117,
      "activations/layer2_attention_weight_min": -32.41032791137695,
      "activations/layer3_attention_weight_max": 88.60551452636719,
      "activations/layer3_attention_weight_min": -91.77039337158203,
      "activations/layer4_attention_weight_max": 115.16475677490234,
      "activations/layer4_attention_weight_min": -117.25670623779297,
      "activations/layer5_attention_weight_max": 54.91744613647461,
      "activations/layer5_attention_weight_min": -61.76585388183594,
      "activations/layer6_attention_weight_max": 47.74348831176758,
      "activations/layer6_attention_weight_min": -52.095584869384766,
      "activations/layer7_attention_weight_max": 85.47193145751953,
      "activations/layer7_attention_weight_min": -87.85757446289062,
      "activations/layer8_attention_weight_max": 39.15018844604492,
      "activations/layer8_attention_weight_min": -40.38037109375,
      "activations/layer9_attention_weight_max": 33.47692108154297,
      "activations/layer9_attention_weight_min": -34.157291412353516,
      "epoch": 8.44,
      "learning_rate": 9.654507575757574e-05,
      "loss": 2.7953,
      "step": 145250
    },
    {
      "activations/layer0_attention_weight_max": 15.170092582702637,
      "activations/layer0_attention_weight_min": -11.384208679199219,
      "activations/layer10_attention_weight_max": 32.023651123046875,
      "activations/layer10_attention_weight_min": -31.946273803710938,
      "activations/layer11_attention_weight_max": 34.26927947998047,
      "activations/layer11_attention_weight_min": -30.50662612915039,
      "activations/layer12_attention_weight_max": 32.39543914794922,
      "activations/layer12_attention_weight_min": -26.445907592773438,
      "activations/layer13_attention_weight_max": 42.492897033691406,
      "activations/layer13_attention_weight_min": -32.19956588745117,
      "activations/layer14_attention_weight_max": 51.13951110839844,
      "activations/layer14_attention_weight_min": -38.10452651977539,
      "activations/layer15_attention_weight_max": 41.77050018310547,
      "activations/layer15_attention_weight_min": -30.982641220092773,
      "activations/layer16_attention_weight_max": 37.03108596801758,
      "activations/layer16_attention_weight_min": -27.983144760131836,
      "activations/layer17_attention_weight_max": 39.26480484008789,
      "activations/layer17_attention_weight_min": -25.469995498657227,
      "activations/layer18_attention_weight_max": 33.26553726196289,
      "activations/layer18_attention_weight_min": -20.50120735168457,
      "activations/layer19_attention_weight_max": 48.850399017333984,
      "activations/layer19_attention_weight_min": -29.341995239257812,
      "activations/layer1_attention_weight_max": 17.649009704589844,
      "activations/layer1_attention_weight_min": -14.47076416015625,
      "activations/layer20_attention_weight_max": 36.225215911865234,
      "activations/layer20_attention_weight_min": -24.992115020751953,
      "activations/layer21_attention_weight_max": 32.15633773803711,
      "activations/layer21_attention_weight_min": -21.805877685546875,
      "activations/layer22_attention_weight_max": 50.956119537353516,
      "activations/layer22_attention_weight_min": -28.614147186279297,
      "activations/layer23_attention_weight_max": 51.98155212402344,
      "activations/layer23_attention_weight_min": -21.015188217163086,
      "activations/layer2_attention_weight_max": 32.28843688964844,
      "activations/layer2_attention_weight_min": -32.036048889160156,
      "activations/layer3_attention_weight_max": 89.52580261230469,
      "activations/layer3_attention_weight_min": -92.2078628540039,
      "activations/layer4_attention_weight_max": 115.66817474365234,
      "activations/layer4_attention_weight_min": -112.9246826171875,
      "activations/layer5_attention_weight_max": 57.19398498535156,
      "activations/layer5_attention_weight_min": -61.95090866088867,
      "activations/layer6_attention_weight_max": 48.00687026977539,
      "activations/layer6_attention_weight_min": -50.001731872558594,
      "activations/layer7_attention_weight_max": 87.58231353759766,
      "activations/layer7_attention_weight_min": -87.88419342041016,
      "activations/layer8_attention_weight_max": 38.15443420410156,
      "activations/layer8_attention_weight_min": -37.85117721557617,
      "activations/layer9_attention_weight_max": 34.54463577270508,
      "activations/layer9_attention_weight_min": -33.50592803955078,
      "epoch": 8.44,
      "learning_rate": 9.652613636363636e-05,
      "loss": 2.8024,
      "step": 145300
    },
    {
      "activations/layer0_attention_weight_max": 15.19773006439209,
      "activations/layer0_attention_weight_min": -11.52103328704834,
      "activations/layer10_attention_weight_max": 33.843631744384766,
      "activations/layer10_attention_weight_min": -31.585063934326172,
      "activations/layer11_attention_weight_max": 36.80484390258789,
      "activations/layer11_attention_weight_min": -32.889366149902344,
      "activations/layer12_attention_weight_max": 26.96491813659668,
      "activations/layer12_attention_weight_min": -26.688520431518555,
      "activations/layer13_attention_weight_max": 47.936546325683594,
      "activations/layer13_attention_weight_min": -34.07048034667969,
      "activations/layer14_attention_weight_max": 55.17372131347656,
      "activations/layer14_attention_weight_min": -39.587318420410156,
      "activations/layer15_attention_weight_max": 44.01577377319336,
      "activations/layer15_attention_weight_min": -32.47264862060547,
      "activations/layer16_attention_weight_max": 42.039039611816406,
      "activations/layer16_attention_weight_min": -31.327014923095703,
      "activations/layer17_attention_weight_max": 40.539127349853516,
      "activations/layer17_attention_weight_min": -25.709400177001953,
      "activations/layer18_attention_weight_max": 34.98802947998047,
      "activations/layer18_attention_weight_min": -24.354793548583984,
      "activations/layer19_attention_weight_max": 45.29656982421875,
      "activations/layer19_attention_weight_min": -31.187726974487305,
      "activations/layer1_attention_weight_max": 16.563344955444336,
      "activations/layer1_attention_weight_min": -15.268190383911133,
      "activations/layer20_attention_weight_max": 36.48642349243164,
      "activations/layer20_attention_weight_min": -24.517284393310547,
      "activations/layer21_attention_weight_max": 34.04521560668945,
      "activations/layer21_attention_weight_min": -23.356842041015625,
      "activations/layer22_attention_weight_max": 63.046142578125,
      "activations/layer22_attention_weight_min": -28.848833084106445,
      "activations/layer23_attention_weight_max": 49.54677963256836,
      "activations/layer23_attention_weight_min": -22.75176239013672,
      "activations/layer2_attention_weight_max": 31.931278228759766,
      "activations/layer2_attention_weight_min": -31.28638458251953,
      "activations/layer3_attention_weight_max": 95.22257995605469,
      "activations/layer3_attention_weight_min": -93.25283813476562,
      "activations/layer4_attention_weight_max": 122.86331939697266,
      "activations/layer4_attention_weight_min": -115.2137451171875,
      "activations/layer5_attention_weight_max": 56.677520751953125,
      "activations/layer5_attention_weight_min": -62.19626998901367,
      "activations/layer6_attention_weight_max": 51.9052848815918,
      "activations/layer6_attention_weight_min": -53.04304122924805,
      "activations/layer7_attention_weight_max": 92.93062591552734,
      "activations/layer7_attention_weight_min": -92.82089233398438,
      "activations/layer8_attention_weight_max": 39.98106002807617,
      "activations/layer8_attention_weight_min": -38.71269226074219,
      "activations/layer9_attention_weight_max": 37.163108825683594,
      "activations/layer9_attention_weight_min": -33.78750228881836,
      "epoch": 8.45,
      "learning_rate": 9.650719696969696e-05,
      "loss": 2.8177,
      "step": 145350
    },
    {
      "activations/layer0_attention_weight_max": 15.257761001586914,
      "activations/layer0_attention_weight_min": -11.19023609161377,
      "activations/layer10_attention_weight_max": 30.93028450012207,
      "activations/layer10_attention_weight_min": -30.453519821166992,
      "activations/layer11_attention_weight_max": 32.411354064941406,
      "activations/layer11_attention_weight_min": -27.841062545776367,
      "activations/layer12_attention_weight_max": 33.883644104003906,
      "activations/layer12_attention_weight_min": -27.988521575927734,
      "activations/layer13_attention_weight_max": 37.9505500793457,
      "activations/layer13_attention_weight_min": -33.882179260253906,
      "activations/layer14_attention_weight_max": 47.21693801879883,
      "activations/layer14_attention_weight_min": -38.84564971923828,
      "activations/layer15_attention_weight_max": 36.31812286376953,
      "activations/layer15_attention_weight_min": -30.8797664642334,
      "activations/layer16_attention_weight_max": 34.84833526611328,
      "activations/layer16_attention_weight_min": -28.597381591796875,
      "activations/layer17_attention_weight_max": 34.7624397277832,
      "activations/layer17_attention_weight_min": -25.87362289428711,
      "activations/layer18_attention_weight_max": 35.344966888427734,
      "activations/layer18_attention_weight_min": -20.632720947265625,
      "activations/layer19_attention_weight_max": 36.31467819213867,
      "activations/layer19_attention_weight_min": -30.050363540649414,
      "activations/layer1_attention_weight_max": 17.173410415649414,
      "activations/layer1_attention_weight_min": -14.423298835754395,
      "activations/layer20_attention_weight_max": 33.54399108886719,
      "activations/layer20_attention_weight_min": -24.3825740814209,
      "activations/layer21_attention_weight_max": 31.04154396057129,
      "activations/layer21_attention_weight_min": -22.579370498657227,
      "activations/layer22_attention_weight_max": 45.86255645751953,
      "activations/layer22_attention_weight_min": -28.18122673034668,
      "activations/layer23_attention_weight_max": 36.885475158691406,
      "activations/layer23_attention_weight_min": -24.581100463867188,
      "activations/layer2_attention_weight_max": 32.070457458496094,
      "activations/layer2_attention_weight_min": -32.689002990722656,
      "activations/layer3_attention_weight_max": 90.74846649169922,
      "activations/layer3_attention_weight_min": -92.80137634277344,
      "activations/layer4_attention_weight_max": 117.809814453125,
      "activations/layer4_attention_weight_min": -117.70909881591797,
      "activations/layer5_attention_weight_max": 57.451576232910156,
      "activations/layer5_attention_weight_min": -59.82949447631836,
      "activations/layer6_attention_weight_max": 48.82341766357422,
      "activations/layer6_attention_weight_min": -50.985633850097656,
      "activations/layer7_attention_weight_max": 88.1718978881836,
      "activations/layer7_attention_weight_min": -89.48916625976562,
      "activations/layer8_attention_weight_max": 41.32916259765625,
      "activations/layer8_attention_weight_min": -39.44831085205078,
      "activations/layer9_attention_weight_max": 32.822837829589844,
      "activations/layer9_attention_weight_min": -31.579456329345703,
      "epoch": 8.45,
      "learning_rate": 9.648825757575757e-05,
      "loss": 2.7961,
      "step": 145400
    },
    {
      "activations/layer0_attention_weight_max": 15.226897239685059,
      "activations/layer0_attention_weight_min": -10.91325855255127,
      "activations/layer10_attention_weight_max": 33.97751235961914,
      "activations/layer10_attention_weight_min": -33.906005859375,
      "activations/layer11_attention_weight_max": 34.32685089111328,
      "activations/layer11_attention_weight_min": -31.10512924194336,
      "activations/layer12_attention_weight_max": 23.00078582763672,
      "activations/layer12_attention_weight_min": -25.046253204345703,
      "activations/layer13_attention_weight_max": 43.57304763793945,
      "activations/layer13_attention_weight_min": -34.713539123535156,
      "activations/layer14_attention_weight_max": 47.10939025878906,
      "activations/layer14_attention_weight_min": -39.58753204345703,
      "activations/layer15_attention_weight_max": 37.02684783935547,
      "activations/layer15_attention_weight_min": -29.94599723815918,
      "activations/layer16_attention_weight_max": 31.08295440673828,
      "activations/layer16_attention_weight_min": -28.89084243774414,
      "activations/layer17_attention_weight_max": 33.6168098449707,
      "activations/layer17_attention_weight_min": -25.745065689086914,
      "activations/layer18_attention_weight_max": 35.397403717041016,
      "activations/layer18_attention_weight_min": -22.46132469177246,
      "activations/layer19_attention_weight_max": 35.473655700683594,
      "activations/layer19_attention_weight_min": -29.8963565826416,
      "activations/layer1_attention_weight_max": 16.687755584716797,
      "activations/layer1_attention_weight_min": -14.330967903137207,
      "activations/layer20_attention_weight_max": 29.5499210357666,
      "activations/layer20_attention_weight_min": -25.329349517822266,
      "activations/layer21_attention_weight_max": 33.51453399658203,
      "activations/layer21_attention_weight_min": -22.509204864501953,
      "activations/layer22_attention_weight_max": 45.831390380859375,
      "activations/layer22_attention_weight_min": -29.674213409423828,
      "activations/layer23_attention_weight_max": 36.98012161254883,
      "activations/layer23_attention_weight_min": -24.17023468017578,
      "activations/layer2_attention_weight_max": 32.03937530517578,
      "activations/layer2_attention_weight_min": -31.019445419311523,
      "activations/layer3_attention_weight_max": 85.70463562011719,
      "activations/layer3_attention_weight_min": -90.3970718383789,
      "activations/layer4_attention_weight_max": 119.48320770263672,
      "activations/layer4_attention_weight_min": -118.48451232910156,
      "activations/layer5_attention_weight_max": 58.334136962890625,
      "activations/layer5_attention_weight_min": -63.6936149597168,
      "activations/layer6_attention_weight_max": 48.04588317871094,
      "activations/layer6_attention_weight_min": -52.219112396240234,
      "activations/layer7_attention_weight_max": 92.34656524658203,
      "activations/layer7_attention_weight_min": -90.85814666748047,
      "activations/layer8_attention_weight_max": 40.44326400756836,
      "activations/layer8_attention_weight_min": -41.8240852355957,
      "activations/layer9_attention_weight_max": 36.49305725097656,
      "activations/layer9_attention_weight_min": -34.72236251831055,
      "epoch": 8.45,
      "learning_rate": 9.646931818181817e-05,
      "loss": 2.8101,
      "step": 145450
    },
    {
      "activations/layer0_attention_weight_max": 16.08721923828125,
      "activations/layer0_attention_weight_min": -12.075453758239746,
      "activations/layer10_attention_weight_max": 46.62211608886719,
      "activations/layer10_attention_weight_min": -42.680545806884766,
      "activations/layer11_attention_weight_max": 48.8690299987793,
      "activations/layer11_attention_weight_min": -42.576175689697266,
      "activations/layer12_attention_weight_max": 35.66390609741211,
      "activations/layer12_attention_weight_min": -26.935041427612305,
      "activations/layer13_attention_weight_max": 55.45621871948242,
      "activations/layer13_attention_weight_min": -37.8914680480957,
      "activations/layer14_attention_weight_max": 54.34263610839844,
      "activations/layer14_attention_weight_min": -41.51877975463867,
      "activations/layer15_attention_weight_max": 54.11581802368164,
      "activations/layer15_attention_weight_min": -33.66724395751953,
      "activations/layer16_attention_weight_max": 33.571109771728516,
      "activations/layer16_attention_weight_min": -28.577274322509766,
      "activations/layer17_attention_weight_max": 34.695281982421875,
      "activations/layer17_attention_weight_min": -28.866037368774414,
      "activations/layer18_attention_weight_max": 33.512203216552734,
      "activations/layer18_attention_weight_min": -23.46646499633789,
      "activations/layer19_attention_weight_max": 38.24020767211914,
      "activations/layer19_attention_weight_min": -31.3773136138916,
      "activations/layer1_attention_weight_max": 16.660818099975586,
      "activations/layer1_attention_weight_min": -15.794024467468262,
      "activations/layer20_attention_weight_max": 30.656448364257812,
      "activations/layer20_attention_weight_min": -25.211925506591797,
      "activations/layer21_attention_weight_max": 30.506200790405273,
      "activations/layer21_attention_weight_min": -24.70013999938965,
      "activations/layer22_attention_weight_max": 46.63986587524414,
      "activations/layer22_attention_weight_min": -32.05123519897461,
      "activations/layer23_attention_weight_max": 36.49150085449219,
      "activations/layer23_attention_weight_min": -25.873146057128906,
      "activations/layer2_attention_weight_max": 34.24433898925781,
      "activations/layer2_attention_weight_min": -33.359378814697266,
      "activations/layer3_attention_weight_max": 96.31049346923828,
      "activations/layer3_attention_weight_min": -98.85604858398438,
      "activations/layer4_attention_weight_max": 124.04747772216797,
      "activations/layer4_attention_weight_min": -117.15166473388672,
      "activations/layer5_attention_weight_max": 60.493221282958984,
      "activations/layer5_attention_weight_min": -59.509727478027344,
      "activations/layer6_attention_weight_max": 58.44013595581055,
      "activations/layer6_attention_weight_min": -52.58928680419922,
      "activations/layer7_attention_weight_max": 112.77265930175781,
      "activations/layer7_attention_weight_min": -94.82929229736328,
      "activations/layer8_attention_weight_max": 52.49858093261719,
      "activations/layer8_attention_weight_min": -48.777915954589844,
      "activations/layer9_attention_weight_max": 47.33073806762695,
      "activations/layer9_attention_weight_min": -41.855926513671875,
      "epoch": 8.45,
      "learning_rate": 9.645037878787878e-05,
      "loss": 2.7973,
      "step": 145500
    },
    {
      "activations/layer0_attention_weight_max": 15.788692474365234,
      "activations/layer0_attention_weight_min": -12.304707527160645,
      "activations/layer10_attention_weight_max": 34.267189025878906,
      "activations/layer10_attention_weight_min": -33.44636535644531,
      "activations/layer11_attention_weight_max": 34.534156799316406,
      "activations/layer11_attention_weight_min": -31.896482467651367,
      "activations/layer12_attention_weight_max": 25.930265426635742,
      "activations/layer12_attention_weight_min": -25.460269927978516,
      "activations/layer13_attention_weight_max": 39.260433197021484,
      "activations/layer13_attention_weight_min": -33.027496337890625,
      "activations/layer14_attention_weight_max": 50.63525390625,
      "activations/layer14_attention_weight_min": -36.96078109741211,
      "activations/layer15_attention_weight_max": 35.22935485839844,
      "activations/layer15_attention_weight_min": -30.96993637084961,
      "activations/layer16_attention_weight_max": 35.246498107910156,
      "activations/layer16_attention_weight_min": -29.024444580078125,
      "activations/layer17_attention_weight_max": 33.461849212646484,
      "activations/layer17_attention_weight_min": -25.768512725830078,
      "activations/layer18_attention_weight_max": 31.29107093811035,
      "activations/layer18_attention_weight_min": -21.702722549438477,
      "activations/layer19_attention_weight_max": 37.00276565551758,
      "activations/layer19_attention_weight_min": -31.859338760375977,
      "activations/layer1_attention_weight_max": 17.701810836791992,
      "activations/layer1_attention_weight_min": -15.636258125305176,
      "activations/layer20_attention_weight_max": 30.239728927612305,
      "activations/layer20_attention_weight_min": -26.165483474731445,
      "activations/layer21_attention_weight_max": 27.862930297851562,
      "activations/layer21_attention_weight_min": -25.013107299804688,
      "activations/layer22_attention_weight_max": 46.42556381225586,
      "activations/layer22_attention_weight_min": -29.914440155029297,
      "activations/layer23_attention_weight_max": 38.12752914428711,
      "activations/layer23_attention_weight_min": -23.59198760986328,
      "activations/layer2_attention_weight_max": 34.09473419189453,
      "activations/layer2_attention_weight_min": -33.77499008178711,
      "activations/layer3_attention_weight_max": 93.00714111328125,
      "activations/layer3_attention_weight_min": -96.97834014892578,
      "activations/layer4_attention_weight_max": 121.96028137207031,
      "activations/layer4_attention_weight_min": -122.3706283569336,
      "activations/layer5_attention_weight_max": 57.568580627441406,
      "activations/layer5_attention_weight_min": -63.32148742675781,
      "activations/layer6_attention_weight_max": 47.99073791503906,
      "activations/layer6_attention_weight_min": -52.91902160644531,
      "activations/layer7_attention_weight_max": 86.19722747802734,
      "activations/layer7_attention_weight_min": -89.21511840820312,
      "activations/layer8_attention_weight_max": 43.28913116455078,
      "activations/layer8_attention_weight_min": -43.155372619628906,
      "activations/layer9_attention_weight_max": 37.6451301574707,
      "activations/layer9_attention_weight_min": -34.65763473510742,
      "epoch": 8.46,
      "learning_rate": 9.643143939393939e-05,
      "loss": 2.8164,
      "step": 145550
    },
    {
      "activations/layer0_attention_weight_max": 16.54163360595703,
      "activations/layer0_attention_weight_min": -11.634333610534668,
      "activations/layer10_attention_weight_max": 33.679962158203125,
      "activations/layer10_attention_weight_min": -33.42286682128906,
      "activations/layer11_attention_weight_max": 36.79222869873047,
      "activations/layer11_attention_weight_min": -31.433246612548828,
      "activations/layer12_attention_weight_max": 24.04086685180664,
      "activations/layer12_attention_weight_min": -25.03911781311035,
      "activations/layer13_attention_weight_max": 39.7136116027832,
      "activations/layer13_attention_weight_min": -36.08054733276367,
      "activations/layer14_attention_weight_max": 56.078121185302734,
      "activations/layer14_attention_weight_min": -42.860023498535156,
      "activations/layer15_attention_weight_max": 40.071266174316406,
      "activations/layer15_attention_weight_min": -33.285980224609375,
      "activations/layer16_attention_weight_max": 35.72092819213867,
      "activations/layer16_attention_weight_min": -29.04482650756836,
      "activations/layer17_attention_weight_max": 34.68295669555664,
      "activations/layer17_attention_weight_min": -27.225601196289062,
      "activations/layer18_attention_weight_max": 33.8825798034668,
      "activations/layer18_attention_weight_min": -22.38729476928711,
      "activations/layer19_attention_weight_max": 42.76396942138672,
      "activations/layer19_attention_weight_min": -33.352779388427734,
      "activations/layer1_attention_weight_max": 16.094573974609375,
      "activations/layer1_attention_weight_min": -13.84785270690918,
      "activations/layer20_attention_weight_max": 34.72107696533203,
      "activations/layer20_attention_weight_min": -25.597551345825195,
      "activations/layer21_attention_weight_max": 31.237884521484375,
      "activations/layer21_attention_weight_min": -23.60030174255371,
      "activations/layer22_attention_weight_max": 48.80777359008789,
      "activations/layer22_attention_weight_min": -28.886268615722656,
      "activations/layer23_attention_weight_max": 40.321937561035156,
      "activations/layer23_attention_weight_min": -23.47958755493164,
      "activations/layer2_attention_weight_max": 34.061180114746094,
      "activations/layer2_attention_weight_min": -31.516599655151367,
      "activations/layer3_attention_weight_max": 91.89218139648438,
      "activations/layer3_attention_weight_min": -95.34713745117188,
      "activations/layer4_attention_weight_max": 120.9017562866211,
      "activations/layer4_attention_weight_min": -121.0344467163086,
      "activations/layer5_attention_weight_max": 58.05522918701172,
      "activations/layer5_attention_weight_min": -64.07044219970703,
      "activations/layer6_attention_weight_max": 52.86885070800781,
      "activations/layer6_attention_weight_min": -56.258731842041016,
      "activations/layer7_attention_weight_max": 93.48902130126953,
      "activations/layer7_attention_weight_min": -97.15769958496094,
      "activations/layer8_attention_weight_max": 42.95022201538086,
      "activations/layer8_attention_weight_min": -43.8170166015625,
      "activations/layer9_attention_weight_max": 34.74472427368164,
      "activations/layer9_attention_weight_min": -35.75459671020508,
      "epoch": 8.46,
      "learning_rate": 9.641249999999999e-05,
      "loss": 2.8049,
      "step": 145600
    },
    {
      "activations/layer0_attention_weight_max": 15.533465385437012,
      "activations/layer0_attention_weight_min": -11.362802505493164,
      "activations/layer10_attention_weight_max": 31.606290817260742,
      "activations/layer10_attention_weight_min": -32.03769302368164,
      "activations/layer11_attention_weight_max": 31.911420822143555,
      "activations/layer11_attention_weight_min": -31.467811584472656,
      "activations/layer12_attention_weight_max": 24.323583602905273,
      "activations/layer12_attention_weight_min": -23.782609939575195,
      "activations/layer13_attention_weight_max": 37.305389404296875,
      "activations/layer13_attention_weight_min": -36.60889434814453,
      "activations/layer14_attention_weight_max": 43.717529296875,
      "activations/layer14_attention_weight_min": -39.00638961791992,
      "activations/layer15_attention_weight_max": 36.05141830444336,
      "activations/layer15_attention_weight_min": -32.99872970581055,
      "activations/layer16_attention_weight_max": 31.366777420043945,
      "activations/layer16_attention_weight_min": -29.818546295166016,
      "activations/layer17_attention_weight_max": 33.6151237487793,
      "activations/layer17_attention_weight_min": -27.86219596862793,
      "activations/layer18_attention_weight_max": 30.865829467773438,
      "activations/layer18_attention_weight_min": -24.62294578552246,
      "activations/layer19_attention_weight_max": 41.13467788696289,
      "activations/layer19_attention_weight_min": -31.254058837890625,
      "activations/layer1_attention_weight_max": 16.682371139526367,
      "activations/layer1_attention_weight_min": -14.474526405334473,
      "activations/layer20_attention_weight_max": 32.6168212890625,
      "activations/layer20_attention_weight_min": -26.350866317749023,
      "activations/layer21_attention_weight_max": 29.883792877197266,
      "activations/layer21_attention_weight_min": -24.50949478149414,
      "activations/layer22_attention_weight_max": 47.8944206237793,
      "activations/layer22_attention_weight_min": -30.884355545043945,
      "activations/layer23_attention_weight_max": 40.81471252441406,
      "activations/layer23_attention_weight_min": -24.219932556152344,
      "activations/layer2_attention_weight_max": 33.089111328125,
      "activations/layer2_attention_weight_min": -32.587860107421875,
      "activations/layer3_attention_weight_max": 91.4280014038086,
      "activations/layer3_attention_weight_min": -95.7662124633789,
      "activations/layer4_attention_weight_max": 116.9283218383789,
      "activations/layer4_attention_weight_min": -116.8508071899414,
      "activations/layer5_attention_weight_max": 55.61787033081055,
      "activations/layer5_attention_weight_min": -61.361534118652344,
      "activations/layer6_attention_weight_max": 46.569087982177734,
      "activations/layer6_attention_weight_min": -49.61696243286133,
      "activations/layer7_attention_weight_max": 86.19328308105469,
      "activations/layer7_attention_weight_min": -88.0697021484375,
      "activations/layer8_attention_weight_max": 37.17890930175781,
      "activations/layer8_attention_weight_min": -40.5943717956543,
      "activations/layer9_attention_weight_max": 32.88828659057617,
      "activations/layer9_attention_weight_min": -34.511600494384766,
      "epoch": 8.46,
      "learning_rate": 9.63935606060606e-05,
      "loss": 2.7994,
      "step": 145650
    },
    {
      "activations/layer0_attention_weight_max": 17.276641845703125,
      "activations/layer0_attention_weight_min": -11.348685264587402,
      "activations/layer10_attention_weight_max": 34.051605224609375,
      "activations/layer10_attention_weight_min": -34.97016143798828,
      "activations/layer11_attention_weight_max": 36.41462326049805,
      "activations/layer11_attention_weight_min": -30.84435272216797,
      "activations/layer12_attention_weight_max": 23.63860511779785,
      "activations/layer12_attention_weight_min": -28.142724990844727,
      "activations/layer13_attention_weight_max": 43.14864730834961,
      "activations/layer13_attention_weight_min": -32.018402099609375,
      "activations/layer14_attention_weight_max": 55.440059661865234,
      "activations/layer14_attention_weight_min": -38.65964126586914,
      "activations/layer15_attention_weight_max": 35.158958435058594,
      "activations/layer15_attention_weight_min": -29.183456420898438,
      "activations/layer16_attention_weight_max": 30.646663665771484,
      "activations/layer16_attention_weight_min": -27.83132553100586,
      "activations/layer17_attention_weight_max": 33.9022102355957,
      "activations/layer17_attention_weight_min": -24.306318283081055,
      "activations/layer18_attention_weight_max": 27.981895446777344,
      "activations/layer18_attention_weight_min": -20.870925903320312,
      "activations/layer19_attention_weight_max": 33.80557632446289,
      "activations/layer19_attention_weight_min": -29.639806747436523,
      "activations/layer1_attention_weight_max": 16.427005767822266,
      "activations/layer1_attention_weight_min": -13.46863842010498,
      "activations/layer20_attention_weight_max": 30.21905517578125,
      "activations/layer20_attention_weight_min": -24.358055114746094,
      "activations/layer21_attention_weight_max": 31.175884246826172,
      "activations/layer21_attention_weight_min": -22.665868759155273,
      "activations/layer22_attention_weight_max": 47.48362731933594,
      "activations/layer22_attention_weight_min": -28.037981033325195,
      "activations/layer23_attention_weight_max": 33.851558685302734,
      "activations/layer23_attention_weight_min": -23.907073974609375,
      "activations/layer2_attention_weight_max": 31.320331573486328,
      "activations/layer2_attention_weight_min": -31.301666259765625,
      "activations/layer3_attention_weight_max": 89.52305603027344,
      "activations/layer3_attention_weight_min": -92.41891479492188,
      "activations/layer4_attention_weight_max": 118.57085418701172,
      "activations/layer4_attention_weight_min": -119.38616180419922,
      "activations/layer5_attention_weight_max": 54.42168426513672,
      "activations/layer5_attention_weight_min": -60.6252555847168,
      "activations/layer6_attention_weight_max": 47.31136703491211,
      "activations/layer6_attention_weight_min": -49.16261672973633,
      "activations/layer7_attention_weight_max": 87.6443099975586,
      "activations/layer7_attention_weight_min": -88.46660614013672,
      "activations/layer8_attention_weight_max": 40.588558197021484,
      "activations/layer8_attention_weight_min": -41.260337829589844,
      "activations/layer9_attention_weight_max": 37.336219787597656,
      "activations/layer9_attention_weight_min": -34.58717727661133,
      "epoch": 8.47,
      "learning_rate": 9.637462121212121e-05,
      "loss": 2.8012,
      "step": 145700
    },
    {
      "activations/layer0_attention_weight_max": 15.937823295593262,
      "activations/layer0_attention_weight_min": -12.041302680969238,
      "activations/layer10_attention_weight_max": 29.79143524169922,
      "activations/layer10_attention_weight_min": -32.81645965576172,
      "activations/layer11_attention_weight_max": 30.08357810974121,
      "activations/layer11_attention_weight_min": -31.063810348510742,
      "activations/layer12_attention_weight_max": 23.042661666870117,
      "activations/layer12_attention_weight_min": -26.91693878173828,
      "activations/layer13_attention_weight_max": 39.6170654296875,
      "activations/layer13_attention_weight_min": -33.796661376953125,
      "activations/layer14_attention_weight_max": 43.7900390625,
      "activations/layer14_attention_weight_min": -38.25102996826172,
      "activations/layer15_attention_weight_max": 36.1533203125,
      "activations/layer15_attention_weight_min": -30.550935745239258,
      "activations/layer16_attention_weight_max": 32.7773551940918,
      "activations/layer16_attention_weight_min": -27.835561752319336,
      "activations/layer17_attention_weight_max": 31.819427490234375,
      "activations/layer17_attention_weight_min": -25.192222595214844,
      "activations/layer18_attention_weight_max": 28.7858829498291,
      "activations/layer18_attention_weight_min": -21.833383560180664,
      "activations/layer19_attention_weight_max": 37.189552307128906,
      "activations/layer19_attention_weight_min": -31.458404541015625,
      "activations/layer1_attention_weight_max": 17.601741790771484,
      "activations/layer1_attention_weight_min": -15.07243537902832,
      "activations/layer20_attention_weight_max": 31.09221076965332,
      "activations/layer20_attention_weight_min": -26.503049850463867,
      "activations/layer21_attention_weight_max": 26.380584716796875,
      "activations/layer21_attention_weight_min": -22.879663467407227,
      "activations/layer22_attention_weight_max": 43.166866302490234,
      "activations/layer22_attention_weight_min": -29.545848846435547,
      "activations/layer23_attention_weight_max": 32.659393310546875,
      "activations/layer23_attention_weight_min": -23.62623405456543,
      "activations/layer2_attention_weight_max": 29.719717025756836,
      "activations/layer2_attention_weight_min": -29.2491455078125,
      "activations/layer3_attention_weight_max": 84.513916015625,
      "activations/layer3_attention_weight_min": -86.25814056396484,
      "activations/layer4_attention_weight_max": 113.76185607910156,
      "activations/layer4_attention_weight_min": -109.67091369628906,
      "activations/layer5_attention_weight_max": 52.13429260253906,
      "activations/layer5_attention_weight_min": -62.98952865600586,
      "activations/layer6_attention_weight_max": 45.710819244384766,
      "activations/layer6_attention_weight_min": -49.50917053222656,
      "activations/layer7_attention_weight_max": 85.74768829345703,
      "activations/layer7_attention_weight_min": -83.89551544189453,
      "activations/layer8_attention_weight_max": 35.644309997558594,
      "activations/layer8_attention_weight_min": -38.4599723815918,
      "activations/layer9_attention_weight_max": 31.690521240234375,
      "activations/layer9_attention_weight_min": -32.25430679321289,
      "epoch": 8.47,
      "learning_rate": 9.635568181818182e-05,
      "loss": 2.7999,
      "step": 145750
    },
    {
      "activations/layer0_attention_weight_max": 16.22477912902832,
      "activations/layer0_attention_weight_min": -11.247316360473633,
      "activations/layer10_attention_weight_max": 32.335262298583984,
      "activations/layer10_attention_weight_min": -29.695232391357422,
      "activations/layer11_attention_weight_max": 33.95427703857422,
      "activations/layer11_attention_weight_min": -29.84618377685547,
      "activations/layer12_attention_weight_max": 36.72168731689453,
      "activations/layer12_attention_weight_min": -24.097196578979492,
      "activations/layer13_attention_weight_max": 46.94850540161133,
      "activations/layer13_attention_weight_min": -31.980491638183594,
      "activations/layer14_attention_weight_max": 44.061790466308594,
      "activations/layer14_attention_weight_min": -41.7072639465332,
      "activations/layer15_attention_weight_max": 43.06087875366211,
      "activations/layer15_attention_weight_min": -32.3614387512207,
      "activations/layer16_attention_weight_max": 32.443603515625,
      "activations/layer16_attention_weight_min": -29.080036163330078,
      "activations/layer17_attention_weight_max": 30.675554275512695,
      "activations/layer17_attention_weight_min": -24.816024780273438,
      "activations/layer18_attention_weight_max": 30.169837951660156,
      "activations/layer18_attention_weight_min": -21.982507705688477,
      "activations/layer19_attention_weight_max": 34.9295768737793,
      "activations/layer19_attention_weight_min": -31.441761016845703,
      "activations/layer1_attention_weight_max": 16.386674880981445,
      "activations/layer1_attention_weight_min": -13.445385932922363,
      "activations/layer20_attention_weight_max": 28.368528366088867,
      "activations/layer20_attention_weight_min": -24.360492706298828,
      "activations/layer21_attention_weight_max": 30.693984985351562,
      "activations/layer21_attention_weight_min": -23.648649215698242,
      "activations/layer22_attention_weight_max": 42.42210006713867,
      "activations/layer22_attention_weight_min": -29.1036376953125,
      "activations/layer23_attention_weight_max": 33.32475280761719,
      "activations/layer23_attention_weight_min": -23.769702911376953,
      "activations/layer2_attention_weight_max": 31.542804718017578,
      "activations/layer2_attention_weight_min": -31.316434860229492,
      "activations/layer3_attention_weight_max": 88.61225128173828,
      "activations/layer3_attention_weight_min": -93.94676208496094,
      "activations/layer4_attention_weight_max": 120.341064453125,
      "activations/layer4_attention_weight_min": -117.17350006103516,
      "activations/layer5_attention_weight_max": 54.241302490234375,
      "activations/layer5_attention_weight_min": -63.26337814331055,
      "activations/layer6_attention_weight_max": 46.94561004638672,
      "activations/layer6_attention_weight_min": -51.03669738769531,
      "activations/layer7_attention_weight_max": 84.05120849609375,
      "activations/layer7_attention_weight_min": -86.50101470947266,
      "activations/layer8_attention_weight_max": 38.821128845214844,
      "activations/layer8_attention_weight_min": -37.54905700683594,
      "activations/layer9_attention_weight_max": 35.8432502746582,
      "activations/layer9_attention_weight_min": -31.71408462524414,
      "epoch": 8.47,
      "learning_rate": 9.633674242424241e-05,
      "loss": 2.797,
      "step": 145800
    },
    {
      "activations/layer0_attention_weight_max": 16.71608543395996,
      "activations/layer0_attention_weight_min": -12.524772644042969,
      "activations/layer10_attention_weight_max": 33.17299270629883,
      "activations/layer10_attention_weight_min": -30.793004989624023,
      "activations/layer11_attention_weight_max": 30.277732849121094,
      "activations/layer11_attention_weight_min": -28.84906578063965,
      "activations/layer12_attention_weight_max": 23.106056213378906,
      "activations/layer12_attention_weight_min": -25.710779190063477,
      "activations/layer13_attention_weight_max": 40.40721893310547,
      "activations/layer13_attention_weight_min": -35.05139923095703,
      "activations/layer14_attention_weight_max": 51.86666488647461,
      "activations/layer14_attention_weight_min": -42.18952941894531,
      "activations/layer15_attention_weight_max": 35.1865234375,
      "activations/layer15_attention_weight_min": -33.72270584106445,
      "activations/layer16_attention_weight_max": 32.11085891723633,
      "activations/layer16_attention_weight_min": -30.52106475830078,
      "activations/layer17_attention_weight_max": 35.90108108520508,
      "activations/layer17_attention_weight_min": -24.950105667114258,
      "activations/layer18_attention_weight_max": 32.198543548583984,
      "activations/layer18_attention_weight_min": -22.08413314819336,
      "activations/layer19_attention_weight_max": 33.675167083740234,
      "activations/layer19_attention_weight_min": -29.9971981048584,
      "activations/layer1_attention_weight_max": 16.55855941772461,
      "activations/layer1_attention_weight_min": -15.596758842468262,
      "activations/layer20_attention_weight_max": 28.7598876953125,
      "activations/layer20_attention_weight_min": -24.707536697387695,
      "activations/layer21_attention_weight_max": 27.59153175354004,
      "activations/layer21_attention_weight_min": -22.357666015625,
      "activations/layer22_attention_weight_max": 45.390586853027344,
      "activations/layer22_attention_weight_min": -28.64175033569336,
      "activations/layer23_attention_weight_max": 37.326698303222656,
      "activations/layer23_attention_weight_min": -24.509620666503906,
      "activations/layer2_attention_weight_max": 32.46055603027344,
      "activations/layer2_attention_weight_min": -30.434837341308594,
      "activations/layer3_attention_weight_max": 87.2718276977539,
      "activations/layer3_attention_weight_min": -87.31277465820312,
      "activations/layer4_attention_weight_max": 112.81074523925781,
      "activations/layer4_attention_weight_min": -107.00532531738281,
      "activations/layer5_attention_weight_max": 54.990745544433594,
      "activations/layer5_attention_weight_min": -61.54752731323242,
      "activations/layer6_attention_weight_max": 47.18721008300781,
      "activations/layer6_attention_weight_min": -48.35221481323242,
      "activations/layer7_attention_weight_max": 86.7608871459961,
      "activations/layer7_attention_weight_min": -83.22418212890625,
      "activations/layer8_attention_weight_max": 40.23532485961914,
      "activations/layer8_attention_weight_min": -39.52212142944336,
      "activations/layer9_attention_weight_max": 34.95362091064453,
      "activations/layer9_attention_weight_min": -33.79212188720703,
      "epoch": 8.47,
      "learning_rate": 9.631780303030303e-05,
      "loss": 2.8041,
      "step": 145850
    },
    {
      "activations/layer0_attention_weight_max": 15.970773696899414,
      "activations/layer0_attention_weight_min": -10.458227157592773,
      "activations/layer10_attention_weight_max": 35.7011833190918,
      "activations/layer10_attention_weight_min": -32.97711181640625,
      "activations/layer11_attention_weight_max": 36.03400802612305,
      "activations/layer11_attention_weight_min": -32.34864044189453,
      "activations/layer12_attention_weight_max": 25.981592178344727,
      "activations/layer12_attention_weight_min": -24.07785415649414,
      "activations/layer13_attention_weight_max": 44.21688461303711,
      "activations/layer13_attention_weight_min": -32.594417572021484,
      "activations/layer14_attention_weight_max": 47.441612243652344,
      "activations/layer14_attention_weight_min": -36.2049674987793,
      "activations/layer15_attention_weight_max": 37.11144256591797,
      "activations/layer15_attention_weight_min": -28.833662033081055,
      "activations/layer16_attention_weight_max": 30.52789306640625,
      "activations/layer16_attention_weight_min": -28.687294006347656,
      "activations/layer17_attention_weight_max": 34.03737258911133,
      "activations/layer17_attention_weight_min": -23.770986557006836,
      "activations/layer18_attention_weight_max": 36.033931732177734,
      "activations/layer18_attention_weight_min": -20.833667755126953,
      "activations/layer19_attention_weight_max": 37.094730377197266,
      "activations/layer19_attention_weight_min": -29.065814971923828,
      "activations/layer1_attention_weight_max": 19.710330963134766,
      "activations/layer1_attention_weight_min": -16.135452270507812,
      "activations/layer20_attention_weight_max": 35.452308654785156,
      "activations/layer20_attention_weight_min": -24.399309158325195,
      "activations/layer21_attention_weight_max": 35.157752990722656,
      "activations/layer21_attention_weight_min": -22.38971519470215,
      "activations/layer22_attention_weight_max": 43.50180435180664,
      "activations/layer22_attention_weight_min": -26.3433895111084,
      "activations/layer23_attention_weight_max": 32.86762237548828,
      "activations/layer23_attention_weight_min": -22.960037231445312,
      "activations/layer2_attention_weight_max": 31.222396850585938,
      "activations/layer2_attention_weight_min": -32.90472412109375,
      "activations/layer3_attention_weight_max": 93.4591293334961,
      "activations/layer3_attention_weight_min": -96.02826690673828,
      "activations/layer4_attention_weight_max": 121.97198486328125,
      "activations/layer4_attention_weight_min": -117.61898040771484,
      "activations/layer5_attention_weight_max": 58.81132507324219,
      "activations/layer5_attention_weight_min": -58.00603485107422,
      "activations/layer6_attention_weight_max": 48.62132263183594,
      "activations/layer6_attention_weight_min": -49.55376052856445,
      "activations/layer7_attention_weight_max": 88.59981536865234,
      "activations/layer7_attention_weight_min": -87.50059509277344,
      "activations/layer8_attention_weight_max": 43.720340728759766,
      "activations/layer8_attention_weight_min": -43.173622131347656,
      "activations/layer9_attention_weight_max": 37.856689453125,
      "activations/layer9_attention_weight_min": -34.45771789550781,
      "epoch": 8.48,
      "learning_rate": 9.629886363636361e-05,
      "loss": 2.7855,
      "step": 145900
    },
    {
      "activations/layer0_attention_weight_max": 14.783413887023926,
      "activations/layer0_attention_weight_min": -12.296353340148926,
      "activations/layer10_attention_weight_max": 36.412200927734375,
      "activations/layer10_attention_weight_min": -33.05799865722656,
      "activations/layer11_attention_weight_max": 35.20531463623047,
      "activations/layer11_attention_weight_min": -32.915260314941406,
      "activations/layer12_attention_weight_max": 23.170372009277344,
      "activations/layer12_attention_weight_min": -24.331003189086914,
      "activations/layer13_attention_weight_max": 38.70929718017578,
      "activations/layer13_attention_weight_min": -33.89420700073242,
      "activations/layer14_attention_weight_max": 47.09510040283203,
      "activations/layer14_attention_weight_min": -37.23495101928711,
      "activations/layer15_attention_weight_max": 36.485877990722656,
      "activations/layer15_attention_weight_min": -30.811595916748047,
      "activations/layer16_attention_weight_max": 30.014102935791016,
      "activations/layer16_attention_weight_min": -26.02035903930664,
      "activations/layer17_attention_weight_max": 33.12996292114258,
      "activations/layer17_attention_weight_min": -25.223886489868164,
      "activations/layer18_attention_weight_max": 28.170188903808594,
      "activations/layer18_attention_weight_min": -20.91736602783203,
      "activations/layer19_attention_weight_max": 35.654354095458984,
      "activations/layer19_attention_weight_min": -30.173681259155273,
      "activations/layer1_attention_weight_max": 16.095020294189453,
      "activations/layer1_attention_weight_min": -14.144482612609863,
      "activations/layer20_attention_weight_max": 30.456602096557617,
      "activations/layer20_attention_weight_min": -25.538318634033203,
      "activations/layer21_attention_weight_max": 26.103940963745117,
      "activations/layer21_attention_weight_min": -23.614145278930664,
      "activations/layer22_attention_weight_max": 45.52606201171875,
      "activations/layer22_attention_weight_min": -27.891517639160156,
      "activations/layer23_attention_weight_max": 32.16879653930664,
      "activations/layer23_attention_weight_min": -22.30237579345703,
      "activations/layer2_attention_weight_max": 31.075902938842773,
      "activations/layer2_attention_weight_min": -32.280982971191406,
      "activations/layer3_attention_weight_max": 88.81717681884766,
      "activations/layer3_attention_weight_min": -92.98511505126953,
      "activations/layer4_attention_weight_max": 116.04227447509766,
      "activations/layer4_attention_weight_min": -118.33036041259766,
      "activations/layer5_attention_weight_max": 55.18296432495117,
      "activations/layer5_attention_weight_min": -66.14288330078125,
      "activations/layer6_attention_weight_max": 49.88603591918945,
      "activations/layer6_attention_weight_min": -52.82111358642578,
      "activations/layer7_attention_weight_max": 91.06434631347656,
      "activations/layer7_attention_weight_min": -91.07079315185547,
      "activations/layer8_attention_weight_max": 42.39519119262695,
      "activations/layer8_attention_weight_min": -42.95805740356445,
      "activations/layer9_attention_weight_max": 39.656822204589844,
      "activations/layer9_attention_weight_min": -34.4397087097168,
      "epoch": 8.48,
      "learning_rate": 9.628030303030303e-05,
      "loss": 2.8018,
      "step": 145950
    },
    {
      "activations/layer0_attention_weight_max": 15.27513599395752,
      "activations/layer0_attention_weight_min": -11.909802436828613,
      "activations/layer10_attention_weight_max": 33.50421142578125,
      "activations/layer10_attention_weight_min": -32.28632354736328,
      "activations/layer11_attention_weight_max": 38.874969482421875,
      "activations/layer11_attention_weight_min": -32.83769226074219,
      "activations/layer12_attention_weight_max": 37.523414611816406,
      "activations/layer12_attention_weight_min": -23.319271087646484,
      "activations/layer13_attention_weight_max": 49.309959411621094,
      "activations/layer13_attention_weight_min": -32.1905632019043,
      "activations/layer14_attention_weight_max": 64.35611724853516,
      "activations/layer14_attention_weight_min": -41.95304870605469,
      "activations/layer15_attention_weight_max": 66.03898620605469,
      "activations/layer15_attention_weight_min": -35.65117645263672,
      "activations/layer16_attention_weight_max": 41.34457015991211,
      "activations/layer16_attention_weight_min": -26.881120681762695,
      "activations/layer17_attention_weight_max": 53.25615692138672,
      "activations/layer17_attention_weight_min": -25.576324462890625,
      "activations/layer18_attention_weight_max": 48.8289680480957,
      "activations/layer18_attention_weight_min": -24.019786834716797,
      "activations/layer19_attention_weight_max": 47.355037689208984,
      "activations/layer19_attention_weight_min": -30.08096694946289,
      "activations/layer1_attention_weight_max": 17.126657485961914,
      "activations/layer1_attention_weight_min": -13.558151245117188,
      "activations/layer20_attention_weight_max": 36.493003845214844,
      "activations/layer20_attention_weight_min": -24.171199798583984,
      "activations/layer21_attention_weight_max": 37.19364547729492,
      "activations/layer21_attention_weight_min": -23.30475425720215,
      "activations/layer22_attention_weight_max": 61.10291290283203,
      "activations/layer22_attention_weight_min": -30.165056228637695,
      "activations/layer23_attention_weight_max": 45.18048095703125,
      "activations/layer23_attention_weight_min": -25.292484283447266,
      "activations/layer2_attention_weight_max": 31.840208053588867,
      "activations/layer2_attention_weight_min": -31.38949203491211,
      "activations/layer3_attention_weight_max": 91.57119750976562,
      "activations/layer3_attention_weight_min": -91.8759765625,
      "activations/layer4_attention_weight_max": 117.27318572998047,
      "activations/layer4_attention_weight_min": -112.9507827758789,
      "activations/layer5_attention_weight_max": 54.5290641784668,
      "activations/layer5_attention_weight_min": -64.47962951660156,
      "activations/layer6_attention_weight_max": 48.04494094848633,
      "activations/layer6_attention_weight_min": -49.54405975341797,
      "activations/layer7_attention_weight_max": 88.51679229736328,
      "activations/layer7_attention_weight_min": -86.92083740234375,
      "activations/layer8_attention_weight_max": 40.448890686035156,
      "activations/layer8_attention_weight_min": -42.5318717956543,
      "activations/layer9_attention_weight_max": 35.194393157958984,
      "activations/layer9_attention_weight_min": -33.07662582397461,
      "epoch": 8.48,
      "learning_rate": 9.626136363636362e-05,
      "loss": 2.8093,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.5143,
      "eval_samples_per_second": 504.326,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.5143,
      "eval_openwebtext_samples_per_second": 504.326,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 2.0124,
      "eval_wikitext_samples_per_second": 226.6,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_lambada_loss": 2.609375,
      "eval_lambada_ppl": 13.59055410055989,
      "eval_lambada_runtime": 9.5941,
      "eval_lambada_samples_per_second": 507.497,
      "step": 146000
    },
    {
      "activations/layer0_attention_weight_max": 15.897650718688965,
      "activations/layer0_attention_weight_min": -11.369879722595215,
      "activations/layer10_attention_weight_max": 30.968252182006836,
      "activations/layer10_attention_weight_min": -33.4666633605957,
      "activations/layer11_attention_weight_max": 31.184803009033203,
      "activations/layer11_attention_weight_min": -31.212520599365234,
      "activations/layer12_attention_weight_max": 27.180274963378906,
      "activations/layer12_attention_weight_min": -27.00245475769043,
      "activations/layer13_attention_weight_max": 36.57561492919922,
      "activations/layer13_attention_weight_min": -34.89069366455078,
      "activations/layer14_attention_weight_max": 43.713706970214844,
      "activations/layer14_attention_weight_min": -40.91645812988281,
      "activations/layer15_attention_weight_max": 36.24217224121094,
      "activations/layer15_attention_weight_min": -33.09086608886719,
      "activations/layer16_attention_weight_max": 31.14711570739746,
      "activations/layer16_attention_weight_min": -28.466585159301758,
      "activations/layer17_attention_weight_max": 32.5093879699707,
      "activations/layer17_attention_weight_min": -28.225242614746094,
      "activations/layer18_attention_weight_max": 28.8066463470459,
      "activations/layer18_attention_weight_min": -22.380630493164062,
      "activations/layer19_attention_weight_max": 37.79245376586914,
      "activations/layer19_attention_weight_min": -32.5455322265625,
      "activations/layer1_attention_weight_max": 16.97649383544922,
      "activations/layer1_attention_weight_min": -14.4771728515625,
      "activations/layer20_attention_weight_max": 32.570579528808594,
      "activations/layer20_attention_weight_min": -25.869699478149414,
      "activations/layer21_attention_weight_max": 27.147872924804688,
      "activations/layer21_attention_weight_min": -25.08182144165039,
      "activations/layer22_attention_weight_max": 43.38376998901367,
      "activations/layer22_attention_weight_min": -30.82914161682129,
      "activations/layer23_attention_weight_max": 35.87195587158203,
      "activations/layer23_attention_weight_min": -25.155872344970703,
      "activations/layer2_attention_weight_max": 32.47301483154297,
      "activations/layer2_attention_weight_min": -33.682254791259766,
      "activations/layer3_attention_weight_max": 92.46233367919922,
      "activations/layer3_attention_weight_min": -94.52705383300781,
      "activations/layer4_attention_weight_max": 120.48590087890625,
      "activations/layer4_attention_weight_min": -115.549560546875,
      "activations/layer5_attention_weight_max": 56.58643341064453,
      "activations/layer5_attention_weight_min": -62.99050521850586,
      "activations/layer6_attention_weight_max": 48.406272888183594,
      "activations/layer6_attention_weight_min": -50.30947494506836,
      "activations/layer7_attention_weight_max": 86.99879455566406,
      "activations/layer7_attention_weight_min": -89.7195053100586,
      "activations/layer8_attention_weight_max": 41.52860641479492,
      "activations/layer8_attention_weight_min": -41.80656814575195,
      "activations/layer9_attention_weight_max": 33.62483596801758,
      "activations/layer9_attention_weight_min": -34.635257720947266,
      "epoch": 8.49,
      "learning_rate": 9.624242424242424e-05,
      "loss": 2.8146,
      "step": 146050
    },
    {
      "activations/layer0_attention_weight_max": 16.170143127441406,
      "activations/layer0_attention_weight_min": -12.313551902770996,
      "activations/layer10_attention_weight_max": 32.61960220336914,
      "activations/layer10_attention_weight_min": -32.64788818359375,
      "activations/layer11_attention_weight_max": 33.17311096191406,
      "activations/layer11_attention_weight_min": -30.935407638549805,
      "activations/layer12_attention_weight_max": 25.180545806884766,
      "activations/layer12_attention_weight_min": -25.485944747924805,
      "activations/layer13_attention_weight_max": 37.57769012451172,
      "activations/layer13_attention_weight_min": -32.81489181518555,
      "activations/layer14_attention_weight_max": 48.979888916015625,
      "activations/layer14_attention_weight_min": -41.524478912353516,
      "activations/layer15_attention_weight_max": 36.595863342285156,
      "activations/layer15_attention_weight_min": -29.97673225402832,
      "activations/layer16_attention_weight_max": 33.70598602294922,
      "activations/layer16_attention_weight_min": -29.922603607177734,
      "activations/layer17_attention_weight_max": 37.49483108520508,
      "activations/layer17_attention_weight_min": -25.45832633972168,
      "activations/layer18_attention_weight_max": 32.57777786254883,
      "activations/layer18_attention_weight_min": -23.567506790161133,
      "activations/layer19_attention_weight_max": 37.39400100708008,
      "activations/layer19_attention_weight_min": -31.54831886291504,
      "activations/layer1_attention_weight_max": 20.774362564086914,
      "activations/layer1_attention_weight_min": -17.592350006103516,
      "activations/layer20_attention_weight_max": 31.761716842651367,
      "activations/layer20_attention_weight_min": -25.579130172729492,
      "activations/layer21_attention_weight_max": 32.189937591552734,
      "activations/layer21_attention_weight_min": -25.6817626953125,
      "activations/layer22_attention_weight_max": 50.29930114746094,
      "activations/layer22_attention_weight_min": -32.47792434692383,
      "activations/layer23_attention_weight_max": 42.09834671020508,
      "activations/layer23_attention_weight_min": -25.734149932861328,
      "activations/layer2_attention_weight_max": 32.421932220458984,
      "activations/layer2_attention_weight_min": -32.346336364746094,
      "activations/layer3_attention_weight_max": 95.59761810302734,
      "activations/layer3_attention_weight_min": -96.09517669677734,
      "activations/layer4_attention_weight_max": 122.0576171875,
      "activations/layer4_attention_weight_min": -124.4637680053711,
      "activations/layer5_attention_weight_max": 56.3359489440918,
      "activations/layer5_attention_weight_min": -66.39076232910156,
      "activations/layer6_attention_weight_max": 49.74886703491211,
      "activations/layer6_attention_weight_min": -53.765018463134766,
      "activations/layer7_attention_weight_max": 94.88542938232422,
      "activations/layer7_attention_weight_min": -88.1649398803711,
      "activations/layer8_attention_weight_max": 41.538902282714844,
      "activations/layer8_attention_weight_min": -42.57102584838867,
      "activations/layer9_attention_weight_max": 35.395301818847656,
      "activations/layer9_attention_weight_min": -34.15668487548828,
      "epoch": 8.49,
      "learning_rate": 9.622348484848484e-05,
      "loss": 2.8055,
      "step": 146100
    },
    {
      "activations/layer0_attention_weight_max": 16.24761962890625,
      "activations/layer0_attention_weight_min": -11.810928344726562,
      "activations/layer10_attention_weight_max": 30.32260513305664,
      "activations/layer10_attention_weight_min": -29.21559715270996,
      "activations/layer11_attention_weight_max": 31.84089469909668,
      "activations/layer11_attention_weight_min": -28.25835418701172,
      "activations/layer12_attention_weight_max": 30.159147262573242,
      "activations/layer12_attention_weight_min": -26.649269104003906,
      "activations/layer13_attention_weight_max": 37.81008529663086,
      "activations/layer13_attention_weight_min": -32.95149230957031,
      "activations/layer14_attention_weight_max": 48.17539978027344,
      "activations/layer14_attention_weight_min": -34.5380859375,
      "activations/layer15_attention_weight_max": 37.83903121948242,
      "activations/layer15_attention_weight_min": -30.522607803344727,
      "activations/layer16_attention_weight_max": 31.383106231689453,
      "activations/layer16_attention_weight_min": -26.77611541748047,
      "activations/layer17_attention_weight_max": 33.02687454223633,
      "activations/layer17_attention_weight_min": -24.63543701171875,
      "activations/layer18_attention_weight_max": 34.3734130859375,
      "activations/layer18_attention_weight_min": -20.854293823242188,
      "activations/layer19_attention_weight_max": 38.07462692260742,
      "activations/layer19_attention_weight_min": -30.759403228759766,
      "activations/layer1_attention_weight_max": 17.653335571289062,
      "activations/layer1_attention_weight_min": -14.034904479980469,
      "activations/layer20_attention_weight_max": 29.61300277709961,
      "activations/layer20_attention_weight_min": -27.61174964904785,
      "activations/layer21_attention_weight_max": 29.028507232666016,
      "activations/layer21_attention_weight_min": -23.187763214111328,
      "activations/layer22_attention_weight_max": 41.8835334777832,
      "activations/layer22_attention_weight_min": -28.8747615814209,
      "activations/layer23_attention_weight_max": 33.099082946777344,
      "activations/layer23_attention_weight_min": -24.116138458251953,
      "activations/layer2_attention_weight_max": 30.87305450439453,
      "activations/layer2_attention_weight_min": -29.534698486328125,
      "activations/layer3_attention_weight_max": 89.06181335449219,
      "activations/layer3_attention_weight_min": -91.50642395019531,
      "activations/layer4_attention_weight_max": 114.54122161865234,
      "activations/layer4_attention_weight_min": -119.48995208740234,
      "activations/layer5_attention_weight_max": 54.04975128173828,
      "activations/layer5_attention_weight_min": -61.47087860107422,
      "activations/layer6_attention_weight_max": 45.50127410888672,
      "activations/layer6_attention_weight_min": -48.914222717285156,
      "activations/layer7_attention_weight_max": 87.43901062011719,
      "activations/layer7_attention_weight_min": -85.410400390625,
      "activations/layer8_attention_weight_max": 38.63243103027344,
      "activations/layer8_attention_weight_min": -38.660221099853516,
      "activations/layer9_attention_weight_max": 34.21753692626953,
      "activations/layer9_attention_weight_min": -31.772119522094727,
      "epoch": 8.49,
      "learning_rate": 9.620454545454544e-05,
      "loss": 2.7827,
      "step": 146150
    },
    {
      "activations/layer0_attention_weight_max": 16.386695861816406,
      "activations/layer0_attention_weight_min": -12.808032989501953,
      "activations/layer10_attention_weight_max": 32.03193664550781,
      "activations/layer10_attention_weight_min": -32.959964752197266,
      "activations/layer11_attention_weight_max": 33.10089111328125,
      "activations/layer11_attention_weight_min": -30.089126586914062,
      "activations/layer12_attention_weight_max": 23.362937927246094,
      "activations/layer12_attention_weight_min": -26.609418869018555,
      "activations/layer13_attention_weight_max": 37.987850189208984,
      "activations/layer13_attention_weight_min": -30.92759132385254,
      "activations/layer14_attention_weight_max": 47.647029876708984,
      "activations/layer14_attention_weight_min": -40.751853942871094,
      "activations/layer15_attention_weight_max": 37.26667022705078,
      "activations/layer15_attention_weight_min": -30.33571434020996,
      "activations/layer16_attention_weight_max": 32.48471450805664,
      "activations/layer16_attention_weight_min": -27.93301010131836,
      "activations/layer17_attention_weight_max": 36.21809387207031,
      "activations/layer17_attention_weight_min": -23.920679092407227,
      "activations/layer18_attention_weight_max": 31.81391716003418,
      "activations/layer18_attention_weight_min": -21.959680557250977,
      "activations/layer19_attention_weight_max": 35.21000289916992,
      "activations/layer19_attention_weight_min": -30.647775650024414,
      "activations/layer1_attention_weight_max": 17.631900787353516,
      "activations/layer1_attention_weight_min": -13.741986274719238,
      "activations/layer20_attention_weight_max": 31.8497314453125,
      "activations/layer20_attention_weight_min": -23.981155395507812,
      "activations/layer21_attention_weight_max": 33.201881408691406,
      "activations/layer21_attention_weight_min": -23.052963256835938,
      "activations/layer22_attention_weight_max": 47.15266799926758,
      "activations/layer22_attention_weight_min": -29.997447967529297,
      "activations/layer23_attention_weight_max": 38.20854949951172,
      "activations/layer23_attention_weight_min": -25.543197631835938,
      "activations/layer2_attention_weight_max": 31.614879608154297,
      "activations/layer2_attention_weight_min": -30.365131378173828,
      "activations/layer3_attention_weight_max": 87.6396713256836,
      "activations/layer3_attention_weight_min": -87.61720275878906,
      "activations/layer4_attention_weight_max": 118.0855712890625,
      "activations/layer4_attention_weight_min": -109.99346923828125,
      "activations/layer5_attention_weight_max": 53.424217224121094,
      "activations/layer5_attention_weight_min": -59.15007019042969,
      "activations/layer6_attention_weight_max": 47.431575775146484,
      "activations/layer6_attention_weight_min": -51.8420295715332,
      "activations/layer7_attention_weight_max": 86.05764770507812,
      "activations/layer7_attention_weight_min": -83.70001220703125,
      "activations/layer8_attention_weight_max": 39.02057647705078,
      "activations/layer8_attention_weight_min": -38.807518005371094,
      "activations/layer9_attention_weight_max": 34.581844329833984,
      "activations/layer9_attention_weight_min": -33.35172653198242,
      "epoch": 8.5,
      "learning_rate": 9.618560606060605e-05,
      "loss": 2.8017,
      "step": 146200
    },
    {
      "activations/layer0_attention_weight_max": 15.287881851196289,
      "activations/layer0_attention_weight_min": -11.213905334472656,
      "activations/layer10_attention_weight_max": 34.87046432495117,
      "activations/layer10_attention_weight_min": -32.68949890136719,
      "activations/layer11_attention_weight_max": 32.84912872314453,
      "activations/layer11_attention_weight_min": -30.872135162353516,
      "activations/layer12_attention_weight_max": 23.99882698059082,
      "activations/layer12_attention_weight_min": -25.715763092041016,
      "activations/layer13_attention_weight_max": 40.93830871582031,
      "activations/layer13_attention_weight_min": -35.604087829589844,
      "activations/layer14_attention_weight_max": 55.39434051513672,
      "activations/layer14_attention_weight_min": -42.356201171875,
      "activations/layer15_attention_weight_max": 37.20595932006836,
      "activations/layer15_attention_weight_min": -32.517189025878906,
      "activations/layer16_attention_weight_max": 31.8275146484375,
      "activations/layer16_attention_weight_min": -28.75217628479004,
      "activations/layer17_attention_weight_max": 34.017738342285156,
      "activations/layer17_attention_weight_min": -27.35479164123535,
      "activations/layer18_attention_weight_max": 31.60066032409668,
      "activations/layer18_attention_weight_min": -23.026296615600586,
      "activations/layer19_attention_weight_max": 36.9708137512207,
      "activations/layer19_attention_weight_min": -32.252044677734375,
      "activations/layer1_attention_weight_max": 19.691381454467773,
      "activations/layer1_attention_weight_min": -16.798555374145508,
      "activations/layer20_attention_weight_max": 32.82170104980469,
      "activations/layer20_attention_weight_min": -26.669071197509766,
      "activations/layer21_attention_weight_max": 34.125946044921875,
      "activations/layer21_attention_weight_min": -23.50539207458496,
      "activations/layer22_attention_weight_max": 47.43511199951172,
      "activations/layer22_attention_weight_min": -30.769895553588867,
      "activations/layer23_attention_weight_max": 37.20736312866211,
      "activations/layer23_attention_weight_min": -23.14594268798828,
      "activations/layer2_attention_weight_max": 32.979034423828125,
      "activations/layer2_attention_weight_min": -30.867828369140625,
      "activations/layer3_attention_weight_max": 91.94929504394531,
      "activations/layer3_attention_weight_min": -94.46797943115234,
      "activations/layer4_attention_weight_max": 122.84754943847656,
      "activations/layer4_attention_weight_min": -123.26300048828125,
      "activations/layer5_attention_weight_max": 57.37763595581055,
      "activations/layer5_attention_weight_min": -66.83295440673828,
      "activations/layer6_attention_weight_max": 51.054805755615234,
      "activations/layer6_attention_weight_min": -52.17104721069336,
      "activations/layer7_attention_weight_max": 97.9799575805664,
      "activations/layer7_attention_weight_min": -90.48204040527344,
      "activations/layer8_attention_weight_max": 41.12129592895508,
      "activations/layer8_attention_weight_min": -41.74611282348633,
      "activations/layer9_attention_weight_max": 38.35084533691406,
      "activations/layer9_attention_weight_min": -36.089717864990234,
      "epoch": 8.5,
      "learning_rate": 9.616666666666666e-05,
      "loss": 2.7963,
      "step": 146250
    },
    {
      "activations/layer0_attention_weight_max": 16.00574493408203,
      "activations/layer0_attention_weight_min": -11.723865509033203,
      "activations/layer10_attention_weight_max": 33.1292724609375,
      "activations/layer10_attention_weight_min": -32.2772102355957,
      "activations/layer11_attention_weight_max": 30.631362915039062,
      "activations/layer11_attention_weight_min": -29.0361328125,
      "activations/layer12_attention_weight_max": 24.2070255279541,
      "activations/layer12_attention_weight_min": -25.193973541259766,
      "activations/layer13_attention_weight_max": 39.454185485839844,
      "activations/layer13_attention_weight_min": -33.183380126953125,
      "activations/layer14_attention_weight_max": 50.215049743652344,
      "activations/layer14_attention_weight_min": -40.91801834106445,
      "activations/layer15_attention_weight_max": 35.489253997802734,
      "activations/layer15_attention_weight_min": -30.707117080688477,
      "activations/layer16_attention_weight_max": 32.06193161010742,
      "activations/layer16_attention_weight_min": -27.3570556640625,
      "activations/layer17_attention_weight_max": 32.81342315673828,
      "activations/layer17_attention_weight_min": -25.851655960083008,
      "activations/layer18_attention_weight_max": 30.840665817260742,
      "activations/layer18_attention_weight_min": -22.166200637817383,
      "activations/layer19_attention_weight_max": 35.90607833862305,
      "activations/layer19_attention_weight_min": -30.83725929260254,
      "activations/layer1_attention_weight_max": 17.545244216918945,
      "activations/layer1_attention_weight_min": -14.620505332946777,
      "activations/layer20_attention_weight_max": 30.63878059387207,
      "activations/layer20_attention_weight_min": -25.448030471801758,
      "activations/layer21_attention_weight_max": 34.66853713989258,
      "activations/layer21_attention_weight_min": -24.533885955810547,
      "activations/layer22_attention_weight_max": 46.75912857055664,
      "activations/layer22_attention_weight_min": -29.96407127380371,
      "activations/layer23_attention_weight_max": 35.33909225463867,
      "activations/layer23_attention_weight_min": -24.48225975036621,
      "activations/layer2_attention_weight_max": 32.153900146484375,
      "activations/layer2_attention_weight_min": -31.720989227294922,
      "activations/layer3_attention_weight_max": 92.3446273803711,
      "activations/layer3_attention_weight_min": -92.76647186279297,
      "activations/layer4_attention_weight_max": 123.3137435913086,
      "activations/layer4_attention_weight_min": -115.7803955078125,
      "activations/layer5_attention_weight_max": 58.55192565917969,
      "activations/layer5_attention_weight_min": -62.26987838745117,
      "activations/layer6_attention_weight_max": 50.77254104614258,
      "activations/layer6_attention_weight_min": -52.2729606628418,
      "activations/layer7_attention_weight_max": 92.42439270019531,
      "activations/layer7_attention_weight_min": -86.59042358398438,
      "activations/layer8_attention_weight_max": 42.10160827636719,
      "activations/layer8_attention_weight_min": -42.14149856567383,
      "activations/layer9_attention_weight_max": 35.59724426269531,
      "activations/layer9_attention_weight_min": -32.90778732299805,
      "epoch": 8.5,
      "learning_rate": 9.614772727272727e-05,
      "loss": 2.8049,
      "step": 146300
    },
    {
      "activations/layer0_attention_weight_max": 16.077016830444336,
      "activations/layer0_attention_weight_min": -12.55325698852539,
      "activations/layer10_attention_weight_max": 33.22789001464844,
      "activations/layer10_attention_weight_min": -31.950515747070312,
      "activations/layer11_attention_weight_max": 32.965293884277344,
      "activations/layer11_attention_weight_min": -30.980688095092773,
      "activations/layer12_attention_weight_max": 24.327356338500977,
      "activations/layer12_attention_weight_min": -26.21853256225586,
      "activations/layer13_attention_weight_max": 41.25811767578125,
      "activations/layer13_attention_weight_min": -32.85250473022461,
      "activations/layer14_attention_weight_max": 47.267398834228516,
      "activations/layer14_attention_weight_min": -37.113006591796875,
      "activations/layer15_attention_weight_max": 38.03565979003906,
      "activations/layer15_attention_weight_min": -30.223678588867188,
      "activations/layer16_attention_weight_max": 32.33173370361328,
      "activations/layer16_attention_weight_min": -27.73912239074707,
      "activations/layer17_attention_weight_max": 35.75839614868164,
      "activations/layer17_attention_weight_min": -24.586496353149414,
      "activations/layer18_attention_weight_max": 32.46432113647461,
      "activations/layer18_attention_weight_min": -22.088953018188477,
      "activations/layer19_attention_weight_max": 32.653438568115234,
      "activations/layer19_attention_weight_min": -30.29941177368164,
      "activations/layer1_attention_weight_max": 17.375030517578125,
      "activations/layer1_attention_weight_min": -13.27925968170166,
      "activations/layer20_attention_weight_max": 29.154945373535156,
      "activations/layer20_attention_weight_min": -24.883634567260742,
      "activations/layer21_attention_weight_max": 31.61911964416504,
      "activations/layer21_attention_weight_min": -23.758073806762695,
      "activations/layer22_attention_weight_max": 45.94174575805664,
      "activations/layer22_attention_weight_min": -31.080060958862305,
      "activations/layer23_attention_weight_max": 37.245079040527344,
      "activations/layer23_attention_weight_min": -26.464628219604492,
      "activations/layer2_attention_weight_max": 32.49919128417969,
      "activations/layer2_attention_weight_min": -31.546606063842773,
      "activations/layer3_attention_weight_max": 91.48507690429688,
      "activations/layer3_attention_weight_min": -93.88766479492188,
      "activations/layer4_attention_weight_max": 120.56108856201172,
      "activations/layer4_attention_weight_min": -116.93875885009766,
      "activations/layer5_attention_weight_max": 56.608238220214844,
      "activations/layer5_attention_weight_min": -63.051177978515625,
      "activations/layer6_attention_weight_max": 50.1017951965332,
      "activations/layer6_attention_weight_min": -51.243648529052734,
      "activations/layer7_attention_weight_max": 91.05213165283203,
      "activations/layer7_attention_weight_min": -89.0291976928711,
      "activations/layer8_attention_weight_max": 40.25533676147461,
      "activations/layer8_attention_weight_min": -39.04887008666992,
      "activations/layer9_attention_weight_max": 35.60472106933594,
      "activations/layer9_attention_weight_min": -33.44948196411133,
      "epoch": 8.5,
      "learning_rate": 9.612878787878787e-05,
      "loss": 2.8109,
      "step": 146350
    },
    {
      "activations/layer0_attention_weight_max": 16.067224502563477,
      "activations/layer0_attention_weight_min": -11.806009292602539,
      "activations/layer10_attention_weight_max": 33.969581604003906,
      "activations/layer10_attention_weight_min": -32.49529266357422,
      "activations/layer11_attention_weight_max": 32.519981384277344,
      "activations/layer11_attention_weight_min": -30.419849395751953,
      "activations/layer12_attention_weight_max": 20.25856590270996,
      "activations/layer12_attention_weight_min": -26.529193878173828,
      "activations/layer13_attention_weight_max": 34.96337127685547,
      "activations/layer13_attention_weight_min": -34.67458724975586,
      "activations/layer14_attention_weight_max": 47.164276123046875,
      "activations/layer14_attention_weight_min": -41.73940658569336,
      "activations/layer15_attention_weight_max": 32.850135803222656,
      "activations/layer15_attention_weight_min": -31.979028701782227,
      "activations/layer16_attention_weight_max": 32.06187057495117,
      "activations/layer16_attention_weight_min": -28.304555892944336,
      "activations/layer17_attention_weight_max": 29.857730865478516,
      "activations/layer17_attention_weight_min": -24.763181686401367,
      "activations/layer18_attention_weight_max": 29.585140228271484,
      "activations/layer18_attention_weight_min": -23.222944259643555,
      "activations/layer19_attention_weight_max": 34.22642135620117,
      "activations/layer19_attention_weight_min": -30.52351951599121,
      "activations/layer1_attention_weight_max": 18.82697105407715,
      "activations/layer1_attention_weight_min": -14.767671585083008,
      "activations/layer20_attention_weight_max": 31.25527000427246,
      "activations/layer20_attention_weight_min": -23.84488296508789,
      "activations/layer21_attention_weight_max": 29.006132125854492,
      "activations/layer21_attention_weight_min": -23.501846313476562,
      "activations/layer22_attention_weight_max": 45.93958282470703,
      "activations/layer22_attention_weight_min": -27.342662811279297,
      "activations/layer23_attention_weight_max": 32.49329376220703,
      "activations/layer23_attention_weight_min": -23.066747665405273,
      "activations/layer2_attention_weight_max": 31.611112594604492,
      "activations/layer2_attention_weight_min": -31.46752166748047,
      "activations/layer3_attention_weight_max": 91.14632415771484,
      "activations/layer3_attention_weight_min": -93.00544738769531,
      "activations/layer4_attention_weight_max": 123.5842056274414,
      "activations/layer4_attention_weight_min": -122.5723876953125,
      "activations/layer5_attention_weight_max": 55.86907958984375,
      "activations/layer5_attention_weight_min": -60.32868576049805,
      "activations/layer6_attention_weight_max": 49.25222396850586,
      "activations/layer6_attention_weight_min": -50.233333587646484,
      "activations/layer7_attention_weight_max": 86.13264465332031,
      "activations/layer7_attention_weight_min": -84.61204528808594,
      "activations/layer8_attention_weight_max": 38.67422866821289,
      "activations/layer8_attention_weight_min": -40.74811553955078,
      "activations/layer9_attention_weight_max": 35.41764450073242,
      "activations/layer9_attention_weight_min": -32.92361831665039,
      "epoch": 8.51,
      "learning_rate": 9.610984848484847e-05,
      "loss": 2.7956,
      "step": 146400
    },
    {
      "activations/layer0_attention_weight_max": 15.648494720458984,
      "activations/layer0_attention_weight_min": -12.701000213623047,
      "activations/layer10_attention_weight_max": 35.576629638671875,
      "activations/layer10_attention_weight_min": -32.088844299316406,
      "activations/layer11_attention_weight_max": 32.98235321044922,
      "activations/layer11_attention_weight_min": -30.112342834472656,
      "activations/layer12_attention_weight_max": 29.23386573791504,
      "activations/layer12_attention_weight_min": -37.006492614746094,
      "activations/layer13_attention_weight_max": 39.2519416809082,
      "activations/layer13_attention_weight_min": -32.963714599609375,
      "activations/layer14_attention_weight_max": 53.42970275878906,
      "activations/layer14_attention_weight_min": -38.14162063598633,
      "activations/layer15_attention_weight_max": 36.562034606933594,
      "activations/layer15_attention_weight_min": -28.38175392150879,
      "activations/layer16_attention_weight_max": 34.87887954711914,
      "activations/layer16_attention_weight_min": -26.58955955505371,
      "activations/layer17_attention_weight_max": 36.45370864868164,
      "activations/layer17_attention_weight_min": -24.901691436767578,
      "activations/layer18_attention_weight_max": 33.106536865234375,
      "activations/layer18_attention_weight_min": -22.200389862060547,
      "activations/layer19_attention_weight_max": 38.98242950439453,
      "activations/layer19_attention_weight_min": -30.028432846069336,
      "activations/layer1_attention_weight_max": 17.19696044921875,
      "activations/layer1_attention_weight_min": -14.248089790344238,
      "activations/layer20_attention_weight_max": 34.531883239746094,
      "activations/layer20_attention_weight_min": -24.900678634643555,
      "activations/layer21_attention_weight_max": 32.795509338378906,
      "activations/layer21_attention_weight_min": -23.29180908203125,
      "activations/layer22_attention_weight_max": 51.0875358581543,
      "activations/layer22_attention_weight_min": -29.80113983154297,
      "activations/layer23_attention_weight_max": 38.039337158203125,
      "activations/layer23_attention_weight_min": -24.677661895751953,
      "activations/layer2_attention_weight_max": 31.195194244384766,
      "activations/layer2_attention_weight_min": -32.05900573730469,
      "activations/layer3_attention_weight_max": 90.68006896972656,
      "activations/layer3_attention_weight_min": -95.39716339111328,
      "activations/layer4_attention_weight_max": 118.98522186279297,
      "activations/layer4_attention_weight_min": -118.84761810302734,
      "activations/layer5_attention_weight_max": 54.74503707885742,
      "activations/layer5_attention_weight_min": -61.059547424316406,
      "activations/layer6_attention_weight_max": 48.798789978027344,
      "activations/layer6_attention_weight_min": -50.83256530761719,
      "activations/layer7_attention_weight_max": 85.4941177368164,
      "activations/layer7_attention_weight_min": -91.31087493896484,
      "activations/layer8_attention_weight_max": 40.54419708251953,
      "activations/layer8_attention_weight_min": -41.93153762817383,
      "activations/layer9_attention_weight_max": 35.779197692871094,
      "activations/layer9_attention_weight_min": -34.6408805847168,
      "epoch": 8.51,
      "learning_rate": 9.609090909090909e-05,
      "loss": 2.794,
      "step": 146450
    },
    {
      "activations/layer0_attention_weight_max": 16.420917510986328,
      "activations/layer0_attention_weight_min": -11.694899559020996,
      "activations/layer10_attention_weight_max": 33.929847717285156,
      "activations/layer10_attention_weight_min": -31.89000701904297,
      "activations/layer11_attention_weight_max": 31.903717041015625,
      "activations/layer11_attention_weight_min": -31.027463912963867,
      "activations/layer12_attention_weight_max": 22.134082794189453,
      "activations/layer12_attention_weight_min": -25.090641021728516,
      "activations/layer13_attention_weight_max": 37.42977523803711,
      "activations/layer13_attention_weight_min": -32.98194885253906,
      "activations/layer14_attention_weight_max": 52.883853912353516,
      "activations/layer14_attention_weight_min": -44.7081413269043,
      "activations/layer15_attention_weight_max": 34.65669250488281,
      "activations/layer15_attention_weight_min": -31.00674819946289,
      "activations/layer16_attention_weight_max": 30.886669158935547,
      "activations/layer16_attention_weight_min": -27.12680435180664,
      "activations/layer17_attention_weight_max": 33.25322341918945,
      "activations/layer17_attention_weight_min": -26.239503860473633,
      "activations/layer18_attention_weight_max": 30.693805694580078,
      "activations/layer18_attention_weight_min": -22.87397575378418,
      "activations/layer19_attention_weight_max": 39.447166442871094,
      "activations/layer19_attention_weight_min": -30.013700485229492,
      "activations/layer1_attention_weight_max": 16.46397590637207,
      "activations/layer1_attention_weight_min": -14.76953411102295,
      "activations/layer20_attention_weight_max": 31.522748947143555,
      "activations/layer20_attention_weight_min": -25.220918655395508,
      "activations/layer21_attention_weight_max": 32.50419235229492,
      "activations/layer21_attention_weight_min": -24.689678192138672,
      "activations/layer22_attention_weight_max": 44.19536590576172,
      "activations/layer22_attention_weight_min": -28.868757247924805,
      "activations/layer23_attention_weight_max": 38.49119186401367,
      "activations/layer23_attention_weight_min": -23.322509765625,
      "activations/layer2_attention_weight_max": 33.631282806396484,
      "activations/layer2_attention_weight_min": -33.19097137451172,
      "activations/layer3_attention_weight_max": 89.83475494384766,
      "activations/layer3_attention_weight_min": -94.61556243896484,
      "activations/layer4_attention_weight_max": 118.91349029541016,
      "activations/layer4_attention_weight_min": -115.85731506347656,
      "activations/layer5_attention_weight_max": 51.77657699584961,
      "activations/layer5_attention_weight_min": -61.58495330810547,
      "activations/layer6_attention_weight_max": 46.650577545166016,
      "activations/layer6_attention_weight_min": -53.42546463012695,
      "activations/layer7_attention_weight_max": 85.58641815185547,
      "activations/layer7_attention_weight_min": -91.99260711669922,
      "activations/layer8_attention_weight_max": 41.39412307739258,
      "activations/layer8_attention_weight_min": -43.86058807373047,
      "activations/layer9_attention_weight_max": 33.79986572265625,
      "activations/layer9_attention_weight_min": -35.51538848876953,
      "epoch": 8.51,
      "learning_rate": 9.607196969696968e-05,
      "loss": 2.8212,
      "step": 146500
    },
    {
      "activations/layer0_attention_weight_max": 14.596247673034668,
      "activations/layer0_attention_weight_min": -12.244463920593262,
      "activations/layer10_attention_weight_max": 34.655662536621094,
      "activations/layer10_attention_weight_min": -32.650665283203125,
      "activations/layer11_attention_weight_max": 34.91698455810547,
      "activations/layer11_attention_weight_min": -31.062244415283203,
      "activations/layer12_attention_weight_max": 30.655332565307617,
      "activations/layer12_attention_weight_min": -25.854455947875977,
      "activations/layer13_attention_weight_max": 42.40693283081055,
      "activations/layer13_attention_weight_min": -34.83785629272461,
      "activations/layer14_attention_weight_max": 53.36806869506836,
      "activations/layer14_attention_weight_min": -42.39152908325195,
      "activations/layer15_attention_weight_max": 37.606483459472656,
      "activations/layer15_attention_weight_min": -30.456327438354492,
      "activations/layer16_attention_weight_max": 36.70165252685547,
      "activations/layer16_attention_weight_min": -27.599063873291016,
      "activations/layer17_attention_weight_max": 35.922454833984375,
      "activations/layer17_attention_weight_min": -25.820037841796875,
      "activations/layer18_attention_weight_max": 33.40249252319336,
      "activations/layer18_attention_weight_min": -23.2106876373291,
      "activations/layer19_attention_weight_max": 39.44108581542969,
      "activations/layer19_attention_weight_min": -30.768152236938477,
      "activations/layer1_attention_weight_max": 16.909814834594727,
      "activations/layer1_attention_weight_min": -13.347558975219727,
      "activations/layer20_attention_weight_max": 42.24439239501953,
      "activations/layer20_attention_weight_min": -24.66457748413086,
      "activations/layer21_attention_weight_max": 33.94355010986328,
      "activations/layer21_attention_weight_min": -22.72743797302246,
      "activations/layer22_attention_weight_max": 52.76885986328125,
      "activations/layer22_attention_weight_min": -31.00953483581543,
      "activations/layer23_attention_weight_max": 34.78083801269531,
      "activations/layer23_attention_weight_min": -24.581701278686523,
      "activations/layer2_attention_weight_max": 31.66315460205078,
      "activations/layer2_attention_weight_min": -30.50421714782715,
      "activations/layer3_attention_weight_max": 93.21244049072266,
      "activations/layer3_attention_weight_min": -91.40013122558594,
      "activations/layer4_attention_weight_max": 122.31900787353516,
      "activations/layer4_attention_weight_min": -114.98101043701172,
      "activations/layer5_attention_weight_max": 58.5258903503418,
      "activations/layer5_attention_weight_min": -61.796043395996094,
      "activations/layer6_attention_weight_max": 49.676429748535156,
      "activations/layer6_attention_weight_min": -50.69379806518555,
      "activations/layer7_attention_weight_max": 92.20054626464844,
      "activations/layer7_attention_weight_min": -87.66886901855469,
      "activations/layer8_attention_weight_max": 40.35899353027344,
      "activations/layer8_attention_weight_min": -42.533103942871094,
      "activations/layer9_attention_weight_max": 36.3149299621582,
      "activations/layer9_attention_weight_min": -35.17481994628906,
      "epoch": 8.52,
      "learning_rate": 9.605303030303029e-05,
      "loss": 2.804,
      "step": 146550
    },
    {
      "activations/layer0_attention_weight_max": 15.376214981079102,
      "activations/layer0_attention_weight_min": -12.477299690246582,
      "activations/layer10_attention_weight_max": 35.17983627319336,
      "activations/layer10_attention_weight_min": -33.58838653564453,
      "activations/layer11_attention_weight_max": 33.82600402832031,
      "activations/layer11_attention_weight_min": -30.428050994873047,
      "activations/layer12_attention_weight_max": 23.893299102783203,
      "activations/layer12_attention_weight_min": -25.29180145263672,
      "activations/layer13_attention_weight_max": 40.2469482421875,
      "activations/layer13_attention_weight_min": -33.5308837890625,
      "activations/layer14_attention_weight_max": 54.24474334716797,
      "activations/layer14_attention_weight_min": -41.74312210083008,
      "activations/layer15_attention_weight_max": 38.416908264160156,
      "activations/layer15_attention_weight_min": -32.229530334472656,
      "activations/layer16_attention_weight_max": 31.3265380859375,
      "activations/layer16_attention_weight_min": -26.831668853759766,
      "activations/layer17_attention_weight_max": 35.03859329223633,
      "activations/layer17_attention_weight_min": -26.177148818969727,
      "activations/layer18_attention_weight_max": 29.951339721679688,
      "activations/layer18_attention_weight_min": -21.47364616394043,
      "activations/layer19_attention_weight_max": 34.71723556518555,
      "activations/layer19_attention_weight_min": -30.42428970336914,
      "activations/layer1_attention_weight_max": 16.13677215576172,
      "activations/layer1_attention_weight_min": -14.655088424682617,
      "activations/layer20_attention_weight_max": 31.817583084106445,
      "activations/layer20_attention_weight_min": -24.007047653198242,
      "activations/layer21_attention_weight_max": 32.69757843017578,
      "activations/layer21_attention_weight_min": -21.694656372070312,
      "activations/layer22_attention_weight_max": 43.358642578125,
      "activations/layer22_attention_weight_min": -29.69740867614746,
      "activations/layer23_attention_weight_max": 34.3307991027832,
      "activations/layer23_attention_weight_min": -23.8488826751709,
      "activations/layer2_attention_weight_max": 34.002166748046875,
      "activations/layer2_attention_weight_min": -33.44005584716797,
      "activations/layer3_attention_weight_max": 92.60929107666016,
      "activations/layer3_attention_weight_min": -94.34320068359375,
      "activations/layer4_attention_weight_max": 123.7558364868164,
      "activations/layer4_attention_weight_min": -123.58248138427734,
      "activations/layer5_attention_weight_max": 56.741729736328125,
      "activations/layer5_attention_weight_min": -62.96208190917969,
      "activations/layer6_attention_weight_max": 49.891441345214844,
      "activations/layer6_attention_weight_min": -54.06224822998047,
      "activations/layer7_attention_weight_max": 92.44635009765625,
      "activations/layer7_attention_weight_min": -97.39292907714844,
      "activations/layer8_attention_weight_max": 42.7792854309082,
      "activations/layer8_attention_weight_min": -44.37623977661133,
      "activations/layer9_attention_weight_max": 38.004600524902344,
      "activations/layer9_attention_weight_min": -37.16604232788086,
      "epoch": 8.52,
      "learning_rate": 9.60340909090909e-05,
      "loss": 2.8085,
      "step": 146600
    },
    {
      "activations/layer0_attention_weight_max": 15.588857650756836,
      "activations/layer0_attention_weight_min": -11.803847312927246,
      "activations/layer10_attention_weight_max": 32.20510482788086,
      "activations/layer10_attention_weight_min": -32.44826126098633,
      "activations/layer11_attention_weight_max": 34.63817596435547,
      "activations/layer11_attention_weight_min": -31.443946838378906,
      "activations/layer12_attention_weight_max": 22.999122619628906,
      "activations/layer12_attention_weight_min": -23.87217903137207,
      "activations/layer13_attention_weight_max": 38.53029251098633,
      "activations/layer13_attention_weight_min": -31.908639907836914,
      "activations/layer14_attention_weight_max": 44.54087829589844,
      "activations/layer14_attention_weight_min": -39.78453063964844,
      "activations/layer15_attention_weight_max": 34.88905715942383,
      "activations/layer15_attention_weight_min": -28.78837776184082,
      "activations/layer16_attention_weight_max": 34.55581283569336,
      "activations/layer16_attention_weight_min": -28.143945693969727,
      "activations/layer17_attention_weight_max": 37.046783447265625,
      "activations/layer17_attention_weight_min": -25.5565128326416,
      "activations/layer18_attention_weight_max": 31.837215423583984,
      "activations/layer18_attention_weight_min": -21.970993041992188,
      "activations/layer19_attention_weight_max": 37.6683235168457,
      "activations/layer19_attention_weight_min": -30.682327270507812,
      "activations/layer1_attention_weight_max": 16.665996551513672,
      "activations/layer1_attention_weight_min": -14.388138771057129,
      "activations/layer20_attention_weight_max": 30.2019100189209,
      "activations/layer20_attention_weight_min": -23.688310623168945,
      "activations/layer21_attention_weight_max": 28.80837631225586,
      "activations/layer21_attention_weight_min": -22.19834327697754,
      "activations/layer22_attention_weight_max": 42.88795852661133,
      "activations/layer22_attention_weight_min": -31.090112686157227,
      "activations/layer23_attention_weight_max": 34.94572067260742,
      "activations/layer23_attention_weight_min": -26.16707420349121,
      "activations/layer2_attention_weight_max": 33.39456558227539,
      "activations/layer2_attention_weight_min": -32.076995849609375,
      "activations/layer3_attention_weight_max": 93.4786148071289,
      "activations/layer3_attention_weight_min": -93.71829986572266,
      "activations/layer4_attention_weight_max": 120.40467834472656,
      "activations/layer4_attention_weight_min": -117.50072479248047,
      "activations/layer5_attention_weight_max": 56.1919059753418,
      "activations/layer5_attention_weight_min": -62.754295349121094,
      "activations/layer6_attention_weight_max": 49.366188049316406,
      "activations/layer6_attention_weight_min": -49.34010314941406,
      "activations/layer7_attention_weight_max": 90.24288940429688,
      "activations/layer7_attention_weight_min": -87.92144012451172,
      "activations/layer8_attention_weight_max": 41.6176643371582,
      "activations/layer8_attention_weight_min": -41.357017517089844,
      "activations/layer9_attention_weight_max": 35.11271286010742,
      "activations/layer9_attention_weight_min": -34.27172088623047,
      "epoch": 8.52,
      "learning_rate": 9.60151515151515e-05,
      "loss": 2.7998,
      "step": 146650
    },
    {
      "activations/layer0_attention_weight_max": 14.947132110595703,
      "activations/layer0_attention_weight_min": -12.832293510437012,
      "activations/layer10_attention_weight_max": 35.66069030761719,
      "activations/layer10_attention_weight_min": -32.1279182434082,
      "activations/layer11_attention_weight_max": 35.180355072021484,
      "activations/layer11_attention_weight_min": -30.145063400268555,
      "activations/layer12_attention_weight_max": 24.706525802612305,
      "activations/layer12_attention_weight_min": -24.439865112304688,
      "activations/layer13_attention_weight_max": 41.40462875366211,
      "activations/layer13_attention_weight_min": -35.10874938964844,
      "activations/layer14_attention_weight_max": 48.10295486450195,
      "activations/layer14_attention_weight_min": -39.88074493408203,
      "activations/layer15_attention_weight_max": 33.845191955566406,
      "activations/layer15_attention_weight_min": -30.12440299987793,
      "activations/layer16_attention_weight_max": 30.537874221801758,
      "activations/layer16_attention_weight_min": -28.689449310302734,
      "activations/layer17_attention_weight_max": 31.994409561157227,
      "activations/layer17_attention_weight_min": -24.247610092163086,
      "activations/layer18_attention_weight_max": 34.1664924621582,
      "activations/layer18_attention_weight_min": -23.011716842651367,
      "activations/layer19_attention_weight_max": 33.70936965942383,
      "activations/layer19_attention_weight_min": -31.798791885375977,
      "activations/layer1_attention_weight_max": 16.933252334594727,
      "activations/layer1_attention_weight_min": -16.262699127197266,
      "activations/layer20_attention_weight_max": 28.60038948059082,
      "activations/layer20_attention_weight_min": -24.3945369720459,
      "activations/layer21_attention_weight_max": 29.741729736328125,
      "activations/layer21_attention_weight_min": -23.707693099975586,
      "activations/layer22_attention_weight_max": 48.85655212402344,
      "activations/layer22_attention_weight_min": -28.20543670654297,
      "activations/layer23_attention_weight_max": 36.96247100830078,
      "activations/layer23_attention_weight_min": -24.0679988861084,
      "activations/layer2_attention_weight_max": 32.813804626464844,
      "activations/layer2_attention_weight_min": -32.330814361572266,
      "activations/layer3_attention_weight_max": 92.48013305664062,
      "activations/layer3_attention_weight_min": -93.29466247558594,
      "activations/layer4_attention_weight_max": 119.15726470947266,
      "activations/layer4_attention_weight_min": -121.80550384521484,
      "activations/layer5_attention_weight_max": 54.80854034423828,
      "activations/layer5_attention_weight_min": -61.81745910644531,
      "activations/layer6_attention_weight_max": 47.463375091552734,
      "activations/layer6_attention_weight_min": -52.25884246826172,
      "activations/layer7_attention_weight_max": 86.30274963378906,
      "activations/layer7_attention_weight_min": -94.65875244140625,
      "activations/layer8_attention_weight_max": 40.7756462097168,
      "activations/layer8_attention_weight_min": -41.465362548828125,
      "activations/layer9_attention_weight_max": 36.3038444519043,
      "activations/layer9_attention_weight_min": -35.374874114990234,
      "epoch": 8.52,
      "learning_rate": 9.599621212121211e-05,
      "loss": 2.8175,
      "step": 146700
    },
    {
      "activations/layer0_attention_weight_max": 14.760757446289062,
      "activations/layer0_attention_weight_min": -11.247085571289062,
      "activations/layer10_attention_weight_max": 34.53382110595703,
      "activations/layer10_attention_weight_min": -34.622310638427734,
      "activations/layer11_attention_weight_max": 35.660560607910156,
      "activations/layer11_attention_weight_min": -34.47882843017578,
      "activations/layer12_attention_weight_max": 33.027164459228516,
      "activations/layer12_attention_weight_min": -24.85718536376953,
      "activations/layer13_attention_weight_max": 45.55305099487305,
      "activations/layer13_attention_weight_min": -33.8380012512207,
      "activations/layer14_attention_weight_max": 64.02027893066406,
      "activations/layer14_attention_weight_min": -40.090309143066406,
      "activations/layer15_attention_weight_max": 38.525856018066406,
      "activations/layer15_attention_weight_min": -31.926958084106445,
      "activations/layer16_attention_weight_max": 36.561744689941406,
      "activations/layer16_attention_weight_min": -28.382280349731445,
      "activations/layer17_attention_weight_max": 35.461647033691406,
      "activations/layer17_attention_weight_min": -26.53745460510254,
      "activations/layer18_attention_weight_max": 36.03916549682617,
      "activations/layer18_attention_weight_min": -20.45370101928711,
      "activations/layer19_attention_weight_max": 37.77952194213867,
      "activations/layer19_attention_weight_min": -29.706439971923828,
      "activations/layer1_attention_weight_max": 16.706192016601562,
      "activations/layer1_attention_weight_min": -14.895857810974121,
      "activations/layer20_attention_weight_max": 35.876564025878906,
      "activations/layer20_attention_weight_min": -22.97408676147461,
      "activations/layer21_attention_weight_max": 30.470428466796875,
      "activations/layer21_attention_weight_min": -22.095468521118164,
      "activations/layer22_attention_weight_max": 47.28794860839844,
      "activations/layer22_attention_weight_min": -29.5859432220459,
      "activations/layer23_attention_weight_max": 34.34840774536133,
      "activations/layer23_attention_weight_min": -21.648338317871094,
      "activations/layer2_attention_weight_max": 31.169668197631836,
      "activations/layer2_attention_weight_min": -32.15608215332031,
      "activations/layer3_attention_weight_max": 96.78762817382812,
      "activations/layer3_attention_weight_min": -97.52928924560547,
      "activations/layer4_attention_weight_max": 128.71922302246094,
      "activations/layer4_attention_weight_min": -126.64066314697266,
      "activations/layer5_attention_weight_max": 62.585968017578125,
      "activations/layer5_attention_weight_min": -65.98670959472656,
      "activations/layer6_attention_weight_max": 53.41118240356445,
      "activations/layer6_attention_weight_min": -57.385032653808594,
      "activations/layer7_attention_weight_max": 101.3799057006836,
      "activations/layer7_attention_weight_min": -95.04130554199219,
      "activations/layer8_attention_weight_max": 45.399574279785156,
      "activations/layer8_attention_weight_min": -49.58644485473633,
      "activations/layer9_attention_weight_max": 39.81611633300781,
      "activations/layer9_attention_weight_min": -37.39009475708008,
      "epoch": 8.53,
      "learning_rate": 9.597727272727272e-05,
      "loss": 2.7915,
      "step": 146750
    },
    {
      "activations/layer0_attention_weight_max": 15.2648344039917,
      "activations/layer0_attention_weight_min": -11.799092292785645,
      "activations/layer10_attention_weight_max": 32.2188835144043,
      "activations/layer10_attention_weight_min": -31.499876022338867,
      "activations/layer11_attention_weight_max": 32.95151901245117,
      "activations/layer11_attention_weight_min": -30.766647338867188,
      "activations/layer12_attention_weight_max": 24.19989013671875,
      "activations/layer12_attention_weight_min": -25.277238845825195,
      "activations/layer13_attention_weight_max": 39.21529769897461,
      "activations/layer13_attention_weight_min": -33.4550666809082,
      "activations/layer14_attention_weight_max": 51.595951080322266,
      "activations/layer14_attention_weight_min": -40.94167709350586,
      "activations/layer15_attention_weight_max": 38.12273406982422,
      "activations/layer15_attention_weight_min": -29.845754623413086,
      "activations/layer16_attention_weight_max": 40.26564025878906,
      "activations/layer16_attention_weight_min": -28.6888484954834,
      "activations/layer17_attention_weight_max": 36.79166793823242,
      "activations/layer17_attention_weight_min": -25.381519317626953,
      "activations/layer18_attention_weight_max": 29.711585998535156,
      "activations/layer18_attention_weight_min": -22.239002227783203,
      "activations/layer19_attention_weight_max": 38.795528411865234,
      "activations/layer19_attention_weight_min": -33.00735092163086,
      "activations/layer1_attention_weight_max": 15.999000549316406,
      "activations/layer1_attention_weight_min": -14.038800239562988,
      "activations/layer20_attention_weight_max": 32.346614837646484,
      "activations/layer20_attention_weight_min": -26.889114379882812,
      "activations/layer21_attention_weight_max": 31.69455909729004,
      "activations/layer21_attention_weight_min": -23.78143882751465,
      "activations/layer22_attention_weight_max": 49.57102584838867,
      "activations/layer22_attention_weight_min": -29.590675354003906,
      "activations/layer23_attention_weight_max": 39.53026580810547,
      "activations/layer23_attention_weight_min": -24.868383407592773,
      "activations/layer2_attention_weight_max": 32.212276458740234,
      "activations/layer2_attention_weight_min": -31.8546199798584,
      "activations/layer3_attention_weight_max": 91.89431762695312,
      "activations/layer3_attention_weight_min": -93.41130065917969,
      "activations/layer4_attention_weight_max": 117.71414947509766,
      "activations/layer4_attention_weight_min": -122.13703918457031,
      "activations/layer5_attention_weight_max": 57.00141525268555,
      "activations/layer5_attention_weight_min": -60.96916961669922,
      "activations/layer6_attention_weight_max": 52.000362396240234,
      "activations/layer6_attention_weight_min": -48.26118469238281,
      "activations/layer7_attention_weight_max": 93.32049560546875,
      "activations/layer7_attention_weight_min": -87.5511474609375,
      "activations/layer8_attention_weight_max": 39.37519454956055,
      "activations/layer8_attention_weight_min": -39.73021697998047,
      "activations/layer9_attention_weight_max": 34.38264083862305,
      "activations/layer9_attention_weight_min": -32.95110321044922,
      "epoch": 8.53,
      "learning_rate": 9.595833333333333e-05,
      "loss": 2.8125,
      "step": 146800
    },
    {
      "activations/layer0_attention_weight_max": 15.316412925720215,
      "activations/layer0_attention_weight_min": -11.670266151428223,
      "activations/layer10_attention_weight_max": 33.422767639160156,
      "activations/layer10_attention_weight_min": -35.41347885131836,
      "activations/layer11_attention_weight_max": 31.81149673461914,
      "activations/layer11_attention_weight_min": -33.057586669921875,
      "activations/layer12_attention_weight_max": 23.873573303222656,
      "activations/layer12_attention_weight_min": -25.034805297851562,
      "activations/layer13_attention_weight_max": 40.54829406738281,
      "activations/layer13_attention_weight_min": -33.56023406982422,
      "activations/layer14_attention_weight_max": 52.08964538574219,
      "activations/layer14_attention_weight_min": -41.08928298950195,
      "activations/layer15_attention_weight_max": 37.48786926269531,
      "activations/layer15_attention_weight_min": -33.25127029418945,
      "activations/layer16_attention_weight_max": 32.500099182128906,
      "activations/layer16_attention_weight_min": -27.01313591003418,
      "activations/layer17_attention_weight_max": 34.42283630371094,
      "activations/layer17_attention_weight_min": -24.33745574951172,
      "activations/layer18_attention_weight_max": 34.419010162353516,
      "activations/layer18_attention_weight_min": -20.630084991455078,
      "activations/layer19_attention_weight_max": 37.59122848510742,
      "activations/layer19_attention_weight_min": -29.626184463500977,
      "activations/layer1_attention_weight_max": 16.703872680664062,
      "activations/layer1_attention_weight_min": -14.465479850769043,
      "activations/layer20_attention_weight_max": 32.35985565185547,
      "activations/layer20_attention_weight_min": -23.15953254699707,
      "activations/layer21_attention_weight_max": 28.93162727355957,
      "activations/layer21_attention_weight_min": -22.564401626586914,
      "activations/layer22_attention_weight_max": 45.894248962402344,
      "activations/layer22_attention_weight_min": -28.826250076293945,
      "activations/layer23_attention_weight_max": 40.13637161254883,
      "activations/layer23_attention_weight_min": -22.99927520751953,
      "activations/layer2_attention_weight_max": 31.17876434326172,
      "activations/layer2_attention_weight_min": -30.696205139160156,
      "activations/layer3_attention_weight_max": 98.62776947021484,
      "activations/layer3_attention_weight_min": -95.15829467773438,
      "activations/layer4_attention_weight_max": 121.37670135498047,
      "activations/layer4_attention_weight_min": -116.2584228515625,
      "activations/layer5_attention_weight_max": 54.42754364013672,
      "activations/layer5_attention_weight_min": -60.63159942626953,
      "activations/layer6_attention_weight_max": 47.311256408691406,
      "activations/layer6_attention_weight_min": -50.74321365356445,
      "activations/layer7_attention_weight_max": 90.6444320678711,
      "activations/layer7_attention_weight_min": -88.94701385498047,
      "activations/layer8_attention_weight_max": 41.77200698852539,
      "activations/layer8_attention_weight_min": -39.31085205078125,
      "activations/layer9_attention_weight_max": 38.01522445678711,
      "activations/layer9_attention_weight_min": -34.99801254272461,
      "epoch": 8.53,
      "learning_rate": 9.593939393939393e-05,
      "loss": 2.7966,
      "step": 146850
    },
    {
      "activations/layer0_attention_weight_max": 15.057268142700195,
      "activations/layer0_attention_weight_min": -11.813169479370117,
      "activations/layer10_attention_weight_max": 47.36094665527344,
      "activations/layer10_attention_weight_min": -44.85733413696289,
      "activations/layer11_attention_weight_max": 51.5123405456543,
      "activations/layer11_attention_weight_min": -44.964622497558594,
      "activations/layer12_attention_weight_max": 37.50837707519531,
      "activations/layer12_attention_weight_min": -27.3277530670166,
      "activations/layer13_attention_weight_max": 62.77000045776367,
      "activations/layer13_attention_weight_min": -39.17329788208008,
      "activations/layer14_attention_weight_max": 70.61371612548828,
      "activations/layer14_attention_weight_min": -47.773502349853516,
      "activations/layer15_attention_weight_max": 75.21976470947266,
      "activations/layer15_attention_weight_min": -38.32052230834961,
      "activations/layer16_attention_weight_max": 42.354103088378906,
      "activations/layer16_attention_weight_min": -28.845502853393555,
      "activations/layer17_attention_weight_max": 33.95695495605469,
      "activations/layer17_attention_weight_min": -26.754419326782227,
      "activations/layer18_attention_weight_max": 30.648883819580078,
      "activations/layer18_attention_weight_min": -23.860309600830078,
      "activations/layer19_attention_weight_max": 39.172821044921875,
      "activations/layer19_attention_weight_min": -30.789403915405273,
      "activations/layer1_attention_weight_max": 18.12613868713379,
      "activations/layer1_attention_weight_min": -13.989197731018066,
      "activations/layer20_attention_weight_max": 29.97062873840332,
      "activations/layer20_attention_weight_min": -23.33133316040039,
      "activations/layer21_attention_weight_max": 27.207786560058594,
      "activations/layer21_attention_weight_min": -22.355224609375,
      "activations/layer22_attention_weight_max": 54.2396354675293,
      "activations/layer22_attention_weight_min": -27.80595588684082,
      "activations/layer23_attention_weight_max": 37.81845474243164,
      "activations/layer23_attention_weight_min": -22.164663314819336,
      "activations/layer2_attention_weight_max": 33.84614562988281,
      "activations/layer2_attention_weight_min": -32.354496002197266,
      "activations/layer3_attention_weight_max": 99.53472900390625,
      "activations/layer3_attention_weight_min": -100.28659057617188,
      "activations/layer4_attention_weight_max": 126.06743621826172,
      "activations/layer4_attention_weight_min": -120.56774139404297,
      "activations/layer5_attention_weight_max": 62.98069381713867,
      "activations/layer5_attention_weight_min": -61.44263458251953,
      "activations/layer6_attention_weight_max": 54.00273132324219,
      "activations/layer6_attention_weight_min": -49.72178649902344,
      "activations/layer7_attention_weight_max": 111.39116668701172,
      "activations/layer7_attention_weight_min": -106.63508605957031,
      "activations/layer8_attention_weight_max": 49.30323791503906,
      "activations/layer8_attention_weight_min": -52.66155242919922,
      "activations/layer9_attention_weight_max": 47.93301010131836,
      "activations/layer9_attention_weight_min": -46.388755798339844,
      "epoch": 8.54,
      "learning_rate": 9.592045454545454e-05,
      "loss": 2.7949,
      "step": 146900
    },
    {
      "activations/layer0_attention_weight_max": 15.337401390075684,
      "activations/layer0_attention_weight_min": -13.049925804138184,
      "activations/layer10_attention_weight_max": 39.41518783569336,
      "activations/layer10_attention_weight_min": -36.53675842285156,
      "activations/layer11_attention_weight_max": 38.2984733581543,
      "activations/layer11_attention_weight_min": -34.0618896484375,
      "activations/layer12_attention_weight_max": 34.143367767333984,
      "activations/layer12_attention_weight_min": -26.13609504699707,
      "activations/layer13_attention_weight_max": 47.33180618286133,
      "activations/layer13_attention_weight_min": -34.31621551513672,
      "activations/layer14_attention_weight_max": 65.20808410644531,
      "activations/layer14_attention_weight_min": -47.963775634765625,
      "activations/layer15_attention_weight_max": 39.57032775878906,
      "activations/layer15_attention_weight_min": -32.31425857543945,
      "activations/layer16_attention_weight_max": 34.389892578125,
      "activations/layer16_attention_weight_min": -29.011322021484375,
      "activations/layer17_attention_weight_max": 36.2975959777832,
      "activations/layer17_attention_weight_min": -25.256980895996094,
      "activations/layer18_attention_weight_max": 34.809600830078125,
      "activations/layer18_attention_weight_min": -21.5688533782959,
      "activations/layer19_attention_weight_max": 39.6251335144043,
      "activations/layer19_attention_weight_min": -32.14999008178711,
      "activations/layer1_attention_weight_max": 17.2445125579834,
      "activations/layer1_attention_weight_min": -16.62127113342285,
      "activations/layer20_attention_weight_max": 34.83157730102539,
      "activations/layer20_attention_weight_min": -24.170055389404297,
      "activations/layer21_attention_weight_max": 33.89167022705078,
      "activations/layer21_attention_weight_min": -23.28261375427246,
      "activations/layer22_attention_weight_max": 51.42049026489258,
      "activations/layer22_attention_weight_min": -29.59168243408203,
      "activations/layer23_attention_weight_max": 41.22273635864258,
      "activations/layer23_attention_weight_min": -24.47604751586914,
      "activations/layer2_attention_weight_max": 34.27317810058594,
      "activations/layer2_attention_weight_min": -33.75572204589844,
      "activations/layer3_attention_weight_max": 95.67376708984375,
      "activations/layer3_attention_weight_min": -94.96775817871094,
      "activations/layer4_attention_weight_max": 124.27591705322266,
      "activations/layer4_attention_weight_min": -123.2175521850586,
      "activations/layer5_attention_weight_max": 57.55527877807617,
      "activations/layer5_attention_weight_min": -65.21215057373047,
      "activations/layer6_attention_weight_max": 51.80400085449219,
      "activations/layer6_attention_weight_min": -52.786521911621094,
      "activations/layer7_attention_weight_max": 98.95509338378906,
      "activations/layer7_attention_weight_min": -99.82447814941406,
      "activations/layer8_attention_weight_max": 46.825016021728516,
      "activations/layer8_attention_weight_min": -44.7213020324707,
      "activations/layer9_attention_weight_max": 39.864681243896484,
      "activations/layer9_attention_weight_min": -37.99549102783203,
      "epoch": 8.54,
      "learning_rate": 9.590151515151514e-05,
      "loss": 2.7891,
      "step": 146950
    },
    {
      "activations/layer0_attention_weight_max": 14.911358833312988,
      "activations/layer0_attention_weight_min": -12.216272354125977,
      "activations/layer10_attention_weight_max": 36.272125244140625,
      "activations/layer10_attention_weight_min": -33.89032745361328,
      "activations/layer11_attention_weight_max": 38.465171813964844,
      "activations/layer11_attention_weight_min": -33.825836181640625,
      "activations/layer12_attention_weight_max": 49.205989837646484,
      "activations/layer12_attention_weight_min": -26.11654281616211,
      "activations/layer13_attention_weight_max": 50.10359191894531,
      "activations/layer13_attention_weight_min": -32.888763427734375,
      "activations/layer14_attention_weight_max": 51.30176544189453,
      "activations/layer14_attention_weight_min": -38.77959442138672,
      "activations/layer15_attention_weight_max": 47.91801834106445,
      "activations/layer15_attention_weight_min": -29.47832489013672,
      "activations/layer16_attention_weight_max": 33.58992385864258,
      "activations/layer16_attention_weight_min": -27.84322738647461,
      "activations/layer17_attention_weight_max": 30.594308853149414,
      "activations/layer17_attention_weight_min": -24.672046661376953,
      "activations/layer18_attention_weight_max": 29.746139526367188,
      "activations/layer18_attention_weight_min": -21.285512924194336,
      "activations/layer19_attention_weight_max": 37.319026947021484,
      "activations/layer19_attention_weight_min": -30.05918312072754,
      "activations/layer1_attention_weight_max": 17.019311904907227,
      "activations/layer1_attention_weight_min": -13.925239562988281,
      "activations/layer20_attention_weight_max": 33.002197265625,
      "activations/layer20_attention_weight_min": -23.892045974731445,
      "activations/layer21_attention_weight_max": 29.73140525817871,
      "activations/layer21_attention_weight_min": -23.577817916870117,
      "activations/layer22_attention_weight_max": 44.72207260131836,
      "activations/layer22_attention_weight_min": -26.844554901123047,
      "activations/layer23_attention_weight_max": 36.95091247558594,
      "activations/layer23_attention_weight_min": -26.176498413085938,
      "activations/layer2_attention_weight_max": 31.725173950195312,
      "activations/layer2_attention_weight_min": -31.50886344909668,
      "activations/layer3_attention_weight_max": 89.87873077392578,
      "activations/layer3_attention_weight_min": -91.61250305175781,
      "activations/layer4_attention_weight_max": 119.90160369873047,
      "activations/layer4_attention_weight_min": -115.74718475341797,
      "activations/layer5_attention_weight_max": 58.13964080810547,
      "activations/layer5_attention_weight_min": -63.6824951171875,
      "activations/layer6_attention_weight_max": 51.33304214477539,
      "activations/layer6_attention_weight_min": -51.880165100097656,
      "activations/layer7_attention_weight_max": 95.63394927978516,
      "activations/layer7_attention_weight_min": -89.86605072021484,
      "activations/layer8_attention_weight_max": 40.919456481933594,
      "activations/layer8_attention_weight_min": -40.77312088012695,
      "activations/layer9_attention_weight_max": 35.75796127319336,
      "activations/layer9_attention_weight_min": -33.80302429199219,
      "epoch": 8.54,
      "learning_rate": 9.588257575757576e-05,
      "loss": 2.8034,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.4959,
      "eval_samples_per_second": 505.42,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.4959,
      "eval_openwebtext_samples_per_second": 505.42,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 2.0114,
      "eval_wikitext_samples_per_second": 226.705,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_lambada_loss": 2.5625,
      "eval_lambada_ppl": 12.968197316969134,
      "eval_lambada_runtime": 9.6028,
      "eval_lambada_samples_per_second": 507.041,
      "step": 147000
    },
    {
      "activations/layer0_attention_weight_max": 17.261394500732422,
      "activations/layer0_attention_weight_min": -10.950085639953613,
      "activations/layer10_attention_weight_max": 35.52177047729492,
      "activations/layer10_attention_weight_min": -31.557687759399414,
      "activations/layer11_attention_weight_max": 38.033203125,
      "activations/layer11_attention_weight_min": -31.144485473632812,
      "activations/layer12_attention_weight_max": 37.759239196777344,
      "activations/layer12_attention_weight_min": -28.062397003173828,
      "activations/layer13_attention_weight_max": 52.71403121948242,
      "activations/layer13_attention_weight_min": -33.24012756347656,
      "activations/layer14_attention_weight_max": 49.35075759887695,
      "activations/layer14_attention_weight_min": -40.01076889038086,
      "activations/layer15_attention_weight_max": 53.106407165527344,
      "activations/layer15_attention_weight_min": -29.632427215576172,
      "activations/layer16_attention_weight_max": 32.17009735107422,
      "activations/layer16_attention_weight_min": -27.0146484375,
      "activations/layer17_attention_weight_max": 29.22173500061035,
      "activations/layer17_attention_weight_min": -27.462947845458984,
      "activations/layer18_attention_weight_max": 31.043100357055664,
      "activations/layer18_attention_weight_min": -22.778526306152344,
      "activations/layer19_attention_weight_max": 38.340789794921875,
      "activations/layer19_attention_weight_min": -30.383115768432617,
      "activations/layer1_attention_weight_max": 17.80105209350586,
      "activations/layer1_attention_weight_min": -13.72348403930664,
      "activations/layer20_attention_weight_max": 30.84039306640625,
      "activations/layer20_attention_weight_min": -24.241180419921875,
      "activations/layer21_attention_weight_max": 30.902902603149414,
      "activations/layer21_attention_weight_min": -23.25295639038086,
      "activations/layer22_attention_weight_max": 44.990013122558594,
      "activations/layer22_attention_weight_min": -29.016952514648438,
      "activations/layer23_attention_weight_max": 36.59739303588867,
      "activations/layer23_attention_weight_min": -22.802780151367188,
      "activations/layer2_attention_weight_max": 32.46584701538086,
      "activations/layer2_attention_weight_min": -30.935802459716797,
      "activations/layer3_attention_weight_max": 92.7145004272461,
      "activations/layer3_attention_weight_min": -95.27745819091797,
      "activations/layer4_attention_weight_max": 120.03175354003906,
      "activations/layer4_attention_weight_min": -115.05921936035156,
      "activations/layer5_attention_weight_max": 59.734615325927734,
      "activations/layer5_attention_weight_min": -63.08395004272461,
      "activations/layer6_attention_weight_max": 47.2859992980957,
      "activations/layer6_attention_weight_min": -50.210208892822266,
      "activations/layer7_attention_weight_max": 92.99935150146484,
      "activations/layer7_attention_weight_min": -89.95686340332031,
      "activations/layer8_attention_weight_max": 41.77663040161133,
      "activations/layer8_attention_weight_min": -38.8454704284668,
      "activations/layer9_attention_weight_max": 35.6196403503418,
      "activations/layer9_attention_weight_min": -32.304298400878906,
      "epoch": 8.54,
      "learning_rate": 9.586363636363635e-05,
      "loss": 2.8142,
      "step": 147050
    },
    {
      "activations/layer0_attention_weight_max": 15.918622016906738,
      "activations/layer0_attention_weight_min": -12.752745628356934,
      "activations/layer10_attention_weight_max": 33.345420837402344,
      "activations/layer10_attention_weight_min": -31.99169158935547,
      "activations/layer11_attention_weight_max": 34.56578063964844,
      "activations/layer11_attention_weight_min": -30.57954978942871,
      "activations/layer12_attention_weight_max": 31.422569274902344,
      "activations/layer12_attention_weight_min": -24.60416030883789,
      "activations/layer13_attention_weight_max": 39.82311248779297,
      "activations/layer13_attention_weight_min": -32.355682373046875,
      "activations/layer14_attention_weight_max": 48.97149658203125,
      "activations/layer14_attention_weight_min": -37.114723205566406,
      "activations/layer15_attention_weight_max": 37.610618591308594,
      "activations/layer15_attention_weight_min": -29.34339714050293,
      "activations/layer16_attention_weight_max": 38.53156661987305,
      "activations/layer16_attention_weight_min": -29.370588302612305,
      "activations/layer17_attention_weight_max": 34.55573272705078,
      "activations/layer17_attention_weight_min": -25.32110595703125,
      "activations/layer18_attention_weight_max": 29.985403060913086,
      "activations/layer18_attention_weight_min": -21.310407638549805,
      "activations/layer19_attention_weight_max": 38.206546783447266,
      "activations/layer19_attention_weight_min": -30.43071937561035,
      "activations/layer1_attention_weight_max": 16.12641716003418,
      "activations/layer1_attention_weight_min": -14.601698875427246,
      "activations/layer20_attention_weight_max": 30.82354736328125,
      "activations/layer20_attention_weight_min": -22.69228172302246,
      "activations/layer21_attention_weight_max": 29.885971069335938,
      "activations/layer21_attention_weight_min": -21.783843994140625,
      "activations/layer22_attention_weight_max": 45.66870880126953,
      "activations/layer22_attention_weight_min": -27.9727725982666,
      "activations/layer23_attention_weight_max": 34.83644104003906,
      "activations/layer23_attention_weight_min": -23.319318771362305,
      "activations/layer2_attention_weight_max": 31.066219329833984,
      "activations/layer2_attention_weight_min": -32.57038116455078,
      "activations/layer3_attention_weight_max": 89.51361846923828,
      "activations/layer3_attention_weight_min": -92.40589904785156,
      "activations/layer4_attention_weight_max": 111.51899719238281,
      "activations/layer4_attention_weight_min": -116.35572814941406,
      "activations/layer5_attention_weight_max": 53.95922088623047,
      "activations/layer5_attention_weight_min": -60.26313781738281,
      "activations/layer6_attention_weight_max": 46.939369201660156,
      "activations/layer6_attention_weight_min": -51.07453918457031,
      "activations/layer7_attention_weight_max": 93.71549224853516,
      "activations/layer7_attention_weight_min": -89.79508972167969,
      "activations/layer8_attention_weight_max": 38.821434020996094,
      "activations/layer8_attention_weight_min": -40.302711486816406,
      "activations/layer9_attention_weight_max": 34.3592414855957,
      "activations/layer9_attention_weight_min": -33.96677780151367,
      "epoch": 8.55,
      "learning_rate": 9.584469696969696e-05,
      "loss": 2.8133,
      "step": 147100
    },
    {
      "activations/layer0_attention_weight_max": 15.227848052978516,
      "activations/layer0_attention_weight_min": -12.674749374389648,
      "activations/layer10_attention_weight_max": 32.77144241333008,
      "activations/layer10_attention_weight_min": -31.39457893371582,
      "activations/layer11_attention_weight_max": 32.226646423339844,
      "activations/layer11_attention_weight_min": -30.188213348388672,
      "activations/layer12_attention_weight_max": 23.754486083984375,
      "activations/layer12_attention_weight_min": -26.692636489868164,
      "activations/layer13_attention_weight_max": 38.81243896484375,
      "activations/layer13_attention_weight_min": -32.49775695800781,
      "activations/layer14_attention_weight_max": 47.72513961791992,
      "activations/layer14_attention_weight_min": -40.018951416015625,
      "activations/layer15_attention_weight_max": 36.758331298828125,
      "activations/layer15_attention_weight_min": -30.51683807373047,
      "activations/layer16_attention_weight_max": 32.92720413208008,
      "activations/layer16_attention_weight_min": -26.485078811645508,
      "activations/layer17_attention_weight_max": 31.608402252197266,
      "activations/layer17_attention_weight_min": -24.148767471313477,
      "activations/layer18_attention_weight_max": 26.984195709228516,
      "activations/layer18_attention_weight_min": -20.95416831970215,
      "activations/layer19_attention_weight_max": 33.37556076049805,
      "activations/layer19_attention_weight_min": -30.779245376586914,
      "activations/layer1_attention_weight_max": 16.304136276245117,
      "activations/layer1_attention_weight_min": -15.710677146911621,
      "activations/layer20_attention_weight_max": 29.432403564453125,
      "activations/layer20_attention_weight_min": -24.27544403076172,
      "activations/layer21_attention_weight_max": 30.46492576599121,
      "activations/layer21_attention_weight_min": -22.47970199584961,
      "activations/layer22_attention_weight_max": 49.964420318603516,
      "activations/layer22_attention_weight_min": -28.451745986938477,
      "activations/layer23_attention_weight_max": 34.31553649902344,
      "activations/layer23_attention_weight_min": -23.17031478881836,
      "activations/layer2_attention_weight_max": 31.805225372314453,
      "activations/layer2_attention_weight_min": -31.740293502807617,
      "activations/layer3_attention_weight_max": 96.11552429199219,
      "activations/layer3_attention_weight_min": -97.4853744506836,
      "activations/layer4_attention_weight_max": 121.61034393310547,
      "activations/layer4_attention_weight_min": -116.7597427368164,
      "activations/layer5_attention_weight_max": 53.68217086791992,
      "activations/layer5_attention_weight_min": -59.39585494995117,
      "activations/layer6_attention_weight_max": 46.731021881103516,
      "activations/layer6_attention_weight_min": -52.82734298706055,
      "activations/layer7_attention_weight_max": 91.01224517822266,
      "activations/layer7_attention_weight_min": -96.50537872314453,
      "activations/layer8_attention_weight_max": 39.98427200317383,
      "activations/layer8_attention_weight_min": -40.134056091308594,
      "activations/layer9_attention_weight_max": 33.54397201538086,
      "activations/layer9_attention_weight_min": -35.51783752441406,
      "epoch": 8.55,
      "learning_rate": 9.582613636363635e-05,
      "loss": 2.8116,
      "step": 147150
    },
    {
      "activations/layer0_attention_weight_max": 15.321657180786133,
      "activations/layer0_attention_weight_min": -12.719683647155762,
      "activations/layer10_attention_weight_max": 30.9566650390625,
      "activations/layer10_attention_weight_min": -32.949302673339844,
      "activations/layer11_attention_weight_max": 32.316375732421875,
      "activations/layer11_attention_weight_min": -31.69684600830078,
      "activations/layer12_attention_weight_max": 27.041767120361328,
      "activations/layer12_attention_weight_min": -25.632814407348633,
      "activations/layer13_attention_weight_max": 40.73259735107422,
      "activations/layer13_attention_weight_min": -33.56825256347656,
      "activations/layer14_attention_weight_max": 49.05087661743164,
      "activations/layer14_attention_weight_min": -39.697174072265625,
      "activations/layer15_attention_weight_max": 38.34152603149414,
      "activations/layer15_attention_weight_min": -33.05739974975586,
      "activations/layer16_attention_weight_max": 38.7660026550293,
      "activations/layer16_attention_weight_min": -29.062580108642578,
      "activations/layer17_attention_weight_max": 38.4728889465332,
      "activations/layer17_attention_weight_min": -25.352060317993164,
      "activations/layer18_attention_weight_max": 32.52735900878906,
      "activations/layer18_attention_weight_min": -23.753202438354492,
      "activations/layer19_attention_weight_max": 44.74123764038086,
      "activations/layer19_attention_weight_min": -32.71786117553711,
      "activations/layer1_attention_weight_max": 16.63481330871582,
      "activations/layer1_attention_weight_min": -14.084698677062988,
      "activations/layer20_attention_weight_max": 40.32218551635742,
      "activations/layer20_attention_weight_min": -25.47361946105957,
      "activations/layer21_attention_weight_max": 38.7126579284668,
      "activations/layer21_attention_weight_min": -22.86838722229004,
      "activations/layer22_attention_weight_max": 55.42290496826172,
      "activations/layer22_attention_weight_min": -29.885196685791016,
      "activations/layer23_attention_weight_max": 45.687721252441406,
      "activations/layer23_attention_weight_min": -24.38142204284668,
      "activations/layer2_attention_weight_max": 32.76744842529297,
      "activations/layer2_attention_weight_min": -33.80826187133789,
      "activations/layer3_attention_weight_max": 96.20169067382812,
      "activations/layer3_attention_weight_min": -94.42149353027344,
      "activations/layer4_attention_weight_max": 124.94332122802734,
      "activations/layer4_attention_weight_min": -121.10772705078125,
      "activations/layer5_attention_weight_max": 57.28911590576172,
      "activations/layer5_attention_weight_min": -65.49636840820312,
      "activations/layer6_attention_weight_max": 52.44546127319336,
      "activations/layer6_attention_weight_min": -53.23419952392578,
      "activations/layer7_attention_weight_max": 90.56533813476562,
      "activations/layer7_attention_weight_min": -94.89134979248047,
      "activations/layer8_attention_weight_max": 40.64501190185547,
      "activations/layer8_attention_weight_min": -40.478050231933594,
      "activations/layer9_attention_weight_max": 33.47774887084961,
      "activations/layer9_attention_weight_min": -34.276119232177734,
      "epoch": 8.55,
      "learning_rate": 9.580719696969697e-05,
      "loss": 2.79,
      "step": 147200
    },
    {
      "activations/layer0_attention_weight_max": 15.974422454833984,
      "activations/layer0_attention_weight_min": -12.091753959655762,
      "activations/layer10_attention_weight_max": 32.24281692504883,
      "activations/layer10_attention_weight_min": -31.333709716796875,
      "activations/layer11_attention_weight_max": 32.040626525878906,
      "activations/layer11_attention_weight_min": -27.842849731445312,
      "activations/layer12_attention_weight_max": 23.090673446655273,
      "activations/layer12_attention_weight_min": -25.4584903717041,
      "activations/layer13_attention_weight_max": 39.72655487060547,
      "activations/layer13_attention_weight_min": -33.87132263183594,
      "activations/layer14_attention_weight_max": 48.72941970825195,
      "activations/layer14_attention_weight_min": -37.16631317138672,
      "activations/layer15_attention_weight_max": 37.628662109375,
      "activations/layer15_attention_weight_min": -30.667055130004883,
      "activations/layer16_attention_weight_max": 33.66739273071289,
      "activations/layer16_attention_weight_min": -28.189538955688477,
      "activations/layer17_attention_weight_max": 33.42911148071289,
      "activations/layer17_attention_weight_min": -25.480146408081055,
      "activations/layer18_attention_weight_max": 31.50603675842285,
      "activations/layer18_attention_weight_min": -22.21925163269043,
      "activations/layer19_attention_weight_max": 37.40300369262695,
      "activations/layer19_attention_weight_min": -32.53867721557617,
      "activations/layer1_attention_weight_max": 16.807519912719727,
      "activations/layer1_attention_weight_min": -14.50085163116455,
      "activations/layer20_attention_weight_max": 31.47563934326172,
      "activations/layer20_attention_weight_min": -24.121896743774414,
      "activations/layer21_attention_weight_max": 30.142370223999023,
      "activations/layer21_attention_weight_min": -23.6101016998291,
      "activations/layer22_attention_weight_max": 46.703704833984375,
      "activations/layer22_attention_weight_min": -31.736536026000977,
      "activations/layer23_attention_weight_max": 36.221397399902344,
      "activations/layer23_attention_weight_min": -25.460514068603516,
      "activations/layer2_attention_weight_max": 32.524505615234375,
      "activations/layer2_attention_weight_min": -31.4636287689209,
      "activations/layer3_attention_weight_max": 92.35929870605469,
      "activations/layer3_attention_weight_min": -95.87836456298828,
      "activations/layer4_attention_weight_max": 117.3956527709961,
      "activations/layer4_attention_weight_min": -119.5052719116211,
      "activations/layer5_attention_weight_max": 55.624019622802734,
      "activations/layer5_attention_weight_min": -60.762264251708984,
      "activations/layer6_attention_weight_max": 50.81236267089844,
      "activations/layer6_attention_weight_min": -52.29533386230469,
      "activations/layer7_attention_weight_max": 94.83291625976562,
      "activations/layer7_attention_weight_min": -86.81954956054688,
      "activations/layer8_attention_weight_max": 45.02730178833008,
      "activations/layer8_attention_weight_min": -42.67606735229492,
      "activations/layer9_attention_weight_max": 35.94699478149414,
      "activations/layer9_attention_weight_min": -34.76267623901367,
      "epoch": 8.56,
      "learning_rate": 9.578825757575756e-05,
      "loss": 2.8168,
      "step": 147250
    },
    {
      "activations/layer0_attention_weight_max": 14.726792335510254,
      "activations/layer0_attention_weight_min": -12.736980438232422,
      "activations/layer10_attention_weight_max": 32.00153350830078,
      "activations/layer10_attention_weight_min": -31.235422134399414,
      "activations/layer11_attention_weight_max": 32.67649459838867,
      "activations/layer11_attention_weight_min": -28.76185417175293,
      "activations/layer12_attention_weight_max": 23.9202823638916,
      "activations/layer12_attention_weight_min": -27.709455490112305,
      "activations/layer13_attention_weight_max": 41.46186065673828,
      "activations/layer13_attention_weight_min": -32.62797546386719,
      "activations/layer14_attention_weight_max": 44.91099166870117,
      "activations/layer14_attention_weight_min": -37.861061096191406,
      "activations/layer15_attention_weight_max": 34.39719772338867,
      "activations/layer15_attention_weight_min": -28.306884765625,
      "activations/layer16_attention_weight_max": 30.35207176208496,
      "activations/layer16_attention_weight_min": -26.847124099731445,
      "activations/layer17_attention_weight_max": 33.83671569824219,
      "activations/layer17_attention_weight_min": -23.798139572143555,
      "activations/layer18_attention_weight_max": 30.262685775756836,
      "activations/layer18_attention_weight_min": -21.306594848632812,
      "activations/layer19_attention_weight_max": 36.88510513305664,
      "activations/layer19_attention_weight_min": -31.732603073120117,
      "activations/layer1_attention_weight_max": 16.154268264770508,
      "activations/layer1_attention_weight_min": -14.319753646850586,
      "activations/layer20_attention_weight_max": 31.68974494934082,
      "activations/layer20_attention_weight_min": -26.42479705810547,
      "activations/layer21_attention_weight_max": 30.215286254882812,
      "activations/layer21_attention_weight_min": -24.15666389465332,
      "activations/layer22_attention_weight_max": 45.09303283691406,
      "activations/layer22_attention_weight_min": -30.34688949584961,
      "activations/layer23_attention_weight_max": 36.06739044189453,
      "activations/layer23_attention_weight_min": -23.984329223632812,
      "activations/layer2_attention_weight_max": 29.90070152282715,
      "activations/layer2_attention_weight_min": -30.086524963378906,
      "activations/layer3_attention_weight_max": 85.24461364746094,
      "activations/layer3_attention_weight_min": -87.3583984375,
      "activations/layer4_attention_weight_max": 116.04679107666016,
      "activations/layer4_attention_weight_min": -111.4362564086914,
      "activations/layer5_attention_weight_max": 53.53691101074219,
      "activations/layer5_attention_weight_min": -60.823463439941406,
      "activations/layer6_attention_weight_max": 46.970787048339844,
      "activations/layer6_attention_weight_min": -48.90004348754883,
      "activations/layer7_attention_weight_max": 91.42898559570312,
      "activations/layer7_attention_weight_min": -84.61076354980469,
      "activations/layer8_attention_weight_max": 40.452144622802734,
      "activations/layer8_attention_weight_min": -38.66456985473633,
      "activations/layer9_attention_weight_max": 35.01894760131836,
      "activations/layer9_attention_weight_min": -34.32382583618164,
      "epoch": 8.56,
      "learning_rate": 9.576931818181817e-05,
      "loss": 2.817,
      "step": 147300
    },
    {
      "activations/layer0_attention_weight_max": 15.36728286743164,
      "activations/layer0_attention_weight_min": -12.624999046325684,
      "activations/layer10_attention_weight_max": 32.376136779785156,
      "activations/layer10_attention_weight_min": -30.31401252746582,
      "activations/layer11_attention_weight_max": 31.52901268005371,
      "activations/layer11_attention_weight_min": -30.75556182861328,
      "activations/layer12_attention_weight_max": 21.79918098449707,
      "activations/layer12_attention_weight_min": -27.046688079833984,
      "activations/layer13_attention_weight_max": 41.63578414916992,
      "activations/layer13_attention_weight_min": -33.063018798828125,
      "activations/layer14_attention_weight_max": 52.78759002685547,
      "activations/layer14_attention_weight_min": -37.380516052246094,
      "activations/layer15_attention_weight_max": 35.54058837890625,
      "activations/layer15_attention_weight_min": -31.019197463989258,
      "activations/layer16_attention_weight_max": 33.01174545288086,
      "activations/layer16_attention_weight_min": -26.670534133911133,
      "activations/layer17_attention_weight_max": 35.709346771240234,
      "activations/layer17_attention_weight_min": -24.43924903869629,
      "activations/layer18_attention_weight_max": 32.395626068115234,
      "activations/layer18_attention_weight_min": -22.221168518066406,
      "activations/layer19_attention_weight_max": 39.815303802490234,
      "activations/layer19_attention_weight_min": -29.821929931640625,
      "activations/layer1_attention_weight_max": 16.107446670532227,
      "activations/layer1_attention_weight_min": -15.546244621276855,
      "activations/layer20_attention_weight_max": 32.434661865234375,
      "activations/layer20_attention_weight_min": -24.12841033935547,
      "activations/layer21_attention_weight_max": 33.557254791259766,
      "activations/layer21_attention_weight_min": -23.099706649780273,
      "activations/layer22_attention_weight_max": 52.1542854309082,
      "activations/layer22_attention_weight_min": -27.55154037475586,
      "activations/layer23_attention_weight_max": 34.91986846923828,
      "activations/layer23_attention_weight_min": -25.945880889892578,
      "activations/layer2_attention_weight_max": 31.68738555908203,
      "activations/layer2_attention_weight_min": -30.87546157836914,
      "activations/layer3_attention_weight_max": 89.39447784423828,
      "activations/layer3_attention_weight_min": -88.97356414794922,
      "activations/layer4_attention_weight_max": 114.01500701904297,
      "activations/layer4_attention_weight_min": -116.81792449951172,
      "activations/layer5_attention_weight_max": 53.24515914916992,
      "activations/layer5_attention_weight_min": -61.19139862060547,
      "activations/layer6_attention_weight_max": 45.52204895019531,
      "activations/layer6_attention_weight_min": -50.33720397949219,
      "activations/layer7_attention_weight_max": 86.74066162109375,
      "activations/layer7_attention_weight_min": -83.21218872070312,
      "activations/layer8_attention_weight_max": 39.166805267333984,
      "activations/layer8_attention_weight_min": -39.3791389465332,
      "activations/layer9_attention_weight_max": 35.42978286743164,
      "activations/layer9_attention_weight_min": -33.77594757080078,
      "epoch": 8.56,
      "learning_rate": 9.575037878787879e-05,
      "loss": 2.7975,
      "step": 147350
    },
    {
      "activations/layer0_attention_weight_max": 15.971901893615723,
      "activations/layer0_attention_weight_min": -12.569483757019043,
      "activations/layer10_attention_weight_max": 30.33373260498047,
      "activations/layer10_attention_weight_min": -29.779714584350586,
      "activations/layer11_attention_weight_max": 30.513275146484375,
      "activations/layer11_attention_weight_min": -28.00909423828125,
      "activations/layer12_attention_weight_max": 22.681928634643555,
      "activations/layer12_attention_weight_min": -26.07417106628418,
      "activations/layer13_attention_weight_max": 36.87229537963867,
      "activations/layer13_attention_weight_min": -32.39727020263672,
      "activations/layer14_attention_weight_max": 42.225608825683594,
      "activations/layer14_attention_weight_min": -36.57918167114258,
      "activations/layer15_attention_weight_max": 31.881521224975586,
      "activations/layer15_attention_weight_min": -29.239301681518555,
      "activations/layer16_attention_weight_max": 29.464452743530273,
      "activations/layer16_attention_weight_min": -26.750757217407227,
      "activations/layer17_attention_weight_max": 30.297250747680664,
      "activations/layer17_attention_weight_min": -24.003353118896484,
      "activations/layer18_attention_weight_max": 31.501205444335938,
      "activations/layer18_attention_weight_min": -22.333768844604492,
      "activations/layer19_attention_weight_max": 31.493906021118164,
      "activations/layer19_attention_weight_min": -30.322784423828125,
      "activations/layer1_attention_weight_max": 16.747243881225586,
      "activations/layer1_attention_weight_min": -14.480037689208984,
      "activations/layer20_attention_weight_max": 28.35418128967285,
      "activations/layer20_attention_weight_min": -22.965221405029297,
      "activations/layer21_attention_weight_max": 26.70865821838379,
      "activations/layer21_attention_weight_min": -20.984424591064453,
      "activations/layer22_attention_weight_max": 42.97731018066406,
      "activations/layer22_attention_weight_min": -30.75233268737793,
      "activations/layer23_attention_weight_max": 33.45105743408203,
      "activations/layer23_attention_weight_min": -23.64192008972168,
      "activations/layer2_attention_weight_max": 30.351028442382812,
      "activations/layer2_attention_weight_min": -30.27273941040039,
      "activations/layer3_attention_weight_max": 87.1169204711914,
      "activations/layer3_attention_weight_min": -88.89608001708984,
      "activations/layer4_attention_weight_max": 113.98546600341797,
      "activations/layer4_attention_weight_min": -111.81636810302734,
      "activations/layer5_attention_weight_max": 53.74385070800781,
      "activations/layer5_attention_weight_min": -60.54169845581055,
      "activations/layer6_attention_weight_max": 47.245731353759766,
      "activations/layer6_attention_weight_min": -50.5799446105957,
      "activations/layer7_attention_weight_max": 86.42236328125,
      "activations/layer7_attention_weight_min": -84.35440826416016,
      "activations/layer8_attention_weight_max": 38.86028289794922,
      "activations/layer8_attention_weight_min": -39.81951904296875,
      "activations/layer9_attention_weight_max": 32.84457015991211,
      "activations/layer9_attention_weight_min": -32.511985778808594,
      "epoch": 8.56,
      "learning_rate": 9.573143939393937e-05,
      "loss": 2.8099,
      "step": 147400
    },
    {
      "activations/layer0_attention_weight_max": 15.892653465270996,
      "activations/layer0_attention_weight_min": -12.709394454956055,
      "activations/layer10_attention_weight_max": 33.72706604003906,
      "activations/layer10_attention_weight_min": -32.35518264770508,
      "activations/layer11_attention_weight_max": 31.9056453704834,
      "activations/layer11_attention_weight_min": -28.6380615234375,
      "activations/layer12_attention_weight_max": 24.658292770385742,
      "activations/layer12_attention_weight_min": -24.1719913482666,
      "activations/layer13_attention_weight_max": 36.0113525390625,
      "activations/layer13_attention_weight_min": -31.313541412353516,
      "activations/layer14_attention_weight_max": 43.90118408203125,
      "activations/layer14_attention_weight_min": -37.14834213256836,
      "activations/layer15_attention_weight_max": 32.975013732910156,
      "activations/layer15_attention_weight_min": -30.10489273071289,
      "activations/layer16_attention_weight_max": 31.819257736206055,
      "activations/layer16_attention_weight_min": -27.075098037719727,
      "activations/layer17_attention_weight_max": 31.140058517456055,
      "activations/layer17_attention_weight_min": -27.226104736328125,
      "activations/layer18_attention_weight_max": 32.30974578857422,
      "activations/layer18_attention_weight_min": -21.831501007080078,
      "activations/layer19_attention_weight_max": 31.91178321838379,
      "activations/layer19_attention_weight_min": -30.12631607055664,
      "activations/layer1_attention_weight_max": 17.225868225097656,
      "activations/layer1_attention_weight_min": -14.854995727539062,
      "activations/layer20_attention_weight_max": 27.626903533935547,
      "activations/layer20_attention_weight_min": -24.177698135375977,
      "activations/layer21_attention_weight_max": 29.670391082763672,
      "activations/layer21_attention_weight_min": -23.081010818481445,
      "activations/layer22_attention_weight_max": 42.38872528076172,
      "activations/layer22_attention_weight_min": -28.44049644470215,
      "activations/layer23_attention_weight_max": 33.16024398803711,
      "activations/layer23_attention_weight_min": -24.66029930114746,
      "activations/layer2_attention_weight_max": 29.229711532592773,
      "activations/layer2_attention_weight_min": -31.045181274414062,
      "activations/layer3_attention_weight_max": 86.72115325927734,
      "activations/layer3_attention_weight_min": -89.34932708740234,
      "activations/layer4_attention_weight_max": 117.65438079833984,
      "activations/layer4_attention_weight_min": -114.98095703125,
      "activations/layer5_attention_weight_max": 51.982479095458984,
      "activations/layer5_attention_weight_min": -64.03678894042969,
      "activations/layer6_attention_weight_max": 47.666385650634766,
      "activations/layer6_attention_weight_min": -50.612083435058594,
      "activations/layer7_attention_weight_max": 85.35124206542969,
      "activations/layer7_attention_weight_min": -83.94779205322266,
      "activations/layer8_attention_weight_max": 38.97122573852539,
      "activations/layer8_attention_weight_min": -38.60112762451172,
      "activations/layer9_attention_weight_max": 34.17659378051758,
      "activations/layer9_attention_weight_min": -32.67384338378906,
      "epoch": 8.57,
      "learning_rate": 9.571249999999999e-05,
      "loss": 2.8081,
      "step": 147450
    },
    {
      "activations/layer0_attention_weight_max": 15.921625137329102,
      "activations/layer0_attention_weight_min": -12.026777267456055,
      "activations/layer10_attention_weight_max": 34.373687744140625,
      "activations/layer10_attention_weight_min": -33.411190032958984,
      "activations/layer11_attention_weight_max": 33.748992919921875,
      "activations/layer11_attention_weight_min": -31.816448211669922,
      "activations/layer12_attention_weight_max": 37.26491928100586,
      "activations/layer12_attention_weight_min": -30.971208572387695,
      "activations/layer13_attention_weight_max": 44.17676544189453,
      "activations/layer13_attention_weight_min": -34.543617248535156,
      "activations/layer14_attention_weight_max": 55.67453384399414,
      "activations/layer14_attention_weight_min": -41.922645568847656,
      "activations/layer15_attention_weight_max": 37.52933883666992,
      "activations/layer15_attention_weight_min": -31.290822982788086,
      "activations/layer16_attention_weight_max": 34.197021484375,
      "activations/layer16_attention_weight_min": -27.382383346557617,
      "activations/layer17_attention_weight_max": 34.599910736083984,
      "activations/layer17_attention_weight_min": -26.1224308013916,
      "activations/layer18_attention_weight_max": 35.880584716796875,
      "activations/layer18_attention_weight_min": -23.354381561279297,
      "activations/layer19_attention_weight_max": 39.49314880371094,
      "activations/layer19_attention_weight_min": -33.334373474121094,
      "activations/layer1_attention_weight_max": 16.7526798248291,
      "activations/layer1_attention_weight_min": -13.472166061401367,
      "activations/layer20_attention_weight_max": 32.04306411743164,
      "activations/layer20_attention_weight_min": -24.8001651763916,
      "activations/layer21_attention_weight_max": 32.48451614379883,
      "activations/layer21_attention_weight_min": -24.432947158813477,
      "activations/layer22_attention_weight_max": 54.28561782836914,
      "activations/layer22_attention_weight_min": -29.796085357666016,
      "activations/layer23_attention_weight_max": 40.174346923828125,
      "activations/layer23_attention_weight_min": -24.2275447845459,
      "activations/layer2_attention_weight_max": 31.880809783935547,
      "activations/layer2_attention_weight_min": -32.89057159423828,
      "activations/layer3_attention_weight_max": 95.19263458251953,
      "activations/layer3_attention_weight_min": -96.69896697998047,
      "activations/layer4_attention_weight_max": 121.64293670654297,
      "activations/layer4_attention_weight_min": -116.49527740478516,
      "activations/layer5_attention_weight_max": 55.401424407958984,
      "activations/layer5_attention_weight_min": -60.753150939941406,
      "activations/layer6_attention_weight_max": 50.188377380371094,
      "activations/layer6_attention_weight_min": -51.460689544677734,
      "activations/layer7_attention_weight_max": 88.33753204345703,
      "activations/layer7_attention_weight_min": -87.37747955322266,
      "activations/layer8_attention_weight_max": 40.32406234741211,
      "activations/layer8_attention_weight_min": -40.73383331298828,
      "activations/layer9_attention_weight_max": 36.5090217590332,
      "activations/layer9_attention_weight_min": -33.128604888916016,
      "epoch": 8.57,
      "learning_rate": 9.56935606060606e-05,
      "loss": 2.798,
      "step": 147500
    },
    {
      "activations/layer0_attention_weight_max": 15.427119255065918,
      "activations/layer0_attention_weight_min": -12.864372253417969,
      "activations/layer10_attention_weight_max": 31.913936614990234,
      "activations/layer10_attention_weight_min": -32.133392333984375,
      "activations/layer11_attention_weight_max": 29.127166748046875,
      "activations/layer11_attention_weight_min": -28.626693725585938,
      "activations/layer12_attention_weight_max": 23.855085372924805,
      "activations/layer12_attention_weight_min": -23.837005615234375,
      "activations/layer13_attention_weight_max": 37.715293884277344,
      "activations/layer13_attention_weight_min": -31.501754760742188,
      "activations/layer14_attention_weight_max": 45.70196533203125,
      "activations/layer14_attention_weight_min": -40.309715270996094,
      "activations/layer15_attention_weight_max": 36.31550216674805,
      "activations/layer15_attention_weight_min": -28.726917266845703,
      "activations/layer16_attention_weight_max": 30.986865997314453,
      "activations/layer16_attention_weight_min": -27.552833557128906,
      "activations/layer17_attention_weight_max": 33.44657516479492,
      "activations/layer17_attention_weight_min": -24.546405792236328,
      "activations/layer18_attention_weight_max": 33.853973388671875,
      "activations/layer18_attention_weight_min": -21.924116134643555,
      "activations/layer19_attention_weight_max": 35.507381439208984,
      "activations/layer19_attention_weight_min": -29.124753952026367,
      "activations/layer1_attention_weight_max": 16.749536514282227,
      "activations/layer1_attention_weight_min": -14.700204849243164,
      "activations/layer20_attention_weight_max": 28.657941818237305,
      "activations/layer20_attention_weight_min": -24.384197235107422,
      "activations/layer21_attention_weight_max": 28.80824851989746,
      "activations/layer21_attention_weight_min": -24.029150009155273,
      "activations/layer22_attention_weight_max": 45.6197624206543,
      "activations/layer22_attention_weight_min": -29.32804298400879,
      "activations/layer23_attention_weight_max": 33.23441696166992,
      "activations/layer23_attention_weight_min": -25.3192195892334,
      "activations/layer2_attention_weight_max": 31.419174194335938,
      "activations/layer2_attention_weight_min": -31.538711547851562,
      "activations/layer3_attention_weight_max": 93.58172607421875,
      "activations/layer3_attention_weight_min": -91.46511840820312,
      "activations/layer4_attention_weight_max": 119.3447036743164,
      "activations/layer4_attention_weight_min": -116.02961730957031,
      "activations/layer5_attention_weight_max": 55.28376388549805,
      "activations/layer5_attention_weight_min": -60.8051872253418,
      "activations/layer6_attention_weight_max": 48.70676040649414,
      "activations/layer6_attention_weight_min": -50.86884307861328,
      "activations/layer7_attention_weight_max": 85.37922668457031,
      "activations/layer7_attention_weight_min": -86.36344146728516,
      "activations/layer8_attention_weight_max": 42.18008804321289,
      "activations/layer8_attention_weight_min": -38.03839111328125,
      "activations/layer9_attention_weight_max": 32.3648796081543,
      "activations/layer9_attention_weight_min": -34.15852355957031,
      "epoch": 8.57,
      "learning_rate": 9.56746212121212e-05,
      "loss": 2.79,
      "step": 147550
    },
    {
      "activations/layer0_attention_weight_max": 15.439961433410645,
      "activations/layer0_attention_weight_min": -11.842902183532715,
      "activations/layer10_attention_weight_max": 33.94499588012695,
      "activations/layer10_attention_weight_min": -32.73328399658203,
      "activations/layer11_attention_weight_max": 34.03665542602539,
      "activations/layer11_attention_weight_min": -29.95650291442871,
      "activations/layer12_attention_weight_max": 25.098764419555664,
      "activations/layer12_attention_weight_min": -24.69131851196289,
      "activations/layer13_attention_weight_max": 41.083160400390625,
      "activations/layer13_attention_weight_min": -32.0279426574707,
      "activations/layer14_attention_weight_max": 47.014015197753906,
      "activations/layer14_attention_weight_min": -38.42924880981445,
      "activations/layer15_attention_weight_max": 36.60548400878906,
      "activations/layer15_attention_weight_min": -29.38665771484375,
      "activations/layer16_attention_weight_max": 32.70972442626953,
      "activations/layer16_attention_weight_min": -27.136367797851562,
      "activations/layer17_attention_weight_max": 33.58555221557617,
      "activations/layer17_attention_weight_min": -24.57171058654785,
      "activations/layer18_attention_weight_max": 30.958969116210938,
      "activations/layer18_attention_weight_min": -21.829879760742188,
      "activations/layer19_attention_weight_max": 34.69469451904297,
      "activations/layer19_attention_weight_min": -29.746091842651367,
      "activations/layer1_attention_weight_max": 16.167369842529297,
      "activations/layer1_attention_weight_min": -14.255977630615234,
      "activations/layer20_attention_weight_max": 29.763933181762695,
      "activations/layer20_attention_weight_min": -25.260986328125,
      "activations/layer21_attention_weight_max": 29.087345123291016,
      "activations/layer21_attention_weight_min": -23.768091201782227,
      "activations/layer22_attention_weight_max": 46.58753967285156,
      "activations/layer22_attention_weight_min": -29.964874267578125,
      "activations/layer23_attention_weight_max": 34.978759765625,
      "activations/layer23_attention_weight_min": -24.325546264648438,
      "activations/layer2_attention_weight_max": 33.97827911376953,
      "activations/layer2_attention_weight_min": -32.889625549316406,
      "activations/layer3_attention_weight_max": 91.24189758300781,
      "activations/layer3_attention_weight_min": -96.24087524414062,
      "activations/layer4_attention_weight_max": 117.66475677490234,
      "activations/layer4_attention_weight_min": -122.51324462890625,
      "activations/layer5_attention_weight_max": 54.84098815917969,
      "activations/layer5_attention_weight_min": -63.647483825683594,
      "activations/layer6_attention_weight_max": 48.473018646240234,
      "activations/layer6_attention_weight_min": -51.9287223815918,
      "activations/layer7_attention_weight_max": 91.6238784790039,
      "activations/layer7_attention_weight_min": -91.12007904052734,
      "activations/layer8_attention_weight_max": 40.4051399230957,
      "activations/layer8_attention_weight_min": -42.61895751953125,
      "activations/layer9_attention_weight_max": 35.519371032714844,
      "activations/layer9_attention_weight_min": -33.82366180419922,
      "epoch": 8.58,
      "learning_rate": 9.56556818181818e-05,
      "loss": 2.8131,
      "step": 147600
    },
    {
      "activations/layer0_attention_weight_max": 16.421903610229492,
      "activations/layer0_attention_weight_min": -12.190820693969727,
      "activations/layer10_attention_weight_max": 34.917213439941406,
      "activations/layer10_attention_weight_min": -33.63095474243164,
      "activations/layer11_attention_weight_max": 33.772789001464844,
      "activations/layer11_attention_weight_min": -32.701141357421875,
      "activations/layer12_attention_weight_max": 22.98163604736328,
      "activations/layer12_attention_weight_min": -24.329240798950195,
      "activations/layer13_attention_weight_max": 39.04391098022461,
      "activations/layer13_attention_weight_min": -34.0168571472168,
      "activations/layer14_attention_weight_max": 48.30185317993164,
      "activations/layer14_attention_weight_min": -41.084129333496094,
      "activations/layer15_attention_weight_max": 36.50693893432617,
      "activations/layer15_attention_weight_min": -33.41932678222656,
      "activations/layer16_attention_weight_max": 33.24895477294922,
      "activations/layer16_attention_weight_min": -30.237442016601562,
      "activations/layer17_attention_weight_max": 37.14799499511719,
      "activations/layer17_attention_weight_min": -25.559965133666992,
      "activations/layer18_attention_weight_max": 32.71229553222656,
      "activations/layer18_attention_weight_min": -21.57843780517578,
      "activations/layer19_attention_weight_max": 39.96106719970703,
      "activations/layer19_attention_weight_min": -32.18309783935547,
      "activations/layer1_attention_weight_max": 17.108320236206055,
      "activations/layer1_attention_weight_min": -14.743330955505371,
      "activations/layer20_attention_weight_max": 34.82969665527344,
      "activations/layer20_attention_weight_min": -25.519474029541016,
      "activations/layer21_attention_weight_max": 32.11616897583008,
      "activations/layer21_attention_weight_min": -23.506900787353516,
      "activations/layer22_attention_weight_max": 46.24927520751953,
      "activations/layer22_attention_weight_min": -30.783935546875,
      "activations/layer23_attention_weight_max": 37.85221862792969,
      "activations/layer23_attention_weight_min": -24.125913619995117,
      "activations/layer2_attention_weight_max": 32.81708526611328,
      "activations/layer2_attention_weight_min": -33.24208068847656,
      "activations/layer3_attention_weight_max": 96.5879898071289,
      "activations/layer3_attention_weight_min": -93.13770294189453,
      "activations/layer4_attention_weight_max": 123.10269927978516,
      "activations/layer4_attention_weight_min": -116.371337890625,
      "activations/layer5_attention_weight_max": 56.28266143798828,
      "activations/layer5_attention_weight_min": -61.67728805541992,
      "activations/layer6_attention_weight_max": 50.25403594970703,
      "activations/layer6_attention_weight_min": -51.7066764831543,
      "activations/layer7_attention_weight_max": 94.13905334472656,
      "activations/layer7_attention_weight_min": -92.65875244140625,
      "activations/layer8_attention_weight_max": 40.913631439208984,
      "activations/layer8_attention_weight_min": -43.01679611206055,
      "activations/layer9_attention_weight_max": 36.38374710083008,
      "activations/layer9_attention_weight_min": -35.32648849487305,
      "epoch": 8.58,
      "learning_rate": 9.563674242424242e-05,
      "loss": 2.7986,
      "step": 147650
    },
    {
      "activations/layer0_attention_weight_max": 15.620609283447266,
      "activations/layer0_attention_weight_min": -11.978238105773926,
      "activations/layer10_attention_weight_max": 32.41801834106445,
      "activations/layer10_attention_weight_min": -31.24746322631836,
      "activations/layer11_attention_weight_max": 32.128326416015625,
      "activations/layer11_attention_weight_min": -29.59333038330078,
      "activations/layer12_attention_weight_max": 22.52255630493164,
      "activations/layer12_attention_weight_min": -25.851598739624023,
      "activations/layer13_attention_weight_max": 38.278465270996094,
      "activations/layer13_attention_weight_min": -34.294681549072266,
      "activations/layer14_attention_weight_max": 47.39290237426758,
      "activations/layer14_attention_weight_min": -38.8469123840332,
      "activations/layer15_attention_weight_max": 36.61845397949219,
      "activations/layer15_attention_weight_min": -29.62082862854004,
      "activations/layer16_attention_weight_max": 34.582305908203125,
      "activations/layer16_attention_weight_min": -28.939655303955078,
      "activations/layer17_attention_weight_max": 32.09416961669922,
      "activations/layer17_attention_weight_min": -25.54709815979004,
      "activations/layer18_attention_weight_max": 32.087066650390625,
      "activations/layer18_attention_weight_min": -22.031967163085938,
      "activations/layer19_attention_weight_max": 43.542842864990234,
      "activations/layer19_attention_weight_min": -33.09870910644531,
      "activations/layer1_attention_weight_max": 18.065013885498047,
      "activations/layer1_attention_weight_min": -15.83195972442627,
      "activations/layer20_attention_weight_max": 31.03619956970215,
      "activations/layer20_attention_weight_min": -23.701171875,
      "activations/layer21_attention_weight_max": 31.75588607788086,
      "activations/layer21_attention_weight_min": -22.504226684570312,
      "activations/layer22_attention_weight_max": 53.67292022705078,
      "activations/layer22_attention_weight_min": -32.36310577392578,
      "activations/layer23_attention_weight_max": 40.276512145996094,
      "activations/layer23_attention_weight_min": -26.712942123413086,
      "activations/layer2_attention_weight_max": 32.08977508544922,
      "activations/layer2_attention_weight_min": -31.60323143005371,
      "activations/layer3_attention_weight_max": 95.30531311035156,
      "activations/layer3_attention_weight_min": -97.38835906982422,
      "activations/layer4_attention_weight_max": 119.9218978881836,
      "activations/layer4_attention_weight_min": -117.9028549194336,
      "activations/layer5_attention_weight_max": 54.21099090576172,
      "activations/layer5_attention_weight_min": -61.912757873535156,
      "activations/layer6_attention_weight_max": 48.74766540527344,
      "activations/layer6_attention_weight_min": -55.38577651977539,
      "activations/layer7_attention_weight_max": 93.33663177490234,
      "activations/layer7_attention_weight_min": -91.76732635498047,
      "activations/layer8_attention_weight_max": 39.76581573486328,
      "activations/layer8_attention_weight_min": -38.630550384521484,
      "activations/layer9_attention_weight_max": 35.39244842529297,
      "activations/layer9_attention_weight_min": -34.183040618896484,
      "epoch": 8.58,
      "learning_rate": 9.561780303030302e-05,
      "loss": 2.8096,
      "step": 147700
    },
    {
      "activations/layer0_attention_weight_max": 15.623204231262207,
      "activations/layer0_attention_weight_min": -11.833474159240723,
      "activations/layer10_attention_weight_max": 36.874481201171875,
      "activations/layer10_attention_weight_min": -33.24272537231445,
      "activations/layer11_attention_weight_max": 33.98326873779297,
      "activations/layer11_attention_weight_min": -32.261085510253906,
      "activations/layer12_attention_weight_max": 27.538148880004883,
      "activations/layer12_attention_weight_min": -25.112342834472656,
      "activations/layer13_attention_weight_max": 39.36336898803711,
      "activations/layer13_attention_weight_min": -35.7801399230957,
      "activations/layer14_attention_weight_max": 51.78190612792969,
      "activations/layer14_attention_weight_min": -45.48048400878906,
      "activations/layer15_attention_weight_max": 36.13225173950195,
      "activations/layer15_attention_weight_min": -32.16466522216797,
      "activations/layer16_attention_weight_max": 32.360870361328125,
      "activations/layer16_attention_weight_min": -28.014272689819336,
      "activations/layer17_attention_weight_max": 34.145267486572266,
      "activations/layer17_attention_weight_min": -25.765127182006836,
      "activations/layer18_attention_weight_max": 28.6871280670166,
      "activations/layer18_attention_weight_min": -22.188377380371094,
      "activations/layer19_attention_weight_max": 38.09561538696289,
      "activations/layer19_attention_weight_min": -30.020919799804688,
      "activations/layer1_attention_weight_max": 16.33914566040039,
      "activations/layer1_attention_weight_min": -14.593680381774902,
      "activations/layer20_attention_weight_max": 30.995975494384766,
      "activations/layer20_attention_weight_min": -25.471967697143555,
      "activations/layer21_attention_weight_max": 27.8253231048584,
      "activations/layer21_attention_weight_min": -22.759521484375,
      "activations/layer22_attention_weight_max": 45.51546096801758,
      "activations/layer22_attention_weight_min": -29.55498504638672,
      "activations/layer23_attention_weight_max": 35.44981002807617,
      "activations/layer23_attention_weight_min": -24.106712341308594,
      "activations/layer2_attention_weight_max": 33.669830322265625,
      "activations/layer2_attention_weight_min": -33.14466094970703,
      "activations/layer3_attention_weight_max": 100.15547180175781,
      "activations/layer3_attention_weight_min": -99.50897979736328,
      "activations/layer4_attention_weight_max": 130.57269287109375,
      "activations/layer4_attention_weight_min": -124.3055419921875,
      "activations/layer5_attention_weight_max": 60.014404296875,
      "activations/layer5_attention_weight_min": -65.20391082763672,
      "activations/layer6_attention_weight_max": 53.16875076293945,
      "activations/layer6_attention_weight_min": -55.971107482910156,
      "activations/layer7_attention_weight_max": 95.61175537109375,
      "activations/layer7_attention_weight_min": -93.49935913085938,
      "activations/layer8_attention_weight_max": 44.91425323486328,
      "activations/layer8_attention_weight_min": -45.32722091674805,
      "activations/layer9_attention_weight_max": 36.69538116455078,
      "activations/layer9_attention_weight_min": -36.350101470947266,
      "epoch": 8.59,
      "learning_rate": 9.559886363636364e-05,
      "loss": 2.7995,
      "step": 147750
    },
    {
      "activations/layer0_attention_weight_max": 15.271394729614258,
      "activations/layer0_attention_weight_min": -11.995346069335938,
      "activations/layer10_attention_weight_max": 33.783203125,
      "activations/layer10_attention_weight_min": -31.19553565979004,
      "activations/layer11_attention_weight_max": 33.27536392211914,
      "activations/layer11_attention_weight_min": -31.61269187927246,
      "activations/layer12_attention_weight_max": 25.387794494628906,
      "activations/layer12_attention_weight_min": -27.04304313659668,
      "activations/layer13_attention_weight_max": 43.713863372802734,
      "activations/layer13_attention_weight_min": -34.16566467285156,
      "activations/layer14_attention_weight_max": 58.91720962524414,
      "activations/layer14_attention_weight_min": -41.73622512817383,
      "activations/layer15_attention_weight_max": 41.597110748291016,
      "activations/layer15_attention_weight_min": -29.957752227783203,
      "activations/layer16_attention_weight_max": 42.062164306640625,
      "activations/layer16_attention_weight_min": -27.902860641479492,
      "activations/layer17_attention_weight_max": 38.48161315917969,
      "activations/layer17_attention_weight_min": -26.18121910095215,
      "activations/layer18_attention_weight_max": 36.93783187866211,
      "activations/layer18_attention_weight_min": -21.78190040588379,
      "activations/layer19_attention_weight_max": 46.3742561340332,
      "activations/layer19_attention_weight_min": -30.809240341186523,
      "activations/layer1_attention_weight_max": 16.254396438598633,
      "activations/layer1_attention_weight_min": -14.0477876663208,
      "activations/layer20_attention_weight_max": 37.61267852783203,
      "activations/layer20_attention_weight_min": -24.901073455810547,
      "activations/layer21_attention_weight_max": 38.79948806762695,
      "activations/layer21_attention_weight_min": -22.95738410949707,
      "activations/layer22_attention_weight_max": 61.82362365722656,
      "activations/layer22_attention_weight_min": -29.404293060302734,
      "activations/layer23_attention_weight_max": 36.627349853515625,
      "activations/layer23_attention_weight_min": -23.8330078125,
      "activations/layer2_attention_weight_max": 32.46131134033203,
      "activations/layer2_attention_weight_min": -30.913291931152344,
      "activations/layer3_attention_weight_max": 89.8370132446289,
      "activations/layer3_attention_weight_min": -91.00060272216797,
      "activations/layer4_attention_weight_max": 114.13750457763672,
      "activations/layer4_attention_weight_min": -114.6513900756836,
      "activations/layer5_attention_weight_max": 52.037567138671875,
      "activations/layer5_attention_weight_min": -62.213478088378906,
      "activations/layer6_attention_weight_max": 46.80819320678711,
      "activations/layer6_attention_weight_min": -49.85109329223633,
      "activations/layer7_attention_weight_max": 92.77207946777344,
      "activations/layer7_attention_weight_min": -88.5424575805664,
      "activations/layer8_attention_weight_max": 39.420894622802734,
      "activations/layer8_attention_weight_min": -39.299869537353516,
      "activations/layer9_attention_weight_max": 36.49353790283203,
      "activations/layer9_attention_weight_min": -33.4375114440918,
      "epoch": 8.59,
      "learning_rate": 9.557992424242423e-05,
      "loss": 2.8006,
      "step": 147800
    },
    {
      "activations/layer0_attention_weight_max": 15.352499961853027,
      "activations/layer0_attention_weight_min": -11.784266471862793,
      "activations/layer10_attention_weight_max": 37.86676788330078,
      "activations/layer10_attention_weight_min": -36.231849670410156,
      "activations/layer11_attention_weight_max": 39.56688690185547,
      "activations/layer11_attention_weight_min": -37.51291275024414,
      "activations/layer12_attention_weight_max": 27.22024154663086,
      "activations/layer12_attention_weight_min": -26.567014694213867,
      "activations/layer13_attention_weight_max": 43.3535270690918,
      "activations/layer13_attention_weight_min": -31.90312385559082,
      "activations/layer14_attention_weight_max": 44.067420959472656,
      "activations/layer14_attention_weight_min": -36.147850036621094,
      "activations/layer15_attention_weight_max": 40.078243255615234,
      "activations/layer15_attention_weight_min": -28.620405197143555,
      "activations/layer16_attention_weight_max": 31.64937400817871,
      "activations/layer16_attention_weight_min": -26.64341926574707,
      "activations/layer17_attention_weight_max": 32.8642578125,
      "activations/layer17_attention_weight_min": -24.759883880615234,
      "activations/layer18_attention_weight_max": 31.914690017700195,
      "activations/layer18_attention_weight_min": -21.370296478271484,
      "activations/layer19_attention_weight_max": 34.85740280151367,
      "activations/layer19_attention_weight_min": -30.6777400970459,
      "activations/layer1_attention_weight_max": 17.091285705566406,
      "activations/layer1_attention_weight_min": -14.792540550231934,
      "activations/layer20_attention_weight_max": 27.477121353149414,
      "activations/layer20_attention_weight_min": -24.9392147064209,
      "activations/layer21_attention_weight_max": 26.121929168701172,
      "activations/layer21_attention_weight_min": -22.636449813842773,
      "activations/layer22_attention_weight_max": 45.77729034423828,
      "activations/layer22_attention_weight_min": -30.745168685913086,
      "activations/layer23_attention_weight_max": 36.92652893066406,
      "activations/layer23_attention_weight_min": -26.443214416503906,
      "activations/layer2_attention_weight_max": 34.54588317871094,
      "activations/layer2_attention_weight_min": -34.294612884521484,
      "activations/layer3_attention_weight_max": 91.87318420410156,
      "activations/layer3_attention_weight_min": -94.872314453125,
      "activations/layer4_attention_weight_max": 117.75719451904297,
      "activations/layer4_attention_weight_min": -116.90648651123047,
      "activations/layer5_attention_weight_max": 56.544593811035156,
      "activations/layer5_attention_weight_min": -60.98683166503906,
      "activations/layer6_attention_weight_max": 48.43910217285156,
      "activations/layer6_attention_weight_min": -52.31606674194336,
      "activations/layer7_attention_weight_max": 99.90287780761719,
      "activations/layer7_attention_weight_min": -96.18382263183594,
      "activations/layer8_attention_weight_max": 45.53351974487305,
      "activations/layer8_attention_weight_min": -41.59318161010742,
      "activations/layer9_attention_weight_max": 38.16438674926758,
      "activations/layer9_attention_weight_min": -37.522823333740234,
      "epoch": 8.59,
      "learning_rate": 9.556098484848484e-05,
      "loss": 2.8084,
      "step": 147850
    },
    {
      "activations/layer0_attention_weight_max": 17.00031852722168,
      "activations/layer0_attention_weight_min": -11.066146850585938,
      "activations/layer10_attention_weight_max": 31.403820037841797,
      "activations/layer10_attention_weight_min": -31.582847595214844,
      "activations/layer11_attention_weight_max": 30.854263305664062,
      "activations/layer11_attention_weight_min": -30.150314331054688,
      "activations/layer12_attention_weight_max": 23.231645584106445,
      "activations/layer12_attention_weight_min": -28.866117477416992,
      "activations/layer13_attention_weight_max": 37.83149719238281,
      "activations/layer13_attention_weight_min": -31.795564651489258,
      "activations/layer14_attention_weight_max": 47.123477935791016,
      "activations/layer14_attention_weight_min": -37.81465148925781,
      "activations/layer15_attention_weight_max": 36.77444076538086,
      "activations/layer15_attention_weight_min": -31.536218643188477,
      "activations/layer16_attention_weight_max": 29.08517837524414,
      "activations/layer16_attention_weight_min": -26.604795455932617,
      "activations/layer17_attention_weight_max": 31.337305068969727,
      "activations/layer17_attention_weight_min": -23.807464599609375,
      "activations/layer18_attention_weight_max": 27.741779327392578,
      "activations/layer18_attention_weight_min": -21.19329261779785,
      "activations/layer19_attention_weight_max": 31.596532821655273,
      "activations/layer19_attention_weight_min": -29.297515869140625,
      "activations/layer1_attention_weight_max": 16.981374740600586,
      "activations/layer1_attention_weight_min": -15.21113109588623,
      "activations/layer20_attention_weight_max": 29.431108474731445,
      "activations/layer20_attention_weight_min": -23.991308212280273,
      "activations/layer21_attention_weight_max": 29.41705322265625,
      "activations/layer21_attention_weight_min": -21.632001876831055,
      "activations/layer22_attention_weight_max": 47.10742950439453,
      "activations/layer22_attention_weight_min": -27.56071662902832,
      "activations/layer23_attention_weight_max": 32.05997085571289,
      "activations/layer23_attention_weight_min": -22.878910064697266,
      "activations/layer2_attention_weight_max": 32.92768859863281,
      "activations/layer2_attention_weight_min": -31.169017791748047,
      "activations/layer3_attention_weight_max": 94.15702819824219,
      "activations/layer3_attention_weight_min": -92.80565643310547,
      "activations/layer4_attention_weight_max": 118.6880874633789,
      "activations/layer4_attention_weight_min": -116.91922760009766,
      "activations/layer5_attention_weight_max": 56.5177116394043,
      "activations/layer5_attention_weight_min": -61.376277923583984,
      "activations/layer6_attention_weight_max": 47.31273651123047,
      "activations/layer6_attention_weight_min": -55.651092529296875,
      "activations/layer7_attention_weight_max": 90.41962432861328,
      "activations/layer7_attention_weight_min": -91.87763214111328,
      "activations/layer8_attention_weight_max": 39.57613754272461,
      "activations/layer8_attention_weight_min": -41.15486526489258,
      "activations/layer9_attention_weight_max": 32.50617599487305,
      "activations/layer9_attention_weight_min": -33.840450286865234,
      "epoch": 8.59,
      "learning_rate": 9.554204545454546e-05,
      "loss": 2.8035,
      "step": 147900
    },
    {
      "activations/layer0_attention_weight_max": 16.411945343017578,
      "activations/layer0_attention_weight_min": -12.5516357421875,
      "activations/layer10_attention_weight_max": 32.29060363769531,
      "activations/layer10_attention_weight_min": -31.072790145874023,
      "activations/layer11_attention_weight_max": 29.88277244567871,
      "activations/layer11_attention_weight_min": -29.2757625579834,
      "activations/layer12_attention_weight_max": 24.33322525024414,
      "activations/layer12_attention_weight_min": -25.310190200805664,
      "activations/layer13_attention_weight_max": 41.691925048828125,
      "activations/layer13_attention_weight_min": -34.17007064819336,
      "activations/layer14_attention_weight_max": 52.99277114868164,
      "activations/layer14_attention_weight_min": -39.912837982177734,
      "activations/layer15_attention_weight_max": 34.54859161376953,
      "activations/layer15_attention_weight_min": -31.21622657775879,
      "activations/layer16_attention_weight_max": 30.884204864501953,
      "activations/layer16_attention_weight_min": -28.613000869750977,
      "activations/layer17_attention_weight_max": 34.7244758605957,
      "activations/layer17_attention_weight_min": -25.421356201171875,
      "activations/layer18_attention_weight_max": 40.8550910949707,
      "activations/layer18_attention_weight_min": -23.566547393798828,
      "activations/layer19_attention_weight_max": 36.07584762573242,
      "activations/layer19_attention_weight_min": -30.128446578979492,
      "activations/layer1_attention_weight_max": 16.840593338012695,
      "activations/layer1_attention_weight_min": -14.897652626037598,
      "activations/layer20_attention_weight_max": 30.32433319091797,
      "activations/layer20_attention_weight_min": -24.807716369628906,
      "activations/layer21_attention_weight_max": 36.67353439331055,
      "activations/layer21_attention_weight_min": -24.281009674072266,
      "activations/layer22_attention_weight_max": 48.03691864013672,
      "activations/layer22_attention_weight_min": -30.872331619262695,
      "activations/layer23_attention_weight_max": 39.462669372558594,
      "activations/layer23_attention_weight_min": -23.239913940429688,
      "activations/layer2_attention_weight_max": 32.47963333129883,
      "activations/layer2_attention_weight_min": -32.92681884765625,
      "activations/layer3_attention_weight_max": 96.0628662109375,
      "activations/layer3_attention_weight_min": -95.17790222167969,
      "activations/layer4_attention_weight_max": 117.27632141113281,
      "activations/layer4_attention_weight_min": -115.09232330322266,
      "activations/layer5_attention_weight_max": 53.3275146484375,
      "activations/layer5_attention_weight_min": -63.699337005615234,
      "activations/layer6_attention_weight_max": 45.36451721191406,
      "activations/layer6_attention_weight_min": -51.03303909301758,
      "activations/layer7_attention_weight_max": 89.07283782958984,
      "activations/layer7_attention_weight_min": -93.87754821777344,
      "activations/layer8_attention_weight_max": 38.70486831665039,
      "activations/layer8_attention_weight_min": -39.169254302978516,
      "activations/layer9_attention_weight_max": 34.177040100097656,
      "activations/layer9_attention_weight_min": -32.70597457885742,
      "epoch": 8.6,
      "learning_rate": 9.552310606060604e-05,
      "loss": 2.7944,
      "step": 147950
    },
    {
      "activations/layer0_attention_weight_max": 15.774995803833008,
      "activations/layer0_attention_weight_min": -12.345949172973633,
      "activations/layer10_attention_weight_max": 32.8664436340332,
      "activations/layer10_attention_weight_min": -32.703407287597656,
      "activations/layer11_attention_weight_max": 32.40730285644531,
      "activations/layer11_attention_weight_min": -29.812728881835938,
      "activations/layer12_attention_weight_max": 23.150102615356445,
      "activations/layer12_attention_weight_min": -27.11326789855957,
      "activations/layer13_attention_weight_max": 38.8853645324707,
      "activations/layer13_attention_weight_min": -31.720561981201172,
      "activations/layer14_attention_weight_max": 48.7556037902832,
      "activations/layer14_attention_weight_min": -39.12788391113281,
      "activations/layer15_attention_weight_max": 37.638221740722656,
      "activations/layer15_attention_weight_min": -29.90894889831543,
      "activations/layer16_attention_weight_max": 31.359912872314453,
      "activations/layer16_attention_weight_min": -25.571033477783203,
      "activations/layer17_attention_weight_max": 32.682987213134766,
      "activations/layer17_attention_weight_min": -25.088186264038086,
      "activations/layer18_attention_weight_max": 31.054704666137695,
      "activations/layer18_attention_weight_min": -21.101215362548828,
      "activations/layer19_attention_weight_max": 38.19042205810547,
      "activations/layer19_attention_weight_min": -28.557193756103516,
      "activations/layer1_attention_weight_max": 17.019390106201172,
      "activations/layer1_attention_weight_min": -14.539324760437012,
      "activations/layer20_attention_weight_max": 28.481855392456055,
      "activations/layer20_attention_weight_min": -23.359394073486328,
      "activations/layer21_attention_weight_max": 28.129125595092773,
      "activations/layer21_attention_weight_min": -22.781179428100586,
      "activations/layer22_attention_weight_max": 44.4835205078125,
      "activations/layer22_attention_weight_min": -27.403059005737305,
      "activations/layer23_attention_weight_max": 37.357017517089844,
      "activations/layer23_attention_weight_min": -24.548213958740234,
      "activations/layer2_attention_weight_max": 31.776456832885742,
      "activations/layer2_attention_weight_min": -30.875938415527344,
      "activations/layer3_attention_weight_max": 92.99427032470703,
      "activations/layer3_attention_weight_min": -92.0238265991211,
      "activations/layer4_attention_weight_max": 122.58576965332031,
      "activations/layer4_attention_weight_min": -117.3393325805664,
      "activations/layer5_attention_weight_max": 56.68074035644531,
      "activations/layer5_attention_weight_min": -62.48698425292969,
      "activations/layer6_attention_weight_max": 49.51975631713867,
      "activations/layer6_attention_weight_min": -50.97486877441406,
      "activations/layer7_attention_weight_max": 95.24905395507812,
      "activations/layer7_attention_weight_min": -89.66456604003906,
      "activations/layer8_attention_weight_max": 40.895538330078125,
      "activations/layer8_attention_weight_min": -41.177032470703125,
      "activations/layer9_attention_weight_max": 36.55094528198242,
      "activations/layer9_attention_weight_min": -34.1109733581543,
      "epoch": 8.6,
      "learning_rate": 9.550416666666666e-05,
      "loss": 2.8055,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.5388,
      "eval_samples_per_second": 502.883,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.5388,
      "eval_openwebtext_samples_per_second": 502.883,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 2.0197,
      "eval_wikitext_samples_per_second": 225.777,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_lambada_loss": 2.5859375,
      "eval_lambada_ppl": 13.275729253905588,
      "eval_lambada_runtime": 9.653,
      "eval_lambada_samples_per_second": 504.405,
      "step": 148000
    },
    {
      "activations/layer0_attention_weight_max": 15.563567161560059,
      "activations/layer0_attention_weight_min": -11.980531692504883,
      "activations/layer10_attention_weight_max": 32.84791564941406,
      "activations/layer10_attention_weight_min": -32.35608673095703,
      "activations/layer11_attention_weight_max": 35.54890441894531,
      "activations/layer11_attention_weight_min": -30.654829025268555,
      "activations/layer12_attention_weight_max": 38.649932861328125,
      "activations/layer12_attention_weight_min": -26.328201293945312,
      "activations/layer13_attention_weight_max": 41.09145736694336,
      "activations/layer13_attention_weight_min": -33.17280197143555,
      "activations/layer14_attention_weight_max": 58.827884674072266,
      "activations/layer14_attention_weight_min": -39.76548767089844,
      "activations/layer15_attention_weight_max": 38.124168395996094,
      "activations/layer15_attention_weight_min": -30.135173797607422,
      "activations/layer16_attention_weight_max": 35.704105377197266,
      "activations/layer16_attention_weight_min": -27.70033073425293,
      "activations/layer17_attention_weight_max": 41.87639617919922,
      "activations/layer17_attention_weight_min": -27.57853126525879,
      "activations/layer18_attention_weight_max": 33.355926513671875,
      "activations/layer18_attention_weight_min": -23.871212005615234,
      "activations/layer19_attention_weight_max": 37.87870788574219,
      "activations/layer19_attention_weight_min": -31.863988876342773,
      "activations/layer1_attention_weight_max": 16.161428451538086,
      "activations/layer1_attention_weight_min": -14.885214805603027,
      "activations/layer20_attention_weight_max": 30.87834358215332,
      "activations/layer20_attention_weight_min": -26.8458194732666,
      "activations/layer21_attention_weight_max": 27.3349666595459,
      "activations/layer21_attention_weight_min": -26.96401023864746,
      "activations/layer22_attention_weight_max": 47.424888610839844,
      "activations/layer22_attention_weight_min": -32.19390106201172,
      "activations/layer23_attention_weight_max": 39.13822555541992,
      "activations/layer23_attention_weight_min": -23.52739715576172,
      "activations/layer2_attention_weight_max": 32.560508728027344,
      "activations/layer2_attention_weight_min": -32.73202133178711,
      "activations/layer3_attention_weight_max": 98.28245544433594,
      "activations/layer3_attention_weight_min": -100.0299072265625,
      "activations/layer4_attention_weight_max": 121.64334869384766,
      "activations/layer4_attention_weight_min": -120.26092529296875,
      "activations/layer5_attention_weight_max": 56.41329574584961,
      "activations/layer5_attention_weight_min": -63.15715789794922,
      "activations/layer6_attention_weight_max": 49.961727142333984,
      "activations/layer6_attention_weight_min": -51.040225982666016,
      "activations/layer7_attention_weight_max": 90.09629821777344,
      "activations/layer7_attention_weight_min": -87.91136932373047,
      "activations/layer8_attention_weight_max": 38.35820770263672,
      "activations/layer8_attention_weight_min": -39.14338684082031,
      "activations/layer9_attention_weight_max": 33.57160568237305,
      "activations/layer9_attention_weight_min": -34.947784423828125,
      "epoch": 8.6,
      "learning_rate": 9.548522727272727e-05,
      "loss": 2.8122,
      "step": 148050
    },
    {
      "activations/layer0_attention_weight_max": 15.493453025817871,
      "activations/layer0_attention_weight_min": -11.31178092956543,
      "activations/layer10_attention_weight_max": 33.24516677856445,
      "activations/layer10_attention_weight_min": -33.366024017333984,
      "activations/layer11_attention_weight_max": 32.85425567626953,
      "activations/layer11_attention_weight_min": -31.008214950561523,
      "activations/layer12_attention_weight_max": 24.42132568359375,
      "activations/layer12_attention_weight_min": -25.04412269592285,
      "activations/layer13_attention_weight_max": 40.4221076965332,
      "activations/layer13_attention_weight_min": -34.63396453857422,
      "activations/layer14_attention_weight_max": 50.04835891723633,
      "activations/layer14_attention_weight_min": -43.035804748535156,
      "activations/layer15_attention_weight_max": 33.75713348388672,
      "activations/layer15_attention_weight_min": -30.584753036499023,
      "activations/layer16_attention_weight_max": 30.1021728515625,
      "activations/layer16_attention_weight_min": -27.196563720703125,
      "activations/layer17_attention_weight_max": 30.386444091796875,
      "activations/layer17_attention_weight_min": -25.518709182739258,
      "activations/layer18_attention_weight_max": 30.463481903076172,
      "activations/layer18_attention_weight_min": -22.318864822387695,
      "activations/layer19_attention_weight_max": 37.031593322753906,
      "activations/layer19_attention_weight_min": -29.024494171142578,
      "activations/layer1_attention_weight_max": 16.455257415771484,
      "activations/layer1_attention_weight_min": -14.379568099975586,
      "activations/layer20_attention_weight_max": 29.473133087158203,
      "activations/layer20_attention_weight_min": -25.711137771606445,
      "activations/layer21_attention_weight_max": 29.93583106994629,
      "activations/layer21_attention_weight_min": -23.002683639526367,
      "activations/layer22_attention_weight_max": 41.33567428588867,
      "activations/layer22_attention_weight_min": -29.363033294677734,
      "activations/layer23_attention_weight_max": 36.451751708984375,
      "activations/layer23_attention_weight_min": -24.839309692382812,
      "activations/layer2_attention_weight_max": 30.627256393432617,
      "activations/layer2_attention_weight_min": -32.16854476928711,
      "activations/layer3_attention_weight_max": 98.09193420410156,
      "activations/layer3_attention_weight_min": -98.66895294189453,
      "activations/layer4_attention_weight_max": 122.33458709716797,
      "activations/layer4_attention_weight_min": -119.67041015625,
      "activations/layer5_attention_weight_max": 56.36201095581055,
      "activations/layer5_attention_weight_min": -64.31591796875,
      "activations/layer6_attention_weight_max": 48.431888580322266,
      "activations/layer6_attention_weight_min": -51.38047409057617,
      "activations/layer7_attention_weight_max": 87.58767700195312,
      "activations/layer7_attention_weight_min": -89.9355239868164,
      "activations/layer8_attention_weight_max": 37.77642059326172,
      "activations/layer8_attention_weight_min": -40.6182746887207,
      "activations/layer9_attention_weight_max": 32.42001724243164,
      "activations/layer9_attention_weight_min": -32.6234245300293,
      "epoch": 8.61,
      "learning_rate": 9.546628787878786e-05,
      "loss": 2.8016,
      "step": 148100
    },
    {
      "activations/layer0_attention_weight_max": 15.784377098083496,
      "activations/layer0_attention_weight_min": -11.980828285217285,
      "activations/layer10_attention_weight_max": 30.45253562927246,
      "activations/layer10_attention_weight_min": -31.685497283935547,
      "activations/layer11_attention_weight_max": 32.380374908447266,
      "activations/layer11_attention_weight_min": -32.370304107666016,
      "activations/layer12_attention_weight_max": 33.13098907470703,
      "activations/layer12_attention_weight_min": -24.314237594604492,
      "activations/layer13_attention_weight_max": 35.76051712036133,
      "activations/layer13_attention_weight_min": -36.565834045410156,
      "activations/layer14_attention_weight_max": 59.96260452270508,
      "activations/layer14_attention_weight_min": -45.121177673339844,
      "activations/layer15_attention_weight_max": 38.99125671386719,
      "activations/layer15_attention_weight_min": -32.725669860839844,
      "activations/layer16_attention_weight_max": 35.24565887451172,
      "activations/layer16_attention_weight_min": -28.272783279418945,
      "activations/layer17_attention_weight_max": 41.41642761230469,
      "activations/layer17_attention_weight_min": -28.008272171020508,
      "activations/layer18_attention_weight_max": 32.9283332824707,
      "activations/layer18_attention_weight_min": -23.608463287353516,
      "activations/layer19_attention_weight_max": 36.617801666259766,
      "activations/layer19_attention_weight_min": -30.038131713867188,
      "activations/layer1_attention_weight_max": 16.68524932861328,
      "activations/layer1_attention_weight_min": -13.318392753601074,
      "activations/layer20_attention_weight_max": 34.708587646484375,
      "activations/layer20_attention_weight_min": -24.187257766723633,
      "activations/layer21_attention_weight_max": 31.464170455932617,
      "activations/layer21_attention_weight_min": -22.967782974243164,
      "activations/layer22_attention_weight_max": 46.284027099609375,
      "activations/layer22_attention_weight_min": -31.209259033203125,
      "activations/layer23_attention_weight_max": 34.227718353271484,
      "activations/layer23_attention_weight_min": -25.45917510986328,
      "activations/layer2_attention_weight_max": 33.19020080566406,
      "activations/layer2_attention_weight_min": -33.40376281738281,
      "activations/layer3_attention_weight_max": 95.7715072631836,
      "activations/layer3_attention_weight_min": -100.99779510498047,
      "activations/layer4_attention_weight_max": 120.84364318847656,
      "activations/layer4_attention_weight_min": -120.9210205078125,
      "activations/layer5_attention_weight_max": 58.294395446777344,
      "activations/layer5_attention_weight_min": -65.13992309570312,
      "activations/layer6_attention_weight_max": 48.70614242553711,
      "activations/layer6_attention_weight_min": -53.78028869628906,
      "activations/layer7_attention_weight_max": 93.78272247314453,
      "activations/layer7_attention_weight_min": -95.97052764892578,
      "activations/layer8_attention_weight_max": 38.6422004699707,
      "activations/layer8_attention_weight_min": -44.150081634521484,
      "activations/layer9_attention_weight_max": 32.59775924682617,
      "activations/layer9_attention_weight_min": -34.17506790161133,
      "epoch": 8.61,
      "learning_rate": 9.544772727272725e-05,
      "loss": 2.7974,
      "step": 148150
    },
    {
      "activations/layer0_attention_weight_max": 15.926628112792969,
      "activations/layer0_attention_weight_min": -12.41244888305664,
      "activations/layer10_attention_weight_max": 35.110958099365234,
      "activations/layer10_attention_weight_min": -34.60660171508789,
      "activations/layer11_attention_weight_max": 32.699615478515625,
      "activations/layer11_attention_weight_min": -31.976287841796875,
      "activations/layer12_attention_weight_max": 24.052865982055664,
      "activations/layer12_attention_weight_min": -25.902938842773438,
      "activations/layer13_attention_weight_max": 40.15104675292969,
      "activations/layer13_attention_weight_min": -36.7751579284668,
      "activations/layer14_attention_weight_max": 47.78425216674805,
      "activations/layer14_attention_weight_min": -41.38710021972656,
      "activations/layer15_attention_weight_max": 37.22126770019531,
      "activations/layer15_attention_weight_min": -29.931922912597656,
      "activations/layer16_attention_weight_max": 33.02729034423828,
      "activations/layer16_attention_weight_min": -28.202104568481445,
      "activations/layer17_attention_weight_max": 33.53372573852539,
      "activations/layer17_attention_weight_min": -26.56540870666504,
      "activations/layer18_attention_weight_max": 31.48982048034668,
      "activations/layer18_attention_weight_min": -22.29111099243164,
      "activations/layer19_attention_weight_max": 38.67915725708008,
      "activations/layer19_attention_weight_min": -34.46963882446289,
      "activations/layer1_attention_weight_max": 16.5711612701416,
      "activations/layer1_attention_weight_min": -14.374479293823242,
      "activations/layer20_attention_weight_max": 29.670318603515625,
      "activations/layer20_attention_weight_min": -25.63234519958496,
      "activations/layer21_attention_weight_max": 29.427165985107422,
      "activations/layer21_attention_weight_min": -23.7386531829834,
      "activations/layer22_attention_weight_max": 54.43730545043945,
      "activations/layer22_attention_weight_min": -29.82400131225586,
      "activations/layer23_attention_weight_max": 37.29536437988281,
      "activations/layer23_attention_weight_min": -26.25347900390625,
      "activations/layer2_attention_weight_max": 32.49504089355469,
      "activations/layer2_attention_weight_min": -32.91194152832031,
      "activations/layer3_attention_weight_max": 94.13160705566406,
      "activations/layer3_attention_weight_min": -96.7370834350586,
      "activations/layer4_attention_weight_max": 126.16302490234375,
      "activations/layer4_attention_weight_min": -120.00640869140625,
      "activations/layer5_attention_weight_max": 58.90652084350586,
      "activations/layer5_attention_weight_min": -63.22566223144531,
      "activations/layer6_attention_weight_max": 48.88597106933594,
      "activations/layer6_attention_weight_min": -51.83530044555664,
      "activations/layer7_attention_weight_max": 92.97897338867188,
      "activations/layer7_attention_weight_min": -97.94075012207031,
      "activations/layer8_attention_weight_max": 41.85345458984375,
      "activations/layer8_attention_weight_min": -43.36680603027344,
      "activations/layer9_attention_weight_max": 35.9500617980957,
      "activations/layer9_attention_weight_min": -35.56470489501953,
      "epoch": 8.61,
      "learning_rate": 9.542878787878787e-05,
      "loss": 2.8055,
      "step": 148200
    },
    {
      "activations/layer0_attention_weight_max": 16.01009750366211,
      "activations/layer0_attention_weight_min": -12.603615760803223,
      "activations/layer10_attention_weight_max": 32.08707809448242,
      "activations/layer10_attention_weight_min": -32.32231521606445,
      "activations/layer11_attention_weight_max": 33.140625,
      "activations/layer11_attention_weight_min": -29.890464782714844,
      "activations/layer12_attention_weight_max": 24.09716033935547,
      "activations/layer12_attention_weight_min": -23.783029556274414,
      "activations/layer13_attention_weight_max": 39.67723083496094,
      "activations/layer13_attention_weight_min": -34.804718017578125,
      "activations/layer14_attention_weight_max": 51.764442443847656,
      "activations/layer14_attention_weight_min": -38.51540756225586,
      "activations/layer15_attention_weight_max": 36.849422454833984,
      "activations/layer15_attention_weight_min": -33.6030387878418,
      "activations/layer16_attention_weight_max": 31.357803344726562,
      "activations/layer16_attention_weight_min": -27.292680740356445,
      "activations/layer17_attention_weight_max": 35.12167739868164,
      "activations/layer17_attention_weight_min": -25.50928497314453,
      "activations/layer18_attention_weight_max": 30.9993953704834,
      "activations/layer18_attention_weight_min": -20.999921798706055,
      "activations/layer19_attention_weight_max": 36.381839752197266,
      "activations/layer19_attention_weight_min": -31.427616119384766,
      "activations/layer1_attention_weight_max": 16.489986419677734,
      "activations/layer1_attention_weight_min": -13.050985336303711,
      "activations/layer20_attention_weight_max": 31.19769859313965,
      "activations/layer20_attention_weight_min": -25.224889755249023,
      "activations/layer21_attention_weight_max": 27.76278305053711,
      "activations/layer21_attention_weight_min": -23.709142684936523,
      "activations/layer22_attention_weight_max": 45.06768035888672,
      "activations/layer22_attention_weight_min": -30.849586486816406,
      "activations/layer23_attention_weight_max": 38.90274429321289,
      "activations/layer23_attention_weight_min": -26.61652946472168,
      "activations/layer2_attention_weight_max": 33.20156478881836,
      "activations/layer2_attention_weight_min": -30.562658309936523,
      "activations/layer3_attention_weight_max": 96.48589324951172,
      "activations/layer3_attention_weight_min": -94.96072387695312,
      "activations/layer4_attention_weight_max": 122.22820281982422,
      "activations/layer4_attention_weight_min": -119.89546203613281,
      "activations/layer5_attention_weight_max": 54.66017150878906,
      "activations/layer5_attention_weight_min": -62.53825378417969,
      "activations/layer6_attention_weight_max": 46.880001068115234,
      "activations/layer6_attention_weight_min": -50.96303176879883,
      "activations/layer7_attention_weight_max": 87.99470520019531,
      "activations/layer7_attention_weight_min": -90.05963134765625,
      "activations/layer8_attention_weight_max": 40.63554000854492,
      "activations/layer8_attention_weight_min": -41.110408782958984,
      "activations/layer9_attention_weight_max": 36.48312759399414,
      "activations/layer9_attention_weight_min": -34.17036437988281,
      "epoch": 8.61,
      "learning_rate": 9.540984848484848e-05,
      "loss": 2.8128,
      "step": 148250
    },
    {
      "activations/layer0_attention_weight_max": 15.848042488098145,
      "activations/layer0_attention_weight_min": -12.54751205444336,
      "activations/layer10_attention_weight_max": 46.68882751464844,
      "activations/layer10_attention_weight_min": -42.0267448425293,
      "activations/layer11_attention_weight_max": 46.858367919921875,
      "activations/layer11_attention_weight_min": -40.09410095214844,
      "activations/layer12_attention_weight_max": 33.19191360473633,
      "activations/layer12_attention_weight_min": -32.34141159057617,
      "activations/layer13_attention_weight_max": 49.20109176635742,
      "activations/layer13_attention_weight_min": -34.91380310058594,
      "activations/layer14_attention_weight_max": 64.55692291259766,
      "activations/layer14_attention_weight_min": -47.41598129272461,
      "activations/layer15_attention_weight_max": 42.37389373779297,
      "activations/layer15_attention_weight_min": -31.701257705688477,
      "activations/layer16_attention_weight_max": 34.45920181274414,
      "activations/layer16_attention_weight_min": -28.31789779663086,
      "activations/layer17_attention_weight_max": 38.84125900268555,
      "activations/layer17_attention_weight_min": -25.38118553161621,
      "activations/layer18_attention_weight_max": 30.48920249938965,
      "activations/layer18_attention_weight_min": -22.07380485534668,
      "activations/layer19_attention_weight_max": 36.541847229003906,
      "activations/layer19_attention_weight_min": -30.626819610595703,
      "activations/layer1_attention_weight_max": 16.856435775756836,
      "activations/layer1_attention_weight_min": -14.813859939575195,
      "activations/layer20_attention_weight_max": 29.86113739013672,
      "activations/layer20_attention_weight_min": -23.97957420349121,
      "activations/layer21_attention_weight_max": 27.620498657226562,
      "activations/layer21_attention_weight_min": -22.929418563842773,
      "activations/layer22_attention_weight_max": 49.42768096923828,
      "activations/layer22_attention_weight_min": -28.406620025634766,
      "activations/layer23_attention_weight_max": 33.22657012939453,
      "activations/layer23_attention_weight_min": -23.186630249023438,
      "activations/layer2_attention_weight_max": 35.342613220214844,
      "activations/layer2_attention_weight_min": -35.921417236328125,
      "activations/layer3_attention_weight_max": 104.13070678710938,
      "activations/layer3_attention_weight_min": -104.36902618408203,
      "activations/layer4_attention_weight_max": 128.29649353027344,
      "activations/layer4_attention_weight_min": -126.9410400390625,
      "activations/layer5_attention_weight_max": 58.19725036621094,
      "activations/layer5_attention_weight_min": -62.62771224975586,
      "activations/layer6_attention_weight_max": 55.651344299316406,
      "activations/layer6_attention_weight_min": -55.88148880004883,
      "activations/layer7_attention_weight_max": 113.52679443359375,
      "activations/layer7_attention_weight_min": -103.99541473388672,
      "activations/layer8_attention_weight_max": 52.82518768310547,
      "activations/layer8_attention_weight_min": -51.7199821472168,
      "activations/layer9_attention_weight_max": 49.15442657470703,
      "activations/layer9_attention_weight_min": -44.1282958984375,
      "epoch": 8.62,
      "learning_rate": 9.539090909090908e-05,
      "loss": 2.8164,
      "step": 148300
    },
    {
      "activations/layer0_attention_weight_max": 15.242288589477539,
      "activations/layer0_attention_weight_min": -12.684914588928223,
      "activations/layer10_attention_weight_max": 30.033065795898438,
      "activations/layer10_attention_weight_min": -35.65443801879883,
      "activations/layer11_attention_weight_max": 30.71730613708496,
      "activations/layer11_attention_weight_min": -29.211502075195312,
      "activations/layer12_attention_weight_max": 22.308029174804688,
      "activations/layer12_attention_weight_min": -27.704076766967773,
      "activations/layer13_attention_weight_max": 37.21192932128906,
      "activations/layer13_attention_weight_min": -31.767044067382812,
      "activations/layer14_attention_weight_max": 46.63493728637695,
      "activations/layer14_attention_weight_min": -42.134952545166016,
      "activations/layer15_attention_weight_max": 35.339351654052734,
      "activations/layer15_attention_weight_min": -28.95770835876465,
      "activations/layer16_attention_weight_max": 30.93916893005371,
      "activations/layer16_attention_weight_min": -26.544946670532227,
      "activations/layer17_attention_weight_max": 32.84677505493164,
      "activations/layer17_attention_weight_min": -25.399274826049805,
      "activations/layer18_attention_weight_max": 31.86310386657715,
      "activations/layer18_attention_weight_min": -22.725303649902344,
      "activations/layer19_attention_weight_max": 33.707035064697266,
      "activations/layer19_attention_weight_min": -30.217117309570312,
      "activations/layer1_attention_weight_max": 16.643104553222656,
      "activations/layer1_attention_weight_min": -14.148089408874512,
      "activations/layer20_attention_weight_max": 31.28997230529785,
      "activations/layer20_attention_weight_min": -24.317277908325195,
      "activations/layer21_attention_weight_max": 30.906217575073242,
      "activations/layer21_attention_weight_min": -23.571147918701172,
      "activations/layer22_attention_weight_max": 45.212371826171875,
      "activations/layer22_attention_weight_min": -31.35346031188965,
      "activations/layer23_attention_weight_max": 35.79997253417969,
      "activations/layer23_attention_weight_min": -25.49665069580078,
      "activations/layer2_attention_weight_max": 31.12511444091797,
      "activations/layer2_attention_weight_min": -30.51661491394043,
      "activations/layer3_attention_weight_max": 91.40863800048828,
      "activations/layer3_attention_weight_min": -92.53083038330078,
      "activations/layer4_attention_weight_max": 115.21646881103516,
      "activations/layer4_attention_weight_min": -115.14989471435547,
      "activations/layer5_attention_weight_max": 51.842735290527344,
      "activations/layer5_attention_weight_min": -62.838890075683594,
      "activations/layer6_attention_weight_max": 46.53114318847656,
      "activations/layer6_attention_weight_min": -48.0886116027832,
      "activations/layer7_attention_weight_max": 86.43952178955078,
      "activations/layer7_attention_weight_min": -83.83182525634766,
      "activations/layer8_attention_weight_max": 36.967342376708984,
      "activations/layer8_attention_weight_min": -35.99602127075195,
      "activations/layer9_attention_weight_max": 31.6169490814209,
      "activations/layer9_attention_weight_min": -32.12234115600586,
      "epoch": 8.62,
      "learning_rate": 9.537196969696969e-05,
      "loss": 2.7874,
      "step": 148350
    },
    {
      "activations/layer0_attention_weight_max": 16.085901260375977,
      "activations/layer0_attention_weight_min": -11.174385070800781,
      "activations/layer10_attention_weight_max": 35.13623046875,
      "activations/layer10_attention_weight_min": -32.24074172973633,
      "activations/layer11_attention_weight_max": 35.265132904052734,
      "activations/layer11_attention_weight_min": -30.247879028320312,
      "activations/layer12_attention_weight_max": 32.88542938232422,
      "activations/layer12_attention_weight_min": -27.514209747314453,
      "activations/layer13_attention_weight_max": 48.086830139160156,
      "activations/layer13_attention_weight_min": -33.07642364501953,
      "activations/layer14_attention_weight_max": 61.37250518798828,
      "activations/layer14_attention_weight_min": -39.734920501708984,
      "activations/layer15_attention_weight_max": 46.0591926574707,
      "activations/layer15_attention_weight_min": -33.531150817871094,
      "activations/layer16_attention_weight_max": 41.641578674316406,
      "activations/layer16_attention_weight_min": -31.334630966186523,
      "activations/layer17_attention_weight_max": 47.53551483154297,
      "activations/layer17_attention_weight_min": -28.61848258972168,
      "activations/layer18_attention_weight_max": 38.223121643066406,
      "activations/layer18_attention_weight_min": -25.827495574951172,
      "activations/layer19_attention_weight_max": 47.12174987792969,
      "activations/layer19_attention_weight_min": -34.588157653808594,
      "activations/layer1_attention_weight_max": 16.521472930908203,
      "activations/layer1_attention_weight_min": -14.909956932067871,
      "activations/layer20_attention_weight_max": 41.693809509277344,
      "activations/layer20_attention_weight_min": -28.078094482421875,
      "activations/layer21_attention_weight_max": 38.31338119506836,
      "activations/layer21_attention_weight_min": -25.61279296875,
      "activations/layer22_attention_weight_max": 57.56602096557617,
      "activations/layer22_attention_weight_min": -30.829010009765625,
      "activations/layer23_attention_weight_max": 39.44843292236328,
      "activations/layer23_attention_weight_min": -26.645689010620117,
      "activations/layer2_attention_weight_max": 32.97962951660156,
      "activations/layer2_attention_weight_min": -32.02179718017578,
      "activations/layer3_attention_weight_max": 95.32371520996094,
      "activations/layer3_attention_weight_min": -97.3102798461914,
      "activations/layer4_attention_weight_max": 117.93572998046875,
      "activations/layer4_attention_weight_min": -120.952392578125,
      "activations/layer5_attention_weight_max": 59.231502532958984,
      "activations/layer5_attention_weight_min": -63.209774017333984,
      "activations/layer6_attention_weight_max": 50.79102325439453,
      "activations/layer6_attention_weight_min": -53.24526596069336,
      "activations/layer7_attention_weight_max": 90.97261047363281,
      "activations/layer7_attention_weight_min": -94.94204711914062,
      "activations/layer8_attention_weight_max": 40.20139694213867,
      "activations/layer8_attention_weight_min": -42.833709716796875,
      "activations/layer9_attention_weight_max": 37.197242736816406,
      "activations/layer9_attention_weight_min": -34.389404296875,
      "epoch": 8.62,
      "learning_rate": 9.535303030303029e-05,
      "loss": 2.8251,
      "step": 148400
    },
    {
      "activations/layer0_attention_weight_max": 15.795870780944824,
      "activations/layer0_attention_weight_min": -12.024598121643066,
      "activations/layer10_attention_weight_max": 30.682180404663086,
      "activations/layer10_attention_weight_min": -31.87492561340332,
      "activations/layer11_attention_weight_max": 28.654029846191406,
      "activations/layer11_attention_weight_min": -28.42880630493164,
      "activations/layer12_attention_weight_max": 24.980972290039062,
      "activations/layer12_attention_weight_min": -29.28506851196289,
      "activations/layer13_attention_weight_max": 42.186431884765625,
      "activations/layer13_attention_weight_min": -33.59116744995117,
      "activations/layer14_attention_weight_max": 47.44900894165039,
      "activations/layer14_attention_weight_min": -35.79195022583008,
      "activations/layer15_attention_weight_max": 38.38178253173828,
      "activations/layer15_attention_weight_min": -30.1857852935791,
      "activations/layer16_attention_weight_max": 33.05679702758789,
      "activations/layer16_attention_weight_min": -27.44200897216797,
      "activations/layer17_attention_weight_max": 37.23213195800781,
      "activations/layer17_attention_weight_min": -24.895221710205078,
      "activations/layer18_attention_weight_max": 37.18141174316406,
      "activations/layer18_attention_weight_min": -22.381444931030273,
      "activations/layer19_attention_weight_max": 38.17388153076172,
      "activations/layer19_attention_weight_min": -30.379972457885742,
      "activations/layer1_attention_weight_max": 16.47217559814453,
      "activations/layer1_attention_weight_min": -13.297930717468262,
      "activations/layer20_attention_weight_max": 30.735185623168945,
      "activations/layer20_attention_weight_min": -24.030513763427734,
      "activations/layer21_attention_weight_max": 30.626522064208984,
      "activations/layer21_attention_weight_min": -23.46820640563965,
      "activations/layer22_attention_weight_max": 43.860992431640625,
      "activations/layer22_attention_weight_min": -30.144622802734375,
      "activations/layer23_attention_weight_max": 34.22309112548828,
      "activations/layer23_attention_weight_min": -25.712871551513672,
      "activations/layer2_attention_weight_max": 31.52638816833496,
      "activations/layer2_attention_weight_min": -33.244483947753906,
      "activations/layer3_attention_weight_max": 93.96977996826172,
      "activations/layer3_attention_weight_min": -98.11180877685547,
      "activations/layer4_attention_weight_max": 117.75911712646484,
      "activations/layer4_attention_weight_min": -121.95206451416016,
      "activations/layer5_attention_weight_max": 54.207969665527344,
      "activations/layer5_attention_weight_min": -62.37702941894531,
      "activations/layer6_attention_weight_max": 47.52967071533203,
      "activations/layer6_attention_weight_min": -52.70893478393555,
      "activations/layer7_attention_weight_max": 86.76670837402344,
      "activations/layer7_attention_weight_min": -85.48098754882812,
      "activations/layer8_attention_weight_max": 39.23432540893555,
      "activations/layer8_attention_weight_min": -40.99662780761719,
      "activations/layer9_attention_weight_max": 32.63053512573242,
      "activations/layer9_attention_weight_min": -32.9504508972168,
      "epoch": 8.63,
      "learning_rate": 9.53340909090909e-05,
      "loss": 2.8057,
      "step": 148450
    },
    {
      "activations/layer0_attention_weight_max": 15.2977933883667,
      "activations/layer0_attention_weight_min": -12.1781587600708,
      "activations/layer10_attention_weight_max": 32.74860763549805,
      "activations/layer10_attention_weight_min": -31.75717544555664,
      "activations/layer11_attention_weight_max": 34.324134826660156,
      "activations/layer11_attention_weight_min": -30.719966888427734,
      "activations/layer12_attention_weight_max": 26.481826782226562,
      "activations/layer12_attention_weight_min": -26.47257423400879,
      "activations/layer13_attention_weight_max": 42.030372619628906,
      "activations/layer13_attention_weight_min": -31.270944595336914,
      "activations/layer14_attention_weight_max": 52.368412017822266,
      "activations/layer14_attention_weight_min": -39.1577033996582,
      "activations/layer15_attention_weight_max": 42.16086959838867,
      "activations/layer15_attention_weight_min": -30.163793563842773,
      "activations/layer16_attention_weight_max": 32.15594482421875,
      "activations/layer16_attention_weight_min": -28.40031623840332,
      "activations/layer17_attention_weight_max": 31.584449768066406,
      "activations/layer17_attention_weight_min": -25.91883087158203,
      "activations/layer18_attention_weight_max": 32.69040298461914,
      "activations/layer18_attention_weight_min": -22.4993953704834,
      "activations/layer19_attention_weight_max": 42.80742263793945,
      "activations/layer19_attention_weight_min": -31.3503475189209,
      "activations/layer1_attention_weight_max": 15.81807804107666,
      "activations/layer1_attention_weight_min": -14.411141395568848,
      "activations/layer20_attention_weight_max": 30.880144119262695,
      "activations/layer20_attention_weight_min": -24.48424530029297,
      "activations/layer21_attention_weight_max": 29.322856903076172,
      "activations/layer21_attention_weight_min": -23.469131469726562,
      "activations/layer22_attention_weight_max": 43.96000289916992,
      "activations/layer22_attention_weight_min": -30.780683517456055,
      "activations/layer23_attention_weight_max": 38.1047477722168,
      "activations/layer23_attention_weight_min": -24.160327911376953,
      "activations/layer2_attention_weight_max": 33.03649139404297,
      "activations/layer2_attention_weight_min": -31.98508071899414,
      "activations/layer3_attention_weight_max": 97.7902603149414,
      "activations/layer3_attention_weight_min": -99.0101547241211,
      "activations/layer4_attention_weight_max": 125.61067962646484,
      "activations/layer4_attention_weight_min": -121.45479583740234,
      "activations/layer5_attention_weight_max": 55.66767501831055,
      "activations/layer5_attention_weight_min": -63.98679733276367,
      "activations/layer6_attention_weight_max": 49.85460662841797,
      "activations/layer6_attention_weight_min": -54.73448181152344,
      "activations/layer7_attention_weight_max": 91.92958068847656,
      "activations/layer7_attention_weight_min": -91.56935119628906,
      "activations/layer8_attention_weight_max": 39.27717590332031,
      "activations/layer8_attention_weight_min": -40.78624725341797,
      "activations/layer9_attention_weight_max": 36.19293212890625,
      "activations/layer9_attention_weight_min": -34.42311096191406,
      "epoch": 8.63,
      "learning_rate": 9.53155303030303e-05,
      "loss": 2.8055,
      "step": 148500
    },
    {
      "activations/layer0_attention_weight_max": 16.478975296020508,
      "activations/layer0_attention_weight_min": -12.982194900512695,
      "activations/layer10_attention_weight_max": 32.702064514160156,
      "activations/layer10_attention_weight_min": -32.63447952270508,
      "activations/layer11_attention_weight_max": 32.62388610839844,
      "activations/layer11_attention_weight_min": -29.555700302124023,
      "activations/layer12_attention_weight_max": 24.112939834594727,
      "activations/layer12_attention_weight_min": -26.77317237854004,
      "activations/layer13_attention_weight_max": 40.133705139160156,
      "activations/layer13_attention_weight_min": -31.27155303955078,
      "activations/layer14_attention_weight_max": 45.047122955322266,
      "activations/layer14_attention_weight_min": -35.02887725830078,
      "activations/layer15_attention_weight_max": 36.70890808105469,
      "activations/layer15_attention_weight_min": -29.756603240966797,
      "activations/layer16_attention_weight_max": 33.51710510253906,
      "activations/layer16_attention_weight_min": -26.35771942138672,
      "activations/layer17_attention_weight_max": 36.674076080322266,
      "activations/layer17_attention_weight_min": -23.6805362701416,
      "activations/layer18_attention_weight_max": 32.9033203125,
      "activations/layer18_attention_weight_min": -22.47076988220215,
      "activations/layer19_attention_weight_max": 35.34048843383789,
      "activations/layer19_attention_weight_min": -29.494094848632812,
      "activations/layer1_attention_weight_max": 16.5362491607666,
      "activations/layer1_attention_weight_min": -14.779781341552734,
      "activations/layer20_attention_weight_max": 27.904617309570312,
      "activations/layer20_attention_weight_min": -23.482376098632812,
      "activations/layer21_attention_weight_max": 29.249847412109375,
      "activations/layer21_attention_weight_min": -21.939697265625,
      "activations/layer22_attention_weight_max": 46.68731689453125,
      "activations/layer22_attention_weight_min": -26.236055374145508,
      "activations/layer23_attention_weight_max": 36.53826141357422,
      "activations/layer23_attention_weight_min": -22.90082550048828,
      "activations/layer2_attention_weight_max": 32.307533264160156,
      "activations/layer2_attention_weight_min": -31.397720336914062,
      "activations/layer3_attention_weight_max": 93.96737670898438,
      "activations/layer3_attention_weight_min": -93.96887969970703,
      "activations/layer4_attention_weight_max": 119.37264251708984,
      "activations/layer4_attention_weight_min": -114.08939361572266,
      "activations/layer5_attention_weight_max": 53.71343994140625,
      "activations/layer5_attention_weight_min": -61.34281921386719,
      "activations/layer6_attention_weight_max": 47.62693786621094,
      "activations/layer6_attention_weight_min": -51.054527282714844,
      "activations/layer7_attention_weight_max": 87.6221694946289,
      "activations/layer7_attention_weight_min": -90.81306457519531,
      "activations/layer8_attention_weight_max": 39.276878356933594,
      "activations/layer8_attention_weight_min": -39.00001907348633,
      "activations/layer9_attention_weight_max": 34.700164794921875,
      "activations/layer9_attention_weight_min": -34.08706283569336,
      "epoch": 8.63,
      "learning_rate": 9.529659090909091e-05,
      "loss": 2.7928,
      "step": 148550
    },
    {
      "activations/layer0_attention_weight_max": 15.63038158416748,
      "activations/layer0_attention_weight_min": -12.420697212219238,
      "activations/layer10_attention_weight_max": 35.53351593017578,
      "activations/layer10_attention_weight_min": -31.613155364990234,
      "activations/layer11_attention_weight_max": 32.702545166015625,
      "activations/layer11_attention_weight_min": -31.436582565307617,
      "activations/layer12_attention_weight_max": 21.77318000793457,
      "activations/layer12_attention_weight_min": -24.876859664916992,
      "activations/layer13_attention_weight_max": 38.4158935546875,
      "activations/layer13_attention_weight_min": -32.058895111083984,
      "activations/layer14_attention_weight_max": 46.17890167236328,
      "activations/layer14_attention_weight_min": -37.687522888183594,
      "activations/layer15_attention_weight_max": 35.3819580078125,
      "activations/layer15_attention_weight_min": -29.766443252563477,
      "activations/layer16_attention_weight_max": 31.555118560791016,
      "activations/layer16_attention_weight_min": -28.794506072998047,
      "activations/layer17_attention_weight_max": 33.94464111328125,
      "activations/layer17_attention_weight_min": -25.629297256469727,
      "activations/layer18_attention_weight_max": 31.32417106628418,
      "activations/layer18_attention_weight_min": -24.812715530395508,
      "activations/layer19_attention_weight_max": 35.43512725830078,
      "activations/layer19_attention_weight_min": -29.7362117767334,
      "activations/layer1_attention_weight_max": 16.415590286254883,
      "activations/layer1_attention_weight_min": -13.585944175720215,
      "activations/layer20_attention_weight_max": 31.411476135253906,
      "activations/layer20_attention_weight_min": -25.505582809448242,
      "activations/layer21_attention_weight_max": 30.170623779296875,
      "activations/layer21_attention_weight_min": -23.6439208984375,
      "activations/layer22_attention_weight_max": 48.2491569519043,
      "activations/layer22_attention_weight_min": -29.31614112854004,
      "activations/layer23_attention_weight_max": 34.24824905395508,
      "activations/layer23_attention_weight_min": -25.311010360717773,
      "activations/layer2_attention_weight_max": 34.029869079589844,
      "activations/layer2_attention_weight_min": -33.136905670166016,
      "activations/layer3_attention_weight_max": 97.72245788574219,
      "activations/layer3_attention_weight_min": -96.50444793701172,
      "activations/layer4_attention_weight_max": 117.59191131591797,
      "activations/layer4_attention_weight_min": -115.94219207763672,
      "activations/layer5_attention_weight_max": 57.277217864990234,
      "activations/layer5_attention_weight_min": -60.49684143066406,
      "activations/layer6_attention_weight_max": 48.35746765136719,
      "activations/layer6_attention_weight_min": -51.83993148803711,
      "activations/layer7_attention_weight_max": 89.62748718261719,
      "activations/layer7_attention_weight_min": -88.20816802978516,
      "activations/layer8_attention_weight_max": 39.69557571411133,
      "activations/layer8_attention_weight_min": -39.6839599609375,
      "activations/layer9_attention_weight_max": 34.885765075683594,
      "activations/layer9_attention_weight_min": -32.741939544677734,
      "epoch": 8.63,
      "learning_rate": 9.52776515151515e-05,
      "loss": 2.8017,
      "step": 148600
    },
    {
      "activations/layer0_attention_weight_max": 15.27762508392334,
      "activations/layer0_attention_weight_min": -12.096776008605957,
      "activations/layer10_attention_weight_max": 37.42947006225586,
      "activations/layer10_attention_weight_min": -32.3891487121582,
      "activations/layer11_attention_weight_max": 38.01657485961914,
      "activations/layer11_attention_weight_min": -31.00441551208496,
      "activations/layer12_attention_weight_max": 28.622907638549805,
      "activations/layer12_attention_weight_min": -27.340024948120117,
      "activations/layer13_attention_weight_max": 43.756656646728516,
      "activations/layer13_attention_weight_min": -33.8178825378418,
      "activations/layer14_attention_weight_max": 54.39475631713867,
      "activations/layer14_attention_weight_min": -41.7868766784668,
      "activations/layer15_attention_weight_max": 43.63772964477539,
      "activations/layer15_attention_weight_min": -32.85005569458008,
      "activations/layer16_attention_weight_max": 34.32796859741211,
      "activations/layer16_attention_weight_min": -27.857650756835938,
      "activations/layer17_attention_weight_max": 32.0460319519043,
      "activations/layer17_attention_weight_min": -25.30086898803711,
      "activations/layer18_attention_weight_max": 33.82182693481445,
      "activations/layer18_attention_weight_min": -23.99835968017578,
      "activations/layer19_attention_weight_max": 34.01512908935547,
      "activations/layer19_attention_weight_min": -31.354299545288086,
      "activations/layer1_attention_weight_max": 18.1237735748291,
      "activations/layer1_attention_weight_min": -14.293917655944824,
      "activations/layer20_attention_weight_max": 29.656753540039062,
      "activations/layer20_attention_weight_min": -26.097379684448242,
      "activations/layer21_attention_weight_max": 30.854387283325195,
      "activations/layer21_attention_weight_min": -24.25227928161621,
      "activations/layer22_attention_weight_max": 41.319950103759766,
      "activations/layer22_attention_weight_min": -28.743629455566406,
      "activations/layer23_attention_weight_max": 33.323638916015625,
      "activations/layer23_attention_weight_min": -22.68600845336914,
      "activations/layer2_attention_weight_max": 34.05548858642578,
      "activations/layer2_attention_weight_min": -33.99652099609375,
      "activations/layer3_attention_weight_max": 95.1033935546875,
      "activations/layer3_attention_weight_min": -97.63346099853516,
      "activations/layer4_attention_weight_max": 124.8350601196289,
      "activations/layer4_attention_weight_min": -117.03447723388672,
      "activations/layer5_attention_weight_max": 54.97098922729492,
      "activations/layer5_attention_weight_min": -61.7942008972168,
      "activations/layer6_attention_weight_max": 49.42168045043945,
      "activations/layer6_attention_weight_min": -50.87721252441406,
      "activations/layer7_attention_weight_max": 96.03872680664062,
      "activations/layer7_attention_weight_min": -89.01055908203125,
      "activations/layer8_attention_weight_max": 41.92134475708008,
      "activations/layer8_attention_weight_min": -38.58241653442383,
      "activations/layer9_attention_weight_max": 37.08716583251953,
      "activations/layer9_attention_weight_min": -32.534053802490234,
      "epoch": 8.64,
      "learning_rate": 9.525871212121211e-05,
      "loss": 2.8062,
      "step": 148650
    },
    {
      "activations/layer0_attention_weight_max": 15.068167686462402,
      "activations/layer0_attention_weight_min": -12.291994094848633,
      "activations/layer10_attention_weight_max": 29.46633529663086,
      "activations/layer10_attention_weight_min": -29.972105026245117,
      "activations/layer11_attention_weight_max": 30.42372703552246,
      "activations/layer11_attention_weight_min": -28.888731002807617,
      "activations/layer12_attention_weight_max": 21.138839721679688,
      "activations/layer12_attention_weight_min": -24.244524002075195,
      "activations/layer13_attention_weight_max": 35.12788009643555,
      "activations/layer13_attention_weight_min": -32.33709716796875,
      "activations/layer14_attention_weight_max": 41.052093505859375,
      "activations/layer14_attention_weight_min": -36.202552795410156,
      "activations/layer15_attention_weight_max": 33.18687057495117,
      "activations/layer15_attention_weight_min": -28.425655364990234,
      "activations/layer16_attention_weight_max": 32.05081558227539,
      "activations/layer16_attention_weight_min": -27.188554763793945,
      "activations/layer17_attention_weight_max": 32.66289138793945,
      "activations/layer17_attention_weight_min": -24.942323684692383,
      "activations/layer18_attention_weight_max": 27.375289916992188,
      "activations/layer18_attention_weight_min": -21.95768928527832,
      "activations/layer19_attention_weight_max": 33.68802261352539,
      "activations/layer19_attention_weight_min": -30.26263999938965,
      "activations/layer1_attention_weight_max": 16.50507926940918,
      "activations/layer1_attention_weight_min": -13.510581970214844,
      "activations/layer20_attention_weight_max": 29.39923095703125,
      "activations/layer20_attention_weight_min": -23.603496551513672,
      "activations/layer21_attention_weight_max": 27.05986213684082,
      "activations/layer21_attention_weight_min": -23.90562629699707,
      "activations/layer22_attention_weight_max": 43.074058532714844,
      "activations/layer22_attention_weight_min": -29.845075607299805,
      "activations/layer23_attention_weight_max": 33.42766189575195,
      "activations/layer23_attention_weight_min": -23.10356903076172,
      "activations/layer2_attention_weight_max": 30.518171310424805,
      "activations/layer2_attention_weight_min": -30.748380661010742,
      "activations/layer3_attention_weight_max": 91.14462280273438,
      "activations/layer3_attention_weight_min": -94.11898803710938,
      "activations/layer4_attention_weight_max": 116.63957977294922,
      "activations/layer4_attention_weight_min": -114.07818603515625,
      "activations/layer5_attention_weight_max": 53.36269760131836,
      "activations/layer5_attention_weight_min": -58.350181579589844,
      "activations/layer6_attention_weight_max": 45.00260925292969,
      "activations/layer6_attention_weight_min": -47.52487564086914,
      "activations/layer7_attention_weight_max": 86.32645416259766,
      "activations/layer7_attention_weight_min": -85.39593505859375,
      "activations/layer8_attention_weight_max": 39.69020462036133,
      "activations/layer8_attention_weight_min": -38.22478103637695,
      "activations/layer9_attention_weight_max": 32.344329833984375,
      "activations/layer9_attention_weight_min": -30.956491470336914,
      "epoch": 8.64,
      "learning_rate": 9.523977272727273e-05,
      "loss": 2.7979,
      "step": 148700
    },
    {
      "activations/layer0_attention_weight_max": 15.677840232849121,
      "activations/layer0_attention_weight_min": -13.817434310913086,
      "activations/layer10_attention_weight_max": 33.70912551879883,
      "activations/layer10_attention_weight_min": -31.780508041381836,
      "activations/layer11_attention_weight_max": 33.531089782714844,
      "activations/layer11_attention_weight_min": -31.313207626342773,
      "activations/layer12_attention_weight_max": 23.389034271240234,
      "activations/layer12_attention_weight_min": -25.91199493408203,
      "activations/layer13_attention_weight_max": 36.70260238647461,
      "activations/layer13_attention_weight_min": -32.62083435058594,
      "activations/layer14_attention_weight_max": 45.642974853515625,
      "activations/layer14_attention_weight_min": -36.36594009399414,
      "activations/layer15_attention_weight_max": 35.676841735839844,
      "activations/layer15_attention_weight_min": -30.300525665283203,
      "activations/layer16_attention_weight_max": 32.306236267089844,
      "activations/layer16_attention_weight_min": -26.54071044921875,
      "activations/layer17_attention_weight_max": 36.16875457763672,
      "activations/layer17_attention_weight_min": -25.139568328857422,
      "activations/layer18_attention_weight_max": 31.450340270996094,
      "activations/layer18_attention_weight_min": -23.04702377319336,
      "activations/layer19_attention_weight_max": 37.12837600708008,
      "activations/layer19_attention_weight_min": -33.63302230834961,
      "activations/layer1_attention_weight_max": 16.44941520690918,
      "activations/layer1_attention_weight_min": -14.548972129821777,
      "activations/layer20_attention_weight_max": 31.030017852783203,
      "activations/layer20_attention_weight_min": -24.79671859741211,
      "activations/layer21_attention_weight_max": 31.009885787963867,
      "activations/layer21_attention_weight_min": -21.4732666015625,
      "activations/layer22_attention_weight_max": 47.66327667236328,
      "activations/layer22_attention_weight_min": -29.744213104248047,
      "activations/layer23_attention_weight_max": 38.244510650634766,
      "activations/layer23_attention_weight_min": -25.08125877380371,
      "activations/layer2_attention_weight_max": 34.035911560058594,
      "activations/layer2_attention_weight_min": -35.006141662597656,
      "activations/layer3_attention_weight_max": 98.56172180175781,
      "activations/layer3_attention_weight_min": -100.53606414794922,
      "activations/layer4_attention_weight_max": 124.08223724365234,
      "activations/layer4_attention_weight_min": -121.9394302368164,
      "activations/layer5_attention_weight_max": 56.275611877441406,
      "activations/layer5_attention_weight_min": -63.287506103515625,
      "activations/layer6_attention_weight_max": 50.914676666259766,
      "activations/layer6_attention_weight_min": -51.49559783935547,
      "activations/layer7_attention_weight_max": 94.84071350097656,
      "activations/layer7_attention_weight_min": -89.73950958251953,
      "activations/layer8_attention_weight_max": 40.56096267700195,
      "activations/layer8_attention_weight_min": -39.331817626953125,
      "activations/layer9_attention_weight_max": 33.990108489990234,
      "activations/layer9_attention_weight_min": -34.4201774597168,
      "epoch": 8.64,
      "learning_rate": 9.522083333333331e-05,
      "loss": 2.8072,
      "step": 148750
    },
    {
      "activations/layer0_attention_weight_max": 15.646280288696289,
      "activations/layer0_attention_weight_min": -13.244047164916992,
      "activations/layer10_attention_weight_max": 35.71211624145508,
      "activations/layer10_attention_weight_min": -35.744964599609375,
      "activations/layer11_attention_weight_max": 40.750850677490234,
      "activations/layer11_attention_weight_min": -33.755279541015625,
      "activations/layer12_attention_weight_max": 58.69121551513672,
      "activations/layer12_attention_weight_min": -25.239450454711914,
      "activations/layer13_attention_weight_max": 76.03370666503906,
      "activations/layer13_attention_weight_min": -40.62158966064453,
      "activations/layer14_attention_weight_max": 56.244056701660156,
      "activations/layer14_attention_weight_min": -36.65378189086914,
      "activations/layer15_attention_weight_max": 59.712188720703125,
      "activations/layer15_attention_weight_min": -30.065719604492188,
      "activations/layer16_attention_weight_max": 36.47403335571289,
      "activations/layer16_attention_weight_min": -27.678842544555664,
      "activations/layer17_attention_weight_max": 34.00688552856445,
      "activations/layer17_attention_weight_min": -24.77573585510254,
      "activations/layer18_attention_weight_max": 31.614294052124023,
      "activations/layer18_attention_weight_min": -22.8154239654541,
      "activations/layer19_attention_weight_max": 39.66877365112305,
      "activations/layer19_attention_weight_min": -31.16116714477539,
      "activations/layer1_attention_weight_max": 17.161773681640625,
      "activations/layer1_attention_weight_min": -15.226388931274414,
      "activations/layer20_attention_weight_max": 30.078096389770508,
      "activations/layer20_attention_weight_min": -27.50657844543457,
      "activations/layer21_attention_weight_max": 28.98134994506836,
      "activations/layer21_attention_weight_min": -24.463823318481445,
      "activations/layer22_attention_weight_max": 51.039588928222656,
      "activations/layer22_attention_weight_min": -28.788097381591797,
      "activations/layer23_attention_weight_max": 41.75413513183594,
      "activations/layer23_attention_weight_min": -24.27145004272461,
      "activations/layer2_attention_weight_max": 33.75165557861328,
      "activations/layer2_attention_weight_min": -33.21892166137695,
      "activations/layer3_attention_weight_max": 92.49451446533203,
      "activations/layer3_attention_weight_min": -99.0273666381836,
      "activations/layer4_attention_weight_max": 116.62430572509766,
      "activations/layer4_attention_weight_min": -121.99556732177734,
      "activations/layer5_attention_weight_max": 55.87269592285156,
      "activations/layer5_attention_weight_min": -63.73002243041992,
      "activations/layer6_attention_weight_max": 49.24253463745117,
      "activations/layer6_attention_weight_min": -50.249229431152344,
      "activations/layer7_attention_weight_max": 94.35099792480469,
      "activations/layer7_attention_weight_min": -89.19216918945312,
      "activations/layer8_attention_weight_max": 44.199867248535156,
      "activations/layer8_attention_weight_min": -44.68928527832031,
      "activations/layer9_attention_weight_max": 40.85504150390625,
      "activations/layer9_attention_weight_min": -39.50460433959961,
      "epoch": 8.65,
      "learning_rate": 9.520189393939393e-05,
      "loss": 2.7974,
      "step": 148800
    },
    {
      "activations/layer0_attention_weight_max": 15.589571952819824,
      "activations/layer0_attention_weight_min": -11.841904640197754,
      "activations/layer10_attention_weight_max": 36.73283386230469,
      "activations/layer10_attention_weight_min": -32.79651641845703,
      "activations/layer11_attention_weight_max": 32.92469024658203,
      "activations/layer11_attention_weight_min": -31.775381088256836,
      "activations/layer12_attention_weight_max": 23.89177703857422,
      "activations/layer12_attention_weight_min": -28.939956665039062,
      "activations/layer13_attention_weight_max": 41.246788024902344,
      "activations/layer13_attention_weight_min": -36.4827995300293,
      "activations/layer14_attention_weight_max": 47.94596862792969,
      "activations/layer14_attention_weight_min": -40.845054626464844,
      "activations/layer15_attention_weight_max": 36.60150909423828,
      "activations/layer15_attention_weight_min": -29.768596649169922,
      "activations/layer16_attention_weight_max": 35.83485794067383,
      "activations/layer16_attention_weight_min": -27.654985427856445,
      "activations/layer17_attention_weight_max": 34.26808547973633,
      "activations/layer17_attention_weight_min": -27.572999954223633,
      "activations/layer18_attention_weight_max": 33.47418212890625,
      "activations/layer18_attention_weight_min": -22.985305786132812,
      "activations/layer19_attention_weight_max": 37.19952392578125,
      "activations/layer19_attention_weight_min": -32.626914978027344,
      "activations/layer1_attention_weight_max": 16.183551788330078,
      "activations/layer1_attention_weight_min": -15.399168968200684,
      "activations/layer20_attention_weight_max": 30.987071990966797,
      "activations/layer20_attention_weight_min": -25.760290145874023,
      "activations/layer21_attention_weight_max": 29.919750213623047,
      "activations/layer21_attention_weight_min": -24.96323013305664,
      "activations/layer22_attention_weight_max": 49.75250244140625,
      "activations/layer22_attention_weight_min": -30.911645889282227,
      "activations/layer23_attention_weight_max": 37.298580169677734,
      "activations/layer23_attention_weight_min": -24.040538787841797,
      "activations/layer2_attention_weight_max": 34.02230453491211,
      "activations/layer2_attention_weight_min": -32.828330993652344,
      "activations/layer3_attention_weight_max": 96.13704681396484,
      "activations/layer3_attention_weight_min": -101.5198745727539,
      "activations/layer4_attention_weight_max": 120.33924865722656,
      "activations/layer4_attention_weight_min": -125.38996887207031,
      "activations/layer5_attention_weight_max": 54.56995391845703,
      "activations/layer5_attention_weight_min": -62.20073699951172,
      "activations/layer6_attention_weight_max": 50.744205474853516,
      "activations/layer6_attention_weight_min": -52.27104568481445,
      "activations/layer7_attention_weight_max": 88.4140853881836,
      "activations/layer7_attention_weight_min": -87.54734802246094,
      "activations/layer8_attention_weight_max": 39.72990798950195,
      "activations/layer8_attention_weight_min": -40.10872268676758,
      "activations/layer9_attention_weight_max": 37.332481384277344,
      "activations/layer9_attention_weight_min": -34.705230712890625,
      "epoch": 8.65,
      "learning_rate": 9.518295454545454e-05,
      "loss": 2.8065,
      "step": 148850
    },
    {
      "activations/layer0_attention_weight_max": 14.371706008911133,
      "activations/layer0_attention_weight_min": -12.41313362121582,
      "activations/layer10_attention_weight_max": 34.24026107788086,
      "activations/layer10_attention_weight_min": -30.475637435913086,
      "activations/layer11_attention_weight_max": 31.89138412475586,
      "activations/layer11_attention_weight_min": -29.418933868408203,
      "activations/layer12_attention_weight_max": 23.191791534423828,
      "activations/layer12_attention_weight_min": -32.05094909667969,
      "activations/layer13_attention_weight_max": 37.6928596496582,
      "activations/layer13_attention_weight_min": -32.42134475708008,
      "activations/layer14_attention_weight_max": 49.68342208862305,
      "activations/layer14_attention_weight_min": -40.73759460449219,
      "activations/layer15_attention_weight_max": 34.713375091552734,
      "activations/layer15_attention_weight_min": -29.556434631347656,
      "activations/layer16_attention_weight_max": 34.156700134277344,
      "activations/layer16_attention_weight_min": -26.841461181640625,
      "activations/layer17_attention_weight_max": 34.35667419433594,
      "activations/layer17_attention_weight_min": -26.311613082885742,
      "activations/layer18_attention_weight_max": 31.872882843017578,
      "activations/layer18_attention_weight_min": -21.587942123413086,
      "activations/layer19_attention_weight_max": 40.95654296875,
      "activations/layer19_attention_weight_min": -31.162717819213867,
      "activations/layer1_attention_weight_max": 16.963184356689453,
      "activations/layer1_attention_weight_min": -13.877031326293945,
      "activations/layer20_attention_weight_max": 31.02971649169922,
      "activations/layer20_attention_weight_min": -23.488487243652344,
      "activations/layer21_attention_weight_max": 31.381155014038086,
      "activations/layer21_attention_weight_min": -23.043140411376953,
      "activations/layer22_attention_weight_max": 45.833946228027344,
      "activations/layer22_attention_weight_min": -27.530149459838867,
      "activations/layer23_attention_weight_max": 36.66929626464844,
      "activations/layer23_attention_weight_min": -21.906509399414062,
      "activations/layer2_attention_weight_max": 32.3642578125,
      "activations/layer2_attention_weight_min": -30.603515625,
      "activations/layer3_attention_weight_max": 93.35320281982422,
      "activations/layer3_attention_weight_min": -92.9295425415039,
      "activations/layer4_attention_weight_max": 118.8077392578125,
      "activations/layer4_attention_weight_min": -113.2135238647461,
      "activations/layer5_attention_weight_max": 57.39563751220703,
      "activations/layer5_attention_weight_min": -62.45118713378906,
      "activations/layer6_attention_weight_max": 49.236351013183594,
      "activations/layer6_attention_weight_min": -49.69977569580078,
      "activations/layer7_attention_weight_max": 94.4176254272461,
      "activations/layer7_attention_weight_min": -88.4117660522461,
      "activations/layer8_attention_weight_max": 41.8242301940918,
      "activations/layer8_attention_weight_min": -38.84675979614258,
      "activations/layer9_attention_weight_max": 35.68709945678711,
      "activations/layer9_attention_weight_min": -33.30826950073242,
      "epoch": 8.65,
      "learning_rate": 9.516401515151513e-05,
      "loss": 2.8271,
      "step": 148900
    },
    {
      "activations/layer0_attention_weight_max": 15.737615585327148,
      "activations/layer0_attention_weight_min": -12.615870475769043,
      "activations/layer10_attention_weight_max": 36.15424346923828,
      "activations/layer10_attention_weight_min": -34.62067413330078,
      "activations/layer11_attention_weight_max": 34.657623291015625,
      "activations/layer11_attention_weight_min": -33.079345703125,
      "activations/layer12_attention_weight_max": 28.31521224975586,
      "activations/layer12_attention_weight_min": -28.09406852722168,
      "activations/layer13_attention_weight_max": 48.88991928100586,
      "activations/layer13_attention_weight_min": -37.01615524291992,
      "activations/layer14_attention_weight_max": 59.869232177734375,
      "activations/layer14_attention_weight_min": -44.88541030883789,
      "activations/layer15_attention_weight_max": 42.84689712524414,
      "activations/layer15_attention_weight_min": -34.543556213378906,
      "activations/layer16_attention_weight_max": 37.4169921875,
      "activations/layer16_attention_weight_min": -28.760751724243164,
      "activations/layer17_attention_weight_max": 39.26250076293945,
      "activations/layer17_attention_weight_min": -27.33696174621582,
      "activations/layer18_attention_weight_max": 36.06058883666992,
      "activations/layer18_attention_weight_min": -23.136062622070312,
      "activations/layer19_attention_weight_max": 43.570396423339844,
      "activations/layer19_attention_weight_min": -35.02412414550781,
      "activations/layer1_attention_weight_max": 16.97901725769043,
      "activations/layer1_attention_weight_min": -14.695548057556152,
      "activations/layer20_attention_weight_max": 38.58616638183594,
      "activations/layer20_attention_weight_min": -26.38055419921875,
      "activations/layer21_attention_weight_max": 38.097782135009766,
      "activations/layer21_attention_weight_min": -25.305686950683594,
      "activations/layer22_attention_weight_max": 62.979393005371094,
      "activations/layer22_attention_weight_min": -31.24462890625,
      "activations/layer23_attention_weight_max": 45.111175537109375,
      "activations/layer23_attention_weight_min": -25.51453399658203,
      "activations/layer2_attention_weight_max": 33.57668685913086,
      "activations/layer2_attention_weight_min": -33.805572509765625,
      "activations/layer3_attention_weight_max": 97.13633728027344,
      "activations/layer3_attention_weight_min": -97.3307113647461,
      "activations/layer4_attention_weight_max": 125.73686981201172,
      "activations/layer4_attention_weight_min": -119.77689361572266,
      "activations/layer5_attention_weight_max": 57.772369384765625,
      "activations/layer5_attention_weight_min": -62.71142578125,
      "activations/layer6_attention_weight_max": 52.370540618896484,
      "activations/layer6_attention_weight_min": -52.56114959716797,
      "activations/layer7_attention_weight_max": 98.89025115966797,
      "activations/layer7_attention_weight_min": -96.1323471069336,
      "activations/layer8_attention_weight_max": 43.72024154663086,
      "activations/layer8_attention_weight_min": -43.48451232910156,
      "activations/layer9_attention_weight_max": 39.56467819213867,
      "activations/layer9_attention_weight_min": -37.92377471923828,
      "epoch": 8.65,
      "learning_rate": 9.514507575757575e-05,
      "loss": 2.8121,
      "step": 148950
    },
    {
      "activations/layer0_attention_weight_max": 16.063323974609375,
      "activations/layer0_attention_weight_min": -13.306478500366211,
      "activations/layer10_attention_weight_max": 35.56886672973633,
      "activations/layer10_attention_weight_min": -35.06634521484375,
      "activations/layer11_attention_weight_max": 34.24789047241211,
      "activations/layer11_attention_weight_min": -31.287961959838867,
      "activations/layer12_attention_weight_max": 27.43667221069336,
      "activations/layer12_attention_weight_min": -26.76681900024414,
      "activations/layer13_attention_weight_max": 39.62004470825195,
      "activations/layer13_attention_weight_min": -33.53829574584961,
      "activations/layer14_attention_weight_max": 45.8552131652832,
      "activations/layer14_attention_weight_min": -37.70627212524414,
      "activations/layer15_attention_weight_max": 35.79485321044922,
      "activations/layer15_attention_weight_min": -30.22646713256836,
      "activations/layer16_attention_weight_max": 30.625728607177734,
      "activations/layer16_attention_weight_min": -25.963762283325195,
      "activations/layer17_attention_weight_max": 34.82426071166992,
      "activations/layer17_attention_weight_min": -24.40825653076172,
      "activations/layer18_attention_weight_max": 34.81914520263672,
      "activations/layer18_attention_weight_min": -22.09563636779785,
      "activations/layer19_attention_weight_max": 36.494712829589844,
      "activations/layer19_attention_weight_min": -31.81660270690918,
      "activations/layer1_attention_weight_max": 16.176626205444336,
      "activations/layer1_attention_weight_min": -15.1322660446167,
      "activations/layer20_attention_weight_max": 30.721372604370117,
      "activations/layer20_attention_weight_min": -26.273412704467773,
      "activations/layer21_attention_weight_max": 32.135169982910156,
      "activations/layer21_attention_weight_min": -23.608911514282227,
      "activations/layer22_attention_weight_max": 48.20322036743164,
      "activations/layer22_attention_weight_min": -29.62214469909668,
      "activations/layer23_attention_weight_max": 39.952613830566406,
      "activations/layer23_attention_weight_min": -25.061683654785156,
      "activations/layer2_attention_weight_max": 32.74446487426758,
      "activations/layer2_attention_weight_min": -33.706336975097656,
      "activations/layer3_attention_weight_max": 95.5202407836914,
      "activations/layer3_attention_weight_min": -97.66287994384766,
      "activations/layer4_attention_weight_max": 120.14632415771484,
      "activations/layer4_attention_weight_min": -120.04756927490234,
      "activations/layer5_attention_weight_max": 54.6556396484375,
      "activations/layer5_attention_weight_min": -59.48368453979492,
      "activations/layer6_attention_weight_max": 47.421424865722656,
      "activations/layer6_attention_weight_min": -50.13603591918945,
      "activations/layer7_attention_weight_max": 87.69428253173828,
      "activations/layer7_attention_weight_min": -86.91643524169922,
      "activations/layer8_attention_weight_max": 38.798736572265625,
      "activations/layer8_attention_weight_min": -38.53471755981445,
      "activations/layer9_attention_weight_max": 35.48479080200195,
      "activations/layer9_attention_weight_min": -33.009056091308594,
      "epoch": 8.66,
      "learning_rate": 9.512613636363636e-05,
      "loss": 2.7936,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.5659,
      "eval_samples_per_second": 501.29,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.5659,
      "eval_openwebtext_samples_per_second": 501.29,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0189,
      "eval_wikitext_samples_per_second": 225.87,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_lambada_loss": 2.5625,
      "eval_lambada_ppl": 12.968197316969134,
      "eval_lambada_runtime": 9.6121,
      "eval_lambada_samples_per_second": 506.547,
      "step": 149000
    },
    {
      "activations/layer0_attention_weight_max": 16.59372901916504,
      "activations/layer0_attention_weight_min": -12.982308387756348,
      "activations/layer10_attention_weight_max": 31.223711013793945,
      "activations/layer10_attention_weight_min": -30.66973876953125,
      "activations/layer11_attention_weight_max": 32.63529968261719,
      "activations/layer11_attention_weight_min": -29.34675407409668,
      "activations/layer12_attention_weight_max": 25.06640625,
      "activations/layer12_attention_weight_min": -27.98672103881836,
      "activations/layer13_attention_weight_max": 43.07255554199219,
      "activations/layer13_attention_weight_min": -33.3150520324707,
      "activations/layer14_attention_weight_max": 58.94247055053711,
      "activations/layer14_attention_weight_min": -46.48997497558594,
      "activations/layer15_attention_weight_max": 41.9804801940918,
      "activations/layer15_attention_weight_min": -30.58429718017578,
      "activations/layer16_attention_weight_max": 35.27904510498047,
      "activations/layer16_attention_weight_min": -28.954660415649414,
      "activations/layer17_attention_weight_max": 33.86197280883789,
      "activations/layer17_attention_weight_min": -26.393253326416016,
      "activations/layer18_attention_weight_max": 33.29292297363281,
      "activations/layer18_attention_weight_min": -22.102882385253906,
      "activations/layer19_attention_weight_max": 39.729366302490234,
      "activations/layer19_attention_weight_min": -33.48501205444336,
      "activations/layer1_attention_weight_max": 17.729379653930664,
      "activations/layer1_attention_weight_min": -15.22465991973877,
      "activations/layer20_attention_weight_max": 31.849679946899414,
      "activations/layer20_attention_weight_min": -25.93746566772461,
      "activations/layer21_attention_weight_max": 34.75835418701172,
      "activations/layer21_attention_weight_min": -26.46293830871582,
      "activations/layer22_attention_weight_max": 53.89976119995117,
      "activations/layer22_attention_weight_min": -29.762531280517578,
      "activations/layer23_attention_weight_max": 38.81713104248047,
      "activations/layer23_attention_weight_min": -26.113256454467773,
      "activations/layer2_attention_weight_max": 32.569969177246094,
      "activations/layer2_attention_weight_min": -31.455631256103516,
      "activations/layer3_attention_weight_max": 94.50552368164062,
      "activations/layer3_attention_weight_min": -93.08168029785156,
      "activations/layer4_attention_weight_max": 115.58123016357422,
      "activations/layer4_attention_weight_min": -112.87861633300781,
      "activations/layer5_attention_weight_max": 55.16139221191406,
      "activations/layer5_attention_weight_min": -60.14459228515625,
      "activations/layer6_attention_weight_max": 47.771446228027344,
      "activations/layer6_attention_weight_min": -51.841793060302734,
      "activations/layer7_attention_weight_max": 85.10803985595703,
      "activations/layer7_attention_weight_min": -85.96654510498047,
      "activations/layer8_attention_weight_max": 39.31730270385742,
      "activations/layer8_attention_weight_min": -40.64598846435547,
      "activations/layer9_attention_weight_max": 33.71376419067383,
      "activations/layer9_attention_weight_min": -33.226768493652344,
      "epoch": 8.66,
      "learning_rate": 9.510719696969696e-05,
      "loss": 2.7924,
      "step": 149050
    },
    {
      "activations/layer0_attention_weight_max": 15.272223472595215,
      "activations/layer0_attention_weight_min": -12.994338035583496,
      "activations/layer10_attention_weight_max": 35.07170867919922,
      "activations/layer10_attention_weight_min": -34.820106506347656,
      "activations/layer11_attention_weight_max": 34.36910629272461,
      "activations/layer11_attention_weight_min": -32.89482498168945,
      "activations/layer12_attention_weight_max": 23.29633903503418,
      "activations/layer12_attention_weight_min": -26.05510902404785,
      "activations/layer13_attention_weight_max": 40.96940994262695,
      "activations/layer13_attention_weight_min": -33.498451232910156,
      "activations/layer14_attention_weight_max": 55.079322814941406,
      "activations/layer14_attention_weight_min": -40.97795867919922,
      "activations/layer15_attention_weight_max": 38.669212341308594,
      "activations/layer15_attention_weight_min": -29.6735782623291,
      "activations/layer16_attention_weight_max": 34.97098922729492,
      "activations/layer16_attention_weight_min": -26.107229232788086,
      "activations/layer17_attention_weight_max": 38.20066833496094,
      "activations/layer17_attention_weight_min": -26.8012752532959,
      "activations/layer18_attention_weight_max": 37.42182540893555,
      "activations/layer18_attention_weight_min": -24.69900131225586,
      "activations/layer19_attention_weight_max": 41.53781509399414,
      "activations/layer19_attention_weight_min": -30.4321346282959,
      "activations/layer1_attention_weight_max": 16.34720802307129,
      "activations/layer1_attention_weight_min": -15.087952613830566,
      "activations/layer20_attention_weight_max": 33.04356384277344,
      "activations/layer20_attention_weight_min": -24.344512939453125,
      "activations/layer21_attention_weight_max": 34.76465606689453,
      "activations/layer21_attention_weight_min": -22.643341064453125,
      "activations/layer22_attention_weight_max": 50.501651763916016,
      "activations/layer22_attention_weight_min": -29.40043830871582,
      "activations/layer23_attention_weight_max": 40.789268493652344,
      "activations/layer23_attention_weight_min": -23.69647789001465,
      "activations/layer2_attention_weight_max": 30.746124267578125,
      "activations/layer2_attention_weight_min": -30.525875091552734,
      "activations/layer3_attention_weight_max": 92.22582244873047,
      "activations/layer3_attention_weight_min": -96.25791931152344,
      "activations/layer4_attention_weight_max": 116.869140625,
      "activations/layer4_attention_weight_min": -117.33900451660156,
      "activations/layer5_attention_weight_max": 56.978553771972656,
      "activations/layer5_attention_weight_min": -63.28384017944336,
      "activations/layer6_attention_weight_max": 49.337928771972656,
      "activations/layer6_attention_weight_min": -53.038230895996094,
      "activations/layer7_attention_weight_max": 94.1301040649414,
      "activations/layer7_attention_weight_min": -96.3564453125,
      "activations/layer8_attention_weight_max": 42.450992584228516,
      "activations/layer8_attention_weight_min": -46.60917663574219,
      "activations/layer9_attention_weight_max": 36.255149841308594,
      "activations/layer9_attention_weight_min": -36.06136703491211,
      "epoch": 8.66,
      "learning_rate": 9.508825757575757e-05,
      "loss": 2.8196,
      "step": 149100
    },
    {
      "activations/layer0_attention_weight_max": 14.777941703796387,
      "activations/layer0_attention_weight_min": -13.673848152160645,
      "activations/layer10_attention_weight_max": 37.211612701416016,
      "activations/layer10_attention_weight_min": -34.640926361083984,
      "activations/layer11_attention_weight_max": 38.31568145751953,
      "activations/layer11_attention_weight_min": -34.39622497558594,
      "activations/layer12_attention_weight_max": 25.820470809936523,
      "activations/layer12_attention_weight_min": -28.15497398376465,
      "activations/layer13_attention_weight_max": 35.9879264831543,
      "activations/layer13_attention_weight_min": -32.2580451965332,
      "activations/layer14_attention_weight_max": 48.609920501708984,
      "activations/layer14_attention_weight_min": -36.45449447631836,
      "activations/layer15_attention_weight_max": 36.2491340637207,
      "activations/layer15_attention_weight_min": -29.001968383789062,
      "activations/layer16_attention_weight_max": 35.96261978149414,
      "activations/layer16_attention_weight_min": -27.736923217773438,
      "activations/layer17_attention_weight_max": 31.299652099609375,
      "activations/layer17_attention_weight_min": -24.295068740844727,
      "activations/layer18_attention_weight_max": 29.849218368530273,
      "activations/layer18_attention_weight_min": -20.528169631958008,
      "activations/layer19_attention_weight_max": 35.41325759887695,
      "activations/layer19_attention_weight_min": -30.851572036743164,
      "activations/layer1_attention_weight_max": 16.96885108947754,
      "activations/layer1_attention_weight_min": -13.957867622375488,
      "activations/layer20_attention_weight_max": 28.186511993408203,
      "activations/layer20_attention_weight_min": -26.306907653808594,
      "activations/layer21_attention_weight_max": 26.358999252319336,
      "activations/layer21_attention_weight_min": -21.755144119262695,
      "activations/layer22_attention_weight_max": 46.95572280883789,
      "activations/layer22_attention_weight_min": -28.19810676574707,
      "activations/layer23_attention_weight_max": 35.093894958496094,
      "activations/layer23_attention_weight_min": -22.905637741088867,
      "activations/layer2_attention_weight_max": 35.287841796875,
      "activations/layer2_attention_weight_min": -32.60796356201172,
      "activations/layer3_attention_weight_max": 95.74907684326172,
      "activations/layer3_attention_weight_min": -94.02378845214844,
      "activations/layer4_attention_weight_max": 122.22132873535156,
      "activations/layer4_attention_weight_min": -117.34673309326172,
      "activations/layer5_attention_weight_max": 52.96208953857422,
      "activations/layer5_attention_weight_min": -60.903297424316406,
      "activations/layer6_attention_weight_max": 46.988426208496094,
      "activations/layer6_attention_weight_min": -50.69283676147461,
      "activations/layer7_attention_weight_max": 101.58601379394531,
      "activations/layer7_attention_weight_min": -93.77510070800781,
      "activations/layer8_attention_weight_max": 42.52220153808594,
      "activations/layer8_attention_weight_min": -42.40605545043945,
      "activations/layer9_attention_weight_max": 40.23344421386719,
      "activations/layer9_attention_weight_min": -36.82319259643555,
      "epoch": 8.67,
      "learning_rate": 9.506931818181817e-05,
      "loss": 2.803,
      "step": 149150
    },
    {
      "activations/layer0_attention_weight_max": 14.945484161376953,
      "activations/layer0_attention_weight_min": -12.91090202331543,
      "activations/layer10_attention_weight_max": 34.05638122558594,
      "activations/layer10_attention_weight_min": -33.9593391418457,
      "activations/layer11_attention_weight_max": 35.5252799987793,
      "activations/layer11_attention_weight_min": -34.615699768066406,
      "activations/layer12_attention_weight_max": 26.066591262817383,
      "activations/layer12_attention_weight_min": -30.333885192871094,
      "activations/layer13_attention_weight_max": 47.64267349243164,
      "activations/layer13_attention_weight_min": -34.411476135253906,
      "activations/layer14_attention_weight_max": 46.21017837524414,
      "activations/layer14_attention_weight_min": -41.27019500732422,
      "activations/layer15_attention_weight_max": 36.71506118774414,
      "activations/layer15_attention_weight_min": -33.01484680175781,
      "activations/layer16_attention_weight_max": 30.219526290893555,
      "activations/layer16_attention_weight_min": -26.566431045532227,
      "activations/layer17_attention_weight_max": 31.72553825378418,
      "activations/layer17_attention_weight_min": -24.352558135986328,
      "activations/layer18_attention_weight_max": 29.468156814575195,
      "activations/layer18_attention_weight_min": -20.408723831176758,
      "activations/layer19_attention_weight_max": 37.5817985534668,
      "activations/layer19_attention_weight_min": -29.15003776550293,
      "activations/layer1_attention_weight_max": 17.36318016052246,
      "activations/layer1_attention_weight_min": -15.588637351989746,
      "activations/layer20_attention_weight_max": 29.451276779174805,
      "activations/layer20_attention_weight_min": -25.051191329956055,
      "activations/layer21_attention_weight_max": 29.03318214416504,
      "activations/layer21_attention_weight_min": -23.969396591186523,
      "activations/layer22_attention_weight_max": 44.961883544921875,
      "activations/layer22_attention_weight_min": -27.965059280395508,
      "activations/layer23_attention_weight_max": 36.167747497558594,
      "activations/layer23_attention_weight_min": -24.314048767089844,
      "activations/layer2_attention_weight_max": 35.00825500488281,
      "activations/layer2_attention_weight_min": -35.0054931640625,
      "activations/layer3_attention_weight_max": 100.31332397460938,
      "activations/layer3_attention_weight_min": -101.49266052246094,
      "activations/layer4_attention_weight_max": 126.4169692993164,
      "activations/layer4_attention_weight_min": -119.99176025390625,
      "activations/layer5_attention_weight_max": 53.853878021240234,
      "activations/layer5_attention_weight_min": -62.955482482910156,
      "activations/layer6_attention_weight_max": 48.80351638793945,
      "activations/layer6_attention_weight_min": -51.791873931884766,
      "activations/layer7_attention_weight_max": 105.65851593017578,
      "activations/layer7_attention_weight_min": -87.83190155029297,
      "activations/layer8_attention_weight_max": 42.495574951171875,
      "activations/layer8_attention_weight_min": -42.01420211791992,
      "activations/layer9_attention_weight_max": 36.76470184326172,
      "activations/layer9_attention_weight_min": -34.290283203125,
      "epoch": 8.67,
      "learning_rate": 9.505037878787878e-05,
      "loss": 2.7991,
      "step": 149200
    },
    {
      "activations/layer0_attention_weight_max": 17.14967155456543,
      "activations/layer0_attention_weight_min": -13.145214080810547,
      "activations/layer10_attention_weight_max": 30.30117416381836,
      "activations/layer10_attention_weight_min": -31.44249725341797,
      "activations/layer11_attention_weight_max": 32.04436492919922,
      "activations/layer11_attention_weight_min": -28.136329650878906,
      "activations/layer12_attention_weight_max": 21.7179012298584,
      "activations/layer12_attention_weight_min": -23.346464157104492,
      "activations/layer13_attention_weight_max": 40.29925537109375,
      "activations/layer13_attention_weight_min": -32.234947204589844,
      "activations/layer14_attention_weight_max": 53.6678466796875,
      "activations/layer14_attention_weight_min": -46.387603759765625,
      "activations/layer15_attention_weight_max": 35.052452087402344,
      "activations/layer15_attention_weight_min": -28.36136817932129,
      "activations/layer16_attention_weight_max": 30.899686813354492,
      "activations/layer16_attention_weight_min": -28.487010955810547,
      "activations/layer17_attention_weight_max": 30.770673751831055,
      "activations/layer17_attention_weight_min": -25.977798461914062,
      "activations/layer18_attention_weight_max": 30.19550132751465,
      "activations/layer18_attention_weight_min": -21.628583908081055,
      "activations/layer19_attention_weight_max": 33.94907760620117,
      "activations/layer19_attention_weight_min": -31.637441635131836,
      "activations/layer1_attention_weight_max": 16.593690872192383,
      "activations/layer1_attention_weight_min": -15.843728065490723,
      "activations/layer20_attention_weight_max": 28.53005599975586,
      "activations/layer20_attention_weight_min": -23.52277946472168,
      "activations/layer21_attention_weight_max": 27.676742553710938,
      "activations/layer21_attention_weight_min": -23.628454208374023,
      "activations/layer22_attention_weight_max": 49.01882553100586,
      "activations/layer22_attention_weight_min": -29.886505126953125,
      "activations/layer23_attention_weight_max": 37.2714729309082,
      "activations/layer23_attention_weight_min": -25.582901000976562,
      "activations/layer2_attention_weight_max": 34.49361038208008,
      "activations/layer2_attention_weight_min": -32.29249572753906,
      "activations/layer3_attention_weight_max": 96.40919494628906,
      "activations/layer3_attention_weight_min": -96.6685791015625,
      "activations/layer4_attention_weight_max": 116.10787200927734,
      "activations/layer4_attention_weight_min": -114.86445617675781,
      "activations/layer5_attention_weight_max": 54.40373229980469,
      "activations/layer5_attention_weight_min": -63.8216667175293,
      "activations/layer6_attention_weight_max": 46.92039489746094,
      "activations/layer6_attention_weight_min": -50.426021575927734,
      "activations/layer7_attention_weight_max": 83.58394622802734,
      "activations/layer7_attention_weight_min": -83.5936050415039,
      "activations/layer8_attention_weight_max": 38.98952102661133,
      "activations/layer8_attention_weight_min": -38.00632095336914,
      "activations/layer9_attention_weight_max": 33.97325134277344,
      "activations/layer9_attention_weight_min": -32.21920394897461,
      "epoch": 8.67,
      "learning_rate": 9.503143939393938e-05,
      "loss": 2.8058,
      "step": 149250
    },
    {
      "activations/layer0_attention_weight_max": 16.47543716430664,
      "activations/layer0_attention_weight_min": -12.529314041137695,
      "activations/layer10_attention_weight_max": 35.476112365722656,
      "activations/layer10_attention_weight_min": -31.33414649963379,
      "activations/layer11_attention_weight_max": 37.3058967590332,
      "activations/layer11_attention_weight_min": -30.202571868896484,
      "activations/layer12_attention_weight_max": 23.11333656311035,
      "activations/layer12_attention_weight_min": -24.616703033447266,
      "activations/layer13_attention_weight_max": 40.02427291870117,
      "activations/layer13_attention_weight_min": -32.89627456665039,
      "activations/layer14_attention_weight_max": 48.266563415527344,
      "activations/layer14_attention_weight_min": -37.704185485839844,
      "activations/layer15_attention_weight_max": 36.01679992675781,
      "activations/layer15_attention_weight_min": -30.482702255249023,
      "activations/layer16_attention_weight_max": 32.90362548828125,
      "activations/layer16_attention_weight_min": -27.66939926147461,
      "activations/layer17_attention_weight_max": 33.67829513549805,
      "activations/layer17_attention_weight_min": -25.096595764160156,
      "activations/layer18_attention_weight_max": 32.85044479370117,
      "activations/layer18_attention_weight_min": -21.96644401550293,
      "activations/layer19_attention_weight_max": 34.92220687866211,
      "activations/layer19_attention_weight_min": -31.445362091064453,
      "activations/layer1_attention_weight_max": 16.572036743164062,
      "activations/layer1_attention_weight_min": -15.066165924072266,
      "activations/layer20_attention_weight_max": 28.40576171875,
      "activations/layer20_attention_weight_min": -25.04157066345215,
      "activations/layer21_attention_weight_max": 28.50029754638672,
      "activations/layer21_attention_weight_min": -23.4185791015625,
      "activations/layer22_attention_weight_max": 44.65936279296875,
      "activations/layer22_attention_weight_min": -28.725671768188477,
      "activations/layer23_attention_weight_max": 37.03486633300781,
      "activations/layer23_attention_weight_min": -22.68316650390625,
      "activations/layer2_attention_weight_max": 34.458274841308594,
      "activations/layer2_attention_weight_min": -31.458852767944336,
      "activations/layer3_attention_weight_max": 101.0589828491211,
      "activations/layer3_attention_weight_min": -94.85377502441406,
      "activations/layer4_attention_weight_max": 122.18885803222656,
      "activations/layer4_attention_weight_min": -115.73897552490234,
      "activations/layer5_attention_weight_max": 55.89515686035156,
      "activations/layer5_attention_weight_min": -63.005836486816406,
      "activations/layer6_attention_weight_max": 49.699607849121094,
      "activations/layer6_attention_weight_min": -50.376956939697266,
      "activations/layer7_attention_weight_max": 86.91865539550781,
      "activations/layer7_attention_weight_min": -86.20345306396484,
      "activations/layer8_attention_weight_max": 40.909420013427734,
      "activations/layer8_attention_weight_min": -39.60078430175781,
      "activations/layer9_attention_weight_max": 35.669822692871094,
      "activations/layer9_attention_weight_min": -32.615230560302734,
      "epoch": 8.68,
      "learning_rate": 9.501249999999999e-05,
      "loss": 2.8105,
      "step": 149300
    },
    {
      "activations/layer0_attention_weight_max": 15.493663787841797,
      "activations/layer0_attention_weight_min": -13.009596824645996,
      "activations/layer10_attention_weight_max": 34.69743728637695,
      "activations/layer10_attention_weight_min": -32.103797912597656,
      "activations/layer11_attention_weight_max": 33.801204681396484,
      "activations/layer11_attention_weight_min": -30.195890426635742,
      "activations/layer12_attention_weight_max": 27.079435348510742,
      "activations/layer12_attention_weight_min": -27.88056182861328,
      "activations/layer13_attention_weight_max": 39.49282455444336,
      "activations/layer13_attention_weight_min": -32.95735549926758,
      "activations/layer14_attention_weight_max": 50.679683685302734,
      "activations/layer14_attention_weight_min": -38.902095794677734,
      "activations/layer15_attention_weight_max": 38.680519104003906,
      "activations/layer15_attention_weight_min": -30.569377899169922,
      "activations/layer16_attention_weight_max": 33.6900634765625,
      "activations/layer16_attention_weight_min": -27.558107376098633,
      "activations/layer17_attention_weight_max": 35.46627426147461,
      "activations/layer17_attention_weight_min": -26.347000122070312,
      "activations/layer18_attention_weight_max": 34.15718078613281,
      "activations/layer18_attention_weight_min": -22.25511360168457,
      "activations/layer19_attention_weight_max": 40.097625732421875,
      "activations/layer19_attention_weight_min": -31.652414321899414,
      "activations/layer1_attention_weight_max": 16.536680221557617,
      "activations/layer1_attention_weight_min": -15.219917297363281,
      "activations/layer20_attention_weight_max": 30.69103240966797,
      "activations/layer20_attention_weight_min": -27.698631286621094,
      "activations/layer21_attention_weight_max": 31.548112869262695,
      "activations/layer21_attention_weight_min": -23.909311294555664,
      "activations/layer22_attention_weight_max": 45.37931442260742,
      "activations/layer22_attention_weight_min": -28.490983963012695,
      "activations/layer23_attention_weight_max": 37.50294494628906,
      "activations/layer23_attention_weight_min": -22.408681869506836,
      "activations/layer2_attention_weight_max": 33.31913375854492,
      "activations/layer2_attention_weight_min": -31.264690399169922,
      "activations/layer3_attention_weight_max": 93.63487243652344,
      "activations/layer3_attention_weight_min": -92.84111785888672,
      "activations/layer4_attention_weight_max": 116.92342376708984,
      "activations/layer4_attention_weight_min": -121.111572265625,
      "activations/layer5_attention_weight_max": 53.409568786621094,
      "activations/layer5_attention_weight_min": -61.463985443115234,
      "activations/layer6_attention_weight_max": 47.22844696044922,
      "activations/layer6_attention_weight_min": -51.822689056396484,
      "activations/layer7_attention_weight_max": 91.85543060302734,
      "activations/layer7_attention_weight_min": -90.1799545288086,
      "activations/layer8_attention_weight_max": 39.393131256103516,
      "activations/layer8_attention_weight_min": -41.148040771484375,
      "activations/layer9_attention_weight_max": 36.13031768798828,
      "activations/layer9_attention_weight_min": -35.2998161315918,
      "epoch": 8.68,
      "learning_rate": 9.49935606060606e-05,
      "loss": 2.8213,
      "step": 149350
    },
    {
      "activations/layer0_attention_weight_max": 15.248644828796387,
      "activations/layer0_attention_weight_min": -11.75135612487793,
      "activations/layer10_attention_weight_max": 34.09016418457031,
      "activations/layer10_attention_weight_min": -34.32484817504883,
      "activations/layer11_attention_weight_max": 35.296600341796875,
      "activations/layer11_attention_weight_min": -30.786624908447266,
      "activations/layer12_attention_weight_max": 25.21082305908203,
      "activations/layer12_attention_weight_min": -25.317359924316406,
      "activations/layer13_attention_weight_max": 49.9682731628418,
      "activations/layer13_attention_weight_min": -33.46558380126953,
      "activations/layer14_attention_weight_max": 75.62462615966797,
      "activations/layer14_attention_weight_min": -47.29487991333008,
      "activations/layer15_attention_weight_max": 39.578033447265625,
      "activations/layer15_attention_weight_min": -32.73005294799805,
      "activations/layer16_attention_weight_max": 38.82196807861328,
      "activations/layer16_attention_weight_min": -26.61476707458496,
      "activations/layer17_attention_weight_max": 34.78473663330078,
      "activations/layer17_attention_weight_min": -25.165857315063477,
      "activations/layer18_attention_weight_max": 28.449424743652344,
      "activations/layer18_attention_weight_min": -22.495023727416992,
      "activations/layer19_attention_weight_max": 41.445472717285156,
      "activations/layer19_attention_weight_min": -29.6882381439209,
      "activations/layer1_attention_weight_max": 19.15177345275879,
      "activations/layer1_attention_weight_min": -16.62114906311035,
      "activations/layer20_attention_weight_max": 32.34077835083008,
      "activations/layer20_attention_weight_min": -24.146907806396484,
      "activations/layer21_attention_weight_max": 30.57491683959961,
      "activations/layer21_attention_weight_min": -21.175708770751953,
      "activations/layer22_attention_weight_max": 55.06385803222656,
      "activations/layer22_attention_weight_min": -29.364641189575195,
      "activations/layer23_attention_weight_max": 43.48100280761719,
      "activations/layer23_attention_weight_min": -25.85797119140625,
      "activations/layer2_attention_weight_max": 33.774391174316406,
      "activations/layer2_attention_weight_min": -34.462928771972656,
      "activations/layer3_attention_weight_max": 100.201416015625,
      "activations/layer3_attention_weight_min": -105.32672882080078,
      "activations/layer4_attention_weight_max": 122.71484375,
      "activations/layer4_attention_weight_min": -124.2048568725586,
      "activations/layer5_attention_weight_max": 55.10622787475586,
      "activations/layer5_attention_weight_min": -60.79279327392578,
      "activations/layer6_attention_weight_max": 46.709903717041016,
      "activations/layer6_attention_weight_min": -50.07020568847656,
      "activations/layer7_attention_weight_max": 89.31805419921875,
      "activations/layer7_attention_weight_min": -88.01171112060547,
      "activations/layer8_attention_weight_max": 43.92013931274414,
      "activations/layer8_attention_weight_min": -42.315765380859375,
      "activations/layer9_attention_weight_max": 36.42257308959961,
      "activations/layer9_attention_weight_min": -35.811851501464844,
      "epoch": 8.68,
      "learning_rate": 9.497462121212122e-05,
      "loss": 2.8056,
      "step": 149400
    },
    {
      "activations/layer0_attention_weight_max": 14.273737907409668,
      "activations/layer0_attention_weight_min": -11.484273910522461,
      "activations/layer10_attention_weight_max": 34.361976623535156,
      "activations/layer10_attention_weight_min": -34.36845016479492,
      "activations/layer11_attention_weight_max": 34.18747329711914,
      "activations/layer11_attention_weight_min": -32.86781692504883,
      "activations/layer12_attention_weight_max": 23.83695411682129,
      "activations/layer12_attention_weight_min": -25.127599716186523,
      "activations/layer13_attention_weight_max": 39.92003631591797,
      "activations/layer13_attention_weight_min": -35.42851257324219,
      "activations/layer14_attention_weight_max": 47.52595520019531,
      "activations/layer14_attention_weight_min": -38.762115478515625,
      "activations/layer15_attention_weight_max": 38.790733337402344,
      "activations/layer15_attention_weight_min": -32.95069885253906,
      "activations/layer16_attention_weight_max": 34.38280487060547,
      "activations/layer16_attention_weight_min": -28.44247817993164,
      "activations/layer17_attention_weight_max": 36.30137634277344,
      "activations/layer17_attention_weight_min": -24.830238342285156,
      "activations/layer18_attention_weight_max": 30.625398635864258,
      "activations/layer18_attention_weight_min": -19.680532455444336,
      "activations/layer19_attention_weight_max": 38.13393020629883,
      "activations/layer19_attention_weight_min": -31.92482566833496,
      "activations/layer1_attention_weight_max": 17.11133575439453,
      "activations/layer1_attention_weight_min": -12.73961353302002,
      "activations/layer20_attention_weight_max": 32.81140899658203,
      "activations/layer20_attention_weight_min": -24.429004669189453,
      "activations/layer21_attention_weight_max": 30.056142807006836,
      "activations/layer21_attention_weight_min": -23.597055435180664,
      "activations/layer22_attention_weight_max": 44.74104690551758,
      "activations/layer22_attention_weight_min": -27.829601287841797,
      "activations/layer23_attention_weight_max": 37.05765914916992,
      "activations/layer23_attention_weight_min": -23.79756736755371,
      "activations/layer2_attention_weight_max": 28.435791015625,
      "activations/layer2_attention_weight_min": -28.472496032714844,
      "activations/layer3_attention_weight_max": 83.84996032714844,
      "activations/layer3_attention_weight_min": -87.90398406982422,
      "activations/layer4_attention_weight_max": 108.74993133544922,
      "activations/layer4_attention_weight_min": -113.0301742553711,
      "activations/layer5_attention_weight_max": 52.297119140625,
      "activations/layer5_attention_weight_min": -63.98820114135742,
      "activations/layer6_attention_weight_max": 46.01369857788086,
      "activations/layer6_attention_weight_min": -51.15235137939453,
      "activations/layer7_attention_weight_max": 84.2972412109375,
      "activations/layer7_attention_weight_min": -92.64398193359375,
      "activations/layer8_attention_weight_max": 38.149784088134766,
      "activations/layer8_attention_weight_min": -40.78663635253906,
      "activations/layer9_attention_weight_max": 33.95760726928711,
      "activations/layer9_attention_weight_min": -35.148494720458984,
      "epoch": 8.68,
      "learning_rate": 9.49556818181818e-05,
      "loss": 2.8049,
      "step": 149450
    },
    {
      "activations/layer0_attention_weight_max": 15.686917304992676,
      "activations/layer0_attention_weight_min": -12.744874000549316,
      "activations/layer10_attention_weight_max": 33.846839904785156,
      "activations/layer10_attention_weight_min": -34.0336799621582,
      "activations/layer11_attention_weight_max": 34.44188690185547,
      "activations/layer11_attention_weight_min": -33.725791931152344,
      "activations/layer12_attention_weight_max": 30.497777938842773,
      "activations/layer12_attention_weight_min": -23.57400894165039,
      "activations/layer13_attention_weight_max": 41.19020462036133,
      "activations/layer13_attention_weight_min": -33.29935836791992,
      "activations/layer14_attention_weight_max": 56.614097595214844,
      "activations/layer14_attention_weight_min": -43.14809036254883,
      "activations/layer15_attention_weight_max": 38.27829360961914,
      "activations/layer15_attention_weight_min": -29.838167190551758,
      "activations/layer16_attention_weight_max": 37.80890655517578,
      "activations/layer16_attention_weight_min": -27.774398803710938,
      "activations/layer17_attention_weight_max": 34.368812561035156,
      "activations/layer17_attention_weight_min": -24.64223861694336,
      "activations/layer18_attention_weight_max": 32.735923767089844,
      "activations/layer18_attention_weight_min": -22.99036979675293,
      "activations/layer19_attention_weight_max": 45.84884262084961,
      "activations/layer19_attention_weight_min": -29.022581100463867,
      "activations/layer1_attention_weight_max": 16.53016471862793,
      "activations/layer1_attention_weight_min": -13.936211585998535,
      "activations/layer20_attention_weight_max": 33.185489654541016,
      "activations/layer20_attention_weight_min": -24.5004940032959,
      "activations/layer21_attention_weight_max": 34.706504821777344,
      "activations/layer21_attention_weight_min": -22.05727195739746,
      "activations/layer22_attention_weight_max": 49.6833610534668,
      "activations/layer22_attention_weight_min": -26.851104736328125,
      "activations/layer23_attention_weight_max": 41.286869049072266,
      "activations/layer23_attention_weight_min": -22.85536766052246,
      "activations/layer2_attention_weight_max": 32.009620666503906,
      "activations/layer2_attention_weight_min": -30.912765502929688,
      "activations/layer3_attention_weight_max": 93.18497467041016,
      "activations/layer3_attention_weight_min": -97.3419189453125,
      "activations/layer4_attention_weight_max": 122.94937896728516,
      "activations/layer4_attention_weight_min": -123.16167449951172,
      "activations/layer5_attention_weight_max": 56.60057067871094,
      "activations/layer5_attention_weight_min": -63.846397399902344,
      "activations/layer6_attention_weight_max": 49.419654846191406,
      "activations/layer6_attention_weight_min": -53.60731506347656,
      "activations/layer7_attention_weight_max": 91.1624984741211,
      "activations/layer7_attention_weight_min": -94.0213851928711,
      "activations/layer8_attention_weight_max": 42.96471405029297,
      "activations/layer8_attention_weight_min": -43.01707077026367,
      "activations/layer9_attention_weight_max": 35.745452880859375,
      "activations/layer9_attention_weight_min": -35.6356201171875,
      "epoch": 8.69,
      "learning_rate": 9.493674242424242e-05,
      "loss": 2.8019,
      "step": 149500
    },
    {
      "activations/layer0_attention_weight_max": 14.575825691223145,
      "activations/layer0_attention_weight_min": -12.587459564208984,
      "activations/layer10_attention_weight_max": 32.406002044677734,
      "activations/layer10_attention_weight_min": -32.521583557128906,
      "activations/layer11_attention_weight_max": 34.771820068359375,
      "activations/layer11_attention_weight_min": -32.693138122558594,
      "activations/layer12_attention_weight_max": 25.270139694213867,
      "activations/layer12_attention_weight_min": -26.69894027709961,
      "activations/layer13_attention_weight_max": 38.06147766113281,
      "activations/layer13_attention_weight_min": -33.09629440307617,
      "activations/layer14_attention_weight_max": 46.87254333496094,
      "activations/layer14_attention_weight_min": -38.817710876464844,
      "activations/layer15_attention_weight_max": 36.092864990234375,
      "activations/layer15_attention_weight_min": -30.72819709777832,
      "activations/layer16_attention_weight_max": 32.78422164916992,
      "activations/layer16_attention_weight_min": -27.33609390258789,
      "activations/layer17_attention_weight_max": 37.11394119262695,
      "activations/layer17_attention_weight_min": -25.210086822509766,
      "activations/layer18_attention_weight_max": 36.035953521728516,
      "activations/layer18_attention_weight_min": -22.0974063873291,
      "activations/layer19_attention_weight_max": 39.582603454589844,
      "activations/layer19_attention_weight_min": -30.156600952148438,
      "activations/layer1_attention_weight_max": 16.36379623413086,
      "activations/layer1_attention_weight_min": -15.189842224121094,
      "activations/layer20_attention_weight_max": 33.04812240600586,
      "activations/layer20_attention_weight_min": -24.67791175842285,
      "activations/layer21_attention_weight_max": 32.22597122192383,
      "activations/layer21_attention_weight_min": -24.01873779296875,
      "activations/layer22_attention_weight_max": 44.714393615722656,
      "activations/layer22_attention_weight_min": -29.821504592895508,
      "activations/layer23_attention_weight_max": 37.85346984863281,
      "activations/layer23_attention_weight_min": -22.980255126953125,
      "activations/layer2_attention_weight_max": 33.67864990234375,
      "activations/layer2_attention_weight_min": -30.601226806640625,
      "activations/layer3_attention_weight_max": 93.69055938720703,
      "activations/layer3_attention_weight_min": -95.4959716796875,
      "activations/layer4_attention_weight_max": 121.85062408447266,
      "activations/layer4_attention_weight_min": -117.32769012451172,
      "activations/layer5_attention_weight_max": 57.031829833984375,
      "activations/layer5_attention_weight_min": -64.04713439941406,
      "activations/layer6_attention_weight_max": 50.98393249511719,
      "activations/layer6_attention_weight_min": -51.35466384887695,
      "activations/layer7_attention_weight_max": 92.49254608154297,
      "activations/layer7_attention_weight_min": -90.33415985107422,
      "activations/layer8_attention_weight_max": 39.32694625854492,
      "activations/layer8_attention_weight_min": -41.60017395019531,
      "activations/layer9_attention_weight_max": 33.75780487060547,
      "activations/layer9_attention_weight_min": -33.04080581665039,
      "epoch": 8.69,
      "learning_rate": 9.491780303030303e-05,
      "loss": 2.8128,
      "step": 149550
    },
    {
      "activations/layer0_attention_weight_max": 15.65455150604248,
      "activations/layer0_attention_weight_min": -12.91576099395752,
      "activations/layer10_attention_weight_max": 34.41585159301758,
      "activations/layer10_attention_weight_min": -31.647520065307617,
      "activations/layer11_attention_weight_max": 32.3851432800293,
      "activations/layer11_attention_weight_min": -31.351675033569336,
      "activations/layer12_attention_weight_max": 23.322830200195312,
      "activations/layer12_attention_weight_min": -25.58371353149414,
      "activations/layer13_attention_weight_max": 39.87165069580078,
      "activations/layer13_attention_weight_min": -32.89957809448242,
      "activations/layer14_attention_weight_max": 44.64070510864258,
      "activations/layer14_attention_weight_min": -38.89828109741211,
      "activations/layer15_attention_weight_max": 36.300743103027344,
      "activations/layer15_attention_weight_min": -33.11079788208008,
      "activations/layer16_attention_weight_max": 34.22274398803711,
      "activations/layer16_attention_weight_min": -30.255945205688477,
      "activations/layer17_attention_weight_max": 37.37891387939453,
      "activations/layer17_attention_weight_min": -26.472991943359375,
      "activations/layer18_attention_weight_max": 34.388771057128906,
      "activations/layer18_attention_weight_min": -23.186552047729492,
      "activations/layer19_attention_weight_max": 37.7537956237793,
      "activations/layer19_attention_weight_min": -33.88749313354492,
      "activations/layer1_attention_weight_max": 16.334217071533203,
      "activations/layer1_attention_weight_min": -14.43813419342041,
      "activations/layer20_attention_weight_max": 32.15235900878906,
      "activations/layer20_attention_weight_min": -26.06230926513672,
      "activations/layer21_attention_weight_max": 32.56239700317383,
      "activations/layer21_attention_weight_min": -23.580753326416016,
      "activations/layer22_attention_weight_max": 54.91750717163086,
      "activations/layer22_attention_weight_min": -29.52290153503418,
      "activations/layer23_attention_weight_max": 37.72196578979492,
      "activations/layer23_attention_weight_min": -24.122236251831055,
      "activations/layer2_attention_weight_max": 31.90382194519043,
      "activations/layer2_attention_weight_min": -32.58189010620117,
      "activations/layer3_attention_weight_max": 92.60578918457031,
      "activations/layer3_attention_weight_min": -92.86333465576172,
      "activations/layer4_attention_weight_max": 119.22965240478516,
      "activations/layer4_attention_weight_min": -115.6333236694336,
      "activations/layer5_attention_weight_max": 54.286155700683594,
      "activations/layer5_attention_weight_min": -63.6666374206543,
      "activations/layer6_attention_weight_max": 51.791587829589844,
      "activations/layer6_attention_weight_min": -53.35637664794922,
      "activations/layer7_attention_weight_max": 87.31271362304688,
      "activations/layer7_attention_weight_min": -89.31462860107422,
      "activations/layer8_attention_weight_max": 41.6241569519043,
      "activations/layer8_attention_weight_min": -41.647090911865234,
      "activations/layer9_attention_weight_max": 35.72011184692383,
      "activations/layer9_attention_weight_min": -34.123477935791016,
      "epoch": 8.69,
      "learning_rate": 9.489886363636362e-05,
      "loss": 2.7987,
      "step": 149600
    },
    {
      "activations/layer0_attention_weight_max": 16.42730140686035,
      "activations/layer0_attention_weight_min": -12.822049140930176,
      "activations/layer10_attention_weight_max": 34.959232330322266,
      "activations/layer10_attention_weight_min": -32.175987243652344,
      "activations/layer11_attention_weight_max": 35.19219970703125,
      "activations/layer11_attention_weight_min": -32.20011901855469,
      "activations/layer12_attention_weight_max": 33.48224639892578,
      "activations/layer12_attention_weight_min": -25.451133728027344,
      "activations/layer13_attention_weight_max": 40.91050720214844,
      "activations/layer13_attention_weight_min": -34.999549865722656,
      "activations/layer14_attention_weight_max": 49.82755661010742,
      "activations/layer14_attention_weight_min": -39.744361877441406,
      "activations/layer15_attention_weight_max": 42.821720123291016,
      "activations/layer15_attention_weight_min": -31.733346939086914,
      "activations/layer16_attention_weight_max": 35.58419418334961,
      "activations/layer16_attention_weight_min": -29.99443817138672,
      "activations/layer17_attention_weight_max": 35.35948944091797,
      "activations/layer17_attention_weight_min": -26.902311325073242,
      "activations/layer18_attention_weight_max": 29.753559112548828,
      "activations/layer18_attention_weight_min": -22.291336059570312,
      "activations/layer19_attention_weight_max": 38.493656158447266,
      "activations/layer19_attention_weight_min": -32.646575927734375,
      "activations/layer1_attention_weight_max": 17.823688507080078,
      "activations/layer1_attention_weight_min": -15.635533332824707,
      "activations/layer20_attention_weight_max": 34.477073669433594,
      "activations/layer20_attention_weight_min": -25.288230895996094,
      "activations/layer21_attention_weight_max": 30.25685691833496,
      "activations/layer21_attention_weight_min": -23.071807861328125,
      "activations/layer22_attention_weight_max": 49.128238677978516,
      "activations/layer22_attention_weight_min": -29.326454162597656,
      "activations/layer23_attention_weight_max": 37.500892639160156,
      "activations/layer23_attention_weight_min": -22.788557052612305,
      "activations/layer2_attention_weight_max": 34.24742126464844,
      "activations/layer2_attention_weight_min": -34.457298278808594,
      "activations/layer3_attention_weight_max": 93.86894989013672,
      "activations/layer3_attention_weight_min": -98.3404769897461,
      "activations/layer4_attention_weight_max": 119.45936584472656,
      "activations/layer4_attention_weight_min": -116.14772033691406,
      "activations/layer5_attention_weight_max": 56.38434600830078,
      "activations/layer5_attention_weight_min": -62.56538391113281,
      "activations/layer6_attention_weight_max": 48.92328643798828,
      "activations/layer6_attention_weight_min": -51.9636344909668,
      "activations/layer7_attention_weight_max": 92.23541259765625,
      "activations/layer7_attention_weight_min": -89.55784606933594,
      "activations/layer8_attention_weight_max": 41.288578033447266,
      "activations/layer8_attention_weight_min": -40.95285415649414,
      "activations/layer9_attention_weight_max": 36.47150421142578,
      "activations/layer9_attention_weight_min": -34.181976318359375,
      "epoch": 8.7,
      "learning_rate": 9.487992424242424e-05,
      "loss": 2.7987,
      "step": 149650
    },
    {
      "activations/layer0_attention_weight_max": 15.304041862487793,
      "activations/layer0_attention_weight_min": -12.85615062713623,
      "activations/layer10_attention_weight_max": 35.045196533203125,
      "activations/layer10_attention_weight_min": -33.511959075927734,
      "activations/layer11_attention_weight_max": 35.41950225830078,
      "activations/layer11_attention_weight_min": -32.70069885253906,
      "activations/layer12_attention_weight_max": 25.38238525390625,
      "activations/layer12_attention_weight_min": -24.018741607666016,
      "activations/layer13_attention_weight_max": 42.859718322753906,
      "activations/layer13_attention_weight_min": -34.19929122924805,
      "activations/layer14_attention_weight_max": 50.422210693359375,
      "activations/layer14_attention_weight_min": -40.85050582885742,
      "activations/layer15_attention_weight_max": 37.54168701171875,
      "activations/layer15_attention_weight_min": -29.678787231445312,
      "activations/layer16_attention_weight_max": 33.651973724365234,
      "activations/layer16_attention_weight_min": -27.930139541625977,
      "activations/layer17_attention_weight_max": 37.515899658203125,
      "activations/layer17_attention_weight_min": -24.827234268188477,
      "activations/layer18_attention_weight_max": 33.46869659423828,
      "activations/layer18_attention_weight_min": -20.733877182006836,
      "activations/layer19_attention_weight_max": 37.938175201416016,
      "activations/layer19_attention_weight_min": -29.9597110748291,
      "activations/layer1_attention_weight_max": 16.77126693725586,
      "activations/layer1_attention_weight_min": -15.328680992126465,
      "activations/layer20_attention_weight_max": 34.94801712036133,
      "activations/layer20_attention_weight_min": -25.172517776489258,
      "activations/layer21_attention_weight_max": 31.41542625427246,
      "activations/layer21_attention_weight_min": -23.45281982421875,
      "activations/layer22_attention_weight_max": 49.67927551269531,
      "activations/layer22_attention_weight_min": -28.893144607543945,
      "activations/layer23_attention_weight_max": 36.76017379760742,
      "activations/layer23_attention_weight_min": -23.261245727539062,
      "activations/layer2_attention_weight_max": 34.72602844238281,
      "activations/layer2_attention_weight_min": -33.32624816894531,
      "activations/layer3_attention_weight_max": 102.4610824584961,
      "activations/layer3_attention_weight_min": -101.62590026855469,
      "activations/layer4_attention_weight_max": 125.8674087524414,
      "activations/layer4_attention_weight_min": -126.27498626708984,
      "activations/layer5_attention_weight_max": 56.72404479980469,
      "activations/layer5_attention_weight_min": -64.82502746582031,
      "activations/layer6_attention_weight_max": 49.822933197021484,
      "activations/layer6_attention_weight_min": -52.59962844848633,
      "activations/layer7_attention_weight_max": 95.12397766113281,
      "activations/layer7_attention_weight_min": -94.09732055664062,
      "activations/layer8_attention_weight_max": 44.333221435546875,
      "activations/layer8_attention_weight_min": -43.32448959350586,
      "activations/layer9_attention_weight_max": 36.22239303588867,
      "activations/layer9_attention_weight_min": -35.1004753112793,
      "epoch": 8.7,
      "learning_rate": 9.486098484848484e-05,
      "loss": 2.7933,
      "step": 149700
    },
    {
      "activations/layer0_attention_weight_max": 15.634723663330078,
      "activations/layer0_attention_weight_min": -11.793027877807617,
      "activations/layer10_attention_weight_max": 30.388607025146484,
      "activations/layer10_attention_weight_min": -31.361629486083984,
      "activations/layer11_attention_weight_max": 33.769325256347656,
      "activations/layer11_attention_weight_min": -30.572494506835938,
      "activations/layer12_attention_weight_max": 24.36390495300293,
      "activations/layer12_attention_weight_min": -24.70865249633789,
      "activations/layer13_attention_weight_max": 42.46684646606445,
      "activations/layer13_attention_weight_min": -32.32745361328125,
      "activations/layer14_attention_weight_max": 55.16653823852539,
      "activations/layer14_attention_weight_min": -38.910255432128906,
      "activations/layer15_attention_weight_max": 37.802310943603516,
      "activations/layer15_attention_weight_min": -30.289417266845703,
      "activations/layer16_attention_weight_max": 31.913671493530273,
      "activations/layer16_attention_weight_min": -27.898279190063477,
      "activations/layer17_attention_weight_max": 40.079166412353516,
      "activations/layer17_attention_weight_min": -25.207460403442383,
      "activations/layer18_attention_weight_max": 39.390567779541016,
      "activations/layer18_attention_weight_min": -20.942970275878906,
      "activations/layer19_attention_weight_max": 41.28218460083008,
      "activations/layer19_attention_weight_min": -30.41688346862793,
      "activations/layer1_attention_weight_max": 17.071969985961914,
      "activations/layer1_attention_weight_min": -14.094454765319824,
      "activations/layer20_attention_weight_max": 33.24458312988281,
      "activations/layer20_attention_weight_min": -24.967693328857422,
      "activations/layer21_attention_weight_max": 32.97190856933594,
      "activations/layer21_attention_weight_min": -23.960948944091797,
      "activations/layer22_attention_weight_max": 45.97321319580078,
      "activations/layer22_attention_weight_min": -29.052820205688477,
      "activations/layer23_attention_weight_max": 41.48259353637695,
      "activations/layer23_attention_weight_min": -24.698972702026367,
      "activations/layer2_attention_weight_max": 32.32105255126953,
      "activations/layer2_attention_weight_min": -30.523113250732422,
      "activations/layer3_attention_weight_max": 95.37718200683594,
      "activations/layer3_attention_weight_min": -92.04974365234375,
      "activations/layer4_attention_weight_max": 122.21916961669922,
      "activations/layer4_attention_weight_min": -114.91368865966797,
      "activations/layer5_attention_weight_max": 55.260459899902344,
      "activations/layer5_attention_weight_min": -60.51046371459961,
      "activations/layer6_attention_weight_max": 49.51206970214844,
      "activations/layer6_attention_weight_min": -53.256752014160156,
      "activations/layer7_attention_weight_max": 88.5833511352539,
      "activations/layer7_attention_weight_min": -90.22278594970703,
      "activations/layer8_attention_weight_max": 40.1574821472168,
      "activations/layer8_attention_weight_min": -41.83650588989258,
      "activations/layer9_attention_weight_max": 32.86909103393555,
      "activations/layer9_attention_weight_min": -32.37587356567383,
      "epoch": 8.7,
      "learning_rate": 9.484204545454544e-05,
      "loss": 2.8001,
      "step": 149750
    },
    {
      "activations/layer0_attention_weight_max": 16.107877731323242,
      "activations/layer0_attention_weight_min": -12.556136131286621,
      "activations/layer10_attention_weight_max": 32.37361526489258,
      "activations/layer10_attention_weight_min": -34.189674377441406,
      "activations/layer11_attention_weight_max": 32.553070068359375,
      "activations/layer11_attention_weight_min": -31.958236694335938,
      "activations/layer12_attention_weight_max": 22.23284912109375,
      "activations/layer12_attention_weight_min": -24.23955535888672,
      "activations/layer13_attention_weight_max": 41.204185485839844,
      "activations/layer13_attention_weight_min": -36.234188079833984,
      "activations/layer14_attention_weight_max": 48.23872756958008,
      "activations/layer14_attention_weight_min": -41.89418411254883,
      "activations/layer15_attention_weight_max": 33.495235443115234,
      "activations/layer15_attention_weight_min": -30.181676864624023,
      "activations/layer16_attention_weight_max": 30.803104400634766,
      "activations/layer16_attention_weight_min": -27.79054832458496,
      "activations/layer17_attention_weight_max": 34.51922607421875,
      "activations/layer17_attention_weight_min": -25.537809371948242,
      "activations/layer18_attention_weight_max": 30.530813217163086,
      "activations/layer18_attention_weight_min": -21.882917404174805,
      "activations/layer19_attention_weight_max": 34.711116790771484,
      "activations/layer19_attention_weight_min": -31.152368545532227,
      "activations/layer1_attention_weight_max": 16.426904678344727,
      "activations/layer1_attention_weight_min": -15.600838661193848,
      "activations/layer20_attention_weight_max": 31.41795539855957,
      "activations/layer20_attention_weight_min": -23.945226669311523,
      "activations/layer21_attention_weight_max": 32.30099868774414,
      "activations/layer21_attention_weight_min": -23.06056022644043,
      "activations/layer22_attention_weight_max": 46.8606071472168,
      "activations/layer22_attention_weight_min": -30.23858642578125,
      "activations/layer23_attention_weight_max": 38.347991943359375,
      "activations/layer23_attention_weight_min": -24.464431762695312,
      "activations/layer2_attention_weight_max": 32.00697708129883,
      "activations/layer2_attention_weight_min": -33.63111877441406,
      "activations/layer3_attention_weight_max": 92.3246841430664,
      "activations/layer3_attention_weight_min": -93.3536376953125,
      "activations/layer4_attention_weight_max": 118.9189224243164,
      "activations/layer4_attention_weight_min": -117.23321533203125,
      "activations/layer5_attention_weight_max": 54.64695739746094,
      "activations/layer5_attention_weight_min": -62.388221740722656,
      "activations/layer6_attention_weight_max": 49.37617492675781,
      "activations/layer6_attention_weight_min": -53.60732650756836,
      "activations/layer7_attention_weight_max": 92.80500793457031,
      "activations/layer7_attention_weight_min": -97.36590576171875,
      "activations/layer8_attention_weight_max": 40.93622589111328,
      "activations/layer8_attention_weight_min": -41.326416015625,
      "activations/layer9_attention_weight_max": 35.619102478027344,
      "activations/layer9_attention_weight_min": -33.420631408691406,
      "epoch": 8.7,
      "learning_rate": 9.482310606060605e-05,
      "loss": 2.8165,
      "step": 149800
    },
    {
      "activations/layer0_attention_weight_max": 16.010202407836914,
      "activations/layer0_attention_weight_min": -12.829480171203613,
      "activations/layer10_attention_weight_max": 31.88425064086914,
      "activations/layer10_attention_weight_min": -32.247657775878906,
      "activations/layer11_attention_weight_max": 29.797895431518555,
      "activations/layer11_attention_weight_min": -31.29000473022461,
      "activations/layer12_attention_weight_max": 23.76015853881836,
      "activations/layer12_attention_weight_min": -24.66638946533203,
      "activations/layer13_attention_weight_max": 38.62715530395508,
      "activations/layer13_attention_weight_min": -34.26750183105469,
      "activations/layer14_attention_weight_max": 50.259822845458984,
      "activations/layer14_attention_weight_min": -38.280303955078125,
      "activations/layer15_attention_weight_max": 35.654449462890625,
      "activations/layer15_attention_weight_min": -32.708675384521484,
      "activations/layer16_attention_weight_max": 40.30007553100586,
      "activations/layer16_attention_weight_min": -31.660322189331055,
      "activations/layer17_attention_weight_max": 32.43971252441406,
      "activations/layer17_attention_weight_min": -26.774127960205078,
      "activations/layer18_attention_weight_max": 30.651657104492188,
      "activations/layer18_attention_weight_min": -22.238994598388672,
      "activations/layer19_attention_weight_max": 50.3758659362793,
      "activations/layer19_attention_weight_min": -30.9886417388916,
      "activations/layer1_attention_weight_max": 16.94078826904297,
      "activations/layer1_attention_weight_min": -14.266703605651855,
      "activations/layer20_attention_weight_max": 30.099754333496094,
      "activations/layer20_attention_weight_min": -24.26482391357422,
      "activations/layer21_attention_weight_max": 31.97847557067871,
      "activations/layer21_attention_weight_min": -23.850725173950195,
      "activations/layer22_attention_weight_max": 47.871768951416016,
      "activations/layer22_attention_weight_min": -28.055986404418945,
      "activations/layer23_attention_weight_max": 39.85401916503906,
      "activations/layer23_attention_weight_min": -22.763389587402344,
      "activations/layer2_attention_weight_max": 34.7843132019043,
      "activations/layer2_attention_weight_min": -31.61581039428711,
      "activations/layer3_attention_weight_max": 92.41973876953125,
      "activations/layer3_attention_weight_min": -90.97432708740234,
      "activations/layer4_attention_weight_max": 123.53023529052734,
      "activations/layer4_attention_weight_min": -114.44825744628906,
      "activations/layer5_attention_weight_max": 53.83406066894531,
      "activations/layer5_attention_weight_min": -62.27112579345703,
      "activations/layer6_attention_weight_max": 48.149410247802734,
      "activations/layer6_attention_weight_min": -51.96961975097656,
      "activations/layer7_attention_weight_max": 87.48174285888672,
      "activations/layer7_attention_weight_min": -85.3935546875,
      "activations/layer8_attention_weight_max": 38.30385971069336,
      "activations/layer8_attention_weight_min": -39.15279769897461,
      "activations/layer9_attention_weight_max": 31.99515724182129,
      "activations/layer9_attention_weight_min": -33.00367736816406,
      "epoch": 8.71,
      "learning_rate": 9.480416666666666e-05,
      "loss": 2.8016,
      "step": 149850
    },
    {
      "activations/layer0_attention_weight_max": 14.790571212768555,
      "activations/layer0_attention_weight_min": -13.221013069152832,
      "activations/layer10_attention_weight_max": 31.40395736694336,
      "activations/layer10_attention_weight_min": -31.46554183959961,
      "activations/layer11_attention_weight_max": 33.475486755371094,
      "activations/layer11_attention_weight_min": -32.06391143798828,
      "activations/layer12_attention_weight_max": 22.76017189025879,
      "activations/layer12_attention_weight_min": -23.796533584594727,
      "activations/layer13_attention_weight_max": 41.366756439208984,
      "activations/layer13_attention_weight_min": -32.56633758544922,
      "activations/layer14_attention_weight_max": 48.040767669677734,
      "activations/layer14_attention_weight_min": -40.060237884521484,
      "activations/layer15_attention_weight_max": 39.92906951904297,
      "activations/layer15_attention_weight_min": -30.801544189453125,
      "activations/layer16_attention_weight_max": 30.350934982299805,
      "activations/layer16_attention_weight_min": -26.335975646972656,
      "activations/layer17_attention_weight_max": 32.610843658447266,
      "activations/layer17_attention_weight_min": -24.94849967956543,
      "activations/layer18_attention_weight_max": 32.688743591308594,
      "activations/layer18_attention_weight_min": -20.770370483398438,
      "activations/layer19_attention_weight_max": 35.059303283691406,
      "activations/layer19_attention_weight_min": -29.23333740234375,
      "activations/layer1_attention_weight_max": 16.750930786132812,
      "activations/layer1_attention_weight_min": -14.354997634887695,
      "activations/layer20_attention_weight_max": 29.506473541259766,
      "activations/layer20_attention_weight_min": -25.4932804107666,
      "activations/layer21_attention_weight_max": 29.133800506591797,
      "activations/layer21_attention_weight_min": -23.182092666625977,
      "activations/layer22_attention_weight_max": 43.064266204833984,
      "activations/layer22_attention_weight_min": -29.30276107788086,
      "activations/layer23_attention_weight_max": 37.13295364379883,
      "activations/layer23_attention_weight_min": -24.49422836303711,
      "activations/layer2_attention_weight_max": 33.48781204223633,
      "activations/layer2_attention_weight_min": -33.081016540527344,
      "activations/layer3_attention_weight_max": 92.45625305175781,
      "activations/layer3_attention_weight_min": -92.59683990478516,
      "activations/layer4_attention_weight_max": 117.46968078613281,
      "activations/layer4_attention_weight_min": -115.96076965332031,
      "activations/layer5_attention_weight_max": 54.839378356933594,
      "activations/layer5_attention_weight_min": -61.15659713745117,
      "activations/layer6_attention_weight_max": 51.46170425415039,
      "activations/layer6_attention_weight_min": -53.30952835083008,
      "activations/layer7_attention_weight_max": 96.4239273071289,
      "activations/layer7_attention_weight_min": -96.69206237792969,
      "activations/layer8_attention_weight_max": 41.74800491333008,
      "activations/layer8_attention_weight_min": -43.50756072998047,
      "activations/layer9_attention_weight_max": 35.039710998535156,
      "activations/layer9_attention_weight_min": -35.120460510253906,
      "epoch": 8.71,
      "learning_rate": 9.478522727272727e-05,
      "loss": 2.8052,
      "step": 149900
    },
    {
      "activations/layer0_attention_weight_max": 15.857375144958496,
      "activations/layer0_attention_weight_min": -12.552750587463379,
      "activations/layer10_attention_weight_max": 35.73067092895508,
      "activations/layer10_attention_weight_min": -35.082489013671875,
      "activations/layer11_attention_weight_max": 35.63566207885742,
      "activations/layer11_attention_weight_min": -34.39244842529297,
      "activations/layer12_attention_weight_max": 29.190351486206055,
      "activations/layer12_attention_weight_min": -26.07762336730957,
      "activations/layer13_attention_weight_max": 48.41761016845703,
      "activations/layer13_attention_weight_min": -37.40830993652344,
      "activations/layer14_attention_weight_max": 57.68474197387695,
      "activations/layer14_attention_weight_min": -45.74119186401367,
      "activations/layer15_attention_weight_max": 40.03948211669922,
      "activations/layer15_attention_weight_min": -29.537765502929688,
      "activations/layer16_attention_weight_max": 35.441890716552734,
      "activations/layer16_attention_weight_min": -28.28699493408203,
      "activations/layer17_attention_weight_max": 36.27131271362305,
      "activations/layer17_attention_weight_min": -27.910709381103516,
      "activations/layer18_attention_weight_max": 30.74151039123535,
      "activations/layer18_attention_weight_min": -20.939538955688477,
      "activations/layer19_attention_weight_max": 38.031368255615234,
      "activations/layer19_attention_weight_min": -28.952482223510742,
      "activations/layer1_attention_weight_max": 17.23651695251465,
      "activations/layer1_attention_weight_min": -15.168807029724121,
      "activations/layer20_attention_weight_max": 31.15847396850586,
      "activations/layer20_attention_weight_min": -23.583850860595703,
      "activations/layer21_attention_weight_max": 31.453704833984375,
      "activations/layer21_attention_weight_min": -22.26617431640625,
      "activations/layer22_attention_weight_max": 50.72310256958008,
      "activations/layer22_attention_weight_min": -32.2104377746582,
      "activations/layer23_attention_weight_max": 38.781131744384766,
      "activations/layer23_attention_weight_min": -23.776248931884766,
      "activations/layer2_attention_weight_max": 32.65412139892578,
      "activations/layer2_attention_weight_min": -33.28767395019531,
      "activations/layer3_attention_weight_max": 95.349853515625,
      "activations/layer3_attention_weight_min": -96.57601928710938,
      "activations/layer4_attention_weight_max": 123.47950744628906,
      "activations/layer4_attention_weight_min": -118.77816009521484,
      "activations/layer5_attention_weight_max": 55.33420944213867,
      "activations/layer5_attention_weight_min": -61.32817840576172,
      "activations/layer6_attention_weight_max": 50.09138107299805,
      "activations/layer6_attention_weight_min": -55.12422180175781,
      "activations/layer7_attention_weight_max": 97.80931091308594,
      "activations/layer7_attention_weight_min": -99.79489135742188,
      "activations/layer8_attention_weight_max": 45.310523986816406,
      "activations/layer8_attention_weight_min": -46.33546829223633,
      "activations/layer9_attention_weight_max": 38.686058044433594,
      "activations/layer9_attention_weight_min": -39.30099105834961,
      "epoch": 8.71,
      "learning_rate": 9.476628787878787e-05,
      "loss": 2.7922,
      "step": 149950
    },
    {
      "activations/layer0_attention_weight_max": 16.317506790161133,
      "activations/layer0_attention_weight_min": -12.481313705444336,
      "activations/layer10_attention_weight_max": 33.73207092285156,
      "activations/layer10_attention_weight_min": -31.700773239135742,
      "activations/layer11_attention_weight_max": 34.657371520996094,
      "activations/layer11_attention_weight_min": -31.59564208984375,
      "activations/layer12_attention_weight_max": 24.599313735961914,
      "activations/layer12_attention_weight_min": -25.63368034362793,
      "activations/layer13_attention_weight_max": 40.90971755981445,
      "activations/layer13_attention_weight_min": -34.117027282714844,
      "activations/layer14_attention_weight_max": 46.50360107421875,
      "activations/layer14_attention_weight_min": -38.78544998168945,
      "activations/layer15_attention_weight_max": 39.243804931640625,
      "activations/layer15_attention_weight_min": -31.549589157104492,
      "activations/layer16_attention_weight_max": 35.27815246582031,
      "activations/layer16_attention_weight_min": -27.706092834472656,
      "activations/layer17_attention_weight_max": 33.103424072265625,
      "activations/layer17_attention_weight_min": -24.961753845214844,
      "activations/layer18_attention_weight_max": 35.37403106689453,
      "activations/layer18_attention_weight_min": -23.425277709960938,
      "activations/layer19_attention_weight_max": 38.4716911315918,
      "activations/layer19_attention_weight_min": -29.803415298461914,
      "activations/layer1_attention_weight_max": 16.838233947753906,
      "activations/layer1_attention_weight_min": -15.106941223144531,
      "activations/layer20_attention_weight_max": 31.74392318725586,
      "activations/layer20_attention_weight_min": -24.152408599853516,
      "activations/layer21_attention_weight_max": 30.43658447265625,
      "activations/layer21_attention_weight_min": -22.795663833618164,
      "activations/layer22_attention_weight_max": 45.955440521240234,
      "activations/layer22_attention_weight_min": -29.21367073059082,
      "activations/layer23_attention_weight_max": 36.719852447509766,
      "activations/layer23_attention_weight_min": -25.241966247558594,
      "activations/layer2_attention_weight_max": 31.433147430419922,
      "activations/layer2_attention_weight_min": -31.545211791992188,
      "activations/layer3_attention_weight_max": 92.61052703857422,
      "activations/layer3_attention_weight_min": -93.59103393554688,
      "activations/layer4_attention_weight_max": 121.86444091796875,
      "activations/layer4_attention_weight_min": -116.72997283935547,
      "activations/layer5_attention_weight_max": 56.76245880126953,
      "activations/layer5_attention_weight_min": -61.65637969970703,
      "activations/layer6_attention_weight_max": 48.40121841430664,
      "activations/layer6_attention_weight_min": -53.6722526550293,
      "activations/layer7_attention_weight_max": 87.8413314819336,
      "activations/layer7_attention_weight_min": -92.48760986328125,
      "activations/layer8_attention_weight_max": 41.94224548339844,
      "activations/layer8_attention_weight_min": -41.8035888671875,
      "activations/layer9_attention_weight_max": 37.98524856567383,
      "activations/layer9_attention_weight_min": -34.21696090698242,
      "epoch": 8.72,
      "learning_rate": 9.474734848484847e-05,
      "loss": 2.7941,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.4836,
      "eval_samples_per_second": 506.154,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.4836,
      "eval_openwebtext_samples_per_second": 506.154,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 2.0323,
      "eval_wikitext_samples_per_second": 224.376,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.6426,
      "eval_lambada_samples_per_second": 504.949,
      "step": 150000
    },
    {
      "activations/layer0_attention_weight_max": 16.50726318359375,
      "activations/layer0_attention_weight_min": -13.538031578063965,
      "activations/layer10_attention_weight_max": 32.5126953125,
      "activations/layer10_attention_weight_min": -31.262454986572266,
      "activations/layer11_attention_weight_max": 30.924808502197266,
      "activations/layer11_attention_weight_min": -31.35934066772461,
      "activations/layer12_attention_weight_max": 22.4483642578125,
      "activations/layer12_attention_weight_min": -24.259824752807617,
      "activations/layer13_attention_weight_max": 37.30879592895508,
      "activations/layer13_attention_weight_min": -34.152496337890625,
      "activations/layer14_attention_weight_max": 45.02197265625,
      "activations/layer14_attention_weight_min": -37.544307708740234,
      "activations/layer15_attention_weight_max": 36.212894439697266,
      "activations/layer15_attention_weight_min": -32.963985443115234,
      "activations/layer16_attention_weight_max": 33.132843017578125,
      "activations/layer16_attention_weight_min": -28.036685943603516,
      "activations/layer17_attention_weight_max": 33.754154205322266,
      "activations/layer17_attention_weight_min": -25.723804473876953,
      "activations/layer18_attention_weight_max": 33.22258758544922,
      "activations/layer18_attention_weight_min": -25.099851608276367,
      "activations/layer19_attention_weight_max": 39.496482849121094,
      "activations/layer19_attention_weight_min": -30.077747344970703,
      "activations/layer1_attention_weight_max": 16.614032745361328,
      "activations/layer1_attention_weight_min": -14.435690879821777,
      "activations/layer20_attention_weight_max": 31.547250747680664,
      "activations/layer20_attention_weight_min": -25.556583404541016,
      "activations/layer21_attention_weight_max": 32.88630676269531,
      "activations/layer21_attention_weight_min": -23.02646827697754,
      "activations/layer22_attention_weight_max": 45.22935485839844,
      "activations/layer22_attention_weight_min": -28.83863067626953,
      "activations/layer23_attention_weight_max": 37.05242919921875,
      "activations/layer23_attention_weight_min": -22.14377212524414,
      "activations/layer2_attention_weight_max": 33.10032653808594,
      "activations/layer2_attention_weight_min": -31.774303436279297,
      "activations/layer3_attention_weight_max": 90.17182159423828,
      "activations/layer3_attention_weight_min": -92.82473754882812,
      "activations/layer4_attention_weight_max": 116.16844940185547,
      "activations/layer4_attention_weight_min": -119.96392059326172,
      "activations/layer5_attention_weight_max": 53.78657531738281,
      "activations/layer5_attention_weight_min": -60.56444549560547,
      "activations/layer6_attention_weight_max": 45.24528884887695,
      "activations/layer6_attention_weight_min": -51.75086975097656,
      "activations/layer7_attention_weight_max": 90.09579467773438,
      "activations/layer7_attention_weight_min": -89.76158905029297,
      "activations/layer8_attention_weight_max": 38.54740524291992,
      "activations/layer8_attention_weight_min": -40.6247444152832,
      "activations/layer9_attention_weight_max": 33.90303421020508,
      "activations/layer9_attention_weight_min": -32.86872863769531,
      "epoch": 8.72,
      "learning_rate": 9.472840909090909e-05,
      "loss": 2.8088,
      "step": 150050
    },
    {
      "activations/layer0_attention_weight_max": 15.228549003601074,
      "activations/layer0_attention_weight_min": -12.874710083007812,
      "activations/layer10_attention_weight_max": 31.44146728515625,
      "activations/layer10_attention_weight_min": -34.96290588378906,
      "activations/layer11_attention_weight_max": 32.93622589111328,
      "activations/layer11_attention_weight_min": -31.5445613861084,
      "activations/layer12_attention_weight_max": 38.342506408691406,
      "activations/layer12_attention_weight_min": -27.07197380065918,
      "activations/layer13_attention_weight_max": 38.727142333984375,
      "activations/layer13_attention_weight_min": -32.09156799316406,
      "activations/layer14_attention_weight_max": 48.10249328613281,
      "activations/layer14_attention_weight_min": -39.55949783325195,
      "activations/layer15_attention_weight_max": 34.28834915161133,
      "activations/layer15_attention_weight_min": -30.424381256103516,
      "activations/layer16_attention_weight_max": 31.42668914794922,
      "activations/layer16_attention_weight_min": -27.41996955871582,
      "activations/layer17_attention_weight_max": 32.684444427490234,
      "activations/layer17_attention_weight_min": -26.0213565826416,
      "activations/layer18_attention_weight_max": 31.09819221496582,
      "activations/layer18_attention_weight_min": -23.031721115112305,
      "activations/layer19_attention_weight_max": 37.7894172668457,
      "activations/layer19_attention_weight_min": -30.7049560546875,
      "activations/layer1_attention_weight_max": 17.216041564941406,
      "activations/layer1_attention_weight_min": -13.913619041442871,
      "activations/layer20_attention_weight_max": 30.006481170654297,
      "activations/layer20_attention_weight_min": -25.234769821166992,
      "activations/layer21_attention_weight_max": 28.53664779663086,
      "activations/layer21_attention_weight_min": -23.455331802368164,
      "activations/layer22_attention_weight_max": 45.169837951660156,
      "activations/layer22_attention_weight_min": -27.43216323852539,
      "activations/layer23_attention_weight_max": 33.06536102294922,
      "activations/layer23_attention_weight_min": -21.785884857177734,
      "activations/layer2_attention_weight_max": 31.90908432006836,
      "activations/layer2_attention_weight_min": -30.45389175415039,
      "activations/layer3_attention_weight_max": 94.33615112304688,
      "activations/layer3_attention_weight_min": -93.74224090576172,
      "activations/layer4_attention_weight_max": 117.2153091430664,
      "activations/layer4_attention_weight_min": -116.1049575805664,
      "activations/layer5_attention_weight_max": 54.986351013183594,
      "activations/layer5_attention_weight_min": -62.78546142578125,
      "activations/layer6_attention_weight_max": 46.286521911621094,
      "activations/layer6_attention_weight_min": -50.435768127441406,
      "activations/layer7_attention_weight_max": 89.84821319580078,
      "activations/layer7_attention_weight_min": -86.8642578125,
      "activations/layer8_attention_weight_max": 39.73280715942383,
      "activations/layer8_attention_weight_min": -41.39345932006836,
      "activations/layer9_attention_weight_max": 33.917823791503906,
      "activations/layer9_attention_weight_min": -35.58888244628906,
      "epoch": 8.72,
      "learning_rate": 9.470946969696968e-05,
      "loss": 2.8085,
      "step": 150100
    },
    {
      "activations/layer0_attention_weight_max": 16.27750587463379,
      "activations/layer0_attention_weight_min": -13.272363662719727,
      "activations/layer10_attention_weight_max": 34.37847900390625,
      "activations/layer10_attention_weight_min": -34.32475280761719,
      "activations/layer11_attention_weight_max": 36.459922790527344,
      "activations/layer11_attention_weight_min": -32.255313873291016,
      "activations/layer12_attention_weight_max": 25.202661514282227,
      "activations/layer12_attention_weight_min": -24.98676109313965,
      "activations/layer13_attention_weight_max": 43.304420471191406,
      "activations/layer13_attention_weight_min": -33.06770324707031,
      "activations/layer14_attention_weight_max": 52.67433547973633,
      "activations/layer14_attention_weight_min": -39.09285354614258,
      "activations/layer15_attention_weight_max": 42.260250091552734,
      "activations/layer15_attention_weight_min": -31.633533477783203,
      "activations/layer16_attention_weight_max": 34.140071868896484,
      "activations/layer16_attention_weight_min": -27.24077796936035,
      "activations/layer17_attention_weight_max": 37.61013412475586,
      "activations/layer17_attention_weight_min": -25.334489822387695,
      "activations/layer18_attention_weight_max": 30.278972625732422,
      "activations/layer18_attention_weight_min": -21.81888771057129,
      "activations/layer19_attention_weight_max": 38.415096282958984,
      "activations/layer19_attention_weight_min": -31.123865127563477,
      "activations/layer1_attention_weight_max": 16.78225326538086,
      "activations/layer1_attention_weight_min": -14.613425254821777,
      "activations/layer20_attention_weight_max": 30.736860275268555,
      "activations/layer20_attention_weight_min": -23.065290451049805,
      "activations/layer21_attention_weight_max": 29.835342407226562,
      "activations/layer21_attention_weight_min": -21.372238159179688,
      "activations/layer22_attention_weight_max": 48.46311569213867,
      "activations/layer22_attention_weight_min": -28.698078155517578,
      "activations/layer23_attention_weight_max": 36.47980499267578,
      "activations/layer23_attention_weight_min": -24.730558395385742,
      "activations/layer2_attention_weight_max": 31.701183319091797,
      "activations/layer2_attention_weight_min": -31.903675079345703,
      "activations/layer3_attention_weight_max": 94.60826873779297,
      "activations/layer3_attention_weight_min": -95.26377868652344,
      "activations/layer4_attention_weight_max": 119.7159423828125,
      "activations/layer4_attention_weight_min": -121.91385650634766,
      "activations/layer5_attention_weight_max": 55.596641540527344,
      "activations/layer5_attention_weight_min": -66.09049224853516,
      "activations/layer6_attention_weight_max": 49.63365936279297,
      "activations/layer6_attention_weight_min": -55.647727966308594,
      "activations/layer7_attention_weight_max": 94.75419616699219,
      "activations/layer7_attention_weight_min": -99.69754791259766,
      "activations/layer8_attention_weight_max": 39.5158576965332,
      "activations/layer8_attention_weight_min": -46.70451354980469,
      "activations/layer9_attention_weight_max": 36.71134567260742,
      "activations/layer9_attention_weight_min": -36.70078659057617,
      "epoch": 8.72,
      "learning_rate": 9.46909090909091e-05,
      "loss": 2.8236,
      "step": 150150
    },
    {
      "activations/layer0_attention_weight_max": 15.690143585205078,
      "activations/layer0_attention_weight_min": -11.434349060058594,
      "activations/layer10_attention_weight_max": 30.789451599121094,
      "activations/layer10_attention_weight_min": -30.9776668548584,
      "activations/layer11_attention_weight_max": 31.34316635131836,
      "activations/layer11_attention_weight_min": -29.308889389038086,
      "activations/layer12_attention_weight_max": 24.60943031311035,
      "activations/layer12_attention_weight_min": -24.577285766601562,
      "activations/layer13_attention_weight_max": 42.62818908691406,
      "activations/layer13_attention_weight_min": -31.97856330871582,
      "activations/layer14_attention_weight_max": 53.499481201171875,
      "activations/layer14_attention_weight_min": -40.69978332519531,
      "activations/layer15_attention_weight_max": 39.36682891845703,
      "activations/layer15_attention_weight_min": -29.537500381469727,
      "activations/layer16_attention_weight_max": 32.86911392211914,
      "activations/layer16_attention_weight_min": -27.034072875976562,
      "activations/layer17_attention_weight_max": 35.61928176879883,
      "activations/layer17_attention_weight_min": -26.51219367980957,
      "activations/layer18_attention_weight_max": 29.489694595336914,
      "activations/layer18_attention_weight_min": -20.696563720703125,
      "activations/layer19_attention_weight_max": 39.14561080932617,
      "activations/layer19_attention_weight_min": -28.654224395751953,
      "activations/layer1_attention_weight_max": 16.772504806518555,
      "activations/layer1_attention_weight_min": -15.881927490234375,
      "activations/layer20_attention_weight_max": 30.16203498840332,
      "activations/layer20_attention_weight_min": -22.87870979309082,
      "activations/layer21_attention_weight_max": 28.710988998413086,
      "activations/layer21_attention_weight_min": -21.785213470458984,
      "activations/layer22_attention_weight_max": 46.49241638183594,
      "activations/layer22_attention_weight_min": -30.27091407775879,
      "activations/layer23_attention_weight_max": 39.894927978515625,
      "activations/layer23_attention_weight_min": -23.11904525756836,
      "activations/layer2_attention_weight_max": 33.24404525756836,
      "activations/layer2_attention_weight_min": -32.55525588989258,
      "activations/layer3_attention_weight_max": 95.79186248779297,
      "activations/layer3_attention_weight_min": -93.75497436523438,
      "activations/layer4_attention_weight_max": 118.30921936035156,
      "activations/layer4_attention_weight_min": -115.29534912109375,
      "activations/layer5_attention_weight_max": 54.28858184814453,
      "activations/layer5_attention_weight_min": -63.1072998046875,
      "activations/layer6_attention_weight_max": 50.38530349731445,
      "activations/layer6_attention_weight_min": -51.85381317138672,
      "activations/layer7_attention_weight_max": 90.06419372558594,
      "activations/layer7_attention_weight_min": -88.32349395751953,
      "activations/layer8_attention_weight_max": 39.89695358276367,
      "activations/layer8_attention_weight_min": -39.626617431640625,
      "activations/layer9_attention_weight_max": 33.750553131103516,
      "activations/layer9_attention_weight_min": -31.674917221069336,
      "epoch": 8.73,
      "learning_rate": 9.467196969696968e-05,
      "loss": 2.8161,
      "step": 150200
    },
    {
      "activations/layer0_attention_weight_max": 15.684454917907715,
      "activations/layer0_attention_weight_min": -12.985553741455078,
      "activations/layer10_attention_weight_max": 33.13397979736328,
      "activations/layer10_attention_weight_min": -31.09782600402832,
      "activations/layer11_attention_weight_max": 31.65291404724121,
      "activations/layer11_attention_weight_min": -30.943092346191406,
      "activations/layer12_attention_weight_max": 24.365127563476562,
      "activations/layer12_attention_weight_min": -26.561235427856445,
      "activations/layer13_attention_weight_max": 39.6522102355957,
      "activations/layer13_attention_weight_min": -33.769962310791016,
      "activations/layer14_attention_weight_max": 46.366878509521484,
      "activations/layer14_attention_weight_min": -36.926456451416016,
      "activations/layer15_attention_weight_max": 37.975250244140625,
      "activations/layer15_attention_weight_min": -30.523143768310547,
      "activations/layer16_attention_weight_max": 35.06898880004883,
      "activations/layer16_attention_weight_min": -26.310251235961914,
      "activations/layer17_attention_weight_max": 33.40703582763672,
      "activations/layer17_attention_weight_min": -25.45756721496582,
      "activations/layer18_attention_weight_max": 32.22858428955078,
      "activations/layer18_attention_weight_min": -21.421754837036133,
      "activations/layer19_attention_weight_max": 33.29429244995117,
      "activations/layer19_attention_weight_min": -29.83002281188965,
      "activations/layer1_attention_weight_max": 17.203710556030273,
      "activations/layer1_attention_weight_min": -15.93245792388916,
      "activations/layer20_attention_weight_max": 28.041967391967773,
      "activations/layer20_attention_weight_min": -23.772483825683594,
      "activations/layer21_attention_weight_max": 28.699026107788086,
      "activations/layer21_attention_weight_min": -23.506576538085938,
      "activations/layer22_attention_weight_max": 43.05620574951172,
      "activations/layer22_attention_weight_min": -30.37311553955078,
      "activations/layer23_attention_weight_max": 34.26938247680664,
      "activations/layer23_attention_weight_min": -24.830684661865234,
      "activations/layer2_attention_weight_max": 32.05908203125,
      "activations/layer2_attention_weight_min": -30.47591209411621,
      "activations/layer3_attention_weight_max": 93.13251495361328,
      "activations/layer3_attention_weight_min": -94.67833709716797,
      "activations/layer4_attention_weight_max": 120.1108627319336,
      "activations/layer4_attention_weight_min": -115.70267486572266,
      "activations/layer5_attention_weight_max": 53.824913024902344,
      "activations/layer5_attention_weight_min": -62.351070404052734,
      "activations/layer6_attention_weight_max": 47.84031677246094,
      "activations/layer6_attention_weight_min": -50.12644577026367,
      "activations/layer7_attention_weight_max": 88.20958709716797,
      "activations/layer7_attention_weight_min": -89.42321014404297,
      "activations/layer8_attention_weight_max": 40.9785041809082,
      "activations/layer8_attention_weight_min": -38.26726150512695,
      "activations/layer9_attention_weight_max": 34.92564010620117,
      "activations/layer9_attention_weight_min": -32.531368255615234,
      "epoch": 8.73,
      "learning_rate": 9.46530303030303e-05,
      "loss": 2.7872,
      "step": 150250
    },
    {
      "activations/layer0_attention_weight_max": 15.383899688720703,
      "activations/layer0_attention_weight_min": -12.285255432128906,
      "activations/layer10_attention_weight_max": 29.44131088256836,
      "activations/layer10_attention_weight_min": -30.87686538696289,
      "activations/layer11_attention_weight_max": 30.25033187866211,
      "activations/layer11_attention_weight_min": -28.601974487304688,
      "activations/layer12_attention_weight_max": 23.593523025512695,
      "activations/layer12_attention_weight_min": -27.092954635620117,
      "activations/layer13_attention_weight_max": 40.65045166015625,
      "activations/layer13_attention_weight_min": -31.27100372314453,
      "activations/layer14_attention_weight_max": 43.66122055053711,
      "activations/layer14_attention_weight_min": -34.79548263549805,
      "activations/layer15_attention_weight_max": 37.68590545654297,
      "activations/layer15_attention_weight_min": -28.8944091796875,
      "activations/layer16_attention_weight_max": 36.236698150634766,
      "activations/layer16_attention_weight_min": -26.793560028076172,
      "activations/layer17_attention_weight_max": 34.955745697021484,
      "activations/layer17_attention_weight_min": -25.859643936157227,
      "activations/layer18_attention_weight_max": 38.66911315917969,
      "activations/layer18_attention_weight_min": -22.95412826538086,
      "activations/layer19_attention_weight_max": 42.325592041015625,
      "activations/layer19_attention_weight_min": -29.229572296142578,
      "activations/layer1_attention_weight_max": 16.95415687561035,
      "activations/layer1_attention_weight_min": -14.323602676391602,
      "activations/layer20_attention_weight_max": 34.16804504394531,
      "activations/layer20_attention_weight_min": -23.458738327026367,
      "activations/layer21_attention_weight_max": 31.75564956665039,
      "activations/layer21_attention_weight_min": -23.710124969482422,
      "activations/layer22_attention_weight_max": 48.480709075927734,
      "activations/layer22_attention_weight_min": -28.015869140625,
      "activations/layer23_attention_weight_max": 41.73683166503906,
      "activations/layer23_attention_weight_min": -23.11225128173828,
      "activations/layer2_attention_weight_max": 34.24424743652344,
      "activations/layer2_attention_weight_min": -31.78228759765625,
      "activations/layer3_attention_weight_max": 96.31163024902344,
      "activations/layer3_attention_weight_min": -93.76261138916016,
      "activations/layer4_attention_weight_max": 119.07257080078125,
      "activations/layer4_attention_weight_min": -115.013916015625,
      "activations/layer5_attention_weight_max": 53.34873962402344,
      "activations/layer5_attention_weight_min": -60.3577880859375,
      "activations/layer6_attention_weight_max": 46.38351058959961,
      "activations/layer6_attention_weight_min": -49.16341018676758,
      "activations/layer7_attention_weight_max": 83.12017059326172,
      "activations/layer7_attention_weight_min": -86.06107330322266,
      "activations/layer8_attention_weight_max": 37.96834945678711,
      "activations/layer8_attention_weight_min": -37.63459014892578,
      "activations/layer9_attention_weight_max": 30.80585289001465,
      "activations/layer9_attention_weight_min": -31.579069137573242,
      "epoch": 8.73,
      "learning_rate": 9.46340909090909e-05,
      "loss": 2.8049,
      "step": 150300
    },
    {
      "activations/layer0_attention_weight_max": 15.469009399414062,
      "activations/layer0_attention_weight_min": -12.63107681274414,
      "activations/layer10_attention_weight_max": 31.443613052368164,
      "activations/layer10_attention_weight_min": -34.75750732421875,
      "activations/layer11_attention_weight_max": 33.36136245727539,
      "activations/layer11_attention_weight_min": -32.023651123046875,
      "activations/layer12_attention_weight_max": 22.894132614135742,
      "activations/layer12_attention_weight_min": -25.630929946899414,
      "activations/layer13_attention_weight_max": 37.19269943237305,
      "activations/layer13_attention_weight_min": -32.109649658203125,
      "activations/layer14_attention_weight_max": 44.68781280517578,
      "activations/layer14_attention_weight_min": -36.685638427734375,
      "activations/layer15_attention_weight_max": 35.45503234863281,
      "activations/layer15_attention_weight_min": -30.52403450012207,
      "activations/layer16_attention_weight_max": 33.822471618652344,
      "activations/layer16_attention_weight_min": -28.179210662841797,
      "activations/layer17_attention_weight_max": 32.46549606323242,
      "activations/layer17_attention_weight_min": -25.916444778442383,
      "activations/layer18_attention_weight_max": 31.434696197509766,
      "activations/layer18_attention_weight_min": -22.10323143005371,
      "activations/layer19_attention_weight_max": 35.20301055908203,
      "activations/layer19_attention_weight_min": -31.60296058654785,
      "activations/layer1_attention_weight_max": 16.855871200561523,
      "activations/layer1_attention_weight_min": -15.317707061767578,
      "activations/layer20_attention_weight_max": 28.4765682220459,
      "activations/layer20_attention_weight_min": -26.99618148803711,
      "activations/layer21_attention_weight_max": 28.662498474121094,
      "activations/layer21_attention_weight_min": -23.83727264404297,
      "activations/layer22_attention_weight_max": 49.98210906982422,
      "activations/layer22_attention_weight_min": -32.98015213012695,
      "activations/layer23_attention_weight_max": 40.85247039794922,
      "activations/layer23_attention_weight_min": -23.936330795288086,
      "activations/layer2_attention_weight_max": 33.3463020324707,
      "activations/layer2_attention_weight_min": -32.161556243896484,
      "activations/layer3_attention_weight_max": 99.60236358642578,
      "activations/layer3_attention_weight_min": -93.78314208984375,
      "activations/layer4_attention_weight_max": 123.88719940185547,
      "activations/layer4_attention_weight_min": -115.45940399169922,
      "activations/layer5_attention_weight_max": 59.363197326660156,
      "activations/layer5_attention_weight_min": -58.461360931396484,
      "activations/layer6_attention_weight_max": 46.44993591308594,
      "activations/layer6_attention_weight_min": -48.64485549926758,
      "activations/layer7_attention_weight_max": 89.52542114257812,
      "activations/layer7_attention_weight_min": -85.6095962524414,
      "activations/layer8_attention_weight_max": 38.91775894165039,
      "activations/layer8_attention_weight_min": -43.29143524169922,
      "activations/layer9_attention_weight_max": 34.58882141113281,
      "activations/layer9_attention_weight_min": -35.152488708496094,
      "epoch": 8.74,
      "learning_rate": 9.46151515151515e-05,
      "loss": 2.7972,
      "step": 150350
    },
    {
      "activations/layer0_attention_weight_max": 15.612733840942383,
      "activations/layer0_attention_weight_min": -12.821483612060547,
      "activations/layer10_attention_weight_max": 33.21023178100586,
      "activations/layer10_attention_weight_min": -31.224056243896484,
      "activations/layer11_attention_weight_max": 37.92123794555664,
      "activations/layer11_attention_weight_min": -30.86174774169922,
      "activations/layer12_attention_weight_max": 26.893630981445312,
      "activations/layer12_attention_weight_min": -22.50196647644043,
      "activations/layer13_attention_weight_max": 38.375389099121094,
      "activations/layer13_attention_weight_min": -32.098907470703125,
      "activations/layer14_attention_weight_max": 50.914886474609375,
      "activations/layer14_attention_weight_min": -43.42214584350586,
      "activations/layer15_attention_weight_max": 33.60578536987305,
      "activations/layer15_attention_weight_min": -30.46994400024414,
      "activations/layer16_attention_weight_max": 31.08219337463379,
      "activations/layer16_attention_weight_min": -30.344375610351562,
      "activations/layer17_attention_weight_max": 33.287086486816406,
      "activations/layer17_attention_weight_min": -28.1065731048584,
      "activations/layer18_attention_weight_max": 31.7014217376709,
      "activations/layer18_attention_weight_min": -21.802324295043945,
      "activations/layer19_attention_weight_max": 33.90365219116211,
      "activations/layer19_attention_weight_min": -29.80352783203125,
      "activations/layer1_attention_weight_max": 16.419153213500977,
      "activations/layer1_attention_weight_min": -16.468645095825195,
      "activations/layer20_attention_weight_max": 29.48114013671875,
      "activations/layer20_attention_weight_min": -26.086313247680664,
      "activations/layer21_attention_weight_max": 30.40818214416504,
      "activations/layer21_attention_weight_min": -22.452392578125,
      "activations/layer22_attention_weight_max": 48.828453063964844,
      "activations/layer22_attention_weight_min": -29.17584800720215,
      "activations/layer23_attention_weight_max": 32.98655319213867,
      "activations/layer23_attention_weight_min": -23.009984970092773,
      "activations/layer2_attention_weight_max": 34.766075134277344,
      "activations/layer2_attention_weight_min": -32.630916595458984,
      "activations/layer3_attention_weight_max": 101.06075286865234,
      "activations/layer3_attention_weight_min": -98.13456726074219,
      "activations/layer4_attention_weight_max": 126.45467376708984,
      "activations/layer4_attention_weight_min": -126.6020278930664,
      "activations/layer5_attention_weight_max": 56.325843811035156,
      "activations/layer5_attention_weight_min": -68.04325103759766,
      "activations/layer6_attention_weight_max": 48.62417221069336,
      "activations/layer6_attention_weight_min": -51.536067962646484,
      "activations/layer7_attention_weight_max": 92.5971908569336,
      "activations/layer7_attention_weight_min": -95.22111511230469,
      "activations/layer8_attention_weight_max": 40.939239501953125,
      "activations/layer8_attention_weight_min": -39.98969650268555,
      "activations/layer9_attention_weight_max": 36.548988342285156,
      "activations/layer9_attention_weight_min": -33.404014587402344,
      "epoch": 8.74,
      "learning_rate": 9.459621212121212e-05,
      "loss": 2.8066,
      "step": 150400
    },
    {
      "activations/layer0_attention_weight_max": 15.464858055114746,
      "activations/layer0_attention_weight_min": -14.19736099243164,
      "activations/layer10_attention_weight_max": 38.21131896972656,
      "activations/layer10_attention_weight_min": -36.80642318725586,
      "activations/layer11_attention_weight_max": 33.54187774658203,
      "activations/layer11_attention_weight_min": -32.930908203125,
      "activations/layer12_attention_weight_max": 24.298627853393555,
      "activations/layer12_attention_weight_min": -24.161548614501953,
      "activations/layer13_attention_weight_max": 45.51244354248047,
      "activations/layer13_attention_weight_min": -33.66689682006836,
      "activations/layer14_attention_weight_max": 48.078372955322266,
      "activations/layer14_attention_weight_min": -37.43035125732422,
      "activations/layer15_attention_weight_max": 37.46621322631836,
      "activations/layer15_attention_weight_min": -30.982006072998047,
      "activations/layer16_attention_weight_max": 30.64274024963379,
      "activations/layer16_attention_weight_min": -27.75472640991211,
      "activations/layer17_attention_weight_max": 33.51031494140625,
      "activations/layer17_attention_weight_min": -25.275997161865234,
      "activations/layer18_attention_weight_max": 34.8785400390625,
      "activations/layer18_attention_weight_min": -21.855527877807617,
      "activations/layer19_attention_weight_max": 35.84251022338867,
      "activations/layer19_attention_weight_min": -30.09364891052246,
      "activations/layer1_attention_weight_max": 16.584596633911133,
      "activations/layer1_attention_weight_min": -14.722926139831543,
      "activations/layer20_attention_weight_max": 30.31393051147461,
      "activations/layer20_attention_weight_min": -24.949872970581055,
      "activations/layer21_attention_weight_max": 28.868541717529297,
      "activations/layer21_attention_weight_min": -23.552871704101562,
      "activations/layer22_attention_weight_max": 47.86782455444336,
      "activations/layer22_attention_weight_min": -29.58716583251953,
      "activations/layer23_attention_weight_max": 33.32786178588867,
      "activations/layer23_attention_weight_min": -25.2182674407959,
      "activations/layer2_attention_weight_max": 28.94598388671875,
      "activations/layer2_attention_weight_min": -29.540977478027344,
      "activations/layer3_attention_weight_max": 86.97599792480469,
      "activations/layer3_attention_weight_min": -91.18598175048828,
      "activations/layer4_attention_weight_max": 113.07918548583984,
      "activations/layer4_attention_weight_min": -115.40718078613281,
      "activations/layer5_attention_weight_max": 52.80239486694336,
      "activations/layer5_attention_weight_min": -65.81956481933594,
      "activations/layer6_attention_weight_max": 46.74808120727539,
      "activations/layer6_attention_weight_min": -53.18311309814453,
      "activations/layer7_attention_weight_max": 88.7073745727539,
      "activations/layer7_attention_weight_min": -101.55775451660156,
      "activations/layer8_attention_weight_max": 43.12495040893555,
      "activations/layer8_attention_weight_min": -43.813785552978516,
      "activations/layer9_attention_weight_max": 38.71580505371094,
      "activations/layer9_attention_weight_min": -37.05410385131836,
      "epoch": 8.74,
      "learning_rate": 9.457727272727272e-05,
      "loss": 2.791,
      "step": 150450
    },
    {
      "activations/layer0_attention_weight_max": 15.90401840209961,
      "activations/layer0_attention_weight_min": -13.440347671508789,
      "activations/layer10_attention_weight_max": 33.58522033691406,
      "activations/layer10_attention_weight_min": -32.40252685546875,
      "activations/layer11_attention_weight_max": 33.191368103027344,
      "activations/layer11_attention_weight_min": -31.30730438232422,
      "activations/layer12_attention_weight_max": 24.441909790039062,
      "activations/layer12_attention_weight_min": -26.680335998535156,
      "activations/layer13_attention_weight_max": 40.733699798583984,
      "activations/layer13_attention_weight_min": -31.729164123535156,
      "activations/layer14_attention_weight_max": 48.77316665649414,
      "activations/layer14_attention_weight_min": -43.073123931884766,
      "activations/layer15_attention_weight_max": 39.518531799316406,
      "activations/layer15_attention_weight_min": -28.935733795166016,
      "activations/layer16_attention_weight_max": 33.81510543823242,
      "activations/layer16_attention_weight_min": -27.727258682250977,
      "activations/layer17_attention_weight_max": 34.56481170654297,
      "activations/layer17_attention_weight_min": -26.313697814941406,
      "activations/layer18_attention_weight_max": 32.62923812866211,
      "activations/layer18_attention_weight_min": -23.786903381347656,
      "activations/layer19_attention_weight_max": 36.47187423706055,
      "activations/layer19_attention_weight_min": -29.833877563476562,
      "activations/layer1_attention_weight_max": 16.01705551147461,
      "activations/layer1_attention_weight_min": -14.888317108154297,
      "activations/layer20_attention_weight_max": 30.14215087890625,
      "activations/layer20_attention_weight_min": -23.48751449584961,
      "activations/layer21_attention_weight_max": 30.694162368774414,
      "activations/layer21_attention_weight_min": -21.404069900512695,
      "activations/layer22_attention_weight_max": 42.35641860961914,
      "activations/layer22_attention_weight_min": -29.661027908325195,
      "activations/layer23_attention_weight_max": 34.96403121948242,
      "activations/layer23_attention_weight_min": -24.374563217163086,
      "activations/layer2_attention_weight_max": 31.894817352294922,
      "activations/layer2_attention_weight_min": -31.692991256713867,
      "activations/layer3_attention_weight_max": 96.2025375366211,
      "activations/layer3_attention_weight_min": -99.57403564453125,
      "activations/layer4_attention_weight_max": 116.66815185546875,
      "activations/layer4_attention_weight_min": -121.40398406982422,
      "activations/layer5_attention_weight_max": 54.86375427246094,
      "activations/layer5_attention_weight_min": -63.881614685058594,
      "activations/layer6_attention_weight_max": 48.06260681152344,
      "activations/layer6_attention_weight_min": -56.51381301879883,
      "activations/layer7_attention_weight_max": 91.5697250366211,
      "activations/layer7_attention_weight_min": -89.9632339477539,
      "activations/layer8_attention_weight_max": 41.99333953857422,
      "activations/layer8_attention_weight_min": -44.339263916015625,
      "activations/layer9_attention_weight_max": 37.34123229980469,
      "activations/layer9_attention_weight_min": -34.18929672241211,
      "epoch": 8.74,
      "learning_rate": 9.455833333333332e-05,
      "loss": 2.8169,
      "step": 150500
    },
    {
      "activations/layer0_attention_weight_max": 15.969467163085938,
      "activations/layer0_attention_weight_min": -13.372268676757812,
      "activations/layer10_attention_weight_max": 30.83779525756836,
      "activations/layer10_attention_weight_min": -32.403472900390625,
      "activations/layer11_attention_weight_max": 29.998395919799805,
      "activations/layer11_attention_weight_min": -30.28077507019043,
      "activations/layer12_attention_weight_max": 22.82126808166504,
      "activations/layer12_attention_weight_min": -27.631593704223633,
      "activations/layer13_attention_weight_max": 35.72258758544922,
      "activations/layer13_attention_weight_min": -32.120941162109375,
      "activations/layer14_attention_weight_max": 48.25764083862305,
      "activations/layer14_attention_weight_min": -36.308223724365234,
      "activations/layer15_attention_weight_max": 36.205726623535156,
      "activations/layer15_attention_weight_min": -32.20925521850586,
      "activations/layer16_attention_weight_max": 32.6295166015625,
      "activations/layer16_attention_weight_min": -29.391929626464844,
      "activations/layer17_attention_weight_max": 32.2171516418457,
      "activations/layer17_attention_weight_min": -26.11248207092285,
      "activations/layer18_attention_weight_max": 29.43425941467285,
      "activations/layer18_attention_weight_min": -22.49213409423828,
      "activations/layer19_attention_weight_max": 37.32687759399414,
      "activations/layer19_attention_weight_min": -30.28635025024414,
      "activations/layer1_attention_weight_max": 16.23392105102539,
      "activations/layer1_attention_weight_min": -15.275858879089355,
      "activations/layer20_attention_weight_max": 29.813884735107422,
      "activations/layer20_attention_weight_min": -24.944250106811523,
      "activations/layer21_attention_weight_max": 32.24988555908203,
      "activations/layer21_attention_weight_min": -25.106958389282227,
      "activations/layer22_attention_weight_max": 56.09461975097656,
      "activations/layer22_attention_weight_min": -31.483518600463867,
      "activations/layer23_attention_weight_max": 37.88413619995117,
      "activations/layer23_attention_weight_min": -25.74953842163086,
      "activations/layer2_attention_weight_max": 32.45765686035156,
      "activations/layer2_attention_weight_min": -33.23176574707031,
      "activations/layer3_attention_weight_max": 92.78060150146484,
      "activations/layer3_attention_weight_min": -98.07671356201172,
      "activations/layer4_attention_weight_max": 110.76934051513672,
      "activations/layer4_attention_weight_min": -122.15534973144531,
      "activations/layer5_attention_weight_max": 52.033348083496094,
      "activations/layer5_attention_weight_min": -62.849891662597656,
      "activations/layer6_attention_weight_max": 47.45875549316406,
      "activations/layer6_attention_weight_min": -51.645572662353516,
      "activations/layer7_attention_weight_max": 83.19588470458984,
      "activations/layer7_attention_weight_min": -92.81615447998047,
      "activations/layer8_attention_weight_max": 37.903865814208984,
      "activations/layer8_attention_weight_min": -43.502037048339844,
      "activations/layer9_attention_weight_max": 32.82356643676758,
      "activations/layer9_attention_weight_min": -33.49416732788086,
      "epoch": 8.75,
      "learning_rate": 9.453939393939393e-05,
      "loss": 2.8091,
      "step": 150550
    },
    {
      "activations/layer0_attention_weight_max": 15.74050521850586,
      "activations/layer0_attention_weight_min": -13.627333641052246,
      "activations/layer10_attention_weight_max": 34.4100456237793,
      "activations/layer10_attention_weight_min": -32.21225357055664,
      "activations/layer11_attention_weight_max": 36.015289306640625,
      "activations/layer11_attention_weight_min": -31.363811492919922,
      "activations/layer12_attention_weight_max": 25.70841407775879,
      "activations/layer12_attention_weight_min": -25.737201690673828,
      "activations/layer13_attention_weight_max": 41.136924743652344,
      "activations/layer13_attention_weight_min": -33.814666748046875,
      "activations/layer14_attention_weight_max": 52.49742126464844,
      "activations/layer14_attention_weight_min": -40.39958953857422,
      "activations/layer15_attention_weight_max": 39.31068801879883,
      "activations/layer15_attention_weight_min": -32.35207748413086,
      "activations/layer16_attention_weight_max": 32.76414108276367,
      "activations/layer16_attention_weight_min": -28.43657684326172,
      "activations/layer17_attention_weight_max": 33.916847229003906,
      "activations/layer17_attention_weight_min": -27.25408935546875,
      "activations/layer18_attention_weight_max": 34.47767639160156,
      "activations/layer18_attention_weight_min": -22.718103408813477,
      "activations/layer19_attention_weight_max": 36.12143325805664,
      "activations/layer19_attention_weight_min": -33.67276382446289,
      "activations/layer1_attention_weight_max": 17.209611892700195,
      "activations/layer1_attention_weight_min": -15.251470565795898,
      "activations/layer20_attention_weight_max": 29.489816665649414,
      "activations/layer20_attention_weight_min": -25.12933921813965,
      "activations/layer21_attention_weight_max": 29.81443977355957,
      "activations/layer21_attention_weight_min": -24.957916259765625,
      "activations/layer22_attention_weight_max": 46.74675369262695,
      "activations/layer22_attention_weight_min": -31.16260528564453,
      "activations/layer23_attention_weight_max": 38.59794616699219,
      "activations/layer23_attention_weight_min": -25.557371139526367,
      "activations/layer2_attention_weight_max": 33.205291748046875,
      "activations/layer2_attention_weight_min": -31.47052001953125,
      "activations/layer3_attention_weight_max": 96.34687805175781,
      "activations/layer3_attention_weight_min": -95.62161254882812,
      "activations/layer4_attention_weight_max": 119.86553192138672,
      "activations/layer4_attention_weight_min": -119.3946762084961,
      "activations/layer5_attention_weight_max": 54.414649963378906,
      "activations/layer5_attention_weight_min": -65.65711212158203,
      "activations/layer6_attention_weight_max": 49.18003845214844,
      "activations/layer6_attention_weight_min": -53.95423126220703,
      "activations/layer7_attention_weight_max": 87.46981811523438,
      "activations/layer7_attention_weight_min": -93.29994201660156,
      "activations/layer8_attention_weight_max": 41.1263427734375,
      "activations/layer8_attention_weight_min": -40.70610427856445,
      "activations/layer9_attention_weight_max": 36.889713287353516,
      "activations/layer9_attention_weight_min": -34.760475158691406,
      "epoch": 8.75,
      "learning_rate": 9.452045454545454e-05,
      "loss": 2.8155,
      "step": 150600
    },
    {
      "activations/layer0_attention_weight_max": 14.981941223144531,
      "activations/layer0_attention_weight_min": -13.643054008483887,
      "activations/layer10_attention_weight_max": 32.323673248291016,
      "activations/layer10_attention_weight_min": -31.407297134399414,
      "activations/layer11_attention_weight_max": 31.724933624267578,
      "activations/layer11_attention_weight_min": -30.751996994018555,
      "activations/layer12_attention_weight_max": 23.192163467407227,
      "activations/layer12_attention_weight_min": -24.010589599609375,
      "activations/layer13_attention_weight_max": 36.854095458984375,
      "activations/layer13_attention_weight_min": -33.94812774658203,
      "activations/layer14_attention_weight_max": 45.45309829711914,
      "activations/layer14_attention_weight_min": -41.77182388305664,
      "activations/layer15_attention_weight_max": 32.6305046081543,
      "activations/layer15_attention_weight_min": -31.26665496826172,
      "activations/layer16_attention_weight_max": 28.356435775756836,
      "activations/layer16_attention_weight_min": -28.318260192871094,
      "activations/layer17_attention_weight_max": 33.923133850097656,
      "activations/layer17_attention_weight_min": -25.434925079345703,
      "activations/layer18_attention_weight_max": 30.002599716186523,
      "activations/layer18_attention_weight_min": -22.402496337890625,
      "activations/layer19_attention_weight_max": 31.106979370117188,
      "activations/layer19_attention_weight_min": -30.66297721862793,
      "activations/layer1_attention_weight_max": 16.240955352783203,
      "activations/layer1_attention_weight_min": -13.553762435913086,
      "activations/layer20_attention_weight_max": 27.7860050201416,
      "activations/layer20_attention_weight_min": -23.080766677856445,
      "activations/layer21_attention_weight_max": 27.985992431640625,
      "activations/layer21_attention_weight_min": -22.803050994873047,
      "activations/layer22_attention_weight_max": 42.47029495239258,
      "activations/layer22_attention_weight_min": -27.22518539428711,
      "activations/layer23_attention_weight_max": 32.97907257080078,
      "activations/layer23_attention_weight_min": -24.813201904296875,
      "activations/layer2_attention_weight_max": 31.265483856201172,
      "activations/layer2_attention_weight_min": -32.377342224121094,
      "activations/layer3_attention_weight_max": 91.06414794921875,
      "activations/layer3_attention_weight_min": -95.15911102294922,
      "activations/layer4_attention_weight_max": 119.73055267333984,
      "activations/layer4_attention_weight_min": -115.79036712646484,
      "activations/layer5_attention_weight_max": 54.96575164794922,
      "activations/layer5_attention_weight_min": -61.36920166015625,
      "activations/layer6_attention_weight_max": 50.54042434692383,
      "activations/layer6_attention_weight_min": -52.09965515136719,
      "activations/layer7_attention_weight_max": 89.27651977539062,
      "activations/layer7_attention_weight_min": -90.26585388183594,
      "activations/layer8_attention_weight_max": 40.624271392822266,
      "activations/layer8_attention_weight_min": -41.845130920410156,
      "activations/layer9_attention_weight_max": 34.966087341308594,
      "activations/layer9_attention_weight_min": -33.925453186035156,
      "epoch": 8.75,
      "learning_rate": 9.450151515151515e-05,
      "loss": 2.8152,
      "step": 150650
    },
    {
      "activations/layer0_attention_weight_max": 16.043704986572266,
      "activations/layer0_attention_weight_min": -12.685895919799805,
      "activations/layer10_attention_weight_max": 37.02880096435547,
      "activations/layer10_attention_weight_min": -34.44805908203125,
      "activations/layer11_attention_weight_max": 37.066864013671875,
      "activations/layer11_attention_weight_min": -32.528282165527344,
      "activations/layer12_attention_weight_max": 38.25090789794922,
      "activations/layer12_attention_weight_min": -25.075660705566406,
      "activations/layer13_attention_weight_max": 50.03871536254883,
      "activations/layer13_attention_weight_min": -32.620140075683594,
      "activations/layer14_attention_weight_max": 54.760414123535156,
      "activations/layer14_attention_weight_min": -42.78398513793945,
      "activations/layer15_attention_weight_max": 38.931461334228516,
      "activations/layer15_attention_weight_min": -30.11909294128418,
      "activations/layer16_attention_weight_max": 33.65201187133789,
      "activations/layer16_attention_weight_min": -26.165132522583008,
      "activations/layer17_attention_weight_max": 35.4876823425293,
      "activations/layer17_attention_weight_min": -26.617881774902344,
      "activations/layer18_attention_weight_max": 31.282283782958984,
      "activations/layer18_attention_weight_min": -20.72942543029785,
      "activations/layer19_attention_weight_max": 36.52156066894531,
      "activations/layer19_attention_weight_min": -29.138357162475586,
      "activations/layer1_attention_weight_max": 16.920543670654297,
      "activations/layer1_attention_weight_min": -14.746527671813965,
      "activations/layer20_attention_weight_max": 30.8366756439209,
      "activations/layer20_attention_weight_min": -23.138351440429688,
      "activations/layer21_attention_weight_max": 31.477170944213867,
      "activations/layer21_attention_weight_min": -21.26799201965332,
      "activations/layer22_attention_weight_max": 46.45065689086914,
      "activations/layer22_attention_weight_min": -29.24851417541504,
      "activations/layer23_attention_weight_max": 37.67377853393555,
      "activations/layer23_attention_weight_min": -24.09108543395996,
      "activations/layer2_attention_weight_max": 33.82683563232422,
      "activations/layer2_attention_weight_min": -34.302982330322266,
      "activations/layer3_attention_weight_max": 97.45624542236328,
      "activations/layer3_attention_weight_min": -102.76691436767578,
      "activations/layer4_attention_weight_max": 119.87940979003906,
      "activations/layer4_attention_weight_min": -127.27717590332031,
      "activations/layer5_attention_weight_max": 60.119972229003906,
      "activations/layer5_attention_weight_min": -65.56893920898438,
      "activations/layer6_attention_weight_max": 49.70030212402344,
      "activations/layer6_attention_weight_min": -52.265804290771484,
      "activations/layer7_attention_weight_max": 100.15906524658203,
      "activations/layer7_attention_weight_min": -95.61046600341797,
      "activations/layer8_attention_weight_max": 43.09111404418945,
      "activations/layer8_attention_weight_min": -45.62483596801758,
      "activations/layer9_attention_weight_max": 38.19559860229492,
      "activations/layer9_attention_weight_min": -36.23116683959961,
      "epoch": 8.76,
      "learning_rate": 9.448257575757575e-05,
      "loss": 2.8133,
      "step": 150700
    },
    {
      "activations/layer0_attention_weight_max": 16.38909149169922,
      "activations/layer0_attention_weight_min": -12.57791519165039,
      "activations/layer10_attention_weight_max": 32.83546829223633,
      "activations/layer10_attention_weight_min": -32.523658752441406,
      "activations/layer11_attention_weight_max": 32.59310531616211,
      "activations/layer11_attention_weight_min": -30.764999389648438,
      "activations/layer12_attention_weight_max": 24.04555892944336,
      "activations/layer12_attention_weight_min": -24.471370697021484,
      "activations/layer13_attention_weight_max": 38.72103500366211,
      "activations/layer13_attention_weight_min": -36.27602767944336,
      "activations/layer14_attention_weight_max": 43.50300598144531,
      "activations/layer14_attention_weight_min": -38.033416748046875,
      "activations/layer15_attention_weight_max": 37.53908920288086,
      "activations/layer15_attention_weight_min": -29.751882553100586,
      "activations/layer16_attention_weight_max": 33.80419921875,
      "activations/layer16_attention_weight_min": -27.786788940429688,
      "activations/layer17_attention_weight_max": 32.4552116394043,
      "activations/layer17_attention_weight_min": -25.956663131713867,
      "activations/layer18_attention_weight_max": 32.1426887512207,
      "activations/layer18_attention_weight_min": -23.18096160888672,
      "activations/layer19_attention_weight_max": 35.591854095458984,
      "activations/layer19_attention_weight_min": -30.304977416992188,
      "activations/layer1_attention_weight_max": 16.3865966796875,
      "activations/layer1_attention_weight_min": -14.590181350708008,
      "activations/layer20_attention_weight_max": 30.623178482055664,
      "activations/layer20_attention_weight_min": -24.444002151489258,
      "activations/layer21_attention_weight_max": 28.617156982421875,
      "activations/layer21_attention_weight_min": -23.957286834716797,
      "activations/layer22_attention_weight_max": 47.599754333496094,
      "activations/layer22_attention_weight_min": -29.5593318939209,
      "activations/layer23_attention_weight_max": 35.95655059814453,
      "activations/layer23_attention_weight_min": -23.919010162353516,
      "activations/layer2_attention_weight_max": 31.68256187438965,
      "activations/layer2_attention_weight_min": -32.629913330078125,
      "activations/layer3_attention_weight_max": 89.23151397705078,
      "activations/layer3_attention_weight_min": -92.29849243164062,
      "activations/layer4_attention_weight_max": 116.45672607421875,
      "activations/layer4_attention_weight_min": -114.96868133544922,
      "activations/layer5_attention_weight_max": 54.3272819519043,
      "activations/layer5_attention_weight_min": -62.72611999511719,
      "activations/layer6_attention_weight_max": 47.33280944824219,
      "activations/layer6_attention_weight_min": -50.01155090332031,
      "activations/layer7_attention_weight_max": 86.19659423828125,
      "activations/layer7_attention_weight_min": -89.2693099975586,
      "activations/layer8_attention_weight_max": 41.4674072265625,
      "activations/layer8_attention_weight_min": -42.71635818481445,
      "activations/layer9_attention_weight_max": 33.162513732910156,
      "activations/layer9_attention_weight_min": -33.458683013916016,
      "epoch": 8.76,
      "learning_rate": 9.446363636363635e-05,
      "loss": 2.7794,
      "step": 150750
    },
    {
      "activations/layer0_attention_weight_max": 16.13905143737793,
      "activations/layer0_attention_weight_min": -12.642902374267578,
      "activations/layer10_attention_weight_max": 33.88127517700195,
      "activations/layer10_attention_weight_min": -32.142513275146484,
      "activations/layer11_attention_weight_max": 33.70232009887695,
      "activations/layer11_attention_weight_min": -30.821208953857422,
      "activations/layer12_attention_weight_max": 24.552953720092773,
      "activations/layer12_attention_weight_min": -24.044910430908203,
      "activations/layer13_attention_weight_max": 38.922821044921875,
      "activations/layer13_attention_weight_min": -31.889934539794922,
      "activations/layer14_attention_weight_max": 47.93743896484375,
      "activations/layer14_attention_weight_min": -37.69440841674805,
      "activations/layer15_attention_weight_max": 39.297943115234375,
      "activations/layer15_attention_weight_min": -32.499324798583984,
      "activations/layer16_attention_weight_max": 36.3953742980957,
      "activations/layer16_attention_weight_min": -26.880523681640625,
      "activations/layer17_attention_weight_max": 37.16011047363281,
      "activations/layer17_attention_weight_min": -25.447237014770508,
      "activations/layer18_attention_weight_max": 34.090545654296875,
      "activations/layer18_attention_weight_min": -22.07355499267578,
      "activations/layer19_attention_weight_max": 43.26736068725586,
      "activations/layer19_attention_weight_min": -33.14116287231445,
      "activations/layer1_attention_weight_max": 16.595767974853516,
      "activations/layer1_attention_weight_min": -13.915431022644043,
      "activations/layer20_attention_weight_max": 39.13352966308594,
      "activations/layer20_attention_weight_min": -26.069047927856445,
      "activations/layer21_attention_weight_max": 32.7164306640625,
      "activations/layer21_attention_weight_min": -23.325286865234375,
      "activations/layer22_attention_weight_max": 57.98466110229492,
      "activations/layer22_attention_weight_min": -29.972850799560547,
      "activations/layer23_attention_weight_max": 44.188819885253906,
      "activations/layer23_attention_weight_min": -25.469039916992188,
      "activations/layer2_attention_weight_max": 34.05873107910156,
      "activations/layer2_attention_weight_min": -31.552120208740234,
      "activations/layer3_attention_weight_max": 95.24072265625,
      "activations/layer3_attention_weight_min": -95.88232421875,
      "activations/layer4_attention_weight_max": 117.98712158203125,
      "activations/layer4_attention_weight_min": -118.0752944946289,
      "activations/layer5_attention_weight_max": 55.182830810546875,
      "activations/layer5_attention_weight_min": -62.4831657409668,
      "activations/layer6_attention_weight_max": 48.91968536376953,
      "activations/layer6_attention_weight_min": -53.99107360839844,
      "activations/layer7_attention_weight_max": 93.47203063964844,
      "activations/layer7_attention_weight_min": -96.40725708007812,
      "activations/layer8_attention_weight_max": 41.158119201660156,
      "activations/layer8_attention_weight_min": -41.73301315307617,
      "activations/layer9_attention_weight_max": 35.36599349975586,
      "activations/layer9_attention_weight_min": -34.093990325927734,
      "epoch": 8.76,
      "learning_rate": 9.444469696969697e-05,
      "loss": 2.8077,
      "step": 150800
    },
    {
      "activations/layer0_attention_weight_max": 16.688034057617188,
      "activations/layer0_attention_weight_min": -13.107922554016113,
      "activations/layer10_attention_weight_max": 35.308494567871094,
      "activations/layer10_attention_weight_min": -34.09324264526367,
      "activations/layer11_attention_weight_max": 33.52568435668945,
      "activations/layer11_attention_weight_min": -32.322879791259766,
      "activations/layer12_attention_weight_max": 25.174943923950195,
      "activations/layer12_attention_weight_min": -25.304706573486328,
      "activations/layer13_attention_weight_max": 41.084693908691406,
      "activations/layer13_attention_weight_min": -37.19612121582031,
      "activations/layer14_attention_weight_max": 51.12394332885742,
      "activations/layer14_attention_weight_min": -43.82817459106445,
      "activations/layer15_attention_weight_max": 35.01228713989258,
      "activations/layer15_attention_weight_min": -29.21609115600586,
      "activations/layer16_attention_weight_max": 30.048503875732422,
      "activations/layer16_attention_weight_min": -27.260616302490234,
      "activations/layer17_attention_weight_max": 32.14975357055664,
      "activations/layer17_attention_weight_min": -25.936195373535156,
      "activations/layer18_attention_weight_max": 32.71190643310547,
      "activations/layer18_attention_weight_min": -21.490398406982422,
      "activations/layer19_attention_weight_max": 34.41561508178711,
      "activations/layer19_attention_weight_min": -29.40485191345215,
      "activations/layer1_attention_weight_max": 17.006593704223633,
      "activations/layer1_attention_weight_min": -15.914690017700195,
      "activations/layer20_attention_weight_max": 28.3536434173584,
      "activations/layer20_attention_weight_min": -24.965726852416992,
      "activations/layer21_attention_weight_max": 31.905670166015625,
      "activations/layer21_attention_weight_min": -24.11589241027832,
      "activations/layer22_attention_weight_max": 45.567230224609375,
      "activations/layer22_attention_weight_min": -29.17517852783203,
      "activations/layer23_attention_weight_max": 36.44165802001953,
      "activations/layer23_attention_weight_min": -24.998655319213867,
      "activations/layer2_attention_weight_max": 34.09046173095703,
      "activations/layer2_attention_weight_min": -34.218502044677734,
      "activations/layer3_attention_weight_max": 95.34288787841797,
      "activations/layer3_attention_weight_min": -99.94445037841797,
      "activations/layer4_attention_weight_max": 116.8191909790039,
      "activations/layer4_attention_weight_min": -117.414306640625,
      "activations/layer5_attention_weight_max": 56.931129455566406,
      "activations/layer5_attention_weight_min": -60.5133171081543,
      "activations/layer6_attention_weight_max": 48.246192932128906,
      "activations/layer6_attention_weight_min": -51.95056915283203,
      "activations/layer7_attention_weight_max": 88.98799133300781,
      "activations/layer7_attention_weight_min": -94.07545471191406,
      "activations/layer8_attention_weight_max": 43.013145446777344,
      "activations/layer8_attention_weight_min": -44.654727935791016,
      "activations/layer9_attention_weight_max": 38.21038055419922,
      "activations/layer9_attention_weight_min": -35.58407211303711,
      "epoch": 8.77,
      "learning_rate": 9.442575757575756e-05,
      "loss": 2.8097,
      "step": 150850
    },
    {
      "activations/layer0_attention_weight_max": 16.01689910888672,
      "activations/layer0_attention_weight_min": -13.170970916748047,
      "activations/layer10_attention_weight_max": 35.23614501953125,
      "activations/layer10_attention_weight_min": -33.93477249145508,
      "activations/layer11_attention_weight_max": 34.01990509033203,
      "activations/layer11_attention_weight_min": -31.454206466674805,
      "activations/layer12_attention_weight_max": 24.328516006469727,
      "activations/layer12_attention_weight_min": -25.0318603515625,
      "activations/layer13_attention_weight_max": 40.765052795410156,
      "activations/layer13_attention_weight_min": -34.58116912841797,
      "activations/layer14_attention_weight_max": 49.21016311645508,
      "activations/layer14_attention_weight_min": -41.404815673828125,
      "activations/layer15_attention_weight_max": 36.670326232910156,
      "activations/layer15_attention_weight_min": -29.758529663085938,
      "activations/layer16_attention_weight_max": 32.03959655761719,
      "activations/layer16_attention_weight_min": -28.269567489624023,
      "activations/layer17_attention_weight_max": 34.14140701293945,
      "activations/layer17_attention_weight_min": -24.877410888671875,
      "activations/layer18_attention_weight_max": 31.654415130615234,
      "activations/layer18_attention_weight_min": -22.74363136291504,
      "activations/layer19_attention_weight_max": 37.38357162475586,
      "activations/layer19_attention_weight_min": -29.933338165283203,
      "activations/layer1_attention_weight_max": 16.69137954711914,
      "activations/layer1_attention_weight_min": -15.347825050354004,
      "activations/layer20_attention_weight_max": 30.900230407714844,
      "activations/layer20_attention_weight_min": -23.813539505004883,
      "activations/layer21_attention_weight_max": 27.54521942138672,
      "activations/layer21_attention_weight_min": -22.592397689819336,
      "activations/layer22_attention_weight_max": 50.891910552978516,
      "activations/layer22_attention_weight_min": -28.683963775634766,
      "activations/layer23_attention_weight_max": 36.825592041015625,
      "activations/layer23_attention_weight_min": -24.385440826416016,
      "activations/layer2_attention_weight_max": 33.1419563293457,
      "activations/layer2_attention_weight_min": -31.24049949645996,
      "activations/layer3_attention_weight_max": 93.84220886230469,
      "activations/layer3_attention_weight_min": -97.69599151611328,
      "activations/layer4_attention_weight_max": 115.96012115478516,
      "activations/layer4_attention_weight_min": -116.04622650146484,
      "activations/layer5_attention_weight_max": 54.97779846191406,
      "activations/layer5_attention_weight_min": -64.82765197753906,
      "activations/layer6_attention_weight_max": 49.45952606201172,
      "activations/layer6_attention_weight_min": -50.002098083496094,
      "activations/layer7_attention_weight_max": 94.27013397216797,
      "activations/layer7_attention_weight_min": -87.2479476928711,
      "activations/layer8_attention_weight_max": 43.25497817993164,
      "activations/layer8_attention_weight_min": -42.89906692504883,
      "activations/layer9_attention_weight_max": 35.502540588378906,
      "activations/layer9_attention_weight_min": -32.93606185913086,
      "epoch": 8.77,
      "learning_rate": 9.440681818181817e-05,
      "loss": 2.7943,
      "step": 150900
    },
    {
      "activations/layer0_attention_weight_max": 16.33893394470215,
      "activations/layer0_attention_weight_min": -12.892687797546387,
      "activations/layer10_attention_weight_max": 32.97645950317383,
      "activations/layer10_attention_weight_min": -30.983327865600586,
      "activations/layer11_attention_weight_max": 32.04167175292969,
      "activations/layer11_attention_weight_min": -29.863475799560547,
      "activations/layer12_attention_weight_max": 21.889175415039062,
      "activations/layer12_attention_weight_min": -24.317188262939453,
      "activations/layer13_attention_weight_max": 36.959896087646484,
      "activations/layer13_attention_weight_min": -31.834373474121094,
      "activations/layer14_attention_weight_max": 46.05462646484375,
      "activations/layer14_attention_weight_min": -36.933353424072266,
      "activations/layer15_attention_weight_max": 36.57200241088867,
      "activations/layer15_attention_weight_min": -29.414161682128906,
      "activations/layer16_attention_weight_max": 32.64556884765625,
      "activations/layer16_attention_weight_min": -26.986303329467773,
      "activations/layer17_attention_weight_max": 34.00495529174805,
      "activations/layer17_attention_weight_min": -24.86097526550293,
      "activations/layer18_attention_weight_max": 31.582988739013672,
      "activations/layer18_attention_weight_min": -21.0456600189209,
      "activations/layer19_attention_weight_max": 37.443931579589844,
      "activations/layer19_attention_weight_min": -29.69144058227539,
      "activations/layer1_attention_weight_max": 16.758480072021484,
      "activations/layer1_attention_weight_min": -14.316320419311523,
      "activations/layer20_attention_weight_max": 32.45014190673828,
      "activations/layer20_attention_weight_min": -23.132925033569336,
      "activations/layer21_attention_weight_max": 27.62270164489746,
      "activations/layer21_attention_weight_min": -21.625205993652344,
      "activations/layer22_attention_weight_max": 44.91641616821289,
      "activations/layer22_attention_weight_min": -30.025142669677734,
      "activations/layer23_attention_weight_max": 38.442176818847656,
      "activations/layer23_attention_weight_min": -21.906818389892578,
      "activations/layer2_attention_weight_max": 32.1907958984375,
      "activations/layer2_attention_weight_min": -33.42490768432617,
      "activations/layer3_attention_weight_max": 95.66223907470703,
      "activations/layer3_attention_weight_min": -98.88641357421875,
      "activations/layer4_attention_weight_max": 119.77717590332031,
      "activations/layer4_attention_weight_min": -120.87162017822266,
      "activations/layer5_attention_weight_max": 54.751007080078125,
      "activations/layer5_attention_weight_min": -64.06321716308594,
      "activations/layer6_attention_weight_max": 48.204078674316406,
      "activations/layer6_attention_weight_min": -51.532508850097656,
      "activations/layer7_attention_weight_max": 90.40907287597656,
      "activations/layer7_attention_weight_min": -87.86511993408203,
      "activations/layer8_attention_weight_max": 38.7401237487793,
      "activations/layer8_attention_weight_min": -40.31770324707031,
      "activations/layer9_attention_weight_max": 34.687110900878906,
      "activations/layer9_attention_weight_min": -33.201168060302734,
      "epoch": 8.77,
      "learning_rate": 9.438787878787879e-05,
      "loss": 2.7871,
      "step": 150950
    },
    {
      "activations/layer0_attention_weight_max": 15.193472862243652,
      "activations/layer0_attention_weight_min": -12.401922225952148,
      "activations/layer10_attention_weight_max": 33.312129974365234,
      "activations/layer10_attention_weight_min": -31.75919532775879,
      "activations/layer11_attention_weight_max": 31.2532958984375,
      "activations/layer11_attention_weight_min": -32.146934509277344,
      "activations/layer12_attention_weight_max": 23.7387752532959,
      "activations/layer12_attention_weight_min": -24.5720272064209,
      "activations/layer13_attention_weight_max": 39.13886642456055,
      "activations/layer13_attention_weight_min": -32.13176727294922,
      "activations/layer14_attention_weight_max": 45.59921646118164,
      "activations/layer14_attention_weight_min": -37.09593963623047,
      "activations/layer15_attention_weight_max": 35.86890411376953,
      "activations/layer15_attention_weight_min": -29.9035587310791,
      "activations/layer16_attention_weight_max": 30.974807739257812,
      "activations/layer16_attention_weight_min": -27.430564880371094,
      "activations/layer17_attention_weight_max": 31.39808464050293,
      "activations/layer17_attention_weight_min": -26.14310646057129,
      "activations/layer18_attention_weight_max": 31.115800857543945,
      "activations/layer18_attention_weight_min": -22.51584243774414,
      "activations/layer19_attention_weight_max": 36.366737365722656,
      "activations/layer19_attention_weight_min": -29.489736557006836,
      "activations/layer1_attention_weight_max": 16.584402084350586,
      "activations/layer1_attention_weight_min": -15.779038429260254,
      "activations/layer20_attention_weight_max": 30.55242156982422,
      "activations/layer20_attention_weight_min": -24.692514419555664,
      "activations/layer21_attention_weight_max": 26.439231872558594,
      "activations/layer21_attention_weight_min": -24.396495819091797,
      "activations/layer22_attention_weight_max": 42.12664794921875,
      "activations/layer22_attention_weight_min": -30.870405197143555,
      "activations/layer23_attention_weight_max": 32.89185333251953,
      "activations/layer23_attention_weight_min": -23.504636764526367,
      "activations/layer2_attention_weight_max": 32.382503509521484,
      "activations/layer2_attention_weight_min": -32.5266227722168,
      "activations/layer3_attention_weight_max": 93.60639190673828,
      "activations/layer3_attention_weight_min": -96.7565689086914,
      "activations/layer4_attention_weight_max": 116.66057586669922,
      "activations/layer4_attention_weight_min": -123.08528137207031,
      "activations/layer5_attention_weight_max": 55.447593688964844,
      "activations/layer5_attention_weight_min": -64.20693969726562,
      "activations/layer6_attention_weight_max": 45.243412017822266,
      "activations/layer6_attention_weight_min": -52.65248489379883,
      "activations/layer7_attention_weight_max": 89.69654083251953,
      "activations/layer7_attention_weight_min": -99.33493041992188,
      "activations/layer8_attention_weight_max": 41.19915771484375,
      "activations/layer8_attention_weight_min": -41.928070068359375,
      "activations/layer9_attention_weight_max": 35.42263412475586,
      "activations/layer9_attention_weight_min": -35.69065475463867,
      "epoch": 8.77,
      "learning_rate": 9.436893939393937e-05,
      "loss": 2.786,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.548,
      "eval_samples_per_second": 502.343,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.548,
      "eval_openwebtext_samples_per_second": 502.343,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_wikitext_loss": 2.978515625,
      "eval_wikitext_ppl": 19.658614220966218,
      "eval_wikitext_runtime": 2.0009,
      "eval_wikitext_samples_per_second": 227.903,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_lambada_loss": 2.53515625,
      "eval_lambada_ppl": 12.618402319374093,
      "eval_lambada_runtime": 9.6706,
      "eval_lambada_samples_per_second": 503.486,
      "step": 151000
    },
    {
      "activations/layer0_attention_weight_max": 16.158649444580078,
      "activations/layer0_attention_weight_min": -13.73958683013916,
      "activations/layer10_attention_weight_max": 31.215593338012695,
      "activations/layer10_attention_weight_min": -31.943931579589844,
      "activations/layer11_attention_weight_max": 31.69084930419922,
      "activations/layer11_attention_weight_min": -31.5277156829834,
      "activations/layer12_attention_weight_max": 27.852027893066406,
      "activations/layer12_attention_weight_min": -24.123992919921875,
      "activations/layer13_attention_weight_max": 37.73794174194336,
      "activations/layer13_attention_weight_min": -33.02751922607422,
      "activations/layer14_attention_weight_max": 53.66450881958008,
      "activations/layer14_attention_weight_min": -40.84693908691406,
      "activations/layer15_attention_weight_max": 35.84086227416992,
      "activations/layer15_attention_weight_min": -34.65932846069336,
      "activations/layer16_attention_weight_max": 32.977378845214844,
      "activations/layer16_attention_weight_min": -26.936603546142578,
      "activations/layer17_attention_weight_max": 30.309139251708984,
      "activations/layer17_attention_weight_min": -26.807788848876953,
      "activations/layer18_attention_weight_max": 32.79360580444336,
      "activations/layer18_attention_weight_min": -23.056198120117188,
      "activations/layer19_attention_weight_max": 41.04328536987305,
      "activations/layer19_attention_weight_min": -33.19204330444336,
      "activations/layer1_attention_weight_max": 16.558502197265625,
      "activations/layer1_attention_weight_min": -13.493391990661621,
      "activations/layer20_attention_weight_max": 31.714448928833008,
      "activations/layer20_attention_weight_min": -27.3399658203125,
      "activations/layer21_attention_weight_max": 28.51541519165039,
      "activations/layer21_attention_weight_min": -24.704898834228516,
      "activations/layer22_attention_weight_max": 45.296504974365234,
      "activations/layer22_attention_weight_min": -30.40045166015625,
      "activations/layer23_attention_weight_max": 38.04515838623047,
      "activations/layer23_attention_weight_min": -23.681255340576172,
      "activations/layer2_attention_weight_max": 31.653961181640625,
      "activations/layer2_attention_weight_min": -31.46497344970703,
      "activations/layer3_attention_weight_max": 92.06005859375,
      "activations/layer3_attention_weight_min": -91.73994445800781,
      "activations/layer4_attention_weight_max": 109.80391693115234,
      "activations/layer4_attention_weight_min": -108.889892578125,
      "activations/layer5_attention_weight_max": 53.405059814453125,
      "activations/layer5_attention_weight_min": -61.58124542236328,
      "activations/layer6_attention_weight_max": 46.973655700683594,
      "activations/layer6_attention_weight_min": -48.67800521850586,
      "activations/layer7_attention_weight_max": 88.82881927490234,
      "activations/layer7_attention_weight_min": -89.07887268066406,
      "activations/layer8_attention_weight_max": 39.46524429321289,
      "activations/layer8_attention_weight_min": -40.17144012451172,
      "activations/layer9_attention_weight_max": 34.0716667175293,
      "activations/layer9_attention_weight_min": -32.85190963745117,
      "epoch": 8.78,
      "learning_rate": 9.434999999999999e-05,
      "loss": 2.8087,
      "step": 151050
    },
    {
      "activations/layer0_attention_weight_max": 16.125198364257812,
      "activations/layer0_attention_weight_min": -12.34620189666748,
      "activations/layer10_attention_weight_max": 31.817663192749023,
      "activations/layer10_attention_weight_min": -35.54195022583008,
      "activations/layer11_attention_weight_max": 32.935245513916016,
      "activations/layer11_attention_weight_min": -29.78497314453125,
      "activations/layer12_attention_weight_max": 24.921649932861328,
      "activations/layer12_attention_weight_min": -29.17557144165039,
      "activations/layer13_attention_weight_max": 41.45123291015625,
      "activations/layer13_attention_weight_min": -35.16752624511719,
      "activations/layer14_attention_weight_max": 48.03250503540039,
      "activations/layer14_attention_weight_min": -39.230838775634766,
      "activations/layer15_attention_weight_max": 36.85392379760742,
      "activations/layer15_attention_weight_min": -31.92888832092285,
      "activations/layer16_attention_weight_max": 36.6043586730957,
      "activations/layer16_attention_weight_min": -29.057109832763672,
      "activations/layer17_attention_weight_max": 35.78886413574219,
      "activations/layer17_attention_weight_min": -25.380821228027344,
      "activations/layer18_attention_weight_max": 34.357059478759766,
      "activations/layer18_attention_weight_min": -23.264850616455078,
      "activations/layer19_attention_weight_max": 43.061336517333984,
      "activations/layer19_attention_weight_min": -31.48085594177246,
      "activations/layer1_attention_weight_max": 16.685623168945312,
      "activations/layer1_attention_weight_min": -14.503098487854004,
      "activations/layer20_attention_weight_max": 37.29350662231445,
      "activations/layer20_attention_weight_min": -25.37455940246582,
      "activations/layer21_attention_weight_max": 35.97134017944336,
      "activations/layer21_attention_weight_min": -22.76474952697754,
      "activations/layer22_attention_weight_max": 55.88638687133789,
      "activations/layer22_attention_weight_min": -28.04863166809082,
      "activations/layer23_attention_weight_max": 44.9152946472168,
      "activations/layer23_attention_weight_min": -25.077123641967773,
      "activations/layer2_attention_weight_max": 33.37757110595703,
      "activations/layer2_attention_weight_min": -30.70387077331543,
      "activations/layer3_attention_weight_max": 90.47977447509766,
      "activations/layer3_attention_weight_min": -93.107177734375,
      "activations/layer4_attention_weight_max": 116.30571746826172,
      "activations/layer4_attention_weight_min": -110.14649963378906,
      "activations/layer5_attention_weight_max": 53.34050750732422,
      "activations/layer5_attention_weight_min": -63.758968353271484,
      "activations/layer6_attention_weight_max": 47.48807144165039,
      "activations/layer6_attention_weight_min": -50.01604080200195,
      "activations/layer7_attention_weight_max": 84.522705078125,
      "activations/layer7_attention_weight_min": -83.13095092773438,
      "activations/layer8_attention_weight_max": 40.69429397583008,
      "activations/layer8_attention_weight_min": -39.15357971191406,
      "activations/layer9_attention_weight_max": 33.75335693359375,
      "activations/layer9_attention_weight_min": -34.05118942260742,
      "epoch": 8.78,
      "learning_rate": 9.43310606060606e-05,
      "loss": 2.8143,
      "step": 151100
    },
    {
      "activations/layer0_attention_weight_max": 16.163597106933594,
      "activations/layer0_attention_weight_min": -12.982841491699219,
      "activations/layer10_attention_weight_max": 33.58012771606445,
      "activations/layer10_attention_weight_min": -35.12396240234375,
      "activations/layer11_attention_weight_max": 33.091064453125,
      "activations/layer11_attention_weight_min": -32.075172424316406,
      "activations/layer12_attention_weight_max": 25.03728675842285,
      "activations/layer12_attention_weight_min": -25.341691970825195,
      "activations/layer13_attention_weight_max": 39.89829635620117,
      "activations/layer13_attention_weight_min": -35.16689682006836,
      "activations/layer14_attention_weight_max": 52.997108459472656,
      "activations/layer14_attention_weight_min": -42.62498474121094,
      "activations/layer15_attention_weight_max": 35.50676345825195,
      "activations/layer15_attention_weight_min": -32.49481201171875,
      "activations/layer16_attention_weight_max": 33.161617279052734,
      "activations/layer16_attention_weight_min": -31.531957626342773,
      "activations/layer17_attention_weight_max": 35.75359344482422,
      "activations/layer17_attention_weight_min": -26.90460205078125,
      "activations/layer18_attention_weight_max": 30.688982009887695,
      "activations/layer18_attention_weight_min": -24.92916488647461,
      "activations/layer19_attention_weight_max": 40.002689361572266,
      "activations/layer19_attention_weight_min": -32.84316635131836,
      "activations/layer1_attention_weight_max": 15.795246124267578,
      "activations/layer1_attention_weight_min": -14.79199504852295,
      "activations/layer20_attention_weight_max": 35.48991012573242,
      "activations/layer20_attention_weight_min": -26.470129013061523,
      "activations/layer21_attention_weight_max": 35.901676177978516,
      "activations/layer21_attention_weight_min": -26.277908325195312,
      "activations/layer22_attention_weight_max": 57.91880416870117,
      "activations/layer22_attention_weight_min": -31.922449111938477,
      "activations/layer23_attention_weight_max": 39.065364837646484,
      "activations/layer23_attention_weight_min": -26.139087677001953,
      "activations/layer2_attention_weight_max": 33.58888244628906,
      "activations/layer2_attention_weight_min": -32.4517822265625,
      "activations/layer3_attention_weight_max": 93.48714447021484,
      "activations/layer3_attention_weight_min": -98.47930908203125,
      "activations/layer4_attention_weight_max": 114.4022445678711,
      "activations/layer4_attention_weight_min": -119.15665435791016,
      "activations/layer5_attention_weight_max": 52.14553451538086,
      "activations/layer5_attention_weight_min": -59.76006317138672,
      "activations/layer6_attention_weight_max": 49.84339904785156,
      "activations/layer6_attention_weight_min": -50.58943557739258,
      "activations/layer7_attention_weight_max": 87.82545471191406,
      "activations/layer7_attention_weight_min": -89.66068267822266,
      "activations/layer8_attention_weight_max": 39.86114501953125,
      "activations/layer8_attention_weight_min": -42.18977355957031,
      "activations/layer9_attention_weight_max": 34.9248161315918,
      "activations/layer9_attention_weight_min": -35.90730667114258,
      "epoch": 8.78,
      "learning_rate": 9.43121212121212e-05,
      "loss": 2.8016,
      "step": 151150
    },
    {
      "activations/layer0_attention_weight_max": 15.152234077453613,
      "activations/layer0_attention_weight_min": -12.116778373718262,
      "activations/layer10_attention_weight_max": 37.39218521118164,
      "activations/layer10_attention_weight_min": -34.20539093017578,
      "activations/layer11_attention_weight_max": 38.16451644897461,
      "activations/layer11_attention_weight_min": -32.2679443359375,
      "activations/layer12_attention_weight_max": 27.647058486938477,
      "activations/layer12_attention_weight_min": -27.693016052246094,
      "activations/layer13_attention_weight_max": 37.57659912109375,
      "activations/layer13_attention_weight_min": -32.91135025024414,
      "activations/layer14_attention_weight_max": 43.88581848144531,
      "activations/layer14_attention_weight_min": -38.112220764160156,
      "activations/layer15_attention_weight_max": 33.92007064819336,
      "activations/layer15_attention_weight_min": -30.344600677490234,
      "activations/layer16_attention_weight_max": 32.997344970703125,
      "activations/layer16_attention_weight_min": -27.901243209838867,
      "activations/layer17_attention_weight_max": 31.641897201538086,
      "activations/layer17_attention_weight_min": -25.780311584472656,
      "activations/layer18_attention_weight_max": 29.17138671875,
      "activations/layer18_attention_weight_min": -21.543916702270508,
      "activations/layer19_attention_weight_max": 36.256568908691406,
      "activations/layer19_attention_weight_min": -29.53791618347168,
      "activations/layer1_attention_weight_max": 17.11542510986328,
      "activations/layer1_attention_weight_min": -14.311912536621094,
      "activations/layer20_attention_weight_max": 28.123882293701172,
      "activations/layer20_attention_weight_min": -22.901704788208008,
      "activations/layer21_attention_weight_max": 28.20009422302246,
      "activations/layer21_attention_weight_min": -21.73929786682129,
      "activations/layer22_attention_weight_max": 42.13273239135742,
      "activations/layer22_attention_weight_min": -27.555118560791016,
      "activations/layer23_attention_weight_max": 37.58913803100586,
      "activations/layer23_attention_weight_min": -23.392345428466797,
      "activations/layer2_attention_weight_max": 32.77455520629883,
      "activations/layer2_attention_weight_min": -34.001216888427734,
      "activations/layer3_attention_weight_max": 95.62926483154297,
      "activations/layer3_attention_weight_min": -96.83736419677734,
      "activations/layer4_attention_weight_max": 118.5271224975586,
      "activations/layer4_attention_weight_min": -115.16472625732422,
      "activations/layer5_attention_weight_max": 58.04010772705078,
      "activations/layer5_attention_weight_min": -60.592369079589844,
      "activations/layer6_attention_weight_max": 48.60268020629883,
      "activations/layer6_attention_weight_min": -49.05036163330078,
      "activations/layer7_attention_weight_max": 106.1971206665039,
      "activations/layer7_attention_weight_min": -90.9610366821289,
      "activations/layer8_attention_weight_max": 48.454444885253906,
      "activations/layer8_attention_weight_min": -41.786048889160156,
      "activations/layer9_attention_weight_max": 36.65996551513672,
      "activations/layer9_attention_weight_min": -35.90157699584961,
      "epoch": 8.79,
      "learning_rate": 9.429318181818181e-05,
      "loss": 2.7903,
      "step": 151200
    },
    {
      "activations/layer0_attention_weight_max": 16.277210235595703,
      "activations/layer0_attention_weight_min": -12.78038215637207,
      "activations/layer10_attention_weight_max": 31.32254409790039,
      "activations/layer10_attention_weight_min": -32.745269775390625,
      "activations/layer11_attention_weight_max": 32.17040252685547,
      "activations/layer11_attention_weight_min": -30.456401824951172,
      "activations/layer12_attention_weight_max": 28.530733108520508,
      "activations/layer12_attention_weight_min": -23.722248077392578,
      "activations/layer13_attention_weight_max": 38.610504150390625,
      "activations/layer13_attention_weight_min": -33.727439880371094,
      "activations/layer14_attention_weight_max": 49.858489990234375,
      "activations/layer14_attention_weight_min": -40.38990020751953,
      "activations/layer15_attention_weight_max": 35.584320068359375,
      "activations/layer15_attention_weight_min": -32.002716064453125,
      "activations/layer16_attention_weight_max": 29.313512802124023,
      "activations/layer16_attention_weight_min": -27.46791648864746,
      "activations/layer17_attention_weight_max": 36.935997009277344,
      "activations/layer17_attention_weight_min": -25.07134246826172,
      "activations/layer18_attention_weight_max": 32.71832275390625,
      "activations/layer18_attention_weight_min": -21.69080352783203,
      "activations/layer19_attention_weight_max": 33.26770782470703,
      "activations/layer19_attention_weight_min": -30.432392120361328,
      "activations/layer1_attention_weight_max": 16.859304428100586,
      "activations/layer1_attention_weight_min": -14.182018280029297,
      "activations/layer20_attention_weight_max": 27.076457977294922,
      "activations/layer20_attention_weight_min": -23.789134979248047,
      "activations/layer21_attention_weight_max": 28.191082000732422,
      "activations/layer21_attention_weight_min": -22.926544189453125,
      "activations/layer22_attention_weight_max": 49.14468765258789,
      "activations/layer22_attention_weight_min": -28.553180694580078,
      "activations/layer23_attention_weight_max": 34.01091003417969,
      "activations/layer23_attention_weight_min": -25.229045867919922,
      "activations/layer2_attention_weight_max": 31.826690673828125,
      "activations/layer2_attention_weight_min": -31.079936981201172,
      "activations/layer3_attention_weight_max": 92.89027404785156,
      "activations/layer3_attention_weight_min": -95.07396697998047,
      "activations/layer4_attention_weight_max": 113.71944427490234,
      "activations/layer4_attention_weight_min": -120.1485824584961,
      "activations/layer5_attention_weight_max": 52.24006652832031,
      "activations/layer5_attention_weight_min": -64.73828125,
      "activations/layer6_attention_weight_max": 44.94211196899414,
      "activations/layer6_attention_weight_min": -51.54922103881836,
      "activations/layer7_attention_weight_max": 85.95695495605469,
      "activations/layer7_attention_weight_min": -90.92144012451172,
      "activations/layer8_attention_weight_max": 37.97795486450195,
      "activations/layer8_attention_weight_min": -39.86763381958008,
      "activations/layer9_attention_weight_max": 33.697322845458984,
      "activations/layer9_attention_weight_min": -32.830101013183594,
      "epoch": 8.79,
      "learning_rate": 9.427424242424241e-05,
      "loss": 2.795,
      "step": 151250
    },
    {
      "activations/layer0_attention_weight_max": 15.620972633361816,
      "activations/layer0_attention_weight_min": -12.478693008422852,
      "activations/layer10_attention_weight_max": 32.208839416503906,
      "activations/layer10_attention_weight_min": -32.311073303222656,
      "activations/layer11_attention_weight_max": 32.86927795410156,
      "activations/layer11_attention_weight_min": -30.410436630249023,
      "activations/layer12_attention_weight_max": 25.734098434448242,
      "activations/layer12_attention_weight_min": -23.58100700378418,
      "activations/layer13_attention_weight_max": 44.30338668823242,
      "activations/layer13_attention_weight_min": -34.346458435058594,
      "activations/layer14_attention_weight_max": 45.65110778808594,
      "activations/layer14_attention_weight_min": -37.20698165893555,
      "activations/layer15_attention_weight_max": 39.23554229736328,
      "activations/layer15_attention_weight_min": -30.483064651489258,
      "activations/layer16_attention_weight_max": 32.508785247802734,
      "activations/layer16_attention_weight_min": -28.595643997192383,
      "activations/layer17_attention_weight_max": 33.62264633178711,
      "activations/layer17_attention_weight_min": -25.655101776123047,
      "activations/layer18_attention_weight_max": 32.1899299621582,
      "activations/layer18_attention_weight_min": -22.524381637573242,
      "activations/layer19_attention_weight_max": 37.2757682800293,
      "activations/layer19_attention_weight_min": -30.495349884033203,
      "activations/layer1_attention_weight_max": 16.470678329467773,
      "activations/layer1_attention_weight_min": -14.732885360717773,
      "activations/layer20_attention_weight_max": 32.82324981689453,
      "activations/layer20_attention_weight_min": -25.143144607543945,
      "activations/layer21_attention_weight_max": 30.198226928710938,
      "activations/layer21_attention_weight_min": -23.420257568359375,
      "activations/layer22_attention_weight_max": 46.863807678222656,
      "activations/layer22_attention_weight_min": -30.55906867980957,
      "activations/layer23_attention_weight_max": 41.65695571899414,
      "activations/layer23_attention_weight_min": -26.788555145263672,
      "activations/layer2_attention_weight_max": 32.197731018066406,
      "activations/layer2_attention_weight_min": -32.19536590576172,
      "activations/layer3_attention_weight_max": 91.22183990478516,
      "activations/layer3_attention_weight_min": -93.33065032958984,
      "activations/layer4_attention_weight_max": 117.16783905029297,
      "activations/layer4_attention_weight_min": -115.24464416503906,
      "activations/layer5_attention_weight_max": 54.92698669433594,
      "activations/layer5_attention_weight_min": -59.9404296875,
      "activations/layer6_attention_weight_max": 46.60277557373047,
      "activations/layer6_attention_weight_min": -49.69739532470703,
      "activations/layer7_attention_weight_max": 87.42186737060547,
      "activations/layer7_attention_weight_min": -88.606201171875,
      "activations/layer8_attention_weight_max": 38.78014373779297,
      "activations/layer8_attention_weight_min": -41.36738967895508,
      "activations/layer9_attention_weight_max": 36.20404815673828,
      "activations/layer9_attention_weight_min": -36.362815856933594,
      "epoch": 8.79,
      "learning_rate": 9.425530303030302e-05,
      "loss": 2.7911,
      "step": 151300
    },
    {
      "activations/layer0_attention_weight_max": 16.113977432250977,
      "activations/layer0_attention_weight_min": -12.693873405456543,
      "activations/layer10_attention_weight_max": 33.473785400390625,
      "activations/layer10_attention_weight_min": -32.789886474609375,
      "activations/layer11_attention_weight_max": 39.25811767578125,
      "activations/layer11_attention_weight_min": -35.03136444091797,
      "activations/layer12_attention_weight_max": 24.321866989135742,
      "activations/layer12_attention_weight_min": -28.22699737548828,
      "activations/layer13_attention_weight_max": 41.626914978027344,
      "activations/layer13_attention_weight_min": -34.68006134033203,
      "activations/layer14_attention_weight_max": 46.827518463134766,
      "activations/layer14_attention_weight_min": -37.271400451660156,
      "activations/layer15_attention_weight_max": 36.18753433227539,
      "activations/layer15_attention_weight_min": -29.818750381469727,
      "activations/layer16_attention_weight_max": 33.903656005859375,
      "activations/layer16_attention_weight_min": -28.250070571899414,
      "activations/layer17_attention_weight_max": 35.22217559814453,
      "activations/layer17_attention_weight_min": -25.000812530517578,
      "activations/layer18_attention_weight_max": 32.844451904296875,
      "activations/layer18_attention_weight_min": -21.225521087646484,
      "activations/layer19_attention_weight_max": 40.84210968017578,
      "activations/layer19_attention_weight_min": -30.12529945373535,
      "activations/layer1_attention_weight_max": 15.976298332214355,
      "activations/layer1_attention_weight_min": -13.877816200256348,
      "activations/layer20_attention_weight_max": 33.312191009521484,
      "activations/layer20_attention_weight_min": -24.253660202026367,
      "activations/layer21_attention_weight_max": 30.601842880249023,
      "activations/layer21_attention_weight_min": -23.044490814208984,
      "activations/layer22_attention_weight_max": 48.12273406982422,
      "activations/layer22_attention_weight_min": -28.6143798828125,
      "activations/layer23_attention_weight_max": 36.88216018676758,
      "activations/layer23_attention_weight_min": -23.59661865234375,
      "activations/layer2_attention_weight_max": 32.33259963989258,
      "activations/layer2_attention_weight_min": -33.717437744140625,
      "activations/layer3_attention_weight_max": 95.72613525390625,
      "activations/layer3_attention_weight_min": -98.9369888305664,
      "activations/layer4_attention_weight_max": 119.3126220703125,
      "activations/layer4_attention_weight_min": -117.4068603515625,
      "activations/layer5_attention_weight_max": 55.296810150146484,
      "activations/layer5_attention_weight_min": -60.87696075439453,
      "activations/layer6_attention_weight_max": 49.69597244262695,
      "activations/layer6_attention_weight_min": -48.49991989135742,
      "activations/layer7_attention_weight_max": 98.3140869140625,
      "activations/layer7_attention_weight_min": -88.33123016357422,
      "activations/layer8_attention_weight_max": 41.66838455200195,
      "activations/layer8_attention_weight_min": -40.42410659790039,
      "activations/layer9_attention_weight_max": 34.42744064331055,
      "activations/layer9_attention_weight_min": -33.35151290893555,
      "epoch": 8.79,
      "learning_rate": 9.423636363636363e-05,
      "loss": 2.8021,
      "step": 151350
    },
    {
      "activations/layer0_attention_weight_max": 15.701577186584473,
      "activations/layer0_attention_weight_min": -12.164112091064453,
      "activations/layer10_attention_weight_max": 40.398719787597656,
      "activations/layer10_attention_weight_min": -35.52247619628906,
      "activations/layer11_attention_weight_max": 41.76136779785156,
      "activations/layer11_attention_weight_min": -35.967193603515625,
      "activations/layer12_attention_weight_max": 25.558969497680664,
      "activations/layer12_attention_weight_min": -24.028806686401367,
      "activations/layer13_attention_weight_max": 40.83357238769531,
      "activations/layer13_attention_weight_min": -32.658714294433594,
      "activations/layer14_attention_weight_max": 50.552528381347656,
      "activations/layer14_attention_weight_min": -37.430633544921875,
      "activations/layer15_attention_weight_max": 36.84274673461914,
      "activations/layer15_attention_weight_min": -29.491724014282227,
      "activations/layer16_attention_weight_max": 34.56056594848633,
      "activations/layer16_attention_weight_min": -27.774520874023438,
      "activations/layer17_attention_weight_max": 33.836605072021484,
      "activations/layer17_attention_weight_min": -25.113149642944336,
      "activations/layer18_attention_weight_max": 30.9189453125,
      "activations/layer18_attention_weight_min": -22.755598068237305,
      "activations/layer19_attention_weight_max": 40.20104217529297,
      "activations/layer19_attention_weight_min": -32.4364013671875,
      "activations/layer1_attention_weight_max": 17.446378707885742,
      "activations/layer1_attention_weight_min": -13.993792533874512,
      "activations/layer20_attention_weight_max": 31.10899543762207,
      "activations/layer20_attention_weight_min": -26.125337600708008,
      "activations/layer21_attention_weight_max": 31.041717529296875,
      "activations/layer21_attention_weight_min": -24.63443374633789,
      "activations/layer22_attention_weight_max": 48.35606384277344,
      "activations/layer22_attention_weight_min": -29.81800079345703,
      "activations/layer23_attention_weight_max": 37.52223205566406,
      "activations/layer23_attention_weight_min": -23.587711334228516,
      "activations/layer2_attention_weight_max": 32.99424743652344,
      "activations/layer2_attention_weight_min": -30.434751510620117,
      "activations/layer3_attention_weight_max": 96.1227035522461,
      "activations/layer3_attention_weight_min": -92.44064331054688,
      "activations/layer4_attention_weight_max": 119.13846588134766,
      "activations/layer4_attention_weight_min": -114.60472869873047,
      "activations/layer5_attention_weight_max": 54.70111083984375,
      "activations/layer5_attention_weight_min": -59.517906188964844,
      "activations/layer6_attention_weight_max": 47.34693908691406,
      "activations/layer6_attention_weight_min": -52.315223693847656,
      "activations/layer7_attention_weight_max": 102.60133361816406,
      "activations/layer7_attention_weight_min": -88.71922302246094,
      "activations/layer8_attention_weight_max": 48.083213806152344,
      "activations/layer8_attention_weight_min": -43.34556198120117,
      "activations/layer9_attention_weight_max": 41.80826187133789,
      "activations/layer9_attention_weight_min": -36.6822624206543,
      "epoch": 8.8,
      "learning_rate": 9.421742424242423e-05,
      "loss": 2.7976,
      "step": 151400
    },
    {
      "activations/layer0_attention_weight_max": 16.351009368896484,
      "activations/layer0_attention_weight_min": -12.963236808776855,
      "activations/layer10_attention_weight_max": 32.51240921020508,
      "activations/layer10_attention_weight_min": -30.650774002075195,
      "activations/layer11_attention_weight_max": 31.489547729492188,
      "activations/layer11_attention_weight_min": -29.6087703704834,
      "activations/layer12_attention_weight_max": 22.696189880371094,
      "activations/layer12_attention_weight_min": -24.897525787353516,
      "activations/layer13_attention_weight_max": 39.45158004760742,
      "activations/layer13_attention_weight_min": -32.47167205810547,
      "activations/layer14_attention_weight_max": 45.3238525390625,
      "activations/layer14_attention_weight_min": -39.159481048583984,
      "activations/layer15_attention_weight_max": 36.338565826416016,
      "activations/layer15_attention_weight_min": -30.564722061157227,
      "activations/layer16_attention_weight_max": 35.42233657836914,
      "activations/layer16_attention_weight_min": -27.584972381591797,
      "activations/layer17_attention_weight_max": 31.275651931762695,
      "activations/layer17_attention_weight_min": -25.90952491760254,
      "activations/layer18_attention_weight_max": 30.842031478881836,
      "activations/layer18_attention_weight_min": -21.44942283630371,
      "activations/layer19_attention_weight_max": 38.15095520019531,
      "activations/layer19_attention_weight_min": -30.6304874420166,
      "activations/layer1_attention_weight_max": 16.26822280883789,
      "activations/layer1_attention_weight_min": -14.822668075561523,
      "activations/layer20_attention_weight_max": 29.468948364257812,
      "activations/layer20_attention_weight_min": -22.967660903930664,
      "activations/layer21_attention_weight_max": 26.115001678466797,
      "activations/layer21_attention_weight_min": -22.110471725463867,
      "activations/layer22_attention_weight_max": 43.45284652709961,
      "activations/layer22_attention_weight_min": -30.340375900268555,
      "activations/layer23_attention_weight_max": 34.235618591308594,
      "activations/layer23_attention_weight_min": -23.396282196044922,
      "activations/layer2_attention_weight_max": 32.17668151855469,
      "activations/layer2_attention_weight_min": -31.07636260986328,
      "activations/layer3_attention_weight_max": 94.63794708251953,
      "activations/layer3_attention_weight_min": -93.58882141113281,
      "activations/layer4_attention_weight_max": 119.0520248413086,
      "activations/layer4_attention_weight_min": -114.03873443603516,
      "activations/layer5_attention_weight_max": 55.928802490234375,
      "activations/layer5_attention_weight_min": -61.430335998535156,
      "activations/layer6_attention_weight_max": 47.67805862426758,
      "activations/layer6_attention_weight_min": -52.021812438964844,
      "activations/layer7_attention_weight_max": 89.38156127929688,
      "activations/layer7_attention_weight_min": -84.46495056152344,
      "activations/layer8_attention_weight_max": 39.63545227050781,
      "activations/layer8_attention_weight_min": -40.364051818847656,
      "activations/layer9_attention_weight_max": 34.13671112060547,
      "activations/layer9_attention_weight_min": -34.13300704956055,
      "epoch": 8.8,
      "learning_rate": 9.419848484848484e-05,
      "loss": 2.7977,
      "step": 151450
    },
    {
      "activations/layer0_attention_weight_max": 16.29218101501465,
      "activations/layer0_attention_weight_min": -12.885212898254395,
      "activations/layer10_attention_weight_max": 31.938777923583984,
      "activations/layer10_attention_weight_min": -32.46338653564453,
      "activations/layer11_attention_weight_max": 33.08827209472656,
      "activations/layer11_attention_weight_min": -30.216625213623047,
      "activations/layer12_attention_weight_max": 24.286819458007812,
      "activations/layer12_attention_weight_min": -24.6121768951416,
      "activations/layer13_attention_weight_max": 42.49586486816406,
      "activations/layer13_attention_weight_min": -33.66560745239258,
      "activations/layer14_attention_weight_max": 53.07979965209961,
      "activations/layer14_attention_weight_min": -39.590457916259766,
      "activations/layer15_attention_weight_max": 37.31999969482422,
      "activations/layer15_attention_weight_min": -31.1575984954834,
      "activations/layer16_attention_weight_max": 33.75033950805664,
      "activations/layer16_attention_weight_min": -30.210708618164062,
      "activations/layer17_attention_weight_max": 34.315223693847656,
      "activations/layer17_attention_weight_min": -26.02750587463379,
      "activations/layer18_attention_weight_max": 33.3465690612793,
      "activations/layer18_attention_weight_min": -22.012662887573242,
      "activations/layer19_attention_weight_max": 40.59722137451172,
      "activations/layer19_attention_weight_min": -29.73431968688965,
      "activations/layer1_attention_weight_max": 16.701086044311523,
      "activations/layer1_attention_weight_min": -14.601214408874512,
      "activations/layer20_attention_weight_max": 35.28824996948242,
      "activations/layer20_attention_weight_min": -24.932804107666016,
      "activations/layer21_attention_weight_max": 31.48863983154297,
      "activations/layer21_attention_weight_min": -22.772619247436523,
      "activations/layer22_attention_weight_max": 54.192325592041016,
      "activations/layer22_attention_weight_min": -29.10959815979004,
      "activations/layer23_attention_weight_max": 38.96506881713867,
      "activations/layer23_attention_weight_min": -22.45306968688965,
      "activations/layer2_attention_weight_max": 31.15218734741211,
      "activations/layer2_attention_weight_min": -30.36263084411621,
      "activations/layer3_attention_weight_max": 91.40939331054688,
      "activations/layer3_attention_weight_min": -90.52881622314453,
      "activations/layer4_attention_weight_max": 115.49346923828125,
      "activations/layer4_attention_weight_min": -115.133544921875,
      "activations/layer5_attention_weight_max": 52.17252731323242,
      "activations/layer5_attention_weight_min": -63.447566986083984,
      "activations/layer6_attention_weight_max": 46.80513381958008,
      "activations/layer6_attention_weight_min": -52.60768127441406,
      "activations/layer7_attention_weight_max": 90.3675308227539,
      "activations/layer7_attention_weight_min": -90.65519714355469,
      "activations/layer8_attention_weight_max": 39.29191589355469,
      "activations/layer8_attention_weight_min": -40.65388107299805,
      "activations/layer9_attention_weight_max": 34.45458984375,
      "activations/layer9_attention_weight_min": -32.540714263916016,
      "epoch": 8.8,
      "learning_rate": 9.417954545454546e-05,
      "loss": 2.7927,
      "step": 151500
    },
    {
      "activations/layer0_attention_weight_max": 15.869154930114746,
      "activations/layer0_attention_weight_min": -12.127181053161621,
      "activations/layer10_attention_weight_max": 43.70440673828125,
      "activations/layer10_attention_weight_min": -37.8238525390625,
      "activations/layer11_attention_weight_max": 42.769737243652344,
      "activations/layer11_attention_weight_min": -35.854644775390625,
      "activations/layer12_attention_weight_max": 24.29680061340332,
      "activations/layer12_attention_weight_min": -41.160247802734375,
      "activations/layer13_attention_weight_max": 39.33940505981445,
      "activations/layer13_attention_weight_min": -33.46914291381836,
      "activations/layer14_attention_weight_max": 57.819580078125,
      "activations/layer14_attention_weight_min": -46.31747055053711,
      "activations/layer15_attention_weight_max": 35.923404693603516,
      "activations/layer15_attention_weight_min": -30.840349197387695,
      "activations/layer16_attention_weight_max": 34.25907516479492,
      "activations/layer16_attention_weight_min": -27.330461502075195,
      "activations/layer17_attention_weight_max": 30.5274600982666,
      "activations/layer17_attention_weight_min": -24.867536544799805,
      "activations/layer18_attention_weight_max": 27.6480770111084,
      "activations/layer18_attention_weight_min": -21.59605598449707,
      "activations/layer19_attention_weight_max": 34.50847244262695,
      "activations/layer19_attention_weight_min": -28.790063858032227,
      "activations/layer1_attention_weight_max": 17.299457550048828,
      "activations/layer1_attention_weight_min": -15.126836776733398,
      "activations/layer20_attention_weight_max": 28.178691864013672,
      "activations/layer20_attention_weight_min": -21.47801971435547,
      "activations/layer21_attention_weight_max": 27.50606918334961,
      "activations/layer21_attention_weight_min": -20.913654327392578,
      "activations/layer22_attention_weight_max": 42.040958404541016,
      "activations/layer22_attention_weight_min": -28.89290428161621,
      "activations/layer23_attention_weight_max": 34.55535888671875,
      "activations/layer23_attention_weight_min": -24.863325119018555,
      "activations/layer2_attention_weight_max": 34.853328704833984,
      "activations/layer2_attention_weight_min": -35.930931091308594,
      "activations/layer3_attention_weight_max": 101.1859130859375,
      "activations/layer3_attention_weight_min": -102.0210952758789,
      "activations/layer4_attention_weight_max": 126.22627258300781,
      "activations/layer4_attention_weight_min": -123.11671447753906,
      "activations/layer5_attention_weight_max": 59.263587951660156,
      "activations/layer5_attention_weight_min": -62.66162109375,
      "activations/layer6_attention_weight_max": 54.978843688964844,
      "activations/layer6_attention_weight_min": -52.950008392333984,
      "activations/layer7_attention_weight_max": 114.05986785888672,
      "activations/layer7_attention_weight_min": -105.42528533935547,
      "activations/layer8_attention_weight_max": 52.8770751953125,
      "activations/layer8_attention_weight_min": -48.661865234375,
      "activations/layer9_attention_weight_max": 44.330223083496094,
      "activations/layer9_attention_weight_min": -40.627349853515625,
      "epoch": 8.81,
      "learning_rate": 9.416060606060605e-05,
      "loss": 2.8307,
      "step": 151550
    },
    {
      "activations/layer0_attention_weight_max": 15.52702808380127,
      "activations/layer0_attention_weight_min": -12.606520652770996,
      "activations/layer10_attention_weight_max": 32.89976501464844,
      "activations/layer10_attention_weight_min": -31.906932830810547,
      "activations/layer11_attention_weight_max": 35.31303405761719,
      "activations/layer11_attention_weight_min": -33.141204833984375,
      "activations/layer12_attention_weight_max": 26.712684631347656,
      "activations/layer12_attention_weight_min": -24.488677978515625,
      "activations/layer13_attention_weight_max": 40.10417938232422,
      "activations/layer13_attention_weight_min": -34.0223503112793,
      "activations/layer14_attention_weight_max": 47.84109878540039,
      "activations/layer14_attention_weight_min": -37.644317626953125,
      "activations/layer15_attention_weight_max": 38.59220886230469,
      "activations/layer15_attention_weight_min": -30.169775009155273,
      "activations/layer16_attention_weight_max": 33.38808059692383,
      "activations/layer16_attention_weight_min": -27.293901443481445,
      "activations/layer17_attention_weight_max": 32.19944763183594,
      "activations/layer17_attention_weight_min": -25.498544692993164,
      "activations/layer18_attention_weight_max": 31.319091796875,
      "activations/layer18_attention_weight_min": -21.79299545288086,
      "activations/layer19_attention_weight_max": 37.506832122802734,
      "activations/layer19_attention_weight_min": -30.344953536987305,
      "activations/layer1_attention_weight_max": 16.164337158203125,
      "activations/layer1_attention_weight_min": -14.496927261352539,
      "activations/layer20_attention_weight_max": 31.146770477294922,
      "activations/layer20_attention_weight_min": -24.348915100097656,
      "activations/layer21_attention_weight_max": 32.116554260253906,
      "activations/layer21_attention_weight_min": -24.31011199951172,
      "activations/layer22_attention_weight_max": 46.95281982421875,
      "activations/layer22_attention_weight_min": -28.85785484313965,
      "activations/layer23_attention_weight_max": 34.07486343383789,
      "activations/layer23_attention_weight_min": -24.71999168395996,
      "activations/layer2_attention_weight_max": 32.443023681640625,
      "activations/layer2_attention_weight_min": -30.809837341308594,
      "activations/layer3_attention_weight_max": 95.53919219970703,
      "activations/layer3_attention_weight_min": -93.99948120117188,
      "activations/layer4_attention_weight_max": 119.39471435546875,
      "activations/layer4_attention_weight_min": -114.93135833740234,
      "activations/layer5_attention_weight_max": 52.9096794128418,
      "activations/layer5_attention_weight_min": -63.24851989746094,
      "activations/layer6_attention_weight_max": 49.11220932006836,
      "activations/layer6_attention_weight_min": -49.19951248168945,
      "activations/layer7_attention_weight_max": 83.08870697021484,
      "activations/layer7_attention_weight_min": -85.27583312988281,
      "activations/layer8_attention_weight_max": 39.897010803222656,
      "activations/layer8_attention_weight_min": -39.01631546020508,
      "activations/layer9_attention_weight_max": 33.56553268432617,
      "activations/layer9_attention_weight_min": -34.76679611206055,
      "epoch": 8.81,
      "learning_rate": 9.414166666666666e-05,
      "loss": 2.7966,
      "step": 151600
    },
    {
      "activations/layer0_attention_weight_max": 14.928829193115234,
      "activations/layer0_attention_weight_min": -13.037675857543945,
      "activations/layer10_attention_weight_max": 33.75263214111328,
      "activations/layer10_attention_weight_min": -35.31116485595703,
      "activations/layer11_attention_weight_max": 33.196937561035156,
      "activations/layer11_attention_weight_min": -32.2249755859375,
      "activations/layer12_attention_weight_max": 22.311548233032227,
      "activations/layer12_attention_weight_min": -26.322959899902344,
      "activations/layer13_attention_weight_max": 38.71464920043945,
      "activations/layer13_attention_weight_min": -37.087074279785156,
      "activations/layer14_attention_weight_max": 51.54185104370117,
      "activations/layer14_attention_weight_min": -39.036006927490234,
      "activations/layer15_attention_weight_max": 37.28882598876953,
      "activations/layer15_attention_weight_min": -30.78653907775879,
      "activations/layer16_attention_weight_max": 38.9181022644043,
      "activations/layer16_attention_weight_min": -31.00563621520996,
      "activations/layer17_attention_weight_max": 36.258270263671875,
      "activations/layer17_attention_weight_min": -27.0682430267334,
      "activations/layer18_attention_weight_max": 30.007427215576172,
      "activations/layer18_attention_weight_min": -20.672679901123047,
      "activations/layer19_attention_weight_max": 34.85771179199219,
      "activations/layer19_attention_weight_min": -33.319183349609375,
      "activations/layer1_attention_weight_max": 18.53897476196289,
      "activations/layer1_attention_weight_min": -14.89672565460205,
      "activations/layer20_attention_weight_max": 33.09850311279297,
      "activations/layer20_attention_weight_min": -23.141677856445312,
      "activations/layer21_attention_weight_max": 31.37082290649414,
      "activations/layer21_attention_weight_min": -22.3576602935791,
      "activations/layer22_attention_weight_max": 43.22317886352539,
      "activations/layer22_attention_weight_min": -31.215614318847656,
      "activations/layer23_attention_weight_max": 35.333274841308594,
      "activations/layer23_attention_weight_min": -25.27698516845703,
      "activations/layer2_attention_weight_max": 33.934226989746094,
      "activations/layer2_attention_weight_min": -34.064910888671875,
      "activations/layer3_attention_weight_max": 98.8916015625,
      "activations/layer3_attention_weight_min": -102.47322082519531,
      "activations/layer4_attention_weight_max": 122.74964904785156,
      "activations/layer4_attention_weight_min": -121.6574935913086,
      "activations/layer5_attention_weight_max": 56.30670166015625,
      "activations/layer5_attention_weight_min": -63.69927978515625,
      "activations/layer6_attention_weight_max": 49.61317825317383,
      "activations/layer6_attention_weight_min": -56.26237487792969,
      "activations/layer7_attention_weight_max": 91.69908905029297,
      "activations/layer7_attention_weight_min": -93.35267639160156,
      "activations/layer8_attention_weight_max": 41.438472747802734,
      "activations/layer8_attention_weight_min": -44.60054397583008,
      "activations/layer9_attention_weight_max": 36.55514144897461,
      "activations/layer9_attention_weight_min": -38.31495666503906,
      "epoch": 8.81,
      "learning_rate": 9.412272727272728e-05,
      "loss": 2.7975,
      "step": 151650
    },
    {
      "activations/layer0_attention_weight_max": 16.67502212524414,
      "activations/layer0_attention_weight_min": -12.99168872833252,
      "activations/layer10_attention_weight_max": 32.23353958129883,
      "activations/layer10_attention_weight_min": -31.348648071289062,
      "activations/layer11_attention_weight_max": 30.13072967529297,
      "activations/layer11_attention_weight_min": -27.95672607421875,
      "activations/layer12_attention_weight_max": 23.153539657592773,
      "activations/layer12_attention_weight_min": -27.24156951904297,
      "activations/layer13_attention_weight_max": 36.62047576904297,
      "activations/layer13_attention_weight_min": -33.37504577636719,
      "activations/layer14_attention_weight_max": 39.80216598510742,
      "activations/layer14_attention_weight_min": -34.43031311035156,
      "activations/layer15_attention_weight_max": 33.146240234375,
      "activations/layer15_attention_weight_min": -29.286151885986328,
      "activations/layer16_attention_weight_max": 29.385560989379883,
      "activations/layer16_attention_weight_min": -27.633451461791992,
      "activations/layer17_attention_weight_max": 31.61734390258789,
      "activations/layer17_attention_weight_min": -25.361299514770508,
      "activations/layer18_attention_weight_max": 32.63311004638672,
      "activations/layer18_attention_weight_min": -22.0313663482666,
      "activations/layer19_attention_weight_max": 30.636621475219727,
      "activations/layer19_attention_weight_min": -28.524118423461914,
      "activations/layer1_attention_weight_max": 15.642879486083984,
      "activations/layer1_attention_weight_min": -13.47603988647461,
      "activations/layer20_attention_weight_max": 27.011730194091797,
      "activations/layer20_attention_weight_min": -23.43844985961914,
      "activations/layer21_attention_weight_max": 28.47492027282715,
      "activations/layer21_attention_weight_min": -22.613616943359375,
      "activations/layer22_attention_weight_max": 41.54029083251953,
      "activations/layer22_attention_weight_min": -30.291337966918945,
      "activations/layer23_attention_weight_max": 32.80000686645508,
      "activations/layer23_attention_weight_min": -28.218463897705078,
      "activations/layer2_attention_weight_max": 29.564281463623047,
      "activations/layer2_attention_weight_min": -28.228361129760742,
      "activations/layer3_attention_weight_max": 88.19029998779297,
      "activations/layer3_attention_weight_min": -87.08583068847656,
      "activations/layer4_attention_weight_max": 112.2569580078125,
      "activations/layer4_attention_weight_min": -108.23839569091797,
      "activations/layer5_attention_weight_max": 54.334373474121094,
      "activations/layer5_attention_weight_min": -58.536376953125,
      "activations/layer6_attention_weight_max": 46.192771911621094,
      "activations/layer6_attention_weight_min": -48.979984283447266,
      "activations/layer7_attention_weight_max": 86.89766693115234,
      "activations/layer7_attention_weight_min": -86.10614776611328,
      "activations/layer8_attention_weight_max": 37.007869720458984,
      "activations/layer8_attention_weight_min": -39.99137878417969,
      "activations/layer9_attention_weight_max": 34.97575378417969,
      "activations/layer9_attention_weight_min": -32.345489501953125,
      "epoch": 8.81,
      "learning_rate": 9.410378787878786e-05,
      "loss": 2.7918,
      "step": 151700
    },
    {
      "activations/layer0_attention_weight_max": 15.00666618347168,
      "activations/layer0_attention_weight_min": -12.667445182800293,
      "activations/layer10_attention_weight_max": 30.0919189453125,
      "activations/layer10_attention_weight_min": -32.62519454956055,
      "activations/layer11_attention_weight_max": 31.709348678588867,
      "activations/layer11_attention_weight_min": -29.904569625854492,
      "activations/layer12_attention_weight_max": 22.868974685668945,
      "activations/layer12_attention_weight_min": -28.03641128540039,
      "activations/layer13_attention_weight_max": 37.221553802490234,
      "activations/layer13_attention_weight_min": -31.96428108215332,
      "activations/layer14_attention_weight_max": 46.90788650512695,
      "activations/layer14_attention_weight_min": -37.93875503540039,
      "activations/layer15_attention_weight_max": 35.02141189575195,
      "activations/layer15_attention_weight_min": -29.910226821899414,
      "activations/layer16_attention_weight_max": 32.98180389404297,
      "activations/layer16_attention_weight_min": -28.750932693481445,
      "activations/layer17_attention_weight_max": 32.8462028503418,
      "activations/layer17_attention_weight_min": -25.702913284301758,
      "activations/layer18_attention_weight_max": 32.90241241455078,
      "activations/layer18_attention_weight_min": -21.3145809173584,
      "activations/layer19_attention_weight_max": 37.30031967163086,
      "activations/layer19_attention_weight_min": -30.26593589782715,
      "activations/layer1_attention_weight_max": 17.386322021484375,
      "activations/layer1_attention_weight_min": -16.22679328918457,
      "activations/layer20_attention_weight_max": 31.918067932128906,
      "activations/layer20_attention_weight_min": -23.583768844604492,
      "activations/layer21_attention_weight_max": 31.1228084564209,
      "activations/layer21_attention_weight_min": -22.2796630859375,
      "activations/layer22_attention_weight_max": 51.03538513183594,
      "activations/layer22_attention_weight_min": -28.210758209228516,
      "activations/layer23_attention_weight_max": 36.23387908935547,
      "activations/layer23_attention_weight_min": -23.400543212890625,
      "activations/layer2_attention_weight_max": 30.396039962768555,
      "activations/layer2_attention_weight_min": -29.58430290222168,
      "activations/layer3_attention_weight_max": 90.99761962890625,
      "activations/layer3_attention_weight_min": -89.59480285644531,
      "activations/layer4_attention_weight_max": 110.89635467529297,
      "activations/layer4_attention_weight_min": -107.54691314697266,
      "activations/layer5_attention_weight_max": 55.72495651245117,
      "activations/layer5_attention_weight_min": -60.852394104003906,
      "activations/layer6_attention_weight_max": 47.15986633300781,
      "activations/layer6_attention_weight_min": -48.65040969848633,
      "activations/layer7_attention_weight_max": 87.4625015258789,
      "activations/layer7_attention_weight_min": -87.17064666748047,
      "activations/layer8_attention_weight_max": 39.01155471801758,
      "activations/layer8_attention_weight_min": -42.28477478027344,
      "activations/layer9_attention_weight_max": 32.60099411010742,
      "activations/layer9_attention_weight_min": -32.636314392089844,
      "epoch": 8.82,
      "learning_rate": 9.408484848484848e-05,
      "loss": 2.8178,
      "step": 151750
    },
    {
      "activations/layer0_attention_weight_max": 15.591449737548828,
      "activations/layer0_attention_weight_min": -12.055632591247559,
      "activations/layer10_attention_weight_max": 35.590431213378906,
      "activations/layer10_attention_weight_min": -33.04501724243164,
      "activations/layer11_attention_weight_max": 38.436309814453125,
      "activations/layer11_attention_weight_min": -31.787200927734375,
      "activations/layer12_attention_weight_max": 44.02143478393555,
      "activations/layer12_attention_weight_min": -23.509605407714844,
      "activations/layer13_attention_weight_max": 54.73068618774414,
      "activations/layer13_attention_weight_min": -31.640487670898438,
      "activations/layer14_attention_weight_max": 51.12175369262695,
      "activations/layer14_attention_weight_min": -40.668601989746094,
      "activations/layer15_attention_weight_max": 42.65040588378906,
      "activations/layer15_attention_weight_min": -30.649036407470703,
      "activations/layer16_attention_weight_max": 33.68901824951172,
      "activations/layer16_attention_weight_min": -26.99461555480957,
      "activations/layer17_attention_weight_max": 37.394039154052734,
      "activations/layer17_attention_weight_min": -25.289857864379883,
      "activations/layer18_attention_weight_max": 33.04780960083008,
      "activations/layer18_attention_weight_min": -21.929431915283203,
      "activations/layer19_attention_weight_max": 39.207279205322266,
      "activations/layer19_attention_weight_min": -29.90720558166504,
      "activations/layer1_attention_weight_max": 17.822189331054688,
      "activations/layer1_attention_weight_min": -13.961520195007324,
      "activations/layer20_attention_weight_max": 34.98310852050781,
      "activations/layer20_attention_weight_min": -23.511734008789062,
      "activations/layer21_attention_weight_max": 35.14375305175781,
      "activations/layer21_attention_weight_min": -21.35521697998047,
      "activations/layer22_attention_weight_max": 52.80375289916992,
      "activations/layer22_attention_weight_min": -28.354543685913086,
      "activations/layer23_attention_weight_max": 37.02136993408203,
      "activations/layer23_attention_weight_min": -21.687816619873047,
      "activations/layer2_attention_weight_max": 31.520353317260742,
      "activations/layer2_attention_weight_min": -32.62909698486328,
      "activations/layer3_attention_weight_max": 88.88275146484375,
      "activations/layer3_attention_weight_min": -94.3062515258789,
      "activations/layer4_attention_weight_max": 107.7828369140625,
      "activations/layer4_attention_weight_min": -112.31896209716797,
      "activations/layer5_attention_weight_max": 53.35407257080078,
      "activations/layer5_attention_weight_min": -61.75019073486328,
      "activations/layer6_attention_weight_max": 47.140193939208984,
      "activations/layer6_attention_weight_min": -49.20012664794922,
      "activations/layer7_attention_weight_max": 86.41143798828125,
      "activations/layer7_attention_weight_min": -86.35760498046875,
      "activations/layer8_attention_weight_max": 42.903564453125,
      "activations/layer8_attention_weight_min": -39.872623443603516,
      "activations/layer9_attention_weight_max": 33.9761848449707,
      "activations/layer9_attention_weight_min": -33.49544906616211,
      "epoch": 8.82,
      "learning_rate": 9.406590909090908e-05,
      "loss": 2.8075,
      "step": 151800
    },
    {
      "activations/layer0_attention_weight_max": 15.662837982177734,
      "activations/layer0_attention_weight_min": -12.120264053344727,
      "activations/layer10_attention_weight_max": 35.90785217285156,
      "activations/layer10_attention_weight_min": -36.568931579589844,
      "activations/layer11_attention_weight_max": 36.86603927612305,
      "activations/layer11_attention_weight_min": -36.24007034301758,
      "activations/layer12_attention_weight_max": 26.26436996459961,
      "activations/layer12_attention_weight_min": -30.556808471679688,
      "activations/layer13_attention_weight_max": 40.76062774658203,
      "activations/layer13_attention_weight_min": -35.293331146240234,
      "activations/layer14_attention_weight_max": 49.79025650024414,
      "activations/layer14_attention_weight_min": -38.91587448120117,
      "activations/layer15_attention_weight_max": 40.20051574707031,
      "activations/layer15_attention_weight_min": -29.984983444213867,
      "activations/layer16_attention_weight_max": 31.67688751220703,
      "activations/layer16_attention_weight_min": -28.354049682617188,
      "activations/layer17_attention_weight_max": 36.12547302246094,
      "activations/layer17_attention_weight_min": -25.25039291381836,
      "activations/layer18_attention_weight_max": 32.69402313232422,
      "activations/layer18_attention_weight_min": -22.0186824798584,
      "activations/layer19_attention_weight_max": 40.7860221862793,
      "activations/layer19_attention_weight_min": -30.91829490661621,
      "activations/layer1_attention_weight_max": 17.73876953125,
      "activations/layer1_attention_weight_min": -15.197094917297363,
      "activations/layer20_attention_weight_max": 30.98967933654785,
      "activations/layer20_attention_weight_min": -24.52297019958496,
      "activations/layer21_attention_weight_max": 31.605527877807617,
      "activations/layer21_attention_weight_min": -23.164688110351562,
      "activations/layer22_attention_weight_max": 51.10917282104492,
      "activations/layer22_attention_weight_min": -30.055574417114258,
      "activations/layer23_attention_weight_max": 38.72013473510742,
      "activations/layer23_attention_weight_min": -24.218931198120117,
      "activations/layer2_attention_weight_max": 31.848651885986328,
      "activations/layer2_attention_weight_min": -30.950477600097656,
      "activations/layer3_attention_weight_max": 92.92283630371094,
      "activations/layer3_attention_weight_min": -93.39901733398438,
      "activations/layer4_attention_weight_max": 114.77125549316406,
      "activations/layer4_attention_weight_min": -114.20832824707031,
      "activations/layer5_attention_weight_max": 53.19078826904297,
      "activations/layer5_attention_weight_min": -58.75373840332031,
      "activations/layer6_attention_weight_max": 48.21278381347656,
      "activations/layer6_attention_weight_min": -51.51253128051758,
      "activations/layer7_attention_weight_max": 95.92591094970703,
      "activations/layer7_attention_weight_min": -98.09928894042969,
      "activations/layer8_attention_weight_max": 44.51881408691406,
      "activations/layer8_attention_weight_min": -44.51970672607422,
      "activations/layer9_attention_weight_max": 38.20876693725586,
      "activations/layer9_attention_weight_min": -36.965572357177734,
      "epoch": 8.82,
      "learning_rate": 9.404696969696968e-05,
      "loss": 2.8128,
      "step": 151850
    },
    {
      "activations/layer0_attention_weight_max": 14.797933578491211,
      "activations/layer0_attention_weight_min": -11.469697952270508,
      "activations/layer10_attention_weight_max": 34.000213623046875,
      "activations/layer10_attention_weight_min": -33.471221923828125,
      "activations/layer11_attention_weight_max": 32.44477844238281,
      "activations/layer11_attention_weight_min": -31.8437557220459,
      "activations/layer12_attention_weight_max": 24.409927368164062,
      "activations/layer12_attention_weight_min": -25.144256591796875,
      "activations/layer13_attention_weight_max": 40.25312423706055,
      "activations/layer13_attention_weight_min": -36.082603454589844,
      "activations/layer14_attention_weight_max": 53.600067138671875,
      "activations/layer14_attention_weight_min": -39.79003143310547,
      "activations/layer15_attention_weight_max": 40.90281677246094,
      "activations/layer15_attention_weight_min": -33.32950973510742,
      "activations/layer16_attention_weight_max": 35.96986770629883,
      "activations/layer16_attention_weight_min": -28.65849494934082,
      "activations/layer17_attention_weight_max": 35.67055130004883,
      "activations/layer17_attention_weight_min": -27.230138778686523,
      "activations/layer18_attention_weight_max": 33.78925323486328,
      "activations/layer18_attention_weight_min": -23.298721313476562,
      "activations/layer19_attention_weight_max": 41.67433166503906,
      "activations/layer19_attention_weight_min": -33.29408645629883,
      "activations/layer1_attention_weight_max": 16.172426223754883,
      "activations/layer1_attention_weight_min": -15.95816421508789,
      "activations/layer20_attention_weight_max": 34.746334075927734,
      "activations/layer20_attention_weight_min": -27.010650634765625,
      "activations/layer21_attention_weight_max": 31.982267379760742,
      "activations/layer21_attention_weight_min": -22.945655822753906,
      "activations/layer22_attention_weight_max": 49.956932067871094,
      "activations/layer22_attention_weight_min": -28.18494415283203,
      "activations/layer23_attention_weight_max": 41.114463806152344,
      "activations/layer23_attention_weight_min": -23.513683319091797,
      "activations/layer2_attention_weight_max": 30.472797393798828,
      "activations/layer2_attention_weight_min": -30.669708251953125,
      "activations/layer3_attention_weight_max": 91.07230377197266,
      "activations/layer3_attention_weight_min": -92.0794677734375,
      "activations/layer4_attention_weight_max": 113.01045227050781,
      "activations/layer4_attention_weight_min": -110.0611801147461,
      "activations/layer5_attention_weight_max": 52.76560592651367,
      "activations/layer5_attention_weight_min": -61.566864013671875,
      "activations/layer6_attention_weight_max": 46.362735748291016,
      "activations/layer6_attention_weight_min": -50.41068649291992,
      "activations/layer7_attention_weight_max": 96.98373413085938,
      "activations/layer7_attention_weight_min": -88.94910430908203,
      "activations/layer8_attention_weight_max": 41.42616271972656,
      "activations/layer8_attention_weight_min": -41.00476837158203,
      "activations/layer9_attention_weight_max": 36.906368255615234,
      "activations/layer9_attention_weight_min": -34.54972457885742,
      "epoch": 8.83,
      "learning_rate": 9.40280303030303e-05,
      "loss": 2.789,
      "step": 151900
    },
    {
      "activations/layer0_attention_weight_max": 15.675617218017578,
      "activations/layer0_attention_weight_min": -12.28303050994873,
      "activations/layer10_attention_weight_max": 32.54574203491211,
      "activations/layer10_attention_weight_min": -31.08407211303711,
      "activations/layer11_attention_weight_max": 31.642536163330078,
      "activations/layer11_attention_weight_min": -30.708271026611328,
      "activations/layer12_attention_weight_max": 23.875179290771484,
      "activations/layer12_attention_weight_min": -23.977474212646484,
      "activations/layer13_attention_weight_max": 35.455116271972656,
      "activations/layer13_attention_weight_min": -33.39773178100586,
      "activations/layer14_attention_weight_max": 44.2462272644043,
      "activations/layer14_attention_weight_min": -36.61933898925781,
      "activations/layer15_attention_weight_max": 34.078163146972656,
      "activations/layer15_attention_weight_min": -30.78331756591797,
      "activations/layer16_attention_weight_max": 30.173336029052734,
      "activations/layer16_attention_weight_min": -29.322134017944336,
      "activations/layer17_attention_weight_max": 31.94214630126953,
      "activations/layer17_attention_weight_min": -24.438570022583008,
      "activations/layer18_attention_weight_max": 31.267330169677734,
      "activations/layer18_attention_weight_min": -20.889263153076172,
      "activations/layer19_attention_weight_max": 33.93593215942383,
      "activations/layer19_attention_weight_min": -30.303991317749023,
      "activations/layer1_attention_weight_max": 16.607192993164062,
      "activations/layer1_attention_weight_min": -15.69939136505127,
      "activations/layer20_attention_weight_max": 27.312875747680664,
      "activations/layer20_attention_weight_min": -23.633054733276367,
      "activations/layer21_attention_weight_max": 26.491506576538086,
      "activations/layer21_attention_weight_min": -21.948715209960938,
      "activations/layer22_attention_weight_max": 41.653907775878906,
      "activations/layer22_attention_weight_min": -29.78317642211914,
      "activations/layer23_attention_weight_max": 35.616363525390625,
      "activations/layer23_attention_weight_min": -27.591413497924805,
      "activations/layer2_attention_weight_max": 32.52367401123047,
      "activations/layer2_attention_weight_min": -31.115550994873047,
      "activations/layer3_attention_weight_max": 96.78699493408203,
      "activations/layer3_attention_weight_min": -92.51158142089844,
      "activations/layer4_attention_weight_max": 118.1470947265625,
      "activations/layer4_attention_weight_min": -111.5834732055664,
      "activations/layer5_attention_weight_max": 52.92057418823242,
      "activations/layer5_attention_weight_min": -59.2672233581543,
      "activations/layer6_attention_weight_max": 47.35484313964844,
      "activations/layer6_attention_weight_min": -48.40721130371094,
      "activations/layer7_attention_weight_max": 88.81848907470703,
      "activations/layer7_attention_weight_min": -86.24397277832031,
      "activations/layer8_attention_weight_max": 39.22719192504883,
      "activations/layer8_attention_weight_min": -39.42111587524414,
      "activations/layer9_attention_weight_max": 36.73246383666992,
      "activations/layer9_attention_weight_min": -33.7822380065918,
      "epoch": 8.83,
      "learning_rate": 9.40090909090909e-05,
      "loss": 2.7903,
      "step": 151950
    },
    {
      "activations/layer0_attention_weight_max": 16.238739013671875,
      "activations/layer0_attention_weight_min": -12.1231107711792,
      "activations/layer10_attention_weight_max": 34.332672119140625,
      "activations/layer10_attention_weight_min": -31.171884536743164,
      "activations/layer11_attention_weight_max": 31.582759857177734,
      "activations/layer11_attention_weight_min": -30.08884048461914,
      "activations/layer12_attention_weight_max": 21.725025177001953,
      "activations/layer12_attention_weight_min": -30.811927795410156,
      "activations/layer13_attention_weight_max": 34.78726577758789,
      "activations/layer13_attention_weight_min": -31.7255802154541,
      "activations/layer14_attention_weight_max": 43.94110107421875,
      "activations/layer14_attention_weight_min": -40.93830108642578,
      "activations/layer15_attention_weight_max": 36.51557922363281,
      "activations/layer15_attention_weight_min": -31.37691307067871,
      "activations/layer16_attention_weight_max": 30.421585083007812,
      "activations/layer16_attention_weight_min": -30.33391571044922,
      "activations/layer17_attention_weight_max": 33.61744689941406,
      "activations/layer17_attention_weight_min": -25.443431854248047,
      "activations/layer18_attention_weight_max": 33.383636474609375,
      "activations/layer18_attention_weight_min": -21.847049713134766,
      "activations/layer19_attention_weight_max": 32.03575897216797,
      "activations/layer19_attention_weight_min": -28.388397216796875,
      "activations/layer1_attention_weight_max": 15.885748863220215,
      "activations/layer1_attention_weight_min": -16.45762062072754,
      "activations/layer20_attention_weight_max": 29.57900047302246,
      "activations/layer20_attention_weight_min": -22.733001708984375,
      "activations/layer21_attention_weight_max": 34.985595703125,
      "activations/layer21_attention_weight_min": -22.52720069885254,
      "activations/layer22_attention_weight_max": 44.175479888916016,
      "activations/layer22_attention_weight_min": -26.94365882873535,
      "activations/layer23_attention_weight_max": 34.46050262451172,
      "activations/layer23_attention_weight_min": -24.246524810791016,
      "activations/layer2_attention_weight_max": 30.795822143554688,
      "activations/layer2_attention_weight_min": -30.604644775390625,
      "activations/layer3_attention_weight_max": 92.18204498291016,
      "activations/layer3_attention_weight_min": -94.69705200195312,
      "activations/layer4_attention_weight_max": 117.59391021728516,
      "activations/layer4_attention_weight_min": -116.72269439697266,
      "activations/layer5_attention_weight_max": 55.15422058105469,
      "activations/layer5_attention_weight_min": -61.67411804199219,
      "activations/layer6_attention_weight_max": 46.63964080810547,
      "activations/layer6_attention_weight_min": -50.6301383972168,
      "activations/layer7_attention_weight_max": 87.61876678466797,
      "activations/layer7_attention_weight_min": -90.4841537475586,
      "activations/layer8_attention_weight_max": 40.58016586303711,
      "activations/layer8_attention_weight_min": -40.236656188964844,
      "activations/layer9_attention_weight_max": 34.68819046020508,
      "activations/layer9_attention_weight_min": -34.638973236083984,
      "epoch": 8.83,
      "learning_rate": 9.399015151515151e-05,
      "loss": 2.8199,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.5075,
      "eval_samples_per_second": 504.73,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.5075,
      "eval_openwebtext_samples_per_second": 504.73,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9942,
      "eval_wikitext_samples_per_second": 228.663,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_lambada_loss": 2.599609375,
      "eval_lambada_ppl": 13.458479789399725,
      "eval_lambada_runtime": 9.609,
      "eval_lambada_samples_per_second": 506.711,
      "step": 152000
    },
    {
      "activations/layer0_attention_weight_max": 15.507315635681152,
      "activations/layer0_attention_weight_min": -13.490612030029297,
      "activations/layer10_attention_weight_max": 33.127685546875,
      "activations/layer10_attention_weight_min": -31.814815521240234,
      "activations/layer11_attention_weight_max": 32.21782684326172,
      "activations/layer11_attention_weight_min": -30.145153045654297,
      "activations/layer12_attention_weight_max": 23.262042999267578,
      "activations/layer12_attention_weight_min": -27.30636215209961,
      "activations/layer13_attention_weight_max": 38.24046325683594,
      "activations/layer13_attention_weight_min": -33.653411865234375,
      "activations/layer14_attention_weight_max": 48.094383239746094,
      "activations/layer14_attention_weight_min": -43.60265350341797,
      "activations/layer15_attention_weight_max": 37.034568786621094,
      "activations/layer15_attention_weight_min": -31.390562057495117,
      "activations/layer16_attention_weight_max": 30.513553619384766,
      "activations/layer16_attention_weight_min": -29.795883178710938,
      "activations/layer17_attention_weight_max": 34.27058029174805,
      "activations/layer17_attention_weight_min": -26.699901580810547,
      "activations/layer18_attention_weight_max": 32.33766174316406,
      "activations/layer18_attention_weight_min": -25.460105895996094,
      "activations/layer19_attention_weight_max": 35.43632125854492,
      "activations/layer19_attention_weight_min": -32.24893569946289,
      "activations/layer1_attention_weight_max": 16.25994300842285,
      "activations/layer1_attention_weight_min": -13.677607536315918,
      "activations/layer20_attention_weight_max": 31.733579635620117,
      "activations/layer20_attention_weight_min": -27.291431427001953,
      "activations/layer21_attention_weight_max": 32.419376373291016,
      "activations/layer21_attention_weight_min": -24.300500869750977,
      "activations/layer22_attention_weight_max": 45.0711555480957,
      "activations/layer22_attention_weight_min": -30.79056167602539,
      "activations/layer23_attention_weight_max": 37.30413055419922,
      "activations/layer23_attention_weight_min": -25.09644889831543,
      "activations/layer2_attention_weight_max": 30.316526412963867,
      "activations/layer2_attention_weight_min": -30.902267456054688,
      "activations/layer3_attention_weight_max": 90.97652435302734,
      "activations/layer3_attention_weight_min": -93.21736907958984,
      "activations/layer4_attention_weight_max": 114.73146057128906,
      "activations/layer4_attention_weight_min": -112.93138885498047,
      "activations/layer5_attention_weight_max": 53.63633346557617,
      "activations/layer5_attention_weight_min": -63.962989807128906,
      "activations/layer6_attention_weight_max": 48.888877868652344,
      "activations/layer6_attention_weight_min": -51.47148513793945,
      "activations/layer7_attention_weight_max": 89.8892822265625,
      "activations/layer7_attention_weight_min": -91.63713836669922,
      "activations/layer8_attention_weight_max": 41.105838775634766,
      "activations/layer8_attention_weight_min": -45.6848258972168,
      "activations/layer9_attention_weight_max": 36.58430099487305,
      "activations/layer9_attention_weight_min": -34.44474411010742,
      "epoch": 8.83,
      "learning_rate": 9.397121212121211e-05,
      "loss": 2.8043,
      "step": 152050
    },
    {
      "activations/layer0_attention_weight_max": 15.601759910583496,
      "activations/layer0_attention_weight_min": -12.477522850036621,
      "activations/layer10_attention_weight_max": 34.912227630615234,
      "activations/layer10_attention_weight_min": -31.83102798461914,
      "activations/layer11_attention_weight_max": 32.846092224121094,
      "activations/layer11_attention_weight_min": -31.205631256103516,
      "activations/layer12_attention_weight_max": 22.960546493530273,
      "activations/layer12_attention_weight_min": -33.71514892578125,
      "activations/layer13_attention_weight_max": 35.115570068359375,
      "activations/layer13_attention_weight_min": -32.1807746887207,
      "activations/layer14_attention_weight_max": 44.5814208984375,
      "activations/layer14_attention_weight_min": -38.5377082824707,
      "activations/layer15_attention_weight_max": 35.25904083251953,
      "activations/layer15_attention_weight_min": -30.754350662231445,
      "activations/layer16_attention_weight_max": 32.01750946044922,
      "activations/layer16_attention_weight_min": -26.438884735107422,
      "activations/layer17_attention_weight_max": 33.4913215637207,
      "activations/layer17_attention_weight_min": -24.593673706054688,
      "activations/layer18_attention_weight_max": 31.73641586303711,
      "activations/layer18_attention_weight_min": -22.42650032043457,
      "activations/layer19_attention_weight_max": 33.85367965698242,
      "activations/layer19_attention_weight_min": -31.07142448425293,
      "activations/layer1_attention_weight_max": 16.19154930114746,
      "activations/layer1_attention_weight_min": -13.793091773986816,
      "activations/layer20_attention_weight_max": 30.859466552734375,
      "activations/layer20_attention_weight_min": -25.529067993164062,
      "activations/layer21_attention_weight_max": 32.848304748535156,
      "activations/layer21_attention_weight_min": -24.591821670532227,
      "activations/layer22_attention_weight_max": 42.761810302734375,
      "activations/layer22_attention_weight_min": -29.62824249267578,
      "activations/layer23_attention_weight_max": 33.285526275634766,
      "activations/layer23_attention_weight_min": -24.429874420166016,
      "activations/layer2_attention_weight_max": 30.607587814331055,
      "activations/layer2_attention_weight_min": -30.9090633392334,
      "activations/layer3_attention_weight_max": 98.07344055175781,
      "activations/layer3_attention_weight_min": -94.98607635498047,
      "activations/layer4_attention_weight_max": 117.15437316894531,
      "activations/layer4_attention_weight_min": -118.42162322998047,
      "activations/layer5_attention_weight_max": 54.166351318359375,
      "activations/layer5_attention_weight_min": -64.48123168945312,
      "activations/layer6_attention_weight_max": 47.317359924316406,
      "activations/layer6_attention_weight_min": -51.30205154418945,
      "activations/layer7_attention_weight_max": 89.94882202148438,
      "activations/layer7_attention_weight_min": -87.48506164550781,
      "activations/layer8_attention_weight_max": 40.32278060913086,
      "activations/layer8_attention_weight_min": -42.089820861816406,
      "activations/layer9_attention_weight_max": 35.138851165771484,
      "activations/layer9_attention_weight_min": -34.93431091308594,
      "epoch": 8.84,
      "learning_rate": 9.395227272727272e-05,
      "loss": 2.7817,
      "step": 152100
    },
    {
      "activations/layer0_attention_weight_max": 15.07304859161377,
      "activations/layer0_attention_weight_min": -12.49427318572998,
      "activations/layer10_attention_weight_max": 33.490272521972656,
      "activations/layer10_attention_weight_min": -33.68638610839844,
      "activations/layer11_attention_weight_max": 32.060020446777344,
      "activations/layer11_attention_weight_min": -31.59332847595215,
      "activations/layer12_attention_weight_max": 26.561561584472656,
      "activations/layer12_attention_weight_min": -31.973024368286133,
      "activations/layer13_attention_weight_max": 41.03888702392578,
      "activations/layer13_attention_weight_min": -34.213497161865234,
      "activations/layer14_attention_weight_max": 48.37468338012695,
      "activations/layer14_attention_weight_min": -37.39570999145508,
      "activations/layer15_attention_weight_max": 41.47992706298828,
      "activations/layer15_attention_weight_min": -34.26008605957031,
      "activations/layer16_attention_weight_max": 34.91288757324219,
      "activations/layer16_attention_weight_min": -28.826784133911133,
      "activations/layer17_attention_weight_max": 39.10090637207031,
      "activations/layer17_attention_weight_min": -27.299467086791992,
      "activations/layer18_attention_weight_max": 38.686248779296875,
      "activations/layer18_attention_weight_min": -22.937618255615234,
      "activations/layer19_attention_weight_max": 41.10988998413086,
      "activations/layer19_attention_weight_min": -36.144161224365234,
      "activations/layer1_attention_weight_max": 15.96147346496582,
      "activations/layer1_attention_weight_min": -12.952996253967285,
      "activations/layer20_attention_weight_max": 37.72601318359375,
      "activations/layer20_attention_weight_min": -31.9924259185791,
      "activations/layer21_attention_weight_max": 36.457763671875,
      "activations/layer21_attention_weight_min": -31.000993728637695,
      "activations/layer22_attention_weight_max": 52.43537139892578,
      "activations/layer22_attention_weight_min": -32.1219367980957,
      "activations/layer23_attention_weight_max": 40.096527099609375,
      "activations/layer23_attention_weight_min": -22.74061393737793,
      "activations/layer2_attention_weight_max": 32.26133346557617,
      "activations/layer2_attention_weight_min": -30.27602767944336,
      "activations/layer3_attention_weight_max": 91.65907287597656,
      "activations/layer3_attention_weight_min": -92.86370849609375,
      "activations/layer4_attention_weight_max": 117.7688980102539,
      "activations/layer4_attention_weight_min": -114.78050994873047,
      "activations/layer5_attention_weight_max": 51.72930145263672,
      "activations/layer5_attention_weight_min": -62.397552490234375,
      "activations/layer6_attention_weight_max": 46.28130340576172,
      "activations/layer6_attention_weight_min": -50.02524185180664,
      "activations/layer7_attention_weight_max": 87.43559265136719,
      "activations/layer7_attention_weight_min": -91.43968963623047,
      "activations/layer8_attention_weight_max": 38.58879470825195,
      "activations/layer8_attention_weight_min": -39.55704879760742,
      "activations/layer9_attention_weight_max": 32.336708068847656,
      "activations/layer9_attention_weight_min": -33.73981857299805,
      "epoch": 8.84,
      "learning_rate": 9.393333333333333e-05,
      "loss": 2.7879,
      "step": 152150
    },
    {
      "activations/layer0_attention_weight_max": 14.583563804626465,
      "activations/layer0_attention_weight_min": -12.818538665771484,
      "activations/layer10_attention_weight_max": 36.548805236816406,
      "activations/layer10_attention_weight_min": -35.57269287109375,
      "activations/layer11_attention_weight_max": 39.62871551513672,
      "activations/layer11_attention_weight_min": -32.794334411621094,
      "activations/layer12_attention_weight_max": 27.52312469482422,
      "activations/layer12_attention_weight_min": -27.69585609436035,
      "activations/layer13_attention_weight_max": 42.050758361816406,
      "activations/layer13_attention_weight_min": -35.67329025268555,
      "activations/layer14_attention_weight_max": 63.83778762817383,
      "activations/layer14_attention_weight_min": -45.81572723388672,
      "activations/layer15_attention_weight_max": 40.94528579711914,
      "activations/layer15_attention_weight_min": -32.87799835205078,
      "activations/layer16_attention_weight_max": 36.017086029052734,
      "activations/layer16_attention_weight_min": -27.90875816345215,
      "activations/layer17_attention_weight_max": 42.08381652832031,
      "activations/layer17_attention_weight_min": -26.420923233032227,
      "activations/layer18_attention_weight_max": 35.69140625,
      "activations/layer18_attention_weight_min": -25.921415328979492,
      "activations/layer19_attention_weight_max": 37.920291900634766,
      "activations/layer19_attention_weight_min": -30.86713218688965,
      "activations/layer1_attention_weight_max": 17.19663429260254,
      "activations/layer1_attention_weight_min": -14.708300590515137,
      "activations/layer20_attention_weight_max": 28.42987060546875,
      "activations/layer20_attention_weight_min": -23.9520320892334,
      "activations/layer21_attention_weight_max": 26.567541122436523,
      "activations/layer21_attention_weight_min": -21.992746353149414,
      "activations/layer22_attention_weight_max": 45.67412567138672,
      "activations/layer22_attention_weight_min": -29.880048751831055,
      "activations/layer23_attention_weight_max": 33.65589904785156,
      "activations/layer23_attention_weight_min": -23.706958770751953,
      "activations/layer2_attention_weight_max": 31.59441375732422,
      "activations/layer2_attention_weight_min": -31.86227035522461,
      "activations/layer3_attention_weight_max": 100.60569763183594,
      "activations/layer3_attention_weight_min": -103.23446655273438,
      "activations/layer4_attention_weight_max": 128.2257537841797,
      "activations/layer4_attention_weight_min": -126.06892395019531,
      "activations/layer5_attention_weight_max": 60.07823181152344,
      "activations/layer5_attention_weight_min": -64.50538635253906,
      "activations/layer6_attention_weight_max": 50.023521423339844,
      "activations/layer6_attention_weight_min": -54.424964904785156,
      "activations/layer7_attention_weight_max": 95.98552703857422,
      "activations/layer7_attention_weight_min": -93.53736877441406,
      "activations/layer8_attention_weight_max": 44.96026611328125,
      "activations/layer8_attention_weight_min": -45.56784439086914,
      "activations/layer9_attention_weight_max": 37.22208786010742,
      "activations/layer9_attention_weight_min": -36.661460876464844,
      "epoch": 8.84,
      "learning_rate": 9.391439393939395e-05,
      "loss": 2.7927,
      "step": 152200
    },
    {
      "activations/layer0_attention_weight_max": 15.465071678161621,
      "activations/layer0_attention_weight_min": -12.13956356048584,
      "activations/layer10_attention_weight_max": 34.00927734375,
      "activations/layer10_attention_weight_min": -34.794246673583984,
      "activations/layer11_attention_weight_max": 33.79306411743164,
      "activations/layer11_attention_weight_min": -31.058324813842773,
      "activations/layer12_attention_weight_max": 22.200496673583984,
      "activations/layer12_attention_weight_min": -24.907272338867188,
      "activations/layer13_attention_weight_max": 37.58889389038086,
      "activations/layer13_attention_weight_min": -32.83735275268555,
      "activations/layer14_attention_weight_max": 44.67621994018555,
      "activations/layer14_attention_weight_min": -41.77260208129883,
      "activations/layer15_attention_weight_max": 36.07699203491211,
      "activations/layer15_attention_weight_min": -30.592504501342773,
      "activations/layer16_attention_weight_max": 31.68531036376953,
      "activations/layer16_attention_weight_min": -27.08235740661621,
      "activations/layer17_attention_weight_max": 31.3867130279541,
      "activations/layer17_attention_weight_min": -24.74707794189453,
      "activations/layer18_attention_weight_max": 29.496774673461914,
      "activations/layer18_attention_weight_min": -23.73581314086914,
      "activations/layer19_attention_weight_max": 33.61396408081055,
      "activations/layer19_attention_weight_min": -29.211355209350586,
      "activations/layer1_attention_weight_max": 16.160402297973633,
      "activations/layer1_attention_weight_min": -13.230490684509277,
      "activations/layer20_attention_weight_max": 30.277055740356445,
      "activations/layer20_attention_weight_min": -25.779930114746094,
      "activations/layer21_attention_weight_max": 29.006418228149414,
      "activations/layer21_attention_weight_min": -22.75554847717285,
      "activations/layer22_attention_weight_max": 41.566139221191406,
      "activations/layer22_attention_weight_min": -28.593063354492188,
      "activations/layer23_attention_weight_max": 32.467472076416016,
      "activations/layer23_attention_weight_min": -22.912628173828125,
      "activations/layer2_attention_weight_max": 30.897762298583984,
      "activations/layer2_attention_weight_min": -32.01759719848633,
      "activations/layer3_attention_weight_max": 92.93225860595703,
      "activations/layer3_attention_weight_min": -95.3769760131836,
      "activations/layer4_attention_weight_max": 116.47722625732422,
      "activations/layer4_attention_weight_min": -121.538330078125,
      "activations/layer5_attention_weight_max": 56.97553634643555,
      "activations/layer5_attention_weight_min": -63.910518646240234,
      "activations/layer6_attention_weight_max": 47.45975875854492,
      "activations/layer6_attention_weight_min": -52.44807434082031,
      "activations/layer7_attention_weight_max": 91.32808685302734,
      "activations/layer7_attention_weight_min": -97.41429138183594,
      "activations/layer8_attention_weight_max": 43.35974884033203,
      "activations/layer8_attention_weight_min": -44.35432815551758,
      "activations/layer9_attention_weight_max": 35.75751876831055,
      "activations/layer9_attention_weight_min": -36.92763900756836,
      "epoch": 8.85,
      "learning_rate": 9.389545454545453e-05,
      "loss": 2.7702,
      "step": 152250
    },
    {
      "activations/layer0_attention_weight_max": 16.013582229614258,
      "activations/layer0_attention_weight_min": -13.344393730163574,
      "activations/layer10_attention_weight_max": 37.444862365722656,
      "activations/layer10_attention_weight_min": -35.43980407714844,
      "activations/layer11_attention_weight_max": 37.78048324584961,
      "activations/layer11_attention_weight_min": -33.7803955078125,
      "activations/layer12_attention_weight_max": 27.29793930053711,
      "activations/layer12_attention_weight_min": -24.683128356933594,
      "activations/layer13_attention_weight_max": 43.66075897216797,
      "activations/layer13_attention_weight_min": -31.614477157592773,
      "activations/layer14_attention_weight_max": 53.41951370239258,
      "activations/layer14_attention_weight_min": -38.89155578613281,
      "activations/layer15_attention_weight_max": 37.86582565307617,
      "activations/layer15_attention_weight_min": -31.175798416137695,
      "activations/layer16_attention_weight_max": 38.01495361328125,
      "activations/layer16_attention_weight_min": -29.058347702026367,
      "activations/layer17_attention_weight_max": 31.871124267578125,
      "activations/layer17_attention_weight_min": -25.293039321899414,
      "activations/layer18_attention_weight_max": 29.73879623413086,
      "activations/layer18_attention_weight_min": -23.195735931396484,
      "activations/layer19_attention_weight_max": 37.609344482421875,
      "activations/layer19_attention_weight_min": -30.816511154174805,
      "activations/layer1_attention_weight_max": 16.30154037475586,
      "activations/layer1_attention_weight_min": -14.278499603271484,
      "activations/layer20_attention_weight_max": 33.42097854614258,
      "activations/layer20_attention_weight_min": -24.742170333862305,
      "activations/layer21_attention_weight_max": 30.614744186401367,
      "activations/layer21_attention_weight_min": -25.63286590576172,
      "activations/layer22_attention_weight_max": 51.28070831298828,
      "activations/layer22_attention_weight_min": -30.370285034179688,
      "activations/layer23_attention_weight_max": 37.22969055175781,
      "activations/layer23_attention_weight_min": -24.83645248413086,
      "activations/layer2_attention_weight_max": 32.162376403808594,
      "activations/layer2_attention_weight_min": -32.76953887939453,
      "activations/layer3_attention_weight_max": 97.15818786621094,
      "activations/layer3_attention_weight_min": -99.1475830078125,
      "activations/layer4_attention_weight_max": 128.32752990722656,
      "activations/layer4_attention_weight_min": -123.12078094482422,
      "activations/layer5_attention_weight_max": 57.547332763671875,
      "activations/layer5_attention_weight_min": -61.037513732910156,
      "activations/layer6_attention_weight_max": 51.621524810791016,
      "activations/layer6_attention_weight_min": -55.27283477783203,
      "activations/layer7_attention_weight_max": 92.51817321777344,
      "activations/layer7_attention_weight_min": -98.89017486572266,
      "activations/layer8_attention_weight_max": 43.0124626159668,
      "activations/layer8_attention_weight_min": -45.485252380371094,
      "activations/layer9_attention_weight_max": 38.79294967651367,
      "activations/layer9_attention_weight_min": -36.3234977722168,
      "epoch": 8.85,
      "learning_rate": 9.387651515151515e-05,
      "loss": 2.7617,
      "step": 152300
    },
    {
      "activations/layer0_attention_weight_max": 16.644081115722656,
      "activations/layer0_attention_weight_min": -11.311506271362305,
      "activations/layer10_attention_weight_max": 45.24428176879883,
      "activations/layer10_attention_weight_min": -41.87493896484375,
      "activations/layer11_attention_weight_max": 46.22069549560547,
      "activations/layer11_attention_weight_min": -38.73480987548828,
      "activations/layer12_attention_weight_max": 28.763324737548828,
      "activations/layer12_attention_weight_min": -26.589330673217773,
      "activations/layer13_attention_weight_max": 47.460975646972656,
      "activations/layer13_attention_weight_min": -37.026214599609375,
      "activations/layer14_attention_weight_max": 55.55763244628906,
      "activations/layer14_attention_weight_min": -43.069908142089844,
      "activations/layer15_attention_weight_max": 50.215335845947266,
      "activations/layer15_attention_weight_min": -36.11052703857422,
      "activations/layer16_attention_weight_max": 35.299964904785156,
      "activations/layer16_attention_weight_min": -28.5147705078125,
      "activations/layer17_attention_weight_max": 34.222373962402344,
      "activations/layer17_attention_weight_min": -27.177892684936523,
      "activations/layer18_attention_weight_max": 30.51451301574707,
      "activations/layer18_attention_weight_min": -24.40234375,
      "activations/layer19_attention_weight_max": 40.36422348022461,
      "activations/layer19_attention_weight_min": -29.652618408203125,
      "activations/layer1_attention_weight_max": 16.76161003112793,
      "activations/layer1_attention_weight_min": -15.437006950378418,
      "activations/layer20_attention_weight_max": 33.16173553466797,
      "activations/layer20_attention_weight_min": -24.83030891418457,
      "activations/layer21_attention_weight_max": 30.34566879272461,
      "activations/layer21_attention_weight_min": -23.274906158447266,
      "activations/layer22_attention_weight_max": 55.63332748413086,
      "activations/layer22_attention_weight_min": -28.33002281188965,
      "activations/layer23_attention_weight_max": 33.78130340576172,
      "activations/layer23_attention_weight_min": -24.27098846435547,
      "activations/layer2_attention_weight_max": 34.791015625,
      "activations/layer2_attention_weight_min": -34.699981689453125,
      "activations/layer3_attention_weight_max": 109.47684478759766,
      "activations/layer3_attention_weight_min": -110.15556335449219,
      "activations/layer4_attention_weight_max": 134.89279174804688,
      "activations/layer4_attention_weight_min": -132.70533752441406,
      "activations/layer5_attention_weight_max": 61.147979736328125,
      "activations/layer5_attention_weight_min": -64.57353973388672,
      "activations/layer6_attention_weight_max": 53.081275939941406,
      "activations/layer6_attention_weight_min": -55.9022216796875,
      "activations/layer7_attention_weight_max": 110.01318359375,
      "activations/layer7_attention_weight_min": -102.7769775390625,
      "activations/layer8_attention_weight_max": 51.53594207763672,
      "activations/layer8_attention_weight_min": -49.34270095825195,
      "activations/layer9_attention_weight_max": 44.12529373168945,
      "activations/layer9_attention_weight_min": -41.24403381347656,
      "epoch": 8.85,
      "learning_rate": 9.385757575757574e-05,
      "loss": 2.7887,
      "step": 152350
    },
    {
      "activations/layer0_attention_weight_max": 15.834783554077148,
      "activations/layer0_attention_weight_min": -12.40395450592041,
      "activations/layer10_attention_weight_max": 32.35104751586914,
      "activations/layer10_attention_weight_min": -32.91078186035156,
      "activations/layer11_attention_weight_max": 32.84552001953125,
      "activations/layer11_attention_weight_min": -31.772193908691406,
      "activations/layer12_attention_weight_max": 26.04974937438965,
      "activations/layer12_attention_weight_min": -27.62704849243164,
      "activations/layer13_attention_weight_max": 42.554359436035156,
      "activations/layer13_attention_weight_min": -35.23514175415039,
      "activations/layer14_attention_weight_max": 50.01089096069336,
      "activations/layer14_attention_weight_min": -41.15593719482422,
      "activations/layer15_attention_weight_max": 36.46183395385742,
      "activations/layer15_attention_weight_min": -30.067951202392578,
      "activations/layer16_attention_weight_max": 32.60957717895508,
      "activations/layer16_attention_weight_min": -28.695589065551758,
      "activations/layer17_attention_weight_max": 33.42277526855469,
      "activations/layer17_attention_weight_min": -25.96868133544922,
      "activations/layer18_attention_weight_max": 31.206748962402344,
      "activations/layer18_attention_weight_min": -21.59791374206543,
      "activations/layer19_attention_weight_max": 42.422462463378906,
      "activations/layer19_attention_weight_min": -30.519506454467773,
      "activations/layer1_attention_weight_max": 17.404037475585938,
      "activations/layer1_attention_weight_min": -14.088836669921875,
      "activations/layer20_attention_weight_max": 33.03145980834961,
      "activations/layer20_attention_weight_min": -24.888805389404297,
      "activations/layer21_attention_weight_max": 32.872615814208984,
      "activations/layer21_attention_weight_min": -24.28325843811035,
      "activations/layer22_attention_weight_max": 54.05497360229492,
      "activations/layer22_attention_weight_min": -31.647357940673828,
      "activations/layer23_attention_weight_max": 45.93988037109375,
      "activations/layer23_attention_weight_min": -26.048690795898438,
      "activations/layer2_attention_weight_max": 32.205657958984375,
      "activations/layer2_attention_weight_min": -32.54575729370117,
      "activations/layer3_attention_weight_max": 101.7469253540039,
      "activations/layer3_attention_weight_min": -102.72299194335938,
      "activations/layer4_attention_weight_max": 119.77996063232422,
      "activations/layer4_attention_weight_min": -119.3799819946289,
      "activations/layer5_attention_weight_max": 55.33059310913086,
      "activations/layer5_attention_weight_min": -61.3231315612793,
      "activations/layer6_attention_weight_max": 47.14423370361328,
      "activations/layer6_attention_weight_min": -50.351112365722656,
      "activations/layer7_attention_weight_max": 89.2173843383789,
      "activations/layer7_attention_weight_min": -93.48553466796875,
      "activations/layer8_attention_weight_max": 39.1038703918457,
      "activations/layer8_attention_weight_min": -41.79855728149414,
      "activations/layer9_attention_weight_max": 33.4474983215332,
      "activations/layer9_attention_weight_min": -34.11104965209961,
      "epoch": 8.86,
      "learning_rate": 9.383863636363635e-05,
      "loss": 2.7896,
      "step": 152400
    },
    {
      "activations/layer0_attention_weight_max": 16.15648651123047,
      "activations/layer0_attention_weight_min": -10.782198905944824,
      "activations/layer10_attention_weight_max": 31.58024024963379,
      "activations/layer10_attention_weight_min": -29.937192916870117,
      "activations/layer11_attention_weight_max": 32.072208404541016,
      "activations/layer11_attention_weight_min": -30.539419174194336,
      "activations/layer12_attention_weight_max": 25.871442794799805,
      "activations/layer12_attention_weight_min": -32.836326599121094,
      "activations/layer13_attention_weight_max": 36.17221450805664,
      "activations/layer13_attention_weight_min": -33.86640167236328,
      "activations/layer14_attention_weight_max": 42.85115432739258,
      "activations/layer14_attention_weight_min": -39.312801361083984,
      "activations/layer15_attention_weight_max": 34.348934173583984,
      "activations/layer15_attention_weight_min": -31.53812599182129,
      "activations/layer16_attention_weight_max": 30.504711151123047,
      "activations/layer16_attention_weight_min": -28.481077194213867,
      "activations/layer17_attention_weight_max": 31.190101623535156,
      "activations/layer17_attention_weight_min": -26.103792190551758,
      "activations/layer18_attention_weight_max": 31.763492584228516,
      "activations/layer18_attention_weight_min": -21.509443283081055,
      "activations/layer19_attention_weight_max": 33.73666000366211,
      "activations/layer19_attention_weight_min": -30.886493682861328,
      "activations/layer1_attention_weight_max": 17.37335205078125,
      "activations/layer1_attention_weight_min": -12.932417869567871,
      "activations/layer20_attention_weight_max": 30.007633209228516,
      "activations/layer20_attention_weight_min": -23.647634506225586,
      "activations/layer21_attention_weight_max": 27.284147262573242,
      "activations/layer21_attention_weight_min": -21.833232879638672,
      "activations/layer22_attention_weight_max": 42.02690505981445,
      "activations/layer22_attention_weight_min": -27.771604537963867,
      "activations/layer23_attention_weight_max": 34.39778137207031,
      "activations/layer23_attention_weight_min": -22.5523681640625,
      "activations/layer2_attention_weight_max": 32.474998474121094,
      "activations/layer2_attention_weight_min": -32.00468444824219,
      "activations/layer3_attention_weight_max": 97.71955871582031,
      "activations/layer3_attention_weight_min": -98.04881286621094,
      "activations/layer4_attention_weight_max": 126.1849365234375,
      "activations/layer4_attention_weight_min": -118.6580581665039,
      "activations/layer5_attention_weight_max": 57.88450241088867,
      "activations/layer5_attention_weight_min": -63.439353942871094,
      "activations/layer6_attention_weight_max": 47.631649017333984,
      "activations/layer6_attention_weight_min": -49.2590446472168,
      "activations/layer7_attention_weight_max": 90.50604248046875,
      "activations/layer7_attention_weight_min": -89.53558349609375,
      "activations/layer8_attention_weight_max": 42.030250549316406,
      "activations/layer8_attention_weight_min": -39.448486328125,
      "activations/layer9_attention_weight_max": 33.67219924926758,
      "activations/layer9_attention_weight_min": -33.003639221191406,
      "epoch": 8.86,
      "learning_rate": 9.381969696969697e-05,
      "loss": 2.7827,
      "step": 152450
    },
    {
      "activations/layer0_attention_weight_max": 15.57400894165039,
      "activations/layer0_attention_weight_min": -12.111248016357422,
      "activations/layer10_attention_weight_max": 34.757408142089844,
      "activations/layer10_attention_weight_min": -34.48538589477539,
      "activations/layer11_attention_weight_max": 37.26778030395508,
      "activations/layer11_attention_weight_min": -31.888927459716797,
      "activations/layer12_attention_weight_max": 24.956748962402344,
      "activations/layer12_attention_weight_min": -26.87122917175293,
      "activations/layer13_attention_weight_max": 49.03994369506836,
      "activations/layer13_attention_weight_min": -32.93617630004883,
      "activations/layer14_attention_weight_max": 46.379512786865234,
      "activations/layer14_attention_weight_min": -41.47297668457031,
      "activations/layer15_attention_weight_max": 56.47005081176758,
      "activations/layer15_attention_weight_min": -32.67037582397461,
      "activations/layer16_attention_weight_max": 34.77313995361328,
      "activations/layer16_attention_weight_min": -26.916196823120117,
      "activations/layer17_attention_weight_max": 31.83979034423828,
      "activations/layer17_attention_weight_min": -26.392614364624023,
      "activations/layer18_attention_weight_max": 33.1824836730957,
      "activations/layer18_attention_weight_min": -22.21245002746582,
      "activations/layer19_attention_weight_max": 35.119293212890625,
      "activations/layer19_attention_weight_min": -31.36531639099121,
      "activations/layer1_attention_weight_max": 16.756881713867188,
      "activations/layer1_attention_weight_min": -13.664966583251953,
      "activations/layer20_attention_weight_max": 30.274717330932617,
      "activations/layer20_attention_weight_min": -25.712486267089844,
      "activations/layer21_attention_weight_max": 31.135202407836914,
      "activations/layer21_attention_weight_min": -23.398075103759766,
      "activations/layer22_attention_weight_max": 46.356361389160156,
      "activations/layer22_attention_weight_min": -31.20096778869629,
      "activations/layer23_attention_weight_max": 36.03834915161133,
      "activations/layer23_attention_weight_min": -24.929479598999023,
      "activations/layer2_attention_weight_max": 31.17388916015625,
      "activations/layer2_attention_weight_min": -31.6805477142334,
      "activations/layer3_attention_weight_max": 94.77325439453125,
      "activations/layer3_attention_weight_min": -96.5044937133789,
      "activations/layer4_attention_weight_max": 119.71215057373047,
      "activations/layer4_attention_weight_min": -123.14338684082031,
      "activations/layer5_attention_weight_max": 54.56655502319336,
      "activations/layer5_attention_weight_min": -61.13153076171875,
      "activations/layer6_attention_weight_max": 48.42818069458008,
      "activations/layer6_attention_weight_min": -51.2358512878418,
      "activations/layer7_attention_weight_max": 90.92335510253906,
      "activations/layer7_attention_weight_min": -100.98702239990234,
      "activations/layer8_attention_weight_max": 43.81850051879883,
      "activations/layer8_attention_weight_min": -44.76933670043945,
      "activations/layer9_attention_weight_max": 38.62405014038086,
      "activations/layer9_attention_weight_min": -37.09793472290039,
      "epoch": 8.86,
      "learning_rate": 9.380075757575757e-05,
      "loss": 2.7673,
      "step": 152500
    },
    {
      "activations/layer0_attention_weight_max": 15.681380271911621,
      "activations/layer0_attention_weight_min": -11.601524353027344,
      "activations/layer10_attention_weight_max": 35.39789581298828,
      "activations/layer10_attention_weight_min": -32.74195861816406,
      "activations/layer11_attention_weight_max": 34.307289123535156,
      "activations/layer11_attention_weight_min": -32.01258087158203,
      "activations/layer12_attention_weight_max": 23.942655563354492,
      "activations/layer12_attention_weight_min": -27.0308895111084,
      "activations/layer13_attention_weight_max": 37.72481918334961,
      "activations/layer13_attention_weight_min": -34.231441497802734,
      "activations/layer14_attention_weight_max": 47.36435317993164,
      "activations/layer14_attention_weight_min": -39.309696197509766,
      "activations/layer15_attention_weight_max": 33.64887237548828,
      "activations/layer15_attention_weight_min": -28.685115814208984,
      "activations/layer16_attention_weight_max": 29.751708984375,
      "activations/layer16_attention_weight_min": -28.491621017456055,
      "activations/layer17_attention_weight_max": 30.111494064331055,
      "activations/layer17_attention_weight_min": -23.926605224609375,
      "activations/layer18_attention_weight_max": 28.1876277923584,
      "activations/layer18_attention_weight_min": -21.232046127319336,
      "activations/layer19_attention_weight_max": 33.05952835083008,
      "activations/layer19_attention_weight_min": -29.349044799804688,
      "activations/layer1_attention_weight_max": 16.834232330322266,
      "activations/layer1_attention_weight_min": -13.637446403503418,
      "activations/layer20_attention_weight_max": 29.37447738647461,
      "activations/layer20_attention_weight_min": -24.970352172851562,
      "activations/layer21_attention_weight_max": 27.372055053710938,
      "activations/layer21_attention_weight_min": -22.139923095703125,
      "activations/layer22_attention_weight_max": 44.6196174621582,
      "activations/layer22_attention_weight_min": -27.468130111694336,
      "activations/layer23_attention_weight_max": 34.698814392089844,
      "activations/layer23_attention_weight_min": -25.272878646850586,
      "activations/layer2_attention_weight_max": 30.99251365661621,
      "activations/layer2_attention_weight_min": -31.73062515258789,
      "activations/layer3_attention_weight_max": 94.48362731933594,
      "activations/layer3_attention_weight_min": -94.00190734863281,
      "activations/layer4_attention_weight_max": 120.81062316894531,
      "activations/layer4_attention_weight_min": -117.9482192993164,
      "activations/layer5_attention_weight_max": 55.039581298828125,
      "activations/layer5_attention_weight_min": -62.589290618896484,
      "activations/layer6_attention_weight_max": 50.19779968261719,
      "activations/layer6_attention_weight_min": -50.34014129638672,
      "activations/layer7_attention_weight_max": 95.37129211425781,
      "activations/layer7_attention_weight_min": -87.73137664794922,
      "activations/layer8_attention_weight_max": 42.3885383605957,
      "activations/layer8_attention_weight_min": -41.942325592041016,
      "activations/layer9_attention_weight_max": 36.42580795288086,
      "activations/layer9_attention_weight_min": -35.9786262512207,
      "epoch": 8.86,
      "learning_rate": 9.378181818181817e-05,
      "loss": 2.7945,
      "step": 152550
    },
    {
      "activations/layer0_attention_weight_max": 15.082050323486328,
      "activations/layer0_attention_weight_min": -11.773720741271973,
      "activations/layer10_attention_weight_max": 32.6234016418457,
      "activations/layer10_attention_weight_min": -33.687286376953125,
      "activations/layer11_attention_weight_max": 33.16846466064453,
      "activations/layer11_attention_weight_min": -33.842586517333984,
      "activations/layer12_attention_weight_max": 24.545440673828125,
      "activations/layer12_attention_weight_min": -24.46752166748047,
      "activations/layer13_attention_weight_max": 40.151607513427734,
      "activations/layer13_attention_weight_min": -33.58441925048828,
      "activations/layer14_attention_weight_max": 47.54972457885742,
      "activations/layer14_attention_weight_min": -39.641178131103516,
      "activations/layer15_attention_weight_max": 36.68301010131836,
      "activations/layer15_attention_weight_min": -31.378738403320312,
      "activations/layer16_attention_weight_max": 31.749149322509766,
      "activations/layer16_attention_weight_min": -27.546283721923828,
      "activations/layer17_attention_weight_max": 31.392419815063477,
      "activations/layer17_attention_weight_min": -25.406513214111328,
      "activations/layer18_attention_weight_max": 34.24516296386719,
      "activations/layer18_attention_weight_min": -23.073930740356445,
      "activations/layer19_attention_weight_max": 34.055152893066406,
      "activations/layer19_attention_weight_min": -29.438230514526367,
      "activations/layer1_attention_weight_max": 17.68380355834961,
      "activations/layer1_attention_weight_min": -14.890599250793457,
      "activations/layer20_attention_weight_max": 28.180973052978516,
      "activations/layer20_attention_weight_min": -23.685728073120117,
      "activations/layer21_attention_weight_max": 30.699718475341797,
      "activations/layer21_attention_weight_min": -22.980356216430664,
      "activations/layer22_attention_weight_max": 48.42509078979492,
      "activations/layer22_attention_weight_min": -28.28852081298828,
      "activations/layer23_attention_weight_max": 35.416805267333984,
      "activations/layer23_attention_weight_min": -23.910797119140625,
      "activations/layer2_attention_weight_max": 31.160327911376953,
      "activations/layer2_attention_weight_min": -31.514713287353516,
      "activations/layer3_attention_weight_max": 96.92835235595703,
      "activations/layer3_attention_weight_min": -99.20938110351562,
      "activations/layer4_attention_weight_max": 116.29127502441406,
      "activations/layer4_attention_weight_min": -115.2705078125,
      "activations/layer5_attention_weight_max": 52.86994934082031,
      "activations/layer5_attention_weight_min": -62.42646789550781,
      "activations/layer6_attention_weight_max": 48.35965347290039,
      "activations/layer6_attention_weight_min": -51.733253479003906,
      "activations/layer7_attention_weight_max": 89.2313232421875,
      "activations/layer7_attention_weight_min": -92.87203979492188,
      "activations/layer8_attention_weight_max": 40.341129302978516,
      "activations/layer8_attention_weight_min": -41.321205139160156,
      "activations/layer9_attention_weight_max": 34.3974609375,
      "activations/layer9_attention_weight_min": -34.52354049682617,
      "epoch": 8.87,
      "learning_rate": 9.376325757575756e-05,
      "loss": 2.7993,
      "step": 152600
    },
    {
      "activations/layer0_attention_weight_max": 15.62835693359375,
      "activations/layer0_attention_weight_min": -12.056482315063477,
      "activations/layer10_attention_weight_max": 30.9141845703125,
      "activations/layer10_attention_weight_min": -31.545236587524414,
      "activations/layer11_attention_weight_max": 30.672414779663086,
      "activations/layer11_attention_weight_min": -29.733436584472656,
      "activations/layer12_attention_weight_max": 23.354795455932617,
      "activations/layer12_attention_weight_min": -26.50825309753418,
      "activations/layer13_attention_weight_max": 37.90827941894531,
      "activations/layer13_attention_weight_min": -34.252227783203125,
      "activations/layer14_attention_weight_max": 46.90132141113281,
      "activations/layer14_attention_weight_min": -37.60886001586914,
      "activations/layer15_attention_weight_max": 33.95719528198242,
      "activations/layer15_attention_weight_min": -29.996679306030273,
      "activations/layer16_attention_weight_max": 29.975215911865234,
      "activations/layer16_attention_weight_min": -28.56692123413086,
      "activations/layer17_attention_weight_max": 32.599430084228516,
      "activations/layer17_attention_weight_min": -26.4794864654541,
      "activations/layer18_attention_weight_max": 32.97256851196289,
      "activations/layer18_attention_weight_min": -24.715248107910156,
      "activations/layer19_attention_weight_max": 33.09571075439453,
      "activations/layer19_attention_weight_min": -30.294870376586914,
      "activations/layer1_attention_weight_max": 16.686521530151367,
      "activations/layer1_attention_weight_min": -13.942252159118652,
      "activations/layer20_attention_weight_max": 29.107280731201172,
      "activations/layer20_attention_weight_min": -25.438371658325195,
      "activations/layer21_attention_weight_max": 27.824588775634766,
      "activations/layer21_attention_weight_min": -23.68459129333496,
      "activations/layer22_attention_weight_max": 44.98394775390625,
      "activations/layer22_attention_weight_min": -28.897642135620117,
      "activations/layer23_attention_weight_max": 32.975582122802734,
      "activations/layer23_attention_weight_min": -24.896663665771484,
      "activations/layer2_attention_weight_max": 30.709457397460938,
      "activations/layer2_attention_weight_min": -31.554935455322266,
      "activations/layer3_attention_weight_max": 93.99781799316406,
      "activations/layer3_attention_weight_min": -93.4874496459961,
      "activations/layer4_attention_weight_max": 112.6009292602539,
      "activations/layer4_attention_weight_min": -114.5393295288086,
      "activations/layer5_attention_weight_max": 52.995582580566406,
      "activations/layer5_attention_weight_min": -59.18940353393555,
      "activations/layer6_attention_weight_max": 43.296024322509766,
      "activations/layer6_attention_weight_min": -50.35026550292969,
      "activations/layer7_attention_weight_max": 82.88372802734375,
      "activations/layer7_attention_weight_min": -90.1631088256836,
      "activations/layer8_attention_weight_max": 37.22169876098633,
      "activations/layer8_attention_weight_min": -40.52480697631836,
      "activations/layer9_attention_weight_max": 32.688663482666016,
      "activations/layer9_attention_weight_min": -32.0041618347168,
      "epoch": 8.87,
      "learning_rate": 9.374431818181818e-05,
      "loss": 2.7855,
      "step": 152650
    },
    {
      "activations/layer0_attention_weight_max": 14.282023429870605,
      "activations/layer0_attention_weight_min": -12.47938060760498,
      "activations/layer10_attention_weight_max": 32.20892333984375,
      "activations/layer10_attention_weight_min": -31.035106658935547,
      "activations/layer11_attention_weight_max": 33.84984588623047,
      "activations/layer11_attention_weight_min": -29.24794578552246,
      "activations/layer12_attention_weight_max": 31.284252166748047,
      "activations/layer12_attention_weight_min": -25.405405044555664,
      "activations/layer13_attention_weight_max": 42.00364685058594,
      "activations/layer13_attention_weight_min": -32.31706237792969,
      "activations/layer14_attention_weight_max": 51.68163299560547,
      "activations/layer14_attention_weight_min": -40.48848342895508,
      "activations/layer15_attention_weight_max": 38.77372741699219,
      "activations/layer15_attention_weight_min": -29.01015853881836,
      "activations/layer16_attention_weight_max": 38.69852066040039,
      "activations/layer16_attention_weight_min": -27.36136817932129,
      "activations/layer17_attention_weight_max": 32.58308792114258,
      "activations/layer17_attention_weight_min": -28.409284591674805,
      "activations/layer18_attention_weight_max": 31.693031311035156,
      "activations/layer18_attention_weight_min": -22.448341369628906,
      "activations/layer19_attention_weight_max": 36.11910629272461,
      "activations/layer19_attention_weight_min": -30.12872314453125,
      "activations/layer1_attention_weight_max": 16.950193405151367,
      "activations/layer1_attention_weight_min": -13.815706253051758,
      "activations/layer20_attention_weight_max": 32.01881790161133,
      "activations/layer20_attention_weight_min": -23.267423629760742,
      "activations/layer21_attention_weight_max": 30.543025970458984,
      "activations/layer21_attention_weight_min": -23.839508056640625,
      "activations/layer22_attention_weight_max": 45.15802001953125,
      "activations/layer22_attention_weight_min": -28.549360275268555,
      "activations/layer23_attention_weight_max": 38.893882751464844,
      "activations/layer23_attention_weight_min": -25.396438598632812,
      "activations/layer2_attention_weight_max": 31.943645477294922,
      "activations/layer2_attention_weight_min": -31.685951232910156,
      "activations/layer3_attention_weight_max": 93.7608413696289,
      "activations/layer3_attention_weight_min": -96.6192626953125,
      "activations/layer4_attention_weight_max": 115.1739273071289,
      "activations/layer4_attention_weight_min": -116.2992935180664,
      "activations/layer5_attention_weight_max": 53.26564025878906,
      "activations/layer5_attention_weight_min": -61.2458610534668,
      "activations/layer6_attention_weight_max": 46.706180572509766,
      "activations/layer6_attention_weight_min": -50.782188415527344,
      "activations/layer7_attention_weight_max": 88.98566436767578,
      "activations/layer7_attention_weight_min": -87.23188781738281,
      "activations/layer8_attention_weight_max": 43.60478973388672,
      "activations/layer8_attention_weight_min": -40.31920623779297,
      "activations/layer9_attention_weight_max": 36.14618682861328,
      "activations/layer9_attention_weight_min": -32.372806549072266,
      "epoch": 8.87,
      "learning_rate": 9.372537878787878e-05,
      "loss": 2.7884,
      "step": 152700
    },
    {
      "activations/layer0_attention_weight_max": 15.317631721496582,
      "activations/layer0_attention_weight_min": -11.951253890991211,
      "activations/layer10_attention_weight_max": 33.20286560058594,
      "activations/layer10_attention_weight_min": -30.76401138305664,
      "activations/layer11_attention_weight_max": 32.51683044433594,
      "activations/layer11_attention_weight_min": -30.262218475341797,
      "activations/layer12_attention_weight_max": 24.094125747680664,
      "activations/layer12_attention_weight_min": -24.671199798583984,
      "activations/layer13_attention_weight_max": 40.774627685546875,
      "activations/layer13_attention_weight_min": -34.785118103027344,
      "activations/layer14_attention_weight_max": 48.827571868896484,
      "activations/layer14_attention_weight_min": -40.049720764160156,
      "activations/layer15_attention_weight_max": 37.60888671875,
      "activations/layer15_attention_weight_min": -30.466102600097656,
      "activations/layer16_attention_weight_max": 34.598663330078125,
      "activations/layer16_attention_weight_min": -27.53755760192871,
      "activations/layer17_attention_weight_max": 32.73366165161133,
      "activations/layer17_attention_weight_min": -24.9787540435791,
      "activations/layer18_attention_weight_max": 31.28782844543457,
      "activations/layer18_attention_weight_min": -21.787025451660156,
      "activations/layer19_attention_weight_max": 39.78981399536133,
      "activations/layer19_attention_weight_min": -32.332576751708984,
      "activations/layer1_attention_weight_max": 16.093015670776367,
      "activations/layer1_attention_weight_min": -14.046012878417969,
      "activations/layer20_attention_weight_max": 31.973770141601562,
      "activations/layer20_attention_weight_min": -26.394811630249023,
      "activations/layer21_attention_weight_max": 29.999753952026367,
      "activations/layer21_attention_weight_min": -24.81187629699707,
      "activations/layer22_attention_weight_max": 53.870849609375,
      "activations/layer22_attention_weight_min": -31.71470069885254,
      "activations/layer23_attention_weight_max": 42.965614318847656,
      "activations/layer23_attention_weight_min": -22.71694564819336,
      "activations/layer2_attention_weight_max": 31.36740493774414,
      "activations/layer2_attention_weight_min": -32.367462158203125,
      "activations/layer3_attention_weight_max": 93.67793273925781,
      "activations/layer3_attention_weight_min": -95.76332092285156,
      "activations/layer4_attention_weight_max": 118.4396743774414,
      "activations/layer4_attention_weight_min": -116.07913970947266,
      "activations/layer5_attention_weight_max": 55.91789245605469,
      "activations/layer5_attention_weight_min": -61.08979797363281,
      "activations/layer6_attention_weight_max": 49.76365661621094,
      "activations/layer6_attention_weight_min": -50.624656677246094,
      "activations/layer7_attention_weight_max": 92.32743072509766,
      "activations/layer7_attention_weight_min": -92.1215591430664,
      "activations/layer8_attention_weight_max": 42.22603225708008,
      "activations/layer8_attention_weight_min": -43.92656326293945,
      "activations/layer9_attention_weight_max": 36.18001174926758,
      "activations/layer9_attention_weight_min": -33.16880798339844,
      "epoch": 8.88,
      "learning_rate": 9.370643939393939e-05,
      "loss": 2.8016,
      "step": 152750
    },
    {
      "activations/layer0_attention_weight_max": 14.94968318939209,
      "activations/layer0_attention_weight_min": -12.542072296142578,
      "activations/layer10_attention_weight_max": 33.99560546875,
      "activations/layer10_attention_weight_min": -33.69136047363281,
      "activations/layer11_attention_weight_max": 35.809268951416016,
      "activations/layer11_attention_weight_min": -32.03282165527344,
      "activations/layer12_attention_weight_max": 26.018770217895508,
      "activations/layer12_attention_weight_min": -25.41507911682129,
      "activations/layer13_attention_weight_max": 46.88454818725586,
      "activations/layer13_attention_weight_min": -34.70220947265625,
      "activations/layer14_attention_weight_max": 51.524112701416016,
      "activations/layer14_attention_weight_min": -44.82201385498047,
      "activations/layer15_attention_weight_max": 39.20738983154297,
      "activations/layer15_attention_weight_min": -32.46837615966797,
      "activations/layer16_attention_weight_max": 36.03113555908203,
      "activations/layer16_attention_weight_min": -27.996013641357422,
      "activations/layer17_attention_weight_max": 35.727909088134766,
      "activations/layer17_attention_weight_min": -25.044391632080078,
      "activations/layer18_attention_weight_max": 33.289894104003906,
      "activations/layer18_attention_weight_min": -21.761241912841797,
      "activations/layer19_attention_weight_max": 38.37180709838867,
      "activations/layer19_attention_weight_min": -31.858366012573242,
      "activations/layer1_attention_weight_max": 16.09273910522461,
      "activations/layer1_attention_weight_min": -14.196126937866211,
      "activations/layer20_attention_weight_max": 29.879926681518555,
      "activations/layer20_attention_weight_min": -24.855224609375,
      "activations/layer21_attention_weight_max": 31.88573455810547,
      "activations/layer21_attention_weight_min": -24.4315128326416,
      "activations/layer22_attention_weight_max": 50.57763671875,
      "activations/layer22_attention_weight_min": -30.817792892456055,
      "activations/layer23_attention_weight_max": 41.71946716308594,
      "activations/layer23_attention_weight_min": -23.843189239501953,
      "activations/layer2_attention_weight_max": 33.57284164428711,
      "activations/layer2_attention_weight_min": -32.27806854248047,
      "activations/layer3_attention_weight_max": 98.82012939453125,
      "activations/layer3_attention_weight_min": -98.51067352294922,
      "activations/layer4_attention_weight_max": 122.1948471069336,
      "activations/layer4_attention_weight_min": -121.76399230957031,
      "activations/layer5_attention_weight_max": 56.17427062988281,
      "activations/layer5_attention_weight_min": -65.20138549804688,
      "activations/layer6_attention_weight_max": 49.25379180908203,
      "activations/layer6_attention_weight_min": -52.30342483520508,
      "activations/layer7_attention_weight_max": 99.52599334716797,
      "activations/layer7_attention_weight_min": -94.25341796875,
      "activations/layer8_attention_weight_max": 44.10063552856445,
      "activations/layer8_attention_weight_min": -42.291473388671875,
      "activations/layer9_attention_weight_max": 37.90842056274414,
      "activations/layer9_attention_weight_min": -36.80158996582031,
      "epoch": 8.88,
      "learning_rate": 9.36875e-05,
      "loss": 2.7815,
      "step": 152800
    },
    {
      "activations/layer0_attention_weight_max": 14.563552856445312,
      "activations/layer0_attention_weight_min": -12.184678077697754,
      "activations/layer10_attention_weight_max": 34.88196563720703,
      "activations/layer10_attention_weight_min": -34.26097106933594,
      "activations/layer11_attention_weight_max": 36.304298400878906,
      "activations/layer11_attention_weight_min": -32.045997619628906,
      "activations/layer12_attention_weight_max": 26.10769271850586,
      "activations/layer12_attention_weight_min": -25.41061019897461,
      "activations/layer13_attention_weight_max": 48.75534439086914,
      "activations/layer13_attention_weight_min": -36.223873138427734,
      "activations/layer14_attention_weight_max": 56.34147262573242,
      "activations/layer14_attention_weight_min": -39.06626892089844,
      "activations/layer15_attention_weight_max": 43.81364822387695,
      "activations/layer15_attention_weight_min": -31.373268127441406,
      "activations/layer16_attention_weight_max": 35.35935974121094,
      "activations/layer16_attention_weight_min": -28.25461769104004,
      "activations/layer17_attention_weight_max": 42.79359817504883,
      "activations/layer17_attention_weight_min": -25.59663963317871,
      "activations/layer18_attention_weight_max": 32.4876823425293,
      "activations/layer18_attention_weight_min": -21.6712703704834,
      "activations/layer19_attention_weight_max": 36.97126388549805,
      "activations/layer19_attention_weight_min": -30.402273178100586,
      "activations/layer1_attention_weight_max": 16.495729446411133,
      "activations/layer1_attention_weight_min": -13.205835342407227,
      "activations/layer20_attention_weight_max": 34.79100036621094,
      "activations/layer20_attention_weight_min": -27.722965240478516,
      "activations/layer21_attention_weight_max": 32.31166458129883,
      "activations/layer21_attention_weight_min": -22.863265991210938,
      "activations/layer22_attention_weight_max": 47.10639190673828,
      "activations/layer22_attention_weight_min": -27.641233444213867,
      "activations/layer23_attention_weight_max": 37.13288497924805,
      "activations/layer23_attention_weight_min": -23.428585052490234,
      "activations/layer2_attention_weight_max": 30.906532287597656,
      "activations/layer2_attention_weight_min": -30.976154327392578,
      "activations/layer3_attention_weight_max": 94.59516143798828,
      "activations/layer3_attention_weight_min": -96.11805725097656,
      "activations/layer4_attention_weight_max": 121.3980712890625,
      "activations/layer4_attention_weight_min": -116.4511489868164,
      "activations/layer5_attention_weight_max": 55.35345458984375,
      "activations/layer5_attention_weight_min": -63.58721923828125,
      "activations/layer6_attention_weight_max": 48.18087387084961,
      "activations/layer6_attention_weight_min": -51.8740119934082,
      "activations/layer7_attention_weight_max": 92.55145263671875,
      "activations/layer7_attention_weight_min": -93.50787353515625,
      "activations/layer8_attention_weight_max": 41.939022064208984,
      "activations/layer8_attention_weight_min": -42.24128723144531,
      "activations/layer9_attention_weight_max": 37.82046127319336,
      "activations/layer9_attention_weight_min": -35.54172134399414,
      "epoch": 8.88,
      "learning_rate": 9.36685606060606e-05,
      "loss": 2.8042,
      "step": 152850
    },
    {
      "activations/layer0_attention_weight_max": 15.495564460754395,
      "activations/layer0_attention_weight_min": -11.657574653625488,
      "activations/layer10_attention_weight_max": 32.13842010498047,
      "activations/layer10_attention_weight_min": -32.199710845947266,
      "activations/layer11_attention_weight_max": 31.649978637695312,
      "activations/layer11_attention_weight_min": -30.764759063720703,
      "activations/layer12_attention_weight_max": 27.71201515197754,
      "activations/layer12_attention_weight_min": -25.23228645324707,
      "activations/layer13_attention_weight_max": 37.43695068359375,
      "activations/layer13_attention_weight_min": -36.46385955810547,
      "activations/layer14_attention_weight_max": 45.950050354003906,
      "activations/layer14_attention_weight_min": -37.39107131958008,
      "activations/layer15_attention_weight_max": 35.81296920776367,
      "activations/layer15_attention_weight_min": -29.591630935668945,
      "activations/layer16_attention_weight_max": 30.661962509155273,
      "activations/layer16_attention_weight_min": -28.399477005004883,
      "activations/layer17_attention_weight_max": 32.45219802856445,
      "activations/layer17_attention_weight_min": -25.23137092590332,
      "activations/layer18_attention_weight_max": 32.77497482299805,
      "activations/layer18_attention_weight_min": -21.840129852294922,
      "activations/layer19_attention_weight_max": 33.347259521484375,
      "activations/layer19_attention_weight_min": -31.47661781311035,
      "activations/layer1_attention_weight_max": 16.28557777404785,
      "activations/layer1_attention_weight_min": -13.682991027832031,
      "activations/layer20_attention_weight_max": 28.8421630859375,
      "activations/layer20_attention_weight_min": -25.053621292114258,
      "activations/layer21_attention_weight_max": 32.031314849853516,
      "activations/layer21_attention_weight_min": -23.878141403198242,
      "activations/layer22_attention_weight_max": 43.05841064453125,
      "activations/layer22_attention_weight_min": -29.47739601135254,
      "activations/layer23_attention_weight_max": 36.469146728515625,
      "activations/layer23_attention_weight_min": -23.413360595703125,
      "activations/layer2_attention_weight_max": 29.635465621948242,
      "activations/layer2_attention_weight_min": -31.438507080078125,
      "activations/layer3_attention_weight_max": 94.52579498291016,
      "activations/layer3_attention_weight_min": -97.39905548095703,
      "activations/layer4_attention_weight_max": 115.5016098022461,
      "activations/layer4_attention_weight_min": -116.19132232666016,
      "activations/layer5_attention_weight_max": 51.454586029052734,
      "activations/layer5_attention_weight_min": -58.90015411376953,
      "activations/layer6_attention_weight_max": 47.03949737548828,
      "activations/layer6_attention_weight_min": -48.73411560058594,
      "activations/layer7_attention_weight_max": 91.88356018066406,
      "activations/layer7_attention_weight_min": -93.6984634399414,
      "activations/layer8_attention_weight_max": 38.47304916381836,
      "activations/layer8_attention_weight_min": -41.34697341918945,
      "activations/layer9_attention_weight_max": 34.555686950683594,
      "activations/layer9_attention_weight_min": -33.286277770996094,
      "epoch": 8.88,
      "learning_rate": 9.364962121212121e-05,
      "loss": 2.7779,
      "step": 152900
    },
    {
      "activations/layer0_attention_weight_max": 15.5408935546875,
      "activations/layer0_attention_weight_min": -12.046625137329102,
      "activations/layer10_attention_weight_max": 32.41643524169922,
      "activations/layer10_attention_weight_min": -32.861454010009766,
      "activations/layer11_attention_weight_max": 30.84748077392578,
      "activations/layer11_attention_weight_min": -30.188312530517578,
      "activations/layer12_attention_weight_max": 39.0401725769043,
      "activations/layer12_attention_weight_min": -24.463132858276367,
      "activations/layer13_attention_weight_max": 43.132850646972656,
      "activations/layer13_attention_weight_min": -32.806846618652344,
      "activations/layer14_attention_weight_max": 44.69698715209961,
      "activations/layer14_attention_weight_min": -41.02584457397461,
      "activations/layer15_attention_weight_max": 34.12534713745117,
      "activations/layer15_attention_weight_min": -31.5173282623291,
      "activations/layer16_attention_weight_max": 29.67656135559082,
      "activations/layer16_attention_weight_min": -33.07439041137695,
      "activations/layer17_attention_weight_max": 31.644832611083984,
      "activations/layer17_attention_weight_min": -25.564687728881836,
      "activations/layer18_attention_weight_max": 28.166770935058594,
      "activations/layer18_attention_weight_min": -22.176176071166992,
      "activations/layer19_attention_weight_max": 34.542320251464844,
      "activations/layer19_attention_weight_min": -33.44247055053711,
      "activations/layer1_attention_weight_max": 16.65636444091797,
      "activations/layer1_attention_weight_min": -14.084213256835938,
      "activations/layer20_attention_weight_max": 31.875286102294922,
      "activations/layer20_attention_weight_min": -24.08094596862793,
      "activations/layer21_attention_weight_max": 28.450517654418945,
      "activations/layer21_attention_weight_min": -22.893783569335938,
      "activations/layer22_attention_weight_max": 45.34027862548828,
      "activations/layer22_attention_weight_min": -27.545034408569336,
      "activations/layer23_attention_weight_max": 36.85963439941406,
      "activations/layer23_attention_weight_min": -22.834232330322266,
      "activations/layer2_attention_weight_max": 32.266231536865234,
      "activations/layer2_attention_weight_min": -31.268917083740234,
      "activations/layer3_attention_weight_max": 95.0107421875,
      "activations/layer3_attention_weight_min": -96.05960083007812,
      "activations/layer4_attention_weight_max": 113.96871185302734,
      "activations/layer4_attention_weight_min": -116.24486541748047,
      "activations/layer5_attention_weight_max": 52.445953369140625,
      "activations/layer5_attention_weight_min": -64.48625183105469,
      "activations/layer6_attention_weight_max": 50.79853820800781,
      "activations/layer6_attention_weight_min": -50.32023239135742,
      "activations/layer7_attention_weight_max": 90.23702239990234,
      "activations/layer7_attention_weight_min": -86.95370483398438,
      "activations/layer8_attention_weight_max": 40.361881256103516,
      "activations/layer8_attention_weight_min": -39.035888671875,
      "activations/layer9_attention_weight_max": 34.58603286743164,
      "activations/layer9_attention_weight_min": -33.2291374206543,
      "epoch": 8.89,
      "learning_rate": 9.36306818181818e-05,
      "loss": 2.7906,
      "step": 152950
    },
    {
      "activations/layer0_attention_weight_max": 14.815293312072754,
      "activations/layer0_attention_weight_min": -11.814485549926758,
      "activations/layer10_attention_weight_max": 34.542724609375,
      "activations/layer10_attention_weight_min": -33.497291564941406,
      "activations/layer11_attention_weight_max": 35.43079376220703,
      "activations/layer11_attention_weight_min": -30.639928817749023,
      "activations/layer12_attention_weight_max": 46.40532684326172,
      "activations/layer12_attention_weight_min": -27.634658813476562,
      "activations/layer13_attention_weight_max": 48.35926818847656,
      "activations/layer13_attention_weight_min": -34.57566452026367,
      "activations/layer14_attention_weight_max": 48.77421951293945,
      "activations/layer14_attention_weight_min": -43.06291580200195,
      "activations/layer15_attention_weight_max": 35.097896575927734,
      "activations/layer15_attention_weight_min": -29.6552677154541,
      "activations/layer16_attention_weight_max": 32.48663330078125,
      "activations/layer16_attention_weight_min": -29.42881202697754,
      "activations/layer17_attention_weight_max": 34.30956268310547,
      "activations/layer17_attention_weight_min": -25.37344741821289,
      "activations/layer18_attention_weight_max": 32.83347702026367,
      "activations/layer18_attention_weight_min": -22.778614044189453,
      "activations/layer19_attention_weight_max": 38.02423095703125,
      "activations/layer19_attention_weight_min": -30.655935287475586,
      "activations/layer1_attention_weight_max": 16.055089950561523,
      "activations/layer1_attention_weight_min": -13.6322660446167,
      "activations/layer20_attention_weight_max": 29.656187057495117,
      "activations/layer20_attention_weight_min": -23.14386558532715,
      "activations/layer21_attention_weight_max": 29.95290184020996,
      "activations/layer21_attention_weight_min": -25.012537002563477,
      "activations/layer22_attention_weight_max": 48.613059997558594,
      "activations/layer22_attention_weight_min": -29.596765518188477,
      "activations/layer23_attention_weight_max": 36.3201904296875,
      "activations/layer23_attention_weight_min": -23.728626251220703,
      "activations/layer2_attention_weight_max": 32.36792755126953,
      "activations/layer2_attention_weight_min": -35.3992805480957,
      "activations/layer3_attention_weight_max": 100.6942367553711,
      "activations/layer3_attention_weight_min": -105.28746795654297,
      "activations/layer4_attention_weight_max": 123.47712707519531,
      "activations/layer4_attention_weight_min": -125.33853912353516,
      "activations/layer5_attention_weight_max": 56.240867614746094,
      "activations/layer5_attention_weight_min": -64.9083023071289,
      "activations/layer6_attention_weight_max": 50.22290802001953,
      "activations/layer6_attention_weight_min": -53.76239776611328,
      "activations/layer7_attention_weight_max": 91.82955169677734,
      "activations/layer7_attention_weight_min": -97.95153045654297,
      "activations/layer8_attention_weight_max": 42.52202224731445,
      "activations/layer8_attention_weight_min": -40.91574478149414,
      "activations/layer9_attention_weight_max": 34.62080001831055,
      "activations/layer9_attention_weight_min": -34.19063186645508,
      "epoch": 8.89,
      "learning_rate": 9.361174242424241e-05,
      "loss": 2.7909,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.8124,
      "eval_samples_per_second": 487.265,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.8124,
      "eval_openwebtext_samples_per_second": 487.265,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9063,
      "eval_wikitext_samples_per_second": 239.213,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_lambada_loss": 2.541015625,
      "eval_lambada_ppl": 12.692555303381242,
      "eval_lambada_runtime": 9.4926,
      "eval_lambada_samples_per_second": 512.924,
      "step": 153000
    },
    {
      "activations/layer0_attention_weight_max": 14.942404747009277,
      "activations/layer0_attention_weight_min": -12.607872009277344,
      "activations/layer10_attention_weight_max": 35.203765869140625,
      "activations/layer10_attention_weight_min": -34.75104522705078,
      "activations/layer11_attention_weight_max": 31.594295501708984,
      "activations/layer11_attention_weight_min": -32.54043960571289,
      "activations/layer12_attention_weight_max": 26.074867248535156,
      "activations/layer12_attention_weight_min": -24.465911865234375,
      "activations/layer13_attention_weight_max": 41.98805236816406,
      "activations/layer13_attention_weight_min": -38.87531280517578,
      "activations/layer14_attention_weight_max": 47.548892974853516,
      "activations/layer14_attention_weight_min": -38.39907455444336,
      "activations/layer15_attention_weight_max": 34.91082763671875,
      "activations/layer15_attention_weight_min": -30.675275802612305,
      "activations/layer16_attention_weight_max": 30.54395294189453,
      "activations/layer16_attention_weight_min": -28.29541778564453,
      "activations/layer17_attention_weight_max": 30.95022964477539,
      "activations/layer17_attention_weight_min": -24.15732192993164,
      "activations/layer18_attention_weight_max": 31.297077178955078,
      "activations/layer18_attention_weight_min": -22.576074600219727,
      "activations/layer19_attention_weight_max": 35.45525360107422,
      "activations/layer19_attention_weight_min": -29.69788932800293,
      "activations/layer1_attention_weight_max": 16.28305435180664,
      "activations/layer1_attention_weight_min": -13.017721176147461,
      "activations/layer20_attention_weight_max": 29.910449981689453,
      "activations/layer20_attention_weight_min": -24.017805099487305,
      "activations/layer21_attention_weight_max": 25.965957641601562,
      "activations/layer21_attention_weight_min": -23.54557991027832,
      "activations/layer22_attention_weight_max": 47.3867301940918,
      "activations/layer22_attention_weight_min": -29.24238395690918,
      "activations/layer23_attention_weight_max": 36.53690719604492,
      "activations/layer23_attention_weight_min": -24.40888214111328,
      "activations/layer2_attention_weight_max": 31.729206085205078,
      "activations/layer2_attention_weight_min": -32.27454376220703,
      "activations/layer3_attention_weight_max": 95.46880340576172,
      "activations/layer3_attention_weight_min": -96.53658294677734,
      "activations/layer4_attention_weight_max": 117.31904602050781,
      "activations/layer4_attention_weight_min": -122.35382080078125,
      "activations/layer5_attention_weight_max": 55.512908935546875,
      "activations/layer5_attention_weight_min": -62.59322738647461,
      "activations/layer6_attention_weight_max": 50.45846176147461,
      "activations/layer6_attention_weight_min": -56.81065368652344,
      "activations/layer7_attention_weight_max": 95.60182189941406,
      "activations/layer7_attention_weight_min": -94.7900619506836,
      "activations/layer8_attention_weight_max": 42.71612548828125,
      "activations/layer8_attention_weight_min": -44.7011604309082,
      "activations/layer9_attention_weight_max": 36.47261047363281,
      "activations/layer9_attention_weight_min": -37.13699722290039,
      "epoch": 8.89,
      "learning_rate": 9.359280303030303e-05,
      "loss": 2.8015,
      "step": 153050
    },
    {
      "activations/layer0_attention_weight_max": 15.075091361999512,
      "activations/layer0_attention_weight_min": -12.516729354858398,
      "activations/layer10_attention_weight_max": 33.28514099121094,
      "activations/layer10_attention_weight_min": -31.90114402770996,
      "activations/layer11_attention_weight_max": 32.801387786865234,
      "activations/layer11_attention_weight_min": -29.439205169677734,
      "activations/layer12_attention_weight_max": 22.164596557617188,
      "activations/layer12_attention_weight_min": -26.336055755615234,
      "activations/layer13_attention_weight_max": 34.14078903198242,
      "activations/layer13_attention_weight_min": -34.288429260253906,
      "activations/layer14_attention_weight_max": 40.59896469116211,
      "activations/layer14_attention_weight_min": -34.15180587768555,
      "activations/layer15_attention_weight_max": 33.77145767211914,
      "activations/layer15_attention_weight_min": -29.794963836669922,
      "activations/layer16_attention_weight_max": 30.776948928833008,
      "activations/layer16_attention_weight_min": -26.794801712036133,
      "activations/layer17_attention_weight_max": 31.702911376953125,
      "activations/layer17_attention_weight_min": -23.745115280151367,
      "activations/layer18_attention_weight_max": 30.95286750793457,
      "activations/layer18_attention_weight_min": -20.141502380371094,
      "activations/layer19_attention_weight_max": 33.8156852722168,
      "activations/layer19_attention_weight_min": -30.548099517822266,
      "activations/layer1_attention_weight_max": 16.86876106262207,
      "activations/layer1_attention_weight_min": -14.096868515014648,
      "activations/layer20_attention_weight_max": 30.37060546875,
      "activations/layer20_attention_weight_min": -23.356525421142578,
      "activations/layer21_attention_weight_max": 32.93788146972656,
      "activations/layer21_attention_weight_min": -21.795856475830078,
      "activations/layer22_attention_weight_max": 42.40032958984375,
      "activations/layer22_attention_weight_min": -26.3001766204834,
      "activations/layer23_attention_weight_max": 29.952634811401367,
      "activations/layer23_attention_weight_min": -23.15163230895996,
      "activations/layer2_attention_weight_max": 33.199424743652344,
      "activations/layer2_attention_weight_min": -31.977773666381836,
      "activations/layer3_attention_weight_max": 92.8958740234375,
      "activations/layer3_attention_weight_min": -96.97313690185547,
      "activations/layer4_attention_weight_max": 117.78533935546875,
      "activations/layer4_attention_weight_min": -114.75318908691406,
      "activations/layer5_attention_weight_max": 52.87421417236328,
      "activations/layer5_attention_weight_min": -64.1835708618164,
      "activations/layer6_attention_weight_max": 46.1522102355957,
      "activations/layer6_attention_weight_min": -49.37503433227539,
      "activations/layer7_attention_weight_max": 90.41922760009766,
      "activations/layer7_attention_weight_min": -89.82918548583984,
      "activations/layer8_attention_weight_max": 40.54246520996094,
      "activations/layer8_attention_weight_min": -39.72784423828125,
      "activations/layer9_attention_weight_max": 32.86229705810547,
      "activations/layer9_attention_weight_min": -33.846004486083984,
      "epoch": 8.9,
      "learning_rate": 9.357386363636362e-05,
      "loss": 2.794,
      "step": 153100
    },
    {
      "activations/layer0_attention_weight_max": 16.27057647705078,
      "activations/layer0_attention_weight_min": -12.837493896484375,
      "activations/layer10_attention_weight_max": 37.29261016845703,
      "activations/layer10_attention_weight_min": -30.983537673950195,
      "activations/layer11_attention_weight_max": 37.466922760009766,
      "activations/layer11_attention_weight_min": -32.110328674316406,
      "activations/layer12_attention_weight_max": 23.763721466064453,
      "activations/layer12_attention_weight_min": -27.585283279418945,
      "activations/layer13_attention_weight_max": 38.09489440917969,
      "activations/layer13_attention_weight_min": -32.031280517578125,
      "activations/layer14_attention_weight_max": 57.560218811035156,
      "activations/layer14_attention_weight_min": -42.93787384033203,
      "activations/layer15_attention_weight_max": 38.70363998413086,
      "activations/layer15_attention_weight_min": -29.64955711364746,
      "activations/layer16_attention_weight_max": 37.969303131103516,
      "activations/layer16_attention_weight_min": -30.487821578979492,
      "activations/layer17_attention_weight_max": 34.068016052246094,
      "activations/layer17_attention_weight_min": -25.9799861907959,
      "activations/layer18_attention_weight_max": 30.930654525756836,
      "activations/layer18_attention_weight_min": -23.327320098876953,
      "activations/layer19_attention_weight_max": 41.1742057800293,
      "activations/layer19_attention_weight_min": -32.70138168334961,
      "activations/layer1_attention_weight_max": 16.102529525756836,
      "activations/layer1_attention_weight_min": -14.22805404663086,
      "activations/layer20_attention_weight_max": 34.28910446166992,
      "activations/layer20_attention_weight_min": -26.182695388793945,
      "activations/layer21_attention_weight_max": 32.30731964111328,
      "activations/layer21_attention_weight_min": -25.888845443725586,
      "activations/layer22_attention_weight_max": 45.534423828125,
      "activations/layer22_attention_weight_min": -29.80158805847168,
      "activations/layer23_attention_weight_max": 37.026275634765625,
      "activations/layer23_attention_weight_min": -25.45929527282715,
      "activations/layer2_attention_weight_max": 32.46754455566406,
      "activations/layer2_attention_weight_min": -31.995494842529297,
      "activations/layer3_attention_weight_max": 95.28582000732422,
      "activations/layer3_attention_weight_min": -97.9045181274414,
      "activations/layer4_attention_weight_max": 117.08975982666016,
      "activations/layer4_attention_weight_min": -115.40013885498047,
      "activations/layer5_attention_weight_max": 54.31707763671875,
      "activations/layer5_attention_weight_min": -62.320106506347656,
      "activations/layer6_attention_weight_max": 48.32801818847656,
      "activations/layer6_attention_weight_min": -53.466590881347656,
      "activations/layer7_attention_weight_max": 94.30318450927734,
      "activations/layer7_attention_weight_min": -87.98774719238281,
      "activations/layer8_attention_weight_max": 43.22195053100586,
      "activations/layer8_attention_weight_min": -40.25936508178711,
      "activations/layer9_attention_weight_max": 35.502628326416016,
      "activations/layer9_attention_weight_min": -34.33535385131836,
      "epoch": 8.9,
      "learning_rate": 9.355492424242423e-05,
      "loss": 2.7904,
      "step": 153150
    },
    {
      "activations/layer0_attention_weight_max": 15.745408058166504,
      "activations/layer0_attention_weight_min": -11.918598175048828,
      "activations/layer10_attention_weight_max": 34.43872833251953,
      "activations/layer10_attention_weight_min": -31.63930892944336,
      "activations/layer11_attention_weight_max": 36.81337356567383,
      "activations/layer11_attention_weight_min": -31.81795310974121,
      "activations/layer12_attention_weight_max": 45.19483947753906,
      "activations/layer12_attention_weight_min": -25.75409507751465,
      "activations/layer13_attention_weight_max": 46.52791976928711,
      "activations/layer13_attention_weight_min": -37.43016052246094,
      "activations/layer14_attention_weight_max": 50.768890380859375,
      "activations/layer14_attention_weight_min": -40.73336410522461,
      "activations/layer15_attention_weight_max": 40.449859619140625,
      "activations/layer15_attention_weight_min": -34.135841369628906,
      "activations/layer16_attention_weight_max": 37.679710388183594,
      "activations/layer16_attention_weight_min": -29.64535903930664,
      "activations/layer17_attention_weight_max": 34.21957015991211,
      "activations/layer17_attention_weight_min": -26.59739112854004,
      "activations/layer18_attention_weight_max": 33.296688079833984,
      "activations/layer18_attention_weight_min": -24.633031845092773,
      "activations/layer19_attention_weight_max": 38.61954116821289,
      "activations/layer19_attention_weight_min": -33.7719841003418,
      "activations/layer1_attention_weight_max": 16.95619010925293,
      "activations/layer1_attention_weight_min": -15.028573036193848,
      "activations/layer20_attention_weight_max": 34.24162292480469,
      "activations/layer20_attention_weight_min": -27.267892837524414,
      "activations/layer21_attention_weight_max": 35.977783203125,
      "activations/layer21_attention_weight_min": -24.380651473999023,
      "activations/layer22_attention_weight_max": 52.84689712524414,
      "activations/layer22_attention_weight_min": -30.243324279785156,
      "activations/layer23_attention_weight_max": 41.370338439941406,
      "activations/layer23_attention_weight_min": -26.73621940612793,
      "activations/layer2_attention_weight_max": 32.2966423034668,
      "activations/layer2_attention_weight_min": -31.53548812866211,
      "activations/layer3_attention_weight_max": 98.70502471923828,
      "activations/layer3_attention_weight_min": -98.7325668334961,
      "activations/layer4_attention_weight_max": 121.85247039794922,
      "activations/layer4_attention_weight_min": -120.55809783935547,
      "activations/layer5_attention_weight_max": 55.97791290283203,
      "activations/layer5_attention_weight_min": -62.1353874206543,
      "activations/layer6_attention_weight_max": 50.263328552246094,
      "activations/layer6_attention_weight_min": -51.503700256347656,
      "activations/layer7_attention_weight_max": 91.59829711914062,
      "activations/layer7_attention_weight_min": -90.22864532470703,
      "activations/layer8_attention_weight_max": 43.28567886352539,
      "activations/layer8_attention_weight_min": -44.105018615722656,
      "activations/layer9_attention_weight_max": 37.693565368652344,
      "activations/layer9_attention_weight_min": -36.83451843261719,
      "epoch": 8.9,
      "learning_rate": 9.353598484848485e-05,
      "loss": 2.8021,
      "step": 153200
    },
    {
      "activations/layer0_attention_weight_max": 14.46611499786377,
      "activations/layer0_attention_weight_min": -12.152161598205566,
      "activations/layer10_attention_weight_max": 33.46479034423828,
      "activations/layer10_attention_weight_min": -34.42954635620117,
      "activations/layer11_attention_weight_max": 34.82012176513672,
      "activations/layer11_attention_weight_min": -30.82961082458496,
      "activations/layer12_attention_weight_max": 33.42314910888672,
      "activations/layer12_attention_weight_min": -25.22738265991211,
      "activations/layer13_attention_weight_max": 45.047183990478516,
      "activations/layer13_attention_weight_min": -34.336097717285156,
      "activations/layer14_attention_weight_max": 51.002445220947266,
      "activations/layer14_attention_weight_min": -40.12916946411133,
      "activations/layer15_attention_weight_max": 38.789791107177734,
      "activations/layer15_attention_weight_min": -31.378034591674805,
      "activations/layer16_attention_weight_max": 34.73405075073242,
      "activations/layer16_attention_weight_min": -27.64452362060547,
      "activations/layer17_attention_weight_max": 38.43738555908203,
      "activations/layer17_attention_weight_min": -25.39025115966797,
      "activations/layer18_attention_weight_max": 36.28814697265625,
      "activations/layer18_attention_weight_min": -22.337783813476562,
      "activations/layer19_attention_weight_max": 37.615089416503906,
      "activations/layer19_attention_weight_min": -29.381168365478516,
      "activations/layer1_attention_weight_max": 16.364042282104492,
      "activations/layer1_attention_weight_min": -15.063882827758789,
      "activations/layer20_attention_weight_max": 34.45072937011719,
      "activations/layer20_attention_weight_min": -25.84218406677246,
      "activations/layer21_attention_weight_max": 34.86198425292969,
      "activations/layer21_attention_weight_min": -24.15264129638672,
      "activations/layer22_attention_weight_max": 56.260658264160156,
      "activations/layer22_attention_weight_min": -29.79217529296875,
      "activations/layer23_attention_weight_max": 43.76035690307617,
      "activations/layer23_attention_weight_min": -23.894512176513672,
      "activations/layer2_attention_weight_max": 31.377416610717773,
      "activations/layer2_attention_weight_min": -31.339834213256836,
      "activations/layer3_attention_weight_max": 91.74896240234375,
      "activations/layer3_attention_weight_min": -93.64730072021484,
      "activations/layer4_attention_weight_max": 115.20404052734375,
      "activations/layer4_attention_weight_min": -112.89247131347656,
      "activations/layer5_attention_weight_max": 54.23014450073242,
      "activations/layer5_attention_weight_min": -62.02724838256836,
      "activations/layer6_attention_weight_max": 46.00538635253906,
      "activations/layer6_attention_weight_min": -50.92341232299805,
      "activations/layer7_attention_weight_max": 86.01409912109375,
      "activations/layer7_attention_weight_min": -89.555908203125,
      "activations/layer8_attention_weight_max": 40.779693603515625,
      "activations/layer8_attention_weight_min": -41.94596481323242,
      "activations/layer9_attention_weight_max": 35.55513381958008,
      "activations/layer9_attention_weight_min": -36.399017333984375,
      "epoch": 8.9,
      "learning_rate": 9.351704545454545e-05,
      "loss": 2.8008,
      "step": 153250
    },
    {
      "activations/layer0_attention_weight_max": 15.295434951782227,
      "activations/layer0_attention_weight_min": -12.04125690460205,
      "activations/layer10_attention_weight_max": 31.40606689453125,
      "activations/layer10_attention_weight_min": -31.933935165405273,
      "activations/layer11_attention_weight_max": 30.137603759765625,
      "activations/layer11_attention_weight_min": -29.00701141357422,
      "activations/layer12_attention_weight_max": 21.811120986938477,
      "activations/layer12_attention_weight_min": -23.818326950073242,
      "activations/layer13_attention_weight_max": 35.19451904296875,
      "activations/layer13_attention_weight_min": -31.95488929748535,
      "activations/layer14_attention_weight_max": 44.01321029663086,
      "activations/layer14_attention_weight_min": -38.88755798339844,
      "activations/layer15_attention_weight_max": 32.65378189086914,
      "activations/layer15_attention_weight_min": -30.702098846435547,
      "activations/layer16_attention_weight_max": 30.43699836730957,
      "activations/layer16_attention_weight_min": -27.689125061035156,
      "activations/layer17_attention_weight_max": 29.858083724975586,
      "activations/layer17_attention_weight_min": -25.396169662475586,
      "activations/layer18_attention_weight_max": 28.008060455322266,
      "activations/layer18_attention_weight_min": -21.36437225341797,
      "activations/layer19_attention_weight_max": 34.8830680847168,
      "activations/layer19_attention_weight_min": -31.910505294799805,
      "activations/layer1_attention_weight_max": 17.765409469604492,
      "activations/layer1_attention_weight_min": -15.837657928466797,
      "activations/layer20_attention_weight_max": 27.704017639160156,
      "activations/layer20_attention_weight_min": -24.762657165527344,
      "activations/layer21_attention_weight_max": 28.682476043701172,
      "activations/layer21_attention_weight_min": -24.349834442138672,
      "activations/layer22_attention_weight_max": 42.62421798706055,
      "activations/layer22_attention_weight_min": -29.769466400146484,
      "activations/layer23_attention_weight_max": 33.187191009521484,
      "activations/layer23_attention_weight_min": -24.07073974609375,
      "activations/layer2_attention_weight_max": 32.107086181640625,
      "activations/layer2_attention_weight_min": -31.84034538269043,
      "activations/layer3_attention_weight_max": 93.57201385498047,
      "activations/layer3_attention_weight_min": -94.04153442382812,
      "activations/layer4_attention_weight_max": 114.13541412353516,
      "activations/layer4_attention_weight_min": -112.98469543457031,
      "activations/layer5_attention_weight_max": 52.2071647644043,
      "activations/layer5_attention_weight_min": -59.806739807128906,
      "activations/layer6_attention_weight_max": 44.9587516784668,
      "activations/layer6_attention_weight_min": -51.11140441894531,
      "activations/layer7_attention_weight_max": 83.36199951171875,
      "activations/layer7_attention_weight_min": -84.2198257446289,
      "activations/layer8_attention_weight_max": 38.070594787597656,
      "activations/layer8_attention_weight_min": -39.56508255004883,
      "activations/layer9_attention_weight_max": 35.23249435424805,
      "activations/layer9_attention_weight_min": -32.5279655456543,
      "epoch": 8.91,
      "learning_rate": 9.349810606060605e-05,
      "loss": 2.8271,
      "step": 153300
    },
    {
      "activations/layer0_attention_weight_max": 15.722027778625488,
      "activations/layer0_attention_weight_min": -12.362299919128418,
      "activations/layer10_attention_weight_max": 30.94389533996582,
      "activations/layer10_attention_weight_min": -31.95123863220215,
      "activations/layer11_attention_weight_max": 29.843175888061523,
      "activations/layer11_attention_weight_min": -29.136642456054688,
      "activations/layer12_attention_weight_max": 19.777896881103516,
      "activations/layer12_attention_weight_min": -24.959571838378906,
      "activations/layer13_attention_weight_max": 33.43282699584961,
      "activations/layer13_attention_weight_min": -35.266136169433594,
      "activations/layer14_attention_weight_max": 41.7734489440918,
      "activations/layer14_attention_weight_min": -38.54355239868164,
      "activations/layer15_attention_weight_max": 33.1334114074707,
      "activations/layer15_attention_weight_min": -29.896949768066406,
      "activations/layer16_attention_weight_max": 28.914764404296875,
      "activations/layer16_attention_weight_min": -27.59726905822754,
      "activations/layer17_attention_weight_max": 34.049415588378906,
      "activations/layer17_attention_weight_min": -25.875017166137695,
      "activations/layer18_attention_weight_max": 28.990291595458984,
      "activations/layer18_attention_weight_min": -21.553789138793945,
      "activations/layer19_attention_weight_max": 36.43447494506836,
      "activations/layer19_attention_weight_min": -30.88217544555664,
      "activations/layer1_attention_weight_max": 15.88231372833252,
      "activations/layer1_attention_weight_min": -14.214295387268066,
      "activations/layer20_attention_weight_max": 30.133981704711914,
      "activations/layer20_attention_weight_min": -26.15700912475586,
      "activations/layer21_attention_weight_max": 29.581987380981445,
      "activations/layer21_attention_weight_min": -24.486600875854492,
      "activations/layer22_attention_weight_max": 42.324546813964844,
      "activations/layer22_attention_weight_min": -30.790342330932617,
      "activations/layer23_attention_weight_max": 31.97789764404297,
      "activations/layer23_attention_weight_min": -24.164939880371094,
      "activations/layer2_attention_weight_max": 34.3145866394043,
      "activations/layer2_attention_weight_min": -31.227359771728516,
      "activations/layer3_attention_weight_max": 96.96115112304688,
      "activations/layer3_attention_weight_min": -96.25946807861328,
      "activations/layer4_attention_weight_max": 113.40743255615234,
      "activations/layer4_attention_weight_min": -114.8360824584961,
      "activations/layer5_attention_weight_max": 53.464027404785156,
      "activations/layer5_attention_weight_min": -61.24180221557617,
      "activations/layer6_attention_weight_max": 46.41151809692383,
      "activations/layer6_attention_weight_min": -49.568565368652344,
      "activations/layer7_attention_weight_max": 88.46526336669922,
      "activations/layer7_attention_weight_min": -85.94525909423828,
      "activations/layer8_attention_weight_max": 38.544288635253906,
      "activations/layer8_attention_weight_min": -39.47417068481445,
      "activations/layer9_attention_weight_max": 33.42818832397461,
      "activations/layer9_attention_weight_min": -31.83247947692871,
      "epoch": 8.91,
      "learning_rate": 9.347916666666666e-05,
      "loss": 2.7945,
      "step": 153350
    },
    {
      "activations/layer0_attention_weight_max": 14.562751770019531,
      "activations/layer0_attention_weight_min": -12.179250717163086,
      "activations/layer10_attention_weight_max": 33.22549819946289,
      "activations/layer10_attention_weight_min": -30.868200302124023,
      "activations/layer11_attention_weight_max": 31.359615325927734,
      "activations/layer11_attention_weight_min": -30.121219635009766,
      "activations/layer12_attention_weight_max": 26.621456146240234,
      "activations/layer12_attention_weight_min": -25.84038543701172,
      "activations/layer13_attention_weight_max": 42.3187255859375,
      "activations/layer13_attention_weight_min": -33.929569244384766,
      "activations/layer14_attention_weight_max": 45.146278381347656,
      "activations/layer14_attention_weight_min": -36.72658920288086,
      "activations/layer15_attention_weight_max": 44.755592346191406,
      "activations/layer15_attention_weight_min": -30.032846450805664,
      "activations/layer16_attention_weight_max": 29.68537139892578,
      "activations/layer16_attention_weight_min": -27.657514572143555,
      "activations/layer17_attention_weight_max": 30.792280197143555,
      "activations/layer17_attention_weight_min": -25.4205379486084,
      "activations/layer18_attention_weight_max": 30.523216247558594,
      "activations/layer18_attention_weight_min": -22.692718505859375,
      "activations/layer19_attention_weight_max": 34.29293441772461,
      "activations/layer19_attention_weight_min": -30.775104522705078,
      "activations/layer1_attention_weight_max": 16.86928939819336,
      "activations/layer1_attention_weight_min": -15.490839958190918,
      "activations/layer20_attention_weight_max": 27.296592712402344,
      "activations/layer20_attention_weight_min": -24.447649002075195,
      "activations/layer21_attention_weight_max": 26.696483612060547,
      "activations/layer21_attention_weight_min": -23.711196899414062,
      "activations/layer22_attention_weight_max": 45.4913330078125,
      "activations/layer22_attention_weight_min": -30.26606559753418,
      "activations/layer23_attention_weight_max": 31.18732261657715,
      "activations/layer23_attention_weight_min": -25.33320426940918,
      "activations/layer2_attention_weight_max": 31.74152374267578,
      "activations/layer2_attention_weight_min": -32.89524841308594,
      "activations/layer3_attention_weight_max": 96.60888671875,
      "activations/layer3_attention_weight_min": -98.80355834960938,
      "activations/layer4_attention_weight_max": 116.22020721435547,
      "activations/layer4_attention_weight_min": -116.85884857177734,
      "activations/layer5_attention_weight_max": 53.69143295288086,
      "activations/layer5_attention_weight_min": -59.35103988647461,
      "activations/layer6_attention_weight_max": 46.49800109863281,
      "activations/layer6_attention_weight_min": -48.23090744018555,
      "activations/layer7_attention_weight_max": 86.06501770019531,
      "activations/layer7_attention_weight_min": -86.54180908203125,
      "activations/layer8_attention_weight_max": 39.6172981262207,
      "activations/layer8_attention_weight_min": -40.67182540893555,
      "activations/layer9_attention_weight_max": 33.43547058105469,
      "activations/layer9_attention_weight_min": -32.06977844238281,
      "epoch": 8.91,
      "learning_rate": 9.346022727272727e-05,
      "loss": 2.7995,
      "step": 153400
    },
    {
      "activations/layer0_attention_weight_max": 14.48486614227295,
      "activations/layer0_attention_weight_min": -12.45962142944336,
      "activations/layer10_attention_weight_max": 31.543163299560547,
      "activations/layer10_attention_weight_min": -32.35659408569336,
      "activations/layer11_attention_weight_max": 33.22882843017578,
      "activations/layer11_attention_weight_min": -31.975528717041016,
      "activations/layer12_attention_weight_max": 24.961647033691406,
      "activations/layer12_attention_weight_min": -26.60861587524414,
      "activations/layer13_attention_weight_max": 42.42328643798828,
      "activations/layer13_attention_weight_min": -33.904685974121094,
      "activations/layer14_attention_weight_max": 58.61262893676758,
      "activations/layer14_attention_weight_min": -41.23391342163086,
      "activations/layer15_attention_weight_max": 38.371826171875,
      "activations/layer15_attention_weight_min": -33.68806838989258,
      "activations/layer16_attention_weight_max": 35.71482849121094,
      "activations/layer16_attention_weight_min": -27.713085174560547,
      "activations/layer17_attention_weight_max": 37.888301849365234,
      "activations/layer17_attention_weight_min": -24.76186180114746,
      "activations/layer18_attention_weight_max": 32.4123420715332,
      "activations/layer18_attention_weight_min": -21.88865852355957,
      "activations/layer19_attention_weight_max": 37.60263442993164,
      "activations/layer19_attention_weight_min": -29.21782875061035,
      "activations/layer1_attention_weight_max": 16.448089599609375,
      "activations/layer1_attention_weight_min": -14.514187812805176,
      "activations/layer20_attention_weight_max": 32.68942642211914,
      "activations/layer20_attention_weight_min": -24.45815086364746,
      "activations/layer21_attention_weight_max": 35.08013916015625,
      "activations/layer21_attention_weight_min": -22.939653396606445,
      "activations/layer22_attention_weight_max": 52.253211975097656,
      "activations/layer22_attention_weight_min": -28.502025604248047,
      "activations/layer23_attention_weight_max": 38.55296325683594,
      "activations/layer23_attention_weight_min": -24.43740463256836,
      "activations/layer2_attention_weight_max": 30.997343063354492,
      "activations/layer2_attention_weight_min": -32.323394775390625,
      "activations/layer3_attention_weight_max": 94.5721435546875,
      "activations/layer3_attention_weight_min": -100.3023452758789,
      "activations/layer4_attention_weight_max": 115.96785736083984,
      "activations/layer4_attention_weight_min": -118.87932586669922,
      "activations/layer5_attention_weight_max": 55.14468002319336,
      "activations/layer5_attention_weight_min": -64.04905700683594,
      "activations/layer6_attention_weight_max": 47.66529846191406,
      "activations/layer6_attention_weight_min": -52.61964797973633,
      "activations/layer7_attention_weight_max": 87.95628356933594,
      "activations/layer7_attention_weight_min": -89.74415588378906,
      "activations/layer8_attention_weight_max": 40.01728439331055,
      "activations/layer8_attention_weight_min": -40.0120964050293,
      "activations/layer9_attention_weight_max": 32.83454132080078,
      "activations/layer9_attention_weight_min": -34.33430480957031,
      "epoch": 8.92,
      "learning_rate": 9.344128787878787e-05,
      "loss": 2.7985,
      "step": 153450
    },
    {
      "activations/layer0_attention_weight_max": 16.54390525817871,
      "activations/layer0_attention_weight_min": -12.223456382751465,
      "activations/layer10_attention_weight_max": 33.407691955566406,
      "activations/layer10_attention_weight_min": -31.855113983154297,
      "activations/layer11_attention_weight_max": 35.22903060913086,
      "activations/layer11_attention_weight_min": -31.697433471679688,
      "activations/layer12_attention_weight_max": 24.430631637573242,
      "activations/layer12_attention_weight_min": -25.606307983398438,
      "activations/layer13_attention_weight_max": 39.09489059448242,
      "activations/layer13_attention_weight_min": -34.377410888671875,
      "activations/layer14_attention_weight_max": 48.036739349365234,
      "activations/layer14_attention_weight_min": -42.32674789428711,
      "activations/layer15_attention_weight_max": 38.7241096496582,
      "activations/layer15_attention_weight_min": -30.741600036621094,
      "activations/layer16_attention_weight_max": 30.2666015625,
      "activations/layer16_attention_weight_min": -27.50104522705078,
      "activations/layer17_attention_weight_max": 32.23833465576172,
      "activations/layer17_attention_weight_min": -24.85247039794922,
      "activations/layer18_attention_weight_max": 31.257417678833008,
      "activations/layer18_attention_weight_min": -21.99457359313965,
      "activations/layer19_attention_weight_max": 34.53669357299805,
      "activations/layer19_attention_weight_min": -29.979196548461914,
      "activations/layer1_attention_weight_max": 16.957706451416016,
      "activations/layer1_attention_weight_min": -15.566702842712402,
      "activations/layer20_attention_weight_max": 32.927310943603516,
      "activations/layer20_attention_weight_min": -22.755081176757812,
      "activations/layer21_attention_weight_max": 29.873123168945312,
      "activations/layer21_attention_weight_min": -22.64459228515625,
      "activations/layer22_attention_weight_max": 51.13499069213867,
      "activations/layer22_attention_weight_min": -31.207393646240234,
      "activations/layer23_attention_weight_max": 37.97480392456055,
      "activations/layer23_attention_weight_min": -23.982376098632812,
      "activations/layer2_attention_weight_max": 31.264507293701172,
      "activations/layer2_attention_weight_min": -31.277111053466797,
      "activations/layer3_attention_weight_max": 94.01815795898438,
      "activations/layer3_attention_weight_min": -92.78104400634766,
      "activations/layer4_attention_weight_max": 118.6058120727539,
      "activations/layer4_attention_weight_min": -117.27313232421875,
      "activations/layer5_attention_weight_max": 55.97242736816406,
      "activations/layer5_attention_weight_min": -60.68379211425781,
      "activations/layer6_attention_weight_max": 47.211124420166016,
      "activations/layer6_attention_weight_min": -50.58167266845703,
      "activations/layer7_attention_weight_max": 91.86605072021484,
      "activations/layer7_attention_weight_min": -87.47935485839844,
      "activations/layer8_attention_weight_max": 42.103824615478516,
      "activations/layer8_attention_weight_min": -40.81507110595703,
      "activations/layer9_attention_weight_max": 37.99918746948242,
      "activations/layer9_attention_weight_min": -33.353641510009766,
      "epoch": 8.92,
      "learning_rate": 9.342234848484847e-05,
      "loss": 2.7974,
      "step": 153500
    },
    {
      "activations/layer0_attention_weight_max": 15.168911933898926,
      "activations/layer0_attention_weight_min": -11.879264831542969,
      "activations/layer10_attention_weight_max": 37.29749298095703,
      "activations/layer10_attention_weight_min": -36.91269302368164,
      "activations/layer11_attention_weight_max": 36.450382232666016,
      "activations/layer11_attention_weight_min": -34.572017669677734,
      "activations/layer12_attention_weight_max": 25.385395050048828,
      "activations/layer12_attention_weight_min": -24.93157386779785,
      "activations/layer13_attention_weight_max": 43.049686431884766,
      "activations/layer13_attention_weight_min": -35.4510498046875,
      "activations/layer14_attention_weight_max": 51.02649688720703,
      "activations/layer14_attention_weight_min": -39.781803131103516,
      "activations/layer15_attention_weight_max": 40.39213562011719,
      "activations/layer15_attention_weight_min": -32.23702621459961,
      "activations/layer16_attention_weight_max": 34.79265594482422,
      "activations/layer16_attention_weight_min": -26.66551971435547,
      "activations/layer17_attention_weight_max": 39.118797302246094,
      "activations/layer17_attention_weight_min": -25.85879135131836,
      "activations/layer18_attention_weight_max": 35.2170295715332,
      "activations/layer18_attention_weight_min": -20.850929260253906,
      "activations/layer19_attention_weight_max": 37.377288818359375,
      "activations/layer19_attention_weight_min": -31.334020614624023,
      "activations/layer1_attention_weight_max": 15.94393539428711,
      "activations/layer1_attention_weight_min": -13.606942176818848,
      "activations/layer20_attention_weight_max": 34.48480987548828,
      "activations/layer20_attention_weight_min": -23.04532814025879,
      "activations/layer21_attention_weight_max": 31.3505916595459,
      "activations/layer21_attention_weight_min": -23.1829776763916,
      "activations/layer22_attention_weight_max": 48.8299674987793,
      "activations/layer22_attention_weight_min": -28.917945861816406,
      "activations/layer23_attention_weight_max": 39.560821533203125,
      "activations/layer23_attention_weight_min": -24.118667602539062,
      "activations/layer2_attention_weight_max": 31.889060974121094,
      "activations/layer2_attention_weight_min": -31.90167236328125,
      "activations/layer3_attention_weight_max": 97.54386901855469,
      "activations/layer3_attention_weight_min": -97.44220733642578,
      "activations/layer4_attention_weight_max": 120.36897277832031,
      "activations/layer4_attention_weight_min": -119.57781219482422,
      "activations/layer5_attention_weight_max": 57.240386962890625,
      "activations/layer5_attention_weight_min": -62.90083312988281,
      "activations/layer6_attention_weight_max": 49.85030746459961,
      "activations/layer6_attention_weight_min": -51.02014923095703,
      "activations/layer7_attention_weight_max": 104.11183166503906,
      "activations/layer7_attention_weight_min": -91.2177505493164,
      "activations/layer8_attention_weight_max": 48.298851013183594,
      "activations/layer8_attention_weight_min": -47.444095611572266,
      "activations/layer9_attention_weight_max": 40.58974838256836,
      "activations/layer9_attention_weight_min": -38.056007385253906,
      "epoch": 8.92,
      "learning_rate": 9.340340909090908e-05,
      "loss": 2.8016,
      "step": 153550
    },
    {
      "activations/layer0_attention_weight_max": 14.48582649230957,
      "activations/layer0_attention_weight_min": -11.677295684814453,
      "activations/layer10_attention_weight_max": 34.6131591796875,
      "activations/layer10_attention_weight_min": -34.01334762573242,
      "activations/layer11_attention_weight_max": 34.41722106933594,
      "activations/layer11_attention_weight_min": -32.555389404296875,
      "activations/layer12_attention_weight_max": 25.578842163085938,
      "activations/layer12_attention_weight_min": -24.55705451965332,
      "activations/layer13_attention_weight_max": 42.87495422363281,
      "activations/layer13_attention_weight_min": -35.03818893432617,
      "activations/layer14_attention_weight_max": 49.886898040771484,
      "activations/layer14_attention_weight_min": -40.90995407104492,
      "activations/layer15_attention_weight_max": 37.59309768676758,
      "activations/layer15_attention_weight_min": -33.42247009277344,
      "activations/layer16_attention_weight_max": 33.273643493652344,
      "activations/layer16_attention_weight_min": -28.343984603881836,
      "activations/layer17_attention_weight_max": 33.581153869628906,
      "activations/layer17_attention_weight_min": -25.29155158996582,
      "activations/layer18_attention_weight_max": 33.437034606933594,
      "activations/layer18_attention_weight_min": -21.953609466552734,
      "activations/layer19_attention_weight_max": 37.412742614746094,
      "activations/layer19_attention_weight_min": -31.483121871948242,
      "activations/layer1_attention_weight_max": 16.189130783081055,
      "activations/layer1_attention_weight_min": -15.47700309753418,
      "activations/layer20_attention_weight_max": 32.19157409667969,
      "activations/layer20_attention_weight_min": -24.41018295288086,
      "activations/layer21_attention_weight_max": 31.66191291809082,
      "activations/layer21_attention_weight_min": -23.530681610107422,
      "activations/layer22_attention_weight_max": 48.83109664916992,
      "activations/layer22_attention_weight_min": -31.580509185791016,
      "activations/layer23_attention_weight_max": 35.709815979003906,
      "activations/layer23_attention_weight_min": -23.602169036865234,
      "activations/layer2_attention_weight_max": 32.776405334472656,
      "activations/layer2_attention_weight_min": -31.203954696655273,
      "activations/layer3_attention_weight_max": 95.69786071777344,
      "activations/layer3_attention_weight_min": -95.32122039794922,
      "activations/layer4_attention_weight_max": 121.1847152709961,
      "activations/layer4_attention_weight_min": -115.50569915771484,
      "activations/layer5_attention_weight_max": 54.940338134765625,
      "activations/layer5_attention_weight_min": -60.693695068359375,
      "activations/layer6_attention_weight_max": 47.36248016357422,
      "activations/layer6_attention_weight_min": -51.95920181274414,
      "activations/layer7_attention_weight_max": 95.03860473632812,
      "activations/layer7_attention_weight_min": -95.09651947021484,
      "activations/layer8_attention_weight_max": 42.53145980834961,
      "activations/layer8_attention_weight_min": -42.85725021362305,
      "activations/layer9_attention_weight_max": 38.41910171508789,
      "activations/layer9_attention_weight_min": -34.366004943847656,
      "epoch": 8.92,
      "learning_rate": 9.33844696969697e-05,
      "loss": 2.8012,
      "step": 153600
    },
    {
      "activations/layer0_attention_weight_max": 15.534199714660645,
      "activations/layer0_attention_weight_min": -12.625932693481445,
      "activations/layer10_attention_weight_max": 34.35896682739258,
      "activations/layer10_attention_weight_min": -33.18095397949219,
      "activations/layer11_attention_weight_max": 33.05982208251953,
      "activations/layer11_attention_weight_min": -31.651973724365234,
      "activations/layer12_attention_weight_max": 23.182315826416016,
      "activations/layer12_attention_weight_min": -28.534791946411133,
      "activations/layer13_attention_weight_max": 39.810638427734375,
      "activations/layer13_attention_weight_min": -33.71282196044922,
      "activations/layer14_attention_weight_max": 46.156654357910156,
      "activations/layer14_attention_weight_min": -37.582672119140625,
      "activations/layer15_attention_weight_max": 34.62013244628906,
      "activations/layer15_attention_weight_min": -29.941688537597656,
      "activations/layer16_attention_weight_max": 30.910030364990234,
      "activations/layer16_attention_weight_min": -28.07474708557129,
      "activations/layer17_attention_weight_max": 33.6322135925293,
      "activations/layer17_attention_weight_min": -24.91815948486328,
      "activations/layer18_attention_weight_max": 32.08621597290039,
      "activations/layer18_attention_weight_min": -23.013898849487305,
      "activations/layer19_attention_weight_max": 35.723880767822266,
      "activations/layer19_attention_weight_min": -29.902780532836914,
      "activations/layer1_attention_weight_max": 16.50216293334961,
      "activations/layer1_attention_weight_min": -14.333056449890137,
      "activations/layer20_attention_weight_max": 29.280576705932617,
      "activations/layer20_attention_weight_min": -23.367876052856445,
      "activations/layer21_attention_weight_max": 30.942045211791992,
      "activations/layer21_attention_weight_min": -23.056499481201172,
      "activations/layer22_attention_weight_max": 44.34885025024414,
      "activations/layer22_attention_weight_min": -27.501693725585938,
      "activations/layer23_attention_weight_max": 35.405845642089844,
      "activations/layer23_attention_weight_min": -24.49396514892578,
      "activations/layer2_attention_weight_max": 33.96841049194336,
      "activations/layer2_attention_weight_min": -32.20629119873047,
      "activations/layer3_attention_weight_max": 96.47834777832031,
      "activations/layer3_attention_weight_min": -98.31747436523438,
      "activations/layer4_attention_weight_max": 117.59193420410156,
      "activations/layer4_attention_weight_min": -117.72637176513672,
      "activations/layer5_attention_weight_max": 56.44265365600586,
      "activations/layer5_attention_weight_min": -60.66548156738281,
      "activations/layer6_attention_weight_max": 50.674739837646484,
      "activations/layer6_attention_weight_min": -53.00489044189453,
      "activations/layer7_attention_weight_max": 96.1763687133789,
      "activations/layer7_attention_weight_min": -92.5635986328125,
      "activations/layer8_attention_weight_max": 44.0928840637207,
      "activations/layer8_attention_weight_min": -42.019168853759766,
      "activations/layer9_attention_weight_max": 35.505226135253906,
      "activations/layer9_attention_weight_min": -33.51913833618164,
      "epoch": 8.93,
      "learning_rate": 9.336553030303029e-05,
      "loss": 2.8038,
      "step": 153650
    },
    {
      "activations/layer0_attention_weight_max": 14.471648216247559,
      "activations/layer0_attention_weight_min": -12.522151947021484,
      "activations/layer10_attention_weight_max": 31.70927619934082,
      "activations/layer10_attention_weight_min": -32.5715217590332,
      "activations/layer11_attention_weight_max": 32.218475341796875,
      "activations/layer11_attention_weight_min": -30.340057373046875,
      "activations/layer12_attention_weight_max": 23.67963218688965,
      "activations/layer12_attention_weight_min": -24.542741775512695,
      "activations/layer13_attention_weight_max": 38.20794677734375,
      "activations/layer13_attention_weight_min": -33.65374755859375,
      "activations/layer14_attention_weight_max": 45.69198989868164,
      "activations/layer14_attention_weight_min": -38.30997848510742,
      "activations/layer15_attention_weight_max": 36.4860954284668,
      "activations/layer15_attention_weight_min": -32.36921310424805,
      "activations/layer16_attention_weight_max": 31.543594360351562,
      "activations/layer16_attention_weight_min": -27.247148513793945,
      "activations/layer17_attention_weight_max": 33.27880859375,
      "activations/layer17_attention_weight_min": -25.290205001831055,
      "activations/layer18_attention_weight_max": 30.757591247558594,
      "activations/layer18_attention_weight_min": -21.973438262939453,
      "activations/layer19_attention_weight_max": 35.479915618896484,
      "activations/layer19_attention_weight_min": -29.939472198486328,
      "activations/layer1_attention_weight_max": 16.41173553466797,
      "activations/layer1_attention_weight_min": -15.071235656738281,
      "activations/layer20_attention_weight_max": 30.818897247314453,
      "activations/layer20_attention_weight_min": -26.07183265686035,
      "activations/layer21_attention_weight_max": 28.457096099853516,
      "activations/layer21_attention_weight_min": -23.781461715698242,
      "activations/layer22_attention_weight_max": 47.4578971862793,
      "activations/layer22_attention_weight_min": -29.951263427734375,
      "activations/layer23_attention_weight_max": 34.33278274536133,
      "activations/layer23_attention_weight_min": -23.89703941345215,
      "activations/layer2_attention_weight_max": 32.85952377319336,
      "activations/layer2_attention_weight_min": -34.286598205566406,
      "activations/layer3_attention_weight_max": 95.82315063476562,
      "activations/layer3_attention_weight_min": -101.6423110961914,
      "activations/layer4_attention_weight_max": 115.2378921508789,
      "activations/layer4_attention_weight_min": -117.76344299316406,
      "activations/layer5_attention_weight_max": 54.3534049987793,
      "activations/layer5_attention_weight_min": -65.40071105957031,
      "activations/layer6_attention_weight_max": 47.51057815551758,
      "activations/layer6_attention_weight_min": -52.38726043701172,
      "activations/layer7_attention_weight_max": 90.14519500732422,
      "activations/layer7_attention_weight_min": -86.1886978149414,
      "activations/layer8_attention_weight_max": 39.49745178222656,
      "activations/layer8_attention_weight_min": -40.40422821044922,
      "activations/layer9_attention_weight_max": 34.69794845581055,
      "activations/layer9_attention_weight_min": -35.4304313659668,
      "epoch": 8.93,
      "learning_rate": 9.33465909090909e-05,
      "loss": 2.8083,
      "step": 153700
    },
    {
      "activations/layer0_attention_weight_max": 15.634969711303711,
      "activations/layer0_attention_weight_min": -13.088674545288086,
      "activations/layer10_attention_weight_max": 31.554210662841797,
      "activations/layer10_attention_weight_min": -34.738922119140625,
      "activations/layer11_attention_weight_max": 31.71645736694336,
      "activations/layer11_attention_weight_min": -32.16596603393555,
      "activations/layer12_attention_weight_max": 22.70856285095215,
      "activations/layer12_attention_weight_min": -26.066591262817383,
      "activations/layer13_attention_weight_max": 34.83554458618164,
      "activations/layer13_attention_weight_min": -34.640106201171875,
      "activations/layer14_attention_weight_max": 47.230438232421875,
      "activations/layer14_attention_weight_min": -38.25328063964844,
      "activations/layer15_attention_weight_max": 35.0828857421875,
      "activations/layer15_attention_weight_min": -30.759130477905273,
      "activations/layer16_attention_weight_max": 31.978160858154297,
      "activations/layer16_attention_weight_min": -28.677061080932617,
      "activations/layer17_attention_weight_max": 33.711151123046875,
      "activations/layer17_attention_weight_min": -27.04691505432129,
      "activations/layer18_attention_weight_max": 33.300350189208984,
      "activations/layer18_attention_weight_min": -22.53143310546875,
      "activations/layer19_attention_weight_max": 34.369972229003906,
      "activations/layer19_attention_weight_min": -32.61661911010742,
      "activations/layer1_attention_weight_max": 16.731550216674805,
      "activations/layer1_attention_weight_min": -14.297181129455566,
      "activations/layer20_attention_weight_max": 31.757850646972656,
      "activations/layer20_attention_weight_min": -27.523374557495117,
      "activations/layer21_attention_weight_max": 29.66117286682129,
      "activations/layer21_attention_weight_min": -26.214834213256836,
      "activations/layer22_attention_weight_max": 46.666351318359375,
      "activations/layer22_attention_weight_min": -30.164201736450195,
      "activations/layer23_attention_weight_max": 33.17210388183594,
      "activations/layer23_attention_weight_min": -24.834501266479492,
      "activations/layer2_attention_weight_max": 32.42628479003906,
      "activations/layer2_attention_weight_min": -33.144386291503906,
      "activations/layer3_attention_weight_max": 95.63064575195312,
      "activations/layer3_attention_weight_min": -97.16890716552734,
      "activations/layer4_attention_weight_max": 118.08829498291016,
      "activations/layer4_attention_weight_min": -115.8238525390625,
      "activations/layer5_attention_weight_max": 52.205833435058594,
      "activations/layer5_attention_weight_min": -59.64111328125,
      "activations/layer6_attention_weight_max": 45.93404769897461,
      "activations/layer6_attention_weight_min": -48.40660858154297,
      "activations/layer7_attention_weight_max": 84.67369842529297,
      "activations/layer7_attention_weight_min": -92.18476867675781,
      "activations/layer8_attention_weight_max": 40.858280181884766,
      "activations/layer8_attention_weight_min": -41.060150146484375,
      "activations/layer9_attention_weight_max": 35.108978271484375,
      "activations/layer9_attention_weight_min": -34.40717697143555,
      "epoch": 8.93,
      "learning_rate": 9.332765151515152e-05,
      "loss": 2.7976,
      "step": 153750
    },
    {
      "activations/layer0_attention_weight_max": 16.410242080688477,
      "activations/layer0_attention_weight_min": -10.787217140197754,
      "activations/layer10_attention_weight_max": 35.366661071777344,
      "activations/layer10_attention_weight_min": -34.82850646972656,
      "activations/layer11_attention_weight_max": 33.50716018676758,
      "activations/layer11_attention_weight_min": -30.639867782592773,
      "activations/layer12_attention_weight_max": 24.699981689453125,
      "activations/layer12_attention_weight_min": -26.286806106567383,
      "activations/layer13_attention_weight_max": 45.92587661743164,
      "activations/layer13_attention_weight_min": -34.47181701660156,
      "activations/layer14_attention_weight_max": 57.88974380493164,
      "activations/layer14_attention_weight_min": -44.09672164916992,
      "activations/layer15_attention_weight_max": 40.436275482177734,
      "activations/layer15_attention_weight_min": -31.90964126586914,
      "activations/layer16_attention_weight_max": 34.301170349121094,
      "activations/layer16_attention_weight_min": -27.349124908447266,
      "activations/layer17_attention_weight_max": 35.87424087524414,
      "activations/layer17_attention_weight_min": -25.361665725708008,
      "activations/layer18_attention_weight_max": 35.062950134277344,
      "activations/layer18_attention_weight_min": -22.180517196655273,
      "activations/layer19_attention_weight_max": 40.867942810058594,
      "activations/layer19_attention_weight_min": -30.986677169799805,
      "activations/layer1_attention_weight_max": 16.284608840942383,
      "activations/layer1_attention_weight_min": -15.433658599853516,
      "activations/layer20_attention_weight_max": 34.617557525634766,
      "activations/layer20_attention_weight_min": -24.275339126586914,
      "activations/layer21_attention_weight_max": 33.0401611328125,
      "activations/layer21_attention_weight_min": -21.753942489624023,
      "activations/layer22_attention_weight_max": 56.785369873046875,
      "activations/layer22_attention_weight_min": -30.208148956298828,
      "activations/layer23_attention_weight_max": 44.51540756225586,
      "activations/layer23_attention_weight_min": -22.689802169799805,
      "activations/layer2_attention_weight_max": 33.90473937988281,
      "activations/layer2_attention_weight_min": -33.07245635986328,
      "activations/layer3_attention_weight_max": 99.65215301513672,
      "activations/layer3_attention_weight_min": -98.7400131225586,
      "activations/layer4_attention_weight_max": 121.0329360961914,
      "activations/layer4_attention_weight_min": -119.26861572265625,
      "activations/layer5_attention_weight_max": 58.917938232421875,
      "activations/layer5_attention_weight_min": -62.47314453125,
      "activations/layer6_attention_weight_max": 51.4047737121582,
      "activations/layer6_attention_weight_min": -51.57395935058594,
      "activations/layer7_attention_weight_max": 92.29076385498047,
      "activations/layer7_attention_weight_min": -92.33710479736328,
      "activations/layer8_attention_weight_max": 42.66438674926758,
      "activations/layer8_attention_weight_min": -41.125244140625,
      "activations/layer9_attention_weight_max": 35.932918548583984,
      "activations/layer9_attention_weight_min": -35.3834228515625,
      "epoch": 8.94,
      "learning_rate": 9.33087121212121e-05,
      "loss": 2.7895,
      "step": 153800
    },
    {
      "activations/layer0_attention_weight_max": 15.253820419311523,
      "activations/layer0_attention_weight_min": -11.710474967956543,
      "activations/layer10_attention_weight_max": 34.2042236328125,
      "activations/layer10_attention_weight_min": -34.229087829589844,
      "activations/layer11_attention_weight_max": 34.79728698730469,
      "activations/layer11_attention_weight_min": -33.31184387207031,
      "activations/layer12_attention_weight_max": 24.12981414794922,
      "activations/layer12_attention_weight_min": -25.191104888916016,
      "activations/layer13_attention_weight_max": 38.97792434692383,
      "activations/layer13_attention_weight_min": -33.49433135986328,
      "activations/layer14_attention_weight_max": 49.904720306396484,
      "activations/layer14_attention_weight_min": -38.494720458984375,
      "activations/layer15_attention_weight_max": 35.03299331665039,
      "activations/layer15_attention_weight_min": -32.15592575073242,
      "activations/layer16_attention_weight_max": 33.92582702636719,
      "activations/layer16_attention_weight_min": -28.70536994934082,
      "activations/layer17_attention_weight_max": 34.791141510009766,
      "activations/layer17_attention_weight_min": -25.144065856933594,
      "activations/layer18_attention_weight_max": 29.298845291137695,
      "activations/layer18_attention_weight_min": -21.616201400756836,
      "activations/layer19_attention_weight_max": 34.54401397705078,
      "activations/layer19_attention_weight_min": -30.427160263061523,
      "activations/layer1_attention_weight_max": 17.711462020874023,
      "activations/layer1_attention_weight_min": -15.345059394836426,
      "activations/layer20_attention_weight_max": 29.219831466674805,
      "activations/layer20_attention_weight_min": -25.72637176513672,
      "activations/layer21_attention_weight_max": 28.18404197692871,
      "activations/layer21_attention_weight_min": -22.998031616210938,
      "activations/layer22_attention_weight_max": 45.400352478027344,
      "activations/layer22_attention_weight_min": -30.126888275146484,
      "activations/layer23_attention_weight_max": 35.95000457763672,
      "activations/layer23_attention_weight_min": -24.475040435791016,
      "activations/layer2_attention_weight_max": 33.944915771484375,
      "activations/layer2_attention_weight_min": -34.50383377075195,
      "activations/layer3_attention_weight_max": 97.49102783203125,
      "activations/layer3_attention_weight_min": -100.98477935791016,
      "activations/layer4_attention_weight_max": 121.49177551269531,
      "activations/layer4_attention_weight_min": -122.8714828491211,
      "activations/layer5_attention_weight_max": 56.297630310058594,
      "activations/layer5_attention_weight_min": -63.2658805847168,
      "activations/layer6_attention_weight_max": 50.76438522338867,
      "activations/layer6_attention_weight_min": -53.263694763183594,
      "activations/layer7_attention_weight_max": 94.90038299560547,
      "activations/layer7_attention_weight_min": -88.36822509765625,
      "activations/layer8_attention_weight_max": 40.450439453125,
      "activations/layer8_attention_weight_min": -41.70295333862305,
      "activations/layer9_attention_weight_max": 34.751991271972656,
      "activations/layer9_attention_weight_min": -36.026058197021484,
      "epoch": 8.94,
      "learning_rate": 9.32901515151515e-05,
      "loss": 2.8076,
      "step": 153850
    },
    {
      "activations/layer0_attention_weight_max": 15.861933708190918,
      "activations/layer0_attention_weight_min": -12.515684127807617,
      "activations/layer10_attention_weight_max": 31.771621704101562,
      "activations/layer10_attention_weight_min": -30.3319149017334,
      "activations/layer11_attention_weight_max": 31.441009521484375,
      "activations/layer11_attention_weight_min": -29.486492156982422,
      "activations/layer12_attention_weight_max": 29.42135238647461,
      "activations/layer12_attention_weight_min": -28.37125015258789,
      "activations/layer13_attention_weight_max": 40.548126220703125,
      "activations/layer13_attention_weight_min": -34.020904541015625,
      "activations/layer14_attention_weight_max": 55.15910339355469,
      "activations/layer14_attention_weight_min": -43.621788024902344,
      "activations/layer15_attention_weight_max": 36.39415740966797,
      "activations/layer15_attention_weight_min": -32.10300827026367,
      "activations/layer16_attention_weight_max": 35.41970443725586,
      "activations/layer16_attention_weight_min": -27.432554244995117,
      "activations/layer17_attention_weight_max": 31.171003341674805,
      "activations/layer17_attention_weight_min": -25.989852905273438,
      "activations/layer18_attention_weight_max": 29.387807846069336,
      "activations/layer18_attention_weight_min": -22.311546325683594,
      "activations/layer19_attention_weight_max": 37.4759407043457,
      "activations/layer19_attention_weight_min": -29.74493980407715,
      "activations/layer1_attention_weight_max": 17.020809173583984,
      "activations/layer1_attention_weight_min": -15.176199913024902,
      "activations/layer20_attention_weight_max": 33.477291107177734,
      "activations/layer20_attention_weight_min": -25.345247268676758,
      "activations/layer21_attention_weight_max": 33.48162078857422,
      "activations/layer21_attention_weight_min": -23.495542526245117,
      "activations/layer22_attention_weight_max": 43.76823043823242,
      "activations/layer22_attention_weight_min": -27.31668472290039,
      "activations/layer23_attention_weight_max": 34.329437255859375,
      "activations/layer23_attention_weight_min": -24.1461181640625,
      "activations/layer2_attention_weight_max": 31.45473861694336,
      "activations/layer2_attention_weight_min": -31.805988311767578,
      "activations/layer3_attention_weight_max": 97.04949951171875,
      "activations/layer3_attention_weight_min": -102.74942016601562,
      "activations/layer4_attention_weight_max": 117.73876190185547,
      "activations/layer4_attention_weight_min": -111.78221893310547,
      "activations/layer5_attention_weight_max": 56.040374755859375,
      "activations/layer5_attention_weight_min": -63.0112419128418,
      "activations/layer6_attention_weight_max": 47.618141174316406,
      "activations/layer6_attention_weight_min": -48.94225311279297,
      "activations/layer7_attention_weight_max": 91.64351654052734,
      "activations/layer7_attention_weight_min": -92.06010437011719,
      "activations/layer8_attention_weight_max": 41.27051544189453,
      "activations/layer8_attention_weight_min": -39.1796989440918,
      "activations/layer9_attention_weight_max": 36.08470153808594,
      "activations/layer9_attention_weight_min": -32.43111801147461,
      "epoch": 8.94,
      "learning_rate": 9.327121212121211e-05,
      "loss": 2.8034,
      "step": 153900
    },
    {
      "activations/layer0_attention_weight_max": 15.454439163208008,
      "activations/layer0_attention_weight_min": -12.053787231445312,
      "activations/layer10_attention_weight_max": 32.992401123046875,
      "activations/layer10_attention_weight_min": -31.732473373413086,
      "activations/layer11_attention_weight_max": 32.366546630859375,
      "activations/layer11_attention_weight_min": -30.765851974487305,
      "activations/layer12_attention_weight_max": 22.70670509338379,
      "activations/layer12_attention_weight_min": -26.65979766845703,
      "activations/layer13_attention_weight_max": 37.4170036315918,
      "activations/layer13_attention_weight_min": -34.755516052246094,
      "activations/layer14_attention_weight_max": 44.09651184082031,
      "activations/layer14_attention_weight_min": -39.50666046142578,
      "activations/layer15_attention_weight_max": 36.471900939941406,
      "activations/layer15_attention_weight_min": -30.673770904541016,
      "activations/layer16_attention_weight_max": 30.468379974365234,
      "activations/layer16_attention_weight_min": -27.39038848876953,
      "activations/layer17_attention_weight_max": 34.44045639038086,
      "activations/layer17_attention_weight_min": -25.150096893310547,
      "activations/layer18_attention_weight_max": 31.57769775390625,
      "activations/layer18_attention_weight_min": -24.49400520324707,
      "activations/layer19_attention_weight_max": 38.628963470458984,
      "activations/layer19_attention_weight_min": -33.63286590576172,
      "activations/layer1_attention_weight_max": 16.034074783325195,
      "activations/layer1_attention_weight_min": -14.140772819519043,
      "activations/layer20_attention_weight_max": 37.8817138671875,
      "activations/layer20_attention_weight_min": -26.055606842041016,
      "activations/layer21_attention_weight_max": 36.30744552612305,
      "activations/layer21_attention_weight_min": -24.496061325073242,
      "activations/layer22_attention_weight_max": 45.77045440673828,
      "activations/layer22_attention_weight_min": -29.077213287353516,
      "activations/layer23_attention_weight_max": 33.258174896240234,
      "activations/layer23_attention_weight_min": -23.751873016357422,
      "activations/layer2_attention_weight_max": 32.44780731201172,
      "activations/layer2_attention_weight_min": -31.59480857849121,
      "activations/layer3_attention_weight_max": 98.24120330810547,
      "activations/layer3_attention_weight_min": -96.1070785522461,
      "activations/layer4_attention_weight_max": 122.79033660888672,
      "activations/layer4_attention_weight_min": -120.58332824707031,
      "activations/layer5_attention_weight_max": 55.48897933959961,
      "activations/layer5_attention_weight_min": -61.49944305419922,
      "activations/layer6_attention_weight_max": 48.59917449951172,
      "activations/layer6_attention_weight_min": -48.085689544677734,
      "activations/layer7_attention_weight_max": 88.58905029296875,
      "activations/layer7_attention_weight_min": -85.95185089111328,
      "activations/layer8_attention_weight_max": 42.7695426940918,
      "activations/layer8_attention_weight_min": -42.0721321105957,
      "activations/layer9_attention_weight_max": 35.96757507324219,
      "activations/layer9_attention_weight_min": -32.506046295166016,
      "epoch": 8.95,
      "learning_rate": 9.325227272727273e-05,
      "loss": 2.7875,
      "step": 153950
    },
    {
      "activations/layer0_attention_weight_max": 15.842720985412598,
      "activations/layer0_attention_weight_min": -12.54639720916748,
      "activations/layer10_attention_weight_max": 35.4378776550293,
      "activations/layer10_attention_weight_min": -33.1728515625,
      "activations/layer11_attention_weight_max": 33.31540298461914,
      "activations/layer11_attention_weight_min": -32.62554931640625,
      "activations/layer12_attention_weight_max": 25.079307556152344,
      "activations/layer12_attention_weight_min": -26.764305114746094,
      "activations/layer13_attention_weight_max": 41.13601303100586,
      "activations/layer13_attention_weight_min": -34.99761199951172,
      "activations/layer14_attention_weight_max": 46.405094146728516,
      "activations/layer14_attention_weight_min": -38.149627685546875,
      "activations/layer15_attention_weight_max": 35.93743896484375,
      "activations/layer15_attention_weight_min": -32.81058120727539,
      "activations/layer16_attention_weight_max": 31.23240852355957,
      "activations/layer16_attention_weight_min": -29.764623641967773,
      "activations/layer17_attention_weight_max": 31.246219635009766,
      "activations/layer17_attention_weight_min": -26.604381561279297,
      "activations/layer18_attention_weight_max": 30.002132415771484,
      "activations/layer18_attention_weight_min": -22.562294006347656,
      "activations/layer19_attention_weight_max": 36.34883117675781,
      "activations/layer19_attention_weight_min": -29.820926666259766,
      "activations/layer1_attention_weight_max": 16.403905868530273,
      "activations/layer1_attention_weight_min": -13.696975708007812,
      "activations/layer20_attention_weight_max": 33.01581954956055,
      "activations/layer20_attention_weight_min": -25.057126998901367,
      "activations/layer21_attention_weight_max": 29.7093505859375,
      "activations/layer21_attention_weight_min": -23.671764373779297,
      "activations/layer22_attention_weight_max": 47.27530288696289,
      "activations/layer22_attention_weight_min": -30.458467483520508,
      "activations/layer23_attention_weight_max": 32.99620819091797,
      "activations/layer23_attention_weight_min": -23.839221954345703,
      "activations/layer2_attention_weight_max": 30.883237838745117,
      "activations/layer2_attention_weight_min": -31.450546264648438,
      "activations/layer3_attention_weight_max": 95.35986328125,
      "activations/layer3_attention_weight_min": -93.97132873535156,
      "activations/layer4_attention_weight_max": 116.61634826660156,
      "activations/layer4_attention_weight_min": -116.71571350097656,
      "activations/layer5_attention_weight_max": 54.04155731201172,
      "activations/layer5_attention_weight_min": -65.986083984375,
      "activations/layer6_attention_weight_max": 46.93911361694336,
      "activations/layer6_attention_weight_min": -57.1453971862793,
      "activations/layer7_attention_weight_max": 90.38216400146484,
      "activations/layer7_attention_weight_min": -89.47615051269531,
      "activations/layer8_attention_weight_max": 41.52301788330078,
      "activations/layer8_attention_weight_min": -43.985511779785156,
      "activations/layer9_attention_weight_max": 38.0443000793457,
      "activations/layer9_attention_weight_min": -34.314697265625,
      "epoch": 8.95,
      "learning_rate": 9.323333333333333e-05,
      "loss": 2.7934,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.4023,
      "eval_samples_per_second": 511.048,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.4023,
      "eval_openwebtext_samples_per_second": 511.048,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.8989,
      "eval_wikitext_samples_per_second": 240.144,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_lambada_loss": 2.5,
      "eval_lambada_ppl": 12.182493960703473,
      "eval_lambada_runtime": 9.5132,
      "eval_lambada_samples_per_second": 511.814,
      "step": 154000
    },
    {
      "activations/layer0_attention_weight_max": 16.78544044494629,
      "activations/layer0_attention_weight_min": -11.755288124084473,
      "activations/layer10_attention_weight_max": 32.42280578613281,
      "activations/layer10_attention_weight_min": -32.31208419799805,
      "activations/layer11_attention_weight_max": 32.688507080078125,
      "activations/layer11_attention_weight_min": -33.290985107421875,
      "activations/layer12_attention_weight_max": 21.808303833007812,
      "activations/layer12_attention_weight_min": -28.259401321411133,
      "activations/layer13_attention_weight_max": 38.11553955078125,
      "activations/layer13_attention_weight_min": -32.07696533203125,
      "activations/layer14_attention_weight_max": 46.872222900390625,
      "activations/layer14_attention_weight_min": -37.04801940917969,
      "activations/layer15_attention_weight_max": 36.42576217651367,
      "activations/layer15_attention_weight_min": -31.4013614654541,
      "activations/layer16_attention_weight_max": 32.186492919921875,
      "activations/layer16_attention_weight_min": -27.36662483215332,
      "activations/layer17_attention_weight_max": 33.761226654052734,
      "activations/layer17_attention_weight_min": -26.06376075744629,
      "activations/layer18_attention_weight_max": 30.8712215423584,
      "activations/layer18_attention_weight_min": -21.685930252075195,
      "activations/layer19_attention_weight_max": 37.125057220458984,
      "activations/layer19_attention_weight_min": -31.791152954101562,
      "activations/layer1_attention_weight_max": 16.649503707885742,
      "activations/layer1_attention_weight_min": -16.215517044067383,
      "activations/layer20_attention_weight_max": 33.69424819946289,
      "activations/layer20_attention_weight_min": -26.236454010009766,
      "activations/layer21_attention_weight_max": 36.09358596801758,
      "activations/layer21_attention_weight_min": -24.21377944946289,
      "activations/layer22_attention_weight_max": 46.71479034423828,
      "activations/layer22_attention_weight_min": -30.28982925415039,
      "activations/layer23_attention_weight_max": 36.33774185180664,
      "activations/layer23_attention_weight_min": -22.30450439453125,
      "activations/layer2_attention_weight_max": 31.511714935302734,
      "activations/layer2_attention_weight_min": -32.104461669921875,
      "activations/layer3_attention_weight_max": 94.3191146850586,
      "activations/layer3_attention_weight_min": -97.02066802978516,
      "activations/layer4_attention_weight_max": 122.3138198852539,
      "activations/layer4_attention_weight_min": -120.3602523803711,
      "activations/layer5_attention_weight_max": 58.7243766784668,
      "activations/layer5_attention_weight_min": -63.34666442871094,
      "activations/layer6_attention_weight_max": 45.81370544433594,
      "activations/layer6_attention_weight_min": -48.95092010498047,
      "activations/layer7_attention_weight_max": 91.24605560302734,
      "activations/layer7_attention_weight_min": -89.02981567382812,
      "activations/layer8_attention_weight_max": 40.406803131103516,
      "activations/layer8_attention_weight_min": -41.25886917114258,
      "activations/layer9_attention_weight_max": 31.998947143554688,
      "activations/layer9_attention_weight_min": -35.95359420776367,
      "epoch": 8.95,
      "learning_rate": 9.321439393939393e-05,
      "loss": 2.8053,
      "step": 154050
    },
    {
      "activations/layer0_attention_weight_max": 15.603189468383789,
      "activations/layer0_attention_weight_min": -11.620218276977539,
      "activations/layer10_attention_weight_max": 32.39774703979492,
      "activations/layer10_attention_weight_min": -31.561519622802734,
      "activations/layer11_attention_weight_max": 34.47773361206055,
      "activations/layer11_attention_weight_min": -31.401487350463867,
      "activations/layer12_attention_weight_max": 38.388309478759766,
      "activations/layer12_attention_weight_min": -23.562118530273438,
      "activations/layer13_attention_weight_max": 45.55286407470703,
      "activations/layer13_attention_weight_min": -31.42754554748535,
      "activations/layer14_attention_weight_max": 46.29505157470703,
      "activations/layer14_attention_weight_min": -38.119014739990234,
      "activations/layer15_attention_weight_max": 41.970767974853516,
      "activations/layer15_attention_weight_min": -30.373897552490234,
      "activations/layer16_attention_weight_max": 32.91592788696289,
      "activations/layer16_attention_weight_min": -29.185129165649414,
      "activations/layer17_attention_weight_max": 35.86203384399414,
      "activations/layer17_attention_weight_min": -24.664344787597656,
      "activations/layer18_attention_weight_max": 32.59225845336914,
      "activations/layer18_attention_weight_min": -21.86833381652832,
      "activations/layer19_attention_weight_max": 35.402183532714844,
      "activations/layer19_attention_weight_min": -29.518024444580078,
      "activations/layer1_attention_weight_max": 15.693541526794434,
      "activations/layer1_attention_weight_min": -14.724452018737793,
      "activations/layer20_attention_weight_max": 31.651277542114258,
      "activations/layer20_attention_weight_min": -22.597476959228516,
      "activations/layer21_attention_weight_max": 31.141965866088867,
      "activations/layer21_attention_weight_min": -21.091638565063477,
      "activations/layer22_attention_weight_max": 45.36195373535156,
      "activations/layer22_attention_weight_min": -29.559492111206055,
      "activations/layer23_attention_weight_max": 38.40583419799805,
      "activations/layer23_attention_weight_min": -26.634449005126953,
      "activations/layer2_attention_weight_max": 34.268531799316406,
      "activations/layer2_attention_weight_min": -32.76746368408203,
      "activations/layer3_attention_weight_max": 94.54424285888672,
      "activations/layer3_attention_weight_min": -93.59866333007812,
      "activations/layer4_attention_weight_max": 119.306396484375,
      "activations/layer4_attention_weight_min": -114.72920989990234,
      "activations/layer5_attention_weight_max": 54.656307220458984,
      "activations/layer5_attention_weight_min": -64.79560852050781,
      "activations/layer6_attention_weight_max": 49.63626480102539,
      "activations/layer6_attention_weight_min": -49.60227966308594,
      "activations/layer7_attention_weight_max": 91.12158966064453,
      "activations/layer7_attention_weight_min": -91.39810180664062,
      "activations/layer8_attention_weight_max": 39.30546951293945,
      "activations/layer8_attention_weight_min": -39.73574447631836,
      "activations/layer9_attention_weight_max": 32.601009368896484,
      "activations/layer9_attention_weight_min": -33.35905075073242,
      "epoch": 8.95,
      "learning_rate": 9.319545454545454e-05,
      "loss": 2.7976,
      "step": 154100
    },
    {
      "activations/layer0_attention_weight_max": 15.249462127685547,
      "activations/layer0_attention_weight_min": -11.570531845092773,
      "activations/layer10_attention_weight_max": 33.00636291503906,
      "activations/layer10_attention_weight_min": -30.215360641479492,
      "activations/layer11_attention_weight_max": 30.531597137451172,
      "activations/layer11_attention_weight_min": -31.441823959350586,
      "activations/layer12_attention_weight_max": 25.27869415283203,
      "activations/layer12_attention_weight_min": -25.79399871826172,
      "activations/layer13_attention_weight_max": 38.49290466308594,
      "activations/layer13_attention_weight_min": -30.82880401611328,
      "activations/layer14_attention_weight_max": 46.31802749633789,
      "activations/layer14_attention_weight_min": -36.22422790527344,
      "activations/layer15_attention_weight_max": 34.326698303222656,
      "activations/layer15_attention_weight_min": -30.37912368774414,
      "activations/layer16_attention_weight_max": 28.876880645751953,
      "activations/layer16_attention_weight_min": -25.327068328857422,
      "activations/layer17_attention_weight_max": 30.37260627746582,
      "activations/layer17_attention_weight_min": -24.69095802307129,
      "activations/layer18_attention_weight_max": 30.791963577270508,
      "activations/layer18_attention_weight_min": -20.86841583251953,
      "activations/layer19_attention_weight_max": 31.57745361328125,
      "activations/layer19_attention_weight_min": -28.21673583984375,
      "activations/layer1_attention_weight_max": 16.642471313476562,
      "activations/layer1_attention_weight_min": -13.792183876037598,
      "activations/layer20_attention_weight_max": 31.80385971069336,
      "activations/layer20_attention_weight_min": -23.059629440307617,
      "activations/layer21_attention_weight_max": 30.120359420776367,
      "activations/layer21_attention_weight_min": -21.866191864013672,
      "activations/layer22_attention_weight_max": 45.70984649658203,
      "activations/layer22_attention_weight_min": -26.90135955810547,
      "activations/layer23_attention_weight_max": 32.88567352294922,
      "activations/layer23_attention_weight_min": -21.457334518432617,
      "activations/layer2_attention_weight_max": 31.017911911010742,
      "activations/layer2_attention_weight_min": -31.028465270996094,
      "activations/layer3_attention_weight_max": 89.89716339111328,
      "activations/layer3_attention_weight_min": -92.02190399169922,
      "activations/layer4_attention_weight_max": 113.4997787475586,
      "activations/layer4_attention_weight_min": -112.30669403076172,
      "activations/layer5_attention_weight_max": 53.6444091796875,
      "activations/layer5_attention_weight_min": -60.93280792236328,
      "activations/layer6_attention_weight_max": 45.0594596862793,
      "activations/layer6_attention_weight_min": -47.98708724975586,
      "activations/layer7_attention_weight_max": 88.84142303466797,
      "activations/layer7_attention_weight_min": -81.41679382324219,
      "activations/layer8_attention_weight_max": 39.894371032714844,
      "activations/layer8_attention_weight_min": -39.939205169677734,
      "activations/layer9_attention_weight_max": 35.115264892578125,
      "activations/layer9_attention_weight_min": -33.587867736816406,
      "epoch": 8.96,
      "learning_rate": 9.317651515151515e-05,
      "loss": 2.7972,
      "step": 154150
    },
    {
      "activations/layer0_attention_weight_max": 15.033349990844727,
      "activations/layer0_attention_weight_min": -11.675443649291992,
      "activations/layer10_attention_weight_max": 34.39881134033203,
      "activations/layer10_attention_weight_min": -31.895584106445312,
      "activations/layer11_attention_weight_max": 37.353111267089844,
      "activations/layer11_attention_weight_min": -33.02846908569336,
      "activations/layer12_attention_weight_max": 29.129243850708008,
      "activations/layer12_attention_weight_min": -26.081083297729492,
      "activations/layer13_attention_weight_max": 39.90637969970703,
      "activations/layer13_attention_weight_min": -32.825584411621094,
      "activations/layer14_attention_weight_max": 49.4205322265625,
      "activations/layer14_attention_weight_min": -40.97444534301758,
      "activations/layer15_attention_weight_max": 35.7093505859375,
      "activations/layer15_attention_weight_min": -31.06593894958496,
      "activations/layer16_attention_weight_max": 30.377717971801758,
      "activations/layer16_attention_weight_min": -27.971349716186523,
      "activations/layer17_attention_weight_max": 33.930233001708984,
      "activations/layer17_attention_weight_min": -25.99246597290039,
      "activations/layer18_attention_weight_max": 30.57314682006836,
      "activations/layer18_attention_weight_min": -23.060932159423828,
      "activations/layer19_attention_weight_max": 38.7286262512207,
      "activations/layer19_attention_weight_min": -31.81282615661621,
      "activations/layer1_attention_weight_max": 16.152864456176758,
      "activations/layer1_attention_weight_min": -15.49620532989502,
      "activations/layer20_attention_weight_max": 32.619686126708984,
      "activations/layer20_attention_weight_min": -25.087318420410156,
      "activations/layer21_attention_weight_max": 29.947629928588867,
      "activations/layer21_attention_weight_min": -24.094249725341797,
      "activations/layer22_attention_weight_max": 44.76324462890625,
      "activations/layer22_attention_weight_min": -31.430932998657227,
      "activations/layer23_attention_weight_max": 39.90476989746094,
      "activations/layer23_attention_weight_min": -25.944021224975586,
      "activations/layer2_attention_weight_max": 33.30678939819336,
      "activations/layer2_attention_weight_min": -33.869815826416016,
      "activations/layer3_attention_weight_max": 94.23086547851562,
      "activations/layer3_attention_weight_min": -96.60323333740234,
      "activations/layer4_attention_weight_max": 117.6512222290039,
      "activations/layer4_attention_weight_min": -113.64595794677734,
      "activations/layer5_attention_weight_max": 55.13155746459961,
      "activations/layer5_attention_weight_min": -60.559112548828125,
      "activations/layer6_attention_weight_max": 45.44701385498047,
      "activations/layer6_attention_weight_min": -49.736366271972656,
      "activations/layer7_attention_weight_max": 93.45887756347656,
      "activations/layer7_attention_weight_min": -83.32811737060547,
      "activations/layer8_attention_weight_max": 40.998443603515625,
      "activations/layer8_attention_weight_min": -40.24589538574219,
      "activations/layer9_attention_weight_max": 34.46964645385742,
      "activations/layer9_attention_weight_min": -32.95292663574219,
      "epoch": 8.96,
      "learning_rate": 9.315757575757575e-05,
      "loss": 2.7897,
      "step": 154200
    },
    {
      "activations/layer0_attention_weight_max": 14.60205078125,
      "activations/layer0_attention_weight_min": -12.448748588562012,
      "activations/layer10_attention_weight_max": 33.072235107421875,
      "activations/layer10_attention_weight_min": -31.773971557617188,
      "activations/layer11_attention_weight_max": 34.02582550048828,
      "activations/layer11_attention_weight_min": -30.74061393737793,
      "activations/layer12_attention_weight_max": 28.262067794799805,
      "activations/layer12_attention_weight_min": -24.6470947265625,
      "activations/layer13_attention_weight_max": 37.62793731689453,
      "activations/layer13_attention_weight_min": -31.611042022705078,
      "activations/layer14_attention_weight_max": 49.68880081176758,
      "activations/layer14_attention_weight_min": -40.149776458740234,
      "activations/layer15_attention_weight_max": 35.59718704223633,
      "activations/layer15_attention_weight_min": -30.961227416992188,
      "activations/layer16_attention_weight_max": 33.00717544555664,
      "activations/layer16_attention_weight_min": -27.449670791625977,
      "activations/layer17_attention_weight_max": 39.20466613769531,
      "activations/layer17_attention_weight_min": -23.991411209106445,
      "activations/layer18_attention_weight_max": 33.4554557800293,
      "activations/layer18_attention_weight_min": -21.71969223022461,
      "activations/layer19_attention_weight_max": 36.703094482421875,
      "activations/layer19_attention_weight_min": -29.566843032836914,
      "activations/layer1_attention_weight_max": 15.894723892211914,
      "activations/layer1_attention_weight_min": -14.315544128417969,
      "activations/layer20_attention_weight_max": 31.472808837890625,
      "activations/layer20_attention_weight_min": -24.678415298461914,
      "activations/layer21_attention_weight_max": 29.862974166870117,
      "activations/layer21_attention_weight_min": -21.741491317749023,
      "activations/layer22_attention_weight_max": 45.53128433227539,
      "activations/layer22_attention_weight_min": -28.37674903869629,
      "activations/layer23_attention_weight_max": 36.962303161621094,
      "activations/layer23_attention_weight_min": -22.872894287109375,
      "activations/layer2_attention_weight_max": 32.62066650390625,
      "activations/layer2_attention_weight_min": -31.046855926513672,
      "activations/layer3_attention_weight_max": 90.8081283569336,
      "activations/layer3_attention_weight_min": -93.71585083007812,
      "activations/layer4_attention_weight_max": 115.54000091552734,
      "activations/layer4_attention_weight_min": -111.17085266113281,
      "activations/layer5_attention_weight_max": 61.43854522705078,
      "activations/layer5_attention_weight_min": -61.73213195800781,
      "activations/layer6_attention_weight_max": 50.33345413208008,
      "activations/layer6_attention_weight_min": -49.9394645690918,
      "activations/layer7_attention_weight_max": 91.2823715209961,
      "activations/layer7_attention_weight_min": -98.0246353149414,
      "activations/layer8_attention_weight_max": 39.804405212402344,
      "activations/layer8_attention_weight_min": -40.64736557006836,
      "activations/layer9_attention_weight_max": 33.775634765625,
      "activations/layer9_attention_weight_min": -34.049346923828125,
      "epoch": 8.96,
      "learning_rate": 9.313863636363635e-05,
      "loss": 2.7972,
      "step": 154250
    },
    {
      "activations/layer0_attention_weight_max": 14.309891700744629,
      "activations/layer0_attention_weight_min": -11.788415908813477,
      "activations/layer10_attention_weight_max": 31.336870193481445,
      "activations/layer10_attention_weight_min": -37.587677001953125,
      "activations/layer11_attention_weight_max": 31.251882553100586,
      "activations/layer11_attention_weight_min": -35.12712860107422,
      "activations/layer12_attention_weight_max": 23.419450759887695,
      "activations/layer12_attention_weight_min": -30.753902435302734,
      "activations/layer13_attention_weight_max": 40.14909362792969,
      "activations/layer13_attention_weight_min": -32.216705322265625,
      "activations/layer14_attention_weight_max": 42.942203521728516,
      "activations/layer14_attention_weight_min": -35.7501106262207,
      "activations/layer15_attention_weight_max": 34.132720947265625,
      "activations/layer15_attention_weight_min": -32.11219787597656,
      "activations/layer16_attention_weight_max": 34.43832778930664,
      "activations/layer16_attention_weight_min": -29.282001495361328,
      "activations/layer17_attention_weight_max": 35.07974624633789,
      "activations/layer17_attention_weight_min": -27.4451847076416,
      "activations/layer18_attention_weight_max": 31.671287536621094,
      "activations/layer18_attention_weight_min": -22.41181182861328,
      "activations/layer19_attention_weight_max": 34.71586990356445,
      "activations/layer19_attention_weight_min": -30.5604305267334,
      "activations/layer1_attention_weight_max": 16.723649978637695,
      "activations/layer1_attention_weight_min": -15.185667991638184,
      "activations/layer20_attention_weight_max": 30.400585174560547,
      "activations/layer20_attention_weight_min": -24.388242721557617,
      "activations/layer21_attention_weight_max": 28.270755767822266,
      "activations/layer21_attention_weight_min": -22.556846618652344,
      "activations/layer22_attention_weight_max": 45.25869369506836,
      "activations/layer22_attention_weight_min": -29.21128273010254,
      "activations/layer23_attention_weight_max": 33.892574310302734,
      "activations/layer23_attention_weight_min": -24.781272888183594,
      "activations/layer2_attention_weight_max": 34.01852035522461,
      "activations/layer2_attention_weight_min": -32.47930145263672,
      "activations/layer3_attention_weight_max": 97.27619934082031,
      "activations/layer3_attention_weight_min": -99.4307632446289,
      "activations/layer4_attention_weight_max": 120.02404022216797,
      "activations/layer4_attention_weight_min": -118.09942626953125,
      "activations/layer5_attention_weight_max": 54.127227783203125,
      "activations/layer5_attention_weight_min": -60.56718444824219,
      "activations/layer6_attention_weight_max": 48.1306266784668,
      "activations/layer6_attention_weight_min": -49.84028244018555,
      "activations/layer7_attention_weight_max": 95.09849548339844,
      "activations/layer7_attention_weight_min": -89.45511627197266,
      "activations/layer8_attention_weight_max": 39.35210037231445,
      "activations/layer8_attention_weight_min": -42.21031951904297,
      "activations/layer9_attention_weight_max": 33.36931610107422,
      "activations/layer9_attention_weight_min": -36.200679779052734,
      "epoch": 8.97,
      "learning_rate": 9.311969696969696e-05,
      "loss": 2.8001,
      "step": 154300
    },
    {
      "activations/layer0_attention_weight_max": 14.472996711730957,
      "activations/layer0_attention_weight_min": -11.687057495117188,
      "activations/layer10_attention_weight_max": 34.1736946105957,
      "activations/layer10_attention_weight_min": -33.236244201660156,
      "activations/layer11_attention_weight_max": 32.19510269165039,
      "activations/layer11_attention_weight_min": -31.078781127929688,
      "activations/layer12_attention_weight_max": 22.88098907470703,
      "activations/layer12_attention_weight_min": -24.546409606933594,
      "activations/layer13_attention_weight_max": 41.86424255371094,
      "activations/layer13_attention_weight_min": -33.850303649902344,
      "activations/layer14_attention_weight_max": 44.475013732910156,
      "activations/layer14_attention_weight_min": -37.26645278930664,
      "activations/layer15_attention_weight_max": 37.814231872558594,
      "activations/layer15_attention_weight_min": -30.97561264038086,
      "activations/layer16_attention_weight_max": 32.20798110961914,
      "activations/layer16_attention_weight_min": -28.005937576293945,
      "activations/layer17_attention_weight_max": 32.19113540649414,
      "activations/layer17_attention_weight_min": -24.938491821289062,
      "activations/layer18_attention_weight_max": 30.95884895324707,
      "activations/layer18_attention_weight_min": -22.201004028320312,
      "activations/layer19_attention_weight_max": 39.13753890991211,
      "activations/layer19_attention_weight_min": -28.509565353393555,
      "activations/layer1_attention_weight_max": 16.144058227539062,
      "activations/layer1_attention_weight_min": -14.613334655761719,
      "activations/layer20_attention_weight_max": 29.954143524169922,
      "activations/layer20_attention_weight_min": -22.966766357421875,
      "activations/layer21_attention_weight_max": 29.472824096679688,
      "activations/layer21_attention_weight_min": -22.313344955444336,
      "activations/layer22_attention_weight_max": 42.838218688964844,
      "activations/layer22_attention_weight_min": -28.183305740356445,
      "activations/layer23_attention_weight_max": 35.859169006347656,
      "activations/layer23_attention_weight_min": -21.549245834350586,
      "activations/layer2_attention_weight_max": 32.30949401855469,
      "activations/layer2_attention_weight_min": -32.99300003051758,
      "activations/layer3_attention_weight_max": 100.9783706665039,
      "activations/layer3_attention_weight_min": -104.9205551147461,
      "activations/layer4_attention_weight_max": 123.79620361328125,
      "activations/layer4_attention_weight_min": -122.85798645019531,
      "activations/layer5_attention_weight_max": 56.32389450073242,
      "activations/layer5_attention_weight_min": -63.35934066772461,
      "activations/layer6_attention_weight_max": 48.04325485229492,
      "activations/layer6_attention_weight_min": -50.47520446777344,
      "activations/layer7_attention_weight_max": 88.96614837646484,
      "activations/layer7_attention_weight_min": -87.1874771118164,
      "activations/layer8_attention_weight_max": 39.370689392089844,
      "activations/layer8_attention_weight_min": -42.13380813598633,
      "activations/layer9_attention_weight_max": 33.97799301147461,
      "activations/layer9_attention_weight_min": -34.434444427490234,
      "epoch": 8.97,
      "learning_rate": 9.310075757575758e-05,
      "loss": 2.791,
      "step": 154350
    },
    {
      "activations/layer0_attention_weight_max": 14.871071815490723,
      "activations/layer0_attention_weight_min": -12.394156455993652,
      "activations/layer10_attention_weight_max": 33.15780258178711,
      "activations/layer10_attention_weight_min": -31.39585304260254,
      "activations/layer11_attention_weight_max": 32.61875915527344,
      "activations/layer11_attention_weight_min": -30.512435913085938,
      "activations/layer12_attention_weight_max": 23.652849197387695,
      "activations/layer12_attention_weight_min": -26.658126831054688,
      "activations/layer13_attention_weight_max": 38.643333435058594,
      "activations/layer13_attention_weight_min": -34.732078552246094,
      "activations/layer14_attention_weight_max": 48.497989654541016,
      "activations/layer14_attention_weight_min": -40.54814147949219,
      "activations/layer15_attention_weight_max": 35.97690200805664,
      "activations/layer15_attention_weight_min": -31.86208152770996,
      "activations/layer16_attention_weight_max": 31.427526473999023,
      "activations/layer16_attention_weight_min": -27.177106857299805,
      "activations/layer17_attention_weight_max": 33.39397048950195,
      "activations/layer17_attention_weight_min": -25.977458953857422,
      "activations/layer18_attention_weight_max": 35.08775329589844,
      "activations/layer18_attention_weight_min": -22.94847297668457,
      "activations/layer19_attention_weight_max": 36.6023063659668,
      "activations/layer19_attention_weight_min": -32.2025032043457,
      "activations/layer1_attention_weight_max": 17.027332305908203,
      "activations/layer1_attention_weight_min": -15.130058288574219,
      "activations/layer20_attention_weight_max": 29.843894958496094,
      "activations/layer20_attention_weight_min": -24.192827224731445,
      "activations/layer21_attention_weight_max": 29.647579193115234,
      "activations/layer21_attention_weight_min": -24.04250144958496,
      "activations/layer22_attention_weight_max": 43.85062789916992,
      "activations/layer22_attention_weight_min": -30.4298152923584,
      "activations/layer23_attention_weight_max": 35.727088928222656,
      "activations/layer23_attention_weight_min": -24.22454071044922,
      "activations/layer2_attention_weight_max": 32.95631408691406,
      "activations/layer2_attention_weight_min": -33.209102630615234,
      "activations/layer3_attention_weight_max": 95.94147491455078,
      "activations/layer3_attention_weight_min": -99.40418243408203,
      "activations/layer4_attention_weight_max": 122.67461395263672,
      "activations/layer4_attention_weight_min": -122.29206848144531,
      "activations/layer5_attention_weight_max": 54.99577331542969,
      "activations/layer5_attention_weight_min": -63.96590805053711,
      "activations/layer6_attention_weight_max": 48.61021041870117,
      "activations/layer6_attention_weight_min": -51.207542419433594,
      "activations/layer7_attention_weight_max": 88.9666519165039,
      "activations/layer7_attention_weight_min": -90.06989288330078,
      "activations/layer8_attention_weight_max": 38.89632034301758,
      "activations/layer8_attention_weight_min": -42.22212219238281,
      "activations/layer9_attention_weight_max": 33.212684631347656,
      "activations/layer9_attention_weight_min": -34.054752349853516,
      "epoch": 8.97,
      "learning_rate": 9.308181818181817e-05,
      "loss": 2.7959,
      "step": 154400
    },
    {
      "activations/layer0_attention_weight_max": 14.88542366027832,
      "activations/layer0_attention_weight_min": -11.797039985656738,
      "activations/layer10_attention_weight_max": 33.90930938720703,
      "activations/layer10_attention_weight_min": -31.248558044433594,
      "activations/layer11_attention_weight_max": 32.10820770263672,
      "activations/layer11_attention_weight_min": -31.419296264648438,
      "activations/layer12_attention_weight_max": 26.40507698059082,
      "activations/layer12_attention_weight_min": -24.87067985534668,
      "activations/layer13_attention_weight_max": 41.1130485534668,
      "activations/layer13_attention_weight_min": -32.72165298461914,
      "activations/layer14_attention_weight_max": 51.167327880859375,
      "activations/layer14_attention_weight_min": -39.34103012084961,
      "activations/layer15_attention_weight_max": 37.349143981933594,
      "activations/layer15_attention_weight_min": -31.267955780029297,
      "activations/layer16_attention_weight_max": 32.40324783325195,
      "activations/layer16_attention_weight_min": -26.62875747680664,
      "activations/layer17_attention_weight_max": 35.578372955322266,
      "activations/layer17_attention_weight_min": -25.271257400512695,
      "activations/layer18_attention_weight_max": 31.687118530273438,
      "activations/layer18_attention_weight_min": -24.020627975463867,
      "activations/layer19_attention_weight_max": 34.18341064453125,
      "activations/layer19_attention_weight_min": -28.926929473876953,
      "activations/layer1_attention_weight_max": 15.938095092773438,
      "activations/layer1_attention_weight_min": -14.767541885375977,
      "activations/layer20_attention_weight_max": 31.64901351928711,
      "activations/layer20_attention_weight_min": -25.236927032470703,
      "activations/layer21_attention_weight_max": 30.175743103027344,
      "activations/layer21_attention_weight_min": -25.76921844482422,
      "activations/layer22_attention_weight_max": 49.54694366455078,
      "activations/layer22_attention_weight_min": -30.076210021972656,
      "activations/layer23_attention_weight_max": 36.83200454711914,
      "activations/layer23_attention_weight_min": -23.735633850097656,
      "activations/layer2_attention_weight_max": 32.67218780517578,
      "activations/layer2_attention_weight_min": -33.192405700683594,
      "activations/layer3_attention_weight_max": 92.0421371459961,
      "activations/layer3_attention_weight_min": -97.76751708984375,
      "activations/layer4_attention_weight_max": 114.47628021240234,
      "activations/layer4_attention_weight_min": -113.00518798828125,
      "activations/layer5_attention_weight_max": 52.562355041503906,
      "activations/layer5_attention_weight_min": -61.24992370605469,
      "activations/layer6_attention_weight_max": 47.48461151123047,
      "activations/layer6_attention_weight_min": -49.478919982910156,
      "activations/layer7_attention_weight_max": 90.09062957763672,
      "activations/layer7_attention_weight_min": -90.25678253173828,
      "activations/layer8_attention_weight_max": 41.64030456542969,
      "activations/layer8_attention_weight_min": -38.66877365112305,
      "activations/layer9_attention_weight_max": 34.091156005859375,
      "activations/layer9_attention_weight_min": -32.239967346191406,
      "epoch": 8.97,
      "learning_rate": 9.306287878787878e-05,
      "loss": 2.7969,
      "step": 154450
    },
    {
      "activations/layer0_attention_weight_max": 14.991186141967773,
      "activations/layer0_attention_weight_min": -12.285006523132324,
      "activations/layer10_attention_weight_max": 32.851043701171875,
      "activations/layer10_attention_weight_min": -30.473960876464844,
      "activations/layer11_attention_weight_max": 35.868797302246094,
      "activations/layer11_attention_weight_min": -30.63673973083496,
      "activations/layer12_attention_weight_max": 23.127424240112305,
      "activations/layer12_attention_weight_min": -24.41926383972168,
      "activations/layer13_attention_weight_max": 40.90983581542969,
      "activations/layer13_attention_weight_min": -32.207176208496094,
      "activations/layer14_attention_weight_max": 47.55729293823242,
      "activations/layer14_attention_weight_min": -40.90974044799805,
      "activations/layer15_attention_weight_max": 38.7347526550293,
      "activations/layer15_attention_weight_min": -30.29662322998047,
      "activations/layer16_attention_weight_max": 30.221424102783203,
      "activations/layer16_attention_weight_min": -27.934396743774414,
      "activations/layer17_attention_weight_max": 34.85216522216797,
      "activations/layer17_attention_weight_min": -24.733591079711914,
      "activations/layer18_attention_weight_max": 30.996313095092773,
      "activations/layer18_attention_weight_min": -23.044872283935547,
      "activations/layer19_attention_weight_max": 34.49066925048828,
      "activations/layer19_attention_weight_min": -31.093168258666992,
      "activations/layer1_attention_weight_max": 16.61737823486328,
      "activations/layer1_attention_weight_min": -14.857513427734375,
      "activations/layer20_attention_weight_max": 30.80095100402832,
      "activations/layer20_attention_weight_min": -23.83434295654297,
      "activations/layer21_attention_weight_max": 29.27623176574707,
      "activations/layer21_attention_weight_min": -23.062053680419922,
      "activations/layer22_attention_weight_max": 50.47807693481445,
      "activations/layer22_attention_weight_min": -29.198389053344727,
      "activations/layer23_attention_weight_max": 34.68560028076172,
      "activations/layer23_attention_weight_min": -23.85597801208496,
      "activations/layer2_attention_weight_max": 32.79497528076172,
      "activations/layer2_attention_weight_min": -32.73478317260742,
      "activations/layer3_attention_weight_max": 97.01201629638672,
      "activations/layer3_attention_weight_min": -100.74375915527344,
      "activations/layer4_attention_weight_max": 119.78133392333984,
      "activations/layer4_attention_weight_min": -124.4929428100586,
      "activations/layer5_attention_weight_max": 54.37266159057617,
      "activations/layer5_attention_weight_min": -63.44745635986328,
      "activations/layer6_attention_weight_max": 47.86206817626953,
      "activations/layer6_attention_weight_min": -53.00041580200195,
      "activations/layer7_attention_weight_max": 93.4880142211914,
      "activations/layer7_attention_weight_min": -92.96003723144531,
      "activations/layer8_attention_weight_max": 42.26887512207031,
      "activations/layer8_attention_weight_min": -39.28963851928711,
      "activations/layer9_attention_weight_max": 37.152565002441406,
      "activations/layer9_attention_weight_min": -34.40886306762695,
      "epoch": 8.98,
      "learning_rate": 9.30439393939394e-05,
      "loss": 2.7982,
      "step": 154500
    },
    {
      "activations/layer0_attention_weight_max": 15.476705551147461,
      "activations/layer0_attention_weight_min": -12.801785469055176,
      "activations/layer10_attention_weight_max": 31.388540267944336,
      "activations/layer10_attention_weight_min": -30.49752426147461,
      "activations/layer11_attention_weight_max": 30.434965133666992,
      "activations/layer11_attention_weight_min": -29.143421173095703,
      "activations/layer12_attention_weight_max": 22.75040054321289,
      "activations/layer12_attention_weight_min": -25.127241134643555,
      "activations/layer13_attention_weight_max": 40.217620849609375,
      "activations/layer13_attention_weight_min": -34.683284759521484,
      "activations/layer14_attention_weight_max": 46.552001953125,
      "activations/layer14_attention_weight_min": -38.241966247558594,
      "activations/layer15_attention_weight_max": 35.02043533325195,
      "activations/layer15_attention_weight_min": -29.551172256469727,
      "activations/layer16_attention_weight_max": 29.020767211914062,
      "activations/layer16_attention_weight_min": -28.221464157104492,
      "activations/layer17_attention_weight_max": 34.735530853271484,
      "activations/layer17_attention_weight_min": -24.56858253479004,
      "activations/layer18_attention_weight_max": 30.09443473815918,
      "activations/layer18_attention_weight_min": -21.47430419921875,
      "activations/layer19_attention_weight_max": 33.73908233642578,
      "activations/layer19_attention_weight_min": -29.5122013092041,
      "activations/layer1_attention_weight_max": 17.407886505126953,
      "activations/layer1_attention_weight_min": -14.619675636291504,
      "activations/layer20_attention_weight_max": 30.89445686340332,
      "activations/layer20_attention_weight_min": -24.296451568603516,
      "activations/layer21_attention_weight_max": 25.943870544433594,
      "activations/layer21_attention_weight_min": -24.153690338134766,
      "activations/layer22_attention_weight_max": 55.24217224121094,
      "activations/layer22_attention_weight_min": -28.326887130737305,
      "activations/layer23_attention_weight_max": 36.1365966796875,
      "activations/layer23_attention_weight_min": -21.479360580444336,
      "activations/layer2_attention_weight_max": 34.70009231567383,
      "activations/layer2_attention_weight_min": -33.87544631958008,
      "activations/layer3_attention_weight_max": 98.93582916259766,
      "activations/layer3_attention_weight_min": -102.58350372314453,
      "activations/layer4_attention_weight_max": 117.91446685791016,
      "activations/layer4_attention_weight_min": -115.96294403076172,
      "activations/layer5_attention_weight_max": 52.626060485839844,
      "activations/layer5_attention_weight_min": -61.35810852050781,
      "activations/layer6_attention_weight_max": 46.81513595581055,
      "activations/layer6_attention_weight_min": -54.01841735839844,
      "activations/layer7_attention_weight_max": 86.96111297607422,
      "activations/layer7_attention_weight_min": -94.90980529785156,
      "activations/layer8_attention_weight_max": 38.57118225097656,
      "activations/layer8_attention_weight_min": -40.72587966918945,
      "activations/layer9_attention_weight_max": 33.37510299682617,
      "activations/layer9_attention_weight_min": -34.17138671875,
      "epoch": 8.98,
      "learning_rate": 9.302499999999998e-05,
      "loss": 2.8022,
      "step": 154550
    },
    {
      "activations/layer0_attention_weight_max": 15.925883293151855,
      "activations/layer0_attention_weight_min": -11.907210350036621,
      "activations/layer10_attention_weight_max": 36.17488098144531,
      "activations/layer10_attention_weight_min": -32.210205078125,
      "activations/layer11_attention_weight_max": 33.733367919921875,
      "activations/layer11_attention_weight_min": -31.34565544128418,
      "activations/layer12_attention_weight_max": 24.699031829833984,
      "activations/layer12_attention_weight_min": -25.852893829345703,
      "activations/layer13_attention_weight_max": 40.22772979736328,
      "activations/layer13_attention_weight_min": -33.441070556640625,
      "activations/layer14_attention_weight_max": 49.07426452636719,
      "activations/layer14_attention_weight_min": -39.33330535888672,
      "activations/layer15_attention_weight_max": 36.54780578613281,
      "activations/layer15_attention_weight_min": -33.19083786010742,
      "activations/layer16_attention_weight_max": 29.382959365844727,
      "activations/layer16_attention_weight_min": -28.44468116760254,
      "activations/layer17_attention_weight_max": 31.239370346069336,
      "activations/layer17_attention_weight_min": -25.489294052124023,
      "activations/layer18_attention_weight_max": 30.390207290649414,
      "activations/layer18_attention_weight_min": -20.821197509765625,
      "activations/layer19_attention_weight_max": 35.1981086730957,
      "activations/layer19_attention_weight_min": -32.12626647949219,
      "activations/layer1_attention_weight_max": 16.47127914428711,
      "activations/layer1_attention_weight_min": -15.401326179504395,
      "activations/layer20_attention_weight_max": 28.595556259155273,
      "activations/layer20_attention_weight_min": -26.714820861816406,
      "activations/layer21_attention_weight_max": 27.710317611694336,
      "activations/layer21_attention_weight_min": -23.253049850463867,
      "activations/layer22_attention_weight_max": 42.834266662597656,
      "activations/layer22_attention_weight_min": -29.95201301574707,
      "activations/layer23_attention_weight_max": 35.52904510498047,
      "activations/layer23_attention_weight_min": -22.297727584838867,
      "activations/layer2_attention_weight_max": 33.971824645996094,
      "activations/layer2_attention_weight_min": -33.80986785888672,
      "activations/layer3_attention_weight_max": 97.12490844726562,
      "activations/layer3_attention_weight_min": -95.7012710571289,
      "activations/layer4_attention_weight_max": 117.44039154052734,
      "activations/layer4_attention_weight_min": -115.3887939453125,
      "activations/layer5_attention_weight_max": 52.536617279052734,
      "activations/layer5_attention_weight_min": -62.771759033203125,
      "activations/layer6_attention_weight_max": 43.33613204956055,
      "activations/layer6_attention_weight_min": -49.66339874267578,
      "activations/layer7_attention_weight_max": 89.06431579589844,
      "activations/layer7_attention_weight_min": -94.36515808105469,
      "activations/layer8_attention_weight_max": 39.63173294067383,
      "activations/layer8_attention_weight_min": -41.04743576049805,
      "activations/layer9_attention_weight_max": 37.49273681640625,
      "activations/layer9_attention_weight_min": -34.871604919433594,
      "epoch": 8.98,
      "learning_rate": 9.30060606060606e-05,
      "loss": 2.8169,
      "step": 154600
    },
    {
      "activations/layer0_attention_weight_max": 15.501886367797852,
      "activations/layer0_attention_weight_min": -12.040522575378418,
      "activations/layer10_attention_weight_max": 30.229013442993164,
      "activations/layer10_attention_weight_min": -29.106096267700195,
      "activations/layer11_attention_weight_max": 30.573955535888672,
      "activations/layer11_attention_weight_min": -28.899044036865234,
      "activations/layer12_attention_weight_max": 24.277801513671875,
      "activations/layer12_attention_weight_min": -28.979969024658203,
      "activations/layer13_attention_weight_max": 40.66817855834961,
      "activations/layer13_attention_weight_min": -31.748353958129883,
      "activations/layer14_attention_weight_max": 47.11378860473633,
      "activations/layer14_attention_weight_min": -35.56976318359375,
      "activations/layer15_attention_weight_max": 36.92304611206055,
      "activations/layer15_attention_weight_min": -27.74591636657715,
      "activations/layer16_attention_weight_max": 29.675884246826172,
      "activations/layer16_attention_weight_min": -25.995765686035156,
      "activations/layer17_attention_weight_max": 35.75660705566406,
      "activations/layer17_attention_weight_min": -26.38693618774414,
      "activations/layer18_attention_weight_max": 33.37774658203125,
      "activations/layer18_attention_weight_min": -22.72062110900879,
      "activations/layer19_attention_weight_max": 37.052024841308594,
      "activations/layer19_attention_weight_min": -31.541900634765625,
      "activations/layer1_attention_weight_max": 16.51490020751953,
      "activations/layer1_attention_weight_min": -15.512431144714355,
      "activations/layer20_attention_weight_max": 33.98989486694336,
      "activations/layer20_attention_weight_min": -23.48580551147461,
      "activations/layer21_attention_weight_max": 34.13452911376953,
      "activations/layer21_attention_weight_min": -24.48937225341797,
      "activations/layer22_attention_weight_max": 56.22986602783203,
      "activations/layer22_attention_weight_min": -31.24427604675293,
      "activations/layer23_attention_weight_max": 35.62314224243164,
      "activations/layer23_attention_weight_min": -22.965717315673828,
      "activations/layer2_attention_weight_max": 30.23367691040039,
      "activations/layer2_attention_weight_min": -30.138463973999023,
      "activations/layer3_attention_weight_max": 91.21370697021484,
      "activations/layer3_attention_weight_min": -90.09259796142578,
      "activations/layer4_attention_weight_max": 109.33221435546875,
      "activations/layer4_attention_weight_min": -111.42681121826172,
      "activations/layer5_attention_weight_max": 51.247413635253906,
      "activations/layer5_attention_weight_min": -57.46503448486328,
      "activations/layer6_attention_weight_max": 46.74003982543945,
      "activations/layer6_attention_weight_min": -47.3569450378418,
      "activations/layer7_attention_weight_max": 81.65684509277344,
      "activations/layer7_attention_weight_min": -84.36968231201172,
      "activations/layer8_attention_weight_max": 36.729400634765625,
      "activations/layer8_attention_weight_min": -38.55027389526367,
      "activations/layer9_attention_weight_max": 32.38595199584961,
      "activations/layer9_attention_weight_min": -31.852035522460938,
      "epoch": 8.99,
      "learning_rate": 9.29871212121212e-05,
      "loss": 2.8156,
      "step": 154650
    },
    {
      "activations/layer0_attention_weight_max": 15.272109031677246,
      "activations/layer0_attention_weight_min": -11.869434356689453,
      "activations/layer10_attention_weight_max": 29.956090927124023,
      "activations/layer10_attention_weight_min": -34.92595291137695,
      "activations/layer11_attention_weight_max": 29.538978576660156,
      "activations/layer11_attention_weight_min": -32.139198303222656,
      "activations/layer12_attention_weight_max": 20.370691299438477,
      "activations/layer12_attention_weight_min": -25.34415626525879,
      "activations/layer13_attention_weight_max": 34.03170394897461,
      "activations/layer13_attention_weight_min": -32.03139114379883,
      "activations/layer14_attention_weight_max": 42.22801208496094,
      "activations/layer14_attention_weight_min": -38.007415771484375,
      "activations/layer15_attention_weight_max": 32.8038330078125,
      "activations/layer15_attention_weight_min": -32.411041259765625,
      "activations/layer16_attention_weight_max": 28.087541580200195,
      "activations/layer16_attention_weight_min": -27.167325973510742,
      "activations/layer17_attention_weight_max": 31.70535659790039,
      "activations/layer17_attention_weight_min": -24.967086791992188,
      "activations/layer18_attention_weight_max": 28.4349308013916,
      "activations/layer18_attention_weight_min": -23.117753982543945,
      "activations/layer19_attention_weight_max": 34.027767181396484,
      "activations/layer19_attention_weight_min": -30.526020050048828,
      "activations/layer1_attention_weight_max": 16.170679092407227,
      "activations/layer1_attention_weight_min": -14.824851036071777,
      "activations/layer20_attention_weight_max": 32.115478515625,
      "activations/layer20_attention_weight_min": -23.643901824951172,
      "activations/layer21_attention_weight_max": 28.57672119140625,
      "activations/layer21_attention_weight_min": -22.65118408203125,
      "activations/layer22_attention_weight_max": 43.96598815917969,
      "activations/layer22_attention_weight_min": -29.363162994384766,
      "activations/layer23_attention_weight_max": 35.52922058105469,
      "activations/layer23_attention_weight_min": -23.37887954711914,
      "activations/layer2_attention_weight_max": 33.36389923095703,
      "activations/layer2_attention_weight_min": -34.278961181640625,
      "activations/layer3_attention_weight_max": 95.85087585449219,
      "activations/layer3_attention_weight_min": -98.17991638183594,
      "activations/layer4_attention_weight_max": 121.00933837890625,
      "activations/layer4_attention_weight_min": -122.56391906738281,
      "activations/layer5_attention_weight_max": 54.33003234863281,
      "activations/layer5_attention_weight_min": -61.37101364135742,
      "activations/layer6_attention_weight_max": 48.143245697021484,
      "activations/layer6_attention_weight_min": -49.173831939697266,
      "activations/layer7_attention_weight_max": 88.66548156738281,
      "activations/layer7_attention_weight_min": -94.77973937988281,
      "activations/layer8_attention_weight_max": 39.815223693847656,
      "activations/layer8_attention_weight_min": -41.10525894165039,
      "activations/layer9_attention_weight_max": 31.335418701171875,
      "activations/layer9_attention_weight_min": -33.43312454223633,
      "epoch": 8.99,
      "learning_rate": 9.29681818181818e-05,
      "loss": 2.8106,
      "step": 154700
    },
    {
      "activations/layer0_attention_weight_max": 16.614818572998047,
      "activations/layer0_attention_weight_min": -12.849895477294922,
      "activations/layer10_attention_weight_max": 35.150634765625,
      "activations/layer10_attention_weight_min": -31.64634895324707,
      "activations/layer11_attention_weight_max": 34.128787994384766,
      "activations/layer11_attention_weight_min": -30.69432830810547,
      "activations/layer12_attention_weight_max": 24.500850677490234,
      "activations/layer12_attention_weight_min": -27.409786224365234,
      "activations/layer13_attention_weight_max": 37.3962287902832,
      "activations/layer13_attention_weight_min": -34.585655212402344,
      "activations/layer14_attention_weight_max": 43.26810073852539,
      "activations/layer14_attention_weight_min": -38.51448440551758,
      "activations/layer15_attention_weight_max": 35.08238983154297,
      "activations/layer15_attention_weight_min": -31.278263092041016,
      "activations/layer16_attention_weight_max": 32.61383819580078,
      "activations/layer16_attention_weight_min": -27.999736785888672,
      "activations/layer17_attention_weight_max": 33.01801300048828,
      "activations/layer17_attention_weight_min": -24.764123916625977,
      "activations/layer18_attention_weight_max": 33.70512008666992,
      "activations/layer18_attention_weight_min": -20.99947738647461,
      "activations/layer19_attention_weight_max": 38.61518478393555,
      "activations/layer19_attention_weight_min": -29.790067672729492,
      "activations/layer1_attention_weight_max": 18.371047973632812,
      "activations/layer1_attention_weight_min": -14.325556755065918,
      "activations/layer20_attention_weight_max": 30.796268463134766,
      "activations/layer20_attention_weight_min": -25.032514572143555,
      "activations/layer21_attention_weight_max": 27.13767433166504,
      "activations/layer21_attention_weight_min": -23.054534912109375,
      "activations/layer22_attention_weight_max": 44.50553512573242,
      "activations/layer22_attention_weight_min": -28.671968460083008,
      "activations/layer23_attention_weight_max": 39.44084930419922,
      "activations/layer23_attention_weight_min": -24.63018035888672,
      "activations/layer2_attention_weight_max": 32.5936279296875,
      "activations/layer2_attention_weight_min": -32.55158233642578,
      "activations/layer3_attention_weight_max": 100.30166625976562,
      "activations/layer3_attention_weight_min": -99.3274917602539,
      "activations/layer4_attention_weight_max": 121.18607330322266,
      "activations/layer4_attention_weight_min": -117.39387512207031,
      "activations/layer5_attention_weight_max": 55.284847259521484,
      "activations/layer5_attention_weight_min": -62.685298919677734,
      "activations/layer6_attention_weight_max": 47.37311935424805,
      "activations/layer6_attention_weight_min": -51.225101470947266,
      "activations/layer7_attention_weight_max": 91.3865966796875,
      "activations/layer7_attention_weight_min": -89.35897064208984,
      "activations/layer8_attention_weight_max": 41.619014739990234,
      "activations/layer8_attention_weight_min": -41.147640228271484,
      "activations/layer9_attention_weight_max": 35.86587142944336,
      "activations/layer9_attention_weight_min": -34.30164337158203,
      "epoch": 8.99,
      "learning_rate": 9.294924242424242e-05,
      "loss": 2.8105,
      "step": 154750
    },
    {
      "activations/layer0_attention_weight_max": 14.951484680175781,
      "activations/layer0_attention_weight_min": -12.643635749816895,
      "activations/layer10_attention_weight_max": 40.28789138793945,
      "activations/layer10_attention_weight_min": -39.801185607910156,
      "activations/layer11_attention_weight_max": 38.371620178222656,
      "activations/layer11_attention_weight_min": -36.96299743652344,
      "activations/layer12_attention_weight_max": 27.552465438842773,
      "activations/layer12_attention_weight_min": -26.636796951293945,
      "activations/layer13_attention_weight_max": 49.620872497558594,
      "activations/layer13_attention_weight_min": -38.98188400268555,
      "activations/layer14_attention_weight_max": 67.10063934326172,
      "activations/layer14_attention_weight_min": -49.88423538208008,
      "activations/layer15_attention_weight_max": 42.25504684448242,
      "activations/layer15_attention_weight_min": -33.5588264465332,
      "activations/layer16_attention_weight_max": 39.99934768676758,
      "activations/layer16_attention_weight_min": -28.03052520751953,
      "activations/layer17_attention_weight_max": 38.44154739379883,
      "activations/layer17_attention_weight_min": -26.119535446166992,
      "activations/layer18_attention_weight_max": 34.89384841918945,
      "activations/layer18_attention_weight_min": -22.729999542236328,
      "activations/layer19_attention_weight_max": 47.994163513183594,
      "activations/layer19_attention_weight_min": -31.047409057617188,
      "activations/layer1_attention_weight_max": 17.10034942626953,
      "activations/layer1_attention_weight_min": -14.557066917419434,
      "activations/layer20_attention_weight_max": 36.00552749633789,
      "activations/layer20_attention_weight_min": -23.56971549987793,
      "activations/layer21_attention_weight_max": 32.96485137939453,
      "activations/layer21_attention_weight_min": -23.19439125061035,
      "activations/layer22_attention_weight_max": 60.68120193481445,
      "activations/layer22_attention_weight_min": -31.368623733520508,
      "activations/layer23_attention_weight_max": 45.197052001953125,
      "activations/layer23_attention_weight_min": -22.33529281616211,
      "activations/layer2_attention_weight_max": 34.38435363769531,
      "activations/layer2_attention_weight_min": -32.34678268432617,
      "activations/layer3_attention_weight_max": 97.43870544433594,
      "activations/layer3_attention_weight_min": -99.6122055053711,
      "activations/layer4_attention_weight_max": 124.69439697265625,
      "activations/layer4_attention_weight_min": -125.08528900146484,
      "activations/layer5_attention_weight_max": 57.04431915283203,
      "activations/layer5_attention_weight_min": -67.9307861328125,
      "activations/layer6_attention_weight_max": 50.270267486572266,
      "activations/layer6_attention_weight_min": -52.584468841552734,
      "activations/layer7_attention_weight_max": 98.74432373046875,
      "activations/layer7_attention_weight_min": -104.39974975585938,
      "activations/layer8_attention_weight_max": 47.93404006958008,
      "activations/layer8_attention_weight_min": -51.61507034301758,
      "activations/layer9_attention_weight_max": 42.33967208862305,
      "activations/layer9_attention_weight_min": -43.319332122802734,
      "epoch": 8.99,
      "learning_rate": 9.293030303030302e-05,
      "loss": 2.8089,
      "step": 154800
    },
    {
      "activations/layer0_attention_weight_max": 15.311689376831055,
      "activations/layer0_attention_weight_min": -11.833022117614746,
      "activations/layer10_attention_weight_max": 32.48499298095703,
      "activations/layer10_attention_weight_min": -31.75358009338379,
      "activations/layer11_attention_weight_max": 32.264713287353516,
      "activations/layer11_attention_weight_min": -30.186798095703125,
      "activations/layer12_attention_weight_max": 24.37550163269043,
      "activations/layer12_attention_weight_min": -23.124773025512695,
      "activations/layer13_attention_weight_max": 40.62087631225586,
      "activations/layer13_attention_weight_min": -32.975467681884766,
      "activations/layer14_attention_weight_max": 48.319068908691406,
      "activations/layer14_attention_weight_min": -39.54782485961914,
      "activations/layer15_attention_weight_max": 35.977142333984375,
      "activations/layer15_attention_weight_min": -30.95339584350586,
      "activations/layer16_attention_weight_max": 32.11957550048828,
      "activations/layer16_attention_weight_min": -26.668821334838867,
      "activations/layer17_attention_weight_max": 29.874094009399414,
      "activations/layer17_attention_weight_min": -23.86076545715332,
      "activations/layer18_attention_weight_max": 30.217138290405273,
      "activations/layer18_attention_weight_min": -23.439105987548828,
      "activations/layer19_attention_weight_max": 33.46338653564453,
      "activations/layer19_attention_weight_min": -31.466955184936523,
      "activations/layer1_attention_weight_max": 16.59490203857422,
      "activations/layer1_attention_weight_min": -14.832191467285156,
      "activations/layer20_attention_weight_max": 31.66836166381836,
      "activations/layer20_attention_weight_min": -23.472566604614258,
      "activations/layer21_attention_weight_max": 30.310287475585938,
      "activations/layer21_attention_weight_min": -24.1309757232666,
      "activations/layer22_attention_weight_max": 44.31243133544922,
      "activations/layer22_attention_weight_min": -29.95819091796875,
      "activations/layer23_attention_weight_max": 33.56047439575195,
      "activations/layer23_attention_weight_min": -23.886093139648438,
      "activations/layer2_attention_weight_max": 30.86885643005371,
      "activations/layer2_attention_weight_min": -32.6707649230957,
      "activations/layer3_attention_weight_max": 94.48516082763672,
      "activations/layer3_attention_weight_min": -100.39960479736328,
      "activations/layer4_attention_weight_max": 115.0364761352539,
      "activations/layer4_attention_weight_min": -118.27314758300781,
      "activations/layer5_attention_weight_max": 52.68385314941406,
      "activations/layer5_attention_weight_min": -60.56657409667969,
      "activations/layer6_attention_weight_max": 46.02137756347656,
      "activations/layer6_attention_weight_min": -49.60234451293945,
      "activations/layer7_attention_weight_max": 87.01488494873047,
      "activations/layer7_attention_weight_min": -83.77404022216797,
      "activations/layer8_attention_weight_max": 38.9879035949707,
      "activations/layer8_attention_weight_min": -40.186405181884766,
      "activations/layer9_attention_weight_max": 34.541255950927734,
      "activations/layer9_attention_weight_min": -31.650190353393555,
      "epoch": 9.0,
      "learning_rate": 9.291136363636363e-05,
      "loss": 2.8095,
      "step": 154850
    },
    {
      "activations/layer0_attention_weight_max": 14.892754554748535,
      "activations/layer0_attention_weight_min": -12.531030654907227,
      "activations/layer10_attention_weight_max": 34.22071075439453,
      "activations/layer10_attention_weight_min": -32.75981521606445,
      "activations/layer11_attention_weight_max": 33.20276641845703,
      "activations/layer11_attention_weight_min": -32.233680725097656,
      "activations/layer12_attention_weight_max": 24.029098510742188,
      "activations/layer12_attention_weight_min": -24.713558197021484,
      "activations/layer13_attention_weight_max": 40.86135482788086,
      "activations/layer13_attention_weight_min": -34.84062957763672,
      "activations/layer14_attention_weight_max": 52.9244384765625,
      "activations/layer14_attention_weight_min": -41.81536102294922,
      "activations/layer15_attention_weight_max": 37.558345794677734,
      "activations/layer15_attention_weight_min": -30.438302993774414,
      "activations/layer16_attention_weight_max": 33.530174255371094,
      "activations/layer16_attention_weight_min": -27.51940155029297,
      "activations/layer17_attention_weight_max": 32.62654113769531,
      "activations/layer17_attention_weight_min": -25.24650001525879,
      "activations/layer18_attention_weight_max": 32.98231887817383,
      "activations/layer18_attention_weight_min": -22.837528228759766,
      "activations/layer19_attention_weight_max": 38.007049560546875,
      "activations/layer19_attention_weight_min": -29.65787124633789,
      "activations/layer1_attention_weight_max": 16.640193939208984,
      "activations/layer1_attention_weight_min": -15.886247634887695,
      "activations/layer20_attention_weight_max": 31.12750244140625,
      "activations/layer20_attention_weight_min": -25.794384002685547,
      "activations/layer21_attention_weight_max": 28.61202621459961,
      "activations/layer21_attention_weight_min": -22.399354934692383,
      "activations/layer22_attention_weight_max": 47.75483703613281,
      "activations/layer22_attention_weight_min": -30.323654174804688,
      "activations/layer23_attention_weight_max": 37.420162200927734,
      "activations/layer23_attention_weight_min": -24.56563377380371,
      "activations/layer2_attention_weight_max": 32.53105163574219,
      "activations/layer2_attention_weight_min": -33.013885498046875,
      "activations/layer3_attention_weight_max": 94.2562484741211,
      "activations/layer3_attention_weight_min": -101.1720962524414,
      "activations/layer4_attention_weight_max": 120.58387756347656,
      "activations/layer4_attention_weight_min": -120.5506362915039,
      "activations/layer5_attention_weight_max": 53.09420394897461,
      "activations/layer5_attention_weight_min": -61.909828186035156,
      "activations/layer6_attention_weight_max": 46.632835388183594,
      "activations/layer6_attention_weight_min": -47.35689163208008,
      "activations/layer7_attention_weight_max": 88.61874389648438,
      "activations/layer7_attention_weight_min": -87.05448913574219,
      "activations/layer8_attention_weight_max": 40.47544860839844,
      "activations/layer8_attention_weight_min": -40.13225555419922,
      "activations/layer9_attention_weight_max": 35.45549774169922,
      "activations/layer9_attention_weight_min": -35.20543670654297,
      "epoch": 9.0,
      "learning_rate": 9.289242424242424e-05,
      "loss": 2.8421,
      "step": 154900
    },
    {
      "activations/layer0_attention_weight_max": 14.985188484191895,
      "activations/layer0_attention_weight_min": -12.415821075439453,
      "activations/layer10_attention_weight_max": 31.295879364013672,
      "activations/layer10_attention_weight_min": -30.92790412902832,
      "activations/layer11_attention_weight_max": 32.33300018310547,
      "activations/layer11_attention_weight_min": -29.931835174560547,
      "activations/layer12_attention_weight_max": 41.223419189453125,
      "activations/layer12_attention_weight_min": -24.10520362854004,
      "activations/layer13_attention_weight_max": 43.414371490478516,
      "activations/layer13_attention_weight_min": -32.255409240722656,
      "activations/layer14_attention_weight_max": 46.82072830200195,
      "activations/layer14_attention_weight_min": -36.16825866699219,
      "activations/layer15_attention_weight_max": 34.51087188720703,
      "activations/layer15_attention_weight_min": -29.193973541259766,
      "activations/layer16_attention_weight_max": 31.826993942260742,
      "activations/layer16_attention_weight_min": -28.3945255279541,
      "activations/layer17_attention_weight_max": 32.419044494628906,
      "activations/layer17_attention_weight_min": -24.740610122680664,
      "activations/layer18_attention_weight_max": 34.380489349365234,
      "activations/layer18_attention_weight_min": -21.12629508972168,
      "activations/layer19_attention_weight_max": 36.57444381713867,
      "activations/layer19_attention_weight_min": -29.5515079498291,
      "activations/layer1_attention_weight_max": 16.74294090270996,
      "activations/layer1_attention_weight_min": -13.816588401794434,
      "activations/layer20_attention_weight_max": 33.72615432739258,
      "activations/layer20_attention_weight_min": -24.276090621948242,
      "activations/layer21_attention_weight_max": 37.57644271850586,
      "activations/layer21_attention_weight_min": -23.311668395996094,
      "activations/layer22_attention_weight_max": 46.79509735107422,
      "activations/layer22_attention_weight_min": -28.139455795288086,
      "activations/layer23_attention_weight_max": 37.264793395996094,
      "activations/layer23_attention_weight_min": -25.90481948852539,
      "activations/layer2_attention_weight_max": 32.39120864868164,
      "activations/layer2_attention_weight_min": -31.69710350036621,
      "activations/layer3_attention_weight_max": 91.5162124633789,
      "activations/layer3_attention_weight_min": -92.60453796386719,
      "activations/layer4_attention_weight_max": 113.57498931884766,
      "activations/layer4_attention_weight_min": -117.76959228515625,
      "activations/layer5_attention_weight_max": 53.534236907958984,
      "activations/layer5_attention_weight_min": -62.508644104003906,
      "activations/layer6_attention_weight_max": 46.81633758544922,
      "activations/layer6_attention_weight_min": -50.01691818237305,
      "activations/layer7_attention_weight_max": 85.16487884521484,
      "activations/layer7_attention_weight_min": -85.87519836425781,
      "activations/layer8_attention_weight_max": 40.098018646240234,
      "activations/layer8_attention_weight_min": -39.50101852416992,
      "activations/layer9_attention_weight_max": 32.85390090942383,
      "activations/layer9_attention_weight_min": -32.08948516845703,
      "epoch": 9.0,
      "learning_rate": 9.287348484848484e-05,
      "loss": 2.7805,
      "step": 154950
    },
    {
      "activations/layer0_attention_weight_max": 15.696840286254883,
      "activations/layer0_attention_weight_min": -12.90777587890625,
      "activations/layer10_attention_weight_max": 33.72433090209961,
      "activations/layer10_attention_weight_min": -37.64656448364258,
      "activations/layer11_attention_weight_max": 32.36220169067383,
      "activations/layer11_attention_weight_min": -35.548057556152344,
      "activations/layer12_attention_weight_max": 25.71261215209961,
      "activations/layer12_attention_weight_min": -26.01627540588379,
      "activations/layer13_attention_weight_max": 42.536094665527344,
      "activations/layer13_attention_weight_min": -34.020530700683594,
      "activations/layer14_attention_weight_max": 45.09597396850586,
      "activations/layer14_attention_weight_min": -40.615272521972656,
      "activations/layer15_attention_weight_max": 35.7048225402832,
      "activations/layer15_attention_weight_min": -33.17511749267578,
      "activations/layer16_attention_weight_max": 32.48805236816406,
      "activations/layer16_attention_weight_min": -27.508590698242188,
      "activations/layer17_attention_weight_max": 31.278146743774414,
      "activations/layer17_attention_weight_min": -24.927560806274414,
      "activations/layer18_attention_weight_max": 28.636802673339844,
      "activations/layer18_attention_weight_min": -22.25263023376465,
      "activations/layer19_attention_weight_max": 37.53640365600586,
      "activations/layer19_attention_weight_min": -31.86152458190918,
      "activations/layer1_attention_weight_max": 17.977216720581055,
      "activations/layer1_attention_weight_min": -16.802949905395508,
      "activations/layer20_attention_weight_max": 31.15960121154785,
      "activations/layer20_attention_weight_min": -26.11705207824707,
      "activations/layer21_attention_weight_max": 30.95583152770996,
      "activations/layer21_attention_weight_min": -24.33915901184082,
      "activations/layer22_attention_weight_max": 46.31330108642578,
      "activations/layer22_attention_weight_min": -30.763628005981445,
      "activations/layer23_attention_weight_max": 36.549434661865234,
      "activations/layer23_attention_weight_min": -25.674518585205078,
      "activations/layer2_attention_weight_max": 31.527273178100586,
      "activations/layer2_attention_weight_min": -33.21681594848633,
      "activations/layer3_attention_weight_max": 93.4909896850586,
      "activations/layer3_attention_weight_min": -100.45429229736328,
      "activations/layer4_attention_weight_max": 117.25677490234375,
      "activations/layer4_attention_weight_min": -118.910888671875,
      "activations/layer5_attention_weight_max": 52.96412658691406,
      "activations/layer5_attention_weight_min": -62.68115234375,
      "activations/layer6_attention_weight_max": 47.452884674072266,
      "activations/layer6_attention_weight_min": -51.49713897705078,
      "activations/layer7_attention_weight_max": 89.37139892578125,
      "activations/layer7_attention_weight_min": -92.68155670166016,
      "activations/layer8_attention_weight_max": 39.829872131347656,
      "activations/layer8_attention_weight_min": -42.9694709777832,
      "activations/layer9_attention_weight_max": 34.3393440246582,
      "activations/layer9_attention_weight_min": -36.35774612426758,
      "epoch": 9.01,
      "learning_rate": 9.285454545454545e-05,
      "loss": 2.7967,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4632,
      "eval_samples_per_second": 507.376,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4632,
      "eval_openwebtext_samples_per_second": 507.376,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 1.9851,
      "eval_wikitext_samples_per_second": 229.714,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_lambada_loss": 2.5546875,
      "eval_lambada_ppl": 12.867278004795136,
      "eval_lambada_runtime": 9.5728,
      "eval_lambada_samples_per_second": 508.628,
      "step": 155000
    },
    {
      "activations/layer0_attention_weight_max": 15.57599925994873,
      "activations/layer0_attention_weight_min": -12.264303207397461,
      "activations/layer10_attention_weight_max": 31.10556983947754,
      "activations/layer10_attention_weight_min": -30.22859764099121,
      "activations/layer11_attention_weight_max": 30.507850646972656,
      "activations/layer11_attention_weight_min": -29.976158142089844,
      "activations/layer12_attention_weight_max": 23.33111572265625,
      "activations/layer12_attention_weight_min": -24.37837028503418,
      "activations/layer13_attention_weight_max": 37.556434631347656,
      "activations/layer13_attention_weight_min": -34.534454345703125,
      "activations/layer14_attention_weight_max": 46.290550231933594,
      "activations/layer14_attention_weight_min": -43.09751892089844,
      "activations/layer15_attention_weight_max": 32.43445587158203,
      "activations/layer15_attention_weight_min": -30.04180335998535,
      "activations/layer16_attention_weight_max": 28.04354476928711,
      "activations/layer16_attention_weight_min": -29.307111740112305,
      "activations/layer17_attention_weight_max": 32.6179084777832,
      "activations/layer17_attention_weight_min": -27.332481384277344,
      "activations/layer18_attention_weight_max": 32.573246002197266,
      "activations/layer18_attention_weight_min": -23.017841339111328,
      "activations/layer19_attention_weight_max": 34.675289154052734,
      "activations/layer19_attention_weight_min": -31.6392879486084,
      "activations/layer1_attention_weight_max": 16.592432022094727,
      "activations/layer1_attention_weight_min": -15.111101150512695,
      "activations/layer20_attention_weight_max": 29.377784729003906,
      "activations/layer20_attention_weight_min": -23.58769416809082,
      "activations/layer21_attention_weight_max": 30.760618209838867,
      "activations/layer21_attention_weight_min": -23.556997299194336,
      "activations/layer22_attention_weight_max": 41.17781066894531,
      "activations/layer22_attention_weight_min": -29.369667053222656,
      "activations/layer23_attention_weight_max": 33.67952346801758,
      "activations/layer23_attention_weight_min": -25.117996215820312,
      "activations/layer2_attention_weight_max": 32.1058464050293,
      "activations/layer2_attention_weight_min": -31.085140228271484,
      "activations/layer3_attention_weight_max": 94.23735809326172,
      "activations/layer3_attention_weight_min": -93.27413177490234,
      "activations/layer4_attention_weight_max": 119.66438293457031,
      "activations/layer4_attention_weight_min": -110.7978286743164,
      "activations/layer5_attention_weight_max": 53.92271041870117,
      "activations/layer5_attention_weight_min": -60.15251922607422,
      "activations/layer6_attention_weight_max": 47.08525466918945,
      "activations/layer6_attention_weight_min": -48.5576057434082,
      "activations/layer7_attention_weight_max": 86.62247467041016,
      "activations/layer7_attention_weight_min": -85.762451171875,
      "activations/layer8_attention_weight_max": 38.13566589355469,
      "activations/layer8_attention_weight_min": -39.16316604614258,
      "activations/layer9_attention_weight_max": 31.080434799194336,
      "activations/layer9_attention_weight_min": -32.86395263671875,
      "epoch": 9.01,
      "learning_rate": 9.283560606060607e-05,
      "loss": 2.7854,
      "step": 155050
    },
    {
      "activations/layer0_attention_weight_max": 15.748200416564941,
      "activations/layer0_attention_weight_min": -12.753560066223145,
      "activations/layer10_attention_weight_max": 34.358673095703125,
      "activations/layer10_attention_weight_min": -35.481201171875,
      "activations/layer11_attention_weight_max": 32.9874153137207,
      "activations/layer11_attention_weight_min": -33.58008575439453,
      "activations/layer12_attention_weight_max": 34.617740631103516,
      "activations/layer12_attention_weight_min": -24.84980583190918,
      "activations/layer13_attention_weight_max": 41.6265983581543,
      "activations/layer13_attention_weight_min": -32.452484130859375,
      "activations/layer14_attention_weight_max": 39.27103805541992,
      "activations/layer14_attention_weight_min": -34.902259826660156,
      "activations/layer15_attention_weight_max": 36.750282287597656,
      "activations/layer15_attention_weight_min": -29.811634063720703,
      "activations/layer16_attention_weight_max": 30.287378311157227,
      "activations/layer16_attention_weight_min": -27.086408615112305,
      "activations/layer17_attention_weight_max": 31.12247085571289,
      "activations/layer17_attention_weight_min": -25.70978355407715,
      "activations/layer18_attention_weight_max": 29.526460647583008,
      "activations/layer18_attention_weight_min": -22.364212036132812,
      "activations/layer19_attention_weight_max": 29.54850196838379,
      "activations/layer19_attention_weight_min": -30.4583683013916,
      "activations/layer1_attention_weight_max": 16.716983795166016,
      "activations/layer1_attention_weight_min": -14.03252124786377,
      "activations/layer20_attention_weight_max": 27.128978729248047,
      "activations/layer20_attention_weight_min": -22.84221076965332,
      "activations/layer21_attention_weight_max": 25.12289810180664,
      "activations/layer21_attention_weight_min": -23.334447860717773,
      "activations/layer22_attention_weight_max": 40.66851806640625,
      "activations/layer22_attention_weight_min": -32.58122634887695,
      "activations/layer23_attention_weight_max": 33.05377960205078,
      "activations/layer23_attention_weight_min": -27.228973388671875,
      "activations/layer2_attention_weight_max": 30.751911163330078,
      "activations/layer2_attention_weight_min": -33.70294189453125,
      "activations/layer3_attention_weight_max": 93.90409088134766,
      "activations/layer3_attention_weight_min": -96.17228698730469,
      "activations/layer4_attention_weight_max": 116.91029357910156,
      "activations/layer4_attention_weight_min": -116.92372131347656,
      "activations/layer5_attention_weight_max": 52.538291931152344,
      "activations/layer5_attention_weight_min": -60.319190979003906,
      "activations/layer6_attention_weight_max": 45.23148727416992,
      "activations/layer6_attention_weight_min": -46.917049407958984,
      "activations/layer7_attention_weight_max": 87.28483581542969,
      "activations/layer7_attention_weight_min": -91.81190490722656,
      "activations/layer8_attention_weight_max": 39.85152053833008,
      "activations/layer8_attention_weight_min": -44.57815933227539,
      "activations/layer9_attention_weight_max": 33.97627639770508,
      "activations/layer9_attention_weight_min": -36.820064544677734,
      "epoch": 9.01,
      "learning_rate": 9.281666666666665e-05,
      "loss": 2.7888,
      "step": 155100
    },
    {
      "activations/layer0_attention_weight_max": 15.244537353515625,
      "activations/layer0_attention_weight_min": -11.293599128723145,
      "activations/layer10_attention_weight_max": 32.590782165527344,
      "activations/layer10_attention_weight_min": -30.74802017211914,
      "activations/layer11_attention_weight_max": 35.29700469970703,
      "activations/layer11_attention_weight_min": -30.30797576904297,
      "activations/layer12_attention_weight_max": 24.09067153930664,
      "activations/layer12_attention_weight_min": -25.0288028717041,
      "activations/layer13_attention_weight_max": 36.5081901550293,
      "activations/layer13_attention_weight_min": -32.63796615600586,
      "activations/layer14_attention_weight_max": 46.93965148925781,
      "activations/layer14_attention_weight_min": -41.397056579589844,
      "activations/layer15_attention_weight_max": 34.416282653808594,
      "activations/layer15_attention_weight_min": -29.700719833374023,
      "activations/layer16_attention_weight_max": 30.657726287841797,
      "activations/layer16_attention_weight_min": -28.96523094177246,
      "activations/layer17_attention_weight_max": 36.5815315246582,
      "activations/layer17_attention_weight_min": -25.38585090637207,
      "activations/layer18_attention_weight_max": 28.488481521606445,
      "activations/layer18_attention_weight_min": -20.685068130493164,
      "activations/layer19_attention_weight_max": 33.10159683227539,
      "activations/layer19_attention_weight_min": -29.750778198242188,
      "activations/layer1_attention_weight_max": 18.576833724975586,
      "activations/layer1_attention_weight_min": -14.667497634887695,
      "activations/layer20_attention_weight_max": 27.617353439331055,
      "activations/layer20_attention_weight_min": -23.522628784179688,
      "activations/layer21_attention_weight_max": 26.101524353027344,
      "activations/layer21_attention_weight_min": -22.033672332763672,
      "activations/layer22_attention_weight_max": 42.86563491821289,
      "activations/layer22_attention_weight_min": -28.392108917236328,
      "activations/layer23_attention_weight_max": 35.63941955566406,
      "activations/layer23_attention_weight_min": -22.0177059173584,
      "activations/layer2_attention_weight_max": 33.36518859863281,
      "activations/layer2_attention_weight_min": -33.183685302734375,
      "activations/layer3_attention_weight_max": 95.2310791015625,
      "activations/layer3_attention_weight_min": -96.88165283203125,
      "activations/layer4_attention_weight_max": 116.43780517578125,
      "activations/layer4_attention_weight_min": -115.887451171875,
      "activations/layer5_attention_weight_max": 53.239315032958984,
      "activations/layer5_attention_weight_min": -62.61583709716797,
      "activations/layer6_attention_weight_max": 46.13504409790039,
      "activations/layer6_attention_weight_min": -51.20054626464844,
      "activations/layer7_attention_weight_max": 88.32312774658203,
      "activations/layer7_attention_weight_min": -87.89885711669922,
      "activations/layer8_attention_weight_max": 40.65812301635742,
      "activations/layer8_attention_weight_min": -40.33552932739258,
      "activations/layer9_attention_weight_max": 34.27388000488281,
      "activations/layer9_attention_weight_min": -33.671627044677734,
      "epoch": 9.02,
      "learning_rate": 9.279772727272727e-05,
      "loss": 2.7968,
      "step": 155150
    },
    {
      "activations/layer0_attention_weight_max": 16.36142349243164,
      "activations/layer0_attention_weight_min": -11.582619667053223,
      "activations/layer10_attention_weight_max": 32.181915283203125,
      "activations/layer10_attention_weight_min": -33.22725296020508,
      "activations/layer11_attention_weight_max": 30.90926170349121,
      "activations/layer11_attention_weight_min": -31.627544403076172,
      "activations/layer12_attention_weight_max": 24.69497299194336,
      "activations/layer12_attention_weight_min": -23.287128448486328,
      "activations/layer13_attention_weight_max": 41.02412033081055,
      "activations/layer13_attention_weight_min": -33.15324783325195,
      "activations/layer14_attention_weight_max": 45.50300598144531,
      "activations/layer14_attention_weight_min": -35.156822204589844,
      "activations/layer15_attention_weight_max": 38.05976867675781,
      "activations/layer15_attention_weight_min": -29.804447174072266,
      "activations/layer16_attention_weight_max": 33.9172477722168,
      "activations/layer16_attention_weight_min": -27.737518310546875,
      "activations/layer17_attention_weight_max": 36.33065414428711,
      "activations/layer17_attention_weight_min": -26.220123291015625,
      "activations/layer18_attention_weight_max": 33.3997917175293,
      "activations/layer18_attention_weight_min": -23.114513397216797,
      "activations/layer19_attention_weight_max": 33.49971389770508,
      "activations/layer19_attention_weight_min": -31.962726593017578,
      "activations/layer1_attention_weight_max": 16.09213638305664,
      "activations/layer1_attention_weight_min": -14.71396255493164,
      "activations/layer20_attention_weight_max": 30.655948638916016,
      "activations/layer20_attention_weight_min": -24.342735290527344,
      "activations/layer21_attention_weight_max": 26.932071685791016,
      "activations/layer21_attention_weight_min": -22.460643768310547,
      "activations/layer22_attention_weight_max": 44.875770568847656,
      "activations/layer22_attention_weight_min": -30.800806045532227,
      "activations/layer23_attention_weight_max": 35.46717071533203,
      "activations/layer23_attention_weight_min": -25.257095336914062,
      "activations/layer2_attention_weight_max": 32.26762390136719,
      "activations/layer2_attention_weight_min": -33.66456985473633,
      "activations/layer3_attention_weight_max": 95.21854400634766,
      "activations/layer3_attention_weight_min": -100.0624008178711,
      "activations/layer4_attention_weight_max": 117.49837493896484,
      "activations/layer4_attention_weight_min": -116.76690673828125,
      "activations/layer5_attention_weight_max": 51.67722702026367,
      "activations/layer5_attention_weight_min": -61.228675842285156,
      "activations/layer6_attention_weight_max": 46.942115783691406,
      "activations/layer6_attention_weight_min": -50.55107879638672,
      "activations/layer7_attention_weight_max": 89.04964447021484,
      "activations/layer7_attention_weight_min": -86.67208099365234,
      "activations/layer8_attention_weight_max": 38.49020004272461,
      "activations/layer8_attention_weight_min": -38.444923400878906,
      "activations/layer9_attention_weight_max": 32.77720642089844,
      "activations/layer9_attention_weight_min": -32.99177932739258,
      "epoch": 9.02,
      "learning_rate": 9.277878787878786e-05,
      "loss": 2.7847,
      "step": 155200
    },
    {
      "activations/layer0_attention_weight_max": 15.471200942993164,
      "activations/layer0_attention_weight_min": -11.599101066589355,
      "activations/layer10_attention_weight_max": 31.63375473022461,
      "activations/layer10_attention_weight_min": -33.162147521972656,
      "activations/layer11_attention_weight_max": 31.161659240722656,
      "activations/layer11_attention_weight_min": -32.27267837524414,
      "activations/layer12_attention_weight_max": 25.241413116455078,
      "activations/layer12_attention_weight_min": -27.187231063842773,
      "activations/layer13_attention_weight_max": 45.74625015258789,
      "activations/layer13_attention_weight_min": -34.93986892700195,
      "activations/layer14_attention_weight_max": 50.18875503540039,
      "activations/layer14_attention_weight_min": -40.49431610107422,
      "activations/layer15_attention_weight_max": 37.95745086669922,
      "activations/layer15_attention_weight_min": -31.86371612548828,
      "activations/layer16_attention_weight_max": 33.812599182128906,
      "activations/layer16_attention_weight_min": -28.470989227294922,
      "activations/layer17_attention_weight_max": 35.62112808227539,
      "activations/layer17_attention_weight_min": -25.433021545410156,
      "activations/layer18_attention_weight_max": 36.72885513305664,
      "activations/layer18_attention_weight_min": -21.301733016967773,
      "activations/layer19_attention_weight_max": 38.611122131347656,
      "activations/layer19_attention_weight_min": -28.4848690032959,
      "activations/layer1_attention_weight_max": 17.275583267211914,
      "activations/layer1_attention_weight_min": -14.24234676361084,
      "activations/layer20_attention_weight_max": 31.288963317871094,
      "activations/layer20_attention_weight_min": -23.936521530151367,
      "activations/layer21_attention_weight_max": 32.521514892578125,
      "activations/layer21_attention_weight_min": -21.831424713134766,
      "activations/layer22_attention_weight_max": 50.58232498168945,
      "activations/layer22_attention_weight_min": -28.202919006347656,
      "activations/layer23_attention_weight_max": 40.71796417236328,
      "activations/layer23_attention_weight_min": -23.471914291381836,
      "activations/layer2_attention_weight_max": 31.76811981201172,
      "activations/layer2_attention_weight_min": -32.84077072143555,
      "activations/layer3_attention_weight_max": 91.57905578613281,
      "activations/layer3_attention_weight_min": -96.39514923095703,
      "activations/layer4_attention_weight_max": 118.80004119873047,
      "activations/layer4_attention_weight_min": -120.1019287109375,
      "activations/layer5_attention_weight_max": 54.64823532104492,
      "activations/layer5_attention_weight_min": -60.489742279052734,
      "activations/layer6_attention_weight_max": 46.642738342285156,
      "activations/layer6_attention_weight_min": -52.088260650634766,
      "activations/layer7_attention_weight_max": 97.01905059814453,
      "activations/layer7_attention_weight_min": -91.11791229248047,
      "activations/layer8_attention_weight_max": 38.014930725097656,
      "activations/layer8_attention_weight_min": -39.64607620239258,
      "activations/layer9_attention_weight_max": 34.45621871948242,
      "activations/layer9_attention_weight_min": -33.874603271484375,
      "epoch": 9.02,
      "learning_rate": 9.275984848484847e-05,
      "loss": 2.7927,
      "step": 155250
    },
    {
      "activations/layer0_attention_weight_max": 15.276651382446289,
      "activations/layer0_attention_weight_min": -11.70821475982666,
      "activations/layer10_attention_weight_max": 35.655242919921875,
      "activations/layer10_attention_weight_min": -33.850093841552734,
      "activations/layer11_attention_weight_max": 31.855161666870117,
      "activations/layer11_attention_weight_min": -31.514766693115234,
      "activations/layer12_attention_weight_max": 25.18950843811035,
      "activations/layer12_attention_weight_min": -25.15400505065918,
      "activations/layer13_attention_weight_max": 40.64443588256836,
      "activations/layer13_attention_weight_min": -37.00627136230469,
      "activations/layer14_attention_weight_max": 53.73131561279297,
      "activations/layer14_attention_weight_min": -41.62274932861328,
      "activations/layer15_attention_weight_max": 37.80422592163086,
      "activations/layer15_attention_weight_min": -32.424964904785156,
      "activations/layer16_attention_weight_max": 32.09817886352539,
      "activations/layer16_attention_weight_min": -29.39783477783203,
      "activations/layer17_attention_weight_max": 33.604061126708984,
      "activations/layer17_attention_weight_min": -26.188953399658203,
      "activations/layer18_attention_weight_max": 36.99042510986328,
      "activations/layer18_attention_weight_min": -22.369808197021484,
      "activations/layer19_attention_weight_max": 36.11975860595703,
      "activations/layer19_attention_weight_min": -30.318754196166992,
      "activations/layer1_attention_weight_max": 16.308313369750977,
      "activations/layer1_attention_weight_min": -14.214688301086426,
      "activations/layer20_attention_weight_max": 30.705930709838867,
      "activations/layer20_attention_weight_min": -24.61770248413086,
      "activations/layer21_attention_weight_max": 30.815494537353516,
      "activations/layer21_attention_weight_min": -23.37096405029297,
      "activations/layer22_attention_weight_max": 49.520870208740234,
      "activations/layer22_attention_weight_min": -31.92812728881836,
      "activations/layer23_attention_weight_max": 38.28314971923828,
      "activations/layer23_attention_weight_min": -27.521793365478516,
      "activations/layer2_attention_weight_max": 32.940155029296875,
      "activations/layer2_attention_weight_min": -33.730369567871094,
      "activations/layer3_attention_weight_max": 96.50038146972656,
      "activations/layer3_attention_weight_min": -96.46463775634766,
      "activations/layer4_attention_weight_max": 121.32634735107422,
      "activations/layer4_attention_weight_min": -123.8249740600586,
      "activations/layer5_attention_weight_max": 54.063438415527344,
      "activations/layer5_attention_weight_min": -67.21739196777344,
      "activations/layer6_attention_weight_max": 47.85399627685547,
      "activations/layer6_attention_weight_min": -53.910099029541016,
      "activations/layer7_attention_weight_max": 96.23392486572266,
      "activations/layer7_attention_weight_min": -93.31065368652344,
      "activations/layer8_attention_weight_max": 43.648475646972656,
      "activations/layer8_attention_weight_min": -44.29753112792969,
      "activations/layer9_attention_weight_max": 36.81397247314453,
      "activations/layer9_attention_weight_min": -36.28878402709961,
      "epoch": 9.02,
      "learning_rate": 9.274090909090909e-05,
      "loss": 2.8127,
      "step": 155300
    },
    {
      "activations/layer0_attention_weight_max": 15.273168563842773,
      "activations/layer0_attention_weight_min": -11.54034423828125,
      "activations/layer10_attention_weight_max": 30.72705078125,
      "activations/layer10_attention_weight_min": -31.567039489746094,
      "activations/layer11_attention_weight_max": 29.338546752929688,
      "activations/layer11_attention_weight_min": -29.811168670654297,
      "activations/layer12_attention_weight_max": 22.701642990112305,
      "activations/layer12_attention_weight_min": -35.69447708129883,
      "activations/layer13_attention_weight_max": 35.30735397338867,
      "activations/layer13_attention_weight_min": -34.35565948486328,
      "activations/layer14_attention_weight_max": 48.27030563354492,
      "activations/layer14_attention_weight_min": -42.60335922241211,
      "activations/layer15_attention_weight_max": 34.62443542480469,
      "activations/layer15_attention_weight_min": -29.01047706604004,
      "activations/layer16_attention_weight_max": 36.14268493652344,
      "activations/layer16_attention_weight_min": -28.459232330322266,
      "activations/layer17_attention_weight_max": 36.3721923828125,
      "activations/layer17_attention_weight_min": -25.162185668945312,
      "activations/layer18_attention_weight_max": 30.8765811920166,
      "activations/layer18_attention_weight_min": -21.434619903564453,
      "activations/layer19_attention_weight_max": 39.929744720458984,
      "activations/layer19_attention_weight_min": -30.731281280517578,
      "activations/layer1_attention_weight_max": 17.820091247558594,
      "activations/layer1_attention_weight_min": -13.905580520629883,
      "activations/layer20_attention_weight_max": 34.749385833740234,
      "activations/layer20_attention_weight_min": -25.58063316345215,
      "activations/layer21_attention_weight_max": 31.5855712890625,
      "activations/layer21_attention_weight_min": -23.86760139465332,
      "activations/layer22_attention_weight_max": 50.31256103515625,
      "activations/layer22_attention_weight_min": -29.3387508392334,
      "activations/layer23_attention_weight_max": 39.955604553222656,
      "activations/layer23_attention_weight_min": -23.04511260986328,
      "activations/layer2_attention_weight_max": 34.66999435424805,
      "activations/layer2_attention_weight_min": -31.966434478759766,
      "activations/layer3_attention_weight_max": 101.80197143554688,
      "activations/layer3_attention_weight_min": -93.50440979003906,
      "activations/layer4_attention_weight_max": 120.8956527709961,
      "activations/layer4_attention_weight_min": -114.84461975097656,
      "activations/layer5_attention_weight_max": 53.289947509765625,
      "activations/layer5_attention_weight_min": -63.14683532714844,
      "activations/layer6_attention_weight_max": 45.63882827758789,
      "activations/layer6_attention_weight_min": -49.19636535644531,
      "activations/layer7_attention_weight_max": 85.69204711914062,
      "activations/layer7_attention_weight_min": -88.1893310546875,
      "activations/layer8_attention_weight_max": 39.01670455932617,
      "activations/layer8_attention_weight_min": -39.7969970703125,
      "activations/layer9_attention_weight_max": 33.383419036865234,
      "activations/layer9_attention_weight_min": -33.34918975830078,
      "epoch": 9.03,
      "learning_rate": 9.272196969696969e-05,
      "loss": 2.8066,
      "step": 155350
    },
    {
      "activations/layer0_attention_weight_max": 15.500043869018555,
      "activations/layer0_attention_weight_min": -12.417591094970703,
      "activations/layer10_attention_weight_max": 35.353363037109375,
      "activations/layer10_attention_weight_min": -32.16162109375,
      "activations/layer11_attention_weight_max": 35.63523864746094,
      "activations/layer11_attention_weight_min": -31.795743942260742,
      "activations/layer12_attention_weight_max": 23.74308967590332,
      "activations/layer12_attention_weight_min": -26.733293533325195,
      "activations/layer13_attention_weight_max": 39.67653274536133,
      "activations/layer13_attention_weight_min": -33.52589416503906,
      "activations/layer14_attention_weight_max": 45.76712417602539,
      "activations/layer14_attention_weight_min": -38.800315856933594,
      "activations/layer15_attention_weight_max": 37.277610778808594,
      "activations/layer15_attention_weight_min": -31.127168655395508,
      "activations/layer16_attention_weight_max": 33.19884490966797,
      "activations/layer16_attention_weight_min": -27.380008697509766,
      "activations/layer17_attention_weight_max": 34.6955680847168,
      "activations/layer17_attention_weight_min": -25.289308547973633,
      "activations/layer18_attention_weight_max": 32.4217643737793,
      "activations/layer18_attention_weight_min": -23.697858810424805,
      "activations/layer19_attention_weight_max": 33.532901763916016,
      "activations/layer19_attention_weight_min": -30.867685317993164,
      "activations/layer1_attention_weight_max": 16.270259857177734,
      "activations/layer1_attention_weight_min": -14.854167938232422,
      "activations/layer20_attention_weight_max": 29.620649337768555,
      "activations/layer20_attention_weight_min": -25.428112030029297,
      "activations/layer21_attention_weight_max": 29.89153289794922,
      "activations/layer21_attention_weight_min": -23.324697494506836,
      "activations/layer22_attention_weight_max": 50.26390075683594,
      "activations/layer22_attention_weight_min": -28.274215698242188,
      "activations/layer23_attention_weight_max": 36.176353454589844,
      "activations/layer23_attention_weight_min": -26.14694595336914,
      "activations/layer2_attention_weight_max": 32.541927337646484,
      "activations/layer2_attention_weight_min": -33.84539031982422,
      "activations/layer3_attention_weight_max": 99.42481994628906,
      "activations/layer3_attention_weight_min": -100.80744934082031,
      "activations/layer4_attention_weight_max": 116.5305404663086,
      "activations/layer4_attention_weight_min": -117.69933319091797,
      "activations/layer5_attention_weight_max": 54.68785095214844,
      "activations/layer5_attention_weight_min": -60.64598083496094,
      "activations/layer6_attention_weight_max": 47.69785690307617,
      "activations/layer6_attention_weight_min": -50.05351638793945,
      "activations/layer7_attention_weight_max": 90.17912292480469,
      "activations/layer7_attention_weight_min": -87.29026794433594,
      "activations/layer8_attention_weight_max": 41.7548713684082,
      "activations/layer8_attention_weight_min": -40.4721565246582,
      "activations/layer9_attention_weight_max": 38.81324005126953,
      "activations/layer9_attention_weight_min": -32.68268585205078,
      "epoch": 9.03,
      "learning_rate": 9.270303030303029e-05,
      "loss": 2.7936,
      "step": 155400
    },
    {
      "activations/layer0_attention_weight_max": 16.72553062438965,
      "activations/layer0_attention_weight_min": -12.571587562561035,
      "activations/layer10_attention_weight_max": 34.33042907714844,
      "activations/layer10_attention_weight_min": -32.75863265991211,
      "activations/layer11_attention_weight_max": 33.937416076660156,
      "activations/layer11_attention_weight_min": -31.909812927246094,
      "activations/layer12_attention_weight_max": 24.083740234375,
      "activations/layer12_attention_weight_min": -30.914220809936523,
      "activations/layer13_attention_weight_max": 40.85755920410156,
      "activations/layer13_attention_weight_min": -33.86405563354492,
      "activations/layer14_attention_weight_max": 49.773075103759766,
      "activations/layer14_attention_weight_min": -39.87663650512695,
      "activations/layer15_attention_weight_max": 38.54738998413086,
      "activations/layer15_attention_weight_min": -32.5185661315918,
      "activations/layer16_attention_weight_max": 34.266353607177734,
      "activations/layer16_attention_weight_min": -29.76439094543457,
      "activations/layer17_attention_weight_max": 35.40298080444336,
      "activations/layer17_attention_weight_min": -26.213415145874023,
      "activations/layer18_attention_weight_max": 30.717975616455078,
      "activations/layer18_attention_weight_min": -21.43620491027832,
      "activations/layer19_attention_weight_max": 40.6608772277832,
      "activations/layer19_attention_weight_min": -30.152313232421875,
      "activations/layer1_attention_weight_max": 16.371049880981445,
      "activations/layer1_attention_weight_min": -14.447467803955078,
      "activations/layer20_attention_weight_max": 39.45023727416992,
      "activations/layer20_attention_weight_min": -22.7989559173584,
      "activations/layer21_attention_weight_max": 30.48471450805664,
      "activations/layer21_attention_weight_min": -22.07254409790039,
      "activations/layer22_attention_weight_max": 51.30295181274414,
      "activations/layer22_attention_weight_min": -28.865320205688477,
      "activations/layer23_attention_weight_max": 38.54419708251953,
      "activations/layer23_attention_weight_min": -22.445377349853516,
      "activations/layer2_attention_weight_max": 32.99608612060547,
      "activations/layer2_attention_weight_min": -33.22650909423828,
      "activations/layer3_attention_weight_max": 96.28973388671875,
      "activations/layer3_attention_weight_min": -98.0281753540039,
      "activations/layer4_attention_weight_max": 119.07757568359375,
      "activations/layer4_attention_weight_min": -118.05924224853516,
      "activations/layer5_attention_weight_max": 55.76866149902344,
      "activations/layer5_attention_weight_min": -60.24835968017578,
      "activations/layer6_attention_weight_max": 50.178199768066406,
      "activations/layer6_attention_weight_min": -51.8057975769043,
      "activations/layer7_attention_weight_max": 94.17019653320312,
      "activations/layer7_attention_weight_min": -95.68605041503906,
      "activations/layer8_attention_weight_max": 41.89344787597656,
      "activations/layer8_attention_weight_min": -43.46552658081055,
      "activations/layer9_attention_weight_max": 38.16975021362305,
      "activations/layer9_attention_weight_min": -35.201080322265625,
      "epoch": 9.03,
      "learning_rate": 9.26840909090909e-05,
      "loss": 2.7936,
      "step": 155450
    },
    {
      "activations/layer0_attention_weight_max": 15.47479248046875,
      "activations/layer0_attention_weight_min": -11.976158142089844,
      "activations/layer10_attention_weight_max": 30.73828125,
      "activations/layer10_attention_weight_min": -29.8205623626709,
      "activations/layer11_attention_weight_max": 30.505046844482422,
      "activations/layer11_attention_weight_min": -28.98366355895996,
      "activations/layer12_attention_weight_max": 23.20598602294922,
      "activations/layer12_attention_weight_min": -25.61638641357422,
      "activations/layer13_attention_weight_max": 36.962459564208984,
      "activations/layer13_attention_weight_min": -34.16691207885742,
      "activations/layer14_attention_weight_max": 44.60356521606445,
      "activations/layer14_attention_weight_min": -42.37299346923828,
      "activations/layer15_attention_weight_max": 34.958370208740234,
      "activations/layer15_attention_weight_min": -30.82819938659668,
      "activations/layer16_attention_weight_max": 33.532569885253906,
      "activations/layer16_attention_weight_min": -28.987478256225586,
      "activations/layer17_attention_weight_max": 32.06145095825195,
      "activations/layer17_attention_weight_min": -25.455171585083008,
      "activations/layer18_attention_weight_max": 30.43488883972168,
      "activations/layer18_attention_weight_min": -20.8821964263916,
      "activations/layer19_attention_weight_max": 36.36375045776367,
      "activations/layer19_attention_weight_min": -32.12980651855469,
      "activations/layer1_attention_weight_max": 16.958663940429688,
      "activations/layer1_attention_weight_min": -17.48973274230957,
      "activations/layer20_attention_weight_max": 31.011629104614258,
      "activations/layer20_attention_weight_min": -26.697954177856445,
      "activations/layer21_attention_weight_max": 29.3015193939209,
      "activations/layer21_attention_weight_min": -24.238256454467773,
      "activations/layer22_attention_weight_max": 44.47893524169922,
      "activations/layer22_attention_weight_min": -30.31428337097168,
      "activations/layer23_attention_weight_max": 35.00666427612305,
      "activations/layer23_attention_weight_min": -24.235807418823242,
      "activations/layer2_attention_weight_max": 33.87900161743164,
      "activations/layer2_attention_weight_min": -31.769878387451172,
      "activations/layer3_attention_weight_max": 95.49884033203125,
      "activations/layer3_attention_weight_min": -94.05736541748047,
      "activations/layer4_attention_weight_max": 119.5701675415039,
      "activations/layer4_attention_weight_min": -110.7799301147461,
      "activations/layer5_attention_weight_max": 56.75408172607422,
      "activations/layer5_attention_weight_min": -58.4873161315918,
      "activations/layer6_attention_weight_max": 49.39662551879883,
      "activations/layer6_attention_weight_min": -48.68065643310547,
      "activations/layer7_attention_weight_max": 89.3731460571289,
      "activations/layer7_attention_weight_min": -86.09165954589844,
      "activations/layer8_attention_weight_max": 38.921241760253906,
      "activations/layer8_attention_weight_min": -39.740936279296875,
      "activations/layer9_attention_weight_max": 31.888349533081055,
      "activations/layer9_attention_weight_min": -31.60959243774414,
      "epoch": 9.04,
      "learning_rate": 9.266515151515151e-05,
      "loss": 2.7972,
      "step": 155500
    },
    {
      "activations/layer0_attention_weight_max": 13.98525333404541,
      "activations/layer0_attention_weight_min": -12.077948570251465,
      "activations/layer10_attention_weight_max": 39.12686538696289,
      "activations/layer10_attention_weight_min": -34.9273796081543,
      "activations/layer11_attention_weight_max": 36.46452331542969,
      "activations/layer11_attention_weight_min": -35.847679138183594,
      "activations/layer12_attention_weight_max": 23.677629470825195,
      "activations/layer12_attention_weight_min": -26.933610916137695,
      "activations/layer13_attention_weight_max": 41.2972297668457,
      "activations/layer13_attention_weight_min": -37.44219970703125,
      "activations/layer14_attention_weight_max": 48.21662902832031,
      "activations/layer14_attention_weight_min": -41.25080871582031,
      "activations/layer15_attention_weight_max": 37.48033905029297,
      "activations/layer15_attention_weight_min": -33.09456253051758,
      "activations/layer16_attention_weight_max": 33.20343780517578,
      "activations/layer16_attention_weight_min": -27.559415817260742,
      "activations/layer17_attention_weight_max": 34.39931869506836,
      "activations/layer17_attention_weight_min": -24.645587921142578,
      "activations/layer18_attention_weight_max": 31.09271240234375,
      "activations/layer18_attention_weight_min": -21.351451873779297,
      "activations/layer19_attention_weight_max": 34.59055709838867,
      "activations/layer19_attention_weight_min": -30.7083683013916,
      "activations/layer1_attention_weight_max": 17.04604721069336,
      "activations/layer1_attention_weight_min": -15.220949172973633,
      "activations/layer20_attention_weight_max": 31.919937133789062,
      "activations/layer20_attention_weight_min": -23.84426498413086,
      "activations/layer21_attention_weight_max": 31.0908260345459,
      "activations/layer21_attention_weight_min": -23.25572395324707,
      "activations/layer22_attention_weight_max": 46.494789123535156,
      "activations/layer22_attention_weight_min": -30.549945831298828,
      "activations/layer23_attention_weight_max": 35.46806335449219,
      "activations/layer23_attention_weight_min": -23.718502044677734,
      "activations/layer2_attention_weight_max": 32.518577575683594,
      "activations/layer2_attention_weight_min": -33.56949996948242,
      "activations/layer3_attention_weight_max": 103.8494873046875,
      "activations/layer3_attention_weight_min": -105.74119567871094,
      "activations/layer4_attention_weight_max": 127.12092590332031,
      "activations/layer4_attention_weight_min": -123.42523193359375,
      "activations/layer5_attention_weight_max": 58.11609649658203,
      "activations/layer5_attention_weight_min": -66.20767974853516,
      "activations/layer6_attention_weight_max": 54.07792282104492,
      "activations/layer6_attention_weight_min": -54.56178283691406,
      "activations/layer7_attention_weight_max": 95.0607681274414,
      "activations/layer7_attention_weight_min": -93.90259552001953,
      "activations/layer8_attention_weight_max": 44.6884651184082,
      "activations/layer8_attention_weight_min": -43.78466033935547,
      "activations/layer9_attention_weight_max": 41.350502014160156,
      "activations/layer9_attention_weight_min": -37.690372467041016,
      "epoch": 9.04,
      "learning_rate": 9.264621212121211e-05,
      "loss": 2.7739,
      "step": 155550
    },
    {
      "activations/layer0_attention_weight_max": 14.84517765045166,
      "activations/layer0_attention_weight_min": -11.867464065551758,
      "activations/layer10_attention_weight_max": 36.3983268737793,
      "activations/layer10_attention_weight_min": -34.430381774902344,
      "activations/layer11_attention_weight_max": 38.956642150878906,
      "activations/layer11_attention_weight_min": -33.79326248168945,
      "activations/layer12_attention_weight_max": 46.132259368896484,
      "activations/layer12_attention_weight_min": -27.900789260864258,
      "activations/layer13_attention_weight_max": 58.9797477722168,
      "activations/layer13_attention_weight_min": -36.9537239074707,
      "activations/layer14_attention_weight_max": 51.14609909057617,
      "activations/layer14_attention_weight_min": -37.94983673095703,
      "activations/layer15_attention_weight_max": 44.439144134521484,
      "activations/layer15_attention_weight_min": -30.994338989257812,
      "activations/layer16_attention_weight_max": 36.406105041503906,
      "activations/layer16_attention_weight_min": -28.563005447387695,
      "activations/layer17_attention_weight_max": 36.53791809082031,
      "activations/layer17_attention_weight_min": -27.425678253173828,
      "activations/layer18_attention_weight_max": 30.82758903503418,
      "activations/layer18_attention_weight_min": -25.82143211364746,
      "activations/layer19_attention_weight_max": 36.3921012878418,
      "activations/layer19_attention_weight_min": -31.38968276977539,
      "activations/layer1_attention_weight_max": 17.0808048248291,
      "activations/layer1_attention_weight_min": -15.103443145751953,
      "activations/layer20_attention_weight_max": 27.689109802246094,
      "activations/layer20_attention_weight_min": -23.781530380249023,
      "activations/layer21_attention_weight_max": 27.183439254760742,
      "activations/layer21_attention_weight_min": -23.225759506225586,
      "activations/layer22_attention_weight_max": 40.7839469909668,
      "activations/layer22_attention_weight_min": -30.24159812927246,
      "activations/layer23_attention_weight_max": 36.98509216308594,
      "activations/layer23_attention_weight_min": -27.16994285583496,
      "activations/layer2_attention_weight_max": 32.18983459472656,
      "activations/layer2_attention_weight_min": -34.037109375,
      "activations/layer3_attention_weight_max": 101.5576400756836,
      "activations/layer3_attention_weight_min": -101.0923080444336,
      "activations/layer4_attention_weight_max": 117.2933578491211,
      "activations/layer4_attention_weight_min": -118.66719055175781,
      "activations/layer5_attention_weight_max": 54.060218811035156,
      "activations/layer5_attention_weight_min": -62.29633331298828,
      "activations/layer6_attention_weight_max": 48.40144729614258,
      "activations/layer6_attention_weight_min": -49.159969329833984,
      "activations/layer7_attention_weight_max": 97.76509094238281,
      "activations/layer7_attention_weight_min": -86.16133117675781,
      "activations/layer8_attention_weight_max": 45.38239288330078,
      "activations/layer8_attention_weight_min": -40.254913330078125,
      "activations/layer9_attention_weight_max": 36.44343185424805,
      "activations/layer9_attention_weight_min": -34.87398910522461,
      "epoch": 9.04,
      "learning_rate": 9.262727272727271e-05,
      "loss": 2.7923,
      "step": 155600
    },
    {
      "activations/layer0_attention_weight_max": 14.642594337463379,
      "activations/layer0_attention_weight_min": -12.005874633789062,
      "activations/layer10_attention_weight_max": 36.79251480102539,
      "activations/layer10_attention_weight_min": -34.486480712890625,
      "activations/layer11_attention_weight_max": 37.562538146972656,
      "activations/layer11_attention_weight_min": -35.888797760009766,
      "activations/layer12_attention_weight_max": 41.2426872253418,
      "activations/layer12_attention_weight_min": -25.00827980041504,
      "activations/layer13_attention_weight_max": 40.97816848754883,
      "activations/layer13_attention_weight_min": -32.77811813354492,
      "activations/layer14_attention_weight_max": 51.279380798339844,
      "activations/layer14_attention_weight_min": -39.31637954711914,
      "activations/layer15_attention_weight_max": 41.171566009521484,
      "activations/layer15_attention_weight_min": -31.235971450805664,
      "activations/layer16_attention_weight_max": 33.67292022705078,
      "activations/layer16_attention_weight_min": -28.122438430786133,
      "activations/layer17_attention_weight_max": 37.57829666137695,
      "activations/layer17_attention_weight_min": -24.791919708251953,
      "activations/layer18_attention_weight_max": 33.676795959472656,
      "activations/layer18_attention_weight_min": -21.783891677856445,
      "activations/layer19_attention_weight_max": 42.542335510253906,
      "activations/layer19_attention_weight_min": -30.859106063842773,
      "activations/layer1_attention_weight_max": 17.705429077148438,
      "activations/layer1_attention_weight_min": -15.525386810302734,
      "activations/layer20_attention_weight_max": 36.21348571777344,
      "activations/layer20_attention_weight_min": -23.377593994140625,
      "activations/layer21_attention_weight_max": 35.151763916015625,
      "activations/layer21_attention_weight_min": -23.59128761291504,
      "activations/layer22_attention_weight_max": 50.15046691894531,
      "activations/layer22_attention_weight_min": -32.2385139465332,
      "activations/layer23_attention_weight_max": 37.989601135253906,
      "activations/layer23_attention_weight_min": -25.727493286132812,
      "activations/layer2_attention_weight_max": 32.50299072265625,
      "activations/layer2_attention_weight_min": -33.058380126953125,
      "activations/layer3_attention_weight_max": 98.69927978515625,
      "activations/layer3_attention_weight_min": -98.14672088623047,
      "activations/layer4_attention_weight_max": 122.9893798828125,
      "activations/layer4_attention_weight_min": -119.8934097290039,
      "activations/layer5_attention_weight_max": 53.906761169433594,
      "activations/layer5_attention_weight_min": -62.35517120361328,
      "activations/layer6_attention_weight_max": 47.92632293701172,
      "activations/layer6_attention_weight_min": -50.903480529785156,
      "activations/layer7_attention_weight_max": 89.79087829589844,
      "activations/layer7_attention_weight_min": -92.69792175292969,
      "activations/layer8_attention_weight_max": 39.874298095703125,
      "activations/layer8_attention_weight_min": -41.22093200683594,
      "activations/layer9_attention_weight_max": 38.316184997558594,
      "activations/layer9_attention_weight_min": -35.347389221191406,
      "epoch": 9.04,
      "learning_rate": 9.260833333333333e-05,
      "loss": 2.7822,
      "step": 155650
    },
    {
      "activations/layer0_attention_weight_max": 16.558923721313477,
      "activations/layer0_attention_weight_min": -12.678339004516602,
      "activations/layer10_attention_weight_max": 32.182498931884766,
      "activations/layer10_attention_weight_min": -32.935096740722656,
      "activations/layer11_attention_weight_max": 33.11237335205078,
      "activations/layer11_attention_weight_min": -30.282554626464844,
      "activations/layer12_attention_weight_max": 26.125900268554688,
      "activations/layer12_attention_weight_min": -24.50037384033203,
      "activations/layer13_attention_weight_max": 39.45759963989258,
      "activations/layer13_attention_weight_min": -31.22975730895996,
      "activations/layer14_attention_weight_max": 46.5045051574707,
      "activations/layer14_attention_weight_min": -33.793601989746094,
      "activations/layer15_attention_weight_max": 39.63082504272461,
      "activations/layer15_attention_weight_min": -31.439889907836914,
      "activations/layer16_attention_weight_max": 34.43902587890625,
      "activations/layer16_attention_weight_min": -26.985450744628906,
      "activations/layer17_attention_weight_max": 34.775753021240234,
      "activations/layer17_attention_weight_min": -25.638660430908203,
      "activations/layer18_attention_weight_max": 31.415550231933594,
      "activations/layer18_attention_weight_min": -22.786149978637695,
      "activations/layer19_attention_weight_max": 38.37418746948242,
      "activations/layer19_attention_weight_min": -29.42768669128418,
      "activations/layer1_attention_weight_max": 17.706571578979492,
      "activations/layer1_attention_weight_min": -15.92177677154541,
      "activations/layer20_attention_weight_max": 33.02153015136719,
      "activations/layer20_attention_weight_min": -23.6173152923584,
      "activations/layer21_attention_weight_max": 31.94890022277832,
      "activations/layer21_attention_weight_min": -23.853641510009766,
      "activations/layer22_attention_weight_max": 48.10687255859375,
      "activations/layer22_attention_weight_min": -29.414623260498047,
      "activations/layer23_attention_weight_max": 36.5762939453125,
      "activations/layer23_attention_weight_min": -24.670194625854492,
      "activations/layer2_attention_weight_max": 31.231313705444336,
      "activations/layer2_attention_weight_min": -31.000730514526367,
      "activations/layer3_attention_weight_max": 96.73087310791016,
      "activations/layer3_attention_weight_min": -95.6510238647461,
      "activations/layer4_attention_weight_max": 118.32135009765625,
      "activations/layer4_attention_weight_min": -116.25511932373047,
      "activations/layer5_attention_weight_max": 53.92478942871094,
      "activations/layer5_attention_weight_min": -65.48966979980469,
      "activations/layer6_attention_weight_max": 46.91643142700195,
      "activations/layer6_attention_weight_min": -48.13814163208008,
      "activations/layer7_attention_weight_max": 89.36023712158203,
      "activations/layer7_attention_weight_min": -89.01083374023438,
      "activations/layer8_attention_weight_max": 39.14686965942383,
      "activations/layer8_attention_weight_min": -39.246665954589844,
      "activations/layer9_attention_weight_max": 35.04689025878906,
      "activations/layer9_attention_weight_min": -35.422054290771484,
      "epoch": 9.05,
      "learning_rate": 9.258939393939394e-05,
      "loss": 2.7873,
      "step": 155700
    },
    {
      "activations/layer0_attention_weight_max": 16.13994598388672,
      "activations/layer0_attention_weight_min": -13.19279956817627,
      "activations/layer10_attention_weight_max": 34.5074462890625,
      "activations/layer10_attention_weight_min": -33.43269729614258,
      "activations/layer11_attention_weight_max": 34.30638122558594,
      "activations/layer11_attention_weight_min": -33.53388977050781,
      "activations/layer12_attention_weight_max": 32.84334182739258,
      "activations/layer12_attention_weight_min": -39.56815719604492,
      "activations/layer13_attention_weight_max": 38.739601135253906,
      "activations/layer13_attention_weight_min": -32.36576461791992,
      "activations/layer14_attention_weight_max": 45.7882080078125,
      "activations/layer14_attention_weight_min": -40.50116729736328,
      "activations/layer15_attention_weight_max": 38.36336898803711,
      "activations/layer15_attention_weight_min": -31.437955856323242,
      "activations/layer16_attention_weight_max": 32.3216552734375,
      "activations/layer16_attention_weight_min": -26.263500213623047,
      "activations/layer17_attention_weight_max": 33.86625289916992,
      "activations/layer17_attention_weight_min": -29.09986686706543,
      "activations/layer18_attention_weight_max": 33.421287536621094,
      "activations/layer18_attention_weight_min": -21.167728424072266,
      "activations/layer19_attention_weight_max": 34.55167007446289,
      "activations/layer19_attention_weight_min": -28.381061553955078,
      "activations/layer1_attention_weight_max": 17.221445083618164,
      "activations/layer1_attention_weight_min": -14.846607208251953,
      "activations/layer20_attention_weight_max": 30.82391929626465,
      "activations/layer20_attention_weight_min": -22.476469039916992,
      "activations/layer21_attention_weight_max": 36.62919616699219,
      "activations/layer21_attention_weight_min": -22.25987434387207,
      "activations/layer22_attention_weight_max": 49.921451568603516,
      "activations/layer22_attention_weight_min": -28.40007209777832,
      "activations/layer23_attention_weight_max": 33.190940856933594,
      "activations/layer23_attention_weight_min": -22.813648223876953,
      "activations/layer2_attention_weight_max": 32.94817352294922,
      "activations/layer2_attention_weight_min": -33.804054260253906,
      "activations/layer3_attention_weight_max": 93.71649169921875,
      "activations/layer3_attention_weight_min": -100.35469818115234,
      "activations/layer4_attention_weight_max": 115.928955078125,
      "activations/layer4_attention_weight_min": -116.2530746459961,
      "activations/layer5_attention_weight_max": 53.47034454345703,
      "activations/layer5_attention_weight_min": -60.227569580078125,
      "activations/layer6_attention_weight_max": 46.80876541137695,
      "activations/layer6_attention_weight_min": -48.534671783447266,
      "activations/layer7_attention_weight_max": 91.8292007446289,
      "activations/layer7_attention_weight_min": -86.06208801269531,
      "activations/layer8_attention_weight_max": 40.5460090637207,
      "activations/layer8_attention_weight_min": -41.871788024902344,
      "activations/layer9_attention_weight_max": 34.87370300292969,
      "activations/layer9_attention_weight_min": -34.4044303894043,
      "epoch": 9.05,
      "learning_rate": 9.257045454545453e-05,
      "loss": 2.7936,
      "step": 155750
    },
    {
      "activations/layer0_attention_weight_max": 15.322969436645508,
      "activations/layer0_attention_weight_min": -11.57364559173584,
      "activations/layer10_attention_weight_max": 33.01947784423828,
      "activations/layer10_attention_weight_min": -31.783660888671875,
      "activations/layer11_attention_weight_max": 33.39931869506836,
      "activations/layer11_attention_weight_min": -30.197818756103516,
      "activations/layer12_attention_weight_max": 22.29935646057129,
      "activations/layer12_attention_weight_min": -26.428512573242188,
      "activations/layer13_attention_weight_max": 37.63303756713867,
      "activations/layer13_attention_weight_min": -33.23206329345703,
      "activations/layer14_attention_weight_max": 45.66706466674805,
      "activations/layer14_attention_weight_min": -35.600303649902344,
      "activations/layer15_attention_weight_max": 35.511417388916016,
      "activations/layer15_attention_weight_min": -30.89706039428711,
      "activations/layer16_attention_weight_max": 32.20169448852539,
      "activations/layer16_attention_weight_min": -27.484603881835938,
      "activations/layer17_attention_weight_max": 36.40658187866211,
      "activations/layer17_attention_weight_min": -25.354888916015625,
      "activations/layer18_attention_weight_max": 31.765811920166016,
      "activations/layer18_attention_weight_min": -21.617647171020508,
      "activations/layer19_attention_weight_max": 43.367855072021484,
      "activations/layer19_attention_weight_min": -32.85161590576172,
      "activations/layer1_attention_weight_max": 16.2464599609375,
      "activations/layer1_attention_weight_min": -14.77374267578125,
      "activations/layer20_attention_weight_max": 36.449642181396484,
      "activations/layer20_attention_weight_min": -26.29912757873535,
      "activations/layer21_attention_weight_max": 31.049007415771484,
      "activations/layer21_attention_weight_min": -23.420005798339844,
      "activations/layer22_attention_weight_max": 57.388492584228516,
      "activations/layer22_attention_weight_min": -33.6417236328125,
      "activations/layer23_attention_weight_max": 44.316429138183594,
      "activations/layer23_attention_weight_min": -26.024333953857422,
      "activations/layer2_attention_weight_max": 32.98653030395508,
      "activations/layer2_attention_weight_min": -31.37433624267578,
      "activations/layer3_attention_weight_max": 100.17988586425781,
      "activations/layer3_attention_weight_min": -94.54682922363281,
      "activations/layer4_attention_weight_max": 119.99981689453125,
      "activations/layer4_attention_weight_min": -113.03166961669922,
      "activations/layer5_attention_weight_max": 56.117000579833984,
      "activations/layer5_attention_weight_min": -58.93146896362305,
      "activations/layer6_attention_weight_max": 49.037452697753906,
      "activations/layer6_attention_weight_min": -49.33870315551758,
      "activations/layer7_attention_weight_max": 88.30511474609375,
      "activations/layer7_attention_weight_min": -88.7041244506836,
      "activations/layer8_attention_weight_max": 40.08979415893555,
      "activations/layer8_attention_weight_min": -45.70800018310547,
      "activations/layer9_attention_weight_max": 33.08475875854492,
      "activations/layer9_attention_weight_min": -36.31909942626953,
      "epoch": 9.05,
      "learning_rate": 9.255151515151514e-05,
      "loss": 2.7765,
      "step": 155800
    },
    {
      "activations/layer0_attention_weight_max": 15.823552131652832,
      "activations/layer0_attention_weight_min": -12.279046058654785,
      "activations/layer10_attention_weight_max": 34.146514892578125,
      "activations/layer10_attention_weight_min": -32.22809600830078,
      "activations/layer11_attention_weight_max": 33.60332489013672,
      "activations/layer11_attention_weight_min": -30.68903160095215,
      "activations/layer12_attention_weight_max": 28.695785522460938,
      "activations/layer12_attention_weight_min": -27.187658309936523,
      "activations/layer13_attention_weight_max": 37.06714630126953,
      "activations/layer13_attention_weight_min": -31.578584671020508,
      "activations/layer14_attention_weight_max": 46.790157318115234,
      "activations/layer14_attention_weight_min": -37.20009994506836,
      "activations/layer15_attention_weight_max": 35.238563537597656,
      "activations/layer15_attention_weight_min": -29.338638305664062,
      "activations/layer16_attention_weight_max": 32.28289794921875,
      "activations/layer16_attention_weight_min": -26.475425720214844,
      "activations/layer17_attention_weight_max": 32.54956817626953,
      "activations/layer17_attention_weight_min": -24.29485321044922,
      "activations/layer18_attention_weight_max": 31.1638126373291,
      "activations/layer18_attention_weight_min": -23.44683837890625,
      "activations/layer19_attention_weight_max": 37.144161224365234,
      "activations/layer19_attention_weight_min": -30.0445556640625,
      "activations/layer1_attention_weight_max": 16.50637435913086,
      "activations/layer1_attention_weight_min": -14.945672988891602,
      "activations/layer20_attention_weight_max": 29.880319595336914,
      "activations/layer20_attention_weight_min": -25.81878662109375,
      "activations/layer21_attention_weight_max": 29.156042098999023,
      "activations/layer21_attention_weight_min": -23.270980834960938,
      "activations/layer22_attention_weight_max": 46.99837112426758,
      "activations/layer22_attention_weight_min": -29.053634643554688,
      "activations/layer23_attention_weight_max": 33.177188873291016,
      "activations/layer23_attention_weight_min": -23.220022201538086,
      "activations/layer2_attention_weight_max": 32.86951446533203,
      "activations/layer2_attention_weight_min": -33.05540466308594,
      "activations/layer3_attention_weight_max": 99.07968139648438,
      "activations/layer3_attention_weight_min": -97.07704162597656,
      "activations/layer4_attention_weight_max": 119.9522705078125,
      "activations/layer4_attention_weight_min": -115.01312255859375,
      "activations/layer5_attention_weight_max": 53.41432189941406,
      "activations/layer5_attention_weight_min": -58.618404388427734,
      "activations/layer6_attention_weight_max": 47.44845962524414,
      "activations/layer6_attention_weight_min": -47.902896881103516,
      "activations/layer7_attention_weight_max": 90.08899688720703,
      "activations/layer7_attention_weight_min": -84.64683532714844,
      "activations/layer8_attention_weight_max": 41.972373962402344,
      "activations/layer8_attention_weight_min": -40.01054763793945,
      "activations/layer9_attention_weight_max": 37.01988983154297,
      "activations/layer9_attention_weight_min": -33.80900573730469,
      "epoch": 9.06,
      "learning_rate": 9.253257575757576e-05,
      "loss": 2.7897,
      "step": 155850
    },
    {
      "activations/layer0_attention_weight_max": 15.681679725646973,
      "activations/layer0_attention_weight_min": -12.616724967956543,
      "activations/layer10_attention_weight_max": 34.03560256958008,
      "activations/layer10_attention_weight_min": -34.03673553466797,
      "activations/layer11_attention_weight_max": 35.33416748046875,
      "activations/layer11_attention_weight_min": -35.609336853027344,
      "activations/layer12_attention_weight_max": 39.33635711669922,
      "activations/layer12_attention_weight_min": -25.359046936035156,
      "activations/layer13_attention_weight_max": 52.07902908325195,
      "activations/layer13_attention_weight_min": -35.33089828491211,
      "activations/layer14_attention_weight_max": 70.0462646484375,
      "activations/layer14_attention_weight_min": -43.656402587890625,
      "activations/layer15_attention_weight_max": 43.689056396484375,
      "activations/layer15_attention_weight_min": -31.376766204833984,
      "activations/layer16_attention_weight_max": 36.395118713378906,
      "activations/layer16_attention_weight_min": -27.842853546142578,
      "activations/layer17_attention_weight_max": 36.14045333862305,
      "activations/layer17_attention_weight_min": -25.9760799407959,
      "activations/layer18_attention_weight_max": 31.633642196655273,
      "activations/layer18_attention_weight_min": -23.2886962890625,
      "activations/layer19_attention_weight_max": 38.93281173706055,
      "activations/layer19_attention_weight_min": -29.942758560180664,
      "activations/layer1_attention_weight_max": 16.766706466674805,
      "activations/layer1_attention_weight_min": -14.373754501342773,
      "activations/layer20_attention_weight_max": 32.53888702392578,
      "activations/layer20_attention_weight_min": -23.852014541625977,
      "activations/layer21_attention_weight_max": 29.60764503479004,
      "activations/layer21_attention_weight_min": -22.009641647338867,
      "activations/layer22_attention_weight_max": 50.75423049926758,
      "activations/layer22_attention_weight_min": -29.32373809814453,
      "activations/layer23_attention_weight_max": 42.22508239746094,
      "activations/layer23_attention_weight_min": -23.594512939453125,
      "activations/layer2_attention_weight_max": 33.94207763671875,
      "activations/layer2_attention_weight_min": -35.466487884521484,
      "activations/layer3_attention_weight_max": 96.38693237304688,
      "activations/layer3_attention_weight_min": -100.2090835571289,
      "activations/layer4_attention_weight_max": 118.41490936279297,
      "activations/layer4_attention_weight_min": -118.11626434326172,
      "activations/layer5_attention_weight_max": 56.08637619018555,
      "activations/layer5_attention_weight_min": -64.54930114746094,
      "activations/layer6_attention_weight_max": 48.32991027832031,
      "activations/layer6_attention_weight_min": -50.232418060302734,
      "activations/layer7_attention_weight_max": 91.84001159667969,
      "activations/layer7_attention_weight_min": -90.78228759765625,
      "activations/layer8_attention_weight_max": 42.63702392578125,
      "activations/layer8_attention_weight_min": -42.746002197265625,
      "activations/layer9_attention_weight_max": 37.66130065917969,
      "activations/layer9_attention_weight_min": -35.312164306640625,
      "epoch": 9.06,
      "learning_rate": 9.251363636363635e-05,
      "loss": 2.7763,
      "step": 155900
    },
    {
      "activations/layer0_attention_weight_max": 15.532089233398438,
      "activations/layer0_attention_weight_min": -11.869494438171387,
      "activations/layer10_attention_weight_max": 33.65745544433594,
      "activations/layer10_attention_weight_min": -32.61993408203125,
      "activations/layer11_attention_weight_max": 35.12548828125,
      "activations/layer11_attention_weight_min": -33.39185333251953,
      "activations/layer12_attention_weight_max": 23.195066452026367,
      "activations/layer12_attention_weight_min": -37.44486999511719,
      "activations/layer13_attention_weight_max": 42.3290901184082,
      "activations/layer13_attention_weight_min": -40.428009033203125,
      "activations/layer14_attention_weight_max": 48.58209991455078,
      "activations/layer14_attention_weight_min": -45.76630401611328,
      "activations/layer15_attention_weight_max": 36.103267669677734,
      "activations/layer15_attention_weight_min": -32.28913497924805,
      "activations/layer16_attention_weight_max": 29.614126205444336,
      "activations/layer16_attention_weight_min": -28.309913635253906,
      "activations/layer17_attention_weight_max": 30.7979679107666,
      "activations/layer17_attention_weight_min": -31.29155921936035,
      "activations/layer18_attention_weight_max": 30.346359252929688,
      "activations/layer18_attention_weight_min": -24.709123611450195,
      "activations/layer19_attention_weight_max": 32.643062591552734,
      "activations/layer19_attention_weight_min": -30.302722930908203,
      "activations/layer1_attention_weight_max": 16.90094757080078,
      "activations/layer1_attention_weight_min": -13.852867126464844,
      "activations/layer20_attention_weight_max": 27.468658447265625,
      "activations/layer20_attention_weight_min": -26.49336814880371,
      "activations/layer21_attention_weight_max": 27.306297302246094,
      "activations/layer21_attention_weight_min": -23.072898864746094,
      "activations/layer22_attention_weight_max": 45.97370910644531,
      "activations/layer22_attention_weight_min": -30.918291091918945,
      "activations/layer23_attention_weight_max": 34.64110565185547,
      "activations/layer23_attention_weight_min": -23.389196395874023,
      "activations/layer2_attention_weight_max": 32.716270446777344,
      "activations/layer2_attention_weight_min": -32.04621124267578,
      "activations/layer3_attention_weight_max": 95.6664047241211,
      "activations/layer3_attention_weight_min": -96.102783203125,
      "activations/layer4_attention_weight_max": 117.74063873291016,
      "activations/layer4_attention_weight_min": -114.89342498779297,
      "activations/layer5_attention_weight_max": 57.9687614440918,
      "activations/layer5_attention_weight_min": -60.818111419677734,
      "activations/layer6_attention_weight_max": 46.758731842041016,
      "activations/layer6_attention_weight_min": -50.32395935058594,
      "activations/layer7_attention_weight_max": 91.48322296142578,
      "activations/layer7_attention_weight_min": -83.77804565429688,
      "activations/layer8_attention_weight_max": 40.113975524902344,
      "activations/layer8_attention_weight_min": -38.29933547973633,
      "activations/layer9_attention_weight_max": 33.9310302734375,
      "activations/layer9_attention_weight_min": -35.27467727661133,
      "epoch": 9.06,
      "learning_rate": 9.249469696969696e-05,
      "loss": 2.7953,
      "step": 155950
    },
    {
      "activations/layer0_attention_weight_max": 15.445253372192383,
      "activations/layer0_attention_weight_min": -11.775115013122559,
      "activations/layer10_attention_weight_max": 31.6833553314209,
      "activations/layer10_attention_weight_min": -32.16964340209961,
      "activations/layer11_attention_weight_max": 33.277198791503906,
      "activations/layer11_attention_weight_min": -31.321157455444336,
      "activations/layer12_attention_weight_max": 28.39704132080078,
      "activations/layer12_attention_weight_min": -28.790117263793945,
      "activations/layer13_attention_weight_max": 37.7959098815918,
      "activations/layer13_attention_weight_min": -33.57486343383789,
      "activations/layer14_attention_weight_max": 48.3952751159668,
      "activations/layer14_attention_weight_min": -40.010353088378906,
      "activations/layer15_attention_weight_max": 34.23909378051758,
      "activations/layer15_attention_weight_min": -30.335126876831055,
      "activations/layer16_attention_weight_max": 29.472415924072266,
      "activations/layer16_attention_weight_min": -28.896652221679688,
      "activations/layer17_attention_weight_max": 31.694692611694336,
      "activations/layer17_attention_weight_min": -23.713706970214844,
      "activations/layer18_attention_weight_max": 35.34846496582031,
      "activations/layer18_attention_weight_min": -22.597209930419922,
      "activations/layer19_attention_weight_max": 36.09231948852539,
      "activations/layer19_attention_weight_min": -28.918888092041016,
      "activations/layer1_attention_weight_max": 15.918458938598633,
      "activations/layer1_attention_weight_min": -14.745318412780762,
      "activations/layer20_attention_weight_max": 27.772979736328125,
      "activations/layer20_attention_weight_min": -23.14916229248047,
      "activations/layer21_attention_weight_max": 27.532194137573242,
      "activations/layer21_attention_weight_min": -22.144126892089844,
      "activations/layer22_attention_weight_max": 42.65743637084961,
      "activations/layer22_attention_weight_min": -27.54358673095703,
      "activations/layer23_attention_weight_max": 36.24114227294922,
      "activations/layer23_attention_weight_min": -23.612598419189453,
      "activations/layer2_attention_weight_max": 33.286109924316406,
      "activations/layer2_attention_weight_min": -34.05385971069336,
      "activations/layer3_attention_weight_max": 100.79267883300781,
      "activations/layer3_attention_weight_min": -101.0326156616211,
      "activations/layer4_attention_weight_max": 118.17115783691406,
      "activations/layer4_attention_weight_min": -117.47102355957031,
      "activations/layer5_attention_weight_max": 55.49020004272461,
      "activations/layer5_attention_weight_min": -62.07208251953125,
      "activations/layer6_attention_weight_max": 47.13357162475586,
      "activations/layer6_attention_weight_min": -53.25270080566406,
      "activations/layer7_attention_weight_max": 88.41632843017578,
      "activations/layer7_attention_weight_min": -86.42608642578125,
      "activations/layer8_attention_weight_max": 42.18294143676758,
      "activations/layer8_attention_weight_min": -39.732933044433594,
      "activations/layer9_attention_weight_max": 33.96309280395508,
      "activations/layer9_attention_weight_min": -34.21456527709961,
      "epoch": 9.06,
      "learning_rate": 9.247575757575758e-05,
      "loss": 2.8112,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4503,
      "eval_samples_per_second": 508.149,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4503,
      "eval_openwebtext_samples_per_second": 508.149,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.9355,
      "eval_wikitext_samples_per_second": 235.595,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_lambada_loss": 2.525390625,
      "eval_lambada_ppl": 12.495775472672934,
      "eval_lambada_runtime": 9.5221,
      "eval_lambada_samples_per_second": 511.336,
      "step": 156000
    },
    {
      "activations/layer0_attention_weight_max": 14.535943031311035,
      "activations/layer0_attention_weight_min": -11.311944007873535,
      "activations/layer10_attention_weight_max": 33.36195373535156,
      "activations/layer10_attention_weight_min": -30.832632064819336,
      "activations/layer11_attention_weight_max": 32.417884826660156,
      "activations/layer11_attention_weight_min": -29.803640365600586,
      "activations/layer12_attention_weight_max": 37.9134407043457,
      "activations/layer12_attention_weight_min": -26.718530654907227,
      "activations/layer13_attention_weight_max": 49.25044250488281,
      "activations/layer13_attention_weight_min": -33.00456237792969,
      "activations/layer14_attention_weight_max": 54.49681854248047,
      "activations/layer14_attention_weight_min": -36.874393463134766,
      "activations/layer15_attention_weight_max": 37.41732406616211,
      "activations/layer15_attention_weight_min": -29.318803787231445,
      "activations/layer16_attention_weight_max": 34.34908676147461,
      "activations/layer16_attention_weight_min": -27.14573097229004,
      "activations/layer17_attention_weight_max": 35.528751373291016,
      "activations/layer17_attention_weight_min": -24.096981048583984,
      "activations/layer18_attention_weight_max": 31.217477798461914,
      "activations/layer18_attention_weight_min": -20.372777938842773,
      "activations/layer19_attention_weight_max": 40.71333312988281,
      "activations/layer19_attention_weight_min": -30.125837326049805,
      "activations/layer1_attention_weight_max": 16.3810977935791,
      "activations/layer1_attention_weight_min": -14.413697242736816,
      "activations/layer20_attention_weight_max": 34.06443786621094,
      "activations/layer20_attention_weight_min": -24.856544494628906,
      "activations/layer21_attention_weight_max": 31.289154052734375,
      "activations/layer21_attention_weight_min": -22.946483612060547,
      "activations/layer22_attention_weight_max": 57.642494201660156,
      "activations/layer22_attention_weight_min": -30.14842987060547,
      "activations/layer23_attention_weight_max": 41.05582046508789,
      "activations/layer23_attention_weight_min": -24.773883819580078,
      "activations/layer2_attention_weight_max": 32.838619232177734,
      "activations/layer2_attention_weight_min": -32.84544372558594,
      "activations/layer3_attention_weight_max": 98.64686584472656,
      "activations/layer3_attention_weight_min": -100.38406372070312,
      "activations/layer4_attention_weight_max": 119.0914535522461,
      "activations/layer4_attention_weight_min": -114.8652572631836,
      "activations/layer5_attention_weight_max": 53.6759033203125,
      "activations/layer5_attention_weight_min": -62.04774475097656,
      "activations/layer6_attention_weight_max": 46.4400749206543,
      "activations/layer6_attention_weight_min": -48.29805374145508,
      "activations/layer7_attention_weight_max": 85.99816131591797,
      "activations/layer7_attention_weight_min": -88.96732330322266,
      "activations/layer8_attention_weight_max": 43.4510612487793,
      "activations/layer8_attention_weight_min": -41.33122253417969,
      "activations/layer9_attention_weight_max": 34.47461700439453,
      "activations/layer9_attention_weight_min": -34.45755386352539,
      "epoch": 9.07,
      "learning_rate": 9.245719696969697e-05,
      "loss": 2.7973,
      "step": 156050
    },
    {
      "activations/layer0_attention_weight_max": 15.742748260498047,
      "activations/layer0_attention_weight_min": -11.38935375213623,
      "activations/layer10_attention_weight_max": 31.407915115356445,
      "activations/layer10_attention_weight_min": -30.53489875793457,
      "activations/layer11_attention_weight_max": 35.836090087890625,
      "activations/layer11_attention_weight_min": -29.930011749267578,
      "activations/layer12_attention_weight_max": 21.788599014282227,
      "activations/layer12_attention_weight_min": -32.88960647583008,
      "activations/layer13_attention_weight_max": 35.66630172729492,
      "activations/layer13_attention_weight_min": -32.42909240722656,
      "activations/layer14_attention_weight_max": 45.26020050048828,
      "activations/layer14_attention_weight_min": -34.597633361816406,
      "activations/layer15_attention_weight_max": 35.69102478027344,
      "activations/layer15_attention_weight_min": -29.1146240234375,
      "activations/layer16_attention_weight_max": 33.48820877075195,
      "activations/layer16_attention_weight_min": -27.12431526184082,
      "activations/layer17_attention_weight_max": 32.14689254760742,
      "activations/layer17_attention_weight_min": -25.247926712036133,
      "activations/layer18_attention_weight_max": 28.1239070892334,
      "activations/layer18_attention_weight_min": -20.78129768371582,
      "activations/layer19_attention_weight_max": 36.664024353027344,
      "activations/layer19_attention_weight_min": -29.5729923248291,
      "activations/layer1_attention_weight_max": 16.068628311157227,
      "activations/layer1_attention_weight_min": -14.332340240478516,
      "activations/layer20_attention_weight_max": 29.895145416259766,
      "activations/layer20_attention_weight_min": -24.32965660095215,
      "activations/layer21_attention_weight_max": 29.097185134887695,
      "activations/layer21_attention_weight_min": -21.294803619384766,
      "activations/layer22_attention_weight_max": 46.42449951171875,
      "activations/layer22_attention_weight_min": -27.501964569091797,
      "activations/layer23_attention_weight_max": 35.896278381347656,
      "activations/layer23_attention_weight_min": -23.639842987060547,
      "activations/layer2_attention_weight_max": 31.781192779541016,
      "activations/layer2_attention_weight_min": -32.81845474243164,
      "activations/layer3_attention_weight_max": 97.66194152832031,
      "activations/layer3_attention_weight_min": -97.13264465332031,
      "activations/layer4_attention_weight_max": 119.8714599609375,
      "activations/layer4_attention_weight_min": -117.07828521728516,
      "activations/layer5_attention_weight_max": 52.162940979003906,
      "activations/layer5_attention_weight_min": -59.897178649902344,
      "activations/layer6_attention_weight_max": 46.69713592529297,
      "activations/layer6_attention_weight_min": -50.67732620239258,
      "activations/layer7_attention_weight_max": 95.2693099975586,
      "activations/layer7_attention_weight_min": -82.97521209716797,
      "activations/layer8_attention_weight_max": 41.01112747192383,
      "activations/layer8_attention_weight_min": -41.61183166503906,
      "activations/layer9_attention_weight_max": 34.11125946044922,
      "activations/layer9_attention_weight_min": -32.736183166503906,
      "epoch": 9.07,
      "learning_rate": 9.243825757575757e-05,
      "loss": 2.7935,
      "step": 156100
    },
    {
      "activations/layer0_attention_weight_max": 15.11186695098877,
      "activations/layer0_attention_weight_min": -10.763747215270996,
      "activations/layer10_attention_weight_max": 33.66054153442383,
      "activations/layer10_attention_weight_min": -31.426076889038086,
      "activations/layer11_attention_weight_max": 33.110172271728516,
      "activations/layer11_attention_weight_min": -29.850704193115234,
      "activations/layer12_attention_weight_max": 41.568511962890625,
      "activations/layer12_attention_weight_min": -28.076210021972656,
      "activations/layer13_attention_weight_max": 45.27568435668945,
      "activations/layer13_attention_weight_min": -34.666046142578125,
      "activations/layer14_attention_weight_max": 45.86967086791992,
      "activations/layer14_attention_weight_min": -38.456092834472656,
      "activations/layer15_attention_weight_max": 35.45240783691406,
      "activations/layer15_attention_weight_min": -31.82619285583496,
      "activations/layer16_attention_weight_max": 33.183509826660156,
      "activations/layer16_attention_weight_min": -27.742494583129883,
      "activations/layer17_attention_weight_max": 31.110050201416016,
      "activations/layer17_attention_weight_min": -25.42760467529297,
      "activations/layer18_attention_weight_max": 30.493913650512695,
      "activations/layer18_attention_weight_min": -22.052433013916016,
      "activations/layer19_attention_weight_max": 35.60798263549805,
      "activations/layer19_attention_weight_min": -30.52273941040039,
      "activations/layer1_attention_weight_max": 16.821937561035156,
      "activations/layer1_attention_weight_min": -14.034745216369629,
      "activations/layer20_attention_weight_max": 30.684602737426758,
      "activations/layer20_attention_weight_min": -24.559629440307617,
      "activations/layer21_attention_weight_max": 28.60483169555664,
      "activations/layer21_attention_weight_min": -23.83367156982422,
      "activations/layer22_attention_weight_max": 45.68309020996094,
      "activations/layer22_attention_weight_min": -30.65486717224121,
      "activations/layer23_attention_weight_max": 32.89567947387695,
      "activations/layer23_attention_weight_min": -25.808368682861328,
      "activations/layer2_attention_weight_max": 33.555076599121094,
      "activations/layer2_attention_weight_min": -31.098045349121094,
      "activations/layer3_attention_weight_max": 97.15386199951172,
      "activations/layer3_attention_weight_min": -93.43394470214844,
      "activations/layer4_attention_weight_max": 116.4698257446289,
      "activations/layer4_attention_weight_min": -117.3138198852539,
      "activations/layer5_attention_weight_max": 53.76817321777344,
      "activations/layer5_attention_weight_min": -60.331695556640625,
      "activations/layer6_attention_weight_max": 44.30215835571289,
      "activations/layer6_attention_weight_min": -46.427738189697266,
      "activations/layer7_attention_weight_max": 96.52615356445312,
      "activations/layer7_attention_weight_min": -88.37223815917969,
      "activations/layer8_attention_weight_max": 40.30809020996094,
      "activations/layer8_attention_weight_min": -40.40542221069336,
      "activations/layer9_attention_weight_max": 35.334049224853516,
      "activations/layer9_attention_weight_min": -35.6020622253418,
      "epoch": 9.07,
      "learning_rate": 9.241931818181817e-05,
      "loss": 2.7906,
      "step": 156150
    },
    {
      "activations/layer0_attention_weight_max": 15.986945152282715,
      "activations/layer0_attention_weight_min": -12.011493682861328,
      "activations/layer10_attention_weight_max": 34.41699981689453,
      "activations/layer10_attention_weight_min": -30.546890258789062,
      "activations/layer11_attention_weight_max": 33.263389587402344,
      "activations/layer11_attention_weight_min": -29.90520477294922,
      "activations/layer12_attention_weight_max": 23.670230865478516,
      "activations/layer12_attention_weight_min": -32.352142333984375,
      "activations/layer13_attention_weight_max": 40.396446228027344,
      "activations/layer13_attention_weight_min": -30.538393020629883,
      "activations/layer14_attention_weight_max": 49.097320556640625,
      "activations/layer14_attention_weight_min": -35.5262451171875,
      "activations/layer15_attention_weight_max": 34.384490966796875,
      "activations/layer15_attention_weight_min": -28.490768432617188,
      "activations/layer16_attention_weight_max": 31.834077835083008,
      "activations/layer16_attention_weight_min": -27.116031646728516,
      "activations/layer17_attention_weight_max": 33.74379348754883,
      "activations/layer17_attention_weight_min": -24.509000778198242,
      "activations/layer18_attention_weight_max": 29.77926254272461,
      "activations/layer18_attention_weight_min": -22.61251449584961,
      "activations/layer19_attention_weight_max": 35.76336669921875,
      "activations/layer19_attention_weight_min": -29.676237106323242,
      "activations/layer1_attention_weight_max": 16.57190704345703,
      "activations/layer1_attention_weight_min": -14.494507789611816,
      "activations/layer20_attention_weight_max": 30.194061279296875,
      "activations/layer20_attention_weight_min": -23.785093307495117,
      "activations/layer21_attention_weight_max": 29.362987518310547,
      "activations/layer21_attention_weight_min": -22.50482940673828,
      "activations/layer22_attention_weight_max": 48.768280029296875,
      "activations/layer22_attention_weight_min": -28.498960494995117,
      "activations/layer23_attention_weight_max": 36.976287841796875,
      "activations/layer23_attention_weight_min": -24.481290817260742,
      "activations/layer2_attention_weight_max": 31.94771957397461,
      "activations/layer2_attention_weight_min": -33.06256866455078,
      "activations/layer3_attention_weight_max": 92.30056762695312,
      "activations/layer3_attention_weight_min": -98.20343017578125,
      "activations/layer4_attention_weight_max": 114.68305969238281,
      "activations/layer4_attention_weight_min": -115.30891418457031,
      "activations/layer5_attention_weight_max": 54.9822998046875,
      "activations/layer5_attention_weight_min": -61.658164978027344,
      "activations/layer6_attention_weight_max": 47.5753059387207,
      "activations/layer6_attention_weight_min": -48.81800842285156,
      "activations/layer7_attention_weight_max": 85.85382080078125,
      "activations/layer7_attention_weight_min": -83.5770034790039,
      "activations/layer8_attention_weight_max": 39.615516662597656,
      "activations/layer8_attention_weight_min": -39.787017822265625,
      "activations/layer9_attention_weight_max": 35.1124382019043,
      "activations/layer9_attention_weight_min": -33.12697982788086,
      "epoch": 9.08,
      "learning_rate": 9.240037878787879e-05,
      "loss": 2.7829,
      "step": 156200
    },
    {
      "activations/layer0_attention_weight_max": 15.334057807922363,
      "activations/layer0_attention_weight_min": -12.037540435791016,
      "activations/layer10_attention_weight_max": 33.95485305786133,
      "activations/layer10_attention_weight_min": -35.485382080078125,
      "activations/layer11_attention_weight_max": 31.44919204711914,
      "activations/layer11_attention_weight_min": -33.427921295166016,
      "activations/layer12_attention_weight_max": 23.367408752441406,
      "activations/layer12_attention_weight_min": -24.60184669494629,
      "activations/layer13_attention_weight_max": 40.46458435058594,
      "activations/layer13_attention_weight_min": -35.457786560058594,
      "activations/layer14_attention_weight_max": 46.07769012451172,
      "activations/layer14_attention_weight_min": -38.558746337890625,
      "activations/layer15_attention_weight_max": 33.92154312133789,
      "activations/layer15_attention_weight_min": -30.304859161376953,
      "activations/layer16_attention_weight_max": 32.39491653442383,
      "activations/layer16_attention_weight_min": -28.255578994750977,
      "activations/layer17_attention_weight_max": 32.46826171875,
      "activations/layer17_attention_weight_min": -24.026275634765625,
      "activations/layer18_attention_weight_max": 30.826223373413086,
      "activations/layer18_attention_weight_min": -21.045080184936523,
      "activations/layer19_attention_weight_max": 37.04463195800781,
      "activations/layer19_attention_weight_min": -29.462121963500977,
      "activations/layer1_attention_weight_max": 16.4981746673584,
      "activations/layer1_attention_weight_min": -16.54932975769043,
      "activations/layer20_attention_weight_max": 30.716026306152344,
      "activations/layer20_attention_weight_min": -23.605985641479492,
      "activations/layer21_attention_weight_max": 27.946727752685547,
      "activations/layer21_attention_weight_min": -23.045005798339844,
      "activations/layer22_attention_weight_max": 48.6905403137207,
      "activations/layer22_attention_weight_min": -30.0033016204834,
      "activations/layer23_attention_weight_max": 38.67029571533203,
      "activations/layer23_attention_weight_min": -23.483749389648438,
      "activations/layer2_attention_weight_max": 32.080753326416016,
      "activations/layer2_attention_weight_min": -34.22221374511719,
      "activations/layer3_attention_weight_max": 98.45851135253906,
      "activations/layer3_attention_weight_min": -100.48296356201172,
      "activations/layer4_attention_weight_max": 117.43959045410156,
      "activations/layer4_attention_weight_min": -121.9539794921875,
      "activations/layer5_attention_weight_max": 55.124813079833984,
      "activations/layer5_attention_weight_min": -61.99990463256836,
      "activations/layer6_attention_weight_max": 45.126651763916016,
      "activations/layer6_attention_weight_min": -50.01606369018555,
      "activations/layer7_attention_weight_max": 90.33943939208984,
      "activations/layer7_attention_weight_min": -92.69506072998047,
      "activations/layer8_attention_weight_max": 40.06285095214844,
      "activations/layer8_attention_weight_min": -43.240203857421875,
      "activations/layer9_attention_weight_max": 35.526756286621094,
      "activations/layer9_attention_weight_min": -35.58271408081055,
      "epoch": 9.08,
      "learning_rate": 9.238143939393939e-05,
      "loss": 2.8011,
      "step": 156250
    },
    {
      "activations/layer0_attention_weight_max": 14.710127830505371,
      "activations/layer0_attention_weight_min": -11.67299747467041,
      "activations/layer10_attention_weight_max": 32.84360885620117,
      "activations/layer10_attention_weight_min": -29.96636199951172,
      "activations/layer11_attention_weight_max": 31.957191467285156,
      "activations/layer11_attention_weight_min": -29.897777557373047,
      "activations/layer12_attention_weight_max": 23.69596290588379,
      "activations/layer12_attention_weight_min": -25.052595138549805,
      "activations/layer13_attention_weight_max": 38.837799072265625,
      "activations/layer13_attention_weight_min": -31.03826141357422,
      "activations/layer14_attention_weight_max": 49.20115280151367,
      "activations/layer14_attention_weight_min": -40.629329681396484,
      "activations/layer15_attention_weight_max": 35.77081298828125,
      "activations/layer15_attention_weight_min": -28.775653839111328,
      "activations/layer16_attention_weight_max": 31.736133575439453,
      "activations/layer16_attention_weight_min": -26.153045654296875,
      "activations/layer17_attention_weight_max": 33.756221771240234,
      "activations/layer17_attention_weight_min": -25.297861099243164,
      "activations/layer18_attention_weight_max": 33.66548156738281,
      "activations/layer18_attention_weight_min": -20.77711296081543,
      "activations/layer19_attention_weight_max": 34.55877685546875,
      "activations/layer19_attention_weight_min": -29.123859405517578,
      "activations/layer1_attention_weight_max": 16.37964630126953,
      "activations/layer1_attention_weight_min": -13.300995826721191,
      "activations/layer20_attention_weight_max": 29.70966148376465,
      "activations/layer20_attention_weight_min": -22.50011444091797,
      "activations/layer21_attention_weight_max": 31.058799743652344,
      "activations/layer21_attention_weight_min": -21.628385543823242,
      "activations/layer22_attention_weight_max": 44.857810974121094,
      "activations/layer22_attention_weight_min": -28.223529815673828,
      "activations/layer23_attention_weight_max": 35.26847839355469,
      "activations/layer23_attention_weight_min": -24.817626953125,
      "activations/layer2_attention_weight_max": 32.05632781982422,
      "activations/layer2_attention_weight_min": -33.28813171386719,
      "activations/layer3_attention_weight_max": 99.72001647949219,
      "activations/layer3_attention_weight_min": -101.43682861328125,
      "activations/layer4_attention_weight_max": 116.82976531982422,
      "activations/layer4_attention_weight_min": -116.38480377197266,
      "activations/layer5_attention_weight_max": 54.89217758178711,
      "activations/layer5_attention_weight_min": -61.31563949584961,
      "activations/layer6_attention_weight_max": 51.39585494995117,
      "activations/layer6_attention_weight_min": -49.88079071044922,
      "activations/layer7_attention_weight_max": 92.06106567382812,
      "activations/layer7_attention_weight_min": -85.79109191894531,
      "activations/layer8_attention_weight_max": 41.019622802734375,
      "activations/layer8_attention_weight_min": -38.81294631958008,
      "activations/layer9_attention_weight_max": 34.89647674560547,
      "activations/layer9_attention_weight_min": -31.565427780151367,
      "epoch": 9.08,
      "learning_rate": 9.236249999999999e-05,
      "loss": 2.7771,
      "step": 156300
    },
    {
      "activations/layer0_attention_weight_max": 17.331504821777344,
      "activations/layer0_attention_weight_min": -11.857565879821777,
      "activations/layer10_attention_weight_max": 31.08817481994629,
      "activations/layer10_attention_weight_min": -31.209911346435547,
      "activations/layer11_attention_weight_max": 31.068511962890625,
      "activations/layer11_attention_weight_min": -30.027202606201172,
      "activations/layer12_attention_weight_max": 22.870996475219727,
      "activations/layer12_attention_weight_min": -26.062339782714844,
      "activations/layer13_attention_weight_max": 37.03486251831055,
      "activations/layer13_attention_weight_min": -36.91309356689453,
      "activations/layer14_attention_weight_max": 42.00463104248047,
      "activations/layer14_attention_weight_min": -42.02215576171875,
      "activations/layer15_attention_weight_max": 34.9265022277832,
      "activations/layer15_attention_weight_min": -30.953617095947266,
      "activations/layer16_attention_weight_max": 29.46756935119629,
      "activations/layer16_attention_weight_min": -27.957849502563477,
      "activations/layer17_attention_weight_max": 31.41777229309082,
      "activations/layer17_attention_weight_min": -26.41616439819336,
      "activations/layer18_attention_weight_max": 29.521310806274414,
      "activations/layer18_attention_weight_min": -21.105985641479492,
      "activations/layer19_attention_weight_max": 31.019620895385742,
      "activations/layer19_attention_weight_min": -29.54182243347168,
      "activations/layer1_attention_weight_max": 16.85480499267578,
      "activations/layer1_attention_weight_min": -14.23513126373291,
      "activations/layer20_attention_weight_max": 27.98716926574707,
      "activations/layer20_attention_weight_min": -23.64451789855957,
      "activations/layer21_attention_weight_max": 25.92032241821289,
      "activations/layer21_attention_weight_min": -23.330163955688477,
      "activations/layer22_attention_weight_max": 42.24557113647461,
      "activations/layer22_attention_weight_min": -27.336572647094727,
      "activations/layer23_attention_weight_max": 36.49968719482422,
      "activations/layer23_attention_weight_min": -24.628433227539062,
      "activations/layer2_attention_weight_max": 33.10384750366211,
      "activations/layer2_attention_weight_min": -32.989383697509766,
      "activations/layer3_attention_weight_max": 92.60469055175781,
      "activations/layer3_attention_weight_min": -97.70355224609375,
      "activations/layer4_attention_weight_max": 117.43202209472656,
      "activations/layer4_attention_weight_min": -114.92253875732422,
      "activations/layer5_attention_weight_max": 51.04602813720703,
      "activations/layer5_attention_weight_min": -62.032283782958984,
      "activations/layer6_attention_weight_max": 46.051597595214844,
      "activations/layer6_attention_weight_min": -47.28919219970703,
      "activations/layer7_attention_weight_max": 82.35148620605469,
      "activations/layer7_attention_weight_min": -89.95104217529297,
      "activations/layer8_attention_weight_max": 38.2075309753418,
      "activations/layer8_attention_weight_min": -39.41862106323242,
      "activations/layer9_attention_weight_max": 31.756067276000977,
      "activations/layer9_attention_weight_min": -32.098384857177734,
      "epoch": 9.08,
      "learning_rate": 9.234356060606059e-05,
      "loss": 2.7829,
      "step": 156350
    },
    {
      "activations/layer0_attention_weight_max": 15.171249389648438,
      "activations/layer0_attention_weight_min": -13.405582427978516,
      "activations/layer10_attention_weight_max": 33.546470642089844,
      "activations/layer10_attention_weight_min": -33.180702209472656,
      "activations/layer11_attention_weight_max": 33.44313430786133,
      "activations/layer11_attention_weight_min": -31.816099166870117,
      "activations/layer12_attention_weight_max": 26.309762954711914,
      "activations/layer12_attention_weight_min": -26.803998947143555,
      "activations/layer13_attention_weight_max": 41.966552734375,
      "activations/layer13_attention_weight_min": -33.76092529296875,
      "activations/layer14_attention_weight_max": 47.59031295776367,
      "activations/layer14_attention_weight_min": -37.49782943725586,
      "activations/layer15_attention_weight_max": 37.5824089050293,
      "activations/layer15_attention_weight_min": -33.007110595703125,
      "activations/layer16_attention_weight_max": 30.824352264404297,
      "activations/layer16_attention_weight_min": -28.895021438598633,
      "activations/layer17_attention_weight_max": 33.957523345947266,
      "activations/layer17_attention_weight_min": -28.018627166748047,
      "activations/layer18_attention_weight_max": 31.733989715576172,
      "activations/layer18_attention_weight_min": -23.090694427490234,
      "activations/layer19_attention_weight_max": 35.215641021728516,
      "activations/layer19_attention_weight_min": -29.589569091796875,
      "activations/layer1_attention_weight_max": 16.568134307861328,
      "activations/layer1_attention_weight_min": -14.275274276733398,
      "activations/layer20_attention_weight_max": 31.044525146484375,
      "activations/layer20_attention_weight_min": -23.41777229309082,
      "activations/layer21_attention_weight_max": 28.29397201538086,
      "activations/layer21_attention_weight_min": -22.069082260131836,
      "activations/layer22_attention_weight_max": 46.46342086791992,
      "activations/layer22_attention_weight_min": -30.680973052978516,
      "activations/layer23_attention_weight_max": 35.94270324707031,
      "activations/layer23_attention_weight_min": -23.61566925048828,
      "activations/layer2_attention_weight_max": 32.57511901855469,
      "activations/layer2_attention_weight_min": -33.68354034423828,
      "activations/layer3_attention_weight_max": 95.7222900390625,
      "activations/layer3_attention_weight_min": -98.16594696044922,
      "activations/layer4_attention_weight_max": 120.68831634521484,
      "activations/layer4_attention_weight_min": -116.94384002685547,
      "activations/layer5_attention_weight_max": 56.83802032470703,
      "activations/layer5_attention_weight_min": -60.62856674194336,
      "activations/layer6_attention_weight_max": 47.619144439697266,
      "activations/layer6_attention_weight_min": -52.974239349365234,
      "activations/layer7_attention_weight_max": 93.60990142822266,
      "activations/layer7_attention_weight_min": -88.76539611816406,
      "activations/layer8_attention_weight_max": 42.78444290161133,
      "activations/layer8_attention_weight_min": -41.48357391357422,
      "activations/layer9_attention_weight_max": 33.78873825073242,
      "activations/layer9_attention_weight_min": -36.84368133544922,
      "epoch": 9.09,
      "learning_rate": 9.23246212121212e-05,
      "loss": 2.8051,
      "step": 156400
    },
    {
      "activations/layer0_attention_weight_max": 15.475478172302246,
      "activations/layer0_attention_weight_min": -11.395559310913086,
      "activations/layer10_attention_weight_max": 34.349090576171875,
      "activations/layer10_attention_weight_min": -31.397720336914062,
      "activations/layer11_attention_weight_max": 32.921180725097656,
      "activations/layer11_attention_weight_min": -31.244823455810547,
      "activations/layer12_attention_weight_max": 24.33648681640625,
      "activations/layer12_attention_weight_min": -24.024463653564453,
      "activations/layer13_attention_weight_max": 40.82280349731445,
      "activations/layer13_attention_weight_min": -33.3582878112793,
      "activations/layer14_attention_weight_max": 53.2459716796875,
      "activations/layer14_attention_weight_min": -44.165306091308594,
      "activations/layer15_attention_weight_max": 36.74953079223633,
      "activations/layer15_attention_weight_min": -31.86239242553711,
      "activations/layer16_attention_weight_max": 34.17182540893555,
      "activations/layer16_attention_weight_min": -28.633150100708008,
      "activations/layer17_attention_weight_max": 33.25044631958008,
      "activations/layer17_attention_weight_min": -25.60455894470215,
      "activations/layer18_attention_weight_max": 31.46612548828125,
      "activations/layer18_attention_weight_min": -21.908220291137695,
      "activations/layer19_attention_weight_max": 38.98801040649414,
      "activations/layer19_attention_weight_min": -31.325510025024414,
      "activations/layer1_attention_weight_max": 16.80220603942871,
      "activations/layer1_attention_weight_min": -13.685319900512695,
      "activations/layer20_attention_weight_max": 30.865943908691406,
      "activations/layer20_attention_weight_min": -24.93483543395996,
      "activations/layer21_attention_weight_max": 33.0252685546875,
      "activations/layer21_attention_weight_min": -22.700769424438477,
      "activations/layer22_attention_weight_max": 51.05644989013672,
      "activations/layer22_attention_weight_min": -27.15239906311035,
      "activations/layer23_attention_weight_max": 36.23319625854492,
      "activations/layer23_attention_weight_min": -23.613067626953125,
      "activations/layer2_attention_weight_max": 32.830318450927734,
      "activations/layer2_attention_weight_min": -32.745731353759766,
      "activations/layer3_attention_weight_max": 98.50049591064453,
      "activations/layer3_attention_weight_min": -95.0947265625,
      "activations/layer4_attention_weight_max": 124.17192840576172,
      "activations/layer4_attention_weight_min": -114.93730926513672,
      "activations/layer5_attention_weight_max": 55.09513473510742,
      "activations/layer5_attention_weight_min": -63.53971862792969,
      "activations/layer6_attention_weight_max": 46.85050964355469,
      "activations/layer6_attention_weight_min": -51.473060607910156,
      "activations/layer7_attention_weight_max": 97.0347900390625,
      "activations/layer7_attention_weight_min": -92.62340545654297,
      "activations/layer8_attention_weight_max": 41.9860725402832,
      "activations/layer8_attention_weight_min": -41.428531646728516,
      "activations/layer9_attention_weight_max": 36.010162353515625,
      "activations/layer9_attention_weight_min": -34.73298263549805,
      "epoch": 9.09,
      "learning_rate": 9.230568181818182e-05,
      "loss": 2.7936,
      "step": 156450
    },
    {
      "activations/layer0_attention_weight_max": 16.222986221313477,
      "activations/layer0_attention_weight_min": -11.711773872375488,
      "activations/layer10_attention_weight_max": 31.56944465637207,
      "activations/layer10_attention_weight_min": -32.30524826049805,
      "activations/layer11_attention_weight_max": 30.347049713134766,
      "activations/layer11_attention_weight_min": -32.34017562866211,
      "activations/layer12_attention_weight_max": 22.945337295532227,
      "activations/layer12_attention_weight_min": -23.728525161743164,
      "activations/layer13_attention_weight_max": 37.318702697753906,
      "activations/layer13_attention_weight_min": -31.77187728881836,
      "activations/layer14_attention_weight_max": 39.72051239013672,
      "activations/layer14_attention_weight_min": -36.22336959838867,
      "activations/layer15_attention_weight_max": 35.37842559814453,
      "activations/layer15_attention_weight_min": -30.273588180541992,
      "activations/layer16_attention_weight_max": 31.245723724365234,
      "activations/layer16_attention_weight_min": -28.239253997802734,
      "activations/layer17_attention_weight_max": 33.622005462646484,
      "activations/layer17_attention_weight_min": -24.85057830810547,
      "activations/layer18_attention_weight_max": 29.805513381958008,
      "activations/layer18_attention_weight_min": -22.49678611755371,
      "activations/layer19_attention_weight_max": 38.2518424987793,
      "activations/layer19_attention_weight_min": -34.03838348388672,
      "activations/layer1_attention_weight_max": 16.032434463500977,
      "activations/layer1_attention_weight_min": -14.160534858703613,
      "activations/layer20_attention_weight_max": 30.834606170654297,
      "activations/layer20_attention_weight_min": -26.686450958251953,
      "activations/layer21_attention_weight_max": 29.30021095275879,
      "activations/layer21_attention_weight_min": -23.714860916137695,
      "activations/layer22_attention_weight_max": 45.959835052490234,
      "activations/layer22_attention_weight_min": -30.359365463256836,
      "activations/layer23_attention_weight_max": 36.787235260009766,
      "activations/layer23_attention_weight_min": -22.56825065612793,
      "activations/layer2_attention_weight_max": 33.190223693847656,
      "activations/layer2_attention_weight_min": -33.27091979980469,
      "activations/layer3_attention_weight_max": 95.97909545898438,
      "activations/layer3_attention_weight_min": -99.596923828125,
      "activations/layer4_attention_weight_max": 119.53536224365234,
      "activations/layer4_attention_weight_min": -116.43025207519531,
      "activations/layer5_attention_weight_max": 53.33897399902344,
      "activations/layer5_attention_weight_min": -61.21499252319336,
      "activations/layer6_attention_weight_max": 44.99563980102539,
      "activations/layer6_attention_weight_min": -50.036258697509766,
      "activations/layer7_attention_weight_max": 87.31487274169922,
      "activations/layer7_attention_weight_min": -93.37974548339844,
      "activations/layer8_attention_weight_max": 37.277557373046875,
      "activations/layer8_attention_weight_min": -39.547855377197266,
      "activations/layer9_attention_weight_max": 31.891233444213867,
      "activations/layer9_attention_weight_min": -33.01766586303711,
      "epoch": 9.09,
      "learning_rate": 9.228674242424241e-05,
      "loss": 2.7859,
      "step": 156500
    },
    {
      "activations/layer0_attention_weight_max": 16.443574905395508,
      "activations/layer0_attention_weight_min": -12.433293342590332,
      "activations/layer10_attention_weight_max": 39.451290130615234,
      "activations/layer10_attention_weight_min": -36.708160400390625,
      "activations/layer11_attention_weight_max": 39.44343185424805,
      "activations/layer11_attention_weight_min": -33.558570861816406,
      "activations/layer12_attention_weight_max": 26.153587341308594,
      "activations/layer12_attention_weight_min": -28.002479553222656,
      "activations/layer13_attention_weight_max": 43.73114776611328,
      "activations/layer13_attention_weight_min": -34.243492126464844,
      "activations/layer14_attention_weight_max": 53.697975158691406,
      "activations/layer14_attention_weight_min": -42.7497673034668,
      "activations/layer15_attention_weight_max": 55.61723709106445,
      "activations/layer15_attention_weight_min": -36.12888717651367,
      "activations/layer16_attention_weight_max": 35.138885498046875,
      "activations/layer16_attention_weight_min": -28.08144760131836,
      "activations/layer17_attention_weight_max": 37.0651741027832,
      "activations/layer17_attention_weight_min": -25.756500244140625,
      "activations/layer18_attention_weight_max": 31.561443328857422,
      "activations/layer18_attention_weight_min": -22.152936935424805,
      "activations/layer19_attention_weight_max": 34.75544357299805,
      "activations/layer19_attention_weight_min": -31.465988159179688,
      "activations/layer1_attention_weight_max": 17.19112777709961,
      "activations/layer1_attention_weight_min": -14.623252868652344,
      "activations/layer20_attention_weight_max": 28.969913482666016,
      "activations/layer20_attention_weight_min": -25.148143768310547,
      "activations/layer21_attention_weight_max": 30.302982330322266,
      "activations/layer21_attention_weight_min": -22.613079071044922,
      "activations/layer22_attention_weight_max": 41.96533966064453,
      "activations/layer22_attention_weight_min": -30.54066276550293,
      "activations/layer23_attention_weight_max": 32.44558334350586,
      "activations/layer23_attention_weight_min": -24.056684494018555,
      "activations/layer2_attention_weight_max": 36.31289291381836,
      "activations/layer2_attention_weight_min": -33.740814208984375,
      "activations/layer3_attention_weight_max": 103.01434326171875,
      "activations/layer3_attention_weight_min": -102.41857147216797,
      "activations/layer4_attention_weight_max": 128.274658203125,
      "activations/layer4_attention_weight_min": -124.79743957519531,
      "activations/layer5_attention_weight_max": 56.860504150390625,
      "activations/layer5_attention_weight_min": -66.77841186523438,
      "activations/layer6_attention_weight_max": 48.48405838012695,
      "activations/layer6_attention_weight_min": -52.72136306762695,
      "activations/layer7_attention_weight_max": 95.06485748291016,
      "activations/layer7_attention_weight_min": -93.76548767089844,
      "activations/layer8_attention_weight_max": 45.096981048583984,
      "activations/layer8_attention_weight_min": -45.047752380371094,
      "activations/layer9_attention_weight_max": 40.27997589111328,
      "activations/layer9_attention_weight_min": -37.5670280456543,
      "epoch": 9.1,
      "learning_rate": 9.226780303030302e-05,
      "loss": 2.7802,
      "step": 156550
    },
    {
      "activations/layer0_attention_weight_max": 15.186843872070312,
      "activations/layer0_attention_weight_min": -12.208913803100586,
      "activations/layer10_attention_weight_max": 33.367950439453125,
      "activations/layer10_attention_weight_min": -31.15021324157715,
      "activations/layer11_attention_weight_max": 32.123931884765625,
      "activations/layer11_attention_weight_min": -29.653789520263672,
      "activations/layer12_attention_weight_max": 21.537616729736328,
      "activations/layer12_attention_weight_min": -23.66748809814453,
      "activations/layer13_attention_weight_max": 35.227020263671875,
      "activations/layer13_attention_weight_min": -33.595333099365234,
      "activations/layer14_attention_weight_max": 48.907737731933594,
      "activations/layer14_attention_weight_min": -38.55827331542969,
      "activations/layer15_attention_weight_max": 36.542259216308594,
      "activations/layer15_attention_weight_min": -30.499296188354492,
      "activations/layer16_attention_weight_max": 31.556838989257812,
      "activations/layer16_attention_weight_min": -27.94403076171875,
      "activations/layer17_attention_weight_max": 32.39348602294922,
      "activations/layer17_attention_weight_min": -26.62444496154785,
      "activations/layer18_attention_weight_max": 31.22671127319336,
      "activations/layer18_attention_weight_min": -21.048933029174805,
      "activations/layer19_attention_weight_max": 37.13991165161133,
      "activations/layer19_attention_weight_min": -31.531522750854492,
      "activations/layer1_attention_weight_max": 15.99700927734375,
      "activations/layer1_attention_weight_min": -15.06450366973877,
      "activations/layer20_attention_weight_max": 31.582712173461914,
      "activations/layer20_attention_weight_min": -24.671951293945312,
      "activations/layer21_attention_weight_max": 33.11306381225586,
      "activations/layer21_attention_weight_min": -24.48528289794922,
      "activations/layer22_attention_weight_max": 42.52253341674805,
      "activations/layer22_attention_weight_min": -28.00642204284668,
      "activations/layer23_attention_weight_max": 33.856075286865234,
      "activations/layer23_attention_weight_min": -23.47061538696289,
      "activations/layer2_attention_weight_max": 32.87099075317383,
      "activations/layer2_attention_weight_min": -33.14728546142578,
      "activations/layer3_attention_weight_max": 106.21739196777344,
      "activations/layer3_attention_weight_min": -103.61187744140625,
      "activations/layer4_attention_weight_max": 122.55989074707031,
      "activations/layer4_attention_weight_min": -117.5450439453125,
      "activations/layer5_attention_weight_max": 52.73749923706055,
      "activations/layer5_attention_weight_min": -61.63512420654297,
      "activations/layer6_attention_weight_max": 46.54877471923828,
      "activations/layer6_attention_weight_min": -51.25751876831055,
      "activations/layer7_attention_weight_max": 91.98645782470703,
      "activations/layer7_attention_weight_min": -87.18843841552734,
      "activations/layer8_attention_weight_max": 39.77299880981445,
      "activations/layer8_attention_weight_min": -41.23775863647461,
      "activations/layer9_attention_weight_max": 34.187705993652344,
      "activations/layer9_attention_weight_min": -33.266971588134766,
      "epoch": 9.1,
      "learning_rate": 9.224886363636364e-05,
      "loss": 2.8013,
      "step": 156600
    },
    {
      "activations/layer0_attention_weight_max": 15.743759155273438,
      "activations/layer0_attention_weight_min": -12.931031227111816,
      "activations/layer10_attention_weight_max": 30.977453231811523,
      "activations/layer10_attention_weight_min": -30.820213317871094,
      "activations/layer11_attention_weight_max": 30.700658798217773,
      "activations/layer11_attention_weight_min": -29.782896041870117,
      "activations/layer12_attention_weight_max": 24.315078735351562,
      "activations/layer12_attention_weight_min": -23.905941009521484,
      "activations/layer13_attention_weight_max": 38.004573822021484,
      "activations/layer13_attention_weight_min": -33.44092559814453,
      "activations/layer14_attention_weight_max": 47.896541595458984,
      "activations/layer14_attention_weight_min": -36.55332946777344,
      "activations/layer15_attention_weight_max": 37.171409606933594,
      "activations/layer15_attention_weight_min": -30.442955017089844,
      "activations/layer16_attention_weight_max": 35.08588790893555,
      "activations/layer16_attention_weight_min": -27.452268600463867,
      "activations/layer17_attention_weight_max": 33.615570068359375,
      "activations/layer17_attention_weight_min": -24.85519790649414,
      "activations/layer18_attention_weight_max": 29.481889724731445,
      "activations/layer18_attention_weight_min": -20.822038650512695,
      "activations/layer19_attention_weight_max": 38.57268524169922,
      "activations/layer19_attention_weight_min": -30.0698184967041,
      "activations/layer1_attention_weight_max": 17.300273895263672,
      "activations/layer1_attention_weight_min": -14.99721908569336,
      "activations/layer20_attention_weight_max": 30.94573211669922,
      "activations/layer20_attention_weight_min": -23.982852935791016,
      "activations/layer21_attention_weight_max": 30.428428649902344,
      "activations/layer21_attention_weight_min": -22.93962287902832,
      "activations/layer22_attention_weight_max": 45.244014739990234,
      "activations/layer22_attention_weight_min": -29.803878784179688,
      "activations/layer23_attention_weight_max": 38.615966796875,
      "activations/layer23_attention_weight_min": -24.02766227722168,
      "activations/layer2_attention_weight_max": 32.378841400146484,
      "activations/layer2_attention_weight_min": -32.861602783203125,
      "activations/layer3_attention_weight_max": 95.13065338134766,
      "activations/layer3_attention_weight_min": -95.3609390258789,
      "activations/layer4_attention_weight_max": 116.46846008300781,
      "activations/layer4_attention_weight_min": -115.43511962890625,
      "activations/layer5_attention_weight_max": 57.07731628417969,
      "activations/layer5_attention_weight_min": -60.20909881591797,
      "activations/layer6_attention_weight_max": 47.03507614135742,
      "activations/layer6_attention_weight_min": -46.65793228149414,
      "activations/layer7_attention_weight_max": 88.66043853759766,
      "activations/layer7_attention_weight_min": -83.16650390625,
      "activations/layer8_attention_weight_max": 39.26957321166992,
      "activations/layer8_attention_weight_min": -37.48664474487305,
      "activations/layer9_attention_weight_max": 32.494712829589844,
      "activations/layer9_attention_weight_min": -31.488906860351562,
      "epoch": 9.1,
      "learning_rate": 9.222992424242423e-05,
      "loss": 2.8018,
      "step": 156650
    },
    {
      "activations/layer0_attention_weight_max": 15.06407356262207,
      "activations/layer0_attention_weight_min": -12.914420127868652,
      "activations/layer10_attention_weight_max": 40.68506622314453,
      "activations/layer10_attention_weight_min": -34.21170425415039,
      "activations/layer11_attention_weight_max": 40.376708984375,
      "activations/layer11_attention_weight_min": -33.10728454589844,
      "activations/layer12_attention_weight_max": 23.6843318939209,
      "activations/layer12_attention_weight_min": -24.322132110595703,
      "activations/layer13_attention_weight_max": 43.767154693603516,
      "activations/layer13_attention_weight_min": -33.84344482421875,
      "activations/layer14_attention_weight_max": 51.320648193359375,
      "activations/layer14_attention_weight_min": -38.15897750854492,
      "activations/layer15_attention_weight_max": 41.85164260864258,
      "activations/layer15_attention_weight_min": -28.54298210144043,
      "activations/layer16_attention_weight_max": 32.19630813598633,
      "activations/layer16_attention_weight_min": -27.225475311279297,
      "activations/layer17_attention_weight_max": 31.187585830688477,
      "activations/layer17_attention_weight_min": -27.557369232177734,
      "activations/layer18_attention_weight_max": 28.986492156982422,
      "activations/layer18_attention_weight_min": -22.06489372253418,
      "activations/layer19_attention_weight_max": 33.74818801879883,
      "activations/layer19_attention_weight_min": -30.52935791015625,
      "activations/layer1_attention_weight_max": 18.064632415771484,
      "activations/layer1_attention_weight_min": -14.891202926635742,
      "activations/layer20_attention_weight_max": 28.633764266967773,
      "activations/layer20_attention_weight_min": -22.07521629333496,
      "activations/layer21_attention_weight_max": 26.677642822265625,
      "activations/layer21_attention_weight_min": -23.10304069519043,
      "activations/layer22_attention_weight_max": 42.78505325317383,
      "activations/layer22_attention_weight_min": -28.749591827392578,
      "activations/layer23_attention_weight_max": 33.88164520263672,
      "activations/layer23_attention_weight_min": -24.127593994140625,
      "activations/layer2_attention_weight_max": 34.326416015625,
      "activations/layer2_attention_weight_min": -34.66002655029297,
      "activations/layer3_attention_weight_max": 94.82320404052734,
      "activations/layer3_attention_weight_min": -96.13738250732422,
      "activations/layer4_attention_weight_max": 115.007080078125,
      "activations/layer4_attention_weight_min": -112.89188385009766,
      "activations/layer5_attention_weight_max": 53.60685348510742,
      "activations/layer5_attention_weight_min": -63.86168670654297,
      "activations/layer6_attention_weight_max": 47.19816589355469,
      "activations/layer6_attention_weight_min": -48.79231643676758,
      "activations/layer7_attention_weight_max": 101.59751892089844,
      "activations/layer7_attention_weight_min": -89.92652130126953,
      "activations/layer8_attention_weight_max": 44.86525344848633,
      "activations/layer8_attention_weight_min": -42.07124328613281,
      "activations/layer9_attention_weight_max": 38.844051361083984,
      "activations/layer9_attention_weight_min": -34.75811004638672,
      "epoch": 9.11,
      "learning_rate": 9.221098484848484e-05,
      "loss": 2.7953,
      "step": 156700
    },
    {
      "activations/layer0_attention_weight_max": 15.570502281188965,
      "activations/layer0_attention_weight_min": -12.587040901184082,
      "activations/layer10_attention_weight_max": 31.75449562072754,
      "activations/layer10_attention_weight_min": -31.500972747802734,
      "activations/layer11_attention_weight_max": 32.63957595825195,
      "activations/layer11_attention_weight_min": -29.90218162536621,
      "activations/layer12_attention_weight_max": 24.65687370300293,
      "activations/layer12_attention_weight_min": -27.798934936523438,
      "activations/layer13_attention_weight_max": 39.69161605834961,
      "activations/layer13_attention_weight_min": -34.88669967651367,
      "activations/layer14_attention_weight_max": 48.801414489746094,
      "activations/layer14_attention_weight_min": -37.65565872192383,
      "activations/layer15_attention_weight_max": 37.07713317871094,
      "activations/layer15_attention_weight_min": -30.07513427734375,
      "activations/layer16_attention_weight_max": 34.117610931396484,
      "activations/layer16_attention_weight_min": -27.363435745239258,
      "activations/layer17_attention_weight_max": 36.6524543762207,
      "activations/layer17_attention_weight_min": -25.987903594970703,
      "activations/layer18_attention_weight_max": 28.820608139038086,
      "activations/layer18_attention_weight_min": -21.304500579833984,
      "activations/layer19_attention_weight_max": 35.9845085144043,
      "activations/layer19_attention_weight_min": -30.203948974609375,
      "activations/layer1_attention_weight_max": 16.076522827148438,
      "activations/layer1_attention_weight_min": -15.37269115447998,
      "activations/layer20_attention_weight_max": 31.83831214904785,
      "activations/layer20_attention_weight_min": -24.19611358642578,
      "activations/layer21_attention_weight_max": 27.486276626586914,
      "activations/layer21_attention_weight_min": -23.062856674194336,
      "activations/layer22_attention_weight_max": 46.89396286010742,
      "activations/layer22_attention_weight_min": -28.29363441467285,
      "activations/layer23_attention_weight_max": 39.356475830078125,
      "activations/layer23_attention_weight_min": -23.540206909179688,
      "activations/layer2_attention_weight_max": 33.95490646362305,
      "activations/layer2_attention_weight_min": -31.732540130615234,
      "activations/layer3_attention_weight_max": 94.56273651123047,
      "activations/layer3_attention_weight_min": -94.5001449584961,
      "activations/layer4_attention_weight_max": 118.09679412841797,
      "activations/layer4_attention_weight_min": -116.11946868896484,
      "activations/layer5_attention_weight_max": 52.51813507080078,
      "activations/layer5_attention_weight_min": -60.653358459472656,
      "activations/layer6_attention_weight_max": 46.12125015258789,
      "activations/layer6_attention_weight_min": -48.558738708496094,
      "activations/layer7_attention_weight_max": 94.34620666503906,
      "activations/layer7_attention_weight_min": -89.08338165283203,
      "activations/layer8_attention_weight_max": 41.76790237426758,
      "activations/layer8_attention_weight_min": -42.666847229003906,
      "activations/layer9_attention_weight_max": 36.62736892700195,
      "activations/layer9_attention_weight_min": -34.0107421875,
      "epoch": 9.11,
      "learning_rate": 9.219204545454546e-05,
      "loss": 2.8111,
      "step": 156750
    },
    {
      "activations/layer0_attention_weight_max": 15.489398002624512,
      "activations/layer0_attention_weight_min": -11.916028022766113,
      "activations/layer10_attention_weight_max": 36.16151809692383,
      "activations/layer10_attention_weight_min": -33.602813720703125,
      "activations/layer11_attention_weight_max": 35.717247009277344,
      "activations/layer11_attention_weight_min": -33.54220962524414,
      "activations/layer12_attention_weight_max": 28.958438873291016,
      "activations/layer12_attention_weight_min": -26.448158264160156,
      "activations/layer13_attention_weight_max": 47.010433197021484,
      "activations/layer13_attention_weight_min": -34.32978439331055,
      "activations/layer14_attention_weight_max": 63.28549575805664,
      "activations/layer14_attention_weight_min": -39.902793884277344,
      "activations/layer15_attention_weight_max": 41.30649185180664,
      "activations/layer15_attention_weight_min": -32.362003326416016,
      "activations/layer16_attention_weight_max": 37.66276931762695,
      "activations/layer16_attention_weight_min": -27.787370681762695,
      "activations/layer17_attention_weight_max": 37.622432708740234,
      "activations/layer17_attention_weight_min": -27.03236198425293,
      "activations/layer18_attention_weight_max": 34.4414176940918,
      "activations/layer18_attention_weight_min": -22.62458610534668,
      "activations/layer19_attention_weight_max": 44.989315032958984,
      "activations/layer19_attention_weight_min": -31.459630966186523,
      "activations/layer1_attention_weight_max": 16.427520751953125,
      "activations/layer1_attention_weight_min": -14.437973022460938,
      "activations/layer20_attention_weight_max": 37.48421096801758,
      "activations/layer20_attention_weight_min": -24.294816970825195,
      "activations/layer21_attention_weight_max": 32.40937423706055,
      "activations/layer21_attention_weight_min": -25.160966873168945,
      "activations/layer22_attention_weight_max": 56.628753662109375,
      "activations/layer22_attention_weight_min": -29.29486846923828,
      "activations/layer23_attention_weight_max": 40.67326354980469,
      "activations/layer23_attention_weight_min": -24.04366111755371,
      "activations/layer2_attention_weight_max": 33.56708526611328,
      "activations/layer2_attention_weight_min": -32.88697052001953,
      "activations/layer3_attention_weight_max": 99.7314453125,
      "activations/layer3_attention_weight_min": -101.58062744140625,
      "activations/layer4_attention_weight_max": 123.1838150024414,
      "activations/layer4_attention_weight_min": -117.38177490234375,
      "activations/layer5_attention_weight_max": 57.59708786010742,
      "activations/layer5_attention_weight_min": -63.82889938354492,
      "activations/layer6_attention_weight_max": 48.881595611572266,
      "activations/layer6_attention_weight_min": -49.500057220458984,
      "activations/layer7_attention_weight_max": 90.45283508300781,
      "activations/layer7_attention_weight_min": -90.27751922607422,
      "activations/layer8_attention_weight_max": 43.702701568603516,
      "activations/layer8_attention_weight_min": -41.997215270996094,
      "activations/layer9_attention_weight_max": 39.765777587890625,
      "activations/layer9_attention_weight_min": -35.55113220214844,
      "epoch": 9.11,
      "learning_rate": 9.217310606060604e-05,
      "loss": 2.7882,
      "step": 156800
    },
    {
      "activations/layer0_attention_weight_max": 16.59897232055664,
      "activations/layer0_attention_weight_min": -12.181827545166016,
      "activations/layer10_attention_weight_max": 32.54440689086914,
      "activations/layer10_attention_weight_min": -31.613662719726562,
      "activations/layer11_attention_weight_max": 33.38854217529297,
      "activations/layer11_attention_weight_min": -30.452848434448242,
      "activations/layer12_attention_weight_max": 25.816308975219727,
      "activations/layer12_attention_weight_min": -24.147626876831055,
      "activations/layer13_attention_weight_max": 48.43959045410156,
      "activations/layer13_attention_weight_min": -31.047813415527344,
      "activations/layer14_attention_weight_max": 52.253662109375,
      "activations/layer14_attention_weight_min": -38.58934020996094,
      "activations/layer15_attention_weight_max": 38.76494598388672,
      "activations/layer15_attention_weight_min": -29.663665771484375,
      "activations/layer16_attention_weight_max": 31.043106079101562,
      "activations/layer16_attention_weight_min": -27.792022705078125,
      "activations/layer17_attention_weight_max": 33.019004821777344,
      "activations/layer17_attention_weight_min": -24.16522979736328,
      "activations/layer18_attention_weight_max": 28.867229461669922,
      "activations/layer18_attention_weight_min": -20.782726287841797,
      "activations/layer19_attention_weight_max": 31.291616439819336,
      "activations/layer19_attention_weight_min": -30.60813331604004,
      "activations/layer1_attention_weight_max": 17.187297821044922,
      "activations/layer1_attention_weight_min": -13.439499855041504,
      "activations/layer20_attention_weight_max": 26.418190002441406,
      "activations/layer20_attention_weight_min": -24.722251892089844,
      "activations/layer21_attention_weight_max": 27.537437438964844,
      "activations/layer21_attention_weight_min": -23.04776954650879,
      "activations/layer22_attention_weight_max": 38.1854248046875,
      "activations/layer22_attention_weight_min": -27.606855392456055,
      "activations/layer23_attention_weight_max": 30.903478622436523,
      "activations/layer23_attention_weight_min": -28.180370330810547,
      "activations/layer2_attention_weight_max": 32.35596466064453,
      "activations/layer2_attention_weight_min": -32.83207702636719,
      "activations/layer3_attention_weight_max": 99.4017333984375,
      "activations/layer3_attention_weight_min": -98.40229034423828,
      "activations/layer4_attention_weight_max": 118.75042724609375,
      "activations/layer4_attention_weight_min": -115.6279067993164,
      "activations/layer5_attention_weight_max": 55.85464096069336,
      "activations/layer5_attention_weight_min": -62.18266296386719,
      "activations/layer6_attention_weight_max": 47.25050735473633,
      "activations/layer6_attention_weight_min": -55.074649810791016,
      "activations/layer7_attention_weight_max": 88.49930572509766,
      "activations/layer7_attention_weight_min": -96.88311004638672,
      "activations/layer8_attention_weight_max": 38.86785125732422,
      "activations/layer8_attention_weight_min": -41.61190414428711,
      "activations/layer9_attention_weight_max": 33.15815353393555,
      "activations/layer9_attention_weight_min": -32.4148063659668,
      "epoch": 9.11,
      "learning_rate": 9.215416666666666e-05,
      "loss": 2.8018,
      "step": 156850
    },
    {
      "activations/layer0_attention_weight_max": 15.718013763427734,
      "activations/layer0_attention_weight_min": -12.092248916625977,
      "activations/layer10_attention_weight_max": 46.411163330078125,
      "activations/layer10_attention_weight_min": -45.79559326171875,
      "activations/layer11_attention_weight_max": 44.89345932006836,
      "activations/layer11_attention_weight_min": -43.89570236206055,
      "activations/layer12_attention_weight_max": 26.92826271057129,
      "activations/layer12_attention_weight_min": -28.885534286499023,
      "activations/layer13_attention_weight_max": 48.338035583496094,
      "activations/layer13_attention_weight_min": -38.939964294433594,
      "activations/layer14_attention_weight_max": 59.837860107421875,
      "activations/layer14_attention_weight_min": -48.27389144897461,
      "activations/layer15_attention_weight_max": 49.52969741821289,
      "activations/layer15_attention_weight_min": -35.48588180541992,
      "activations/layer16_attention_weight_max": 30.98683738708496,
      "activations/layer16_attention_weight_min": -26.577774047851562,
      "activations/layer17_attention_weight_max": 35.71537399291992,
      "activations/layer17_attention_weight_min": -25.792640686035156,
      "activations/layer18_attention_weight_max": 31.910776138305664,
      "activations/layer18_attention_weight_min": -22.747289657592773,
      "activations/layer19_attention_weight_max": 34.34931564331055,
      "activations/layer19_attention_weight_min": -29.552932739257812,
      "activations/layer1_attention_weight_max": 16.818784713745117,
      "activations/layer1_attention_weight_min": -16.789175033569336,
      "activations/layer20_attention_weight_max": 28.589981079101562,
      "activations/layer20_attention_weight_min": -23.812353134155273,
      "activations/layer21_attention_weight_max": 27.5092830657959,
      "activations/layer21_attention_weight_min": -22.7423095703125,
      "activations/layer22_attention_weight_max": 44.49901580810547,
      "activations/layer22_attention_weight_min": -27.826780319213867,
      "activations/layer23_attention_weight_max": 32.87623596191406,
      "activations/layer23_attention_weight_min": -24.966489791870117,
      "activations/layer2_attention_weight_max": 36.818138122558594,
      "activations/layer2_attention_weight_min": -37.165435791015625,
      "activations/layer3_attention_weight_max": 111.45449829101562,
      "activations/layer3_attention_weight_min": -115.52302551269531,
      "activations/layer4_attention_weight_max": 138.43556213378906,
      "activations/layer4_attention_weight_min": -137.60635375976562,
      "activations/layer5_attention_weight_max": 63.29273986816406,
      "activations/layer5_attention_weight_min": -62.82831573486328,
      "activations/layer6_attention_weight_max": 52.27657699584961,
      "activations/layer6_attention_weight_min": -49.990623474121094,
      "activations/layer7_attention_weight_max": 109.3211669921875,
      "activations/layer7_attention_weight_min": -97.71009826660156,
      "activations/layer8_attention_weight_max": 52.81931686401367,
      "activations/layer8_attention_weight_min": -52.621944427490234,
      "activations/layer9_attention_weight_max": 47.47683334350586,
      "activations/layer9_attention_weight_min": -43.96803283691406,
      "epoch": 9.12,
      "learning_rate": 9.213522727272726e-05,
      "loss": 2.8125,
      "step": 156900
    },
    {
      "activations/layer0_attention_weight_max": 16.491426467895508,
      "activations/layer0_attention_weight_min": -11.901874542236328,
      "activations/layer10_attention_weight_max": 34.046913146972656,
      "activations/layer10_attention_weight_min": -34.982215881347656,
      "activations/layer11_attention_weight_max": 36.42779541015625,
      "activations/layer11_attention_weight_min": -32.31406784057617,
      "activations/layer12_attention_weight_max": 24.018150329589844,
      "activations/layer12_attention_weight_min": -27.23472023010254,
      "activations/layer13_attention_weight_max": 38.902130126953125,
      "activations/layer13_attention_weight_min": -34.52634048461914,
      "activations/layer14_attention_weight_max": 48.43318176269531,
      "activations/layer14_attention_weight_min": -39.69627380371094,
      "activations/layer15_attention_weight_max": 36.72072982788086,
      "activations/layer15_attention_weight_min": -30.2150821685791,
      "activations/layer16_attention_weight_max": 35.945735931396484,
      "activations/layer16_attention_weight_min": -27.074058532714844,
      "activations/layer17_attention_weight_max": 33.79960632324219,
      "activations/layer17_attention_weight_min": -25.68366241455078,
      "activations/layer18_attention_weight_max": 29.55868148803711,
      "activations/layer18_attention_weight_min": -22.847131729125977,
      "activations/layer19_attention_weight_max": 32.39318084716797,
      "activations/layer19_attention_weight_min": -31.450902938842773,
      "activations/layer1_attention_weight_max": 16.68866729736328,
      "activations/layer1_attention_weight_min": -15.5761137008667,
      "activations/layer20_attention_weight_max": 31.44566535949707,
      "activations/layer20_attention_weight_min": -24.675397872924805,
      "activations/layer21_attention_weight_max": 27.26182746887207,
      "activations/layer21_attention_weight_min": -23.428380966186523,
      "activations/layer22_attention_weight_max": 46.34062957763672,
      "activations/layer22_attention_weight_min": -27.09590721130371,
      "activations/layer23_attention_weight_max": 40.82318115234375,
      "activations/layer23_attention_weight_min": -23.900266647338867,
      "activations/layer2_attention_weight_max": 35.441585540771484,
      "activations/layer2_attention_weight_min": -34.14999771118164,
      "activations/layer3_attention_weight_max": 98.06786346435547,
      "activations/layer3_attention_weight_min": -98.80810546875,
      "activations/layer4_attention_weight_max": 118.60725402832031,
      "activations/layer4_attention_weight_min": -121.79537200927734,
      "activations/layer5_attention_weight_max": 55.355003356933594,
      "activations/layer5_attention_weight_min": -65.06388092041016,
      "activations/layer6_attention_weight_max": 46.013980865478516,
      "activations/layer6_attention_weight_min": -50.660213470458984,
      "activations/layer7_attention_weight_max": 89.42108154296875,
      "activations/layer7_attention_weight_min": -93.91287994384766,
      "activations/layer8_attention_weight_max": 39.9852409362793,
      "activations/layer8_attention_weight_min": -42.44793701171875,
      "activations/layer9_attention_weight_max": 35.500274658203125,
      "activations/layer9_attention_weight_min": -34.75611114501953,
      "epoch": 9.12,
      "learning_rate": 9.211628787878788e-05,
      "loss": 2.8045,
      "step": 156950
    },
    {
      "activations/layer0_attention_weight_max": 14.961008071899414,
      "activations/layer0_attention_weight_min": -11.76927375793457,
      "activations/layer10_attention_weight_max": 31.79775047302246,
      "activations/layer10_attention_weight_min": -33.03754806518555,
      "activations/layer11_attention_weight_max": 31.307212829589844,
      "activations/layer11_attention_weight_min": -29.025188446044922,
      "activations/layer12_attention_weight_max": 22.380802154541016,
      "activations/layer12_attention_weight_min": -24.73975372314453,
      "activations/layer13_attention_weight_max": 35.69672393798828,
      "activations/layer13_attention_weight_min": -33.33967971801758,
      "activations/layer14_attention_weight_max": 44.244651794433594,
      "activations/layer14_attention_weight_min": -35.90494155883789,
      "activations/layer15_attention_weight_max": 33.99721145629883,
      "activations/layer15_attention_weight_min": -29.858251571655273,
      "activations/layer16_attention_weight_max": 29.108257293701172,
      "activations/layer16_attention_weight_min": -28.15972900390625,
      "activations/layer17_attention_weight_max": 30.567007064819336,
      "activations/layer17_attention_weight_min": -26.92832374572754,
      "activations/layer18_attention_weight_max": 27.783536911010742,
      "activations/layer18_attention_weight_min": -22.597856521606445,
      "activations/layer19_attention_weight_max": 32.64551544189453,
      "activations/layer19_attention_weight_min": -32.30259323120117,
      "activations/layer1_attention_weight_max": 15.677871704101562,
      "activations/layer1_attention_weight_min": -14.213926315307617,
      "activations/layer20_attention_weight_max": 27.753150939941406,
      "activations/layer20_attention_weight_min": -24.26315689086914,
      "activations/layer21_attention_weight_max": 28.831924438476562,
      "activations/layer21_attention_weight_min": -23.994522094726562,
      "activations/layer22_attention_weight_max": 39.78486633300781,
      "activations/layer22_attention_weight_min": -29.65273094177246,
      "activations/layer23_attention_weight_max": 34.49162292480469,
      "activations/layer23_attention_weight_min": -25.475706100463867,
      "activations/layer2_attention_weight_max": 33.643104553222656,
      "activations/layer2_attention_weight_min": -32.69413375854492,
      "activations/layer3_attention_weight_max": 96.78746032714844,
      "activations/layer3_attention_weight_min": -98.86905670166016,
      "activations/layer4_attention_weight_max": 118.33354949951172,
      "activations/layer4_attention_weight_min": -112.96339416503906,
      "activations/layer5_attention_weight_max": 54.37568283081055,
      "activations/layer5_attention_weight_min": -64.10836029052734,
      "activations/layer6_attention_weight_max": 49.3455696105957,
      "activations/layer6_attention_weight_min": -50.225337982177734,
      "activations/layer7_attention_weight_max": 90.05542755126953,
      "activations/layer7_attention_weight_min": -86.96453857421875,
      "activations/layer8_attention_weight_max": 38.34833908081055,
      "activations/layer8_attention_weight_min": -39.80671310424805,
      "activations/layer9_attention_weight_max": 32.633514404296875,
      "activations/layer9_attention_weight_min": -33.84961700439453,
      "epoch": 9.12,
      "learning_rate": 9.209734848484848e-05,
      "loss": 2.7901,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_loss": 2.75,
      "eval_runtime": 8.4572,
      "eval_samples_per_second": 507.735,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.4572,
      "eval_openwebtext_samples_per_second": 507.735,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 1.9131,
      "eval_wikitext_samples_per_second": 238.361,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.5073,
      "eval_lambada_samples_per_second": 512.133,
      "step": 157000
    },
    {
      "activations/layer0_attention_weight_max": 15.212652206420898,
      "activations/layer0_attention_weight_min": -12.504545211791992,
      "activations/layer10_attention_weight_max": 31.982254028320312,
      "activations/layer10_attention_weight_min": -32.73135757446289,
      "activations/layer11_attention_weight_max": 32.51982116699219,
      "activations/layer11_attention_weight_min": -30.568065643310547,
      "activations/layer12_attention_weight_max": 29.321792602539062,
      "activations/layer12_attention_weight_min": -24.467262268066406,
      "activations/layer13_attention_weight_max": 36.2407341003418,
      "activations/layer13_attention_weight_min": -32.18763732910156,
      "activations/layer14_attention_weight_max": 39.91740036010742,
      "activations/layer14_attention_weight_min": -36.603641510009766,
      "activations/layer15_attention_weight_max": 33.90324783325195,
      "activations/layer15_attention_weight_min": -29.772825241088867,
      "activations/layer16_attention_weight_max": 28.705280303955078,
      "activations/layer16_attention_weight_min": -27.382715225219727,
      "activations/layer17_attention_weight_max": 33.35182571411133,
      "activations/layer17_attention_weight_min": -24.925559997558594,
      "activations/layer18_attention_weight_max": 29.218481063842773,
      "activations/layer18_attention_weight_min": -22.032045364379883,
      "activations/layer19_attention_weight_max": 32.63858413696289,
      "activations/layer19_attention_weight_min": -31.313016891479492,
      "activations/layer1_attention_weight_max": 16.348752975463867,
      "activations/layer1_attention_weight_min": -15.264453887939453,
      "activations/layer20_attention_weight_max": 30.31749153137207,
      "activations/layer20_attention_weight_min": -28.39882469177246,
      "activations/layer21_attention_weight_max": 28.379329681396484,
      "activations/layer21_attention_weight_min": -25.245615005493164,
      "activations/layer22_attention_weight_max": 43.79624557495117,
      "activations/layer22_attention_weight_min": -30.389556884765625,
      "activations/layer23_attention_weight_max": 32.49997329711914,
      "activations/layer23_attention_weight_min": -24.911319732666016,
      "activations/layer2_attention_weight_max": 34.15679931640625,
      "activations/layer2_attention_weight_min": -33.70691680908203,
      "activations/layer3_attention_weight_max": 96.1579360961914,
      "activations/layer3_attention_weight_min": -97.56843566894531,
      "activations/layer4_attention_weight_max": 122.81658935546875,
      "activations/layer4_attention_weight_min": -122.7129898071289,
      "activations/layer5_attention_weight_max": 57.78738021850586,
      "activations/layer5_attention_weight_min": -62.36913299560547,
      "activations/layer6_attention_weight_max": 48.24141311645508,
      "activations/layer6_attention_weight_min": -52.1047477722168,
      "activations/layer7_attention_weight_max": 91.34271240234375,
      "activations/layer7_attention_weight_min": -94.77192687988281,
      "activations/layer8_attention_weight_max": 43.24505615234375,
      "activations/layer8_attention_weight_min": -41.69619369506836,
      "activations/layer9_attention_weight_max": 35.7325553894043,
      "activations/layer9_attention_weight_min": -34.27656936645508,
      "epoch": 9.13,
      "learning_rate": 9.207840909090908e-05,
      "loss": 2.7893,
      "step": 157050
    },
    {
      "activations/layer0_attention_weight_max": 16.035165786743164,
      "activations/layer0_attention_weight_min": -12.422619819641113,
      "activations/layer10_attention_weight_max": 31.769424438476562,
      "activations/layer10_attention_weight_min": -31.583404541015625,
      "activations/layer11_attention_weight_max": 31.444570541381836,
      "activations/layer11_attention_weight_min": -30.68133544921875,
      "activations/layer12_attention_weight_max": 22.891510009765625,
      "activations/layer12_attention_weight_min": -26.154438018798828,
      "activations/layer13_attention_weight_max": 39.060768127441406,
      "activations/layer13_attention_weight_min": -31.066553115844727,
      "activations/layer14_attention_weight_max": 45.35211944580078,
      "activations/layer14_attention_weight_min": -36.13892364501953,
      "activations/layer15_attention_weight_max": 37.03008270263672,
      "activations/layer15_attention_weight_min": -29.125442504882812,
      "activations/layer16_attention_weight_max": 32.12025451660156,
      "activations/layer16_attention_weight_min": -29.05597496032715,
      "activations/layer17_attention_weight_max": 32.86757278442383,
      "activations/layer17_attention_weight_min": -25.21859359741211,
      "activations/layer18_attention_weight_max": 28.62696647644043,
      "activations/layer18_attention_weight_min": -22.989585876464844,
      "activations/layer19_attention_weight_max": 38.77228546142578,
      "activations/layer19_attention_weight_min": -28.806350708007812,
      "activations/layer1_attention_weight_max": 16.25472068786621,
      "activations/layer1_attention_weight_min": -15.97120189666748,
      "activations/layer20_attention_weight_max": 31.80004119873047,
      "activations/layer20_attention_weight_min": -23.928800582885742,
      "activations/layer21_attention_weight_max": 30.477611541748047,
      "activations/layer21_attention_weight_min": -22.144638061523438,
      "activations/layer22_attention_weight_max": 44.01310729980469,
      "activations/layer22_attention_weight_min": -27.721521377563477,
      "activations/layer23_attention_weight_max": 36.28701400756836,
      "activations/layer23_attention_weight_min": -22.526763916015625,
      "activations/layer2_attention_weight_max": 33.36884307861328,
      "activations/layer2_attention_weight_min": -31.987775802612305,
      "activations/layer3_attention_weight_max": 98.02265167236328,
      "activations/layer3_attention_weight_min": -93.25384521484375,
      "activations/layer4_attention_weight_max": 118.13323211669922,
      "activations/layer4_attention_weight_min": -114.5810546875,
      "activations/layer5_attention_weight_max": 54.9063720703125,
      "activations/layer5_attention_weight_min": -60.620277404785156,
      "activations/layer6_attention_weight_max": 46.667842864990234,
      "activations/layer6_attention_weight_min": -49.46928024291992,
      "activations/layer7_attention_weight_max": 85.25377655029297,
      "activations/layer7_attention_weight_min": -83.68876647949219,
      "activations/layer8_attention_weight_max": 40.98828887939453,
      "activations/layer8_attention_weight_min": -40.44070053100586,
      "activations/layer9_attention_weight_max": 34.315616607666016,
      "activations/layer9_attention_weight_min": -31.417713165283203,
      "epoch": 9.13,
      "learning_rate": 9.20594696969697e-05,
      "loss": 2.7908,
      "step": 157100
    },
    {
      "activations/layer0_attention_weight_max": 15.851951599121094,
      "activations/layer0_attention_weight_min": -13.097590446472168,
      "activations/layer10_attention_weight_max": 33.79930877685547,
      "activations/layer10_attention_weight_min": -34.33380126953125,
      "activations/layer11_attention_weight_max": 35.180606842041016,
      "activations/layer11_attention_weight_min": -31.063922882080078,
      "activations/layer12_attention_weight_max": 26.50753402709961,
      "activations/layer12_attention_weight_min": -24.63486671447754,
      "activations/layer13_attention_weight_max": 44.21175003051758,
      "activations/layer13_attention_weight_min": -36.08030700683594,
      "activations/layer14_attention_weight_max": 58.85987091064453,
      "activations/layer14_attention_weight_min": -41.126094818115234,
      "activations/layer15_attention_weight_max": 41.00049591064453,
      "activations/layer15_attention_weight_min": -31.115304946899414,
      "activations/layer16_attention_weight_max": 36.12473678588867,
      "activations/layer16_attention_weight_min": -27.071001052856445,
      "activations/layer17_attention_weight_max": 34.7183837890625,
      "activations/layer17_attention_weight_min": -24.50230598449707,
      "activations/layer18_attention_weight_max": 30.52606201171875,
      "activations/layer18_attention_weight_min": -21.87002182006836,
      "activations/layer19_attention_weight_max": 41.30485153198242,
      "activations/layer19_attention_weight_min": -31.203336715698242,
      "activations/layer1_attention_weight_max": 16.62590980529785,
      "activations/layer1_attention_weight_min": -14.445747375488281,
      "activations/layer20_attention_weight_max": 34.16878890991211,
      "activations/layer20_attention_weight_min": -24.463926315307617,
      "activations/layer21_attention_weight_max": 35.234336853027344,
      "activations/layer21_attention_weight_min": -22.21427345275879,
      "activations/layer22_attention_weight_max": 46.64390182495117,
      "activations/layer22_attention_weight_min": -29.714921951293945,
      "activations/layer23_attention_weight_max": 41.47065353393555,
      "activations/layer23_attention_weight_min": -26.208927154541016,
      "activations/layer2_attention_weight_max": 33.107784271240234,
      "activations/layer2_attention_weight_min": -32.24180603027344,
      "activations/layer3_attention_weight_max": 94.17940521240234,
      "activations/layer3_attention_weight_min": -94.2065200805664,
      "activations/layer4_attention_weight_max": 115.67095184326172,
      "activations/layer4_attention_weight_min": -110.18553924560547,
      "activations/layer5_attention_weight_max": 54.6190299987793,
      "activations/layer5_attention_weight_min": -59.06772994995117,
      "activations/layer6_attention_weight_max": 48.795955657958984,
      "activations/layer6_attention_weight_min": -49.494632720947266,
      "activations/layer7_attention_weight_max": 95.09100341796875,
      "activations/layer7_attention_weight_min": -91.35867309570312,
      "activations/layer8_attention_weight_max": 42.0092887878418,
      "activations/layer8_attention_weight_min": -42.39775466918945,
      "activations/layer9_attention_weight_max": 35.74744415283203,
      "activations/layer9_attention_weight_min": -34.83646774291992,
      "epoch": 9.13,
      "learning_rate": 9.204053030303031e-05,
      "loss": 2.7985,
      "step": 157150
    },
    {
      "activations/layer0_attention_weight_max": 15.60646915435791,
      "activations/layer0_attention_weight_min": -11.995765686035156,
      "activations/layer10_attention_weight_max": 32.017066955566406,
      "activations/layer10_attention_weight_min": -33.05015563964844,
      "activations/layer11_attention_weight_max": 31.660512924194336,
      "activations/layer11_attention_weight_min": -32.04449462890625,
      "activations/layer12_attention_weight_max": 22.16817855834961,
      "activations/layer12_attention_weight_min": -23.83069610595703,
      "activations/layer13_attention_weight_max": 36.81056213378906,
      "activations/layer13_attention_weight_min": -33.6300163269043,
      "activations/layer14_attention_weight_max": 42.50653839111328,
      "activations/layer14_attention_weight_min": -39.25753402709961,
      "activations/layer15_attention_weight_max": 32.956626892089844,
      "activations/layer15_attention_weight_min": -29.371488571166992,
      "activations/layer16_attention_weight_max": 29.272153854370117,
      "activations/layer16_attention_weight_min": -26.50972557067871,
      "activations/layer17_attention_weight_max": 33.928192138671875,
      "activations/layer17_attention_weight_min": -24.177549362182617,
      "activations/layer18_attention_weight_max": 32.30479049682617,
      "activations/layer18_attention_weight_min": -21.926111221313477,
      "activations/layer19_attention_weight_max": 34.8094596862793,
      "activations/layer19_attention_weight_min": -30.574020385742188,
      "activations/layer1_attention_weight_max": 16.061901092529297,
      "activations/layer1_attention_weight_min": -14.981988906860352,
      "activations/layer20_attention_weight_max": 29.43551254272461,
      "activations/layer20_attention_weight_min": -25.18837547302246,
      "activations/layer21_attention_weight_max": 29.90959930419922,
      "activations/layer21_attention_weight_min": -23.256269454956055,
      "activations/layer22_attention_weight_max": 39.45198059082031,
      "activations/layer22_attention_weight_min": -28.42708396911621,
      "activations/layer23_attention_weight_max": 33.36308670043945,
      "activations/layer23_attention_weight_min": -23.543441772460938,
      "activations/layer2_attention_weight_max": 33.95153045654297,
      "activations/layer2_attention_weight_min": -32.93464279174805,
      "activations/layer3_attention_weight_max": 97.66692352294922,
      "activations/layer3_attention_weight_min": -95.92757415771484,
      "activations/layer4_attention_weight_max": 122.6179428100586,
      "activations/layer4_attention_weight_min": -113.430908203125,
      "activations/layer5_attention_weight_max": 54.97910690307617,
      "activations/layer5_attention_weight_min": -59.41450500488281,
      "activations/layer6_attention_weight_max": 48.776371002197266,
      "activations/layer6_attention_weight_min": -50.37691116333008,
      "activations/layer7_attention_weight_max": 93.10022735595703,
      "activations/layer7_attention_weight_min": -85.09976196289062,
      "activations/layer8_attention_weight_max": 40.25505828857422,
      "activations/layer8_attention_weight_min": -41.0278205871582,
      "activations/layer9_attention_weight_max": 34.38668441772461,
      "activations/layer9_attention_weight_min": -33.419673919677734,
      "epoch": 9.13,
      "learning_rate": 9.20215909090909e-05,
      "loss": 2.7887,
      "step": 157200
    },
    {
      "activations/layer0_attention_weight_max": 14.651124000549316,
      "activations/layer0_attention_weight_min": -13.146588325500488,
      "activations/layer10_attention_weight_max": 31.215322494506836,
      "activations/layer10_attention_weight_min": -31.08536720275879,
      "activations/layer11_attention_weight_max": 30.257617950439453,
      "activations/layer11_attention_weight_min": -29.543445587158203,
      "activations/layer12_attention_weight_max": 22.29743194580078,
      "activations/layer12_attention_weight_min": -25.966354370117188,
      "activations/layer13_attention_weight_max": 34.085453033447266,
      "activations/layer13_attention_weight_min": -32.602195739746094,
      "activations/layer14_attention_weight_max": 47.50339889526367,
      "activations/layer14_attention_weight_min": -39.407344818115234,
      "activations/layer15_attention_weight_max": 33.616233825683594,
      "activations/layer15_attention_weight_min": -29.413738250732422,
      "activations/layer16_attention_weight_max": 30.155948638916016,
      "activations/layer16_attention_weight_min": -29.260284423828125,
      "activations/layer17_attention_weight_max": 31.3532657623291,
      "activations/layer17_attention_weight_min": -26.059202194213867,
      "activations/layer18_attention_weight_max": 31.32282066345215,
      "activations/layer18_attention_weight_min": -23.51839828491211,
      "activations/layer19_attention_weight_max": 34.5306282043457,
      "activations/layer19_attention_weight_min": -31.366352081298828,
      "activations/layer1_attention_weight_max": 17.153650283813477,
      "activations/layer1_attention_weight_min": -15.117240905761719,
      "activations/layer20_attention_weight_max": 30.465469360351562,
      "activations/layer20_attention_weight_min": -26.328413009643555,
      "activations/layer21_attention_weight_max": 32.14548110961914,
      "activations/layer21_attention_weight_min": -25.302146911621094,
      "activations/layer22_attention_weight_max": 45.01878356933594,
      "activations/layer22_attention_weight_min": -28.112934112548828,
      "activations/layer23_attention_weight_max": 36.1341552734375,
      "activations/layer23_attention_weight_min": -23.938209533691406,
      "activations/layer2_attention_weight_max": 32.94959259033203,
      "activations/layer2_attention_weight_min": -31.673002243041992,
      "activations/layer3_attention_weight_max": 93.53209686279297,
      "activations/layer3_attention_weight_min": -96.88230895996094,
      "activations/layer4_attention_weight_max": 118.48067474365234,
      "activations/layer4_attention_weight_min": -114.05558013916016,
      "activations/layer5_attention_weight_max": 54.39646911621094,
      "activations/layer5_attention_weight_min": -62.78622055053711,
      "activations/layer6_attention_weight_max": 46.80887985229492,
      "activations/layer6_attention_weight_min": -50.979957580566406,
      "activations/layer7_attention_weight_max": 87.52057647705078,
      "activations/layer7_attention_weight_min": -86.73812866210938,
      "activations/layer8_attention_weight_max": 39.33066177368164,
      "activations/layer8_attention_weight_min": -38.571048736572266,
      "activations/layer9_attention_weight_max": 33.080074310302734,
      "activations/layer9_attention_weight_min": -32.79412078857422,
      "epoch": 9.14,
      "learning_rate": 9.200303030303029e-05,
      "loss": 2.7886,
      "step": 157250
    },
    {
      "activations/layer0_attention_weight_max": 15.364968299865723,
      "activations/layer0_attention_weight_min": -11.370357513427734,
      "activations/layer10_attention_weight_max": 33.82844161987305,
      "activations/layer10_attention_weight_min": -32.87842559814453,
      "activations/layer11_attention_weight_max": 32.20787048339844,
      "activations/layer11_attention_weight_min": -30.268272399902344,
      "activations/layer12_attention_weight_max": 47.40199661254883,
      "activations/layer12_attention_weight_min": -26.502811431884766,
      "activations/layer13_attention_weight_max": 49.3670768737793,
      "activations/layer13_attention_weight_min": -31.430471420288086,
      "activations/layer14_attention_weight_max": 45.08019256591797,
      "activations/layer14_attention_weight_min": -37.453765869140625,
      "activations/layer15_attention_weight_max": 40.650146484375,
      "activations/layer15_attention_weight_min": -29.133319854736328,
      "activations/layer16_attention_weight_max": 32.36852264404297,
      "activations/layer16_attention_weight_min": -30.268936157226562,
      "activations/layer17_attention_weight_max": 32.13740921020508,
      "activations/layer17_attention_weight_min": -26.369909286499023,
      "activations/layer18_attention_weight_max": 30.687143325805664,
      "activations/layer18_attention_weight_min": -21.854188919067383,
      "activations/layer19_attention_weight_max": 33.200164794921875,
      "activations/layer19_attention_weight_min": -30.350004196166992,
      "activations/layer1_attention_weight_max": 16.396699905395508,
      "activations/layer1_attention_weight_min": -15.690770149230957,
      "activations/layer20_attention_weight_max": 30.957611083984375,
      "activations/layer20_attention_weight_min": -22.87656021118164,
      "activations/layer21_attention_weight_max": 29.611520767211914,
      "activations/layer21_attention_weight_min": -23.05364227294922,
      "activations/layer22_attention_weight_max": 45.0173454284668,
      "activations/layer22_attention_weight_min": -29.20310401916504,
      "activations/layer23_attention_weight_max": 35.03718948364258,
      "activations/layer23_attention_weight_min": -22.24708366394043,
      "activations/layer2_attention_weight_max": 32.58720779418945,
      "activations/layer2_attention_weight_min": -32.76677703857422,
      "activations/layer3_attention_weight_max": 98.00126647949219,
      "activations/layer3_attention_weight_min": -96.3399887084961,
      "activations/layer4_attention_weight_max": 119.69559478759766,
      "activations/layer4_attention_weight_min": -118.2739028930664,
      "activations/layer5_attention_weight_max": 55.84374237060547,
      "activations/layer5_attention_weight_min": -63.15813446044922,
      "activations/layer6_attention_weight_max": 46.9710807800293,
      "activations/layer6_attention_weight_min": -47.70405960083008,
      "activations/layer7_attention_weight_max": 88.05366516113281,
      "activations/layer7_attention_weight_min": -86.64402770996094,
      "activations/layer8_attention_weight_max": 41.50566101074219,
      "activations/layer8_attention_weight_min": -38.531494140625,
      "activations/layer9_attention_weight_max": 32.60149002075195,
      "activations/layer9_attention_weight_min": -32.279685974121094,
      "epoch": 9.14,
      "learning_rate": 9.19840909090909e-05,
      "loss": 2.7992,
      "step": 157300
    },
    {
      "activations/layer0_attention_weight_max": 16.299551010131836,
      "activations/layer0_attention_weight_min": -12.811613082885742,
      "activations/layer10_attention_weight_max": 30.063140869140625,
      "activations/layer10_attention_weight_min": -31.754117965698242,
      "activations/layer11_attention_weight_max": 32.23501205444336,
      "activations/layer11_attention_weight_min": -29.44198226928711,
      "activations/layer12_attention_weight_max": 28.485105514526367,
      "activations/layer12_attention_weight_min": -26.763912200927734,
      "activations/layer13_attention_weight_max": 44.27042770385742,
      "activations/layer13_attention_weight_min": -33.97203063964844,
      "activations/layer14_attention_weight_max": 47.2341423034668,
      "activations/layer14_attention_weight_min": -37.026065826416016,
      "activations/layer15_attention_weight_max": 35.40715408325195,
      "activations/layer15_attention_weight_min": -30.605772018432617,
      "activations/layer16_attention_weight_max": 33.509437561035156,
      "activations/layer16_attention_weight_min": -26.585994720458984,
      "activations/layer17_attention_weight_max": 31.34138298034668,
      "activations/layer17_attention_weight_min": -24.639755249023438,
      "activations/layer18_attention_weight_max": 30.101041793823242,
      "activations/layer18_attention_weight_min": -21.516565322875977,
      "activations/layer19_attention_weight_max": 37.82497024536133,
      "activations/layer19_attention_weight_min": -31.927579879760742,
      "activations/layer1_attention_weight_max": 16.481555938720703,
      "activations/layer1_attention_weight_min": -15.25529956817627,
      "activations/layer20_attention_weight_max": 30.032548904418945,
      "activations/layer20_attention_weight_min": -24.17172622680664,
      "activations/layer21_attention_weight_max": 29.989665985107422,
      "activations/layer21_attention_weight_min": -22.85221290588379,
      "activations/layer22_attention_weight_max": 41.59782028198242,
      "activations/layer22_attention_weight_min": -29.955795288085938,
      "activations/layer23_attention_weight_max": 33.58283996582031,
      "activations/layer23_attention_weight_min": -25.095687866210938,
      "activations/layer2_attention_weight_max": 32.659725189208984,
      "activations/layer2_attention_weight_min": -32.35512161254883,
      "activations/layer3_attention_weight_max": 92.25064086914062,
      "activations/layer3_attention_weight_min": -94.65694427490234,
      "activations/layer4_attention_weight_max": 113.3831787109375,
      "activations/layer4_attention_weight_min": -118.58013916015625,
      "activations/layer5_attention_weight_max": 52.34837341308594,
      "activations/layer5_attention_weight_min": -61.60050964355469,
      "activations/layer6_attention_weight_max": 47.549354553222656,
      "activations/layer6_attention_weight_min": -47.85348129272461,
      "activations/layer7_attention_weight_max": 88.7129135131836,
      "activations/layer7_attention_weight_min": -83.11204528808594,
      "activations/layer8_attention_weight_max": 38.6871223449707,
      "activations/layer8_attention_weight_min": -40.40482711791992,
      "activations/layer9_attention_weight_max": 31.046995162963867,
      "activations/layer9_attention_weight_min": -34.28760528564453,
      "epoch": 9.14,
      "learning_rate": 9.196515151515152e-05,
      "loss": 2.8062,
      "step": 157350
    },
    {
      "activations/layer0_attention_weight_max": 15.89444637298584,
      "activations/layer0_attention_weight_min": -11.334229469299316,
      "activations/layer10_attention_weight_max": 34.80430221557617,
      "activations/layer10_attention_weight_min": -34.69694900512695,
      "activations/layer11_attention_weight_max": 33.889305114746094,
      "activations/layer11_attention_weight_min": -31.94155502319336,
      "activations/layer12_attention_weight_max": 35.5349235534668,
      "activations/layer12_attention_weight_min": -25.593460083007812,
      "activations/layer13_attention_weight_max": 41.524169921875,
      "activations/layer13_attention_weight_min": -35.23186492919922,
      "activations/layer14_attention_weight_max": 54.36832809448242,
      "activations/layer14_attention_weight_min": -41.01689529418945,
      "activations/layer15_attention_weight_max": 38.48654556274414,
      "activations/layer15_attention_weight_min": -32.96341323852539,
      "activations/layer16_attention_weight_max": 35.562744140625,
      "activations/layer16_attention_weight_min": -30.121362686157227,
      "activations/layer17_attention_weight_max": 36.563453674316406,
      "activations/layer17_attention_weight_min": -26.7945556640625,
      "activations/layer18_attention_weight_max": 34.39448547363281,
      "activations/layer18_attention_weight_min": -22.062801361083984,
      "activations/layer19_attention_weight_max": 40.52739715576172,
      "activations/layer19_attention_weight_min": -30.988956451416016,
      "activations/layer1_attention_weight_max": 16.196931838989258,
      "activations/layer1_attention_weight_min": -14.838172912597656,
      "activations/layer20_attention_weight_max": 34.65544891357422,
      "activations/layer20_attention_weight_min": -24.04445457458496,
      "activations/layer21_attention_weight_max": 32.29057693481445,
      "activations/layer21_attention_weight_min": -22.079687118530273,
      "activations/layer22_attention_weight_max": 46.8472785949707,
      "activations/layer22_attention_weight_min": -27.607975006103516,
      "activations/layer23_attention_weight_max": 42.128387451171875,
      "activations/layer23_attention_weight_min": -21.868003845214844,
      "activations/layer2_attention_weight_max": 33.459449768066406,
      "activations/layer2_attention_weight_min": -33.85844421386719,
      "activations/layer3_attention_weight_max": 91.83235931396484,
      "activations/layer3_attention_weight_min": -96.22340393066406,
      "activations/layer4_attention_weight_max": 115.66151428222656,
      "activations/layer4_attention_weight_min": -114.84541320800781,
      "activations/layer5_attention_weight_max": 54.00886154174805,
      "activations/layer5_attention_weight_min": -60.31504440307617,
      "activations/layer6_attention_weight_max": 48.59791946411133,
      "activations/layer6_attention_weight_min": -49.7972297668457,
      "activations/layer7_attention_weight_max": 91.61717987060547,
      "activations/layer7_attention_weight_min": -89.13182830810547,
      "activations/layer8_attention_weight_max": 41.78858947753906,
      "activations/layer8_attention_weight_min": -43.2989387512207,
      "activations/layer9_attention_weight_max": 34.50050735473633,
      "activations/layer9_attention_weight_min": -35.21269226074219,
      "epoch": 9.15,
      "learning_rate": 9.19462121212121e-05,
      "loss": 2.7754,
      "step": 157400
    },
    {
      "activations/layer0_attention_weight_max": 15.604413986206055,
      "activations/layer0_attention_weight_min": -11.940605163574219,
      "activations/layer10_attention_weight_max": 34.49391174316406,
      "activations/layer10_attention_weight_min": -34.49024963378906,
      "activations/layer11_attention_weight_max": 34.82725524902344,
      "activations/layer11_attention_weight_min": -32.56266784667969,
      "activations/layer12_attention_weight_max": 27.75283432006836,
      "activations/layer12_attention_weight_min": -25.31946563720703,
      "activations/layer13_attention_weight_max": 40.07194137573242,
      "activations/layer13_attention_weight_min": -36.708580017089844,
      "activations/layer14_attention_weight_max": 53.24272918701172,
      "activations/layer14_attention_weight_min": -44.29374313354492,
      "activations/layer15_attention_weight_max": 42.57693862915039,
      "activations/layer15_attention_weight_min": -32.22544860839844,
      "activations/layer16_attention_weight_max": 39.05867004394531,
      "activations/layer16_attention_weight_min": -28.738664627075195,
      "activations/layer17_attention_weight_max": 34.74129867553711,
      "activations/layer17_attention_weight_min": -25.488666534423828,
      "activations/layer18_attention_weight_max": 31.98640251159668,
      "activations/layer18_attention_weight_min": -23.83980369567871,
      "activations/layer19_attention_weight_max": 42.52510452270508,
      "activations/layer19_attention_weight_min": -31.122848510742188,
      "activations/layer1_attention_weight_max": 17.814197540283203,
      "activations/layer1_attention_weight_min": -14.431021690368652,
      "activations/layer20_attention_weight_max": 34.91218566894531,
      "activations/layer20_attention_weight_min": -25.470491409301758,
      "activations/layer21_attention_weight_max": 31.921154022216797,
      "activations/layer21_attention_weight_min": -25.475982666015625,
      "activations/layer22_attention_weight_max": 49.70567321777344,
      "activations/layer22_attention_weight_min": -29.474918365478516,
      "activations/layer23_attention_weight_max": 35.0259895324707,
      "activations/layer23_attention_weight_min": -25.443960189819336,
      "activations/layer2_attention_weight_max": 33.96269607543945,
      "activations/layer2_attention_weight_min": -32.565895080566406,
      "activations/layer3_attention_weight_max": 97.52854919433594,
      "activations/layer3_attention_weight_min": -94.4259262084961,
      "activations/layer4_attention_weight_max": 124.79467010498047,
      "activations/layer4_attention_weight_min": -115.8509521484375,
      "activations/layer5_attention_weight_max": 58.814453125,
      "activations/layer5_attention_weight_min": -59.39210510253906,
      "activations/layer6_attention_weight_max": 50.75216293334961,
      "activations/layer6_attention_weight_min": -52.00008773803711,
      "activations/layer7_attention_weight_max": 90.06100463867188,
      "activations/layer7_attention_weight_min": -94.47721099853516,
      "activations/layer8_attention_weight_max": 42.28409957885742,
      "activations/layer8_attention_weight_min": -43.743492126464844,
      "activations/layer9_attention_weight_max": 36.62489700317383,
      "activations/layer9_attention_weight_min": -36.95697021484375,
      "epoch": 9.15,
      "learning_rate": 9.192727272727272e-05,
      "loss": 2.7848,
      "step": 157450
    },
    {
      "activations/layer0_attention_weight_max": 16.108247756958008,
      "activations/layer0_attention_weight_min": -12.030502319335938,
      "activations/layer10_attention_weight_max": 40.1231575012207,
      "activations/layer10_attention_weight_min": -35.63908386230469,
      "activations/layer11_attention_weight_max": 44.26158142089844,
      "activations/layer11_attention_weight_min": -37.04606246948242,
      "activations/layer12_attention_weight_max": 32.75822067260742,
      "activations/layer12_attention_weight_min": -25.807960510253906,
      "activations/layer13_attention_weight_max": 58.59912109375,
      "activations/layer13_attention_weight_min": -36.39835739135742,
      "activations/layer14_attention_weight_max": 66.94005584716797,
      "activations/layer14_attention_weight_min": -40.08238220214844,
      "activations/layer15_attention_weight_max": 64.76361083984375,
      "activations/layer15_attention_weight_min": -33.70975875854492,
      "activations/layer16_attention_weight_max": 40.748023986816406,
      "activations/layer16_attention_weight_min": -27.309097290039062,
      "activations/layer17_attention_weight_max": 39.930301666259766,
      "activations/layer17_attention_weight_min": -26.861560821533203,
      "activations/layer18_attention_weight_max": 34.52722930908203,
      "activations/layer18_attention_weight_min": -24.928810119628906,
      "activations/layer19_attention_weight_max": 40.433834075927734,
      "activations/layer19_attention_weight_min": -32.17000198364258,
      "activations/layer1_attention_weight_max": 16.02019500732422,
      "activations/layer1_attention_weight_min": -13.953417778015137,
      "activations/layer20_attention_weight_max": 34.54740905761719,
      "activations/layer20_attention_weight_min": -26.12274742126465,
      "activations/layer21_attention_weight_max": 31.77134132385254,
      "activations/layer21_attention_weight_min": -23.10929298400879,
      "activations/layer22_attention_weight_max": 52.37502670288086,
      "activations/layer22_attention_weight_min": -29.524898529052734,
      "activations/layer23_attention_weight_max": 37.30921173095703,
      "activations/layer23_attention_weight_min": -23.688154220581055,
      "activations/layer2_attention_weight_max": 33.869842529296875,
      "activations/layer2_attention_weight_min": -32.87567901611328,
      "activations/layer3_attention_weight_max": 93.43128967285156,
      "activations/layer3_attention_weight_min": -98.78339385986328,
      "activations/layer4_attention_weight_max": 113.0006103515625,
      "activations/layer4_attention_weight_min": -112.31317138671875,
      "activations/layer5_attention_weight_max": 52.85210418701172,
      "activations/layer5_attention_weight_min": -61.78768539428711,
      "activations/layer6_attention_weight_max": 46.79494094848633,
      "activations/layer6_attention_weight_min": -49.83319854736328,
      "activations/layer7_attention_weight_max": 96.18464660644531,
      "activations/layer7_attention_weight_min": -94.80194854736328,
      "activations/layer8_attention_weight_max": 47.79298400878906,
      "activations/layer8_attention_weight_min": -43.71220016479492,
      "activations/layer9_attention_weight_max": 41.721153259277344,
      "activations/layer9_attention_weight_min": -37.27484893798828,
      "epoch": 9.15,
      "learning_rate": 9.190833333333332e-05,
      "loss": 2.7995,
      "step": 157500
    },
    {
      "activations/layer0_attention_weight_max": 14.856646537780762,
      "activations/layer0_attention_weight_min": -12.88975715637207,
      "activations/layer10_attention_weight_max": 34.705665588378906,
      "activations/layer10_attention_weight_min": -33.85185241699219,
      "activations/layer11_attention_weight_max": 37.136192321777344,
      "activations/layer11_attention_weight_min": -33.7395133972168,
      "activations/layer12_attention_weight_max": 24.478965759277344,
      "activations/layer12_attention_weight_min": -28.869991302490234,
      "activations/layer13_attention_weight_max": 39.699676513671875,
      "activations/layer13_attention_weight_min": -31.741424560546875,
      "activations/layer14_attention_weight_max": 52.30339813232422,
      "activations/layer14_attention_weight_min": -38.41307830810547,
      "activations/layer15_attention_weight_max": 38.04621124267578,
      "activations/layer15_attention_weight_min": -28.857513427734375,
      "activations/layer16_attention_weight_max": 33.56850814819336,
      "activations/layer16_attention_weight_min": -27.739038467407227,
      "activations/layer17_attention_weight_max": 34.61725616455078,
      "activations/layer17_attention_weight_min": -24.713069915771484,
      "activations/layer18_attention_weight_max": 29.881248474121094,
      "activations/layer18_attention_weight_min": -21.347726821899414,
      "activations/layer19_attention_weight_max": 39.05013656616211,
      "activations/layer19_attention_weight_min": -31.556852340698242,
      "activations/layer1_attention_weight_max": 16.068037033081055,
      "activations/layer1_attention_weight_min": -14.041608810424805,
      "activations/layer20_attention_weight_max": 34.427066802978516,
      "activations/layer20_attention_weight_min": -24.525646209716797,
      "activations/layer21_attention_weight_max": 30.447439193725586,
      "activations/layer21_attention_weight_min": -22.680112838745117,
      "activations/layer22_attention_weight_max": 46.750877380371094,
      "activations/layer22_attention_weight_min": -29.008516311645508,
      "activations/layer23_attention_weight_max": 35.15749740600586,
      "activations/layer23_attention_weight_min": -22.31591033935547,
      "activations/layer2_attention_weight_max": 32.31842803955078,
      "activations/layer2_attention_weight_min": -32.700645446777344,
      "activations/layer3_attention_weight_max": 95.7559585571289,
      "activations/layer3_attention_weight_min": -97.94831848144531,
      "activations/layer4_attention_weight_max": 116.69339752197266,
      "activations/layer4_attention_weight_min": -115.29552459716797,
      "activations/layer5_attention_weight_max": 53.62945556640625,
      "activations/layer5_attention_weight_min": -59.094146728515625,
      "activations/layer6_attention_weight_max": 48.520503997802734,
      "activations/layer6_attention_weight_min": -48.18111801147461,
      "activations/layer7_attention_weight_max": 89.19428253173828,
      "activations/layer7_attention_weight_min": -86.93205261230469,
      "activations/layer8_attention_weight_max": 40.766448974609375,
      "activations/layer8_attention_weight_min": -43.41862487792969,
      "activations/layer9_attention_weight_max": 36.63080596923828,
      "activations/layer9_attention_weight_min": -34.9349250793457,
      "epoch": 9.15,
      "learning_rate": 9.188977272727273e-05,
      "loss": 2.7951,
      "step": 157550
    },
    {
      "activations/layer0_attention_weight_max": 15.320330619812012,
      "activations/layer0_attention_weight_min": -12.976637840270996,
      "activations/layer10_attention_weight_max": 33.04487609863281,
      "activations/layer10_attention_weight_min": -32.767757415771484,
      "activations/layer11_attention_weight_max": 31.876502990722656,
      "activations/layer11_attention_weight_min": -31.886058807373047,
      "activations/layer12_attention_weight_max": 27.679397583007812,
      "activations/layer12_attention_weight_min": -26.017343521118164,
      "activations/layer13_attention_weight_max": 39.05365753173828,
      "activations/layer13_attention_weight_min": -35.335731506347656,
      "activations/layer14_attention_weight_max": 54.378379821777344,
      "activations/layer14_attention_weight_min": -41.158470153808594,
      "activations/layer15_attention_weight_max": 35.91177749633789,
      "activations/layer15_attention_weight_min": -32.309200286865234,
      "activations/layer16_attention_weight_max": 39.114933013916016,
      "activations/layer16_attention_weight_min": -27.731117248535156,
      "activations/layer17_attention_weight_max": 32.761985778808594,
      "activations/layer17_attention_weight_min": -26.773244857788086,
      "activations/layer18_attention_weight_max": 30.656314849853516,
      "activations/layer18_attention_weight_min": -22.93764877319336,
      "activations/layer19_attention_weight_max": 42.27184295654297,
      "activations/layer19_attention_weight_min": -29.42337989807129,
      "activations/layer1_attention_weight_max": 17.143890380859375,
      "activations/layer1_attention_weight_min": -15.609244346618652,
      "activations/layer20_attention_weight_max": 29.58489990234375,
      "activations/layer20_attention_weight_min": -24.494976043701172,
      "activations/layer21_attention_weight_max": 31.600772857666016,
      "activations/layer21_attention_weight_min": -21.53264045715332,
      "activations/layer22_attention_weight_max": 45.1688232421875,
      "activations/layer22_attention_weight_min": -28.25251579284668,
      "activations/layer23_attention_weight_max": 39.32950973510742,
      "activations/layer23_attention_weight_min": -21.897382736206055,
      "activations/layer2_attention_weight_max": 33.1591682434082,
      "activations/layer2_attention_weight_min": -32.8558349609375,
      "activations/layer3_attention_weight_max": 94.85737609863281,
      "activations/layer3_attention_weight_min": -98.44825744628906,
      "activations/layer4_attention_weight_max": 117.3758773803711,
      "activations/layer4_attention_weight_min": -118.06194305419922,
      "activations/layer5_attention_weight_max": 53.23630905151367,
      "activations/layer5_attention_weight_min": -62.170867919921875,
      "activations/layer6_attention_weight_max": 47.182594299316406,
      "activations/layer6_attention_weight_min": -49.188438415527344,
      "activations/layer7_attention_weight_max": 84.30608367919922,
      "activations/layer7_attention_weight_min": -90.30374908447266,
      "activations/layer8_attention_weight_max": 39.59404373168945,
      "activations/layer8_attention_weight_min": -41.6639518737793,
      "activations/layer9_attention_weight_max": 33.68079376220703,
      "activations/layer9_attention_weight_min": -35.83767318725586,
      "epoch": 9.16,
      "learning_rate": 9.187083333333333e-05,
      "loss": 2.7746,
      "step": 157600
    },
    {
      "activations/layer0_attention_weight_max": 16.41579246520996,
      "activations/layer0_attention_weight_min": -12.603840827941895,
      "activations/layer10_attention_weight_max": 32.583858489990234,
      "activations/layer10_attention_weight_min": -31.28360366821289,
      "activations/layer11_attention_weight_max": 33.78799819946289,
      "activations/layer11_attention_weight_min": -29.722034454345703,
      "activations/layer12_attention_weight_max": 22.460323333740234,
      "activations/layer12_attention_weight_min": -22.97393035888672,
      "activations/layer13_attention_weight_max": 35.39682388305664,
      "activations/layer13_attention_weight_min": -33.70553207397461,
      "activations/layer14_attention_weight_max": 47.469844818115234,
      "activations/layer14_attention_weight_min": -41.04896545410156,
      "activations/layer15_attention_weight_max": 33.69976043701172,
      "activations/layer15_attention_weight_min": -30.511598587036133,
      "activations/layer16_attention_weight_max": 32.543209075927734,
      "activations/layer16_attention_weight_min": -28.3590030670166,
      "activations/layer17_attention_weight_max": 32.05104446411133,
      "activations/layer17_attention_weight_min": -25.37626838684082,
      "activations/layer18_attention_weight_max": 31.012306213378906,
      "activations/layer18_attention_weight_min": -23.07811164855957,
      "activations/layer19_attention_weight_max": 33.815303802490234,
      "activations/layer19_attention_weight_min": -29.166685104370117,
      "activations/layer1_attention_weight_max": 17.183212280273438,
      "activations/layer1_attention_weight_min": -14.346203804016113,
      "activations/layer20_attention_weight_max": 28.610029220581055,
      "activations/layer20_attention_weight_min": -25.045835494995117,
      "activations/layer21_attention_weight_max": 27.07036590576172,
      "activations/layer21_attention_weight_min": -22.534042358398438,
      "activations/layer22_attention_weight_max": 47.269527435302734,
      "activations/layer22_attention_weight_min": -29.869659423828125,
      "activations/layer23_attention_weight_max": 42.84393310546875,
      "activations/layer23_attention_weight_min": -25.945533752441406,
      "activations/layer2_attention_weight_max": 33.38220977783203,
      "activations/layer2_attention_weight_min": -34.326942443847656,
      "activations/layer3_attention_weight_max": 95.96111297607422,
      "activations/layer3_attention_weight_min": -100.85723876953125,
      "activations/layer4_attention_weight_max": 121.37525939941406,
      "activations/layer4_attention_weight_min": -124.13201141357422,
      "activations/layer5_attention_weight_max": 54.890899658203125,
      "activations/layer5_attention_weight_min": -66.58223724365234,
      "activations/layer6_attention_weight_max": 49.44258117675781,
      "activations/layer6_attention_weight_min": -52.34299087524414,
      "activations/layer7_attention_weight_max": 90.64530944824219,
      "activations/layer7_attention_weight_min": -87.73697662353516,
      "activations/layer8_attention_weight_max": 41.21114730834961,
      "activations/layer8_attention_weight_min": -41.508541107177734,
      "activations/layer9_attention_weight_max": 34.60445785522461,
      "activations/layer9_attention_weight_min": -33.58695602416992,
      "epoch": 9.16,
      "learning_rate": 9.185189393939393e-05,
      "loss": 2.7751,
      "step": 157650
    },
    {
      "activations/layer0_attention_weight_max": 15.712409019470215,
      "activations/layer0_attention_weight_min": -11.898301124572754,
      "activations/layer10_attention_weight_max": 34.585205078125,
      "activations/layer10_attention_weight_min": -33.6480827331543,
      "activations/layer11_attention_weight_max": 38.455970764160156,
      "activations/layer11_attention_weight_min": -34.08633041381836,
      "activations/layer12_attention_weight_max": 24.299819946289062,
      "activations/layer12_attention_weight_min": -25.33909797668457,
      "activations/layer13_attention_weight_max": 41.31422424316406,
      "activations/layer13_attention_weight_min": -33.177955627441406,
      "activations/layer14_attention_weight_max": 45.8004264831543,
      "activations/layer14_attention_weight_min": -38.499549865722656,
      "activations/layer15_attention_weight_max": 35.43840026855469,
      "activations/layer15_attention_weight_min": -31.217565536499023,
      "activations/layer16_attention_weight_max": 31.259403228759766,
      "activations/layer16_attention_weight_min": -27.883724212646484,
      "activations/layer17_attention_weight_max": 32.29405975341797,
      "activations/layer17_attention_weight_min": -27.67708396911621,
      "activations/layer18_attention_weight_max": 31.737455368041992,
      "activations/layer18_attention_weight_min": -23.400896072387695,
      "activations/layer19_attention_weight_max": 39.527950286865234,
      "activations/layer19_attention_weight_min": -31.641103744506836,
      "activations/layer1_attention_weight_max": 17.737524032592773,
      "activations/layer1_attention_weight_min": -14.067432403564453,
      "activations/layer20_attention_weight_max": 33.76979446411133,
      "activations/layer20_attention_weight_min": -26.654102325439453,
      "activations/layer21_attention_weight_max": 33.367637634277344,
      "activations/layer21_attention_weight_min": -24.66823387145996,
      "activations/layer22_attention_weight_max": 47.044498443603516,
      "activations/layer22_attention_weight_min": -30.653331756591797,
      "activations/layer23_attention_weight_max": 37.12303161621094,
      "activations/layer23_attention_weight_min": -24.892377853393555,
      "activations/layer2_attention_weight_max": 32.863677978515625,
      "activations/layer2_attention_weight_min": -31.932464599609375,
      "activations/layer3_attention_weight_max": 96.59268188476562,
      "activations/layer3_attention_weight_min": -94.54464721679688,
      "activations/layer4_attention_weight_max": 113.98052978515625,
      "activations/layer4_attention_weight_min": -112.85320281982422,
      "activations/layer5_attention_weight_max": 53.648189544677734,
      "activations/layer5_attention_weight_min": -66.17608642578125,
      "activations/layer6_attention_weight_max": 48.560733795166016,
      "activations/layer6_attention_weight_min": -49.34837341308594,
      "activations/layer7_attention_weight_max": 91.11201477050781,
      "activations/layer7_attention_weight_min": -88.42960357666016,
      "activations/layer8_attention_weight_max": 43.018829345703125,
      "activations/layer8_attention_weight_min": -41.144378662109375,
      "activations/layer9_attention_weight_max": 36.45345687866211,
      "activations/layer9_attention_weight_min": -34.7208366394043,
      "epoch": 9.16,
      "learning_rate": 9.183295454545454e-05,
      "loss": 2.7985,
      "step": 157700
    },
    {
      "activations/layer0_attention_weight_max": 16.000093460083008,
      "activations/layer0_attention_weight_min": -12.415189743041992,
      "activations/layer10_attention_weight_max": 32.730384826660156,
      "activations/layer10_attention_weight_min": -33.5753173828125,
      "activations/layer11_attention_weight_max": 34.62226104736328,
      "activations/layer11_attention_weight_min": -32.93682098388672,
      "activations/layer12_attention_weight_max": 24.174354553222656,
      "activations/layer12_attention_weight_min": -26.79425621032715,
      "activations/layer13_attention_weight_max": 40.27838134765625,
      "activations/layer13_attention_weight_min": -33.463417053222656,
      "activations/layer14_attention_weight_max": 49.87368392944336,
      "activations/layer14_attention_weight_min": -40.70515060424805,
      "activations/layer15_attention_weight_max": 36.31719970703125,
      "activations/layer15_attention_weight_min": -29.95241928100586,
      "activations/layer16_attention_weight_max": 32.60144805908203,
      "activations/layer16_attention_weight_min": -28.5460262298584,
      "activations/layer17_attention_weight_max": 32.863956451416016,
      "activations/layer17_attention_weight_min": -27.36463165283203,
      "activations/layer18_attention_weight_max": 28.51475715637207,
      "activations/layer18_attention_weight_min": -22.349966049194336,
      "activations/layer19_attention_weight_max": 38.811256408691406,
      "activations/layer19_attention_weight_min": -31.094358444213867,
      "activations/layer1_attention_weight_max": 16.746257781982422,
      "activations/layer1_attention_weight_min": -14.802842140197754,
      "activations/layer20_attention_weight_max": 30.488115310668945,
      "activations/layer20_attention_weight_min": -24.618024826049805,
      "activations/layer21_attention_weight_max": 29.44350242614746,
      "activations/layer21_attention_weight_min": -23.84404754638672,
      "activations/layer22_attention_weight_max": 49.17756652832031,
      "activations/layer22_attention_weight_min": -27.544784545898438,
      "activations/layer23_attention_weight_max": 37.14579391479492,
      "activations/layer23_attention_weight_min": -23.96611785888672,
      "activations/layer2_attention_weight_max": 33.8649787902832,
      "activations/layer2_attention_weight_min": -34.307289123535156,
      "activations/layer3_attention_weight_max": 99.06185150146484,
      "activations/layer3_attention_weight_min": -101.23585510253906,
      "activations/layer4_attention_weight_max": 126.5183334350586,
      "activations/layer4_attention_weight_min": -124.4590072631836,
      "activations/layer5_attention_weight_max": 55.24181365966797,
      "activations/layer5_attention_weight_min": -62.271907806396484,
      "activations/layer6_attention_weight_max": 48.72382354736328,
      "activations/layer6_attention_weight_min": -53.28632354736328,
      "activations/layer7_attention_weight_max": 87.98711395263672,
      "activations/layer7_attention_weight_min": -94.54312133789062,
      "activations/layer8_attention_weight_max": 40.03159713745117,
      "activations/layer8_attention_weight_min": -42.79158401489258,
      "activations/layer9_attention_weight_max": 35.146488189697266,
      "activations/layer9_attention_weight_min": -34.30276107788086,
      "epoch": 9.17,
      "learning_rate": 9.181401515151515e-05,
      "loss": 2.7947,
      "step": 157750
    },
    {
      "activations/layer0_attention_weight_max": 16.922317504882812,
      "activations/layer0_attention_weight_min": -12.44519329071045,
      "activations/layer10_attention_weight_max": 33.69320297241211,
      "activations/layer10_attention_weight_min": -30.965274810791016,
      "activations/layer11_attention_weight_max": 34.32192611694336,
      "activations/layer11_attention_weight_min": -30.832183837890625,
      "activations/layer12_attention_weight_max": 22.82299041748047,
      "activations/layer12_attention_weight_min": -22.749855041503906,
      "activations/layer13_attention_weight_max": 34.04435348510742,
      "activations/layer13_attention_weight_min": -31.9321346282959,
      "activations/layer14_attention_weight_max": 41.26648712158203,
      "activations/layer14_attention_weight_min": -35.93635940551758,
      "activations/layer15_attention_weight_max": 34.758121490478516,
      "activations/layer15_attention_weight_min": -28.73796844482422,
      "activations/layer16_attention_weight_max": 30.594327926635742,
      "activations/layer16_attention_weight_min": -25.990938186645508,
      "activations/layer17_attention_weight_max": 34.21001434326172,
      "activations/layer17_attention_weight_min": -22.954574584960938,
      "activations/layer18_attention_weight_max": 29.602741241455078,
      "activations/layer18_attention_weight_min": -21.36748504638672,
      "activations/layer19_attention_weight_max": 35.6933479309082,
      "activations/layer19_attention_weight_min": -30.251724243164062,
      "activations/layer1_attention_weight_max": 16.82219123840332,
      "activations/layer1_attention_weight_min": -16.1241512298584,
      "activations/layer20_attention_weight_max": 28.154211044311523,
      "activations/layer20_attention_weight_min": -24.38047218322754,
      "activations/layer21_attention_weight_max": 27.136638641357422,
      "activations/layer21_attention_weight_min": -23.569292068481445,
      "activations/layer22_attention_weight_max": 44.333431243896484,
      "activations/layer22_attention_weight_min": -29.556089401245117,
      "activations/layer23_attention_weight_max": 36.70774841308594,
      "activations/layer23_attention_weight_min": -23.124404907226562,
      "activations/layer2_attention_weight_max": 32.143672943115234,
      "activations/layer2_attention_weight_min": -32.95564651489258,
      "activations/layer3_attention_weight_max": 94.49732208251953,
      "activations/layer3_attention_weight_min": -91.07695007324219,
      "activations/layer4_attention_weight_max": 117.40785217285156,
      "activations/layer4_attention_weight_min": -114.06391906738281,
      "activations/layer5_attention_weight_max": 52.84385299682617,
      "activations/layer5_attention_weight_min": -63.952392578125,
      "activations/layer6_attention_weight_max": 45.841434478759766,
      "activations/layer6_attention_weight_min": -51.62950897216797,
      "activations/layer7_attention_weight_max": 84.73200225830078,
      "activations/layer7_attention_weight_min": -87.99879455566406,
      "activations/layer8_attention_weight_max": 39.22620391845703,
      "activations/layer8_attention_weight_min": -39.41048049926758,
      "activations/layer9_attention_weight_max": 34.196075439453125,
      "activations/layer9_attention_weight_min": -32.922725677490234,
      "epoch": 9.17,
      "learning_rate": 9.179507575757575e-05,
      "loss": 2.7949,
      "step": 157800
    },
    {
      "activations/layer0_attention_weight_max": 15.839616775512695,
      "activations/layer0_attention_weight_min": -12.398713111877441,
      "activations/layer10_attention_weight_max": 33.970359802246094,
      "activations/layer10_attention_weight_min": -33.4028205871582,
      "activations/layer11_attention_weight_max": 35.642486572265625,
      "activations/layer11_attention_weight_min": -30.994672775268555,
      "activations/layer12_attention_weight_max": 25.37899398803711,
      "activations/layer12_attention_weight_min": -26.36515235900879,
      "activations/layer13_attention_weight_max": 42.62508773803711,
      "activations/layer13_attention_weight_min": -33.20813751220703,
      "activations/layer14_attention_weight_max": 52.353660583496094,
      "activations/layer14_attention_weight_min": -43.46055221557617,
      "activations/layer15_attention_weight_max": 39.41552734375,
      "activations/layer15_attention_weight_min": -29.620258331298828,
      "activations/layer16_attention_weight_max": 31.993961334228516,
      "activations/layer16_attention_weight_min": -27.255531311035156,
      "activations/layer17_attention_weight_max": 32.32093811035156,
      "activations/layer17_attention_weight_min": -25.348304748535156,
      "activations/layer18_attention_weight_max": 29.640850067138672,
      "activations/layer18_attention_weight_min": -23.52743148803711,
      "activations/layer19_attention_weight_max": 34.122982025146484,
      "activations/layer19_attention_weight_min": -31.35504150390625,
      "activations/layer1_attention_weight_max": 16.506671905517578,
      "activations/layer1_attention_weight_min": -15.42148208618164,
      "activations/layer20_attention_weight_max": 29.248872756958008,
      "activations/layer20_attention_weight_min": -24.519933700561523,
      "activations/layer21_attention_weight_max": 28.226985931396484,
      "activations/layer21_attention_weight_min": -22.22884178161621,
      "activations/layer22_attention_weight_max": 46.490909576416016,
      "activations/layer22_attention_weight_min": -32.44084548950195,
      "activations/layer23_attention_weight_max": 35.04627227783203,
      "activations/layer23_attention_weight_min": -23.799053192138672,
      "activations/layer2_attention_weight_max": 33.8929443359375,
      "activations/layer2_attention_weight_min": -32.117637634277344,
      "activations/layer3_attention_weight_max": 92.5370101928711,
      "activations/layer3_attention_weight_min": -93.36334228515625,
      "activations/layer4_attention_weight_max": 113.19829559326172,
      "activations/layer4_attention_weight_min": -115.5367431640625,
      "activations/layer5_attention_weight_max": 55.83645248413086,
      "activations/layer5_attention_weight_min": -60.43627166748047,
      "activations/layer6_attention_weight_max": 48.4368782043457,
      "activations/layer6_attention_weight_min": -52.10323715209961,
      "activations/layer7_attention_weight_max": 89.59563446044922,
      "activations/layer7_attention_weight_min": -97.2615966796875,
      "activations/layer8_attention_weight_max": 41.69811248779297,
      "activations/layer8_attention_weight_min": -44.26744079589844,
      "activations/layer9_attention_weight_max": 34.4306526184082,
      "activations/layer9_attention_weight_min": -35.730281829833984,
      "epoch": 9.17,
      "learning_rate": 9.177613636363635e-05,
      "loss": 2.8059,
      "step": 157850
    },
    {
      "activations/layer0_attention_weight_max": 15.80446720123291,
      "activations/layer0_attention_weight_min": -12.53181266784668,
      "activations/layer10_attention_weight_max": 32.96332931518555,
      "activations/layer10_attention_weight_min": -32.065948486328125,
      "activations/layer11_attention_weight_max": 32.261985778808594,
      "activations/layer11_attention_weight_min": -32.55882263183594,
      "activations/layer12_attention_weight_max": 23.597198486328125,
      "activations/layer12_attention_weight_min": -29.162460327148438,
      "activations/layer13_attention_weight_max": 37.0232048034668,
      "activations/layer13_attention_weight_min": -33.790245056152344,
      "activations/layer14_attention_weight_max": 43.64371109008789,
      "activations/layer14_attention_weight_min": -36.69064712524414,
      "activations/layer15_attention_weight_max": 34.22132873535156,
      "activations/layer15_attention_weight_min": -29.88210678100586,
      "activations/layer16_attention_weight_max": 31.88828468322754,
      "activations/layer16_attention_weight_min": -27.712547302246094,
      "activations/layer17_attention_weight_max": 31.513383865356445,
      "activations/layer17_attention_weight_min": -24.830434799194336,
      "activations/layer18_attention_weight_max": 31.713523864746094,
      "activations/layer18_attention_weight_min": -22.360454559326172,
      "activations/layer19_attention_weight_max": 32.49611282348633,
      "activations/layer19_attention_weight_min": -32.036678314208984,
      "activations/layer1_attention_weight_max": 16.137731552124023,
      "activations/layer1_attention_weight_min": -15.090999603271484,
      "activations/layer20_attention_weight_max": 27.705461502075195,
      "activations/layer20_attention_weight_min": -24.252788543701172,
      "activations/layer21_attention_weight_max": 27.261747360229492,
      "activations/layer21_attention_weight_min": -22.0866756439209,
      "activations/layer22_attention_weight_max": 41.16780090332031,
      "activations/layer22_attention_weight_min": -27.955286026000977,
      "activations/layer23_attention_weight_max": 34.23280334472656,
      "activations/layer23_attention_weight_min": -23.147621154785156,
      "activations/layer2_attention_weight_max": 32.35855484008789,
      "activations/layer2_attention_weight_min": -32.53249740600586,
      "activations/layer3_attention_weight_max": 91.00025939941406,
      "activations/layer3_attention_weight_min": -94.92464447021484,
      "activations/layer4_attention_weight_max": 115.71026611328125,
      "activations/layer4_attention_weight_min": -115.58424377441406,
      "activations/layer5_attention_weight_max": 55.87261199951172,
      "activations/layer5_attention_weight_min": -62.20660400390625,
      "activations/layer6_attention_weight_max": 46.92427062988281,
      "activations/layer6_attention_weight_min": -50.64803695678711,
      "activations/layer7_attention_weight_max": 85.85759735107422,
      "activations/layer7_attention_weight_min": -88.6355972290039,
      "activations/layer8_attention_weight_max": 39.768165588378906,
      "activations/layer8_attention_weight_min": -40.18225860595703,
      "activations/layer9_attention_weight_max": 35.320037841796875,
      "activations/layer9_attention_weight_min": -33.20805358886719,
      "epoch": 9.17,
      "learning_rate": 9.175719696969696e-05,
      "loss": 2.8053,
      "step": 157900
    },
    {
      "activations/layer0_attention_weight_max": 15.897109985351562,
      "activations/layer0_attention_weight_min": -12.247261047363281,
      "activations/layer10_attention_weight_max": 33.831932067871094,
      "activations/layer10_attention_weight_min": -33.123329162597656,
      "activations/layer11_attention_weight_max": 31.060482025146484,
      "activations/layer11_attention_weight_min": -30.78252410888672,
      "activations/layer12_attention_weight_max": 22.66252326965332,
      "activations/layer12_attention_weight_min": -28.963525772094727,
      "activations/layer13_attention_weight_max": 37.0699462890625,
      "activations/layer13_attention_weight_min": -33.118934631347656,
      "activations/layer14_attention_weight_max": 44.88654327392578,
      "activations/layer14_attention_weight_min": -39.15586853027344,
      "activations/layer15_attention_weight_max": 34.37192916870117,
      "activations/layer15_attention_weight_min": -30.756216049194336,
      "activations/layer16_attention_weight_max": 32.64712905883789,
      "activations/layer16_attention_weight_min": -27.575895309448242,
      "activations/layer17_attention_weight_max": 33.53215408325195,
      "activations/layer17_attention_weight_min": -25.42535972595215,
      "activations/layer18_attention_weight_max": 32.250457763671875,
      "activations/layer18_attention_weight_min": -21.9980525970459,
      "activations/layer19_attention_weight_max": 39.02310562133789,
      "activations/layer19_attention_weight_min": -32.32267379760742,
      "activations/layer1_attention_weight_max": 15.722772598266602,
      "activations/layer1_attention_weight_min": -14.332859992980957,
      "activations/layer20_attention_weight_max": 31.77260398864746,
      "activations/layer20_attention_weight_min": -25.028783798217773,
      "activations/layer21_attention_weight_max": 29.398466110229492,
      "activations/layer21_attention_weight_min": -25.39281463623047,
      "activations/layer22_attention_weight_max": 44.08294677734375,
      "activations/layer22_attention_weight_min": -31.32562828063965,
      "activations/layer23_attention_weight_max": 34.146236419677734,
      "activations/layer23_attention_weight_min": -24.52583122253418,
      "activations/layer2_attention_weight_max": 33.269195556640625,
      "activations/layer2_attention_weight_min": -31.26079559326172,
      "activations/layer3_attention_weight_max": 93.57744598388672,
      "activations/layer3_attention_weight_min": -92.58070373535156,
      "activations/layer4_attention_weight_max": 117.3634033203125,
      "activations/layer4_attention_weight_min": -114.96284484863281,
      "activations/layer5_attention_weight_max": 55.395713806152344,
      "activations/layer5_attention_weight_min": -63.279212951660156,
      "activations/layer6_attention_weight_max": 48.70946502685547,
      "activations/layer6_attention_weight_min": -50.601322174072266,
      "activations/layer7_attention_weight_max": 92.99747467041016,
      "activations/layer7_attention_weight_min": -89.50112915039062,
      "activations/layer8_attention_weight_max": 40.588504791259766,
      "activations/layer8_attention_weight_min": -40.809329986572266,
      "activations/layer9_attention_weight_max": 34.62367248535156,
      "activations/layer9_attention_weight_min": -33.43088912963867,
      "epoch": 9.18,
      "learning_rate": 9.173825757575758e-05,
      "loss": 2.7967,
      "step": 157950
    },
    {
      "activations/layer0_attention_weight_max": 15.99130630493164,
      "activations/layer0_attention_weight_min": -11.762147903442383,
      "activations/layer10_attention_weight_max": 33.616539001464844,
      "activations/layer10_attention_weight_min": -33.9226188659668,
      "activations/layer11_attention_weight_max": 35.57170867919922,
      "activations/layer11_attention_weight_min": -32.392974853515625,
      "activations/layer12_attention_weight_max": 25.772396087646484,
      "activations/layer12_attention_weight_min": -25.775476455688477,
      "activations/layer13_attention_weight_max": 44.625083923339844,
      "activations/layer13_attention_weight_min": -35.619693756103516,
      "activations/layer14_attention_weight_max": 50.323001861572266,
      "activations/layer14_attention_weight_min": -42.82734680175781,
      "activations/layer15_attention_weight_max": 38.42119216918945,
      "activations/layer15_attention_weight_min": -31.1840877532959,
      "activations/layer16_attention_weight_max": 34.002586364746094,
      "activations/layer16_attention_weight_min": -28.25053596496582,
      "activations/layer17_attention_weight_max": 38.094852447509766,
      "activations/layer17_attention_weight_min": -28.89236068725586,
      "activations/layer18_attention_weight_max": 36.9594612121582,
      "activations/layer18_attention_weight_min": -24.291467666625977,
      "activations/layer19_attention_weight_max": 41.999507904052734,
      "activations/layer19_attention_weight_min": -33.36767578125,
      "activations/layer1_attention_weight_max": 17.547237396240234,
      "activations/layer1_attention_weight_min": -14.110511779785156,
      "activations/layer20_attention_weight_max": 34.8820915222168,
      "activations/layer20_attention_weight_min": -25.957202911376953,
      "activations/layer21_attention_weight_max": 36.51599884033203,
      "activations/layer21_attention_weight_min": -26.341861724853516,
      "activations/layer22_attention_weight_max": 62.187957763671875,
      "activations/layer22_attention_weight_min": -35.1934814453125,
      "activations/layer23_attention_weight_max": 44.642608642578125,
      "activations/layer23_attention_weight_min": -22.538028717041016,
      "activations/layer2_attention_weight_max": 33.98088836669922,
      "activations/layer2_attention_weight_min": -33.97954559326172,
      "activations/layer3_attention_weight_max": 96.62322998046875,
      "activations/layer3_attention_weight_min": -100.5411148071289,
      "activations/layer4_attention_weight_max": 122.0057144165039,
      "activations/layer4_attention_weight_min": -122.74285125732422,
      "activations/layer5_attention_weight_max": 54.7336540222168,
      "activations/layer5_attention_weight_min": -63.49031066894531,
      "activations/layer6_attention_weight_max": 47.893680572509766,
      "activations/layer6_attention_weight_min": -51.24790573120117,
      "activations/layer7_attention_weight_max": 92.7671127319336,
      "activations/layer7_attention_weight_min": -94.04436492919922,
      "activations/layer8_attention_weight_max": 42.120662689208984,
      "activations/layer8_attention_weight_min": -43.68922805786133,
      "activations/layer9_attention_weight_max": 36.76824951171875,
      "activations/layer9_attention_weight_min": -35.0919075012207,
      "epoch": 9.18,
      "learning_rate": 9.171931818181817e-05,
      "loss": 2.8015,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4991,
      "eval_samples_per_second": 505.228,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4991,
      "eval_openwebtext_samples_per_second": 505.228,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 1.9006,
      "eval_wikitext_samples_per_second": 239.918,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_lambada_loss": 2.689453125,
      "eval_lambada_ppl": 14.723621737689907,
      "eval_lambada_runtime": 9.531,
      "eval_lambada_samples_per_second": 510.862,
      "step": 158000
    },
    {
      "activations/layer0_attention_weight_max": 15.5701265335083,
      "activations/layer0_attention_weight_min": -12.276920318603516,
      "activations/layer10_attention_weight_max": 35.36063003540039,
      "activations/layer10_attention_weight_min": -33.142066955566406,
      "activations/layer11_attention_weight_max": 34.53168869018555,
      "activations/layer11_attention_weight_min": -32.349632263183594,
      "activations/layer12_attention_weight_max": 27.46046257019043,
      "activations/layer12_attention_weight_min": -28.32485580444336,
      "activations/layer13_attention_weight_max": 43.80068588256836,
      "activations/layer13_attention_weight_min": -32.074851989746094,
      "activations/layer14_attention_weight_max": 53.1450309753418,
      "activations/layer14_attention_weight_min": -37.77068328857422,
      "activations/layer15_attention_weight_max": 41.72517013549805,
      "activations/layer15_attention_weight_min": -32.000396728515625,
      "activations/layer16_attention_weight_max": 36.249271392822266,
      "activations/layer16_attention_weight_min": -28.771390914916992,
      "activations/layer17_attention_weight_max": 37.34109878540039,
      "activations/layer17_attention_weight_min": -25.026508331298828,
      "activations/layer18_attention_weight_max": 32.78179168701172,
      "activations/layer18_attention_weight_min": -21.52594566345215,
      "activations/layer19_attention_weight_max": 39.479248046875,
      "activations/layer19_attention_weight_min": -31.119653701782227,
      "activations/layer1_attention_weight_max": 17.5037841796875,
      "activations/layer1_attention_weight_min": -13.489375114440918,
      "activations/layer20_attention_weight_max": 34.63786315917969,
      "activations/layer20_attention_weight_min": -22.656021118164062,
      "activations/layer21_attention_weight_max": 33.01877212524414,
      "activations/layer21_attention_weight_min": -22.448486328125,
      "activations/layer22_attention_weight_max": 49.83714294433594,
      "activations/layer22_attention_weight_min": -26.352983474731445,
      "activations/layer23_attention_weight_max": 40.632545471191406,
      "activations/layer23_attention_weight_min": -22.51968765258789,
      "activations/layer2_attention_weight_max": 33.50587463378906,
      "activations/layer2_attention_weight_min": -32.37950897216797,
      "activations/layer3_attention_weight_max": 95.42706298828125,
      "activations/layer3_attention_weight_min": -94.30139923095703,
      "activations/layer4_attention_weight_max": 120.31253814697266,
      "activations/layer4_attention_weight_min": -113.2863540649414,
      "activations/layer5_attention_weight_max": 55.046199798583984,
      "activations/layer5_attention_weight_min": -61.12110137939453,
      "activations/layer6_attention_weight_max": 49.35433578491211,
      "activations/layer6_attention_weight_min": -50.739173889160156,
      "activations/layer7_attention_weight_max": 92.03410339355469,
      "activations/layer7_attention_weight_min": -92.80167388916016,
      "activations/layer8_attention_weight_max": 44.238285064697266,
      "activations/layer8_attention_weight_min": -41.14997100830078,
      "activations/layer9_attention_weight_max": 38.582393646240234,
      "activations/layer9_attention_weight_min": -35.788028717041016,
      "epoch": 9.18,
      "learning_rate": 9.170037878787878e-05,
      "loss": 2.7895,
      "step": 158050
    },
    {
      "activations/layer0_attention_weight_max": 15.480267524719238,
      "activations/layer0_attention_weight_min": -11.93420696258545,
      "activations/layer10_attention_weight_max": 38.34526443481445,
      "activations/layer10_attention_weight_min": -32.63177490234375,
      "activations/layer11_attention_weight_max": 41.122901916503906,
      "activations/layer11_attention_weight_min": -34.4921989440918,
      "activations/layer12_attention_weight_max": 26.807249069213867,
      "activations/layer12_attention_weight_min": -25.016382217407227,
      "activations/layer13_attention_weight_max": 47.20323944091797,
      "activations/layer13_attention_weight_min": -35.76637268066406,
      "activations/layer14_attention_weight_max": 49.901214599609375,
      "activations/layer14_attention_weight_min": -45.128787994384766,
      "activations/layer15_attention_weight_max": 41.59598159790039,
      "activations/layer15_attention_weight_min": -33.983585357666016,
      "activations/layer16_attention_weight_max": 38.10268783569336,
      "activations/layer16_attention_weight_min": -28.810731887817383,
      "activations/layer17_attention_weight_max": 34.29777526855469,
      "activations/layer17_attention_weight_min": -25.831560134887695,
      "activations/layer18_attention_weight_max": 31.24416732788086,
      "activations/layer18_attention_weight_min": -22.930768966674805,
      "activations/layer19_attention_weight_max": 36.18362808227539,
      "activations/layer19_attention_weight_min": -30.308521270751953,
      "activations/layer1_attention_weight_max": 16.931215286254883,
      "activations/layer1_attention_weight_min": -16.610536575317383,
      "activations/layer20_attention_weight_max": 30.187166213989258,
      "activations/layer20_attention_weight_min": -24.781644821166992,
      "activations/layer21_attention_weight_max": 28.774858474731445,
      "activations/layer21_attention_weight_min": -24.8593807220459,
      "activations/layer22_attention_weight_max": 46.505096435546875,
      "activations/layer22_attention_weight_min": -30.23944091796875,
      "activations/layer23_attention_weight_max": 36.318885803222656,
      "activations/layer23_attention_weight_min": -24.59332847595215,
      "activations/layer2_attention_weight_max": 33.00091552734375,
      "activations/layer2_attention_weight_min": -33.188697814941406,
      "activations/layer3_attention_weight_max": 92.0562744140625,
      "activations/layer3_attention_weight_min": -94.96095275878906,
      "activations/layer4_attention_weight_max": 116.02310943603516,
      "activations/layer4_attention_weight_min": -113.64332580566406,
      "activations/layer5_attention_weight_max": 55.335811614990234,
      "activations/layer5_attention_weight_min": -60.77867889404297,
      "activations/layer6_attention_weight_max": 48.01182174682617,
      "activations/layer6_attention_weight_min": -49.07500457763672,
      "activations/layer7_attention_weight_max": 95.46459197998047,
      "activations/layer7_attention_weight_min": -87.51880645751953,
      "activations/layer8_attention_weight_max": 43.58086395263672,
      "activations/layer8_attention_weight_min": -40.735504150390625,
      "activations/layer9_attention_weight_max": 37.03934860229492,
      "activations/layer9_attention_weight_min": -33.04514694213867,
      "epoch": 9.19,
      "learning_rate": 9.16814393939394e-05,
      "loss": 2.8048,
      "step": 158100
    },
    {
      "activations/layer0_attention_weight_max": 16.381227493286133,
      "activations/layer0_attention_weight_min": -13.204547882080078,
      "activations/layer10_attention_weight_max": 37.144962310791016,
      "activations/layer10_attention_weight_min": -31.108518600463867,
      "activations/layer11_attention_weight_max": 36.19563293457031,
      "activations/layer11_attention_weight_min": -29.944854736328125,
      "activations/layer12_attention_weight_max": 24.52960968017578,
      "activations/layer12_attention_weight_min": -23.52944564819336,
      "activations/layer13_attention_weight_max": 40.65828323364258,
      "activations/layer13_attention_weight_min": -31.865814208984375,
      "activations/layer14_attention_weight_max": 43.353179931640625,
      "activations/layer14_attention_weight_min": -34.64756774902344,
      "activations/layer15_attention_weight_max": 34.137149810791016,
      "activations/layer15_attention_weight_min": -28.487905502319336,
      "activations/layer16_attention_weight_max": 31.690881729125977,
      "activations/layer16_attention_weight_min": -26.787538528442383,
      "activations/layer17_attention_weight_max": 32.73811340332031,
      "activations/layer17_attention_weight_min": -26.2755184173584,
      "activations/layer18_attention_weight_max": 32.552001953125,
      "activations/layer18_attention_weight_min": -22.697998046875,
      "activations/layer19_attention_weight_max": 36.58272171020508,
      "activations/layer19_attention_weight_min": -30.683034896850586,
      "activations/layer1_attention_weight_max": 17.258691787719727,
      "activations/layer1_attention_weight_min": -16.25828742980957,
      "activations/layer20_attention_weight_max": 32.557613372802734,
      "activations/layer20_attention_weight_min": -23.325361251831055,
      "activations/layer21_attention_weight_max": 29.372154235839844,
      "activations/layer21_attention_weight_min": -24.106592178344727,
      "activations/layer22_attention_weight_max": 53.57757568359375,
      "activations/layer22_attention_weight_min": -28.0316162109375,
      "activations/layer23_attention_weight_max": 39.28800964355469,
      "activations/layer23_attention_weight_min": -23.808263778686523,
      "activations/layer2_attention_weight_max": 34.25630569458008,
      "activations/layer2_attention_weight_min": -34.485557556152344,
      "activations/layer3_attention_weight_max": 95.96570587158203,
      "activations/layer3_attention_weight_min": -94.96100616455078,
      "activations/layer4_attention_weight_max": 119.43004608154297,
      "activations/layer4_attention_weight_min": -115.31383514404297,
      "activations/layer5_attention_weight_max": 54.93872833251953,
      "activations/layer5_attention_weight_min": -62.464622497558594,
      "activations/layer6_attention_weight_max": 47.15340805053711,
      "activations/layer6_attention_weight_min": -49.551700592041016,
      "activations/layer7_attention_weight_max": 91.46147155761719,
      "activations/layer7_attention_weight_min": -88.73826599121094,
      "activations/layer8_attention_weight_max": 41.92293930053711,
      "activations/layer8_attention_weight_min": -41.75740432739258,
      "activations/layer9_attention_weight_max": 35.328975677490234,
      "activations/layer9_attention_weight_min": -35.45736312866211,
      "epoch": 9.19,
      "learning_rate": 9.166249999999999e-05,
      "loss": 2.7887,
      "step": 158150
    },
    {
      "activations/layer0_attention_weight_max": 16.316131591796875,
      "activations/layer0_attention_weight_min": -13.487320899963379,
      "activations/layer10_attention_weight_max": 36.64045715332031,
      "activations/layer10_attention_weight_min": -33.9632568359375,
      "activations/layer11_attention_weight_max": 34.11284255981445,
      "activations/layer11_attention_weight_min": -32.69647216796875,
      "activations/layer12_attention_weight_max": 36.20717239379883,
      "activations/layer12_attention_weight_min": -24.80289649963379,
      "activations/layer13_attention_weight_max": 43.02427291870117,
      "activations/layer13_attention_weight_min": -37.94997787475586,
      "activations/layer14_attention_weight_max": 45.36371994018555,
      "activations/layer14_attention_weight_min": -40.070167541503906,
      "activations/layer15_attention_weight_max": 36.79805374145508,
      "activations/layer15_attention_weight_min": -35.81621551513672,
      "activations/layer16_attention_weight_max": 35.39611053466797,
      "activations/layer16_attention_weight_min": -28.942424774169922,
      "activations/layer17_attention_weight_max": 36.04049301147461,
      "activations/layer17_attention_weight_min": -28.20846176147461,
      "activations/layer18_attention_weight_max": 34.497886657714844,
      "activations/layer18_attention_weight_min": -24.667640686035156,
      "activations/layer19_attention_weight_max": 41.32106399536133,
      "activations/layer19_attention_weight_min": -35.242130279541016,
      "activations/layer1_attention_weight_max": 16.1936092376709,
      "activations/layer1_attention_weight_min": -14.2599458694458,
      "activations/layer20_attention_weight_max": 35.01717758178711,
      "activations/layer20_attention_weight_min": -26.944358825683594,
      "activations/layer21_attention_weight_max": 30.03973960876465,
      "activations/layer21_attention_weight_min": -24.047372817993164,
      "activations/layer22_attention_weight_max": 50.014278411865234,
      "activations/layer22_attention_weight_min": -31.215351104736328,
      "activations/layer23_attention_weight_max": 38.49425506591797,
      "activations/layer23_attention_weight_min": -24.41848373413086,
      "activations/layer2_attention_weight_max": 33.27154541015625,
      "activations/layer2_attention_weight_min": -32.97751998901367,
      "activations/layer3_attention_weight_max": 97.66175842285156,
      "activations/layer3_attention_weight_min": -97.92579650878906,
      "activations/layer4_attention_weight_max": 120.5806655883789,
      "activations/layer4_attention_weight_min": -121.49263000488281,
      "activations/layer5_attention_weight_max": 55.0769157409668,
      "activations/layer5_attention_weight_min": -63.14832305908203,
      "activations/layer6_attention_weight_max": 48.43533706665039,
      "activations/layer6_attention_weight_min": -50.41872024536133,
      "activations/layer7_attention_weight_max": 91.96381378173828,
      "activations/layer7_attention_weight_min": -85.97315979003906,
      "activations/layer8_attention_weight_max": 41.72750473022461,
      "activations/layer8_attention_weight_min": -39.4975471496582,
      "activations/layer9_attention_weight_max": 35.08946228027344,
      "activations/layer9_attention_weight_min": -34.55873489379883,
      "epoch": 9.19,
      "learning_rate": 9.16435606060606e-05,
      "loss": 2.785,
      "step": 158200
    },
    {
      "activations/layer0_attention_weight_max": 16.451757431030273,
      "activations/layer0_attention_weight_min": -12.398027420043945,
      "activations/layer10_attention_weight_max": 35.57783889770508,
      "activations/layer10_attention_weight_min": -32.326873779296875,
      "activations/layer11_attention_weight_max": 31.32440185546875,
      "activations/layer11_attention_weight_min": -30.755958557128906,
      "activations/layer12_attention_weight_max": 23.64369010925293,
      "activations/layer12_attention_weight_min": -24.78329849243164,
      "activations/layer13_attention_weight_max": 39.18121337890625,
      "activations/layer13_attention_weight_min": -32.831565856933594,
      "activations/layer14_attention_weight_max": 51.002933502197266,
      "activations/layer14_attention_weight_min": -39.009708404541016,
      "activations/layer15_attention_weight_max": 36.18534469604492,
      "activations/layer15_attention_weight_min": -30.395889282226562,
      "activations/layer16_attention_weight_max": 34.65653610229492,
      "activations/layer16_attention_weight_min": -29.575416564941406,
      "activations/layer17_attention_weight_max": 35.773414611816406,
      "activations/layer17_attention_weight_min": -26.874277114868164,
      "activations/layer18_attention_weight_max": 34.13749313354492,
      "activations/layer18_attention_weight_min": -23.773029327392578,
      "activations/layer19_attention_weight_max": 34.7808952331543,
      "activations/layer19_attention_weight_min": -32.40869140625,
      "activations/layer1_attention_weight_max": 15.840328216552734,
      "activations/layer1_attention_weight_min": -14.467609405517578,
      "activations/layer20_attention_weight_max": 31.242795944213867,
      "activations/layer20_attention_weight_min": -25.008134841918945,
      "activations/layer21_attention_weight_max": 29.1408748626709,
      "activations/layer21_attention_weight_min": -24.114688873291016,
      "activations/layer22_attention_weight_max": 50.92287826538086,
      "activations/layer22_attention_weight_min": -29.673229217529297,
      "activations/layer23_attention_weight_max": 35.888118743896484,
      "activations/layer23_attention_weight_min": -25.364994049072266,
      "activations/layer2_attention_weight_max": 32.36617660522461,
      "activations/layer2_attention_weight_min": -33.01036834716797,
      "activations/layer3_attention_weight_max": 96.42414855957031,
      "activations/layer3_attention_weight_min": -98.53459930419922,
      "activations/layer4_attention_weight_max": 121.29237365722656,
      "activations/layer4_attention_weight_min": -117.43279266357422,
      "activations/layer5_attention_weight_max": 53.11750793457031,
      "activations/layer5_attention_weight_min": -65.10455322265625,
      "activations/layer6_attention_weight_max": 45.15061569213867,
      "activations/layer6_attention_weight_min": -49.83348846435547,
      "activations/layer7_attention_weight_max": 89.25709533691406,
      "activations/layer7_attention_weight_min": -97.90342712402344,
      "activations/layer8_attention_weight_max": 38.727134704589844,
      "activations/layer8_attention_weight_min": -40.86307907104492,
      "activations/layer9_attention_weight_max": 34.35213088989258,
      "activations/layer9_attention_weight_min": -31.77692222595215,
      "epoch": 9.2,
      "learning_rate": 9.16246212121212e-05,
      "loss": 2.7833,
      "step": 158250
    },
    {
      "activations/layer0_attention_weight_max": 15.886322021484375,
      "activations/layer0_attention_weight_min": -13.06314754486084,
      "activations/layer10_attention_weight_max": 31.475418090820312,
      "activations/layer10_attention_weight_min": -31.071640014648438,
      "activations/layer11_attention_weight_max": 32.88702392578125,
      "activations/layer11_attention_weight_min": -30.093238830566406,
      "activations/layer12_attention_weight_max": 24.09581756591797,
      "activations/layer12_attention_weight_min": -26.14557456970215,
      "activations/layer13_attention_weight_max": 38.180381774902344,
      "activations/layer13_attention_weight_min": -31.326126098632812,
      "activations/layer14_attention_weight_max": 44.032352447509766,
      "activations/layer14_attention_weight_min": -35.93784713745117,
      "activations/layer15_attention_weight_max": 37.124176025390625,
      "activations/layer15_attention_weight_min": -29.563644409179688,
      "activations/layer16_attention_weight_max": 31.107332229614258,
      "activations/layer16_attention_weight_min": -26.517438888549805,
      "activations/layer17_attention_weight_max": 36.03126525878906,
      "activations/layer17_attention_weight_min": -25.694271087646484,
      "activations/layer18_attention_weight_max": 31.64177131652832,
      "activations/layer18_attention_weight_min": -21.597200393676758,
      "activations/layer19_attention_weight_max": 37.376853942871094,
      "activations/layer19_attention_weight_min": -31.308948516845703,
      "activations/layer1_attention_weight_max": 16.213117599487305,
      "activations/layer1_attention_weight_min": -14.271013259887695,
      "activations/layer20_attention_weight_max": 29.670974731445312,
      "activations/layer20_attention_weight_min": -25.130582809448242,
      "activations/layer21_attention_weight_max": 30.14565658569336,
      "activations/layer21_attention_weight_min": -23.69399070739746,
      "activations/layer22_attention_weight_max": 45.02008056640625,
      "activations/layer22_attention_weight_min": -29.874040603637695,
      "activations/layer23_attention_weight_max": 37.11382293701172,
      "activations/layer23_attention_weight_min": -26.12586784362793,
      "activations/layer2_attention_weight_max": 33.827552795410156,
      "activations/layer2_attention_weight_min": -32.68711853027344,
      "activations/layer3_attention_weight_max": 96.77928924560547,
      "activations/layer3_attention_weight_min": -98.87498474121094,
      "activations/layer4_attention_weight_max": 118.68214416503906,
      "activations/layer4_attention_weight_min": -114.59661865234375,
      "activations/layer5_attention_weight_max": 56.13621520996094,
      "activations/layer5_attention_weight_min": -62.65165710449219,
      "activations/layer6_attention_weight_max": 49.20777893066406,
      "activations/layer6_attention_weight_min": -48.779579162597656,
      "activations/layer7_attention_weight_max": 91.37327575683594,
      "activations/layer7_attention_weight_min": -90.98162841796875,
      "activations/layer8_attention_weight_max": 41.28452682495117,
      "activations/layer8_attention_weight_min": -39.256473541259766,
      "activations/layer9_attention_weight_max": 33.981719970703125,
      "activations/layer9_attention_weight_min": -32.538333892822266,
      "epoch": 9.2,
      "learning_rate": 9.16056818181818e-05,
      "loss": 2.7884,
      "step": 158300
    },
    {
      "activations/layer0_attention_weight_max": 16.11991310119629,
      "activations/layer0_attention_weight_min": -12.247448921203613,
      "activations/layer10_attention_weight_max": 32.27962875366211,
      "activations/layer10_attention_weight_min": -31.491811752319336,
      "activations/layer11_attention_weight_max": 31.101749420166016,
      "activations/layer11_attention_weight_min": -31.574071884155273,
      "activations/layer12_attention_weight_max": 22.793725967407227,
      "activations/layer12_attention_weight_min": -24.74593734741211,
      "activations/layer13_attention_weight_max": 38.42716979980469,
      "activations/layer13_attention_weight_min": -34.02151870727539,
      "activations/layer14_attention_weight_max": 44.95108413696289,
      "activations/layer14_attention_weight_min": -39.82209396362305,
      "activations/layer15_attention_weight_max": 35.63494873046875,
      "activations/layer15_attention_weight_min": -31.107479095458984,
      "activations/layer16_attention_weight_max": 31.77952003479004,
      "activations/layer16_attention_weight_min": -27.58247947692871,
      "activations/layer17_attention_weight_max": 35.83551025390625,
      "activations/layer17_attention_weight_min": -24.68109130859375,
      "activations/layer18_attention_weight_max": 33.71808624267578,
      "activations/layer18_attention_weight_min": -21.4110107421875,
      "activations/layer19_attention_weight_max": 35.27313995361328,
      "activations/layer19_attention_weight_min": -31.81829833984375,
      "activations/layer1_attention_weight_max": 16.630949020385742,
      "activations/layer1_attention_weight_min": -13.9546480178833,
      "activations/layer20_attention_weight_max": 31.875818252563477,
      "activations/layer20_attention_weight_min": -23.7188663482666,
      "activations/layer21_attention_weight_max": 28.236936569213867,
      "activations/layer21_attention_weight_min": -23.383403778076172,
      "activations/layer22_attention_weight_max": 43.25827407836914,
      "activations/layer22_attention_weight_min": -28.194204330444336,
      "activations/layer23_attention_weight_max": 34.572166442871094,
      "activations/layer23_attention_weight_min": -22.63128662109375,
      "activations/layer2_attention_weight_max": 31.89883804321289,
      "activations/layer2_attention_weight_min": -32.424652099609375,
      "activations/layer3_attention_weight_max": 95.06973266601562,
      "activations/layer3_attention_weight_min": -99.9175033569336,
      "activations/layer4_attention_weight_max": 114.26690673828125,
      "activations/layer4_attention_weight_min": -120.4721450805664,
      "activations/layer5_attention_weight_max": 52.709205627441406,
      "activations/layer5_attention_weight_min": -64.45600128173828,
      "activations/layer6_attention_weight_max": 45.65581512451172,
      "activations/layer6_attention_weight_min": -50.26192855834961,
      "activations/layer7_attention_weight_max": 87.5489730834961,
      "activations/layer7_attention_weight_min": -89.33783721923828,
      "activations/layer8_attention_weight_max": 41.366512298583984,
      "activations/layer8_attention_weight_min": -45.40785598754883,
      "activations/layer9_attention_weight_max": 34.36808395385742,
      "activations/layer9_attention_weight_min": -33.300865173339844,
      "epoch": 9.2,
      "learning_rate": 9.158674242424242e-05,
      "loss": 2.7973,
      "step": 158350
    },
    {
      "activations/layer0_attention_weight_max": 15.849763870239258,
      "activations/layer0_attention_weight_min": -12.701986312866211,
      "activations/layer10_attention_weight_max": 39.261314392089844,
      "activations/layer10_attention_weight_min": -35.853759765625,
      "activations/layer11_attention_weight_max": 40.208805084228516,
      "activations/layer11_attention_weight_min": -34.39986038208008,
      "activations/layer12_attention_weight_max": 24.734092712402344,
      "activations/layer12_attention_weight_min": -37.1561393737793,
      "activations/layer13_attention_weight_max": 43.0285530090332,
      "activations/layer13_attention_weight_min": -33.85270690917969,
      "activations/layer14_attention_weight_max": 60.31767654418945,
      "activations/layer14_attention_weight_min": -43.43424987792969,
      "activations/layer15_attention_weight_max": 46.83267593383789,
      "activations/layer15_attention_weight_min": -31.93661117553711,
      "activations/layer16_attention_weight_max": 39.01324462890625,
      "activations/layer16_attention_weight_min": -26.689489364624023,
      "activations/layer17_attention_weight_max": 35.34682083129883,
      "activations/layer17_attention_weight_min": -25.11318588256836,
      "activations/layer18_attention_weight_max": 31.268146514892578,
      "activations/layer18_attention_weight_min": -23.39259910583496,
      "activations/layer19_attention_weight_max": 37.035945892333984,
      "activations/layer19_attention_weight_min": -32.34869384765625,
      "activations/layer1_attention_weight_max": 18.04448699951172,
      "activations/layer1_attention_weight_min": -14.833419799804688,
      "activations/layer20_attention_weight_max": 32.49980163574219,
      "activations/layer20_attention_weight_min": -23.77188491821289,
      "activations/layer21_attention_weight_max": 30.04007339477539,
      "activations/layer21_attention_weight_min": -22.74074935913086,
      "activations/layer22_attention_weight_max": 48.89776611328125,
      "activations/layer22_attention_weight_min": -31.705625534057617,
      "activations/layer23_attention_weight_max": 39.03106689453125,
      "activations/layer23_attention_weight_min": -24.26195526123047,
      "activations/layer2_attention_weight_max": 35.6111946105957,
      "activations/layer2_attention_weight_min": -33.21942901611328,
      "activations/layer3_attention_weight_max": 101.3333969116211,
      "activations/layer3_attention_weight_min": -101.68929290771484,
      "activations/layer4_attention_weight_max": 125.26323699951172,
      "activations/layer4_attention_weight_min": -121.18802642822266,
      "activations/layer5_attention_weight_max": 56.0265007019043,
      "activations/layer5_attention_weight_min": -60.940887451171875,
      "activations/layer6_attention_weight_max": 48.96537399291992,
      "activations/layer6_attention_weight_min": -53.34426498413086,
      "activations/layer7_attention_weight_max": 100.26776123046875,
      "activations/layer7_attention_weight_min": -92.98152923583984,
      "activations/layer8_attention_weight_max": 44.473331451416016,
      "activations/layer8_attention_weight_min": -44.39707946777344,
      "activations/layer9_attention_weight_max": 40.99090576171875,
      "activations/layer9_attention_weight_min": -36.417137145996094,
      "epoch": 9.2,
      "learning_rate": 9.156780303030302e-05,
      "loss": 2.8049,
      "step": 158400
    },
    {
      "activations/layer0_attention_weight_max": 14.484189987182617,
      "activations/layer0_attention_weight_min": -12.12733268737793,
      "activations/layer10_attention_weight_max": 29.438737869262695,
      "activations/layer10_attention_weight_min": -31.3493709564209,
      "activations/layer11_attention_weight_max": 30.74860382080078,
      "activations/layer11_attention_weight_min": -30.567943572998047,
      "activations/layer12_attention_weight_max": 21.159332275390625,
      "activations/layer12_attention_weight_min": -27.057138442993164,
      "activations/layer13_attention_weight_max": 35.413726806640625,
      "activations/layer13_attention_weight_min": -31.093181610107422,
      "activations/layer14_attention_weight_max": 48.51687240600586,
      "activations/layer14_attention_weight_min": -41.91293716430664,
      "activations/layer15_attention_weight_max": 34.08261489868164,
      "activations/layer15_attention_weight_min": -31.227529525756836,
      "activations/layer16_attention_weight_max": 28.63858413696289,
      "activations/layer16_attention_weight_min": -27.123327255249023,
      "activations/layer17_attention_weight_max": 31.026222229003906,
      "activations/layer17_attention_weight_min": -23.933876037597656,
      "activations/layer18_attention_weight_max": 31.10786247253418,
      "activations/layer18_attention_weight_min": -22.763608932495117,
      "activations/layer19_attention_weight_max": 30.760257720947266,
      "activations/layer19_attention_weight_min": -30.6617431640625,
      "activations/layer1_attention_weight_max": 16.45781898498535,
      "activations/layer1_attention_weight_min": -14.759847640991211,
      "activations/layer20_attention_weight_max": 27.750288009643555,
      "activations/layer20_attention_weight_min": -25.405611038208008,
      "activations/layer21_attention_weight_max": 28.79933738708496,
      "activations/layer21_attention_weight_min": -22.726659774780273,
      "activations/layer22_attention_weight_max": 41.80982208251953,
      "activations/layer22_attention_weight_min": -28.62216567993164,
      "activations/layer23_attention_weight_max": 32.55772399902344,
      "activations/layer23_attention_weight_min": -25.816085815429688,
      "activations/layer2_attention_weight_max": 31.775962829589844,
      "activations/layer2_attention_weight_min": -32.65667724609375,
      "activations/layer3_attention_weight_max": 96.97660064697266,
      "activations/layer3_attention_weight_min": -95.1629638671875,
      "activations/layer4_attention_weight_max": 114.66495513916016,
      "activations/layer4_attention_weight_min": -115.9138412475586,
      "activations/layer5_attention_weight_max": 54.804710388183594,
      "activations/layer5_attention_weight_min": -62.09420394897461,
      "activations/layer6_attention_weight_max": 45.96643829345703,
      "activations/layer6_attention_weight_min": -48.50838088989258,
      "activations/layer7_attention_weight_max": 93.84212493896484,
      "activations/layer7_attention_weight_min": -87.3891372680664,
      "activations/layer8_attention_weight_max": 36.22896957397461,
      "activations/layer8_attention_weight_min": -40.39361572265625,
      "activations/layer9_attention_weight_max": 33.159263610839844,
      "activations/layer9_attention_weight_min": -32.212730407714844,
      "epoch": 9.21,
      "learning_rate": 9.154886363636363e-05,
      "loss": 2.8082,
      "step": 158450
    },
    {
      "activations/layer0_attention_weight_max": 16.633426666259766,
      "activations/layer0_attention_weight_min": -12.530609130859375,
      "activations/layer10_attention_weight_max": 35.628387451171875,
      "activations/layer10_attention_weight_min": -33.62521743774414,
      "activations/layer11_attention_weight_max": 38.66151428222656,
      "activations/layer11_attention_weight_min": -32.362342834472656,
      "activations/layer12_attention_weight_max": 38.28641891479492,
      "activations/layer12_attention_weight_min": -25.88065528869629,
      "activations/layer13_attention_weight_max": 47.06515121459961,
      "activations/layer13_attention_weight_min": -33.557891845703125,
      "activations/layer14_attention_weight_max": 50.48781967163086,
      "activations/layer14_attention_weight_min": -38.135475158691406,
      "activations/layer15_attention_weight_max": 45.403568267822266,
      "activations/layer15_attention_weight_min": -31.605548858642578,
      "activations/layer16_attention_weight_max": 37.29463577270508,
      "activations/layer16_attention_weight_min": -27.6640567779541,
      "activations/layer17_attention_weight_max": 35.10233688354492,
      "activations/layer17_attention_weight_min": -26.078113555908203,
      "activations/layer18_attention_weight_max": 30.794633865356445,
      "activations/layer18_attention_weight_min": -22.83205795288086,
      "activations/layer19_attention_weight_max": 39.011905670166016,
      "activations/layer19_attention_weight_min": -33.13236618041992,
      "activations/layer1_attention_weight_max": 16.7047176361084,
      "activations/layer1_attention_weight_min": -15.199013710021973,
      "activations/layer20_attention_weight_max": 32.437713623046875,
      "activations/layer20_attention_weight_min": -25.11665916442871,
      "activations/layer21_attention_weight_max": 30.569869995117188,
      "activations/layer21_attention_weight_min": -22.523395538330078,
      "activations/layer22_attention_weight_max": 49.10309600830078,
      "activations/layer22_attention_weight_min": -28.37420654296875,
      "activations/layer23_attention_weight_max": 36.85008239746094,
      "activations/layer23_attention_weight_min": -24.59749984741211,
      "activations/layer2_attention_weight_max": 34.882415771484375,
      "activations/layer2_attention_weight_min": -34.836849212646484,
      "activations/layer3_attention_weight_max": 98.20503234863281,
      "activations/layer3_attention_weight_min": -100.74320983886719,
      "activations/layer4_attention_weight_max": 118.66349029541016,
      "activations/layer4_attention_weight_min": -119.4698486328125,
      "activations/layer5_attention_weight_max": 57.31203842163086,
      "activations/layer5_attention_weight_min": -61.08849334716797,
      "activations/layer6_attention_weight_max": 47.2201042175293,
      "activations/layer6_attention_weight_min": -51.169742584228516,
      "activations/layer7_attention_weight_max": 92.3779296875,
      "activations/layer7_attention_weight_min": -89.72306823730469,
      "activations/layer8_attention_weight_max": 40.8895149230957,
      "activations/layer8_attention_weight_min": -40.80232620239258,
      "activations/layer9_attention_weight_max": 35.1403923034668,
      "activations/layer9_attention_weight_min": -34.190364837646484,
      "epoch": 9.21,
      "learning_rate": 9.152992424242424e-05,
      "loss": 2.7936,
      "step": 158500
    },
    {
      "activations/layer0_attention_weight_max": 15.357752799987793,
      "activations/layer0_attention_weight_min": -12.987563133239746,
      "activations/layer10_attention_weight_max": 30.7315731048584,
      "activations/layer10_attention_weight_min": -29.847434997558594,
      "activations/layer11_attention_weight_max": 32.790252685546875,
      "activations/layer11_attention_weight_min": -30.15883445739746,
      "activations/layer12_attention_weight_max": 24.126420974731445,
      "activations/layer12_attention_weight_min": -31.635231018066406,
      "activations/layer13_attention_weight_max": 36.85801315307617,
      "activations/layer13_attention_weight_min": -33.296382904052734,
      "activations/layer14_attention_weight_max": 42.37682342529297,
      "activations/layer14_attention_weight_min": -39.05776596069336,
      "activations/layer15_attention_weight_max": 32.85781478881836,
      "activations/layer15_attention_weight_min": -30.122905731201172,
      "activations/layer16_attention_weight_max": 30.81103515625,
      "activations/layer16_attention_weight_min": -27.10102653503418,
      "activations/layer17_attention_weight_max": 32.094303131103516,
      "activations/layer17_attention_weight_min": -25.16083526611328,
      "activations/layer18_attention_weight_max": 30.598819732666016,
      "activations/layer18_attention_weight_min": -21.23238754272461,
      "activations/layer19_attention_weight_max": 32.25568771362305,
      "activations/layer19_attention_weight_min": -29.039779663085938,
      "activations/layer1_attention_weight_max": 17.30809211730957,
      "activations/layer1_attention_weight_min": -17.96809196472168,
      "activations/layer20_attention_weight_max": 28.597421646118164,
      "activations/layer20_attention_weight_min": -23.15920066833496,
      "activations/layer21_attention_weight_max": 27.578908920288086,
      "activations/layer21_attention_weight_min": -22.78389549255371,
      "activations/layer22_attention_weight_max": 47.700069427490234,
      "activations/layer22_attention_weight_min": -26.889495849609375,
      "activations/layer23_attention_weight_max": 35.747039794921875,
      "activations/layer23_attention_weight_min": -22.365760803222656,
      "activations/layer2_attention_weight_max": 35.18165588378906,
      "activations/layer2_attention_weight_min": -33.80543518066406,
      "activations/layer3_attention_weight_max": 98.11289978027344,
      "activations/layer3_attention_weight_min": -98.40892028808594,
      "activations/layer4_attention_weight_max": 118.34127044677734,
      "activations/layer4_attention_weight_min": -111.2721176147461,
      "activations/layer5_attention_weight_max": 57.40938949584961,
      "activations/layer5_attention_weight_min": -59.45317077636719,
      "activations/layer6_attention_weight_max": 44.01539611816406,
      "activations/layer6_attention_weight_min": -48.52790832519531,
      "activations/layer7_attention_weight_max": 96.59580993652344,
      "activations/layer7_attention_weight_min": -84.78819274902344,
      "activations/layer8_attention_weight_max": 42.76933288574219,
      "activations/layer8_attention_weight_min": -39.92902755737305,
      "activations/layer9_attention_weight_max": 32.47002410888672,
      "activations/layer9_attention_weight_min": -32.681156158447266,
      "epoch": 9.21,
      "learning_rate": 9.151098484848484e-05,
      "loss": 2.7868,
      "step": 158550
    },
    {
      "activations/layer0_attention_weight_max": 15.83516788482666,
      "activations/layer0_attention_weight_min": -12.13487720489502,
      "activations/layer10_attention_weight_max": 32.93007278442383,
      "activations/layer10_attention_weight_min": -33.127891540527344,
      "activations/layer11_attention_weight_max": 32.92638397216797,
      "activations/layer11_attention_weight_min": -31.059898376464844,
      "activations/layer12_attention_weight_max": 25.819095611572266,
      "activations/layer12_attention_weight_min": -23.953582763671875,
      "activations/layer13_attention_weight_max": 41.676734924316406,
      "activations/layer13_attention_weight_min": -33.97758865356445,
      "activations/layer14_attention_weight_max": 52.418975830078125,
      "activations/layer14_attention_weight_min": -41.397010803222656,
      "activations/layer15_attention_weight_max": 39.771968841552734,
      "activations/layer15_attention_weight_min": -30.917491912841797,
      "activations/layer16_attention_weight_max": 35.626346588134766,
      "activations/layer16_attention_weight_min": -29.341100692749023,
      "activations/layer17_attention_weight_max": 34.0467643737793,
      "activations/layer17_attention_weight_min": -25.978954315185547,
      "activations/layer18_attention_weight_max": 37.63220977783203,
      "activations/layer18_attention_weight_min": -22.961191177368164,
      "activations/layer19_attention_weight_max": 39.719322204589844,
      "activations/layer19_attention_weight_min": -31.143573760986328,
      "activations/layer1_attention_weight_max": 16.196678161621094,
      "activations/layer1_attention_weight_min": -16.418241500854492,
      "activations/layer20_attention_weight_max": 34.47952651977539,
      "activations/layer20_attention_weight_min": -26.876075744628906,
      "activations/layer21_attention_weight_max": 32.696998596191406,
      "activations/layer21_attention_weight_min": -23.252151489257812,
      "activations/layer22_attention_weight_max": 46.910369873046875,
      "activations/layer22_attention_weight_min": -29.180683135986328,
      "activations/layer23_attention_weight_max": 39.50065994262695,
      "activations/layer23_attention_weight_min": -22.379987716674805,
      "activations/layer2_attention_weight_max": 33.979820251464844,
      "activations/layer2_attention_weight_min": -33.847530364990234,
      "activations/layer3_attention_weight_max": 97.47283935546875,
      "activations/layer3_attention_weight_min": -98.75782775878906,
      "activations/layer4_attention_weight_max": 119.21013641357422,
      "activations/layer4_attention_weight_min": -115.9582748413086,
      "activations/layer5_attention_weight_max": 54.155120849609375,
      "activations/layer5_attention_weight_min": -61.779232025146484,
      "activations/layer6_attention_weight_max": 50.28911209106445,
      "activations/layer6_attention_weight_min": -50.8908576965332,
      "activations/layer7_attention_weight_max": 90.50558471679688,
      "activations/layer7_attention_weight_min": -93.92243194580078,
      "activations/layer8_attention_weight_max": 40.47393035888672,
      "activations/layer8_attention_weight_min": -41.89788055419922,
      "activations/layer9_attention_weight_max": 33.80488586425781,
      "activations/layer9_attention_weight_min": -32.368873596191406,
      "epoch": 9.22,
      "learning_rate": 9.149204545454545e-05,
      "loss": 2.7982,
      "step": 158600
    },
    {
      "activations/layer0_attention_weight_max": 15.1591215133667,
      "activations/layer0_attention_weight_min": -12.201008796691895,
      "activations/layer10_attention_weight_max": 32.6307258605957,
      "activations/layer10_attention_weight_min": -35.19941329956055,
      "activations/layer11_attention_weight_max": 33.66582489013672,
      "activations/layer11_attention_weight_min": -31.107215881347656,
      "activations/layer12_attention_weight_max": 25.547855377197266,
      "activations/layer12_attention_weight_min": -26.148311614990234,
      "activations/layer13_attention_weight_max": 40.02872848510742,
      "activations/layer13_attention_weight_min": -37.5928955078125,
      "activations/layer14_attention_weight_max": 47.46916198730469,
      "activations/layer14_attention_weight_min": -38.26762771606445,
      "activations/layer15_attention_weight_max": 38.085140228271484,
      "activations/layer15_attention_weight_min": -28.726184844970703,
      "activations/layer16_attention_weight_max": 31.62397003173828,
      "activations/layer16_attention_weight_min": -27.484018325805664,
      "activations/layer17_attention_weight_max": 33.94094467163086,
      "activations/layer17_attention_weight_min": -25.991653442382812,
      "activations/layer18_attention_weight_max": 31.60075569152832,
      "activations/layer18_attention_weight_min": -21.60415267944336,
      "activations/layer19_attention_weight_max": 40.134429931640625,
      "activations/layer19_attention_weight_min": -32.38861846923828,
      "activations/layer1_attention_weight_max": 15.89306354522705,
      "activations/layer1_attention_weight_min": -16.814939498901367,
      "activations/layer20_attention_weight_max": 34.98585891723633,
      "activations/layer20_attention_weight_min": -24.14105224609375,
      "activations/layer21_attention_weight_max": 29.49078941345215,
      "activations/layer21_attention_weight_min": -23.820384979248047,
      "activations/layer22_attention_weight_max": 43.40255355834961,
      "activations/layer22_attention_weight_min": -29.838396072387695,
      "activations/layer23_attention_weight_max": 35.26416778564453,
      "activations/layer23_attention_weight_min": -21.531368255615234,
      "activations/layer2_attention_weight_max": 32.56926727294922,
      "activations/layer2_attention_weight_min": -34.13237380981445,
      "activations/layer3_attention_weight_max": 95.43028259277344,
      "activations/layer3_attention_weight_min": -96.36090087890625,
      "activations/layer4_attention_weight_max": 116.66975402832031,
      "activations/layer4_attention_weight_min": -116.26580810546875,
      "activations/layer5_attention_weight_max": 55.28904342651367,
      "activations/layer5_attention_weight_min": -61.983699798583984,
      "activations/layer6_attention_weight_max": 47.907352447509766,
      "activations/layer6_attention_weight_min": -51.72264862060547,
      "activations/layer7_attention_weight_max": 91.61894226074219,
      "activations/layer7_attention_weight_min": -96.95565795898438,
      "activations/layer8_attention_weight_max": 40.9588737487793,
      "activations/layer8_attention_weight_min": -41.76554870605469,
      "activations/layer9_attention_weight_max": 36.46156311035156,
      "activations/layer9_attention_weight_min": -36.25002670288086,
      "epoch": 9.22,
      "learning_rate": 9.147310606060605e-05,
      "loss": 2.8067,
      "step": 158650
    },
    {
      "activations/layer0_attention_weight_max": 14.749505996704102,
      "activations/layer0_attention_weight_min": -12.866406440734863,
      "activations/layer10_attention_weight_max": 32.06708908081055,
      "activations/layer10_attention_weight_min": -31.19405174255371,
      "activations/layer11_attention_weight_max": 31.425893783569336,
      "activations/layer11_attention_weight_min": -30.96979522705078,
      "activations/layer12_attention_weight_max": 31.0823917388916,
      "activations/layer12_attention_weight_min": -27.044391632080078,
      "activations/layer13_attention_weight_max": 37.6982421875,
      "activations/layer13_attention_weight_min": -36.38629150390625,
      "activations/layer14_attention_weight_max": 43.914588928222656,
      "activations/layer14_attention_weight_min": -38.0845832824707,
      "activations/layer15_attention_weight_max": 33.099124908447266,
      "activations/layer15_attention_weight_min": -30.534156799316406,
      "activations/layer16_attention_weight_max": 30.37884521484375,
      "activations/layer16_attention_weight_min": -28.762054443359375,
      "activations/layer17_attention_weight_max": 31.262163162231445,
      "activations/layer17_attention_weight_min": -25.516969680786133,
      "activations/layer18_attention_weight_max": 27.988876342773438,
      "activations/layer18_attention_weight_min": -20.553321838378906,
      "activations/layer19_attention_weight_max": 30.947071075439453,
      "activations/layer19_attention_weight_min": -29.275821685791016,
      "activations/layer1_attention_weight_max": 17.022493362426758,
      "activations/layer1_attention_weight_min": -14.482341766357422,
      "activations/layer20_attention_weight_max": 27.83555793762207,
      "activations/layer20_attention_weight_min": -24.33466339111328,
      "activations/layer21_attention_weight_max": 27.927654266357422,
      "activations/layer21_attention_weight_min": -22.382450103759766,
      "activations/layer22_attention_weight_max": 41.65412139892578,
      "activations/layer22_attention_weight_min": -27.064836502075195,
      "activations/layer23_attention_weight_max": 33.09064483642578,
      "activations/layer23_attention_weight_min": -23.390487670898438,
      "activations/layer2_attention_weight_max": 33.54440689086914,
      "activations/layer2_attention_weight_min": -34.98431396484375,
      "activations/layer3_attention_weight_max": 95.6646728515625,
      "activations/layer3_attention_weight_min": -99.6837158203125,
      "activations/layer4_attention_weight_max": 117.27809143066406,
      "activations/layer4_attention_weight_min": -115.50687408447266,
      "activations/layer5_attention_weight_max": 54.243255615234375,
      "activations/layer5_attention_weight_min": -62.08372116088867,
      "activations/layer6_attention_weight_max": 46.174495697021484,
      "activations/layer6_attention_weight_min": -51.45243835449219,
      "activations/layer7_attention_weight_max": 89.87458801269531,
      "activations/layer7_attention_weight_min": -89.56209564208984,
      "activations/layer8_attention_weight_max": 38.92168045043945,
      "activations/layer8_attention_weight_min": -40.414432525634766,
      "activations/layer9_attention_weight_max": 32.57623291015625,
      "activations/layer9_attention_weight_min": -34.08445358276367,
      "epoch": 9.22,
      "learning_rate": 9.145416666666666e-05,
      "loss": 2.8004,
      "step": 158700
    },
    {
      "activations/layer0_attention_weight_max": 15.540407180786133,
      "activations/layer0_attention_weight_min": -12.21323013305664,
      "activations/layer10_attention_weight_max": 34.4968376159668,
      "activations/layer10_attention_weight_min": -33.702850341796875,
      "activations/layer11_attention_weight_max": 36.54816436767578,
      "activations/layer11_attention_weight_min": -31.30348777770996,
      "activations/layer12_attention_weight_max": 27.057676315307617,
      "activations/layer12_attention_weight_min": -23.921588897705078,
      "activations/layer13_attention_weight_max": 38.49064636230469,
      "activations/layer13_attention_weight_min": -32.33732604980469,
      "activations/layer14_attention_weight_max": 44.41767120361328,
      "activations/layer14_attention_weight_min": -37.60706329345703,
      "activations/layer15_attention_weight_max": 37.09870147705078,
      "activations/layer15_attention_weight_min": -30.698780059814453,
      "activations/layer16_attention_weight_max": 32.122108459472656,
      "activations/layer16_attention_weight_min": -26.316390991210938,
      "activations/layer17_attention_weight_max": 37.84541320800781,
      "activations/layer17_attention_weight_min": -27.42514991760254,
      "activations/layer18_attention_weight_max": 32.40485763549805,
      "activations/layer18_attention_weight_min": -24.575725555419922,
      "activations/layer19_attention_weight_max": 39.50822830200195,
      "activations/layer19_attention_weight_min": -29.302204132080078,
      "activations/layer1_attention_weight_max": 18.1251277923584,
      "activations/layer1_attention_weight_min": -16.164796829223633,
      "activations/layer20_attention_weight_max": 36.07164001464844,
      "activations/layer20_attention_weight_min": -23.490093231201172,
      "activations/layer21_attention_weight_max": 32.274452209472656,
      "activations/layer21_attention_weight_min": -22.0977783203125,
      "activations/layer22_attention_weight_max": 48.68889617919922,
      "activations/layer22_attention_weight_min": -29.43949317932129,
      "activations/layer23_attention_weight_max": 40.819252014160156,
      "activations/layer23_attention_weight_min": -25.389171600341797,
      "activations/layer2_attention_weight_max": 34.72875213623047,
      "activations/layer2_attention_weight_min": -35.109825134277344,
      "activations/layer3_attention_weight_max": 101.45172882080078,
      "activations/layer3_attention_weight_min": -99.07866668701172,
      "activations/layer4_attention_weight_max": 125.8665771484375,
      "activations/layer4_attention_weight_min": -116.18082427978516,
      "activations/layer5_attention_weight_max": 57.87751388549805,
      "activations/layer5_attention_weight_min": -63.01183319091797,
      "activations/layer6_attention_weight_max": 51.78648376464844,
      "activations/layer6_attention_weight_min": -50.26505661010742,
      "activations/layer7_attention_weight_max": 92.2982177734375,
      "activations/layer7_attention_weight_min": -88.63365173339844,
      "activations/layer8_attention_weight_max": 41.84745788574219,
      "activations/layer8_attention_weight_min": -41.70683670043945,
      "activations/layer9_attention_weight_max": 34.08073806762695,
      "activations/layer9_attention_weight_min": -34.0916862487793,
      "epoch": 9.22,
      "learning_rate": 9.143522727272727e-05,
      "loss": 2.795,
      "step": 158750
    },
    {
      "activations/layer0_attention_weight_max": 15.768080711364746,
      "activations/layer0_attention_weight_min": -12.255186080932617,
      "activations/layer10_attention_weight_max": 32.36096954345703,
      "activations/layer10_attention_weight_min": -32.69037628173828,
      "activations/layer11_attention_weight_max": 33.032562255859375,
      "activations/layer11_attention_weight_min": -29.686172485351562,
      "activations/layer12_attention_weight_max": 25.397825241088867,
      "activations/layer12_attention_weight_min": -24.9366512298584,
      "activations/layer13_attention_weight_max": 38.147010803222656,
      "activations/layer13_attention_weight_min": -32.40472412109375,
      "activations/layer14_attention_weight_max": 42.563331604003906,
      "activations/layer14_attention_weight_min": -33.717567443847656,
      "activations/layer15_attention_weight_max": 34.202884674072266,
      "activations/layer15_attention_weight_min": -29.637418746948242,
      "activations/layer16_attention_weight_max": 29.024911880493164,
      "activations/layer16_attention_weight_min": -27.15512466430664,
      "activations/layer17_attention_weight_max": 30.248769760131836,
      "activations/layer17_attention_weight_min": -24.489776611328125,
      "activations/layer18_attention_weight_max": 33.10948944091797,
      "activations/layer18_attention_weight_min": -24.92673110961914,
      "activations/layer19_attention_weight_max": 36.762943267822266,
      "activations/layer19_attention_weight_min": -29.941986083984375,
      "activations/layer1_attention_weight_max": 17.345417022705078,
      "activations/layer1_attention_weight_min": -14.546285629272461,
      "activations/layer20_attention_weight_max": 28.118438720703125,
      "activations/layer20_attention_weight_min": -23.211631774902344,
      "activations/layer21_attention_weight_max": 27.29899024963379,
      "activations/layer21_attention_weight_min": -21.92435646057129,
      "activations/layer22_attention_weight_max": 43.809593200683594,
      "activations/layer22_attention_weight_min": -28.901020050048828,
      "activations/layer23_attention_weight_max": 34.65332794189453,
      "activations/layer23_attention_weight_min": -24.546445846557617,
      "activations/layer2_attention_weight_max": 32.220455169677734,
      "activations/layer2_attention_weight_min": -33.477779388427734,
      "activations/layer3_attention_weight_max": 96.83628845214844,
      "activations/layer3_attention_weight_min": -99.5727767944336,
      "activations/layer4_attention_weight_max": 115.9229965209961,
      "activations/layer4_attention_weight_min": -112.65997314453125,
      "activations/layer5_attention_weight_max": 51.31019592285156,
      "activations/layer5_attention_weight_min": -61.46476364135742,
      "activations/layer6_attention_weight_max": 43.5447883605957,
      "activations/layer6_attention_weight_min": -49.48096466064453,
      "activations/layer7_attention_weight_max": 85.95036315917969,
      "activations/layer7_attention_weight_min": -83.92727661132812,
      "activations/layer8_attention_weight_max": 37.99122619628906,
      "activations/layer8_attention_weight_min": -40.354148864746094,
      "activations/layer9_attention_weight_max": 34.15510177612305,
      "activations/layer9_attention_weight_min": -32.720760345458984,
      "epoch": 9.23,
      "learning_rate": 9.141628787878786e-05,
      "loss": 2.804,
      "step": 158800
    },
    {
      "activations/layer0_attention_weight_max": 16.40606689453125,
      "activations/layer0_attention_weight_min": -12.409069061279297,
      "activations/layer10_attention_weight_max": 33.997589111328125,
      "activations/layer10_attention_weight_min": -30.88797950744629,
      "activations/layer11_attention_weight_max": 30.478992462158203,
      "activations/layer11_attention_weight_min": -28.74083709716797,
      "activations/layer12_attention_weight_max": 22.700782775878906,
      "activations/layer12_attention_weight_min": -25.856361389160156,
      "activations/layer13_attention_weight_max": 36.94367599487305,
      "activations/layer13_attention_weight_min": -33.67432403564453,
      "activations/layer14_attention_weight_max": 45.874698638916016,
      "activations/layer14_attention_weight_min": -37.957359313964844,
      "activations/layer15_attention_weight_max": 32.789180755615234,
      "activations/layer15_attention_weight_min": -31.41396713256836,
      "activations/layer16_attention_weight_max": 32.39688491821289,
      "activations/layer16_attention_weight_min": -27.364269256591797,
      "activations/layer17_attention_weight_max": 28.81109046936035,
      "activations/layer17_attention_weight_min": -26.267751693725586,
      "activations/layer18_attention_weight_max": 30.201852798461914,
      "activations/layer18_attention_weight_min": -23.76683807373047,
      "activations/layer19_attention_weight_max": 33.60954284667969,
      "activations/layer19_attention_weight_min": -30.5754451751709,
      "activations/layer1_attention_weight_max": 16.94278907775879,
      "activations/layer1_attention_weight_min": -15.859720230102539,
      "activations/layer20_attention_weight_max": 26.557767868041992,
      "activations/layer20_attention_weight_min": -23.273632049560547,
      "activations/layer21_attention_weight_max": 26.468822479248047,
      "activations/layer21_attention_weight_min": -22.719118118286133,
      "activations/layer22_attention_weight_max": 43.785614013671875,
      "activations/layer22_attention_weight_min": -26.65667724609375,
      "activations/layer23_attention_weight_max": 34.88348388671875,
      "activations/layer23_attention_weight_min": -23.34686279296875,
      "activations/layer2_attention_weight_max": 36.02104568481445,
      "activations/layer2_attention_weight_min": -32.58776092529297,
      "activations/layer3_attention_weight_max": 95.34857940673828,
      "activations/layer3_attention_weight_min": -94.98336791992188,
      "activations/layer4_attention_weight_max": 113.61297607421875,
      "activations/layer4_attention_weight_min": -117.3431625366211,
      "activations/layer5_attention_weight_max": 55.305572509765625,
      "activations/layer5_attention_weight_min": -63.93474578857422,
      "activations/layer6_attention_weight_max": 48.05968475341797,
      "activations/layer6_attention_weight_min": -48.97138595581055,
      "activations/layer7_attention_weight_max": 85.20571899414062,
      "activations/layer7_attention_weight_min": -85.04483795166016,
      "activations/layer8_attention_weight_max": 37.9998779296875,
      "activations/layer8_attention_weight_min": -38.743350982666016,
      "activations/layer9_attention_weight_max": 32.366153717041016,
      "activations/layer9_attention_weight_min": -34.0216178894043,
      "epoch": 9.23,
      "learning_rate": 9.139734848484847e-05,
      "loss": 2.7861,
      "step": 158850
    },
    {
      "activations/layer0_attention_weight_max": 16.334308624267578,
      "activations/layer0_attention_weight_min": -11.793683052062988,
      "activations/layer10_attention_weight_max": 37.39982223510742,
      "activations/layer10_attention_weight_min": -33.61299514770508,
      "activations/layer11_attention_weight_max": 36.441314697265625,
      "activations/layer11_attention_weight_min": -32.03121566772461,
      "activations/layer12_attention_weight_max": 23.622074127197266,
      "activations/layer12_attention_weight_min": -24.74928092956543,
      "activations/layer13_attention_weight_max": 38.19187927246094,
      "activations/layer13_attention_weight_min": -35.547611236572266,
      "activations/layer14_attention_weight_max": 45.48991394042969,
      "activations/layer14_attention_weight_min": -40.568870544433594,
      "activations/layer15_attention_weight_max": 36.396263122558594,
      "activations/layer15_attention_weight_min": -32.325260162353516,
      "activations/layer16_attention_weight_max": 31.821033477783203,
      "activations/layer16_attention_weight_min": -29.383407592773438,
      "activations/layer17_attention_weight_max": 32.807159423828125,
      "activations/layer17_attention_weight_min": -27.259540557861328,
      "activations/layer18_attention_weight_max": 30.415903091430664,
      "activations/layer18_attention_weight_min": -22.626691818237305,
      "activations/layer19_attention_weight_max": 33.6870231628418,
      "activations/layer19_attention_weight_min": -33.154537200927734,
      "activations/layer1_attention_weight_max": 17.65456199645996,
      "activations/layer1_attention_weight_min": -14.015950202941895,
      "activations/layer20_attention_weight_max": 29.042713165283203,
      "activations/layer20_attention_weight_min": -23.706586837768555,
      "activations/layer21_attention_weight_max": 30.80794906616211,
      "activations/layer21_attention_weight_min": -22.770771026611328,
      "activations/layer22_attention_weight_max": 44.14227294921875,
      "activations/layer22_attention_weight_min": -28.90375518798828,
      "activations/layer23_attention_weight_max": 33.032615661621094,
      "activations/layer23_attention_weight_min": -26.42056655883789,
      "activations/layer2_attention_weight_max": 34.89845275878906,
      "activations/layer2_attention_weight_min": -35.1251106262207,
      "activations/layer3_attention_weight_max": 101.12993621826172,
      "activations/layer3_attention_weight_min": -102.70042419433594,
      "activations/layer4_attention_weight_max": 123.90943908691406,
      "activations/layer4_attention_weight_min": -116.39466857910156,
      "activations/layer5_attention_weight_max": 56.773590087890625,
      "activations/layer5_attention_weight_min": -61.68212890625,
      "activations/layer6_attention_weight_max": 49.06922149658203,
      "activations/layer6_attention_weight_min": -49.782691955566406,
      "activations/layer7_attention_weight_max": 89.2370376586914,
      "activations/layer7_attention_weight_min": -86.15208435058594,
      "activations/layer8_attention_weight_max": 41.62233352661133,
      "activations/layer8_attention_weight_min": -38.8408088684082,
      "activations/layer9_attention_weight_max": 37.13373947143555,
      "activations/layer9_attention_weight_min": -33.260459899902344,
      "epoch": 9.23,
      "learning_rate": 9.137840909090909e-05,
      "loss": 2.8009,
      "step": 158900
    },
    {
      "activations/layer0_attention_weight_max": 15.311429023742676,
      "activations/layer0_attention_weight_min": -11.34947681427002,
      "activations/layer10_attention_weight_max": 34.85626983642578,
      "activations/layer10_attention_weight_min": -31.288890838623047,
      "activations/layer11_attention_weight_max": 36.364009857177734,
      "activations/layer11_attention_weight_min": -30.286542892456055,
      "activations/layer12_attention_weight_max": 26.109745025634766,
      "activations/layer12_attention_weight_min": -23.54072380065918,
      "activations/layer13_attention_weight_max": 43.25587844848633,
      "activations/layer13_attention_weight_min": -32.142555236816406,
      "activations/layer14_attention_weight_max": 50.00706481933594,
      "activations/layer14_attention_weight_min": -35.335243225097656,
      "activations/layer15_attention_weight_max": 39.7653694152832,
      "activations/layer15_attention_weight_min": -31.01146697998047,
      "activations/layer16_attention_weight_max": 34.266700744628906,
      "activations/layer16_attention_weight_min": -26.08537483215332,
      "activations/layer17_attention_weight_max": 33.782901763916016,
      "activations/layer17_attention_weight_min": -24.760744094848633,
      "activations/layer18_attention_weight_max": 33.834075927734375,
      "activations/layer18_attention_weight_min": -24.703550338745117,
      "activations/layer19_attention_weight_max": 34.56327438354492,
      "activations/layer19_attention_weight_min": -30.509336471557617,
      "activations/layer1_attention_weight_max": 16.668907165527344,
      "activations/layer1_attention_weight_min": -14.617853164672852,
      "activations/layer20_attention_weight_max": 27.908063888549805,
      "activations/layer20_attention_weight_min": -23.98946189880371,
      "activations/layer21_attention_weight_max": 28.444656372070312,
      "activations/layer21_attention_weight_min": -22.912588119506836,
      "activations/layer22_attention_weight_max": 49.02622985839844,
      "activations/layer22_attention_weight_min": -29.080617904663086,
      "activations/layer23_attention_weight_max": 36.31005859375,
      "activations/layer23_attention_weight_min": -24.32201385498047,
      "activations/layer2_attention_weight_max": 32.14678192138672,
      "activations/layer2_attention_weight_min": -32.47907257080078,
      "activations/layer3_attention_weight_max": 94.06916046142578,
      "activations/layer3_attention_weight_min": -95.56504821777344,
      "activations/layer4_attention_weight_max": 111.01165771484375,
      "activations/layer4_attention_weight_min": -113.99725341796875,
      "activations/layer5_attention_weight_max": 53.69095993041992,
      "activations/layer5_attention_weight_min": -60.191951751708984,
      "activations/layer6_attention_weight_max": 44.998600006103516,
      "activations/layer6_attention_weight_min": -51.74125289916992,
      "activations/layer7_attention_weight_max": 85.41869354248047,
      "activations/layer7_attention_weight_min": -93.12478637695312,
      "activations/layer8_attention_weight_max": 37.96525573730469,
      "activations/layer8_attention_weight_min": -38.826683044433594,
      "activations/layer9_attention_weight_max": 32.87251663208008,
      "activations/layer9_attention_weight_min": -32.12531280517578,
      "epoch": 9.24,
      "learning_rate": 9.135946969696969e-05,
      "loss": 2.8025,
      "step": 158950
    },
    {
      "activations/layer0_attention_weight_max": 15.378477096557617,
      "activations/layer0_attention_weight_min": -11.396845817565918,
      "activations/layer10_attention_weight_max": 33.031288146972656,
      "activations/layer10_attention_weight_min": -31.61519432067871,
      "activations/layer11_attention_weight_max": 32.199161529541016,
      "activations/layer11_attention_weight_min": -30.799617767333984,
      "activations/layer12_attention_weight_max": 27.255563735961914,
      "activations/layer12_attention_weight_min": -23.646406173706055,
      "activations/layer13_attention_weight_max": 38.3746337890625,
      "activations/layer13_attention_weight_min": -34.43016052246094,
      "activations/layer14_attention_weight_max": 49.362186431884766,
      "activations/layer14_attention_weight_min": -41.940452575683594,
      "activations/layer15_attention_weight_max": 35.83460235595703,
      "activations/layer15_attention_weight_min": -32.03098678588867,
      "activations/layer16_attention_weight_max": 31.17564582824707,
      "activations/layer16_attention_weight_min": -29.08841896057129,
      "activations/layer17_attention_weight_max": 34.63174057006836,
      "activations/layer17_attention_weight_min": -25.814422607421875,
      "activations/layer18_attention_weight_max": 30.305782318115234,
      "activations/layer18_attention_weight_min": -22.120555877685547,
      "activations/layer19_attention_weight_max": 38.8797492980957,
      "activations/layer19_attention_weight_min": -31.248342514038086,
      "activations/layer1_attention_weight_max": 16.281124114990234,
      "activations/layer1_attention_weight_min": -14.113347053527832,
      "activations/layer20_attention_weight_max": 32.15544128417969,
      "activations/layer20_attention_weight_min": -25.458932876586914,
      "activations/layer21_attention_weight_max": 32.10019302368164,
      "activations/layer21_attention_weight_min": -24.110923767089844,
      "activations/layer22_attention_weight_max": 47.279869079589844,
      "activations/layer22_attention_weight_min": -29.008617401123047,
      "activations/layer23_attention_weight_max": 38.05666732788086,
      "activations/layer23_attention_weight_min": -23.44545555114746,
      "activations/layer2_attention_weight_max": 31.481258392333984,
      "activations/layer2_attention_weight_min": -30.574459075927734,
      "activations/layer3_attention_weight_max": 91.40100860595703,
      "activations/layer3_attention_weight_min": -87.5121078491211,
      "activations/layer4_attention_weight_max": 119.19322967529297,
      "activations/layer4_attention_weight_min": -108.6132583618164,
      "activations/layer5_attention_weight_max": 52.21427917480469,
      "activations/layer5_attention_weight_min": -63.4942626953125,
      "activations/layer6_attention_weight_max": 46.85613250732422,
      "activations/layer6_attention_weight_min": -51.48167037963867,
      "activations/layer7_attention_weight_max": 90.32481384277344,
      "activations/layer7_attention_weight_min": -90.49105072021484,
      "activations/layer8_attention_weight_max": 38.1163444519043,
      "activations/layer8_attention_weight_min": -40.27880096435547,
      "activations/layer9_attention_weight_max": 33.04682540893555,
      "activations/layer9_attention_weight_min": -32.04299545288086,
      "epoch": 9.24,
      "learning_rate": 9.134053030303029e-05,
      "loss": 2.8081,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.4472,
      "eval_samples_per_second": 508.336,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.4472,
      "eval_openwebtext_samples_per_second": 508.336,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_wikitext_loss": 2.978515625,
      "eval_wikitext_ppl": 19.658614220966218,
      "eval_wikitext_runtime": 1.9035,
      "eval_wikitext_samples_per_second": 239.555,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_lambada_loss": 2.60546875,
      "eval_lambada_ppl": 13.537569551425428,
      "eval_lambada_runtime": 9.5094,
      "eval_lambada_samples_per_second": 512.02,
      "step": 159000
    },
    {
      "activations/layer0_attention_weight_max": 14.751002311706543,
      "activations/layer0_attention_weight_min": -13.103066444396973,
      "activations/layer10_attention_weight_max": 35.291847229003906,
      "activations/layer10_attention_weight_min": -32.676605224609375,
      "activations/layer11_attention_weight_max": 35.3442497253418,
      "activations/layer11_attention_weight_min": -31.213279724121094,
      "activations/layer12_attention_weight_max": 33.851898193359375,
      "activations/layer12_attention_weight_min": -26.70571517944336,
      "activations/layer13_attention_weight_max": 44.11339569091797,
      "activations/layer13_attention_weight_min": -34.1673469543457,
      "activations/layer14_attention_weight_max": 43.97355270385742,
      "activations/layer14_attention_weight_min": -35.15854263305664,
      "activations/layer15_attention_weight_max": 37.80110168457031,
      "activations/layer15_attention_weight_min": -30.45540428161621,
      "activations/layer16_attention_weight_max": 30.710399627685547,
      "activations/layer16_attention_weight_min": -26.013408660888672,
      "activations/layer17_attention_weight_max": 34.79226303100586,
      "activations/layer17_attention_weight_min": -25.054821014404297,
      "activations/layer18_attention_weight_max": 31.937509536743164,
      "activations/layer18_attention_weight_min": -21.685108184814453,
      "activations/layer19_attention_weight_max": 32.28566360473633,
      "activations/layer19_attention_weight_min": -29.701059341430664,
      "activations/layer1_attention_weight_max": 17.59271812438965,
      "activations/layer1_attention_weight_min": -13.0619478225708,
      "activations/layer20_attention_weight_max": 30.411130905151367,
      "activations/layer20_attention_weight_min": -24.460737228393555,
      "activations/layer21_attention_weight_max": 30.415691375732422,
      "activations/layer21_attention_weight_min": -24.673492431640625,
      "activations/layer22_attention_weight_max": 45.98957824707031,
      "activations/layer22_attention_weight_min": -29.32170295715332,
      "activations/layer23_attention_weight_max": 36.081722259521484,
      "activations/layer23_attention_weight_min": -24.035545349121094,
      "activations/layer2_attention_weight_max": 33.68658447265625,
      "activations/layer2_attention_weight_min": -32.37923049926758,
      "activations/layer3_attention_weight_max": 93.06964111328125,
      "activations/layer3_attention_weight_min": -91.80670166015625,
      "activations/layer4_attention_weight_max": 114.48426818847656,
      "activations/layer4_attention_weight_min": -117.39845275878906,
      "activations/layer5_attention_weight_max": 56.660682678222656,
      "activations/layer5_attention_weight_min": -68.29257202148438,
      "activations/layer6_attention_weight_max": 46.046085357666016,
      "activations/layer6_attention_weight_min": -50.237606048583984,
      "activations/layer7_attention_weight_max": 91.5377197265625,
      "activations/layer7_attention_weight_min": -93.96587371826172,
      "activations/layer8_attention_weight_max": 38.127525329589844,
      "activations/layer8_attention_weight_min": -40.766265869140625,
      "activations/layer9_attention_weight_max": 35.48469924926758,
      "activations/layer9_attention_weight_min": -33.12275314331055,
      "epoch": 9.24,
      "learning_rate": 9.132159090909091e-05,
      "loss": 2.7897,
      "step": 159050
    },
    {
      "activations/layer0_attention_weight_max": 15.4627103805542,
      "activations/layer0_attention_weight_min": -13.00071907043457,
      "activations/layer10_attention_weight_max": 31.791906356811523,
      "activations/layer10_attention_weight_min": -32.32386779785156,
      "activations/layer11_attention_weight_max": 30.803251266479492,
      "activations/layer11_attention_weight_min": -28.749462127685547,
      "activations/layer12_attention_weight_max": 26.148216247558594,
      "activations/layer12_attention_weight_min": -26.512107849121094,
      "activations/layer13_attention_weight_max": 39.52965545654297,
      "activations/layer13_attention_weight_min": -32.63199996948242,
      "activations/layer14_attention_weight_max": 49.002262115478516,
      "activations/layer14_attention_weight_min": -38.626407623291016,
      "activations/layer15_attention_weight_max": 36.59248733520508,
      "activations/layer15_attention_weight_min": -29.015697479248047,
      "activations/layer16_attention_weight_max": 33.643402099609375,
      "activations/layer16_attention_weight_min": -26.789339065551758,
      "activations/layer17_attention_weight_max": 37.23464584350586,
      "activations/layer17_attention_weight_min": -24.391916275024414,
      "activations/layer18_attention_weight_max": 30.693706512451172,
      "activations/layer18_attention_weight_min": -21.29534912109375,
      "activations/layer19_attention_weight_max": 37.2579460144043,
      "activations/layer19_attention_weight_min": -30.264923095703125,
      "activations/layer1_attention_weight_max": 18.501737594604492,
      "activations/layer1_attention_weight_min": -13.99824047088623,
      "activations/layer20_attention_weight_max": 30.58180809020996,
      "activations/layer20_attention_weight_min": -24.799537658691406,
      "activations/layer21_attention_weight_max": 29.032821655273438,
      "activations/layer21_attention_weight_min": -23.231351852416992,
      "activations/layer22_attention_weight_max": 42.68169021606445,
      "activations/layer22_attention_weight_min": -29.604537963867188,
      "activations/layer23_attention_weight_max": 38.78204345703125,
      "activations/layer23_attention_weight_min": -25.18836212158203,
      "activations/layer2_attention_weight_max": 30.23758316040039,
      "activations/layer2_attention_weight_min": -31.543140411376953,
      "activations/layer3_attention_weight_max": 91.96952819824219,
      "activations/layer3_attention_weight_min": -90.74630737304688,
      "activations/layer4_attention_weight_max": 116.37921142578125,
      "activations/layer4_attention_weight_min": -109.21331024169922,
      "activations/layer5_attention_weight_max": 54.5761833190918,
      "activations/layer5_attention_weight_min": -61.48371887207031,
      "activations/layer6_attention_weight_max": 47.10462951660156,
      "activations/layer6_attention_weight_min": -47.770286560058594,
      "activations/layer7_attention_weight_max": 82.32694244384766,
      "activations/layer7_attention_weight_min": -83.8202133178711,
      "activations/layer8_attention_weight_max": 35.479698181152344,
      "activations/layer8_attention_weight_min": -38.973514556884766,
      "activations/layer9_attention_weight_max": 33.06460952758789,
      "activations/layer9_attention_weight_min": -30.67133140563965,
      "epoch": 9.24,
      "learning_rate": 9.130265151515151e-05,
      "loss": 2.7955,
      "step": 159100
    },
    {
      "activations/layer0_attention_weight_max": 15.276395797729492,
      "activations/layer0_attention_weight_min": -12.259057998657227,
      "activations/layer10_attention_weight_max": 28.9853572845459,
      "activations/layer10_attention_weight_min": -30.369400024414062,
      "activations/layer11_attention_weight_max": 29.67414093017578,
      "activations/layer11_attention_weight_min": -30.90511703491211,
      "activations/layer12_attention_weight_max": 22.32366180419922,
      "activations/layer12_attention_weight_min": -24.944456100463867,
      "activations/layer13_attention_weight_max": 35.34739685058594,
      "activations/layer13_attention_weight_min": -32.35784912109375,
      "activations/layer14_attention_weight_max": 45.89083480834961,
      "activations/layer14_attention_weight_min": -40.207271575927734,
      "activations/layer15_attention_weight_max": 36.18225860595703,
      "activations/layer15_attention_weight_min": -32.12445831298828,
      "activations/layer16_attention_weight_max": 35.93334197998047,
      "activations/layer16_attention_weight_min": -27.808547973632812,
      "activations/layer17_attention_weight_max": 34.477989196777344,
      "activations/layer17_attention_weight_min": -25.16058349609375,
      "activations/layer18_attention_weight_max": 32.89298629760742,
      "activations/layer18_attention_weight_min": -21.420421600341797,
      "activations/layer19_attention_weight_max": 39.989112854003906,
      "activations/layer19_attention_weight_min": -35.34365463256836,
      "activations/layer1_attention_weight_max": 16.659103393554688,
      "activations/layer1_attention_weight_min": -13.844030380249023,
      "activations/layer20_attention_weight_max": 30.363548278808594,
      "activations/layer20_attention_weight_min": -24.463274002075195,
      "activations/layer21_attention_weight_max": 30.142332077026367,
      "activations/layer21_attention_weight_min": -22.703886032104492,
      "activations/layer22_attention_weight_max": 47.599849700927734,
      "activations/layer22_attention_weight_min": -30.16444206237793,
      "activations/layer23_attention_weight_max": 37.49916076660156,
      "activations/layer23_attention_weight_min": -26.16832733154297,
      "activations/layer2_attention_weight_max": 34.851810455322266,
      "activations/layer2_attention_weight_min": -35.46544647216797,
      "activations/layer3_attention_weight_max": 99.2000961303711,
      "activations/layer3_attention_weight_min": -103.91471862792969,
      "activations/layer4_attention_weight_max": 122.08077239990234,
      "activations/layer4_attention_weight_min": -112.93852996826172,
      "activations/layer5_attention_weight_max": 55.287803649902344,
      "activations/layer5_attention_weight_min": -59.394805908203125,
      "activations/layer6_attention_weight_max": 50.83403396606445,
      "activations/layer6_attention_weight_min": -51.47890090942383,
      "activations/layer7_attention_weight_max": 87.77542114257812,
      "activations/layer7_attention_weight_min": -92.62364959716797,
      "activations/layer8_attention_weight_max": 38.29834747314453,
      "activations/layer8_attention_weight_min": -40.76449966430664,
      "activations/layer9_attention_weight_max": 31.109872817993164,
      "activations/layer9_attention_weight_min": -32.50303268432617,
      "epoch": 9.25,
      "learning_rate": 9.128371212121211e-05,
      "loss": 2.7974,
      "step": 159150
    },
    {
      "activations/layer0_attention_weight_max": 15.21505069732666,
      "activations/layer0_attention_weight_min": -11.376908302307129,
      "activations/layer10_attention_weight_max": 33.33839797973633,
      "activations/layer10_attention_weight_min": -30.6607666015625,
      "activations/layer11_attention_weight_max": 33.128841400146484,
      "activations/layer11_attention_weight_min": -30.339279174804688,
      "activations/layer12_attention_weight_max": 24.998319625854492,
      "activations/layer12_attention_weight_min": -27.2761287689209,
      "activations/layer13_attention_weight_max": 40.27741622924805,
      "activations/layer13_attention_weight_min": -32.005184173583984,
      "activations/layer14_attention_weight_max": 51.810203552246094,
      "activations/layer14_attention_weight_min": -39.72577667236328,
      "activations/layer15_attention_weight_max": 37.159481048583984,
      "activations/layer15_attention_weight_min": -30.848390579223633,
      "activations/layer16_attention_weight_max": 33.37406921386719,
      "activations/layer16_attention_weight_min": -26.95770263671875,
      "activations/layer17_attention_weight_max": 34.14339065551758,
      "activations/layer17_attention_weight_min": -25.62523078918457,
      "activations/layer18_attention_weight_max": 30.895761489868164,
      "activations/layer18_attention_weight_min": -22.062896728515625,
      "activations/layer19_attention_weight_max": 37.440303802490234,
      "activations/layer19_attention_weight_min": -30.579792022705078,
      "activations/layer1_attention_weight_max": 16.341510772705078,
      "activations/layer1_attention_weight_min": -14.887423515319824,
      "activations/layer20_attention_weight_max": 33.855220794677734,
      "activations/layer20_attention_weight_min": -23.69708824157715,
      "activations/layer21_attention_weight_max": 30.09676742553711,
      "activations/layer21_attention_weight_min": -22.134428024291992,
      "activations/layer22_attention_weight_max": 59.536590576171875,
      "activations/layer22_attention_weight_min": -29.536624908447266,
      "activations/layer23_attention_weight_max": 39.994171142578125,
      "activations/layer23_attention_weight_min": -22.96259307861328,
      "activations/layer2_attention_weight_max": 31.697437286376953,
      "activations/layer2_attention_weight_min": -31.34382438659668,
      "activations/layer3_attention_weight_max": 96.27623748779297,
      "activations/layer3_attention_weight_min": -93.68313598632812,
      "activations/layer4_attention_weight_max": 117.09283447265625,
      "activations/layer4_attention_weight_min": -111.81914520263672,
      "activations/layer5_attention_weight_max": 55.11572265625,
      "activations/layer5_attention_weight_min": -60.892494201660156,
      "activations/layer6_attention_weight_max": 46.65348434448242,
      "activations/layer6_attention_weight_min": -49.70061492919922,
      "activations/layer7_attention_weight_max": 89.21963500976562,
      "activations/layer7_attention_weight_min": -90.3120346069336,
      "activations/layer8_attention_weight_max": 40.9542236328125,
      "activations/layer8_attention_weight_min": -41.66731643676758,
      "activations/layer9_attention_weight_max": 35.71376419067383,
      "activations/layer9_attention_weight_min": -33.847190856933594,
      "epoch": 9.25,
      "learning_rate": 9.126515151515151e-05,
      "loss": 2.7773,
      "step": 159200
    },
    {
      "activations/layer0_attention_weight_max": 14.669272422790527,
      "activations/layer0_attention_weight_min": -12.839067459106445,
      "activations/layer10_attention_weight_max": 34.498268127441406,
      "activations/layer10_attention_weight_min": -32.113922119140625,
      "activations/layer11_attention_weight_max": 35.29134750366211,
      "activations/layer11_attention_weight_min": -30.957319259643555,
      "activations/layer12_attention_weight_max": 22.322425842285156,
      "activations/layer12_attention_weight_min": -25.198923110961914,
      "activations/layer13_attention_weight_max": 36.8562126159668,
      "activations/layer13_attention_weight_min": -35.48774719238281,
      "activations/layer14_attention_weight_max": 45.36241912841797,
      "activations/layer14_attention_weight_min": -38.09919738769531,
      "activations/layer15_attention_weight_max": 34.352508544921875,
      "activations/layer15_attention_weight_min": -31.805936813354492,
      "activations/layer16_attention_weight_max": 30.41432762145996,
      "activations/layer16_attention_weight_min": -26.8306941986084,
      "activations/layer17_attention_weight_max": 31.92365837097168,
      "activations/layer17_attention_weight_min": -25.3865909576416,
      "activations/layer18_attention_weight_max": 29.486858367919922,
      "activations/layer18_attention_weight_min": -23.554731369018555,
      "activations/layer19_attention_weight_max": 37.743282318115234,
      "activations/layer19_attention_weight_min": -32.703224182128906,
      "activations/layer1_attention_weight_max": 16.61839485168457,
      "activations/layer1_attention_weight_min": -14.946288108825684,
      "activations/layer20_attention_weight_max": 31.330259323120117,
      "activations/layer20_attention_weight_min": -25.12160873413086,
      "activations/layer21_attention_weight_max": 31.2161865234375,
      "activations/layer21_attention_weight_min": -25.241567611694336,
      "activations/layer22_attention_weight_max": 47.72172927856445,
      "activations/layer22_attention_weight_min": -32.19749069213867,
      "activations/layer23_attention_weight_max": 34.32701110839844,
      "activations/layer23_attention_weight_min": -23.80915069580078,
      "activations/layer2_attention_weight_max": 31.37941551208496,
      "activations/layer2_attention_weight_min": -32.25355529785156,
      "activations/layer3_attention_weight_max": 95.42940521240234,
      "activations/layer3_attention_weight_min": -91.54647827148438,
      "activations/layer4_attention_weight_max": 115.1096420288086,
      "activations/layer4_attention_weight_min": -111.0543441772461,
      "activations/layer5_attention_weight_max": 53.29470443725586,
      "activations/layer5_attention_weight_min": -61.755615234375,
      "activations/layer6_attention_weight_max": 48.41319274902344,
      "activations/layer6_attention_weight_min": -51.29940414428711,
      "activations/layer7_attention_weight_max": 94.1197738647461,
      "activations/layer7_attention_weight_min": -84.46363830566406,
      "activations/layer8_attention_weight_max": 40.76792907714844,
      "activations/layer8_attention_weight_min": -41.001373291015625,
      "activations/layer9_attention_weight_max": 33.84003829956055,
      "activations/layer9_attention_weight_min": -32.17710494995117,
      "epoch": 9.25,
      "learning_rate": 9.124621212121212e-05,
      "loss": 2.7916,
      "step": 159250
    },
    {
      "activations/layer0_attention_weight_max": 16.329282760620117,
      "activations/layer0_attention_weight_min": -11.760163307189941,
      "activations/layer10_attention_weight_max": 32.13538360595703,
      "activations/layer10_attention_weight_min": -31.514719009399414,
      "activations/layer11_attention_weight_max": 31.497997283935547,
      "activations/layer11_attention_weight_min": -30.01276206970215,
      "activations/layer12_attention_weight_max": 23.127214431762695,
      "activations/layer12_attention_weight_min": -27.0628604888916,
      "activations/layer13_attention_weight_max": 39.953834533691406,
      "activations/layer13_attention_weight_min": -34.419776916503906,
      "activations/layer14_attention_weight_max": 46.9534797668457,
      "activations/layer14_attention_weight_min": -39.566314697265625,
      "activations/layer15_attention_weight_max": 37.055641174316406,
      "activations/layer15_attention_weight_min": -31.513193130493164,
      "activations/layer16_attention_weight_max": 32.907203674316406,
      "activations/layer16_attention_weight_min": -27.69235610961914,
      "activations/layer17_attention_weight_max": 31.817373275756836,
      "activations/layer17_attention_weight_min": -25.14699363708496,
      "activations/layer18_attention_weight_max": 28.899747848510742,
      "activations/layer18_attention_weight_min": -22.00277328491211,
      "activations/layer19_attention_weight_max": 37.28592300415039,
      "activations/layer19_attention_weight_min": -30.09712028503418,
      "activations/layer1_attention_weight_max": 16.071945190429688,
      "activations/layer1_attention_weight_min": -14.260265350341797,
      "activations/layer20_attention_weight_max": 31.423215866088867,
      "activations/layer20_attention_weight_min": -24.523839950561523,
      "activations/layer21_attention_weight_max": 28.851932525634766,
      "activations/layer21_attention_weight_min": -23.276622772216797,
      "activations/layer22_attention_weight_max": 47.923423767089844,
      "activations/layer22_attention_weight_min": -29.850278854370117,
      "activations/layer23_attention_weight_max": 37.42086410522461,
      "activations/layer23_attention_weight_min": -23.970975875854492,
      "activations/layer2_attention_weight_max": 34.655216217041016,
      "activations/layer2_attention_weight_min": -33.58705139160156,
      "activations/layer3_attention_weight_max": 96.45816802978516,
      "activations/layer3_attention_weight_min": -98.73838806152344,
      "activations/layer4_attention_weight_max": 116.00667572021484,
      "activations/layer4_attention_weight_min": -112.7655258178711,
      "activations/layer5_attention_weight_max": 51.70918273925781,
      "activations/layer5_attention_weight_min": -60.740325927734375,
      "activations/layer6_attention_weight_max": 48.06632614135742,
      "activations/layer6_attention_weight_min": -48.93643569946289,
      "activations/layer7_attention_weight_max": 88.22401428222656,
      "activations/layer7_attention_weight_min": -97.05751037597656,
      "activations/layer8_attention_weight_max": 39.44707489013672,
      "activations/layer8_attention_weight_min": -41.12824630737305,
      "activations/layer9_attention_weight_max": 33.20164108276367,
      "activations/layer9_attention_weight_min": -32.64344787597656,
      "epoch": 9.26,
      "learning_rate": 9.122727272727272e-05,
      "loss": 2.7945,
      "step": 159300
    },
    {
      "activations/layer0_attention_weight_max": 16.507444381713867,
      "activations/layer0_attention_weight_min": -11.448851585388184,
      "activations/layer10_attention_weight_max": 33.36368942260742,
      "activations/layer10_attention_weight_min": -33.183780670166016,
      "activations/layer11_attention_weight_max": 34.054466247558594,
      "activations/layer11_attention_weight_min": -31.8375244140625,
      "activations/layer12_attention_weight_max": 41.25662612915039,
      "activations/layer12_attention_weight_min": -38.148963928222656,
      "activations/layer13_attention_weight_max": 46.224754333496094,
      "activations/layer13_attention_weight_min": -33.88636779785156,
      "activations/layer14_attention_weight_max": 41.16756820678711,
      "activations/layer14_attention_weight_min": -39.8100471496582,
      "activations/layer15_attention_weight_max": 33.197532653808594,
      "activations/layer15_attention_weight_min": -31.108278274536133,
      "activations/layer16_attention_weight_max": 29.451303482055664,
      "activations/layer16_attention_weight_min": -28.28435516357422,
      "activations/layer17_attention_weight_max": 35.174747467041016,
      "activations/layer17_attention_weight_min": -24.429935455322266,
      "activations/layer18_attention_weight_max": 29.46898078918457,
      "activations/layer18_attention_weight_min": -22.371564865112305,
      "activations/layer19_attention_weight_max": 35.63775634765625,
      "activations/layer19_attention_weight_min": -30.06646156311035,
      "activations/layer1_attention_weight_max": 16.574922561645508,
      "activations/layer1_attention_weight_min": -13.764841079711914,
      "activations/layer20_attention_weight_max": 30.96901512145996,
      "activations/layer20_attention_weight_min": -25.14655113220215,
      "activations/layer21_attention_weight_max": 29.29585838317871,
      "activations/layer21_attention_weight_min": -22.699419021606445,
      "activations/layer22_attention_weight_max": 50.6853141784668,
      "activations/layer22_attention_weight_min": -29.444992065429688,
      "activations/layer23_attention_weight_max": 37.252655029296875,
      "activations/layer23_attention_weight_min": -23.29798126220703,
      "activations/layer2_attention_weight_max": 33.376243591308594,
      "activations/layer2_attention_weight_min": -34.67708969116211,
      "activations/layer3_attention_weight_max": 96.58219909667969,
      "activations/layer3_attention_weight_min": -101.88591766357422,
      "activations/layer4_attention_weight_max": 117.86419677734375,
      "activations/layer4_attention_weight_min": -116.2977294921875,
      "activations/layer5_attention_weight_max": 54.804378509521484,
      "activations/layer5_attention_weight_min": -62.764488220214844,
      "activations/layer6_attention_weight_max": 48.78419494628906,
      "activations/layer6_attention_weight_min": -50.38640594482422,
      "activations/layer7_attention_weight_max": 88.75397491455078,
      "activations/layer7_attention_weight_min": -92.04528045654297,
      "activations/layer8_attention_weight_max": 38.828006744384766,
      "activations/layer8_attention_weight_min": -40.463809967041016,
      "activations/layer9_attention_weight_max": 34.227088928222656,
      "activations/layer9_attention_weight_min": -35.1750602722168,
      "epoch": 9.26,
      "learning_rate": 9.120833333333333e-05,
      "loss": 2.7841,
      "step": 159350
    },
    {
      "activations/layer0_attention_weight_max": 15.95728874206543,
      "activations/layer0_attention_weight_min": -12.197189331054688,
      "activations/layer10_attention_weight_max": 35.225624084472656,
      "activations/layer10_attention_weight_min": -31.579601287841797,
      "activations/layer11_attention_weight_max": 35.84730529785156,
      "activations/layer11_attention_weight_min": -30.994626998901367,
      "activations/layer12_attention_weight_max": 34.25379180908203,
      "activations/layer12_attention_weight_min": -24.04416847229004,
      "activations/layer13_attention_weight_max": 37.849876403808594,
      "activations/layer13_attention_weight_min": -32.02465057373047,
      "activations/layer14_attention_weight_max": 43.2007942199707,
      "activations/layer14_attention_weight_min": -35.2838020324707,
      "activations/layer15_attention_weight_max": 33.376670837402344,
      "activations/layer15_attention_weight_min": -29.803939819335938,
      "activations/layer16_attention_weight_max": 33.34310531616211,
      "activations/layer16_attention_weight_min": -27.70499610900879,
      "activations/layer17_attention_weight_max": 31.0065860748291,
      "activations/layer17_attention_weight_min": -27.35914421081543,
      "activations/layer18_attention_weight_max": 30.202260971069336,
      "activations/layer18_attention_weight_min": -24.141952514648438,
      "activations/layer19_attention_weight_max": 37.12772750854492,
      "activations/layer19_attention_weight_min": -32.62302780151367,
      "activations/layer1_attention_weight_max": 16.25807762145996,
      "activations/layer1_attention_weight_min": -16.98324966430664,
      "activations/layer20_attention_weight_max": 32.43172073364258,
      "activations/layer20_attention_weight_min": -25.942882537841797,
      "activations/layer21_attention_weight_max": 33.18224334716797,
      "activations/layer21_attention_weight_min": -24.364999771118164,
      "activations/layer22_attention_weight_max": 47.914268493652344,
      "activations/layer22_attention_weight_min": -31.86563491821289,
      "activations/layer23_attention_weight_max": 35.1673583984375,
      "activations/layer23_attention_weight_min": -25.266584396362305,
      "activations/layer2_attention_weight_max": 33.0450325012207,
      "activations/layer2_attention_weight_min": -33.160369873046875,
      "activations/layer3_attention_weight_max": 91.9996109008789,
      "activations/layer3_attention_weight_min": -93.93047332763672,
      "activations/layer4_attention_weight_max": 113.09382629394531,
      "activations/layer4_attention_weight_min": -110.02474212646484,
      "activations/layer5_attention_weight_max": 53.46718978881836,
      "activations/layer5_attention_weight_min": -62.66099548339844,
      "activations/layer6_attention_weight_max": 46.40122604370117,
      "activations/layer6_attention_weight_min": -49.273048400878906,
      "activations/layer7_attention_weight_max": 93.14501190185547,
      "activations/layer7_attention_weight_min": -89.67882537841797,
      "activations/layer8_attention_weight_max": 42.445884704589844,
      "activations/layer8_attention_weight_min": -41.39113998413086,
      "activations/layer9_attention_weight_max": 37.28792190551758,
      "activations/layer9_attention_weight_min": -34.974605560302734,
      "epoch": 9.26,
      "learning_rate": 9.118939393939392e-05,
      "loss": 2.7874,
      "step": 159400
    },
    {
      "activations/layer0_attention_weight_max": 16.03456687927246,
      "activations/layer0_attention_weight_min": -12.72321605682373,
      "activations/layer10_attention_weight_max": 34.3585090637207,
      "activations/layer10_attention_weight_min": -32.26614761352539,
      "activations/layer11_attention_weight_max": 35.11244583129883,
      "activations/layer11_attention_weight_min": -30.88400650024414,
      "activations/layer12_attention_weight_max": 25.93163299560547,
      "activations/layer12_attention_weight_min": -23.644798278808594,
      "activations/layer13_attention_weight_max": 42.14606475830078,
      "activations/layer13_attention_weight_min": -32.27988052368164,
      "activations/layer14_attention_weight_max": 51.03176498413086,
      "activations/layer14_attention_weight_min": -39.40524673461914,
      "activations/layer15_attention_weight_max": 39.27064514160156,
      "activations/layer15_attention_weight_min": -30.30430030822754,
      "activations/layer16_attention_weight_max": 36.701725006103516,
      "activations/layer16_attention_weight_min": -26.720396041870117,
      "activations/layer17_attention_weight_max": 38.55507278442383,
      "activations/layer17_attention_weight_min": -25.967016220092773,
      "activations/layer18_attention_weight_max": 34.3098030090332,
      "activations/layer18_attention_weight_min": -23.111528396606445,
      "activations/layer19_attention_weight_max": 40.93718338012695,
      "activations/layer19_attention_weight_min": -30.55938720703125,
      "activations/layer1_attention_weight_max": 17.26812171936035,
      "activations/layer1_attention_weight_min": -15.33592700958252,
      "activations/layer20_attention_weight_max": 36.59408187866211,
      "activations/layer20_attention_weight_min": -24.5828857421875,
      "activations/layer21_attention_weight_max": 34.854103088378906,
      "activations/layer21_attention_weight_min": -24.218795776367188,
      "activations/layer22_attention_weight_max": 50.48567199707031,
      "activations/layer22_attention_weight_min": -30.110580444335938,
      "activations/layer23_attention_weight_max": 37.56422424316406,
      "activations/layer23_attention_weight_min": -24.14946746826172,
      "activations/layer2_attention_weight_max": 33.516136169433594,
      "activations/layer2_attention_weight_min": -32.552001953125,
      "activations/layer3_attention_weight_max": 94.49520111083984,
      "activations/layer3_attention_weight_min": -98.29426574707031,
      "activations/layer4_attention_weight_max": 120.3749008178711,
      "activations/layer4_attention_weight_min": -117.33316802978516,
      "activations/layer5_attention_weight_max": 54.60479736328125,
      "activations/layer5_attention_weight_min": -64.27766418457031,
      "activations/layer6_attention_weight_max": 49.16081237792969,
      "activations/layer6_attention_weight_min": -51.05478286743164,
      "activations/layer7_attention_weight_max": 91.86631774902344,
      "activations/layer7_attention_weight_min": -93.8458023071289,
      "activations/layer8_attention_weight_max": 41.064727783203125,
      "activations/layer8_attention_weight_min": -40.486515045166016,
      "activations/layer9_attention_weight_max": 36.14728927612305,
      "activations/layer9_attention_weight_min": -34.228824615478516,
      "epoch": 9.26,
      "learning_rate": 9.117045454545454e-05,
      "loss": 2.7773,
      "step": 159450
    },
    {
      "activations/layer0_attention_weight_max": 14.685221672058105,
      "activations/layer0_attention_weight_min": -12.22212028503418,
      "activations/layer10_attention_weight_max": 33.16717529296875,
      "activations/layer10_attention_weight_min": -31.487003326416016,
      "activations/layer11_attention_weight_max": 30.533109664916992,
      "activations/layer11_attention_weight_min": -30.5323486328125,
      "activations/layer12_attention_weight_max": 29.620214462280273,
      "activations/layer12_attention_weight_min": -23.560409545898438,
      "activations/layer13_attention_weight_max": 43.401424407958984,
      "activations/layer13_attention_weight_min": -32.5864372253418,
      "activations/layer14_attention_weight_max": 46.4002571105957,
      "activations/layer14_attention_weight_min": -38.54859924316406,
      "activations/layer15_attention_weight_max": 35.696929931640625,
      "activations/layer15_attention_weight_min": -30.48363494873047,
      "activations/layer16_attention_weight_max": 35.36060333251953,
      "activations/layer16_attention_weight_min": -29.670808792114258,
      "activations/layer17_attention_weight_max": 31.266855239868164,
      "activations/layer17_attention_weight_min": -25.81737518310547,
      "activations/layer18_attention_weight_max": 31.454349517822266,
      "activations/layer18_attention_weight_min": -21.3758602142334,
      "activations/layer19_attention_weight_max": 36.90860366821289,
      "activations/layer19_attention_weight_min": -30.094202041625977,
      "activations/layer1_attention_weight_max": 17.606096267700195,
      "activations/layer1_attention_weight_min": -14.263118743896484,
      "activations/layer20_attention_weight_max": 27.817564010620117,
      "activations/layer20_attention_weight_min": -22.880287170410156,
      "activations/layer21_attention_weight_max": 27.645137786865234,
      "activations/layer21_attention_weight_min": -22.7197322845459,
      "activations/layer22_attention_weight_max": 45.26223373413086,
      "activations/layer22_attention_weight_min": -28.063100814819336,
      "activations/layer23_attention_weight_max": 33.20410919189453,
      "activations/layer23_attention_weight_min": -23.169981002807617,
      "activations/layer2_attention_weight_max": 32.071537017822266,
      "activations/layer2_attention_weight_min": -32.740909576416016,
      "activations/layer3_attention_weight_max": 93.2778091430664,
      "activations/layer3_attention_weight_min": -97.52017211914062,
      "activations/layer4_attention_weight_max": 113.81497955322266,
      "activations/layer4_attention_weight_min": -109.6618881225586,
      "activations/layer5_attention_weight_max": 50.74322509765625,
      "activations/layer5_attention_weight_min": -62.02573776245117,
      "activations/layer6_attention_weight_max": 45.48716354370117,
      "activations/layer6_attention_weight_min": -48.21607208251953,
      "activations/layer7_attention_weight_max": 87.0904769897461,
      "activations/layer7_attention_weight_min": -93.98835754394531,
      "activations/layer8_attention_weight_max": 39.21461486816406,
      "activations/layer8_attention_weight_min": -40.82320022583008,
      "activations/layer9_attention_weight_max": 35.338008880615234,
      "activations/layer9_attention_weight_min": -34.2865104675293,
      "epoch": 9.27,
      "learning_rate": 9.115151515151515e-05,
      "loss": 2.7941,
      "step": 159500
    },
    {
      "activations/layer0_attention_weight_max": 13.767887115478516,
      "activations/layer0_attention_weight_min": -11.476667404174805,
      "activations/layer10_attention_weight_max": 32.36992645263672,
      "activations/layer10_attention_weight_min": -31.02858543395996,
      "activations/layer11_attention_weight_max": 34.129615783691406,
      "activations/layer11_attention_weight_min": -29.262609481811523,
      "activations/layer12_attention_weight_max": 23.29151725769043,
      "activations/layer12_attention_weight_min": -23.5118350982666,
      "activations/layer13_attention_weight_max": 37.2678337097168,
      "activations/layer13_attention_weight_min": -31.3599910736084,
      "activations/layer14_attention_weight_max": 48.49735641479492,
      "activations/layer14_attention_weight_min": -39.531219482421875,
      "activations/layer15_attention_weight_max": 37.672428131103516,
      "activations/layer15_attention_weight_min": -29.677875518798828,
      "activations/layer16_attention_weight_max": 32.791927337646484,
      "activations/layer16_attention_weight_min": -28.309127807617188,
      "activations/layer17_attention_weight_max": 34.72782897949219,
      "activations/layer17_attention_weight_min": -26.880876541137695,
      "activations/layer18_attention_weight_max": 35.36427688598633,
      "activations/layer18_attention_weight_min": -20.695234298706055,
      "activations/layer19_attention_weight_max": 32.96849060058594,
      "activations/layer19_attention_weight_min": -30.41360855102539,
      "activations/layer1_attention_weight_max": 16.242752075195312,
      "activations/layer1_attention_weight_min": -15.878472328186035,
      "activations/layer20_attention_weight_max": 29.537498474121094,
      "activations/layer20_attention_weight_min": -23.297718048095703,
      "activations/layer21_attention_weight_max": 29.946983337402344,
      "activations/layer21_attention_weight_min": -22.302099227905273,
      "activations/layer22_attention_weight_max": 46.56032943725586,
      "activations/layer22_attention_weight_min": -29.964942932128906,
      "activations/layer23_attention_weight_max": 34.86681365966797,
      "activations/layer23_attention_weight_min": -25.81298828125,
      "activations/layer2_attention_weight_max": 31.588886260986328,
      "activations/layer2_attention_weight_min": -31.272167205810547,
      "activations/layer3_attention_weight_max": 91.7574234008789,
      "activations/layer3_attention_weight_min": -93.73650360107422,
      "activations/layer4_attention_weight_max": 115.38060760498047,
      "activations/layer4_attention_weight_min": -112.38777923583984,
      "activations/layer5_attention_weight_max": 53.574607849121094,
      "activations/layer5_attention_weight_min": -65.48570251464844,
      "activations/layer6_attention_weight_max": 46.66279220581055,
      "activations/layer6_attention_weight_min": -50.99794006347656,
      "activations/layer7_attention_weight_max": 90.32318115234375,
      "activations/layer7_attention_weight_min": -91.0409927368164,
      "activations/layer8_attention_weight_max": 38.4759635925293,
      "activations/layer8_attention_weight_min": -39.25076675415039,
      "activations/layer9_attention_weight_max": 34.321014404296875,
      "activations/layer9_attention_weight_min": -32.96963119506836,
      "epoch": 9.27,
      "learning_rate": 9.113257575757574e-05,
      "loss": 2.798,
      "step": 159550
    },
    {
      "activations/layer0_attention_weight_max": 16.417098999023438,
      "activations/layer0_attention_weight_min": -12.019715309143066,
      "activations/layer10_attention_weight_max": 32.63011932373047,
      "activations/layer10_attention_weight_min": -31.438236236572266,
      "activations/layer11_attention_weight_max": 33.200016021728516,
      "activations/layer11_attention_weight_min": -29.946819305419922,
      "activations/layer12_attention_weight_max": 25.536720275878906,
      "activations/layer12_attention_weight_min": -26.021970748901367,
      "activations/layer13_attention_weight_max": 41.405372619628906,
      "activations/layer13_attention_weight_min": -30.87343406677246,
      "activations/layer14_attention_weight_max": 50.402626037597656,
      "activations/layer14_attention_weight_min": -37.782081604003906,
      "activations/layer15_attention_weight_max": 39.45565414428711,
      "activations/layer15_attention_weight_min": -31.235815048217773,
      "activations/layer16_attention_weight_max": 34.30777359008789,
      "activations/layer16_attention_weight_min": -27.16827392578125,
      "activations/layer17_attention_weight_max": 32.81429672241211,
      "activations/layer17_attention_weight_min": -23.547958374023438,
      "activations/layer18_attention_weight_max": 32.26424026489258,
      "activations/layer18_attention_weight_min": -20.95844841003418,
      "activations/layer19_attention_weight_max": 40.42451095581055,
      "activations/layer19_attention_weight_min": -31.571487426757812,
      "activations/layer1_attention_weight_max": 17.276844024658203,
      "activations/layer1_attention_weight_min": -14.363900184631348,
      "activations/layer20_attention_weight_max": 33.95333480834961,
      "activations/layer20_attention_weight_min": -25.774120330810547,
      "activations/layer21_attention_weight_max": 30.750741958618164,
      "activations/layer21_attention_weight_min": -24.24730682373047,
      "activations/layer22_attention_weight_max": 51.18317413330078,
      "activations/layer22_attention_weight_min": -32.85887908935547,
      "activations/layer23_attention_weight_max": 39.067420959472656,
      "activations/layer23_attention_weight_min": -23.46506118774414,
      "activations/layer2_attention_weight_max": 31.81932830810547,
      "activations/layer2_attention_weight_min": -32.011165618896484,
      "activations/layer3_attention_weight_max": 97.77671813964844,
      "activations/layer3_attention_weight_min": -96.2730941772461,
      "activations/layer4_attention_weight_max": 121.5254135131836,
      "activations/layer4_attention_weight_min": -114.49847412109375,
      "activations/layer5_attention_weight_max": 54.69817352294922,
      "activations/layer5_attention_weight_min": -62.4807243347168,
      "activations/layer6_attention_weight_max": 48.61982345581055,
      "activations/layer6_attention_weight_min": -49.6537971496582,
      "activations/layer7_attention_weight_max": 86.53309631347656,
      "activations/layer7_attention_weight_min": -95.18739318847656,
      "activations/layer8_attention_weight_max": 39.050899505615234,
      "activations/layer8_attention_weight_min": -40.25361251831055,
      "activations/layer9_attention_weight_max": 32.940853118896484,
      "activations/layer9_attention_weight_min": -31.905439376831055,
      "epoch": 9.27,
      "learning_rate": 9.111363636363635e-05,
      "loss": 2.7854,
      "step": 159600
    },
    {
      "activations/layer0_attention_weight_max": 15.727971076965332,
      "activations/layer0_attention_weight_min": -12.725876808166504,
      "activations/layer10_attention_weight_max": 37.925140380859375,
      "activations/layer10_attention_weight_min": -35.444068908691406,
      "activations/layer11_attention_weight_max": 37.16566467285156,
      "activations/layer11_attention_weight_min": -33.450645446777344,
      "activations/layer12_attention_weight_max": 25.501585006713867,
      "activations/layer12_attention_weight_min": -23.3607177734375,
      "activations/layer13_attention_weight_max": 42.20020294189453,
      "activations/layer13_attention_weight_min": -32.705596923828125,
      "activations/layer14_attention_weight_max": 58.58806228637695,
      "activations/layer14_attention_weight_min": -38.88385772705078,
      "activations/layer15_attention_weight_max": 40.3351936340332,
      "activations/layer15_attention_weight_min": -30.209514617919922,
      "activations/layer16_attention_weight_max": 35.433475494384766,
      "activations/layer16_attention_weight_min": -27.890832901000977,
      "activations/layer17_attention_weight_max": 36.091670989990234,
      "activations/layer17_attention_weight_min": -27.469491958618164,
      "activations/layer18_attention_weight_max": 31.701908111572266,
      "activations/layer18_attention_weight_min": -21.16473960876465,
      "activations/layer19_attention_weight_max": 38.4439697265625,
      "activations/layer19_attention_weight_min": -31.643646240234375,
      "activations/layer1_attention_weight_max": 16.381210327148438,
      "activations/layer1_attention_weight_min": -14.500968933105469,
      "activations/layer20_attention_weight_max": 33.36467742919922,
      "activations/layer20_attention_weight_min": -24.400667190551758,
      "activations/layer21_attention_weight_max": 32.48191452026367,
      "activations/layer21_attention_weight_min": -22.70638084411621,
      "activations/layer22_attention_weight_max": 47.2652473449707,
      "activations/layer22_attention_weight_min": -28.423357009887695,
      "activations/layer23_attention_weight_max": 36.41162109375,
      "activations/layer23_attention_weight_min": -24.718578338623047,
      "activations/layer2_attention_weight_max": 33.36974334716797,
      "activations/layer2_attention_weight_min": -33.30831527709961,
      "activations/layer3_attention_weight_max": 96.80648040771484,
      "activations/layer3_attention_weight_min": -101.20189666748047,
      "activations/layer4_attention_weight_max": 120.30909729003906,
      "activations/layer4_attention_weight_min": -119.5864028930664,
      "activations/layer5_attention_weight_max": 55.348628997802734,
      "activations/layer5_attention_weight_min": -62.18180847167969,
      "activations/layer6_attention_weight_max": 48.57217025756836,
      "activations/layer6_attention_weight_min": -50.81047821044922,
      "activations/layer7_attention_weight_max": 97.92556762695312,
      "activations/layer7_attention_weight_min": -91.35652923583984,
      "activations/layer8_attention_weight_max": 43.3906364440918,
      "activations/layer8_attention_weight_min": -43.036190032958984,
      "activations/layer9_attention_weight_max": 39.43216323852539,
      "activations/layer9_attention_weight_min": -37.008235931396484,
      "epoch": 9.28,
      "learning_rate": 9.109469696969697e-05,
      "loss": 2.7876,
      "step": 159650
    },
    {
      "activations/layer0_attention_weight_max": 15.572587013244629,
      "activations/layer0_attention_weight_min": -12.559795379638672,
      "activations/layer10_attention_weight_max": 32.024227142333984,
      "activations/layer10_attention_weight_min": -32.77466583251953,
      "activations/layer11_attention_weight_max": 30.879966735839844,
      "activations/layer11_attention_weight_min": -29.755142211914062,
      "activations/layer12_attention_weight_max": 24.22049331665039,
      "activations/layer12_attention_weight_min": -25.951095581054688,
      "activations/layer13_attention_weight_max": 38.279579162597656,
      "activations/layer13_attention_weight_min": -34.86960220336914,
      "activations/layer14_attention_weight_max": 44.19649887084961,
      "activations/layer14_attention_weight_min": -37.382484436035156,
      "activations/layer15_attention_weight_max": 35.79989242553711,
      "activations/layer15_attention_weight_min": -30.089914321899414,
      "activations/layer16_attention_weight_max": 29.48814582824707,
      "activations/layer16_attention_weight_min": -28.07962417602539,
      "activations/layer17_attention_weight_max": 29.686519622802734,
      "activations/layer17_attention_weight_min": -25.933876037597656,
      "activations/layer18_attention_weight_max": 29.235477447509766,
      "activations/layer18_attention_weight_min": -26.987272262573242,
      "activations/layer19_attention_weight_max": 35.5068473815918,
      "activations/layer19_attention_weight_min": -29.830686569213867,
      "activations/layer1_attention_weight_max": 16.824726104736328,
      "activations/layer1_attention_weight_min": -14.19437026977539,
      "activations/layer20_attention_weight_max": 27.831462860107422,
      "activations/layer20_attention_weight_min": -25.045791625976562,
      "activations/layer21_attention_weight_max": 26.615032196044922,
      "activations/layer21_attention_weight_min": -24.667526245117188,
      "activations/layer22_attention_weight_max": 39.166297912597656,
      "activations/layer22_attention_weight_min": -28.387483596801758,
      "activations/layer23_attention_weight_max": 31.25086784362793,
      "activations/layer23_attention_weight_min": -23.835674285888672,
      "activations/layer2_attention_weight_max": 31.207035064697266,
      "activations/layer2_attention_weight_min": -34.12602233886719,
      "activations/layer3_attention_weight_max": 91.505859375,
      "activations/layer3_attention_weight_min": -95.74472045898438,
      "activations/layer4_attention_weight_max": 117.3194351196289,
      "activations/layer4_attention_weight_min": -116.97027587890625,
      "activations/layer5_attention_weight_max": 54.62049102783203,
      "activations/layer5_attention_weight_min": -60.4591178894043,
      "activations/layer6_attention_weight_max": 49.965633392333984,
      "activations/layer6_attention_weight_min": -49.39741897583008,
      "activations/layer7_attention_weight_max": 87.89652252197266,
      "activations/layer7_attention_weight_min": -88.59686279296875,
      "activations/layer8_attention_weight_max": 40.09437561035156,
      "activations/layer8_attention_weight_min": -39.31055450439453,
      "activations/layer9_attention_weight_max": 35.020355224609375,
      "activations/layer9_attention_weight_min": -32.91487503051758,
      "epoch": 9.28,
      "learning_rate": 9.107575757575757e-05,
      "loss": 2.7919,
      "step": 159700
    },
    {
      "activations/layer0_attention_weight_max": 15.343432426452637,
      "activations/layer0_attention_weight_min": -11.643000602722168,
      "activations/layer10_attention_weight_max": 35.239707946777344,
      "activations/layer10_attention_weight_min": -33.59416580200195,
      "activations/layer11_attention_weight_max": 35.207481384277344,
      "activations/layer11_attention_weight_min": -32.30891799926758,
      "activations/layer12_attention_weight_max": 27.085695266723633,
      "activations/layer12_attention_weight_min": -26.09098243713379,
      "activations/layer13_attention_weight_max": 44.809146881103516,
      "activations/layer13_attention_weight_min": -35.53642654418945,
      "activations/layer14_attention_weight_max": 48.95644760131836,
      "activations/layer14_attention_weight_min": -37.43378829956055,
      "activations/layer15_attention_weight_max": 40.49308395385742,
      "activations/layer15_attention_weight_min": -30.944828033447266,
      "activations/layer16_attention_weight_max": 35.222389221191406,
      "activations/layer16_attention_weight_min": -30.12084197998047,
      "activations/layer17_attention_weight_max": 39.28409957885742,
      "activations/layer17_attention_weight_min": -26.97124671936035,
      "activations/layer18_attention_weight_max": 38.42198181152344,
      "activations/layer18_attention_weight_min": -23.56516456604004,
      "activations/layer19_attention_weight_max": 40.9718132019043,
      "activations/layer19_attention_weight_min": -31.232513427734375,
      "activations/layer1_attention_weight_max": 16.9639892578125,
      "activations/layer1_attention_weight_min": -14.206891059875488,
      "activations/layer20_attention_weight_max": 33.50138854980469,
      "activations/layer20_attention_weight_min": -24.597368240356445,
      "activations/layer21_attention_weight_max": 34.82454299926758,
      "activations/layer21_attention_weight_min": -23.883983612060547,
      "activations/layer22_attention_weight_max": 54.18888473510742,
      "activations/layer22_attention_weight_min": -29.75032615661621,
      "activations/layer23_attention_weight_max": 40.83222579956055,
      "activations/layer23_attention_weight_min": -25.602554321289062,
      "activations/layer2_attention_weight_max": 32.948665618896484,
      "activations/layer2_attention_weight_min": -31.95248794555664,
      "activations/layer3_attention_weight_max": 94.51750946044922,
      "activations/layer3_attention_weight_min": -92.22176361083984,
      "activations/layer4_attention_weight_max": 121.08977508544922,
      "activations/layer4_attention_weight_min": -115.86639404296875,
      "activations/layer5_attention_weight_max": 56.433631896972656,
      "activations/layer5_attention_weight_min": -61.877052307128906,
      "activations/layer6_attention_weight_max": 49.71546173095703,
      "activations/layer6_attention_weight_min": -52.231998443603516,
      "activations/layer7_attention_weight_max": 96.4936294555664,
      "activations/layer7_attention_weight_min": -92.07762145996094,
      "activations/layer8_attention_weight_max": 41.42329788208008,
      "activations/layer8_attention_weight_min": -41.32080841064453,
      "activations/layer9_attention_weight_max": 35.63650894165039,
      "activations/layer9_attention_weight_min": -34.72683334350586,
      "epoch": 9.28,
      "learning_rate": 9.105681818181817e-05,
      "loss": 2.8048,
      "step": 159750
    },
    {
      "activations/layer0_attention_weight_max": 16.295629501342773,
      "activations/layer0_attention_weight_min": -12.824441909790039,
      "activations/layer10_attention_weight_max": 37.4925537109375,
      "activations/layer10_attention_weight_min": -32.449066162109375,
      "activations/layer11_attention_weight_max": 31.378265380859375,
      "activations/layer11_attention_weight_min": -29.709545135498047,
      "activations/layer12_attention_weight_max": 22.288156509399414,
      "activations/layer12_attention_weight_min": -27.575761795043945,
      "activations/layer13_attention_weight_max": 39.57598114013672,
      "activations/layer13_attention_weight_min": -33.29930114746094,
      "activations/layer14_attention_weight_max": 45.71635055541992,
      "activations/layer14_attention_weight_min": -38.264312744140625,
      "activations/layer15_attention_weight_max": 37.48982238769531,
      "activations/layer15_attention_weight_min": -30.108531951904297,
      "activations/layer16_attention_weight_max": 36.04151153564453,
      "activations/layer16_attention_weight_min": -26.52699851989746,
      "activations/layer17_attention_weight_max": 31.892038345336914,
      "activations/layer17_attention_weight_min": -23.649765014648438,
      "activations/layer18_attention_weight_max": 30.02479362487793,
      "activations/layer18_attention_weight_min": -21.42836570739746,
      "activations/layer19_attention_weight_max": 35.89402770996094,
      "activations/layer19_attention_weight_min": -31.030813217163086,
      "activations/layer1_attention_weight_max": 15.578641891479492,
      "activations/layer1_attention_weight_min": -13.70588493347168,
      "activations/layer20_attention_weight_max": 33.10832977294922,
      "activations/layer20_attention_weight_min": -24.49431037902832,
      "activations/layer21_attention_weight_max": 33.96059799194336,
      "activations/layer21_attention_weight_min": -24.561412811279297,
      "activations/layer22_attention_weight_max": 43.06401062011719,
      "activations/layer22_attention_weight_min": -30.05194854736328,
      "activations/layer23_attention_weight_max": 37.657958984375,
      "activations/layer23_attention_weight_min": -24.811620712280273,
      "activations/layer2_attention_weight_max": 33.0606689453125,
      "activations/layer2_attention_weight_min": -33.534969329833984,
      "activations/layer3_attention_weight_max": 95.33686065673828,
      "activations/layer3_attention_weight_min": -99.21481323242188,
      "activations/layer4_attention_weight_max": 115.10450744628906,
      "activations/layer4_attention_weight_min": -116.68155670166016,
      "activations/layer5_attention_weight_max": 57.56261444091797,
      "activations/layer5_attention_weight_min": -60.513153076171875,
      "activations/layer6_attention_weight_max": 50.90345001220703,
      "activations/layer6_attention_weight_min": -51.03852844238281,
      "activations/layer7_attention_weight_max": 92.76203918457031,
      "activations/layer7_attention_weight_min": -90.00506591796875,
      "activations/layer8_attention_weight_max": 40.5939826965332,
      "activations/layer8_attention_weight_min": -42.24372482299805,
      "activations/layer9_attention_weight_max": 37.97526168823242,
      "activations/layer9_attention_weight_min": -34.79941177368164,
      "epoch": 9.29,
      "learning_rate": 9.103787878787879e-05,
      "loss": 2.8011,
      "step": 159800
    },
    {
      "activations/layer0_attention_weight_max": 15.48818588256836,
      "activations/layer0_attention_weight_min": -13.069838523864746,
      "activations/layer10_attention_weight_max": 35.279388427734375,
      "activations/layer10_attention_weight_min": -30.902341842651367,
      "activations/layer11_attention_weight_max": 37.04493713378906,
      "activations/layer11_attention_weight_min": -30.394277572631836,
      "activations/layer12_attention_weight_max": 25.867216110229492,
      "activations/layer12_attention_weight_min": -28.567867279052734,
      "activations/layer13_attention_weight_max": 39.82069778442383,
      "activations/layer13_attention_weight_min": -34.293365478515625,
      "activations/layer14_attention_weight_max": 51.250335693359375,
      "activations/layer14_attention_weight_min": -42.98710250854492,
      "activations/layer15_attention_weight_max": 34.72703552246094,
      "activations/layer15_attention_weight_min": -29.940372467041016,
      "activations/layer16_attention_weight_max": 32.12192916870117,
      "activations/layer16_attention_weight_min": -26.645919799804688,
      "activations/layer17_attention_weight_max": 34.32820510864258,
      "activations/layer17_attention_weight_min": -25.31433868408203,
      "activations/layer18_attention_weight_max": 30.39539909362793,
      "activations/layer18_attention_weight_min": -21.862627029418945,
      "activations/layer19_attention_weight_max": 38.21644973754883,
      "activations/layer19_attention_weight_min": -29.678022384643555,
      "activations/layer1_attention_weight_max": 16.971492767333984,
      "activations/layer1_attention_weight_min": -14.238448143005371,
      "activations/layer20_attention_weight_max": 31.500125885009766,
      "activations/layer20_attention_weight_min": -24.636695861816406,
      "activations/layer21_attention_weight_max": 30.432418823242188,
      "activations/layer21_attention_weight_min": -23.32603645324707,
      "activations/layer22_attention_weight_max": 46.833274841308594,
      "activations/layer22_attention_weight_min": -30.872896194458008,
      "activations/layer23_attention_weight_max": 36.12643814086914,
      "activations/layer23_attention_weight_min": -25.178802490234375,
      "activations/layer2_attention_weight_max": 34.40258026123047,
      "activations/layer2_attention_weight_min": -32.981895446777344,
      "activations/layer3_attention_weight_max": 95.48754119873047,
      "activations/layer3_attention_weight_min": -94.34429168701172,
      "activations/layer4_attention_weight_max": 116.1855239868164,
      "activations/layer4_attention_weight_min": -109.18110656738281,
      "activations/layer5_attention_weight_max": 57.039772033691406,
      "activations/layer5_attention_weight_min": -62.49531173706055,
      "activations/layer6_attention_weight_max": 45.907527923583984,
      "activations/layer6_attention_weight_min": -49.687923431396484,
      "activations/layer7_attention_weight_max": 96.99337005615234,
      "activations/layer7_attention_weight_min": -87.85944366455078,
      "activations/layer8_attention_weight_max": 41.03670120239258,
      "activations/layer8_attention_weight_min": -40.01973342895508,
      "activations/layer9_attention_weight_max": 33.81494140625,
      "activations/layer9_attention_weight_min": -33.84425735473633,
      "epoch": 9.29,
      "learning_rate": 9.101893939393939e-05,
      "loss": 2.7957,
      "step": 159850
    },
    {
      "activations/layer0_attention_weight_max": 15.529281616210938,
      "activations/layer0_attention_weight_min": -12.242681503295898,
      "activations/layer10_attention_weight_max": 33.17816162109375,
      "activations/layer10_attention_weight_min": -30.9984188079834,
      "activations/layer11_attention_weight_max": 31.552104949951172,
      "activations/layer11_attention_weight_min": -29.00543212890625,
      "activations/layer12_attention_weight_max": 26.860031127929688,
      "activations/layer12_attention_weight_min": -24.6068058013916,
      "activations/layer13_attention_weight_max": 36.118282318115234,
      "activations/layer13_attention_weight_min": -30.750452041625977,
      "activations/layer14_attention_weight_max": 42.00435256958008,
      "activations/layer14_attention_weight_min": -36.27180862426758,
      "activations/layer15_attention_weight_max": 33.5969352722168,
      "activations/layer15_attention_weight_min": -30.38262176513672,
      "activations/layer16_attention_weight_max": 31.344762802124023,
      "activations/layer16_attention_weight_min": -27.686302185058594,
      "activations/layer17_attention_weight_max": 32.2219123840332,
      "activations/layer17_attention_weight_min": -25.042682647705078,
      "activations/layer18_attention_weight_max": 29.20806884765625,
      "activations/layer18_attention_weight_min": -24.002853393554688,
      "activations/layer19_attention_weight_max": 35.8852653503418,
      "activations/layer19_attention_weight_min": -31.706802368164062,
      "activations/layer1_attention_weight_max": 16.86591148376465,
      "activations/layer1_attention_weight_min": -14.974769592285156,
      "activations/layer20_attention_weight_max": 27.632699966430664,
      "activations/layer20_attention_weight_min": -25.220064163208008,
      "activations/layer21_attention_weight_max": 26.604379653930664,
      "activations/layer21_attention_weight_min": -23.697906494140625,
      "activations/layer22_attention_weight_max": 46.043670654296875,
      "activations/layer22_attention_weight_min": -32.458438873291016,
      "activations/layer23_attention_weight_max": 36.183128356933594,
      "activations/layer23_attention_weight_min": -24.00343132019043,
      "activations/layer2_attention_weight_max": 35.18836975097656,
      "activations/layer2_attention_weight_min": -31.25102996826172,
      "activations/layer3_attention_weight_max": 96.97804260253906,
      "activations/layer3_attention_weight_min": -95.6825180053711,
      "activations/layer4_attention_weight_max": 120.9889907836914,
      "activations/layer4_attention_weight_min": -111.07051849365234,
      "activations/layer5_attention_weight_max": 54.196285247802734,
      "activations/layer5_attention_weight_min": -61.335731506347656,
      "activations/layer6_attention_weight_max": 45.964839935302734,
      "activations/layer6_attention_weight_min": -46.5068244934082,
      "activations/layer7_attention_weight_max": 91.84864807128906,
      "activations/layer7_attention_weight_min": -84.46114349365234,
      "activations/layer8_attention_weight_max": 39.98966598510742,
      "activations/layer8_attention_weight_min": -39.08894729614258,
      "activations/layer9_attention_weight_max": 34.517208099365234,
      "activations/layer9_attention_weight_min": -33.889442443847656,
      "epoch": 9.29,
      "learning_rate": 9.099999999999999e-05,
      "loss": 2.8032,
      "step": 159900
    },
    {
      "activations/layer0_attention_weight_max": 15.521079063415527,
      "activations/layer0_attention_weight_min": -12.230517387390137,
      "activations/layer10_attention_weight_max": 30.091270446777344,
      "activations/layer10_attention_weight_min": -31.249914169311523,
      "activations/layer11_attention_weight_max": 32.79899215698242,
      "activations/layer11_attention_weight_min": -29.077672958374023,
      "activations/layer12_attention_weight_max": 22.301916122436523,
      "activations/layer12_attention_weight_min": -27.187761306762695,
      "activations/layer13_attention_weight_max": 38.6609001159668,
      "activations/layer13_attention_weight_min": -32.419185638427734,
      "activations/layer14_attention_weight_max": 46.11994171142578,
      "activations/layer14_attention_weight_min": -38.10484313964844,
      "activations/layer15_attention_weight_max": 37.29289245605469,
      "activations/layer15_attention_weight_min": -30.480613708496094,
      "activations/layer16_attention_weight_max": 33.454097747802734,
      "activations/layer16_attention_weight_min": -29.90350341796875,
      "activations/layer17_attention_weight_max": 33.73533248901367,
      "activations/layer17_attention_weight_min": -24.55991554260254,
      "activations/layer18_attention_weight_max": 35.10813903808594,
      "activations/layer18_attention_weight_min": -24.059865951538086,
      "activations/layer19_attention_weight_max": 37.07492446899414,
      "activations/layer19_attention_weight_min": -31.341949462890625,
      "activations/layer1_attention_weight_max": 16.592470169067383,
      "activations/layer1_attention_weight_min": -14.332544326782227,
      "activations/layer20_attention_weight_max": 34.849998474121094,
      "activations/layer20_attention_weight_min": -25.35881233215332,
      "activations/layer21_attention_weight_max": 32.7294921875,
      "activations/layer21_attention_weight_min": -22.86359405517578,
      "activations/layer22_attention_weight_max": 49.51451110839844,
      "activations/layer22_attention_weight_min": -28.133100509643555,
      "activations/layer23_attention_weight_max": 37.417686462402344,
      "activations/layer23_attention_weight_min": -24.20073699951172,
      "activations/layer2_attention_weight_max": 32.03515625,
      "activations/layer2_attention_weight_min": -31.10767364501953,
      "activations/layer3_attention_weight_max": 94.8958740234375,
      "activations/layer3_attention_weight_min": -93.84170532226562,
      "activations/layer4_attention_weight_max": 119.15567779541016,
      "activations/layer4_attention_weight_min": -114.24281311035156,
      "activations/layer5_attention_weight_max": 56.00674057006836,
      "activations/layer5_attention_weight_min": -64.11714172363281,
      "activations/layer6_attention_weight_max": 47.5439338684082,
      "activations/layer6_attention_weight_min": -48.20120620727539,
      "activations/layer7_attention_weight_max": 85.94367218017578,
      "activations/layer7_attention_weight_min": -89.61803436279297,
      "activations/layer8_attention_weight_max": 38.032039642333984,
      "activations/layer8_attention_weight_min": -39.521888732910156,
      "activations/layer9_attention_weight_max": 33.3010368347168,
      "activations/layer9_attention_weight_min": -33.321983337402344,
      "epoch": 9.29,
      "learning_rate": 9.098106060606059e-05,
      "loss": 2.7906,
      "step": 159950
    },
    {
      "activations/layer0_attention_weight_max": 15.934648513793945,
      "activations/layer0_attention_weight_min": -13.21216869354248,
      "activations/layer10_attention_weight_max": 34.144561767578125,
      "activations/layer10_attention_weight_min": -31.2977352142334,
      "activations/layer11_attention_weight_max": 34.09854507446289,
      "activations/layer11_attention_weight_min": -31.81502914428711,
      "activations/layer12_attention_weight_max": 24.576515197753906,
      "activations/layer12_attention_weight_min": -23.58266830444336,
      "activations/layer13_attention_weight_max": 39.274436950683594,
      "activations/layer13_attention_weight_min": -33.33483123779297,
      "activations/layer14_attention_weight_max": 48.550750732421875,
      "activations/layer14_attention_weight_min": -36.76564407348633,
      "activations/layer15_attention_weight_max": 35.39177703857422,
      "activations/layer15_attention_weight_min": -29.0745849609375,
      "activations/layer16_attention_weight_max": 33.57788848876953,
      "activations/layer16_attention_weight_min": -27.67998695373535,
      "activations/layer17_attention_weight_max": 35.57828140258789,
      "activations/layer17_attention_weight_min": -24.79838752746582,
      "activations/layer18_attention_weight_max": 31.184734344482422,
      "activations/layer18_attention_weight_min": -21.492509841918945,
      "activations/layer19_attention_weight_max": 35.36819839477539,
      "activations/layer19_attention_weight_min": -30.699298858642578,
      "activations/layer1_attention_weight_max": 17.528900146484375,
      "activations/layer1_attention_weight_min": -15.994926452636719,
      "activations/layer20_attention_weight_max": 29.8826904296875,
      "activations/layer20_attention_weight_min": -26.315128326416016,
      "activations/layer21_attention_weight_max": 27.99009132385254,
      "activations/layer21_attention_weight_min": -24.027965545654297,
      "activations/layer22_attention_weight_max": 47.226036071777344,
      "activations/layer22_attention_weight_min": -29.398685455322266,
      "activations/layer23_attention_weight_max": 33.74939727783203,
      "activations/layer23_attention_weight_min": -25.1417236328125,
      "activations/layer2_attention_weight_max": 31.539508819580078,
      "activations/layer2_attention_weight_min": -32.73509979248047,
      "activations/layer3_attention_weight_max": 93.94657897949219,
      "activations/layer3_attention_weight_min": -97.1630859375,
      "activations/layer4_attention_weight_max": 113.09364318847656,
      "activations/layer4_attention_weight_min": -116.90632629394531,
      "activations/layer5_attention_weight_max": 53.242431640625,
      "activations/layer5_attention_weight_min": -63.13595199584961,
      "activations/layer6_attention_weight_max": 46.62401580810547,
      "activations/layer6_attention_weight_min": -49.1068000793457,
      "activations/layer7_attention_weight_max": 94.35444641113281,
      "activations/layer7_attention_weight_min": -88.34856414794922,
      "activations/layer8_attention_weight_max": 41.574623107910156,
      "activations/layer8_attention_weight_min": -43.07942581176758,
      "activations/layer9_attention_weight_max": 34.380008697509766,
      "activations/layer9_attention_weight_min": -32.42316818237305,
      "epoch": 9.3,
      "learning_rate": 9.09621212121212e-05,
      "loss": 2.8118,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4378,
      "eval_samples_per_second": 508.9,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4378,
      "eval_openwebtext_samples_per_second": 508.9,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 1.8982,
      "eval_wikitext_samples_per_second": 240.227,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_lambada_loss": 2.505859375,
      "eval_lambada_ppl": 12.254085296672228,
      "eval_lambada_runtime": 9.4844,
      "eval_lambada_samples_per_second": 513.37,
      "step": 160000
    },
    {
      "activations/layer0_attention_weight_max": 15.825265884399414,
      "activations/layer0_attention_weight_min": -12.4039306640625,
      "activations/layer10_attention_weight_max": 32.93465042114258,
      "activations/layer10_attention_weight_min": -32.24872970581055,
      "activations/layer11_attention_weight_max": 34.642616271972656,
      "activations/layer11_attention_weight_min": -31.80166244506836,
      "activations/layer12_attention_weight_max": 22.915756225585938,
      "activations/layer12_attention_weight_min": -27.842649459838867,
      "activations/layer13_attention_weight_max": 39.34495162963867,
      "activations/layer13_attention_weight_min": -34.431480407714844,
      "activations/layer14_attention_weight_max": 48.249332427978516,
      "activations/layer14_attention_weight_min": -41.7303352355957,
      "activations/layer15_attention_weight_max": 37.99692153930664,
      "activations/layer15_attention_weight_min": -32.289424896240234,
      "activations/layer16_attention_weight_max": 31.990245819091797,
      "activations/layer16_attention_weight_min": -29.01014518737793,
      "activations/layer17_attention_weight_max": 33.52839660644531,
      "activations/layer17_attention_weight_min": -25.68405532836914,
      "activations/layer18_attention_weight_max": 30.115365982055664,
      "activations/layer18_attention_weight_min": -21.52003288269043,
      "activations/layer19_attention_weight_max": 35.18360137939453,
      "activations/layer19_attention_weight_min": -30.879209518432617,
      "activations/layer1_attention_weight_max": 17.278303146362305,
      "activations/layer1_attention_weight_min": -14.582674026489258,
      "activations/layer20_attention_weight_max": 30.915843963623047,
      "activations/layer20_attention_weight_min": -24.343326568603516,
      "activations/layer21_attention_weight_max": 29.54174041748047,
      "activations/layer21_attention_weight_min": -23.76852798461914,
      "activations/layer22_attention_weight_max": 43.321834564208984,
      "activations/layer22_attention_weight_min": -28.52830696105957,
      "activations/layer23_attention_weight_max": 38.56623840332031,
      "activations/layer23_attention_weight_min": -24.92911148071289,
      "activations/layer2_attention_weight_max": 32.8213996887207,
      "activations/layer2_attention_weight_min": -30.223060607910156,
      "activations/layer3_attention_weight_max": 92.526611328125,
      "activations/layer3_attention_weight_min": -90.05036926269531,
      "activations/layer4_attention_weight_max": 117.30294036865234,
      "activations/layer4_attention_weight_min": -112.6231689453125,
      "activations/layer5_attention_weight_max": 57.349822998046875,
      "activations/layer5_attention_weight_min": -59.57820129394531,
      "activations/layer6_attention_weight_max": 50.60911560058594,
      "activations/layer6_attention_weight_min": -51.68238830566406,
      "activations/layer7_attention_weight_max": 101.1986083984375,
      "activations/layer7_attention_weight_min": -92.2253189086914,
      "activations/layer8_attention_weight_max": 41.00883483886719,
      "activations/layer8_attention_weight_min": -40.54840850830078,
      "activations/layer9_attention_weight_max": 34.496192932128906,
      "activations/layer9_attention_weight_min": -33.05167007446289,
      "epoch": 9.3,
      "learning_rate": 9.094318181818182e-05,
      "loss": 2.8022,
      "step": 160050
    },
    {
      "activations/layer0_attention_weight_max": 15.081814765930176,
      "activations/layer0_attention_weight_min": -12.802018165588379,
      "activations/layer10_attention_weight_max": 34.6215705871582,
      "activations/layer10_attention_weight_min": -36.76401138305664,
      "activations/layer11_attention_weight_max": 33.31816864013672,
      "activations/layer11_attention_weight_min": -30.99590301513672,
      "activations/layer12_attention_weight_max": 24.613901138305664,
      "activations/layer12_attention_weight_min": -28.414491653442383,
      "activations/layer13_attention_weight_max": 41.34247589111328,
      "activations/layer13_attention_weight_min": -30.905555725097656,
      "activations/layer14_attention_weight_max": 47.311336517333984,
      "activations/layer14_attention_weight_min": -39.45039749145508,
      "activations/layer15_attention_weight_max": 38.275611877441406,
      "activations/layer15_attention_weight_min": -31.587915420532227,
      "activations/layer16_attention_weight_max": 33.19569396972656,
      "activations/layer16_attention_weight_min": -27.6469669342041,
      "activations/layer17_attention_weight_max": 35.18194580078125,
      "activations/layer17_attention_weight_min": -25.01549530029297,
      "activations/layer18_attention_weight_max": 32.23564910888672,
      "activations/layer18_attention_weight_min": -22.570293426513672,
      "activations/layer19_attention_weight_max": 41.02694320678711,
      "activations/layer19_attention_weight_min": -30.823017120361328,
      "activations/layer1_attention_weight_max": 17.780853271484375,
      "activations/layer1_attention_weight_min": -15.114912033081055,
      "activations/layer20_attention_weight_max": 38.35092544555664,
      "activations/layer20_attention_weight_min": -25.697025299072266,
      "activations/layer21_attention_weight_max": 32.1638069152832,
      "activations/layer21_attention_weight_min": -23.050552368164062,
      "activations/layer22_attention_weight_max": 49.00335693359375,
      "activations/layer22_attention_weight_min": -27.959810256958008,
      "activations/layer23_attention_weight_max": 38.638919830322266,
      "activations/layer23_attention_weight_min": -22.878108978271484,
      "activations/layer2_attention_weight_max": 34.568885803222656,
      "activations/layer2_attention_weight_min": -32.87944030761719,
      "activations/layer3_attention_weight_max": 94.7187728881836,
      "activations/layer3_attention_weight_min": -96.23193359375,
      "activations/layer4_attention_weight_max": 115.13945770263672,
      "activations/layer4_attention_weight_min": -113.38811492919922,
      "activations/layer5_attention_weight_max": 53.15926742553711,
      "activations/layer5_attention_weight_min": -62.815513610839844,
      "activations/layer6_attention_weight_max": 46.87107467651367,
      "activations/layer6_attention_weight_min": -50.06813049316406,
      "activations/layer7_attention_weight_max": 89.44054412841797,
      "activations/layer7_attention_weight_min": -90.14390563964844,
      "activations/layer8_attention_weight_max": 41.87089920043945,
      "activations/layer8_attention_weight_min": -42.58489227294922,
      "activations/layer9_attention_weight_max": 35.69182586669922,
      "activations/layer9_attention_weight_min": -35.40545654296875,
      "epoch": 9.3,
      "learning_rate": 9.092424242424241e-05,
      "loss": 2.8025,
      "step": 160100
    },
    {
      "activations/layer0_attention_weight_max": 14.29924201965332,
      "activations/layer0_attention_weight_min": -12.434650421142578,
      "activations/layer10_attention_weight_max": 35.303924560546875,
      "activations/layer10_attention_weight_min": -33.7982177734375,
      "activations/layer11_attention_weight_max": 34.396453857421875,
      "activations/layer11_attention_weight_min": -30.811079025268555,
      "activations/layer12_attention_weight_max": 24.99212074279785,
      "activations/layer12_attention_weight_min": -25.84671974182129,
      "activations/layer13_attention_weight_max": 51.48505783081055,
      "activations/layer13_attention_weight_min": -37.94453048706055,
      "activations/layer14_attention_weight_max": 51.79248046875,
      "activations/layer14_attention_weight_min": -42.90309524536133,
      "activations/layer15_attention_weight_max": 35.312713623046875,
      "activations/layer15_attention_weight_min": -31.166000366210938,
      "activations/layer16_attention_weight_max": 35.030696868896484,
      "activations/layer16_attention_weight_min": -29.227928161621094,
      "activations/layer17_attention_weight_max": 31.83313751220703,
      "activations/layer17_attention_weight_min": -26.394323348999023,
      "activations/layer18_attention_weight_max": 36.59086608886719,
      "activations/layer18_attention_weight_min": -23.688695907592773,
      "activations/layer19_attention_weight_max": 39.298675537109375,
      "activations/layer19_attention_weight_min": -31.96843910217285,
      "activations/layer1_attention_weight_max": 17.36594581604004,
      "activations/layer1_attention_weight_min": -16.578399658203125,
      "activations/layer20_attention_weight_max": 32.958229064941406,
      "activations/layer20_attention_weight_min": -24.901273727416992,
      "activations/layer21_attention_weight_max": 33.242862701416016,
      "activations/layer21_attention_weight_min": -24.187936782836914,
      "activations/layer22_attention_weight_max": 51.615325927734375,
      "activations/layer22_attention_weight_min": -32.56782531738281,
      "activations/layer23_attention_weight_max": 39.0452995300293,
      "activations/layer23_attention_weight_min": -24.360214233398438,
      "activations/layer2_attention_weight_max": 33.761383056640625,
      "activations/layer2_attention_weight_min": -34.38127517700195,
      "activations/layer3_attention_weight_max": 99.35545349121094,
      "activations/layer3_attention_weight_min": -103.65280151367188,
      "activations/layer4_attention_weight_max": 120.72252655029297,
      "activations/layer4_attention_weight_min": -119.92988586425781,
      "activations/layer5_attention_weight_max": 57.00619125366211,
      "activations/layer5_attention_weight_min": -61.353729248046875,
      "activations/layer6_attention_weight_max": 49.08335876464844,
      "activations/layer6_attention_weight_min": -52.0752067565918,
      "activations/layer7_attention_weight_max": 93.80906677246094,
      "activations/layer7_attention_weight_min": -93.27761840820312,
      "activations/layer8_attention_weight_max": 42.51412582397461,
      "activations/layer8_attention_weight_min": -43.8968391418457,
      "activations/layer9_attention_weight_max": 36.59413528442383,
      "activations/layer9_attention_weight_min": -35.716064453125,
      "epoch": 9.31,
      "learning_rate": 9.090530303030302e-05,
      "loss": 2.7962,
      "step": 160150
    },
    {
      "activations/layer0_attention_weight_max": 15.59631061553955,
      "activations/layer0_attention_weight_min": -12.637807846069336,
      "activations/layer10_attention_weight_max": 38.69417190551758,
      "activations/layer10_attention_weight_min": -36.03947448730469,
      "activations/layer11_attention_weight_max": 41.64191818237305,
      "activations/layer11_attention_weight_min": -35.7333984375,
      "activations/layer12_attention_weight_max": 33.2373161315918,
      "activations/layer12_attention_weight_min": -25.749658584594727,
      "activations/layer13_attention_weight_max": 48.9235954284668,
      "activations/layer13_attention_weight_min": -36.341007232666016,
      "activations/layer14_attention_weight_max": 60.3073616027832,
      "activations/layer14_attention_weight_min": -46.00801086425781,
      "activations/layer15_attention_weight_max": 44.38775634765625,
      "activations/layer15_attention_weight_min": -33.985450744628906,
      "activations/layer16_attention_weight_max": 38.82599639892578,
      "activations/layer16_attention_weight_min": -26.679407119750977,
      "activations/layer17_attention_weight_max": 35.8182373046875,
      "activations/layer17_attention_weight_min": -26.410287857055664,
      "activations/layer18_attention_weight_max": 31.52815055847168,
      "activations/layer18_attention_weight_min": -25.527503967285156,
      "activations/layer19_attention_weight_max": 43.84433364868164,
      "activations/layer19_attention_weight_min": -29.41383934020996,
      "activations/layer1_attention_weight_max": 17.078147888183594,
      "activations/layer1_attention_weight_min": -15.319208145141602,
      "activations/layer20_attention_weight_max": 39.92702865600586,
      "activations/layer20_attention_weight_min": -24.14907455444336,
      "activations/layer21_attention_weight_max": 41.0338249206543,
      "activations/layer21_attention_weight_min": -22.721364974975586,
      "activations/layer22_attention_weight_max": 54.110740661621094,
      "activations/layer22_attention_weight_min": -28.586627960205078,
      "activations/layer23_attention_weight_max": 43.48424530029297,
      "activations/layer23_attention_weight_min": -23.981847763061523,
      "activations/layer2_attention_weight_max": 32.44200134277344,
      "activations/layer2_attention_weight_min": -34.259639739990234,
      "activations/layer3_attention_weight_max": 97.878173828125,
      "activations/layer3_attention_weight_min": -98.47871398925781,
      "activations/layer4_attention_weight_max": 120.26726531982422,
      "activations/layer4_attention_weight_min": -120.3591537475586,
      "activations/layer5_attention_weight_max": 52.584896087646484,
      "activations/layer5_attention_weight_min": -62.70370101928711,
      "activations/layer6_attention_weight_max": 47.20506286621094,
      "activations/layer6_attention_weight_min": -48.19855880737305,
      "activations/layer7_attention_weight_max": 97.44694519042969,
      "activations/layer7_attention_weight_min": -90.24180603027344,
      "activations/layer8_attention_weight_max": 44.35303497314453,
      "activations/layer8_attention_weight_min": -42.36674880981445,
      "activations/layer9_attention_weight_max": 38.12855911254883,
      "activations/layer9_attention_weight_min": -34.04362869262695,
      "epoch": 9.31,
      "learning_rate": 9.088636363636364e-05,
      "loss": 2.7994,
      "step": 160200
    },
    {
      "activations/layer0_attention_weight_max": 15.453502655029297,
      "activations/layer0_attention_weight_min": -12.366318702697754,
      "activations/layer10_attention_weight_max": 29.67061424255371,
      "activations/layer10_attention_weight_min": -31.882579803466797,
      "activations/layer11_attention_weight_max": 33.9788932800293,
      "activations/layer11_attention_weight_min": -31.28345489501953,
      "activations/layer12_attention_weight_max": 32.69904327392578,
      "activations/layer12_attention_weight_min": -26.788541793823242,
      "activations/layer13_attention_weight_max": 39.56092834472656,
      "activations/layer13_attention_weight_min": -31.398229598999023,
      "activations/layer14_attention_weight_max": 57.34022903442383,
      "activations/layer14_attention_weight_min": -42.62190628051758,
      "activations/layer15_attention_weight_max": 38.97431182861328,
      "activations/layer15_attention_weight_min": -31.438852310180664,
      "activations/layer16_attention_weight_max": 30.593908309936523,
      "activations/layer16_attention_weight_min": -27.26771354675293,
      "activations/layer17_attention_weight_max": 31.91424560546875,
      "activations/layer17_attention_weight_min": -24.040742874145508,
      "activations/layer18_attention_weight_max": 28.323040008544922,
      "activations/layer18_attention_weight_min": -21.93455696105957,
      "activations/layer19_attention_weight_max": 40.3930778503418,
      "activations/layer19_attention_weight_min": -32.780311584472656,
      "activations/layer1_attention_weight_max": 16.34238052368164,
      "activations/layer1_attention_weight_min": -14.695669174194336,
      "activations/layer20_attention_weight_max": 29.670427322387695,
      "activations/layer20_attention_weight_min": -24.0660343170166,
      "activations/layer21_attention_weight_max": 27.584836959838867,
      "activations/layer21_attention_weight_min": -24.112258911132812,
      "activations/layer22_attention_weight_max": 43.97563552856445,
      "activations/layer22_attention_weight_min": -29.03360939025879,
      "activations/layer23_attention_weight_max": 36.080406188964844,
      "activations/layer23_attention_weight_min": -26.488014221191406,
      "activations/layer2_attention_weight_max": 30.82025909423828,
      "activations/layer2_attention_weight_min": -30.142776489257812,
      "activations/layer3_attention_weight_max": 88.98247528076172,
      "activations/layer3_attention_weight_min": -89.58975219726562,
      "activations/layer4_attention_weight_max": 110.15543365478516,
      "activations/layer4_attention_weight_min": -111.4166259765625,
      "activations/layer5_attention_weight_max": 51.65977096557617,
      "activations/layer5_attention_weight_min": -58.655452728271484,
      "activations/layer6_attention_weight_max": 44.65282440185547,
      "activations/layer6_attention_weight_min": -48.933326721191406,
      "activations/layer7_attention_weight_max": 85.98023223876953,
      "activations/layer7_attention_weight_min": -89.0629653930664,
      "activations/layer8_attention_weight_max": 37.76531982421875,
      "activations/layer8_attention_weight_min": -39.26234436035156,
      "activations/layer9_attention_weight_max": 30.972904205322266,
      "activations/layer9_attention_weight_min": -31.924421310424805,
      "epoch": 9.31,
      "learning_rate": 9.086742424242423e-05,
      "loss": 2.7873,
      "step": 160250
    },
    {
      "activations/layer0_attention_weight_max": 15.451799392700195,
      "activations/layer0_attention_weight_min": -11.91369915008545,
      "activations/layer10_attention_weight_max": 31.903013229370117,
      "activations/layer10_attention_weight_min": -30.531896591186523,
      "activations/layer11_attention_weight_max": 30.855224609375,
      "activations/layer11_attention_weight_min": -30.747419357299805,
      "activations/layer12_attention_weight_max": 22.120004653930664,
      "activations/layer12_attention_weight_min": -27.564279556274414,
      "activations/layer13_attention_weight_max": 38.071144104003906,
      "activations/layer13_attention_weight_min": -32.62199401855469,
      "activations/layer14_attention_weight_max": 39.70283126831055,
      "activations/layer14_attention_weight_min": -35.93981170654297,
      "activations/layer15_attention_weight_max": 35.55746078491211,
      "activations/layer15_attention_weight_min": -29.873159408569336,
      "activations/layer16_attention_weight_max": 30.23301887512207,
      "activations/layer16_attention_weight_min": -27.59020233154297,
      "activations/layer17_attention_weight_max": 32.44740676879883,
      "activations/layer17_attention_weight_min": -23.635318756103516,
      "activations/layer18_attention_weight_max": 30.261953353881836,
      "activations/layer18_attention_weight_min": -21.51800537109375,
      "activations/layer19_attention_weight_max": 32.225425720214844,
      "activations/layer19_attention_weight_min": -28.601816177368164,
      "activations/layer1_attention_weight_max": 17.577167510986328,
      "activations/layer1_attention_weight_min": -15.018047332763672,
      "activations/layer20_attention_weight_max": 27.96114158630371,
      "activations/layer20_attention_weight_min": -24.507734298706055,
      "activations/layer21_attention_weight_max": 27.999919891357422,
      "activations/layer21_attention_weight_min": -23.509511947631836,
      "activations/layer22_attention_weight_max": 40.98809814453125,
      "activations/layer22_attention_weight_min": -29.01890754699707,
      "activations/layer23_attention_weight_max": 33.069129943847656,
      "activations/layer23_attention_weight_min": -24.690475463867188,
      "activations/layer2_attention_weight_max": 33.60203552246094,
      "activations/layer2_attention_weight_min": -34.2208251953125,
      "activations/layer3_attention_weight_max": 93.46749877929688,
      "activations/layer3_attention_weight_min": -99.99687957763672,
      "activations/layer4_attention_weight_max": 118.5698471069336,
      "activations/layer4_attention_weight_min": -118.00479888916016,
      "activations/layer5_attention_weight_max": 53.61783981323242,
      "activations/layer5_attention_weight_min": -64.02196502685547,
      "activations/layer6_attention_weight_max": 47.730899810791016,
      "activations/layer6_attention_weight_min": -48.361854553222656,
      "activations/layer7_attention_weight_max": 89.10327911376953,
      "activations/layer7_attention_weight_min": -87.71138000488281,
      "activations/layer8_attention_weight_max": 40.88050079345703,
      "activations/layer8_attention_weight_min": -39.6080207824707,
      "activations/layer9_attention_weight_max": 34.4193000793457,
      "activations/layer9_attention_weight_min": -31.806869506835938,
      "epoch": 9.31,
      "learning_rate": 9.084848484848484e-05,
      "loss": 2.7925,
      "step": 160300
    },
    {
      "activations/layer0_attention_weight_max": 14.27440071105957,
      "activations/layer0_attention_weight_min": -12.042706489562988,
      "activations/layer10_attention_weight_max": 30.000370025634766,
      "activations/layer10_attention_weight_min": -31.639999389648438,
      "activations/layer11_attention_weight_max": 32.86976623535156,
      "activations/layer11_attention_weight_min": -32.073402404785156,
      "activations/layer12_attention_weight_max": 22.60685157775879,
      "activations/layer12_attention_weight_min": -25.565147399902344,
      "activations/layer13_attention_weight_max": 35.05854415893555,
      "activations/layer13_attention_weight_min": -32.58470916748047,
      "activations/layer14_attention_weight_max": 52.82962417602539,
      "activations/layer14_attention_weight_min": -38.01526641845703,
      "activations/layer15_attention_weight_max": 35.34943389892578,
      "activations/layer15_attention_weight_min": -30.325037002563477,
      "activations/layer16_attention_weight_max": 36.11064910888672,
      "activations/layer16_attention_weight_min": -28.41421127319336,
      "activations/layer17_attention_weight_max": 31.67066192626953,
      "activations/layer17_attention_weight_min": -25.103328704833984,
      "activations/layer18_attention_weight_max": 30.376094818115234,
      "activations/layer18_attention_weight_min": -24.877500534057617,
      "activations/layer19_attention_weight_max": 41.1555290222168,
      "activations/layer19_attention_weight_min": -31.460840225219727,
      "activations/layer1_attention_weight_max": 16.492446899414062,
      "activations/layer1_attention_weight_min": -13.551844596862793,
      "activations/layer20_attention_weight_max": 34.85111618041992,
      "activations/layer20_attention_weight_min": -26.727705001831055,
      "activations/layer21_attention_weight_max": 30.485776901245117,
      "activations/layer21_attention_weight_min": -22.257347106933594,
      "activations/layer22_attention_weight_max": 44.167625427246094,
      "activations/layer22_attention_weight_min": -29.972530364990234,
      "activations/layer23_attention_weight_max": 37.32554626464844,
      "activations/layer23_attention_weight_min": -23.63239097595215,
      "activations/layer2_attention_weight_max": 32.83320617675781,
      "activations/layer2_attention_weight_min": -33.462890625,
      "activations/layer3_attention_weight_max": 95.35863494873047,
      "activations/layer3_attention_weight_min": -98.34197998046875,
      "activations/layer4_attention_weight_max": 117.3476791381836,
      "activations/layer4_attention_weight_min": -111.6971664428711,
      "activations/layer5_attention_weight_max": 57.280879974365234,
      "activations/layer5_attention_weight_min": -57.68143844604492,
      "activations/layer6_attention_weight_max": 46.89905548095703,
      "activations/layer6_attention_weight_min": -51.69248962402344,
      "activations/layer7_attention_weight_max": 92.72173309326172,
      "activations/layer7_attention_weight_min": -93.77447509765625,
      "activations/layer8_attention_weight_max": 38.94987487792969,
      "activations/layer8_attention_weight_min": -41.99783706665039,
      "activations/layer9_attention_weight_max": 32.69161605834961,
      "activations/layer9_attention_weight_min": -34.45610427856445,
      "epoch": 9.32,
      "learning_rate": 9.082992424242423e-05,
      "loss": 2.7974,
      "step": 160350
    },
    {
      "activations/layer0_attention_weight_max": 15.26314926147461,
      "activations/layer0_attention_weight_min": -12.830430030822754,
      "activations/layer10_attention_weight_max": 31.876285552978516,
      "activations/layer10_attention_weight_min": -32.226890563964844,
      "activations/layer11_attention_weight_max": 31.032272338867188,
      "activations/layer11_attention_weight_min": -30.588180541992188,
      "activations/layer12_attention_weight_max": 24.05560302734375,
      "activations/layer12_attention_weight_min": -26.965713500976562,
      "activations/layer13_attention_weight_max": 45.272438049316406,
      "activations/layer13_attention_weight_min": -32.387535095214844,
      "activations/layer14_attention_weight_max": 44.295352935791016,
      "activations/layer14_attention_weight_min": -36.50075912475586,
      "activations/layer15_attention_weight_max": 37.257110595703125,
      "activations/layer15_attention_weight_min": -32.24910354614258,
      "activations/layer16_attention_weight_max": 33.081478118896484,
      "activations/layer16_attention_weight_min": -29.76667022705078,
      "activations/layer17_attention_weight_max": 33.258602142333984,
      "activations/layer17_attention_weight_min": -26.12993049621582,
      "activations/layer18_attention_weight_max": 30.253822326660156,
      "activations/layer18_attention_weight_min": -23.928247451782227,
      "activations/layer19_attention_weight_max": 43.38977813720703,
      "activations/layer19_attention_weight_min": -34.0264778137207,
      "activations/layer1_attention_weight_max": 18.570783615112305,
      "activations/layer1_attention_weight_min": -17.146230697631836,
      "activations/layer20_attention_weight_max": 34.504722595214844,
      "activations/layer20_attention_weight_min": -27.361459732055664,
      "activations/layer21_attention_weight_max": 36.05937576293945,
      "activations/layer21_attention_weight_min": -24.552934646606445,
      "activations/layer22_attention_weight_max": 49.2746696472168,
      "activations/layer22_attention_weight_min": -31.038991928100586,
      "activations/layer23_attention_weight_max": 40.35664367675781,
      "activations/layer23_attention_weight_min": -26.44184684753418,
      "activations/layer2_attention_weight_max": 32.209388732910156,
      "activations/layer2_attention_weight_min": -32.3070068359375,
      "activations/layer3_attention_weight_max": 94.80989074707031,
      "activations/layer3_attention_weight_min": -98.38358306884766,
      "activations/layer4_attention_weight_max": 123.24732971191406,
      "activations/layer4_attention_weight_min": -117.8062744140625,
      "activations/layer5_attention_weight_max": 55.53860092163086,
      "activations/layer5_attention_weight_min": -63.76799011230469,
      "activations/layer6_attention_weight_max": 47.80124282836914,
      "activations/layer6_attention_weight_min": -53.43696212768555,
      "activations/layer7_attention_weight_max": 94.4392318725586,
      "activations/layer7_attention_weight_min": -93.31188201904297,
      "activations/layer8_attention_weight_max": 40.373046875,
      "activations/layer8_attention_weight_min": -41.62437438964844,
      "activations/layer9_attention_weight_max": 34.829376220703125,
      "activations/layer9_attention_weight_min": -33.07148742675781,
      "epoch": 9.32,
      "learning_rate": 9.081098484848485e-05,
      "loss": 2.8044,
      "step": 160400
    },
    {
      "activations/layer0_attention_weight_max": 16.083208084106445,
      "activations/layer0_attention_weight_min": -12.927783966064453,
      "activations/layer10_attention_weight_max": 32.0804557800293,
      "activations/layer10_attention_weight_min": -33.16451644897461,
      "activations/layer11_attention_weight_max": 34.28224182128906,
      "activations/layer11_attention_weight_min": -31.529964447021484,
      "activations/layer12_attention_weight_max": 22.94352149963379,
      "activations/layer12_attention_weight_min": -32.43269348144531,
      "activations/layer13_attention_weight_max": 38.461509704589844,
      "activations/layer13_attention_weight_min": -32.924583435058594,
      "activations/layer14_attention_weight_max": 47.59375,
      "activations/layer14_attention_weight_min": -37.528846740722656,
      "activations/layer15_attention_weight_max": 35.67460632324219,
      "activations/layer15_attention_weight_min": -29.781925201416016,
      "activations/layer16_attention_weight_max": 32.52056884765625,
      "activations/layer16_attention_weight_min": -28.127334594726562,
      "activations/layer17_attention_weight_max": 30.565778732299805,
      "activations/layer17_attention_weight_min": -26.24155044555664,
      "activations/layer18_attention_weight_max": 29.5390625,
      "activations/layer18_attention_weight_min": -21.890884399414062,
      "activations/layer19_attention_weight_max": 32.70049285888672,
      "activations/layer19_attention_weight_min": -33.63164138793945,
      "activations/layer1_attention_weight_max": 17.06361961364746,
      "activations/layer1_attention_weight_min": -15.237459182739258,
      "activations/layer20_attention_weight_max": 29.242942810058594,
      "activations/layer20_attention_weight_min": -25.677715301513672,
      "activations/layer21_attention_weight_max": 27.457229614257812,
      "activations/layer21_attention_weight_min": -24.812847137451172,
      "activations/layer22_attention_weight_max": 44.3165283203125,
      "activations/layer22_attention_weight_min": -30.773517608642578,
      "activations/layer23_attention_weight_max": 34.998085021972656,
      "activations/layer23_attention_weight_min": -27.325393676757812,
      "activations/layer2_attention_weight_max": 33.425209045410156,
      "activations/layer2_attention_weight_min": -32.12565612792969,
      "activations/layer3_attention_weight_max": 90.83837127685547,
      "activations/layer3_attention_weight_min": -92.29006958007812,
      "activations/layer4_attention_weight_max": 112.58739471435547,
      "activations/layer4_attention_weight_min": -114.05333709716797,
      "activations/layer5_attention_weight_max": 58.80238342285156,
      "activations/layer5_attention_weight_min": -62.130836486816406,
      "activations/layer6_attention_weight_max": 47.29609298706055,
      "activations/layer6_attention_weight_min": -50.56863021850586,
      "activations/layer7_attention_weight_max": 86.28203582763672,
      "activations/layer7_attention_weight_min": -86.34237670898438,
      "activations/layer8_attention_weight_max": 39.74122619628906,
      "activations/layer8_attention_weight_min": -40.36473846435547,
      "activations/layer9_attention_weight_max": 33.76744842529297,
      "activations/layer9_attention_weight_min": -33.901668548583984,
      "epoch": 9.32,
      "learning_rate": 9.079242424242424e-05,
      "loss": 2.7834,
      "step": 160450
    },
    {
      "activations/layer0_attention_weight_max": 16.529006958007812,
      "activations/layer0_attention_weight_min": -13.059693336486816,
      "activations/layer10_attention_weight_max": 34.617897033691406,
      "activations/layer10_attention_weight_min": -32.25074005126953,
      "activations/layer11_attention_weight_max": 34.562721252441406,
      "activations/layer11_attention_weight_min": -30.691953659057617,
      "activations/layer12_attention_weight_max": 23.19499969482422,
      "activations/layer12_attention_weight_min": -24.554176330566406,
      "activations/layer13_attention_weight_max": 36.895294189453125,
      "activations/layer13_attention_weight_min": -33.630767822265625,
      "activations/layer14_attention_weight_max": 42.39796829223633,
      "activations/layer14_attention_weight_min": -37.109703063964844,
      "activations/layer15_attention_weight_max": 34.37732696533203,
      "activations/layer15_attention_weight_min": -31.05449867248535,
      "activations/layer16_attention_weight_max": 30.631547927856445,
      "activations/layer16_attention_weight_min": -28.508813858032227,
      "activations/layer17_attention_weight_max": 33.63093948364258,
      "activations/layer17_attention_weight_min": -27.443452835083008,
      "activations/layer18_attention_weight_max": 29.99347496032715,
      "activations/layer18_attention_weight_min": -23.20943260192871,
      "activations/layer19_attention_weight_max": 32.942543029785156,
      "activations/layer19_attention_weight_min": -32.0081901550293,
      "activations/layer1_attention_weight_max": 18.280763626098633,
      "activations/layer1_attention_weight_min": -16.450355529785156,
      "activations/layer20_attention_weight_max": 31.5939884185791,
      "activations/layer20_attention_weight_min": -25.160860061645508,
      "activations/layer21_attention_weight_max": 32.09058380126953,
      "activations/layer21_attention_weight_min": -23.533885955810547,
      "activations/layer22_attention_weight_max": 43.26824188232422,
      "activations/layer22_attention_weight_min": -28.9578914642334,
      "activations/layer23_attention_weight_max": 33.652374267578125,
      "activations/layer23_attention_weight_min": -24.12813377380371,
      "activations/layer2_attention_weight_max": 30.264331817626953,
      "activations/layer2_attention_weight_min": -29.75601577758789,
      "activations/layer3_attention_weight_max": 91.35489654541016,
      "activations/layer3_attention_weight_min": -89.18030548095703,
      "activations/layer4_attention_weight_max": 115.64869689941406,
      "activations/layer4_attention_weight_min": -115.8222885131836,
      "activations/layer5_attention_weight_max": 53.43914031982422,
      "activations/layer5_attention_weight_min": -63.56395721435547,
      "activations/layer6_attention_weight_max": 48.23727035522461,
      "activations/layer6_attention_weight_min": -51.39728927612305,
      "activations/layer7_attention_weight_max": 90.94734191894531,
      "activations/layer7_attention_weight_min": -91.79641723632812,
      "activations/layer8_attention_weight_max": 40.594886779785156,
      "activations/layer8_attention_weight_min": -39.61003112792969,
      "activations/layer9_attention_weight_max": 34.39105224609375,
      "activations/layer9_attention_weight_min": -33.21812057495117,
      "epoch": 9.33,
      "learning_rate": 9.077348484848484e-05,
      "loss": 2.7974,
      "step": 160500
    },
    {
      "activations/layer0_attention_weight_max": 16.21019172668457,
      "activations/layer0_attention_weight_min": -12.535839080810547,
      "activations/layer10_attention_weight_max": 32.28675842285156,
      "activations/layer10_attention_weight_min": -36.006832122802734,
      "activations/layer11_attention_weight_max": 31.92425537109375,
      "activations/layer11_attention_weight_min": -33.75206756591797,
      "activations/layer12_attention_weight_max": 23.3817195892334,
      "activations/layer12_attention_weight_min": -24.478485107421875,
      "activations/layer13_attention_weight_max": 38.84025955200195,
      "activations/layer13_attention_weight_min": -34.19157409667969,
      "activations/layer14_attention_weight_max": 45.17349624633789,
      "activations/layer14_attention_weight_min": -37.685646057128906,
      "activations/layer15_attention_weight_max": 35.53337860107422,
      "activations/layer15_attention_weight_min": -29.768938064575195,
      "activations/layer16_attention_weight_max": 32.465213775634766,
      "activations/layer16_attention_weight_min": -26.25875473022461,
      "activations/layer17_attention_weight_max": 35.10653305053711,
      "activations/layer17_attention_weight_min": -24.805835723876953,
      "activations/layer18_attention_weight_max": 34.778724670410156,
      "activations/layer18_attention_weight_min": -23.24285316467285,
      "activations/layer19_attention_weight_max": 36.49251937866211,
      "activations/layer19_attention_weight_min": -32.60579299926758,
      "activations/layer1_attention_weight_max": 17.256187438964844,
      "activations/layer1_attention_weight_min": -14.871979713439941,
      "activations/layer20_attention_weight_max": 30.660892486572266,
      "activations/layer20_attention_weight_min": -25.243131637573242,
      "activations/layer21_attention_weight_max": 30.103302001953125,
      "activations/layer21_attention_weight_min": -22.866104125976562,
      "activations/layer22_attention_weight_max": 46.24243927001953,
      "activations/layer22_attention_weight_min": -28.103235244750977,
      "activations/layer23_attention_weight_max": 37.91057205200195,
      "activations/layer23_attention_weight_min": -24.168315887451172,
      "activations/layer2_attention_weight_max": 32.07142639160156,
      "activations/layer2_attention_weight_min": -32.33810043334961,
      "activations/layer3_attention_weight_max": 93.29067993164062,
      "activations/layer3_attention_weight_min": -97.91130828857422,
      "activations/layer4_attention_weight_max": 113.925537109375,
      "activations/layer4_attention_weight_min": -112.52410125732422,
      "activations/layer5_attention_weight_max": 54.12787628173828,
      "activations/layer5_attention_weight_min": -60.476131439208984,
      "activations/layer6_attention_weight_max": 48.206905364990234,
      "activations/layer6_attention_weight_min": -47.96138000488281,
      "activations/layer7_attention_weight_max": 88.47579956054688,
      "activations/layer7_attention_weight_min": -98.8053970336914,
      "activations/layer8_attention_weight_max": 40.78523254394531,
      "activations/layer8_attention_weight_min": -43.86149978637695,
      "activations/layer9_attention_weight_max": 33.28853225708008,
      "activations/layer9_attention_weight_min": -35.27072525024414,
      "epoch": 9.33,
      "learning_rate": 9.075454545454544e-05,
      "loss": 2.788,
      "step": 160550
    },
    {
      "activations/layer0_attention_weight_max": 16.033750534057617,
      "activations/layer0_attention_weight_min": -11.573537826538086,
      "activations/layer10_attention_weight_max": 30.71695327758789,
      "activations/layer10_attention_weight_min": -29.96653175354004,
      "activations/layer11_attention_weight_max": 31.215085983276367,
      "activations/layer11_attention_weight_min": -29.113670349121094,
      "activations/layer12_attention_weight_max": 22.275344848632812,
      "activations/layer12_attention_weight_min": -25.98029327392578,
      "activations/layer13_attention_weight_max": 34.04543685913086,
      "activations/layer13_attention_weight_min": -33.21651077270508,
      "activations/layer14_attention_weight_max": 40.4728889465332,
      "activations/layer14_attention_weight_min": -36.53049087524414,
      "activations/layer15_attention_weight_max": 35.38698959350586,
      "activations/layer15_attention_weight_min": -30.908817291259766,
      "activations/layer16_attention_weight_max": 29.619497299194336,
      "activations/layer16_attention_weight_min": -28.75420570373535,
      "activations/layer17_attention_weight_max": 32.40458679199219,
      "activations/layer17_attention_weight_min": -26.822429656982422,
      "activations/layer18_attention_weight_max": 36.61368179321289,
      "activations/layer18_attention_weight_min": -23.527713775634766,
      "activations/layer19_attention_weight_max": 32.71446990966797,
      "activations/layer19_attention_weight_min": -30.895435333251953,
      "activations/layer1_attention_weight_max": 16.509946823120117,
      "activations/layer1_attention_weight_min": -14.257342338562012,
      "activations/layer20_attention_weight_max": 27.006040573120117,
      "activations/layer20_attention_weight_min": -25.165727615356445,
      "activations/layer21_attention_weight_max": 30.872665405273438,
      "activations/layer21_attention_weight_min": -23.677745819091797,
      "activations/layer22_attention_weight_max": 43.30561065673828,
      "activations/layer22_attention_weight_min": -29.1065731048584,
      "activations/layer23_attention_weight_max": 33.30755615234375,
      "activations/layer23_attention_weight_min": -22.508522033691406,
      "activations/layer2_attention_weight_max": 32.50975036621094,
      "activations/layer2_attention_weight_min": -31.01990509033203,
      "activations/layer3_attention_weight_max": 95.47623443603516,
      "activations/layer3_attention_weight_min": -96.76936340332031,
      "activations/layer4_attention_weight_max": 118.0834732055664,
      "activations/layer4_attention_weight_min": -114.40764617919922,
      "activations/layer5_attention_weight_max": 54.718074798583984,
      "activations/layer5_attention_weight_min": -60.78199768066406,
      "activations/layer6_attention_weight_max": 46.64995574951172,
      "activations/layer6_attention_weight_min": -48.41447448730469,
      "activations/layer7_attention_weight_max": 83.85979461669922,
      "activations/layer7_attention_weight_min": -85.85480499267578,
      "activations/layer8_attention_weight_max": 39.53933334350586,
      "activations/layer8_attention_weight_min": -39.83160400390625,
      "activations/layer9_attention_weight_max": 33.64431381225586,
      "activations/layer9_attention_weight_min": -32.77956771850586,
      "epoch": 9.33,
      "learning_rate": 9.073560606060606e-05,
      "loss": 2.8125,
      "step": 160600
    },
    {
      "activations/layer0_attention_weight_max": 15.366174697875977,
      "activations/layer0_attention_weight_min": -12.489073753356934,
      "activations/layer10_attention_weight_max": 32.528480529785156,
      "activations/layer10_attention_weight_min": -32.516693115234375,
      "activations/layer11_attention_weight_max": 34.219207763671875,
      "activations/layer11_attention_weight_min": -32.28069305419922,
      "activations/layer12_attention_weight_max": 23.266666412353516,
      "activations/layer12_attention_weight_min": -24.94451332092285,
      "activations/layer13_attention_weight_max": 38.614253997802734,
      "activations/layer13_attention_weight_min": -34.68492889404297,
      "activations/layer14_attention_weight_max": 44.78700256347656,
      "activations/layer14_attention_weight_min": -38.27183532714844,
      "activations/layer15_attention_weight_max": 37.59315872192383,
      "activations/layer15_attention_weight_min": -31.695127487182617,
      "activations/layer16_attention_weight_max": 32.56125259399414,
      "activations/layer16_attention_weight_min": -28.03254508972168,
      "activations/layer17_attention_weight_max": 35.96403884887695,
      "activations/layer17_attention_weight_min": -25.47333335876465,
      "activations/layer18_attention_weight_max": 34.44627380371094,
      "activations/layer18_attention_weight_min": -22.030372619628906,
      "activations/layer19_attention_weight_max": 40.810157775878906,
      "activations/layer19_attention_weight_min": -30.090097427368164,
      "activations/layer1_attention_weight_max": 16.593063354492188,
      "activations/layer1_attention_weight_min": -14.265203475952148,
      "activations/layer20_attention_weight_max": 34.58892059326172,
      "activations/layer20_attention_weight_min": -26.320444107055664,
      "activations/layer21_attention_weight_max": 32.78254699707031,
      "activations/layer21_attention_weight_min": -22.691259384155273,
      "activations/layer22_attention_weight_max": 49.6118049621582,
      "activations/layer22_attention_weight_min": -27.66944122314453,
      "activations/layer23_attention_weight_max": 38.77210998535156,
      "activations/layer23_attention_weight_min": -22.50835418701172,
      "activations/layer2_attention_weight_max": 32.79505157470703,
      "activations/layer2_attention_weight_min": -33.3186149597168,
      "activations/layer3_attention_weight_max": 93.17686462402344,
      "activations/layer3_attention_weight_min": -94.68280029296875,
      "activations/layer4_attention_weight_max": 113.17280578613281,
      "activations/layer4_attention_weight_min": -111.45130920410156,
      "activations/layer5_attention_weight_max": 51.86531066894531,
      "activations/layer5_attention_weight_min": -62.03449249267578,
      "activations/layer6_attention_weight_max": 46.568721771240234,
      "activations/layer6_attention_weight_min": -50.47836685180664,
      "activations/layer7_attention_weight_max": 91.97916412353516,
      "activations/layer7_attention_weight_min": -92.3698501586914,
      "activations/layer8_attention_weight_max": 40.911251068115234,
      "activations/layer8_attention_weight_min": -41.19287872314453,
      "activations/layer9_attention_weight_max": 33.89858627319336,
      "activations/layer9_attention_weight_min": -32.28373336791992,
      "epoch": 9.33,
      "learning_rate": 9.071666666666666e-05,
      "loss": 2.7946,
      "step": 160650
    },
    {
      "activations/layer0_attention_weight_max": 15.167837142944336,
      "activations/layer0_attention_weight_min": -12.529330253601074,
      "activations/layer10_attention_weight_max": 31.2739315032959,
      "activations/layer10_attention_weight_min": -32.774173736572266,
      "activations/layer11_attention_weight_max": 29.552183151245117,
      "activations/layer11_attention_weight_min": -30.201011657714844,
      "activations/layer12_attention_weight_max": 23.37424087524414,
      "activations/layer12_attention_weight_min": -23.347347259521484,
      "activations/layer13_attention_weight_max": 36.73664855957031,
      "activations/layer13_attention_weight_min": -31.01024627685547,
      "activations/layer14_attention_weight_max": 45.75429916381836,
      "activations/layer14_attention_weight_min": -39.319732666015625,
      "activations/layer15_attention_weight_max": 34.246124267578125,
      "activations/layer15_attention_weight_min": -29.612974166870117,
      "activations/layer16_attention_weight_max": 28.866758346557617,
      "activations/layer16_attention_weight_min": -26.230852127075195,
      "activations/layer17_attention_weight_max": 33.77098083496094,
      "activations/layer17_attention_weight_min": -23.556161880493164,
      "activations/layer18_attention_weight_max": 36.45915985107422,
      "activations/layer18_attention_weight_min": -20.802227020263672,
      "activations/layer19_attention_weight_max": 36.326690673828125,
      "activations/layer19_attention_weight_min": -30.89670753479004,
      "activations/layer1_attention_weight_max": 17.441858291625977,
      "activations/layer1_attention_weight_min": -13.815070152282715,
      "activations/layer20_attention_weight_max": 32.990352630615234,
      "activations/layer20_attention_weight_min": -24.159568786621094,
      "activations/layer21_attention_weight_max": 32.084712982177734,
      "activations/layer21_attention_weight_min": -23.13606834411621,
      "activations/layer22_attention_weight_max": 43.59294891357422,
      "activations/layer22_attention_weight_min": -27.48041534423828,
      "activations/layer23_attention_weight_max": 33.29098129272461,
      "activations/layer23_attention_weight_min": -23.98589324951172,
      "activations/layer2_attention_weight_max": 31.289813995361328,
      "activations/layer2_attention_weight_min": -32.72722625732422,
      "activations/layer3_attention_weight_max": 86.39181518554688,
      "activations/layer3_attention_weight_min": -91.1481704711914,
      "activations/layer4_attention_weight_max": 109.25587463378906,
      "activations/layer4_attention_weight_min": -113.83768463134766,
      "activations/layer5_attention_weight_max": 51.632442474365234,
      "activations/layer5_attention_weight_min": -61.05158233642578,
      "activations/layer6_attention_weight_max": 44.57758331298828,
      "activations/layer6_attention_weight_min": -47.638248443603516,
      "activations/layer7_attention_weight_max": 81.26361846923828,
      "activations/layer7_attention_weight_min": -83.3936996459961,
      "activations/layer8_attention_weight_max": 39.78456115722656,
      "activations/layer8_attention_weight_min": -39.6475830078125,
      "activations/layer9_attention_weight_max": 32.42043685913086,
      "activations/layer9_attention_weight_min": -32.33283615112305,
      "epoch": 9.34,
      "learning_rate": 9.069772727272726e-05,
      "loss": 2.7843,
      "step": 160700
    },
    {
      "activations/layer0_attention_weight_max": 16.002132415771484,
      "activations/layer0_attention_weight_min": -12.185081481933594,
      "activations/layer10_attention_weight_max": 41.23879623413086,
      "activations/layer10_attention_weight_min": -35.66029357910156,
      "activations/layer11_attention_weight_max": 40.711551666259766,
      "activations/layer11_attention_weight_min": -33.568199157714844,
      "activations/layer12_attention_weight_max": 27.926132202148438,
      "activations/layer12_attention_weight_min": -27.633625030517578,
      "activations/layer13_attention_weight_max": 44.13678741455078,
      "activations/layer13_attention_weight_min": -33.385101318359375,
      "activations/layer14_attention_weight_max": 63.84486770629883,
      "activations/layer14_attention_weight_min": -40.82713317871094,
      "activations/layer15_attention_weight_max": 41.01139831542969,
      "activations/layer15_attention_weight_min": -29.9752254486084,
      "activations/layer16_attention_weight_max": 39.96569061279297,
      "activations/layer16_attention_weight_min": -27.275053024291992,
      "activations/layer17_attention_weight_max": 49.046974182128906,
      "activations/layer17_attention_weight_min": -27.0234432220459,
      "activations/layer18_attention_weight_max": 44.1107063293457,
      "activations/layer18_attention_weight_min": -25.051326751708984,
      "activations/layer19_attention_weight_max": 46.12723922729492,
      "activations/layer19_attention_weight_min": -29.161457061767578,
      "activations/layer1_attention_weight_max": 17.445091247558594,
      "activations/layer1_attention_weight_min": -14.403212547302246,
      "activations/layer20_attention_weight_max": 40.42734909057617,
      "activations/layer20_attention_weight_min": -22.85975456237793,
      "activations/layer21_attention_weight_max": 38.731483459472656,
      "activations/layer21_attention_weight_min": -21.852685928344727,
      "activations/layer22_attention_weight_max": 66.4017105102539,
      "activations/layer22_attention_weight_min": -28.309743881225586,
      "activations/layer23_attention_weight_max": 46.0383415222168,
      "activations/layer23_attention_weight_min": -22.89924430847168,
      "activations/layer2_attention_weight_max": 34.46070098876953,
      "activations/layer2_attention_weight_min": -33.72091293334961,
      "activations/layer3_attention_weight_max": 90.72722625732422,
      "activations/layer3_attention_weight_min": -92.4046401977539,
      "activations/layer4_attention_weight_max": 112.98870849609375,
      "activations/layer4_attention_weight_min": -110.64800262451172,
      "activations/layer5_attention_weight_max": 53.65106964111328,
      "activations/layer5_attention_weight_min": -60.11659240722656,
      "activations/layer6_attention_weight_max": 48.00370407104492,
      "activations/layer6_attention_weight_min": -52.030860900878906,
      "activations/layer7_attention_weight_max": 91.02006530761719,
      "activations/layer7_attention_weight_min": -88.49760437011719,
      "activations/layer8_attention_weight_max": 44.367618560791016,
      "activations/layer8_attention_weight_min": -44.01802444458008,
      "activations/layer9_attention_weight_max": 41.24077224731445,
      "activations/layer9_attention_weight_min": -36.86695861816406,
      "epoch": 9.34,
      "learning_rate": 9.067878787878787e-05,
      "loss": 2.7766,
      "step": 160750
    },
    {
      "activations/layer0_attention_weight_max": 16.61699867248535,
      "activations/layer0_attention_weight_min": -12.081585884094238,
      "activations/layer10_attention_weight_max": 30.49898338317871,
      "activations/layer10_attention_weight_min": -32.03666305541992,
      "activations/layer11_attention_weight_max": 29.80282211303711,
      "activations/layer11_attention_weight_min": -31.055423736572266,
      "activations/layer12_attention_weight_max": 22.646812438964844,
      "activations/layer12_attention_weight_min": -24.802383422851562,
      "activations/layer13_attention_weight_max": 37.19794464111328,
      "activations/layer13_attention_weight_min": -33.41476058959961,
      "activations/layer14_attention_weight_max": 43.72988510131836,
      "activations/layer14_attention_weight_min": -37.38918685913086,
      "activations/layer15_attention_weight_max": 34.8453483581543,
      "activations/layer15_attention_weight_min": -30.428529739379883,
      "activations/layer16_attention_weight_max": 31.59986114501953,
      "activations/layer16_attention_weight_min": -27.144025802612305,
      "activations/layer17_attention_weight_max": 30.445730209350586,
      "activations/layer17_attention_weight_min": -27.958261489868164,
      "activations/layer18_attention_weight_max": 29.78335189819336,
      "activations/layer18_attention_weight_min": -24.045373916625977,
      "activations/layer19_attention_weight_max": 34.1391716003418,
      "activations/layer19_attention_weight_min": -34.53744125366211,
      "activations/layer1_attention_weight_max": 16.380722045898438,
      "activations/layer1_attention_weight_min": -14.39321517944336,
      "activations/layer20_attention_weight_max": 29.28911590576172,
      "activations/layer20_attention_weight_min": -24.23851776123047,
      "activations/layer21_attention_weight_max": 28.457876205444336,
      "activations/layer21_attention_weight_min": -22.547801971435547,
      "activations/layer22_attention_weight_max": 49.802364349365234,
      "activations/layer22_attention_weight_min": -33.32133865356445,
      "activations/layer23_attention_weight_max": 36.317413330078125,
      "activations/layer23_attention_weight_min": -27.051984786987305,
      "activations/layer2_attention_weight_max": 32.750274658203125,
      "activations/layer2_attention_weight_min": -32.71037673950195,
      "activations/layer3_attention_weight_max": 91.95915985107422,
      "activations/layer3_attention_weight_min": -93.86497497558594,
      "activations/layer4_attention_weight_max": 111.36333465576172,
      "activations/layer4_attention_weight_min": -113.7752456665039,
      "activations/layer5_attention_weight_max": 53.20276641845703,
      "activations/layer5_attention_weight_min": -65.05348205566406,
      "activations/layer6_attention_weight_max": 45.201637268066406,
      "activations/layer6_attention_weight_min": -48.6724967956543,
      "activations/layer7_attention_weight_max": 88.0578842163086,
      "activations/layer7_attention_weight_min": -87.67073822021484,
      "activations/layer8_attention_weight_max": 38.504005432128906,
      "activations/layer8_attention_weight_min": -42.01161575317383,
      "activations/layer9_attention_weight_max": 33.36660385131836,
      "activations/layer9_attention_weight_min": -34.88053512573242,
      "epoch": 9.34,
      "learning_rate": 9.065984848484848e-05,
      "loss": 2.7939,
      "step": 160800
    },
    {
      "activations/layer0_attention_weight_max": 15.060133934020996,
      "activations/layer0_attention_weight_min": -12.203961372375488,
      "activations/layer10_attention_weight_max": 36.52470779418945,
      "activations/layer10_attention_weight_min": -31.879201889038086,
      "activations/layer11_attention_weight_max": 34.631683349609375,
      "activations/layer11_attention_weight_min": -32.1613883972168,
      "activations/layer12_attention_weight_max": 25.443103790283203,
      "activations/layer12_attention_weight_min": -25.111408233642578,
      "activations/layer13_attention_weight_max": 42.289154052734375,
      "activations/layer13_attention_weight_min": -33.00395965576172,
      "activations/layer14_attention_weight_max": 53.438209533691406,
      "activations/layer14_attention_weight_min": -39.99460220336914,
      "activations/layer15_attention_weight_max": 36.715293884277344,
      "activations/layer15_attention_weight_min": -30.716890335083008,
      "activations/layer16_attention_weight_max": 32.17587661743164,
      "activations/layer16_attention_weight_min": -28.756174087524414,
      "activations/layer17_attention_weight_max": 33.483978271484375,
      "activations/layer17_attention_weight_min": -26.111948013305664,
      "activations/layer18_attention_weight_max": 34.96967697143555,
      "activations/layer18_attention_weight_min": -21.043888092041016,
      "activations/layer19_attention_weight_max": 37.60594940185547,
      "activations/layer19_attention_weight_min": -30.093647003173828,
      "activations/layer1_attention_weight_max": 17.16290283203125,
      "activations/layer1_attention_weight_min": -14.135936737060547,
      "activations/layer20_attention_weight_max": 30.437211990356445,
      "activations/layer20_attention_weight_min": -23.650981903076172,
      "activations/layer21_attention_weight_max": 32.5114631652832,
      "activations/layer21_attention_weight_min": -22.307891845703125,
      "activations/layer22_attention_weight_max": 43.91133117675781,
      "activations/layer22_attention_weight_min": -28.60157585144043,
      "activations/layer23_attention_weight_max": 34.91563415527344,
      "activations/layer23_attention_weight_min": -23.24391746520996,
      "activations/layer2_attention_weight_max": 34.3484992980957,
      "activations/layer2_attention_weight_min": -31.27652359008789,
      "activations/layer3_attention_weight_max": 94.0503158569336,
      "activations/layer3_attention_weight_min": -94.43135833740234,
      "activations/layer4_attention_weight_max": 116.84407806396484,
      "activations/layer4_attention_weight_min": -115.91771697998047,
      "activations/layer5_attention_weight_max": 53.77955627441406,
      "activations/layer5_attention_weight_min": -67.17594909667969,
      "activations/layer6_attention_weight_max": 47.455806732177734,
      "activations/layer6_attention_weight_min": -52.23575973510742,
      "activations/layer7_attention_weight_max": 90.6080093383789,
      "activations/layer7_attention_weight_min": -94.73428344726562,
      "activations/layer8_attention_weight_max": 40.38187789916992,
      "activations/layer8_attention_weight_min": -41.882286071777344,
      "activations/layer9_attention_weight_max": 34.18466567993164,
      "activations/layer9_attention_weight_min": -35.10321044921875,
      "epoch": 9.35,
      "learning_rate": 9.064090909090909e-05,
      "loss": 2.798,
      "step": 160850
    },
    {
      "activations/layer0_attention_weight_max": 16.632892608642578,
      "activations/layer0_attention_weight_min": -11.76669979095459,
      "activations/layer10_attention_weight_max": 29.663450241088867,
      "activations/layer10_attention_weight_min": -30.62959861755371,
      "activations/layer11_attention_weight_max": 30.659446716308594,
      "activations/layer11_attention_weight_min": -28.996944427490234,
      "activations/layer12_attention_weight_max": 23.250816345214844,
      "activations/layer12_attention_weight_min": -23.85291290283203,
      "activations/layer13_attention_weight_max": 41.282470703125,
      "activations/layer13_attention_weight_min": -32.804344177246094,
      "activations/layer14_attention_weight_max": 43.940284729003906,
      "activations/layer14_attention_weight_min": -36.46734619140625,
      "activations/layer15_attention_weight_max": 33.61345672607422,
      "activations/layer15_attention_weight_min": -28.347957611083984,
      "activations/layer16_attention_weight_max": 29.962581634521484,
      "activations/layer16_attention_weight_min": -25.311172485351562,
      "activations/layer17_attention_weight_max": 29.494314193725586,
      "activations/layer17_attention_weight_min": -25.155094146728516,
      "activations/layer18_attention_weight_max": 31.837331771850586,
      "activations/layer18_attention_weight_min": -20.997604370117188,
      "activations/layer19_attention_weight_max": 32.7725715637207,
      "activations/layer19_attention_weight_min": -29.177396774291992,
      "activations/layer1_attention_weight_max": 17.521581649780273,
      "activations/layer1_attention_weight_min": -14.199522972106934,
      "activations/layer20_attention_weight_max": 28.272045135498047,
      "activations/layer20_attention_weight_min": -23.768404006958008,
      "activations/layer21_attention_weight_max": 25.657115936279297,
      "activations/layer21_attention_weight_min": -22.275623321533203,
      "activations/layer22_attention_weight_max": 41.619911193847656,
      "activations/layer22_attention_weight_min": -28.43407440185547,
      "activations/layer23_attention_weight_max": 35.51226806640625,
      "activations/layer23_attention_weight_min": -22.422950744628906,
      "activations/layer2_attention_weight_max": 32.040672302246094,
      "activations/layer2_attention_weight_min": -31.631473541259766,
      "activations/layer3_attention_weight_max": 91.99284362792969,
      "activations/layer3_attention_weight_min": -92.19190216064453,
      "activations/layer4_attention_weight_max": 111.84415435791016,
      "activations/layer4_attention_weight_min": -110.27678680419922,
      "activations/layer5_attention_weight_max": 51.778533935546875,
      "activations/layer5_attention_weight_min": -61.43339157104492,
      "activations/layer6_attention_weight_max": 44.75765609741211,
      "activations/layer6_attention_weight_min": -48.982784271240234,
      "activations/layer7_attention_weight_max": 85.44451904296875,
      "activations/layer7_attention_weight_min": -85.923095703125,
      "activations/layer8_attention_weight_max": 41.567813873291016,
      "activations/layer8_attention_weight_min": -40.62856674194336,
      "activations/layer9_attention_weight_max": 33.18099594116211,
      "activations/layer9_attention_weight_min": -31.4445743560791,
      "epoch": 9.35,
      "learning_rate": 9.062196969696968e-05,
      "loss": 2.7988,
      "step": 160900
    },
    {
      "activations/layer0_attention_weight_max": 16.400976181030273,
      "activations/layer0_attention_weight_min": -12.337423324584961,
      "activations/layer10_attention_weight_max": 34.74650955200195,
      "activations/layer10_attention_weight_min": -33.73337173461914,
      "activations/layer11_attention_weight_max": 34.60356140136719,
      "activations/layer11_attention_weight_min": -31.666044235229492,
      "activations/layer12_attention_weight_max": 30.733060836791992,
      "activations/layer12_attention_weight_min": -27.77668571472168,
      "activations/layer13_attention_weight_max": 37.76250457763672,
      "activations/layer13_attention_weight_min": -32.62726974487305,
      "activations/layer14_attention_weight_max": 42.61915588378906,
      "activations/layer14_attention_weight_min": -38.031734466552734,
      "activations/layer15_attention_weight_max": 36.42597579956055,
      "activations/layer15_attention_weight_min": -32.531646728515625,
      "activations/layer16_attention_weight_max": 31.38975715637207,
      "activations/layer16_attention_weight_min": -27.56566047668457,
      "activations/layer17_attention_weight_max": 41.32109451293945,
      "activations/layer17_attention_weight_min": -27.57572364807129,
      "activations/layer18_attention_weight_max": 38.112701416015625,
      "activations/layer18_attention_weight_min": -22.8150691986084,
      "activations/layer19_attention_weight_max": 37.5122184753418,
      "activations/layer19_attention_weight_min": -32.91169738769531,
      "activations/layer1_attention_weight_max": 17.664884567260742,
      "activations/layer1_attention_weight_min": -13.345658302307129,
      "activations/layer20_attention_weight_max": 31.55727195739746,
      "activations/layer20_attention_weight_min": -27.330509185791016,
      "activations/layer21_attention_weight_max": 30.319089889526367,
      "activations/layer21_attention_weight_min": -25.346811294555664,
      "activations/layer22_attention_weight_max": 48.77162170410156,
      "activations/layer22_attention_weight_min": -32.25746536254883,
      "activations/layer23_attention_weight_max": 36.024234771728516,
      "activations/layer23_attention_weight_min": -24.729923248291016,
      "activations/layer2_attention_weight_max": 33.69233703613281,
      "activations/layer2_attention_weight_min": -31.112783432006836,
      "activations/layer3_attention_weight_max": 92.80763244628906,
      "activations/layer3_attention_weight_min": -91.03482818603516,
      "activations/layer4_attention_weight_max": 118.5889663696289,
      "activations/layer4_attention_weight_min": -108.03255462646484,
      "activations/layer5_attention_weight_max": 54.851356506347656,
      "activations/layer5_attention_weight_min": -60.45284652709961,
      "activations/layer6_attention_weight_max": 46.15581512451172,
      "activations/layer6_attention_weight_min": -46.26460266113281,
      "activations/layer7_attention_weight_max": 95.42218780517578,
      "activations/layer7_attention_weight_min": -86.97203826904297,
      "activations/layer8_attention_weight_max": 41.83657455444336,
      "activations/layer8_attention_weight_min": -38.68801498413086,
      "activations/layer9_attention_weight_max": 37.01333999633789,
      "activations/layer9_attention_weight_min": -34.005035400390625,
      "epoch": 9.35,
      "learning_rate": 9.06030303030303e-05,
      "loss": 2.7812,
      "step": 160950
    },
    {
      "activations/layer0_attention_weight_max": 16.038789749145508,
      "activations/layer0_attention_weight_min": -12.866789817810059,
      "activations/layer10_attention_weight_max": 33.75140380859375,
      "activations/layer10_attention_weight_min": -34.28007507324219,
      "activations/layer11_attention_weight_max": 33.16921615600586,
      "activations/layer11_attention_weight_min": -32.142967224121094,
      "activations/layer12_attention_weight_max": 26.239681243896484,
      "activations/layer12_attention_weight_min": -24.38845443725586,
      "activations/layer13_attention_weight_max": 45.37141799926758,
      "activations/layer13_attention_weight_min": -34.526023864746094,
      "activations/layer14_attention_weight_max": 55.86985778808594,
      "activations/layer14_attention_weight_min": -41.90578842163086,
      "activations/layer15_attention_weight_max": 39.09368896484375,
      "activations/layer15_attention_weight_min": -33.14937210083008,
      "activations/layer16_attention_weight_max": 34.882232666015625,
      "activations/layer16_attention_weight_min": -27.302143096923828,
      "activations/layer17_attention_weight_max": 34.5070915222168,
      "activations/layer17_attention_weight_min": -25.17758560180664,
      "activations/layer18_attention_weight_max": 31.573686599731445,
      "activations/layer18_attention_weight_min": -22.446849822998047,
      "activations/layer19_attention_weight_max": 42.62568664550781,
      "activations/layer19_attention_weight_min": -29.87489128112793,
      "activations/layer1_attention_weight_max": 16.42381477355957,
      "activations/layer1_attention_weight_min": -13.765791893005371,
      "activations/layer20_attention_weight_max": 32.797271728515625,
      "activations/layer20_attention_weight_min": -23.437084197998047,
      "activations/layer21_attention_weight_max": 29.255937576293945,
      "activations/layer21_attention_weight_min": -22.4191951751709,
      "activations/layer22_attention_weight_max": 44.17371368408203,
      "activations/layer22_attention_weight_min": -27.41268539428711,
      "activations/layer23_attention_weight_max": 40.76231002807617,
      "activations/layer23_attention_weight_min": -23.960453033447266,
      "activations/layer2_attention_weight_max": 32.07297897338867,
      "activations/layer2_attention_weight_min": -31.44906234741211,
      "activations/layer3_attention_weight_max": 90.32266235351562,
      "activations/layer3_attention_weight_min": -93.75723266601562,
      "activations/layer4_attention_weight_max": 113.68627166748047,
      "activations/layer4_attention_weight_min": -114.52809143066406,
      "activations/layer5_attention_weight_max": 54.32316207885742,
      "activations/layer5_attention_weight_min": -63.19001770019531,
      "activations/layer6_attention_weight_max": 47.225425720214844,
      "activations/layer6_attention_weight_min": -51.725242614746094,
      "activations/layer7_attention_weight_max": 90.27184295654297,
      "activations/layer7_attention_weight_min": -94.05227661132812,
      "activations/layer8_attention_weight_max": 43.1827392578125,
      "activations/layer8_attention_weight_min": -43.47304916381836,
      "activations/layer9_attention_weight_max": 36.09455490112305,
      "activations/layer9_attention_weight_min": -37.536659240722656,
      "epoch": 9.36,
      "learning_rate": 9.058409090909091e-05,
      "loss": 2.8023,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.4416,
      "eval_samples_per_second": 508.671,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.4416,
      "eval_openwebtext_samples_per_second": 508.671,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9264,
      "eval_wikitext_samples_per_second": 236.707,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_lambada_loss": 2.548828125,
      "eval_lambada_ppl": 12.792104248423877,
      "eval_lambada_runtime": 9.5121,
      "eval_lambada_samples_per_second": 511.875,
      "step": 161000
    },
    {
      "activations/layer0_attention_weight_max": 17.08660316467285,
      "activations/layer0_attention_weight_min": -12.417984008789062,
      "activations/layer10_attention_weight_max": 31.21255874633789,
      "activations/layer10_attention_weight_min": -30.78081512451172,
      "activations/layer11_attention_weight_max": 34.03746032714844,
      "activations/layer11_attention_weight_min": -32.50083923339844,
      "activations/layer12_attention_weight_max": 41.68578338623047,
      "activations/layer12_attention_weight_min": -27.293800354003906,
      "activations/layer13_attention_weight_max": 47.50508499145508,
      "activations/layer13_attention_weight_min": -33.535404205322266,
      "activations/layer14_attention_weight_max": 53.42082977294922,
      "activations/layer14_attention_weight_min": -37.024803161621094,
      "activations/layer15_attention_weight_max": 36.18357849121094,
      "activations/layer15_attention_weight_min": -31.125993728637695,
      "activations/layer16_attention_weight_max": 36.38718795776367,
      "activations/layer16_attention_weight_min": -27.59424591064453,
      "activations/layer17_attention_weight_max": 34.45869827270508,
      "activations/layer17_attention_weight_min": -27.332225799560547,
      "activations/layer18_attention_weight_max": 32.560699462890625,
      "activations/layer18_attention_weight_min": -24.357675552368164,
      "activations/layer19_attention_weight_max": 45.01679611206055,
      "activations/layer19_attention_weight_min": -29.131988525390625,
      "activations/layer1_attention_weight_max": 17.36028289794922,
      "activations/layer1_attention_weight_min": -13.816031455993652,
      "activations/layer20_attention_weight_max": 36.25669479370117,
      "activations/layer20_attention_weight_min": -24.14353370666504,
      "activations/layer21_attention_weight_max": 36.74472427368164,
      "activations/layer21_attention_weight_min": -24.118621826171875,
      "activations/layer22_attention_weight_max": 54.72162628173828,
      "activations/layer22_attention_weight_min": -28.46638298034668,
      "activations/layer23_attention_weight_max": 41.646453857421875,
      "activations/layer23_attention_weight_min": -23.174800872802734,
      "activations/layer2_attention_weight_max": 32.19898223876953,
      "activations/layer2_attention_weight_min": -33.88691711425781,
      "activations/layer3_attention_weight_max": 91.25386810302734,
      "activations/layer3_attention_weight_min": -90.98075866699219,
      "activations/layer4_attention_weight_max": 112.15279388427734,
      "activations/layer4_attention_weight_min": -109.76065826416016,
      "activations/layer5_attention_weight_max": 50.01792526245117,
      "activations/layer5_attention_weight_min": -59.62669372558594,
      "activations/layer6_attention_weight_max": 44.00273132324219,
      "activations/layer6_attention_weight_min": -50.33897018432617,
      "activations/layer7_attention_weight_max": 86.92501068115234,
      "activations/layer7_attention_weight_min": -85.52350616455078,
      "activations/layer8_attention_weight_max": 37.339969635009766,
      "activations/layer8_attention_weight_min": -38.347618103027344,
      "activations/layer9_attention_weight_max": 31.55409812927246,
      "activations/layer9_attention_weight_min": -32.27085494995117,
      "epoch": 9.36,
      "learning_rate": 9.05651515151515e-05,
      "loss": 2.7885,
      "step": 161050
    },
    {
      "activations/layer0_attention_weight_max": 14.630331993103027,
      "activations/layer0_attention_weight_min": -11.963370323181152,
      "activations/layer10_attention_weight_max": 34.47576904296875,
      "activations/layer10_attention_weight_min": -32.247222900390625,
      "activations/layer11_attention_weight_max": 32.136741638183594,
      "activations/layer11_attention_weight_min": -29.649503707885742,
      "activations/layer12_attention_weight_max": 42.37030029296875,
      "activations/layer12_attention_weight_min": -28.60359001159668,
      "activations/layer13_attention_weight_max": 51.10688400268555,
      "activations/layer13_attention_weight_min": -35.9146614074707,
      "activations/layer14_attention_weight_max": 44.11597442626953,
      "activations/layer14_attention_weight_min": -41.36936569213867,
      "activations/layer15_attention_weight_max": 38.06435012817383,
      "activations/layer15_attention_weight_min": -30.73780632019043,
      "activations/layer16_attention_weight_max": 34.624149322509766,
      "activations/layer16_attention_weight_min": -28.49681854248047,
      "activations/layer17_attention_weight_max": 30.15348243713379,
      "activations/layer17_attention_weight_min": -25.065479278564453,
      "activations/layer18_attention_weight_max": 31.77688217163086,
      "activations/layer18_attention_weight_min": -23.19519805908203,
      "activations/layer19_attention_weight_max": 39.19314193725586,
      "activations/layer19_attention_weight_min": -30.62879753112793,
      "activations/layer1_attention_weight_max": 17.424503326416016,
      "activations/layer1_attention_weight_min": -14.799044609069824,
      "activations/layer20_attention_weight_max": 31.085649490356445,
      "activations/layer20_attention_weight_min": -24.67632484436035,
      "activations/layer21_attention_weight_max": 32.93273162841797,
      "activations/layer21_attention_weight_min": -22.01873779296875,
      "activations/layer22_attention_weight_max": 46.87683868408203,
      "activations/layer22_attention_weight_min": -29.224817276000977,
      "activations/layer23_attention_weight_max": 43.79453659057617,
      "activations/layer23_attention_weight_min": -24.392757415771484,
      "activations/layer2_attention_weight_max": 33.12445068359375,
      "activations/layer2_attention_weight_min": -32.243160247802734,
      "activations/layer3_attention_weight_max": 92.2579345703125,
      "activations/layer3_attention_weight_min": -93.4371109008789,
      "activations/layer4_attention_weight_max": 117.15262603759766,
      "activations/layer4_attention_weight_min": -111.6552505493164,
      "activations/layer5_attention_weight_max": 57.67097091674805,
      "activations/layer5_attention_weight_min": -62.971553802490234,
      "activations/layer6_attention_weight_max": 49.38322830200195,
      "activations/layer6_attention_weight_min": -48.62662124633789,
      "activations/layer7_attention_weight_max": 90.4522933959961,
      "activations/layer7_attention_weight_min": -90.86090850830078,
      "activations/layer8_attention_weight_max": 41.00358963012695,
      "activations/layer8_attention_weight_min": -41.530242919921875,
      "activations/layer9_attention_weight_max": 35.44319152832031,
      "activations/layer9_attention_weight_min": -34.8387565612793,
      "epoch": 9.36,
      "learning_rate": 9.054621212121211e-05,
      "loss": 2.7873,
      "step": 161100
    },
    {
      "activations/layer0_attention_weight_max": 15.47468090057373,
      "activations/layer0_attention_weight_min": -11.727533340454102,
      "activations/layer10_attention_weight_max": 34.36138916015625,
      "activations/layer10_attention_weight_min": -33.95944595336914,
      "activations/layer11_attention_weight_max": 33.12131118774414,
      "activations/layer11_attention_weight_min": -32.3267822265625,
      "activations/layer12_attention_weight_max": 38.17081832885742,
      "activations/layer12_attention_weight_min": -27.739896774291992,
      "activations/layer13_attention_weight_max": 45.24930191040039,
      "activations/layer13_attention_weight_min": -37.34416580200195,
      "activations/layer14_attention_weight_max": 55.4720458984375,
      "activations/layer14_attention_weight_min": -45.200660705566406,
      "activations/layer15_attention_weight_max": 41.34453582763672,
      "activations/layer15_attention_weight_min": -36.553043365478516,
      "activations/layer16_attention_weight_max": 38.03679656982422,
      "activations/layer16_attention_weight_min": -30.20461082458496,
      "activations/layer17_attention_weight_max": 37.8751106262207,
      "activations/layer17_attention_weight_min": -30.554407119750977,
      "activations/layer18_attention_weight_max": 34.066802978515625,
      "activations/layer18_attention_weight_min": -24.255516052246094,
      "activations/layer19_attention_weight_max": 37.826473236083984,
      "activations/layer19_attention_weight_min": -30.049253463745117,
      "activations/layer1_attention_weight_max": 15.739041328430176,
      "activations/layer1_attention_weight_min": -14.199249267578125,
      "activations/layer20_attention_weight_max": 31.977155685424805,
      "activations/layer20_attention_weight_min": -25.185123443603516,
      "activations/layer21_attention_weight_max": 31.504608154296875,
      "activations/layer21_attention_weight_min": -26.06595230102539,
      "activations/layer22_attention_weight_max": 56.53247833251953,
      "activations/layer22_attention_weight_min": -29.88774299621582,
      "activations/layer23_attention_weight_max": 36.51939392089844,
      "activations/layer23_attention_weight_min": -25.108388900756836,
      "activations/layer2_attention_weight_max": 33.566856384277344,
      "activations/layer2_attention_weight_min": -31.769865036010742,
      "activations/layer3_attention_weight_max": 98.43568420410156,
      "activations/layer3_attention_weight_min": -95.37255859375,
      "activations/layer4_attention_weight_max": 121.09622955322266,
      "activations/layer4_attention_weight_min": -117.0118179321289,
      "activations/layer5_attention_weight_max": 60.09637451171875,
      "activations/layer5_attention_weight_min": -60.618404388427734,
      "activations/layer6_attention_weight_max": 52.07653045654297,
      "activations/layer6_attention_weight_min": -50.41617202758789,
      "activations/layer7_attention_weight_max": 96.1585922241211,
      "activations/layer7_attention_weight_min": -93.36479187011719,
      "activations/layer8_attention_weight_max": 42.93964385986328,
      "activations/layer8_attention_weight_min": -42.91913604736328,
      "activations/layer9_attention_weight_max": 38.005218505859375,
      "activations/layer9_attention_weight_min": -36.75050735473633,
      "epoch": 9.36,
      "learning_rate": 9.052727272727273e-05,
      "loss": 2.8139,
      "step": 161150
    },
    {
      "activations/layer0_attention_weight_max": 16.167322158813477,
      "activations/layer0_attention_weight_min": -11.57875919342041,
      "activations/layer10_attention_weight_max": 34.2802734375,
      "activations/layer10_attention_weight_min": -34.69173812866211,
      "activations/layer11_attention_weight_max": 35.08982849121094,
      "activations/layer11_attention_weight_min": -31.536128997802734,
      "activations/layer12_attention_weight_max": 25.002946853637695,
      "activations/layer12_attention_weight_min": -24.80019760131836,
      "activations/layer13_attention_weight_max": 39.31333541870117,
      "activations/layer13_attention_weight_min": -35.990760803222656,
      "activations/layer14_attention_weight_max": 50.93327331542969,
      "activations/layer14_attention_weight_min": -40.334510803222656,
      "activations/layer15_attention_weight_max": 37.61697769165039,
      "activations/layer15_attention_weight_min": -33.73621368408203,
      "activations/layer16_attention_weight_max": 33.54146194458008,
      "activations/layer16_attention_weight_min": -27.77595329284668,
      "activations/layer17_attention_weight_max": 32.0650634765625,
      "activations/layer17_attention_weight_min": -26.22701072692871,
      "activations/layer18_attention_weight_max": 30.390399932861328,
      "activations/layer18_attention_weight_min": -22.807573318481445,
      "activations/layer19_attention_weight_max": 37.735870361328125,
      "activations/layer19_attention_weight_min": -32.24544906616211,
      "activations/layer1_attention_weight_max": 16.50732421875,
      "activations/layer1_attention_weight_min": -15.410449028015137,
      "activations/layer20_attention_weight_max": 29.2435245513916,
      "activations/layer20_attention_weight_min": -24.515541076660156,
      "activations/layer21_attention_weight_max": 27.98757553100586,
      "activations/layer21_attention_weight_min": -22.626842498779297,
      "activations/layer22_attention_weight_max": 49.778465270996094,
      "activations/layer22_attention_weight_min": -29.76802635192871,
      "activations/layer23_attention_weight_max": 38.269691467285156,
      "activations/layer23_attention_weight_min": -25.065990447998047,
      "activations/layer2_attention_weight_max": 32.12367248535156,
      "activations/layer2_attention_weight_min": -32.15218734741211,
      "activations/layer3_attention_weight_max": 94.77446746826172,
      "activations/layer3_attention_weight_min": -98.99446105957031,
      "activations/layer4_attention_weight_max": 115.31693267822266,
      "activations/layer4_attention_weight_min": -116.59532928466797,
      "activations/layer5_attention_weight_max": 56.91425704956055,
      "activations/layer5_attention_weight_min": -61.925148010253906,
      "activations/layer6_attention_weight_max": 48.43384552001953,
      "activations/layer6_attention_weight_min": -52.063133239746094,
      "activations/layer7_attention_weight_max": 93.03430938720703,
      "activations/layer7_attention_weight_min": -95.97384643554688,
      "activations/layer8_attention_weight_max": 41.6515007019043,
      "activations/layer8_attention_weight_min": -46.76693344116211,
      "activations/layer9_attention_weight_max": 37.42512893676758,
      "activations/layer9_attention_weight_min": -37.47028732299805,
      "epoch": 9.37,
      "learning_rate": 9.050833333333332e-05,
      "loss": 2.8117,
      "step": 161200
    },
    {
      "activations/layer0_attention_weight_max": 16.217041015625,
      "activations/layer0_attention_weight_min": -12.867761611938477,
      "activations/layer10_attention_weight_max": 32.58527374267578,
      "activations/layer10_attention_weight_min": -33.194400787353516,
      "activations/layer11_attention_weight_max": 32.07177734375,
      "activations/layer11_attention_weight_min": -31.102787017822266,
      "activations/layer12_attention_weight_max": 32.86064147949219,
      "activations/layer12_attention_weight_min": -26.194089889526367,
      "activations/layer13_attention_weight_max": 43.64962387084961,
      "activations/layer13_attention_weight_min": -34.607601165771484,
      "activations/layer14_attention_weight_max": 42.580848693847656,
      "activations/layer14_attention_weight_min": -37.3790168762207,
      "activations/layer15_attention_weight_max": 39.87076950073242,
      "activations/layer15_attention_weight_min": -30.129663467407227,
      "activations/layer16_attention_weight_max": 35.30464172363281,
      "activations/layer16_attention_weight_min": -27.923603057861328,
      "activations/layer17_attention_weight_max": 32.84309768676758,
      "activations/layer17_attention_weight_min": -26.777769088745117,
      "activations/layer18_attention_weight_max": 28.812475204467773,
      "activations/layer18_attention_weight_min": -23.247957229614258,
      "activations/layer19_attention_weight_max": 35.08301544189453,
      "activations/layer19_attention_weight_min": -30.922033309936523,
      "activations/layer1_attention_weight_max": 17.74798583984375,
      "activations/layer1_attention_weight_min": -16.189382553100586,
      "activations/layer20_attention_weight_max": 33.4970817565918,
      "activations/layer20_attention_weight_min": -25.364017486572266,
      "activations/layer21_attention_weight_max": 29.68987274169922,
      "activations/layer21_attention_weight_min": -23.937803268432617,
      "activations/layer22_attention_weight_max": 43.512813568115234,
      "activations/layer22_attention_weight_min": -28.05116844177246,
      "activations/layer23_attention_weight_max": 36.57570266723633,
      "activations/layer23_attention_weight_min": -24.33844757080078,
      "activations/layer2_attention_weight_max": 36.45738220214844,
      "activations/layer2_attention_weight_min": -35.138710021972656,
      "activations/layer3_attention_weight_max": 96.81884002685547,
      "activations/layer3_attention_weight_min": -96.53250885009766,
      "activations/layer4_attention_weight_max": 122.13069152832031,
      "activations/layer4_attention_weight_min": -117.1249771118164,
      "activations/layer5_attention_weight_max": 58.5017204284668,
      "activations/layer5_attention_weight_min": -63.06478500366211,
      "activations/layer6_attention_weight_max": 48.9477424621582,
      "activations/layer6_attention_weight_min": -51.85124206542969,
      "activations/layer7_attention_weight_max": 90.8342056274414,
      "activations/layer7_attention_weight_min": -86.41064453125,
      "activations/layer8_attention_weight_max": 42.00699996948242,
      "activations/layer8_attention_weight_min": -41.812591552734375,
      "activations/layer9_attention_weight_max": 32.28811264038086,
      "activations/layer9_attention_weight_min": -33.49169158935547,
      "epoch": 9.37,
      "learning_rate": 9.048939393939393e-05,
      "loss": 2.7976,
      "step": 161250
    },
    {
      "activations/layer0_attention_weight_max": 15.381102561950684,
      "activations/layer0_attention_weight_min": -11.944035530090332,
      "activations/layer10_attention_weight_max": 37.65966033935547,
      "activations/layer10_attention_weight_min": -35.226844787597656,
      "activations/layer11_attention_weight_max": 35.8131103515625,
      "activations/layer11_attention_weight_min": -31.230802536010742,
      "activations/layer12_attention_weight_max": 25.814125061035156,
      "activations/layer12_attention_weight_min": -25.94657325744629,
      "activations/layer13_attention_weight_max": 41.83984375,
      "activations/layer13_attention_weight_min": -36.43735885620117,
      "activations/layer14_attention_weight_max": 51.91455078125,
      "activations/layer14_attention_weight_min": -38.944480895996094,
      "activations/layer15_attention_weight_max": 35.55646896362305,
      "activations/layer15_attention_weight_min": -31.800140380859375,
      "activations/layer16_attention_weight_max": 33.06234359741211,
      "activations/layer16_attention_weight_min": -30.06548309326172,
      "activations/layer17_attention_weight_max": 33.44364547729492,
      "activations/layer17_attention_weight_min": -26.99103546142578,
      "activations/layer18_attention_weight_max": 30.862262725830078,
      "activations/layer18_attention_weight_min": -22.967580795288086,
      "activations/layer19_attention_weight_max": 33.819427490234375,
      "activations/layer19_attention_weight_min": -30.49165916442871,
      "activations/layer1_attention_weight_max": 16.311738967895508,
      "activations/layer1_attention_weight_min": -15.040802001953125,
      "activations/layer20_attention_weight_max": 31.261402130126953,
      "activations/layer20_attention_weight_min": -25.133337020874023,
      "activations/layer21_attention_weight_max": 28.76059913635254,
      "activations/layer21_attention_weight_min": -23.320873260498047,
      "activations/layer22_attention_weight_max": 42.924686431884766,
      "activations/layer22_attention_weight_min": -29.26485252380371,
      "activations/layer23_attention_weight_max": 35.359832763671875,
      "activations/layer23_attention_weight_min": -22.630081176757812,
      "activations/layer2_attention_weight_max": 37.096099853515625,
      "activations/layer2_attention_weight_min": -32.40765380859375,
      "activations/layer3_attention_weight_max": 93.47549438476562,
      "activations/layer3_attention_weight_min": -97.8857192993164,
      "activations/layer4_attention_weight_max": 117.63932037353516,
      "activations/layer4_attention_weight_min": -120.99736785888672,
      "activations/layer5_attention_weight_max": 55.59459686279297,
      "activations/layer5_attention_weight_min": -65.09904479980469,
      "activations/layer6_attention_weight_max": 51.07755661010742,
      "activations/layer6_attention_weight_min": -52.64112091064453,
      "activations/layer7_attention_weight_max": 95.92710876464844,
      "activations/layer7_attention_weight_min": -95.13684844970703,
      "activations/layer8_attention_weight_max": 42.05541229248047,
      "activations/layer8_attention_weight_min": -43.15126037597656,
      "activations/layer9_attention_weight_max": 38.16210174560547,
      "activations/layer9_attention_weight_min": -35.559112548828125,
      "epoch": 9.37,
      "learning_rate": 9.047045454545453e-05,
      "loss": 2.7988,
      "step": 161300
    },
    {
      "activations/layer0_attention_weight_max": 14.48255729675293,
      "activations/layer0_attention_weight_min": -13.141481399536133,
      "activations/layer10_attention_weight_max": 34.362579345703125,
      "activations/layer10_attention_weight_min": -33.90979766845703,
      "activations/layer11_attention_weight_max": 35.724273681640625,
      "activations/layer11_attention_weight_min": -34.25518035888672,
      "activations/layer12_attention_weight_max": 24.779281616210938,
      "activations/layer12_attention_weight_min": -26.110563278198242,
      "activations/layer13_attention_weight_max": 39.50635528564453,
      "activations/layer13_attention_weight_min": -33.76507568359375,
      "activations/layer14_attention_weight_max": 43.206626892089844,
      "activations/layer14_attention_weight_min": -40.00252151489258,
      "activations/layer15_attention_weight_max": 34.6014518737793,
      "activations/layer15_attention_weight_min": -30.562105178833008,
      "activations/layer16_attention_weight_max": 28.402128219604492,
      "activations/layer16_attention_weight_min": -26.576845169067383,
      "activations/layer17_attention_weight_max": 31.895166397094727,
      "activations/layer17_attention_weight_min": -24.863733291625977,
      "activations/layer18_attention_weight_max": 31.062114715576172,
      "activations/layer18_attention_weight_min": -24.217260360717773,
      "activations/layer19_attention_weight_max": 30.432458877563477,
      "activations/layer19_attention_weight_min": -30.284698486328125,
      "activations/layer1_attention_weight_max": 16.255962371826172,
      "activations/layer1_attention_weight_min": -14.638260841369629,
      "activations/layer20_attention_weight_max": 25.86543846130371,
      "activations/layer20_attention_weight_min": -25.079862594604492,
      "activations/layer21_attention_weight_max": 27.609403610229492,
      "activations/layer21_attention_weight_min": -23.1772518157959,
      "activations/layer22_attention_weight_max": 42.521697998046875,
      "activations/layer22_attention_weight_min": -34.559383392333984,
      "activations/layer23_attention_weight_max": 32.78285598754883,
      "activations/layer23_attention_weight_min": -24.263057708740234,
      "activations/layer2_attention_weight_max": 30.47900390625,
      "activations/layer2_attention_weight_min": -32.05143737792969,
      "activations/layer3_attention_weight_max": 88.873046875,
      "activations/layer3_attention_weight_min": -94.34835815429688,
      "activations/layer4_attention_weight_max": 114.91204071044922,
      "activations/layer4_attention_weight_min": -116.06558990478516,
      "activations/layer5_attention_weight_max": 52.93048095703125,
      "activations/layer5_attention_weight_min": -63.931026458740234,
      "activations/layer6_attention_weight_max": 47.860897064208984,
      "activations/layer6_attention_weight_min": -51.381893157958984,
      "activations/layer7_attention_weight_max": 86.23492431640625,
      "activations/layer7_attention_weight_min": -91.15756225585938,
      "activations/layer8_attention_weight_max": 39.8023567199707,
      "activations/layer8_attention_weight_min": -43.060699462890625,
      "activations/layer9_attention_weight_max": 35.357627868652344,
      "activations/layer9_attention_weight_min": -35.643218994140625,
      "epoch": 9.38,
      "learning_rate": 9.045151515151515e-05,
      "loss": 2.7921,
      "step": 161350
    },
    {
      "activations/layer0_attention_weight_max": 15.841721534729004,
      "activations/layer0_attention_weight_min": -12.867769241333008,
      "activations/layer10_attention_weight_max": 31.18968963623047,
      "activations/layer10_attention_weight_min": -30.913528442382812,
      "activations/layer11_attention_weight_max": 34.82538986206055,
      "activations/layer11_attention_weight_min": -35.8144645690918,
      "activations/layer12_attention_weight_max": 22.00761604309082,
      "activations/layer12_attention_weight_min": -26.496503829956055,
      "activations/layer13_attention_weight_max": 36.69743347167969,
      "activations/layer13_attention_weight_min": -30.632640838623047,
      "activations/layer14_attention_weight_max": 45.89494705200195,
      "activations/layer14_attention_weight_min": -37.729225158691406,
      "activations/layer15_attention_weight_max": 32.527523040771484,
      "activations/layer15_attention_weight_min": -30.15910530090332,
      "activations/layer16_attention_weight_max": 31.328397750854492,
      "activations/layer16_attention_weight_min": -28.12031364440918,
      "activations/layer17_attention_weight_max": 32.217445373535156,
      "activations/layer17_attention_weight_min": -27.513378143310547,
      "activations/layer18_attention_weight_max": 32.534976959228516,
      "activations/layer18_attention_weight_min": -25.63414764404297,
      "activations/layer19_attention_weight_max": 35.22014617919922,
      "activations/layer19_attention_weight_min": -34.12312698364258,
      "activations/layer1_attention_weight_max": 16.377779006958008,
      "activations/layer1_attention_weight_min": -14.950247764587402,
      "activations/layer20_attention_weight_max": 29.160110473632812,
      "activations/layer20_attention_weight_min": -25.374773025512695,
      "activations/layer21_attention_weight_max": 29.447948455810547,
      "activations/layer21_attention_weight_min": -24.887557983398438,
      "activations/layer22_attention_weight_max": 44.50770568847656,
      "activations/layer22_attention_weight_min": -30.413280487060547,
      "activations/layer23_attention_weight_max": 35.493408203125,
      "activations/layer23_attention_weight_min": -24.397932052612305,
      "activations/layer2_attention_weight_max": 32.63777160644531,
      "activations/layer2_attention_weight_min": -33.08026123046875,
      "activations/layer3_attention_weight_max": 92.7169418334961,
      "activations/layer3_attention_weight_min": -94.41376495361328,
      "activations/layer4_attention_weight_max": 114.53514862060547,
      "activations/layer4_attention_weight_min": -113.65209197998047,
      "activations/layer5_attention_weight_max": 57.930328369140625,
      "activations/layer5_attention_weight_min": -63.774574279785156,
      "activations/layer6_attention_weight_max": 46.88138198852539,
      "activations/layer6_attention_weight_min": -51.03678894042969,
      "activations/layer7_attention_weight_max": 97.8390121459961,
      "activations/layer7_attention_weight_min": -86.30814361572266,
      "activations/layer8_attention_weight_max": 40.682437896728516,
      "activations/layer8_attention_weight_min": -39.32194137573242,
      "activations/layer9_attention_weight_max": 32.45036315917969,
      "activations/layer9_attention_weight_min": -37.746490478515625,
      "epoch": 9.38,
      "learning_rate": 9.043257575757575e-05,
      "loss": 2.8128,
      "step": 161400
    },
    {
      "activations/layer0_attention_weight_max": 14.83596420288086,
      "activations/layer0_attention_weight_min": -12.261856079101562,
      "activations/layer10_attention_weight_max": 33.59845733642578,
      "activations/layer10_attention_weight_min": -34.16901779174805,
      "activations/layer11_attention_weight_max": 33.37366485595703,
      "activations/layer11_attention_weight_min": -33.17717361450195,
      "activations/layer12_attention_weight_max": 22.92990493774414,
      "activations/layer12_attention_weight_min": -26.464052200317383,
      "activations/layer13_attention_weight_max": 37.49409103393555,
      "activations/layer13_attention_weight_min": -33.473506927490234,
      "activations/layer14_attention_weight_max": 52.572364807128906,
      "activations/layer14_attention_weight_min": -42.03634262084961,
      "activations/layer15_attention_weight_max": 38.87580871582031,
      "activations/layer15_attention_weight_min": -33.5830078125,
      "activations/layer16_attention_weight_max": 31.862003326416016,
      "activations/layer16_attention_weight_min": -29.156471252441406,
      "activations/layer17_attention_weight_max": 35.50550079345703,
      "activations/layer17_attention_weight_min": -26.623647689819336,
      "activations/layer18_attention_weight_max": 31.11211585998535,
      "activations/layer18_attention_weight_min": -23.454593658447266,
      "activations/layer19_attention_weight_max": 34.38324737548828,
      "activations/layer19_attention_weight_min": -32.96080017089844,
      "activations/layer1_attention_weight_max": 15.994709968566895,
      "activations/layer1_attention_weight_min": -15.27475357055664,
      "activations/layer20_attention_weight_max": 29.997446060180664,
      "activations/layer20_attention_weight_min": -27.068395614624023,
      "activations/layer21_attention_weight_max": 29.84307861328125,
      "activations/layer21_attention_weight_min": -24.593236923217773,
      "activations/layer22_attention_weight_max": 45.65254211425781,
      "activations/layer22_attention_weight_min": -30.78482437133789,
      "activations/layer23_attention_weight_max": 35.44568634033203,
      "activations/layer23_attention_weight_min": -25.543724060058594,
      "activations/layer2_attention_weight_max": 32.6312370300293,
      "activations/layer2_attention_weight_min": -33.00243377685547,
      "activations/layer3_attention_weight_max": 97.83718872070312,
      "activations/layer3_attention_weight_min": -97.18397521972656,
      "activations/layer4_attention_weight_max": 124.51274871826172,
      "activations/layer4_attention_weight_min": -123.93944549560547,
      "activations/layer5_attention_weight_max": 56.11428451538086,
      "activations/layer5_attention_weight_min": -66.25050354003906,
      "activations/layer6_attention_weight_max": 50.16078567504883,
      "activations/layer6_attention_weight_min": -51.8066520690918,
      "activations/layer7_attention_weight_max": 88.66719818115234,
      "activations/layer7_attention_weight_min": -92.8408432006836,
      "activations/layer8_attention_weight_max": 43.261131286621094,
      "activations/layer8_attention_weight_min": -41.939456939697266,
      "activations/layer9_attention_weight_max": 34.28158950805664,
      "activations/layer9_attention_weight_min": -33.769893646240234,
      "epoch": 9.38,
      "learning_rate": 9.041363636363635e-05,
      "loss": 2.7974,
      "step": 161450
    },
    {
      "activations/layer0_attention_weight_max": 16.38161849975586,
      "activations/layer0_attention_weight_min": -12.300965309143066,
      "activations/layer10_attention_weight_max": 34.07970428466797,
      "activations/layer10_attention_weight_min": -31.892967224121094,
      "activations/layer11_attention_weight_max": 34.50242614746094,
      "activations/layer11_attention_weight_min": -30.941823959350586,
      "activations/layer12_attention_weight_max": 23.779006958007812,
      "activations/layer12_attention_weight_min": -23.14873504638672,
      "activations/layer13_attention_weight_max": 40.124900817871094,
      "activations/layer13_attention_weight_min": -32.635990142822266,
      "activations/layer14_attention_weight_max": 47.2051887512207,
      "activations/layer14_attention_weight_min": -37.00696563720703,
      "activations/layer15_attention_weight_max": 34.152408599853516,
      "activations/layer15_attention_weight_min": -29.902616500854492,
      "activations/layer16_attention_weight_max": 30.762182235717773,
      "activations/layer16_attention_weight_min": -28.902891159057617,
      "activations/layer17_attention_weight_max": 34.42421340942383,
      "activations/layer17_attention_weight_min": -24.041345596313477,
      "activations/layer18_attention_weight_max": 29.42556381225586,
      "activations/layer18_attention_weight_min": -21.67544937133789,
      "activations/layer19_attention_weight_max": 34.06394958496094,
      "activations/layer19_attention_weight_min": -28.25996971130371,
      "activations/layer1_attention_weight_max": 17.179798126220703,
      "activations/layer1_attention_weight_min": -15.5867338180542,
      "activations/layer20_attention_weight_max": 31.150285720825195,
      "activations/layer20_attention_weight_min": -26.009138107299805,
      "activations/layer21_attention_weight_max": 28.66831398010254,
      "activations/layer21_attention_weight_min": -22.540590286254883,
      "activations/layer22_attention_weight_max": 43.11151885986328,
      "activations/layer22_attention_weight_min": -28.3533878326416,
      "activations/layer23_attention_weight_max": 36.753082275390625,
      "activations/layer23_attention_weight_min": -23.546972274780273,
      "activations/layer2_attention_weight_max": 30.24553680419922,
      "activations/layer2_attention_weight_min": -30.03705596923828,
      "activations/layer3_attention_weight_max": 90.77339935302734,
      "activations/layer3_attention_weight_min": -93.10822296142578,
      "activations/layer4_attention_weight_max": 111.22828674316406,
      "activations/layer4_attention_weight_min": -110.25627899169922,
      "activations/layer5_attention_weight_max": 52.258087158203125,
      "activations/layer5_attention_weight_min": -59.53379821777344,
      "activations/layer6_attention_weight_max": 48.83479309082031,
      "activations/layer6_attention_weight_min": -47.015132904052734,
      "activations/layer7_attention_weight_max": 86.64993286132812,
      "activations/layer7_attention_weight_min": -86.94771575927734,
      "activations/layer8_attention_weight_max": 42.178192138671875,
      "activations/layer8_attention_weight_min": -40.920982360839844,
      "activations/layer9_attention_weight_max": 35.41453552246094,
      "activations/layer9_attention_weight_min": -34.38007736206055,
      "epoch": 9.38,
      "learning_rate": 9.039469696969696e-05,
      "loss": 2.8072,
      "step": 161500
    },
    {
      "activations/layer0_attention_weight_max": 15.15877628326416,
      "activations/layer0_attention_weight_min": -11.286639213562012,
      "activations/layer10_attention_weight_max": 31.42453956604004,
      "activations/layer10_attention_weight_min": -33.28631591796875,
      "activations/layer11_attention_weight_max": 33.955833435058594,
      "activations/layer11_attention_weight_min": -34.671791076660156,
      "activations/layer12_attention_weight_max": 30.38797378540039,
      "activations/layer12_attention_weight_min": -25.238685607910156,
      "activations/layer13_attention_weight_max": 54.8763427734375,
      "activations/layer13_attention_weight_min": -36.522926330566406,
      "activations/layer14_attention_weight_max": 51.17124557495117,
      "activations/layer14_attention_weight_min": -40.76185607910156,
      "activations/layer15_attention_weight_max": 46.06031799316406,
      "activations/layer15_attention_weight_min": -32.88177490234375,
      "activations/layer16_attention_weight_max": 32.27490234375,
      "activations/layer16_attention_weight_min": -27.31006622314453,
      "activations/layer17_attention_weight_max": 32.44692611694336,
      "activations/layer17_attention_weight_min": -26.85509490966797,
      "activations/layer18_attention_weight_max": 31.368980407714844,
      "activations/layer18_attention_weight_min": -22.3129940032959,
      "activations/layer19_attention_weight_max": 36.98173904418945,
      "activations/layer19_attention_weight_min": -31.257080078125,
      "activations/layer1_attention_weight_max": 16.60103988647461,
      "activations/layer1_attention_weight_min": -14.921415328979492,
      "activations/layer20_attention_weight_max": 30.46474266052246,
      "activations/layer20_attention_weight_min": -23.303150177001953,
      "activations/layer21_attention_weight_max": 29.226924896240234,
      "activations/layer21_attention_weight_min": -23.120866775512695,
      "activations/layer22_attention_weight_max": 48.871482849121094,
      "activations/layer22_attention_weight_min": -29.425704956054688,
      "activations/layer23_attention_weight_max": 37.784149169921875,
      "activations/layer23_attention_weight_min": -23.668941497802734,
      "activations/layer2_attention_weight_max": 30.828845977783203,
      "activations/layer2_attention_weight_min": -31.208059310913086,
      "activations/layer3_attention_weight_max": 92.05097198486328,
      "activations/layer3_attention_weight_min": -93.58240509033203,
      "activations/layer4_attention_weight_max": 121.32225799560547,
      "activations/layer4_attention_weight_min": -117.6725082397461,
      "activations/layer5_attention_weight_max": 55.830074310302734,
      "activations/layer5_attention_weight_min": -61.38482666015625,
      "activations/layer6_attention_weight_max": 47.07810592651367,
      "activations/layer6_attention_weight_min": -49.30854415893555,
      "activations/layer7_attention_weight_max": 91.58982849121094,
      "activations/layer7_attention_weight_min": -86.61336517333984,
      "activations/layer8_attention_weight_max": 40.520076751708984,
      "activations/layer8_attention_weight_min": -40.74616622924805,
      "activations/layer9_attention_weight_max": 33.23295211791992,
      "activations/layer9_attention_weight_min": -34.12384033203125,
      "epoch": 9.39,
      "learning_rate": 9.037575757575758e-05,
      "loss": 2.7903,
      "step": 161550
    },
    {
      "activations/layer0_attention_weight_max": 15.46103286743164,
      "activations/layer0_attention_weight_min": -11.992942810058594,
      "activations/layer10_attention_weight_max": 30.296342849731445,
      "activations/layer10_attention_weight_min": -30.83072280883789,
      "activations/layer11_attention_weight_max": 31.407848358154297,
      "activations/layer11_attention_weight_min": -28.91510772705078,
      "activations/layer12_attention_weight_max": 23.059326171875,
      "activations/layer12_attention_weight_min": -24.68271827697754,
      "activations/layer13_attention_weight_max": 38.26130294799805,
      "activations/layer13_attention_weight_min": -34.446998596191406,
      "activations/layer14_attention_weight_max": 45.04582977294922,
      "activations/layer14_attention_weight_min": -38.24435043334961,
      "activations/layer15_attention_weight_max": 35.57888412475586,
      "activations/layer15_attention_weight_min": -30.761442184448242,
      "activations/layer16_attention_weight_max": 30.05199432373047,
      "activations/layer16_attention_weight_min": -28.973398208618164,
      "activations/layer17_attention_weight_max": 33.21038055419922,
      "activations/layer17_attention_weight_min": -24.830921173095703,
      "activations/layer18_attention_weight_max": 33.41957473754883,
      "activations/layer18_attention_weight_min": -23.137022018432617,
      "activations/layer19_attention_weight_max": 35.75071334838867,
      "activations/layer19_attention_weight_min": -34.51184844970703,
      "activations/layer1_attention_weight_max": 18.02012062072754,
      "activations/layer1_attention_weight_min": -14.848145484924316,
      "activations/layer20_attention_weight_max": 33.96092987060547,
      "activations/layer20_attention_weight_min": -26.421672821044922,
      "activations/layer21_attention_weight_max": 32.45858383178711,
      "activations/layer21_attention_weight_min": -23.797571182250977,
      "activations/layer22_attention_weight_max": 45.11119079589844,
      "activations/layer22_attention_weight_min": -32.54737091064453,
      "activations/layer23_attention_weight_max": 38.213600158691406,
      "activations/layer23_attention_weight_min": -25.311092376708984,
      "activations/layer2_attention_weight_max": 31.672664642333984,
      "activations/layer2_attention_weight_min": -33.955169677734375,
      "activations/layer3_attention_weight_max": 93.07649230957031,
      "activations/layer3_attention_weight_min": -95.78916931152344,
      "activations/layer4_attention_weight_max": 118.35192108154297,
      "activations/layer4_attention_weight_min": -113.3241958618164,
      "activations/layer5_attention_weight_max": 54.6217041015625,
      "activations/layer5_attention_weight_min": -63.2412223815918,
      "activations/layer6_attention_weight_max": 46.472049713134766,
      "activations/layer6_attention_weight_min": -47.9631462097168,
      "activations/layer7_attention_weight_max": 85.7452392578125,
      "activations/layer7_attention_weight_min": -82.8023681640625,
      "activations/layer8_attention_weight_max": 37.76054382324219,
      "activations/layer8_attention_weight_min": -38.805152893066406,
      "activations/layer9_attention_weight_max": 33.41106033325195,
      "activations/layer9_attention_weight_min": -32.171607971191406,
      "epoch": 9.39,
      "learning_rate": 9.035681818181817e-05,
      "loss": 2.8052,
      "step": 161600
    },
    {
      "activations/layer0_attention_weight_max": 15.999135971069336,
      "activations/layer0_attention_weight_min": -12.818817138671875,
      "activations/layer10_attention_weight_max": 32.636295318603516,
      "activations/layer10_attention_weight_min": -33.19501495361328,
      "activations/layer11_attention_weight_max": 30.52975845336914,
      "activations/layer11_attention_weight_min": -31.674291610717773,
      "activations/layer12_attention_weight_max": 23.543100357055664,
      "activations/layer12_attention_weight_min": -24.544931411743164,
      "activations/layer13_attention_weight_max": 35.92864227294922,
      "activations/layer13_attention_weight_min": -32.76106262207031,
      "activations/layer14_attention_weight_max": 47.342926025390625,
      "activations/layer14_attention_weight_min": -36.8494758605957,
      "activations/layer15_attention_weight_max": 34.348663330078125,
      "activations/layer15_attention_weight_min": -30.559795379638672,
      "activations/layer16_attention_weight_max": 32.900047302246094,
      "activations/layer16_attention_weight_min": -27.303081512451172,
      "activations/layer17_attention_weight_max": 31.69509506225586,
      "activations/layer17_attention_weight_min": -25.223976135253906,
      "activations/layer18_attention_weight_max": 31.550662994384766,
      "activations/layer18_attention_weight_min": -23.045272827148438,
      "activations/layer19_attention_weight_max": 40.261932373046875,
      "activations/layer19_attention_weight_min": -31.501760482788086,
      "activations/layer1_attention_weight_max": 16.45040512084961,
      "activations/layer1_attention_weight_min": -13.910924911499023,
      "activations/layer20_attention_weight_max": 32.38753128051758,
      "activations/layer20_attention_weight_min": -25.045909881591797,
      "activations/layer21_attention_weight_max": 31.18907928466797,
      "activations/layer21_attention_weight_min": -25.649295806884766,
      "activations/layer22_attention_weight_max": 46.23918533325195,
      "activations/layer22_attention_weight_min": -32.053890228271484,
      "activations/layer23_attention_weight_max": 37.60888671875,
      "activations/layer23_attention_weight_min": -24.760509490966797,
      "activations/layer2_attention_weight_max": 30.991559982299805,
      "activations/layer2_attention_weight_min": -31.163333892822266,
      "activations/layer3_attention_weight_max": 93.2762451171875,
      "activations/layer3_attention_weight_min": -93.38782501220703,
      "activations/layer4_attention_weight_max": 117.43660736083984,
      "activations/layer4_attention_weight_min": -117.75555419921875,
      "activations/layer5_attention_weight_max": 53.70515441894531,
      "activations/layer5_attention_weight_min": -66.40681457519531,
      "activations/layer6_attention_weight_max": 44.68643569946289,
      "activations/layer6_attention_weight_min": -49.81570053100586,
      "activations/layer7_attention_weight_max": 87.28897094726562,
      "activations/layer7_attention_weight_min": -87.68392181396484,
      "activations/layer8_attention_weight_max": 39.46296310424805,
      "activations/layer8_attention_weight_min": -42.419090270996094,
      "activations/layer9_attention_weight_max": 34.3978157043457,
      "activations/layer9_attention_weight_min": -34.94172668457031,
      "epoch": 9.39,
      "learning_rate": 9.033787878787878e-05,
      "loss": 2.8013,
      "step": 161650
    },
    {
      "activations/layer0_attention_weight_max": 14.9371976852417,
      "activations/layer0_attention_weight_min": -11.415923118591309,
      "activations/layer10_attention_weight_max": 34.93886947631836,
      "activations/layer10_attention_weight_min": -33.51921463012695,
      "activations/layer11_attention_weight_max": 31.49117088317871,
      "activations/layer11_attention_weight_min": -31.54253387451172,
      "activations/layer12_attention_weight_max": 23.889400482177734,
      "activations/layer12_attention_weight_min": -24.44451332092285,
      "activations/layer13_attention_weight_max": 47.91762161254883,
      "activations/layer13_attention_weight_min": -34.27105712890625,
      "activations/layer14_attention_weight_max": 46.58831787109375,
      "activations/layer14_attention_weight_min": -37.45151138305664,
      "activations/layer15_attention_weight_max": 36.44081115722656,
      "activations/layer15_attention_weight_min": -32.86658477783203,
      "activations/layer16_attention_weight_max": 35.9548454284668,
      "activations/layer16_attention_weight_min": -30.40918731689453,
      "activations/layer17_attention_weight_max": 33.766265869140625,
      "activations/layer17_attention_weight_min": -26.334375381469727,
      "activations/layer18_attention_weight_max": 34.44449996948242,
      "activations/layer18_attention_weight_min": -21.433177947998047,
      "activations/layer19_attention_weight_max": 40.47667694091797,
      "activations/layer19_attention_weight_min": -32.73243713378906,
      "activations/layer1_attention_weight_max": 16.69230842590332,
      "activations/layer1_attention_weight_min": -13.03091812133789,
      "activations/layer20_attention_weight_max": 35.48063278198242,
      "activations/layer20_attention_weight_min": -26.56219482421875,
      "activations/layer21_attention_weight_max": 39.102596282958984,
      "activations/layer21_attention_weight_min": -23.565786361694336,
      "activations/layer22_attention_weight_max": 48.55082321166992,
      "activations/layer22_attention_weight_min": -28.627283096313477,
      "activations/layer23_attention_weight_max": 38.05661392211914,
      "activations/layer23_attention_weight_min": -24.486949920654297,
      "activations/layer2_attention_weight_max": 30.759700775146484,
      "activations/layer2_attention_weight_min": -29.961040496826172,
      "activations/layer3_attention_weight_max": 88.68370819091797,
      "activations/layer3_attention_weight_min": -88.85308837890625,
      "activations/layer4_attention_weight_max": 111.08232879638672,
      "activations/layer4_attention_weight_min": -114.28045654296875,
      "activations/layer5_attention_weight_max": 54.199562072753906,
      "activations/layer5_attention_weight_min": -61.47922897338867,
      "activations/layer6_attention_weight_max": 46.04203796386719,
      "activations/layer6_attention_weight_min": -52.53215026855469,
      "activations/layer7_attention_weight_max": 91.38397979736328,
      "activations/layer7_attention_weight_min": -95.90289306640625,
      "activations/layer8_attention_weight_max": 39.47776412963867,
      "activations/layer8_attention_weight_min": -42.83891677856445,
      "activations/layer9_attention_weight_max": 35.210693359375,
      "activations/layer9_attention_weight_min": -35.57392120361328,
      "epoch": 9.4,
      "learning_rate": 9.03189393939394e-05,
      "loss": 2.7885,
      "step": 161700
    },
    {
      "activations/layer0_attention_weight_max": 15.359262466430664,
      "activations/layer0_attention_weight_min": -12.560426712036133,
      "activations/layer10_attention_weight_max": 31.65024757385254,
      "activations/layer10_attention_weight_min": -31.423437118530273,
      "activations/layer11_attention_weight_max": 31.1585693359375,
      "activations/layer11_attention_weight_min": -30.151565551757812,
      "activations/layer12_attention_weight_max": 24.213851928710938,
      "activations/layer12_attention_weight_min": -24.24161148071289,
      "activations/layer13_attention_weight_max": 39.96745681762695,
      "activations/layer13_attention_weight_min": -32.164276123046875,
      "activations/layer14_attention_weight_max": 50.278621673583984,
      "activations/layer14_attention_weight_min": -38.61757278442383,
      "activations/layer15_attention_weight_max": 34.582035064697266,
      "activations/layer15_attention_weight_min": -30.505170822143555,
      "activations/layer16_attention_weight_max": 29.479551315307617,
      "activations/layer16_attention_weight_min": -25.3261775970459,
      "activations/layer17_attention_weight_max": 31.61150550842285,
      "activations/layer17_attention_weight_min": -24.243064880371094,
      "activations/layer18_attention_weight_max": 30.165767669677734,
      "activations/layer18_attention_weight_min": -23.18355941772461,
      "activations/layer19_attention_weight_max": 33.2966194152832,
      "activations/layer19_attention_weight_min": -28.120824813842773,
      "activations/layer1_attention_weight_max": 16.206222534179688,
      "activations/layer1_attention_weight_min": -14.505565643310547,
      "activations/layer20_attention_weight_max": 28.761550903320312,
      "activations/layer20_attention_weight_min": -24.204479217529297,
      "activations/layer21_attention_weight_max": 29.93617820739746,
      "activations/layer21_attention_weight_min": -21.988454818725586,
      "activations/layer22_attention_weight_max": 51.19498825073242,
      "activations/layer22_attention_weight_min": -27.399860382080078,
      "activations/layer23_attention_weight_max": 36.12154006958008,
      "activations/layer23_attention_weight_min": -23.729888916015625,
      "activations/layer2_attention_weight_max": 30.325742721557617,
      "activations/layer2_attention_weight_min": -33.359046936035156,
      "activations/layer3_attention_weight_max": 89.40459442138672,
      "activations/layer3_attention_weight_min": -91.30431365966797,
      "activations/layer4_attention_weight_max": 110.24787902832031,
      "activations/layer4_attention_weight_min": -113.72005462646484,
      "activations/layer5_attention_weight_max": 54.680885314941406,
      "activations/layer5_attention_weight_min": -61.58845520019531,
      "activations/layer6_attention_weight_max": 46.50388717651367,
      "activations/layer6_attention_weight_min": -50.0451545715332,
      "activations/layer7_attention_weight_max": 92.34080505371094,
      "activations/layer7_attention_weight_min": -91.98300170898438,
      "activations/layer8_attention_weight_max": 40.110748291015625,
      "activations/layer8_attention_weight_min": -41.56930160522461,
      "activations/layer9_attention_weight_max": 33.46736526489258,
      "activations/layer9_attention_weight_min": -34.57099533081055,
      "epoch": 9.4,
      "learning_rate": 9.029999999999999e-05,
      "loss": 2.8035,
      "step": 161750
    },
    {
      "activations/layer0_attention_weight_max": 15.000431060791016,
      "activations/layer0_attention_weight_min": -11.893681526184082,
      "activations/layer10_attention_weight_max": 34.64849853515625,
      "activations/layer10_attention_weight_min": -31.909423828125,
      "activations/layer11_attention_weight_max": 33.588993072509766,
      "activations/layer11_attention_weight_min": -31.331069946289062,
      "activations/layer12_attention_weight_max": 25.434200286865234,
      "activations/layer12_attention_weight_min": -24.824071884155273,
      "activations/layer13_attention_weight_max": 38.971405029296875,
      "activations/layer13_attention_weight_min": -34.488487243652344,
      "activations/layer14_attention_weight_max": 53.039794921875,
      "activations/layer14_attention_weight_min": -40.80216598510742,
      "activations/layer15_attention_weight_max": 34.949073791503906,
      "activations/layer15_attention_weight_min": -31.34001350402832,
      "activations/layer16_attention_weight_max": 31.432971954345703,
      "activations/layer16_attention_weight_min": -27.4774169921875,
      "activations/layer17_attention_weight_max": 31.67573356628418,
      "activations/layer17_attention_weight_min": -27.818801879882812,
      "activations/layer18_attention_weight_max": 28.55817222595215,
      "activations/layer18_attention_weight_min": -23.07927703857422,
      "activations/layer19_attention_weight_max": 34.278587341308594,
      "activations/layer19_attention_weight_min": -29.64552879333496,
      "activations/layer1_attention_weight_max": 16.708202362060547,
      "activations/layer1_attention_weight_min": -14.378555297851562,
      "activations/layer20_attention_weight_max": 28.312000274658203,
      "activations/layer20_attention_weight_min": -25.926231384277344,
      "activations/layer21_attention_weight_max": 25.757474899291992,
      "activations/layer21_attention_weight_min": -23.74834632873535,
      "activations/layer22_attention_weight_max": 48.19223403930664,
      "activations/layer22_attention_weight_min": -29.53541374206543,
      "activations/layer23_attention_weight_max": 35.5654182434082,
      "activations/layer23_attention_weight_min": -24.235748291015625,
      "activations/layer2_attention_weight_max": 30.249969482421875,
      "activations/layer2_attention_weight_min": -30.419797897338867,
      "activations/layer3_attention_weight_max": 89.12519073486328,
      "activations/layer3_attention_weight_min": -90.19698333740234,
      "activations/layer4_attention_weight_max": 118.3856430053711,
      "activations/layer4_attention_weight_min": -112.93062591552734,
      "activations/layer5_attention_weight_max": 53.132930755615234,
      "activations/layer5_attention_weight_min": -66.76101684570312,
      "activations/layer6_attention_weight_max": 45.868324279785156,
      "activations/layer6_attention_weight_min": -50.22428512573242,
      "activations/layer7_attention_weight_max": 92.98866271972656,
      "activations/layer7_attention_weight_min": -96.43085479736328,
      "activations/layer8_attention_weight_max": 40.38482666015625,
      "activations/layer8_attention_weight_min": -42.18403625488281,
      "activations/layer9_attention_weight_max": 37.26180648803711,
      "activations/layer9_attention_weight_min": -34.31003189086914,
      "epoch": 9.4,
      "learning_rate": 9.02810606060606e-05,
      "loss": 2.7861,
      "step": 161800
    },
    {
      "activations/layer0_attention_weight_max": 15.969183921813965,
      "activations/layer0_attention_weight_min": -11.642189979553223,
      "activations/layer10_attention_weight_max": 36.254798889160156,
      "activations/layer10_attention_weight_min": -37.70897674560547,
      "activations/layer11_attention_weight_max": 34.580257415771484,
      "activations/layer11_attention_weight_min": -35.85206604003906,
      "activations/layer12_attention_weight_max": 26.6688175201416,
      "activations/layer12_attention_weight_min": -26.552427291870117,
      "activations/layer13_attention_weight_max": 41.0468864440918,
      "activations/layer13_attention_weight_min": -36.30488586425781,
      "activations/layer14_attention_weight_max": 46.732643127441406,
      "activations/layer14_attention_weight_min": -39.6052131652832,
      "activations/layer15_attention_weight_max": 38.747310638427734,
      "activations/layer15_attention_weight_min": -33.67057418823242,
      "activations/layer16_attention_weight_max": 30.596105575561523,
      "activations/layer16_attention_weight_min": -28.599592208862305,
      "activations/layer17_attention_weight_max": 33.53682327270508,
      "activations/layer17_attention_weight_min": -27.84986686706543,
      "activations/layer18_attention_weight_max": 31.166479110717773,
      "activations/layer18_attention_weight_min": -25.033233642578125,
      "activations/layer19_attention_weight_max": 33.55707931518555,
      "activations/layer19_attention_weight_min": -31.816816329956055,
      "activations/layer1_attention_weight_max": 17.126916885375977,
      "activations/layer1_attention_weight_min": -11.702752113342285,
      "activations/layer20_attention_weight_max": 30.424203872680664,
      "activations/layer20_attention_weight_min": -27.502891540527344,
      "activations/layer21_attention_weight_max": 32.01161575317383,
      "activations/layer21_attention_weight_min": -25.520055770874023,
      "activations/layer22_attention_weight_max": 50.19770812988281,
      "activations/layer22_attention_weight_min": -32.928401947021484,
      "activations/layer23_attention_weight_max": 40.49419403076172,
      "activations/layer23_attention_weight_min": -28.000892639160156,
      "activations/layer2_attention_weight_max": 31.365116119384766,
      "activations/layer2_attention_weight_min": -31.73556137084961,
      "activations/layer3_attention_weight_max": 91.82120513916016,
      "activations/layer3_attention_weight_min": -90.5967025756836,
      "activations/layer4_attention_weight_max": 116.75093078613281,
      "activations/layer4_attention_weight_min": -113.02359771728516,
      "activations/layer5_attention_weight_max": 55.061546325683594,
      "activations/layer5_attention_weight_min": -63.995269775390625,
      "activations/layer6_attention_weight_max": 45.91916275024414,
      "activations/layer6_attention_weight_min": -50.230262756347656,
      "activations/layer7_attention_weight_max": 89.03165435791016,
      "activations/layer7_attention_weight_min": -94.84109497070312,
      "activations/layer8_attention_weight_max": 43.48442077636719,
      "activations/layer8_attention_weight_min": -44.09919357299805,
      "activations/layer9_attention_weight_max": 37.195613861083984,
      "activations/layer9_attention_weight_min": -37.300113677978516,
      "epoch": 9.4,
      "learning_rate": 9.02621212121212e-05,
      "loss": 2.7853,
      "step": 161850
    },
    {
      "activations/layer0_attention_weight_max": 15.555550575256348,
      "activations/layer0_attention_weight_min": -11.698163032531738,
      "activations/layer10_attention_weight_max": 35.068111419677734,
      "activations/layer10_attention_weight_min": -35.18244552612305,
      "activations/layer11_attention_weight_max": 33.796382904052734,
      "activations/layer11_attention_weight_min": -33.054351806640625,
      "activations/layer12_attention_weight_max": 25.7189998626709,
      "activations/layer12_attention_weight_min": -25.51164436340332,
      "activations/layer13_attention_weight_max": 39.64362716674805,
      "activations/layer13_attention_weight_min": -34.229087829589844,
      "activations/layer14_attention_weight_max": 45.67197036743164,
      "activations/layer14_attention_weight_min": -44.461212158203125,
      "activations/layer15_attention_weight_max": 33.976192474365234,
      "activations/layer15_attention_weight_min": -30.28396987915039,
      "activations/layer16_attention_weight_max": 29.841983795166016,
      "activations/layer16_attention_weight_min": -27.311981201171875,
      "activations/layer17_attention_weight_max": 30.645442962646484,
      "activations/layer17_attention_weight_min": -25.429790496826172,
      "activations/layer18_attention_weight_max": 30.66802406311035,
      "activations/layer18_attention_weight_min": -22.50582504272461,
      "activations/layer19_attention_weight_max": 38.222652435302734,
      "activations/layer19_attention_weight_min": -29.985279083251953,
      "activations/layer1_attention_weight_max": 16.9844970703125,
      "activations/layer1_attention_weight_min": -14.013440132141113,
      "activations/layer20_attention_weight_max": 28.301292419433594,
      "activations/layer20_attention_weight_min": -24.025318145751953,
      "activations/layer21_attention_weight_max": 29.25448226928711,
      "activations/layer21_attention_weight_min": -23.83853530883789,
      "activations/layer22_attention_weight_max": 43.68907165527344,
      "activations/layer22_attention_weight_min": -28.07647705078125,
      "activations/layer23_attention_weight_max": 39.41116714477539,
      "activations/layer23_attention_weight_min": -24.37018394470215,
      "activations/layer2_attention_weight_max": 30.54717254638672,
      "activations/layer2_attention_weight_min": -31.071903228759766,
      "activations/layer3_attention_weight_max": 89.29776763916016,
      "activations/layer3_attention_weight_min": -94.62875366210938,
      "activations/layer4_attention_weight_max": 110.9909439086914,
      "activations/layer4_attention_weight_min": -110.73712158203125,
      "activations/layer5_attention_weight_max": 51.516929626464844,
      "activations/layer5_attention_weight_min": -60.24189758300781,
      "activations/layer6_attention_weight_max": 44.253623962402344,
      "activations/layer6_attention_weight_min": -47.49631118774414,
      "activations/layer7_attention_weight_max": 87.29105377197266,
      "activations/layer7_attention_weight_min": -91.40855407714844,
      "activations/layer8_attention_weight_max": 41.22975158691406,
      "activations/layer8_attention_weight_min": -42.50202941894531,
      "activations/layer9_attention_weight_max": 35.427433013916016,
      "activations/layer9_attention_weight_min": -37.711891174316406,
      "epoch": 9.41,
      "learning_rate": 9.02431818181818e-05,
      "loss": 2.8005,
      "step": 161900
    },
    {
      "activations/layer0_attention_weight_max": 16.176889419555664,
      "activations/layer0_attention_weight_min": -12.164276123046875,
      "activations/layer10_attention_weight_max": 33.753456115722656,
      "activations/layer10_attention_weight_min": -34.4925422668457,
      "activations/layer11_attention_weight_max": 30.30633544921875,
      "activations/layer11_attention_weight_min": -32.39090347290039,
      "activations/layer12_attention_weight_max": 25.096710205078125,
      "activations/layer12_attention_weight_min": -24.4553165435791,
      "activations/layer13_attention_weight_max": 39.11089324951172,
      "activations/layer13_attention_weight_min": -32.73860168457031,
      "activations/layer14_attention_weight_max": 46.07539749145508,
      "activations/layer14_attention_weight_min": -38.042015075683594,
      "activations/layer15_attention_weight_max": 38.28489303588867,
      "activations/layer15_attention_weight_min": -32.2432861328125,
      "activations/layer16_attention_weight_max": 31.823091506958008,
      "activations/layer16_attention_weight_min": -27.700790405273438,
      "activations/layer17_attention_weight_max": 34.28322219848633,
      "activations/layer17_attention_weight_min": -25.718576431274414,
      "activations/layer18_attention_weight_max": 35.33088302612305,
      "activations/layer18_attention_weight_min": -21.585651397705078,
      "activations/layer19_attention_weight_max": 37.26481628417969,
      "activations/layer19_attention_weight_min": -32.40818405151367,
      "activations/layer1_attention_weight_max": 17.24591636657715,
      "activations/layer1_attention_weight_min": -16.274314880371094,
      "activations/layer20_attention_weight_max": 30.359724044799805,
      "activations/layer20_attention_weight_min": -25.043495178222656,
      "activations/layer21_attention_weight_max": 29.09332275390625,
      "activations/layer21_attention_weight_min": -23.573753356933594,
      "activations/layer22_attention_weight_max": 46.61980438232422,
      "activations/layer22_attention_weight_min": -30.864009857177734,
      "activations/layer23_attention_weight_max": 34.55872344970703,
      "activations/layer23_attention_weight_min": -24.08822250366211,
      "activations/layer2_attention_weight_max": 33.20478439331055,
      "activations/layer2_attention_weight_min": -29.75833511352539,
      "activations/layer3_attention_weight_max": 89.27642059326172,
      "activations/layer3_attention_weight_min": -87.19139862060547,
      "activations/layer4_attention_weight_max": 113.57674407958984,
      "activations/layer4_attention_weight_min": -113.65087127685547,
      "activations/layer5_attention_weight_max": 53.50581359863281,
      "activations/layer5_attention_weight_min": -64.65074920654297,
      "activations/layer6_attention_weight_max": 47.65103530883789,
      "activations/layer6_attention_weight_min": -50.297054290771484,
      "activations/layer7_attention_weight_max": 95.16094970703125,
      "activations/layer7_attention_weight_min": -87.1659164428711,
      "activations/layer8_attention_weight_max": 40.49025344848633,
      "activations/layer8_attention_weight_min": -42.925437927246094,
      "activations/layer9_attention_weight_max": 33.29292678833008,
      "activations/layer9_attention_weight_min": -35.615169525146484,
      "epoch": 9.41,
      "learning_rate": 9.022424242424242e-05,
      "loss": 2.7777,
      "step": 161950
    },
    {
      "activations/layer0_attention_weight_max": 16.818920135498047,
      "activations/layer0_attention_weight_min": -12.734614372253418,
      "activations/layer10_attention_weight_max": 36.33185577392578,
      "activations/layer10_attention_weight_min": -37.90376663208008,
      "activations/layer11_attention_weight_max": 37.36897277832031,
      "activations/layer11_attention_weight_min": -35.78678894042969,
      "activations/layer12_attention_weight_max": 26.96565055847168,
      "activations/layer12_attention_weight_min": -31.400165557861328,
      "activations/layer13_attention_weight_max": 43.775150299072266,
      "activations/layer13_attention_weight_min": -35.08991241455078,
      "activations/layer14_attention_weight_max": 56.55860137939453,
      "activations/layer14_attention_weight_min": -42.64295959472656,
      "activations/layer15_attention_weight_max": 40.28239440917969,
      "activations/layer15_attention_weight_min": -30.259014129638672,
      "activations/layer16_attention_weight_max": 36.32516860961914,
      "activations/layer16_attention_weight_min": -29.53474998474121,
      "activations/layer17_attention_weight_max": 36.38753128051758,
      "activations/layer17_attention_weight_min": -25.31111717224121,
      "activations/layer18_attention_weight_max": 32.93600845336914,
      "activations/layer18_attention_weight_min": -22.95149803161621,
      "activations/layer19_attention_weight_max": 36.81106185913086,
      "activations/layer19_attention_weight_min": -30.447904586791992,
      "activations/layer1_attention_weight_max": 21.570770263671875,
      "activations/layer1_attention_weight_min": -21.603670120239258,
      "activations/layer20_attention_weight_max": 29.38515853881836,
      "activations/layer20_attention_weight_min": -24.120420455932617,
      "activations/layer21_attention_weight_max": 29.280370712280273,
      "activations/layer21_attention_weight_min": -23.246835708618164,
      "activations/layer22_attention_weight_max": 46.495849609375,
      "activations/layer22_attention_weight_min": -28.79480743408203,
      "activations/layer23_attention_weight_max": 38.749412536621094,
      "activations/layer23_attention_weight_min": -25.72626495361328,
      "activations/layer2_attention_weight_max": 31.902591705322266,
      "activations/layer2_attention_weight_min": -33.591773986816406,
      "activations/layer3_attention_weight_max": 93.10181427001953,
      "activations/layer3_attention_weight_min": -97.01904296875,
      "activations/layer4_attention_weight_max": 119.3171615600586,
      "activations/layer4_attention_weight_min": -116.69026184082031,
      "activations/layer5_attention_weight_max": 58.424224853515625,
      "activations/layer5_attention_weight_min": -63.925048828125,
      "activations/layer6_attention_weight_max": 50.01769256591797,
      "activations/layer6_attention_weight_min": -51.400699615478516,
      "activations/layer7_attention_weight_max": 97.14401245117188,
      "activations/layer7_attention_weight_min": -94.45117950439453,
      "activations/layer8_attention_weight_max": 44.74299621582031,
      "activations/layer8_attention_weight_min": -46.118770599365234,
      "activations/layer9_attention_weight_max": 37.87101364135742,
      "activations/layer9_attention_weight_min": -35.97331237792969,
      "epoch": 9.41,
      "learning_rate": 9.020530303030302e-05,
      "loss": 2.8072,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4279,
      "eval_samples_per_second": 509.496,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4279,
      "eval_openwebtext_samples_per_second": 509.496,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 1.9155,
      "eval_wikitext_samples_per_second": 238.061,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_lambada_loss": 2.62109375,
      "eval_lambada_ppl": 13.750755251693265,
      "eval_lambada_runtime": 9.5104,
      "eval_lambada_samples_per_second": 511.964,
      "step": 162000
    },
    {
      "activations/layer0_attention_weight_max": 15.221458435058594,
      "activations/layer0_attention_weight_min": -11.832361221313477,
      "activations/layer10_attention_weight_max": 33.634864807128906,
      "activations/layer10_attention_weight_min": -33.39429473876953,
      "activations/layer11_attention_weight_max": 33.86759948730469,
      "activations/layer11_attention_weight_min": -31.07850456237793,
      "activations/layer12_attention_weight_max": 26.69506072998047,
      "activations/layer12_attention_weight_min": -26.180728912353516,
      "activations/layer13_attention_weight_max": 40.8173828125,
      "activations/layer13_attention_weight_min": -34.88970184326172,
      "activations/layer14_attention_weight_max": 52.04854202270508,
      "activations/layer14_attention_weight_min": -42.06003189086914,
      "activations/layer15_attention_weight_max": 40.16978454589844,
      "activations/layer15_attention_weight_min": -31.353166580200195,
      "activations/layer16_attention_weight_max": 35.02751922607422,
      "activations/layer16_attention_weight_min": -26.167814254760742,
      "activations/layer17_attention_weight_max": 34.85820770263672,
      "activations/layer17_attention_weight_min": -23.635494232177734,
      "activations/layer18_attention_weight_max": 30.451271057128906,
      "activations/layer18_attention_weight_min": -21.78244972229004,
      "activations/layer19_attention_weight_max": 33.51945114135742,
      "activations/layer19_attention_weight_min": -27.714826583862305,
      "activations/layer1_attention_weight_max": 16.681440353393555,
      "activations/layer1_attention_weight_min": -13.673240661621094,
      "activations/layer20_attention_weight_max": 27.58244514465332,
      "activations/layer20_attention_weight_min": -22.142637252807617,
      "activations/layer21_attention_weight_max": 26.603580474853516,
      "activations/layer21_attention_weight_min": -20.57200050354004,
      "activations/layer22_attention_weight_max": 46.829654693603516,
      "activations/layer22_attention_weight_min": -28.878395080566406,
      "activations/layer23_attention_weight_max": 34.16449737548828,
      "activations/layer23_attention_weight_min": -23.01895523071289,
      "activations/layer2_attention_weight_max": 30.251178741455078,
      "activations/layer2_attention_weight_min": -28.661300659179688,
      "activations/layer3_attention_weight_max": 85.68646240234375,
      "activations/layer3_attention_weight_min": -87.81033325195312,
      "activations/layer4_attention_weight_max": 114.9953384399414,
      "activations/layer4_attention_weight_min": -112.53034973144531,
      "activations/layer5_attention_weight_max": 55.862728118896484,
      "activations/layer5_attention_weight_min": -63.501277923583984,
      "activations/layer6_attention_weight_max": 48.040138244628906,
      "activations/layer6_attention_weight_min": -49.87158966064453,
      "activations/layer7_attention_weight_max": 89.08658599853516,
      "activations/layer7_attention_weight_min": -92.34659576416016,
      "activations/layer8_attention_weight_max": 41.62049865722656,
      "activations/layer8_attention_weight_min": -42.64808654785156,
      "activations/layer9_attention_weight_max": 37.05247497558594,
      "activations/layer9_attention_weight_min": -35.514347076416016,
      "epoch": 9.42,
      "learning_rate": 9.018636363636364e-05,
      "loss": 2.7655,
      "step": 162050
    },
    {
      "activations/layer0_attention_weight_max": 15.295778274536133,
      "activations/layer0_attention_weight_min": -12.52708911895752,
      "activations/layer10_attention_weight_max": 31.520538330078125,
      "activations/layer10_attention_weight_min": -32.26081848144531,
      "activations/layer11_attention_weight_max": 31.65321159362793,
      "activations/layer11_attention_weight_min": -30.469764709472656,
      "activations/layer12_attention_weight_max": 23.829984664916992,
      "activations/layer12_attention_weight_min": -26.93905258178711,
      "activations/layer13_attention_weight_max": 38.53725814819336,
      "activations/layer13_attention_weight_min": -34.608177185058594,
      "activations/layer14_attention_weight_max": 46.18537139892578,
      "activations/layer14_attention_weight_min": -43.91724395751953,
      "activations/layer15_attention_weight_max": 37.435916900634766,
      "activations/layer15_attention_weight_min": -33.150428771972656,
      "activations/layer16_attention_weight_max": 27.432966232299805,
      "activations/layer16_attention_weight_min": -28.775121688842773,
      "activations/layer17_attention_weight_max": 30.140127182006836,
      "activations/layer17_attention_weight_min": -24.623254776000977,
      "activations/layer18_attention_weight_max": 29.148794174194336,
      "activations/layer18_attention_weight_min": -22.34741973876953,
      "activations/layer19_attention_weight_max": 34.47955322265625,
      "activations/layer19_attention_weight_min": -30.487577438354492,
      "activations/layer1_attention_weight_max": 16.774866104125977,
      "activations/layer1_attention_weight_min": -13.520874977111816,
      "activations/layer20_attention_weight_max": 29.467573165893555,
      "activations/layer20_attention_weight_min": -24.44780158996582,
      "activations/layer21_attention_weight_max": 31.74307632446289,
      "activations/layer21_attention_weight_min": -22.416894912719727,
      "activations/layer22_attention_weight_max": 42.796974182128906,
      "activations/layer22_attention_weight_min": -29.124780654907227,
      "activations/layer23_attention_weight_max": 34.29137420654297,
      "activations/layer23_attention_weight_min": -24.840421676635742,
      "activations/layer2_attention_weight_max": 30.930267333984375,
      "activations/layer2_attention_weight_min": -29.984289169311523,
      "activations/layer3_attention_weight_max": 87.63878631591797,
      "activations/layer3_attention_weight_min": -91.79851531982422,
      "activations/layer4_attention_weight_max": 113.28136444091797,
      "activations/layer4_attention_weight_min": -107.0727767944336,
      "activations/layer5_attention_weight_max": 51.14240264892578,
      "activations/layer5_attention_weight_min": -59.377655029296875,
      "activations/layer6_attention_weight_max": 46.295204162597656,
      "activations/layer6_attention_weight_min": -50.81550979614258,
      "activations/layer7_attention_weight_max": 87.996826171875,
      "activations/layer7_attention_weight_min": -95.46190643310547,
      "activations/layer8_attention_weight_max": 38.209041595458984,
      "activations/layer8_attention_weight_min": -38.363868713378906,
      "activations/layer9_attention_weight_max": 33.98957443237305,
      "activations/layer9_attention_weight_min": -33.36404800415039,
      "epoch": 9.42,
      "learning_rate": 9.016742424242424e-05,
      "loss": 2.8075,
      "step": 162100
    },
    {
      "activations/layer0_attention_weight_max": 15.318700790405273,
      "activations/layer0_attention_weight_min": -12.596855163574219,
      "activations/layer10_attention_weight_max": 35.3404426574707,
      "activations/layer10_attention_weight_min": -36.76708984375,
      "activations/layer11_attention_weight_max": 35.68133544921875,
      "activations/layer11_attention_weight_min": -33.50445556640625,
      "activations/layer12_attention_weight_max": 27.29054069519043,
      "activations/layer12_attention_weight_min": -27.045499801635742,
      "activations/layer13_attention_weight_max": 48.584537506103516,
      "activations/layer13_attention_weight_min": -36.218833923339844,
      "activations/layer14_attention_weight_max": 56.864540100097656,
      "activations/layer14_attention_weight_min": -43.71417236328125,
      "activations/layer15_attention_weight_max": 39.43598175048828,
      "activations/layer15_attention_weight_min": -34.07594299316406,
      "activations/layer16_attention_weight_max": 35.70193862915039,
      "activations/layer16_attention_weight_min": -28.980241775512695,
      "activations/layer17_attention_weight_max": 37.14409255981445,
      "activations/layer17_attention_weight_min": -26.690248489379883,
      "activations/layer18_attention_weight_max": 34.43999481201172,
      "activations/layer18_attention_weight_min": -24.603473663330078,
      "activations/layer19_attention_weight_max": 43.550113677978516,
      "activations/layer19_attention_weight_min": -31.044137954711914,
      "activations/layer1_attention_weight_max": 16.24711799621582,
      "activations/layer1_attention_weight_min": -14.518054962158203,
      "activations/layer20_attention_weight_max": 35.50279235839844,
      "activations/layer20_attention_weight_min": -24.250993728637695,
      "activations/layer21_attention_weight_max": 35.05098342895508,
      "activations/layer21_attention_weight_min": -23.01748275756836,
      "activations/layer22_attention_weight_max": 58.71577453613281,
      "activations/layer22_attention_weight_min": -31.198486328125,
      "activations/layer23_attention_weight_max": 44.756004333496094,
      "activations/layer23_attention_weight_min": -24.313932418823242,
      "activations/layer2_attention_weight_max": 34.877288818359375,
      "activations/layer2_attention_weight_min": -30.7747859954834,
      "activations/layer3_attention_weight_max": 100.58922576904297,
      "activations/layer3_attention_weight_min": -95.7594985961914,
      "activations/layer4_attention_weight_max": 121.43611145019531,
      "activations/layer4_attention_weight_min": -120.21569061279297,
      "activations/layer5_attention_weight_max": 54.991302490234375,
      "activations/layer5_attention_weight_min": -61.45707321166992,
      "activations/layer6_attention_weight_max": 47.956520080566406,
      "activations/layer6_attention_weight_min": -49.33022689819336,
      "activations/layer7_attention_weight_max": 97.39442443847656,
      "activations/layer7_attention_weight_min": -101.21324157714844,
      "activations/layer8_attention_weight_max": 46.66570281982422,
      "activations/layer8_attention_weight_min": -46.05373001098633,
      "activations/layer9_attention_weight_max": 38.76008987426758,
      "activations/layer9_attention_weight_min": -38.66724395751953,
      "epoch": 9.42,
      "learning_rate": 9.014848484848484e-05,
      "loss": 2.7841,
      "step": 162150
    },
    {
      "activations/layer0_attention_weight_max": 15.743976593017578,
      "activations/layer0_attention_weight_min": -12.435388565063477,
      "activations/layer10_attention_weight_max": 32.41969680786133,
      "activations/layer10_attention_weight_min": -33.82211685180664,
      "activations/layer11_attention_weight_max": 32.89618682861328,
      "activations/layer11_attention_weight_min": -32.9769401550293,
      "activations/layer12_attention_weight_max": 24.37218475341797,
      "activations/layer12_attention_weight_min": -26.842220306396484,
      "activations/layer13_attention_weight_max": 37.3335075378418,
      "activations/layer13_attention_weight_min": -32.68896484375,
      "activations/layer14_attention_weight_max": 41.30866241455078,
      "activations/layer14_attention_weight_min": -38.02030563354492,
      "activations/layer15_attention_weight_max": 34.370147705078125,
      "activations/layer15_attention_weight_min": -29.748008728027344,
      "activations/layer16_attention_weight_max": 29.506521224975586,
      "activations/layer16_attention_weight_min": -28.551542282104492,
      "activations/layer17_attention_weight_max": 31.52192497253418,
      "activations/layer17_attention_weight_min": -26.174358367919922,
      "activations/layer18_attention_weight_max": 30.948942184448242,
      "activations/layer18_attention_weight_min": -22.86139488220215,
      "activations/layer19_attention_weight_max": 34.48186492919922,
      "activations/layer19_attention_weight_min": -29.817838668823242,
      "activations/layer1_attention_weight_max": 16.46663475036621,
      "activations/layer1_attention_weight_min": -15.41098403930664,
      "activations/layer20_attention_weight_max": 29.63007164001465,
      "activations/layer20_attention_weight_min": -24.047847747802734,
      "activations/layer21_attention_weight_max": 30.05045509338379,
      "activations/layer21_attention_weight_min": -23.597837448120117,
      "activations/layer22_attention_weight_max": 45.4210319519043,
      "activations/layer22_attention_weight_min": -29.084428787231445,
      "activations/layer23_attention_weight_max": 35.935081481933594,
      "activations/layer23_attention_weight_min": -24.827350616455078,
      "activations/layer2_attention_weight_max": 31.215322494506836,
      "activations/layer2_attention_weight_min": -32.31753158569336,
      "activations/layer3_attention_weight_max": 93.29336547851562,
      "activations/layer3_attention_weight_min": -93.58700561523438,
      "activations/layer4_attention_weight_max": 116.99482727050781,
      "activations/layer4_attention_weight_min": -112.870361328125,
      "activations/layer5_attention_weight_max": 53.38335418701172,
      "activations/layer5_attention_weight_min": -57.3072624206543,
      "activations/layer6_attention_weight_max": 44.947914123535156,
      "activations/layer6_attention_weight_min": -49.24878692626953,
      "activations/layer7_attention_weight_max": 89.86199951171875,
      "activations/layer7_attention_weight_min": -87.14965057373047,
      "activations/layer8_attention_weight_max": 39.653751373291016,
      "activations/layer8_attention_weight_min": -39.404544830322266,
      "activations/layer9_attention_weight_max": 35.4839973449707,
      "activations/layer9_attention_weight_min": -35.06144332885742,
      "epoch": 9.42,
      "learning_rate": 9.012954545454545e-05,
      "loss": 2.8092,
      "step": 162200
    },
    {
      "activations/layer0_attention_weight_max": 15.005802154541016,
      "activations/layer0_attention_weight_min": -12.572157859802246,
      "activations/layer10_attention_weight_max": 35.78860855102539,
      "activations/layer10_attention_weight_min": -35.63565444946289,
      "activations/layer11_attention_weight_max": 36.73095703125,
      "activations/layer11_attention_weight_min": -34.16200256347656,
      "activations/layer12_attention_weight_max": 23.38799285888672,
      "activations/layer12_attention_weight_min": -26.13898468017578,
      "activations/layer13_attention_weight_max": 37.79345703125,
      "activations/layer13_attention_weight_min": -34.04935073852539,
      "activations/layer14_attention_weight_max": 47.61979293823242,
      "activations/layer14_attention_weight_min": -43.1989631652832,
      "activations/layer15_attention_weight_max": 34.36762619018555,
      "activations/layer15_attention_weight_min": -29.55709457397461,
      "activations/layer16_attention_weight_max": 31.406055450439453,
      "activations/layer16_attention_weight_min": -27.025814056396484,
      "activations/layer17_attention_weight_max": 33.21722412109375,
      "activations/layer17_attention_weight_min": -25.837505340576172,
      "activations/layer18_attention_weight_max": 30.513038635253906,
      "activations/layer18_attention_weight_min": -21.527204513549805,
      "activations/layer19_attention_weight_max": 36.0780143737793,
      "activations/layer19_attention_weight_min": -28.58815574645996,
      "activations/layer1_attention_weight_max": 16.831865310668945,
      "activations/layer1_attention_weight_min": -14.439027786254883,
      "activations/layer20_attention_weight_max": 31.703723907470703,
      "activations/layer20_attention_weight_min": -23.759227752685547,
      "activations/layer21_attention_weight_max": 29.311758041381836,
      "activations/layer21_attention_weight_min": -22.049222946166992,
      "activations/layer22_attention_weight_max": 50.55283737182617,
      "activations/layer22_attention_weight_min": -29.41571044921875,
      "activations/layer23_attention_weight_max": 33.608299255371094,
      "activations/layer23_attention_weight_min": -23.16309356689453,
      "activations/layer2_attention_weight_max": 30.51168441772461,
      "activations/layer2_attention_weight_min": -31.369552612304688,
      "activations/layer3_attention_weight_max": 93.61640930175781,
      "activations/layer3_attention_weight_min": -92.82687377929688,
      "activations/layer4_attention_weight_max": 120.4311294555664,
      "activations/layer4_attention_weight_min": -117.27449035644531,
      "activations/layer5_attention_weight_max": 55.33002471923828,
      "activations/layer5_attention_weight_min": -62.514564514160156,
      "activations/layer6_attention_weight_max": 47.54822540283203,
      "activations/layer6_attention_weight_min": -51.047176361083984,
      "activations/layer7_attention_weight_max": 93.32288360595703,
      "activations/layer7_attention_weight_min": -95.28019714355469,
      "activations/layer8_attention_weight_max": 42.22212219238281,
      "activations/layer8_attention_weight_min": -44.47150421142578,
      "activations/layer9_attention_weight_max": 37.60167694091797,
      "activations/layer9_attention_weight_min": -35.21049118041992,
      "epoch": 9.43,
      "learning_rate": 9.011060606060605e-05,
      "loss": 2.7851,
      "step": 162250
    },
    {
      "activations/layer0_attention_weight_max": 15.532756805419922,
      "activations/layer0_attention_weight_min": -12.142823219299316,
      "activations/layer10_attention_weight_max": 31.939483642578125,
      "activations/layer10_attention_weight_min": -33.15271759033203,
      "activations/layer11_attention_weight_max": 32.4115104675293,
      "activations/layer11_attention_weight_min": -32.077049255371094,
      "activations/layer12_attention_weight_max": 23.3204288482666,
      "activations/layer12_attention_weight_min": -25.662817001342773,
      "activations/layer13_attention_weight_max": 37.2188835144043,
      "activations/layer13_attention_weight_min": -32.01443099975586,
      "activations/layer14_attention_weight_max": 51.501399993896484,
      "activations/layer14_attention_weight_min": -39.08357238769531,
      "activations/layer15_attention_weight_max": 35.64562225341797,
      "activations/layer15_attention_weight_min": -30.613906860351562,
      "activations/layer16_attention_weight_max": 31.171403884887695,
      "activations/layer16_attention_weight_min": -27.621675491333008,
      "activations/layer17_attention_weight_max": 35.67447280883789,
      "activations/layer17_attention_weight_min": -26.774822235107422,
      "activations/layer18_attention_weight_max": 30.086271286010742,
      "activations/layer18_attention_weight_min": -22.333484649658203,
      "activations/layer19_attention_weight_max": 33.55674362182617,
      "activations/layer19_attention_weight_min": -30.835988998413086,
      "activations/layer1_attention_weight_max": 17.031452178955078,
      "activations/layer1_attention_weight_min": -15.26902961730957,
      "activations/layer20_attention_weight_max": 31.14293098449707,
      "activations/layer20_attention_weight_min": -26.21221160888672,
      "activations/layer21_attention_weight_max": 30.504262924194336,
      "activations/layer21_attention_weight_min": -24.10007095336914,
      "activations/layer22_attention_weight_max": 44.66740798950195,
      "activations/layer22_attention_weight_min": -29.77779769897461,
      "activations/layer23_attention_weight_max": 32.460262298583984,
      "activations/layer23_attention_weight_min": -24.3253173828125,
      "activations/layer2_attention_weight_max": 30.652494430541992,
      "activations/layer2_attention_weight_min": -30.45685386657715,
      "activations/layer3_attention_weight_max": 88.69258117675781,
      "activations/layer3_attention_weight_min": -88.48582458496094,
      "activations/layer4_attention_weight_max": 115.80126190185547,
      "activations/layer4_attention_weight_min": -115.69222259521484,
      "activations/layer5_attention_weight_max": 55.33297348022461,
      "activations/layer5_attention_weight_min": -64.1120376586914,
      "activations/layer6_attention_weight_max": 47.142452239990234,
      "activations/layer6_attention_weight_min": -49.03385543823242,
      "activations/layer7_attention_weight_max": 90.5003662109375,
      "activations/layer7_attention_weight_min": -89.07523345947266,
      "activations/layer8_attention_weight_max": 40.520538330078125,
      "activations/layer8_attention_weight_min": -39.57386016845703,
      "activations/layer9_attention_weight_max": 34.19526290893555,
      "activations/layer9_attention_weight_min": -34.580963134765625,
      "epoch": 9.43,
      "learning_rate": 9.009166666666666e-05,
      "loss": 2.794,
      "step": 162300
    },
    {
      "activations/layer0_attention_weight_max": 16.11609649658203,
      "activations/layer0_attention_weight_min": -12.249530792236328,
      "activations/layer10_attention_weight_max": 35.18621826171875,
      "activations/layer10_attention_weight_min": -34.845550537109375,
      "activations/layer11_attention_weight_max": 36.564002990722656,
      "activations/layer11_attention_weight_min": -33.97984313964844,
      "activations/layer12_attention_weight_max": 28.96682357788086,
      "activations/layer12_attention_weight_min": -28.321582794189453,
      "activations/layer13_attention_weight_max": 47.50975036621094,
      "activations/layer13_attention_weight_min": -34.83232879638672,
      "activations/layer14_attention_weight_max": 58.26545333862305,
      "activations/layer14_attention_weight_min": -39.42632293701172,
      "activations/layer15_attention_weight_max": 41.07340621948242,
      "activations/layer15_attention_weight_min": -30.498912811279297,
      "activations/layer16_attention_weight_max": 41.098907470703125,
      "activations/layer16_attention_weight_min": -29.41811752319336,
      "activations/layer17_attention_weight_max": 38.10970687866211,
      "activations/layer17_attention_weight_min": -26.59856414794922,
      "activations/layer18_attention_weight_max": 35.46196746826172,
      "activations/layer18_attention_weight_min": -23.68520736694336,
      "activations/layer19_attention_weight_max": 44.18711471557617,
      "activations/layer19_attention_weight_min": -29.94256591796875,
      "activations/layer1_attention_weight_max": 16.817968368530273,
      "activations/layer1_attention_weight_min": -14.720476150512695,
      "activations/layer20_attention_weight_max": 36.12456512451172,
      "activations/layer20_attention_weight_min": -23.525449752807617,
      "activations/layer21_attention_weight_max": 34.27833938598633,
      "activations/layer21_attention_weight_min": -22.174400329589844,
      "activations/layer22_attention_weight_max": 55.40019607543945,
      "activations/layer22_attention_weight_min": -30.519678115844727,
      "activations/layer23_attention_weight_max": 39.787418365478516,
      "activations/layer23_attention_weight_min": -26.027881622314453,
      "activations/layer2_attention_weight_max": 32.52034378051758,
      "activations/layer2_attention_weight_min": -32.040863037109375,
      "activations/layer3_attention_weight_max": 91.1143569946289,
      "activations/layer3_attention_weight_min": -92.34185791015625,
      "activations/layer4_attention_weight_max": 112.37892150878906,
      "activations/layer4_attention_weight_min": -118.21561431884766,
      "activations/layer5_attention_weight_max": 54.91918182373047,
      "activations/layer5_attention_weight_min": -64.95703125,
      "activations/layer6_attention_weight_max": 48.221317291259766,
      "activations/layer6_attention_weight_min": -50.50660705566406,
      "activations/layer7_attention_weight_max": 90.84983825683594,
      "activations/layer7_attention_weight_min": -95.14442443847656,
      "activations/layer8_attention_weight_max": 42.5350227355957,
      "activations/layer8_attention_weight_min": -41.60319137573242,
      "activations/layer9_attention_weight_max": 34.535888671875,
      "activations/layer9_attention_weight_min": -35.32417678833008,
      "epoch": 9.43,
      "learning_rate": 9.007272727272727e-05,
      "loss": 2.7967,
      "step": 162350
    },
    {
      "activations/layer0_attention_weight_max": 15.940413475036621,
      "activations/layer0_attention_weight_min": -11.040741920471191,
      "activations/layer10_attention_weight_max": 31.499601364135742,
      "activations/layer10_attention_weight_min": -31.62358283996582,
      "activations/layer11_attention_weight_max": 32.52671813964844,
      "activations/layer11_attention_weight_min": -31.652172088623047,
      "activations/layer12_attention_weight_max": 24.382431030273438,
      "activations/layer12_attention_weight_min": -26.0253849029541,
      "activations/layer13_attention_weight_max": 41.786834716796875,
      "activations/layer13_attention_weight_min": -36.9182243347168,
      "activations/layer14_attention_weight_max": 52.62248229980469,
      "activations/layer14_attention_weight_min": -44.36450958251953,
      "activations/layer15_attention_weight_max": 36.386688232421875,
      "activations/layer15_attention_weight_min": -32.459388732910156,
      "activations/layer16_attention_weight_max": 31.589067459106445,
      "activations/layer16_attention_weight_min": -29.39389991760254,
      "activations/layer17_attention_weight_max": 32.504825592041016,
      "activations/layer17_attention_weight_min": -26.93440055847168,
      "activations/layer18_attention_weight_max": 31.11260986328125,
      "activations/layer18_attention_weight_min": -22.97393798828125,
      "activations/layer19_attention_weight_max": 39.9688720703125,
      "activations/layer19_attention_weight_min": -36.223426818847656,
      "activations/layer1_attention_weight_max": 16.833581924438477,
      "activations/layer1_attention_weight_min": -14.799039840698242,
      "activations/layer20_attention_weight_max": 31.35734748840332,
      "activations/layer20_attention_weight_min": -25.858924865722656,
      "activations/layer21_attention_weight_max": 30.62422752380371,
      "activations/layer21_attention_weight_min": -23.90195083618164,
      "activations/layer22_attention_weight_max": 52.38997268676758,
      "activations/layer22_attention_weight_min": -32.24566650390625,
      "activations/layer23_attention_weight_max": 38.369293212890625,
      "activations/layer23_attention_weight_min": -26.840360641479492,
      "activations/layer2_attention_weight_max": 30.707927703857422,
      "activations/layer2_attention_weight_min": -31.717700958251953,
      "activations/layer3_attention_weight_max": 94.51887512207031,
      "activations/layer3_attention_weight_min": -95.25365447998047,
      "activations/layer4_attention_weight_max": 124.04634094238281,
      "activations/layer4_attention_weight_min": -119.34020233154297,
      "activations/layer5_attention_weight_max": 56.79206466674805,
      "activations/layer5_attention_weight_min": -64.5215835571289,
      "activations/layer6_attention_weight_max": 48.24710464477539,
      "activations/layer6_attention_weight_min": -48.98076248168945,
      "activations/layer7_attention_weight_max": 88.05609130859375,
      "activations/layer7_attention_weight_min": -90.05635833740234,
      "activations/layer8_attention_weight_max": 43.069454193115234,
      "activations/layer8_attention_weight_min": -42.098201751708984,
      "activations/layer9_attention_weight_max": 31.675207138061523,
      "activations/layer9_attention_weight_min": -33.76569747924805,
      "epoch": 9.44,
      "learning_rate": 9.005378787878786e-05,
      "loss": 2.8028,
      "step": 162400
    },
    {
      "activations/layer0_attention_weight_max": 14.758136749267578,
      "activations/layer0_attention_weight_min": -12.344053268432617,
      "activations/layer10_attention_weight_max": 33.887855529785156,
      "activations/layer10_attention_weight_min": -34.41121292114258,
      "activations/layer11_attention_weight_max": 32.06512451171875,
      "activations/layer11_attention_weight_min": -31.39520263671875,
      "activations/layer12_attention_weight_max": 22.455244064331055,
      "activations/layer12_attention_weight_min": -23.224044799804688,
      "activations/layer13_attention_weight_max": 37.898284912109375,
      "activations/layer13_attention_weight_min": -34.20627212524414,
      "activations/layer14_attention_weight_max": 47.16521072387695,
      "activations/layer14_attention_weight_min": -36.08924102783203,
      "activations/layer15_attention_weight_max": 34.56723403930664,
      "activations/layer15_attention_weight_min": -31.098087310791016,
      "activations/layer16_attention_weight_max": 32.60220718383789,
      "activations/layer16_attention_weight_min": -27.413524627685547,
      "activations/layer17_attention_weight_max": 34.11506652832031,
      "activations/layer17_attention_weight_min": -26.099136352539062,
      "activations/layer18_attention_weight_max": 31.65888786315918,
      "activations/layer18_attention_weight_min": -22.395788192749023,
      "activations/layer19_attention_weight_max": 37.49013900756836,
      "activations/layer19_attention_weight_min": -32.39023971557617,
      "activations/layer1_attention_weight_max": 17.13623809814453,
      "activations/layer1_attention_weight_min": -15.428470611572266,
      "activations/layer20_attention_weight_max": 33.681514739990234,
      "activations/layer20_attention_weight_min": -25.91082763671875,
      "activations/layer21_attention_weight_max": 31.66391372680664,
      "activations/layer21_attention_weight_min": -24.502458572387695,
      "activations/layer22_attention_weight_max": 45.904136657714844,
      "activations/layer22_attention_weight_min": -32.46757507324219,
      "activations/layer23_attention_weight_max": 35.86577606201172,
      "activations/layer23_attention_weight_min": -24.251258850097656,
      "activations/layer2_attention_weight_max": 31.565555572509766,
      "activations/layer2_attention_weight_min": -30.532611846923828,
      "activations/layer3_attention_weight_max": 87.43267822265625,
      "activations/layer3_attention_weight_min": -91.80415344238281,
      "activations/layer4_attention_weight_max": 111.06434631347656,
      "activations/layer4_attention_weight_min": -110.57450103759766,
      "activations/layer5_attention_weight_max": 58.13503646850586,
      "activations/layer5_attention_weight_min": -60.57488250732422,
      "activations/layer6_attention_weight_max": 44.80363464355469,
      "activations/layer6_attention_weight_min": -48.20063018798828,
      "activations/layer7_attention_weight_max": 88.81815338134766,
      "activations/layer7_attention_weight_min": -89.74268341064453,
      "activations/layer8_attention_weight_max": 38.776302337646484,
      "activations/layer8_attention_weight_min": -41.02531814575195,
      "activations/layer9_attention_weight_max": 32.11460494995117,
      "activations/layer9_attention_weight_min": -35.05661392211914,
      "epoch": 9.44,
      "learning_rate": 9.003484848484847e-05,
      "loss": 2.8018,
      "step": 162450
    },
    {
      "activations/layer0_attention_weight_max": 14.776531219482422,
      "activations/layer0_attention_weight_min": -12.089496612548828,
      "activations/layer10_attention_weight_max": 35.60309600830078,
      "activations/layer10_attention_weight_min": -30.533306121826172,
      "activations/layer11_attention_weight_max": 33.1514892578125,
      "activations/layer11_attention_weight_min": -29.215049743652344,
      "activations/layer12_attention_weight_max": 27.557113647460938,
      "activations/layer12_attention_weight_min": -25.13166046142578,
      "activations/layer13_attention_weight_max": 43.333473205566406,
      "activations/layer13_attention_weight_min": -35.45729064941406,
      "activations/layer14_attention_weight_max": 49.606475830078125,
      "activations/layer14_attention_weight_min": -42.57331848144531,
      "activations/layer15_attention_weight_max": 34.72783660888672,
      "activations/layer15_attention_weight_min": -31.093305587768555,
      "activations/layer16_attention_weight_max": 33.082061767578125,
      "activations/layer16_attention_weight_min": -27.83495330810547,
      "activations/layer17_attention_weight_max": 33.31841278076172,
      "activations/layer17_attention_weight_min": -26.352415084838867,
      "activations/layer18_attention_weight_max": 30.777915954589844,
      "activations/layer18_attention_weight_min": -22.465709686279297,
      "activations/layer19_attention_weight_max": 34.43400192260742,
      "activations/layer19_attention_weight_min": -29.77863121032715,
      "activations/layer1_attention_weight_max": 16.71537208557129,
      "activations/layer1_attention_weight_min": -13.803293228149414,
      "activations/layer20_attention_weight_max": 31.451486587524414,
      "activations/layer20_attention_weight_min": -25.1511173248291,
      "activations/layer21_attention_weight_max": 29.653520584106445,
      "activations/layer21_attention_weight_min": -23.1593074798584,
      "activations/layer22_attention_weight_max": 46.404197692871094,
      "activations/layer22_attention_weight_min": -31.437639236450195,
      "activations/layer23_attention_weight_max": 34.92755889892578,
      "activations/layer23_attention_weight_min": -25.440834045410156,
      "activations/layer2_attention_weight_max": 29.871448516845703,
      "activations/layer2_attention_weight_min": -29.601978302001953,
      "activations/layer3_attention_weight_max": 88.54875183105469,
      "activations/layer3_attention_weight_min": -85.9954605102539,
      "activations/layer4_attention_weight_max": 111.30543518066406,
      "activations/layer4_attention_weight_min": -111.60668182373047,
      "activations/layer5_attention_weight_max": 53.091548919677734,
      "activations/layer5_attention_weight_min": -62.678077697753906,
      "activations/layer6_attention_weight_max": 46.34626770019531,
      "activations/layer6_attention_weight_min": -49.17007064819336,
      "activations/layer7_attention_weight_max": 87.71200561523438,
      "activations/layer7_attention_weight_min": -87.59935760498047,
      "activations/layer8_attention_weight_max": 40.316490173339844,
      "activations/layer8_attention_weight_min": -39.609737396240234,
      "activations/layer9_attention_weight_max": 35.61834716796875,
      "activations/layer9_attention_weight_min": -30.8028621673584,
      "epoch": 9.44,
      "learning_rate": 9.001590909090909e-05,
      "loss": 2.8078,
      "step": 162500
    },
    {
      "activations/layer0_attention_weight_max": 15.679903030395508,
      "activations/layer0_attention_weight_min": -12.161075592041016,
      "activations/layer10_attention_weight_max": 35.897010803222656,
      "activations/layer10_attention_weight_min": -34.08526611328125,
      "activations/layer11_attention_weight_max": 33.956993103027344,
      "activations/layer11_attention_weight_min": -33.38374328613281,
      "activations/layer12_attention_weight_max": 22.023563385009766,
      "activations/layer12_attention_weight_min": -26.07892417907715,
      "activations/layer13_attention_weight_max": 37.523826599121094,
      "activations/layer13_attention_weight_min": -32.84133529663086,
      "activations/layer14_attention_weight_max": 43.99200439453125,
      "activations/layer14_attention_weight_min": -37.22148132324219,
      "activations/layer15_attention_weight_max": 32.270450592041016,
      "activations/layer15_attention_weight_min": -29.872400283813477,
      "activations/layer16_attention_weight_max": 31.86437225341797,
      "activations/layer16_attention_weight_min": -26.1960391998291,
      "activations/layer17_attention_weight_max": 29.381818771362305,
      "activations/layer17_attention_weight_min": -25.550779342651367,
      "activations/layer18_attention_weight_max": 30.76958656311035,
      "activations/layer18_attention_weight_min": -21.310720443725586,
      "activations/layer19_attention_weight_max": 34.71112060546875,
      "activations/layer19_attention_weight_min": -32.395721435546875,
      "activations/layer1_attention_weight_max": 16.296329498291016,
      "activations/layer1_attention_weight_min": -14.705331802368164,
      "activations/layer20_attention_weight_max": 28.494173049926758,
      "activations/layer20_attention_weight_min": -23.578086853027344,
      "activations/layer21_attention_weight_max": 28.575040817260742,
      "activations/layer21_attention_weight_min": -24.465124130249023,
      "activations/layer22_attention_weight_max": 45.31245803833008,
      "activations/layer22_attention_weight_min": -28.925914764404297,
      "activations/layer23_attention_weight_max": 34.06841278076172,
      "activations/layer23_attention_weight_min": -26.145902633666992,
      "activations/layer2_attention_weight_max": 31.32467269897461,
      "activations/layer2_attention_weight_min": -30.675947189331055,
      "activations/layer3_attention_weight_max": 92.39641571044922,
      "activations/layer3_attention_weight_min": -91.09634399414062,
      "activations/layer4_attention_weight_max": 115.59569549560547,
      "activations/layer4_attention_weight_min": -114.61687469482422,
      "activations/layer5_attention_weight_max": 54.27875518798828,
      "activations/layer5_attention_weight_min": -63.386566162109375,
      "activations/layer6_attention_weight_max": 47.118492126464844,
      "activations/layer6_attention_weight_min": -49.37018585205078,
      "activations/layer7_attention_weight_max": 90.78570556640625,
      "activations/layer7_attention_weight_min": -92.99077606201172,
      "activations/layer8_attention_weight_max": 42.6795768737793,
      "activations/layer8_attention_weight_min": -44.926231384277344,
      "activations/layer9_attention_weight_max": 37.427059173583984,
      "activations/layer9_attention_weight_min": -35.580955505371094,
      "epoch": 9.45,
      "learning_rate": 8.999696969696968e-05,
      "loss": 2.8068,
      "step": 162550
    },
    {
      "activations/layer0_attention_weight_max": 16.112245559692383,
      "activations/layer0_attention_weight_min": -11.714254379272461,
      "activations/layer10_attention_weight_max": 35.671607971191406,
      "activations/layer10_attention_weight_min": -32.41745376586914,
      "activations/layer11_attention_weight_max": 37.63935089111328,
      "activations/layer11_attention_weight_min": -34.113582611083984,
      "activations/layer12_attention_weight_max": 24.548730850219727,
      "activations/layer12_attention_weight_min": -30.41636848449707,
      "activations/layer13_attention_weight_max": 47.94041442871094,
      "activations/layer13_attention_weight_min": -36.706077575683594,
      "activations/layer14_attention_weight_max": 50.012088775634766,
      "activations/layer14_attention_weight_min": -40.8475227355957,
      "activations/layer15_attention_weight_max": 45.09943771362305,
      "activations/layer15_attention_weight_min": -34.0675163269043,
      "activations/layer16_attention_weight_max": 30.019636154174805,
      "activations/layer16_attention_weight_min": -27.240568161010742,
      "activations/layer17_attention_weight_max": 32.79989242553711,
      "activations/layer17_attention_weight_min": -24.996110916137695,
      "activations/layer18_attention_weight_max": 34.3195686340332,
      "activations/layer18_attention_weight_min": -22.553640365600586,
      "activations/layer19_attention_weight_max": 33.60055160522461,
      "activations/layer19_attention_weight_min": -28.31477928161621,
      "activations/layer1_attention_weight_max": 18.0123291015625,
      "activations/layer1_attention_weight_min": -14.741015434265137,
      "activations/layer20_attention_weight_max": 29.25072479248047,
      "activations/layer20_attention_weight_min": -22.587934494018555,
      "activations/layer21_attention_weight_max": 31.16020965576172,
      "activations/layer21_attention_weight_min": -21.22760009765625,
      "activations/layer22_attention_weight_max": 44.057498931884766,
      "activations/layer22_attention_weight_min": -26.10947036743164,
      "activations/layer23_attention_weight_max": 34.0216064453125,
      "activations/layer23_attention_weight_min": -22.84661865234375,
      "activations/layer2_attention_weight_max": 30.51825714111328,
      "activations/layer2_attention_weight_min": -32.104652404785156,
      "activations/layer3_attention_weight_max": 91.69161987304688,
      "activations/layer3_attention_weight_min": -95.6771011352539,
      "activations/layer4_attention_weight_max": 116.5042495727539,
      "activations/layer4_attention_weight_min": -119.36102294921875,
      "activations/layer5_attention_weight_max": 53.3834342956543,
      "activations/layer5_attention_weight_min": -64.708740234375,
      "activations/layer6_attention_weight_max": 46.2811393737793,
      "activations/layer6_attention_weight_min": -48.43087387084961,
      "activations/layer7_attention_weight_max": 92.91293334960938,
      "activations/layer7_attention_weight_min": -89.93946838378906,
      "activations/layer8_attention_weight_max": 41.289756774902344,
      "activations/layer8_attention_weight_min": -40.020912170410156,
      "activations/layer9_attention_weight_max": 34.217647552490234,
      "activations/layer9_attention_weight_min": -33.04645919799805,
      "epoch": 9.45,
      "learning_rate": 8.997803030303029e-05,
      "loss": 2.8019,
      "step": 162600
    },
    {
      "activations/layer0_attention_weight_max": 15.44405460357666,
      "activations/layer0_attention_weight_min": -12.379240989685059,
      "activations/layer10_attention_weight_max": 36.31690216064453,
      "activations/layer10_attention_weight_min": -32.81916427612305,
      "activations/layer11_attention_weight_max": 37.76495361328125,
      "activations/layer11_attention_weight_min": -32.931846618652344,
      "activations/layer12_attention_weight_max": 24.6193904876709,
      "activations/layer12_attention_weight_min": -26.33626937866211,
      "activations/layer13_attention_weight_max": 46.501399993896484,
      "activations/layer13_attention_weight_min": -33.433555603027344,
      "activations/layer14_attention_weight_max": 50.649532318115234,
      "activations/layer14_attention_weight_min": -41.623321533203125,
      "activations/layer15_attention_weight_max": 44.497589111328125,
      "activations/layer15_attention_weight_min": -31.886184692382812,
      "activations/layer16_attention_weight_max": 32.65726089477539,
      "activations/layer16_attention_weight_min": -28.941152572631836,
      "activations/layer17_attention_weight_max": 35.2299690246582,
      "activations/layer17_attention_weight_min": -26.958999633789062,
      "activations/layer18_attention_weight_max": 31.709196090698242,
      "activations/layer18_attention_weight_min": -23.270742416381836,
      "activations/layer19_attention_weight_max": 36.73493194580078,
      "activations/layer19_attention_weight_min": -30.76968765258789,
      "activations/layer1_attention_weight_max": 16.639381408691406,
      "activations/layer1_attention_weight_min": -13.85401725769043,
      "activations/layer20_attention_weight_max": 32.93635559082031,
      "activations/layer20_attention_weight_min": -25.604711532592773,
      "activations/layer21_attention_weight_max": 31.910057067871094,
      "activations/layer21_attention_weight_min": -22.99562644958496,
      "activations/layer22_attention_weight_max": 42.139469146728516,
      "activations/layer22_attention_weight_min": -32.859127044677734,
      "activations/layer23_attention_weight_max": 37.831878662109375,
      "activations/layer23_attention_weight_min": -24.80562973022461,
      "activations/layer2_attention_weight_max": 33.523780822753906,
      "activations/layer2_attention_weight_min": -31.48019027709961,
      "activations/layer3_attention_weight_max": 96.33039855957031,
      "activations/layer3_attention_weight_min": -98.6996078491211,
      "activations/layer4_attention_weight_max": 114.2380599975586,
      "activations/layer4_attention_weight_min": -122.8498306274414,
      "activations/layer5_attention_weight_max": 54.679439544677734,
      "activations/layer5_attention_weight_min": -61.075767517089844,
      "activations/layer6_attention_weight_max": 46.58035659790039,
      "activations/layer6_attention_weight_min": -50.611854553222656,
      "activations/layer7_attention_weight_max": 90.56653594970703,
      "activations/layer7_attention_weight_min": -96.03599548339844,
      "activations/layer8_attention_weight_max": 42.9954719543457,
      "activations/layer8_attention_weight_min": -42.866233825683594,
      "activations/layer9_attention_weight_max": 37.2099609375,
      "activations/layer9_attention_weight_min": -36.935935974121094,
      "epoch": 9.45,
      "learning_rate": 8.995909090909091e-05,
      "loss": 2.772,
      "step": 162650
    },
    {
      "activations/layer0_attention_weight_max": 15.374789237976074,
      "activations/layer0_attention_weight_min": -12.194507598876953,
      "activations/layer10_attention_weight_max": 32.660743713378906,
      "activations/layer10_attention_weight_min": -30.873245239257812,
      "activations/layer11_attention_weight_max": 31.225311279296875,
      "activations/layer11_attention_weight_min": -31.243568420410156,
      "activations/layer12_attention_weight_max": 21.788328170776367,
      "activations/layer12_attention_weight_min": -25.22458267211914,
      "activations/layer13_attention_weight_max": 37.54084396362305,
      "activations/layer13_attention_weight_min": -34.022945404052734,
      "activations/layer14_attention_weight_max": 44.76348876953125,
      "activations/layer14_attention_weight_min": -37.8323860168457,
      "activations/layer15_attention_weight_max": 32.8992805480957,
      "activations/layer15_attention_weight_min": -29.36313247680664,
      "activations/layer16_attention_weight_max": 29.54701805114746,
      "activations/layer16_attention_weight_min": -28.718481063842773,
      "activations/layer17_attention_weight_max": 30.733816146850586,
      "activations/layer17_attention_weight_min": -25.72937774658203,
      "activations/layer18_attention_weight_max": 30.217931747436523,
      "activations/layer18_attention_weight_min": -25.0484676361084,
      "activations/layer19_attention_weight_max": 35.002193450927734,
      "activations/layer19_attention_weight_min": -30.97174644470215,
      "activations/layer1_attention_weight_max": 15.768061637878418,
      "activations/layer1_attention_weight_min": -15.307584762573242,
      "activations/layer20_attention_weight_max": 28.456212997436523,
      "activations/layer20_attention_weight_min": -25.42983055114746,
      "activations/layer21_attention_weight_max": 28.540573120117188,
      "activations/layer21_attention_weight_min": -24.080888748168945,
      "activations/layer22_attention_weight_max": 43.91131591796875,
      "activations/layer22_attention_weight_min": -30.211795806884766,
      "activations/layer23_attention_weight_max": 36.34941482543945,
      "activations/layer23_attention_weight_min": -25.689483642578125,
      "activations/layer2_attention_weight_max": 34.922515869140625,
      "activations/layer2_attention_weight_min": -32.328216552734375,
      "activations/layer3_attention_weight_max": 96.37125396728516,
      "activations/layer3_attention_weight_min": -94.03536987304688,
      "activations/layer4_attention_weight_max": 123.97689056396484,
      "activations/layer4_attention_weight_min": -115.29046630859375,
      "activations/layer5_attention_weight_max": 55.890464782714844,
      "activations/layer5_attention_weight_min": -67.264892578125,
      "activations/layer6_attention_weight_max": 47.7328987121582,
      "activations/layer6_attention_weight_min": -51.50896453857422,
      "activations/layer7_attention_weight_max": 87.1288070678711,
      "activations/layer7_attention_weight_min": -89.9693832397461,
      "activations/layer8_attention_weight_max": 38.78140640258789,
      "activations/layer8_attention_weight_min": -40.153770446777344,
      "activations/layer9_attention_weight_max": 33.56877899169922,
      "activations/layer9_attention_weight_min": -32.87772750854492,
      "epoch": 9.45,
      "learning_rate": 8.994015151515151e-05,
      "loss": 2.7911,
      "step": 162700
    },
    {
      "activations/layer0_attention_weight_max": 15.470582962036133,
      "activations/layer0_attention_weight_min": -11.709478378295898,
      "activations/layer10_attention_weight_max": 36.08695602416992,
      "activations/layer10_attention_weight_min": -34.00041580200195,
      "activations/layer11_attention_weight_max": 34.75735855102539,
      "activations/layer11_attention_weight_min": -32.417701721191406,
      "activations/layer12_attention_weight_max": 24.65639877319336,
      "activations/layer12_attention_weight_min": -25.05368995666504,
      "activations/layer13_attention_weight_max": 42.67140579223633,
      "activations/layer13_attention_weight_min": -34.93779373168945,
      "activations/layer14_attention_weight_max": 56.82316970825195,
      "activations/layer14_attention_weight_min": -42.543052673339844,
      "activations/layer15_attention_weight_max": 38.727840423583984,
      "activations/layer15_attention_weight_min": -31.311922073364258,
      "activations/layer16_attention_weight_max": 35.3297004699707,
      "activations/layer16_attention_weight_min": -26.535837173461914,
      "activations/layer17_attention_weight_max": 33.7263298034668,
      "activations/layer17_attention_weight_min": -25.579967498779297,
      "activations/layer18_attention_weight_max": 30.290508270263672,
      "activations/layer18_attention_weight_min": -21.19426155090332,
      "activations/layer19_attention_weight_max": 35.952850341796875,
      "activations/layer19_attention_weight_min": -30.61635398864746,
      "activations/layer1_attention_weight_max": 15.643437385559082,
      "activations/layer1_attention_weight_min": -15.002105712890625,
      "activations/layer20_attention_weight_max": 32.84532928466797,
      "activations/layer20_attention_weight_min": -23.797643661499023,
      "activations/layer21_attention_weight_max": 30.657268524169922,
      "activations/layer21_attention_weight_min": -23.670211791992188,
      "activations/layer22_attention_weight_max": 50.0135498046875,
      "activations/layer22_attention_weight_min": -27.914833068847656,
      "activations/layer23_attention_weight_max": 37.66126251220703,
      "activations/layer23_attention_weight_min": -26.67091941833496,
      "activations/layer2_attention_weight_max": 30.101764678955078,
      "activations/layer2_attention_weight_min": -31.989234924316406,
      "activations/layer3_attention_weight_max": 91.85868835449219,
      "activations/layer3_attention_weight_min": -92.48612976074219,
      "activations/layer4_attention_weight_max": 117.92677307128906,
      "activations/layer4_attention_weight_min": -114.59224700927734,
      "activations/layer5_attention_weight_max": 53.57941436767578,
      "activations/layer5_attention_weight_min": -61.76519012451172,
      "activations/layer6_attention_weight_max": 45.32749557495117,
      "activations/layer6_attention_weight_min": -48.46361541748047,
      "activations/layer7_attention_weight_max": 94.38008117675781,
      "activations/layer7_attention_weight_min": -91.2568588256836,
      "activations/layer8_attention_weight_max": 40.13069152832031,
      "activations/layer8_attention_weight_min": -42.511192321777344,
      "activations/layer9_attention_weight_max": 35.0605583190918,
      "activations/layer9_attention_weight_min": -33.50824737548828,
      "epoch": 9.46,
      "learning_rate": 8.992121212121211e-05,
      "loss": 2.7884,
      "step": 162750
    },
    {
      "activations/layer0_attention_weight_max": 15.301645278930664,
      "activations/layer0_attention_weight_min": -12.499810218811035,
      "activations/layer10_attention_weight_max": 31.743911743164062,
      "activations/layer10_attention_weight_min": -32.2689094543457,
      "activations/layer11_attention_weight_max": 32.63928985595703,
      "activations/layer11_attention_weight_min": -31.373640060424805,
      "activations/layer12_attention_weight_max": 22.45214080810547,
      "activations/layer12_attention_weight_min": -22.96660614013672,
      "activations/layer13_attention_weight_max": 37.985023498535156,
      "activations/layer13_attention_weight_min": -32.54774475097656,
      "activations/layer14_attention_weight_max": 47.454833984375,
      "activations/layer14_attention_weight_min": -39.011070251464844,
      "activations/layer15_attention_weight_max": 36.356388092041016,
      "activations/layer15_attention_weight_min": -29.680204391479492,
      "activations/layer16_attention_weight_max": 34.19727325439453,
      "activations/layer16_attention_weight_min": -28.70733070373535,
      "activations/layer17_attention_weight_max": 37.09413528442383,
      "activations/layer17_attention_weight_min": -24.946218490600586,
      "activations/layer18_attention_weight_max": 32.80116271972656,
      "activations/layer18_attention_weight_min": -21.965167999267578,
      "activations/layer19_attention_weight_max": 40.54586410522461,
      "activations/layer19_attention_weight_min": -33.85676574707031,
      "activations/layer1_attention_weight_max": 16.97799301147461,
      "activations/layer1_attention_weight_min": -14.386375427246094,
      "activations/layer20_attention_weight_max": 32.01350784301758,
      "activations/layer20_attention_weight_min": -24.734228134155273,
      "activations/layer21_attention_weight_max": 32.27490234375,
      "activations/layer21_attention_weight_min": -23.42146873474121,
      "activations/layer22_attention_weight_max": 46.90718460083008,
      "activations/layer22_attention_weight_min": -29.32642364501953,
      "activations/layer23_attention_weight_max": 37.708282470703125,
      "activations/layer23_attention_weight_min": -23.687728881835938,
      "activations/layer2_attention_weight_max": 31.6284236907959,
      "activations/layer2_attention_weight_min": -32.44915771484375,
      "activations/layer3_attention_weight_max": 90.61652374267578,
      "activations/layer3_attention_weight_min": -95.6922836303711,
      "activations/layer4_attention_weight_max": 115.4674072265625,
      "activations/layer4_attention_weight_min": -114.98571014404297,
      "activations/layer5_attention_weight_max": 53.97331237792969,
      "activations/layer5_attention_weight_min": -62.582515716552734,
      "activations/layer6_attention_weight_max": 45.70236587524414,
      "activations/layer6_attention_weight_min": -49.083858489990234,
      "activations/layer7_attention_weight_max": 89.56839752197266,
      "activations/layer7_attention_weight_min": -93.25045013427734,
      "activations/layer8_attention_weight_max": 40.05976867675781,
      "activations/layer8_attention_weight_min": -40.70259475708008,
      "activations/layer9_attention_weight_max": 35.31398391723633,
      "activations/layer9_attention_weight_min": -34.18418884277344,
      "epoch": 9.46,
      "learning_rate": 8.990227272727271e-05,
      "loss": 2.8015,
      "step": 162800
    },
    {
      "activations/layer0_attention_weight_max": 15.257104873657227,
      "activations/layer0_attention_weight_min": -11.685498237609863,
      "activations/layer10_attention_weight_max": 32.022125244140625,
      "activations/layer10_attention_weight_min": -32.51912307739258,
      "activations/layer11_attention_weight_max": 32.133949279785156,
      "activations/layer11_attention_weight_min": -30.38081932067871,
      "activations/layer12_attention_weight_max": 22.059036254882812,
      "activations/layer12_attention_weight_min": -28.729143142700195,
      "activations/layer13_attention_weight_max": 34.60734176635742,
      "activations/layer13_attention_weight_min": -32.418914794921875,
      "activations/layer14_attention_weight_max": 41.19491195678711,
      "activations/layer14_attention_weight_min": -38.2133903503418,
      "activations/layer15_attention_weight_max": 32.423866271972656,
      "activations/layer15_attention_weight_min": -30.695091247558594,
      "activations/layer16_attention_weight_max": 29.43442726135254,
      "activations/layer16_attention_weight_min": -27.61465072631836,
      "activations/layer17_attention_weight_max": 31.047340393066406,
      "activations/layer17_attention_weight_min": -25.15169334411621,
      "activations/layer18_attention_weight_max": 28.85576057434082,
      "activations/layer18_attention_weight_min": -23.335933685302734,
      "activations/layer19_attention_weight_max": 35.63393020629883,
      "activations/layer19_attention_weight_min": -28.195663452148438,
      "activations/layer1_attention_weight_max": 16.38196563720703,
      "activations/layer1_attention_weight_min": -14.339035034179688,
      "activations/layer20_attention_weight_max": 27.928709030151367,
      "activations/layer20_attention_weight_min": -25.149274826049805,
      "activations/layer21_attention_weight_max": 28.169897079467773,
      "activations/layer21_attention_weight_min": -24.626224517822266,
      "activations/layer22_attention_weight_max": 46.0158576965332,
      "activations/layer22_attention_weight_min": -29.538707733154297,
      "activations/layer23_attention_weight_max": 40.109317779541016,
      "activations/layer23_attention_weight_min": -22.374807357788086,
      "activations/layer2_attention_weight_max": 31.52107048034668,
      "activations/layer2_attention_weight_min": -31.846599578857422,
      "activations/layer3_attention_weight_max": 92.17971801757812,
      "activations/layer3_attention_weight_min": -95.08541107177734,
      "activations/layer4_attention_weight_max": 118.60426330566406,
      "activations/layer4_attention_weight_min": -115.98970031738281,
      "activations/layer5_attention_weight_max": 57.11491394042969,
      "activations/layer5_attention_weight_min": -62.22783660888672,
      "activations/layer6_attention_weight_max": 49.92213821411133,
      "activations/layer6_attention_weight_min": -51.38867950439453,
      "activations/layer7_attention_weight_max": 88.05988311767578,
      "activations/layer7_attention_weight_min": -91.29023742675781,
      "activations/layer8_attention_weight_max": 40.82398223876953,
      "activations/layer8_attention_weight_min": -43.25310516357422,
      "activations/layer9_attention_weight_max": 34.3315544128418,
      "activations/layer9_attention_weight_min": -34.05015563964844,
      "epoch": 9.46,
      "learning_rate": 8.988333333333333e-05,
      "loss": 2.7963,
      "step": 162850
    },
    {
      "activations/layer0_attention_weight_max": 14.284908294677734,
      "activations/layer0_attention_weight_min": -11.504626274108887,
      "activations/layer10_attention_weight_max": 35.68558883666992,
      "activations/layer10_attention_weight_min": -32.890380859375,
      "activations/layer11_attention_weight_max": 34.823829650878906,
      "activations/layer11_attention_weight_min": -32.35205078125,
      "activations/layer12_attention_weight_max": 25.65000343322754,
      "activations/layer12_attention_weight_min": -26.011011123657227,
      "activations/layer13_attention_weight_max": 44.48705291748047,
      "activations/layer13_attention_weight_min": -33.039180755615234,
      "activations/layer14_attention_weight_max": 58.620826721191406,
      "activations/layer14_attention_weight_min": -44.6612434387207,
      "activations/layer15_attention_weight_max": 41.90461349487305,
      "activations/layer15_attention_weight_min": -31.144392013549805,
      "activations/layer16_attention_weight_max": 36.05617904663086,
      "activations/layer16_attention_weight_min": -28.173561096191406,
      "activations/layer17_attention_weight_max": 35.80961227416992,
      "activations/layer17_attention_weight_min": -25.978174209594727,
      "activations/layer18_attention_weight_max": 35.734657287597656,
      "activations/layer18_attention_weight_min": -22.38010597229004,
      "activations/layer19_attention_weight_max": 41.459590911865234,
      "activations/layer19_attention_weight_min": -29.843708038330078,
      "activations/layer1_attention_weight_max": 16.22849464416504,
      "activations/layer1_attention_weight_min": -15.01650333404541,
      "activations/layer20_attention_weight_max": 34.11314392089844,
      "activations/layer20_attention_weight_min": -24.195785522460938,
      "activations/layer21_attention_weight_max": 34.28752899169922,
      "activations/layer21_attention_weight_min": -23.067092895507812,
      "activations/layer22_attention_weight_max": 55.04671096801758,
      "activations/layer22_attention_weight_min": -29.472808837890625,
      "activations/layer23_attention_weight_max": 38.83034133911133,
      "activations/layer23_attention_weight_min": -23.2968807220459,
      "activations/layer2_attention_weight_max": 32.234771728515625,
      "activations/layer2_attention_weight_min": -33.78657150268555,
      "activations/layer3_attention_weight_max": 94.94469451904297,
      "activations/layer3_attention_weight_min": -100.2811508178711,
      "activations/layer4_attention_weight_max": 120.36937713623047,
      "activations/layer4_attention_weight_min": -120.16263580322266,
      "activations/layer5_attention_weight_max": 53.60594940185547,
      "activations/layer5_attention_weight_min": -63.691139221191406,
      "activations/layer6_attention_weight_max": 47.558589935302734,
      "activations/layer6_attention_weight_min": -51.31631088256836,
      "activations/layer7_attention_weight_max": 91.29114532470703,
      "activations/layer7_attention_weight_min": -91.70247650146484,
      "activations/layer8_attention_weight_max": 41.671836853027344,
      "activations/layer8_attention_weight_min": -42.495872497558594,
      "activations/layer9_attention_weight_max": 36.501155853271484,
      "activations/layer9_attention_weight_min": -34.12495040893555,
      "epoch": 9.47,
      "learning_rate": 8.986439393939394e-05,
      "loss": 2.7886,
      "step": 162900
    },
    {
      "activations/layer0_attention_weight_max": 16.631134033203125,
      "activations/layer0_attention_weight_min": -12.12178897857666,
      "activations/layer10_attention_weight_max": 33.922359466552734,
      "activations/layer10_attention_weight_min": -34.80652618408203,
      "activations/layer11_attention_weight_max": 32.61449432373047,
      "activations/layer11_attention_weight_min": -31.306560516357422,
      "activations/layer12_attention_weight_max": 22.9521427154541,
      "activations/layer12_attention_weight_min": -34.44352340698242,
      "activations/layer13_attention_weight_max": 34.8701171875,
      "activations/layer13_attention_weight_min": -32.23052215576172,
      "activations/layer14_attention_weight_max": 42.44310760498047,
      "activations/layer14_attention_weight_min": -38.13835144042969,
      "activations/layer15_attention_weight_max": 35.85956573486328,
      "activations/layer15_attention_weight_min": -29.432043075561523,
      "activations/layer16_attention_weight_max": 34.56426239013672,
      "activations/layer16_attention_weight_min": -26.1689510345459,
      "activations/layer17_attention_weight_max": 35.30712127685547,
      "activations/layer17_attention_weight_min": -23.890499114990234,
      "activations/layer18_attention_weight_max": 33.306068420410156,
      "activations/layer18_attention_weight_min": -20.94327163696289,
      "activations/layer19_attention_weight_max": 35.4402961730957,
      "activations/layer19_attention_weight_min": -31.032926559448242,
      "activations/layer1_attention_weight_max": 17.320707321166992,
      "activations/layer1_attention_weight_min": -14.07107162475586,
      "activations/layer20_attention_weight_max": 29.8403377532959,
      "activations/layer20_attention_weight_min": -26.460399627685547,
      "activations/layer21_attention_weight_max": 30.056987762451172,
      "activations/layer21_attention_weight_min": -22.553794860839844,
      "activations/layer22_attention_weight_max": 49.74009704589844,
      "activations/layer22_attention_weight_min": -29.775468826293945,
      "activations/layer23_attention_weight_max": 34.550621032714844,
      "activations/layer23_attention_weight_min": -22.994667053222656,
      "activations/layer2_attention_weight_max": 32.192039489746094,
      "activations/layer2_attention_weight_min": -32.933990478515625,
      "activations/layer3_attention_weight_max": 94.74407196044922,
      "activations/layer3_attention_weight_min": -100.59888458251953,
      "activations/layer4_attention_weight_max": 118.0787353515625,
      "activations/layer4_attention_weight_min": -118.86418914794922,
      "activations/layer5_attention_weight_max": 56.29249572753906,
      "activations/layer5_attention_weight_min": -63.99113464355469,
      "activations/layer6_attention_weight_max": 46.11294937133789,
      "activations/layer6_attention_weight_min": -49.70558166503906,
      "activations/layer7_attention_weight_max": 93.96044921875,
      "activations/layer7_attention_weight_min": -93.90428161621094,
      "activations/layer8_attention_weight_max": 40.47953414916992,
      "activations/layer8_attention_weight_min": -42.09196090698242,
      "activations/layer9_attention_weight_max": 32.39109420776367,
      "activations/layer9_attention_weight_min": -35.100929260253906,
      "epoch": 9.47,
      "learning_rate": 8.984545454545453e-05,
      "loss": 2.7933,
      "step": 162950
    },
    {
      "activations/layer0_attention_weight_max": 15.370176315307617,
      "activations/layer0_attention_weight_min": -12.39842414855957,
      "activations/layer10_attention_weight_max": 33.63804244995117,
      "activations/layer10_attention_weight_min": -31.049808502197266,
      "activations/layer11_attention_weight_max": 32.73029708862305,
      "activations/layer11_attention_weight_min": -31.96657943725586,
      "activations/layer12_attention_weight_max": 29.00653839111328,
      "activations/layer12_attention_weight_min": -26.046960830688477,
      "activations/layer13_attention_weight_max": 43.0169563293457,
      "activations/layer13_attention_weight_min": -36.89620590209961,
      "activations/layer14_attention_weight_max": 58.00230026245117,
      "activations/layer14_attention_weight_min": -43.457740783691406,
      "activations/layer15_attention_weight_max": 44.80878448486328,
      "activations/layer15_attention_weight_min": -32.5949592590332,
      "activations/layer16_attention_weight_max": 34.53875732421875,
      "activations/layer16_attention_weight_min": -27.550317764282227,
      "activations/layer17_attention_weight_max": 34.14217758178711,
      "activations/layer17_attention_weight_min": -25.128196716308594,
      "activations/layer18_attention_weight_max": 32.4295768737793,
      "activations/layer18_attention_weight_min": -22.078981399536133,
      "activations/layer19_attention_weight_max": 36.528133392333984,
      "activations/layer19_attention_weight_min": -30.51002311706543,
      "activations/layer1_attention_weight_max": 16.640758514404297,
      "activations/layer1_attention_weight_min": -12.652034759521484,
      "activations/layer20_attention_weight_max": 33.55390548706055,
      "activations/layer20_attention_weight_min": -23.640666961669922,
      "activations/layer21_attention_weight_max": 29.86356544494629,
      "activations/layer21_attention_weight_min": -22.228931427001953,
      "activations/layer22_attention_weight_max": 44.555503845214844,
      "activations/layer22_attention_weight_min": -30.15760612487793,
      "activations/layer23_attention_weight_max": 33.915489196777344,
      "activations/layer23_attention_weight_min": -23.442947387695312,
      "activations/layer2_attention_weight_max": 30.357772827148438,
      "activations/layer2_attention_weight_min": -30.226028442382812,
      "activations/layer3_attention_weight_max": 88.94184112548828,
      "activations/layer3_attention_weight_min": -86.36245727539062,
      "activations/layer4_attention_weight_max": 115.60674285888672,
      "activations/layer4_attention_weight_min": -112.1918716430664,
      "activations/layer5_attention_weight_max": 56.653106689453125,
      "activations/layer5_attention_weight_min": -63.69865036010742,
      "activations/layer6_attention_weight_max": 46.51322937011719,
      "activations/layer6_attention_weight_min": -51.975955963134766,
      "activations/layer7_attention_weight_max": 90.6908950805664,
      "activations/layer7_attention_weight_min": -87.6484146118164,
      "activations/layer8_attention_weight_max": 40.24031448364258,
      "activations/layer8_attention_weight_min": -38.92008590698242,
      "activations/layer9_attention_weight_max": 34.76470947265625,
      "activations/layer9_attention_weight_min": -33.100502014160156,
      "epoch": 9.47,
      "learning_rate": 8.982689393939392e-05,
      "loss": 2.8035,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_loss": 2.75,
      "eval_runtime": 8.4318,
      "eval_samples_per_second": 509.261,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.4318,
      "eval_openwebtext_samples_per_second": 509.261,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 1.9286,
      "eval_wikitext_samples_per_second": 236.442,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5206,
      "eval_lambada_samples_per_second": 511.418,
      "step": 163000
    },
    {
      "activations/layer0_attention_weight_max": 15.197273254394531,
      "activations/layer0_attention_weight_min": -12.261634826660156,
      "activations/layer10_attention_weight_max": 37.32550048828125,
      "activations/layer10_attention_weight_min": -31.771976470947266,
      "activations/layer11_attention_weight_max": 36.73219299316406,
      "activations/layer11_attention_weight_min": -30.987348556518555,
      "activations/layer12_attention_weight_max": 24.88909149169922,
      "activations/layer12_attention_weight_min": -24.42103385925293,
      "activations/layer13_attention_weight_max": 41.08797073364258,
      "activations/layer13_attention_weight_min": -34.185340881347656,
      "activations/layer14_attention_weight_max": 50.239105224609375,
      "activations/layer14_attention_weight_min": -38.187103271484375,
      "activations/layer15_attention_weight_max": 39.14200210571289,
      "activations/layer15_attention_weight_min": -31.708728790283203,
      "activations/layer16_attention_weight_max": 36.69437026977539,
      "activations/layer16_attention_weight_min": -30.80522918701172,
      "activations/layer17_attention_weight_max": 35.606407165527344,
      "activations/layer17_attention_weight_min": -27.585779190063477,
      "activations/layer18_attention_weight_max": 40.2530632019043,
      "activations/layer18_attention_weight_min": -25.447669982910156,
      "activations/layer19_attention_weight_max": 44.8648681640625,
      "activations/layer19_attention_weight_min": -31.335317611694336,
      "activations/layer1_attention_weight_max": 16.323993682861328,
      "activations/layer1_attention_weight_min": -14.776230812072754,
      "activations/layer20_attention_weight_max": 35.22649002075195,
      "activations/layer20_attention_weight_min": -23.78221893310547,
      "activations/layer21_attention_weight_max": 35.71034622192383,
      "activations/layer21_attention_weight_min": -23.583446502685547,
      "activations/layer22_attention_weight_max": 54.948280334472656,
      "activations/layer22_attention_weight_min": -29.41179656982422,
      "activations/layer23_attention_weight_max": 39.7469482421875,
      "activations/layer23_attention_weight_min": -24.03396987915039,
      "activations/layer2_attention_weight_max": 32.123497009277344,
      "activations/layer2_attention_weight_min": -32.19919204711914,
      "activations/layer3_attention_weight_max": 94.0576400756836,
      "activations/layer3_attention_weight_min": -93.85857391357422,
      "activations/layer4_attention_weight_max": 117.94813537597656,
      "activations/layer4_attention_weight_min": -117.81389617919922,
      "activations/layer5_attention_weight_max": 53.39280319213867,
      "activations/layer5_attention_weight_min": -62.721336364746094,
      "activations/layer6_attention_weight_max": 45.96696472167969,
      "activations/layer6_attention_weight_min": -51.62672424316406,
      "activations/layer7_attention_weight_max": 92.24247741699219,
      "activations/layer7_attention_weight_min": -90.5278549194336,
      "activations/layer8_attention_weight_max": 41.51089096069336,
      "activations/layer8_attention_weight_min": -42.109779357910156,
      "activations/layer9_attention_weight_max": 35.66600799560547,
      "activations/layer9_attention_weight_min": -33.713138580322266,
      "epoch": 9.47,
      "learning_rate": 8.980795454545454e-05,
      "loss": 2.8004,
      "step": 163050
    },
    {
      "activations/layer0_attention_weight_max": 16.248332977294922,
      "activations/layer0_attention_weight_min": -11.449760437011719,
      "activations/layer10_attention_weight_max": 31.45485496520996,
      "activations/layer10_attention_weight_min": -32.025230407714844,
      "activations/layer11_attention_weight_max": 32.48577880859375,
      "activations/layer11_attention_weight_min": -31.57848358154297,
      "activations/layer12_attention_weight_max": 24.98467254638672,
      "activations/layer12_attention_weight_min": -24.84671974182129,
      "activations/layer13_attention_weight_max": 36.58794403076172,
      "activations/layer13_attention_weight_min": -31.142608642578125,
      "activations/layer14_attention_weight_max": 50.785919189453125,
      "activations/layer14_attention_weight_min": -38.25666809082031,
      "activations/layer15_attention_weight_max": 35.07246398925781,
      "activations/layer15_attention_weight_min": -30.407337188720703,
      "activations/layer16_attention_weight_max": 29.802734375,
      "activations/layer16_attention_weight_min": -28.80967140197754,
      "activations/layer17_attention_weight_max": 34.861732482910156,
      "activations/layer17_attention_weight_min": -25.44194984436035,
      "activations/layer18_attention_weight_max": 32.096248626708984,
      "activations/layer18_attention_weight_min": -23.24608039855957,
      "activations/layer19_attention_weight_max": 36.57233428955078,
      "activations/layer19_attention_weight_min": -32.68358612060547,
      "activations/layer1_attention_weight_max": 16.264619827270508,
      "activations/layer1_attention_weight_min": -15.01491641998291,
      "activations/layer20_attention_weight_max": 30.810579299926758,
      "activations/layer20_attention_weight_min": -24.870277404785156,
      "activations/layer21_attention_weight_max": 30.073062896728516,
      "activations/layer21_attention_weight_min": -23.233051300048828,
      "activations/layer22_attention_weight_max": 47.5248908996582,
      "activations/layer22_attention_weight_min": -29.438446044921875,
      "activations/layer23_attention_weight_max": 40.71229553222656,
      "activations/layer23_attention_weight_min": -24.913177490234375,
      "activations/layer2_attention_weight_max": 32.65074920654297,
      "activations/layer2_attention_weight_min": -32.54722595214844,
      "activations/layer3_attention_weight_max": 93.81520080566406,
      "activations/layer3_attention_weight_min": -93.5854721069336,
      "activations/layer4_attention_weight_max": 117.09037017822266,
      "activations/layer4_attention_weight_min": -115.4300537109375,
      "activations/layer5_attention_weight_max": 54.99571228027344,
      "activations/layer5_attention_weight_min": -60.5657958984375,
      "activations/layer6_attention_weight_max": 48.59281539916992,
      "activations/layer6_attention_weight_min": -46.7934684753418,
      "activations/layer7_attention_weight_max": 84.8022689819336,
      "activations/layer7_attention_weight_min": -84.67195129394531,
      "activations/layer8_attention_weight_max": 39.30707550048828,
      "activations/layer8_attention_weight_min": -39.181495666503906,
      "activations/layer9_attention_weight_max": 33.41315460205078,
      "activations/layer9_attention_weight_min": -32.016353607177734,
      "epoch": 9.48,
      "learning_rate": 8.978901515151515e-05,
      "loss": 2.7861,
      "step": 163100
    },
    {
      "activations/layer0_attention_weight_max": 15.396783828735352,
      "activations/layer0_attention_weight_min": -11.809369087219238,
      "activations/layer10_attention_weight_max": 34.580509185791016,
      "activations/layer10_attention_weight_min": -32.068214416503906,
      "activations/layer11_attention_weight_max": 32.583961486816406,
      "activations/layer11_attention_weight_min": -30.378765106201172,
      "activations/layer12_attention_weight_max": 23.17259979248047,
      "activations/layer12_attention_weight_min": -24.786094665527344,
      "activations/layer13_attention_weight_max": 39.20094299316406,
      "activations/layer13_attention_weight_min": -36.08080291748047,
      "activations/layer14_attention_weight_max": 48.03327941894531,
      "activations/layer14_attention_weight_min": -36.95137023925781,
      "activations/layer15_attention_weight_max": 35.86381912231445,
      "activations/layer15_attention_weight_min": -30.5168514251709,
      "activations/layer16_attention_weight_max": 28.228893280029297,
      "activations/layer16_attention_weight_min": -27.049156188964844,
      "activations/layer17_attention_weight_max": 31.101295471191406,
      "activations/layer17_attention_weight_min": -25.454099655151367,
      "activations/layer18_attention_weight_max": 31.4460506439209,
      "activations/layer18_attention_weight_min": -23.17251205444336,
      "activations/layer19_attention_weight_max": 33.17451477050781,
      "activations/layer19_attention_weight_min": -31.375471115112305,
      "activations/layer1_attention_weight_max": 16.339529037475586,
      "activations/layer1_attention_weight_min": -15.151439666748047,
      "activations/layer20_attention_weight_max": 30.209077835083008,
      "activations/layer20_attention_weight_min": -26.90869140625,
      "activations/layer21_attention_weight_max": 29.608592987060547,
      "activations/layer21_attention_weight_min": -25.46832275390625,
      "activations/layer22_attention_weight_max": 48.541526794433594,
      "activations/layer22_attention_weight_min": -29.626705169677734,
      "activations/layer23_attention_weight_max": 37.66896057128906,
      "activations/layer23_attention_weight_min": -24.33513641357422,
      "activations/layer2_attention_weight_max": 30.32050323486328,
      "activations/layer2_attention_weight_min": -27.707401275634766,
      "activations/layer3_attention_weight_max": 82.00457000732422,
      "activations/layer3_attention_weight_min": -83.65669250488281,
      "activations/layer4_attention_weight_max": 107.91008758544922,
      "activations/layer4_attention_weight_min": -103.59366607666016,
      "activations/layer5_attention_weight_max": 51.22492980957031,
      "activations/layer5_attention_weight_min": -56.091888427734375,
      "activations/layer6_attention_weight_max": 45.29401397705078,
      "activations/layer6_attention_weight_min": -49.12662124633789,
      "activations/layer7_attention_weight_max": 84.5176010131836,
      "activations/layer7_attention_weight_min": -90.65496063232422,
      "activations/layer8_attention_weight_max": 39.14314270019531,
      "activations/layer8_attention_weight_min": -40.26616287231445,
      "activations/layer9_attention_weight_max": 35.41542434692383,
      "activations/layer9_attention_weight_min": -34.47782897949219,
      "epoch": 9.48,
      "learning_rate": 8.977007575757574e-05,
      "loss": 2.7935,
      "step": 163150
    },
    {
      "activations/layer0_attention_weight_max": 15.067571640014648,
      "activations/layer0_attention_weight_min": -12.320890426635742,
      "activations/layer10_attention_weight_max": 32.2174072265625,
      "activations/layer10_attention_weight_min": -32.0815544128418,
      "activations/layer11_attention_weight_max": 31.560279846191406,
      "activations/layer11_attention_weight_min": -31.669193267822266,
      "activations/layer12_attention_weight_max": 22.07696533203125,
      "activations/layer12_attention_weight_min": -25.211246490478516,
      "activations/layer13_attention_weight_max": 36.29237365722656,
      "activations/layer13_attention_weight_min": -34.87224578857422,
      "activations/layer14_attention_weight_max": 46.18716049194336,
      "activations/layer14_attention_weight_min": -36.84516525268555,
      "activations/layer15_attention_weight_max": 33.76384735107422,
      "activations/layer15_attention_weight_min": -30.843263626098633,
      "activations/layer16_attention_weight_max": 29.92685317993164,
      "activations/layer16_attention_weight_min": -27.598512649536133,
      "activations/layer17_attention_weight_max": 31.632165908813477,
      "activations/layer17_attention_weight_min": -25.8886661529541,
      "activations/layer18_attention_weight_max": 31.74239730834961,
      "activations/layer18_attention_weight_min": -21.884946823120117,
      "activations/layer19_attention_weight_max": 32.33871078491211,
      "activations/layer19_attention_weight_min": -28.4669132232666,
      "activations/layer1_attention_weight_max": 17.689992904663086,
      "activations/layer1_attention_weight_min": -17.50666046142578,
      "activations/layer20_attention_weight_max": 27.822629928588867,
      "activations/layer20_attention_weight_min": -24.20365333557129,
      "activations/layer21_attention_weight_max": 29.76298713684082,
      "activations/layer21_attention_weight_min": -23.060623168945312,
      "activations/layer22_attention_weight_max": 47.3213996887207,
      "activations/layer22_attention_weight_min": -28.154159545898438,
      "activations/layer23_attention_weight_max": 35.7061767578125,
      "activations/layer23_attention_weight_min": -24.103973388671875,
      "activations/layer2_attention_weight_max": 30.450756072998047,
      "activations/layer2_attention_weight_min": -31.447643280029297,
      "activations/layer3_attention_weight_max": 90.51587677001953,
      "activations/layer3_attention_weight_min": -97.76443481445312,
      "activations/layer4_attention_weight_max": 111.04137420654297,
      "activations/layer4_attention_weight_min": -107.58483123779297,
      "activations/layer5_attention_weight_max": 51.92997741699219,
      "activations/layer5_attention_weight_min": -57.967979431152344,
      "activations/layer6_attention_weight_max": 44.572853088378906,
      "activations/layer6_attention_weight_min": -46.01105499267578,
      "activations/layer7_attention_weight_max": 84.27439880371094,
      "activations/layer7_attention_weight_min": -85.08612823486328,
      "activations/layer8_attention_weight_max": 37.811851501464844,
      "activations/layer8_attention_weight_min": -38.6926155090332,
      "activations/layer9_attention_weight_max": 32.61588668823242,
      "activations/layer9_attention_weight_min": -33.12184143066406,
      "epoch": 9.48,
      "learning_rate": 8.975113636363635e-05,
      "loss": 2.8068,
      "step": 163200
    },
    {
      "activations/layer0_attention_weight_max": 14.961216926574707,
      "activations/layer0_attention_weight_min": -12.679844856262207,
      "activations/layer10_attention_weight_max": 31.71653175354004,
      "activations/layer10_attention_weight_min": -31.297372817993164,
      "activations/layer11_attention_weight_max": 30.470815658569336,
      "activations/layer11_attention_weight_min": -29.810043334960938,
      "activations/layer12_attention_weight_max": 22.916162490844727,
      "activations/layer12_attention_weight_min": -26.343158721923828,
      "activations/layer13_attention_weight_max": 37.032222747802734,
      "activations/layer13_attention_weight_min": -32.34525680541992,
      "activations/layer14_attention_weight_max": 44.87521743774414,
      "activations/layer14_attention_weight_min": -36.69463348388672,
      "activations/layer15_attention_weight_max": 36.46123504638672,
      "activations/layer15_attention_weight_min": -29.85569953918457,
      "activations/layer16_attention_weight_max": 31.457836151123047,
      "activations/layer16_attention_weight_min": -26.851289749145508,
      "activations/layer17_attention_weight_max": 35.77893829345703,
      "activations/layer17_attention_weight_min": -25.432403564453125,
      "activations/layer18_attention_weight_max": 30.741594314575195,
      "activations/layer18_attention_weight_min": -21.60645866394043,
      "activations/layer19_attention_weight_max": 36.971527099609375,
      "activations/layer19_attention_weight_min": -29.590978622436523,
      "activations/layer1_attention_weight_max": 17.77678871154785,
      "activations/layer1_attention_weight_min": -14.943465232849121,
      "activations/layer20_attention_weight_max": 34.97856140136719,
      "activations/layer20_attention_weight_min": -24.1672420501709,
      "activations/layer21_attention_weight_max": 31.265026092529297,
      "activations/layer21_attention_weight_min": -23.147781372070312,
      "activations/layer22_attention_weight_max": 45.80722427368164,
      "activations/layer22_attention_weight_min": -30.045190811157227,
      "activations/layer23_attention_weight_max": 39.003971099853516,
      "activations/layer23_attention_weight_min": -25.834638595581055,
      "activations/layer2_attention_weight_max": 30.842559814453125,
      "activations/layer2_attention_weight_min": -29.21025848388672,
      "activations/layer3_attention_weight_max": 87.50577545166016,
      "activations/layer3_attention_weight_min": -84.15492248535156,
      "activations/layer4_attention_weight_max": 108.140625,
      "activations/layer4_attention_weight_min": -110.77528381347656,
      "activations/layer5_attention_weight_max": 52.82447052001953,
      "activations/layer5_attention_weight_min": -57.282562255859375,
      "activations/layer6_attention_weight_max": 48.03512954711914,
      "activations/layer6_attention_weight_min": -49.2602653503418,
      "activations/layer7_attention_weight_max": 91.02500915527344,
      "activations/layer7_attention_weight_min": -83.46836853027344,
      "activations/layer8_attention_weight_max": 37.5628662109375,
      "activations/layer8_attention_weight_min": -37.518150329589844,
      "activations/layer9_attention_weight_max": 31.877805709838867,
      "activations/layer9_attention_weight_min": -31.090194702148438,
      "epoch": 9.49,
      "learning_rate": 8.973219696969697e-05,
      "loss": 2.7878,
      "step": 163250
    },
    {
      "activations/layer0_attention_weight_max": 16.01756477355957,
      "activations/layer0_attention_weight_min": -12.49837875366211,
      "activations/layer10_attention_weight_max": 32.44782638549805,
      "activations/layer10_attention_weight_min": -31.324399948120117,
      "activations/layer11_attention_weight_max": 35.47901153564453,
      "activations/layer11_attention_weight_min": -30.45050621032715,
      "activations/layer12_attention_weight_max": 23.522945404052734,
      "activations/layer12_attention_weight_min": -24.51412582397461,
      "activations/layer13_attention_weight_max": 39.5374641418457,
      "activations/layer13_attention_weight_min": -32.96982955932617,
      "activations/layer14_attention_weight_max": 54.17977523803711,
      "activations/layer14_attention_weight_min": -36.639217376708984,
      "activations/layer15_attention_weight_max": 36.20356750488281,
      "activations/layer15_attention_weight_min": -30.35428810119629,
      "activations/layer16_attention_weight_max": 34.85095977783203,
      "activations/layer16_attention_weight_min": -28.612567901611328,
      "activations/layer17_attention_weight_max": 33.23264694213867,
      "activations/layer17_attention_weight_min": -25.600807189941406,
      "activations/layer18_attention_weight_max": 31.194862365722656,
      "activations/layer18_attention_weight_min": -21.163602828979492,
      "activations/layer19_attention_weight_max": 38.82588577270508,
      "activations/layer19_attention_weight_min": -29.937665939331055,
      "activations/layer1_attention_weight_max": 17.188114166259766,
      "activations/layer1_attention_weight_min": -15.382800102233887,
      "activations/layer20_attention_weight_max": 31.769739151000977,
      "activations/layer20_attention_weight_min": -24.945554733276367,
      "activations/layer21_attention_weight_max": 31.337377548217773,
      "activations/layer21_attention_weight_min": -22.138246536254883,
      "activations/layer22_attention_weight_max": 44.2225341796875,
      "activations/layer22_attention_weight_min": -27.189517974853516,
      "activations/layer23_attention_weight_max": 34.94546890258789,
      "activations/layer23_attention_weight_min": -24.894506454467773,
      "activations/layer2_attention_weight_max": 31.572093963623047,
      "activations/layer2_attention_weight_min": -29.817333221435547,
      "activations/layer3_attention_weight_max": 88.54252624511719,
      "activations/layer3_attention_weight_min": -89.46630859375,
      "activations/layer4_attention_weight_max": 108.9015121459961,
      "activations/layer4_attention_weight_min": -107.27388000488281,
      "activations/layer5_attention_weight_max": 51.73468017578125,
      "activations/layer5_attention_weight_min": -58.86802673339844,
      "activations/layer6_attention_weight_max": 45.69483947753906,
      "activations/layer6_attention_weight_min": -48.156471252441406,
      "activations/layer7_attention_weight_max": 88.32489776611328,
      "activations/layer7_attention_weight_min": -86.5715103149414,
      "activations/layer8_attention_weight_max": 39.08760452270508,
      "activations/layer8_attention_weight_min": -41.197818756103516,
      "activations/layer9_attention_weight_max": 33.146175384521484,
      "activations/layer9_attention_weight_min": -32.139244079589844,
      "epoch": 9.49,
      "learning_rate": 8.971325757575756e-05,
      "loss": 2.8006,
      "step": 163300
    },
    {
      "activations/layer0_attention_weight_max": 16.26206398010254,
      "activations/layer0_attention_weight_min": -12.589435577392578,
      "activations/layer10_attention_weight_max": 37.047603607177734,
      "activations/layer10_attention_weight_min": -34.824981689453125,
      "activations/layer11_attention_weight_max": 36.508209228515625,
      "activations/layer11_attention_weight_min": -35.03886795043945,
      "activations/layer12_attention_weight_max": 30.26104164123535,
      "activations/layer12_attention_weight_min": -25.182573318481445,
      "activations/layer13_attention_weight_max": 46.9217414855957,
      "activations/layer13_attention_weight_min": -36.40738296508789,
      "activations/layer14_attention_weight_max": 56.602500915527344,
      "activations/layer14_attention_weight_min": -46.917179107666016,
      "activations/layer15_attention_weight_max": 39.178741455078125,
      "activations/layer15_attention_weight_min": -33.93526077270508,
      "activations/layer16_attention_weight_max": 33.7248649597168,
      "activations/layer16_attention_weight_min": -28.34092903137207,
      "activations/layer17_attention_weight_max": 34.012874603271484,
      "activations/layer17_attention_weight_min": -27.540571212768555,
      "activations/layer18_attention_weight_max": 29.816635131835938,
      "activations/layer18_attention_weight_min": -22.937856674194336,
      "activations/layer19_attention_weight_max": 34.18398666381836,
      "activations/layer19_attention_weight_min": -30.098892211914062,
      "activations/layer1_attention_weight_max": 16.796916961669922,
      "activations/layer1_attention_weight_min": -14.466564178466797,
      "activations/layer20_attention_weight_max": 30.88475227355957,
      "activations/layer20_attention_weight_min": -26.55349349975586,
      "activations/layer21_attention_weight_max": 32.617794036865234,
      "activations/layer21_attention_weight_min": -24.58652114868164,
      "activations/layer22_attention_weight_max": 43.71226119995117,
      "activations/layer22_attention_weight_min": -29.549192428588867,
      "activations/layer23_attention_weight_max": 34.68556594848633,
      "activations/layer23_attention_weight_min": -25.698116302490234,
      "activations/layer2_attention_weight_max": 32.88939666748047,
      "activations/layer2_attention_weight_min": -32.54719543457031,
      "activations/layer3_attention_weight_max": 90.44564819335938,
      "activations/layer3_attention_weight_min": -90.8251724243164,
      "activations/layer4_attention_weight_max": 116.9433822631836,
      "activations/layer4_attention_weight_min": -116.49633026123047,
      "activations/layer5_attention_weight_max": 54.86022186279297,
      "activations/layer5_attention_weight_min": -64.54437255859375,
      "activations/layer6_attention_weight_max": 46.79118347167969,
      "activations/layer6_attention_weight_min": -52.79869079589844,
      "activations/layer7_attention_weight_max": 88.75263977050781,
      "activations/layer7_attention_weight_min": -93.86856079101562,
      "activations/layer8_attention_weight_max": 42.4547004699707,
      "activations/layer8_attention_weight_min": -45.75358200073242,
      "activations/layer9_attention_weight_max": 38.771568298339844,
      "activations/layer9_attention_weight_min": -37.4535026550293,
      "epoch": 9.49,
      "learning_rate": 8.969431818181817e-05,
      "loss": 2.7937,
      "step": 163350
    },
    {
      "activations/layer0_attention_weight_max": 14.917388916015625,
      "activations/layer0_attention_weight_min": -11.83910083770752,
      "activations/layer10_attention_weight_max": 34.96597671508789,
      "activations/layer10_attention_weight_min": -32.5204963684082,
      "activations/layer11_attention_weight_max": 33.58110809326172,
      "activations/layer11_attention_weight_min": -31.541296005249023,
      "activations/layer12_attention_weight_max": 24.065580368041992,
      "activations/layer12_attention_weight_min": -24.967239379882812,
      "activations/layer13_attention_weight_max": 38.53022384643555,
      "activations/layer13_attention_weight_min": -33.756370544433594,
      "activations/layer14_attention_weight_max": 46.909698486328125,
      "activations/layer14_attention_weight_min": -38.16965866088867,
      "activations/layer15_attention_weight_max": 36.873592376708984,
      "activations/layer15_attention_weight_min": -31.38117027282715,
      "activations/layer16_attention_weight_max": 35.29834747314453,
      "activations/layer16_attention_weight_min": -27.140127182006836,
      "activations/layer17_attention_weight_max": 35.75831604003906,
      "activations/layer17_attention_weight_min": -25.483844757080078,
      "activations/layer18_attention_weight_max": 30.493192672729492,
      "activations/layer18_attention_weight_min": -21.253454208374023,
      "activations/layer19_attention_weight_max": 36.818572998046875,
      "activations/layer19_attention_weight_min": -29.96285057067871,
      "activations/layer1_attention_weight_max": 15.629976272583008,
      "activations/layer1_attention_weight_min": -14.58318042755127,
      "activations/layer20_attention_weight_max": 30.612884521484375,
      "activations/layer20_attention_weight_min": -23.989763259887695,
      "activations/layer21_attention_weight_max": 30.137670516967773,
      "activations/layer21_attention_weight_min": -22.070331573486328,
      "activations/layer22_attention_weight_max": 53.17281723022461,
      "activations/layer22_attention_weight_min": -27.548187255859375,
      "activations/layer23_attention_weight_max": 39.26966094970703,
      "activations/layer23_attention_weight_min": -22.976926803588867,
      "activations/layer2_attention_weight_max": 31.30206871032715,
      "activations/layer2_attention_weight_min": -30.560199737548828,
      "activations/layer3_attention_weight_max": 88.40296936035156,
      "activations/layer3_attention_weight_min": -87.5047836303711,
      "activations/layer4_attention_weight_max": 112.95751953125,
      "activations/layer4_attention_weight_min": -110.15787506103516,
      "activations/layer5_attention_weight_max": 51.247676849365234,
      "activations/layer5_attention_weight_min": -61.67839813232422,
      "activations/layer6_attention_weight_max": 45.71985626220703,
      "activations/layer6_attention_weight_min": -48.444740295410156,
      "activations/layer7_attention_weight_max": 91.42195129394531,
      "activations/layer7_attention_weight_min": -90.04496002197266,
      "activations/layer8_attention_weight_max": 41.62727737426758,
      "activations/layer8_attention_weight_min": -42.22984313964844,
      "activations/layer9_attention_weight_max": 33.51113510131836,
      "activations/layer9_attention_weight_min": -33.97572326660156,
      "epoch": 9.49,
      "learning_rate": 8.967537878787879e-05,
      "loss": 2.7792,
      "step": 163400
    },
    {
      "activations/layer0_attention_weight_max": 15.476643562316895,
      "activations/layer0_attention_weight_min": -12.389577865600586,
      "activations/layer10_attention_weight_max": 33.91581726074219,
      "activations/layer10_attention_weight_min": -33.16146469116211,
      "activations/layer11_attention_weight_max": 34.05156326293945,
      "activations/layer11_attention_weight_min": -31.095569610595703,
      "activations/layer12_attention_weight_max": 24.223194122314453,
      "activations/layer12_attention_weight_min": -24.08356285095215,
      "activations/layer13_attention_weight_max": 39.41706466674805,
      "activations/layer13_attention_weight_min": -35.80039596557617,
      "activations/layer14_attention_weight_max": 50.8991813659668,
      "activations/layer14_attention_weight_min": -38.91799545288086,
      "activations/layer15_attention_weight_max": 35.818931579589844,
      "activations/layer15_attention_weight_min": -31.34742546081543,
      "activations/layer16_attention_weight_max": 31.678560256958008,
      "activations/layer16_attention_weight_min": -28.9127140045166,
      "activations/layer17_attention_weight_max": 34.17230224609375,
      "activations/layer17_attention_weight_min": -25.51896095275879,
      "activations/layer18_attention_weight_max": 29.625770568847656,
      "activations/layer18_attention_weight_min": -21.1575984954834,
      "activations/layer19_attention_weight_max": 36.063350677490234,
      "activations/layer19_attention_weight_min": -31.551807403564453,
      "activations/layer1_attention_weight_max": 17.136333465576172,
      "activations/layer1_attention_weight_min": -14.090666770935059,
      "activations/layer20_attention_weight_max": 29.712749481201172,
      "activations/layer20_attention_weight_min": -25.877084732055664,
      "activations/layer21_attention_weight_max": 33.43118667602539,
      "activations/layer21_attention_weight_min": -24.687204360961914,
      "activations/layer22_attention_weight_max": 41.558319091796875,
      "activations/layer22_attention_weight_min": -28.292789459228516,
      "activations/layer23_attention_weight_max": 37.2876091003418,
      "activations/layer23_attention_weight_min": -22.917011260986328,
      "activations/layer2_attention_weight_max": 31.767841339111328,
      "activations/layer2_attention_weight_min": -30.424415588378906,
      "activations/layer3_attention_weight_max": 91.44356536865234,
      "activations/layer3_attention_weight_min": -91.88008880615234,
      "activations/layer4_attention_weight_max": 114.36744689941406,
      "activations/layer4_attention_weight_min": -113.47027587890625,
      "activations/layer5_attention_weight_max": 56.34466552734375,
      "activations/layer5_attention_weight_min": -59.745399475097656,
      "activations/layer6_attention_weight_max": 46.55833053588867,
      "activations/layer6_attention_weight_min": -49.473934173583984,
      "activations/layer7_attention_weight_max": 88.36300659179688,
      "activations/layer7_attention_weight_min": -87.6261978149414,
      "activations/layer8_attention_weight_max": 38.69844436645508,
      "activations/layer8_attention_weight_min": -39.63551712036133,
      "activations/layer9_attention_weight_max": 35.0817985534668,
      "activations/layer9_attention_weight_min": -33.33318328857422,
      "epoch": 9.5,
      "learning_rate": 8.965643939393939e-05,
      "loss": 2.7997,
      "step": 163450
    },
    {
      "activations/layer0_attention_weight_max": 14.904247283935547,
      "activations/layer0_attention_weight_min": -12.374953269958496,
      "activations/layer10_attention_weight_max": 33.601593017578125,
      "activations/layer10_attention_weight_min": -31.77945327758789,
      "activations/layer11_attention_weight_max": 32.75440216064453,
      "activations/layer11_attention_weight_min": -31.364004135131836,
      "activations/layer12_attention_weight_max": 24.42821502685547,
      "activations/layer12_attention_weight_min": -27.207115173339844,
      "activations/layer13_attention_weight_max": 40.051048278808594,
      "activations/layer13_attention_weight_min": -32.00750732421875,
      "activations/layer14_attention_weight_max": 44.13865280151367,
      "activations/layer14_attention_weight_min": -37.81047439575195,
      "activations/layer15_attention_weight_max": 34.62657165527344,
      "activations/layer15_attention_weight_min": -29.18991470336914,
      "activations/layer16_attention_weight_max": 31.21608543395996,
      "activations/layer16_attention_weight_min": -27.92253875732422,
      "activations/layer17_attention_weight_max": 32.491355895996094,
      "activations/layer17_attention_weight_min": -25.12729835510254,
      "activations/layer18_attention_weight_max": 31.166784286499023,
      "activations/layer18_attention_weight_min": -21.529869079589844,
      "activations/layer19_attention_weight_max": 34.87545394897461,
      "activations/layer19_attention_weight_min": -30.95696449279785,
      "activations/layer1_attention_weight_max": 16.62388801574707,
      "activations/layer1_attention_weight_min": -14.428312301635742,
      "activations/layer20_attention_weight_max": 29.01849365234375,
      "activations/layer20_attention_weight_min": -27.533958435058594,
      "activations/layer21_attention_weight_max": 30.765079498291016,
      "activations/layer21_attention_weight_min": -26.011337280273438,
      "activations/layer22_attention_weight_max": 50.216590881347656,
      "activations/layer22_attention_weight_min": -33.395172119140625,
      "activations/layer23_attention_weight_max": 36.742919921875,
      "activations/layer23_attention_weight_min": -26.456268310546875,
      "activations/layer2_attention_weight_max": 33.64777374267578,
      "activations/layer2_attention_weight_min": -31.26631736755371,
      "activations/layer3_attention_weight_max": 90.87659454345703,
      "activations/layer3_attention_weight_min": -89.96227264404297,
      "activations/layer4_attention_weight_max": 111.58438873291016,
      "activations/layer4_attention_weight_min": -110.32795715332031,
      "activations/layer5_attention_weight_max": 52.3525505065918,
      "activations/layer5_attention_weight_min": -61.19750213623047,
      "activations/layer6_attention_weight_max": 46.18559265136719,
      "activations/layer6_attention_weight_min": -50.70539474487305,
      "activations/layer7_attention_weight_max": 85.87030029296875,
      "activations/layer7_attention_weight_min": -85.9699935913086,
      "activations/layer8_attention_weight_max": 41.398033142089844,
      "activations/layer8_attention_weight_min": -40.25407791137695,
      "activations/layer9_attention_weight_max": 35.69913101196289,
      "activations/layer9_attention_weight_min": -34.25352478027344,
      "epoch": 9.5,
      "learning_rate": 8.963749999999999e-05,
      "loss": 2.7983,
      "step": 163500
    },
    {
      "activations/layer0_attention_weight_max": 14.53093147277832,
      "activations/layer0_attention_weight_min": -12.775758743286133,
      "activations/layer10_attention_weight_max": 31.363628387451172,
      "activations/layer10_attention_weight_min": -31.784238815307617,
      "activations/layer11_attention_weight_max": 31.204710006713867,
      "activations/layer11_attention_weight_min": -30.29467010498047,
      "activations/layer12_attention_weight_max": 23.06046485900879,
      "activations/layer12_attention_weight_min": -25.950519561767578,
      "activations/layer13_attention_weight_max": 37.43964767456055,
      "activations/layer13_attention_weight_min": -34.00712585449219,
      "activations/layer14_attention_weight_max": 42.74797821044922,
      "activations/layer14_attention_weight_min": -36.5835075378418,
      "activations/layer15_attention_weight_max": 36.135250091552734,
      "activations/layer15_attention_weight_min": -29.87830352783203,
      "activations/layer16_attention_weight_max": 30.261844635009766,
      "activations/layer16_attention_weight_min": -27.941434860229492,
      "activations/layer17_attention_weight_max": 30.354440689086914,
      "activations/layer17_attention_weight_min": -26.91143226623535,
      "activations/layer18_attention_weight_max": 31.03325080871582,
      "activations/layer18_attention_weight_min": -22.93061637878418,
      "activations/layer19_attention_weight_max": 38.76621627807617,
      "activations/layer19_attention_weight_min": -31.0073299407959,
      "activations/layer1_attention_weight_max": 17.670738220214844,
      "activations/layer1_attention_weight_min": -15.636384963989258,
      "activations/layer20_attention_weight_max": 28.94631004333496,
      "activations/layer20_attention_weight_min": -24.02727699279785,
      "activations/layer21_attention_weight_max": 29.919023513793945,
      "activations/layer21_attention_weight_min": -22.520233154296875,
      "activations/layer22_attention_weight_max": 45.37200927734375,
      "activations/layer22_attention_weight_min": -30.387714385986328,
      "activations/layer23_attention_weight_max": 35.60276794433594,
      "activations/layer23_attention_weight_min": -23.168006896972656,
      "activations/layer2_attention_weight_max": 33.076778411865234,
      "activations/layer2_attention_weight_min": -30.8834285736084,
      "activations/layer3_attention_weight_max": 87.36290740966797,
      "activations/layer3_attention_weight_min": -87.65040588378906,
      "activations/layer4_attention_weight_max": 112.82647705078125,
      "activations/layer4_attention_weight_min": -109.47706604003906,
      "activations/layer5_attention_weight_max": 51.68377685546875,
      "activations/layer5_attention_weight_min": -60.418819427490234,
      "activations/layer6_attention_weight_max": 45.80739974975586,
      "activations/layer6_attention_weight_min": -48.38875198364258,
      "activations/layer7_attention_weight_max": 90.2003402709961,
      "activations/layer7_attention_weight_min": -85.25747680664062,
      "activations/layer8_attention_weight_max": 42.238460540771484,
      "activations/layer8_attention_weight_min": -44.182884216308594,
      "activations/layer9_attention_weight_max": 33.18433380126953,
      "activations/layer9_attention_weight_min": -35.30852127075195,
      "epoch": 9.5,
      "learning_rate": 8.961856060606059e-05,
      "loss": 2.7874,
      "step": 163550
    },
    {
      "activations/layer0_attention_weight_max": 15.31848430633545,
      "activations/layer0_attention_weight_min": -11.812660217285156,
      "activations/layer10_attention_weight_max": 33.07960891723633,
      "activations/layer10_attention_weight_min": -33.91846466064453,
      "activations/layer11_attention_weight_max": 37.15673065185547,
      "activations/layer11_attention_weight_min": -32.62979507446289,
      "activations/layer12_attention_weight_max": 26.248750686645508,
      "activations/layer12_attention_weight_min": -24.678550720214844,
      "activations/layer13_attention_weight_max": 38.844112396240234,
      "activations/layer13_attention_weight_min": -33.0660514831543,
      "activations/layer14_attention_weight_max": 52.6427001953125,
      "activations/layer14_attention_weight_min": -36.586116790771484,
      "activations/layer15_attention_weight_max": 36.38937759399414,
      "activations/layer15_attention_weight_min": -31.9506893157959,
      "activations/layer16_attention_weight_max": 31.715986251831055,
      "activations/layer16_attention_weight_min": -28.327369689941406,
      "activations/layer17_attention_weight_max": 30.915462493896484,
      "activations/layer17_attention_weight_min": -24.646623611450195,
      "activations/layer18_attention_weight_max": 30.079875946044922,
      "activations/layer18_attention_weight_min": -22.51566505432129,
      "activations/layer19_attention_weight_max": 39.486507415771484,
      "activations/layer19_attention_weight_min": -30.232593536376953,
      "activations/layer1_attention_weight_max": 16.577411651611328,
      "activations/layer1_attention_weight_min": -14.64017391204834,
      "activations/layer20_attention_weight_max": 28.702608108520508,
      "activations/layer20_attention_weight_min": -24.739662170410156,
      "activations/layer21_attention_weight_max": 28.142168045043945,
      "activations/layer21_attention_weight_min": -23.241914749145508,
      "activations/layer22_attention_weight_max": 52.32994079589844,
      "activations/layer22_attention_weight_min": -28.74856185913086,
      "activations/layer23_attention_weight_max": 39.6285285949707,
      "activations/layer23_attention_weight_min": -24.817716598510742,
      "activations/layer2_attention_weight_max": 30.652484893798828,
      "activations/layer2_attention_weight_min": -31.82891845703125,
      "activations/layer3_attention_weight_max": 89.92210388183594,
      "activations/layer3_attention_weight_min": -91.77775573730469,
      "activations/layer4_attention_weight_max": 116.97342681884766,
      "activations/layer4_attention_weight_min": -114.8114242553711,
      "activations/layer5_attention_weight_max": 56.51182174682617,
      "activations/layer5_attention_weight_min": -62.17607116699219,
      "activations/layer6_attention_weight_max": 46.14247512817383,
      "activations/layer6_attention_weight_min": -49.92945098876953,
      "activations/layer7_attention_weight_max": 92.76411437988281,
      "activations/layer7_attention_weight_min": -89.944091796875,
      "activations/layer8_attention_weight_max": 40.834388732910156,
      "activations/layer8_attention_weight_min": -41.50213623046875,
      "activations/layer9_attention_weight_max": 34.634578704833984,
      "activations/layer9_attention_weight_min": -34.573036193847656,
      "epoch": 9.51,
      "learning_rate": 8.95996212121212e-05,
      "loss": 2.7964,
      "step": 163600
    },
    {
      "activations/layer0_attention_weight_max": 15.627806663513184,
      "activations/layer0_attention_weight_min": -11.297385215759277,
      "activations/layer10_attention_weight_max": 39.809051513671875,
      "activations/layer10_attention_weight_min": -34.170291900634766,
      "activations/layer11_attention_weight_max": 42.544132232666016,
      "activations/layer11_attention_weight_min": -35.19207763671875,
      "activations/layer12_attention_weight_max": 25.364566802978516,
      "activations/layer12_attention_weight_min": -29.140592575073242,
      "activations/layer13_attention_weight_max": 39.52742385864258,
      "activations/layer13_attention_weight_min": -32.63351821899414,
      "activations/layer14_attention_weight_max": 47.20992660522461,
      "activations/layer14_attention_weight_min": -38.07334899902344,
      "activations/layer15_attention_weight_max": 37.20160675048828,
      "activations/layer15_attention_weight_min": -32.654640197753906,
      "activations/layer16_attention_weight_max": 33.06889724731445,
      "activations/layer16_attention_weight_min": -30.10832405090332,
      "activations/layer17_attention_weight_max": 33.83452606201172,
      "activations/layer17_attention_weight_min": -27.656143188476562,
      "activations/layer18_attention_weight_max": 35.42646408081055,
      "activations/layer18_attention_weight_min": -23.066282272338867,
      "activations/layer19_attention_weight_max": 36.79854965209961,
      "activations/layer19_attention_weight_min": -33.627830505371094,
      "activations/layer1_attention_weight_max": 16.402973175048828,
      "activations/layer1_attention_weight_min": -15.027530670166016,
      "activations/layer20_attention_weight_max": 33.22394561767578,
      "activations/layer20_attention_weight_min": -24.94677734375,
      "activations/layer21_attention_weight_max": 31.337560653686523,
      "activations/layer21_attention_weight_min": -23.106475830078125,
      "activations/layer22_attention_weight_max": 49.47779083251953,
      "activations/layer22_attention_weight_min": -29.30184555053711,
      "activations/layer23_attention_weight_max": 36.90319061279297,
      "activations/layer23_attention_weight_min": -23.544044494628906,
      "activations/layer2_attention_weight_max": 33.22722625732422,
      "activations/layer2_attention_weight_min": -32.40705871582031,
      "activations/layer3_attention_weight_max": 94.69072723388672,
      "activations/layer3_attention_weight_min": -96.02554321289062,
      "activations/layer4_attention_weight_max": 124.2215805053711,
      "activations/layer4_attention_weight_min": -114.50421142578125,
      "activations/layer5_attention_weight_max": 58.775177001953125,
      "activations/layer5_attention_weight_min": -62.70841979980469,
      "activations/layer6_attention_weight_max": 45.58415985107422,
      "activations/layer6_attention_weight_min": -47.638465881347656,
      "activations/layer7_attention_weight_max": 108.1133041381836,
      "activations/layer7_attention_weight_min": -92.16549682617188,
      "activations/layer8_attention_weight_max": 47.16345977783203,
      "activations/layer8_attention_weight_min": -42.350563049316406,
      "activations/layer9_attention_weight_max": 37.129207611083984,
      "activations/layer9_attention_weight_min": -36.73020553588867,
      "epoch": 9.51,
      "learning_rate": 8.958068181818182e-05,
      "loss": 2.8004,
      "step": 163650
    },
    {
      "activations/layer0_attention_weight_max": 15.139822959899902,
      "activations/layer0_attention_weight_min": -12.663490295410156,
      "activations/layer10_attention_weight_max": 34.726524353027344,
      "activations/layer10_attention_weight_min": -35.757293701171875,
      "activations/layer11_attention_weight_max": 36.053306579589844,
      "activations/layer11_attention_weight_min": -32.128448486328125,
      "activations/layer12_attention_weight_max": 24.678049087524414,
      "activations/layer12_attention_weight_min": -25.022851943969727,
      "activations/layer13_attention_weight_max": 41.68606185913086,
      "activations/layer13_attention_weight_min": -33.98539733886719,
      "activations/layer14_attention_weight_max": 44.74748229980469,
      "activations/layer14_attention_weight_min": -39.469032287597656,
      "activations/layer15_attention_weight_max": 40.31367111206055,
      "activations/layer15_attention_weight_min": -31.92620086669922,
      "activations/layer16_attention_weight_max": 30.851652145385742,
      "activations/layer16_attention_weight_min": -29.46088409423828,
      "activations/layer17_attention_weight_max": 33.767799377441406,
      "activations/layer17_attention_weight_min": -25.81162452697754,
      "activations/layer18_attention_weight_max": 30.493000030517578,
      "activations/layer18_attention_weight_min": -21.99566078186035,
      "activations/layer19_attention_weight_max": 35.395023345947266,
      "activations/layer19_attention_weight_min": -33.3488655090332,
      "activations/layer1_attention_weight_max": 16.73737335205078,
      "activations/layer1_attention_weight_min": -14.22120475769043,
      "activations/layer20_attention_weight_max": 28.748077392578125,
      "activations/layer20_attention_weight_min": -27.163679122924805,
      "activations/layer21_attention_weight_max": 31.510295867919922,
      "activations/layer21_attention_weight_min": -23.070837020874023,
      "activations/layer22_attention_weight_max": 48.2334098815918,
      "activations/layer22_attention_weight_min": -31.18488883972168,
      "activations/layer23_attention_weight_max": 35.48442077636719,
      "activations/layer23_attention_weight_min": -25.56014060974121,
      "activations/layer2_attention_weight_max": 31.325056076049805,
      "activations/layer2_attention_weight_min": -31.80475616455078,
      "activations/layer3_attention_weight_max": 91.99642181396484,
      "activations/layer3_attention_weight_min": -96.22957611083984,
      "activations/layer4_attention_weight_max": 119.53376770019531,
      "activations/layer4_attention_weight_min": -117.5101089477539,
      "activations/layer5_attention_weight_max": 55.99707794189453,
      "activations/layer5_attention_weight_min": -61.30031967163086,
      "activations/layer6_attention_weight_max": 49.86629867553711,
      "activations/layer6_attention_weight_min": -51.04258346557617,
      "activations/layer7_attention_weight_max": 97.93282318115234,
      "activations/layer7_attention_weight_min": -94.06221008300781,
      "activations/layer8_attention_weight_max": 43.468990325927734,
      "activations/layer8_attention_weight_min": -43.71989440917969,
      "activations/layer9_attention_weight_max": 38.303810119628906,
      "activations/layer9_attention_weight_min": -37.056068420410156,
      "epoch": 9.51,
      "learning_rate": 8.956174242424241e-05,
      "loss": 2.778,
      "step": 163700
    },
    {
      "activations/layer0_attention_weight_max": 15.467147827148438,
      "activations/layer0_attention_weight_min": -13.751108169555664,
      "activations/layer10_attention_weight_max": 33.737457275390625,
      "activations/layer10_attention_weight_min": -32.57215881347656,
      "activations/layer11_attention_weight_max": 35.24909591674805,
      "activations/layer11_attention_weight_min": -30.69562530517578,
      "activations/layer12_attention_weight_max": 35.14716720581055,
      "activations/layer12_attention_weight_min": -26.232202529907227,
      "activations/layer13_attention_weight_max": 40.683467864990234,
      "activations/layer13_attention_weight_min": -35.12010955810547,
      "activations/layer14_attention_weight_max": 45.58405303955078,
      "activations/layer14_attention_weight_min": -39.27120590209961,
      "activations/layer15_attention_weight_max": 34.9375,
      "activations/layer15_attention_weight_min": -30.44995880126953,
      "activations/layer16_attention_weight_max": 32.31975555419922,
      "activations/layer16_attention_weight_min": -27.751773834228516,
      "activations/layer17_attention_weight_max": 31.729923248291016,
      "activations/layer17_attention_weight_min": -24.300878524780273,
      "activations/layer18_attention_weight_max": 29.314720153808594,
      "activations/layer18_attention_weight_min": -21.25851058959961,
      "activations/layer19_attention_weight_max": 35.511356353759766,
      "activations/layer19_attention_weight_min": -31.05244255065918,
      "activations/layer1_attention_weight_max": 16.60169219970703,
      "activations/layer1_attention_weight_min": -15.164289474487305,
      "activations/layer20_attention_weight_max": 33.60391616821289,
      "activations/layer20_attention_weight_min": -24.29637908935547,
      "activations/layer21_attention_weight_max": 30.88927459716797,
      "activations/layer21_attention_weight_min": -23.658409118652344,
      "activations/layer22_attention_weight_max": 44.38579177856445,
      "activations/layer22_attention_weight_min": -27.399784088134766,
      "activations/layer23_attention_weight_max": 38.445255279541016,
      "activations/layer23_attention_weight_min": -24.392263412475586,
      "activations/layer2_attention_weight_max": 31.31678009033203,
      "activations/layer2_attention_weight_min": -30.222389221191406,
      "activations/layer3_attention_weight_max": 87.39068603515625,
      "activations/layer3_attention_weight_min": -92.252685546875,
      "activations/layer4_attention_weight_max": 110.23109436035156,
      "activations/layer4_attention_weight_min": -111.07293701171875,
      "activations/layer5_attention_weight_max": 52.43544006347656,
      "activations/layer5_attention_weight_min": -64.51380920410156,
      "activations/layer6_attention_weight_max": 47.6493034362793,
      "activations/layer6_attention_weight_min": -52.32511520385742,
      "activations/layer7_attention_weight_max": 88.32066345214844,
      "activations/layer7_attention_weight_min": -95.97045135498047,
      "activations/layer8_attention_weight_max": 38.57670974731445,
      "activations/layer8_attention_weight_min": -41.29685974121094,
      "activations/layer9_attention_weight_max": 33.518001556396484,
      "activations/layer9_attention_weight_min": -34.411346435546875,
      "epoch": 9.51,
      "learning_rate": 8.954280303030302e-05,
      "loss": 2.7867,
      "step": 163750
    },
    {
      "activations/layer0_attention_weight_max": 15.317208290100098,
      "activations/layer0_attention_weight_min": -12.1487398147583,
      "activations/layer10_attention_weight_max": 31.392059326171875,
      "activations/layer10_attention_weight_min": -31.185161590576172,
      "activations/layer11_attention_weight_max": 30.8701229095459,
      "activations/layer11_attention_weight_min": -33.443031311035156,
      "activations/layer12_attention_weight_max": 26.87681007385254,
      "activations/layer12_attention_weight_min": -26.948959350585938,
      "activations/layer13_attention_weight_max": 39.94829559326172,
      "activations/layer13_attention_weight_min": -30.788681030273438,
      "activations/layer14_attention_weight_max": 44.976383209228516,
      "activations/layer14_attention_weight_min": -36.775428771972656,
      "activations/layer15_attention_weight_max": 36.175987243652344,
      "activations/layer15_attention_weight_min": -30.601600646972656,
      "activations/layer16_attention_weight_max": 32.17939376831055,
      "activations/layer16_attention_weight_min": -25.91381072998047,
      "activations/layer17_attention_weight_max": 34.46302795410156,
      "activations/layer17_attention_weight_min": -25.160663604736328,
      "activations/layer18_attention_weight_max": 29.479328155517578,
      "activations/layer18_attention_weight_min": -22.33468246459961,
      "activations/layer19_attention_weight_max": 36.62629318237305,
      "activations/layer19_attention_weight_min": -28.624021530151367,
      "activations/layer1_attention_weight_max": 16.583032608032227,
      "activations/layer1_attention_weight_min": -14.324457168579102,
      "activations/layer20_attention_weight_max": 33.163143157958984,
      "activations/layer20_attention_weight_min": -24.008214950561523,
      "activations/layer21_attention_weight_max": 30.840627670288086,
      "activations/layer21_attention_weight_min": -22.37721061706543,
      "activations/layer22_attention_weight_max": 48.07914352416992,
      "activations/layer22_attention_weight_min": -29.27144432067871,
      "activations/layer23_attention_weight_max": 34.208290100097656,
      "activations/layer23_attention_weight_min": -24.884357452392578,
      "activations/layer2_attention_weight_max": 32.22506332397461,
      "activations/layer2_attention_weight_min": -30.56026840209961,
      "activations/layer3_attention_weight_max": 92.4226303100586,
      "activations/layer3_attention_weight_min": -94.12271881103516,
      "activations/layer4_attention_weight_max": 112.29571533203125,
      "activations/layer4_attention_weight_min": -112.09972381591797,
      "activations/layer5_attention_weight_max": 51.821041107177734,
      "activations/layer5_attention_weight_min": -61.28643035888672,
      "activations/layer6_attention_weight_max": 47.62815475463867,
      "activations/layer6_attention_weight_min": -46.39558029174805,
      "activations/layer7_attention_weight_max": 84.35456848144531,
      "activations/layer7_attention_weight_min": -84.45704650878906,
      "activations/layer8_attention_weight_max": 37.71046447753906,
      "activations/layer8_attention_weight_min": -38.25825500488281,
      "activations/layer9_attention_weight_max": 30.832448959350586,
      "activations/layer9_attention_weight_min": -32.10816192626953,
      "epoch": 9.52,
      "learning_rate": 8.952386363636364e-05,
      "loss": 2.809,
      "step": 163800
    },
    {
      "activations/layer0_attention_weight_max": 15.57581615447998,
      "activations/layer0_attention_weight_min": -12.854394912719727,
      "activations/layer10_attention_weight_max": 41.87004470825195,
      "activations/layer10_attention_weight_min": -37.19661331176758,
      "activations/layer11_attention_weight_max": 42.04978942871094,
      "activations/layer11_attention_weight_min": -37.07444763183594,
      "activations/layer12_attention_weight_max": 27.029809951782227,
      "activations/layer12_attention_weight_min": -27.37084197998047,
      "activations/layer13_attention_weight_max": 45.80791091918945,
      "activations/layer13_attention_weight_min": -35.96554946899414,
      "activations/layer14_attention_weight_max": 60.88985824584961,
      "activations/layer14_attention_weight_min": -43.00922393798828,
      "activations/layer15_attention_weight_max": 40.61985778808594,
      "activations/layer15_attention_weight_min": -33.437171936035156,
      "activations/layer16_attention_weight_max": 37.21446228027344,
      "activations/layer16_attention_weight_min": -28.20635986328125,
      "activations/layer17_attention_weight_max": 37.3353385925293,
      "activations/layer17_attention_weight_min": -28.434419631958008,
      "activations/layer18_attention_weight_max": 33.61842727661133,
      "activations/layer18_attention_weight_min": -26.172237396240234,
      "activations/layer19_attention_weight_max": 41.35404586791992,
      "activations/layer19_attention_weight_min": -33.181907653808594,
      "activations/layer1_attention_weight_max": 16.209701538085938,
      "activations/layer1_attention_weight_min": -13.999370574951172,
      "activations/layer20_attention_weight_max": 36.00350570678711,
      "activations/layer20_attention_weight_min": -24.963829040527344,
      "activations/layer21_attention_weight_max": 34.78740692138672,
      "activations/layer21_attention_weight_min": -23.744863510131836,
      "activations/layer22_attention_weight_max": 53.81031799316406,
      "activations/layer22_attention_weight_min": -32.1955451965332,
      "activations/layer23_attention_weight_max": 37.48406219482422,
      "activations/layer23_attention_weight_min": -24.29097557067871,
      "activations/layer2_attention_weight_max": 32.795860290527344,
      "activations/layer2_attention_weight_min": -31.762020111083984,
      "activations/layer3_attention_weight_max": 94.0429916381836,
      "activations/layer3_attention_weight_min": -97.00157928466797,
      "activations/layer4_attention_weight_max": 119.06493377685547,
      "activations/layer4_attention_weight_min": -116.05340576171875,
      "activations/layer5_attention_weight_max": 52.86463928222656,
      "activations/layer5_attention_weight_min": -60.47520446777344,
      "activations/layer6_attention_weight_max": 46.5097541809082,
      "activations/layer6_attention_weight_min": -50.21244430541992,
      "activations/layer7_attention_weight_max": 99.10579681396484,
      "activations/layer7_attention_weight_min": -93.74472045898438,
      "activations/layer8_attention_weight_max": 47.84421920776367,
      "activations/layer8_attention_weight_min": -44.86742401123047,
      "activations/layer9_attention_weight_max": 42.01736831665039,
      "activations/layer9_attention_weight_min": -40.438419342041016,
      "epoch": 9.52,
      "learning_rate": 8.950492424242423e-05,
      "loss": 2.7955,
      "step": 163850
    },
    {
      "activations/layer0_attention_weight_max": 15.718099594116211,
      "activations/layer0_attention_weight_min": -13.189888954162598,
      "activations/layer10_attention_weight_max": 32.502784729003906,
      "activations/layer10_attention_weight_min": -30.604019165039062,
      "activations/layer11_attention_weight_max": 31.690536499023438,
      "activations/layer11_attention_weight_min": -29.072681427001953,
      "activations/layer12_attention_weight_max": 25.406225204467773,
      "activations/layer12_attention_weight_min": -23.874176025390625,
      "activations/layer13_attention_weight_max": 43.79526138305664,
      "activations/layer13_attention_weight_min": -33.19097137451172,
      "activations/layer14_attention_weight_max": 44.87346267700195,
      "activations/layer14_attention_weight_min": -37.649009704589844,
      "activations/layer15_attention_weight_max": 37.382423400878906,
      "activations/layer15_attention_weight_min": -32.53718948364258,
      "activations/layer16_attention_weight_max": 34.03791427612305,
      "activations/layer16_attention_weight_min": -28.350706100463867,
      "activations/layer17_attention_weight_max": 35.242191314697266,
      "activations/layer17_attention_weight_min": -24.334129333496094,
      "activations/layer18_attention_weight_max": 33.5026969909668,
      "activations/layer18_attention_weight_min": -21.910783767700195,
      "activations/layer19_attention_weight_max": 35.984859466552734,
      "activations/layer19_attention_weight_min": -29.32033348083496,
      "activations/layer1_attention_weight_max": 16.883724212646484,
      "activations/layer1_attention_weight_min": -14.402142524719238,
      "activations/layer20_attention_weight_max": 31.956829071044922,
      "activations/layer20_attention_weight_min": -22.744911193847656,
      "activations/layer21_attention_weight_max": 32.196876525878906,
      "activations/layer21_attention_weight_min": -21.248449325561523,
      "activations/layer22_attention_weight_max": 47.265838623046875,
      "activations/layer22_attention_weight_min": -30.26993179321289,
      "activations/layer23_attention_weight_max": 35.590293884277344,
      "activations/layer23_attention_weight_min": -25.394123077392578,
      "activations/layer2_attention_weight_max": 31.056913375854492,
      "activations/layer2_attention_weight_min": -30.087276458740234,
      "activations/layer3_attention_weight_max": 89.41313934326172,
      "activations/layer3_attention_weight_min": -88.95735168457031,
      "activations/layer4_attention_weight_max": 110.8876724243164,
      "activations/layer4_attention_weight_min": -108.76787567138672,
      "activations/layer5_attention_weight_max": 51.66841125488281,
      "activations/layer5_attention_weight_min": -60.51798629760742,
      "activations/layer6_attention_weight_max": 44.60715103149414,
      "activations/layer6_attention_weight_min": -49.646339416503906,
      "activations/layer7_attention_weight_max": 91.1861343383789,
      "activations/layer7_attention_weight_min": -87.59127807617188,
      "activations/layer8_attention_weight_max": 39.83792495727539,
      "activations/layer8_attention_weight_min": -42.23897171020508,
      "activations/layer9_attention_weight_max": 34.54531478881836,
      "activations/layer9_attention_weight_min": -33.206146240234375,
      "epoch": 9.52,
      "learning_rate": 8.948598484848484e-05,
      "loss": 2.8005,
      "step": 163900
    },
    {
      "activations/layer0_attention_weight_max": 14.968445777893066,
      "activations/layer0_attention_weight_min": -12.162176132202148,
      "activations/layer10_attention_weight_max": 32.813331604003906,
      "activations/layer10_attention_weight_min": -33.45710754394531,
      "activations/layer11_attention_weight_max": 33.14019012451172,
      "activations/layer11_attention_weight_min": -32.479087829589844,
      "activations/layer12_attention_weight_max": 23.771154403686523,
      "activations/layer12_attention_weight_min": -24.398027420043945,
      "activations/layer13_attention_weight_max": 37.481056213378906,
      "activations/layer13_attention_weight_min": -33.607704162597656,
      "activations/layer14_attention_weight_max": 45.62955093383789,
      "activations/layer14_attention_weight_min": -43.296348571777344,
      "activations/layer15_attention_weight_max": 35.88264846801758,
      "activations/layer15_attention_weight_min": -29.754161834716797,
      "activations/layer16_attention_weight_max": 31.534204483032227,
      "activations/layer16_attention_weight_min": -27.70077133178711,
      "activations/layer17_attention_weight_max": 31.212533950805664,
      "activations/layer17_attention_weight_min": -25.610464096069336,
      "activations/layer18_attention_weight_max": 29.366878509521484,
      "activations/layer18_attention_weight_min": -22.187843322753906,
      "activations/layer19_attention_weight_max": 35.81547164916992,
      "activations/layer19_attention_weight_min": -31.495365142822266,
      "activations/layer1_attention_weight_max": 17.069913864135742,
      "activations/layer1_attention_weight_min": -15.534479141235352,
      "activations/layer20_attention_weight_max": 29.787498474121094,
      "activations/layer20_attention_weight_min": -24.533092498779297,
      "activations/layer21_attention_weight_max": 28.047794342041016,
      "activations/layer21_attention_weight_min": -23.040283203125,
      "activations/layer22_attention_weight_max": 46.745849609375,
      "activations/layer22_attention_weight_min": -29.975997924804688,
      "activations/layer23_attention_weight_max": 36.580223083496094,
      "activations/layer23_attention_weight_min": -23.318464279174805,
      "activations/layer2_attention_weight_max": 31.31549644470215,
      "activations/layer2_attention_weight_min": -32.14251708984375,
      "activations/layer3_attention_weight_max": 90.143310546875,
      "activations/layer3_attention_weight_min": -91.76713562011719,
      "activations/layer4_attention_weight_max": 113.8672866821289,
      "activations/layer4_attention_weight_min": -113.64067840576172,
      "activations/layer5_attention_weight_max": 52.87179183959961,
      "activations/layer5_attention_weight_min": -60.064048767089844,
      "activations/layer6_attention_weight_max": 45.31842803955078,
      "activations/layer6_attention_weight_min": -50.17832946777344,
      "activations/layer7_attention_weight_max": 90.78345489501953,
      "activations/layer7_attention_weight_min": -91.56788635253906,
      "activations/layer8_attention_weight_max": 40.97666549682617,
      "activations/layer8_attention_weight_min": -42.60488510131836,
      "activations/layer9_attention_weight_max": 35.24082565307617,
      "activations/layer9_attention_weight_min": -35.15156936645508,
      "epoch": 9.53,
      "learning_rate": 8.946704545454544e-05,
      "loss": 2.7786,
      "step": 163950
    },
    {
      "activations/layer0_attention_weight_max": 15.388460159301758,
      "activations/layer0_attention_weight_min": -13.314931869506836,
      "activations/layer10_attention_weight_max": 31.030099868774414,
      "activations/layer10_attention_weight_min": -30.54683494567871,
      "activations/layer11_attention_weight_max": 31.100933074951172,
      "activations/layer11_attention_weight_min": -29.289228439331055,
      "activations/layer12_attention_weight_max": 22.61709213256836,
      "activations/layer12_attention_weight_min": -23.61289405822754,
      "activations/layer13_attention_weight_max": 37.98503494262695,
      "activations/layer13_attention_weight_min": -31.5894718170166,
      "activations/layer14_attention_weight_max": 45.522300720214844,
      "activations/layer14_attention_weight_min": -35.23580551147461,
      "activations/layer15_attention_weight_max": 34.406002044677734,
      "activations/layer15_attention_weight_min": -30.41924285888672,
      "activations/layer16_attention_weight_max": 31.80544090270996,
      "activations/layer16_attention_weight_min": -25.80823516845703,
      "activations/layer17_attention_weight_max": 36.640037536621094,
      "activations/layer17_attention_weight_min": -23.759305953979492,
      "activations/layer18_attention_weight_max": 32.03178405761719,
      "activations/layer18_attention_weight_min": -21.852645874023438,
      "activations/layer19_attention_weight_max": 34.38060760498047,
      "activations/layer19_attention_weight_min": -30.295652389526367,
      "activations/layer1_attention_weight_max": 16.641983032226562,
      "activations/layer1_attention_weight_min": -14.583778381347656,
      "activations/layer20_attention_weight_max": 30.650178909301758,
      "activations/layer20_attention_weight_min": -25.74603271484375,
      "activations/layer21_attention_weight_max": 29.197385787963867,
      "activations/layer21_attention_weight_min": -24.114444732666016,
      "activations/layer22_attention_weight_max": 48.628578186035156,
      "activations/layer22_attention_weight_min": -30.20784568786621,
      "activations/layer23_attention_weight_max": 33.11612319946289,
      "activations/layer23_attention_weight_min": -22.89040184020996,
      "activations/layer2_attention_weight_max": 32.478118896484375,
      "activations/layer2_attention_weight_min": -31.080198287963867,
      "activations/layer3_attention_weight_max": 91.4040756225586,
      "activations/layer3_attention_weight_min": -87.94129180908203,
      "activations/layer4_attention_weight_max": 112.42474365234375,
      "activations/layer4_attention_weight_min": -110.9855728149414,
      "activations/layer5_attention_weight_max": 51.60969161987305,
      "activations/layer5_attention_weight_min": -59.006954193115234,
      "activations/layer6_attention_weight_max": 44.12533187866211,
      "activations/layer6_attention_weight_min": -46.620052337646484,
      "activations/layer7_attention_weight_max": 88.67030334472656,
      "activations/layer7_attention_weight_min": -84.26300048828125,
      "activations/layer8_attention_weight_max": 38.48162841796875,
      "activations/layer8_attention_weight_min": -40.13218307495117,
      "activations/layer9_attention_weight_max": 33.46675491333008,
      "activations/layer9_attention_weight_min": -32.29340744018555,
      "epoch": 9.53,
      "learning_rate": 8.944810606060605e-05,
      "loss": 2.7749,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.435,
      "eval_samples_per_second": 509.069,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.435,
      "eval_openwebtext_samples_per_second": 509.069,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9276,
      "eval_wikitext_samples_per_second": 236.568,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_lambada_loss": 2.625,
      "eval_lambada_ppl": 13.804574186067095,
      "eval_lambada_runtime": 9.5129,
      "eval_lambada_samples_per_second": 511.83,
      "step": 164000
    },
    {
      "activations/layer0_attention_weight_max": 15.108220100402832,
      "activations/layer0_attention_weight_min": -12.80417537689209,
      "activations/layer10_attention_weight_max": 35.566673278808594,
      "activations/layer10_attention_weight_min": -32.505096435546875,
      "activations/layer11_attention_weight_max": 31.781871795654297,
      "activations/layer11_attention_weight_min": -30.16628074645996,
      "activations/layer12_attention_weight_max": 24.13477897644043,
      "activations/layer12_attention_weight_min": -27.65261459350586,
      "activations/layer13_attention_weight_max": 40.26913070678711,
      "activations/layer13_attention_weight_min": -35.91743087768555,
      "activations/layer14_attention_weight_max": 46.23025894165039,
      "activations/layer14_attention_weight_min": -37.5935173034668,
      "activations/layer15_attention_weight_max": 37.960350036621094,
      "activations/layer15_attention_weight_min": -30.528831481933594,
      "activations/layer16_attention_weight_max": 31.21242332458496,
      "activations/layer16_attention_weight_min": -27.150535583496094,
      "activations/layer17_attention_weight_max": 34.4947395324707,
      "activations/layer17_attention_weight_min": -25.847414016723633,
      "activations/layer18_attention_weight_max": 31.9750919342041,
      "activations/layer18_attention_weight_min": -22.54912757873535,
      "activations/layer19_attention_weight_max": 36.488990783691406,
      "activations/layer19_attention_weight_min": -30.300399780273438,
      "activations/layer1_attention_weight_max": 17.37310028076172,
      "activations/layer1_attention_weight_min": -13.17908763885498,
      "activations/layer20_attention_weight_max": 32.61351013183594,
      "activations/layer20_attention_weight_min": -25.93798065185547,
      "activations/layer21_attention_weight_max": 32.65633773803711,
      "activations/layer21_attention_weight_min": -24.498266220092773,
      "activations/layer22_attention_weight_max": 45.88243103027344,
      "activations/layer22_attention_weight_min": -30.356252670288086,
      "activations/layer23_attention_weight_max": 38.96517562866211,
      "activations/layer23_attention_weight_min": -25.509923934936523,
      "activations/layer2_attention_weight_max": 34.27859878540039,
      "activations/layer2_attention_weight_min": -31.566547393798828,
      "activations/layer3_attention_weight_max": 98.56958770751953,
      "activations/layer3_attention_weight_min": -94.79243469238281,
      "activations/layer4_attention_weight_max": 123.1072006225586,
      "activations/layer4_attention_weight_min": -114.99238586425781,
      "activations/layer5_attention_weight_max": 58.34844970703125,
      "activations/layer5_attention_weight_min": -63.567649841308594,
      "activations/layer6_attention_weight_max": 50.574501037597656,
      "activations/layer6_attention_weight_min": -50.86396789550781,
      "activations/layer7_attention_weight_max": 91.74006652832031,
      "activations/layer7_attention_weight_min": -87.64141845703125,
      "activations/layer8_attention_weight_max": 41.58047866821289,
      "activations/layer8_attention_weight_min": -43.67896270751953,
      "activations/layer9_attention_weight_max": 36.804256439208984,
      "activations/layer9_attention_weight_min": -35.26079177856445,
      "epoch": 9.53,
      "learning_rate": 8.942954545454544e-05,
      "loss": 2.7912,
      "step": 164050
    },
    {
      "activations/layer0_attention_weight_max": 16.309261322021484,
      "activations/layer0_attention_weight_min": -12.300029754638672,
      "activations/layer10_attention_weight_max": 30.815855026245117,
      "activations/layer10_attention_weight_min": -32.42378616333008,
      "activations/layer11_attention_weight_max": 30.31198501586914,
      "activations/layer11_attention_weight_min": -31.1083984375,
      "activations/layer12_attention_weight_max": 21.331100463867188,
      "activations/layer12_attention_weight_min": -27.891393661499023,
      "activations/layer13_attention_weight_max": 34.42107391357422,
      "activations/layer13_attention_weight_min": -35.08867263793945,
      "activations/layer14_attention_weight_max": 40.22759246826172,
      "activations/layer14_attention_weight_min": -36.22060012817383,
      "activations/layer15_attention_weight_max": 33.223209381103516,
      "activations/layer15_attention_weight_min": -30.20970344543457,
      "activations/layer16_attention_weight_max": 28.640844345092773,
      "activations/layer16_attention_weight_min": -30.953617095947266,
      "activations/layer17_attention_weight_max": 31.937408447265625,
      "activations/layer17_attention_weight_min": -26.669416427612305,
      "activations/layer18_attention_weight_max": 30.14421272277832,
      "activations/layer18_attention_weight_min": -22.950334548950195,
      "activations/layer19_attention_weight_max": 33.64522933959961,
      "activations/layer19_attention_weight_min": -29.834012985229492,
      "activations/layer1_attention_weight_max": 16.417072296142578,
      "activations/layer1_attention_weight_min": -15.271747589111328,
      "activations/layer20_attention_weight_max": 29.536808013916016,
      "activations/layer20_attention_weight_min": -24.504060745239258,
      "activations/layer21_attention_weight_max": 27.60063362121582,
      "activations/layer21_attention_weight_min": -22.697866439819336,
      "activations/layer22_attention_weight_max": 42.25774383544922,
      "activations/layer22_attention_weight_min": -27.911184310913086,
      "activations/layer23_attention_weight_max": 36.7568473815918,
      "activations/layer23_attention_weight_min": -22.483922958374023,
      "activations/layer2_attention_weight_max": 29.51567840576172,
      "activations/layer2_attention_weight_min": -31.350515365600586,
      "activations/layer3_attention_weight_max": 87.69226837158203,
      "activations/layer3_attention_weight_min": -92.13948822021484,
      "activations/layer4_attention_weight_max": 116.23979949951172,
      "activations/layer4_attention_weight_min": -120.07197570800781,
      "activations/layer5_attention_weight_max": 62.256954193115234,
      "activations/layer5_attention_weight_min": -63.90156555175781,
      "activations/layer6_attention_weight_max": 48.29186248779297,
      "activations/layer6_attention_weight_min": -52.549129486083984,
      "activations/layer7_attention_weight_max": 90.77678680419922,
      "activations/layer7_attention_weight_min": -90.15396118164062,
      "activations/layer8_attention_weight_max": 39.14900588989258,
      "activations/layer8_attention_weight_min": -43.42628479003906,
      "activations/layer9_attention_weight_max": 34.84097671508789,
      "activations/layer9_attention_weight_min": -35.74538803100586,
      "epoch": 9.54,
      "learning_rate": 8.941060606060605e-05,
      "loss": 2.8097,
      "step": 164100
    },
    {
      "activations/layer0_attention_weight_max": 15.04898452758789,
      "activations/layer0_attention_weight_min": -12.858798027038574,
      "activations/layer10_attention_weight_max": 35.96848678588867,
      "activations/layer10_attention_weight_min": -34.38567352294922,
      "activations/layer11_attention_weight_max": 35.92457580566406,
      "activations/layer11_attention_weight_min": -35.64824676513672,
      "activations/layer12_attention_weight_max": 25.394060134887695,
      "activations/layer12_attention_weight_min": -26.236995697021484,
      "activations/layer13_attention_weight_max": 43.283817291259766,
      "activations/layer13_attention_weight_min": -37.6885871887207,
      "activations/layer14_attention_weight_max": 57.52998733520508,
      "activations/layer14_attention_weight_min": -45.644412994384766,
      "activations/layer15_attention_weight_max": 43.541168212890625,
      "activations/layer15_attention_weight_min": -33.72261428833008,
      "activations/layer16_attention_weight_max": 31.85260772705078,
      "activations/layer16_attention_weight_min": -27.37877082824707,
      "activations/layer17_attention_weight_max": 34.80084991455078,
      "activations/layer17_attention_weight_min": -25.746816635131836,
      "activations/layer18_attention_weight_max": 32.09825134277344,
      "activations/layer18_attention_weight_min": -22.272544860839844,
      "activations/layer19_attention_weight_max": 41.886844635009766,
      "activations/layer19_attention_weight_min": -27.430191040039062,
      "activations/layer1_attention_weight_max": 17.50094985961914,
      "activations/layer1_attention_weight_min": -14.5592679977417,
      "activations/layer20_attention_weight_max": 32.83599090576172,
      "activations/layer20_attention_weight_min": -24.691333770751953,
      "activations/layer21_attention_weight_max": 31.875621795654297,
      "activations/layer21_attention_weight_min": -22.275915145874023,
      "activations/layer22_attention_weight_max": 51.12099838256836,
      "activations/layer22_attention_weight_min": -29.25919532775879,
      "activations/layer23_attention_weight_max": 36.67127227783203,
      "activations/layer23_attention_weight_min": -26.54829978942871,
      "activations/layer2_attention_weight_max": 31.093364715576172,
      "activations/layer2_attention_weight_min": -29.053211212158203,
      "activations/layer3_attention_weight_max": 89.24369812011719,
      "activations/layer3_attention_weight_min": -85.62720489501953,
      "activations/layer4_attention_weight_max": 110.93502044677734,
      "activations/layer4_attention_weight_min": -108.08197021484375,
      "activations/layer5_attention_weight_max": 54.006446838378906,
      "activations/layer5_attention_weight_min": -59.799903869628906,
      "activations/layer6_attention_weight_max": 49.18439483642578,
      "activations/layer6_attention_weight_min": -49.2238883972168,
      "activations/layer7_attention_weight_max": 89.60328674316406,
      "activations/layer7_attention_weight_min": -90.65626525878906,
      "activations/layer8_attention_weight_max": 41.164306640625,
      "activations/layer8_attention_weight_min": -42.51176834106445,
      "activations/layer9_attention_weight_max": 36.10392761230469,
      "activations/layer9_attention_weight_min": -35.18593215942383,
      "epoch": 9.54,
      "learning_rate": 8.939166666666667e-05,
      "loss": 2.778,
      "step": 164150
    },
    {
      "activations/layer0_attention_weight_max": 15.668203353881836,
      "activations/layer0_attention_weight_min": -12.592008590698242,
      "activations/layer10_attention_weight_max": 33.902679443359375,
      "activations/layer10_attention_weight_min": -33.14649200439453,
      "activations/layer11_attention_weight_max": 33.40342712402344,
      "activations/layer11_attention_weight_min": -31.061573028564453,
      "activations/layer12_attention_weight_max": 23.31948471069336,
      "activations/layer12_attention_weight_min": -23.26521110534668,
      "activations/layer13_attention_weight_max": 36.695106506347656,
      "activations/layer13_attention_weight_min": -31.998374938964844,
      "activations/layer14_attention_weight_max": 46.3525505065918,
      "activations/layer14_attention_weight_min": -42.350040435791016,
      "activations/layer15_attention_weight_max": 33.55297088623047,
      "activations/layer15_attention_weight_min": -29.72407341003418,
      "activations/layer16_attention_weight_max": 29.232873916625977,
      "activations/layer16_attention_weight_min": -28.880895614624023,
      "activations/layer17_attention_weight_max": 35.76773452758789,
      "activations/layer17_attention_weight_min": -26.14690589904785,
      "activations/layer18_attention_weight_max": 32.8504753112793,
      "activations/layer18_attention_weight_min": -22.0865535736084,
      "activations/layer19_attention_weight_max": 33.921260833740234,
      "activations/layer19_attention_weight_min": -31.114633560180664,
      "activations/layer1_attention_weight_max": 16.69295883178711,
      "activations/layer1_attention_weight_min": -13.75615119934082,
      "activations/layer20_attention_weight_max": 31.341856002807617,
      "activations/layer20_attention_weight_min": -25.205476760864258,
      "activations/layer21_attention_weight_max": 28.05788230895996,
      "activations/layer21_attention_weight_min": -22.757261276245117,
      "activations/layer22_attention_weight_max": 44.92566680908203,
      "activations/layer22_attention_weight_min": -28.00441551208496,
      "activations/layer23_attention_weight_max": 31.86920166015625,
      "activations/layer23_attention_weight_min": -24.171438217163086,
      "activations/layer2_attention_weight_max": 31.034086227416992,
      "activations/layer2_attention_weight_min": -30.50996971130371,
      "activations/layer3_attention_weight_max": 93.36981201171875,
      "activations/layer3_attention_weight_min": -92.99624633789062,
      "activations/layer4_attention_weight_max": 118.68251037597656,
      "activations/layer4_attention_weight_min": -115.82185363769531,
      "activations/layer5_attention_weight_max": 59.422401428222656,
      "activations/layer5_attention_weight_min": -61.55868148803711,
      "activations/layer6_attention_weight_max": 48.554325103759766,
      "activations/layer6_attention_weight_min": -53.04390335083008,
      "activations/layer7_attention_weight_max": 91.50980377197266,
      "activations/layer7_attention_weight_min": -96.18514251708984,
      "activations/layer8_attention_weight_max": 40.767704010009766,
      "activations/layer8_attention_weight_min": -41.909549713134766,
      "activations/layer9_attention_weight_max": 35.96329879760742,
      "activations/layer9_attention_weight_min": -34.41859436035156,
      "epoch": 9.54,
      "learning_rate": 8.937272727272727e-05,
      "loss": 2.7961,
      "step": 164200
    },
    {
      "activations/layer0_attention_weight_max": 15.190582275390625,
      "activations/layer0_attention_weight_min": -12.134008407592773,
      "activations/layer10_attention_weight_max": 34.49288558959961,
      "activations/layer10_attention_weight_min": -32.69353103637695,
      "activations/layer11_attention_weight_max": 34.400726318359375,
      "activations/layer11_attention_weight_min": -32.0133056640625,
      "activations/layer12_attention_weight_max": 42.62317657470703,
      "activations/layer12_attention_weight_min": -27.357410430908203,
      "activations/layer13_attention_weight_max": 52.05329132080078,
      "activations/layer13_attention_weight_min": -35.52819061279297,
      "activations/layer14_attention_weight_max": 44.19591522216797,
      "activations/layer14_attention_weight_min": -39.38541793823242,
      "activations/layer15_attention_weight_max": 37.75640106201172,
      "activations/layer15_attention_weight_min": -31.36359977722168,
      "activations/layer16_attention_weight_max": 32.015140533447266,
      "activations/layer16_attention_weight_min": -28.272968292236328,
      "activations/layer17_attention_weight_max": 32.67787551879883,
      "activations/layer17_attention_weight_min": -24.692703247070312,
      "activations/layer18_attention_weight_max": 30.375572204589844,
      "activations/layer18_attention_weight_min": -21.05967903137207,
      "activations/layer19_attention_weight_max": 35.06583786010742,
      "activations/layer19_attention_weight_min": -30.937610626220703,
      "activations/layer1_attention_weight_max": 18.68212127685547,
      "activations/layer1_attention_weight_min": -14.216533660888672,
      "activations/layer20_attention_weight_max": 31.8746337890625,
      "activations/layer20_attention_weight_min": -24.677734375,
      "activations/layer21_attention_weight_max": 29.52236557006836,
      "activations/layer21_attention_weight_min": -23.103641510009766,
      "activations/layer22_attention_weight_max": 42.98068618774414,
      "activations/layer22_attention_weight_min": -28.028345108032227,
      "activations/layer23_attention_weight_max": 33.89553451538086,
      "activations/layer23_attention_weight_min": -22.007888793945312,
      "activations/layer2_attention_weight_max": 33.84392166137695,
      "activations/layer2_attention_weight_min": -32.08013916015625,
      "activations/layer3_attention_weight_max": 90.7786636352539,
      "activations/layer3_attention_weight_min": -90.82850646972656,
      "activations/layer4_attention_weight_max": 113.6439437866211,
      "activations/layer4_attention_weight_min": -109.99310302734375,
      "activations/layer5_attention_weight_max": 52.946022033691406,
      "activations/layer5_attention_weight_min": -61.25768280029297,
      "activations/layer6_attention_weight_max": 47.85531997680664,
      "activations/layer6_attention_weight_min": -48.8840446472168,
      "activations/layer7_attention_weight_max": 94.3459701538086,
      "activations/layer7_attention_weight_min": -88.32581329345703,
      "activations/layer8_attention_weight_max": 42.20415115356445,
      "activations/layer8_attention_weight_min": -40.663848876953125,
      "activations/layer9_attention_weight_max": 35.90359878540039,
      "activations/layer9_attention_weight_min": -32.85932540893555,
      "epoch": 9.54,
      "learning_rate": 8.935378787878787e-05,
      "loss": 2.7956,
      "step": 164250
    },
    {
      "activations/layer0_attention_weight_max": 14.847195625305176,
      "activations/layer0_attention_weight_min": -12.087427139282227,
      "activations/layer10_attention_weight_max": 33.362823486328125,
      "activations/layer10_attention_weight_min": -32.0870361328125,
      "activations/layer11_attention_weight_max": 34.333953857421875,
      "activations/layer11_attention_weight_min": -31.853546142578125,
      "activations/layer12_attention_weight_max": 24.72423553466797,
      "activations/layer12_attention_weight_min": -26.707862854003906,
      "activations/layer13_attention_weight_max": 38.550270080566406,
      "activations/layer13_attention_weight_min": -34.1694221496582,
      "activations/layer14_attention_weight_max": 45.04047775268555,
      "activations/layer14_attention_weight_min": -39.758628845214844,
      "activations/layer15_attention_weight_max": 36.05910110473633,
      "activations/layer15_attention_weight_min": -29.508949279785156,
      "activations/layer16_attention_weight_max": 33.81239700317383,
      "activations/layer16_attention_weight_min": -27.044118881225586,
      "activations/layer17_attention_weight_max": 35.24258804321289,
      "activations/layer17_attention_weight_min": -25.26808738708496,
      "activations/layer18_attention_weight_max": 33.766868591308594,
      "activations/layer18_attention_weight_min": -22.295488357543945,
      "activations/layer19_attention_weight_max": 32.31217575073242,
      "activations/layer19_attention_weight_min": -29.482471466064453,
      "activations/layer1_attention_weight_max": 16.915851593017578,
      "activations/layer1_attention_weight_min": -15.711204528808594,
      "activations/layer20_attention_weight_max": 31.4639892578125,
      "activations/layer20_attention_weight_min": -24.189048767089844,
      "activations/layer21_attention_weight_max": 29.91290283203125,
      "activations/layer21_attention_weight_min": -23.41807746887207,
      "activations/layer22_attention_weight_max": 44.66260528564453,
      "activations/layer22_attention_weight_min": -27.221948623657227,
      "activations/layer23_attention_weight_max": 33.33287811279297,
      "activations/layer23_attention_weight_min": -22.08498191833496,
      "activations/layer2_attention_weight_max": 29.342697143554688,
      "activations/layer2_attention_weight_min": -29.436492919921875,
      "activations/layer3_attention_weight_max": 87.83036041259766,
      "activations/layer3_attention_weight_min": -85.54918670654297,
      "activations/layer4_attention_weight_max": 110.20025634765625,
      "activations/layer4_attention_weight_min": -107.36427307128906,
      "activations/layer5_attention_weight_max": 55.768280029296875,
      "activations/layer5_attention_weight_min": -59.701515197753906,
      "activations/layer6_attention_weight_max": 47.29825210571289,
      "activations/layer6_attention_weight_min": -49.82866668701172,
      "activations/layer7_attention_weight_max": 91.95795440673828,
      "activations/layer7_attention_weight_min": -87.63676452636719,
      "activations/layer8_attention_weight_max": 38.8288459777832,
      "activations/layer8_attention_weight_min": -40.19490051269531,
      "activations/layer9_attention_weight_max": 34.4779167175293,
      "activations/layer9_attention_weight_min": -33.26711654663086,
      "epoch": 9.55,
      "learning_rate": 8.933484848484847e-05,
      "loss": 2.7891,
      "step": 164300
    },
    {
      "activations/layer0_attention_weight_max": 15.585440635681152,
      "activations/layer0_attention_weight_min": -11.927699089050293,
      "activations/layer10_attention_weight_max": 32.060264587402344,
      "activations/layer10_attention_weight_min": -30.320341110229492,
      "activations/layer11_attention_weight_max": 32.18622970581055,
      "activations/layer11_attention_weight_min": -30.100566864013672,
      "activations/layer12_attention_weight_max": 26.487375259399414,
      "activations/layer12_attention_weight_min": -26.03734016418457,
      "activations/layer13_attention_weight_max": 42.42877960205078,
      "activations/layer13_attention_weight_min": -33.7242317199707,
      "activations/layer14_attention_weight_max": 49.5695915222168,
      "activations/layer14_attention_weight_min": -40.2461051940918,
      "activations/layer15_attention_weight_max": 34.87236022949219,
      "activations/layer15_attention_weight_min": -31.091808319091797,
      "activations/layer16_attention_weight_max": 31.71580696105957,
      "activations/layer16_attention_weight_min": -27.824703216552734,
      "activations/layer17_attention_weight_max": 32.066158294677734,
      "activations/layer17_attention_weight_min": -25.575456619262695,
      "activations/layer18_attention_weight_max": 31.124441146850586,
      "activations/layer18_attention_weight_min": -23.188161849975586,
      "activations/layer19_attention_weight_max": 35.85158920288086,
      "activations/layer19_attention_weight_min": -31.336225509643555,
      "activations/layer1_attention_weight_max": 15.863945960998535,
      "activations/layer1_attention_weight_min": -15.51799488067627,
      "activations/layer20_attention_weight_max": 29.957962036132812,
      "activations/layer20_attention_weight_min": -23.887174606323242,
      "activations/layer21_attention_weight_max": 29.46622657775879,
      "activations/layer21_attention_weight_min": -22.836288452148438,
      "activations/layer22_attention_weight_max": 51.72922897338867,
      "activations/layer22_attention_weight_min": -31.165449142456055,
      "activations/layer23_attention_weight_max": 38.52385711669922,
      "activations/layer23_attention_weight_min": -25.00606918334961,
      "activations/layer2_attention_weight_max": 30.719959259033203,
      "activations/layer2_attention_weight_min": -31.682355880737305,
      "activations/layer3_attention_weight_max": 89.43816375732422,
      "activations/layer3_attention_weight_min": -94.56734466552734,
      "activations/layer4_attention_weight_max": 111.71142578125,
      "activations/layer4_attention_weight_min": -109.76000213623047,
      "activations/layer5_attention_weight_max": 53.63587951660156,
      "activations/layer5_attention_weight_min": -60.780433654785156,
      "activations/layer6_attention_weight_max": 46.968238830566406,
      "activations/layer6_attention_weight_min": -49.32784652709961,
      "activations/layer7_attention_weight_max": 90.65482330322266,
      "activations/layer7_attention_weight_min": -89.80313873291016,
      "activations/layer8_attention_weight_max": 42.950111389160156,
      "activations/layer8_attention_weight_min": -39.70427703857422,
      "activations/layer9_attention_weight_max": 33.58323669433594,
      "activations/layer9_attention_weight_min": -31.355527877807617,
      "epoch": 9.55,
      "learning_rate": 8.931590909090909e-05,
      "loss": 2.7935,
      "step": 164350
    },
    {
      "activations/layer0_attention_weight_max": 15.633626937866211,
      "activations/layer0_attention_weight_min": -12.70881175994873,
      "activations/layer10_attention_weight_max": 31.964139938354492,
      "activations/layer10_attention_weight_min": -32.93291091918945,
      "activations/layer11_attention_weight_max": 31.64003562927246,
      "activations/layer11_attention_weight_min": -31.513214111328125,
      "activations/layer12_attention_weight_max": 23.18065071105957,
      "activations/layer12_attention_weight_min": -25.2163028717041,
      "activations/layer13_attention_weight_max": 35.83528518676758,
      "activations/layer13_attention_weight_min": -35.55852508544922,
      "activations/layer14_attention_weight_max": 43.316593170166016,
      "activations/layer14_attention_weight_min": -37.020877838134766,
      "activations/layer15_attention_weight_max": 34.5845947265625,
      "activations/layer15_attention_weight_min": -30.78905487060547,
      "activations/layer16_attention_weight_max": 31.26628303527832,
      "activations/layer16_attention_weight_min": -29.46285057067871,
      "activations/layer17_attention_weight_max": 31.551416397094727,
      "activations/layer17_attention_weight_min": -25.992589950561523,
      "activations/layer18_attention_weight_max": 32.65327835083008,
      "activations/layer18_attention_weight_min": -21.29723358154297,
      "activations/layer19_attention_weight_max": 33.552433013916016,
      "activations/layer19_attention_weight_min": -29.56817054748535,
      "activations/layer1_attention_weight_max": 16.524568557739258,
      "activations/layer1_attention_weight_min": -14.240582466125488,
      "activations/layer20_attention_weight_max": 29.630325317382812,
      "activations/layer20_attention_weight_min": -25.259965896606445,
      "activations/layer21_attention_weight_max": 30.06899070739746,
      "activations/layer21_attention_weight_min": -23.678457260131836,
      "activations/layer22_attention_weight_max": 47.691036224365234,
      "activations/layer22_attention_weight_min": -32.94403076171875,
      "activations/layer23_attention_weight_max": 36.95446014404297,
      "activations/layer23_attention_weight_min": -23.870643615722656,
      "activations/layer2_attention_weight_max": 31.054115295410156,
      "activations/layer2_attention_weight_min": -31.522666931152344,
      "activations/layer3_attention_weight_max": 90.70685577392578,
      "activations/layer3_attention_weight_min": -91.5240707397461,
      "activations/layer4_attention_weight_max": 115.81947326660156,
      "activations/layer4_attention_weight_min": -111.18907928466797,
      "activations/layer5_attention_weight_max": 55.126060485839844,
      "activations/layer5_attention_weight_min": -62.64338302612305,
      "activations/layer6_attention_weight_max": 47.169700622558594,
      "activations/layer6_attention_weight_min": -49.75956344604492,
      "activations/layer7_attention_weight_max": 88.33413696289062,
      "activations/layer7_attention_weight_min": -92.15617370605469,
      "activations/layer8_attention_weight_max": 41.61836242675781,
      "activations/layer8_attention_weight_min": -40.73262023925781,
      "activations/layer9_attention_weight_max": 36.3283576965332,
      "activations/layer9_attention_weight_min": -35.21579360961914,
      "epoch": 9.55,
      "learning_rate": 8.92969696969697e-05,
      "loss": 2.8069,
      "step": 164400
    },
    {
      "activations/layer0_attention_weight_max": 15.739986419677734,
      "activations/layer0_attention_weight_min": -12.595829010009766,
      "activations/layer10_attention_weight_max": 33.790802001953125,
      "activations/layer10_attention_weight_min": -32.97175216674805,
      "activations/layer11_attention_weight_max": 31.50790786743164,
      "activations/layer11_attention_weight_min": -30.08623504638672,
      "activations/layer12_attention_weight_max": 21.961971282958984,
      "activations/layer12_attention_weight_min": -24.71487808227539,
      "activations/layer13_attention_weight_max": 38.83526611328125,
      "activations/layer13_attention_weight_min": -31.994197845458984,
      "activations/layer14_attention_weight_max": 44.78994369506836,
      "activations/layer14_attention_weight_min": -37.41122055053711,
      "activations/layer15_attention_weight_max": 33.64052200317383,
      "activations/layer15_attention_weight_min": -30.8170166015625,
      "activations/layer16_attention_weight_max": 27.949615478515625,
      "activations/layer16_attention_weight_min": -28.75055503845215,
      "activations/layer17_attention_weight_max": 30.03192901611328,
      "activations/layer17_attention_weight_min": -25.0234375,
      "activations/layer18_attention_weight_max": 31.602312088012695,
      "activations/layer18_attention_weight_min": -21.58672523498535,
      "activations/layer19_attention_weight_max": 33.502681732177734,
      "activations/layer19_attention_weight_min": -30.290740966796875,
      "activations/layer1_attention_weight_max": 16.7262020111084,
      "activations/layer1_attention_weight_min": -15.857309341430664,
      "activations/layer20_attention_weight_max": 27.667949676513672,
      "activations/layer20_attention_weight_min": -23.962011337280273,
      "activations/layer21_attention_weight_max": 31.488609313964844,
      "activations/layer21_attention_weight_min": -21.076818466186523,
      "activations/layer22_attention_weight_max": 42.253448486328125,
      "activations/layer22_attention_weight_min": -28.47153091430664,
      "activations/layer23_attention_weight_max": 34.180362701416016,
      "activations/layer23_attention_weight_min": -23.51711654663086,
      "activations/layer2_attention_weight_max": 30.80023193359375,
      "activations/layer2_attention_weight_min": -29.465435028076172,
      "activations/layer3_attention_weight_max": 93.3277359008789,
      "activations/layer3_attention_weight_min": -92.15646362304688,
      "activations/layer4_attention_weight_max": 116.8922348022461,
      "activations/layer4_attention_weight_min": -114.77876281738281,
      "activations/layer5_attention_weight_max": 55.67759323120117,
      "activations/layer5_attention_weight_min": -63.06412887573242,
      "activations/layer6_attention_weight_max": 49.03968048095703,
      "activations/layer6_attention_weight_min": -51.25312042236328,
      "activations/layer7_attention_weight_max": 93.61367797851562,
      "activations/layer7_attention_weight_min": -91.88907623291016,
      "activations/layer8_attention_weight_max": 43.69851303100586,
      "activations/layer8_attention_weight_min": -42.557464599609375,
      "activations/layer9_attention_weight_max": 37.6292724609375,
      "activations/layer9_attention_weight_min": -34.31730270385742,
      "epoch": 9.56,
      "learning_rate": 8.927803030303029e-05,
      "loss": 2.8,
      "step": 164450
    },
    {
      "activations/layer0_attention_weight_max": 15.218822479248047,
      "activations/layer0_attention_weight_min": -12.534214973449707,
      "activations/layer10_attention_weight_max": 32.97797775268555,
      "activations/layer10_attention_weight_min": -32.879364013671875,
      "activations/layer11_attention_weight_max": 33.79927062988281,
      "activations/layer11_attention_weight_min": -31.059249877929688,
      "activations/layer12_attention_weight_max": 23.85335922241211,
      "activations/layer12_attention_weight_min": -26.198583602905273,
      "activations/layer13_attention_weight_max": 39.23024368286133,
      "activations/layer13_attention_weight_min": -34.47132873535156,
      "activations/layer14_attention_weight_max": 47.65256118774414,
      "activations/layer14_attention_weight_min": -37.67772674560547,
      "activations/layer15_attention_weight_max": 39.1503791809082,
      "activations/layer15_attention_weight_min": -30.806886672973633,
      "activations/layer16_attention_weight_max": 32.06262969970703,
      "activations/layer16_attention_weight_min": -27.632144927978516,
      "activations/layer17_attention_weight_max": 33.251277923583984,
      "activations/layer17_attention_weight_min": -26.6806583404541,
      "activations/layer18_attention_weight_max": 29.398971557617188,
      "activations/layer18_attention_weight_min": -24.852140426635742,
      "activations/layer19_attention_weight_max": 39.86765670776367,
      "activations/layer19_attention_weight_min": -31.33734130859375,
      "activations/layer1_attention_weight_max": 15.998249053955078,
      "activations/layer1_attention_weight_min": -13.35766887664795,
      "activations/layer20_attention_weight_max": 29.49517059326172,
      "activations/layer20_attention_weight_min": -24.97414779663086,
      "activations/layer21_attention_weight_max": 30.18038558959961,
      "activations/layer21_attention_weight_min": -21.9527645111084,
      "activations/layer22_attention_weight_max": 41.66820526123047,
      "activations/layer22_attention_weight_min": -26.661212921142578,
      "activations/layer23_attention_weight_max": 35.48775100708008,
      "activations/layer23_attention_weight_min": -23.451406478881836,
      "activations/layer2_attention_weight_max": 30.607086181640625,
      "activations/layer2_attention_weight_min": -30.50337028503418,
      "activations/layer3_attention_weight_max": 90.37458038330078,
      "activations/layer3_attention_weight_min": -91.43013000488281,
      "activations/layer4_attention_weight_max": 111.10393524169922,
      "activations/layer4_attention_weight_min": -114.8298110961914,
      "activations/layer5_attention_weight_max": 52.543235778808594,
      "activations/layer5_attention_weight_min": -61.16577911376953,
      "activations/layer6_attention_weight_max": 45.283897399902344,
      "activations/layer6_attention_weight_min": -51.58534622192383,
      "activations/layer7_attention_weight_max": 88.90633392333984,
      "activations/layer7_attention_weight_min": -92.00738525390625,
      "activations/layer8_attention_weight_max": 39.703365325927734,
      "activations/layer8_attention_weight_min": -40.584693908691406,
      "activations/layer9_attention_weight_max": 35.90800094604492,
      "activations/layer9_attention_weight_min": -33.78676986694336,
      "epoch": 9.56,
      "learning_rate": 8.92590909090909e-05,
      "loss": 2.7846,
      "step": 164500
    },
    {
      "activations/layer0_attention_weight_max": 16.56403160095215,
      "activations/layer0_attention_weight_min": -12.982731819152832,
      "activations/layer10_attention_weight_max": 33.3071174621582,
      "activations/layer10_attention_weight_min": -33.261329650878906,
      "activations/layer11_attention_weight_max": 34.84031677246094,
      "activations/layer11_attention_weight_min": -32.935081481933594,
      "activations/layer12_attention_weight_max": 33.77299880981445,
      "activations/layer12_attention_weight_min": -26.308547973632812,
      "activations/layer13_attention_weight_max": 39.70185852050781,
      "activations/layer13_attention_weight_min": -34.66099548339844,
      "activations/layer14_attention_weight_max": 46.71240997314453,
      "activations/layer14_attention_weight_min": -41.052879333496094,
      "activations/layer15_attention_weight_max": 38.946754455566406,
      "activations/layer15_attention_weight_min": -34.738685607910156,
      "activations/layer16_attention_weight_max": 33.93027114868164,
      "activations/layer16_attention_weight_min": -29.374406814575195,
      "activations/layer17_attention_weight_max": 37.22049331665039,
      "activations/layer17_attention_weight_min": -26.459474563598633,
      "activations/layer18_attention_weight_max": 33.629425048828125,
      "activations/layer18_attention_weight_min": -21.974328994750977,
      "activations/layer19_attention_weight_max": 38.545196533203125,
      "activations/layer19_attention_weight_min": -31.274824142456055,
      "activations/layer1_attention_weight_max": 16.861303329467773,
      "activations/layer1_attention_weight_min": -13.984362602233887,
      "activations/layer20_attention_weight_max": 30.64073944091797,
      "activations/layer20_attention_weight_min": -26.355073928833008,
      "activations/layer21_attention_weight_max": 30.001066207885742,
      "activations/layer21_attention_weight_min": -23.47825050354004,
      "activations/layer22_attention_weight_max": 44.30060577392578,
      "activations/layer22_attention_weight_min": -30.21738624572754,
      "activations/layer23_attention_weight_max": 33.62236785888672,
      "activations/layer23_attention_weight_min": -25.263700485229492,
      "activations/layer2_attention_weight_max": 31.3721981048584,
      "activations/layer2_attention_weight_min": -30.90664291381836,
      "activations/layer3_attention_weight_max": 96.68860626220703,
      "activations/layer3_attention_weight_min": -91.28512573242188,
      "activations/layer4_attention_weight_max": 122.2350845336914,
      "activations/layer4_attention_weight_min": -112.95074462890625,
      "activations/layer5_attention_weight_max": 55.17668914794922,
      "activations/layer5_attention_weight_min": -61.882301330566406,
      "activations/layer6_attention_weight_max": 47.52089309692383,
      "activations/layer6_attention_weight_min": -51.512962341308594,
      "activations/layer7_attention_weight_max": 88.41340637207031,
      "activations/layer7_attention_weight_min": -85.32456970214844,
      "activations/layer8_attention_weight_max": 42.76095962524414,
      "activations/layer8_attention_weight_min": -41.07393264770508,
      "activations/layer9_attention_weight_max": 35.150142669677734,
      "activations/layer9_attention_weight_min": -32.723697662353516,
      "epoch": 9.56,
      "learning_rate": 8.924015151515152e-05,
      "loss": 2.7912,
      "step": 164550
    },
    {
      "activations/layer0_attention_weight_max": 17.317970275878906,
      "activations/layer0_attention_weight_min": -11.73575496673584,
      "activations/layer10_attention_weight_max": 31.802507400512695,
      "activations/layer10_attention_weight_min": -33.60193634033203,
      "activations/layer11_attention_weight_max": 33.06235885620117,
      "activations/layer11_attention_weight_min": -31.052152633666992,
      "activations/layer12_attention_weight_max": 23.25576400756836,
      "activations/layer12_attention_weight_min": -39.92198944091797,
      "activations/layer13_attention_weight_max": 39.09593963623047,
      "activations/layer13_attention_weight_min": -32.80239486694336,
      "activations/layer14_attention_weight_max": 52.068817138671875,
      "activations/layer14_attention_weight_min": -37.77232360839844,
      "activations/layer15_attention_weight_max": 45.93907165527344,
      "activations/layer15_attention_weight_min": -31.038516998291016,
      "activations/layer16_attention_weight_max": 32.14375305175781,
      "activations/layer16_attention_weight_min": -26.257461547851562,
      "activations/layer17_attention_weight_max": 38.32736587524414,
      "activations/layer17_attention_weight_min": -24.220191955566406,
      "activations/layer18_attention_weight_max": 33.27146530151367,
      "activations/layer18_attention_weight_min": -22.928422927856445,
      "activations/layer19_attention_weight_max": 39.706119537353516,
      "activations/layer19_attention_weight_min": -30.75274085998535,
      "activations/layer1_attention_weight_max": 16.405736923217773,
      "activations/layer1_attention_weight_min": -14.011518478393555,
      "activations/layer20_attention_weight_max": 36.26068115234375,
      "activations/layer20_attention_weight_min": -25.226573944091797,
      "activations/layer21_attention_weight_max": 36.43096923828125,
      "activations/layer21_attention_weight_min": -24.3887882232666,
      "activations/layer22_attention_weight_max": 48.8945198059082,
      "activations/layer22_attention_weight_min": -29.574588775634766,
      "activations/layer23_attention_weight_max": 36.65792465209961,
      "activations/layer23_attention_weight_min": -24.69525718688965,
      "activations/layer2_attention_weight_max": 31.7402400970459,
      "activations/layer2_attention_weight_min": -31.59158706665039,
      "activations/layer3_attention_weight_max": 93.62592315673828,
      "activations/layer3_attention_weight_min": -93.97323608398438,
      "activations/layer4_attention_weight_max": 118.5315170288086,
      "activations/layer4_attention_weight_min": -110.8159408569336,
      "activations/layer5_attention_weight_max": 56.805442810058594,
      "activations/layer5_attention_weight_min": -68.5901107788086,
      "activations/layer6_attention_weight_max": 47.907222747802734,
      "activations/layer6_attention_weight_min": -48.94422912597656,
      "activations/layer7_attention_weight_max": 97.74592590332031,
      "activations/layer7_attention_weight_min": -89.93153381347656,
      "activations/layer8_attention_weight_max": 42.495849609375,
      "activations/layer8_attention_weight_min": -41.31244659423828,
      "activations/layer9_attention_weight_max": 32.960350036621094,
      "activations/layer9_attention_weight_min": -41.09485626220703,
      "epoch": 9.56,
      "learning_rate": 8.922121212121211e-05,
      "loss": 2.7872,
      "step": 164600
    },
    {
      "activations/layer0_attention_weight_max": 15.544720649719238,
      "activations/layer0_attention_weight_min": -12.451674461364746,
      "activations/layer10_attention_weight_max": 30.79934310913086,
      "activations/layer10_attention_weight_min": -30.836143493652344,
      "activations/layer11_attention_weight_max": 29.960386276245117,
      "activations/layer11_attention_weight_min": -29.21967887878418,
      "activations/layer12_attention_weight_max": 23.140621185302734,
      "activations/layer12_attention_weight_min": -27.0062313079834,
      "activations/layer13_attention_weight_max": 36.164669036865234,
      "activations/layer13_attention_weight_min": -33.77082061767578,
      "activations/layer14_attention_weight_max": 39.742977142333984,
      "activations/layer14_attention_weight_min": -38.14569854736328,
      "activations/layer15_attention_weight_max": 32.164306640625,
      "activations/layer15_attention_weight_min": -29.660242080688477,
      "activations/layer16_attention_weight_max": 27.25133514404297,
      "activations/layer16_attention_weight_min": -28.235124588012695,
      "activations/layer17_attention_weight_max": 29.50135612487793,
      "activations/layer17_attention_weight_min": -24.86209487915039,
      "activations/layer18_attention_weight_max": 32.32770538330078,
      "activations/layer18_attention_weight_min": -24.21857261657715,
      "activations/layer19_attention_weight_max": 29.723026275634766,
      "activations/layer19_attention_weight_min": -29.36508560180664,
      "activations/layer1_attention_weight_max": 16.490854263305664,
      "activations/layer1_attention_weight_min": -15.99062442779541,
      "activations/layer20_attention_weight_max": 30.136056900024414,
      "activations/layer20_attention_weight_min": -23.360782623291016,
      "activations/layer21_attention_weight_max": 26.968570709228516,
      "activations/layer21_attention_weight_min": -23.67382049560547,
      "activations/layer22_attention_weight_max": 45.427433013916016,
      "activations/layer22_attention_weight_min": -27.554597854614258,
      "activations/layer23_attention_weight_max": 31.418338775634766,
      "activations/layer23_attention_weight_min": -24.342153549194336,
      "activations/layer2_attention_weight_max": 30.650836944580078,
      "activations/layer2_attention_weight_min": -28.785091400146484,
      "activations/layer3_attention_weight_max": 84.59383392333984,
      "activations/layer3_attention_weight_min": -86.8401870727539,
      "activations/layer4_attention_weight_max": 111.38134002685547,
      "activations/layer4_attention_weight_min": -111.57814025878906,
      "activations/layer5_attention_weight_max": 52.697845458984375,
      "activations/layer5_attention_weight_min": -59.011348724365234,
      "activations/layer6_attention_weight_max": 44.686851501464844,
      "activations/layer6_attention_weight_min": -47.09223937988281,
      "activations/layer7_attention_weight_max": 84.68486022949219,
      "activations/layer7_attention_weight_min": -86.90959167480469,
      "activations/layer8_attention_weight_max": 39.135738372802734,
      "activations/layer8_attention_weight_min": -39.75044250488281,
      "activations/layer9_attention_weight_max": 33.03369903564453,
      "activations/layer9_attention_weight_min": -32.48717498779297,
      "epoch": 9.57,
      "learning_rate": 8.920227272727272e-05,
      "loss": 2.7871,
      "step": 164650
    },
    {
      "activations/layer0_attention_weight_max": 17.21635627746582,
      "activations/layer0_attention_weight_min": -11.15235710144043,
      "activations/layer10_attention_weight_max": 31.791479110717773,
      "activations/layer10_attention_weight_min": -32.364097595214844,
      "activations/layer11_attention_weight_max": 29.84204864501953,
      "activations/layer11_attention_weight_min": -31.362476348876953,
      "activations/layer12_attention_weight_max": 22.656198501586914,
      "activations/layer12_attention_weight_min": -25.079273223876953,
      "activations/layer13_attention_weight_max": 36.50250244140625,
      "activations/layer13_attention_weight_min": -33.720035552978516,
      "activations/layer14_attention_weight_max": 39.56126022338867,
      "activations/layer14_attention_weight_min": -38.70173263549805,
      "activations/layer15_attention_weight_max": 34.21786880493164,
      "activations/layer15_attention_weight_min": -30.262500762939453,
      "activations/layer16_attention_weight_max": 30.156587600708008,
      "activations/layer16_attention_weight_min": -27.81508445739746,
      "activations/layer17_attention_weight_max": 32.42033004760742,
      "activations/layer17_attention_weight_min": -24.86228370666504,
      "activations/layer18_attention_weight_max": 27.588167190551758,
      "activations/layer18_attention_weight_min": -24.74169921875,
      "activations/layer19_attention_weight_max": 33.81555938720703,
      "activations/layer19_attention_weight_min": -31.24127769470215,
      "activations/layer1_attention_weight_max": 16.886844635009766,
      "activations/layer1_attention_weight_min": -13.484702110290527,
      "activations/layer20_attention_weight_max": 27.906890869140625,
      "activations/layer20_attention_weight_min": -24.22185707092285,
      "activations/layer21_attention_weight_max": 26.497020721435547,
      "activations/layer21_attention_weight_min": -25.997833251953125,
      "activations/layer22_attention_weight_max": 41.07493209838867,
      "activations/layer22_attention_weight_min": -29.977319717407227,
      "activations/layer23_attention_weight_max": 31.99538803100586,
      "activations/layer23_attention_weight_min": -25.0705623626709,
      "activations/layer2_attention_weight_max": 30.219635009765625,
      "activations/layer2_attention_weight_min": -29.336322784423828,
      "activations/layer3_attention_weight_max": 87.91453552246094,
      "activations/layer3_attention_weight_min": -86.93617248535156,
      "activations/layer4_attention_weight_max": 109.57745361328125,
      "activations/layer4_attention_weight_min": -113.5312271118164,
      "activations/layer5_attention_weight_max": 53.33531951904297,
      "activations/layer5_attention_weight_min": -63.00472640991211,
      "activations/layer6_attention_weight_max": 44.72931671142578,
      "activations/layer6_attention_weight_min": -48.9454460144043,
      "activations/layer7_attention_weight_max": 88.34146118164062,
      "activations/layer7_attention_weight_min": -87.65495300292969,
      "activations/layer8_attention_weight_max": 39.111270904541016,
      "activations/layer8_attention_weight_min": -38.203128814697266,
      "activations/layer9_attention_weight_max": 32.46107864379883,
      "activations/layer9_attention_weight_min": -31.57472038269043,
      "epoch": 9.57,
      "learning_rate": 8.918333333333332e-05,
      "loss": 2.7906,
      "step": 164700
    },
    {
      "activations/layer0_attention_weight_max": 15.667736053466797,
      "activations/layer0_attention_weight_min": -11.776598930358887,
      "activations/layer10_attention_weight_max": 34.86670684814453,
      "activations/layer10_attention_weight_min": -31.501861572265625,
      "activations/layer11_attention_weight_max": 34.353721618652344,
      "activations/layer11_attention_weight_min": -31.05337905883789,
      "activations/layer12_attention_weight_max": 24.028757095336914,
      "activations/layer12_attention_weight_min": -26.193317413330078,
      "activations/layer13_attention_weight_max": 40.3945426940918,
      "activations/layer13_attention_weight_min": -32.10759735107422,
      "activations/layer14_attention_weight_max": 44.91828536987305,
      "activations/layer14_attention_weight_min": -37.30724334716797,
      "activations/layer15_attention_weight_max": 35.23865509033203,
      "activations/layer15_attention_weight_min": -30.6680908203125,
      "activations/layer16_attention_weight_max": 30.951038360595703,
      "activations/layer16_attention_weight_min": -27.289499282836914,
      "activations/layer17_attention_weight_max": 32.357757568359375,
      "activations/layer17_attention_weight_min": -24.28135108947754,
      "activations/layer18_attention_weight_max": 30.88947105407715,
      "activations/layer18_attention_weight_min": -21.385950088500977,
      "activations/layer19_attention_weight_max": 33.03959274291992,
      "activations/layer19_attention_weight_min": -30.465118408203125,
      "activations/layer1_attention_weight_max": 16.36054039001465,
      "activations/layer1_attention_weight_min": -13.371776580810547,
      "activations/layer20_attention_weight_max": 29.417137145996094,
      "activations/layer20_attention_weight_min": -24.55531883239746,
      "activations/layer21_attention_weight_max": 30.288330078125,
      "activations/layer21_attention_weight_min": -24.6125431060791,
      "activations/layer22_attention_weight_max": 44.165321350097656,
      "activations/layer22_attention_weight_min": -28.59807777404785,
      "activations/layer23_attention_weight_max": 32.7618408203125,
      "activations/layer23_attention_weight_min": -23.996734619140625,
      "activations/layer2_attention_weight_max": 30.36798858642578,
      "activations/layer2_attention_weight_min": -29.821247100830078,
      "activations/layer3_attention_weight_max": 88.18670654296875,
      "activations/layer3_attention_weight_min": -85.95848083496094,
      "activations/layer4_attention_weight_max": 108.72676849365234,
      "activations/layer4_attention_weight_min": -105.67545318603516,
      "activations/layer5_attention_weight_max": 51.5108757019043,
      "activations/layer5_attention_weight_min": -60.092018127441406,
      "activations/layer6_attention_weight_max": 45.3797721862793,
      "activations/layer6_attention_weight_min": -49.3514289855957,
      "activations/layer7_attention_weight_max": 84.74853515625,
      "activations/layer7_attention_weight_min": -84.56179809570312,
      "activations/layer8_attention_weight_max": 40.681785583496094,
      "activations/layer8_attention_weight_min": -40.16263961791992,
      "activations/layer9_attention_weight_max": 35.76990509033203,
      "activations/layer9_attention_weight_min": -34.330780029296875,
      "epoch": 9.57,
      "learning_rate": 8.916477272727273e-05,
      "loss": 2.8023,
      "step": 164750
    },
    {
      "activations/layer0_attention_weight_max": 15.879023551940918,
      "activations/layer0_attention_weight_min": -12.11575698852539,
      "activations/layer10_attention_weight_max": 33.097049713134766,
      "activations/layer10_attention_weight_min": -33.6002197265625,
      "activations/layer11_attention_weight_max": 34.184329986572266,
      "activations/layer11_attention_weight_min": -33.00300598144531,
      "activations/layer12_attention_weight_max": 25.58799934387207,
      "activations/layer12_attention_weight_min": -26.289579391479492,
      "activations/layer13_attention_weight_max": 39.065425872802734,
      "activations/layer13_attention_weight_min": -35.74229049682617,
      "activations/layer14_attention_weight_max": 50.8065071105957,
      "activations/layer14_attention_weight_min": -37.926918029785156,
      "activations/layer15_attention_weight_max": 38.18342590332031,
      "activations/layer15_attention_weight_min": -34.06647491455078,
      "activations/layer16_attention_weight_max": 33.334800720214844,
      "activations/layer16_attention_weight_min": -29.353130340576172,
      "activations/layer17_attention_weight_max": 35.80972671508789,
      "activations/layer17_attention_weight_min": -27.92814064025879,
      "activations/layer18_attention_weight_max": 30.876136779785156,
      "activations/layer18_attention_weight_min": -22.572153091430664,
      "activations/layer19_attention_weight_max": 39.2121696472168,
      "activations/layer19_attention_weight_min": -30.97031021118164,
      "activations/layer1_attention_weight_max": 17.277116775512695,
      "activations/layer1_attention_weight_min": -14.076704025268555,
      "activations/layer20_attention_weight_max": 31.86052703857422,
      "activations/layer20_attention_weight_min": -23.31614875793457,
      "activations/layer21_attention_weight_max": 30.64067840576172,
      "activations/layer21_attention_weight_min": -23.438024520874023,
      "activations/layer22_attention_weight_max": 53.528602600097656,
      "activations/layer22_attention_weight_min": -29.600238800048828,
      "activations/layer23_attention_weight_max": 41.704673767089844,
      "activations/layer23_attention_weight_min": -24.54098892211914,
      "activations/layer2_attention_weight_max": 31.148590087890625,
      "activations/layer2_attention_weight_min": -29.815908432006836,
      "activations/layer3_attention_weight_max": 92.88034057617188,
      "activations/layer3_attention_weight_min": -90.93569946289062,
      "activations/layer4_attention_weight_max": 115.19085693359375,
      "activations/layer4_attention_weight_min": -117.01014709472656,
      "activations/layer5_attention_weight_max": 57.601966857910156,
      "activations/layer5_attention_weight_min": -63.972503662109375,
      "activations/layer6_attention_weight_max": 46.25715637207031,
      "activations/layer6_attention_weight_min": -51.32857131958008,
      "activations/layer7_attention_weight_max": 86.454833984375,
      "activations/layer7_attention_weight_min": -90.26487731933594,
      "activations/layer8_attention_weight_max": 39.206111907958984,
      "activations/layer8_attention_weight_min": -43.47650909423828,
      "activations/layer9_attention_weight_max": 35.32549285888672,
      "activations/layer9_attention_weight_min": -35.050540924072266,
      "epoch": 9.58,
      "learning_rate": 8.914583333333332e-05,
      "loss": 2.7815,
      "step": 164800
    },
    {
      "activations/layer0_attention_weight_max": 15.847013473510742,
      "activations/layer0_attention_weight_min": -12.501579284667969,
      "activations/layer10_attention_weight_max": 35.42151641845703,
      "activations/layer10_attention_weight_min": -30.946012496948242,
      "activations/layer11_attention_weight_max": 36.605125427246094,
      "activations/layer11_attention_weight_min": -31.73426055908203,
      "activations/layer12_attention_weight_max": 26.855026245117188,
      "activations/layer12_attention_weight_min": -24.84493637084961,
      "activations/layer13_attention_weight_max": 37.94337844848633,
      "activations/layer13_attention_weight_min": -33.84812927246094,
      "activations/layer14_attention_weight_max": 44.2872428894043,
      "activations/layer14_attention_weight_min": -38.15762710571289,
      "activations/layer15_attention_weight_max": 34.37835693359375,
      "activations/layer15_attention_weight_min": -32.191959381103516,
      "activations/layer16_attention_weight_max": 32.56499481201172,
      "activations/layer16_attention_weight_min": -26.96337127685547,
      "activations/layer17_attention_weight_max": 32.13319778442383,
      "activations/layer17_attention_weight_min": -25.42449378967285,
      "activations/layer18_attention_weight_max": 28.48731803894043,
      "activations/layer18_attention_weight_min": -22.891883850097656,
      "activations/layer19_attention_weight_max": 34.085975646972656,
      "activations/layer19_attention_weight_min": -30.366809844970703,
      "activations/layer1_attention_weight_max": 16.18025016784668,
      "activations/layer1_attention_weight_min": -15.365828514099121,
      "activations/layer20_attention_weight_max": 28.875919342041016,
      "activations/layer20_attention_weight_min": -24.10896110534668,
      "activations/layer21_attention_weight_max": 29.353160858154297,
      "activations/layer21_attention_weight_min": -21.919836044311523,
      "activations/layer22_attention_weight_max": 42.6401481628418,
      "activations/layer22_attention_weight_min": -27.24480628967285,
      "activations/layer23_attention_weight_max": 35.646785736083984,
      "activations/layer23_attention_weight_min": -22.74532127380371,
      "activations/layer2_attention_weight_max": 32.66797637939453,
      "activations/layer2_attention_weight_min": -31.79065704345703,
      "activations/layer3_attention_weight_max": 94.21690368652344,
      "activations/layer3_attention_weight_min": -93.81198120117188,
      "activations/layer4_attention_weight_max": 122.92955780029297,
      "activations/layer4_attention_weight_min": -118.01274871826172,
      "activations/layer5_attention_weight_max": 56.376426696777344,
      "activations/layer5_attention_weight_min": -58.57636642456055,
      "activations/layer6_attention_weight_max": 48.37796401977539,
      "activations/layer6_attention_weight_min": -49.244503021240234,
      "activations/layer7_attention_weight_max": 96.73506927490234,
      "activations/layer7_attention_weight_min": -89.91490173339844,
      "activations/layer8_attention_weight_max": 41.323509216308594,
      "activations/layer8_attention_weight_min": -41.347137451171875,
      "activations/layer9_attention_weight_max": 33.93465042114258,
      "activations/layer9_attention_weight_min": -32.78795623779297,
      "epoch": 9.58,
      "learning_rate": 8.912689393939393e-05,
      "loss": 2.7742,
      "step": 164850
    },
    {
      "activations/layer0_attention_weight_max": 15.875747680664062,
      "activations/layer0_attention_weight_min": -11.451803207397461,
      "activations/layer10_attention_weight_max": 33.25508499145508,
      "activations/layer10_attention_weight_min": -33.33696746826172,
      "activations/layer11_attention_weight_max": 35.87671661376953,
      "activations/layer11_attention_weight_min": -35.05189514160156,
      "activations/layer12_attention_weight_max": 24.131805419921875,
      "activations/layer12_attention_weight_min": -24.567901611328125,
      "activations/layer13_attention_weight_max": 38.68727493286133,
      "activations/layer13_attention_weight_min": -32.145931243896484,
      "activations/layer14_attention_weight_max": 48.294769287109375,
      "activations/layer14_attention_weight_min": -36.78718185424805,
      "activations/layer15_attention_weight_max": 33.97063064575195,
      "activations/layer15_attention_weight_min": -29.59355926513672,
      "activations/layer16_attention_weight_max": 29.398942947387695,
      "activations/layer16_attention_weight_min": -29.345365524291992,
      "activations/layer17_attention_weight_max": 31.29435920715332,
      "activations/layer17_attention_weight_min": -25.043928146362305,
      "activations/layer18_attention_weight_max": 38.62227249145508,
      "activations/layer18_attention_weight_min": -21.47958755493164,
      "activations/layer19_attention_weight_max": 36.47330856323242,
      "activations/layer19_attention_weight_min": -30.153202056884766,
      "activations/layer1_attention_weight_max": 17.848539352416992,
      "activations/layer1_attention_weight_min": -16.10637855529785,
      "activations/layer20_attention_weight_max": 32.481502532958984,
      "activations/layer20_attention_weight_min": -26.039827346801758,
      "activations/layer21_attention_weight_max": 35.73335647583008,
      "activations/layer21_attention_weight_min": -24.018503189086914,
      "activations/layer22_attention_weight_max": 48.60845184326172,
      "activations/layer22_attention_weight_min": -27.958454132080078,
      "activations/layer23_attention_weight_max": 35.714717864990234,
      "activations/layer23_attention_weight_min": -23.45321273803711,
      "activations/layer2_attention_weight_max": 30.306821823120117,
      "activations/layer2_attention_weight_min": -29.92577362060547,
      "activations/layer3_attention_weight_max": 89.50480651855469,
      "activations/layer3_attention_weight_min": -87.3891372680664,
      "activations/layer4_attention_weight_max": 110.82391357421875,
      "activations/layer4_attention_weight_min": -109.66121673583984,
      "activations/layer5_attention_weight_max": 55.64064025878906,
      "activations/layer5_attention_weight_min": -57.09745407104492,
      "activations/layer6_attention_weight_max": 47.38249588012695,
      "activations/layer6_attention_weight_min": -48.28817367553711,
      "activations/layer7_attention_weight_max": 89.26644897460938,
      "activations/layer7_attention_weight_min": -87.7258529663086,
      "activations/layer8_attention_weight_max": 40.31575393676758,
      "activations/layer8_attention_weight_min": -40.769832611083984,
      "activations/layer9_attention_weight_max": 34.5645866394043,
      "activations/layer9_attention_weight_min": -32.33079147338867,
      "epoch": 9.58,
      "learning_rate": 8.910795454545453e-05,
      "loss": 2.7921,
      "step": 164900
    },
    {
      "activations/layer0_attention_weight_max": 15.422065734863281,
      "activations/layer0_attention_weight_min": -12.251433372497559,
      "activations/layer10_attention_weight_max": 35.25666046142578,
      "activations/layer10_attention_weight_min": -33.98212432861328,
      "activations/layer11_attention_weight_max": 37.21333694458008,
      "activations/layer11_attention_weight_min": -33.151817321777344,
      "activations/layer12_attention_weight_max": 24.78744125366211,
      "activations/layer12_attention_weight_min": -28.831205368041992,
      "activations/layer13_attention_weight_max": 38.46379470825195,
      "activations/layer13_attention_weight_min": -34.72239303588867,
      "activations/layer14_attention_weight_max": 43.09696960449219,
      "activations/layer14_attention_weight_min": -37.527915954589844,
      "activations/layer15_attention_weight_max": 34.9345817565918,
      "activations/layer15_attention_weight_min": -33.38630676269531,
      "activations/layer16_attention_weight_max": 31.00028419494629,
      "activations/layer16_attention_weight_min": -28.369117736816406,
      "activations/layer17_attention_weight_max": 32.50679016113281,
      "activations/layer17_attention_weight_min": -25.64735984802246,
      "activations/layer18_attention_weight_max": 27.72566032409668,
      "activations/layer18_attention_weight_min": -22.247539520263672,
      "activations/layer19_attention_weight_max": 35.61100769042969,
      "activations/layer19_attention_weight_min": -29.662260055541992,
      "activations/layer1_attention_weight_max": 16.583316802978516,
      "activations/layer1_attention_weight_min": -16.14068603515625,
      "activations/layer20_attention_weight_max": 32.0269660949707,
      "activations/layer20_attention_weight_min": -24.44378662109375,
      "activations/layer21_attention_weight_max": 30.396738052368164,
      "activations/layer21_attention_weight_min": -23.207847595214844,
      "activations/layer22_attention_weight_max": 41.91226577758789,
      "activations/layer22_attention_weight_min": -29.709150314331055,
      "activations/layer23_attention_weight_max": 32.68840026855469,
      "activations/layer23_attention_weight_min": -22.539358139038086,
      "activations/layer2_attention_weight_max": 30.390567779541016,
      "activations/layer2_attention_weight_min": -30.66470718383789,
      "activations/layer3_attention_weight_max": 87.0177001953125,
      "activations/layer3_attention_weight_min": -91.04348754882812,
      "activations/layer4_attention_weight_max": 108.68074035644531,
      "activations/layer4_attention_weight_min": -107.8070297241211,
      "activations/layer5_attention_weight_max": 52.292171478271484,
      "activations/layer5_attention_weight_min": -61.52770233154297,
      "activations/layer6_attention_weight_max": 45.32927703857422,
      "activations/layer6_attention_weight_min": -53.15890121459961,
      "activations/layer7_attention_weight_max": 90.37020111083984,
      "activations/layer7_attention_weight_min": -91.16380310058594,
      "activations/layer8_attention_weight_max": 41.92014694213867,
      "activations/layer8_attention_weight_min": -42.113277435302734,
      "activations/layer9_attention_weight_max": 38.31901931762695,
      "activations/layer9_attention_weight_min": -35.1450309753418,
      "epoch": 9.58,
      "learning_rate": 8.908901515151515e-05,
      "loss": 2.7697,
      "step": 164950
    },
    {
      "activations/layer0_attention_weight_max": 15.908823013305664,
      "activations/layer0_attention_weight_min": -12.642372131347656,
      "activations/layer10_attention_weight_max": 34.61381530761719,
      "activations/layer10_attention_weight_min": -32.862117767333984,
      "activations/layer11_attention_weight_max": 33.7332878112793,
      "activations/layer11_attention_weight_min": -30.890357971191406,
      "activations/layer12_attention_weight_max": 23.785255432128906,
      "activations/layer12_attention_weight_min": -23.762998580932617,
      "activations/layer13_attention_weight_max": 38.613807678222656,
      "activations/layer13_attention_weight_min": -35.03279495239258,
      "activations/layer14_attention_weight_max": 49.29267883300781,
      "activations/layer14_attention_weight_min": -37.68830871582031,
      "activations/layer15_attention_weight_max": 37.26811981201172,
      "activations/layer15_attention_weight_min": -31.05290985107422,
      "activations/layer16_attention_weight_max": 32.02126693725586,
      "activations/layer16_attention_weight_min": -27.20927619934082,
      "activations/layer17_attention_weight_max": 33.963470458984375,
      "activations/layer17_attention_weight_min": -25.306358337402344,
      "activations/layer18_attention_weight_max": 33.34991455078125,
      "activations/layer18_attention_weight_min": -21.880739212036133,
      "activations/layer19_attention_weight_max": 34.257659912109375,
      "activations/layer19_attention_weight_min": -28.894886016845703,
      "activations/layer1_attention_weight_max": 16.42768096923828,
      "activations/layer1_attention_weight_min": -13.178293228149414,
      "activations/layer20_attention_weight_max": 28.391427993774414,
      "activations/layer20_attention_weight_min": -25.30678367614746,
      "activations/layer21_attention_weight_max": 28.90511131286621,
      "activations/layer21_attention_weight_min": -23.111940383911133,
      "activations/layer22_attention_weight_max": 44.21913146972656,
      "activations/layer22_attention_weight_min": -29.74974822998047,
      "activations/layer23_attention_weight_max": 31.422107696533203,
      "activations/layer23_attention_weight_min": -23.31024742126465,
      "activations/layer2_attention_weight_max": 30.79837417602539,
      "activations/layer2_attention_weight_min": -30.372648239135742,
      "activations/layer3_attention_weight_max": 86.42730712890625,
      "activations/layer3_attention_weight_min": -85.9541015625,
      "activations/layer4_attention_weight_max": 111.0250473022461,
      "activations/layer4_attention_weight_min": -105.7138900756836,
      "activations/layer5_attention_weight_max": 53.02631759643555,
      "activations/layer5_attention_weight_min": -60.90782928466797,
      "activations/layer6_attention_weight_max": 45.195701599121094,
      "activations/layer6_attention_weight_min": -49.42290496826172,
      "activations/layer7_attention_weight_max": 85.70378112792969,
      "activations/layer7_attention_weight_min": -87.62930297851562,
      "activations/layer8_attention_weight_max": 44.79652786254883,
      "activations/layer8_attention_weight_min": -42.375205993652344,
      "activations/layer9_attention_weight_max": 37.61711502075195,
      "activations/layer9_attention_weight_min": -35.81563949584961,
      "epoch": 9.59,
      "learning_rate": 8.907007575757575e-05,
      "loss": 2.8006,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4454,
      "eval_samples_per_second": 508.445,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4454,
      "eval_openwebtext_samples_per_second": 508.445,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 1.9502,
      "eval_wikitext_samples_per_second": 233.82,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_lambada_loss": 2.5546875,
      "eval_lambada_ppl": 12.867278004795136,
      "eval_lambada_runtime": 9.5103,
      "eval_lambada_samples_per_second": 511.97,
      "step": 165000
    },
    {
      "activations/layer0_attention_weight_max": 15.468474388122559,
      "activations/layer0_attention_weight_min": -13.201238632202148,
      "activations/layer10_attention_weight_max": 34.08837127685547,
      "activations/layer10_attention_weight_min": -31.301937103271484,
      "activations/layer11_attention_weight_max": 32.474403381347656,
      "activations/layer11_attention_weight_min": -30.06593894958496,
      "activations/layer12_attention_weight_max": 25.469501495361328,
      "activations/layer12_attention_weight_min": -26.386598587036133,
      "activations/layer13_attention_weight_max": 41.36505126953125,
      "activations/layer13_attention_weight_min": -37.37832260131836,
      "activations/layer14_attention_weight_max": 48.769168853759766,
      "activations/layer14_attention_weight_min": -39.668846130371094,
      "activations/layer15_attention_weight_max": 37.97138977050781,
      "activations/layer15_attention_weight_min": -31.679975509643555,
      "activations/layer16_attention_weight_max": 33.82797622680664,
      "activations/layer16_attention_weight_min": -30.805908203125,
      "activations/layer17_attention_weight_max": 31.34254264831543,
      "activations/layer17_attention_weight_min": -26.751474380493164,
      "activations/layer18_attention_weight_max": 28.488117218017578,
      "activations/layer18_attention_weight_min": -21.913835525512695,
      "activations/layer19_attention_weight_max": 34.032955169677734,
      "activations/layer19_attention_weight_min": -30.173892974853516,
      "activations/layer1_attention_weight_max": 18.666744232177734,
      "activations/layer1_attention_weight_min": -14.910847663879395,
      "activations/layer20_attention_weight_max": 29.39751434326172,
      "activations/layer20_attention_weight_min": -25.304349899291992,
      "activations/layer21_attention_weight_max": 29.92820167541504,
      "activations/layer21_attention_weight_min": -23.383703231811523,
      "activations/layer22_attention_weight_max": 41.54111099243164,
      "activations/layer22_attention_weight_min": -28.51413345336914,
      "activations/layer23_attention_weight_max": 32.345252990722656,
      "activations/layer23_attention_weight_min": -23.8700008392334,
      "activations/layer2_attention_weight_max": 29.880229949951172,
      "activations/layer2_attention_weight_min": -28.63890838623047,
      "activations/layer3_attention_weight_max": 80.52238464355469,
      "activations/layer3_attention_weight_min": -83.60066986083984,
      "activations/layer4_attention_weight_max": 105.63934326171875,
      "activations/layer4_attention_weight_min": -105.99491119384766,
      "activations/layer5_attention_weight_max": 52.95766830444336,
      "activations/layer5_attention_weight_min": -60.253475189208984,
      "activations/layer6_attention_weight_max": 45.85405349731445,
      "activations/layer6_attention_weight_min": -48.35204315185547,
      "activations/layer7_attention_weight_max": 90.71797180175781,
      "activations/layer7_attention_weight_min": -84.58251953125,
      "activations/layer8_attention_weight_max": 41.5801887512207,
      "activations/layer8_attention_weight_min": -40.40372848510742,
      "activations/layer9_attention_weight_max": 35.76740264892578,
      "activations/layer9_attention_weight_min": -32.89951705932617,
      "epoch": 9.59,
      "learning_rate": 8.905113636363635e-05,
      "loss": 2.7818,
      "step": 165050
    },
    {
      "activations/layer0_attention_weight_max": 15.40174674987793,
      "activations/layer0_attention_weight_min": -12.05656909942627,
      "activations/layer10_attention_weight_max": 34.862823486328125,
      "activations/layer10_attention_weight_min": -34.00657653808594,
      "activations/layer11_attention_weight_max": 35.68843460083008,
      "activations/layer11_attention_weight_min": -31.797414779663086,
      "activations/layer12_attention_weight_max": 24.15966033935547,
      "activations/layer12_attention_weight_min": -25.301288604736328,
      "activations/layer13_attention_weight_max": 38.765846252441406,
      "activations/layer13_attention_weight_min": -34.103485107421875,
      "activations/layer14_attention_weight_max": 48.190887451171875,
      "activations/layer14_attention_weight_min": -40.03726577758789,
      "activations/layer15_attention_weight_max": 35.666717529296875,
      "activations/layer15_attention_weight_min": -32.22740936279297,
      "activations/layer16_attention_weight_max": 32.52519989013672,
      "activations/layer16_attention_weight_min": -29.66288948059082,
      "activations/layer17_attention_weight_max": 32.12958526611328,
      "activations/layer17_attention_weight_min": -25.140167236328125,
      "activations/layer18_attention_weight_max": 30.541790008544922,
      "activations/layer18_attention_weight_min": -23.64787483215332,
      "activations/layer19_attention_weight_max": 36.330142974853516,
      "activations/layer19_attention_weight_min": -29.966161727905273,
      "activations/layer1_attention_weight_max": 17.92237091064453,
      "activations/layer1_attention_weight_min": -14.764361381530762,
      "activations/layer20_attention_weight_max": 33.43296813964844,
      "activations/layer20_attention_weight_min": -25.765939712524414,
      "activations/layer21_attention_weight_max": 32.47332000732422,
      "activations/layer21_attention_weight_min": -24.66941261291504,
      "activations/layer22_attention_weight_max": 45.863059997558594,
      "activations/layer22_attention_weight_min": -29.686046600341797,
      "activations/layer23_attention_weight_max": 35.29352569580078,
      "activations/layer23_attention_weight_min": -23.80258560180664,
      "activations/layer2_attention_weight_max": 31.087833404541016,
      "activations/layer2_attention_weight_min": -30.862201690673828,
      "activations/layer3_attention_weight_max": 88.49557495117188,
      "activations/layer3_attention_weight_min": -93.66910552978516,
      "activations/layer4_attention_weight_max": 118.16082000732422,
      "activations/layer4_attention_weight_min": -110.96611785888672,
      "activations/layer5_attention_weight_max": 53.134246826171875,
      "activations/layer5_attention_weight_min": -64.69400024414062,
      "activations/layer6_attention_weight_max": 49.74009323120117,
      "activations/layer6_attention_weight_min": -48.422908782958984,
      "activations/layer7_attention_weight_max": 92.83369445800781,
      "activations/layer7_attention_weight_min": -91.26905822753906,
      "activations/layer8_attention_weight_max": 43.789581298828125,
      "activations/layer8_attention_weight_min": -41.141666412353516,
      "activations/layer9_attention_weight_max": 35.45177459716797,
      "activations/layer9_attention_weight_min": -35.26107406616211,
      "epoch": 9.59,
      "learning_rate": 8.903219696969697e-05,
      "loss": 2.7896,
      "step": 165100
    },
    {
      "activations/layer0_attention_weight_max": 15.528759002685547,
      "activations/layer0_attention_weight_min": -12.501906394958496,
      "activations/layer10_attention_weight_max": 31.254165649414062,
      "activations/layer10_attention_weight_min": -32.13640213012695,
      "activations/layer11_attention_weight_max": 30.326793670654297,
      "activations/layer11_attention_weight_min": -29.496625900268555,
      "activations/layer12_attention_weight_max": 24.849903106689453,
      "activations/layer12_attention_weight_min": -36.89591598510742,
      "activations/layer13_attention_weight_max": 39.79397201538086,
      "activations/layer13_attention_weight_min": -36.34199523925781,
      "activations/layer14_attention_weight_max": 52.99489212036133,
      "activations/layer14_attention_weight_min": -42.32912826538086,
      "activations/layer15_attention_weight_max": 37.94499206542969,
      "activations/layer15_attention_weight_min": -30.824718475341797,
      "activations/layer16_attention_weight_max": 30.932085037231445,
      "activations/layer16_attention_weight_min": -27.599355697631836,
      "activations/layer17_attention_weight_max": 37.49215316772461,
      "activations/layer17_attention_weight_min": -26.80840492248535,
      "activations/layer18_attention_weight_max": 34.85774230957031,
      "activations/layer18_attention_weight_min": -22.637426376342773,
      "activations/layer19_attention_weight_max": 33.433284759521484,
      "activations/layer19_attention_weight_min": -29.94644546508789,
      "activations/layer1_attention_weight_max": 17.13279914855957,
      "activations/layer1_attention_weight_min": -13.843393325805664,
      "activations/layer20_attention_weight_max": 29.6730899810791,
      "activations/layer20_attention_weight_min": -24.244718551635742,
      "activations/layer21_attention_weight_max": 31.65291404724121,
      "activations/layer21_attention_weight_min": -23.599224090576172,
      "activations/layer22_attention_weight_max": 48.02690887451172,
      "activations/layer22_attention_weight_min": -29.59893798828125,
      "activations/layer23_attention_weight_max": 34.04124069213867,
      "activations/layer23_attention_weight_min": -26.154874801635742,
      "activations/layer2_attention_weight_max": 26.023754119873047,
      "activations/layer2_attention_weight_min": -26.0959529876709,
      "activations/layer3_attention_weight_max": 77.87911987304688,
      "activations/layer3_attention_weight_min": -77.1376953125,
      "activations/layer4_attention_weight_max": 102.5055160522461,
      "activations/layer4_attention_weight_min": -99.11506652832031,
      "activations/layer5_attention_weight_max": 52.501373291015625,
      "activations/layer5_attention_weight_min": -59.595298767089844,
      "activations/layer6_attention_weight_max": 43.02710723876953,
      "activations/layer6_attention_weight_min": -46.60135269165039,
      "activations/layer7_attention_weight_max": 86.11872863769531,
      "activations/layer7_attention_weight_min": -80.09252166748047,
      "activations/layer8_attention_weight_max": 38.419647216796875,
      "activations/layer8_attention_weight_min": -41.11644744873047,
      "activations/layer9_attention_weight_max": 32.8342399597168,
      "activations/layer9_attention_weight_min": -33.08956527709961,
      "epoch": 9.6,
      "learning_rate": 8.901325757575758e-05,
      "loss": 2.8005,
      "step": 165150
    },
    {
      "activations/layer0_attention_weight_max": 16.139144897460938,
      "activations/layer0_attention_weight_min": -12.484071731567383,
      "activations/layer10_attention_weight_max": 32.16325759887695,
      "activations/layer10_attention_weight_min": -32.07795715332031,
      "activations/layer11_attention_weight_max": 33.00749588012695,
      "activations/layer11_attention_weight_min": -31.902639389038086,
      "activations/layer12_attention_weight_max": 23.17859649658203,
      "activations/layer12_attention_weight_min": -25.788084030151367,
      "activations/layer13_attention_weight_max": 36.77399444580078,
      "activations/layer13_attention_weight_min": -33.04823684692383,
      "activations/layer14_attention_weight_max": 43.18177795410156,
      "activations/layer14_attention_weight_min": -38.774776458740234,
      "activations/layer15_attention_weight_max": 35.35192108154297,
      "activations/layer15_attention_weight_min": -29.923686981201172,
      "activations/layer16_attention_weight_max": 31.570112228393555,
      "activations/layer16_attention_weight_min": -27.265872955322266,
      "activations/layer17_attention_weight_max": 36.764713287353516,
      "activations/layer17_attention_weight_min": -25.427642822265625,
      "activations/layer18_attention_weight_max": 37.347496032714844,
      "activations/layer18_attention_weight_min": -23.78010368347168,
      "activations/layer19_attention_weight_max": 36.39869689941406,
      "activations/layer19_attention_weight_min": -31.522001266479492,
      "activations/layer1_attention_weight_max": 17.605432510375977,
      "activations/layer1_attention_weight_min": -15.334527969360352,
      "activations/layer20_attention_weight_max": 29.885986328125,
      "activations/layer20_attention_weight_min": -26.79006576538086,
      "activations/layer21_attention_weight_max": 31.84297752380371,
      "activations/layer21_attention_weight_min": -24.704296112060547,
      "activations/layer22_attention_weight_max": 46.6574592590332,
      "activations/layer22_attention_weight_min": -31.33231544494629,
      "activations/layer23_attention_weight_max": 35.63360595703125,
      "activations/layer23_attention_weight_min": -24.962528228759766,
      "activations/layer2_attention_weight_max": 29.699542999267578,
      "activations/layer2_attention_weight_min": -28.557897567749023,
      "activations/layer3_attention_weight_max": 84.71305084228516,
      "activations/layer3_attention_weight_min": -89.43079376220703,
      "activations/layer4_attention_weight_max": 113.59122467041016,
      "activations/layer4_attention_weight_min": -112.03108215332031,
      "activations/layer5_attention_weight_max": 54.95173263549805,
      "activations/layer5_attention_weight_min": -61.86481857299805,
      "activations/layer6_attention_weight_max": 45.25419998168945,
      "activations/layer6_attention_weight_min": -50.620582580566406,
      "activations/layer7_attention_weight_max": 88.008544921875,
      "activations/layer7_attention_weight_min": -91.53279876708984,
      "activations/layer8_attention_weight_max": 41.6565055847168,
      "activations/layer8_attention_weight_min": -42.09553909301758,
      "activations/layer9_attention_weight_max": 35.493717193603516,
      "activations/layer9_attention_weight_min": -34.54124450683594,
      "epoch": 9.6,
      "learning_rate": 8.899431818181817e-05,
      "loss": 2.805,
      "step": 165200
    },
    {
      "activations/layer0_attention_weight_max": 15.385764122009277,
      "activations/layer0_attention_weight_min": -12.4002103805542,
      "activations/layer10_attention_weight_max": 29.623611450195312,
      "activations/layer10_attention_weight_min": -30.33188247680664,
      "activations/layer11_attention_weight_max": 30.540355682373047,
      "activations/layer11_attention_weight_min": -29.56972312927246,
      "activations/layer12_attention_weight_max": 33.78590393066406,
      "activations/layer12_attention_weight_min": -26.729251861572266,
      "activations/layer13_attention_weight_max": 35.92502975463867,
      "activations/layer13_attention_weight_min": -37.72747802734375,
      "activations/layer14_attention_weight_max": 44.5528450012207,
      "activations/layer14_attention_weight_min": -38.7864875793457,
      "activations/layer15_attention_weight_max": 33.68238830566406,
      "activations/layer15_attention_weight_min": -31.871543884277344,
      "activations/layer16_attention_weight_max": 30.72149658203125,
      "activations/layer16_attention_weight_min": -29.25172996520996,
      "activations/layer17_attention_weight_max": 32.557552337646484,
      "activations/layer17_attention_weight_min": -26.098247528076172,
      "activations/layer18_attention_weight_max": 28.464218139648438,
      "activations/layer18_attention_weight_min": -22.00542449951172,
      "activations/layer19_attention_weight_max": 38.60624313354492,
      "activations/layer19_attention_weight_min": -31.14192008972168,
      "activations/layer1_attention_weight_max": 16.694915771484375,
      "activations/layer1_attention_weight_min": -13.665986061096191,
      "activations/layer20_attention_weight_max": 30.673954010009766,
      "activations/layer20_attention_weight_min": -26.248003005981445,
      "activations/layer21_attention_weight_max": 28.446697235107422,
      "activations/layer21_attention_weight_min": -23.992076873779297,
      "activations/layer22_attention_weight_max": 48.97600173950195,
      "activations/layer22_attention_weight_min": -28.981889724731445,
      "activations/layer23_attention_weight_max": 36.54131317138672,
      "activations/layer23_attention_weight_min": -26.486968994140625,
      "activations/layer2_attention_weight_max": 28.115650177001953,
      "activations/layer2_attention_weight_min": -28.848325729370117,
      "activations/layer3_attention_weight_max": 87.52605438232422,
      "activations/layer3_attention_weight_min": -87.77872467041016,
      "activations/layer4_attention_weight_max": 107.90740966796875,
      "activations/layer4_attention_weight_min": -111.34227752685547,
      "activations/layer5_attention_weight_max": 50.62615203857422,
      "activations/layer5_attention_weight_min": -59.532772064208984,
      "activations/layer6_attention_weight_max": 42.57728576660156,
      "activations/layer6_attention_weight_min": -49.94580841064453,
      "activations/layer7_attention_weight_max": 87.7847900390625,
      "activations/layer7_attention_weight_min": -82.01470184326172,
      "activations/layer8_attention_weight_max": 37.154319763183594,
      "activations/layer8_attention_weight_min": -38.21910858154297,
      "activations/layer9_attention_weight_max": 31.611127853393555,
      "activations/layer9_attention_weight_min": -31.200551986694336,
      "epoch": 9.6,
      "learning_rate": 8.897537878787878e-05,
      "loss": 2.7999,
      "step": 165250
    },
    {
      "activations/layer0_attention_weight_max": 15.633731842041016,
      "activations/layer0_attention_weight_min": -12.174427032470703,
      "activations/layer10_attention_weight_max": 33.673919677734375,
      "activations/layer10_attention_weight_min": -33.23158645629883,
      "activations/layer11_attention_weight_max": 34.141761779785156,
      "activations/layer11_attention_weight_min": -32.770320892333984,
      "activations/layer12_attention_weight_max": 41.92267990112305,
      "activations/layer12_attention_weight_min": -26.211280822753906,
      "activations/layer13_attention_weight_max": 40.31493377685547,
      "activations/layer13_attention_weight_min": -34.851531982421875,
      "activations/layer14_attention_weight_max": 50.234466552734375,
      "activations/layer14_attention_weight_min": -39.81425476074219,
      "activations/layer15_attention_weight_max": 35.27885437011719,
      "activations/layer15_attention_weight_min": -31.927061080932617,
      "activations/layer16_attention_weight_max": 37.21212387084961,
      "activations/layer16_attention_weight_min": -28.627470016479492,
      "activations/layer17_attention_weight_max": 31.796878814697266,
      "activations/layer17_attention_weight_min": -27.688932418823242,
      "activations/layer18_attention_weight_max": 27.878623962402344,
      "activations/layer18_attention_weight_min": -23.436176300048828,
      "activations/layer19_attention_weight_max": 41.3311882019043,
      "activations/layer19_attention_weight_min": -35.213584899902344,
      "activations/layer1_attention_weight_max": 17.68650245666504,
      "activations/layer1_attention_weight_min": -17.424697875976562,
      "activations/layer20_attention_weight_max": 34.3879280090332,
      "activations/layer20_attention_weight_min": -24.83974266052246,
      "activations/layer21_attention_weight_max": 35.117218017578125,
      "activations/layer21_attention_weight_min": -24.10587501525879,
      "activations/layer22_attention_weight_max": 46.1057014465332,
      "activations/layer22_attention_weight_min": -30.732725143432617,
      "activations/layer23_attention_weight_max": 36.38587188720703,
      "activations/layer23_attention_weight_min": -24.518247604370117,
      "activations/layer2_attention_weight_max": 29.931411743164062,
      "activations/layer2_attention_weight_min": -28.743675231933594,
      "activations/layer3_attention_weight_max": 86.67007446289062,
      "activations/layer3_attention_weight_min": -89.28250885009766,
      "activations/layer4_attention_weight_max": 112.7300033569336,
      "activations/layer4_attention_weight_min": -112.90470123291016,
      "activations/layer5_attention_weight_max": 52.823665618896484,
      "activations/layer5_attention_weight_min": -59.38534164428711,
      "activations/layer6_attention_weight_max": 45.1262321472168,
      "activations/layer6_attention_weight_min": -50.435115814208984,
      "activations/layer7_attention_weight_max": 88.87825012207031,
      "activations/layer7_attention_weight_min": -87.46954345703125,
      "activations/layer8_attention_weight_max": 41.60182571411133,
      "activations/layer8_attention_weight_min": -43.49726867675781,
      "activations/layer9_attention_weight_max": 33.64156723022461,
      "activations/layer9_attention_weight_min": -35.2336540222168,
      "epoch": 9.6,
      "learning_rate": 8.89564393939394e-05,
      "loss": 2.793,
      "step": 165300
    },
    {
      "activations/layer0_attention_weight_max": 14.900678634643555,
      "activations/layer0_attention_weight_min": -11.838476181030273,
      "activations/layer10_attention_weight_max": 43.84667205810547,
      "activations/layer10_attention_weight_min": -43.0836067199707,
      "activations/layer11_attention_weight_max": 44.46507263183594,
      "activations/layer11_attention_weight_min": -46.46583557128906,
      "activations/layer12_attention_weight_max": 40.57028579711914,
      "activations/layer12_attention_weight_min": -29.96457290649414,
      "activations/layer13_attention_weight_max": 52.96015167236328,
      "activations/layer13_attention_weight_min": -41.60398864746094,
      "activations/layer14_attention_weight_max": 61.926143646240234,
      "activations/layer14_attention_weight_min": -53.097618103027344,
      "activations/layer15_attention_weight_max": 42.50756072998047,
      "activations/layer15_attention_weight_min": -38.02593231201172,
      "activations/layer16_attention_weight_max": 39.8828010559082,
      "activations/layer16_attention_weight_min": -31.342981338500977,
      "activations/layer17_attention_weight_max": 36.39616775512695,
      "activations/layer17_attention_weight_min": -32.137413024902344,
      "activations/layer18_attention_weight_max": 33.59978103637695,
      "activations/layer18_attention_weight_min": -25.838090896606445,
      "activations/layer19_attention_weight_max": 41.29339599609375,
      "activations/layer19_attention_weight_min": -30.047727584838867,
      "activations/layer1_attention_weight_max": 16.55385971069336,
      "activations/layer1_attention_weight_min": -14.306877136230469,
      "activations/layer20_attention_weight_max": 31.63357162475586,
      "activations/layer20_attention_weight_min": -25.958690643310547,
      "activations/layer21_attention_weight_max": 27.959884643554688,
      "activations/layer21_attention_weight_min": -22.706005096435547,
      "activations/layer22_attention_weight_max": 46.11505126953125,
      "activations/layer22_attention_weight_min": -31.018890380859375,
      "activations/layer23_attention_weight_max": 35.542388916015625,
      "activations/layer23_attention_weight_min": -23.64073944091797,
      "activations/layer2_attention_weight_max": 32.58148956298828,
      "activations/layer2_attention_weight_min": -34.1805419921875,
      "activations/layer3_attention_weight_max": 99.39637756347656,
      "activations/layer3_attention_weight_min": -108.1992416381836,
      "activations/layer4_attention_weight_max": 123.3495101928711,
      "activations/layer4_attention_weight_min": -122.88970947265625,
      "activations/layer5_attention_weight_max": 56.24714279174805,
      "activations/layer5_attention_weight_min": -58.60946273803711,
      "activations/layer6_attention_weight_max": 49.81132125854492,
      "activations/layer6_attention_weight_min": -50.86917495727539,
      "activations/layer7_attention_weight_max": 111.9565658569336,
      "activations/layer7_attention_weight_min": -111.54661560058594,
      "activations/layer8_attention_weight_max": 52.461544036865234,
      "activations/layer8_attention_weight_min": -52.628334045410156,
      "activations/layer9_attention_weight_max": 45.15607833862305,
      "activations/layer9_attention_weight_min": -43.61810302734375,
      "epoch": 9.61,
      "learning_rate": 8.893749999999999e-05,
      "loss": 2.7946,
      "step": 165350
    },
    {
      "activations/layer0_attention_weight_max": 15.19930362701416,
      "activations/layer0_attention_weight_min": -12.431772232055664,
      "activations/layer10_attention_weight_max": 33.557899475097656,
      "activations/layer10_attention_weight_min": -32.20286560058594,
      "activations/layer11_attention_weight_max": 31.631420135498047,
      "activations/layer11_attention_weight_min": -29.778396606445312,
      "activations/layer12_attention_weight_max": 25.15323829650879,
      "activations/layer12_attention_weight_min": -25.36590003967285,
      "activations/layer13_attention_weight_max": 40.883636474609375,
      "activations/layer13_attention_weight_min": -33.46220016479492,
      "activations/layer14_attention_weight_max": 44.85929489135742,
      "activations/layer14_attention_weight_min": -37.59256362915039,
      "activations/layer15_attention_weight_max": 38.92369842529297,
      "activations/layer15_attention_weight_min": -30.241491317749023,
      "activations/layer16_attention_weight_max": 33.80973434448242,
      "activations/layer16_attention_weight_min": -28.647397994995117,
      "activations/layer17_attention_weight_max": 34.542964935302734,
      "activations/layer17_attention_weight_min": -24.981149673461914,
      "activations/layer18_attention_weight_max": 31.1850528717041,
      "activations/layer18_attention_weight_min": -23.75594711303711,
      "activations/layer19_attention_weight_max": 36.16083908081055,
      "activations/layer19_attention_weight_min": -33.17934799194336,
      "activations/layer1_attention_weight_max": 17.237041473388672,
      "activations/layer1_attention_weight_min": -14.865652084350586,
      "activations/layer20_attention_weight_max": 31.884014129638672,
      "activations/layer20_attention_weight_min": -26.012407302856445,
      "activations/layer21_attention_weight_max": 30.12779426574707,
      "activations/layer21_attention_weight_min": -25.98324966430664,
      "activations/layer22_attention_weight_max": 47.49782943725586,
      "activations/layer22_attention_weight_min": -32.38562774658203,
      "activations/layer23_attention_weight_max": 35.99787139892578,
      "activations/layer23_attention_weight_min": -28.28327751159668,
      "activations/layer2_attention_weight_max": 31.61787986755371,
      "activations/layer2_attention_weight_min": -30.158206939697266,
      "activations/layer3_attention_weight_max": 94.83479309082031,
      "activations/layer3_attention_weight_min": -93.64962768554688,
      "activations/layer4_attention_weight_max": 117.5372314453125,
      "activations/layer4_attention_weight_min": -115.50099182128906,
      "activations/layer5_attention_weight_max": 52.46064758300781,
      "activations/layer5_attention_weight_min": -60.018829345703125,
      "activations/layer6_attention_weight_max": 47.02231979370117,
      "activations/layer6_attention_weight_min": -51.817405700683594,
      "activations/layer7_attention_weight_max": 91.01127624511719,
      "activations/layer7_attention_weight_min": -94.41395568847656,
      "activations/layer8_attention_weight_max": 41.1393928527832,
      "activations/layer8_attention_weight_min": -42.42216873168945,
      "activations/layer9_attention_weight_max": 34.288818359375,
      "activations/layer9_attention_weight_min": -35.08512878417969,
      "epoch": 9.61,
      "learning_rate": 8.89185606060606e-05,
      "loss": 2.8107,
      "step": 165400
    },
    {
      "activations/layer0_attention_weight_max": 15.300948143005371,
      "activations/layer0_attention_weight_min": -11.974733352661133,
      "activations/layer10_attention_weight_max": 34.38972854614258,
      "activations/layer10_attention_weight_min": -34.45033645629883,
      "activations/layer11_attention_weight_max": 34.36724090576172,
      "activations/layer11_attention_weight_min": -33.64574432373047,
      "activations/layer12_attention_weight_max": 25.348310470581055,
      "activations/layer12_attention_weight_min": -27.224502563476562,
      "activations/layer13_attention_weight_max": 43.66854476928711,
      "activations/layer13_attention_weight_min": -32.78010940551758,
      "activations/layer14_attention_weight_max": 50.616111755371094,
      "activations/layer14_attention_weight_min": -36.92682647705078,
      "activations/layer15_attention_weight_max": 35.51519012451172,
      "activations/layer15_attention_weight_min": -29.274995803833008,
      "activations/layer16_attention_weight_max": 30.651960372924805,
      "activations/layer16_attention_weight_min": -26.199607849121094,
      "activations/layer17_attention_weight_max": 33.52214431762695,
      "activations/layer17_attention_weight_min": -25.636560440063477,
      "activations/layer18_attention_weight_max": 30.830289840698242,
      "activations/layer18_attention_weight_min": -22.411794662475586,
      "activations/layer19_attention_weight_max": 34.09577560424805,
      "activations/layer19_attention_weight_min": -30.169973373413086,
      "activations/layer1_attention_weight_max": 16.197994232177734,
      "activations/layer1_attention_weight_min": -13.765070915222168,
      "activations/layer20_attention_weight_max": 30.201841354370117,
      "activations/layer20_attention_weight_min": -24.690128326416016,
      "activations/layer21_attention_weight_max": 29.791118621826172,
      "activations/layer21_attention_weight_min": -23.211505889892578,
      "activations/layer22_attention_weight_max": 45.82648468017578,
      "activations/layer22_attention_weight_min": -29.50773811340332,
      "activations/layer23_attention_weight_max": 35.922359466552734,
      "activations/layer23_attention_weight_min": -25.684452056884766,
      "activations/layer2_attention_weight_max": 30.266340255737305,
      "activations/layer2_attention_weight_min": -29.783721923828125,
      "activations/layer3_attention_weight_max": 92.47462463378906,
      "activations/layer3_attention_weight_min": -91.26653289794922,
      "activations/layer4_attention_weight_max": 118.74403381347656,
      "activations/layer4_attention_weight_min": -117.01397705078125,
      "activations/layer5_attention_weight_max": 52.87551498413086,
      "activations/layer5_attention_weight_min": -60.21238327026367,
      "activations/layer6_attention_weight_max": 46.86823272705078,
      "activations/layer6_attention_weight_min": -49.530643463134766,
      "activations/layer7_attention_weight_max": 91.37938690185547,
      "activations/layer7_attention_weight_min": -90.24314880371094,
      "activations/layer8_attention_weight_max": 40.39410400390625,
      "activations/layer8_attention_weight_min": -41.719242095947266,
      "activations/layer9_attention_weight_max": 36.08530807495117,
      "activations/layer9_attention_weight_min": -34.64433670043945,
      "epoch": 9.61,
      "learning_rate": 8.88996212121212e-05,
      "loss": 2.7982,
      "step": 165450
    },
    {
      "activations/layer0_attention_weight_max": 15.172528266906738,
      "activations/layer0_attention_weight_min": -13.31005859375,
      "activations/layer10_attention_weight_max": 35.58235549926758,
      "activations/layer10_attention_weight_min": -34.93067169189453,
      "activations/layer11_attention_weight_max": 33.636985778808594,
      "activations/layer11_attention_weight_min": -32.760250091552734,
      "activations/layer12_attention_weight_max": 29.948728561401367,
      "activations/layer12_attention_weight_min": -26.45073890686035,
      "activations/layer13_attention_weight_max": 40.2314453125,
      "activations/layer13_attention_weight_min": -35.3648796081543,
      "activations/layer14_attention_weight_max": 49.313499450683594,
      "activations/layer14_attention_weight_min": -42.58103561401367,
      "activations/layer15_attention_weight_max": 35.75975036621094,
      "activations/layer15_attention_weight_min": -30.75575065612793,
      "activations/layer16_attention_weight_max": 33.61286163330078,
      "activations/layer16_attention_weight_min": -31.09031105041504,
      "activations/layer17_attention_weight_max": 33.92267990112305,
      "activations/layer17_attention_weight_min": -27.335620880126953,
      "activations/layer18_attention_weight_max": 29.75214958190918,
      "activations/layer18_attention_weight_min": -22.713733673095703,
      "activations/layer19_attention_weight_max": 35.181495666503906,
      "activations/layer19_attention_weight_min": -33.63894271850586,
      "activations/layer1_attention_weight_max": 16.62502098083496,
      "activations/layer1_attention_weight_min": -14.713441848754883,
      "activations/layer20_attention_weight_max": 31.74660873413086,
      "activations/layer20_attention_weight_min": -27.72756004333496,
      "activations/layer21_attention_weight_max": 27.436107635498047,
      "activations/layer21_attention_weight_min": -26.909704208374023,
      "activations/layer22_attention_weight_max": 44.871864318847656,
      "activations/layer22_attention_weight_min": -30.895845413208008,
      "activations/layer23_attention_weight_max": 34.252254486083984,
      "activations/layer23_attention_weight_min": -24.51418685913086,
      "activations/layer2_attention_weight_max": 30.035078048706055,
      "activations/layer2_attention_weight_min": -30.357769012451172,
      "activations/layer3_attention_weight_max": 93.0200424194336,
      "activations/layer3_attention_weight_min": -98.35265350341797,
      "activations/layer4_attention_weight_max": 115.7601318359375,
      "activations/layer4_attention_weight_min": -116.34488677978516,
      "activations/layer5_attention_weight_max": 51.517608642578125,
      "activations/layer5_attention_weight_min": -64.34471130371094,
      "activations/layer6_attention_weight_max": 47.803035736083984,
      "activations/layer6_attention_weight_min": -50.65260314941406,
      "activations/layer7_attention_weight_max": 89.21701049804688,
      "activations/layer7_attention_weight_min": -92.24322509765625,
      "activations/layer8_attention_weight_max": 40.76805114746094,
      "activations/layer8_attention_weight_min": -47.33381652832031,
      "activations/layer9_attention_weight_max": 35.68669509887695,
      "activations/layer9_attention_weight_min": -35.3020133972168,
      "epoch": 9.62,
      "learning_rate": 8.88806818181818e-05,
      "loss": 2.7927,
      "step": 165500
    },
    {
      "activations/layer0_attention_weight_max": 15.066078186035156,
      "activations/layer0_attention_weight_min": -12.587054252624512,
      "activations/layer10_attention_weight_max": 37.025848388671875,
      "activations/layer10_attention_weight_min": -33.903350830078125,
      "activations/layer11_attention_weight_max": 34.59404754638672,
      "activations/layer11_attention_weight_min": -32.688514709472656,
      "activations/layer12_attention_weight_max": 26.34856414794922,
      "activations/layer12_attention_weight_min": -27.774919509887695,
      "activations/layer13_attention_weight_max": 41.442955017089844,
      "activations/layer13_attention_weight_min": -33.007286071777344,
      "activations/layer14_attention_weight_max": 51.584495544433594,
      "activations/layer14_attention_weight_min": -38.762393951416016,
      "activations/layer15_attention_weight_max": 43.61781692504883,
      "activations/layer15_attention_weight_min": -31.374853134155273,
      "activations/layer16_attention_weight_max": 31.077680587768555,
      "activations/layer16_attention_weight_min": -26.62177848815918,
      "activations/layer17_attention_weight_max": 35.32039260864258,
      "activations/layer17_attention_weight_min": -26.395328521728516,
      "activations/layer18_attention_weight_max": 33.97509002685547,
      "activations/layer18_attention_weight_min": -23.319005966186523,
      "activations/layer19_attention_weight_max": 36.53165054321289,
      "activations/layer19_attention_weight_min": -32.2581901550293,
      "activations/layer1_attention_weight_max": 17.010162353515625,
      "activations/layer1_attention_weight_min": -15.012152671813965,
      "activations/layer20_attention_weight_max": 31.947521209716797,
      "activations/layer20_attention_weight_min": -26.116548538208008,
      "activations/layer21_attention_weight_max": 32.5902099609375,
      "activations/layer21_attention_weight_min": -25.35700035095215,
      "activations/layer22_attention_weight_max": 49.63401794433594,
      "activations/layer22_attention_weight_min": -32.62854766845703,
      "activations/layer23_attention_weight_max": 38.9252815246582,
      "activations/layer23_attention_weight_min": -25.57197380065918,
      "activations/layer2_attention_weight_max": 31.543960571289062,
      "activations/layer2_attention_weight_min": -30.061412811279297,
      "activations/layer3_attention_weight_max": 97.55851745605469,
      "activations/layer3_attention_weight_min": -94.30367279052734,
      "activations/layer4_attention_weight_max": 122.5579605102539,
      "activations/layer4_attention_weight_min": -116.95123291015625,
      "activations/layer5_attention_weight_max": 55.89000701904297,
      "activations/layer5_attention_weight_min": -60.18928527832031,
      "activations/layer6_attention_weight_max": 49.22895050048828,
      "activations/layer6_attention_weight_min": -53.17043685913086,
      "activations/layer7_attention_weight_max": 89.19859313964844,
      "activations/layer7_attention_weight_min": -96.35113525390625,
      "activations/layer8_attention_weight_max": 44.551536560058594,
      "activations/layer8_attention_weight_min": -45.2993049621582,
      "activations/layer9_attention_weight_max": 37.52946472167969,
      "activations/layer9_attention_weight_min": -35.260799407958984,
      "epoch": 9.62,
      "learning_rate": 8.886174242424242e-05,
      "loss": 2.7934,
      "step": 165550
    },
    {
      "activations/layer0_attention_weight_max": 14.846600532531738,
      "activations/layer0_attention_weight_min": -12.291563034057617,
      "activations/layer10_attention_weight_max": 34.764564514160156,
      "activations/layer10_attention_weight_min": -33.2022819519043,
      "activations/layer11_attention_weight_max": 34.390594482421875,
      "activations/layer11_attention_weight_min": -31.535985946655273,
      "activations/layer12_attention_weight_max": 25.986652374267578,
      "activations/layer12_attention_weight_min": -25.236759185791016,
      "activations/layer13_attention_weight_max": 41.8238639831543,
      "activations/layer13_attention_weight_min": -33.785186767578125,
      "activations/layer14_attention_weight_max": 47.011383056640625,
      "activations/layer14_attention_weight_min": -36.716590881347656,
      "activations/layer15_attention_weight_max": 39.62417221069336,
      "activations/layer15_attention_weight_min": -31.021268844604492,
      "activations/layer16_attention_weight_max": 32.44768142700195,
      "activations/layer16_attention_weight_min": -28.60948371887207,
      "activations/layer17_attention_weight_max": 36.71540069580078,
      "activations/layer17_attention_weight_min": -27.0977783203125,
      "activations/layer18_attention_weight_max": 34.31424331665039,
      "activations/layer18_attention_weight_min": -22.41138458251953,
      "activations/layer19_attention_weight_max": 37.390987396240234,
      "activations/layer19_attention_weight_min": -30.93186378479004,
      "activations/layer1_attention_weight_max": 16.812517166137695,
      "activations/layer1_attention_weight_min": -15.414313316345215,
      "activations/layer20_attention_weight_max": 30.525938034057617,
      "activations/layer20_attention_weight_min": -26.24904441833496,
      "activations/layer21_attention_weight_max": 33.30495071411133,
      "activations/layer21_attention_weight_min": -23.133956909179688,
      "activations/layer22_attention_weight_max": 51.57038116455078,
      "activations/layer22_attention_weight_min": -28.79304313659668,
      "activations/layer23_attention_weight_max": 39.59026336669922,
      "activations/layer23_attention_weight_min": -23.180692672729492,
      "activations/layer2_attention_weight_max": 30.901342391967773,
      "activations/layer2_attention_weight_min": -30.140165328979492,
      "activations/layer3_attention_weight_max": 87.19467163085938,
      "activations/layer3_attention_weight_min": -89.0680923461914,
      "activations/layer4_attention_weight_max": 110.37902069091797,
      "activations/layer4_attention_weight_min": -111.09577941894531,
      "activations/layer5_attention_weight_max": 52.38902282714844,
      "activations/layer5_attention_weight_min": -62.14829635620117,
      "activations/layer6_attention_weight_max": 46.794185638427734,
      "activations/layer6_attention_weight_min": -50.66722869873047,
      "activations/layer7_attention_weight_max": 88.35079956054688,
      "activations/layer7_attention_weight_min": -89.48668670654297,
      "activations/layer8_attention_weight_max": 42.304325103759766,
      "activations/layer8_attention_weight_min": -41.54671859741211,
      "activations/layer9_attention_weight_max": 35.91126251220703,
      "activations/layer9_attention_weight_min": -35.45245361328125,
      "epoch": 9.62,
      "learning_rate": 8.884280303030302e-05,
      "loss": 2.801,
      "step": 165600
    },
    {
      "activations/layer0_attention_weight_max": 14.360690116882324,
      "activations/layer0_attention_weight_min": -11.218588829040527,
      "activations/layer10_attention_weight_max": 34.626094818115234,
      "activations/layer10_attention_weight_min": -32.49787521362305,
      "activations/layer11_attention_weight_max": 34.70227813720703,
      "activations/layer11_attention_weight_min": -31.51911735534668,
      "activations/layer12_attention_weight_max": 22.8403377532959,
      "activations/layer12_attention_weight_min": -22.856708526611328,
      "activations/layer13_attention_weight_max": 38.10530090332031,
      "activations/layer13_attention_weight_min": -31.87723731994629,
      "activations/layer14_attention_weight_max": 47.331966400146484,
      "activations/layer14_attention_weight_min": -40.1469841003418,
      "activations/layer15_attention_weight_max": 37.231712341308594,
      "activations/layer15_attention_weight_min": -29.135377883911133,
      "activations/layer16_attention_weight_max": 33.070919036865234,
      "activations/layer16_attention_weight_min": -25.8977108001709,
      "activations/layer17_attention_weight_max": 37.04835891723633,
      "activations/layer17_attention_weight_min": -23.357158660888672,
      "activations/layer18_attention_weight_max": 33.02254104614258,
      "activations/layer18_attention_weight_min": -20.51565170288086,
      "activations/layer19_attention_weight_max": 36.12858200073242,
      "activations/layer19_attention_weight_min": -30.31026268005371,
      "activations/layer1_attention_weight_max": 16.241668701171875,
      "activations/layer1_attention_weight_min": -15.364907264709473,
      "activations/layer20_attention_weight_max": 29.6967716217041,
      "activations/layer20_attention_weight_min": -22.963584899902344,
      "activations/layer21_attention_weight_max": 32.1624641418457,
      "activations/layer21_attention_weight_min": -22.047958374023438,
      "activations/layer22_attention_weight_max": 48.80893325805664,
      "activations/layer22_attention_weight_min": -27.742782592773438,
      "activations/layer23_attention_weight_max": 37.053916931152344,
      "activations/layer23_attention_weight_min": -25.112606048583984,
      "activations/layer2_attention_weight_max": 30.184772491455078,
      "activations/layer2_attention_weight_min": -29.981197357177734,
      "activations/layer3_attention_weight_max": 91.60802459716797,
      "activations/layer3_attention_weight_min": -94.83001708984375,
      "activations/layer4_attention_weight_max": 115.60047912597656,
      "activations/layer4_attention_weight_min": -119.6689453125,
      "activations/layer5_attention_weight_max": 54.41402816772461,
      "activations/layer5_attention_weight_min": -66.50784301757812,
      "activations/layer6_attention_weight_max": 47.03728485107422,
      "activations/layer6_attention_weight_min": -51.49040222167969,
      "activations/layer7_attention_weight_max": 89.91895294189453,
      "activations/layer7_attention_weight_min": -90.21975708007812,
      "activations/layer8_attention_weight_max": 40.823917388916016,
      "activations/layer8_attention_weight_min": -42.293521881103516,
      "activations/layer9_attention_weight_max": 35.33576583862305,
      "activations/layer9_attention_weight_min": -33.8581428527832,
      "epoch": 9.63,
      "learning_rate": 8.882386363636362e-05,
      "loss": 2.802,
      "step": 165650
    },
    {
      "activations/layer0_attention_weight_max": 15.245079040527344,
      "activations/layer0_attention_weight_min": -12.523587226867676,
      "activations/layer10_attention_weight_max": 31.06743049621582,
      "activations/layer10_attention_weight_min": -30.84773826599121,
      "activations/layer11_attention_weight_max": 31.279499053955078,
      "activations/layer11_attention_weight_min": -29.025127410888672,
      "activations/layer12_attention_weight_max": 22.441162109375,
      "activations/layer12_attention_weight_min": -23.341400146484375,
      "activations/layer13_attention_weight_max": 36.79469299316406,
      "activations/layer13_attention_weight_min": -32.38303756713867,
      "activations/layer14_attention_weight_max": 41.264739990234375,
      "activations/layer14_attention_weight_min": -37.10192108154297,
      "activations/layer15_attention_weight_max": 33.52207565307617,
      "activations/layer15_attention_weight_min": -30.99210548400879,
      "activations/layer16_attention_weight_max": 29.028844833374023,
      "activations/layer16_attention_weight_min": -29.641881942749023,
      "activations/layer17_attention_weight_max": 32.69890594482422,
      "activations/layer17_attention_weight_min": -25.979860305786133,
      "activations/layer18_attention_weight_max": 30.08399772644043,
      "activations/layer18_attention_weight_min": -21.89559555053711,
      "activations/layer19_attention_weight_max": 33.1565055847168,
      "activations/layer19_attention_weight_min": -29.654783248901367,
      "activations/layer1_attention_weight_max": 16.873878479003906,
      "activations/layer1_attention_weight_min": -13.868135452270508,
      "activations/layer20_attention_weight_max": 33.156246185302734,
      "activations/layer20_attention_weight_min": -24.774818420410156,
      "activations/layer21_attention_weight_max": 30.570171356201172,
      "activations/layer21_attention_weight_min": -22.234695434570312,
      "activations/layer22_attention_weight_max": 43.916847229003906,
      "activations/layer22_attention_weight_min": -26.823505401611328,
      "activations/layer23_attention_weight_max": 33.263763427734375,
      "activations/layer23_attention_weight_min": -23.571035385131836,
      "activations/layer2_attention_weight_max": 30.002559661865234,
      "activations/layer2_attention_weight_min": -31.072355270385742,
      "activations/layer3_attention_weight_max": 91.88353729248047,
      "activations/layer3_attention_weight_min": -92.99678802490234,
      "activations/layer4_attention_weight_max": 115.990234375,
      "activations/layer4_attention_weight_min": -119.62811279296875,
      "activations/layer5_attention_weight_max": 53.51641845703125,
      "activations/layer5_attention_weight_min": -63.79705047607422,
      "activations/layer6_attention_weight_max": 47.54572677612305,
      "activations/layer6_attention_weight_min": -51.50118637084961,
      "activations/layer7_attention_weight_max": 89.75862121582031,
      "activations/layer7_attention_weight_min": -89.97610473632812,
      "activations/layer8_attention_weight_max": 40.66770553588867,
      "activations/layer8_attention_weight_min": -43.23474884033203,
      "activations/layer9_attention_weight_max": 32.887908935546875,
      "activations/layer9_attention_weight_min": -33.65818405151367,
      "epoch": 9.63,
      "learning_rate": 8.880492424242424e-05,
      "loss": 2.7973,
      "step": 165700
    },
    {
      "activations/layer0_attention_weight_max": 15.418309211730957,
      "activations/layer0_attention_weight_min": -13.651463508605957,
      "activations/layer10_attention_weight_max": 34.95690155029297,
      "activations/layer10_attention_weight_min": -33.36673355102539,
      "activations/layer11_attention_weight_max": 35.99519348144531,
      "activations/layer11_attention_weight_min": -31.771398544311523,
      "activations/layer12_attention_weight_max": 35.915016174316406,
      "activations/layer12_attention_weight_min": -24.67206573486328,
      "activations/layer13_attention_weight_max": 44.43196105957031,
      "activations/layer13_attention_weight_min": -34.733245849609375,
      "activations/layer14_attention_weight_max": 49.30972671508789,
      "activations/layer14_attention_weight_min": -38.01197052001953,
      "activations/layer15_attention_weight_max": 48.44918441772461,
      "activations/layer15_attention_weight_min": -30.504377365112305,
      "activations/layer16_attention_weight_max": 33.31294631958008,
      "activations/layer16_attention_weight_min": -30.015031814575195,
      "activations/layer17_attention_weight_max": 37.64265441894531,
      "activations/layer17_attention_weight_min": -27.00912857055664,
      "activations/layer18_attention_weight_max": 34.865234375,
      "activations/layer18_attention_weight_min": -23.323423385620117,
      "activations/layer19_attention_weight_max": 38.44857406616211,
      "activations/layer19_attention_weight_min": -30.5266056060791,
      "activations/layer1_attention_weight_max": 16.273483276367188,
      "activations/layer1_attention_weight_min": -14.326911926269531,
      "activations/layer20_attention_weight_max": 34.57858657836914,
      "activations/layer20_attention_weight_min": -24.776073455810547,
      "activations/layer21_attention_weight_max": 33.07667541503906,
      "activations/layer21_attention_weight_min": -25.37584114074707,
      "activations/layer22_attention_weight_max": 50.773231506347656,
      "activations/layer22_attention_weight_min": -29.65022087097168,
      "activations/layer23_attention_weight_max": 37.80316162109375,
      "activations/layer23_attention_weight_min": -24.43224334716797,
      "activations/layer2_attention_weight_max": 29.163936614990234,
      "activations/layer2_attention_weight_min": -29.14101791381836,
      "activations/layer3_attention_weight_max": 90.1457748413086,
      "activations/layer3_attention_weight_min": -89.84930419921875,
      "activations/layer4_attention_weight_max": 109.45012664794922,
      "activations/layer4_attention_weight_min": -106.47725677490234,
      "activations/layer5_attention_weight_max": 50.59797668457031,
      "activations/layer5_attention_weight_min": -61.44272994995117,
      "activations/layer6_attention_weight_max": 45.83669662475586,
      "activations/layer6_attention_weight_min": -49.4056510925293,
      "activations/layer7_attention_weight_max": 87.79805755615234,
      "activations/layer7_attention_weight_min": -87.77837371826172,
      "activations/layer8_attention_weight_max": 39.71332550048828,
      "activations/layer8_attention_weight_min": -41.50357437133789,
      "activations/layer9_attention_weight_max": 34.674617767333984,
      "activations/layer9_attention_weight_min": -33.74655532836914,
      "epoch": 9.63,
      "learning_rate": 8.878598484848484e-05,
      "loss": 2.7823,
      "step": 165750
    },
    {
      "activations/layer0_attention_weight_max": 15.150989532470703,
      "activations/layer0_attention_weight_min": -11.711498260498047,
      "activations/layer10_attention_weight_max": 34.09009552001953,
      "activations/layer10_attention_weight_min": -32.65823745727539,
      "activations/layer11_attention_weight_max": 37.69867706298828,
      "activations/layer11_attention_weight_min": -31.70471954345703,
      "activations/layer12_attention_weight_max": 30.771873474121094,
      "activations/layer12_attention_weight_min": -27.420364379882812,
      "activations/layer13_attention_weight_max": 42.28107452392578,
      "activations/layer13_attention_weight_min": -34.28436279296875,
      "activations/layer14_attention_weight_max": 58.58113479614258,
      "activations/layer14_attention_weight_min": -44.995182037353516,
      "activations/layer15_attention_weight_max": 41.72018814086914,
      "activations/layer15_attention_weight_min": -32.17799758911133,
      "activations/layer16_attention_weight_max": 36.86026382446289,
      "activations/layer16_attention_weight_min": -28.711706161499023,
      "activations/layer17_attention_weight_max": 34.29134750366211,
      "activations/layer17_attention_weight_min": -27.4779109954834,
      "activations/layer18_attention_weight_max": 31.664791107177734,
      "activations/layer18_attention_weight_min": -23.33791732788086,
      "activations/layer19_attention_weight_max": 43.41596603393555,
      "activations/layer19_attention_weight_min": -32.61600875854492,
      "activations/layer1_attention_weight_max": 17.286401748657227,
      "activations/layer1_attention_weight_min": -15.935736656188965,
      "activations/layer20_attention_weight_max": 36.654335021972656,
      "activations/layer20_attention_weight_min": -25.70723533630371,
      "activations/layer21_attention_weight_max": 32.43281173706055,
      "activations/layer21_attention_weight_min": -26.530874252319336,
      "activations/layer22_attention_weight_max": 54.229393005371094,
      "activations/layer22_attention_weight_min": -30.241914749145508,
      "activations/layer23_attention_weight_max": 39.44263458251953,
      "activations/layer23_attention_weight_min": -23.71768569946289,
      "activations/layer2_attention_weight_max": 30.262739181518555,
      "activations/layer2_attention_weight_min": -29.421863555908203,
      "activations/layer3_attention_weight_max": 89.50131225585938,
      "activations/layer3_attention_weight_min": -90.02175903320312,
      "activations/layer4_attention_weight_max": 114.21776580810547,
      "activations/layer4_attention_weight_min": -114.3471450805664,
      "activations/layer5_attention_weight_max": 52.352630615234375,
      "activations/layer5_attention_weight_min": -60.74358367919922,
      "activations/layer6_attention_weight_max": 46.55377960205078,
      "activations/layer6_attention_weight_min": -49.139286041259766,
      "activations/layer7_attention_weight_max": 89.36599731445312,
      "activations/layer7_attention_weight_min": -89.26081085205078,
      "activations/layer8_attention_weight_max": 40.91160202026367,
      "activations/layer8_attention_weight_min": -41.89132308959961,
      "activations/layer9_attention_weight_max": 35.81193923950195,
      "activations/layer9_attention_weight_min": -34.565555572509766,
      "epoch": 9.63,
      "learning_rate": 8.876704545454545e-05,
      "loss": 2.7883,
      "step": 165800
    },
    {
      "activations/layer0_attention_weight_max": 15.215141296386719,
      "activations/layer0_attention_weight_min": -11.415047645568848,
      "activations/layer10_attention_weight_max": 35.381771087646484,
      "activations/layer10_attention_weight_min": -34.87760543823242,
      "activations/layer11_attention_weight_max": 35.56431579589844,
      "activations/layer11_attention_weight_min": -33.21120834350586,
      "activations/layer12_attention_weight_max": 26.93952178955078,
      "activations/layer12_attention_weight_min": -25.0914249420166,
      "activations/layer13_attention_weight_max": 43.93080139160156,
      "activations/layer13_attention_weight_min": -34.21337890625,
      "activations/layer14_attention_weight_max": 51.515323638916016,
      "activations/layer14_attention_weight_min": -41.81797409057617,
      "activations/layer15_attention_weight_max": 39.64267349243164,
      "activations/layer15_attention_weight_min": -32.044063568115234,
      "activations/layer16_attention_weight_max": 31.825145721435547,
      "activations/layer16_attention_weight_min": -27.825010299682617,
      "activations/layer17_attention_weight_max": 35.13374710083008,
      "activations/layer17_attention_weight_min": -25.8511905670166,
      "activations/layer18_attention_weight_max": 33.36863708496094,
      "activations/layer18_attention_weight_min": -22.25926971435547,
      "activations/layer19_attention_weight_max": 37.773529052734375,
      "activations/layer19_attention_weight_min": -31.60291862487793,
      "activations/layer1_attention_weight_max": 16.480205535888672,
      "activations/layer1_attention_weight_min": -14.092562675476074,
      "activations/layer20_attention_weight_max": 32.273887634277344,
      "activations/layer20_attention_weight_min": -24.184404373168945,
      "activations/layer21_attention_weight_max": 29.422760009765625,
      "activations/layer21_attention_weight_min": -22.838258743286133,
      "activations/layer22_attention_weight_max": 57.41931915283203,
      "activations/layer22_attention_weight_min": -28.38286018371582,
      "activations/layer23_attention_weight_max": 37.579437255859375,
      "activations/layer23_attention_weight_min": -24.75518798828125,
      "activations/layer2_attention_weight_max": 30.475109100341797,
      "activations/layer2_attention_weight_min": -29.212535858154297,
      "activations/layer3_attention_weight_max": 87.37902069091797,
      "activations/layer3_attention_weight_min": -88.80817413330078,
      "activations/layer4_attention_weight_max": 114.043701171875,
      "activations/layer4_attention_weight_min": -111.54080963134766,
      "activations/layer5_attention_weight_max": 57.08379364013672,
      "activations/layer5_attention_weight_min": -63.4148063659668,
      "activations/layer6_attention_weight_max": 46.62773513793945,
      "activations/layer6_attention_weight_min": -50.01817321777344,
      "activations/layer7_attention_weight_max": 92.43460845947266,
      "activations/layer7_attention_weight_min": -92.14704132080078,
      "activations/layer8_attention_weight_max": 41.17477035522461,
      "activations/layer8_attention_weight_min": -44.45259475708008,
      "activations/layer9_attention_weight_max": 36.547969818115234,
      "activations/layer9_attention_weight_min": -35.03816223144531,
      "epoch": 9.64,
      "learning_rate": 8.874810606060604e-05,
      "loss": 2.791,
      "step": 165850
    },
    {
      "activations/layer0_attention_weight_max": 14.279891967773438,
      "activations/layer0_attention_weight_min": -11.958671569824219,
      "activations/layer10_attention_weight_max": 33.23423385620117,
      "activations/layer10_attention_weight_min": -34.42543411254883,
      "activations/layer11_attention_weight_max": 33.65629577636719,
      "activations/layer11_attention_weight_min": -31.030248641967773,
      "activations/layer12_attention_weight_max": 22.701311111450195,
      "activations/layer12_attention_weight_min": -24.36064338684082,
      "activations/layer13_attention_weight_max": 36.84298324584961,
      "activations/layer13_attention_weight_min": -35.45341110229492,
      "activations/layer14_attention_weight_max": 44.04648971557617,
      "activations/layer14_attention_weight_min": -36.975257873535156,
      "activations/layer15_attention_weight_max": 34.97523880004883,
      "activations/layer15_attention_weight_min": -31.64906883239746,
      "activations/layer16_attention_weight_max": 33.56360626220703,
      "activations/layer16_attention_weight_min": -28.873361587524414,
      "activations/layer17_attention_weight_max": 34.932010650634766,
      "activations/layer17_attention_weight_min": -25.89059829711914,
      "activations/layer18_attention_weight_max": 31.60247802734375,
      "activations/layer18_attention_weight_min": -21.330480575561523,
      "activations/layer19_attention_weight_max": 37.5616340637207,
      "activations/layer19_attention_weight_min": -30.703495025634766,
      "activations/layer1_attention_weight_max": 16.9888858795166,
      "activations/layer1_attention_weight_min": -13.698044776916504,
      "activations/layer20_attention_weight_max": 30.853551864624023,
      "activations/layer20_attention_weight_min": -24.558757781982422,
      "activations/layer21_attention_weight_max": 29.33556365966797,
      "activations/layer21_attention_weight_min": -24.134105682373047,
      "activations/layer22_attention_weight_max": 48.9002799987793,
      "activations/layer22_attention_weight_min": -30.99017333984375,
      "activations/layer23_attention_weight_max": 35.8712158203125,
      "activations/layer23_attention_weight_min": -26.193283081054688,
      "activations/layer2_attention_weight_max": 30.831098556518555,
      "activations/layer2_attention_weight_min": -29.69667625427246,
      "activations/layer3_attention_weight_max": 91.77607727050781,
      "activations/layer3_attention_weight_min": -89.54405212402344,
      "activations/layer4_attention_weight_max": 114.19367980957031,
      "activations/layer4_attention_weight_min": -114.08576965332031,
      "activations/layer5_attention_weight_max": 54.021358489990234,
      "activations/layer5_attention_weight_min": -61.636810302734375,
      "activations/layer6_attention_weight_max": 46.816001892089844,
      "activations/layer6_attention_weight_min": -49.77770233154297,
      "activations/layer7_attention_weight_max": 89.12161254882812,
      "activations/layer7_attention_weight_min": -87.05104064941406,
      "activations/layer8_attention_weight_max": 43.52775955200195,
      "activations/layer8_attention_weight_min": -43.41893005371094,
      "activations/layer9_attention_weight_max": 35.68314743041992,
      "activations/layer9_attention_weight_min": -34.6945686340332,
      "epoch": 9.64,
      "learning_rate": 8.872916666666666e-05,
      "loss": 2.7895,
      "step": 165900
    },
    {
      "activations/layer0_attention_weight_max": 17.064516067504883,
      "activations/layer0_attention_weight_min": -12.393183708190918,
      "activations/layer10_attention_weight_max": 32.71281051635742,
      "activations/layer10_attention_weight_min": -33.000091552734375,
      "activations/layer11_attention_weight_max": 32.7973747253418,
      "activations/layer11_attention_weight_min": -31.40346908569336,
      "activations/layer12_attention_weight_max": 23.779870986938477,
      "activations/layer12_attention_weight_min": -27.055742263793945,
      "activations/layer13_attention_weight_max": 39.36883544921875,
      "activations/layer13_attention_weight_min": -34.73521423339844,
      "activations/layer14_attention_weight_max": 45.034263610839844,
      "activations/layer14_attention_weight_min": -40.89834213256836,
      "activations/layer15_attention_weight_max": 37.07518005371094,
      "activations/layer15_attention_weight_min": -31.27854347229004,
      "activations/layer16_attention_weight_max": 33.365745544433594,
      "activations/layer16_attention_weight_min": -28.803434371948242,
      "activations/layer17_attention_weight_max": 33.30001449584961,
      "activations/layer17_attention_weight_min": -26.030359268188477,
      "activations/layer18_attention_weight_max": 33.89084243774414,
      "activations/layer18_attention_weight_min": -22.715084075927734,
      "activations/layer19_attention_weight_max": 39.7122917175293,
      "activations/layer19_attention_weight_min": -30.710132598876953,
      "activations/layer1_attention_weight_max": 15.990690231323242,
      "activations/layer1_attention_weight_min": -14.265896797180176,
      "activations/layer20_attention_weight_max": 29.049121856689453,
      "activations/layer20_attention_weight_min": -23.616931915283203,
      "activations/layer21_attention_weight_max": 30.966930389404297,
      "activations/layer21_attention_weight_min": -26.25213623046875,
      "activations/layer22_attention_weight_max": 45.83256912231445,
      "activations/layer22_attention_weight_min": -32.21437454223633,
      "activations/layer23_attention_weight_max": 42.65071105957031,
      "activations/layer23_attention_weight_min": -27.002044677734375,
      "activations/layer2_attention_weight_max": 31.054420471191406,
      "activations/layer2_attention_weight_min": -30.34484100341797,
      "activations/layer3_attention_weight_max": 89.99301147460938,
      "activations/layer3_attention_weight_min": -92.80311584472656,
      "activations/layer4_attention_weight_max": 112.19603729248047,
      "activations/layer4_attention_weight_min": -113.32325744628906,
      "activations/layer5_attention_weight_max": 54.33135986328125,
      "activations/layer5_attention_weight_min": -61.59890365600586,
      "activations/layer6_attention_weight_max": 46.62635040283203,
      "activations/layer6_attention_weight_min": -51.378482818603516,
      "activations/layer7_attention_weight_max": 91.32972717285156,
      "activations/layer7_attention_weight_min": -88.18366241455078,
      "activations/layer8_attention_weight_max": 45.17085647583008,
      "activations/layer8_attention_weight_min": -41.91081619262695,
      "activations/layer9_attention_weight_max": 34.039764404296875,
      "activations/layer9_attention_weight_min": -32.69761657714844,
      "epoch": 9.64,
      "learning_rate": 8.871022727272727e-05,
      "loss": 2.8097,
      "step": 165950
    },
    {
      "activations/layer0_attention_weight_max": 14.739019393920898,
      "activations/layer0_attention_weight_min": -12.246347427368164,
      "activations/layer10_attention_weight_max": 33.661170959472656,
      "activations/layer10_attention_weight_min": -29.824539184570312,
      "activations/layer11_attention_weight_max": 36.16769790649414,
      "activations/layer11_attention_weight_min": -32.672996520996094,
      "activations/layer12_attention_weight_max": 22.774337768554688,
      "activations/layer12_attention_weight_min": -25.63941192626953,
      "activations/layer13_attention_weight_max": 36.33176040649414,
      "activations/layer13_attention_weight_min": -34.72050094604492,
      "activations/layer14_attention_weight_max": 46.53195571899414,
      "activations/layer14_attention_weight_min": -40.513004302978516,
      "activations/layer15_attention_weight_max": 33.732421875,
      "activations/layer15_attention_weight_min": -30.801563262939453,
      "activations/layer16_attention_weight_max": 34.14958572387695,
      "activations/layer16_attention_weight_min": -28.126325607299805,
      "activations/layer17_attention_weight_max": 35.30645751953125,
      "activations/layer17_attention_weight_min": -25.716949462890625,
      "activations/layer18_attention_weight_max": 34.428855895996094,
      "activations/layer18_attention_weight_min": -23.185951232910156,
      "activations/layer19_attention_weight_max": 38.718421936035156,
      "activations/layer19_attention_weight_min": -35.22404098510742,
      "activations/layer1_attention_weight_max": 15.750422477722168,
      "activations/layer1_attention_weight_min": -15.013110160827637,
      "activations/layer20_attention_weight_max": 31.20848274230957,
      "activations/layer20_attention_weight_min": -28.0942325592041,
      "activations/layer21_attention_weight_max": 30.317302703857422,
      "activations/layer21_attention_weight_min": -25.28006362915039,
      "activations/layer22_attention_weight_max": 49.35640335083008,
      "activations/layer22_attention_weight_min": -29.87692642211914,
      "activations/layer23_attention_weight_max": 35.41427230834961,
      "activations/layer23_attention_weight_min": -26.72479820251465,
      "activations/layer2_attention_weight_max": 30.71808624267578,
      "activations/layer2_attention_weight_min": -30.883712768554688,
      "activations/layer3_attention_weight_max": 92.18975830078125,
      "activations/layer3_attention_weight_min": -91.70367431640625,
      "activations/layer4_attention_weight_max": 113.3858871459961,
      "activations/layer4_attention_weight_min": -112.34004974365234,
      "activations/layer5_attention_weight_max": 53.41044235229492,
      "activations/layer5_attention_weight_min": -59.82435607910156,
      "activations/layer6_attention_weight_max": 44.02104187011719,
      "activations/layer6_attention_weight_min": -48.45024108886719,
      "activations/layer7_attention_weight_max": 91.82429504394531,
      "activations/layer7_attention_weight_min": -97.46833801269531,
      "activations/layer8_attention_weight_max": 41.39128494262695,
      "activations/layer8_attention_weight_min": -39.32478332519531,
      "activations/layer9_attention_weight_max": 34.10607147216797,
      "activations/layer9_attention_weight_min": -31.225534439086914,
      "epoch": 9.65,
      "learning_rate": 8.869128787878786e-05,
      "loss": 2.7906,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.4898,
      "eval_samples_per_second": 505.782,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.4898,
      "eval_openwebtext_samples_per_second": 505.782,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9142,
      "eval_wikitext_samples_per_second": 238.214,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.5238,
      "eval_lambada_samples_per_second": 511.243,
      "step": 166000
    },
    {
      "activations/layer0_attention_weight_max": 14.974618911743164,
      "activations/layer0_attention_weight_min": -12.91010856628418,
      "activations/layer10_attention_weight_max": 33.103267669677734,
      "activations/layer10_attention_weight_min": -30.128475189208984,
      "activations/layer11_attention_weight_max": 34.34135055541992,
      "activations/layer11_attention_weight_min": -29.443981170654297,
      "activations/layer12_attention_weight_max": 31.316822052001953,
      "activations/layer12_attention_weight_min": -24.63458824157715,
      "activations/layer13_attention_weight_max": 40.59339141845703,
      "activations/layer13_attention_weight_min": -33.344478607177734,
      "activations/layer14_attention_weight_max": 45.9940299987793,
      "activations/layer14_attention_weight_min": -38.0069694519043,
      "activations/layer15_attention_weight_max": 38.25688171386719,
      "activations/layer15_attention_weight_min": -32.788536071777344,
      "activations/layer16_attention_weight_max": 34.02574157714844,
      "activations/layer16_attention_weight_min": -28.746875762939453,
      "activations/layer17_attention_weight_max": 36.087039947509766,
      "activations/layer17_attention_weight_min": -26.284862518310547,
      "activations/layer18_attention_weight_max": 35.992828369140625,
      "activations/layer18_attention_weight_min": -22.657499313354492,
      "activations/layer19_attention_weight_max": 37.74711227416992,
      "activations/layer19_attention_weight_min": -29.611661911010742,
      "activations/layer1_attention_weight_max": 16.450578689575195,
      "activations/layer1_attention_weight_min": -12.92686939239502,
      "activations/layer20_attention_weight_max": 33.15610885620117,
      "activations/layer20_attention_weight_min": -23.738479614257812,
      "activations/layer21_attention_weight_max": 32.25992202758789,
      "activations/layer21_attention_weight_min": -23.18794822692871,
      "activations/layer22_attention_weight_max": 45.286216735839844,
      "activations/layer22_attention_weight_min": -30.345584869384766,
      "activations/layer23_attention_weight_max": 42.081787109375,
      "activations/layer23_attention_weight_min": -26.947200775146484,
      "activations/layer2_attention_weight_max": 30.250333786010742,
      "activations/layer2_attention_weight_min": -27.90186309814453,
      "activations/layer3_attention_weight_max": 85.4688720703125,
      "activations/layer3_attention_weight_min": -83.41480255126953,
      "activations/layer4_attention_weight_max": 110.17037200927734,
      "activations/layer4_attention_weight_min": -104.77298736572266,
      "activations/layer5_attention_weight_max": 51.262916564941406,
      "activations/layer5_attention_weight_min": -59.314918518066406,
      "activations/layer6_attention_weight_max": 43.80821228027344,
      "activations/layer6_attention_weight_min": -45.57817459106445,
      "activations/layer7_attention_weight_max": 85.6034927368164,
      "activations/layer7_attention_weight_min": -80.8341293334961,
      "activations/layer8_attention_weight_max": 39.28474426269531,
      "activations/layer8_attention_weight_min": -37.02559280395508,
      "activations/layer9_attention_weight_max": 32.04206466674805,
      "activations/layer9_attention_weight_min": -32.300628662109375,
      "epoch": 9.65,
      "learning_rate": 8.867234848484848e-05,
      "loss": 2.7904,
      "step": 166050
    },
    {
      "activations/layer0_attention_weight_max": 14.902371406555176,
      "activations/layer0_attention_weight_min": -11.778014183044434,
      "activations/layer10_attention_weight_max": 33.72538757324219,
      "activations/layer10_attention_weight_min": -33.500553131103516,
      "activations/layer11_attention_weight_max": 36.03325653076172,
      "activations/layer11_attention_weight_min": -31.77172088623047,
      "activations/layer12_attention_weight_max": 40.14627456665039,
      "activations/layer12_attention_weight_min": -31.00902557373047,
      "activations/layer13_attention_weight_max": 49.75185775756836,
      "activations/layer13_attention_weight_min": -32.10375213623047,
      "activations/layer14_attention_weight_max": 50.3260612487793,
      "activations/layer14_attention_weight_min": -36.393333435058594,
      "activations/layer15_attention_weight_max": 40.00309753417969,
      "activations/layer15_attention_weight_min": -31.49270248413086,
      "activations/layer16_attention_weight_max": 33.03515625,
      "activations/layer16_attention_weight_min": -27.543563842773438,
      "activations/layer17_attention_weight_max": 33.51262283325195,
      "activations/layer17_attention_weight_min": -24.93313980102539,
      "activations/layer18_attention_weight_max": 31.191646575927734,
      "activations/layer18_attention_weight_min": -22.676576614379883,
      "activations/layer19_attention_weight_max": 33.695796966552734,
      "activations/layer19_attention_weight_min": -32.01593017578125,
      "activations/layer1_attention_weight_max": 16.123456954956055,
      "activations/layer1_attention_weight_min": -14.889979362487793,
      "activations/layer20_attention_weight_max": 30.997440338134766,
      "activations/layer20_attention_weight_min": -24.246477127075195,
      "activations/layer21_attention_weight_max": 32.671016693115234,
      "activations/layer21_attention_weight_min": -25.02748680114746,
      "activations/layer22_attention_weight_max": 47.810150146484375,
      "activations/layer22_attention_weight_min": -29.831392288208008,
      "activations/layer23_attention_weight_max": 33.86994934082031,
      "activations/layer23_attention_weight_min": -24.939895629882812,
      "activations/layer2_attention_weight_max": 32.76820755004883,
      "activations/layer2_attention_weight_min": -31.551115036010742,
      "activations/layer3_attention_weight_max": 91.56815338134766,
      "activations/layer3_attention_weight_min": -93.91352081298828,
      "activations/layer4_attention_weight_max": 111.38531494140625,
      "activations/layer4_attention_weight_min": -115.25135803222656,
      "activations/layer5_attention_weight_max": 53.677303314208984,
      "activations/layer5_attention_weight_min": -61.113311767578125,
      "activations/layer6_attention_weight_max": 44.685001373291016,
      "activations/layer6_attention_weight_min": -49.41413116455078,
      "activations/layer7_attention_weight_max": 94.56351470947266,
      "activations/layer7_attention_weight_min": -93.54291534423828,
      "activations/layer8_attention_weight_max": 40.41294479370117,
      "activations/layer8_attention_weight_min": -41.66544723510742,
      "activations/layer9_attention_weight_max": 35.93674850463867,
      "activations/layer9_attention_weight_min": -35.92073440551758,
      "epoch": 9.65,
      "learning_rate": 8.865340909090909e-05,
      "loss": 2.7802,
      "step": 166100
    },
    {
      "activations/layer0_attention_weight_max": 14.810218811035156,
      "activations/layer0_attention_weight_min": -12.878671646118164,
      "activations/layer10_attention_weight_max": 33.420509338378906,
      "activations/layer10_attention_weight_min": -32.52404022216797,
      "activations/layer11_attention_weight_max": 32.793357849121094,
      "activations/layer11_attention_weight_min": -31.40167236328125,
      "activations/layer12_attention_weight_max": 28.93204116821289,
      "activations/layer12_attention_weight_min": -27.85525131225586,
      "activations/layer13_attention_weight_max": 44.777610778808594,
      "activations/layer13_attention_weight_min": -33.14984893798828,
      "activations/layer14_attention_weight_max": 51.49513626098633,
      "activations/layer14_attention_weight_min": -38.567447662353516,
      "activations/layer15_attention_weight_max": 38.51860427856445,
      "activations/layer15_attention_weight_min": -32.33850860595703,
      "activations/layer16_attention_weight_max": 34.38327407836914,
      "activations/layer16_attention_weight_min": -29.926355361938477,
      "activations/layer17_attention_weight_max": 31.778818130493164,
      "activations/layer17_attention_weight_min": -25.6624755859375,
      "activations/layer18_attention_weight_max": 32.067962646484375,
      "activations/layer18_attention_weight_min": -23.088909149169922,
      "activations/layer19_attention_weight_max": 40.77730941772461,
      "activations/layer19_attention_weight_min": -31.737024307250977,
      "activations/layer1_attention_weight_max": 17.04438591003418,
      "activations/layer1_attention_weight_min": -13.827232360839844,
      "activations/layer20_attention_weight_max": 33.33544158935547,
      "activations/layer20_attention_weight_min": -25.522459030151367,
      "activations/layer21_attention_weight_max": 32.5073127746582,
      "activations/layer21_attention_weight_min": -26.0050048828125,
      "activations/layer22_attention_weight_max": 45.96493911743164,
      "activations/layer22_attention_weight_min": -28.279125213623047,
      "activations/layer23_attention_weight_max": 40.939449310302734,
      "activations/layer23_attention_weight_min": -24.45099639892578,
      "activations/layer2_attention_weight_max": 31.96328353881836,
      "activations/layer2_attention_weight_min": -29.714611053466797,
      "activations/layer3_attention_weight_max": 90.32185363769531,
      "activations/layer3_attention_weight_min": -88.25959014892578,
      "activations/layer4_attention_weight_max": 113.72159576416016,
      "activations/layer4_attention_weight_min": -112.21324920654297,
      "activations/layer5_attention_weight_max": 52.03239059448242,
      "activations/layer5_attention_weight_min": -61.14921569824219,
      "activations/layer6_attention_weight_max": 45.399410247802734,
      "activations/layer6_attention_weight_min": -48.269596099853516,
      "activations/layer7_attention_weight_max": 88.90526580810547,
      "activations/layer7_attention_weight_min": -88.82803344726562,
      "activations/layer8_attention_weight_max": 40.722389221191406,
      "activations/layer8_attention_weight_min": -41.06175231933594,
      "activations/layer9_attention_weight_max": 35.59577178955078,
      "activations/layer9_attention_weight_min": -34.62710189819336,
      "epoch": 9.65,
      "learning_rate": 8.863446969696968e-05,
      "loss": 2.7799,
      "step": 166150
    },
    {
      "activations/layer0_attention_weight_max": 15.551072120666504,
      "activations/layer0_attention_weight_min": -12.02431869506836,
      "activations/layer10_attention_weight_max": 35.201019287109375,
      "activations/layer10_attention_weight_min": -32.57361602783203,
      "activations/layer11_attention_weight_max": 34.483707427978516,
      "activations/layer11_attention_weight_min": -31.690580368041992,
      "activations/layer12_attention_weight_max": 22.58716583251953,
      "activations/layer12_attention_weight_min": -26.033262252807617,
      "activations/layer13_attention_weight_max": 36.63555908203125,
      "activations/layer13_attention_weight_min": -33.70979690551758,
      "activations/layer14_attention_weight_max": 46.4925651550293,
      "activations/layer14_attention_weight_min": -44.47062301635742,
      "activations/layer15_attention_weight_max": 34.329559326171875,
      "activations/layer15_attention_weight_min": -30.665449142456055,
      "activations/layer16_attention_weight_max": 33.90314483642578,
      "activations/layer16_attention_weight_min": -28.294057846069336,
      "activations/layer17_attention_weight_max": 34.29345703125,
      "activations/layer17_attention_weight_min": -25.803220748901367,
      "activations/layer18_attention_weight_max": 30.879756927490234,
      "activations/layer18_attention_weight_min": -23.812862396240234,
      "activations/layer19_attention_weight_max": 36.30765151977539,
      "activations/layer19_attention_weight_min": -30.323837280273438,
      "activations/layer1_attention_weight_max": 16.996747970581055,
      "activations/layer1_attention_weight_min": -13.612865447998047,
      "activations/layer20_attention_weight_max": 29.655101776123047,
      "activations/layer20_attention_weight_min": -23.253496170043945,
      "activations/layer21_attention_weight_max": 28.580820083618164,
      "activations/layer21_attention_weight_min": -21.809114456176758,
      "activations/layer22_attention_weight_max": 46.01308822631836,
      "activations/layer22_attention_weight_min": -29.269840240478516,
      "activations/layer23_attention_weight_max": 35.337486267089844,
      "activations/layer23_attention_weight_min": -24.122495651245117,
      "activations/layer2_attention_weight_max": 30.344459533691406,
      "activations/layer2_attention_weight_min": -30.969350814819336,
      "activations/layer3_attention_weight_max": 88.64221954345703,
      "activations/layer3_attention_weight_min": -90.45819854736328,
      "activations/layer4_attention_weight_max": 117.8490219116211,
      "activations/layer4_attention_weight_min": -113.15675354003906,
      "activations/layer5_attention_weight_max": 55.26519012451172,
      "activations/layer5_attention_weight_min": -59.15168380737305,
      "activations/layer6_attention_weight_max": 46.857505798339844,
      "activations/layer6_attention_weight_min": -49.77381896972656,
      "activations/layer7_attention_weight_max": 88.57966613769531,
      "activations/layer7_attention_weight_min": -89.1471176147461,
      "activations/layer8_attention_weight_max": 41.28377914428711,
      "activations/layer8_attention_weight_min": -40.92536926269531,
      "activations/layer9_attention_weight_max": 35.78132247924805,
      "activations/layer9_attention_weight_min": -35.67929458618164,
      "epoch": 9.66,
      "learning_rate": 8.86155303030303e-05,
      "loss": 2.7952,
      "step": 166200
    },
    {
      "activations/layer0_attention_weight_max": 15.34766674041748,
      "activations/layer0_attention_weight_min": -11.707367897033691,
      "activations/layer10_attention_weight_max": 36.47794723510742,
      "activations/layer10_attention_weight_min": -34.34521484375,
      "activations/layer11_attention_weight_max": 33.999027252197266,
      "activations/layer11_attention_weight_min": -33.97332763671875,
      "activations/layer12_attention_weight_max": 25.22870445251465,
      "activations/layer12_attention_weight_min": -27.107328414916992,
      "activations/layer13_attention_weight_max": 38.07616424560547,
      "activations/layer13_attention_weight_min": -35.379310607910156,
      "activations/layer14_attention_weight_max": 47.725135803222656,
      "activations/layer14_attention_weight_min": -40.13793182373047,
      "activations/layer15_attention_weight_max": 39.52683639526367,
      "activations/layer15_attention_weight_min": -30.650753021240234,
      "activations/layer16_attention_weight_max": 32.211097717285156,
      "activations/layer16_attention_weight_min": -27.985021591186523,
      "activations/layer17_attention_weight_max": 34.317264556884766,
      "activations/layer17_attention_weight_min": -24.688886642456055,
      "activations/layer18_attention_weight_max": 29.725666046142578,
      "activations/layer18_attention_weight_min": -22.927326202392578,
      "activations/layer19_attention_weight_max": 39.55982208251953,
      "activations/layer19_attention_weight_min": -33.54193115234375,
      "activations/layer1_attention_weight_max": 17.0028076171875,
      "activations/layer1_attention_weight_min": -16.260887145996094,
      "activations/layer20_attention_weight_max": 29.12565803527832,
      "activations/layer20_attention_weight_min": -25.55519676208496,
      "activations/layer21_attention_weight_max": 28.230621337890625,
      "activations/layer21_attention_weight_min": -24.556177139282227,
      "activations/layer22_attention_weight_max": 44.07624435424805,
      "activations/layer22_attention_weight_min": -32.21897506713867,
      "activations/layer23_attention_weight_max": 34.225868225097656,
      "activations/layer23_attention_weight_min": -25.18270492553711,
      "activations/layer2_attention_weight_max": 31.545318603515625,
      "activations/layer2_attention_weight_min": -30.719764709472656,
      "activations/layer3_attention_weight_max": 91.31739044189453,
      "activations/layer3_attention_weight_min": -94.28170013427734,
      "activations/layer4_attention_weight_max": 114.34736633300781,
      "activations/layer4_attention_weight_min": -114.59539031982422,
      "activations/layer5_attention_weight_max": 53.416107177734375,
      "activations/layer5_attention_weight_min": -59.25041198730469,
      "activations/layer6_attention_weight_max": 49.80424118041992,
      "activations/layer6_attention_weight_min": -52.1666374206543,
      "activations/layer7_attention_weight_max": 90.93911743164062,
      "activations/layer7_attention_weight_min": -96.60698699951172,
      "activations/layer8_attention_weight_max": 44.43151092529297,
      "activations/layer8_attention_weight_min": -45.26198196411133,
      "activations/layer9_attention_weight_max": 36.967140197753906,
      "activations/layer9_attention_weight_min": -35.075965881347656,
      "epoch": 9.66,
      "learning_rate": 8.859659090909091e-05,
      "loss": 2.7974,
      "step": 166250
    },
    {
      "activations/layer0_attention_weight_max": 14.654341697692871,
      "activations/layer0_attention_weight_min": -12.424051284790039,
      "activations/layer10_attention_weight_max": 32.33875274658203,
      "activations/layer10_attention_weight_min": -31.75609588623047,
      "activations/layer11_attention_weight_max": 34.32451629638672,
      "activations/layer11_attention_weight_min": -29.717777252197266,
      "activations/layer12_attention_weight_max": 26.55748176574707,
      "activations/layer12_attention_weight_min": -24.45549774169922,
      "activations/layer13_attention_weight_max": 42.96314239501953,
      "activations/layer13_attention_weight_min": -34.76568603515625,
      "activations/layer14_attention_weight_max": 47.679725646972656,
      "activations/layer14_attention_weight_min": -41.7813835144043,
      "activations/layer15_attention_weight_max": 38.160186767578125,
      "activations/layer15_attention_weight_min": -31.231992721557617,
      "activations/layer16_attention_weight_max": 32.68680191040039,
      "activations/layer16_attention_weight_min": -28.955514907836914,
      "activations/layer17_attention_weight_max": 36.138450622558594,
      "activations/layer17_attention_weight_min": -24.972774505615234,
      "activations/layer18_attention_weight_max": 34.011192321777344,
      "activations/layer18_attention_weight_min": -22.364456176757812,
      "activations/layer19_attention_weight_max": 37.75402069091797,
      "activations/layer19_attention_weight_min": -31.92447853088379,
      "activations/layer1_attention_weight_max": 15.740812301635742,
      "activations/layer1_attention_weight_min": -13.885014533996582,
      "activations/layer20_attention_weight_max": 33.22018051147461,
      "activations/layer20_attention_weight_min": -26.930498123168945,
      "activations/layer21_attention_weight_max": 31.867046356201172,
      "activations/layer21_attention_weight_min": -24.886459350585938,
      "activations/layer22_attention_weight_max": 51.51958465576172,
      "activations/layer22_attention_weight_min": -29.30886459350586,
      "activations/layer23_attention_weight_max": 42.976463317871094,
      "activations/layer23_attention_weight_min": -23.746061325073242,
      "activations/layer2_attention_weight_max": 30.693099975585938,
      "activations/layer2_attention_weight_min": -30.183334350585938,
      "activations/layer3_attention_weight_max": 88.67684173583984,
      "activations/layer3_attention_weight_min": -91.06155395507812,
      "activations/layer4_attention_weight_max": 110.69258880615234,
      "activations/layer4_attention_weight_min": -111.91346740722656,
      "activations/layer5_attention_weight_max": 54.17805480957031,
      "activations/layer5_attention_weight_min": -61.713226318359375,
      "activations/layer6_attention_weight_max": 45.18974304199219,
      "activations/layer6_attention_weight_min": -50.6179084777832,
      "activations/layer7_attention_weight_max": 86.35211944580078,
      "activations/layer7_attention_weight_min": -86.70800018310547,
      "activations/layer8_attention_weight_max": 41.670555114746094,
      "activations/layer8_attention_weight_min": -39.57618713378906,
      "activations/layer9_attention_weight_max": 33.90831756591797,
      "activations/layer9_attention_weight_min": -31.793323516845703,
      "epoch": 9.66,
      "learning_rate": 8.857765151515151e-05,
      "loss": 2.7966,
      "step": 166300
    },
    {
      "activations/layer0_attention_weight_max": 15.503843307495117,
      "activations/layer0_attention_weight_min": -12.535781860351562,
      "activations/layer10_attention_weight_max": 32.298118591308594,
      "activations/layer10_attention_weight_min": -32.6777229309082,
      "activations/layer11_attention_weight_max": 32.57105255126953,
      "activations/layer11_attention_weight_min": -30.959508895874023,
      "activations/layer12_attention_weight_max": 24.24793815612793,
      "activations/layer12_attention_weight_min": -27.573135375976562,
      "activations/layer13_attention_weight_max": 42.7298469543457,
      "activations/layer13_attention_weight_min": -33.5012321472168,
      "activations/layer14_attention_weight_max": 56.26359176635742,
      "activations/layer14_attention_weight_min": -40.017879486083984,
      "activations/layer15_attention_weight_max": 37.7294807434082,
      "activations/layer15_attention_weight_min": -32.566585540771484,
      "activations/layer16_attention_weight_max": 35.99227523803711,
      "activations/layer16_attention_weight_min": -30.136585235595703,
      "activations/layer17_attention_weight_max": 35.055545806884766,
      "activations/layer17_attention_weight_min": -27.344898223876953,
      "activations/layer18_attention_weight_max": 34.282474517822266,
      "activations/layer18_attention_weight_min": -24.5073299407959,
      "activations/layer19_attention_weight_max": 39.132747650146484,
      "activations/layer19_attention_weight_min": -31.595932006835938,
      "activations/layer1_attention_weight_max": 15.51968002319336,
      "activations/layer1_attention_weight_min": -14.08894157409668,
      "activations/layer20_attention_weight_max": 32.63058853149414,
      "activations/layer20_attention_weight_min": -25.0161075592041,
      "activations/layer21_attention_weight_max": 29.91051483154297,
      "activations/layer21_attention_weight_min": -24.872289657592773,
      "activations/layer22_attention_weight_max": 53.090614318847656,
      "activations/layer22_attention_weight_min": -32.21370315551758,
      "activations/layer23_attention_weight_max": 38.34795379638672,
      "activations/layer23_attention_weight_min": -25.801219940185547,
      "activations/layer2_attention_weight_max": 30.401779174804688,
      "activations/layer2_attention_weight_min": -30.236652374267578,
      "activations/layer3_attention_weight_max": 93.59986877441406,
      "activations/layer3_attention_weight_min": -92.15546417236328,
      "activations/layer4_attention_weight_max": 117.83033752441406,
      "activations/layer4_attention_weight_min": -113.3222885131836,
      "activations/layer5_attention_weight_max": 53.010658264160156,
      "activations/layer5_attention_weight_min": -64.6146011352539,
      "activations/layer6_attention_weight_max": 47.81441116333008,
      "activations/layer6_attention_weight_min": -49.054378509521484,
      "activations/layer7_attention_weight_max": 89.00269317626953,
      "activations/layer7_attention_weight_min": -89.38021087646484,
      "activations/layer8_attention_weight_max": 40.97587203979492,
      "activations/layer8_attention_weight_min": -40.95965576171875,
      "activations/layer9_attention_weight_max": 33.31637954711914,
      "activations/layer9_attention_weight_min": -33.0734977722168,
      "epoch": 9.67,
      "learning_rate": 8.855871212121211e-05,
      "loss": 2.7964,
      "step": 166350
    },
    {
      "activations/layer0_attention_weight_max": 16.013742446899414,
      "activations/layer0_attention_weight_min": -12.39836597442627,
      "activations/layer10_attention_weight_max": 32.79063415527344,
      "activations/layer10_attention_weight_min": -32.76472854614258,
      "activations/layer11_attention_weight_max": 32.89362335205078,
      "activations/layer11_attention_weight_min": -30.86456298828125,
      "activations/layer12_attention_weight_max": 23.882553100585938,
      "activations/layer12_attention_weight_min": -23.358610153198242,
      "activations/layer13_attention_weight_max": 38.21851348876953,
      "activations/layer13_attention_weight_min": -33.6348876953125,
      "activations/layer14_attention_weight_max": 45.212921142578125,
      "activations/layer14_attention_weight_min": -39.46711349487305,
      "activations/layer15_attention_weight_max": 35.25617218017578,
      "activations/layer15_attention_weight_min": -30.09617805480957,
      "activations/layer16_attention_weight_max": 30.68699073791504,
      "activations/layer16_attention_weight_min": -27.622114181518555,
      "activations/layer17_attention_weight_max": 29.928619384765625,
      "activations/layer17_attention_weight_min": -25.033967971801758,
      "activations/layer18_attention_weight_max": 30.30596351623535,
      "activations/layer18_attention_weight_min": -23.514345169067383,
      "activations/layer19_attention_weight_max": 30.7661190032959,
      "activations/layer19_attention_weight_min": -28.745553970336914,
      "activations/layer1_attention_weight_max": 16.030502319335938,
      "activations/layer1_attention_weight_min": -14.23015308380127,
      "activations/layer20_attention_weight_max": 28.156761169433594,
      "activations/layer20_attention_weight_min": -24.118694305419922,
      "activations/layer21_attention_weight_max": 26.760601043701172,
      "activations/layer21_attention_weight_min": -22.51262855529785,
      "activations/layer22_attention_weight_max": 45.736392974853516,
      "activations/layer22_attention_weight_min": -31.039318084716797,
      "activations/layer23_attention_weight_max": 36.37250518798828,
      "activations/layer23_attention_weight_min": -24.13359832763672,
      "activations/layer2_attention_weight_max": 29.284709930419922,
      "activations/layer2_attention_weight_min": -30.619586944580078,
      "activations/layer3_attention_weight_max": 92.42493438720703,
      "activations/layer3_attention_weight_min": -94.04644775390625,
      "activations/layer4_attention_weight_max": 114.80362701416016,
      "activations/layer4_attention_weight_min": -117.24578094482422,
      "activations/layer5_attention_weight_max": 53.669822692871094,
      "activations/layer5_attention_weight_min": -63.41895294189453,
      "activations/layer6_attention_weight_max": 45.815635681152344,
      "activations/layer6_attention_weight_min": -51.70908737182617,
      "activations/layer7_attention_weight_max": 94.77352905273438,
      "activations/layer7_attention_weight_min": -90.35942840576172,
      "activations/layer8_attention_weight_max": 41.18057632446289,
      "activations/layer8_attention_weight_min": -41.992801666259766,
      "activations/layer9_attention_weight_max": 34.732635498046875,
      "activations/layer9_attention_weight_min": -34.818214416503906,
      "epoch": 9.67,
      "learning_rate": 8.853977272727271e-05,
      "loss": 2.7883,
      "step": 166400
    },
    {
      "activations/layer0_attention_weight_max": 15.380585670471191,
      "activations/layer0_attention_weight_min": -11.998828887939453,
      "activations/layer10_attention_weight_max": 33.52390670776367,
      "activations/layer10_attention_weight_min": -32.60300064086914,
      "activations/layer11_attention_weight_max": 31.444538116455078,
      "activations/layer11_attention_weight_min": -31.230693817138672,
      "activations/layer12_attention_weight_max": 37.71455001831055,
      "activations/layer12_attention_weight_min": -30.549816131591797,
      "activations/layer13_attention_weight_max": 50.17438507080078,
      "activations/layer13_attention_weight_min": -32.95252990722656,
      "activations/layer14_attention_weight_max": 47.91110610961914,
      "activations/layer14_attention_weight_min": -38.15341567993164,
      "activations/layer15_attention_weight_max": 37.86515808105469,
      "activations/layer15_attention_weight_min": -30.833810806274414,
      "activations/layer16_attention_weight_max": 33.064002990722656,
      "activations/layer16_attention_weight_min": -26.165857315063477,
      "activations/layer17_attention_weight_max": 35.34544372558594,
      "activations/layer17_attention_weight_min": -26.431711196899414,
      "activations/layer18_attention_weight_max": 31.968563079833984,
      "activations/layer18_attention_weight_min": -24.24028205871582,
      "activations/layer19_attention_weight_max": 36.789066314697266,
      "activations/layer19_attention_weight_min": -31.5617618560791,
      "activations/layer1_attention_weight_max": 16.42948341369629,
      "activations/layer1_attention_weight_min": -13.49170207977295,
      "activations/layer20_attention_weight_max": 32.587337493896484,
      "activations/layer20_attention_weight_min": -24.83189582824707,
      "activations/layer21_attention_weight_max": 30.1370906829834,
      "activations/layer21_attention_weight_min": -25.3349666595459,
      "activations/layer22_attention_weight_max": 46.29104232788086,
      "activations/layer22_attention_weight_min": -31.444461822509766,
      "activations/layer23_attention_weight_max": 37.309715270996094,
      "activations/layer23_attention_weight_min": -24.742774963378906,
      "activations/layer2_attention_weight_max": 31.29377555847168,
      "activations/layer2_attention_weight_min": -30.216123580932617,
      "activations/layer3_attention_weight_max": 92.836181640625,
      "activations/layer3_attention_weight_min": -93.36155700683594,
      "activations/layer4_attention_weight_max": 115.47640228271484,
      "activations/layer4_attention_weight_min": -116.52718353271484,
      "activations/layer5_attention_weight_max": 55.75565719604492,
      "activations/layer5_attention_weight_min": -64.94843292236328,
      "activations/layer6_attention_weight_max": 46.684818267822266,
      "activations/layer6_attention_weight_min": -49.8442268371582,
      "activations/layer7_attention_weight_max": 88.59650421142578,
      "activations/layer7_attention_weight_min": -97.72473907470703,
      "activations/layer8_attention_weight_max": 41.56468963623047,
      "activations/layer8_attention_weight_min": -43.218414306640625,
      "activations/layer9_attention_weight_max": 35.29890823364258,
      "activations/layer9_attention_weight_min": -34.64972686767578,
      "epoch": 9.67,
      "learning_rate": 8.852083333333333e-05,
      "loss": 2.7849,
      "step": 166450
    },
    {
      "activations/layer0_attention_weight_max": 15.310888290405273,
      "activations/layer0_attention_weight_min": -12.647062301635742,
      "activations/layer10_attention_weight_max": 32.978515625,
      "activations/layer10_attention_weight_min": -32.570438385009766,
      "activations/layer11_attention_weight_max": 32.61738204956055,
      "activations/layer11_attention_weight_min": -30.673110961914062,
      "activations/layer12_attention_weight_max": 22.990705490112305,
      "activations/layer12_attention_weight_min": -24.951736450195312,
      "activations/layer13_attention_weight_max": 37.708168029785156,
      "activations/layer13_attention_weight_min": -33.136104583740234,
      "activations/layer14_attention_weight_max": 49.706478118896484,
      "activations/layer14_attention_weight_min": -39.26228332519531,
      "activations/layer15_attention_weight_max": 36.104190826416016,
      "activations/layer15_attention_weight_min": -31.518531799316406,
      "activations/layer16_attention_weight_max": 36.65182876586914,
      "activations/layer16_attention_weight_min": -28.173099517822266,
      "activations/layer17_attention_weight_max": 33.1043815612793,
      "activations/layer17_attention_weight_min": -26.232688903808594,
      "activations/layer18_attention_weight_max": 32.0347900390625,
      "activations/layer18_attention_weight_min": -23.218507766723633,
      "activations/layer19_attention_weight_max": 38.42484664916992,
      "activations/layer19_attention_weight_min": -31.0240421295166,
      "activations/layer1_attention_weight_max": 16.39650535583496,
      "activations/layer1_attention_weight_min": -15.445867538452148,
      "activations/layer20_attention_weight_max": 34.59561538696289,
      "activations/layer20_attention_weight_min": -26.371498107910156,
      "activations/layer21_attention_weight_max": 32.53373718261719,
      "activations/layer21_attention_weight_min": -24.655193328857422,
      "activations/layer22_attention_weight_max": 46.30009841918945,
      "activations/layer22_attention_weight_min": -29.950321197509766,
      "activations/layer23_attention_weight_max": 41.17529296875,
      "activations/layer23_attention_weight_min": -24.15357780456543,
      "activations/layer2_attention_weight_max": 33.60558319091797,
      "activations/layer2_attention_weight_min": -31.527563095092773,
      "activations/layer3_attention_weight_max": 98.37089538574219,
      "activations/layer3_attention_weight_min": -100.42548370361328,
      "activations/layer4_attention_weight_max": 124.93055725097656,
      "activations/layer4_attention_weight_min": -113.7733154296875,
      "activations/layer5_attention_weight_max": 54.304195404052734,
      "activations/layer5_attention_weight_min": -59.22136688232422,
      "activations/layer6_attention_weight_max": 47.830509185791016,
      "activations/layer6_attention_weight_min": -51.40641784667969,
      "activations/layer7_attention_weight_max": 89.841064453125,
      "activations/layer7_attention_weight_min": -89.5970230102539,
      "activations/layer8_attention_weight_max": 39.41151428222656,
      "activations/layer8_attention_weight_min": -38.73292922973633,
      "activations/layer9_attention_weight_max": 35.36533737182617,
      "activations/layer9_attention_weight_min": -32.179405212402344,
      "epoch": 9.67,
      "learning_rate": 8.850189393939394e-05,
      "loss": 2.7786,
      "step": 166500
    },
    {
      "activations/layer0_attention_weight_max": 14.929874420166016,
      "activations/layer0_attention_weight_min": -13.164380073547363,
      "activations/layer10_attention_weight_max": 33.964698791503906,
      "activations/layer10_attention_weight_min": -32.07073211669922,
      "activations/layer11_attention_weight_max": 38.271087646484375,
      "activations/layer11_attention_weight_min": -32.39784240722656,
      "activations/layer12_attention_weight_max": 26.234437942504883,
      "activations/layer12_attention_weight_min": -27.196365356445312,
      "activations/layer13_attention_weight_max": 40.620452880859375,
      "activations/layer13_attention_weight_min": -32.26386260986328,
      "activations/layer14_attention_weight_max": 48.82910919189453,
      "activations/layer14_attention_weight_min": -37.73068618774414,
      "activations/layer15_attention_weight_max": 37.83444595336914,
      "activations/layer15_attention_weight_min": -32.96519470214844,
      "activations/layer16_attention_weight_max": 33.635066986083984,
      "activations/layer16_attention_weight_min": -28.45515251159668,
      "activations/layer17_attention_weight_max": 33.13300704956055,
      "activations/layer17_attention_weight_min": -23.91118049621582,
      "activations/layer18_attention_weight_max": 32.659263610839844,
      "activations/layer18_attention_weight_min": -22.488758087158203,
      "activations/layer19_attention_weight_max": 38.24213790893555,
      "activations/layer19_attention_weight_min": -29.152942657470703,
      "activations/layer1_attention_weight_max": 17.14075469970703,
      "activations/layer1_attention_weight_min": -14.383352279663086,
      "activations/layer20_attention_weight_max": 28.572959899902344,
      "activations/layer20_attention_weight_min": -24.704078674316406,
      "activations/layer21_attention_weight_max": 30.122270584106445,
      "activations/layer21_attention_weight_min": -22.50057029724121,
      "activations/layer22_attention_weight_max": 44.33623123168945,
      "activations/layer22_attention_weight_min": -29.107669830322266,
      "activations/layer23_attention_weight_max": 37.82010269165039,
      "activations/layer23_attention_weight_min": -22.69324493408203,
      "activations/layer2_attention_weight_max": 30.510292053222656,
      "activations/layer2_attention_weight_min": -31.554847717285156,
      "activations/layer3_attention_weight_max": 89.24757385253906,
      "activations/layer3_attention_weight_min": -92.49533081054688,
      "activations/layer4_attention_weight_max": 113.59661865234375,
      "activations/layer4_attention_weight_min": -116.33905029296875,
      "activations/layer5_attention_weight_max": 58.95316696166992,
      "activations/layer5_attention_weight_min": -58.59588623046875,
      "activations/layer6_attention_weight_max": 45.18617248535156,
      "activations/layer6_attention_weight_min": -47.1566276550293,
      "activations/layer7_attention_weight_max": 87.27738952636719,
      "activations/layer7_attention_weight_min": -84.78691864013672,
      "activations/layer8_attention_weight_max": 40.67940902709961,
      "activations/layer8_attention_weight_min": -39.043601989746094,
      "activations/layer9_attention_weight_max": 35.32719039916992,
      "activations/layer9_attention_weight_min": -33.453277587890625,
      "epoch": 9.68,
      "learning_rate": 8.848295454545453e-05,
      "loss": 2.783,
      "step": 166550
    },
    {
      "activations/layer0_attention_weight_max": 14.313735961914062,
      "activations/layer0_attention_weight_min": -11.766351699829102,
      "activations/layer10_attention_weight_max": 34.20466613769531,
      "activations/layer10_attention_weight_min": -34.084197998046875,
      "activations/layer11_attention_weight_max": 38.23707962036133,
      "activations/layer11_attention_weight_min": -33.01331329345703,
      "activations/layer12_attention_weight_max": 30.353492736816406,
      "activations/layer12_attention_weight_min": -26.0246639251709,
      "activations/layer13_attention_weight_max": 41.50261306762695,
      "activations/layer13_attention_weight_min": -35.020164489746094,
      "activations/layer14_attention_weight_max": 49.45615768432617,
      "activations/layer14_attention_weight_min": -39.67323303222656,
      "activations/layer15_attention_weight_max": 36.06599044799805,
      "activations/layer15_attention_weight_min": -31.222105026245117,
      "activations/layer16_attention_weight_max": 29.68008041381836,
      "activations/layer16_attention_weight_min": -28.892101287841797,
      "activations/layer17_attention_weight_max": 35.56578063964844,
      "activations/layer17_attention_weight_min": -25.47995376586914,
      "activations/layer18_attention_weight_max": 30.77557373046875,
      "activations/layer18_attention_weight_min": -22.814258575439453,
      "activations/layer19_attention_weight_max": 32.00031280517578,
      "activations/layer19_attention_weight_min": -34.21981430053711,
      "activations/layer1_attention_weight_max": 16.78513526916504,
      "activations/layer1_attention_weight_min": -14.891151428222656,
      "activations/layer20_attention_weight_max": 27.921890258789062,
      "activations/layer20_attention_weight_min": -27.207733154296875,
      "activations/layer21_attention_weight_max": 31.390439987182617,
      "activations/layer21_attention_weight_min": -23.23367691040039,
      "activations/layer22_attention_weight_max": 46.16191864013672,
      "activations/layer22_attention_weight_min": -32.93951416015625,
      "activations/layer23_attention_weight_max": 35.70478439331055,
      "activations/layer23_attention_weight_min": -22.63448143005371,
      "activations/layer2_attention_weight_max": 31.721960067749023,
      "activations/layer2_attention_weight_min": -30.901220321655273,
      "activations/layer3_attention_weight_max": 92.09955596923828,
      "activations/layer3_attention_weight_min": -94.87198638916016,
      "activations/layer4_attention_weight_max": 117.42242431640625,
      "activations/layer4_attention_weight_min": -118.4020767211914,
      "activations/layer5_attention_weight_max": 52.19912338256836,
      "activations/layer5_attention_weight_min": -59.75080871582031,
      "activations/layer6_attention_weight_max": 47.30256652832031,
      "activations/layer6_attention_weight_min": -49.13667678833008,
      "activations/layer7_attention_weight_max": 88.54215240478516,
      "activations/layer7_attention_weight_min": -89.47570037841797,
      "activations/layer8_attention_weight_max": 42.50648880004883,
      "activations/layer8_attention_weight_min": -46.02473449707031,
      "activations/layer9_attention_weight_max": 35.32554244995117,
      "activations/layer9_attention_weight_min": -36.977664947509766,
      "epoch": 9.68,
      "learning_rate": 8.846401515151515e-05,
      "loss": 2.7857,
      "step": 166600
    },
    {
      "activations/layer0_attention_weight_max": 14.945085525512695,
      "activations/layer0_attention_weight_min": -11.933625221252441,
      "activations/layer10_attention_weight_max": 36.6098518371582,
      "activations/layer10_attention_weight_min": -36.22481155395508,
      "activations/layer11_attention_weight_max": 34.834407806396484,
      "activations/layer11_attention_weight_min": -35.057044982910156,
      "activations/layer12_attention_weight_max": 29.97748374938965,
      "activations/layer12_attention_weight_min": -32.233394622802734,
      "activations/layer13_attention_weight_max": 41.89579391479492,
      "activations/layer13_attention_weight_min": -34.6518669128418,
      "activations/layer14_attention_weight_max": 48.70912170410156,
      "activations/layer14_attention_weight_min": -36.358028411865234,
      "activations/layer15_attention_weight_max": 45.11389923095703,
      "activations/layer15_attention_weight_min": -31.17397117614746,
      "activations/layer16_attention_weight_max": 33.70387268066406,
      "activations/layer16_attention_weight_min": -26.171411514282227,
      "activations/layer17_attention_weight_max": 34.163856506347656,
      "activations/layer17_attention_weight_min": -25.45754051208496,
      "activations/layer18_attention_weight_max": 31.562536239624023,
      "activations/layer18_attention_weight_min": -23.50762367248535,
      "activations/layer19_attention_weight_max": 34.96369552612305,
      "activations/layer19_attention_weight_min": -30.60267448425293,
      "activations/layer1_attention_weight_max": 17.368553161621094,
      "activations/layer1_attention_weight_min": -12.373876571655273,
      "activations/layer20_attention_weight_max": 29.853199005126953,
      "activations/layer20_attention_weight_min": -25.551170349121094,
      "activations/layer21_attention_weight_max": 30.035043716430664,
      "activations/layer21_attention_weight_min": -22.821969985961914,
      "activations/layer22_attention_weight_max": 52.28946304321289,
      "activations/layer22_attention_weight_min": -28.684934616088867,
      "activations/layer23_attention_weight_max": 36.2337760925293,
      "activations/layer23_attention_weight_min": -22.054981231689453,
      "activations/layer2_attention_weight_max": 31.426925659179688,
      "activations/layer2_attention_weight_min": -31.602787017822266,
      "activations/layer3_attention_weight_max": 97.31979370117188,
      "activations/layer3_attention_weight_min": -97.2988052368164,
      "activations/layer4_attention_weight_max": 118.23291015625,
      "activations/layer4_attention_weight_min": -117.75709533691406,
      "activations/layer5_attention_weight_max": 55.202064514160156,
      "activations/layer5_attention_weight_min": -60.846778869628906,
      "activations/layer6_attention_weight_max": 47.719112396240234,
      "activations/layer6_attention_weight_min": -49.176876068115234,
      "activations/layer7_attention_weight_max": 94.20277404785156,
      "activations/layer7_attention_weight_min": -91.39556884765625,
      "activations/layer8_attention_weight_max": 40.93489456176758,
      "activations/layer8_attention_weight_min": -44.59663772583008,
      "activations/layer9_attention_weight_max": 35.693565368652344,
      "activations/layer9_attention_weight_min": -36.83834457397461,
      "epoch": 9.68,
      "learning_rate": 8.844507575757576e-05,
      "loss": 2.7743,
      "step": 166650
    },
    {
      "activations/layer0_attention_weight_max": 16.028182983398438,
      "activations/layer0_attention_weight_min": -11.668168067932129,
      "activations/layer10_attention_weight_max": 32.16115188598633,
      "activations/layer10_attention_weight_min": -33.56842041015625,
      "activations/layer11_attention_weight_max": 32.59190368652344,
      "activations/layer11_attention_weight_min": -33.74098205566406,
      "activations/layer12_attention_weight_max": 22.998971939086914,
      "activations/layer12_attention_weight_min": -26.578327178955078,
      "activations/layer13_attention_weight_max": 37.6646614074707,
      "activations/layer13_attention_weight_min": -33.32682800292969,
      "activations/layer14_attention_weight_max": 51.93838119506836,
      "activations/layer14_attention_weight_min": -41.226356506347656,
      "activations/layer15_attention_weight_max": 35.26823043823242,
      "activations/layer15_attention_weight_min": -31.382719039916992,
      "activations/layer16_attention_weight_max": 34.64965057373047,
      "activations/layer16_attention_weight_min": -28.472122192382812,
      "activations/layer17_attention_weight_max": 31.06920051574707,
      "activations/layer17_attention_weight_min": -24.925086975097656,
      "activations/layer18_attention_weight_max": 30.347736358642578,
      "activations/layer18_attention_weight_min": -21.452497482299805,
      "activations/layer19_attention_weight_max": 35.35734558105469,
      "activations/layer19_attention_weight_min": -30.18001365661621,
      "activations/layer1_attention_weight_max": 16.3729248046875,
      "activations/layer1_attention_weight_min": -16.03997802734375,
      "activations/layer20_attention_weight_max": 28.482189178466797,
      "activations/layer20_attention_weight_min": -22.826072692871094,
      "activations/layer21_attention_weight_max": 29.005538940429688,
      "activations/layer21_attention_weight_min": -21.535465240478516,
      "activations/layer22_attention_weight_max": 42.934478759765625,
      "activations/layer22_attention_weight_min": -29.184934616088867,
      "activations/layer23_attention_weight_max": 34.21585464477539,
      "activations/layer23_attention_weight_min": -25.224319458007812,
      "activations/layer2_attention_weight_max": 31.5317440032959,
      "activations/layer2_attention_weight_min": -31.973331451416016,
      "activations/layer3_attention_weight_max": 90.64290618896484,
      "activations/layer3_attention_weight_min": -93.86750793457031,
      "activations/layer4_attention_weight_max": 112.08097076416016,
      "activations/layer4_attention_weight_min": -111.71876525878906,
      "activations/layer5_attention_weight_max": 54.82564926147461,
      "activations/layer5_attention_weight_min": -62.55419921875,
      "activations/layer6_attention_weight_max": 47.76548385620117,
      "activations/layer6_attention_weight_min": -49.57128143310547,
      "activations/layer7_attention_weight_max": 98.79447937011719,
      "activations/layer7_attention_weight_min": -90.05093383789062,
      "activations/layer8_attention_weight_max": 40.47053909301758,
      "activations/layer8_attention_weight_min": -42.945133209228516,
      "activations/layer9_attention_weight_max": 32.251136779785156,
      "activations/layer9_attention_weight_min": -34.04099655151367,
      "epoch": 9.69,
      "learning_rate": 8.842613636363635e-05,
      "loss": 2.7845,
      "step": 166700
    },
    {
      "activations/layer0_attention_weight_max": 15.55940055847168,
      "activations/layer0_attention_weight_min": -12.131183624267578,
      "activations/layer10_attention_weight_max": 35.923316955566406,
      "activations/layer10_attention_weight_min": -33.02682113647461,
      "activations/layer11_attention_weight_max": 35.39875411987305,
      "activations/layer11_attention_weight_min": -33.79730224609375,
      "activations/layer12_attention_weight_max": 26.65836524963379,
      "activations/layer12_attention_weight_min": -24.63656234741211,
      "activations/layer13_attention_weight_max": 41.29472732543945,
      "activations/layer13_attention_weight_min": -36.362220764160156,
      "activations/layer14_attention_weight_max": 57.784088134765625,
      "activations/layer14_attention_weight_min": -43.521942138671875,
      "activations/layer15_attention_weight_max": 39.72289276123047,
      "activations/layer15_attention_weight_min": -31.313377380371094,
      "activations/layer16_attention_weight_max": 33.25233459472656,
      "activations/layer16_attention_weight_min": -29.943824768066406,
      "activations/layer17_attention_weight_max": 35.996055603027344,
      "activations/layer17_attention_weight_min": -27.252660751342773,
      "activations/layer18_attention_weight_max": 33.00333786010742,
      "activations/layer18_attention_weight_min": -23.09170913696289,
      "activations/layer19_attention_weight_max": 42.849853515625,
      "activations/layer19_attention_weight_min": -30.989383697509766,
      "activations/layer1_attention_weight_max": 16.988784790039062,
      "activations/layer1_attention_weight_min": -14.557310104370117,
      "activations/layer20_attention_weight_max": 35.252098083496094,
      "activations/layer20_attention_weight_min": -24.769775390625,
      "activations/layer21_attention_weight_max": 36.199981689453125,
      "activations/layer21_attention_weight_min": -23.548349380493164,
      "activations/layer22_attention_weight_max": 55.96282958984375,
      "activations/layer22_attention_weight_min": -28.91910743713379,
      "activations/layer23_attention_weight_max": 44.23122787475586,
      "activations/layer23_attention_weight_min": -24.062294006347656,
      "activations/layer2_attention_weight_max": 30.87753677368164,
      "activations/layer2_attention_weight_min": -31.787128448486328,
      "activations/layer3_attention_weight_max": 96.83452606201172,
      "activations/layer3_attention_weight_min": -96.53324127197266,
      "activations/layer4_attention_weight_max": 118.2755126953125,
      "activations/layer4_attention_weight_min": -116.3395004272461,
      "activations/layer5_attention_weight_max": 57.02678680419922,
      "activations/layer5_attention_weight_min": -61.79471969604492,
      "activations/layer6_attention_weight_max": 47.517086029052734,
      "activations/layer6_attention_weight_min": -50.06087875366211,
      "activations/layer7_attention_weight_max": 89.52040100097656,
      "activations/layer7_attention_weight_min": -92.09061431884766,
      "activations/layer8_attention_weight_max": 40.90819549560547,
      "activations/layer8_attention_weight_min": -41.48064041137695,
      "activations/layer9_attention_weight_max": 36.59520721435547,
      "activations/layer9_attention_weight_min": -37.5894660949707,
      "epoch": 9.69,
      "learning_rate": 8.840719696969696e-05,
      "loss": 2.7906,
      "step": 166750
    },
    {
      "activations/layer0_attention_weight_max": 14.870158195495605,
      "activations/layer0_attention_weight_min": -12.641373634338379,
      "activations/layer10_attention_weight_max": 33.25225830078125,
      "activations/layer10_attention_weight_min": -33.56071853637695,
      "activations/layer11_attention_weight_max": 32.2042236328125,
      "activations/layer11_attention_weight_min": -31.565649032592773,
      "activations/layer12_attention_weight_max": 23.66102409362793,
      "activations/layer12_attention_weight_min": -27.531614303588867,
      "activations/layer13_attention_weight_max": 39.1607551574707,
      "activations/layer13_attention_weight_min": -33.80473709106445,
      "activations/layer14_attention_weight_max": 40.56877136230469,
      "activations/layer14_attention_weight_min": -35.40654754638672,
      "activations/layer15_attention_weight_max": 34.463623046875,
      "activations/layer15_attention_weight_min": -30.638608932495117,
      "activations/layer16_attention_weight_max": 31.920360565185547,
      "activations/layer16_attention_weight_min": -29.872413635253906,
      "activations/layer17_attention_weight_max": 32.88954162597656,
      "activations/layer17_attention_weight_min": -26.228731155395508,
      "activations/layer18_attention_weight_max": 32.61753845214844,
      "activations/layer18_attention_weight_min": -22.831056594848633,
      "activations/layer19_attention_weight_max": 37.30451202392578,
      "activations/layer19_attention_weight_min": -33.06816482543945,
      "activations/layer1_attention_weight_max": 17.164587020874023,
      "activations/layer1_attention_weight_min": -13.829236030578613,
      "activations/layer20_attention_weight_max": 33.82634353637695,
      "activations/layer20_attention_weight_min": -28.334321975708008,
      "activations/layer21_attention_weight_max": 30.22038459777832,
      "activations/layer21_attention_weight_min": -26.204931259155273,
      "activations/layer22_attention_weight_max": 48.308555603027344,
      "activations/layer22_attention_weight_min": -28.724132537841797,
      "activations/layer23_attention_weight_max": 37.373558044433594,
      "activations/layer23_attention_weight_min": -24.117652893066406,
      "activations/layer2_attention_weight_max": 32.396915435791016,
      "activations/layer2_attention_weight_min": -32.347076416015625,
      "activations/layer3_attention_weight_max": 99.88790130615234,
      "activations/layer3_attention_weight_min": -95.4351577758789,
      "activations/layer4_attention_weight_max": 124.16545867919922,
      "activations/layer4_attention_weight_min": -117.79100799560547,
      "activations/layer5_attention_weight_max": 56.806068420410156,
      "activations/layer5_attention_weight_min": -62.596473693847656,
      "activations/layer6_attention_weight_max": 50.90568542480469,
      "activations/layer6_attention_weight_min": -50.053062438964844,
      "activations/layer7_attention_weight_max": 86.8743667602539,
      "activations/layer7_attention_weight_min": -90.4580307006836,
      "activations/layer8_attention_weight_max": 41.51597595214844,
      "activations/layer8_attention_weight_min": -43.096614837646484,
      "activations/layer9_attention_weight_max": 33.35616683959961,
      "activations/layer9_attention_weight_min": -33.81802749633789,
      "epoch": 9.69,
      "learning_rate": 8.838825757575758e-05,
      "loss": 2.7937,
      "step": 166800
    },
    {
      "activations/layer0_attention_weight_max": 16.058412551879883,
      "activations/layer0_attention_weight_min": -12.933195114135742,
      "activations/layer10_attention_weight_max": 33.22108840942383,
      "activations/layer10_attention_weight_min": -35.720909118652344,
      "activations/layer11_attention_weight_max": 33.63994598388672,
      "activations/layer11_attention_weight_min": -35.26575469970703,
      "activations/layer12_attention_weight_max": 25.33098793029785,
      "activations/layer12_attention_weight_min": -27.056961059570312,
      "activations/layer13_attention_weight_max": 38.78704071044922,
      "activations/layer13_attention_weight_min": -35.71527862548828,
      "activations/layer14_attention_weight_max": 47.19540786743164,
      "activations/layer14_attention_weight_min": -44.61174011230469,
      "activations/layer15_attention_weight_max": 36.0217170715332,
      "activations/layer15_attention_weight_min": -33.70361328125,
      "activations/layer16_attention_weight_max": 33.800376892089844,
      "activations/layer16_attention_weight_min": -29.23423957824707,
      "activations/layer17_attention_weight_max": 32.25994873046875,
      "activations/layer17_attention_weight_min": -26.410202026367188,
      "activations/layer18_attention_weight_max": 31.641586303710938,
      "activations/layer18_attention_weight_min": -22.734386444091797,
      "activations/layer19_attention_weight_max": 33.86600112915039,
      "activations/layer19_attention_weight_min": -33.06748580932617,
      "activations/layer1_attention_weight_max": 16.644243240356445,
      "activations/layer1_attention_weight_min": -15.502638816833496,
      "activations/layer20_attention_weight_max": 31.935312271118164,
      "activations/layer20_attention_weight_min": -25.93907928466797,
      "activations/layer21_attention_weight_max": 29.97197151184082,
      "activations/layer21_attention_weight_min": -24.499298095703125,
      "activations/layer22_attention_weight_max": 47.04852294921875,
      "activations/layer22_attention_weight_min": -30.90435028076172,
      "activations/layer23_attention_weight_max": 34.22788619995117,
      "activations/layer23_attention_weight_min": -25.10040283203125,
      "activations/layer2_attention_weight_max": 30.67327117919922,
      "activations/layer2_attention_weight_min": -29.967662811279297,
      "activations/layer3_attention_weight_max": 89.69940185546875,
      "activations/layer3_attention_weight_min": -92.3995361328125,
      "activations/layer4_attention_weight_max": 114.541748046875,
      "activations/layer4_attention_weight_min": -116.64763641357422,
      "activations/layer5_attention_weight_max": 54.090816497802734,
      "activations/layer5_attention_weight_min": -61.661705017089844,
      "activations/layer6_attention_weight_max": 46.46540069580078,
      "activations/layer6_attention_weight_min": -52.473670959472656,
      "activations/layer7_attention_weight_max": 87.08727264404297,
      "activations/layer7_attention_weight_min": -89.15182495117188,
      "activations/layer8_attention_weight_max": 40.93254089355469,
      "activations/layer8_attention_weight_min": -43.363380432128906,
      "activations/layer9_attention_weight_max": 34.9900016784668,
      "activations/layer9_attention_weight_min": -33.75881576538086,
      "epoch": 9.69,
      "learning_rate": 8.836931818181817e-05,
      "loss": 2.7957,
      "step": 166850
    },
    {
      "activations/layer0_attention_weight_max": 16.04547882080078,
      "activations/layer0_attention_weight_min": -12.814044952392578,
      "activations/layer10_attention_weight_max": 32.07367706298828,
      "activations/layer10_attention_weight_min": -35.90802764892578,
      "activations/layer11_attention_weight_max": 34.34221267700195,
      "activations/layer11_attention_weight_min": -33.894775390625,
      "activations/layer12_attention_weight_max": 27.424726486206055,
      "activations/layer12_attention_weight_min": -26.16029167175293,
      "activations/layer13_attention_weight_max": 41.621089935302734,
      "activations/layer13_attention_weight_min": -34.96842956542969,
      "activations/layer14_attention_weight_max": 55.54591369628906,
      "activations/layer14_attention_weight_min": -44.51197052001953,
      "activations/layer15_attention_weight_max": 41.018585205078125,
      "activations/layer15_attention_weight_min": -33.238285064697266,
      "activations/layer16_attention_weight_max": 33.6717529296875,
      "activations/layer16_attention_weight_min": -28.93165397644043,
      "activations/layer17_attention_weight_max": 34.55058288574219,
      "activations/layer17_attention_weight_min": -26.00501251220703,
      "activations/layer18_attention_weight_max": 31.20827865600586,
      "activations/layer18_attention_weight_min": -22.18337059020996,
      "activations/layer19_attention_weight_max": 37.68196487426758,
      "activations/layer19_attention_weight_min": -31.890180587768555,
      "activations/layer1_attention_weight_max": 16.231111526489258,
      "activations/layer1_attention_weight_min": -17.118722915649414,
      "activations/layer20_attention_weight_max": 30.473249435424805,
      "activations/layer20_attention_weight_min": -23.832443237304688,
      "activations/layer21_attention_weight_max": 27.153045654296875,
      "activations/layer21_attention_weight_min": -22.918020248413086,
      "activations/layer22_attention_weight_max": 50.577754974365234,
      "activations/layer22_attention_weight_min": -28.903459548950195,
      "activations/layer23_attention_weight_max": 38.231361389160156,
      "activations/layer23_attention_weight_min": -26.204345703125,
      "activations/layer2_attention_weight_max": 31.18563461303711,
      "activations/layer2_attention_weight_min": -29.55437469482422,
      "activations/layer3_attention_weight_max": 89.20173645019531,
      "activations/layer3_attention_weight_min": -86.50215148925781,
      "activations/layer4_attention_weight_max": 114.79341125488281,
      "activations/layer4_attention_weight_min": -108.24320220947266,
      "activations/layer5_attention_weight_max": 50.739601135253906,
      "activations/layer5_attention_weight_min": -59.2939567565918,
      "activations/layer6_attention_weight_max": 46.804771423339844,
      "activations/layer6_attention_weight_min": -48.97454071044922,
      "activations/layer7_attention_weight_max": 91.55452728271484,
      "activations/layer7_attention_weight_min": -99.63665771484375,
      "activations/layer8_attention_weight_max": 41.79391098022461,
      "activations/layer8_attention_weight_min": -46.36421203613281,
      "activations/layer9_attention_weight_max": 34.774314880371094,
      "activations/layer9_attention_weight_min": -36.7806510925293,
      "epoch": 9.7,
      "learning_rate": 8.835075757575756e-05,
      "loss": 2.7881,
      "step": 166900
    },
    {
      "activations/layer0_attention_weight_max": 14.574264526367188,
      "activations/layer0_attention_weight_min": -12.587401390075684,
      "activations/layer10_attention_weight_max": 38.922576904296875,
      "activations/layer10_attention_weight_min": -34.851348876953125,
      "activations/layer11_attention_weight_max": 41.468345642089844,
      "activations/layer11_attention_weight_min": -36.4261589050293,
      "activations/layer12_attention_weight_max": 39.32822799682617,
      "activations/layer12_attention_weight_min": -27.43766212463379,
      "activations/layer13_attention_weight_max": 50.94953155517578,
      "activations/layer13_attention_weight_min": -39.249629974365234,
      "activations/layer14_attention_weight_max": 75.2542953491211,
      "activations/layer14_attention_weight_min": -47.557830810546875,
      "activations/layer15_attention_weight_max": 66.47343444824219,
      "activations/layer15_attention_weight_min": -37.7891845703125,
      "activations/layer16_attention_weight_max": 40.43174743652344,
      "activations/layer16_attention_weight_min": -28.903921127319336,
      "activations/layer17_attention_weight_max": 39.926124572753906,
      "activations/layer17_attention_weight_min": -30.71592903137207,
      "activations/layer18_attention_weight_max": 38.129981994628906,
      "activations/layer18_attention_weight_min": -27.261022567749023,
      "activations/layer19_attention_weight_max": 57.606319427490234,
      "activations/layer19_attention_weight_min": -36.5556755065918,
      "activations/layer1_attention_weight_max": 16.08011245727539,
      "activations/layer1_attention_weight_min": -14.590901374816895,
      "activations/layer20_attention_weight_max": 45.13407516479492,
      "activations/layer20_attention_weight_min": -26.953508377075195,
      "activations/layer21_attention_weight_max": 40.21489715576172,
      "activations/layer21_attention_weight_min": -24.583633422851562,
      "activations/layer22_attention_weight_max": 70.94058227539062,
      "activations/layer22_attention_weight_min": -31.95292091369629,
      "activations/layer23_attention_weight_max": 48.27424240112305,
      "activations/layer23_attention_weight_min": -21.155874252319336,
      "activations/layer2_attention_weight_max": 32.52545166015625,
      "activations/layer2_attention_weight_min": -29.88903045654297,
      "activations/layer3_attention_weight_max": 100.62308502197266,
      "activations/layer3_attention_weight_min": -92.72882080078125,
      "activations/layer4_attention_weight_max": 123.46327209472656,
      "activations/layer4_attention_weight_min": -111.40057373046875,
      "activations/layer5_attention_weight_max": 56.80296325683594,
      "activations/layer5_attention_weight_min": -58.64594650268555,
      "activations/layer6_attention_weight_max": 51.78569793701172,
      "activations/layer6_attention_weight_min": -48.399112701416016,
      "activations/layer7_attention_weight_max": 102.99565887451172,
      "activations/layer7_attention_weight_min": -88.38844299316406,
      "activations/layer8_attention_weight_max": 45.438507080078125,
      "activations/layer8_attention_weight_min": -40.93117141723633,
      "activations/layer9_attention_weight_max": 37.78562927246094,
      "activations/layer9_attention_weight_min": -34.60845184326172,
      "epoch": 9.7,
      "learning_rate": 8.833181818181817e-05,
      "loss": 2.8048,
      "step": 166950
    },
    {
      "activations/layer0_attention_weight_max": 16.2763614654541,
      "activations/layer0_attention_weight_min": -12.337417602539062,
      "activations/layer10_attention_weight_max": 32.93985366821289,
      "activations/layer10_attention_weight_min": -32.432899475097656,
      "activations/layer11_attention_weight_max": 32.52012634277344,
      "activations/layer11_attention_weight_min": -32.2661018371582,
      "activations/layer12_attention_weight_max": 25.403486251831055,
      "activations/layer12_attention_weight_min": -25.111793518066406,
      "activations/layer13_attention_weight_max": 40.89338684082031,
      "activations/layer13_attention_weight_min": -34.26955032348633,
      "activations/layer14_attention_weight_max": 43.644142150878906,
      "activations/layer14_attention_weight_min": -40.971702575683594,
      "activations/layer15_attention_weight_max": 34.489749908447266,
      "activations/layer15_attention_weight_min": -30.453584671020508,
      "activations/layer16_attention_weight_max": 30.128108978271484,
      "activations/layer16_attention_weight_min": -27.320396423339844,
      "activations/layer17_attention_weight_max": 33.27842330932617,
      "activations/layer17_attention_weight_min": -25.809795379638672,
      "activations/layer18_attention_weight_max": 29.852270126342773,
      "activations/layer18_attention_weight_min": -21.95370864868164,
      "activations/layer19_attention_weight_max": 36.35845947265625,
      "activations/layer19_attention_weight_min": -30.20928382873535,
      "activations/layer1_attention_weight_max": 16.237686157226562,
      "activations/layer1_attention_weight_min": -14.248541831970215,
      "activations/layer20_attention_weight_max": 28.171152114868164,
      "activations/layer20_attention_weight_min": -23.83716583251953,
      "activations/layer21_attention_weight_max": 30.622833251953125,
      "activations/layer21_attention_weight_min": -23.114582061767578,
      "activations/layer22_attention_weight_max": 45.29899215698242,
      "activations/layer22_attention_weight_min": -29.10028839111328,
      "activations/layer23_attention_weight_max": 32.60780715942383,
      "activations/layer23_attention_weight_min": -29.947582244873047,
      "activations/layer2_attention_weight_max": 29.413341522216797,
      "activations/layer2_attention_weight_min": -30.373275756835938,
      "activations/layer3_attention_weight_max": 93.13665008544922,
      "activations/layer3_attention_weight_min": -91.83578491210938,
      "activations/layer4_attention_weight_max": 116.31849670410156,
      "activations/layer4_attention_weight_min": -113.363525390625,
      "activations/layer5_attention_weight_max": 54.65484619140625,
      "activations/layer5_attention_weight_min": -61.474239349365234,
      "activations/layer6_attention_weight_max": 47.58981704711914,
      "activations/layer6_attention_weight_min": -49.392189025878906,
      "activations/layer7_attention_weight_max": 91.97095489501953,
      "activations/layer7_attention_weight_min": -96.5113296508789,
      "activations/layer8_attention_weight_max": 40.71430587768555,
      "activations/layer8_attention_weight_min": -40.77180480957031,
      "activations/layer9_attention_weight_max": 34.457786560058594,
      "activations/layer9_attention_weight_min": -34.305477142333984,
      "epoch": 9.7,
      "learning_rate": 8.831287878787879e-05,
      "loss": 2.7939,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.4635,
      "eval_samples_per_second": 507.353,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.4635,
      "eval_openwebtext_samples_per_second": 507.353,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 1.9287,
      "eval_wikitext_samples_per_second": 236.426,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_lambada_loss": 2.556640625,
      "eval_lambada_ppl": 12.892433965519283,
      "eval_lambada_runtime": 9.5213,
      "eval_lambada_samples_per_second": 511.382,
      "step": 167000
    },
    {
      "activations/layer0_attention_weight_max": 15.490555763244629,
      "activations/layer0_attention_weight_min": -12.326340675354004,
      "activations/layer10_attention_weight_max": 33.449867248535156,
      "activations/layer10_attention_weight_min": -32.73566436767578,
      "activations/layer11_attention_weight_max": 31.647886276245117,
      "activations/layer11_attention_weight_min": -31.1536865234375,
      "activations/layer12_attention_weight_max": 25.54856300354004,
      "activations/layer12_attention_weight_min": -24.173669815063477,
      "activations/layer13_attention_weight_max": 36.43155288696289,
      "activations/layer13_attention_weight_min": -34.62431335449219,
      "activations/layer14_attention_weight_max": 42.63191604614258,
      "activations/layer14_attention_weight_min": -36.45232009887695,
      "activations/layer15_attention_weight_max": 35.193599700927734,
      "activations/layer15_attention_weight_min": -31.90863037109375,
      "activations/layer16_attention_weight_max": 33.041996002197266,
      "activations/layer16_attention_weight_min": -28.614044189453125,
      "activations/layer17_attention_weight_max": 31.14804458618164,
      "activations/layer17_attention_weight_min": -24.4073543548584,
      "activations/layer18_attention_weight_max": 33.90467834472656,
      "activations/layer18_attention_weight_min": -23.53044319152832,
      "activations/layer19_attention_weight_max": 35.47246170043945,
      "activations/layer19_attention_weight_min": -32.81401062011719,
      "activations/layer1_attention_weight_max": 16.592653274536133,
      "activations/layer1_attention_weight_min": -12.053447723388672,
      "activations/layer20_attention_weight_max": 28.403480529785156,
      "activations/layer20_attention_weight_min": -24.1396541595459,
      "activations/layer21_attention_weight_max": 27.742412567138672,
      "activations/layer21_attention_weight_min": -22.57754898071289,
      "activations/layer22_attention_weight_max": 50.338523864746094,
      "activations/layer22_attention_weight_min": -31.73594093322754,
      "activations/layer23_attention_weight_max": 35.80814743041992,
      "activations/layer23_attention_weight_min": -25.836585998535156,
      "activations/layer2_attention_weight_max": 31.930639266967773,
      "activations/layer2_attention_weight_min": -30.775531768798828,
      "activations/layer3_attention_weight_max": 90.97683715820312,
      "activations/layer3_attention_weight_min": -89.6640396118164,
      "activations/layer4_attention_weight_max": 117.57561492919922,
      "activations/layer4_attention_weight_min": -107.1064682006836,
      "activations/layer5_attention_weight_max": 55.76478576660156,
      "activations/layer5_attention_weight_min": -62.71007537841797,
      "activations/layer6_attention_weight_max": 47.91645431518555,
      "activations/layer6_attention_weight_min": -48.09391784667969,
      "activations/layer7_attention_weight_max": 86.61318969726562,
      "activations/layer7_attention_weight_min": -93.49835205078125,
      "activations/layer8_attention_weight_max": 42.1951789855957,
      "activations/layer8_attention_weight_min": -40.470062255859375,
      "activations/layer9_attention_weight_max": 34.9920539855957,
      "activations/layer9_attention_weight_min": -33.51728820800781,
      "epoch": 9.71,
      "learning_rate": 8.829393939393939e-05,
      "loss": 2.7907,
      "step": 167050
    },
    {
      "activations/layer0_attention_weight_max": 14.86937427520752,
      "activations/layer0_attention_weight_min": -11.834311485290527,
      "activations/layer10_attention_weight_max": 36.39581298828125,
      "activations/layer10_attention_weight_min": -32.031070709228516,
      "activations/layer11_attention_weight_max": 37.81416320800781,
      "activations/layer11_attention_weight_min": -32.076194763183594,
      "activations/layer12_attention_weight_max": 35.752708435058594,
      "activations/layer12_attention_weight_min": -24.64735221862793,
      "activations/layer13_attention_weight_max": 53.6294059753418,
      "activations/layer13_attention_weight_min": -35.37413787841797,
      "activations/layer14_attention_weight_max": 60.960731506347656,
      "activations/layer14_attention_weight_min": -42.62621307373047,
      "activations/layer15_attention_weight_max": 47.770484924316406,
      "activations/layer15_attention_weight_min": -33.69990539550781,
      "activations/layer16_attention_weight_max": 40.36928939819336,
      "activations/layer16_attention_weight_min": -29.145959854125977,
      "activations/layer17_attention_weight_max": 39.22300338745117,
      "activations/layer17_attention_weight_min": -28.187976837158203,
      "activations/layer18_attention_weight_max": 34.227203369140625,
      "activations/layer18_attention_weight_min": -24.43089485168457,
      "activations/layer19_attention_weight_max": 44.997047424316406,
      "activations/layer19_attention_weight_min": -30.95903968811035,
      "activations/layer1_attention_weight_max": 16.453876495361328,
      "activations/layer1_attention_weight_min": -13.314472198486328,
      "activations/layer20_attention_weight_max": 40.12158203125,
      "activations/layer20_attention_weight_min": -25.981605529785156,
      "activations/layer21_attention_weight_max": 41.90401840209961,
      "activations/layer21_attention_weight_min": -23.2423038482666,
      "activations/layer22_attention_weight_max": 54.60710906982422,
      "activations/layer22_attention_weight_min": -28.704471588134766,
      "activations/layer23_attention_weight_max": 39.202369689941406,
      "activations/layer23_attention_weight_min": -23.45662498474121,
      "activations/layer2_attention_weight_max": 28.238800048828125,
      "activations/layer2_attention_weight_min": -28.609764099121094,
      "activations/layer3_attention_weight_max": 83.96037292480469,
      "activations/layer3_attention_weight_min": -84.55252075195312,
      "activations/layer4_attention_weight_max": 108.5868911743164,
      "activations/layer4_attention_weight_min": -107.57466888427734,
      "activations/layer5_attention_weight_max": 51.388328552246094,
      "activations/layer5_attention_weight_min": -62.5747184753418,
      "activations/layer6_attention_weight_max": 45.02619171142578,
      "activations/layer6_attention_weight_min": -47.5366096496582,
      "activations/layer7_attention_weight_max": 86.77902221679688,
      "activations/layer7_attention_weight_min": -90.81824493408203,
      "activations/layer8_attention_weight_max": 41.955780029296875,
      "activations/layer8_attention_weight_min": -39.95536804199219,
      "activations/layer9_attention_weight_max": 37.801719665527344,
      "activations/layer9_attention_weight_min": -33.02629470825195,
      "epoch": 9.71,
      "learning_rate": 8.827499999999999e-05,
      "loss": 2.8114,
      "step": 167100
    },
    {
      "activations/layer0_attention_weight_max": 15.535454750061035,
      "activations/layer0_attention_weight_min": -12.932307243347168,
      "activations/layer10_attention_weight_max": 38.951881408691406,
      "activations/layer10_attention_weight_min": -35.50777053833008,
      "activations/layer11_attention_weight_max": 40.1202392578125,
      "activations/layer11_attention_weight_min": -34.62312316894531,
      "activations/layer12_attention_weight_max": 29.39812469482422,
      "activations/layer12_attention_weight_min": -28.737289428710938,
      "activations/layer13_attention_weight_max": 42.11244583129883,
      "activations/layer13_attention_weight_min": -34.510250091552734,
      "activations/layer14_attention_weight_max": 50.530269622802734,
      "activations/layer14_attention_weight_min": -41.30027389526367,
      "activations/layer15_attention_weight_max": 43.596736907958984,
      "activations/layer15_attention_weight_min": -31.631179809570312,
      "activations/layer16_attention_weight_max": 34.61762237548828,
      "activations/layer16_attention_weight_min": -29.97881507873535,
      "activations/layer17_attention_weight_max": 33.13367462158203,
      "activations/layer17_attention_weight_min": -26.57395362854004,
      "activations/layer18_attention_weight_max": 32.55347442626953,
      "activations/layer18_attention_weight_min": -22.80455207824707,
      "activations/layer19_attention_weight_max": 37.52864456176758,
      "activations/layer19_attention_weight_min": -30.18251609802246,
      "activations/layer1_attention_weight_max": 17.2231502532959,
      "activations/layer1_attention_weight_min": -13.985028266906738,
      "activations/layer20_attention_weight_max": 31.302928924560547,
      "activations/layer20_attention_weight_min": -24.25665283203125,
      "activations/layer21_attention_weight_max": 28.802133560180664,
      "activations/layer21_attention_weight_min": -23.994766235351562,
      "activations/layer22_attention_weight_max": 47.72684097290039,
      "activations/layer22_attention_weight_min": -27.33904457092285,
      "activations/layer23_attention_weight_max": 38.85905456542969,
      "activations/layer23_attention_weight_min": -22.976110458374023,
      "activations/layer2_attention_weight_max": 31.630599975585938,
      "activations/layer2_attention_weight_min": -30.430633544921875,
      "activations/layer3_attention_weight_max": 98.13756561279297,
      "activations/layer3_attention_weight_min": -97.75878143310547,
      "activations/layer4_attention_weight_max": 124.0648193359375,
      "activations/layer4_attention_weight_min": -114.51631927490234,
      "activations/layer5_attention_weight_max": 61.73101043701172,
      "activations/layer5_attention_weight_min": -58.56696319580078,
      "activations/layer6_attention_weight_max": 49.625770568847656,
      "activations/layer6_attention_weight_min": -51.17374801635742,
      "activations/layer7_attention_weight_max": 101.21338653564453,
      "activations/layer7_attention_weight_min": -94.67474365234375,
      "activations/layer8_attention_weight_max": 46.95655059814453,
      "activations/layer8_attention_weight_min": -46.026546478271484,
      "activations/layer9_attention_weight_max": 39.554901123046875,
      "activations/layer9_attention_weight_min": -36.90877914428711,
      "epoch": 9.71,
      "learning_rate": 8.825606060606059e-05,
      "loss": 2.7888,
      "step": 167150
    },
    {
      "activations/layer0_attention_weight_max": 16.149089813232422,
      "activations/layer0_attention_weight_min": -13.155692100524902,
      "activations/layer10_attention_weight_max": 32.64060592651367,
      "activations/layer10_attention_weight_min": -32.57156753540039,
      "activations/layer11_attention_weight_max": 32.088706970214844,
      "activations/layer11_attention_weight_min": -29.801403045654297,
      "activations/layer12_attention_weight_max": 25.0654354095459,
      "activations/layer12_attention_weight_min": -23.446372985839844,
      "activations/layer13_attention_weight_max": 45.69734191894531,
      "activations/layer13_attention_weight_min": -32.739288330078125,
      "activations/layer14_attention_weight_max": 52.791160583496094,
      "activations/layer14_attention_weight_min": -39.05418014526367,
      "activations/layer15_attention_weight_max": 37.986488342285156,
      "activations/layer15_attention_weight_min": -31.109073638916016,
      "activations/layer16_attention_weight_max": 36.231048583984375,
      "activations/layer16_attention_weight_min": -25.871536254882812,
      "activations/layer17_attention_weight_max": 34.00614929199219,
      "activations/layer17_attention_weight_min": -25.678466796875,
      "activations/layer18_attention_weight_max": 32.858612060546875,
      "activations/layer18_attention_weight_min": -22.956945419311523,
      "activations/layer19_attention_weight_max": 40.22184371948242,
      "activations/layer19_attention_weight_min": -31.207611083984375,
      "activations/layer1_attention_weight_max": 17.999189376831055,
      "activations/layer1_attention_weight_min": -14.382705688476562,
      "activations/layer20_attention_weight_max": 32.61870574951172,
      "activations/layer20_attention_weight_min": -27.095008850097656,
      "activations/layer21_attention_weight_max": 32.57306671142578,
      "activations/layer21_attention_weight_min": -24.381189346313477,
      "activations/layer22_attention_weight_max": 53.266082763671875,
      "activations/layer22_attention_weight_min": -30.038681030273438,
      "activations/layer23_attention_weight_max": 37.82442855834961,
      "activations/layer23_attention_weight_min": -26.00194549560547,
      "activations/layer2_attention_weight_max": 30.859479904174805,
      "activations/layer2_attention_weight_min": -29.614713668823242,
      "activations/layer3_attention_weight_max": 86.79734802246094,
      "activations/layer3_attention_weight_min": -85.63726043701172,
      "activations/layer4_attention_weight_max": 104.93863677978516,
      "activations/layer4_attention_weight_min": -102.87690734863281,
      "activations/layer5_attention_weight_max": 50.9869384765625,
      "activations/layer5_attention_weight_min": -57.66731262207031,
      "activations/layer6_attention_weight_max": 46.43946075439453,
      "activations/layer6_attention_weight_min": -47.45100784301758,
      "activations/layer7_attention_weight_max": 91.43094635009766,
      "activations/layer7_attention_weight_min": -87.82427215576172,
      "activations/layer8_attention_weight_max": 39.19832992553711,
      "activations/layer8_attention_weight_min": -39.632083892822266,
      "activations/layer9_attention_weight_max": 34.62421798706055,
      "activations/layer9_attention_weight_min": -31.646947860717773,
      "epoch": 9.72,
      "learning_rate": 8.823712121212121e-05,
      "loss": 2.8023,
      "step": 167200
    },
    {
      "activations/layer0_attention_weight_max": 15.265153884887695,
      "activations/layer0_attention_weight_min": -12.790858268737793,
      "activations/layer10_attention_weight_max": 35.162750244140625,
      "activations/layer10_attention_weight_min": -35.13711166381836,
      "activations/layer11_attention_weight_max": 35.744789123535156,
      "activations/layer11_attention_weight_min": -32.88612365722656,
      "activations/layer12_attention_weight_max": 27.4576473236084,
      "activations/layer12_attention_weight_min": -25.543649673461914,
      "activations/layer13_attention_weight_max": 44.74534225463867,
      "activations/layer13_attention_weight_min": -36.93975830078125,
      "activations/layer14_attention_weight_max": 50.69822692871094,
      "activations/layer14_attention_weight_min": -43.5277214050293,
      "activations/layer15_attention_weight_max": 36.613399505615234,
      "activations/layer15_attention_weight_min": -33.42633819580078,
      "activations/layer16_attention_weight_max": 32.41737365722656,
      "activations/layer16_attention_weight_min": -29.363439559936523,
      "activations/layer17_attention_weight_max": 30.905914306640625,
      "activations/layer17_attention_weight_min": -26.818737030029297,
      "activations/layer18_attention_weight_max": 33.29508590698242,
      "activations/layer18_attention_weight_min": -22.936786651611328,
      "activations/layer19_attention_weight_max": 34.555511474609375,
      "activations/layer19_attention_weight_min": -32.02248764038086,
      "activations/layer1_attention_weight_max": 16.14659881591797,
      "activations/layer1_attention_weight_min": -15.758590698242188,
      "activations/layer20_attention_weight_max": 31.256263732910156,
      "activations/layer20_attention_weight_min": -25.331499099731445,
      "activations/layer21_attention_weight_max": 27.940921783447266,
      "activations/layer21_attention_weight_min": -25.510385513305664,
      "activations/layer22_attention_weight_max": 40.86066436767578,
      "activations/layer22_attention_weight_min": -32.43486022949219,
      "activations/layer23_attention_weight_max": 35.29573059082031,
      "activations/layer23_attention_weight_min": -22.925031661987305,
      "activations/layer2_attention_weight_max": 31.149635314941406,
      "activations/layer2_attention_weight_min": -31.385116577148438,
      "activations/layer3_attention_weight_max": 91.03292846679688,
      "activations/layer3_attention_weight_min": -92.08948516845703,
      "activations/layer4_attention_weight_max": 112.9760513305664,
      "activations/layer4_attention_weight_min": -110.4315414428711,
      "activations/layer5_attention_weight_max": 52.77216720581055,
      "activations/layer5_attention_weight_min": -58.58494186401367,
      "activations/layer6_attention_weight_max": 47.257755279541016,
      "activations/layer6_attention_weight_min": -46.84605026245117,
      "activations/layer7_attention_weight_max": 95.53150177001953,
      "activations/layer7_attention_weight_min": -93.1919937133789,
      "activations/layer8_attention_weight_max": 42.357276916503906,
      "activations/layer8_attention_weight_min": -41.2427864074707,
      "activations/layer9_attention_weight_max": 37.62687301635742,
      "activations/layer9_attention_weight_min": -35.4256706237793,
      "epoch": 9.72,
      "learning_rate": 8.821818181818181e-05,
      "loss": 2.804,
      "step": 167250
    },
    {
      "activations/layer0_attention_weight_max": 15.570215225219727,
      "activations/layer0_attention_weight_min": -12.393819808959961,
      "activations/layer10_attention_weight_max": 31.638113021850586,
      "activations/layer10_attention_weight_min": -32.42879104614258,
      "activations/layer11_attention_weight_max": 31.95503044128418,
      "activations/layer11_attention_weight_min": -30.82546615600586,
      "activations/layer12_attention_weight_max": 23.433868408203125,
      "activations/layer12_attention_weight_min": -25.443214416503906,
      "activations/layer13_attention_weight_max": 39.436012268066406,
      "activations/layer13_attention_weight_min": -31.33732795715332,
      "activations/layer14_attention_weight_max": 44.417747497558594,
      "activations/layer14_attention_weight_min": -35.782989501953125,
      "activations/layer15_attention_weight_max": 36.05282974243164,
      "activations/layer15_attention_weight_min": -29.555673599243164,
      "activations/layer16_attention_weight_max": 32.574527740478516,
      "activations/layer16_attention_weight_min": -28.254806518554688,
      "activations/layer17_attention_weight_max": 32.28269577026367,
      "activations/layer17_attention_weight_min": -25.882572174072266,
      "activations/layer18_attention_weight_max": 31.021461486816406,
      "activations/layer18_attention_weight_min": -22.81148910522461,
      "activations/layer19_attention_weight_max": 34.248966217041016,
      "activations/layer19_attention_weight_min": -32.06490707397461,
      "activations/layer1_attention_weight_max": 16.697202682495117,
      "activations/layer1_attention_weight_min": -16.056278228759766,
      "activations/layer20_attention_weight_max": 29.642711639404297,
      "activations/layer20_attention_weight_min": -26.816287994384766,
      "activations/layer21_attention_weight_max": 30.88468360900879,
      "activations/layer21_attention_weight_min": -24.866907119750977,
      "activations/layer22_attention_weight_max": 44.66364288330078,
      "activations/layer22_attention_weight_min": -31.324087142944336,
      "activations/layer23_attention_weight_max": 34.88336944580078,
      "activations/layer23_attention_weight_min": -25.307331085205078,
      "activations/layer2_attention_weight_max": 30.557476043701172,
      "activations/layer2_attention_weight_min": -30.035751342773438,
      "activations/layer3_attention_weight_max": 91.05036163330078,
      "activations/layer3_attention_weight_min": -90.15270233154297,
      "activations/layer4_attention_weight_max": 111.80667877197266,
      "activations/layer4_attention_weight_min": -109.42009735107422,
      "activations/layer5_attention_weight_max": 55.68828582763672,
      "activations/layer5_attention_weight_min": -60.104488372802734,
      "activations/layer6_attention_weight_max": 46.52056121826172,
      "activations/layer6_attention_weight_min": -48.14057922363281,
      "activations/layer7_attention_weight_max": 83.05878448486328,
      "activations/layer7_attention_weight_min": -84.73808288574219,
      "activations/layer8_attention_weight_max": 38.557369232177734,
      "activations/layer8_attention_weight_min": -40.84590530395508,
      "activations/layer9_attention_weight_max": 33.75726318359375,
      "activations/layer9_attention_weight_min": -32.40668869018555,
      "epoch": 9.72,
      "learning_rate": 8.819924242424241e-05,
      "loss": 2.8074,
      "step": 167300
    },
    {
      "activations/layer0_attention_weight_max": 16.00101089477539,
      "activations/layer0_attention_weight_min": -11.966477394104004,
      "activations/layer10_attention_weight_max": 35.30815505981445,
      "activations/layer10_attention_weight_min": -33.38016128540039,
      "activations/layer11_attention_weight_max": 33.0095329284668,
      "activations/layer11_attention_weight_min": -32.036094665527344,
      "activations/layer12_attention_weight_max": 36.724159240722656,
      "activations/layer12_attention_weight_min": -25.009775161743164,
      "activations/layer13_attention_weight_max": 44.068153381347656,
      "activations/layer13_attention_weight_min": -34.37172317504883,
      "activations/layer14_attention_weight_max": 46.70689010620117,
      "activations/layer14_attention_weight_min": -39.180721282958984,
      "activations/layer15_attention_weight_max": 40.08536148071289,
      "activations/layer15_attention_weight_min": -31.151615142822266,
      "activations/layer16_attention_weight_max": 32.85700225830078,
      "activations/layer16_attention_weight_min": -28.93672752380371,
      "activations/layer17_attention_weight_max": 31.41900634765625,
      "activations/layer17_attention_weight_min": -25.66171646118164,
      "activations/layer18_attention_weight_max": 32.20616912841797,
      "activations/layer18_attention_weight_min": -22.514171600341797,
      "activations/layer19_attention_weight_max": 39.80004119873047,
      "activations/layer19_attention_weight_min": -34.68846893310547,
      "activations/layer1_attention_weight_max": 16.27431297302246,
      "activations/layer1_attention_weight_min": -13.132596015930176,
      "activations/layer20_attention_weight_max": 31.330738067626953,
      "activations/layer20_attention_weight_min": -23.670793533325195,
      "activations/layer21_attention_weight_max": 30.27202033996582,
      "activations/layer21_attention_weight_min": -23.26828956604004,
      "activations/layer22_attention_weight_max": 43.592376708984375,
      "activations/layer22_attention_weight_min": -28.255340576171875,
      "activations/layer23_attention_weight_max": 33.577972412109375,
      "activations/layer23_attention_weight_min": -24.929203033447266,
      "activations/layer2_attention_weight_max": 31.791728973388672,
      "activations/layer2_attention_weight_min": -30.897809982299805,
      "activations/layer3_attention_weight_max": 93.5392074584961,
      "activations/layer3_attention_weight_min": -92.8168716430664,
      "activations/layer4_attention_weight_max": 112.2436752319336,
      "activations/layer4_attention_weight_min": -109.8729476928711,
      "activations/layer5_attention_weight_max": 52.01620101928711,
      "activations/layer5_attention_weight_min": -59.817596435546875,
      "activations/layer6_attention_weight_max": 46.369728088378906,
      "activations/layer6_attention_weight_min": -48.88996887207031,
      "activations/layer7_attention_weight_max": 93.00491333007812,
      "activations/layer7_attention_weight_min": -87.21533966064453,
      "activations/layer8_attention_weight_max": 41.5926399230957,
      "activations/layer8_attention_weight_min": -43.104061126708984,
      "activations/layer9_attention_weight_max": 34.1535758972168,
      "activations/layer9_attention_weight_min": -33.6150016784668,
      "epoch": 9.72,
      "learning_rate": 8.818030303030303e-05,
      "loss": 2.8124,
      "step": 167350
    },
    {
      "activations/layer0_attention_weight_max": 15.733298301696777,
      "activations/layer0_attention_weight_min": -12.510262489318848,
      "activations/layer10_attention_weight_max": 31.506065368652344,
      "activations/layer10_attention_weight_min": -32.823394775390625,
      "activations/layer11_attention_weight_max": 30.216571807861328,
      "activations/layer11_attention_weight_min": -33.306968688964844,
      "activations/layer12_attention_weight_max": 22.098955154418945,
      "activations/layer12_attention_weight_min": -27.685630798339844,
      "activations/layer13_attention_weight_max": 34.325191497802734,
      "activations/layer13_attention_weight_min": -32.997066497802734,
      "activations/layer14_attention_weight_max": 38.63321304321289,
      "activations/layer14_attention_weight_min": -35.55870819091797,
      "activations/layer15_attention_weight_max": 33.15712356567383,
      "activations/layer15_attention_weight_min": -30.257097244262695,
      "activations/layer16_attention_weight_max": 29.9416561126709,
      "activations/layer16_attention_weight_min": -25.445276260375977,
      "activations/layer17_attention_weight_max": 32.94291687011719,
      "activations/layer17_attention_weight_min": -25.10877227783203,
      "activations/layer18_attention_weight_max": 30.9296932220459,
      "activations/layer18_attention_weight_min": -23.83264923095703,
      "activations/layer19_attention_weight_max": 35.519020080566406,
      "activations/layer19_attention_weight_min": -28.892440795898438,
      "activations/layer1_attention_weight_max": 16.272876739501953,
      "activations/layer1_attention_weight_min": -14.31341552734375,
      "activations/layer20_attention_weight_max": 26.712480545043945,
      "activations/layer20_attention_weight_min": -23.918689727783203,
      "activations/layer21_attention_weight_max": 27.13570213317871,
      "activations/layer21_attention_weight_min": -24.221874237060547,
      "activations/layer22_attention_weight_max": 40.93918991088867,
      "activations/layer22_attention_weight_min": -29.066436767578125,
      "activations/layer23_attention_weight_max": 32.22848892211914,
      "activations/layer23_attention_weight_min": -24.456716537475586,
      "activations/layer2_attention_weight_max": 29.522504806518555,
      "activations/layer2_attention_weight_min": -30.586957931518555,
      "activations/layer3_attention_weight_max": 91.49551391601562,
      "activations/layer3_attention_weight_min": -86.5751724243164,
      "activations/layer4_attention_weight_max": 112.31734466552734,
      "activations/layer4_attention_weight_min": -107.32594299316406,
      "activations/layer5_attention_weight_max": 50.495277404785156,
      "activations/layer5_attention_weight_min": -60.747955322265625,
      "activations/layer6_attention_weight_max": 43.91667556762695,
      "activations/layer6_attention_weight_min": -47.099239349365234,
      "activations/layer7_attention_weight_max": 83.57292938232422,
      "activations/layer7_attention_weight_min": -88.1578140258789,
      "activations/layer8_attention_weight_max": 36.80826950073242,
      "activations/layer8_attention_weight_min": -38.47446823120117,
      "activations/layer9_attention_weight_max": 33.31399917602539,
      "activations/layer9_attention_weight_min": -33.266258239746094,
      "epoch": 9.73,
      "learning_rate": 8.816136363636364e-05,
      "loss": 2.7952,
      "step": 167400
    },
    {
      "activations/layer0_attention_weight_max": 15.966262817382812,
      "activations/layer0_attention_weight_min": -13.279410362243652,
      "activations/layer10_attention_weight_max": 41.01921081542969,
      "activations/layer10_attention_weight_min": -38.37346267700195,
      "activations/layer11_attention_weight_max": 41.37748718261719,
      "activations/layer11_attention_weight_min": -36.00869369506836,
      "activations/layer12_attention_weight_max": 29.268823623657227,
      "activations/layer12_attention_weight_min": -27.459508895874023,
      "activations/layer13_attention_weight_max": 48.864662170410156,
      "activations/layer13_attention_weight_min": -34.94782257080078,
      "activations/layer14_attention_weight_max": 59.1228141784668,
      "activations/layer14_attention_weight_min": -50.29011154174805,
      "activations/layer15_attention_weight_max": 39.30412292480469,
      "activations/layer15_attention_weight_min": -31.77247428894043,
      "activations/layer16_attention_weight_max": 30.62067413330078,
      "activations/layer16_attention_weight_min": -29.589946746826172,
      "activations/layer17_attention_weight_max": 29.77321434020996,
      "activations/layer17_attention_weight_min": -26.130752563476562,
      "activations/layer18_attention_weight_max": 32.69191360473633,
      "activations/layer18_attention_weight_min": -24.23218536376953,
      "activations/layer19_attention_weight_max": 35.39414596557617,
      "activations/layer19_attention_weight_min": -31.40378189086914,
      "activations/layer1_attention_weight_max": 16.01190185546875,
      "activations/layer1_attention_weight_min": -15.017231941223145,
      "activations/layer20_attention_weight_max": 29.354007720947266,
      "activations/layer20_attention_weight_min": -24.01671600341797,
      "activations/layer21_attention_weight_max": 29.88828468322754,
      "activations/layer21_attention_weight_min": -22.907182693481445,
      "activations/layer22_attention_weight_max": 48.5981559753418,
      "activations/layer22_attention_weight_min": -29.756256103515625,
      "activations/layer23_attention_weight_max": 38.295616149902344,
      "activations/layer23_attention_weight_min": -26.41765594482422,
      "activations/layer2_attention_weight_max": 34.06202697753906,
      "activations/layer2_attention_weight_min": -31.38323211669922,
      "activations/layer3_attention_weight_max": 92.3244400024414,
      "activations/layer3_attention_weight_min": -93.05433654785156,
      "activations/layer4_attention_weight_max": 118.07716369628906,
      "activations/layer4_attention_weight_min": -116.80812072753906,
      "activations/layer5_attention_weight_max": 55.27177047729492,
      "activations/layer5_attention_weight_min": -61.91967010498047,
      "activations/layer6_attention_weight_max": 48.83677291870117,
      "activations/layer6_attention_weight_min": -49.514305114746094,
      "activations/layer7_attention_weight_max": 98.38980102539062,
      "activations/layer7_attention_weight_min": -94.07823181152344,
      "activations/layer8_attention_weight_max": 47.49759292602539,
      "activations/layer8_attention_weight_min": -47.1021728515625,
      "activations/layer9_attention_weight_max": 40.61318588256836,
      "activations/layer9_attention_weight_min": -38.37720489501953,
      "epoch": 9.73,
      "learning_rate": 8.814242424242423e-05,
      "loss": 2.7828,
      "step": 167450
    },
    {
      "activations/layer0_attention_weight_max": 15.97716999053955,
      "activations/layer0_attention_weight_min": -11.952804565429688,
      "activations/layer10_attention_weight_max": 33.4378662109375,
      "activations/layer10_attention_weight_min": -34.82218933105469,
      "activations/layer11_attention_weight_max": 34.1607666015625,
      "activations/layer11_attention_weight_min": -33.446434020996094,
      "activations/layer12_attention_weight_max": 38.72019577026367,
      "activations/layer12_attention_weight_min": -26.037378311157227,
      "activations/layer13_attention_weight_max": 42.3170166015625,
      "activations/layer13_attention_weight_min": -33.185150146484375,
      "activations/layer14_attention_weight_max": 46.70976638793945,
      "activations/layer14_attention_weight_min": -37.36048126220703,
      "activations/layer15_attention_weight_max": 36.0806770324707,
      "activations/layer15_attention_weight_min": -33.094215393066406,
      "activations/layer16_attention_weight_max": 31.94382667541504,
      "activations/layer16_attention_weight_min": -29.359291076660156,
      "activations/layer17_attention_weight_max": 33.00505447387695,
      "activations/layer17_attention_weight_min": -26.53598976135254,
      "activations/layer18_attention_weight_max": 32.8178825378418,
      "activations/layer18_attention_weight_min": -21.716062545776367,
      "activations/layer19_attention_weight_max": 36.85527038574219,
      "activations/layer19_attention_weight_min": -31.264394760131836,
      "activations/layer1_attention_weight_max": 16.7359561920166,
      "activations/layer1_attention_weight_min": -14.410362243652344,
      "activations/layer20_attention_weight_max": 30.915130615234375,
      "activations/layer20_attention_weight_min": -23.900901794433594,
      "activations/layer21_attention_weight_max": 29.484464645385742,
      "activations/layer21_attention_weight_min": -23.183395385742188,
      "activations/layer22_attention_weight_max": 46.41120910644531,
      "activations/layer22_attention_weight_min": -29.056514739990234,
      "activations/layer23_attention_weight_max": 38.62331008911133,
      "activations/layer23_attention_weight_min": -24.38295555114746,
      "activations/layer2_attention_weight_max": 31.55301284790039,
      "activations/layer2_attention_weight_min": -31.89011001586914,
      "activations/layer3_attention_weight_max": 95.00640869140625,
      "activations/layer3_attention_weight_min": -95.93154907226562,
      "activations/layer4_attention_weight_max": 117.7342529296875,
      "activations/layer4_attention_weight_min": -115.0394287109375,
      "activations/layer5_attention_weight_max": 57.362056732177734,
      "activations/layer5_attention_weight_min": -63.34461975097656,
      "activations/layer6_attention_weight_max": 49.47403335571289,
      "activations/layer6_attention_weight_min": -51.40018844604492,
      "activations/layer7_attention_weight_max": 96.79486083984375,
      "activations/layer7_attention_weight_min": -94.18029022216797,
      "activations/layer8_attention_weight_max": 40.02547836303711,
      "activations/layer8_attention_weight_min": -42.09651565551758,
      "activations/layer9_attention_weight_max": 35.20640182495117,
      "activations/layer9_attention_weight_min": -33.90018844604492,
      "epoch": 9.73,
      "learning_rate": 8.812386363636362e-05,
      "loss": 2.7893,
      "step": 167500
    },
    {
      "activations/layer0_attention_weight_max": 16.068552017211914,
      "activations/layer0_attention_weight_min": -12.333683967590332,
      "activations/layer10_attention_weight_max": 36.606021881103516,
      "activations/layer10_attention_weight_min": -33.091392517089844,
      "activations/layer11_attention_weight_max": 36.356712341308594,
      "activations/layer11_attention_weight_min": -32.35669708251953,
      "activations/layer12_attention_weight_max": 26.646066665649414,
      "activations/layer12_attention_weight_min": -26.83387565612793,
      "activations/layer13_attention_weight_max": 46.45962905883789,
      "activations/layer13_attention_weight_min": -35.28654098510742,
      "activations/layer14_attention_weight_max": 46.85923767089844,
      "activations/layer14_attention_weight_min": -38.87904357910156,
      "activations/layer15_attention_weight_max": 38.246646881103516,
      "activations/layer15_attention_weight_min": -30.135047912597656,
      "activations/layer16_attention_weight_max": 33.34596252441406,
      "activations/layer16_attention_weight_min": -29.47774887084961,
      "activations/layer17_attention_weight_max": 35.86024475097656,
      "activations/layer17_attention_weight_min": -24.85164451599121,
      "activations/layer18_attention_weight_max": 33.72151184082031,
      "activations/layer18_attention_weight_min": -21.200611114501953,
      "activations/layer19_attention_weight_max": 40.56635665893555,
      "activations/layer19_attention_weight_min": -30.891128540039062,
      "activations/layer1_attention_weight_max": 16.07563018798828,
      "activations/layer1_attention_weight_min": -13.503129959106445,
      "activations/layer20_attention_weight_max": 33.39331817626953,
      "activations/layer20_attention_weight_min": -24.43630599975586,
      "activations/layer21_attention_weight_max": 32.595314025878906,
      "activations/layer21_attention_weight_min": -23.381500244140625,
      "activations/layer22_attention_weight_max": 54.960487365722656,
      "activations/layer22_attention_weight_min": -32.045127868652344,
      "activations/layer23_attention_weight_max": 40.195587158203125,
      "activations/layer23_attention_weight_min": -23.20765495300293,
      "activations/layer2_attention_weight_max": 31.460447311401367,
      "activations/layer2_attention_weight_min": -32.68616485595703,
      "activations/layer3_attention_weight_max": 89.14347076416016,
      "activations/layer3_attention_weight_min": -90.16629791259766,
      "activations/layer4_attention_weight_max": 111.5789794921875,
      "activations/layer4_attention_weight_min": -112.07933807373047,
      "activations/layer5_attention_weight_max": 51.19477081298828,
      "activations/layer5_attention_weight_min": -61.770721435546875,
      "activations/layer6_attention_weight_max": 45.65635299682617,
      "activations/layer6_attention_weight_min": -48.41708755493164,
      "activations/layer7_attention_weight_max": 87.822998046875,
      "activations/layer7_attention_weight_min": -88.58763885498047,
      "activations/layer8_attention_weight_max": 42.482017517089844,
      "activations/layer8_attention_weight_min": -40.15812301635742,
      "activations/layer9_attention_weight_max": 36.555667877197266,
      "activations/layer9_attention_weight_min": -34.81166458129883,
      "epoch": 9.74,
      "learning_rate": 8.810492424242423e-05,
      "loss": 2.7895,
      "step": 167550
    },
    {
      "activations/layer0_attention_weight_max": 15.119661331176758,
      "activations/layer0_attention_weight_min": -11.387106895446777,
      "activations/layer10_attention_weight_max": 37.630615234375,
      "activations/layer10_attention_weight_min": -34.361656188964844,
      "activations/layer11_attention_weight_max": 39.849246978759766,
      "activations/layer11_attention_weight_min": -36.839332580566406,
      "activations/layer12_attention_weight_max": 21.83376693725586,
      "activations/layer12_attention_weight_min": -26.247465133666992,
      "activations/layer13_attention_weight_max": 36.503143310546875,
      "activations/layer13_attention_weight_min": -35.51725769042969,
      "activations/layer14_attention_weight_max": 42.01525115966797,
      "activations/layer14_attention_weight_min": -37.624000549316406,
      "activations/layer15_attention_weight_max": 39.346160888671875,
      "activations/layer15_attention_weight_min": -35.65036392211914,
      "activations/layer16_attention_weight_max": 31.65477180480957,
      "activations/layer16_attention_weight_min": -27.83619499206543,
      "activations/layer17_attention_weight_max": 29.75897216796875,
      "activations/layer17_attention_weight_min": -25.097517013549805,
      "activations/layer18_attention_weight_max": 28.6055850982666,
      "activations/layer18_attention_weight_min": -21.476463317871094,
      "activations/layer19_attention_weight_max": 31.39418601989746,
      "activations/layer19_attention_weight_min": -29.92828369140625,
      "activations/layer1_attention_weight_max": 18.227418899536133,
      "activations/layer1_attention_weight_min": -14.798609733581543,
      "activations/layer20_attention_weight_max": 29.6633243560791,
      "activations/layer20_attention_weight_min": -23.475690841674805,
      "activations/layer21_attention_weight_max": 29.415945053100586,
      "activations/layer21_attention_weight_min": -21.84674644470215,
      "activations/layer22_attention_weight_max": 40.237937927246094,
      "activations/layer22_attention_weight_min": -29.132402420043945,
      "activations/layer23_attention_weight_max": 31.617843627929688,
      "activations/layer23_attention_weight_min": -22.218170166015625,
      "activations/layer2_attention_weight_max": 32.04404830932617,
      "activations/layer2_attention_weight_min": -31.94261932373047,
      "activations/layer3_attention_weight_max": 90.84626007080078,
      "activations/layer3_attention_weight_min": -91.9615249633789,
      "activations/layer4_attention_weight_max": 110.39640045166016,
      "activations/layer4_attention_weight_min": -108.2290267944336,
      "activations/layer5_attention_weight_max": 53.344520568847656,
      "activations/layer5_attention_weight_min": -59.587093353271484,
      "activations/layer6_attention_weight_max": 48.42073440551758,
      "activations/layer6_attention_weight_min": -46.17314910888672,
      "activations/layer7_attention_weight_max": 98.15631103515625,
      "activations/layer7_attention_weight_min": -89.5175552368164,
      "activations/layer8_attention_weight_max": 43.38045120239258,
      "activations/layer8_attention_weight_min": -40.59932327270508,
      "activations/layer9_attention_weight_max": 35.02701187133789,
      "activations/layer9_attention_weight_min": -33.622615814208984,
      "epoch": 9.74,
      "learning_rate": 8.808598484848485e-05,
      "loss": 2.7812,
      "step": 167600
    },
    {
      "activations/layer0_attention_weight_max": 15.350198745727539,
      "activations/layer0_attention_weight_min": -12.231228828430176,
      "activations/layer10_attention_weight_max": 32.50416946411133,
      "activations/layer10_attention_weight_min": -33.42362976074219,
      "activations/layer11_attention_weight_max": 32.087677001953125,
      "activations/layer11_attention_weight_min": -34.86923599243164,
      "activations/layer12_attention_weight_max": 25.110761642456055,
      "activations/layer12_attention_weight_min": -27.022584915161133,
      "activations/layer13_attention_weight_max": 41.575660705566406,
      "activations/layer13_attention_weight_min": -38.78056716918945,
      "activations/layer14_attention_weight_max": 50.23724365234375,
      "activations/layer14_attention_weight_min": -44.049232482910156,
      "activations/layer15_attention_weight_max": 37.925384521484375,
      "activations/layer15_attention_weight_min": -36.20177459716797,
      "activations/layer16_attention_weight_max": 32.66283416748047,
      "activations/layer16_attention_weight_min": -29.55142593383789,
      "activations/layer17_attention_weight_max": 34.08580780029297,
      "activations/layer17_attention_weight_min": -26.254531860351562,
      "activations/layer18_attention_weight_max": 34.27739334106445,
      "activations/layer18_attention_weight_min": -22.761695861816406,
      "activations/layer19_attention_weight_max": 37.85056686401367,
      "activations/layer19_attention_weight_min": -31.295047760009766,
      "activations/layer1_attention_weight_max": 16.048120498657227,
      "activations/layer1_attention_weight_min": -13.926225662231445,
      "activations/layer20_attention_weight_max": 33.2327995300293,
      "activations/layer20_attention_weight_min": -26.294143676757812,
      "activations/layer21_attention_weight_max": 33.814231872558594,
      "activations/layer21_attention_weight_min": -24.97731590270996,
      "activations/layer22_attention_weight_max": 54.204490661621094,
      "activations/layer22_attention_weight_min": -30.01694679260254,
      "activations/layer23_attention_weight_max": 40.90338134765625,
      "activations/layer23_attention_weight_min": -26.743940353393555,
      "activations/layer2_attention_weight_max": 31.238370895385742,
      "activations/layer2_attention_weight_min": -32.74127960205078,
      "activations/layer3_attention_weight_max": 91.12222290039062,
      "activations/layer3_attention_weight_min": -102.23664093017578,
      "activations/layer4_attention_weight_max": 110.15142059326172,
      "activations/layer4_attention_weight_min": -109.37763977050781,
      "activations/layer5_attention_weight_max": 52.77961730957031,
      "activations/layer5_attention_weight_min": -59.22734832763672,
      "activations/layer6_attention_weight_max": 46.1970100402832,
      "activations/layer6_attention_weight_min": -50.12025833129883,
      "activations/layer7_attention_weight_max": 90.44989776611328,
      "activations/layer7_attention_weight_min": -89.54134368896484,
      "activations/layer8_attention_weight_max": 39.12387466430664,
      "activations/layer8_attention_weight_min": -41.4692497253418,
      "activations/layer9_attention_weight_max": 33.39353942871094,
      "activations/layer9_attention_weight_min": -35.01486587524414,
      "epoch": 9.74,
      "learning_rate": 8.806704545454544e-05,
      "loss": 2.7825,
      "step": 167650
    },
    {
      "activations/layer0_attention_weight_max": 15.803018569946289,
      "activations/layer0_attention_weight_min": -12.735823631286621,
      "activations/layer10_attention_weight_max": 31.33463478088379,
      "activations/layer10_attention_weight_min": -34.61861038208008,
      "activations/layer11_attention_weight_max": 33.055538177490234,
      "activations/layer11_attention_weight_min": -30.54808807373047,
      "activations/layer12_attention_weight_max": 24.257749557495117,
      "activations/layer12_attention_weight_min": -26.333696365356445,
      "activations/layer13_attention_weight_max": 47.50075912475586,
      "activations/layer13_attention_weight_min": -35.50651168823242,
      "activations/layer14_attention_weight_max": 48.2868766784668,
      "activations/layer14_attention_weight_min": -39.13040542602539,
      "activations/layer15_attention_weight_max": 36.22563552856445,
      "activations/layer15_attention_weight_min": -31.207828521728516,
      "activations/layer16_attention_weight_max": 31.577648162841797,
      "activations/layer16_attention_weight_min": -28.00167465209961,
      "activations/layer17_attention_weight_max": 32.43213653564453,
      "activations/layer17_attention_weight_min": -26.284090042114258,
      "activations/layer18_attention_weight_max": 34.03396987915039,
      "activations/layer18_attention_weight_min": -22.771984100341797,
      "activations/layer19_attention_weight_max": 39.78994369506836,
      "activations/layer19_attention_weight_min": -31.042984008789062,
      "activations/layer1_attention_weight_max": 16.369979858398438,
      "activations/layer1_attention_weight_min": -15.022335052490234,
      "activations/layer20_attention_weight_max": 33.22402572631836,
      "activations/layer20_attention_weight_min": -26.256458282470703,
      "activations/layer21_attention_weight_max": 31.667022705078125,
      "activations/layer21_attention_weight_min": -23.876169204711914,
      "activations/layer22_attention_weight_max": 51.6839599609375,
      "activations/layer22_attention_weight_min": -30.7181396484375,
      "activations/layer23_attention_weight_max": 46.73739242553711,
      "activations/layer23_attention_weight_min": -25.635826110839844,
      "activations/layer2_attention_weight_max": 31.466388702392578,
      "activations/layer2_attention_weight_min": -32.02705001831055,
      "activations/layer3_attention_weight_max": 90.72846984863281,
      "activations/layer3_attention_weight_min": -97.79923248291016,
      "activations/layer4_attention_weight_max": 113.66596984863281,
      "activations/layer4_attention_weight_min": -116.35052490234375,
      "activations/layer5_attention_weight_max": 52.854976654052734,
      "activations/layer5_attention_weight_min": -62.57093811035156,
      "activations/layer6_attention_weight_max": 46.04948806762695,
      "activations/layer6_attention_weight_min": -54.48743438720703,
      "activations/layer7_attention_weight_max": 89.30118560791016,
      "activations/layer7_attention_weight_min": -96.48710632324219,
      "activations/layer8_attention_weight_max": 41.35419845581055,
      "activations/layer8_attention_weight_min": -42.49509811401367,
      "activations/layer9_attention_weight_max": 33.05979537963867,
      "activations/layer9_attention_weight_min": -33.714412689208984,
      "epoch": 9.74,
      "learning_rate": 8.804810606060605e-05,
      "loss": 2.7994,
      "step": 167700
    },
    {
      "activations/layer0_attention_weight_max": 14.57166862487793,
      "activations/layer0_attention_weight_min": -10.595135688781738,
      "activations/layer10_attention_weight_max": 36.16435241699219,
      "activations/layer10_attention_weight_min": -33.26520538330078,
      "activations/layer11_attention_weight_max": 38.087684631347656,
      "activations/layer11_attention_weight_min": -34.54368591308594,
      "activations/layer12_attention_weight_max": 25.986722946166992,
      "activations/layer12_attention_weight_min": -25.25320816040039,
      "activations/layer13_attention_weight_max": 48.16508102416992,
      "activations/layer13_attention_weight_min": -33.10710906982422,
      "activations/layer14_attention_weight_max": 48.39236068725586,
      "activations/layer14_attention_weight_min": -37.98830795288086,
      "activations/layer15_attention_weight_max": 45.48438262939453,
      "activations/layer15_attention_weight_min": -30.572246551513672,
      "activations/layer16_attention_weight_max": 32.9420051574707,
      "activations/layer16_attention_weight_min": -26.68305206298828,
      "activations/layer17_attention_weight_max": 34.127227783203125,
      "activations/layer17_attention_weight_min": -27.120372772216797,
      "activations/layer18_attention_weight_max": 30.6479434967041,
      "activations/layer18_attention_weight_min": -21.522920608520508,
      "activations/layer19_attention_weight_max": 37.72671127319336,
      "activations/layer19_attention_weight_min": -29.871740341186523,
      "activations/layer1_attention_weight_max": 17.201133728027344,
      "activations/layer1_attention_weight_min": -13.697687149047852,
      "activations/layer20_attention_weight_max": 30.924949645996094,
      "activations/layer20_attention_weight_min": -24.4892520904541,
      "activations/layer21_attention_weight_max": 27.483823776245117,
      "activations/layer21_attention_weight_min": -21.930044174194336,
      "activations/layer22_attention_weight_max": 46.910640716552734,
      "activations/layer22_attention_weight_min": -29.138282775878906,
      "activations/layer23_attention_weight_max": 38.458133697509766,
      "activations/layer23_attention_weight_min": -23.926565170288086,
      "activations/layer2_attention_weight_max": 32.588844299316406,
      "activations/layer2_attention_weight_min": -32.747528076171875,
      "activations/layer3_attention_weight_max": 92.31327056884766,
      "activations/layer3_attention_weight_min": -93.85026550292969,
      "activations/layer4_attention_weight_max": 112.0475082397461,
      "activations/layer4_attention_weight_min": -113.00604248046875,
      "activations/layer5_attention_weight_max": 52.60536193847656,
      "activations/layer5_attention_weight_min": -63.16459274291992,
      "activations/layer6_attention_weight_max": 47.03288269042969,
      "activations/layer6_attention_weight_min": -51.104957580566406,
      "activations/layer7_attention_weight_max": 92.07579803466797,
      "activations/layer7_attention_weight_min": -96.07290649414062,
      "activations/layer8_attention_weight_max": 43.2931022644043,
      "activations/layer8_attention_weight_min": -43.42781448364258,
      "activations/layer9_attention_weight_max": 36.273765563964844,
      "activations/layer9_attention_weight_min": -35.85124206542969,
      "epoch": 9.75,
      "learning_rate": 8.802916666666665e-05,
      "loss": 2.8029,
      "step": 167750
    },
    {
      "activations/layer0_attention_weight_max": 16.0052547454834,
      "activations/layer0_attention_weight_min": -12.095335006713867,
      "activations/layer10_attention_weight_max": 35.932621002197266,
      "activations/layer10_attention_weight_min": -32.69746780395508,
      "activations/layer11_attention_weight_max": 33.56257629394531,
      "activations/layer11_attention_weight_min": -35.25303649902344,
      "activations/layer12_attention_weight_max": 23.6132869720459,
      "activations/layer12_attention_weight_min": -24.19943618774414,
      "activations/layer13_attention_weight_max": 36.998779296875,
      "activations/layer13_attention_weight_min": -33.24956130981445,
      "activations/layer14_attention_weight_max": 42.932403564453125,
      "activations/layer14_attention_weight_min": -36.79536437988281,
      "activations/layer15_attention_weight_max": 34.78571701049805,
      "activations/layer15_attention_weight_min": -29.62529754638672,
      "activations/layer16_attention_weight_max": 31.941389083862305,
      "activations/layer16_attention_weight_min": -29.314281463623047,
      "activations/layer17_attention_weight_max": 33.227020263671875,
      "activations/layer17_attention_weight_min": -25.70616340637207,
      "activations/layer18_attention_weight_max": 32.8076057434082,
      "activations/layer18_attention_weight_min": -21.960735321044922,
      "activations/layer19_attention_weight_max": 34.77298355102539,
      "activations/layer19_attention_weight_min": -31.80354118347168,
      "activations/layer1_attention_weight_max": 16.441204071044922,
      "activations/layer1_attention_weight_min": -14.162548065185547,
      "activations/layer20_attention_weight_max": 34.28842544555664,
      "activations/layer20_attention_weight_min": -24.715913772583008,
      "activations/layer21_attention_weight_max": 30.791271209716797,
      "activations/layer21_attention_weight_min": -23.681053161621094,
      "activations/layer22_attention_weight_max": 45.05879211425781,
      "activations/layer22_attention_weight_min": -29.361772537231445,
      "activations/layer23_attention_weight_max": 35.69310760498047,
      "activations/layer23_attention_weight_min": -24.504886627197266,
      "activations/layer2_attention_weight_max": 31.26966094970703,
      "activations/layer2_attention_weight_min": -30.72241973876953,
      "activations/layer3_attention_weight_max": 91.02103424072266,
      "activations/layer3_attention_weight_min": -92.7650146484375,
      "activations/layer4_attention_weight_max": 114.75665283203125,
      "activations/layer4_attention_weight_min": -114.86998748779297,
      "activations/layer5_attention_weight_max": 52.136104583740234,
      "activations/layer5_attention_weight_min": -58.9200439453125,
      "activations/layer6_attention_weight_max": 44.07093811035156,
      "activations/layer6_attention_weight_min": -48.31526565551758,
      "activations/layer7_attention_weight_max": 84.3087158203125,
      "activations/layer7_attention_weight_min": -86.71672058105469,
      "activations/layer8_attention_weight_max": 40.539703369140625,
      "activations/layer8_attention_weight_min": -39.21550750732422,
      "activations/layer9_attention_weight_max": 33.90462875366211,
      "activations/layer9_attention_weight_min": -33.82770538330078,
      "epoch": 9.75,
      "learning_rate": 8.801022727272727e-05,
      "loss": 2.8207,
      "step": 167800
    },
    {
      "activations/layer0_attention_weight_max": 16.08558464050293,
      "activations/layer0_attention_weight_min": -11.603927612304688,
      "activations/layer10_attention_weight_max": 33.21076583862305,
      "activations/layer10_attention_weight_min": -32.3687858581543,
      "activations/layer11_attention_weight_max": 30.881345748901367,
      "activations/layer11_attention_weight_min": -30.217529296875,
      "activations/layer12_attention_weight_max": 24.029354095458984,
      "activations/layer12_attention_weight_min": -24.02178192138672,
      "activations/layer13_attention_weight_max": 39.63435363769531,
      "activations/layer13_attention_weight_min": -34.045692443847656,
      "activations/layer14_attention_weight_max": 45.59196090698242,
      "activations/layer14_attention_weight_min": -39.80821228027344,
      "activations/layer15_attention_weight_max": 35.036685943603516,
      "activations/layer15_attention_weight_min": -32.2028923034668,
      "activations/layer16_attention_weight_max": 30.780916213989258,
      "activations/layer16_attention_weight_min": -29.033864974975586,
      "activations/layer17_attention_weight_max": 33.702117919921875,
      "activations/layer17_attention_weight_min": -26.566328048706055,
      "activations/layer18_attention_weight_max": 36.27694320678711,
      "activations/layer18_attention_weight_min": -25.455171585083008,
      "activations/layer19_attention_weight_max": 42.1281623840332,
      "activations/layer19_attention_weight_min": -34.00266647338867,
      "activations/layer1_attention_weight_max": 16.023204803466797,
      "activations/layer1_attention_weight_min": -15.408233642578125,
      "activations/layer20_attention_weight_max": 33.66581726074219,
      "activations/layer20_attention_weight_min": -26.12626838684082,
      "activations/layer21_attention_weight_max": 36.44953155517578,
      "activations/layer21_attention_weight_min": -23.598173141479492,
      "activations/layer22_attention_weight_max": 50.966793060302734,
      "activations/layer22_attention_weight_min": -29.904314041137695,
      "activations/layer23_attention_weight_max": 39.211605072021484,
      "activations/layer23_attention_weight_min": -24.47744369506836,
      "activations/layer2_attention_weight_max": 30.976924896240234,
      "activations/layer2_attention_weight_min": -31.818777084350586,
      "activations/layer3_attention_weight_max": 89.515380859375,
      "activations/layer3_attention_weight_min": -91.62911987304688,
      "activations/layer4_attention_weight_max": 110.00945281982422,
      "activations/layer4_attention_weight_min": -114.77678680419922,
      "activations/layer5_attention_weight_max": 50.88530731201172,
      "activations/layer5_attention_weight_min": -61.35706329345703,
      "activations/layer6_attention_weight_max": 44.442344665527344,
      "activations/layer6_attention_weight_min": -48.3690071105957,
      "activations/layer7_attention_weight_max": 87.79972076416016,
      "activations/layer7_attention_weight_min": -86.70738220214844,
      "activations/layer8_attention_weight_max": 39.28792190551758,
      "activations/layer8_attention_weight_min": -38.47496795654297,
      "activations/layer9_attention_weight_max": 33.53877258300781,
      "activations/layer9_attention_weight_min": -33.620059967041016,
      "epoch": 9.75,
      "learning_rate": 8.799128787878787e-05,
      "loss": 2.7969,
      "step": 167850
    },
    {
      "activations/layer0_attention_weight_max": 15.33676528930664,
      "activations/layer0_attention_weight_min": -12.120399475097656,
      "activations/layer10_attention_weight_max": 32.595401763916016,
      "activations/layer10_attention_weight_min": -32.728675842285156,
      "activations/layer11_attention_weight_max": 34.89976119995117,
      "activations/layer11_attention_weight_min": -30.204425811767578,
      "activations/layer12_attention_weight_max": 40.66018295288086,
      "activations/layer12_attention_weight_min": -25.273635864257812,
      "activations/layer13_attention_weight_max": 41.11335372924805,
      "activations/layer13_attention_weight_min": -32.95210266113281,
      "activations/layer14_attention_weight_max": 46.523658752441406,
      "activations/layer14_attention_weight_min": -42.422813415527344,
      "activations/layer15_attention_weight_max": 37.37717056274414,
      "activations/layer15_attention_weight_min": -31.748580932617188,
      "activations/layer16_attention_weight_max": 31.659650802612305,
      "activations/layer16_attention_weight_min": -27.74339485168457,
      "activations/layer17_attention_weight_max": 32.43093490600586,
      "activations/layer17_attention_weight_min": -25.634084701538086,
      "activations/layer18_attention_weight_max": 33.23537063598633,
      "activations/layer18_attention_weight_min": -23.025712966918945,
      "activations/layer19_attention_weight_max": 35.45297622680664,
      "activations/layer19_attention_weight_min": -31.035184860229492,
      "activations/layer1_attention_weight_max": 17.513246536254883,
      "activations/layer1_attention_weight_min": -14.126163482666016,
      "activations/layer20_attention_weight_max": 31.164323806762695,
      "activations/layer20_attention_weight_min": -25.52834129333496,
      "activations/layer21_attention_weight_max": 31.871355056762695,
      "activations/layer21_attention_weight_min": -22.736597061157227,
      "activations/layer22_attention_weight_max": 44.42192077636719,
      "activations/layer22_attention_weight_min": -29.124101638793945,
      "activations/layer23_attention_weight_max": 34.931129455566406,
      "activations/layer23_attention_weight_min": -24.992931365966797,
      "activations/layer2_attention_weight_max": 31.547086715698242,
      "activations/layer2_attention_weight_min": -31.03388214111328,
      "activations/layer3_attention_weight_max": 89.18411254882812,
      "activations/layer3_attention_weight_min": -92.12114715576172,
      "activations/layer4_attention_weight_max": 113.8768081665039,
      "activations/layer4_attention_weight_min": -109.38448333740234,
      "activations/layer5_attention_weight_max": 50.01569366455078,
      "activations/layer5_attention_weight_min": -62.00565719604492,
      "activations/layer6_attention_weight_max": 46.05231857299805,
      "activations/layer6_attention_weight_min": -47.731143951416016,
      "activations/layer7_attention_weight_max": 92.5294418334961,
      "activations/layer7_attention_weight_min": -89.61639404296875,
      "activations/layer8_attention_weight_max": 40.79115676879883,
      "activations/layer8_attention_weight_min": -39.9255256652832,
      "activations/layer9_attention_weight_max": 37.44224166870117,
      "activations/layer9_attention_weight_min": -35.0166130065918,
      "epoch": 9.76,
      "learning_rate": 8.797234848484847e-05,
      "loss": 2.7896,
      "step": 167900
    },
    {
      "activations/layer0_attention_weight_max": 15.317862510681152,
      "activations/layer0_attention_weight_min": -11.799683570861816,
      "activations/layer10_attention_weight_max": 33.49809646606445,
      "activations/layer10_attention_weight_min": -31.818084716796875,
      "activations/layer11_attention_weight_max": 35.045284271240234,
      "activations/layer11_attention_weight_min": -32.37623977661133,
      "activations/layer12_attention_weight_max": 24.742847442626953,
      "activations/layer12_attention_weight_min": -25.0001277923584,
      "activations/layer13_attention_weight_max": 39.759788513183594,
      "activations/layer13_attention_weight_min": -32.47120666503906,
      "activations/layer14_attention_weight_max": 50.70260238647461,
      "activations/layer14_attention_weight_min": -38.21712112426758,
      "activations/layer15_attention_weight_max": 38.83686447143555,
      "activations/layer15_attention_weight_min": -34.20729064941406,
      "activations/layer16_attention_weight_max": 36.55708312988281,
      "activations/layer16_attention_weight_min": -30.058040618896484,
      "activations/layer17_attention_weight_max": 36.942928314208984,
      "activations/layer17_attention_weight_min": -25.779905319213867,
      "activations/layer18_attention_weight_max": 35.50662612915039,
      "activations/layer18_attention_weight_min": -24.214427947998047,
      "activations/layer19_attention_weight_max": 45.310089111328125,
      "activations/layer19_attention_weight_min": -32.848819732666016,
      "activations/layer1_attention_weight_max": 17.384296417236328,
      "activations/layer1_attention_weight_min": -14.453057289123535,
      "activations/layer20_attention_weight_max": 42.86893081665039,
      "activations/layer20_attention_weight_min": -26.64744758605957,
      "activations/layer21_attention_weight_max": 37.43408203125,
      "activations/layer21_attention_weight_min": -24.346086502075195,
      "activations/layer22_attention_weight_max": 49.76847457885742,
      "activations/layer22_attention_weight_min": -30.68811798095703,
      "activations/layer23_attention_weight_max": 41.5936393737793,
      "activations/layer23_attention_weight_min": -25.75465965270996,
      "activations/layer2_attention_weight_max": 31.779346466064453,
      "activations/layer2_attention_weight_min": -33.51372528076172,
      "activations/layer3_attention_weight_max": 95.77731323242188,
      "activations/layer3_attention_weight_min": -96.48566436767578,
      "activations/layer4_attention_weight_max": 119.36190032958984,
      "activations/layer4_attention_weight_min": -120.97603607177734,
      "activations/layer5_attention_weight_max": 56.02362823486328,
      "activations/layer5_attention_weight_min": -63.37704086303711,
      "activations/layer6_attention_weight_max": 48.37205123901367,
      "activations/layer6_attention_weight_min": -49.61219024658203,
      "activations/layer7_attention_weight_max": 91.00619506835938,
      "activations/layer7_attention_weight_min": -94.9785385131836,
      "activations/layer8_attention_weight_max": 40.795127868652344,
      "activations/layer8_attention_weight_min": -42.693885803222656,
      "activations/layer9_attention_weight_max": 35.69686508178711,
      "activations/layer9_attention_weight_min": -33.33435821533203,
      "epoch": 9.76,
      "learning_rate": 8.795340909090909e-05,
      "loss": 2.7994,
      "step": 167950
    },
    {
      "activations/layer0_attention_weight_max": 15.951798439025879,
      "activations/layer0_attention_weight_min": -11.34898567199707,
      "activations/layer10_attention_weight_max": 31.81845474243164,
      "activations/layer10_attention_weight_min": -32.721534729003906,
      "activations/layer11_attention_weight_max": 34.4405632019043,
      "activations/layer11_attention_weight_min": -32.34560012817383,
      "activations/layer12_attention_weight_max": 47.62543869018555,
      "activations/layer12_attention_weight_min": -26.656435012817383,
      "activations/layer13_attention_weight_max": 49.13202667236328,
      "activations/layer13_attention_weight_min": -33.24784851074219,
      "activations/layer14_attention_weight_max": 47.73193359375,
      "activations/layer14_attention_weight_min": -38.52837371826172,
      "activations/layer15_attention_weight_max": 35.68419647216797,
      "activations/layer15_attention_weight_min": -31.081703186035156,
      "activations/layer16_attention_weight_max": 33.8868522644043,
      "activations/layer16_attention_weight_min": -26.623380661010742,
      "activations/layer17_attention_weight_max": 35.94966125488281,
      "activations/layer17_attention_weight_min": -24.68058204650879,
      "activations/layer18_attention_weight_max": 31.061214447021484,
      "activations/layer18_attention_weight_min": -22.417709350585938,
      "activations/layer19_attention_weight_max": 35.532470703125,
      "activations/layer19_attention_weight_min": -32.6749382019043,
      "activations/layer1_attention_weight_max": 16.59243392944336,
      "activations/layer1_attention_weight_min": -14.948378562927246,
      "activations/layer20_attention_weight_max": 32.40491485595703,
      "activations/layer20_attention_weight_min": -25.854087829589844,
      "activations/layer21_attention_weight_max": 32.01013946533203,
      "activations/layer21_attention_weight_min": -24.417461395263672,
      "activations/layer22_attention_weight_max": 47.514102935791016,
      "activations/layer22_attention_weight_min": -30.1767578125,
      "activations/layer23_attention_weight_max": 36.75838851928711,
      "activations/layer23_attention_weight_min": -25.821685791015625,
      "activations/layer2_attention_weight_max": 31.781261444091797,
      "activations/layer2_attention_weight_min": -32.60680389404297,
      "activations/layer3_attention_weight_max": 90.78547668457031,
      "activations/layer3_attention_weight_min": -88.7989501953125,
      "activations/layer4_attention_weight_max": 111.23890686035156,
      "activations/layer4_attention_weight_min": -106.27696990966797,
      "activations/layer5_attention_weight_max": 50.842979431152344,
      "activations/layer5_attention_weight_min": -57.078529357910156,
      "activations/layer6_attention_weight_max": 46.88827896118164,
      "activations/layer6_attention_weight_min": -48.50518035888672,
      "activations/layer7_attention_weight_max": 86.4821548461914,
      "activations/layer7_attention_weight_min": -84.09639739990234,
      "activations/layer8_attention_weight_max": 40.23384094238281,
      "activations/layer8_attention_weight_min": -40.96815872192383,
      "activations/layer9_attention_weight_max": 34.67194747924805,
      "activations/layer9_attention_weight_min": -35.53022003173828,
      "epoch": 9.76,
      "learning_rate": 8.793446969696969e-05,
      "loss": 2.7963,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.465,
      "eval_samples_per_second": 507.263,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.465,
      "eval_openwebtext_samples_per_second": 507.263,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 1.9761,
      "eval_wikitext_samples_per_second": 230.753,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_lambada_loss": 2.587890625,
      "eval_lambada_ppl": 13.301683750542033,
      "eval_lambada_runtime": 9.5139,
      "eval_lambada_samples_per_second": 511.78,
      "step": 168000
    },
    {
      "activations/layer0_attention_weight_max": 15.800472259521484,
      "activations/layer0_attention_weight_min": -11.940678596496582,
      "activations/layer10_attention_weight_max": 28.342721939086914,
      "activations/layer10_attention_weight_min": -29.46133041381836,
      "activations/layer11_attention_weight_max": 27.91475486755371,
      "activations/layer11_attention_weight_min": -29.586095809936523,
      "activations/layer12_attention_weight_max": 22.704166412353516,
      "activations/layer12_attention_weight_min": -23.407859802246094,
      "activations/layer13_attention_weight_max": 36.91590118408203,
      "activations/layer13_attention_weight_min": -34.303977966308594,
      "activations/layer14_attention_weight_max": 49.59710693359375,
      "activations/layer14_attention_weight_min": -35.7199821472168,
      "activations/layer15_attention_weight_max": 32.21476364135742,
      "activations/layer15_attention_weight_min": -28.845460891723633,
      "activations/layer16_attention_weight_max": 31.327238082885742,
      "activations/layer16_attention_weight_min": -25.616743087768555,
      "activations/layer17_attention_weight_max": 29.42901611328125,
      "activations/layer17_attention_weight_min": -24.59981346130371,
      "activations/layer18_attention_weight_max": 30.25947380065918,
      "activations/layer18_attention_weight_min": -23.594688415527344,
      "activations/layer19_attention_weight_max": 35.28775405883789,
      "activations/layer19_attention_weight_min": -28.928543090820312,
      "activations/layer1_attention_weight_max": 17.243555068969727,
      "activations/layer1_attention_weight_min": -16.55984115600586,
      "activations/layer20_attention_weight_max": 27.432409286499023,
      "activations/layer20_attention_weight_min": -24.02632713317871,
      "activations/layer21_attention_weight_max": 25.57212257385254,
      "activations/layer21_attention_weight_min": -22.61187171936035,
      "activations/layer22_attention_weight_max": 37.70266342163086,
      "activations/layer22_attention_weight_min": -27.9067440032959,
      "activations/layer23_attention_weight_max": 34.30267333984375,
      "activations/layer23_attention_weight_min": -24.445056915283203,
      "activations/layer2_attention_weight_max": 30.99587059020996,
      "activations/layer2_attention_weight_min": -32.07468795776367,
      "activations/layer3_attention_weight_max": 91.37657928466797,
      "activations/layer3_attention_weight_min": -93.29486846923828,
      "activations/layer4_attention_weight_max": 112.1258773803711,
      "activations/layer4_attention_weight_min": -108.4801254272461,
      "activations/layer5_attention_weight_max": 51.96049499511719,
      "activations/layer5_attention_weight_min": -59.604759216308594,
      "activations/layer6_attention_weight_max": 46.207462310791016,
      "activations/layer6_attention_weight_min": -48.080379486083984,
      "activations/layer7_attention_weight_max": 82.84379577636719,
      "activations/layer7_attention_weight_min": -87.93611907958984,
      "activations/layer8_attention_weight_max": 36.970306396484375,
      "activations/layer8_attention_weight_min": -39.54240036010742,
      "activations/layer9_attention_weight_max": 30.84749412536621,
      "activations/layer9_attention_weight_min": -30.959253311157227,
      "epoch": 9.76,
      "learning_rate": 8.791553030303029e-05,
      "loss": 2.7907,
      "step": 168050
    },
    {
      "activations/layer0_attention_weight_max": 15.358380317687988,
      "activations/layer0_attention_weight_min": -12.310431480407715,
      "activations/layer10_attention_weight_max": 31.451452255249023,
      "activations/layer10_attention_weight_min": -33.5465087890625,
      "activations/layer11_attention_weight_max": 32.35181427001953,
      "activations/layer11_attention_weight_min": -32.763084411621094,
      "activations/layer12_attention_weight_max": 24.58476448059082,
      "activations/layer12_attention_weight_min": -23.72171974182129,
      "activations/layer13_attention_weight_max": 40.03668975830078,
      "activations/layer13_attention_weight_min": -34.018272399902344,
      "activations/layer14_attention_weight_max": 47.69741439819336,
      "activations/layer14_attention_weight_min": -37.435997009277344,
      "activations/layer15_attention_weight_max": 38.8004150390625,
      "activations/layer15_attention_weight_min": -31.202775955200195,
      "activations/layer16_attention_weight_max": 31.71146583557129,
      "activations/layer16_attention_weight_min": -26.79617691040039,
      "activations/layer17_attention_weight_max": 32.8874397277832,
      "activations/layer17_attention_weight_min": -24.495309829711914,
      "activations/layer18_attention_weight_max": 32.787899017333984,
      "activations/layer18_attention_weight_min": -22.04178237915039,
      "activations/layer19_attention_weight_max": 35.97269058227539,
      "activations/layer19_attention_weight_min": -29.990537643432617,
      "activations/layer1_attention_weight_max": 15.90402603149414,
      "activations/layer1_attention_weight_min": -14.713518142700195,
      "activations/layer20_attention_weight_max": 33.897308349609375,
      "activations/layer20_attention_weight_min": -24.295957565307617,
      "activations/layer21_attention_weight_max": 31.30890464782715,
      "activations/layer21_attention_weight_min": -23.207260131835938,
      "activations/layer22_attention_weight_max": 49.6950798034668,
      "activations/layer22_attention_weight_min": -29.32132911682129,
      "activations/layer23_attention_weight_max": 35.86602020263672,
      "activations/layer23_attention_weight_min": -23.6102294921875,
      "activations/layer2_attention_weight_max": 32.434059143066406,
      "activations/layer2_attention_weight_min": -34.12408447265625,
      "activations/layer3_attention_weight_max": 95.94689178466797,
      "activations/layer3_attention_weight_min": -99.77786254882812,
      "activations/layer4_attention_weight_max": 117.1172866821289,
      "activations/layer4_attention_weight_min": -115.28533172607422,
      "activations/layer5_attention_weight_max": 53.639434814453125,
      "activations/layer5_attention_weight_min": -65.56730651855469,
      "activations/layer6_attention_weight_max": 47.13867950439453,
      "activations/layer6_attention_weight_min": -52.58115768432617,
      "activations/layer7_attention_weight_max": 90.03392791748047,
      "activations/layer7_attention_weight_min": -96.94044494628906,
      "activations/layer8_attention_weight_max": 40.92479705810547,
      "activations/layer8_attention_weight_min": -42.620384216308594,
      "activations/layer9_attention_weight_max": 33.72789001464844,
      "activations/layer9_attention_weight_min": -35.570980072021484,
      "epoch": 9.77,
      "learning_rate": 8.78965909090909e-05,
      "loss": 2.805,
      "step": 168100
    },
    {
      "activations/layer0_attention_weight_max": 15.69591236114502,
      "activations/layer0_attention_weight_min": -12.069005012512207,
      "activations/layer10_attention_weight_max": 32.17875671386719,
      "activations/layer10_attention_weight_min": -30.55489730834961,
      "activations/layer11_attention_weight_max": 35.207149505615234,
      "activations/layer11_attention_weight_min": -30.74616813659668,
      "activations/layer12_attention_weight_max": 22.23119354248047,
      "activations/layer12_attention_weight_min": -23.8271427154541,
      "activations/layer13_attention_weight_max": 37.5659294128418,
      "activations/layer13_attention_weight_min": -30.95868682861328,
      "activations/layer14_attention_weight_max": 44.51552963256836,
      "activations/layer14_attention_weight_min": -36.92969512939453,
      "activations/layer15_attention_weight_max": 37.44955825805664,
      "activations/layer15_attention_weight_min": -29.5312442779541,
      "activations/layer16_attention_weight_max": 32.05721664428711,
      "activations/layer16_attention_weight_min": -28.04015350341797,
      "activations/layer17_attention_weight_max": 34.89448165893555,
      "activations/layer17_attention_weight_min": -23.479684829711914,
      "activations/layer18_attention_weight_max": 35.20643615722656,
      "activations/layer18_attention_weight_min": -22.407705307006836,
      "activations/layer19_attention_weight_max": 38.2569694519043,
      "activations/layer19_attention_weight_min": -31.445755004882812,
      "activations/layer1_attention_weight_max": 16.640687942504883,
      "activations/layer1_attention_weight_min": -14.759476661682129,
      "activations/layer20_attention_weight_max": 30.8853702545166,
      "activations/layer20_attention_weight_min": -24.89406394958496,
      "activations/layer21_attention_weight_max": 30.669492721557617,
      "activations/layer21_attention_weight_min": -26.073171615600586,
      "activations/layer22_attention_weight_max": 51.71122360229492,
      "activations/layer22_attention_weight_min": -32.12015914916992,
      "activations/layer23_attention_weight_max": 39.96894073486328,
      "activations/layer23_attention_weight_min": -26.19550895690918,
      "activations/layer2_attention_weight_max": 32.10234832763672,
      "activations/layer2_attention_weight_min": -30.482311248779297,
      "activations/layer3_attention_weight_max": 93.91728210449219,
      "activations/layer3_attention_weight_min": -88.6971664428711,
      "activations/layer4_attention_weight_max": 116.00988006591797,
      "activations/layer4_attention_weight_min": -107.3641128540039,
      "activations/layer5_attention_weight_max": 55.896873474121094,
      "activations/layer5_attention_weight_min": -60.890777587890625,
      "activations/layer6_attention_weight_max": 45.636940002441406,
      "activations/layer6_attention_weight_min": -49.28723907470703,
      "activations/layer7_attention_weight_max": 88.68636322021484,
      "activations/layer7_attention_weight_min": -91.64189147949219,
      "activations/layer8_attention_weight_max": 38.58574676513672,
      "activations/layer8_attention_weight_min": -41.785099029541016,
      "activations/layer9_attention_weight_max": 35.3067741394043,
      "activations/layer9_attention_weight_min": -32.44149398803711,
      "epoch": 9.77,
      "learning_rate": 8.787765151515152e-05,
      "loss": 2.7915,
      "step": 168150
    },
    {
      "activations/layer0_attention_weight_max": 15.222418785095215,
      "activations/layer0_attention_weight_min": -11.53089714050293,
      "activations/layer10_attention_weight_max": 36.46809005737305,
      "activations/layer10_attention_weight_min": -33.16447830200195,
      "activations/layer11_attention_weight_max": 33.273746490478516,
      "activations/layer11_attention_weight_min": -32.36676788330078,
      "activations/layer12_attention_weight_max": 27.880802154541016,
      "activations/layer12_attention_weight_min": -26.737367630004883,
      "activations/layer13_attention_weight_max": 47.91661071777344,
      "activations/layer13_attention_weight_min": -34.74204635620117,
      "activations/layer14_attention_weight_max": 55.74851989746094,
      "activations/layer14_attention_weight_min": -39.850799560546875,
      "activations/layer15_attention_weight_max": 40.5438346862793,
      "activations/layer15_attention_weight_min": -31.07268524169922,
      "activations/layer16_attention_weight_max": 35.83946990966797,
      "activations/layer16_attention_weight_min": -26.90375328063965,
      "activations/layer17_attention_weight_max": 34.80180740356445,
      "activations/layer17_attention_weight_min": -26.311519622802734,
      "activations/layer18_attention_weight_max": 33.94099426269531,
      "activations/layer18_attention_weight_min": -22.96017074584961,
      "activations/layer19_attention_weight_max": 38.00675582885742,
      "activations/layer19_attention_weight_min": -32.138187408447266,
      "activations/layer1_attention_weight_max": 16.989742279052734,
      "activations/layer1_attention_weight_min": -13.969707489013672,
      "activations/layer20_attention_weight_max": 31.264427185058594,
      "activations/layer20_attention_weight_min": -26.291309356689453,
      "activations/layer21_attention_weight_max": 32.4765625,
      "activations/layer21_attention_weight_min": -22.68118667602539,
      "activations/layer22_attention_weight_max": 50.08554458618164,
      "activations/layer22_attention_weight_min": -30.03276824951172,
      "activations/layer23_attention_weight_max": 41.33995819091797,
      "activations/layer23_attention_weight_min": -25.284528732299805,
      "activations/layer2_attention_weight_max": 34.02573776245117,
      "activations/layer2_attention_weight_min": -34.84388732910156,
      "activations/layer3_attention_weight_max": 96.38645935058594,
      "activations/layer3_attention_weight_min": -99.81842803955078,
      "activations/layer4_attention_weight_max": 117.69392395019531,
      "activations/layer4_attention_weight_min": -116.80073547363281,
      "activations/layer5_attention_weight_max": 54.190521240234375,
      "activations/layer5_attention_weight_min": -61.966163635253906,
      "activations/layer6_attention_weight_max": 45.53572082519531,
      "activations/layer6_attention_weight_min": -48.5219841003418,
      "activations/layer7_attention_weight_max": 91.74021911621094,
      "activations/layer7_attention_weight_min": -89.23275756835938,
      "activations/layer8_attention_weight_max": 41.404014587402344,
      "activations/layer8_attention_weight_min": -44.41219711303711,
      "activations/layer9_attention_weight_max": 36.5545654296875,
      "activations/layer9_attention_weight_min": -35.26425552368164,
      "epoch": 9.77,
      "learning_rate": 8.785871212121211e-05,
      "loss": 2.7847,
      "step": 168200
    },
    {
      "activations/layer0_attention_weight_max": 15.444124221801758,
      "activations/layer0_attention_weight_min": -12.196215629577637,
      "activations/layer10_attention_weight_max": 32.678184509277344,
      "activations/layer10_attention_weight_min": -34.903053283691406,
      "activations/layer11_attention_weight_max": 33.231910705566406,
      "activations/layer11_attention_weight_min": -31.31517791748047,
      "activations/layer12_attention_weight_max": 24.50503158569336,
      "activations/layer12_attention_weight_min": -25.57839584350586,
      "activations/layer13_attention_weight_max": 40.59895324707031,
      "activations/layer13_attention_weight_min": -34.436973571777344,
      "activations/layer14_attention_weight_max": 45.95083999633789,
      "activations/layer14_attention_weight_min": -37.91848373413086,
      "activations/layer15_attention_weight_max": 37.863224029541016,
      "activations/layer15_attention_weight_min": -32.350738525390625,
      "activations/layer16_attention_weight_max": 33.433589935302734,
      "activations/layer16_attention_weight_min": -28.231998443603516,
      "activations/layer17_attention_weight_max": 34.18256378173828,
      "activations/layer17_attention_weight_min": -27.097864151000977,
      "activations/layer18_attention_weight_max": 31.323287963867188,
      "activations/layer18_attention_weight_min": -23.323936462402344,
      "activations/layer19_attention_weight_max": 39.28995895385742,
      "activations/layer19_attention_weight_min": -33.3111457824707,
      "activations/layer1_attention_weight_max": 18.207836151123047,
      "activations/layer1_attention_weight_min": -16.50888442993164,
      "activations/layer20_attention_weight_max": 34.42708206176758,
      "activations/layer20_attention_weight_min": -27.939769744873047,
      "activations/layer21_attention_weight_max": 30.80663299560547,
      "activations/layer21_attention_weight_min": -25.068832397460938,
      "activations/layer22_attention_weight_max": 50.72195816040039,
      "activations/layer22_attention_weight_min": -30.662364959716797,
      "activations/layer23_attention_weight_max": 36.53184509277344,
      "activations/layer23_attention_weight_min": -25.943235397338867,
      "activations/layer2_attention_weight_max": 35.520469665527344,
      "activations/layer2_attention_weight_min": -31.6980037689209,
      "activations/layer3_attention_weight_max": 96.98491668701172,
      "activations/layer3_attention_weight_min": -93.34258270263672,
      "activations/layer4_attention_weight_max": 122.87670135498047,
      "activations/layer4_attention_weight_min": -110.91890716552734,
      "activations/layer5_attention_weight_max": 55.659664154052734,
      "activations/layer5_attention_weight_min": -60.03327178955078,
      "activations/layer6_attention_weight_max": 48.5989875793457,
      "activations/layer6_attention_weight_min": -48.93800354003906,
      "activations/layer7_attention_weight_max": 93.16879272460938,
      "activations/layer7_attention_weight_min": -92.4104232788086,
      "activations/layer8_attention_weight_max": 41.42764663696289,
      "activations/layer8_attention_weight_min": -41.21726989746094,
      "activations/layer9_attention_weight_max": 34.59043502807617,
      "activations/layer9_attention_weight_min": -34.10496520996094,
      "epoch": 9.78,
      "learning_rate": 8.783977272727272e-05,
      "loss": 2.7983,
      "step": 168250
    },
    {
      "activations/layer0_attention_weight_max": 15.327966690063477,
      "activations/layer0_attention_weight_min": -12.787550926208496,
      "activations/layer10_attention_weight_max": 30.046245574951172,
      "activations/layer10_attention_weight_min": -30.213741302490234,
      "activations/layer11_attention_weight_max": 31.116161346435547,
      "activations/layer11_attention_weight_min": -31.248252868652344,
      "activations/layer12_attention_weight_max": 30.497583389282227,
      "activations/layer12_attention_weight_min": -26.726259231567383,
      "activations/layer13_attention_weight_max": 36.5179443359375,
      "activations/layer13_attention_weight_min": -33.24400329589844,
      "activations/layer14_attention_weight_max": 43.499114990234375,
      "activations/layer14_attention_weight_min": -34.78862762451172,
      "activations/layer15_attention_weight_max": 33.176300048828125,
      "activations/layer15_attention_weight_min": -31.582258224487305,
      "activations/layer16_attention_weight_max": 32.61252212524414,
      "activations/layer16_attention_weight_min": -29.170461654663086,
      "activations/layer17_attention_weight_max": 31.674903869628906,
      "activations/layer17_attention_weight_min": -25.441896438598633,
      "activations/layer18_attention_weight_max": 30.259531021118164,
      "activations/layer18_attention_weight_min": -22.071918487548828,
      "activations/layer19_attention_weight_max": 35.94141387939453,
      "activations/layer19_attention_weight_min": -29.52341651916504,
      "activations/layer1_attention_weight_max": 17.08749008178711,
      "activations/layer1_attention_weight_min": -16.228811264038086,
      "activations/layer20_attention_weight_max": 28.169986724853516,
      "activations/layer20_attention_weight_min": -25.747913360595703,
      "activations/layer21_attention_weight_max": 30.307510375976562,
      "activations/layer21_attention_weight_min": -23.9910888671875,
      "activations/layer22_attention_weight_max": 40.234222412109375,
      "activations/layer22_attention_weight_min": -30.13387680053711,
      "activations/layer23_attention_weight_max": 39.455108642578125,
      "activations/layer23_attention_weight_min": -23.541439056396484,
      "activations/layer2_attention_weight_max": 30.916980743408203,
      "activations/layer2_attention_weight_min": -33.37403106689453,
      "activations/layer3_attention_weight_max": 90.47740173339844,
      "activations/layer3_attention_weight_min": -97.0674819946289,
      "activations/layer4_attention_weight_max": 107.56166076660156,
      "activations/layer4_attention_weight_min": -108.83441162109375,
      "activations/layer5_attention_weight_max": 52.612083435058594,
      "activations/layer5_attention_weight_min": -65.2035903930664,
      "activations/layer6_attention_weight_max": 45.9578742980957,
      "activations/layer6_attention_weight_min": -49.73790740966797,
      "activations/layer7_attention_weight_max": 89.04434967041016,
      "activations/layer7_attention_weight_min": -87.85536193847656,
      "activations/layer8_attention_weight_max": 37.08464050292969,
      "activations/layer8_attention_weight_min": -38.81212615966797,
      "activations/layer9_attention_weight_max": 32.29941177368164,
      "activations/layer9_attention_weight_min": -32.03105163574219,
      "epoch": 9.78,
      "learning_rate": 8.782083333333332e-05,
      "loss": 2.7978,
      "step": 168300
    },
    {
      "activations/layer0_attention_weight_max": 15.797747611999512,
      "activations/layer0_attention_weight_min": -12.258310317993164,
      "activations/layer10_attention_weight_max": 31.079320907592773,
      "activations/layer10_attention_weight_min": -32.57810974121094,
      "activations/layer11_attention_weight_max": 32.78425216674805,
      "activations/layer11_attention_weight_min": -30.221906661987305,
      "activations/layer12_attention_weight_max": 38.80887985229492,
      "activations/layer12_attention_weight_min": -30.42461585998535,
      "activations/layer13_attention_weight_max": 44.827247619628906,
      "activations/layer13_attention_weight_min": -32.518287658691406,
      "activations/layer14_attention_weight_max": 42.055450439453125,
      "activations/layer14_attention_weight_min": -38.50102996826172,
      "activations/layer15_attention_weight_max": 35.69667053222656,
      "activations/layer15_attention_weight_min": -31.673612594604492,
      "activations/layer16_attention_weight_max": 29.53538703918457,
      "activations/layer16_attention_weight_min": -26.9407901763916,
      "activations/layer17_attention_weight_max": 32.64916229248047,
      "activations/layer17_attention_weight_min": -26.287126541137695,
      "activations/layer18_attention_weight_max": 30.27785873413086,
      "activations/layer18_attention_weight_min": -24.471132278442383,
      "activations/layer19_attention_weight_max": 33.970298767089844,
      "activations/layer19_attention_weight_min": -31.32608413696289,
      "activations/layer1_attention_weight_max": 17.65926170349121,
      "activations/layer1_attention_weight_min": -14.41503620147705,
      "activations/layer20_attention_weight_max": 29.088722229003906,
      "activations/layer20_attention_weight_min": -25.340721130371094,
      "activations/layer21_attention_weight_max": 29.015499114990234,
      "activations/layer21_attention_weight_min": -24.876415252685547,
      "activations/layer22_attention_weight_max": 39.746177673339844,
      "activations/layer22_attention_weight_min": -30.978179931640625,
      "activations/layer23_attention_weight_max": 35.51353454589844,
      "activations/layer23_attention_weight_min": -23.309873580932617,
      "activations/layer2_attention_weight_max": 32.21234130859375,
      "activations/layer2_attention_weight_min": -31.99871826171875,
      "activations/layer3_attention_weight_max": 92.31275939941406,
      "activations/layer3_attention_weight_min": -92.2332534790039,
      "activations/layer4_attention_weight_max": 114.0989990234375,
      "activations/layer4_attention_weight_min": -112.8515625,
      "activations/layer5_attention_weight_max": 51.26581573486328,
      "activations/layer5_attention_weight_min": -59.95256805419922,
      "activations/layer6_attention_weight_max": 44.611671447753906,
      "activations/layer6_attention_weight_min": -48.585384368896484,
      "activations/layer7_attention_weight_max": 90.7938003540039,
      "activations/layer7_attention_weight_min": -84.81733703613281,
      "activations/layer8_attention_weight_max": 38.73780822753906,
      "activations/layer8_attention_weight_min": -40.34088897705078,
      "activations/layer9_attention_weight_max": 34.717559814453125,
      "activations/layer9_attention_weight_min": -36.20151138305664,
      "epoch": 9.78,
      "learning_rate": 8.780189393939393e-05,
      "loss": 2.8,
      "step": 168350
    },
    {
      "activations/layer0_attention_weight_max": 15.260258674621582,
      "activations/layer0_attention_weight_min": -11.982983589172363,
      "activations/layer10_attention_weight_max": 31.205644607543945,
      "activations/layer10_attention_weight_min": -31.005126953125,
      "activations/layer11_attention_weight_max": 30.761493682861328,
      "activations/layer11_attention_weight_min": -30.164501190185547,
      "activations/layer12_attention_weight_max": 22.39472198486328,
      "activations/layer12_attention_weight_min": -25.47553825378418,
      "activations/layer13_attention_weight_max": 38.365535736083984,
      "activations/layer13_attention_weight_min": -34.844669342041016,
      "activations/layer14_attention_weight_max": 41.46717071533203,
      "activations/layer14_attention_weight_min": -35.49353790283203,
      "activations/layer15_attention_weight_max": 32.99396514892578,
      "activations/layer15_attention_weight_min": -31.88654899597168,
      "activations/layer16_attention_weight_max": 31.632917404174805,
      "activations/layer16_attention_weight_min": -26.11412811279297,
      "activations/layer17_attention_weight_max": 30.440643310546875,
      "activations/layer17_attention_weight_min": -26.56537437438965,
      "activations/layer18_attention_weight_max": 33.020782470703125,
      "activations/layer18_attention_weight_min": -21.270565032958984,
      "activations/layer19_attention_weight_max": 37.624664306640625,
      "activations/layer19_attention_weight_min": -29.520715713500977,
      "activations/layer1_attention_weight_max": 16.882349014282227,
      "activations/layer1_attention_weight_min": -15.136539459228516,
      "activations/layer20_attention_weight_max": 29.505504608154297,
      "activations/layer20_attention_weight_min": -21.965669631958008,
      "activations/layer21_attention_weight_max": 27.706016540527344,
      "activations/layer21_attention_weight_min": -22.077821731567383,
      "activations/layer22_attention_weight_max": 42.99957275390625,
      "activations/layer22_attention_weight_min": -28.410127639770508,
      "activations/layer23_attention_weight_max": 34.24928283691406,
      "activations/layer23_attention_weight_min": -23.758100509643555,
      "activations/layer2_attention_weight_max": 33.01270294189453,
      "activations/layer2_attention_weight_min": -32.585693359375,
      "activations/layer3_attention_weight_max": 93.85582733154297,
      "activations/layer3_attention_weight_min": -94.98357391357422,
      "activations/layer4_attention_weight_max": 112.87259674072266,
      "activations/layer4_attention_weight_min": -108.77245330810547,
      "activations/layer5_attention_weight_max": 52.512001037597656,
      "activations/layer5_attention_weight_min": -63.64716720581055,
      "activations/layer6_attention_weight_max": 45.78527069091797,
      "activations/layer6_attention_weight_min": -47.283851623535156,
      "activations/layer7_attention_weight_max": 90.82276916503906,
      "activations/layer7_attention_weight_min": -85.7566146850586,
      "activations/layer8_attention_weight_max": 39.35799026489258,
      "activations/layer8_attention_weight_min": -38.91920852661133,
      "activations/layer9_attention_weight_max": 33.840858459472656,
      "activations/layer9_attention_weight_min": -33.43537902832031,
      "epoch": 9.78,
      "learning_rate": 8.778295454545454e-05,
      "loss": 2.802,
      "step": 168400
    },
    {
      "activations/layer0_attention_weight_max": 15.801924705505371,
      "activations/layer0_attention_weight_min": -12.299628257751465,
      "activations/layer10_attention_weight_max": 35.7420539855957,
      "activations/layer10_attention_weight_min": -33.910804748535156,
      "activations/layer11_attention_weight_max": 32.639610290527344,
      "activations/layer11_attention_weight_min": -32.22361373901367,
      "activations/layer12_attention_weight_max": 23.93238639831543,
      "activations/layer12_attention_weight_min": -24.40782928466797,
      "activations/layer13_attention_weight_max": 38.671443939208984,
      "activations/layer13_attention_weight_min": -33.983856201171875,
      "activations/layer14_attention_weight_max": 51.43560791015625,
      "activations/layer14_attention_weight_min": -39.014671325683594,
      "activations/layer15_attention_weight_max": 35.697113037109375,
      "activations/layer15_attention_weight_min": -31.072908401489258,
      "activations/layer16_attention_weight_max": 31.988235473632812,
      "activations/layer16_attention_weight_min": -27.549650192260742,
      "activations/layer17_attention_weight_max": 35.4655876159668,
      "activations/layer17_attention_weight_min": -26.46825408935547,
      "activations/layer18_attention_weight_max": 30.486135482788086,
      "activations/layer18_attention_weight_min": -22.4083251953125,
      "activations/layer19_attention_weight_max": 33.678279876708984,
      "activations/layer19_attention_weight_min": -33.35042953491211,
      "activations/layer1_attention_weight_max": 17.134342193603516,
      "activations/layer1_attention_weight_min": -15.448770523071289,
      "activations/layer20_attention_weight_max": 31.575780868530273,
      "activations/layer20_attention_weight_min": -26.83538246154785,
      "activations/layer21_attention_weight_max": 28.392393112182617,
      "activations/layer21_attention_weight_min": -24.475961685180664,
      "activations/layer22_attention_weight_max": 45.2461051940918,
      "activations/layer22_attention_weight_min": -33.690555572509766,
      "activations/layer23_attention_weight_max": 36.25760269165039,
      "activations/layer23_attention_weight_min": -24.976491928100586,
      "activations/layer2_attention_weight_max": 31.876705169677734,
      "activations/layer2_attention_weight_min": -32.13959884643555,
      "activations/layer3_attention_weight_max": 91.80831909179688,
      "activations/layer3_attention_weight_min": -94.19338989257812,
      "activations/layer4_attention_weight_max": 110.6953125,
      "activations/layer4_attention_weight_min": -111.5682144165039,
      "activations/layer5_attention_weight_max": 53.245582580566406,
      "activations/layer5_attention_weight_min": -62.93288803100586,
      "activations/layer6_attention_weight_max": 48.11861801147461,
      "activations/layer6_attention_weight_min": -50.48192596435547,
      "activations/layer7_attention_weight_max": 89.39698791503906,
      "activations/layer7_attention_weight_min": -89.07388305664062,
      "activations/layer8_attention_weight_max": 41.519744873046875,
      "activations/layer8_attention_weight_min": -43.53577423095703,
      "activations/layer9_attention_weight_max": 38.07414627075195,
      "activations/layer9_attention_weight_min": -35.45890426635742,
      "epoch": 9.79,
      "learning_rate": 8.776401515151514e-05,
      "loss": 2.7655,
      "step": 168450
    },
    {
      "activations/layer0_attention_weight_max": 16.71462059020996,
      "activations/layer0_attention_weight_min": -12.431589126586914,
      "activations/layer10_attention_weight_max": 31.89239501953125,
      "activations/layer10_attention_weight_min": -34.49118423461914,
      "activations/layer11_attention_weight_max": 30.511127471923828,
      "activations/layer11_attention_weight_min": -31.79035758972168,
      "activations/layer12_attention_weight_max": 34.384910583496094,
      "activations/layer12_attention_weight_min": -26.33507537841797,
      "activations/layer13_attention_weight_max": 43.522525787353516,
      "activations/layer13_attention_weight_min": -35.07979202270508,
      "activations/layer14_attention_weight_max": 44.48604965209961,
      "activations/layer14_attention_weight_min": -39.9168815612793,
      "activations/layer15_attention_weight_max": 36.182334899902344,
      "activations/layer15_attention_weight_min": -31.735851287841797,
      "activations/layer16_attention_weight_max": 30.2596435546875,
      "activations/layer16_attention_weight_min": -27.791683197021484,
      "activations/layer17_attention_weight_max": 33.28055191040039,
      "activations/layer17_attention_weight_min": -24.53243637084961,
      "activations/layer18_attention_weight_max": 29.057212829589844,
      "activations/layer18_attention_weight_min": -23.412477493286133,
      "activations/layer19_attention_weight_max": 34.730648040771484,
      "activations/layer19_attention_weight_min": -30.732275009155273,
      "activations/layer1_attention_weight_max": 15.947672843933105,
      "activations/layer1_attention_weight_min": -14.64256477355957,
      "activations/layer20_attention_weight_max": 29.282535552978516,
      "activations/layer20_attention_weight_min": -26.73922348022461,
      "activations/layer21_attention_weight_max": 29.828426361083984,
      "activations/layer21_attention_weight_min": -26.06874656677246,
      "activations/layer22_attention_weight_max": 42.875633239746094,
      "activations/layer22_attention_weight_min": -31.124441146850586,
      "activations/layer23_attention_weight_max": 33.488346099853516,
      "activations/layer23_attention_weight_min": -24.052921295166016,
      "activations/layer2_attention_weight_max": 33.185813903808594,
      "activations/layer2_attention_weight_min": -33.338218688964844,
      "activations/layer3_attention_weight_max": 92.19192504882812,
      "activations/layer3_attention_weight_min": -94.2525405883789,
      "activations/layer4_attention_weight_max": 113.49718475341797,
      "activations/layer4_attention_weight_min": -119.52510833740234,
      "activations/layer5_attention_weight_max": 55.179603576660156,
      "activations/layer5_attention_weight_min": -63.79598617553711,
      "activations/layer6_attention_weight_max": 46.17869186401367,
      "activations/layer6_attention_weight_min": -49.77802658081055,
      "activations/layer7_attention_weight_max": 92.1160659790039,
      "activations/layer7_attention_weight_min": -92.49638366699219,
      "activations/layer8_attention_weight_max": 40.010963439941406,
      "activations/layer8_attention_weight_min": -39.935699462890625,
      "activations/layer9_attention_weight_max": 35.09269332885742,
      "activations/layer9_attention_weight_min": -34.91672134399414,
      "epoch": 9.79,
      "learning_rate": 8.774507575757574e-05,
      "loss": 2.79,
      "step": 168500
    },
    {
      "activations/layer0_attention_weight_max": 15.906177520751953,
      "activations/layer0_attention_weight_min": -11.70228385925293,
      "activations/layer10_attention_weight_max": 34.69548416137695,
      "activations/layer10_attention_weight_min": -33.2296257019043,
      "activations/layer11_attention_weight_max": 32.63903045654297,
      "activations/layer11_attention_weight_min": -32.16224670410156,
      "activations/layer12_attention_weight_max": 27.90304183959961,
      "activations/layer12_attention_weight_min": -27.084884643554688,
      "activations/layer13_attention_weight_max": 38.556427001953125,
      "activations/layer13_attention_weight_min": -32.27781677246094,
      "activations/layer14_attention_weight_max": 48.20808792114258,
      "activations/layer14_attention_weight_min": -37.949317932128906,
      "activations/layer15_attention_weight_max": 37.170284271240234,
      "activations/layer15_attention_weight_min": -31.80477523803711,
      "activations/layer16_attention_weight_max": 32.519081115722656,
      "activations/layer16_attention_weight_min": -27.655426025390625,
      "activations/layer17_attention_weight_max": 33.38363265991211,
      "activations/layer17_attention_weight_min": -27.962142944335938,
      "activations/layer18_attention_weight_max": 30.735525131225586,
      "activations/layer18_attention_weight_min": -22.16526222229004,
      "activations/layer19_attention_weight_max": 35.45059585571289,
      "activations/layer19_attention_weight_min": -30.303049087524414,
      "activations/layer1_attention_weight_max": 16.862380981445312,
      "activations/layer1_attention_weight_min": -14.409165382385254,
      "activations/layer20_attention_weight_max": 29.352588653564453,
      "activations/layer20_attention_weight_min": -24.150781631469727,
      "activations/layer21_attention_weight_max": 33.014163970947266,
      "activations/layer21_attention_weight_min": -24.459897994995117,
      "activations/layer22_attention_weight_max": 43.32069396972656,
      "activations/layer22_attention_weight_min": -30.234773635864258,
      "activations/layer23_attention_weight_max": 33.60643768310547,
      "activations/layer23_attention_weight_min": -24.78322982788086,
      "activations/layer2_attention_weight_max": 35.822105407714844,
      "activations/layer2_attention_weight_min": -32.972930908203125,
      "activations/layer3_attention_weight_max": 91.82040405273438,
      "activations/layer3_attention_weight_min": -93.5206298828125,
      "activations/layer4_attention_weight_max": 112.85749816894531,
      "activations/layer4_attention_weight_min": -111.219482421875,
      "activations/layer5_attention_weight_max": 52.65964889526367,
      "activations/layer5_attention_weight_min": -59.90853500366211,
      "activations/layer6_attention_weight_max": 45.83287048339844,
      "activations/layer6_attention_weight_min": -51.76740646362305,
      "activations/layer7_attention_weight_max": 89.49715423583984,
      "activations/layer7_attention_weight_min": -92.28185272216797,
      "activations/layer8_attention_weight_max": 39.71190643310547,
      "activations/layer8_attention_weight_min": -43.204986572265625,
      "activations/layer9_attention_weight_max": 32.540679931640625,
      "activations/layer9_attention_weight_min": -33.50562286376953,
      "epoch": 9.79,
      "learning_rate": 8.772613636363636e-05,
      "loss": 2.7952,
      "step": 168550
    },
    {
      "activations/layer0_attention_weight_max": 16.684640884399414,
      "activations/layer0_attention_weight_min": -13.007248878479004,
      "activations/layer10_attention_weight_max": 31.101131439208984,
      "activations/layer10_attention_weight_min": -31.875865936279297,
      "activations/layer11_attention_weight_max": 31.29646873474121,
      "activations/layer11_attention_weight_min": -30.663223266601562,
      "activations/layer12_attention_weight_max": 23.36099624633789,
      "activations/layer12_attention_weight_min": -26.04405403137207,
      "activations/layer13_attention_weight_max": 37.37849807739258,
      "activations/layer13_attention_weight_min": -34.98273468017578,
      "activations/layer14_attention_weight_max": 43.04114532470703,
      "activations/layer14_attention_weight_min": -37.320655822753906,
      "activations/layer15_attention_weight_max": 36.36025619506836,
      "activations/layer15_attention_weight_min": -33.08404541015625,
      "activations/layer16_attention_weight_max": 32.56303024291992,
      "activations/layer16_attention_weight_min": -28.949331283569336,
      "activations/layer17_attention_weight_max": 38.891571044921875,
      "activations/layer17_attention_weight_min": -27.05393409729004,
      "activations/layer18_attention_weight_max": 38.143863677978516,
      "activations/layer18_attention_weight_min": -23.488054275512695,
      "activations/layer19_attention_weight_max": 39.71010971069336,
      "activations/layer19_attention_weight_min": -33.176082611083984,
      "activations/layer1_attention_weight_max": 16.50416374206543,
      "activations/layer1_attention_weight_min": -14.579545021057129,
      "activations/layer20_attention_weight_max": 30.829206466674805,
      "activations/layer20_attention_weight_min": -25.819551467895508,
      "activations/layer21_attention_weight_max": 28.31557273864746,
      "activations/layer21_attention_weight_min": -23.5621395111084,
      "activations/layer22_attention_weight_max": 47.02315139770508,
      "activations/layer22_attention_weight_min": -29.907020568847656,
      "activations/layer23_attention_weight_max": 37.45489501953125,
      "activations/layer23_attention_weight_min": -29.592031478881836,
      "activations/layer2_attention_weight_max": 31.589994430541992,
      "activations/layer2_attention_weight_min": -32.38372039794922,
      "activations/layer3_attention_weight_max": 94.76337432861328,
      "activations/layer3_attention_weight_min": -95.8467788696289,
      "activations/layer4_attention_weight_max": 115.28144836425781,
      "activations/layer4_attention_weight_min": -109.99555969238281,
      "activations/layer5_attention_weight_max": 50.79230499267578,
      "activations/layer5_attention_weight_min": -59.69508361816406,
      "activations/layer6_attention_weight_max": 43.94615936279297,
      "activations/layer6_attention_weight_min": -47.243186950683594,
      "activations/layer7_attention_weight_max": 83.29976654052734,
      "activations/layer7_attention_weight_min": -85.25593566894531,
      "activations/layer8_attention_weight_max": 38.19171142578125,
      "activations/layer8_attention_weight_min": -41.51181411743164,
      "activations/layer9_attention_weight_max": 32.38863754272461,
      "activations/layer9_attention_weight_min": -33.69630813598633,
      "epoch": 9.8,
      "learning_rate": 8.770719696969696e-05,
      "loss": 2.7841,
      "step": 168600
    },
    {
      "activations/layer0_attention_weight_max": 14.615303993225098,
      "activations/layer0_attention_weight_min": -12.288495063781738,
      "activations/layer10_attention_weight_max": 31.115196228027344,
      "activations/layer10_attention_weight_min": -31.375646591186523,
      "activations/layer11_attention_weight_max": 31.9761962890625,
      "activations/layer11_attention_weight_min": -29.440357208251953,
      "activations/layer12_attention_weight_max": 27.737699508666992,
      "activations/layer12_attention_weight_min": -30.94721221923828,
      "activations/layer13_attention_weight_max": 35.513328552246094,
      "activations/layer13_attention_weight_min": -31.728504180908203,
      "activations/layer14_attention_weight_max": 40.91962432861328,
      "activations/layer14_attention_weight_min": -34.899139404296875,
      "activations/layer15_attention_weight_max": 34.289024353027344,
      "activations/layer15_attention_weight_min": -31.005970001220703,
      "activations/layer16_attention_weight_max": 29.57743263244629,
      "activations/layer16_attention_weight_min": -28.152143478393555,
      "activations/layer17_attention_weight_max": 33.75517272949219,
      "activations/layer17_attention_weight_min": -25.67601203918457,
      "activations/layer18_attention_weight_max": 31.132497787475586,
      "activations/layer18_attention_weight_min": -22.201059341430664,
      "activations/layer19_attention_weight_max": 34.82331466674805,
      "activations/layer19_attention_weight_min": -30.503061294555664,
      "activations/layer1_attention_weight_max": 16.661775588989258,
      "activations/layer1_attention_weight_min": -15.026232719421387,
      "activations/layer20_attention_weight_max": 30.276670455932617,
      "activations/layer20_attention_weight_min": -25.7624568939209,
      "activations/layer21_attention_weight_max": 28.057523727416992,
      "activations/layer21_attention_weight_min": -25.036550521850586,
      "activations/layer22_attention_weight_max": 41.755126953125,
      "activations/layer22_attention_weight_min": -28.542280197143555,
      "activations/layer23_attention_weight_max": 33.20354461669922,
      "activations/layer23_attention_weight_min": -25.010425567626953,
      "activations/layer2_attention_weight_max": 33.08457946777344,
      "activations/layer2_attention_weight_min": -33.16068649291992,
      "activations/layer3_attention_weight_max": 94.38218688964844,
      "activations/layer3_attention_weight_min": -93.18408203125,
      "activations/layer4_attention_weight_max": 114.87313079833984,
      "activations/layer4_attention_weight_min": -111.31746673583984,
      "activations/layer5_attention_weight_max": 54.911109924316406,
      "activations/layer5_attention_weight_min": -61.78905487060547,
      "activations/layer6_attention_weight_max": 47.707801818847656,
      "activations/layer6_attention_weight_min": -48.23057556152344,
      "activations/layer7_attention_weight_max": 90.0601806640625,
      "activations/layer7_attention_weight_min": -84.08129119873047,
      "activations/layer8_attention_weight_max": 40.35592269897461,
      "activations/layer8_attention_weight_min": -40.01782989501953,
      "activations/layer9_attention_weight_max": 32.168304443359375,
      "activations/layer9_attention_weight_min": -30.943185806274414,
      "epoch": 9.8,
      "learning_rate": 8.768825757575758e-05,
      "loss": 2.7913,
      "step": 168650
    },
    {
      "activations/layer0_attention_weight_max": 16.26188087463379,
      "activations/layer0_attention_weight_min": -13.393784523010254,
      "activations/layer10_attention_weight_max": 32.5521125793457,
      "activations/layer10_attention_weight_min": -31.97467041015625,
      "activations/layer11_attention_weight_max": 32.38492965698242,
      "activations/layer11_attention_weight_min": -31.076635360717773,
      "activations/layer12_attention_weight_max": 23.4942626953125,
      "activations/layer12_attention_weight_min": -22.678869247436523,
      "activations/layer13_attention_weight_max": 35.99397277832031,
      "activations/layer13_attention_weight_min": -32.559261322021484,
      "activations/layer14_attention_weight_max": 51.86129379272461,
      "activations/layer14_attention_weight_min": -39.630401611328125,
      "activations/layer15_attention_weight_max": 36.46553039550781,
      "activations/layer15_attention_weight_min": -30.586164474487305,
      "activations/layer16_attention_weight_max": 32.90409469604492,
      "activations/layer16_attention_weight_min": -29.41476058959961,
      "activations/layer17_attention_weight_max": 33.32331848144531,
      "activations/layer17_attention_weight_min": -26.27229118347168,
      "activations/layer18_attention_weight_max": 30.337167739868164,
      "activations/layer18_attention_weight_min": -22.507835388183594,
      "activations/layer19_attention_weight_max": 38.34223937988281,
      "activations/layer19_attention_weight_min": -32.61845016479492,
      "activations/layer1_attention_weight_max": 16.800830841064453,
      "activations/layer1_attention_weight_min": -14.79478931427002,
      "activations/layer20_attention_weight_max": 33.53131103515625,
      "activations/layer20_attention_weight_min": -25.911592483520508,
      "activations/layer21_attention_weight_max": 32.769020080566406,
      "activations/layer21_attention_weight_min": -23.332504272460938,
      "activations/layer22_attention_weight_max": 49.31988525390625,
      "activations/layer22_attention_weight_min": -29.415592193603516,
      "activations/layer23_attention_weight_max": 42.21815490722656,
      "activations/layer23_attention_weight_min": -23.425931930541992,
      "activations/layer2_attention_weight_max": 31.456642150878906,
      "activations/layer2_attention_weight_min": -34.45262145996094,
      "activations/layer3_attention_weight_max": 94.18315887451172,
      "activations/layer3_attention_weight_min": -98.38859558105469,
      "activations/layer4_attention_weight_max": 113.64302825927734,
      "activations/layer4_attention_weight_min": -116.42291259765625,
      "activations/layer5_attention_weight_max": 54.137107849121094,
      "activations/layer5_attention_weight_min": -66.15802001953125,
      "activations/layer6_attention_weight_max": 44.8622932434082,
      "activations/layer6_attention_weight_min": -52.92462921142578,
      "activations/layer7_attention_weight_max": 85.71285247802734,
      "activations/layer7_attention_weight_min": -89.6104507446289,
      "activations/layer8_attention_weight_max": 40.633338928222656,
      "activations/layer8_attention_weight_min": -41.94417190551758,
      "activations/layer9_attention_weight_max": 34.11540222167969,
      "activations/layer9_attention_weight_min": -33.855316162109375,
      "epoch": 9.8,
      "learning_rate": 8.766931818181818e-05,
      "loss": 2.7847,
      "step": 168700
    },
    {
      "activations/layer0_attention_weight_max": 16.69235610961914,
      "activations/layer0_attention_weight_min": -12.949478149414062,
      "activations/layer10_attention_weight_max": 38.78517532348633,
      "activations/layer10_attention_weight_min": -36.181941986083984,
      "activations/layer11_attention_weight_max": 39.51576232910156,
      "activations/layer11_attention_weight_min": -34.616275787353516,
      "activations/layer12_attention_weight_max": 26.268638610839844,
      "activations/layer12_attention_weight_min": -28.95777702331543,
      "activations/layer13_attention_weight_max": 43.377864837646484,
      "activations/layer13_attention_weight_min": -34.76123046875,
      "activations/layer14_attention_weight_max": 49.08256149291992,
      "activations/layer14_attention_weight_min": -40.97609329223633,
      "activations/layer15_attention_weight_max": 38.807613372802734,
      "activations/layer15_attention_weight_min": -33.47905349731445,
      "activations/layer16_attention_weight_max": 34.13100814819336,
      "activations/layer16_attention_weight_min": -27.075664520263672,
      "activations/layer17_attention_weight_max": 33.101192474365234,
      "activations/layer17_attention_weight_min": -26.918283462524414,
      "activations/layer18_attention_weight_max": 33.970306396484375,
      "activations/layer18_attention_weight_min": -25.566381454467773,
      "activations/layer19_attention_weight_max": 37.44527816772461,
      "activations/layer19_attention_weight_min": -32.00333786010742,
      "activations/layer1_attention_weight_max": 18.488067626953125,
      "activations/layer1_attention_weight_min": -15.634424209594727,
      "activations/layer20_attention_weight_max": 32.760860443115234,
      "activations/layer20_attention_weight_min": -26.94218635559082,
      "activations/layer21_attention_weight_max": 35.909507751464844,
      "activations/layer21_attention_weight_min": -25.968164443969727,
      "activations/layer22_attention_weight_max": 46.201087951660156,
      "activations/layer22_attention_weight_min": -31.4571590423584,
      "activations/layer23_attention_weight_max": 38.10558319091797,
      "activations/layer23_attention_weight_min": -26.21881675720215,
      "activations/layer2_attention_weight_max": 35.38591003417969,
      "activations/layer2_attention_weight_min": -33.950740814208984,
      "activations/layer3_attention_weight_max": 100.20606994628906,
      "activations/layer3_attention_weight_min": -101.27266693115234,
      "activations/layer4_attention_weight_max": 119.95861053466797,
      "activations/layer4_attention_weight_min": -118.05702209472656,
      "activations/layer5_attention_weight_max": 53.694454193115234,
      "activations/layer5_attention_weight_min": -60.51464080810547,
      "activations/layer6_attention_weight_max": 48.007564544677734,
      "activations/layer6_attention_weight_min": -52.0255241394043,
      "activations/layer7_attention_weight_max": 96.77131652832031,
      "activations/layer7_attention_weight_min": -93.59625244140625,
      "activations/layer8_attention_weight_max": 45.419944763183594,
      "activations/layer8_attention_weight_min": -45.085636138916016,
      "activations/layer9_attention_weight_max": 41.38824462890625,
      "activations/layer9_attention_weight_min": -39.05067825317383,
      "epoch": 9.81,
      "learning_rate": 8.765037878787878e-05,
      "loss": 2.7923,
      "step": 168750
    },
    {
      "activations/layer0_attention_weight_max": 15.448380470275879,
      "activations/layer0_attention_weight_min": -13.27200984954834,
      "activations/layer10_attention_weight_max": 30.751293182373047,
      "activations/layer10_attention_weight_min": -30.095861434936523,
      "activations/layer11_attention_weight_max": 30.058860778808594,
      "activations/layer11_attention_weight_min": -29.454036712646484,
      "activations/layer12_attention_weight_max": 22.55332374572754,
      "activations/layer12_attention_weight_min": -24.557966232299805,
      "activations/layer13_attention_weight_max": 34.53900146484375,
      "activations/layer13_attention_weight_min": -33.165340423583984,
      "activations/layer14_attention_weight_max": 50.1134147644043,
      "activations/layer14_attention_weight_min": -41.070098876953125,
      "activations/layer15_attention_weight_max": 35.66765594482422,
      "activations/layer15_attention_weight_min": -30.376041412353516,
      "activations/layer16_attention_weight_max": 32.60489273071289,
      "activations/layer16_attention_weight_min": -27.017168045043945,
      "activations/layer17_attention_weight_max": 32.12821960449219,
      "activations/layer17_attention_weight_min": -25.60306167602539,
      "activations/layer18_attention_weight_max": 31.090044021606445,
      "activations/layer18_attention_weight_min": -21.76886749267578,
      "activations/layer19_attention_weight_max": 37.037208557128906,
      "activations/layer19_attention_weight_min": -32.48345184326172,
      "activations/layer1_attention_weight_max": 16.238866806030273,
      "activations/layer1_attention_weight_min": -15.5009765625,
      "activations/layer20_attention_weight_max": 31.685428619384766,
      "activations/layer20_attention_weight_min": -25.1119384765625,
      "activations/layer21_attention_weight_max": 31.034568786621094,
      "activations/layer21_attention_weight_min": -24.425765991210938,
      "activations/layer22_attention_weight_max": 43.36804962158203,
      "activations/layer22_attention_weight_min": -27.621845245361328,
      "activations/layer23_attention_weight_max": 35.099632263183594,
      "activations/layer23_attention_weight_min": -25.87213897705078,
      "activations/layer2_attention_weight_max": 34.23560333251953,
      "activations/layer2_attention_weight_min": -32.538543701171875,
      "activations/layer3_attention_weight_max": 91.99430847167969,
      "activations/layer3_attention_weight_min": -93.95940399169922,
      "activations/layer4_attention_weight_max": 110.19332122802734,
      "activations/layer4_attention_weight_min": -113.1324462890625,
      "activations/layer5_attention_weight_max": 52.953102111816406,
      "activations/layer5_attention_weight_min": -62.58318328857422,
      "activations/layer6_attention_weight_max": 44.96741485595703,
      "activations/layer6_attention_weight_min": -47.07723617553711,
      "activations/layer7_attention_weight_max": 86.09044647216797,
      "activations/layer7_attention_weight_min": -85.0774917602539,
      "activations/layer8_attention_weight_max": 36.63261413574219,
      "activations/layer8_attention_weight_min": -38.963375091552734,
      "activations/layer9_attention_weight_max": 30.96311378479004,
      "activations/layer9_attention_weight_min": -31.06049919128418,
      "epoch": 9.81,
      "learning_rate": 8.76314393939394e-05,
      "loss": 2.7827,
      "step": 168800
    },
    {
      "activations/layer0_attention_weight_max": 14.597342491149902,
      "activations/layer0_attention_weight_min": -12.207429885864258,
      "activations/layer10_attention_weight_max": 31.728439331054688,
      "activations/layer10_attention_weight_min": -31.576675415039062,
      "activations/layer11_attention_weight_max": 33.95063781738281,
      "activations/layer11_attention_weight_min": -33.285194396972656,
      "activations/layer12_attention_weight_max": 24.60527992248535,
      "activations/layer12_attention_weight_min": -24.80906867980957,
      "activations/layer13_attention_weight_max": 42.56842803955078,
      "activations/layer13_attention_weight_min": -32.12223815917969,
      "activations/layer14_attention_weight_max": 47.1822624206543,
      "activations/layer14_attention_weight_min": -40.22103500366211,
      "activations/layer15_attention_weight_max": 38.495635986328125,
      "activations/layer15_attention_weight_min": -30.591243743896484,
      "activations/layer16_attention_weight_max": 32.46360397338867,
      "activations/layer16_attention_weight_min": -27.924211502075195,
      "activations/layer17_attention_weight_max": 33.595863342285156,
      "activations/layer17_attention_weight_min": -24.230737686157227,
      "activations/layer18_attention_weight_max": 31.917095184326172,
      "activations/layer18_attention_weight_min": -21.898103713989258,
      "activations/layer19_attention_weight_max": 35.81199264526367,
      "activations/layer19_attention_weight_min": -30.59221839904785,
      "activations/layer1_attention_weight_max": 16.126956939697266,
      "activations/layer1_attention_weight_min": -14.88775634765625,
      "activations/layer20_attention_weight_max": 30.436504364013672,
      "activations/layer20_attention_weight_min": -24.27532196044922,
      "activations/layer21_attention_weight_max": 28.911149978637695,
      "activations/layer21_attention_weight_min": -23.17829704284668,
      "activations/layer22_attention_weight_max": 50.75867462158203,
      "activations/layer22_attention_weight_min": -29.909320831298828,
      "activations/layer23_attention_weight_max": 35.08001708984375,
      "activations/layer23_attention_weight_min": -22.86549186706543,
      "activations/layer2_attention_weight_max": 32.441925048828125,
      "activations/layer2_attention_weight_min": -31.647144317626953,
      "activations/layer3_attention_weight_max": 94.0185317993164,
      "activations/layer3_attention_weight_min": -93.81958770751953,
      "activations/layer4_attention_weight_max": 111.26422119140625,
      "activations/layer4_attention_weight_min": -110.53114318847656,
      "activations/layer5_attention_weight_max": 52.16999435424805,
      "activations/layer5_attention_weight_min": -59.7001953125,
      "activations/layer6_attention_weight_max": 44.44399642944336,
      "activations/layer6_attention_weight_min": -47.74311828613281,
      "activations/layer7_attention_weight_max": 87.52293395996094,
      "activations/layer7_attention_weight_min": -87.00950622558594,
      "activations/layer8_attention_weight_max": 40.78275680541992,
      "activations/layer8_attention_weight_min": -41.125091552734375,
      "activations/layer9_attention_weight_max": 35.67681121826172,
      "activations/layer9_attention_weight_min": -32.95077133178711,
      "epoch": 9.81,
      "learning_rate": 8.761249999999998e-05,
      "loss": 2.7999,
      "step": 168850
    },
    {
      "activations/layer0_attention_weight_max": 15.03723430633545,
      "activations/layer0_attention_weight_min": -12.579227447509766,
      "activations/layer10_attention_weight_max": 31.944644927978516,
      "activations/layer10_attention_weight_min": -32.166221618652344,
      "activations/layer11_attention_weight_max": 36.111209869384766,
      "activations/layer11_attention_weight_min": -34.65259552001953,
      "activations/layer12_attention_weight_max": 23.113235473632812,
      "activations/layer12_attention_weight_min": -25.466407775878906,
      "activations/layer13_attention_weight_max": 36.410770416259766,
      "activations/layer13_attention_weight_min": -32.59130096435547,
      "activations/layer14_attention_weight_max": 47.32548904418945,
      "activations/layer14_attention_weight_min": -38.04477310180664,
      "activations/layer15_attention_weight_max": 33.246612548828125,
      "activations/layer15_attention_weight_min": -30.45425796508789,
      "activations/layer16_attention_weight_max": 29.688507080078125,
      "activations/layer16_attention_weight_min": -27.05690574645996,
      "activations/layer17_attention_weight_max": 32.111446380615234,
      "activations/layer17_attention_weight_min": -24.880094528198242,
      "activations/layer18_attention_weight_max": 31.865995407104492,
      "activations/layer18_attention_weight_min": -24.197181701660156,
      "activations/layer19_attention_weight_max": 31.32795524597168,
      "activations/layer19_attention_weight_min": -29.902631759643555,
      "activations/layer1_attention_weight_max": 16.348966598510742,
      "activations/layer1_attention_weight_min": -14.236559867858887,
      "activations/layer20_attention_weight_max": 29.055908203125,
      "activations/layer20_attention_weight_min": -24.558305740356445,
      "activations/layer21_attention_weight_max": 28.306304931640625,
      "activations/layer21_attention_weight_min": -22.970388412475586,
      "activations/layer22_attention_weight_max": 46.6449089050293,
      "activations/layer22_attention_weight_min": -30.95154571533203,
      "activations/layer23_attention_weight_max": 34.29533767700195,
      "activations/layer23_attention_weight_min": -24.008588790893555,
      "activations/layer2_attention_weight_max": 32.56519317626953,
      "activations/layer2_attention_weight_min": -31.662094116210938,
      "activations/layer3_attention_weight_max": 90.51142120361328,
      "activations/layer3_attention_weight_min": -92.17121887207031,
      "activations/layer4_attention_weight_max": 112.70684814453125,
      "activations/layer4_attention_weight_min": -113.21807861328125,
      "activations/layer5_attention_weight_max": 52.079524993896484,
      "activations/layer5_attention_weight_min": -61.81757354736328,
      "activations/layer6_attention_weight_max": 45.51536560058594,
      "activations/layer6_attention_weight_min": -48.1161994934082,
      "activations/layer7_attention_weight_max": 84.93266296386719,
      "activations/layer7_attention_weight_min": -89.61870574951172,
      "activations/layer8_attention_weight_max": 40.41539001464844,
      "activations/layer8_attention_weight_min": -40.574424743652344,
      "activations/layer9_attention_weight_max": 33.13851547241211,
      "activations/layer9_attention_weight_min": -33.59851837158203,
      "epoch": 9.81,
      "learning_rate": 8.75935606060606e-05,
      "loss": 2.7865,
      "step": 168900
    },
    {
      "activations/layer0_attention_weight_max": 15.973355293273926,
      "activations/layer0_attention_weight_min": -12.926102638244629,
      "activations/layer10_attention_weight_max": 31.093425750732422,
      "activations/layer10_attention_weight_min": -32.134918212890625,
      "activations/layer11_attention_weight_max": 34.971012115478516,
      "activations/layer11_attention_weight_min": -30.81183624267578,
      "activations/layer12_attention_weight_max": 27.47986602783203,
      "activations/layer12_attention_weight_min": -25.815195083618164,
      "activations/layer13_attention_weight_max": 45.12816619873047,
      "activations/layer13_attention_weight_min": -36.78477096557617,
      "activations/layer14_attention_weight_max": 57.82395553588867,
      "activations/layer14_attention_weight_min": -40.30744934082031,
      "activations/layer15_attention_weight_max": 38.72320556640625,
      "activations/layer15_attention_weight_min": -32.49098205566406,
      "activations/layer16_attention_weight_max": 35.63783645629883,
      "activations/layer16_attention_weight_min": -28.887310028076172,
      "activations/layer17_attention_weight_max": 33.338287353515625,
      "activations/layer17_attention_weight_min": -25.92791748046875,
      "activations/layer18_attention_weight_max": 33.42591094970703,
      "activations/layer18_attention_weight_min": -23.416013717651367,
      "activations/layer19_attention_weight_max": 40.10677719116211,
      "activations/layer19_attention_weight_min": -31.87542152404785,
      "activations/layer1_attention_weight_max": 16.320585250854492,
      "activations/layer1_attention_weight_min": -15.445575714111328,
      "activations/layer20_attention_weight_max": 34.851783752441406,
      "activations/layer20_attention_weight_min": -26.55791664123535,
      "activations/layer21_attention_weight_max": 35.36286544799805,
      "activations/layer21_attention_weight_min": -24.517745971679688,
      "activations/layer22_attention_weight_max": 50.47000503540039,
      "activations/layer22_attention_weight_min": -30.24232292175293,
      "activations/layer23_attention_weight_max": 38.72984313964844,
      "activations/layer23_attention_weight_min": -25.424095153808594,
      "activations/layer2_attention_weight_max": 32.3939323425293,
      "activations/layer2_attention_weight_min": -32.219913482666016,
      "activations/layer3_attention_weight_max": 91.56649780273438,
      "activations/layer3_attention_weight_min": -95.68646240234375,
      "activations/layer4_attention_weight_max": 109.9942398071289,
      "activations/layer4_attention_weight_min": -111.29597473144531,
      "activations/layer5_attention_weight_max": 55.64325714111328,
      "activations/layer5_attention_weight_min": -61.313838958740234,
      "activations/layer6_attention_weight_max": 45.183834075927734,
      "activations/layer6_attention_weight_min": -50.2484016418457,
      "activations/layer7_attention_weight_max": 83.51823425292969,
      "activations/layer7_attention_weight_min": -89.01628875732422,
      "activations/layer8_attention_weight_max": 39.627662658691406,
      "activations/layer8_attention_weight_min": -41.389198303222656,
      "activations/layer9_attention_weight_max": 31.93853187561035,
      "activations/layer9_attention_weight_min": -32.77791213989258,
      "epoch": 9.82,
      "learning_rate": 8.757462121212121e-05,
      "loss": 2.7831,
      "step": 168950
    },
    {
      "activations/layer0_attention_weight_max": 14.751690864562988,
      "activations/layer0_attention_weight_min": -12.549957275390625,
      "activations/layer10_attention_weight_max": 36.57523727416992,
      "activations/layer10_attention_weight_min": -31.730192184448242,
      "activations/layer11_attention_weight_max": 34.370792388916016,
      "activations/layer11_attention_weight_min": -33.53547668457031,
      "activations/layer12_attention_weight_max": 28.957538604736328,
      "activations/layer12_attention_weight_min": -26.074865341186523,
      "activations/layer13_attention_weight_max": 50.66958236694336,
      "activations/layer13_attention_weight_min": -36.20805740356445,
      "activations/layer14_attention_weight_max": 54.19300079345703,
      "activations/layer14_attention_weight_min": -41.29507827758789,
      "activations/layer15_attention_weight_max": 48.17033767700195,
      "activations/layer15_attention_weight_min": -33.651363372802734,
      "activations/layer16_attention_weight_max": 38.282989501953125,
      "activations/layer16_attention_weight_min": -29.164033889770508,
      "activations/layer17_attention_weight_max": 37.654296875,
      "activations/layer17_attention_weight_min": -26.18113899230957,
      "activations/layer18_attention_weight_max": 34.82518768310547,
      "activations/layer18_attention_weight_min": -22.631006240844727,
      "activations/layer19_attention_weight_max": 40.739009857177734,
      "activations/layer19_attention_weight_min": -31.961179733276367,
      "activations/layer1_attention_weight_max": 16.401540756225586,
      "activations/layer1_attention_weight_min": -13.408604621887207,
      "activations/layer20_attention_weight_max": 35.60821533203125,
      "activations/layer20_attention_weight_min": -24.484567642211914,
      "activations/layer21_attention_weight_max": 35.72417449951172,
      "activations/layer21_attention_weight_min": -22.056949615478516,
      "activations/layer22_attention_weight_max": 49.5505256652832,
      "activations/layer22_attention_weight_min": -29.78451156616211,
      "activations/layer23_attention_weight_max": 42.04295349121094,
      "activations/layer23_attention_weight_min": -24.082061767578125,
      "activations/layer2_attention_weight_max": 33.49648666381836,
      "activations/layer2_attention_weight_min": -32.69847869873047,
      "activations/layer3_attention_weight_max": 97.69699096679688,
      "activations/layer3_attention_weight_min": -98.5535659790039,
      "activations/layer4_attention_weight_max": 120.1396713256836,
      "activations/layer4_attention_weight_min": -115.0780258178711,
      "activations/layer5_attention_weight_max": 54.686546325683594,
      "activations/layer5_attention_weight_min": -67.80935668945312,
      "activations/layer6_attention_weight_max": 48.23965072631836,
      "activations/layer6_attention_weight_min": -49.75035858154297,
      "activations/layer7_attention_weight_max": 91.82142639160156,
      "activations/layer7_attention_weight_min": -87.12958526611328,
      "activations/layer8_attention_weight_max": 41.00318908691406,
      "activations/layer8_attention_weight_min": -40.08665466308594,
      "activations/layer9_attention_weight_max": 34.36491012573242,
      "activations/layer9_attention_weight_min": -33.8338737487793,
      "epoch": 9.82,
      "learning_rate": 8.75556818181818e-05,
      "loss": 2.8127,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.4141,
      "eval_samples_per_second": 510.333,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.4141,
      "eval_openwebtext_samples_per_second": 510.333,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 1.969,
      "eval_wikitext_samples_per_second": 231.595,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_lambada_loss": 2.52734375,
      "eval_lambada_ppl": 12.520205133467638,
      "eval_lambada_runtime": 9.5139,
      "eval_lambada_samples_per_second": 511.779,
      "step": 169000
    },
    {
      "activations/layer0_attention_weight_max": 15.140168190002441,
      "activations/layer0_attention_weight_min": -12.176920890808105,
      "activations/layer10_attention_weight_max": 34.88396453857422,
      "activations/layer10_attention_weight_min": -34.091758728027344,
      "activations/layer11_attention_weight_max": 35.79822540283203,
      "activations/layer11_attention_weight_min": -34.73484420776367,
      "activations/layer12_attention_weight_max": 25.270151138305664,
      "activations/layer12_attention_weight_min": -25.941062927246094,
      "activations/layer13_attention_weight_max": 40.84001541137695,
      "activations/layer13_attention_weight_min": -36.064491271972656,
      "activations/layer14_attention_weight_max": 50.013343811035156,
      "activations/layer14_attention_weight_min": -38.04609298706055,
      "activations/layer15_attention_weight_max": 39.22378158569336,
      "activations/layer15_attention_weight_min": -32.256839752197266,
      "activations/layer16_attention_weight_max": 33.05716323852539,
      "activations/layer16_attention_weight_min": -28.062007904052734,
      "activations/layer17_attention_weight_max": 34.82142639160156,
      "activations/layer17_attention_weight_min": -25.45966339111328,
      "activations/layer18_attention_weight_max": 32.00650405883789,
      "activations/layer18_attention_weight_min": -21.73941993713379,
      "activations/layer19_attention_weight_max": 32.66484069824219,
      "activations/layer19_attention_weight_min": -29.57594108581543,
      "activations/layer1_attention_weight_max": 16.415651321411133,
      "activations/layer1_attention_weight_min": -16.09031867980957,
      "activations/layer20_attention_weight_max": 30.323986053466797,
      "activations/layer20_attention_weight_min": -25.4370059967041,
      "activations/layer21_attention_weight_max": 28.645549774169922,
      "activations/layer21_attention_weight_min": -23.981157302856445,
      "activations/layer22_attention_weight_max": 43.35987854003906,
      "activations/layer22_attention_weight_min": -30.2779541015625,
      "activations/layer23_attention_weight_max": 33.23130798339844,
      "activations/layer23_attention_weight_min": -25.61817741394043,
      "activations/layer2_attention_weight_max": 31.621322631835938,
      "activations/layer2_attention_weight_min": -32.57347869873047,
      "activations/layer3_attention_weight_max": 92.64315032958984,
      "activations/layer3_attention_weight_min": -92.25171661376953,
      "activations/layer4_attention_weight_max": 114.1069564819336,
      "activations/layer4_attention_weight_min": -112.94093322753906,
      "activations/layer5_attention_weight_max": 51.59096908569336,
      "activations/layer5_attention_weight_min": -60.98406219482422,
      "activations/layer6_attention_weight_max": 47.42009353637695,
      "activations/layer6_attention_weight_min": -50.7108039855957,
      "activations/layer7_attention_weight_max": 90.36312103271484,
      "activations/layer7_attention_weight_min": -92.98943328857422,
      "activations/layer8_attention_weight_max": 43.84900665283203,
      "activations/layer8_attention_weight_min": -40.632347106933594,
      "activations/layer9_attention_weight_max": 36.1683235168457,
      "activations/layer9_attention_weight_min": -33.43244552612305,
      "epoch": 9.82,
      "learning_rate": 8.753674242424241e-05,
      "loss": 2.7924,
      "step": 169050
    },
    {
      "activations/layer0_attention_weight_max": 15.201271057128906,
      "activations/layer0_attention_weight_min": -12.499275207519531,
      "activations/layer10_attention_weight_max": 30.009984970092773,
      "activations/layer10_attention_weight_min": -31.42416763305664,
      "activations/layer11_attention_weight_max": 30.9370174407959,
      "activations/layer11_attention_weight_min": -30.579185485839844,
      "activations/layer12_attention_weight_max": 22.9957218170166,
      "activations/layer12_attention_weight_min": -25.09001922607422,
      "activations/layer13_attention_weight_max": 38.42730712890625,
      "activations/layer13_attention_weight_min": -36.392364501953125,
      "activations/layer14_attention_weight_max": 52.443580627441406,
      "activations/layer14_attention_weight_min": -38.46963119506836,
      "activations/layer15_attention_weight_max": 34.217132568359375,
      "activations/layer15_attention_weight_min": -30.912363052368164,
      "activations/layer16_attention_weight_max": 31.010498046875,
      "activations/layer16_attention_weight_min": -28.264944076538086,
      "activations/layer17_attention_weight_max": 34.14988327026367,
      "activations/layer17_attention_weight_min": -26.217044830322266,
      "activations/layer18_attention_weight_max": 34.330352783203125,
      "activations/layer18_attention_weight_min": -25.979415893554688,
      "activations/layer19_attention_weight_max": 39.69108963012695,
      "activations/layer19_attention_weight_min": -31.14943504333496,
      "activations/layer1_attention_weight_max": 17.625669479370117,
      "activations/layer1_attention_weight_min": -15.472345352172852,
      "activations/layer20_attention_weight_max": 36.10606002807617,
      "activations/layer20_attention_weight_min": -26.69976234436035,
      "activations/layer21_attention_weight_max": 37.19749069213867,
      "activations/layer21_attention_weight_min": -24.88724708557129,
      "activations/layer22_attention_weight_max": 51.371524810791016,
      "activations/layer22_attention_weight_min": -31.742294311523438,
      "activations/layer23_attention_weight_max": 39.150447845458984,
      "activations/layer23_attention_weight_min": -25.642065048217773,
      "activations/layer2_attention_weight_max": 31.675148010253906,
      "activations/layer2_attention_weight_min": -33.95447540283203,
      "activations/layer3_attention_weight_max": 92.95246124267578,
      "activations/layer3_attention_weight_min": -98.5764389038086,
      "activations/layer4_attention_weight_max": 111.40128326416016,
      "activations/layer4_attention_weight_min": -113.00716400146484,
      "activations/layer5_attention_weight_max": 52.1114501953125,
      "activations/layer5_attention_weight_min": -62.7254638671875,
      "activations/layer6_attention_weight_max": 44.88920211791992,
      "activations/layer6_attention_weight_min": -47.40201187133789,
      "activations/layer7_attention_weight_max": 87.20121002197266,
      "activations/layer7_attention_weight_min": -84.50877380371094,
      "activations/layer8_attention_weight_max": 39.17318344116211,
      "activations/layer8_attention_weight_min": -41.09193801879883,
      "activations/layer9_attention_weight_max": 32.7484245300293,
      "activations/layer9_attention_weight_min": -35.334861755371094,
      "epoch": 9.83,
      "learning_rate": 8.751780303030303e-05,
      "loss": 2.7926,
      "step": 169100
    },
    {
      "activations/layer0_attention_weight_max": 15.425409317016602,
      "activations/layer0_attention_weight_min": -12.0939302444458,
      "activations/layer10_attention_weight_max": 33.551292419433594,
      "activations/layer10_attention_weight_min": -34.93457794189453,
      "activations/layer11_attention_weight_max": 37.31710433959961,
      "activations/layer11_attention_weight_min": -37.693267822265625,
      "activations/layer12_attention_weight_max": 36.9801139831543,
      "activations/layer12_attention_weight_min": -29.657556533813477,
      "activations/layer13_attention_weight_max": 53.642311096191406,
      "activations/layer13_attention_weight_min": -40.65425109863281,
      "activations/layer14_attention_weight_max": 59.84282302856445,
      "activations/layer14_attention_weight_min": -50.817665100097656,
      "activations/layer15_attention_weight_max": 47.46629333496094,
      "activations/layer15_attention_weight_min": -36.84878921508789,
      "activations/layer16_attention_weight_max": 33.47328567504883,
      "activations/layer16_attention_weight_min": -31.97539520263672,
      "activations/layer17_attention_weight_max": 29.98492431640625,
      "activations/layer17_attention_weight_min": -29.853240966796875,
      "activations/layer18_attention_weight_max": 30.825490951538086,
      "activations/layer18_attention_weight_min": -24.765941619873047,
      "activations/layer19_attention_weight_max": 37.05564880371094,
      "activations/layer19_attention_weight_min": -36.21113204956055,
      "activations/layer1_attention_weight_max": 16.67662239074707,
      "activations/layer1_attention_weight_min": -14.589998245239258,
      "activations/layer20_attention_weight_max": 30.1704044342041,
      "activations/layer20_attention_weight_min": -27.650083541870117,
      "activations/layer21_attention_weight_max": 28.734020233154297,
      "activations/layer21_attention_weight_min": -24.743392944335938,
      "activations/layer22_attention_weight_max": 47.216156005859375,
      "activations/layer22_attention_weight_min": -33.53740310668945,
      "activations/layer23_attention_weight_max": 33.73143768310547,
      "activations/layer23_attention_weight_min": -28.427230834960938,
      "activations/layer2_attention_weight_max": 33.41410827636719,
      "activations/layer2_attention_weight_min": -34.431427001953125,
      "activations/layer3_attention_weight_max": 94.77651977539062,
      "activations/layer3_attention_weight_min": -98.15313720703125,
      "activations/layer4_attention_weight_max": 115.17009735107422,
      "activations/layer4_attention_weight_min": -117.75054931640625,
      "activations/layer5_attention_weight_max": 53.85231018066406,
      "activations/layer5_attention_weight_min": -61.602813720703125,
      "activations/layer6_attention_weight_max": 45.34096908569336,
      "activations/layer6_attention_weight_min": -48.459529876708984,
      "activations/layer7_attention_weight_max": 90.41986083984375,
      "activations/layer7_attention_weight_min": -99.57266998291016,
      "activations/layer8_attention_weight_max": 40.60954666137695,
      "activations/layer8_attention_weight_min": -43.48366165161133,
      "activations/layer9_attention_weight_max": 34.970279693603516,
      "activations/layer9_attention_weight_min": -34.39104080200195,
      "epoch": 9.83,
      "learning_rate": 8.749886363636363e-05,
      "loss": 2.7869,
      "step": 169150
    },
    {
      "activations/layer0_attention_weight_max": 14.7537841796875,
      "activations/layer0_attention_weight_min": -13.1132230758667,
      "activations/layer10_attention_weight_max": 32.436161041259766,
      "activations/layer10_attention_weight_min": -30.162132263183594,
      "activations/layer11_attention_weight_max": 32.68338394165039,
      "activations/layer11_attention_weight_min": -32.097740173339844,
      "activations/layer12_attention_weight_max": 25.178192138671875,
      "activations/layer12_attention_weight_min": -24.05424690246582,
      "activations/layer13_attention_weight_max": 38.4459114074707,
      "activations/layer13_attention_weight_min": -35.05828857421875,
      "activations/layer14_attention_weight_max": 44.96382141113281,
      "activations/layer14_attention_weight_min": -41.18254089355469,
      "activations/layer15_attention_weight_max": 34.75576400756836,
      "activations/layer15_attention_weight_min": -30.726993560791016,
      "activations/layer16_attention_weight_max": 30.80621337890625,
      "activations/layer16_attention_weight_min": -29.348176956176758,
      "activations/layer17_attention_weight_max": 34.54756546020508,
      "activations/layer17_attention_weight_min": -26.8219051361084,
      "activations/layer18_attention_weight_max": 36.72057342529297,
      "activations/layer18_attention_weight_min": -22.76626968383789,
      "activations/layer19_attention_weight_max": 35.15592956542969,
      "activations/layer19_attention_weight_min": -30.927011489868164,
      "activations/layer1_attention_weight_max": 16.69852066040039,
      "activations/layer1_attention_weight_min": -13.970094680786133,
      "activations/layer20_attention_weight_max": 28.826480865478516,
      "activations/layer20_attention_weight_min": -24.292905807495117,
      "activations/layer21_attention_weight_max": 33.04555130004883,
      "activations/layer21_attention_weight_min": -23.524892807006836,
      "activations/layer22_attention_weight_max": 44.44124984741211,
      "activations/layer22_attention_weight_min": -30.0406551361084,
      "activations/layer23_attention_weight_max": 32.933162689208984,
      "activations/layer23_attention_weight_min": -23.23923110961914,
      "activations/layer2_attention_weight_max": 30.910507202148438,
      "activations/layer2_attention_weight_min": -33.61292266845703,
      "activations/layer3_attention_weight_max": 92.91590118408203,
      "activations/layer3_attention_weight_min": -91.8995132446289,
      "activations/layer4_attention_weight_max": 119.07398986816406,
      "activations/layer4_attention_weight_min": -111.02021789550781,
      "activations/layer5_attention_weight_max": 52.515357971191406,
      "activations/layer5_attention_weight_min": -59.12628173828125,
      "activations/layer6_attention_weight_max": 45.26927947998047,
      "activations/layer6_attention_weight_min": -46.94992446899414,
      "activations/layer7_attention_weight_max": 87.60791778564453,
      "activations/layer7_attention_weight_min": -82.74490356445312,
      "activations/layer8_attention_weight_max": 39.147762298583984,
      "activations/layer8_attention_weight_min": -40.00813293457031,
      "activations/layer9_attention_weight_max": 32.58934020996094,
      "activations/layer9_attention_weight_min": -31.823251724243164,
      "epoch": 9.83,
      "learning_rate": 8.747992424242423e-05,
      "loss": 2.7988,
      "step": 169200
    },
    {
      "activations/layer0_attention_weight_max": 15.824707984924316,
      "activations/layer0_attention_weight_min": -13.111059188842773,
      "activations/layer10_attention_weight_max": 35.56850051879883,
      "activations/layer10_attention_weight_min": -33.50120544433594,
      "activations/layer11_attention_weight_max": 36.726192474365234,
      "activations/layer11_attention_weight_min": -32.801597595214844,
      "activations/layer12_attention_weight_max": 26.719966888427734,
      "activations/layer12_attention_weight_min": -24.670284271240234,
      "activations/layer13_attention_weight_max": 41.88916778564453,
      "activations/layer13_attention_weight_min": -33.282657623291016,
      "activations/layer14_attention_weight_max": 52.799835205078125,
      "activations/layer14_attention_weight_min": -37.99471664428711,
      "activations/layer15_attention_weight_max": 38.513938903808594,
      "activations/layer15_attention_weight_min": -30.143705368041992,
      "activations/layer16_attention_weight_max": 33.552978515625,
      "activations/layer16_attention_weight_min": -28.358678817749023,
      "activations/layer17_attention_weight_max": 34.16218948364258,
      "activations/layer17_attention_weight_min": -25.106374740600586,
      "activations/layer18_attention_weight_max": 34.51209259033203,
      "activations/layer18_attention_weight_min": -24.08216094970703,
      "activations/layer19_attention_weight_max": 41.316070556640625,
      "activations/layer19_attention_weight_min": -29.8748836517334,
      "activations/layer1_attention_weight_max": 16.009937286376953,
      "activations/layer1_attention_weight_min": -15.977871894836426,
      "activations/layer20_attention_weight_max": 32.32049560546875,
      "activations/layer20_attention_weight_min": -23.55055046081543,
      "activations/layer21_attention_weight_max": 30.318021774291992,
      "activations/layer21_attention_weight_min": -21.670392990112305,
      "activations/layer22_attention_weight_max": 56.32575225830078,
      "activations/layer22_attention_weight_min": -27.086288452148438,
      "activations/layer23_attention_weight_max": 46.25324249267578,
      "activations/layer23_attention_weight_min": -22.530010223388672,
      "activations/layer2_attention_weight_max": 33.16822052001953,
      "activations/layer2_attention_weight_min": -33.09061050415039,
      "activations/layer3_attention_weight_max": 92.77861022949219,
      "activations/layer3_attention_weight_min": -91.69664001464844,
      "activations/layer4_attention_weight_max": 114.17449951171875,
      "activations/layer4_attention_weight_min": -113.31114959716797,
      "activations/layer5_attention_weight_max": 52.499839782714844,
      "activations/layer5_attention_weight_min": -60.583702087402344,
      "activations/layer6_attention_weight_max": 46.01472091674805,
      "activations/layer6_attention_weight_min": -49.89418029785156,
      "activations/layer7_attention_weight_max": 89.65228271484375,
      "activations/layer7_attention_weight_min": -96.94405364990234,
      "activations/layer8_attention_weight_max": 40.204288482666016,
      "activations/layer8_attention_weight_min": -41.12217330932617,
      "activations/layer9_attention_weight_max": 35.54769515991211,
      "activations/layer9_attention_weight_min": -34.1037712097168,
      "epoch": 9.83,
      "learning_rate": 8.746098484848483e-05,
      "loss": 2.7972,
      "step": 169250
    },
    {
      "activations/layer0_attention_weight_max": 15.171259880065918,
      "activations/layer0_attention_weight_min": -12.981356620788574,
      "activations/layer10_attention_weight_max": 31.424453735351562,
      "activations/layer10_attention_weight_min": -30.876060485839844,
      "activations/layer11_attention_weight_max": 32.552032470703125,
      "activations/layer11_attention_weight_min": -30.586225509643555,
      "activations/layer12_attention_weight_max": 25.9304141998291,
      "activations/layer12_attention_weight_min": -25.570037841796875,
      "activations/layer13_attention_weight_max": 39.1454963684082,
      "activations/layer13_attention_weight_min": -31.49709129333496,
      "activations/layer14_attention_weight_max": 50.80826187133789,
      "activations/layer14_attention_weight_min": -41.19365692138672,
      "activations/layer15_attention_weight_max": 39.87343978881836,
      "activations/layer15_attention_weight_min": -30.58255386352539,
      "activations/layer16_attention_weight_max": 34.85220718383789,
      "activations/layer16_attention_weight_min": -26.659000396728516,
      "activations/layer17_attention_weight_max": 33.980377197265625,
      "activations/layer17_attention_weight_min": -26.140911102294922,
      "activations/layer18_attention_weight_max": 31.679832458496094,
      "activations/layer18_attention_weight_min": -24.72829818725586,
      "activations/layer19_attention_weight_max": 40.06875228881836,
      "activations/layer19_attention_weight_min": -29.80716323852539,
      "activations/layer1_attention_weight_max": 17.08403968811035,
      "activations/layer1_attention_weight_min": -13.851365089416504,
      "activations/layer20_attention_weight_max": 35.7670783996582,
      "activations/layer20_attention_weight_min": -24.22962760925293,
      "activations/layer21_attention_weight_max": 35.40129470825195,
      "activations/layer21_attention_weight_min": -22.428232192993164,
      "activations/layer22_attention_weight_max": 50.625877380371094,
      "activations/layer22_attention_weight_min": -27.95340347290039,
      "activations/layer23_attention_weight_max": 41.32270812988281,
      "activations/layer23_attention_weight_min": -25.236331939697266,
      "activations/layer2_attention_weight_max": 31.91765594482422,
      "activations/layer2_attention_weight_min": -32.35797882080078,
      "activations/layer3_attention_weight_max": 96.30146789550781,
      "activations/layer3_attention_weight_min": -95.79085540771484,
      "activations/layer4_attention_weight_max": 115.6240463256836,
      "activations/layer4_attention_weight_min": -110.4927749633789,
      "activations/layer5_attention_weight_max": 53.07721710205078,
      "activations/layer5_attention_weight_min": -59.60309982299805,
      "activations/layer6_attention_weight_max": 45.23629379272461,
      "activations/layer6_attention_weight_min": -52.70618438720703,
      "activations/layer7_attention_weight_max": 88.14820098876953,
      "activations/layer7_attention_weight_min": -88.3231430053711,
      "activations/layer8_attention_weight_max": 38.08012390136719,
      "activations/layer8_attention_weight_min": -39.31950378417969,
      "activations/layer9_attention_weight_max": 31.931304931640625,
      "activations/layer9_attention_weight_min": -32.24649429321289,
      "epoch": 9.84,
      "learning_rate": 8.744204545454545e-05,
      "loss": 2.807,
      "step": 169300
    },
    {
      "activations/layer0_attention_weight_max": 16.05028533935547,
      "activations/layer0_attention_weight_min": -12.914668083190918,
      "activations/layer10_attention_weight_max": 30.766258239746094,
      "activations/layer10_attention_weight_min": -32.005645751953125,
      "activations/layer11_attention_weight_max": 31.231822967529297,
      "activations/layer11_attention_weight_min": -33.429405212402344,
      "activations/layer12_attention_weight_max": 22.178869247436523,
      "activations/layer12_attention_weight_min": -25.236419677734375,
      "activations/layer13_attention_weight_max": 34.70801544189453,
      "activations/layer13_attention_weight_min": -33.16262435913086,
      "activations/layer14_attention_weight_max": 42.180931091308594,
      "activations/layer14_attention_weight_min": -37.707733154296875,
      "activations/layer15_attention_weight_max": 35.2036247253418,
      "activations/layer15_attention_weight_min": -30.422637939453125,
      "activations/layer16_attention_weight_max": 31.735164642333984,
      "activations/layer16_attention_weight_min": -29.253677368164062,
      "activations/layer17_attention_weight_max": 30.37116241455078,
      "activations/layer17_attention_weight_min": -24.958574295043945,
      "activations/layer18_attention_weight_max": 30.568864822387695,
      "activations/layer18_attention_weight_min": -21.34644317626953,
      "activations/layer19_attention_weight_max": 33.86572265625,
      "activations/layer19_attention_weight_min": -28.74054527282715,
      "activations/layer1_attention_weight_max": 16.32996368408203,
      "activations/layer1_attention_weight_min": -13.279754638671875,
      "activations/layer20_attention_weight_max": 27.7625675201416,
      "activations/layer20_attention_weight_min": -22.988008499145508,
      "activations/layer21_attention_weight_max": 30.084062576293945,
      "activations/layer21_attention_weight_min": -23.649723052978516,
      "activations/layer22_attention_weight_max": 40.347511291503906,
      "activations/layer22_attention_weight_min": -27.83057403564453,
      "activations/layer23_attention_weight_max": 32.52544403076172,
      "activations/layer23_attention_weight_min": -22.43878173828125,
      "activations/layer2_attention_weight_max": 31.050168991088867,
      "activations/layer2_attention_weight_min": -31.386045455932617,
      "activations/layer3_attention_weight_max": 90.91519165039062,
      "activations/layer3_attention_weight_min": -90.83120727539062,
      "activations/layer4_attention_weight_max": 110.92935943603516,
      "activations/layer4_attention_weight_min": -107.32401275634766,
      "activations/layer5_attention_weight_max": 52.52464294433594,
      "activations/layer5_attention_weight_min": -59.77561569213867,
      "activations/layer6_attention_weight_max": 46.12429428100586,
      "activations/layer6_attention_weight_min": -49.122337341308594,
      "activations/layer7_attention_weight_max": 89.7863998413086,
      "activations/layer7_attention_weight_min": -87.1724624633789,
      "activations/layer8_attention_weight_max": 39.27498245239258,
      "activations/layer8_attention_weight_min": -43.16645431518555,
      "activations/layer9_attention_weight_max": 32.69236373901367,
      "activations/layer9_attention_weight_min": -32.55282211303711,
      "epoch": 9.84,
      "learning_rate": 8.742310606060605e-05,
      "loss": 2.8005,
      "step": 169350
    },
    {
      "activations/layer0_attention_weight_max": 15.334142684936523,
      "activations/layer0_attention_weight_min": -12.10059928894043,
      "activations/layer10_attention_weight_max": 32.9193115234375,
      "activations/layer10_attention_weight_min": -31.263425827026367,
      "activations/layer11_attention_weight_max": 32.776390075683594,
      "activations/layer11_attention_weight_min": -32.54298400878906,
      "activations/layer12_attention_weight_max": 33.975120544433594,
      "activations/layer12_attention_weight_min": -23.47941017150879,
      "activations/layer13_attention_weight_max": 43.51553726196289,
      "activations/layer13_attention_weight_min": -33.26630783081055,
      "activations/layer14_attention_weight_max": 42.8330078125,
      "activations/layer14_attention_weight_min": -38.11952209472656,
      "activations/layer15_attention_weight_max": 41.00381851196289,
      "activations/layer15_attention_weight_min": -30.801862716674805,
      "activations/layer16_attention_weight_max": 31.585742950439453,
      "activations/layer16_attention_weight_min": -28.455631256103516,
      "activations/layer17_attention_weight_max": 31.280080795288086,
      "activations/layer17_attention_weight_min": -26.590442657470703,
      "activations/layer18_attention_weight_max": 30.321550369262695,
      "activations/layer18_attention_weight_min": -21.543903350830078,
      "activations/layer19_attention_weight_max": 35.62054443359375,
      "activations/layer19_attention_weight_min": -30.39554214477539,
      "activations/layer1_attention_weight_max": 17.641504287719727,
      "activations/layer1_attention_weight_min": -15.247782707214355,
      "activations/layer20_attention_weight_max": 31.00067710876465,
      "activations/layer20_attention_weight_min": -24.364152908325195,
      "activations/layer21_attention_weight_max": 32.54018783569336,
      "activations/layer21_attention_weight_min": -24.235748291015625,
      "activations/layer22_attention_weight_max": 44.73676300048828,
      "activations/layer22_attention_weight_min": -28.493425369262695,
      "activations/layer23_attention_weight_max": 36.2818603515625,
      "activations/layer23_attention_weight_min": -24.099363327026367,
      "activations/layer2_attention_weight_max": 34.961368560791016,
      "activations/layer2_attention_weight_min": -35.0399169921875,
      "activations/layer3_attention_weight_max": 92.79883575439453,
      "activations/layer3_attention_weight_min": -95.75957489013672,
      "activations/layer4_attention_weight_max": 109.92827606201172,
      "activations/layer4_attention_weight_min": -114.8582763671875,
      "activations/layer5_attention_weight_max": 50.10807800292969,
      "activations/layer5_attention_weight_min": -63.49030685424805,
      "activations/layer6_attention_weight_max": 45.70137405395508,
      "activations/layer6_attention_weight_min": -48.617469787597656,
      "activations/layer7_attention_weight_max": 85.39270782470703,
      "activations/layer7_attention_weight_min": -86.25460052490234,
      "activations/layer8_attention_weight_max": 39.39140701293945,
      "activations/layer8_attention_weight_min": -40.87424087524414,
      "activations/layer9_attention_weight_max": 34.134883880615234,
      "activations/layer9_attention_weight_min": -32.77709197998047,
      "epoch": 9.84,
      "learning_rate": 8.740416666666665e-05,
      "loss": 2.7969,
      "step": 169400
    },
    {
      "activations/layer0_attention_weight_max": 14.998078346252441,
      "activations/layer0_attention_weight_min": -12.690143585205078,
      "activations/layer10_attention_weight_max": 34.20907211303711,
      "activations/layer10_attention_weight_min": -35.7793083190918,
      "activations/layer11_attention_weight_max": 35.80225372314453,
      "activations/layer11_attention_weight_min": -33.52838134765625,
      "activations/layer12_attention_weight_max": 25.93640899658203,
      "activations/layer12_attention_weight_min": -25.682035446166992,
      "activations/layer13_attention_weight_max": 38.685726165771484,
      "activations/layer13_attention_weight_min": -33.215240478515625,
      "activations/layer14_attention_weight_max": 46.31562042236328,
      "activations/layer14_attention_weight_min": -36.357330322265625,
      "activations/layer15_attention_weight_max": 37.85958480834961,
      "activations/layer15_attention_weight_min": -32.2013053894043,
      "activations/layer16_attention_weight_max": 34.599525451660156,
      "activations/layer16_attention_weight_min": -28.12801742553711,
      "activations/layer17_attention_weight_max": 34.869468688964844,
      "activations/layer17_attention_weight_min": -25.82077980041504,
      "activations/layer18_attention_weight_max": 32.537723541259766,
      "activations/layer18_attention_weight_min": -23.22491455078125,
      "activations/layer19_attention_weight_max": 36.2503776550293,
      "activations/layer19_attention_weight_min": -31.74015235900879,
      "activations/layer1_attention_weight_max": 17.180130004882812,
      "activations/layer1_attention_weight_min": -14.269315719604492,
      "activations/layer20_attention_weight_max": 31.91671371459961,
      "activations/layer20_attention_weight_min": -25.415283203125,
      "activations/layer21_attention_weight_max": 30.031179428100586,
      "activations/layer21_attention_weight_min": -23.546939849853516,
      "activations/layer22_attention_weight_max": 46.25587463378906,
      "activations/layer22_attention_weight_min": -29.033437728881836,
      "activations/layer23_attention_weight_max": 36.24811553955078,
      "activations/layer23_attention_weight_min": -25.51589012145996,
      "activations/layer2_attention_weight_max": 33.061248779296875,
      "activations/layer2_attention_weight_min": -34.199745178222656,
      "activations/layer3_attention_weight_max": 96.60253143310547,
      "activations/layer3_attention_weight_min": -94.40198516845703,
      "activations/layer4_attention_weight_max": 120.70769500732422,
      "activations/layer4_attention_weight_min": -114.22704315185547,
      "activations/layer5_attention_weight_max": 54.59574508666992,
      "activations/layer5_attention_weight_min": -61.33866882324219,
      "activations/layer6_attention_weight_max": 48.86399841308594,
      "activations/layer6_attention_weight_min": -51.80801773071289,
      "activations/layer7_attention_weight_max": 86.03775787353516,
      "activations/layer7_attention_weight_min": -88.37432098388672,
      "activations/layer8_attention_weight_max": 41.605525970458984,
      "activations/layer8_attention_weight_min": -41.350440979003906,
      "activations/layer9_attention_weight_max": 34.15375518798828,
      "activations/layer9_attention_weight_min": -34.3093376159668,
      "epoch": 9.85,
      "learning_rate": 8.738522727272727e-05,
      "loss": 2.7994,
      "step": 169450
    },
    {
      "activations/layer0_attention_weight_max": 15.640434265136719,
      "activations/layer0_attention_weight_min": -12.129860877990723,
      "activations/layer10_attention_weight_max": 41.972076416015625,
      "activations/layer10_attention_weight_min": -38.484344482421875,
      "activations/layer11_attention_weight_max": 40.169090270996094,
      "activations/layer11_attention_weight_min": -35.269657135009766,
      "activations/layer12_attention_weight_max": 29.632230758666992,
      "activations/layer12_attention_weight_min": -28.942541122436523,
      "activations/layer13_attention_weight_max": 49.403018951416016,
      "activations/layer13_attention_weight_min": -34.30952453613281,
      "activations/layer14_attention_weight_max": 57.60454559326172,
      "activations/layer14_attention_weight_min": -39.54930114746094,
      "activations/layer15_attention_weight_max": 51.75053405761719,
      "activations/layer15_attention_weight_min": -33.29351806640625,
      "activations/layer16_attention_weight_max": 37.46183776855469,
      "activations/layer16_attention_weight_min": -27.985185623168945,
      "activations/layer17_attention_weight_max": 38.51472854614258,
      "activations/layer17_attention_weight_min": -28.296144485473633,
      "activations/layer18_attention_weight_max": 32.45493698120117,
      "activations/layer18_attention_weight_min": -22.197471618652344,
      "activations/layer19_attention_weight_max": 37.031394958496094,
      "activations/layer19_attention_weight_min": -30.24005126953125,
      "activations/layer1_attention_weight_max": 17.00078010559082,
      "activations/layer1_attention_weight_min": -15.42072868347168,
      "activations/layer20_attention_weight_max": 31.86968994140625,
      "activations/layer20_attention_weight_min": -26.265714645385742,
      "activations/layer21_attention_weight_max": 30.184017181396484,
      "activations/layer21_attention_weight_min": -23.7204647064209,
      "activations/layer22_attention_weight_max": 50.95030212402344,
      "activations/layer22_attention_weight_min": -31.00943946838379,
      "activations/layer23_attention_weight_max": 39.47264099121094,
      "activations/layer23_attention_weight_min": -24.26532745361328,
      "activations/layer2_attention_weight_max": 33.153114318847656,
      "activations/layer2_attention_weight_min": -33.86486053466797,
      "activations/layer3_attention_weight_max": 101.76342010498047,
      "activations/layer3_attention_weight_min": -102.50104522705078,
      "activations/layer4_attention_weight_max": 121.7978286743164,
      "activations/layer4_attention_weight_min": -124.42852783203125,
      "activations/layer5_attention_weight_max": 57.92817687988281,
      "activations/layer5_attention_weight_min": -64.42044067382812,
      "activations/layer6_attention_weight_max": 49.65224838256836,
      "activations/layer6_attention_weight_min": -53.50332260131836,
      "activations/layer7_attention_weight_max": 99.65133666992188,
      "activations/layer7_attention_weight_min": -99.74560546875,
      "activations/layer8_attention_weight_max": 47.59318161010742,
      "activations/layer8_attention_weight_min": -47.0632438659668,
      "activations/layer9_attention_weight_max": 42.43219757080078,
      "activations/layer9_attention_weight_min": -38.02619171142578,
      "epoch": 9.85,
      "learning_rate": 8.736628787878788e-05,
      "loss": 2.7908,
      "step": 169500
    },
    {
      "activations/layer0_attention_weight_max": 15.481075286865234,
      "activations/layer0_attention_weight_min": -12.079090118408203,
      "activations/layer10_attention_weight_max": 32.5996208190918,
      "activations/layer10_attention_weight_min": -33.972686767578125,
      "activations/layer11_attention_weight_max": 30.458721160888672,
      "activations/layer11_attention_weight_min": -31.088560104370117,
      "activations/layer12_attention_weight_max": 22.939237594604492,
      "activations/layer12_attention_weight_min": -29.029922485351562,
      "activations/layer13_attention_weight_max": 37.64341735839844,
      "activations/layer13_attention_weight_min": -35.5119743347168,
      "activations/layer14_attention_weight_max": 42.53251647949219,
      "activations/layer14_attention_weight_min": -41.00056457519531,
      "activations/layer15_attention_weight_max": 34.77098083496094,
      "activations/layer15_attention_weight_min": -31.965831756591797,
      "activations/layer16_attention_weight_max": 30.00360870361328,
      "activations/layer16_attention_weight_min": -27.3592472076416,
      "activations/layer17_attention_weight_max": 32.84828186035156,
      "activations/layer17_attention_weight_min": -25.322364807128906,
      "activations/layer18_attention_weight_max": 31.054677963256836,
      "activations/layer18_attention_weight_min": -21.436288833618164,
      "activations/layer19_attention_weight_max": 33.9113883972168,
      "activations/layer19_attention_weight_min": -29.02753257751465,
      "activations/layer1_attention_weight_max": 16.457246780395508,
      "activations/layer1_attention_weight_min": -14.362700462341309,
      "activations/layer20_attention_weight_max": 30.403669357299805,
      "activations/layer20_attention_weight_min": -22.978410720825195,
      "activations/layer21_attention_weight_max": 32.76325225830078,
      "activations/layer21_attention_weight_min": -21.592208862304688,
      "activations/layer22_attention_weight_max": 46.04279327392578,
      "activations/layer22_attention_weight_min": -29.10110092163086,
      "activations/layer23_attention_weight_max": 35.325660705566406,
      "activations/layer23_attention_weight_min": -23.453222274780273,
      "activations/layer2_attention_weight_max": 32.545902252197266,
      "activations/layer2_attention_weight_min": -31.85296630859375,
      "activations/layer3_attention_weight_max": 88.79753875732422,
      "activations/layer3_attention_weight_min": -92.54762268066406,
      "activations/layer4_attention_weight_max": 108.40254974365234,
      "activations/layer4_attention_weight_min": -108.62397766113281,
      "activations/layer5_attention_weight_max": 53.844512939453125,
      "activations/layer5_attention_weight_min": -65.88446044921875,
      "activations/layer6_attention_weight_max": 45.604774475097656,
      "activations/layer6_attention_weight_min": -50.386966705322266,
      "activations/layer7_attention_weight_max": 87.34233856201172,
      "activations/layer7_attention_weight_min": -90.6531753540039,
      "activations/layer8_attention_weight_max": 42.41008758544922,
      "activations/layer8_attention_weight_min": -40.76066589355469,
      "activations/layer9_attention_weight_max": 31.483976364135742,
      "activations/layer9_attention_weight_min": -34.9187126159668,
      "epoch": 9.85,
      "learning_rate": 8.734734848484847e-05,
      "loss": 2.7979,
      "step": 169550
    },
    {
      "activations/layer0_attention_weight_max": 14.364411354064941,
      "activations/layer0_attention_weight_min": -12.876236915588379,
      "activations/layer10_attention_weight_max": 33.38237380981445,
      "activations/layer10_attention_weight_min": -30.476463317871094,
      "activations/layer11_attention_weight_max": 31.7065486907959,
      "activations/layer11_attention_weight_min": -30.978315353393555,
      "activations/layer12_attention_weight_max": 22.645063400268555,
      "activations/layer12_attention_weight_min": -23.26725196838379,
      "activations/layer13_attention_weight_max": 36.558963775634766,
      "activations/layer13_attention_weight_min": -32.43339157104492,
      "activations/layer14_attention_weight_max": 49.97248840332031,
      "activations/layer14_attention_weight_min": -40.63835525512695,
      "activations/layer15_attention_weight_max": 36.231842041015625,
      "activations/layer15_attention_weight_min": -33.70441818237305,
      "activations/layer16_attention_weight_max": 31.900222778320312,
      "activations/layer16_attention_weight_min": -28.72037124633789,
      "activations/layer17_attention_weight_max": 32.0446891784668,
      "activations/layer17_attention_weight_min": -25.71719741821289,
      "activations/layer18_attention_weight_max": 29.93947982788086,
      "activations/layer18_attention_weight_min": -23.08626937866211,
      "activations/layer19_attention_weight_max": 37.66417694091797,
      "activations/layer19_attention_weight_min": -30.285303115844727,
      "activations/layer1_attention_weight_max": 16.090574264526367,
      "activations/layer1_attention_weight_min": -13.989779472351074,
      "activations/layer20_attention_weight_max": 32.80790710449219,
      "activations/layer20_attention_weight_min": -24.46831512451172,
      "activations/layer21_attention_weight_max": 31.82023811340332,
      "activations/layer21_attention_weight_min": -24.399436950683594,
      "activations/layer22_attention_weight_max": 47.991329193115234,
      "activations/layer22_attention_weight_min": -29.440364837646484,
      "activations/layer23_attention_weight_max": 34.22737121582031,
      "activations/layer23_attention_weight_min": -24.24908447265625,
      "activations/layer2_attention_weight_max": 31.54102325439453,
      "activations/layer2_attention_weight_min": -32.33446502685547,
      "activations/layer3_attention_weight_max": 93.75017547607422,
      "activations/layer3_attention_weight_min": -94.62053680419922,
      "activations/layer4_attention_weight_max": 114.36797332763672,
      "activations/layer4_attention_weight_min": -111.98221588134766,
      "activations/layer5_attention_weight_max": 53.33506774902344,
      "activations/layer5_attention_weight_min": -61.03779602050781,
      "activations/layer6_attention_weight_max": 46.89009475708008,
      "activations/layer6_attention_weight_min": -48.98317337036133,
      "activations/layer7_attention_weight_max": 86.98373413085938,
      "activations/layer7_attention_weight_min": -88.72037506103516,
      "activations/layer8_attention_weight_max": 40.21052169799805,
      "activations/layer8_attention_weight_min": -41.158668518066406,
      "activations/layer9_attention_weight_max": 33.36909866333008,
      "activations/layer9_attention_weight_min": -33.38994216918945,
      "epoch": 9.85,
      "learning_rate": 8.732840909090909e-05,
      "loss": 2.7905,
      "step": 169600
    },
    {
      "activations/layer0_attention_weight_max": 14.264030456542969,
      "activations/layer0_attention_weight_min": -12.26963996887207,
      "activations/layer10_attention_weight_max": 32.651912689208984,
      "activations/layer10_attention_weight_min": -32.30221939086914,
      "activations/layer11_attention_weight_max": 32.148677825927734,
      "activations/layer11_attention_weight_min": -32.57696533203125,
      "activations/layer12_attention_weight_max": 22.54293441772461,
      "activations/layer12_attention_weight_min": -24.692039489746094,
      "activations/layer13_attention_weight_max": 34.988460540771484,
      "activations/layer13_attention_weight_min": -34.85047149658203,
      "activations/layer14_attention_weight_max": 42.38120651245117,
      "activations/layer14_attention_weight_min": -39.996334075927734,
      "activations/layer15_attention_weight_max": 33.308868408203125,
      "activations/layer15_attention_weight_min": -32.730079650878906,
      "activations/layer16_attention_weight_max": 29.522085189819336,
      "activations/layer16_attention_weight_min": -31.089534759521484,
      "activations/layer17_attention_weight_max": 29.43990707397461,
      "activations/layer17_attention_weight_min": -27.34673500061035,
      "activations/layer18_attention_weight_max": 28.837190628051758,
      "activations/layer18_attention_weight_min": -23.37251091003418,
      "activations/layer19_attention_weight_max": 35.04155349731445,
      "activations/layer19_attention_weight_min": -31.17282485961914,
      "activations/layer1_attention_weight_max": 16.464441299438477,
      "activations/layer1_attention_weight_min": -16.349693298339844,
      "activations/layer20_attention_weight_max": 26.519634246826172,
      "activations/layer20_attention_weight_min": -25.70723533630371,
      "activations/layer21_attention_weight_max": 26.956768035888672,
      "activations/layer21_attention_weight_min": -24.946117401123047,
      "activations/layer22_attention_weight_max": 43.50816345214844,
      "activations/layer22_attention_weight_min": -30.558759689331055,
      "activations/layer23_attention_weight_max": 31.2169246673584,
      "activations/layer23_attention_weight_min": -23.730331420898438,
      "activations/layer2_attention_weight_max": 32.645423889160156,
      "activations/layer2_attention_weight_min": -35.79168701171875,
      "activations/layer3_attention_weight_max": 91.92012023925781,
      "activations/layer3_attention_weight_min": -96.95025634765625,
      "activations/layer4_attention_weight_max": 112.35607147216797,
      "activations/layer4_attention_weight_min": -116.03169250488281,
      "activations/layer5_attention_weight_max": 51.76373291015625,
      "activations/layer5_attention_weight_min": -60.893524169921875,
      "activations/layer6_attention_weight_max": 45.244667053222656,
      "activations/layer6_attention_weight_min": -46.52790069580078,
      "activations/layer7_attention_weight_max": 84.59717559814453,
      "activations/layer7_attention_weight_min": -89.62396240234375,
      "activations/layer8_attention_weight_max": 38.38936233520508,
      "activations/layer8_attention_weight_min": -41.10508346557617,
      "activations/layer9_attention_weight_max": 33.83316421508789,
      "activations/layer9_attention_weight_min": -33.114501953125,
      "epoch": 9.86,
      "learning_rate": 8.73094696969697e-05,
      "loss": 2.7974,
      "step": 169650
    },
    {
      "activations/layer0_attention_weight_max": 14.970842361450195,
      "activations/layer0_attention_weight_min": -13.021248817443848,
      "activations/layer10_attention_weight_max": 34.70013427734375,
      "activations/layer10_attention_weight_min": -35.39045333862305,
      "activations/layer11_attention_weight_max": 35.518463134765625,
      "activations/layer11_attention_weight_min": -36.148475646972656,
      "activations/layer12_attention_weight_max": 42.44973373413086,
      "activations/layer12_attention_weight_min": -26.73676300048828,
      "activations/layer13_attention_weight_max": 53.38764953613281,
      "activations/layer13_attention_weight_min": -37.308197021484375,
      "activations/layer14_attention_weight_max": 47.41830825805664,
      "activations/layer14_attention_weight_min": -43.47228240966797,
      "activations/layer15_attention_weight_max": 40.821067810058594,
      "activations/layer15_attention_weight_min": -37.61832809448242,
      "activations/layer16_attention_weight_max": 35.35981369018555,
      "activations/layer16_attention_weight_min": -29.822002410888672,
      "activations/layer17_attention_weight_max": 38.65471267700195,
      "activations/layer17_attention_weight_min": -27.4939022064209,
      "activations/layer18_attention_weight_max": 35.41530990600586,
      "activations/layer18_attention_weight_min": -26.054943084716797,
      "activations/layer19_attention_weight_max": 45.955257415771484,
      "activations/layer19_attention_weight_min": -31.15900993347168,
      "activations/layer1_attention_weight_max": 16.651718139648438,
      "activations/layer1_attention_weight_min": -17.275920867919922,
      "activations/layer20_attention_weight_max": 39.9157600402832,
      "activations/layer20_attention_weight_min": -25.66658592224121,
      "activations/layer21_attention_weight_max": 36.078269958496094,
      "activations/layer21_attention_weight_min": -24.851415634155273,
      "activations/layer22_attention_weight_max": 55.97544860839844,
      "activations/layer22_attention_weight_min": -31.516357421875,
      "activations/layer23_attention_weight_max": 42.89516830444336,
      "activations/layer23_attention_weight_min": -25.7824764251709,
      "activations/layer2_attention_weight_max": 33.742469787597656,
      "activations/layer2_attention_weight_min": -35.66636276245117,
      "activations/layer3_attention_weight_max": 100.21398162841797,
      "activations/layer3_attention_weight_min": -98.0324478149414,
      "activations/layer4_attention_weight_max": 119.58319091796875,
      "activations/layer4_attention_weight_min": -115.15850830078125,
      "activations/layer5_attention_weight_max": 55.09309387207031,
      "activations/layer5_attention_weight_min": -63.57679748535156,
      "activations/layer6_attention_weight_max": 46.893348693847656,
      "activations/layer6_attention_weight_min": -52.638206481933594,
      "activations/layer7_attention_weight_max": 95.34309387207031,
      "activations/layer7_attention_weight_min": -90.57732391357422,
      "activations/layer8_attention_weight_max": 41.16878890991211,
      "activations/layer8_attention_weight_min": -45.3995246887207,
      "activations/layer9_attention_weight_max": 35.683929443359375,
      "activations/layer9_attention_weight_min": -34.99485397338867,
      "epoch": 9.86,
      "learning_rate": 8.729053030303029e-05,
      "loss": 2.7864,
      "step": 169700
    },
    {
      "activations/layer0_attention_weight_max": 15.582669258117676,
      "activations/layer0_attention_weight_min": -11.863927841186523,
      "activations/layer10_attention_weight_max": 35.03352355957031,
      "activations/layer10_attention_weight_min": -33.446144104003906,
      "activations/layer11_attention_weight_max": 36.56573486328125,
      "activations/layer11_attention_weight_min": -34.29837417602539,
      "activations/layer12_attention_weight_max": 29.108631134033203,
      "activations/layer12_attention_weight_min": -26.700210571289062,
      "activations/layer13_attention_weight_max": 45.06642150878906,
      "activations/layer13_attention_weight_min": -34.2006721496582,
      "activations/layer14_attention_weight_max": 58.00323486328125,
      "activations/layer14_attention_weight_min": -42.41094970703125,
      "activations/layer15_attention_weight_max": 43.515350341796875,
      "activations/layer15_attention_weight_min": -35.013065338134766,
      "activations/layer16_attention_weight_max": 32.90968704223633,
      "activations/layer16_attention_weight_min": -28.86322021484375,
      "activations/layer17_attention_weight_max": 32.8461799621582,
      "activations/layer17_attention_weight_min": -27.54429054260254,
      "activations/layer18_attention_weight_max": 31.904130935668945,
      "activations/layer18_attention_weight_min": -25.08400535583496,
      "activations/layer19_attention_weight_max": 37.38895797729492,
      "activations/layer19_attention_weight_min": -32.32123947143555,
      "activations/layer1_attention_weight_max": 17.646310806274414,
      "activations/layer1_attention_weight_min": -15.174440383911133,
      "activations/layer20_attention_weight_max": 29.45062828063965,
      "activations/layer20_attention_weight_min": -26.203533172607422,
      "activations/layer21_attention_weight_max": 30.64272689819336,
      "activations/layer21_attention_weight_min": -23.609512329101562,
      "activations/layer22_attention_weight_max": 52.10563278198242,
      "activations/layer22_attention_weight_min": -30.135498046875,
      "activations/layer23_attention_weight_max": 40.5283203125,
      "activations/layer23_attention_weight_min": -24.994224548339844,
      "activations/layer2_attention_weight_max": 30.70427703857422,
      "activations/layer2_attention_weight_min": -33.00335693359375,
      "activations/layer3_attention_weight_max": 92.44403839111328,
      "activations/layer3_attention_weight_min": -94.69947052001953,
      "activations/layer4_attention_weight_max": 110.0865478515625,
      "activations/layer4_attention_weight_min": -111.32225799560547,
      "activations/layer5_attention_weight_max": 53.63035202026367,
      "activations/layer5_attention_weight_min": -63.89598083496094,
      "activations/layer6_attention_weight_max": 44.664432525634766,
      "activations/layer6_attention_weight_min": -48.806400299072266,
      "activations/layer7_attention_weight_max": 97.71829223632812,
      "activations/layer7_attention_weight_min": -85.37660217285156,
      "activations/layer8_attention_weight_max": 41.678123474121094,
      "activations/layer8_attention_weight_min": -41.234840393066406,
      "activations/layer9_attention_weight_max": 36.27859878540039,
      "activations/layer9_attention_weight_min": -34.015689849853516,
      "epoch": 9.86,
      "learning_rate": 8.72715909090909e-05,
      "loss": 2.782,
      "step": 169750
    },
    {
      "activations/layer0_attention_weight_max": 14.523188591003418,
      "activations/layer0_attention_weight_min": -11.345611572265625,
      "activations/layer10_attention_weight_max": 33.645896911621094,
      "activations/layer10_attention_weight_min": -30.9842586517334,
      "activations/layer11_attention_weight_max": 35.11029815673828,
      "activations/layer11_attention_weight_min": -32.04386520385742,
      "activations/layer12_attention_weight_max": 26.3323917388916,
      "activations/layer12_attention_weight_min": -25.716840744018555,
      "activations/layer13_attention_weight_max": 43.31635665893555,
      "activations/layer13_attention_weight_min": -35.628379821777344,
      "activations/layer14_attention_weight_max": 51.89971923828125,
      "activations/layer14_attention_weight_min": -38.66634750366211,
      "activations/layer15_attention_weight_max": 38.447959899902344,
      "activations/layer15_attention_weight_min": -30.46473503112793,
      "activations/layer16_attention_weight_max": 34.141056060791016,
      "activations/layer16_attention_weight_min": -27.090105056762695,
      "activations/layer17_attention_weight_max": 35.87689208984375,
      "activations/layer17_attention_weight_min": -25.487380981445312,
      "activations/layer18_attention_weight_max": 36.767486572265625,
      "activations/layer18_attention_weight_min": -23.917490005493164,
      "activations/layer19_attention_weight_max": 41.620452880859375,
      "activations/layer19_attention_weight_min": -31.980939865112305,
      "activations/layer1_attention_weight_max": 17.1744441986084,
      "activations/layer1_attention_weight_min": -15.604857444763184,
      "activations/layer20_attention_weight_max": 32.71889114379883,
      "activations/layer20_attention_weight_min": -27.231101989746094,
      "activations/layer21_attention_weight_max": 32.48508071899414,
      "activations/layer21_attention_weight_min": -25.871915817260742,
      "activations/layer22_attention_weight_max": 49.04130935668945,
      "activations/layer22_attention_weight_min": -30.48826789855957,
      "activations/layer23_attention_weight_max": 45.27288055419922,
      "activations/layer23_attention_weight_min": -26.033613204956055,
      "activations/layer2_attention_weight_max": 32.48729705810547,
      "activations/layer2_attention_weight_min": -32.52271270751953,
      "activations/layer3_attention_weight_max": 90.47315979003906,
      "activations/layer3_attention_weight_min": -91.67446899414062,
      "activations/layer4_attention_weight_max": 109.7776870727539,
      "activations/layer4_attention_weight_min": -114.19685363769531,
      "activations/layer5_attention_weight_max": 54.19919967651367,
      "activations/layer5_attention_weight_min": -61.24772262573242,
      "activations/layer6_attention_weight_max": 44.062068939208984,
      "activations/layer6_attention_weight_min": -49.4748420715332,
      "activations/layer7_attention_weight_max": 92.4289321899414,
      "activations/layer7_attention_weight_min": -85.04405212402344,
      "activations/layer8_attention_weight_max": 39.66683578491211,
      "activations/layer8_attention_weight_min": -39.836326599121094,
      "activations/layer9_attention_weight_max": 36.41868209838867,
      "activations/layer9_attention_weight_min": -32.59450149536133,
      "epoch": 9.87,
      "learning_rate": 8.72526515151515e-05,
      "loss": 2.8124,
      "step": 169800
    },
    {
      "activations/layer0_attention_weight_max": 15.297154426574707,
      "activations/layer0_attention_weight_min": -12.16257095336914,
      "activations/layer10_attention_weight_max": 33.602325439453125,
      "activations/layer10_attention_weight_min": -33.97252655029297,
      "activations/layer11_attention_weight_max": 35.58375549316406,
      "activations/layer11_attention_weight_min": -31.97669219970703,
      "activations/layer12_attention_weight_max": 34.4995231628418,
      "activations/layer12_attention_weight_min": -24.776371002197266,
      "activations/layer13_attention_weight_max": 39.929317474365234,
      "activations/layer13_attention_weight_min": -33.243560791015625,
      "activations/layer14_attention_weight_max": 47.61947250366211,
      "activations/layer14_attention_weight_min": -37.27132034301758,
      "activations/layer15_attention_weight_max": 37.38340377807617,
      "activations/layer15_attention_weight_min": -31.848176956176758,
      "activations/layer16_attention_weight_max": 32.93437957763672,
      "activations/layer16_attention_weight_min": -29.76463508605957,
      "activations/layer17_attention_weight_max": 30.908260345458984,
      "activations/layer17_attention_weight_min": -25.472782135009766,
      "activations/layer18_attention_weight_max": 34.74394989013672,
      "activations/layer18_attention_weight_min": -24.721485137939453,
      "activations/layer19_attention_weight_max": 41.57590866088867,
      "activations/layer19_attention_weight_min": -34.265438079833984,
      "activations/layer1_attention_weight_max": 16.81869125366211,
      "activations/layer1_attention_weight_min": -16.0576114654541,
      "activations/layer20_attention_weight_max": 33.567378997802734,
      "activations/layer20_attention_weight_min": -26.936660766601562,
      "activations/layer21_attention_weight_max": 31.02785873413086,
      "activations/layer21_attention_weight_min": -24.449996948242188,
      "activations/layer22_attention_weight_max": 52.8276481628418,
      "activations/layer22_attention_weight_min": -31.65546417236328,
      "activations/layer23_attention_weight_max": 38.78053283691406,
      "activations/layer23_attention_weight_min": -26.75250816345215,
      "activations/layer2_attention_weight_max": 32.87928009033203,
      "activations/layer2_attention_weight_min": -34.159420013427734,
      "activations/layer3_attention_weight_max": 95.8703384399414,
      "activations/layer3_attention_weight_min": -98.96681213378906,
      "activations/layer4_attention_weight_max": 114.1399154663086,
      "activations/layer4_attention_weight_min": -115.11775207519531,
      "activations/layer5_attention_weight_max": 54.03298568725586,
      "activations/layer5_attention_weight_min": -62.653839111328125,
      "activations/layer6_attention_weight_max": 46.338645935058594,
      "activations/layer6_attention_weight_min": -51.54836654663086,
      "activations/layer7_attention_weight_max": 88.8707504272461,
      "activations/layer7_attention_weight_min": -90.27838897705078,
      "activations/layer8_attention_weight_max": 40.380645751953125,
      "activations/layer8_attention_weight_min": -43.0294075012207,
      "activations/layer9_attention_weight_max": 34.81233596801758,
      "activations/layer9_attention_weight_min": -35.031150817871094,
      "epoch": 9.87,
      "learning_rate": 8.72337121212121e-05,
      "loss": 2.7834,
      "step": 169850
    },
    {
      "activations/layer0_attention_weight_max": 15.686487197875977,
      "activations/layer0_attention_weight_min": -12.002820014953613,
      "activations/layer10_attention_weight_max": 31.01520347595215,
      "activations/layer10_attention_weight_min": -30.957317352294922,
      "activations/layer11_attention_weight_max": 33.24359893798828,
      "activations/layer11_attention_weight_min": -29.65746307373047,
      "activations/layer12_attention_weight_max": 25.392250061035156,
      "activations/layer12_attention_weight_min": -25.495868682861328,
      "activations/layer13_attention_weight_max": 37.4462890625,
      "activations/layer13_attention_weight_min": -32.92407989501953,
      "activations/layer14_attention_weight_max": 45.53128433227539,
      "activations/layer14_attention_weight_min": -36.956321716308594,
      "activations/layer15_attention_weight_max": 35.820011138916016,
      "activations/layer15_attention_weight_min": -30.916725158691406,
      "activations/layer16_attention_weight_max": 35.19969940185547,
      "activations/layer16_attention_weight_min": -28.029796600341797,
      "activations/layer17_attention_weight_max": 32.37670135498047,
      "activations/layer17_attention_weight_min": -24.829471588134766,
      "activations/layer18_attention_weight_max": 33.7059326171875,
      "activations/layer18_attention_weight_min": -22.077640533447266,
      "activations/layer19_attention_weight_max": 37.537654876708984,
      "activations/layer19_attention_weight_min": -29.313199996948242,
      "activations/layer1_attention_weight_max": 16.915348052978516,
      "activations/layer1_attention_weight_min": -14.184447288513184,
      "activations/layer20_attention_weight_max": 30.31761360168457,
      "activations/layer20_attention_weight_min": -24.172306060791016,
      "activations/layer21_attention_weight_max": 29.00084114074707,
      "activations/layer21_attention_weight_min": -23.017929077148438,
      "activations/layer22_attention_weight_max": 44.95072555541992,
      "activations/layer22_attention_weight_min": -27.77834701538086,
      "activations/layer23_attention_weight_max": 36.50332260131836,
      "activations/layer23_attention_weight_min": -24.11768913269043,
      "activations/layer2_attention_weight_max": 33.380027770996094,
      "activations/layer2_attention_weight_min": -30.224830627441406,
      "activations/layer3_attention_weight_max": 89.40287017822266,
      "activations/layer3_attention_weight_min": -87.30755615234375,
      "activations/layer4_attention_weight_max": 116.16607666015625,
      "activations/layer4_attention_weight_min": -109.38629150390625,
      "activations/layer5_attention_weight_max": 54.93828582763672,
      "activations/layer5_attention_weight_min": -60.511627197265625,
      "activations/layer6_attention_weight_max": 43.76472854614258,
      "activations/layer6_attention_weight_min": -47.332664489746094,
      "activations/layer7_attention_weight_max": 83.09260559082031,
      "activations/layer7_attention_weight_min": -86.1370849609375,
      "activations/layer8_attention_weight_max": 38.493255615234375,
      "activations/layer8_attention_weight_min": -39.335147857666016,
      "activations/layer9_attention_weight_max": 32.327781677246094,
      "activations/layer9_attention_weight_min": -31.42488670349121,
      "epoch": 9.87,
      "learning_rate": 8.721477272727272e-05,
      "loss": 2.7928,
      "step": 169900
    },
    {
      "activations/layer0_attention_weight_max": 15.424379348754883,
      "activations/layer0_attention_weight_min": -12.809013366699219,
      "activations/layer10_attention_weight_max": 31.647830963134766,
      "activations/layer10_attention_weight_min": -29.94740867614746,
      "activations/layer11_attention_weight_max": 33.09721755981445,
      "activations/layer11_attention_weight_min": -29.679166793823242,
      "activations/layer12_attention_weight_max": 23.618772506713867,
      "activations/layer12_attention_weight_min": -24.391813278198242,
      "activations/layer13_attention_weight_max": 35.94540786743164,
      "activations/layer13_attention_weight_min": -34.281158447265625,
      "activations/layer14_attention_weight_max": 45.446693420410156,
      "activations/layer14_attention_weight_min": -36.692665100097656,
      "activations/layer15_attention_weight_max": 32.63786315917969,
      "activations/layer15_attention_weight_min": -29.816038131713867,
      "activations/layer16_attention_weight_max": 30.719453811645508,
      "activations/layer16_attention_weight_min": -26.353025436401367,
      "activations/layer17_attention_weight_max": 32.40078353881836,
      "activations/layer17_attention_weight_min": -25.171056747436523,
      "activations/layer18_attention_weight_max": 34.190757751464844,
      "activations/layer18_attention_weight_min": -22.444679260253906,
      "activations/layer19_attention_weight_max": 31.408920288085938,
      "activations/layer19_attention_weight_min": -31.404743194580078,
      "activations/layer1_attention_weight_max": 16.870695114135742,
      "activations/layer1_attention_weight_min": -17.52726936340332,
      "activations/layer20_attention_weight_max": 27.849002838134766,
      "activations/layer20_attention_weight_min": -24.530967712402344,
      "activations/layer21_attention_weight_max": 29.671844482421875,
      "activations/layer21_attention_weight_min": -23.470006942749023,
      "activations/layer22_attention_weight_max": 42.97321319580078,
      "activations/layer22_attention_weight_min": -30.150930404663086,
      "activations/layer23_attention_weight_max": 30.58517074584961,
      "activations/layer23_attention_weight_min": -25.61977767944336,
      "activations/layer2_attention_weight_max": 32.9047737121582,
      "activations/layer2_attention_weight_min": -32.888668060302734,
      "activations/layer3_attention_weight_max": 93.38064575195312,
      "activations/layer3_attention_weight_min": -93.50540924072266,
      "activations/layer4_attention_weight_max": 112.8816909790039,
      "activations/layer4_attention_weight_min": -110.72633361816406,
      "activations/layer5_attention_weight_max": 53.327362060546875,
      "activations/layer5_attention_weight_min": -62.37371063232422,
      "activations/layer6_attention_weight_max": 45.080875396728516,
      "activations/layer6_attention_weight_min": -49.15800094604492,
      "activations/layer7_attention_weight_max": 84.56343841552734,
      "activations/layer7_attention_weight_min": -94.80496978759766,
      "activations/layer8_attention_weight_max": 38.583370208740234,
      "activations/layer8_attention_weight_min": -41.41189193725586,
      "activations/layer9_attention_weight_max": 34.202293395996094,
      "activations/layer9_attention_weight_min": -33.2058219909668,
      "epoch": 9.88,
      "learning_rate": 8.719583333333332e-05,
      "loss": 2.7749,
      "step": 169950
    },
    {
      "activations/layer0_attention_weight_max": 15.538890838623047,
      "activations/layer0_attention_weight_min": -12.568177223205566,
      "activations/layer10_attention_weight_max": 31.34249496459961,
      "activations/layer10_attention_weight_min": -32.4852180480957,
      "activations/layer11_attention_weight_max": 30.86383628845215,
      "activations/layer11_attention_weight_min": -31.126264572143555,
      "activations/layer12_attention_weight_max": 24.2268009185791,
      "activations/layer12_attention_weight_min": -26.70581817626953,
      "activations/layer13_attention_weight_max": 40.027061462402344,
      "activations/layer13_attention_weight_min": -36.15855026245117,
      "activations/layer14_attention_weight_max": 54.35585021972656,
      "activations/layer14_attention_weight_min": -45.5425910949707,
      "activations/layer15_attention_weight_max": 36.42268371582031,
      "activations/layer15_attention_weight_min": -31.418704986572266,
      "activations/layer16_attention_weight_max": 30.622024536132812,
      "activations/layer16_attention_weight_min": -27.793310165405273,
      "activations/layer17_attention_weight_max": 33.65685272216797,
      "activations/layer17_attention_weight_min": -25.092248916625977,
      "activations/layer18_attention_weight_max": 30.728609085083008,
      "activations/layer18_attention_weight_min": -22.33909797668457,
      "activations/layer19_attention_weight_max": 33.96755599975586,
      "activations/layer19_attention_weight_min": -32.14577102661133,
      "activations/layer1_attention_weight_max": 16.728147506713867,
      "activations/layer1_attention_weight_min": -14.909087181091309,
      "activations/layer20_attention_weight_max": 34.563968658447266,
      "activations/layer20_attention_weight_min": -23.751232147216797,
      "activations/layer21_attention_weight_max": 29.582489013671875,
      "activations/layer21_attention_weight_min": -22.23356056213379,
      "activations/layer22_attention_weight_max": 47.72007369995117,
      "activations/layer22_attention_weight_min": -30.65835189819336,
      "activations/layer23_attention_weight_max": 40.15330505371094,
      "activations/layer23_attention_weight_min": -23.827735900878906,
      "activations/layer2_attention_weight_max": 33.860595703125,
      "activations/layer2_attention_weight_min": -31.56671142578125,
      "activations/layer3_attention_weight_max": 98.33638763427734,
      "activations/layer3_attention_weight_min": -93.10166931152344,
      "activations/layer4_attention_weight_max": 117.55250549316406,
      "activations/layer4_attention_weight_min": -119.4389877319336,
      "activations/layer5_attention_weight_max": 55.97615051269531,
      "activations/layer5_attention_weight_min": -62.55389404296875,
      "activations/layer6_attention_weight_max": 45.8184928894043,
      "activations/layer6_attention_weight_min": -48.95882034301758,
      "activations/layer7_attention_weight_max": 86.73001098632812,
      "activations/layer7_attention_weight_min": -86.9803466796875,
      "activations/layer8_attention_weight_max": 38.76777267456055,
      "activations/layer8_attention_weight_min": -39.855960845947266,
      "activations/layer9_attention_weight_max": 31.86012077331543,
      "activations/layer9_attention_weight_min": -32.34178161621094,
      "epoch": 9.88,
      "learning_rate": 8.717689393939394e-05,
      "loss": 2.7865,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.4637,
      "eval_samples_per_second": 507.345,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.4637,
      "eval_openwebtext_samples_per_second": 507.345,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9422,
      "eval_wikitext_samples_per_second": 234.781,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.4839,
      "eval_lambada_samples_per_second": 513.395,
      "step": 170000
    },
    {
      "activations/layer0_attention_weight_max": 17.670421600341797,
      "activations/layer0_attention_weight_min": -12.780308723449707,
      "activations/layer10_attention_weight_max": 32.383033752441406,
      "activations/layer10_attention_weight_min": -33.328826904296875,
      "activations/layer11_attention_weight_max": 33.547279357910156,
      "activations/layer11_attention_weight_min": -31.68324851989746,
      "activations/layer12_attention_weight_max": 24.31330680847168,
      "activations/layer12_attention_weight_min": -24.38967514038086,
      "activations/layer13_attention_weight_max": 41.577735900878906,
      "activations/layer13_attention_weight_min": -33.51845932006836,
      "activations/layer14_attention_weight_max": 57.81511306762695,
      "activations/layer14_attention_weight_min": -39.978092193603516,
      "activations/layer15_attention_weight_max": 35.48497772216797,
      "activations/layer15_attention_weight_min": -30.548622131347656,
      "activations/layer16_attention_weight_max": 31.32759666442871,
      "activations/layer16_attention_weight_min": -27.381254196166992,
      "activations/layer17_attention_weight_max": 31.78643226623535,
      "activations/layer17_attention_weight_min": -26.63384246826172,
      "activations/layer18_attention_weight_max": 29.750179290771484,
      "activations/layer18_attention_weight_min": -23.231966018676758,
      "activations/layer19_attention_weight_max": 35.86346435546875,
      "activations/layer19_attention_weight_min": -32.4730224609375,
      "activations/layer1_attention_weight_max": 16.43720817565918,
      "activations/layer1_attention_weight_min": -14.70377254486084,
      "activations/layer20_attention_weight_max": 32.47832489013672,
      "activations/layer20_attention_weight_min": -25.16197967529297,
      "activations/layer21_attention_weight_max": 32.664039611816406,
      "activations/layer21_attention_weight_min": -24.204885482788086,
      "activations/layer22_attention_weight_max": 47.044151306152344,
      "activations/layer22_attention_weight_min": -27.99894905090332,
      "activations/layer23_attention_weight_max": 37.07490539550781,
      "activations/layer23_attention_weight_min": -23.143447875976562,
      "activations/layer2_attention_weight_max": 32.37855529785156,
      "activations/layer2_attention_weight_min": -33.64942932128906,
      "activations/layer3_attention_weight_max": 93.79863739013672,
      "activations/layer3_attention_weight_min": -94.55254364013672,
      "activations/layer4_attention_weight_max": 107.65042877197266,
      "activations/layer4_attention_weight_min": -110.23725891113281,
      "activations/layer5_attention_weight_max": 53.898048400878906,
      "activations/layer5_attention_weight_min": -61.638484954833984,
      "activations/layer6_attention_weight_max": 44.350040435791016,
      "activations/layer6_attention_weight_min": -50.30280685424805,
      "activations/layer7_attention_weight_max": 87.60594177246094,
      "activations/layer7_attention_weight_min": -91.30345153808594,
      "activations/layer8_attention_weight_max": 38.20366287231445,
      "activations/layer8_attention_weight_min": -42.528934478759766,
      "activations/layer9_attention_weight_max": 33.5362434387207,
      "activations/layer9_attention_weight_min": -33.4399528503418,
      "epoch": 9.88,
      "learning_rate": 8.715795454545454e-05,
      "loss": 2.7901,
      "step": 170050
    },
    {
      "activations/layer0_attention_weight_max": 15.10033893585205,
      "activations/layer0_attention_weight_min": -12.701424598693848,
      "activations/layer10_attention_weight_max": 37.06072998046875,
      "activations/layer10_attention_weight_min": -36.49934768676758,
      "activations/layer11_attention_weight_max": 36.34308624267578,
      "activations/layer11_attention_weight_min": -34.341552734375,
      "activations/layer12_attention_weight_max": 27.571836471557617,
      "activations/layer12_attention_weight_min": -26.120241165161133,
      "activations/layer13_attention_weight_max": 42.10744094848633,
      "activations/layer13_attention_weight_min": -36.50425338745117,
      "activations/layer14_attention_weight_max": 54.04564666748047,
      "activations/layer14_attention_weight_min": -44.64040756225586,
      "activations/layer15_attention_weight_max": 47.91604995727539,
      "activations/layer15_attention_weight_min": -34.31843185424805,
      "activations/layer16_attention_weight_max": 32.359066009521484,
      "activations/layer16_attention_weight_min": -28.362916946411133,
      "activations/layer17_attention_weight_max": 36.43048095703125,
      "activations/layer17_attention_weight_min": -26.52293586730957,
      "activations/layer18_attention_weight_max": 33.86341857910156,
      "activations/layer18_attention_weight_min": -22.582279205322266,
      "activations/layer19_attention_weight_max": 39.10380172729492,
      "activations/layer19_attention_weight_min": -34.19931411743164,
      "activations/layer1_attention_weight_max": 17.049781799316406,
      "activations/layer1_attention_weight_min": -14.259894371032715,
      "activations/layer20_attention_weight_max": 34.41016387939453,
      "activations/layer20_attention_weight_min": -25.043806076049805,
      "activations/layer21_attention_weight_max": 32.16756057739258,
      "activations/layer21_attention_weight_min": -24.45005226135254,
      "activations/layer22_attention_weight_max": 44.15007019042969,
      "activations/layer22_attention_weight_min": -29.814682006835938,
      "activations/layer23_attention_weight_max": 39.463966369628906,
      "activations/layer23_attention_weight_min": -25.324981689453125,
      "activations/layer2_attention_weight_max": 32.71317672729492,
      "activations/layer2_attention_weight_min": -33.136531829833984,
      "activations/layer3_attention_weight_max": 99.39573669433594,
      "activations/layer3_attention_weight_min": -107.47105407714844,
      "activations/layer4_attention_weight_max": 123.62591552734375,
      "activations/layer4_attention_weight_min": -123.36809539794922,
      "activations/layer5_attention_weight_max": 59.29758834838867,
      "activations/layer5_attention_weight_min": -61.81990051269531,
      "activations/layer6_attention_weight_max": 49.80373001098633,
      "activations/layer6_attention_weight_min": -52.70874786376953,
      "activations/layer7_attention_weight_max": 98.34468078613281,
      "activations/layer7_attention_weight_min": -94.93986511230469,
      "activations/layer8_attention_weight_max": 45.147972106933594,
      "activations/layer8_attention_weight_min": -45.50120544433594,
      "activations/layer9_attention_weight_max": 38.24755859375,
      "activations/layer9_attention_weight_min": -37.899349212646484,
      "epoch": 9.88,
      "learning_rate": 8.713939393939393e-05,
      "loss": 2.7921,
      "step": 170100
    },
    {
      "activations/layer0_attention_weight_max": 15.558951377868652,
      "activations/layer0_attention_weight_min": -11.801671028137207,
      "activations/layer10_attention_weight_max": 31.818050384521484,
      "activations/layer10_attention_weight_min": -31.756826400756836,
      "activations/layer11_attention_weight_max": 32.71347427368164,
      "activations/layer11_attention_weight_min": -32.09414291381836,
      "activations/layer12_attention_weight_max": 23.336618423461914,
      "activations/layer12_attention_weight_min": -24.139244079589844,
      "activations/layer13_attention_weight_max": 36.66210174560547,
      "activations/layer13_attention_weight_min": -33.24673080444336,
      "activations/layer14_attention_weight_max": 41.76395797729492,
      "activations/layer14_attention_weight_min": -37.00953674316406,
      "activations/layer15_attention_weight_max": 33.240135192871094,
      "activations/layer15_attention_weight_min": -29.916173934936523,
      "activations/layer16_attention_weight_max": 32.36507797241211,
      "activations/layer16_attention_weight_min": -28.234525680541992,
      "activations/layer17_attention_weight_max": 30.43592643737793,
      "activations/layer17_attention_weight_min": -24.765329360961914,
      "activations/layer18_attention_weight_max": 32.054935455322266,
      "activations/layer18_attention_weight_min": -22.598724365234375,
      "activations/layer19_attention_weight_max": 40.97880172729492,
      "activations/layer19_attention_weight_min": -32.433067321777344,
      "activations/layer1_attention_weight_max": 16.730134963989258,
      "activations/layer1_attention_weight_min": -16.12535858154297,
      "activations/layer20_attention_weight_max": 30.4490966796875,
      "activations/layer20_attention_weight_min": -24.140758514404297,
      "activations/layer21_attention_weight_max": 32.15812301635742,
      "activations/layer21_attention_weight_min": -23.74222183227539,
      "activations/layer22_attention_weight_max": 46.49009704589844,
      "activations/layer22_attention_weight_min": -30.12882423400879,
      "activations/layer23_attention_weight_max": 37.09672927856445,
      "activations/layer23_attention_weight_min": -25.581615447998047,
      "activations/layer2_attention_weight_max": 31.219770431518555,
      "activations/layer2_attention_weight_min": -32.28578186035156,
      "activations/layer3_attention_weight_max": 91.37371826171875,
      "activations/layer3_attention_weight_min": -92.53409576416016,
      "activations/layer4_attention_weight_max": 111.14641571044922,
      "activations/layer4_attention_weight_min": -112.29182434082031,
      "activations/layer5_attention_weight_max": 52.075035095214844,
      "activations/layer5_attention_weight_min": -62.233150482177734,
      "activations/layer6_attention_weight_max": 46.038719177246094,
      "activations/layer6_attention_weight_min": -51.97398376464844,
      "activations/layer7_attention_weight_max": 89.37429809570312,
      "activations/layer7_attention_weight_min": -93.98949432373047,
      "activations/layer8_attention_weight_max": 41.86468505859375,
      "activations/layer8_attention_weight_min": -42.58631896972656,
      "activations/layer9_attention_weight_max": 33.40189743041992,
      "activations/layer9_attention_weight_min": -35.42890930175781,
      "epoch": 9.89,
      "learning_rate": 8.712045454545453e-05,
      "loss": 2.7829,
      "step": 170150
    },
    {
      "activations/layer0_attention_weight_max": 15.888904571533203,
      "activations/layer0_attention_weight_min": -12.743903160095215,
      "activations/layer10_attention_weight_max": 31.235380172729492,
      "activations/layer10_attention_weight_min": -34.846370697021484,
      "activations/layer11_attention_weight_max": 31.941448211669922,
      "activations/layer11_attention_weight_min": -31.136472702026367,
      "activations/layer12_attention_weight_max": 25.83507537841797,
      "activations/layer12_attention_weight_min": -25.35413360595703,
      "activations/layer13_attention_weight_max": 38.96672439575195,
      "activations/layer13_attention_weight_min": -34.97834777832031,
      "activations/layer14_attention_weight_max": 43.230186462402344,
      "activations/layer14_attention_weight_min": -37.155487060546875,
      "activations/layer15_attention_weight_max": 35.58776092529297,
      "activations/layer15_attention_weight_min": -29.74976921081543,
      "activations/layer16_attention_weight_max": 31.47780418395996,
      "activations/layer16_attention_weight_min": -25.54537010192871,
      "activations/layer17_attention_weight_max": 33.626529693603516,
      "activations/layer17_attention_weight_min": -24.400117874145508,
      "activations/layer18_attention_weight_max": 30.58465576171875,
      "activations/layer18_attention_weight_min": -21.383609771728516,
      "activations/layer19_attention_weight_max": 35.458797454833984,
      "activations/layer19_attention_weight_min": -30.9906005859375,
      "activations/layer1_attention_weight_max": 17.02074432373047,
      "activations/layer1_attention_weight_min": -14.411296844482422,
      "activations/layer20_attention_weight_max": 31.32049560546875,
      "activations/layer20_attention_weight_min": -23.506240844726562,
      "activations/layer21_attention_weight_max": 28.59393882751465,
      "activations/layer21_attention_weight_min": -22.091678619384766,
      "activations/layer22_attention_weight_max": 45.629234313964844,
      "activations/layer22_attention_weight_min": -27.45760726928711,
      "activations/layer23_attention_weight_max": 34.188140869140625,
      "activations/layer23_attention_weight_min": -24.0087947845459,
      "activations/layer2_attention_weight_max": 30.590991973876953,
      "activations/layer2_attention_weight_min": -31.053434371948242,
      "activations/layer3_attention_weight_max": 87.09230041503906,
      "activations/layer3_attention_weight_min": -89.00775909423828,
      "activations/layer4_attention_weight_max": 104.30525970458984,
      "activations/layer4_attention_weight_min": -104.16124725341797,
      "activations/layer5_attention_weight_max": 48.799400329589844,
      "activations/layer5_attention_weight_min": -60.55267333984375,
      "activations/layer6_attention_weight_max": 44.00886535644531,
      "activations/layer6_attention_weight_min": -47.11751937866211,
      "activations/layer7_attention_weight_max": 85.44759368896484,
      "activations/layer7_attention_weight_min": -87.11153411865234,
      "activations/layer8_attention_weight_max": 38.133514404296875,
      "activations/layer8_attention_weight_min": -41.819976806640625,
      "activations/layer9_attention_weight_max": 33.519981384277344,
      "activations/layer9_attention_weight_min": -34.05380630493164,
      "epoch": 9.89,
      "learning_rate": 8.710151515151515e-05,
      "loss": 2.7832,
      "step": 170200
    },
    {
      "activations/layer0_attention_weight_max": 15.368989944458008,
      "activations/layer0_attention_weight_min": -12.878326416015625,
      "activations/layer10_attention_weight_max": 29.751787185668945,
      "activations/layer10_attention_weight_min": -30.29399299621582,
      "activations/layer11_attention_weight_max": 31.8792781829834,
      "activations/layer11_attention_weight_min": -30.84328269958496,
      "activations/layer12_attention_weight_max": 22.545011520385742,
      "activations/layer12_attention_weight_min": -25.29279899597168,
      "activations/layer13_attention_weight_max": 38.393829345703125,
      "activations/layer13_attention_weight_min": -35.04938507080078,
      "activations/layer14_attention_weight_max": 45.77088928222656,
      "activations/layer14_attention_weight_min": -40.681339263916016,
      "activations/layer15_attention_weight_max": 33.747440338134766,
      "activations/layer15_attention_weight_min": -30.652063369750977,
      "activations/layer16_attention_weight_max": 31.140274047851562,
      "activations/layer16_attention_weight_min": -28.648094177246094,
      "activations/layer17_attention_weight_max": 32.9066047668457,
      "activations/layer17_attention_weight_min": -27.037412643432617,
      "activations/layer18_attention_weight_max": 31.637428283691406,
      "activations/layer18_attention_weight_min": -23.36135482788086,
      "activations/layer19_attention_weight_max": 34.70722579956055,
      "activations/layer19_attention_weight_min": -33.55888748168945,
      "activations/layer1_attention_weight_max": 16.897968292236328,
      "activations/layer1_attention_weight_min": -13.925905227661133,
      "activations/layer20_attention_weight_max": 30.30312728881836,
      "activations/layer20_attention_weight_min": -25.94805908203125,
      "activations/layer21_attention_weight_max": 29.170318603515625,
      "activations/layer21_attention_weight_min": -24.884862899780273,
      "activations/layer22_attention_weight_max": 42.84844970703125,
      "activations/layer22_attention_weight_min": -31.464555740356445,
      "activations/layer23_attention_weight_max": 33.92247009277344,
      "activations/layer23_attention_weight_min": -27.938291549682617,
      "activations/layer2_attention_weight_max": 32.62339782714844,
      "activations/layer2_attention_weight_min": -31.526458740234375,
      "activations/layer3_attention_weight_max": 89.10611724853516,
      "activations/layer3_attention_weight_min": -92.81340026855469,
      "activations/layer4_attention_weight_max": 109.62794494628906,
      "activations/layer4_attention_weight_min": -105.27764129638672,
      "activations/layer5_attention_weight_max": 50.40223693847656,
      "activations/layer5_attention_weight_min": -58.50836181640625,
      "activations/layer6_attention_weight_max": 46.271400451660156,
      "activations/layer6_attention_weight_min": -48.917625427246094,
      "activations/layer7_attention_weight_max": 86.0384292602539,
      "activations/layer7_attention_weight_min": -86.00402069091797,
      "activations/layer8_attention_weight_max": 38.82761764526367,
      "activations/layer8_attention_weight_min": -41.71686553955078,
      "activations/layer9_attention_weight_max": 33.01124954223633,
      "activations/layer9_attention_weight_min": -31.5400390625,
      "epoch": 9.89,
      "learning_rate": 8.708257575757576e-05,
      "loss": 2.8011,
      "step": 170250
    },
    {
      "activations/layer0_attention_weight_max": 14.844507217407227,
      "activations/layer0_attention_weight_min": -12.117568016052246,
      "activations/layer10_attention_weight_max": 33.754146575927734,
      "activations/layer10_attention_weight_min": -32.02326583862305,
      "activations/layer11_attention_weight_max": 34.6032600402832,
      "activations/layer11_attention_weight_min": -32.830841064453125,
      "activations/layer12_attention_weight_max": 24.73641586303711,
      "activations/layer12_attention_weight_min": -26.38982582092285,
      "activations/layer13_attention_weight_max": 39.214683532714844,
      "activations/layer13_attention_weight_min": -33.06772994995117,
      "activations/layer14_attention_weight_max": 50.63716125488281,
      "activations/layer14_attention_weight_min": -40.283775329589844,
      "activations/layer15_attention_weight_max": 35.5844841003418,
      "activations/layer15_attention_weight_min": -30.473134994506836,
      "activations/layer16_attention_weight_max": 31.38102912902832,
      "activations/layer16_attention_weight_min": -27.525693893432617,
      "activations/layer17_attention_weight_max": 32.99557113647461,
      "activations/layer17_attention_weight_min": -25.02555274963379,
      "activations/layer18_attention_weight_max": 30.154380798339844,
      "activations/layer18_attention_weight_min": -23.007631301879883,
      "activations/layer19_attention_weight_max": 36.74468231201172,
      "activations/layer19_attention_weight_min": -32.038597106933594,
      "activations/layer1_attention_weight_max": 17.19620704650879,
      "activations/layer1_attention_weight_min": -15.346813201904297,
      "activations/layer20_attention_weight_max": 29.388498306274414,
      "activations/layer20_attention_weight_min": -24.985532760620117,
      "activations/layer21_attention_weight_max": 28.808469772338867,
      "activations/layer21_attention_weight_min": -23.585081100463867,
      "activations/layer22_attention_weight_max": 56.351139068603516,
      "activations/layer22_attention_weight_min": -28.62546157836914,
      "activations/layer23_attention_weight_max": 35.72273254394531,
      "activations/layer23_attention_weight_min": -22.37738609313965,
      "activations/layer2_attention_weight_max": 32.6806755065918,
      "activations/layer2_attention_weight_min": -32.06034469604492,
      "activations/layer3_attention_weight_max": 96.82683563232422,
      "activations/layer3_attention_weight_min": -96.3363265991211,
      "activations/layer4_attention_weight_max": 116.9700698852539,
      "activations/layer4_attention_weight_min": -115.37187957763672,
      "activations/layer5_attention_weight_max": 58.28269958496094,
      "activations/layer5_attention_weight_min": -63.41807174682617,
      "activations/layer6_attention_weight_max": 49.901981353759766,
      "activations/layer6_attention_weight_min": -49.90178680419922,
      "activations/layer7_attention_weight_max": 90.27252960205078,
      "activations/layer7_attention_weight_min": -90.75753021240234,
      "activations/layer8_attention_weight_max": 43.5426139831543,
      "activations/layer8_attention_weight_min": -41.95643997192383,
      "activations/layer9_attention_weight_max": 35.376224517822266,
      "activations/layer9_attention_weight_min": -34.56234359741211,
      "epoch": 9.9,
      "learning_rate": 8.706363636363635e-05,
      "loss": 2.7787,
      "step": 170300
    },
    {
      "activations/layer0_attention_weight_max": 16.023738861083984,
      "activations/layer0_attention_weight_min": -12.090579986572266,
      "activations/layer10_attention_weight_max": 31.099510192871094,
      "activations/layer10_attention_weight_min": -31.10643768310547,
      "activations/layer11_attention_weight_max": 31.065282821655273,
      "activations/layer11_attention_weight_min": -29.954809188842773,
      "activations/layer12_attention_weight_max": 29.6709041595459,
      "activations/layer12_attention_weight_min": -31.0198917388916,
      "activations/layer13_attention_weight_max": 36.40221405029297,
      "activations/layer13_attention_weight_min": -34.12129592895508,
      "activations/layer14_attention_weight_max": 42.86984634399414,
      "activations/layer14_attention_weight_min": -38.454349517822266,
      "activations/layer15_attention_weight_max": 36.601226806640625,
      "activations/layer15_attention_weight_min": -31.12131690979004,
      "activations/layer16_attention_weight_max": 32.067779541015625,
      "activations/layer16_attention_weight_min": -27.657176971435547,
      "activations/layer17_attention_weight_max": 31.891998291015625,
      "activations/layer17_attention_weight_min": -25.9063663482666,
      "activations/layer18_attention_weight_max": 28.507909774780273,
      "activations/layer18_attention_weight_min": -22.64889907836914,
      "activations/layer19_attention_weight_max": 33.81136703491211,
      "activations/layer19_attention_weight_min": -31.107349395751953,
      "activations/layer1_attention_weight_max": 16.67984962463379,
      "activations/layer1_attention_weight_min": -14.81289005279541,
      "activations/layer20_attention_weight_max": 28.171724319458008,
      "activations/layer20_attention_weight_min": -23.97199058532715,
      "activations/layer21_attention_weight_max": 27.17857551574707,
      "activations/layer21_attention_weight_min": -22.73240852355957,
      "activations/layer22_attention_weight_max": 41.79742431640625,
      "activations/layer22_attention_weight_min": -27.330041885375977,
      "activations/layer23_attention_weight_max": 35.233211517333984,
      "activations/layer23_attention_weight_min": -23.879133224487305,
      "activations/layer2_attention_weight_max": 31.494361877441406,
      "activations/layer2_attention_weight_min": -29.363353729248047,
      "activations/layer3_attention_weight_max": 91.88477325439453,
      "activations/layer3_attention_weight_min": -90.622802734375,
      "activations/layer4_attention_weight_max": 110.78590393066406,
      "activations/layer4_attention_weight_min": -109.62340545654297,
      "activations/layer5_attention_weight_max": 50.926204681396484,
      "activations/layer5_attention_weight_min": -64.53983306884766,
      "activations/layer6_attention_weight_max": 44.514976501464844,
      "activations/layer6_attention_weight_min": -47.42807388305664,
      "activations/layer7_attention_weight_max": 88.93518829345703,
      "activations/layer7_attention_weight_min": -88.54427337646484,
      "activations/layer8_attention_weight_max": 39.066165924072266,
      "activations/layer8_attention_weight_min": -39.457149505615234,
      "activations/layer9_attention_weight_max": 34.60605239868164,
      "activations/layer9_attention_weight_min": -32.38571548461914,
      "epoch": 9.9,
      "learning_rate": 8.704469696969696e-05,
      "loss": 2.7936,
      "step": 170350
    },
    {
      "activations/layer0_attention_weight_max": 15.339691162109375,
      "activations/layer0_attention_weight_min": -12.716575622558594,
      "activations/layer10_attention_weight_max": 39.873687744140625,
      "activations/layer10_attention_weight_min": -36.621826171875,
      "activations/layer11_attention_weight_max": 41.99076843261719,
      "activations/layer11_attention_weight_min": -36.79034423828125,
      "activations/layer12_attention_weight_max": 23.842178344726562,
      "activations/layer12_attention_weight_min": -30.778806686401367,
      "activations/layer13_attention_weight_max": 38.609832763671875,
      "activations/layer13_attention_weight_min": -32.14777755737305,
      "activations/layer14_attention_weight_max": 39.621376037597656,
      "activations/layer14_attention_weight_min": -34.28728103637695,
      "activations/layer15_attention_weight_max": 39.125328063964844,
      "activations/layer15_attention_weight_min": -30.42508888244629,
      "activations/layer16_attention_weight_max": 30.194713592529297,
      "activations/layer16_attention_weight_min": -26.334529876708984,
      "activations/layer17_attention_weight_max": 30.585763931274414,
      "activations/layer17_attention_weight_min": -27.426292419433594,
      "activations/layer18_attention_weight_max": 29.426076889038086,
      "activations/layer18_attention_weight_min": -21.501264572143555,
      "activations/layer19_attention_weight_max": 31.55984878540039,
      "activations/layer19_attention_weight_min": -29.481725692749023,
      "activations/layer1_attention_weight_max": 17.62183380126953,
      "activations/layer1_attention_weight_min": -15.785649299621582,
      "activations/layer20_attention_weight_max": 27.73284912109375,
      "activations/layer20_attention_weight_min": -24.302061080932617,
      "activations/layer21_attention_weight_max": 24.360055923461914,
      "activations/layer21_attention_weight_min": -22.99834632873535,
      "activations/layer22_attention_weight_max": 41.63215637207031,
      "activations/layer22_attention_weight_min": -32.01453399658203,
      "activations/layer23_attention_weight_max": 33.66691970825195,
      "activations/layer23_attention_weight_min": -23.28793716430664,
      "activations/layer2_attention_weight_max": 31.66420555114746,
      "activations/layer2_attention_weight_min": -31.150592803955078,
      "activations/layer3_attention_weight_max": 93.16424560546875,
      "activations/layer3_attention_weight_min": -93.21300506591797,
      "activations/layer4_attention_weight_max": 108.99007415771484,
      "activations/layer4_attention_weight_min": -108.22643280029297,
      "activations/layer5_attention_weight_max": 54.903404235839844,
      "activations/layer5_attention_weight_min": -58.92913818359375,
      "activations/layer6_attention_weight_max": 48.3712043762207,
      "activations/layer6_attention_weight_min": -47.12383270263672,
      "activations/layer7_attention_weight_max": 101.17076110839844,
      "activations/layer7_attention_weight_min": -91.5076904296875,
      "activations/layer8_attention_weight_max": 46.73350524902344,
      "activations/layer8_attention_weight_min": -39.49413299560547,
      "activations/layer9_attention_weight_max": 38.0782585144043,
      "activations/layer9_attention_weight_min": -35.537818908691406,
      "epoch": 9.9,
      "learning_rate": 8.702575757575758e-05,
      "loss": 2.788,
      "step": 170400
    },
    {
      "activations/layer0_attention_weight_max": 15.389655113220215,
      "activations/layer0_attention_weight_min": -12.851834297180176,
      "activations/layer10_attention_weight_max": 33.78767776489258,
      "activations/layer10_attention_weight_min": -34.23097610473633,
      "activations/layer11_attention_weight_max": 31.966720581054688,
      "activations/layer11_attention_weight_min": -31.364904403686523,
      "activations/layer12_attention_weight_max": 23.693296432495117,
      "activations/layer12_attention_weight_min": -26.047103881835938,
      "activations/layer13_attention_weight_max": 39.916534423828125,
      "activations/layer13_attention_weight_min": -34.33222198486328,
      "activations/layer14_attention_weight_max": 42.59025192260742,
      "activations/layer14_attention_weight_min": -37.72633361816406,
      "activations/layer15_attention_weight_max": 35.61807632446289,
      "activations/layer15_attention_weight_min": -29.78308868408203,
      "activations/layer16_attention_weight_max": 29.175487518310547,
      "activations/layer16_attention_weight_min": -27.25826644897461,
      "activations/layer17_attention_weight_max": 31.193872451782227,
      "activations/layer17_attention_weight_min": -24.30921173095703,
      "activations/layer18_attention_weight_max": 32.70207595825195,
      "activations/layer18_attention_weight_min": -23.8564453125,
      "activations/layer19_attention_weight_max": 34.72736740112305,
      "activations/layer19_attention_weight_min": -29.618642807006836,
      "activations/layer1_attention_weight_max": 16.697908401489258,
      "activations/layer1_attention_weight_min": -15.485335350036621,
      "activations/layer20_attention_weight_max": 29.660839080810547,
      "activations/layer20_attention_weight_min": -24.80011558532715,
      "activations/layer21_attention_weight_max": 30.338157653808594,
      "activations/layer21_attention_weight_min": -22.5823917388916,
      "activations/layer22_attention_weight_max": 48.4063835144043,
      "activations/layer22_attention_weight_min": -29.13702392578125,
      "activations/layer23_attention_weight_max": 37.722137451171875,
      "activations/layer23_attention_weight_min": -23.184329986572266,
      "activations/layer2_attention_weight_max": 31.725322723388672,
      "activations/layer2_attention_weight_min": -33.44572067260742,
      "activations/layer3_attention_weight_max": 97.44515991210938,
      "activations/layer3_attention_weight_min": -96.88880920410156,
      "activations/layer4_attention_weight_max": 118.95903778076172,
      "activations/layer4_attention_weight_min": -116.87999725341797,
      "activations/layer5_attention_weight_max": 55.40436553955078,
      "activations/layer5_attention_weight_min": -63.18067932128906,
      "activations/layer6_attention_weight_max": 48.226661682128906,
      "activations/layer6_attention_weight_min": -49.89777755737305,
      "activations/layer7_attention_weight_max": 92.7937240600586,
      "activations/layer7_attention_weight_min": -91.90538787841797,
      "activations/layer8_attention_weight_max": 40.580848693847656,
      "activations/layer8_attention_weight_min": -42.05908966064453,
      "activations/layer9_attention_weight_max": 33.244850158691406,
      "activations/layer9_attention_weight_min": -33.54488754272461,
      "epoch": 9.9,
      "learning_rate": 8.700681818181817e-05,
      "loss": 2.7779,
      "step": 170450
    },
    {
      "activations/layer0_attention_weight_max": 15.706493377685547,
      "activations/layer0_attention_weight_min": -12.80731201171875,
      "activations/layer10_attention_weight_max": 33.661556243896484,
      "activations/layer10_attention_weight_min": -32.229217529296875,
      "activations/layer11_attention_weight_max": 36.034732818603516,
      "activations/layer11_attention_weight_min": -31.202747344970703,
      "activations/layer12_attention_weight_max": 24.77218246459961,
      "activations/layer12_attention_weight_min": -26.24540901184082,
      "activations/layer13_attention_weight_max": 40.3614501953125,
      "activations/layer13_attention_weight_min": -33.54595947265625,
      "activations/layer14_attention_weight_max": 57.15727996826172,
      "activations/layer14_attention_weight_min": -41.80877685546875,
      "activations/layer15_attention_weight_max": 36.42593002319336,
      "activations/layer15_attention_weight_min": -31.3645076751709,
      "activations/layer16_attention_weight_max": 35.24037551879883,
      "activations/layer16_attention_weight_min": -27.408235549926758,
      "activations/layer17_attention_weight_max": 33.540897369384766,
      "activations/layer17_attention_weight_min": -26.582019805908203,
      "activations/layer18_attention_weight_max": 31.900611877441406,
      "activations/layer18_attention_weight_min": -21.238237380981445,
      "activations/layer19_attention_weight_max": 38.04242706298828,
      "activations/layer19_attention_weight_min": -31.468862533569336,
      "activations/layer1_attention_weight_max": 16.491024017333984,
      "activations/layer1_attention_weight_min": -14.310956001281738,
      "activations/layer20_attention_weight_max": 35.64069366455078,
      "activations/layer20_attention_weight_min": -24.50848960876465,
      "activations/layer21_attention_weight_max": 30.34883689880371,
      "activations/layer21_attention_weight_min": -22.647666931152344,
      "activations/layer22_attention_weight_max": 45.45134353637695,
      "activations/layer22_attention_weight_min": -29.623310089111328,
      "activations/layer23_attention_weight_max": 38.27128601074219,
      "activations/layer23_attention_weight_min": -23.885282516479492,
      "activations/layer2_attention_weight_max": 32.353118896484375,
      "activations/layer2_attention_weight_min": -31.32281494140625,
      "activations/layer3_attention_weight_max": 89.77591705322266,
      "activations/layer3_attention_weight_min": -90.37248992919922,
      "activations/layer4_attention_weight_max": 107.2593002319336,
      "activations/layer4_attention_weight_min": -108.37947845458984,
      "activations/layer5_attention_weight_max": 52.09912109375,
      "activations/layer5_attention_weight_min": -60.387359619140625,
      "activations/layer6_attention_weight_max": 44.304771423339844,
      "activations/layer6_attention_weight_min": -50.39854049682617,
      "activations/layer7_attention_weight_max": 87.88196563720703,
      "activations/layer7_attention_weight_min": -92.06853485107422,
      "activations/layer8_attention_weight_max": 39.862701416015625,
      "activations/layer8_attention_weight_min": -42.402828216552734,
      "activations/layer9_attention_weight_max": 34.825984954833984,
      "activations/layer9_attention_weight_min": -33.17390441894531,
      "epoch": 9.91,
      "learning_rate": 8.698787878787878e-05,
      "loss": 2.8082,
      "step": 170500
    },
    {
      "activations/layer0_attention_weight_max": 15.561963081359863,
      "activations/layer0_attention_weight_min": -13.697994232177734,
      "activations/layer10_attention_weight_max": 34.1278076171875,
      "activations/layer10_attention_weight_min": -33.767547607421875,
      "activations/layer11_attention_weight_max": 34.75342559814453,
      "activations/layer11_attention_weight_min": -31.582386016845703,
      "activations/layer12_attention_weight_max": 24.43199348449707,
      "activations/layer12_attention_weight_min": -25.298500061035156,
      "activations/layer13_attention_weight_max": 37.46720886230469,
      "activations/layer13_attention_weight_min": -34.407981872558594,
      "activations/layer14_attention_weight_max": 45.5706672668457,
      "activations/layer14_attention_weight_min": -41.64887237548828,
      "activations/layer15_attention_weight_max": 37.58564758300781,
      "activations/layer15_attention_weight_min": -33.72172164916992,
      "activations/layer16_attention_weight_max": 34.04097366333008,
      "activations/layer16_attention_weight_min": -26.179481506347656,
      "activations/layer17_attention_weight_max": 31.847116470336914,
      "activations/layer17_attention_weight_min": -26.86057472229004,
      "activations/layer18_attention_weight_max": 32.4887809753418,
      "activations/layer18_attention_weight_min": -24.26917266845703,
      "activations/layer19_attention_weight_max": 36.83303451538086,
      "activations/layer19_attention_weight_min": -29.033233642578125,
      "activations/layer1_attention_weight_max": 16.702003479003906,
      "activations/layer1_attention_weight_min": -14.295075416564941,
      "activations/layer20_attention_weight_max": 29.543346405029297,
      "activations/layer20_attention_weight_min": -25.766605377197266,
      "activations/layer21_attention_weight_max": 27.619842529296875,
      "activations/layer21_attention_weight_min": -22.965829849243164,
      "activations/layer22_attention_weight_max": 41.4422607421875,
      "activations/layer22_attention_weight_min": -28.951753616333008,
      "activations/layer23_attention_weight_max": 31.12202262878418,
      "activations/layer23_attention_weight_min": -23.14869499206543,
      "activations/layer2_attention_weight_max": 32.401123046875,
      "activations/layer2_attention_weight_min": -32.35612487792969,
      "activations/layer3_attention_weight_max": 95.5535888671875,
      "activations/layer3_attention_weight_min": -96.77772521972656,
      "activations/layer4_attention_weight_max": 114.8035888671875,
      "activations/layer4_attention_weight_min": -110.89794921875,
      "activations/layer5_attention_weight_max": 54.14806365966797,
      "activations/layer5_attention_weight_min": -64.26066589355469,
      "activations/layer6_attention_weight_max": 44.75743865966797,
      "activations/layer6_attention_weight_min": -46.887428283691406,
      "activations/layer7_attention_weight_max": 90.72423553466797,
      "activations/layer7_attention_weight_min": -87.2862548828125,
      "activations/layer8_attention_weight_max": 40.11030960083008,
      "activations/layer8_attention_weight_min": -40.06557083129883,
      "activations/layer9_attention_weight_max": 32.9813117980957,
      "activations/layer9_attention_weight_min": -32.55598831176758,
      "epoch": 9.91,
      "learning_rate": 8.696893939393938e-05,
      "loss": 2.8003,
      "step": 170550
    },
    {
      "activations/layer0_attention_weight_max": 16.38665008544922,
      "activations/layer0_attention_weight_min": -12.224599838256836,
      "activations/layer10_attention_weight_max": 30.393234252929688,
      "activations/layer10_attention_weight_min": -31.47745132446289,
      "activations/layer11_attention_weight_max": 31.05916404724121,
      "activations/layer11_attention_weight_min": -29.97292709350586,
      "activations/layer12_attention_weight_max": 24.063308715820312,
      "activations/layer12_attention_weight_min": -28.684640884399414,
      "activations/layer13_attention_weight_max": 35.874656677246094,
      "activations/layer13_attention_weight_min": -32.98493957519531,
      "activations/layer14_attention_weight_max": 43.16178512573242,
      "activations/layer14_attention_weight_min": -37.40153503417969,
      "activations/layer15_attention_weight_max": 36.05133056640625,
      "activations/layer15_attention_weight_min": -31.960844039916992,
      "activations/layer16_attention_weight_max": 33.668785095214844,
      "activations/layer16_attention_weight_min": -28.138925552368164,
      "activations/layer17_attention_weight_max": 32.353694915771484,
      "activations/layer17_attention_weight_min": -25.216766357421875,
      "activations/layer18_attention_weight_max": 32.093753814697266,
      "activations/layer18_attention_weight_min": -22.76651954650879,
      "activations/layer19_attention_weight_max": 36.27665710449219,
      "activations/layer19_attention_weight_min": -31.074588775634766,
      "activations/layer1_attention_weight_max": 17.41107940673828,
      "activations/layer1_attention_weight_min": -14.722397804260254,
      "activations/layer20_attention_weight_max": 29.86347007751465,
      "activations/layer20_attention_weight_min": -25.047550201416016,
      "activations/layer21_attention_weight_max": 33.308956146240234,
      "activations/layer21_attention_weight_min": -23.634984970092773,
      "activations/layer22_attention_weight_max": 47.66655731201172,
      "activations/layer22_attention_weight_min": -28.627336502075195,
      "activations/layer23_attention_weight_max": 36.4092903137207,
      "activations/layer23_attention_weight_min": -23.44753646850586,
      "activations/layer2_attention_weight_max": 31.923263549804688,
      "activations/layer2_attention_weight_min": -32.363006591796875,
      "activations/layer3_attention_weight_max": 95.55203247070312,
      "activations/layer3_attention_weight_min": -95.15445709228516,
      "activations/layer4_attention_weight_max": 113.97505187988281,
      "activations/layer4_attention_weight_min": -116.31798553466797,
      "activations/layer5_attention_weight_max": 55.63998031616211,
      "activations/layer5_attention_weight_min": -61.273223876953125,
      "activations/layer6_attention_weight_max": 47.48024368286133,
      "activations/layer6_attention_weight_min": -48.823421478271484,
      "activations/layer7_attention_weight_max": 88.69463348388672,
      "activations/layer7_attention_weight_min": -85.17105865478516,
      "activations/layer8_attention_weight_max": 38.69158172607422,
      "activations/layer8_attention_weight_min": -41.6302375793457,
      "activations/layer9_attention_weight_max": 33.10221481323242,
      "activations/layer9_attention_weight_min": -32.09918975830078,
      "epoch": 9.91,
      "learning_rate": 8.694999999999999e-05,
      "loss": 2.7828,
      "step": 170600
    },
    {
      "activations/layer0_attention_weight_max": 15.148380279541016,
      "activations/layer0_attention_weight_min": -12.523164749145508,
      "activations/layer10_attention_weight_max": 38.949684143066406,
      "activations/layer10_attention_weight_min": -37.4390869140625,
      "activations/layer11_attention_weight_max": 40.79985046386719,
      "activations/layer11_attention_weight_min": -40.10665512084961,
      "activations/layer12_attention_weight_max": 44.11518859863281,
      "activations/layer12_attention_weight_min": -27.89092254638672,
      "activations/layer13_attention_weight_max": 68.13681030273438,
      "activations/layer13_attention_weight_min": -39.789451599121094,
      "activations/layer14_attention_weight_max": 56.43111801147461,
      "activations/layer14_attention_weight_min": -41.161346435546875,
      "activations/layer15_attention_weight_max": 64.80593872070312,
      "activations/layer15_attention_weight_min": -37.663272857666016,
      "activations/layer16_attention_weight_max": 41.50006866455078,
      "activations/layer16_attention_weight_min": -26.421781539916992,
      "activations/layer17_attention_weight_max": 32.962059020996094,
      "activations/layer17_attention_weight_min": -28.516218185424805,
      "activations/layer18_attention_weight_max": 29.356754302978516,
      "activations/layer18_attention_weight_min": -24.604873657226562,
      "activations/layer19_attention_weight_max": 31.304880142211914,
      "activations/layer19_attention_weight_min": -30.623126983642578,
      "activations/layer1_attention_weight_max": 17.561487197875977,
      "activations/layer1_attention_weight_min": -14.64683723449707,
      "activations/layer20_attention_weight_max": 27.724435806274414,
      "activations/layer20_attention_weight_min": -24.67540740966797,
      "activations/layer21_attention_weight_max": 28.372282028198242,
      "activations/layer21_attention_weight_min": -22.470285415649414,
      "activations/layer22_attention_weight_max": 42.329166412353516,
      "activations/layer22_attention_weight_min": -29.51342010498047,
      "activations/layer23_attention_weight_max": 33.86480712890625,
      "activations/layer23_attention_weight_min": -24.39902114868164,
      "activations/layer2_attention_weight_max": 32.676788330078125,
      "activations/layer2_attention_weight_min": -32.794254302978516,
      "activations/layer3_attention_weight_max": 92.40421295166016,
      "activations/layer3_attention_weight_min": -98.32817840576172,
      "activations/layer4_attention_weight_max": 113.64336395263672,
      "activations/layer4_attention_weight_min": -113.07486724853516,
      "activations/layer5_attention_weight_max": 57.76821517944336,
      "activations/layer5_attention_weight_min": -59.16726303100586,
      "activations/layer6_attention_weight_max": 47.07697677612305,
      "activations/layer6_attention_weight_min": -47.458255767822266,
      "activations/layer7_attention_weight_max": 101.02071380615234,
      "activations/layer7_attention_weight_min": -92.94632720947266,
      "activations/layer8_attention_weight_max": 47.119041442871094,
      "activations/layer8_attention_weight_min": -43.73238754272461,
      "activations/layer9_attention_weight_max": 40.309730529785156,
      "activations/layer9_attention_weight_min": -38.46870040893555,
      "epoch": 9.92,
      "learning_rate": 8.69310606060606e-05,
      "loss": 2.7867,
      "step": 170650
    },
    {
      "activations/layer0_attention_weight_max": 16.454723358154297,
      "activations/layer0_attention_weight_min": -12.269129753112793,
      "activations/layer10_attention_weight_max": 38.64839553833008,
      "activations/layer10_attention_weight_min": -33.2642822265625,
      "activations/layer11_attention_weight_max": 41.2586784362793,
      "activations/layer11_attention_weight_min": -33.08903121948242,
      "activations/layer12_attention_weight_max": 24.3748836517334,
      "activations/layer12_attention_weight_min": -26.11200714111328,
      "activations/layer13_attention_weight_max": 39.13297653198242,
      "activations/layer13_attention_weight_min": -34.24510192871094,
      "activations/layer14_attention_weight_max": 47.12014389038086,
      "activations/layer14_attention_weight_min": -35.99810028076172,
      "activations/layer15_attention_weight_max": 38.41844940185547,
      "activations/layer15_attention_weight_min": -32.49052810668945,
      "activations/layer16_attention_weight_max": 37.273292541503906,
      "activations/layer16_attention_weight_min": -29.22213363647461,
      "activations/layer17_attention_weight_max": 37.555747985839844,
      "activations/layer17_attention_weight_min": -25.18311882019043,
      "activations/layer18_attention_weight_max": 30.412593841552734,
      "activations/layer18_attention_weight_min": -21.864858627319336,
      "activations/layer19_attention_weight_max": 43.411373138427734,
      "activations/layer19_attention_weight_min": -32.96475601196289,
      "activations/layer1_attention_weight_max": 16.296358108520508,
      "activations/layer1_attention_weight_min": -14.322100639343262,
      "activations/layer20_attention_weight_max": 36.18382263183594,
      "activations/layer20_attention_weight_min": -26.680753707885742,
      "activations/layer21_attention_weight_max": 33.52009201049805,
      "activations/layer21_attention_weight_min": -24.5578556060791,
      "activations/layer22_attention_weight_max": 49.27134323120117,
      "activations/layer22_attention_weight_min": -30.654998779296875,
      "activations/layer23_attention_weight_max": 42.71771240234375,
      "activations/layer23_attention_weight_min": -22.20252227783203,
      "activations/layer2_attention_weight_max": 34.185821533203125,
      "activations/layer2_attention_weight_min": -32.52638626098633,
      "activations/layer3_attention_weight_max": 95.52587890625,
      "activations/layer3_attention_weight_min": -94.40821075439453,
      "activations/layer4_attention_weight_max": 115.71185302734375,
      "activations/layer4_attention_weight_min": -112.62877655029297,
      "activations/layer5_attention_weight_max": 51.7380485534668,
      "activations/layer5_attention_weight_min": -62.75294494628906,
      "activations/layer6_attention_weight_max": 44.3411750793457,
      "activations/layer6_attention_weight_min": -47.08479309082031,
      "activations/layer7_attention_weight_max": 92.49839782714844,
      "activations/layer7_attention_weight_min": -89.69490051269531,
      "activations/layer8_attention_weight_max": 42.23468780517578,
      "activations/layer8_attention_weight_min": -39.9264030456543,
      "activations/layer9_attention_weight_max": 35.79283905029297,
      "activations/layer9_attention_weight_min": -32.416412353515625,
      "epoch": 9.92,
      "learning_rate": 8.69121212121212e-05,
      "loss": 2.7921,
      "step": 170700
    },
    {
      "activations/layer0_attention_weight_max": 15.401586532592773,
      "activations/layer0_attention_weight_min": -12.589208602905273,
      "activations/layer10_attention_weight_max": 31.527812957763672,
      "activations/layer10_attention_weight_min": -30.777402877807617,
      "activations/layer11_attention_weight_max": 30.56319236755371,
      "activations/layer11_attention_weight_min": -29.96206283569336,
      "activations/layer12_attention_weight_max": 28.571582794189453,
      "activations/layer12_attention_weight_min": -24.256641387939453,
      "activations/layer13_attention_weight_max": 41.53672409057617,
      "activations/layer13_attention_weight_min": -33.043060302734375,
      "activations/layer14_attention_weight_max": 52.84303665161133,
      "activations/layer14_attention_weight_min": -40.1075553894043,
      "activations/layer15_attention_weight_max": 35.5666618347168,
      "activations/layer15_attention_weight_min": -30.603818893432617,
      "activations/layer16_attention_weight_max": 29.904033660888672,
      "activations/layer16_attention_weight_min": -27.834701538085938,
      "activations/layer17_attention_weight_max": 33.299556732177734,
      "activations/layer17_attention_weight_min": -25.157751083374023,
      "activations/layer18_attention_weight_max": 33.173126220703125,
      "activations/layer18_attention_weight_min": -22.355100631713867,
      "activations/layer19_attention_weight_max": 39.24766540527344,
      "activations/layer19_attention_weight_min": -30.868499755859375,
      "activations/layer1_attention_weight_max": 16.7509822845459,
      "activations/layer1_attention_weight_min": -16.002351760864258,
      "activations/layer20_attention_weight_max": 32.71297836303711,
      "activations/layer20_attention_weight_min": -24.86528778076172,
      "activations/layer21_attention_weight_max": 30.105682373046875,
      "activations/layer21_attention_weight_min": -23.684734344482422,
      "activations/layer22_attention_weight_max": 45.376243591308594,
      "activations/layer22_attention_weight_min": -30.73833465576172,
      "activations/layer23_attention_weight_max": 39.40286636352539,
      "activations/layer23_attention_weight_min": -23.215801239013672,
      "activations/layer2_attention_weight_max": 31.918231964111328,
      "activations/layer2_attention_weight_min": -32.25312805175781,
      "activations/layer3_attention_weight_max": 93.59947967529297,
      "activations/layer3_attention_weight_min": -95.73514556884766,
      "activations/layer4_attention_weight_max": 114.10009002685547,
      "activations/layer4_attention_weight_min": -110.3099365234375,
      "activations/layer5_attention_weight_max": 49.877098083496094,
      "activations/layer5_attention_weight_min": -61.414894104003906,
      "activations/layer6_attention_weight_max": 45.90019989013672,
      "activations/layer6_attention_weight_min": -46.848976135253906,
      "activations/layer7_attention_weight_max": 94.66387176513672,
      "activations/layer7_attention_weight_min": -91.5413818359375,
      "activations/layer8_attention_weight_max": 38.7143440246582,
      "activations/layer8_attention_weight_min": -40.45091247558594,
      "activations/layer9_attention_weight_max": 33.07285690307617,
      "activations/layer9_attention_weight_min": -33.17393493652344,
      "epoch": 9.92,
      "learning_rate": 8.689318181818182e-05,
      "loss": 2.7948,
      "step": 170750
    },
    {
      "activations/layer0_attention_weight_max": 15.867772102355957,
      "activations/layer0_attention_weight_min": -12.500712394714355,
      "activations/layer10_attention_weight_max": 30.88222885131836,
      "activations/layer10_attention_weight_min": -31.62839126586914,
      "activations/layer11_attention_weight_max": 32.762046813964844,
      "activations/layer11_attention_weight_min": -32.96845245361328,
      "activations/layer12_attention_weight_max": 31.907180786132812,
      "activations/layer12_attention_weight_min": -24.86566925048828,
      "activations/layer13_attention_weight_max": 38.53725051879883,
      "activations/layer13_attention_weight_min": -34.02541732788086,
      "activations/layer14_attention_weight_max": 46.411598205566406,
      "activations/layer14_attention_weight_min": -39.6205940246582,
      "activations/layer15_attention_weight_max": 37.93119430541992,
      "activations/layer15_attention_weight_min": -31.908767700195312,
      "activations/layer16_attention_weight_max": 34.94631576538086,
      "activations/layer16_attention_weight_min": -29.717660903930664,
      "activations/layer17_attention_weight_max": 34.86545181274414,
      "activations/layer17_attention_weight_min": -25.67270851135254,
      "activations/layer18_attention_weight_max": 33.111358642578125,
      "activations/layer18_attention_weight_min": -23.776966094970703,
      "activations/layer19_attention_weight_max": 36.43938446044922,
      "activations/layer19_attention_weight_min": -33.37565994262695,
      "activations/layer1_attention_weight_max": 17.19449806213379,
      "activations/layer1_attention_weight_min": -17.579532623291016,
      "activations/layer20_attention_weight_max": 31.315671920776367,
      "activations/layer20_attention_weight_min": -23.861677169799805,
      "activations/layer21_attention_weight_max": 33.30448532104492,
      "activations/layer21_attention_weight_min": -24.14287757873535,
      "activations/layer22_attention_weight_max": 48.75811767578125,
      "activations/layer22_attention_weight_min": -28.66733741760254,
      "activations/layer23_attention_weight_max": 35.551414489746094,
      "activations/layer23_attention_weight_min": -23.77096176147461,
      "activations/layer2_attention_weight_max": 32.16165542602539,
      "activations/layer2_attention_weight_min": -31.738075256347656,
      "activations/layer3_attention_weight_max": 89.48298645019531,
      "activations/layer3_attention_weight_min": -95.58709716796875,
      "activations/layer4_attention_weight_max": 111.12674713134766,
      "activations/layer4_attention_weight_min": -114.3590316772461,
      "activations/layer5_attention_weight_max": 53.9135856628418,
      "activations/layer5_attention_weight_min": -65.49161529541016,
      "activations/layer6_attention_weight_max": 45.569114685058594,
      "activations/layer6_attention_weight_min": -48.69424819946289,
      "activations/layer7_attention_weight_max": 87.11236572265625,
      "activations/layer7_attention_weight_min": -89.71730041503906,
      "activations/layer8_attention_weight_max": 38.26050567626953,
      "activations/layer8_attention_weight_min": -41.19379425048828,
      "activations/layer9_attention_weight_max": 33.295860290527344,
      "activations/layer9_attention_weight_min": -32.959434509277344,
      "epoch": 9.92,
      "learning_rate": 8.687424242424242e-05,
      "loss": 2.7934,
      "step": 170800
    },
    {
      "activations/layer0_attention_weight_max": 15.342752456665039,
      "activations/layer0_attention_weight_min": -11.534329414367676,
      "activations/layer10_attention_weight_max": 33.31092834472656,
      "activations/layer10_attention_weight_min": -33.22139358520508,
      "activations/layer11_attention_weight_max": 34.83487319946289,
      "activations/layer11_attention_weight_min": -33.96357727050781,
      "activations/layer12_attention_weight_max": 23.892297744750977,
      "activations/layer12_attention_weight_min": -26.05510902404785,
      "activations/layer13_attention_weight_max": 38.61610794067383,
      "activations/layer13_attention_weight_min": -34.25897979736328,
      "activations/layer14_attention_weight_max": 44.25558853149414,
      "activations/layer14_attention_weight_min": -39.710784912109375,
      "activations/layer15_attention_weight_max": 38.59466552734375,
      "activations/layer15_attention_weight_min": -33.32594299316406,
      "activations/layer16_attention_weight_max": 31.08984375,
      "activations/layer16_attention_weight_min": -29.150632858276367,
      "activations/layer17_attention_weight_max": 33.19624710083008,
      "activations/layer17_attention_weight_min": -27.963502883911133,
      "activations/layer18_attention_weight_max": 31.910709381103516,
      "activations/layer18_attention_weight_min": -22.820924758911133,
      "activations/layer19_attention_weight_max": 39.231258392333984,
      "activations/layer19_attention_weight_min": -32.51603317260742,
      "activations/layer1_attention_weight_max": 16.826723098754883,
      "activations/layer1_attention_weight_min": -15.874418258666992,
      "activations/layer20_attention_weight_max": 30.06944465637207,
      "activations/layer20_attention_weight_min": -26.09673500061035,
      "activations/layer21_attention_weight_max": 28.31084442138672,
      "activations/layer21_attention_weight_min": -25.18825340270996,
      "activations/layer22_attention_weight_max": 45.41241455078125,
      "activations/layer22_attention_weight_min": -34.9620475769043,
      "activations/layer23_attention_weight_max": 38.45307159423828,
      "activations/layer23_attention_weight_min": -28.935508728027344,
      "activations/layer2_attention_weight_max": 31.798770904541016,
      "activations/layer2_attention_weight_min": -31.000646591186523,
      "activations/layer3_attention_weight_max": 91.95703125,
      "activations/layer3_attention_weight_min": -93.54560089111328,
      "activations/layer4_attention_weight_max": 112.65438079833984,
      "activations/layer4_attention_weight_min": -115.41744232177734,
      "activations/layer5_attention_weight_max": 54.22398376464844,
      "activations/layer5_attention_weight_min": -60.30531311035156,
      "activations/layer6_attention_weight_max": 45.97168731689453,
      "activations/layer6_attention_weight_min": -48.51366424560547,
      "activations/layer7_attention_weight_max": 87.64993286132812,
      "activations/layer7_attention_weight_min": -93.3008804321289,
      "activations/layer8_attention_weight_max": 40.2630729675293,
      "activations/layer8_attention_weight_min": -40.2509765625,
      "activations/layer9_attention_weight_max": 34.5404167175293,
      "activations/layer9_attention_weight_min": -33.0377082824707,
      "epoch": 9.93,
      "learning_rate": 8.685530303030302e-05,
      "loss": 2.7868,
      "step": 170850
    },
    {
      "activations/layer0_attention_weight_max": 15.591859817504883,
      "activations/layer0_attention_weight_min": -12.600217819213867,
      "activations/layer10_attention_weight_max": 31.988561630249023,
      "activations/layer10_attention_weight_min": -34.23809051513672,
      "activations/layer11_attention_weight_max": 32.962440490722656,
      "activations/layer11_attention_weight_min": -31.141815185546875,
      "activations/layer12_attention_weight_max": 24.31805992126465,
      "activations/layer12_attention_weight_min": -24.819416046142578,
      "activations/layer13_attention_weight_max": 42.20315170288086,
      "activations/layer13_attention_weight_min": -37.952842712402344,
      "activations/layer14_attention_weight_max": 46.69898986816406,
      "activations/layer14_attention_weight_min": -39.874786376953125,
      "activations/layer15_attention_weight_max": 36.546836853027344,
      "activations/layer15_attention_weight_min": -31.825212478637695,
      "activations/layer16_attention_weight_max": 32.37051773071289,
      "activations/layer16_attention_weight_min": -29.64640998840332,
      "activations/layer17_attention_weight_max": 34.17798614501953,
      "activations/layer17_attention_weight_min": -27.16025733947754,
      "activations/layer18_attention_weight_max": 30.99162483215332,
      "activations/layer18_attention_weight_min": -23.1839599609375,
      "activations/layer19_attention_weight_max": 34.72768783569336,
      "activations/layer19_attention_weight_min": -33.02787399291992,
      "activations/layer1_attention_weight_max": 18.917715072631836,
      "activations/layer1_attention_weight_min": -15.143628120422363,
      "activations/layer20_attention_weight_max": 27.604379653930664,
      "activations/layer20_attention_weight_min": -25.510879516601562,
      "activations/layer21_attention_weight_max": 28.65513038635254,
      "activations/layer21_attention_weight_min": -22.105361938476562,
      "activations/layer22_attention_weight_max": 46.174072265625,
      "activations/layer22_attention_weight_min": -31.251502990722656,
      "activations/layer23_attention_weight_max": 34.13349914550781,
      "activations/layer23_attention_weight_min": -23.55531120300293,
      "activations/layer2_attention_weight_max": 31.159231185913086,
      "activations/layer2_attention_weight_min": -32.575538635253906,
      "activations/layer3_attention_weight_max": 88.94562530517578,
      "activations/layer3_attention_weight_min": -92.89598083496094,
      "activations/layer4_attention_weight_max": 108.52335357666016,
      "activations/layer4_attention_weight_min": -108.83049774169922,
      "activations/layer5_attention_weight_max": 50.88642883300781,
      "activations/layer5_attention_weight_min": -63.441009521484375,
      "activations/layer6_attention_weight_max": 42.61799240112305,
      "activations/layer6_attention_weight_min": -48.18455505371094,
      "activations/layer7_attention_weight_max": 87.68301391601562,
      "activations/layer7_attention_weight_min": -86.59818267822266,
      "activations/layer8_attention_weight_max": 40.42295455932617,
      "activations/layer8_attention_weight_min": -39.73768997192383,
      "activations/layer9_attention_weight_max": 32.093563079833984,
      "activations/layer9_attention_weight_min": -32.822349548339844,
      "epoch": 9.93,
      "learning_rate": 8.683636363636364e-05,
      "loss": 2.7874,
      "step": 170900
    },
    {
      "activations/layer0_attention_weight_max": 15.11155891418457,
      "activations/layer0_attention_weight_min": -12.338138580322266,
      "activations/layer10_attention_weight_max": 31.303659439086914,
      "activations/layer10_attention_weight_min": -32.83891296386719,
      "activations/layer11_attention_weight_max": 32.45912170410156,
      "activations/layer11_attention_weight_min": -32.11610794067383,
      "activations/layer12_attention_weight_max": 25.037992477416992,
      "activations/layer12_attention_weight_min": -25.533607482910156,
      "activations/layer13_attention_weight_max": 39.2147102355957,
      "activations/layer13_attention_weight_min": -34.35951614379883,
      "activations/layer14_attention_weight_max": 46.55873107910156,
      "activations/layer14_attention_weight_min": -38.063194274902344,
      "activations/layer15_attention_weight_max": 36.03232192993164,
      "activations/layer15_attention_weight_min": -31.21951675415039,
      "activations/layer16_attention_weight_max": 30.547380447387695,
      "activations/layer16_attention_weight_min": -28.423173904418945,
      "activations/layer17_attention_weight_max": 30.908180236816406,
      "activations/layer17_attention_weight_min": -25.29410171508789,
      "activations/layer18_attention_weight_max": 29.83993148803711,
      "activations/layer18_attention_weight_min": -22.39076805114746,
      "activations/layer19_attention_weight_max": 32.35121536254883,
      "activations/layer19_attention_weight_min": -32.75082778930664,
      "activations/layer1_attention_weight_max": 16.463502883911133,
      "activations/layer1_attention_weight_min": -14.470478057861328,
      "activations/layer20_attention_weight_max": 28.15435028076172,
      "activations/layer20_attention_weight_min": -24.321025848388672,
      "activations/layer21_attention_weight_max": 30.078372955322266,
      "activations/layer21_attention_weight_min": -22.31096076965332,
      "activations/layer22_attention_weight_max": 41.95270538330078,
      "activations/layer22_attention_weight_min": -30.115299224853516,
      "activations/layer23_attention_weight_max": 32.253807067871094,
      "activations/layer23_attention_weight_min": -27.277807235717773,
      "activations/layer2_attention_weight_max": 31.980314254760742,
      "activations/layer2_attention_weight_min": -31.65167999267578,
      "activations/layer3_attention_weight_max": 89.85562133789062,
      "activations/layer3_attention_weight_min": -93.18882751464844,
      "activations/layer4_attention_weight_max": 110.6806411743164,
      "activations/layer4_attention_weight_min": -111.44171142578125,
      "activations/layer5_attention_weight_max": 52.14945983886719,
      "activations/layer5_attention_weight_min": -63.200374603271484,
      "activations/layer6_attention_weight_max": 46.09562301635742,
      "activations/layer6_attention_weight_min": -47.62723922729492,
      "activations/layer7_attention_weight_max": 87.85466003417969,
      "activations/layer7_attention_weight_min": -88.84031677246094,
      "activations/layer8_attention_weight_max": 38.72682571411133,
      "activations/layer8_attention_weight_min": -39.098243713378906,
      "activations/layer9_attention_weight_max": 31.195581436157227,
      "activations/layer9_attention_weight_min": -33.414039611816406,
      "epoch": 9.93,
      "learning_rate": 8.681742424242422e-05,
      "loss": 2.7828,
      "step": 170950
    },
    {
      "activations/layer0_attention_weight_max": 15.812084197998047,
      "activations/layer0_attention_weight_min": -12.247291564941406,
      "activations/layer10_attention_weight_max": 33.102142333984375,
      "activations/layer10_attention_weight_min": -33.297943115234375,
      "activations/layer11_attention_weight_max": 31.916217803955078,
      "activations/layer11_attention_weight_min": -32.21500778198242,
      "activations/layer12_attention_weight_max": 24.551937103271484,
      "activations/layer12_attention_weight_min": -24.59880828857422,
      "activations/layer13_attention_weight_max": 35.596473693847656,
      "activations/layer13_attention_weight_min": -32.855281829833984,
      "activations/layer14_attention_weight_max": 44.9207649230957,
      "activations/layer14_attention_weight_min": -40.763221740722656,
      "activations/layer15_attention_weight_max": 33.47737503051758,
      "activations/layer15_attention_weight_min": -31.856800079345703,
      "activations/layer16_attention_weight_max": 28.51638412475586,
      "activations/layer16_attention_weight_min": -29.132339477539062,
      "activations/layer17_attention_weight_max": 28.639856338500977,
      "activations/layer17_attention_weight_min": -25.593107223510742,
      "activations/layer18_attention_weight_max": 28.50210189819336,
      "activations/layer18_attention_weight_min": -23.473440170288086,
      "activations/layer19_attention_weight_max": 31.871030807495117,
      "activations/layer19_attention_weight_min": -31.113943099975586,
      "activations/layer1_attention_weight_max": 17.131298065185547,
      "activations/layer1_attention_weight_min": -16.199386596679688,
      "activations/layer20_attention_weight_max": 25.473575592041016,
      "activations/layer20_attention_weight_min": -22.816951751708984,
      "activations/layer21_attention_weight_max": 26.140342712402344,
      "activations/layer21_attention_weight_min": -23.36025047302246,
      "activations/layer22_attention_weight_max": 40.97262954711914,
      "activations/layer22_attention_weight_min": -28.269062042236328,
      "activations/layer23_attention_weight_max": 31.831153869628906,
      "activations/layer23_attention_weight_min": -24.26813507080078,
      "activations/layer2_attention_weight_max": 32.07502746582031,
      "activations/layer2_attention_weight_min": -33.53712463378906,
      "activations/layer3_attention_weight_max": 91.89279174804688,
      "activations/layer3_attention_weight_min": -96.91043853759766,
      "activations/layer4_attention_weight_max": 109.89641571044922,
      "activations/layer4_attention_weight_min": -113.44245910644531,
      "activations/layer5_attention_weight_max": 54.85313034057617,
      "activations/layer5_attention_weight_min": -61.206661224365234,
      "activations/layer6_attention_weight_max": 47.495243072509766,
      "activations/layer6_attention_weight_min": -49.669246673583984,
      "activations/layer7_attention_weight_max": 94.12737274169922,
      "activations/layer7_attention_weight_min": -89.67488861083984,
      "activations/layer8_attention_weight_max": 41.9847526550293,
      "activations/layer8_attention_weight_min": -41.65830993652344,
      "activations/layer9_attention_weight_max": 34.65147018432617,
      "activations/layer9_attention_weight_min": -35.42285919189453,
      "epoch": 9.94,
      "learning_rate": 8.679848484848484e-05,
      "loss": 2.7802,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.1836,
      "eval_samples_per_second": 524.709,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.1836,
      "eval_openwebtext_samples_per_second": 524.709,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.607,
      "eval_wikitext_samples_per_second": 283.756,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_lambada_loss": 2.572265625,
      "eval_lambada_ppl": 13.095460258581369,
      "eval_lambada_runtime": 9.1763,
      "eval_lambada_samples_per_second": 530.607,
      "step": 171000
    },
    {
      "activations/layer0_attention_weight_max": 15.76918888092041,
      "activations/layer0_attention_weight_min": -12.610154151916504,
      "activations/layer10_attention_weight_max": 34.43865203857422,
      "activations/layer10_attention_weight_min": -33.31949996948242,
      "activations/layer11_attention_weight_max": 34.55073547363281,
      "activations/layer11_attention_weight_min": -31.433094024658203,
      "activations/layer12_attention_weight_max": 24.4400577545166,
      "activations/layer12_attention_weight_min": -24.151809692382812,
      "activations/layer13_attention_weight_max": 38.48157501220703,
      "activations/layer13_attention_weight_min": -32.49258804321289,
      "activations/layer14_attention_weight_max": 57.71198272705078,
      "activations/layer14_attention_weight_min": -37.44843673706055,
      "activations/layer15_attention_weight_max": 39.336830139160156,
      "activations/layer15_attention_weight_min": -31.775604248046875,
      "activations/layer16_attention_weight_max": 36.075496673583984,
      "activations/layer16_attention_weight_min": -27.574663162231445,
      "activations/layer17_attention_weight_max": 34.819210052490234,
      "activations/layer17_attention_weight_min": -25.720073699951172,
      "activations/layer18_attention_weight_max": 32.20770263671875,
      "activations/layer18_attention_weight_min": -22.45980453491211,
      "activations/layer19_attention_weight_max": 40.660640716552734,
      "activations/layer19_attention_weight_min": -29.6630916595459,
      "activations/layer1_attention_weight_max": 17.326696395874023,
      "activations/layer1_attention_weight_min": -14.74865436553955,
      "activations/layer20_attention_weight_max": 32.70412826538086,
      "activations/layer20_attention_weight_min": -25.275972366333008,
      "activations/layer21_attention_weight_max": 29.98479461669922,
      "activations/layer21_attention_weight_min": -23.484447479248047,
      "activations/layer22_attention_weight_max": 48.78230667114258,
      "activations/layer22_attention_weight_min": -28.642480850219727,
      "activations/layer23_attention_weight_max": 41.33509063720703,
      "activations/layer23_attention_weight_min": -25.519750595092773,
      "activations/layer2_attention_weight_max": 30.895198822021484,
      "activations/layer2_attention_weight_min": -31.806354522705078,
      "activations/layer3_attention_weight_max": 94.46522521972656,
      "activations/layer3_attention_weight_min": -93.53068542480469,
      "activations/layer4_attention_weight_max": 114.63482666015625,
      "activations/layer4_attention_weight_min": -109.52603912353516,
      "activations/layer5_attention_weight_max": 57.02570343017578,
      "activations/layer5_attention_weight_min": -61.57250213623047,
      "activations/layer6_attention_weight_max": 48.69597244262695,
      "activations/layer6_attention_weight_min": -47.98484802246094,
      "activations/layer7_attention_weight_max": 95.33985900878906,
      "activations/layer7_attention_weight_min": -90.59640502929688,
      "activations/layer8_attention_weight_max": 41.752140045166016,
      "activations/layer8_attention_weight_min": -40.563514709472656,
      "activations/layer9_attention_weight_max": 34.8958625793457,
      "activations/layer9_attention_weight_min": -34.33488845825195,
      "epoch": 9.94,
      "learning_rate": 8.677954545454545e-05,
      "loss": 2.7726,
      "step": 171050
    },
    {
      "activations/layer0_attention_weight_max": 15.292485237121582,
      "activations/layer0_attention_weight_min": -11.797977447509766,
      "activations/layer10_attention_weight_max": 33.62423324584961,
      "activations/layer10_attention_weight_min": -31.21318817138672,
      "activations/layer11_attention_weight_max": 32.1492919921875,
      "activations/layer11_attention_weight_min": -30.64731216430664,
      "activations/layer12_attention_weight_max": 33.42202377319336,
      "activations/layer12_attention_weight_min": -29.20779037475586,
      "activations/layer13_attention_weight_max": 45.741573333740234,
      "activations/layer13_attention_weight_min": -33.90407943725586,
      "activations/layer14_attention_weight_max": 45.05782699584961,
      "activations/layer14_attention_weight_min": -44.26173782348633,
      "activations/layer15_attention_weight_max": 36.90877151489258,
      "activations/layer15_attention_weight_min": -32.25204086303711,
      "activations/layer16_attention_weight_max": 33.917076110839844,
      "activations/layer16_attention_weight_min": -27.738750457763672,
      "activations/layer17_attention_weight_max": 31.223567962646484,
      "activations/layer17_attention_weight_min": -25.032285690307617,
      "activations/layer18_attention_weight_max": 28.4445858001709,
      "activations/layer18_attention_weight_min": -21.727737426757812,
      "activations/layer19_attention_weight_max": 35.242801666259766,
      "activations/layer19_attention_weight_min": -28.57697105407715,
      "activations/layer1_attention_weight_max": 16.652551651000977,
      "activations/layer1_attention_weight_min": -14.917903900146484,
      "activations/layer20_attention_weight_max": 30.427350997924805,
      "activations/layer20_attention_weight_min": -22.413225173950195,
      "activations/layer21_attention_weight_max": 25.00351905822754,
      "activations/layer21_attention_weight_min": -21.480823516845703,
      "activations/layer22_attention_weight_max": 40.80950927734375,
      "activations/layer22_attention_weight_min": -27.376585006713867,
      "activations/layer23_attention_weight_max": 29.558652877807617,
      "activations/layer23_attention_weight_min": -26.65426254272461,
      "activations/layer2_attention_weight_max": 35.31507110595703,
      "activations/layer2_attention_weight_min": -33.37312316894531,
      "activations/layer3_attention_weight_max": 93.311767578125,
      "activations/layer3_attention_weight_min": -96.68560791015625,
      "activations/layer4_attention_weight_max": 109.95295715332031,
      "activations/layer4_attention_weight_min": -113.3190689086914,
      "activations/layer5_attention_weight_max": 52.04932403564453,
      "activations/layer5_attention_weight_min": -60.40039825439453,
      "activations/layer6_attention_weight_max": 45.93027114868164,
      "activations/layer6_attention_weight_min": -49.64110565185547,
      "activations/layer7_attention_weight_max": 87.5920639038086,
      "activations/layer7_attention_weight_min": -86.31782531738281,
      "activations/layer8_attention_weight_max": 38.855079650878906,
      "activations/layer8_attention_weight_min": -40.34518814086914,
      "activations/layer9_attention_weight_max": 36.350067138671875,
      "activations/layer9_attention_weight_min": -32.53817367553711,
      "epoch": 9.94,
      "learning_rate": 8.676060606060604e-05,
      "loss": 2.8001,
      "step": 171100
    },
    {
      "activations/layer0_attention_weight_max": 15.791893005371094,
      "activations/layer0_attention_weight_min": -12.451360702514648,
      "activations/layer10_attention_weight_max": 31.119186401367188,
      "activations/layer10_attention_weight_min": -31.865262985229492,
      "activations/layer11_attention_weight_max": 30.995567321777344,
      "activations/layer11_attention_weight_min": -31.35820770263672,
      "activations/layer12_attention_weight_max": 25.2135066986084,
      "activations/layer12_attention_weight_min": -25.11562728881836,
      "activations/layer13_attention_weight_max": 39.30508041381836,
      "activations/layer13_attention_weight_min": -37.1871223449707,
      "activations/layer14_attention_weight_max": 47.15961456298828,
      "activations/layer14_attention_weight_min": -41.8411750793457,
      "activations/layer15_attention_weight_max": 35.501190185546875,
      "activations/layer15_attention_weight_min": -31.938709259033203,
      "activations/layer16_attention_weight_max": 31.021923065185547,
      "activations/layer16_attention_weight_min": -29.091766357421875,
      "activations/layer17_attention_weight_max": 34.478187561035156,
      "activations/layer17_attention_weight_min": -25.424976348876953,
      "activations/layer18_attention_weight_max": 31.6693058013916,
      "activations/layer18_attention_weight_min": -21.956701278686523,
      "activations/layer19_attention_weight_max": 35.31707763671875,
      "activations/layer19_attention_weight_min": -31.001785278320312,
      "activations/layer1_attention_weight_max": 17.514728546142578,
      "activations/layer1_attention_weight_min": -14.985815048217773,
      "activations/layer20_attention_weight_max": 29.82613754272461,
      "activations/layer20_attention_weight_min": -26.971691131591797,
      "activations/layer21_attention_weight_max": 30.47667121887207,
      "activations/layer21_attention_weight_min": -25.279373168945312,
      "activations/layer22_attention_weight_max": 48.55368423461914,
      "activations/layer22_attention_weight_min": -31.592445373535156,
      "activations/layer23_attention_weight_max": 35.510066986083984,
      "activations/layer23_attention_weight_min": -24.098066329956055,
      "activations/layer2_attention_weight_max": 33.72997283935547,
      "activations/layer2_attention_weight_min": -31.88741683959961,
      "activations/layer3_attention_weight_max": 94.85433197021484,
      "activations/layer3_attention_weight_min": -98.4549560546875,
      "activations/layer4_attention_weight_max": 114.27107238769531,
      "activations/layer4_attention_weight_min": -114.07806396484375,
      "activations/layer5_attention_weight_max": 57.71672058105469,
      "activations/layer5_attention_weight_min": -63.088966369628906,
      "activations/layer6_attention_weight_max": 50.07814407348633,
      "activations/layer6_attention_weight_min": -49.696128845214844,
      "activations/layer7_attention_weight_max": 96.3541259765625,
      "activations/layer7_attention_weight_min": -86.97640228271484,
      "activations/layer8_attention_weight_max": 40.33403778076172,
      "activations/layer8_attention_weight_min": -40.950645446777344,
      "activations/layer9_attention_weight_max": 32.016334533691406,
      "activations/layer9_attention_weight_min": -33.16161346435547,
      "epoch": 9.94,
      "learning_rate": 8.674166666666666e-05,
      "loss": 2.798,
      "step": 171150
    },
    {
      "activations/layer0_attention_weight_max": 15.9271879196167,
      "activations/layer0_attention_weight_min": -12.318284034729004,
      "activations/layer10_attention_weight_max": 30.760650634765625,
      "activations/layer10_attention_weight_min": -29.802303314208984,
      "activations/layer11_attention_weight_max": 32.09782791137695,
      "activations/layer11_attention_weight_min": -30.366535186767578,
      "activations/layer12_attention_weight_max": 25.76569938659668,
      "activations/layer12_attention_weight_min": -23.50786781311035,
      "activations/layer13_attention_weight_max": 37.54836654663086,
      "activations/layer13_attention_weight_min": -31.26482582092285,
      "activations/layer14_attention_weight_max": 45.66743850708008,
      "activations/layer14_attention_weight_min": -36.30680465698242,
      "activations/layer15_attention_weight_max": 37.97903823852539,
      "activations/layer15_attention_weight_min": -32.184478759765625,
      "activations/layer16_attention_weight_max": 31.184736251831055,
      "activations/layer16_attention_weight_min": -27.94441032409668,
      "activations/layer17_attention_weight_max": 32.414772033691406,
      "activations/layer17_attention_weight_min": -25.67746925354004,
      "activations/layer18_attention_weight_max": 29.955974578857422,
      "activations/layer18_attention_weight_min": -21.893789291381836,
      "activations/layer19_attention_weight_max": 33.79838180541992,
      "activations/layer19_attention_weight_min": -30.928424835205078,
      "activations/layer1_attention_weight_max": 17.800765991210938,
      "activations/layer1_attention_weight_min": -13.905425071716309,
      "activations/layer20_attention_weight_max": 29.10250473022461,
      "activations/layer20_attention_weight_min": -24.610509872436523,
      "activations/layer21_attention_weight_max": 29.391489028930664,
      "activations/layer21_attention_weight_min": -23.902891159057617,
      "activations/layer22_attention_weight_max": 48.04334259033203,
      "activations/layer22_attention_weight_min": -31.176021575927734,
      "activations/layer23_attention_weight_max": 40.09672546386719,
      "activations/layer23_attention_weight_min": -24.784345626831055,
      "activations/layer2_attention_weight_max": 32.1624870300293,
      "activations/layer2_attention_weight_min": -32.53453063964844,
      "activations/layer3_attention_weight_max": 92.55694580078125,
      "activations/layer3_attention_weight_min": -96.80371856689453,
      "activations/layer4_attention_weight_max": 109.95501708984375,
      "activations/layer4_attention_weight_min": -112.22248840332031,
      "activations/layer5_attention_weight_max": 53.01870346069336,
      "activations/layer5_attention_weight_min": -61.05341720581055,
      "activations/layer6_attention_weight_max": 45.65571594238281,
      "activations/layer6_attention_weight_min": -47.41209411621094,
      "activations/layer7_attention_weight_max": 91.05077362060547,
      "activations/layer7_attention_weight_min": -87.1348876953125,
      "activations/layer8_attention_weight_max": 41.16173553466797,
      "activations/layer8_attention_weight_min": -40.94643020629883,
      "activations/layer9_attention_weight_max": 34.53153610229492,
      "activations/layer9_attention_weight_min": -33.298519134521484,
      "epoch": 9.95,
      "learning_rate": 8.672272727272727e-05,
      "loss": 2.7847,
      "step": 171200
    },
    {
      "activations/layer0_attention_weight_max": 16.178647994995117,
      "activations/layer0_attention_weight_min": -12.378511428833008,
      "activations/layer10_attention_weight_max": 32.65169143676758,
      "activations/layer10_attention_weight_min": -31.8551082611084,
      "activations/layer11_attention_weight_max": 32.3873291015625,
      "activations/layer11_attention_weight_min": -31.951251983642578,
      "activations/layer12_attention_weight_max": 23.581226348876953,
      "activations/layer12_attention_weight_min": -25.434341430664062,
      "activations/layer13_attention_weight_max": 35.80354309082031,
      "activations/layer13_attention_weight_min": -35.37900924682617,
      "activations/layer14_attention_weight_max": 42.43315887451172,
      "activations/layer14_attention_weight_min": -36.69859313964844,
      "activations/layer15_attention_weight_max": 34.16891860961914,
      "activations/layer15_attention_weight_min": -30.301530838012695,
      "activations/layer16_attention_weight_max": 30.83683204650879,
      "activations/layer16_attention_weight_min": -27.499448776245117,
      "activations/layer17_attention_weight_max": 31.698490142822266,
      "activations/layer17_attention_weight_min": -26.98179817199707,
      "activations/layer18_attention_weight_max": 32.593284606933594,
      "activations/layer18_attention_weight_min": -22.988048553466797,
      "activations/layer19_attention_weight_max": 32.948795318603516,
      "activations/layer19_attention_weight_min": -29.67006492614746,
      "activations/layer1_attention_weight_max": 15.805045127868652,
      "activations/layer1_attention_weight_min": -14.824745178222656,
      "activations/layer20_attention_weight_max": 31.56368064880371,
      "activations/layer20_attention_weight_min": -23.40656089782715,
      "activations/layer21_attention_weight_max": 30.361894607543945,
      "activations/layer21_attention_weight_min": -22.371896743774414,
      "activations/layer22_attention_weight_max": 43.683265686035156,
      "activations/layer22_attention_weight_min": -30.496139526367188,
      "activations/layer23_attention_weight_max": 35.04440689086914,
      "activations/layer23_attention_weight_min": -25.509414672851562,
      "activations/layer2_attention_weight_max": 33.67304992675781,
      "activations/layer2_attention_weight_min": -32.54849624633789,
      "activations/layer3_attention_weight_max": 92.3388442993164,
      "activations/layer3_attention_weight_min": -95.53166961669922,
      "activations/layer4_attention_weight_max": 112.41747283935547,
      "activations/layer4_attention_weight_min": -113.4908447265625,
      "activations/layer5_attention_weight_max": 53.3590087890625,
      "activations/layer5_attention_weight_min": -62.01337814331055,
      "activations/layer6_attention_weight_max": 45.48700714111328,
      "activations/layer6_attention_weight_min": -48.1125373840332,
      "activations/layer7_attention_weight_max": 85.40631866455078,
      "activations/layer7_attention_weight_min": -91.7316665649414,
      "activations/layer8_attention_weight_max": 39.20561599731445,
      "activations/layer8_attention_weight_min": -40.31184768676758,
      "activations/layer9_attention_weight_max": 33.49168014526367,
      "activations/layer9_attention_weight_min": -33.13882064819336,
      "epoch": 9.95,
      "learning_rate": 8.670378787878787e-05,
      "loss": 2.8112,
      "step": 171250
    },
    {
      "activations/layer0_attention_weight_max": 15.975732803344727,
      "activations/layer0_attention_weight_min": -12.27524471282959,
      "activations/layer10_attention_weight_max": 34.9890022277832,
      "activations/layer10_attention_weight_min": -33.1884765625,
      "activations/layer11_attention_weight_max": 34.358909606933594,
      "activations/layer11_attention_weight_min": -31.777891159057617,
      "activations/layer12_attention_weight_max": 24.41215705871582,
      "activations/layer12_attention_weight_min": -29.32278060913086,
      "activations/layer13_attention_weight_max": 38.8655891418457,
      "activations/layer13_attention_weight_min": -34.14371871948242,
      "activations/layer14_attention_weight_max": 45.59401321411133,
      "activations/layer14_attention_weight_min": -37.46387481689453,
      "activations/layer15_attention_weight_max": 36.18815612792969,
      "activations/layer15_attention_weight_min": -30.678625106811523,
      "activations/layer16_attention_weight_max": 33.56487274169922,
      "activations/layer16_attention_weight_min": -26.21720314025879,
      "activations/layer17_attention_weight_max": 35.455013275146484,
      "activations/layer17_attention_weight_min": -26.60982322692871,
      "activations/layer18_attention_weight_max": 32.72589874267578,
      "activations/layer18_attention_weight_min": -22.967296600341797,
      "activations/layer19_attention_weight_max": 34.6596565246582,
      "activations/layer19_attention_weight_min": -32.1429557800293,
      "activations/layer1_attention_weight_max": 16.920244216918945,
      "activations/layer1_attention_weight_min": -16.078073501586914,
      "activations/layer20_attention_weight_max": 28.698631286621094,
      "activations/layer20_attention_weight_min": -25.97760009765625,
      "activations/layer21_attention_weight_max": 28.12371826171875,
      "activations/layer21_attention_weight_min": -24.58336639404297,
      "activations/layer22_attention_weight_max": 48.6409797668457,
      "activations/layer22_attention_weight_min": -32.34899139404297,
      "activations/layer23_attention_weight_max": 35.919639587402344,
      "activations/layer23_attention_weight_min": -27.583919525146484,
      "activations/layer2_attention_weight_max": 33.873573303222656,
      "activations/layer2_attention_weight_min": -32.714393615722656,
      "activations/layer3_attention_weight_max": 91.84874725341797,
      "activations/layer3_attention_weight_min": -92.44398498535156,
      "activations/layer4_attention_weight_max": 116.01187896728516,
      "activations/layer4_attention_weight_min": -113.48406982421875,
      "activations/layer5_attention_weight_max": 53.040794372558594,
      "activations/layer5_attention_weight_min": -63.74872589111328,
      "activations/layer6_attention_weight_max": 47.795597076416016,
      "activations/layer6_attention_weight_min": -51.71806716918945,
      "activations/layer7_attention_weight_max": 93.64547729492188,
      "activations/layer7_attention_weight_min": -89.24303436279297,
      "activations/layer8_attention_weight_max": 41.430545806884766,
      "activations/layer8_attention_weight_min": -42.95357894897461,
      "activations/layer9_attention_weight_max": 35.48551559448242,
      "activations/layer9_attention_weight_min": -34.16545104980469,
      "epoch": 9.95,
      "learning_rate": 8.668484848484847e-05,
      "loss": 2.7897,
      "step": 171300
    },
    {
      "activations/layer0_attention_weight_max": 16.370012283325195,
      "activations/layer0_attention_weight_min": -12.314123153686523,
      "activations/layer10_attention_weight_max": 31.066246032714844,
      "activations/layer10_attention_weight_min": -33.42969512939453,
      "activations/layer11_attention_weight_max": 30.5963134765625,
      "activations/layer11_attention_weight_min": -29.795936584472656,
      "activations/layer12_attention_weight_max": 21.82342529296875,
      "activations/layer12_attention_weight_min": -25.037063598632812,
      "activations/layer13_attention_weight_max": 35.823143005371094,
      "activations/layer13_attention_weight_min": -34.29462814331055,
      "activations/layer14_attention_weight_max": 43.80685043334961,
      "activations/layer14_attention_weight_min": -38.59431838989258,
      "activations/layer15_attention_weight_max": 36.20660400390625,
      "activations/layer15_attention_weight_min": -33.37771224975586,
      "activations/layer16_attention_weight_max": 31.98546600341797,
      "activations/layer16_attention_weight_min": -28.929460525512695,
      "activations/layer17_attention_weight_max": 31.14259910583496,
      "activations/layer17_attention_weight_min": -25.28916358947754,
      "activations/layer18_attention_weight_max": 28.21467399597168,
      "activations/layer18_attention_weight_min": -22.11836051940918,
      "activations/layer19_attention_weight_max": 39.32008743286133,
      "activations/layer19_attention_weight_min": -30.523893356323242,
      "activations/layer1_attention_weight_max": 16.9147891998291,
      "activations/layer1_attention_weight_min": -14.169937133789062,
      "activations/layer20_attention_weight_max": 33.16234588623047,
      "activations/layer20_attention_weight_min": -26.23329734802246,
      "activations/layer21_attention_weight_max": 30.33959197998047,
      "activations/layer21_attention_weight_min": -23.0146541595459,
      "activations/layer22_attention_weight_max": 39.937435150146484,
      "activations/layer22_attention_weight_min": -27.499645233154297,
      "activations/layer23_attention_weight_max": 31.859281539916992,
      "activations/layer23_attention_weight_min": -23.347135543823242,
      "activations/layer2_attention_weight_max": 33.34345245361328,
      "activations/layer2_attention_weight_min": -31.467567443847656,
      "activations/layer3_attention_weight_max": 92.2882080078125,
      "activations/layer3_attention_weight_min": -93.3492660522461,
      "activations/layer4_attention_weight_max": 108.8338851928711,
      "activations/layer4_attention_weight_min": -108.3121566772461,
      "activations/layer5_attention_weight_max": 54.49189758300781,
      "activations/layer5_attention_weight_min": -60.948341369628906,
      "activations/layer6_attention_weight_max": 44.702640533447266,
      "activations/layer6_attention_weight_min": -46.939369201660156,
      "activations/layer7_attention_weight_max": 93.72413635253906,
      "activations/layer7_attention_weight_min": -91.24869537353516,
      "activations/layer8_attention_weight_max": 40.04243850708008,
      "activations/layer8_attention_weight_min": -38.716957092285156,
      "activations/layer9_attention_weight_max": 31.7685604095459,
      "activations/layer9_attention_weight_min": -33.1440315246582,
      "epoch": 9.96,
      "learning_rate": 8.666628787878787e-05,
      "loss": 2.7975,
      "step": 171350
    },
    {
      "activations/layer0_attention_weight_max": 16.135282516479492,
      "activations/layer0_attention_weight_min": -11.64377498626709,
      "activations/layer10_attention_weight_max": 30.63229751586914,
      "activations/layer10_attention_weight_min": -31.837352752685547,
      "activations/layer11_attention_weight_max": 32.49434280395508,
      "activations/layer11_attention_weight_min": -31.573057174682617,
      "activations/layer12_attention_weight_max": 23.28506088256836,
      "activations/layer12_attention_weight_min": -24.78976058959961,
      "activations/layer13_attention_weight_max": 37.06529998779297,
      "activations/layer13_attention_weight_min": -33.42301559448242,
      "activations/layer14_attention_weight_max": 45.355106353759766,
      "activations/layer14_attention_weight_min": -37.09267044067383,
      "activations/layer15_attention_weight_max": 34.489994049072266,
      "activations/layer15_attention_weight_min": -32.45022201538086,
      "activations/layer16_attention_weight_max": 32.39605712890625,
      "activations/layer16_attention_weight_min": -29.92238426208496,
      "activations/layer17_attention_weight_max": 35.65270233154297,
      "activations/layer17_attention_weight_min": -27.250253677368164,
      "activations/layer18_attention_weight_max": 29.300373077392578,
      "activations/layer18_attention_weight_min": -21.161996841430664,
      "activations/layer19_attention_weight_max": 40.236412048339844,
      "activations/layer19_attention_weight_min": -31.020307540893555,
      "activations/layer1_attention_weight_max": 17.37232780456543,
      "activations/layer1_attention_weight_min": -14.638346672058105,
      "activations/layer20_attention_weight_max": 35.71266555786133,
      "activations/layer20_attention_weight_min": -24.39451789855957,
      "activations/layer21_attention_weight_max": 32.53061294555664,
      "activations/layer21_attention_weight_min": -23.666677474975586,
      "activations/layer22_attention_weight_max": 42.50559997558594,
      "activations/layer22_attention_weight_min": -29.01458740234375,
      "activations/layer23_attention_weight_max": 37.51130676269531,
      "activations/layer23_attention_weight_min": -25.907608032226562,
      "activations/layer2_attention_weight_max": 33.18064880371094,
      "activations/layer2_attention_weight_min": -32.92931365966797,
      "activations/layer3_attention_weight_max": 92.12042236328125,
      "activations/layer3_attention_weight_min": -91.794677734375,
      "activations/layer4_attention_weight_max": 111.97844696044922,
      "activations/layer4_attention_weight_min": -109.57476806640625,
      "activations/layer5_attention_weight_max": 54.395652770996094,
      "activations/layer5_attention_weight_min": -62.699119567871094,
      "activations/layer6_attention_weight_max": 45.759849548339844,
      "activations/layer6_attention_weight_min": -46.41959762573242,
      "activations/layer7_attention_weight_max": 90.20256042480469,
      "activations/layer7_attention_weight_min": -88.58656311035156,
      "activations/layer8_attention_weight_max": 39.4608154296875,
      "activations/layer8_attention_weight_min": -39.777923583984375,
      "activations/layer9_attention_weight_max": 32.58662033081055,
      "activations/layer9_attention_weight_min": -33.269287109375,
      "epoch": 9.96,
      "learning_rate": 8.664734848484848e-05,
      "loss": 2.8076,
      "step": 171400
    },
    {
      "activations/layer0_attention_weight_max": 15.176925659179688,
      "activations/layer0_attention_weight_min": -12.61043930053711,
      "activations/layer10_attention_weight_max": 34.063907623291016,
      "activations/layer10_attention_weight_min": -34.263954162597656,
      "activations/layer11_attention_weight_max": 34.822044372558594,
      "activations/layer11_attention_weight_min": -32.53995132446289,
      "activations/layer12_attention_weight_max": 25.059974670410156,
      "activations/layer12_attention_weight_min": -25.966676712036133,
      "activations/layer13_attention_weight_max": 40.37223815917969,
      "activations/layer13_attention_weight_min": -34.2856330871582,
      "activations/layer14_attention_weight_max": 45.39334487915039,
      "activations/layer14_attention_weight_min": -40.759613037109375,
      "activations/layer15_attention_weight_max": 34.43254089355469,
      "activations/layer15_attention_weight_min": -32.831024169921875,
      "activations/layer16_attention_weight_max": 29.37309455871582,
      "activations/layer16_attention_weight_min": -27.44189453125,
      "activations/layer17_attention_weight_max": 32.41139221191406,
      "activations/layer17_attention_weight_min": -24.4315128326416,
      "activations/layer18_attention_weight_max": 32.349937438964844,
      "activations/layer18_attention_weight_min": -24.192392349243164,
      "activations/layer19_attention_weight_max": 31.717798233032227,
      "activations/layer19_attention_weight_min": -30.83302116394043,
      "activations/layer1_attention_weight_max": 17.11630630493164,
      "activations/layer1_attention_weight_min": -15.521418571472168,
      "activations/layer20_attention_weight_max": 30.239171981811523,
      "activations/layer20_attention_weight_min": -25.25539207458496,
      "activations/layer21_attention_weight_max": 34.389320373535156,
      "activations/layer21_attention_weight_min": -23.915660858154297,
      "activations/layer22_attention_weight_max": 44.278995513916016,
      "activations/layer22_attention_weight_min": -31.4647216796875,
      "activations/layer23_attention_weight_max": 34.98078536987305,
      "activations/layer23_attention_weight_min": -24.762601852416992,
      "activations/layer2_attention_weight_max": 33.980926513671875,
      "activations/layer2_attention_weight_min": -34.160125732421875,
      "activations/layer3_attention_weight_max": 94.84138488769531,
      "activations/layer3_attention_weight_min": -99.01624298095703,
      "activations/layer4_attention_weight_max": 116.32002258300781,
      "activations/layer4_attention_weight_min": -114.17408752441406,
      "activations/layer5_attention_weight_max": 53.52449035644531,
      "activations/layer5_attention_weight_min": -60.99866485595703,
      "activations/layer6_attention_weight_max": 46.31475067138672,
      "activations/layer6_attention_weight_min": -51.09489822387695,
      "activations/layer7_attention_weight_max": 85.47708129882812,
      "activations/layer7_attention_weight_min": -91.7132568359375,
      "activations/layer8_attention_weight_max": 41.2972526550293,
      "activations/layer8_attention_weight_min": -42.24303436279297,
      "activations/layer9_attention_weight_max": 33.93585205078125,
      "activations/layer9_attention_weight_min": -37.523738861083984,
      "epoch": 9.96,
      "learning_rate": 8.662840909090908e-05,
      "loss": 2.8029,
      "step": 171450
    },
    {
      "activations/layer0_attention_weight_max": 15.958074569702148,
      "activations/layer0_attention_weight_min": -11.424654006958008,
      "activations/layer10_attention_weight_max": 38.358558654785156,
      "activations/layer10_attention_weight_min": -33.516822814941406,
      "activations/layer11_attention_weight_max": 36.263267517089844,
      "activations/layer11_attention_weight_min": -34.21976089477539,
      "activations/layer12_attention_weight_max": 44.50734329223633,
      "activations/layer12_attention_weight_min": -25.60724449157715,
      "activations/layer13_attention_weight_max": 49.63227081298828,
      "activations/layer13_attention_weight_min": -36.317481994628906,
      "activations/layer14_attention_weight_max": 52.95014572143555,
      "activations/layer14_attention_weight_min": -41.23771667480469,
      "activations/layer15_attention_weight_max": 40.114444732666016,
      "activations/layer15_attention_weight_min": -33.78363800048828,
      "activations/layer16_attention_weight_max": 37.29108810424805,
      "activations/layer16_attention_weight_min": -27.684659957885742,
      "activations/layer17_attention_weight_max": 34.379493713378906,
      "activations/layer17_attention_weight_min": -25.265378952026367,
      "activations/layer18_attention_weight_max": 33.232879638671875,
      "activations/layer18_attention_weight_min": -23.302242279052734,
      "activations/layer19_attention_weight_max": 38.44966125488281,
      "activations/layer19_attention_weight_min": -31.62302589416504,
      "activations/layer1_attention_weight_max": 16.479246139526367,
      "activations/layer1_attention_weight_min": -14.292817115783691,
      "activations/layer20_attention_weight_max": 32.000545501708984,
      "activations/layer20_attention_weight_min": -26.789676666259766,
      "activations/layer21_attention_weight_max": 39.58649444580078,
      "activations/layer21_attention_weight_min": -25.141965866088867,
      "activations/layer22_attention_weight_max": 47.53477478027344,
      "activations/layer22_attention_weight_min": -30.986371994018555,
      "activations/layer23_attention_weight_max": 39.950531005859375,
      "activations/layer23_attention_weight_min": -23.012983322143555,
      "activations/layer2_attention_weight_max": 33.76441955566406,
      "activations/layer2_attention_weight_min": -34.08695602416992,
      "activations/layer3_attention_weight_max": 100.55828094482422,
      "activations/layer3_attention_weight_min": -104.75701904296875,
      "activations/layer4_attention_weight_max": 114.994140625,
      "activations/layer4_attention_weight_min": -111.8935317993164,
      "activations/layer5_attention_weight_max": 51.92865753173828,
      "activations/layer5_attention_weight_min": -61.65178680419922,
      "activations/layer6_attention_weight_max": 44.465450286865234,
      "activations/layer6_attention_weight_min": -49.79752731323242,
      "activations/layer7_attention_weight_max": 92.57754516601562,
      "activations/layer7_attention_weight_min": -92.73590087890625,
      "activations/layer8_attention_weight_max": 42.22278594970703,
      "activations/layer8_attention_weight_min": -41.622406005859375,
      "activations/layer9_attention_weight_max": 37.07145309448242,
      "activations/layer9_attention_weight_min": -34.234825134277344,
      "epoch": 9.97,
      "learning_rate": 8.66094696969697e-05,
      "loss": 2.7884,
      "step": 171500
    },
    {
      "activations/layer0_attention_weight_max": 16.211164474487305,
      "activations/layer0_attention_weight_min": -12.216639518737793,
      "activations/layer10_attention_weight_max": 31.96445655822754,
      "activations/layer10_attention_weight_min": -31.075725555419922,
      "activations/layer11_attention_weight_max": 36.33761215209961,
      "activations/layer11_attention_weight_min": -31.530941009521484,
      "activations/layer12_attention_weight_max": 26.887998580932617,
      "activations/layer12_attention_weight_min": -28.21257972717285,
      "activations/layer13_attention_weight_max": 37.79896926879883,
      "activations/layer13_attention_weight_min": -31.433795928955078,
      "activations/layer14_attention_weight_max": 41.40953826904297,
      "activations/layer14_attention_weight_min": -34.22822570800781,
      "activations/layer15_attention_weight_max": 34.460628509521484,
      "activations/layer15_attention_weight_min": -27.960161209106445,
      "activations/layer16_attention_weight_max": 29.410465240478516,
      "activations/layer16_attention_weight_min": -27.449983596801758,
      "activations/layer17_attention_weight_max": 31.971338272094727,
      "activations/layer17_attention_weight_min": -25.088098526000977,
      "activations/layer18_attention_weight_max": 32.88240432739258,
      "activations/layer18_attention_weight_min": -21.765932083129883,
      "activations/layer19_attention_weight_max": 31.112985610961914,
      "activations/layer19_attention_weight_min": -30.87578773498535,
      "activations/layer1_attention_weight_max": 16.806964874267578,
      "activations/layer1_attention_weight_min": -15.806514739990234,
      "activations/layer20_attention_weight_max": 24.62883949279785,
      "activations/layer20_attention_weight_min": -23.505815505981445,
      "activations/layer21_attention_weight_max": 25.1552734375,
      "activations/layer21_attention_weight_min": -24.14006805419922,
      "activations/layer22_attention_weight_max": 40.08381652832031,
      "activations/layer22_attention_weight_min": -28.15241241455078,
      "activations/layer23_attention_weight_max": 32.35563278198242,
      "activations/layer23_attention_weight_min": -23.839933395385742,
      "activations/layer2_attention_weight_max": 34.259307861328125,
      "activations/layer2_attention_weight_min": -34.419097900390625,
      "activations/layer3_attention_weight_max": 93.63062286376953,
      "activations/layer3_attention_weight_min": -97.63762664794922,
      "activations/layer4_attention_weight_max": 115.49348449707031,
      "activations/layer4_attention_weight_min": -109.77764129638672,
      "activations/layer5_attention_weight_max": 52.34523010253906,
      "activations/layer5_attention_weight_min": -59.340126037597656,
      "activations/layer6_attention_weight_max": 45.953025817871094,
      "activations/layer6_attention_weight_min": -49.213558197021484,
      "activations/layer7_attention_weight_max": 95.20427703857422,
      "activations/layer7_attention_weight_min": -86.06249237060547,
      "activations/layer8_attention_weight_max": 40.23091125488281,
      "activations/layer8_attention_weight_min": -38.637760162353516,
      "activations/layer9_attention_weight_max": 32.31940841674805,
      "activations/layer9_attention_weight_min": -31.093923568725586,
      "epoch": 9.97,
      "learning_rate": 8.65905303030303e-05,
      "loss": 2.7854,
      "step": 171550
    },
    {
      "activations/layer0_attention_weight_max": 15.056863784790039,
      "activations/layer0_attention_weight_min": -12.356893539428711,
      "activations/layer10_attention_weight_max": 36.9406852722168,
      "activations/layer10_attention_weight_min": -35.059669494628906,
      "activations/layer11_attention_weight_max": 36.49190902709961,
      "activations/layer11_attention_weight_min": -34.42705535888672,
      "activations/layer12_attention_weight_max": 23.848670959472656,
      "activations/layer12_attention_weight_min": -25.178218841552734,
      "activations/layer13_attention_weight_max": 38.048606872558594,
      "activations/layer13_attention_weight_min": -34.235408782958984,
      "activations/layer14_attention_weight_max": 42.320106506347656,
      "activations/layer14_attention_weight_min": -38.29179763793945,
      "activations/layer15_attention_weight_max": 35.760746002197266,
      "activations/layer15_attention_weight_min": -30.654224395751953,
      "activations/layer16_attention_weight_max": 34.68131637573242,
      "activations/layer16_attention_weight_min": -28.124122619628906,
      "activations/layer17_attention_weight_max": 31.08089828491211,
      "activations/layer17_attention_weight_min": -24.139562606811523,
      "activations/layer18_attention_weight_max": 31.433122634887695,
      "activations/layer18_attention_weight_min": -21.668455123901367,
      "activations/layer19_attention_weight_max": 37.37888717651367,
      "activations/layer19_attention_weight_min": -31.600284576416016,
      "activations/layer1_attention_weight_max": 16.642168045043945,
      "activations/layer1_attention_weight_min": -15.736918449401855,
      "activations/layer20_attention_weight_max": 33.34099578857422,
      "activations/layer20_attention_weight_min": -23.561277389526367,
      "activations/layer21_attention_weight_max": 32.60075759887695,
      "activations/layer21_attention_weight_min": -22.781795501708984,
      "activations/layer22_attention_weight_max": 43.7961311340332,
      "activations/layer22_attention_weight_min": -26.72072410583496,
      "activations/layer23_attention_weight_max": 36.2584228515625,
      "activations/layer23_attention_weight_min": -23.619665145874023,
      "activations/layer2_attention_weight_max": 34.727386474609375,
      "activations/layer2_attention_weight_min": -33.659400939941406,
      "activations/layer3_attention_weight_max": 102.77955627441406,
      "activations/layer3_attention_weight_min": -101.08126068115234,
      "activations/layer4_attention_weight_max": 119.66064453125,
      "activations/layer4_attention_weight_min": -118.17695617675781,
      "activations/layer5_attention_weight_max": 55.66909408569336,
      "activations/layer5_attention_weight_min": -63.735008239746094,
      "activations/layer6_attention_weight_max": 46.71302795410156,
      "activations/layer6_attention_weight_min": -49.51691436767578,
      "activations/layer7_attention_weight_max": 91.71006774902344,
      "activations/layer7_attention_weight_min": -90.83282470703125,
      "activations/layer8_attention_weight_max": 44.36669158935547,
      "activations/layer8_attention_weight_min": -42.335975646972656,
      "activations/layer9_attention_weight_max": 37.79087448120117,
      "activations/layer9_attention_weight_min": -35.524715423583984,
      "epoch": 9.97,
      "learning_rate": 8.65715909090909e-05,
      "loss": 2.7851,
      "step": 171600
    },
    {
      "activations/layer0_attention_weight_max": 15.274984359741211,
      "activations/layer0_attention_weight_min": -12.189505577087402,
      "activations/layer10_attention_weight_max": 37.26117706298828,
      "activations/layer10_attention_weight_min": -32.68941879272461,
      "activations/layer11_attention_weight_max": 37.16841506958008,
      "activations/layer11_attention_weight_min": -33.729881286621094,
      "activations/layer12_attention_weight_max": 26.29126739501953,
      "activations/layer12_attention_weight_min": -23.55939292907715,
      "activations/layer13_attention_weight_max": 46.09126281738281,
      "activations/layer13_attention_weight_min": -33.17977523803711,
      "activations/layer14_attention_weight_max": 48.96827697753906,
      "activations/layer14_attention_weight_min": -39.0200309753418,
      "activations/layer15_attention_weight_max": 36.187137603759766,
      "activations/layer15_attention_weight_min": -30.552799224853516,
      "activations/layer16_attention_weight_max": 30.968338012695312,
      "activations/layer16_attention_weight_min": -29.452775955200195,
      "activations/layer17_attention_weight_max": 33.190494537353516,
      "activations/layer17_attention_weight_min": -25.323328018188477,
      "activations/layer18_attention_weight_max": 33.60597610473633,
      "activations/layer18_attention_weight_min": -22.55417823791504,
      "activations/layer19_attention_weight_max": 33.59638595581055,
      "activations/layer19_attention_weight_min": -31.544864654541016,
      "activations/layer1_attention_weight_max": 16.67072296142578,
      "activations/layer1_attention_weight_min": -15.781209945678711,
      "activations/layer20_attention_weight_max": 28.16063117980957,
      "activations/layer20_attention_weight_min": -26.008981704711914,
      "activations/layer21_attention_weight_max": 30.71922492980957,
      "activations/layer21_attention_weight_min": -22.6834716796875,
      "activations/layer22_attention_weight_max": 45.27653884887695,
      "activations/layer22_attention_weight_min": -30.222408294677734,
      "activations/layer23_attention_weight_max": 34.879127502441406,
      "activations/layer23_attention_weight_min": -23.934980392456055,
      "activations/layer2_attention_weight_max": 31.830806732177734,
      "activations/layer2_attention_weight_min": -34.62419891357422,
      "activations/layer3_attention_weight_max": 95.71968078613281,
      "activations/layer3_attention_weight_min": -99.90669250488281,
      "activations/layer4_attention_weight_max": 116.75279998779297,
      "activations/layer4_attention_weight_min": -116.515869140625,
      "activations/layer5_attention_weight_max": 54.01398468017578,
      "activations/layer5_attention_weight_min": -64.06365966796875,
      "activations/layer6_attention_weight_max": 48.63225173950195,
      "activations/layer6_attention_weight_min": -48.562198638916016,
      "activations/layer7_attention_weight_max": 93.99842071533203,
      "activations/layer7_attention_weight_min": -89.66156005859375,
      "activations/layer8_attention_weight_max": 44.62744903564453,
      "activations/layer8_attention_weight_min": -41.812599182128906,
      "activations/layer9_attention_weight_max": 38.6270866394043,
      "activations/layer9_attention_weight_min": -35.626190185546875,
      "epoch": 9.97,
      "learning_rate": 8.655265151515151e-05,
      "loss": 2.7697,
      "step": 171650
    },
    {
      "activations/layer0_attention_weight_max": 15.82999324798584,
      "activations/layer0_attention_weight_min": -13.651118278503418,
      "activations/layer10_attention_weight_max": 36.00840377807617,
      "activations/layer10_attention_weight_min": -32.27320098876953,
      "activations/layer11_attention_weight_max": 36.684783935546875,
      "activations/layer11_attention_weight_min": -32.39649200439453,
      "activations/layer12_attention_weight_max": 22.980913162231445,
      "activations/layer12_attention_weight_min": -25.48872184753418,
      "activations/layer13_attention_weight_max": 36.6085090637207,
      "activations/layer13_attention_weight_min": -31.912734985351562,
      "activations/layer14_attention_weight_max": 47.2514762878418,
      "activations/layer14_attention_weight_min": -35.92816925048828,
      "activations/layer15_attention_weight_max": 34.94639587402344,
      "activations/layer15_attention_weight_min": -29.748878479003906,
      "activations/layer16_attention_weight_max": 31.306798934936523,
      "activations/layer16_attention_weight_min": -25.372575759887695,
      "activations/layer17_attention_weight_max": 31.644025802612305,
      "activations/layer17_attention_weight_min": -23.637277603149414,
      "activations/layer18_attention_weight_max": 31.394535064697266,
      "activations/layer18_attention_weight_min": -20.884056091308594,
      "activations/layer19_attention_weight_max": 33.02073669433594,
      "activations/layer19_attention_weight_min": -27.91202735900879,
      "activations/layer1_attention_weight_max": 16.367305755615234,
      "activations/layer1_attention_weight_min": -15.741081237792969,
      "activations/layer20_attention_weight_max": 30.235139846801758,
      "activations/layer20_attention_weight_min": -23.15728759765625,
      "activations/layer21_attention_weight_max": 33.204429626464844,
      "activations/layer21_attention_weight_min": -22.530298233032227,
      "activations/layer22_attention_weight_max": 44.212852478027344,
      "activations/layer22_attention_weight_min": -27.57084083557129,
      "activations/layer23_attention_weight_max": 32.98448944091797,
      "activations/layer23_attention_weight_min": -24.49120330810547,
      "activations/layer2_attention_weight_max": 33.769065856933594,
      "activations/layer2_attention_weight_min": -36.742916107177734,
      "activations/layer3_attention_weight_max": 95.71001434326172,
      "activations/layer3_attention_weight_min": -98.22712707519531,
      "activations/layer4_attention_weight_max": 116.87373352050781,
      "activations/layer4_attention_weight_min": -117.86018371582031,
      "activations/layer5_attention_weight_max": 58.013633728027344,
      "activations/layer5_attention_weight_min": -62.175628662109375,
      "activations/layer6_attention_weight_max": 48.13084411621094,
      "activations/layer6_attention_weight_min": -49.82174301147461,
      "activations/layer7_attention_weight_max": 91.41414642333984,
      "activations/layer7_attention_weight_min": -92.83042907714844,
      "activations/layer8_attention_weight_max": 41.53498840332031,
      "activations/layer8_attention_weight_min": -42.72926712036133,
      "activations/layer9_attention_weight_max": 36.08660125732422,
      "activations/layer9_attention_weight_min": -33.863746643066406,
      "epoch": 9.98,
      "learning_rate": 8.65337121212121e-05,
      "loss": 2.8103,
      "step": 171700
    },
    {
      "activations/layer0_attention_weight_max": 15.43065357208252,
      "activations/layer0_attention_weight_min": -12.618022918701172,
      "activations/layer10_attention_weight_max": 40.19425582885742,
      "activations/layer10_attention_weight_min": -36.36748504638672,
      "activations/layer11_attention_weight_max": 45.254005432128906,
      "activations/layer11_attention_weight_min": -37.40800476074219,
      "activations/layer12_attention_weight_max": 32.75624084472656,
      "activations/layer12_attention_weight_min": -28.840091705322266,
      "activations/layer13_attention_weight_max": 53.34663772583008,
      "activations/layer13_attention_weight_min": -36.89976501464844,
      "activations/layer14_attention_weight_max": 68.03926849365234,
      "activations/layer14_attention_weight_min": -45.42048645019531,
      "activations/layer15_attention_weight_max": 52.233585357666016,
      "activations/layer15_attention_weight_min": -32.34467697143555,
      "activations/layer16_attention_weight_max": 44.16611099243164,
      "activations/layer16_attention_weight_min": -30.164819717407227,
      "activations/layer17_attention_weight_max": 34.521907806396484,
      "activations/layer17_attention_weight_min": -26.2150936126709,
      "activations/layer18_attention_weight_max": 34.04267501831055,
      "activations/layer18_attention_weight_min": -23.873144149780273,
      "activations/layer19_attention_weight_max": 43.31023406982422,
      "activations/layer19_attention_weight_min": -32.732547760009766,
      "activations/layer1_attention_weight_max": 16.598384857177734,
      "activations/layer1_attention_weight_min": -15.813387870788574,
      "activations/layer20_attention_weight_max": 34.17610549926758,
      "activations/layer20_attention_weight_min": -25.140226364135742,
      "activations/layer21_attention_weight_max": 32.411537170410156,
      "activations/layer21_attention_weight_min": -24.015567779541016,
      "activations/layer22_attention_weight_max": 55.639522552490234,
      "activations/layer22_attention_weight_min": -28.642242431640625,
      "activations/layer23_attention_weight_max": 42.26766586303711,
      "activations/layer23_attention_weight_min": -24.170490264892578,
      "activations/layer2_attention_weight_max": 33.72377395629883,
      "activations/layer2_attention_weight_min": -32.8858642578125,
      "activations/layer3_attention_weight_max": 95.30532836914062,
      "activations/layer3_attention_weight_min": -96.70623779296875,
      "activations/layer4_attention_weight_max": 118.17756652832031,
      "activations/layer4_attention_weight_min": -114.15103912353516,
      "activations/layer5_attention_weight_max": 56.729530334472656,
      "activations/layer5_attention_weight_min": -62.70022201538086,
      "activations/layer6_attention_weight_max": 48.70472717285156,
      "activations/layer6_attention_weight_min": -51.11885452270508,
      "activations/layer7_attention_weight_max": 98.38841247558594,
      "activations/layer7_attention_weight_min": -91.86260986328125,
      "activations/layer8_attention_weight_max": 44.53630828857422,
      "activations/layer8_attention_weight_min": -44.24219512939453,
      "activations/layer9_attention_weight_max": 39.046546936035156,
      "activations/layer9_attention_weight_min": -36.63786315917969,
      "epoch": 9.98,
      "learning_rate": 8.651477272727272e-05,
      "loss": 2.7991,
      "step": 171750
    },
    {
      "activations/layer0_attention_weight_max": 16.40704917907715,
      "activations/layer0_attention_weight_min": -11.54283332824707,
      "activations/layer10_attention_weight_max": 34.35329055786133,
      "activations/layer10_attention_weight_min": -31.754674911499023,
      "activations/layer11_attention_weight_max": 32.623321533203125,
      "activations/layer11_attention_weight_min": -31.74043083190918,
      "activations/layer12_attention_weight_max": 22.92742156982422,
      "activations/layer12_attention_weight_min": -26.204181671142578,
      "activations/layer13_attention_weight_max": 36.333797454833984,
      "activations/layer13_attention_weight_min": -33.850379943847656,
      "activations/layer14_attention_weight_max": 41.18946075439453,
      "activations/layer14_attention_weight_min": -39.47042465209961,
      "activations/layer15_attention_weight_max": 33.666263580322266,
      "activations/layer15_attention_weight_min": -32.694644927978516,
      "activations/layer16_attention_weight_max": 29.3911190032959,
      "activations/layer16_attention_weight_min": -29.116487503051758,
      "activations/layer17_attention_weight_max": 33.129913330078125,
      "activations/layer17_attention_weight_min": -25.648595809936523,
      "activations/layer18_attention_weight_max": 31.851802825927734,
      "activations/layer18_attention_weight_min": -22.001184463500977,
      "activations/layer19_attention_weight_max": 34.04171371459961,
      "activations/layer19_attention_weight_min": -30.30860710144043,
      "activations/layer1_attention_weight_max": 17.198583602905273,
      "activations/layer1_attention_weight_min": -15.253226280212402,
      "activations/layer20_attention_weight_max": 27.735870361328125,
      "activations/layer20_attention_weight_min": -24.23763656616211,
      "activations/layer21_attention_weight_max": 29.00114631652832,
      "activations/layer21_attention_weight_min": -22.201858520507812,
      "activations/layer22_attention_weight_max": 41.36083221435547,
      "activations/layer22_attention_weight_min": -28.93500518798828,
      "activations/layer23_attention_weight_max": 34.018775939941406,
      "activations/layer23_attention_weight_min": -23.284198760986328,
      "activations/layer2_attention_weight_max": 33.23719024658203,
      "activations/layer2_attention_weight_min": -32.97098159790039,
      "activations/layer3_attention_weight_max": 94.04432678222656,
      "activations/layer3_attention_weight_min": -96.68633270263672,
      "activations/layer4_attention_weight_max": 116.34801483154297,
      "activations/layer4_attention_weight_min": -112.81951141357422,
      "activations/layer5_attention_weight_max": 52.778865814208984,
      "activations/layer5_attention_weight_min": -63.167724609375,
      "activations/layer6_attention_weight_max": 45.97233200073242,
      "activations/layer6_attention_weight_min": -51.18319320678711,
      "activations/layer7_attention_weight_max": 87.52639770507812,
      "activations/layer7_attention_weight_min": -86.34758758544922,
      "activations/layer8_attention_weight_max": 41.1746711730957,
      "activations/layer8_attention_weight_min": -41.124942779541016,
      "activations/layer9_attention_weight_max": 37.09663009643555,
      "activations/layer9_attention_weight_min": -35.51716613769531,
      "epoch": 9.98,
      "learning_rate": 8.649583333333333e-05,
      "loss": 2.7939,
      "step": 171800
    },
    {
      "activations/layer0_attention_weight_max": 15.609317779541016,
      "activations/layer0_attention_weight_min": -12.016705513000488,
      "activations/layer10_attention_weight_max": 32.140663146972656,
      "activations/layer10_attention_weight_min": -30.711729049682617,
      "activations/layer11_attention_weight_max": 31.226499557495117,
      "activations/layer11_attention_weight_min": -29.941518783569336,
      "activations/layer12_attention_weight_max": 23.680999755859375,
      "activations/layer12_attention_weight_min": -24.13172149658203,
      "activations/layer13_attention_weight_max": 35.97914123535156,
      "activations/layer13_attention_weight_min": -32.98225402832031,
      "activations/layer14_attention_weight_max": 45.63094711303711,
      "activations/layer14_attention_weight_min": -37.75340270996094,
      "activations/layer15_attention_weight_max": 36.02216720581055,
      "activations/layer15_attention_weight_min": -29.78740692138672,
      "activations/layer16_attention_weight_max": 33.22275161743164,
      "activations/layer16_attention_weight_min": -28.61894416809082,
      "activations/layer17_attention_weight_max": 29.60344886779785,
      "activations/layer17_attention_weight_min": -25.65960121154785,
      "activations/layer18_attention_weight_max": 29.861167907714844,
      "activations/layer18_attention_weight_min": -22.497936248779297,
      "activations/layer19_attention_weight_max": 33.76424026489258,
      "activations/layer19_attention_weight_min": -28.365055084228516,
      "activations/layer1_attention_weight_max": 17.620508193969727,
      "activations/layer1_attention_weight_min": -14.183987617492676,
      "activations/layer20_attention_weight_max": 28.459150314331055,
      "activations/layer20_attention_weight_min": -23.225149154663086,
      "activations/layer21_attention_weight_max": 27.68813133239746,
      "activations/layer21_attention_weight_min": -22.01047706604004,
      "activations/layer22_attention_weight_max": 43.741817474365234,
      "activations/layer22_attention_weight_min": -29.209585189819336,
      "activations/layer23_attention_weight_max": 34.62669372558594,
      "activations/layer23_attention_weight_min": -23.15166473388672,
      "activations/layer2_attention_weight_max": 31.30978775024414,
      "activations/layer2_attention_weight_min": -33.143829345703125,
      "activations/layer3_attention_weight_max": 91.55703735351562,
      "activations/layer3_attention_weight_min": -95.71505737304688,
      "activations/layer4_attention_weight_max": 112.41686248779297,
      "activations/layer4_attention_weight_min": -111.125,
      "activations/layer5_attention_weight_max": 52.50120544433594,
      "activations/layer5_attention_weight_min": -59.75828552246094,
      "activations/layer6_attention_weight_max": 48.94895935058594,
      "activations/layer6_attention_weight_min": -50.07465362548828,
      "activations/layer7_attention_weight_max": 94.32921600341797,
      "activations/layer7_attention_weight_min": -86.34288024902344,
      "activations/layer8_attention_weight_max": 40.78720474243164,
      "activations/layer8_attention_weight_min": -40.191200256347656,
      "activations/layer9_attention_weight_max": 32.63393783569336,
      "activations/layer9_attention_weight_min": -31.744388580322266,
      "epoch": 9.99,
      "learning_rate": 8.647689393939392e-05,
      "loss": 2.7861,
      "step": 171850
    },
    {
      "activations/layer0_attention_weight_max": 16.770267486572266,
      "activations/layer0_attention_weight_min": -11.19006633758545,
      "activations/layer10_attention_weight_max": 38.58026123046875,
      "activations/layer10_attention_weight_min": -36.26639175415039,
      "activations/layer11_attention_weight_max": 35.97232437133789,
      "activations/layer11_attention_weight_min": -35.137245178222656,
      "activations/layer12_attention_weight_max": 39.36993408203125,
      "activations/layer12_attention_weight_min": -27.504533767700195,
      "activations/layer13_attention_weight_max": 52.80875778198242,
      "activations/layer13_attention_weight_min": -35.95670700073242,
      "activations/layer14_attention_weight_max": 42.46907043457031,
      "activations/layer14_attention_weight_min": -36.27375411987305,
      "activations/layer15_attention_weight_max": 36.814903259277344,
      "activations/layer15_attention_weight_min": -31.487268447875977,
      "activations/layer16_attention_weight_max": 29.33763313293457,
      "activations/layer16_attention_weight_min": -26.67112922668457,
      "activations/layer17_attention_weight_max": 31.77998161315918,
      "activations/layer17_attention_weight_min": -25.741228103637695,
      "activations/layer18_attention_weight_max": 28.496389389038086,
      "activations/layer18_attention_weight_min": -22.867712020874023,
      "activations/layer19_attention_weight_max": 34.384727478027344,
      "activations/layer19_attention_weight_min": -29.73137855529785,
      "activations/layer1_attention_weight_max": 17.013628005981445,
      "activations/layer1_attention_weight_min": -16.68039894104004,
      "activations/layer20_attention_weight_max": 29.699682235717773,
      "activations/layer20_attention_weight_min": -24.458866119384766,
      "activations/layer21_attention_weight_max": 28.706499099731445,
      "activations/layer21_attention_weight_min": -24.596399307250977,
      "activations/layer22_attention_weight_max": 43.13200759887695,
      "activations/layer22_attention_weight_min": -30.305444717407227,
      "activations/layer23_attention_weight_max": 31.967063903808594,
      "activations/layer23_attention_weight_min": -26.8880672454834,
      "activations/layer2_attention_weight_max": 36.220130920410156,
      "activations/layer2_attention_weight_min": -39.02536392211914,
      "activations/layer3_attention_weight_max": 109.59173583984375,
      "activations/layer3_attention_weight_min": -107.14163208007812,
      "activations/layer4_attention_weight_max": 130.14134216308594,
      "activations/layer4_attention_weight_min": -124.36394500732422,
      "activations/layer5_attention_weight_max": 63.60734176635742,
      "activations/layer5_attention_weight_min": -68.15072631835938,
      "activations/layer6_attention_weight_max": 51.721920013427734,
      "activations/layer6_attention_weight_min": -50.600074768066406,
      "activations/layer7_attention_weight_max": 98.519775390625,
      "activations/layer7_attention_weight_min": -97.19507598876953,
      "activations/layer8_attention_weight_max": 45.3347053527832,
      "activations/layer8_attention_weight_min": -43.475791931152344,
      "activations/layer9_attention_weight_max": 38.114662170410156,
      "activations/layer9_attention_weight_min": -36.20730209350586,
      "epoch": 9.99,
      "learning_rate": 8.645795454545454e-05,
      "loss": 2.7911,
      "step": 171900
    },
    {
      "activations/layer0_attention_weight_max": 15.128643035888672,
      "activations/layer0_attention_weight_min": -13.088658332824707,
      "activations/layer10_attention_weight_max": 34.858009338378906,
      "activations/layer10_attention_weight_min": -31.948007583618164,
      "activations/layer11_attention_weight_max": 33.13410186767578,
      "activations/layer11_attention_weight_min": -32.301361083984375,
      "activations/layer12_attention_weight_max": 28.805150985717773,
      "activations/layer12_attention_weight_min": -24.08283233642578,
      "activations/layer13_attention_weight_max": 40.437232971191406,
      "activations/layer13_attention_weight_min": -34.658111572265625,
      "activations/layer14_attention_weight_max": 51.67191696166992,
      "activations/layer14_attention_weight_min": -39.10829162597656,
      "activations/layer15_attention_weight_max": 36.590450286865234,
      "activations/layer15_attention_weight_min": -31.95810317993164,
      "activations/layer16_attention_weight_max": 33.43275451660156,
      "activations/layer16_attention_weight_min": -26.645063400268555,
      "activations/layer17_attention_weight_max": 35.44646072387695,
      "activations/layer17_attention_weight_min": -26.1049747467041,
      "activations/layer18_attention_weight_max": 36.515621185302734,
      "activations/layer18_attention_weight_min": -23.417278289794922,
      "activations/layer19_attention_weight_max": 40.302860260009766,
      "activations/layer19_attention_weight_min": -32.431480407714844,
      "activations/layer1_attention_weight_max": 17.264049530029297,
      "activations/layer1_attention_weight_min": -14.25261402130127,
      "activations/layer20_attention_weight_max": 31.162240982055664,
      "activations/layer20_attention_weight_min": -27.306472778320312,
      "activations/layer21_attention_weight_max": 37.768959045410156,
      "activations/layer21_attention_weight_min": -25.173843383789062,
      "activations/layer22_attention_weight_max": 47.2962646484375,
      "activations/layer22_attention_weight_min": -31.088682174682617,
      "activations/layer23_attention_weight_max": 36.0097541809082,
      "activations/layer23_attention_weight_min": -25.197566986083984,
      "activations/layer2_attention_weight_max": 32.66657257080078,
      "activations/layer2_attention_weight_min": -33.62543487548828,
      "activations/layer3_attention_weight_max": 90.87987518310547,
      "activations/layer3_attention_weight_min": -95.76412963867188,
      "activations/layer4_attention_weight_max": 110.5107192993164,
      "activations/layer4_attention_weight_min": -113.89399719238281,
      "activations/layer5_attention_weight_max": 52.554473876953125,
      "activations/layer5_attention_weight_min": -57.782100677490234,
      "activations/layer6_attention_weight_max": 45.70311737060547,
      "activations/layer6_attention_weight_min": -48.11343002319336,
      "activations/layer7_attention_weight_max": 89.57173156738281,
      "activations/layer7_attention_weight_min": -88.92115783691406,
      "activations/layer8_attention_weight_max": 42.80947494506836,
      "activations/layer8_attention_weight_min": -40.20888137817383,
      "activations/layer9_attention_weight_max": 34.894649505615234,
      "activations/layer9_attention_weight_min": -33.58254623413086,
      "epoch": 9.99,
      "learning_rate": 8.643901515151515e-05,
      "loss": 2.7999,
      "step": 171950
    },
    {
      "activations/layer0_attention_weight_max": 15.831086158752441,
      "activations/layer0_attention_weight_min": -12.769654273986816,
      "activations/layer10_attention_weight_max": 33.30916213989258,
      "activations/layer10_attention_weight_min": -34.652870178222656,
      "activations/layer11_attention_weight_max": 32.30720520019531,
      "activations/layer11_attention_weight_min": -32.12172317504883,
      "activations/layer12_attention_weight_max": 25.90335464477539,
      "activations/layer12_attention_weight_min": -30.755765914916992,
      "activations/layer13_attention_weight_max": 40.849151611328125,
      "activations/layer13_attention_weight_min": -34.59929656982422,
      "activations/layer14_attention_weight_max": 49.52376937866211,
      "activations/layer14_attention_weight_min": -38.78684997558594,
      "activations/layer15_attention_weight_max": 36.95091247558594,
      "activations/layer15_attention_weight_min": -31.62204360961914,
      "activations/layer16_attention_weight_max": 32.02503204345703,
      "activations/layer16_attention_weight_min": -28.538551330566406,
      "activations/layer17_attention_weight_max": 36.04582595825195,
      "activations/layer17_attention_weight_min": -27.592124938964844,
      "activations/layer18_attention_weight_max": 35.64621353149414,
      "activations/layer18_attention_weight_min": -23.040184020996094,
      "activations/layer19_attention_weight_max": 37.02256393432617,
      "activations/layer19_attention_weight_min": -31.170236587524414,
      "activations/layer1_attention_weight_max": 16.650924682617188,
      "activations/layer1_attention_weight_min": -15.943796157836914,
      "activations/layer20_attention_weight_max": 31.6761531829834,
      "activations/layer20_attention_weight_min": -24.570934295654297,
      "activations/layer21_attention_weight_max": 31.370967864990234,
      "activations/layer21_attention_weight_min": -24.82260513305664,
      "activations/layer22_attention_weight_max": 61.12378692626953,
      "activations/layer22_attention_weight_min": -29.171106338500977,
      "activations/layer23_attention_weight_max": 38.09709167480469,
      "activations/layer23_attention_weight_min": -24.85794448852539,
      "activations/layer2_attention_weight_max": 34.77676773071289,
      "activations/layer2_attention_weight_min": -33.37175750732422,
      "activations/layer3_attention_weight_max": 97.33495330810547,
      "activations/layer3_attention_weight_min": -96.66581726074219,
      "activations/layer4_attention_weight_max": 121.76708984375,
      "activations/layer4_attention_weight_min": -112.98072814941406,
      "activations/layer5_attention_weight_max": 54.858001708984375,
      "activations/layer5_attention_weight_min": -63.495025634765625,
      "activations/layer6_attention_weight_max": 47.08010482788086,
      "activations/layer6_attention_weight_min": -49.2577018737793,
      "activations/layer7_attention_weight_max": 87.04635620117188,
      "activations/layer7_attention_weight_min": -85.89675903320312,
      "activations/layer8_attention_weight_max": 41.207305908203125,
      "activations/layer8_attention_weight_min": -40.43236541748047,
      "activations/layer9_attention_weight_max": 35.66917419433594,
      "activations/layer9_attention_weight_min": -34.91244888305664,
      "epoch": 9.99,
      "learning_rate": 8.642007575757575e-05,
      "loss": 2.7995,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.3515,
      "eval_samples_per_second": 514.159,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.3515,
      "eval_openwebtext_samples_per_second": 514.159,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.9446,
      "eval_wikitext_samples_per_second": 234.493,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_lambada_loss": 2.541015625,
      "eval_lambada_ppl": 12.692555303381242,
      "eval_lambada_runtime": 9.569,
      "eval_lambada_samples_per_second": 508.831,
      "step": 172000
    },
    {
      "activations/layer0_attention_weight_max": 16.073238372802734,
      "activations/layer0_attention_weight_min": -11.95715045928955,
      "activations/layer10_attention_weight_max": 39.659217834472656,
      "activations/layer10_attention_weight_min": -35.11211395263672,
      "activations/layer11_attention_weight_max": 36.56194305419922,
      "activations/layer11_attention_weight_min": -30.869205474853516,
      "activations/layer12_attention_weight_max": 25.19978141784668,
      "activations/layer12_attention_weight_min": -26.65709114074707,
      "activations/layer13_attention_weight_max": 38.36402130126953,
      "activations/layer13_attention_weight_min": -35.39453125,
      "activations/layer14_attention_weight_max": 45.738834381103516,
      "activations/layer14_attention_weight_min": -39.61905288696289,
      "activations/layer15_attention_weight_max": 35.83536911010742,
      "activations/layer15_attention_weight_min": -31.134994506835938,
      "activations/layer16_attention_weight_max": 29.613988876342773,
      "activations/layer16_attention_weight_min": -27.35430145263672,
      "activations/layer17_attention_weight_max": 30.0537109375,
      "activations/layer17_attention_weight_min": -26.832944869995117,
      "activations/layer18_attention_weight_max": 27.935298919677734,
      "activations/layer18_attention_weight_min": -24.545896530151367,
      "activations/layer19_attention_weight_max": 31.821630477905273,
      "activations/layer19_attention_weight_min": -30.373300552368164,
      "activations/layer1_attention_weight_max": 16.279773712158203,
      "activations/layer1_attention_weight_min": -14.677668571472168,
      "activations/layer20_attention_weight_max": 27.98309898376465,
      "activations/layer20_attention_weight_min": -24.532514572143555,
      "activations/layer21_attention_weight_max": 25.512845993041992,
      "activations/layer21_attention_weight_min": -23.22589111328125,
      "activations/layer22_attention_weight_max": 45.7054443359375,
      "activations/layer22_attention_weight_min": -28.504093170166016,
      "activations/layer23_attention_weight_max": 39.57234191894531,
      "activations/layer23_attention_weight_min": -25.35915184020996,
      "activations/layer2_attention_weight_max": 30.053550720214844,
      "activations/layer2_attention_weight_min": -30.87152671813965,
      "activations/layer3_attention_weight_max": 91.47019958496094,
      "activations/layer3_attention_weight_min": -97.41944885253906,
      "activations/layer4_attention_weight_max": 117.33055877685547,
      "activations/layer4_attention_weight_min": -113.10298156738281,
      "activations/layer5_attention_weight_max": 53.945858001708984,
      "activations/layer5_attention_weight_min": -67.29242706298828,
      "activations/layer6_attention_weight_max": 47.84372329711914,
      "activations/layer6_attention_weight_min": -47.926902770996094,
      "activations/layer7_attention_weight_max": 96.7484130859375,
      "activations/layer7_attention_weight_min": -91.2591781616211,
      "activations/layer8_attention_weight_max": 45.70466613769531,
      "activations/layer8_attention_weight_min": -44.305450439453125,
      "activations/layer9_attention_weight_max": 37.76731491088867,
      "activations/layer9_attention_weight_min": -34.634334564208984,
      "epoch": 10.0,
      "learning_rate": 8.640113636363635e-05,
      "loss": 2.7914,
      "step": 172050
    },
    {
      "activations/layer0_attention_weight_max": 15.070587158203125,
      "activations/layer0_attention_weight_min": -11.770224571228027,
      "activations/layer10_attention_weight_max": 32.337493896484375,
      "activations/layer10_attention_weight_min": -30.212841033935547,
      "activations/layer11_attention_weight_max": 32.36274719238281,
      "activations/layer11_attention_weight_min": -30.163867950439453,
      "activations/layer12_attention_weight_max": 23.78146743774414,
      "activations/layer12_attention_weight_min": -25.202056884765625,
      "activations/layer13_attention_weight_max": 40.356529235839844,
      "activations/layer13_attention_weight_min": -32.942588806152344,
      "activations/layer14_attention_weight_max": 50.95414352416992,
      "activations/layer14_attention_weight_min": -39.95275115966797,
      "activations/layer15_attention_weight_max": 43.907352447509766,
      "activations/layer15_attention_weight_min": -32.07621383666992,
      "activations/layer16_attention_weight_max": 34.70132827758789,
      "activations/layer16_attention_weight_min": -27.324420928955078,
      "activations/layer17_attention_weight_max": 33.438846588134766,
      "activations/layer17_attention_weight_min": -26.09382438659668,
      "activations/layer18_attention_weight_max": 29.549034118652344,
      "activations/layer18_attention_weight_min": -23.01876449584961,
      "activations/layer19_attention_weight_max": 38.919002532958984,
      "activations/layer19_attention_weight_min": -34.2802734375,
      "activations/layer1_attention_weight_max": 16.95599937438965,
      "activations/layer1_attention_weight_min": -17.4554500579834,
      "activations/layer20_attention_weight_max": 32.982540130615234,
      "activations/layer20_attention_weight_min": -25.66975975036621,
      "activations/layer21_attention_weight_max": 31.491241455078125,
      "activations/layer21_attention_weight_min": -24.13249969482422,
      "activations/layer22_attention_weight_max": 50.3720817565918,
      "activations/layer22_attention_weight_min": -32.91743469238281,
      "activations/layer23_attention_weight_max": 36.25234603881836,
      "activations/layer23_attention_weight_min": -24.605743408203125,
      "activations/layer2_attention_weight_max": 30.975645065307617,
      "activations/layer2_attention_weight_min": -33.50027084350586,
      "activations/layer3_attention_weight_max": 92.63502502441406,
      "activations/layer3_attention_weight_min": -91.87434387207031,
      "activations/layer4_attention_weight_max": 118.03741455078125,
      "activations/layer4_attention_weight_min": -110.85579681396484,
      "activations/layer5_attention_weight_max": 54.13821029663086,
      "activations/layer5_attention_weight_min": -60.9155387878418,
      "activations/layer6_attention_weight_max": 46.060115814208984,
      "activations/layer6_attention_weight_min": -47.397071838378906,
      "activations/layer7_attention_weight_max": 90.9744644165039,
      "activations/layer7_attention_weight_min": -86.90965270996094,
      "activations/layer8_attention_weight_max": 38.671756744384766,
      "activations/layer8_attention_weight_min": -37.303550720214844,
      "activations/layer9_attention_weight_max": 31.855453491210938,
      "activations/layer9_attention_weight_min": -31.199583053588867,
      "epoch": 10.0,
      "learning_rate": 8.638219696969696e-05,
      "loss": 2.7939,
      "step": 172100
    },
    {
      "activations/layer0_attention_weight_max": 16.84315299987793,
      "activations/layer0_attention_weight_min": -11.503878593444824,
      "activations/layer10_attention_weight_max": 32.9105110168457,
      "activations/layer10_attention_weight_min": -31.194913864135742,
      "activations/layer11_attention_weight_max": 32.519859313964844,
      "activations/layer11_attention_weight_min": -30.919452667236328,
      "activations/layer12_attention_weight_max": 28.802343368530273,
      "activations/layer12_attention_weight_min": -23.839269638061523,
      "activations/layer13_attention_weight_max": 38.56639099121094,
      "activations/layer13_attention_weight_min": -35.59108352661133,
      "activations/layer14_attention_weight_max": 43.67928695678711,
      "activations/layer14_attention_weight_min": -37.78505325317383,
      "activations/layer15_attention_weight_max": 33.424949645996094,
      "activations/layer15_attention_weight_min": -31.49024772644043,
      "activations/layer16_attention_weight_max": 30.98936653137207,
      "activations/layer16_attention_weight_min": -30.02104949951172,
      "activations/layer17_attention_weight_max": 32.79085159301758,
      "activations/layer17_attention_weight_min": -26.942106246948242,
      "activations/layer18_attention_weight_max": 33.08543395996094,
      "activations/layer18_attention_weight_min": -22.198095321655273,
      "activations/layer19_attention_weight_max": 33.24519729614258,
      "activations/layer19_attention_weight_min": -30.94632339477539,
      "activations/layer1_attention_weight_max": 16.35224151611328,
      "activations/layer1_attention_weight_min": -15.010684967041016,
      "activations/layer20_attention_weight_max": 29.512060165405273,
      "activations/layer20_attention_weight_min": -24.505935668945312,
      "activations/layer21_attention_weight_max": 27.62247085571289,
      "activations/layer21_attention_weight_min": -23.374576568603516,
      "activations/layer22_attention_weight_max": 47.49857711791992,
      "activations/layer22_attention_weight_min": -29.412458419799805,
      "activations/layer23_attention_weight_max": 36.63544464111328,
      "activations/layer23_attention_weight_min": -24.060321807861328,
      "activations/layer2_attention_weight_max": 30.696096420288086,
      "activations/layer2_attention_weight_min": -31.148479461669922,
      "activations/layer3_attention_weight_max": 90.95660400390625,
      "activations/layer3_attention_weight_min": -92.90816497802734,
      "activations/layer4_attention_weight_max": 112.4568099975586,
      "activations/layer4_attention_weight_min": -113.62044525146484,
      "activations/layer5_attention_weight_max": 54.27964782714844,
      "activations/layer5_attention_weight_min": -61.68738555908203,
      "activations/layer6_attention_weight_max": 46.278785705566406,
      "activations/layer6_attention_weight_min": -50.196739196777344,
      "activations/layer7_attention_weight_max": 91.04679107666016,
      "activations/layer7_attention_weight_min": -90.8175048828125,
      "activations/layer8_attention_weight_max": 39.57281494140625,
      "activations/layer8_attention_weight_min": -39.306983947753906,
      "activations/layer9_attention_weight_max": 33.86622619628906,
      "activations/layer9_attention_weight_min": -32.24504089355469,
      "epoch": 10.0,
      "learning_rate": 8.636287878787878e-05,
      "loss": 2.8464,
      "step": 172150
    },
    {
      "activations/layer0_attention_weight_max": 15.498797416687012,
      "activations/layer0_attention_weight_min": -11.96084976196289,
      "activations/layer10_attention_weight_max": 29.293188095092773,
      "activations/layer10_attention_weight_min": -31.51579475402832,
      "activations/layer11_attention_weight_max": 29.663564682006836,
      "activations/layer11_attention_weight_min": -29.21619415283203,
      "activations/layer12_attention_weight_max": 38.53766632080078,
      "activations/layer12_attention_weight_min": -33.74154281616211,
      "activations/layer13_attention_weight_max": 39.72596740722656,
      "activations/layer13_attention_weight_min": -32.461708068847656,
      "activations/layer14_attention_weight_max": 48.7674674987793,
      "activations/layer14_attention_weight_min": -38.42451477050781,
      "activations/layer15_attention_weight_max": 36.99690628051758,
      "activations/layer15_attention_weight_min": -29.36431884765625,
      "activations/layer16_attention_weight_max": 32.18242263793945,
      "activations/layer16_attention_weight_min": -27.779699325561523,
      "activations/layer17_attention_weight_max": 33.335784912109375,
      "activations/layer17_attention_weight_min": -24.999841690063477,
      "activations/layer18_attention_weight_max": 29.433664321899414,
      "activations/layer18_attention_weight_min": -22.218170166015625,
      "activations/layer19_attention_weight_max": 36.547550201416016,
      "activations/layer19_attention_weight_min": -31.274381637573242,
      "activations/layer1_attention_weight_max": 16.371719360351562,
      "activations/layer1_attention_weight_min": -13.930023193359375,
      "activations/layer20_attention_weight_max": 32.1158332824707,
      "activations/layer20_attention_weight_min": -25.060020446777344,
      "activations/layer21_attention_weight_max": 31.362010955810547,
      "activations/layer21_attention_weight_min": -23.79323387145996,
      "activations/layer22_attention_weight_max": 49.069698333740234,
      "activations/layer22_attention_weight_min": -29.2839412689209,
      "activations/layer23_attention_weight_max": 35.11400604248047,
      "activations/layer23_attention_weight_min": -22.855566024780273,
      "activations/layer2_attention_weight_max": 31.1956844329834,
      "activations/layer2_attention_weight_min": -32.47370147705078,
      "activations/layer3_attention_weight_max": 89.8613510131836,
      "activations/layer3_attention_weight_min": -91.97582244873047,
      "activations/layer4_attention_weight_max": 111.3669204711914,
      "activations/layer4_attention_weight_min": -110.23302459716797,
      "activations/layer5_attention_weight_max": 52.321441650390625,
      "activations/layer5_attention_weight_min": -63.076744079589844,
      "activations/layer6_attention_weight_max": 43.4470329284668,
      "activations/layer6_attention_weight_min": -47.84127426147461,
      "activations/layer7_attention_weight_max": 84.60356903076172,
      "activations/layer7_attention_weight_min": -83.37773895263672,
      "activations/layer8_attention_weight_max": 37.28263854980469,
      "activations/layer8_attention_weight_min": -38.64965057373047,
      "activations/layer9_attention_weight_max": 31.47785758972168,
      "activations/layer9_attention_weight_min": -34.15916061401367,
      "epoch": 10.01,
      "learning_rate": 8.634393939393938e-05,
      "loss": 2.7897,
      "step": 172200
    },
    {
      "activations/layer0_attention_weight_max": 16.655893325805664,
      "activations/layer0_attention_weight_min": -12.365638732910156,
      "activations/layer10_attention_weight_max": 32.82972717285156,
      "activations/layer10_attention_weight_min": -30.296249389648438,
      "activations/layer11_attention_weight_max": 33.538169860839844,
      "activations/layer11_attention_weight_min": -30.056835174560547,
      "activations/layer12_attention_weight_max": 40.9077033996582,
      "activations/layer12_attention_weight_min": -25.244474411010742,
      "activations/layer13_attention_weight_max": 46.32322311401367,
      "activations/layer13_attention_weight_min": -33.67270278930664,
      "activations/layer14_attention_weight_max": 50.419342041015625,
      "activations/layer14_attention_weight_min": -39.6956672668457,
      "activations/layer15_attention_weight_max": 34.45244598388672,
      "activations/layer15_attention_weight_min": -31.957578659057617,
      "activations/layer16_attention_weight_max": 31.24538230895996,
      "activations/layer16_attention_weight_min": -27.882097244262695,
      "activations/layer17_attention_weight_max": 31.814504623413086,
      "activations/layer17_attention_weight_min": -25.85923957824707,
      "activations/layer18_attention_weight_max": 29.46527671813965,
      "activations/layer18_attention_weight_min": -22.526540756225586,
      "activations/layer19_attention_weight_max": 37.673954010009766,
      "activations/layer19_attention_weight_min": -30.235692977905273,
      "activations/layer1_attention_weight_max": 16.381214141845703,
      "activations/layer1_attention_weight_min": -14.449897766113281,
      "activations/layer20_attention_weight_max": 29.031198501586914,
      "activations/layer20_attention_weight_min": -23.969642639160156,
      "activations/layer21_attention_weight_max": 26.942289352416992,
      "activations/layer21_attention_weight_min": -22.356496810913086,
      "activations/layer22_attention_weight_max": 41.433982849121094,
      "activations/layer22_attention_weight_min": -27.53990936279297,
      "activations/layer23_attention_weight_max": 40.47673034667969,
      "activations/layer23_attention_weight_min": -24.018917083740234,
      "activations/layer2_attention_weight_max": 31.933874130249023,
      "activations/layer2_attention_weight_min": -33.097225189208984,
      "activations/layer3_attention_weight_max": 91.35662078857422,
      "activations/layer3_attention_weight_min": -96.37501525878906,
      "activations/layer4_attention_weight_max": 110.56453704833984,
      "activations/layer4_attention_weight_min": -113.69978332519531,
      "activations/layer5_attention_weight_max": 52.095802307128906,
      "activations/layer5_attention_weight_min": -63.836517333984375,
      "activations/layer6_attention_weight_max": 44.598472595214844,
      "activations/layer6_attention_weight_min": -47.48603439331055,
      "activations/layer7_attention_weight_max": 91.28516387939453,
      "activations/layer7_attention_weight_min": -91.90254211425781,
      "activations/layer8_attention_weight_max": 37.8603401184082,
      "activations/layer8_attention_weight_min": -40.7841682434082,
      "activations/layer9_attention_weight_max": 31.73150062561035,
      "activations/layer9_attention_weight_min": -32.26339340209961,
      "epoch": 10.01,
      "learning_rate": 8.6325e-05,
      "loss": 2.7911,
      "step": 172250
    },
    {
      "activations/layer0_attention_weight_max": 15.76729679107666,
      "activations/layer0_attention_weight_min": -12.336705207824707,
      "activations/layer10_attention_weight_max": 35.395843505859375,
      "activations/layer10_attention_weight_min": -32.268768310546875,
      "activations/layer11_attention_weight_max": 31.195953369140625,
      "activations/layer11_attention_weight_min": -30.478343963623047,
      "activations/layer12_attention_weight_max": 23.688230514526367,
      "activations/layer12_attention_weight_min": -24.404754638671875,
      "activations/layer13_attention_weight_max": 38.53874206542969,
      "activations/layer13_attention_weight_min": -34.38920211791992,
      "activations/layer14_attention_weight_max": 42.541847229003906,
      "activations/layer14_attention_weight_min": -39.86973190307617,
      "activations/layer15_attention_weight_max": 35.39309310913086,
      "activations/layer15_attention_weight_min": -30.34849739074707,
      "activations/layer16_attention_weight_max": 34.809814453125,
      "activations/layer16_attention_weight_min": -27.16325569152832,
      "activations/layer17_attention_weight_max": 35.22423553466797,
      "activations/layer17_attention_weight_min": -25.3760986328125,
      "activations/layer18_attention_weight_max": 32.01359558105469,
      "activations/layer18_attention_weight_min": -22.46385383605957,
      "activations/layer19_attention_weight_max": 36.463191986083984,
      "activations/layer19_attention_weight_min": -29.630542755126953,
      "activations/layer1_attention_weight_max": 16.60759735107422,
      "activations/layer1_attention_weight_min": -15.761161804199219,
      "activations/layer20_attention_weight_max": 31.272308349609375,
      "activations/layer20_attention_weight_min": -24.010576248168945,
      "activations/layer21_attention_weight_max": 28.66209602355957,
      "activations/layer21_attention_weight_min": -22.016071319580078,
      "activations/layer22_attention_weight_max": 44.19525909423828,
      "activations/layer22_attention_weight_min": -28.518104553222656,
      "activations/layer23_attention_weight_max": 33.70988464355469,
      "activations/layer23_attention_weight_min": -24.96426010131836,
      "activations/layer2_attention_weight_max": 32.13178253173828,
      "activations/layer2_attention_weight_min": -32.71158981323242,
      "activations/layer3_attention_weight_max": 95.66783142089844,
      "activations/layer3_attention_weight_min": -97.2972412109375,
      "activations/layer4_attention_weight_max": 114.06111907958984,
      "activations/layer4_attention_weight_min": -123.57337951660156,
      "activations/layer5_attention_weight_max": 53.02212142944336,
      "activations/layer5_attention_weight_min": -64.78189849853516,
      "activations/layer6_attention_weight_max": 46.653648376464844,
      "activations/layer6_attention_weight_min": -49.9170036315918,
      "activations/layer7_attention_weight_max": 89.91798400878906,
      "activations/layer7_attention_weight_min": -92.2890396118164,
      "activations/layer8_attention_weight_max": 39.53961181640625,
      "activations/layer8_attention_weight_min": -42.88435745239258,
      "activations/layer9_attention_weight_max": 34.09666061401367,
      "activations/layer9_attention_weight_min": -33.821712493896484,
      "epoch": 10.01,
      "learning_rate": 8.630606060606061e-05,
      "loss": 2.7785,
      "step": 172300
    },
    {
      "activations/layer0_attention_weight_max": 15.808100700378418,
      "activations/layer0_attention_weight_min": -12.925154685974121,
      "activations/layer10_attention_weight_max": 32.591556549072266,
      "activations/layer10_attention_weight_min": -31.631927490234375,
      "activations/layer11_attention_weight_max": 35.49736022949219,
      "activations/layer11_attention_weight_min": -32.27741622924805,
      "activations/layer12_attention_weight_max": 30.367637634277344,
      "activations/layer12_attention_weight_min": -24.917997360229492,
      "activations/layer13_attention_weight_max": 37.754268646240234,
      "activations/layer13_attention_weight_min": -34.86164474487305,
      "activations/layer14_attention_weight_max": 46.085716247558594,
      "activations/layer14_attention_weight_min": -40.597965240478516,
      "activations/layer15_attention_weight_max": 36.42483901977539,
      "activations/layer15_attention_weight_min": -30.422691345214844,
      "activations/layer16_attention_weight_max": 32.93168640136719,
      "activations/layer16_attention_weight_min": -28.410057067871094,
      "activations/layer17_attention_weight_max": 33.7658805847168,
      "activations/layer17_attention_weight_min": -25.257333755493164,
      "activations/layer18_attention_weight_max": 32.22063064575195,
      "activations/layer18_attention_weight_min": -23.043119430541992,
      "activations/layer19_attention_weight_max": 37.27643966674805,
      "activations/layer19_attention_weight_min": -28.838010787963867,
      "activations/layer1_attention_weight_max": 17.194257736206055,
      "activations/layer1_attention_weight_min": -16.50051498413086,
      "activations/layer20_attention_weight_max": 31.6900691986084,
      "activations/layer20_attention_weight_min": -26.119888305664062,
      "activations/layer21_attention_weight_max": 32.0174446105957,
      "activations/layer21_attention_weight_min": -23.25136375427246,
      "activations/layer22_attention_weight_max": 49.03803253173828,
      "activations/layer22_attention_weight_min": -28.6856689453125,
      "activations/layer23_attention_weight_max": 39.066139221191406,
      "activations/layer23_attention_weight_min": -22.89520835876465,
      "activations/layer2_attention_weight_max": 33.06389236450195,
      "activations/layer2_attention_weight_min": -33.545501708984375,
      "activations/layer3_attention_weight_max": 92.15446472167969,
      "activations/layer3_attention_weight_min": -94.40518188476562,
      "activations/layer4_attention_weight_max": 110.30805969238281,
      "activations/layer4_attention_weight_min": -112.84209442138672,
      "activations/layer5_attention_weight_max": 55.95453643798828,
      "activations/layer5_attention_weight_min": -64.22340393066406,
      "activations/layer6_attention_weight_max": 46.6738166809082,
      "activations/layer6_attention_weight_min": -48.66606903076172,
      "activations/layer7_attention_weight_max": 86.86473083496094,
      "activations/layer7_attention_weight_min": -83.98578643798828,
      "activations/layer8_attention_weight_max": 41.846317291259766,
      "activations/layer8_attention_weight_min": -41.56146240234375,
      "activations/layer9_attention_weight_max": 32.860557556152344,
      "activations/layer9_attention_weight_min": -32.27614974975586,
      "epoch": 10.01,
      "learning_rate": 8.62871212121212e-05,
      "loss": 2.7811,
      "step": 172350
    },
    {
      "activations/layer0_attention_weight_max": 15.806364059448242,
      "activations/layer0_attention_weight_min": -12.02672004699707,
      "activations/layer10_attention_weight_max": 33.8272590637207,
      "activations/layer10_attention_weight_min": -35.72816848754883,
      "activations/layer11_attention_weight_max": 32.73622131347656,
      "activations/layer11_attention_weight_min": -30.845443725585938,
      "activations/layer12_attention_weight_max": 33.33669662475586,
      "activations/layer12_attention_weight_min": -27.243362426757812,
      "activations/layer13_attention_weight_max": 38.76247787475586,
      "activations/layer13_attention_weight_min": -34.906856536865234,
      "activations/layer14_attention_weight_max": 47.6792106628418,
      "activations/layer14_attention_weight_min": -44.57001495361328,
      "activations/layer15_attention_weight_max": 35.70087432861328,
      "activations/layer15_attention_weight_min": -30.243778228759766,
      "activations/layer16_attention_weight_max": 29.662050247192383,
      "activations/layer16_attention_weight_min": -27.50305938720703,
      "activations/layer17_attention_weight_max": 33.19527053833008,
      "activations/layer17_attention_weight_min": -25.59971046447754,
      "activations/layer18_attention_weight_max": 33.9531135559082,
      "activations/layer18_attention_weight_min": -23.362335205078125,
      "activations/layer19_attention_weight_max": 38.8913688659668,
      "activations/layer19_attention_weight_min": -29.20615577697754,
      "activations/layer1_attention_weight_max": 16.413097381591797,
      "activations/layer1_attention_weight_min": -14.933307647705078,
      "activations/layer20_attention_weight_max": 29.4913387298584,
      "activations/layer20_attention_weight_min": -25.317148208618164,
      "activations/layer21_attention_weight_max": 30.474828720092773,
      "activations/layer21_attention_weight_min": -22.89406394958496,
      "activations/layer22_attention_weight_max": 48.76041793823242,
      "activations/layer22_attention_weight_min": -29.43407440185547,
      "activations/layer23_attention_weight_max": 39.33228302001953,
      "activations/layer23_attention_weight_min": -25.566680908203125,
      "activations/layer2_attention_weight_max": 32.471038818359375,
      "activations/layer2_attention_weight_min": -33.34424591064453,
      "activations/layer3_attention_weight_max": 91.802001953125,
      "activations/layer3_attention_weight_min": -94.78834533691406,
      "activations/layer4_attention_weight_max": 113.24337005615234,
      "activations/layer4_attention_weight_min": -118.14210510253906,
      "activations/layer5_attention_weight_max": 55.532936096191406,
      "activations/layer5_attention_weight_min": -65.62637329101562,
      "activations/layer6_attention_weight_max": 49.054100036621094,
      "activations/layer6_attention_weight_min": -53.63111877441406,
      "activations/layer7_attention_weight_max": 93.14290618896484,
      "activations/layer7_attention_weight_min": -93.62310028076172,
      "activations/layer8_attention_weight_max": 42.5551643371582,
      "activations/layer8_attention_weight_min": -42.45320129394531,
      "activations/layer9_attention_weight_max": 35.3031120300293,
      "activations/layer9_attention_weight_min": -36.16225814819336,
      "epoch": 10.02,
      "learning_rate": 8.626856060606059e-05,
      "loss": 2.7766,
      "step": 172400
    },
    {
      "activations/layer0_attention_weight_max": 15.12403392791748,
      "activations/layer0_attention_weight_min": -11.356313705444336,
      "activations/layer10_attention_weight_max": 42.478790283203125,
      "activations/layer10_attention_weight_min": -39.58734130859375,
      "activations/layer11_attention_weight_max": 45.65133285522461,
      "activations/layer11_attention_weight_min": -42.27940368652344,
      "activations/layer12_attention_weight_max": 35.806434631347656,
      "activations/layer12_attention_weight_min": -26.120018005371094,
      "activations/layer13_attention_weight_max": 44.89440155029297,
      "activations/layer13_attention_weight_min": -34.94179916381836,
      "activations/layer14_attention_weight_max": 54.596656799316406,
      "activations/layer14_attention_weight_min": -38.725929260253906,
      "activations/layer15_attention_weight_max": 39.772552490234375,
      "activations/layer15_attention_weight_min": -31.195648193359375,
      "activations/layer16_attention_weight_max": 39.3015251159668,
      "activations/layer16_attention_weight_min": -28.117416381835938,
      "activations/layer17_attention_weight_max": 42.56159973144531,
      "activations/layer17_attention_weight_min": -26.806066513061523,
      "activations/layer18_attention_weight_max": 37.447872161865234,
      "activations/layer18_attention_weight_min": -25.62738037109375,
      "activations/layer19_attention_weight_max": 35.88258743286133,
      "activations/layer19_attention_weight_min": -30.041030883789062,
      "activations/layer1_attention_weight_max": 18.212310791015625,
      "activations/layer1_attention_weight_min": -15.808182716369629,
      "activations/layer20_attention_weight_max": 32.666568756103516,
      "activations/layer20_attention_weight_min": -24.809280395507812,
      "activations/layer21_attention_weight_max": 32.02920913696289,
      "activations/layer21_attention_weight_min": -23.185312271118164,
      "activations/layer22_attention_weight_max": 46.943275451660156,
      "activations/layer22_attention_weight_min": -29.448720932006836,
      "activations/layer23_attention_weight_max": 35.917510986328125,
      "activations/layer23_attention_weight_min": -24.738563537597656,
      "activations/layer2_attention_weight_max": 34.90049362182617,
      "activations/layer2_attention_weight_min": -34.596435546875,
      "activations/layer3_attention_weight_max": 93.26554107666016,
      "activations/layer3_attention_weight_min": -95.68329620361328,
      "activations/layer4_attention_weight_max": 112.67237854003906,
      "activations/layer4_attention_weight_min": -115.84561920166016,
      "activations/layer5_attention_weight_max": 56.19038009643555,
      "activations/layer5_attention_weight_min": -62.814544677734375,
      "activations/layer6_attention_weight_max": 47.21131896972656,
      "activations/layer6_attention_weight_min": -48.87621307373047,
      "activations/layer7_attention_weight_max": 104.77367401123047,
      "activations/layer7_attention_weight_min": -91.88473510742188,
      "activations/layer8_attention_weight_max": 48.16608428955078,
      "activations/layer8_attention_weight_min": -43.81930923461914,
      "activations/layer9_attention_weight_max": 43.336326599121094,
      "activations/layer9_attention_weight_min": -38.14780807495117,
      "epoch": 10.02,
      "learning_rate": 8.62496212121212e-05,
      "loss": 2.7731,
      "step": 172450
    },
    {
      "activations/layer0_attention_weight_max": 16.008798599243164,
      "activations/layer0_attention_weight_min": -12.483031272888184,
      "activations/layer10_attention_weight_max": 30.341703414916992,
      "activations/layer10_attention_weight_min": -31.839767456054688,
      "activations/layer11_attention_weight_max": 31.792919158935547,
      "activations/layer11_attention_weight_min": -29.97793960571289,
      "activations/layer12_attention_weight_max": 23.546964645385742,
      "activations/layer12_attention_weight_min": -26.573057174682617,
      "activations/layer13_attention_weight_max": 36.51669692993164,
      "activations/layer13_attention_weight_min": -34.19518280029297,
      "activations/layer14_attention_weight_max": 43.22243881225586,
      "activations/layer14_attention_weight_min": -43.70724868774414,
      "activations/layer15_attention_weight_max": 33.265533447265625,
      "activations/layer15_attention_weight_min": -30.847932815551758,
      "activations/layer16_attention_weight_max": 28.55829429626465,
      "activations/layer16_attention_weight_min": -28.8646240234375,
      "activations/layer17_attention_weight_max": 29.526575088500977,
      "activations/layer17_attention_weight_min": -24.24095916748047,
      "activations/layer18_attention_weight_max": 28.15838623046875,
      "activations/layer18_attention_weight_min": -23.17963218688965,
      "activations/layer19_attention_weight_max": 33.990867614746094,
      "activations/layer19_attention_weight_min": -29.427152633666992,
      "activations/layer1_attention_weight_max": 16.18337059020996,
      "activations/layer1_attention_weight_min": -15.704955101013184,
      "activations/layer20_attention_weight_max": 29.209081649780273,
      "activations/layer20_attention_weight_min": -21.95673942565918,
      "activations/layer21_attention_weight_max": 27.902421951293945,
      "activations/layer21_attention_weight_min": -21.325653076171875,
      "activations/layer22_attention_weight_max": 45.348472595214844,
      "activations/layer22_attention_weight_min": -28.541522979736328,
      "activations/layer23_attention_weight_max": 33.59693145751953,
      "activations/layer23_attention_weight_min": -25.419034957885742,
      "activations/layer2_attention_weight_max": 31.830724716186523,
      "activations/layer2_attention_weight_min": -34.292198181152344,
      "activations/layer3_attention_weight_max": 90.96271514892578,
      "activations/layer3_attention_weight_min": -95.77720642089844,
      "activations/layer4_attention_weight_max": 109.5238037109375,
      "activations/layer4_attention_weight_min": -116.08556365966797,
      "activations/layer5_attention_weight_max": 52.334434509277344,
      "activations/layer5_attention_weight_min": -61.58549499511719,
      "activations/layer6_attention_weight_max": 45.23924255371094,
      "activations/layer6_attention_weight_min": -48.180057525634766,
      "activations/layer7_attention_weight_max": 84.20921325683594,
      "activations/layer7_attention_weight_min": -88.88764953613281,
      "activations/layer8_attention_weight_max": 40.03089904785156,
      "activations/layer8_attention_weight_min": -37.52164077758789,
      "activations/layer9_attention_weight_max": 31.80495262145996,
      "activations/layer9_attention_weight_min": -31.991987228393555,
      "epoch": 10.02,
      "learning_rate": 8.623068181818182e-05,
      "loss": 2.7683,
      "step": 172500
    },
    {
      "activations/layer0_attention_weight_max": 15.691972732543945,
      "activations/layer0_attention_weight_min": -11.435567855834961,
      "activations/layer10_attention_weight_max": 34.639278411865234,
      "activations/layer10_attention_weight_min": -33.23732376098633,
      "activations/layer11_attention_weight_max": 34.69374084472656,
      "activations/layer11_attention_weight_min": -33.2219123840332,
      "activations/layer12_attention_weight_max": 22.87973403930664,
      "activations/layer12_attention_weight_min": -26.203630447387695,
      "activations/layer13_attention_weight_max": 36.865821838378906,
      "activations/layer13_attention_weight_min": -32.33509826660156,
      "activations/layer14_attention_weight_max": 42.653602600097656,
      "activations/layer14_attention_weight_min": -37.13532638549805,
      "activations/layer15_attention_weight_max": 35.65441131591797,
      "activations/layer15_attention_weight_min": -30.332847595214844,
      "activations/layer16_attention_weight_max": 29.097881317138672,
      "activations/layer16_attention_weight_min": -28.218486785888672,
      "activations/layer17_attention_weight_max": 30.83659553527832,
      "activations/layer17_attention_weight_min": -25.08230209350586,
      "activations/layer18_attention_weight_max": 30.426815032958984,
      "activations/layer18_attention_weight_min": -23.82888412475586,
      "activations/layer19_attention_weight_max": 32.383968353271484,
      "activations/layer19_attention_weight_min": -31.706594467163086,
      "activations/layer1_attention_weight_max": 16.20906639099121,
      "activations/layer1_attention_weight_min": -15.013127326965332,
      "activations/layer20_attention_weight_max": 27.379863739013672,
      "activations/layer20_attention_weight_min": -24.82017707824707,
      "activations/layer21_attention_weight_max": 27.718069076538086,
      "activations/layer21_attention_weight_min": -22.445289611816406,
      "activations/layer22_attention_weight_max": 44.801307678222656,
      "activations/layer22_attention_weight_min": -30.098236083984375,
      "activations/layer23_attention_weight_max": 35.55200958251953,
      "activations/layer23_attention_weight_min": -24.650917053222656,
      "activations/layer2_attention_weight_max": 32.07090377807617,
      "activations/layer2_attention_weight_min": -35.096073150634766,
      "activations/layer3_attention_weight_max": 97.93780517578125,
      "activations/layer3_attention_weight_min": -102.38902282714844,
      "activations/layer4_attention_weight_max": 118.1207504272461,
      "activations/layer4_attention_weight_min": -117.74315643310547,
      "activations/layer5_attention_weight_max": 55.49709701538086,
      "activations/layer5_attention_weight_min": -59.89406967163086,
      "activations/layer6_attention_weight_max": 45.744625091552734,
      "activations/layer6_attention_weight_min": -50.73490524291992,
      "activations/layer7_attention_weight_max": 91.0655746459961,
      "activations/layer7_attention_weight_min": -90.14781951904297,
      "activations/layer8_attention_weight_max": 41.32694625854492,
      "activations/layer8_attention_weight_min": -42.71553421020508,
      "activations/layer9_attention_weight_max": 36.23569869995117,
      "activations/layer9_attention_weight_min": -33.80697250366211,
      "epoch": 10.03,
      "learning_rate": 8.621174242424241e-05,
      "loss": 2.791,
      "step": 172550
    },
    {
      "activations/layer0_attention_weight_max": 15.504444122314453,
      "activations/layer0_attention_weight_min": -12.724709510803223,
      "activations/layer10_attention_weight_max": 31.636709213256836,
      "activations/layer10_attention_weight_min": -33.44471740722656,
      "activations/layer11_attention_weight_max": 32.36017990112305,
      "activations/layer11_attention_weight_min": -29.780988693237305,
      "activations/layer12_attention_weight_max": 23.024616241455078,
      "activations/layer12_attention_weight_min": -25.207138061523438,
      "activations/layer13_attention_weight_max": 40.19684982299805,
      "activations/layer13_attention_weight_min": -34.48908615112305,
      "activations/layer14_attention_weight_max": 43.9234619140625,
      "activations/layer14_attention_weight_min": -36.5928840637207,
      "activations/layer15_attention_weight_max": 36.19575500488281,
      "activations/layer15_attention_weight_min": -30.019088745117188,
      "activations/layer16_attention_weight_max": 31.69718360900879,
      "activations/layer16_attention_weight_min": -28.226865768432617,
      "activations/layer17_attention_weight_max": 30.451196670532227,
      "activations/layer17_attention_weight_min": -26.29789161682129,
      "activations/layer18_attention_weight_max": 29.613779067993164,
      "activations/layer18_attention_weight_min": -22.63690757751465,
      "activations/layer19_attention_weight_max": 33.26027297973633,
      "activations/layer19_attention_weight_min": -32.44845199584961,
      "activations/layer1_attention_weight_max": 16.038389205932617,
      "activations/layer1_attention_weight_min": -14.826807975769043,
      "activations/layer20_attention_weight_max": 27.802330017089844,
      "activations/layer20_attention_weight_min": -25.924976348876953,
      "activations/layer21_attention_weight_max": 27.990976333618164,
      "activations/layer21_attention_weight_min": -23.678983688354492,
      "activations/layer22_attention_weight_max": 41.82105255126953,
      "activations/layer22_attention_weight_min": -31.04956817626953,
      "activations/layer23_attention_weight_max": 34.4300651550293,
      "activations/layer23_attention_weight_min": -24.43875503540039,
      "activations/layer2_attention_weight_max": 31.93415069580078,
      "activations/layer2_attention_weight_min": -33.28765869140625,
      "activations/layer3_attention_weight_max": 96.33086395263672,
      "activations/layer3_attention_weight_min": -96.94823455810547,
      "activations/layer4_attention_weight_max": 111.04859924316406,
      "activations/layer4_attention_weight_min": -109.82965087890625,
      "activations/layer5_attention_weight_max": 52.92197799682617,
      "activations/layer5_attention_weight_min": -60.1153678894043,
      "activations/layer6_attention_weight_max": 46.39488220214844,
      "activations/layer6_attention_weight_min": -49.56913375854492,
      "activations/layer7_attention_weight_max": 91.49071502685547,
      "activations/layer7_attention_weight_min": -84.717529296875,
      "activations/layer8_attention_weight_max": 40.9603385925293,
      "activations/layer8_attention_weight_min": -41.40053176879883,
      "activations/layer9_attention_weight_max": 33.37529754638672,
      "activations/layer9_attention_weight_min": -32.42312240600586,
      "epoch": 10.03,
      "learning_rate": 8.619280303030302e-05,
      "loss": 2.799,
      "step": 172600
    },
    {
      "activations/layer0_attention_weight_max": 15.249153137207031,
      "activations/layer0_attention_weight_min": -12.147323608398438,
      "activations/layer10_attention_weight_max": 33.2314453125,
      "activations/layer10_attention_weight_min": -31.323637008666992,
      "activations/layer11_attention_weight_max": 32.21116638183594,
      "activations/layer11_attention_weight_min": -31.908292770385742,
      "activations/layer12_attention_weight_max": 24.7457332611084,
      "activations/layer12_attention_weight_min": -24.00589370727539,
      "activations/layer13_attention_weight_max": 38.87049865722656,
      "activations/layer13_attention_weight_min": -34.596988677978516,
      "activations/layer14_attention_weight_max": 45.721397399902344,
      "activations/layer14_attention_weight_min": -36.334678649902344,
      "activations/layer15_attention_weight_max": 37.754905700683594,
      "activations/layer15_attention_weight_min": -31.93610382080078,
      "activations/layer16_attention_weight_max": 36.90215301513672,
      "activations/layer16_attention_weight_min": -29.035491943359375,
      "activations/layer17_attention_weight_max": 37.044639587402344,
      "activations/layer17_attention_weight_min": -26.687744140625,
      "activations/layer18_attention_weight_max": 41.390113830566406,
      "activations/layer18_attention_weight_min": -22.9304256439209,
      "activations/layer19_attention_weight_max": 44.08889389038086,
      "activations/layer19_attention_weight_min": -31.889551162719727,
      "activations/layer1_attention_weight_max": 17.994882583618164,
      "activations/layer1_attention_weight_min": -14.801278114318848,
      "activations/layer20_attention_weight_max": 36.387855529785156,
      "activations/layer20_attention_weight_min": -25.412179946899414,
      "activations/layer21_attention_weight_max": 41.607730865478516,
      "activations/layer21_attention_weight_min": -23.66330909729004,
      "activations/layer22_attention_weight_max": 49.55909729003906,
      "activations/layer22_attention_weight_min": -28.278841018676758,
      "activations/layer23_attention_weight_max": 34.56816101074219,
      "activations/layer23_attention_weight_min": -24.911911010742188,
      "activations/layer2_attention_weight_max": 30.661273956298828,
      "activations/layer2_attention_weight_min": -31.980937957763672,
      "activations/layer3_attention_weight_max": 90.0687255859375,
      "activations/layer3_attention_weight_min": -94.78539276123047,
      "activations/layer4_attention_weight_max": 107.69233703613281,
      "activations/layer4_attention_weight_min": -110.93731689453125,
      "activations/layer5_attention_weight_max": 52.31838607788086,
      "activations/layer5_attention_weight_min": -60.922447204589844,
      "activations/layer6_attention_weight_max": 44.858829498291016,
      "activations/layer6_attention_weight_min": -46.97150802612305,
      "activations/layer7_attention_weight_max": 85.80077362060547,
      "activations/layer7_attention_weight_min": -84.73291015625,
      "activations/layer8_attention_weight_max": 38.172508239746094,
      "activations/layer8_attention_weight_min": -39.28083801269531,
      "activations/layer9_attention_weight_max": 33.82331466674805,
      "activations/layer9_attention_weight_min": -32.39529800415039,
      "epoch": 10.03,
      "learning_rate": 8.617386363636363e-05,
      "loss": 2.784,
      "step": 172650
    },
    {
      "activations/layer0_attention_weight_max": 15.969064712524414,
      "activations/layer0_attention_weight_min": -12.473589897155762,
      "activations/layer10_attention_weight_max": 32.557762145996094,
      "activations/layer10_attention_weight_min": -32.1501350402832,
      "activations/layer11_attention_weight_max": 31.994535446166992,
      "activations/layer11_attention_weight_min": -29.68524932861328,
      "activations/layer12_attention_weight_max": 26.12187957763672,
      "activations/layer12_attention_weight_min": -23.954605102539062,
      "activations/layer13_attention_weight_max": 40.173919677734375,
      "activations/layer13_attention_weight_min": -32.91910171508789,
      "activations/layer14_attention_weight_max": 48.988155364990234,
      "activations/layer14_attention_weight_min": -37.63216781616211,
      "activations/layer15_attention_weight_max": 37.880157470703125,
      "activations/layer15_attention_weight_min": -29.9599552154541,
      "activations/layer16_attention_weight_max": 30.947139739990234,
      "activations/layer16_attention_weight_min": -28.161880493164062,
      "activations/layer17_attention_weight_max": 29.772274017333984,
      "activations/layer17_attention_weight_min": -25.378541946411133,
      "activations/layer18_attention_weight_max": 29.832298278808594,
      "activations/layer18_attention_weight_min": -22.888113021850586,
      "activations/layer19_attention_weight_max": 34.288326263427734,
      "activations/layer19_attention_weight_min": -29.21976089477539,
      "activations/layer1_attention_weight_max": 16.470823287963867,
      "activations/layer1_attention_weight_min": -14.843659400939941,
      "activations/layer20_attention_weight_max": 30.88835334777832,
      "activations/layer20_attention_weight_min": -24.538366317749023,
      "activations/layer21_attention_weight_max": 28.612913131713867,
      "activations/layer21_attention_weight_min": -22.994285583496094,
      "activations/layer22_attention_weight_max": 45.50523376464844,
      "activations/layer22_attention_weight_min": -27.409679412841797,
      "activations/layer23_attention_weight_max": 34.896522521972656,
      "activations/layer23_attention_weight_min": -25.05590057373047,
      "activations/layer2_attention_weight_max": 34.33921813964844,
      "activations/layer2_attention_weight_min": -32.91889190673828,
      "activations/layer3_attention_weight_max": 89.96925354003906,
      "activations/layer3_attention_weight_min": -95.70671081542969,
      "activations/layer4_attention_weight_max": 111.64391326904297,
      "activations/layer4_attention_weight_min": -116.4013442993164,
      "activations/layer5_attention_weight_max": 50.4727783203125,
      "activations/layer5_attention_weight_min": -58.14985656738281,
      "activations/layer6_attention_weight_max": 47.29359817504883,
      "activations/layer6_attention_weight_min": -48.22767639160156,
      "activations/layer7_attention_weight_max": 87.51719665527344,
      "activations/layer7_attention_weight_min": -90.85094451904297,
      "activations/layer8_attention_weight_max": 42.41096878051758,
      "activations/layer8_attention_weight_min": -40.0469856262207,
      "activations/layer9_attention_weight_max": 34.70964050292969,
      "activations/layer9_attention_weight_min": -32.7140007019043,
      "epoch": 10.03,
      "learning_rate": 8.615492424242423e-05,
      "loss": 2.7774,
      "step": 172700
    },
    {
      "activations/layer0_attention_weight_max": 15.515192031860352,
      "activations/layer0_attention_weight_min": -12.775914192199707,
      "activations/layer10_attention_weight_max": 34.75132369995117,
      "activations/layer10_attention_weight_min": -34.55667495727539,
      "activations/layer11_attention_weight_max": 37.41797637939453,
      "activations/layer11_attention_weight_min": -35.6334228515625,
      "activations/layer12_attention_weight_max": 34.297935485839844,
      "activations/layer12_attention_weight_min": -31.181798934936523,
      "activations/layer13_attention_weight_max": 47.051753997802734,
      "activations/layer13_attention_weight_min": -35.721405029296875,
      "activations/layer14_attention_weight_max": 58.52446365356445,
      "activations/layer14_attention_weight_min": -41.46043014526367,
      "activations/layer15_attention_weight_max": 42.19245910644531,
      "activations/layer15_attention_weight_min": -31.021963119506836,
      "activations/layer16_attention_weight_max": 40.6230354309082,
      "activations/layer16_attention_weight_min": -28.2674503326416,
      "activations/layer17_attention_weight_max": 37.570213317871094,
      "activations/layer17_attention_weight_min": -24.48191261291504,
      "activations/layer18_attention_weight_max": 41.265045166015625,
      "activations/layer18_attention_weight_min": -22.50968360900879,
      "activations/layer19_attention_weight_max": 51.88871383666992,
      "activations/layer19_attention_weight_min": -31.726770401000977,
      "activations/layer1_attention_weight_max": 17.55292510986328,
      "activations/layer1_attention_weight_min": -15.156095504760742,
      "activations/layer20_attention_weight_max": 41.56741714477539,
      "activations/layer20_attention_weight_min": -24.245237350463867,
      "activations/layer21_attention_weight_max": 41.51240539550781,
      "activations/layer21_attention_weight_min": -23.165842056274414,
      "activations/layer22_attention_weight_max": 60.276092529296875,
      "activations/layer22_attention_weight_min": -29.50339698791504,
      "activations/layer23_attention_weight_max": 48.77322006225586,
      "activations/layer23_attention_weight_min": -25.12149429321289,
      "activations/layer2_attention_weight_max": 37.42096710205078,
      "activations/layer2_attention_weight_min": -35.638084411621094,
      "activations/layer3_attention_weight_max": 106.69022369384766,
      "activations/layer3_attention_weight_min": -101.53902435302734,
      "activations/layer4_attention_weight_max": 125.61358642578125,
      "activations/layer4_attention_weight_min": -111.92486572265625,
      "activations/layer5_attention_weight_max": 55.40713119506836,
      "activations/layer5_attention_weight_min": -61.40260314941406,
      "activations/layer6_attention_weight_max": 48.83622741699219,
      "activations/layer6_attention_weight_min": -48.22188949584961,
      "activations/layer7_attention_weight_max": 96.95907592773438,
      "activations/layer7_attention_weight_min": -90.12427520751953,
      "activations/layer8_attention_weight_max": 44.229026794433594,
      "activations/layer8_attention_weight_min": -41.74485397338867,
      "activations/layer9_attention_weight_max": 34.140933990478516,
      "activations/layer9_attention_weight_min": -37.031307220458984,
      "epoch": 10.04,
      "learning_rate": 8.613598484848484e-05,
      "loss": 2.7715,
      "step": 172750
    },
    {
      "activations/layer0_attention_weight_max": 15.839560508728027,
      "activations/layer0_attention_weight_min": -12.675590515136719,
      "activations/layer10_attention_weight_max": 32.976959228515625,
      "activations/layer10_attention_weight_min": -32.496944427490234,
      "activations/layer11_attention_weight_max": 31.93600082397461,
      "activations/layer11_attention_weight_min": -30.787841796875,
      "activations/layer12_attention_weight_max": 25.013891220092773,
      "activations/layer12_attention_weight_min": -34.2669677734375,
      "activations/layer13_attention_weight_max": 35.775917053222656,
      "activations/layer13_attention_weight_min": -33.247798919677734,
      "activations/layer14_attention_weight_max": 43.738258361816406,
      "activations/layer14_attention_weight_min": -36.906307220458984,
      "activations/layer15_attention_weight_max": 33.36296844482422,
      "activations/layer15_attention_weight_min": -30.694480895996094,
      "activations/layer16_attention_weight_max": 29.489931106567383,
      "activations/layer16_attention_weight_min": -28.047283172607422,
      "activations/layer17_attention_weight_max": 31.78453254699707,
      "activations/layer17_attention_weight_min": -25.98802947998047,
      "activations/layer18_attention_weight_max": 31.834270477294922,
      "activations/layer18_attention_weight_min": -24.033607482910156,
      "activations/layer19_attention_weight_max": 34.82602310180664,
      "activations/layer19_attention_weight_min": -32.210853576660156,
      "activations/layer1_attention_weight_max": 16.709962844848633,
      "activations/layer1_attention_weight_min": -13.633758544921875,
      "activations/layer20_attention_weight_max": 26.711641311645508,
      "activations/layer20_attention_weight_min": -26.644115447998047,
      "activations/layer21_attention_weight_max": 28.764633178710938,
      "activations/layer21_attention_weight_min": -24.53987693786621,
      "activations/layer22_attention_weight_max": 47.6155891418457,
      "activations/layer22_attention_weight_min": -31.282529830932617,
      "activations/layer23_attention_weight_max": 36.249637603759766,
      "activations/layer23_attention_weight_min": -23.65467643737793,
      "activations/layer2_attention_weight_max": 30.872180938720703,
      "activations/layer2_attention_weight_min": -32.9300537109375,
      "activations/layer3_attention_weight_max": 94.66223907470703,
      "activations/layer3_attention_weight_min": -93.70210266113281,
      "activations/layer4_attention_weight_max": 113.217529296875,
      "activations/layer4_attention_weight_min": -110.06575012207031,
      "activations/layer5_attention_weight_max": 52.558135986328125,
      "activations/layer5_attention_weight_min": -61.25299835205078,
      "activations/layer6_attention_weight_max": 44.25392150878906,
      "activations/layer6_attention_weight_min": -47.98381042480469,
      "activations/layer7_attention_weight_max": 85.84864044189453,
      "activations/layer7_attention_weight_min": -89.2045669555664,
      "activations/layer8_attention_weight_max": 38.56248474121094,
      "activations/layer8_attention_weight_min": -40.13648223876953,
      "activations/layer9_attention_weight_max": 32.415401458740234,
      "activations/layer9_attention_weight_min": -33.35916519165039,
      "epoch": 10.04,
      "learning_rate": 8.611704545454544e-05,
      "loss": 2.7835,
      "step": 172800
    },
    {
      "activations/layer0_attention_weight_max": 15.645346641540527,
      "activations/layer0_attention_weight_min": -11.932829856872559,
      "activations/layer10_attention_weight_max": 31.29779624938965,
      "activations/layer10_attention_weight_min": -31.117462158203125,
      "activations/layer11_attention_weight_max": 36.80608367919922,
      "activations/layer11_attention_weight_min": -30.818206787109375,
      "activations/layer12_attention_weight_max": 22.88591194152832,
      "activations/layer12_attention_weight_min": -34.43486404418945,
      "activations/layer13_attention_weight_max": 39.66443634033203,
      "activations/layer13_attention_weight_min": -32.2157096862793,
      "activations/layer14_attention_weight_max": 45.742576599121094,
      "activations/layer14_attention_weight_min": -37.93534851074219,
      "activations/layer15_attention_weight_max": 40.86533737182617,
      "activations/layer15_attention_weight_min": -29.44957160949707,
      "activations/layer16_attention_weight_max": 32.90904998779297,
      "activations/layer16_attention_weight_min": -28.36357307434082,
      "activations/layer17_attention_weight_max": 33.24753189086914,
      "activations/layer17_attention_weight_min": -24.330888748168945,
      "activations/layer18_attention_weight_max": 32.93349075317383,
      "activations/layer18_attention_weight_min": -22.6394100189209,
      "activations/layer19_attention_weight_max": 36.28055953979492,
      "activations/layer19_attention_weight_min": -34.44273376464844,
      "activations/layer1_attention_weight_max": 17.43572998046875,
      "activations/layer1_attention_weight_min": -14.630627632141113,
      "activations/layer20_attention_weight_max": 32.04933547973633,
      "activations/layer20_attention_weight_min": -24.303665161132812,
      "activations/layer21_attention_weight_max": 33.0391731262207,
      "activations/layer21_attention_weight_min": -25.42098045349121,
      "activations/layer22_attention_weight_max": 48.50913619995117,
      "activations/layer22_attention_weight_min": -29.757259368896484,
      "activations/layer23_attention_weight_max": 36.572898864746094,
      "activations/layer23_attention_weight_min": -26.651756286621094,
      "activations/layer2_attention_weight_max": 34.128360748291016,
      "activations/layer2_attention_weight_min": -32.91168212890625,
      "activations/layer3_attention_weight_max": 97.71843719482422,
      "activations/layer3_attention_weight_min": -98.11133575439453,
      "activations/layer4_attention_weight_max": 114.02186584472656,
      "activations/layer4_attention_weight_min": -112.77291107177734,
      "activations/layer5_attention_weight_max": 53.608604431152344,
      "activations/layer5_attention_weight_min": -67.12959289550781,
      "activations/layer6_attention_weight_max": 45.149898529052734,
      "activations/layer6_attention_weight_min": -47.695152282714844,
      "activations/layer7_attention_weight_max": 95.51155090332031,
      "activations/layer7_attention_weight_min": -94.65543365478516,
      "activations/layer8_attention_weight_max": 39.7687873840332,
      "activations/layer8_attention_weight_min": -43.415279388427734,
      "activations/layer9_attention_weight_max": 32.020938873291016,
      "activations/layer9_attention_weight_min": -38.43137741088867,
      "epoch": 10.04,
      "learning_rate": 8.609810606060606e-05,
      "loss": 2.7941,
      "step": 172850
    },
    {
      "activations/layer0_attention_weight_max": 16.471010208129883,
      "activations/layer0_attention_weight_min": -13.701749801635742,
      "activations/layer10_attention_weight_max": 34.28873062133789,
      "activations/layer10_attention_weight_min": -32.589534759521484,
      "activations/layer11_attention_weight_max": 35.415428161621094,
      "activations/layer11_attention_weight_min": -31.598297119140625,
      "activations/layer12_attention_weight_max": 23.711606979370117,
      "activations/layer12_attention_weight_min": -26.155683517456055,
      "activations/layer13_attention_weight_max": 39.44898223876953,
      "activations/layer13_attention_weight_min": -36.26847457885742,
      "activations/layer14_attention_weight_max": 43.553016662597656,
      "activations/layer14_attention_weight_min": -38.69242477416992,
      "activations/layer15_attention_weight_max": 35.62385559082031,
      "activations/layer15_attention_weight_min": -32.2553596496582,
      "activations/layer16_attention_weight_max": 32.72050094604492,
      "activations/layer16_attention_weight_min": -27.29515266418457,
      "activations/layer17_attention_weight_max": 31.59671401977539,
      "activations/layer17_attention_weight_min": -25.528697967529297,
      "activations/layer18_attention_weight_max": 32.828468322753906,
      "activations/layer18_attention_weight_min": -22.26455307006836,
      "activations/layer19_attention_weight_max": 33.9005241394043,
      "activations/layer19_attention_weight_min": -30.659799575805664,
      "activations/layer1_attention_weight_max": 17.143102645874023,
      "activations/layer1_attention_weight_min": -15.008811950683594,
      "activations/layer20_attention_weight_max": 28.944236755371094,
      "activations/layer20_attention_weight_min": -24.671951293945312,
      "activations/layer21_attention_weight_max": 28.887828826904297,
      "activations/layer21_attention_weight_min": -23.397008895874023,
      "activations/layer22_attention_weight_max": 41.86378479003906,
      "activations/layer22_attention_weight_min": -30.329439163208008,
      "activations/layer23_attention_weight_max": 35.300331115722656,
      "activations/layer23_attention_weight_min": -23.596656799316406,
      "activations/layer2_attention_weight_max": 31.269163131713867,
      "activations/layer2_attention_weight_min": -33.499839782714844,
      "activations/layer3_attention_weight_max": 90.89887237548828,
      "activations/layer3_attention_weight_min": -93.41202545166016,
      "activations/layer4_attention_weight_max": 108.75477600097656,
      "activations/layer4_attention_weight_min": -112.77254486083984,
      "activations/layer5_attention_weight_max": 53.06123352050781,
      "activations/layer5_attention_weight_min": -60.04651641845703,
      "activations/layer6_attention_weight_max": 47.46864700317383,
      "activations/layer6_attention_weight_min": -51.84018325805664,
      "activations/layer7_attention_weight_max": 87.59698486328125,
      "activations/layer7_attention_weight_min": -89.54462432861328,
      "activations/layer8_attention_weight_max": 40.991127014160156,
      "activations/layer8_attention_weight_min": -40.10873794555664,
      "activations/layer9_attention_weight_max": 34.64237976074219,
      "activations/layer9_attention_weight_min": -34.23590850830078,
      "epoch": 10.05,
      "learning_rate": 8.607954545454545e-05,
      "loss": 2.7965,
      "step": 172900
    },
    {
      "activations/layer0_attention_weight_max": 16.676271438598633,
      "activations/layer0_attention_weight_min": -12.618253707885742,
      "activations/layer10_attention_weight_max": 43.875709533691406,
      "activations/layer10_attention_weight_min": -40.151241302490234,
      "activations/layer11_attention_weight_max": 45.614540100097656,
      "activations/layer11_attention_weight_min": -37.8623046875,
      "activations/layer12_attention_weight_max": 32.19644546508789,
      "activations/layer12_attention_weight_min": -25.63250160217285,
      "activations/layer13_attention_weight_max": 57.574005126953125,
      "activations/layer13_attention_weight_min": -36.7841682434082,
      "activations/layer14_attention_weight_max": 56.20998001098633,
      "activations/layer14_attention_weight_min": -39.671974182128906,
      "activations/layer15_attention_weight_max": 63.87276840209961,
      "activations/layer15_attention_weight_min": -34.91875076293945,
      "activations/layer16_attention_weight_max": 46.06789016723633,
      "activations/layer16_attention_weight_min": -27.29585075378418,
      "activations/layer17_attention_weight_max": 36.18257141113281,
      "activations/layer17_attention_weight_min": -26.505014419555664,
      "activations/layer18_attention_weight_max": 39.00591278076172,
      "activations/layer18_attention_weight_min": -23.54967498779297,
      "activations/layer19_attention_weight_max": 44.833744049072266,
      "activations/layer19_attention_weight_min": -33.970802307128906,
      "activations/layer1_attention_weight_max": 16.202011108398438,
      "activations/layer1_attention_weight_min": -14.06540584564209,
      "activations/layer20_attention_weight_max": 34.527156829833984,
      "activations/layer20_attention_weight_min": -26.947002410888672,
      "activations/layer21_attention_weight_max": 33.46673583984375,
      "activations/layer21_attention_weight_min": -22.647180557250977,
      "activations/layer22_attention_weight_max": 57.70916748046875,
      "activations/layer22_attention_weight_min": -29.720972061157227,
      "activations/layer23_attention_weight_max": 40.66339874267578,
      "activations/layer23_attention_weight_min": -22.102375030517578,
      "activations/layer2_attention_weight_max": 34.581321716308594,
      "activations/layer2_attention_weight_min": -32.803531646728516,
      "activations/layer3_attention_weight_max": 95.05833435058594,
      "activations/layer3_attention_weight_min": -93.6427001953125,
      "activations/layer4_attention_weight_max": 115.10285949707031,
      "activations/layer4_attention_weight_min": -112.49635314941406,
      "activations/layer5_attention_weight_max": 57.06761169433594,
      "activations/layer5_attention_weight_min": -63.576324462890625,
      "activations/layer6_attention_weight_max": 53.69059371948242,
      "activations/layer6_attention_weight_min": -49.3527946472168,
      "activations/layer7_attention_weight_max": 122.17047119140625,
      "activations/layer7_attention_weight_min": -102.42526245117188,
      "activations/layer8_attention_weight_max": 54.09095001220703,
      "activations/layer8_attention_weight_min": -48.39981460571289,
      "activations/layer9_attention_weight_max": 44.693206787109375,
      "activations/layer9_attention_weight_min": -39.30314254760742,
      "epoch": 10.05,
      "learning_rate": 8.606060606060605e-05,
      "loss": 2.7809,
      "step": 172950
    },
    {
      "activations/layer0_attention_weight_max": 16.444168090820312,
      "activations/layer0_attention_weight_min": -13.290057182312012,
      "activations/layer10_attention_weight_max": 31.616323471069336,
      "activations/layer10_attention_weight_min": -33.2435188293457,
      "activations/layer11_attention_weight_max": 31.006200790405273,
      "activations/layer11_attention_weight_min": -32.937564849853516,
      "activations/layer12_attention_weight_max": 24.81031608581543,
      "activations/layer12_attention_weight_min": -24.93825912475586,
      "activations/layer13_attention_weight_max": 42.14116287231445,
      "activations/layer13_attention_weight_min": -33.5459098815918,
      "activations/layer14_attention_weight_max": 43.419742584228516,
      "activations/layer14_attention_weight_min": -38.91748809814453,
      "activations/layer15_attention_weight_max": 38.88676452636719,
      "activations/layer15_attention_weight_min": -29.84476661682129,
      "activations/layer16_attention_weight_max": 31.495893478393555,
      "activations/layer16_attention_weight_min": -27.42947769165039,
      "activations/layer17_attention_weight_max": 33.787353515625,
      "activations/layer17_attention_weight_min": -24.60791778564453,
      "activations/layer18_attention_weight_max": 31.61717414855957,
      "activations/layer18_attention_weight_min": -21.741662979125977,
      "activations/layer19_attention_weight_max": 35.652366638183594,
      "activations/layer19_attention_weight_min": -30.108932495117188,
      "activations/layer1_attention_weight_max": 16.382003784179688,
      "activations/layer1_attention_weight_min": -14.011788368225098,
      "activations/layer20_attention_weight_max": 28.956968307495117,
      "activations/layer20_attention_weight_min": -24.515151977539062,
      "activations/layer21_attention_weight_max": 29.14408302307129,
      "activations/layer21_attention_weight_min": -24.026670455932617,
      "activations/layer22_attention_weight_max": 46.13473892211914,
      "activations/layer22_attention_weight_min": -27.99468994140625,
      "activations/layer23_attention_weight_max": 36.36576461791992,
      "activations/layer23_attention_weight_min": -24.301227569580078,
      "activations/layer2_attention_weight_max": 31.3436279296875,
      "activations/layer2_attention_weight_min": -31.488162994384766,
      "activations/layer3_attention_weight_max": 88.74510192871094,
      "activations/layer3_attention_weight_min": -93.84243774414062,
      "activations/layer4_attention_weight_max": 104.8396224975586,
      "activations/layer4_attention_weight_min": -108.7618179321289,
      "activations/layer5_attention_weight_max": 50.906715393066406,
      "activations/layer5_attention_weight_min": -59.833229064941406,
      "activations/layer6_attention_weight_max": 43.37525939941406,
      "activations/layer6_attention_weight_min": -49.34125518798828,
      "activations/layer7_attention_weight_max": 84.82493591308594,
      "activations/layer7_attention_weight_min": -89.25011444091797,
      "activations/layer8_attention_weight_max": 37.3057975769043,
      "activations/layer8_attention_weight_min": -41.71946716308594,
      "activations/layer9_attention_weight_max": 32.31806945800781,
      "activations/layer9_attention_weight_min": -34.20795822143555,
      "epoch": 10.05,
      "learning_rate": 8.604166666666665e-05,
      "loss": 2.7911,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.515,
      "eval_samples_per_second": 504.286,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.515,
      "eval_openwebtext_samples_per_second": 504.286,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 1.9627,
      "eval_wikitext_samples_per_second": 232.335,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.5143,
      "eval_lambada_samples_per_second": 511.755,
      "step": 173000
    },
    {
      "activations/layer0_attention_weight_max": 15.706645965576172,
      "activations/layer0_attention_weight_min": -11.87035083770752,
      "activations/layer10_attention_weight_max": 32.11880874633789,
      "activations/layer10_attention_weight_min": -34.298221588134766,
      "activations/layer11_attention_weight_max": 30.913190841674805,
      "activations/layer11_attention_weight_min": -29.919631958007812,
      "activations/layer12_attention_weight_max": 22.963733673095703,
      "activations/layer12_attention_weight_min": -24.057966232299805,
      "activations/layer13_attention_weight_max": 37.71905517578125,
      "activations/layer13_attention_weight_min": -31.606481552124023,
      "activations/layer14_attention_weight_max": 44.95524597167969,
      "activations/layer14_attention_weight_min": -37.190818786621094,
      "activations/layer15_attention_weight_max": 35.963165283203125,
      "activations/layer15_attention_weight_min": -29.63710594177246,
      "activations/layer16_attention_weight_max": 30.142595291137695,
      "activations/layer16_attention_weight_min": -28.578197479248047,
      "activations/layer17_attention_weight_max": 32.05074691772461,
      "activations/layer17_attention_weight_min": -25.332853317260742,
      "activations/layer18_attention_weight_max": 30.020893096923828,
      "activations/layer18_attention_weight_min": -25.04620933532715,
      "activations/layer19_attention_weight_max": 34.5072135925293,
      "activations/layer19_attention_weight_min": -32.51530075073242,
      "activations/layer1_attention_weight_max": 16.987110137939453,
      "activations/layer1_attention_weight_min": -15.661749839782715,
      "activations/layer20_attention_weight_max": 30.425142288208008,
      "activations/layer20_attention_weight_min": -24.713834762573242,
      "activations/layer21_attention_weight_max": 30.217458724975586,
      "activations/layer21_attention_weight_min": -24.166213989257812,
      "activations/layer22_attention_weight_max": 49.31595230102539,
      "activations/layer22_attention_weight_min": -31.37583351135254,
      "activations/layer23_attention_weight_max": 36.53775405883789,
      "activations/layer23_attention_weight_min": -25.588619232177734,
      "activations/layer2_attention_weight_max": 31.95150375366211,
      "activations/layer2_attention_weight_min": -31.60792350769043,
      "activations/layer3_attention_weight_max": 91.96402740478516,
      "activations/layer3_attention_weight_min": -95.9616470336914,
      "activations/layer4_attention_weight_max": 112.6728744506836,
      "activations/layer4_attention_weight_min": -103.04058837890625,
      "activations/layer5_attention_weight_max": 52.54082489013672,
      "activations/layer5_attention_weight_min": -58.93146896362305,
      "activations/layer6_attention_weight_max": 44.90568542480469,
      "activations/layer6_attention_weight_min": -47.687129974365234,
      "activations/layer7_attention_weight_max": 88.31854248046875,
      "activations/layer7_attention_weight_min": -84.99212646484375,
      "activations/layer8_attention_weight_max": 40.73831558227539,
      "activations/layer8_attention_weight_min": -40.428470611572266,
      "activations/layer9_attention_weight_max": 32.96723556518555,
      "activations/layer9_attention_weight_min": -33.91598129272461,
      "epoch": 10.06,
      "learning_rate": 8.602272727272727e-05,
      "loss": 2.781,
      "step": 173050
    },
    {
      "activations/layer0_attention_weight_max": 15.66981029510498,
      "activations/layer0_attention_weight_min": -11.923511505126953,
      "activations/layer10_attention_weight_max": 34.195838928222656,
      "activations/layer10_attention_weight_min": -32.26605987548828,
      "activations/layer11_attention_weight_max": 31.918502807617188,
      "activations/layer11_attention_weight_min": -32.50579833984375,
      "activations/layer12_attention_weight_max": 24.684961318969727,
      "activations/layer12_attention_weight_min": -26.04214096069336,
      "activations/layer13_attention_weight_max": 37.48845672607422,
      "activations/layer13_attention_weight_min": -36.0192756652832,
      "activations/layer14_attention_weight_max": 47.228477478027344,
      "activations/layer14_attention_weight_min": -39.93319320678711,
      "activations/layer15_attention_weight_max": 34.50328063964844,
      "activations/layer15_attention_weight_min": -31.94315528869629,
      "activations/layer16_attention_weight_max": 30.74919891357422,
      "activations/layer16_attention_weight_min": -28.680416107177734,
      "activations/layer17_attention_weight_max": 31.00814437866211,
      "activations/layer17_attention_weight_min": -24.518024444580078,
      "activations/layer18_attention_weight_max": 30.782114028930664,
      "activations/layer18_attention_weight_min": -24.78218650817871,
      "activations/layer19_attention_weight_max": 32.91747283935547,
      "activations/layer19_attention_weight_min": -30.471960067749023,
      "activations/layer1_attention_weight_max": 15.886894226074219,
      "activations/layer1_attention_weight_min": -15.366965293884277,
      "activations/layer20_attention_weight_max": 29.227947235107422,
      "activations/layer20_attention_weight_min": -24.97869300842285,
      "activations/layer21_attention_weight_max": 31.863964080810547,
      "activations/layer21_attention_weight_min": -23.667936325073242,
      "activations/layer22_attention_weight_max": 43.76496505737305,
      "activations/layer22_attention_weight_min": -30.69224739074707,
      "activations/layer23_attention_weight_max": 33.39651107788086,
      "activations/layer23_attention_weight_min": -27.57164764404297,
      "activations/layer2_attention_weight_max": 30.916954040527344,
      "activations/layer2_attention_weight_min": -32.39284133911133,
      "activations/layer3_attention_weight_max": 93.0551986694336,
      "activations/layer3_attention_weight_min": -94.5417709350586,
      "activations/layer4_attention_weight_max": 111.92900848388672,
      "activations/layer4_attention_weight_min": -108.55885314941406,
      "activations/layer5_attention_weight_max": 51.509071350097656,
      "activations/layer5_attention_weight_min": -63.944580078125,
      "activations/layer6_attention_weight_max": 47.368099212646484,
      "activations/layer6_attention_weight_min": -51.74800491333008,
      "activations/layer7_attention_weight_max": 85.19505310058594,
      "activations/layer7_attention_weight_min": -88.80317687988281,
      "activations/layer8_attention_weight_max": 38.25353240966797,
      "activations/layer8_attention_weight_min": -39.32062911987305,
      "activations/layer9_attention_weight_max": 33.275596618652344,
      "activations/layer9_attention_weight_min": -33.70344161987305,
      "epoch": 10.06,
      "learning_rate": 8.600378787878788e-05,
      "loss": 2.7818,
      "step": 173100
    },
    {
      "activations/layer0_attention_weight_max": 15.53822135925293,
      "activations/layer0_attention_weight_min": -11.483511924743652,
      "activations/layer10_attention_weight_max": 32.498374938964844,
      "activations/layer10_attention_weight_min": -31.224782943725586,
      "activations/layer11_attention_weight_max": 32.13519287109375,
      "activations/layer11_attention_weight_min": -30.333242416381836,
      "activations/layer12_attention_weight_max": 25.14507484436035,
      "activations/layer12_attention_weight_min": -24.760955810546875,
      "activations/layer13_attention_weight_max": 39.087303161621094,
      "activations/layer13_attention_weight_min": -33.30779266357422,
      "activations/layer14_attention_weight_max": 46.411529541015625,
      "activations/layer14_attention_weight_min": -39.14488983154297,
      "activations/layer15_attention_weight_max": 38.37458038330078,
      "activations/layer15_attention_weight_min": -30.162935256958008,
      "activations/layer16_attention_weight_max": 33.08492660522461,
      "activations/layer16_attention_weight_min": -27.635347366333008,
      "activations/layer17_attention_weight_max": 31.33115577697754,
      "activations/layer17_attention_weight_min": -25.879413604736328,
      "activations/layer18_attention_weight_max": 32.1640739440918,
      "activations/layer18_attention_weight_min": -22.9550724029541,
      "activations/layer19_attention_weight_max": 35.8542594909668,
      "activations/layer19_attention_weight_min": -31.195755004882812,
      "activations/layer1_attention_weight_max": 17.932600021362305,
      "activations/layer1_attention_weight_min": -16.037084579467773,
      "activations/layer20_attention_weight_max": 31.31838035583496,
      "activations/layer20_attention_weight_min": -25.600563049316406,
      "activations/layer21_attention_weight_max": 30.428293228149414,
      "activations/layer21_attention_weight_min": -25.784048080444336,
      "activations/layer22_attention_weight_max": 54.66574478149414,
      "activations/layer22_attention_weight_min": -31.912744522094727,
      "activations/layer23_attention_weight_max": 37.430450439453125,
      "activations/layer23_attention_weight_min": -25.068830490112305,
      "activations/layer2_attention_weight_max": 34.714202880859375,
      "activations/layer2_attention_weight_min": -33.80576705932617,
      "activations/layer3_attention_weight_max": 96.53318786621094,
      "activations/layer3_attention_weight_min": -96.21729278564453,
      "activations/layer4_attention_weight_max": 113.80439758300781,
      "activations/layer4_attention_weight_min": -111.03810119628906,
      "activations/layer5_attention_weight_max": 54.76244354248047,
      "activations/layer5_attention_weight_min": -59.562889099121094,
      "activations/layer6_attention_weight_max": 46.81703567504883,
      "activations/layer6_attention_weight_min": -48.55429458618164,
      "activations/layer7_attention_weight_max": 94.86565399169922,
      "activations/layer7_attention_weight_min": -91.93354034423828,
      "activations/layer8_attention_weight_max": 41.14046859741211,
      "activations/layer8_attention_weight_min": -40.171905517578125,
      "activations/layer9_attention_weight_max": 33.94268035888672,
      "activations/layer9_attention_weight_min": -33.230403900146484,
      "epoch": 10.06,
      "learning_rate": 8.598484848484847e-05,
      "loss": 2.7609,
      "step": 173150
    },
    {
      "activations/layer0_attention_weight_max": 15.663604736328125,
      "activations/layer0_attention_weight_min": -12.07459545135498,
      "activations/layer10_attention_weight_max": 34.35701370239258,
      "activations/layer10_attention_weight_min": -33.83758544921875,
      "activations/layer11_attention_weight_max": 33.93755340576172,
      "activations/layer11_attention_weight_min": -33.8399658203125,
      "activations/layer12_attention_weight_max": 24.53811264038086,
      "activations/layer12_attention_weight_min": -27.026126861572266,
      "activations/layer13_attention_weight_max": 38.03935623168945,
      "activations/layer13_attention_weight_min": -36.32963562011719,
      "activations/layer14_attention_weight_max": 40.80387878417969,
      "activations/layer14_attention_weight_min": -39.23892593383789,
      "activations/layer15_attention_weight_max": 34.114967346191406,
      "activations/layer15_attention_weight_min": -30.856639862060547,
      "activations/layer16_attention_weight_max": 28.808794021606445,
      "activations/layer16_attention_weight_min": -26.154895782470703,
      "activations/layer17_attention_weight_max": 30.565927505493164,
      "activations/layer17_attention_weight_min": -25.03953742980957,
      "activations/layer18_attention_weight_max": 28.061124801635742,
      "activations/layer18_attention_weight_min": -23.367036819458008,
      "activations/layer19_attention_weight_max": 33.262542724609375,
      "activations/layer19_attention_weight_min": -31.150793075561523,
      "activations/layer1_attention_weight_max": 16.533849716186523,
      "activations/layer1_attention_weight_min": -14.81906509399414,
      "activations/layer20_attention_weight_max": 29.78221893310547,
      "activations/layer20_attention_weight_min": -26.719892501831055,
      "activations/layer21_attention_weight_max": 28.01581382751465,
      "activations/layer21_attention_weight_min": -26.004913330078125,
      "activations/layer22_attention_weight_max": 41.82411193847656,
      "activations/layer22_attention_weight_min": -30.745264053344727,
      "activations/layer23_attention_weight_max": 32.93760299682617,
      "activations/layer23_attention_weight_min": -25.97615623474121,
      "activations/layer2_attention_weight_max": 28.165143966674805,
      "activations/layer2_attention_weight_min": -30.7504940032959,
      "activations/layer3_attention_weight_max": 92.19115447998047,
      "activations/layer3_attention_weight_min": -93.74073028564453,
      "activations/layer4_attention_weight_max": 111.84786224365234,
      "activations/layer4_attention_weight_min": -109.0328140258789,
      "activations/layer5_attention_weight_max": 53.548988342285156,
      "activations/layer5_attention_weight_min": -62.78837585449219,
      "activations/layer6_attention_weight_max": 46.473716735839844,
      "activations/layer6_attention_weight_min": -49.389163970947266,
      "activations/layer7_attention_weight_max": 88.07369232177734,
      "activations/layer7_attention_weight_min": -91.66171264648438,
      "activations/layer8_attention_weight_max": 40.326175689697266,
      "activations/layer8_attention_weight_min": -41.82426071166992,
      "activations/layer9_attention_weight_max": 34.98185729980469,
      "activations/layer9_attention_weight_min": -36.12252426147461,
      "epoch": 10.06,
      "learning_rate": 8.596590909090909e-05,
      "loss": 2.7718,
      "step": 173200
    },
    {
      "activations/layer0_attention_weight_max": 14.576863288879395,
      "activations/layer0_attention_weight_min": -12.76172924041748,
      "activations/layer10_attention_weight_max": 35.98147201538086,
      "activations/layer10_attention_weight_min": -32.08136749267578,
      "activations/layer11_attention_weight_max": 37.37223815917969,
      "activations/layer11_attention_weight_min": -32.39410400390625,
      "activations/layer12_attention_weight_max": 24.69567108154297,
      "activations/layer12_attention_weight_min": -24.780338287353516,
      "activations/layer13_attention_weight_max": 39.320091247558594,
      "activations/layer13_attention_weight_min": -32.467552185058594,
      "activations/layer14_attention_weight_max": 44.92892074584961,
      "activations/layer14_attention_weight_min": -37.38655090332031,
      "activations/layer15_attention_weight_max": 37.78733825683594,
      "activations/layer15_attention_weight_min": -31.137115478515625,
      "activations/layer16_attention_weight_max": 34.096221923828125,
      "activations/layer16_attention_weight_min": -26.370906829833984,
      "activations/layer17_attention_weight_max": 35.38762664794922,
      "activations/layer17_attention_weight_min": -23.598237991333008,
      "activations/layer18_attention_weight_max": 31.60226058959961,
      "activations/layer18_attention_weight_min": -21.825950622558594,
      "activations/layer19_attention_weight_max": 33.781211853027344,
      "activations/layer19_attention_weight_min": -28.461393356323242,
      "activations/layer1_attention_weight_max": 18.10458755493164,
      "activations/layer1_attention_weight_min": -15.440207481384277,
      "activations/layer20_attention_weight_max": 31.411680221557617,
      "activations/layer20_attention_weight_min": -24.60299301147461,
      "activations/layer21_attention_weight_max": 27.197172164916992,
      "activations/layer21_attention_weight_min": -22.115341186523438,
      "activations/layer22_attention_weight_max": 43.946781158447266,
      "activations/layer22_attention_weight_min": -26.029905319213867,
      "activations/layer23_attention_weight_max": 32.718204498291016,
      "activations/layer23_attention_weight_min": -23.682737350463867,
      "activations/layer2_attention_weight_max": 32.85091781616211,
      "activations/layer2_attention_weight_min": -32.85308074951172,
      "activations/layer3_attention_weight_max": 92.51409149169922,
      "activations/layer3_attention_weight_min": -97.51099395751953,
      "activations/layer4_attention_weight_max": 108.04833221435547,
      "activations/layer4_attention_weight_min": -108.42433166503906,
      "activations/layer5_attention_weight_max": 51.670188903808594,
      "activations/layer5_attention_weight_min": -60.802833557128906,
      "activations/layer6_attention_weight_max": 46.11240768432617,
      "activations/layer6_attention_weight_min": -47.63570785522461,
      "activations/layer7_attention_weight_max": 98.69625854492188,
      "activations/layer7_attention_weight_min": -88.11713409423828,
      "activations/layer8_attention_weight_max": 46.4821891784668,
      "activations/layer8_attention_weight_min": -43.49248504638672,
      "activations/layer9_attention_weight_max": 38.150474548339844,
      "activations/layer9_attention_weight_min": -38.100868225097656,
      "epoch": 10.07,
      "learning_rate": 8.59469696969697e-05,
      "loss": 2.7853,
      "step": 173250
    },
    {
      "activations/layer0_attention_weight_max": 15.63398265838623,
      "activations/layer0_attention_weight_min": -12.048848152160645,
      "activations/layer10_attention_weight_max": 30.320585250854492,
      "activations/layer10_attention_weight_min": -30.5972900390625,
      "activations/layer11_attention_weight_max": 28.861631393432617,
      "activations/layer11_attention_weight_min": -30.13608169555664,
      "activations/layer12_attention_weight_max": 22.542001724243164,
      "activations/layer12_attention_weight_min": -25.79934310913086,
      "activations/layer13_attention_weight_max": 33.72705078125,
      "activations/layer13_attention_weight_min": -33.99298858642578,
      "activations/layer14_attention_weight_max": 41.353336334228516,
      "activations/layer14_attention_weight_min": -36.35298538208008,
      "activations/layer15_attention_weight_max": 32.1870002746582,
      "activations/layer15_attention_weight_min": -29.76103401184082,
      "activations/layer16_attention_weight_max": 27.589967727661133,
      "activations/layer16_attention_weight_min": -28.34610939025879,
      "activations/layer17_attention_weight_max": 27.665000915527344,
      "activations/layer17_attention_weight_min": -26.56109046936035,
      "activations/layer18_attention_weight_max": 28.14959716796875,
      "activations/layer18_attention_weight_min": -24.319564819335938,
      "activations/layer19_attention_weight_max": 30.9548397064209,
      "activations/layer19_attention_weight_min": -29.29572868347168,
      "activations/layer1_attention_weight_max": 17.30113410949707,
      "activations/layer1_attention_weight_min": -15.848406791687012,
      "activations/layer20_attention_weight_max": 26.63277816772461,
      "activations/layer20_attention_weight_min": -23.36605453491211,
      "activations/layer21_attention_weight_max": 26.324825286865234,
      "activations/layer21_attention_weight_min": -22.55915069580078,
      "activations/layer22_attention_weight_max": 40.870758056640625,
      "activations/layer22_attention_weight_min": -30.317169189453125,
      "activations/layer23_attention_weight_max": 31.012968063354492,
      "activations/layer23_attention_weight_min": -25.868764877319336,
      "activations/layer2_attention_weight_max": 30.40435791015625,
      "activations/layer2_attention_weight_min": -31.298629760742188,
      "activations/layer3_attention_weight_max": 89.51313781738281,
      "activations/layer3_attention_weight_min": -91.33672332763672,
      "activations/layer4_attention_weight_max": 106.9012222290039,
      "activations/layer4_attention_weight_min": -107.3568115234375,
      "activations/layer5_attention_weight_max": 50.971092224121094,
      "activations/layer5_attention_weight_min": -60.482521057128906,
      "activations/layer6_attention_weight_max": 42.380916595458984,
      "activations/layer6_attention_weight_min": -45.84676742553711,
      "activations/layer7_attention_weight_max": 81.78602600097656,
      "activations/layer7_attention_weight_min": -85.41378021240234,
      "activations/layer8_attention_weight_max": 39.375431060791016,
      "activations/layer8_attention_weight_min": -41.87274169921875,
      "activations/layer9_attention_weight_max": 33.38557052612305,
      "activations/layer9_attention_weight_min": -31.904584884643555,
      "epoch": 10.07,
      "learning_rate": 8.592803030303029e-05,
      "loss": 2.779,
      "step": 173300
    },
    {
      "activations/layer0_attention_weight_max": 16.156864166259766,
      "activations/layer0_attention_weight_min": -12.07662582397461,
      "activations/layer10_attention_weight_max": 32.4606819152832,
      "activations/layer10_attention_weight_min": -30.547805786132812,
      "activations/layer11_attention_weight_max": 31.533157348632812,
      "activations/layer11_attention_weight_min": -27.93623161315918,
      "activations/layer12_attention_weight_max": 22.870441436767578,
      "activations/layer12_attention_weight_min": -23.60647201538086,
      "activations/layer13_attention_weight_max": 37.5123405456543,
      "activations/layer13_attention_weight_min": -31.77829933166504,
      "activations/layer14_attention_weight_max": 46.98184585571289,
      "activations/layer14_attention_weight_min": -38.45005798339844,
      "activations/layer15_attention_weight_max": 36.61061477661133,
      "activations/layer15_attention_weight_min": -30.000669479370117,
      "activations/layer16_attention_weight_max": 31.077653884887695,
      "activations/layer16_attention_weight_min": -27.349681854248047,
      "activations/layer17_attention_weight_max": 35.90810012817383,
      "activations/layer17_attention_weight_min": -24.26786231994629,
      "activations/layer18_attention_weight_max": 37.458946228027344,
      "activations/layer18_attention_weight_min": -21.417144775390625,
      "activations/layer19_attention_weight_max": 39.94232177734375,
      "activations/layer19_attention_weight_min": -31.480274200439453,
      "activations/layer1_attention_weight_max": 16.85460090637207,
      "activations/layer1_attention_weight_min": -14.679425239562988,
      "activations/layer20_attention_weight_max": 31.19011688232422,
      "activations/layer20_attention_weight_min": -23.273906707763672,
      "activations/layer21_attention_weight_max": 35.260536193847656,
      "activations/layer21_attention_weight_min": -21.880359649658203,
      "activations/layer22_attention_weight_max": 43.58629608154297,
      "activations/layer22_attention_weight_min": -29.905513763427734,
      "activations/layer23_attention_weight_max": 40.131690979003906,
      "activations/layer23_attention_weight_min": -21.88135528564453,
      "activations/layer2_attention_weight_max": 31.483640670776367,
      "activations/layer2_attention_weight_min": -32.57670593261719,
      "activations/layer3_attention_weight_max": 95.68923950195312,
      "activations/layer3_attention_weight_min": -99.62932586669922,
      "activations/layer4_attention_weight_max": 117.80950164794922,
      "activations/layer4_attention_weight_min": -112.0208969116211,
      "activations/layer5_attention_weight_max": 53.90729522705078,
      "activations/layer5_attention_weight_min": -62.469764709472656,
      "activations/layer6_attention_weight_max": 46.20267868041992,
      "activations/layer6_attention_weight_min": -53.093360900878906,
      "activations/layer7_attention_weight_max": 88.57421112060547,
      "activations/layer7_attention_weight_min": -94.7362060546875,
      "activations/layer8_attention_weight_max": 40.05129623413086,
      "activations/layer8_attention_weight_min": -40.08295822143555,
      "activations/layer9_attention_weight_max": 33.23025131225586,
      "activations/layer9_attention_weight_min": -31.8673038482666,
      "epoch": 10.07,
      "learning_rate": 8.59090909090909e-05,
      "loss": 2.7889,
      "step": 173350
    },
    {
      "activations/layer0_attention_weight_max": 16.941452026367188,
      "activations/layer0_attention_weight_min": -12.564881324768066,
      "activations/layer10_attention_weight_max": 31.171329498291016,
      "activations/layer10_attention_weight_min": -33.73674392700195,
      "activations/layer11_attention_weight_max": 31.880605697631836,
      "activations/layer11_attention_weight_min": -32.014259338378906,
      "activations/layer12_attention_weight_max": 22.455440521240234,
      "activations/layer12_attention_weight_min": -25.499975204467773,
      "activations/layer13_attention_weight_max": 37.02290344238281,
      "activations/layer13_attention_weight_min": -34.395015716552734,
      "activations/layer14_attention_weight_max": 46.164459228515625,
      "activations/layer14_attention_weight_min": -42.04570770263672,
      "activations/layer15_attention_weight_max": 36.6851921081543,
      "activations/layer15_attention_weight_min": -30.314802169799805,
      "activations/layer16_attention_weight_max": 29.975994110107422,
      "activations/layer16_attention_weight_min": -27.201139450073242,
      "activations/layer17_attention_weight_max": 28.68118667602539,
      "activations/layer17_attention_weight_min": -23.948556900024414,
      "activations/layer18_attention_weight_max": 29.640033721923828,
      "activations/layer18_attention_weight_min": -22.389299392700195,
      "activations/layer19_attention_weight_max": 34.1513557434082,
      "activations/layer19_attention_weight_min": -31.31699562072754,
      "activations/layer1_attention_weight_max": 16.602649688720703,
      "activations/layer1_attention_weight_min": -15.025960922241211,
      "activations/layer20_attention_weight_max": 30.037914276123047,
      "activations/layer20_attention_weight_min": -24.42670249938965,
      "activations/layer21_attention_weight_max": 27.15176010131836,
      "activations/layer21_attention_weight_min": -23.707046508789062,
      "activations/layer22_attention_weight_max": 46.2801399230957,
      "activations/layer22_attention_weight_min": -30.029584884643555,
      "activations/layer23_attention_weight_max": 33.15190124511719,
      "activations/layer23_attention_weight_min": -26.535030364990234,
      "activations/layer2_attention_weight_max": 30.56831169128418,
      "activations/layer2_attention_weight_min": -31.179080963134766,
      "activations/layer3_attention_weight_max": 86.51173400878906,
      "activations/layer3_attention_weight_min": -91.7352523803711,
      "activations/layer4_attention_weight_max": 103.21471405029297,
      "activations/layer4_attention_weight_min": -108.4458236694336,
      "activations/layer5_attention_weight_max": 50.831703186035156,
      "activations/layer5_attention_weight_min": -59.95459747314453,
      "activations/layer6_attention_weight_max": 44.586299896240234,
      "activations/layer6_attention_weight_min": -47.17638397216797,
      "activations/layer7_attention_weight_max": 90.5849838256836,
      "activations/layer7_attention_weight_min": -92.27017974853516,
      "activations/layer8_attention_weight_max": 40.570011138916016,
      "activations/layer8_attention_weight_min": -44.075557708740234,
      "activations/layer9_attention_weight_max": 34.49210739135742,
      "activations/layer9_attention_weight_min": -35.86236572265625,
      "epoch": 10.08,
      "learning_rate": 8.58901515151515e-05,
      "loss": 2.7971,
      "step": 173400
    },
    {
      "activations/layer0_attention_weight_max": 15.741613388061523,
      "activations/layer0_attention_weight_min": -12.433354377746582,
      "activations/layer10_attention_weight_max": 37.892486572265625,
      "activations/layer10_attention_weight_min": -33.351951599121094,
      "activations/layer11_attention_weight_max": 33.51111602783203,
      "activations/layer11_attention_weight_min": -32.04258346557617,
      "activations/layer12_attention_weight_max": 32.3295783996582,
      "activations/layer12_attention_weight_min": -30.817550659179688,
      "activations/layer13_attention_weight_max": 41.55892562866211,
      "activations/layer13_attention_weight_min": -32.85231018066406,
      "activations/layer14_attention_weight_max": 53.33890914916992,
      "activations/layer14_attention_weight_min": -38.405982971191406,
      "activations/layer15_attention_weight_max": 39.39680099487305,
      "activations/layer15_attention_weight_min": -31.36972999572754,
      "activations/layer16_attention_weight_max": 31.377038955688477,
      "activations/layer16_attention_weight_min": -27.09293556213379,
      "activations/layer17_attention_weight_max": 33.5128059387207,
      "activations/layer17_attention_weight_min": -25.321632385253906,
      "activations/layer18_attention_weight_max": 32.15696334838867,
      "activations/layer18_attention_weight_min": -23.664831161499023,
      "activations/layer19_attention_weight_max": 34.576454162597656,
      "activations/layer19_attention_weight_min": -30.855295181274414,
      "activations/layer1_attention_weight_max": 16.825515747070312,
      "activations/layer1_attention_weight_min": -15.464240074157715,
      "activations/layer20_attention_weight_max": 30.64370346069336,
      "activations/layer20_attention_weight_min": -24.59370994567871,
      "activations/layer21_attention_weight_max": 28.544328689575195,
      "activations/layer21_attention_weight_min": -22.749134063720703,
      "activations/layer22_attention_weight_max": 45.841346740722656,
      "activations/layer22_attention_weight_min": -29.456708908081055,
      "activations/layer23_attention_weight_max": 34.14479064941406,
      "activations/layer23_attention_weight_min": -26.680557250976562,
      "activations/layer2_attention_weight_max": 31.03687286376953,
      "activations/layer2_attention_weight_min": -32.32109832763672,
      "activations/layer3_attention_weight_max": 91.04414367675781,
      "activations/layer3_attention_weight_min": -91.92595672607422,
      "activations/layer4_attention_weight_max": 108.59637451171875,
      "activations/layer4_attention_weight_min": -115.0420150756836,
      "activations/layer5_attention_weight_max": 51.226356506347656,
      "activations/layer5_attention_weight_min": -66.86929321289062,
      "activations/layer6_attention_weight_max": 46.39815139770508,
      "activations/layer6_attention_weight_min": -48.154056549072266,
      "activations/layer7_attention_weight_max": 89.26642608642578,
      "activations/layer7_attention_weight_min": -90.89545440673828,
      "activations/layer8_attention_weight_max": 40.0362663269043,
      "activations/layer8_attention_weight_min": -41.98206329345703,
      "activations/layer9_attention_weight_max": 36.247684478759766,
      "activations/layer9_attention_weight_min": -33.83002853393555,
      "epoch": 10.08,
      "learning_rate": 8.587121212121211e-05,
      "loss": 2.7896,
      "step": 173450
    },
    {
      "activations/layer0_attention_weight_max": 15.217157363891602,
      "activations/layer0_attention_weight_min": -12.474516868591309,
      "activations/layer10_attention_weight_max": 31.349884033203125,
      "activations/layer10_attention_weight_min": -30.868478775024414,
      "activations/layer11_attention_weight_max": 31.263425827026367,
      "activations/layer11_attention_weight_min": -29.12250328063965,
      "activations/layer12_attention_weight_max": 23.380416870117188,
      "activations/layer12_attention_weight_min": -26.437435150146484,
      "activations/layer13_attention_weight_max": 38.04650115966797,
      "activations/layer13_attention_weight_min": -31.51627540588379,
      "activations/layer14_attention_weight_max": 48.04710388183594,
      "activations/layer14_attention_weight_min": -37.43831253051758,
      "activations/layer15_attention_weight_max": 37.665897369384766,
      "activations/layer15_attention_weight_min": -31.518558502197266,
      "activations/layer16_attention_weight_max": 36.188697814941406,
      "activations/layer16_attention_weight_min": -26.519500732421875,
      "activations/layer17_attention_weight_max": 35.343143463134766,
      "activations/layer17_attention_weight_min": -26.110477447509766,
      "activations/layer18_attention_weight_max": 33.832096099853516,
      "activations/layer18_attention_weight_min": -22.559343338012695,
      "activations/layer19_attention_weight_max": 43.27711868286133,
      "activations/layer19_attention_weight_min": -31.82779312133789,
      "activations/layer1_attention_weight_max": 16.490097045898438,
      "activations/layer1_attention_weight_min": -13.201659202575684,
      "activations/layer20_attention_weight_max": 34.9990348815918,
      "activations/layer20_attention_weight_min": -24.497777938842773,
      "activations/layer21_attention_weight_max": 32.60886764526367,
      "activations/layer21_attention_weight_min": -24.071985244750977,
      "activations/layer22_attention_weight_max": 51.03110122680664,
      "activations/layer22_attention_weight_min": -30.204635620117188,
      "activations/layer23_attention_weight_max": 38.568851470947266,
      "activations/layer23_attention_weight_min": -24.25196075439453,
      "activations/layer2_attention_weight_max": 31.101184844970703,
      "activations/layer2_attention_weight_min": -31.119911193847656,
      "activations/layer3_attention_weight_max": 85.96508026123047,
      "activations/layer3_attention_weight_min": -88.82395935058594,
      "activations/layer4_attention_weight_max": 103.97383117675781,
      "activations/layer4_attention_weight_min": -104.5496826171875,
      "activations/layer5_attention_weight_max": 49.29554748535156,
      "activations/layer5_attention_weight_min": -60.31764221191406,
      "activations/layer6_attention_weight_max": 44.00724792480469,
      "activations/layer6_attention_weight_min": -46.20109176635742,
      "activations/layer7_attention_weight_max": 87.65653991699219,
      "activations/layer7_attention_weight_min": -85.5431137084961,
      "activations/layer8_attention_weight_max": 36.97800827026367,
      "activations/layer8_attention_weight_min": -41.16628646850586,
      "activations/layer9_attention_weight_max": 30.95608901977539,
      "activations/layer9_attention_weight_min": -33.55231475830078,
      "epoch": 10.08,
      "learning_rate": 8.585227272727272e-05,
      "loss": 2.7981,
      "step": 173500
    },
    {
      "activations/layer0_attention_weight_max": 16.918869018554688,
      "activations/layer0_attention_weight_min": -11.548813819885254,
      "activations/layer10_attention_weight_max": 31.96575927734375,
      "activations/layer10_attention_weight_min": -30.34752082824707,
      "activations/layer11_attention_weight_max": 31.228248596191406,
      "activations/layer11_attention_weight_min": -29.679859161376953,
      "activations/layer12_attention_weight_max": 24.40926742553711,
      "activations/layer12_attention_weight_min": -27.345096588134766,
      "activations/layer13_attention_weight_max": 40.73891067504883,
      "activations/layer13_attention_weight_min": -32.97489929199219,
      "activations/layer14_attention_weight_max": 52.483863830566406,
      "activations/layer14_attention_weight_min": -41.34109115600586,
      "activations/layer15_attention_weight_max": 36.147220611572266,
      "activations/layer15_attention_weight_min": -30.19135284423828,
      "activations/layer16_attention_weight_max": 30.856754302978516,
      "activations/layer16_attention_weight_min": -28.506242752075195,
      "activations/layer17_attention_weight_max": 30.651031494140625,
      "activations/layer17_attention_weight_min": -25.237682342529297,
      "activations/layer18_attention_weight_max": 32.455013275146484,
      "activations/layer18_attention_weight_min": -22.820323944091797,
      "activations/layer19_attention_weight_max": 38.54564666748047,
      "activations/layer19_attention_weight_min": -33.421119689941406,
      "activations/layer1_attention_weight_max": 16.378555297851562,
      "activations/layer1_attention_weight_min": -13.798227310180664,
      "activations/layer20_attention_weight_max": 30.41109275817871,
      "activations/layer20_attention_weight_min": -25.89318084716797,
      "activations/layer21_attention_weight_max": 28.96090316772461,
      "activations/layer21_attention_weight_min": -25.412038803100586,
      "activations/layer22_attention_weight_max": 45.74943923950195,
      "activations/layer22_attention_weight_min": -32.298789978027344,
      "activations/layer23_attention_weight_max": 39.12812042236328,
      "activations/layer23_attention_weight_min": -29.025348663330078,
      "activations/layer2_attention_weight_max": 30.443988800048828,
      "activations/layer2_attention_weight_min": -31.73731803894043,
      "activations/layer3_attention_weight_max": 91.61848449707031,
      "activations/layer3_attention_weight_min": -93.25812530517578,
      "activations/layer4_attention_weight_max": 114.95354461669922,
      "activations/layer4_attention_weight_min": -111.44261932373047,
      "activations/layer5_attention_weight_max": 54.262569427490234,
      "activations/layer5_attention_weight_min": -63.80847930908203,
      "activations/layer6_attention_weight_max": 46.542396545410156,
      "activations/layer6_attention_weight_min": -51.699398040771484,
      "activations/layer7_attention_weight_max": 90.0989990234375,
      "activations/layer7_attention_weight_min": -90.72589111328125,
      "activations/layer8_attention_weight_max": 39.18302536010742,
      "activations/layer8_attention_weight_min": -41.60384750366211,
      "activations/layer9_attention_weight_max": 33.124107360839844,
      "activations/layer9_attention_weight_min": -31.33526039123535,
      "epoch": 10.08,
      "learning_rate": 8.583333333333332e-05,
      "loss": 2.7799,
      "step": 173550
    },
    {
      "activations/layer0_attention_weight_max": 15.516777038574219,
      "activations/layer0_attention_weight_min": -12.710680961608887,
      "activations/layer10_attention_weight_max": 30.336929321289062,
      "activations/layer10_attention_weight_min": -32.43552780151367,
      "activations/layer11_attention_weight_max": 30.840009689331055,
      "activations/layer11_attention_weight_min": -32.30946350097656,
      "activations/layer12_attention_weight_max": 23.023399353027344,
      "activations/layer12_attention_weight_min": -24.490251541137695,
      "activations/layer13_attention_weight_max": 36.629207611083984,
      "activations/layer13_attention_weight_min": -32.35783386230469,
      "activations/layer14_attention_weight_max": 50.88385772705078,
      "activations/layer14_attention_weight_min": -40.346439361572266,
      "activations/layer15_attention_weight_max": 35.81607437133789,
      "activations/layer15_attention_weight_min": -31.42115592956543,
      "activations/layer16_attention_weight_max": 30.771839141845703,
      "activations/layer16_attention_weight_min": -28.239295959472656,
      "activations/layer17_attention_weight_max": 32.0106086730957,
      "activations/layer17_attention_weight_min": -25.626800537109375,
      "activations/layer18_attention_weight_max": 30.392179489135742,
      "activations/layer18_attention_weight_min": -21.43179702758789,
      "activations/layer19_attention_weight_max": 35.65500259399414,
      "activations/layer19_attention_weight_min": -31.799772262573242,
      "activations/layer1_attention_weight_max": 17.144643783569336,
      "activations/layer1_attention_weight_min": -14.092460632324219,
      "activations/layer20_attention_weight_max": 29.84218978881836,
      "activations/layer20_attention_weight_min": -24.520023345947266,
      "activations/layer21_attention_weight_max": 28.8731746673584,
      "activations/layer21_attention_weight_min": -23.495927810668945,
      "activations/layer22_attention_weight_max": 44.91183853149414,
      "activations/layer22_attention_weight_min": -28.7338924407959,
      "activations/layer23_attention_weight_max": 32.356971740722656,
      "activations/layer23_attention_weight_min": -24.845693588256836,
      "activations/layer2_attention_weight_max": 32.818878173828125,
      "activations/layer2_attention_weight_min": -33.15839385986328,
      "activations/layer3_attention_weight_max": 95.27930450439453,
      "activations/layer3_attention_weight_min": -99.34519958496094,
      "activations/layer4_attention_weight_max": 114.45552062988281,
      "activations/layer4_attention_weight_min": -112.8299560546875,
      "activations/layer5_attention_weight_max": 50.625953674316406,
      "activations/layer5_attention_weight_min": -63.537132263183594,
      "activations/layer6_attention_weight_max": 45.15393829345703,
      "activations/layer6_attention_weight_min": -48.73179626464844,
      "activations/layer7_attention_weight_max": 86.74994659423828,
      "activations/layer7_attention_weight_min": -90.64710235595703,
      "activations/layer8_attention_weight_max": 38.95026779174805,
      "activations/layer8_attention_weight_min": -43.8224983215332,
      "activations/layer9_attention_weight_max": 34.0765495300293,
      "activations/layer9_attention_weight_min": -33.210777282714844,
      "epoch": 10.09,
      "learning_rate": 8.581439393939394e-05,
      "loss": 2.7786,
      "step": 173600
    },
    {
      "activations/layer0_attention_weight_max": 15.936043739318848,
      "activations/layer0_attention_weight_min": -12.74659252166748,
      "activations/layer10_attention_weight_max": 32.280094146728516,
      "activations/layer10_attention_weight_min": -29.776351928710938,
      "activations/layer11_attention_weight_max": 31.52386474609375,
      "activations/layer11_attention_weight_min": -31.2508544921875,
      "activations/layer12_attention_weight_max": 25.64774513244629,
      "activations/layer12_attention_weight_min": -32.224300384521484,
      "activations/layer13_attention_weight_max": 36.07102584838867,
      "activations/layer13_attention_weight_min": -30.45151710510254,
      "activations/layer14_attention_weight_max": 42.914920806884766,
      "activations/layer14_attention_weight_min": -37.720821380615234,
      "activations/layer15_attention_weight_max": 34.808692932128906,
      "activations/layer15_attention_weight_min": -32.196006774902344,
      "activations/layer16_attention_weight_max": 29.72453498840332,
      "activations/layer16_attention_weight_min": -29.35093116760254,
      "activations/layer17_attention_weight_max": 31.907649993896484,
      "activations/layer17_attention_weight_min": -25.230121612548828,
      "activations/layer18_attention_weight_max": 30.78863525390625,
      "activations/layer18_attention_weight_min": -23.24278450012207,
      "activations/layer19_attention_weight_max": 32.738372802734375,
      "activations/layer19_attention_weight_min": -32.012691497802734,
      "activations/layer1_attention_weight_max": 17.340091705322266,
      "activations/layer1_attention_weight_min": -14.064620971679688,
      "activations/layer20_attention_weight_max": 31.32914924621582,
      "activations/layer20_attention_weight_min": -25.979623794555664,
      "activations/layer21_attention_weight_max": 29.183774948120117,
      "activations/layer21_attention_weight_min": -25.113895416259766,
      "activations/layer22_attention_weight_max": 46.00168228149414,
      "activations/layer22_attention_weight_min": -31.16941261291504,
      "activations/layer23_attention_weight_max": 34.40010070800781,
      "activations/layer23_attention_weight_min": -26.850975036621094,
      "activations/layer2_attention_weight_max": 33.37583923339844,
      "activations/layer2_attention_weight_min": -32.30509948730469,
      "activations/layer3_attention_weight_max": 98.63106536865234,
      "activations/layer3_attention_weight_min": -95.6444091796875,
      "activations/layer4_attention_weight_max": 108.83296966552734,
      "activations/layer4_attention_weight_min": -109.03197479248047,
      "activations/layer5_attention_weight_max": 51.49602508544922,
      "activations/layer5_attention_weight_min": -60.07182312011719,
      "activations/layer6_attention_weight_max": 48.118125915527344,
      "activations/layer6_attention_weight_min": -50.087303161621094,
      "activations/layer7_attention_weight_max": 89.28007507324219,
      "activations/layer7_attention_weight_min": -94.60481262207031,
      "activations/layer8_attention_weight_max": 43.47462844848633,
      "activations/layer8_attention_weight_min": -42.13446044921875,
      "activations/layer9_attention_weight_max": 31.679615020751953,
      "activations/layer9_attention_weight_min": -31.7706298828125,
      "epoch": 10.09,
      "learning_rate": 8.579545454545454e-05,
      "loss": 2.7653,
      "step": 173650
    },
    {
      "activations/layer0_attention_weight_max": 15.834051132202148,
      "activations/layer0_attention_weight_min": -12.341142654418945,
      "activations/layer10_attention_weight_max": 41.08968734741211,
      "activations/layer10_attention_weight_min": -35.05641555786133,
      "activations/layer11_attention_weight_max": 45.32396697998047,
      "activations/layer11_attention_weight_min": -37.19340515136719,
      "activations/layer12_attention_weight_max": 26.655000686645508,
      "activations/layer12_attention_weight_min": -28.10247230529785,
      "activations/layer13_attention_weight_max": 36.55125427246094,
      "activations/layer13_attention_weight_min": -33.820465087890625,
      "activations/layer14_attention_weight_max": 49.4224967956543,
      "activations/layer14_attention_weight_min": -43.658145904541016,
      "activations/layer15_attention_weight_max": 43.44575500488281,
      "activations/layer15_attention_weight_min": -30.804109573364258,
      "activations/layer16_attention_weight_max": 32.71163558959961,
      "activations/layer16_attention_weight_min": -28.489849090576172,
      "activations/layer17_attention_weight_max": 28.294647216796875,
      "activations/layer17_attention_weight_min": -26.808155059814453,
      "activations/layer18_attention_weight_max": 28.974218368530273,
      "activations/layer18_attention_weight_min": -23.37653350830078,
      "activations/layer19_attention_weight_max": 39.19310760498047,
      "activations/layer19_attention_weight_min": -29.600006103515625,
      "activations/layer1_attention_weight_max": 17.32634925842285,
      "activations/layer1_attention_weight_min": -16.65842628479004,
      "activations/layer20_attention_weight_max": 32.07283401489258,
      "activations/layer20_attention_weight_min": -25.157316207885742,
      "activations/layer21_attention_weight_max": 30.02433967590332,
      "activations/layer21_attention_weight_min": -25.30058479309082,
      "activations/layer22_attention_weight_max": 45.998207092285156,
      "activations/layer22_attention_weight_min": -29.613786697387695,
      "activations/layer23_attention_weight_max": 35.67971420288086,
      "activations/layer23_attention_weight_min": -24.15341567993164,
      "activations/layer2_attention_weight_max": 35.142887115478516,
      "activations/layer2_attention_weight_min": -36.70140838623047,
      "activations/layer3_attention_weight_max": 100.42701721191406,
      "activations/layer3_attention_weight_min": -105.05940246582031,
      "activations/layer4_attention_weight_max": 118.80150604248047,
      "activations/layer4_attention_weight_min": -117.51795959472656,
      "activations/layer5_attention_weight_max": 56.287071228027344,
      "activations/layer5_attention_weight_min": -63.085960388183594,
      "activations/layer6_attention_weight_max": 47.19900894165039,
      "activations/layer6_attention_weight_min": -51.553062438964844,
      "activations/layer7_attention_weight_max": 105.38096618652344,
      "activations/layer7_attention_weight_min": -95.69303131103516,
      "activations/layer8_attention_weight_max": 46.60878372192383,
      "activations/layer8_attention_weight_min": -44.95988464355469,
      "activations/layer9_attention_weight_max": 38.517364501953125,
      "activations/layer9_attention_weight_min": -36.73688888549805,
      "epoch": 10.09,
      "learning_rate": 8.577651515151514e-05,
      "loss": 2.7801,
      "step": 173700
    },
    {
      "activations/layer0_attention_weight_max": 16.2041072845459,
      "activations/layer0_attention_weight_min": -12.637900352478027,
      "activations/layer10_attention_weight_max": 31.52057456970215,
      "activations/layer10_attention_weight_min": -32.645103454589844,
      "activations/layer11_attention_weight_max": 30.6958065032959,
      "activations/layer11_attention_weight_min": -30.832754135131836,
      "activations/layer12_attention_weight_max": 22.39590835571289,
      "activations/layer12_attention_weight_min": -25.901844024658203,
      "activations/layer13_attention_weight_max": 37.6699333190918,
      "activations/layer13_attention_weight_min": -34.035369873046875,
      "activations/layer14_attention_weight_max": 41.93848419189453,
      "activations/layer14_attention_weight_min": -38.37517547607422,
      "activations/layer15_attention_weight_max": 32.20313262939453,
      "activations/layer15_attention_weight_min": -30.76482582092285,
      "activations/layer16_attention_weight_max": 29.70185089111328,
      "activations/layer16_attention_weight_min": -28.985898971557617,
      "activations/layer17_attention_weight_max": 30.16314697265625,
      "activations/layer17_attention_weight_min": -26.020244598388672,
      "activations/layer18_attention_weight_max": 31.10704231262207,
      "activations/layer18_attention_weight_min": -24.230438232421875,
      "activations/layer19_attention_weight_max": 30.717771530151367,
      "activations/layer19_attention_weight_min": -32.67428207397461,
      "activations/layer1_attention_weight_max": 16.155641555786133,
      "activations/layer1_attention_weight_min": -14.85589599609375,
      "activations/layer20_attention_weight_max": 27.877696990966797,
      "activations/layer20_attention_weight_min": -23.852949142456055,
      "activations/layer21_attention_weight_max": 26.657817840576172,
      "activations/layer21_attention_weight_min": -22.936187744140625,
      "activations/layer22_attention_weight_max": 42.305538177490234,
      "activations/layer22_attention_weight_min": -30.30598258972168,
      "activations/layer23_attention_weight_max": 34.22325134277344,
      "activations/layer23_attention_weight_min": -27.25722312927246,
      "activations/layer2_attention_weight_max": 33.30908966064453,
      "activations/layer2_attention_weight_min": -32.480369567871094,
      "activations/layer3_attention_weight_max": 94.64615631103516,
      "activations/layer3_attention_weight_min": -97.17269897460938,
      "activations/layer4_attention_weight_max": 113.0057373046875,
      "activations/layer4_attention_weight_min": -116.43526458740234,
      "activations/layer5_attention_weight_max": 50.704978942871094,
      "activations/layer5_attention_weight_min": -62.18012619018555,
      "activations/layer6_attention_weight_max": 43.68208694458008,
      "activations/layer6_attention_weight_min": -48.81454849243164,
      "activations/layer7_attention_weight_max": 83.1531982421875,
      "activations/layer7_attention_weight_min": -87.8424301147461,
      "activations/layer8_attention_weight_max": 37.87651824951172,
      "activations/layer8_attention_weight_min": -39.938507080078125,
      "activations/layer9_attention_weight_max": 31.400909423828125,
      "activations/layer9_attention_weight_min": -32.447017669677734,
      "epoch": 10.1,
      "learning_rate": 8.575757575757576e-05,
      "loss": 2.7833,
      "step": 173750
    },
    {
      "activations/layer0_attention_weight_max": 16.03944969177246,
      "activations/layer0_attention_weight_min": -12.64859676361084,
      "activations/layer10_attention_weight_max": 35.12377166748047,
      "activations/layer10_attention_weight_min": -32.67656326293945,
      "activations/layer11_attention_weight_max": 34.91445541381836,
      "activations/layer11_attention_weight_min": -32.214141845703125,
      "activations/layer12_attention_weight_max": 25.898799896240234,
      "activations/layer12_attention_weight_min": -25.987689971923828,
      "activations/layer13_attention_weight_max": 42.80571365356445,
      "activations/layer13_attention_weight_min": -34.336830139160156,
      "activations/layer14_attention_weight_max": 47.85834503173828,
      "activations/layer14_attention_weight_min": -38.53162384033203,
      "activations/layer15_attention_weight_max": 41.82575225830078,
      "activations/layer15_attention_weight_min": -34.324851989746094,
      "activations/layer16_attention_weight_max": 36.95560836791992,
      "activations/layer16_attention_weight_min": -28.246274948120117,
      "activations/layer17_attention_weight_max": 33.73180389404297,
      "activations/layer17_attention_weight_min": -25.32724952697754,
      "activations/layer18_attention_weight_max": 33.324771881103516,
      "activations/layer18_attention_weight_min": -22.857250213623047,
      "activations/layer19_attention_weight_max": 38.69285202026367,
      "activations/layer19_attention_weight_min": -30.58414077758789,
      "activations/layer1_attention_weight_max": 17.27252197265625,
      "activations/layer1_attention_weight_min": -15.223506927490234,
      "activations/layer20_attention_weight_max": 30.805400848388672,
      "activations/layer20_attention_weight_min": -24.48944664001465,
      "activations/layer21_attention_weight_max": 30.655517578125,
      "activations/layer21_attention_weight_min": -23.15499496459961,
      "activations/layer22_attention_weight_max": 54.866641998291016,
      "activations/layer22_attention_weight_min": -28.78740692138672,
      "activations/layer23_attention_weight_max": 41.44347381591797,
      "activations/layer23_attention_weight_min": -25.575637817382812,
      "activations/layer2_attention_weight_max": 32.19773483276367,
      "activations/layer2_attention_weight_min": -32.608375549316406,
      "activations/layer3_attention_weight_max": 91.14149475097656,
      "activations/layer3_attention_weight_min": -93.82335662841797,
      "activations/layer4_attention_weight_max": 110.84990692138672,
      "activations/layer4_attention_weight_min": -112.57905578613281,
      "activations/layer5_attention_weight_max": 52.285091400146484,
      "activations/layer5_attention_weight_min": -62.94025421142578,
      "activations/layer6_attention_weight_max": 45.92967987060547,
      "activations/layer6_attention_weight_min": -50.75563049316406,
      "activations/layer7_attention_weight_max": 95.84729766845703,
      "activations/layer7_attention_weight_min": -89.87592315673828,
      "activations/layer8_attention_weight_max": 39.932899475097656,
      "activations/layer8_attention_weight_min": -45.19124984741211,
      "activations/layer9_attention_weight_max": 35.071590423583984,
      "activations/layer9_attention_weight_min": -33.992122650146484,
      "epoch": 10.1,
      "learning_rate": 8.573863636363634e-05,
      "loss": 2.7795,
      "step": 173800
    },
    {
      "activations/layer0_attention_weight_max": 16.26513671875,
      "activations/layer0_attention_weight_min": -12.444074630737305,
      "activations/layer10_attention_weight_max": 33.369380950927734,
      "activations/layer10_attention_weight_min": -31.92613983154297,
      "activations/layer11_attention_weight_max": 32.89100646972656,
      "activations/layer11_attention_weight_min": -32.79986572265625,
      "activations/layer12_attention_weight_max": 23.130773544311523,
      "activations/layer12_attention_weight_min": -23.412353515625,
      "activations/layer13_attention_weight_max": 36.3765754699707,
      "activations/layer13_attention_weight_min": -32.59850311279297,
      "activations/layer14_attention_weight_max": 46.01957702636719,
      "activations/layer14_attention_weight_min": -34.78754806518555,
      "activations/layer15_attention_weight_max": 35.977108001708984,
      "activations/layer15_attention_weight_min": -29.87175178527832,
      "activations/layer16_attention_weight_max": 30.440425872802734,
      "activations/layer16_attention_weight_min": -26.258893966674805,
      "activations/layer17_attention_weight_max": 31.686832427978516,
      "activations/layer17_attention_weight_min": -24.996999740600586,
      "activations/layer18_attention_weight_max": 31.130435943603516,
      "activations/layer18_attention_weight_min": -22.09501075744629,
      "activations/layer19_attention_weight_max": 34.04110336303711,
      "activations/layer19_attention_weight_min": -29.43613624572754,
      "activations/layer1_attention_weight_max": 16.957992553710938,
      "activations/layer1_attention_weight_min": -13.958423614501953,
      "activations/layer20_attention_weight_max": 31.694623947143555,
      "activations/layer20_attention_weight_min": -26.124834060668945,
      "activations/layer21_attention_weight_max": 30.978660583496094,
      "activations/layer21_attention_weight_min": -23.15485191345215,
      "activations/layer22_attention_weight_max": 45.07135772705078,
      "activations/layer22_attention_weight_min": -28.908796310424805,
      "activations/layer23_attention_weight_max": 37.360530853271484,
      "activations/layer23_attention_weight_min": -26.767492294311523,
      "activations/layer2_attention_weight_max": 31.71792221069336,
      "activations/layer2_attention_weight_min": -33.860111236572266,
      "activations/layer3_attention_weight_max": 91.0770492553711,
      "activations/layer3_attention_weight_min": -94.87899017333984,
      "activations/layer4_attention_weight_max": 109.85137939453125,
      "activations/layer4_attention_weight_min": -109.43916320800781,
      "activations/layer5_attention_weight_max": 49.34438705444336,
      "activations/layer5_attention_weight_min": -61.35118103027344,
      "activations/layer6_attention_weight_max": 43.75162887573242,
      "activations/layer6_attention_weight_min": -48.021183013916016,
      "activations/layer7_attention_weight_max": 85.7216796875,
      "activations/layer7_attention_weight_min": -86.40616607666016,
      "activations/layer8_attention_weight_max": 41.268550872802734,
      "activations/layer8_attention_weight_min": -43.76527786254883,
      "activations/layer9_attention_weight_max": 33.786128997802734,
      "activations/layer9_attention_weight_min": -34.5705680847168,
      "epoch": 10.1,
      "learning_rate": 8.571969696969696e-05,
      "loss": 2.7904,
      "step": 173850
    },
    {
      "activations/layer0_attention_weight_max": 16.176651000976562,
      "activations/layer0_attention_weight_min": -12.420745849609375,
      "activations/layer10_attention_weight_max": 30.24698257446289,
      "activations/layer10_attention_weight_min": -30.474651336669922,
      "activations/layer11_attention_weight_max": 30.95663070678711,
      "activations/layer11_attention_weight_min": -30.346921920776367,
      "activations/layer12_attention_weight_max": 33.99896240234375,
      "activations/layer12_attention_weight_min": -25.87558364868164,
      "activations/layer13_attention_weight_max": 43.115379333496094,
      "activations/layer13_attention_weight_min": -34.285499572753906,
      "activations/layer14_attention_weight_max": 44.6069450378418,
      "activations/layer14_attention_weight_min": -38.30692672729492,
      "activations/layer15_attention_weight_max": 36.87652587890625,
      "activations/layer15_attention_weight_min": -32.3952751159668,
      "activations/layer16_attention_weight_max": 30.30812644958496,
      "activations/layer16_attention_weight_min": -29.623882293701172,
      "activations/layer17_attention_weight_max": 30.29570960998535,
      "activations/layer17_attention_weight_min": -25.757492065429688,
      "activations/layer18_attention_weight_max": 31.24172592163086,
      "activations/layer18_attention_weight_min": -22.118993759155273,
      "activations/layer19_attention_weight_max": 33.013641357421875,
      "activations/layer19_attention_weight_min": -29.438831329345703,
      "activations/layer1_attention_weight_max": 16.666955947875977,
      "activations/layer1_attention_weight_min": -14.266613960266113,
      "activations/layer20_attention_weight_max": 27.252506256103516,
      "activations/layer20_attention_weight_min": -25.258304595947266,
      "activations/layer21_attention_weight_max": 26.765995025634766,
      "activations/layer21_attention_weight_min": -23.875524520874023,
      "activations/layer22_attention_weight_max": 46.54651641845703,
      "activations/layer22_attention_weight_min": -28.802902221679688,
      "activations/layer23_attention_weight_max": 32.20601272583008,
      "activations/layer23_attention_weight_min": -24.844282150268555,
      "activations/layer2_attention_weight_max": 32.57614517211914,
      "activations/layer2_attention_weight_min": -31.27747344970703,
      "activations/layer3_attention_weight_max": 93.75475311279297,
      "activations/layer3_attention_weight_min": -90.86317443847656,
      "activations/layer4_attention_weight_max": 114.57193756103516,
      "activations/layer4_attention_weight_min": -109.02617645263672,
      "activations/layer5_attention_weight_max": 50.28672790527344,
      "activations/layer5_attention_weight_min": -59.28324890136719,
      "activations/layer6_attention_weight_max": 44.68606185913086,
      "activations/layer6_attention_weight_min": -47.571495056152344,
      "activations/layer7_attention_weight_max": 83.64229583740234,
      "activations/layer7_attention_weight_min": -83.64202880859375,
      "activations/layer8_attention_weight_max": 39.68718338012695,
      "activations/layer8_attention_weight_min": -40.34531021118164,
      "activations/layer9_attention_weight_max": 30.03841209411621,
      "activations/layer9_attention_weight_min": -31.877065658569336,
      "epoch": 10.1,
      "learning_rate": 8.570075757575757e-05,
      "loss": 2.7879,
      "step": 173900
    },
    {
      "activations/layer0_attention_weight_max": 15.387155532836914,
      "activations/layer0_attention_weight_min": -12.620267868041992,
      "activations/layer10_attention_weight_max": 33.97489929199219,
      "activations/layer10_attention_weight_min": -31.608116149902344,
      "activations/layer11_attention_weight_max": 38.15794372558594,
      "activations/layer11_attention_weight_min": -32.013431549072266,
      "activations/layer12_attention_weight_max": 23.310745239257812,
      "activations/layer12_attention_weight_min": -25.0323486328125,
      "activations/layer13_attention_weight_max": 37.46839904785156,
      "activations/layer13_attention_weight_min": -34.472259521484375,
      "activations/layer14_attention_weight_max": 42.695499420166016,
      "activations/layer14_attention_weight_min": -38.72114181518555,
      "activations/layer15_attention_weight_max": 34.74664306640625,
      "activations/layer15_attention_weight_min": -30.897977828979492,
      "activations/layer16_attention_weight_max": 29.739900588989258,
      "activations/layer16_attention_weight_min": -28.880098342895508,
      "activations/layer17_attention_weight_max": 30.542152404785156,
      "activations/layer17_attention_weight_min": -26.09630584716797,
      "activations/layer18_attention_weight_max": 29.515743255615234,
      "activations/layer18_attention_weight_min": -21.904701232910156,
      "activations/layer19_attention_weight_max": 32.026790618896484,
      "activations/layer19_attention_weight_min": -29.35467529296875,
      "activations/layer1_attention_weight_max": 16.60445785522461,
      "activations/layer1_attention_weight_min": -14.270830154418945,
      "activations/layer20_attention_weight_max": 27.845136642456055,
      "activations/layer20_attention_weight_min": -24.708330154418945,
      "activations/layer21_attention_weight_max": 28.144378662109375,
      "activations/layer21_attention_weight_min": -22.585153579711914,
      "activations/layer22_attention_weight_max": 42.09855270385742,
      "activations/layer22_attention_weight_min": -26.476947784423828,
      "activations/layer23_attention_weight_max": 37.0853385925293,
      "activations/layer23_attention_weight_min": -22.963817596435547,
      "activations/layer2_attention_weight_max": 34.21418762207031,
      "activations/layer2_attention_weight_min": -33.239837646484375,
      "activations/layer3_attention_weight_max": 99.89512634277344,
      "activations/layer3_attention_weight_min": -96.8726806640625,
      "activations/layer4_attention_weight_max": 120.82757568359375,
      "activations/layer4_attention_weight_min": -113.16920471191406,
      "activations/layer5_attention_weight_max": 54.396541595458984,
      "activations/layer5_attention_weight_min": -66.85633850097656,
      "activations/layer6_attention_weight_max": 46.069374084472656,
      "activations/layer6_attention_weight_min": -50.31193161010742,
      "activations/layer7_attention_weight_max": 88.0865707397461,
      "activations/layer7_attention_weight_min": -94.25692749023438,
      "activations/layer8_attention_weight_max": 42.88245391845703,
      "activations/layer8_attention_weight_min": -42.72412872314453,
      "activations/layer9_attention_weight_max": 34.30491256713867,
      "activations/layer9_attention_weight_min": -33.432254791259766,
      "epoch": 10.11,
      "learning_rate": 8.568181818181816e-05,
      "loss": 2.7965,
      "step": 173950
    },
    {
      "activations/layer0_attention_weight_max": 16.483657836914062,
      "activations/layer0_attention_weight_min": -11.581656455993652,
      "activations/layer10_attention_weight_max": 33.548553466796875,
      "activations/layer10_attention_weight_min": -32.35151290893555,
      "activations/layer11_attention_weight_max": 31.070537567138672,
      "activations/layer11_attention_weight_min": -29.783462524414062,
      "activations/layer12_attention_weight_max": 23.215591430664062,
      "activations/layer12_attention_weight_min": -26.336286544799805,
      "activations/layer13_attention_weight_max": 40.3983154296875,
      "activations/layer13_attention_weight_min": -36.17123031616211,
      "activations/layer14_attention_weight_max": 46.46793746948242,
      "activations/layer14_attention_weight_min": -38.229007720947266,
      "activations/layer15_attention_weight_max": 37.07550811767578,
      "activations/layer15_attention_weight_min": -31.6730899810791,
      "activations/layer16_attention_weight_max": 29.299789428710938,
      "activations/layer16_attention_weight_min": -28.0075740814209,
      "activations/layer17_attention_weight_max": 32.38787078857422,
      "activations/layer17_attention_weight_min": -25.801589965820312,
      "activations/layer18_attention_weight_max": 32.01731491088867,
      "activations/layer18_attention_weight_min": -22.69413185119629,
      "activations/layer19_attention_weight_max": 33.64963912963867,
      "activations/layer19_attention_weight_min": -31.443359375,
      "activations/layer1_attention_weight_max": 17.158523559570312,
      "activations/layer1_attention_weight_min": -13.853440284729004,
      "activations/layer20_attention_weight_max": 29.052478790283203,
      "activations/layer20_attention_weight_min": -25.763036727905273,
      "activations/layer21_attention_weight_max": 29.5596981048584,
      "activations/layer21_attention_weight_min": -25.179397583007812,
      "activations/layer22_attention_weight_max": 47.61573791503906,
      "activations/layer22_attention_weight_min": -29.200761795043945,
      "activations/layer23_attention_weight_max": 32.720924377441406,
      "activations/layer23_attention_weight_min": -23.29336166381836,
      "activations/layer2_attention_weight_max": 32.486915588378906,
      "activations/layer2_attention_weight_min": -32.09827423095703,
      "activations/layer3_attention_weight_max": 91.77838134765625,
      "activations/layer3_attention_weight_min": -93.2480697631836,
      "activations/layer4_attention_weight_max": 113.71815490722656,
      "activations/layer4_attention_weight_min": -111.4741439819336,
      "activations/layer5_attention_weight_max": 51.151729583740234,
      "activations/layer5_attention_weight_min": -61.41325378417969,
      "activations/layer6_attention_weight_max": 44.10744094848633,
      "activations/layer6_attention_weight_min": -51.454044342041016,
      "activations/layer7_attention_weight_max": 90.60800170898438,
      "activations/layer7_attention_weight_min": -88.4525375366211,
      "activations/layer8_attention_weight_max": 39.64744567871094,
      "activations/layer8_attention_weight_min": -42.32857131958008,
      "activations/layer9_attention_weight_max": 33.857730865478516,
      "activations/layer9_attention_weight_min": -33.9326171875,
      "epoch": 10.11,
      "learning_rate": 8.566325757575755e-05,
      "loss": 2.7785,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4729,
      "eval_samples_per_second": 506.791,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4729,
      "eval_openwebtext_samples_per_second": 506.791,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9057,
      "eval_wikitext_samples_per_second": 239.278,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_lambada_loss": 2.55078125,
      "eval_lambada_ppl": 12.817113241929157,
      "eval_lambada_runtime": 9.5102,
      "eval_lambada_samples_per_second": 511.978,
      "step": 174000
    },
    {
      "activations/layer0_attention_weight_max": 16.19451332092285,
      "activations/layer0_attention_weight_min": -12.283307075500488,
      "activations/layer10_attention_weight_max": 35.21049880981445,
      "activations/layer10_attention_weight_min": -34.40887451171875,
      "activations/layer11_attention_weight_max": 35.359092712402344,
      "activations/layer11_attention_weight_min": -31.21300506591797,
      "activations/layer12_attention_weight_max": 25.79848861694336,
      "activations/layer12_attention_weight_min": -25.791330337524414,
      "activations/layer13_attention_weight_max": 42.124732971191406,
      "activations/layer13_attention_weight_min": -33.978939056396484,
      "activations/layer14_attention_weight_max": 42.92144775390625,
      "activations/layer14_attention_weight_min": -37.02677917480469,
      "activations/layer15_attention_weight_max": 38.30433654785156,
      "activations/layer15_attention_weight_min": -31.132509231567383,
      "activations/layer16_attention_weight_max": 31.72337532043457,
      "activations/layer16_attention_weight_min": -27.177038192749023,
      "activations/layer17_attention_weight_max": 33.97003173828125,
      "activations/layer17_attention_weight_min": -24.08972930908203,
      "activations/layer18_attention_weight_max": 31.20928382873535,
      "activations/layer18_attention_weight_min": -21.58515167236328,
      "activations/layer19_attention_weight_max": 33.015010833740234,
      "activations/layer19_attention_weight_min": -30.150236129760742,
      "activations/layer1_attention_weight_max": 17.33938217163086,
      "activations/layer1_attention_weight_min": -15.820887565612793,
      "activations/layer20_attention_weight_max": 28.615386962890625,
      "activations/layer20_attention_weight_min": -24.4978084564209,
      "activations/layer21_attention_weight_max": 27.018444061279297,
      "activations/layer21_attention_weight_min": -23.757097244262695,
      "activations/layer22_attention_weight_max": 47.58222961425781,
      "activations/layer22_attention_weight_min": -29.28617286682129,
      "activations/layer23_attention_weight_max": 36.283546447753906,
      "activations/layer23_attention_weight_min": -22.26354217529297,
      "activations/layer2_attention_weight_max": 32.771095275878906,
      "activations/layer2_attention_weight_min": -33.37702941894531,
      "activations/layer3_attention_weight_max": 98.97577667236328,
      "activations/layer3_attention_weight_min": -98.8599853515625,
      "activations/layer4_attention_weight_max": 124.53065490722656,
      "activations/layer4_attention_weight_min": -115.49260711669922,
      "activations/layer5_attention_weight_max": 56.475521087646484,
      "activations/layer5_attention_weight_min": -67.87013244628906,
      "activations/layer6_attention_weight_max": 48.56282043457031,
      "activations/layer6_attention_weight_min": -51.605716705322266,
      "activations/layer7_attention_weight_max": 90.57886505126953,
      "activations/layer7_attention_weight_min": -90.92674255371094,
      "activations/layer8_attention_weight_max": 41.78907012939453,
      "activations/layer8_attention_weight_min": -41.77262878417969,
      "activations/layer9_attention_weight_max": 36.45292663574219,
      "activations/layer9_attention_weight_min": -34.777076721191406,
      "epoch": 10.11,
      "learning_rate": 8.564431818181817e-05,
      "loss": 2.7954,
      "step": 174050
    },
    {
      "activations/layer0_attention_weight_max": 15.803088188171387,
      "activations/layer0_attention_weight_min": -12.45429515838623,
      "activations/layer10_attention_weight_max": 31.19272232055664,
      "activations/layer10_attention_weight_min": -28.830535888671875,
      "activations/layer11_attention_weight_max": 29.024673461914062,
      "activations/layer11_attention_weight_min": -28.744651794433594,
      "activations/layer12_attention_weight_max": 23.177820205688477,
      "activations/layer12_attention_weight_min": -24.032983779907227,
      "activations/layer13_attention_weight_max": 36.136940002441406,
      "activations/layer13_attention_weight_min": -33.061073303222656,
      "activations/layer14_attention_weight_max": 42.03769302368164,
      "activations/layer14_attention_weight_min": -33.41122817993164,
      "activations/layer15_attention_weight_max": 35.82209396362305,
      "activations/layer15_attention_weight_min": -29.426250457763672,
      "activations/layer16_attention_weight_max": 32.0207633972168,
      "activations/layer16_attention_weight_min": -26.34397315979004,
      "activations/layer17_attention_weight_max": 34.371944427490234,
      "activations/layer17_attention_weight_min": -26.051992416381836,
      "activations/layer18_attention_weight_max": 32.00912094116211,
      "activations/layer18_attention_weight_min": -24.238025665283203,
      "activations/layer19_attention_weight_max": 41.80512619018555,
      "activations/layer19_attention_weight_min": -31.708677291870117,
      "activations/layer1_attention_weight_max": 16.34038734436035,
      "activations/layer1_attention_weight_min": -14.02177906036377,
      "activations/layer20_attention_weight_max": 38.02626037597656,
      "activations/layer20_attention_weight_min": -24.621337890625,
      "activations/layer21_attention_weight_max": 30.573184967041016,
      "activations/layer21_attention_weight_min": -24.073991775512695,
      "activations/layer22_attention_weight_max": 48.47321319580078,
      "activations/layer22_attention_weight_min": -30.040422439575195,
      "activations/layer23_attention_weight_max": 37.50013732910156,
      "activations/layer23_attention_weight_min": -26.966222763061523,
      "activations/layer2_attention_weight_max": 31.578596115112305,
      "activations/layer2_attention_weight_min": -31.757659912109375,
      "activations/layer3_attention_weight_max": 98.30402374267578,
      "activations/layer3_attention_weight_min": -91.15435791015625,
      "activations/layer4_attention_weight_max": 119.82097625732422,
      "activations/layer4_attention_weight_min": -104.35475158691406,
      "activations/layer5_attention_weight_max": 50.54791259765625,
      "activations/layer5_attention_weight_min": -59.15465545654297,
      "activations/layer6_attention_weight_max": 43.2332878112793,
      "activations/layer6_attention_weight_min": -49.01628875732422,
      "activations/layer7_attention_weight_max": 83.19143676757812,
      "activations/layer7_attention_weight_min": -84.29132843017578,
      "activations/layer8_attention_weight_max": 35.66567611694336,
      "activations/layer8_attention_weight_min": -35.84543228149414,
      "activations/layer9_attention_weight_max": 29.039508819580078,
      "activations/layer9_attention_weight_min": -31.379724502563477,
      "epoch": 10.12,
      "learning_rate": 8.562537878787878e-05,
      "loss": 2.7645,
      "step": 174100
    },
    {
      "activations/layer0_attention_weight_max": 15.303313255310059,
      "activations/layer0_attention_weight_min": -12.351359367370605,
      "activations/layer10_attention_weight_max": 31.370798110961914,
      "activations/layer10_attention_weight_min": -33.98344802856445,
      "activations/layer11_attention_weight_max": 31.80666160583496,
      "activations/layer11_attention_weight_min": -31.231327056884766,
      "activations/layer12_attention_weight_max": 25.35346221923828,
      "activations/layer12_attention_weight_min": -24.283109664916992,
      "activations/layer13_attention_weight_max": 40.39778137207031,
      "activations/layer13_attention_weight_min": -33.651920318603516,
      "activations/layer14_attention_weight_max": 49.38148880004883,
      "activations/layer14_attention_weight_min": -39.53389358520508,
      "activations/layer15_attention_weight_max": 34.98601531982422,
      "activations/layer15_attention_weight_min": -31.351882934570312,
      "activations/layer16_attention_weight_max": 33.052127838134766,
      "activations/layer16_attention_weight_min": -27.80203628540039,
      "activations/layer17_attention_weight_max": 33.259037017822266,
      "activations/layer17_attention_weight_min": -25.098960876464844,
      "activations/layer18_attention_weight_max": 32.28055953979492,
      "activations/layer18_attention_weight_min": -23.74064826965332,
      "activations/layer19_attention_weight_max": 37.02692794799805,
      "activations/layer19_attention_weight_min": -31.658117294311523,
      "activations/layer1_attention_weight_max": 17.825836181640625,
      "activations/layer1_attention_weight_min": -15.531708717346191,
      "activations/layer20_attention_weight_max": 31.72667694091797,
      "activations/layer20_attention_weight_min": -26.94036293029785,
      "activations/layer21_attention_weight_max": 30.74100685119629,
      "activations/layer21_attention_weight_min": -25.97243309020996,
      "activations/layer22_attention_weight_max": 42.00175476074219,
      "activations/layer22_attention_weight_min": -32.096923828125,
      "activations/layer23_attention_weight_max": 38.03844451904297,
      "activations/layer23_attention_weight_min": -27.12267303466797,
      "activations/layer2_attention_weight_max": 32.31972885131836,
      "activations/layer2_attention_weight_min": -31.886404037475586,
      "activations/layer3_attention_weight_max": 92.84549713134766,
      "activations/layer3_attention_weight_min": -91.65385437011719,
      "activations/layer4_attention_weight_max": 113.47322845458984,
      "activations/layer4_attention_weight_min": -111.59122467041016,
      "activations/layer5_attention_weight_max": 52.64056396484375,
      "activations/layer5_attention_weight_min": -62.7121696472168,
      "activations/layer6_attention_weight_max": 45.99454116821289,
      "activations/layer6_attention_weight_min": -48.33634948730469,
      "activations/layer7_attention_weight_max": 91.99398803710938,
      "activations/layer7_attention_weight_min": -86.44351196289062,
      "activations/layer8_attention_weight_max": 38.23227310180664,
      "activations/layer8_attention_weight_min": -39.1090202331543,
      "activations/layer9_attention_weight_max": 30.8538761138916,
      "activations/layer9_attention_weight_min": -30.815387725830078,
      "epoch": 10.12,
      "learning_rate": 8.560643939393939e-05,
      "loss": 2.7749,
      "step": 174150
    },
    {
      "activations/layer0_attention_weight_max": 15.793055534362793,
      "activations/layer0_attention_weight_min": -12.167938232421875,
      "activations/layer10_attention_weight_max": 33.840003967285156,
      "activations/layer10_attention_weight_min": -32.1707649230957,
      "activations/layer11_attention_weight_max": 34.26385498046875,
      "activations/layer11_attention_weight_min": -30.486515045166016,
      "activations/layer12_attention_weight_max": 23.78807258605957,
      "activations/layer12_attention_weight_min": -25.14073944091797,
      "activations/layer13_attention_weight_max": 38.822288513183594,
      "activations/layer13_attention_weight_min": -35.0692253112793,
      "activations/layer14_attention_weight_max": 43.69056701660156,
      "activations/layer14_attention_weight_min": -40.59689712524414,
      "activations/layer15_attention_weight_max": 35.941558837890625,
      "activations/layer15_attention_weight_min": -29.8396053314209,
      "activations/layer16_attention_weight_max": 31.522769927978516,
      "activations/layer16_attention_weight_min": -28.794342041015625,
      "activations/layer17_attention_weight_max": 33.7768440246582,
      "activations/layer17_attention_weight_min": -28.356346130371094,
      "activations/layer18_attention_weight_max": 32.868865966796875,
      "activations/layer18_attention_weight_min": -22.82528305053711,
      "activations/layer19_attention_weight_max": 33.78584671020508,
      "activations/layer19_attention_weight_min": -31.494871139526367,
      "activations/layer1_attention_weight_max": 16.348730087280273,
      "activations/layer1_attention_weight_min": -14.86842155456543,
      "activations/layer20_attention_weight_max": 29.68919563293457,
      "activations/layer20_attention_weight_min": -24.922496795654297,
      "activations/layer21_attention_weight_max": 30.658748626708984,
      "activations/layer21_attention_weight_min": -25.145463943481445,
      "activations/layer22_attention_weight_max": 45.7187614440918,
      "activations/layer22_attention_weight_min": -29.93543243408203,
      "activations/layer23_attention_weight_max": 36.099449157714844,
      "activations/layer23_attention_weight_min": -26.23578453063965,
      "activations/layer2_attention_weight_max": 32.371551513671875,
      "activations/layer2_attention_weight_min": -32.26805877685547,
      "activations/layer3_attention_weight_max": 94.03445434570312,
      "activations/layer3_attention_weight_min": -93.1820297241211,
      "activations/layer4_attention_weight_max": 114.27009582519531,
      "activations/layer4_attention_weight_min": -111.9879379272461,
      "activations/layer5_attention_weight_max": 55.992515563964844,
      "activations/layer5_attention_weight_min": -62.90896224975586,
      "activations/layer6_attention_weight_max": 45.45746994018555,
      "activations/layer6_attention_weight_min": -48.0390510559082,
      "activations/layer7_attention_weight_max": 89.70297241210938,
      "activations/layer7_attention_weight_min": -90.44390106201172,
      "activations/layer8_attention_weight_max": 41.003395080566406,
      "activations/layer8_attention_weight_min": -41.53757095336914,
      "activations/layer9_attention_weight_max": 37.85829162597656,
      "activations/layer9_attention_weight_min": -34.42985916137695,
      "epoch": 10.12,
      "learning_rate": 8.558749999999999e-05,
      "loss": 2.7803,
      "step": 174200
    },
    {
      "activations/layer0_attention_weight_max": 16.231853485107422,
      "activations/layer0_attention_weight_min": -11.321163177490234,
      "activations/layer10_attention_weight_max": 33.18696594238281,
      "activations/layer10_attention_weight_min": -30.58788299560547,
      "activations/layer11_attention_weight_max": 35.766990661621094,
      "activations/layer11_attention_weight_min": -30.89971160888672,
      "activations/layer12_attention_weight_max": 24.07975959777832,
      "activations/layer12_attention_weight_min": -28.219913482666016,
      "activations/layer13_attention_weight_max": 36.97126388549805,
      "activations/layer13_attention_weight_min": -33.715938568115234,
      "activations/layer14_attention_weight_max": 45.59821701049805,
      "activations/layer14_attention_weight_min": -37.51444625854492,
      "activations/layer15_attention_weight_max": 35.80833435058594,
      "activations/layer15_attention_weight_min": -31.145496368408203,
      "activations/layer16_attention_weight_max": 33.841487884521484,
      "activations/layer16_attention_weight_min": -26.677473068237305,
      "activations/layer17_attention_weight_max": 32.34136199951172,
      "activations/layer17_attention_weight_min": -24.65790367126465,
      "activations/layer18_attention_weight_max": 30.689697265625,
      "activations/layer18_attention_weight_min": -21.1854190826416,
      "activations/layer19_attention_weight_max": 40.15321731567383,
      "activations/layer19_attention_weight_min": -29.916839599609375,
      "activations/layer1_attention_weight_max": 16.724365234375,
      "activations/layer1_attention_weight_min": -14.19039249420166,
      "activations/layer20_attention_weight_max": 31.430418014526367,
      "activations/layer20_attention_weight_min": -23.772430419921875,
      "activations/layer21_attention_weight_max": 33.82917022705078,
      "activations/layer21_attention_weight_min": -22.643091201782227,
      "activations/layer22_attention_weight_max": 43.83932876586914,
      "activations/layer22_attention_weight_min": -27.74117088317871,
      "activations/layer23_attention_weight_max": 43.23997497558594,
      "activations/layer23_attention_weight_min": -22.719160079956055,
      "activations/layer2_attention_weight_max": 31.899795532226562,
      "activations/layer2_attention_weight_min": -34.89083480834961,
      "activations/layer3_attention_weight_max": 94.39055633544922,
      "activations/layer3_attention_weight_min": -100.32209777832031,
      "activations/layer4_attention_weight_max": 120.16991424560547,
      "activations/layer4_attention_weight_min": -117.99442291259766,
      "activations/layer5_attention_weight_max": 54.76374816894531,
      "activations/layer5_attention_weight_min": -61.490333557128906,
      "activations/layer6_attention_weight_max": 45.9847526550293,
      "activations/layer6_attention_weight_min": -50.02366256713867,
      "activations/layer7_attention_weight_max": 93.15919494628906,
      "activations/layer7_attention_weight_min": -96.38925170898438,
      "activations/layer8_attention_weight_max": 41.58748245239258,
      "activations/layer8_attention_weight_min": -39.3569221496582,
      "activations/layer9_attention_weight_max": 34.417415618896484,
      "activations/layer9_attention_weight_min": -32.73032760620117,
      "epoch": 10.12,
      "learning_rate": 8.55685606060606e-05,
      "loss": 2.8003,
      "step": 174250
    },
    {
      "activations/layer0_attention_weight_max": 15.853646278381348,
      "activations/layer0_attention_weight_min": -12.867186546325684,
      "activations/layer10_attention_weight_max": 33.44239044189453,
      "activations/layer10_attention_weight_min": -34.52496337890625,
      "activations/layer11_attention_weight_max": 31.270980834960938,
      "activations/layer11_attention_weight_min": -31.97364044189453,
      "activations/layer12_attention_weight_max": 23.082345962524414,
      "activations/layer12_attention_weight_min": -25.24148941040039,
      "activations/layer13_attention_weight_max": 37.54817581176758,
      "activations/layer13_attention_weight_min": -31.600452423095703,
      "activations/layer14_attention_weight_max": 40.80318069458008,
      "activations/layer14_attention_weight_min": -35.45109558105469,
      "activations/layer15_attention_weight_max": 36.94131851196289,
      "activations/layer15_attention_weight_min": -30.805147171020508,
      "activations/layer16_attention_weight_max": 33.89403533935547,
      "activations/layer16_attention_weight_min": -27.74480628967285,
      "activations/layer17_attention_weight_max": 35.27290725708008,
      "activations/layer17_attention_weight_min": -24.362260818481445,
      "activations/layer18_attention_weight_max": 32.638057708740234,
      "activations/layer18_attention_weight_min": -21.603721618652344,
      "activations/layer19_attention_weight_max": 35.71782302856445,
      "activations/layer19_attention_weight_min": -30.01722526550293,
      "activations/layer1_attention_weight_max": 16.17898941040039,
      "activations/layer1_attention_weight_min": -14.765690803527832,
      "activations/layer20_attention_weight_max": 33.51434326171875,
      "activations/layer20_attention_weight_min": -24.405317306518555,
      "activations/layer21_attention_weight_max": 33.29785919189453,
      "activations/layer21_attention_weight_min": -22.22536277770996,
      "activations/layer22_attention_weight_max": 48.37420654296875,
      "activations/layer22_attention_weight_min": -29.204994201660156,
      "activations/layer23_attention_weight_max": 35.73069763183594,
      "activations/layer23_attention_weight_min": -24.51708221435547,
      "activations/layer2_attention_weight_max": 30.218852996826172,
      "activations/layer2_attention_weight_min": -32.15978240966797,
      "activations/layer3_attention_weight_max": 91.5168685913086,
      "activations/layer3_attention_weight_min": -92.97848510742188,
      "activations/layer4_attention_weight_max": 111.94225311279297,
      "activations/layer4_attention_weight_min": -107.40520477294922,
      "activations/layer5_attention_weight_max": 52.03326416015625,
      "activations/layer5_attention_weight_min": -59.90543746948242,
      "activations/layer6_attention_weight_max": 47.47389221191406,
      "activations/layer6_attention_weight_min": -47.14596176147461,
      "activations/layer7_attention_weight_max": 94.10991668701172,
      "activations/layer7_attention_weight_min": -89.46601867675781,
      "activations/layer8_attention_weight_max": 41.23572540283203,
      "activations/layer8_attention_weight_min": -40.22742462158203,
      "activations/layer9_attention_weight_max": 34.6972541809082,
      "activations/layer9_attention_weight_min": -34.77075958251953,
      "epoch": 10.13,
      "learning_rate": 8.55496212121212e-05,
      "loss": 2.7778,
      "step": 174300
    },
    {
      "activations/layer0_attention_weight_max": 15.812612533569336,
      "activations/layer0_attention_weight_min": -13.194504737854004,
      "activations/layer10_attention_weight_max": 36.33671188354492,
      "activations/layer10_attention_weight_min": -33.34249496459961,
      "activations/layer11_attention_weight_max": 32.8139762878418,
      "activations/layer11_attention_weight_min": -31.597599029541016,
      "activations/layer12_attention_weight_max": 25.784852981567383,
      "activations/layer12_attention_weight_min": -25.503934860229492,
      "activations/layer13_attention_weight_max": 41.405086517333984,
      "activations/layer13_attention_weight_min": -35.88047790527344,
      "activations/layer14_attention_weight_max": 44.71007537841797,
      "activations/layer14_attention_weight_min": -36.63526153564453,
      "activations/layer15_attention_weight_max": 35.54518508911133,
      "activations/layer15_attention_weight_min": -32.287288665771484,
      "activations/layer16_attention_weight_max": 33.297367095947266,
      "activations/layer16_attention_weight_min": -27.81059455871582,
      "activations/layer17_attention_weight_max": 31.06830596923828,
      "activations/layer17_attention_weight_min": -26.65262222290039,
      "activations/layer18_attention_weight_max": 28.45838737487793,
      "activations/layer18_attention_weight_min": -22.05862808227539,
      "activations/layer19_attention_weight_max": 37.23185348510742,
      "activations/layer19_attention_weight_min": -31.72450065612793,
      "activations/layer1_attention_weight_max": 18.72284698486328,
      "activations/layer1_attention_weight_min": -15.474347114562988,
      "activations/layer20_attention_weight_max": 30.116680145263672,
      "activations/layer20_attention_weight_min": -25.52059555053711,
      "activations/layer21_attention_weight_max": 27.736875534057617,
      "activations/layer21_attention_weight_min": -22.700822830200195,
      "activations/layer22_attention_weight_max": 48.19364547729492,
      "activations/layer22_attention_weight_min": -29.734975814819336,
      "activations/layer23_attention_weight_max": 36.84455490112305,
      "activations/layer23_attention_weight_min": -23.557104110717773,
      "activations/layer2_attention_weight_max": 32.33016586303711,
      "activations/layer2_attention_weight_min": -33.19964599609375,
      "activations/layer3_attention_weight_max": 92.9443130493164,
      "activations/layer3_attention_weight_min": -93.41905212402344,
      "activations/layer4_attention_weight_max": 113.36116027832031,
      "activations/layer4_attention_weight_min": -116.2517318725586,
      "activations/layer5_attention_weight_max": 52.12169647216797,
      "activations/layer5_attention_weight_min": -66.11546325683594,
      "activations/layer6_attention_weight_max": 47.48081588745117,
      "activations/layer6_attention_weight_min": -49.10456848144531,
      "activations/layer7_attention_weight_max": 92.95660400390625,
      "activations/layer7_attention_weight_min": -89.28982543945312,
      "activations/layer8_attention_weight_max": 40.38431167602539,
      "activations/layer8_attention_weight_min": -39.64973831176758,
      "activations/layer9_attention_weight_max": 35.62996292114258,
      "activations/layer9_attention_weight_min": -33.36472702026367,
      "epoch": 10.13,
      "learning_rate": 8.553068181818182e-05,
      "loss": 2.7824,
      "step": 174350
    },
    {
      "activations/layer0_attention_weight_max": 15.613104820251465,
      "activations/layer0_attention_weight_min": -12.600790023803711,
      "activations/layer10_attention_weight_max": 32.87235641479492,
      "activations/layer10_attention_weight_min": -33.56214904785156,
      "activations/layer11_attention_weight_max": 35.07075500488281,
      "activations/layer11_attention_weight_min": -31.625202178955078,
      "activations/layer12_attention_weight_max": 22.843523025512695,
      "activations/layer12_attention_weight_min": -23.501930236816406,
      "activations/layer13_attention_weight_max": 37.65106201171875,
      "activations/layer13_attention_weight_min": -32.15258026123047,
      "activations/layer14_attention_weight_max": 44.546478271484375,
      "activations/layer14_attention_weight_min": -37.44369125366211,
      "activations/layer15_attention_weight_max": 39.905921936035156,
      "activations/layer15_attention_weight_min": -31.54979705810547,
      "activations/layer16_attention_weight_max": 34.38993453979492,
      "activations/layer16_attention_weight_min": -26.390033721923828,
      "activations/layer17_attention_weight_max": 33.77119064331055,
      "activations/layer17_attention_weight_min": -26.162967681884766,
      "activations/layer18_attention_weight_max": 37.03278732299805,
      "activations/layer18_attention_weight_min": -23.349281311035156,
      "activations/layer19_attention_weight_max": 39.000038146972656,
      "activations/layer19_attention_weight_min": -30.700275421142578,
      "activations/layer1_attention_weight_max": 17.141504287719727,
      "activations/layer1_attention_weight_min": -13.304500579833984,
      "activations/layer20_attention_weight_max": 31.369701385498047,
      "activations/layer20_attention_weight_min": -24.019989013671875,
      "activations/layer21_attention_weight_max": 33.03401565551758,
      "activations/layer21_attention_weight_min": -22.09478187561035,
      "activations/layer22_attention_weight_max": 46.85331344604492,
      "activations/layer22_attention_weight_min": -29.23133659362793,
      "activations/layer23_attention_weight_max": 37.509544372558594,
      "activations/layer23_attention_weight_min": -23.056806564331055,
      "activations/layer2_attention_weight_max": 31.878036499023438,
      "activations/layer2_attention_weight_min": -31.600772857666016,
      "activations/layer3_attention_weight_max": 95.61762237548828,
      "activations/layer3_attention_weight_min": -93.06822204589844,
      "activations/layer4_attention_weight_max": 109.90101623535156,
      "activations/layer4_attention_weight_min": -111.98249816894531,
      "activations/layer5_attention_weight_max": 53.4389762878418,
      "activations/layer5_attention_weight_min": -60.337501525878906,
      "activations/layer6_attention_weight_max": 45.0927619934082,
      "activations/layer6_attention_weight_min": -47.68961715698242,
      "activations/layer7_attention_weight_max": 89.88591003417969,
      "activations/layer7_attention_weight_min": -87.17779541015625,
      "activations/layer8_attention_weight_max": 39.41718292236328,
      "activations/layer8_attention_weight_min": -38.620513916015625,
      "activations/layer9_attention_weight_max": 32.65029525756836,
      "activations/layer9_attention_weight_min": -32.296714782714844,
      "epoch": 10.13,
      "learning_rate": 8.551174242424242e-05,
      "loss": 2.7933,
      "step": 174400
    },
    {
      "activations/layer0_attention_weight_max": 14.917630195617676,
      "activations/layer0_attention_weight_min": -12.94843864440918,
      "activations/layer10_attention_weight_max": 33.61587905883789,
      "activations/layer10_attention_weight_min": -32.63245391845703,
      "activations/layer11_attention_weight_max": 31.65829849243164,
      "activations/layer11_attention_weight_min": -31.40411376953125,
      "activations/layer12_attention_weight_max": 23.490406036376953,
      "activations/layer12_attention_weight_min": -25.707529067993164,
      "activations/layer13_attention_weight_max": 36.17695236206055,
      "activations/layer13_attention_weight_min": -32.652137756347656,
      "activations/layer14_attention_weight_max": 46.16261291503906,
      "activations/layer14_attention_weight_min": -38.36888122558594,
      "activations/layer15_attention_weight_max": 36.198509216308594,
      "activations/layer15_attention_weight_min": -31.891210556030273,
      "activations/layer16_attention_weight_max": 29.190189361572266,
      "activations/layer16_attention_weight_min": -26.270648956298828,
      "activations/layer17_attention_weight_max": 31.221052169799805,
      "activations/layer17_attention_weight_min": -24.42456817626953,
      "activations/layer18_attention_weight_max": 28.250642776489258,
      "activations/layer18_attention_weight_min": -23.137224197387695,
      "activations/layer19_attention_weight_max": 31.790760040283203,
      "activations/layer19_attention_weight_min": -30.3107852935791,
      "activations/layer1_attention_weight_max": 16.562326431274414,
      "activations/layer1_attention_weight_min": -14.220858573913574,
      "activations/layer20_attention_weight_max": 30.654584884643555,
      "activations/layer20_attention_weight_min": -26.483198165893555,
      "activations/layer21_attention_weight_max": 28.628843307495117,
      "activations/layer21_attention_weight_min": -24.85173225402832,
      "activations/layer22_attention_weight_max": 44.30965042114258,
      "activations/layer22_attention_weight_min": -30.82965087890625,
      "activations/layer23_attention_weight_max": 32.440399169921875,
      "activations/layer23_attention_weight_min": -24.420690536499023,
      "activations/layer2_attention_weight_max": 31.61335563659668,
      "activations/layer2_attention_weight_min": -32.30814743041992,
      "activations/layer3_attention_weight_max": 93.51163482666016,
      "activations/layer3_attention_weight_min": -93.72774505615234,
      "activations/layer4_attention_weight_max": 114.14222717285156,
      "activations/layer4_attention_weight_min": -111.0279541015625,
      "activations/layer5_attention_weight_max": 52.750362396240234,
      "activations/layer5_attention_weight_min": -59.98692321777344,
      "activations/layer6_attention_weight_max": 45.33556365966797,
      "activations/layer6_attention_weight_min": -47.63768768310547,
      "activations/layer7_attention_weight_max": 88.59962463378906,
      "activations/layer7_attention_weight_min": -87.16566467285156,
      "activations/layer8_attention_weight_max": 42.70912551879883,
      "activations/layer8_attention_weight_min": -41.03741455078125,
      "activations/layer9_attention_weight_max": 34.83890914916992,
      "activations/layer9_attention_weight_min": -33.54515838623047,
      "epoch": 10.14,
      "learning_rate": 8.549280303030302e-05,
      "loss": 2.7847,
      "step": 174450
    },
    {
      "activations/layer0_attention_weight_max": 15.624369621276855,
      "activations/layer0_attention_weight_min": -13.087489128112793,
      "activations/layer10_attention_weight_max": 30.832340240478516,
      "activations/layer10_attention_weight_min": -30.163036346435547,
      "activations/layer11_attention_weight_max": 30.700973510742188,
      "activations/layer11_attention_weight_min": -31.368637084960938,
      "activations/layer12_attention_weight_max": 24.30305290222168,
      "activations/layer12_attention_weight_min": -25.984045028686523,
      "activations/layer13_attention_weight_max": 37.4340934753418,
      "activations/layer13_attention_weight_min": -31.65157699584961,
      "activations/layer14_attention_weight_max": 46.97188186645508,
      "activations/layer14_attention_weight_min": -38.74198532104492,
      "activations/layer15_attention_weight_max": 36.150672912597656,
      "activations/layer15_attention_weight_min": -30.45783805847168,
      "activations/layer16_attention_weight_max": 31.942644119262695,
      "activations/layer16_attention_weight_min": -28.366533279418945,
      "activations/layer17_attention_weight_max": 33.673797607421875,
      "activations/layer17_attention_weight_min": -23.92022705078125,
      "activations/layer18_attention_weight_max": 32.500736236572266,
      "activations/layer18_attention_weight_min": -20.91767692565918,
      "activations/layer19_attention_weight_max": 40.21017074584961,
      "activations/layer19_attention_weight_min": -30.6197452545166,
      "activations/layer1_attention_weight_max": 16.6158504486084,
      "activations/layer1_attention_weight_min": -13.219583511352539,
      "activations/layer20_attention_weight_max": 33.75246810913086,
      "activations/layer20_attention_weight_min": -24.28827667236328,
      "activations/layer21_attention_weight_max": 34.035858154296875,
      "activations/layer21_attention_weight_min": -23.716650009155273,
      "activations/layer22_attention_weight_max": 46.48911666870117,
      "activations/layer22_attention_weight_min": -29.8590145111084,
      "activations/layer23_attention_weight_max": 40.82407760620117,
      "activations/layer23_attention_weight_min": -26.18254852294922,
      "activations/layer2_attention_weight_max": 31.933307647705078,
      "activations/layer2_attention_weight_min": -31.7507381439209,
      "activations/layer3_attention_weight_max": 92.94563293457031,
      "activations/layer3_attention_weight_min": -95.2872314453125,
      "activations/layer4_attention_weight_max": 112.824462890625,
      "activations/layer4_attention_weight_min": -110.255126953125,
      "activations/layer5_attention_weight_max": 52.77412414550781,
      "activations/layer5_attention_weight_min": -60.19493865966797,
      "activations/layer6_attention_weight_max": 45.64372253417969,
      "activations/layer6_attention_weight_min": -49.320377349853516,
      "activations/layer7_attention_weight_max": 88.14402770996094,
      "activations/layer7_attention_weight_min": -86.80052185058594,
      "activations/layer8_attention_weight_max": 38.16020965576172,
      "activations/layer8_attention_weight_min": -40.92396926879883,
      "activations/layer9_attention_weight_max": 32.59950256347656,
      "activations/layer9_attention_weight_min": -31.939395904541016,
      "epoch": 10.14,
      "learning_rate": 8.547386363636364e-05,
      "loss": 2.7908,
      "step": 174500
    },
    {
      "activations/layer0_attention_weight_max": 16.035459518432617,
      "activations/layer0_attention_weight_min": -11.878055572509766,
      "activations/layer10_attention_weight_max": 39.466400146484375,
      "activations/layer10_attention_weight_min": -36.261962890625,
      "activations/layer11_attention_weight_max": 34.973960876464844,
      "activations/layer11_attention_weight_min": -32.57486343383789,
      "activations/layer12_attention_weight_max": 27.088010787963867,
      "activations/layer12_attention_weight_min": -26.40166473388672,
      "activations/layer13_attention_weight_max": 42.01811981201172,
      "activations/layer13_attention_weight_min": -35.744197845458984,
      "activations/layer14_attention_weight_max": 51.98052978515625,
      "activations/layer14_attention_weight_min": -43.790565490722656,
      "activations/layer15_attention_weight_max": 42.74247741699219,
      "activations/layer15_attention_weight_min": -32.2296028137207,
      "activations/layer16_attention_weight_max": 33.50259780883789,
      "activations/layer16_attention_weight_min": -27.215866088867188,
      "activations/layer17_attention_weight_max": 30.174381256103516,
      "activations/layer17_attention_weight_min": -27.131366729736328,
      "activations/layer18_attention_weight_max": 28.311992645263672,
      "activations/layer18_attention_weight_min": -24.871530532836914,
      "activations/layer19_attention_weight_max": 40.2832145690918,
      "activations/layer19_attention_weight_min": -33.04738998413086,
      "activations/layer1_attention_weight_max": 16.726404190063477,
      "activations/layer1_attention_weight_min": -14.658496856689453,
      "activations/layer20_attention_weight_max": 34.37910079956055,
      "activations/layer20_attention_weight_min": -24.82552146911621,
      "activations/layer21_attention_weight_max": 30.338335037231445,
      "activations/layer21_attention_weight_min": -24.4296875,
      "activations/layer22_attention_weight_max": 51.4869384765625,
      "activations/layer22_attention_weight_min": -32.754512786865234,
      "activations/layer23_attention_weight_max": 36.13125991821289,
      "activations/layer23_attention_weight_min": -28.014949798583984,
      "activations/layer2_attention_weight_max": 35.44041442871094,
      "activations/layer2_attention_weight_min": -35.315059661865234,
      "activations/layer3_attention_weight_max": 101.05020141601562,
      "activations/layer3_attention_weight_min": -104.90792846679688,
      "activations/layer4_attention_weight_max": 125.10363006591797,
      "activations/layer4_attention_weight_min": -125.05364990234375,
      "activations/layer5_attention_weight_max": 56.042686462402344,
      "activations/layer5_attention_weight_min": -59.13994598388672,
      "activations/layer6_attention_weight_max": 49.07846450805664,
      "activations/layer6_attention_weight_min": -49.21762466430664,
      "activations/layer7_attention_weight_max": 103.34262084960938,
      "activations/layer7_attention_weight_min": -99.60012817382812,
      "activations/layer8_attention_weight_max": 43.999412536621094,
      "activations/layer8_attention_weight_min": -43.57150650024414,
      "activations/layer9_attention_weight_max": 37.911739349365234,
      "activations/layer9_attention_weight_min": -36.49833297729492,
      "epoch": 10.14,
      "learning_rate": 8.545492424242422e-05,
      "loss": 2.7868,
      "step": 174550
    },
    {
      "activations/layer0_attention_weight_max": 16.455543518066406,
      "activations/layer0_attention_weight_min": -12.801959037780762,
      "activations/layer10_attention_weight_max": 35.12127685546875,
      "activations/layer10_attention_weight_min": -32.77266311645508,
      "activations/layer11_attention_weight_max": 34.74787139892578,
      "activations/layer11_attention_weight_min": -31.049503326416016,
      "activations/layer12_attention_weight_max": 25.679672241210938,
      "activations/layer12_attention_weight_min": -26.196449279785156,
      "activations/layer13_attention_weight_max": 39.59099578857422,
      "activations/layer13_attention_weight_min": -37.09458541870117,
      "activations/layer14_attention_weight_max": 43.6862907409668,
      "activations/layer14_attention_weight_min": -40.189796447753906,
      "activations/layer15_attention_weight_max": 38.077301025390625,
      "activations/layer15_attention_weight_min": -33.75975036621094,
      "activations/layer16_attention_weight_max": 33.98447036743164,
      "activations/layer16_attention_weight_min": -27.949064254760742,
      "activations/layer17_attention_weight_max": 33.576072692871094,
      "activations/layer17_attention_weight_min": -26.440610885620117,
      "activations/layer18_attention_weight_max": 31.719675064086914,
      "activations/layer18_attention_weight_min": -25.22165870666504,
      "activations/layer19_attention_weight_max": 34.10490417480469,
      "activations/layer19_attention_weight_min": -31.922988891601562,
      "activations/layer1_attention_weight_max": 17.492109298706055,
      "activations/layer1_attention_weight_min": -14.459542274475098,
      "activations/layer20_attention_weight_max": 29.310951232910156,
      "activations/layer20_attention_weight_min": -26.005205154418945,
      "activations/layer21_attention_weight_max": 32.00600814819336,
      "activations/layer21_attention_weight_min": -24.071767807006836,
      "activations/layer22_attention_weight_max": 43.77497100830078,
      "activations/layer22_attention_weight_min": -31.710405349731445,
      "activations/layer23_attention_weight_max": 35.9332275390625,
      "activations/layer23_attention_weight_min": -25.668058395385742,
      "activations/layer2_attention_weight_max": 33.25658416748047,
      "activations/layer2_attention_weight_min": -32.72637939453125,
      "activations/layer3_attention_weight_max": 96.07820129394531,
      "activations/layer3_attention_weight_min": -93.52183532714844,
      "activations/layer4_attention_weight_max": 120.37666320800781,
      "activations/layer4_attention_weight_min": -112.66024017333984,
      "activations/layer5_attention_weight_max": 59.52975082397461,
      "activations/layer5_attention_weight_min": -64.43633270263672,
      "activations/layer6_attention_weight_max": 47.941322326660156,
      "activations/layer6_attention_weight_min": -49.486305236816406,
      "activations/layer7_attention_weight_max": 95.50283813476562,
      "activations/layer7_attention_weight_min": -92.93643951416016,
      "activations/layer8_attention_weight_max": 43.293800354003906,
      "activations/layer8_attention_weight_min": -42.192195892333984,
      "activations/layer9_attention_weight_max": 36.696754455566406,
      "activations/layer9_attention_weight_min": -33.83757400512695,
      "epoch": 10.15,
      "learning_rate": 8.543598484848484e-05,
      "loss": 2.7927,
      "step": 174600
    },
    {
      "activations/layer0_attention_weight_max": 15.951908111572266,
      "activations/layer0_attention_weight_min": -12.871811866760254,
      "activations/layer10_attention_weight_max": 32.852996826171875,
      "activations/layer10_attention_weight_min": -31.635421752929688,
      "activations/layer11_attention_weight_max": 34.658233642578125,
      "activations/layer11_attention_weight_min": -31.808324813842773,
      "activations/layer12_attention_weight_max": 23.26926612854004,
      "activations/layer12_attention_weight_min": -25.7856502532959,
      "activations/layer13_attention_weight_max": 35.151161193847656,
      "activations/layer13_attention_weight_min": -30.33208656311035,
      "activations/layer14_attention_weight_max": 42.20073699951172,
      "activations/layer14_attention_weight_min": -36.16727828979492,
      "activations/layer15_attention_weight_max": 34.58631134033203,
      "activations/layer15_attention_weight_min": -30.835773468017578,
      "activations/layer16_attention_weight_max": 30.64264678955078,
      "activations/layer16_attention_weight_min": -26.720733642578125,
      "activations/layer17_attention_weight_max": 30.58408546447754,
      "activations/layer17_attention_weight_min": -25.31389617919922,
      "activations/layer18_attention_weight_max": 30.992692947387695,
      "activations/layer18_attention_weight_min": -23.3089656829834,
      "activations/layer19_attention_weight_max": 36.735443115234375,
      "activations/layer19_attention_weight_min": -29.783588409423828,
      "activations/layer1_attention_weight_max": 17.033945083618164,
      "activations/layer1_attention_weight_min": -14.86433219909668,
      "activations/layer20_attention_weight_max": 30.49202537536621,
      "activations/layer20_attention_weight_min": -24.848325729370117,
      "activations/layer21_attention_weight_max": 30.51805305480957,
      "activations/layer21_attention_weight_min": -23.780254364013672,
      "activations/layer22_attention_weight_max": 48.266231536865234,
      "activations/layer22_attention_weight_min": -29.33139991760254,
      "activations/layer23_attention_weight_max": 36.45561981201172,
      "activations/layer23_attention_weight_min": -24.09175682067871,
      "activations/layer2_attention_weight_max": 32.05813980102539,
      "activations/layer2_attention_weight_min": -34.20964050292969,
      "activations/layer3_attention_weight_max": 91.03763580322266,
      "activations/layer3_attention_weight_min": -93.95919799804688,
      "activations/layer4_attention_weight_max": 108.17107391357422,
      "activations/layer4_attention_weight_min": -115.0414810180664,
      "activations/layer5_attention_weight_max": 50.97292709350586,
      "activations/layer5_attention_weight_min": -58.771541595458984,
      "activations/layer6_attention_weight_max": 45.26552963256836,
      "activations/layer6_attention_weight_min": -48.680091857910156,
      "activations/layer7_attention_weight_max": 94.25037384033203,
      "activations/layer7_attention_weight_min": -88.43840789794922,
      "activations/layer8_attention_weight_max": 39.375389099121094,
      "activations/layer8_attention_weight_min": -40.54413986206055,
      "activations/layer9_attention_weight_max": 33.57313537597656,
      "activations/layer9_attention_weight_min": -33.02783203125,
      "epoch": 10.15,
      "learning_rate": 8.541704545454545e-05,
      "loss": 2.7833,
      "step": 174650
    },
    {
      "activations/layer0_attention_weight_max": 15.810503959655762,
      "activations/layer0_attention_weight_min": -12.247855186462402,
      "activations/layer10_attention_weight_max": 37.20182418823242,
      "activations/layer10_attention_weight_min": -33.799461364746094,
      "activations/layer11_attention_weight_max": 36.50469207763672,
      "activations/layer11_attention_weight_min": -36.719810485839844,
      "activations/layer12_attention_weight_max": 25.365747451782227,
      "activations/layer12_attention_weight_min": -29.202733993530273,
      "activations/layer13_attention_weight_max": 38.15987777709961,
      "activations/layer13_attention_weight_min": -35.328460693359375,
      "activations/layer14_attention_weight_max": 42.23647689819336,
      "activations/layer14_attention_weight_min": -38.59488296508789,
      "activations/layer15_attention_weight_max": 38.46697998046875,
      "activations/layer15_attention_weight_min": -32.33736038208008,
      "activations/layer16_attention_weight_max": 30.605159759521484,
      "activations/layer16_attention_weight_min": -27.445112228393555,
      "activations/layer17_attention_weight_max": 30.18548011779785,
      "activations/layer17_attention_weight_min": -24.51026153564453,
      "activations/layer18_attention_weight_max": 31.50179672241211,
      "activations/layer18_attention_weight_min": -23.85833740234375,
      "activations/layer19_attention_weight_max": 30.500808715820312,
      "activations/layer19_attention_weight_min": -30.271316528320312,
      "activations/layer1_attention_weight_max": 17.06517219543457,
      "activations/layer1_attention_weight_min": -15.011331558227539,
      "activations/layer20_attention_weight_max": 28.295352935791016,
      "activations/layer20_attention_weight_min": -23.6429500579834,
      "activations/layer21_attention_weight_max": 28.3138427734375,
      "activations/layer21_attention_weight_min": -25.43611717224121,
      "activations/layer22_attention_weight_max": 41.77421951293945,
      "activations/layer22_attention_weight_min": -29.079708099365234,
      "activations/layer23_attention_weight_max": 32.621368408203125,
      "activations/layer23_attention_weight_min": -26.398326873779297,
      "activations/layer2_attention_weight_max": 32.20406723022461,
      "activations/layer2_attention_weight_min": -33.057621002197266,
      "activations/layer3_attention_weight_max": 92.25721740722656,
      "activations/layer3_attention_weight_min": -94.53069305419922,
      "activations/layer4_attention_weight_max": 114.9580307006836,
      "activations/layer4_attention_weight_min": -110.09346771240234,
      "activations/layer5_attention_weight_max": 53.47827911376953,
      "activations/layer5_attention_weight_min": -60.43592071533203,
      "activations/layer6_attention_weight_max": 47.661964416503906,
      "activations/layer6_attention_weight_min": -47.55746078491211,
      "activations/layer7_attention_weight_max": 96.75642395019531,
      "activations/layer7_attention_weight_min": -90.01090240478516,
      "activations/layer8_attention_weight_max": 43.297279357910156,
      "activations/layer8_attention_weight_min": -42.811988830566406,
      "activations/layer9_attention_weight_max": 38.01961898803711,
      "activations/layer9_attention_weight_min": -35.4570426940918,
      "epoch": 10.15,
      "learning_rate": 8.539810606060604e-05,
      "loss": 2.8009,
      "step": 174700
    },
    {
      "activations/layer0_attention_weight_max": 16.052215576171875,
      "activations/layer0_attention_weight_min": -12.90805435180664,
      "activations/layer10_attention_weight_max": 31.627859115600586,
      "activations/layer10_attention_weight_min": -31.17558479309082,
      "activations/layer11_attention_weight_max": 32.24639892578125,
      "activations/layer11_attention_weight_min": -31.058429718017578,
      "activations/layer12_attention_weight_max": 23.90363121032715,
      "activations/layer12_attention_weight_min": -26.233783721923828,
      "activations/layer13_attention_weight_max": 37.81260681152344,
      "activations/layer13_attention_weight_min": -35.2902717590332,
      "activations/layer14_attention_weight_max": 42.96356964111328,
      "activations/layer14_attention_weight_min": -41.87455368041992,
      "activations/layer15_attention_weight_max": 33.26863098144531,
      "activations/layer15_attention_weight_min": -29.927017211914062,
      "activations/layer16_attention_weight_max": 31.329435348510742,
      "activations/layer16_attention_weight_min": -28.404022216796875,
      "activations/layer17_attention_weight_max": 30.990190505981445,
      "activations/layer17_attention_weight_min": -25.427770614624023,
      "activations/layer18_attention_weight_max": 28.602760314941406,
      "activations/layer18_attention_weight_min": -22.166709899902344,
      "activations/layer19_attention_weight_max": 33.17719268798828,
      "activations/layer19_attention_weight_min": -31.071752548217773,
      "activations/layer1_attention_weight_max": 16.94246482849121,
      "activations/layer1_attention_weight_min": -13.276101112365723,
      "activations/layer20_attention_weight_max": 29.60402488708496,
      "activations/layer20_attention_weight_min": -25.869001388549805,
      "activations/layer21_attention_weight_max": 27.339982986450195,
      "activations/layer21_attention_weight_min": -25.526762008666992,
      "activations/layer22_attention_weight_max": 46.04490661621094,
      "activations/layer22_attention_weight_min": -32.081356048583984,
      "activations/layer23_attention_weight_max": 34.78828048706055,
      "activations/layer23_attention_weight_min": -25.87500762939453,
      "activations/layer2_attention_weight_max": 31.82349967956543,
      "activations/layer2_attention_weight_min": -31.36629867553711,
      "activations/layer3_attention_weight_max": 90.82843780517578,
      "activations/layer3_attention_weight_min": -93.31051635742188,
      "activations/layer4_attention_weight_max": 110.8638687133789,
      "activations/layer4_attention_weight_min": -110.49652099609375,
      "activations/layer5_attention_weight_max": 54.7745475769043,
      "activations/layer5_attention_weight_min": -61.549468994140625,
      "activations/layer6_attention_weight_max": 44.81996154785156,
      "activations/layer6_attention_weight_min": -46.82735061645508,
      "activations/layer7_attention_weight_max": 85.57263946533203,
      "activations/layer7_attention_weight_min": -86.20954895019531,
      "activations/layer8_attention_weight_max": 39.4755744934082,
      "activations/layer8_attention_weight_min": -40.30611038208008,
      "activations/layer9_attention_weight_max": 31.10618019104004,
      "activations/layer9_attention_weight_min": -32.448909759521484,
      "epoch": 10.15,
      "learning_rate": 8.537916666666666e-05,
      "loss": 2.7981,
      "step": 174750
    },
    {
      "activations/layer0_attention_weight_max": 16.339590072631836,
      "activations/layer0_attention_weight_min": -12.579523086547852,
      "activations/layer10_attention_weight_max": 34.036346435546875,
      "activations/layer10_attention_weight_min": -32.667015075683594,
      "activations/layer11_attention_weight_max": 31.726696014404297,
      "activations/layer11_attention_weight_min": -33.03953170776367,
      "activations/layer12_attention_weight_max": 23.014083862304688,
      "activations/layer12_attention_weight_min": -25.238523483276367,
      "activations/layer13_attention_weight_max": 38.548702239990234,
      "activations/layer13_attention_weight_min": -32.06182098388672,
      "activations/layer14_attention_weight_max": 43.69206237792969,
      "activations/layer14_attention_weight_min": -38.53622055053711,
      "activations/layer15_attention_weight_max": 35.6458625793457,
      "activations/layer15_attention_weight_min": -31.787662506103516,
      "activations/layer16_attention_weight_max": 28.510889053344727,
      "activations/layer16_attention_weight_min": -26.721681594848633,
      "activations/layer17_attention_weight_max": 30.62956428527832,
      "activations/layer17_attention_weight_min": -25.47435188293457,
      "activations/layer18_attention_weight_max": 27.923912048339844,
      "activations/layer18_attention_weight_min": -22.219791412353516,
      "activations/layer19_attention_weight_max": 30.966764450073242,
      "activations/layer19_attention_weight_min": -29.88260841369629,
      "activations/layer1_attention_weight_max": 16.728939056396484,
      "activations/layer1_attention_weight_min": -13.98678970336914,
      "activations/layer20_attention_weight_max": 27.450719833374023,
      "activations/layer20_attention_weight_min": -26.490089416503906,
      "activations/layer21_attention_weight_max": 26.91807746887207,
      "activations/layer21_attention_weight_min": -23.2344970703125,
      "activations/layer22_attention_weight_max": 39.291099548339844,
      "activations/layer22_attention_weight_min": -29.867633819580078,
      "activations/layer23_attention_weight_max": 33.13412094116211,
      "activations/layer23_attention_weight_min": -24.974912643432617,
      "activations/layer2_attention_weight_max": 31.483930587768555,
      "activations/layer2_attention_weight_min": -32.30096435546875,
      "activations/layer3_attention_weight_max": 93.06517028808594,
      "activations/layer3_attention_weight_min": -98.02556610107422,
      "activations/layer4_attention_weight_max": 110.61832427978516,
      "activations/layer4_attention_weight_min": -111.2738265991211,
      "activations/layer5_attention_weight_max": 53.140499114990234,
      "activations/layer5_attention_weight_min": -60.23262023925781,
      "activations/layer6_attention_weight_max": 46.25514602661133,
      "activations/layer6_attention_weight_min": -48.19868087768555,
      "activations/layer7_attention_weight_max": 96.31353759765625,
      "activations/layer7_attention_weight_min": -84.82113647460938,
      "activations/layer8_attention_weight_max": 41.20479965209961,
      "activations/layer8_attention_weight_min": -39.83898162841797,
      "activations/layer9_attention_weight_max": 33.53366470336914,
      "activations/layer9_attention_weight_min": -32.76862716674805,
      "epoch": 10.16,
      "learning_rate": 8.536022727272727e-05,
      "loss": 2.7829,
      "step": 174800
    },
    {
      "activations/layer0_attention_weight_max": 16.295597076416016,
      "activations/layer0_attention_weight_min": -12.728425025939941,
      "activations/layer10_attention_weight_max": 32.05967712402344,
      "activations/layer10_attention_weight_min": -31.51778793334961,
      "activations/layer11_attention_weight_max": 31.38168716430664,
      "activations/layer11_attention_weight_min": -33.31282424926758,
      "activations/layer12_attention_weight_max": 22.81353759765625,
      "activations/layer12_attention_weight_min": -25.901823043823242,
      "activations/layer13_attention_weight_max": 35.87028121948242,
      "activations/layer13_attention_weight_min": -32.35124206542969,
      "activations/layer14_attention_weight_max": 42.445743560791016,
      "activations/layer14_attention_weight_min": -35.320045471191406,
      "activations/layer15_attention_weight_max": 35.34486770629883,
      "activations/layer15_attention_weight_min": -32.8583984375,
      "activations/layer16_attention_weight_max": 31.147382736206055,
      "activations/layer16_attention_weight_min": -27.656970977783203,
      "activations/layer17_attention_weight_max": 35.80271911621094,
      "activations/layer17_attention_weight_min": -25.169776916503906,
      "activations/layer18_attention_weight_max": 35.014835357666016,
      "activations/layer18_attention_weight_min": -23.46705436706543,
      "activations/layer19_attention_weight_max": 37.36703109741211,
      "activations/layer19_attention_weight_min": -30.052289962768555,
      "activations/layer1_attention_weight_max": 16.457063674926758,
      "activations/layer1_attention_weight_min": -13.561151504516602,
      "activations/layer20_attention_weight_max": 31.925872802734375,
      "activations/layer20_attention_weight_min": -25.376325607299805,
      "activations/layer21_attention_weight_max": 36.644248962402344,
      "activations/layer21_attention_weight_min": -24.19194984436035,
      "activations/layer22_attention_weight_max": 45.108299255371094,
      "activations/layer22_attention_weight_min": -29.026670455932617,
      "activations/layer23_attention_weight_max": 37.9036865234375,
      "activations/layer23_attention_weight_min": -26.16823959350586,
      "activations/layer2_attention_weight_max": 31.197654724121094,
      "activations/layer2_attention_weight_min": -31.715635299682617,
      "activations/layer3_attention_weight_max": 90.34239196777344,
      "activations/layer3_attention_weight_min": -92.10572814941406,
      "activations/layer4_attention_weight_max": 109.0779800415039,
      "activations/layer4_attention_weight_min": -113.73963928222656,
      "activations/layer5_attention_weight_max": 52.17821502685547,
      "activations/layer5_attention_weight_min": -61.428802490234375,
      "activations/layer6_attention_weight_max": 45.16863250732422,
      "activations/layer6_attention_weight_min": -46.678428649902344,
      "activations/layer7_attention_weight_max": 86.65745544433594,
      "activations/layer7_attention_weight_min": -87.32306671142578,
      "activations/layer8_attention_weight_max": 39.233211517333984,
      "activations/layer8_attention_weight_min": -40.04844665527344,
      "activations/layer9_attention_weight_max": 31.3830623626709,
      "activations/layer9_attention_weight_min": -32.79081344604492,
      "epoch": 10.16,
      "learning_rate": 8.534128787878787e-05,
      "loss": 2.7928,
      "step": 174850
    },
    {
      "activations/layer0_attention_weight_max": 15.621719360351562,
      "activations/layer0_attention_weight_min": -12.300558090209961,
      "activations/layer10_attention_weight_max": 33.619300842285156,
      "activations/layer10_attention_weight_min": -32.328712463378906,
      "activations/layer11_attention_weight_max": 32.416168212890625,
      "activations/layer11_attention_weight_min": -31.288646697998047,
      "activations/layer12_attention_weight_max": 24.222862243652344,
      "activations/layer12_attention_weight_min": -24.830257415771484,
      "activations/layer13_attention_weight_max": 41.740013122558594,
      "activations/layer13_attention_weight_min": -35.244712829589844,
      "activations/layer14_attention_weight_max": 43.72132110595703,
      "activations/layer14_attention_weight_min": -39.61388397216797,
      "activations/layer15_attention_weight_max": 36.19440841674805,
      "activations/layer15_attention_weight_min": -32.85017395019531,
      "activations/layer16_attention_weight_max": 30.484758377075195,
      "activations/layer16_attention_weight_min": -29.052398681640625,
      "activations/layer17_attention_weight_max": 30.982675552368164,
      "activations/layer17_attention_weight_min": -26.025583267211914,
      "activations/layer18_attention_weight_max": 31.5798397064209,
      "activations/layer18_attention_weight_min": -23.569194793701172,
      "activations/layer19_attention_weight_max": 37.8219108581543,
      "activations/layer19_attention_weight_min": -31.44174575805664,
      "activations/layer1_attention_weight_max": 17.64259147644043,
      "activations/layer1_attention_weight_min": -14.167737007141113,
      "activations/layer20_attention_weight_max": 28.399938583374023,
      "activations/layer20_attention_weight_min": -25.372303009033203,
      "activations/layer21_attention_weight_max": 27.667856216430664,
      "activations/layer21_attention_weight_min": -25.2528018951416,
      "activations/layer22_attention_weight_max": 44.8148307800293,
      "activations/layer22_attention_weight_min": -33.14595413208008,
      "activations/layer23_attention_weight_max": 35.34521484375,
      "activations/layer23_attention_weight_min": -24.716596603393555,
      "activations/layer2_attention_weight_max": 32.86974334716797,
      "activations/layer2_attention_weight_min": -32.042503356933594,
      "activations/layer3_attention_weight_max": 96.53224182128906,
      "activations/layer3_attention_weight_min": -95.55119323730469,
      "activations/layer4_attention_weight_max": 117.47021484375,
      "activations/layer4_attention_weight_min": -111.07135009765625,
      "activations/layer5_attention_weight_max": 56.20182418823242,
      "activations/layer5_attention_weight_min": -62.16786193847656,
      "activations/layer6_attention_weight_max": 46.81563186645508,
      "activations/layer6_attention_weight_min": -48.3666877746582,
      "activations/layer7_attention_weight_max": 95.38419342041016,
      "activations/layer7_attention_weight_min": -95.4007568359375,
      "activations/layer8_attention_weight_max": 42.45158767700195,
      "activations/layer8_attention_weight_min": -42.26702880859375,
      "activations/layer9_attention_weight_max": 35.372196197509766,
      "activations/layer9_attention_weight_min": -32.83145523071289,
      "epoch": 10.16,
      "learning_rate": 8.532234848484848e-05,
      "loss": 2.787,
      "step": 174900
    },
    {
      "activations/layer0_attention_weight_max": 15.711874008178711,
      "activations/layer0_attention_weight_min": -12.413498878479004,
      "activations/layer10_attention_weight_max": 31.90949058532715,
      "activations/layer10_attention_weight_min": -30.517295837402344,
      "activations/layer11_attention_weight_max": 32.44245147705078,
      "activations/layer11_attention_weight_min": -29.129873275756836,
      "activations/layer12_attention_weight_max": 24.62202262878418,
      "activations/layer12_attention_weight_min": -25.42650604248047,
      "activations/layer13_attention_weight_max": 37.18425750732422,
      "activations/layer13_attention_weight_min": -33.590721130371094,
      "activations/layer14_attention_weight_max": 48.922874450683594,
      "activations/layer14_attention_weight_min": -40.86100387573242,
      "activations/layer15_attention_weight_max": 35.4901008605957,
      "activations/layer15_attention_weight_min": -30.90363311767578,
      "activations/layer16_attention_weight_max": 37.49365997314453,
      "activations/layer16_attention_weight_min": -27.435474395751953,
      "activations/layer17_attention_weight_max": 34.41407775878906,
      "activations/layer17_attention_weight_min": -24.847131729125977,
      "activations/layer18_attention_weight_max": 31.754060745239258,
      "activations/layer18_attention_weight_min": -21.587324142456055,
      "activations/layer19_attention_weight_max": 38.90327835083008,
      "activations/layer19_attention_weight_min": -31.200515747070312,
      "activations/layer1_attention_weight_max": 16.995574951171875,
      "activations/layer1_attention_weight_min": -14.08682632446289,
      "activations/layer20_attention_weight_max": 35.61019515991211,
      "activations/layer20_attention_weight_min": -25.53682518005371,
      "activations/layer21_attention_weight_max": 34.1331787109375,
      "activations/layer21_attention_weight_min": -24.405813217163086,
      "activations/layer22_attention_weight_max": 48.295135498046875,
      "activations/layer22_attention_weight_min": -30.463356018066406,
      "activations/layer23_attention_weight_max": 38.34130096435547,
      "activations/layer23_attention_weight_min": -23.70123291015625,
      "activations/layer2_attention_weight_max": 33.782623291015625,
      "activations/layer2_attention_weight_min": -32.109500885009766,
      "activations/layer3_attention_weight_max": 94.70991516113281,
      "activations/layer3_attention_weight_min": -93.27196502685547,
      "activations/layer4_attention_weight_max": 111.98360443115234,
      "activations/layer4_attention_weight_min": -110.98783111572266,
      "activations/layer5_attention_weight_max": 57.64384078979492,
      "activations/layer5_attention_weight_min": -65.07789611816406,
      "activations/layer6_attention_weight_max": 45.61261749267578,
      "activations/layer6_attention_weight_min": -47.38272476196289,
      "activations/layer7_attention_weight_max": 92.27310943603516,
      "activations/layer7_attention_weight_min": -91.0975341796875,
      "activations/layer8_attention_weight_max": 41.18206024169922,
      "activations/layer8_attention_weight_min": -39.906063079833984,
      "activations/layer9_attention_weight_max": 32.579437255859375,
      "activations/layer9_attention_weight_min": -31.705244064331055,
      "epoch": 10.17,
      "learning_rate": 8.530340909090909e-05,
      "loss": 2.7956,
      "step": 174950
    },
    {
      "activations/layer0_attention_weight_max": 16.252986907958984,
      "activations/layer0_attention_weight_min": -12.405339241027832,
      "activations/layer10_attention_weight_max": 31.361797332763672,
      "activations/layer10_attention_weight_min": -30.080867767333984,
      "activations/layer11_attention_weight_max": 30.960636138916016,
      "activations/layer11_attention_weight_min": -29.828168869018555,
      "activations/layer12_attention_weight_max": 22.489688873291016,
      "activations/layer12_attention_weight_min": -24.916549682617188,
      "activations/layer13_attention_weight_max": 35.4564323425293,
      "activations/layer13_attention_weight_min": -32.60919189453125,
      "activations/layer14_attention_weight_max": 44.60087585449219,
      "activations/layer14_attention_weight_min": -39.918861389160156,
      "activations/layer15_attention_weight_max": 34.408935546875,
      "activations/layer15_attention_weight_min": -28.767333984375,
      "activations/layer16_attention_weight_max": 32.6418571472168,
      "activations/layer16_attention_weight_min": -26.69231605529785,
      "activations/layer17_attention_weight_max": 31.314022064208984,
      "activations/layer17_attention_weight_min": -25.762775421142578,
      "activations/layer18_attention_weight_max": 31.285306930541992,
      "activations/layer18_attention_weight_min": -21.7928524017334,
      "activations/layer19_attention_weight_max": 37.12495040893555,
      "activations/layer19_attention_weight_min": -30.3605899810791,
      "activations/layer1_attention_weight_max": 15.873143196105957,
      "activations/layer1_attention_weight_min": -15.033220291137695,
      "activations/layer20_attention_weight_max": 30.12810707092285,
      "activations/layer20_attention_weight_min": -25.535356521606445,
      "activations/layer21_attention_weight_max": 28.676847457885742,
      "activations/layer21_attention_weight_min": -25.483074188232422,
      "activations/layer22_attention_weight_max": 49.91315460205078,
      "activations/layer22_attention_weight_min": -30.276594161987305,
      "activations/layer23_attention_weight_max": 36.170066833496094,
      "activations/layer23_attention_weight_min": -27.803739547729492,
      "activations/layer2_attention_weight_max": 34.65203857421875,
      "activations/layer2_attention_weight_min": -33.916175842285156,
      "activations/layer3_attention_weight_max": 95.0475845336914,
      "activations/layer3_attention_weight_min": -98.5283432006836,
      "activations/layer4_attention_weight_max": 113.56536865234375,
      "activations/layer4_attention_weight_min": -117.98503875732422,
      "activations/layer5_attention_weight_max": 52.92057800292969,
      "activations/layer5_attention_weight_min": -63.24866485595703,
      "activations/layer6_attention_weight_max": 47.10683822631836,
      "activations/layer6_attention_weight_min": -50.23130416870117,
      "activations/layer7_attention_weight_max": 91.60871124267578,
      "activations/layer7_attention_weight_min": -89.97334289550781,
      "activations/layer8_attention_weight_max": 39.97844314575195,
      "activations/layer8_attention_weight_min": -41.66577911376953,
      "activations/layer9_attention_weight_max": 32.282100677490234,
      "activations/layer9_attention_weight_min": -32.5509147644043,
      "epoch": 10.17,
      "learning_rate": 8.528446969696969e-05,
      "loss": 2.7928,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.5079,
      "eval_samples_per_second": 504.706,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.5079,
      "eval_openwebtext_samples_per_second": 504.706,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9396,
      "eval_wikitext_samples_per_second": 235.103,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.5148,
      "eval_lambada_samples_per_second": 511.727,
      "step": 175000
    },
    {
      "activations/layer0_attention_weight_max": 16.371341705322266,
      "activations/layer0_attention_weight_min": -13.578035354614258,
      "activations/layer10_attention_weight_max": 35.51190185546875,
      "activations/layer10_attention_weight_min": -31.26486587524414,
      "activations/layer11_attention_weight_max": 33.06858444213867,
      "activations/layer11_attention_weight_min": -30.725658416748047,
      "activations/layer12_attention_weight_max": 28.645906448364258,
      "activations/layer12_attention_weight_min": -23.76274871826172,
      "activations/layer13_attention_weight_max": 39.3210563659668,
      "activations/layer13_attention_weight_min": -36.15458297729492,
      "activations/layer14_attention_weight_max": 44.76169967651367,
      "activations/layer14_attention_weight_min": -39.923667907714844,
      "activations/layer15_attention_weight_max": 38.21250534057617,
      "activations/layer15_attention_weight_min": -31.719846725463867,
      "activations/layer16_attention_weight_max": 29.700986862182617,
      "activations/layer16_attention_weight_min": -27.46757698059082,
      "activations/layer17_attention_weight_max": 33.5567512512207,
      "activations/layer17_attention_weight_min": -23.991676330566406,
      "activations/layer18_attention_weight_max": 32.22386932373047,
      "activations/layer18_attention_weight_min": -23.04690933227539,
      "activations/layer19_attention_weight_max": 34.142391204833984,
      "activations/layer19_attention_weight_min": -30.065458297729492,
      "activations/layer1_attention_weight_max": 16.40303611755371,
      "activations/layer1_attention_weight_min": -15.026718139648438,
      "activations/layer20_attention_weight_max": 31.001724243164062,
      "activations/layer20_attention_weight_min": -24.222505569458008,
      "activations/layer21_attention_weight_max": 29.56069564819336,
      "activations/layer21_attention_weight_min": -23.35819435119629,
      "activations/layer22_attention_weight_max": 42.010807037353516,
      "activations/layer22_attention_weight_min": -29.06248664855957,
      "activations/layer23_attention_weight_max": 34.4509162902832,
      "activations/layer23_attention_weight_min": -23.155012130737305,
      "activations/layer2_attention_weight_max": 31.23950958251953,
      "activations/layer2_attention_weight_min": -31.223155975341797,
      "activations/layer3_attention_weight_max": 92.52748107910156,
      "activations/layer3_attention_weight_min": -92.23522186279297,
      "activations/layer4_attention_weight_max": 112.8978500366211,
      "activations/layer4_attention_weight_min": -111.4272689819336,
      "activations/layer5_attention_weight_max": 52.71488571166992,
      "activations/layer5_attention_weight_min": -61.663787841796875,
      "activations/layer6_attention_weight_max": 47.4015998840332,
      "activations/layer6_attention_weight_min": -50.6351203918457,
      "activations/layer7_attention_weight_max": 95.29183959960938,
      "activations/layer7_attention_weight_min": -91.54932403564453,
      "activations/layer8_attention_weight_max": 40.39033508300781,
      "activations/layer8_attention_weight_min": -41.34749221801758,
      "activations/layer9_attention_weight_max": 35.92720413208008,
      "activations/layer9_attention_weight_min": -32.30959701538086,
      "epoch": 10.17,
      "learning_rate": 8.526553030303029e-05,
      "loss": 2.8002,
      "step": 175050
    },
    {
      "activations/layer0_attention_weight_max": 15.723803520202637,
      "activations/layer0_attention_weight_min": -12.820743560791016,
      "activations/layer10_attention_weight_max": 32.594024658203125,
      "activations/layer10_attention_weight_min": -31.852386474609375,
      "activations/layer11_attention_weight_max": 33.43568420410156,
      "activations/layer11_attention_weight_min": -32.2538948059082,
      "activations/layer12_attention_weight_max": 24.491418838500977,
      "activations/layer12_attention_weight_min": -24.381874084472656,
      "activations/layer13_attention_weight_max": 40.1841926574707,
      "activations/layer13_attention_weight_min": -32.919429779052734,
      "activations/layer14_attention_weight_max": 49.307273864746094,
      "activations/layer14_attention_weight_min": -36.88008117675781,
      "activations/layer15_attention_weight_max": 35.73866271972656,
      "activations/layer15_attention_weight_min": -30.210613250732422,
      "activations/layer16_attention_weight_max": 30.558378219604492,
      "activations/layer16_attention_weight_min": -28.750953674316406,
      "activations/layer17_attention_weight_max": 31.002193450927734,
      "activations/layer17_attention_weight_min": -25.51907730102539,
      "activations/layer18_attention_weight_max": 31.128389358520508,
      "activations/layer18_attention_weight_min": -23.326536178588867,
      "activations/layer19_attention_weight_max": 35.72185134887695,
      "activations/layer19_attention_weight_min": -32.185550689697266,
      "activations/layer1_attention_weight_max": 16.861003875732422,
      "activations/layer1_attention_weight_min": -14.25585651397705,
      "activations/layer20_attention_weight_max": 29.931682586669922,
      "activations/layer20_attention_weight_min": -24.98198127746582,
      "activations/layer21_attention_weight_max": 30.01350975036621,
      "activations/layer21_attention_weight_min": -24.53265953063965,
      "activations/layer22_attention_weight_max": 51.07371520996094,
      "activations/layer22_attention_weight_min": -29.13457489013672,
      "activations/layer23_attention_weight_max": 37.36060333251953,
      "activations/layer23_attention_weight_min": -23.870269775390625,
      "activations/layer2_attention_weight_max": 30.176992416381836,
      "activations/layer2_attention_weight_min": -32.407588958740234,
      "activations/layer3_attention_weight_max": 91.66153717041016,
      "activations/layer3_attention_weight_min": -94.51420593261719,
      "activations/layer4_attention_weight_max": 115.34203338623047,
      "activations/layer4_attention_weight_min": -113.2306900024414,
      "activations/layer5_attention_weight_max": 53.437904357910156,
      "activations/layer5_attention_weight_min": -61.52369689941406,
      "activations/layer6_attention_weight_max": 45.4374885559082,
      "activations/layer6_attention_weight_min": -50.55891799926758,
      "activations/layer7_attention_weight_max": 91.00413513183594,
      "activations/layer7_attention_weight_min": -98.1788330078125,
      "activations/layer8_attention_weight_max": 40.9058837890625,
      "activations/layer8_attention_weight_min": -42.1649284362793,
      "activations/layer9_attention_weight_max": 34.808387756347656,
      "activations/layer9_attention_weight_min": -34.467708587646484,
      "epoch": 10.17,
      "learning_rate": 8.52465909090909e-05,
      "loss": 2.7937,
      "step": 175100
    },
    {
      "activations/layer0_attention_weight_max": 15.408377647399902,
      "activations/layer0_attention_weight_min": -13.199670791625977,
      "activations/layer10_attention_weight_max": 34.10870361328125,
      "activations/layer10_attention_weight_min": -33.097755432128906,
      "activations/layer11_attention_weight_max": 34.636566162109375,
      "activations/layer11_attention_weight_min": -32.30778121948242,
      "activations/layer12_attention_weight_max": 26.03495216369629,
      "activations/layer12_attention_weight_min": -34.29227828979492,
      "activations/layer13_attention_weight_max": 41.63265609741211,
      "activations/layer13_attention_weight_min": -35.59184646606445,
      "activations/layer14_attention_weight_max": 49.786190032958984,
      "activations/layer14_attention_weight_min": -39.09926986694336,
      "activations/layer15_attention_weight_max": 38.79711151123047,
      "activations/layer15_attention_weight_min": -31.856090545654297,
      "activations/layer16_attention_weight_max": 31.01241111755371,
      "activations/layer16_attention_weight_min": -29.348695755004883,
      "activations/layer17_attention_weight_max": 32.054420471191406,
      "activations/layer17_attention_weight_min": -27.177616119384766,
      "activations/layer18_attention_weight_max": 31.62554359436035,
      "activations/layer18_attention_weight_min": -24.27694320678711,
      "activations/layer19_attention_weight_max": 39.339683532714844,
      "activations/layer19_attention_weight_min": -32.97104263305664,
      "activations/layer1_attention_weight_max": 17.445404052734375,
      "activations/layer1_attention_weight_min": -15.517821311950684,
      "activations/layer20_attention_weight_max": 29.101123809814453,
      "activations/layer20_attention_weight_min": -27.176740646362305,
      "activations/layer21_attention_weight_max": 28.911046981811523,
      "activations/layer21_attention_weight_min": -25.263092041015625,
      "activations/layer22_attention_weight_max": 46.93120193481445,
      "activations/layer22_attention_weight_min": -29.763916015625,
      "activations/layer23_attention_weight_max": 35.57505798339844,
      "activations/layer23_attention_weight_min": -24.2237606048584,
      "activations/layer2_attention_weight_max": 33.54772186279297,
      "activations/layer2_attention_weight_min": -34.39431381225586,
      "activations/layer3_attention_weight_max": 92.7656478881836,
      "activations/layer3_attention_weight_min": -94.22793579101562,
      "activations/layer4_attention_weight_max": 116.17162322998047,
      "activations/layer4_attention_weight_min": -112.07826232910156,
      "activations/layer5_attention_weight_max": 51.1455078125,
      "activations/layer5_attention_weight_min": -63.86418914794922,
      "activations/layer6_attention_weight_max": 46.447147369384766,
      "activations/layer6_attention_weight_min": -51.65528106689453,
      "activations/layer7_attention_weight_max": 95.78776550292969,
      "activations/layer7_attention_weight_min": -95.21405792236328,
      "activations/layer8_attention_weight_max": 42.28411865234375,
      "activations/layer8_attention_weight_min": -41.20497131347656,
      "activations/layer9_attention_weight_max": 35.39274597167969,
      "activations/layer9_attention_weight_min": -33.46840286254883,
      "epoch": 10.18,
      "learning_rate": 8.522765151515151e-05,
      "loss": 2.7875,
      "step": 175150
    },
    {
      "activations/layer0_attention_weight_max": 15.64148998260498,
      "activations/layer0_attention_weight_min": -13.036036491394043,
      "activations/layer10_attention_weight_max": 32.1488151550293,
      "activations/layer10_attention_weight_min": -32.63642883300781,
      "activations/layer11_attention_weight_max": 31.942874908447266,
      "activations/layer11_attention_weight_min": -30.52810287475586,
      "activations/layer12_attention_weight_max": 22.279890060424805,
      "activations/layer12_attention_weight_min": -25.190683364868164,
      "activations/layer13_attention_weight_max": 36.4277458190918,
      "activations/layer13_attention_weight_min": -32.97142791748047,
      "activations/layer14_attention_weight_max": 41.42252731323242,
      "activations/layer14_attention_weight_min": -37.272560119628906,
      "activations/layer15_attention_weight_max": 33.46228790283203,
      "activations/layer15_attention_weight_min": -30.664888381958008,
      "activations/layer16_attention_weight_max": 32.678245544433594,
      "activations/layer16_attention_weight_min": -27.985910415649414,
      "activations/layer17_attention_weight_max": 32.19691848754883,
      "activations/layer17_attention_weight_min": -24.55324935913086,
      "activations/layer18_attention_weight_max": 31.3862247467041,
      "activations/layer18_attention_weight_min": -22.232805252075195,
      "activations/layer19_attention_weight_max": 34.01762008666992,
      "activations/layer19_attention_weight_min": -31.25482177734375,
      "activations/layer1_attention_weight_max": 15.96860122680664,
      "activations/layer1_attention_weight_min": -14.213445663452148,
      "activations/layer20_attention_weight_max": 27.120193481445312,
      "activations/layer20_attention_weight_min": -26.50055503845215,
      "activations/layer21_attention_weight_max": 27.966381072998047,
      "activations/layer21_attention_weight_min": -24.23980140686035,
      "activations/layer22_attention_weight_max": 40.397369384765625,
      "activations/layer22_attention_weight_min": -29.10480308532715,
      "activations/layer23_attention_weight_max": 31.750051498413086,
      "activations/layer23_attention_weight_min": -23.15949058532715,
      "activations/layer2_attention_weight_max": 34.636817932128906,
      "activations/layer2_attention_weight_min": -30.508129119873047,
      "activations/layer3_attention_weight_max": 89.88995361328125,
      "activations/layer3_attention_weight_min": -89.09050750732422,
      "activations/layer4_attention_weight_max": 107.7717514038086,
      "activations/layer4_attention_weight_min": -109.1546859741211,
      "activations/layer5_attention_weight_max": 50.62122344970703,
      "activations/layer5_attention_weight_min": -60.99144744873047,
      "activations/layer6_attention_weight_max": 44.074989318847656,
      "activations/layer6_attention_weight_min": -49.55305862426758,
      "activations/layer7_attention_weight_max": 87.95063781738281,
      "activations/layer7_attention_weight_min": -93.57254791259766,
      "activations/layer8_attention_weight_max": 39.7398796081543,
      "activations/layer8_attention_weight_min": -41.83806228637695,
      "activations/layer9_attention_weight_max": 32.37901306152344,
      "activations/layer9_attention_weight_min": -31.815174102783203,
      "epoch": 10.18,
      "learning_rate": 8.520871212121212e-05,
      "loss": 2.7898,
      "step": 175200
    },
    {
      "activations/layer0_attention_weight_max": 15.711669921875,
      "activations/layer0_attention_weight_min": -12.97900390625,
      "activations/layer10_attention_weight_max": 31.48329734802246,
      "activations/layer10_attention_weight_min": -31.575469970703125,
      "activations/layer11_attention_weight_max": 32.516387939453125,
      "activations/layer11_attention_weight_min": -30.428070068359375,
      "activations/layer12_attention_weight_max": 23.25423812866211,
      "activations/layer12_attention_weight_min": -31.24717903137207,
      "activations/layer13_attention_weight_max": 38.161346435546875,
      "activations/layer13_attention_weight_min": -33.75383758544922,
      "activations/layer14_attention_weight_max": 43.6895751953125,
      "activations/layer14_attention_weight_min": -36.63385772705078,
      "activations/layer15_attention_weight_max": 32.26470947265625,
      "activations/layer15_attention_weight_min": -29.497331619262695,
      "activations/layer16_attention_weight_max": 28.613719940185547,
      "activations/layer16_attention_weight_min": -25.859142303466797,
      "activations/layer17_attention_weight_max": 27.34417152404785,
      "activations/layer17_attention_weight_min": -24.32778549194336,
      "activations/layer18_attention_weight_max": 28.035400390625,
      "activations/layer18_attention_weight_min": -22.549421310424805,
      "activations/layer19_attention_weight_max": 30.30562400817871,
      "activations/layer19_attention_weight_min": -28.49370765686035,
      "activations/layer1_attention_weight_max": 18.377052307128906,
      "activations/layer1_attention_weight_min": -15.045475006103516,
      "activations/layer20_attention_weight_max": 25.703519821166992,
      "activations/layer20_attention_weight_min": -23.363441467285156,
      "activations/layer21_attention_weight_max": 25.5029296875,
      "activations/layer21_attention_weight_min": -21.942306518554688,
      "activations/layer22_attention_weight_max": 42.222293853759766,
      "activations/layer22_attention_weight_min": -28.19437599182129,
      "activations/layer23_attention_weight_max": 34.62583923339844,
      "activations/layer23_attention_weight_min": -24.15457534790039,
      "activations/layer2_attention_weight_max": 33.686676025390625,
      "activations/layer2_attention_weight_min": -34.38589096069336,
      "activations/layer3_attention_weight_max": 94.9646987915039,
      "activations/layer3_attention_weight_min": -93.25907897949219,
      "activations/layer4_attention_weight_max": 112.53069305419922,
      "activations/layer4_attention_weight_min": -111.5634994506836,
      "activations/layer5_attention_weight_max": 54.706024169921875,
      "activations/layer5_attention_weight_min": -58.54774475097656,
      "activations/layer6_attention_weight_max": 43.690589904785156,
      "activations/layer6_attention_weight_min": -44.65336608886719,
      "activations/layer7_attention_weight_max": 89.2691421508789,
      "activations/layer7_attention_weight_min": -88.02421569824219,
      "activations/layer8_attention_weight_max": 39.1959114074707,
      "activations/layer8_attention_weight_min": -40.03947830200195,
      "activations/layer9_attention_weight_max": 31.707921981811523,
      "activations/layer9_attention_weight_min": -32.9083366394043,
      "epoch": 10.18,
      "learning_rate": 8.518977272727271e-05,
      "loss": 2.7889,
      "step": 175250
    },
    {
      "activations/layer0_attention_weight_max": 15.100086212158203,
      "activations/layer0_attention_weight_min": -13.064029693603516,
      "activations/layer10_attention_weight_max": 36.88993835449219,
      "activations/layer10_attention_weight_min": -37.69500732421875,
      "activations/layer11_attention_weight_max": 37.251197814941406,
      "activations/layer11_attention_weight_min": -35.04446029663086,
      "activations/layer12_attention_weight_max": 38.0277214050293,
      "activations/layer12_attention_weight_min": -30.615253448486328,
      "activations/layer13_attention_weight_max": 45.4825553894043,
      "activations/layer13_attention_weight_min": -38.769954681396484,
      "activations/layer14_attention_weight_max": 49.73102951049805,
      "activations/layer14_attention_weight_min": -46.103843688964844,
      "activations/layer15_attention_weight_max": 40.24276351928711,
      "activations/layer15_attention_weight_min": -33.58926010131836,
      "activations/layer16_attention_weight_max": 35.19275665283203,
      "activations/layer16_attention_weight_min": -27.299333572387695,
      "activations/layer17_attention_weight_max": 34.48860168457031,
      "activations/layer17_attention_weight_min": -26.829021453857422,
      "activations/layer18_attention_weight_max": 33.65473175048828,
      "activations/layer18_attention_weight_min": -23.55193328857422,
      "activations/layer19_attention_weight_max": 37.1109619140625,
      "activations/layer19_attention_weight_min": -30.11452293395996,
      "activations/layer1_attention_weight_max": 17.84103775024414,
      "activations/layer1_attention_weight_min": -13.968835830688477,
      "activations/layer20_attention_weight_max": 33.70625686645508,
      "activations/layer20_attention_weight_min": -23.475444793701172,
      "activations/layer21_attention_weight_max": 31.3995361328125,
      "activations/layer21_attention_weight_min": -23.284135818481445,
      "activations/layer22_attention_weight_max": 47.42226791381836,
      "activations/layer22_attention_weight_min": -30.850725173950195,
      "activations/layer23_attention_weight_max": 41.47624206542969,
      "activations/layer23_attention_weight_min": -27.38422203063965,
      "activations/layer2_attention_weight_max": 32.506107330322266,
      "activations/layer2_attention_weight_min": -34.4554557800293,
      "activations/layer3_attention_weight_max": 101.54207611083984,
      "activations/layer3_attention_weight_min": -104.49022674560547,
      "activations/layer4_attention_weight_max": 118.25764465332031,
      "activations/layer4_attention_weight_min": -122.2574234008789,
      "activations/layer5_attention_weight_max": 53.93998718261719,
      "activations/layer5_attention_weight_min": -65.36386108398438,
      "activations/layer6_attention_weight_max": 47.82014465332031,
      "activations/layer6_attention_weight_min": -51.259883880615234,
      "activations/layer7_attention_weight_max": 95.39154815673828,
      "activations/layer7_attention_weight_min": -97.58417510986328,
      "activations/layer8_attention_weight_max": 46.54069900512695,
      "activations/layer8_attention_weight_min": -46.57857131958008,
      "activations/layer9_attention_weight_max": 36.53146743774414,
      "activations/layer9_attention_weight_min": -37.39051055908203,
      "epoch": 10.19,
      "learning_rate": 8.517083333333333e-05,
      "loss": 2.796,
      "step": 175300
    },
    {
      "activations/layer0_attention_weight_max": 16.175443649291992,
      "activations/layer0_attention_weight_min": -12.845918655395508,
      "activations/layer10_attention_weight_max": 31.18361473083496,
      "activations/layer10_attention_weight_min": -30.139537811279297,
      "activations/layer11_attention_weight_max": 30.69852066040039,
      "activations/layer11_attention_weight_min": -29.937583923339844,
      "activations/layer12_attention_weight_max": 23.59752655029297,
      "activations/layer12_attention_weight_min": -27.22492027282715,
      "activations/layer13_attention_weight_max": 37.377315521240234,
      "activations/layer13_attention_weight_min": -33.19428253173828,
      "activations/layer14_attention_weight_max": 46.842166900634766,
      "activations/layer14_attention_weight_min": -38.58653259277344,
      "activations/layer15_attention_weight_max": 37.086517333984375,
      "activations/layer15_attention_weight_min": -30.306066513061523,
      "activations/layer16_attention_weight_max": 31.039201736450195,
      "activations/layer16_attention_weight_min": -27.018861770629883,
      "activations/layer17_attention_weight_max": 33.08354568481445,
      "activations/layer17_attention_weight_min": -25.446374893188477,
      "activations/layer18_attention_weight_max": 32.25798797607422,
      "activations/layer18_attention_weight_min": -23.64894676208496,
      "activations/layer19_attention_weight_max": 37.25957489013672,
      "activations/layer19_attention_weight_min": -30.9417667388916,
      "activations/layer1_attention_weight_max": 16.808879852294922,
      "activations/layer1_attention_weight_min": -13.665754318237305,
      "activations/layer20_attention_weight_max": 29.497093200683594,
      "activations/layer20_attention_weight_min": -24.220279693603516,
      "activations/layer21_attention_weight_max": 29.154203414916992,
      "activations/layer21_attention_weight_min": -23.5164794921875,
      "activations/layer22_attention_weight_max": 49.084590911865234,
      "activations/layer22_attention_weight_min": -29.641984939575195,
      "activations/layer23_attention_weight_max": 40.336402893066406,
      "activations/layer23_attention_weight_min": -26.15822982788086,
      "activations/layer2_attention_weight_max": 29.524038314819336,
      "activations/layer2_attention_weight_min": -31.470298767089844,
      "activations/layer3_attention_weight_max": 86.81109619140625,
      "activations/layer3_attention_weight_min": -87.97718811035156,
      "activations/layer4_attention_weight_max": 109.14471435546875,
      "activations/layer4_attention_weight_min": -108.85420227050781,
      "activations/layer5_attention_weight_max": 51.42061996459961,
      "activations/layer5_attention_weight_min": -62.20222473144531,
      "activations/layer6_attention_weight_max": 41.41143798828125,
      "activations/layer6_attention_weight_min": -47.94918441772461,
      "activations/layer7_attention_weight_max": 85.03474426269531,
      "activations/layer7_attention_weight_min": -84.54080200195312,
      "activations/layer8_attention_weight_max": 37.967918395996094,
      "activations/layer8_attention_weight_min": -37.862911224365234,
      "activations/layer9_attention_weight_max": 31.465742111206055,
      "activations/layer9_attention_weight_min": -31.851581573486328,
      "epoch": 10.19,
      "learning_rate": 8.515189393939394e-05,
      "loss": 2.7806,
      "step": 175350
    },
    {
      "activations/layer0_attention_weight_max": 15.213858604431152,
      "activations/layer0_attention_weight_min": -12.775341987609863,
      "activations/layer10_attention_weight_max": 34.98465347290039,
      "activations/layer10_attention_weight_min": -33.54298400878906,
      "activations/layer11_attention_weight_max": 35.22822570800781,
      "activations/layer11_attention_weight_min": -32.569515228271484,
      "activations/layer12_attention_weight_max": 25.296253204345703,
      "activations/layer12_attention_weight_min": -26.320865631103516,
      "activations/layer13_attention_weight_max": 40.76607131958008,
      "activations/layer13_attention_weight_min": -37.758140563964844,
      "activations/layer14_attention_weight_max": 47.39521789550781,
      "activations/layer14_attention_weight_min": -40.71084213256836,
      "activations/layer15_attention_weight_max": 39.10090637207031,
      "activations/layer15_attention_weight_min": -32.35152053833008,
      "activations/layer16_attention_weight_max": 34.24524688720703,
      "activations/layer16_attention_weight_min": -28.87306022644043,
      "activations/layer17_attention_weight_max": 35.17919921875,
      "activations/layer17_attention_weight_min": -28.125951766967773,
      "activations/layer18_attention_weight_max": 30.787755966186523,
      "activations/layer18_attention_weight_min": -25.12055015563965,
      "activations/layer19_attention_weight_max": 36.44016647338867,
      "activations/layer19_attention_weight_min": -31.686853408813477,
      "activations/layer1_attention_weight_max": 17.11356544494629,
      "activations/layer1_attention_weight_min": -14.8297758102417,
      "activations/layer20_attention_weight_max": 30.401866912841797,
      "activations/layer20_attention_weight_min": -26.911453247070312,
      "activations/layer21_attention_weight_max": 31.0388240814209,
      "activations/layer21_attention_weight_min": -25.013212203979492,
      "activations/layer22_attention_weight_max": 53.38241958618164,
      "activations/layer22_attention_weight_min": -29.30881118774414,
      "activations/layer23_attention_weight_max": 38.01408767700195,
      "activations/layer23_attention_weight_min": -26.188308715820312,
      "activations/layer2_attention_weight_max": 33.62142562866211,
      "activations/layer2_attention_weight_min": -33.19774627685547,
      "activations/layer3_attention_weight_max": 94.16397857666016,
      "activations/layer3_attention_weight_min": -94.19430541992188,
      "activations/layer4_attention_weight_max": 113.4135513305664,
      "activations/layer4_attention_weight_min": -114.6456298828125,
      "activations/layer5_attention_weight_max": 52.91785430908203,
      "activations/layer5_attention_weight_min": -65.38081359863281,
      "activations/layer6_attention_weight_max": 44.74336624145508,
      "activations/layer6_attention_weight_min": -50.415653228759766,
      "activations/layer7_attention_weight_max": 89.1554183959961,
      "activations/layer7_attention_weight_min": -92.18976593017578,
      "activations/layer8_attention_weight_max": 41.599517822265625,
      "activations/layer8_attention_weight_min": -42.83884811401367,
      "activations/layer9_attention_weight_max": 38.61491775512695,
      "activations/layer9_attention_weight_min": -33.80677795410156,
      "epoch": 10.19,
      "learning_rate": 8.513295454545453e-05,
      "loss": 2.7852,
      "step": 175400
    },
    {
      "activations/layer0_attention_weight_max": 15.422383308410645,
      "activations/layer0_attention_weight_min": -12.736425399780273,
      "activations/layer10_attention_weight_max": 34.09013748168945,
      "activations/layer10_attention_weight_min": -31.679073333740234,
      "activations/layer11_attention_weight_max": 33.93535614013672,
      "activations/layer11_attention_weight_min": -30.64901351928711,
      "activations/layer12_attention_weight_max": 23.648353576660156,
      "activations/layer12_attention_weight_min": -27.96739387512207,
      "activations/layer13_attention_weight_max": 37.40543746948242,
      "activations/layer13_attention_weight_min": -32.52136993408203,
      "activations/layer14_attention_weight_max": 46.376495361328125,
      "activations/layer14_attention_weight_min": -40.442569732666016,
      "activations/layer15_attention_weight_max": 35.514774322509766,
      "activations/layer15_attention_weight_min": -30.376304626464844,
      "activations/layer16_attention_weight_max": 35.11077117919922,
      "activations/layer16_attention_weight_min": -26.88994598388672,
      "activations/layer17_attention_weight_max": 34.33121109008789,
      "activations/layer17_attention_weight_min": -25.49759864807129,
      "activations/layer18_attention_weight_max": 32.64426040649414,
      "activations/layer18_attention_weight_min": -24.395099639892578,
      "activations/layer19_attention_weight_max": 37.74459457397461,
      "activations/layer19_attention_weight_min": -29.63396644592285,
      "activations/layer1_attention_weight_max": 16.676523208618164,
      "activations/layer1_attention_weight_min": -14.721779823303223,
      "activations/layer20_attention_weight_max": 31.605987548828125,
      "activations/layer20_attention_weight_min": -24.333459854125977,
      "activations/layer21_attention_weight_max": 29.70787811279297,
      "activations/layer21_attention_weight_min": -24.188621520996094,
      "activations/layer22_attention_weight_max": 49.06097412109375,
      "activations/layer22_attention_weight_min": -30.904802322387695,
      "activations/layer23_attention_weight_max": 45.33616638183594,
      "activations/layer23_attention_weight_min": -24.684568405151367,
      "activations/layer2_attention_weight_max": 31.79505157470703,
      "activations/layer2_attention_weight_min": -32.34673309326172,
      "activations/layer3_attention_weight_max": 93.80139923095703,
      "activations/layer3_attention_weight_min": -92.7248306274414,
      "activations/layer4_attention_weight_max": 111.8908462524414,
      "activations/layer4_attention_weight_min": -108.17242431640625,
      "activations/layer5_attention_weight_max": 51.83747100830078,
      "activations/layer5_attention_weight_min": -57.80133819580078,
      "activations/layer6_attention_weight_max": 44.18114471435547,
      "activations/layer6_attention_weight_min": -49.33386993408203,
      "activations/layer7_attention_weight_max": 87.87201690673828,
      "activations/layer7_attention_weight_min": -92.53790283203125,
      "activations/layer8_attention_weight_max": 40.173439025878906,
      "activations/layer8_attention_weight_min": -39.27455520629883,
      "activations/layer9_attention_weight_max": 33.24896240234375,
      "activations/layer9_attention_weight_min": -33.013492584228516,
      "epoch": 10.19,
      "learning_rate": 8.511401515151515e-05,
      "loss": 2.7969,
      "step": 175450
    },
    {
      "activations/layer0_attention_weight_max": 15.430183410644531,
      "activations/layer0_attention_weight_min": -12.661290168762207,
      "activations/layer10_attention_weight_max": 45.45124053955078,
      "activations/layer10_attention_weight_min": -39.72859573364258,
      "activations/layer11_attention_weight_max": 47.67252731323242,
      "activations/layer11_attention_weight_min": -41.729331970214844,
      "activations/layer12_attention_weight_max": 24.009078979492188,
      "activations/layer12_attention_weight_min": -26.82221794128418,
      "activations/layer13_attention_weight_max": 36.776100158691406,
      "activations/layer13_attention_weight_min": -33.825870513916016,
      "activations/layer14_attention_weight_max": 45.25419616699219,
      "activations/layer14_attention_weight_min": -37.673057556152344,
      "activations/layer15_attention_weight_max": 35.68894577026367,
      "activations/layer15_attention_weight_min": -30.984785079956055,
      "activations/layer16_attention_weight_max": 30.75269317626953,
      "activations/layer16_attention_weight_min": -29.976171493530273,
      "activations/layer17_attention_weight_max": 31.794973373413086,
      "activations/layer17_attention_weight_min": -25.377222061157227,
      "activations/layer18_attention_weight_max": 30.192440032958984,
      "activations/layer18_attention_weight_min": -22.61302375793457,
      "activations/layer19_attention_weight_max": 32.64088439941406,
      "activations/layer19_attention_weight_min": -32.35405349731445,
      "activations/layer1_attention_weight_max": 17.19373321533203,
      "activations/layer1_attention_weight_min": -14.379286766052246,
      "activations/layer20_attention_weight_max": 32.06324005126953,
      "activations/layer20_attention_weight_min": -23.601343154907227,
      "activations/layer21_attention_weight_max": 31.318527221679688,
      "activations/layer21_attention_weight_min": -23.352102279663086,
      "activations/layer22_attention_weight_max": 44.35633850097656,
      "activations/layer22_attention_weight_min": -29.14940071105957,
      "activations/layer23_attention_weight_max": 36.449310302734375,
      "activations/layer23_attention_weight_min": -24.601316452026367,
      "activations/layer2_attention_weight_max": 34.04301071166992,
      "activations/layer2_attention_weight_min": -36.02180099487305,
      "activations/layer3_attention_weight_max": 99.41924285888672,
      "activations/layer3_attention_weight_min": -104.26973724365234,
      "activations/layer4_attention_weight_max": 121.52559661865234,
      "activations/layer4_attention_weight_min": -129.43951416015625,
      "activations/layer5_attention_weight_max": 58.15224838256836,
      "activations/layer5_attention_weight_min": -64.53732299804688,
      "activations/layer6_attention_weight_max": 49.42155075073242,
      "activations/layer6_attention_weight_min": -52.15489196777344,
      "activations/layer7_attention_weight_max": 108.52412414550781,
      "activations/layer7_attention_weight_min": -106.96302795410156,
      "activations/layer8_attention_weight_max": 50.68278884887695,
      "activations/layer8_attention_weight_min": -47.07673263549805,
      "activations/layer9_attention_weight_max": 43.32906723022461,
      "activations/layer9_attention_weight_min": -40.29636764526367,
      "epoch": 10.2,
      "learning_rate": 8.509507575757575e-05,
      "loss": 2.7826,
      "step": 175500
    },
    {
      "activations/layer0_attention_weight_max": 16.013282775878906,
      "activations/layer0_attention_weight_min": -12.992030143737793,
      "activations/layer10_attention_weight_max": 35.30427169799805,
      "activations/layer10_attention_weight_min": -34.748390197753906,
      "activations/layer11_attention_weight_max": 34.322021484375,
      "activations/layer11_attention_weight_min": -32.704620361328125,
      "activations/layer12_attention_weight_max": 43.02593994140625,
      "activations/layer12_attention_weight_min": -27.225622177124023,
      "activations/layer13_attention_weight_max": 45.93762969970703,
      "activations/layer13_attention_weight_min": -35.041770935058594,
      "activations/layer14_attention_weight_max": 54.27701187133789,
      "activations/layer14_attention_weight_min": -40.77790069580078,
      "activations/layer15_attention_weight_max": 41.85139465332031,
      "activations/layer15_attention_weight_min": -33.33527374267578,
      "activations/layer16_attention_weight_max": 33.87063980102539,
      "activations/layer16_attention_weight_min": -29.138185501098633,
      "activations/layer17_attention_weight_max": 38.796112060546875,
      "activations/layer17_attention_weight_min": -25.684173583984375,
      "activations/layer18_attention_weight_max": 32.450897216796875,
      "activations/layer18_attention_weight_min": -25.17607879638672,
      "activations/layer19_attention_weight_max": 37.16360092163086,
      "activations/layer19_attention_weight_min": -32.62873077392578,
      "activations/layer1_attention_weight_max": 17.863384246826172,
      "activations/layer1_attention_weight_min": -16.913766860961914,
      "activations/layer20_attention_weight_max": 32.70526885986328,
      "activations/layer20_attention_weight_min": -25.797256469726562,
      "activations/layer21_attention_weight_max": 30.76598358154297,
      "activations/layer21_attention_weight_min": -26.531925201416016,
      "activations/layer22_attention_weight_max": 56.62460708618164,
      "activations/layer22_attention_weight_min": -32.21507263183594,
      "activations/layer23_attention_weight_max": 39.005775451660156,
      "activations/layer23_attention_weight_min": -24.8187255859375,
      "activations/layer2_attention_weight_max": 32.96278381347656,
      "activations/layer2_attention_weight_min": -34.10283279418945,
      "activations/layer3_attention_weight_max": 94.30084228515625,
      "activations/layer3_attention_weight_min": -95.58732604980469,
      "activations/layer4_attention_weight_max": 110.69791412353516,
      "activations/layer4_attention_weight_min": -111.1199722290039,
      "activations/layer5_attention_weight_max": 51.76634979248047,
      "activations/layer5_attention_weight_min": -63.683067321777344,
      "activations/layer6_attention_weight_max": 44.4453010559082,
      "activations/layer6_attention_weight_min": -49.05934143066406,
      "activations/layer7_attention_weight_max": 88.53035736083984,
      "activations/layer7_attention_weight_min": -89.69821166992188,
      "activations/layer8_attention_weight_max": 40.64601135253906,
      "activations/layer8_attention_weight_min": -40.08332443237305,
      "activations/layer9_attention_weight_max": 35.43641662597656,
      "activations/layer9_attention_weight_min": -32.88905334472656,
      "epoch": 10.2,
      "learning_rate": 8.507613636363635e-05,
      "loss": 2.7964,
      "step": 175550
    },
    {
      "activations/layer0_attention_weight_max": 16.423051834106445,
      "activations/layer0_attention_weight_min": -12.62220287322998,
      "activations/layer10_attention_weight_max": 32.62274932861328,
      "activations/layer10_attention_weight_min": -31.045455932617188,
      "activations/layer11_attention_weight_max": 32.0971565246582,
      "activations/layer11_attention_weight_min": -31.50598907470703,
      "activations/layer12_attention_weight_max": 25.246667861938477,
      "activations/layer12_attention_weight_min": -35.58354568481445,
      "activations/layer13_attention_weight_max": 40.528099060058594,
      "activations/layer13_attention_weight_min": -32.87519073486328,
      "activations/layer14_attention_weight_max": 44.23520278930664,
      "activations/layer14_attention_weight_min": -37.696510314941406,
      "activations/layer15_attention_weight_max": 36.07075881958008,
      "activations/layer15_attention_weight_min": -30.676780700683594,
      "activations/layer16_attention_weight_max": 31.405410766601562,
      "activations/layer16_attention_weight_min": -27.58498191833496,
      "activations/layer17_attention_weight_max": 29.97951316833496,
      "activations/layer17_attention_weight_min": -25.807863235473633,
      "activations/layer18_attention_weight_max": 33.127708435058594,
      "activations/layer18_attention_weight_min": -22.144224166870117,
      "activations/layer19_attention_weight_max": 32.52735900878906,
      "activations/layer19_attention_weight_min": -30.32588768005371,
      "activations/layer1_attention_weight_max": 16.904518127441406,
      "activations/layer1_attention_weight_min": -14.892256736755371,
      "activations/layer20_attention_weight_max": 31.432044982910156,
      "activations/layer20_attention_weight_min": -22.780536651611328,
      "activations/layer21_attention_weight_max": 33.24834442138672,
      "activations/layer21_attention_weight_min": -23.115705490112305,
      "activations/layer22_attention_weight_max": 44.04828643798828,
      "activations/layer22_attention_weight_min": -27.772695541381836,
      "activations/layer23_attention_weight_max": 30.856199264526367,
      "activations/layer23_attention_weight_min": -22.736051559448242,
      "activations/layer2_attention_weight_max": 31.77955436706543,
      "activations/layer2_attention_weight_min": -34.718746185302734,
      "activations/layer3_attention_weight_max": 92.74755859375,
      "activations/layer3_attention_weight_min": -92.40619659423828,
      "activations/layer4_attention_weight_max": 110.2745361328125,
      "activations/layer4_attention_weight_min": -108.8043212890625,
      "activations/layer5_attention_weight_max": 54.943111419677734,
      "activations/layer5_attention_weight_min": -60.75032043457031,
      "activations/layer6_attention_weight_max": 48.252559661865234,
      "activations/layer6_attention_weight_min": -47.50212478637695,
      "activations/layer7_attention_weight_max": 89.17462158203125,
      "activations/layer7_attention_weight_min": -90.25166320800781,
      "activations/layer8_attention_weight_max": 40.34895706176758,
      "activations/layer8_attention_weight_min": -41.64670181274414,
      "activations/layer9_attention_weight_max": 33.430747985839844,
      "activations/layer9_attention_weight_min": -33.94868850708008,
      "epoch": 10.2,
      "learning_rate": 8.505719696969696e-05,
      "loss": 2.7912,
      "step": 175600
    },
    {
      "activations/layer0_attention_weight_max": 16.484777450561523,
      "activations/layer0_attention_weight_min": -13.43881607055664,
      "activations/layer10_attention_weight_max": 35.13585662841797,
      "activations/layer10_attention_weight_min": -30.76934814453125,
      "activations/layer11_attention_weight_max": 32.618038177490234,
      "activations/layer11_attention_weight_min": -30.285863876342773,
      "activations/layer12_attention_weight_max": 24.27136993408203,
      "activations/layer12_attention_weight_min": -25.180204391479492,
      "activations/layer13_attention_weight_max": 38.51069641113281,
      "activations/layer13_attention_weight_min": -34.33693313598633,
      "activations/layer14_attention_weight_max": 44.28512954711914,
      "activations/layer14_attention_weight_min": -36.65276336669922,
      "activations/layer15_attention_weight_max": 35.328636169433594,
      "activations/layer15_attention_weight_min": -31.583709716796875,
      "activations/layer16_attention_weight_max": 32.07054138183594,
      "activations/layer16_attention_weight_min": -26.92410659790039,
      "activations/layer17_attention_weight_max": 28.455509185791016,
      "activations/layer17_attention_weight_min": -25.716238021850586,
      "activations/layer18_attention_weight_max": 30.11962890625,
      "activations/layer18_attention_weight_min": -24.903480529785156,
      "activations/layer19_attention_weight_max": 41.74409484863281,
      "activations/layer19_attention_weight_min": -31.884611129760742,
      "activations/layer1_attention_weight_max": 16.420867919921875,
      "activations/layer1_attention_weight_min": -14.459504127502441,
      "activations/layer20_attention_weight_max": 30.21974754333496,
      "activations/layer20_attention_weight_min": -26.474246978759766,
      "activations/layer21_attention_weight_max": 32.7164421081543,
      "activations/layer21_attention_weight_min": -25.02106285095215,
      "activations/layer22_attention_weight_max": 49.7286262512207,
      "activations/layer22_attention_weight_min": -30.658472061157227,
      "activations/layer23_attention_weight_max": 37.0347900390625,
      "activations/layer23_attention_weight_min": -28.036304473876953,
      "activations/layer2_attention_weight_max": 32.39667510986328,
      "activations/layer2_attention_weight_min": -33.495819091796875,
      "activations/layer3_attention_weight_max": 90.75929260253906,
      "activations/layer3_attention_weight_min": -93.2202377319336,
      "activations/layer4_attention_weight_max": 113.10367584228516,
      "activations/layer4_attention_weight_min": -108.12921142578125,
      "activations/layer5_attention_weight_max": 52.83116149902344,
      "activations/layer5_attention_weight_min": -61.312782287597656,
      "activations/layer6_attention_weight_max": 46.90964889526367,
      "activations/layer6_attention_weight_min": -49.028663635253906,
      "activations/layer7_attention_weight_max": 96.30378723144531,
      "activations/layer7_attention_weight_min": -88.73741149902344,
      "activations/layer8_attention_weight_max": 40.12750244140625,
      "activations/layer8_attention_weight_min": -41.355804443359375,
      "activations/layer9_attention_weight_max": 34.810001373291016,
      "activations/layer9_attention_weight_min": -32.16050720214844,
      "epoch": 10.21,
      "learning_rate": 8.503825757575757e-05,
      "loss": 2.7909,
      "step": 175650
    },
    {
      "activations/layer0_attention_weight_max": 15.636685371398926,
      "activations/layer0_attention_weight_min": -11.489606857299805,
      "activations/layer10_attention_weight_max": 31.158178329467773,
      "activations/layer10_attention_weight_min": -29.589590072631836,
      "activations/layer11_attention_weight_max": 31.690221786499023,
      "activations/layer11_attention_weight_min": -29.30109405517578,
      "activations/layer12_attention_weight_max": 34.516048431396484,
      "activations/layer12_attention_weight_min": -29.32929039001465,
      "activations/layer13_attention_weight_max": 40.716583251953125,
      "activations/layer13_attention_weight_min": -34.114803314208984,
      "activations/layer14_attention_weight_max": 43.73634338378906,
      "activations/layer14_attention_weight_min": -38.24411392211914,
      "activations/layer15_attention_weight_max": 35.89223861694336,
      "activations/layer15_attention_weight_min": -32.913455963134766,
      "activations/layer16_attention_weight_max": 31.005924224853516,
      "activations/layer16_attention_weight_min": -27.9609375,
      "activations/layer17_attention_weight_max": 34.828163146972656,
      "activations/layer17_attention_weight_min": -26.748199462890625,
      "activations/layer18_attention_weight_max": 38.1573371887207,
      "activations/layer18_attention_weight_min": -23.3397159576416,
      "activations/layer19_attention_weight_max": 38.01200485229492,
      "activations/layer19_attention_weight_min": -31.57614517211914,
      "activations/layer1_attention_weight_max": 15.92654800415039,
      "activations/layer1_attention_weight_min": -15.818713188171387,
      "activations/layer20_attention_weight_max": 34.62510681152344,
      "activations/layer20_attention_weight_min": -25.899812698364258,
      "activations/layer21_attention_weight_max": 30.51498794555664,
      "activations/layer21_attention_weight_min": -24.138580322265625,
      "activations/layer22_attention_weight_max": 45.84765625,
      "activations/layer22_attention_weight_min": -31.176578521728516,
      "activations/layer23_attention_weight_max": 40.49079895019531,
      "activations/layer23_attention_weight_min": -25.95314598083496,
      "activations/layer2_attention_weight_max": 32.707374572753906,
      "activations/layer2_attention_weight_min": -32.84652328491211,
      "activations/layer3_attention_weight_max": 88.75699615478516,
      "activations/layer3_attention_weight_min": -92.10572814941406,
      "activations/layer4_attention_weight_max": 105.85736083984375,
      "activations/layer4_attention_weight_min": -107.7620620727539,
      "activations/layer5_attention_weight_max": 50.457820892333984,
      "activations/layer5_attention_weight_min": -59.79864501953125,
      "activations/layer6_attention_weight_max": 41.23109436035156,
      "activations/layer6_attention_weight_min": -45.101810455322266,
      "activations/layer7_attention_weight_max": 87.23001098632812,
      "activations/layer7_attention_weight_min": -85.7636489868164,
      "activations/layer8_attention_weight_max": 37.58345413208008,
      "activations/layer8_attention_weight_min": -38.667747497558594,
      "activations/layer9_attention_weight_max": 32.80506134033203,
      "activations/layer9_attention_weight_min": -30.73988151550293,
      "epoch": 10.21,
      "learning_rate": 8.501931818181818e-05,
      "loss": 2.788,
      "step": 175700
    },
    {
      "activations/layer0_attention_weight_max": 15.210026741027832,
      "activations/layer0_attention_weight_min": -12.608078002929688,
      "activations/layer10_attention_weight_max": 29.649370193481445,
      "activations/layer10_attention_weight_min": -30.635271072387695,
      "activations/layer11_attention_weight_max": 30.531286239624023,
      "activations/layer11_attention_weight_min": -29.852712631225586,
      "activations/layer12_attention_weight_max": 25.47740936279297,
      "activations/layer12_attention_weight_min": -25.85365104675293,
      "activations/layer13_attention_weight_max": 37.903099060058594,
      "activations/layer13_attention_weight_min": -31.469329833984375,
      "activations/layer14_attention_weight_max": 43.86269760131836,
      "activations/layer14_attention_weight_min": -34.86991882324219,
      "activations/layer15_attention_weight_max": 35.47433853149414,
      "activations/layer15_attention_weight_min": -30.11082649230957,
      "activations/layer16_attention_weight_max": 33.86243438720703,
      "activations/layer16_attention_weight_min": -27.03990364074707,
      "activations/layer17_attention_weight_max": 32.803367614746094,
      "activations/layer17_attention_weight_min": -25.404428482055664,
      "activations/layer18_attention_weight_max": 31.585487365722656,
      "activations/layer18_attention_weight_min": -21.917924880981445,
      "activations/layer19_attention_weight_max": 36.5728874206543,
      "activations/layer19_attention_weight_min": -31.324045181274414,
      "activations/layer1_attention_weight_max": 16.429927825927734,
      "activations/layer1_attention_weight_min": -14.511451721191406,
      "activations/layer20_attention_weight_max": 27.0242977142334,
      "activations/layer20_attention_weight_min": -25.54507064819336,
      "activations/layer21_attention_weight_max": 29.46408462524414,
      "activations/layer21_attention_weight_min": -23.584014892578125,
      "activations/layer22_attention_weight_max": 45.771278381347656,
      "activations/layer22_attention_weight_min": -27.52588653564453,
      "activations/layer23_attention_weight_max": 33.70879364013672,
      "activations/layer23_attention_weight_min": -24.728513717651367,
      "activations/layer2_attention_weight_max": 27.609609603881836,
      "activations/layer2_attention_weight_min": -29.331546783447266,
      "activations/layer3_attention_weight_max": 81.51655578613281,
      "activations/layer3_attention_weight_min": -82.2895736694336,
      "activations/layer4_attention_weight_max": 101.97402954101562,
      "activations/layer4_attention_weight_min": -97.87210083007812,
      "activations/layer5_attention_weight_max": 51.97312927246094,
      "activations/layer5_attention_weight_min": -56.92023468017578,
      "activations/layer6_attention_weight_max": 42.844295501708984,
      "activations/layer6_attention_weight_min": -45.292911529541016,
      "activations/layer7_attention_weight_max": 86.5662841796875,
      "activations/layer7_attention_weight_min": -83.26414489746094,
      "activations/layer8_attention_weight_max": 37.62300491333008,
      "activations/layer8_attention_weight_min": -41.294166564941406,
      "activations/layer9_attention_weight_max": 31.582660675048828,
      "activations/layer9_attention_weight_min": -31.097986221313477,
      "epoch": 10.21,
      "learning_rate": 8.500037878787878e-05,
      "loss": 2.7794,
      "step": 175750
    },
    {
      "activations/layer0_attention_weight_max": 16.084104537963867,
      "activations/layer0_attention_weight_min": -12.944241523742676,
      "activations/layer10_attention_weight_max": 34.13542938232422,
      "activations/layer10_attention_weight_min": -31.778276443481445,
      "activations/layer11_attention_weight_max": 37.028846740722656,
      "activations/layer11_attention_weight_min": -32.334739685058594,
      "activations/layer12_attention_weight_max": 25.878435134887695,
      "activations/layer12_attention_weight_min": -25.101707458496094,
      "activations/layer13_attention_weight_max": 39.35445022583008,
      "activations/layer13_attention_weight_min": -35.566200256347656,
      "activations/layer14_attention_weight_max": 46.66557312011719,
      "activations/layer14_attention_weight_min": -39.003421783447266,
      "activations/layer15_attention_weight_max": 35.75352096557617,
      "activations/layer15_attention_weight_min": -32.348201751708984,
      "activations/layer16_attention_weight_max": 32.34473419189453,
      "activations/layer16_attention_weight_min": -28.944076538085938,
      "activations/layer17_attention_weight_max": 30.584619522094727,
      "activations/layer17_attention_weight_min": -23.679954528808594,
      "activations/layer18_attention_weight_max": 31.84112548828125,
      "activations/layer18_attention_weight_min": -23.895370483398438,
      "activations/layer19_attention_weight_max": 33.57381820678711,
      "activations/layer19_attention_weight_min": -31.148818969726562,
      "activations/layer1_attention_weight_max": 16.935163497924805,
      "activations/layer1_attention_weight_min": -15.437320709228516,
      "activations/layer20_attention_weight_max": 28.91636848449707,
      "activations/layer20_attention_weight_min": -24.26093101501465,
      "activations/layer21_attention_weight_max": 26.085880279541016,
      "activations/layer21_attention_weight_min": -21.830791473388672,
      "activations/layer22_attention_weight_max": 47.738685607910156,
      "activations/layer22_attention_weight_min": -27.969623565673828,
      "activations/layer23_attention_weight_max": 36.2079963684082,
      "activations/layer23_attention_weight_min": -24.88521385192871,
      "activations/layer2_attention_weight_max": 32.70203399658203,
      "activations/layer2_attention_weight_min": -33.52825927734375,
      "activations/layer3_attention_weight_max": 94.10932922363281,
      "activations/layer3_attention_weight_min": -97.19397735595703,
      "activations/layer4_attention_weight_max": 112.68214416503906,
      "activations/layer4_attention_weight_min": -118.97774505615234,
      "activations/layer5_attention_weight_max": 52.8829345703125,
      "activations/layer5_attention_weight_min": -63.459259033203125,
      "activations/layer6_attention_weight_max": 45.78976058959961,
      "activations/layer6_attention_weight_min": -47.90694046020508,
      "activations/layer7_attention_weight_max": 89.115478515625,
      "activations/layer7_attention_weight_min": -88.28031158447266,
      "activations/layer8_attention_weight_max": 42.694679260253906,
      "activations/layer8_attention_weight_min": -40.67308807373047,
      "activations/layer9_attention_weight_max": 35.52367401123047,
      "activations/layer9_attention_weight_min": -33.204288482666016,
      "epoch": 10.21,
      "learning_rate": 8.498143939393938e-05,
      "loss": 2.7752,
      "step": 175800
    },
    {
      "activations/layer0_attention_weight_max": 17.28510284423828,
      "activations/layer0_attention_weight_min": -11.82194709777832,
      "activations/layer10_attention_weight_max": 32.1894416809082,
      "activations/layer10_attention_weight_min": -30.01805305480957,
      "activations/layer11_attention_weight_max": 30.952434539794922,
      "activations/layer11_attention_weight_min": -29.07913589477539,
      "activations/layer12_attention_weight_max": 26.401880264282227,
      "activations/layer12_attention_weight_min": -25.366601943969727,
      "activations/layer13_attention_weight_max": 42.06801986694336,
      "activations/layer13_attention_weight_min": -34.62677764892578,
      "activations/layer14_attention_weight_max": 48.78223419189453,
      "activations/layer14_attention_weight_min": -39.72415542602539,
      "activations/layer15_attention_weight_max": 40.814048767089844,
      "activations/layer15_attention_weight_min": -33.390174865722656,
      "activations/layer16_attention_weight_max": 35.231136322021484,
      "activations/layer16_attention_weight_min": -27.778976440429688,
      "activations/layer17_attention_weight_max": 36.298370361328125,
      "activations/layer17_attention_weight_min": -25.20033073425293,
      "activations/layer18_attention_weight_max": 35.364524841308594,
      "activations/layer18_attention_weight_min": -24.090612411499023,
      "activations/layer19_attention_weight_max": 40.96572494506836,
      "activations/layer19_attention_weight_min": -34.73616409301758,
      "activations/layer1_attention_weight_max": 16.559059143066406,
      "activations/layer1_attention_weight_min": -15.806917190551758,
      "activations/layer20_attention_weight_max": 38.653404235839844,
      "activations/layer20_attention_weight_min": -25.95956039428711,
      "activations/layer21_attention_weight_max": 34.49652099609375,
      "activations/layer21_attention_weight_min": -25.4702205657959,
      "activations/layer22_attention_weight_max": 54.32024002075195,
      "activations/layer22_attention_weight_min": -30.490184783935547,
      "activations/layer23_attention_weight_max": 41.18438720703125,
      "activations/layer23_attention_weight_min": -28.72350311279297,
      "activations/layer2_attention_weight_max": 31.14541244506836,
      "activations/layer2_attention_weight_min": -32.32222366333008,
      "activations/layer3_attention_weight_max": 90.5814208984375,
      "activations/layer3_attention_weight_min": -92.70182800292969,
      "activations/layer4_attention_weight_max": 111.18807220458984,
      "activations/layer4_attention_weight_min": -113.8682632446289,
      "activations/layer5_attention_weight_max": 51.27729797363281,
      "activations/layer5_attention_weight_min": -58.13551330566406,
      "activations/layer6_attention_weight_max": 45.38399887084961,
      "activations/layer6_attention_weight_min": -45.56476974487305,
      "activations/layer7_attention_weight_max": 88.24278259277344,
      "activations/layer7_attention_weight_min": -83.68173217773438,
      "activations/layer8_attention_weight_max": 39.615055084228516,
      "activations/layer8_attention_weight_min": -37.20166015625,
      "activations/layer9_attention_weight_max": 33.16475296020508,
      "activations/layer9_attention_weight_min": -31.562742233276367,
      "epoch": 10.22,
      "learning_rate": 8.49625e-05,
      "loss": 2.785,
      "step": 175850
    },
    {
      "activations/layer0_attention_weight_max": 15.562374114990234,
      "activations/layer0_attention_weight_min": -13.150779724121094,
      "activations/layer10_attention_weight_max": 46.276329040527344,
      "activations/layer10_attention_weight_min": -43.58415222167969,
      "activations/layer11_attention_weight_max": 48.74604415893555,
      "activations/layer11_attention_weight_min": -43.166927337646484,
      "activations/layer12_attention_weight_max": 48.33821487426758,
      "activations/layer12_attention_weight_min": -31.020414352416992,
      "activations/layer13_attention_weight_max": 71.74436950683594,
      "activations/layer13_attention_weight_min": -40.77360534667969,
      "activations/layer14_attention_weight_max": 62.62422180175781,
      "activations/layer14_attention_weight_min": -41.436248779296875,
      "activations/layer15_attention_weight_max": 61.4443473815918,
      "activations/layer15_attention_weight_min": -37.322845458984375,
      "activations/layer16_attention_weight_max": 40.662353515625,
      "activations/layer16_attention_weight_min": -28.640979766845703,
      "activations/layer17_attention_weight_max": 28.427947998046875,
      "activations/layer17_attention_weight_min": -28.96816062927246,
      "activations/layer18_attention_weight_max": 27.20265007019043,
      "activations/layer18_attention_weight_min": -24.57783317565918,
      "activations/layer19_attention_weight_max": 34.10783004760742,
      "activations/layer19_attention_weight_min": -31.323331832885742,
      "activations/layer1_attention_weight_max": 17.325923919677734,
      "activations/layer1_attention_weight_min": -14.950817108154297,
      "activations/layer20_attention_weight_max": 28.805618286132812,
      "activations/layer20_attention_weight_min": -24.946062088012695,
      "activations/layer21_attention_weight_max": 28.249582290649414,
      "activations/layer21_attention_weight_min": -24.120012283325195,
      "activations/layer22_attention_weight_max": 45.21975326538086,
      "activations/layer22_attention_weight_min": -32.69041442871094,
      "activations/layer23_attention_weight_max": 34.02044677734375,
      "activations/layer23_attention_weight_min": -25.886411666870117,
      "activations/layer2_attention_weight_max": 36.0142936706543,
      "activations/layer2_attention_weight_min": -37.68000030517578,
      "activations/layer3_attention_weight_max": 102.79298400878906,
      "activations/layer3_attention_weight_min": -110.23961639404297,
      "activations/layer4_attention_weight_max": 125.55196380615234,
      "activations/layer4_attention_weight_min": -124.3859634399414,
      "activations/layer5_attention_weight_max": 63.026851654052734,
      "activations/layer5_attention_weight_min": -63.00187301635742,
      "activations/layer6_attention_weight_max": 50.48509979248047,
      "activations/layer6_attention_weight_min": -52.01467514038086,
      "activations/layer7_attention_weight_max": 113.63218688964844,
      "activations/layer7_attention_weight_min": -116.05614471435547,
      "activations/layer8_attention_weight_max": 54.37550354003906,
      "activations/layer8_attention_weight_min": -52.88050842285156,
      "activations/layer9_attention_weight_max": 47.04273223876953,
      "activations/layer9_attention_weight_min": -43.9221305847168,
      "epoch": 10.22,
      "learning_rate": 8.49435606060606e-05,
      "loss": 2.7858,
      "step": 175900
    },
    {
      "activations/layer0_attention_weight_max": 15.037720680236816,
      "activations/layer0_attention_weight_min": -13.159655570983887,
      "activations/layer10_attention_weight_max": 31.960205078125,
      "activations/layer10_attention_weight_min": -31.273435592651367,
      "activations/layer11_attention_weight_max": 30.52430534362793,
      "activations/layer11_attention_weight_min": -29.63486099243164,
      "activations/layer12_attention_weight_max": 23.24663734436035,
      "activations/layer12_attention_weight_min": -25.06056022644043,
      "activations/layer13_attention_weight_max": 35.973670959472656,
      "activations/layer13_attention_weight_min": -33.40402603149414,
      "activations/layer14_attention_weight_max": 48.52510452270508,
      "activations/layer14_attention_weight_min": -35.759822845458984,
      "activations/layer15_attention_weight_max": 35.65991973876953,
      "activations/layer15_attention_weight_min": -31.368436813354492,
      "activations/layer16_attention_weight_max": 32.283851623535156,
      "activations/layer16_attention_weight_min": -26.116790771484375,
      "activations/layer17_attention_weight_max": 34.08932876586914,
      "activations/layer17_attention_weight_min": -25.507198333740234,
      "activations/layer18_attention_weight_max": 30.850004196166992,
      "activations/layer18_attention_weight_min": -21.91780662536621,
      "activations/layer19_attention_weight_max": 38.680477142333984,
      "activations/layer19_attention_weight_min": -32.47895812988281,
      "activations/layer1_attention_weight_max": 17.20096206665039,
      "activations/layer1_attention_weight_min": -14.651033401489258,
      "activations/layer20_attention_weight_max": 32.000797271728516,
      "activations/layer20_attention_weight_min": -26.873638153076172,
      "activations/layer21_attention_weight_max": 30.516189575195312,
      "activations/layer21_attention_weight_min": -26.087491989135742,
      "activations/layer22_attention_weight_max": 46.87176513671875,
      "activations/layer22_attention_weight_min": -31.073347091674805,
      "activations/layer23_attention_weight_max": 43.311500549316406,
      "activations/layer23_attention_weight_min": -23.495559692382812,
      "activations/layer2_attention_weight_max": 31.57040023803711,
      "activations/layer2_attention_weight_min": -32.21001434326172,
      "activations/layer3_attention_weight_max": 87.87154388427734,
      "activations/layer3_attention_weight_min": -89.79505157470703,
      "activations/layer4_attention_weight_max": 108.17411041259766,
      "activations/layer4_attention_weight_min": -106.4510726928711,
      "activations/layer5_attention_weight_max": 49.861083984375,
      "activations/layer5_attention_weight_min": -58.76026153564453,
      "activations/layer6_attention_weight_max": 42.13546371459961,
      "activations/layer6_attention_weight_min": -48.024662017822266,
      "activations/layer7_attention_weight_max": 83.3644027709961,
      "activations/layer7_attention_weight_min": -89.23214721679688,
      "activations/layer8_attention_weight_max": 37.30994415283203,
      "activations/layer8_attention_weight_min": -38.70549392700195,
      "activations/layer9_attention_weight_max": 33.802371978759766,
      "activations/layer9_attention_weight_min": -32.32644271850586,
      "epoch": 10.22,
      "learning_rate": 8.49246212121212e-05,
      "loss": 2.7845,
      "step": 175950
    },
    {
      "activations/layer0_attention_weight_max": 15.601415634155273,
      "activations/layer0_attention_weight_min": -13.148541450500488,
      "activations/layer10_attention_weight_max": 33.50510787963867,
      "activations/layer10_attention_weight_min": -32.859580993652344,
      "activations/layer11_attention_weight_max": 34.009857177734375,
      "activations/layer11_attention_weight_min": -35.08364486694336,
      "activations/layer12_attention_weight_max": 24.451087951660156,
      "activations/layer12_attention_weight_min": -25.24864387512207,
      "activations/layer13_attention_weight_max": 40.72627639770508,
      "activations/layer13_attention_weight_min": -35.75704574584961,
      "activations/layer14_attention_weight_max": 50.41382598876953,
      "activations/layer14_attention_weight_min": -39.13270568847656,
      "activations/layer15_attention_weight_max": 35.552188873291016,
      "activations/layer15_attention_weight_min": -32.10458755493164,
      "activations/layer16_attention_weight_max": 34.26588439941406,
      "activations/layer16_attention_weight_min": -30.564191818237305,
      "activations/layer17_attention_weight_max": 32.65497589111328,
      "activations/layer17_attention_weight_min": -26.941831588745117,
      "activations/layer18_attention_weight_max": 33.476104736328125,
      "activations/layer18_attention_weight_min": -23.383739471435547,
      "activations/layer19_attention_weight_max": 40.50582504272461,
      "activations/layer19_attention_weight_min": -31.78013038635254,
      "activations/layer1_attention_weight_max": 17.915992736816406,
      "activations/layer1_attention_weight_min": -15.540708541870117,
      "activations/layer20_attention_weight_max": 32.50656509399414,
      "activations/layer20_attention_weight_min": -24.795564651489258,
      "activations/layer21_attention_weight_max": 31.11163902282715,
      "activations/layer21_attention_weight_min": -23.811765670776367,
      "activations/layer22_attention_weight_max": 52.19742202758789,
      "activations/layer22_attention_weight_min": -30.43772315979004,
      "activations/layer23_attention_weight_max": 40.762176513671875,
      "activations/layer23_attention_weight_min": -25.29636001586914,
      "activations/layer2_attention_weight_max": 33.168949127197266,
      "activations/layer2_attention_weight_min": -32.926536560058594,
      "activations/layer3_attention_weight_max": 90.9307632446289,
      "activations/layer3_attention_weight_min": -92.27371215820312,
      "activations/layer4_attention_weight_max": 110.5776596069336,
      "activations/layer4_attention_weight_min": -111.95877838134766,
      "activations/layer5_attention_weight_max": 53.205360412597656,
      "activations/layer5_attention_weight_min": -63.07337951660156,
      "activations/layer6_attention_weight_max": 46.62228775024414,
      "activations/layer6_attention_weight_min": -49.75382614135742,
      "activations/layer7_attention_weight_max": 89.55496215820312,
      "activations/layer7_attention_weight_min": -94.64118957519531,
      "activations/layer8_attention_weight_max": 39.37449645996094,
      "activations/layer8_attention_weight_min": -41.8504638671875,
      "activations/layer9_attention_weight_max": 35.29266357421875,
      "activations/layer9_attention_weight_min": -34.32805252075195,
      "epoch": 10.23,
      "learning_rate": 8.490568181818182e-05,
      "loss": 2.7983,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4964,
      "eval_samples_per_second": 505.39,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4964,
      "eval_openwebtext_samples_per_second": 505.39,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 2.017,
      "eval_wikitext_samples_per_second": 226.076,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_lambada_loss": 2.494140625,
      "eval_lambada_ppl": 12.111320878668954,
      "eval_lambada_runtime": 9.5275,
      "eval_lambada_samples_per_second": 511.046,
      "step": 176000
    },
    {
      "activations/layer0_attention_weight_max": 15.89098834991455,
      "activations/layer0_attention_weight_min": -12.84790325164795,
      "activations/layer10_attention_weight_max": 29.96588706970215,
      "activations/layer10_attention_weight_min": -31.96725845336914,
      "activations/layer11_attention_weight_max": 30.207763671875,
      "activations/layer11_attention_weight_min": -31.132192611694336,
      "activations/layer12_attention_weight_max": 26.27814483642578,
      "activations/layer12_attention_weight_min": -26.636964797973633,
      "activations/layer13_attention_weight_max": 35.69038391113281,
      "activations/layer13_attention_weight_min": -34.25634002685547,
      "activations/layer14_attention_weight_max": 41.87870407104492,
      "activations/layer14_attention_weight_min": -39.165626525878906,
      "activations/layer15_attention_weight_max": 33.312774658203125,
      "activations/layer15_attention_weight_min": -31.376483917236328,
      "activations/layer16_attention_weight_max": 28.662565231323242,
      "activations/layer16_attention_weight_min": -26.66419219970703,
      "activations/layer17_attention_weight_max": 29.24509048461914,
      "activations/layer17_attention_weight_min": -23.853515625,
      "activations/layer18_attention_weight_max": 29.379228591918945,
      "activations/layer18_attention_weight_min": -22.128379821777344,
      "activations/layer19_attention_weight_max": 30.774351119995117,
      "activations/layer19_attention_weight_min": -31.780990600585938,
      "activations/layer1_attention_weight_max": 17.102367401123047,
      "activations/layer1_attention_weight_min": -15.08558177947998,
      "activations/layer20_attention_weight_max": 27.097129821777344,
      "activations/layer20_attention_weight_min": -24.464431762695312,
      "activations/layer21_attention_weight_max": 28.726055145263672,
      "activations/layer21_attention_weight_min": -24.523441314697266,
      "activations/layer22_attention_weight_max": 43.29163360595703,
      "activations/layer22_attention_weight_min": -28.740177154541016,
      "activations/layer23_attention_weight_max": 33.65098571777344,
      "activations/layer23_attention_weight_min": -24.81444549560547,
      "activations/layer2_attention_weight_max": 31.07442855834961,
      "activations/layer2_attention_weight_min": -33.65754699707031,
      "activations/layer3_attention_weight_max": 91.00039672851562,
      "activations/layer3_attention_weight_min": -92.6806411743164,
      "activations/layer4_attention_weight_max": 108.53277587890625,
      "activations/layer4_attention_weight_min": -112.21949768066406,
      "activations/layer5_attention_weight_max": 51.096553802490234,
      "activations/layer5_attention_weight_min": -61.724510192871094,
      "activations/layer6_attention_weight_max": 43.20768356323242,
      "activations/layer6_attention_weight_min": -46.895694732666016,
      "activations/layer7_attention_weight_max": 82.9000473022461,
      "activations/layer7_attention_weight_min": -83.90677642822266,
      "activations/layer8_attention_weight_max": 36.06061553955078,
      "activations/layer8_attention_weight_min": -38.36467742919922,
      "activations/layer9_attention_weight_max": 29.699588775634766,
      "activations/layer9_attention_weight_min": -31.924640655517578,
      "epoch": 10.23,
      "learning_rate": 8.488712121212121e-05,
      "loss": 2.7932,
      "step": 176050
    },
    {
      "activations/layer0_attention_weight_max": 15.772195816040039,
      "activations/layer0_attention_weight_min": -12.29073429107666,
      "activations/layer10_attention_weight_max": 33.71141815185547,
      "activations/layer10_attention_weight_min": -30.792306900024414,
      "activations/layer11_attention_weight_max": 32.10082244873047,
      "activations/layer11_attention_weight_min": -32.31578826904297,
      "activations/layer12_attention_weight_max": 24.71379852294922,
      "activations/layer12_attention_weight_min": -23.638126373291016,
      "activations/layer13_attention_weight_max": 38.57330322265625,
      "activations/layer13_attention_weight_min": -34.4980583190918,
      "activations/layer14_attention_weight_max": 44.33348846435547,
      "activations/layer14_attention_weight_min": -36.241180419921875,
      "activations/layer15_attention_weight_max": 38.68762969970703,
      "activations/layer15_attention_weight_min": -30.60315704345703,
      "activations/layer16_attention_weight_max": 31.902006149291992,
      "activations/layer16_attention_weight_min": -27.207439422607422,
      "activations/layer17_attention_weight_max": 32.270931243896484,
      "activations/layer17_attention_weight_min": -25.797576904296875,
      "activations/layer18_attention_weight_max": 30.827434539794922,
      "activations/layer18_attention_weight_min": -22.266645431518555,
      "activations/layer19_attention_weight_max": 33.01837921142578,
      "activations/layer19_attention_weight_min": -29.92679786682129,
      "activations/layer1_attention_weight_max": 16.368406295776367,
      "activations/layer1_attention_weight_min": -14.877395629882812,
      "activations/layer20_attention_weight_max": 28.176429748535156,
      "activations/layer20_attention_weight_min": -24.674304962158203,
      "activations/layer21_attention_weight_max": 26.862667083740234,
      "activations/layer21_attention_weight_min": -23.77138328552246,
      "activations/layer22_attention_weight_max": 44.407630920410156,
      "activations/layer22_attention_weight_min": -28.75710678100586,
      "activations/layer23_attention_weight_max": 35.64173126220703,
      "activations/layer23_attention_weight_min": -25.555843353271484,
      "activations/layer2_attention_weight_max": 31.075490951538086,
      "activations/layer2_attention_weight_min": -33.40877914428711,
      "activations/layer3_attention_weight_max": 92.08535766601562,
      "activations/layer3_attention_weight_min": -96.82465362548828,
      "activations/layer4_attention_weight_max": 110.13753509521484,
      "activations/layer4_attention_weight_min": -108.66997528076172,
      "activations/layer5_attention_weight_max": 54.60606002807617,
      "activations/layer5_attention_weight_min": -59.854244232177734,
      "activations/layer6_attention_weight_max": 48.66496658325195,
      "activations/layer6_attention_weight_min": -48.23439025878906,
      "activations/layer7_attention_weight_max": 93.20068359375,
      "activations/layer7_attention_weight_min": -86.25772857666016,
      "activations/layer8_attention_weight_max": 41.55133056640625,
      "activations/layer8_attention_weight_min": -38.747947692871094,
      "activations/layer9_attention_weight_max": 34.8302001953125,
      "activations/layer9_attention_weight_min": -32.20322036743164,
      "epoch": 10.23,
      "learning_rate": 8.486818181818182e-05,
      "loss": 2.7827,
      "step": 176100
    },
    {
      "activations/layer0_attention_weight_max": 15.656373023986816,
      "activations/layer0_attention_weight_min": -13.043882369995117,
      "activations/layer10_attention_weight_max": 35.36857223510742,
      "activations/layer10_attention_weight_min": -33.66407012939453,
      "activations/layer11_attention_weight_max": 34.613399505615234,
      "activations/layer11_attention_weight_min": -34.12664794921875,
      "activations/layer12_attention_weight_max": 26.271913528442383,
      "activations/layer12_attention_weight_min": -24.936845779418945,
      "activations/layer13_attention_weight_max": 41.611778259277344,
      "activations/layer13_attention_weight_min": -36.24833297729492,
      "activations/layer14_attention_weight_max": 51.82142639160156,
      "activations/layer14_attention_weight_min": -39.818763732910156,
      "activations/layer15_attention_weight_max": 41.76884078979492,
      "activations/layer15_attention_weight_min": -32.96453094482422,
      "activations/layer16_attention_weight_max": 35.1429328918457,
      "activations/layer16_attention_weight_min": -29.889604568481445,
      "activations/layer17_attention_weight_max": 34.940486907958984,
      "activations/layer17_attention_weight_min": -25.24348258972168,
      "activations/layer18_attention_weight_max": 32.142242431640625,
      "activations/layer18_attention_weight_min": -22.81805992126465,
      "activations/layer19_attention_weight_max": 42.03843307495117,
      "activations/layer19_attention_weight_min": -34.474246978759766,
      "activations/layer1_attention_weight_max": 16.58220863342285,
      "activations/layer1_attention_weight_min": -12.947049140930176,
      "activations/layer20_attention_weight_max": 39.31883239746094,
      "activations/layer20_attention_weight_min": -26.432409286499023,
      "activations/layer21_attention_weight_max": 34.71254348754883,
      "activations/layer21_attention_weight_min": -25.85148811340332,
      "activations/layer22_attention_weight_max": 53.20600128173828,
      "activations/layer22_attention_weight_min": -32.04981231689453,
      "activations/layer23_attention_weight_max": 40.4124755859375,
      "activations/layer23_attention_weight_min": -26.635025024414062,
      "activations/layer2_attention_weight_max": 30.840017318725586,
      "activations/layer2_attention_weight_min": -31.6175594329834,
      "activations/layer3_attention_weight_max": 92.1911392211914,
      "activations/layer3_attention_weight_min": -92.21459197998047,
      "activations/layer4_attention_weight_max": 114.10025787353516,
      "activations/layer4_attention_weight_min": -113.0575180053711,
      "activations/layer5_attention_weight_max": 54.94523620605469,
      "activations/layer5_attention_weight_min": -62.77621078491211,
      "activations/layer6_attention_weight_max": 49.065223693847656,
      "activations/layer6_attention_weight_min": -47.84403991699219,
      "activations/layer7_attention_weight_max": 91.17218780517578,
      "activations/layer7_attention_weight_min": -90.04048156738281,
      "activations/layer8_attention_weight_max": 40.686424255371094,
      "activations/layer8_attention_weight_min": -40.89052200317383,
      "activations/layer9_attention_weight_max": 34.159393310546875,
      "activations/layer9_attention_weight_min": -32.422142028808594,
      "epoch": 10.24,
      "learning_rate": 8.484924242424241e-05,
      "loss": 2.7986,
      "step": 176150
    },
    {
      "activations/layer0_attention_weight_max": 15.875205993652344,
      "activations/layer0_attention_weight_min": -13.282694816589355,
      "activations/layer10_attention_weight_max": 33.96915054321289,
      "activations/layer10_attention_weight_min": -32.50796127319336,
      "activations/layer11_attention_weight_max": 33.086151123046875,
      "activations/layer11_attention_weight_min": -33.058807373046875,
      "activations/layer12_attention_weight_max": 27.067541122436523,
      "activations/layer12_attention_weight_min": -28.268217086791992,
      "activations/layer13_attention_weight_max": 39.94348907470703,
      "activations/layer13_attention_weight_min": -34.400306701660156,
      "activations/layer14_attention_weight_max": 46.73196792602539,
      "activations/layer14_attention_weight_min": -37.12863540649414,
      "activations/layer15_attention_weight_max": 37.19865036010742,
      "activations/layer15_attention_weight_min": -30.830663681030273,
      "activations/layer16_attention_weight_max": 31.637601852416992,
      "activations/layer16_attention_weight_min": -28.191415786743164,
      "activations/layer17_attention_weight_max": 32.84233474731445,
      "activations/layer17_attention_weight_min": -23.86493682861328,
      "activations/layer18_attention_weight_max": 34.09972381591797,
      "activations/layer18_attention_weight_min": -21.777544021606445,
      "activations/layer19_attention_weight_max": 39.739463806152344,
      "activations/layer19_attention_weight_min": -30.454212188720703,
      "activations/layer1_attention_weight_max": 16.6922550201416,
      "activations/layer1_attention_weight_min": -13.422444343566895,
      "activations/layer20_attention_weight_max": 35.225311279296875,
      "activations/layer20_attention_weight_min": -23.260047912597656,
      "activations/layer21_attention_weight_max": 32.3011589050293,
      "activations/layer21_attention_weight_min": -23.259841918945312,
      "activations/layer22_attention_weight_max": 51.72783660888672,
      "activations/layer22_attention_weight_min": -28.494043350219727,
      "activations/layer23_attention_weight_max": 39.120399475097656,
      "activations/layer23_attention_weight_min": -22.902111053466797,
      "activations/layer2_attention_weight_max": 34.16357421875,
      "activations/layer2_attention_weight_min": -32.88799285888672,
      "activations/layer3_attention_weight_max": 94.6318359375,
      "activations/layer3_attention_weight_min": -97.03369903564453,
      "activations/layer4_attention_weight_max": 116.07076263427734,
      "activations/layer4_attention_weight_min": -113.18192291259766,
      "activations/layer5_attention_weight_max": 53.72532653808594,
      "activations/layer5_attention_weight_min": -64.48667907714844,
      "activations/layer6_attention_weight_max": 45.793296813964844,
      "activations/layer6_attention_weight_min": -47.65642547607422,
      "activations/layer7_attention_weight_max": 87.26361083984375,
      "activations/layer7_attention_weight_min": -89.09989166259766,
      "activations/layer8_attention_weight_max": 41.38111877441406,
      "activations/layer8_attention_weight_min": -40.17972946166992,
      "activations/layer9_attention_weight_max": 33.9457893371582,
      "activations/layer9_attention_weight_min": -34.80397033691406,
      "epoch": 10.24,
      "learning_rate": 8.483030303030303e-05,
      "loss": 2.7863,
      "step": 176200
    },
    {
      "activations/layer0_attention_weight_max": 16.030500411987305,
      "activations/layer0_attention_weight_min": -13.612632751464844,
      "activations/layer10_attention_weight_max": 32.33143997192383,
      "activations/layer10_attention_weight_min": -30.476186752319336,
      "activations/layer11_attention_weight_max": 32.440826416015625,
      "activations/layer11_attention_weight_min": -29.694622039794922,
      "activations/layer12_attention_weight_max": 24.866056442260742,
      "activations/layer12_attention_weight_min": -24.700347900390625,
      "activations/layer13_attention_weight_max": 42.39491653442383,
      "activations/layer13_attention_weight_min": -34.90894317626953,
      "activations/layer14_attention_weight_max": 48.83598709106445,
      "activations/layer14_attention_weight_min": -38.726593017578125,
      "activations/layer15_attention_weight_max": 39.19246292114258,
      "activations/layer15_attention_weight_min": -31.835596084594727,
      "activations/layer16_attention_weight_max": 35.4462890625,
      "activations/layer16_attention_weight_min": -27.590166091918945,
      "activations/layer17_attention_weight_max": 32.18608856201172,
      "activations/layer17_attention_weight_min": -25.15293312072754,
      "activations/layer18_attention_weight_max": 30.83936882019043,
      "activations/layer18_attention_weight_min": -23.095252990722656,
      "activations/layer19_attention_weight_max": 38.14567947387695,
      "activations/layer19_attention_weight_min": -30.066816329956055,
      "activations/layer1_attention_weight_max": 16.60985565185547,
      "activations/layer1_attention_weight_min": -15.803712844848633,
      "activations/layer20_attention_weight_max": 32.45337677001953,
      "activations/layer20_attention_weight_min": -26.931522369384766,
      "activations/layer21_attention_weight_max": 32.18943786621094,
      "activations/layer21_attention_weight_min": -24.69833755493164,
      "activations/layer22_attention_weight_max": 50.519775390625,
      "activations/layer22_attention_weight_min": -29.39956283569336,
      "activations/layer23_attention_weight_max": 36.01213073730469,
      "activations/layer23_attention_weight_min": -24.83070182800293,
      "activations/layer2_attention_weight_max": 31.9300537109375,
      "activations/layer2_attention_weight_min": -33.412620544433594,
      "activations/layer3_attention_weight_max": 90.89012145996094,
      "activations/layer3_attention_weight_min": -95.67617797851562,
      "activations/layer4_attention_weight_max": 109.85294342041016,
      "activations/layer4_attention_weight_min": -116.690673828125,
      "activations/layer5_attention_weight_max": 52.595924377441406,
      "activations/layer5_attention_weight_min": -64.3778076171875,
      "activations/layer6_attention_weight_max": 45.1758918762207,
      "activations/layer6_attention_weight_min": -50.235939025878906,
      "activations/layer7_attention_weight_max": 88.46575164794922,
      "activations/layer7_attention_weight_min": -84.80430603027344,
      "activations/layer8_attention_weight_max": 40.72157669067383,
      "activations/layer8_attention_weight_min": -39.2078742980957,
      "activations/layer9_attention_weight_max": 32.380672454833984,
      "activations/layer9_attention_weight_min": -31.307479858398438,
      "epoch": 10.24,
      "learning_rate": 8.481136363636363e-05,
      "loss": 2.7765,
      "step": 176250
    },
    {
      "activations/layer0_attention_weight_max": 16.0825138092041,
      "activations/layer0_attention_weight_min": -12.569478988647461,
      "activations/layer10_attention_weight_max": 31.025436401367188,
      "activations/layer10_attention_weight_min": -32.157012939453125,
      "activations/layer11_attention_weight_max": 31.138797760009766,
      "activations/layer11_attention_weight_min": -30.258682250976562,
      "activations/layer12_attention_weight_max": 24.38418960571289,
      "activations/layer12_attention_weight_min": -24.336938858032227,
      "activations/layer13_attention_weight_max": 40.693824768066406,
      "activations/layer13_attention_weight_min": -34.156211853027344,
      "activations/layer14_attention_weight_max": 41.65697479248047,
      "activations/layer14_attention_weight_min": -36.01350784301758,
      "activations/layer15_attention_weight_max": 35.655330657958984,
      "activations/layer15_attention_weight_min": -31.25002670288086,
      "activations/layer16_attention_weight_max": 32.127445220947266,
      "activations/layer16_attention_weight_min": -30.62836456298828,
      "activations/layer17_attention_weight_max": 30.793325424194336,
      "activations/layer17_attention_weight_min": -26.764028549194336,
      "activations/layer18_attention_weight_max": 29.695377349853516,
      "activations/layer18_attention_weight_min": -23.185117721557617,
      "activations/layer19_attention_weight_max": 35.87284469604492,
      "activations/layer19_attention_weight_min": -32.591880798339844,
      "activations/layer1_attention_weight_max": 17.4215145111084,
      "activations/layer1_attention_weight_min": -14.568825721740723,
      "activations/layer20_attention_weight_max": 30.77305793762207,
      "activations/layer20_attention_weight_min": -25.917884826660156,
      "activations/layer21_attention_weight_max": 31.457223892211914,
      "activations/layer21_attention_weight_min": -24.401647567749023,
      "activations/layer22_attention_weight_max": 48.30128479003906,
      "activations/layer22_attention_weight_min": -28.789522171020508,
      "activations/layer23_attention_weight_max": 37.95793914794922,
      "activations/layer23_attention_weight_min": -23.12189292907715,
      "activations/layer2_attention_weight_max": 31.290054321289062,
      "activations/layer2_attention_weight_min": -32.66105651855469,
      "activations/layer3_attention_weight_max": 90.16809844970703,
      "activations/layer3_attention_weight_min": -93.4544906616211,
      "activations/layer4_attention_weight_max": 111.3018798828125,
      "activations/layer4_attention_weight_min": -111.23201751708984,
      "activations/layer5_attention_weight_max": 53.34153747558594,
      "activations/layer5_attention_weight_min": -60.0848503112793,
      "activations/layer6_attention_weight_max": 42.991207122802734,
      "activations/layer6_attention_weight_min": -47.059017181396484,
      "activations/layer7_attention_weight_max": 89.99754333496094,
      "activations/layer7_attention_weight_min": -88.11304473876953,
      "activations/layer8_attention_weight_max": 39.81565475463867,
      "activations/layer8_attention_weight_min": -42.41777801513672,
      "activations/layer9_attention_weight_max": 32.681640625,
      "activations/layer9_attention_weight_min": -33.2056770324707,
      "epoch": 10.24,
      "learning_rate": 8.479242424242423e-05,
      "loss": 2.7914,
      "step": 176300
    },
    {
      "activations/layer0_attention_weight_max": 15.016538619995117,
      "activations/layer0_attention_weight_min": -12.201109886169434,
      "activations/layer10_attention_weight_max": 31.154319763183594,
      "activations/layer10_attention_weight_min": -31.454557418823242,
      "activations/layer11_attention_weight_max": 31.76752471923828,
      "activations/layer11_attention_weight_min": -29.80535125732422,
      "activations/layer12_attention_weight_max": 34.63789749145508,
      "activations/layer12_attention_weight_min": -28.79538917541504,
      "activations/layer13_attention_weight_max": 37.8438835144043,
      "activations/layer13_attention_weight_min": -34.872581481933594,
      "activations/layer14_attention_weight_max": 48.768741607666016,
      "activations/layer14_attention_weight_min": -39.45768356323242,
      "activations/layer15_attention_weight_max": 38.35406494140625,
      "activations/layer15_attention_weight_min": -33.54020690917969,
      "activations/layer16_attention_weight_max": 34.965171813964844,
      "activations/layer16_attention_weight_min": -29.63348388671875,
      "activations/layer17_attention_weight_max": 32.15338897705078,
      "activations/layer17_attention_weight_min": -26.686906814575195,
      "activations/layer18_attention_weight_max": 31.201534271240234,
      "activations/layer18_attention_weight_min": -22.88788604736328,
      "activations/layer19_attention_weight_max": 39.44074630737305,
      "activations/layer19_attention_weight_min": -32.76254653930664,
      "activations/layer1_attention_weight_max": 17.101579666137695,
      "activations/layer1_attention_weight_min": -15.89328670501709,
      "activations/layer20_attention_weight_max": 33.35971450805664,
      "activations/layer20_attention_weight_min": -24.53973960876465,
      "activations/layer21_attention_weight_max": 30.744035720825195,
      "activations/layer21_attention_weight_min": -24.23008155822754,
      "activations/layer22_attention_weight_max": 44.573631286621094,
      "activations/layer22_attention_weight_min": -31.301986694335938,
      "activations/layer23_attention_weight_max": 34.297943115234375,
      "activations/layer23_attention_weight_min": -25.538061141967773,
      "activations/layer2_attention_weight_max": 32.169715881347656,
      "activations/layer2_attention_weight_min": -32.86174392700195,
      "activations/layer3_attention_weight_max": 94.60783386230469,
      "activations/layer3_attention_weight_min": -94.47643280029297,
      "activations/layer4_attention_weight_max": 115.51019287109375,
      "activations/layer4_attention_weight_min": -113.95594787597656,
      "activations/layer5_attention_weight_max": 55.16374206542969,
      "activations/layer5_attention_weight_min": -65.1710205078125,
      "activations/layer6_attention_weight_max": 47.122840881347656,
      "activations/layer6_attention_weight_min": -49.02768325805664,
      "activations/layer7_attention_weight_max": 95.59793090820312,
      "activations/layer7_attention_weight_min": -91.26419067382812,
      "activations/layer8_attention_weight_max": 40.21502685546875,
      "activations/layer8_attention_weight_min": -42.19325256347656,
      "activations/layer9_attention_weight_max": 33.300960540771484,
      "activations/layer9_attention_weight_min": -33.21135711669922,
      "epoch": 10.25,
      "learning_rate": 8.477348484848484e-05,
      "loss": 2.7843,
      "step": 176350
    },
    {
      "activations/layer0_attention_weight_max": 15.289884567260742,
      "activations/layer0_attention_weight_min": -12.699864387512207,
      "activations/layer10_attention_weight_max": 33.07622528076172,
      "activations/layer10_attention_weight_min": -31.165035247802734,
      "activations/layer11_attention_weight_max": 33.65732192993164,
      "activations/layer11_attention_weight_min": -30.47991180419922,
      "activations/layer12_attention_weight_max": 22.849700927734375,
      "activations/layer12_attention_weight_min": -24.555017471313477,
      "activations/layer13_attention_weight_max": 35.98473358154297,
      "activations/layer13_attention_weight_min": -35.62542724609375,
      "activations/layer14_attention_weight_max": 41.826210021972656,
      "activations/layer14_attention_weight_min": -39.09010314941406,
      "activations/layer15_attention_weight_max": 34.69190216064453,
      "activations/layer15_attention_weight_min": -35.358482360839844,
      "activations/layer16_attention_weight_max": 29.057533264160156,
      "activations/layer16_attention_weight_min": -28.515119552612305,
      "activations/layer17_attention_weight_max": 30.14829444885254,
      "activations/layer17_attention_weight_min": -27.03533363342285,
      "activations/layer18_attention_weight_max": 31.00019645690918,
      "activations/layer18_attention_weight_min": -23.05646324157715,
      "activations/layer19_attention_weight_max": 32.13621520996094,
      "activations/layer19_attention_weight_min": -30.612991333007812,
      "activations/layer1_attention_weight_max": 15.968273162841797,
      "activations/layer1_attention_weight_min": -14.37076473236084,
      "activations/layer20_attention_weight_max": 26.760086059570312,
      "activations/layer20_attention_weight_min": -26.088224411010742,
      "activations/layer21_attention_weight_max": 27.763242721557617,
      "activations/layer21_attention_weight_min": -23.275482177734375,
      "activations/layer22_attention_weight_max": 40.69602966308594,
      "activations/layer22_attention_weight_min": -30.62919807434082,
      "activations/layer23_attention_weight_max": 30.9404354095459,
      "activations/layer23_attention_weight_min": -25.275115966796875,
      "activations/layer2_attention_weight_max": 32.714866638183594,
      "activations/layer2_attention_weight_min": -32.858245849609375,
      "activations/layer3_attention_weight_max": 95.21210479736328,
      "activations/layer3_attention_weight_min": -100.54398345947266,
      "activations/layer4_attention_weight_max": 109.71600341796875,
      "activations/layer4_attention_weight_min": -107.7793960571289,
      "activations/layer5_attention_weight_max": 52.37632751464844,
      "activations/layer5_attention_weight_min": -59.52862548828125,
      "activations/layer6_attention_weight_max": 44.16834259033203,
      "activations/layer6_attention_weight_min": -47.60004425048828,
      "activations/layer7_attention_weight_max": 86.8459243774414,
      "activations/layer7_attention_weight_min": -85.48194885253906,
      "activations/layer8_attention_weight_max": 39.1358528137207,
      "activations/layer8_attention_weight_min": -39.70917510986328,
      "activations/layer9_attention_weight_max": 35.445770263671875,
      "activations/layer9_attention_weight_min": -32.28107833862305,
      "epoch": 10.25,
      "learning_rate": 8.475454545454544e-05,
      "loss": 2.776,
      "step": 176400
    },
    {
      "activations/layer0_attention_weight_max": 15.348609924316406,
      "activations/layer0_attention_weight_min": -12.480304718017578,
      "activations/layer10_attention_weight_max": 33.003379821777344,
      "activations/layer10_attention_weight_min": -34.12821960449219,
      "activations/layer11_attention_weight_max": 32.12024688720703,
      "activations/layer11_attention_weight_min": -33.725982666015625,
      "activations/layer12_attention_weight_max": 24.997310638427734,
      "activations/layer12_attention_weight_min": -27.920047760009766,
      "activations/layer13_attention_weight_max": 38.38633728027344,
      "activations/layer13_attention_weight_min": -33.64069366455078,
      "activations/layer14_attention_weight_max": 48.34309387207031,
      "activations/layer14_attention_weight_min": -37.7911491394043,
      "activations/layer15_attention_weight_max": 38.141021728515625,
      "activations/layer15_attention_weight_min": -32.038204193115234,
      "activations/layer16_attention_weight_max": 32.82928466796875,
      "activations/layer16_attention_weight_min": -27.77054786682129,
      "activations/layer17_attention_weight_max": 34.16824722290039,
      "activations/layer17_attention_weight_min": -29.97068977355957,
      "activations/layer18_attention_weight_max": 32.21258544921875,
      "activations/layer18_attention_weight_min": -24.126455307006836,
      "activations/layer19_attention_weight_max": 34.12784957885742,
      "activations/layer19_attention_weight_min": -34.038265228271484,
      "activations/layer1_attention_weight_max": 16.813589096069336,
      "activations/layer1_attention_weight_min": -15.470209121704102,
      "activations/layer20_attention_weight_max": 30.000938415527344,
      "activations/layer20_attention_weight_min": -25.02604866027832,
      "activations/layer21_attention_weight_max": 28.104503631591797,
      "activations/layer21_attention_weight_min": -23.711620330810547,
      "activations/layer22_attention_weight_max": 46.67448806762695,
      "activations/layer22_attention_weight_min": -29.89923095703125,
      "activations/layer23_attention_weight_max": 38.507057189941406,
      "activations/layer23_attention_weight_min": -25.949073791503906,
      "activations/layer2_attention_weight_max": 32.19157409667969,
      "activations/layer2_attention_weight_min": -34.657291412353516,
      "activations/layer3_attention_weight_max": 98.01471710205078,
      "activations/layer3_attention_weight_min": -95.49325561523438,
      "activations/layer4_attention_weight_max": 114.77363586425781,
      "activations/layer4_attention_weight_min": -108.7447738647461,
      "activations/layer5_attention_weight_max": 55.08953857421875,
      "activations/layer5_attention_weight_min": -62.7750358581543,
      "activations/layer6_attention_weight_max": 46.50462341308594,
      "activations/layer6_attention_weight_min": -49.86940383911133,
      "activations/layer7_attention_weight_max": 89.28931427001953,
      "activations/layer7_attention_weight_min": -90.8171615600586,
      "activations/layer8_attention_weight_max": 39.59086990356445,
      "activations/layer8_attention_weight_min": -42.060611724853516,
      "activations/layer9_attention_weight_max": 33.326473236083984,
      "activations/layer9_attention_weight_min": -34.080867767333984,
      "epoch": 10.25,
      "learning_rate": 8.473560606060606e-05,
      "loss": 2.7705,
      "step": 176450
    },
    {
      "activations/layer0_attention_weight_max": 16.180438995361328,
      "activations/layer0_attention_weight_min": -13.019832611083984,
      "activations/layer10_attention_weight_max": 45.012855529785156,
      "activations/layer10_attention_weight_min": -43.060935974121094,
      "activations/layer11_attention_weight_max": 45.69776153564453,
      "activations/layer11_attention_weight_min": -43.25765609741211,
      "activations/layer12_attention_weight_max": 28.611709594726562,
      "activations/layer12_attention_weight_min": -29.871753692626953,
      "activations/layer13_attention_weight_max": 38.200496673583984,
      "activations/layer13_attention_weight_min": -34.52242660522461,
      "activations/layer14_attention_weight_max": 47.47846603393555,
      "activations/layer14_attention_weight_min": -42.553348541259766,
      "activations/layer15_attention_weight_max": 43.05953598022461,
      "activations/layer15_attention_weight_min": -36.75082778930664,
      "activations/layer16_attention_weight_max": 32.985599517822266,
      "activations/layer16_attention_weight_min": -27.6451473236084,
      "activations/layer17_attention_weight_max": 31.89828109741211,
      "activations/layer17_attention_weight_min": -27.6057071685791,
      "activations/layer18_attention_weight_max": 30.274953842163086,
      "activations/layer18_attention_weight_min": -22.57549285888672,
      "activations/layer19_attention_weight_max": 41.439605712890625,
      "activations/layer19_attention_weight_min": -30.927175521850586,
      "activations/layer1_attention_weight_max": 16.673355102539062,
      "activations/layer1_attention_weight_min": -14.601316452026367,
      "activations/layer20_attention_weight_max": 33.327392578125,
      "activations/layer20_attention_weight_min": -22.224342346191406,
      "activations/layer21_attention_weight_max": 29.631771087646484,
      "activations/layer21_attention_weight_min": -21.410982131958008,
      "activations/layer22_attention_weight_max": 51.60091018676758,
      "activations/layer22_attention_weight_min": -28.54764175415039,
      "activations/layer23_attention_weight_max": 40.09228515625,
      "activations/layer23_attention_weight_min": -24.20049285888672,
      "activations/layer2_attention_weight_max": 33.7081413269043,
      "activations/layer2_attention_weight_min": -37.392581939697266,
      "activations/layer3_attention_weight_max": 99.9310531616211,
      "activations/layer3_attention_weight_min": -104.33419036865234,
      "activations/layer4_attention_weight_max": 120.65177154541016,
      "activations/layer4_attention_weight_min": -117.11781311035156,
      "activations/layer5_attention_weight_max": 56.78416442871094,
      "activations/layer5_attention_weight_min": -61.820594787597656,
      "activations/layer6_attention_weight_max": 48.73330307006836,
      "activations/layer6_attention_weight_min": -48.42831039428711,
      "activations/layer7_attention_weight_max": 104.22777557373047,
      "activations/layer7_attention_weight_min": -97.7959213256836,
      "activations/layer8_attention_weight_max": 49.06249237060547,
      "activations/layer8_attention_weight_min": -44.77524185180664,
      "activations/layer9_attention_weight_max": 42.73563003540039,
      "activations/layer9_attention_weight_min": -42.088863372802734,
      "epoch": 10.26,
      "learning_rate": 8.471666666666666e-05,
      "loss": 2.7898,
      "step": 176500
    },
    {
      "activations/layer0_attention_weight_max": 16.114845275878906,
      "activations/layer0_attention_weight_min": -12.07351016998291,
      "activations/layer10_attention_weight_max": 33.800411224365234,
      "activations/layer10_attention_weight_min": -33.94284439086914,
      "activations/layer11_attention_weight_max": 33.15185546875,
      "activations/layer11_attention_weight_min": -32.831092834472656,
      "activations/layer12_attention_weight_max": 25.32223129272461,
      "activations/layer12_attention_weight_min": -24.52969741821289,
      "activations/layer13_attention_weight_max": 39.956642150878906,
      "activations/layer13_attention_weight_min": -36.153404235839844,
      "activations/layer14_attention_weight_max": 45.92662811279297,
      "activations/layer14_attention_weight_min": -37.05662155151367,
      "activations/layer15_attention_weight_max": 37.426544189453125,
      "activations/layer15_attention_weight_min": -31.459468841552734,
      "activations/layer16_attention_weight_max": 30.94072723388672,
      "activations/layer16_attention_weight_min": -29.16974449157715,
      "activations/layer17_attention_weight_max": 32.17777633666992,
      "activations/layer17_attention_weight_min": -25.71879005432129,
      "activations/layer18_attention_weight_max": 32.076759338378906,
      "activations/layer18_attention_weight_min": -22.212703704833984,
      "activations/layer19_attention_weight_max": 38.42911911010742,
      "activations/layer19_attention_weight_min": -31.2575740814209,
      "activations/layer1_attention_weight_max": 17.389360427856445,
      "activations/layer1_attention_weight_min": -14.427765846252441,
      "activations/layer20_attention_weight_max": 31.843948364257812,
      "activations/layer20_attention_weight_min": -22.910236358642578,
      "activations/layer21_attention_weight_max": 30.302122116088867,
      "activations/layer21_attention_weight_min": -21.91689109802246,
      "activations/layer22_attention_weight_max": 47.95062255859375,
      "activations/layer22_attention_weight_min": -28.96014404296875,
      "activations/layer23_attention_weight_max": 37.12257385253906,
      "activations/layer23_attention_weight_min": -24.506513595581055,
      "activations/layer2_attention_weight_max": 32.29109191894531,
      "activations/layer2_attention_weight_min": -33.812217712402344,
      "activations/layer3_attention_weight_max": 95.5035400390625,
      "activations/layer3_attention_weight_min": -96.23267364501953,
      "activations/layer4_attention_weight_max": 115.19732666015625,
      "activations/layer4_attention_weight_min": -115.24895477294922,
      "activations/layer5_attention_weight_max": 53.27227020263672,
      "activations/layer5_attention_weight_min": -63.59607696533203,
      "activations/layer6_attention_weight_max": 48.64872741699219,
      "activations/layer6_attention_weight_min": -52.94389343261719,
      "activations/layer7_attention_weight_max": 97.6098403930664,
      "activations/layer7_attention_weight_min": -91.11390686035156,
      "activations/layer8_attention_weight_max": 42.32027816772461,
      "activations/layer8_attention_weight_min": -44.11763000488281,
      "activations/layer9_attention_weight_max": 35.41720199584961,
      "activations/layer9_attention_weight_min": -36.34772872924805,
      "epoch": 10.26,
      "learning_rate": 8.469772727272726e-05,
      "loss": 2.7797,
      "step": 176550
    },
    {
      "activations/layer0_attention_weight_max": 14.725468635559082,
      "activations/layer0_attention_weight_min": -12.411778450012207,
      "activations/layer10_attention_weight_max": 36.23304748535156,
      "activations/layer10_attention_weight_min": -34.99605178833008,
      "activations/layer11_attention_weight_max": 37.325225830078125,
      "activations/layer11_attention_weight_min": -34.8933219909668,
      "activations/layer12_attention_weight_max": 26.55553436279297,
      "activations/layer12_attention_weight_min": -24.425779342651367,
      "activations/layer13_attention_weight_max": 40.45751190185547,
      "activations/layer13_attention_weight_min": -33.911895751953125,
      "activations/layer14_attention_weight_max": 51.298160552978516,
      "activations/layer14_attention_weight_min": -37.91206741333008,
      "activations/layer15_attention_weight_max": 43.95614242553711,
      "activations/layer15_attention_weight_min": -31.625642776489258,
      "activations/layer16_attention_weight_max": 32.55197525024414,
      "activations/layer16_attention_weight_min": -26.96023941040039,
      "activations/layer17_attention_weight_max": 32.79780960083008,
      "activations/layer17_attention_weight_min": -23.96813201904297,
      "activations/layer18_attention_weight_max": 32.05803680419922,
      "activations/layer18_attention_weight_min": -21.306459426879883,
      "activations/layer19_attention_weight_max": 39.67841720581055,
      "activations/layer19_attention_weight_min": -30.510889053344727,
      "activations/layer1_attention_weight_max": 17.716291427612305,
      "activations/layer1_attention_weight_min": -16.708587646484375,
      "activations/layer20_attention_weight_max": 34.18159866333008,
      "activations/layer20_attention_weight_min": -23.881908416748047,
      "activations/layer21_attention_weight_max": 31.55166244506836,
      "activations/layer21_attention_weight_min": -23.565025329589844,
      "activations/layer22_attention_weight_max": 55.95439529418945,
      "activations/layer22_attention_weight_min": -29.379718780517578,
      "activations/layer23_attention_weight_max": 37.93423080444336,
      "activations/layer23_attention_weight_min": -23.298194885253906,
      "activations/layer2_attention_weight_max": 32.44086456298828,
      "activations/layer2_attention_weight_min": -34.726688385009766,
      "activations/layer3_attention_weight_max": 102.846923828125,
      "activations/layer3_attention_weight_min": -102.4425277709961,
      "activations/layer4_attention_weight_max": 117.67237854003906,
      "activations/layer4_attention_weight_min": -113.60176849365234,
      "activations/layer5_attention_weight_max": 55.00578308105469,
      "activations/layer5_attention_weight_min": -63.212181091308594,
      "activations/layer6_attention_weight_max": 46.205196380615234,
      "activations/layer6_attention_weight_min": -46.43634796142578,
      "activations/layer7_attention_weight_max": 89.74532318115234,
      "activations/layer7_attention_weight_min": -90.01283264160156,
      "activations/layer8_attention_weight_max": 41.785362243652344,
      "activations/layer8_attention_weight_min": -42.14807891845703,
      "activations/layer9_attention_weight_max": 35.76389694213867,
      "activations/layer9_attention_weight_min": -34.945899963378906,
      "epoch": 10.26,
      "learning_rate": 8.467878787878788e-05,
      "loss": 2.7965,
      "step": 176600
    },
    {
      "activations/layer0_attention_weight_max": 14.802987098693848,
      "activations/layer0_attention_weight_min": -12.509814262390137,
      "activations/layer10_attention_weight_max": 45.96345138549805,
      "activations/layer10_attention_weight_min": -41.88895797729492,
      "activations/layer11_attention_weight_max": 46.613975524902344,
      "activations/layer11_attention_weight_min": -41.20111846923828,
      "activations/layer12_attention_weight_max": 33.90503692626953,
      "activations/layer12_attention_weight_min": -28.76621437072754,
      "activations/layer13_attention_weight_max": 51.142112731933594,
      "activations/layer13_attention_weight_min": -39.985286712646484,
      "activations/layer14_attention_weight_max": 67.14849853515625,
      "activations/layer14_attention_weight_min": -48.76350402832031,
      "activations/layer15_attention_weight_max": 47.2359504699707,
      "activations/layer15_attention_weight_min": -36.76750946044922,
      "activations/layer16_attention_weight_max": 37.71725845336914,
      "activations/layer16_attention_weight_min": -28.285951614379883,
      "activations/layer17_attention_weight_max": 37.073883056640625,
      "activations/layer17_attention_weight_min": -29.85392951965332,
      "activations/layer18_attention_weight_max": 38.366172790527344,
      "activations/layer18_attention_weight_min": -25.300479888916016,
      "activations/layer19_attention_weight_max": 46.42649459838867,
      "activations/layer19_attention_weight_min": -32.678707122802734,
      "activations/layer1_attention_weight_max": 16.42302131652832,
      "activations/layer1_attention_weight_min": -14.975663185119629,
      "activations/layer20_attention_weight_max": 36.162254333496094,
      "activations/layer20_attention_weight_min": -24.53338050842285,
      "activations/layer21_attention_weight_max": 32.81132888793945,
      "activations/layer21_attention_weight_min": -24.232656478881836,
      "activations/layer22_attention_weight_max": 71.98423767089844,
      "activations/layer22_attention_weight_min": -32.141082763671875,
      "activations/layer23_attention_weight_max": 48.29498291015625,
      "activations/layer23_attention_weight_min": -21.791906356811523,
      "activations/layer2_attention_weight_max": 35.22138214111328,
      "activations/layer2_attention_weight_min": -36.589664459228516,
      "activations/layer3_attention_weight_max": 107.18658447265625,
      "activations/layer3_attention_weight_min": -109.69478607177734,
      "activations/layer4_attention_weight_max": 129.66383361816406,
      "activations/layer4_attention_weight_min": -126.19707489013672,
      "activations/layer5_attention_weight_max": 57.4954948425293,
      "activations/layer5_attention_weight_min": -62.22013473510742,
      "activations/layer6_attention_weight_max": 50.619876861572266,
      "activations/layer6_attention_weight_min": -51.37166213989258,
      "activations/layer7_attention_weight_max": 119.96910095214844,
      "activations/layer7_attention_weight_min": -107.9782943725586,
      "activations/layer8_attention_weight_max": 55.02286148071289,
      "activations/layer8_attention_weight_min": -53.06496047973633,
      "activations/layer9_attention_weight_max": 46.61726379394531,
      "activations/layer9_attention_weight_min": -43.83192443847656,
      "epoch": 10.26,
      "learning_rate": 8.465984848484847e-05,
      "loss": 2.775,
      "step": 176650
    },
    {
      "activations/layer0_attention_weight_max": 16.30852508544922,
      "activations/layer0_attention_weight_min": -13.616033554077148,
      "activations/layer10_attention_weight_max": 33.662635803222656,
      "activations/layer10_attention_weight_min": -33.39695358276367,
      "activations/layer11_attention_weight_max": 31.630840301513672,
      "activations/layer11_attention_weight_min": -32.43510437011719,
      "activations/layer12_attention_weight_max": 24.799406051635742,
      "activations/layer12_attention_weight_min": -24.850051879882812,
      "activations/layer13_attention_weight_max": 40.45135498046875,
      "activations/layer13_attention_weight_min": -34.309722900390625,
      "activations/layer14_attention_weight_max": 43.10213088989258,
      "activations/layer14_attention_weight_min": -37.07619857788086,
      "activations/layer15_attention_weight_max": 36.93913650512695,
      "activations/layer15_attention_weight_min": -33.65843963623047,
      "activations/layer16_attention_weight_max": 34.08363342285156,
      "activations/layer16_attention_weight_min": -29.802892684936523,
      "activations/layer17_attention_weight_max": 35.467445373535156,
      "activations/layer17_attention_weight_min": -26.846755981445312,
      "activations/layer18_attention_weight_max": 38.888519287109375,
      "activations/layer18_attention_weight_min": -24.18036460876465,
      "activations/layer19_attention_weight_max": 41.67456817626953,
      "activations/layer19_attention_weight_min": -31.081899642944336,
      "activations/layer1_attention_weight_max": 15.989295959472656,
      "activations/layer1_attention_weight_min": -16.13888931274414,
      "activations/layer20_attention_weight_max": 33.5745735168457,
      "activations/layer20_attention_weight_min": -24.816598892211914,
      "activations/layer21_attention_weight_max": 37.186065673828125,
      "activations/layer21_attention_weight_min": -25.62604331970215,
      "activations/layer22_attention_weight_max": 49.110389709472656,
      "activations/layer22_attention_weight_min": -30.19405746459961,
      "activations/layer23_attention_weight_max": 45.493621826171875,
      "activations/layer23_attention_weight_min": -25.9498348236084,
      "activations/layer2_attention_weight_max": 31.984182357788086,
      "activations/layer2_attention_weight_min": -32.2447509765625,
      "activations/layer3_attention_weight_max": 97.01697540283203,
      "activations/layer3_attention_weight_min": -97.70858001708984,
      "activations/layer4_attention_weight_max": 113.91297912597656,
      "activations/layer4_attention_weight_min": -112.63826751708984,
      "activations/layer5_attention_weight_max": 55.60557174682617,
      "activations/layer5_attention_weight_min": -60.98740768432617,
      "activations/layer6_attention_weight_max": 48.61565399169922,
      "activations/layer6_attention_weight_min": -50.387271881103516,
      "activations/layer7_attention_weight_max": 91.80610656738281,
      "activations/layer7_attention_weight_min": -90.9041976928711,
      "activations/layer8_attention_weight_max": 41.715301513671875,
      "activations/layer8_attention_weight_min": -41.98201370239258,
      "activations/layer9_attention_weight_max": 34.21330261230469,
      "activations/layer9_attention_weight_min": -33.907196044921875,
      "epoch": 10.27,
      "learning_rate": 8.464090909090908e-05,
      "loss": 2.793,
      "step": 176700
    },
    {
      "activations/layer0_attention_weight_max": 15.186511039733887,
      "activations/layer0_attention_weight_min": -11.382214546203613,
      "activations/layer10_attention_weight_max": 34.673370361328125,
      "activations/layer10_attention_weight_min": -34.80968475341797,
      "activations/layer11_attention_weight_max": 34.5889892578125,
      "activations/layer11_attention_weight_min": -33.527198791503906,
      "activations/layer12_attention_weight_max": 26.21050453186035,
      "activations/layer12_attention_weight_min": -26.403526306152344,
      "activations/layer13_attention_weight_max": 41.866451263427734,
      "activations/layer13_attention_weight_min": -36.12939453125,
      "activations/layer14_attention_weight_max": 50.77836227416992,
      "activations/layer14_attention_weight_min": -40.58967971801758,
      "activations/layer15_attention_weight_max": 38.82948303222656,
      "activations/layer15_attention_weight_min": -31.431041717529297,
      "activations/layer16_attention_weight_max": 32.52714538574219,
      "activations/layer16_attention_weight_min": -28.817310333251953,
      "activations/layer17_attention_weight_max": 32.39190673828125,
      "activations/layer17_attention_weight_min": -26.047988891601562,
      "activations/layer18_attention_weight_max": 32.74530029296875,
      "activations/layer18_attention_weight_min": -22.14760971069336,
      "activations/layer19_attention_weight_max": 37.8939323425293,
      "activations/layer19_attention_weight_min": -31.60169792175293,
      "activations/layer1_attention_weight_max": 16.386110305786133,
      "activations/layer1_attention_weight_min": -13.83790111541748,
      "activations/layer20_attention_weight_max": 33.75607681274414,
      "activations/layer20_attention_weight_min": -25.807573318481445,
      "activations/layer21_attention_weight_max": 32.87574005126953,
      "activations/layer21_attention_weight_min": -24.203258514404297,
      "activations/layer22_attention_weight_max": 52.40455627441406,
      "activations/layer22_attention_weight_min": -30.455278396606445,
      "activations/layer23_attention_weight_max": 36.11759948730469,
      "activations/layer23_attention_weight_min": -25.22004508972168,
      "activations/layer2_attention_weight_max": 32.404109954833984,
      "activations/layer2_attention_weight_min": -35.16333770751953,
      "activations/layer3_attention_weight_max": 94.07389068603516,
      "activations/layer3_attention_weight_min": -95.8211669921875,
      "activations/layer4_attention_weight_max": 114.9945068359375,
      "activations/layer4_attention_weight_min": -115.80409240722656,
      "activations/layer5_attention_weight_max": 52.632469177246094,
      "activations/layer5_attention_weight_min": -62.827659606933594,
      "activations/layer6_attention_weight_max": 48.04252243041992,
      "activations/layer6_attention_weight_min": -49.96598434448242,
      "activations/layer7_attention_weight_max": 91.49308013916016,
      "activations/layer7_attention_weight_min": -96.7431869506836,
      "activations/layer8_attention_weight_max": 41.57533264160156,
      "activations/layer8_attention_weight_min": -44.453330993652344,
      "activations/layer9_attention_weight_max": 35.73063659667969,
      "activations/layer9_attention_weight_min": -35.77987289428711,
      "epoch": 10.27,
      "learning_rate": 8.46219696969697e-05,
      "loss": 2.7805,
      "step": 176750
    },
    {
      "activations/layer0_attention_weight_max": 16.234052658081055,
      "activations/layer0_attention_weight_min": -12.920443534851074,
      "activations/layer10_attention_weight_max": 32.09002685546875,
      "activations/layer10_attention_weight_min": -32.261837005615234,
      "activations/layer11_attention_weight_max": 33.998931884765625,
      "activations/layer11_attention_weight_min": -30.761877059936523,
      "activations/layer12_attention_weight_max": 24.892257690429688,
      "activations/layer12_attention_weight_min": -25.800153732299805,
      "activations/layer13_attention_weight_max": 39.530540466308594,
      "activations/layer13_attention_weight_min": -36.946632385253906,
      "activations/layer14_attention_weight_max": 47.53387451171875,
      "activations/layer14_attention_weight_min": -42.85725402832031,
      "activations/layer15_attention_weight_max": 38.90334701538086,
      "activations/layer15_attention_weight_min": -31.257017135620117,
      "activations/layer16_attention_weight_max": 33.78500747680664,
      "activations/layer16_attention_weight_min": -30.051715850830078,
      "activations/layer17_attention_weight_max": 31.515533447265625,
      "activations/layer17_attention_weight_min": -25.270219802856445,
      "activations/layer18_attention_weight_max": 34.28606414794922,
      "activations/layer18_attention_weight_min": -23.708587646484375,
      "activations/layer19_attention_weight_max": 37.20705795288086,
      "activations/layer19_attention_weight_min": -33.07826614379883,
      "activations/layer1_attention_weight_max": 15.658357620239258,
      "activations/layer1_attention_weight_min": -13.991251945495605,
      "activations/layer20_attention_weight_max": 29.699295043945312,
      "activations/layer20_attention_weight_min": -24.660572052001953,
      "activations/layer21_attention_weight_max": 32.94076156616211,
      "activations/layer21_attention_weight_min": -26.022478103637695,
      "activations/layer22_attention_weight_max": 49.39028549194336,
      "activations/layer22_attention_weight_min": -29.77504539489746,
      "activations/layer23_attention_weight_max": 38.06731414794922,
      "activations/layer23_attention_weight_min": -24.90130615234375,
      "activations/layer2_attention_weight_max": 31.030067443847656,
      "activations/layer2_attention_weight_min": -32.10173416137695,
      "activations/layer3_attention_weight_max": 96.5480728149414,
      "activations/layer3_attention_weight_min": -93.3175277709961,
      "activations/layer4_attention_weight_max": 116.59748840332031,
      "activations/layer4_attention_weight_min": -114.33509826660156,
      "activations/layer5_attention_weight_max": 55.95402145385742,
      "activations/layer5_attention_weight_min": -62.598480224609375,
      "activations/layer6_attention_weight_max": 45.921714782714844,
      "activations/layer6_attention_weight_min": -49.10120391845703,
      "activations/layer7_attention_weight_max": 90.1104965209961,
      "activations/layer7_attention_weight_min": -90.9195327758789,
      "activations/layer8_attention_weight_max": 39.036190032958984,
      "activations/layer8_attention_weight_min": -39.0540657043457,
      "activations/layer9_attention_weight_max": 31.683652877807617,
      "activations/layer9_attention_weight_min": -32.16216278076172,
      "epoch": 10.27,
      "learning_rate": 8.460303030303028e-05,
      "loss": 2.792,
      "step": 176800
    },
    {
      "activations/layer0_attention_weight_max": 16.98484992980957,
      "activations/layer0_attention_weight_min": -13.114788055419922,
      "activations/layer10_attention_weight_max": 33.401031494140625,
      "activations/layer10_attention_weight_min": -34.58806228637695,
      "activations/layer11_attention_weight_max": 34.39101028442383,
      "activations/layer11_attention_weight_min": -33.095314025878906,
      "activations/layer12_attention_weight_max": 24.25735092163086,
      "activations/layer12_attention_weight_min": -26.809598922729492,
      "activations/layer13_attention_weight_max": 38.35430908203125,
      "activations/layer13_attention_weight_min": -33.246116638183594,
      "activations/layer14_attention_weight_max": 41.36063003540039,
      "activations/layer14_attention_weight_min": -37.8327751159668,
      "activations/layer15_attention_weight_max": 34.255836486816406,
      "activations/layer15_attention_weight_min": -30.38414764404297,
      "activations/layer16_attention_weight_max": 31.55657196044922,
      "activations/layer16_attention_weight_min": -27.868698120117188,
      "activations/layer17_attention_weight_max": 30.71769142150879,
      "activations/layer17_attention_weight_min": -25.973337173461914,
      "activations/layer18_attention_weight_max": 30.58331871032715,
      "activations/layer18_attention_weight_min": -23.447208404541016,
      "activations/layer19_attention_weight_max": 32.313514709472656,
      "activations/layer19_attention_weight_min": -33.617733001708984,
      "activations/layer1_attention_weight_max": 17.481245040893555,
      "activations/layer1_attention_weight_min": -14.022292137145996,
      "activations/layer20_attention_weight_max": 29.5172061920166,
      "activations/layer20_attention_weight_min": -28.941600799560547,
      "activations/layer21_attention_weight_max": 27.35577964782715,
      "activations/layer21_attention_weight_min": -26.84093475341797,
      "activations/layer22_attention_weight_max": 40.69667053222656,
      "activations/layer22_attention_weight_min": -33.559783935546875,
      "activations/layer23_attention_weight_max": 31.37533187866211,
      "activations/layer23_attention_weight_min": -24.101539611816406,
      "activations/layer2_attention_weight_max": 32.143516540527344,
      "activations/layer2_attention_weight_min": -32.14674758911133,
      "activations/layer3_attention_weight_max": 94.1238784790039,
      "activations/layer3_attention_weight_min": -94.41796875,
      "activations/layer4_attention_weight_max": 114.0124282836914,
      "activations/layer4_attention_weight_min": -111.15338134765625,
      "activations/layer5_attention_weight_max": 50.422386169433594,
      "activations/layer5_attention_weight_min": -58.73827362060547,
      "activations/layer6_attention_weight_max": 46.406837463378906,
      "activations/layer6_attention_weight_min": -48.62405776977539,
      "activations/layer7_attention_weight_max": 88.77075958251953,
      "activations/layer7_attention_weight_min": -91.21984100341797,
      "activations/layer8_attention_weight_max": 39.5688362121582,
      "activations/layer8_attention_weight_min": -42.97508239746094,
      "activations/layer9_attention_weight_max": 35.588104248046875,
      "activations/layer9_attention_weight_min": -34.80763626098633,
      "epoch": 10.28,
      "learning_rate": 8.45840909090909e-05,
      "loss": 2.8021,
      "step": 176850
    },
    {
      "activations/layer0_attention_weight_max": 14.724626541137695,
      "activations/layer0_attention_weight_min": -12.92602825164795,
      "activations/layer10_attention_weight_max": 43.108360290527344,
      "activations/layer10_attention_weight_min": -38.40865707397461,
      "activations/layer11_attention_weight_max": 42.96241760253906,
      "activations/layer11_attention_weight_min": -39.64551544189453,
      "activations/layer12_attention_weight_max": 28.224992752075195,
      "activations/layer12_attention_weight_min": -26.09030532836914,
      "activations/layer13_attention_weight_max": 46.10807800292969,
      "activations/layer13_attention_weight_min": -35.78042221069336,
      "activations/layer14_attention_weight_max": 55.808372497558594,
      "activations/layer14_attention_weight_min": -43.4553108215332,
      "activations/layer15_attention_weight_max": 42.805606842041016,
      "activations/layer15_attention_weight_min": -33.71644592285156,
      "activations/layer16_attention_weight_max": 35.46864700317383,
      "activations/layer16_attention_weight_min": -30.10553550720215,
      "activations/layer17_attention_weight_max": 38.91042709350586,
      "activations/layer17_attention_weight_min": -27.323711395263672,
      "activations/layer18_attention_weight_max": 30.78145980834961,
      "activations/layer18_attention_weight_min": -24.274030685424805,
      "activations/layer19_attention_weight_max": 36.79721450805664,
      "activations/layer19_attention_weight_min": -32.07581329345703,
      "activations/layer1_attention_weight_max": 17.29993438720703,
      "activations/layer1_attention_weight_min": -15.625280380249023,
      "activations/layer20_attention_weight_max": 28.601173400878906,
      "activations/layer20_attention_weight_min": -24.175500869750977,
      "activations/layer21_attention_weight_max": 30.28891372680664,
      "activations/layer21_attention_weight_min": -23.300846099853516,
      "activations/layer22_attention_weight_max": 49.22038650512695,
      "activations/layer22_attention_weight_min": -32.56842803955078,
      "activations/layer23_attention_weight_max": 36.389156341552734,
      "activations/layer23_attention_weight_min": -26.9342041015625,
      "activations/layer2_attention_weight_max": 34.929813385009766,
      "activations/layer2_attention_weight_min": -34.40167999267578,
      "activations/layer3_attention_weight_max": 98.60169219970703,
      "activations/layer3_attention_weight_min": -97.9905014038086,
      "activations/layer4_attention_weight_max": 113.170654296875,
      "activations/layer4_attention_weight_min": -110.3499984741211,
      "activations/layer5_attention_weight_max": 55.83209991455078,
      "activations/layer5_attention_weight_min": -61.99407196044922,
      "activations/layer6_attention_weight_max": 46.87559127807617,
      "activations/layer6_attention_weight_min": -47.92533493041992,
      "activations/layer7_attention_weight_max": 101.57441711425781,
      "activations/layer7_attention_weight_min": -94.68206024169922,
      "activations/layer8_attention_weight_max": 46.365352630615234,
      "activations/layer8_attention_weight_min": -43.39515686035156,
      "activations/layer9_attention_weight_max": 42.34613037109375,
      "activations/layer9_attention_weight_min": -36.787174224853516,
      "epoch": 10.28,
      "learning_rate": 8.456515151515151e-05,
      "loss": 2.7783,
      "step": 176900
    },
    {
      "activations/layer0_attention_weight_max": 14.370659828186035,
      "activations/layer0_attention_weight_min": -12.668514251708984,
      "activations/layer10_attention_weight_max": 34.69580078125,
      "activations/layer10_attention_weight_min": -30.769460678100586,
      "activations/layer11_attention_weight_max": 34.57952117919922,
      "activations/layer11_attention_weight_min": -30.814735412597656,
      "activations/layer12_attention_weight_max": 25.794307708740234,
      "activations/layer12_attention_weight_min": -25.753183364868164,
      "activations/layer13_attention_weight_max": 43.19502639770508,
      "activations/layer13_attention_weight_min": -33.577392578125,
      "activations/layer14_attention_weight_max": 47.80881118774414,
      "activations/layer14_attention_weight_min": -38.175785064697266,
      "activations/layer15_attention_weight_max": 38.88840103149414,
      "activations/layer15_attention_weight_min": -31.552820205688477,
      "activations/layer16_attention_weight_max": 34.43752670288086,
      "activations/layer16_attention_weight_min": -28.945762634277344,
      "activations/layer17_attention_weight_max": 33.8369140625,
      "activations/layer17_attention_weight_min": -25.675355911254883,
      "activations/layer18_attention_weight_max": 35.23502731323242,
      "activations/layer18_attention_weight_min": -22.305004119873047,
      "activations/layer19_attention_weight_max": 41.18685531616211,
      "activations/layer19_attention_weight_min": -31.030628204345703,
      "activations/layer1_attention_weight_max": 17.005102157592773,
      "activations/layer1_attention_weight_min": -15.066901206970215,
      "activations/layer20_attention_weight_max": 36.1324462890625,
      "activations/layer20_attention_weight_min": -26.2307071685791,
      "activations/layer21_attention_weight_max": 33.93777847290039,
      "activations/layer21_attention_weight_min": -22.47908592224121,
      "activations/layer22_attention_weight_max": 53.615501403808594,
      "activations/layer22_attention_weight_min": -28.447521209716797,
      "activations/layer23_attention_weight_max": 43.05633544921875,
      "activations/layer23_attention_weight_min": -24.99536895751953,
      "activations/layer2_attention_weight_max": 31.80952262878418,
      "activations/layer2_attention_weight_min": -33.11665344238281,
      "activations/layer3_attention_weight_max": 94.12831115722656,
      "activations/layer3_attention_weight_min": -95.62686920166016,
      "activations/layer4_attention_weight_max": 108.56847381591797,
      "activations/layer4_attention_weight_min": -115.99433898925781,
      "activations/layer5_attention_weight_max": 52.652671813964844,
      "activations/layer5_attention_weight_min": -60.5373649597168,
      "activations/layer6_attention_weight_max": 43.87649917602539,
      "activations/layer6_attention_weight_min": -52.0615348815918,
      "activations/layer7_attention_weight_max": 86.56941986083984,
      "activations/layer7_attention_weight_min": -95.41236114501953,
      "activations/layer8_attention_weight_max": 38.12996292114258,
      "activations/layer8_attention_weight_min": -39.16744613647461,
      "activations/layer9_attention_weight_max": 33.381317138671875,
      "activations/layer9_attention_weight_min": -32.98557662963867,
      "epoch": 10.28,
      "learning_rate": 8.454621212121212e-05,
      "loss": 2.7898,
      "step": 176950
    },
    {
      "activations/layer0_attention_weight_max": 16.079666137695312,
      "activations/layer0_attention_weight_min": -13.353392601013184,
      "activations/layer10_attention_weight_max": 35.111690521240234,
      "activations/layer10_attention_weight_min": -33.52396011352539,
      "activations/layer11_attention_weight_max": 36.10756301879883,
      "activations/layer11_attention_weight_min": -32.65153503417969,
      "activations/layer12_attention_weight_max": 25.61190414428711,
      "activations/layer12_attention_weight_min": -26.39777183532715,
      "activations/layer13_attention_weight_max": 45.15167999267578,
      "activations/layer13_attention_weight_min": -36.04792404174805,
      "activations/layer14_attention_weight_max": 46.95500183105469,
      "activations/layer14_attention_weight_min": -40.71255874633789,
      "activations/layer15_attention_weight_max": 34.807884216308594,
      "activations/layer15_attention_weight_min": -32.66022491455078,
      "activations/layer16_attention_weight_max": 29.722463607788086,
      "activations/layer16_attention_weight_min": -28.601999282836914,
      "activations/layer17_attention_weight_max": 31.480472564697266,
      "activations/layer17_attention_weight_min": -26.398862838745117,
      "activations/layer18_attention_weight_max": 31.707073211669922,
      "activations/layer18_attention_weight_min": -22.425472259521484,
      "activations/layer19_attention_weight_max": 37.69887924194336,
      "activations/layer19_attention_weight_min": -30.064329147338867,
      "activations/layer1_attention_weight_max": 16.39457893371582,
      "activations/layer1_attention_weight_min": -15.048627853393555,
      "activations/layer20_attention_weight_max": 30.631608963012695,
      "activations/layer20_attention_weight_min": -24.785560607910156,
      "activations/layer21_attention_weight_max": 29.509389877319336,
      "activations/layer21_attention_weight_min": -25.217208862304688,
      "activations/layer22_attention_weight_max": 48.662811279296875,
      "activations/layer22_attention_weight_min": -30.5455379486084,
      "activations/layer23_attention_weight_max": 41.32667922973633,
      "activations/layer23_attention_weight_min": -23.68536949157715,
      "activations/layer2_attention_weight_max": 32.90534973144531,
      "activations/layer2_attention_weight_min": -33.6748161315918,
      "activations/layer3_attention_weight_max": 97.12313079833984,
      "activations/layer3_attention_weight_min": -100.2376708984375,
      "activations/layer4_attention_weight_max": 112.7599105834961,
      "activations/layer4_attention_weight_min": -111.927490234375,
      "activations/layer5_attention_weight_max": 53.800376892089844,
      "activations/layer5_attention_weight_min": -61.83331298828125,
      "activations/layer6_attention_weight_max": 45.72560501098633,
      "activations/layer6_attention_weight_min": -48.61823272705078,
      "activations/layer7_attention_weight_max": 91.62786102294922,
      "activations/layer7_attention_weight_min": -93.94081115722656,
      "activations/layer8_attention_weight_max": 42.75299072265625,
      "activations/layer8_attention_weight_min": -42.206851959228516,
      "activations/layer9_attention_weight_max": 37.45722579956055,
      "activations/layer9_attention_weight_min": -34.27920150756836,
      "epoch": 10.28,
      "learning_rate": 8.452727272727272e-05,
      "loss": 2.7787,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_loss": 2.73828125,
      "eval_runtime": 9.0098,
      "eval_samples_per_second": 476.592,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_openwebtext_loss": 2.73828125,
      "eval_openwebtext_ppl": 15.460389702669161,
      "eval_openwebtext_runtime": 9.0098,
      "eval_openwebtext_samples_per_second": 476.592,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 1.9048,
      "eval_wikitext_samples_per_second": 239.392,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.529,
      "eval_lambada_samples_per_second": 510.967,
      "step": 177000
    },
    {
      "activations/layer0_attention_weight_max": 16.127689361572266,
      "activations/layer0_attention_weight_min": -12.662970542907715,
      "activations/layer10_attention_weight_max": 38.763633728027344,
      "activations/layer10_attention_weight_min": -39.357330322265625,
      "activations/layer11_attention_weight_max": 39.723480224609375,
      "activations/layer11_attention_weight_min": -38.330902099609375,
      "activations/layer12_attention_weight_max": 29.241540908813477,
      "activations/layer12_attention_weight_min": -29.57537078857422,
      "activations/layer13_attention_weight_max": 47.58519744873047,
      "activations/layer13_attention_weight_min": -40.1655158996582,
      "activations/layer14_attention_weight_max": 54.10762405395508,
      "activations/layer14_attention_weight_min": -46.23291778564453,
      "activations/layer15_attention_weight_max": 38.745338439941406,
      "activations/layer15_attention_weight_min": -34.22754669189453,
      "activations/layer16_attention_weight_max": 32.04163360595703,
      "activations/layer16_attention_weight_min": -28.040990829467773,
      "activations/layer17_attention_weight_max": 31.634033203125,
      "activations/layer17_attention_weight_min": -27.226808547973633,
      "activations/layer18_attention_weight_max": 28.216976165771484,
      "activations/layer18_attention_weight_min": -25.238603591918945,
      "activations/layer19_attention_weight_max": 42.91444396972656,
      "activations/layer19_attention_weight_min": -30.097707748413086,
      "activations/layer1_attention_weight_max": 16.727737426757812,
      "activations/layer1_attention_weight_min": -16.48605728149414,
      "activations/layer20_attention_weight_max": 27.772472381591797,
      "activations/layer20_attention_weight_min": -23.48773765563965,
      "activations/layer21_attention_weight_max": 25.101831436157227,
      "activations/layer21_attention_weight_min": -24.25758934020996,
      "activations/layer22_attention_weight_max": 40.33774948120117,
      "activations/layer22_attention_weight_min": -28.720056533813477,
      "activations/layer23_attention_weight_max": 34.179931640625,
      "activations/layer23_attention_weight_min": -25.591724395751953,
      "activations/layer2_attention_weight_max": 33.718135833740234,
      "activations/layer2_attention_weight_min": -39.18950271606445,
      "activations/layer3_attention_weight_max": 102.57794952392578,
      "activations/layer3_attention_weight_min": -111.50222778320312,
      "activations/layer4_attention_weight_max": 116.47672271728516,
      "activations/layer4_attention_weight_min": -118.2998046875,
      "activations/layer5_attention_weight_max": 52.340084075927734,
      "activations/layer5_attention_weight_min": -62.14501190185547,
      "activations/layer6_attention_weight_max": 48.37169647216797,
      "activations/layer6_attention_weight_min": -53.29433059692383,
      "activations/layer7_attention_weight_max": 104.3878173828125,
      "activations/layer7_attention_weight_min": -107.53583526611328,
      "activations/layer8_attention_weight_max": 48.935482025146484,
      "activations/layer8_attention_weight_min": -49.266395568847656,
      "activations/layer9_attention_weight_max": 39.19673156738281,
      "activations/layer9_attention_weight_min": -39.11952209472656,
      "epoch": 10.29,
      "learning_rate": 8.450833333333333e-05,
      "loss": 2.7612,
      "step": 177050
    },
    {
      "activations/layer0_attention_weight_max": 15.727561950683594,
      "activations/layer0_attention_weight_min": -13.292638778686523,
      "activations/layer10_attention_weight_max": 31.74479866027832,
      "activations/layer10_attention_weight_min": -33.21795654296875,
      "activations/layer11_attention_weight_max": 32.46961975097656,
      "activations/layer11_attention_weight_min": -30.761531829833984,
      "activations/layer12_attention_weight_max": 23.539222717285156,
      "activations/layer12_attention_weight_min": -23.945817947387695,
      "activations/layer13_attention_weight_max": 37.31678009033203,
      "activations/layer13_attention_weight_min": -33.27619552612305,
      "activations/layer14_attention_weight_max": 42.24831771850586,
      "activations/layer14_attention_weight_min": -37.11240005493164,
      "activations/layer15_attention_weight_max": 35.28410720825195,
      "activations/layer15_attention_weight_min": -30.123388290405273,
      "activations/layer16_attention_weight_max": 29.415109634399414,
      "activations/layer16_attention_weight_min": -26.527637481689453,
      "activations/layer17_attention_weight_max": 31.916093826293945,
      "activations/layer17_attention_weight_min": -26.65933609008789,
      "activations/layer18_attention_weight_max": 32.66706466674805,
      "activations/layer18_attention_weight_min": -22.152467727661133,
      "activations/layer19_attention_weight_max": 32.780696868896484,
      "activations/layer19_attention_weight_min": -30.722644805908203,
      "activations/layer1_attention_weight_max": 16.609050750732422,
      "activations/layer1_attention_weight_min": -13.839969635009766,
      "activations/layer20_attention_weight_max": 28.46138572692871,
      "activations/layer20_attention_weight_min": -24.8271484375,
      "activations/layer21_attention_weight_max": 28.96877098083496,
      "activations/layer21_attention_weight_min": -23.653331756591797,
      "activations/layer22_attention_weight_max": 43.40357208251953,
      "activations/layer22_attention_weight_min": -30.178850173950195,
      "activations/layer23_attention_weight_max": 35.977882385253906,
      "activations/layer23_attention_weight_min": -24.152379989624023,
      "activations/layer2_attention_weight_max": 32.391700744628906,
      "activations/layer2_attention_weight_min": -34.07632827758789,
      "activations/layer3_attention_weight_max": 100.01744079589844,
      "activations/layer3_attention_weight_min": -103.53143310546875,
      "activations/layer4_attention_weight_max": 122.52949523925781,
      "activations/layer4_attention_weight_min": -121.08296966552734,
      "activations/layer5_attention_weight_max": 53.60097885131836,
      "activations/layer5_attention_weight_min": -62.31549072265625,
      "activations/layer6_attention_weight_max": 44.11103820800781,
      "activations/layer6_attention_weight_min": -49.101261138916016,
      "activations/layer7_attention_weight_max": 88.1733627319336,
      "activations/layer7_attention_weight_min": -89.16624450683594,
      "activations/layer8_attention_weight_max": 40.10185241699219,
      "activations/layer8_attention_weight_min": -40.418399810791016,
      "activations/layer9_attention_weight_max": 32.46449661254883,
      "activations/layer9_attention_weight_min": -32.5504264831543,
      "epoch": 10.29,
      "learning_rate": 8.448939393939393e-05,
      "loss": 2.7871,
      "step": 177100
    },
    {
      "activations/layer0_attention_weight_max": 16.044801712036133,
      "activations/layer0_attention_weight_min": -12.674389839172363,
      "activations/layer10_attention_weight_max": 41.021385192871094,
      "activations/layer10_attention_weight_min": -35.950111389160156,
      "activations/layer11_attention_weight_max": 43.241973876953125,
      "activations/layer11_attention_weight_min": -38.93883514404297,
      "activations/layer12_attention_weight_max": 27.839067459106445,
      "activations/layer12_attention_weight_min": -30.43659210205078,
      "activations/layer13_attention_weight_max": 39.844482421875,
      "activations/layer13_attention_weight_min": -33.72381591796875,
      "activations/layer14_attention_weight_max": 48.637351989746094,
      "activations/layer14_attention_weight_min": -38.692970275878906,
      "activations/layer15_attention_weight_max": 40.309898376464844,
      "activations/layer15_attention_weight_min": -31.4760799407959,
      "activations/layer16_attention_weight_max": 32.71012878417969,
      "activations/layer16_attention_weight_min": -28.927473068237305,
      "activations/layer17_attention_weight_max": 30.96563720703125,
      "activations/layer17_attention_weight_min": -25.236557006835938,
      "activations/layer18_attention_weight_max": 31.201778411865234,
      "activations/layer18_attention_weight_min": -23.026798248291016,
      "activations/layer19_attention_weight_max": 37.176570892333984,
      "activations/layer19_attention_weight_min": -31.36116600036621,
      "activations/layer1_attention_weight_max": 17.596038818359375,
      "activations/layer1_attention_weight_min": -14.958236694335938,
      "activations/layer20_attention_weight_max": 28.346513748168945,
      "activations/layer20_attention_weight_min": -24.022489547729492,
      "activations/layer21_attention_weight_max": 26.59916877746582,
      "activations/layer21_attention_weight_min": -23.300975799560547,
      "activations/layer22_attention_weight_max": 45.070091247558594,
      "activations/layer22_attention_weight_min": -31.19318389892578,
      "activations/layer23_attention_weight_max": 35.58625793457031,
      "activations/layer23_attention_weight_min": -24.00305938720703,
      "activations/layer2_attention_weight_max": 37.024715423583984,
      "activations/layer2_attention_weight_min": -37.42933654785156,
      "activations/layer3_attention_weight_max": 106.16215515136719,
      "activations/layer3_attention_weight_min": -104.34686279296875,
      "activations/layer4_attention_weight_max": 125.17142486572266,
      "activations/layer4_attention_weight_min": -118.04638671875,
      "activations/layer5_attention_weight_max": 58.215423583984375,
      "activations/layer5_attention_weight_min": -63.95024871826172,
      "activations/layer6_attention_weight_max": 47.415531158447266,
      "activations/layer6_attention_weight_min": -50.28562927246094,
      "activations/layer7_attention_weight_max": 103.3415756225586,
      "activations/layer7_attention_weight_min": -95.46527099609375,
      "activations/layer8_attention_weight_max": 45.59213638305664,
      "activations/layer8_attention_weight_min": -44.26042175292969,
      "activations/layer9_attention_weight_max": 39.95786666870117,
      "activations/layer9_attention_weight_min": -36.14942169189453,
      "epoch": 10.29,
      "learning_rate": 8.447045454545453e-05,
      "loss": 2.7716,
      "step": 177150
    },
    {
      "activations/layer0_attention_weight_max": 15.596412658691406,
      "activations/layer0_attention_weight_min": -12.86355972290039,
      "activations/layer10_attention_weight_max": 35.908531188964844,
      "activations/layer10_attention_weight_min": -34.671630859375,
      "activations/layer11_attention_weight_max": 36.03539276123047,
      "activations/layer11_attention_weight_min": -34.11932373046875,
      "activations/layer12_attention_weight_max": 25.209259033203125,
      "activations/layer12_attention_weight_min": -25.000465393066406,
      "activations/layer13_attention_weight_max": 39.10622024536133,
      "activations/layer13_attention_weight_min": -35.005615234375,
      "activations/layer14_attention_weight_max": 45.21161651611328,
      "activations/layer14_attention_weight_min": -37.47275924682617,
      "activations/layer15_attention_weight_max": 37.670230865478516,
      "activations/layer15_attention_weight_min": -30.425384521484375,
      "activations/layer16_attention_weight_max": 33.21928024291992,
      "activations/layer16_attention_weight_min": -28.333452224731445,
      "activations/layer17_attention_weight_max": 33.17795181274414,
      "activations/layer17_attention_weight_min": -23.889801025390625,
      "activations/layer18_attention_weight_max": 32.94253921508789,
      "activations/layer18_attention_weight_min": -23.832170486450195,
      "activations/layer19_attention_weight_max": 35.67786407470703,
      "activations/layer19_attention_weight_min": -30.181135177612305,
      "activations/layer1_attention_weight_max": 17.97974967956543,
      "activations/layer1_attention_weight_min": -14.66447925567627,
      "activations/layer20_attention_weight_max": 30.007036209106445,
      "activations/layer20_attention_weight_min": -24.57579231262207,
      "activations/layer21_attention_weight_max": 31.166057586669922,
      "activations/layer21_attention_weight_min": -24.272830963134766,
      "activations/layer22_attention_weight_max": 47.861785888671875,
      "activations/layer22_attention_weight_min": -31.746673583984375,
      "activations/layer23_attention_weight_max": 37.84272766113281,
      "activations/layer23_attention_weight_min": -23.186664581298828,
      "activations/layer2_attention_weight_max": 33.82353591918945,
      "activations/layer2_attention_weight_min": -33.082054138183594,
      "activations/layer3_attention_weight_max": 103.98936462402344,
      "activations/layer3_attention_weight_min": -102.9997787475586,
      "activations/layer4_attention_weight_max": 120.9952621459961,
      "activations/layer4_attention_weight_min": -121.31884765625,
      "activations/layer5_attention_weight_max": 54.27140808105469,
      "activations/layer5_attention_weight_min": -64.58865356445312,
      "activations/layer6_attention_weight_max": 48.4260368347168,
      "activations/layer6_attention_weight_min": -51.052101135253906,
      "activations/layer7_attention_weight_max": 97.84696197509766,
      "activations/layer7_attention_weight_min": -97.02818298339844,
      "activations/layer8_attention_weight_max": 44.01370620727539,
      "activations/layer8_attention_weight_min": -43.5817756652832,
      "activations/layer9_attention_weight_max": 37.527645111083984,
      "activations/layer9_attention_weight_min": -34.79976272583008,
      "epoch": 10.3,
      "learning_rate": 8.445151515151514e-05,
      "loss": 2.7747,
      "step": 177200
    },
    {
      "activations/layer0_attention_weight_max": 15.918198585510254,
      "activations/layer0_attention_weight_min": -12.079469680786133,
      "activations/layer10_attention_weight_max": 42.39945602416992,
      "activations/layer10_attention_weight_min": -41.24208068847656,
      "activations/layer11_attention_weight_max": 43.39506149291992,
      "activations/layer11_attention_weight_min": -43.113033294677734,
      "activations/layer12_attention_weight_max": 24.13895606994629,
      "activations/layer12_attention_weight_min": -30.443742752075195,
      "activations/layer13_attention_weight_max": 37.41558074951172,
      "activations/layer13_attention_weight_min": -37.08347702026367,
      "activations/layer14_attention_weight_max": 44.55327606201172,
      "activations/layer14_attention_weight_min": -39.20018768310547,
      "activations/layer15_attention_weight_max": 35.422607421875,
      "activations/layer15_attention_weight_min": -32.81568145751953,
      "activations/layer16_attention_weight_max": 30.020992279052734,
      "activations/layer16_attention_weight_min": -28.357816696166992,
      "activations/layer17_attention_weight_max": 37.161258697509766,
      "activations/layer17_attention_weight_min": -25.888362884521484,
      "activations/layer18_attention_weight_max": 31.732881546020508,
      "activations/layer18_attention_weight_min": -22.342771530151367,
      "activations/layer19_attention_weight_max": 36.45399856567383,
      "activations/layer19_attention_weight_min": -31.39240837097168,
      "activations/layer1_attention_weight_max": 16.736486434936523,
      "activations/layer1_attention_weight_min": -14.8771333694458,
      "activations/layer20_attention_weight_max": 30.8536376953125,
      "activations/layer20_attention_weight_min": -25.4818058013916,
      "activations/layer21_attention_weight_max": 31.552169799804688,
      "activations/layer21_attention_weight_min": -23.577951431274414,
      "activations/layer22_attention_weight_max": 51.0267448425293,
      "activations/layer22_attention_weight_min": -31.473163604736328,
      "activations/layer23_attention_weight_max": 35.38861846923828,
      "activations/layer23_attention_weight_min": -24.994144439697266,
      "activations/layer2_attention_weight_max": 33.19894790649414,
      "activations/layer2_attention_weight_min": -34.17368698120117,
      "activations/layer3_attention_weight_max": 95.68607330322266,
      "activations/layer3_attention_weight_min": -98.56159973144531,
      "activations/layer4_attention_weight_max": 111.0663070678711,
      "activations/layer4_attention_weight_min": -116.61372375488281,
      "activations/layer5_attention_weight_max": 60.239768981933594,
      "activations/layer5_attention_weight_min": -63.15758514404297,
      "activations/layer6_attention_weight_max": 48.42306900024414,
      "activations/layer6_attention_weight_min": -48.80604934692383,
      "activations/layer7_attention_weight_max": 113.5840835571289,
      "activations/layer7_attention_weight_min": -93.72039794921875,
      "activations/layer8_attention_weight_max": 52.42207717895508,
      "activations/layer8_attention_weight_min": -45.665958404541016,
      "activations/layer9_attention_weight_max": 40.69633483886719,
      "activations/layer9_attention_weight_min": -41.36373519897461,
      "epoch": 10.3,
      "learning_rate": 8.443257575757575e-05,
      "loss": 2.7987,
      "step": 177250
    },
    {
      "activations/layer0_attention_weight_max": 15.727635383605957,
      "activations/layer0_attention_weight_min": -11.428216934204102,
      "activations/layer10_attention_weight_max": 32.271671295166016,
      "activations/layer10_attention_weight_min": -31.688688278198242,
      "activations/layer11_attention_weight_max": 31.946849822998047,
      "activations/layer11_attention_weight_min": -32.9278564453125,
      "activations/layer12_attention_weight_max": 23.17778968811035,
      "activations/layer12_attention_weight_min": -27.21673011779785,
      "activations/layer13_attention_weight_max": 36.12107467651367,
      "activations/layer13_attention_weight_min": -33.891807556152344,
      "activations/layer14_attention_weight_max": 43.7092399597168,
      "activations/layer14_attention_weight_min": -36.52678298950195,
      "activations/layer15_attention_weight_max": 36.65653610229492,
      "activations/layer15_attention_weight_min": -31.008703231811523,
      "activations/layer16_attention_weight_max": 31.162256240844727,
      "activations/layer16_attention_weight_min": -28.55452537536621,
      "activations/layer17_attention_weight_max": 36.001068115234375,
      "activations/layer17_attention_weight_min": -26.782678604125977,
      "activations/layer18_attention_weight_max": 32.215667724609375,
      "activations/layer18_attention_weight_min": -23.346710205078125,
      "activations/layer19_attention_weight_max": 36.19681930541992,
      "activations/layer19_attention_weight_min": -30.543241500854492,
      "activations/layer1_attention_weight_max": 16.936037063598633,
      "activations/layer1_attention_weight_min": -14.207405090332031,
      "activations/layer20_attention_weight_max": 35.26111602783203,
      "activations/layer20_attention_weight_min": -24.876890182495117,
      "activations/layer21_attention_weight_max": 33.456451416015625,
      "activations/layer21_attention_weight_min": -23.58610725402832,
      "activations/layer22_attention_weight_max": 46.37772750854492,
      "activations/layer22_attention_weight_min": -28.199487686157227,
      "activations/layer23_attention_weight_max": 37.2807502746582,
      "activations/layer23_attention_weight_min": -22.952678680419922,
      "activations/layer2_attention_weight_max": 33.62526321411133,
      "activations/layer2_attention_weight_min": -33.50267028808594,
      "activations/layer3_attention_weight_max": 93.26930236816406,
      "activations/layer3_attention_weight_min": -95.80879974365234,
      "activations/layer4_attention_weight_max": 111.6740951538086,
      "activations/layer4_attention_weight_min": -108.06202697753906,
      "activations/layer5_attention_weight_max": 51.31150817871094,
      "activations/layer5_attention_weight_min": -61.83751678466797,
      "activations/layer6_attention_weight_max": 43.41600036621094,
      "activations/layer6_attention_weight_min": -45.86021423339844,
      "activations/layer7_attention_weight_max": 85.4802017211914,
      "activations/layer7_attention_weight_min": -91.25338745117188,
      "activations/layer8_attention_weight_max": 36.55088424682617,
      "activations/layer8_attention_weight_min": -39.47092819213867,
      "activations/layer9_attention_weight_max": 32.51002502441406,
      "activations/layer9_attention_weight_min": -32.00004577636719,
      "epoch": 10.3,
      "learning_rate": 8.441363636363637e-05,
      "loss": 2.7929,
      "step": 177300
    },
    {
      "activations/layer0_attention_weight_max": 16.02248764038086,
      "activations/layer0_attention_weight_min": -12.944993019104004,
      "activations/layer10_attention_weight_max": 32.38550567626953,
      "activations/layer10_attention_weight_min": -34.81956100463867,
      "activations/layer11_attention_weight_max": 31.995847702026367,
      "activations/layer11_attention_weight_min": -34.674232482910156,
      "activations/layer12_attention_weight_max": 26.30608558654785,
      "activations/layer12_attention_weight_min": -26.29035758972168,
      "activations/layer13_attention_weight_max": 41.279048919677734,
      "activations/layer13_attention_weight_min": -35.06292724609375,
      "activations/layer14_attention_weight_max": 46.807899475097656,
      "activations/layer14_attention_weight_min": -39.11198425292969,
      "activations/layer15_attention_weight_max": 40.30442810058594,
      "activations/layer15_attention_weight_min": -32.74342727661133,
      "activations/layer16_attention_weight_max": 36.32159423828125,
      "activations/layer16_attention_weight_min": -29.413766860961914,
      "activations/layer17_attention_weight_max": 40.683650970458984,
      "activations/layer17_attention_weight_min": -26.58647918701172,
      "activations/layer18_attention_weight_max": 40.50116729736328,
      "activations/layer18_attention_weight_min": -25.50350570678711,
      "activations/layer19_attention_weight_max": 44.67698287963867,
      "activations/layer19_attention_weight_min": -32.406150817871094,
      "activations/layer1_attention_weight_max": 16.991958618164062,
      "activations/layer1_attention_weight_min": -13.911627769470215,
      "activations/layer20_attention_weight_max": 38.894500732421875,
      "activations/layer20_attention_weight_min": -24.57103729248047,
      "activations/layer21_attention_weight_max": 38.5394287109375,
      "activations/layer21_attention_weight_min": -24.7318058013916,
      "activations/layer22_attention_weight_max": 49.5681037902832,
      "activations/layer22_attention_weight_min": -30.45032501220703,
      "activations/layer23_attention_weight_max": 42.248756408691406,
      "activations/layer23_attention_weight_min": -24.3419189453125,
      "activations/layer2_attention_weight_max": 32.97852325439453,
      "activations/layer2_attention_weight_min": -34.304779052734375,
      "activations/layer3_attention_weight_max": 97.22572326660156,
      "activations/layer3_attention_weight_min": -99.69389343261719,
      "activations/layer4_attention_weight_max": 112.6480484008789,
      "activations/layer4_attention_weight_min": -112.31290435791016,
      "activations/layer5_attention_weight_max": 51.03658676147461,
      "activations/layer5_attention_weight_min": -63.73877716064453,
      "activations/layer6_attention_weight_max": 46.19053268432617,
      "activations/layer6_attention_weight_min": -52.440223693847656,
      "activations/layer7_attention_weight_max": 90.17493438720703,
      "activations/layer7_attention_weight_min": -98.05206298828125,
      "activations/layer8_attention_weight_max": 42.107242584228516,
      "activations/layer8_attention_weight_min": -41.30305862426758,
      "activations/layer9_attention_weight_max": 33.889225006103516,
      "activations/layer9_attention_weight_min": -34.80897903442383,
      "epoch": 10.31,
      "learning_rate": 8.439469696969695e-05,
      "loss": 2.7786,
      "step": 177350
    },
    {
      "activations/layer0_attention_weight_max": 15.243017196655273,
      "activations/layer0_attention_weight_min": -12.633156776428223,
      "activations/layer10_attention_weight_max": 32.82252502441406,
      "activations/layer10_attention_weight_min": -32.55766677856445,
      "activations/layer11_attention_weight_max": 31.64674949645996,
      "activations/layer11_attention_weight_min": -30.52713394165039,
      "activations/layer12_attention_weight_max": 24.894805908203125,
      "activations/layer12_attention_weight_min": -24.382862091064453,
      "activations/layer13_attention_weight_max": 38.39519500732422,
      "activations/layer13_attention_weight_min": -33.50240707397461,
      "activations/layer14_attention_weight_max": 41.98143768310547,
      "activations/layer14_attention_weight_min": -35.614410400390625,
      "activations/layer15_attention_weight_max": 34.43260955810547,
      "activations/layer15_attention_weight_min": -31.027761459350586,
      "activations/layer16_attention_weight_max": 28.737991333007812,
      "activations/layer16_attention_weight_min": -28.197927474975586,
      "activations/layer17_attention_weight_max": 33.41041946411133,
      "activations/layer17_attention_weight_min": -25.09710121154785,
      "activations/layer18_attention_weight_max": 30.722909927368164,
      "activations/layer18_attention_weight_min": -21.07329559326172,
      "activations/layer19_attention_weight_max": 31.9728946685791,
      "activations/layer19_attention_weight_min": -29.6977596282959,
      "activations/layer1_attention_weight_max": 16.246318817138672,
      "activations/layer1_attention_weight_min": -14.014730453491211,
      "activations/layer20_attention_weight_max": 27.86696434020996,
      "activations/layer20_attention_weight_min": -24.845237731933594,
      "activations/layer21_attention_weight_max": 29.70221519470215,
      "activations/layer21_attention_weight_min": -23.818565368652344,
      "activations/layer22_attention_weight_max": 40.210968017578125,
      "activations/layer22_attention_weight_min": -29.29199981689453,
      "activations/layer23_attention_weight_max": 34.46253204345703,
      "activations/layer23_attention_weight_min": -23.390735626220703,
      "activations/layer2_attention_weight_max": 32.02499008178711,
      "activations/layer2_attention_weight_min": -33.43798828125,
      "activations/layer3_attention_weight_max": 93.09398651123047,
      "activations/layer3_attention_weight_min": -93.62487030029297,
      "activations/layer4_attention_weight_max": 110.06157684326172,
      "activations/layer4_attention_weight_min": -107.66674041748047,
      "activations/layer5_attention_weight_max": 52.35566329956055,
      "activations/layer5_attention_weight_min": -59.66262435913086,
      "activations/layer6_attention_weight_max": 47.895381927490234,
      "activations/layer6_attention_weight_min": -46.9180793762207,
      "activations/layer7_attention_weight_max": 88.8679428100586,
      "activations/layer7_attention_weight_min": -88.0093994140625,
      "activations/layer8_attention_weight_max": 40.6252555847168,
      "activations/layer8_attention_weight_min": -44.103965759277344,
      "activations/layer9_attention_weight_max": 33.04267501831055,
      "activations/layer9_attention_weight_min": -33.84699249267578,
      "epoch": 10.31,
      "learning_rate": 8.437575757575757e-05,
      "loss": 2.7594,
      "step": 177400
    },
    {
      "activations/layer0_attention_weight_max": 15.91479206085205,
      "activations/layer0_attention_weight_min": -13.146376609802246,
      "activations/layer10_attention_weight_max": 33.71467971801758,
      "activations/layer10_attention_weight_min": -31.940427780151367,
      "activations/layer11_attention_weight_max": 33.1379508972168,
      "activations/layer11_attention_weight_min": -29.783245086669922,
      "activations/layer12_attention_weight_max": 23.71272087097168,
      "activations/layer12_attention_weight_min": -25.242389678955078,
      "activations/layer13_attention_weight_max": 36.89550018310547,
      "activations/layer13_attention_weight_min": -33.0562858581543,
      "activations/layer14_attention_weight_max": 43.69017791748047,
      "activations/layer14_attention_weight_min": -35.88033676147461,
      "activations/layer15_attention_weight_max": 34.0991325378418,
      "activations/layer15_attention_weight_min": -32.31418991088867,
      "activations/layer16_attention_weight_max": 31.36082649230957,
      "activations/layer16_attention_weight_min": -27.819704055786133,
      "activations/layer17_attention_weight_max": 31.65045166015625,
      "activations/layer17_attention_weight_min": -25.142244338989258,
      "activations/layer18_attention_weight_max": 32.0323600769043,
      "activations/layer18_attention_weight_min": -23.650333404541016,
      "activations/layer19_attention_weight_max": 33.41480255126953,
      "activations/layer19_attention_weight_min": -29.365610122680664,
      "activations/layer1_attention_weight_max": 16.466299057006836,
      "activations/layer1_attention_weight_min": -14.704140663146973,
      "activations/layer20_attention_weight_max": 30.046062469482422,
      "activations/layer20_attention_weight_min": -24.38726234436035,
      "activations/layer21_attention_weight_max": 29.839916229248047,
      "activations/layer21_attention_weight_min": -23.924386978149414,
      "activations/layer22_attention_weight_max": 43.55850601196289,
      "activations/layer22_attention_weight_min": -29.287860870361328,
      "activations/layer23_attention_weight_max": 34.516387939453125,
      "activations/layer23_attention_weight_min": -24.363059997558594,
      "activations/layer2_attention_weight_max": 33.610267639160156,
      "activations/layer2_attention_weight_min": -33.70323944091797,
      "activations/layer3_attention_weight_max": 88.06578063964844,
      "activations/layer3_attention_weight_min": -90.85442352294922,
      "activations/layer4_attention_weight_max": 108.1173324584961,
      "activations/layer4_attention_weight_min": -116.11199188232422,
      "activations/layer5_attention_weight_max": 54.15515899658203,
      "activations/layer5_attention_weight_min": -60.81672286987305,
      "activations/layer6_attention_weight_max": 42.87147903442383,
      "activations/layer6_attention_weight_min": -50.387020111083984,
      "activations/layer7_attention_weight_max": 87.6175765991211,
      "activations/layer7_attention_weight_min": -84.5743637084961,
      "activations/layer8_attention_weight_max": 40.293575286865234,
      "activations/layer8_attention_weight_min": -39.315303802490234,
      "activations/layer9_attention_weight_max": 35.256832122802734,
      "activations/layer9_attention_weight_min": -33.83077621459961,
      "epoch": 10.31,
      "learning_rate": 8.435681818181818e-05,
      "loss": 2.7832,
      "step": 177450
    },
    {
      "activations/layer0_attention_weight_max": 15.45529842376709,
      "activations/layer0_attention_weight_min": -13.005738258361816,
      "activations/layer10_attention_weight_max": 30.167068481445312,
      "activations/layer10_attention_weight_min": -31.592111587524414,
      "activations/layer11_attention_weight_max": 30.14971923828125,
      "activations/layer11_attention_weight_min": -30.032821655273438,
      "activations/layer12_attention_weight_max": 23.517736434936523,
      "activations/layer12_attention_weight_min": -23.072608947753906,
      "activations/layer13_attention_weight_max": 36.78010177612305,
      "activations/layer13_attention_weight_min": -32.34375762939453,
      "activations/layer14_attention_weight_max": 41.297061920166016,
      "activations/layer14_attention_weight_min": -33.635520935058594,
      "activations/layer15_attention_weight_max": 34.843448638916016,
      "activations/layer15_attention_weight_min": -30.63697052001953,
      "activations/layer16_attention_weight_max": 30.060131072998047,
      "activations/layer16_attention_weight_min": -26.466365814208984,
      "activations/layer17_attention_weight_max": 31.81058120727539,
      "activations/layer17_attention_weight_min": -26.090190887451172,
      "activations/layer18_attention_weight_max": 31.640560150146484,
      "activations/layer18_attention_weight_min": -24.239492416381836,
      "activations/layer19_attention_weight_max": 34.99361038208008,
      "activations/layer19_attention_weight_min": -29.365385055541992,
      "activations/layer1_attention_weight_max": 17.018598556518555,
      "activations/layer1_attention_weight_min": -14.802924156188965,
      "activations/layer20_attention_weight_max": 29.33026695251465,
      "activations/layer20_attention_weight_min": -24.190181732177734,
      "activations/layer21_attention_weight_max": 29.33884048461914,
      "activations/layer21_attention_weight_min": -23.152145385742188,
      "activations/layer22_attention_weight_max": 41.90644073486328,
      "activations/layer22_attention_weight_min": -28.546953201293945,
      "activations/layer23_attention_weight_max": 36.27821731567383,
      "activations/layer23_attention_weight_min": -26.121997833251953,
      "activations/layer2_attention_weight_max": 32.308502197265625,
      "activations/layer2_attention_weight_min": -33.3070068359375,
      "activations/layer3_attention_weight_max": 94.61852264404297,
      "activations/layer3_attention_weight_min": -97.22398376464844,
      "activations/layer4_attention_weight_max": 111.3283462524414,
      "activations/layer4_attention_weight_min": -111.87895965576172,
      "activations/layer5_attention_weight_max": 50.867637634277344,
      "activations/layer5_attention_weight_min": -58.852333068847656,
      "activations/layer6_attention_weight_max": 42.008331298828125,
      "activations/layer6_attention_weight_min": -47.654579162597656,
      "activations/layer7_attention_weight_max": 79.69640350341797,
      "activations/layer7_attention_weight_min": -81.7695083618164,
      "activations/layer8_attention_weight_max": 36.99647903442383,
      "activations/layer8_attention_weight_min": -37.642852783203125,
      "activations/layer9_attention_weight_max": 31.94489097595215,
      "activations/layer9_attention_weight_min": -32.177616119384766,
      "epoch": 10.31,
      "learning_rate": 8.433787878787877e-05,
      "loss": 2.7957,
      "step": 177500
    },
    {
      "activations/layer0_attention_weight_max": 15.529694557189941,
      "activations/layer0_attention_weight_min": -13.610543251037598,
      "activations/layer10_attention_weight_max": 33.73031997680664,
      "activations/layer10_attention_weight_min": -32.519805908203125,
      "activations/layer11_attention_weight_max": 31.165281295776367,
      "activations/layer11_attention_weight_min": -29.506072998046875,
      "activations/layer12_attention_weight_max": 23.09516716003418,
      "activations/layer12_attention_weight_min": -24.540237426757812,
      "activations/layer13_attention_weight_max": 36.15076446533203,
      "activations/layer13_attention_weight_min": -33.20671463012695,
      "activations/layer14_attention_weight_max": 52.04330062866211,
      "activations/layer14_attention_weight_min": -41.845550537109375,
      "activations/layer15_attention_weight_max": 32.752872467041016,
      "activations/layer15_attention_weight_min": -30.715612411499023,
      "activations/layer16_attention_weight_max": 27.738868713378906,
      "activations/layer16_attention_weight_min": -27.113004684448242,
      "activations/layer17_attention_weight_max": 31.60344886779785,
      "activations/layer17_attention_weight_min": -24.938419342041016,
      "activations/layer18_attention_weight_max": 33.30103302001953,
      "activations/layer18_attention_weight_min": -23.310747146606445,
      "activations/layer19_attention_weight_max": 32.13835144042969,
      "activations/layer19_attention_weight_min": -29.56060791015625,
      "activations/layer1_attention_weight_max": 17.006235122680664,
      "activations/layer1_attention_weight_min": -12.859023094177246,
      "activations/layer20_attention_weight_max": 29.177888870239258,
      "activations/layer20_attention_weight_min": -24.038911819458008,
      "activations/layer21_attention_weight_max": 28.318126678466797,
      "activations/layer21_attention_weight_min": -22.93011474609375,
      "activations/layer22_attention_weight_max": 44.423439025878906,
      "activations/layer22_attention_weight_min": -30.01802635192871,
      "activations/layer23_attention_weight_max": 34.72184753417969,
      "activations/layer23_attention_weight_min": -26.08998680114746,
      "activations/layer2_attention_weight_max": 31.983592987060547,
      "activations/layer2_attention_weight_min": -32.55755615234375,
      "activations/layer3_attention_weight_max": 95.01250457763672,
      "activations/layer3_attention_weight_min": -94.51708984375,
      "activations/layer4_attention_weight_max": 115.71063232421875,
      "activations/layer4_attention_weight_min": -112.38264465332031,
      "activations/layer5_attention_weight_max": 51.86125183105469,
      "activations/layer5_attention_weight_min": -60.09022521972656,
      "activations/layer6_attention_weight_max": 44.14127731323242,
      "activations/layer6_attention_weight_min": -48.877140045166016,
      "activations/layer7_attention_weight_max": 85.57799530029297,
      "activations/layer7_attention_weight_min": -84.07086181640625,
      "activations/layer8_attention_weight_max": 40.00814437866211,
      "activations/layer8_attention_weight_min": -39.97357940673828,
      "activations/layer9_attention_weight_max": 33.02658462524414,
      "activations/layer9_attention_weight_min": -31.675617218017578,
      "epoch": 10.32,
      "learning_rate": 8.431893939393939e-05,
      "loss": 2.7808,
      "step": 177550
    },
    {
      "activations/layer0_attention_weight_max": 15.859230041503906,
      "activations/layer0_attention_weight_min": -13.504075050354004,
      "activations/layer10_attention_weight_max": 30.394731521606445,
      "activations/layer10_attention_weight_min": -29.999784469604492,
      "activations/layer11_attention_weight_max": 29.66846466064453,
      "activations/layer11_attention_weight_min": -29.81403923034668,
      "activations/layer12_attention_weight_max": 22.83514976501465,
      "activations/layer12_attention_weight_min": -25.226526260375977,
      "activations/layer13_attention_weight_max": 34.889930725097656,
      "activations/layer13_attention_weight_min": -32.278533935546875,
      "activations/layer14_attention_weight_max": 43.956939697265625,
      "activations/layer14_attention_weight_min": -35.15052795410156,
      "activations/layer15_attention_weight_max": 32.864898681640625,
      "activations/layer15_attention_weight_min": -31.39732551574707,
      "activations/layer16_attention_weight_max": 28.786027908325195,
      "activations/layer16_attention_weight_min": -28.22239875793457,
      "activations/layer17_attention_weight_max": 32.68044662475586,
      "activations/layer17_attention_weight_min": -25.178253173828125,
      "activations/layer18_attention_weight_max": 31.342021942138672,
      "activations/layer18_attention_weight_min": -23.01244354248047,
      "activations/layer19_attention_weight_max": 34.60033416748047,
      "activations/layer19_attention_weight_min": -28.890003204345703,
      "activations/layer1_attention_weight_max": 20.387861251831055,
      "activations/layer1_attention_weight_min": -17.232202529907227,
      "activations/layer20_attention_weight_max": 29.581642150878906,
      "activations/layer20_attention_weight_min": -26.692033767700195,
      "activations/layer21_attention_weight_max": 26.97604751586914,
      "activations/layer21_attention_weight_min": -23.486167907714844,
      "activations/layer22_attention_weight_max": 48.35481262207031,
      "activations/layer22_attention_weight_min": -27.73542594909668,
      "activations/layer23_attention_weight_max": 36.51104736328125,
      "activations/layer23_attention_weight_min": -25.78394317626953,
      "activations/layer2_attention_weight_max": 33.65625762939453,
      "activations/layer2_attention_weight_min": -33.18402099609375,
      "activations/layer3_attention_weight_max": 96.34564208984375,
      "activations/layer3_attention_weight_min": -92.12773895263672,
      "activations/layer4_attention_weight_max": 117.03627014160156,
      "activations/layer4_attention_weight_min": -113.28165435791016,
      "activations/layer5_attention_weight_max": 52.801536560058594,
      "activations/layer5_attention_weight_min": -62.96657943725586,
      "activations/layer6_attention_weight_max": 42.346656799316406,
      "activations/layer6_attention_weight_min": -45.9608039855957,
      "activations/layer7_attention_weight_max": 86.62198638916016,
      "activations/layer7_attention_weight_min": -82.13980102539062,
      "activations/layer8_attention_weight_max": 35.97252655029297,
      "activations/layer8_attention_weight_min": -39.49131774902344,
      "activations/layer9_attention_weight_max": 29.937091827392578,
      "activations/layer9_attention_weight_min": -31.372888565063477,
      "epoch": 10.32,
      "learning_rate": 8.43e-05,
      "loss": 2.7915,
      "step": 177600
    },
    {
      "activations/layer0_attention_weight_max": 15.748416900634766,
      "activations/layer0_attention_weight_min": -12.945395469665527,
      "activations/layer10_attention_weight_max": 37.87364196777344,
      "activations/layer10_attention_weight_min": -34.58951187133789,
      "activations/layer11_attention_weight_max": 39.13383102416992,
      "activations/layer11_attention_weight_min": -34.46033477783203,
      "activations/layer12_attention_weight_max": 43.033851623535156,
      "activations/layer12_attention_weight_min": -28.625837326049805,
      "activations/layer13_attention_weight_max": 61.243614196777344,
      "activations/layer13_attention_weight_min": -32.44866943359375,
      "activations/layer14_attention_weight_max": 50.211769104003906,
      "activations/layer14_attention_weight_min": -36.35643768310547,
      "activations/layer15_attention_weight_max": 60.5980339050293,
      "activations/layer15_attention_weight_min": -31.598026275634766,
      "activations/layer16_attention_weight_max": 33.81275939941406,
      "activations/layer16_attention_weight_min": -25.52432632446289,
      "activations/layer17_attention_weight_max": 30.097843170166016,
      "activations/layer17_attention_weight_min": -24.319278717041016,
      "activations/layer18_attention_weight_max": 27.787145614624023,
      "activations/layer18_attention_weight_min": -21.70730209350586,
      "activations/layer19_attention_weight_max": 34.90145492553711,
      "activations/layer19_attention_weight_min": -29.9381160736084,
      "activations/layer1_attention_weight_max": 17.708589553833008,
      "activations/layer1_attention_weight_min": -13.882844924926758,
      "activations/layer20_attention_weight_max": 28.311607360839844,
      "activations/layer20_attention_weight_min": -22.062976837158203,
      "activations/layer21_attention_weight_max": 25.913503646850586,
      "activations/layer21_attention_weight_min": -21.551437377929688,
      "activations/layer22_attention_weight_max": 42.628501892089844,
      "activations/layer22_attention_weight_min": -28.587291717529297,
      "activations/layer23_attention_weight_max": 30.9073486328125,
      "activations/layer23_attention_weight_min": -21.91925048828125,
      "activations/layer2_attention_weight_max": 33.75041198730469,
      "activations/layer2_attention_weight_min": -35.00647735595703,
      "activations/layer3_attention_weight_max": 99.83637237548828,
      "activations/layer3_attention_weight_min": -101.30903625488281,
      "activations/layer4_attention_weight_max": 117.51947021484375,
      "activations/layer4_attention_weight_min": -115.60684967041016,
      "activations/layer5_attention_weight_max": 58.526878356933594,
      "activations/layer5_attention_weight_min": -61.45929718017578,
      "activations/layer6_attention_weight_max": 46.78959274291992,
      "activations/layer6_attention_weight_min": -48.187503814697266,
      "activations/layer7_attention_weight_max": 101.7421646118164,
      "activations/layer7_attention_weight_min": -91.64985656738281,
      "activations/layer8_attention_weight_max": 45.03450012207031,
      "activations/layer8_attention_weight_min": -42.48073196411133,
      "activations/layer9_attention_weight_max": 38.60292434692383,
      "activations/layer9_attention_weight_min": -37.4044075012207,
      "epoch": 10.32,
      "learning_rate": 8.428106060606059e-05,
      "loss": 2.7874,
      "step": 177650
    },
    {
      "activations/layer0_attention_weight_max": 16.08674430847168,
      "activations/layer0_attention_weight_min": -12.465591430664062,
      "activations/layer10_attention_weight_max": 32.686946868896484,
      "activations/layer10_attention_weight_min": -35.623504638671875,
      "activations/layer11_attention_weight_max": 34.58577346801758,
      "activations/layer11_attention_weight_min": -34.00277328491211,
      "activations/layer12_attention_weight_max": 26.24763298034668,
      "activations/layer12_attention_weight_min": -27.7513484954834,
      "activations/layer13_attention_weight_max": 37.70380783081055,
      "activations/layer13_attention_weight_min": -34.06208419799805,
      "activations/layer14_attention_weight_max": 46.49713897705078,
      "activations/layer14_attention_weight_min": -38.07503128051758,
      "activations/layer15_attention_weight_max": 37.37486267089844,
      "activations/layer15_attention_weight_min": -36.933223724365234,
      "activations/layer16_attention_weight_max": 32.3857421875,
      "activations/layer16_attention_weight_min": -32.209449768066406,
      "activations/layer17_attention_weight_max": 33.27727127075195,
      "activations/layer17_attention_weight_min": -30.842668533325195,
      "activations/layer18_attention_weight_max": 29.478328704833984,
      "activations/layer18_attention_weight_min": -25.259910583496094,
      "activations/layer19_attention_weight_max": 43.468910217285156,
      "activations/layer19_attention_weight_min": -33.95724105834961,
      "activations/layer1_attention_weight_max": 16.741260528564453,
      "activations/layer1_attention_weight_min": -14.200835227966309,
      "activations/layer20_attention_weight_max": 32.75461196899414,
      "activations/layer20_attention_weight_min": -26.481443405151367,
      "activations/layer21_attention_weight_max": 30.000091552734375,
      "activations/layer21_attention_weight_min": -24.957387924194336,
      "activations/layer22_attention_weight_max": 48.33579635620117,
      "activations/layer22_attention_weight_min": -32.786869049072266,
      "activations/layer23_attention_weight_max": 36.12456512451172,
      "activations/layer23_attention_weight_min": -25.849498748779297,
      "activations/layer2_attention_weight_max": 33.18538284301758,
      "activations/layer2_attention_weight_min": -35.67525100708008,
      "activations/layer3_attention_weight_max": 98.10140991210938,
      "activations/layer3_attention_weight_min": -102.87389373779297,
      "activations/layer4_attention_weight_max": 120.72480773925781,
      "activations/layer4_attention_weight_min": -115.4542236328125,
      "activations/layer5_attention_weight_max": 60.23274230957031,
      "activations/layer5_attention_weight_min": -60.7408561706543,
      "activations/layer6_attention_weight_max": 45.80055618286133,
      "activations/layer6_attention_weight_min": -48.29627227783203,
      "activations/layer7_attention_weight_max": 86.8856201171875,
      "activations/layer7_attention_weight_min": -97.90544128417969,
      "activations/layer8_attention_weight_max": 40.742008209228516,
      "activations/layer8_attention_weight_min": -44.66005325317383,
      "activations/layer9_attention_weight_max": 33.31240463256836,
      "activations/layer9_attention_weight_min": -34.854007720947266,
      "epoch": 10.33,
      "learning_rate": 8.42621212121212e-05,
      "loss": 2.7688,
      "step": 177700
    },
    {
      "activations/layer0_attention_weight_max": 15.8185453414917,
      "activations/layer0_attention_weight_min": -12.94033432006836,
      "activations/layer10_attention_weight_max": 39.52373504638672,
      "activations/layer10_attention_weight_min": -33.014984130859375,
      "activations/layer11_attention_weight_max": 39.03683090209961,
      "activations/layer11_attention_weight_min": -34.83573913574219,
      "activations/layer12_attention_weight_max": 29.723302841186523,
      "activations/layer12_attention_weight_min": -27.135486602783203,
      "activations/layer13_attention_weight_max": 42.95302963256836,
      "activations/layer13_attention_weight_min": -33.773555755615234,
      "activations/layer14_attention_weight_max": 52.4512939453125,
      "activations/layer14_attention_weight_min": -37.836605072021484,
      "activations/layer15_attention_weight_max": 37.289817810058594,
      "activations/layer15_attention_weight_min": -31.30095100402832,
      "activations/layer16_attention_weight_max": 33.27735137939453,
      "activations/layer16_attention_weight_min": -27.674327850341797,
      "activations/layer17_attention_weight_max": 30.96293830871582,
      "activations/layer17_attention_weight_min": -23.782312393188477,
      "activations/layer18_attention_weight_max": 29.50112533569336,
      "activations/layer18_attention_weight_min": -20.801279067993164,
      "activations/layer19_attention_weight_max": 36.02796173095703,
      "activations/layer19_attention_weight_min": -30.06282615661621,
      "activations/layer1_attention_weight_max": 17.005083084106445,
      "activations/layer1_attention_weight_min": -16.65949058532715,
      "activations/layer20_attention_weight_max": 30.308137893676758,
      "activations/layer20_attention_weight_min": -25.934762954711914,
      "activations/layer21_attention_weight_max": 25.929454803466797,
      "activations/layer21_attention_weight_min": -24.212230682373047,
      "activations/layer22_attention_weight_max": 43.085811614990234,
      "activations/layer22_attention_weight_min": -30.408187866210938,
      "activations/layer23_attention_weight_max": 30.63540267944336,
      "activations/layer23_attention_weight_min": -24.74332046508789,
      "activations/layer2_attention_weight_max": 36.11626434326172,
      "activations/layer2_attention_weight_min": -37.13479232788086,
      "activations/layer3_attention_weight_max": 100.76826477050781,
      "activations/layer3_attention_weight_min": -103.21135711669922,
      "activations/layer4_attention_weight_max": 116.70775604248047,
      "activations/layer4_attention_weight_min": -114.15215301513672,
      "activations/layer5_attention_weight_max": 56.45092010498047,
      "activations/layer5_attention_weight_min": -64.68914031982422,
      "activations/layer6_attention_weight_max": 51.291221618652344,
      "activations/layer6_attention_weight_min": -50.62614059448242,
      "activations/layer7_attention_weight_max": 100.18192291259766,
      "activations/layer7_attention_weight_min": -103.59086608886719,
      "activations/layer8_attention_weight_max": 47.74071502685547,
      "activations/layer8_attention_weight_min": -44.73762512207031,
      "activations/layer9_attention_weight_max": 39.12641525268555,
      "activations/layer9_attention_weight_min": -35.57276153564453,
      "epoch": 10.33,
      "learning_rate": 8.424318181818181e-05,
      "loss": 2.7854,
      "step": 177750
    },
    {
      "activations/layer0_attention_weight_max": 15.582586288452148,
      "activations/layer0_attention_weight_min": -12.255444526672363,
      "activations/layer10_attention_weight_max": 32.973785400390625,
      "activations/layer10_attention_weight_min": -32.11688232421875,
      "activations/layer11_attention_weight_max": 34.04391860961914,
      "activations/layer11_attention_weight_min": -32.58046340942383,
      "activations/layer12_attention_weight_max": 26.899885177612305,
      "activations/layer12_attention_weight_min": -26.432687759399414,
      "activations/layer13_attention_weight_max": 39.386497497558594,
      "activations/layer13_attention_weight_min": -34.06525421142578,
      "activations/layer14_attention_weight_max": 43.837860107421875,
      "activations/layer14_attention_weight_min": -36.07929992675781,
      "activations/layer15_attention_weight_max": 34.54195785522461,
      "activations/layer15_attention_weight_min": -29.927629470825195,
      "activations/layer16_attention_weight_max": 30.754653930664062,
      "activations/layer16_attention_weight_min": -28.366031646728516,
      "activations/layer17_attention_weight_max": 33.481510162353516,
      "activations/layer17_attention_weight_min": -24.95125389099121,
      "activations/layer18_attention_weight_max": 32.92668533325195,
      "activations/layer18_attention_weight_min": -22.240840911865234,
      "activations/layer19_attention_weight_max": 34.72280502319336,
      "activations/layer19_attention_weight_min": -30.507333755493164,
      "activations/layer1_attention_weight_max": 16.433818817138672,
      "activations/layer1_attention_weight_min": -14.427993774414062,
      "activations/layer20_attention_weight_max": 33.939117431640625,
      "activations/layer20_attention_weight_min": -25.703710556030273,
      "activations/layer21_attention_weight_max": 29.4738826751709,
      "activations/layer21_attention_weight_min": -24.103050231933594,
      "activations/layer22_attention_weight_max": 48.12788772583008,
      "activations/layer22_attention_weight_min": -32.91111373901367,
      "activations/layer23_attention_weight_max": 31.533246994018555,
      "activations/layer23_attention_weight_min": -27.4334716796875,
      "activations/layer2_attention_weight_max": 32.38099670410156,
      "activations/layer2_attention_weight_min": -33.35111618041992,
      "activations/layer3_attention_weight_max": 97.61073303222656,
      "activations/layer3_attention_weight_min": -96.07061004638672,
      "activations/layer4_attention_weight_max": 120.4113540649414,
      "activations/layer4_attention_weight_min": -118.03409576416016,
      "activations/layer5_attention_weight_max": 54.384185791015625,
      "activations/layer5_attention_weight_min": -63.716552734375,
      "activations/layer6_attention_weight_max": 47.12320327758789,
      "activations/layer6_attention_weight_min": -49.27424621582031,
      "activations/layer7_attention_weight_max": 90.32331848144531,
      "activations/layer7_attention_weight_min": -93.20818328857422,
      "activations/layer8_attention_weight_max": 42.34954833984375,
      "activations/layer8_attention_weight_min": -42.47291946411133,
      "activations/layer9_attention_weight_max": 34.2030029296875,
      "activations/layer9_attention_weight_min": -34.75156784057617,
      "epoch": 10.33,
      "learning_rate": 8.422424242424242e-05,
      "loss": 2.7773,
      "step": 177800
    },
    {
      "activations/layer0_attention_weight_max": 16.13995361328125,
      "activations/layer0_attention_weight_min": -11.820405006408691,
      "activations/layer10_attention_weight_max": 30.114788055419922,
      "activations/layer10_attention_weight_min": -29.977659225463867,
      "activations/layer11_attention_weight_max": 30.907581329345703,
      "activations/layer11_attention_weight_min": -29.800735473632812,
      "activations/layer12_attention_weight_max": 22.17107391357422,
      "activations/layer12_attention_weight_min": -27.23723602294922,
      "activations/layer13_attention_weight_max": 35.48637008666992,
      "activations/layer13_attention_weight_min": -32.72111511230469,
      "activations/layer14_attention_weight_max": 40.13253402709961,
      "activations/layer14_attention_weight_min": -35.58993148803711,
      "activations/layer15_attention_weight_max": 33.63808059692383,
      "activations/layer15_attention_weight_min": -31.755428314208984,
      "activations/layer16_attention_weight_max": 29.320512771606445,
      "activations/layer16_attention_weight_min": -32.417930603027344,
      "activations/layer17_attention_weight_max": 30.830942153930664,
      "activations/layer17_attention_weight_min": -25.29417610168457,
      "activations/layer18_attention_weight_max": 28.262081146240234,
      "activations/layer18_attention_weight_min": -21.702566146850586,
      "activations/layer19_attention_weight_max": 33.324031829833984,
      "activations/layer19_attention_weight_min": -33.19632339477539,
      "activations/layer1_attention_weight_max": 16.54233741760254,
      "activations/layer1_attention_weight_min": -14.136557579040527,
      "activations/layer20_attention_weight_max": 26.699384689331055,
      "activations/layer20_attention_weight_min": -24.020172119140625,
      "activations/layer21_attention_weight_max": 28.088022232055664,
      "activations/layer21_attention_weight_min": -23.23113441467285,
      "activations/layer22_attention_weight_max": 44.48009490966797,
      "activations/layer22_attention_weight_min": -28.39094352722168,
      "activations/layer23_attention_weight_max": 33.7911262512207,
      "activations/layer23_attention_weight_min": -25.128089904785156,
      "activations/layer2_attention_weight_max": 32.99401092529297,
      "activations/layer2_attention_weight_min": -32.2834358215332,
      "activations/layer3_attention_weight_max": 92.33306884765625,
      "activations/layer3_attention_weight_min": -93.46220397949219,
      "activations/layer4_attention_weight_max": 114.03929901123047,
      "activations/layer4_attention_weight_min": -112.46703338623047,
      "activations/layer5_attention_weight_max": 50.863094329833984,
      "activations/layer5_attention_weight_min": -61.31380844116211,
      "activations/layer6_attention_weight_max": 44.230037689208984,
      "activations/layer6_attention_weight_min": -49.1551628112793,
      "activations/layer7_attention_weight_max": 91.95867156982422,
      "activations/layer7_attention_weight_min": -83.28771209716797,
      "activations/layer8_attention_weight_max": 40.51830291748047,
      "activations/layer8_attention_weight_min": -37.17502212524414,
      "activations/layer9_attention_weight_max": 30.51377296447754,
      "activations/layer9_attention_weight_min": -31.314838409423828,
      "epoch": 10.33,
      "learning_rate": 8.420530303030302e-05,
      "loss": 2.7731,
      "step": 177850
    },
    {
      "activations/layer0_attention_weight_max": 15.186700820922852,
      "activations/layer0_attention_weight_min": -13.448057174682617,
      "activations/layer10_attention_weight_max": 35.749298095703125,
      "activations/layer10_attention_weight_min": -33.50880432128906,
      "activations/layer11_attention_weight_max": 32.618343353271484,
      "activations/layer11_attention_weight_min": -31.036575317382812,
      "activations/layer12_attention_weight_max": 22.85818099975586,
      "activations/layer12_attention_weight_min": -25.50413703918457,
      "activations/layer13_attention_weight_max": 36.94657516479492,
      "activations/layer13_attention_weight_min": -33.651695251464844,
      "activations/layer14_attention_weight_max": 42.87044906616211,
      "activations/layer14_attention_weight_min": -37.401912689208984,
      "activations/layer15_attention_weight_max": 34.66238021850586,
      "activations/layer15_attention_weight_min": -31.46446990966797,
      "activations/layer16_attention_weight_max": 31.1329345703125,
      "activations/layer16_attention_weight_min": -27.358997344970703,
      "activations/layer17_attention_weight_max": 32.969276428222656,
      "activations/layer17_attention_weight_min": -24.900341033935547,
      "activations/layer18_attention_weight_max": 31.8813533782959,
      "activations/layer18_attention_weight_min": -22.678180694580078,
      "activations/layer19_attention_weight_max": 34.00522994995117,
      "activations/layer19_attention_weight_min": -29.75259780883789,
      "activations/layer1_attention_weight_max": 16.701284408569336,
      "activations/layer1_attention_weight_min": -14.458024024963379,
      "activations/layer20_attention_weight_max": 30.736122131347656,
      "activations/layer20_attention_weight_min": -24.649694442749023,
      "activations/layer21_attention_weight_max": 29.974027633666992,
      "activations/layer21_attention_weight_min": -24.039154052734375,
      "activations/layer22_attention_weight_max": 43.64360809326172,
      "activations/layer22_attention_weight_min": -28.708677291870117,
      "activations/layer23_attention_weight_max": 34.37929916381836,
      "activations/layer23_attention_weight_min": -24.9470272064209,
      "activations/layer2_attention_weight_max": 34.38002014160156,
      "activations/layer2_attention_weight_min": -34.131187438964844,
      "activations/layer3_attention_weight_max": 100.91548156738281,
      "activations/layer3_attention_weight_min": -100.25299835205078,
      "activations/layer4_attention_weight_max": 117.14996337890625,
      "activations/layer4_attention_weight_min": -122.68187713623047,
      "activations/layer5_attention_weight_max": 53.86492156982422,
      "activations/layer5_attention_weight_min": -62.741920471191406,
      "activations/layer6_attention_weight_max": 45.88225173950195,
      "activations/layer6_attention_weight_min": -50.56150436401367,
      "activations/layer7_attention_weight_max": 89.78243255615234,
      "activations/layer7_attention_weight_min": -92.46112823486328,
      "activations/layer8_attention_weight_max": 41.75154113769531,
      "activations/layer8_attention_weight_min": -43.0340690612793,
      "activations/layer9_attention_weight_max": 35.547698974609375,
      "activations/layer9_attention_weight_min": -34.13447952270508,
      "epoch": 10.34,
      "learning_rate": 8.418636363636363e-05,
      "loss": 2.7937,
      "step": 177900
    },
    {
      "activations/layer0_attention_weight_max": 16.380130767822266,
      "activations/layer0_attention_weight_min": -12.136775970458984,
      "activations/layer10_attention_weight_max": 33.488525390625,
      "activations/layer10_attention_weight_min": -34.4029426574707,
      "activations/layer11_attention_weight_max": 32.38201904296875,
      "activations/layer11_attention_weight_min": -33.0079345703125,
      "activations/layer12_attention_weight_max": 24.837873458862305,
      "activations/layer12_attention_weight_min": -25.76692008972168,
      "activations/layer13_attention_weight_max": 39.51483917236328,
      "activations/layer13_attention_weight_min": -35.142601013183594,
      "activations/layer14_attention_weight_max": 48.469417572021484,
      "activations/layer14_attention_weight_min": -38.64202117919922,
      "activations/layer15_attention_weight_max": 37.476593017578125,
      "activations/layer15_attention_weight_min": -32.83566665649414,
      "activations/layer16_attention_weight_max": 32.58367156982422,
      "activations/layer16_attention_weight_min": -28.184444427490234,
      "activations/layer17_attention_weight_max": 34.084049224853516,
      "activations/layer17_attention_weight_min": -27.817340850830078,
      "activations/layer18_attention_weight_max": 33.80287551879883,
      "activations/layer18_attention_weight_min": -23.611982345581055,
      "activations/layer19_attention_weight_max": 39.649322509765625,
      "activations/layer19_attention_weight_min": -31.154037475585938,
      "activations/layer1_attention_weight_max": 16.326278686523438,
      "activations/layer1_attention_weight_min": -14.340187072753906,
      "activations/layer20_attention_weight_max": 33.10499954223633,
      "activations/layer20_attention_weight_min": -25.02644920349121,
      "activations/layer21_attention_weight_max": 31.19530487060547,
      "activations/layer21_attention_weight_min": -25.189603805541992,
      "activations/layer22_attention_weight_max": 48.725921630859375,
      "activations/layer22_attention_weight_min": -30.235240936279297,
      "activations/layer23_attention_weight_max": 38.49985885620117,
      "activations/layer23_attention_weight_min": -23.97212791442871,
      "activations/layer2_attention_weight_max": 34.15712356567383,
      "activations/layer2_attention_weight_min": -34.728004455566406,
      "activations/layer3_attention_weight_max": 94.32792663574219,
      "activations/layer3_attention_weight_min": -92.20248413085938,
      "activations/layer4_attention_weight_max": 116.72769927978516,
      "activations/layer4_attention_weight_min": -109.37873840332031,
      "activations/layer5_attention_weight_max": 55.89497375488281,
      "activations/layer5_attention_weight_min": -59.09918212890625,
      "activations/layer6_attention_weight_max": 45.2057991027832,
      "activations/layer6_attention_weight_min": -46.97246551513672,
      "activations/layer7_attention_weight_max": 89.36148071289062,
      "activations/layer7_attention_weight_min": -91.2695541381836,
      "activations/layer8_attention_weight_max": 39.45708465576172,
      "activations/layer8_attention_weight_min": -40.39023208618164,
      "activations/layer9_attention_weight_max": 33.042137145996094,
      "activations/layer9_attention_weight_min": -33.90864181518555,
      "epoch": 10.34,
      "learning_rate": 8.416742424242424e-05,
      "loss": 2.7854,
      "step": 177950
    },
    {
      "activations/layer0_attention_weight_max": 15.72818660736084,
      "activations/layer0_attention_weight_min": -12.700742721557617,
      "activations/layer10_attention_weight_max": 33.7894172668457,
      "activations/layer10_attention_weight_min": -33.98231506347656,
      "activations/layer11_attention_weight_max": 35.014305114746094,
      "activations/layer11_attention_weight_min": -32.339134216308594,
      "activations/layer12_attention_weight_max": 31.67937469482422,
      "activations/layer12_attention_weight_min": -26.723215103149414,
      "activations/layer13_attention_weight_max": 40.426551818847656,
      "activations/layer13_attention_weight_min": -32.136474609375,
      "activations/layer14_attention_weight_max": 43.76461410522461,
      "activations/layer14_attention_weight_min": -35.88004684448242,
      "activations/layer15_attention_weight_max": 36.84018325805664,
      "activations/layer15_attention_weight_min": -29.844417572021484,
      "activations/layer16_attention_weight_max": 33.21565628051758,
      "activations/layer16_attention_weight_min": -28.669221878051758,
      "activations/layer17_attention_weight_max": 31.53511619567871,
      "activations/layer17_attention_weight_min": -26.94066619873047,
      "activations/layer18_attention_weight_max": 32.016597747802734,
      "activations/layer18_attention_weight_min": -22.999549865722656,
      "activations/layer19_attention_weight_max": 36.45191192626953,
      "activations/layer19_attention_weight_min": -33.04043197631836,
      "activations/layer1_attention_weight_max": 16.227676391601562,
      "activations/layer1_attention_weight_min": -14.049405097961426,
      "activations/layer20_attention_weight_max": 32.25178909301758,
      "activations/layer20_attention_weight_min": -27.202180862426758,
      "activations/layer21_attention_weight_max": 28.938077926635742,
      "activations/layer21_attention_weight_min": -25.640249252319336,
      "activations/layer22_attention_weight_max": 48.039222717285156,
      "activations/layer22_attention_weight_min": -34.22023010253906,
      "activations/layer23_attention_weight_max": 35.73450469970703,
      "activations/layer23_attention_weight_min": -24.90843963623047,
      "activations/layer2_attention_weight_max": 32.854827880859375,
      "activations/layer2_attention_weight_min": -34.25022888183594,
      "activations/layer3_attention_weight_max": 97.72700500488281,
      "activations/layer3_attention_weight_min": -100.48334503173828,
      "activations/layer4_attention_weight_max": 110.6769790649414,
      "activations/layer4_attention_weight_min": -115.11732482910156,
      "activations/layer5_attention_weight_max": 55.856910705566406,
      "activations/layer5_attention_weight_min": -60.293235778808594,
      "activations/layer6_attention_weight_max": 46.145652770996094,
      "activations/layer6_attention_weight_min": -49.41371154785156,
      "activations/layer7_attention_weight_max": 94.28447723388672,
      "activations/layer7_attention_weight_min": -95.11762237548828,
      "activations/layer8_attention_weight_max": 41.14497756958008,
      "activations/layer8_attention_weight_min": -40.51686096191406,
      "activations/layer9_attention_weight_max": 33.33723068237305,
      "activations/layer9_attention_weight_min": -34.24363327026367,
      "epoch": 10.34,
      "learning_rate": 8.414848484848484e-05,
      "loss": 2.7918,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_loss": 2.73828125,
      "eval_runtime": 8.4867,
      "eval_samples_per_second": 505.968,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_openwebtext_loss": 2.73828125,
      "eval_openwebtext_ppl": 15.460389702669161,
      "eval_openwebtext_runtime": 8.4867,
      "eval_openwebtext_samples_per_second": 505.968,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_wikitext_loss": 2.9609375,
      "eval_wikitext_ppl": 19.31607208722833,
      "eval_wikitext_runtime": 1.9304,
      "eval_wikitext_samples_per_second": 236.215,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.5306,
      "eval_lambada_samples_per_second": 510.883,
      "step": 178000
    },
    {
      "activations/layer0_attention_weight_max": 16.226877212524414,
      "activations/layer0_attention_weight_min": -12.128925323486328,
      "activations/layer10_attention_weight_max": 35.91232681274414,
      "activations/layer10_attention_weight_min": -33.954750061035156,
      "activations/layer11_attention_weight_max": 42.394569396972656,
      "activations/layer11_attention_weight_min": -34.558345794677734,
      "activations/layer12_attention_weight_max": 30.747474670410156,
      "activations/layer12_attention_weight_min": -27.841670989990234,
      "activations/layer13_attention_weight_max": 57.54207229614258,
      "activations/layer13_attention_weight_min": -33.391963958740234,
      "activations/layer14_attention_weight_max": 51.42353820800781,
      "activations/layer14_attention_weight_min": -36.807403564453125,
      "activations/layer15_attention_weight_max": 49.61206817626953,
      "activations/layer15_attention_weight_min": -32.46696472167969,
      "activations/layer16_attention_weight_max": 33.14854431152344,
      "activations/layer16_attention_weight_min": -28.255184173583984,
      "activations/layer17_attention_weight_max": 30.97473907470703,
      "activations/layer17_attention_weight_min": -27.383012771606445,
      "activations/layer18_attention_weight_max": 31.030929565429688,
      "activations/layer18_attention_weight_min": -24.119251251220703,
      "activations/layer19_attention_weight_max": 39.827842712402344,
      "activations/layer19_attention_weight_min": -30.5487003326416,
      "activations/layer1_attention_weight_max": 16.7869815826416,
      "activations/layer1_attention_weight_min": -13.058380126953125,
      "activations/layer20_attention_weight_max": 34.40625762939453,
      "activations/layer20_attention_weight_min": -25.238481521606445,
      "activations/layer21_attention_weight_max": 33.541282653808594,
      "activations/layer21_attention_weight_min": -23.16373634338379,
      "activations/layer22_attention_weight_max": 58.5954704284668,
      "activations/layer22_attention_weight_min": -29.84617042541504,
      "activations/layer23_attention_weight_max": 38.66341018676758,
      "activations/layer23_attention_weight_min": -23.74249267578125,
      "activations/layer2_attention_weight_max": 34.75827407836914,
      "activations/layer2_attention_weight_min": -35.14687728881836,
      "activations/layer3_attention_weight_max": 94.0125732421875,
      "activations/layer3_attention_weight_min": -99.43599700927734,
      "activations/layer4_attention_weight_max": 115.421875,
      "activations/layer4_attention_weight_min": -113.79610443115234,
      "activations/layer5_attention_weight_max": 53.28342819213867,
      "activations/layer5_attention_weight_min": -63.75599670410156,
      "activations/layer6_attention_weight_max": 43.957828521728516,
      "activations/layer6_attention_weight_min": -47.02567672729492,
      "activations/layer7_attention_weight_max": 94.3027572631836,
      "activations/layer7_attention_weight_min": -92.56144714355469,
      "activations/layer8_attention_weight_max": 40.40061950683594,
      "activations/layer8_attention_weight_min": -40.91510772705078,
      "activations/layer9_attention_weight_max": 37.48293685913086,
      "activations/layer9_attention_weight_min": -34.82040786743164,
      "epoch": 10.35,
      "learning_rate": 8.412954545454544e-05,
      "loss": 2.7822,
      "step": 178050
    },
    {
      "activations/layer0_attention_weight_max": 14.777349472045898,
      "activations/layer0_attention_weight_min": -12.736845016479492,
      "activations/layer10_attention_weight_max": 40.9932861328125,
      "activations/layer10_attention_weight_min": -40.41112518310547,
      "activations/layer11_attention_weight_max": 45.356201171875,
      "activations/layer11_attention_weight_min": -41.249717712402344,
      "activations/layer12_attention_weight_max": 28.80263900756836,
      "activations/layer12_attention_weight_min": -31.646812438964844,
      "activations/layer13_attention_weight_max": 40.038116455078125,
      "activations/layer13_attention_weight_min": -35.83057403564453,
      "activations/layer14_attention_weight_max": 49.82785415649414,
      "activations/layer14_attention_weight_min": -41.4792594909668,
      "activations/layer15_attention_weight_max": 40.28130340576172,
      "activations/layer15_attention_weight_min": -33.58329391479492,
      "activations/layer16_attention_weight_max": 36.56311798095703,
      "activations/layer16_attention_weight_min": -27.963762283325195,
      "activations/layer17_attention_weight_max": 32.263248443603516,
      "activations/layer17_attention_weight_min": -25.822860717773438,
      "activations/layer18_attention_weight_max": 34.4903678894043,
      "activations/layer18_attention_weight_min": -22.988054275512695,
      "activations/layer19_attention_weight_max": 40.250667572021484,
      "activations/layer19_attention_weight_min": -30.715280532836914,
      "activations/layer1_attention_weight_max": 16.569517135620117,
      "activations/layer1_attention_weight_min": -14.393477439880371,
      "activations/layer20_attention_weight_max": 30.968883514404297,
      "activations/layer20_attention_weight_min": -25.38695526123047,
      "activations/layer21_attention_weight_max": 29.967105865478516,
      "activations/layer21_attention_weight_min": -23.94573974609375,
      "activations/layer22_attention_weight_max": 49.4282112121582,
      "activations/layer22_attention_weight_min": -29.60970115661621,
      "activations/layer23_attention_weight_max": 34.931671142578125,
      "activations/layer23_attention_weight_min": -24.523250579833984,
      "activations/layer2_attention_weight_max": 37.78968811035156,
      "activations/layer2_attention_weight_min": -37.29726791381836,
      "activations/layer3_attention_weight_max": 108.58189392089844,
      "activations/layer3_attention_weight_min": -109.62239074707031,
      "activations/layer4_attention_weight_max": 125.31709289550781,
      "activations/layer4_attention_weight_min": -128.9418487548828,
      "activations/layer5_attention_weight_max": 59.571563720703125,
      "activations/layer5_attention_weight_min": -63.512474060058594,
      "activations/layer6_attention_weight_max": 51.84988784790039,
      "activations/layer6_attention_weight_min": -51.979713439941406,
      "activations/layer7_attention_weight_max": 108.96353912353516,
      "activations/layer7_attention_weight_min": -106.71571350097656,
      "activations/layer8_attention_weight_max": 51.977806091308594,
      "activations/layer8_attention_weight_min": -51.52378845214844,
      "activations/layer9_attention_weight_max": 45.60795974731445,
      "activations/layer9_attention_weight_min": -43.57293701171875,
      "epoch": 10.35,
      "learning_rate": 8.411098484848483e-05,
      "loss": 2.7962,
      "step": 178100
    },
    {
      "activations/layer0_attention_weight_max": 16.496509552001953,
      "activations/layer0_attention_weight_min": -13.15234088897705,
      "activations/layer10_attention_weight_max": 34.497711181640625,
      "activations/layer10_attention_weight_min": -31.555906295776367,
      "activations/layer11_attention_weight_max": 35.81310272216797,
      "activations/layer11_attention_weight_min": -32.09333419799805,
      "activations/layer12_attention_weight_max": 34.01926803588867,
      "activations/layer12_attention_weight_min": -25.646039962768555,
      "activations/layer13_attention_weight_max": 53.434791564941406,
      "activations/layer13_attention_weight_min": -33.73067092895508,
      "activations/layer14_attention_weight_max": 62.29381561279297,
      "activations/layer14_attention_weight_min": -38.464752197265625,
      "activations/layer15_attention_weight_max": 55.23737335205078,
      "activations/layer15_attention_weight_min": -31.140087127685547,
      "activations/layer16_attention_weight_max": 39.650611877441406,
      "activations/layer16_attention_weight_min": -27.477190017700195,
      "activations/layer17_attention_weight_max": 41.67014694213867,
      "activations/layer17_attention_weight_min": -26.76164436340332,
      "activations/layer18_attention_weight_max": 36.620880126953125,
      "activations/layer18_attention_weight_min": -20.808427810668945,
      "activations/layer19_attention_weight_max": 37.90950012207031,
      "activations/layer19_attention_weight_min": -29.688749313354492,
      "activations/layer1_attention_weight_max": 16.61220932006836,
      "activations/layer1_attention_weight_min": -16.477041244506836,
      "activations/layer20_attention_weight_max": 33.0616569519043,
      "activations/layer20_attention_weight_min": -25.257888793945312,
      "activations/layer21_attention_weight_max": 34.081993103027344,
      "activations/layer21_attention_weight_min": -22.9027042388916,
      "activations/layer22_attention_weight_max": 61.78277587890625,
      "activations/layer22_attention_weight_min": -28.116960525512695,
      "activations/layer23_attention_weight_max": 40.98005294799805,
      "activations/layer23_attention_weight_min": -23.93246078491211,
      "activations/layer2_attention_weight_max": 33.09310531616211,
      "activations/layer2_attention_weight_min": -35.27500915527344,
      "activations/layer3_attention_weight_max": 95.30451965332031,
      "activations/layer3_attention_weight_min": -100.197509765625,
      "activations/layer4_attention_weight_max": 115.61824798583984,
      "activations/layer4_attention_weight_min": -115.03668212890625,
      "activations/layer5_attention_weight_max": 52.816062927246094,
      "activations/layer5_attention_weight_min": -67.25918579101562,
      "activations/layer6_attention_weight_max": 45.34938049316406,
      "activations/layer6_attention_weight_min": -48.6264533996582,
      "activations/layer7_attention_weight_max": 88.95122528076172,
      "activations/layer7_attention_weight_min": -92.89385223388672,
      "activations/layer8_attention_weight_max": 38.95229721069336,
      "activations/layer8_attention_weight_min": -41.61552047729492,
      "activations/layer9_attention_weight_max": 33.97391128540039,
      "activations/layer9_attention_weight_min": -32.29685592651367,
      "epoch": 10.35,
      "learning_rate": 8.409204545454545e-05,
      "loss": 2.7759,
      "step": 178150
    },
    {
      "activations/layer0_attention_weight_max": 15.427887916564941,
      "activations/layer0_attention_weight_min": -11.579147338867188,
      "activations/layer10_attention_weight_max": 30.871639251708984,
      "activations/layer10_attention_weight_min": -30.310428619384766,
      "activations/layer11_attention_weight_max": 31.558807373046875,
      "activations/layer11_attention_weight_min": -29.673633575439453,
      "activations/layer12_attention_weight_max": 25.39040756225586,
      "activations/layer12_attention_weight_min": -27.72834014892578,
      "activations/layer13_attention_weight_max": 41.448204040527344,
      "activations/layer13_attention_weight_min": -34.88747787475586,
      "activations/layer14_attention_weight_max": 49.814125061035156,
      "activations/layer14_attention_weight_min": -41.47826385498047,
      "activations/layer15_attention_weight_max": 35.60447692871094,
      "activations/layer15_attention_weight_min": -31.124359130859375,
      "activations/layer16_attention_weight_max": 32.20588302612305,
      "activations/layer16_attention_weight_min": -27.7751522064209,
      "activations/layer17_attention_weight_max": 32.9512939453125,
      "activations/layer17_attention_weight_min": -25.649335861206055,
      "activations/layer18_attention_weight_max": 31.937959671020508,
      "activations/layer18_attention_weight_min": -23.51786994934082,
      "activations/layer19_attention_weight_max": 36.25045394897461,
      "activations/layer19_attention_weight_min": -30.7426700592041,
      "activations/layer1_attention_weight_max": 16.578327178955078,
      "activations/layer1_attention_weight_min": -14.433348655700684,
      "activations/layer20_attention_weight_max": 33.186214447021484,
      "activations/layer20_attention_weight_min": -25.041831970214844,
      "activations/layer21_attention_weight_max": 31.381175994873047,
      "activations/layer21_attention_weight_min": -24.139297485351562,
      "activations/layer22_attention_weight_max": 50.09234619140625,
      "activations/layer22_attention_weight_min": -31.099939346313477,
      "activations/layer23_attention_weight_max": 35.86711502075195,
      "activations/layer23_attention_weight_min": -24.611713409423828,
      "activations/layer2_attention_weight_max": 33.034027099609375,
      "activations/layer2_attention_weight_min": -33.79739761352539,
      "activations/layer3_attention_weight_max": 95.29469299316406,
      "activations/layer3_attention_weight_min": -98.7957992553711,
      "activations/layer4_attention_weight_max": 110.80582427978516,
      "activations/layer4_attention_weight_min": -117.71240234375,
      "activations/layer5_attention_weight_max": 56.92257308959961,
      "activations/layer5_attention_weight_min": -65.08560180664062,
      "activations/layer6_attention_weight_max": 43.12975311279297,
      "activations/layer6_attention_weight_min": -47.766719818115234,
      "activations/layer7_attention_weight_max": 89.05268859863281,
      "activations/layer7_attention_weight_min": -89.33782958984375,
      "activations/layer8_attention_weight_max": 37.074825286865234,
      "activations/layer8_attention_weight_min": -38.310054779052734,
      "activations/layer9_attention_weight_max": 31.895435333251953,
      "activations/layer9_attention_weight_min": -33.815975189208984,
      "epoch": 10.35,
      "learning_rate": 8.407310606060606e-05,
      "loss": 2.7642,
      "step": 178200
    },
    {
      "activations/layer0_attention_weight_max": 16.066057205200195,
      "activations/layer0_attention_weight_min": -12.139616012573242,
      "activations/layer10_attention_weight_max": 33.8357048034668,
      "activations/layer10_attention_weight_min": -32.769287109375,
      "activations/layer11_attention_weight_max": 32.68360137939453,
      "activations/layer11_attention_weight_min": -31.622844696044922,
      "activations/layer12_attention_weight_max": 27.432880401611328,
      "activations/layer12_attention_weight_min": -25.782068252563477,
      "activations/layer13_attention_weight_max": 41.638526916503906,
      "activations/layer13_attention_weight_min": -34.55305480957031,
      "activations/layer14_attention_weight_max": 48.626258850097656,
      "activations/layer14_attention_weight_min": -41.15570831298828,
      "activations/layer15_attention_weight_max": 39.50376892089844,
      "activations/layer15_attention_weight_min": -32.321712493896484,
      "activations/layer16_attention_weight_max": 32.416847229003906,
      "activations/layer16_attention_weight_min": -28.758190155029297,
      "activations/layer17_attention_weight_max": 36.322635650634766,
      "activations/layer17_attention_weight_min": -27.45033073425293,
      "activations/layer18_attention_weight_max": 32.78519058227539,
      "activations/layer18_attention_weight_min": -22.810192108154297,
      "activations/layer19_attention_weight_max": 36.06458282470703,
      "activations/layer19_attention_weight_min": -32.78160095214844,
      "activations/layer1_attention_weight_max": 17.78620147705078,
      "activations/layer1_attention_weight_min": -13.548598289489746,
      "activations/layer20_attention_weight_max": 31.421581268310547,
      "activations/layer20_attention_weight_min": -27.205814361572266,
      "activations/layer21_attention_weight_max": 31.40890884399414,
      "activations/layer21_attention_weight_min": -24.538915634155273,
      "activations/layer22_attention_weight_max": 45.53118896484375,
      "activations/layer22_attention_weight_min": -29.466089248657227,
      "activations/layer23_attention_weight_max": 36.063087463378906,
      "activations/layer23_attention_weight_min": -23.77009391784668,
      "activations/layer2_attention_weight_max": 32.00613784790039,
      "activations/layer2_attention_weight_min": -33.6849365234375,
      "activations/layer3_attention_weight_max": 93.2507095336914,
      "activations/layer3_attention_weight_min": -97.00254821777344,
      "activations/layer4_attention_weight_max": 114.59915924072266,
      "activations/layer4_attention_weight_min": -116.1816635131836,
      "activations/layer5_attention_weight_max": 55.698665618896484,
      "activations/layer5_attention_weight_min": -63.562679290771484,
      "activations/layer6_attention_weight_max": 46.82551574707031,
      "activations/layer6_attention_weight_min": -50.879703521728516,
      "activations/layer7_attention_weight_max": 93.524169921875,
      "activations/layer7_attention_weight_min": -95.91036987304688,
      "activations/layer8_attention_weight_max": 40.54892349243164,
      "activations/layer8_attention_weight_min": -41.978363037109375,
      "activations/layer9_attention_weight_max": 34.91722106933594,
      "activations/layer9_attention_weight_min": -33.092891693115234,
      "epoch": 10.36,
      "learning_rate": 8.405416666666665e-05,
      "loss": 2.78,
      "step": 178250
    },
    {
      "activations/layer0_attention_weight_max": 15.902189254760742,
      "activations/layer0_attention_weight_min": -12.865764617919922,
      "activations/layer10_attention_weight_max": 33.45457458496094,
      "activations/layer10_attention_weight_min": -32.013648986816406,
      "activations/layer11_attention_weight_max": 33.47221755981445,
      "activations/layer11_attention_weight_min": -31.04696273803711,
      "activations/layer12_attention_weight_max": 26.047653198242188,
      "activations/layer12_attention_weight_min": -24.43757438659668,
      "activations/layer13_attention_weight_max": 35.36945724487305,
      "activations/layer13_attention_weight_min": -32.1600456237793,
      "activations/layer14_attention_weight_max": 41.0202522277832,
      "activations/layer14_attention_weight_min": -35.65998458862305,
      "activations/layer15_attention_weight_max": 33.368568420410156,
      "activations/layer15_attention_weight_min": -30.065305709838867,
      "activations/layer16_attention_weight_max": 31.62762451171875,
      "activations/layer16_attention_weight_min": -28.100486755371094,
      "activations/layer17_attention_weight_max": 31.25018310546875,
      "activations/layer17_attention_weight_min": -24.225526809692383,
      "activations/layer18_attention_weight_max": 32.86384582519531,
      "activations/layer18_attention_weight_min": -22.120820999145508,
      "activations/layer19_attention_weight_max": 35.39150619506836,
      "activations/layer19_attention_weight_min": -28.274124145507812,
      "activations/layer1_attention_weight_max": 17.56588363647461,
      "activations/layer1_attention_weight_min": -15.029211044311523,
      "activations/layer20_attention_weight_max": 30.932920455932617,
      "activations/layer20_attention_weight_min": -24.00469398498535,
      "activations/layer21_attention_weight_max": 34.244529724121094,
      "activations/layer21_attention_weight_min": -23.13755226135254,
      "activations/layer22_attention_weight_max": 41.63859939575195,
      "activations/layer22_attention_weight_min": -30.375085830688477,
      "activations/layer23_attention_weight_max": 34.542930603027344,
      "activations/layer23_attention_weight_min": -23.219524383544922,
      "activations/layer2_attention_weight_max": 32.2380256652832,
      "activations/layer2_attention_weight_min": -32.54567337036133,
      "activations/layer3_attention_weight_max": 95.95581817626953,
      "activations/layer3_attention_weight_min": -95.03687286376953,
      "activations/layer4_attention_weight_max": 107.6045150756836,
      "activations/layer4_attention_weight_min": -110.95770263671875,
      "activations/layer5_attention_weight_max": 52.81812286376953,
      "activations/layer5_attention_weight_min": -62.3919792175293,
      "activations/layer6_attention_weight_max": 45.09706497192383,
      "activations/layer6_attention_weight_min": -46.802608489990234,
      "activations/layer7_attention_weight_max": 87.73336029052734,
      "activations/layer7_attention_weight_min": -87.44625091552734,
      "activations/layer8_attention_weight_max": 42.89699172973633,
      "activations/layer8_attention_weight_min": -41.72580337524414,
      "activations/layer9_attention_weight_max": 35.95072937011719,
      "activations/layer9_attention_weight_min": -33.08476638793945,
      "epoch": 10.36,
      "learning_rate": 8.403522727272727e-05,
      "loss": 2.7907,
      "step": 178300
    },
    {
      "activations/layer0_attention_weight_max": 15.875093460083008,
      "activations/layer0_attention_weight_min": -12.842275619506836,
      "activations/layer10_attention_weight_max": 38.71173858642578,
      "activations/layer10_attention_weight_min": -37.590431213378906,
      "activations/layer11_attention_weight_max": 40.64832305908203,
      "activations/layer11_attention_weight_min": -34.919036865234375,
      "activations/layer12_attention_weight_max": 23.660234451293945,
      "activations/layer12_attention_weight_min": -27.66739273071289,
      "activations/layer13_attention_weight_max": 39.02145004272461,
      "activations/layer13_attention_weight_min": -34.371856689453125,
      "activations/layer14_attention_weight_max": 40.62781524658203,
      "activations/layer14_attention_weight_min": -36.96898651123047,
      "activations/layer15_attention_weight_max": 34.42304611206055,
      "activations/layer15_attention_weight_min": -32.047607421875,
      "activations/layer16_attention_weight_max": 27.744396209716797,
      "activations/layer16_attention_weight_min": -27.37636375427246,
      "activations/layer17_attention_weight_max": 30.52814483642578,
      "activations/layer17_attention_weight_min": -24.630329132080078,
      "activations/layer18_attention_weight_max": 31.86969566345215,
      "activations/layer18_attention_weight_min": -21.16236114501953,
      "activations/layer19_attention_weight_max": 30.346771240234375,
      "activations/layer19_attention_weight_min": -31.095203399658203,
      "activations/layer1_attention_weight_max": 16.785371780395508,
      "activations/layer1_attention_weight_min": -14.636096954345703,
      "activations/layer20_attention_weight_max": 26.282115936279297,
      "activations/layer20_attention_weight_min": -24.778564453125,
      "activations/layer21_attention_weight_max": 27.017515182495117,
      "activations/layer21_attention_weight_min": -23.177749633789062,
      "activations/layer22_attention_weight_max": 40.67836380004883,
      "activations/layer22_attention_weight_min": -28.716068267822266,
      "activations/layer23_attention_weight_max": 35.407779693603516,
      "activations/layer23_attention_weight_min": -25.937660217285156,
      "activations/layer2_attention_weight_max": 34.329620361328125,
      "activations/layer2_attention_weight_min": -33.88672637939453,
      "activations/layer3_attention_weight_max": 100.99834442138672,
      "activations/layer3_attention_weight_min": -103.3423843383789,
      "activations/layer4_attention_weight_max": 115.8310775756836,
      "activations/layer4_attention_weight_min": -114.19398498535156,
      "activations/layer5_attention_weight_max": 51.6343994140625,
      "activations/layer5_attention_weight_min": -62.51061248779297,
      "activations/layer6_attention_weight_max": 45.15576934814453,
      "activations/layer6_attention_weight_min": -47.700626373291016,
      "activations/layer7_attention_weight_max": 93.84013366699219,
      "activations/layer7_attention_weight_min": -93.83849334716797,
      "activations/layer8_attention_weight_max": 42.963722229003906,
      "activations/layer8_attention_weight_min": -44.72622299194336,
      "activations/layer9_attention_weight_max": 37.48984909057617,
      "activations/layer9_attention_weight_min": -35.534515380859375,
      "epoch": 10.36,
      "learning_rate": 8.401628787878787e-05,
      "loss": 2.7777,
      "step": 178350
    },
    {
      "activations/layer0_attention_weight_max": 16.238086700439453,
      "activations/layer0_attention_weight_min": -12.382816314697266,
      "activations/layer10_attention_weight_max": 33.220672607421875,
      "activations/layer10_attention_weight_min": -31.58211326599121,
      "activations/layer11_attention_weight_max": 34.19628143310547,
      "activations/layer11_attention_weight_min": -30.98902130126953,
      "activations/layer12_attention_weight_max": 44.16968536376953,
      "activations/layer12_attention_weight_min": -25.125381469726562,
      "activations/layer13_attention_weight_max": 48.006141662597656,
      "activations/layer13_attention_weight_min": -37.2556266784668,
      "activations/layer14_attention_weight_max": 43.4669189453125,
      "activations/layer14_attention_weight_min": -35.9190788269043,
      "activations/layer15_attention_weight_max": 38.383724212646484,
      "activations/layer15_attention_weight_min": -32.02739715576172,
      "activations/layer16_attention_weight_max": 32.679786682128906,
      "activations/layer16_attention_weight_min": -27.530338287353516,
      "activations/layer17_attention_weight_max": 30.74818229675293,
      "activations/layer17_attention_weight_min": -25.45469856262207,
      "activations/layer18_attention_weight_max": 32.70842361450195,
      "activations/layer18_attention_weight_min": -25.724815368652344,
      "activations/layer19_attention_weight_max": 34.87186813354492,
      "activations/layer19_attention_weight_min": -30.25876235961914,
      "activations/layer1_attention_weight_max": 18.123199462890625,
      "activations/layer1_attention_weight_min": -17.196325302124023,
      "activations/layer20_attention_weight_max": 28.296585083007812,
      "activations/layer20_attention_weight_min": -25.593355178833008,
      "activations/layer21_attention_weight_max": 27.950292587280273,
      "activations/layer21_attention_weight_min": -24.82295799255371,
      "activations/layer22_attention_weight_max": 45.3901252746582,
      "activations/layer22_attention_weight_min": -29.685104370117188,
      "activations/layer23_attention_weight_max": 33.731544494628906,
      "activations/layer23_attention_weight_min": -26.27444076538086,
      "activations/layer2_attention_weight_max": 31.59067153930664,
      "activations/layer2_attention_weight_min": -32.76582717895508,
      "activations/layer3_attention_weight_max": 92.49622344970703,
      "activations/layer3_attention_weight_min": -95.906005859375,
      "activations/layer4_attention_weight_max": 110.84991455078125,
      "activations/layer4_attention_weight_min": -114.7891616821289,
      "activations/layer5_attention_weight_max": 51.892086029052734,
      "activations/layer5_attention_weight_min": -60.560115814208984,
      "activations/layer6_attention_weight_max": 44.384517669677734,
      "activations/layer6_attention_weight_min": -48.71220016479492,
      "activations/layer7_attention_weight_max": 88.79827880859375,
      "activations/layer7_attention_weight_min": -94.2367172241211,
      "activations/layer8_attention_weight_max": 40.193687438964844,
      "activations/layer8_attention_weight_min": -44.63649368286133,
      "activations/layer9_attention_weight_max": 34.63549041748047,
      "activations/layer9_attention_weight_min": -33.98765182495117,
      "epoch": 10.37,
      "learning_rate": 8.399734848484847e-05,
      "loss": 2.788,
      "step": 178400
    },
    {
      "activations/layer0_attention_weight_max": 15.386927604675293,
      "activations/layer0_attention_weight_min": -12.904866218566895,
      "activations/layer10_attention_weight_max": 31.471670150756836,
      "activations/layer10_attention_weight_min": -32.256622314453125,
      "activations/layer11_attention_weight_max": 30.929672241210938,
      "activations/layer11_attention_weight_min": -31.142009735107422,
      "activations/layer12_attention_weight_max": 38.8428840637207,
      "activations/layer12_attention_weight_min": -25.437068939208984,
      "activations/layer13_attention_weight_max": 39.229225158691406,
      "activations/layer13_attention_weight_min": -35.09897994995117,
      "activations/layer14_attention_weight_max": 43.62045669555664,
      "activations/layer14_attention_weight_min": -41.87294006347656,
      "activations/layer15_attention_weight_max": 37.03443908691406,
      "activations/layer15_attention_weight_min": -31.78887939453125,
      "activations/layer16_attention_weight_max": 31.479976654052734,
      "activations/layer16_attention_weight_min": -29.58897590637207,
      "activations/layer17_attention_weight_max": 34.54872512817383,
      "activations/layer17_attention_weight_min": -25.687374114990234,
      "activations/layer18_attention_weight_max": 31.626298904418945,
      "activations/layer18_attention_weight_min": -23.370561599731445,
      "activations/layer19_attention_weight_max": 36.04658126831055,
      "activations/layer19_attention_weight_min": -31.50945472717285,
      "activations/layer1_attention_weight_max": 18.314369201660156,
      "activations/layer1_attention_weight_min": -16.203861236572266,
      "activations/layer20_attention_weight_max": 29.50808334350586,
      "activations/layer20_attention_weight_min": -24.81757164001465,
      "activations/layer21_attention_weight_max": 30.15068817138672,
      "activations/layer21_attention_weight_min": -24.966381072998047,
      "activations/layer22_attention_weight_max": 43.751678466796875,
      "activations/layer22_attention_weight_min": -28.57265853881836,
      "activations/layer23_attention_weight_max": 37.46535110473633,
      "activations/layer23_attention_weight_min": -24.20102310180664,
      "activations/layer2_attention_weight_max": 34.73783493041992,
      "activations/layer2_attention_weight_min": -35.485679626464844,
      "activations/layer3_attention_weight_max": 96.08590698242188,
      "activations/layer3_attention_weight_min": -97.06432342529297,
      "activations/layer4_attention_weight_max": 114.74848937988281,
      "activations/layer4_attention_weight_min": -118.20619201660156,
      "activations/layer5_attention_weight_max": 57.513851165771484,
      "activations/layer5_attention_weight_min": -65.07589721679688,
      "activations/layer6_attention_weight_max": 43.6659049987793,
      "activations/layer6_attention_weight_min": -49.100257873535156,
      "activations/layer7_attention_weight_max": 90.37039184570312,
      "activations/layer7_attention_weight_min": -88.70792388916016,
      "activations/layer8_attention_weight_max": 38.99390411376953,
      "activations/layer8_attention_weight_min": -42.28916549682617,
      "activations/layer9_attention_weight_max": 32.2879524230957,
      "activations/layer9_attention_weight_min": -33.28943634033203,
      "epoch": 10.37,
      "learning_rate": 8.397840909090909e-05,
      "loss": 2.7744,
      "step": 178450
    },
    {
      "activations/layer0_attention_weight_max": 16.23183250427246,
      "activations/layer0_attention_weight_min": -12.518288612365723,
      "activations/layer10_attention_weight_max": 35.50572967529297,
      "activations/layer10_attention_weight_min": -36.26621627807617,
      "activations/layer11_attention_weight_max": 36.45340347290039,
      "activations/layer11_attention_weight_min": -36.1741943359375,
      "activations/layer12_attention_weight_max": 24.731990814208984,
      "activations/layer12_attention_weight_min": -25.534854888916016,
      "activations/layer13_attention_weight_max": 38.99294662475586,
      "activations/layer13_attention_weight_min": -36.408958435058594,
      "activations/layer14_attention_weight_max": 42.22376251220703,
      "activations/layer14_attention_weight_min": -38.56511306762695,
      "activations/layer15_attention_weight_max": 36.186546325683594,
      "activations/layer15_attention_weight_min": -32.52101135253906,
      "activations/layer16_attention_weight_max": 31.37288475036621,
      "activations/layer16_attention_weight_min": -29.6287784576416,
      "activations/layer17_attention_weight_max": 32.6326904296875,
      "activations/layer17_attention_weight_min": -26.41860008239746,
      "activations/layer18_attention_weight_max": 33.0489501953125,
      "activations/layer18_attention_weight_min": -23.33767318725586,
      "activations/layer19_attention_weight_max": 36.98592758178711,
      "activations/layer19_attention_weight_min": -32.04067611694336,
      "activations/layer1_attention_weight_max": 16.486997604370117,
      "activations/layer1_attention_weight_min": -13.858572006225586,
      "activations/layer20_attention_weight_max": 32.85124206542969,
      "activations/layer20_attention_weight_min": -27.12179183959961,
      "activations/layer21_attention_weight_max": 35.156883239746094,
      "activations/layer21_attention_weight_min": -24.94944190979004,
      "activations/layer22_attention_weight_max": 44.764366149902344,
      "activations/layer22_attention_weight_min": -29.13666534423828,
      "activations/layer23_attention_weight_max": 37.4299430847168,
      "activations/layer23_attention_weight_min": -24.196266174316406,
      "activations/layer2_attention_weight_max": 33.82041931152344,
      "activations/layer2_attention_weight_min": -33.830108642578125,
      "activations/layer3_attention_weight_max": 102.39444732666016,
      "activations/layer3_attention_weight_min": -101.71479797363281,
      "activations/layer4_attention_weight_max": 118.6646957397461,
      "activations/layer4_attention_weight_min": -119.5776596069336,
      "activations/layer5_attention_weight_max": 54.399566650390625,
      "activations/layer5_attention_weight_min": -64.96459197998047,
      "activations/layer6_attention_weight_max": 47.25253677368164,
      "activations/layer6_attention_weight_min": -49.67906951904297,
      "activations/layer7_attention_weight_max": 92.57841491699219,
      "activations/layer7_attention_weight_min": -95.97537994384766,
      "activations/layer8_attention_weight_max": 41.8662223815918,
      "activations/layer8_attention_weight_min": -41.97779083251953,
      "activations/layer9_attention_weight_max": 38.20103454589844,
      "activations/layer9_attention_weight_min": -36.028926849365234,
      "epoch": 10.37,
      "learning_rate": 8.395946969696969e-05,
      "loss": 2.7912,
      "step": 178500
    },
    {
      "activations/layer0_attention_weight_max": 16.584692001342773,
      "activations/layer0_attention_weight_min": -11.898581504821777,
      "activations/layer10_attention_weight_max": 34.52650833129883,
      "activations/layer10_attention_weight_min": -32.8455696105957,
      "activations/layer11_attention_weight_max": 32.5617790222168,
      "activations/layer11_attention_weight_min": -30.79128074645996,
      "activations/layer12_attention_weight_max": 28.38109016418457,
      "activations/layer12_attention_weight_min": -25.420303344726562,
      "activations/layer13_attention_weight_max": 39.75984573364258,
      "activations/layer13_attention_weight_min": -32.48714828491211,
      "activations/layer14_attention_weight_max": 50.10022735595703,
      "activations/layer14_attention_weight_min": -39.98060989379883,
      "activations/layer15_attention_weight_max": 34.199188232421875,
      "activations/layer15_attention_weight_min": -30.340665817260742,
      "activations/layer16_attention_weight_max": 30.841840744018555,
      "activations/layer16_attention_weight_min": -27.409141540527344,
      "activations/layer17_attention_weight_max": 31.38572883605957,
      "activations/layer17_attention_weight_min": -24.33780860900879,
      "activations/layer18_attention_weight_max": 28.75105094909668,
      "activations/layer18_attention_weight_min": -20.797542572021484,
      "activations/layer19_attention_weight_max": 37.11904525756836,
      "activations/layer19_attention_weight_min": -30.04835319519043,
      "activations/layer1_attention_weight_max": 17.042076110839844,
      "activations/layer1_attention_weight_min": -16.253131866455078,
      "activations/layer20_attention_weight_max": 30.77239227294922,
      "activations/layer20_attention_weight_min": -24.433670043945312,
      "activations/layer21_attention_weight_max": 30.753543853759766,
      "activations/layer21_attention_weight_min": -22.604476928710938,
      "activations/layer22_attention_weight_max": 44.54880142211914,
      "activations/layer22_attention_weight_min": -28.107454299926758,
      "activations/layer23_attention_weight_max": 36.80813980102539,
      "activations/layer23_attention_weight_min": -23.14375877380371,
      "activations/layer2_attention_weight_max": 33.478614807128906,
      "activations/layer2_attention_weight_min": -35.7970085144043,
      "activations/layer3_attention_weight_max": 99.05242919921875,
      "activations/layer3_attention_weight_min": -98.17823028564453,
      "activations/layer4_attention_weight_max": 123.89759826660156,
      "activations/layer4_attention_weight_min": -117.18843078613281,
      "activations/layer5_attention_weight_max": 56.80381393432617,
      "activations/layer5_attention_weight_min": -64.06617736816406,
      "activations/layer6_attention_weight_max": 47.01335144042969,
      "activations/layer6_attention_weight_min": -47.44942092895508,
      "activations/layer7_attention_weight_max": 89.06205749511719,
      "activations/layer7_attention_weight_min": -94.91500091552734,
      "activations/layer8_attention_weight_max": 40.96625518798828,
      "activations/layer8_attention_weight_min": -40.56365203857422,
      "activations/layer9_attention_weight_max": 33.9677619934082,
      "activations/layer9_attention_weight_min": -32.9870719909668,
      "epoch": 10.37,
      "learning_rate": 8.39405303030303e-05,
      "loss": 2.7902,
      "step": 178550
    },
    {
      "activations/layer0_attention_weight_max": 15.650279998779297,
      "activations/layer0_attention_weight_min": -12.372184753417969,
      "activations/layer10_attention_weight_max": 35.97563552856445,
      "activations/layer10_attention_weight_min": -33.2700309753418,
      "activations/layer11_attention_weight_max": 40.45942687988281,
      "activations/layer11_attention_weight_min": -38.62086486816406,
      "activations/layer12_attention_weight_max": 25.740495681762695,
      "activations/layer12_attention_weight_min": -28.606348037719727,
      "activations/layer13_attention_weight_max": 42.659767150878906,
      "activations/layer13_attention_weight_min": -35.1029167175293,
      "activations/layer14_attention_weight_max": 52.05364990234375,
      "activations/layer14_attention_weight_min": -40.499122619628906,
      "activations/layer15_attention_weight_max": 43.816165924072266,
      "activations/layer15_attention_weight_min": -33.545536041259766,
      "activations/layer16_attention_weight_max": 35.52914047241211,
      "activations/layer16_attention_weight_min": -28.15391731262207,
      "activations/layer17_attention_weight_max": 31.842153549194336,
      "activations/layer17_attention_weight_min": -26.070768356323242,
      "activations/layer18_attention_weight_max": 31.35955810546875,
      "activations/layer18_attention_weight_min": -22.185014724731445,
      "activations/layer19_attention_weight_max": 37.146018981933594,
      "activations/layer19_attention_weight_min": -30.69510269165039,
      "activations/layer1_attention_weight_max": 17.289928436279297,
      "activations/layer1_attention_weight_min": -14.15890121459961,
      "activations/layer20_attention_weight_max": 29.647167205810547,
      "activations/layer20_attention_weight_min": -23.16766357421875,
      "activations/layer21_attention_weight_max": 30.19542694091797,
      "activations/layer21_attention_weight_min": -23.74118423461914,
      "activations/layer22_attention_weight_max": 46.85542678833008,
      "activations/layer22_attention_weight_min": -28.490434646606445,
      "activations/layer23_attention_weight_max": 36.34193801879883,
      "activations/layer23_attention_weight_min": -24.804548263549805,
      "activations/layer2_attention_weight_max": 34.03666687011719,
      "activations/layer2_attention_weight_min": -35.89324951171875,
      "activations/layer3_attention_weight_max": 97.96669006347656,
      "activations/layer3_attention_weight_min": -100.55777740478516,
      "activations/layer4_attention_weight_max": 116.25943756103516,
      "activations/layer4_attention_weight_min": -108.97877502441406,
      "activations/layer5_attention_weight_max": 57.204505920410156,
      "activations/layer5_attention_weight_min": -61.20606231689453,
      "activations/layer6_attention_weight_max": 44.86652374267578,
      "activations/layer6_attention_weight_min": -47.2952766418457,
      "activations/layer7_attention_weight_max": 97.06373596191406,
      "activations/layer7_attention_weight_min": -102.32344055175781,
      "activations/layer8_attention_weight_max": 41.147071838378906,
      "activations/layer8_attention_weight_min": -42.80913162231445,
      "activations/layer9_attention_weight_max": 34.38645935058594,
      "activations/layer9_attention_weight_min": -35.22964859008789,
      "epoch": 10.38,
      "learning_rate": 8.39215909090909e-05,
      "loss": 2.7736,
      "step": 178600
    },
    {
      "activations/layer0_attention_weight_max": 16.177452087402344,
      "activations/layer0_attention_weight_min": -12.470637321472168,
      "activations/layer10_attention_weight_max": 32.60642623901367,
      "activations/layer10_attention_weight_min": -33.1760139465332,
      "activations/layer11_attention_weight_max": 32.37517547607422,
      "activations/layer11_attention_weight_min": -31.28969383239746,
      "activations/layer12_attention_weight_max": 23.781654357910156,
      "activations/layer12_attention_weight_min": -27.011295318603516,
      "activations/layer13_attention_weight_max": 38.583251953125,
      "activations/layer13_attention_weight_min": -37.41334915161133,
      "activations/layer14_attention_weight_max": 46.30248260498047,
      "activations/layer14_attention_weight_min": -37.927799224853516,
      "activations/layer15_attention_weight_max": 35.01686096191406,
      "activations/layer15_attention_weight_min": -31.65320587158203,
      "activations/layer16_attention_weight_max": 31.333539962768555,
      "activations/layer16_attention_weight_min": -27.884931564331055,
      "activations/layer17_attention_weight_max": 33.12625503540039,
      "activations/layer17_attention_weight_min": -25.651166915893555,
      "activations/layer18_attention_weight_max": 31.02529525756836,
      "activations/layer18_attention_weight_min": -23.871339797973633,
      "activations/layer19_attention_weight_max": 35.76216506958008,
      "activations/layer19_attention_weight_min": -32.02313995361328,
      "activations/layer1_attention_weight_max": 17.265512466430664,
      "activations/layer1_attention_weight_min": -14.766095161437988,
      "activations/layer20_attention_weight_max": 30.976694107055664,
      "activations/layer20_attention_weight_min": -24.86572265625,
      "activations/layer21_attention_weight_max": 30.077831268310547,
      "activations/layer21_attention_weight_min": -26.301815032958984,
      "activations/layer22_attention_weight_max": 43.43916320800781,
      "activations/layer22_attention_weight_min": -30.88688850402832,
      "activations/layer23_attention_weight_max": 34.01737594604492,
      "activations/layer23_attention_weight_min": -26.39444351196289,
      "activations/layer2_attention_weight_max": 33.33550262451172,
      "activations/layer2_attention_weight_min": -33.24602127075195,
      "activations/layer3_attention_weight_max": 94.962646484375,
      "activations/layer3_attention_weight_min": -97.73455047607422,
      "activations/layer4_attention_weight_max": 114.5641098022461,
      "activations/layer4_attention_weight_min": -114.00276947021484,
      "activations/layer5_attention_weight_max": 55.15080261230469,
      "activations/layer5_attention_weight_min": -61.122920989990234,
      "activations/layer6_attention_weight_max": 45.57444763183594,
      "activations/layer6_attention_weight_min": -49.29185485839844,
      "activations/layer7_attention_weight_max": 92.29447174072266,
      "activations/layer7_attention_weight_min": -89.47388458251953,
      "activations/layer8_attention_weight_max": 40.12847900390625,
      "activations/layer8_attention_weight_min": -41.422767639160156,
      "activations/layer9_attention_weight_max": 33.03889083862305,
      "activations/layer9_attention_weight_min": -33.328956604003906,
      "epoch": 10.38,
      "learning_rate": 8.39026515151515e-05,
      "loss": 2.7808,
      "step": 178650
    },
    {
      "activations/layer0_attention_weight_max": 16.23615074157715,
      "activations/layer0_attention_weight_min": -11.51058578491211,
      "activations/layer10_attention_weight_max": 30.919677734375,
      "activations/layer10_attention_weight_min": -33.30651092529297,
      "activations/layer11_attention_weight_max": 30.88380241394043,
      "activations/layer11_attention_weight_min": -31.817108154296875,
      "activations/layer12_attention_weight_max": 35.7507209777832,
      "activations/layer12_attention_weight_min": -25.071670532226562,
      "activations/layer13_attention_weight_max": 49.828975677490234,
      "activations/layer13_attention_weight_min": -34.37058639526367,
      "activations/layer14_attention_weight_max": 51.5097770690918,
      "activations/layer14_attention_weight_min": -40.026676177978516,
      "activations/layer15_attention_weight_max": 40.75252914428711,
      "activations/layer15_attention_weight_min": -33.52076721191406,
      "activations/layer16_attention_weight_max": 36.979156494140625,
      "activations/layer16_attention_weight_min": -30.112810134887695,
      "activations/layer17_attention_weight_max": 37.66374588012695,
      "activations/layer17_attention_weight_min": -26.182422637939453,
      "activations/layer18_attention_weight_max": 34.384464263916016,
      "activations/layer18_attention_weight_min": -23.5018310546875,
      "activations/layer19_attention_weight_max": 40.672462463378906,
      "activations/layer19_attention_weight_min": -31.40814208984375,
      "activations/layer1_attention_weight_max": 17.207853317260742,
      "activations/layer1_attention_weight_min": -14.238800048828125,
      "activations/layer20_attention_weight_max": 37.03929138183594,
      "activations/layer20_attention_weight_min": -25.06649398803711,
      "activations/layer21_attention_weight_max": 35.21454620361328,
      "activations/layer21_attention_weight_min": -24.2734432220459,
      "activations/layer22_attention_weight_max": 46.6227912902832,
      "activations/layer22_attention_weight_min": -30.84273910522461,
      "activations/layer23_attention_weight_max": 33.7103271484375,
      "activations/layer23_attention_weight_min": -25.980955123901367,
      "activations/layer2_attention_weight_max": 31.187406539916992,
      "activations/layer2_attention_weight_min": -32.725887298583984,
      "activations/layer3_attention_weight_max": 90.51448822021484,
      "activations/layer3_attention_weight_min": -96.90386962890625,
      "activations/layer4_attention_weight_max": 110.42305755615234,
      "activations/layer4_attention_weight_min": -111.72444915771484,
      "activations/layer5_attention_weight_max": 50.93921661376953,
      "activations/layer5_attention_weight_min": -60.215728759765625,
      "activations/layer6_attention_weight_max": 43.89844512939453,
      "activations/layer6_attention_weight_min": -48.68378448486328,
      "activations/layer7_attention_weight_max": 89.39742279052734,
      "activations/layer7_attention_weight_min": -93.38841247558594,
      "activations/layer8_attention_weight_max": 38.753440856933594,
      "activations/layer8_attention_weight_min": -44.68196105957031,
      "activations/layer9_attention_weight_max": 31.03004264831543,
      "activations/layer9_attention_weight_min": -33.25035858154297,
      "epoch": 10.38,
      "learning_rate": 8.388371212121212e-05,
      "loss": 2.8055,
      "step": 178700
    },
    {
      "activations/layer0_attention_weight_max": 15.275151252746582,
      "activations/layer0_attention_weight_min": -12.312395095825195,
      "activations/layer10_attention_weight_max": 30.06682014465332,
      "activations/layer10_attention_weight_min": -30.777738571166992,
      "activations/layer11_attention_weight_max": 31.855993270874023,
      "activations/layer11_attention_weight_min": -32.35443878173828,
      "activations/layer12_attention_weight_max": 22.766040802001953,
      "activations/layer12_attention_weight_min": -26.08204460144043,
      "activations/layer13_attention_weight_max": 35.9849967956543,
      "activations/layer13_attention_weight_min": -32.8700065612793,
      "activations/layer14_attention_weight_max": 38.32323455810547,
      "activations/layer14_attention_weight_min": -36.44715881347656,
      "activations/layer15_attention_weight_max": 32.95481491088867,
      "activations/layer15_attention_weight_min": -30.903295516967773,
      "activations/layer16_attention_weight_max": 29.7865047454834,
      "activations/layer16_attention_weight_min": -27.165199279785156,
      "activations/layer17_attention_weight_max": 30.275272369384766,
      "activations/layer17_attention_weight_min": -25.367469787597656,
      "activations/layer18_attention_weight_max": 31.642187118530273,
      "activations/layer18_attention_weight_min": -24.458209991455078,
      "activations/layer19_attention_weight_max": 34.31403732299805,
      "activations/layer19_attention_weight_min": -30.710752487182617,
      "activations/layer1_attention_weight_max": 16.176387786865234,
      "activations/layer1_attention_weight_min": -14.506982803344727,
      "activations/layer20_attention_weight_max": 29.220678329467773,
      "activations/layer20_attention_weight_min": -24.12813949584961,
      "activations/layer21_attention_weight_max": 33.4736328125,
      "activations/layer21_attention_weight_min": -23.61437225341797,
      "activations/layer22_attention_weight_max": 42.95045852661133,
      "activations/layer22_attention_weight_min": -30.65980339050293,
      "activations/layer23_attention_weight_max": 32.19517517089844,
      "activations/layer23_attention_weight_min": -23.12763214111328,
      "activations/layer2_attention_weight_max": 31.516254425048828,
      "activations/layer2_attention_weight_min": -32.86259841918945,
      "activations/layer3_attention_weight_max": 93.07659912109375,
      "activations/layer3_attention_weight_min": -93.21189880371094,
      "activations/layer4_attention_weight_max": 111.65836334228516,
      "activations/layer4_attention_weight_min": -108.4527359008789,
      "activations/layer5_attention_weight_max": 50.67879867553711,
      "activations/layer5_attention_weight_min": -59.2711296081543,
      "activations/layer6_attention_weight_max": 42.70502471923828,
      "activations/layer6_attention_weight_min": -45.97898864746094,
      "activations/layer7_attention_weight_max": 86.0516357421875,
      "activations/layer7_attention_weight_min": -82.26248931884766,
      "activations/layer8_attention_weight_max": 36.46990203857422,
      "activations/layer8_attention_weight_min": -39.30712890625,
      "activations/layer9_attention_weight_max": 31.740432739257812,
      "activations/layer9_attention_weight_min": -32.89202880859375,
      "epoch": 10.39,
      "learning_rate": 8.386477272727272e-05,
      "loss": 2.7815,
      "step": 178750
    },
    {
      "activations/layer0_attention_weight_max": 15.671503067016602,
      "activations/layer0_attention_weight_min": -12.0999755859375,
      "activations/layer10_attention_weight_max": 32.966678619384766,
      "activations/layer10_attention_weight_min": -32.34541702270508,
      "activations/layer11_attention_weight_max": 33.38666534423828,
      "activations/layer11_attention_weight_min": -32.28956604003906,
      "activations/layer12_attention_weight_max": 24.384315490722656,
      "activations/layer12_attention_weight_min": -26.3702335357666,
      "activations/layer13_attention_weight_max": 40.164608001708984,
      "activations/layer13_attention_weight_min": -34.06809997558594,
      "activations/layer14_attention_weight_max": 47.71091079711914,
      "activations/layer14_attention_weight_min": -38.31794738769531,
      "activations/layer15_attention_weight_max": 37.95747756958008,
      "activations/layer15_attention_weight_min": -33.0270881652832,
      "activations/layer16_attention_weight_max": 31.796293258666992,
      "activations/layer16_attention_weight_min": -30.672557830810547,
      "activations/layer17_attention_weight_max": 34.06570816040039,
      "activations/layer17_attention_weight_min": -26.428186416625977,
      "activations/layer18_attention_weight_max": 38.192474365234375,
      "activations/layer18_attention_weight_min": -23.243389129638672,
      "activations/layer19_attention_weight_max": 35.19868087768555,
      "activations/layer19_attention_weight_min": -30.643497467041016,
      "activations/layer1_attention_weight_max": 17.185068130493164,
      "activations/layer1_attention_weight_min": -14.079482078552246,
      "activations/layer20_attention_weight_max": 33.516448974609375,
      "activations/layer20_attention_weight_min": -24.559247970581055,
      "activations/layer21_attention_weight_max": 34.57108688354492,
      "activations/layer21_attention_weight_min": -24.759145736694336,
      "activations/layer22_attention_weight_max": 48.351402282714844,
      "activations/layer22_attention_weight_min": -31.18729019165039,
      "activations/layer23_attention_weight_max": 37.806541442871094,
      "activations/layer23_attention_weight_min": -26.224605560302734,
      "activations/layer2_attention_weight_max": 33.884986877441406,
      "activations/layer2_attention_weight_min": -32.093177795410156,
      "activations/layer3_attention_weight_max": 97.74092864990234,
      "activations/layer3_attention_weight_min": -95.80298614501953,
      "activations/layer4_attention_weight_max": 114.7669677734375,
      "activations/layer4_attention_weight_min": -119.17302703857422,
      "activations/layer5_attention_weight_max": 54.196495056152344,
      "activations/layer5_attention_weight_min": -67.46720123291016,
      "activations/layer6_attention_weight_max": 46.773277282714844,
      "activations/layer6_attention_weight_min": -48.55807876586914,
      "activations/layer7_attention_weight_max": 91.416015625,
      "activations/layer7_attention_weight_min": -87.84606170654297,
      "activations/layer8_attention_weight_max": 40.855289459228516,
      "activations/layer8_attention_weight_min": -40.95118713378906,
      "activations/layer9_attention_weight_max": 32.79552459716797,
      "activations/layer9_attention_weight_min": -32.98624038696289,
      "epoch": 10.39,
      "learning_rate": 8.384583333333332e-05,
      "loss": 2.7829,
      "step": 178800
    },
    {
      "activations/layer0_attention_weight_max": 16.127288818359375,
      "activations/layer0_attention_weight_min": -11.379413604736328,
      "activations/layer10_attention_weight_max": 36.462913513183594,
      "activations/layer10_attention_weight_min": -32.136131286621094,
      "activations/layer11_attention_weight_max": 37.8414421081543,
      "activations/layer11_attention_weight_min": -33.6586799621582,
      "activations/layer12_attention_weight_max": 25.910234451293945,
      "activations/layer12_attention_weight_min": -25.545345306396484,
      "activations/layer13_attention_weight_max": 41.68452453613281,
      "activations/layer13_attention_weight_min": -34.534996032714844,
      "activations/layer14_attention_weight_max": 41.78609848022461,
      "activations/layer14_attention_weight_min": -36.95423889160156,
      "activations/layer15_attention_weight_max": 38.80666732788086,
      "activations/layer15_attention_weight_min": -30.265527725219727,
      "activations/layer16_attention_weight_max": 37.46621322631836,
      "activations/layer16_attention_weight_min": -26.967466354370117,
      "activations/layer17_attention_weight_max": 35.29184341430664,
      "activations/layer17_attention_weight_min": -25.809083938598633,
      "activations/layer18_attention_weight_max": 30.552669525146484,
      "activations/layer18_attention_weight_min": -22.70669174194336,
      "activations/layer19_attention_weight_max": 36.00408935546875,
      "activations/layer19_attention_weight_min": -29.580646514892578,
      "activations/layer1_attention_weight_max": 17.135263442993164,
      "activations/layer1_attention_weight_min": -13.818806648254395,
      "activations/layer20_attention_weight_max": 31.282453536987305,
      "activations/layer20_attention_weight_min": -24.015302658081055,
      "activations/layer21_attention_weight_max": 31.18771743774414,
      "activations/layer21_attention_weight_min": -23.771718978881836,
      "activations/layer22_attention_weight_max": 48.62082290649414,
      "activations/layer22_attention_weight_min": -30.40542984008789,
      "activations/layer23_attention_weight_max": 36.91496658325195,
      "activations/layer23_attention_weight_min": -22.923940658569336,
      "activations/layer2_attention_weight_max": 34.652366638183594,
      "activations/layer2_attention_weight_min": -34.70457077026367,
      "activations/layer3_attention_weight_max": 98.34825897216797,
      "activations/layer3_attention_weight_min": -100.29694366455078,
      "activations/layer4_attention_weight_max": 114.18009185791016,
      "activations/layer4_attention_weight_min": -116.12139129638672,
      "activations/layer5_attention_weight_max": 54.52822494506836,
      "activations/layer5_attention_weight_min": -62.86216354370117,
      "activations/layer6_attention_weight_max": 43.414432525634766,
      "activations/layer6_attention_weight_min": -49.22509002685547,
      "activations/layer7_attention_weight_max": 99.03374481201172,
      "activations/layer7_attention_weight_min": -94.69144439697266,
      "activations/layer8_attention_weight_max": 42.93674087524414,
      "activations/layer8_attention_weight_min": -42.6429328918457,
      "activations/layer9_attention_weight_max": 36.77341079711914,
      "activations/layer9_attention_weight_min": -33.33027648925781,
      "epoch": 10.39,
      "learning_rate": 8.382689393939394e-05,
      "loss": 2.7964,
      "step": 178850
    },
    {
      "activations/layer0_attention_weight_max": 16.826068878173828,
      "activations/layer0_attention_weight_min": -11.711009979248047,
      "activations/layer10_attention_weight_max": 37.73036575317383,
      "activations/layer10_attention_weight_min": -35.15068435668945,
      "activations/layer11_attention_weight_max": 38.5190315246582,
      "activations/layer11_attention_weight_min": -33.91581344604492,
      "activations/layer12_attention_weight_max": 33.752220153808594,
      "activations/layer12_attention_weight_min": -30.65862464904785,
      "activations/layer13_attention_weight_max": 45.333560943603516,
      "activations/layer13_attention_weight_min": -36.010955810546875,
      "activations/layer14_attention_weight_max": 57.44138717651367,
      "activations/layer14_attention_weight_min": -43.06140899658203,
      "activations/layer15_attention_weight_max": 42.721588134765625,
      "activations/layer15_attention_weight_min": -35.77355194091797,
      "activations/layer16_attention_weight_max": 37.133033752441406,
      "activations/layer16_attention_weight_min": -29.669185638427734,
      "activations/layer17_attention_weight_max": 35.57627487182617,
      "activations/layer17_attention_weight_min": -25.04522705078125,
      "activations/layer18_attention_weight_max": 36.329891204833984,
      "activations/layer18_attention_weight_min": -23.76194953918457,
      "activations/layer19_attention_weight_max": 38.87741470336914,
      "activations/layer19_attention_weight_min": -29.293685913085938,
      "activations/layer1_attention_weight_max": 16.676021575927734,
      "activations/layer1_attention_weight_min": -13.696535110473633,
      "activations/layer20_attention_weight_max": 32.3421630859375,
      "activations/layer20_attention_weight_min": -23.566959381103516,
      "activations/layer21_attention_weight_max": 31.175064086914062,
      "activations/layer21_attention_weight_min": -22.8806095123291,
      "activations/layer22_attention_weight_max": 52.40293884277344,
      "activations/layer22_attention_weight_min": -28.84385871887207,
      "activations/layer23_attention_weight_max": 40.24192428588867,
      "activations/layer23_attention_weight_min": -25.452289581298828,
      "activations/layer2_attention_weight_max": 34.22182846069336,
      "activations/layer2_attention_weight_min": -35.45476531982422,
      "activations/layer3_attention_weight_max": 96.88275909423828,
      "activations/layer3_attention_weight_min": -97.353759765625,
      "activations/layer4_attention_weight_max": 117.32128143310547,
      "activations/layer4_attention_weight_min": -115.13750457763672,
      "activations/layer5_attention_weight_max": 53.99451446533203,
      "activations/layer5_attention_weight_min": -60.6368408203125,
      "activations/layer6_attention_weight_max": 44.78254318237305,
      "activations/layer6_attention_weight_min": -47.2396354675293,
      "activations/layer7_attention_weight_max": 98.03076934814453,
      "activations/layer7_attention_weight_min": -91.05756378173828,
      "activations/layer8_attention_weight_max": 41.19058609008789,
      "activations/layer8_attention_weight_min": -41.884796142578125,
      "activations/layer9_attention_weight_max": 38.64689254760742,
      "activations/layer9_attention_weight_min": -33.64162826538086,
      "epoch": 10.4,
      "learning_rate": 8.380795454545453e-05,
      "loss": 2.8009,
      "step": 178900
    },
    {
      "activations/layer0_attention_weight_max": 16.08027458190918,
      "activations/layer0_attention_weight_min": -13.25838565826416,
      "activations/layer10_attention_weight_max": 35.33245849609375,
      "activations/layer10_attention_weight_min": -34.0548095703125,
      "activations/layer11_attention_weight_max": 38.935630798339844,
      "activations/layer11_attention_weight_min": -34.87013244628906,
      "activations/layer12_attention_weight_max": 38.00349426269531,
      "activations/layer12_attention_weight_min": -26.421693801879883,
      "activations/layer13_attention_weight_max": 46.93405532836914,
      "activations/layer13_attention_weight_min": -34.72397994995117,
      "activations/layer14_attention_weight_max": 52.43870162963867,
      "activations/layer14_attention_weight_min": -38.344390869140625,
      "activations/layer15_attention_weight_max": 45.637210845947266,
      "activations/layer15_attention_weight_min": -36.314029693603516,
      "activations/layer16_attention_weight_max": 37.26400375366211,
      "activations/layer16_attention_weight_min": -29.351078033447266,
      "activations/layer17_attention_weight_max": 33.698036193847656,
      "activations/layer17_attention_weight_min": -27.229583740234375,
      "activations/layer18_attention_weight_max": 31.284042358398438,
      "activations/layer18_attention_weight_min": -23.190488815307617,
      "activations/layer19_attention_weight_max": 37.97468185424805,
      "activations/layer19_attention_weight_min": -31.602426528930664,
      "activations/layer1_attention_weight_max": 16.836061477661133,
      "activations/layer1_attention_weight_min": -15.321758270263672,
      "activations/layer20_attention_weight_max": 37.864952087402344,
      "activations/layer20_attention_weight_min": -27.169536590576172,
      "activations/layer21_attention_weight_max": 34.279563903808594,
      "activations/layer21_attention_weight_min": -24.591262817382812,
      "activations/layer22_attention_weight_max": 46.723655700683594,
      "activations/layer22_attention_weight_min": -29.973115921020508,
      "activations/layer23_attention_weight_max": 34.740169525146484,
      "activations/layer23_attention_weight_min": -24.368297576904297,
      "activations/layer2_attention_weight_max": 36.19866943359375,
      "activations/layer2_attention_weight_min": -34.16456985473633,
      "activations/layer3_attention_weight_max": 96.19979095458984,
      "activations/layer3_attention_weight_min": -95.42633819580078,
      "activations/layer4_attention_weight_max": 112.02066802978516,
      "activations/layer4_attention_weight_min": -118.69313049316406,
      "activations/layer5_attention_weight_max": 52.70906066894531,
      "activations/layer5_attention_weight_min": -61.27860641479492,
      "activations/layer6_attention_weight_max": 45.84783172607422,
      "activations/layer6_attention_weight_min": -49.83738327026367,
      "activations/layer7_attention_weight_max": 93.27445983886719,
      "activations/layer7_attention_weight_min": -90.90736389160156,
      "activations/layer8_attention_weight_max": 44.78364562988281,
      "activations/layer8_attention_weight_min": -43.75341796875,
      "activations/layer9_attention_weight_max": 36.53119659423828,
      "activations/layer9_attention_weight_min": -36.69524002075195,
      "epoch": 10.4,
      "learning_rate": 8.378901515151514e-05,
      "loss": 2.8033,
      "step": 178950
    },
    {
      "activations/layer0_attention_weight_max": 16.445764541625977,
      "activations/layer0_attention_weight_min": -12.53857135772705,
      "activations/layer10_attention_weight_max": 47.17673873901367,
      "activations/layer10_attention_weight_min": -43.03512954711914,
      "activations/layer11_attention_weight_max": 48.13227462768555,
      "activations/layer11_attention_weight_min": -44.55669403076172,
      "activations/layer12_attention_weight_max": 29.64406967163086,
      "activations/layer12_attention_weight_min": -26.23552703857422,
      "activations/layer13_attention_weight_max": 47.068477630615234,
      "activations/layer13_attention_weight_min": -33.11115646362305,
      "activations/layer14_attention_weight_max": 53.582088470458984,
      "activations/layer14_attention_weight_min": -41.4695930480957,
      "activations/layer15_attention_weight_max": 54.21686935424805,
      "activations/layer15_attention_weight_min": -35.2253303527832,
      "activations/layer16_attention_weight_max": 31.554920196533203,
      "activations/layer16_attention_weight_min": -27.82056427001953,
      "activations/layer17_attention_weight_max": 31.132211685180664,
      "activations/layer17_attention_weight_min": -27.293745040893555,
      "activations/layer18_attention_weight_max": 28.934823989868164,
      "activations/layer18_attention_weight_min": -25.513608932495117,
      "activations/layer19_attention_weight_max": 34.11092758178711,
      "activations/layer19_attention_weight_min": -31.304264068603516,
      "activations/layer1_attention_weight_max": 17.73520851135254,
      "activations/layer1_attention_weight_min": -14.28145694732666,
      "activations/layer20_attention_weight_max": 27.486970901489258,
      "activations/layer20_attention_weight_min": -24.91615867614746,
      "activations/layer21_attention_weight_max": 27.163650512695312,
      "activations/layer21_attention_weight_min": -23.797306060791016,
      "activations/layer22_attention_weight_max": 49.91225814819336,
      "activations/layer22_attention_weight_min": -30.979639053344727,
      "activations/layer23_attention_weight_max": 33.35789489746094,
      "activations/layer23_attention_weight_min": -24.16993522644043,
      "activations/layer2_attention_weight_max": 36.456382751464844,
      "activations/layer2_attention_weight_min": -37.13584899902344,
      "activations/layer3_attention_weight_max": 108.34330749511719,
      "activations/layer3_attention_weight_min": -108.19486236572266,
      "activations/layer4_attention_weight_max": 131.24618530273438,
      "activations/layer4_attention_weight_min": -119.11444091796875,
      "activations/layer5_attention_weight_max": 56.38282775878906,
      "activations/layer5_attention_weight_min": -60.95783233642578,
      "activations/layer6_attention_weight_max": 48.77626037597656,
      "activations/layer6_attention_weight_min": -47.33225631713867,
      "activations/layer7_attention_weight_max": 111.11390686035156,
      "activations/layer7_attention_weight_min": -96.32469940185547,
      "activations/layer8_attention_weight_max": 53.06744384765625,
      "activations/layer8_attention_weight_min": -50.05329895019531,
      "activations/layer9_attention_weight_max": 43.45469665527344,
      "activations/layer9_attention_weight_min": -40.57882308959961,
      "epoch": 10.4,
      "learning_rate": 8.377007575757576e-05,
      "loss": 2.7881,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_loss": 2.73828125,
      "eval_runtime": 8.4578,
      "eval_samples_per_second": 507.696,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_openwebtext_loss": 2.73828125,
      "eval_openwebtext_ppl": 15.460389702669161,
      "eval_openwebtext_runtime": 8.4578,
      "eval_openwebtext_samples_per_second": 507.696,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_wikitext_loss": 2.9609375,
      "eval_wikitext_ppl": 19.31607208722833,
      "eval_wikitext_runtime": 2.0204,
      "eval_wikitext_samples_per_second": 225.698,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_lambada_loss": 2.578125,
      "eval_lambada_ppl": 13.172416709211404,
      "eval_lambada_runtime": 9.5279,
      "eval_lambada_samples_per_second": 511.024,
      "step": 179000
    },
    {
      "activations/layer0_attention_weight_max": 15.475565910339355,
      "activations/layer0_attention_weight_min": -12.99435806274414,
      "activations/layer10_attention_weight_max": 35.73697280883789,
      "activations/layer10_attention_weight_min": -32.97599411010742,
      "activations/layer11_attention_weight_max": 38.33871841430664,
      "activations/layer11_attention_weight_min": -36.380577087402344,
      "activations/layer12_attention_weight_max": 50.57461929321289,
      "activations/layer12_attention_weight_min": -33.16570281982422,
      "activations/layer13_attention_weight_max": 60.574180603027344,
      "activations/layer13_attention_weight_min": -33.057525634765625,
      "activations/layer14_attention_weight_max": 58.85033416748047,
      "activations/layer14_attention_weight_min": -37.07607650756836,
      "activations/layer15_attention_weight_max": 59.908992767333984,
      "activations/layer15_attention_weight_min": -32.6917839050293,
      "activations/layer16_attention_weight_max": 35.05158996582031,
      "activations/layer16_attention_weight_min": -27.662918090820312,
      "activations/layer17_attention_weight_max": 37.504051208496094,
      "activations/layer17_attention_weight_min": -26.64521598815918,
      "activations/layer18_attention_weight_max": 32.20834732055664,
      "activations/layer18_attention_weight_min": -24.089946746826172,
      "activations/layer19_attention_weight_max": 38.83300018310547,
      "activations/layer19_attention_weight_min": -31.442615509033203,
      "activations/layer1_attention_weight_max": 16.78202247619629,
      "activations/layer1_attention_weight_min": -14.188655853271484,
      "activations/layer20_attention_weight_max": 31.66795539855957,
      "activations/layer20_attention_weight_min": -25.916574478149414,
      "activations/layer21_attention_weight_max": 29.91757583618164,
      "activations/layer21_attention_weight_min": -24.08746337890625,
      "activations/layer22_attention_weight_max": 48.65602111816406,
      "activations/layer22_attention_weight_min": -29.213348388671875,
      "activations/layer23_attention_weight_max": 36.773868560791016,
      "activations/layer23_attention_weight_min": -24.537124633789062,
      "activations/layer2_attention_weight_max": 32.800575256347656,
      "activations/layer2_attention_weight_min": -33.555992126464844,
      "activations/layer3_attention_weight_max": 95.59208679199219,
      "activations/layer3_attention_weight_min": -98.7996826171875,
      "activations/layer4_attention_weight_max": 114.57561492919922,
      "activations/layer4_attention_weight_min": -111.68306732177734,
      "activations/layer5_attention_weight_max": 58.84586715698242,
      "activations/layer5_attention_weight_min": -64.99531555175781,
      "activations/layer6_attention_weight_max": 46.813167572021484,
      "activations/layer6_attention_weight_min": -47.90532302856445,
      "activations/layer7_attention_weight_max": 99.63688659667969,
      "activations/layer7_attention_weight_min": -89.55516052246094,
      "activations/layer8_attention_weight_max": 42.33454132080078,
      "activations/layer8_attention_weight_min": -43.06502151489258,
      "activations/layer9_attention_weight_max": 36.11480712890625,
      "activations/layer9_attention_weight_min": -34.16093063354492,
      "epoch": 10.4,
      "learning_rate": 8.375113636363636e-05,
      "loss": 2.7865,
      "step": 179050
    },
    {
      "activations/layer0_attention_weight_max": 15.12597370147705,
      "activations/layer0_attention_weight_min": -13.108023643493652,
      "activations/layer10_attention_weight_max": 31.218765258789062,
      "activations/layer10_attention_weight_min": -34.38351058959961,
      "activations/layer11_attention_weight_max": 32.22734069824219,
      "activations/layer11_attention_weight_min": -33.724082946777344,
      "activations/layer12_attention_weight_max": 24.343921661376953,
      "activations/layer12_attention_weight_min": -23.4925594329834,
      "activations/layer13_attention_weight_max": 38.15934371948242,
      "activations/layer13_attention_weight_min": -32.489906311035156,
      "activations/layer14_attention_weight_max": 49.507381439208984,
      "activations/layer14_attention_weight_min": -39.94276809692383,
      "activations/layer15_attention_weight_max": 35.407981872558594,
      "activations/layer15_attention_weight_min": -31.553220748901367,
      "activations/layer16_attention_weight_max": 31.973461151123047,
      "activations/layer16_attention_weight_min": -27.652362823486328,
      "activations/layer17_attention_weight_max": 34.8799934387207,
      "activations/layer17_attention_weight_min": -25.957523345947266,
      "activations/layer18_attention_weight_max": 33.81463623046875,
      "activations/layer18_attention_weight_min": -24.86813735961914,
      "activations/layer19_attention_weight_max": 33.652347564697266,
      "activations/layer19_attention_weight_min": -31.70554542541504,
      "activations/layer1_attention_weight_max": 17.245281219482422,
      "activations/layer1_attention_weight_min": -15.741389274597168,
      "activations/layer20_attention_weight_max": 29.205554962158203,
      "activations/layer20_attention_weight_min": -26.1054744720459,
      "activations/layer21_attention_weight_max": 30.199005126953125,
      "activations/layer21_attention_weight_min": -24.136167526245117,
      "activations/layer22_attention_weight_max": 44.916908264160156,
      "activations/layer22_attention_weight_min": -29.692523956298828,
      "activations/layer23_attention_weight_max": 34.12268829345703,
      "activations/layer23_attention_weight_min": -24.22710418701172,
      "activations/layer2_attention_weight_max": 32.15627670288086,
      "activations/layer2_attention_weight_min": -33.69096755981445,
      "activations/layer3_attention_weight_max": 95.81838989257812,
      "activations/layer3_attention_weight_min": -99.30377960205078,
      "activations/layer4_attention_weight_max": 113.27278137207031,
      "activations/layer4_attention_weight_min": -113.26827239990234,
      "activations/layer5_attention_weight_max": 55.332237243652344,
      "activations/layer5_attention_weight_min": -61.617828369140625,
      "activations/layer6_attention_weight_max": 44.22053909301758,
      "activations/layer6_attention_weight_min": -49.62626266479492,
      "activations/layer7_attention_weight_max": 92.44808197021484,
      "activations/layer7_attention_weight_min": -89.54046630859375,
      "activations/layer8_attention_weight_max": 39.90607452392578,
      "activations/layer8_attention_weight_min": -42.468994140625,
      "activations/layer9_attention_weight_max": 33.30805206298828,
      "activations/layer9_attention_weight_min": -34.701961517333984,
      "epoch": 10.41,
      "learning_rate": 8.373219696969696e-05,
      "loss": 2.7734,
      "step": 179100
    },
    {
      "activations/layer0_attention_weight_max": 15.987138748168945,
      "activations/layer0_attention_weight_min": -12.907289505004883,
      "activations/layer10_attention_weight_max": 38.77720260620117,
      "activations/layer10_attention_weight_min": -35.97967529296875,
      "activations/layer11_attention_weight_max": 40.42364501953125,
      "activations/layer11_attention_weight_min": -37.92107009887695,
      "activations/layer12_attention_weight_max": 26.580665588378906,
      "activations/layer12_attention_weight_min": -29.153295516967773,
      "activations/layer13_attention_weight_max": 48.426307678222656,
      "activations/layer13_attention_weight_min": -38.88030242919922,
      "activations/layer14_attention_weight_max": 59.45657730102539,
      "activations/layer14_attention_weight_min": -43.59092330932617,
      "activations/layer15_attention_weight_max": 50.051570892333984,
      "activations/layer15_attention_weight_min": -36.01987838745117,
      "activations/layer16_attention_weight_max": 32.376731872558594,
      "activations/layer16_attention_weight_min": -30.386119842529297,
      "activations/layer17_attention_weight_max": 31.54385757446289,
      "activations/layer17_attention_weight_min": -28.993825912475586,
      "activations/layer18_attention_weight_max": 31.445247650146484,
      "activations/layer18_attention_weight_min": -22.981237411499023,
      "activations/layer19_attention_weight_max": 37.54789352416992,
      "activations/layer19_attention_weight_min": -32.41703414916992,
      "activations/layer1_attention_weight_max": 16.453357696533203,
      "activations/layer1_attention_weight_min": -15.222097396850586,
      "activations/layer20_attention_weight_max": 31.617687225341797,
      "activations/layer20_attention_weight_min": -24.829607009887695,
      "activations/layer21_attention_weight_max": 29.248868942260742,
      "activations/layer21_attention_weight_min": -24.2060604095459,
      "activations/layer22_attention_weight_max": 51.45573806762695,
      "activations/layer22_attention_weight_min": -31.89073944091797,
      "activations/layer23_attention_weight_max": 33.502010345458984,
      "activations/layer23_attention_weight_min": -23.961143493652344,
      "activations/layer2_attention_weight_max": 34.57963943481445,
      "activations/layer2_attention_weight_min": -33.421051025390625,
      "activations/layer3_attention_weight_max": 95.71039581298828,
      "activations/layer3_attention_weight_min": -98.04147338867188,
      "activations/layer4_attention_weight_max": 116.3140869140625,
      "activations/layer4_attention_weight_min": -117.40118408203125,
      "activations/layer5_attention_weight_max": 57.59529113769531,
      "activations/layer5_attention_weight_min": -64.18254089355469,
      "activations/layer6_attention_weight_max": 46.738643646240234,
      "activations/layer6_attention_weight_min": -49.441505432128906,
      "activations/layer7_attention_weight_max": 101.69661712646484,
      "activations/layer7_attention_weight_min": -92.54927062988281,
      "activations/layer8_attention_weight_max": 46.848716735839844,
      "activations/layer8_attention_weight_min": -42.19648361206055,
      "activations/layer9_attention_weight_max": 37.60612869262695,
      "activations/layer9_attention_weight_min": -35.653446197509766,
      "epoch": 10.41,
      "learning_rate": 8.371363636363635e-05,
      "loss": 2.7766,
      "step": 179150
    },
    {
      "activations/layer0_attention_weight_max": 15.938587188720703,
      "activations/layer0_attention_weight_min": -13.514341354370117,
      "activations/layer10_attention_weight_max": 33.25016784667969,
      "activations/layer10_attention_weight_min": -35.358421325683594,
      "activations/layer11_attention_weight_max": 32.62550354003906,
      "activations/layer11_attention_weight_min": -32.42875671386719,
      "activations/layer12_attention_weight_max": 26.667667388916016,
      "activations/layer12_attention_weight_min": -25.382658004760742,
      "activations/layer13_attention_weight_max": 43.40330505371094,
      "activations/layer13_attention_weight_min": -36.53319549560547,
      "activations/layer14_attention_weight_max": 47.5823974609375,
      "activations/layer14_attention_weight_min": -39.65403747558594,
      "activations/layer15_attention_weight_max": 41.79585266113281,
      "activations/layer15_attention_weight_min": -32.515804290771484,
      "activations/layer16_attention_weight_max": 34.28292465209961,
      "activations/layer16_attention_weight_min": -29.373289108276367,
      "activations/layer17_attention_weight_max": 34.93878173828125,
      "activations/layer17_attention_weight_min": -26.219247817993164,
      "activations/layer18_attention_weight_max": 31.160995483398438,
      "activations/layer18_attention_weight_min": -23.65709114074707,
      "activations/layer19_attention_weight_max": 35.73649215698242,
      "activations/layer19_attention_weight_min": -31.844287872314453,
      "activations/layer1_attention_weight_max": 15.84223461151123,
      "activations/layer1_attention_weight_min": -14.276116371154785,
      "activations/layer20_attention_weight_max": 29.077373504638672,
      "activations/layer20_attention_weight_min": -24.653762817382812,
      "activations/layer21_attention_weight_max": 31.637266159057617,
      "activations/layer21_attention_weight_min": -24.58583641052246,
      "activations/layer22_attention_weight_max": 44.51909637451172,
      "activations/layer22_attention_weight_min": -29.910799026489258,
      "activations/layer23_attention_weight_max": 35.75267791748047,
      "activations/layer23_attention_weight_min": -25.837743759155273,
      "activations/layer2_attention_weight_max": 32.25324630737305,
      "activations/layer2_attention_weight_min": -34.79357147216797,
      "activations/layer3_attention_weight_max": 97.18714904785156,
      "activations/layer3_attention_weight_min": -99.6583251953125,
      "activations/layer4_attention_weight_max": 117.66104888916016,
      "activations/layer4_attention_weight_min": -116.50325775146484,
      "activations/layer5_attention_weight_max": 53.275264739990234,
      "activations/layer5_attention_weight_min": -62.39134979248047,
      "activations/layer6_attention_weight_max": 45.552574157714844,
      "activations/layer6_attention_weight_min": -48.73877716064453,
      "activations/layer7_attention_weight_max": 96.98162841796875,
      "activations/layer7_attention_weight_min": -98.58485412597656,
      "activations/layer8_attention_weight_max": 40.927467346191406,
      "activations/layer8_attention_weight_min": -42.4887580871582,
      "activations/layer9_attention_weight_max": 34.1259651184082,
      "activations/layer9_attention_weight_min": -34.930824279785156,
      "epoch": 10.41,
      "learning_rate": 8.369469696969696e-05,
      "loss": 2.7707,
      "step": 179200
    },
    {
      "activations/layer0_attention_weight_max": 16.68878936767578,
      "activations/layer0_attention_weight_min": -11.474733352661133,
      "activations/layer10_attention_weight_max": 31.130977630615234,
      "activations/layer10_attention_weight_min": -33.23711013793945,
      "activations/layer11_attention_weight_max": 31.08120346069336,
      "activations/layer11_attention_weight_min": -30.396930694580078,
      "activations/layer12_attention_weight_max": 23.762781143188477,
      "activations/layer12_attention_weight_min": -25.05316925048828,
      "activations/layer13_attention_weight_max": 38.957000732421875,
      "activations/layer13_attention_weight_min": -36.06916809082031,
      "activations/layer14_attention_weight_max": 44.69648742675781,
      "activations/layer14_attention_weight_min": -40.33719253540039,
      "activations/layer15_attention_weight_max": 34.88554763793945,
      "activations/layer15_attention_weight_min": -31.10274887084961,
      "activations/layer16_attention_weight_max": 30.602994918823242,
      "activations/layer16_attention_weight_min": -26.944517135620117,
      "activations/layer17_attention_weight_max": 30.435773849487305,
      "activations/layer17_attention_weight_min": -25.90230369567871,
      "activations/layer18_attention_weight_max": 31.468900680541992,
      "activations/layer18_attention_weight_min": -23.658525466918945,
      "activations/layer19_attention_weight_max": 33.36088180541992,
      "activations/layer19_attention_weight_min": -30.6761474609375,
      "activations/layer1_attention_weight_max": 16.16720962524414,
      "activations/layer1_attention_weight_min": -15.307263374328613,
      "activations/layer20_attention_weight_max": 28.959983825683594,
      "activations/layer20_attention_weight_min": -24.206069946289062,
      "activations/layer21_attention_weight_max": 26.61408042907715,
      "activations/layer21_attention_weight_min": -23.538549423217773,
      "activations/layer22_attention_weight_max": 39.739402770996094,
      "activations/layer22_attention_weight_min": -30.96053695678711,
      "activations/layer23_attention_weight_max": 32.40985870361328,
      "activations/layer23_attention_weight_min": -25.28192710876465,
      "activations/layer2_attention_weight_max": 33.2485466003418,
      "activations/layer2_attention_weight_min": -33.607582092285156,
      "activations/layer3_attention_weight_max": 92.0947036743164,
      "activations/layer3_attention_weight_min": -97.65433502197266,
      "activations/layer4_attention_weight_max": 113.21070861816406,
      "activations/layer4_attention_weight_min": -114.0811996459961,
      "activations/layer5_attention_weight_max": 52.133087158203125,
      "activations/layer5_attention_weight_min": -63.081966400146484,
      "activations/layer6_attention_weight_max": 43.39791488647461,
      "activations/layer6_attention_weight_min": -49.4348258972168,
      "activations/layer7_attention_weight_max": 88.52962493896484,
      "activations/layer7_attention_weight_min": -89.10399627685547,
      "activations/layer8_attention_weight_max": 38.8959846496582,
      "activations/layer8_attention_weight_min": -40.617862701416016,
      "activations/layer9_attention_weight_max": 31.868173599243164,
      "activations/layer9_attention_weight_min": -33.10972213745117,
      "epoch": 10.42,
      "learning_rate": 8.367575757575757e-05,
      "loss": 2.7662,
      "step": 179250
    },
    {
      "activations/layer0_attention_weight_max": 15.768036842346191,
      "activations/layer0_attention_weight_min": -12.4603853225708,
      "activations/layer10_attention_weight_max": 37.540992736816406,
      "activations/layer10_attention_weight_min": -34.999324798583984,
      "activations/layer11_attention_weight_max": 38.17218780517578,
      "activations/layer11_attention_weight_min": -37.487693786621094,
      "activations/layer12_attention_weight_max": 25.552627563476562,
      "activations/layer12_attention_weight_min": -26.094846725463867,
      "activations/layer13_attention_weight_max": 41.99479675292969,
      "activations/layer13_attention_weight_min": -33.70185470581055,
      "activations/layer14_attention_weight_max": 47.35951232910156,
      "activations/layer14_attention_weight_min": -35.735877990722656,
      "activations/layer15_attention_weight_max": 40.86436462402344,
      "activations/layer15_attention_weight_min": -31.13752555847168,
      "activations/layer16_attention_weight_max": 35.50740432739258,
      "activations/layer16_attention_weight_min": -27.99457550048828,
      "activations/layer17_attention_weight_max": 35.0217170715332,
      "activations/layer17_attention_weight_min": -24.788646697998047,
      "activations/layer18_attention_weight_max": 35.70749282836914,
      "activations/layer18_attention_weight_min": -21.631601333618164,
      "activations/layer19_attention_weight_max": 38.5903434753418,
      "activations/layer19_attention_weight_min": -30.813711166381836,
      "activations/layer1_attention_weight_max": 17.301788330078125,
      "activations/layer1_attention_weight_min": -13.46970272064209,
      "activations/layer20_attention_weight_max": 34.22078323364258,
      "activations/layer20_attention_weight_min": -24.668432235717773,
      "activations/layer21_attention_weight_max": 35.22090148925781,
      "activations/layer21_attention_weight_min": -23.482656478881836,
      "activations/layer22_attention_weight_max": 45.34583282470703,
      "activations/layer22_attention_weight_min": -29.93903923034668,
      "activations/layer23_attention_weight_max": 36.59710693359375,
      "activations/layer23_attention_weight_min": -23.559219360351562,
      "activations/layer2_attention_weight_max": 32.133331298828125,
      "activations/layer2_attention_weight_min": -32.5853157043457,
      "activations/layer3_attention_weight_max": 98.31986236572266,
      "activations/layer3_attention_weight_min": -96.54362487792969,
      "activations/layer4_attention_weight_max": 119.32376861572266,
      "activations/layer4_attention_weight_min": -116.60540771484375,
      "activations/layer5_attention_weight_max": 54.28246307373047,
      "activations/layer5_attention_weight_min": -61.49168395996094,
      "activations/layer6_attention_weight_max": 47.03014373779297,
      "activations/layer6_attention_weight_min": -50.24113464355469,
      "activations/layer7_attention_weight_max": 94.7342300415039,
      "activations/layer7_attention_weight_min": -89.00752258300781,
      "activations/layer8_attention_weight_max": 40.78776168823242,
      "activations/layer8_attention_weight_min": -42.58552551269531,
      "activations/layer9_attention_weight_max": 36.41170883178711,
      "activations/layer9_attention_weight_min": -36.47937774658203,
      "epoch": 10.42,
      "learning_rate": 8.365681818181818e-05,
      "loss": 2.7655,
      "step": 179300
    },
    {
      "activations/layer0_attention_weight_max": 15.840317726135254,
      "activations/layer0_attention_weight_min": -12.060962677001953,
      "activations/layer10_attention_weight_max": 34.17828369140625,
      "activations/layer10_attention_weight_min": -32.12154769897461,
      "activations/layer11_attention_weight_max": 34.17378234863281,
      "activations/layer11_attention_weight_min": -31.75477409362793,
      "activations/layer12_attention_weight_max": 25.224727630615234,
      "activations/layer12_attention_weight_min": -24.923728942871094,
      "activations/layer13_attention_weight_max": 36.14696502685547,
      "activations/layer13_attention_weight_min": -35.45585632324219,
      "activations/layer14_attention_weight_max": 42.59440994262695,
      "activations/layer14_attention_weight_min": -41.09476089477539,
      "activations/layer15_attention_weight_max": 32.86699676513672,
      "activations/layer15_attention_weight_min": -32.23371124267578,
      "activations/layer16_attention_weight_max": 31.339847564697266,
      "activations/layer16_attention_weight_min": -27.72828483581543,
      "activations/layer17_attention_weight_max": 29.50958251953125,
      "activations/layer17_attention_weight_min": -27.739255905151367,
      "activations/layer18_attention_weight_max": 29.755891799926758,
      "activations/layer18_attention_weight_min": -24.108293533325195,
      "activations/layer19_attention_weight_max": 33.37736129760742,
      "activations/layer19_attention_weight_min": -31.709787368774414,
      "activations/layer1_attention_weight_max": 16.2783145904541,
      "activations/layer1_attention_weight_min": -14.44294548034668,
      "activations/layer20_attention_weight_max": 27.651521682739258,
      "activations/layer20_attention_weight_min": -26.014198303222656,
      "activations/layer21_attention_weight_max": 27.981231689453125,
      "activations/layer21_attention_weight_min": -24.935911178588867,
      "activations/layer22_attention_weight_max": 42.53611373901367,
      "activations/layer22_attention_weight_min": -30.00332260131836,
      "activations/layer23_attention_weight_max": 35.54554748535156,
      "activations/layer23_attention_weight_min": -23.44899559020996,
      "activations/layer2_attention_weight_max": 34.08673095703125,
      "activations/layer2_attention_weight_min": -34.93253707885742,
      "activations/layer3_attention_weight_max": 94.8454360961914,
      "activations/layer3_attention_weight_min": -99.60828399658203,
      "activations/layer4_attention_weight_max": 118.45863342285156,
      "activations/layer4_attention_weight_min": -112.91168212890625,
      "activations/layer5_attention_weight_max": 54.57505416870117,
      "activations/layer5_attention_weight_min": -62.86304473876953,
      "activations/layer6_attention_weight_max": 46.03217315673828,
      "activations/layer6_attention_weight_min": -49.55488967895508,
      "activations/layer7_attention_weight_max": 90.16853332519531,
      "activations/layer7_attention_weight_min": -97.5135498046875,
      "activations/layer8_attention_weight_max": 39.59174728393555,
      "activations/layer8_attention_weight_min": -41.30794906616211,
      "activations/layer9_attention_weight_max": 33.93497085571289,
      "activations/layer9_attention_weight_min": -34.6489372253418,
      "epoch": 10.42,
      "learning_rate": 8.363787878787878e-05,
      "loss": 2.7677,
      "step": 179350
    },
    {
      "activations/layer0_attention_weight_max": 16.416563034057617,
      "activations/layer0_attention_weight_min": -12.617732048034668,
      "activations/layer10_attention_weight_max": 32.669647216796875,
      "activations/layer10_attention_weight_min": -32.54243469238281,
      "activations/layer11_attention_weight_max": 34.12293243408203,
      "activations/layer11_attention_weight_min": -32.540313720703125,
      "activations/layer12_attention_weight_max": 24.1319637298584,
      "activations/layer12_attention_weight_min": -26.888731002807617,
      "activations/layer13_attention_weight_max": 38.05477523803711,
      "activations/layer13_attention_weight_min": -36.34307861328125,
      "activations/layer14_attention_weight_max": 44.92610168457031,
      "activations/layer14_attention_weight_min": -39.889686584472656,
      "activations/layer15_attention_weight_max": 35.4661750793457,
      "activations/layer15_attention_weight_min": -32.237911224365234,
      "activations/layer16_attention_weight_max": 31.64525604248047,
      "activations/layer16_attention_weight_min": -26.31634521484375,
      "activations/layer17_attention_weight_max": 31.154504776000977,
      "activations/layer17_attention_weight_min": -25.86057472229004,
      "activations/layer18_attention_weight_max": 29.923044204711914,
      "activations/layer18_attention_weight_min": -22.888896942138672,
      "activations/layer19_attention_weight_max": 34.39342498779297,
      "activations/layer19_attention_weight_min": -35.77821350097656,
      "activations/layer1_attention_weight_max": 17.62177276611328,
      "activations/layer1_attention_weight_min": -14.607142448425293,
      "activations/layer20_attention_weight_max": 30.866924285888672,
      "activations/layer20_attention_weight_min": -26.27397918701172,
      "activations/layer21_attention_weight_max": 28.862646102905273,
      "activations/layer21_attention_weight_min": -24.512893676757812,
      "activations/layer22_attention_weight_max": 42.99108123779297,
      "activations/layer22_attention_weight_min": -29.992719650268555,
      "activations/layer23_attention_weight_max": 32.04518127441406,
      "activations/layer23_attention_weight_min": -22.08216094970703,
      "activations/layer2_attention_weight_max": 35.373291015625,
      "activations/layer2_attention_weight_min": -35.48530197143555,
      "activations/layer3_attention_weight_max": 94.52404022216797,
      "activations/layer3_attention_weight_min": -96.62982177734375,
      "activations/layer4_attention_weight_max": 113.41316223144531,
      "activations/layer4_attention_weight_min": -117.05958557128906,
      "activations/layer5_attention_weight_max": 52.881553649902344,
      "activations/layer5_attention_weight_min": -60.89111328125,
      "activations/layer6_attention_weight_max": 46.592491149902344,
      "activations/layer6_attention_weight_min": -50.40110397338867,
      "activations/layer7_attention_weight_max": 91.53968811035156,
      "activations/layer7_attention_weight_min": -91.47631072998047,
      "activations/layer8_attention_weight_max": 42.36160659790039,
      "activations/layer8_attention_weight_min": -42.60585021972656,
      "activations/layer9_attention_weight_max": 35.27836608886719,
      "activations/layer9_attention_weight_min": -33.63650894165039,
      "epoch": 10.42,
      "learning_rate": 8.361893939393938e-05,
      "loss": 2.7799,
      "step": 179400
    },
    {
      "activations/layer0_attention_weight_max": 15.487961769104004,
      "activations/layer0_attention_weight_min": -12.963496208190918,
      "activations/layer10_attention_weight_max": 33.964927673339844,
      "activations/layer10_attention_weight_min": -32.53693771362305,
      "activations/layer11_attention_weight_max": 35.16218566894531,
      "activations/layer11_attention_weight_min": -32.2443733215332,
      "activations/layer12_attention_weight_max": 28.987646102905273,
      "activations/layer12_attention_weight_min": -28.210664749145508,
      "activations/layer13_attention_weight_max": 39.393310546875,
      "activations/layer13_attention_weight_min": -33.20182800292969,
      "activations/layer14_attention_weight_max": 44.72147750854492,
      "activations/layer14_attention_weight_min": -36.117610931396484,
      "activations/layer15_attention_weight_max": 39.35148239135742,
      "activations/layer15_attention_weight_min": -31.081811904907227,
      "activations/layer16_attention_weight_max": 31.845840454101562,
      "activations/layer16_attention_weight_min": -27.98065185546875,
      "activations/layer17_attention_weight_max": 34.93123245239258,
      "activations/layer17_attention_weight_min": -27.000585556030273,
      "activations/layer18_attention_weight_max": 31.385164260864258,
      "activations/layer18_attention_weight_min": -23.672344207763672,
      "activations/layer19_attention_weight_max": 37.540374755859375,
      "activations/layer19_attention_weight_min": -31.841552734375,
      "activations/layer1_attention_weight_max": 17.528650283813477,
      "activations/layer1_attention_weight_min": -16.939245223999023,
      "activations/layer20_attention_weight_max": 32.66740417480469,
      "activations/layer20_attention_weight_min": -27.257366180419922,
      "activations/layer21_attention_weight_max": 33.3111457824707,
      "activations/layer21_attention_weight_min": -24.425418853759766,
      "activations/layer22_attention_weight_max": 45.40033721923828,
      "activations/layer22_attention_weight_min": -31.047550201416016,
      "activations/layer23_attention_weight_max": 38.972572326660156,
      "activations/layer23_attention_weight_min": -24.322629928588867,
      "activations/layer2_attention_weight_max": 33.33819580078125,
      "activations/layer2_attention_weight_min": -32.71809005737305,
      "activations/layer3_attention_weight_max": 100.35267639160156,
      "activations/layer3_attention_weight_min": -101.57984924316406,
      "activations/layer4_attention_weight_max": 119.71150207519531,
      "activations/layer4_attention_weight_min": -117.32816314697266,
      "activations/layer5_attention_weight_max": 56.68968963623047,
      "activations/layer5_attention_weight_min": -63.191200256347656,
      "activations/layer6_attention_weight_max": 47.51677322387695,
      "activations/layer6_attention_weight_min": -50.93558120727539,
      "activations/layer7_attention_weight_max": 95.3846664428711,
      "activations/layer7_attention_weight_min": -95.17388916015625,
      "activations/layer8_attention_weight_max": 42.20258331298828,
      "activations/layer8_attention_weight_min": -41.3021354675293,
      "activations/layer9_attention_weight_max": 35.979156494140625,
      "activations/layer9_attention_weight_min": -35.23771286010742,
      "epoch": 10.43,
      "learning_rate": 8.36e-05,
      "loss": 2.7418,
      "step": 179450
    },
    {
      "activations/layer0_attention_weight_max": 15.800378799438477,
      "activations/layer0_attention_weight_min": -12.2262601852417,
      "activations/layer10_attention_weight_max": 31.774242401123047,
      "activations/layer10_attention_weight_min": -32.76728057861328,
      "activations/layer11_attention_weight_max": 32.0226936340332,
      "activations/layer11_attention_weight_min": -32.09661102294922,
      "activations/layer12_attention_weight_max": 24.33658790588379,
      "activations/layer12_attention_weight_min": -25.899009704589844,
      "activations/layer13_attention_weight_max": 42.28656768798828,
      "activations/layer13_attention_weight_min": -35.229454040527344,
      "activations/layer14_attention_weight_max": 43.31864547729492,
      "activations/layer14_attention_weight_min": -36.1854133605957,
      "activations/layer15_attention_weight_max": 33.77592086791992,
      "activations/layer15_attention_weight_min": -32.93376922607422,
      "activations/layer16_attention_weight_max": 31.882368087768555,
      "activations/layer16_attention_weight_min": -27.654998779296875,
      "activations/layer17_attention_weight_max": 32.276145935058594,
      "activations/layer17_attention_weight_min": -26.060039520263672,
      "activations/layer18_attention_weight_max": 32.447174072265625,
      "activations/layer18_attention_weight_min": -23.407852172851562,
      "activations/layer19_attention_weight_max": 33.766231536865234,
      "activations/layer19_attention_weight_min": -30.076690673828125,
      "activations/layer1_attention_weight_max": 16.291982650756836,
      "activations/layer1_attention_weight_min": -14.232780456542969,
      "activations/layer20_attention_weight_max": 28.820940017700195,
      "activations/layer20_attention_weight_min": -24.80925750732422,
      "activations/layer21_attention_weight_max": 29.38408088684082,
      "activations/layer21_attention_weight_min": -24.545976638793945,
      "activations/layer22_attention_weight_max": 43.85713195800781,
      "activations/layer22_attention_weight_min": -30.89034652709961,
      "activations/layer23_attention_weight_max": 35.777565002441406,
      "activations/layer23_attention_weight_min": -25.38373374938965,
      "activations/layer2_attention_weight_max": 30.814054489135742,
      "activations/layer2_attention_weight_min": -32.748779296875,
      "activations/layer3_attention_weight_max": 94.23311614990234,
      "activations/layer3_attention_weight_min": -94.92979431152344,
      "activations/layer4_attention_weight_max": 113.89261627197266,
      "activations/layer4_attention_weight_min": -112.57340240478516,
      "activations/layer5_attention_weight_max": 54.58123779296875,
      "activations/layer5_attention_weight_min": -62.248069763183594,
      "activations/layer6_attention_weight_max": 47.6761589050293,
      "activations/layer6_attention_weight_min": -48.69309616088867,
      "activations/layer7_attention_weight_max": 95.46810150146484,
      "activations/layer7_attention_weight_min": -92.01278686523438,
      "activations/layer8_attention_weight_max": 42.62510681152344,
      "activations/layer8_attention_weight_min": -41.569313049316406,
      "activations/layer9_attention_weight_max": 34.3792839050293,
      "activations/layer9_attention_weight_min": -33.7368049621582,
      "epoch": 10.43,
      "learning_rate": 8.35810606060606e-05,
      "loss": 2.7848,
      "step": 179500
    },
    {
      "activations/layer0_attention_weight_max": 15.450066566467285,
      "activations/layer0_attention_weight_min": -12.454363822937012,
      "activations/layer10_attention_weight_max": 32.503822326660156,
      "activations/layer10_attention_weight_min": -32.00140380859375,
      "activations/layer11_attention_weight_max": 33.4361686706543,
      "activations/layer11_attention_weight_min": -32.286964416503906,
      "activations/layer12_attention_weight_max": 21.805967330932617,
      "activations/layer12_attention_weight_min": -23.654834747314453,
      "activations/layer13_attention_weight_max": 34.73249435424805,
      "activations/layer13_attention_weight_min": -33.19879913330078,
      "activations/layer14_attention_weight_max": 40.4377555847168,
      "activations/layer14_attention_weight_min": -36.04027557373047,
      "activations/layer15_attention_weight_max": 32.13510513305664,
      "activations/layer15_attention_weight_min": -31.853378295898438,
      "activations/layer16_attention_weight_max": 27.19605255126953,
      "activations/layer16_attention_weight_min": -26.82139015197754,
      "activations/layer17_attention_weight_max": 30.873315811157227,
      "activations/layer17_attention_weight_min": -26.581649780273438,
      "activations/layer18_attention_weight_max": 29.40908432006836,
      "activations/layer18_attention_weight_min": -22.610597610473633,
      "activations/layer19_attention_weight_max": 31.306350708007812,
      "activations/layer19_attention_weight_min": -29.668136596679688,
      "activations/layer1_attention_weight_max": 16.30508804321289,
      "activations/layer1_attention_weight_min": -14.882710456848145,
      "activations/layer20_attention_weight_max": 27.027294158935547,
      "activations/layer20_attention_weight_min": -23.50991439819336,
      "activations/layer21_attention_weight_max": 28.57539939880371,
      "activations/layer21_attention_weight_min": -22.113204956054688,
      "activations/layer22_attention_weight_max": 39.065608978271484,
      "activations/layer22_attention_weight_min": -29.76564598083496,
      "activations/layer23_attention_weight_max": 29.957935333251953,
      "activations/layer23_attention_weight_min": -25.67377471923828,
      "activations/layer2_attention_weight_max": 34.13306427001953,
      "activations/layer2_attention_weight_min": -35.71390151977539,
      "activations/layer3_attention_weight_max": 96.30633544921875,
      "activations/layer3_attention_weight_min": -101.7088623046875,
      "activations/layer4_attention_weight_max": 117.06396484375,
      "activations/layer4_attention_weight_min": -117.65602111816406,
      "activations/layer5_attention_weight_max": 52.34235763549805,
      "activations/layer5_attention_weight_min": -61.096839904785156,
      "activations/layer6_attention_weight_max": 45.42816925048828,
      "activations/layer6_attention_weight_min": -47.70643615722656,
      "activations/layer7_attention_weight_max": 87.18370056152344,
      "activations/layer7_attention_weight_min": -91.19384002685547,
      "activations/layer8_attention_weight_max": 39.88309097290039,
      "activations/layer8_attention_weight_min": -42.63739776611328,
      "activations/layer9_attention_weight_max": 33.0994987487793,
      "activations/layer9_attention_weight_min": -32.868160247802734,
      "epoch": 10.43,
      "learning_rate": 8.35625e-05,
      "loss": 2.7828,
      "step": 179550
    },
    {
      "activations/layer0_attention_weight_max": 15.209881782531738,
      "activations/layer0_attention_weight_min": -12.002907752990723,
      "activations/layer10_attention_weight_max": 36.16676712036133,
      "activations/layer10_attention_weight_min": -36.279823303222656,
      "activations/layer11_attention_weight_max": 33.141075134277344,
      "activations/layer11_attention_weight_min": -36.24839401245117,
      "activations/layer12_attention_weight_max": 24.56133460998535,
      "activations/layer12_attention_weight_min": -25.842721939086914,
      "activations/layer13_attention_weight_max": 39.786407470703125,
      "activations/layer13_attention_weight_min": -36.06037139892578,
      "activations/layer14_attention_weight_max": 48.74189758300781,
      "activations/layer14_attention_weight_min": -41.99851608276367,
      "activations/layer15_attention_weight_max": 35.824161529541016,
      "activations/layer15_attention_weight_min": -31.477928161621094,
      "activations/layer16_attention_weight_max": 33.138980865478516,
      "activations/layer16_attention_weight_min": -28.295654296875,
      "activations/layer17_attention_weight_max": 30.56710433959961,
      "activations/layer17_attention_weight_min": -25.28825569152832,
      "activations/layer18_attention_weight_max": 29.26237678527832,
      "activations/layer18_attention_weight_min": -22.89581298828125,
      "activations/layer19_attention_weight_max": 39.0489501953125,
      "activations/layer19_attention_weight_min": -29.70401954650879,
      "activations/layer1_attention_weight_max": 16.831560134887695,
      "activations/layer1_attention_weight_min": -15.36461067199707,
      "activations/layer20_attention_weight_max": 32.11325454711914,
      "activations/layer20_attention_weight_min": -24.889724731445312,
      "activations/layer21_attention_weight_max": 34.69902420043945,
      "activations/layer21_attention_weight_min": -25.32782554626465,
      "activations/layer22_attention_weight_max": 44.2225341796875,
      "activations/layer22_attention_weight_min": -30.913206100463867,
      "activations/layer23_attention_weight_max": 34.74446105957031,
      "activations/layer23_attention_weight_min": -24.439748764038086,
      "activations/layer2_attention_weight_max": 34.059669494628906,
      "activations/layer2_attention_weight_min": -32.818382263183594,
      "activations/layer3_attention_weight_max": 101.31404113769531,
      "activations/layer3_attention_weight_min": -96.05037689208984,
      "activations/layer4_attention_weight_max": 118.85179901123047,
      "activations/layer4_attention_weight_min": -117.63385009765625,
      "activations/layer5_attention_weight_max": 55.28063201904297,
      "activations/layer5_attention_weight_min": -64.732666015625,
      "activations/layer6_attention_weight_max": 46.53583526611328,
      "activations/layer6_attention_weight_min": -52.44431686401367,
      "activations/layer7_attention_weight_max": 96.8258285522461,
      "activations/layer7_attention_weight_min": -99.0180892944336,
      "activations/layer8_attention_weight_max": 40.56403350830078,
      "activations/layer8_attention_weight_min": -42.9002685546875,
      "activations/layer9_attention_weight_max": 36.12619400024414,
      "activations/layer9_attention_weight_min": -38.17416763305664,
      "epoch": 10.44,
      "learning_rate": 8.354356060606059e-05,
      "loss": 2.7727,
      "step": 179600
    },
    {
      "activations/layer0_attention_weight_max": 16.72171401977539,
      "activations/layer0_attention_weight_min": -12.853408813476562,
      "activations/layer10_attention_weight_max": 33.80677795410156,
      "activations/layer10_attention_weight_min": -33.21194076538086,
      "activations/layer11_attention_weight_max": 35.137001037597656,
      "activations/layer11_attention_weight_min": -32.44432830810547,
      "activations/layer12_attention_weight_max": 22.471208572387695,
      "activations/layer12_attention_weight_min": -26.982084274291992,
      "activations/layer13_attention_weight_max": 36.04471206665039,
      "activations/layer13_attention_weight_min": -34.71062469482422,
      "activations/layer14_attention_weight_max": 48.82802200317383,
      "activations/layer14_attention_weight_min": -38.339874267578125,
      "activations/layer15_attention_weight_max": 38.04171371459961,
      "activations/layer15_attention_weight_min": -30.64982795715332,
      "activations/layer16_attention_weight_max": 32.62114715576172,
      "activations/layer16_attention_weight_min": -28.134599685668945,
      "activations/layer17_attention_weight_max": 35.901485443115234,
      "activations/layer17_attention_weight_min": -25.178462982177734,
      "activations/layer18_attention_weight_max": 38.62023162841797,
      "activations/layer18_attention_weight_min": -21.608692169189453,
      "activations/layer19_attention_weight_max": 37.73735427856445,
      "activations/layer19_attention_weight_min": -30.28750991821289,
      "activations/layer1_attention_weight_max": 17.310325622558594,
      "activations/layer1_attention_weight_min": -14.184089660644531,
      "activations/layer20_attention_weight_max": 30.542516708374023,
      "activations/layer20_attention_weight_min": -24.103303909301758,
      "activations/layer21_attention_weight_max": 32.97993850708008,
      "activations/layer21_attention_weight_min": -24.390701293945312,
      "activations/layer22_attention_weight_max": 42.59648132324219,
      "activations/layer22_attention_weight_min": -27.860105514526367,
      "activations/layer23_attention_weight_max": 36.25172424316406,
      "activations/layer23_attention_weight_min": -23.522323608398438,
      "activations/layer2_attention_weight_max": 33.66194152832031,
      "activations/layer2_attention_weight_min": -34.08345031738281,
      "activations/layer3_attention_weight_max": 96.71342468261719,
      "activations/layer3_attention_weight_min": -93.76282501220703,
      "activations/layer4_attention_weight_max": 119.6192398071289,
      "activations/layer4_attention_weight_min": -114.6670913696289,
      "activations/layer5_attention_weight_max": 53.26548767089844,
      "activations/layer5_attention_weight_min": -65.31193542480469,
      "activations/layer6_attention_weight_max": 46.692447662353516,
      "activations/layer6_attention_weight_min": -49.46045684814453,
      "activations/layer7_attention_weight_max": 91.7847671508789,
      "activations/layer7_attention_weight_min": -90.59493255615234,
      "activations/layer8_attention_weight_max": 41.570594787597656,
      "activations/layer8_attention_weight_min": -40.56629943847656,
      "activations/layer9_attention_weight_max": 33.57789611816406,
      "activations/layer9_attention_weight_min": -32.904762268066406,
      "epoch": 10.44,
      "learning_rate": 8.352462121212121e-05,
      "loss": 2.776,
      "step": 179650
    },
    {
      "activations/layer0_attention_weight_max": 16.220571517944336,
      "activations/layer0_attention_weight_min": -11.689522743225098,
      "activations/layer10_attention_weight_max": 33.954288482666016,
      "activations/layer10_attention_weight_min": -33.58944320678711,
      "activations/layer11_attention_weight_max": 34.480316162109375,
      "activations/layer11_attention_weight_min": -33.73301696777344,
      "activations/layer12_attention_weight_max": 26.296138763427734,
      "activations/layer12_attention_weight_min": -28.085670471191406,
      "activations/layer13_attention_weight_max": 40.797584533691406,
      "activations/layer13_attention_weight_min": -39.59922790527344,
      "activations/layer14_attention_weight_max": 49.67855453491211,
      "activations/layer14_attention_weight_min": -40.0871467590332,
      "activations/layer15_attention_weight_max": 36.926025390625,
      "activations/layer15_attention_weight_min": -34.242584228515625,
      "activations/layer16_attention_weight_max": 31.007816314697266,
      "activations/layer16_attention_weight_min": -32.02898025512695,
      "activations/layer17_attention_weight_max": 34.686519622802734,
      "activations/layer17_attention_weight_min": -27.98627471923828,
      "activations/layer18_attention_weight_max": 30.669328689575195,
      "activations/layer18_attention_weight_min": -23.493709564208984,
      "activations/layer19_attention_weight_max": 37.61558532714844,
      "activations/layer19_attention_weight_min": -34.3585319519043,
      "activations/layer1_attention_weight_max": 16.644386291503906,
      "activations/layer1_attention_weight_min": -11.704999923706055,
      "activations/layer20_attention_weight_max": 31.693241119384766,
      "activations/layer20_attention_weight_min": -26.734342575073242,
      "activations/layer21_attention_weight_max": 28.31930923461914,
      "activations/layer21_attention_weight_min": -26.543291091918945,
      "activations/layer22_attention_weight_max": 44.5630989074707,
      "activations/layer22_attention_weight_min": -30.27273178100586,
      "activations/layer23_attention_weight_max": 32.90019226074219,
      "activations/layer23_attention_weight_min": -26.09601402282715,
      "activations/layer2_attention_weight_max": 27.498456954956055,
      "activations/layer2_attention_weight_min": -24.95533561706543,
      "activations/layer3_attention_weight_max": 91.5501480102539,
      "activations/layer3_attention_weight_min": -83.30350494384766,
      "activations/layer4_attention_weight_max": 115.13968658447266,
      "activations/layer4_attention_weight_min": -106.71150970458984,
      "activations/layer5_attention_weight_max": 56.58039855957031,
      "activations/layer5_attention_weight_min": -65.90702056884766,
      "activations/layer6_attention_weight_max": 50.47051239013672,
      "activations/layer6_attention_weight_min": -51.47024154663086,
      "activations/layer7_attention_weight_max": 95.84322357177734,
      "activations/layer7_attention_weight_min": -94.74564361572266,
      "activations/layer8_attention_weight_max": 44.9223518371582,
      "activations/layer8_attention_weight_min": -48.05195617675781,
      "activations/layer9_attention_weight_max": 35.85300827026367,
      "activations/layer9_attention_weight_min": -35.908363342285156,
      "epoch": 10.44,
      "learning_rate": 8.350568181818182e-05,
      "loss": 2.7761,
      "step": 179700
    },
    {
      "activations/layer0_attention_weight_max": 15.661072731018066,
      "activations/layer0_attention_weight_min": -12.834144592285156,
      "activations/layer10_attention_weight_max": 36.79246520996094,
      "activations/layer10_attention_weight_min": -33.78982162475586,
      "activations/layer11_attention_weight_max": 32.91864013671875,
      "activations/layer11_attention_weight_min": -32.75885009765625,
      "activations/layer12_attention_weight_max": 24.505048751831055,
      "activations/layer12_attention_weight_min": -27.780227661132812,
      "activations/layer13_attention_weight_max": 39.380760192871094,
      "activations/layer13_attention_weight_min": -33.960777282714844,
      "activations/layer14_attention_weight_max": 41.56841278076172,
      "activations/layer14_attention_weight_min": -39.125267028808594,
      "activations/layer15_attention_weight_max": 35.44493865966797,
      "activations/layer15_attention_weight_min": -31.635211944580078,
      "activations/layer16_attention_weight_max": 31.198192596435547,
      "activations/layer16_attention_weight_min": -27.90542221069336,
      "activations/layer17_attention_weight_max": 29.999536514282227,
      "activations/layer17_attention_weight_min": -26.593103408813477,
      "activations/layer18_attention_weight_max": 30.48690414428711,
      "activations/layer18_attention_weight_min": -24.100936889648438,
      "activations/layer19_attention_weight_max": 38.8700065612793,
      "activations/layer19_attention_weight_min": -33.11289978027344,
      "activations/layer1_attention_weight_max": 16.412967681884766,
      "activations/layer1_attention_weight_min": -15.464662551879883,
      "activations/layer20_attention_weight_max": 30.918724060058594,
      "activations/layer20_attention_weight_min": -25.334787368774414,
      "activations/layer21_attention_weight_max": 30.621456146240234,
      "activations/layer21_attention_weight_min": -24.57167625427246,
      "activations/layer22_attention_weight_max": 48.04669952392578,
      "activations/layer22_attention_weight_min": -31.558345794677734,
      "activations/layer23_attention_weight_max": 35.901588439941406,
      "activations/layer23_attention_weight_min": -26.37379264831543,
      "activations/layer2_attention_weight_max": 33.4002799987793,
      "activations/layer2_attention_weight_min": -34.02016067504883,
      "activations/layer3_attention_weight_max": 100.36837005615234,
      "activations/layer3_attention_weight_min": -97.01148223876953,
      "activations/layer4_attention_weight_max": 118.75589752197266,
      "activations/layer4_attention_weight_min": -115.409423828125,
      "activations/layer5_attention_weight_max": 55.02252197265625,
      "activations/layer5_attention_weight_min": -63.27598190307617,
      "activations/layer6_attention_weight_max": 45.870933532714844,
      "activations/layer6_attention_weight_min": -48.5460090637207,
      "activations/layer7_attention_weight_max": 90.71363830566406,
      "activations/layer7_attention_weight_min": -100.89138793945312,
      "activations/layer8_attention_weight_max": 39.21193313598633,
      "activations/layer8_attention_weight_min": -42.54077911376953,
      "activations/layer9_attention_weight_max": 36.228431701660156,
      "activations/layer9_attention_weight_min": -33.504817962646484,
      "epoch": 10.44,
      "learning_rate": 8.348674242424241e-05,
      "loss": 2.7816,
      "step": 179750
    },
    {
      "activations/layer0_attention_weight_max": 16.10401725769043,
      "activations/layer0_attention_weight_min": -12.933635711669922,
      "activations/layer10_attention_weight_max": 33.77665710449219,
      "activations/layer10_attention_weight_min": -33.654579162597656,
      "activations/layer11_attention_weight_max": 34.48860168457031,
      "activations/layer11_attention_weight_min": -34.354827880859375,
      "activations/layer12_attention_weight_max": 26.428770065307617,
      "activations/layer12_attention_weight_min": -27.298429489135742,
      "activations/layer13_attention_weight_max": 40.931114196777344,
      "activations/layer13_attention_weight_min": -35.85010528564453,
      "activations/layer14_attention_weight_max": 46.53865432739258,
      "activations/layer14_attention_weight_min": -39.815887451171875,
      "activations/layer15_attention_weight_max": 37.05977249145508,
      "activations/layer15_attention_weight_min": -31.554189682006836,
      "activations/layer16_attention_weight_max": 31.219852447509766,
      "activations/layer16_attention_weight_min": -28.43327522277832,
      "activations/layer17_attention_weight_max": 30.124902725219727,
      "activations/layer17_attention_weight_min": -26.994089126586914,
      "activations/layer18_attention_weight_max": 31.242807388305664,
      "activations/layer18_attention_weight_min": -23.408672332763672,
      "activations/layer19_attention_weight_max": 32.99913024902344,
      "activations/layer19_attention_weight_min": -30.80735206604004,
      "activations/layer1_attention_weight_max": 17.904830932617188,
      "activations/layer1_attention_weight_min": -15.533329010009766,
      "activations/layer20_attention_weight_max": 28.03767967224121,
      "activations/layer20_attention_weight_min": -24.712512969970703,
      "activations/layer21_attention_weight_max": 28.3125,
      "activations/layer21_attention_weight_min": -24.39542579650879,
      "activations/layer22_attention_weight_max": 48.34228515625,
      "activations/layer22_attention_weight_min": -35.55250930786133,
      "activations/layer23_attention_weight_max": 33.833656311035156,
      "activations/layer23_attention_weight_min": -27.449819564819336,
      "activations/layer2_attention_weight_max": 33.95890426635742,
      "activations/layer2_attention_weight_min": -34.38125991821289,
      "activations/layer3_attention_weight_max": 91.49617004394531,
      "activations/layer3_attention_weight_min": -93.19964599609375,
      "activations/layer4_attention_weight_max": 112.88691711425781,
      "activations/layer4_attention_weight_min": -113.98526763916016,
      "activations/layer5_attention_weight_max": 53.454994201660156,
      "activations/layer5_attention_weight_min": -64.65744018554688,
      "activations/layer6_attention_weight_max": 45.47749328613281,
      "activations/layer6_attention_weight_min": -50.68378448486328,
      "activations/layer7_attention_weight_max": 91.84114837646484,
      "activations/layer7_attention_weight_min": -94.33250427246094,
      "activations/layer8_attention_weight_max": 40.29130935668945,
      "activations/layer8_attention_weight_min": -42.80659866333008,
      "activations/layer9_attention_weight_max": 34.36294174194336,
      "activations/layer9_attention_weight_min": -35.31781768798828,
      "epoch": 10.45,
      "learning_rate": 8.346780303030303e-05,
      "loss": 2.7779,
      "step": 179800
    },
    {
      "activations/layer0_attention_weight_max": 15.742942810058594,
      "activations/layer0_attention_weight_min": -12.558669090270996,
      "activations/layer10_attention_weight_max": 36.8302001953125,
      "activations/layer10_attention_weight_min": -35.198875427246094,
      "activations/layer11_attention_weight_max": 38.006343841552734,
      "activations/layer11_attention_weight_min": -35.39513397216797,
      "activations/layer12_attention_weight_max": 25.90526580810547,
      "activations/layer12_attention_weight_min": -27.166135787963867,
      "activations/layer13_attention_weight_max": 42.52125549316406,
      "activations/layer13_attention_weight_min": -34.976375579833984,
      "activations/layer14_attention_weight_max": 48.80046081542969,
      "activations/layer14_attention_weight_min": -39.473201751708984,
      "activations/layer15_attention_weight_max": 45.39762878417969,
      "activations/layer15_attention_weight_min": -34.61452102661133,
      "activations/layer16_attention_weight_max": 32.71353530883789,
      "activations/layer16_attention_weight_min": -29.820541381835938,
      "activations/layer17_attention_weight_max": 39.38814163208008,
      "activations/layer17_attention_weight_min": -27.16112518310547,
      "activations/layer18_attention_weight_max": 33.501007080078125,
      "activations/layer18_attention_weight_min": -23.373241424560547,
      "activations/layer19_attention_weight_max": 40.97629928588867,
      "activations/layer19_attention_weight_min": -32.99287414550781,
      "activations/layer1_attention_weight_max": 17.45475196838379,
      "activations/layer1_attention_weight_min": -14.851956367492676,
      "activations/layer20_attention_weight_max": 31.961034774780273,
      "activations/layer20_attention_weight_min": -27.950254440307617,
      "activations/layer21_attention_weight_max": 31.887052536010742,
      "activations/layer21_attention_weight_min": -26.13655662536621,
      "activations/layer22_attention_weight_max": 57.02119827270508,
      "activations/layer22_attention_weight_min": -32.978294372558594,
      "activations/layer23_attention_weight_max": 40.59474182128906,
      "activations/layer23_attention_weight_min": -25.608013153076172,
      "activations/layer2_attention_weight_max": 33.312400817871094,
      "activations/layer2_attention_weight_min": -33.54842758178711,
      "activations/layer3_attention_weight_max": 97.66107940673828,
      "activations/layer3_attention_weight_min": -98.1360855102539,
      "activations/layer4_attention_weight_max": 120.17424774169922,
      "activations/layer4_attention_weight_min": -116.11775207519531,
      "activations/layer5_attention_weight_max": 55.99420166015625,
      "activations/layer5_attention_weight_min": -63.4075927734375,
      "activations/layer6_attention_weight_max": 48.185813903808594,
      "activations/layer6_attention_weight_min": -51.19828414916992,
      "activations/layer7_attention_weight_max": 94.80265808105469,
      "activations/layer7_attention_weight_min": -100.90999603271484,
      "activations/layer8_attention_weight_max": 43.05321502685547,
      "activations/layer8_attention_weight_min": -43.88431930541992,
      "activations/layer9_attention_weight_max": 39.099178314208984,
      "activations/layer9_attention_weight_min": -35.83597946166992,
      "epoch": 10.45,
      "learning_rate": 8.344886363636363e-05,
      "loss": 2.7694,
      "step": 179850
    },
    {
      "activations/layer0_attention_weight_max": 15.796290397644043,
      "activations/layer0_attention_weight_min": -11.385489463806152,
      "activations/layer10_attention_weight_max": 32.4781608581543,
      "activations/layer10_attention_weight_min": -30.824819564819336,
      "activations/layer11_attention_weight_max": 30.623374938964844,
      "activations/layer11_attention_weight_min": -33.357940673828125,
      "activations/layer12_attention_weight_max": 24.899372100830078,
      "activations/layer12_attention_weight_min": -27.353971481323242,
      "activations/layer13_attention_weight_max": 36.50349807739258,
      "activations/layer13_attention_weight_min": -35.97273254394531,
      "activations/layer14_attention_weight_max": 45.67814636230469,
      "activations/layer14_attention_weight_min": -42.68706512451172,
      "activations/layer15_attention_weight_max": 36.08230209350586,
      "activations/layer15_attention_weight_min": -32.4843635559082,
      "activations/layer16_attention_weight_max": 33.258426666259766,
      "activations/layer16_attention_weight_min": -29.507659912109375,
      "activations/layer17_attention_weight_max": 34.6063232421875,
      "activations/layer17_attention_weight_min": -25.114017486572266,
      "activations/layer18_attention_weight_max": 31.65025520324707,
      "activations/layer18_attention_weight_min": -22.31072235107422,
      "activations/layer19_attention_weight_max": 38.993289947509766,
      "activations/layer19_attention_weight_min": -31.3013973236084,
      "activations/layer1_attention_weight_max": 16.733566284179688,
      "activations/layer1_attention_weight_min": -15.16620922088623,
      "activations/layer20_attention_weight_max": 30.960081100463867,
      "activations/layer20_attention_weight_min": -24.923160552978516,
      "activations/layer21_attention_weight_max": 30.265188217163086,
      "activations/layer21_attention_weight_min": -23.513608932495117,
      "activations/layer22_attention_weight_max": 46.45500564575195,
      "activations/layer22_attention_weight_min": -30.317745208740234,
      "activations/layer23_attention_weight_max": 37.3297004699707,
      "activations/layer23_attention_weight_min": -25.415328979492188,
      "activations/layer2_attention_weight_max": 33.50651550292969,
      "activations/layer2_attention_weight_min": -32.67443084716797,
      "activations/layer3_attention_weight_max": 95.22003173828125,
      "activations/layer3_attention_weight_min": -101.0987319946289,
      "activations/layer4_attention_weight_max": 111.91837310791016,
      "activations/layer4_attention_weight_min": -112.3033218383789,
      "activations/layer5_attention_weight_max": 53.689857482910156,
      "activations/layer5_attention_weight_min": -63.92550277709961,
      "activations/layer6_attention_weight_max": 45.846900939941406,
      "activations/layer6_attention_weight_min": -48.38829803466797,
      "activations/layer7_attention_weight_max": 93.79168701171875,
      "activations/layer7_attention_weight_min": -93.36078643798828,
      "activations/layer8_attention_weight_max": 42.288333892822266,
      "activations/layer8_attention_weight_min": -40.95885467529297,
      "activations/layer9_attention_weight_max": 31.64739990234375,
      "activations/layer9_attention_weight_min": -32.26710510253906,
      "epoch": 10.45,
      "learning_rate": 8.342992424242423e-05,
      "loss": 2.7734,
      "step": 179900
    },
    {
      "activations/layer0_attention_weight_max": 16.077911376953125,
      "activations/layer0_attention_weight_min": -12.283708572387695,
      "activations/layer10_attention_weight_max": 36.366336822509766,
      "activations/layer10_attention_weight_min": -32.83418655395508,
      "activations/layer11_attention_weight_max": 36.70478057861328,
      "activations/layer11_attention_weight_min": -31.543834686279297,
      "activations/layer12_attention_weight_max": 25.205652236938477,
      "activations/layer12_attention_weight_min": -27.295581817626953,
      "activations/layer13_attention_weight_max": 38.4340705871582,
      "activations/layer13_attention_weight_min": -37.4169921875,
      "activations/layer14_attention_weight_max": 52.060340881347656,
      "activations/layer14_attention_weight_min": -39.46564483642578,
      "activations/layer15_attention_weight_max": 37.1260871887207,
      "activations/layer15_attention_weight_min": -34.21220779418945,
      "activations/layer16_attention_weight_max": 33.29545974731445,
      "activations/layer16_attention_weight_min": -27.410085678100586,
      "activations/layer17_attention_weight_max": 29.42818832397461,
      "activations/layer17_attention_weight_min": -26.560293197631836,
      "activations/layer18_attention_weight_max": 31.392704010009766,
      "activations/layer18_attention_weight_min": -22.77260398864746,
      "activations/layer19_attention_weight_max": 35.65056610107422,
      "activations/layer19_attention_weight_min": -32.22492218017578,
      "activations/layer1_attention_weight_max": 17.054454803466797,
      "activations/layer1_attention_weight_min": -13.995094299316406,
      "activations/layer20_attention_weight_max": 29.132600784301758,
      "activations/layer20_attention_weight_min": -25.157541275024414,
      "activations/layer21_attention_weight_max": 31.548870086669922,
      "activations/layer21_attention_weight_min": -25.885589599609375,
      "activations/layer22_attention_weight_max": 47.42405319213867,
      "activations/layer22_attention_weight_min": -30.48052215576172,
      "activations/layer23_attention_weight_max": 32.56684112548828,
      "activations/layer23_attention_weight_min": -25.3289794921875,
      "activations/layer2_attention_weight_max": 35.02088928222656,
      "activations/layer2_attention_weight_min": -36.575782775878906,
      "activations/layer3_attention_weight_max": 105.54408264160156,
      "activations/layer3_attention_weight_min": -108.51345825195312,
      "activations/layer4_attention_weight_max": 120.22784423828125,
      "activations/layer4_attention_weight_min": -116.32125091552734,
      "activations/layer5_attention_weight_max": 55.76125717163086,
      "activations/layer5_attention_weight_min": -59.850337982177734,
      "activations/layer6_attention_weight_max": 47.60295486450195,
      "activations/layer6_attention_weight_min": -47.84690475463867,
      "activations/layer7_attention_weight_max": 98.5459213256836,
      "activations/layer7_attention_weight_min": -93.49610900878906,
      "activations/layer8_attention_weight_max": 43.453792572021484,
      "activations/layer8_attention_weight_min": -42.35406494140625,
      "activations/layer9_attention_weight_max": 33.894039154052734,
      "activations/layer9_attention_weight_min": -33.1278190612793,
      "epoch": 10.46,
      "learning_rate": 8.341098484848484e-05,
      "loss": 2.7704,
      "step": 179950
    },
    {
      "activations/layer0_attention_weight_max": 15.722817420959473,
      "activations/layer0_attention_weight_min": -12.889511108398438,
      "activations/layer10_attention_weight_max": 31.71286392211914,
      "activations/layer10_attention_weight_min": -30.950260162353516,
      "activations/layer11_attention_weight_max": 29.983518600463867,
      "activations/layer11_attention_weight_min": -29.785940170288086,
      "activations/layer12_attention_weight_max": 27.43525505065918,
      "activations/layer12_attention_weight_min": -23.59477996826172,
      "activations/layer13_attention_weight_max": 35.25653839111328,
      "activations/layer13_attention_weight_min": -32.75706481933594,
      "activations/layer14_attention_weight_max": 41.90553283691406,
      "activations/layer14_attention_weight_min": -34.847442626953125,
      "activations/layer15_attention_weight_max": 34.676307678222656,
      "activations/layer15_attention_weight_min": -30.163591384887695,
      "activations/layer16_attention_weight_max": 31.578474044799805,
      "activations/layer16_attention_weight_min": -27.439861297607422,
      "activations/layer17_attention_weight_max": 32.76604080200195,
      "activations/layer17_attention_weight_min": -24.24211883544922,
      "activations/layer18_attention_weight_max": 31.510038375854492,
      "activations/layer18_attention_weight_min": -21.993032455444336,
      "activations/layer19_attention_weight_max": 34.645591735839844,
      "activations/layer19_attention_weight_min": -29.506698608398438,
      "activations/layer1_attention_weight_max": 16.9564266204834,
      "activations/layer1_attention_weight_min": -16.25265121459961,
      "activations/layer20_attention_weight_max": 30.47450828552246,
      "activations/layer20_attention_weight_min": -23.32390022277832,
      "activations/layer21_attention_weight_max": 28.418853759765625,
      "activations/layer21_attention_weight_min": -22.731109619140625,
      "activations/layer22_attention_weight_max": 41.00446319580078,
      "activations/layer22_attention_weight_min": -27.82895851135254,
      "activations/layer23_attention_weight_max": 33.695404052734375,
      "activations/layer23_attention_weight_min": -23.878124237060547,
      "activations/layer2_attention_weight_max": 32.12853240966797,
      "activations/layer2_attention_weight_min": -31.920452117919922,
      "activations/layer3_attention_weight_max": 89.3851318359375,
      "activations/layer3_attention_weight_min": -93.56731414794922,
      "activations/layer4_attention_weight_max": 112.46031951904297,
      "activations/layer4_attention_weight_min": -105.93737030029297,
      "activations/layer5_attention_weight_max": 51.12959289550781,
      "activations/layer5_attention_weight_min": -58.99308776855469,
      "activations/layer6_attention_weight_max": 43.344905853271484,
      "activations/layer6_attention_weight_min": -47.03070831298828,
      "activations/layer7_attention_weight_max": 86.86399841308594,
      "activations/layer7_attention_weight_min": -87.66236114501953,
      "activations/layer8_attention_weight_max": 39.709354400634766,
      "activations/layer8_attention_weight_min": -39.89020919799805,
      "activations/layer9_attention_weight_max": 32.413116455078125,
      "activations/layer9_attention_weight_min": -30.628915786743164,
      "epoch": 10.46,
      "learning_rate": 8.339204545454545e-05,
      "loss": 2.777,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.555,
      "eval_samples_per_second": 501.932,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.555,
      "eval_openwebtext_samples_per_second": 501.932,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 2.0208,
      "eval_wikitext_samples_per_second": 225.656,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_lambada_loss": 2.486328125,
      "eval_lambada_ppl": 12.01706983184115,
      "eval_lambada_runtime": 9.653,
      "eval_lambada_samples_per_second": 504.403,
      "step": 180000
    },
    {
      "activations/layer0_attention_weight_max": 16.018966674804688,
      "activations/layer0_attention_weight_min": -12.973615646362305,
      "activations/layer10_attention_weight_max": 37.59610366821289,
      "activations/layer10_attention_weight_min": -34.4459342956543,
      "activations/layer11_attention_weight_max": 34.78911209106445,
      "activations/layer11_attention_weight_min": -34.29500198364258,
      "activations/layer12_attention_weight_max": 29.361120223999023,
      "activations/layer12_attention_weight_min": -25.093042373657227,
      "activations/layer13_attention_weight_max": 39.37242126464844,
      "activations/layer13_attention_weight_min": -33.33351135253906,
      "activations/layer14_attention_weight_max": 43.553863525390625,
      "activations/layer14_attention_weight_min": -35.94790267944336,
      "activations/layer15_attention_weight_max": 40.09718322753906,
      "activations/layer15_attention_weight_min": -31.235260009765625,
      "activations/layer16_attention_weight_max": 32.35111618041992,
      "activations/layer16_attention_weight_min": -26.893932342529297,
      "activations/layer17_attention_weight_max": 35.6630973815918,
      "activations/layer17_attention_weight_min": -25.074697494506836,
      "activations/layer18_attention_weight_max": 32.2457160949707,
      "activations/layer18_attention_weight_min": -23.81093978881836,
      "activations/layer19_attention_weight_max": 37.00999069213867,
      "activations/layer19_attention_weight_min": -30.832782745361328,
      "activations/layer1_attention_weight_max": 16.963850021362305,
      "activations/layer1_attention_weight_min": -15.10718059539795,
      "activations/layer20_attention_weight_max": 30.366992950439453,
      "activations/layer20_attention_weight_min": -25.910781860351562,
      "activations/layer21_attention_weight_max": 26.94582176208496,
      "activations/layer21_attention_weight_min": -23.45255470275879,
      "activations/layer22_attention_weight_max": 45.49905776977539,
      "activations/layer22_attention_weight_min": -30.17371368408203,
      "activations/layer23_attention_weight_max": 35.22114562988281,
      "activations/layer23_attention_weight_min": -24.139949798583984,
      "activations/layer2_attention_weight_max": 35.08347702026367,
      "activations/layer2_attention_weight_min": -34.31689453125,
      "activations/layer3_attention_weight_max": 95.60401153564453,
      "activations/layer3_attention_weight_min": -97.17571258544922,
      "activations/layer4_attention_weight_max": 115.3886947631836,
      "activations/layer4_attention_weight_min": -117.20809173583984,
      "activations/layer5_attention_weight_max": 54.75282287597656,
      "activations/layer5_attention_weight_min": -60.000640869140625,
      "activations/layer6_attention_weight_max": 46.813209533691406,
      "activations/layer6_attention_weight_min": -50.83157730102539,
      "activations/layer7_attention_weight_max": 98.23233032226562,
      "activations/layer7_attention_weight_min": -92.32551574707031,
      "activations/layer8_attention_weight_max": 45.539100646972656,
      "activations/layer8_attention_weight_min": -45.00294876098633,
      "activations/layer9_attention_weight_max": 39.53580093383789,
      "activations/layer9_attention_weight_min": -35.97846984863281,
      "epoch": 10.46,
      "learning_rate": 8.337310606060606e-05,
      "loss": 2.7676,
      "step": 180050
    },
    {
      "activations/layer0_attention_weight_max": 15.99074935913086,
      "activations/layer0_attention_weight_min": -13.681076049804688,
      "activations/layer10_attention_weight_max": 30.693700790405273,
      "activations/layer10_attention_weight_min": -33.519195556640625,
      "activations/layer11_attention_weight_max": 29.72413444519043,
      "activations/layer11_attention_weight_min": -30.763303756713867,
      "activations/layer12_attention_weight_max": 24.786890029907227,
      "activations/layer12_attention_weight_min": -25.191049575805664,
      "activations/layer13_attention_weight_max": 37.342254638671875,
      "activations/layer13_attention_weight_min": -34.1401252746582,
      "activations/layer14_attention_weight_max": 45.597076416015625,
      "activations/layer14_attention_weight_min": -40.317535400390625,
      "activations/layer15_attention_weight_max": 32.92074203491211,
      "activations/layer15_attention_weight_min": -32.06796646118164,
      "activations/layer16_attention_weight_max": 28.995820999145508,
      "activations/layer16_attention_weight_min": -28.52159309387207,
      "activations/layer17_attention_weight_max": 31.88728904724121,
      "activations/layer17_attention_weight_min": -28.374420166015625,
      "activations/layer18_attention_weight_max": 31.666357040405273,
      "activations/layer18_attention_weight_min": -23.145431518554688,
      "activations/layer19_attention_weight_max": 32.6185417175293,
      "activations/layer19_attention_weight_min": -30.37021255493164,
      "activations/layer1_attention_weight_max": 15.770322799682617,
      "activations/layer1_attention_weight_min": -15.446563720703125,
      "activations/layer20_attention_weight_max": 31.77415657043457,
      "activations/layer20_attention_weight_min": -24.953433990478516,
      "activations/layer21_attention_weight_max": 33.980712890625,
      "activations/layer21_attention_weight_min": -24.039915084838867,
      "activations/layer22_attention_weight_max": 42.811546325683594,
      "activations/layer22_attention_weight_min": -31.36936378479004,
      "activations/layer23_attention_weight_max": 31.73746109008789,
      "activations/layer23_attention_weight_min": -24.21356773376465,
      "activations/layer2_attention_weight_max": 31.453411102294922,
      "activations/layer2_attention_weight_min": -32.946800231933594,
      "activations/layer3_attention_weight_max": 87.3587646484375,
      "activations/layer3_attention_weight_min": -89.43800354003906,
      "activations/layer4_attention_weight_max": 103.4431381225586,
      "activations/layer4_attention_weight_min": -108.5360336303711,
      "activations/layer5_attention_weight_max": 50.4447135925293,
      "activations/layer5_attention_weight_min": -59.231475830078125,
      "activations/layer6_attention_weight_max": 44.256935119628906,
      "activations/layer6_attention_weight_min": -47.010520935058594,
      "activations/layer7_attention_weight_max": 84.65031433105469,
      "activations/layer7_attention_weight_min": -85.78555297851562,
      "activations/layer8_attention_weight_max": 38.652591705322266,
      "activations/layer8_attention_weight_min": -41.801414489746094,
      "activations/layer9_attention_weight_max": 32.342864990234375,
      "activations/layer9_attention_weight_min": -35.499210357666016,
      "epoch": 10.46,
      "learning_rate": 8.335416666666666e-05,
      "loss": 2.7747,
      "step": 180100
    },
    {
      "activations/layer0_attention_weight_max": 16.351043701171875,
      "activations/layer0_attention_weight_min": -12.534012794494629,
      "activations/layer10_attention_weight_max": 31.787607192993164,
      "activations/layer10_attention_weight_min": -34.00135803222656,
      "activations/layer11_attention_weight_max": 29.30738639831543,
      "activations/layer11_attention_weight_min": -32.74669647216797,
      "activations/layer12_attention_weight_max": 24.682661056518555,
      "activations/layer12_attention_weight_min": -25.048667907714844,
      "activations/layer13_attention_weight_max": 37.25115203857422,
      "activations/layer13_attention_weight_min": -34.76890182495117,
      "activations/layer14_attention_weight_max": 42.01845932006836,
      "activations/layer14_attention_weight_min": -39.368858337402344,
      "activations/layer15_attention_weight_max": 33.42959976196289,
      "activations/layer15_attention_weight_min": -33.29926681518555,
      "activations/layer16_attention_weight_max": 31.174510955810547,
      "activations/layer16_attention_weight_min": -28.628541946411133,
      "activations/layer17_attention_weight_max": 32.52934646606445,
      "activations/layer17_attention_weight_min": -25.579700469970703,
      "activations/layer18_attention_weight_max": 29.693336486816406,
      "activations/layer18_attention_weight_min": -22.671842575073242,
      "activations/layer19_attention_weight_max": 32.08247375488281,
      "activations/layer19_attention_weight_min": -31.775766372680664,
      "activations/layer1_attention_weight_max": 16.473665237426758,
      "activations/layer1_attention_weight_min": -16.534164428710938,
      "activations/layer20_attention_weight_max": 28.689712524414062,
      "activations/layer20_attention_weight_min": -25.924060821533203,
      "activations/layer21_attention_weight_max": 26.282268524169922,
      "activations/layer21_attention_weight_min": -23.473875045776367,
      "activations/layer22_attention_weight_max": 40.02396774291992,
      "activations/layer22_attention_weight_min": -29.635536193847656,
      "activations/layer23_attention_weight_max": 30.338123321533203,
      "activations/layer23_attention_weight_min": -24.055397033691406,
      "activations/layer2_attention_weight_max": 31.905378341674805,
      "activations/layer2_attention_weight_min": -31.41067123413086,
      "activations/layer3_attention_weight_max": 86.71320343017578,
      "activations/layer3_attention_weight_min": -90.70770263671875,
      "activations/layer4_attention_weight_max": 106.91191864013672,
      "activations/layer4_attention_weight_min": -105.67974090576172,
      "activations/layer5_attention_weight_max": 50.97357177734375,
      "activations/layer5_attention_weight_min": -61.46946716308594,
      "activations/layer6_attention_weight_max": 42.73514175415039,
      "activations/layer6_attention_weight_min": -46.00605010986328,
      "activations/layer7_attention_weight_max": 86.51930236816406,
      "activations/layer7_attention_weight_min": -85.96326446533203,
      "activations/layer8_attention_weight_max": 36.399314880371094,
      "activations/layer8_attention_weight_min": -39.856754302978516,
      "activations/layer9_attention_weight_max": 31.183542251586914,
      "activations/layer9_attention_weight_min": -34.350830078125,
      "epoch": 10.47,
      "learning_rate": 8.333522727272726e-05,
      "loss": 2.7666,
      "step": 180150
    },
    {
      "activations/layer0_attention_weight_max": 16.073076248168945,
      "activations/layer0_attention_weight_min": -12.250642776489258,
      "activations/layer10_attention_weight_max": 36.762351989746094,
      "activations/layer10_attention_weight_min": -34.10609817504883,
      "activations/layer11_attention_weight_max": 35.452518463134766,
      "activations/layer11_attention_weight_min": -32.4955940246582,
      "activations/layer12_attention_weight_max": 23.95709991455078,
      "activations/layer12_attention_weight_min": -25.417194366455078,
      "activations/layer13_attention_weight_max": 40.78297805786133,
      "activations/layer13_attention_weight_min": -36.47728729248047,
      "activations/layer14_attention_weight_max": 44.92934799194336,
      "activations/layer14_attention_weight_min": -37.01310729980469,
      "activations/layer15_attention_weight_max": 38.87018966674805,
      "activations/layer15_attention_weight_min": -33.75462341308594,
      "activations/layer16_attention_weight_max": 30.760461807250977,
      "activations/layer16_attention_weight_min": -27.220535278320312,
      "activations/layer17_attention_weight_max": 29.469160079956055,
      "activations/layer17_attention_weight_min": -27.119186401367188,
      "activations/layer18_attention_weight_max": 29.311857223510742,
      "activations/layer18_attention_weight_min": -25.336021423339844,
      "activations/layer19_attention_weight_max": 35.53218460083008,
      "activations/layer19_attention_weight_min": -32.16687774658203,
      "activations/layer1_attention_weight_max": 16.66827964782715,
      "activations/layer1_attention_weight_min": -16.095495223999023,
      "activations/layer20_attention_weight_max": 29.017026901245117,
      "activations/layer20_attention_weight_min": -25.04463768005371,
      "activations/layer21_attention_weight_max": 30.683834075927734,
      "activations/layer21_attention_weight_min": -25.144287109375,
      "activations/layer22_attention_weight_max": 41.83737564086914,
      "activations/layer22_attention_weight_min": -31.142107009887695,
      "activations/layer23_attention_weight_max": 32.157676696777344,
      "activations/layer23_attention_weight_min": -23.95578384399414,
      "activations/layer2_attention_weight_max": 32.01646423339844,
      "activations/layer2_attention_weight_min": -33.594581604003906,
      "activations/layer3_attention_weight_max": 96.3730697631836,
      "activations/layer3_attention_weight_min": -95.98453521728516,
      "activations/layer4_attention_weight_max": 112.58715057373047,
      "activations/layer4_attention_weight_min": -115.4156265258789,
      "activations/layer5_attention_weight_max": 55.136192321777344,
      "activations/layer5_attention_weight_min": -64.24388122558594,
      "activations/layer6_attention_weight_max": 46.65047836303711,
      "activations/layer6_attention_weight_min": -47.877403259277344,
      "activations/layer7_attention_weight_max": 94.43208312988281,
      "activations/layer7_attention_weight_min": -90.9977798461914,
      "activations/layer8_attention_weight_max": 43.64228820800781,
      "activations/layer8_attention_weight_min": -40.39474105834961,
      "activations/layer9_attention_weight_max": 37.09449005126953,
      "activations/layer9_attention_weight_min": -33.898189544677734,
      "epoch": 10.47,
      "learning_rate": 8.331628787878788e-05,
      "loss": 2.7789,
      "step": 180200
    },
    {
      "activations/layer0_attention_weight_max": 16.142236709594727,
      "activations/layer0_attention_weight_min": -12.442678451538086,
      "activations/layer10_attention_weight_max": 34.34668731689453,
      "activations/layer10_attention_weight_min": -32.30800247192383,
      "activations/layer11_attention_weight_max": 33.17817687988281,
      "activations/layer11_attention_weight_min": -29.875707626342773,
      "activations/layer12_attention_weight_max": 24.498458862304688,
      "activations/layer12_attention_weight_min": -28.528364181518555,
      "activations/layer13_attention_weight_max": 39.08878707885742,
      "activations/layer13_attention_weight_min": -32.2500114440918,
      "activations/layer14_attention_weight_max": 44.908905029296875,
      "activations/layer14_attention_weight_min": -37.78997039794922,
      "activations/layer15_attention_weight_max": 36.591758728027344,
      "activations/layer15_attention_weight_min": -32.89015579223633,
      "activations/layer16_attention_weight_max": 32.193546295166016,
      "activations/layer16_attention_weight_min": -27.37474822998047,
      "activations/layer17_attention_weight_max": 32.678096771240234,
      "activations/layer17_attention_weight_min": -25.53194236755371,
      "activations/layer18_attention_weight_max": 30.54873275756836,
      "activations/layer18_attention_weight_min": -22.837566375732422,
      "activations/layer19_attention_weight_max": 40.48881912231445,
      "activations/layer19_attention_weight_min": -29.554447174072266,
      "activations/layer1_attention_weight_max": 16.85826873779297,
      "activations/layer1_attention_weight_min": -14.622525215148926,
      "activations/layer20_attention_weight_max": 29.341567993164062,
      "activations/layer20_attention_weight_min": -24.757535934448242,
      "activations/layer21_attention_weight_max": 31.769359588623047,
      "activations/layer21_attention_weight_min": -23.816762924194336,
      "activations/layer22_attention_weight_max": 48.07658767700195,
      "activations/layer22_attention_weight_min": -29.16787338256836,
      "activations/layer23_attention_weight_max": 39.13500213623047,
      "activations/layer23_attention_weight_min": -25.39423370361328,
      "activations/layer2_attention_weight_max": 30.697385787963867,
      "activations/layer2_attention_weight_min": -32.140167236328125,
      "activations/layer3_attention_weight_max": 91.26437377929688,
      "activations/layer3_attention_weight_min": -92.97618865966797,
      "activations/layer4_attention_weight_max": 109.8497543334961,
      "activations/layer4_attention_weight_min": -108.47032928466797,
      "activations/layer5_attention_weight_max": 56.440185546875,
      "activations/layer5_attention_weight_min": -61.89641571044922,
      "activations/layer6_attention_weight_max": 44.704410552978516,
      "activations/layer6_attention_weight_min": -48.12565612792969,
      "activations/layer7_attention_weight_max": 88.15159606933594,
      "activations/layer7_attention_weight_min": -85.06302642822266,
      "activations/layer8_attention_weight_max": 39.754119873046875,
      "activations/layer8_attention_weight_min": -39.1338996887207,
      "activations/layer9_attention_weight_max": 32.22101974487305,
      "activations/layer9_attention_weight_min": -32.060577392578125,
      "epoch": 10.47,
      "learning_rate": 8.329734848484847e-05,
      "loss": 2.7793,
      "step": 180250
    },
    {
      "activations/layer0_attention_weight_max": 16.286794662475586,
      "activations/layer0_attention_weight_min": -13.351293563842773,
      "activations/layer10_attention_weight_max": 30.004894256591797,
      "activations/layer10_attention_weight_min": -31.148265838623047,
      "activations/layer11_attention_weight_max": 30.53030776977539,
      "activations/layer11_attention_weight_min": -29.646011352539062,
      "activations/layer12_attention_weight_max": 23.070066452026367,
      "activations/layer12_attention_weight_min": -24.475318908691406,
      "activations/layer13_attention_weight_max": 35.191471099853516,
      "activations/layer13_attention_weight_min": -32.73591995239258,
      "activations/layer14_attention_weight_max": 40.829532623291016,
      "activations/layer14_attention_weight_min": -36.41599655151367,
      "activations/layer15_attention_weight_max": 37.37076187133789,
      "activations/layer15_attention_weight_min": -30.56195640563965,
      "activations/layer16_attention_weight_max": 32.69429016113281,
      "activations/layer16_attention_weight_min": -27.34112548828125,
      "activations/layer17_attention_weight_max": 31.639421463012695,
      "activations/layer17_attention_weight_min": -24.925107955932617,
      "activations/layer18_attention_weight_max": 29.90510368347168,
      "activations/layer18_attention_weight_min": -23.317546844482422,
      "activations/layer19_attention_weight_max": 31.335758209228516,
      "activations/layer19_attention_weight_min": -29.634124755859375,
      "activations/layer1_attention_weight_max": 15.953401565551758,
      "activations/layer1_attention_weight_min": -16.567380905151367,
      "activations/layer20_attention_weight_max": 27.514657974243164,
      "activations/layer20_attention_weight_min": -24.548721313476562,
      "activations/layer21_attention_weight_max": 28.712738037109375,
      "activations/layer21_attention_weight_min": -24.163890838623047,
      "activations/layer22_attention_weight_max": 44.050697326660156,
      "activations/layer22_attention_weight_min": -31.95856285095215,
      "activations/layer23_attention_weight_max": 32.468101501464844,
      "activations/layer23_attention_weight_min": -23.943510055541992,
      "activations/layer2_attention_weight_max": 31.145780563354492,
      "activations/layer2_attention_weight_min": -32.238521575927734,
      "activations/layer3_attention_weight_max": 86.25727844238281,
      "activations/layer3_attention_weight_min": -91.61181640625,
      "activations/layer4_attention_weight_max": 106.10611724853516,
      "activations/layer4_attention_weight_min": -110.5533447265625,
      "activations/layer5_attention_weight_max": 51.138328552246094,
      "activations/layer5_attention_weight_min": -61.16520690917969,
      "activations/layer6_attention_weight_max": 43.318397521972656,
      "activations/layer6_attention_weight_min": -46.94496536254883,
      "activations/layer7_attention_weight_max": 82.4361572265625,
      "activations/layer7_attention_weight_min": -85.11685180664062,
      "activations/layer8_attention_weight_max": 37.90247344970703,
      "activations/layer8_attention_weight_min": -40.90773391723633,
      "activations/layer9_attention_weight_max": 32.38418960571289,
      "activations/layer9_attention_weight_min": -33.19523239135742,
      "epoch": 10.48,
      "learning_rate": 8.327840909090908e-05,
      "loss": 2.7671,
      "step": 180300
    },
    {
      "activations/layer0_attention_weight_max": 15.940987586975098,
      "activations/layer0_attention_weight_min": -13.08387565612793,
      "activations/layer10_attention_weight_max": 33.300575256347656,
      "activations/layer10_attention_weight_min": -34.41306686401367,
      "activations/layer11_attention_weight_max": 34.34284973144531,
      "activations/layer11_attention_weight_min": -33.44621658325195,
      "activations/layer12_attention_weight_max": 28.643550872802734,
      "activations/layer12_attention_weight_min": -26.70504379272461,
      "activations/layer13_attention_weight_max": 47.58872604370117,
      "activations/layer13_attention_weight_min": -36.75566864013672,
      "activations/layer14_attention_weight_max": 60.68585968017578,
      "activations/layer14_attention_weight_min": -46.093780517578125,
      "activations/layer15_attention_weight_max": 44.22399139404297,
      "activations/layer15_attention_weight_min": -35.94099807739258,
      "activations/layer16_attention_weight_max": 38.43695831298828,
      "activations/layer16_attention_weight_min": -29.210603713989258,
      "activations/layer17_attention_weight_max": 37.36394119262695,
      "activations/layer17_attention_weight_min": -25.6623477935791,
      "activations/layer18_attention_weight_max": 36.988922119140625,
      "activations/layer18_attention_weight_min": -22.374202728271484,
      "activations/layer19_attention_weight_max": 43.378719329833984,
      "activations/layer19_attention_weight_min": -32.2039909362793,
      "activations/layer1_attention_weight_max": 19.882549285888672,
      "activations/layer1_attention_weight_min": -17.817697525024414,
      "activations/layer20_attention_weight_max": 34.103492736816406,
      "activations/layer20_attention_weight_min": -25.48229217529297,
      "activations/layer21_attention_weight_max": 32.45001983642578,
      "activations/layer21_attention_weight_min": -23.25619125366211,
      "activations/layer22_attention_weight_max": 55.14168930053711,
      "activations/layer22_attention_weight_min": -31.556196212768555,
      "activations/layer23_attention_weight_max": 41.13512420654297,
      "activations/layer23_attention_weight_min": -25.994977951049805,
      "activations/layer2_attention_weight_max": 31.206134796142578,
      "activations/layer2_attention_weight_min": -31.276151657104492,
      "activations/layer3_attention_weight_max": 93.64405059814453,
      "activations/layer3_attention_weight_min": -91.1118392944336,
      "activations/layer4_attention_weight_max": 111.57137298583984,
      "activations/layer4_attention_weight_min": -108.1864242553711,
      "activations/layer5_attention_weight_max": 52.16632843017578,
      "activations/layer5_attention_weight_min": -62.88670349121094,
      "activations/layer6_attention_weight_max": 47.80210876464844,
      "activations/layer6_attention_weight_min": -47.489707946777344,
      "activations/layer7_attention_weight_max": 89.66105651855469,
      "activations/layer7_attention_weight_min": -87.0676498413086,
      "activations/layer8_attention_weight_max": 39.58843231201172,
      "activations/layer8_attention_weight_min": -40.21570587158203,
      "activations/layer9_attention_weight_max": 34.410499572753906,
      "activations/layer9_attention_weight_min": -33.533756256103516,
      "epoch": 10.48,
      "learning_rate": 8.32594696969697e-05,
      "loss": 2.788,
      "step": 180350
    },
    {
      "activations/layer0_attention_weight_max": 15.556119918823242,
      "activations/layer0_attention_weight_min": -13.038545608520508,
      "activations/layer10_attention_weight_max": 33.094947814941406,
      "activations/layer10_attention_weight_min": -34.18417739868164,
      "activations/layer11_attention_weight_max": 32.48693084716797,
      "activations/layer11_attention_weight_min": -32.11723709106445,
      "activations/layer12_attention_weight_max": 24.355289459228516,
      "activations/layer12_attention_weight_min": -26.925525665283203,
      "activations/layer13_attention_weight_max": 35.40873336791992,
      "activations/layer13_attention_weight_min": -35.531333923339844,
      "activations/layer14_attention_weight_max": 39.94417190551758,
      "activations/layer14_attention_weight_min": -37.89535903930664,
      "activations/layer15_attention_weight_max": 34.9360237121582,
      "activations/layer15_attention_weight_min": -30.352052688598633,
      "activations/layer16_attention_weight_max": 31.14014434814453,
      "activations/layer16_attention_weight_min": -26.85927391052246,
      "activations/layer17_attention_weight_max": 30.456424713134766,
      "activations/layer17_attention_weight_min": -24.466453552246094,
      "activations/layer18_attention_weight_max": 34.10380172729492,
      "activations/layer18_attention_weight_min": -23.07882308959961,
      "activations/layer19_attention_weight_max": 36.113155364990234,
      "activations/layer19_attention_weight_min": -29.5233154296875,
      "activations/layer1_attention_weight_max": 17.433210372924805,
      "activations/layer1_attention_weight_min": -15.704277038574219,
      "activations/layer20_attention_weight_max": 28.138755798339844,
      "activations/layer20_attention_weight_min": -24.162038803100586,
      "activations/layer21_attention_weight_max": 26.201852798461914,
      "activations/layer21_attention_weight_min": -23.418773651123047,
      "activations/layer22_attention_weight_max": 39.772762298583984,
      "activations/layer22_attention_weight_min": -29.353195190429688,
      "activations/layer23_attention_weight_max": 33.64717102050781,
      "activations/layer23_attention_weight_min": -24.92821502685547,
      "activations/layer2_attention_weight_max": 34.08219909667969,
      "activations/layer2_attention_weight_min": -34.397403717041016,
      "activations/layer3_attention_weight_max": 94.5369644165039,
      "activations/layer3_attention_weight_min": -95.099365234375,
      "activations/layer4_attention_weight_max": 113.66812896728516,
      "activations/layer4_attention_weight_min": -111.88875579833984,
      "activations/layer5_attention_weight_max": 55.83207702636719,
      "activations/layer5_attention_weight_min": -61.72872543334961,
      "activations/layer6_attention_weight_max": 44.631141662597656,
      "activations/layer6_attention_weight_min": -50.62134552001953,
      "activations/layer7_attention_weight_max": 90.579833984375,
      "activations/layer7_attention_weight_min": -98.56458282470703,
      "activations/layer8_attention_weight_max": 40.567256927490234,
      "activations/layer8_attention_weight_min": -42.90153884887695,
      "activations/layer9_attention_weight_max": 33.93782043457031,
      "activations/layer9_attention_weight_min": -35.44605255126953,
      "epoch": 10.48,
      "learning_rate": 8.324053030303028e-05,
      "loss": 2.7865,
      "step": 180400
    },
    {
      "activations/layer0_attention_weight_max": 16.344987869262695,
      "activations/layer0_attention_weight_min": -12.733112335205078,
      "activations/layer10_attention_weight_max": 34.8906135559082,
      "activations/layer10_attention_weight_min": -34.47819900512695,
      "activations/layer11_attention_weight_max": 35.22435760498047,
      "activations/layer11_attention_weight_min": -33.266334533691406,
      "activations/layer12_attention_weight_max": 24.188003540039062,
      "activations/layer12_attention_weight_min": -25.71181869506836,
      "activations/layer13_attention_weight_max": 38.35122299194336,
      "activations/layer13_attention_weight_min": -37.928749084472656,
      "activations/layer14_attention_weight_max": 43.28972244262695,
      "activations/layer14_attention_weight_min": -39.94719314575195,
      "activations/layer15_attention_weight_max": 36.188899993896484,
      "activations/layer15_attention_weight_min": -32.461238861083984,
      "activations/layer16_attention_weight_max": 33.566558837890625,
      "activations/layer16_attention_weight_min": -27.650121688842773,
      "activations/layer17_attention_weight_max": 31.40426254272461,
      "activations/layer17_attention_weight_min": -27.326675415039062,
      "activations/layer18_attention_weight_max": 33.17515182495117,
      "activations/layer18_attention_weight_min": -24.510761260986328,
      "activations/layer19_attention_weight_max": 36.51264572143555,
      "activations/layer19_attention_weight_min": -31.531564712524414,
      "activations/layer1_attention_weight_max": 18.147254943847656,
      "activations/layer1_attention_weight_min": -15.764191627502441,
      "activations/layer20_attention_weight_max": 31.240116119384766,
      "activations/layer20_attention_weight_min": -27.068687438964844,
      "activations/layer21_attention_weight_max": 31.499711990356445,
      "activations/layer21_attention_weight_min": -24.997350692749023,
      "activations/layer22_attention_weight_max": 41.8594970703125,
      "activations/layer22_attention_weight_min": -32.85975646972656,
      "activations/layer23_attention_weight_max": 41.80232238769531,
      "activations/layer23_attention_weight_min": -25.467933654785156,
      "activations/layer2_attention_weight_max": 34.460601806640625,
      "activations/layer2_attention_weight_min": -32.67890548706055,
      "activations/layer3_attention_weight_max": 100.11598205566406,
      "activations/layer3_attention_weight_min": -94.49402618408203,
      "activations/layer4_attention_weight_max": 115.4585952758789,
      "activations/layer4_attention_weight_min": -114.04437255859375,
      "activations/layer5_attention_weight_max": 54.403770446777344,
      "activations/layer5_attention_weight_min": -66.41963195800781,
      "activations/layer6_attention_weight_max": 47.51963806152344,
      "activations/layer6_attention_weight_min": -49.53849792480469,
      "activations/layer7_attention_weight_max": 94.84475708007812,
      "activations/layer7_attention_weight_min": -94.62061309814453,
      "activations/layer8_attention_weight_max": 43.91731262207031,
      "activations/layer8_attention_weight_min": -44.011295318603516,
      "activations/layer9_attention_weight_max": 35.63499069213867,
      "activations/layer9_attention_weight_min": -36.21992111206055,
      "epoch": 10.49,
      "learning_rate": 8.32215909090909e-05,
      "loss": 2.775,
      "step": 180450
    },
    {
      "activations/layer0_attention_weight_max": 16.278522491455078,
      "activations/layer0_attention_weight_min": -13.659363746643066,
      "activations/layer10_attention_weight_max": 35.07721710205078,
      "activations/layer10_attention_weight_min": -32.81522750854492,
      "activations/layer11_attention_weight_max": 33.15654373168945,
      "activations/layer11_attention_weight_min": -32.28547286987305,
      "activations/layer12_attention_weight_max": 23.874866485595703,
      "activations/layer12_attention_weight_min": -24.659271240234375,
      "activations/layer13_attention_weight_max": 39.226959228515625,
      "activations/layer13_attention_weight_min": -35.18681335449219,
      "activations/layer14_attention_weight_max": 42.31545639038086,
      "activations/layer14_attention_weight_min": -36.98826217651367,
      "activations/layer15_attention_weight_max": 35.83991622924805,
      "activations/layer15_attention_weight_min": -32.24448776245117,
      "activations/layer16_attention_weight_max": 31.235979080200195,
      "activations/layer16_attention_weight_min": -27.3681583404541,
      "activations/layer17_attention_weight_max": 30.77942657470703,
      "activations/layer17_attention_weight_min": -26.8939151763916,
      "activations/layer18_attention_weight_max": 29.743404388427734,
      "activations/layer18_attention_weight_min": -23.628463745117188,
      "activations/layer19_attention_weight_max": 35.99735641479492,
      "activations/layer19_attention_weight_min": -30.753875732421875,
      "activations/layer1_attention_weight_max": 17.87982177734375,
      "activations/layer1_attention_weight_min": -17.6907901763916,
      "activations/layer20_attention_weight_max": 27.612354278564453,
      "activations/layer20_attention_weight_min": -24.933879852294922,
      "activations/layer21_attention_weight_max": 27.532989501953125,
      "activations/layer21_attention_weight_min": -25.148191452026367,
      "activations/layer22_attention_weight_max": 41.18841552734375,
      "activations/layer22_attention_weight_min": -29.520912170410156,
      "activations/layer23_attention_weight_max": 38.28127670288086,
      "activations/layer23_attention_weight_min": -25.126541137695312,
      "activations/layer2_attention_weight_max": 33.63970184326172,
      "activations/layer2_attention_weight_min": -34.494239807128906,
      "activations/layer3_attention_weight_max": 97.18283081054688,
      "activations/layer3_attention_weight_min": -96.23605346679688,
      "activations/layer4_attention_weight_max": 117.20106506347656,
      "activations/layer4_attention_weight_min": -117.420166015625,
      "activations/layer5_attention_weight_max": 55.87409210205078,
      "activations/layer5_attention_weight_min": -66.25819396972656,
      "activations/layer6_attention_weight_max": 47.06474685668945,
      "activations/layer6_attention_weight_min": -50.29096603393555,
      "activations/layer7_attention_weight_max": 94.5831527709961,
      "activations/layer7_attention_weight_min": -88.8087387084961,
      "activations/layer8_attention_weight_max": 42.058162689208984,
      "activations/layer8_attention_weight_min": -41.60923385620117,
      "activations/layer9_attention_weight_max": 36.53432083129883,
      "activations/layer9_attention_weight_min": -33.265316009521484,
      "epoch": 10.49,
      "learning_rate": 8.320265151515151e-05,
      "loss": 2.785,
      "step": 180500
    },
    {
      "activations/layer0_attention_weight_max": 16.02499008178711,
      "activations/layer0_attention_weight_min": -11.867369651794434,
      "activations/layer10_attention_weight_max": 42.81184005737305,
      "activations/layer10_attention_weight_min": -37.12421798706055,
      "activations/layer11_attention_weight_max": 43.758689880371094,
      "activations/layer11_attention_weight_min": -38.386905670166016,
      "activations/layer12_attention_weight_max": 36.57474136352539,
      "activations/layer12_attention_weight_min": -29.463077545166016,
      "activations/layer13_attention_weight_max": 64.10543823242188,
      "activations/layer13_attention_weight_min": -44.502174377441406,
      "activations/layer14_attention_weight_max": 69.39773559570312,
      "activations/layer14_attention_weight_min": -47.72356033325195,
      "activations/layer15_attention_weight_max": 64.23751068115234,
      "activations/layer15_attention_weight_min": -38.93940734863281,
      "activations/layer16_attention_weight_max": 49.60721206665039,
      "activations/layer16_attention_weight_min": -29.883150100708008,
      "activations/layer17_attention_weight_max": 39.592979431152344,
      "activations/layer17_attention_weight_min": -28.420835494995117,
      "activations/layer18_attention_weight_max": 38.28986740112305,
      "activations/layer18_attention_weight_min": -24.156959533691406,
      "activations/layer19_attention_weight_max": 47.87363815307617,
      "activations/layer19_attention_weight_min": -32.85600280761719,
      "activations/layer1_attention_weight_max": 18.561607360839844,
      "activations/layer1_attention_weight_min": -15.590332984924316,
      "activations/layer20_attention_weight_max": 34.695884704589844,
      "activations/layer20_attention_weight_min": -24.727460861206055,
      "activations/layer21_attention_weight_max": 30.261058807373047,
      "activations/layer21_attention_weight_min": -23.4829044342041,
      "activations/layer22_attention_weight_max": 56.27524185180664,
      "activations/layer22_attention_weight_min": -30.759660720825195,
      "activations/layer23_attention_weight_max": 36.34544372558594,
      "activations/layer23_attention_weight_min": -23.95615005493164,
      "activations/layer2_attention_weight_max": 34.654090881347656,
      "activations/layer2_attention_weight_min": -33.99432373046875,
      "activations/layer3_attention_weight_max": 99.43815612792969,
      "activations/layer3_attention_weight_min": -99.99971008300781,
      "activations/layer4_attention_weight_max": 112.52921295166016,
      "activations/layer4_attention_weight_min": -115.283935546875,
      "activations/layer5_attention_weight_max": 56.607582092285156,
      "activations/layer5_attention_weight_min": -61.88070297241211,
      "activations/layer6_attention_weight_max": 44.2180290222168,
      "activations/layer6_attention_weight_min": -48.03205108642578,
      "activations/layer7_attention_weight_max": 99.11598205566406,
      "activations/layer7_attention_weight_min": -89.94975280761719,
      "activations/layer8_attention_weight_max": 44.174434661865234,
      "activations/layer8_attention_weight_min": -43.22158432006836,
      "activations/layer9_attention_weight_max": 40.381893157958984,
      "activations/layer9_attention_weight_min": -36.0390739440918,
      "epoch": 10.49,
      "learning_rate": 8.31837121212121e-05,
      "loss": 2.757,
      "step": 180550
    },
    {
      "activations/layer0_attention_weight_max": 15.233604431152344,
      "activations/layer0_attention_weight_min": -13.235788345336914,
      "activations/layer10_attention_weight_max": 33.19879913330078,
      "activations/layer10_attention_weight_min": -33.703392028808594,
      "activations/layer11_attention_weight_max": 34.21371841430664,
      "activations/layer11_attention_weight_min": -33.77967071533203,
      "activations/layer12_attention_weight_max": 25.53622817993164,
      "activations/layer12_attention_weight_min": -25.958942413330078,
      "activations/layer13_attention_weight_max": 39.59357833862305,
      "activations/layer13_attention_weight_min": -33.92982482910156,
      "activations/layer14_attention_weight_max": 44.06315612792969,
      "activations/layer14_attention_weight_min": -36.762271881103516,
      "activations/layer15_attention_weight_max": 34.72169494628906,
      "activations/layer15_attention_weight_min": -32.35869598388672,
      "activations/layer16_attention_weight_max": 29.33696746826172,
      "activations/layer16_attention_weight_min": -27.936689376831055,
      "activations/layer17_attention_weight_max": 30.181344985961914,
      "activations/layer17_attention_weight_min": -26.191028594970703,
      "activations/layer18_attention_weight_max": 30.195829391479492,
      "activations/layer18_attention_weight_min": -23.22354507446289,
      "activations/layer19_attention_weight_max": 36.1447639465332,
      "activations/layer19_attention_weight_min": -31.877262115478516,
      "activations/layer1_attention_weight_max": 21.29570198059082,
      "activations/layer1_attention_weight_min": -16.404062271118164,
      "activations/layer20_attention_weight_max": 29.174114227294922,
      "activations/layer20_attention_weight_min": -26.115739822387695,
      "activations/layer21_attention_weight_max": 28.741369247436523,
      "activations/layer21_attention_weight_min": -24.725059509277344,
      "activations/layer22_attention_weight_max": 48.794334411621094,
      "activations/layer22_attention_weight_min": -30.469173431396484,
      "activations/layer23_attention_weight_max": 36.938968658447266,
      "activations/layer23_attention_weight_min": -24.428241729736328,
      "activations/layer2_attention_weight_max": 34.14646911621094,
      "activations/layer2_attention_weight_min": -34.605064392089844,
      "activations/layer3_attention_weight_max": 101.86253356933594,
      "activations/layer3_attention_weight_min": -97.59896850585938,
      "activations/layer4_attention_weight_max": 118.39583587646484,
      "activations/layer4_attention_weight_min": -114.72505950927734,
      "activations/layer5_attention_weight_max": 56.60205078125,
      "activations/layer5_attention_weight_min": -64.52161407470703,
      "activations/layer6_attention_weight_max": 46.88321304321289,
      "activations/layer6_attention_weight_min": -49.651512145996094,
      "activations/layer7_attention_weight_max": 93.463623046875,
      "activations/layer7_attention_weight_min": -88.6409683227539,
      "activations/layer8_attention_weight_max": 42.196250915527344,
      "activations/layer8_attention_weight_min": -40.72812271118164,
      "activations/layer9_attention_weight_max": 33.689815521240234,
      "activations/layer9_attention_weight_min": -34.6609001159668,
      "epoch": 10.49,
      "learning_rate": 8.316477272727272e-05,
      "loss": 2.7727,
      "step": 180600
    },
    {
      "activations/layer0_attention_weight_max": 16.3842716217041,
      "activations/layer0_attention_weight_min": -12.881790161132812,
      "activations/layer10_attention_weight_max": 35.39400100708008,
      "activations/layer10_attention_weight_min": -34.4505729675293,
      "activations/layer11_attention_weight_max": 38.536537170410156,
      "activations/layer11_attention_weight_min": -35.88859176635742,
      "activations/layer12_attention_weight_max": 27.246679306030273,
      "activations/layer12_attention_weight_min": -24.170988082885742,
      "activations/layer13_attention_weight_max": 43.52311706542969,
      "activations/layer13_attention_weight_min": -34.93563461303711,
      "activations/layer14_attention_weight_max": 55.97666931152344,
      "activations/layer14_attention_weight_min": -41.99140548706055,
      "activations/layer15_attention_weight_max": 41.986488342285156,
      "activations/layer15_attention_weight_min": -33.2818717956543,
      "activations/layer16_attention_weight_max": 39.43646240234375,
      "activations/layer16_attention_weight_min": -25.80055046081543,
      "activations/layer17_attention_weight_max": 38.79977035522461,
      "activations/layer17_attention_weight_min": -25.27242088317871,
      "activations/layer18_attention_weight_max": 32.399147033691406,
      "activations/layer18_attention_weight_min": -23.950054168701172,
      "activations/layer19_attention_weight_max": 41.397796630859375,
      "activations/layer19_attention_weight_min": -31.63702392578125,
      "activations/layer1_attention_weight_max": 17.10064125061035,
      "activations/layer1_attention_weight_min": -15.123272895812988,
      "activations/layer20_attention_weight_max": 35.972129821777344,
      "activations/layer20_attention_weight_min": -26.50047492980957,
      "activations/layer21_attention_weight_max": 31.04738998413086,
      "activations/layer21_attention_weight_min": -25.530405044555664,
      "activations/layer22_attention_weight_max": 50.95908737182617,
      "activations/layer22_attention_weight_min": -30.44544219970703,
      "activations/layer23_attention_weight_max": 36.25157928466797,
      "activations/layer23_attention_weight_min": -28.099994659423828,
      "activations/layer2_attention_weight_max": 33.09037780761719,
      "activations/layer2_attention_weight_min": -31.59990692138672,
      "activations/layer3_attention_weight_max": 93.2576904296875,
      "activations/layer3_attention_weight_min": -96.7098388671875,
      "activations/layer4_attention_weight_max": 113.14607238769531,
      "activations/layer4_attention_weight_min": -115.6006088256836,
      "activations/layer5_attention_weight_max": 55.05242156982422,
      "activations/layer5_attention_weight_min": -65.34776306152344,
      "activations/layer6_attention_weight_max": 51.258811950683594,
      "activations/layer6_attention_weight_min": -49.336246490478516,
      "activations/layer7_attention_weight_max": 96.01152038574219,
      "activations/layer7_attention_weight_min": -93.58625793457031,
      "activations/layer8_attention_weight_max": 42.87778854370117,
      "activations/layer8_attention_weight_min": -42.93742752075195,
      "activations/layer9_attention_weight_max": 35.78483200073242,
      "activations/layer9_attention_weight_min": -34.35261535644531,
      "epoch": 10.5,
      "learning_rate": 8.314583333333333e-05,
      "loss": 2.7772,
      "step": 180650
    },
    {
      "activations/layer0_attention_weight_max": 15.698426246643066,
      "activations/layer0_attention_weight_min": -12.825614929199219,
      "activations/layer10_attention_weight_max": 30.153282165527344,
      "activations/layer10_attention_weight_min": -30.488555908203125,
      "activations/layer11_attention_weight_max": 31.298307418823242,
      "activations/layer11_attention_weight_min": -28.933828353881836,
      "activations/layer12_attention_weight_max": 23.554365158081055,
      "activations/layer12_attention_weight_min": -27.266401290893555,
      "activations/layer13_attention_weight_max": 36.54206848144531,
      "activations/layer13_attention_weight_min": -35.8106803894043,
      "activations/layer14_attention_weight_max": 44.20486068725586,
      "activations/layer14_attention_weight_min": -37.84574508666992,
      "activations/layer15_attention_weight_max": 34.56658935546875,
      "activations/layer15_attention_weight_min": -33.12858200073242,
      "activations/layer16_attention_weight_max": 28.79433822631836,
      "activations/layer16_attention_weight_min": -27.19096565246582,
      "activations/layer17_attention_weight_max": 29.41636848449707,
      "activations/layer17_attention_weight_min": -25.217769622802734,
      "activations/layer18_attention_weight_max": 29.344993591308594,
      "activations/layer18_attention_weight_min": -25.516544342041016,
      "activations/layer19_attention_weight_max": 30.929594039916992,
      "activations/layer19_attention_weight_min": -31.940725326538086,
      "activations/layer1_attention_weight_max": 16.398523330688477,
      "activations/layer1_attention_weight_min": -14.200989723205566,
      "activations/layer20_attention_weight_max": 27.190471649169922,
      "activations/layer20_attention_weight_min": -25.624975204467773,
      "activations/layer21_attention_weight_max": 27.003337860107422,
      "activations/layer21_attention_weight_min": -24.462993621826172,
      "activations/layer22_attention_weight_max": 45.291229248046875,
      "activations/layer22_attention_weight_min": -31.832082748413086,
      "activations/layer23_attention_weight_max": 35.11530303955078,
      "activations/layer23_attention_weight_min": -26.103574752807617,
      "activations/layer2_attention_weight_max": 31.71307373046875,
      "activations/layer2_attention_weight_min": -32.275447845458984,
      "activations/layer3_attention_weight_max": 89.45320892333984,
      "activations/layer3_attention_weight_min": -90.37340545654297,
      "activations/layer4_attention_weight_max": 109.75691986083984,
      "activations/layer4_attention_weight_min": -105.61053466796875,
      "activations/layer5_attention_weight_max": 51.65796661376953,
      "activations/layer5_attention_weight_min": -61.20556640625,
      "activations/layer6_attention_weight_max": 41.95632553100586,
      "activations/layer6_attention_weight_min": -46.63200759887695,
      "activations/layer7_attention_weight_max": 91.57660675048828,
      "activations/layer7_attention_weight_min": -87.32849884033203,
      "activations/layer8_attention_weight_max": 36.96638488769531,
      "activations/layer8_attention_weight_min": -44.108882904052734,
      "activations/layer9_attention_weight_max": 31.48528480529785,
      "activations/layer9_attention_weight_min": -33.15347671508789,
      "epoch": 10.5,
      "learning_rate": 8.312689393939393e-05,
      "loss": 2.7909,
      "step": 180700
    },
    {
      "activations/layer0_attention_weight_max": 14.636263847351074,
      "activations/layer0_attention_weight_min": -12.232869148254395,
      "activations/layer10_attention_weight_max": 31.80832862854004,
      "activations/layer10_attention_weight_min": -30.851572036743164,
      "activations/layer11_attention_weight_max": 33.070011138916016,
      "activations/layer11_attention_weight_min": -32.34983444213867,
      "activations/layer12_attention_weight_max": 24.811601638793945,
      "activations/layer12_attention_weight_min": -24.63010597229004,
      "activations/layer13_attention_weight_max": 40.67576217651367,
      "activations/layer13_attention_weight_min": -34.651912689208984,
      "activations/layer14_attention_weight_max": 46.85897445678711,
      "activations/layer14_attention_weight_min": -39.305030822753906,
      "activations/layer15_attention_weight_max": 37.1428108215332,
      "activations/layer15_attention_weight_min": -32.23692321777344,
      "activations/layer16_attention_weight_max": 30.230802536010742,
      "activations/layer16_attention_weight_min": -28.981489181518555,
      "activations/layer17_attention_weight_max": 31.16407585144043,
      "activations/layer17_attention_weight_min": -27.226022720336914,
      "activations/layer18_attention_weight_max": 31.296852111816406,
      "activations/layer18_attention_weight_min": -23.288272857666016,
      "activations/layer19_attention_weight_max": 33.34674072265625,
      "activations/layer19_attention_weight_min": -31.8420352935791,
      "activations/layer1_attention_weight_max": 17.13813591003418,
      "activations/layer1_attention_weight_min": -14.141548156738281,
      "activations/layer20_attention_weight_max": 28.5666446685791,
      "activations/layer20_attention_weight_min": -25.44152069091797,
      "activations/layer21_attention_weight_max": 30.03391456604004,
      "activations/layer21_attention_weight_min": -24.470914840698242,
      "activations/layer22_attention_weight_max": 47.96595764160156,
      "activations/layer22_attention_weight_min": -31.47995376586914,
      "activations/layer23_attention_weight_max": 38.73139953613281,
      "activations/layer23_attention_weight_min": -24.241458892822266,
      "activations/layer2_attention_weight_max": 34.12997055053711,
      "activations/layer2_attention_weight_min": -31.92407989501953,
      "activations/layer3_attention_weight_max": 92.81514739990234,
      "activations/layer3_attention_weight_min": -94.36953735351562,
      "activations/layer4_attention_weight_max": 107.45238494873047,
      "activations/layer4_attention_weight_min": -107.35637664794922,
      "activations/layer5_attention_weight_max": 51.25133514404297,
      "activations/layer5_attention_weight_min": -61.834651947021484,
      "activations/layer6_attention_weight_max": 47.90943145751953,
      "activations/layer6_attention_weight_min": -47.023250579833984,
      "activations/layer7_attention_weight_max": 95.44772338867188,
      "activations/layer7_attention_weight_min": -90.7027587890625,
      "activations/layer8_attention_weight_max": 42.943389892578125,
      "activations/layer8_attention_weight_min": -38.80031967163086,
      "activations/layer9_attention_weight_max": 33.16084671020508,
      "activations/layer9_attention_weight_min": -32.26968002319336,
      "epoch": 10.5,
      "learning_rate": 8.310795454545454e-05,
      "loss": 2.7808,
      "step": 180750
    },
    {
      "activations/layer0_attention_weight_max": 16.553537368774414,
      "activations/layer0_attention_weight_min": -12.644583702087402,
      "activations/layer10_attention_weight_max": 34.908206939697266,
      "activations/layer10_attention_weight_min": -31.912302017211914,
      "activations/layer11_attention_weight_max": 32.61974334716797,
      "activations/layer11_attention_weight_min": -30.697818756103516,
      "activations/layer12_attention_weight_max": 24.00660514831543,
      "activations/layer12_attention_weight_min": -24.798629760742188,
      "activations/layer13_attention_weight_max": 36.738826751708984,
      "activations/layer13_attention_weight_min": -34.50019454956055,
      "activations/layer14_attention_weight_max": 41.796485900878906,
      "activations/layer14_attention_weight_min": -37.27262878417969,
      "activations/layer15_attention_weight_max": 35.908077239990234,
      "activations/layer15_attention_weight_min": -33.265602111816406,
      "activations/layer16_attention_weight_max": 30.853939056396484,
      "activations/layer16_attention_weight_min": -29.447586059570312,
      "activations/layer17_attention_weight_max": 30.409358978271484,
      "activations/layer17_attention_weight_min": -25.708057403564453,
      "activations/layer18_attention_weight_max": 31.368486404418945,
      "activations/layer18_attention_weight_min": -24.0648250579834,
      "activations/layer19_attention_weight_max": 35.402645111083984,
      "activations/layer19_attention_weight_min": -30.7879695892334,
      "activations/layer1_attention_weight_max": 16.730772018432617,
      "activations/layer1_attention_weight_min": -13.576444625854492,
      "activations/layer20_attention_weight_max": 30.101694107055664,
      "activations/layer20_attention_weight_min": -24.613847732543945,
      "activations/layer21_attention_weight_max": 29.568859100341797,
      "activations/layer21_attention_weight_min": -23.283458709716797,
      "activations/layer22_attention_weight_max": 51.46767807006836,
      "activations/layer22_attention_weight_min": -30.122604370117188,
      "activations/layer23_attention_weight_max": 35.20833206176758,
      "activations/layer23_attention_weight_min": -25.10402488708496,
      "activations/layer2_attention_weight_max": 32.4908332824707,
      "activations/layer2_attention_weight_min": -33.25020217895508,
      "activations/layer3_attention_weight_max": 94.13629150390625,
      "activations/layer3_attention_weight_min": -96.14921569824219,
      "activations/layer4_attention_weight_max": 110.82894134521484,
      "activations/layer4_attention_weight_min": -111.07460021972656,
      "activations/layer5_attention_weight_max": 52.49207305908203,
      "activations/layer5_attention_weight_min": -61.382469177246094,
      "activations/layer6_attention_weight_max": 44.622371673583984,
      "activations/layer6_attention_weight_min": -47.65237045288086,
      "activations/layer7_attention_weight_max": 88.20348358154297,
      "activations/layer7_attention_weight_min": -86.77240753173828,
      "activations/layer8_attention_weight_max": 41.11756896972656,
      "activations/layer8_attention_weight_min": -40.28968048095703,
      "activations/layer9_attention_weight_max": 36.653564453125,
      "activations/layer9_attention_weight_min": -35.02936935424805,
      "epoch": 10.51,
      "learning_rate": 8.308901515151514e-05,
      "loss": 2.7693,
      "step": 180800
    },
    {
      "activations/layer0_attention_weight_max": 15.703441619873047,
      "activations/layer0_attention_weight_min": -12.559954643249512,
      "activations/layer10_attention_weight_max": 30.8848876953125,
      "activations/layer10_attention_weight_min": -35.697776794433594,
      "activations/layer11_attention_weight_max": 32.552223205566406,
      "activations/layer11_attention_weight_min": -31.821651458740234,
      "activations/layer12_attention_weight_max": 24.774696350097656,
      "activations/layer12_attention_weight_min": -25.557798385620117,
      "activations/layer13_attention_weight_max": 39.95891189575195,
      "activations/layer13_attention_weight_min": -32.32710647583008,
      "activations/layer14_attention_weight_max": 40.54019546508789,
      "activations/layer14_attention_weight_min": -36.460411071777344,
      "activations/layer15_attention_weight_max": 34.61824035644531,
      "activations/layer15_attention_weight_min": -31.27587127685547,
      "activations/layer16_attention_weight_max": 31.383800506591797,
      "activations/layer16_attention_weight_min": -28.33020782470703,
      "activations/layer17_attention_weight_max": 29.33343505859375,
      "activations/layer17_attention_weight_min": -24.26906394958496,
      "activations/layer18_attention_weight_max": 32.16145324707031,
      "activations/layer18_attention_weight_min": -24.0124454498291,
      "activations/layer19_attention_weight_max": 35.14457702636719,
      "activations/layer19_attention_weight_min": -30.072484970092773,
      "activations/layer1_attention_weight_max": 16.014617919921875,
      "activations/layer1_attention_weight_min": -14.629530906677246,
      "activations/layer20_attention_weight_max": 26.565715789794922,
      "activations/layer20_attention_weight_min": -24.468578338623047,
      "activations/layer21_attention_weight_max": 26.418624877929688,
      "activations/layer21_attention_weight_min": -22.68572425842285,
      "activations/layer22_attention_weight_max": 43.91289520263672,
      "activations/layer22_attention_weight_min": -27.690109252929688,
      "activations/layer23_attention_weight_max": 36.419097900390625,
      "activations/layer23_attention_weight_min": -24.243698120117188,
      "activations/layer2_attention_weight_max": 34.01551818847656,
      "activations/layer2_attention_weight_min": -33.7823486328125,
      "activations/layer3_attention_weight_max": 101.72962951660156,
      "activations/layer3_attention_weight_min": -99.33800506591797,
      "activations/layer4_attention_weight_max": 116.0353775024414,
      "activations/layer4_attention_weight_min": -119.0359878540039,
      "activations/layer5_attention_weight_max": 52.81874084472656,
      "activations/layer5_attention_weight_min": -62.035972595214844,
      "activations/layer6_attention_weight_max": 45.23972702026367,
      "activations/layer6_attention_weight_min": -51.68452453613281,
      "activations/layer7_attention_weight_max": 85.9332504272461,
      "activations/layer7_attention_weight_min": -88.5578384399414,
      "activations/layer8_attention_weight_max": 39.288970947265625,
      "activations/layer8_attention_weight_min": -42.53163146972656,
      "activations/layer9_attention_weight_max": 33.06874465942383,
      "activations/layer9_attention_weight_min": -36.11124801635742,
      "epoch": 10.51,
      "learning_rate": 8.307007575757575e-05,
      "loss": 2.7699,
      "step": 180850
    },
    {
      "activations/layer0_attention_weight_max": 16.035808563232422,
      "activations/layer0_attention_weight_min": -12.707504272460938,
      "activations/layer10_attention_weight_max": 34.51625442504883,
      "activations/layer10_attention_weight_min": -34.830772399902344,
      "activations/layer11_attention_weight_max": 35.622257232666016,
      "activations/layer11_attention_weight_min": -33.757720947265625,
      "activations/layer12_attention_weight_max": 27.887937545776367,
      "activations/layer12_attention_weight_min": -25.60397720336914,
      "activations/layer13_attention_weight_max": 43.48439407348633,
      "activations/layer13_attention_weight_min": -33.16888427734375,
      "activations/layer14_attention_weight_max": 49.993980407714844,
      "activations/layer14_attention_weight_min": -36.70341491699219,
      "activations/layer15_attention_weight_max": 43.95403289794922,
      "activations/layer15_attention_weight_min": -32.39339828491211,
      "activations/layer16_attention_weight_max": 35.4099235534668,
      "activations/layer16_attention_weight_min": -29.39399528503418,
      "activations/layer17_attention_weight_max": 35.658138275146484,
      "activations/layer17_attention_weight_min": -25.284082412719727,
      "activations/layer18_attention_weight_max": 34.57401657104492,
      "activations/layer18_attention_weight_min": -23.577455520629883,
      "activations/layer19_attention_weight_max": 44.81123352050781,
      "activations/layer19_attention_weight_min": -31.34751319885254,
      "activations/layer1_attention_weight_max": 16.269798278808594,
      "activations/layer1_attention_weight_min": -15.016822814941406,
      "activations/layer20_attention_weight_max": 34.80036544799805,
      "activations/layer20_attention_weight_min": -24.504337310791016,
      "activations/layer21_attention_weight_max": 35.222904205322266,
      "activations/layer21_attention_weight_min": -23.85849380493164,
      "activations/layer22_attention_weight_max": 57.635005950927734,
      "activations/layer22_attention_weight_min": -30.39954948425293,
      "activations/layer23_attention_weight_max": 44.92489242553711,
      "activations/layer23_attention_weight_min": -24.552684783935547,
      "activations/layer2_attention_weight_max": 32.9991569519043,
      "activations/layer2_attention_weight_min": -35.42842483520508,
      "activations/layer3_attention_weight_max": 98.9640884399414,
      "activations/layer3_attention_weight_min": -98.98017883300781,
      "activations/layer4_attention_weight_max": 119.1249008178711,
      "activations/layer4_attention_weight_min": -115.01029205322266,
      "activations/layer5_attention_weight_max": 54.574363708496094,
      "activations/layer5_attention_weight_min": -63.043914794921875,
      "activations/layer6_attention_weight_max": 46.94912338256836,
      "activations/layer6_attention_weight_min": -49.434303283691406,
      "activations/layer7_attention_weight_max": 96.27465057373047,
      "activations/layer7_attention_weight_min": -92.59329986572266,
      "activations/layer8_attention_weight_max": 42.83706283569336,
      "activations/layer8_attention_weight_min": -42.240211486816406,
      "activations/layer9_attention_weight_max": 36.12813949584961,
      "activations/layer9_attention_weight_min": -34.64397430419922,
      "epoch": 10.51,
      "learning_rate": 8.305113636363637e-05,
      "loss": 2.7725,
      "step": 180900
    },
    {
      "activations/layer0_attention_weight_max": 16.17694854736328,
      "activations/layer0_attention_weight_min": -12.81574821472168,
      "activations/layer10_attention_weight_max": 31.52544403076172,
      "activations/layer10_attention_weight_min": -32.190277099609375,
      "activations/layer11_attention_weight_max": 34.360206604003906,
      "activations/layer11_attention_weight_min": -32.37142562866211,
      "activations/layer12_attention_weight_max": 22.785303115844727,
      "activations/layer12_attention_weight_min": -24.68462562561035,
      "activations/layer13_attention_weight_max": 36.01648712158203,
      "activations/layer13_attention_weight_min": -31.272850036621094,
      "activations/layer14_attention_weight_max": 41.84047317504883,
      "activations/layer14_attention_weight_min": -36.97813415527344,
      "activations/layer15_attention_weight_max": 41.49551773071289,
      "activations/layer15_attention_weight_min": -30.059680938720703,
      "activations/layer16_attention_weight_max": 28.171220779418945,
      "activations/layer16_attention_weight_min": -25.831676483154297,
      "activations/layer17_attention_weight_max": 30.66996955871582,
      "activations/layer17_attention_weight_min": -25.842456817626953,
      "activations/layer18_attention_weight_max": 29.76888656616211,
      "activations/layer18_attention_weight_min": -23.505393981933594,
      "activations/layer19_attention_weight_max": 33.438926696777344,
      "activations/layer19_attention_weight_min": -29.299726486206055,
      "activations/layer1_attention_weight_max": 16.589937210083008,
      "activations/layer1_attention_weight_min": -14.369439125061035,
      "activations/layer20_attention_weight_max": 26.301176071166992,
      "activations/layer20_attention_weight_min": -23.770727157592773,
      "activations/layer21_attention_weight_max": 28.33051109313965,
      "activations/layer21_attention_weight_min": -23.09190559387207,
      "activations/layer22_attention_weight_max": 41.709774017333984,
      "activations/layer22_attention_weight_min": -28.48623275756836,
      "activations/layer23_attention_weight_max": 31.571468353271484,
      "activations/layer23_attention_weight_min": -25.019676208496094,
      "activations/layer2_attention_weight_max": 33.427589416503906,
      "activations/layer2_attention_weight_min": -32.543617248535156,
      "activations/layer3_attention_weight_max": 94.57122802734375,
      "activations/layer3_attention_weight_min": -97.63288116455078,
      "activations/layer4_attention_weight_max": 111.90770721435547,
      "activations/layer4_attention_weight_min": -110.92203521728516,
      "activations/layer5_attention_weight_max": 52.20968246459961,
      "activations/layer5_attention_weight_min": -59.522605895996094,
      "activations/layer6_attention_weight_max": 45.02981185913086,
      "activations/layer6_attention_weight_min": -46.5964469909668,
      "activations/layer7_attention_weight_max": 92.54398345947266,
      "activations/layer7_attention_weight_min": -88.81130981445312,
      "activations/layer8_attention_weight_max": 40.5290641784668,
      "activations/layer8_attention_weight_min": -39.42986297607422,
      "activations/layer9_attention_weight_max": 34.94874572753906,
      "activations/layer9_attention_weight_min": -33.83480453491211,
      "epoch": 10.51,
      "learning_rate": 8.303219696969696e-05,
      "loss": 2.785,
      "step": 180950
    },
    {
      "activations/layer0_attention_weight_max": 15.808585166931152,
      "activations/layer0_attention_weight_min": -11.76081371307373,
      "activations/layer10_attention_weight_max": 34.39765548706055,
      "activations/layer10_attention_weight_min": -33.2819938659668,
      "activations/layer11_attention_weight_max": 34.245216369628906,
      "activations/layer11_attention_weight_min": -32.98197937011719,
      "activations/layer12_attention_weight_max": 26.888534545898438,
      "activations/layer12_attention_weight_min": -24.243799209594727,
      "activations/layer13_attention_weight_max": 46.079402923583984,
      "activations/layer13_attention_weight_min": -33.420806884765625,
      "activations/layer14_attention_weight_max": 51.644203186035156,
      "activations/layer14_attention_weight_min": -38.53261184692383,
      "activations/layer15_attention_weight_max": 39.06379318237305,
      "activations/layer15_attention_weight_min": -30.955535888671875,
      "activations/layer16_attention_weight_max": 32.904022216796875,
      "activations/layer16_attention_weight_min": -26.833024978637695,
      "activations/layer17_attention_weight_max": 34.09245300292969,
      "activations/layer17_attention_weight_min": -25.540273666381836,
      "activations/layer18_attention_weight_max": 36.1612663269043,
      "activations/layer18_attention_weight_min": -22.326316833496094,
      "activations/layer19_attention_weight_max": 40.20547866821289,
      "activations/layer19_attention_weight_min": -30.413022994995117,
      "activations/layer1_attention_weight_max": 17.330045700073242,
      "activations/layer1_attention_weight_min": -14.430493354797363,
      "activations/layer20_attention_weight_max": 35.51876449584961,
      "activations/layer20_attention_weight_min": -25.178510665893555,
      "activations/layer21_attention_weight_max": 34.92708206176758,
      "activations/layer21_attention_weight_min": -24.123857498168945,
      "activations/layer22_attention_weight_max": 49.97454833984375,
      "activations/layer22_attention_weight_min": -30.983097076416016,
      "activations/layer23_attention_weight_max": 39.39276123046875,
      "activations/layer23_attention_weight_min": -25.390655517578125,
      "activations/layer2_attention_weight_max": 34.03887176513672,
      "activations/layer2_attention_weight_min": -33.92603302001953,
      "activations/layer3_attention_weight_max": 98.02960968017578,
      "activations/layer3_attention_weight_min": -96.97738647460938,
      "activations/layer4_attention_weight_max": 119.05577850341797,
      "activations/layer4_attention_weight_min": -114.86748504638672,
      "activations/layer5_attention_weight_max": 57.67430877685547,
      "activations/layer5_attention_weight_min": -61.342247009277344,
      "activations/layer6_attention_weight_max": 46.4166145324707,
      "activations/layer6_attention_weight_min": -51.09859848022461,
      "activations/layer7_attention_weight_max": 94.18807983398438,
      "activations/layer7_attention_weight_min": -98.67626190185547,
      "activations/layer8_attention_weight_max": 42.907711029052734,
      "activations/layer8_attention_weight_min": -42.6395263671875,
      "activations/layer9_attention_weight_max": 36.05906295776367,
      "activations/layer9_attention_weight_min": -35.75333786010742,
      "epoch": 10.52,
      "learning_rate": 8.301325757575757e-05,
      "loss": 2.7667,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5181,
      "eval_samples_per_second": 504.106,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5181,
      "eval_openwebtext_samples_per_second": 504.106,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.0155,
      "eval_wikitext_samples_per_second": 226.248,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_lambada_loss": 2.4765625,
      "eval_lambada_ppl": 11.900286792057758,
      "eval_lambada_runtime": 9.5703,
      "eval_lambada_samples_per_second": 508.761,
      "step": 181000
    },
    {
      "activations/layer0_attention_weight_max": 16.440553665161133,
      "activations/layer0_attention_weight_min": -13.25765609741211,
      "activations/layer10_attention_weight_max": 35.21511459350586,
      "activations/layer10_attention_weight_min": -31.291465759277344,
      "activations/layer11_attention_weight_max": 32.72666549682617,
      "activations/layer11_attention_weight_min": -30.09473991394043,
      "activations/layer12_attention_weight_max": 23.83283805847168,
      "activations/layer12_attention_weight_min": -26.480741500854492,
      "activations/layer13_attention_weight_max": 38.59614181518555,
      "activations/layer13_attention_weight_min": -33.13758850097656,
      "activations/layer14_attention_weight_max": 46.312625885009766,
      "activations/layer14_attention_weight_min": -38.741050720214844,
      "activations/layer15_attention_weight_max": 34.31846618652344,
      "activations/layer15_attention_weight_min": -31.12891960144043,
      "activations/layer16_attention_weight_max": 30.191328048706055,
      "activations/layer16_attention_weight_min": -26.17392921447754,
      "activations/layer17_attention_weight_max": 30.91017723083496,
      "activations/layer17_attention_weight_min": -26.383453369140625,
      "activations/layer18_attention_weight_max": 31.77532958984375,
      "activations/layer18_attention_weight_min": -22.519323348999023,
      "activations/layer19_attention_weight_max": 34.96553421020508,
      "activations/layer19_attention_weight_min": -29.01706314086914,
      "activations/layer1_attention_weight_max": 17.501728057861328,
      "activations/layer1_attention_weight_min": -15.075128555297852,
      "activations/layer20_attention_weight_max": 30.39321517944336,
      "activations/layer20_attention_weight_min": -23.661840438842773,
      "activations/layer21_attention_weight_max": 28.154897689819336,
      "activations/layer21_attention_weight_min": -23.99478530883789,
      "activations/layer22_attention_weight_max": 49.1180419921875,
      "activations/layer22_attention_weight_min": -30.76283073425293,
      "activations/layer23_attention_weight_max": 35.334495544433594,
      "activations/layer23_attention_weight_min": -23.46822166442871,
      "activations/layer2_attention_weight_max": 31.98214340209961,
      "activations/layer2_attention_weight_min": -31.89386558532715,
      "activations/layer3_attention_weight_max": 91.9361343383789,
      "activations/layer3_attention_weight_min": -94.41706848144531,
      "activations/layer4_attention_weight_max": 110.9339599609375,
      "activations/layer4_attention_weight_min": -111.61256408691406,
      "activations/layer5_attention_weight_max": 52.94365692138672,
      "activations/layer5_attention_weight_min": -61.294708251953125,
      "activations/layer6_attention_weight_max": 45.07642364501953,
      "activations/layer6_attention_weight_min": -47.134315490722656,
      "activations/layer7_attention_weight_max": 90.47653198242188,
      "activations/layer7_attention_weight_min": -92.92198944091797,
      "activations/layer8_attention_weight_max": 38.28618621826172,
      "activations/layer8_attention_weight_min": -40.9643440246582,
      "activations/layer9_attention_weight_max": 33.75935363769531,
      "activations/layer9_attention_weight_min": -33.543418884277344,
      "epoch": 10.52,
      "learning_rate": 8.299431818181819e-05,
      "loss": 2.7838,
      "step": 181050
    },
    {
      "activations/layer0_attention_weight_max": 16.018203735351562,
      "activations/layer0_attention_weight_min": -12.01376724243164,
      "activations/layer10_attention_weight_max": 37.05352783203125,
      "activations/layer10_attention_weight_min": -37.68779373168945,
      "activations/layer11_attention_weight_max": 39.231040954589844,
      "activations/layer11_attention_weight_min": -38.63414764404297,
      "activations/layer12_attention_weight_max": 27.398454666137695,
      "activations/layer12_attention_weight_min": -27.13707160949707,
      "activations/layer13_attention_weight_max": 38.9852409362793,
      "activations/layer13_attention_weight_min": -34.9752197265625,
      "activations/layer14_attention_weight_max": 40.03300476074219,
      "activations/layer14_attention_weight_min": -40.86028289794922,
      "activations/layer15_attention_weight_max": 40.38734436035156,
      "activations/layer15_attention_weight_min": -36.30270767211914,
      "activations/layer16_attention_weight_max": 29.53481101989746,
      "activations/layer16_attention_weight_min": -25.88799285888672,
      "activations/layer17_attention_weight_max": 31.2757568359375,
      "activations/layer17_attention_weight_min": -26.677040100097656,
      "activations/layer18_attention_weight_max": 31.053504943847656,
      "activations/layer18_attention_weight_min": -23.925106048583984,
      "activations/layer19_attention_weight_max": 32.252403259277344,
      "activations/layer19_attention_weight_min": -31.416135787963867,
      "activations/layer1_attention_weight_max": 16.763328552246094,
      "activations/layer1_attention_weight_min": -16.036096572875977,
      "activations/layer20_attention_weight_max": 29.1708927154541,
      "activations/layer20_attention_weight_min": -25.564395904541016,
      "activations/layer21_attention_weight_max": 26.8272762298584,
      "activations/layer21_attention_weight_min": -24.221763610839844,
      "activations/layer22_attention_weight_max": 42.6616325378418,
      "activations/layer22_attention_weight_min": -30.36536979675293,
      "activations/layer23_attention_weight_max": 33.629459381103516,
      "activations/layer23_attention_weight_min": -24.42354393005371,
      "activations/layer2_attention_weight_max": 33.06764602661133,
      "activations/layer2_attention_weight_min": -34.227195739746094,
      "activations/layer3_attention_weight_max": 93.24392700195312,
      "activations/layer3_attention_weight_min": -99.3533706665039,
      "activations/layer4_attention_weight_max": 112.00737762451172,
      "activations/layer4_attention_weight_min": -108.69823455810547,
      "activations/layer5_attention_weight_max": 58.20787048339844,
      "activations/layer5_attention_weight_min": -61.04323959350586,
      "activations/layer6_attention_weight_max": 46.10995101928711,
      "activations/layer6_attention_weight_min": -43.6876106262207,
      "activations/layer7_attention_weight_max": 100.33686065673828,
      "activations/layer7_attention_weight_min": -94.89949798583984,
      "activations/layer8_attention_weight_max": 47.357486724853516,
      "activations/layer8_attention_weight_min": -45.27447509765625,
      "activations/layer9_attention_weight_max": 35.74052047729492,
      "activations/layer9_attention_weight_min": -36.057456970214844,
      "epoch": 10.52,
      "learning_rate": 8.297537878787877e-05,
      "loss": 2.7725,
      "step": 181100
    },
    {
      "activations/layer0_attention_weight_max": 14.887991905212402,
      "activations/layer0_attention_weight_min": -12.522016525268555,
      "activations/layer10_attention_weight_max": 37.04153823852539,
      "activations/layer10_attention_weight_min": -33.10900115966797,
      "activations/layer11_attention_weight_max": 32.912872314453125,
      "activations/layer11_attention_weight_min": -33.05853271484375,
      "activations/layer12_attention_weight_max": 24.73442840576172,
      "activations/layer12_attention_weight_min": -24.47565269470215,
      "activations/layer13_attention_weight_max": 39.256439208984375,
      "activations/layer13_attention_weight_min": -35.73112106323242,
      "activations/layer14_attention_weight_max": 46.488826751708984,
      "activations/layer14_attention_weight_min": -39.27394485473633,
      "activations/layer15_attention_weight_max": 36.71628189086914,
      "activations/layer15_attention_weight_min": -31.250152587890625,
      "activations/layer16_attention_weight_max": 31.70733642578125,
      "activations/layer16_attention_weight_min": -28.44851303100586,
      "activations/layer17_attention_weight_max": 34.65644836425781,
      "activations/layer17_attention_weight_min": -26.18433952331543,
      "activations/layer18_attention_weight_max": 32.778995513916016,
      "activations/layer18_attention_weight_min": -22.674945831298828,
      "activations/layer19_attention_weight_max": 35.76740646362305,
      "activations/layer19_attention_weight_min": -31.38579750061035,
      "activations/layer1_attention_weight_max": 17.68047523498535,
      "activations/layer1_attention_weight_min": -15.3893404006958,
      "activations/layer20_attention_weight_max": 32.05282974243164,
      "activations/layer20_attention_weight_min": -24.338380813598633,
      "activations/layer21_attention_weight_max": 31.656240463256836,
      "activations/layer21_attention_weight_min": -24.90177345275879,
      "activations/layer22_attention_weight_max": 50.308536529541016,
      "activations/layer22_attention_weight_min": -29.167905807495117,
      "activations/layer23_attention_weight_max": 35.097373962402344,
      "activations/layer23_attention_weight_min": -23.830780029296875,
      "activations/layer2_attention_weight_max": 33.11080551147461,
      "activations/layer2_attention_weight_min": -34.27197265625,
      "activations/layer3_attention_weight_max": 96.65192413330078,
      "activations/layer3_attention_weight_min": -94.9488754272461,
      "activations/layer4_attention_weight_max": 112.9798812866211,
      "activations/layer4_attention_weight_min": -112.1364517211914,
      "activations/layer5_attention_weight_max": 50.85356140136719,
      "activations/layer5_attention_weight_min": -61.22282028198242,
      "activations/layer6_attention_weight_max": 45.68512725830078,
      "activations/layer6_attention_weight_min": -48.11056900024414,
      "activations/layer7_attention_weight_max": 90.3200912475586,
      "activations/layer7_attention_weight_min": -87.64350128173828,
      "activations/layer8_attention_weight_max": 41.27204895019531,
      "activations/layer8_attention_weight_min": -41.1301383972168,
      "activations/layer9_attention_weight_max": 33.7747688293457,
      "activations/layer9_attention_weight_min": -33.88639450073242,
      "epoch": 10.53,
      "learning_rate": 8.295643939393939e-05,
      "loss": 2.78,
      "step": 181150
    },
    {
      "activations/layer0_attention_weight_max": 15.475722312927246,
      "activations/layer0_attention_weight_min": -12.924440383911133,
      "activations/layer10_attention_weight_max": 31.597591400146484,
      "activations/layer10_attention_weight_min": -32.77533721923828,
      "activations/layer11_attention_weight_max": 31.328521728515625,
      "activations/layer11_attention_weight_min": -31.581531524658203,
      "activations/layer12_attention_weight_max": 26.007539749145508,
      "activations/layer12_attention_weight_min": -25.554977416992188,
      "activations/layer13_attention_weight_max": 39.22072219848633,
      "activations/layer13_attention_weight_min": -34.11429977416992,
      "activations/layer14_attention_weight_max": 49.583641052246094,
      "activations/layer14_attention_weight_min": -39.796627044677734,
      "activations/layer15_attention_weight_max": 36.47786331176758,
      "activations/layer15_attention_weight_min": -32.934757232666016,
      "activations/layer16_attention_weight_max": 31.64679718017578,
      "activations/layer16_attention_weight_min": -27.88189697265625,
      "activations/layer17_attention_weight_max": 32.85922622680664,
      "activations/layer17_attention_weight_min": -27.677141189575195,
      "activations/layer18_attention_weight_max": 29.976486206054688,
      "activations/layer18_attention_weight_min": -26.42316246032715,
      "activations/layer19_attention_weight_max": 33.472572326660156,
      "activations/layer19_attention_weight_min": -32.0645751953125,
      "activations/layer1_attention_weight_max": 16.620454788208008,
      "activations/layer1_attention_weight_min": -14.79848575592041,
      "activations/layer20_attention_weight_max": 30.945531845092773,
      "activations/layer20_attention_weight_min": -25.54265594482422,
      "activations/layer21_attention_weight_max": 28.689489364624023,
      "activations/layer21_attention_weight_min": -23.60697364807129,
      "activations/layer22_attention_weight_max": 48.91887664794922,
      "activations/layer22_attention_weight_min": -29.66509246826172,
      "activations/layer23_attention_weight_max": 34.01740646362305,
      "activations/layer23_attention_weight_min": -25.69141960144043,
      "activations/layer2_attention_weight_max": 32.51988220214844,
      "activations/layer2_attention_weight_min": -33.326866149902344,
      "activations/layer3_attention_weight_max": 95.64268493652344,
      "activations/layer3_attention_weight_min": -95.29456329345703,
      "activations/layer4_attention_weight_max": 115.6844253540039,
      "activations/layer4_attention_weight_min": -113.28741455078125,
      "activations/layer5_attention_weight_max": 52.78388595581055,
      "activations/layer5_attention_weight_min": -62.64191818237305,
      "activations/layer6_attention_weight_max": 44.19713592529297,
      "activations/layer6_attention_weight_min": -47.63533020019531,
      "activations/layer7_attention_weight_max": 88.01509857177734,
      "activations/layer7_attention_weight_min": -89.50077819824219,
      "activations/layer8_attention_weight_max": 40.61435317993164,
      "activations/layer8_attention_weight_min": -41.76344680786133,
      "activations/layer9_attention_weight_max": 34.41378402709961,
      "activations/layer9_attention_weight_min": -32.6110954284668,
      "epoch": 10.53,
      "learning_rate": 8.293749999999999e-05,
      "loss": 2.7904,
      "step": 181200
    },
    {
      "activations/layer0_attention_weight_max": 16.066322326660156,
      "activations/layer0_attention_weight_min": -12.640982627868652,
      "activations/layer10_attention_weight_max": 32.9140510559082,
      "activations/layer10_attention_weight_min": -30.474782943725586,
      "activations/layer11_attention_weight_max": 31.779468536376953,
      "activations/layer11_attention_weight_min": -30.61989974975586,
      "activations/layer12_attention_weight_max": 23.85919761657715,
      "activations/layer12_attention_weight_min": -24.05106544494629,
      "activations/layer13_attention_weight_max": 39.045867919921875,
      "activations/layer13_attention_weight_min": -32.39799499511719,
      "activations/layer14_attention_weight_max": 41.56307601928711,
      "activations/layer14_attention_weight_min": -35.87858963012695,
      "activations/layer15_attention_weight_max": 35.930599212646484,
      "activations/layer15_attention_weight_min": -31.987890243530273,
      "activations/layer16_attention_weight_max": 30.47185516357422,
      "activations/layer16_attention_weight_min": -26.42422866821289,
      "activations/layer17_attention_weight_max": 35.15220642089844,
      "activations/layer17_attention_weight_min": -24.56492042541504,
      "activations/layer18_attention_weight_max": 33.2912712097168,
      "activations/layer18_attention_weight_min": -22.349937438964844,
      "activations/layer19_attention_weight_max": 35.78696823120117,
      "activations/layer19_attention_weight_min": -30.571216583251953,
      "activations/layer1_attention_weight_max": 16.244510650634766,
      "activations/layer1_attention_weight_min": -15.246208190917969,
      "activations/layer20_attention_weight_max": 29.985027313232422,
      "activations/layer20_attention_weight_min": -24.222867965698242,
      "activations/layer21_attention_weight_max": 34.22405242919922,
      "activations/layer21_attention_weight_min": -22.38197898864746,
      "activations/layer22_attention_weight_max": 45.8416748046875,
      "activations/layer22_attention_weight_min": -29.794553756713867,
      "activations/layer23_attention_weight_max": 37.66556930541992,
      "activations/layer23_attention_weight_min": -26.217803955078125,
      "activations/layer2_attention_weight_max": 32.19789505004883,
      "activations/layer2_attention_weight_min": -33.11071014404297,
      "activations/layer3_attention_weight_max": 96.27465057373047,
      "activations/layer3_attention_weight_min": -102.19482421875,
      "activations/layer4_attention_weight_max": 115.56673431396484,
      "activations/layer4_attention_weight_min": -119.41863250732422,
      "activations/layer5_attention_weight_max": 53.786495208740234,
      "activations/layer5_attention_weight_min": -61.65044403076172,
      "activations/layer6_attention_weight_max": 45.56351089477539,
      "activations/layer6_attention_weight_min": -49.30833435058594,
      "activations/layer7_attention_weight_max": 89.29821014404297,
      "activations/layer7_attention_weight_min": -90.10140991210938,
      "activations/layer8_attention_weight_max": 41.21037673950195,
      "activations/layer8_attention_weight_min": -39.51079559326172,
      "activations/layer9_attention_weight_max": 34.015804290771484,
      "activations/layer9_attention_weight_min": -33.134521484375,
      "epoch": 10.53,
      "learning_rate": 8.291856060606059e-05,
      "loss": 2.7827,
      "step": 181250
    },
    {
      "activations/layer0_attention_weight_max": 16.07746124267578,
      "activations/layer0_attention_weight_min": -11.911376953125,
      "activations/layer10_attention_weight_max": 32.48476791381836,
      "activations/layer10_attention_weight_min": -34.916419982910156,
      "activations/layer11_attention_weight_max": 33.848384857177734,
      "activations/layer11_attention_weight_min": -32.94435119628906,
      "activations/layer12_attention_weight_max": 25.771188735961914,
      "activations/layer12_attention_weight_min": -27.92941665649414,
      "activations/layer13_attention_weight_max": 40.067665100097656,
      "activations/layer13_attention_weight_min": -34.9810791015625,
      "activations/layer14_attention_weight_max": 46.33815383911133,
      "activations/layer14_attention_weight_min": -41.3890495300293,
      "activations/layer15_attention_weight_max": 35.66092300415039,
      "activations/layer15_attention_weight_min": -32.4266242980957,
      "activations/layer16_attention_weight_max": 30.72971534729004,
      "activations/layer16_attention_weight_min": -27.811906814575195,
      "activations/layer17_attention_weight_max": 33.55284881591797,
      "activations/layer17_attention_weight_min": -26.213159561157227,
      "activations/layer18_attention_weight_max": 31.414901733398438,
      "activations/layer18_attention_weight_min": -23.15082550048828,
      "activations/layer19_attention_weight_max": 35.81138229370117,
      "activations/layer19_attention_weight_min": -30.078886032104492,
      "activations/layer1_attention_weight_max": 18.227983474731445,
      "activations/layer1_attention_weight_min": -16.199399948120117,
      "activations/layer20_attention_weight_max": 29.361120223999023,
      "activations/layer20_attention_weight_min": -24.456132888793945,
      "activations/layer21_attention_weight_max": 30.929916381835938,
      "activations/layer21_attention_weight_min": -24.441068649291992,
      "activations/layer22_attention_weight_max": 48.06871032714844,
      "activations/layer22_attention_weight_min": -30.175758361816406,
      "activations/layer23_attention_weight_max": 38.03864669799805,
      "activations/layer23_attention_weight_min": -23.932376861572266,
      "activations/layer2_attention_weight_max": 33.08418273925781,
      "activations/layer2_attention_weight_min": -34.0277214050293,
      "activations/layer3_attention_weight_max": 97.93038940429688,
      "activations/layer3_attention_weight_min": -101.03297424316406,
      "activations/layer4_attention_weight_max": 119.68758392333984,
      "activations/layer4_attention_weight_min": -121.64190673828125,
      "activations/layer5_attention_weight_max": 54.400115966796875,
      "activations/layer5_attention_weight_min": -63.6788444519043,
      "activations/layer6_attention_weight_max": 45.36388397216797,
      "activations/layer6_attention_weight_min": -48.86212921142578,
      "activations/layer7_attention_weight_max": 91.0401382446289,
      "activations/layer7_attention_weight_min": -95.18923950195312,
      "activations/layer8_attention_weight_max": 40.872772216796875,
      "activations/layer8_attention_weight_min": -43.129215240478516,
      "activations/layer9_attention_weight_max": 33.67573928833008,
      "activations/layer9_attention_weight_min": -34.98025131225586,
      "epoch": 10.53,
      "learning_rate": 8.28996212121212e-05,
      "loss": 2.7974,
      "step": 181300
    },
    {
      "activations/layer0_attention_weight_max": 15.13528060913086,
      "activations/layer0_attention_weight_min": -12.78275203704834,
      "activations/layer10_attention_weight_max": 33.76862335205078,
      "activations/layer10_attention_weight_min": -34.413394927978516,
      "activations/layer11_attention_weight_max": 32.91661834716797,
      "activations/layer11_attention_weight_min": -31.678768157958984,
      "activations/layer12_attention_weight_max": 24.031585693359375,
      "activations/layer12_attention_weight_min": -24.633358001708984,
      "activations/layer13_attention_weight_max": 38.74354553222656,
      "activations/layer13_attention_weight_min": -35.242706298828125,
      "activations/layer14_attention_weight_max": 44.799537658691406,
      "activations/layer14_attention_weight_min": -39.88548278808594,
      "activations/layer15_attention_weight_max": 35.9638671875,
      "activations/layer15_attention_weight_min": -31.05719566345215,
      "activations/layer16_attention_weight_max": 30.461746215820312,
      "activations/layer16_attention_weight_min": -28.16130828857422,
      "activations/layer17_attention_weight_max": 32.554874420166016,
      "activations/layer17_attention_weight_min": -24.868633270263672,
      "activations/layer18_attention_weight_max": 30.610946655273438,
      "activations/layer18_attention_weight_min": -23.00987434387207,
      "activations/layer19_attention_weight_max": 34.994293212890625,
      "activations/layer19_attention_weight_min": -30.17819595336914,
      "activations/layer1_attention_weight_max": 16.84465217590332,
      "activations/layer1_attention_weight_min": -14.381364822387695,
      "activations/layer20_attention_weight_max": 30.883289337158203,
      "activations/layer20_attention_weight_min": -23.250778198242188,
      "activations/layer21_attention_weight_max": 31.48210906982422,
      "activations/layer21_attention_weight_min": -24.671783447265625,
      "activations/layer22_attention_weight_max": 53.0572509765625,
      "activations/layer22_attention_weight_min": -27.897825241088867,
      "activations/layer23_attention_weight_max": 36.419288635253906,
      "activations/layer23_attention_weight_min": -23.857175827026367,
      "activations/layer2_attention_weight_max": 34.16683578491211,
      "activations/layer2_attention_weight_min": -33.31922149658203,
      "activations/layer3_attention_weight_max": 101.1341323852539,
      "activations/layer3_attention_weight_min": -96.4572525024414,
      "activations/layer4_attention_weight_max": 118.9940185546875,
      "activations/layer4_attention_weight_min": -115.2790756225586,
      "activations/layer5_attention_weight_max": 52.63405990600586,
      "activations/layer5_attention_weight_min": -62.457313537597656,
      "activations/layer6_attention_weight_max": 45.123435974121094,
      "activations/layer6_attention_weight_min": -47.16208267211914,
      "activations/layer7_attention_weight_max": 93.65118408203125,
      "activations/layer7_attention_weight_min": -95.76327514648438,
      "activations/layer8_attention_weight_max": 40.599308013916016,
      "activations/layer8_attention_weight_min": -40.981544494628906,
      "activations/layer9_attention_weight_max": 33.799007415771484,
      "activations/layer9_attention_weight_min": -33.00180435180664,
      "epoch": 10.54,
      "learning_rate": 8.288068181818181e-05,
      "loss": 2.7845,
      "step": 181350
    },
    {
      "activations/layer0_attention_weight_max": 15.988872528076172,
      "activations/layer0_attention_weight_min": -12.560432434082031,
      "activations/layer10_attention_weight_max": 36.18938064575195,
      "activations/layer10_attention_weight_min": -36.09463882446289,
      "activations/layer11_attention_weight_max": 34.3250732421875,
      "activations/layer11_attention_weight_min": -34.093162536621094,
      "activations/layer12_attention_weight_max": 25.38998031616211,
      "activations/layer12_attention_weight_min": -25.783985137939453,
      "activations/layer13_attention_weight_max": 41.59306716918945,
      "activations/layer13_attention_weight_min": -36.22597122192383,
      "activations/layer14_attention_weight_max": 50.485992431640625,
      "activations/layer14_attention_weight_min": -39.64603042602539,
      "activations/layer15_attention_weight_max": 37.699337005615234,
      "activations/layer15_attention_weight_min": -31.35843849182129,
      "activations/layer16_attention_weight_max": 30.39983367919922,
      "activations/layer16_attention_weight_min": -29.641876220703125,
      "activations/layer17_attention_weight_max": 34.11152267456055,
      "activations/layer17_attention_weight_min": -26.108612060546875,
      "activations/layer18_attention_weight_max": 33.33866500854492,
      "activations/layer18_attention_weight_min": -23.025991439819336,
      "activations/layer19_attention_weight_max": 40.66218948364258,
      "activations/layer19_attention_weight_min": -31.419355392456055,
      "activations/layer1_attention_weight_max": 18.02857208251953,
      "activations/layer1_attention_weight_min": -15.013301849365234,
      "activations/layer20_attention_weight_max": 31.69209861755371,
      "activations/layer20_attention_weight_min": -25.341392517089844,
      "activations/layer21_attention_weight_max": 30.052249908447266,
      "activations/layer21_attention_weight_min": -24.00725746154785,
      "activations/layer22_attention_weight_max": 49.29526138305664,
      "activations/layer22_attention_weight_min": -29.399364471435547,
      "activations/layer23_attention_weight_max": 36.84593200683594,
      "activations/layer23_attention_weight_min": -25.93311882019043,
      "activations/layer2_attention_weight_max": 32.40699768066406,
      "activations/layer2_attention_weight_min": -32.52998352050781,
      "activations/layer3_attention_weight_max": 97.11654663085938,
      "activations/layer3_attention_weight_min": -100.1493911743164,
      "activations/layer4_attention_weight_max": 117.48988342285156,
      "activations/layer4_attention_weight_min": -120.10737609863281,
      "activations/layer5_attention_weight_max": 59.23773193359375,
      "activations/layer5_attention_weight_min": -61.94840621948242,
      "activations/layer6_attention_weight_max": 47.4686393737793,
      "activations/layer6_attention_weight_min": -50.84627151489258,
      "activations/layer7_attention_weight_max": 96.2607650756836,
      "activations/layer7_attention_weight_min": -95.09010314941406,
      "activations/layer8_attention_weight_max": 43.43071365356445,
      "activations/layer8_attention_weight_min": -46.684532165527344,
      "activations/layer9_attention_weight_max": 36.993289947509766,
      "activations/layer9_attention_weight_min": -36.39970397949219,
      "epoch": 10.54,
      "learning_rate": 8.286174242424242e-05,
      "loss": 2.7855,
      "step": 181400
    },
    {
      "activations/layer0_attention_weight_max": 15.251265525817871,
      "activations/layer0_attention_weight_min": -12.924095153808594,
      "activations/layer10_attention_weight_max": 35.30789566040039,
      "activations/layer10_attention_weight_min": -35.08461380004883,
      "activations/layer11_attention_weight_max": 35.43150329589844,
      "activations/layer11_attention_weight_min": -35.386680603027344,
      "activations/layer12_attention_weight_max": 28.3444766998291,
      "activations/layer12_attention_weight_min": -26.892860412597656,
      "activations/layer13_attention_weight_max": 44.74833679199219,
      "activations/layer13_attention_weight_min": -36.52494812011719,
      "activations/layer14_attention_weight_max": 47.684600830078125,
      "activations/layer14_attention_weight_min": -39.806114196777344,
      "activations/layer15_attention_weight_max": 40.49970245361328,
      "activations/layer15_attention_weight_min": -35.66508865356445,
      "activations/layer16_attention_weight_max": 29.720401763916016,
      "activations/layer16_attention_weight_min": -28.75963020324707,
      "activations/layer17_attention_weight_max": 30.39641761779785,
      "activations/layer17_attention_weight_min": -27.363235473632812,
      "activations/layer18_attention_weight_max": 36.624725341796875,
      "activations/layer18_attention_weight_min": -23.664501190185547,
      "activations/layer19_attention_weight_max": 37.76630783081055,
      "activations/layer19_attention_weight_min": -34.07523727416992,
      "activations/layer1_attention_weight_max": 16.658336639404297,
      "activations/layer1_attention_weight_min": -13.848179817199707,
      "activations/layer20_attention_weight_max": 29.03347396850586,
      "activations/layer20_attention_weight_min": -26.499988555908203,
      "activations/layer21_attention_weight_max": 27.617610931396484,
      "activations/layer21_attention_weight_min": -25.063255310058594,
      "activations/layer22_attention_weight_max": 44.0811767578125,
      "activations/layer22_attention_weight_min": -33.65217590332031,
      "activations/layer23_attention_weight_max": 35.5015869140625,
      "activations/layer23_attention_weight_min": -25.166988372802734,
      "activations/layer2_attention_weight_max": 33.23878479003906,
      "activations/layer2_attention_weight_min": -32.882568359375,
      "activations/layer3_attention_weight_max": 99.22451782226562,
      "activations/layer3_attention_weight_min": -97.05670928955078,
      "activations/layer4_attention_weight_max": 114.37981414794922,
      "activations/layer4_attention_weight_min": -115.87416076660156,
      "activations/layer5_attention_weight_max": 52.32064437866211,
      "activations/layer5_attention_weight_min": -59.94220733642578,
      "activations/layer6_attention_weight_max": 48.754966735839844,
      "activations/layer6_attention_weight_min": -45.902896881103516,
      "activations/layer7_attention_weight_max": 94.93667602539062,
      "activations/layer7_attention_weight_min": -94.79300689697266,
      "activations/layer8_attention_weight_max": 42.54314422607422,
      "activations/layer8_attention_weight_min": -43.50358963012695,
      "activations/layer9_attention_weight_max": 36.84275436401367,
      "activations/layer9_attention_weight_min": -37.41874694824219,
      "epoch": 10.54,
      "learning_rate": 8.284280303030302e-05,
      "loss": 2.7904,
      "step": 181450
    },
    {
      "activations/layer0_attention_weight_max": 15.9814453125,
      "activations/layer0_attention_weight_min": -12.787778854370117,
      "activations/layer10_attention_weight_max": 33.31712341308594,
      "activations/layer10_attention_weight_min": -31.80918312072754,
      "activations/layer11_attention_weight_max": 34.313690185546875,
      "activations/layer11_attention_weight_min": -31.29296112060547,
      "activations/layer12_attention_weight_max": 26.21488380432129,
      "activations/layer12_attention_weight_min": -24.703964233398438,
      "activations/layer13_attention_weight_max": 40.9639892578125,
      "activations/layer13_attention_weight_min": -32.65105438232422,
      "activations/layer14_attention_weight_max": 53.729270935058594,
      "activations/layer14_attention_weight_min": -37.72077941894531,
      "activations/layer15_attention_weight_max": 41.5860481262207,
      "activations/layer15_attention_weight_min": -34.2874641418457,
      "activations/layer16_attention_weight_max": 35.114898681640625,
      "activations/layer16_attention_weight_min": -26.53150177001953,
      "activations/layer17_attention_weight_max": 32.435672760009766,
      "activations/layer17_attention_weight_min": -26.513458251953125,
      "activations/layer18_attention_weight_max": 34.115081787109375,
      "activations/layer18_attention_weight_min": -23.46450424194336,
      "activations/layer19_attention_weight_max": 39.54887771606445,
      "activations/layer19_attention_weight_min": -30.527875900268555,
      "activations/layer1_attention_weight_max": 16.563621520996094,
      "activations/layer1_attention_weight_min": -14.400874137878418,
      "activations/layer20_attention_weight_max": 34.24302673339844,
      "activations/layer20_attention_weight_min": -25.065885543823242,
      "activations/layer21_attention_weight_max": 30.849130630493164,
      "activations/layer21_attention_weight_min": -23.528528213500977,
      "activations/layer22_attention_weight_max": 50.5015754699707,
      "activations/layer22_attention_weight_min": -30.595491409301758,
      "activations/layer23_attention_weight_max": 37.28498077392578,
      "activations/layer23_attention_weight_min": -25.507041931152344,
      "activations/layer2_attention_weight_max": 32.94416046142578,
      "activations/layer2_attention_weight_min": -32.810482025146484,
      "activations/layer3_attention_weight_max": 94.00405883789062,
      "activations/layer3_attention_weight_min": -94.37091827392578,
      "activations/layer4_attention_weight_max": 113.04736328125,
      "activations/layer4_attention_weight_min": -110.34747314453125,
      "activations/layer5_attention_weight_max": 55.611324310302734,
      "activations/layer5_attention_weight_min": -63.175018310546875,
      "activations/layer6_attention_weight_max": 46.76724624633789,
      "activations/layer6_attention_weight_min": -47.41301345825195,
      "activations/layer7_attention_weight_max": 92.50464630126953,
      "activations/layer7_attention_weight_min": -95.02608489990234,
      "activations/layer8_attention_weight_max": 41.96470642089844,
      "activations/layer8_attention_weight_min": -41.871646881103516,
      "activations/layer9_attention_weight_max": 33.890262603759766,
      "activations/layer9_attention_weight_min": -33.59874725341797,
      "epoch": 10.55,
      "learning_rate": 8.282386363636363e-05,
      "loss": 2.7658,
      "step": 181500
    },
    {
      "activations/layer0_attention_weight_max": 15.601903915405273,
      "activations/layer0_attention_weight_min": -13.562952995300293,
      "activations/layer10_attention_weight_max": 31.437591552734375,
      "activations/layer10_attention_weight_min": -33.17668533325195,
      "activations/layer11_attention_weight_max": 32.41318893432617,
      "activations/layer11_attention_weight_min": -30.915748596191406,
      "activations/layer12_attention_weight_max": 24.980792999267578,
      "activations/layer12_attention_weight_min": -24.2020320892334,
      "activations/layer13_attention_weight_max": 38.72124099731445,
      "activations/layer13_attention_weight_min": -34.42689514160156,
      "activations/layer14_attention_weight_max": 48.13218307495117,
      "activations/layer14_attention_weight_min": -39.195865631103516,
      "activations/layer15_attention_weight_max": 37.49038314819336,
      "activations/layer15_attention_weight_min": -28.962154388427734,
      "activations/layer16_attention_weight_max": 31.87908935546875,
      "activations/layer16_attention_weight_min": -28.417678833007812,
      "activations/layer17_attention_weight_max": 35.22744369506836,
      "activations/layer17_attention_weight_min": -26.511014938354492,
      "activations/layer18_attention_weight_max": 33.05734634399414,
      "activations/layer18_attention_weight_min": -24.84606170654297,
      "activations/layer19_attention_weight_max": 38.90067672729492,
      "activations/layer19_attention_weight_min": -31.953100204467773,
      "activations/layer1_attention_weight_max": 16.110017776489258,
      "activations/layer1_attention_weight_min": -13.740340232849121,
      "activations/layer20_attention_weight_max": 33.620849609375,
      "activations/layer20_attention_weight_min": -26.254833221435547,
      "activations/layer21_attention_weight_max": 33.53952407836914,
      "activations/layer21_attention_weight_min": -24.524578094482422,
      "activations/layer22_attention_weight_max": 44.97129440307617,
      "activations/layer22_attention_weight_min": -29.657365798950195,
      "activations/layer23_attention_weight_max": 42.67588424682617,
      "activations/layer23_attention_weight_min": -26.28006362915039,
      "activations/layer2_attention_weight_max": 33.144920349121094,
      "activations/layer2_attention_weight_min": -32.9666748046875,
      "activations/layer3_attention_weight_max": 96.3431396484375,
      "activations/layer3_attention_weight_min": -98.8423080444336,
      "activations/layer4_attention_weight_max": 120.42266845703125,
      "activations/layer4_attention_weight_min": -119.92146301269531,
      "activations/layer5_attention_weight_max": 54.38589096069336,
      "activations/layer5_attention_weight_min": -64.30875396728516,
      "activations/layer6_attention_weight_max": 45.38233184814453,
      "activations/layer6_attention_weight_min": -49.86558151245117,
      "activations/layer7_attention_weight_max": 93.2103500366211,
      "activations/layer7_attention_weight_min": -93.31258392333984,
      "activations/layer8_attention_weight_max": 38.41374588012695,
      "activations/layer8_attention_weight_min": -41.70549011230469,
      "activations/layer9_attention_weight_max": 31.286518096923828,
      "activations/layer9_attention_weight_min": -33.10206604003906,
      "epoch": 10.55,
      "learning_rate": 8.280492424242424e-05,
      "loss": 2.794,
      "step": 181550
    },
    {
      "activations/layer0_attention_weight_max": 15.561798095703125,
      "activations/layer0_attention_weight_min": -12.452170372009277,
      "activations/layer10_attention_weight_max": 32.232513427734375,
      "activations/layer10_attention_weight_min": -32.03253173828125,
      "activations/layer11_attention_weight_max": 33.68983840942383,
      "activations/layer11_attention_weight_min": -32.14377212524414,
      "activations/layer12_attention_weight_max": 27.04145050048828,
      "activations/layer12_attention_weight_min": -26.67994499206543,
      "activations/layer13_attention_weight_max": 44.04819107055664,
      "activations/layer13_attention_weight_min": -35.008724212646484,
      "activations/layer14_attention_weight_max": 47.52114486694336,
      "activations/layer14_attention_weight_min": -38.09986114501953,
      "activations/layer15_attention_weight_max": 38.7247200012207,
      "activations/layer15_attention_weight_min": -31.17607879638672,
      "activations/layer16_attention_weight_max": 31.68126678466797,
      "activations/layer16_attention_weight_min": -27.974626541137695,
      "activations/layer17_attention_weight_max": 33.63228225708008,
      "activations/layer17_attention_weight_min": -26.1760196685791,
      "activations/layer18_attention_weight_max": 31.12013053894043,
      "activations/layer18_attention_weight_min": -22.80276870727539,
      "activations/layer19_attention_weight_max": 35.8714714050293,
      "activations/layer19_attention_weight_min": -30.723093032836914,
      "activations/layer1_attention_weight_max": 16.07868194580078,
      "activations/layer1_attention_weight_min": -14.796562194824219,
      "activations/layer20_attention_weight_max": 30.78219223022461,
      "activations/layer20_attention_weight_min": -24.800270080566406,
      "activations/layer21_attention_weight_max": 30.92216682434082,
      "activations/layer21_attention_weight_min": -23.960779190063477,
      "activations/layer22_attention_weight_max": 56.29098892211914,
      "activations/layer22_attention_weight_min": -30.31970977783203,
      "activations/layer23_attention_weight_max": 38.900657653808594,
      "activations/layer23_attention_weight_min": -25.552440643310547,
      "activations/layer2_attention_weight_max": 32.44803237915039,
      "activations/layer2_attention_weight_min": -32.700477600097656,
      "activations/layer3_attention_weight_max": 92.09671020507812,
      "activations/layer3_attention_weight_min": -94.52356719970703,
      "activations/layer4_attention_weight_max": 114.07608795166016,
      "activations/layer4_attention_weight_min": -110.85235595703125,
      "activations/layer5_attention_weight_max": 52.811317443847656,
      "activations/layer5_attention_weight_min": -61.86426544189453,
      "activations/layer6_attention_weight_max": 43.27781295776367,
      "activations/layer6_attention_weight_min": -46.864871978759766,
      "activations/layer7_attention_weight_max": 88.64478302001953,
      "activations/layer7_attention_weight_min": -90.27919006347656,
      "activations/layer8_attention_weight_max": 40.583370208740234,
      "activations/layer8_attention_weight_min": -41.14250946044922,
      "activations/layer9_attention_weight_max": 31.47003746032715,
      "activations/layer9_attention_weight_min": -32.70357894897461,
      "epoch": 10.55,
      "learning_rate": 8.278598484848484e-05,
      "loss": 2.7722,
      "step": 181600
    },
    {
      "activations/layer0_attention_weight_max": 15.502056121826172,
      "activations/layer0_attention_weight_min": -12.58014965057373,
      "activations/layer10_attention_weight_max": 34.41652297973633,
      "activations/layer10_attention_weight_min": -34.706905364990234,
      "activations/layer11_attention_weight_max": 31.90001678466797,
      "activations/layer11_attention_weight_min": -32.81816482543945,
      "activations/layer12_attention_weight_max": 26.02350616455078,
      "activations/layer12_attention_weight_min": -23.754867553710938,
      "activations/layer13_attention_weight_max": 42.11506271362305,
      "activations/layer13_attention_weight_min": -33.58998107910156,
      "activations/layer14_attention_weight_max": 46.7384147644043,
      "activations/layer14_attention_weight_min": -37.792747497558594,
      "activations/layer15_attention_weight_max": 35.6844596862793,
      "activations/layer15_attention_weight_min": -30.372072219848633,
      "activations/layer16_attention_weight_max": 30.72770881652832,
      "activations/layer16_attention_weight_min": -28.37581443786621,
      "activations/layer17_attention_weight_max": 31.754053115844727,
      "activations/layer17_attention_weight_min": -24.620296478271484,
      "activations/layer18_attention_weight_max": 30.10934066772461,
      "activations/layer18_attention_weight_min": -21.733999252319336,
      "activations/layer19_attention_weight_max": 35.55243682861328,
      "activations/layer19_attention_weight_min": -30.938291549682617,
      "activations/layer1_attention_weight_max": 16.692224502563477,
      "activations/layer1_attention_weight_min": -14.229671478271484,
      "activations/layer20_attention_weight_max": 33.45283126831055,
      "activations/layer20_attention_weight_min": -24.90304183959961,
      "activations/layer21_attention_weight_max": 30.268577575683594,
      "activations/layer21_attention_weight_min": -22.999385833740234,
      "activations/layer22_attention_weight_max": 39.987571716308594,
      "activations/layer22_attention_weight_min": -27.707246780395508,
      "activations/layer23_attention_weight_max": 32.93247985839844,
      "activations/layer23_attention_weight_min": -23.63515281677246,
      "activations/layer2_attention_weight_max": 33.22309875488281,
      "activations/layer2_attention_weight_min": -33.672996520996094,
      "activations/layer3_attention_weight_max": 96.83231353759766,
      "activations/layer3_attention_weight_min": -97.3333511352539,
      "activations/layer4_attention_weight_max": 119.28250885009766,
      "activations/layer4_attention_weight_min": -120.9480972290039,
      "activations/layer5_attention_weight_max": 53.36240768432617,
      "activations/layer5_attention_weight_min": -67.98927307128906,
      "activations/layer6_attention_weight_max": 44.64539337158203,
      "activations/layer6_attention_weight_min": -49.04065704345703,
      "activations/layer7_attention_weight_max": 89.3693618774414,
      "activations/layer7_attention_weight_min": -89.6843490600586,
      "activations/layer8_attention_weight_max": 43.15337371826172,
      "activations/layer8_attention_weight_min": -42.84022903442383,
      "activations/layer9_attention_weight_max": 37.88982391357422,
      "activations/layer9_attention_weight_min": -35.32150650024414,
      "epoch": 10.55,
      "learning_rate": 8.276704545454544e-05,
      "loss": 2.7614,
      "step": 181650
    },
    {
      "activations/layer0_attention_weight_max": 14.717329025268555,
      "activations/layer0_attention_weight_min": -13.707096099853516,
      "activations/layer10_attention_weight_max": 31.152746200561523,
      "activations/layer10_attention_weight_min": -33.06047821044922,
      "activations/layer11_attention_weight_max": 31.870521545410156,
      "activations/layer11_attention_weight_min": -31.946022033691406,
      "activations/layer12_attention_weight_max": 24.071979522705078,
      "activations/layer12_attention_weight_min": -23.28846549987793,
      "activations/layer13_attention_weight_max": 36.2024040222168,
      "activations/layer13_attention_weight_min": -32.56300354003906,
      "activations/layer14_attention_weight_max": 45.66121292114258,
      "activations/layer14_attention_weight_min": -36.7942008972168,
      "activations/layer15_attention_weight_max": 31.744874954223633,
      "activations/layer15_attention_weight_min": -29.0771484375,
      "activations/layer16_attention_weight_max": 27.884870529174805,
      "activations/layer16_attention_weight_min": -28.363384246826172,
      "activations/layer17_attention_weight_max": 30.357540130615234,
      "activations/layer17_attention_weight_min": -24.780649185180664,
      "activations/layer18_attention_weight_max": 29.348793029785156,
      "activations/layer18_attention_weight_min": -22.527280807495117,
      "activations/layer19_attention_weight_max": 30.485910415649414,
      "activations/layer19_attention_weight_min": -29.713790893554688,
      "activations/layer1_attention_weight_max": 16.630935668945312,
      "activations/layer1_attention_weight_min": -14.580178260803223,
      "activations/layer20_attention_weight_max": 26.789615631103516,
      "activations/layer20_attention_weight_min": -24.0651912689209,
      "activations/layer21_attention_weight_max": 28.927743911743164,
      "activations/layer21_attention_weight_min": -24.33774757385254,
      "activations/layer22_attention_weight_max": 38.29454803466797,
      "activations/layer22_attention_weight_min": -28.130104064941406,
      "activations/layer23_attention_weight_max": 32.37459945678711,
      "activations/layer23_attention_weight_min": -24.10080337524414,
      "activations/layer2_attention_weight_max": 33.47624206542969,
      "activations/layer2_attention_weight_min": -33.849395751953125,
      "activations/layer3_attention_weight_max": 98.96134948730469,
      "activations/layer3_attention_weight_min": -97.47346496582031,
      "activations/layer4_attention_weight_max": 117.10863494873047,
      "activations/layer4_attention_weight_min": -118.63128662109375,
      "activations/layer5_attention_weight_max": 53.77330780029297,
      "activations/layer5_attention_weight_min": -62.485809326171875,
      "activations/layer6_attention_weight_max": 45.45722198486328,
      "activations/layer6_attention_weight_min": -47.331180572509766,
      "activations/layer7_attention_weight_max": 89.4538345336914,
      "activations/layer7_attention_weight_min": -88.33148956298828,
      "activations/layer8_attention_weight_max": 39.33100891113281,
      "activations/layer8_attention_weight_min": -40.69709014892578,
      "activations/layer9_attention_weight_max": 33.56338119506836,
      "activations/layer9_attention_weight_min": -33.375545501708984,
      "epoch": 10.56,
      "learning_rate": 8.274810606060606e-05,
      "loss": 2.7857,
      "step": 181700
    },
    {
      "activations/layer0_attention_weight_max": 16.18718719482422,
      "activations/layer0_attention_weight_min": -13.455808639526367,
      "activations/layer10_attention_weight_max": 31.08871841430664,
      "activations/layer10_attention_weight_min": -35.2241325378418,
      "activations/layer11_attention_weight_max": 33.570465087890625,
      "activations/layer11_attention_weight_min": -34.438140869140625,
      "activations/layer12_attention_weight_max": 30.578449249267578,
      "activations/layer12_attention_weight_min": -26.734407424926758,
      "activations/layer13_attention_weight_max": 50.765377044677734,
      "activations/layer13_attention_weight_min": -36.11476516723633,
      "activations/layer14_attention_weight_max": 57.82099533081055,
      "activations/layer14_attention_weight_min": -40.19355392456055,
      "activations/layer15_attention_weight_max": 45.82196044921875,
      "activations/layer15_attention_weight_min": -33.60735321044922,
      "activations/layer16_attention_weight_max": 39.65971755981445,
      "activations/layer16_attention_weight_min": -29.94623374938965,
      "activations/layer17_attention_weight_max": 37.106605529785156,
      "activations/layer17_attention_weight_min": -26.674015045166016,
      "activations/layer18_attention_weight_max": 36.97200012207031,
      "activations/layer18_attention_weight_min": -23.872982025146484,
      "activations/layer19_attention_weight_max": 41.81561279296875,
      "activations/layer19_attention_weight_min": -32.51669692993164,
      "activations/layer1_attention_weight_max": 18.825912475585938,
      "activations/layer1_attention_weight_min": -17.447832107543945,
      "activations/layer20_attention_weight_max": 34.61263656616211,
      "activations/layer20_attention_weight_min": -27.514387130737305,
      "activations/layer21_attention_weight_max": 33.1781120300293,
      "activations/layer21_attention_weight_min": -25.00497817993164,
      "activations/layer22_attention_weight_max": 57.82034683227539,
      "activations/layer22_attention_weight_min": -29.952669143676758,
      "activations/layer23_attention_weight_max": 46.04378890991211,
      "activations/layer23_attention_weight_min": -25.88483428955078,
      "activations/layer2_attention_weight_max": 32.723106384277344,
      "activations/layer2_attention_weight_min": -33.491424560546875,
      "activations/layer3_attention_weight_max": 92.07080841064453,
      "activations/layer3_attention_weight_min": -93.64887237548828,
      "activations/layer4_attention_weight_max": 111.5824966430664,
      "activations/layer4_attention_weight_min": -108.94557189941406,
      "activations/layer5_attention_weight_max": 50.10169219970703,
      "activations/layer5_attention_weight_min": -60.01268768310547,
      "activations/layer6_attention_weight_max": 43.97582244873047,
      "activations/layer6_attention_weight_min": -45.10743713378906,
      "activations/layer7_attention_weight_max": 89.41829681396484,
      "activations/layer7_attention_weight_min": -84.37113952636719,
      "activations/layer8_attention_weight_max": 37.529232025146484,
      "activations/layer8_attention_weight_min": -41.41716003417969,
      "activations/layer9_attention_weight_max": 33.71955490112305,
      "activations/layer9_attention_weight_min": -33.114891052246094,
      "epoch": 10.56,
      "learning_rate": 8.272916666666665e-05,
      "loss": 2.7821,
      "step": 181750
    },
    {
      "activations/layer0_attention_weight_max": 15.437830924987793,
      "activations/layer0_attention_weight_min": -13.052143096923828,
      "activations/layer10_attention_weight_max": 32.512298583984375,
      "activations/layer10_attention_weight_min": -32.19963836669922,
      "activations/layer11_attention_weight_max": 32.06634521484375,
      "activations/layer11_attention_weight_min": -30.664823532104492,
      "activations/layer12_attention_weight_max": 25.54291343688965,
      "activations/layer12_attention_weight_min": -25.74307632446289,
      "activations/layer13_attention_weight_max": 39.92052459716797,
      "activations/layer13_attention_weight_min": -35.96041488647461,
      "activations/layer14_attention_weight_max": 45.47099304199219,
      "activations/layer14_attention_weight_min": -39.32809066772461,
      "activations/layer15_attention_weight_max": 37.9754524230957,
      "activations/layer15_attention_weight_min": -34.01268768310547,
      "activations/layer16_attention_weight_max": 31.18927764892578,
      "activations/layer16_attention_weight_min": -29.1474609375,
      "activations/layer17_attention_weight_max": 31.696664810180664,
      "activations/layer17_attention_weight_min": -27.435903549194336,
      "activations/layer18_attention_weight_max": 30.691940307617188,
      "activations/layer18_attention_weight_min": -23.785083770751953,
      "activations/layer19_attention_weight_max": 33.77846145629883,
      "activations/layer19_attention_weight_min": -31.913785934448242,
      "activations/layer1_attention_weight_max": 15.974833488464355,
      "activations/layer1_attention_weight_min": -13.538003921508789,
      "activations/layer20_attention_weight_max": 33.073890686035156,
      "activations/layer20_attention_weight_min": -26.265331268310547,
      "activations/layer21_attention_weight_max": 29.98512840270996,
      "activations/layer21_attention_weight_min": -24.581064224243164,
      "activations/layer22_attention_weight_max": 46.267494201660156,
      "activations/layer22_attention_weight_min": -32.29823303222656,
      "activations/layer23_attention_weight_max": 34.323875427246094,
      "activations/layer23_attention_weight_min": -24.74688720703125,
      "activations/layer2_attention_weight_max": 33.52178955078125,
      "activations/layer2_attention_weight_min": -31.888080596923828,
      "activations/layer3_attention_weight_max": 96.9068603515625,
      "activations/layer3_attention_weight_min": -96.7464370727539,
      "activations/layer4_attention_weight_max": 117.10872650146484,
      "activations/layer4_attention_weight_min": -116.15279388427734,
      "activations/layer5_attention_weight_max": 57.26247787475586,
      "activations/layer5_attention_weight_min": -62.807308197021484,
      "activations/layer6_attention_weight_max": 44.85306167602539,
      "activations/layer6_attention_weight_min": -48.58357238769531,
      "activations/layer7_attention_weight_max": 92.91121673583984,
      "activations/layer7_attention_weight_min": -91.689453125,
      "activations/layer8_attention_weight_max": 40.49104690551758,
      "activations/layer8_attention_weight_min": -40.37489700317383,
      "activations/layer9_attention_weight_max": 36.023895263671875,
      "activations/layer9_attention_weight_min": -34.724700927734375,
      "epoch": 10.56,
      "learning_rate": 8.271022727272726e-05,
      "loss": 2.7835,
      "step": 181800
    },
    {
      "activations/layer0_attention_weight_max": 15.19370174407959,
      "activations/layer0_attention_weight_min": -12.297038078308105,
      "activations/layer10_attention_weight_max": 34.03499221801758,
      "activations/layer10_attention_weight_min": -32.969757080078125,
      "activations/layer11_attention_weight_max": 35.318992614746094,
      "activations/layer11_attention_weight_min": -32.85065841674805,
      "activations/layer12_attention_weight_max": 24.560312271118164,
      "activations/layer12_attention_weight_min": -25.318479537963867,
      "activations/layer13_attention_weight_max": 38.62031173706055,
      "activations/layer13_attention_weight_min": -33.77955627441406,
      "activations/layer14_attention_weight_max": 47.83170700073242,
      "activations/layer14_attention_weight_min": -40.458946228027344,
      "activations/layer15_attention_weight_max": 41.70726013183594,
      "activations/layer15_attention_weight_min": -30.586408615112305,
      "activations/layer16_attention_weight_max": 30.52361297607422,
      "activations/layer16_attention_weight_min": -25.371688842773438,
      "activations/layer17_attention_weight_max": 30.87148094177246,
      "activations/layer17_attention_weight_min": -24.809829711914062,
      "activations/layer18_attention_weight_max": 32.50275421142578,
      "activations/layer18_attention_weight_min": -24.651443481445312,
      "activations/layer19_attention_weight_max": 34.61851119995117,
      "activations/layer19_attention_weight_min": -33.17494201660156,
      "activations/layer1_attention_weight_max": 16.4770565032959,
      "activations/layer1_attention_weight_min": -14.876510620117188,
      "activations/layer20_attention_weight_max": 26.805042266845703,
      "activations/layer20_attention_weight_min": -25.15450668334961,
      "activations/layer21_attention_weight_max": 26.196123123168945,
      "activations/layer21_attention_weight_min": -23.599578857421875,
      "activations/layer22_attention_weight_max": 41.627620697021484,
      "activations/layer22_attention_weight_min": -31.161075592041016,
      "activations/layer23_attention_weight_max": 35.95292282104492,
      "activations/layer23_attention_weight_min": -22.378202438354492,
      "activations/layer2_attention_weight_max": 32.73097229003906,
      "activations/layer2_attention_weight_min": -32.483333587646484,
      "activations/layer3_attention_weight_max": 90.83088684082031,
      "activations/layer3_attention_weight_min": -96.02806091308594,
      "activations/layer4_attention_weight_max": 107.5701904296875,
      "activations/layer4_attention_weight_min": -110.36800384521484,
      "activations/layer5_attention_weight_max": 52.27145004272461,
      "activations/layer5_attention_weight_min": -66.00602722167969,
      "activations/layer6_attention_weight_max": 42.719520568847656,
      "activations/layer6_attention_weight_min": -45.686363220214844,
      "activations/layer7_attention_weight_max": 83.31561279296875,
      "activations/layer7_attention_weight_min": -88.18905639648438,
      "activations/layer8_attention_weight_max": 39.88468933105469,
      "activations/layer8_attention_weight_min": -40.350215911865234,
      "activations/layer9_attention_weight_max": 33.97751998901367,
      "activations/layer9_attention_weight_min": -35.353816986083984,
      "epoch": 10.57,
      "learning_rate": 8.269128787878788e-05,
      "loss": 2.7891,
      "step": 181850
    },
    {
      "activations/layer0_attention_weight_max": 15.144224166870117,
      "activations/layer0_attention_weight_min": -13.085006713867188,
      "activations/layer10_attention_weight_max": 35.726253509521484,
      "activations/layer10_attention_weight_min": -35.43968200683594,
      "activations/layer11_attention_weight_max": 35.8475456237793,
      "activations/layer11_attention_weight_min": -32.62001419067383,
      "activations/layer12_attention_weight_max": 34.36684036254883,
      "activations/layer12_attention_weight_min": -25.5304012298584,
      "activations/layer13_attention_weight_max": 51.822059631347656,
      "activations/layer13_attention_weight_min": -35.703453063964844,
      "activations/layer14_attention_weight_max": 46.65364456176758,
      "activations/layer14_attention_weight_min": -38.279258728027344,
      "activations/layer15_attention_weight_max": 47.40562057495117,
      "activations/layer15_attention_weight_min": -34.04684829711914,
      "activations/layer16_attention_weight_max": 34.51995086669922,
      "activations/layer16_attention_weight_min": -28.179092407226562,
      "activations/layer17_attention_weight_max": 35.04053497314453,
      "activations/layer17_attention_weight_min": -28.824968338012695,
      "activations/layer18_attention_weight_max": 32.18532943725586,
      "activations/layer18_attention_weight_min": -25.71942710876465,
      "activations/layer19_attention_weight_max": 36.3101921081543,
      "activations/layer19_attention_weight_min": -31.330007553100586,
      "activations/layer1_attention_weight_max": 15.845330238342285,
      "activations/layer1_attention_weight_min": -15.1572265625,
      "activations/layer20_attention_weight_max": 31.566410064697266,
      "activations/layer20_attention_weight_min": -25.36631965637207,
      "activations/layer21_attention_weight_max": 32.11568069458008,
      "activations/layer21_attention_weight_min": -24.372074127197266,
      "activations/layer22_attention_weight_max": 47.020084381103516,
      "activations/layer22_attention_weight_min": -31.97394561767578,
      "activations/layer23_attention_weight_max": 35.93339538574219,
      "activations/layer23_attention_weight_min": -25.383426666259766,
      "activations/layer2_attention_weight_max": 33.14561080932617,
      "activations/layer2_attention_weight_min": -32.57660675048828,
      "activations/layer3_attention_weight_max": 102.13365173339844,
      "activations/layer3_attention_weight_min": -99.98211669921875,
      "activations/layer4_attention_weight_max": 124.26029205322266,
      "activations/layer4_attention_weight_min": -118.0937728881836,
      "activations/layer5_attention_weight_max": 53.800472259521484,
      "activations/layer5_attention_weight_min": -65.19585418701172,
      "activations/layer6_attention_weight_max": 48.541419982910156,
      "activations/layer6_attention_weight_min": -50.89765548706055,
      "activations/layer7_attention_weight_max": 97.67156982421875,
      "activations/layer7_attention_weight_min": -90.3896713256836,
      "activations/layer8_attention_weight_max": 43.626380920410156,
      "activations/layer8_attention_weight_min": -43.67143249511719,
      "activations/layer9_attention_weight_max": 39.07288360595703,
      "activations/layer9_attention_weight_min": -35.15142822265625,
      "epoch": 10.57,
      "learning_rate": 8.267234848484848e-05,
      "loss": 2.7809,
      "step": 181900
    },
    {
      "activations/layer0_attention_weight_max": 15.87719440460205,
      "activations/layer0_attention_weight_min": -13.031172752380371,
      "activations/layer10_attention_weight_max": 36.858245849609375,
      "activations/layer10_attention_weight_min": -33.91010284423828,
      "activations/layer11_attention_weight_max": 36.74869918823242,
      "activations/layer11_attention_weight_min": -35.931697845458984,
      "activations/layer12_attention_weight_max": 24.40953826904297,
      "activations/layer12_attention_weight_min": -25.22114372253418,
      "activations/layer13_attention_weight_max": 38.91193389892578,
      "activations/layer13_attention_weight_min": -34.6735725402832,
      "activations/layer14_attention_weight_max": 43.77334976196289,
      "activations/layer14_attention_weight_min": -37.058799743652344,
      "activations/layer15_attention_weight_max": 42.107242584228516,
      "activations/layer15_attention_weight_min": -30.33690643310547,
      "activations/layer16_attention_weight_max": 31.468563079833984,
      "activations/layer16_attention_weight_min": -27.080707550048828,
      "activations/layer17_attention_weight_max": 30.94644546508789,
      "activations/layer17_attention_weight_min": -25.820281982421875,
      "activations/layer18_attention_weight_max": 31.071001052856445,
      "activations/layer18_attention_weight_min": -23.324705123901367,
      "activations/layer19_attention_weight_max": 32.95771408081055,
      "activations/layer19_attention_weight_min": -31.771329879760742,
      "activations/layer1_attention_weight_max": 16.774789810180664,
      "activations/layer1_attention_weight_min": -14.767610549926758,
      "activations/layer20_attention_weight_max": 28.826770782470703,
      "activations/layer20_attention_weight_min": -25.56842613220215,
      "activations/layer21_attention_weight_max": 29.67632293701172,
      "activations/layer21_attention_weight_min": -26.469816207885742,
      "activations/layer22_attention_weight_max": 42.850521087646484,
      "activations/layer22_attention_weight_min": -30.014156341552734,
      "activations/layer23_attention_weight_max": 33.603553771972656,
      "activations/layer23_attention_weight_min": -27.101558685302734,
      "activations/layer2_attention_weight_max": 33.35718536376953,
      "activations/layer2_attention_weight_min": -33.90571975708008,
      "activations/layer3_attention_weight_max": 94.51675415039062,
      "activations/layer3_attention_weight_min": -98.04764556884766,
      "activations/layer4_attention_weight_max": 114.97674560546875,
      "activations/layer4_attention_weight_min": -111.3633804321289,
      "activations/layer5_attention_weight_max": 52.78461456298828,
      "activations/layer5_attention_weight_min": -63.58442687988281,
      "activations/layer6_attention_weight_max": 46.05250930786133,
      "activations/layer6_attention_weight_min": -47.58103942871094,
      "activations/layer7_attention_weight_max": 95.39979553222656,
      "activations/layer7_attention_weight_min": -88.59284973144531,
      "activations/layer8_attention_weight_max": 42.61934280395508,
      "activations/layer8_attention_weight_min": -41.76798629760742,
      "activations/layer9_attention_weight_max": 36.19709396362305,
      "activations/layer9_attention_weight_min": -33.730594635009766,
      "epoch": 10.57,
      "learning_rate": 8.265378787878787e-05,
      "loss": 2.7816,
      "step": 181950
    },
    {
      "activations/layer0_attention_weight_max": 15.778559684753418,
      "activations/layer0_attention_weight_min": -13.81533432006836,
      "activations/layer10_attention_weight_max": 33.325233459472656,
      "activations/layer10_attention_weight_min": -31.382278442382812,
      "activations/layer11_attention_weight_max": 30.602031707763672,
      "activations/layer11_attention_weight_min": -29.982887268066406,
      "activations/layer12_attention_weight_max": 24.71354866027832,
      "activations/layer12_attention_weight_min": -25.478485107421875,
      "activations/layer13_attention_weight_max": 37.16139602661133,
      "activations/layer13_attention_weight_min": -34.14067459106445,
      "activations/layer14_attention_weight_max": 43.07899856567383,
      "activations/layer14_attention_weight_min": -37.08677291870117,
      "activations/layer15_attention_weight_max": 33.78828430175781,
      "activations/layer15_attention_weight_min": -34.437007904052734,
      "activations/layer16_attention_weight_max": 29.616172790527344,
      "activations/layer16_attention_weight_min": -28.005887985229492,
      "activations/layer17_attention_weight_max": 36.592891693115234,
      "activations/layer17_attention_weight_min": -24.685070037841797,
      "activations/layer18_attention_weight_max": 34.374847412109375,
      "activations/layer18_attention_weight_min": -22.833961486816406,
      "activations/layer19_attention_weight_max": 34.45311737060547,
      "activations/layer19_attention_weight_min": -31.436674118041992,
      "activations/layer1_attention_weight_max": 17.25647735595703,
      "activations/layer1_attention_weight_min": -13.784184455871582,
      "activations/layer20_attention_weight_max": 28.260103225708008,
      "activations/layer20_attention_weight_min": -25.27462387084961,
      "activations/layer21_attention_weight_max": 30.369888305664062,
      "activations/layer21_attention_weight_min": -24.37431526184082,
      "activations/layer22_attention_weight_max": 40.31210708618164,
      "activations/layer22_attention_weight_min": -30.163118362426758,
      "activations/layer23_attention_weight_max": 32.45808410644531,
      "activations/layer23_attention_weight_min": -24.65595245361328,
      "activations/layer2_attention_weight_max": 31.727996826171875,
      "activations/layer2_attention_weight_min": -30.971006393432617,
      "activations/layer3_attention_weight_max": 91.63563537597656,
      "activations/layer3_attention_weight_min": -93.810546875,
      "activations/layer4_attention_weight_max": 111.1776351928711,
      "activations/layer4_attention_weight_min": -115.2318344116211,
      "activations/layer5_attention_weight_max": 53.95079040527344,
      "activations/layer5_attention_weight_min": -64.03543090820312,
      "activations/layer6_attention_weight_max": 43.56135940551758,
      "activations/layer6_attention_weight_min": -47.48885726928711,
      "activations/layer7_attention_weight_max": 87.46946716308594,
      "activations/layer7_attention_weight_min": -89.00518798828125,
      "activations/layer8_attention_weight_max": 39.121089935302734,
      "activations/layer8_attention_weight_min": -40.32448196411133,
      "activations/layer9_attention_weight_max": 35.9255256652832,
      "activations/layer9_attention_weight_min": -31.746103286743164,
      "epoch": 10.58,
      "learning_rate": 8.263484848484847e-05,
      "loss": 2.7941,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5178,
      "eval_samples_per_second": 504.119,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5178,
      "eval_openwebtext_samples_per_second": 504.119,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0177,
      "eval_wikitext_samples_per_second": 225.999,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.6237,
      "eval_lambada_samples_per_second": 505.94,
      "step": 182000
    },
    {
      "activations/layer0_attention_weight_max": 15.665141105651855,
      "activations/layer0_attention_weight_min": -12.917929649353027,
      "activations/layer10_attention_weight_max": 29.60384750366211,
      "activations/layer10_attention_weight_min": -31.99849510192871,
      "activations/layer11_attention_weight_max": 29.54513168334961,
      "activations/layer11_attention_weight_min": -32.069740295410156,
      "activations/layer12_attention_weight_max": 22.303016662597656,
      "activations/layer12_attention_weight_min": -24.67208480834961,
      "activations/layer13_attention_weight_max": 36.70811462402344,
      "activations/layer13_attention_weight_min": -35.72181701660156,
      "activations/layer14_attention_weight_max": 40.23634338378906,
      "activations/layer14_attention_weight_min": -37.049842834472656,
      "activations/layer15_attention_weight_max": 35.280548095703125,
      "activations/layer15_attention_weight_min": -31.573240280151367,
      "activations/layer16_attention_weight_max": 29.156658172607422,
      "activations/layer16_attention_weight_min": -28.39093780517578,
      "activations/layer17_attention_weight_max": 31.5075626373291,
      "activations/layer17_attention_weight_min": -26.10515785217285,
      "activations/layer18_attention_weight_max": 32.311363220214844,
      "activations/layer18_attention_weight_min": -22.300127029418945,
      "activations/layer19_attention_weight_max": 33.85271072387695,
      "activations/layer19_attention_weight_min": -31.148868560791016,
      "activations/layer1_attention_weight_max": 16.920469284057617,
      "activations/layer1_attention_weight_min": -15.30275821685791,
      "activations/layer20_attention_weight_max": 30.665727615356445,
      "activations/layer20_attention_weight_min": -24.590946197509766,
      "activations/layer21_attention_weight_max": 30.00370979309082,
      "activations/layer21_attention_weight_min": -23.173831939697266,
      "activations/layer22_attention_weight_max": 46.57049560546875,
      "activations/layer22_attention_weight_min": -28.656232833862305,
      "activations/layer23_attention_weight_max": 36.16242980957031,
      "activations/layer23_attention_weight_min": -24.049636840820312,
      "activations/layer2_attention_weight_max": 32.231407165527344,
      "activations/layer2_attention_weight_min": -31.75434684753418,
      "activations/layer3_attention_weight_max": 90.932861328125,
      "activations/layer3_attention_weight_min": -94.18244171142578,
      "activations/layer4_attention_weight_max": 110.37383270263672,
      "activations/layer4_attention_weight_min": -108.4655532836914,
      "activations/layer5_attention_weight_max": 51.29697799682617,
      "activations/layer5_attention_weight_min": -61.878265380859375,
      "activations/layer6_attention_weight_max": 44.32276153564453,
      "activations/layer6_attention_weight_min": -48.783851623535156,
      "activations/layer7_attention_weight_max": 86.7071304321289,
      "activations/layer7_attention_weight_min": -90.22228240966797,
      "activations/layer8_attention_weight_max": 37.200984954833984,
      "activations/layer8_attention_weight_min": -42.92941665649414,
      "activations/layer9_attention_weight_max": 33.49769973754883,
      "activations/layer9_attention_weight_min": -33.27140426635742,
      "epoch": 10.58,
      "learning_rate": 8.261590909090909e-05,
      "loss": 2.7874,
      "step": 182050
    },
    {
      "activations/layer0_attention_weight_max": 16.26952362060547,
      "activations/layer0_attention_weight_min": -12.886112213134766,
      "activations/layer10_attention_weight_max": 34.87338638305664,
      "activations/layer10_attention_weight_min": -32.54730987548828,
      "activations/layer11_attention_weight_max": 32.396629333496094,
      "activations/layer11_attention_weight_min": -31.68563461303711,
      "activations/layer12_attention_weight_max": 27.173892974853516,
      "activations/layer12_attention_weight_min": -24.978118896484375,
      "activations/layer13_attention_weight_max": 42.54277801513672,
      "activations/layer13_attention_weight_min": -35.05497360229492,
      "activations/layer14_attention_weight_max": 54.41398239135742,
      "activations/layer14_attention_weight_min": -41.00312042236328,
      "activations/layer15_attention_weight_max": 52.36106872558594,
      "activations/layer15_attention_weight_min": -32.22840118408203,
      "activations/layer16_attention_weight_max": 30.992204666137695,
      "activations/layer16_attention_weight_min": -27.68960952758789,
      "activations/layer17_attention_weight_max": 31.03812026977539,
      "activations/layer17_attention_weight_min": -25.018341064453125,
      "activations/layer18_attention_weight_max": 31.021175384521484,
      "activations/layer18_attention_weight_min": -21.774614334106445,
      "activations/layer19_attention_weight_max": 34.52020263671875,
      "activations/layer19_attention_weight_min": -34.80400466918945,
      "activations/layer1_attention_weight_max": 16.11921501159668,
      "activations/layer1_attention_weight_min": -14.316254615783691,
      "activations/layer20_attention_weight_max": 31.025571823120117,
      "activations/layer20_attention_weight_min": -24.225669860839844,
      "activations/layer21_attention_weight_max": 30.44919776916504,
      "activations/layer21_attention_weight_min": -23.314556121826172,
      "activations/layer22_attention_weight_max": 47.66654586791992,
      "activations/layer22_attention_weight_min": -28.810178756713867,
      "activations/layer23_attention_weight_max": 32.273956298828125,
      "activations/layer23_attention_weight_min": -23.438426971435547,
      "activations/layer2_attention_weight_max": 33.39192581176758,
      "activations/layer2_attention_weight_min": -31.31298828125,
      "activations/layer3_attention_weight_max": 97.49234008789062,
      "activations/layer3_attention_weight_min": -91.8990249633789,
      "activations/layer4_attention_weight_max": 115.3895034790039,
      "activations/layer4_attention_weight_min": -110.53108215332031,
      "activations/layer5_attention_weight_max": 52.81241989135742,
      "activations/layer5_attention_weight_min": -59.712196350097656,
      "activations/layer6_attention_weight_max": 43.88936233520508,
      "activations/layer6_attention_weight_min": -48.35292053222656,
      "activations/layer7_attention_weight_max": 88.62000274658203,
      "activations/layer7_attention_weight_min": -85.9977035522461,
      "activations/layer8_attention_weight_max": 40.95252227783203,
      "activations/layer8_attention_weight_min": -40.63905334472656,
      "activations/layer9_attention_weight_max": 34.07569122314453,
      "activations/layer9_attention_weight_min": -32.5317268371582,
      "epoch": 10.58,
      "learning_rate": 8.259696969696969e-05,
      "loss": 2.7893,
      "step": 182100
    },
    {
      "activations/layer0_attention_weight_max": 16.129016876220703,
      "activations/layer0_attention_weight_min": -12.602530479431152,
      "activations/layer10_attention_weight_max": 40.611446380615234,
      "activations/layer10_attention_weight_min": -38.50578689575195,
      "activations/layer11_attention_weight_max": 43.05094528198242,
      "activations/layer11_attention_weight_min": -39.032318115234375,
      "activations/layer12_attention_weight_max": 30.533126831054688,
      "activations/layer12_attention_weight_min": -28.032588958740234,
      "activations/layer13_attention_weight_max": 62.527488708496094,
      "activations/layer13_attention_weight_min": -38.685420989990234,
      "activations/layer14_attention_weight_max": 70.05069732666016,
      "activations/layer14_attention_weight_min": -45.173919677734375,
      "activations/layer15_attention_weight_max": 77.32728576660156,
      "activations/layer15_attention_weight_min": -38.8064079284668,
      "activations/layer16_attention_weight_max": 45.525447845458984,
      "activations/layer16_attention_weight_min": -27.540929794311523,
      "activations/layer17_attention_weight_max": 33.477867126464844,
      "activations/layer17_attention_weight_min": -29.77143669128418,
      "activations/layer18_attention_weight_max": 38.86790466308594,
      "activations/layer18_attention_weight_min": -26.416011810302734,
      "activations/layer19_attention_weight_max": 45.702720642089844,
      "activations/layer19_attention_weight_min": -30.404541015625,
      "activations/layer1_attention_weight_max": 19.298660278320312,
      "activations/layer1_attention_weight_min": -14.606413841247559,
      "activations/layer20_attention_weight_max": 39.189361572265625,
      "activations/layer20_attention_weight_min": -26.816417694091797,
      "activations/layer21_attention_weight_max": 34.68477249145508,
      "activations/layer21_attention_weight_min": -25.00510025024414,
      "activations/layer22_attention_weight_max": 64.89051818847656,
      "activations/layer22_attention_weight_min": -33.448116302490234,
      "activations/layer23_attention_weight_max": 41.71969223022461,
      "activations/layer23_attention_weight_min": -23.654457092285156,
      "activations/layer2_attention_weight_max": 34.421504974365234,
      "activations/layer2_attention_weight_min": -34.753013610839844,
      "activations/layer3_attention_weight_max": 97.17595672607422,
      "activations/layer3_attention_weight_min": -100.03526306152344,
      "activations/layer4_attention_weight_max": 120.83876037597656,
      "activations/layer4_attention_weight_min": -115.01673889160156,
      "activations/layer5_attention_weight_max": 56.78575897216797,
      "activations/layer5_attention_weight_min": -66.48949432373047,
      "activations/layer6_attention_weight_max": 47.269840240478516,
      "activations/layer6_attention_weight_min": -47.52809143066406,
      "activations/layer7_attention_weight_max": 112.5550537109375,
      "activations/layer7_attention_weight_min": -107.7553939819336,
      "activations/layer8_attention_weight_max": 48.92927932739258,
      "activations/layer8_attention_weight_min": -48.65776824951172,
      "activations/layer9_attention_weight_max": 40.658077239990234,
      "activations/layer9_attention_weight_min": -39.97115707397461,
      "epoch": 10.58,
      "learning_rate": 8.25780303030303e-05,
      "loss": 2.7932,
      "step": 182150
    },
    {
      "activations/layer0_attention_weight_max": 15.665027618408203,
      "activations/layer0_attention_weight_min": -12.489462852478027,
      "activations/layer10_attention_weight_max": 33.079769134521484,
      "activations/layer10_attention_weight_min": -34.4559211730957,
      "activations/layer11_attention_weight_max": 34.71125030517578,
      "activations/layer11_attention_weight_min": -33.358577728271484,
      "activations/layer12_attention_weight_max": 24.374250411987305,
      "activations/layer12_attention_weight_min": -25.122604370117188,
      "activations/layer13_attention_weight_max": 39.29087829589844,
      "activations/layer13_attention_weight_min": -32.539703369140625,
      "activations/layer14_attention_weight_max": 44.0318603515625,
      "activations/layer14_attention_weight_min": -35.9112663269043,
      "activations/layer15_attention_weight_max": 37.514305114746094,
      "activations/layer15_attention_weight_min": -30.27628517150879,
      "activations/layer16_attention_weight_max": 31.75177764892578,
      "activations/layer16_attention_weight_min": -27.14034652709961,
      "activations/layer17_attention_weight_max": 32.042686462402344,
      "activations/layer17_attention_weight_min": -26.194910049438477,
      "activations/layer18_attention_weight_max": 29.431011199951172,
      "activations/layer18_attention_weight_min": -24.737794876098633,
      "activations/layer19_attention_weight_max": 34.72654342651367,
      "activations/layer19_attention_weight_min": -29.946386337280273,
      "activations/layer1_attention_weight_max": 17.654794692993164,
      "activations/layer1_attention_weight_min": -16.7850341796875,
      "activations/layer20_attention_weight_max": 28.92416000366211,
      "activations/layer20_attention_weight_min": -24.808298110961914,
      "activations/layer21_attention_weight_max": 29.13998794555664,
      "activations/layer21_attention_weight_min": -23.26763916015625,
      "activations/layer22_attention_weight_max": 43.04387664794922,
      "activations/layer22_attention_weight_min": -28.99970054626465,
      "activations/layer23_attention_weight_max": 35.17982864379883,
      "activations/layer23_attention_weight_min": -24.04385757446289,
      "activations/layer2_attention_weight_max": 32.925933837890625,
      "activations/layer2_attention_weight_min": -31.16134262084961,
      "activations/layer3_attention_weight_max": 96.666259765625,
      "activations/layer3_attention_weight_min": -92.2996826171875,
      "activations/layer4_attention_weight_max": 122.37361907958984,
      "activations/layer4_attention_weight_min": -112.52819061279297,
      "activations/layer5_attention_weight_max": 54.78025436401367,
      "activations/layer5_attention_weight_min": -61.932308197021484,
      "activations/layer6_attention_weight_max": 45.804405212402344,
      "activations/layer6_attention_weight_min": -48.933162689208984,
      "activations/layer7_attention_weight_max": 96.97589111328125,
      "activations/layer7_attention_weight_min": -88.82633972167969,
      "activations/layer8_attention_weight_max": 42.49050521850586,
      "activations/layer8_attention_weight_min": -40.391605377197266,
      "activations/layer9_attention_weight_max": 36.313350677490234,
      "activations/layer9_attention_weight_min": -35.24589157104492,
      "epoch": 10.59,
      "learning_rate": 8.25590909090909e-05,
      "loss": 2.7952,
      "step": 182200
    },
    {
      "activations/layer0_attention_weight_max": 15.37534236907959,
      "activations/layer0_attention_weight_min": -13.718576431274414,
      "activations/layer10_attention_weight_max": 35.707271575927734,
      "activations/layer10_attention_weight_min": -30.479448318481445,
      "activations/layer11_attention_weight_max": 35.14206314086914,
      "activations/layer11_attention_weight_min": -29.35051727294922,
      "activations/layer12_attention_weight_max": 22.719032287597656,
      "activations/layer12_attention_weight_min": -26.524667739868164,
      "activations/layer13_attention_weight_max": 35.76911163330078,
      "activations/layer13_attention_weight_min": -32.44632339477539,
      "activations/layer14_attention_weight_max": 40.967411041259766,
      "activations/layer14_attention_weight_min": -35.460811614990234,
      "activations/layer15_attention_weight_max": 33.29354476928711,
      "activations/layer15_attention_weight_min": -29.080913543701172,
      "activations/layer16_attention_weight_max": 30.673723220825195,
      "activations/layer16_attention_weight_min": -27.161401748657227,
      "activations/layer17_attention_weight_max": 29.23863410949707,
      "activations/layer17_attention_weight_min": -24.387842178344727,
      "activations/layer18_attention_weight_max": 31.873233795166016,
      "activations/layer18_attention_weight_min": -20.297103881835938,
      "activations/layer19_attention_weight_max": 33.57274627685547,
      "activations/layer19_attention_weight_min": -29.017545700073242,
      "activations/layer1_attention_weight_max": 16.841060638427734,
      "activations/layer1_attention_weight_min": -15.617441177368164,
      "activations/layer20_attention_weight_max": 25.650915145874023,
      "activations/layer20_attention_weight_min": -22.672725677490234,
      "activations/layer21_attention_weight_max": 27.36161231994629,
      "activations/layer21_attention_weight_min": -20.5887451171875,
      "activations/layer22_attention_weight_max": 38.3814582824707,
      "activations/layer22_attention_weight_min": -26.8465518951416,
      "activations/layer23_attention_weight_max": 34.698463439941406,
      "activations/layer23_attention_weight_min": -23.52298355102539,
      "activations/layer2_attention_weight_max": 33.04347229003906,
      "activations/layer2_attention_weight_min": -33.81782913208008,
      "activations/layer3_attention_weight_max": 91.5494613647461,
      "activations/layer3_attention_weight_min": -94.42727661132812,
      "activations/layer4_attention_weight_max": 109.04695892333984,
      "activations/layer4_attention_weight_min": -113.4467544555664,
      "activations/layer5_attention_weight_max": 57.76200866699219,
      "activations/layer5_attention_weight_min": -64.08314514160156,
      "activations/layer6_attention_weight_max": 45.24079895019531,
      "activations/layer6_attention_weight_min": -48.20723342895508,
      "activations/layer7_attention_weight_max": 92.91255187988281,
      "activations/layer7_attention_weight_min": -91.62603759765625,
      "activations/layer8_attention_weight_max": 41.29991912841797,
      "activations/layer8_attention_weight_min": -42.301734924316406,
      "activations/layer9_attention_weight_max": 31.25465965270996,
      "activations/layer9_attention_weight_min": -31.12558364868164,
      "epoch": 10.59,
      "learning_rate": 8.25401515151515e-05,
      "loss": 2.7813,
      "step": 182250
    },
    {
      "activations/layer0_attention_weight_max": 16.31787109375,
      "activations/layer0_attention_weight_min": -12.044282913208008,
      "activations/layer10_attention_weight_max": 35.91339111328125,
      "activations/layer10_attention_weight_min": -35.839664459228516,
      "activations/layer11_attention_weight_max": 36.682350158691406,
      "activations/layer11_attention_weight_min": -36.07661056518555,
      "activations/layer12_attention_weight_max": 32.29729080200195,
      "activations/layer12_attention_weight_min": -27.505460739135742,
      "activations/layer13_attention_weight_max": 54.32157897949219,
      "activations/layer13_attention_weight_min": -39.44181823730469,
      "activations/layer14_attention_weight_max": 68.49311828613281,
      "activations/layer14_attention_weight_min": -44.24354934692383,
      "activations/layer15_attention_weight_max": 48.16237258911133,
      "activations/layer15_attention_weight_min": -34.983116149902344,
      "activations/layer16_attention_weight_max": 41.987632751464844,
      "activations/layer16_attention_weight_min": -28.79521942138672,
      "activations/layer17_attention_weight_max": 42.34370422363281,
      "activations/layer17_attention_weight_min": -26.488182067871094,
      "activations/layer18_attention_weight_max": 34.967708587646484,
      "activations/layer18_attention_weight_min": -21.875280380249023,
      "activations/layer19_attention_weight_max": 47.26612854003906,
      "activations/layer19_attention_weight_min": -31.154876708984375,
      "activations/layer1_attention_weight_max": 15.702922821044922,
      "activations/layer1_attention_weight_min": -14.316882133483887,
      "activations/layer20_attention_weight_max": 38.73598861694336,
      "activations/layer20_attention_weight_min": -24.72715950012207,
      "activations/layer21_attention_weight_max": 36.007423400878906,
      "activations/layer21_attention_weight_min": -22.824825286865234,
      "activations/layer22_attention_weight_max": 62.99225616455078,
      "activations/layer22_attention_weight_min": -29.814960479736328,
      "activations/layer23_attention_weight_max": 47.253379821777344,
      "activations/layer23_attention_weight_min": -22.347585678100586,
      "activations/layer2_attention_weight_max": 33.76416015625,
      "activations/layer2_attention_weight_min": -33.62530517578125,
      "activations/layer3_attention_weight_max": 97.6567153930664,
      "activations/layer3_attention_weight_min": -97.69703674316406,
      "activations/layer4_attention_weight_max": 120.11772918701172,
      "activations/layer4_attention_weight_min": -117.94686126708984,
      "activations/layer5_attention_weight_max": 56.44439697265625,
      "activations/layer5_attention_weight_min": -64.0736083984375,
      "activations/layer6_attention_weight_max": 52.18978500366211,
      "activations/layer6_attention_weight_min": -50.57554626464844,
      "activations/layer7_attention_weight_max": 99.75030517578125,
      "activations/layer7_attention_weight_min": -99.98161315917969,
      "activations/layer8_attention_weight_max": 45.19401931762695,
      "activations/layer8_attention_weight_min": -46.38165283203125,
      "activations/layer9_attention_weight_max": 39.29262924194336,
      "activations/layer9_attention_weight_min": -37.38973617553711,
      "epoch": 10.59,
      "learning_rate": 8.252121212121212e-05,
      "loss": 2.7785,
      "step": 182300
    },
    {
      "activations/layer0_attention_weight_max": 16.064620971679688,
      "activations/layer0_attention_weight_min": -12.792201042175293,
      "activations/layer10_attention_weight_max": 31.181955337524414,
      "activations/layer10_attention_weight_min": -31.412771224975586,
      "activations/layer11_attention_weight_max": 30.055503845214844,
      "activations/layer11_attention_weight_min": -30.675323486328125,
      "activations/layer12_attention_weight_max": 22.751066207885742,
      "activations/layer12_attention_weight_min": -25.053302764892578,
      "activations/layer13_attention_weight_max": 38.77490234375,
      "activations/layer13_attention_weight_min": -33.377052307128906,
      "activations/layer14_attention_weight_max": 46.250816345214844,
      "activations/layer14_attention_weight_min": -37.52117156982422,
      "activations/layer15_attention_weight_max": 40.68645095825195,
      "activations/layer15_attention_weight_min": -31.286439895629883,
      "activations/layer16_attention_weight_max": 33.08368682861328,
      "activations/layer16_attention_weight_min": -30.10643196105957,
      "activations/layer17_attention_weight_max": 31.68643569946289,
      "activations/layer17_attention_weight_min": -23.992393493652344,
      "activations/layer18_attention_weight_max": 33.180999755859375,
      "activations/layer18_attention_weight_min": -21.983341217041016,
      "activations/layer19_attention_weight_max": 33.70405960083008,
      "activations/layer19_attention_weight_min": -31.358448028564453,
      "activations/layer1_attention_weight_max": 16.208126068115234,
      "activations/layer1_attention_weight_min": -15.113602638244629,
      "activations/layer20_attention_weight_max": 31.92885398864746,
      "activations/layer20_attention_weight_min": -26.570348739624023,
      "activations/layer21_attention_weight_max": 29.45023536682129,
      "activations/layer21_attention_weight_min": -22.766525268554688,
      "activations/layer22_attention_weight_max": 45.9860954284668,
      "activations/layer22_attention_weight_min": -28.433143615722656,
      "activations/layer23_attention_weight_max": 37.985267639160156,
      "activations/layer23_attention_weight_min": -22.11034393310547,
      "activations/layer2_attention_weight_max": 31.93665885925293,
      "activations/layer2_attention_weight_min": -34.091957092285156,
      "activations/layer3_attention_weight_max": 91.8037109375,
      "activations/layer3_attention_weight_min": -96.02599334716797,
      "activations/layer4_attention_weight_max": 108.78096008300781,
      "activations/layer4_attention_weight_min": -113.62252044677734,
      "activations/layer5_attention_weight_max": 50.644588470458984,
      "activations/layer5_attention_weight_min": -62.81074142456055,
      "activations/layer6_attention_weight_max": 42.76939010620117,
      "activations/layer6_attention_weight_min": -46.65061950683594,
      "activations/layer7_attention_weight_max": 86.21699523925781,
      "activations/layer7_attention_weight_min": -91.13001251220703,
      "activations/layer8_attention_weight_max": 38.895816802978516,
      "activations/layer8_attention_weight_min": -39.14580154418945,
      "activations/layer9_attention_weight_max": 31.295242309570312,
      "activations/layer9_attention_weight_min": -32.62440872192383,
      "epoch": 10.6,
      "learning_rate": 8.250265151515151e-05,
      "loss": 2.7699,
      "step": 182350
    },
    {
      "activations/layer0_attention_weight_max": 16.32295036315918,
      "activations/layer0_attention_weight_min": -12.009757041931152,
      "activations/layer10_attention_weight_max": 33.75632095336914,
      "activations/layer10_attention_weight_min": -33.4147834777832,
      "activations/layer11_attention_weight_max": 39.70365524291992,
      "activations/layer11_attention_weight_min": -33.14282989501953,
      "activations/layer12_attention_weight_max": 28.138582229614258,
      "activations/layer12_attention_weight_min": -26.694700241088867,
      "activations/layer13_attention_weight_max": 46.2730827331543,
      "activations/layer13_attention_weight_min": -33.84326171875,
      "activations/layer14_attention_weight_max": 44.81954574584961,
      "activations/layer14_attention_weight_min": -36.91427993774414,
      "activations/layer15_attention_weight_max": 40.183441162109375,
      "activations/layer15_attention_weight_min": -31.793445587158203,
      "activations/layer16_attention_weight_max": 31.97991943359375,
      "activations/layer16_attention_weight_min": -29.55524253845215,
      "activations/layer17_attention_weight_max": 30.538286209106445,
      "activations/layer17_attention_weight_min": -24.995319366455078,
      "activations/layer18_attention_weight_max": 29.59502601623535,
      "activations/layer18_attention_weight_min": -22.485546112060547,
      "activations/layer19_attention_weight_max": 34.244903564453125,
      "activations/layer19_attention_weight_min": -30.59647560119629,
      "activations/layer1_attention_weight_max": 17.83099937438965,
      "activations/layer1_attention_weight_min": -14.755105972290039,
      "activations/layer20_attention_weight_max": 33.304954528808594,
      "activations/layer20_attention_weight_min": -24.46474266052246,
      "activations/layer21_attention_weight_max": 29.53628158569336,
      "activations/layer21_attention_weight_min": -24.313241958618164,
      "activations/layer22_attention_weight_max": 42.94392776489258,
      "activations/layer22_attention_weight_min": -28.4659366607666,
      "activations/layer23_attention_weight_max": 35.13740158081055,
      "activations/layer23_attention_weight_min": -24.7061824798584,
      "activations/layer2_attention_weight_max": 32.35787582397461,
      "activations/layer2_attention_weight_min": -33.090450286865234,
      "activations/layer3_attention_weight_max": 96.95630645751953,
      "activations/layer3_attention_weight_min": -101.35484313964844,
      "activations/layer4_attention_weight_max": 112.13215637207031,
      "activations/layer4_attention_weight_min": -111.62389373779297,
      "activations/layer5_attention_weight_max": 54.202449798583984,
      "activations/layer5_attention_weight_min": -61.07073211669922,
      "activations/layer6_attention_weight_max": 43.933677673339844,
      "activations/layer6_attention_weight_min": -47.333797454833984,
      "activations/layer7_attention_weight_max": 94.92684936523438,
      "activations/layer7_attention_weight_min": -84.7691879272461,
      "activations/layer8_attention_weight_max": 40.09402084350586,
      "activations/layer8_attention_weight_min": -40.17722702026367,
      "activations/layer9_attention_weight_max": 32.32481002807617,
      "activations/layer9_attention_weight_min": -33.366188049316406,
      "epoch": 10.6,
      "learning_rate": 8.248371212121213e-05,
      "loss": 2.7807,
      "step": 182400
    },
    {
      "activations/layer0_attention_weight_max": 16.206209182739258,
      "activations/layer0_attention_weight_min": -13.033289909362793,
      "activations/layer10_attention_weight_max": 32.635841369628906,
      "activations/layer10_attention_weight_min": -31.031938552856445,
      "activations/layer11_attention_weight_max": 33.142616271972656,
      "activations/layer11_attention_weight_min": -31.372007369995117,
      "activations/layer12_attention_weight_max": 33.50612258911133,
      "activations/layer12_attention_weight_min": -25.190229415893555,
      "activations/layer13_attention_weight_max": 43.313533782958984,
      "activations/layer13_attention_weight_min": -35.94720458984375,
      "activations/layer14_attention_weight_max": 49.329830169677734,
      "activations/layer14_attention_weight_min": -40.71095657348633,
      "activations/layer15_attention_weight_max": 41.474708557128906,
      "activations/layer15_attention_weight_min": -31.50366973876953,
      "activations/layer16_attention_weight_max": 34.81818771362305,
      "activations/layer16_attention_weight_min": -28.115888595581055,
      "activations/layer17_attention_weight_max": 32.8502311706543,
      "activations/layer17_attention_weight_min": -24.564228057861328,
      "activations/layer18_attention_weight_max": 32.904998779296875,
      "activations/layer18_attention_weight_min": -21.546289443969727,
      "activations/layer19_attention_weight_max": 43.6483154296875,
      "activations/layer19_attention_weight_min": -29.00973892211914,
      "activations/layer1_attention_weight_max": 16.374759674072266,
      "activations/layer1_attention_weight_min": -14.98917293548584,
      "activations/layer20_attention_weight_max": 36.29672622680664,
      "activations/layer20_attention_weight_min": -25.3802433013916,
      "activations/layer21_attention_weight_max": 31.628814697265625,
      "activations/layer21_attention_weight_min": -23.2355899810791,
      "activations/layer22_attention_weight_max": 48.09595489501953,
      "activations/layer22_attention_weight_min": -29.412031173706055,
      "activations/layer23_attention_weight_max": 35.641929626464844,
      "activations/layer23_attention_weight_min": -23.816062927246094,
      "activations/layer2_attention_weight_max": 33.72346115112305,
      "activations/layer2_attention_weight_min": -34.324527740478516,
      "activations/layer3_attention_weight_max": 98.50895690917969,
      "activations/layer3_attention_weight_min": -101.78755187988281,
      "activations/layer4_attention_weight_max": 116.88270568847656,
      "activations/layer4_attention_weight_min": -119.46577453613281,
      "activations/layer5_attention_weight_max": 53.291107177734375,
      "activations/layer5_attention_weight_min": -62.29140853881836,
      "activations/layer6_attention_weight_max": 46.31800079345703,
      "activations/layer6_attention_weight_min": -49.48812484741211,
      "activations/layer7_attention_weight_max": 94.98310089111328,
      "activations/layer7_attention_weight_min": -86.78292083740234,
      "activations/layer8_attention_weight_max": 40.539222717285156,
      "activations/layer8_attention_weight_min": -39.863800048828125,
      "activations/layer9_attention_weight_max": 32.612491607666016,
      "activations/layer9_attention_weight_min": -32.29922103881836,
      "epoch": 10.6,
      "learning_rate": 8.246477272727271e-05,
      "loss": 2.7887,
      "step": 182450
    },
    {
      "activations/layer0_attention_weight_max": 15.38726806640625,
      "activations/layer0_attention_weight_min": -12.807015419006348,
      "activations/layer10_attention_weight_max": 32.08264923095703,
      "activations/layer10_attention_weight_min": -30.657608032226562,
      "activations/layer11_attention_weight_max": 32.051151275634766,
      "activations/layer11_attention_weight_min": -30.687992095947266,
      "activations/layer12_attention_weight_max": 23.69021987915039,
      "activations/layer12_attention_weight_min": -25.462879180908203,
      "activations/layer13_attention_weight_max": 38.47645568847656,
      "activations/layer13_attention_weight_min": -37.555267333984375,
      "activations/layer14_attention_weight_max": 45.19597625732422,
      "activations/layer14_attention_weight_min": -37.61945343017578,
      "activations/layer15_attention_weight_max": 35.915077209472656,
      "activations/layer15_attention_weight_min": -31.990564346313477,
      "activations/layer16_attention_weight_max": 30.055713653564453,
      "activations/layer16_attention_weight_min": -27.48680305480957,
      "activations/layer17_attention_weight_max": 34.88502883911133,
      "activations/layer17_attention_weight_min": -24.96645736694336,
      "activations/layer18_attention_weight_max": 33.20507049560547,
      "activations/layer18_attention_weight_min": -22.671348571777344,
      "activations/layer19_attention_weight_max": 34.709190368652344,
      "activations/layer19_attention_weight_min": -28.805036544799805,
      "activations/layer1_attention_weight_max": 16.58258628845215,
      "activations/layer1_attention_weight_min": -14.926677703857422,
      "activations/layer20_attention_weight_max": 33.395042419433594,
      "activations/layer20_attention_weight_min": -24.380250930786133,
      "activations/layer21_attention_weight_max": 32.626129150390625,
      "activations/layer21_attention_weight_min": -21.985061645507812,
      "activations/layer22_attention_weight_max": 43.077659606933594,
      "activations/layer22_attention_weight_min": -29.26079559326172,
      "activations/layer23_attention_weight_max": 34.00482177734375,
      "activations/layer23_attention_weight_min": -23.010604858398438,
      "activations/layer2_attention_weight_max": 33.049949645996094,
      "activations/layer2_attention_weight_min": -34.16517639160156,
      "activations/layer3_attention_weight_max": 93.98410034179688,
      "activations/layer3_attention_weight_min": -94.87667083740234,
      "activations/layer4_attention_weight_max": 115.12223052978516,
      "activations/layer4_attention_weight_min": -110.8991470336914,
      "activations/layer5_attention_weight_max": 52.717498779296875,
      "activations/layer5_attention_weight_min": -61.470054626464844,
      "activations/layer6_attention_weight_max": 44.665870666503906,
      "activations/layer6_attention_weight_min": -47.03976058959961,
      "activations/layer7_attention_weight_max": 88.38358306884766,
      "activations/layer7_attention_weight_min": -86.0880355834961,
      "activations/layer8_attention_weight_max": 40.043575286865234,
      "activations/layer8_attention_weight_min": -39.08672332763672,
      "activations/layer9_attention_weight_max": 32.58484649658203,
      "activations/layer9_attention_weight_min": -31.0424747467041,
      "epoch": 10.6,
      "learning_rate": 8.244583333333333e-05,
      "loss": 2.7956,
      "step": 182500
    },
    {
      "activations/layer0_attention_weight_max": 16.053447723388672,
      "activations/layer0_attention_weight_min": -12.87558364868164,
      "activations/layer10_attention_weight_max": 31.13492202758789,
      "activations/layer10_attention_weight_min": -30.30459976196289,
      "activations/layer11_attention_weight_max": 32.34138870239258,
      "activations/layer11_attention_weight_min": -30.324504852294922,
      "activations/layer12_attention_weight_max": 23.644102096557617,
      "activations/layer12_attention_weight_min": -27.61585807800293,
      "activations/layer13_attention_weight_max": 37.667205810546875,
      "activations/layer13_attention_weight_min": -33.16089630126953,
      "activations/layer14_attention_weight_max": 44.06449890136719,
      "activations/layer14_attention_weight_min": -38.55418395996094,
      "activations/layer15_attention_weight_max": 34.0807991027832,
      "activations/layer15_attention_weight_min": -31.549705505371094,
      "activations/layer16_attention_weight_max": 29.574920654296875,
      "activations/layer16_attention_weight_min": -27.713224411010742,
      "activations/layer17_attention_weight_max": 33.93096160888672,
      "activations/layer17_attention_weight_min": -26.145709991455078,
      "activations/layer18_attention_weight_max": 30.279325485229492,
      "activations/layer18_attention_weight_min": -22.723243713378906,
      "activations/layer19_attention_weight_max": 33.21523666381836,
      "activations/layer19_attention_weight_min": -29.211294174194336,
      "activations/layer1_attention_weight_max": 17.179222106933594,
      "activations/layer1_attention_weight_min": -14.027920722961426,
      "activations/layer20_attention_weight_max": 30.401512145996094,
      "activations/layer20_attention_weight_min": -24.898963928222656,
      "activations/layer21_attention_weight_max": 29.916812896728516,
      "activations/layer21_attention_weight_min": -24.425071716308594,
      "activations/layer22_attention_weight_max": 48.1003303527832,
      "activations/layer22_attention_weight_min": -30.4608154296875,
      "activations/layer23_attention_weight_max": 35.49120330810547,
      "activations/layer23_attention_weight_min": -25.80215835571289,
      "activations/layer2_attention_weight_max": 32.43173599243164,
      "activations/layer2_attention_weight_min": -32.92903137207031,
      "activations/layer3_attention_weight_max": 91.27705383300781,
      "activations/layer3_attention_weight_min": -93.90725708007812,
      "activations/layer4_attention_weight_max": 110.85717010498047,
      "activations/layer4_attention_weight_min": -111.86161041259766,
      "activations/layer5_attention_weight_max": 50.22965621948242,
      "activations/layer5_attention_weight_min": -60.05763244628906,
      "activations/layer6_attention_weight_max": 42.99372863769531,
      "activations/layer6_attention_weight_min": -44.63685989379883,
      "activations/layer7_attention_weight_max": 82.4076919555664,
      "activations/layer7_attention_weight_min": -82.82432556152344,
      "activations/layer8_attention_weight_max": 36.98954772949219,
      "activations/layer8_attention_weight_min": -38.837005615234375,
      "activations/layer9_attention_weight_max": 31.8648681640625,
      "activations/layer9_attention_weight_min": -31.73763084411621,
      "epoch": 10.61,
      "learning_rate": 8.242689393939394e-05,
      "loss": 2.7806,
      "step": 182550
    },
    {
      "activations/layer0_attention_weight_max": 15.462896347045898,
      "activations/layer0_attention_weight_min": -12.264087677001953,
      "activations/layer10_attention_weight_max": 31.023778915405273,
      "activations/layer10_attention_weight_min": -31.90239715576172,
      "activations/layer11_attention_weight_max": 31.941410064697266,
      "activations/layer11_attention_weight_min": -30.065114974975586,
      "activations/layer12_attention_weight_max": 33.903778076171875,
      "activations/layer12_attention_weight_min": -23.523452758789062,
      "activations/layer13_attention_weight_max": 44.68745040893555,
      "activations/layer13_attention_weight_min": -33.279354095458984,
      "activations/layer14_attention_weight_max": 48.7701301574707,
      "activations/layer14_attention_weight_min": -39.326210021972656,
      "activations/layer15_attention_weight_max": 42.479984283447266,
      "activations/layer15_attention_weight_min": -30.583059310913086,
      "activations/layer16_attention_weight_max": 30.153209686279297,
      "activations/layer16_attention_weight_min": -26.789791107177734,
      "activations/layer17_attention_weight_max": 31.30057144165039,
      "activations/layer17_attention_weight_min": -24.90583610534668,
      "activations/layer18_attention_weight_max": 29.293901443481445,
      "activations/layer18_attention_weight_min": -22.85508918762207,
      "activations/layer19_attention_weight_max": 34.5445556640625,
      "activations/layer19_attention_weight_min": -30.37192726135254,
      "activations/layer1_attention_weight_max": 17.01621437072754,
      "activations/layer1_attention_weight_min": -15.956361770629883,
      "activations/layer20_attention_weight_max": 30.131895065307617,
      "activations/layer20_attention_weight_min": -25.826974868774414,
      "activations/layer21_attention_weight_max": 30.633684158325195,
      "activations/layer21_attention_weight_min": -24.0029354095459,
      "activations/layer22_attention_weight_max": 42.905609130859375,
      "activations/layer22_attention_weight_min": -31.147212982177734,
      "activations/layer23_attention_weight_max": 35.345603942871094,
      "activations/layer23_attention_weight_min": -26.563833236694336,
      "activations/layer2_attention_weight_max": 32.14813232421875,
      "activations/layer2_attention_weight_min": -33.84543991088867,
      "activations/layer3_attention_weight_max": 90.64147186279297,
      "activations/layer3_attention_weight_min": -97.64946746826172,
      "activations/layer4_attention_weight_max": 106.95096588134766,
      "activations/layer4_attention_weight_min": -110.91847229003906,
      "activations/layer5_attention_weight_max": 52.16310119628906,
      "activations/layer5_attention_weight_min": -61.03416061401367,
      "activations/layer6_attention_weight_max": 43.944576263427734,
      "activations/layer6_attention_weight_min": -47.25987243652344,
      "activations/layer7_attention_weight_max": 83.03980255126953,
      "activations/layer7_attention_weight_min": -88.4074478149414,
      "activations/layer8_attention_weight_max": 38.094566345214844,
      "activations/layer8_attention_weight_min": -38.658512115478516,
      "activations/layer9_attention_weight_max": 32.39224624633789,
      "activations/layer9_attention_weight_min": -31.87690544128418,
      "epoch": 10.61,
      "learning_rate": 8.240795454545453e-05,
      "loss": 2.7814,
      "step": 182600
    },
    {
      "activations/layer0_attention_weight_max": 16.208234786987305,
      "activations/layer0_attention_weight_min": -13.709128379821777,
      "activations/layer10_attention_weight_max": 32.980384826660156,
      "activations/layer10_attention_weight_min": -32.08609390258789,
      "activations/layer11_attention_weight_max": 34.242759704589844,
      "activations/layer11_attention_weight_min": -31.486743927001953,
      "activations/layer12_attention_weight_max": 24.083755493164062,
      "activations/layer12_attention_weight_min": -24.187467575073242,
      "activations/layer13_attention_weight_max": 38.42688751220703,
      "activations/layer13_attention_weight_min": -32.565250396728516,
      "activations/layer14_attention_weight_max": 47.14095687866211,
      "activations/layer14_attention_weight_min": -37.33625411987305,
      "activations/layer15_attention_weight_max": 37.96392822265625,
      "activations/layer15_attention_weight_min": -33.317317962646484,
      "activations/layer16_attention_weight_max": 33.608951568603516,
      "activations/layer16_attention_weight_min": -30.29010772705078,
      "activations/layer17_attention_weight_max": 33.900917053222656,
      "activations/layer17_attention_weight_min": -26.9706974029541,
      "activations/layer18_attention_weight_max": 33.60908508300781,
      "activations/layer18_attention_weight_min": -23.708948135375977,
      "activations/layer19_attention_weight_max": 36.25212478637695,
      "activations/layer19_attention_weight_min": -31.170652389526367,
      "activations/layer1_attention_weight_max": 16.012479782104492,
      "activations/layer1_attention_weight_min": -14.823631286621094,
      "activations/layer20_attention_weight_max": 33.17020797729492,
      "activations/layer20_attention_weight_min": -25.65316390991211,
      "activations/layer21_attention_weight_max": 31.584503173828125,
      "activations/layer21_attention_weight_min": -24.2829647064209,
      "activations/layer22_attention_weight_max": 44.15292739868164,
      "activations/layer22_attention_weight_min": -30.2585506439209,
      "activations/layer23_attention_weight_max": 33.210777282714844,
      "activations/layer23_attention_weight_min": -25.91400146484375,
      "activations/layer2_attention_weight_max": 33.71612548828125,
      "activations/layer2_attention_weight_min": -32.221160888671875,
      "activations/layer3_attention_weight_max": 93.55859375,
      "activations/layer3_attention_weight_min": -92.89013671875,
      "activations/layer4_attention_weight_max": 114.9941635131836,
      "activations/layer4_attention_weight_min": -110.93363189697266,
      "activations/layer5_attention_weight_max": 52.66775131225586,
      "activations/layer5_attention_weight_min": -61.22733688354492,
      "activations/layer6_attention_weight_max": 44.34598159790039,
      "activations/layer6_attention_weight_min": -47.43339538574219,
      "activations/layer7_attention_weight_max": 88.30923461914062,
      "activations/layer7_attention_weight_min": -87.1695556640625,
      "activations/layer8_attention_weight_max": 37.57963943481445,
      "activations/layer8_attention_weight_min": -38.94398880004883,
      "activations/layer9_attention_weight_max": 32.25907516479492,
      "activations/layer9_attention_weight_min": -32.33730697631836,
      "epoch": 10.61,
      "learning_rate": 8.238901515151515e-05,
      "loss": 2.7884,
      "step": 182650
    },
    {
      "activations/layer0_attention_weight_max": 16.29161262512207,
      "activations/layer0_attention_weight_min": -13.272537231445312,
      "activations/layer10_attention_weight_max": 39.8215217590332,
      "activations/layer10_attention_weight_min": -36.80034255981445,
      "activations/layer11_attention_weight_max": 43.29418182373047,
      "activations/layer11_attention_weight_min": -36.32225036621094,
      "activations/layer12_attention_weight_max": 38.4580078125,
      "activations/layer12_attention_weight_min": -26.269920349121094,
      "activations/layer13_attention_weight_max": 52.90434646606445,
      "activations/layer13_attention_weight_min": -36.80720901489258,
      "activations/layer14_attention_weight_max": 55.556236267089844,
      "activations/layer14_attention_weight_min": -43.00554656982422,
      "activations/layer15_attention_weight_max": 46.41242599487305,
      "activations/layer15_attention_weight_min": -35.95456314086914,
      "activations/layer16_attention_weight_max": 36.3544807434082,
      "activations/layer16_attention_weight_min": -29.686870574951172,
      "activations/layer17_attention_weight_max": 34.2457160949707,
      "activations/layer17_attention_weight_min": -27.664329528808594,
      "activations/layer18_attention_weight_max": 30.858131408691406,
      "activations/layer18_attention_weight_min": -22.61579704284668,
      "activations/layer19_attention_weight_max": 35.43622970581055,
      "activations/layer19_attention_weight_min": -30.169504165649414,
      "activations/layer1_attention_weight_max": 16.309547424316406,
      "activations/layer1_attention_weight_min": -14.721633911132812,
      "activations/layer20_attention_weight_max": 32.64815139770508,
      "activations/layer20_attention_weight_min": -27.045381546020508,
      "activations/layer21_attention_weight_max": 29.946462631225586,
      "activations/layer21_attention_weight_min": -25.457412719726562,
      "activations/layer22_attention_weight_max": 44.45088195800781,
      "activations/layer22_attention_weight_min": -29.166202545166016,
      "activations/layer23_attention_weight_max": 36.719078063964844,
      "activations/layer23_attention_weight_min": -24.99224853515625,
      "activations/layer2_attention_weight_max": 35.64781951904297,
      "activations/layer2_attention_weight_min": -35.79705047607422,
      "activations/layer3_attention_weight_max": 101.45986938476562,
      "activations/layer3_attention_weight_min": -100.30306243896484,
      "activations/layer4_attention_weight_max": 123.2115478515625,
      "activations/layer4_attention_weight_min": -119.03619384765625,
      "activations/layer5_attention_weight_max": 57.907569885253906,
      "activations/layer5_attention_weight_min": -63.016212463378906,
      "activations/layer6_attention_weight_max": 48.0794792175293,
      "activations/layer6_attention_weight_min": -49.691646575927734,
      "activations/layer7_attention_weight_max": 98.01493072509766,
      "activations/layer7_attention_weight_min": -93.06392669677734,
      "activations/layer8_attention_weight_max": 48.10593795776367,
      "activations/layer8_attention_weight_min": -45.78065872192383,
      "activations/layer9_attention_weight_max": 41.70079040527344,
      "activations/layer9_attention_weight_min": -38.995521545410156,
      "epoch": 10.62,
      "learning_rate": 8.237007575757575e-05,
      "loss": 2.7944,
      "step": 182700
    },
    {
      "activations/layer0_attention_weight_max": 15.964679718017578,
      "activations/layer0_attention_weight_min": -13.090476036071777,
      "activations/layer10_attention_weight_max": 37.27656173706055,
      "activations/layer10_attention_weight_min": -36.39794158935547,
      "activations/layer11_attention_weight_max": 36.36925506591797,
      "activations/layer11_attention_weight_min": -35.74958038330078,
      "activations/layer12_attention_weight_max": 24.933225631713867,
      "activations/layer12_attention_weight_min": -24.365949630737305,
      "activations/layer13_attention_weight_max": 41.20998001098633,
      "activations/layer13_attention_weight_min": -35.57017517089844,
      "activations/layer14_attention_weight_max": 45.836395263671875,
      "activations/layer14_attention_weight_min": -38.612125396728516,
      "activations/layer15_attention_weight_max": 39.412776947021484,
      "activations/layer15_attention_weight_min": -32.64736557006836,
      "activations/layer16_attention_weight_max": 32.26386260986328,
      "activations/layer16_attention_weight_min": -27.915664672851562,
      "activations/layer17_attention_weight_max": 32.747493743896484,
      "activations/layer17_attention_weight_min": -25.237478256225586,
      "activations/layer18_attention_weight_max": 33.343536376953125,
      "activations/layer18_attention_weight_min": -24.28879737854004,
      "activations/layer19_attention_weight_max": 37.69511413574219,
      "activations/layer19_attention_weight_min": -29.95566177368164,
      "activations/layer1_attention_weight_max": 16.590354919433594,
      "activations/layer1_attention_weight_min": -15.521407127380371,
      "activations/layer20_attention_weight_max": 34.77680206298828,
      "activations/layer20_attention_weight_min": -23.548057556152344,
      "activations/layer21_attention_weight_max": 34.63711929321289,
      "activations/layer21_attention_weight_min": -22.685483932495117,
      "activations/layer22_attention_weight_max": 58.25297164916992,
      "activations/layer22_attention_weight_min": -28.72364616394043,
      "activations/layer23_attention_weight_max": 38.9697380065918,
      "activations/layer23_attention_weight_min": -27.36488914489746,
      "activations/layer2_attention_weight_max": 35.43281173706055,
      "activations/layer2_attention_weight_min": -35.39344787597656,
      "activations/layer3_attention_weight_max": 100.52860260009766,
      "activations/layer3_attention_weight_min": -99.61885833740234,
      "activations/layer4_attention_weight_max": 119.8670654296875,
      "activations/layer4_attention_weight_min": -109.96490478515625,
      "activations/layer5_attention_weight_max": 52.79621887207031,
      "activations/layer5_attention_weight_min": -64.59013366699219,
      "activations/layer6_attention_weight_max": 45.69131851196289,
      "activations/layer6_attention_weight_min": -47.74315643310547,
      "activations/layer7_attention_weight_max": 91.06031799316406,
      "activations/layer7_attention_weight_min": -84.9775619506836,
      "activations/layer8_attention_weight_max": 42.54882049560547,
      "activations/layer8_attention_weight_min": -41.84748077392578,
      "activations/layer9_attention_weight_max": 39.46124267578125,
      "activations/layer9_attention_weight_min": -35.19367599487305,
      "epoch": 10.62,
      "learning_rate": 8.235113636363635e-05,
      "loss": 2.7954,
      "step": 182750
    },
    {
      "activations/layer0_attention_weight_max": 14.628450393676758,
      "activations/layer0_attention_weight_min": -12.29055404663086,
      "activations/layer10_attention_weight_max": 36.555091857910156,
      "activations/layer10_attention_weight_min": -35.383544921875,
      "activations/layer11_attention_weight_max": 36.82672119140625,
      "activations/layer11_attention_weight_min": -34.849884033203125,
      "activations/layer12_attention_weight_max": 39.84950256347656,
      "activations/layer12_attention_weight_min": -25.628982543945312,
      "activations/layer13_attention_weight_max": 45.90587615966797,
      "activations/layer13_attention_weight_min": -35.663780212402344,
      "activations/layer14_attention_weight_max": 50.23320388793945,
      "activations/layer14_attention_weight_min": -40.72779846191406,
      "activations/layer15_attention_weight_max": 41.41725540161133,
      "activations/layer15_attention_weight_min": -32.831298828125,
      "activations/layer16_attention_weight_max": 33.89462661743164,
      "activations/layer16_attention_weight_min": -28.210521697998047,
      "activations/layer17_attention_weight_max": 39.432334899902344,
      "activations/layer17_attention_weight_min": -25.72971534729004,
      "activations/layer18_attention_weight_max": 37.65288543701172,
      "activations/layer18_attention_weight_min": -22.963903427124023,
      "activations/layer19_attention_weight_max": 40.84899139404297,
      "activations/layer19_attention_weight_min": -33.41001892089844,
      "activations/layer1_attention_weight_max": 16.473289489746094,
      "activations/layer1_attention_weight_min": -15.730351448059082,
      "activations/layer20_attention_weight_max": 36.884986877441406,
      "activations/layer20_attention_weight_min": -26.53777313232422,
      "activations/layer21_attention_weight_max": 34.9626579284668,
      "activations/layer21_attention_weight_min": -23.1835880279541,
      "activations/layer22_attention_weight_max": 54.90108871459961,
      "activations/layer22_attention_weight_min": -30.069515228271484,
      "activations/layer23_attention_weight_max": 44.93653869628906,
      "activations/layer23_attention_weight_min": -22.593116760253906,
      "activations/layer2_attention_weight_max": 35.987083435058594,
      "activations/layer2_attention_weight_min": -36.079246520996094,
      "activations/layer3_attention_weight_max": 99.70439147949219,
      "activations/layer3_attention_weight_min": -100.11473083496094,
      "activations/layer4_attention_weight_max": 119.80145263671875,
      "activations/layer4_attention_weight_min": -116.36547088623047,
      "activations/layer5_attention_weight_max": 58.977378845214844,
      "activations/layer5_attention_weight_min": -63.67425537109375,
      "activations/layer6_attention_weight_max": 50.94768524169922,
      "activations/layer6_attention_weight_min": -48.259456634521484,
      "activations/layer7_attention_weight_max": 97.43052673339844,
      "activations/layer7_attention_weight_min": -88.87289428710938,
      "activations/layer8_attention_weight_max": 43.85791778564453,
      "activations/layer8_attention_weight_min": -42.85670852661133,
      "activations/layer9_attention_weight_max": 36.74360275268555,
      "activations/layer9_attention_weight_min": -33.99993896484375,
      "epoch": 10.62,
      "learning_rate": 8.233219696969697e-05,
      "loss": 2.7717,
      "step": 182800
    },
    {
      "activations/layer0_attention_weight_max": 16.485013961791992,
      "activations/layer0_attention_weight_min": -11.908672332763672,
      "activations/layer10_attention_weight_max": 30.343841552734375,
      "activations/layer10_attention_weight_min": -32.064178466796875,
      "activations/layer11_attention_weight_max": 30.66034507751465,
      "activations/layer11_attention_weight_min": -30.50373649597168,
      "activations/layer12_attention_weight_max": 23.529699325561523,
      "activations/layer12_attention_weight_min": -23.583900451660156,
      "activations/layer13_attention_weight_max": 35.61766815185547,
      "activations/layer13_attention_weight_min": -32.706199645996094,
      "activations/layer14_attention_weight_max": 41.36283493041992,
      "activations/layer14_attention_weight_min": -35.22408676147461,
      "activations/layer15_attention_weight_max": 33.18482971191406,
      "activations/layer15_attention_weight_min": -30.906145095825195,
      "activations/layer16_attention_weight_max": 31.65987205505371,
      "activations/layer16_attention_weight_min": -28.245038986206055,
      "activations/layer17_attention_weight_max": 32.07572937011719,
      "activations/layer17_attention_weight_min": -25.59325408935547,
      "activations/layer18_attention_weight_max": 33.98140335083008,
      "activations/layer18_attention_weight_min": -23.40013313293457,
      "activations/layer19_attention_weight_max": 36.14866638183594,
      "activations/layer19_attention_weight_min": -34.085052490234375,
      "activations/layer1_attention_weight_max": 17.00044822692871,
      "activations/layer1_attention_weight_min": -14.894729614257812,
      "activations/layer20_attention_weight_max": 30.68587303161621,
      "activations/layer20_attention_weight_min": -25.76235580444336,
      "activations/layer21_attention_weight_max": 32.42091369628906,
      "activations/layer21_attention_weight_min": -24.913131713867188,
      "activations/layer22_attention_weight_max": 45.7462158203125,
      "activations/layer22_attention_weight_min": -30.189815521240234,
      "activations/layer23_attention_weight_max": 38.545223236083984,
      "activations/layer23_attention_weight_min": -24.897441864013672,
      "activations/layer2_attention_weight_max": 32.658203125,
      "activations/layer2_attention_weight_min": -32.794010162353516,
      "activations/layer3_attention_weight_max": 91.52898406982422,
      "activations/layer3_attention_weight_min": -93.26551818847656,
      "activations/layer4_attention_weight_max": 109.85090637207031,
      "activations/layer4_attention_weight_min": -112.34415435791016,
      "activations/layer5_attention_weight_max": 56.16617965698242,
      "activations/layer5_attention_weight_min": -62.07441329956055,
      "activations/layer6_attention_weight_max": 44.1754264831543,
      "activations/layer6_attention_weight_min": -46.628414154052734,
      "activations/layer7_attention_weight_max": 84.83141326904297,
      "activations/layer7_attention_weight_min": -85.67729187011719,
      "activations/layer8_attention_weight_max": 40.61046600341797,
      "activations/layer8_attention_weight_min": -39.94130325317383,
      "activations/layer9_attention_weight_max": 30.495502471923828,
      "activations/layer9_attention_weight_min": -32.210792541503906,
      "epoch": 10.62,
      "learning_rate": 8.231325757575757e-05,
      "loss": 2.775,
      "step": 182850
    },
    {
      "activations/layer0_attention_weight_max": 15.250030517578125,
      "activations/layer0_attention_weight_min": -13.453290939331055,
      "activations/layer10_attention_weight_max": 36.351802825927734,
      "activations/layer10_attention_weight_min": -33.68894577026367,
      "activations/layer11_attention_weight_max": 34.862579345703125,
      "activations/layer11_attention_weight_min": -31.11604881286621,
      "activations/layer12_attention_weight_max": 27.53265953063965,
      "activations/layer12_attention_weight_min": -26.730426788330078,
      "activations/layer13_attention_weight_max": 43.01679992675781,
      "activations/layer13_attention_weight_min": -33.660484313964844,
      "activations/layer14_attention_weight_max": 59.490840911865234,
      "activations/layer14_attention_weight_min": -42.69112014770508,
      "activations/layer15_attention_weight_max": 44.32209777832031,
      "activations/layer15_attention_weight_min": -36.07484436035156,
      "activations/layer16_attention_weight_max": 38.18025588989258,
      "activations/layer16_attention_weight_min": -27.353097915649414,
      "activations/layer17_attention_weight_max": 39.91605758666992,
      "activations/layer17_attention_weight_min": -30.15178108215332,
      "activations/layer18_attention_weight_max": 39.19073486328125,
      "activations/layer18_attention_weight_min": -28.085163116455078,
      "activations/layer19_attention_weight_max": 45.156917572021484,
      "activations/layer19_attention_weight_min": -34.32012176513672,
      "activations/layer1_attention_weight_max": 16.42026138305664,
      "activations/layer1_attention_weight_min": -15.85934829711914,
      "activations/layer20_attention_weight_max": 37.16862106323242,
      "activations/layer20_attention_weight_min": -26.473894119262695,
      "activations/layer21_attention_weight_max": 33.958900451660156,
      "activations/layer21_attention_weight_min": -26.173845291137695,
      "activations/layer22_attention_weight_max": 53.42612838745117,
      "activations/layer22_attention_weight_min": -37.057796478271484,
      "activations/layer23_attention_weight_max": 43.763694763183594,
      "activations/layer23_attention_weight_min": -26.687206268310547,
      "activations/layer2_attention_weight_max": 32.74211120605469,
      "activations/layer2_attention_weight_min": -32.874786376953125,
      "activations/layer3_attention_weight_max": 95.02875518798828,
      "activations/layer3_attention_weight_min": -96.45372772216797,
      "activations/layer4_attention_weight_max": 110.08711242675781,
      "activations/layer4_attention_weight_min": -114.38317108154297,
      "activations/layer5_attention_weight_max": 53.260719299316406,
      "activations/layer5_attention_weight_min": -61.94390869140625,
      "activations/layer6_attention_weight_max": 45.51412582397461,
      "activations/layer6_attention_weight_min": -47.38620376586914,
      "activations/layer7_attention_weight_max": 88.62419128417969,
      "activations/layer7_attention_weight_min": -84.61455535888672,
      "activations/layer8_attention_weight_max": 39.90812301635742,
      "activations/layer8_attention_weight_min": -42.18291091918945,
      "activations/layer9_attention_weight_max": 35.13475799560547,
      "activations/layer9_attention_weight_min": -33.22419357299805,
      "epoch": 10.63,
      "learning_rate": 8.229431818181818e-05,
      "loss": 2.7814,
      "step": 182900
    },
    {
      "activations/layer0_attention_weight_max": 16.398849487304688,
      "activations/layer0_attention_weight_min": -12.706892013549805,
      "activations/layer10_attention_weight_max": 32.3039436340332,
      "activations/layer10_attention_weight_min": -31.975704193115234,
      "activations/layer11_attention_weight_max": 31.443466186523438,
      "activations/layer11_attention_weight_min": -32.114845275878906,
      "activations/layer12_attention_weight_max": 23.53931427001953,
      "activations/layer12_attention_weight_min": -23.44696044921875,
      "activations/layer13_attention_weight_max": 38.84102249145508,
      "activations/layer13_attention_weight_min": -33.525081634521484,
      "activations/layer14_attention_weight_max": 45.613677978515625,
      "activations/layer14_attention_weight_min": -37.52106475830078,
      "activations/layer15_attention_weight_max": 34.69502639770508,
      "activations/layer15_attention_weight_min": -32.43710708618164,
      "activations/layer16_attention_weight_max": 32.18526077270508,
      "activations/layer16_attention_weight_min": -27.125553131103516,
      "activations/layer17_attention_weight_max": 33.24956512451172,
      "activations/layer17_attention_weight_min": -25.957420349121094,
      "activations/layer18_attention_weight_max": 33.47892379760742,
      "activations/layer18_attention_weight_min": -23.20573616027832,
      "activations/layer19_attention_weight_max": 37.14076232910156,
      "activations/layer19_attention_weight_min": -29.841989517211914,
      "activations/layer1_attention_weight_max": 17.35087013244629,
      "activations/layer1_attention_weight_min": -14.75078010559082,
      "activations/layer20_attention_weight_max": 32.566654205322266,
      "activations/layer20_attention_weight_min": -25.123741149902344,
      "activations/layer21_attention_weight_max": 30.833160400390625,
      "activations/layer21_attention_weight_min": -23.61905288696289,
      "activations/layer22_attention_weight_max": 54.41790771484375,
      "activations/layer22_attention_weight_min": -30.411205291748047,
      "activations/layer23_attention_weight_max": 40.00887680053711,
      "activations/layer23_attention_weight_min": -22.660499572753906,
      "activations/layer2_attention_weight_max": 32.7275390625,
      "activations/layer2_attention_weight_min": -33.59660339355469,
      "activations/layer3_attention_weight_max": 94.16126251220703,
      "activations/layer3_attention_weight_min": -92.44386291503906,
      "activations/layer4_attention_weight_max": 117.76893615722656,
      "activations/layer4_attention_weight_min": -113.0568618774414,
      "activations/layer5_attention_weight_max": 54.153011322021484,
      "activations/layer5_attention_weight_min": -61.09245300292969,
      "activations/layer6_attention_weight_max": 46.332576751708984,
      "activations/layer6_attention_weight_min": -47.96693801879883,
      "activations/layer7_attention_weight_max": 96.31573486328125,
      "activations/layer7_attention_weight_min": -87.1229019165039,
      "activations/layer8_attention_weight_max": 40.53759002685547,
      "activations/layer8_attention_weight_min": -42.01301956176758,
      "activations/layer9_attention_weight_max": 31.523807525634766,
      "activations/layer9_attention_weight_min": -32.6679573059082,
      "epoch": 10.63,
      "learning_rate": 8.227537878787878e-05,
      "loss": 2.8026,
      "step": 182950
    },
    {
      "activations/layer0_attention_weight_max": 15.940587997436523,
      "activations/layer0_attention_weight_min": -12.167438507080078,
      "activations/layer10_attention_weight_max": 35.282386779785156,
      "activations/layer10_attention_weight_min": -32.45357131958008,
      "activations/layer11_attention_weight_max": 33.38433837890625,
      "activations/layer11_attention_weight_min": -33.70813751220703,
      "activations/layer12_attention_weight_max": 23.289648056030273,
      "activations/layer12_attention_weight_min": -23.706457138061523,
      "activations/layer13_attention_weight_max": 39.6950798034668,
      "activations/layer13_attention_weight_min": -32.33710861206055,
      "activations/layer14_attention_weight_max": 51.37747573852539,
      "activations/layer14_attention_weight_min": -39.70195388793945,
      "activations/layer15_attention_weight_max": 36.750797271728516,
      "activations/layer15_attention_weight_min": -33.21977996826172,
      "activations/layer16_attention_weight_max": 32.53007125854492,
      "activations/layer16_attention_weight_min": -28.610593795776367,
      "activations/layer17_attention_weight_max": 33.8914909362793,
      "activations/layer17_attention_weight_min": -25.954282760620117,
      "activations/layer18_attention_weight_max": 28.106552124023438,
      "activations/layer18_attention_weight_min": -23.07171058654785,
      "activations/layer19_attention_weight_max": 37.0321044921875,
      "activations/layer19_attention_weight_min": -30.44548988342285,
      "activations/layer1_attention_weight_max": 16.743824005126953,
      "activations/layer1_attention_weight_min": -16.405786514282227,
      "activations/layer20_attention_weight_max": 32.48109817504883,
      "activations/layer20_attention_weight_min": -27.114538192749023,
      "activations/layer21_attention_weight_max": 28.803579330444336,
      "activations/layer21_attention_weight_min": -25.966209411621094,
      "activations/layer22_attention_weight_max": 44.79661560058594,
      "activations/layer22_attention_weight_min": -30.578020095825195,
      "activations/layer23_attention_weight_max": 35.66977310180664,
      "activations/layer23_attention_weight_min": -25.717330932617188,
      "activations/layer2_attention_weight_max": 35.35743713378906,
      "activations/layer2_attention_weight_min": -35.684783935546875,
      "activations/layer3_attention_weight_max": 101.78633880615234,
      "activations/layer3_attention_weight_min": -97.39136505126953,
      "activations/layer4_attention_weight_max": 117.3895034790039,
      "activations/layer4_attention_weight_min": -118.62664794921875,
      "activations/layer5_attention_weight_max": 58.36994552612305,
      "activations/layer5_attention_weight_min": -65.40129089355469,
      "activations/layer6_attention_weight_max": 47.02947235107422,
      "activations/layer6_attention_weight_min": -50.9797248840332,
      "activations/layer7_attention_weight_max": 94.11746215820312,
      "activations/layer7_attention_weight_min": -93.07516479492188,
      "activations/layer8_attention_weight_max": 42.67772674560547,
      "activations/layer8_attention_weight_min": -41.336875915527344,
      "activations/layer9_attention_weight_max": 35.5716667175293,
      "activations/layer9_attention_weight_min": -34.27605056762695,
      "epoch": 10.63,
      "learning_rate": 8.225643939393938e-05,
      "loss": 2.7937,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.56,
      "eval_samples_per_second": 501.634,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.56,
      "eval_openwebtext_samples_per_second": 501.634,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 2.0172,
      "eval_wikitext_samples_per_second": 226.058,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_lambada_loss": 2.5078125,
      "eval_lambada_ppl": 12.278042445054236,
      "eval_lambada_runtime": 9.6215,
      "eval_lambada_samples_per_second": 506.054,
      "step": 183000
    },
    {
      "activations/layer0_attention_weight_max": 15.446834564208984,
      "activations/layer0_attention_weight_min": -12.967631340026855,
      "activations/layer10_attention_weight_max": 35.916969299316406,
      "activations/layer10_attention_weight_min": -33.998023986816406,
      "activations/layer11_attention_weight_max": 34.247520446777344,
      "activations/layer11_attention_weight_min": -31.86567497253418,
      "activations/layer12_attention_weight_max": 24.50812339782715,
      "activations/layer12_attention_weight_min": -24.0352725982666,
      "activations/layer13_attention_weight_max": 39.09651565551758,
      "activations/layer13_attention_weight_min": -35.318511962890625,
      "activations/layer14_attention_weight_max": 44.11445236206055,
      "activations/layer14_attention_weight_min": -37.93522644042969,
      "activations/layer15_attention_weight_max": 37.631805419921875,
      "activations/layer15_attention_weight_min": -31.579477310180664,
      "activations/layer16_attention_weight_max": 33.279319763183594,
      "activations/layer16_attention_weight_min": -28.6732177734375,
      "activations/layer17_attention_weight_max": 33.36122512817383,
      "activations/layer17_attention_weight_min": -27.281923294067383,
      "activations/layer18_attention_weight_max": 33.42478561401367,
      "activations/layer18_attention_weight_min": -26.999120712280273,
      "activations/layer19_attention_weight_max": 40.0706901550293,
      "activations/layer19_attention_weight_min": -33.03943634033203,
      "activations/layer1_attention_weight_max": 17.069616317749023,
      "activations/layer1_attention_weight_min": -15.350550651550293,
      "activations/layer20_attention_weight_max": 28.125625610351562,
      "activations/layer20_attention_weight_min": -25.23192596435547,
      "activations/layer21_attention_weight_max": 28.499168395996094,
      "activations/layer21_attention_weight_min": -23.260251998901367,
      "activations/layer22_attention_weight_max": 42.85191345214844,
      "activations/layer22_attention_weight_min": -32.29044723510742,
      "activations/layer23_attention_weight_max": 37.08673095703125,
      "activations/layer23_attention_weight_min": -27.10738182067871,
      "activations/layer2_attention_weight_max": 34.19915008544922,
      "activations/layer2_attention_weight_min": -31.93353843688965,
      "activations/layer3_attention_weight_max": 95.34585571289062,
      "activations/layer3_attention_weight_min": -92.07070922851562,
      "activations/layer4_attention_weight_max": 117.95943450927734,
      "activations/layer4_attention_weight_min": -109.75118255615234,
      "activations/layer5_attention_weight_max": 53.916805267333984,
      "activations/layer5_attention_weight_min": -60.75538635253906,
      "activations/layer6_attention_weight_max": 45.36650466918945,
      "activations/layer6_attention_weight_min": -49.072635650634766,
      "activations/layer7_attention_weight_max": 91.1205062866211,
      "activations/layer7_attention_weight_min": -87.77410125732422,
      "activations/layer8_attention_weight_max": 43.7105712890625,
      "activations/layer8_attention_weight_min": -43.15766143798828,
      "activations/layer9_attention_weight_max": 36.146018981933594,
      "activations/layer9_attention_weight_min": -34.182518005371094,
      "epoch": 10.64,
      "learning_rate": 8.22375e-05,
      "loss": 2.7781,
      "step": 183050
    },
    {
      "activations/layer0_attention_weight_max": 15.692800521850586,
      "activations/layer0_attention_weight_min": -12.845194816589355,
      "activations/layer10_attention_weight_max": 33.880950927734375,
      "activations/layer10_attention_weight_min": -34.60446548461914,
      "activations/layer11_attention_weight_max": 33.84334182739258,
      "activations/layer11_attention_weight_min": -34.208641052246094,
      "activations/layer12_attention_weight_max": 24.214963912963867,
      "activations/layer12_attention_weight_min": -25.918575286865234,
      "activations/layer13_attention_weight_max": 37.903194427490234,
      "activations/layer13_attention_weight_min": -34.650333404541016,
      "activations/layer14_attention_weight_max": 45.88398361206055,
      "activations/layer14_attention_weight_min": -37.23439407348633,
      "activations/layer15_attention_weight_max": 36.491031646728516,
      "activations/layer15_attention_weight_min": -34.11362838745117,
      "activations/layer16_attention_weight_max": 31.25537872314453,
      "activations/layer16_attention_weight_min": -27.54090118408203,
      "activations/layer17_attention_weight_max": 33.12163543701172,
      "activations/layer17_attention_weight_min": -25.415630340576172,
      "activations/layer18_attention_weight_max": 33.03596496582031,
      "activations/layer18_attention_weight_min": -23.351289749145508,
      "activations/layer19_attention_weight_max": 37.625999450683594,
      "activations/layer19_attention_weight_min": -31.320068359375,
      "activations/layer1_attention_weight_max": 17.63567352294922,
      "activations/layer1_attention_weight_min": -13.793861389160156,
      "activations/layer20_attention_weight_max": 32.80202102661133,
      "activations/layer20_attention_weight_min": -24.54347801208496,
      "activations/layer21_attention_weight_max": 32.031776428222656,
      "activations/layer21_attention_weight_min": -23.95745086669922,
      "activations/layer22_attention_weight_max": 50.3875732421875,
      "activations/layer22_attention_weight_min": -29.65596580505371,
      "activations/layer23_attention_weight_max": 38.894676208496094,
      "activations/layer23_attention_weight_min": -25.111339569091797,
      "activations/layer2_attention_weight_max": 33.7432975769043,
      "activations/layer2_attention_weight_min": -34.098388671875,
      "activations/layer3_attention_weight_max": 92.7899398803711,
      "activations/layer3_attention_weight_min": -95.654296875,
      "activations/layer4_attention_weight_max": 117.80252838134766,
      "activations/layer4_attention_weight_min": -119.61183166503906,
      "activations/layer5_attention_weight_max": 55.2735710144043,
      "activations/layer5_attention_weight_min": -66.48265838623047,
      "activations/layer6_attention_weight_max": 47.13694763183594,
      "activations/layer6_attention_weight_min": -51.74795150756836,
      "activations/layer7_attention_weight_max": 92.54493713378906,
      "activations/layer7_attention_weight_min": -96.23100280761719,
      "activations/layer8_attention_weight_max": 40.28557586669922,
      "activations/layer8_attention_weight_min": -41.40251541137695,
      "activations/layer9_attention_weight_max": 35.4747200012207,
      "activations/layer9_attention_weight_min": -35.58866500854492,
      "epoch": 10.64,
      "learning_rate": 8.221856060606059e-05,
      "loss": 2.7828,
      "step": 183100
    },
    {
      "activations/layer0_attention_weight_max": 15.790519714355469,
      "activations/layer0_attention_weight_min": -13.207634925842285,
      "activations/layer10_attention_weight_max": 30.273515701293945,
      "activations/layer10_attention_weight_min": -30.965978622436523,
      "activations/layer11_attention_weight_max": 30.6209716796875,
      "activations/layer11_attention_weight_min": -28.790037155151367,
      "activations/layer12_attention_weight_max": 22.544050216674805,
      "activations/layer12_attention_weight_min": -25.17816162109375,
      "activations/layer13_attention_weight_max": 37.17873764038086,
      "activations/layer13_attention_weight_min": -35.37566375732422,
      "activations/layer14_attention_weight_max": 44.92228698730469,
      "activations/layer14_attention_weight_min": -39.007118225097656,
      "activations/layer15_attention_weight_max": 34.207149505615234,
      "activations/layer15_attention_weight_min": -31.808979034423828,
      "activations/layer16_attention_weight_max": 31.17057991027832,
      "activations/layer16_attention_weight_min": -29.728229522705078,
      "activations/layer17_attention_weight_max": 29.810462951660156,
      "activations/layer17_attention_weight_min": -25.920225143432617,
      "activations/layer18_attention_weight_max": 32.14863967895508,
      "activations/layer18_attention_weight_min": -20.98839569091797,
      "activations/layer19_attention_weight_max": 35.55268478393555,
      "activations/layer19_attention_weight_min": -30.99915885925293,
      "activations/layer1_attention_weight_max": 17.244930267333984,
      "activations/layer1_attention_weight_min": -14.87303638458252,
      "activations/layer20_attention_weight_max": 29.856029510498047,
      "activations/layer20_attention_weight_min": -23.803428649902344,
      "activations/layer21_attention_weight_max": 28.374801635742188,
      "activations/layer21_attention_weight_min": -22.033483505249023,
      "activations/layer22_attention_weight_max": 41.17129898071289,
      "activations/layer22_attention_weight_min": -29.019210815429688,
      "activations/layer23_attention_weight_max": 32.98038101196289,
      "activations/layer23_attention_weight_min": -25.04738998413086,
      "activations/layer2_attention_weight_max": 31.90506362915039,
      "activations/layer2_attention_weight_min": -32.372154235839844,
      "activations/layer3_attention_weight_max": 87.59549713134766,
      "activations/layer3_attention_weight_min": -90.60462188720703,
      "activations/layer4_attention_weight_max": 108.6280517578125,
      "activations/layer4_attention_weight_min": -111.28828430175781,
      "activations/layer5_attention_weight_max": 50.71891403198242,
      "activations/layer5_attention_weight_min": -63.36035919189453,
      "activations/layer6_attention_weight_max": 40.866783142089844,
      "activations/layer6_attention_weight_min": -45.801231384277344,
      "activations/layer7_attention_weight_max": 84.7964859008789,
      "activations/layer7_attention_weight_min": -86.37870788574219,
      "activations/layer8_attention_weight_max": 36.86468505859375,
      "activations/layer8_attention_weight_min": -39.8723030090332,
      "activations/layer9_attention_weight_max": 30.81314468383789,
      "activations/layer9_attention_weight_min": -30.157339096069336,
      "epoch": 10.64,
      "learning_rate": 8.21996212121212e-05,
      "loss": 2.7841,
      "step": 183150
    },
    {
      "activations/layer0_attention_weight_max": 14.709108352661133,
      "activations/layer0_attention_weight_min": -13.03248119354248,
      "activations/layer10_attention_weight_max": 35.412208557128906,
      "activations/layer10_attention_weight_min": -33.01979064941406,
      "activations/layer11_attention_weight_max": 38.54700469970703,
      "activations/layer11_attention_weight_min": -32.91407775878906,
      "activations/layer12_attention_weight_max": 24.522132873535156,
      "activations/layer12_attention_weight_min": -25.281482696533203,
      "activations/layer13_attention_weight_max": 36.75331115722656,
      "activations/layer13_attention_weight_min": -35.745765686035156,
      "activations/layer14_attention_weight_max": 44.395084381103516,
      "activations/layer14_attention_weight_min": -38.657615661621094,
      "activations/layer15_attention_weight_max": 35.61079025268555,
      "activations/layer15_attention_weight_min": -33.11226272583008,
      "activations/layer16_attention_weight_max": 30.41977882385254,
      "activations/layer16_attention_weight_min": -27.812183380126953,
      "activations/layer17_attention_weight_max": 32.21767807006836,
      "activations/layer17_attention_weight_min": -25.70621681213379,
      "activations/layer18_attention_weight_max": 31.245206832885742,
      "activations/layer18_attention_weight_min": -22.99415397644043,
      "activations/layer19_attention_weight_max": 31.760772705078125,
      "activations/layer19_attention_weight_min": -31.209714889526367,
      "activations/layer1_attention_weight_max": 17.29182243347168,
      "activations/layer1_attention_weight_min": -15.861047744750977,
      "activations/layer20_attention_weight_max": 29.170381546020508,
      "activations/layer20_attention_weight_min": -24.098268508911133,
      "activations/layer21_attention_weight_max": 29.993728637695312,
      "activations/layer21_attention_weight_min": -23.58880043029785,
      "activations/layer22_attention_weight_max": 43.324798583984375,
      "activations/layer22_attention_weight_min": -27.159894943237305,
      "activations/layer23_attention_weight_max": 33.713375091552734,
      "activations/layer23_attention_weight_min": -23.536771774291992,
      "activations/layer2_attention_weight_max": 32.89146423339844,
      "activations/layer2_attention_weight_min": -32.041282653808594,
      "activations/layer3_attention_weight_max": 93.54563903808594,
      "activations/layer3_attention_weight_min": -88.75072479248047,
      "activations/layer4_attention_weight_max": 114.1500244140625,
      "activations/layer4_attention_weight_min": -108.3223876953125,
      "activations/layer5_attention_weight_max": 53.542781829833984,
      "activations/layer5_attention_weight_min": -62.72734832763672,
      "activations/layer6_attention_weight_max": 46.56989288330078,
      "activations/layer6_attention_weight_min": -50.17811584472656,
      "activations/layer7_attention_weight_max": 95.04932403564453,
      "activations/layer7_attention_weight_min": -91.2181396484375,
      "activations/layer8_attention_weight_max": 42.00270080566406,
      "activations/layer8_attention_weight_min": -39.766395568847656,
      "activations/layer9_attention_weight_max": 33.71548843383789,
      "activations/layer9_attention_weight_min": -32.87417221069336,
      "epoch": 10.64,
      "learning_rate": 8.218068181818182e-05,
      "loss": 2.7832,
      "step": 183200
    },
    {
      "activations/layer0_attention_weight_max": 15.253188133239746,
      "activations/layer0_attention_weight_min": -13.1282958984375,
      "activations/layer10_attention_weight_max": 40.63526153564453,
      "activations/layer10_attention_weight_min": -37.776092529296875,
      "activations/layer11_attention_weight_max": 44.28485870361328,
      "activations/layer11_attention_weight_min": -38.604209899902344,
      "activations/layer12_attention_weight_max": 28.34543800354004,
      "activations/layer12_attention_weight_min": -27.527381896972656,
      "activations/layer13_attention_weight_max": 45.81254196166992,
      "activations/layer13_attention_weight_min": -35.59584426879883,
      "activations/layer14_attention_weight_max": 55.420555114746094,
      "activations/layer14_attention_weight_min": -44.59702682495117,
      "activations/layer15_attention_weight_max": 44.30255126953125,
      "activations/layer15_attention_weight_min": -37.374366760253906,
      "activations/layer16_attention_weight_max": 38.727516174316406,
      "activations/layer16_attention_weight_min": -30.048595428466797,
      "activations/layer17_attention_weight_max": 32.318824768066406,
      "activations/layer17_attention_weight_min": -29.23572540283203,
      "activations/layer18_attention_weight_max": 31.056682586669922,
      "activations/layer18_attention_weight_min": -28.338966369628906,
      "activations/layer19_attention_weight_max": 40.515953063964844,
      "activations/layer19_attention_weight_min": -33.09945297241211,
      "activations/layer1_attention_weight_max": 16.978206634521484,
      "activations/layer1_attention_weight_min": -15.931093215942383,
      "activations/layer20_attention_weight_max": 33.69593048095703,
      "activations/layer20_attention_weight_min": -24.963926315307617,
      "activations/layer21_attention_weight_max": 31.584346771240234,
      "activations/layer21_attention_weight_min": -24.63842010498047,
      "activations/layer22_attention_weight_max": 50.8520393371582,
      "activations/layer22_attention_weight_min": -30.386363983154297,
      "activations/layer23_attention_weight_max": 35.35280227661133,
      "activations/layer23_attention_weight_min": -25.78554344177246,
      "activations/layer2_attention_weight_max": 34.87175369262695,
      "activations/layer2_attention_weight_min": -36.24797821044922,
      "activations/layer3_attention_weight_max": 93.98811340332031,
      "activations/layer3_attention_weight_min": -91.61448669433594,
      "activations/layer4_attention_weight_max": 111.1164321899414,
      "activations/layer4_attention_weight_min": -111.4913330078125,
      "activations/layer5_attention_weight_max": 52.30387878417969,
      "activations/layer5_attention_weight_min": -65.91826629638672,
      "activations/layer6_attention_weight_max": 45.32096481323242,
      "activations/layer6_attention_weight_min": -47.24258041381836,
      "activations/layer7_attention_weight_max": 102.21302032470703,
      "activations/layer7_attention_weight_min": -93.17543029785156,
      "activations/layer8_attention_weight_max": 47.394474029541016,
      "activations/layer8_attention_weight_min": -43.83930587768555,
      "activations/layer9_attention_weight_max": 40.55211639404297,
      "activations/layer9_attention_weight_min": -37.481204986572266,
      "epoch": 10.65,
      "learning_rate": 8.21617424242424e-05,
      "loss": 2.791,
      "step": 183250
    },
    {
      "activations/layer0_attention_weight_max": 16.052616119384766,
      "activations/layer0_attention_weight_min": -12.966917991638184,
      "activations/layer10_attention_weight_max": 32.85820388793945,
      "activations/layer10_attention_weight_min": -30.914243698120117,
      "activations/layer11_attention_weight_max": 32.10235595703125,
      "activations/layer11_attention_weight_min": -30.778392791748047,
      "activations/layer12_attention_weight_max": 24.740455627441406,
      "activations/layer12_attention_weight_min": -32.00886917114258,
      "activations/layer13_attention_weight_max": 40.52570343017578,
      "activations/layer13_attention_weight_min": -32.90646743774414,
      "activations/layer14_attention_weight_max": 46.13787841796875,
      "activations/layer14_attention_weight_min": -36.11565017700195,
      "activations/layer15_attention_weight_max": 36.75459289550781,
      "activations/layer15_attention_weight_min": -30.42441749572754,
      "activations/layer16_attention_weight_max": 30.655534744262695,
      "activations/layer16_attention_weight_min": -26.771080017089844,
      "activations/layer17_attention_weight_max": 34.8427734375,
      "activations/layer17_attention_weight_min": -23.895267486572266,
      "activations/layer18_attention_weight_max": 31.905004501342773,
      "activations/layer18_attention_weight_min": -22.685815811157227,
      "activations/layer19_attention_weight_max": 33.84431076049805,
      "activations/layer19_attention_weight_min": -32.86161422729492,
      "activations/layer1_attention_weight_max": 17.988548278808594,
      "activations/layer1_attention_weight_min": -17.92182731628418,
      "activations/layer20_attention_weight_max": 30.732036590576172,
      "activations/layer20_attention_weight_min": -25.256467819213867,
      "activations/layer21_attention_weight_max": 31.089101791381836,
      "activations/layer21_attention_weight_min": -25.29332733154297,
      "activations/layer22_attention_weight_max": 45.469818115234375,
      "activations/layer22_attention_weight_min": -29.275171279907227,
      "activations/layer23_attention_weight_max": 38.80301284790039,
      "activations/layer23_attention_weight_min": -23.360271453857422,
      "activations/layer2_attention_weight_max": 32.911014556884766,
      "activations/layer2_attention_weight_min": -33.53150177001953,
      "activations/layer3_attention_weight_max": 96.70301055908203,
      "activations/layer3_attention_weight_min": -95.43862915039062,
      "activations/layer4_attention_weight_max": 113.34584045410156,
      "activations/layer4_attention_weight_min": -112.65919494628906,
      "activations/layer5_attention_weight_max": 56.11695098876953,
      "activations/layer5_attention_weight_min": -69.46900177001953,
      "activations/layer6_attention_weight_max": 45.52424240112305,
      "activations/layer6_attention_weight_min": -48.96113586425781,
      "activations/layer7_attention_weight_max": 86.81291198730469,
      "activations/layer7_attention_weight_min": -92.33624267578125,
      "activations/layer8_attention_weight_max": 39.707889556884766,
      "activations/layer8_attention_weight_min": -41.510475158691406,
      "activations/layer9_attention_weight_max": 33.475860595703125,
      "activations/layer9_attention_weight_min": -33.60680389404297,
      "epoch": 10.65,
      "learning_rate": 8.214280303030302e-05,
      "loss": 2.7965,
      "step": 183300
    },
    {
      "activations/layer0_attention_weight_max": 15.912087440490723,
      "activations/layer0_attention_weight_min": -12.415400505065918,
      "activations/layer10_attention_weight_max": 35.824378967285156,
      "activations/layer10_attention_weight_min": -33.08787155151367,
      "activations/layer11_attention_weight_max": 36.31926727294922,
      "activations/layer11_attention_weight_min": -33.18518829345703,
      "activations/layer12_attention_weight_max": 26.244892120361328,
      "activations/layer12_attention_weight_min": -24.02819061279297,
      "activations/layer13_attention_weight_max": 43.020633697509766,
      "activations/layer13_attention_weight_min": -35.850669860839844,
      "activations/layer14_attention_weight_max": 47.10330581665039,
      "activations/layer14_attention_weight_min": -36.3809928894043,
      "activations/layer15_attention_weight_max": 46.979530334472656,
      "activations/layer15_attention_weight_min": -31.144428253173828,
      "activations/layer16_attention_weight_max": 32.53435516357422,
      "activations/layer16_attention_weight_min": -28.129558563232422,
      "activations/layer17_attention_weight_max": 31.89781379699707,
      "activations/layer17_attention_weight_min": -24.48703956604004,
      "activations/layer18_attention_weight_max": 33.29100799560547,
      "activations/layer18_attention_weight_min": -23.938283920288086,
      "activations/layer19_attention_weight_max": 38.4932975769043,
      "activations/layer19_attention_weight_min": -31.71390724182129,
      "activations/layer1_attention_weight_max": 16.78418731689453,
      "activations/layer1_attention_weight_min": -15.05933952331543,
      "activations/layer20_attention_weight_max": 31.946008682250977,
      "activations/layer20_attention_weight_min": -24.739389419555664,
      "activations/layer21_attention_weight_max": 30.14657974243164,
      "activations/layer21_attention_weight_min": -23.02962303161621,
      "activations/layer22_attention_weight_max": 41.65191650390625,
      "activations/layer22_attention_weight_min": -29.970727920532227,
      "activations/layer23_attention_weight_max": 36.842323303222656,
      "activations/layer23_attention_weight_min": -23.60125732421875,
      "activations/layer2_attention_weight_max": 33.2098274230957,
      "activations/layer2_attention_weight_min": -31.706504821777344,
      "activations/layer3_attention_weight_max": 92.93243408203125,
      "activations/layer3_attention_weight_min": -93.0582275390625,
      "activations/layer4_attention_weight_max": 108.88116455078125,
      "activations/layer4_attention_weight_min": -108.09639739990234,
      "activations/layer5_attention_weight_max": 51.88804244995117,
      "activations/layer5_attention_weight_min": -59.531646728515625,
      "activations/layer6_attention_weight_max": 44.11382293701172,
      "activations/layer6_attention_weight_min": -47.6818733215332,
      "activations/layer7_attention_weight_max": 93.1957778930664,
      "activations/layer7_attention_weight_min": -87.17969512939453,
      "activations/layer8_attention_weight_max": 39.72611618041992,
      "activations/layer8_attention_weight_min": -40.23659133911133,
      "activations/layer9_attention_weight_max": 35.909912109375,
      "activations/layer9_attention_weight_min": -34.23924255371094,
      "epoch": 10.65,
      "learning_rate": 8.212424242424241e-05,
      "loss": 2.7641,
      "step": 183350
    },
    {
      "activations/layer0_attention_weight_max": 16.161163330078125,
      "activations/layer0_attention_weight_min": -13.496992111206055,
      "activations/layer10_attention_weight_max": 34.952728271484375,
      "activations/layer10_attention_weight_min": -33.596397399902344,
      "activations/layer11_attention_weight_max": 37.47068786621094,
      "activations/layer11_attention_weight_min": -32.66287612915039,
      "activations/layer12_attention_weight_max": 31.136981964111328,
      "activations/layer12_attention_weight_min": -26.691055297851562,
      "activations/layer13_attention_weight_max": 48.5918083190918,
      "activations/layer13_attention_weight_min": -34.9701042175293,
      "activations/layer14_attention_weight_max": 54.247528076171875,
      "activations/layer14_attention_weight_min": -37.50766372680664,
      "activations/layer15_attention_weight_max": 44.05108642578125,
      "activations/layer15_attention_weight_min": -31.053861618041992,
      "activations/layer16_attention_weight_max": 33.99866485595703,
      "activations/layer16_attention_weight_min": -26.827308654785156,
      "activations/layer17_attention_weight_max": 33.82379150390625,
      "activations/layer17_attention_weight_min": -23.32905387878418,
      "activations/layer18_attention_weight_max": 33.24446487426758,
      "activations/layer18_attention_weight_min": -23.163442611694336,
      "activations/layer19_attention_weight_max": 36.27001953125,
      "activations/layer19_attention_weight_min": -31.209440231323242,
      "activations/layer1_attention_weight_max": 16.76506233215332,
      "activations/layer1_attention_weight_min": -14.519134521484375,
      "activations/layer20_attention_weight_max": 32.13889694213867,
      "activations/layer20_attention_weight_min": -22.534650802612305,
      "activations/layer21_attention_weight_max": 31.626188278198242,
      "activations/layer21_attention_weight_min": -22.224185943603516,
      "activations/layer22_attention_weight_max": 48.400821685791016,
      "activations/layer22_attention_weight_min": -27.481599807739258,
      "activations/layer23_attention_weight_max": 36.73045349121094,
      "activations/layer23_attention_weight_min": -23.25063133239746,
      "activations/layer2_attention_weight_max": 35.16455841064453,
      "activations/layer2_attention_weight_min": -34.13318634033203,
      "activations/layer3_attention_weight_max": 97.32612609863281,
      "activations/layer3_attention_weight_min": -104.31026458740234,
      "activations/layer4_attention_weight_max": 113.68505859375,
      "activations/layer4_attention_weight_min": -116.87149810791016,
      "activations/layer5_attention_weight_max": 56.122169494628906,
      "activations/layer5_attention_weight_min": -65.26528930664062,
      "activations/layer6_attention_weight_max": 47.45659255981445,
      "activations/layer6_attention_weight_min": -48.7840461730957,
      "activations/layer7_attention_weight_max": 91.12244415283203,
      "activations/layer7_attention_weight_min": -95.48246765136719,
      "activations/layer8_attention_weight_max": 39.73235321044922,
      "activations/layer8_attention_weight_min": -41.40829086303711,
      "activations/layer9_attention_weight_max": 34.942256927490234,
      "activations/layer9_attention_weight_min": -35.225372314453125,
      "epoch": 10.66,
      "learning_rate": 8.21056818181818e-05,
      "loss": 2.7676,
      "step": 183400
    },
    {
      "activations/layer0_attention_weight_max": 16.70001983642578,
      "activations/layer0_attention_weight_min": -12.557467460632324,
      "activations/layer10_attention_weight_max": 31.343093872070312,
      "activations/layer10_attention_weight_min": -31.097776412963867,
      "activations/layer11_attention_weight_max": 30.120197296142578,
      "activations/layer11_attention_weight_min": -29.006925582885742,
      "activations/layer12_attention_weight_max": 22.302549362182617,
      "activations/layer12_attention_weight_min": -26.339460372924805,
      "activations/layer13_attention_weight_max": 34.554439544677734,
      "activations/layer13_attention_weight_min": -32.54386901855469,
      "activations/layer14_attention_weight_max": 41.62222671508789,
      "activations/layer14_attention_weight_min": -36.44917297363281,
      "activations/layer15_attention_weight_max": 33.106239318847656,
      "activations/layer15_attention_weight_min": -29.98886489868164,
      "activations/layer16_attention_weight_max": 31.050676345825195,
      "activations/layer16_attention_weight_min": -27.686548233032227,
      "activations/layer17_attention_weight_max": 30.92608642578125,
      "activations/layer17_attention_weight_min": -24.953962326049805,
      "activations/layer18_attention_weight_max": 33.00499725341797,
      "activations/layer18_attention_weight_min": -24.049264907836914,
      "activations/layer19_attention_weight_max": 34.80511474609375,
      "activations/layer19_attention_weight_min": -32.98453903198242,
      "activations/layer1_attention_weight_max": 16.64938735961914,
      "activations/layer1_attention_weight_min": -15.617729187011719,
      "activations/layer20_attention_weight_max": 31.241647720336914,
      "activations/layer20_attention_weight_min": -25.070798873901367,
      "activations/layer21_attention_weight_max": 31.686086654663086,
      "activations/layer21_attention_weight_min": -23.18691635131836,
      "activations/layer22_attention_weight_max": 43.270503997802734,
      "activations/layer22_attention_weight_min": -28.35399055480957,
      "activations/layer23_attention_weight_max": 35.54404830932617,
      "activations/layer23_attention_weight_min": -24.811294555664062,
      "activations/layer2_attention_weight_max": 31.8829402923584,
      "activations/layer2_attention_weight_min": -31.82599639892578,
      "activations/layer3_attention_weight_max": 90.05487060546875,
      "activations/layer3_attention_weight_min": -96.17428588867188,
      "activations/layer4_attention_weight_max": 113.62413787841797,
      "activations/layer4_attention_weight_min": -113.28350830078125,
      "activations/layer5_attention_weight_max": 54.177024841308594,
      "activations/layer5_attention_weight_min": -65.53544616699219,
      "activations/layer6_attention_weight_max": 45.42668533325195,
      "activations/layer6_attention_weight_min": -49.00694274902344,
      "activations/layer7_attention_weight_max": 90.99879455566406,
      "activations/layer7_attention_weight_min": -87.35787200927734,
      "activations/layer8_attention_weight_max": 42.222259521484375,
      "activations/layer8_attention_weight_min": -38.640052795410156,
      "activations/layer9_attention_weight_max": 30.746152877807617,
      "activations/layer9_attention_weight_min": -32.17474365234375,
      "epoch": 10.66,
      "learning_rate": 8.208674242424242e-05,
      "loss": 2.7819,
      "step": 183450
    },
    {
      "activations/layer0_attention_weight_max": 15.572232246398926,
      "activations/layer0_attention_weight_min": -12.389301300048828,
      "activations/layer10_attention_weight_max": 29.77409553527832,
      "activations/layer10_attention_weight_min": -29.486326217651367,
      "activations/layer11_attention_weight_max": 30.306699752807617,
      "activations/layer11_attention_weight_min": -30.764936447143555,
      "activations/layer12_attention_weight_max": 25.114521026611328,
      "activations/layer12_attention_weight_min": -25.23084259033203,
      "activations/layer13_attention_weight_max": 37.951900482177734,
      "activations/layer13_attention_weight_min": -36.46836471557617,
      "activations/layer14_attention_weight_max": 44.01596450805664,
      "activations/layer14_attention_weight_min": -38.11199188232422,
      "activations/layer15_attention_weight_max": 34.07140350341797,
      "activations/layer15_attention_weight_min": -32.88030242919922,
      "activations/layer16_attention_weight_max": 31.215023040771484,
      "activations/layer16_attention_weight_min": -26.507404327392578,
      "activations/layer17_attention_weight_max": 31.35097312927246,
      "activations/layer17_attention_weight_min": -25.344623565673828,
      "activations/layer18_attention_weight_max": 31.235448837280273,
      "activations/layer18_attention_weight_min": -23.26202964782715,
      "activations/layer19_attention_weight_max": 38.49632263183594,
      "activations/layer19_attention_weight_min": -30.576526641845703,
      "activations/layer1_attention_weight_max": 16.7189884185791,
      "activations/layer1_attention_weight_min": -14.422146797180176,
      "activations/layer20_attention_weight_max": 32.79594802856445,
      "activations/layer20_attention_weight_min": -25.4176082611084,
      "activations/layer21_attention_weight_max": 32.4515266418457,
      "activations/layer21_attention_weight_min": -22.302316665649414,
      "activations/layer22_attention_weight_max": 46.735069274902344,
      "activations/layer22_attention_weight_min": -29.238725662231445,
      "activations/layer23_attention_weight_max": 34.42424774169922,
      "activations/layer23_attention_weight_min": -25.83003044128418,
      "activations/layer2_attention_weight_max": 30.111064910888672,
      "activations/layer2_attention_weight_min": -30.1527099609375,
      "activations/layer3_attention_weight_max": 85.93488311767578,
      "activations/layer3_attention_weight_min": -90.13583374023438,
      "activations/layer4_attention_weight_max": 100.98617553710938,
      "activations/layer4_attention_weight_min": -104.59149169921875,
      "activations/layer5_attention_weight_max": 50.00096893310547,
      "activations/layer5_attention_weight_min": -58.93825912475586,
      "activations/layer6_attention_weight_max": 40.1791877746582,
      "activations/layer6_attention_weight_min": -45.85274887084961,
      "activations/layer7_attention_weight_max": 80.72994232177734,
      "activations/layer7_attention_weight_min": -87.09880065917969,
      "activations/layer8_attention_weight_max": 36.93406295776367,
      "activations/layer8_attention_weight_min": -37.83085250854492,
      "activations/layer9_attention_weight_max": 30.377553939819336,
      "activations/layer9_attention_weight_min": -30.3873233795166,
      "epoch": 10.66,
      "learning_rate": 8.206780303030303e-05,
      "loss": 2.7863,
      "step": 183500
    },
    {
      "activations/layer0_attention_weight_max": 16.705583572387695,
      "activations/layer0_attention_weight_min": -12.940485000610352,
      "activations/layer10_attention_weight_max": 33.20224380493164,
      "activations/layer10_attention_weight_min": -33.51466751098633,
      "activations/layer11_attention_weight_max": 33.835205078125,
      "activations/layer11_attention_weight_min": -32.30256652832031,
      "activations/layer12_attention_weight_max": 25.719562530517578,
      "activations/layer12_attention_weight_min": -24.710752487182617,
      "activations/layer13_attention_weight_max": 38.66033935546875,
      "activations/layer13_attention_weight_min": -33.31185531616211,
      "activations/layer14_attention_weight_max": 42.92649841308594,
      "activations/layer14_attention_weight_min": -39.0544548034668,
      "activations/layer15_attention_weight_max": 35.9922981262207,
      "activations/layer15_attention_weight_min": -32.023746490478516,
      "activations/layer16_attention_weight_max": 33.18415451049805,
      "activations/layer16_attention_weight_min": -28.948530197143555,
      "activations/layer17_attention_weight_max": 32.728816986083984,
      "activations/layer17_attention_weight_min": -26.180389404296875,
      "activations/layer18_attention_weight_max": 31.2789306640625,
      "activations/layer18_attention_weight_min": -25.0496826171875,
      "activations/layer19_attention_weight_max": 35.35795593261719,
      "activations/layer19_attention_weight_min": -29.636564254760742,
      "activations/layer1_attention_weight_max": 15.94235610961914,
      "activations/layer1_attention_weight_min": -15.693707466125488,
      "activations/layer20_attention_weight_max": 31.48650550842285,
      "activations/layer20_attention_weight_min": -23.35287857055664,
      "activations/layer21_attention_weight_max": 29.762346267700195,
      "activations/layer21_attention_weight_min": -22.492835998535156,
      "activations/layer22_attention_weight_max": 45.00513458251953,
      "activations/layer22_attention_weight_min": -32.44516372680664,
      "activations/layer23_attention_weight_max": 34.08321762084961,
      "activations/layer23_attention_weight_min": -22.17020034790039,
      "activations/layer2_attention_weight_max": 30.563983917236328,
      "activations/layer2_attention_weight_min": -31.491348266601562,
      "activations/layer3_attention_weight_max": 92.22138214111328,
      "activations/layer3_attention_weight_min": -91.00384521484375,
      "activations/layer4_attention_weight_max": 110.9286880493164,
      "activations/layer4_attention_weight_min": -119.42647552490234,
      "activations/layer5_attention_weight_max": 52.54552459716797,
      "activations/layer5_attention_weight_min": -64.85081481933594,
      "activations/layer6_attention_weight_max": 43.569435119628906,
      "activations/layer6_attention_weight_min": -50.359169006347656,
      "activations/layer7_attention_weight_max": 87.09668731689453,
      "activations/layer7_attention_weight_min": -89.14346313476562,
      "activations/layer8_attention_weight_max": 39.98875045776367,
      "activations/layer8_attention_weight_min": -40.56338119506836,
      "activations/layer9_attention_weight_max": 35.207733154296875,
      "activations/layer9_attention_weight_min": -34.22291564941406,
      "epoch": 10.67,
      "learning_rate": 8.204886363636362e-05,
      "loss": 2.7912,
      "step": 183550
    },
    {
      "activations/layer0_attention_weight_max": 14.93508243560791,
      "activations/layer0_attention_weight_min": -12.750248908996582,
      "activations/layer10_attention_weight_max": 35.82447052001953,
      "activations/layer10_attention_weight_min": -34.23380661010742,
      "activations/layer11_attention_weight_max": 33.78290557861328,
      "activations/layer11_attention_weight_min": -33.71197509765625,
      "activations/layer12_attention_weight_max": 25.608665466308594,
      "activations/layer12_attention_weight_min": -27.292940139770508,
      "activations/layer13_attention_weight_max": 39.9193229675293,
      "activations/layer13_attention_weight_min": -35.30628204345703,
      "activations/layer14_attention_weight_max": 48.84385299682617,
      "activations/layer14_attention_weight_min": -39.405914306640625,
      "activations/layer15_attention_weight_max": 41.87797546386719,
      "activations/layer15_attention_weight_min": -35.31310272216797,
      "activations/layer16_attention_weight_max": 33.79133987426758,
      "activations/layer16_attention_weight_min": -29.996734619140625,
      "activations/layer17_attention_weight_max": 30.845224380493164,
      "activations/layer17_attention_weight_min": -28.00521469116211,
      "activations/layer18_attention_weight_max": 30.979412078857422,
      "activations/layer18_attention_weight_min": -23.379411697387695,
      "activations/layer19_attention_weight_max": 37.5966911315918,
      "activations/layer19_attention_weight_min": -30.4472599029541,
      "activations/layer1_attention_weight_max": 16.626667022705078,
      "activations/layer1_attention_weight_min": -14.486736297607422,
      "activations/layer20_attention_weight_max": 34.49052429199219,
      "activations/layer20_attention_weight_min": -24.077463150024414,
      "activations/layer21_attention_weight_max": 33.083221435546875,
      "activations/layer21_attention_weight_min": -24.81816291809082,
      "activations/layer22_attention_weight_max": 45.942935943603516,
      "activations/layer22_attention_weight_min": -28.4140625,
      "activations/layer23_attention_weight_max": 39.99388885498047,
      "activations/layer23_attention_weight_min": -25.296180725097656,
      "activations/layer2_attention_weight_max": 33.50083541870117,
      "activations/layer2_attention_weight_min": -33.09632110595703,
      "activations/layer3_attention_weight_max": 97.63089752197266,
      "activations/layer3_attention_weight_min": -96.15850830078125,
      "activations/layer4_attention_weight_max": 115.9446029663086,
      "activations/layer4_attention_weight_min": -119.45417785644531,
      "activations/layer5_attention_weight_max": 54.51586151123047,
      "activations/layer5_attention_weight_min": -64.64187622070312,
      "activations/layer6_attention_weight_max": 44.9726676940918,
      "activations/layer6_attention_weight_min": -49.07426834106445,
      "activations/layer7_attention_weight_max": 94.91397094726562,
      "activations/layer7_attention_weight_min": -95.34402465820312,
      "activations/layer8_attention_weight_max": 42.426856994628906,
      "activations/layer8_attention_weight_min": -44.280059814453125,
      "activations/layer9_attention_weight_max": 35.11732482910156,
      "activations/layer9_attention_weight_min": -36.047157287597656,
      "epoch": 10.67,
      "learning_rate": 8.202992424242424e-05,
      "loss": 2.7931,
      "step": 183600
    },
    {
      "activations/layer0_attention_weight_max": 15.451643943786621,
      "activations/layer0_attention_weight_min": -13.604454040527344,
      "activations/layer10_attention_weight_max": 32.265586853027344,
      "activations/layer10_attention_weight_min": -30.946453094482422,
      "activations/layer11_attention_weight_max": 32.66728973388672,
      "activations/layer11_attention_weight_min": -30.561147689819336,
      "activations/layer12_attention_weight_max": 25.183202743530273,
      "activations/layer12_attention_weight_min": -24.904712677001953,
      "activations/layer13_attention_weight_max": 36.293758392333984,
      "activations/layer13_attention_weight_min": -33.88507080078125,
      "activations/layer14_attention_weight_max": 45.280982971191406,
      "activations/layer14_attention_weight_min": -36.16605758666992,
      "activations/layer15_attention_weight_max": 36.320030212402344,
      "activations/layer15_attention_weight_min": -33.04867935180664,
      "activations/layer16_attention_weight_max": 29.780012130737305,
      "activations/layer16_attention_weight_min": -27.439634323120117,
      "activations/layer17_attention_weight_max": 30.784318923950195,
      "activations/layer17_attention_weight_min": -25.61244010925293,
      "activations/layer18_attention_weight_max": 30.439687728881836,
      "activations/layer18_attention_weight_min": -22.159883499145508,
      "activations/layer19_attention_weight_max": 33.49861526489258,
      "activations/layer19_attention_weight_min": -30.210248947143555,
      "activations/layer1_attention_weight_max": 17.3113956451416,
      "activations/layer1_attention_weight_min": -16.70697784423828,
      "activations/layer20_attention_weight_max": 30.06133460998535,
      "activations/layer20_attention_weight_min": -23.109813690185547,
      "activations/layer21_attention_weight_max": 27.903221130371094,
      "activations/layer21_attention_weight_min": -23.04776954650879,
      "activations/layer22_attention_weight_max": 46.181365966796875,
      "activations/layer22_attention_weight_min": -29.01123809814453,
      "activations/layer23_attention_weight_max": 34.90672302246094,
      "activations/layer23_attention_weight_min": -25.535757064819336,
      "activations/layer2_attention_weight_max": 31.290950775146484,
      "activations/layer2_attention_weight_min": -32.04688262939453,
      "activations/layer3_attention_weight_max": 90.38334655761719,
      "activations/layer3_attention_weight_min": -97.02503204345703,
      "activations/layer4_attention_weight_max": 107.39275360107422,
      "activations/layer4_attention_weight_min": -105.951904296875,
      "activations/layer5_attention_weight_max": 52.03656768798828,
      "activations/layer5_attention_weight_min": -63.7147216796875,
      "activations/layer6_attention_weight_max": 45.93854522705078,
      "activations/layer6_attention_weight_min": -47.90437316894531,
      "activations/layer7_attention_weight_max": 90.61028289794922,
      "activations/layer7_attention_weight_min": -86.24072265625,
      "activations/layer8_attention_weight_max": 41.55268096923828,
      "activations/layer8_attention_weight_min": -41.54271697998047,
      "activations/layer9_attention_weight_max": 32.47789001464844,
      "activations/layer9_attention_weight_min": -32.424190521240234,
      "epoch": 10.67,
      "learning_rate": 8.201098484848484e-05,
      "loss": 2.7938,
      "step": 183650
    },
    {
      "activations/layer0_attention_weight_max": 15.162083625793457,
      "activations/layer0_attention_weight_min": -11.893250465393066,
      "activations/layer10_attention_weight_max": 33.060516357421875,
      "activations/layer10_attention_weight_min": -30.59015464782715,
      "activations/layer11_attention_weight_max": 32.56791687011719,
      "activations/layer11_attention_weight_min": -30.704458236694336,
      "activations/layer12_attention_weight_max": 25.141752243041992,
      "activations/layer12_attention_weight_min": -26.043617248535156,
      "activations/layer13_attention_weight_max": 35.95917510986328,
      "activations/layer13_attention_weight_min": -34.097721099853516,
      "activations/layer14_attention_weight_max": 43.04376983642578,
      "activations/layer14_attention_weight_min": -37.769588470458984,
      "activations/layer15_attention_weight_max": 35.022029876708984,
      "activations/layer15_attention_weight_min": -30.39950180053711,
      "activations/layer16_attention_weight_max": 34.478126525878906,
      "activations/layer16_attention_weight_min": -27.063304901123047,
      "activations/layer17_attention_weight_max": 35.193302154541016,
      "activations/layer17_attention_weight_min": -24.092693328857422,
      "activations/layer18_attention_weight_max": 32.56184005737305,
      "activations/layer18_attention_weight_min": -21.969703674316406,
      "activations/layer19_attention_weight_max": 34.29085922241211,
      "activations/layer19_attention_weight_min": -29.325057983398438,
      "activations/layer1_attention_weight_max": 16.48832130432129,
      "activations/layer1_attention_weight_min": -14.117080688476562,
      "activations/layer20_attention_weight_max": 30.0833797454834,
      "activations/layer20_attention_weight_min": -23.837873458862305,
      "activations/layer21_attention_weight_max": 29.872604370117188,
      "activations/layer21_attention_weight_min": -22.413393020629883,
      "activations/layer22_attention_weight_max": 40.088504791259766,
      "activations/layer22_attention_weight_min": -32.51961898803711,
      "activations/layer23_attention_weight_max": 33.068641662597656,
      "activations/layer23_attention_weight_min": -24.315418243408203,
      "activations/layer2_attention_weight_max": 31.41928482055664,
      "activations/layer2_attention_weight_min": -30.204679489135742,
      "activations/layer3_attention_weight_max": 86.40726470947266,
      "activations/layer3_attention_weight_min": -89.0712890625,
      "activations/layer4_attention_weight_max": 108.6315689086914,
      "activations/layer4_attention_weight_min": -107.97418212890625,
      "activations/layer5_attention_weight_max": 53.63738250732422,
      "activations/layer5_attention_weight_min": -60.45423126220703,
      "activations/layer6_attention_weight_max": 43.04295349121094,
      "activations/layer6_attention_weight_min": -47.11801528930664,
      "activations/layer7_attention_weight_max": 88.3052749633789,
      "activations/layer7_attention_weight_min": -86.79154968261719,
      "activations/layer8_attention_weight_max": 40.24310302734375,
      "activations/layer8_attention_weight_min": -42.183921813964844,
      "activations/layer9_attention_weight_max": 33.69666290283203,
      "activations/layer9_attention_weight_min": -31.957563400268555,
      "epoch": 10.67,
      "learning_rate": 8.199204545454545e-05,
      "loss": 2.7833,
      "step": 183700
    },
    {
      "activations/layer0_attention_weight_max": 15.656466484069824,
      "activations/layer0_attention_weight_min": -12.629807472229004,
      "activations/layer10_attention_weight_max": 31.53921890258789,
      "activations/layer10_attention_weight_min": -32.777347564697266,
      "activations/layer11_attention_weight_max": 31.147300720214844,
      "activations/layer11_attention_weight_min": -30.638336181640625,
      "activations/layer12_attention_weight_max": 22.5838565826416,
      "activations/layer12_attention_weight_min": -25.048629760742188,
      "activations/layer13_attention_weight_max": 34.86849594116211,
      "activations/layer13_attention_weight_min": -32.459190368652344,
      "activations/layer14_attention_weight_max": 43.3337516784668,
      "activations/layer14_attention_weight_min": -36.29951858520508,
      "activations/layer15_attention_weight_max": 34.67500686645508,
      "activations/layer15_attention_weight_min": -32.25543975830078,
      "activations/layer16_attention_weight_max": 27.8831787109375,
      "activations/layer16_attention_weight_min": -26.31808853149414,
      "activations/layer17_attention_weight_max": 32.81960678100586,
      "activations/layer17_attention_weight_min": -25.415857315063477,
      "activations/layer18_attention_weight_max": 29.855472564697266,
      "activations/layer18_attention_weight_min": -22.713924407958984,
      "activations/layer19_attention_weight_max": 33.19309616088867,
      "activations/layer19_attention_weight_min": -28.519058227539062,
      "activations/layer1_attention_weight_max": 17.330554962158203,
      "activations/layer1_attention_weight_min": -15.574298858642578,
      "activations/layer20_attention_weight_max": 33.27544403076172,
      "activations/layer20_attention_weight_min": -22.70438003540039,
      "activations/layer21_attention_weight_max": 29.97930145263672,
      "activations/layer21_attention_weight_min": -22.544633865356445,
      "activations/layer22_attention_weight_max": 46.568885803222656,
      "activations/layer22_attention_weight_min": -28.077058792114258,
      "activations/layer23_attention_weight_max": 31.594276428222656,
      "activations/layer23_attention_weight_min": -22.908645629882812,
      "activations/layer2_attention_weight_max": 32.744720458984375,
      "activations/layer2_attention_weight_min": -33.21683883666992,
      "activations/layer3_attention_weight_max": 92.66963958740234,
      "activations/layer3_attention_weight_min": -95.10326385498047,
      "activations/layer4_attention_weight_max": 110.70538330078125,
      "activations/layer4_attention_weight_min": -109.33489227294922,
      "activations/layer5_attention_weight_max": 51.35400390625,
      "activations/layer5_attention_weight_min": -58.62272644042969,
      "activations/layer6_attention_weight_max": 42.84067153930664,
      "activations/layer6_attention_weight_min": -46.313499450683594,
      "activations/layer7_attention_weight_max": 85.05475616455078,
      "activations/layer7_attention_weight_min": -88.64037322998047,
      "activations/layer8_attention_weight_max": 38.762847900390625,
      "activations/layer8_attention_weight_min": -39.48945617675781,
      "activations/layer9_attention_weight_max": 34.33230209350586,
      "activations/layer9_attention_weight_min": -34.167240142822266,
      "epoch": 10.68,
      "learning_rate": 8.197310606060605e-05,
      "loss": 2.7704,
      "step": 183750
    },
    {
      "activations/layer0_attention_weight_max": 16.067474365234375,
      "activations/layer0_attention_weight_min": -12.429696083068848,
      "activations/layer10_attention_weight_max": 34.378631591796875,
      "activations/layer10_attention_weight_min": -32.43683624267578,
      "activations/layer11_attention_weight_max": 34.13925552368164,
      "activations/layer11_attention_weight_min": -32.029632568359375,
      "activations/layer12_attention_weight_max": 47.98175811767578,
      "activations/layer12_attention_weight_min": -26.167421340942383,
      "activations/layer13_attention_weight_max": 52.27485656738281,
      "activations/layer13_attention_weight_min": -32.41933822631836,
      "activations/layer14_attention_weight_max": 47.35245895385742,
      "activations/layer14_attention_weight_min": -36.09181213378906,
      "activations/layer15_attention_weight_max": 43.8512077331543,
      "activations/layer15_attention_weight_min": -31.156606674194336,
      "activations/layer16_attention_weight_max": 31.047664642333984,
      "activations/layer16_attention_weight_min": -29.079364776611328,
      "activations/layer17_attention_weight_max": 35.39551544189453,
      "activations/layer17_attention_weight_min": -25.272233963012695,
      "activations/layer18_attention_weight_max": 29.741230010986328,
      "activations/layer18_attention_weight_min": -23.494169235229492,
      "activations/layer19_attention_weight_max": 36.413490295410156,
      "activations/layer19_attention_weight_min": -33.43547821044922,
      "activations/layer1_attention_weight_max": 18.450082778930664,
      "activations/layer1_attention_weight_min": -13.393828392028809,
      "activations/layer20_attention_weight_max": 30.629671096801758,
      "activations/layer20_attention_weight_min": -25.247447967529297,
      "activations/layer21_attention_weight_max": 28.93434715270996,
      "activations/layer21_attention_weight_min": -23.28574562072754,
      "activations/layer22_attention_weight_max": 51.96414566040039,
      "activations/layer22_attention_weight_min": -31.395715713500977,
      "activations/layer23_attention_weight_max": 38.45611572265625,
      "activations/layer23_attention_weight_min": -26.42386245727539,
      "activations/layer2_attention_weight_max": 33.120018005371094,
      "activations/layer2_attention_weight_min": -30.922847747802734,
      "activations/layer3_attention_weight_max": 93.45127868652344,
      "activations/layer3_attention_weight_min": -93.40209197998047,
      "activations/layer4_attention_weight_max": 116.58613586425781,
      "activations/layer4_attention_weight_min": -110.24837493896484,
      "activations/layer5_attention_weight_max": 54.20556640625,
      "activations/layer5_attention_weight_min": -62.54029083251953,
      "activations/layer6_attention_weight_max": 44.23408126831055,
      "activations/layer6_attention_weight_min": -46.74604034423828,
      "activations/layer7_attention_weight_max": 89.55146026611328,
      "activations/layer7_attention_weight_min": -85.03060913085938,
      "activations/layer8_attention_weight_max": 38.11323928833008,
      "activations/layer8_attention_weight_min": -38.82400894165039,
      "activations/layer9_attention_weight_max": 32.54425811767578,
      "activations/layer9_attention_weight_min": -31.705291748046875,
      "epoch": 10.68,
      "learning_rate": 8.195416666666666e-05,
      "loss": 2.7936,
      "step": 183800
    },
    {
      "activations/layer0_attention_weight_max": 16.32431983947754,
      "activations/layer0_attention_weight_min": -13.228765487670898,
      "activations/layer10_attention_weight_max": 33.90839385986328,
      "activations/layer10_attention_weight_min": -31.76401138305664,
      "activations/layer11_attention_weight_max": 35.759944915771484,
      "activations/layer11_attention_weight_min": -33.049503326416016,
      "activations/layer12_attention_weight_max": 22.640705108642578,
      "activations/layer12_attention_weight_min": -23.596012115478516,
      "activations/layer13_attention_weight_max": 36.002525329589844,
      "activations/layer13_attention_weight_min": -32.781333923339844,
      "activations/layer14_attention_weight_max": 40.68245315551758,
      "activations/layer14_attention_weight_min": -36.63142776489258,
      "activations/layer15_attention_weight_max": 33.29412841796875,
      "activations/layer15_attention_weight_min": -33.21088409423828,
      "activations/layer16_attention_weight_max": 29.879457473754883,
      "activations/layer16_attention_weight_min": -28.58457374572754,
      "activations/layer17_attention_weight_max": 34.28894805908203,
      "activations/layer17_attention_weight_min": -25.483051300048828,
      "activations/layer18_attention_weight_max": 29.588640213012695,
      "activations/layer18_attention_weight_min": -23.342931747436523,
      "activations/layer19_attention_weight_max": 31.603071212768555,
      "activations/layer19_attention_weight_min": -31.32210350036621,
      "activations/layer1_attention_weight_max": 16.25640106201172,
      "activations/layer1_attention_weight_min": -15.184535026550293,
      "activations/layer20_attention_weight_max": 29.118144989013672,
      "activations/layer20_attention_weight_min": -26.87505531311035,
      "activations/layer21_attention_weight_max": 27.980770111083984,
      "activations/layer21_attention_weight_min": -23.76592254638672,
      "activations/layer22_attention_weight_max": 40.30710983276367,
      "activations/layer22_attention_weight_min": -30.52210235595703,
      "activations/layer23_attention_weight_max": 32.03635025024414,
      "activations/layer23_attention_weight_min": -23.202381134033203,
      "activations/layer2_attention_weight_max": 33.76205062866211,
      "activations/layer2_attention_weight_min": -31.815855026245117,
      "activations/layer3_attention_weight_max": 91.85598754882812,
      "activations/layer3_attention_weight_min": -99.1910171508789,
      "activations/layer4_attention_weight_max": 106.22712707519531,
      "activations/layer4_attention_weight_min": -109.01838684082031,
      "activations/layer5_attention_weight_max": 50.75956344604492,
      "activations/layer5_attention_weight_min": -62.107566833496094,
      "activations/layer6_attention_weight_max": 44.58530807495117,
      "activations/layer6_attention_weight_min": -48.16626739501953,
      "activations/layer7_attention_weight_max": 89.6307144165039,
      "activations/layer7_attention_weight_min": -88.89702606201172,
      "activations/layer8_attention_weight_max": 39.05315017700195,
      "activations/layer8_attention_weight_min": -42.94942092895508,
      "activations/layer9_attention_weight_max": 33.50794219970703,
      "activations/layer9_attention_weight_min": -31.72088623046875,
      "epoch": 10.68,
      "learning_rate": 8.193522727272727e-05,
      "loss": 2.785,
      "step": 183850
    },
    {
      "activations/layer0_attention_weight_max": 15.674446105957031,
      "activations/layer0_attention_weight_min": -11.824336051940918,
      "activations/layer10_attention_weight_max": 30.09585189819336,
      "activations/layer10_attention_weight_min": -32.188377380371094,
      "activations/layer11_attention_weight_max": 32.40668487548828,
      "activations/layer11_attention_weight_min": -30.88197135925293,
      "activations/layer12_attention_weight_max": 21.067916870117188,
      "activations/layer12_attention_weight_min": -28.853675842285156,
      "activations/layer13_attention_weight_max": 34.18982696533203,
      "activations/layer13_attention_weight_min": -31.761693954467773,
      "activations/layer14_attention_weight_max": 41.54593276977539,
      "activations/layer14_attention_weight_min": -34.02449417114258,
      "activations/layer15_attention_weight_max": 33.52113723754883,
      "activations/layer15_attention_weight_min": -29.790128707885742,
      "activations/layer16_attention_weight_max": 28.900699615478516,
      "activations/layer16_attention_weight_min": -27.41053009033203,
      "activations/layer17_attention_weight_max": 29.512821197509766,
      "activations/layer17_attention_weight_min": -25.72984504699707,
      "activations/layer18_attention_weight_max": 31.623844146728516,
      "activations/layer18_attention_weight_min": -22.536376953125,
      "activations/layer19_attention_weight_max": 33.16822814941406,
      "activations/layer19_attention_weight_min": -30.674365997314453,
      "activations/layer1_attention_weight_max": 16.82566261291504,
      "activations/layer1_attention_weight_min": -15.496171951293945,
      "activations/layer20_attention_weight_max": 30.47097396850586,
      "activations/layer20_attention_weight_min": -25.3182373046875,
      "activations/layer21_attention_weight_max": 28.269283294677734,
      "activations/layer21_attention_weight_min": -24.804519653320312,
      "activations/layer22_attention_weight_max": 47.53900146484375,
      "activations/layer22_attention_weight_min": -31.234848022460938,
      "activations/layer23_attention_weight_max": 35.3962516784668,
      "activations/layer23_attention_weight_min": -27.89661979675293,
      "activations/layer2_attention_weight_max": 31.418302536010742,
      "activations/layer2_attention_weight_min": -32.555580139160156,
      "activations/layer3_attention_weight_max": 90.77928924560547,
      "activations/layer3_attention_weight_min": -99.8862075805664,
      "activations/layer4_attention_weight_max": 116.45714569091797,
      "activations/layer4_attention_weight_min": -110.73341369628906,
      "activations/layer5_attention_weight_max": 51.05957794189453,
      "activations/layer5_attention_weight_min": -60.738521575927734,
      "activations/layer6_attention_weight_max": 44.088069915771484,
      "activations/layer6_attention_weight_min": -48.12416458129883,
      "activations/layer7_attention_weight_max": 88.28496551513672,
      "activations/layer7_attention_weight_min": -86.33797454833984,
      "activations/layer8_attention_weight_max": 39.33765411376953,
      "activations/layer8_attention_weight_min": -41.864524841308594,
      "activations/layer9_attention_weight_max": 31.381322860717773,
      "activations/layer9_attention_weight_min": -32.26388931274414,
      "epoch": 10.69,
      "learning_rate": 8.191628787878787e-05,
      "loss": 2.7968,
      "step": 183900
    },
    {
      "activations/layer0_attention_weight_max": 15.602638244628906,
      "activations/layer0_attention_weight_min": -12.577900886535645,
      "activations/layer10_attention_weight_max": 38.92730712890625,
      "activations/layer10_attention_weight_min": -37.975547790527344,
      "activations/layer11_attention_weight_max": 38.378116607666016,
      "activations/layer11_attention_weight_min": -35.404510498046875,
      "activations/layer12_attention_weight_max": 23.513675689697266,
      "activations/layer12_attention_weight_min": -27.945430755615234,
      "activations/layer13_attention_weight_max": 35.302528381347656,
      "activations/layer13_attention_weight_min": -34.18549728393555,
      "activations/layer14_attention_weight_max": 42.23384475708008,
      "activations/layer14_attention_weight_min": -37.25593948364258,
      "activations/layer15_attention_weight_max": 35.12224197387695,
      "activations/layer15_attention_weight_min": -30.04033088684082,
      "activations/layer16_attention_weight_max": 31.193315505981445,
      "activations/layer16_attention_weight_min": -28.04656219482422,
      "activations/layer17_attention_weight_max": 31.10080909729004,
      "activations/layer17_attention_weight_min": -25.2927188873291,
      "activations/layer18_attention_weight_max": 30.680076599121094,
      "activations/layer18_attention_weight_min": -22.288877487182617,
      "activations/layer19_attention_weight_max": 37.274375915527344,
      "activations/layer19_attention_weight_min": -30.072294235229492,
      "activations/layer1_attention_weight_max": 16.945783615112305,
      "activations/layer1_attention_weight_min": -15.806458473205566,
      "activations/layer20_attention_weight_max": 29.70522689819336,
      "activations/layer20_attention_weight_min": -23.64248275756836,
      "activations/layer21_attention_weight_max": 26.52220916748047,
      "activations/layer21_attention_weight_min": -23.412260055541992,
      "activations/layer22_attention_weight_max": 43.413326263427734,
      "activations/layer22_attention_weight_min": -29.1200008392334,
      "activations/layer23_attention_weight_max": 33.905765533447266,
      "activations/layer23_attention_weight_min": -25.090757369995117,
      "activations/layer2_attention_weight_max": 34.89744186401367,
      "activations/layer2_attention_weight_min": -33.97528839111328,
      "activations/layer3_attention_weight_max": 102.65385437011719,
      "activations/layer3_attention_weight_min": -99.832763671875,
      "activations/layer4_attention_weight_max": 126.09249114990234,
      "activations/layer4_attention_weight_min": -118.77730560302734,
      "activations/layer5_attention_weight_max": 59.05385971069336,
      "activations/layer5_attention_weight_min": -63.248008728027344,
      "activations/layer6_attention_weight_max": 51.38447952270508,
      "activations/layer6_attention_weight_min": -49.21489715576172,
      "activations/layer7_attention_weight_max": 110.84208679199219,
      "activations/layer7_attention_weight_min": -98.29287719726562,
      "activations/layer8_attention_weight_max": 49.34944152832031,
      "activations/layer8_attention_weight_min": -47.14894485473633,
      "activations/layer9_attention_weight_max": 38.20870590209961,
      "activations/layer9_attention_weight_min": -37.79523468017578,
      "epoch": 10.69,
      "learning_rate": 8.189734848484847e-05,
      "loss": 2.7812,
      "step": 183950
    },
    {
      "activations/layer0_attention_weight_max": 15.95906925201416,
      "activations/layer0_attention_weight_min": -13.396270751953125,
      "activations/layer10_attention_weight_max": 32.043922424316406,
      "activations/layer10_attention_weight_min": -32.76966094970703,
      "activations/layer11_attention_weight_max": 32.47433090209961,
      "activations/layer11_attention_weight_min": -32.54380798339844,
      "activations/layer12_attention_weight_max": 23.4196720123291,
      "activations/layer12_attention_weight_min": -25.252588272094727,
      "activations/layer13_attention_weight_max": 34.25714874267578,
      "activations/layer13_attention_weight_min": -33.5509147644043,
      "activations/layer14_attention_weight_max": 39.84474182128906,
      "activations/layer14_attention_weight_min": -39.18595886230469,
      "activations/layer15_attention_weight_max": 33.36605453491211,
      "activations/layer15_attention_weight_min": -31.15669822692871,
      "activations/layer16_attention_weight_max": 28.01110076904297,
      "activations/layer16_attention_weight_min": -28.65793228149414,
      "activations/layer17_attention_weight_max": 34.36758804321289,
      "activations/layer17_attention_weight_min": -26.56850814819336,
      "activations/layer18_attention_weight_max": 31.947776794433594,
      "activations/layer18_attention_weight_min": -25.754968643188477,
      "activations/layer19_attention_weight_max": 34.327091217041016,
      "activations/layer19_attention_weight_min": -31.39767837524414,
      "activations/layer1_attention_weight_max": 16.322601318359375,
      "activations/layer1_attention_weight_min": -13.9627103805542,
      "activations/layer20_attention_weight_max": 30.80867576599121,
      "activations/layer20_attention_weight_min": -24.155942916870117,
      "activations/layer21_attention_weight_max": 30.089616775512695,
      "activations/layer21_attention_weight_min": -23.222381591796875,
      "activations/layer22_attention_weight_max": 46.08718490600586,
      "activations/layer22_attention_weight_min": -28.69475555419922,
      "activations/layer23_attention_weight_max": 37.46992874145508,
      "activations/layer23_attention_weight_min": -25.458141326904297,
      "activations/layer2_attention_weight_max": 32.329444885253906,
      "activations/layer2_attention_weight_min": -32.03365707397461,
      "activations/layer3_attention_weight_max": 88.65518188476562,
      "activations/layer3_attention_weight_min": -91.56105041503906,
      "activations/layer4_attention_weight_max": 108.66706848144531,
      "activations/layer4_attention_weight_min": -111.04485321044922,
      "activations/layer5_attention_weight_max": 50.308990478515625,
      "activations/layer5_attention_weight_min": -62.25578308105469,
      "activations/layer6_attention_weight_max": 42.327877044677734,
      "activations/layer6_attention_weight_min": -47.24043655395508,
      "activations/layer7_attention_weight_max": 86.1932601928711,
      "activations/layer7_attention_weight_min": -91.98534393310547,
      "activations/layer8_attention_weight_max": 39.677513122558594,
      "activations/layer8_attention_weight_min": -41.41389465332031,
      "activations/layer9_attention_weight_max": 33.94599914550781,
      "activations/layer9_attention_weight_min": -33.91012191772461,
      "epoch": 10.69,
      "learning_rate": 8.187840909090909e-05,
      "loss": 2.782,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.5643,
      "eval_samples_per_second": 501.386,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.5643,
      "eval_openwebtext_samples_per_second": 501.386,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 2.0027,
      "eval_wikitext_samples_per_second": 227.691,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_lambada_loss": 2.521484375,
      "eval_lambada_ppl": 12.447059060920614,
      "eval_lambada_runtime": 9.6192,
      "eval_lambada_samples_per_second": 506.178,
      "step": 184000
    },
    {
      "activations/layer0_attention_weight_max": 16.110506057739258,
      "activations/layer0_attention_weight_min": -13.1387300491333,
      "activations/layer10_attention_weight_max": 29.834264755249023,
      "activations/layer10_attention_weight_min": -32.67664337158203,
      "activations/layer11_attention_weight_max": 30.621902465820312,
      "activations/layer11_attention_weight_min": -31.24394989013672,
      "activations/layer12_attention_weight_max": 23.243629455566406,
      "activations/layer12_attention_weight_min": -26.07512855529785,
      "activations/layer13_attention_weight_max": 41.84635543823242,
      "activations/layer13_attention_weight_min": -33.79786682128906,
      "activations/layer14_attention_weight_max": 42.86827850341797,
      "activations/layer14_attention_weight_min": -38.252349853515625,
      "activations/layer15_attention_weight_max": 32.61608123779297,
      "activations/layer15_attention_weight_min": -30.121171951293945,
      "activations/layer16_attention_weight_max": 31.350345611572266,
      "activations/layer16_attention_weight_min": -26.940141677856445,
      "activations/layer17_attention_weight_max": 28.043071746826172,
      "activations/layer17_attention_weight_min": -24.996875762939453,
      "activations/layer18_attention_weight_max": 26.363689422607422,
      "activations/layer18_attention_weight_min": -21.772737503051758,
      "activations/layer19_attention_weight_max": 30.39701271057129,
      "activations/layer19_attention_weight_min": -30.100473403930664,
      "activations/layer1_attention_weight_max": 16.905223846435547,
      "activations/layer1_attention_weight_min": -14.367751121520996,
      "activations/layer20_attention_weight_max": 28.762861251831055,
      "activations/layer20_attention_weight_min": -25.914531707763672,
      "activations/layer21_attention_weight_max": 25.88671875,
      "activations/layer21_attention_weight_min": -23.61419677734375,
      "activations/layer22_attention_weight_max": 39.681373596191406,
      "activations/layer22_attention_weight_min": -31.726627349853516,
      "activations/layer23_attention_weight_max": 33.999942779541016,
      "activations/layer23_attention_weight_min": -27.342559814453125,
      "activations/layer2_attention_weight_max": 33.0705451965332,
      "activations/layer2_attention_weight_min": -32.92520523071289,
      "activations/layer3_attention_weight_max": 89.58834838867188,
      "activations/layer3_attention_weight_min": -91.30464935302734,
      "activations/layer4_attention_weight_max": 106.2252197265625,
      "activations/layer4_attention_weight_min": -112.01849365234375,
      "activations/layer5_attention_weight_max": 53.442100524902344,
      "activations/layer5_attention_weight_min": -65.75077819824219,
      "activations/layer6_attention_weight_max": 44.75221252441406,
      "activations/layer6_attention_weight_min": -49.38861846923828,
      "activations/layer7_attention_weight_max": 90.12379455566406,
      "activations/layer7_attention_weight_min": -89.69541931152344,
      "activations/layer8_attention_weight_max": 38.78582000732422,
      "activations/layer8_attention_weight_min": -42.45692825317383,
      "activations/layer9_attention_weight_max": 32.279170989990234,
      "activations/layer9_attention_weight_min": -33.605342864990234,
      "epoch": 10.69,
      "learning_rate": 8.185946969696968e-05,
      "loss": 2.7881,
      "step": 184050
    },
    {
      "activations/layer0_attention_weight_max": 15.136509895324707,
      "activations/layer0_attention_weight_min": -12.426764488220215,
      "activations/layer10_attention_weight_max": 34.17885971069336,
      "activations/layer10_attention_weight_min": -32.27443313598633,
      "activations/layer11_attention_weight_max": 35.30377960205078,
      "activations/layer11_attention_weight_min": -32.6353759765625,
      "activations/layer12_attention_weight_max": 28.144813537597656,
      "activations/layer12_attention_weight_min": -26.716249465942383,
      "activations/layer13_attention_weight_max": 41.685054779052734,
      "activations/layer13_attention_weight_min": -36.112213134765625,
      "activations/layer14_attention_weight_max": 48.73115158081055,
      "activations/layer14_attention_weight_min": -43.0942268371582,
      "activations/layer15_attention_weight_max": 36.43875503540039,
      "activations/layer15_attention_weight_min": -33.65316390991211,
      "activations/layer16_attention_weight_max": 33.68722152709961,
      "activations/layer16_attention_weight_min": -27.76483726501465,
      "activations/layer17_attention_weight_max": 32.07832336425781,
      "activations/layer17_attention_weight_min": -26.840951919555664,
      "activations/layer18_attention_weight_max": 34.73333740234375,
      "activations/layer18_attention_weight_min": -25.54999542236328,
      "activations/layer19_attention_weight_max": 42.212158203125,
      "activations/layer19_attention_weight_min": -31.84462547302246,
      "activations/layer1_attention_weight_max": 16.71989631652832,
      "activations/layer1_attention_weight_min": -15.368515014648438,
      "activations/layer20_attention_weight_max": 31.803110122680664,
      "activations/layer20_attention_weight_min": -26.234895706176758,
      "activations/layer21_attention_weight_max": 31.701290130615234,
      "activations/layer21_attention_weight_min": -25.038896560668945,
      "activations/layer22_attention_weight_max": 49.5123176574707,
      "activations/layer22_attention_weight_min": -35.866943359375,
      "activations/layer23_attention_weight_max": 38.78590393066406,
      "activations/layer23_attention_weight_min": -28.8551082611084,
      "activations/layer2_attention_weight_max": 33.26451873779297,
      "activations/layer2_attention_weight_min": -32.86504364013672,
      "activations/layer3_attention_weight_max": 93.96871185302734,
      "activations/layer3_attention_weight_min": -96.72541046142578,
      "activations/layer4_attention_weight_max": 113.2603530883789,
      "activations/layer4_attention_weight_min": -112.38250732421875,
      "activations/layer5_attention_weight_max": 54.90504455566406,
      "activations/layer5_attention_weight_min": -62.82387924194336,
      "activations/layer6_attention_weight_max": 44.48126983642578,
      "activations/layer6_attention_weight_min": -48.97421646118164,
      "activations/layer7_attention_weight_max": 90.0123519897461,
      "activations/layer7_attention_weight_min": -87.8547592163086,
      "activations/layer8_attention_weight_max": 39.52876663208008,
      "activations/layer8_attention_weight_min": -41.87308883666992,
      "activations/layer9_attention_weight_max": 35.00314712524414,
      "activations/layer9_attention_weight_min": -32.8449592590332,
      "epoch": 10.7,
      "learning_rate": 8.184053030303029e-05,
      "loss": 2.7765,
      "step": 184100
    },
    {
      "activations/layer0_attention_weight_max": 14.712910652160645,
      "activations/layer0_attention_weight_min": -13.457494735717773,
      "activations/layer10_attention_weight_max": 35.52621841430664,
      "activations/layer10_attention_weight_min": -31.309465408325195,
      "activations/layer11_attention_weight_max": 36.02606964111328,
      "activations/layer11_attention_weight_min": -32.57783126831055,
      "activations/layer12_attention_weight_max": 27.202259063720703,
      "activations/layer12_attention_weight_min": -24.016006469726562,
      "activations/layer13_attention_weight_max": 42.34544372558594,
      "activations/layer13_attention_weight_min": -33.256229400634766,
      "activations/layer14_attention_weight_max": 48.85598373413086,
      "activations/layer14_attention_weight_min": -39.954776763916016,
      "activations/layer15_attention_weight_max": 41.38733673095703,
      "activations/layer15_attention_weight_min": -32.0665283203125,
      "activations/layer16_attention_weight_max": 37.630794525146484,
      "activations/layer16_attention_weight_min": -27.988698959350586,
      "activations/layer17_attention_weight_max": 34.2277717590332,
      "activations/layer17_attention_weight_min": -25.732154846191406,
      "activations/layer18_attention_weight_max": 32.020267486572266,
      "activations/layer18_attention_weight_min": -22.100282669067383,
      "activations/layer19_attention_weight_max": 48.05644226074219,
      "activations/layer19_attention_weight_min": -34.22636032104492,
      "activations/layer1_attention_weight_max": 16.68311309814453,
      "activations/layer1_attention_weight_min": -15.442627906799316,
      "activations/layer20_attention_weight_max": 38.89555740356445,
      "activations/layer20_attention_weight_min": -25.5887393951416,
      "activations/layer21_attention_weight_max": 38.12328338623047,
      "activations/layer21_attention_weight_min": -25.161052703857422,
      "activations/layer22_attention_weight_max": 55.3252067565918,
      "activations/layer22_attention_weight_min": -29.005924224853516,
      "activations/layer23_attention_weight_max": 44.3112907409668,
      "activations/layer23_attention_weight_min": -24.190277099609375,
      "activations/layer2_attention_weight_max": 34.23419952392578,
      "activations/layer2_attention_weight_min": -33.079219818115234,
      "activations/layer3_attention_weight_max": 95.3453369140625,
      "activations/layer3_attention_weight_min": -93.33440399169922,
      "activations/layer4_attention_weight_max": 116.79322814941406,
      "activations/layer4_attention_weight_min": -114.89900970458984,
      "activations/layer5_attention_weight_max": 52.38251495361328,
      "activations/layer5_attention_weight_min": -63.289520263671875,
      "activations/layer6_attention_weight_max": 45.770503997802734,
      "activations/layer6_attention_weight_min": -47.03670120239258,
      "activations/layer7_attention_weight_max": 93.14591979980469,
      "activations/layer7_attention_weight_min": -89.1803207397461,
      "activations/layer8_attention_weight_max": 42.6729621887207,
      "activations/layer8_attention_weight_min": -40.96933364868164,
      "activations/layer9_attention_weight_max": 33.99563980102539,
      "activations/layer9_attention_weight_min": -31.959421157836914,
      "epoch": 10.7,
      "learning_rate": 8.18215909090909e-05,
      "loss": 2.7801,
      "step": 184150
    },
    {
      "activations/layer0_attention_weight_max": 15.604162216186523,
      "activations/layer0_attention_weight_min": -11.8123197555542,
      "activations/layer10_attention_weight_max": 31.84295654296875,
      "activations/layer10_attention_weight_min": -33.15569305419922,
      "activations/layer11_attention_weight_max": 32.03599548339844,
      "activations/layer11_attention_weight_min": -32.15547180175781,
      "activations/layer12_attention_weight_max": 24.99530792236328,
      "activations/layer12_attention_weight_min": -25.56578826904297,
      "activations/layer13_attention_weight_max": 40.30977249145508,
      "activations/layer13_attention_weight_min": -35.44898223876953,
      "activations/layer14_attention_weight_max": 47.12583923339844,
      "activations/layer14_attention_weight_min": -39.61634063720703,
      "activations/layer15_attention_weight_max": 37.22573471069336,
      "activations/layer15_attention_weight_min": -34.073448181152344,
      "activations/layer16_attention_weight_max": 31.726573944091797,
      "activations/layer16_attention_weight_min": -27.392107009887695,
      "activations/layer17_attention_weight_max": 31.00592803955078,
      "activations/layer17_attention_weight_min": -25.517908096313477,
      "activations/layer18_attention_weight_max": 30.11098861694336,
      "activations/layer18_attention_weight_min": -22.517681121826172,
      "activations/layer19_attention_weight_max": 33.035133361816406,
      "activations/layer19_attention_weight_min": -30.117496490478516,
      "activations/layer1_attention_weight_max": 17.153987884521484,
      "activations/layer1_attention_weight_min": -15.86880874633789,
      "activations/layer20_attention_weight_max": 30.91823387145996,
      "activations/layer20_attention_weight_min": -23.937971115112305,
      "activations/layer21_attention_weight_max": 27.89076805114746,
      "activations/layer21_attention_weight_min": -23.6170597076416,
      "activations/layer22_attention_weight_max": 48.81874084472656,
      "activations/layer22_attention_weight_min": -29.83492660522461,
      "activations/layer23_attention_weight_max": 36.875396728515625,
      "activations/layer23_attention_weight_min": -26.243698120117188,
      "activations/layer2_attention_weight_max": 33.01121139526367,
      "activations/layer2_attention_weight_min": -34.68732833862305,
      "activations/layer3_attention_weight_max": 93.25862121582031,
      "activations/layer3_attention_weight_min": -94.77259063720703,
      "activations/layer4_attention_weight_max": 113.90140533447266,
      "activations/layer4_attention_weight_min": -115.84089660644531,
      "activations/layer5_attention_weight_max": 51.80461883544922,
      "activations/layer5_attention_weight_min": -62.49259567260742,
      "activations/layer6_attention_weight_max": 45.26323318481445,
      "activations/layer6_attention_weight_min": -54.07001876831055,
      "activations/layer7_attention_weight_max": 88.0528793334961,
      "activations/layer7_attention_weight_min": -99.8082504272461,
      "activations/layer8_attention_weight_max": 38.47343063354492,
      "activations/layer8_attention_weight_min": -44.163639068603516,
      "activations/layer9_attention_weight_max": 34.275962829589844,
      "activations/layer9_attention_weight_min": -35.501365661621094,
      "epoch": 10.7,
      "learning_rate": 8.180265151515151e-05,
      "loss": 2.7891,
      "step": 184200
    },
    {
      "activations/layer0_attention_weight_max": 16.176654815673828,
      "activations/layer0_attention_weight_min": -12.928159713745117,
      "activations/layer10_attention_weight_max": 34.60516357421875,
      "activations/layer10_attention_weight_min": -31.563310623168945,
      "activations/layer11_attention_weight_max": 33.3819465637207,
      "activations/layer11_attention_weight_min": -33.02861404418945,
      "activations/layer12_attention_weight_max": 23.593936920166016,
      "activations/layer12_attention_weight_min": -23.89148712158203,
      "activations/layer13_attention_weight_max": 36.1157341003418,
      "activations/layer13_attention_weight_min": -33.10498046875,
      "activations/layer14_attention_weight_max": 40.61486053466797,
      "activations/layer14_attention_weight_min": -38.11250686645508,
      "activations/layer15_attention_weight_max": 35.97224426269531,
      "activations/layer15_attention_weight_min": -30.684999465942383,
      "activations/layer16_attention_weight_max": 29.778717041015625,
      "activations/layer16_attention_weight_min": -28.69985580444336,
      "activations/layer17_attention_weight_max": 32.589683532714844,
      "activations/layer17_attention_weight_min": -25.519968032836914,
      "activations/layer18_attention_weight_max": 30.138193130493164,
      "activations/layer18_attention_weight_min": -21.839468002319336,
      "activations/layer19_attention_weight_max": 34.84328079223633,
      "activations/layer19_attention_weight_min": -30.386362075805664,
      "activations/layer1_attention_weight_max": 16.565547943115234,
      "activations/layer1_attention_weight_min": -15.260613441467285,
      "activations/layer20_attention_weight_max": 28.831375122070312,
      "activations/layer20_attention_weight_min": -25.211030960083008,
      "activations/layer21_attention_weight_max": 28.650375366210938,
      "activations/layer21_attention_weight_min": -23.258840560913086,
      "activations/layer22_attention_weight_max": 43.737239837646484,
      "activations/layer22_attention_weight_min": -28.471101760864258,
      "activations/layer23_attention_weight_max": 33.7999153137207,
      "activations/layer23_attention_weight_min": -23.466222763061523,
      "activations/layer2_attention_weight_max": 31.989498138427734,
      "activations/layer2_attention_weight_min": -31.559268951416016,
      "activations/layer3_attention_weight_max": 89.44685363769531,
      "activations/layer3_attention_weight_min": -94.285400390625,
      "activations/layer4_attention_weight_max": 108.34954071044922,
      "activations/layer4_attention_weight_min": -112.26875305175781,
      "activations/layer5_attention_weight_max": 53.27771759033203,
      "activations/layer5_attention_weight_min": -60.22806930541992,
      "activations/layer6_attention_weight_max": 45.26607894897461,
      "activations/layer6_attention_weight_min": -47.364559173583984,
      "activations/layer7_attention_weight_max": 88.76399230957031,
      "activations/layer7_attention_weight_min": -91.16718292236328,
      "activations/layer8_attention_weight_max": 40.13918685913086,
      "activations/layer8_attention_weight_min": -40.12107849121094,
      "activations/layer9_attention_weight_max": 33.17164993286133,
      "activations/layer9_attention_weight_min": -32.96051025390625,
      "epoch": 10.71,
      "learning_rate": 8.178371212121211e-05,
      "loss": 2.7934,
      "step": 184250
    },
    {
      "activations/layer0_attention_weight_max": 16.268287658691406,
      "activations/layer0_attention_weight_min": -12.728494644165039,
      "activations/layer10_attention_weight_max": 31.238664627075195,
      "activations/layer10_attention_weight_min": -32.175045013427734,
      "activations/layer11_attention_weight_max": 30.825347900390625,
      "activations/layer11_attention_weight_min": -31.131080627441406,
      "activations/layer12_attention_weight_max": 26.064762115478516,
      "activations/layer12_attention_weight_min": -30.44867706298828,
      "activations/layer13_attention_weight_max": 39.34617233276367,
      "activations/layer13_attention_weight_min": -33.725711822509766,
      "activations/layer14_attention_weight_max": 48.805084228515625,
      "activations/layer14_attention_weight_min": -37.63397979736328,
      "activations/layer15_attention_weight_max": 37.27070617675781,
      "activations/layer15_attention_weight_min": -30.492708206176758,
      "activations/layer16_attention_weight_max": 33.1038932800293,
      "activations/layer16_attention_weight_min": -28.35435676574707,
      "activations/layer17_attention_weight_max": 35.656517028808594,
      "activations/layer17_attention_weight_min": -25.66185188293457,
      "activations/layer18_attention_weight_max": 39.452423095703125,
      "activations/layer18_attention_weight_min": -24.478940963745117,
      "activations/layer19_attention_weight_max": 38.089630126953125,
      "activations/layer19_attention_weight_min": -29.73691749572754,
      "activations/layer1_attention_weight_max": 16.101844787597656,
      "activations/layer1_attention_weight_min": -15.07813835144043,
      "activations/layer20_attention_weight_max": 33.10345458984375,
      "activations/layer20_attention_weight_min": -25.877599716186523,
      "activations/layer21_attention_weight_max": 36.25560760498047,
      "activations/layer21_attention_weight_min": -23.580007553100586,
      "activations/layer22_attention_weight_max": 46.8275032043457,
      "activations/layer22_attention_weight_min": -31.57573699951172,
      "activations/layer23_attention_weight_max": 33.90460205078125,
      "activations/layer23_attention_weight_min": -26.062673568725586,
      "activations/layer2_attention_weight_max": 33.67631149291992,
      "activations/layer2_attention_weight_min": -31.845600128173828,
      "activations/layer3_attention_weight_max": 91.80863952636719,
      "activations/layer3_attention_weight_min": -91.60029602050781,
      "activations/layer4_attention_weight_max": 112.81558990478516,
      "activations/layer4_attention_weight_min": -111.14186096191406,
      "activations/layer5_attention_weight_max": 55.79728698730469,
      "activations/layer5_attention_weight_min": -65.10353088378906,
      "activations/layer6_attention_weight_max": 45.849891662597656,
      "activations/layer6_attention_weight_min": -48.040042877197266,
      "activations/layer7_attention_weight_max": 94.56442260742188,
      "activations/layer7_attention_weight_min": -89.74874877929688,
      "activations/layer8_attention_weight_max": 42.465606689453125,
      "activations/layer8_attention_weight_min": -41.40915298461914,
      "activations/layer9_attention_weight_max": 32.3068962097168,
      "activations/layer9_attention_weight_min": -34.2498664855957,
      "epoch": 10.71,
      "learning_rate": 8.176477272727272e-05,
      "loss": 2.7867,
      "step": 184300
    },
    {
      "activations/layer0_attention_weight_max": 16.06263542175293,
      "activations/layer0_attention_weight_min": -12.546653747558594,
      "activations/layer10_attention_weight_max": 31.57297706604004,
      "activations/layer10_attention_weight_min": -31.95128631591797,
      "activations/layer11_attention_weight_max": 35.3299560546875,
      "activations/layer11_attention_weight_min": -32.49989318847656,
      "activations/layer12_attention_weight_max": 23.43238639831543,
      "activations/layer12_attention_weight_min": -24.61131477355957,
      "activations/layer13_attention_weight_max": 35.89884567260742,
      "activations/layer13_attention_weight_min": -33.836036682128906,
      "activations/layer14_attention_weight_max": 41.46859359741211,
      "activations/layer14_attention_weight_min": -38.14999008178711,
      "activations/layer15_attention_weight_max": 36.02611541748047,
      "activations/layer15_attention_weight_min": -30.025882720947266,
      "activations/layer16_attention_weight_max": 30.217395782470703,
      "activations/layer16_attention_weight_min": -27.080278396606445,
      "activations/layer17_attention_weight_max": 31.350318908691406,
      "activations/layer17_attention_weight_min": -26.717069625854492,
      "activations/layer18_attention_weight_max": 29.582937240600586,
      "activations/layer18_attention_weight_min": -23.028112411499023,
      "activations/layer19_attention_weight_max": 29.77378273010254,
      "activations/layer19_attention_weight_min": -29.71896743774414,
      "activations/layer1_attention_weight_max": 17.577810287475586,
      "activations/layer1_attention_weight_min": -14.915300369262695,
      "activations/layer20_attention_weight_max": 27.15314292907715,
      "activations/layer20_attention_weight_min": -23.092395782470703,
      "activations/layer21_attention_weight_max": 27.354766845703125,
      "activations/layer21_attention_weight_min": -24.661855697631836,
      "activations/layer22_attention_weight_max": 39.59400939941406,
      "activations/layer22_attention_weight_min": -31.694238662719727,
      "activations/layer23_attention_weight_max": 34.343528747558594,
      "activations/layer23_attention_weight_min": -26.069442749023438,
      "activations/layer2_attention_weight_max": 34.0999870300293,
      "activations/layer2_attention_weight_min": -32.59673309326172,
      "activations/layer3_attention_weight_max": 93.92898559570312,
      "activations/layer3_attention_weight_min": -93.86991119384766,
      "activations/layer4_attention_weight_max": 114.00931549072266,
      "activations/layer4_attention_weight_min": -110.04164123535156,
      "activations/layer5_attention_weight_max": 55.9271240234375,
      "activations/layer5_attention_weight_min": -59.71432113647461,
      "activations/layer6_attention_weight_max": 46.418128967285156,
      "activations/layer6_attention_weight_min": -46.36659622192383,
      "activations/layer7_attention_weight_max": 92.12164306640625,
      "activations/layer7_attention_weight_min": -85.90701293945312,
      "activations/layer8_attention_weight_max": 41.09869384765625,
      "activations/layer8_attention_weight_min": -40.27827835083008,
      "activations/layer9_attention_weight_max": 33.3770866394043,
      "activations/layer9_attention_weight_min": -32.914852142333984,
      "epoch": 10.71,
      "learning_rate": 8.174583333333333e-05,
      "loss": 2.7879,
      "step": 184350
    },
    {
      "activations/layer0_attention_weight_max": 15.04050350189209,
      "activations/layer0_attention_weight_min": -12.85024356842041,
      "activations/layer10_attention_weight_max": 33.33544921875,
      "activations/layer10_attention_weight_min": -31.645708084106445,
      "activations/layer11_attention_weight_max": 32.61838150024414,
      "activations/layer11_attention_weight_min": -31.625364303588867,
      "activations/layer12_attention_weight_max": 24.2252140045166,
      "activations/layer12_attention_weight_min": -25.0245418548584,
      "activations/layer13_attention_weight_max": 39.67937469482422,
      "activations/layer13_attention_weight_min": -34.66859817504883,
      "activations/layer14_attention_weight_max": 42.219913482666016,
      "activations/layer14_attention_weight_min": -35.617122650146484,
      "activations/layer15_attention_weight_max": 34.44442367553711,
      "activations/layer15_attention_weight_min": -30.864826202392578,
      "activations/layer16_attention_weight_max": 32.23662185668945,
      "activations/layer16_attention_weight_min": -28.331363677978516,
      "activations/layer17_attention_weight_max": 33.81966018676758,
      "activations/layer17_attention_weight_min": -26.878318786621094,
      "activations/layer18_attention_weight_max": 30.557823181152344,
      "activations/layer18_attention_weight_min": -23.98695182800293,
      "activations/layer19_attention_weight_max": 34.803253173828125,
      "activations/layer19_attention_weight_min": -30.737579345703125,
      "activations/layer1_attention_weight_max": 16.7539119720459,
      "activations/layer1_attention_weight_min": -16.50794792175293,
      "activations/layer20_attention_weight_max": 28.760047912597656,
      "activations/layer20_attention_weight_min": -25.97292137145996,
      "activations/layer21_attention_weight_max": 30.30885887145996,
      "activations/layer21_attention_weight_min": -24.1801815032959,
      "activations/layer22_attention_weight_max": 41.44519805908203,
      "activations/layer22_attention_weight_min": -30.74733543395996,
      "activations/layer23_attention_weight_max": 36.41285705566406,
      "activations/layer23_attention_weight_min": -27.560157775878906,
      "activations/layer2_attention_weight_max": 33.17947769165039,
      "activations/layer2_attention_weight_min": -35.4370002746582,
      "activations/layer3_attention_weight_max": 95.31861877441406,
      "activations/layer3_attention_weight_min": -102.46233367919922,
      "activations/layer4_attention_weight_max": 110.7073974609375,
      "activations/layer4_attention_weight_min": -115.19939422607422,
      "activations/layer5_attention_weight_max": 49.69025421142578,
      "activations/layer5_attention_weight_min": -60.98680877685547,
      "activations/layer6_attention_weight_max": 43.76700210571289,
      "activations/layer6_attention_weight_min": -48.214271545410156,
      "activations/layer7_attention_weight_max": 90.17115020751953,
      "activations/layer7_attention_weight_min": -86.80340576171875,
      "activations/layer8_attention_weight_max": 39.337730407714844,
      "activations/layer8_attention_weight_min": -40.03546142578125,
      "activations/layer9_attention_weight_max": 33.71012496948242,
      "activations/layer9_attention_weight_min": -33.42999267578125,
      "epoch": 10.71,
      "learning_rate": 8.172689393939393e-05,
      "loss": 2.7892,
      "step": 184400
    },
    {
      "activations/layer0_attention_weight_max": 16.7827091217041,
      "activations/layer0_attention_weight_min": -11.831436157226562,
      "activations/layer10_attention_weight_max": 30.15898895263672,
      "activations/layer10_attention_weight_min": -30.403953552246094,
      "activations/layer11_attention_weight_max": 32.33488464355469,
      "activations/layer11_attention_weight_min": -29.29024887084961,
      "activations/layer12_attention_weight_max": 23.74915313720703,
      "activations/layer12_attention_weight_min": -25.845932006835938,
      "activations/layer13_attention_weight_max": 36.56134796142578,
      "activations/layer13_attention_weight_min": -34.89571762084961,
      "activations/layer14_attention_weight_max": 42.337646484375,
      "activations/layer14_attention_weight_min": -36.52757263183594,
      "activations/layer15_attention_weight_max": 35.00699996948242,
      "activations/layer15_attention_weight_min": -30.438796997070312,
      "activations/layer16_attention_weight_max": 30.80210304260254,
      "activations/layer16_attention_weight_min": -27.787485122680664,
      "activations/layer17_attention_weight_max": 30.582942962646484,
      "activations/layer17_attention_weight_min": -25.958999633789062,
      "activations/layer18_attention_weight_max": 32.819149017333984,
      "activations/layer18_attention_weight_min": -22.6422119140625,
      "activations/layer19_attention_weight_max": 34.37346649169922,
      "activations/layer19_attention_weight_min": -31.91880226135254,
      "activations/layer1_attention_weight_max": 16.91446304321289,
      "activations/layer1_attention_weight_min": -15.137383460998535,
      "activations/layer20_attention_weight_max": 27.4571533203125,
      "activations/layer20_attention_weight_min": -24.491300582885742,
      "activations/layer21_attention_weight_max": 31.21829605102539,
      "activations/layer21_attention_weight_min": -24.201690673828125,
      "activations/layer22_attention_weight_max": 42.60865020751953,
      "activations/layer22_attention_weight_min": -31.890649795532227,
      "activations/layer23_attention_weight_max": 33.6260986328125,
      "activations/layer23_attention_weight_min": -26.873729705810547,
      "activations/layer2_attention_weight_max": 35.46525192260742,
      "activations/layer2_attention_weight_min": -33.236202239990234,
      "activations/layer3_attention_weight_max": 91.9922866821289,
      "activations/layer3_attention_weight_min": -95.22112274169922,
      "activations/layer4_attention_weight_max": 112.9935073852539,
      "activations/layer4_attention_weight_min": -110.13883209228516,
      "activations/layer5_attention_weight_max": 53.6994743347168,
      "activations/layer5_attention_weight_min": -61.868160247802734,
      "activations/layer6_attention_weight_max": 44.74759292602539,
      "activations/layer6_attention_weight_min": -49.083351135253906,
      "activations/layer7_attention_weight_max": 91.31034088134766,
      "activations/layer7_attention_weight_min": -94.4939193725586,
      "activations/layer8_attention_weight_max": 36.976749420166016,
      "activations/layer8_attention_weight_min": -39.85951614379883,
      "activations/layer9_attention_weight_max": 32.16385269165039,
      "activations/layer9_attention_weight_min": -31.529150009155273,
      "epoch": 10.72,
      "learning_rate": 8.170795454545454e-05,
      "loss": 2.7764,
      "step": 184450
    },
    {
      "activations/layer0_attention_weight_max": 16.30953025817871,
      "activations/layer0_attention_weight_min": -13.006011009216309,
      "activations/layer10_attention_weight_max": 32.80743408203125,
      "activations/layer10_attention_weight_min": -32.73517990112305,
      "activations/layer11_attention_weight_max": 31.119848251342773,
      "activations/layer11_attention_weight_min": -31.418088912963867,
      "activations/layer12_attention_weight_max": 24.553003311157227,
      "activations/layer12_attention_weight_min": -24.41446876525879,
      "activations/layer13_attention_weight_max": 40.39733123779297,
      "activations/layer13_attention_weight_min": -32.54438781738281,
      "activations/layer14_attention_weight_max": 43.63691711425781,
      "activations/layer14_attention_weight_min": -36.55888748168945,
      "activations/layer15_attention_weight_max": 35.99274444580078,
      "activations/layer15_attention_weight_min": -31.51722526550293,
      "activations/layer16_attention_weight_max": 33.073486328125,
      "activations/layer16_attention_weight_min": -27.654752731323242,
      "activations/layer17_attention_weight_max": 30.749507904052734,
      "activations/layer17_attention_weight_min": -24.814783096313477,
      "activations/layer18_attention_weight_max": 29.334678649902344,
      "activations/layer18_attention_weight_min": -22.845170974731445,
      "activations/layer19_attention_weight_max": 31.827295303344727,
      "activations/layer19_attention_weight_min": -29.736785888671875,
      "activations/layer1_attention_weight_max": 16.31519889831543,
      "activations/layer1_attention_weight_min": -15.02365493774414,
      "activations/layer20_attention_weight_max": 27.416086196899414,
      "activations/layer20_attention_weight_min": -23.887229919433594,
      "activations/layer21_attention_weight_max": 26.100915908813477,
      "activations/layer21_attention_weight_min": -23.1124324798584,
      "activations/layer22_attention_weight_max": 39.39984893798828,
      "activations/layer22_attention_weight_min": -31.00202751159668,
      "activations/layer23_attention_weight_max": 32.47412109375,
      "activations/layer23_attention_weight_min": -25.22718620300293,
      "activations/layer2_attention_weight_max": 31.699132919311523,
      "activations/layer2_attention_weight_min": -32.852169036865234,
      "activations/layer3_attention_weight_max": 90.53852081298828,
      "activations/layer3_attention_weight_min": -91.2938003540039,
      "activations/layer4_attention_weight_max": 114.2679214477539,
      "activations/layer4_attention_weight_min": -111.66399383544922,
      "activations/layer5_attention_weight_max": 54.21091842651367,
      "activations/layer5_attention_weight_min": -68.56253814697266,
      "activations/layer6_attention_weight_max": 44.32380294799805,
      "activations/layer6_attention_weight_min": -49.1054801940918,
      "activations/layer7_attention_weight_max": 84.32096862792969,
      "activations/layer7_attention_weight_min": -88.82450866699219,
      "activations/layer8_attention_weight_max": 37.886199951171875,
      "activations/layer8_attention_weight_min": -39.51601791381836,
      "activations/layer9_attention_weight_max": 32.28034591674805,
      "activations/layer9_attention_weight_min": -32.12819290161133,
      "epoch": 10.72,
      "learning_rate": 8.168901515151514e-05,
      "loss": 2.784,
      "step": 184500
    },
    {
      "activations/layer0_attention_weight_max": 16.308107376098633,
      "activations/layer0_attention_weight_min": -12.845640182495117,
      "activations/layer10_attention_weight_max": 33.34360122680664,
      "activations/layer10_attention_weight_min": -34.12715530395508,
      "activations/layer11_attention_weight_max": 34.73815155029297,
      "activations/layer11_attention_weight_min": -34.961483001708984,
      "activations/layer12_attention_weight_max": 23.907241821289062,
      "activations/layer12_attention_weight_min": -25.200973510742188,
      "activations/layer13_attention_weight_max": 40.22123336791992,
      "activations/layer13_attention_weight_min": -34.773826599121094,
      "activations/layer14_attention_weight_max": 46.60609817504883,
      "activations/layer14_attention_weight_min": -38.88695526123047,
      "activations/layer15_attention_weight_max": 36.650856018066406,
      "activations/layer15_attention_weight_min": -32.17925262451172,
      "activations/layer16_attention_weight_max": 33.06541442871094,
      "activations/layer16_attention_weight_min": -29.33799934387207,
      "activations/layer17_attention_weight_max": 32.93609619140625,
      "activations/layer17_attention_weight_min": -28.638837814331055,
      "activations/layer18_attention_weight_max": 33.29200744628906,
      "activations/layer18_attention_weight_min": -25.297685623168945,
      "activations/layer19_attention_weight_max": 38.107749938964844,
      "activations/layer19_attention_weight_min": -29.58424949645996,
      "activations/layer1_attention_weight_max": 17.998769760131836,
      "activations/layer1_attention_weight_min": -16.913330078125,
      "activations/layer20_attention_weight_max": 33.921546936035156,
      "activations/layer20_attention_weight_min": -23.92983627319336,
      "activations/layer21_attention_weight_max": 30.933866500854492,
      "activations/layer21_attention_weight_min": -24.036874771118164,
      "activations/layer22_attention_weight_max": 49.07605743408203,
      "activations/layer22_attention_weight_min": -31.004663467407227,
      "activations/layer23_attention_weight_max": 41.495296478271484,
      "activations/layer23_attention_weight_min": -26.067596435546875,
      "activations/layer2_attention_weight_max": 34.04808807373047,
      "activations/layer2_attention_weight_min": -32.72050476074219,
      "activations/layer3_attention_weight_max": 91.45580291748047,
      "activations/layer3_attention_weight_min": -92.33606719970703,
      "activations/layer4_attention_weight_max": 115.88671875,
      "activations/layer4_attention_weight_min": -109.68587493896484,
      "activations/layer5_attention_weight_max": 53.50251388549805,
      "activations/layer5_attention_weight_min": -62.01482391357422,
      "activations/layer6_attention_weight_max": 47.35434341430664,
      "activations/layer6_attention_weight_min": -51.8461799621582,
      "activations/layer7_attention_weight_max": 91.28086853027344,
      "activations/layer7_attention_weight_min": -89.59844207763672,
      "activations/layer8_attention_weight_max": 42.346553802490234,
      "activations/layer8_attention_weight_min": -43.5539436340332,
      "activations/layer9_attention_weight_max": 34.275020599365234,
      "activations/layer9_attention_weight_min": -33.542076110839844,
      "epoch": 10.72,
      "learning_rate": 8.167007575757576e-05,
      "loss": 2.7834,
      "step": 184550
    },
    {
      "activations/layer0_attention_weight_max": 16.926767349243164,
      "activations/layer0_attention_weight_min": -12.161136627197266,
      "activations/layer10_attention_weight_max": 32.2330207824707,
      "activations/layer10_attention_weight_min": -32.0375862121582,
      "activations/layer11_attention_weight_max": 32.33209991455078,
      "activations/layer11_attention_weight_min": -31.472660064697266,
      "activations/layer12_attention_weight_max": 31.80504608154297,
      "activations/layer12_attention_weight_min": -32.899696350097656,
      "activations/layer13_attention_weight_max": 42.665199279785156,
      "activations/layer13_attention_weight_min": -33.65853500366211,
      "activations/layer14_attention_weight_max": 47.47463607788086,
      "activations/layer14_attention_weight_min": -36.713592529296875,
      "activations/layer15_attention_weight_max": 37.22551727294922,
      "activations/layer15_attention_weight_min": -32.2568359375,
      "activations/layer16_attention_weight_max": 34.765846252441406,
      "activations/layer16_attention_weight_min": -28.394323348999023,
      "activations/layer17_attention_weight_max": 31.792240142822266,
      "activations/layer17_attention_weight_min": -27.096200942993164,
      "activations/layer18_attention_weight_max": 32.55113983154297,
      "activations/layer18_attention_weight_min": -24.631200790405273,
      "activations/layer19_attention_weight_max": 37.81656265258789,
      "activations/layer19_attention_weight_min": -31.9123477935791,
      "activations/layer1_attention_weight_max": 16.451568603515625,
      "activations/layer1_attention_weight_min": -15.336089134216309,
      "activations/layer20_attention_weight_max": 33.50204849243164,
      "activations/layer20_attention_weight_min": -25.642066955566406,
      "activations/layer21_attention_weight_max": 32.79723358154297,
      "activations/layer21_attention_weight_min": -24.091014862060547,
      "activations/layer22_attention_weight_max": 49.344696044921875,
      "activations/layer22_attention_weight_min": -28.926898956298828,
      "activations/layer23_attention_weight_max": 34.61457824707031,
      "activations/layer23_attention_weight_min": -22.179683685302734,
      "activations/layer2_attention_weight_max": 34.255950927734375,
      "activations/layer2_attention_weight_min": -32.184120178222656,
      "activations/layer3_attention_weight_max": 90.87873840332031,
      "activations/layer3_attention_weight_min": -92.4531478881836,
      "activations/layer4_attention_weight_max": 108.05322265625,
      "activations/layer4_attention_weight_min": -113.13372802734375,
      "activations/layer5_attention_weight_max": 52.11164474487305,
      "activations/layer5_attention_weight_min": -60.13412094116211,
      "activations/layer6_attention_weight_max": 43.78573989868164,
      "activations/layer6_attention_weight_min": -50.24330139160156,
      "activations/layer7_attention_weight_max": 91.53720092773438,
      "activations/layer7_attention_weight_min": -86.16271209716797,
      "activations/layer8_attention_weight_max": 38.22277069091797,
      "activations/layer8_attention_weight_min": -40.188026428222656,
      "activations/layer9_attention_weight_max": 33.326568603515625,
      "activations/layer9_attention_weight_min": -34.122352600097656,
      "epoch": 10.73,
      "learning_rate": 8.165113636363635e-05,
      "loss": 2.7644,
      "step": 184600
    },
    {
      "activations/layer0_attention_weight_max": 15.099193572998047,
      "activations/layer0_attention_weight_min": -12.33224105834961,
      "activations/layer10_attention_weight_max": 32.97678756713867,
      "activations/layer10_attention_weight_min": -31.41324806213379,
      "activations/layer11_attention_weight_max": 33.7315788269043,
      "activations/layer11_attention_weight_min": -30.27870750427246,
      "activations/layer12_attention_weight_max": 22.96626091003418,
      "activations/layer12_attention_weight_min": -29.5924015045166,
      "activations/layer13_attention_weight_max": 37.01763153076172,
      "activations/layer13_attention_weight_min": -32.87993621826172,
      "activations/layer14_attention_weight_max": 41.96697235107422,
      "activations/layer14_attention_weight_min": -36.75032043457031,
      "activations/layer15_attention_weight_max": 34.33812713623047,
      "activations/layer15_attention_weight_min": -32.93490219116211,
      "activations/layer16_attention_weight_max": 27.545886993408203,
      "activations/layer16_attention_weight_min": -26.671403884887695,
      "activations/layer17_attention_weight_max": 31.01265525817871,
      "activations/layer17_attention_weight_min": -23.66925048828125,
      "activations/layer18_attention_weight_max": 31.244352340698242,
      "activations/layer18_attention_weight_min": -25.04038429260254,
      "activations/layer19_attention_weight_max": 36.20204544067383,
      "activations/layer19_attention_weight_min": -32.915653228759766,
      "activations/layer1_attention_weight_max": 16.804548263549805,
      "activations/layer1_attention_weight_min": -16.020761489868164,
      "activations/layer20_attention_weight_max": 29.442947387695312,
      "activations/layer20_attention_weight_min": -26.788162231445312,
      "activations/layer21_attention_weight_max": 30.052818298339844,
      "activations/layer21_attention_weight_min": -25.26255989074707,
      "activations/layer22_attention_weight_max": 43.47288131713867,
      "activations/layer22_attention_weight_min": -30.703311920166016,
      "activations/layer23_attention_weight_max": 34.23912811279297,
      "activations/layer23_attention_weight_min": -25.871185302734375,
      "activations/layer2_attention_weight_max": 35.09413528442383,
      "activations/layer2_attention_weight_min": -35.18571090698242,
      "activations/layer3_attention_weight_max": 99.22575378417969,
      "activations/layer3_attention_weight_min": -97.76671600341797,
      "activations/layer4_attention_weight_max": 114.47607421875,
      "activations/layer4_attention_weight_min": -110.94493865966797,
      "activations/layer5_attention_weight_max": 54.02659225463867,
      "activations/layer5_attention_weight_min": -62.265533447265625,
      "activations/layer6_attention_weight_max": 45.965721130371094,
      "activations/layer6_attention_weight_min": -45.83475875854492,
      "activations/layer7_attention_weight_max": 95.14705657958984,
      "activations/layer7_attention_weight_min": -89.86106872558594,
      "activations/layer8_attention_weight_max": 39.36772537231445,
      "activations/layer8_attention_weight_min": -39.43952560424805,
      "activations/layer9_attention_weight_max": 32.85021209716797,
      "activations/layer9_attention_weight_min": -33.36003494262695,
      "epoch": 10.73,
      "learning_rate": 8.163219696969696e-05,
      "loss": 2.7852,
      "step": 184650
    },
    {
      "activations/layer0_attention_weight_max": 15.350830078125,
      "activations/layer0_attention_weight_min": -12.693507194519043,
      "activations/layer10_attention_weight_max": 37.37449645996094,
      "activations/layer10_attention_weight_min": -36.10569381713867,
      "activations/layer11_attention_weight_max": 34.70379638671875,
      "activations/layer11_attention_weight_min": -35.41556167602539,
      "activations/layer12_attention_weight_max": 26.48099708557129,
      "activations/layer12_attention_weight_min": -26.107248306274414,
      "activations/layer13_attention_weight_max": 43.27312088012695,
      "activations/layer13_attention_weight_min": -36.94328689575195,
      "activations/layer14_attention_weight_max": 48.0540885925293,
      "activations/layer14_attention_weight_min": -42.093143463134766,
      "activations/layer15_attention_weight_max": 41.0870475769043,
      "activations/layer15_attention_weight_min": -35.39020919799805,
      "activations/layer16_attention_weight_max": 31.958389282226562,
      "activations/layer16_attention_weight_min": -27.174915313720703,
      "activations/layer17_attention_weight_max": 34.50802230834961,
      "activations/layer17_attention_weight_min": -25.59712791442871,
      "activations/layer18_attention_weight_max": 30.03640365600586,
      "activations/layer18_attention_weight_min": -22.694002151489258,
      "activations/layer19_attention_weight_max": 32.481441497802734,
      "activations/layer19_attention_weight_min": -31.540740966796875,
      "activations/layer1_attention_weight_max": 17.165790557861328,
      "activations/layer1_attention_weight_min": -14.362403869628906,
      "activations/layer20_attention_weight_max": 27.59209442138672,
      "activations/layer20_attention_weight_min": -23.478988647460938,
      "activations/layer21_attention_weight_max": 30.138858795166016,
      "activations/layer21_attention_weight_min": -21.286001205444336,
      "activations/layer22_attention_weight_max": 41.833106994628906,
      "activations/layer22_attention_weight_min": -32.015869140625,
      "activations/layer23_attention_weight_max": 35.34858703613281,
      "activations/layer23_attention_weight_min": -26.34424591064453,
      "activations/layer2_attention_weight_max": 33.65201950073242,
      "activations/layer2_attention_weight_min": -32.37153625488281,
      "activations/layer3_attention_weight_max": 93.3691635131836,
      "activations/layer3_attention_weight_min": -94.54315185546875,
      "activations/layer4_attention_weight_max": 114.30195617675781,
      "activations/layer4_attention_weight_min": -110.26582336425781,
      "activations/layer5_attention_weight_max": 51.81470489501953,
      "activations/layer5_attention_weight_min": -63.80671691894531,
      "activations/layer6_attention_weight_max": 44.40360641479492,
      "activations/layer6_attention_weight_min": -47.67041778564453,
      "activations/layer7_attention_weight_max": 89.8118667602539,
      "activations/layer7_attention_weight_min": -92.33189392089844,
      "activations/layer8_attention_weight_max": 43.467376708984375,
      "activations/layer8_attention_weight_min": -42.63992691040039,
      "activations/layer9_attention_weight_max": 39.20956802368164,
      "activations/layer9_attention_weight_min": -36.83497619628906,
      "epoch": 10.73,
      "learning_rate": 8.161325757575758e-05,
      "loss": 2.7743,
      "step": 184700
    },
    {
      "activations/layer0_attention_weight_max": 15.264067649841309,
      "activations/layer0_attention_weight_min": -11.955921173095703,
      "activations/layer10_attention_weight_max": 34.0389289855957,
      "activations/layer10_attention_weight_min": -31.705551147460938,
      "activations/layer11_attention_weight_max": 34.04146194458008,
      "activations/layer11_attention_weight_min": -32.52521514892578,
      "activations/layer12_attention_weight_max": 25.6476993560791,
      "activations/layer12_attention_weight_min": -25.508686065673828,
      "activations/layer13_attention_weight_max": 39.11238098144531,
      "activations/layer13_attention_weight_min": -34.01374435424805,
      "activations/layer14_attention_weight_max": 44.7761116027832,
      "activations/layer14_attention_weight_min": -40.14116668701172,
      "activations/layer15_attention_weight_max": 37.886470794677734,
      "activations/layer15_attention_weight_min": -31.884056091308594,
      "activations/layer16_attention_weight_max": 32.59510803222656,
      "activations/layer16_attention_weight_min": -27.381793975830078,
      "activations/layer17_attention_weight_max": 34.38654327392578,
      "activations/layer17_attention_weight_min": -27.036130905151367,
      "activations/layer18_attention_weight_max": 33.595794677734375,
      "activations/layer18_attention_weight_min": -23.28326416015625,
      "activations/layer19_attention_weight_max": 37.187198638916016,
      "activations/layer19_attention_weight_min": -30.407194137573242,
      "activations/layer1_attention_weight_max": 16.670330047607422,
      "activations/layer1_attention_weight_min": -15.270813941955566,
      "activations/layer20_attention_weight_max": 31.311290740966797,
      "activations/layer20_attention_weight_min": -25.371829986572266,
      "activations/layer21_attention_weight_max": 32.30843734741211,
      "activations/layer21_attention_weight_min": -24.146207809448242,
      "activations/layer22_attention_weight_max": 45.94233703613281,
      "activations/layer22_attention_weight_min": -32.17190933227539,
      "activations/layer23_attention_weight_max": 37.41854476928711,
      "activations/layer23_attention_weight_min": -24.65921401977539,
      "activations/layer2_attention_weight_max": 35.69587707519531,
      "activations/layer2_attention_weight_min": -33.90918731689453,
      "activations/layer3_attention_weight_max": 95.85266876220703,
      "activations/layer3_attention_weight_min": -97.32328796386719,
      "activations/layer4_attention_weight_max": 118.32054901123047,
      "activations/layer4_attention_weight_min": -118.52506256103516,
      "activations/layer5_attention_weight_max": 55.94686508178711,
      "activations/layer5_attention_weight_min": -61.48859405517578,
      "activations/layer6_attention_weight_max": 45.412681579589844,
      "activations/layer6_attention_weight_min": -47.16684341430664,
      "activations/layer7_attention_weight_max": 90.23956298828125,
      "activations/layer7_attention_weight_min": -86.26466369628906,
      "activations/layer8_attention_weight_max": 42.326053619384766,
      "activations/layer8_attention_weight_min": -40.6479377746582,
      "activations/layer9_attention_weight_max": 36.015865325927734,
      "activations/layer9_attention_weight_min": -33.505130767822266,
      "epoch": 10.74,
      "learning_rate": 8.159431818181816e-05,
      "loss": 2.7822,
      "step": 184750
    },
    {
      "activations/layer0_attention_weight_max": 15.83132266998291,
      "activations/layer0_attention_weight_min": -12.71793270111084,
      "activations/layer10_attention_weight_max": 30.209043502807617,
      "activations/layer10_attention_weight_min": -32.87468338012695,
      "activations/layer11_attention_weight_max": 31.524394989013672,
      "activations/layer11_attention_weight_min": -31.468372344970703,
      "activations/layer12_attention_weight_max": 25.9886417388916,
      "activations/layer12_attention_weight_min": -24.691070556640625,
      "activations/layer13_attention_weight_max": 39.72536087036133,
      "activations/layer13_attention_weight_min": -32.71111297607422,
      "activations/layer14_attention_weight_max": 46.6407470703125,
      "activations/layer14_attention_weight_min": -37.96343231201172,
      "activations/layer15_attention_weight_max": 34.81645965576172,
      "activations/layer15_attention_weight_min": -30.746522903442383,
      "activations/layer16_attention_weight_max": 30.643978118896484,
      "activations/layer16_attention_weight_min": -29.08201789855957,
      "activations/layer17_attention_weight_max": 34.029029846191406,
      "activations/layer17_attention_weight_min": -26.116113662719727,
      "activations/layer18_attention_weight_max": 31.684213638305664,
      "activations/layer18_attention_weight_min": -23.99190330505371,
      "activations/layer19_attention_weight_max": 35.78288650512695,
      "activations/layer19_attention_weight_min": -30.899831771850586,
      "activations/layer1_attention_weight_max": 16.75528907775879,
      "activations/layer1_attention_weight_min": -15.827759742736816,
      "activations/layer20_attention_weight_max": 29.55388832092285,
      "activations/layer20_attention_weight_min": -24.452014923095703,
      "activations/layer21_attention_weight_max": 29.988492965698242,
      "activations/layer21_attention_weight_min": -23.60098648071289,
      "activations/layer22_attention_weight_max": 49.91740798950195,
      "activations/layer22_attention_weight_min": -28.9149169921875,
      "activations/layer23_attention_weight_max": 37.42125701904297,
      "activations/layer23_attention_weight_min": -24.843597412109375,
      "activations/layer2_attention_weight_max": 32.79540252685547,
      "activations/layer2_attention_weight_min": -33.51360321044922,
      "activations/layer3_attention_weight_max": 93.4486083984375,
      "activations/layer3_attention_weight_min": -97.11421966552734,
      "activations/layer4_attention_weight_max": 111.53008270263672,
      "activations/layer4_attention_weight_min": -112.88494110107422,
      "activations/layer5_attention_weight_max": 51.061744689941406,
      "activations/layer5_attention_weight_min": -61.470333099365234,
      "activations/layer6_attention_weight_max": 43.4769172668457,
      "activations/layer6_attention_weight_min": -49.88032150268555,
      "activations/layer7_attention_weight_max": 84.98139953613281,
      "activations/layer7_attention_weight_min": -90.7208023071289,
      "activations/layer8_attention_weight_max": 39.17435836791992,
      "activations/layer8_attention_weight_min": -41.259830474853516,
      "activations/layer9_attention_weight_max": 32.78742980957031,
      "activations/layer9_attention_weight_min": -33.16498565673828,
      "epoch": 10.74,
      "learning_rate": 8.157537878787878e-05,
      "loss": 2.7903,
      "step": 184800
    },
    {
      "activations/layer0_attention_weight_max": 15.984797477722168,
      "activations/layer0_attention_weight_min": -13.370888710021973,
      "activations/layer10_attention_weight_max": 32.639991760253906,
      "activations/layer10_attention_weight_min": -32.15840148925781,
      "activations/layer11_attention_weight_max": 32.39154815673828,
      "activations/layer11_attention_weight_min": -32.90700912475586,
      "activations/layer12_attention_weight_max": 23.95343017578125,
      "activations/layer12_attention_weight_min": -25.615421295166016,
      "activations/layer13_attention_weight_max": 38.03690719604492,
      "activations/layer13_attention_weight_min": -35.208194732666016,
      "activations/layer14_attention_weight_max": 43.547698974609375,
      "activations/layer14_attention_weight_min": -39.32796859741211,
      "activations/layer15_attention_weight_max": 35.711280822753906,
      "activations/layer15_attention_weight_min": -32.16917037963867,
      "activations/layer16_attention_weight_max": 30.486446380615234,
      "activations/layer16_attention_weight_min": -28.214271545410156,
      "activations/layer17_attention_weight_max": 30.801862716674805,
      "activations/layer17_attention_weight_min": -23.996078491210938,
      "activations/layer18_attention_weight_max": 30.41615104675293,
      "activations/layer18_attention_weight_min": -22.604957580566406,
      "activations/layer19_attention_weight_max": 31.800399780273438,
      "activations/layer19_attention_weight_min": -31.35623550415039,
      "activations/layer1_attention_weight_max": 16.78693199157715,
      "activations/layer1_attention_weight_min": -14.417829513549805,
      "activations/layer20_attention_weight_max": 28.24262046813965,
      "activations/layer20_attention_weight_min": -23.73828125,
      "activations/layer21_attention_weight_max": 28.4674129486084,
      "activations/layer21_attention_weight_min": -23.712936401367188,
      "activations/layer22_attention_weight_max": 42.04996109008789,
      "activations/layer22_attention_weight_min": -29.545148849487305,
      "activations/layer23_attention_weight_max": 37.12698745727539,
      "activations/layer23_attention_weight_min": -27.745433807373047,
      "activations/layer2_attention_weight_max": 33.329654693603516,
      "activations/layer2_attention_weight_min": -33.32890319824219,
      "activations/layer3_attention_weight_max": 97.22383117675781,
      "activations/layer3_attention_weight_min": -95.54194641113281,
      "activations/layer4_attention_weight_max": 114.94121551513672,
      "activations/layer4_attention_weight_min": -116.7628402709961,
      "activations/layer5_attention_weight_max": 53.94947814941406,
      "activations/layer5_attention_weight_min": -60.795066833496094,
      "activations/layer6_attention_weight_max": 45.44925308227539,
      "activations/layer6_attention_weight_min": -47.88576126098633,
      "activations/layer7_attention_weight_max": 95.49400329589844,
      "activations/layer7_attention_weight_min": -91.19982147216797,
      "activations/layer8_attention_weight_max": 40.13327407836914,
      "activations/layer8_attention_weight_min": -41.15583801269531,
      "activations/layer9_attention_weight_max": 34.21122741699219,
      "activations/layer9_attention_weight_min": -35.14015197753906,
      "epoch": 10.74,
      "learning_rate": 8.155681818181817e-05,
      "loss": 2.794,
      "step": 184850
    },
    {
      "activations/layer0_attention_weight_max": 16.510149002075195,
      "activations/layer0_attention_weight_min": -13.167900085449219,
      "activations/layer10_attention_weight_max": 34.27219009399414,
      "activations/layer10_attention_weight_min": -35.73337936401367,
      "activations/layer11_attention_weight_max": 33.83665466308594,
      "activations/layer11_attention_weight_min": -31.846328735351562,
      "activations/layer12_attention_weight_max": 26.283008575439453,
      "activations/layer12_attention_weight_min": -27.53940200805664,
      "activations/layer13_attention_weight_max": 41.24853515625,
      "activations/layer13_attention_weight_min": -36.14651107788086,
      "activations/layer14_attention_weight_max": 50.09476089477539,
      "activations/layer14_attention_weight_min": -42.912506103515625,
      "activations/layer15_attention_weight_max": 36.77300262451172,
      "activations/layer15_attention_weight_min": -32.70037078857422,
      "activations/layer16_attention_weight_max": 33.47693634033203,
      "activations/layer16_attention_weight_min": -28.400590896606445,
      "activations/layer17_attention_weight_max": 35.081871032714844,
      "activations/layer17_attention_weight_min": -27.023012161254883,
      "activations/layer18_attention_weight_max": 33.1652717590332,
      "activations/layer18_attention_weight_min": -23.383317947387695,
      "activations/layer19_attention_weight_max": 37.169368743896484,
      "activations/layer19_attention_weight_min": -30.512109756469727,
      "activations/layer1_attention_weight_max": 17.77369499206543,
      "activations/layer1_attention_weight_min": -16.252845764160156,
      "activations/layer20_attention_weight_max": 31.42009162902832,
      "activations/layer20_attention_weight_min": -25.422163009643555,
      "activations/layer21_attention_weight_max": 29.938859939575195,
      "activations/layer21_attention_weight_min": -24.977933883666992,
      "activations/layer22_attention_weight_max": 51.292747497558594,
      "activations/layer22_attention_weight_min": -32.24006652832031,
      "activations/layer23_attention_weight_max": 38.4807014465332,
      "activations/layer23_attention_weight_min": -26.6789608001709,
      "activations/layer2_attention_weight_max": 32.622802734375,
      "activations/layer2_attention_weight_min": -32.8285026550293,
      "activations/layer3_attention_weight_max": 92.54064178466797,
      "activations/layer3_attention_weight_min": -94.15453338623047,
      "activations/layer4_attention_weight_max": 113.2528076171875,
      "activations/layer4_attention_weight_min": -116.45623779296875,
      "activations/layer5_attention_weight_max": 53.901336669921875,
      "activations/layer5_attention_weight_min": -67.74307250976562,
      "activations/layer6_attention_weight_max": 46.800838470458984,
      "activations/layer6_attention_weight_min": -50.66633605957031,
      "activations/layer7_attention_weight_max": 91.05027770996094,
      "activations/layer7_attention_weight_min": -92.79814147949219,
      "activations/layer8_attention_weight_max": 40.71963882446289,
      "activations/layer8_attention_weight_min": -42.902313232421875,
      "activations/layer9_attention_weight_max": 34.52863693237305,
      "activations/layer9_attention_weight_min": -34.46653747558594,
      "epoch": 10.74,
      "learning_rate": 8.153787878787879e-05,
      "loss": 2.7774,
      "step": 184900
    },
    {
      "activations/layer0_attention_weight_max": 16.360239028930664,
      "activations/layer0_attention_weight_min": -13.25818157196045,
      "activations/layer10_attention_weight_max": 33.28729248046875,
      "activations/layer10_attention_weight_min": -33.18682861328125,
      "activations/layer11_attention_weight_max": 34.05413818359375,
      "activations/layer11_attention_weight_min": -32.39825439453125,
      "activations/layer12_attention_weight_max": 26.44149398803711,
      "activations/layer12_attention_weight_min": -24.620582580566406,
      "activations/layer13_attention_weight_max": 40.63521957397461,
      "activations/layer13_attention_weight_min": -33.710662841796875,
      "activations/layer14_attention_weight_max": 51.089805603027344,
      "activations/layer14_attention_weight_min": -39.39323806762695,
      "activations/layer15_attention_weight_max": 37.0154914855957,
      "activations/layer15_attention_weight_min": -31.088607788085938,
      "activations/layer16_attention_weight_max": 31.80452537536621,
      "activations/layer16_attention_weight_min": -27.33971405029297,
      "activations/layer17_attention_weight_max": 32.135101318359375,
      "activations/layer17_attention_weight_min": -24.75028419494629,
      "activations/layer18_attention_weight_max": 29.703336715698242,
      "activations/layer18_attention_weight_min": -22.525638580322266,
      "activations/layer19_attention_weight_max": 36.88448715209961,
      "activations/layer19_attention_weight_min": -29.24613380432129,
      "activations/layer1_attention_weight_max": 16.988279342651367,
      "activations/layer1_attention_weight_min": -14.296449661254883,
      "activations/layer20_attention_weight_max": 31.496248245239258,
      "activations/layer20_attention_weight_min": -23.001136779785156,
      "activations/layer21_attention_weight_max": 29.315092086791992,
      "activations/layer21_attention_weight_min": -23.010557174682617,
      "activations/layer22_attention_weight_max": 47.40968322753906,
      "activations/layer22_attention_weight_min": -27.579492568969727,
      "activations/layer23_attention_weight_max": 35.15768814086914,
      "activations/layer23_attention_weight_min": -24.652259826660156,
      "activations/layer2_attention_weight_max": 33.860137939453125,
      "activations/layer2_attention_weight_min": -33.801658630371094,
      "activations/layer3_attention_weight_max": 94.24276733398438,
      "activations/layer3_attention_weight_min": -96.06261444091797,
      "activations/layer4_attention_weight_max": 115.34978485107422,
      "activations/layer4_attention_weight_min": -118.5535888671875,
      "activations/layer5_attention_weight_max": 51.708831787109375,
      "activations/layer5_attention_weight_min": -60.644935607910156,
      "activations/layer6_attention_weight_max": 44.61272048950195,
      "activations/layer6_attention_weight_min": -48.22102355957031,
      "activations/layer7_attention_weight_max": 89.2020034790039,
      "activations/layer7_attention_weight_min": -94.90235137939453,
      "activations/layer8_attention_weight_max": 39.713443756103516,
      "activations/layer8_attention_weight_min": -40.49319839477539,
      "activations/layer9_attention_weight_max": 33.13941192626953,
      "activations/layer9_attention_weight_min": -33.94761276245117,
      "epoch": 10.75,
      "learning_rate": 8.151893939393939e-05,
      "loss": 2.7739,
      "step": 184950
    },
    {
      "activations/layer0_attention_weight_max": 15.949841499328613,
      "activations/layer0_attention_weight_min": -12.625154495239258,
      "activations/layer10_attention_weight_max": 33.654815673828125,
      "activations/layer10_attention_weight_min": -33.416446685791016,
      "activations/layer11_attention_weight_max": 34.181148529052734,
      "activations/layer11_attention_weight_min": -32.450923919677734,
      "activations/layer12_attention_weight_max": 26.708276748657227,
      "activations/layer12_attention_weight_min": -25.998876571655273,
      "activations/layer13_attention_weight_max": 41.77912521362305,
      "activations/layer13_attention_weight_min": -36.54707336425781,
      "activations/layer14_attention_weight_max": 52.65087890625,
      "activations/layer14_attention_weight_min": -41.566749572753906,
      "activations/layer15_attention_weight_max": 42.40053939819336,
      "activations/layer15_attention_weight_min": -33.222373962402344,
      "activations/layer16_attention_weight_max": 35.614356994628906,
      "activations/layer16_attention_weight_min": -29.807331085205078,
      "activations/layer17_attention_weight_max": 39.88454055786133,
      "activations/layer17_attention_weight_min": -27.833269119262695,
      "activations/layer18_attention_weight_max": 34.286537170410156,
      "activations/layer18_attention_weight_min": -25.003820419311523,
      "activations/layer19_attention_weight_max": 42.520381927490234,
      "activations/layer19_attention_weight_min": -35.15700912475586,
      "activations/layer1_attention_weight_max": 17.851839065551758,
      "activations/layer1_attention_weight_min": -16.86126136779785,
      "activations/layer20_attention_weight_max": 34.21945571899414,
      "activations/layer20_attention_weight_min": -27.88438606262207,
      "activations/layer21_attention_weight_max": 33.583885192871094,
      "activations/layer21_attention_weight_min": -26.60057258605957,
      "activations/layer22_attention_weight_max": 53.80438232421875,
      "activations/layer22_attention_weight_min": -33.310237884521484,
      "activations/layer23_attention_weight_max": 42.597572326660156,
      "activations/layer23_attention_weight_min": -25.04706573486328,
      "activations/layer2_attention_weight_max": 33.7221565246582,
      "activations/layer2_attention_weight_min": -32.97355651855469,
      "activations/layer3_attention_weight_max": 99.59431457519531,
      "activations/layer3_attention_weight_min": -101.2864761352539,
      "activations/layer4_attention_weight_max": 112.24139404296875,
      "activations/layer4_attention_weight_min": -113.0594482421875,
      "activations/layer5_attention_weight_max": 53.50726318359375,
      "activations/layer5_attention_weight_min": -62.596771240234375,
      "activations/layer6_attention_weight_max": 49.03120803833008,
      "activations/layer6_attention_weight_min": -49.37133026123047,
      "activations/layer7_attention_weight_max": 93.41361236572266,
      "activations/layer7_attention_weight_min": -89.55661010742188,
      "activations/layer8_attention_weight_max": 41.6219367980957,
      "activations/layer8_attention_weight_min": -41.61391067504883,
      "activations/layer9_attention_weight_max": 35.08868408203125,
      "activations/layer9_attention_weight_min": -33.051082611083984,
      "epoch": 10.75,
      "learning_rate": 8.149999999999999e-05,
      "loss": 2.7866,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5187,
      "eval_samples_per_second": 504.07,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5187,
      "eval_openwebtext_samples_per_second": 504.07,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0046,
      "eval_wikitext_samples_per_second": 227.477,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.5636,
      "eval_lambada_samples_per_second": 509.118,
      "step": 185000
    },
    {
      "activations/layer0_attention_weight_max": 16.89618492126465,
      "activations/layer0_attention_weight_min": -13.227570533752441,
      "activations/layer10_attention_weight_max": 32.19612503051758,
      "activations/layer10_attention_weight_min": -33.667076110839844,
      "activations/layer11_attention_weight_max": 32.99460220336914,
      "activations/layer11_attention_weight_min": -32.35577392578125,
      "activations/layer12_attention_weight_max": 31.90241050720215,
      "activations/layer12_attention_weight_min": -24.548973083496094,
      "activations/layer13_attention_weight_max": 43.22810363769531,
      "activations/layer13_attention_weight_min": -33.788509368896484,
      "activations/layer14_attention_weight_max": 45.934940338134766,
      "activations/layer14_attention_weight_min": -36.926849365234375,
      "activations/layer15_attention_weight_max": 39.042945861816406,
      "activations/layer15_attention_weight_min": -32.49655532836914,
      "activations/layer16_attention_weight_max": 31.54643440246582,
      "activations/layer16_attention_weight_min": -27.332151412963867,
      "activations/layer17_attention_weight_max": 32.050682067871094,
      "activations/layer17_attention_weight_min": -24.45943832397461,
      "activations/layer18_attention_weight_max": 30.1188907623291,
      "activations/layer18_attention_weight_min": -20.98518943786621,
      "activations/layer19_attention_weight_max": 35.437618255615234,
      "activations/layer19_attention_weight_min": -31.575942993164062,
      "activations/layer1_attention_weight_max": 17.36782455444336,
      "activations/layer1_attention_weight_min": -13.663212776184082,
      "activations/layer20_attention_weight_max": 30.476478576660156,
      "activations/layer20_attention_weight_min": -24.491954803466797,
      "activations/layer21_attention_weight_max": 29.426998138427734,
      "activations/layer21_attention_weight_min": -22.027193069458008,
      "activations/layer22_attention_weight_max": 44.2247428894043,
      "activations/layer22_attention_weight_min": -27.6146183013916,
      "activations/layer23_attention_weight_max": 32.29493713378906,
      "activations/layer23_attention_weight_min": -22.45039939880371,
      "activations/layer2_attention_weight_max": 31.122879028320312,
      "activations/layer2_attention_weight_min": -33.664241790771484,
      "activations/layer3_attention_weight_max": 90.25568389892578,
      "activations/layer3_attention_weight_min": -92.55006408691406,
      "activations/layer4_attention_weight_max": 110.23602294921875,
      "activations/layer4_attention_weight_min": -111.201171875,
      "activations/layer5_attention_weight_max": 51.25706481933594,
      "activations/layer5_attention_weight_min": -63.9862060546875,
      "activations/layer6_attention_weight_max": 44.07437515258789,
      "activations/layer6_attention_weight_min": -46.79045104980469,
      "activations/layer7_attention_weight_max": 87.07150268554688,
      "activations/layer7_attention_weight_min": -94.9137954711914,
      "activations/layer8_attention_weight_max": 39.960994720458984,
      "activations/layer8_attention_weight_min": -40.21855163574219,
      "activations/layer9_attention_weight_max": 35.5595817565918,
      "activations/layer9_attention_weight_min": -33.41781997680664,
      "epoch": 10.75,
      "learning_rate": 8.14810606060606e-05,
      "loss": 2.7823,
      "step": 185050
    },
    {
      "activations/layer0_attention_weight_max": 15.60694694519043,
      "activations/layer0_attention_weight_min": -13.740575790405273,
      "activations/layer10_attention_weight_max": 30.52849578857422,
      "activations/layer10_attention_weight_min": -31.776823043823242,
      "activations/layer11_attention_weight_max": 29.40697479248047,
      "activations/layer11_attention_weight_min": -29.651212692260742,
      "activations/layer12_attention_weight_max": 24.317590713500977,
      "activations/layer12_attention_weight_min": -24.532499313354492,
      "activations/layer13_attention_weight_max": 36.692684173583984,
      "activations/layer13_attention_weight_min": -33.83759307861328,
      "activations/layer14_attention_weight_max": 40.06843948364258,
      "activations/layer14_attention_weight_min": -35.16709899902344,
      "activations/layer15_attention_weight_max": 33.71466064453125,
      "activations/layer15_attention_weight_min": -31.526824951171875,
      "activations/layer16_attention_weight_max": 29.01617431640625,
      "activations/layer16_attention_weight_min": -27.390399932861328,
      "activations/layer17_attention_weight_max": 30.089014053344727,
      "activations/layer17_attention_weight_min": -25.408029556274414,
      "activations/layer18_attention_weight_max": 32.54378128051758,
      "activations/layer18_attention_weight_min": -22.614601135253906,
      "activations/layer19_attention_weight_max": 34.92002487182617,
      "activations/layer19_attention_weight_min": -32.29864501953125,
      "activations/layer1_attention_weight_max": 18.09874725341797,
      "activations/layer1_attention_weight_min": -14.187861442565918,
      "activations/layer20_attention_weight_max": 30.361709594726562,
      "activations/layer20_attention_weight_min": -24.939579010009766,
      "activations/layer21_attention_weight_max": 29.09623908996582,
      "activations/layer21_attention_weight_min": -23.863182067871094,
      "activations/layer22_attention_weight_max": 42.58628845214844,
      "activations/layer22_attention_weight_min": -31.220661163330078,
      "activations/layer23_attention_weight_max": 34.7706298828125,
      "activations/layer23_attention_weight_min": -27.895835876464844,
      "activations/layer2_attention_weight_max": 31.11195182800293,
      "activations/layer2_attention_weight_min": -31.233539581298828,
      "activations/layer3_attention_weight_max": 88.14585876464844,
      "activations/layer3_attention_weight_min": -90.72750854492188,
      "activations/layer4_attention_weight_max": 112.35328674316406,
      "activations/layer4_attention_weight_min": -119.7945327758789,
      "activations/layer5_attention_weight_max": 50.35238265991211,
      "activations/layer5_attention_weight_min": -63.71455383300781,
      "activations/layer6_attention_weight_max": 46.067405700683594,
      "activations/layer6_attention_weight_min": -46.23649597167969,
      "activations/layer7_attention_weight_max": 84.51883697509766,
      "activations/layer7_attention_weight_min": -84.4296875,
      "activations/layer8_attention_weight_max": 37.646297454833984,
      "activations/layer8_attention_weight_min": -37.35469436645508,
      "activations/layer9_attention_weight_max": 31.80240821838379,
      "activations/layer9_attention_weight_min": -31.208585739135742,
      "epoch": 10.76,
      "learning_rate": 8.14621212121212e-05,
      "loss": 2.7758,
      "step": 185100
    },
    {
      "activations/layer0_attention_weight_max": 16.340442657470703,
      "activations/layer0_attention_weight_min": -13.45505142211914,
      "activations/layer10_attention_weight_max": 34.15735626220703,
      "activations/layer10_attention_weight_min": -32.91462707519531,
      "activations/layer11_attention_weight_max": 35.09443664550781,
      "activations/layer11_attention_weight_min": -32.37687683105469,
      "activations/layer12_attention_weight_max": 24.937820434570312,
      "activations/layer12_attention_weight_min": -27.881263732910156,
      "activations/layer13_attention_weight_max": 39.14501190185547,
      "activations/layer13_attention_weight_min": -35.73011016845703,
      "activations/layer14_attention_weight_max": 50.623226165771484,
      "activations/layer14_attention_weight_min": -40.211936950683594,
      "activations/layer15_attention_weight_max": 36.966678619384766,
      "activations/layer15_attention_weight_min": -32.3699951171875,
      "activations/layer16_attention_weight_max": 31.928327560424805,
      "activations/layer16_attention_weight_min": -28.292146682739258,
      "activations/layer17_attention_weight_max": 36.60255813598633,
      "activations/layer17_attention_weight_min": -25.837032318115234,
      "activations/layer18_attention_weight_max": 31.363605499267578,
      "activations/layer18_attention_weight_min": -24.622880935668945,
      "activations/layer19_attention_weight_max": 36.20341491699219,
      "activations/layer19_attention_weight_min": -29.368427276611328,
      "activations/layer1_attention_weight_max": 17.071475982666016,
      "activations/layer1_attention_weight_min": -16.435514450073242,
      "activations/layer20_attention_weight_max": 30.751625061035156,
      "activations/layer20_attention_weight_min": -23.533740997314453,
      "activations/layer21_attention_weight_max": 29.47832489013672,
      "activations/layer21_attention_weight_min": -23.089447021484375,
      "activations/layer22_attention_weight_max": 44.55445098876953,
      "activations/layer22_attention_weight_min": -29.76657485961914,
      "activations/layer23_attention_weight_max": 34.02143859863281,
      "activations/layer23_attention_weight_min": -25.52886962890625,
      "activations/layer2_attention_weight_max": 31.682415008544922,
      "activations/layer2_attention_weight_min": -31.37026596069336,
      "activations/layer3_attention_weight_max": 92.66525268554688,
      "activations/layer3_attention_weight_min": -93.96165466308594,
      "activations/layer4_attention_weight_max": 115.98546600341797,
      "activations/layer4_attention_weight_min": -111.20271301269531,
      "activations/layer5_attention_weight_max": 53.00936508178711,
      "activations/layer5_attention_weight_min": -62.19639205932617,
      "activations/layer6_attention_weight_max": 47.06758499145508,
      "activations/layer6_attention_weight_min": -48.68708419799805,
      "activations/layer7_attention_weight_max": 95.2957992553711,
      "activations/layer7_attention_weight_min": -90.99576568603516,
      "activations/layer8_attention_weight_max": 42.62272644042969,
      "activations/layer8_attention_weight_min": -43.00120544433594,
      "activations/layer9_attention_weight_max": 34.15808868408203,
      "activations/layer9_attention_weight_min": -34.34625244140625,
      "epoch": 10.76,
      "learning_rate": 8.144318181818181e-05,
      "loss": 2.7958,
      "step": 185150
    },
    {
      "activations/layer0_attention_weight_max": 16.319774627685547,
      "activations/layer0_attention_weight_min": -12.933256149291992,
      "activations/layer10_attention_weight_max": 38.81327819824219,
      "activations/layer10_attention_weight_min": -35.035682678222656,
      "activations/layer11_attention_weight_max": 35.24472427368164,
      "activations/layer11_attention_weight_min": -33.13374328613281,
      "activations/layer12_attention_weight_max": 28.50034523010254,
      "activations/layer12_attention_weight_min": -26.854663848876953,
      "activations/layer13_attention_weight_max": 47.99210739135742,
      "activations/layer13_attention_weight_min": -36.73188781738281,
      "activations/layer14_attention_weight_max": 53.91062545776367,
      "activations/layer14_attention_weight_min": -41.625885009765625,
      "activations/layer15_attention_weight_max": 48.027225494384766,
      "activations/layer15_attention_weight_min": -37.348365783691406,
      "activations/layer16_attention_weight_max": 39.062896728515625,
      "activations/layer16_attention_weight_min": -29.1720027923584,
      "activations/layer17_attention_weight_max": 40.740455627441406,
      "activations/layer17_attention_weight_min": -27.38438606262207,
      "activations/layer18_attention_weight_max": 33.4565315246582,
      "activations/layer18_attention_weight_min": -23.224767684936523,
      "activations/layer19_attention_weight_max": 43.81625747680664,
      "activations/layer19_attention_weight_min": -32.47119140625,
      "activations/layer1_attention_weight_max": 17.702957153320312,
      "activations/layer1_attention_weight_min": -14.036504745483398,
      "activations/layer20_attention_weight_max": 35.70579528808594,
      "activations/layer20_attention_weight_min": -24.943798065185547,
      "activations/layer21_attention_weight_max": 32.643192291259766,
      "activations/layer21_attention_weight_min": -23.549203872680664,
      "activations/layer22_attention_weight_max": 59.95184326171875,
      "activations/layer22_attention_weight_min": -29.726409912109375,
      "activations/layer23_attention_weight_max": 43.3722038269043,
      "activations/layer23_attention_weight_min": -25.795101165771484,
      "activations/layer2_attention_weight_max": 33.669437408447266,
      "activations/layer2_attention_weight_min": -33.6799430847168,
      "activations/layer3_attention_weight_max": 92.40052795410156,
      "activations/layer3_attention_weight_min": -96.79600524902344,
      "activations/layer4_attention_weight_max": 110.07369232177734,
      "activations/layer4_attention_weight_min": -117.1742935180664,
      "activations/layer5_attention_weight_max": 53.1246452331543,
      "activations/layer5_attention_weight_min": -61.66310119628906,
      "activations/layer6_attention_weight_max": 46.67567443847656,
      "activations/layer6_attention_weight_min": -53.04227066040039,
      "activations/layer7_attention_weight_max": 97.98192596435547,
      "activations/layer7_attention_weight_min": -92.87806701660156,
      "activations/layer8_attention_weight_max": 42.252166748046875,
      "activations/layer8_attention_weight_min": -43.72239303588867,
      "activations/layer9_attention_weight_max": 36.799293518066406,
      "activations/layer9_attention_weight_min": -37.104793548583984,
      "epoch": 10.76,
      "learning_rate": 8.142424242424242e-05,
      "loss": 2.7852,
      "step": 185200
    },
    {
      "activations/layer0_attention_weight_max": 16.60792350769043,
      "activations/layer0_attention_weight_min": -11.841562271118164,
      "activations/layer10_attention_weight_max": 30.253463745117188,
      "activations/layer10_attention_weight_min": -31.527915954589844,
      "activations/layer11_attention_weight_max": 30.518728256225586,
      "activations/layer11_attention_weight_min": -30.1641845703125,
      "activations/layer12_attention_weight_max": 29.260568618774414,
      "activations/layer12_attention_weight_min": -23.817630767822266,
      "activations/layer13_attention_weight_max": 43.17543411254883,
      "activations/layer13_attention_weight_min": -33.921173095703125,
      "activations/layer14_attention_weight_max": 53.21454620361328,
      "activations/layer14_attention_weight_min": -41.91259002685547,
      "activations/layer15_attention_weight_max": 36.14406204223633,
      "activations/layer15_attention_weight_min": -34.15967559814453,
      "activations/layer16_attention_weight_max": 34.70966339111328,
      "activations/layer16_attention_weight_min": -27.893421173095703,
      "activations/layer17_attention_weight_max": 33.80116271972656,
      "activations/layer17_attention_weight_min": -26.5781192779541,
      "activations/layer18_attention_weight_max": 36.06884002685547,
      "activations/layer18_attention_weight_min": -22.646968841552734,
      "activations/layer19_attention_weight_max": 37.23220443725586,
      "activations/layer19_attention_weight_min": -30.837247848510742,
      "activations/layer1_attention_weight_max": 16.925235748291016,
      "activations/layer1_attention_weight_min": -17.155052185058594,
      "activations/layer20_attention_weight_max": 31.188934326171875,
      "activations/layer20_attention_weight_min": -25.549165725708008,
      "activations/layer21_attention_weight_max": 31.187236785888672,
      "activations/layer21_attention_weight_min": -23.74884605407715,
      "activations/layer22_attention_weight_max": 53.86819839477539,
      "activations/layer22_attention_weight_min": -28.671831130981445,
      "activations/layer23_attention_weight_max": 33.379180908203125,
      "activations/layer23_attention_weight_min": -23.409812927246094,
      "activations/layer2_attention_weight_max": 32.366416931152344,
      "activations/layer2_attention_weight_min": -34.088680267333984,
      "activations/layer3_attention_weight_max": 93.44908905029297,
      "activations/layer3_attention_weight_min": -100.34358978271484,
      "activations/layer4_attention_weight_max": 111.052001953125,
      "activations/layer4_attention_weight_min": -113.18025970458984,
      "activations/layer5_attention_weight_max": 52.61322021484375,
      "activations/layer5_attention_weight_min": -59.61286544799805,
      "activations/layer6_attention_weight_max": 46.21620559692383,
      "activations/layer6_attention_weight_min": -49.077125549316406,
      "activations/layer7_attention_weight_max": 85.98932647705078,
      "activations/layer7_attention_weight_min": -92.5748519897461,
      "activations/layer8_attention_weight_max": 40.15431594848633,
      "activations/layer8_attention_weight_min": -41.91463088989258,
      "activations/layer9_attention_weight_max": 30.24470329284668,
      "activations/layer9_attention_weight_min": -31.630430221557617,
      "epoch": 10.76,
      "learning_rate": 8.140530303030302e-05,
      "loss": 2.7755,
      "step": 185250
    },
    {
      "activations/layer0_attention_weight_max": 17.30074691772461,
      "activations/layer0_attention_weight_min": -13.911798477172852,
      "activations/layer10_attention_weight_max": 33.1778450012207,
      "activations/layer10_attention_weight_min": -30.96732521057129,
      "activations/layer11_attention_weight_max": 34.504188537597656,
      "activations/layer11_attention_weight_min": -32.111820220947266,
      "activations/layer12_attention_weight_max": 26.253210067749023,
      "activations/layer12_attention_weight_min": -26.418262481689453,
      "activations/layer13_attention_weight_max": 44.03695297241211,
      "activations/layer13_attention_weight_min": -35.89441680908203,
      "activations/layer14_attention_weight_max": 47.04878234863281,
      "activations/layer14_attention_weight_min": -39.26527786254883,
      "activations/layer15_attention_weight_max": 38.53321838378906,
      "activations/layer15_attention_weight_min": -33.66473388671875,
      "activations/layer16_attention_weight_max": 32.63178634643555,
      "activations/layer16_attention_weight_min": -29.26606559753418,
      "activations/layer17_attention_weight_max": 31.42796516418457,
      "activations/layer17_attention_weight_min": -27.296659469604492,
      "activations/layer18_attention_weight_max": 30.95565414428711,
      "activations/layer18_attention_weight_min": -23.14578628540039,
      "activations/layer19_attention_weight_max": 39.7125244140625,
      "activations/layer19_attention_weight_min": -30.127084732055664,
      "activations/layer1_attention_weight_max": 15.71423625946045,
      "activations/layer1_attention_weight_min": -13.728660583496094,
      "activations/layer20_attention_weight_max": 31.71666145324707,
      "activations/layer20_attention_weight_min": -26.872913360595703,
      "activations/layer21_attention_weight_max": 30.579769134521484,
      "activations/layer21_attention_weight_min": -24.18246841430664,
      "activations/layer22_attention_weight_max": 50.62317657470703,
      "activations/layer22_attention_weight_min": -31.471452713012695,
      "activations/layer23_attention_weight_max": 42.45450973510742,
      "activations/layer23_attention_weight_min": -25.4753475189209,
      "activations/layer2_attention_weight_max": 32.39262390136719,
      "activations/layer2_attention_weight_min": -31.266080856323242,
      "activations/layer3_attention_weight_max": 92.31391143798828,
      "activations/layer3_attention_weight_min": -92.29398345947266,
      "activations/layer4_attention_weight_max": 113.13818359375,
      "activations/layer4_attention_weight_min": -108.76372528076172,
      "activations/layer5_attention_weight_max": 52.423370361328125,
      "activations/layer5_attention_weight_min": -60.94761657714844,
      "activations/layer6_attention_weight_max": 45.18363571166992,
      "activations/layer6_attention_weight_min": -48.17110061645508,
      "activations/layer7_attention_weight_max": 98.40058135986328,
      "activations/layer7_attention_weight_min": -88.38322448730469,
      "activations/layer8_attention_weight_max": 39.982521057128906,
      "activations/layer8_attention_weight_min": -40.23680114746094,
      "activations/layer9_attention_weight_max": 32.02463150024414,
      "activations/layer9_attention_weight_min": -31.32834815979004,
      "epoch": 10.77,
      "learning_rate": 8.138636363636364e-05,
      "loss": 2.7913,
      "step": 185300
    },
    {
      "activations/layer0_attention_weight_max": 16.03599739074707,
      "activations/layer0_attention_weight_min": -11.874678611755371,
      "activations/layer10_attention_weight_max": 40.08610916137695,
      "activations/layer10_attention_weight_min": -37.50186538696289,
      "activations/layer11_attention_weight_max": 42.66944885253906,
      "activations/layer11_attention_weight_min": -37.821449279785156,
      "activations/layer12_attention_weight_max": 41.58481979370117,
      "activations/layer12_attention_weight_min": -26.970518112182617,
      "activations/layer13_attention_weight_max": 49.680397033691406,
      "activations/layer13_attention_weight_min": -35.18011474609375,
      "activations/layer14_attention_weight_max": 47.03681182861328,
      "activations/layer14_attention_weight_min": -38.52861022949219,
      "activations/layer15_attention_weight_max": 52.59260559082031,
      "activations/layer15_attention_weight_min": -34.86140441894531,
      "activations/layer16_attention_weight_max": 35.54869079589844,
      "activations/layer16_attention_weight_min": -28.475723266601562,
      "activations/layer17_attention_weight_max": 34.02418899536133,
      "activations/layer17_attention_weight_min": -27.45233154296875,
      "activations/layer18_attention_weight_max": 33.024688720703125,
      "activations/layer18_attention_weight_min": -25.855920791625977,
      "activations/layer19_attention_weight_max": 40.588436126708984,
      "activations/layer19_attention_weight_min": -32.6554069519043,
      "activations/layer1_attention_weight_max": 17.67241668701172,
      "activations/layer1_attention_weight_min": -14.455135345458984,
      "activations/layer20_attention_weight_max": 35.896812438964844,
      "activations/layer20_attention_weight_min": -25.93532371520996,
      "activations/layer21_attention_weight_max": 34.8829345703125,
      "activations/layer21_attention_weight_min": -24.2467098236084,
      "activations/layer22_attention_weight_max": 51.169185638427734,
      "activations/layer22_attention_weight_min": -30.944557189941406,
      "activations/layer23_attention_weight_max": 36.869361877441406,
      "activations/layer23_attention_weight_min": -23.890335083007812,
      "activations/layer2_attention_weight_max": 37.664466857910156,
      "activations/layer2_attention_weight_min": -35.2696533203125,
      "activations/layer3_attention_weight_max": 101.67548370361328,
      "activations/layer3_attention_weight_min": -103.58790588378906,
      "activations/layer4_attention_weight_max": 121.49971008300781,
      "activations/layer4_attention_weight_min": -118.61150360107422,
      "activations/layer5_attention_weight_max": 56.60736083984375,
      "activations/layer5_attention_weight_min": -62.08634567260742,
      "activations/layer6_attention_weight_max": 46.62749481201172,
      "activations/layer6_attention_weight_min": -48.07551956176758,
      "activations/layer7_attention_weight_max": 104.5652084350586,
      "activations/layer7_attention_weight_min": -95.87653350830078,
      "activations/layer8_attention_weight_max": 47.28202819824219,
      "activations/layer8_attention_weight_min": -47.921722412109375,
      "activations/layer9_attention_weight_max": 37.862239837646484,
      "activations/layer9_attention_weight_min": -36.78878402709961,
      "epoch": 10.77,
      "learning_rate": 8.136742424242423e-05,
      "loss": 2.7705,
      "step": 185350
    },
    {
      "activations/layer0_attention_weight_max": 15.657567977905273,
      "activations/layer0_attention_weight_min": -13.591501235961914,
      "activations/layer10_attention_weight_max": 30.278533935546875,
      "activations/layer10_attention_weight_min": -31.329641342163086,
      "activations/layer11_attention_weight_max": 30.285497665405273,
      "activations/layer11_attention_weight_min": -29.638832092285156,
      "activations/layer12_attention_weight_max": 24.0096378326416,
      "activations/layer12_attention_weight_min": -25.005083084106445,
      "activations/layer13_attention_weight_max": 40.495994567871094,
      "activations/layer13_attention_weight_min": -31.938770294189453,
      "activations/layer14_attention_weight_max": 47.79471206665039,
      "activations/layer14_attention_weight_min": -39.606807708740234,
      "activations/layer15_attention_weight_max": 36.83163833618164,
      "activations/layer15_attention_weight_min": -31.908950805664062,
      "activations/layer16_attention_weight_max": 33.043270111083984,
      "activations/layer16_attention_weight_min": -26.678131103515625,
      "activations/layer17_attention_weight_max": 33.43999099731445,
      "activations/layer17_attention_weight_min": -25.688472747802734,
      "activations/layer18_attention_weight_max": 32.06856155395508,
      "activations/layer18_attention_weight_min": -23.772239685058594,
      "activations/layer19_attention_weight_max": 38.30536651611328,
      "activations/layer19_attention_weight_min": -32.69978713989258,
      "activations/layer1_attention_weight_max": 16.697711944580078,
      "activations/layer1_attention_weight_min": -14.5134859085083,
      "activations/layer20_attention_weight_max": 31.69384765625,
      "activations/layer20_attention_weight_min": -27.149169921875,
      "activations/layer21_attention_weight_max": 35.04652786254883,
      "activations/layer21_attention_weight_min": -24.897268295288086,
      "activations/layer22_attention_weight_max": 50.31314468383789,
      "activations/layer22_attention_weight_min": -32.298892974853516,
      "activations/layer23_attention_weight_max": 38.86496353149414,
      "activations/layer23_attention_weight_min": -26.38238525390625,
      "activations/layer2_attention_weight_max": 34.66728210449219,
      "activations/layer2_attention_weight_min": -33.549964904785156,
      "activations/layer3_attention_weight_max": 93.57284545898438,
      "activations/layer3_attention_weight_min": -98.35902404785156,
      "activations/layer4_attention_weight_max": 118.72222900390625,
      "activations/layer4_attention_weight_min": -110.02050018310547,
      "activations/layer5_attention_weight_max": 56.05591583251953,
      "activations/layer5_attention_weight_min": -61.51264572143555,
      "activations/layer6_attention_weight_max": 46.05498123168945,
      "activations/layer6_attention_weight_min": -46.27299118041992,
      "activations/layer7_attention_weight_max": 88.40985870361328,
      "activations/layer7_attention_weight_min": -85.41395568847656,
      "activations/layer8_attention_weight_max": 38.442779541015625,
      "activations/layer8_attention_weight_min": -40.23790740966797,
      "activations/layer9_attention_weight_max": 32.41062927246094,
      "activations/layer9_attention_weight_min": -32.396484375,
      "epoch": 10.77,
      "learning_rate": 8.134848484848484e-05,
      "loss": 2.7724,
      "step": 185400
    },
    {
      "activations/layer0_attention_weight_max": 15.984142303466797,
      "activations/layer0_attention_weight_min": -13.209005355834961,
      "activations/layer10_attention_weight_max": 34.54461669921875,
      "activations/layer10_attention_weight_min": -35.00623321533203,
      "activations/layer11_attention_weight_max": 34.155269622802734,
      "activations/layer11_attention_weight_min": -31.321142196655273,
      "activations/layer12_attention_weight_max": 25.57858657836914,
      "activations/layer12_attention_weight_min": -28.3253231048584,
      "activations/layer13_attention_weight_max": 42.631492614746094,
      "activations/layer13_attention_weight_min": -35.86634826660156,
      "activations/layer14_attention_weight_max": 44.87238311767578,
      "activations/layer14_attention_weight_min": -36.893306732177734,
      "activations/layer15_attention_weight_max": 38.28852462768555,
      "activations/layer15_attention_weight_min": -32.28221893310547,
      "activations/layer16_attention_weight_max": 33.86715316772461,
      "activations/layer16_attention_weight_min": -28.74131965637207,
      "activations/layer17_attention_weight_max": 33.8603401184082,
      "activations/layer17_attention_weight_min": -25.942895889282227,
      "activations/layer18_attention_weight_max": 31.951692581176758,
      "activations/layer18_attention_weight_min": -22.632957458496094,
      "activations/layer19_attention_weight_max": 36.868717193603516,
      "activations/layer19_attention_weight_min": -30.917205810546875,
      "activations/layer1_attention_weight_max": 15.646474838256836,
      "activations/layer1_attention_weight_min": -16.767831802368164,
      "activations/layer20_attention_weight_max": 31.647048950195312,
      "activations/layer20_attention_weight_min": -24.747526168823242,
      "activations/layer21_attention_weight_max": 30.52532196044922,
      "activations/layer21_attention_weight_min": -23.355606079101562,
      "activations/layer22_attention_weight_max": 49.462684631347656,
      "activations/layer22_attention_weight_min": -28.808856964111328,
      "activations/layer23_attention_weight_max": 41.03462219238281,
      "activations/layer23_attention_weight_min": -25.19997787475586,
      "activations/layer2_attention_weight_max": 32.126373291015625,
      "activations/layer2_attention_weight_min": -33.696128845214844,
      "activations/layer3_attention_weight_max": 97.4930419921875,
      "activations/layer3_attention_weight_min": -97.96836853027344,
      "activations/layer4_attention_weight_max": 117.47582244873047,
      "activations/layer4_attention_weight_min": -119.53580474853516,
      "activations/layer5_attention_weight_max": 52.85480880737305,
      "activations/layer5_attention_weight_min": -63.351402282714844,
      "activations/layer6_attention_weight_max": 45.32719039916992,
      "activations/layer6_attention_weight_min": -48.93185806274414,
      "activations/layer7_attention_weight_max": 93.92784881591797,
      "activations/layer7_attention_weight_min": -93.83723449707031,
      "activations/layer8_attention_weight_max": 41.379127502441406,
      "activations/layer8_attention_weight_min": -42.16765594482422,
      "activations/layer9_attention_weight_max": 33.99605941772461,
      "activations/layer9_attention_weight_min": -33.47093200683594,
      "epoch": 10.78,
      "learning_rate": 8.132954545454546e-05,
      "loss": 2.7937,
      "step": 185450
    },
    {
      "activations/layer0_attention_weight_max": 17.139989852905273,
      "activations/layer0_attention_weight_min": -13.840349197387695,
      "activations/layer10_attention_weight_max": 33.688941955566406,
      "activations/layer10_attention_weight_min": -30.889036178588867,
      "activations/layer11_attention_weight_max": 32.562068939208984,
      "activations/layer11_attention_weight_min": -30.705642700195312,
      "activations/layer12_attention_weight_max": 24.92693328857422,
      "activations/layer12_attention_weight_min": -25.274137496948242,
      "activations/layer13_attention_weight_max": 38.67051696777344,
      "activations/layer13_attention_weight_min": -32.85724639892578,
      "activations/layer14_attention_weight_max": 47.22682189941406,
      "activations/layer14_attention_weight_min": -39.321807861328125,
      "activations/layer15_attention_weight_max": 33.81507873535156,
      "activations/layer15_attention_weight_min": -31.663061141967773,
      "activations/layer16_attention_weight_max": 30.56340217590332,
      "activations/layer16_attention_weight_min": -27.5666446685791,
      "activations/layer17_attention_weight_max": 32.092403411865234,
      "activations/layer17_attention_weight_min": -25.487457275390625,
      "activations/layer18_attention_weight_max": 29.280996322631836,
      "activations/layer18_attention_weight_min": -21.641273498535156,
      "activations/layer19_attention_weight_max": 34.4355354309082,
      "activations/layer19_attention_weight_min": -32.983707427978516,
      "activations/layer1_attention_weight_max": 16.726415634155273,
      "activations/layer1_attention_weight_min": -14.51400089263916,
      "activations/layer20_attention_weight_max": 28.945838928222656,
      "activations/layer20_attention_weight_min": -26.51579475402832,
      "activations/layer21_attention_weight_max": 26.752410888671875,
      "activations/layer21_attention_weight_min": -23.08280372619629,
      "activations/layer22_attention_weight_max": 43.64344787597656,
      "activations/layer22_attention_weight_min": -30.969562530517578,
      "activations/layer23_attention_weight_max": 33.97577667236328,
      "activations/layer23_attention_weight_min": -23.70665740966797,
      "activations/layer2_attention_weight_max": 31.325946807861328,
      "activations/layer2_attention_weight_min": -32.05203628540039,
      "activations/layer3_attention_weight_max": 91.51237487792969,
      "activations/layer3_attention_weight_min": -92.92897033691406,
      "activations/layer4_attention_weight_max": 109.7197265625,
      "activations/layer4_attention_weight_min": -112.93062591552734,
      "activations/layer5_attention_weight_max": 50.48305892944336,
      "activations/layer5_attention_weight_min": -64.88949584960938,
      "activations/layer6_attention_weight_max": 45.69489288330078,
      "activations/layer6_attention_weight_min": -48.77560806274414,
      "activations/layer7_attention_weight_max": 96.7587890625,
      "activations/layer7_attention_weight_min": -90.88473510742188,
      "activations/layer8_attention_weight_max": 42.21581268310547,
      "activations/layer8_attention_weight_min": -41.802940368652344,
      "activations/layer9_attention_weight_max": 35.47359085083008,
      "activations/layer9_attention_weight_min": -35.44169235229492,
      "epoch": 10.78,
      "learning_rate": 8.131060606060604e-05,
      "loss": 2.7965,
      "step": 185500
    },
    {
      "activations/layer0_attention_weight_max": 16.141040802001953,
      "activations/layer0_attention_weight_min": -13.741764068603516,
      "activations/layer10_attention_weight_max": 33.24759292602539,
      "activations/layer10_attention_weight_min": -32.319129943847656,
      "activations/layer11_attention_weight_max": 30.99106216430664,
      "activations/layer11_attention_weight_min": -32.278682708740234,
      "activations/layer12_attention_weight_max": 25.580900192260742,
      "activations/layer12_attention_weight_min": -23.960309982299805,
      "activations/layer13_attention_weight_max": 37.86806106567383,
      "activations/layer13_attention_weight_min": -33.4981803894043,
      "activations/layer14_attention_weight_max": 44.314796447753906,
      "activations/layer14_attention_weight_min": -35.98363494873047,
      "activations/layer15_attention_weight_max": 35.53955841064453,
      "activations/layer15_attention_weight_min": -32.51731491088867,
      "activations/layer16_attention_weight_max": 33.726131439208984,
      "activations/layer16_attention_weight_min": -29.997455596923828,
      "activations/layer17_attention_weight_max": 29.710365295410156,
      "activations/layer17_attention_weight_min": -27.238811492919922,
      "activations/layer18_attention_weight_max": 32.84823226928711,
      "activations/layer18_attention_weight_min": -25.22605323791504,
      "activations/layer19_attention_weight_max": 35.51987838745117,
      "activations/layer19_attention_weight_min": -31.42209815979004,
      "activations/layer1_attention_weight_max": 18.034751892089844,
      "activations/layer1_attention_weight_min": -17.87874412536621,
      "activations/layer20_attention_weight_max": 30.521411895751953,
      "activations/layer20_attention_weight_min": -26.888076782226562,
      "activations/layer21_attention_weight_max": 30.731157302856445,
      "activations/layer21_attention_weight_min": -26.447216033935547,
      "activations/layer22_attention_weight_max": 47.067813873291016,
      "activations/layer22_attention_weight_min": -30.630653381347656,
      "activations/layer23_attention_weight_max": 35.715370178222656,
      "activations/layer23_attention_weight_min": -25.357717514038086,
      "activations/layer2_attention_weight_max": 32.21416473388672,
      "activations/layer2_attention_weight_min": -31.664810180664062,
      "activations/layer3_attention_weight_max": 91.06499481201172,
      "activations/layer3_attention_weight_min": -89.96455383300781,
      "activations/layer4_attention_weight_max": 113.45509338378906,
      "activations/layer4_attention_weight_min": -113.64717102050781,
      "activations/layer5_attention_weight_max": 53.62830352783203,
      "activations/layer5_attention_weight_min": -60.75022888183594,
      "activations/layer6_attention_weight_max": 45.10818862915039,
      "activations/layer6_attention_weight_min": -47.72837829589844,
      "activations/layer7_attention_weight_max": 93.9054183959961,
      "activations/layer7_attention_weight_min": -88.68358612060547,
      "activations/layer8_attention_weight_max": 39.782676696777344,
      "activations/layer8_attention_weight_min": -40.50676345825195,
      "activations/layer9_attention_weight_max": 32.308860778808594,
      "activations/layer9_attention_weight_min": -33.183406829833984,
      "epoch": 10.78,
      "learning_rate": 8.129166666666666e-05,
      "loss": 2.7918,
      "step": 185550
    },
    {
      "activations/layer0_attention_weight_max": 16.993894577026367,
      "activations/layer0_attention_weight_min": -13.578695297241211,
      "activations/layer10_attention_weight_max": 30.469985961914062,
      "activations/layer10_attention_weight_min": -30.740129470825195,
      "activations/layer11_attention_weight_max": 30.443614959716797,
      "activations/layer11_attention_weight_min": -30.75153350830078,
      "activations/layer12_attention_weight_max": 23.246824264526367,
      "activations/layer12_attention_weight_min": -24.485422134399414,
      "activations/layer13_attention_weight_max": 39.18787384033203,
      "activations/layer13_attention_weight_min": -32.5771369934082,
      "activations/layer14_attention_weight_max": 44.744850158691406,
      "activations/layer14_attention_weight_min": -36.55889892578125,
      "activations/layer15_attention_weight_max": 34.960609436035156,
      "activations/layer15_attention_weight_min": -30.33766746520996,
      "activations/layer16_attention_weight_max": 29.818986892700195,
      "activations/layer16_attention_weight_min": -29.219072341918945,
      "activations/layer17_attention_weight_max": 32.80287551879883,
      "activations/layer17_attention_weight_min": -25.785654067993164,
      "activations/layer18_attention_weight_max": 33.48690414428711,
      "activations/layer18_attention_weight_min": -24.023414611816406,
      "activations/layer19_attention_weight_max": 33.94974136352539,
      "activations/layer19_attention_weight_min": -33.323787689208984,
      "activations/layer1_attention_weight_max": 16.687435150146484,
      "activations/layer1_attention_weight_min": -14.715014457702637,
      "activations/layer20_attention_weight_max": 32.987117767333984,
      "activations/layer20_attention_weight_min": -26.42070960998535,
      "activations/layer21_attention_weight_max": 32.68680191040039,
      "activations/layer21_attention_weight_min": -25.70376968383789,
      "activations/layer22_attention_weight_max": 47.58589553833008,
      "activations/layer22_attention_weight_min": -32.32162857055664,
      "activations/layer23_attention_weight_max": 35.204551696777344,
      "activations/layer23_attention_weight_min": -26.156063079833984,
      "activations/layer2_attention_weight_max": 29.569454193115234,
      "activations/layer2_attention_weight_min": -30.005931854248047,
      "activations/layer3_attention_weight_max": 83.31981658935547,
      "activations/layer3_attention_weight_min": -84.17456817626953,
      "activations/layer4_attention_weight_max": 106.52547454833984,
      "activations/layer4_attention_weight_min": -113.57110595703125,
      "activations/layer5_attention_weight_max": 49.47648239135742,
      "activations/layer5_attention_weight_min": -65.29447174072266,
      "activations/layer6_attention_weight_max": 43.40021514892578,
      "activations/layer6_attention_weight_min": -45.94662857055664,
      "activations/layer7_attention_weight_max": 82.59693145751953,
      "activations/layer7_attention_weight_min": -85.48202514648438,
      "activations/layer8_attention_weight_max": 35.74740219116211,
      "activations/layer8_attention_weight_min": -38.02410888671875,
      "activations/layer9_attention_weight_max": 30.6180419921875,
      "activations/layer9_attention_weight_min": -31.780292510986328,
      "epoch": 10.78,
      "learning_rate": 8.127272727272727e-05,
      "loss": 2.7928,
      "step": 185600
    },
    {
      "activations/layer0_attention_weight_max": 15.651519775390625,
      "activations/layer0_attention_weight_min": -12.878713607788086,
      "activations/layer10_attention_weight_max": 31.51901626586914,
      "activations/layer10_attention_weight_min": -33.273277282714844,
      "activations/layer11_attention_weight_max": 30.96047019958496,
      "activations/layer11_attention_weight_min": -32.589454650878906,
      "activations/layer12_attention_weight_max": 25.315534591674805,
      "activations/layer12_attention_weight_min": -25.46768569946289,
      "activations/layer13_attention_weight_max": 39.22576141357422,
      "activations/layer13_attention_weight_min": -36.73972702026367,
      "activations/layer14_attention_weight_max": 42.96140670776367,
      "activations/layer14_attention_weight_min": -38.18290710449219,
      "activations/layer15_attention_weight_max": 35.26261520385742,
      "activations/layer15_attention_weight_min": -31.350250244140625,
      "activations/layer16_attention_weight_max": 32.90430450439453,
      "activations/layer16_attention_weight_min": -28.204864501953125,
      "activations/layer17_attention_weight_max": 32.21959686279297,
      "activations/layer17_attention_weight_min": -26.489669799804688,
      "activations/layer18_attention_weight_max": 33.410213470458984,
      "activations/layer18_attention_weight_min": -26.006418228149414,
      "activations/layer19_attention_weight_max": 37.86806106567383,
      "activations/layer19_attention_weight_min": -32.39656066894531,
      "activations/layer1_attention_weight_max": 16.803064346313477,
      "activations/layer1_attention_weight_min": -15.584233283996582,
      "activations/layer20_attention_weight_max": 32.59004592895508,
      "activations/layer20_attention_weight_min": -26.267276763916016,
      "activations/layer21_attention_weight_max": 33.92561721801758,
      "activations/layer21_attention_weight_min": -25.870708465576172,
      "activations/layer22_attention_weight_max": 51.476505279541016,
      "activations/layer22_attention_weight_min": -33.24899673461914,
      "activations/layer23_attention_weight_max": 34.668670654296875,
      "activations/layer23_attention_weight_min": -26.952964782714844,
      "activations/layer2_attention_weight_max": 34.19546890258789,
      "activations/layer2_attention_weight_min": -35.86778259277344,
      "activations/layer3_attention_weight_max": 98.18771362304688,
      "activations/layer3_attention_weight_min": -101.00350952148438,
      "activations/layer4_attention_weight_max": 122.5606460571289,
      "activations/layer4_attention_weight_min": -128.46641540527344,
      "activations/layer5_attention_weight_max": 56.000335693359375,
      "activations/layer5_attention_weight_min": -68.86180114746094,
      "activations/layer6_attention_weight_max": 47.72309112548828,
      "activations/layer6_attention_weight_min": -51.33965301513672,
      "activations/layer7_attention_weight_max": 92.37394714355469,
      "activations/layer7_attention_weight_min": -94.96003723144531,
      "activations/layer8_attention_weight_max": 40.693111419677734,
      "activations/layer8_attention_weight_min": -41.98590087890625,
      "activations/layer9_attention_weight_max": 32.391807556152344,
      "activations/layer9_attention_weight_min": -33.883056640625,
      "epoch": 10.79,
      "learning_rate": 8.125378787878786e-05,
      "loss": 2.7826,
      "step": 185650
    },
    {
      "activations/layer0_attention_weight_max": 15.413572311401367,
      "activations/layer0_attention_weight_min": -12.870264053344727,
      "activations/layer10_attention_weight_max": 33.213592529296875,
      "activations/layer10_attention_weight_min": -32.35307693481445,
      "activations/layer11_attention_weight_max": 32.707794189453125,
      "activations/layer11_attention_weight_min": -33.763832092285156,
      "activations/layer12_attention_weight_max": 36.885860443115234,
      "activations/layer12_attention_weight_min": -25.621761322021484,
      "activations/layer13_attention_weight_max": 40.550506591796875,
      "activations/layer13_attention_weight_min": -36.872676849365234,
      "activations/layer14_attention_weight_max": 41.91215896606445,
      "activations/layer14_attention_weight_min": -39.44005584716797,
      "activations/layer15_attention_weight_max": 35.96506881713867,
      "activations/layer15_attention_weight_min": -32.27824783325195,
      "activations/layer16_attention_weight_max": 31.276744842529297,
      "activations/layer16_attention_weight_min": -28.915422439575195,
      "activations/layer17_attention_weight_max": 31.918914794921875,
      "activations/layer17_attention_weight_min": -27.02012062072754,
      "activations/layer18_attention_weight_max": 32.2779655456543,
      "activations/layer18_attention_weight_min": -23.767553329467773,
      "activations/layer19_attention_weight_max": 34.34132385253906,
      "activations/layer19_attention_weight_min": -32.195072174072266,
      "activations/layer1_attention_weight_max": 16.82839012145996,
      "activations/layer1_attention_weight_min": -14.43820858001709,
      "activations/layer20_attention_weight_max": 29.082714080810547,
      "activations/layer20_attention_weight_min": -25.11193084716797,
      "activations/layer21_attention_weight_max": 30.03574562072754,
      "activations/layer21_attention_weight_min": -23.634233474731445,
      "activations/layer22_attention_weight_max": 46.24150466918945,
      "activations/layer22_attention_weight_min": -29.83665657043457,
      "activations/layer23_attention_weight_max": 42.74901580810547,
      "activations/layer23_attention_weight_min": -25.09083366394043,
      "activations/layer2_attention_weight_max": 29.94723129272461,
      "activations/layer2_attention_weight_min": -29.251890182495117,
      "activations/layer3_attention_weight_max": 89.91138458251953,
      "activations/layer3_attention_weight_min": -90.38200378417969,
      "activations/layer4_attention_weight_max": 112.75453186035156,
      "activations/layer4_attention_weight_min": -115.8923568725586,
      "activations/layer5_attention_weight_max": 52.84893798828125,
      "activations/layer5_attention_weight_min": -63.040252685546875,
      "activations/layer6_attention_weight_max": 42.7802848815918,
      "activations/layer6_attention_weight_min": -47.5711555480957,
      "activations/layer7_attention_weight_max": 86.04574584960938,
      "activations/layer7_attention_weight_min": -93.95774841308594,
      "activations/layer8_attention_weight_max": 39.593414306640625,
      "activations/layer8_attention_weight_min": -42.35737991333008,
      "activations/layer9_attention_weight_max": 34.101715087890625,
      "activations/layer9_attention_weight_min": -33.400394439697266,
      "epoch": 10.79,
      "learning_rate": 8.123484848484848e-05,
      "loss": 2.7716,
      "step": 185700
    },
    {
      "activations/layer0_attention_weight_max": 16.027360916137695,
      "activations/layer0_attention_weight_min": -13.256382942199707,
      "activations/layer10_attention_weight_max": 31.407011032104492,
      "activations/layer10_attention_weight_min": -32.05093765258789,
      "activations/layer11_attention_weight_max": 31.34180450439453,
      "activations/layer11_attention_weight_min": -29.818470001220703,
      "activations/layer12_attention_weight_max": 24.096147537231445,
      "activations/layer12_attention_weight_min": -24.54654312133789,
      "activations/layer13_attention_weight_max": 39.488502502441406,
      "activations/layer13_attention_weight_min": -34.104122161865234,
      "activations/layer14_attention_weight_max": 44.378421783447266,
      "activations/layer14_attention_weight_min": -37.87992477416992,
      "activations/layer15_attention_weight_max": 38.841163635253906,
      "activations/layer15_attention_weight_min": -31.03404426574707,
      "activations/layer16_attention_weight_max": 30.966617584228516,
      "activations/layer16_attention_weight_min": -28.58551025390625,
      "activations/layer17_attention_weight_max": 31.33406639099121,
      "activations/layer17_attention_weight_min": -25.605897903442383,
      "activations/layer18_attention_weight_max": 31.38518714904785,
      "activations/layer18_attention_weight_min": -23.190462112426758,
      "activations/layer19_attention_weight_max": 33.31117248535156,
      "activations/layer19_attention_weight_min": -31.584775924682617,
      "activations/layer1_attention_weight_max": 17.159151077270508,
      "activations/layer1_attention_weight_min": -14.726866722106934,
      "activations/layer20_attention_weight_max": 28.40612030029297,
      "activations/layer20_attention_weight_min": -24.318960189819336,
      "activations/layer21_attention_weight_max": 30.489286422729492,
      "activations/layer21_attention_weight_min": -22.580020904541016,
      "activations/layer22_attention_weight_max": 44.31919479370117,
      "activations/layer22_attention_weight_min": -29.49251937866211,
      "activations/layer23_attention_weight_max": 33.29986572265625,
      "activations/layer23_attention_weight_min": -25.29828453063965,
      "activations/layer2_attention_weight_max": 31.300230026245117,
      "activations/layer2_attention_weight_min": -31.01613998413086,
      "activations/layer3_attention_weight_max": 91.71454620361328,
      "activations/layer3_attention_weight_min": -91.2109146118164,
      "activations/layer4_attention_weight_max": 116.49893951416016,
      "activations/layer4_attention_weight_min": -114.76458740234375,
      "activations/layer5_attention_weight_max": 53.587196350097656,
      "activations/layer5_attention_weight_min": -63.142784118652344,
      "activations/layer6_attention_weight_max": 45.33201217651367,
      "activations/layer6_attention_weight_min": -50.67057418823242,
      "activations/layer7_attention_weight_max": 92.92483520507812,
      "activations/layer7_attention_weight_min": -89.21681213378906,
      "activations/layer8_attention_weight_max": 39.78971862792969,
      "activations/layer8_attention_weight_min": -40.055233001708984,
      "activations/layer9_attention_weight_max": 35.17318344116211,
      "activations/layer9_attention_weight_min": -31.96912956237793,
      "epoch": 10.79,
      "learning_rate": 8.121590909090908e-05,
      "loss": 2.7826,
      "step": 185750
    },
    {
      "activations/layer0_attention_weight_max": 16.250385284423828,
      "activations/layer0_attention_weight_min": -12.866364479064941,
      "activations/layer10_attention_weight_max": 30.951589584350586,
      "activations/layer10_attention_weight_min": -31.770957946777344,
      "activations/layer11_attention_weight_max": 31.48909568786621,
      "activations/layer11_attention_weight_min": -31.858356475830078,
      "activations/layer12_attention_weight_max": 24.417572021484375,
      "activations/layer12_attention_weight_min": -25.616931915283203,
      "activations/layer13_attention_weight_max": 35.98046875,
      "activations/layer13_attention_weight_min": -35.337181091308594,
      "activations/layer14_attention_weight_max": 43.91053009033203,
      "activations/layer14_attention_weight_min": -40.66044235229492,
      "activations/layer15_attention_weight_max": 31.936586380004883,
      "activations/layer15_attention_weight_min": -30.614458084106445,
      "activations/layer16_attention_weight_max": 29.054645538330078,
      "activations/layer16_attention_weight_min": -28.809146881103516,
      "activations/layer17_attention_weight_max": 29.221967697143555,
      "activations/layer17_attention_weight_min": -25.356372833251953,
      "activations/layer18_attention_weight_max": 29.180673599243164,
      "activations/layer18_attention_weight_min": -22.512563705444336,
      "activations/layer19_attention_weight_max": 34.63082504272461,
      "activations/layer19_attention_weight_min": -30.796218872070312,
      "activations/layer1_attention_weight_max": 17.071184158325195,
      "activations/layer1_attention_weight_min": -15.435922622680664,
      "activations/layer20_attention_weight_max": 30.440406799316406,
      "activations/layer20_attention_weight_min": -25.516450881958008,
      "activations/layer21_attention_weight_max": 29.653417587280273,
      "activations/layer21_attention_weight_min": -25.682861328125,
      "activations/layer22_attention_weight_max": 45.04053497314453,
      "activations/layer22_attention_weight_min": -28.354248046875,
      "activations/layer23_attention_weight_max": 33.248443603515625,
      "activations/layer23_attention_weight_min": -23.962339401245117,
      "activations/layer2_attention_weight_max": 30.530017852783203,
      "activations/layer2_attention_weight_min": -31.609970092773438,
      "activations/layer3_attention_weight_max": 87.12873077392578,
      "activations/layer3_attention_weight_min": -88.47533416748047,
      "activations/layer4_attention_weight_max": 109.08190155029297,
      "activations/layer4_attention_weight_min": -108.17720031738281,
      "activations/layer5_attention_weight_max": 48.850685119628906,
      "activations/layer5_attention_weight_min": -63.42767333984375,
      "activations/layer6_attention_weight_max": 43.88533020019531,
      "activations/layer6_attention_weight_min": -48.029319763183594,
      "activations/layer7_attention_weight_max": 86.07257080078125,
      "activations/layer7_attention_weight_min": -89.8076171875,
      "activations/layer8_attention_weight_max": 39.94490432739258,
      "activations/layer8_attention_weight_min": -40.60414123535156,
      "activations/layer9_attention_weight_max": 35.586639404296875,
      "activations/layer9_attention_weight_min": -34.54096603393555,
      "epoch": 10.8,
      "learning_rate": 8.11969696969697e-05,
      "loss": 2.7843,
      "step": 185800
    },
    {
      "activations/layer0_attention_weight_max": 16.634387969970703,
      "activations/layer0_attention_weight_min": -13.725323677062988,
      "activations/layer10_attention_weight_max": 34.26154327392578,
      "activations/layer10_attention_weight_min": -32.01618576049805,
      "activations/layer11_attention_weight_max": 32.90865707397461,
      "activations/layer11_attention_weight_min": -31.086841583251953,
      "activations/layer12_attention_weight_max": 23.093379974365234,
      "activations/layer12_attention_weight_min": -24.918527603149414,
      "activations/layer13_attention_weight_max": 36.97646713256836,
      "activations/layer13_attention_weight_min": -36.46515655517578,
      "activations/layer14_attention_weight_max": 43.03142166137695,
      "activations/layer14_attention_weight_min": -40.0538444519043,
      "activations/layer15_attention_weight_max": 34.35403060913086,
      "activations/layer15_attention_weight_min": -32.75604248046875,
      "activations/layer16_attention_weight_max": 33.98434829711914,
      "activations/layer16_attention_weight_min": -28.532329559326172,
      "activations/layer17_attention_weight_max": 32.15611267089844,
      "activations/layer17_attention_weight_min": -25.16850471496582,
      "activations/layer18_attention_weight_max": 32.903663635253906,
      "activations/layer18_attention_weight_min": -25.462419509887695,
      "activations/layer19_attention_weight_max": 39.992645263671875,
      "activations/layer19_attention_weight_min": -30.136316299438477,
      "activations/layer1_attention_weight_max": 17.118000030517578,
      "activations/layer1_attention_weight_min": -14.667075157165527,
      "activations/layer20_attention_weight_max": 32.26902770996094,
      "activations/layer20_attention_weight_min": -25.42589569091797,
      "activations/layer21_attention_weight_max": 29.32699966430664,
      "activations/layer21_attention_weight_min": -23.26043701171875,
      "activations/layer22_attention_weight_max": 48.050167083740234,
      "activations/layer22_attention_weight_min": -29.61269187927246,
      "activations/layer23_attention_weight_max": 38.19804382324219,
      "activations/layer23_attention_weight_min": -24.612882614135742,
      "activations/layer2_attention_weight_max": 29.814329147338867,
      "activations/layer2_attention_weight_min": -30.660907745361328,
      "activations/layer3_attention_weight_max": 89.17181396484375,
      "activations/layer3_attention_weight_min": -91.65615844726562,
      "activations/layer4_attention_weight_max": 107.8869857788086,
      "activations/layer4_attention_weight_min": -111.46832275390625,
      "activations/layer5_attention_weight_max": 51.362327575683594,
      "activations/layer5_attention_weight_min": -60.85466003417969,
      "activations/layer6_attention_weight_max": 44.47422790527344,
      "activations/layer6_attention_weight_min": -46.53845977783203,
      "activations/layer7_attention_weight_max": 90.56722259521484,
      "activations/layer7_attention_weight_min": -93.89254760742188,
      "activations/layer8_attention_weight_max": 41.172691345214844,
      "activations/layer8_attention_weight_min": -42.17729568481445,
      "activations/layer9_attention_weight_max": 34.68717956542969,
      "activations/layer9_attention_weight_min": -33.57159423828125,
      "epoch": 10.8,
      "learning_rate": 8.11780303030303e-05,
      "loss": 2.7789,
      "step": 185850
    },
    {
      "activations/layer0_attention_weight_max": 14.723649978637695,
      "activations/layer0_attention_weight_min": -12.460831642150879,
      "activations/layer10_attention_weight_max": 34.385475158691406,
      "activations/layer10_attention_weight_min": -32.278968811035156,
      "activations/layer11_attention_weight_max": 34.3135986328125,
      "activations/layer11_attention_weight_min": -32.42827224731445,
      "activations/layer12_attention_weight_max": 26.11481285095215,
      "activations/layer12_attention_weight_min": -24.496702194213867,
      "activations/layer13_attention_weight_max": 41.27529525756836,
      "activations/layer13_attention_weight_min": -35.0242919921875,
      "activations/layer14_attention_weight_max": 50.28042221069336,
      "activations/layer14_attention_weight_min": -40.54511260986328,
      "activations/layer15_attention_weight_max": 37.083255767822266,
      "activations/layer15_attention_weight_min": -32.01764678955078,
      "activations/layer16_attention_weight_max": 32.989830017089844,
      "activations/layer16_attention_weight_min": -27.126537322998047,
      "activations/layer17_attention_weight_max": 31.81048583984375,
      "activations/layer17_attention_weight_min": -26.132904052734375,
      "activations/layer18_attention_weight_max": 31.885046005249023,
      "activations/layer18_attention_weight_min": -21.687211990356445,
      "activations/layer19_attention_weight_max": 40.93720626831055,
      "activations/layer19_attention_weight_min": -30.648710250854492,
      "activations/layer1_attention_weight_max": 17.09242057800293,
      "activations/layer1_attention_weight_min": -14.49688720703125,
      "activations/layer20_attention_weight_max": 34.6307258605957,
      "activations/layer20_attention_weight_min": -23.567529678344727,
      "activations/layer21_attention_weight_max": 30.65118980407715,
      "activations/layer21_attention_weight_min": -23.23166847229004,
      "activations/layer22_attention_weight_max": 47.50273513793945,
      "activations/layer22_attention_weight_min": -28.55105972290039,
      "activations/layer23_attention_weight_max": 38.88081359863281,
      "activations/layer23_attention_weight_min": -23.833847045898438,
      "activations/layer2_attention_weight_max": 29.67024803161621,
      "activations/layer2_attention_weight_min": -32.702735900878906,
      "activations/layer3_attention_weight_max": 88.68714141845703,
      "activations/layer3_attention_weight_min": -94.56571197509766,
      "activations/layer4_attention_weight_max": 109.9664077758789,
      "activations/layer4_attention_weight_min": -113.6637191772461,
      "activations/layer5_attention_weight_max": 53.30729293823242,
      "activations/layer5_attention_weight_min": -61.15688705444336,
      "activations/layer6_attention_weight_max": 45.11629104614258,
      "activations/layer6_attention_weight_min": -50.687068939208984,
      "activations/layer7_attention_weight_max": 95.08035278320312,
      "activations/layer7_attention_weight_min": -88.55105590820312,
      "activations/layer8_attention_weight_max": 40.12126159667969,
      "activations/layer8_attention_weight_min": -40.70053482055664,
      "activations/layer9_attention_weight_max": 36.16881561279297,
      "activations/layer9_attention_weight_min": -33.783355712890625,
      "epoch": 10.8,
      "learning_rate": 8.11590909090909e-05,
      "loss": 2.7773,
      "step": 185900
    },
    {
      "activations/layer0_attention_weight_max": 15.911358833312988,
      "activations/layer0_attention_weight_min": -11.86020565032959,
      "activations/layer10_attention_weight_max": 33.570220947265625,
      "activations/layer10_attention_weight_min": -32.397125244140625,
      "activations/layer11_attention_weight_max": 30.914371490478516,
      "activations/layer11_attention_weight_min": -31.65118408203125,
      "activations/layer12_attention_weight_max": 23.26537322998047,
      "activations/layer12_attention_weight_min": -25.73589515686035,
      "activations/layer13_attention_weight_max": 37.056236267089844,
      "activations/layer13_attention_weight_min": -34.32457733154297,
      "activations/layer14_attention_weight_max": 43.12255859375,
      "activations/layer14_attention_weight_min": -37.400611877441406,
      "activations/layer15_attention_weight_max": 37.5630989074707,
      "activations/layer15_attention_weight_min": -33.05767059326172,
      "activations/layer16_attention_weight_max": 32.25899887084961,
      "activations/layer16_attention_weight_min": -28.746992111206055,
      "activations/layer17_attention_weight_max": 31.37361717224121,
      "activations/layer17_attention_weight_min": -25.372386932373047,
      "activations/layer18_attention_weight_max": 28.204504013061523,
      "activations/layer18_attention_weight_min": -23.11946678161621,
      "activations/layer19_attention_weight_max": 34.198123931884766,
      "activations/layer19_attention_weight_min": -30.890954971313477,
      "activations/layer1_attention_weight_max": 17.33888816833496,
      "activations/layer1_attention_weight_min": -15.475286483764648,
      "activations/layer20_attention_weight_max": 27.67803192138672,
      "activations/layer20_attention_weight_min": -25.058420181274414,
      "activations/layer21_attention_weight_max": 28.234128952026367,
      "activations/layer21_attention_weight_min": -26.872966766357422,
      "activations/layer22_attention_weight_max": 45.411781311035156,
      "activations/layer22_attention_weight_min": -33.45505142211914,
      "activations/layer23_attention_weight_max": 34.058528900146484,
      "activations/layer23_attention_weight_min": -25.56789779663086,
      "activations/layer2_attention_weight_max": 32.121337890625,
      "activations/layer2_attention_weight_min": -30.791074752807617,
      "activations/layer3_attention_weight_max": 91.28614044189453,
      "activations/layer3_attention_weight_min": -91.472900390625,
      "activations/layer4_attention_weight_max": 112.89044189453125,
      "activations/layer4_attention_weight_min": -116.97145080566406,
      "activations/layer5_attention_weight_max": 52.769874572753906,
      "activations/layer5_attention_weight_min": -62.23257064819336,
      "activations/layer6_attention_weight_max": 43.67559051513672,
      "activations/layer6_attention_weight_min": -47.97191619873047,
      "activations/layer7_attention_weight_max": 87.48303985595703,
      "activations/layer7_attention_weight_min": -90.21272277832031,
      "activations/layer8_attention_weight_max": 38.258026123046875,
      "activations/layer8_attention_weight_min": -39.02387237548828,
      "activations/layer9_attention_weight_max": 33.497581481933594,
      "activations/layer9_attention_weight_min": -32.79106140136719,
      "epoch": 10.8,
      "learning_rate": 8.114015151515151e-05,
      "loss": 2.794,
      "step": 185950
    },
    {
      "activations/layer0_attention_weight_max": 15.795541763305664,
      "activations/layer0_attention_weight_min": -13.81554126739502,
      "activations/layer10_attention_weight_max": 31.495525360107422,
      "activations/layer10_attention_weight_min": -31.798137664794922,
      "activations/layer11_attention_weight_max": 31.10688018798828,
      "activations/layer11_attention_weight_min": -30.775135040283203,
      "activations/layer12_attention_weight_max": 25.59518051147461,
      "activations/layer12_attention_weight_min": -25.115440368652344,
      "activations/layer13_attention_weight_max": 38.3092155456543,
      "activations/layer13_attention_weight_min": -33.53716278076172,
      "activations/layer14_attention_weight_max": 47.37721633911133,
      "activations/layer14_attention_weight_min": -36.591522216796875,
      "activations/layer15_attention_weight_max": 35.5181999206543,
      "activations/layer15_attention_weight_min": -30.044639587402344,
      "activations/layer16_attention_weight_max": 32.051639556884766,
      "activations/layer16_attention_weight_min": -27.923730850219727,
      "activations/layer17_attention_weight_max": 31.288875579833984,
      "activations/layer17_attention_weight_min": -25.333633422851562,
      "activations/layer18_attention_weight_max": 30.978952407836914,
      "activations/layer18_attention_weight_min": -23.981121063232422,
      "activations/layer19_attention_weight_max": 38.132537841796875,
      "activations/layer19_attention_weight_min": -30.999204635620117,
      "activations/layer1_attention_weight_max": 16.61941909790039,
      "activations/layer1_attention_weight_min": -16.54096794128418,
      "activations/layer20_attention_weight_max": 33.98637390136719,
      "activations/layer20_attention_weight_min": -25.10430145263672,
      "activations/layer21_attention_weight_max": 32.78839874267578,
      "activations/layer21_attention_weight_min": -24.663053512573242,
      "activations/layer22_attention_weight_max": 47.24778366088867,
      "activations/layer22_attention_weight_min": -32.38106155395508,
      "activations/layer23_attention_weight_max": 39.21372604370117,
      "activations/layer23_attention_weight_min": -26.849990844726562,
      "activations/layer2_attention_weight_max": 32.942100524902344,
      "activations/layer2_attention_weight_min": -32.471458435058594,
      "activations/layer3_attention_weight_max": 93.87010955810547,
      "activations/layer3_attention_weight_min": -93.1522216796875,
      "activations/layer4_attention_weight_max": 113.21685028076172,
      "activations/layer4_attention_weight_min": -113.17213439941406,
      "activations/layer5_attention_weight_max": 52.714439392089844,
      "activations/layer5_attention_weight_min": -66.44873046875,
      "activations/layer6_attention_weight_max": 44.02406692504883,
      "activations/layer6_attention_weight_min": -49.11992263793945,
      "activations/layer7_attention_weight_max": 88.55255889892578,
      "activations/layer7_attention_weight_min": -86.21747589111328,
      "activations/layer8_attention_weight_max": 39.447628021240234,
      "activations/layer8_attention_weight_min": -39.38554382324219,
      "activations/layer9_attention_weight_max": 32.36349868774414,
      "activations/layer9_attention_weight_min": -32.29667282104492,
      "epoch": 10.81,
      "learning_rate": 8.112121212121211e-05,
      "loss": 2.7773,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.5261,
      "eval_samples_per_second": 503.628,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.5261,
      "eval_openwebtext_samples_per_second": 503.628,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0351,
      "eval_wikitext_samples_per_second": 224.072,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.6239,
      "eval_lambada_samples_per_second": 505.927,
      "step": 186000
    },
    {
      "activations/layer0_attention_weight_max": 15.794360160827637,
      "activations/layer0_attention_weight_min": -13.434160232543945,
      "activations/layer10_attention_weight_max": 31.389781951904297,
      "activations/layer10_attention_weight_min": -32.76178741455078,
      "activations/layer11_attention_weight_max": 31.644447326660156,
      "activations/layer11_attention_weight_min": -30.692920684814453,
      "activations/layer12_attention_weight_max": 32.38011932373047,
      "activations/layer12_attention_weight_min": -27.333261489868164,
      "activations/layer13_attention_weight_max": 40.614479064941406,
      "activations/layer13_attention_weight_min": -34.543487548828125,
      "activations/layer14_attention_weight_max": 48.77779769897461,
      "activations/layer14_attention_weight_min": -39.1016731262207,
      "activations/layer15_attention_weight_max": 36.52743148803711,
      "activations/layer15_attention_weight_min": -31.42164421081543,
      "activations/layer16_attention_weight_max": 33.5954475402832,
      "activations/layer16_attention_weight_min": -29.49472999572754,
      "activations/layer17_attention_weight_max": 29.679019927978516,
      "activations/layer17_attention_weight_min": -25.087486267089844,
      "activations/layer18_attention_weight_max": 30.680644989013672,
      "activations/layer18_attention_weight_min": -23.442874908447266,
      "activations/layer19_attention_weight_max": 34.09808349609375,
      "activations/layer19_attention_weight_min": -31.80887222290039,
      "activations/layer1_attention_weight_max": 17.391565322875977,
      "activations/layer1_attention_weight_min": -14.643023490905762,
      "activations/layer20_attention_weight_max": 28.277496337890625,
      "activations/layer20_attention_weight_min": -24.447660446166992,
      "activations/layer21_attention_weight_max": 29.686662673950195,
      "activations/layer21_attention_weight_min": -24.433292388916016,
      "activations/layer22_attention_weight_max": 45.877197265625,
      "activations/layer22_attention_weight_min": -29.49238395690918,
      "activations/layer23_attention_weight_max": 34.846946716308594,
      "activations/layer23_attention_weight_min": -27.508853912353516,
      "activations/layer2_attention_weight_max": 30.459957122802734,
      "activations/layer2_attention_weight_min": -30.02651023864746,
      "activations/layer3_attention_weight_max": 86.95291900634766,
      "activations/layer3_attention_weight_min": -89.131591796875,
      "activations/layer4_attention_weight_max": 105.98909759521484,
      "activations/layer4_attention_weight_min": -107.68670654296875,
      "activations/layer5_attention_weight_max": 49.436134338378906,
      "activations/layer5_attention_weight_min": -64.51551818847656,
      "activations/layer6_attention_weight_max": 44.837520599365234,
      "activations/layer6_attention_weight_min": -45.10037612915039,
      "activations/layer7_attention_weight_max": 90.11951446533203,
      "activations/layer7_attention_weight_min": -89.90338134765625,
      "activations/layer8_attention_weight_max": 39.48698806762695,
      "activations/layer8_attention_weight_min": -40.563236236572266,
      "activations/layer9_attention_weight_max": 32.25237274169922,
      "activations/layer9_attention_weight_min": -32.670562744140625,
      "epoch": 10.81,
      "learning_rate": 8.110227272727272e-05,
      "loss": 2.7952,
      "step": 186050
    },
    {
      "activations/layer0_attention_weight_max": 14.835846900939941,
      "activations/layer0_attention_weight_min": -13.1039400100708,
      "activations/layer10_attention_weight_max": 31.41575813293457,
      "activations/layer10_attention_weight_min": -32.00400924682617,
      "activations/layer11_attention_weight_max": 31.093494415283203,
      "activations/layer11_attention_weight_min": -30.478559494018555,
      "activations/layer12_attention_weight_max": 24.23731231689453,
      "activations/layer12_attention_weight_min": -23.991315841674805,
      "activations/layer13_attention_weight_max": 41.12824630737305,
      "activations/layer13_attention_weight_min": -32.183834075927734,
      "activations/layer14_attention_weight_max": 47.07324981689453,
      "activations/layer14_attention_weight_min": -36.64216232299805,
      "activations/layer15_attention_weight_max": 36.30429458618164,
      "activations/layer15_attention_weight_min": -31.356853485107422,
      "activations/layer16_attention_weight_max": 32.07523727416992,
      "activations/layer16_attention_weight_min": -27.647581100463867,
      "activations/layer17_attention_weight_max": 30.238306045532227,
      "activations/layer17_attention_weight_min": -25.3098201751709,
      "activations/layer18_attention_weight_max": 31.317358016967773,
      "activations/layer18_attention_weight_min": -24.467769622802734,
      "activations/layer19_attention_weight_max": 34.890933990478516,
      "activations/layer19_attention_weight_min": -32.38190841674805,
      "activations/layer1_attention_weight_max": 16.93499183654785,
      "activations/layer1_attention_weight_min": -15.699238777160645,
      "activations/layer20_attention_weight_max": 30.945457458496094,
      "activations/layer20_attention_weight_min": -25.670982360839844,
      "activations/layer21_attention_weight_max": 29.994277954101562,
      "activations/layer21_attention_weight_min": -25.550765991210938,
      "activations/layer22_attention_weight_max": 45.53818893432617,
      "activations/layer22_attention_weight_min": -31.233600616455078,
      "activations/layer23_attention_weight_max": 34.091552734375,
      "activations/layer23_attention_weight_min": -25.860517501831055,
      "activations/layer2_attention_weight_max": 32.192283630371094,
      "activations/layer2_attention_weight_min": -31.34131622314453,
      "activations/layer3_attention_weight_max": 96.76773834228516,
      "activations/layer3_attention_weight_min": -92.25348663330078,
      "activations/layer4_attention_weight_max": 116.14879608154297,
      "activations/layer4_attention_weight_min": -111.08068084716797,
      "activations/layer5_attention_weight_max": 56.04365539550781,
      "activations/layer5_attention_weight_min": -59.21953582763672,
      "activations/layer6_attention_weight_max": 47.56971740722656,
      "activations/layer6_attention_weight_min": -48.588951110839844,
      "activations/layer7_attention_weight_max": 91.96526336669922,
      "activations/layer7_attention_weight_min": -88.41851806640625,
      "activations/layer8_attention_weight_max": 40.90558624267578,
      "activations/layer8_attention_weight_min": -40.26868438720703,
      "activations/layer9_attention_weight_max": 32.46940231323242,
      "activations/layer9_attention_weight_min": -32.64221954345703,
      "epoch": 10.81,
      "learning_rate": 8.108333333333333e-05,
      "loss": 2.7668,
      "step": 186100
    },
    {
      "activations/layer0_attention_weight_max": 16.865631103515625,
      "activations/layer0_attention_weight_min": -12.64493465423584,
      "activations/layer10_attention_weight_max": 32.03255844116211,
      "activations/layer10_attention_weight_min": -32.23933792114258,
      "activations/layer11_attention_weight_max": 32.692108154296875,
      "activations/layer11_attention_weight_min": -29.757347106933594,
      "activations/layer12_attention_weight_max": 23.715553283691406,
      "activations/layer12_attention_weight_min": -27.231294631958008,
      "activations/layer13_attention_weight_max": 38.00187301635742,
      "activations/layer13_attention_weight_min": -36.85622787475586,
      "activations/layer14_attention_weight_max": 47.2101936340332,
      "activations/layer14_attention_weight_min": -40.825584411621094,
      "activations/layer15_attention_weight_max": 36.56373977661133,
      "activations/layer15_attention_weight_min": -34.02630615234375,
      "activations/layer16_attention_weight_max": 29.393234252929688,
      "activations/layer16_attention_weight_min": -27.552366256713867,
      "activations/layer17_attention_weight_max": 32.48149490356445,
      "activations/layer17_attention_weight_min": -27.17618751525879,
      "activations/layer18_attention_weight_max": 32.29878616333008,
      "activations/layer18_attention_weight_min": -23.182157516479492,
      "activations/layer19_attention_weight_max": 34.8953971862793,
      "activations/layer19_attention_weight_min": -30.069067001342773,
      "activations/layer1_attention_weight_max": 17.076595306396484,
      "activations/layer1_attention_weight_min": -14.176200866699219,
      "activations/layer20_attention_weight_max": 29.278947830200195,
      "activations/layer20_attention_weight_min": -23.162473678588867,
      "activations/layer21_attention_weight_max": 26.636783599853516,
      "activations/layer21_attention_weight_min": -23.412994384765625,
      "activations/layer22_attention_weight_max": 44.02627944946289,
      "activations/layer22_attention_weight_min": -27.304475784301758,
      "activations/layer23_attention_weight_max": 35.0674934387207,
      "activations/layer23_attention_weight_min": -24.829898834228516,
      "activations/layer2_attention_weight_max": 30.908447265625,
      "activations/layer2_attention_weight_min": -32.134578704833984,
      "activations/layer3_attention_weight_max": 87.82403564453125,
      "activations/layer3_attention_weight_min": -90.71107482910156,
      "activations/layer4_attention_weight_max": 106.49803924560547,
      "activations/layer4_attention_weight_min": -112.68231201171875,
      "activations/layer5_attention_weight_max": 50.26833724975586,
      "activations/layer5_attention_weight_min": -58.391719818115234,
      "activations/layer6_attention_weight_max": 42.97407531738281,
      "activations/layer6_attention_weight_min": -46.85763168334961,
      "activations/layer7_attention_weight_max": 83.28521728515625,
      "activations/layer7_attention_weight_min": -86.6293716430664,
      "activations/layer8_attention_weight_max": 38.97795486450195,
      "activations/layer8_attention_weight_min": -39.72377395629883,
      "activations/layer9_attention_weight_max": 32.883270263671875,
      "activations/layer9_attention_weight_min": -31.096210479736328,
      "epoch": 10.82,
      "learning_rate": 8.106439393939395e-05,
      "loss": 2.7888,
      "step": 186150
    },
    {
      "activations/layer0_attention_weight_max": 16.144302368164062,
      "activations/layer0_attention_weight_min": -12.416227340698242,
      "activations/layer10_attention_weight_max": 36.07579803466797,
      "activations/layer10_attention_weight_min": -36.59808349609375,
      "activations/layer11_attention_weight_max": 35.770606994628906,
      "activations/layer11_attention_weight_min": -36.55718231201172,
      "activations/layer12_attention_weight_max": 26.607187271118164,
      "activations/layer12_attention_weight_min": -27.8797664642334,
      "activations/layer13_attention_weight_max": 40.45160675048828,
      "activations/layer13_attention_weight_min": -36.0733642578125,
      "activations/layer14_attention_weight_max": 48.867286682128906,
      "activations/layer14_attention_weight_min": -40.476661682128906,
      "activations/layer15_attention_weight_max": 37.49727249145508,
      "activations/layer15_attention_weight_min": -34.42010498046875,
      "activations/layer16_attention_weight_max": 33.14362335205078,
      "activations/layer16_attention_weight_min": -28.714893341064453,
      "activations/layer17_attention_weight_max": 30.608211517333984,
      "activations/layer17_attention_weight_min": -25.712614059448242,
      "activations/layer18_attention_weight_max": 32.48851776123047,
      "activations/layer18_attention_weight_min": -23.4298152923584,
      "activations/layer19_attention_weight_max": 35.76156997680664,
      "activations/layer19_attention_weight_min": -32.5974235534668,
      "activations/layer1_attention_weight_max": 16.843303680419922,
      "activations/layer1_attention_weight_min": -13.418538093566895,
      "activations/layer20_attention_weight_max": 31.07921028137207,
      "activations/layer20_attention_weight_min": -24.47020149230957,
      "activations/layer21_attention_weight_max": 28.616214752197266,
      "activations/layer21_attention_weight_min": -24.897232055664062,
      "activations/layer22_attention_weight_max": 47.15260314941406,
      "activations/layer22_attention_weight_min": -30.03022003173828,
      "activations/layer23_attention_weight_max": 35.940345764160156,
      "activations/layer23_attention_weight_min": -24.03955078125,
      "activations/layer2_attention_weight_max": 32.032386779785156,
      "activations/layer2_attention_weight_min": -31.706388473510742,
      "activations/layer3_attention_weight_max": 91.33847045898438,
      "activations/layer3_attention_weight_min": -93.77960205078125,
      "activations/layer4_attention_weight_max": 111.02052307128906,
      "activations/layer4_attention_weight_min": -115.4303970336914,
      "activations/layer5_attention_weight_max": 51.18280029296875,
      "activations/layer5_attention_weight_min": -64.05204772949219,
      "activations/layer6_attention_weight_max": 47.12199401855469,
      "activations/layer6_attention_weight_min": -47.08169174194336,
      "activations/layer7_attention_weight_max": 90.66398620605469,
      "activations/layer7_attention_weight_min": -87.41631317138672,
      "activations/layer8_attention_weight_max": 45.58254623413086,
      "activations/layer8_attention_weight_min": -45.136837005615234,
      "activations/layer9_attention_weight_max": 37.47800827026367,
      "activations/layer9_attention_weight_min": -37.51911163330078,
      "epoch": 10.82,
      "learning_rate": 8.104545454545453e-05,
      "loss": 2.793,
      "step": 186200
    },
    {
      "activations/layer0_attention_weight_max": 16.385040283203125,
      "activations/layer0_attention_weight_min": -12.102810859680176,
      "activations/layer10_attention_weight_max": 33.092124938964844,
      "activations/layer10_attention_weight_min": -32.66092300415039,
      "activations/layer11_attention_weight_max": 32.225746154785156,
      "activations/layer11_attention_weight_min": -29.665992736816406,
      "activations/layer12_attention_weight_max": 25.33818244934082,
      "activations/layer12_attention_weight_min": -25.388614654541016,
      "activations/layer13_attention_weight_max": 41.18524932861328,
      "activations/layer13_attention_weight_min": -36.902191162109375,
      "activations/layer14_attention_weight_max": 46.97149658203125,
      "activations/layer14_attention_weight_min": -40.94980239868164,
      "activations/layer15_attention_weight_max": 36.413570404052734,
      "activations/layer15_attention_weight_min": -31.466541290283203,
      "activations/layer16_attention_weight_max": 32.6120719909668,
      "activations/layer16_attention_weight_min": -27.583614349365234,
      "activations/layer17_attention_weight_max": 32.0632209777832,
      "activations/layer17_attention_weight_min": -27.360506057739258,
      "activations/layer18_attention_weight_max": 30.38849449157715,
      "activations/layer18_attention_weight_min": -23.476036071777344,
      "activations/layer19_attention_weight_max": 33.70332336425781,
      "activations/layer19_attention_weight_min": -32.75620651245117,
      "activations/layer1_attention_weight_max": 16.904956817626953,
      "activations/layer1_attention_weight_min": -13.497295379638672,
      "activations/layer20_attention_weight_max": 28.536441802978516,
      "activations/layer20_attention_weight_min": -26.798295974731445,
      "activations/layer21_attention_weight_max": 27.8336181640625,
      "activations/layer21_attention_weight_min": -23.976221084594727,
      "activations/layer22_attention_weight_max": 42.005943298339844,
      "activations/layer22_attention_weight_min": -28.876142501831055,
      "activations/layer23_attention_weight_max": 31.872787475585938,
      "activations/layer23_attention_weight_min": -24.016338348388672,
      "activations/layer2_attention_weight_max": 31.762985229492188,
      "activations/layer2_attention_weight_min": -31.79652214050293,
      "activations/layer3_attention_weight_max": 92.08030700683594,
      "activations/layer3_attention_weight_min": -93.49469757080078,
      "activations/layer4_attention_weight_max": 114.26020812988281,
      "activations/layer4_attention_weight_min": -111.36756896972656,
      "activations/layer5_attention_weight_max": 51.901588439941406,
      "activations/layer5_attention_weight_min": -59.97459411621094,
      "activations/layer6_attention_weight_max": 45.35049057006836,
      "activations/layer6_attention_weight_min": -48.609378814697266,
      "activations/layer7_attention_weight_max": 91.14299011230469,
      "activations/layer7_attention_weight_min": -87.8138656616211,
      "activations/layer8_attention_weight_max": 39.56687545776367,
      "activations/layer8_attention_weight_min": -40.566253662109375,
      "activations/layer9_attention_weight_max": 34.3707160949707,
      "activations/layer9_attention_weight_min": -33.6786994934082,
      "epoch": 10.82,
      "learning_rate": 8.102651515151515e-05,
      "loss": 2.7765,
      "step": 186250
    },
    {
      "activations/layer0_attention_weight_max": 16.117794036865234,
      "activations/layer0_attention_weight_min": -13.049966812133789,
      "activations/layer10_attention_weight_max": 32.7994384765625,
      "activations/layer10_attention_weight_min": -33.2407112121582,
      "activations/layer11_attention_weight_max": 30.51357078552246,
      "activations/layer11_attention_weight_min": -29.935874938964844,
      "activations/layer12_attention_weight_max": 25.00348472595215,
      "activations/layer12_attention_weight_min": -23.49453353881836,
      "activations/layer13_attention_weight_max": 36.72004318237305,
      "activations/layer13_attention_weight_min": -33.25373458862305,
      "activations/layer14_attention_weight_max": 41.60665512084961,
      "activations/layer14_attention_weight_min": -37.96052551269531,
      "activations/layer15_attention_weight_max": 36.717529296875,
      "activations/layer15_attention_weight_min": -30.757564544677734,
      "activations/layer16_attention_weight_max": 31.505733489990234,
      "activations/layer16_attention_weight_min": -28.459300994873047,
      "activations/layer17_attention_weight_max": 31.532461166381836,
      "activations/layer17_attention_weight_min": -25.37487030029297,
      "activations/layer18_attention_weight_max": 30.21644401550293,
      "activations/layer18_attention_weight_min": -24.49286651611328,
      "activations/layer19_attention_weight_max": 33.0936279296875,
      "activations/layer19_attention_weight_min": -31.294782638549805,
      "activations/layer1_attention_weight_max": 17.09101104736328,
      "activations/layer1_attention_weight_min": -14.459388732910156,
      "activations/layer20_attention_weight_max": 29.1734676361084,
      "activations/layer20_attention_weight_min": -26.561084747314453,
      "activations/layer21_attention_weight_max": 31.26409912109375,
      "activations/layer21_attention_weight_min": -27.241905212402344,
      "activations/layer22_attention_weight_max": 44.49468994140625,
      "activations/layer22_attention_weight_min": -29.14393424987793,
      "activations/layer23_attention_weight_max": 34.37897872924805,
      "activations/layer23_attention_weight_min": -23.789772033691406,
      "activations/layer2_attention_weight_max": 31.149429321289062,
      "activations/layer2_attention_weight_min": -31.79934310913086,
      "activations/layer3_attention_weight_max": 91.66902160644531,
      "activations/layer3_attention_weight_min": -93.83740234375,
      "activations/layer4_attention_weight_max": 110.44490051269531,
      "activations/layer4_attention_weight_min": -110.0990982055664,
      "activations/layer5_attention_weight_max": 53.610496520996094,
      "activations/layer5_attention_weight_min": -59.5745964050293,
      "activations/layer6_attention_weight_max": 44.29648971557617,
      "activations/layer6_attention_weight_min": -49.00520324707031,
      "activations/layer7_attention_weight_max": 93.14289093017578,
      "activations/layer7_attention_weight_min": -90.66226959228516,
      "activations/layer8_attention_weight_max": 40.64356994628906,
      "activations/layer8_attention_weight_min": -40.5057373046875,
      "activations/layer9_attention_weight_max": 33.938838958740234,
      "activations/layer9_attention_weight_min": -33.627891540527344,
      "epoch": 10.83,
      "learning_rate": 8.100757575757575e-05,
      "loss": 2.7851,
      "step": 186300
    },
    {
      "activations/layer0_attention_weight_max": 16.195585250854492,
      "activations/layer0_attention_weight_min": -12.827509880065918,
      "activations/layer10_attention_weight_max": 30.69096565246582,
      "activations/layer10_attention_weight_min": -32.47479248046875,
      "activations/layer11_attention_weight_max": 32.28514099121094,
      "activations/layer11_attention_weight_min": -30.585493087768555,
      "activations/layer12_attention_weight_max": 29.9898738861084,
      "activations/layer12_attention_weight_min": -25.99894142150879,
      "activations/layer13_attention_weight_max": 40.165000915527344,
      "activations/layer13_attention_weight_min": -38.04130554199219,
      "activations/layer14_attention_weight_max": 46.05424118041992,
      "activations/layer14_attention_weight_min": -40.14160919189453,
      "activations/layer15_attention_weight_max": 36.710105895996094,
      "activations/layer15_attention_weight_min": -34.56898880004883,
      "activations/layer16_attention_weight_max": 30.28163719177246,
      "activations/layer16_attention_weight_min": -28.98275375366211,
      "activations/layer17_attention_weight_max": 32.62180709838867,
      "activations/layer17_attention_weight_min": -28.240949630737305,
      "activations/layer18_attention_weight_max": 30.957618713378906,
      "activations/layer18_attention_weight_min": -24.021284103393555,
      "activations/layer19_attention_weight_max": 34.300086975097656,
      "activations/layer19_attention_weight_min": -32.434749603271484,
      "activations/layer1_attention_weight_max": 17.27027130126953,
      "activations/layer1_attention_weight_min": -13.886362075805664,
      "activations/layer20_attention_weight_max": 30.381319046020508,
      "activations/layer20_attention_weight_min": -26.903179168701172,
      "activations/layer21_attention_weight_max": 28.640689849853516,
      "activations/layer21_attention_weight_min": -24.859228134155273,
      "activations/layer22_attention_weight_max": 45.2354850769043,
      "activations/layer22_attention_weight_min": -31.409875869750977,
      "activations/layer23_attention_weight_max": 35.19622039794922,
      "activations/layer23_attention_weight_min": -25.302810668945312,
      "activations/layer2_attention_weight_max": 32.62311553955078,
      "activations/layer2_attention_weight_min": -30.667251586914062,
      "activations/layer3_attention_weight_max": 92.1893081665039,
      "activations/layer3_attention_weight_min": -93.15387725830078,
      "activations/layer4_attention_weight_max": 111.9327163696289,
      "activations/layer4_attention_weight_min": -112.7122573852539,
      "activations/layer5_attention_weight_max": 56.04061508178711,
      "activations/layer5_attention_weight_min": -61.50660705566406,
      "activations/layer6_attention_weight_max": 46.664085388183594,
      "activations/layer6_attention_weight_min": -48.51563262939453,
      "activations/layer7_attention_weight_max": 94.14997863769531,
      "activations/layer7_attention_weight_min": -90.906494140625,
      "activations/layer8_attention_weight_max": 39.915283203125,
      "activations/layer8_attention_weight_min": -40.73778533935547,
      "activations/layer9_attention_weight_max": 32.10830307006836,
      "activations/layer9_attention_weight_min": -32.586830139160156,
      "epoch": 10.83,
      "learning_rate": 8.098863636363635e-05,
      "loss": 2.7857,
      "step": 186350
    },
    {
      "activations/layer0_attention_weight_max": 15.800392150878906,
      "activations/layer0_attention_weight_min": -12.380287170410156,
      "activations/layer10_attention_weight_max": 33.05010986328125,
      "activations/layer10_attention_weight_min": -33.06459045410156,
      "activations/layer11_attention_weight_max": 35.76859664916992,
      "activations/layer11_attention_weight_min": -32.2317008972168,
      "activations/layer12_attention_weight_max": 25.224414825439453,
      "activations/layer12_attention_weight_min": -24.61046600341797,
      "activations/layer13_attention_weight_max": 39.12208938598633,
      "activations/layer13_attention_weight_min": -34.66085433959961,
      "activations/layer14_attention_weight_max": 48.0140266418457,
      "activations/layer14_attention_weight_min": -40.387779235839844,
      "activations/layer15_attention_weight_max": 36.636024475097656,
      "activations/layer15_attention_weight_min": -32.292335510253906,
      "activations/layer16_attention_weight_max": 30.048812866210938,
      "activations/layer16_attention_weight_min": -27.674306869506836,
      "activations/layer17_attention_weight_max": 29.907657623291016,
      "activations/layer17_attention_weight_min": -25.463708877563477,
      "activations/layer18_attention_weight_max": 30.02824592590332,
      "activations/layer18_attention_weight_min": -23.7995548248291,
      "activations/layer19_attention_weight_max": 33.83360290527344,
      "activations/layer19_attention_weight_min": -31.428293228149414,
      "activations/layer1_attention_weight_max": 16.766427993774414,
      "activations/layer1_attention_weight_min": -14.388288497924805,
      "activations/layer20_attention_weight_max": 31.5770206451416,
      "activations/layer20_attention_weight_min": -25.42006492614746,
      "activations/layer21_attention_weight_max": 27.16385269165039,
      "activations/layer21_attention_weight_min": -25.542898178100586,
      "activations/layer22_attention_weight_max": 45.786293029785156,
      "activations/layer22_attention_weight_min": -29.54555320739746,
      "activations/layer23_attention_weight_max": 33.48928451538086,
      "activations/layer23_attention_weight_min": -23.12331771850586,
      "activations/layer2_attention_weight_max": 32.35907745361328,
      "activations/layer2_attention_weight_min": -32.00612258911133,
      "activations/layer3_attention_weight_max": 94.45101928710938,
      "activations/layer3_attention_weight_min": -94.12333679199219,
      "activations/layer4_attention_weight_max": 114.5705337524414,
      "activations/layer4_attention_weight_min": -110.33251953125,
      "activations/layer5_attention_weight_max": 52.76380920410156,
      "activations/layer5_attention_weight_min": -61.9713249206543,
      "activations/layer6_attention_weight_max": 44.497276306152344,
      "activations/layer6_attention_weight_min": -50.04219055175781,
      "activations/layer7_attention_weight_max": 89.68701171875,
      "activations/layer7_attention_weight_min": -90.41061401367188,
      "activations/layer8_attention_weight_max": 40.290462493896484,
      "activations/layer8_attention_weight_min": -43.28142547607422,
      "activations/layer9_attention_weight_max": 34.84111022949219,
      "activations/layer9_attention_weight_min": -33.68310546875,
      "epoch": 10.83,
      "learning_rate": 8.096969696969697e-05,
      "loss": 2.7725,
      "step": 186400
    },
    {
      "activations/layer0_attention_weight_max": 14.840005874633789,
      "activations/layer0_attention_weight_min": -13.078614234924316,
      "activations/layer10_attention_weight_max": 31.55807876586914,
      "activations/layer10_attention_weight_min": -32.9162483215332,
      "activations/layer11_attention_weight_max": 32.14543151855469,
      "activations/layer11_attention_weight_min": -31.78744125366211,
      "activations/layer12_attention_weight_max": 23.961532592773438,
      "activations/layer12_attention_weight_min": -25.4866886138916,
      "activations/layer13_attention_weight_max": 38.10400390625,
      "activations/layer13_attention_weight_min": -34.21551513671875,
      "activations/layer14_attention_weight_max": 42.98085021972656,
      "activations/layer14_attention_weight_min": -39.50172805786133,
      "activations/layer15_attention_weight_max": 36.03964614868164,
      "activations/layer15_attention_weight_min": -32.288490295410156,
      "activations/layer16_attention_weight_max": 29.975528717041016,
      "activations/layer16_attention_weight_min": -29.846696853637695,
      "activations/layer17_attention_weight_max": 29.742835998535156,
      "activations/layer17_attention_weight_min": -25.865528106689453,
      "activations/layer18_attention_weight_max": 30.75016975402832,
      "activations/layer18_attention_weight_min": -22.597686767578125,
      "activations/layer19_attention_weight_max": 34.2742805480957,
      "activations/layer19_attention_weight_min": -29.909317016601562,
      "activations/layer1_attention_weight_max": 17.075178146362305,
      "activations/layer1_attention_weight_min": -16.898164749145508,
      "activations/layer20_attention_weight_max": 27.520030975341797,
      "activations/layer20_attention_weight_min": -25.367958068847656,
      "activations/layer21_attention_weight_max": 26.92656135559082,
      "activations/layer21_attention_weight_min": -21.73042869567871,
      "activations/layer22_attention_weight_max": 44.64624786376953,
      "activations/layer22_attention_weight_min": -28.088390350341797,
      "activations/layer23_attention_weight_max": 33.9921875,
      "activations/layer23_attention_weight_min": -23.40890884399414,
      "activations/layer2_attention_weight_max": 30.829116821289062,
      "activations/layer2_attention_weight_min": -32.167747497558594,
      "activations/layer3_attention_weight_max": 95.6237564086914,
      "activations/layer3_attention_weight_min": -99.84579467773438,
      "activations/layer4_attention_weight_max": 112.3205795288086,
      "activations/layer4_attention_weight_min": -117.19300842285156,
      "activations/layer5_attention_weight_max": 56.5658073425293,
      "activations/layer5_attention_weight_min": -66.19587707519531,
      "activations/layer6_attention_weight_max": 44.19717025756836,
      "activations/layer6_attention_weight_min": -47.93345260620117,
      "activations/layer7_attention_weight_max": 91.41244506835938,
      "activations/layer7_attention_weight_min": -91.71983337402344,
      "activations/layer8_attention_weight_max": 39.43274688720703,
      "activations/layer8_attention_weight_min": -42.76265335083008,
      "activations/layer9_attention_weight_max": 33.67829132080078,
      "activations/layer9_attention_weight_min": -32.816036224365234,
      "epoch": 10.83,
      "learning_rate": 8.095075757575757e-05,
      "loss": 2.7799,
      "step": 186450
    },
    {
      "activations/layer0_attention_weight_max": 15.33010196685791,
      "activations/layer0_attention_weight_min": -14.302730560302734,
      "activations/layer10_attention_weight_max": 37.950927734375,
      "activations/layer10_attention_weight_min": -34.98573303222656,
      "activations/layer11_attention_weight_max": 38.382720947265625,
      "activations/layer11_attention_weight_min": -36.511680603027344,
      "activations/layer12_attention_weight_max": 25.470474243164062,
      "activations/layer12_attention_weight_min": -25.890459060668945,
      "activations/layer13_attention_weight_max": 38.59636688232422,
      "activations/layer13_attention_weight_min": -34.49801254272461,
      "activations/layer14_attention_weight_max": 46.45990753173828,
      "activations/layer14_attention_weight_min": -38.11807632446289,
      "activations/layer15_attention_weight_max": 36.52337646484375,
      "activations/layer15_attention_weight_min": -32.600093841552734,
      "activations/layer16_attention_weight_max": 34.73911666870117,
      "activations/layer16_attention_weight_min": -27.3226261138916,
      "activations/layer17_attention_weight_max": 31.144718170166016,
      "activations/layer17_attention_weight_min": -27.82308006286621,
      "activations/layer18_attention_weight_max": 31.526226043701172,
      "activations/layer18_attention_weight_min": -26.060293197631836,
      "activations/layer19_attention_weight_max": 41.435211181640625,
      "activations/layer19_attention_weight_min": -31.430368423461914,
      "activations/layer1_attention_weight_max": 17.414072036743164,
      "activations/layer1_attention_weight_min": -14.09286117553711,
      "activations/layer20_attention_weight_max": 30.25372314453125,
      "activations/layer20_attention_weight_min": -26.81675910949707,
      "activations/layer21_attention_weight_max": 28.681180953979492,
      "activations/layer21_attention_weight_min": -25.422836303710938,
      "activations/layer22_attention_weight_max": 50.50096130371094,
      "activations/layer22_attention_weight_min": -33.16133117675781,
      "activations/layer23_attention_weight_max": 41.79310607910156,
      "activations/layer23_attention_weight_min": -24.397674560546875,
      "activations/layer2_attention_weight_max": 32.59271240234375,
      "activations/layer2_attention_weight_min": -34.24534606933594,
      "activations/layer3_attention_weight_max": 94.92581176757812,
      "activations/layer3_attention_weight_min": -99.21452331542969,
      "activations/layer4_attention_weight_max": 117.26893615722656,
      "activations/layer4_attention_weight_min": -117.90807342529297,
      "activations/layer5_attention_weight_max": 59.167869567871094,
      "activations/layer5_attention_weight_min": -60.8131217956543,
      "activations/layer6_attention_weight_max": 48.02450180053711,
      "activations/layer6_attention_weight_min": -49.22852325439453,
      "activations/layer7_attention_weight_max": 109.70870971679688,
      "activations/layer7_attention_weight_min": -95.57286834716797,
      "activations/layer8_attention_weight_max": 47.474971771240234,
      "activations/layer8_attention_weight_min": -44.374267578125,
      "activations/layer9_attention_weight_max": 37.88314437866211,
      "activations/layer9_attention_weight_min": -36.30072021484375,
      "epoch": 10.84,
      "learning_rate": 8.093181818181817e-05,
      "loss": 2.7934,
      "step": 186500
    },
    {
      "activations/layer0_attention_weight_max": 15.654756546020508,
      "activations/layer0_attention_weight_min": -12.80298900604248,
      "activations/layer10_attention_weight_max": 32.1169319152832,
      "activations/layer10_attention_weight_min": -36.49092483520508,
      "activations/layer11_attention_weight_max": 34.69255065917969,
      "activations/layer11_attention_weight_min": -34.927940368652344,
      "activations/layer12_attention_weight_max": 33.965248107910156,
      "activations/layer12_attention_weight_min": -25.0588321685791,
      "activations/layer13_attention_weight_max": 42.72871398925781,
      "activations/layer13_attention_weight_min": -35.79521179199219,
      "activations/layer14_attention_weight_max": 47.24384689331055,
      "activations/layer14_attention_weight_min": -38.9655647277832,
      "activations/layer15_attention_weight_max": 39.961856842041016,
      "activations/layer15_attention_weight_min": -33.542694091796875,
      "activations/layer16_attention_weight_max": 32.93104934692383,
      "activations/layer16_attention_weight_min": -27.71547508239746,
      "activations/layer17_attention_weight_max": 33.08424758911133,
      "activations/layer17_attention_weight_min": -26.87544822692871,
      "activations/layer18_attention_weight_max": 32.25532150268555,
      "activations/layer18_attention_weight_min": -23.21761703491211,
      "activations/layer19_attention_weight_max": 34.7847785949707,
      "activations/layer19_attention_weight_min": -30.858057022094727,
      "activations/layer1_attention_weight_max": 16.410917282104492,
      "activations/layer1_attention_weight_min": -15.407868385314941,
      "activations/layer20_attention_weight_max": 30.913639068603516,
      "activations/layer20_attention_weight_min": -24.70650291442871,
      "activations/layer21_attention_weight_max": 29.222867965698242,
      "activations/layer21_attention_weight_min": -22.96135139465332,
      "activations/layer22_attention_weight_max": 45.10542678833008,
      "activations/layer22_attention_weight_min": -31.80496597290039,
      "activations/layer23_attention_weight_max": 34.78201675415039,
      "activations/layer23_attention_weight_min": -23.688568115234375,
      "activations/layer2_attention_weight_max": 31.54915428161621,
      "activations/layer2_attention_weight_min": -31.52294158935547,
      "activations/layer3_attention_weight_max": 94.64903259277344,
      "activations/layer3_attention_weight_min": -90.6836929321289,
      "activations/layer4_attention_weight_max": 115.4643783569336,
      "activations/layer4_attention_weight_min": -114.71858215332031,
      "activations/layer5_attention_weight_max": 52.64537811279297,
      "activations/layer5_attention_weight_min": -64.44598388671875,
      "activations/layer6_attention_weight_max": 44.59049606323242,
      "activations/layer6_attention_weight_min": -48.794559478759766,
      "activations/layer7_attention_weight_max": 88.38927459716797,
      "activations/layer7_attention_weight_min": -90.81675720214844,
      "activations/layer8_attention_weight_max": 41.09563064575195,
      "activations/layer8_attention_weight_min": -43.213829040527344,
      "activations/layer9_attention_weight_max": 34.72369384765625,
      "activations/layer9_attention_weight_min": -34.94083786010742,
      "epoch": 10.84,
      "learning_rate": 8.091287878787878e-05,
      "loss": 2.7892,
      "step": 186550
    },
    {
      "activations/layer0_attention_weight_max": 15.36998176574707,
      "activations/layer0_attention_weight_min": -13.883420944213867,
      "activations/layer10_attention_weight_max": 37.2673225402832,
      "activations/layer10_attention_weight_min": -37.72441101074219,
      "activations/layer11_attention_weight_max": 34.58349609375,
      "activations/layer11_attention_weight_min": -34.08168029785156,
      "activations/layer12_attention_weight_max": 28.03486442565918,
      "activations/layer12_attention_weight_min": -25.091964721679688,
      "activations/layer13_attention_weight_max": 40.36322021484375,
      "activations/layer13_attention_weight_min": -36.58442687988281,
      "activations/layer14_attention_weight_max": 60.38274383544922,
      "activations/layer14_attention_weight_min": -39.97818374633789,
      "activations/layer15_attention_weight_max": 44.12192916870117,
      "activations/layer15_attention_weight_min": -36.41101837158203,
      "activations/layer16_attention_weight_max": 34.13606262207031,
      "activations/layer16_attention_weight_min": -28.78429412841797,
      "activations/layer17_attention_weight_max": 33.29801559448242,
      "activations/layer17_attention_weight_min": -26.400466918945312,
      "activations/layer18_attention_weight_max": 33.84587478637695,
      "activations/layer18_attention_weight_min": -23.29081916809082,
      "activations/layer19_attention_weight_max": 40.82802200317383,
      "activations/layer19_attention_weight_min": -32.07744216918945,
      "activations/layer1_attention_weight_max": 15.965665817260742,
      "activations/layer1_attention_weight_min": -13.984601020812988,
      "activations/layer20_attention_weight_max": 31.401371002197266,
      "activations/layer20_attention_weight_min": -25.532941818237305,
      "activations/layer21_attention_weight_max": 31.23556900024414,
      "activations/layer21_attention_weight_min": -24.08455467224121,
      "activations/layer22_attention_weight_max": 51.07905578613281,
      "activations/layer22_attention_weight_min": -33.46799850463867,
      "activations/layer23_attention_weight_max": 39.11968231201172,
      "activations/layer23_attention_weight_min": -29.61736297607422,
      "activations/layer2_attention_weight_max": 33.30094909667969,
      "activations/layer2_attention_weight_min": -32.80187225341797,
      "activations/layer3_attention_weight_max": 97.03631591796875,
      "activations/layer3_attention_weight_min": -96.21964263916016,
      "activations/layer4_attention_weight_max": 110.93560028076172,
      "activations/layer4_attention_weight_min": -112.33013916015625,
      "activations/layer5_attention_weight_max": 52.28657531738281,
      "activations/layer5_attention_weight_min": -61.0712890625,
      "activations/layer6_attention_weight_max": 47.65922164916992,
      "activations/layer6_attention_weight_min": -52.00890350341797,
      "activations/layer7_attention_weight_max": 92.78717041015625,
      "activations/layer7_attention_weight_min": -93.87796783447266,
      "activations/layer8_attention_weight_max": 45.86796951293945,
      "activations/layer8_attention_weight_min": -45.89075469970703,
      "activations/layer9_attention_weight_max": 37.493682861328125,
      "activations/layer9_attention_weight_min": -35.58839797973633,
      "epoch": 10.84,
      "learning_rate": 8.089393939393939e-05,
      "loss": 2.7893,
      "step": 186600
    },
    {
      "activations/layer0_attention_weight_max": 15.063858985900879,
      "activations/layer0_attention_weight_min": -12.045022010803223,
      "activations/layer10_attention_weight_max": 37.26593017578125,
      "activations/layer10_attention_weight_min": -38.265350341796875,
      "activations/layer11_attention_weight_max": 39.50957489013672,
      "activations/layer11_attention_weight_min": -36.881229400634766,
      "activations/layer12_attention_weight_max": 27.639392852783203,
      "activations/layer12_attention_weight_min": -27.44448471069336,
      "activations/layer13_attention_weight_max": 42.718631744384766,
      "activations/layer13_attention_weight_min": -36.96437454223633,
      "activations/layer14_attention_weight_max": 56.09983444213867,
      "activations/layer14_attention_weight_min": -42.322059631347656,
      "activations/layer15_attention_weight_max": 41.4660530090332,
      "activations/layer15_attention_weight_min": -34.81072998046875,
      "activations/layer16_attention_weight_max": 37.19157791137695,
      "activations/layer16_attention_weight_min": -29.72690773010254,
      "activations/layer17_attention_weight_max": 32.087669372558594,
      "activations/layer17_attention_weight_min": -27.74341583251953,
      "activations/layer18_attention_weight_max": 34.00464630126953,
      "activations/layer18_attention_weight_min": -25.145832061767578,
      "activations/layer19_attention_weight_max": 44.777828216552734,
      "activations/layer19_attention_weight_min": -35.23922348022461,
      "activations/layer1_attention_weight_max": 16.748361587524414,
      "activations/layer1_attention_weight_min": -14.51666259765625,
      "activations/layer20_attention_weight_max": 35.518272399902344,
      "activations/layer20_attention_weight_min": -26.448923110961914,
      "activations/layer21_attention_weight_max": 32.885589599609375,
      "activations/layer21_attention_weight_min": -24.0185489654541,
      "activations/layer22_attention_weight_max": 51.58735656738281,
      "activations/layer22_attention_weight_min": -33.89808654785156,
      "activations/layer23_attention_weight_max": 44.03343963623047,
      "activations/layer23_attention_weight_min": -25.055335998535156,
      "activations/layer2_attention_weight_max": 32.66203308105469,
      "activations/layer2_attention_weight_min": -33.56293487548828,
      "activations/layer3_attention_weight_max": 99.70123291015625,
      "activations/layer3_attention_weight_min": -101.70133972167969,
      "activations/layer4_attention_weight_max": 122.81742095947266,
      "activations/layer4_attention_weight_min": -121.03181457519531,
      "activations/layer5_attention_weight_max": 58.02864074707031,
      "activations/layer5_attention_weight_min": -60.91840744018555,
      "activations/layer6_attention_weight_max": 48.28432846069336,
      "activations/layer6_attention_weight_min": -47.46308898925781,
      "activations/layer7_attention_weight_max": 97.920166015625,
      "activations/layer7_attention_weight_min": -99.26475524902344,
      "activations/layer8_attention_weight_max": 45.936805725097656,
      "activations/layer8_attention_weight_min": -45.74275207519531,
      "activations/layer9_attention_weight_max": 36.56538009643555,
      "activations/layer9_attention_weight_min": -37.917240142822266,
      "epoch": 10.85,
      "learning_rate": 8.0875e-05,
      "loss": 2.7799,
      "step": 186650
    },
    {
      "activations/layer0_attention_weight_max": 15.551222801208496,
      "activations/layer0_attention_weight_min": -11.806462287902832,
      "activations/layer10_attention_weight_max": 32.38621139526367,
      "activations/layer10_attention_weight_min": -32.7395133972168,
      "activations/layer11_attention_weight_max": 34.453128814697266,
      "activations/layer11_attention_weight_min": -32.73524475097656,
      "activations/layer12_attention_weight_max": 24.998809814453125,
      "activations/layer12_attention_weight_min": -24.26268768310547,
      "activations/layer13_attention_weight_max": 39.93376159667969,
      "activations/layer13_attention_weight_min": -33.91901397705078,
      "activations/layer14_attention_weight_max": 48.71525573730469,
      "activations/layer14_attention_weight_min": -40.449798583984375,
      "activations/layer15_attention_weight_max": 36.23030471801758,
      "activations/layer15_attention_weight_min": -31.94965362548828,
      "activations/layer16_attention_weight_max": 30.066553115844727,
      "activations/layer16_attention_weight_min": -28.689783096313477,
      "activations/layer17_attention_weight_max": 33.10622024536133,
      "activations/layer17_attention_weight_min": -26.564050674438477,
      "activations/layer18_attention_weight_max": 30.584434509277344,
      "activations/layer18_attention_weight_min": -23.978364944458008,
      "activations/layer19_attention_weight_max": 34.7182502746582,
      "activations/layer19_attention_weight_min": -33.04385757446289,
      "activations/layer1_attention_weight_max": 19.01398277282715,
      "activations/layer1_attention_weight_min": -15.496152877807617,
      "activations/layer20_attention_weight_max": 30.93402099609375,
      "activations/layer20_attention_weight_min": -24.831724166870117,
      "activations/layer21_attention_weight_max": 29.692014694213867,
      "activations/layer21_attention_weight_min": -25.236270904541016,
      "activations/layer22_attention_weight_max": 47.65713119506836,
      "activations/layer22_attention_weight_min": -31.26132583618164,
      "activations/layer23_attention_weight_max": 37.220787048339844,
      "activations/layer23_attention_weight_min": -24.152996063232422,
      "activations/layer2_attention_weight_max": 31.311946868896484,
      "activations/layer2_attention_weight_min": -31.343475341796875,
      "activations/layer3_attention_weight_max": 94.44178009033203,
      "activations/layer3_attention_weight_min": -97.31488800048828,
      "activations/layer4_attention_weight_max": 114.17872619628906,
      "activations/layer4_attention_weight_min": -114.01725006103516,
      "activations/layer5_attention_weight_max": 55.67089080810547,
      "activations/layer5_attention_weight_min": -60.92336654663086,
      "activations/layer6_attention_weight_max": 45.64148712158203,
      "activations/layer6_attention_weight_min": -47.323448181152344,
      "activations/layer7_attention_weight_max": 84.0780258178711,
      "activations/layer7_attention_weight_min": -88.01385498046875,
      "activations/layer8_attention_weight_max": 40.91582107543945,
      "activations/layer8_attention_weight_min": -39.904685974121094,
      "activations/layer9_attention_weight_max": 34.9626579284668,
      "activations/layer9_attention_weight_min": -32.6626091003418,
      "epoch": 10.85,
      "learning_rate": 8.085606060606059e-05,
      "loss": 2.7841,
      "step": 186700
    },
    {
      "activations/layer0_attention_weight_max": 15.841041564941406,
      "activations/layer0_attention_weight_min": -11.783273696899414,
      "activations/layer10_attention_weight_max": 37.2482795715332,
      "activations/layer10_attention_weight_min": -34.26181411743164,
      "activations/layer11_attention_weight_max": 37.63010025024414,
      "activations/layer11_attention_weight_min": -33.46464538574219,
      "activations/layer12_attention_weight_max": 26.478544235229492,
      "activations/layer12_attention_weight_min": -26.234088897705078,
      "activations/layer13_attention_weight_max": 43.18337631225586,
      "activations/layer13_attention_weight_min": -36.275726318359375,
      "activations/layer14_attention_weight_max": 47.40005874633789,
      "activations/layer14_attention_weight_min": -36.65931701660156,
      "activations/layer15_attention_weight_max": 37.8270263671875,
      "activations/layer15_attention_weight_min": -32.13555908203125,
      "activations/layer16_attention_weight_max": 32.993892669677734,
      "activations/layer16_attention_weight_min": -26.855897903442383,
      "activations/layer17_attention_weight_max": 33.595943450927734,
      "activations/layer17_attention_weight_min": -26.8870849609375,
      "activations/layer18_attention_weight_max": 32.970550537109375,
      "activations/layer18_attention_weight_min": -23.157480239868164,
      "activations/layer19_attention_weight_max": 39.454654693603516,
      "activations/layer19_attention_weight_min": -32.161617279052734,
      "activations/layer1_attention_weight_max": 16.785789489746094,
      "activations/layer1_attention_weight_min": -15.905342102050781,
      "activations/layer20_attention_weight_max": 31.48666000366211,
      "activations/layer20_attention_weight_min": -26.511783599853516,
      "activations/layer21_attention_weight_max": 30.24931526184082,
      "activations/layer21_attention_weight_min": -25.874656677246094,
      "activations/layer22_attention_weight_max": 46.47450256347656,
      "activations/layer22_attention_weight_min": -29.599607467651367,
      "activations/layer23_attention_weight_max": 36.75957489013672,
      "activations/layer23_attention_weight_min": -23.399463653564453,
      "activations/layer2_attention_weight_max": 34.34228515625,
      "activations/layer2_attention_weight_min": -32.21874237060547,
      "activations/layer3_attention_weight_max": 91.1319580078125,
      "activations/layer3_attention_weight_min": -95.49365234375,
      "activations/layer4_attention_weight_max": 113.94415283203125,
      "activations/layer4_attention_weight_min": -118.16069793701172,
      "activations/layer5_attention_weight_max": 51.71228790283203,
      "activations/layer5_attention_weight_min": -61.81533432006836,
      "activations/layer6_attention_weight_max": 46.122230529785156,
      "activations/layer6_attention_weight_min": -49.71595764160156,
      "activations/layer7_attention_weight_max": 89.36515808105469,
      "activations/layer7_attention_weight_min": -95.89299774169922,
      "activations/layer8_attention_weight_max": 44.387813568115234,
      "activations/layer8_attention_weight_min": -43.277259826660156,
      "activations/layer9_attention_weight_max": 36.731109619140625,
      "activations/layer9_attention_weight_min": -34.87693405151367,
      "epoch": 10.85,
      "learning_rate": 8.08371212121212e-05,
      "loss": 2.7619,
      "step": 186750
    },
    {
      "activations/layer0_attention_weight_max": 15.610774993896484,
      "activations/layer0_attention_weight_min": -13.38455581665039,
      "activations/layer10_attention_weight_max": 35.321903228759766,
      "activations/layer10_attention_weight_min": -33.468318939208984,
      "activations/layer11_attention_weight_max": 38.79841232299805,
      "activations/layer11_attention_weight_min": -35.04393005371094,
      "activations/layer12_attention_weight_max": 25.191696166992188,
      "activations/layer12_attention_weight_min": -26.544696807861328,
      "activations/layer13_attention_weight_max": 39.918922424316406,
      "activations/layer13_attention_weight_min": -36.004737854003906,
      "activations/layer14_attention_weight_max": 47.725406646728516,
      "activations/layer14_attention_weight_min": -40.875160217285156,
      "activations/layer15_attention_weight_max": 37.76617431640625,
      "activations/layer15_attention_weight_min": -33.29170608520508,
      "activations/layer16_attention_weight_max": 32.734230041503906,
      "activations/layer16_attention_weight_min": -27.09458351135254,
      "activations/layer17_attention_weight_max": 34.243408203125,
      "activations/layer17_attention_weight_min": -25.039663314819336,
      "activations/layer18_attention_weight_max": 33.3824348449707,
      "activations/layer18_attention_weight_min": -22.815853118896484,
      "activations/layer19_attention_weight_max": 37.56165313720703,
      "activations/layer19_attention_weight_min": -31.10068130493164,
      "activations/layer1_attention_weight_max": 16.733688354492188,
      "activations/layer1_attention_weight_min": -14.64844036102295,
      "activations/layer20_attention_weight_max": 32.55888366699219,
      "activations/layer20_attention_weight_min": -23.784587860107422,
      "activations/layer21_attention_weight_max": 29.2486572265625,
      "activations/layer21_attention_weight_min": -22.85108757019043,
      "activations/layer22_attention_weight_max": 51.9554328918457,
      "activations/layer22_attention_weight_min": -30.967344284057617,
      "activations/layer23_attention_weight_max": 37.43402862548828,
      "activations/layer23_attention_weight_min": -23.809120178222656,
      "activations/layer2_attention_weight_max": 31.77383041381836,
      "activations/layer2_attention_weight_min": -32.52241134643555,
      "activations/layer3_attention_weight_max": 97.688232421875,
      "activations/layer3_attention_weight_min": -97.54489135742188,
      "activations/layer4_attention_weight_max": 117.5979232788086,
      "activations/layer4_attention_weight_min": -112.23085021972656,
      "activations/layer5_attention_weight_max": 55.89093017578125,
      "activations/layer5_attention_weight_min": -60.817317962646484,
      "activations/layer6_attention_weight_max": 48.6189079284668,
      "activations/layer6_attention_weight_min": -49.36296844482422,
      "activations/layer7_attention_weight_max": 101.13197326660156,
      "activations/layer7_attention_weight_min": -95.3553466796875,
      "activations/layer8_attention_weight_max": 44.3848876953125,
      "activations/layer8_attention_weight_min": -44.58090591430664,
      "activations/layer9_attention_weight_max": 36.30005645751953,
      "activations/layer9_attention_weight_min": -37.02913284301758,
      "epoch": 10.85,
      "learning_rate": 8.081818181818182e-05,
      "loss": 2.7827,
      "step": 186800
    },
    {
      "activations/layer0_attention_weight_max": 15.409452438354492,
      "activations/layer0_attention_weight_min": -13.210042953491211,
      "activations/layer10_attention_weight_max": 31.477294921875,
      "activations/layer10_attention_weight_min": -31.166349411010742,
      "activations/layer11_attention_weight_max": 31.94536590576172,
      "activations/layer11_attention_weight_min": -30.371414184570312,
      "activations/layer12_attention_weight_max": 23.77617645263672,
      "activations/layer12_attention_weight_min": -25.485706329345703,
      "activations/layer13_attention_weight_max": 38.179405212402344,
      "activations/layer13_attention_weight_min": -35.85205841064453,
      "activations/layer14_attention_weight_max": 42.88811111450195,
      "activations/layer14_attention_weight_min": -38.48008346557617,
      "activations/layer15_attention_weight_max": 35.088321685791016,
      "activations/layer15_attention_weight_min": -31.31149673461914,
      "activations/layer16_attention_weight_max": 29.2282772064209,
      "activations/layer16_attention_weight_min": -27.882919311523438,
      "activations/layer17_attention_weight_max": 31.82807731628418,
      "activations/layer17_attention_weight_min": -26.792945861816406,
      "activations/layer18_attention_weight_max": 31.54010009765625,
      "activations/layer18_attention_weight_min": -25.53945541381836,
      "activations/layer19_attention_weight_max": 33.061710357666016,
      "activations/layer19_attention_weight_min": -31.34235954284668,
      "activations/layer1_attention_weight_max": 15.663741111755371,
      "activations/layer1_attention_weight_min": -14.820951461791992,
      "activations/layer20_attention_weight_max": 29.594846725463867,
      "activations/layer20_attention_weight_min": -23.753984451293945,
      "activations/layer21_attention_weight_max": 31.070011138916016,
      "activations/layer21_attention_weight_min": -24.552379608154297,
      "activations/layer22_attention_weight_max": 42.09817123413086,
      "activations/layer22_attention_weight_min": -30.647579193115234,
      "activations/layer23_attention_weight_max": 36.977962493896484,
      "activations/layer23_attention_weight_min": -23.838478088378906,
      "activations/layer2_attention_weight_max": 31.953975677490234,
      "activations/layer2_attention_weight_min": -33.130531311035156,
      "activations/layer3_attention_weight_max": 96.52935791015625,
      "activations/layer3_attention_weight_min": -97.5520248413086,
      "activations/layer4_attention_weight_max": 116.45263671875,
      "activations/layer4_attention_weight_min": -113.01692199707031,
      "activations/layer5_attention_weight_max": 51.58222198486328,
      "activations/layer5_attention_weight_min": -62.11147689819336,
      "activations/layer6_attention_weight_max": 42.99756622314453,
      "activations/layer6_attention_weight_min": -47.959190368652344,
      "activations/layer7_attention_weight_max": 91.61317443847656,
      "activations/layer7_attention_weight_min": -91.4894027709961,
      "activations/layer8_attention_weight_max": 38.8978271484375,
      "activations/layer8_attention_weight_min": -39.82927322387695,
      "activations/layer9_attention_weight_max": 32.46745300292969,
      "activations/layer9_attention_weight_min": -31.634519577026367,
      "epoch": 10.86,
      "learning_rate": 8.07992424242424e-05,
      "loss": 2.8039,
      "step": 186850
    },
    {
      "activations/layer0_attention_weight_max": 14.405041694641113,
      "activations/layer0_attention_weight_min": -11.943671226501465,
      "activations/layer10_attention_weight_max": 32.732337951660156,
      "activations/layer10_attention_weight_min": -33.5439453125,
      "activations/layer11_attention_weight_max": 33.34039306640625,
      "activations/layer11_attention_weight_min": -32.92958068847656,
      "activations/layer12_attention_weight_max": 23.85248565673828,
      "activations/layer12_attention_weight_min": -27.093341827392578,
      "activations/layer13_attention_weight_max": 40.4455680847168,
      "activations/layer13_attention_weight_min": -31.91092872619629,
      "activations/layer14_attention_weight_max": 45.57448196411133,
      "activations/layer14_attention_weight_min": -40.23229217529297,
      "activations/layer15_attention_weight_max": 39.45564651489258,
      "activations/layer15_attention_weight_min": -30.95583724975586,
      "activations/layer16_attention_weight_max": 33.5516242980957,
      "activations/layer16_attention_weight_min": -28.33013343811035,
      "activations/layer17_attention_weight_max": 33.51736068725586,
      "activations/layer17_attention_weight_min": -26.4528751373291,
      "activations/layer18_attention_weight_max": 30.738107681274414,
      "activations/layer18_attention_weight_min": -23.47640609741211,
      "activations/layer19_attention_weight_max": 38.009117126464844,
      "activations/layer19_attention_weight_min": -30.677331924438477,
      "activations/layer1_attention_weight_max": 16.687026977539062,
      "activations/layer1_attention_weight_min": -15.324629783630371,
      "activations/layer20_attention_weight_max": 28.695018768310547,
      "activations/layer20_attention_weight_min": -25.743364334106445,
      "activations/layer21_attention_weight_max": 29.18455696105957,
      "activations/layer21_attention_weight_min": -24.672027587890625,
      "activations/layer22_attention_weight_max": 48.06724548339844,
      "activations/layer22_attention_weight_min": -30.650102615356445,
      "activations/layer23_attention_weight_max": 33.67705535888672,
      "activations/layer23_attention_weight_min": -25.951210021972656,
      "activations/layer2_attention_weight_max": 32.59001922607422,
      "activations/layer2_attention_weight_min": -33.447940826416016,
      "activations/layer3_attention_weight_max": 93.61762237548828,
      "activations/layer3_attention_weight_min": -93.56388092041016,
      "activations/layer4_attention_weight_max": 113.9865951538086,
      "activations/layer4_attention_weight_min": -112.83658599853516,
      "activations/layer5_attention_weight_max": 56.00370788574219,
      "activations/layer5_attention_weight_min": -61.94252014160156,
      "activations/layer6_attention_weight_max": 44.87120819091797,
      "activations/layer6_attention_weight_min": -47.112430572509766,
      "activations/layer7_attention_weight_max": 95.85655975341797,
      "activations/layer7_attention_weight_min": -91.05269622802734,
      "activations/layer8_attention_weight_max": 40.6541633605957,
      "activations/layer8_attention_weight_min": -42.30781936645508,
      "activations/layer9_attention_weight_max": 34.371788024902344,
      "activations/layer9_attention_weight_min": -34.01967239379883,
      "epoch": 10.86,
      "learning_rate": 8.078030303030302e-05,
      "loss": 2.786,
      "step": 186900
    },
    {
      "activations/layer0_attention_weight_max": 14.639498710632324,
      "activations/layer0_attention_weight_min": -13.285797119140625,
      "activations/layer10_attention_weight_max": 32.48881912231445,
      "activations/layer10_attention_weight_min": -32.520042419433594,
      "activations/layer11_attention_weight_max": 34.16392517089844,
      "activations/layer11_attention_weight_min": -31.993877410888672,
      "activations/layer12_attention_weight_max": 25.863492965698242,
      "activations/layer12_attention_weight_min": -25.446617126464844,
      "activations/layer13_attention_weight_max": 41.83937072753906,
      "activations/layer13_attention_weight_min": -34.554649353027344,
      "activations/layer14_attention_weight_max": 45.62294006347656,
      "activations/layer14_attention_weight_min": -38.64992141723633,
      "activations/layer15_attention_weight_max": 36.42161560058594,
      "activations/layer15_attention_weight_min": -31.243690490722656,
      "activations/layer16_attention_weight_max": 33.19878005981445,
      "activations/layer16_attention_weight_min": -28.587121963500977,
      "activations/layer17_attention_weight_max": 31.298444747924805,
      "activations/layer17_attention_weight_min": -25.77715492248535,
      "activations/layer18_attention_weight_max": 30.92955207824707,
      "activations/layer18_attention_weight_min": -25.489479064941406,
      "activations/layer19_attention_weight_max": 33.54952621459961,
      "activations/layer19_attention_weight_min": -32.96153259277344,
      "activations/layer1_attention_weight_max": 16.242870330810547,
      "activations/layer1_attention_weight_min": -14.813972473144531,
      "activations/layer20_attention_weight_max": 28.4703311920166,
      "activations/layer20_attention_weight_min": -25.434648513793945,
      "activations/layer21_attention_weight_max": 27.38871955871582,
      "activations/layer21_attention_weight_min": -23.99977684020996,
      "activations/layer22_attention_weight_max": 48.720481872558594,
      "activations/layer22_attention_weight_min": -30.145187377929688,
      "activations/layer23_attention_weight_max": 35.012237548828125,
      "activations/layer23_attention_weight_min": -24.67922592163086,
      "activations/layer2_attention_weight_max": 34.55760192871094,
      "activations/layer2_attention_weight_min": -34.95928955078125,
      "activations/layer3_attention_weight_max": 99.3095932006836,
      "activations/layer3_attention_weight_min": -104.19392395019531,
      "activations/layer4_attention_weight_max": 112.94378662109375,
      "activations/layer4_attention_weight_min": -114.54827117919922,
      "activations/layer5_attention_weight_max": 52.815773010253906,
      "activations/layer5_attention_weight_min": -59.44883728027344,
      "activations/layer6_attention_weight_max": 45.34101867675781,
      "activations/layer6_attention_weight_min": -48.88343048095703,
      "activations/layer7_attention_weight_max": 89.03482055664062,
      "activations/layer7_attention_weight_min": -96.07725524902344,
      "activations/layer8_attention_weight_max": 40.44400405883789,
      "activations/layer8_attention_weight_min": -42.44546890258789,
      "activations/layer9_attention_weight_max": 34.51273727416992,
      "activations/layer9_attention_weight_min": -33.44576644897461,
      "epoch": 10.86,
      "learning_rate": 8.076136363636364e-05,
      "loss": 2.7818,
      "step": 186950
    },
    {
      "activations/layer0_attention_weight_max": 15.853581428527832,
      "activations/layer0_attention_weight_min": -12.013936996459961,
      "activations/layer10_attention_weight_max": 31.715286254882812,
      "activations/layer10_attention_weight_min": -33.95316696166992,
      "activations/layer11_attention_weight_max": 32.3049201965332,
      "activations/layer11_attention_weight_min": -32.286041259765625,
      "activations/layer12_attention_weight_max": 24.01202964782715,
      "activations/layer12_attention_weight_min": -29.70319938659668,
      "activations/layer13_attention_weight_max": 38.45110321044922,
      "activations/layer13_attention_weight_min": -32.53792953491211,
      "activations/layer14_attention_weight_max": 41.116695404052734,
      "activations/layer14_attention_weight_min": -36.1053581237793,
      "activations/layer15_attention_weight_max": 34.36386489868164,
      "activations/layer15_attention_weight_min": -31.728721618652344,
      "activations/layer16_attention_weight_max": 30.112899780273438,
      "activations/layer16_attention_weight_min": -25.753726959228516,
      "activations/layer17_attention_weight_max": 29.724077224731445,
      "activations/layer17_attention_weight_min": -24.59211540222168,
      "activations/layer18_attention_weight_max": 28.366281509399414,
      "activations/layer18_attention_weight_min": -22.9412841796875,
      "activations/layer19_attention_weight_max": 32.229915618896484,
      "activations/layer19_attention_weight_min": -29.934972763061523,
      "activations/layer1_attention_weight_max": 16.565526962280273,
      "activations/layer1_attention_weight_min": -13.375568389892578,
      "activations/layer20_attention_weight_max": 29.756561279296875,
      "activations/layer20_attention_weight_min": -24.18425178527832,
      "activations/layer21_attention_weight_max": 28.666215896606445,
      "activations/layer21_attention_weight_min": -22.96453285217285,
      "activations/layer22_attention_weight_max": 46.142642974853516,
      "activations/layer22_attention_weight_min": -28.226526260375977,
      "activations/layer23_attention_weight_max": 35.19093322753906,
      "activations/layer23_attention_weight_min": -23.74799156188965,
      "activations/layer2_attention_weight_max": 29.497060775756836,
      "activations/layer2_attention_weight_min": -29.968048095703125,
      "activations/layer3_attention_weight_max": 88.6029052734375,
      "activations/layer3_attention_weight_min": -94.39623260498047,
      "activations/layer4_attention_weight_max": 112.20660400390625,
      "activations/layer4_attention_weight_min": -112.5809097290039,
      "activations/layer5_attention_weight_max": 55.07831954956055,
      "activations/layer5_attention_weight_min": -62.369667053222656,
      "activations/layer6_attention_weight_max": 45.356380462646484,
      "activations/layer6_attention_weight_min": -51.1172981262207,
      "activations/layer7_attention_weight_max": 92.8006820678711,
      "activations/layer7_attention_weight_min": -91.4136962890625,
      "activations/layer8_attention_weight_max": 40.92549133300781,
      "activations/layer8_attention_weight_min": -41.30656814575195,
      "activations/layer9_attention_weight_max": 34.05751419067383,
      "activations/layer9_attention_weight_min": -33.257484436035156,
      "epoch": 10.87,
      "learning_rate": 8.074242424242422e-05,
      "loss": 2.786,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5157,
      "eval_samples_per_second": 504.243,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5157,
      "eval_openwebtext_samples_per_second": 504.243,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0234,
      "eval_wikitext_samples_per_second": 225.365,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_lambada_loss": 2.53515625,
      "eval_lambada_ppl": 12.618402319374093,
      "eval_lambada_runtime": 9.5921,
      "eval_lambada_samples_per_second": 507.604,
      "step": 187000
    },
    {
      "activations/layer0_attention_weight_max": 15.352737426757812,
      "activations/layer0_attention_weight_min": -12.223037719726562,
      "activations/layer10_attention_weight_max": 31.638608932495117,
      "activations/layer10_attention_weight_min": -33.317901611328125,
      "activations/layer11_attention_weight_max": 30.493635177612305,
      "activations/layer11_attention_weight_min": -30.45418357849121,
      "activations/layer12_attention_weight_max": 24.0773983001709,
      "activations/layer12_attention_weight_min": -22.735069274902344,
      "activations/layer13_attention_weight_max": 38.761962890625,
      "activations/layer13_attention_weight_min": -31.700075149536133,
      "activations/layer14_attention_weight_max": 42.933353424072266,
      "activations/layer14_attention_weight_min": -34.30929183959961,
      "activations/layer15_attention_weight_max": 37.281471252441406,
      "activations/layer15_attention_weight_min": -31.089078903198242,
      "activations/layer16_attention_weight_max": 29.785690307617188,
      "activations/layer16_attention_weight_min": -26.04063606262207,
      "activations/layer17_attention_weight_max": 32.13066482543945,
      "activations/layer17_attention_weight_min": -24.823896408081055,
      "activations/layer18_attention_weight_max": 29.21928596496582,
      "activations/layer18_attention_weight_min": -22.899702072143555,
      "activations/layer19_attention_weight_max": 35.250244140625,
      "activations/layer19_attention_weight_min": -30.67372703552246,
      "activations/layer1_attention_weight_max": 16.66436195373535,
      "activations/layer1_attention_weight_min": -14.451340675354004,
      "activations/layer20_attention_weight_max": 31.525840759277344,
      "activations/layer20_attention_weight_min": -25.858890533447266,
      "activations/layer21_attention_weight_max": 31.17057228088379,
      "activations/layer21_attention_weight_min": -23.333391189575195,
      "activations/layer22_attention_weight_max": 42.85383605957031,
      "activations/layer22_attention_weight_min": -28.12442398071289,
      "activations/layer23_attention_weight_max": 32.04145050048828,
      "activations/layer23_attention_weight_min": -23.620208740234375,
      "activations/layer2_attention_weight_max": 31.62396812438965,
      "activations/layer2_attention_weight_min": -32.10350799560547,
      "activations/layer3_attention_weight_max": 93.81616973876953,
      "activations/layer3_attention_weight_min": -94.3659896850586,
      "activations/layer4_attention_weight_max": 114.08491516113281,
      "activations/layer4_attention_weight_min": -109.60396575927734,
      "activations/layer5_attention_weight_max": 49.57548141479492,
      "activations/layer5_attention_weight_min": -62.96412658691406,
      "activations/layer6_attention_weight_max": 42.99916076660156,
      "activations/layer6_attention_weight_min": -47.886417388916016,
      "activations/layer7_attention_weight_max": 84.98480987548828,
      "activations/layer7_attention_weight_min": -86.79419708251953,
      "activations/layer8_attention_weight_max": 41.77378845214844,
      "activations/layer8_attention_weight_min": -38.83363723754883,
      "activations/layer9_attention_weight_max": 32.688926696777344,
      "activations/layer9_attention_weight_min": -32.886810302734375,
      "epoch": 10.87,
      "learning_rate": 8.072348484848484e-05,
      "loss": 2.7985,
      "step": 187050
    },
    {
      "activations/layer0_attention_weight_max": 16.182661056518555,
      "activations/layer0_attention_weight_min": -12.596260070800781,
      "activations/layer10_attention_weight_max": 34.72818374633789,
      "activations/layer10_attention_weight_min": -33.84991455078125,
      "activations/layer11_attention_weight_max": 33.45812225341797,
      "activations/layer11_attention_weight_min": -31.531095504760742,
      "activations/layer12_attention_weight_max": 25.525054931640625,
      "activations/layer12_attention_weight_min": -24.59585952758789,
      "activations/layer13_attention_weight_max": 40.52020263671875,
      "activations/layer13_attention_weight_min": -32.810577392578125,
      "activations/layer14_attention_weight_max": 42.88327407836914,
      "activations/layer14_attention_weight_min": -34.9459114074707,
      "activations/layer15_attention_weight_max": 39.614070892333984,
      "activations/layer15_attention_weight_min": -30.19056510925293,
      "activations/layer16_attention_weight_max": 29.01650047302246,
      "activations/layer16_attention_weight_min": -28.051361083984375,
      "activations/layer17_attention_weight_max": 30.625864028930664,
      "activations/layer17_attention_weight_min": -25.35896873474121,
      "activations/layer18_attention_weight_max": 30.200363159179688,
      "activations/layer18_attention_weight_min": -22.300317764282227,
      "activations/layer19_attention_weight_max": 32.967830657958984,
      "activations/layer19_attention_weight_min": -30.78382682800293,
      "activations/layer1_attention_weight_max": 16.110897064208984,
      "activations/layer1_attention_weight_min": -15.722261428833008,
      "activations/layer20_attention_weight_max": 29.767366409301758,
      "activations/layer20_attention_weight_min": -24.30414581298828,
      "activations/layer21_attention_weight_max": 28.368911743164062,
      "activations/layer21_attention_weight_min": -22.489084243774414,
      "activations/layer22_attention_weight_max": 44.14310073852539,
      "activations/layer22_attention_weight_min": -30.199068069458008,
      "activations/layer23_attention_weight_max": 35.45464324951172,
      "activations/layer23_attention_weight_min": -24.995407104492188,
      "activations/layer2_attention_weight_max": 32.025604248046875,
      "activations/layer2_attention_weight_min": -33.16724395751953,
      "activations/layer3_attention_weight_max": 95.13166046142578,
      "activations/layer3_attention_weight_min": -94.60363006591797,
      "activations/layer4_attention_weight_max": 113.90413665771484,
      "activations/layer4_attention_weight_min": -112.66023254394531,
      "activations/layer5_attention_weight_max": 55.84563446044922,
      "activations/layer5_attention_weight_min": -60.92157745361328,
      "activations/layer6_attention_weight_max": 48.018062591552734,
      "activations/layer6_attention_weight_min": -48.76508331298828,
      "activations/layer7_attention_weight_max": 94.88106536865234,
      "activations/layer7_attention_weight_min": -92.02436065673828,
      "activations/layer8_attention_weight_max": 41.0472526550293,
      "activations/layer8_attention_weight_min": -42.67725372314453,
      "activations/layer9_attention_weight_max": 34.38560104370117,
      "activations/layer9_attention_weight_min": -34.01691818237305,
      "epoch": 10.87,
      "learning_rate": 8.070454545454545e-05,
      "loss": 2.7863,
      "step": 187100
    },
    {
      "activations/layer0_attention_weight_max": 15.118546485900879,
      "activations/layer0_attention_weight_min": -13.027318000793457,
      "activations/layer10_attention_weight_max": 35.69905471801758,
      "activations/layer10_attention_weight_min": -33.25238800048828,
      "activations/layer11_attention_weight_max": 33.79419708251953,
      "activations/layer11_attention_weight_min": -33.19017791748047,
      "activations/layer12_attention_weight_max": 26.309040069580078,
      "activations/layer12_attention_weight_min": -24.51929473876953,
      "activations/layer13_attention_weight_max": 40.24135971069336,
      "activations/layer13_attention_weight_min": -34.517425537109375,
      "activations/layer14_attention_weight_max": 51.0888786315918,
      "activations/layer14_attention_weight_min": -39.204219818115234,
      "activations/layer15_attention_weight_max": 36.57522201538086,
      "activations/layer15_attention_weight_min": -32.691829681396484,
      "activations/layer16_attention_weight_max": 29.85440444946289,
      "activations/layer16_attention_weight_min": -27.848440170288086,
      "activations/layer17_attention_weight_max": 31.63319206237793,
      "activations/layer17_attention_weight_min": -25.10344886779785,
      "activations/layer18_attention_weight_max": 30.638919830322266,
      "activations/layer18_attention_weight_min": -23.12129783630371,
      "activations/layer19_attention_weight_max": 34.509334564208984,
      "activations/layer19_attention_weight_min": -29.495254516601562,
      "activations/layer1_attention_weight_max": 15.98050594329834,
      "activations/layer1_attention_weight_min": -16.25792694091797,
      "activations/layer20_attention_weight_max": 28.923864364624023,
      "activations/layer20_attention_weight_min": -25.704151153564453,
      "activations/layer21_attention_weight_max": 26.928863525390625,
      "activations/layer21_attention_weight_min": -23.932449340820312,
      "activations/layer22_attention_weight_max": 42.3974494934082,
      "activations/layer22_attention_weight_min": -31.99247932434082,
      "activations/layer23_attention_weight_max": 33.94364929199219,
      "activations/layer23_attention_weight_min": -23.931867599487305,
      "activations/layer2_attention_weight_max": 32.1494026184082,
      "activations/layer2_attention_weight_min": -31.151792526245117,
      "activations/layer3_attention_weight_max": 95.4365005493164,
      "activations/layer3_attention_weight_min": -95.47447204589844,
      "activations/layer4_attention_weight_max": 116.00040435791016,
      "activations/layer4_attention_weight_min": -112.7088851928711,
      "activations/layer5_attention_weight_max": 54.914276123046875,
      "activations/layer5_attention_weight_min": -62.96549987792969,
      "activations/layer6_attention_weight_max": 46.24333190917969,
      "activations/layer6_attention_weight_min": -46.922340393066406,
      "activations/layer7_attention_weight_max": 90.5982437133789,
      "activations/layer7_attention_weight_min": -90.96792602539062,
      "activations/layer8_attention_weight_max": 41.37718963623047,
      "activations/layer8_attention_weight_min": -41.53518295288086,
      "activations/layer9_attention_weight_max": 35.992149353027344,
      "activations/layer9_attention_weight_min": -35.735130310058594,
      "epoch": 10.87,
      "learning_rate": 8.068560606060606e-05,
      "loss": 2.7923,
      "step": 187150
    },
    {
      "activations/layer0_attention_weight_max": 15.812175750732422,
      "activations/layer0_attention_weight_min": -12.977296829223633,
      "activations/layer10_attention_weight_max": 31.680376052856445,
      "activations/layer10_attention_weight_min": -32.2974853515625,
      "activations/layer11_attention_weight_max": 32.21978759765625,
      "activations/layer11_attention_weight_min": -31.283275604248047,
      "activations/layer12_attention_weight_max": 26.04730987548828,
      "activations/layer12_attention_weight_min": -25.096288681030273,
      "activations/layer13_attention_weight_max": 41.162227630615234,
      "activations/layer13_attention_weight_min": -34.245201110839844,
      "activations/layer14_attention_weight_max": 43.580238342285156,
      "activations/layer14_attention_weight_min": -38.373592376708984,
      "activations/layer15_attention_weight_max": 36.84063720703125,
      "activations/layer15_attention_weight_min": -31.089162826538086,
      "activations/layer16_attention_weight_max": 31.217533111572266,
      "activations/layer16_attention_weight_min": -27.05506706237793,
      "activations/layer17_attention_weight_max": 31.62030792236328,
      "activations/layer17_attention_weight_min": -25.027647018432617,
      "activations/layer18_attention_weight_max": 31.634197235107422,
      "activations/layer18_attention_weight_min": -22.58772087097168,
      "activations/layer19_attention_weight_max": 36.000972747802734,
      "activations/layer19_attention_weight_min": -29.2181396484375,
      "activations/layer1_attention_weight_max": 18.254749298095703,
      "activations/layer1_attention_weight_min": -14.936415672302246,
      "activations/layer20_attention_weight_max": 30.718528747558594,
      "activations/layer20_attention_weight_min": -23.739118576049805,
      "activations/layer21_attention_weight_max": 30.85854721069336,
      "activations/layer21_attention_weight_min": -23.58058738708496,
      "activations/layer22_attention_weight_max": 47.26739501953125,
      "activations/layer22_attention_weight_min": -29.38062858581543,
      "activations/layer23_attention_weight_max": 36.52178192138672,
      "activations/layer23_attention_weight_min": -23.686824798583984,
      "activations/layer2_attention_weight_max": 33.243221282958984,
      "activations/layer2_attention_weight_min": -33.01333236694336,
      "activations/layer3_attention_weight_max": 97.63838958740234,
      "activations/layer3_attention_weight_min": -97.21568298339844,
      "activations/layer4_attention_weight_max": 114.09895324707031,
      "activations/layer4_attention_weight_min": -115.04886627197266,
      "activations/layer5_attention_weight_max": 52.706512451171875,
      "activations/layer5_attention_weight_min": -66.95677185058594,
      "activations/layer6_attention_weight_max": 44.92911911010742,
      "activations/layer6_attention_weight_min": -48.745033264160156,
      "activations/layer7_attention_weight_max": 92.9573745727539,
      "activations/layer7_attention_weight_min": -84.94825744628906,
      "activations/layer8_attention_weight_max": 39.07087326049805,
      "activations/layer8_attention_weight_min": -39.69673538208008,
      "activations/layer9_attention_weight_max": 31.5998592376709,
      "activations/layer9_attention_weight_min": -35.0535774230957,
      "epoch": 10.88,
      "learning_rate": 8.066666666666666e-05,
      "loss": 2.7832,
      "step": 187200
    },
    {
      "activations/layer0_attention_weight_max": 15.433953285217285,
      "activations/layer0_attention_weight_min": -12.21656608581543,
      "activations/layer10_attention_weight_max": 31.42636489868164,
      "activations/layer10_attention_weight_min": -31.22293472290039,
      "activations/layer11_attention_weight_max": 31.973674774169922,
      "activations/layer11_attention_weight_min": -31.885887145996094,
      "activations/layer12_attention_weight_max": 24.959884643554688,
      "activations/layer12_attention_weight_min": -25.564208984375,
      "activations/layer13_attention_weight_max": 38.791961669921875,
      "activations/layer13_attention_weight_min": -34.47557067871094,
      "activations/layer14_attention_weight_max": 44.62782669067383,
      "activations/layer14_attention_weight_min": -40.84035110473633,
      "activations/layer15_attention_weight_max": 36.765132904052734,
      "activations/layer15_attention_weight_min": -31.85082244873047,
      "activations/layer16_attention_weight_max": 32.66840362548828,
      "activations/layer16_attention_weight_min": -27.832963943481445,
      "activations/layer17_attention_weight_max": 30.035661697387695,
      "activations/layer17_attention_weight_min": -26.499135971069336,
      "activations/layer18_attention_weight_max": 30.389034271240234,
      "activations/layer18_attention_weight_min": -22.371183395385742,
      "activations/layer19_attention_weight_max": 36.008846282958984,
      "activations/layer19_attention_weight_min": -30.515838623046875,
      "activations/layer1_attention_weight_max": 16.629392623901367,
      "activations/layer1_attention_weight_min": -15.188777923583984,
      "activations/layer20_attention_weight_max": 28.246002197265625,
      "activations/layer20_attention_weight_min": -23.412189483642578,
      "activations/layer21_attention_weight_max": 28.290634155273438,
      "activations/layer21_attention_weight_min": -23.193330764770508,
      "activations/layer22_attention_weight_max": 45.652896881103516,
      "activations/layer22_attention_weight_min": -30.973251342773438,
      "activations/layer23_attention_weight_max": 32.790321350097656,
      "activations/layer23_attention_weight_min": -24.977346420288086,
      "activations/layer2_attention_weight_max": 33.541709899902344,
      "activations/layer2_attention_weight_min": -34.7587890625,
      "activations/layer3_attention_weight_max": 98.39360046386719,
      "activations/layer3_attention_weight_min": -103.25765228271484,
      "activations/layer4_attention_weight_max": 114.45342254638672,
      "activations/layer4_attention_weight_min": -116.84883880615234,
      "activations/layer5_attention_weight_max": 54.033775329589844,
      "activations/layer5_attention_weight_min": -63.688995361328125,
      "activations/layer6_attention_weight_max": 45.752723693847656,
      "activations/layer6_attention_weight_min": -49.16379165649414,
      "activations/layer7_attention_weight_max": 90.39628601074219,
      "activations/layer7_attention_weight_min": -86.37582397460938,
      "activations/layer8_attention_weight_max": 39.47311782836914,
      "activations/layer8_attention_weight_min": -39.840152740478516,
      "activations/layer9_attention_weight_max": 31.665618896484375,
      "activations/layer9_attention_weight_min": -32.85175323486328,
      "epoch": 10.88,
      "learning_rate": 8.064772727272726e-05,
      "loss": 2.7858,
      "step": 187250
    },
    {
      "activations/layer0_attention_weight_max": 15.498504638671875,
      "activations/layer0_attention_weight_min": -12.409383773803711,
      "activations/layer10_attention_weight_max": 31.98018455505371,
      "activations/layer10_attention_weight_min": -35.21272659301758,
      "activations/layer11_attention_weight_max": 33.72574996948242,
      "activations/layer11_attention_weight_min": -32.40149688720703,
      "activations/layer12_attention_weight_max": 25.520912170410156,
      "activations/layer12_attention_weight_min": -24.187820434570312,
      "activations/layer13_attention_weight_max": 40.146705627441406,
      "activations/layer13_attention_weight_min": -34.64128112792969,
      "activations/layer14_attention_weight_max": 51.97697448730469,
      "activations/layer14_attention_weight_min": -39.95612335205078,
      "activations/layer15_attention_weight_max": 40.624061584472656,
      "activations/layer15_attention_weight_min": -31.85663604736328,
      "activations/layer16_attention_weight_max": 37.88470458984375,
      "activations/layer16_attention_weight_min": -27.98185920715332,
      "activations/layer17_attention_weight_max": 35.050819396972656,
      "activations/layer17_attention_weight_min": -25.24469566345215,
      "activations/layer18_attention_weight_max": 36.571163177490234,
      "activations/layer18_attention_weight_min": -25.659029006958008,
      "activations/layer19_attention_weight_max": 39.287837982177734,
      "activations/layer19_attention_weight_min": -31.54864501953125,
      "activations/layer1_attention_weight_max": 17.503847122192383,
      "activations/layer1_attention_weight_min": -16.089956283569336,
      "activations/layer20_attention_weight_max": 34.74820327758789,
      "activations/layer20_attention_weight_min": -26.719663619995117,
      "activations/layer21_attention_weight_max": 34.67963409423828,
      "activations/layer21_attention_weight_min": -25.284141540527344,
      "activations/layer22_attention_weight_max": 49.06206130981445,
      "activations/layer22_attention_weight_min": -32.53845977783203,
      "activations/layer23_attention_weight_max": 43.139495849609375,
      "activations/layer23_attention_weight_min": -26.121387481689453,
      "activations/layer2_attention_weight_max": 33.18857192993164,
      "activations/layer2_attention_weight_min": -32.16863250732422,
      "activations/layer3_attention_weight_max": 91.828369140625,
      "activations/layer3_attention_weight_min": -93.44725799560547,
      "activations/layer4_attention_weight_max": 109.57313537597656,
      "activations/layer4_attention_weight_min": -110.16336822509766,
      "activations/layer5_attention_weight_max": 50.512142181396484,
      "activations/layer5_attention_weight_min": -62.62944030761719,
      "activations/layer6_attention_weight_max": 46.79215621948242,
      "activations/layer6_attention_weight_min": -47.235958099365234,
      "activations/layer7_attention_weight_max": 91.5663833618164,
      "activations/layer7_attention_weight_min": -89.91384887695312,
      "activations/layer8_attention_weight_max": 39.60451889038086,
      "activations/layer8_attention_weight_min": -40.560916900634766,
      "activations/layer9_attention_weight_max": 33.71567916870117,
      "activations/layer9_attention_weight_min": -33.05343246459961,
      "epoch": 10.88,
      "learning_rate": 8.062878787878787e-05,
      "loss": 2.8012,
      "step": 187300
    },
    {
      "activations/layer0_attention_weight_max": 16.02050018310547,
      "activations/layer0_attention_weight_min": -11.921608924865723,
      "activations/layer10_attention_weight_max": 36.165733337402344,
      "activations/layer10_attention_weight_min": -32.01438522338867,
      "activations/layer11_attention_weight_max": 34.73019027709961,
      "activations/layer11_attention_weight_min": -31.47975730895996,
      "activations/layer12_attention_weight_max": 25.4467830657959,
      "activations/layer12_attention_weight_min": -24.964359283447266,
      "activations/layer13_attention_weight_max": 38.859901428222656,
      "activations/layer13_attention_weight_min": -34.07757568359375,
      "activations/layer14_attention_weight_max": 45.89116287231445,
      "activations/layer14_attention_weight_min": -40.36079788208008,
      "activations/layer15_attention_weight_max": 36.63855743408203,
      "activations/layer15_attention_weight_min": -31.404333114624023,
      "activations/layer16_attention_weight_max": 32.93830871582031,
      "activations/layer16_attention_weight_min": -28.004602432250977,
      "activations/layer17_attention_weight_max": 33.78673553466797,
      "activations/layer17_attention_weight_min": -28.19685935974121,
      "activations/layer18_attention_weight_max": 33.36985778808594,
      "activations/layer18_attention_weight_min": -23.957725524902344,
      "activations/layer19_attention_weight_max": 34.792938232421875,
      "activations/layer19_attention_weight_min": -29.22906494140625,
      "activations/layer1_attention_weight_max": 16.77421760559082,
      "activations/layer1_attention_weight_min": -14.87997817993164,
      "activations/layer20_attention_weight_max": 30.218936920166016,
      "activations/layer20_attention_weight_min": -25.49652671813965,
      "activations/layer21_attention_weight_max": 30.992149353027344,
      "activations/layer21_attention_weight_min": -23.280242919921875,
      "activations/layer22_attention_weight_max": 45.23957824707031,
      "activations/layer22_attention_weight_min": -30.90856170654297,
      "activations/layer23_attention_weight_max": 40.28785705566406,
      "activations/layer23_attention_weight_min": -23.06475830078125,
      "activations/layer2_attention_weight_max": 32.750755310058594,
      "activations/layer2_attention_weight_min": -33.61029052734375,
      "activations/layer3_attention_weight_max": 94.41368865966797,
      "activations/layer3_attention_weight_min": -96.30554962158203,
      "activations/layer4_attention_weight_max": 111.71173095703125,
      "activations/layer4_attention_weight_min": -111.2398910522461,
      "activations/layer5_attention_weight_max": 58.62702560424805,
      "activations/layer5_attention_weight_min": -62.46259307861328,
      "activations/layer6_attention_weight_max": 47.096778869628906,
      "activations/layer6_attention_weight_min": -47.9056282043457,
      "activations/layer7_attention_weight_max": 88.04357147216797,
      "activations/layer7_attention_weight_min": -88.34828186035156,
      "activations/layer8_attention_weight_max": 38.49091339111328,
      "activations/layer8_attention_weight_min": -41.124778747558594,
      "activations/layer9_attention_weight_max": 35.91162872314453,
      "activations/layer9_attention_weight_min": -32.4739875793457,
      "epoch": 10.89,
      "learning_rate": 8.060984848484849e-05,
      "loss": 2.789,
      "step": 187350
    },
    {
      "activations/layer0_attention_weight_max": 16.021703720092773,
      "activations/layer0_attention_weight_min": -12.985846519470215,
      "activations/layer10_attention_weight_max": 34.65161895751953,
      "activations/layer10_attention_weight_min": -33.25493621826172,
      "activations/layer11_attention_weight_max": 33.855770111083984,
      "activations/layer11_attention_weight_min": -31.889156341552734,
      "activations/layer12_attention_weight_max": 25.35586166381836,
      "activations/layer12_attention_weight_min": -28.069093704223633,
      "activations/layer13_attention_weight_max": 39.61315155029297,
      "activations/layer13_attention_weight_min": -36.06254196166992,
      "activations/layer14_attention_weight_max": 49.62800979614258,
      "activations/layer14_attention_weight_min": -42.15627670288086,
      "activations/layer15_attention_weight_max": 36.5568962097168,
      "activations/layer15_attention_weight_min": -34.553260803222656,
      "activations/layer16_attention_weight_max": 32.91339874267578,
      "activations/layer16_attention_weight_min": -29.62742805480957,
      "activations/layer17_attention_weight_max": 34.988800048828125,
      "activations/layer17_attention_weight_min": -27.379009246826172,
      "activations/layer18_attention_weight_max": 32.57054138183594,
      "activations/layer18_attention_weight_min": -24.437843322753906,
      "activations/layer19_attention_weight_max": 38.25901794433594,
      "activations/layer19_attention_weight_min": -30.797245025634766,
      "activations/layer1_attention_weight_max": 16.865320205688477,
      "activations/layer1_attention_weight_min": -16.26657485961914,
      "activations/layer20_attention_weight_max": 33.87580108642578,
      "activations/layer20_attention_weight_min": -23.537471771240234,
      "activations/layer21_attention_weight_max": 33.26137161254883,
      "activations/layer21_attention_weight_min": -23.722803115844727,
      "activations/layer22_attention_weight_max": 54.86238479614258,
      "activations/layer22_attention_weight_min": -28.601696014404297,
      "activations/layer23_attention_weight_max": 40.98670959472656,
      "activations/layer23_attention_weight_min": -24.450382232666016,
      "activations/layer2_attention_weight_max": 33.70893859863281,
      "activations/layer2_attention_weight_min": -33.988800048828125,
      "activations/layer3_attention_weight_max": 97.47601318359375,
      "activations/layer3_attention_weight_min": -100.1914291381836,
      "activations/layer4_attention_weight_max": 120.07374572753906,
      "activations/layer4_attention_weight_min": -115.34685516357422,
      "activations/layer5_attention_weight_max": 57.433929443359375,
      "activations/layer5_attention_weight_min": -62.95745849609375,
      "activations/layer6_attention_weight_max": 47.620121002197266,
      "activations/layer6_attention_weight_min": -51.064979553222656,
      "activations/layer7_attention_weight_max": 94.41484069824219,
      "activations/layer7_attention_weight_min": -94.24994659423828,
      "activations/layer8_attention_weight_max": 42.35047149658203,
      "activations/layer8_attention_weight_min": -41.81297302246094,
      "activations/layer9_attention_weight_max": 37.068599700927734,
      "activations/layer9_attention_weight_min": -34.36855697631836,
      "epoch": 10.89,
      "learning_rate": 8.059090909090908e-05,
      "loss": 2.7931,
      "step": 187400
    },
    {
      "activations/layer0_attention_weight_max": 15.340177536010742,
      "activations/layer0_attention_weight_min": -12.056546211242676,
      "activations/layer10_attention_weight_max": 35.3779296875,
      "activations/layer10_attention_weight_min": -33.85255813598633,
      "activations/layer11_attention_weight_max": 33.785152435302734,
      "activations/layer11_attention_weight_min": -33.03717803955078,
      "activations/layer12_attention_weight_max": 23.84140968322754,
      "activations/layer12_attention_weight_min": -24.3027400970459,
      "activations/layer13_attention_weight_max": 37.39931869506836,
      "activations/layer13_attention_weight_min": -33.78776168823242,
      "activations/layer14_attention_weight_max": 41.82744598388672,
      "activations/layer14_attention_weight_min": -35.61679458618164,
      "activations/layer15_attention_weight_max": 36.59236526489258,
      "activations/layer15_attention_weight_min": -31.386808395385742,
      "activations/layer16_attention_weight_max": 31.690563201904297,
      "activations/layer16_attention_weight_min": -28.814958572387695,
      "activations/layer17_attention_weight_max": 29.96441650390625,
      "activations/layer17_attention_weight_min": -25.22342872619629,
      "activations/layer18_attention_weight_max": 30.87643051147461,
      "activations/layer18_attention_weight_min": -21.60480499267578,
      "activations/layer19_attention_weight_max": 37.50118637084961,
      "activations/layer19_attention_weight_min": -30.333005905151367,
      "activations/layer1_attention_weight_max": 16.739761352539062,
      "activations/layer1_attention_weight_min": -17.016403198242188,
      "activations/layer20_attention_weight_max": 29.702526092529297,
      "activations/layer20_attention_weight_min": -24.054912567138672,
      "activations/layer21_attention_weight_max": 27.828752517700195,
      "activations/layer21_attention_weight_min": -23.263141632080078,
      "activations/layer22_attention_weight_max": 43.723514556884766,
      "activations/layer22_attention_weight_min": -27.71109962463379,
      "activations/layer23_attention_weight_max": 35.171356201171875,
      "activations/layer23_attention_weight_min": -26.605695724487305,
      "activations/layer2_attention_weight_max": 32.84360122680664,
      "activations/layer2_attention_weight_min": -32.60482406616211,
      "activations/layer3_attention_weight_max": 95.5307846069336,
      "activations/layer3_attention_weight_min": -96.9958267211914,
      "activations/layer4_attention_weight_max": 121.34429168701172,
      "activations/layer4_attention_weight_min": -113.48731994628906,
      "activations/layer5_attention_weight_max": 54.87628173828125,
      "activations/layer5_attention_weight_min": -65.88600158691406,
      "activations/layer6_attention_weight_max": 44.88662338256836,
      "activations/layer6_attention_weight_min": -51.42918014526367,
      "activations/layer7_attention_weight_max": 101.28841400146484,
      "activations/layer7_attention_weight_min": -96.39120483398438,
      "activations/layer8_attention_weight_max": 42.16838836669922,
      "activations/layer8_attention_weight_min": -40.87958526611328,
      "activations/layer9_attention_weight_max": 35.98594284057617,
      "activations/layer9_attention_weight_min": -34.98082733154297,
      "epoch": 10.89,
      "learning_rate": 8.057196969696969e-05,
      "loss": 2.7847,
      "step": 187450
    },
    {
      "activations/layer0_attention_weight_max": 14.542073249816895,
      "activations/layer0_attention_weight_min": -12.152154922485352,
      "activations/layer10_attention_weight_max": 34.34111785888672,
      "activations/layer10_attention_weight_min": -33.64362335205078,
      "activations/layer11_attention_weight_max": 33.32063293457031,
      "activations/layer11_attention_weight_min": -30.798444747924805,
      "activations/layer12_attention_weight_max": 26.028766632080078,
      "activations/layer12_attention_weight_min": -23.742877960205078,
      "activations/layer13_attention_weight_max": 38.73347473144531,
      "activations/layer13_attention_weight_min": -34.27364730834961,
      "activations/layer14_attention_weight_max": 44.46794509887695,
      "activations/layer14_attention_weight_min": -37.74323272705078,
      "activations/layer15_attention_weight_max": 34.84469223022461,
      "activations/layer15_attention_weight_min": -30.273822784423828,
      "activations/layer16_attention_weight_max": 29.26717758178711,
      "activations/layer16_attention_weight_min": -29.217945098876953,
      "activations/layer17_attention_weight_max": 32.37471389770508,
      "activations/layer17_attention_weight_min": -24.969959259033203,
      "activations/layer18_attention_weight_max": 30.916011810302734,
      "activations/layer18_attention_weight_min": -21.721214294433594,
      "activations/layer19_attention_weight_max": 31.53070068359375,
      "activations/layer19_attention_weight_min": -30.257919311523438,
      "activations/layer1_attention_weight_max": 16.347627639770508,
      "activations/layer1_attention_weight_min": -14.834134101867676,
      "activations/layer20_attention_weight_max": 29.412132263183594,
      "activations/layer20_attention_weight_min": -24.11855125427246,
      "activations/layer21_attention_weight_max": 27.870670318603516,
      "activations/layer21_attention_weight_min": -22.614713668823242,
      "activations/layer22_attention_weight_max": 42.71531295776367,
      "activations/layer22_attention_weight_min": -27.38260269165039,
      "activations/layer23_attention_weight_max": 33.089412689208984,
      "activations/layer23_attention_weight_min": -26.176387786865234,
      "activations/layer2_attention_weight_max": 31.484235763549805,
      "activations/layer2_attention_weight_min": -31.700916290283203,
      "activations/layer3_attention_weight_max": 95.87374114990234,
      "activations/layer3_attention_weight_min": -94.91291809082031,
      "activations/layer4_attention_weight_max": 112.94881439208984,
      "activations/layer4_attention_weight_min": -112.6336441040039,
      "activations/layer5_attention_weight_max": 51.75364685058594,
      "activations/layer5_attention_weight_min": -63.55068588256836,
      "activations/layer6_attention_weight_max": 44.1791877746582,
      "activations/layer6_attention_weight_min": -50.3744010925293,
      "activations/layer7_attention_weight_max": 86.87677001953125,
      "activations/layer7_attention_weight_min": -88.31309509277344,
      "activations/layer8_attention_weight_max": 44.1716194152832,
      "activations/layer8_attention_weight_min": -40.802188873291016,
      "activations/layer9_attention_weight_max": 34.163150787353516,
      "activations/layer9_attention_weight_min": -33.21358108520508,
      "epoch": 10.89,
      "learning_rate": 8.055303030303031e-05,
      "loss": 2.7813,
      "step": 187500
    },
    {
      "activations/layer0_attention_weight_max": 15.60046100616455,
      "activations/layer0_attention_weight_min": -11.824929237365723,
      "activations/layer10_attention_weight_max": 30.79745101928711,
      "activations/layer10_attention_weight_min": -31.333984375,
      "activations/layer11_attention_weight_max": 30.2293701171875,
      "activations/layer11_attention_weight_min": -29.900775909423828,
      "activations/layer12_attention_weight_max": 24.75149917602539,
      "activations/layer12_attention_weight_min": -24.19786834716797,
      "activations/layer13_attention_weight_max": 36.51620101928711,
      "activations/layer13_attention_weight_min": -32.08551025390625,
      "activations/layer14_attention_weight_max": 40.06391525268555,
      "activations/layer14_attention_weight_min": -41.56358337402344,
      "activations/layer15_attention_weight_max": 36.48320770263672,
      "activations/layer15_attention_weight_min": -32.53921127319336,
      "activations/layer16_attention_weight_max": 29.44340705871582,
      "activations/layer16_attention_weight_min": -26.738622665405273,
      "activations/layer17_attention_weight_max": 30.687429428100586,
      "activations/layer17_attention_weight_min": -25.051712036132812,
      "activations/layer18_attention_weight_max": 29.166831970214844,
      "activations/layer18_attention_weight_min": -23.57364273071289,
      "activations/layer19_attention_weight_max": 34.39497756958008,
      "activations/layer19_attention_weight_min": -30.596954345703125,
      "activations/layer1_attention_weight_max": 16.11821937561035,
      "activations/layer1_attention_weight_min": -14.045525550842285,
      "activations/layer20_attention_weight_max": 27.812965393066406,
      "activations/layer20_attention_weight_min": -25.475139617919922,
      "activations/layer21_attention_weight_max": 27.74289894104004,
      "activations/layer21_attention_weight_min": -23.56667137145996,
      "activations/layer22_attention_weight_max": 39.965599060058594,
      "activations/layer22_attention_weight_min": -30.67250633239746,
      "activations/layer23_attention_weight_max": 31.704910278320312,
      "activations/layer23_attention_weight_min": -22.306867599487305,
      "activations/layer2_attention_weight_max": 33.57707595825195,
      "activations/layer2_attention_weight_min": -33.46771240234375,
      "activations/layer3_attention_weight_max": 96.56182861328125,
      "activations/layer3_attention_weight_min": -98.62783813476562,
      "activations/layer4_attention_weight_max": 112.37284088134766,
      "activations/layer4_attention_weight_min": -113.1063461303711,
      "activations/layer5_attention_weight_max": 53.28020477294922,
      "activations/layer5_attention_weight_min": -63.259971618652344,
      "activations/layer6_attention_weight_max": 43.658729553222656,
      "activations/layer6_attention_weight_min": -50.36097717285156,
      "activations/layer7_attention_weight_max": 91.7667236328125,
      "activations/layer7_attention_weight_min": -95.69062805175781,
      "activations/layer8_attention_weight_max": 37.79481887817383,
      "activations/layer8_attention_weight_min": -40.79013442993164,
      "activations/layer9_attention_weight_max": 30.831298828125,
      "activations/layer9_attention_weight_min": -32.28913879394531,
      "epoch": 10.9,
      "learning_rate": 8.05340909090909e-05,
      "loss": 2.7758,
      "step": 187550
    },
    {
      "activations/layer0_attention_weight_max": 15.563763618469238,
      "activations/layer0_attention_weight_min": -12.643034934997559,
      "activations/layer10_attention_weight_max": 35.67460632324219,
      "activations/layer10_attention_weight_min": -33.283409118652344,
      "activations/layer11_attention_weight_max": 35.47941207885742,
      "activations/layer11_attention_weight_min": -34.7181396484375,
      "activations/layer12_attention_weight_max": 28.284080505371094,
      "activations/layer12_attention_weight_min": -26.70596694946289,
      "activations/layer13_attention_weight_max": 43.10847091674805,
      "activations/layer13_attention_weight_min": -33.3945198059082,
      "activations/layer14_attention_weight_max": 40.73456573486328,
      "activations/layer14_attention_weight_min": -36.172698974609375,
      "activations/layer15_attention_weight_max": 34.54930114746094,
      "activations/layer15_attention_weight_min": -33.26974868774414,
      "activations/layer16_attention_weight_max": 30.88570213317871,
      "activations/layer16_attention_weight_min": -29.72995376586914,
      "activations/layer17_attention_weight_max": 32.956092834472656,
      "activations/layer17_attention_weight_min": -25.70169448852539,
      "activations/layer18_attention_weight_max": 32.43583679199219,
      "activations/layer18_attention_weight_min": -22.35764503479004,
      "activations/layer19_attention_weight_max": 36.54206848144531,
      "activations/layer19_attention_weight_min": -32.06044387817383,
      "activations/layer1_attention_weight_max": 17.345115661621094,
      "activations/layer1_attention_weight_min": -14.920101165771484,
      "activations/layer20_attention_weight_max": 31.425352096557617,
      "activations/layer20_attention_weight_min": -24.948040008544922,
      "activations/layer21_attention_weight_max": 32.50110626220703,
      "activations/layer21_attention_weight_min": -24.0689697265625,
      "activations/layer22_attention_weight_max": 45.468719482421875,
      "activations/layer22_attention_weight_min": -29.72108268737793,
      "activations/layer23_attention_weight_max": 35.315338134765625,
      "activations/layer23_attention_weight_min": -23.68497085571289,
      "activations/layer2_attention_weight_max": 34.058326721191406,
      "activations/layer2_attention_weight_min": -35.4848747253418,
      "activations/layer3_attention_weight_max": 101.9695053100586,
      "activations/layer3_attention_weight_min": -109.37576293945312,
      "activations/layer4_attention_weight_max": 114.84941864013672,
      "activations/layer4_attention_weight_min": -121.10621643066406,
      "activations/layer5_attention_weight_max": 53.711082458496094,
      "activations/layer5_attention_weight_min": -61.231624603271484,
      "activations/layer6_attention_weight_max": 44.00654220581055,
      "activations/layer6_attention_weight_min": -52.97264862060547,
      "activations/layer7_attention_weight_max": 91.58326721191406,
      "activations/layer7_attention_weight_min": -97.45609283447266,
      "activations/layer8_attention_weight_max": 44.5240478515625,
      "activations/layer8_attention_weight_min": -44.966758728027344,
      "activations/layer9_attention_weight_max": 36.45552062988281,
      "activations/layer9_attention_weight_min": -37.38478469848633,
      "epoch": 10.9,
      "learning_rate": 8.051515151515151e-05,
      "loss": 2.7685,
      "step": 187600
    },
    {
      "activations/layer0_attention_weight_max": 15.129951477050781,
      "activations/layer0_attention_weight_min": -11.66313362121582,
      "activations/layer10_attention_weight_max": 32.425209045410156,
      "activations/layer10_attention_weight_min": -32.84857177734375,
      "activations/layer11_attention_weight_max": 32.46985626220703,
      "activations/layer11_attention_weight_min": -32.77517318725586,
      "activations/layer12_attention_weight_max": 25.967308044433594,
      "activations/layer12_attention_weight_min": -27.156892776489258,
      "activations/layer13_attention_weight_max": 39.42842483520508,
      "activations/layer13_attention_weight_min": -34.084754943847656,
      "activations/layer14_attention_weight_max": 43.346282958984375,
      "activations/layer14_attention_weight_min": -39.446807861328125,
      "activations/layer15_attention_weight_max": 34.43745040893555,
      "activations/layer15_attention_weight_min": -32.27851104736328,
      "activations/layer16_attention_weight_max": 30.939653396606445,
      "activations/layer16_attention_weight_min": -27.127885818481445,
      "activations/layer17_attention_weight_max": 32.32048416137695,
      "activations/layer17_attention_weight_min": -26.33173942565918,
      "activations/layer18_attention_weight_max": 31.413169860839844,
      "activations/layer18_attention_weight_min": -22.53026580810547,
      "activations/layer19_attention_weight_max": 36.38985824584961,
      "activations/layer19_attention_weight_min": -31.655475616455078,
      "activations/layer1_attention_weight_max": 16.813377380371094,
      "activations/layer1_attention_weight_min": -15.355366706848145,
      "activations/layer20_attention_weight_max": 28.696725845336914,
      "activations/layer20_attention_weight_min": -25.567626953125,
      "activations/layer21_attention_weight_max": 29.101194381713867,
      "activations/layer21_attention_weight_min": -23.584379196166992,
      "activations/layer22_attention_weight_max": 43.67062759399414,
      "activations/layer22_attention_weight_min": -27.28963851928711,
      "activations/layer23_attention_weight_max": 31.708602905273438,
      "activations/layer23_attention_weight_min": -21.602741241455078,
      "activations/layer2_attention_weight_max": 33.38005828857422,
      "activations/layer2_attention_weight_min": -34.807552337646484,
      "activations/layer3_attention_weight_max": 97.11711120605469,
      "activations/layer3_attention_weight_min": -100.96121978759766,
      "activations/layer4_attention_weight_max": 111.69606018066406,
      "activations/layer4_attention_weight_min": -110.50240325927734,
      "activations/layer5_attention_weight_max": 51.154457092285156,
      "activations/layer5_attention_weight_min": -58.774169921875,
      "activations/layer6_attention_weight_max": 42.734195709228516,
      "activations/layer6_attention_weight_min": -47.878929138183594,
      "activations/layer7_attention_weight_max": 89.70134735107422,
      "activations/layer7_attention_weight_min": -88.6032485961914,
      "activations/layer8_attention_weight_max": 38.846649169921875,
      "activations/layer8_attention_weight_min": -40.75578689575195,
      "activations/layer9_attention_weight_max": 33.176795959472656,
      "activations/layer9_attention_weight_min": -35.60990524291992,
      "epoch": 10.9,
      "learning_rate": 8.049621212121211e-05,
      "loss": 2.7753,
      "step": 187650
    },
    {
      "activations/layer0_attention_weight_max": 15.055377006530762,
      "activations/layer0_attention_weight_min": -12.882930755615234,
      "activations/layer10_attention_weight_max": 31.158666610717773,
      "activations/layer10_attention_weight_min": -32.22128677368164,
      "activations/layer11_attention_weight_max": 32.1058349609375,
      "activations/layer11_attention_weight_min": -32.04603576660156,
      "activations/layer12_attention_weight_max": 24.6959285736084,
      "activations/layer12_attention_weight_min": -25.302215576171875,
      "activations/layer13_attention_weight_max": 35.7665901184082,
      "activations/layer13_attention_weight_min": -32.05088806152344,
      "activations/layer14_attention_weight_max": 45.15297317504883,
      "activations/layer14_attention_weight_min": -34.68839645385742,
      "activations/layer15_attention_weight_max": 35.20872116088867,
      "activations/layer15_attention_weight_min": -31.191715240478516,
      "activations/layer16_attention_weight_max": 26.160371780395508,
      "activations/layer16_attention_weight_min": -27.72686004638672,
      "activations/layer17_attention_weight_max": 32.788734436035156,
      "activations/layer17_attention_weight_min": -24.390743255615234,
      "activations/layer18_attention_weight_max": 29.56309700012207,
      "activations/layer18_attention_weight_min": -24.500200271606445,
      "activations/layer19_attention_weight_max": 30.28584098815918,
      "activations/layer19_attention_weight_min": -28.5219669342041,
      "activations/layer1_attention_weight_max": 16.765201568603516,
      "activations/layer1_attention_weight_min": -14.246591567993164,
      "activations/layer20_attention_weight_max": 28.891590118408203,
      "activations/layer20_attention_weight_min": -24.8361759185791,
      "activations/layer21_attention_weight_max": 29.71647071838379,
      "activations/layer21_attention_weight_min": -22.718563079833984,
      "activations/layer22_attention_weight_max": 39.20249557495117,
      "activations/layer22_attention_weight_min": -28.97252655029297,
      "activations/layer23_attention_weight_max": 32.79248046875,
      "activations/layer23_attention_weight_min": -23.570585250854492,
      "activations/layer2_attention_weight_max": 33.56980514526367,
      "activations/layer2_attention_weight_min": -33.31190490722656,
      "activations/layer3_attention_weight_max": 92.70783233642578,
      "activations/layer3_attention_weight_min": -97.85733032226562,
      "activations/layer4_attention_weight_max": 110.52445220947266,
      "activations/layer4_attention_weight_min": -111.85062408447266,
      "activations/layer5_attention_weight_max": 49.82554244995117,
      "activations/layer5_attention_weight_min": -60.264259338378906,
      "activations/layer6_attention_weight_max": 43.83859634399414,
      "activations/layer6_attention_weight_min": -46.22282028198242,
      "activations/layer7_attention_weight_max": 88.48809051513672,
      "activations/layer7_attention_weight_min": -89.06501770019531,
      "activations/layer8_attention_weight_max": 40.30302810668945,
      "activations/layer8_attention_weight_min": -39.69694137573242,
      "activations/layer9_attention_weight_max": 31.401212692260742,
      "activations/layer9_attention_weight_min": -32.227500915527344,
      "epoch": 10.91,
      "learning_rate": 8.047765151515152e-05,
      "loss": 2.773,
      "step": 187700
    },
    {
      "activations/layer0_attention_weight_max": 15.664820671081543,
      "activations/layer0_attention_weight_min": -12.946967124938965,
      "activations/layer10_attention_weight_max": 33.57405090332031,
      "activations/layer10_attention_weight_min": -31.524770736694336,
      "activations/layer11_attention_weight_max": 35.15484619140625,
      "activations/layer11_attention_weight_min": -33.553955078125,
      "activations/layer12_attention_weight_max": 22.942583084106445,
      "activations/layer12_attention_weight_min": -23.888126373291016,
      "activations/layer13_attention_weight_max": 35.480106353759766,
      "activations/layer13_attention_weight_min": -34.49906921386719,
      "activations/layer14_attention_weight_max": 46.93075180053711,
      "activations/layer14_attention_weight_min": -37.162105560302734,
      "activations/layer15_attention_weight_max": 32.98883056640625,
      "activations/layer15_attention_weight_min": -32.079593658447266,
      "activations/layer16_attention_weight_max": 29.275789260864258,
      "activations/layer16_attention_weight_min": -29.88337516784668,
      "activations/layer17_attention_weight_max": 31.981252670288086,
      "activations/layer17_attention_weight_min": -26.941057205200195,
      "activations/layer18_attention_weight_max": 31.712350845336914,
      "activations/layer18_attention_weight_min": -24.157819747924805,
      "activations/layer19_attention_weight_max": 34.028316497802734,
      "activations/layer19_attention_weight_min": -31.3826847076416,
      "activations/layer1_attention_weight_max": 16.94734001159668,
      "activations/layer1_attention_weight_min": -14.264431953430176,
      "activations/layer20_attention_weight_max": 29.412498474121094,
      "activations/layer20_attention_weight_min": -23.61802864074707,
      "activations/layer21_attention_weight_max": 32.077152252197266,
      "activations/layer21_attention_weight_min": -23.741483688354492,
      "activations/layer22_attention_weight_max": 42.21715545654297,
      "activations/layer22_attention_weight_min": -29.455793380737305,
      "activations/layer23_attention_weight_max": 34.201515197753906,
      "activations/layer23_attention_weight_min": -22.80457878112793,
      "activations/layer2_attention_weight_max": 34.24072265625,
      "activations/layer2_attention_weight_min": -33.92803192138672,
      "activations/layer3_attention_weight_max": 97.52174377441406,
      "activations/layer3_attention_weight_min": -99.63568115234375,
      "activations/layer4_attention_weight_max": 114.061767578125,
      "activations/layer4_attention_weight_min": -113.12052154541016,
      "activations/layer5_attention_weight_max": 54.358062744140625,
      "activations/layer5_attention_weight_min": -63.77448272705078,
      "activations/layer6_attention_weight_max": 45.29639434814453,
      "activations/layer6_attention_weight_min": -50.860984802246094,
      "activations/layer7_attention_weight_max": 93.51256561279297,
      "activations/layer7_attention_weight_min": -93.1419906616211,
      "activations/layer8_attention_weight_max": 43.30662536621094,
      "activations/layer8_attention_weight_min": -40.776947021484375,
      "activations/layer9_attention_weight_max": 33.3381462097168,
      "activations/layer9_attention_weight_min": -33.76547622680664,
      "epoch": 10.91,
      "learning_rate": 8.04587121212121e-05,
      "loss": 2.7878,
      "step": 187750
    },
    {
      "activations/layer0_attention_weight_max": 16.18765640258789,
      "activations/layer0_attention_weight_min": -13.565947532653809,
      "activations/layer10_attention_weight_max": 31.939083099365234,
      "activations/layer10_attention_weight_min": -32.15868377685547,
      "activations/layer11_attention_weight_max": 32.001487731933594,
      "activations/layer11_attention_weight_min": -32.10120391845703,
      "activations/layer12_attention_weight_max": 26.701391220092773,
      "activations/layer12_attention_weight_min": -27.1426944732666,
      "activations/layer13_attention_weight_max": 39.09404373168945,
      "activations/layer13_attention_weight_min": -35.337135314941406,
      "activations/layer14_attention_weight_max": 45.55326461791992,
      "activations/layer14_attention_weight_min": -38.86623001098633,
      "activations/layer15_attention_weight_max": 37.03328323364258,
      "activations/layer15_attention_weight_min": -34.20063400268555,
      "activations/layer16_attention_weight_max": 31.084936141967773,
      "activations/layer16_attention_weight_min": -29.631132125854492,
      "activations/layer17_attention_weight_max": 34.695674896240234,
      "activations/layer17_attention_weight_min": -25.394100189208984,
      "activations/layer18_attention_weight_max": 32.41632843017578,
      "activations/layer18_attention_weight_min": -23.087297439575195,
      "activations/layer19_attention_weight_max": 40.29336166381836,
      "activations/layer19_attention_weight_min": -31.675939559936523,
      "activations/layer1_attention_weight_max": 16.52833366394043,
      "activations/layer1_attention_weight_min": -16.173860549926758,
      "activations/layer20_attention_weight_max": 31.845195770263672,
      "activations/layer20_attention_weight_min": -25.91571044921875,
      "activations/layer21_attention_weight_max": 37.03190994262695,
      "activations/layer21_attention_weight_min": -25.21746253967285,
      "activations/layer22_attention_weight_max": 54.00620651245117,
      "activations/layer22_attention_weight_min": -29.661518096923828,
      "activations/layer23_attention_weight_max": 39.0216064453125,
      "activations/layer23_attention_weight_min": -23.33131980895996,
      "activations/layer2_attention_weight_max": 34.147438049316406,
      "activations/layer2_attention_weight_min": -32.98185348510742,
      "activations/layer3_attention_weight_max": 96.06315612792969,
      "activations/layer3_attention_weight_min": -95.3868637084961,
      "activations/layer4_attention_weight_max": 113.1470947265625,
      "activations/layer4_attention_weight_min": -113.89676666259766,
      "activations/layer5_attention_weight_max": 52.26302719116211,
      "activations/layer5_attention_weight_min": -65.242919921875,
      "activations/layer6_attention_weight_max": 46.3886604309082,
      "activations/layer6_attention_weight_min": -47.78853225708008,
      "activations/layer7_attention_weight_max": 91.41999053955078,
      "activations/layer7_attention_weight_min": -87.85301208496094,
      "activations/layer8_attention_weight_max": 39.864532470703125,
      "activations/layer8_attention_weight_min": -41.93978500366211,
      "activations/layer9_attention_weight_max": 33.14761734008789,
      "activations/layer9_attention_weight_min": -33.57281494140625,
      "epoch": 10.91,
      "learning_rate": 8.043977272727272e-05,
      "loss": 2.7776,
      "step": 187800
    },
    {
      "activations/layer0_attention_weight_max": 16.280256271362305,
      "activations/layer0_attention_weight_min": -11.873270034790039,
      "activations/layer10_attention_weight_max": 35.10383605957031,
      "activations/layer10_attention_weight_min": -32.438968658447266,
      "activations/layer11_attention_weight_max": 33.63224411010742,
      "activations/layer11_attention_weight_min": -31.601621627807617,
      "activations/layer12_attention_weight_max": 26.865854263305664,
      "activations/layer12_attention_weight_min": -25.967777252197266,
      "activations/layer13_attention_weight_max": 41.724822998046875,
      "activations/layer13_attention_weight_min": -36.41199493408203,
      "activations/layer14_attention_weight_max": 44.95188903808594,
      "activations/layer14_attention_weight_min": -40.009639739990234,
      "activations/layer15_attention_weight_max": 36.24220657348633,
      "activations/layer15_attention_weight_min": -32.248653411865234,
      "activations/layer16_attention_weight_max": 33.064125061035156,
      "activations/layer16_attention_weight_min": -27.941604614257812,
      "activations/layer17_attention_weight_max": 30.96366310119629,
      "activations/layer17_attention_weight_min": -28.21808624267578,
      "activations/layer18_attention_weight_max": 32.406673431396484,
      "activations/layer18_attention_weight_min": -26.23395538330078,
      "activations/layer19_attention_weight_max": 41.6952018737793,
      "activations/layer19_attention_weight_min": -33.454776763916016,
      "activations/layer1_attention_weight_max": 16.815338134765625,
      "activations/layer1_attention_weight_min": -15.641786575317383,
      "activations/layer20_attention_weight_max": 31.74385643005371,
      "activations/layer20_attention_weight_min": -25.42503547668457,
      "activations/layer21_attention_weight_max": 29.789024353027344,
      "activations/layer21_attention_weight_min": -25.10336685180664,
      "activations/layer22_attention_weight_max": 46.525726318359375,
      "activations/layer22_attention_weight_min": -31.48799705505371,
      "activations/layer23_attention_weight_max": 36.39635467529297,
      "activations/layer23_attention_weight_min": -24.990032196044922,
      "activations/layer2_attention_weight_max": 30.10680389404297,
      "activations/layer2_attention_weight_min": -30.978702545166016,
      "activations/layer3_attention_weight_max": 85.47111511230469,
      "activations/layer3_attention_weight_min": -87.788818359375,
      "activations/layer4_attention_weight_max": 107.6882553100586,
      "activations/layer4_attention_weight_min": -107.56988525390625,
      "activations/layer5_attention_weight_max": 49.58843231201172,
      "activations/layer5_attention_weight_min": -59.30408477783203,
      "activations/layer6_attention_weight_max": 45.66505813598633,
      "activations/layer6_attention_weight_min": -49.389122009277344,
      "activations/layer7_attention_weight_max": 85.16593933105469,
      "activations/layer7_attention_weight_min": -96.1552963256836,
      "activations/layer8_attention_weight_max": 40.24607849121094,
      "activations/layer8_attention_weight_min": -42.0485954284668,
      "activations/layer9_attention_weight_max": 34.22806930541992,
      "activations/layer9_attention_weight_min": -33.435638427734375,
      "epoch": 10.92,
      "learning_rate": 8.042083333333333e-05,
      "loss": 2.7963,
      "step": 187850
    },
    {
      "activations/layer0_attention_weight_max": 15.629048347473145,
      "activations/layer0_attention_weight_min": -13.051029205322266,
      "activations/layer10_attention_weight_max": 32.06108474731445,
      "activations/layer10_attention_weight_min": -32.557838439941406,
      "activations/layer11_attention_weight_max": 32.64764404296875,
      "activations/layer11_attention_weight_min": -31.596294403076172,
      "activations/layer12_attention_weight_max": 26.541183471679688,
      "activations/layer12_attention_weight_min": -27.84514045715332,
      "activations/layer13_attention_weight_max": 39.05583953857422,
      "activations/layer13_attention_weight_min": -36.979530334472656,
      "activations/layer14_attention_weight_max": 48.48676681518555,
      "activations/layer14_attention_weight_min": -42.57319259643555,
      "activations/layer15_attention_weight_max": 37.708641052246094,
      "activations/layer15_attention_weight_min": -36.40255355834961,
      "activations/layer16_attention_weight_max": 31.27726173400879,
      "activations/layer16_attention_weight_min": -29.05925941467285,
      "activations/layer17_attention_weight_max": 34.457881927490234,
      "activations/layer17_attention_weight_min": -27.843101501464844,
      "activations/layer18_attention_weight_max": 31.674898147583008,
      "activations/layer18_attention_weight_min": -22.4224853515625,
      "activations/layer19_attention_weight_max": 32.29950714111328,
      "activations/layer19_attention_weight_min": -30.18471336364746,
      "activations/layer1_attention_weight_max": 16.449068069458008,
      "activations/layer1_attention_weight_min": -13.76181411743164,
      "activations/layer20_attention_weight_max": 30.555625915527344,
      "activations/layer20_attention_weight_min": -23.416366577148438,
      "activations/layer21_attention_weight_max": 31.64629364013672,
      "activations/layer21_attention_weight_min": -22.468631744384766,
      "activations/layer22_attention_weight_max": 45.8115348815918,
      "activations/layer22_attention_weight_min": -27.707977294921875,
      "activations/layer23_attention_weight_max": 36.301082611083984,
      "activations/layer23_attention_weight_min": -23.129087448120117,
      "activations/layer2_attention_weight_max": 30.488662719726562,
      "activations/layer2_attention_weight_min": -33.456363677978516,
      "activations/layer3_attention_weight_max": 92.12372589111328,
      "activations/layer3_attention_weight_min": -95.27562713623047,
      "activations/layer4_attention_weight_max": 114.8216323852539,
      "activations/layer4_attention_weight_min": -118.359375,
      "activations/layer5_attention_weight_max": 52.42733383178711,
      "activations/layer5_attention_weight_min": -65.01793670654297,
      "activations/layer6_attention_weight_max": 43.89016342163086,
      "activations/layer6_attention_weight_min": -47.70709991455078,
      "activations/layer7_attention_weight_max": 84.055419921875,
      "activations/layer7_attention_weight_min": -90.77197265625,
      "activations/layer8_attention_weight_max": 40.232547760009766,
      "activations/layer8_attention_weight_min": -39.883235931396484,
      "activations/layer9_attention_weight_max": 35.23089599609375,
      "activations/layer9_attention_weight_min": -33.397701263427734,
      "epoch": 10.92,
      "learning_rate": 8.040189393939394e-05,
      "loss": 2.7879,
      "step": 187900
    },
    {
      "activations/layer0_attention_weight_max": 15.870219230651855,
      "activations/layer0_attention_weight_min": -13.186030387878418,
      "activations/layer10_attention_weight_max": 33.752655029296875,
      "activations/layer10_attention_weight_min": -32.08098220825195,
      "activations/layer11_attention_weight_max": 34.76908874511719,
      "activations/layer11_attention_weight_min": -32.05596160888672,
      "activations/layer12_attention_weight_max": 25.56019401550293,
      "activations/layer12_attention_weight_min": -25.25327491760254,
      "activations/layer13_attention_weight_max": 37.592185974121094,
      "activations/layer13_attention_weight_min": -35.878780364990234,
      "activations/layer14_attention_weight_max": 42.25068283081055,
      "activations/layer14_attention_weight_min": -40.58767318725586,
      "activations/layer15_attention_weight_max": 35.01170349121094,
      "activations/layer15_attention_weight_min": -32.45890808105469,
      "activations/layer16_attention_weight_max": 30.63987922668457,
      "activations/layer16_attention_weight_min": -27.112932205200195,
      "activations/layer17_attention_weight_max": 32.645057678222656,
      "activations/layer17_attention_weight_min": -24.830307006835938,
      "activations/layer18_attention_weight_max": 29.839778900146484,
      "activations/layer18_attention_weight_min": -22.873910903930664,
      "activations/layer19_attention_weight_max": 34.7945556640625,
      "activations/layer19_attention_weight_min": -30.33744239807129,
      "activations/layer1_attention_weight_max": 17.087779998779297,
      "activations/layer1_attention_weight_min": -14.50145435333252,
      "activations/layer20_attention_weight_max": 29.949617385864258,
      "activations/layer20_attention_weight_min": -24.2832088470459,
      "activations/layer21_attention_weight_max": 27.030029296875,
      "activations/layer21_attention_weight_min": -22.038726806640625,
      "activations/layer22_attention_weight_max": 44.37382125854492,
      "activations/layer22_attention_weight_min": -29.562376022338867,
      "activations/layer23_attention_weight_max": 37.448402404785156,
      "activations/layer23_attention_weight_min": -26.28180694580078,
      "activations/layer2_attention_weight_max": 31.793560028076172,
      "activations/layer2_attention_weight_min": -31.25496482849121,
      "activations/layer3_attention_weight_max": 92.92101287841797,
      "activations/layer3_attention_weight_min": -92.22686004638672,
      "activations/layer4_attention_weight_max": 115.81288146972656,
      "activations/layer4_attention_weight_min": -112.69683837890625,
      "activations/layer5_attention_weight_max": 56.1457633972168,
      "activations/layer5_attention_weight_min": -62.81494140625,
      "activations/layer6_attention_weight_max": 47.90238571166992,
      "activations/layer6_attention_weight_min": -49.40320587158203,
      "activations/layer7_attention_weight_max": 93.14067077636719,
      "activations/layer7_attention_weight_min": -92.20661926269531,
      "activations/layer8_attention_weight_max": 41.43053436279297,
      "activations/layer8_attention_weight_min": -40.764305114746094,
      "activations/layer9_attention_weight_max": 34.60297393798828,
      "activations/layer9_attention_weight_min": -32.88521957397461,
      "epoch": 10.92,
      "learning_rate": 8.038295454545454e-05,
      "loss": 2.7791,
      "step": 187950
    },
    {
      "activations/layer0_attention_weight_max": 16.49748420715332,
      "activations/layer0_attention_weight_min": -13.374584197998047,
      "activations/layer10_attention_weight_max": 29.707311630249023,
      "activations/layer10_attention_weight_min": -31.905941009521484,
      "activations/layer11_attention_weight_max": 29.77499008178711,
      "activations/layer11_attention_weight_min": -31.310474395751953,
      "activations/layer12_attention_weight_max": 22.696487426757812,
      "activations/layer12_attention_weight_min": -24.589378356933594,
      "activations/layer13_attention_weight_max": 34.908653259277344,
      "activations/layer13_attention_weight_min": -33.41484069824219,
      "activations/layer14_attention_weight_max": 42.893150329589844,
      "activations/layer14_attention_weight_min": -37.095603942871094,
      "activations/layer15_attention_weight_max": 34.7631950378418,
      "activations/layer15_attention_weight_min": -31.510738372802734,
      "activations/layer16_attention_weight_max": 30.421749114990234,
      "activations/layer16_attention_weight_min": -29.204208374023438,
      "activations/layer17_attention_weight_max": 30.00712776184082,
      "activations/layer17_attention_weight_min": -26.335235595703125,
      "activations/layer18_attention_weight_max": 31.804977416992188,
      "activations/layer18_attention_weight_min": -24.033588409423828,
      "activations/layer19_attention_weight_max": 33.896236419677734,
      "activations/layer19_attention_weight_min": -30.915115356445312,
      "activations/layer1_attention_weight_max": 16.535375595092773,
      "activations/layer1_attention_weight_min": -14.021334648132324,
      "activations/layer20_attention_weight_max": 28.623395919799805,
      "activations/layer20_attention_weight_min": -24.846269607543945,
      "activations/layer21_attention_weight_max": 29.11772918701172,
      "activations/layer21_attention_weight_min": -23.602842330932617,
      "activations/layer22_attention_weight_max": 42.54644775390625,
      "activations/layer22_attention_weight_min": -28.973562240600586,
      "activations/layer23_attention_weight_max": 32.39702606201172,
      "activations/layer23_attention_weight_min": -26.00041961669922,
      "activations/layer2_attention_weight_max": 31.04767608642578,
      "activations/layer2_attention_weight_min": -32.43635177612305,
      "activations/layer3_attention_weight_max": 93.2198715209961,
      "activations/layer3_attention_weight_min": -94.41468811035156,
      "activations/layer4_attention_weight_max": 111.69593811035156,
      "activations/layer4_attention_weight_min": -114.42414855957031,
      "activations/layer5_attention_weight_max": 52.61831283569336,
      "activations/layer5_attention_weight_min": -62.43632507324219,
      "activations/layer6_attention_weight_max": 45.390907287597656,
      "activations/layer6_attention_weight_min": -50.02727127075195,
      "activations/layer7_attention_weight_max": 84.22605895996094,
      "activations/layer7_attention_weight_min": -89.22906494140625,
      "activations/layer8_attention_weight_max": 39.05039596557617,
      "activations/layer8_attention_weight_min": -43.06784439086914,
      "activations/layer9_attention_weight_max": 31.186542510986328,
      "activations/layer9_attention_weight_min": -32.77146911621094,
      "epoch": 10.92,
      "learning_rate": 8.036401515151514e-05,
      "loss": 2.7741,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5233,
      "eval_samples_per_second": 503.793,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5233,
      "eval_openwebtext_samples_per_second": 503.793,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.997,
      "eval_wikitext_samples_per_second": 228.338,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_lambada_loss": 2.541015625,
      "eval_lambada_ppl": 12.692555303381242,
      "eval_lambada_runtime": 9.6151,
      "eval_lambada_samples_per_second": 506.389,
      "step": 188000
    },
    {
      "activations/layer0_attention_weight_max": 15.840331077575684,
      "activations/layer0_attention_weight_min": -12.512357711791992,
      "activations/layer10_attention_weight_max": 34.269630432128906,
      "activations/layer10_attention_weight_min": -34.23456573486328,
      "activations/layer11_attention_weight_max": 36.086246490478516,
      "activations/layer11_attention_weight_min": -35.004154205322266,
      "activations/layer12_attention_weight_max": 26.267013549804688,
      "activations/layer12_attention_weight_min": -26.386619567871094,
      "activations/layer13_attention_weight_max": 37.68647384643555,
      "activations/layer13_attention_weight_min": -35.823020935058594,
      "activations/layer14_attention_weight_max": 43.668212890625,
      "activations/layer14_attention_weight_min": -38.839107513427734,
      "activations/layer15_attention_weight_max": 37.756954193115234,
      "activations/layer15_attention_weight_min": -33.487369537353516,
      "activations/layer16_attention_weight_max": 33.06821823120117,
      "activations/layer16_attention_weight_min": -27.51525115966797,
      "activations/layer17_attention_weight_max": 36.78940963745117,
      "activations/layer17_attention_weight_min": -26.369972229003906,
      "activations/layer18_attention_weight_max": 32.017757415771484,
      "activations/layer18_attention_weight_min": -21.7874698638916,
      "activations/layer19_attention_weight_max": 35.232418060302734,
      "activations/layer19_attention_weight_min": -31.03206443786621,
      "activations/layer1_attention_weight_max": 16.751066207885742,
      "activations/layer1_attention_weight_min": -14.116650581359863,
      "activations/layer20_attention_weight_max": 34.625423431396484,
      "activations/layer20_attention_weight_min": -24.88766098022461,
      "activations/layer21_attention_weight_max": 30.960405349731445,
      "activations/layer21_attention_weight_min": -22.964263916015625,
      "activations/layer22_attention_weight_max": 43.001319885253906,
      "activations/layer22_attention_weight_min": -30.16511344909668,
      "activations/layer23_attention_weight_max": 36.913509368896484,
      "activations/layer23_attention_weight_min": -25.04876708984375,
      "activations/layer2_attention_weight_max": 33.329307556152344,
      "activations/layer2_attention_weight_min": -32.51776885986328,
      "activations/layer3_attention_weight_max": 94.20667266845703,
      "activations/layer3_attention_weight_min": -92.80400848388672,
      "activations/layer4_attention_weight_max": 117.7159423828125,
      "activations/layer4_attention_weight_min": -116.0435791015625,
      "activations/layer5_attention_weight_max": 56.6037712097168,
      "activations/layer5_attention_weight_min": -66.11039733886719,
      "activations/layer6_attention_weight_max": 47.42841720581055,
      "activations/layer6_attention_weight_min": -49.55522155761719,
      "activations/layer7_attention_weight_max": 92.8560562133789,
      "activations/layer7_attention_weight_min": -98.4267349243164,
      "activations/layer8_attention_weight_max": 42.199344635009766,
      "activations/layer8_attention_weight_min": -44.300472259521484,
      "activations/layer9_attention_weight_max": 34.53356170654297,
      "activations/layer9_attention_weight_min": -33.29364013671875,
      "epoch": 10.93,
      "learning_rate": 8.034507575757575e-05,
      "loss": 2.7882,
      "step": 188050
    },
    {
      "activations/layer0_attention_weight_max": 17.615835189819336,
      "activations/layer0_attention_weight_min": -12.930974960327148,
      "activations/layer10_attention_weight_max": 37.02556228637695,
      "activations/layer10_attention_weight_min": -34.49498748779297,
      "activations/layer11_attention_weight_max": 37.22670364379883,
      "activations/layer11_attention_weight_min": -35.01710510253906,
      "activations/layer12_attention_weight_max": 26.350360870361328,
      "activations/layer12_attention_weight_min": -26.376482009887695,
      "activations/layer13_attention_weight_max": 39.784156799316406,
      "activations/layer13_attention_weight_min": -36.976524353027344,
      "activations/layer14_attention_weight_max": 50.966148376464844,
      "activations/layer14_attention_weight_min": -41.1141242980957,
      "activations/layer15_attention_weight_max": 36.086055755615234,
      "activations/layer15_attention_weight_min": -31.309633255004883,
      "activations/layer16_attention_weight_max": 32.697967529296875,
      "activations/layer16_attention_weight_min": -28.150747299194336,
      "activations/layer17_attention_weight_max": 31.696651458740234,
      "activations/layer17_attention_weight_min": -25.508447647094727,
      "activations/layer18_attention_weight_max": 32.650238037109375,
      "activations/layer18_attention_weight_min": -23.504192352294922,
      "activations/layer19_attention_weight_max": 39.934574127197266,
      "activations/layer19_attention_weight_min": -32.517147064208984,
      "activations/layer1_attention_weight_max": 16.74622344970703,
      "activations/layer1_attention_weight_min": -15.312479019165039,
      "activations/layer20_attention_weight_max": 30.556915283203125,
      "activations/layer20_attention_weight_min": -25.600467681884766,
      "activations/layer21_attention_weight_max": 31.97164535522461,
      "activations/layer21_attention_weight_min": -25.192211151123047,
      "activations/layer22_attention_weight_max": 46.844444274902344,
      "activations/layer22_attention_weight_min": -30.828136444091797,
      "activations/layer23_attention_weight_max": 36.39629364013672,
      "activations/layer23_attention_weight_min": -24.211151123046875,
      "activations/layer2_attention_weight_max": 32.1216926574707,
      "activations/layer2_attention_weight_min": -32.4437141418457,
      "activations/layer3_attention_weight_max": 92.87332916259766,
      "activations/layer3_attention_weight_min": -94.15277862548828,
      "activations/layer4_attention_weight_max": 113.37279510498047,
      "activations/layer4_attention_weight_min": -109.99005126953125,
      "activations/layer5_attention_weight_max": 54.47911071777344,
      "activations/layer5_attention_weight_min": -62.0930290222168,
      "activations/layer6_attention_weight_max": 45.11701583862305,
      "activations/layer6_attention_weight_min": -48.14268493652344,
      "activations/layer7_attention_weight_max": 90.42095184326172,
      "activations/layer7_attention_weight_min": -90.38471984863281,
      "activations/layer8_attention_weight_max": 43.44688034057617,
      "activations/layer8_attention_weight_min": -42.47966003417969,
      "activations/layer9_attention_weight_max": 36.64486312866211,
      "activations/layer9_attention_weight_min": -34.547183990478516,
      "epoch": 10.93,
      "learning_rate": 8.032613636363636e-05,
      "loss": 2.7738,
      "step": 188100
    },
    {
      "activations/layer0_attention_weight_max": 15.162715911865234,
      "activations/layer0_attention_weight_min": -12.550789833068848,
      "activations/layer10_attention_weight_max": 34.92963409423828,
      "activations/layer10_attention_weight_min": -33.28973388671875,
      "activations/layer11_attention_weight_max": 34.03379440307617,
      "activations/layer11_attention_weight_min": -31.814422607421875,
      "activations/layer12_attention_weight_max": 24.43362045288086,
      "activations/layer12_attention_weight_min": -25.713947296142578,
      "activations/layer13_attention_weight_max": 37.54079055786133,
      "activations/layer13_attention_weight_min": -33.68632125854492,
      "activations/layer14_attention_weight_max": 43.74662780761719,
      "activations/layer14_attention_weight_min": -37.30634689331055,
      "activations/layer15_attention_weight_max": 35.816078186035156,
      "activations/layer15_attention_weight_min": -32.342674255371094,
      "activations/layer16_attention_weight_max": 35.41463088989258,
      "activations/layer16_attention_weight_min": -29.70258903503418,
      "activations/layer17_attention_weight_max": 32.791168212890625,
      "activations/layer17_attention_weight_min": -26.848609924316406,
      "activations/layer18_attention_weight_max": 32.36689376831055,
      "activations/layer18_attention_weight_min": -23.2218074798584,
      "activations/layer19_attention_weight_max": 34.86855697631836,
      "activations/layer19_attention_weight_min": -31.882369995117188,
      "activations/layer1_attention_weight_max": 16.072383880615234,
      "activations/layer1_attention_weight_min": -13.80756950378418,
      "activations/layer20_attention_weight_max": 29.59783935546875,
      "activations/layer20_attention_weight_min": -26.04591941833496,
      "activations/layer21_attention_weight_max": 28.513273239135742,
      "activations/layer21_attention_weight_min": -26.039154052734375,
      "activations/layer22_attention_weight_max": 45.77144241333008,
      "activations/layer22_attention_weight_min": -30.34918975830078,
      "activations/layer23_attention_weight_max": 34.943382263183594,
      "activations/layer23_attention_weight_min": -23.279462814331055,
      "activations/layer2_attention_weight_max": 34.3388671875,
      "activations/layer2_attention_weight_min": -32.49797058105469,
      "activations/layer3_attention_weight_max": 100.66422271728516,
      "activations/layer3_attention_weight_min": -99.22020721435547,
      "activations/layer4_attention_weight_max": 119.52550506591797,
      "activations/layer4_attention_weight_min": -118.6755599975586,
      "activations/layer5_attention_weight_max": 55.559410095214844,
      "activations/layer5_attention_weight_min": -63.258018493652344,
      "activations/layer6_attention_weight_max": 48.38524627685547,
      "activations/layer6_attention_weight_min": -48.61296844482422,
      "activations/layer7_attention_weight_max": 91.43830108642578,
      "activations/layer7_attention_weight_min": -93.97240447998047,
      "activations/layer8_attention_weight_max": 40.27299499511719,
      "activations/layer8_attention_weight_min": -43.322479248046875,
      "activations/layer9_attention_weight_max": 34.974647521972656,
      "activations/layer9_attention_weight_min": -33.441734313964844,
      "epoch": 10.93,
      "learning_rate": 8.030719696969696e-05,
      "loss": 2.7674,
      "step": 188150
    },
    {
      "activations/layer0_attention_weight_max": 14.763701438903809,
      "activations/layer0_attention_weight_min": -12.675496101379395,
      "activations/layer10_attention_weight_max": 32.583106994628906,
      "activations/layer10_attention_weight_min": -32.036109924316406,
      "activations/layer11_attention_weight_max": 31.759532928466797,
      "activations/layer11_attention_weight_min": -30.28414535522461,
      "activations/layer12_attention_weight_max": 24.06757354736328,
      "activations/layer12_attention_weight_min": -26.950828552246094,
      "activations/layer13_attention_weight_max": 34.82197570800781,
      "activations/layer13_attention_weight_min": -33.57378387451172,
      "activations/layer14_attention_weight_max": 45.92185974121094,
      "activations/layer14_attention_weight_min": -35.34050369262695,
      "activations/layer15_attention_weight_max": 33.57588577270508,
      "activations/layer15_attention_weight_min": -29.755939483642578,
      "activations/layer16_attention_weight_max": 31.327608108520508,
      "activations/layer16_attention_weight_min": -26.86190414428711,
      "activations/layer17_attention_weight_max": 31.59416389465332,
      "activations/layer17_attention_weight_min": -27.666751861572266,
      "activations/layer18_attention_weight_max": 30.338125228881836,
      "activations/layer18_attention_weight_min": -24.686140060424805,
      "activations/layer19_attention_weight_max": 35.66083908081055,
      "activations/layer19_attention_weight_min": -31.346548080444336,
      "activations/layer1_attention_weight_max": 17.04351043701172,
      "activations/layer1_attention_weight_min": -14.373555183410645,
      "activations/layer20_attention_weight_max": 31.021202087402344,
      "activations/layer20_attention_weight_min": -25.070510864257812,
      "activations/layer21_attention_weight_max": 28.95165252685547,
      "activations/layer21_attention_weight_min": -24.093725204467773,
      "activations/layer22_attention_weight_max": 47.39509963989258,
      "activations/layer22_attention_weight_min": -32.66647720336914,
      "activations/layer23_attention_weight_max": 36.64022445678711,
      "activations/layer23_attention_weight_min": -26.400184631347656,
      "activations/layer2_attention_weight_max": 31.52485466003418,
      "activations/layer2_attention_weight_min": -32.34455871582031,
      "activations/layer3_attention_weight_max": 93.6143798828125,
      "activations/layer3_attention_weight_min": -94.79183959960938,
      "activations/layer4_attention_weight_max": 108.54923248291016,
      "activations/layer4_attention_weight_min": -112.5621566772461,
      "activations/layer5_attention_weight_max": 49.638118743896484,
      "activations/layer5_attention_weight_min": -58.575279235839844,
      "activations/layer6_attention_weight_max": 42.60545349121094,
      "activations/layer6_attention_weight_min": -46.89552307128906,
      "activations/layer7_attention_weight_max": 84.46314239501953,
      "activations/layer7_attention_weight_min": -88.41620635986328,
      "activations/layer8_attention_weight_max": 38.78688049316406,
      "activations/layer8_attention_weight_min": -41.62299346923828,
      "activations/layer9_attention_weight_max": 31.026769638061523,
      "activations/layer9_attention_weight_min": -32.210933685302734,
      "epoch": 10.94,
      "learning_rate": 8.028825757575757e-05,
      "loss": 2.7864,
      "step": 188200
    },
    {
      "activations/layer0_attention_weight_max": 14.423498153686523,
      "activations/layer0_attention_weight_min": -12.264952659606934,
      "activations/layer10_attention_weight_max": 30.50607681274414,
      "activations/layer10_attention_weight_min": -32.295753479003906,
      "activations/layer11_attention_weight_max": 30.851726531982422,
      "activations/layer11_attention_weight_min": -31.24549102783203,
      "activations/layer12_attention_weight_max": 23.20403480529785,
      "activations/layer12_attention_weight_min": -26.750564575195312,
      "activations/layer13_attention_weight_max": 33.686370849609375,
      "activations/layer13_attention_weight_min": -34.01661682128906,
      "activations/layer14_attention_weight_max": 39.744380950927734,
      "activations/layer14_attention_weight_min": -34.61025619506836,
      "activations/layer15_attention_weight_max": 34.317325592041016,
      "activations/layer15_attention_weight_min": -28.72801399230957,
      "activations/layer16_attention_weight_max": 30.01079750061035,
      "activations/layer16_attention_weight_min": -28.822486877441406,
      "activations/layer17_attention_weight_max": 31.569665908813477,
      "activations/layer17_attention_weight_min": -24.567880630493164,
      "activations/layer18_attention_weight_max": 30.86760902404785,
      "activations/layer18_attention_weight_min": -22.42084503173828,
      "activations/layer19_attention_weight_max": 32.618263244628906,
      "activations/layer19_attention_weight_min": -32.031578063964844,
      "activations/layer1_attention_weight_max": 16.63873863220215,
      "activations/layer1_attention_weight_min": -14.448137283325195,
      "activations/layer20_attention_weight_max": 26.895538330078125,
      "activations/layer20_attention_weight_min": -24.261796951293945,
      "activations/layer21_attention_weight_max": 26.22723960876465,
      "activations/layer21_attention_weight_min": -23.208154678344727,
      "activations/layer22_attention_weight_max": 39.92213439941406,
      "activations/layer22_attention_weight_min": -29.36460304260254,
      "activations/layer23_attention_weight_max": 32.84493637084961,
      "activations/layer23_attention_weight_min": -24.864850997924805,
      "activations/layer2_attention_weight_max": 35.77790069580078,
      "activations/layer2_attention_weight_min": -34.160186767578125,
      "activations/layer3_attention_weight_max": 102.12133026123047,
      "activations/layer3_attention_weight_min": -98.48127746582031,
      "activations/layer4_attention_weight_max": 120.32271575927734,
      "activations/layer4_attention_weight_min": -109.9726791381836,
      "activations/layer5_attention_weight_max": 53.27854537963867,
      "activations/layer5_attention_weight_min": -61.311317443847656,
      "activations/layer6_attention_weight_max": 43.34815979003906,
      "activations/layer6_attention_weight_min": -46.316917419433594,
      "activations/layer7_attention_weight_max": 87.93899536132812,
      "activations/layer7_attention_weight_min": -87.41912841796875,
      "activations/layer8_attention_weight_max": 42.69692611694336,
      "activations/layer8_attention_weight_min": -42.102622985839844,
      "activations/layer9_attention_weight_max": 31.18592643737793,
      "activations/layer9_attention_weight_min": -32.7041130065918,
      "epoch": 10.94,
      "learning_rate": 8.026931818181819e-05,
      "loss": 2.7864,
      "step": 188250
    },
    {
      "activations/layer0_attention_weight_max": 14.916793823242188,
      "activations/layer0_attention_weight_min": -12.164671897888184,
      "activations/layer10_attention_weight_max": 29.647716522216797,
      "activations/layer10_attention_weight_min": -31.179513931274414,
      "activations/layer11_attention_weight_max": 30.939624786376953,
      "activations/layer11_attention_weight_min": -29.840408325195312,
      "activations/layer12_attention_weight_max": 23.311634063720703,
      "activations/layer12_attention_weight_min": -24.000247955322266,
      "activations/layer13_attention_weight_max": 35.7843132019043,
      "activations/layer13_attention_weight_min": -32.229270935058594,
      "activations/layer14_attention_weight_max": 47.49773025512695,
      "activations/layer14_attention_weight_min": -36.017513275146484,
      "activations/layer15_attention_weight_max": 38.372188568115234,
      "activations/layer15_attention_weight_min": -31.454374313354492,
      "activations/layer16_attention_weight_max": 31.225557327270508,
      "activations/layer16_attention_weight_min": -26.68406105041504,
      "activations/layer17_attention_weight_max": 32.250587463378906,
      "activations/layer17_attention_weight_min": -27.2974910736084,
      "activations/layer18_attention_weight_max": 30.542339324951172,
      "activations/layer18_attention_weight_min": -22.875341415405273,
      "activations/layer19_attention_weight_max": 34.77012252807617,
      "activations/layer19_attention_weight_min": -31.02427101135254,
      "activations/layer1_attention_weight_max": 16.786874771118164,
      "activations/layer1_attention_weight_min": -13.924722671508789,
      "activations/layer20_attention_weight_max": 31.141849517822266,
      "activations/layer20_attention_weight_min": -24.96954345703125,
      "activations/layer21_attention_weight_max": 29.29947280883789,
      "activations/layer21_attention_weight_min": -24.051992416381836,
      "activations/layer22_attention_weight_max": 46.64931869506836,
      "activations/layer22_attention_weight_min": -30.318378448486328,
      "activations/layer23_attention_weight_max": 33.116939544677734,
      "activations/layer23_attention_weight_min": -25.440786361694336,
      "activations/layer2_attention_weight_max": 33.32598876953125,
      "activations/layer2_attention_weight_min": -32.80683898925781,
      "activations/layer3_attention_weight_max": 89.51473999023438,
      "activations/layer3_attention_weight_min": -96.3291015625,
      "activations/layer4_attention_weight_max": 106.41168212890625,
      "activations/layer4_attention_weight_min": -106.87721252441406,
      "activations/layer5_attention_weight_max": 55.27108383178711,
      "activations/layer5_attention_weight_min": -61.082794189453125,
      "activations/layer6_attention_weight_max": 43.579811096191406,
      "activations/layer6_attention_weight_min": -48.81562423706055,
      "activations/layer7_attention_weight_max": 84.63920593261719,
      "activations/layer7_attention_weight_min": -93.71879577636719,
      "activations/layer8_attention_weight_max": 37.58212661743164,
      "activations/layer8_attention_weight_min": -40.97963333129883,
      "activations/layer9_attention_weight_max": 28.963071823120117,
      "activations/layer9_attention_weight_min": -31.07767105102539,
      "epoch": 10.94,
      "learning_rate": 8.025037878787877e-05,
      "loss": 2.7795,
      "step": 188300
    },
    {
      "activations/layer0_attention_weight_max": 15.154115676879883,
      "activations/layer0_attention_weight_min": -12.97211742401123,
      "activations/layer10_attention_weight_max": 33.24225616455078,
      "activations/layer10_attention_weight_min": -31.59127426147461,
      "activations/layer11_attention_weight_max": 32.05833435058594,
      "activations/layer11_attention_weight_min": -32.94569396972656,
      "activations/layer12_attention_weight_max": 27.400209426879883,
      "activations/layer12_attention_weight_min": -24.288583755493164,
      "activations/layer13_attention_weight_max": 41.845733642578125,
      "activations/layer13_attention_weight_min": -34.642784118652344,
      "activations/layer14_attention_weight_max": 48.04377746582031,
      "activations/layer14_attention_weight_min": -38.6878547668457,
      "activations/layer15_attention_weight_max": 39.058563232421875,
      "activations/layer15_attention_weight_min": -32.06678009033203,
      "activations/layer16_attention_weight_max": 34.84638214111328,
      "activations/layer16_attention_weight_min": -27.176931381225586,
      "activations/layer17_attention_weight_max": 32.63280487060547,
      "activations/layer17_attention_weight_min": -26.479530334472656,
      "activations/layer18_attention_weight_max": 32.300899505615234,
      "activations/layer18_attention_weight_min": -22.119651794433594,
      "activations/layer19_attention_weight_max": 36.798866271972656,
      "activations/layer19_attention_weight_min": -30.45989418029785,
      "activations/layer1_attention_weight_max": 15.944664001464844,
      "activations/layer1_attention_weight_min": -13.284481048583984,
      "activations/layer20_attention_weight_max": 32.74081802368164,
      "activations/layer20_attention_weight_min": -23.538354873657227,
      "activations/layer21_attention_weight_max": 30.746267318725586,
      "activations/layer21_attention_weight_min": -23.17081069946289,
      "activations/layer22_attention_weight_max": 49.149288177490234,
      "activations/layer22_attention_weight_min": -28.839704513549805,
      "activations/layer23_attention_weight_max": 38.47574996948242,
      "activations/layer23_attention_weight_min": -23.7677059173584,
      "activations/layer2_attention_weight_max": 32.502296447753906,
      "activations/layer2_attention_weight_min": -31.693241119384766,
      "activations/layer3_attention_weight_max": 94.42036437988281,
      "activations/layer3_attention_weight_min": -95.13815307617188,
      "activations/layer4_attention_weight_max": 114.93437957763672,
      "activations/layer4_attention_weight_min": -113.7291030883789,
      "activations/layer5_attention_weight_max": 54.6947135925293,
      "activations/layer5_attention_weight_min": -67.53619384765625,
      "activations/layer6_attention_weight_max": 44.61824035644531,
      "activations/layer6_attention_weight_min": -48.56367492675781,
      "activations/layer7_attention_weight_max": 90.4937744140625,
      "activations/layer7_attention_weight_min": -90.63773345947266,
      "activations/layer8_attention_weight_max": 39.80622482299805,
      "activations/layer8_attention_weight_min": -41.64834213256836,
      "activations/layer9_attention_weight_max": 32.540321350097656,
      "activations/layer9_attention_weight_min": -32.42550277709961,
      "epoch": 10.94,
      "learning_rate": 8.023143939393939e-05,
      "loss": 2.7933,
      "step": 188350
    },
    {
      "activations/layer0_attention_weight_max": 15.136110305786133,
      "activations/layer0_attention_weight_min": -12.63085651397705,
      "activations/layer10_attention_weight_max": 33.026126861572266,
      "activations/layer10_attention_weight_min": -33.117218017578125,
      "activations/layer11_attention_weight_max": 34.554359436035156,
      "activations/layer11_attention_weight_min": -32.29011154174805,
      "activations/layer12_attention_weight_max": 24.73225212097168,
      "activations/layer12_attention_weight_min": -26.758899688720703,
      "activations/layer13_attention_weight_max": 42.4797248840332,
      "activations/layer13_attention_weight_min": -33.78122329711914,
      "activations/layer14_attention_weight_max": 48.35537338256836,
      "activations/layer14_attention_weight_min": -37.86288070678711,
      "activations/layer15_attention_weight_max": 41.58684158325195,
      "activations/layer15_attention_weight_min": -33.16190719604492,
      "activations/layer16_attention_weight_max": 30.434354782104492,
      "activations/layer16_attention_weight_min": -26.875347137451172,
      "activations/layer17_attention_weight_max": 31.787723541259766,
      "activations/layer17_attention_weight_min": -26.621845245361328,
      "activations/layer18_attention_weight_max": 32.679725646972656,
      "activations/layer18_attention_weight_min": -23.628067016601562,
      "activations/layer19_attention_weight_max": 35.1870231628418,
      "activations/layer19_attention_weight_min": -30.547510147094727,
      "activations/layer1_attention_weight_max": 17.30550765991211,
      "activations/layer1_attention_weight_min": -14.092411041259766,
      "activations/layer20_attention_weight_max": 29.84710693359375,
      "activations/layer20_attention_weight_min": -24.107206344604492,
      "activations/layer21_attention_weight_max": 30.341405868530273,
      "activations/layer21_attention_weight_min": -22.582191467285156,
      "activations/layer22_attention_weight_max": 46.45335388183594,
      "activations/layer22_attention_weight_min": -28.350114822387695,
      "activations/layer23_attention_weight_max": 32.21531677246094,
      "activations/layer23_attention_weight_min": -24.709129333496094,
      "activations/layer2_attention_weight_max": 32.265708923339844,
      "activations/layer2_attention_weight_min": -33.91932678222656,
      "activations/layer3_attention_weight_max": 93.86285400390625,
      "activations/layer3_attention_weight_min": -100.9871597290039,
      "activations/layer4_attention_weight_max": 110.60381317138672,
      "activations/layer4_attention_weight_min": -117.27125549316406,
      "activations/layer5_attention_weight_max": 56.32926559448242,
      "activations/layer5_attention_weight_min": -62.16184997558594,
      "activations/layer6_attention_weight_max": 45.043636322021484,
      "activations/layer6_attention_weight_min": -48.20533752441406,
      "activations/layer7_attention_weight_max": 92.50960540771484,
      "activations/layer7_attention_weight_min": -89.04943084716797,
      "activations/layer8_attention_weight_max": 41.93036651611328,
      "activations/layer8_attention_weight_min": -43.37622833251953,
      "activations/layer9_attention_weight_max": 34.972930908203125,
      "activations/layer9_attention_weight_min": -34.42634963989258,
      "epoch": 10.95,
      "learning_rate": 8.021249999999999e-05,
      "loss": 2.7887,
      "step": 188400
    },
    {
      "activations/layer0_attention_weight_max": 15.753323554992676,
      "activations/layer0_attention_weight_min": -13.887231826782227,
      "activations/layer10_attention_weight_max": 39.00657653808594,
      "activations/layer10_attention_weight_min": -37.51380157470703,
      "activations/layer11_attention_weight_max": 41.607784271240234,
      "activations/layer11_attention_weight_min": -36.704917907714844,
      "activations/layer12_attention_weight_max": 31.42870330810547,
      "activations/layer12_attention_weight_min": -27.305978775024414,
      "activations/layer13_attention_weight_max": 47.68608474731445,
      "activations/layer13_attention_weight_min": -37.619972229003906,
      "activations/layer14_attention_weight_max": 54.846412658691406,
      "activations/layer14_attention_weight_min": -39.97972869873047,
      "activations/layer15_attention_weight_max": 47.923133850097656,
      "activations/layer15_attention_weight_min": -36.311119079589844,
      "activations/layer16_attention_weight_max": 39.823116302490234,
      "activations/layer16_attention_weight_min": -28.588523864746094,
      "activations/layer17_attention_weight_max": 39.12259292602539,
      "activations/layer17_attention_weight_min": -27.724430084228516,
      "activations/layer18_attention_weight_max": 36.59048080444336,
      "activations/layer18_attention_weight_min": -25.06940269470215,
      "activations/layer19_attention_weight_max": 42.50782775878906,
      "activations/layer19_attention_weight_min": -31.24350357055664,
      "activations/layer1_attention_weight_max": 17.3082275390625,
      "activations/layer1_attention_weight_min": -14.627476692199707,
      "activations/layer20_attention_weight_max": 35.27344512939453,
      "activations/layer20_attention_weight_min": -25.687517166137695,
      "activations/layer21_attention_weight_max": 33.17578125,
      "activations/layer21_attention_weight_min": -23.85422706604004,
      "activations/layer22_attention_weight_max": 55.10620880126953,
      "activations/layer22_attention_weight_min": -30.84099006652832,
      "activations/layer23_attention_weight_max": 46.88100814819336,
      "activations/layer23_attention_weight_min": -26.439029693603516,
      "activations/layer2_attention_weight_max": 32.787384033203125,
      "activations/layer2_attention_weight_min": -32.23835754394531,
      "activations/layer3_attention_weight_max": 97.36552429199219,
      "activations/layer3_attention_weight_min": -97.01231384277344,
      "activations/layer4_attention_weight_max": 111.79487609863281,
      "activations/layer4_attention_weight_min": -111.59613037109375,
      "activations/layer5_attention_weight_max": 52.491607666015625,
      "activations/layer5_attention_weight_min": -62.636470794677734,
      "activations/layer6_attention_weight_max": 46.16231155395508,
      "activations/layer6_attention_weight_min": -50.52806854248047,
      "activations/layer7_attention_weight_max": 96.54164123535156,
      "activations/layer7_attention_weight_min": -96.45133972167969,
      "activations/layer8_attention_weight_max": 44.3563346862793,
      "activations/layer8_attention_weight_min": -48.57136535644531,
      "activations/layer9_attention_weight_max": 38.63396453857422,
      "activations/layer9_attention_weight_min": -39.71378707885742,
      "epoch": 10.95,
      "learning_rate": 8.019356060606059e-05,
      "loss": 2.7623,
      "step": 188450
    },
    {
      "activations/layer0_attention_weight_max": 15.522768020629883,
      "activations/layer0_attention_weight_min": -12.192437171936035,
      "activations/layer10_attention_weight_max": 34.593849182128906,
      "activations/layer10_attention_weight_min": -32.02901077270508,
      "activations/layer11_attention_weight_max": 36.291046142578125,
      "activations/layer11_attention_weight_min": -32.111602783203125,
      "activations/layer12_attention_weight_max": 26.67790412902832,
      "activations/layer12_attention_weight_min": -28.65813446044922,
      "activations/layer13_attention_weight_max": 43.5669059753418,
      "activations/layer13_attention_weight_min": -36.27400588989258,
      "activations/layer14_attention_weight_max": 46.3701286315918,
      "activations/layer14_attention_weight_min": -37.94428634643555,
      "activations/layer15_attention_weight_max": 40.52376937866211,
      "activations/layer15_attention_weight_min": -32.14289093017578,
      "activations/layer16_attention_weight_max": 34.07278823852539,
      "activations/layer16_attention_weight_min": -29.131853103637695,
      "activations/layer17_attention_weight_max": 32.3765754699707,
      "activations/layer17_attention_weight_min": -24.528518676757812,
      "activations/layer18_attention_weight_max": 33.652381896972656,
      "activations/layer18_attention_weight_min": -25.34271240234375,
      "activations/layer19_attention_weight_max": 36.5781364440918,
      "activations/layer19_attention_weight_min": -31.46906852722168,
      "activations/layer1_attention_weight_max": 16.18759536743164,
      "activations/layer1_attention_weight_min": -15.349093437194824,
      "activations/layer20_attention_weight_max": 30.62874412536621,
      "activations/layer20_attention_weight_min": -25.709321975708008,
      "activations/layer21_attention_weight_max": 30.26511573791504,
      "activations/layer21_attention_weight_min": -24.301958084106445,
      "activations/layer22_attention_weight_max": 48.89304733276367,
      "activations/layer22_attention_weight_min": -31.602617263793945,
      "activations/layer23_attention_weight_max": 38.63310623168945,
      "activations/layer23_attention_weight_min": -27.025747299194336,
      "activations/layer2_attention_weight_max": 33.16434860229492,
      "activations/layer2_attention_weight_min": -32.939735412597656,
      "activations/layer3_attention_weight_max": 95.21096801757812,
      "activations/layer3_attention_weight_min": -94.38251495361328,
      "activations/layer4_attention_weight_max": 118.10538482666016,
      "activations/layer4_attention_weight_min": -111.34626770019531,
      "activations/layer5_attention_weight_max": 52.077632904052734,
      "activations/layer5_attention_weight_min": -61.13097381591797,
      "activations/layer6_attention_weight_max": 46.01612854003906,
      "activations/layer6_attention_weight_min": -47.6977653503418,
      "activations/layer7_attention_weight_max": 88.0468521118164,
      "activations/layer7_attention_weight_min": -87.27742004394531,
      "activations/layer8_attention_weight_max": 39.72896194458008,
      "activations/layer8_attention_weight_min": -40.62129592895508,
      "activations/layer9_attention_weight_max": 34.36684799194336,
      "activations/layer9_attention_weight_min": -33.44694900512695,
      "epoch": 10.95,
      "learning_rate": 8.017462121212121e-05,
      "loss": 2.782,
      "step": 188500
    },
    {
      "activations/layer0_attention_weight_max": 15.463272094726562,
      "activations/layer0_attention_weight_min": -13.620882987976074,
      "activations/layer10_attention_weight_max": 32.92425537109375,
      "activations/layer10_attention_weight_min": -34.14383316040039,
      "activations/layer11_attention_weight_max": 34.39320373535156,
      "activations/layer11_attention_weight_min": -33.213279724121094,
      "activations/layer12_attention_weight_max": 24.247129440307617,
      "activations/layer12_attention_weight_min": -26.85267448425293,
      "activations/layer13_attention_weight_max": 37.66068649291992,
      "activations/layer13_attention_weight_min": -34.32855224609375,
      "activations/layer14_attention_weight_max": 42.82524108886719,
      "activations/layer14_attention_weight_min": -38.09736251831055,
      "activations/layer15_attention_weight_max": 35.61268615722656,
      "activations/layer15_attention_weight_min": -32.585914611816406,
      "activations/layer16_attention_weight_max": 31.48431396484375,
      "activations/layer16_attention_weight_min": -29.881120681762695,
      "activations/layer17_attention_weight_max": 32.131622314453125,
      "activations/layer17_attention_weight_min": -25.7379207611084,
      "activations/layer18_attention_weight_max": 31.645145416259766,
      "activations/layer18_attention_weight_min": -22.880815505981445,
      "activations/layer19_attention_weight_max": 36.2551383972168,
      "activations/layer19_attention_weight_min": -31.214920043945312,
      "activations/layer1_attention_weight_max": 16.98811149597168,
      "activations/layer1_attention_weight_min": -16.945249557495117,
      "activations/layer20_attention_weight_max": 28.559730529785156,
      "activations/layer20_attention_weight_min": -22.848751068115234,
      "activations/layer21_attention_weight_max": 28.382389068603516,
      "activations/layer21_attention_weight_min": -22.846973419189453,
      "activations/layer22_attention_weight_max": 40.9176025390625,
      "activations/layer22_attention_weight_min": -31.321449279785156,
      "activations/layer23_attention_weight_max": 33.27201843261719,
      "activations/layer23_attention_weight_min": -26.454578399658203,
      "activations/layer2_attention_weight_max": 33.200992584228516,
      "activations/layer2_attention_weight_min": -31.683155059814453,
      "activations/layer3_attention_weight_max": 96.66771697998047,
      "activations/layer3_attention_weight_min": -97.69205474853516,
      "activations/layer4_attention_weight_max": 114.95343017578125,
      "activations/layer4_attention_weight_min": -116.01513671875,
      "activations/layer5_attention_weight_max": 51.82518005371094,
      "activations/layer5_attention_weight_min": -65.51670837402344,
      "activations/layer6_attention_weight_max": 46.396671295166016,
      "activations/layer6_attention_weight_min": -48.36210250854492,
      "activations/layer7_attention_weight_max": 95.11175537109375,
      "activations/layer7_attention_weight_min": -91.93987274169922,
      "activations/layer8_attention_weight_max": 40.37954330444336,
      "activations/layer8_attention_weight_min": -41.207454681396484,
      "activations/layer9_attention_weight_max": 31.43708610534668,
      "activations/layer9_attention_weight_min": -33.507205963134766,
      "epoch": 10.96,
      "learning_rate": 8.015568181818181e-05,
      "loss": 2.7806,
      "step": 188550
    },
    {
      "activations/layer0_attention_weight_max": 14.723630905151367,
      "activations/layer0_attention_weight_min": -13.020345687866211,
      "activations/layer10_attention_weight_max": 32.450408935546875,
      "activations/layer10_attention_weight_min": -32.766822814941406,
      "activations/layer11_attention_weight_max": 30.823766708374023,
      "activations/layer11_attention_weight_min": -33.66633605957031,
      "activations/layer12_attention_weight_max": 23.944869995117188,
      "activations/layer12_attention_weight_min": -23.92871856689453,
      "activations/layer13_attention_weight_max": 38.76374816894531,
      "activations/layer13_attention_weight_min": -32.929725646972656,
      "activations/layer14_attention_weight_max": 41.366119384765625,
      "activations/layer14_attention_weight_min": -36.26241683959961,
      "activations/layer15_attention_weight_max": 43.092323303222656,
      "activations/layer15_attention_weight_min": -33.08243179321289,
      "activations/layer16_attention_weight_max": 29.29118537902832,
      "activations/layer16_attention_weight_min": -28.664508819580078,
      "activations/layer17_attention_weight_max": 32.98335647583008,
      "activations/layer17_attention_weight_min": -24.69619369506836,
      "activations/layer18_attention_weight_max": 29.937156677246094,
      "activations/layer18_attention_weight_min": -23.7548885345459,
      "activations/layer19_attention_weight_max": 31.588956832885742,
      "activations/layer19_attention_weight_min": -30.653522491455078,
      "activations/layer1_attention_weight_max": 17.316822052001953,
      "activations/layer1_attention_weight_min": -14.958551406860352,
      "activations/layer20_attention_weight_max": 33.116981506347656,
      "activations/layer20_attention_weight_min": -25.067777633666992,
      "activations/layer21_attention_weight_max": 32.62710189819336,
      "activations/layer21_attention_weight_min": -22.776355743408203,
      "activations/layer22_attention_weight_max": 42.028587341308594,
      "activations/layer22_attention_weight_min": -29.18020248413086,
      "activations/layer23_attention_weight_max": 34.83152770996094,
      "activations/layer23_attention_weight_min": -23.877897262573242,
      "activations/layer2_attention_weight_max": 32.89460372924805,
      "activations/layer2_attention_weight_min": -31.247325897216797,
      "activations/layer3_attention_weight_max": 92.0669937133789,
      "activations/layer3_attention_weight_min": -93.28775787353516,
      "activations/layer4_attention_weight_max": 108.39608001708984,
      "activations/layer4_attention_weight_min": -115.51483154296875,
      "activations/layer5_attention_weight_max": 53.92832946777344,
      "activations/layer5_attention_weight_min": -62.727989196777344,
      "activations/layer6_attention_weight_max": 44.15349197387695,
      "activations/layer6_attention_weight_min": -47.00157165527344,
      "activations/layer7_attention_weight_max": 86.87393951416016,
      "activations/layer7_attention_weight_min": -92.01132202148438,
      "activations/layer8_attention_weight_max": 37.32598114013672,
      "activations/layer8_attention_weight_min": -40.20393753051758,
      "activations/layer9_attention_weight_max": 31.630023956298828,
      "activations/layer9_attention_weight_min": -31.971481323242188,
      "epoch": 10.96,
      "learning_rate": 8.013674242424241e-05,
      "loss": 2.8006,
      "step": 188600
    },
    {
      "activations/layer0_attention_weight_max": 14.706307411193848,
      "activations/layer0_attention_weight_min": -12.114314079284668,
      "activations/layer10_attention_weight_max": 33.00811004638672,
      "activations/layer10_attention_weight_min": -35.17951202392578,
      "activations/layer11_attention_weight_max": 33.56048583984375,
      "activations/layer11_attention_weight_min": -33.60262680053711,
      "activations/layer12_attention_weight_max": 25.905961990356445,
      "activations/layer12_attention_weight_min": -27.295564651489258,
      "activations/layer13_attention_weight_max": 39.45802307128906,
      "activations/layer13_attention_weight_min": -36.821651458740234,
      "activations/layer14_attention_weight_max": 49.18424987792969,
      "activations/layer14_attention_weight_min": -38.09442901611328,
      "activations/layer15_attention_weight_max": 35.58968734741211,
      "activations/layer15_attention_weight_min": -31.714035034179688,
      "activations/layer16_attention_weight_max": 31.069751739501953,
      "activations/layer16_attention_weight_min": -27.024980545043945,
      "activations/layer17_attention_weight_max": 31.22020149230957,
      "activations/layer17_attention_weight_min": -25.7186222076416,
      "activations/layer18_attention_weight_max": 30.15709114074707,
      "activations/layer18_attention_weight_min": -21.47488021850586,
      "activations/layer19_attention_weight_max": 34.38019561767578,
      "activations/layer19_attention_weight_min": -30.643537521362305,
      "activations/layer1_attention_weight_max": 15.976407051086426,
      "activations/layer1_attention_weight_min": -14.537084579467773,
      "activations/layer20_attention_weight_max": 30.199384689331055,
      "activations/layer20_attention_weight_min": -25.759563446044922,
      "activations/layer21_attention_weight_max": 30.69873046875,
      "activations/layer21_attention_weight_min": -24.33219337463379,
      "activations/layer22_attention_weight_max": 47.92509841918945,
      "activations/layer22_attention_weight_min": -31.82091522216797,
      "activations/layer23_attention_weight_max": 35.18891525268555,
      "activations/layer23_attention_weight_min": -28.685680389404297,
      "activations/layer2_attention_weight_max": 31.81237030029297,
      "activations/layer2_attention_weight_min": -32.761077880859375,
      "activations/layer3_attention_weight_max": 93.98043823242188,
      "activations/layer3_attention_weight_min": -97.27191162109375,
      "activations/layer4_attention_weight_max": 108.81688690185547,
      "activations/layer4_attention_weight_min": -117.78681945800781,
      "activations/layer5_attention_weight_max": 51.657264709472656,
      "activations/layer5_attention_weight_min": -62.22504425048828,
      "activations/layer6_attention_weight_max": 43.854244232177734,
      "activations/layer6_attention_weight_min": -47.400535583496094,
      "activations/layer7_attention_weight_max": 92.12471771240234,
      "activations/layer7_attention_weight_min": -92.1860580444336,
      "activations/layer8_attention_weight_max": 40.952266693115234,
      "activations/layer8_attention_weight_min": -41.503822326660156,
      "activations/layer9_attention_weight_max": 34.70874786376953,
      "activations/layer9_attention_weight_min": -33.67325973510742,
      "epoch": 10.96,
      "learning_rate": 8.011780303030303e-05,
      "loss": 2.7731,
      "step": 188650
    },
    {
      "activations/layer0_attention_weight_max": 14.871946334838867,
      "activations/layer0_attention_weight_min": -13.036341667175293,
      "activations/layer10_attention_weight_max": 34.19966506958008,
      "activations/layer10_attention_weight_min": -34.406394958496094,
      "activations/layer11_attention_weight_max": 32.191368103027344,
      "activations/layer11_attention_weight_min": -31.917213439941406,
      "activations/layer12_attention_weight_max": 32.389190673828125,
      "activations/layer12_attention_weight_min": -29.408838272094727,
      "activations/layer13_attention_weight_max": 45.05069351196289,
      "activations/layer13_attention_weight_min": -37.51165008544922,
      "activations/layer14_attention_weight_max": 45.14240264892578,
      "activations/layer14_attention_weight_min": -42.274253845214844,
      "activations/layer15_attention_weight_max": 36.7901496887207,
      "activations/layer15_attention_weight_min": -31.921558380126953,
      "activations/layer16_attention_weight_max": 32.17258071899414,
      "activations/layer16_attention_weight_min": -27.854467391967773,
      "activations/layer17_attention_weight_max": 30.88981819152832,
      "activations/layer17_attention_weight_min": -24.959531784057617,
      "activations/layer18_attention_weight_max": 29.775850296020508,
      "activations/layer18_attention_weight_min": -22.266584396362305,
      "activations/layer19_attention_weight_max": 32.745174407958984,
      "activations/layer19_attention_weight_min": -30.318944931030273,
      "activations/layer1_attention_weight_max": 17.386348724365234,
      "activations/layer1_attention_weight_min": -14.474345207214355,
      "activations/layer20_attention_weight_max": 27.525371551513672,
      "activations/layer20_attention_weight_min": -25.06989097595215,
      "activations/layer21_attention_weight_max": 28.095970153808594,
      "activations/layer21_attention_weight_min": -23.387510299682617,
      "activations/layer22_attention_weight_max": 54.6221809387207,
      "activations/layer22_attention_weight_min": -29.311553955078125,
      "activations/layer23_attention_weight_max": 33.381690979003906,
      "activations/layer23_attention_weight_min": -25.081480026245117,
      "activations/layer2_attention_weight_max": 32.24213790893555,
      "activations/layer2_attention_weight_min": -31.998619079589844,
      "activations/layer3_attention_weight_max": 94.07942962646484,
      "activations/layer3_attention_weight_min": -94.43760681152344,
      "activations/layer4_attention_weight_max": 110.77953338623047,
      "activations/layer4_attention_weight_min": -111.8066635131836,
      "activations/layer5_attention_weight_max": 52.066925048828125,
      "activations/layer5_attention_weight_min": -63.22496032714844,
      "activations/layer6_attention_weight_max": 43.11547088623047,
      "activations/layer6_attention_weight_min": -48.54391098022461,
      "activations/layer7_attention_weight_max": 92.6190414428711,
      "activations/layer7_attention_weight_min": -91.39158630371094,
      "activations/layer8_attention_weight_max": 40.247066497802734,
      "activations/layer8_attention_weight_min": -43.344505310058594,
      "activations/layer9_attention_weight_max": 33.927059173583984,
      "activations/layer9_attention_weight_min": -35.46113967895508,
      "epoch": 10.96,
      "learning_rate": 8.009886363636363e-05,
      "loss": 2.7929,
      "step": 188700
    },
    {
      "activations/layer0_attention_weight_max": 15.791295051574707,
      "activations/layer0_attention_weight_min": -12.75106430053711,
      "activations/layer10_attention_weight_max": 31.318363189697266,
      "activations/layer10_attention_weight_min": -31.582462310791016,
      "activations/layer11_attention_weight_max": 31.382701873779297,
      "activations/layer11_attention_weight_min": -30.850997924804688,
      "activations/layer12_attention_weight_max": 24.762651443481445,
      "activations/layer12_attention_weight_min": -24.060958862304688,
      "activations/layer13_attention_weight_max": 38.85454177856445,
      "activations/layer13_attention_weight_min": -34.05131912231445,
      "activations/layer14_attention_weight_max": 48.055908203125,
      "activations/layer14_attention_weight_min": -38.822959899902344,
      "activations/layer15_attention_weight_max": 37.44740676879883,
      "activations/layer15_attention_weight_min": -31.400686264038086,
      "activations/layer16_attention_weight_max": 33.01632308959961,
      "activations/layer16_attention_weight_min": -26.819108963012695,
      "activations/layer17_attention_weight_max": 35.5672492980957,
      "activations/layer17_attention_weight_min": -25.153656005859375,
      "activations/layer18_attention_weight_max": 31.628984451293945,
      "activations/layer18_attention_weight_min": -23.645545959472656,
      "activations/layer19_attention_weight_max": 35.75699996948242,
      "activations/layer19_attention_weight_min": -31.706607818603516,
      "activations/layer1_attention_weight_max": 15.186062812805176,
      "activations/layer1_attention_weight_min": -14.304604530334473,
      "activations/layer20_attention_weight_max": 30.023502349853516,
      "activations/layer20_attention_weight_min": -27.4942569732666,
      "activations/layer21_attention_weight_max": 31.134319305419922,
      "activations/layer21_attention_weight_min": -24.065935134887695,
      "activations/layer22_attention_weight_max": 44.45860290527344,
      "activations/layer22_attention_weight_min": -29.64936637878418,
      "activations/layer23_attention_weight_max": 34.66765594482422,
      "activations/layer23_attention_weight_min": -25.828514099121094,
      "activations/layer2_attention_weight_max": 31.872753143310547,
      "activations/layer2_attention_weight_min": -31.97259521484375,
      "activations/layer3_attention_weight_max": 92.53903198242188,
      "activations/layer3_attention_weight_min": -92.77835083007812,
      "activations/layer4_attention_weight_max": 112.5061264038086,
      "activations/layer4_attention_weight_min": -110.00963592529297,
      "activations/layer5_attention_weight_max": 51.40441131591797,
      "activations/layer5_attention_weight_min": -60.70746994018555,
      "activations/layer6_attention_weight_max": 44.950355529785156,
      "activations/layer6_attention_weight_min": -46.20976257324219,
      "activations/layer7_attention_weight_max": 86.62735748291016,
      "activations/layer7_attention_weight_min": -86.2886734008789,
      "activations/layer8_attention_weight_max": 42.66371154785156,
      "activations/layer8_attention_weight_min": -41.661956787109375,
      "activations/layer9_attention_weight_max": 33.69181442260742,
      "activations/layer9_attention_weight_min": -32.53689193725586,
      "epoch": 10.97,
      "learning_rate": 8.007992424242424e-05,
      "loss": 2.7865,
      "step": 188750
    },
    {
      "activations/layer0_attention_weight_max": 14.981277465820312,
      "activations/layer0_attention_weight_min": -12.878860473632812,
      "activations/layer10_attention_weight_max": 33.735198974609375,
      "activations/layer10_attention_weight_min": -33.267208099365234,
      "activations/layer11_attention_weight_max": 35.21460723876953,
      "activations/layer11_attention_weight_min": -30.96346664428711,
      "activations/layer12_attention_weight_max": 25.31920623779297,
      "activations/layer12_attention_weight_min": -24.60088539123535,
      "activations/layer13_attention_weight_max": 41.114715576171875,
      "activations/layer13_attention_weight_min": -35.104347229003906,
      "activations/layer14_attention_weight_max": 44.63389587402344,
      "activations/layer14_attention_weight_min": -37.14809799194336,
      "activations/layer15_attention_weight_max": 35.86870574951172,
      "activations/layer15_attention_weight_min": -32.2314567565918,
      "activations/layer16_attention_weight_max": 30.256498336791992,
      "activations/layer16_attention_weight_min": -28.537338256835938,
      "activations/layer17_attention_weight_max": 31.639503479003906,
      "activations/layer17_attention_weight_min": -26.277132034301758,
      "activations/layer18_attention_weight_max": 29.529184341430664,
      "activations/layer18_attention_weight_min": -24.189308166503906,
      "activations/layer19_attention_weight_max": 36.04025650024414,
      "activations/layer19_attention_weight_min": -30.11728858947754,
      "activations/layer1_attention_weight_max": 16.128093719482422,
      "activations/layer1_attention_weight_min": -14.036413192749023,
      "activations/layer20_attention_weight_max": 29.610454559326172,
      "activations/layer20_attention_weight_min": -24.532958984375,
      "activations/layer21_attention_weight_max": 28.92660903930664,
      "activations/layer21_attention_weight_min": -23.164846420288086,
      "activations/layer22_attention_weight_max": 45.63539123535156,
      "activations/layer22_attention_weight_min": -29.6484375,
      "activations/layer23_attention_weight_max": 35.715065002441406,
      "activations/layer23_attention_weight_min": -24.267108917236328,
      "activations/layer2_attention_weight_max": 31.736957550048828,
      "activations/layer2_attention_weight_min": -31.884849548339844,
      "activations/layer3_attention_weight_max": 93.54936981201172,
      "activations/layer3_attention_weight_min": -92.71737670898438,
      "activations/layer4_attention_weight_max": 114.5762939453125,
      "activations/layer4_attention_weight_min": -113.69673919677734,
      "activations/layer5_attention_weight_max": 51.260337829589844,
      "activations/layer5_attention_weight_min": -61.00086975097656,
      "activations/layer6_attention_weight_max": 45.872196197509766,
      "activations/layer6_attention_weight_min": -46.91615676879883,
      "activations/layer7_attention_weight_max": 93.40193939208984,
      "activations/layer7_attention_weight_min": -96.78855895996094,
      "activations/layer8_attention_weight_max": 41.096046447753906,
      "activations/layer8_attention_weight_min": -40.554359436035156,
      "activations/layer9_attention_weight_max": 34.11774826049805,
      "activations/layer9_attention_weight_min": -32.661067962646484,
      "epoch": 10.97,
      "learning_rate": 8.006098484848484e-05,
      "loss": 2.7754,
      "step": 188800
    },
    {
      "activations/layer0_attention_weight_max": 14.982012748718262,
      "activations/layer0_attention_weight_min": -11.389281272888184,
      "activations/layer10_attention_weight_max": 33.18027877807617,
      "activations/layer10_attention_weight_min": -31.08092498779297,
      "activations/layer11_attention_weight_max": 33.90240478515625,
      "activations/layer11_attention_weight_min": -31.37129783630371,
      "activations/layer12_attention_weight_max": 29.421091079711914,
      "activations/layer12_attention_weight_min": -23.889429092407227,
      "activations/layer13_attention_weight_max": 39.658897399902344,
      "activations/layer13_attention_weight_min": -34.597957611083984,
      "activations/layer14_attention_weight_max": 43.00725173950195,
      "activations/layer14_attention_weight_min": -36.296119689941406,
      "activations/layer15_attention_weight_max": 34.81501007080078,
      "activations/layer15_attention_weight_min": -32.963314056396484,
      "activations/layer16_attention_weight_max": 29.701251983642578,
      "activations/layer16_attention_weight_min": -27.583433151245117,
      "activations/layer17_attention_weight_max": 34.28507995605469,
      "activations/layer17_attention_weight_min": -25.658876419067383,
      "activations/layer18_attention_weight_max": 34.8825569152832,
      "activations/layer18_attention_weight_min": -22.834917068481445,
      "activations/layer19_attention_weight_max": 37.32990646362305,
      "activations/layer19_attention_weight_min": -32.87793731689453,
      "activations/layer1_attention_weight_max": 16.0941162109375,
      "activations/layer1_attention_weight_min": -15.231440544128418,
      "activations/layer20_attention_weight_max": 28.89049530029297,
      "activations/layer20_attention_weight_min": -24.720048904418945,
      "activations/layer21_attention_weight_max": 30.237565994262695,
      "activations/layer21_attention_weight_min": -23.53690528869629,
      "activations/layer22_attention_weight_max": 45.3596076965332,
      "activations/layer22_attention_weight_min": -28.97658348083496,
      "activations/layer23_attention_weight_max": 36.21807861328125,
      "activations/layer23_attention_weight_min": -23.44040298461914,
      "activations/layer2_attention_weight_max": 32.86689376831055,
      "activations/layer2_attention_weight_min": -33.14779281616211,
      "activations/layer3_attention_weight_max": 93.30017852783203,
      "activations/layer3_attention_weight_min": -96.95829772949219,
      "activations/layer4_attention_weight_max": 113.5845947265625,
      "activations/layer4_attention_weight_min": -113.74903869628906,
      "activations/layer5_attention_weight_max": 54.421714782714844,
      "activations/layer5_attention_weight_min": -62.72540283203125,
      "activations/layer6_attention_weight_max": 43.94470977783203,
      "activations/layer6_attention_weight_min": -45.99348068237305,
      "activations/layer7_attention_weight_max": 85.90011596679688,
      "activations/layer7_attention_weight_min": -91.20096588134766,
      "activations/layer8_attention_weight_max": 39.37602615356445,
      "activations/layer8_attention_weight_min": -39.50679397583008,
      "activations/layer9_attention_weight_max": 30.911502838134766,
      "activations/layer9_attention_weight_min": -32.37665557861328,
      "epoch": 10.97,
      "learning_rate": 8.004204545454545e-05,
      "loss": 2.7708,
      "step": 188850
    },
    {
      "activations/layer0_attention_weight_max": 15.553823471069336,
      "activations/layer0_attention_weight_min": -12.268176078796387,
      "activations/layer10_attention_weight_max": 30.639366149902344,
      "activations/layer10_attention_weight_min": -29.754404067993164,
      "activations/layer11_attention_weight_max": 30.685699462890625,
      "activations/layer11_attention_weight_min": -29.881101608276367,
      "activations/layer12_attention_weight_max": 25.098838806152344,
      "activations/layer12_attention_weight_min": -25.567378997802734,
      "activations/layer13_attention_weight_max": 36.52312469482422,
      "activations/layer13_attention_weight_min": -34.648681640625,
      "activations/layer14_attention_weight_max": 43.53397750854492,
      "activations/layer14_attention_weight_min": -36.90839385986328,
      "activations/layer15_attention_weight_max": 33.89020538330078,
      "activations/layer15_attention_weight_min": -29.009567260742188,
      "activations/layer16_attention_weight_max": 30.172603607177734,
      "activations/layer16_attention_weight_min": -25.751893997192383,
      "activations/layer17_attention_weight_max": 30.340559005737305,
      "activations/layer17_attention_weight_min": -25.340665817260742,
      "activations/layer18_attention_weight_max": 30.055082321166992,
      "activations/layer18_attention_weight_min": -21.185136795043945,
      "activations/layer19_attention_weight_max": 38.734683990478516,
      "activations/layer19_attention_weight_min": -29.046445846557617,
      "activations/layer1_attention_weight_max": 16.55152130126953,
      "activations/layer1_attention_weight_min": -15.299686431884766,
      "activations/layer20_attention_weight_max": 32.155330657958984,
      "activations/layer20_attention_weight_min": -24.641117095947266,
      "activations/layer21_attention_weight_max": 28.154006958007812,
      "activations/layer21_attention_weight_min": -23.03416633605957,
      "activations/layer22_attention_weight_max": 41.91876220703125,
      "activations/layer22_attention_weight_min": -28.300371170043945,
      "activations/layer23_attention_weight_max": 33.891929626464844,
      "activations/layer23_attention_weight_min": -24.647232055664062,
      "activations/layer2_attention_weight_max": 32.610408782958984,
      "activations/layer2_attention_weight_min": -33.1362419128418,
      "activations/layer3_attention_weight_max": 90.87922668457031,
      "activations/layer3_attention_weight_min": -92.74740600585938,
      "activations/layer4_attention_weight_max": 114.63358306884766,
      "activations/layer4_attention_weight_min": -114.29314422607422,
      "activations/layer5_attention_weight_max": 53.00359344482422,
      "activations/layer5_attention_weight_min": -64.1493911743164,
      "activations/layer6_attention_weight_max": 42.839473724365234,
      "activations/layer6_attention_weight_min": -47.183841705322266,
      "activations/layer7_attention_weight_max": 82.9200210571289,
      "activations/layer7_attention_weight_min": -88.51757049560547,
      "activations/layer8_attention_weight_max": 38.87129592895508,
      "activations/layer8_attention_weight_min": -43.13721466064453,
      "activations/layer9_attention_weight_max": 30.539291381835938,
      "activations/layer9_attention_weight_min": -32.76780319213867,
      "epoch": 10.98,
      "learning_rate": 8.002310606060606e-05,
      "loss": 2.7875,
      "step": 188900
    },
    {
      "activations/layer0_attention_weight_max": 15.723170280456543,
      "activations/layer0_attention_weight_min": -11.589879035949707,
      "activations/layer10_attention_weight_max": 32.696842193603516,
      "activations/layer10_attention_weight_min": -34.34577178955078,
      "activations/layer11_attention_weight_max": 33.836856842041016,
      "activations/layer11_attention_weight_min": -32.92073059082031,
      "activations/layer12_attention_weight_max": 25.97972297668457,
      "activations/layer12_attention_weight_min": -29.30617904663086,
      "activations/layer13_attention_weight_max": 41.88262176513672,
      "activations/layer13_attention_weight_min": -33.74949264526367,
      "activations/layer14_attention_weight_max": 45.037315368652344,
      "activations/layer14_attention_weight_min": -36.62391662597656,
      "activations/layer15_attention_weight_max": 37.1915168762207,
      "activations/layer15_attention_weight_min": -30.6934814453125,
      "activations/layer16_attention_weight_max": 32.12337875366211,
      "activations/layer16_attention_weight_min": -27.40938377380371,
      "activations/layer17_attention_weight_max": 33.50822448730469,
      "activations/layer17_attention_weight_min": -25.270448684692383,
      "activations/layer18_attention_weight_max": 35.55524826049805,
      "activations/layer18_attention_weight_min": -22.761106491088867,
      "activations/layer19_attention_weight_max": 34.286800384521484,
      "activations/layer19_attention_weight_min": -31.46665382385254,
      "activations/layer1_attention_weight_max": 17.428573608398438,
      "activations/layer1_attention_weight_min": -14.46286678314209,
      "activations/layer20_attention_weight_max": 29.236663818359375,
      "activations/layer20_attention_weight_min": -25.535831451416016,
      "activations/layer21_attention_weight_max": 28.662076950073242,
      "activations/layer21_attention_weight_min": -24.04216766357422,
      "activations/layer22_attention_weight_max": 41.88713836669922,
      "activations/layer22_attention_weight_min": -31.28451156616211,
      "activations/layer23_attention_weight_max": 34.12263870239258,
      "activations/layer23_attention_weight_min": -25.71236228942871,
      "activations/layer2_attention_weight_max": 33.91551971435547,
      "activations/layer2_attention_weight_min": -34.67472457885742,
      "activations/layer3_attention_weight_max": 96.09956359863281,
      "activations/layer3_attention_weight_min": -98.12116241455078,
      "activations/layer4_attention_weight_max": 115.12451171875,
      "activations/layer4_attention_weight_min": -114.97277069091797,
      "activations/layer5_attention_weight_max": 52.202247619628906,
      "activations/layer5_attention_weight_min": -62.09149169921875,
      "activations/layer6_attention_weight_max": 47.1612663269043,
      "activations/layer6_attention_weight_min": -49.17183303833008,
      "activations/layer7_attention_weight_max": 98.06741333007812,
      "activations/layer7_attention_weight_min": -90.04401397705078,
      "activations/layer8_attention_weight_max": 42.157222747802734,
      "activations/layer8_attention_weight_min": -40.14991760253906,
      "activations/layer9_attention_weight_max": 34.00588607788086,
      "activations/layer9_attention_weight_min": -34.767921447753906,
      "epoch": 10.98,
      "learning_rate": 8.000416666666665e-05,
      "loss": 2.7805,
      "step": 188950
    },
    {
      "activations/layer0_attention_weight_max": 15.198721885681152,
      "activations/layer0_attention_weight_min": -12.586030006408691,
      "activations/layer10_attention_weight_max": 32.731719970703125,
      "activations/layer10_attention_weight_min": -31.51204490661621,
      "activations/layer11_attention_weight_max": 36.99966049194336,
      "activations/layer11_attention_weight_min": -32.319183349609375,
      "activations/layer12_attention_weight_max": 27.181598663330078,
      "activations/layer12_attention_weight_min": -24.90473175048828,
      "activations/layer13_attention_weight_max": 37.58351135253906,
      "activations/layer13_attention_weight_min": -33.373653411865234,
      "activations/layer14_attention_weight_max": 43.25217819213867,
      "activations/layer14_attention_weight_min": -36.323028564453125,
      "activations/layer15_attention_weight_max": 35.35602951049805,
      "activations/layer15_attention_weight_min": -32.17561721801758,
      "activations/layer16_attention_weight_max": 33.38877487182617,
      "activations/layer16_attention_weight_min": -29.954195022583008,
      "activations/layer17_attention_weight_max": 32.019691467285156,
      "activations/layer17_attention_weight_min": -25.522714614868164,
      "activations/layer18_attention_weight_max": 30.49012565612793,
      "activations/layer18_attention_weight_min": -22.85643196105957,
      "activations/layer19_attention_weight_max": 36.000308990478516,
      "activations/layer19_attention_weight_min": -32.748863220214844,
      "activations/layer1_attention_weight_max": 16.10106658935547,
      "activations/layer1_attention_weight_min": -13.945191383361816,
      "activations/layer20_attention_weight_max": 29.347427368164062,
      "activations/layer20_attention_weight_min": -25.263416290283203,
      "activations/layer21_attention_weight_max": 28.347501754760742,
      "activations/layer21_attention_weight_min": -24.25508689880371,
      "activations/layer22_attention_weight_max": 42.74521255493164,
      "activations/layer22_attention_weight_min": -31.429813385009766,
      "activations/layer23_attention_weight_max": 34.97943115234375,
      "activations/layer23_attention_weight_min": -25.86217498779297,
      "activations/layer2_attention_weight_max": 31.742752075195312,
      "activations/layer2_attention_weight_min": -31.590770721435547,
      "activations/layer3_attention_weight_max": 92.28353118896484,
      "activations/layer3_attention_weight_min": -91.05657196044922,
      "activations/layer4_attention_weight_max": 110.88722229003906,
      "activations/layer4_attention_weight_min": -110.974853515625,
      "activations/layer5_attention_weight_max": 51.719154357910156,
      "activations/layer5_attention_weight_min": -63.57395935058594,
      "activations/layer6_attention_weight_max": 44.69501495361328,
      "activations/layer6_attention_weight_min": -48.18197250366211,
      "activations/layer7_attention_weight_max": 88.06210327148438,
      "activations/layer7_attention_weight_min": -86.37550354003906,
      "activations/layer8_attention_weight_max": 41.11635971069336,
      "activations/layer8_attention_weight_min": -39.657955169677734,
      "activations/layer9_attention_weight_max": 35.556522369384766,
      "activations/layer9_attention_weight_min": -32.83319854736328,
      "epoch": 10.98,
      "learning_rate": 7.998522727272726e-05,
      "loss": 2.798,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.527,
      "eval_samples_per_second": 503.578,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.527,
      "eval_openwebtext_samples_per_second": 503.578,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0027,
      "eval_wikitext_samples_per_second": 227.688,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_lambada_loss": 2.521484375,
      "eval_lambada_ppl": 12.447059060920614,
      "eval_lambada_runtime": 9.5851,
      "eval_lambada_samples_per_second": 507.974,
      "step": 189000
    },
    {
      "activations/layer0_attention_weight_max": 14.836554527282715,
      "activations/layer0_attention_weight_min": -11.76435375213623,
      "activations/layer10_attention_weight_max": 30.77034568786621,
      "activations/layer10_attention_weight_min": -30.86758804321289,
      "activations/layer11_attention_weight_max": 31.678260803222656,
      "activations/layer11_attention_weight_min": -29.965925216674805,
      "activations/layer12_attention_weight_max": 21.84398078918457,
      "activations/layer12_attention_weight_min": -25.209671020507812,
      "activations/layer13_attention_weight_max": 35.69032669067383,
      "activations/layer13_attention_weight_min": -32.620887756347656,
      "activations/layer14_attention_weight_max": 44.15057373046875,
      "activations/layer14_attention_weight_min": -40.021183013916016,
      "activations/layer15_attention_weight_max": 36.4383659362793,
      "activations/layer15_attention_weight_min": -32.08150863647461,
      "activations/layer16_attention_weight_max": 30.3369140625,
      "activations/layer16_attention_weight_min": -29.690549850463867,
      "activations/layer17_attention_weight_max": 30.728891372680664,
      "activations/layer17_attention_weight_min": -26.248315811157227,
      "activations/layer18_attention_weight_max": 28.624069213867188,
      "activations/layer18_attention_weight_min": -24.38401222229004,
      "activations/layer19_attention_weight_max": 34.723670959472656,
      "activations/layer19_attention_weight_min": -30.37586784362793,
      "activations/layer1_attention_weight_max": 16.52686882019043,
      "activations/layer1_attention_weight_min": -15.3524808883667,
      "activations/layer20_attention_weight_max": 34.5322380065918,
      "activations/layer20_attention_weight_min": -25.476205825805664,
      "activations/layer21_attention_weight_max": 32.20282745361328,
      "activations/layer21_attention_weight_min": -25.33964729309082,
      "activations/layer22_attention_weight_max": 44.731292724609375,
      "activations/layer22_attention_weight_min": -31.23911476135254,
      "activations/layer23_attention_weight_max": 32.829261779785156,
      "activations/layer23_attention_weight_min": -24.99188804626465,
      "activations/layer2_attention_weight_max": 32.33602523803711,
      "activations/layer2_attention_weight_min": -34.539794921875,
      "activations/layer3_attention_weight_max": 96.20245361328125,
      "activations/layer3_attention_weight_min": -100.48345947265625,
      "activations/layer4_attention_weight_max": 109.46581268310547,
      "activations/layer4_attention_weight_min": -114.27278137207031,
      "activations/layer5_attention_weight_max": 49.51277160644531,
      "activations/layer5_attention_weight_min": -62.758514404296875,
      "activations/layer6_attention_weight_max": 46.17710876464844,
      "activations/layer6_attention_weight_min": -48.20903015136719,
      "activations/layer7_attention_weight_max": 89.06476593017578,
      "activations/layer7_attention_weight_min": -88.45925903320312,
      "activations/layer8_attention_weight_max": 39.05402755737305,
      "activations/layer8_attention_weight_min": -39.95955276489258,
      "activations/layer9_attention_weight_max": 31.257558822631836,
      "activations/layer9_attention_weight_min": -31.68008804321289,
      "epoch": 10.98,
      "learning_rate": 7.996628787878788e-05,
      "loss": 2.7864,
      "step": 189050
    },
    {
      "activations/layer0_attention_weight_max": 15.713242530822754,
      "activations/layer0_attention_weight_min": -12.825389862060547,
      "activations/layer10_attention_weight_max": 31.630245208740234,
      "activations/layer10_attention_weight_min": -32.82743835449219,
      "activations/layer11_attention_weight_max": 32.988731384277344,
      "activations/layer11_attention_weight_min": -31.877660751342773,
      "activations/layer12_attention_weight_max": 24.96289825439453,
      "activations/layer12_attention_weight_min": -27.8254451751709,
      "activations/layer13_attention_weight_max": 37.453155517578125,
      "activations/layer13_attention_weight_min": -33.23161315917969,
      "activations/layer14_attention_weight_max": 47.51022720336914,
      "activations/layer14_attention_weight_min": -39.824066162109375,
      "activations/layer15_attention_weight_max": 37.59526443481445,
      "activations/layer15_attention_weight_min": -30.15403938293457,
      "activations/layer16_attention_weight_max": 31.35376739501953,
      "activations/layer16_attention_weight_min": -29.22819709777832,
      "activations/layer17_attention_weight_max": 31.90440559387207,
      "activations/layer17_attention_weight_min": -25.837427139282227,
      "activations/layer18_attention_weight_max": 30.652767181396484,
      "activations/layer18_attention_weight_min": -23.408166885375977,
      "activations/layer19_attention_weight_max": 33.122676849365234,
      "activations/layer19_attention_weight_min": -32.309532165527344,
      "activations/layer1_attention_weight_max": 17.203083038330078,
      "activations/layer1_attention_weight_min": -15.377035140991211,
      "activations/layer20_attention_weight_max": 28.947437286376953,
      "activations/layer20_attention_weight_min": -25.116899490356445,
      "activations/layer21_attention_weight_max": 27.63095474243164,
      "activations/layer21_attention_weight_min": -25.205583572387695,
      "activations/layer22_attention_weight_max": 41.01374053955078,
      "activations/layer22_attention_weight_min": -31.116859436035156,
      "activations/layer23_attention_weight_max": 32.706390380859375,
      "activations/layer23_attention_weight_min": -25.61130142211914,
      "activations/layer2_attention_weight_max": 32.60271453857422,
      "activations/layer2_attention_weight_min": -33.41428756713867,
      "activations/layer3_attention_weight_max": 94.72652435302734,
      "activations/layer3_attention_weight_min": -95.23591613769531,
      "activations/layer4_attention_weight_max": 114.85884094238281,
      "activations/layer4_attention_weight_min": -114.77174377441406,
      "activations/layer5_attention_weight_max": 52.755428314208984,
      "activations/layer5_attention_weight_min": -63.8554801940918,
      "activations/layer6_attention_weight_max": 44.28989791870117,
      "activations/layer6_attention_weight_min": -49.50385284423828,
      "activations/layer7_attention_weight_max": 91.16481018066406,
      "activations/layer7_attention_weight_min": -92.33800506591797,
      "activations/layer8_attention_weight_max": 40.403587341308594,
      "activations/layer8_attention_weight_min": -44.77515411376953,
      "activations/layer9_attention_weight_max": 35.390201568603516,
      "activations/layer9_attention_weight_min": -36.55925369262695,
      "epoch": 10.99,
      "learning_rate": 7.994734848484847e-05,
      "loss": 2.7789,
      "step": 189100
    },
    {
      "activations/layer0_attention_weight_max": 14.64167308807373,
      "activations/layer0_attention_weight_min": -13.031784057617188,
      "activations/layer10_attention_weight_max": 32.0898551940918,
      "activations/layer10_attention_weight_min": -33.554534912109375,
      "activations/layer11_attention_weight_max": 31.900230407714844,
      "activations/layer11_attention_weight_min": -31.732486724853516,
      "activations/layer12_attention_weight_max": 25.43912696838379,
      "activations/layer12_attention_weight_min": -24.87731170654297,
      "activations/layer13_attention_weight_max": 37.98695755004883,
      "activations/layer13_attention_weight_min": -33.90627670288086,
      "activations/layer14_attention_weight_max": 43.49489212036133,
      "activations/layer14_attention_weight_min": -38.52735137939453,
      "activations/layer15_attention_weight_max": 36.83913040161133,
      "activations/layer15_attention_weight_min": -32.7603645324707,
      "activations/layer16_attention_weight_max": 29.901044845581055,
      "activations/layer16_attention_weight_min": -27.60178565979004,
      "activations/layer17_attention_weight_max": 31.1531925201416,
      "activations/layer17_attention_weight_min": -24.970478057861328,
      "activations/layer18_attention_weight_max": 33.44664764404297,
      "activations/layer18_attention_weight_min": -23.177051544189453,
      "activations/layer19_attention_weight_max": 33.26683044433594,
      "activations/layer19_attention_weight_min": -29.66663932800293,
      "activations/layer1_attention_weight_max": 18.434885025024414,
      "activations/layer1_attention_weight_min": -13.888497352600098,
      "activations/layer20_attention_weight_max": 29.318065643310547,
      "activations/layer20_attention_weight_min": -24.5556640625,
      "activations/layer21_attention_weight_max": 31.355411529541016,
      "activations/layer21_attention_weight_min": -23.077367782592773,
      "activations/layer22_attention_weight_max": 47.92409896850586,
      "activations/layer22_attention_weight_min": -29.06754493713379,
      "activations/layer23_attention_weight_max": 36.255863189697266,
      "activations/layer23_attention_weight_min": -23.368999481201172,
      "activations/layer2_attention_weight_max": 32.66943359375,
      "activations/layer2_attention_weight_min": -33.181922912597656,
      "activations/layer3_attention_weight_max": 95.26249694824219,
      "activations/layer3_attention_weight_min": -100.33846282958984,
      "activations/layer4_attention_weight_max": 113.7171859741211,
      "activations/layer4_attention_weight_min": -114.709228515625,
      "activations/layer5_attention_weight_max": 52.33153533935547,
      "activations/layer5_attention_weight_min": -64.77272033691406,
      "activations/layer6_attention_weight_max": 45.58452224731445,
      "activations/layer6_attention_weight_min": -51.70389175415039,
      "activations/layer7_attention_weight_max": 91.12181854248047,
      "activations/layer7_attention_weight_min": -95.39952850341797,
      "activations/layer8_attention_weight_max": 40.063453674316406,
      "activations/layer8_attention_weight_min": -42.37787628173828,
      "activations/layer9_attention_weight_max": 32.91618347167969,
      "activations/layer9_attention_weight_min": -36.26664352416992,
      "epoch": 10.99,
      "learning_rate": 7.992840909090908e-05,
      "loss": 2.7888,
      "step": 189150
    },
    {
      "activations/layer0_attention_weight_max": 15.7593412399292,
      "activations/layer0_attention_weight_min": -12.4768648147583,
      "activations/layer10_attention_weight_max": 40.87703323364258,
      "activations/layer10_attention_weight_min": -39.49940872192383,
      "activations/layer11_attention_weight_max": 42.71086883544922,
      "activations/layer11_attention_weight_min": -40.539554595947266,
      "activations/layer12_attention_weight_max": 25.817995071411133,
      "activations/layer12_attention_weight_min": -27.423965454101562,
      "activations/layer13_attention_weight_max": 38.181732177734375,
      "activations/layer13_attention_weight_min": -35.76283264160156,
      "activations/layer14_attention_weight_max": 43.84071731567383,
      "activations/layer14_attention_weight_min": -41.31257629394531,
      "activations/layer15_attention_weight_max": 35.993186950683594,
      "activations/layer15_attention_weight_min": -33.83653259277344,
      "activations/layer16_attention_weight_max": 33.096675872802734,
      "activations/layer16_attention_weight_min": -29.338829040527344,
      "activations/layer17_attention_weight_max": 31.064937591552734,
      "activations/layer17_attention_weight_min": -28.172182083129883,
      "activations/layer18_attention_weight_max": 31.54018783569336,
      "activations/layer18_attention_weight_min": -23.174245834350586,
      "activations/layer19_attention_weight_max": 39.323429107666016,
      "activations/layer19_attention_weight_min": -33.63310241699219,
      "activations/layer1_attention_weight_max": 17.942913055419922,
      "activations/layer1_attention_weight_min": -14.178592681884766,
      "activations/layer20_attention_weight_max": 30.390403747558594,
      "activations/layer20_attention_weight_min": -27.006263732910156,
      "activations/layer21_attention_weight_max": 28.137813568115234,
      "activations/layer21_attention_weight_min": -25.74204444885254,
      "activations/layer22_attention_weight_max": 46.51750564575195,
      "activations/layer22_attention_weight_min": -30.447391510009766,
      "activations/layer23_attention_weight_max": 41.17034149169922,
      "activations/layer23_attention_weight_min": -24.629192352294922,
      "activations/layer2_attention_weight_max": 33.75835037231445,
      "activations/layer2_attention_weight_min": -34.99049377441406,
      "activations/layer3_attention_weight_max": 100.7296371459961,
      "activations/layer3_attention_weight_min": -98.11126708984375,
      "activations/layer4_attention_weight_max": 125.46671295166016,
      "activations/layer4_attention_weight_min": -111.16080474853516,
      "activations/layer5_attention_weight_max": 55.56871032714844,
      "activations/layer5_attention_weight_min": -58.161888122558594,
      "activations/layer6_attention_weight_max": 45.59026336669922,
      "activations/layer6_attention_weight_min": -47.236881256103516,
      "activations/layer7_attention_weight_max": 99.03142547607422,
      "activations/layer7_attention_weight_min": -92.25633239746094,
      "activations/layer8_attention_weight_max": 46.298439025878906,
      "activations/layer8_attention_weight_min": -43.66526412963867,
      "activations/layer9_attention_weight_max": 39.34914779663086,
      "activations/layer9_attention_weight_min": -40.22056198120117,
      "epoch": 10.99,
      "learning_rate": 7.99094696969697e-05,
      "loss": 2.8017,
      "step": 189200
    },
    {
      "activations/layer0_attention_weight_max": 15.100839614868164,
      "activations/layer0_attention_weight_min": -12.89281940460205,
      "activations/layer10_attention_weight_max": 39.066741943359375,
      "activations/layer10_attention_weight_min": -35.6114387512207,
      "activations/layer11_attention_weight_max": 41.842857360839844,
      "activations/layer11_attention_weight_min": -36.28083801269531,
      "activations/layer12_attention_weight_max": 27.150148391723633,
      "activations/layer12_attention_weight_min": -27.905353546142578,
      "activations/layer13_attention_weight_max": 42.98896789550781,
      "activations/layer13_attention_weight_min": -36.02772903442383,
      "activations/layer14_attention_weight_max": 48.01689147949219,
      "activations/layer14_attention_weight_min": -42.9116325378418,
      "activations/layer15_attention_weight_max": 39.472190856933594,
      "activations/layer15_attention_weight_min": -34.14670181274414,
      "activations/layer16_attention_weight_max": 33.499786376953125,
      "activations/layer16_attention_weight_min": -29.030773162841797,
      "activations/layer17_attention_weight_max": 32.666481018066406,
      "activations/layer17_attention_weight_min": -27.274961471557617,
      "activations/layer18_attention_weight_max": 30.131855010986328,
      "activations/layer18_attention_weight_min": -27.748924255371094,
      "activations/layer19_attention_weight_max": 39.87553024291992,
      "activations/layer19_attention_weight_min": -32.43259048461914,
      "activations/layer1_attention_weight_max": 17.13941192626953,
      "activations/layer1_attention_weight_min": -14.732321739196777,
      "activations/layer20_attention_weight_max": 30.095224380493164,
      "activations/layer20_attention_weight_min": -26.3717098236084,
      "activations/layer21_attention_weight_max": 28.920236587524414,
      "activations/layer21_attention_weight_min": -26.66625213623047,
      "activations/layer22_attention_weight_max": 43.706695556640625,
      "activations/layer22_attention_weight_min": -33.0105094909668,
      "activations/layer23_attention_weight_max": 35.63688659667969,
      "activations/layer23_attention_weight_min": -26.45003318786621,
      "activations/layer2_attention_weight_max": 35.659549713134766,
      "activations/layer2_attention_weight_min": -34.25291061401367,
      "activations/layer3_attention_weight_max": 94.33587646484375,
      "activations/layer3_attention_weight_min": -96.65977478027344,
      "activations/layer4_attention_weight_max": 111.83943176269531,
      "activations/layer4_attention_weight_min": -110.7861328125,
      "activations/layer5_attention_weight_max": 53.41157150268555,
      "activations/layer5_attention_weight_min": -62.35143280029297,
      "activations/layer6_attention_weight_max": 45.634010314941406,
      "activations/layer6_attention_weight_min": -49.23750305175781,
      "activations/layer7_attention_weight_max": 99.2785873413086,
      "activations/layer7_attention_weight_min": -90.95502471923828,
      "activations/layer8_attention_weight_max": 45.1615104675293,
      "activations/layer8_attention_weight_min": -44.59061813354492,
      "activations/layer9_attention_weight_max": 37.08499526977539,
      "activations/layer9_attention_weight_min": -35.54849624633789,
      "epoch": 11.0,
      "learning_rate": 7.98905303030303e-05,
      "loss": 2.7568,
      "step": 189250
    },
    {
      "activations/layer0_attention_weight_max": 14.50381851196289,
      "activations/layer0_attention_weight_min": -11.81507396697998,
      "activations/layer10_attention_weight_max": 34.49514389038086,
      "activations/layer10_attention_weight_min": -31.052837371826172,
      "activations/layer11_attention_weight_max": 37.57325744628906,
      "activations/layer11_attention_weight_min": -31.633481979370117,
      "activations/layer12_attention_weight_max": 24.711502075195312,
      "activations/layer12_attention_weight_min": -25.397008895874023,
      "activations/layer13_attention_weight_max": 44.77014923095703,
      "activations/layer13_attention_weight_min": -34.8298454284668,
      "activations/layer14_attention_weight_max": 45.500205993652344,
      "activations/layer14_attention_weight_min": -40.43523025512695,
      "activations/layer15_attention_weight_max": 35.62556076049805,
      "activations/layer15_attention_weight_min": -31.811443328857422,
      "activations/layer16_attention_weight_max": 30.267330169677734,
      "activations/layer16_attention_weight_min": -27.225299835205078,
      "activations/layer17_attention_weight_max": 34.492698669433594,
      "activations/layer17_attention_weight_min": -25.385860443115234,
      "activations/layer18_attention_weight_max": 32.31885528564453,
      "activations/layer18_attention_weight_min": -23.08717918395996,
      "activations/layer19_attention_weight_max": 35.96095657348633,
      "activations/layer19_attention_weight_min": -31.768381118774414,
      "activations/layer1_attention_weight_max": 16.447174072265625,
      "activations/layer1_attention_weight_min": -14.211600303649902,
      "activations/layer20_attention_weight_max": 29.367198944091797,
      "activations/layer20_attention_weight_min": -23.902982711791992,
      "activations/layer21_attention_weight_max": 28.517290115356445,
      "activations/layer21_attention_weight_min": -23.375822067260742,
      "activations/layer22_attention_weight_max": 49.135536193847656,
      "activations/layer22_attention_weight_min": -31.983367919921875,
      "activations/layer23_attention_weight_max": 39.18626022338867,
      "activations/layer23_attention_weight_min": -25.82551383972168,
      "activations/layer2_attention_weight_max": 34.52136993408203,
      "activations/layer2_attention_weight_min": -34.02956771850586,
      "activations/layer3_attention_weight_max": 93.873779296875,
      "activations/layer3_attention_weight_min": -94.77852630615234,
      "activations/layer4_attention_weight_max": 111.96544647216797,
      "activations/layer4_attention_weight_min": -109.68990325927734,
      "activations/layer5_attention_weight_max": 53.29411315917969,
      "activations/layer5_attention_weight_min": -60.54928207397461,
      "activations/layer6_attention_weight_max": 44.860721588134766,
      "activations/layer6_attention_weight_min": -47.74645233154297,
      "activations/layer7_attention_weight_max": 94.80683135986328,
      "activations/layer7_attention_weight_min": -89.39070892333984,
      "activations/layer8_attention_weight_max": 42.38818359375,
      "activations/layer8_attention_weight_min": -40.378814697265625,
      "activations/layer9_attention_weight_max": 33.86301803588867,
      "activations/layer9_attention_weight_min": -32.79761505126953,
      "epoch": 11.0,
      "learning_rate": 7.98715909090909e-05,
      "loss": 2.7767,
      "step": 189300
    },
    {
      "activations/layer0_attention_weight_max": 15.693901062011719,
      "activations/layer0_attention_weight_min": -12.82489013671875,
      "activations/layer10_attention_weight_max": 31.33074951171875,
      "activations/layer10_attention_weight_min": -31.174230575561523,
      "activations/layer11_attention_weight_max": 31.651813507080078,
      "activations/layer11_attention_weight_min": -31.135969161987305,
      "activations/layer12_attention_weight_max": 25.440393447875977,
      "activations/layer12_attention_weight_min": -24.169857025146484,
      "activations/layer13_attention_weight_max": 40.15635299682617,
      "activations/layer13_attention_weight_min": -34.64801788330078,
      "activations/layer14_attention_weight_max": 48.530399322509766,
      "activations/layer14_attention_weight_min": -38.22062301635742,
      "activations/layer15_attention_weight_max": 36.65304946899414,
      "activations/layer15_attention_weight_min": -30.726133346557617,
      "activations/layer16_attention_weight_max": 32.0238037109375,
      "activations/layer16_attention_weight_min": -26.682266235351562,
      "activations/layer17_attention_weight_max": 33.62489700317383,
      "activations/layer17_attention_weight_min": -26.02326774597168,
      "activations/layer18_attention_weight_max": 31.97469711303711,
      "activations/layer18_attention_weight_min": -23.805675506591797,
      "activations/layer19_attention_weight_max": 34.37131881713867,
      "activations/layer19_attention_weight_min": -29.25069236755371,
      "activations/layer1_attention_weight_max": 16.30308723449707,
      "activations/layer1_attention_weight_min": -15.119783401489258,
      "activations/layer20_attention_weight_max": 31.20525360107422,
      "activations/layer20_attention_weight_min": -23.5723934173584,
      "activations/layer21_attention_weight_max": 30.15230941772461,
      "activations/layer21_attention_weight_min": -24.140275955200195,
      "activations/layer22_attention_weight_max": 48.17080307006836,
      "activations/layer22_attention_weight_min": -29.25934600830078,
      "activations/layer23_attention_weight_max": 33.53962326049805,
      "activations/layer23_attention_weight_min": -23.958404541015625,
      "activations/layer2_attention_weight_max": 32.32088088989258,
      "activations/layer2_attention_weight_min": -34.3978271484375,
      "activations/layer3_attention_weight_max": 95.54855346679688,
      "activations/layer3_attention_weight_min": -95.84388732910156,
      "activations/layer4_attention_weight_max": 112.4427490234375,
      "activations/layer4_attention_weight_min": -112.49791717529297,
      "activations/layer5_attention_weight_max": 51.96238708496094,
      "activations/layer5_attention_weight_min": -60.1938591003418,
      "activations/layer6_attention_weight_max": 46.577144622802734,
      "activations/layer6_attention_weight_min": -49.428958892822266,
      "activations/layer7_attention_weight_max": 92.97540283203125,
      "activations/layer7_attention_weight_min": -88.23430633544922,
      "activations/layer8_attention_weight_max": 39.87623596191406,
      "activations/layer8_attention_weight_min": -40.80025100708008,
      "activations/layer9_attention_weight_max": 31.450000762939453,
      "activations/layer9_attention_weight_min": -32.00346374511719,
      "epoch": 11.0,
      "learning_rate": 7.98526515151515e-05,
      "loss": 2.831,
      "step": 189350
    },
    {
      "activations/layer0_attention_weight_max": 16.478376388549805,
      "activations/layer0_attention_weight_min": -12.720796585083008,
      "activations/layer10_attention_weight_max": 30.154775619506836,
      "activations/layer10_attention_weight_min": -31.009521484375,
      "activations/layer11_attention_weight_max": 31.163516998291016,
      "activations/layer11_attention_weight_min": -30.620132446289062,
      "activations/layer12_attention_weight_max": 27.043481826782227,
      "activations/layer12_attention_weight_min": -26.748899459838867,
      "activations/layer13_attention_weight_max": 41.29261016845703,
      "activations/layer13_attention_weight_min": -37.655696868896484,
      "activations/layer14_attention_weight_max": 46.55186080932617,
      "activations/layer14_attention_weight_min": -37.64406967163086,
      "activations/layer15_attention_weight_max": 37.18031692504883,
      "activations/layer15_attention_weight_min": -31.153270721435547,
      "activations/layer16_attention_weight_max": 32.747535705566406,
      "activations/layer16_attention_weight_min": -27.006742477416992,
      "activations/layer17_attention_weight_max": 34.008880615234375,
      "activations/layer17_attention_weight_min": -24.25691795349121,
      "activations/layer18_attention_weight_max": 28.87676429748535,
      "activations/layer18_attention_weight_min": -22.67510223388672,
      "activations/layer19_attention_weight_max": 35.12788009643555,
      "activations/layer19_attention_weight_min": -33.20547103881836,
      "activations/layer1_attention_weight_max": 15.981034278869629,
      "activations/layer1_attention_weight_min": -13.069944381713867,
      "activations/layer20_attention_weight_max": 32.330955505371094,
      "activations/layer20_attention_weight_min": -25.348852157592773,
      "activations/layer21_attention_weight_max": 29.403671264648438,
      "activations/layer21_attention_weight_min": -23.26885986328125,
      "activations/layer22_attention_weight_max": 42.15385055541992,
      "activations/layer22_attention_weight_min": -30.01397705078125,
      "activations/layer23_attention_weight_max": 36.63052749633789,
      "activations/layer23_attention_weight_min": -25.514816284179688,
      "activations/layer2_attention_weight_max": 29.661991119384766,
      "activations/layer2_attention_weight_min": -27.98940658569336,
      "activations/layer3_attention_weight_max": 79.60324096679688,
      "activations/layer3_attention_weight_min": -82.37836456298828,
      "activations/layer4_attention_weight_max": 95.6802978515625,
      "activations/layer4_attention_weight_min": -94.13890838623047,
      "activations/layer5_attention_weight_max": 48.81310272216797,
      "activations/layer5_attention_weight_min": -56.65733337402344,
      "activations/layer6_attention_weight_max": 39.81755065917969,
      "activations/layer6_attention_weight_min": -43.83273696899414,
      "activations/layer7_attention_weight_max": 79.1854248046875,
      "activations/layer7_attention_weight_min": -82.020263671875,
      "activations/layer8_attention_weight_max": 36.58835983276367,
      "activations/layer8_attention_weight_min": -38.31226348876953,
      "activations/layer9_attention_weight_max": 30.0103702545166,
      "activations/layer9_attention_weight_min": -29.17280387878418,
      "epoch": 11.01,
      "learning_rate": 7.98340909090909e-05,
      "loss": 2.771,
      "step": 189400
    },
    {
      "activations/layer0_attention_weight_max": 15.859981536865234,
      "activations/layer0_attention_weight_min": -12.96147632598877,
      "activations/layer10_attention_weight_max": 33.721527099609375,
      "activations/layer10_attention_weight_min": -33.2800178527832,
      "activations/layer11_attention_weight_max": 35.326759338378906,
      "activations/layer11_attention_weight_min": -32.107643127441406,
      "activations/layer12_attention_weight_max": 25.23926544189453,
      "activations/layer12_attention_weight_min": -25.36151695251465,
      "activations/layer13_attention_weight_max": 38.16334915161133,
      "activations/layer13_attention_weight_min": -32.46632385253906,
      "activations/layer14_attention_weight_max": 45.70050811767578,
      "activations/layer14_attention_weight_min": -36.445091247558594,
      "activations/layer15_attention_weight_max": 36.433502197265625,
      "activations/layer15_attention_weight_min": -29.94264793395996,
      "activations/layer16_attention_weight_max": 30.58811378479004,
      "activations/layer16_attention_weight_min": -26.941343307495117,
      "activations/layer17_attention_weight_max": 32.025428771972656,
      "activations/layer17_attention_weight_min": -25.01763343811035,
      "activations/layer18_attention_weight_max": 30.052724838256836,
      "activations/layer18_attention_weight_min": -22.801589965820312,
      "activations/layer19_attention_weight_max": 35.9019889831543,
      "activations/layer19_attention_weight_min": -32.79954147338867,
      "activations/layer1_attention_weight_max": 16.74790382385254,
      "activations/layer1_attention_weight_min": -14.175300598144531,
      "activations/layer20_attention_weight_max": 28.52395248413086,
      "activations/layer20_attention_weight_min": -25.901212692260742,
      "activations/layer21_attention_weight_max": 30.345279693603516,
      "activations/layer21_attention_weight_min": -22.63161277770996,
      "activations/layer22_attention_weight_max": 48.328857421875,
      "activations/layer22_attention_weight_min": -29.143009185791016,
      "activations/layer23_attention_weight_max": 37.251121520996094,
      "activations/layer23_attention_weight_min": -24.154813766479492,
      "activations/layer2_attention_weight_max": 33.7183723449707,
      "activations/layer2_attention_weight_min": -34.341285705566406,
      "activations/layer3_attention_weight_max": 94.94141387939453,
      "activations/layer3_attention_weight_min": -96.29845428466797,
      "activations/layer4_attention_weight_max": 114.88579559326172,
      "activations/layer4_attention_weight_min": -112.48282623291016,
      "activations/layer5_attention_weight_max": 52.614219665527344,
      "activations/layer5_attention_weight_min": -61.95608139038086,
      "activations/layer6_attention_weight_max": 47.14493942260742,
      "activations/layer6_attention_weight_min": -50.227577209472656,
      "activations/layer7_attention_weight_max": 99.3003921508789,
      "activations/layer7_attention_weight_min": -94.02606201171875,
      "activations/layer8_attention_weight_max": 42.2191162109375,
      "activations/layer8_attention_weight_min": -43.88606643676758,
      "activations/layer9_attention_weight_max": 35.83323287963867,
      "activations/layer9_attention_weight_min": -33.65557098388672,
      "epoch": 11.01,
      "learning_rate": 7.981515151515151e-05,
      "loss": 2.7721,
      "step": 189450
    },
    {
      "activations/layer0_attention_weight_max": 15.404374122619629,
      "activations/layer0_attention_weight_min": -12.206705093383789,
      "activations/layer10_attention_weight_max": 33.79331970214844,
      "activations/layer10_attention_weight_min": -33.364253997802734,
      "activations/layer11_attention_weight_max": 34.33879089355469,
      "activations/layer11_attention_weight_min": -31.549549102783203,
      "activations/layer12_attention_weight_max": 24.998733520507812,
      "activations/layer12_attention_weight_min": -24.751195907592773,
      "activations/layer13_attention_weight_max": 39.55169677734375,
      "activations/layer13_attention_weight_min": -33.51353454589844,
      "activations/layer14_attention_weight_max": 50.94055938720703,
      "activations/layer14_attention_weight_min": -39.58601760864258,
      "activations/layer15_attention_weight_max": 38.84328842163086,
      "activations/layer15_attention_weight_min": -33.054100036621094,
      "activations/layer16_attention_weight_max": 31.439647674560547,
      "activations/layer16_attention_weight_min": -28.21771240234375,
      "activations/layer17_attention_weight_max": 34.47153854370117,
      "activations/layer17_attention_weight_min": -25.2138671875,
      "activations/layer18_attention_weight_max": 32.621726989746094,
      "activations/layer18_attention_weight_min": -22.80135726928711,
      "activations/layer19_attention_weight_max": 35.22539138793945,
      "activations/layer19_attention_weight_min": -30.51568603515625,
      "activations/layer1_attention_weight_max": 17.847694396972656,
      "activations/layer1_attention_weight_min": -13.253007888793945,
      "activations/layer20_attention_weight_max": 29.10271453857422,
      "activations/layer20_attention_weight_min": -24.170372009277344,
      "activations/layer21_attention_weight_max": 28.234331130981445,
      "activations/layer21_attention_weight_min": -23.2877254486084,
      "activations/layer22_attention_weight_max": 41.99130630493164,
      "activations/layer22_attention_weight_min": -29.62139892578125,
      "activations/layer23_attention_weight_max": 32.53546142578125,
      "activations/layer23_attention_weight_min": -24.851806640625,
      "activations/layer2_attention_weight_max": 33.124881744384766,
      "activations/layer2_attention_weight_min": -31.920604705810547,
      "activations/layer3_attention_weight_max": 92.4787368774414,
      "activations/layer3_attention_weight_min": -91.86190032958984,
      "activations/layer4_attention_weight_max": 110.48294830322266,
      "activations/layer4_attention_weight_min": -112.8438720703125,
      "activations/layer5_attention_weight_max": 52.443817138671875,
      "activations/layer5_attention_weight_min": -65.9461669921875,
      "activations/layer6_attention_weight_max": 47.63670349121094,
      "activations/layer6_attention_weight_min": -50.10248947143555,
      "activations/layer7_attention_weight_max": 92.8781967163086,
      "activations/layer7_attention_weight_min": -96.72999572753906,
      "activations/layer8_attention_weight_max": 43.37163162231445,
      "activations/layer8_attention_weight_min": -43.01043701171875,
      "activations/layer9_attention_weight_max": 36.09169387817383,
      "activations/layer9_attention_weight_min": -34.66063690185547,
      "epoch": 11.01,
      "learning_rate": 7.979621212121212e-05,
      "loss": 2.7809,
      "step": 189500
    },
    {
      "activations/layer0_attention_weight_max": 14.32088851928711,
      "activations/layer0_attention_weight_min": -11.651634216308594,
      "activations/layer10_attention_weight_max": 31.38493537902832,
      "activations/layer10_attention_weight_min": -30.39623260498047,
      "activations/layer11_attention_weight_max": 30.384090423583984,
      "activations/layer11_attention_weight_min": -29.975997924804688,
      "activations/layer12_attention_weight_max": 25.626981735229492,
      "activations/layer12_attention_weight_min": -24.139022827148438,
      "activations/layer13_attention_weight_max": 38.324493408203125,
      "activations/layer13_attention_weight_min": -37.054603576660156,
      "activations/layer14_attention_weight_max": 47.98971176147461,
      "activations/layer14_attention_weight_min": -36.02301025390625,
      "activations/layer15_attention_weight_max": 34.52525329589844,
      "activations/layer15_attention_weight_min": -31.838577270507812,
      "activations/layer16_attention_weight_max": 29.648645401000977,
      "activations/layer16_attention_weight_min": -28.140825271606445,
      "activations/layer17_attention_weight_max": 30.506887435913086,
      "activations/layer17_attention_weight_min": -26.974641799926758,
      "activations/layer18_attention_weight_max": 31.576053619384766,
      "activations/layer18_attention_weight_min": -22.555728912353516,
      "activations/layer19_attention_weight_max": 33.07237243652344,
      "activations/layer19_attention_weight_min": -31.14267349243164,
      "activations/layer1_attention_weight_max": 16.164955139160156,
      "activations/layer1_attention_weight_min": -13.220419883728027,
      "activations/layer20_attention_weight_max": 29.075603485107422,
      "activations/layer20_attention_weight_min": -24.256750106811523,
      "activations/layer21_attention_weight_max": 30.57955551147461,
      "activations/layer21_attention_weight_min": -24.945812225341797,
      "activations/layer22_attention_weight_max": 47.1905517578125,
      "activations/layer22_attention_weight_min": -31.102428436279297,
      "activations/layer23_attention_weight_max": 35.060508728027344,
      "activations/layer23_attention_weight_min": -24.565519332885742,
      "activations/layer2_attention_weight_max": 30.557586669921875,
      "activations/layer2_attention_weight_min": -30.758939743041992,
      "activations/layer3_attention_weight_max": 87.35323333740234,
      "activations/layer3_attention_weight_min": -90.76846313476562,
      "activations/layer4_attention_weight_max": 108.2265853881836,
      "activations/layer4_attention_weight_min": -103.9681396484375,
      "activations/layer5_attention_weight_max": 50.085968017578125,
      "activations/layer5_attention_weight_min": -61.1582145690918,
      "activations/layer6_attention_weight_max": 42.685447692871094,
      "activations/layer6_attention_weight_min": -47.52269744873047,
      "activations/layer7_attention_weight_max": 85.97361755371094,
      "activations/layer7_attention_weight_min": -88.1340103149414,
      "activations/layer8_attention_weight_max": 39.888771057128906,
      "activations/layer8_attention_weight_min": -43.0209846496582,
      "activations/layer9_attention_weight_max": 32.98067855834961,
      "activations/layer9_attention_weight_min": -32.984256744384766,
      "epoch": 11.01,
      "learning_rate": 7.977727272727272e-05,
      "loss": 2.7744,
      "step": 189550
    },
    {
      "activations/layer0_attention_weight_max": 14.95467758178711,
      "activations/layer0_attention_weight_min": -12.880577087402344,
      "activations/layer10_attention_weight_max": 35.51968765258789,
      "activations/layer10_attention_weight_min": -35.0655632019043,
      "activations/layer11_attention_weight_max": 36.43498229980469,
      "activations/layer11_attention_weight_min": -33.708961486816406,
      "activations/layer12_attention_weight_max": 27.885005950927734,
      "activations/layer12_attention_weight_min": -27.686180114746094,
      "activations/layer13_attention_weight_max": 43.93897247314453,
      "activations/layer13_attention_weight_min": -37.90941619873047,
      "activations/layer14_attention_weight_max": 55.437171936035156,
      "activations/layer14_attention_weight_min": -41.39375305175781,
      "activations/layer15_attention_weight_max": 43.40103530883789,
      "activations/layer15_attention_weight_min": -33.714256286621094,
      "activations/layer16_attention_weight_max": 40.29178237915039,
      "activations/layer16_attention_weight_min": -29.453683853149414,
      "activations/layer17_attention_weight_max": 39.35438537597656,
      "activations/layer17_attention_weight_min": -27.85916519165039,
      "activations/layer18_attention_weight_max": 36.69840621948242,
      "activations/layer18_attention_weight_min": -25.1093692779541,
      "activations/layer19_attention_weight_max": 46.157352447509766,
      "activations/layer19_attention_weight_min": -32.9151496887207,
      "activations/layer1_attention_weight_max": 16.739503860473633,
      "activations/layer1_attention_weight_min": -15.415556907653809,
      "activations/layer20_attention_weight_max": 40.53112030029297,
      "activations/layer20_attention_weight_min": -26.11345863342285,
      "activations/layer21_attention_weight_max": 38.33610916137695,
      "activations/layer21_attention_weight_min": -25.14584732055664,
      "activations/layer22_attention_weight_max": 55.83499526977539,
      "activations/layer22_attention_weight_min": -31.392866134643555,
      "activations/layer23_attention_weight_max": 45.7158317565918,
      "activations/layer23_attention_weight_min": -26.14179039001465,
      "activations/layer2_attention_weight_max": 32.75917053222656,
      "activations/layer2_attention_weight_min": -33.07014465332031,
      "activations/layer3_attention_weight_max": 93.34449005126953,
      "activations/layer3_attention_weight_min": -96.74214935302734,
      "activations/layer4_attention_weight_max": 113.4109878540039,
      "activations/layer4_attention_weight_min": -112.71855926513672,
      "activations/layer5_attention_weight_max": 55.64130401611328,
      "activations/layer5_attention_weight_min": -62.79450988769531,
      "activations/layer6_attention_weight_max": 45.6068000793457,
      "activations/layer6_attention_weight_min": -52.2485237121582,
      "activations/layer7_attention_weight_max": 98.3092269897461,
      "activations/layer7_attention_weight_min": -92.7935562133789,
      "activations/layer8_attention_weight_max": 43.8409309387207,
      "activations/layer8_attention_weight_min": -44.097564697265625,
      "activations/layer9_attention_weight_max": 34.806907653808594,
      "activations/layer9_attention_weight_min": -33.928157806396484,
      "epoch": 11.02,
      "learning_rate": 7.975833333333332e-05,
      "loss": 2.7688,
      "step": 189600
    },
    {
      "activations/layer0_attention_weight_max": 14.993513107299805,
      "activations/layer0_attention_weight_min": -12.936993598937988,
      "activations/layer10_attention_weight_max": 30.832054138183594,
      "activations/layer10_attention_weight_min": -33.20573043823242,
      "activations/layer11_attention_weight_max": 31.59487533569336,
      "activations/layer11_attention_weight_min": -30.86963653564453,
      "activations/layer12_attention_weight_max": 23.906293869018555,
      "activations/layer12_attention_weight_min": -25.01468276977539,
      "activations/layer13_attention_weight_max": 36.576045989990234,
      "activations/layer13_attention_weight_min": -34.12367248535156,
      "activations/layer14_attention_weight_max": 41.91611862182617,
      "activations/layer14_attention_weight_min": -35.90666198730469,
      "activations/layer15_attention_weight_max": 33.432613372802734,
      "activations/layer15_attention_weight_min": -32.91815185546875,
      "activations/layer16_attention_weight_max": 28.643943786621094,
      "activations/layer16_attention_weight_min": -29.040760040283203,
      "activations/layer17_attention_weight_max": 32.322998046875,
      "activations/layer17_attention_weight_min": -26.521631240844727,
      "activations/layer18_attention_weight_max": 30.03868293762207,
      "activations/layer18_attention_weight_min": -23.86390495300293,
      "activations/layer19_attention_weight_max": 33.14570236206055,
      "activations/layer19_attention_weight_min": -30.141454696655273,
      "activations/layer1_attention_weight_max": 16.684328079223633,
      "activations/layer1_attention_weight_min": -14.28199291229248,
      "activations/layer20_attention_weight_max": 31.638132095336914,
      "activations/layer20_attention_weight_min": -25.60625457763672,
      "activations/layer21_attention_weight_max": 28.071800231933594,
      "activations/layer21_attention_weight_min": -24.275882720947266,
      "activations/layer22_attention_weight_max": 42.59103012084961,
      "activations/layer22_attention_weight_min": -29.700666427612305,
      "activations/layer23_attention_weight_max": 36.070281982421875,
      "activations/layer23_attention_weight_min": -25.855405807495117,
      "activations/layer2_attention_weight_max": 31.938579559326172,
      "activations/layer2_attention_weight_min": -33.65630340576172,
      "activations/layer3_attention_weight_max": 93.00328063964844,
      "activations/layer3_attention_weight_min": -96.58531188964844,
      "activations/layer4_attention_weight_max": 110.69329833984375,
      "activations/layer4_attention_weight_min": -113.4024887084961,
      "activations/layer5_attention_weight_max": 50.882591247558594,
      "activations/layer5_attention_weight_min": -60.497718811035156,
      "activations/layer6_attention_weight_max": 44.737709045410156,
      "activations/layer6_attention_weight_min": -47.16916275024414,
      "activations/layer7_attention_weight_max": 94.31730651855469,
      "activations/layer7_attention_weight_min": -89.06346893310547,
      "activations/layer8_attention_weight_max": 39.06215286254883,
      "activations/layer8_attention_weight_min": -40.10991287231445,
      "activations/layer9_attention_weight_max": 31.48695945739746,
      "activations/layer9_attention_weight_min": -33.344173431396484,
      "epoch": 11.02,
      "learning_rate": 7.973939393939394e-05,
      "loss": 2.7633,
      "step": 189650
    },
    {
      "activations/layer0_attention_weight_max": 14.971766471862793,
      "activations/layer0_attention_weight_min": -13.09662914276123,
      "activations/layer10_attention_weight_max": 31.687570571899414,
      "activations/layer10_attention_weight_min": -31.498985290527344,
      "activations/layer11_attention_weight_max": 32.72014617919922,
      "activations/layer11_attention_weight_min": -31.328704833984375,
      "activations/layer12_attention_weight_max": 41.962154388427734,
      "activations/layer12_attention_weight_min": -24.454801559448242,
      "activations/layer13_attention_weight_max": 44.07007598876953,
      "activations/layer13_attention_weight_min": -34.8191032409668,
      "activations/layer14_attention_weight_max": 44.455284118652344,
      "activations/layer14_attention_weight_min": -39.76999282836914,
      "activations/layer15_attention_weight_max": 39.885353088378906,
      "activations/layer15_attention_weight_min": -33.24443817138672,
      "activations/layer16_attention_weight_max": 31.004627227783203,
      "activations/layer16_attention_weight_min": -27.891265869140625,
      "activations/layer17_attention_weight_max": 30.883941650390625,
      "activations/layer17_attention_weight_min": -24.85763931274414,
      "activations/layer18_attention_weight_max": 32.338069915771484,
      "activations/layer18_attention_weight_min": -22.39237403869629,
      "activations/layer19_attention_weight_max": 41.316078186035156,
      "activations/layer19_attention_weight_min": -32.71619415283203,
      "activations/layer1_attention_weight_max": 16.46364402770996,
      "activations/layer1_attention_weight_min": -14.801050186157227,
      "activations/layer20_attention_weight_max": 33.88353729248047,
      "activations/layer20_attention_weight_min": -25.30889320373535,
      "activations/layer21_attention_weight_max": 32.945281982421875,
      "activations/layer21_attention_weight_min": -24.162975311279297,
      "activations/layer22_attention_weight_max": 45.31541442871094,
      "activations/layer22_attention_weight_min": -32.127952575683594,
      "activations/layer23_attention_weight_max": 37.529090881347656,
      "activations/layer23_attention_weight_min": -24.323917388916016,
      "activations/layer2_attention_weight_max": 32.6724853515625,
      "activations/layer2_attention_weight_min": -33.56028366088867,
      "activations/layer3_attention_weight_max": 93.0730972290039,
      "activations/layer3_attention_weight_min": -95.5809097290039,
      "activations/layer4_attention_weight_max": 113.92962646484375,
      "activations/layer4_attention_weight_min": -108.87945556640625,
      "activations/layer5_attention_weight_max": 52.59391403198242,
      "activations/layer5_attention_weight_min": -61.51654052734375,
      "activations/layer6_attention_weight_max": 44.50225067138672,
      "activations/layer6_attention_weight_min": -46.63228988647461,
      "activations/layer7_attention_weight_max": 88.67144775390625,
      "activations/layer7_attention_weight_min": -88.57865142822266,
      "activations/layer8_attention_weight_max": 39.81744384765625,
      "activations/layer8_attention_weight_min": -40.12024688720703,
      "activations/layer9_attention_weight_max": 33.55295944213867,
      "activations/layer9_attention_weight_min": -31.669876098632812,
      "epoch": 11.02,
      "learning_rate": 7.972045454545453e-05,
      "loss": 2.7612,
      "step": 189700
    },
    {
      "activations/layer0_attention_weight_max": 14.703911781311035,
      "activations/layer0_attention_weight_min": -13.023276329040527,
      "activations/layer10_attention_weight_max": 34.15887451171875,
      "activations/layer10_attention_weight_min": -33.192996978759766,
      "activations/layer11_attention_weight_max": 31.89482879638672,
      "activations/layer11_attention_weight_min": -32.002071380615234,
      "activations/layer12_attention_weight_max": 25.411792755126953,
      "activations/layer12_attention_weight_min": -24.714874267578125,
      "activations/layer13_attention_weight_max": 40.766845703125,
      "activations/layer13_attention_weight_min": -33.475982666015625,
      "activations/layer14_attention_weight_max": 47.566650390625,
      "activations/layer14_attention_weight_min": -39.71214294433594,
      "activations/layer15_attention_weight_max": 41.2489128112793,
      "activations/layer15_attention_weight_min": -33.05073547363281,
      "activations/layer16_attention_weight_max": 35.56795883178711,
      "activations/layer16_attention_weight_min": -28.183473587036133,
      "activations/layer17_attention_weight_max": 34.395347595214844,
      "activations/layer17_attention_weight_min": -27.774721145629883,
      "activations/layer18_attention_weight_max": 32.692222595214844,
      "activations/layer18_attention_weight_min": -23.483089447021484,
      "activations/layer19_attention_weight_max": 36.06339645385742,
      "activations/layer19_attention_weight_min": -30.01675796508789,
      "activations/layer1_attention_weight_max": 16.97963523864746,
      "activations/layer1_attention_weight_min": -15.200858116149902,
      "activations/layer20_attention_weight_max": 30.85039520263672,
      "activations/layer20_attention_weight_min": -23.974193572998047,
      "activations/layer21_attention_weight_max": 34.78636932373047,
      "activations/layer21_attention_weight_min": -23.715410232543945,
      "activations/layer22_attention_weight_max": 46.01675033569336,
      "activations/layer22_attention_weight_min": -28.460895538330078,
      "activations/layer23_attention_weight_max": 34.273006439208984,
      "activations/layer23_attention_weight_min": -24.492046356201172,
      "activations/layer2_attention_weight_max": 32.01013946533203,
      "activations/layer2_attention_weight_min": -33.15726089477539,
      "activations/layer3_attention_weight_max": 91.08451843261719,
      "activations/layer3_attention_weight_min": -89.69488525390625,
      "activations/layer4_attention_weight_max": 106.74812316894531,
      "activations/layer4_attention_weight_min": -108.73241424560547,
      "activations/layer5_attention_weight_max": 50.74711608886719,
      "activations/layer5_attention_weight_min": -62.31364440917969,
      "activations/layer6_attention_weight_max": 42.80461120605469,
      "activations/layer6_attention_weight_min": -49.8337516784668,
      "activations/layer7_attention_weight_max": 89.91344451904297,
      "activations/layer7_attention_weight_min": -90.79125213623047,
      "activations/layer8_attention_weight_max": 39.47370529174805,
      "activations/layer8_attention_weight_min": -40.497657775878906,
      "activations/layer9_attention_weight_max": 34.24006271362305,
      "activations/layer9_attention_weight_min": -33.73805236816406,
      "epoch": 11.03,
      "learning_rate": 7.970151515151514e-05,
      "loss": 2.7604,
      "step": 189750
    },
    {
      "activations/layer0_attention_weight_max": 14.890746116638184,
      "activations/layer0_attention_weight_min": -13.979755401611328,
      "activations/layer10_attention_weight_max": 29.804916381835938,
      "activations/layer10_attention_weight_min": -31.38838768005371,
      "activations/layer11_attention_weight_max": 29.579259872436523,
      "activations/layer11_attention_weight_min": -31.028059005737305,
      "activations/layer12_attention_weight_max": 22.69956398010254,
      "activations/layer12_attention_weight_min": -24.435775756835938,
      "activations/layer13_attention_weight_max": 34.51359176635742,
      "activations/layer13_attention_weight_min": -33.666114807128906,
      "activations/layer14_attention_weight_max": 41.016605377197266,
      "activations/layer14_attention_weight_min": -38.22511291503906,
      "activations/layer15_attention_weight_max": 34.94381332397461,
      "activations/layer15_attention_weight_min": -30.43769073486328,
      "activations/layer16_attention_weight_max": 28.379316329956055,
      "activations/layer16_attention_weight_min": -27.68498992919922,
      "activations/layer17_attention_weight_max": 30.858102798461914,
      "activations/layer17_attention_weight_min": -27.615093231201172,
      "activations/layer18_attention_weight_max": 30.551359176635742,
      "activations/layer18_attention_weight_min": -24.385334014892578,
      "activations/layer19_attention_weight_max": 33.61343002319336,
      "activations/layer19_attention_weight_min": -32.43391799926758,
      "activations/layer1_attention_weight_max": 16.691822052001953,
      "activations/layer1_attention_weight_min": -15.08221435546875,
      "activations/layer20_attention_weight_max": 29.273181915283203,
      "activations/layer20_attention_weight_min": -25.258703231811523,
      "activations/layer21_attention_weight_max": 27.319326400756836,
      "activations/layer21_attention_weight_min": -24.619840621948242,
      "activations/layer22_attention_weight_max": 47.282413482666016,
      "activations/layer22_attention_weight_min": -32.58485794067383,
      "activations/layer23_attention_weight_max": 37.300926208496094,
      "activations/layer23_attention_weight_min": -27.223560333251953,
      "activations/layer2_attention_weight_max": 32.41404724121094,
      "activations/layer2_attention_weight_min": -32.57810592651367,
      "activations/layer3_attention_weight_max": 91.30113220214844,
      "activations/layer3_attention_weight_min": -93.81354522705078,
      "activations/layer4_attention_weight_max": 109.6345443725586,
      "activations/layer4_attention_weight_min": -109.64583587646484,
      "activations/layer5_attention_weight_max": 56.35430145263672,
      "activations/layer5_attention_weight_min": -59.99263000488281,
      "activations/layer6_attention_weight_max": 43.36486053466797,
      "activations/layer6_attention_weight_min": -46.77229690551758,
      "activations/layer7_attention_weight_max": 84.6375503540039,
      "activations/layer7_attention_weight_min": -86.58647155761719,
      "activations/layer8_attention_weight_max": 38.83820343017578,
      "activations/layer8_attention_weight_min": -40.21132278442383,
      "activations/layer9_attention_weight_max": 30.8918514251709,
      "activations/layer9_attention_weight_min": -32.07505416870117,
      "epoch": 11.03,
      "learning_rate": 7.968257575757576e-05,
      "loss": 2.7646,
      "step": 189800
    },
    {
      "activations/layer0_attention_weight_max": 14.133418083190918,
      "activations/layer0_attention_weight_min": -12.809309959411621,
      "activations/layer10_attention_weight_max": 33.635780334472656,
      "activations/layer10_attention_weight_min": -33.50017547607422,
      "activations/layer11_attention_weight_max": 35.1631965637207,
      "activations/layer11_attention_weight_min": -32.486053466796875,
      "activations/layer12_attention_weight_max": 25.19424819946289,
      "activations/layer12_attention_weight_min": -26.31687355041504,
      "activations/layer13_attention_weight_max": 39.051856994628906,
      "activations/layer13_attention_weight_min": -34.99950408935547,
      "activations/layer14_attention_weight_max": 45.855987548828125,
      "activations/layer14_attention_weight_min": -38.79719543457031,
      "activations/layer15_attention_weight_max": 36.163272857666016,
      "activations/layer15_attention_weight_min": -32.838382720947266,
      "activations/layer16_attention_weight_max": 32.00651550292969,
      "activations/layer16_attention_weight_min": -28.714313507080078,
      "activations/layer17_attention_weight_max": 33.119876861572266,
      "activations/layer17_attention_weight_min": -26.536941528320312,
      "activations/layer18_attention_weight_max": 33.539527893066406,
      "activations/layer18_attention_weight_min": -24.376426696777344,
      "activations/layer19_attention_weight_max": 34.6080322265625,
      "activations/layer19_attention_weight_min": -30.230222702026367,
      "activations/layer1_attention_weight_max": 16.434293746948242,
      "activations/layer1_attention_weight_min": -14.228198051452637,
      "activations/layer20_attention_weight_max": 29.2579288482666,
      "activations/layer20_attention_weight_min": -24.862424850463867,
      "activations/layer21_attention_weight_max": 29.673004150390625,
      "activations/layer21_attention_weight_min": -23.6253662109375,
      "activations/layer22_attention_weight_max": 43.051509857177734,
      "activations/layer22_attention_weight_min": -30.728506088256836,
      "activations/layer23_attention_weight_max": 37.3023567199707,
      "activations/layer23_attention_weight_min": -24.924972534179688,
      "activations/layer2_attention_weight_max": 31.88939666748047,
      "activations/layer2_attention_weight_min": -31.704923629760742,
      "activations/layer3_attention_weight_max": 90.34297180175781,
      "activations/layer3_attention_weight_min": -89.67061614990234,
      "activations/layer4_attention_weight_max": 110.30853271484375,
      "activations/layer4_attention_weight_min": -109.30189514160156,
      "activations/layer5_attention_weight_max": 51.4452018737793,
      "activations/layer5_attention_weight_min": -59.76465606689453,
      "activations/layer6_attention_weight_max": 45.48308563232422,
      "activations/layer6_attention_weight_min": -48.94041061401367,
      "activations/layer7_attention_weight_max": 90.05331420898438,
      "activations/layer7_attention_weight_min": -89.5169448852539,
      "activations/layer8_attention_weight_max": 41.45680236816406,
      "activations/layer8_attention_weight_min": -40.54178237915039,
      "activations/layer9_attention_weight_max": 34.79939270019531,
      "activations/layer9_attention_weight_min": -32.89130783081055,
      "epoch": 11.03,
      "learning_rate": 7.966363636363635e-05,
      "loss": 2.7635,
      "step": 189850
    },
    {
      "activations/layer0_attention_weight_max": 14.902868270874023,
      "activations/layer0_attention_weight_min": -12.251331329345703,
      "activations/layer10_attention_weight_max": 36.78107452392578,
      "activations/layer10_attention_weight_min": -35.24818420410156,
      "activations/layer11_attention_weight_max": 36.42717361450195,
      "activations/layer11_attention_weight_min": -34.362648010253906,
      "activations/layer12_attention_weight_max": 26.13728141784668,
      "activations/layer12_attention_weight_min": -29.212692260742188,
      "activations/layer13_attention_weight_max": 39.44078063964844,
      "activations/layer13_attention_weight_min": -35.0045280456543,
      "activations/layer14_attention_weight_max": 44.83001708984375,
      "activations/layer14_attention_weight_min": -37.67326736450195,
      "activations/layer15_attention_weight_max": 39.033809661865234,
      "activations/layer15_attention_weight_min": -34.1824836730957,
      "activations/layer16_attention_weight_max": 35.8763542175293,
      "activations/layer16_attention_weight_min": -27.095531463623047,
      "activations/layer17_attention_weight_max": 34.31264114379883,
      "activations/layer17_attention_weight_min": -25.128782272338867,
      "activations/layer18_attention_weight_max": 33.75353240966797,
      "activations/layer18_attention_weight_min": -22.094928741455078,
      "activations/layer19_attention_weight_max": 38.377716064453125,
      "activations/layer19_attention_weight_min": -32.213008880615234,
      "activations/layer1_attention_weight_max": 17.22393035888672,
      "activations/layer1_attention_weight_min": -15.212203025817871,
      "activations/layer20_attention_weight_max": 33.61209487915039,
      "activations/layer20_attention_weight_min": -25.769817352294922,
      "activations/layer21_attention_weight_max": 32.48958206176758,
      "activations/layer21_attention_weight_min": -25.578088760375977,
      "activations/layer22_attention_weight_max": 50.982757568359375,
      "activations/layer22_attention_weight_min": -28.890819549560547,
      "activations/layer23_attention_weight_max": 39.40980529785156,
      "activations/layer23_attention_weight_min": -25.353164672851562,
      "activations/layer2_attention_weight_max": 33.577816009521484,
      "activations/layer2_attention_weight_min": -32.91522216796875,
      "activations/layer3_attention_weight_max": 101.22219848632812,
      "activations/layer3_attention_weight_min": -98.97727966308594,
      "activations/layer4_attention_weight_max": 115.0892562866211,
      "activations/layer4_attention_weight_min": -114.7249526977539,
      "activations/layer5_attention_weight_max": 59.51152038574219,
      "activations/layer5_attention_weight_min": -63.22179412841797,
      "activations/layer6_attention_weight_max": 45.08531951904297,
      "activations/layer6_attention_weight_min": -46.216312408447266,
      "activations/layer7_attention_weight_max": 95.29011535644531,
      "activations/layer7_attention_weight_min": -94.62232971191406,
      "activations/layer8_attention_weight_max": 47.5600700378418,
      "activations/layer8_attention_weight_min": -44.946067810058594,
      "activations/layer9_attention_weight_max": 35.05158233642578,
      "activations/layer9_attention_weight_min": -36.8516960144043,
      "epoch": 11.03,
      "learning_rate": 7.964469696969696e-05,
      "loss": 2.7823,
      "step": 189900
    },
    {
      "activations/layer0_attention_weight_max": 14.877848625183105,
      "activations/layer0_attention_weight_min": -11.888848304748535,
      "activations/layer10_attention_weight_max": 30.676456451416016,
      "activations/layer10_attention_weight_min": -31.64272689819336,
      "activations/layer11_attention_weight_max": 29.963132858276367,
      "activations/layer11_attention_weight_min": -29.674732208251953,
      "activations/layer12_attention_weight_max": 22.32496452331543,
      "activations/layer12_attention_weight_min": -25.479515075683594,
      "activations/layer13_attention_weight_max": 34.173519134521484,
      "activations/layer13_attention_weight_min": -33.233985900878906,
      "activations/layer14_attention_weight_max": 43.27934646606445,
      "activations/layer14_attention_weight_min": -35.47378921508789,
      "activations/layer15_attention_weight_max": 34.1622200012207,
      "activations/layer15_attention_weight_min": -30.37409782409668,
      "activations/layer16_attention_weight_max": 31.461666107177734,
      "activations/layer16_attention_weight_min": -27.833484649658203,
      "activations/layer17_attention_weight_max": 34.80580139160156,
      "activations/layer17_attention_weight_min": -24.730844497680664,
      "activations/layer18_attention_weight_max": 31.753665924072266,
      "activations/layer18_attention_weight_min": -22.55108070373535,
      "activations/layer19_attention_weight_max": 33.94546127319336,
      "activations/layer19_attention_weight_min": -30.289783477783203,
      "activations/layer1_attention_weight_max": 16.95380973815918,
      "activations/layer1_attention_weight_min": -15.976033210754395,
      "activations/layer20_attention_weight_max": 31.00905418395996,
      "activations/layer20_attention_weight_min": -24.377586364746094,
      "activations/layer21_attention_weight_max": 28.41855812072754,
      "activations/layer21_attention_weight_min": -23.920400619506836,
      "activations/layer22_attention_weight_max": 47.36280059814453,
      "activations/layer22_attention_weight_min": -30.102046966552734,
      "activations/layer23_attention_weight_max": 34.266075134277344,
      "activations/layer23_attention_weight_min": -24.144399642944336,
      "activations/layer2_attention_weight_max": 32.70988082885742,
      "activations/layer2_attention_weight_min": -33.20130157470703,
      "activations/layer3_attention_weight_max": 95.48494720458984,
      "activations/layer3_attention_weight_min": -97.29033660888672,
      "activations/layer4_attention_weight_max": 115.05494689941406,
      "activations/layer4_attention_weight_min": -115.12806701660156,
      "activations/layer5_attention_weight_max": 53.90326690673828,
      "activations/layer5_attention_weight_min": -63.889678955078125,
      "activations/layer6_attention_weight_max": 44.128108978271484,
      "activations/layer6_attention_weight_min": -48.5892219543457,
      "activations/layer7_attention_weight_max": 88.23711395263672,
      "activations/layer7_attention_weight_min": -89.96188354492188,
      "activations/layer8_attention_weight_max": 42.001556396484375,
      "activations/layer8_attention_weight_min": -44.09435272216797,
      "activations/layer9_attention_weight_max": 31.541584014892578,
      "activations/layer9_attention_weight_min": -33.70566940307617,
      "epoch": 11.04,
      "learning_rate": 7.962575757575758e-05,
      "loss": 2.7815,
      "step": 189950
    },
    {
      "activations/layer0_attention_weight_max": 15.739275932312012,
      "activations/layer0_attention_weight_min": -12.399723052978516,
      "activations/layer10_attention_weight_max": 32.33538818359375,
      "activations/layer10_attention_weight_min": -31.351383209228516,
      "activations/layer11_attention_weight_max": 32.10575866699219,
      "activations/layer11_attention_weight_min": -30.40624237060547,
      "activations/layer12_attention_weight_max": 23.555007934570312,
      "activations/layer12_attention_weight_min": -25.25542640686035,
      "activations/layer13_attention_weight_max": 37.4910888671875,
      "activations/layer13_attention_weight_min": -32.908443450927734,
      "activations/layer14_attention_weight_max": 41.31100082397461,
      "activations/layer14_attention_weight_min": -37.57400894165039,
      "activations/layer15_attention_weight_max": 34.11517333984375,
      "activations/layer15_attention_weight_min": -31.7092342376709,
      "activations/layer16_attention_weight_max": 30.567325592041016,
      "activations/layer16_attention_weight_min": -28.560569763183594,
      "activations/layer17_attention_weight_max": 31.35947036743164,
      "activations/layer17_attention_weight_min": -26.42369842529297,
      "activations/layer18_attention_weight_max": 31.310649871826172,
      "activations/layer18_attention_weight_min": -23.17859649658203,
      "activations/layer19_attention_weight_max": 32.85014343261719,
      "activations/layer19_attention_weight_min": -32.6063346862793,
      "activations/layer1_attention_weight_max": 17.07667350769043,
      "activations/layer1_attention_weight_min": -15.687743186950684,
      "activations/layer20_attention_weight_max": 30.115699768066406,
      "activations/layer20_attention_weight_min": -24.72274398803711,
      "activations/layer21_attention_weight_max": 28.261947631835938,
      "activations/layer21_attention_weight_min": -25.156814575195312,
      "activations/layer22_attention_weight_max": 44.7580451965332,
      "activations/layer22_attention_weight_min": -31.175399780273438,
      "activations/layer23_attention_weight_max": 35.45149612426758,
      "activations/layer23_attention_weight_min": -22.887670516967773,
      "activations/layer2_attention_weight_max": 34.76243591308594,
      "activations/layer2_attention_weight_min": -33.372013092041016,
      "activations/layer3_attention_weight_max": 93.39361572265625,
      "activations/layer3_attention_weight_min": -99.50244140625,
      "activations/layer4_attention_weight_max": 111.49043273925781,
      "activations/layer4_attention_weight_min": -113.8989486694336,
      "activations/layer5_attention_weight_max": 51.00019836425781,
      "activations/layer5_attention_weight_min": -60.043731689453125,
      "activations/layer6_attention_weight_max": 45.47197341918945,
      "activations/layer6_attention_weight_min": -47.160579681396484,
      "activations/layer7_attention_weight_max": 90.47554779052734,
      "activations/layer7_attention_weight_min": -86.29060363769531,
      "activations/layer8_attention_weight_max": 40.754425048828125,
      "activations/layer8_attention_weight_min": -40.36128616333008,
      "activations/layer9_attention_weight_max": 34.09556198120117,
      "activations/layer9_attention_weight_min": -33.35684585571289,
      "epoch": 11.04,
      "learning_rate": 7.960681818181818e-05,
      "loss": 2.7576,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5825,
      "eval_samples_per_second": 500.319,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5825,
      "eval_openwebtext_samples_per_second": 500.319,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.0028,
      "eval_wikitext_samples_per_second": 227.682,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.588,
      "eval_lambada_samples_per_second": 507.822,
      "step": 190000
    },
    {
      "activations/layer0_attention_weight_max": 15.383007049560547,
      "activations/layer0_attention_weight_min": -14.083969116210938,
      "activations/layer10_attention_weight_max": 33.86951446533203,
      "activations/layer10_attention_weight_min": -32.76974868774414,
      "activations/layer11_attention_weight_max": 34.17911911010742,
      "activations/layer11_attention_weight_min": -31.20905303955078,
      "activations/layer12_attention_weight_max": 25.272974014282227,
      "activations/layer12_attention_weight_min": -24.101896286010742,
      "activations/layer13_attention_weight_max": 41.37571716308594,
      "activations/layer13_attention_weight_min": -35.08454132080078,
      "activations/layer14_attention_weight_max": 43.6565055847168,
      "activations/layer14_attention_weight_min": -38.61348342895508,
      "activations/layer15_attention_weight_max": 35.51958465576172,
      "activations/layer15_attention_weight_min": -31.395267486572266,
      "activations/layer16_attention_weight_max": 30.36547088623047,
      "activations/layer16_attention_weight_min": -28.179800033569336,
      "activations/layer17_attention_weight_max": 30.709503173828125,
      "activations/layer17_attention_weight_min": -24.06527328491211,
      "activations/layer18_attention_weight_max": 30.08272933959961,
      "activations/layer18_attention_weight_min": -22.544918060302734,
      "activations/layer19_attention_weight_max": 32.563018798828125,
      "activations/layer19_attention_weight_min": -29.685928344726562,
      "activations/layer1_attention_weight_max": 16.52126693725586,
      "activations/layer1_attention_weight_min": -15.586915969848633,
      "activations/layer20_attention_weight_max": 28.335269927978516,
      "activations/layer20_attention_weight_min": -25.06283187866211,
      "activations/layer21_attention_weight_max": 26.91957664489746,
      "activations/layer21_attention_weight_min": -23.190399169921875,
      "activations/layer22_attention_weight_max": 43.740989685058594,
      "activations/layer22_attention_weight_min": -29.03013801574707,
      "activations/layer23_attention_weight_max": 32.48387145996094,
      "activations/layer23_attention_weight_min": -23.777767181396484,
      "activations/layer2_attention_weight_max": 33.57109451293945,
      "activations/layer2_attention_weight_min": -32.50644302368164,
      "activations/layer3_attention_weight_max": 95.04161071777344,
      "activations/layer3_attention_weight_min": -94.15501403808594,
      "activations/layer4_attention_weight_max": 113.3567886352539,
      "activations/layer4_attention_weight_min": -112.96004486083984,
      "activations/layer5_attention_weight_max": 52.42243957519531,
      "activations/layer5_attention_weight_min": -61.817726135253906,
      "activations/layer6_attention_weight_max": 46.25547409057617,
      "activations/layer6_attention_weight_min": -49.857730865478516,
      "activations/layer7_attention_weight_max": 92.77806091308594,
      "activations/layer7_attention_weight_min": -92.3182144165039,
      "activations/layer8_attention_weight_max": 40.28593826293945,
      "activations/layer8_attention_weight_min": -42.23541259765625,
      "activations/layer9_attention_weight_max": 33.5980339050293,
      "activations/layer9_attention_weight_min": -33.89755630493164,
      "epoch": 11.04,
      "learning_rate": 7.958787878787878e-05,
      "loss": 2.7737,
      "step": 190050
    },
    {
      "activations/layer0_attention_weight_max": 15.41997241973877,
      "activations/layer0_attention_weight_min": -11.898598670959473,
      "activations/layer10_attention_weight_max": 38.454811096191406,
      "activations/layer10_attention_weight_min": -38.00741958618164,
      "activations/layer11_attention_weight_max": 38.643898010253906,
      "activations/layer11_attention_weight_min": -33.80414581298828,
      "activations/layer12_attention_weight_max": 25.944046020507812,
      "activations/layer12_attention_weight_min": -24.37639808654785,
      "activations/layer13_attention_weight_max": 40.7961540222168,
      "activations/layer13_attention_weight_min": -36.44764709472656,
      "activations/layer14_attention_weight_max": 46.95197677612305,
      "activations/layer14_attention_weight_min": -40.01996994018555,
      "activations/layer15_attention_weight_max": 39.0564079284668,
      "activations/layer15_attention_weight_min": -32.78620529174805,
      "activations/layer16_attention_weight_max": 30.795608520507812,
      "activations/layer16_attention_weight_min": -29.00389289855957,
      "activations/layer17_attention_weight_max": 35.837852478027344,
      "activations/layer17_attention_weight_min": -25.810937881469727,
      "activations/layer18_attention_weight_max": 31.719566345214844,
      "activations/layer18_attention_weight_min": -22.321773529052734,
      "activations/layer19_attention_weight_max": 36.76424789428711,
      "activations/layer19_attention_weight_min": -31.48276710510254,
      "activations/layer1_attention_weight_max": 16.201887130737305,
      "activations/layer1_attention_weight_min": -15.039979934692383,
      "activations/layer20_attention_weight_max": 26.964603424072266,
      "activations/layer20_attention_weight_min": -23.875988006591797,
      "activations/layer21_attention_weight_max": 25.112144470214844,
      "activations/layer21_attention_weight_min": -22.324609756469727,
      "activations/layer22_attention_weight_max": 44.82740020751953,
      "activations/layer22_attention_weight_min": -30.276567459106445,
      "activations/layer23_attention_weight_max": 34.89107894897461,
      "activations/layer23_attention_weight_min": -25.937175750732422,
      "activations/layer2_attention_weight_max": 33.008323669433594,
      "activations/layer2_attention_weight_min": -32.76904296875,
      "activations/layer3_attention_weight_max": 96.89856719970703,
      "activations/layer3_attention_weight_min": -94.99870300292969,
      "activations/layer4_attention_weight_max": 117.16230773925781,
      "activations/layer4_attention_weight_min": -118.3197250366211,
      "activations/layer5_attention_weight_max": 53.99209213256836,
      "activations/layer5_attention_weight_min": -63.31793212890625,
      "activations/layer6_attention_weight_max": 46.53119659423828,
      "activations/layer6_attention_weight_min": -48.85478210449219,
      "activations/layer7_attention_weight_max": 92.48381042480469,
      "activations/layer7_attention_weight_min": -92.31378173828125,
      "activations/layer8_attention_weight_max": 43.09865188598633,
      "activations/layer8_attention_weight_min": -45.51182174682617,
      "activations/layer9_attention_weight_max": 38.15333557128906,
      "activations/layer9_attention_weight_min": -37.76204299926758,
      "epoch": 11.05,
      "learning_rate": 7.956893939393938e-05,
      "loss": 2.7788,
      "step": 190100
    },
    {
      "activations/layer0_attention_weight_max": 15.007259368896484,
      "activations/layer0_attention_weight_min": -11.833513259887695,
      "activations/layer10_attention_weight_max": 32.182674407958984,
      "activations/layer10_attention_weight_min": -32.85820388793945,
      "activations/layer11_attention_weight_max": 32.66776657104492,
      "activations/layer11_attention_weight_min": -32.033748626708984,
      "activations/layer12_attention_weight_max": 26.395509719848633,
      "activations/layer12_attention_weight_min": -30.93852996826172,
      "activations/layer13_attention_weight_max": 37.16777801513672,
      "activations/layer13_attention_weight_min": -33.778865814208984,
      "activations/layer14_attention_weight_max": 39.82554626464844,
      "activations/layer14_attention_weight_min": -39.99272155761719,
      "activations/layer15_attention_weight_max": 37.09123229980469,
      "activations/layer15_attention_weight_min": -31.19956398010254,
      "activations/layer16_attention_weight_max": 29.481027603149414,
      "activations/layer16_attention_weight_min": -27.458881378173828,
      "activations/layer17_attention_weight_max": 27.714277267456055,
      "activations/layer17_attention_weight_min": -25.53186798095703,
      "activations/layer18_attention_weight_max": 26.524206161499023,
      "activations/layer18_attention_weight_min": -21.824546813964844,
      "activations/layer19_attention_weight_max": 35.134681701660156,
      "activations/layer19_attention_weight_min": -29.297739028930664,
      "activations/layer1_attention_weight_max": 15.965835571289062,
      "activations/layer1_attention_weight_min": -15.08588695526123,
      "activations/layer20_attention_weight_max": 27.5910701751709,
      "activations/layer20_attention_weight_min": -24.113004684448242,
      "activations/layer21_attention_weight_max": 27.98192024230957,
      "activations/layer21_attention_weight_min": -23.473142623901367,
      "activations/layer22_attention_weight_max": 39.827720642089844,
      "activations/layer22_attention_weight_min": -28.480195999145508,
      "activations/layer23_attention_weight_max": 31.65162467956543,
      "activations/layer23_attention_weight_min": -24.618942260742188,
      "activations/layer2_attention_weight_max": 34.487430572509766,
      "activations/layer2_attention_weight_min": -33.07832336425781,
      "activations/layer3_attention_weight_max": 99.2829360961914,
      "activations/layer3_attention_weight_min": -95.5753402709961,
      "activations/layer4_attention_weight_max": 110.11433410644531,
      "activations/layer4_attention_weight_min": -112.52617645263672,
      "activations/layer5_attention_weight_max": 52.52008056640625,
      "activations/layer5_attention_weight_min": -61.6968994140625,
      "activations/layer6_attention_weight_max": 44.58454513549805,
      "activations/layer6_attention_weight_min": -46.52518844604492,
      "activations/layer7_attention_weight_max": 88.1432113647461,
      "activations/layer7_attention_weight_min": -87.0152359008789,
      "activations/layer8_attention_weight_max": 40.95525360107422,
      "activations/layer8_attention_weight_min": -39.315670013427734,
      "activations/layer9_attention_weight_max": 32.275794982910156,
      "activations/layer9_attention_weight_min": -32.537757873535156,
      "epoch": 11.05,
      "learning_rate": 7.955e-05,
      "loss": 2.7731,
      "step": 190150
    },
    {
      "activations/layer0_attention_weight_max": 15.760071754455566,
      "activations/layer0_attention_weight_min": -12.079429626464844,
      "activations/layer10_attention_weight_max": 31.492473602294922,
      "activations/layer10_attention_weight_min": -29.77620506286621,
      "activations/layer11_attention_weight_max": 32.89900588989258,
      "activations/layer11_attention_weight_min": -29.248191833496094,
      "activations/layer12_attention_weight_max": 25.234479904174805,
      "activations/layer12_attention_weight_min": -24.623369216918945,
      "activations/layer13_attention_weight_max": 39.08364486694336,
      "activations/layer13_attention_weight_min": -33.28790283203125,
      "activations/layer14_attention_weight_max": 46.5509147644043,
      "activations/layer14_attention_weight_min": -38.98440933227539,
      "activations/layer15_attention_weight_max": 37.651302337646484,
      "activations/layer15_attention_weight_min": -31.802722930908203,
      "activations/layer16_attention_weight_max": 31.301223754882812,
      "activations/layer16_attention_weight_min": -27.564861297607422,
      "activations/layer17_attention_weight_max": 33.62392807006836,
      "activations/layer17_attention_weight_min": -24.678184509277344,
      "activations/layer18_attention_weight_max": 32.27349853515625,
      "activations/layer18_attention_weight_min": -22.97530746459961,
      "activations/layer19_attention_weight_max": 33.79579162597656,
      "activations/layer19_attention_weight_min": -32.4595832824707,
      "activations/layer1_attention_weight_max": 16.307514190673828,
      "activations/layer1_attention_weight_min": -14.749826431274414,
      "activations/layer20_attention_weight_max": 28.993606567382812,
      "activations/layer20_attention_weight_min": -27.00187873840332,
      "activations/layer21_attention_weight_max": 35.06182098388672,
      "activations/layer21_attention_weight_min": -26.722124099731445,
      "activations/layer22_attention_weight_max": 42.32011413574219,
      "activations/layer22_attention_weight_min": -31.3901424407959,
      "activations/layer23_attention_weight_max": 35.479183197021484,
      "activations/layer23_attention_weight_min": -25.801122665405273,
      "activations/layer2_attention_weight_max": 33.07872009277344,
      "activations/layer2_attention_weight_min": -31.30410385131836,
      "activations/layer3_attention_weight_max": 90.9136962890625,
      "activations/layer3_attention_weight_min": -90.45079803466797,
      "activations/layer4_attention_weight_max": 110.13021087646484,
      "activations/layer4_attention_weight_min": -108.74234771728516,
      "activations/layer5_attention_weight_max": 53.887725830078125,
      "activations/layer5_attention_weight_min": -65.10132598876953,
      "activations/layer6_attention_weight_max": 42.23082733154297,
      "activations/layer6_attention_weight_min": -46.49620819091797,
      "activations/layer7_attention_weight_max": 83.12909698486328,
      "activations/layer7_attention_weight_min": -90.92947387695312,
      "activations/layer8_attention_weight_max": 40.54794692993164,
      "activations/layer8_attention_weight_min": -38.43962478637695,
      "activations/layer9_attention_weight_max": 30.673450469970703,
      "activations/layer9_attention_weight_min": -30.58469581604004,
      "epoch": 11.05,
      "learning_rate": 7.95310606060606e-05,
      "loss": 2.7715,
      "step": 190200
    },
    {
      "activations/layer0_attention_weight_max": 15.562575340270996,
      "activations/layer0_attention_weight_min": -13.461514472961426,
      "activations/layer10_attention_weight_max": 32.21810531616211,
      "activations/layer10_attention_weight_min": -30.48929214477539,
      "activations/layer11_attention_weight_max": 32.26203918457031,
      "activations/layer11_attention_weight_min": -30.894351959228516,
      "activations/layer12_attention_weight_max": 26.978281021118164,
      "activations/layer12_attention_weight_min": -24.940763473510742,
      "activations/layer13_attention_weight_max": 36.53460693359375,
      "activations/layer13_attention_weight_min": -32.499847412109375,
      "activations/layer14_attention_weight_max": 43.3581428527832,
      "activations/layer14_attention_weight_min": -37.30619430541992,
      "activations/layer15_attention_weight_max": 34.144901275634766,
      "activations/layer15_attention_weight_min": -31.248579025268555,
      "activations/layer16_attention_weight_max": 32.024383544921875,
      "activations/layer16_attention_weight_min": -27.054162979125977,
      "activations/layer17_attention_weight_max": 31.423580169677734,
      "activations/layer17_attention_weight_min": -25.55133056640625,
      "activations/layer18_attention_weight_max": 33.17752456665039,
      "activations/layer18_attention_weight_min": -22.258106231689453,
      "activations/layer19_attention_weight_max": 34.836368560791016,
      "activations/layer19_attention_weight_min": -29.670001983642578,
      "activations/layer1_attention_weight_max": 16.578847885131836,
      "activations/layer1_attention_weight_min": -15.734856605529785,
      "activations/layer20_attention_weight_max": 31.258459091186523,
      "activations/layer20_attention_weight_min": -24.232786178588867,
      "activations/layer21_attention_weight_max": 29.39256477355957,
      "activations/layer21_attention_weight_min": -22.979427337646484,
      "activations/layer22_attention_weight_max": 41.50059127807617,
      "activations/layer22_attention_weight_min": -28.835102081298828,
      "activations/layer23_attention_weight_max": 35.438289642333984,
      "activations/layer23_attention_weight_min": -23.708858489990234,
      "activations/layer2_attention_weight_max": 32.707618713378906,
      "activations/layer2_attention_weight_min": -32.44232177734375,
      "activations/layer3_attention_weight_max": 94.06951141357422,
      "activations/layer3_attention_weight_min": -94.07970428466797,
      "activations/layer4_attention_weight_max": 111.45375061035156,
      "activations/layer4_attention_weight_min": -112.00978088378906,
      "activations/layer5_attention_weight_max": 51.565589904785156,
      "activations/layer5_attention_weight_min": -61.4245719909668,
      "activations/layer6_attention_weight_max": 47.845741271972656,
      "activations/layer6_attention_weight_min": -47.84806823730469,
      "activations/layer7_attention_weight_max": 87.5782470703125,
      "activations/layer7_attention_weight_min": -92.7216796875,
      "activations/layer8_attention_weight_max": 42.625389099121094,
      "activations/layer8_attention_weight_min": -42.88745880126953,
      "activations/layer9_attention_weight_max": 32.83035659790039,
      "activations/layer9_attention_weight_min": -31.335012435913086,
      "epoch": 11.05,
      "learning_rate": 7.95121212121212e-05,
      "loss": 2.7827,
      "step": 190250
    },
    {
      "activations/layer0_attention_weight_max": 16.330934524536133,
      "activations/layer0_attention_weight_min": -11.964308738708496,
      "activations/layer10_attention_weight_max": 36.581939697265625,
      "activations/layer10_attention_weight_min": -34.914100646972656,
      "activations/layer11_attention_weight_max": 35.98223876953125,
      "activations/layer11_attention_weight_min": -33.68641662597656,
      "activations/layer12_attention_weight_max": 31.162940979003906,
      "activations/layer12_attention_weight_min": -26.02791976928711,
      "activations/layer13_attention_weight_max": 43.631160736083984,
      "activations/layer13_attention_weight_min": -35.58645248413086,
      "activations/layer14_attention_weight_max": 53.481319427490234,
      "activations/layer14_attention_weight_min": -37.930973052978516,
      "activations/layer15_attention_weight_max": 37.9191780090332,
      "activations/layer15_attention_weight_min": -32.207271575927734,
      "activations/layer16_attention_weight_max": 33.41820526123047,
      "activations/layer16_attention_weight_min": -27.823829650878906,
      "activations/layer17_attention_weight_max": 35.648319244384766,
      "activations/layer17_attention_weight_min": -25.901111602783203,
      "activations/layer18_attention_weight_max": 33.96228790283203,
      "activations/layer18_attention_weight_min": -22.52159881591797,
      "activations/layer19_attention_weight_max": 41.076534271240234,
      "activations/layer19_attention_weight_min": -33.0069580078125,
      "activations/layer1_attention_weight_max": 16.094493865966797,
      "activations/layer1_attention_weight_min": -14.668684005737305,
      "activations/layer20_attention_weight_max": 36.2197265625,
      "activations/layer20_attention_weight_min": -25.759897232055664,
      "activations/layer21_attention_weight_max": 36.2108268737793,
      "activations/layer21_attention_weight_min": -24.17759895324707,
      "activations/layer22_attention_weight_max": 56.66306686401367,
      "activations/layer22_attention_weight_min": -31.507598876953125,
      "activations/layer23_attention_weight_max": 40.55842971801758,
      "activations/layer23_attention_weight_min": -26.97152328491211,
      "activations/layer2_attention_weight_max": 31.360387802124023,
      "activations/layer2_attention_weight_min": -32.37956237792969,
      "activations/layer3_attention_weight_max": 91.16866302490234,
      "activations/layer3_attention_weight_min": -92.40852355957031,
      "activations/layer4_attention_weight_max": 116.20086669921875,
      "activations/layer4_attention_weight_min": -112.62715148925781,
      "activations/layer5_attention_weight_max": 52.192771911621094,
      "activations/layer5_attention_weight_min": -61.772911071777344,
      "activations/layer6_attention_weight_max": 44.06085968017578,
      "activations/layer6_attention_weight_min": -47.26963424682617,
      "activations/layer7_attention_weight_max": 93.78235626220703,
      "activations/layer7_attention_weight_min": -88.78703308105469,
      "activations/layer8_attention_weight_max": 44.14692687988281,
      "activations/layer8_attention_weight_min": -42.328125,
      "activations/layer9_attention_weight_max": 36.210540771484375,
      "activations/layer9_attention_weight_min": -35.491512298583984,
      "epoch": 11.06,
      "learning_rate": 7.949318181818181e-05,
      "loss": 2.7832,
      "step": 190300
    },
    {
      "activations/layer0_attention_weight_max": 15.104876518249512,
      "activations/layer0_attention_weight_min": -11.895618438720703,
      "activations/layer10_attention_weight_max": 35.61993408203125,
      "activations/layer10_attention_weight_min": -32.63795852661133,
      "activations/layer11_attention_weight_max": 35.68641662597656,
      "activations/layer11_attention_weight_min": -31.488182067871094,
      "activations/layer12_attention_weight_max": 26.666601181030273,
      "activations/layer12_attention_weight_min": -35.19847106933594,
      "activations/layer13_attention_weight_max": 41.29226303100586,
      "activations/layer13_attention_weight_min": -33.90213394165039,
      "activations/layer14_attention_weight_max": 47.923309326171875,
      "activations/layer14_attention_weight_min": -36.31856918334961,
      "activations/layer15_attention_weight_max": 40.269161224365234,
      "activations/layer15_attention_weight_min": -32.356014251708984,
      "activations/layer16_attention_weight_max": 34.78883743286133,
      "activations/layer16_attention_weight_min": -27.145538330078125,
      "activations/layer17_attention_weight_max": 35.69919204711914,
      "activations/layer17_attention_weight_min": -23.51866340637207,
      "activations/layer18_attention_weight_max": 34.47175216674805,
      "activations/layer18_attention_weight_min": -21.998348236083984,
      "activations/layer19_attention_weight_max": 37.14806365966797,
      "activations/layer19_attention_weight_min": -30.650558471679688,
      "activations/layer1_attention_weight_max": 16.343345642089844,
      "activations/layer1_attention_weight_min": -14.762028694152832,
      "activations/layer20_attention_weight_max": 31.910999298095703,
      "activations/layer20_attention_weight_min": -26.25490951538086,
      "activations/layer21_attention_weight_max": 32.9130973815918,
      "activations/layer21_attention_weight_min": -24.09708023071289,
      "activations/layer22_attention_weight_max": 48.02793884277344,
      "activations/layer22_attention_weight_min": -29.54314422607422,
      "activations/layer23_attention_weight_max": 39.341129302978516,
      "activations/layer23_attention_weight_min": -24.422161102294922,
      "activations/layer2_attention_weight_max": 32.991966247558594,
      "activations/layer2_attention_weight_min": -35.760948181152344,
      "activations/layer3_attention_weight_max": 103.8692626953125,
      "activations/layer3_attention_weight_min": -107.00495147705078,
      "activations/layer4_attention_weight_max": 116.6559829711914,
      "activations/layer4_attention_weight_min": -110.14395904541016,
      "activations/layer5_attention_weight_max": 54.76670455932617,
      "activations/layer5_attention_weight_min": -60.0827751159668,
      "activations/layer6_attention_weight_max": 44.95670700073242,
      "activations/layer6_attention_weight_min": -46.51313781738281,
      "activations/layer7_attention_weight_max": 97.6154556274414,
      "activations/layer7_attention_weight_min": -90.10440826416016,
      "activations/layer8_attention_weight_max": 45.046974182128906,
      "activations/layer8_attention_weight_min": -39.61360168457031,
      "activations/layer9_attention_weight_max": 32.14564895629883,
      "activations/layer9_attention_weight_min": -37.12982940673828,
      "epoch": 11.06,
      "learning_rate": 7.947424242424243e-05,
      "loss": 2.7759,
      "step": 190350
    },
    {
      "activations/layer0_attention_weight_max": 15.74449348449707,
      "activations/layer0_attention_weight_min": -13.28635025024414,
      "activations/layer10_attention_weight_max": 32.50364303588867,
      "activations/layer10_attention_weight_min": -31.100353240966797,
      "activations/layer11_attention_weight_max": 29.524017333984375,
      "activations/layer11_attention_weight_min": -29.86599349975586,
      "activations/layer12_attention_weight_max": 21.365314483642578,
      "activations/layer12_attention_weight_min": -23.882959365844727,
      "activations/layer13_attention_weight_max": 34.35561752319336,
      "activations/layer13_attention_weight_min": -31.602951049804688,
      "activations/layer14_attention_weight_max": 41.00630569458008,
      "activations/layer14_attention_weight_min": -34.46775436401367,
      "activations/layer15_attention_weight_max": 33.109127044677734,
      "activations/layer15_attention_weight_min": -30.166364669799805,
      "activations/layer16_attention_weight_max": 30.663700103759766,
      "activations/layer16_attention_weight_min": -28.29094123840332,
      "activations/layer17_attention_weight_max": 29.21029281616211,
      "activations/layer17_attention_weight_min": -25.23646354675293,
      "activations/layer18_attention_weight_max": 32.90409469604492,
      "activations/layer18_attention_weight_min": -22.635120391845703,
      "activations/layer19_attention_weight_max": 40.424434661865234,
      "activations/layer19_attention_weight_min": -30.24322509765625,
      "activations/layer1_attention_weight_max": 16.601173400878906,
      "activations/layer1_attention_weight_min": -15.807746887207031,
      "activations/layer20_attention_weight_max": 29.096405029296875,
      "activations/layer20_attention_weight_min": -25.014596939086914,
      "activations/layer21_attention_weight_max": 34.19432830810547,
      "activations/layer21_attention_weight_min": -23.046701431274414,
      "activations/layer22_attention_weight_max": 45.177650451660156,
      "activations/layer22_attention_weight_min": -29.224416732788086,
      "activations/layer23_attention_weight_max": 36.492767333984375,
      "activations/layer23_attention_weight_min": -26.00116729736328,
      "activations/layer2_attention_weight_max": 34.15729904174805,
      "activations/layer2_attention_weight_min": -33.8405647277832,
      "activations/layer3_attention_weight_max": 99.67679595947266,
      "activations/layer3_attention_weight_min": -96.68820190429688,
      "activations/layer4_attention_weight_max": 115.8273696899414,
      "activations/layer4_attention_weight_min": -112.0163803100586,
      "activations/layer5_attention_weight_max": 54.140316009521484,
      "activations/layer5_attention_weight_min": -60.35223388671875,
      "activations/layer6_attention_weight_max": 46.09572982788086,
      "activations/layer6_attention_weight_min": -47.669395446777344,
      "activations/layer7_attention_weight_max": 91.147705078125,
      "activations/layer7_attention_weight_min": -83.4037857055664,
      "activations/layer8_attention_weight_max": 38.555572509765625,
      "activations/layer8_attention_weight_min": -39.27027130126953,
      "activations/layer9_attention_weight_max": 34.3089485168457,
      "activations/layer9_attention_weight_min": -30.25315284729004,
      "epoch": 11.06,
      "learning_rate": 7.945530303030302e-05,
      "loss": 2.7829,
      "step": 190400
    },
    {
      "activations/layer0_attention_weight_max": 16.134279251098633,
      "activations/layer0_attention_weight_min": -12.934521675109863,
      "activations/layer10_attention_weight_max": 32.55682373046875,
      "activations/layer10_attention_weight_min": -31.856359481811523,
      "activations/layer11_attention_weight_max": 34.76799774169922,
      "activations/layer11_attention_weight_min": -30.443891525268555,
      "activations/layer12_attention_weight_max": 25.357812881469727,
      "activations/layer12_attention_weight_min": -24.464170455932617,
      "activations/layer13_attention_weight_max": 34.59006881713867,
      "activations/layer13_attention_weight_min": -32.724822998046875,
      "activations/layer14_attention_weight_max": 40.60851287841797,
      "activations/layer14_attention_weight_min": -35.93980407714844,
      "activations/layer15_attention_weight_max": 35.37051010131836,
      "activations/layer15_attention_weight_min": -31.2385311126709,
      "activations/layer16_attention_weight_max": 30.723493576049805,
      "activations/layer16_attention_weight_min": -26.9609432220459,
      "activations/layer17_attention_weight_max": 34.251468658447266,
      "activations/layer17_attention_weight_min": -27.31148338317871,
      "activations/layer18_attention_weight_max": 27.188936233520508,
      "activations/layer18_attention_weight_min": -22.764862060546875,
      "activations/layer19_attention_weight_max": 33.86930465698242,
      "activations/layer19_attention_weight_min": -32.64722442626953,
      "activations/layer1_attention_weight_max": 17.061054229736328,
      "activations/layer1_attention_weight_min": -18.722904205322266,
      "activations/layer20_attention_weight_max": 31.755762100219727,
      "activations/layer20_attention_weight_min": -25.499887466430664,
      "activations/layer21_attention_weight_max": 30.581674575805664,
      "activations/layer21_attention_weight_min": -23.619140625,
      "activations/layer22_attention_weight_max": 42.035587310791016,
      "activations/layer22_attention_weight_min": -29.77298355102539,
      "activations/layer23_attention_weight_max": 32.60728454589844,
      "activations/layer23_attention_weight_min": -23.488597869873047,
      "activations/layer2_attention_weight_max": 32.385215759277344,
      "activations/layer2_attention_weight_min": -32.699214935302734,
      "activations/layer3_attention_weight_max": 91.72770690917969,
      "activations/layer3_attention_weight_min": -93.29408264160156,
      "activations/layer4_attention_weight_max": 111.02567291259766,
      "activations/layer4_attention_weight_min": -110.4681167602539,
      "activations/layer5_attention_weight_max": 51.13538360595703,
      "activations/layer5_attention_weight_min": -60.26078796386719,
      "activations/layer6_attention_weight_max": 43.878047943115234,
      "activations/layer6_attention_weight_min": -49.39646530151367,
      "activations/layer7_attention_weight_max": 84.58875274658203,
      "activations/layer7_attention_weight_min": -90.50836944580078,
      "activations/layer8_attention_weight_max": 39.60188674926758,
      "activations/layer8_attention_weight_min": -39.40289306640625,
      "activations/layer9_attention_weight_max": 33.22454071044922,
      "activations/layer9_attention_weight_min": -32.3082160949707,
      "epoch": 11.07,
      "learning_rate": 7.943636363636363e-05,
      "loss": 2.7798,
      "step": 190450
    },
    {
      "activations/layer0_attention_weight_max": 15.384296417236328,
      "activations/layer0_attention_weight_min": -11.602907180786133,
      "activations/layer10_attention_weight_max": 44.13337326049805,
      "activations/layer10_attention_weight_min": -42.87221145629883,
      "activations/layer11_attention_weight_max": 42.0513916015625,
      "activations/layer11_attention_weight_min": -43.490753173828125,
      "activations/layer12_attention_weight_max": 29.291309356689453,
      "activations/layer12_attention_weight_min": -29.7790470123291,
      "activations/layer13_attention_weight_max": 43.90651321411133,
      "activations/layer13_attention_weight_min": -43.50405502319336,
      "activations/layer14_attention_weight_max": 51.272186279296875,
      "activations/layer14_attention_weight_min": -49.53773880004883,
      "activations/layer15_attention_weight_max": 42.325767517089844,
      "activations/layer15_attention_weight_min": -40.003665924072266,
      "activations/layer16_attention_weight_max": 32.71254348754883,
      "activations/layer16_attention_weight_min": -29.730417251586914,
      "activations/layer17_attention_weight_max": 29.34213638305664,
      "activations/layer17_attention_weight_min": -28.88212013244629,
      "activations/layer18_attention_weight_max": 30.830778121948242,
      "activations/layer18_attention_weight_min": -25.97815704345703,
      "activations/layer19_attention_weight_max": 36.576072692871094,
      "activations/layer19_attention_weight_min": -33.13810348510742,
      "activations/layer1_attention_weight_max": 17.363780975341797,
      "activations/layer1_attention_weight_min": -14.071035385131836,
      "activations/layer20_attention_weight_max": 29.93346405029297,
      "activations/layer20_attention_weight_min": -25.304868698120117,
      "activations/layer21_attention_weight_max": 27.39638900756836,
      "activations/layer21_attention_weight_min": -25.39037322998047,
      "activations/layer22_attention_weight_max": 46.74153137207031,
      "activations/layer22_attention_weight_min": -36.84927749633789,
      "activations/layer23_attention_weight_max": 33.66962432861328,
      "activations/layer23_attention_weight_min": -27.393741607666016,
      "activations/layer2_attention_weight_max": 38.725669860839844,
      "activations/layer2_attention_weight_min": -38.2290153503418,
      "activations/layer3_attention_weight_max": 113.95706176757812,
      "activations/layer3_attention_weight_min": -115.94505310058594,
      "activations/layer4_attention_weight_max": 137.0784149169922,
      "activations/layer4_attention_weight_min": -131.2483367919922,
      "activations/layer5_attention_weight_max": 55.89142990112305,
      "activations/layer5_attention_weight_min": -61.12437057495117,
      "activations/layer6_attention_weight_max": 45.2036247253418,
      "activations/layer6_attention_weight_min": -47.134700775146484,
      "activations/layer7_attention_weight_max": 101.41082763671875,
      "activations/layer7_attention_weight_min": -101.62216186523438,
      "activations/layer8_attention_weight_max": 47.577362060546875,
      "activations/layer8_attention_weight_min": -46.40258026123047,
      "activations/layer9_attention_weight_max": 41.62702178955078,
      "activations/layer9_attention_weight_min": -42.09850311279297,
      "epoch": 11.07,
      "learning_rate": 7.941742424242425e-05,
      "loss": 2.7832,
      "step": 190500
    },
    {
      "activations/layer0_attention_weight_max": 15.626985549926758,
      "activations/layer0_attention_weight_min": -12.378514289855957,
      "activations/layer10_attention_weight_max": 31.371868133544922,
      "activations/layer10_attention_weight_min": -31.84941864013672,
      "activations/layer11_attention_weight_max": 32.71717834472656,
      "activations/layer11_attention_weight_min": -31.921878814697266,
      "activations/layer12_attention_weight_max": 24.213531494140625,
      "activations/layer12_attention_weight_min": -29.644939422607422,
      "activations/layer13_attention_weight_max": 39.241783142089844,
      "activations/layer13_attention_weight_min": -34.247982025146484,
      "activations/layer14_attention_weight_max": 42.210853576660156,
      "activations/layer14_attention_weight_min": -37.260597229003906,
      "activations/layer15_attention_weight_max": 37.78764343261719,
      "activations/layer15_attention_weight_min": -32.6389045715332,
      "activations/layer16_attention_weight_max": 31.46636199951172,
      "activations/layer16_attention_weight_min": -29.697574615478516,
      "activations/layer17_attention_weight_max": 35.01460647583008,
      "activations/layer17_attention_weight_min": -26.774869918823242,
      "activations/layer18_attention_weight_max": 33.93291473388672,
      "activations/layer18_attention_weight_min": -23.321998596191406,
      "activations/layer19_attention_weight_max": 36.88589859008789,
      "activations/layer19_attention_weight_min": -31.51459312438965,
      "activations/layer1_attention_weight_max": 16.298351287841797,
      "activations/layer1_attention_weight_min": -14.600711822509766,
      "activations/layer20_attention_weight_max": 30.412635803222656,
      "activations/layer20_attention_weight_min": -23.71995735168457,
      "activations/layer21_attention_weight_max": 30.6627254486084,
      "activations/layer21_attention_weight_min": -24.752832412719727,
      "activations/layer22_attention_weight_max": 45.8449821472168,
      "activations/layer22_attention_weight_min": -28.29081916809082,
      "activations/layer23_attention_weight_max": 34.471797943115234,
      "activations/layer23_attention_weight_min": -23.18582534790039,
      "activations/layer2_attention_weight_max": 32.19655990600586,
      "activations/layer2_attention_weight_min": -33.67980194091797,
      "activations/layer3_attention_weight_max": 95.71682739257812,
      "activations/layer3_attention_weight_min": -98.66876983642578,
      "activations/layer4_attention_weight_max": 115.80086517333984,
      "activations/layer4_attention_weight_min": -111.7180404663086,
      "activations/layer5_attention_weight_max": 55.381629943847656,
      "activations/layer5_attention_weight_min": -63.504722595214844,
      "activations/layer6_attention_weight_max": 44.58763122558594,
      "activations/layer6_attention_weight_min": -47.55612564086914,
      "activations/layer7_attention_weight_max": 89.61193084716797,
      "activations/layer7_attention_weight_min": -90.07433319091797,
      "activations/layer8_attention_weight_max": 39.34633255004883,
      "activations/layer8_attention_weight_min": -43.70091247558594,
      "activations/layer9_attention_weight_max": 32.342132568359375,
      "activations/layer9_attention_weight_min": -32.75385284423828,
      "epoch": 11.07,
      "learning_rate": 7.939848484848483e-05,
      "loss": 2.7751,
      "step": 190550
    },
    {
      "activations/layer0_attention_weight_max": 15.665181159973145,
      "activations/layer0_attention_weight_min": -13.42468547821045,
      "activations/layer10_attention_weight_max": 32.3757438659668,
      "activations/layer10_attention_weight_min": -32.785369873046875,
      "activations/layer11_attention_weight_max": 31.823955535888672,
      "activations/layer11_attention_weight_min": -31.28278160095215,
      "activations/layer12_attention_weight_max": 26.449499130249023,
      "activations/layer12_attention_weight_min": -25.516965866088867,
      "activations/layer13_attention_weight_max": 37.063392639160156,
      "activations/layer13_attention_weight_min": -33.403038024902344,
      "activations/layer14_attention_weight_max": 46.08027648925781,
      "activations/layer14_attention_weight_min": -37.67118835449219,
      "activations/layer15_attention_weight_max": 36.644798278808594,
      "activations/layer15_attention_weight_min": -30.86196517944336,
      "activations/layer16_attention_weight_max": 34.80891799926758,
      "activations/layer16_attention_weight_min": -27.652864456176758,
      "activations/layer17_attention_weight_max": 31.0877628326416,
      "activations/layer17_attention_weight_min": -25.97341537475586,
      "activations/layer18_attention_weight_max": 30.06069564819336,
      "activations/layer18_attention_weight_min": -25.30727195739746,
      "activations/layer19_attention_weight_max": 34.96711349487305,
      "activations/layer19_attention_weight_min": -31.15036392211914,
      "activations/layer1_attention_weight_max": 16.291641235351562,
      "activations/layer1_attention_weight_min": -14.686295509338379,
      "activations/layer20_attention_weight_max": 30.373672485351562,
      "activations/layer20_attention_weight_min": -24.018198013305664,
      "activations/layer21_attention_weight_max": 28.05739402770996,
      "activations/layer21_attention_weight_min": -23.25051498413086,
      "activations/layer22_attention_weight_max": 44.852291107177734,
      "activations/layer22_attention_weight_min": -29.889219284057617,
      "activations/layer23_attention_weight_max": 33.23109817504883,
      "activations/layer23_attention_weight_min": -27.293243408203125,
      "activations/layer2_attention_weight_max": 34.07276916503906,
      "activations/layer2_attention_weight_min": -33.46418762207031,
      "activations/layer3_attention_weight_max": 96.24003601074219,
      "activations/layer3_attention_weight_min": -97.72018432617188,
      "activations/layer4_attention_weight_max": 117.03792572021484,
      "activations/layer4_attention_weight_min": -112.01080322265625,
      "activations/layer5_attention_weight_max": 52.99237060546875,
      "activations/layer5_attention_weight_min": -61.57421875,
      "activations/layer6_attention_weight_max": 44.911373138427734,
      "activations/layer6_attention_weight_min": -49.63621520996094,
      "activations/layer7_attention_weight_max": 89.83065032958984,
      "activations/layer7_attention_weight_min": -89.86419677734375,
      "activations/layer8_attention_weight_max": 38.50685501098633,
      "activations/layer8_attention_weight_min": -40.81448745727539,
      "activations/layer9_attention_weight_max": 32.33545684814453,
      "activations/layer9_attention_weight_min": -32.776573181152344,
      "epoch": 11.07,
      "learning_rate": 7.937954545454545e-05,
      "loss": 2.771,
      "step": 190600
    },
    {
      "activations/layer0_attention_weight_max": 15.29462718963623,
      "activations/layer0_attention_weight_min": -12.261473655700684,
      "activations/layer10_attention_weight_max": 38.25388717651367,
      "activations/layer10_attention_weight_min": -34.245853424072266,
      "activations/layer11_attention_weight_max": 37.16805648803711,
      "activations/layer11_attention_weight_min": -32.71089553833008,
      "activations/layer12_attention_weight_max": 35.89612579345703,
      "activations/layer12_attention_weight_min": -27.225549697875977,
      "activations/layer13_attention_weight_max": 44.82072067260742,
      "activations/layer13_attention_weight_min": -34.559139251708984,
      "activations/layer14_attention_weight_max": 53.37290573120117,
      "activations/layer14_attention_weight_min": -39.89439010620117,
      "activations/layer15_attention_weight_max": 41.33782196044922,
      "activations/layer15_attention_weight_min": -32.89167404174805,
      "activations/layer16_attention_weight_max": 35.248252868652344,
      "activations/layer16_attention_weight_min": -28.948719024658203,
      "activations/layer17_attention_weight_max": 34.98382568359375,
      "activations/layer17_attention_weight_min": -27.269563674926758,
      "activations/layer18_attention_weight_max": 31.579565048217773,
      "activations/layer18_attention_weight_min": -23.18608283996582,
      "activations/layer19_attention_weight_max": 39.46763229370117,
      "activations/layer19_attention_weight_min": -36.086219787597656,
      "activations/layer1_attention_weight_max": 16.03032112121582,
      "activations/layer1_attention_weight_min": -13.870613098144531,
      "activations/layer20_attention_weight_max": 33.35707473754883,
      "activations/layer20_attention_weight_min": -27.185142517089844,
      "activations/layer21_attention_weight_max": 31.241287231445312,
      "activations/layer21_attention_weight_min": -25.250261306762695,
      "activations/layer22_attention_weight_max": 58.73820877075195,
      "activations/layer22_attention_weight_min": -29.834957122802734,
      "activations/layer23_attention_weight_max": 40.121368408203125,
      "activations/layer23_attention_weight_min": -24.27379608154297,
      "activations/layer2_attention_weight_max": 33.880836486816406,
      "activations/layer2_attention_weight_min": -33.53819274902344,
      "activations/layer3_attention_weight_max": 95.87913513183594,
      "activations/layer3_attention_weight_min": -97.7210464477539,
      "activations/layer4_attention_weight_max": 114.57325744628906,
      "activations/layer4_attention_weight_min": -116.4613265991211,
      "activations/layer5_attention_weight_max": 52.50178146362305,
      "activations/layer5_attention_weight_min": -58.607887268066406,
      "activations/layer6_attention_weight_max": 49.10494613647461,
      "activations/layer6_attention_weight_min": -51.35084533691406,
      "activations/layer7_attention_weight_max": 99.93748474121094,
      "activations/layer7_attention_weight_min": -101.27953338623047,
      "activations/layer8_attention_weight_max": 48.00050354003906,
      "activations/layer8_attention_weight_min": -45.4251594543457,
      "activations/layer9_attention_weight_max": 38.08441925048828,
      "activations/layer9_attention_weight_min": -35.67385482788086,
      "epoch": 11.08,
      "learning_rate": 7.936060606060605e-05,
      "loss": 2.7861,
      "step": 190650
    },
    {
      "activations/layer0_attention_weight_max": 15.754778861999512,
      "activations/layer0_attention_weight_min": -13.214781761169434,
      "activations/layer10_attention_weight_max": 32.889923095703125,
      "activations/layer10_attention_weight_min": -32.30625534057617,
      "activations/layer11_attention_weight_max": 32.36660385131836,
      "activations/layer11_attention_weight_min": -31.976734161376953,
      "activations/layer12_attention_weight_max": 25.857559204101562,
      "activations/layer12_attention_weight_min": -26.36960792541504,
      "activations/layer13_attention_weight_max": 38.902523040771484,
      "activations/layer13_attention_weight_min": -34.99713134765625,
      "activations/layer14_attention_weight_max": 43.162174224853516,
      "activations/layer14_attention_weight_min": -39.465599060058594,
      "activations/layer15_attention_weight_max": 36.964385986328125,
      "activations/layer15_attention_weight_min": -33.52366638183594,
      "activations/layer16_attention_weight_max": 30.328365325927734,
      "activations/layer16_attention_weight_min": -27.691566467285156,
      "activations/layer17_attention_weight_max": 31.495161056518555,
      "activations/layer17_attention_weight_min": -27.313148498535156,
      "activations/layer18_attention_weight_max": 32.78982925415039,
      "activations/layer18_attention_weight_min": -25.3116512298584,
      "activations/layer19_attention_weight_max": 38.4017333984375,
      "activations/layer19_attention_weight_min": -33.34748458862305,
      "activations/layer1_attention_weight_max": 17.00530242919922,
      "activations/layer1_attention_weight_min": -14.419781684875488,
      "activations/layer20_attention_weight_max": 30.959054946899414,
      "activations/layer20_attention_weight_min": -26.423572540283203,
      "activations/layer21_attention_weight_max": 31.218185424804688,
      "activations/layer21_attention_weight_min": -24.815885543823242,
      "activations/layer22_attention_weight_max": 51.79998016357422,
      "activations/layer22_attention_weight_min": -32.580291748046875,
      "activations/layer23_attention_weight_max": 37.52836608886719,
      "activations/layer23_attention_weight_min": -24.839942932128906,
      "activations/layer2_attention_weight_max": 33.80189514160156,
      "activations/layer2_attention_weight_min": -34.46595764160156,
      "activations/layer3_attention_weight_max": 97.7643051147461,
      "activations/layer3_attention_weight_min": -101.15005493164062,
      "activations/layer4_attention_weight_max": 118.04471588134766,
      "activations/layer4_attention_weight_min": -114.43795013427734,
      "activations/layer5_attention_weight_max": 53.421958923339844,
      "activations/layer5_attention_weight_min": -66.08235931396484,
      "activations/layer6_attention_weight_max": 44.405784606933594,
      "activations/layer6_attention_weight_min": -49.98310470581055,
      "activations/layer7_attention_weight_max": 90.49321746826172,
      "activations/layer7_attention_weight_min": -91.22286987304688,
      "activations/layer8_attention_weight_max": 42.541481018066406,
      "activations/layer8_attention_weight_min": -40.25541305541992,
      "activations/layer9_attention_weight_max": 31.185474395751953,
      "activations/layer9_attention_weight_min": -32.83614730834961,
      "epoch": 11.08,
      "learning_rate": 7.934166666666665e-05,
      "loss": 2.7613,
      "step": 190700
    },
    {
      "activations/layer0_attention_weight_max": 15.159571647644043,
      "activations/layer0_attention_weight_min": -12.334258079528809,
      "activations/layer10_attention_weight_max": 33.75484085083008,
      "activations/layer10_attention_weight_min": -32.42367935180664,
      "activations/layer11_attention_weight_max": 32.63127517700195,
      "activations/layer11_attention_weight_min": -33.35786056518555,
      "activations/layer12_attention_weight_max": 25.262651443481445,
      "activations/layer12_attention_weight_min": -24.5664119720459,
      "activations/layer13_attention_weight_max": 39.603248596191406,
      "activations/layer13_attention_weight_min": -33.53343200683594,
      "activations/layer14_attention_weight_max": 47.49795913696289,
      "activations/layer14_attention_weight_min": -36.1949577331543,
      "activations/layer15_attention_weight_max": 35.9837646484375,
      "activations/layer15_attention_weight_min": -31.428991317749023,
      "activations/layer16_attention_weight_max": 33.804969787597656,
      "activations/layer16_attention_weight_min": -28.549856185913086,
      "activations/layer17_attention_weight_max": 35.002952575683594,
      "activations/layer17_attention_weight_min": -25.368322372436523,
      "activations/layer18_attention_weight_max": 34.469947814941406,
      "activations/layer18_attention_weight_min": -22.09221649169922,
      "activations/layer19_attention_weight_max": 37.185237884521484,
      "activations/layer19_attention_weight_min": -30.85837173461914,
      "activations/layer1_attention_weight_max": 16.329286575317383,
      "activations/layer1_attention_weight_min": -13.952468872070312,
      "activations/layer20_attention_weight_max": 31.99346923828125,
      "activations/layer20_attention_weight_min": -25.877317428588867,
      "activations/layer21_attention_weight_max": 32.720821380615234,
      "activations/layer21_attention_weight_min": -23.311391830444336,
      "activations/layer22_attention_weight_max": 48.51727294921875,
      "activations/layer22_attention_weight_min": -27.468198776245117,
      "activations/layer23_attention_weight_max": 37.899269104003906,
      "activations/layer23_attention_weight_min": -22.406978607177734,
      "activations/layer2_attention_weight_max": 32.17424392700195,
      "activations/layer2_attention_weight_min": -35.12994384765625,
      "activations/layer3_attention_weight_max": 94.30631256103516,
      "activations/layer3_attention_weight_min": -98.1266098022461,
      "activations/layer4_attention_weight_max": 111.60528564453125,
      "activations/layer4_attention_weight_min": -113.89957427978516,
      "activations/layer5_attention_weight_max": 52.906150817871094,
      "activations/layer5_attention_weight_min": -61.34503936767578,
      "activations/layer6_attention_weight_max": 43.00559997558594,
      "activations/layer6_attention_weight_min": -48.120731353759766,
      "activations/layer7_attention_weight_max": 84.90727996826172,
      "activations/layer7_attention_weight_min": -87.50708770751953,
      "activations/layer8_attention_weight_max": 37.85080337524414,
      "activations/layer8_attention_weight_min": -39.83789825439453,
      "activations/layer9_attention_weight_max": 32.805538177490234,
      "activations/layer9_attention_weight_min": -32.9828987121582,
      "epoch": 11.08,
      "learning_rate": 7.932272727272727e-05,
      "loss": 2.7857,
      "step": 190750
    },
    {
      "activations/layer0_attention_weight_max": 14.890335083007812,
      "activations/layer0_attention_weight_min": -12.894439697265625,
      "activations/layer10_attention_weight_max": 31.619400024414062,
      "activations/layer10_attention_weight_min": -32.096168518066406,
      "activations/layer11_attention_weight_max": 34.28997039794922,
      "activations/layer11_attention_weight_min": -31.049480438232422,
      "activations/layer12_attention_weight_max": 46.55696105957031,
      "activations/layer12_attention_weight_min": -26.60210418701172,
      "activations/layer13_attention_weight_max": 49.81019973754883,
      "activations/layer13_attention_weight_min": -34.22864532470703,
      "activations/layer14_attention_weight_max": 52.25632095336914,
      "activations/layer14_attention_weight_min": -39.52336883544922,
      "activations/layer15_attention_weight_max": 42.95061111450195,
      "activations/layer15_attention_weight_min": -34.1038818359375,
      "activations/layer16_attention_weight_max": 36.56924819946289,
      "activations/layer16_attention_weight_min": -29.549665451049805,
      "activations/layer17_attention_weight_max": 35.1430549621582,
      "activations/layer17_attention_weight_min": -25.669973373413086,
      "activations/layer18_attention_weight_max": 31.970144271850586,
      "activations/layer18_attention_weight_min": -23.714941024780273,
      "activations/layer19_attention_weight_max": 43.85578155517578,
      "activations/layer19_attention_weight_min": -31.330747604370117,
      "activations/layer1_attention_weight_max": 18.271333694458008,
      "activations/layer1_attention_weight_min": -14.214397430419922,
      "activations/layer20_attention_weight_max": 35.336326599121094,
      "activations/layer20_attention_weight_min": -25.450639724731445,
      "activations/layer21_attention_weight_max": 33.12890625,
      "activations/layer21_attention_weight_min": -24.5349178314209,
      "activations/layer22_attention_weight_max": 55.75814437866211,
      "activations/layer22_attention_weight_min": -30.08688735961914,
      "activations/layer23_attention_weight_max": 43.7818603515625,
      "activations/layer23_attention_weight_min": -24.88081169128418,
      "activations/layer2_attention_weight_max": 33.270660400390625,
      "activations/layer2_attention_weight_min": -32.62394714355469,
      "activations/layer3_attention_weight_max": 97.16210174560547,
      "activations/layer3_attention_weight_min": -94.91680908203125,
      "activations/layer4_attention_weight_max": 112.57622528076172,
      "activations/layer4_attention_weight_min": -108.69591522216797,
      "activations/layer5_attention_weight_max": 51.56890106201172,
      "activations/layer5_attention_weight_min": -60.96788024902344,
      "activations/layer6_attention_weight_max": 42.8322639465332,
      "activations/layer6_attention_weight_min": -45.498252868652344,
      "activations/layer7_attention_weight_max": 94.4400863647461,
      "activations/layer7_attention_weight_min": -85.41090393066406,
      "activations/layer8_attention_weight_max": 38.90800476074219,
      "activations/layer8_attention_weight_min": -40.338260650634766,
      "activations/layer9_attention_weight_max": 31.497617721557617,
      "activations/layer9_attention_weight_min": -32.71595764160156,
      "epoch": 11.09,
      "learning_rate": 7.930378787878787e-05,
      "loss": 2.7785,
      "step": 190800
    },
    {
      "activations/layer0_attention_weight_max": 15.704300880432129,
      "activations/layer0_attention_weight_min": -12.544931411743164,
      "activations/layer10_attention_weight_max": 39.994346618652344,
      "activations/layer10_attention_weight_min": -35.382240295410156,
      "activations/layer11_attention_weight_max": 39.90324783325195,
      "activations/layer11_attention_weight_min": -35.52241897583008,
      "activations/layer12_attention_weight_max": 28.301380157470703,
      "activations/layer12_attention_weight_min": -29.930376052856445,
      "activations/layer13_attention_weight_max": 43.61936569213867,
      "activations/layer13_attention_weight_min": -36.19687271118164,
      "activations/layer14_attention_weight_max": 57.9242057800293,
      "activations/layer14_attention_weight_min": -44.32330322265625,
      "activations/layer15_attention_weight_max": 41.55968475341797,
      "activations/layer15_attention_weight_min": -32.2730598449707,
      "activations/layer16_attention_weight_max": 40.991119384765625,
      "activations/layer16_attention_weight_min": -28.796615600585938,
      "activations/layer17_attention_weight_max": 37.29022216796875,
      "activations/layer17_attention_weight_min": -27.099891662597656,
      "activations/layer18_attention_weight_max": 31.937223434448242,
      "activations/layer18_attention_weight_min": -25.46409797668457,
      "activations/layer19_attention_weight_max": 40.02323532104492,
      "activations/layer19_attention_weight_min": -30.88795280456543,
      "activations/layer1_attention_weight_max": 17.03102684020996,
      "activations/layer1_attention_weight_min": -15.505290031433105,
      "activations/layer20_attention_weight_max": 33.94479751586914,
      "activations/layer20_attention_weight_min": -23.924827575683594,
      "activations/layer21_attention_weight_max": 30.60504150390625,
      "activations/layer21_attention_weight_min": -23.89084815979004,
      "activations/layer22_attention_weight_max": 62.54880905151367,
      "activations/layer22_attention_weight_min": -29.947080612182617,
      "activations/layer23_attention_weight_max": 32.49858856201172,
      "activations/layer23_attention_weight_min": -25.824504852294922,
      "activations/layer2_attention_weight_max": 38.814552307128906,
      "activations/layer2_attention_weight_min": -36.513370513916016,
      "activations/layer3_attention_weight_max": 105.07784271240234,
      "activations/layer3_attention_weight_min": -105.85684204101562,
      "activations/layer4_attention_weight_max": 123.6000747680664,
      "activations/layer4_attention_weight_min": -118.88349914550781,
      "activations/layer5_attention_weight_max": 55.60830307006836,
      "activations/layer5_attention_weight_min": -62.409969329833984,
      "activations/layer6_attention_weight_max": 48.86072540283203,
      "activations/layer6_attention_weight_min": -49.42765808105469,
      "activations/layer7_attention_weight_max": 109.9454116821289,
      "activations/layer7_attention_weight_min": -98.83344268798828,
      "activations/layer8_attention_weight_max": 49.67693328857422,
      "activations/layer8_attention_weight_min": -46.337562561035156,
      "activations/layer9_attention_weight_max": 42.78397750854492,
      "activations/layer9_attention_weight_min": -38.25239562988281,
      "epoch": 11.09,
      "learning_rate": 7.928484848484848e-05,
      "loss": 2.7648,
      "step": 190850
    },
    {
      "activations/layer0_attention_weight_max": 15.019148826599121,
      "activations/layer0_attention_weight_min": -12.748720169067383,
      "activations/layer10_attention_weight_max": 38.29595947265625,
      "activations/layer10_attention_weight_min": -36.22926330566406,
      "activations/layer11_attention_weight_max": 33.320716857910156,
      "activations/layer11_attention_weight_min": -34.834712982177734,
      "activations/layer12_attention_weight_max": 26.903520584106445,
      "activations/layer12_attention_weight_min": -29.24753761291504,
      "activations/layer13_attention_weight_max": 40.89812088012695,
      "activations/layer13_attention_weight_min": -36.17278289794922,
      "activations/layer14_attention_weight_max": 44.93977737426758,
      "activations/layer14_attention_weight_min": -39.35879898071289,
      "activations/layer15_attention_weight_max": 36.53532409667969,
      "activations/layer15_attention_weight_min": -32.81262969970703,
      "activations/layer16_attention_weight_max": 32.089927673339844,
      "activations/layer16_attention_weight_min": -27.000755310058594,
      "activations/layer17_attention_weight_max": 34.879127502441406,
      "activations/layer17_attention_weight_min": -26.047889709472656,
      "activations/layer18_attention_weight_max": 33.57475662231445,
      "activations/layer18_attention_weight_min": -24.980287551879883,
      "activations/layer19_attention_weight_max": 34.977195739746094,
      "activations/layer19_attention_weight_min": -29.596288681030273,
      "activations/layer1_attention_weight_max": 17.362829208374023,
      "activations/layer1_attention_weight_min": -15.104972839355469,
      "activations/layer20_attention_weight_max": 32.03331756591797,
      "activations/layer20_attention_weight_min": -24.745084762573242,
      "activations/layer21_attention_weight_max": 33.242652893066406,
      "activations/layer21_attention_weight_min": -24.414043426513672,
      "activations/layer22_attention_weight_max": 53.236324310302734,
      "activations/layer22_attention_weight_min": -30.127485275268555,
      "activations/layer23_attention_weight_max": 35.61534118652344,
      "activations/layer23_attention_weight_min": -25.24236297607422,
      "activations/layer2_attention_weight_max": 35.64349365234375,
      "activations/layer2_attention_weight_min": -34.07183074951172,
      "activations/layer3_attention_weight_max": 94.26134490966797,
      "activations/layer3_attention_weight_min": -98.0252685546875,
      "activations/layer4_attention_weight_max": 110.74958801269531,
      "activations/layer4_attention_weight_min": -122.29033660888672,
      "activations/layer5_attention_weight_max": 51.66551971435547,
      "activations/layer5_attention_weight_min": -60.55500411987305,
      "activations/layer6_attention_weight_max": 45.610836029052734,
      "activations/layer6_attention_weight_min": -48.35905456542969,
      "activations/layer7_attention_weight_max": 98.74972534179688,
      "activations/layer7_attention_weight_min": -98.8187484741211,
      "activations/layer8_attention_weight_max": 43.6821174621582,
      "activations/layer8_attention_weight_min": -45.70332336425781,
      "activations/layer9_attention_weight_max": 36.3671875,
      "activations/layer9_attention_weight_min": -36.589698791503906,
      "epoch": 11.09,
      "learning_rate": 7.926590909090909e-05,
      "loss": 2.776,
      "step": 190900
    },
    {
      "activations/layer0_attention_weight_max": 14.394712448120117,
      "activations/layer0_attention_weight_min": -11.715766906738281,
      "activations/layer10_attention_weight_max": 30.302751541137695,
      "activations/layer10_attention_weight_min": -32.44715881347656,
      "activations/layer11_attention_weight_max": 29.79741668701172,
      "activations/layer11_attention_weight_min": -32.034027099609375,
      "activations/layer12_attention_weight_max": 27.24286460876465,
      "activations/layer12_attention_weight_min": -23.382423400878906,
      "activations/layer13_attention_weight_max": 39.0118522644043,
      "activations/layer13_attention_weight_min": -33.08430862426758,
      "activations/layer14_attention_weight_max": 42.53229522705078,
      "activations/layer14_attention_weight_min": -38.06381607055664,
      "activations/layer15_attention_weight_max": 33.88739776611328,
      "activations/layer15_attention_weight_min": -32.03781509399414,
      "activations/layer16_attention_weight_max": 31.91051483154297,
      "activations/layer16_attention_weight_min": -29.191394805908203,
      "activations/layer17_attention_weight_max": 30.15615463256836,
      "activations/layer17_attention_weight_min": -27.07097816467285,
      "activations/layer18_attention_weight_max": 31.880205154418945,
      "activations/layer18_attention_weight_min": -25.202970504760742,
      "activations/layer19_attention_weight_max": 36.50791931152344,
      "activations/layer19_attention_weight_min": -33.252628326416016,
      "activations/layer1_attention_weight_max": 17.04950523376465,
      "activations/layer1_attention_weight_min": -13.82107925415039,
      "activations/layer20_attention_weight_max": 29.824256896972656,
      "activations/layer20_attention_weight_min": -26.569799423217773,
      "activations/layer21_attention_weight_max": 30.286367416381836,
      "activations/layer21_attention_weight_min": -25.01837158203125,
      "activations/layer22_attention_weight_max": 49.17603302001953,
      "activations/layer22_attention_weight_min": -32.12623977661133,
      "activations/layer23_attention_weight_max": 36.73787307739258,
      "activations/layer23_attention_weight_min": -24.986427307128906,
      "activations/layer2_attention_weight_max": 32.46778869628906,
      "activations/layer2_attention_weight_min": -31.798114776611328,
      "activations/layer3_attention_weight_max": 93.46633911132812,
      "activations/layer3_attention_weight_min": -89.77169799804688,
      "activations/layer4_attention_weight_max": 109.28497314453125,
      "activations/layer4_attention_weight_min": -109.97928619384766,
      "activations/layer5_attention_weight_max": 51.662994384765625,
      "activations/layer5_attention_weight_min": -61.63517761230469,
      "activations/layer6_attention_weight_max": 43.18815231323242,
      "activations/layer6_attention_weight_min": -45.489986419677734,
      "activations/layer7_attention_weight_max": 89.95530700683594,
      "activations/layer7_attention_weight_min": -89.51980590820312,
      "activations/layer8_attention_weight_max": 36.95183181762695,
      "activations/layer8_attention_weight_min": -39.63802719116211,
      "activations/layer9_attention_weight_max": 30.128515243530273,
      "activations/layer9_attention_weight_min": -31.619281768798828,
      "epoch": 11.1,
      "learning_rate": 7.924696969696969e-05,
      "loss": 2.7791,
      "step": 190950
    },
    {
      "activations/layer0_attention_weight_max": 15.46580696105957,
      "activations/layer0_attention_weight_min": -12.144024848937988,
      "activations/layer10_attention_weight_max": 33.6517333984375,
      "activations/layer10_attention_weight_min": -31.143070220947266,
      "activations/layer11_attention_weight_max": 31.24466323852539,
      "activations/layer11_attention_weight_min": -34.078556060791016,
      "activations/layer12_attention_weight_max": 23.84132194519043,
      "activations/layer12_attention_weight_min": -23.58576011657715,
      "activations/layer13_attention_weight_max": 34.23371124267578,
      "activations/layer13_attention_weight_min": -33.67377471923828,
      "activations/layer14_attention_weight_max": 41.799564361572266,
      "activations/layer14_attention_weight_min": -37.1983528137207,
      "activations/layer15_attention_weight_max": 36.555320739746094,
      "activations/layer15_attention_weight_min": -33.63523483276367,
      "activations/layer16_attention_weight_max": 30.288740158081055,
      "activations/layer16_attention_weight_min": -29.255023956298828,
      "activations/layer17_attention_weight_max": 30.705421447753906,
      "activations/layer17_attention_weight_min": -27.3271427154541,
      "activations/layer18_attention_weight_max": 31.637014389038086,
      "activations/layer18_attention_weight_min": -24.38886260986328,
      "activations/layer19_attention_weight_max": 33.52686309814453,
      "activations/layer19_attention_weight_min": -35.51926803588867,
      "activations/layer1_attention_weight_max": 16.606420516967773,
      "activations/layer1_attention_weight_min": -15.770041465759277,
      "activations/layer20_attention_weight_max": 29.288963317871094,
      "activations/layer20_attention_weight_min": -26.63432502746582,
      "activations/layer21_attention_weight_max": 29.03597068786621,
      "activations/layer21_attention_weight_min": -25.923019409179688,
      "activations/layer22_attention_weight_max": 39.49429702758789,
      "activations/layer22_attention_weight_min": -29.5721492767334,
      "activations/layer23_attention_weight_max": 35.19609069824219,
      "activations/layer23_attention_weight_min": -24.26299285888672,
      "activations/layer2_attention_weight_max": 32.52946853637695,
      "activations/layer2_attention_weight_min": -34.13790512084961,
      "activations/layer3_attention_weight_max": 94.77734375,
      "activations/layer3_attention_weight_min": -98.23728942871094,
      "activations/layer4_attention_weight_max": 113.65594482421875,
      "activations/layer4_attention_weight_min": -116.4043197631836,
      "activations/layer5_attention_weight_max": 54.07737731933594,
      "activations/layer5_attention_weight_min": -66.86226654052734,
      "activations/layer6_attention_weight_max": 46.568702697753906,
      "activations/layer6_attention_weight_min": -49.2347526550293,
      "activations/layer7_attention_weight_max": 91.57281494140625,
      "activations/layer7_attention_weight_min": -91.36177062988281,
      "activations/layer8_attention_weight_max": 40.64314651489258,
      "activations/layer8_attention_weight_min": -39.35036849975586,
      "activations/layer9_attention_weight_max": 36.12342834472656,
      "activations/layer9_attention_weight_min": -32.503562927246094,
      "epoch": 11.1,
      "learning_rate": 7.92280303030303e-05,
      "loss": 2.7793,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_loss": 2.734375,
      "eval_runtime": 8.4964,
      "eval_samples_per_second": 505.392,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.4964,
      "eval_openwebtext_samples_per_second": 505.392,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 2.0506,
      "eval_wikitext_samples_per_second": 222.37,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.5927,
      "eval_lambada_samples_per_second": 507.573,
      "step": 191000
    },
    {
      "activations/layer0_attention_weight_max": 14.962254524230957,
      "activations/layer0_attention_weight_min": -12.201253890991211,
      "activations/layer10_attention_weight_max": 31.930118560791016,
      "activations/layer10_attention_weight_min": -31.988597869873047,
      "activations/layer11_attention_weight_max": 33.25885772705078,
      "activations/layer11_attention_weight_min": -31.326025009155273,
      "activations/layer12_attention_weight_max": 23.57383155822754,
      "activations/layer12_attention_weight_min": -30.054784774780273,
      "activations/layer13_attention_weight_max": 40.36046600341797,
      "activations/layer13_attention_weight_min": -33.15671157836914,
      "activations/layer14_attention_weight_max": 46.471195220947266,
      "activations/layer14_attention_weight_min": -40.883541107177734,
      "activations/layer15_attention_weight_max": 35.27222442626953,
      "activations/layer15_attention_weight_min": -34.00847244262695,
      "activations/layer16_attention_weight_max": 32.42454528808594,
      "activations/layer16_attention_weight_min": -26.620159149169922,
      "activations/layer17_attention_weight_max": 30.612878799438477,
      "activations/layer17_attention_weight_min": -27.000253677368164,
      "activations/layer18_attention_weight_max": 30.479642868041992,
      "activations/layer18_attention_weight_min": -24.324064254760742,
      "activations/layer19_attention_weight_max": 38.54489517211914,
      "activations/layer19_attention_weight_min": -33.79384231567383,
      "activations/layer1_attention_weight_max": 16.45268440246582,
      "activations/layer1_attention_weight_min": -15.344779014587402,
      "activations/layer20_attention_weight_max": 31.2398624420166,
      "activations/layer20_attention_weight_min": -25.355501174926758,
      "activations/layer21_attention_weight_max": 30.077829360961914,
      "activations/layer21_attention_weight_min": -24.94949722290039,
      "activations/layer22_attention_weight_max": 52.24604034423828,
      "activations/layer22_attention_weight_min": -30.023799896240234,
      "activations/layer23_attention_weight_max": 40.455570220947266,
      "activations/layer23_attention_weight_min": -24.88710594177246,
      "activations/layer2_attention_weight_max": 33.62309646606445,
      "activations/layer2_attention_weight_min": -33.155975341796875,
      "activations/layer3_attention_weight_max": 96.90872955322266,
      "activations/layer3_attention_weight_min": -100.10835266113281,
      "activations/layer4_attention_weight_max": 111.30084228515625,
      "activations/layer4_attention_weight_min": -114.262451171875,
      "activations/layer5_attention_weight_max": 54.180397033691406,
      "activations/layer5_attention_weight_min": -61.28217697143555,
      "activations/layer6_attention_weight_max": 44.84483337402344,
      "activations/layer6_attention_weight_min": -50.25746154785156,
      "activations/layer7_attention_weight_max": 95.97893524169922,
      "activations/layer7_attention_weight_min": -88.81427764892578,
      "activations/layer8_attention_weight_max": 39.31142044067383,
      "activations/layer8_attention_weight_min": -41.353492736816406,
      "activations/layer9_attention_weight_max": 31.007352828979492,
      "activations/layer9_attention_weight_min": -33.1335563659668,
      "epoch": 11.1,
      "learning_rate": 7.920909090909089e-05,
      "loss": 2.7869,
      "step": 191050
    },
    {
      "activations/layer0_attention_weight_max": 15.484477043151855,
      "activations/layer0_attention_weight_min": -12.923904418945312,
      "activations/layer10_attention_weight_max": 31.706417083740234,
      "activations/layer10_attention_weight_min": -34.14680862426758,
      "activations/layer11_attention_weight_max": 30.49293327331543,
      "activations/layer11_attention_weight_min": -30.92955780029297,
      "activations/layer12_attention_weight_max": 23.50296401977539,
      "activations/layer12_attention_weight_min": -25.925127029418945,
      "activations/layer13_attention_weight_max": 34.98759460449219,
      "activations/layer13_attention_weight_min": -32.71668243408203,
      "activations/layer14_attention_weight_max": 45.06227111816406,
      "activations/layer14_attention_weight_min": -38.66132736206055,
      "activations/layer15_attention_weight_max": 35.84877014160156,
      "activations/layer15_attention_weight_min": -32.7178955078125,
      "activations/layer16_attention_weight_max": 30.68168067932129,
      "activations/layer16_attention_weight_min": -28.8072566986084,
      "activations/layer17_attention_weight_max": 29.550642013549805,
      "activations/layer17_attention_weight_min": -27.135408401489258,
      "activations/layer18_attention_weight_max": 29.925209045410156,
      "activations/layer18_attention_weight_min": -24.594518661499023,
      "activations/layer19_attention_weight_max": 33.76003646850586,
      "activations/layer19_attention_weight_min": -30.69780921936035,
      "activations/layer1_attention_weight_max": 17.34505271911621,
      "activations/layer1_attention_weight_min": -13.881677627563477,
      "activations/layer20_attention_weight_max": 27.864112854003906,
      "activations/layer20_attention_weight_min": -24.045499801635742,
      "activations/layer21_attention_weight_max": 27.948965072631836,
      "activations/layer21_attention_weight_min": -23.99296760559082,
      "activations/layer22_attention_weight_max": 42.0852165222168,
      "activations/layer22_attention_weight_min": -30.54265594482422,
      "activations/layer23_attention_weight_max": 33.426658630371094,
      "activations/layer23_attention_weight_min": -24.090688705444336,
      "activations/layer2_attention_weight_max": 34.156551361083984,
      "activations/layer2_attention_weight_min": -33.649375915527344,
      "activations/layer3_attention_weight_max": 93.34707641601562,
      "activations/layer3_attention_weight_min": -97.71707916259766,
      "activations/layer4_attention_weight_max": 109.06520080566406,
      "activations/layer4_attention_weight_min": -123.19010162353516,
      "activations/layer5_attention_weight_max": 53.704559326171875,
      "activations/layer5_attention_weight_min": -61.86109161376953,
      "activations/layer6_attention_weight_max": 44.553382873535156,
      "activations/layer6_attention_weight_min": -48.16127014160156,
      "activations/layer7_attention_weight_max": 91.61331939697266,
      "activations/layer7_attention_weight_min": -91.27974700927734,
      "activations/layer8_attention_weight_max": 39.45848846435547,
      "activations/layer8_attention_weight_min": -41.267051696777344,
      "activations/layer9_attention_weight_max": 31.871076583862305,
      "activations/layer9_attention_weight_min": -33.79022216796875,
      "epoch": 11.1,
      "learning_rate": 7.91901515151515e-05,
      "loss": 2.7822,
      "step": 191100
    },
    {
      "activations/layer0_attention_weight_max": 15.00155258178711,
      "activations/layer0_attention_weight_min": -13.386466979980469,
      "activations/layer10_attention_weight_max": 33.85254669189453,
      "activations/layer10_attention_weight_min": -34.28263473510742,
      "activations/layer11_attention_weight_max": 34.23237991333008,
      "activations/layer11_attention_weight_min": -33.339439392089844,
      "activations/layer12_attention_weight_max": 28.050418853759766,
      "activations/layer12_attention_weight_min": -25.84625816345215,
      "activations/layer13_attention_weight_max": 51.56007766723633,
      "activations/layer13_attention_weight_min": -36.179439544677734,
      "activations/layer14_attention_weight_max": 52.657405853271484,
      "activations/layer14_attention_weight_min": -40.999107360839844,
      "activations/layer15_attention_weight_max": 46.327720642089844,
      "activations/layer15_attention_weight_min": -33.14704513549805,
      "activations/layer16_attention_weight_max": 36.202667236328125,
      "activations/layer16_attention_weight_min": -29.42616081237793,
      "activations/layer17_attention_weight_max": 33.72224426269531,
      "activations/layer17_attention_weight_min": -26.320749282836914,
      "activations/layer18_attention_weight_max": 34.97731018066406,
      "activations/layer18_attention_weight_min": -23.94387435913086,
      "activations/layer19_attention_weight_max": 48.006954193115234,
      "activations/layer19_attention_weight_min": -32.616180419921875,
      "activations/layer1_attention_weight_max": 17.04535484313965,
      "activations/layer1_attention_weight_min": -14.651052474975586,
      "activations/layer20_attention_weight_max": 39.137081146240234,
      "activations/layer20_attention_weight_min": -25.089988708496094,
      "activations/layer21_attention_weight_max": 34.349613189697266,
      "activations/layer21_attention_weight_min": -24.58732795715332,
      "activations/layer22_attention_weight_max": 60.84006118774414,
      "activations/layer22_attention_weight_min": -32.91928482055664,
      "activations/layer23_attention_weight_max": 52.275733947753906,
      "activations/layer23_attention_weight_min": -25.503955841064453,
      "activations/layer2_attention_weight_max": 33.93461227416992,
      "activations/layer2_attention_weight_min": -33.96696090698242,
      "activations/layer3_attention_weight_max": 102.37066650390625,
      "activations/layer3_attention_weight_min": -99.42221069335938,
      "activations/layer4_attention_weight_max": 117.7279281616211,
      "activations/layer4_attention_weight_min": -115.73404693603516,
      "activations/layer5_attention_weight_max": 53.22309875488281,
      "activations/layer5_attention_weight_min": -59.941062927246094,
      "activations/layer6_attention_weight_max": 45.960628509521484,
      "activations/layer6_attention_weight_min": -50.01095962524414,
      "activations/layer7_attention_weight_max": 95.5817642211914,
      "activations/layer7_attention_weight_min": -102.00690460205078,
      "activations/layer8_attention_weight_max": 40.993804931640625,
      "activations/layer8_attention_weight_min": -43.319053649902344,
      "activations/layer9_attention_weight_max": 34.048133850097656,
      "activations/layer9_attention_weight_min": -36.72962951660156,
      "epoch": 11.11,
      "learning_rate": 7.91715909090909e-05,
      "loss": 2.7776,
      "step": 191150
    },
    {
      "activations/layer0_attention_weight_max": 15.960358619689941,
      "activations/layer0_attention_weight_min": -12.1162748336792,
      "activations/layer10_attention_weight_max": 31.017118453979492,
      "activations/layer10_attention_weight_min": -31.7157039642334,
      "activations/layer11_attention_weight_max": 31.237293243408203,
      "activations/layer11_attention_weight_min": -30.568119049072266,
      "activations/layer12_attention_weight_max": 25.13296127319336,
      "activations/layer12_attention_weight_min": -26.220924377441406,
      "activations/layer13_attention_weight_max": 37.59905242919922,
      "activations/layer13_attention_weight_min": -35.34347152709961,
      "activations/layer14_attention_weight_max": 44.835330963134766,
      "activations/layer14_attention_weight_min": -35.81735610961914,
      "activations/layer15_attention_weight_max": 35.69648742675781,
      "activations/layer15_attention_weight_min": -33.08030319213867,
      "activations/layer16_attention_weight_max": 32.52806091308594,
      "activations/layer16_attention_weight_min": -28.042219161987305,
      "activations/layer17_attention_weight_max": 32.78602600097656,
      "activations/layer17_attention_weight_min": -25.24999237060547,
      "activations/layer18_attention_weight_max": 31.730934143066406,
      "activations/layer18_attention_weight_min": -23.74484634399414,
      "activations/layer19_attention_weight_max": 34.38506317138672,
      "activations/layer19_attention_weight_min": -33.21317672729492,
      "activations/layer1_attention_weight_max": 16.567203521728516,
      "activations/layer1_attention_weight_min": -14.969271659851074,
      "activations/layer20_attention_weight_max": 30.887977600097656,
      "activations/layer20_attention_weight_min": -24.786298751831055,
      "activations/layer21_attention_weight_max": 28.617061614990234,
      "activations/layer21_attention_weight_min": -24.407339096069336,
      "activations/layer22_attention_weight_max": 42.892581939697266,
      "activations/layer22_attention_weight_min": -30.6669921875,
      "activations/layer23_attention_weight_max": 35.07656478881836,
      "activations/layer23_attention_weight_min": -25.715946197509766,
      "activations/layer2_attention_weight_max": 34.85252380371094,
      "activations/layer2_attention_weight_min": -34.25970458984375,
      "activations/layer3_attention_weight_max": 100.38933563232422,
      "activations/layer3_attention_weight_min": -96.8656997680664,
      "activations/layer4_attention_weight_max": 114.10643768310547,
      "activations/layer4_attention_weight_min": -110.87501525878906,
      "activations/layer5_attention_weight_max": 51.91986083984375,
      "activations/layer5_attention_weight_min": -63.39417266845703,
      "activations/layer6_attention_weight_max": 45.91118240356445,
      "activations/layer6_attention_weight_min": -47.260990142822266,
      "activations/layer7_attention_weight_max": 85.37342834472656,
      "activations/layer7_attention_weight_min": -84.21129608154297,
      "activations/layer8_attention_weight_max": 37.463104248046875,
      "activations/layer8_attention_weight_min": -38.91075897216797,
      "activations/layer9_attention_weight_max": 32.42270278930664,
      "activations/layer9_attention_weight_min": -32.50795364379883,
      "epoch": 11.11,
      "learning_rate": 7.915265151515151e-05,
      "loss": 2.7788,
      "step": 191200
    },
    {
      "activations/layer0_attention_weight_max": 17.0821475982666,
      "activations/layer0_attention_weight_min": -12.253689765930176,
      "activations/layer10_attention_weight_max": 28.766984939575195,
      "activations/layer10_attention_weight_min": -29.752986907958984,
      "activations/layer11_attention_weight_max": 28.534866333007812,
      "activations/layer11_attention_weight_min": -30.681859970092773,
      "activations/layer12_attention_weight_max": 25.94138526916504,
      "activations/layer12_attention_weight_min": -26.514183044433594,
      "activations/layer13_attention_weight_max": 36.29066848754883,
      "activations/layer13_attention_weight_min": -31.635852813720703,
      "activations/layer14_attention_weight_max": 41.436195373535156,
      "activations/layer14_attention_weight_min": -35.837310791015625,
      "activations/layer15_attention_weight_max": 37.22077560424805,
      "activations/layer15_attention_weight_min": -29.937475204467773,
      "activations/layer16_attention_weight_max": 32.691280364990234,
      "activations/layer16_attention_weight_min": -26.181793212890625,
      "activations/layer17_attention_weight_max": 33.23273849487305,
      "activations/layer17_attention_weight_min": -24.156667709350586,
      "activations/layer18_attention_weight_max": 29.77879524230957,
      "activations/layer18_attention_weight_min": -23.503835678100586,
      "activations/layer19_attention_weight_max": 39.57658004760742,
      "activations/layer19_attention_weight_min": -31.612777709960938,
      "activations/layer1_attention_weight_max": 17.87677001953125,
      "activations/layer1_attention_weight_min": -15.740374565124512,
      "activations/layer20_attention_weight_max": 32.519588470458984,
      "activations/layer20_attention_weight_min": -25.45819664001465,
      "activations/layer21_attention_weight_max": 32.394569396972656,
      "activations/layer21_attention_weight_min": -23.912731170654297,
      "activations/layer22_attention_weight_max": 42.12558364868164,
      "activations/layer22_attention_weight_min": -27.28866958618164,
      "activations/layer23_attention_weight_max": 34.61785125732422,
      "activations/layer23_attention_weight_min": -24.77154541015625,
      "activations/layer2_attention_weight_max": 34.07762145996094,
      "activations/layer2_attention_weight_min": -32.70198440551758,
      "activations/layer3_attention_weight_max": 94.00065612792969,
      "activations/layer3_attention_weight_min": -95.42123413085938,
      "activations/layer4_attention_weight_max": 111.2166519165039,
      "activations/layer4_attention_weight_min": -105.95243072509766,
      "activations/layer5_attention_weight_max": 50.56081771850586,
      "activations/layer5_attention_weight_min": -56.8333740234375,
      "activations/layer6_attention_weight_max": 42.54658889770508,
      "activations/layer6_attention_weight_min": -43.87354278564453,
      "activations/layer7_attention_weight_max": 88.66526794433594,
      "activations/layer7_attention_weight_min": -83.58660125732422,
      "activations/layer8_attention_weight_max": 36.6532096862793,
      "activations/layer8_attention_weight_min": -38.214508056640625,
      "activations/layer9_attention_weight_max": 29.507577896118164,
      "activations/layer9_attention_weight_min": -30.992700576782227,
      "epoch": 11.11,
      "learning_rate": 7.913371212121211e-05,
      "loss": 2.7755,
      "step": 191250
    },
    {
      "activations/layer0_attention_weight_max": 15.329916000366211,
      "activations/layer0_attention_weight_min": -12.33297348022461,
      "activations/layer10_attention_weight_max": 43.31768798828125,
      "activations/layer10_attention_weight_min": -39.88555908203125,
      "activations/layer11_attention_weight_max": 39.47252655029297,
      "activations/layer11_attention_weight_min": -39.07889938354492,
      "activations/layer12_attention_weight_max": 27.438777923583984,
      "activations/layer12_attention_weight_min": -26.262832641601562,
      "activations/layer13_attention_weight_max": 44.70475769042969,
      "activations/layer13_attention_weight_min": -38.539546966552734,
      "activations/layer14_attention_weight_max": 59.24618148803711,
      "activations/layer14_attention_weight_min": -43.659698486328125,
      "activations/layer15_attention_weight_max": 41.622806549072266,
      "activations/layer15_attention_weight_min": -35.42823028564453,
      "activations/layer16_attention_weight_max": 37.10695266723633,
      "activations/layer16_attention_weight_min": -29.881698608398438,
      "activations/layer17_attention_weight_max": 34.70481872558594,
      "activations/layer17_attention_weight_min": -31.211212158203125,
      "activations/layer18_attention_weight_max": 33.21942138671875,
      "activations/layer18_attention_weight_min": -26.015792846679688,
      "activations/layer19_attention_weight_max": 42.223323822021484,
      "activations/layer19_attention_weight_min": -32.15431213378906,
      "activations/layer1_attention_weight_max": 17.204933166503906,
      "activations/layer1_attention_weight_min": -15.895085334777832,
      "activations/layer20_attention_weight_max": 34.8753547668457,
      "activations/layer20_attention_weight_min": -25.11251449584961,
      "activations/layer21_attention_weight_max": 31.222875595092773,
      "activations/layer21_attention_weight_min": -25.77683448791504,
      "activations/layer22_attention_weight_max": 55.08359146118164,
      "activations/layer22_attention_weight_min": -31.849994659423828,
      "activations/layer23_attention_weight_max": 39.14385223388672,
      "activations/layer23_attention_weight_min": -25.86211395263672,
      "activations/layer2_attention_weight_max": 33.862083435058594,
      "activations/layer2_attention_weight_min": -34.16775131225586,
      "activations/layer3_attention_weight_max": 97.70437622070312,
      "activations/layer3_attention_weight_min": -103.19612121582031,
      "activations/layer4_attention_weight_max": 115.0947265625,
      "activations/layer4_attention_weight_min": -115.6373519897461,
      "activations/layer5_attention_weight_max": 53.794273376464844,
      "activations/layer5_attention_weight_min": -64.6710205078125,
      "activations/layer6_attention_weight_max": 48.0902214050293,
      "activations/layer6_attention_weight_min": -50.12851333618164,
      "activations/layer7_attention_weight_max": 103.2603759765625,
      "activations/layer7_attention_weight_min": -104.05952453613281,
      "activations/layer8_attention_weight_max": 48.09355545043945,
      "activations/layer8_attention_weight_min": -47.5778694152832,
      "activations/layer9_attention_weight_max": 42.78279495239258,
      "activations/layer9_attention_weight_min": -40.45974349975586,
      "epoch": 11.12,
      "learning_rate": 7.911477272727271e-05,
      "loss": 2.7819,
      "step": 191300
    },
    {
      "activations/layer0_attention_weight_max": 15.236973762512207,
      "activations/layer0_attention_weight_min": -12.859732627868652,
      "activations/layer10_attention_weight_max": 34.40388107299805,
      "activations/layer10_attention_weight_min": -33.23246765136719,
      "activations/layer11_attention_weight_max": 35.72798156738281,
      "activations/layer11_attention_weight_min": -31.355091094970703,
      "activations/layer12_attention_weight_max": 24.622520446777344,
      "activations/layer12_attention_weight_min": -24.65269660949707,
      "activations/layer13_attention_weight_max": 38.788536071777344,
      "activations/layer13_attention_weight_min": -32.797847747802734,
      "activations/layer14_attention_weight_max": 42.56267547607422,
      "activations/layer14_attention_weight_min": -35.57488250732422,
      "activations/layer15_attention_weight_max": 35.74458694458008,
      "activations/layer15_attention_weight_min": -31.292707443237305,
      "activations/layer16_attention_weight_max": 31.32225799560547,
      "activations/layer16_attention_weight_min": -26.8702335357666,
      "activations/layer17_attention_weight_max": 32.3297119140625,
      "activations/layer17_attention_weight_min": -25.03054428100586,
      "activations/layer18_attention_weight_max": 29.910104751586914,
      "activations/layer18_attention_weight_min": -23.381534576416016,
      "activations/layer19_attention_weight_max": 36.01017379760742,
      "activations/layer19_attention_weight_min": -31.0585994720459,
      "activations/layer1_attention_weight_max": 18.16636085510254,
      "activations/layer1_attention_weight_min": -15.648760795593262,
      "activations/layer20_attention_weight_max": 29.190765380859375,
      "activations/layer20_attention_weight_min": -24.653879165649414,
      "activations/layer21_attention_weight_max": 29.340286254882812,
      "activations/layer21_attention_weight_min": -23.45902442932129,
      "activations/layer22_attention_weight_max": 44.84409713745117,
      "activations/layer22_attention_weight_min": -30.486772537231445,
      "activations/layer23_attention_weight_max": 34.94778823852539,
      "activations/layer23_attention_weight_min": -22.49697494506836,
      "activations/layer2_attention_weight_max": 31.430343627929688,
      "activations/layer2_attention_weight_min": -34.418365478515625,
      "activations/layer3_attention_weight_max": 96.01293182373047,
      "activations/layer3_attention_weight_min": -96.97673034667969,
      "activations/layer4_attention_weight_max": 114.44551086425781,
      "activations/layer4_attention_weight_min": -111.75322723388672,
      "activations/layer5_attention_weight_max": 53.132625579833984,
      "activations/layer5_attention_weight_min": -62.09442901611328,
      "activations/layer6_attention_weight_max": 45.17976760864258,
      "activations/layer6_attention_weight_min": -47.706417083740234,
      "activations/layer7_attention_weight_max": 96.51165008544922,
      "activations/layer7_attention_weight_min": -89.23827362060547,
      "activations/layer8_attention_weight_max": 41.898094177246094,
      "activations/layer8_attention_weight_min": -42.08266067504883,
      "activations/layer9_attention_weight_max": 33.96904754638672,
      "activations/layer9_attention_weight_min": -35.31072998046875,
      "epoch": 11.12,
      "learning_rate": 7.909583333333333e-05,
      "loss": 2.7736,
      "step": 191350
    },
    {
      "activations/layer0_attention_weight_max": 14.92611026763916,
      "activations/layer0_attention_weight_min": -12.800863265991211,
      "activations/layer10_attention_weight_max": 35.77899169921875,
      "activations/layer10_attention_weight_min": -33.114471435546875,
      "activations/layer11_attention_weight_max": 34.27996063232422,
      "activations/layer11_attention_weight_min": -34.54286575317383,
      "activations/layer12_attention_weight_max": 23.312225341796875,
      "activations/layer12_attention_weight_min": -25.36628532409668,
      "activations/layer13_attention_weight_max": 37.92139434814453,
      "activations/layer13_attention_weight_min": -34.6581916809082,
      "activations/layer14_attention_weight_max": 41.696319580078125,
      "activations/layer14_attention_weight_min": -35.770225524902344,
      "activations/layer15_attention_weight_max": 33.03319549560547,
      "activations/layer15_attention_weight_min": -30.14051055908203,
      "activations/layer16_attention_weight_max": 30.62702178955078,
      "activations/layer16_attention_weight_min": -25.76288414001465,
      "activations/layer17_attention_weight_max": 29.142282485961914,
      "activations/layer17_attention_weight_min": -23.53183937072754,
      "activations/layer18_attention_weight_max": 32.44213104248047,
      "activations/layer18_attention_weight_min": -23.497102737426758,
      "activations/layer19_attention_weight_max": 35.10110092163086,
      "activations/layer19_attention_weight_min": -30.8585262298584,
      "activations/layer1_attention_weight_max": 15.701584815979004,
      "activations/layer1_attention_weight_min": -13.90292739868164,
      "activations/layer20_attention_weight_max": 26.65035629272461,
      "activations/layer20_attention_weight_min": -22.877164840698242,
      "activations/layer21_attention_weight_max": 29.22115707397461,
      "activations/layer21_attention_weight_min": -24.555002212524414,
      "activations/layer22_attention_weight_max": 41.47704315185547,
      "activations/layer22_attention_weight_min": -28.110912322998047,
      "activations/layer23_attention_weight_max": 30.8543758392334,
      "activations/layer23_attention_weight_min": -25.348785400390625,
      "activations/layer2_attention_weight_max": 34.25172805786133,
      "activations/layer2_attention_weight_min": -32.33198165893555,
      "activations/layer3_attention_weight_max": 94.9710464477539,
      "activations/layer3_attention_weight_min": -98.89317321777344,
      "activations/layer4_attention_weight_max": 111.05269622802734,
      "activations/layer4_attention_weight_min": -114.70719146728516,
      "activations/layer5_attention_weight_max": 52.0672607421875,
      "activations/layer5_attention_weight_min": -63.84638977050781,
      "activations/layer6_attention_weight_max": 43.54180145263672,
      "activations/layer6_attention_weight_min": -47.94523239135742,
      "activations/layer7_attention_weight_max": 87.69595336914062,
      "activations/layer7_attention_weight_min": -87.22245788574219,
      "activations/layer8_attention_weight_max": 39.16767883300781,
      "activations/layer8_attention_weight_min": -42.03731155395508,
      "activations/layer9_attention_weight_max": 34.23758316040039,
      "activations/layer9_attention_weight_min": -33.607093811035156,
      "epoch": 11.12,
      "learning_rate": 7.907689393939393e-05,
      "loss": 2.7672,
      "step": 191400
    },
    {
      "activations/layer0_attention_weight_max": 15.473908424377441,
      "activations/layer0_attention_weight_min": -12.250763893127441,
      "activations/layer10_attention_weight_max": 36.10799026489258,
      "activations/layer10_attention_weight_min": -35.69112014770508,
      "activations/layer11_attention_weight_max": 36.54100799560547,
      "activations/layer11_attention_weight_min": -33.69325637817383,
      "activations/layer12_attention_weight_max": 27.69573402404785,
      "activations/layer12_attention_weight_min": -26.434341430664062,
      "activations/layer13_attention_weight_max": 45.8856086730957,
      "activations/layer13_attention_weight_min": -37.415550231933594,
      "activations/layer14_attention_weight_max": 52.07566452026367,
      "activations/layer14_attention_weight_min": -39.59737014770508,
      "activations/layer15_attention_weight_max": 43.550777435302734,
      "activations/layer15_attention_weight_min": -31.446231842041016,
      "activations/layer16_attention_weight_max": 35.00761413574219,
      "activations/layer16_attention_weight_min": -28.23883056640625,
      "activations/layer17_attention_weight_max": 38.587005615234375,
      "activations/layer17_attention_weight_min": -24.811269760131836,
      "activations/layer18_attention_weight_max": 32.112342834472656,
      "activations/layer18_attention_weight_min": -24.929534912109375,
      "activations/layer19_attention_weight_max": 37.4129753112793,
      "activations/layer19_attention_weight_min": -31.42392349243164,
      "activations/layer1_attention_weight_max": 16.87206268310547,
      "activations/layer1_attention_weight_min": -13.310596466064453,
      "activations/layer20_attention_weight_max": 34.60772705078125,
      "activations/layer20_attention_weight_min": -23.501474380493164,
      "activations/layer21_attention_weight_max": 33.28620910644531,
      "activations/layer21_attention_weight_min": -22.74776268005371,
      "activations/layer22_attention_weight_max": 48.30597686767578,
      "activations/layer22_attention_weight_min": -28.019853591918945,
      "activations/layer23_attention_weight_max": 37.91926574707031,
      "activations/layer23_attention_weight_min": -24.16236686706543,
      "activations/layer2_attention_weight_max": 31.620763778686523,
      "activations/layer2_attention_weight_min": -32.550804138183594,
      "activations/layer3_attention_weight_max": 92.27073669433594,
      "activations/layer3_attention_weight_min": -95.23130798339844,
      "activations/layer4_attention_weight_max": 113.14229583740234,
      "activations/layer4_attention_weight_min": -113.09840393066406,
      "activations/layer5_attention_weight_max": 51.140647888183594,
      "activations/layer5_attention_weight_min": -61.7955322265625,
      "activations/layer6_attention_weight_max": 44.65055465698242,
      "activations/layer6_attention_weight_min": -46.25773620605469,
      "activations/layer7_attention_weight_max": 94.7105712890625,
      "activations/layer7_attention_weight_min": -92.152587890625,
      "activations/layer8_attention_weight_max": 42.317710876464844,
      "activations/layer8_attention_weight_min": -43.77740478515625,
      "activations/layer9_attention_weight_max": 36.77676010131836,
      "activations/layer9_attention_weight_min": -35.38886642456055,
      "epoch": 11.12,
      "learning_rate": 7.905795454545453e-05,
      "loss": 2.7829,
      "step": 191450
    },
    {
      "activations/layer0_attention_weight_max": 16.45291519165039,
      "activations/layer0_attention_weight_min": -11.48302936553955,
      "activations/layer10_attention_weight_max": 32.21609115600586,
      "activations/layer10_attention_weight_min": -31.275991439819336,
      "activations/layer11_attention_weight_max": 32.110633850097656,
      "activations/layer11_attention_weight_min": -32.18400955200195,
      "activations/layer12_attention_weight_max": 25.767610549926758,
      "activations/layer12_attention_weight_min": -31.491382598876953,
      "activations/layer13_attention_weight_max": 35.182552337646484,
      "activations/layer13_attention_weight_min": -33.33259201049805,
      "activations/layer14_attention_weight_max": 40.67064666748047,
      "activations/layer14_attention_weight_min": -35.25626754760742,
      "activations/layer15_attention_weight_max": 33.869850158691406,
      "activations/layer15_attention_weight_min": -29.716495513916016,
      "activations/layer16_attention_weight_max": 28.736988067626953,
      "activations/layer16_attention_weight_min": -26.3734188079834,
      "activations/layer17_attention_weight_max": 33.02507781982422,
      "activations/layer17_attention_weight_min": -23.561922073364258,
      "activations/layer18_attention_weight_max": 30.836078643798828,
      "activations/layer18_attention_weight_min": -22.08165168762207,
      "activations/layer19_attention_weight_max": 36.51374435424805,
      "activations/layer19_attention_weight_min": -30.685022354125977,
      "activations/layer1_attention_weight_max": 16.865266799926758,
      "activations/layer1_attention_weight_min": -14.17428970336914,
      "activations/layer20_attention_weight_max": 32.91957473754883,
      "activations/layer20_attention_weight_min": -24.220226287841797,
      "activations/layer21_attention_weight_max": 34.405738830566406,
      "activations/layer21_attention_weight_min": -25.034635543823242,
      "activations/layer22_attention_weight_max": 50.428382873535156,
      "activations/layer22_attention_weight_min": -32.339393615722656,
      "activations/layer23_attention_weight_max": 43.432979583740234,
      "activations/layer23_attention_weight_min": -24.11696434020996,
      "activations/layer2_attention_weight_max": 34.24546813964844,
      "activations/layer2_attention_weight_min": -34.654178619384766,
      "activations/layer3_attention_weight_max": 92.60707092285156,
      "activations/layer3_attention_weight_min": -97.38742065429688,
      "activations/layer4_attention_weight_max": 109.3813247680664,
      "activations/layer4_attention_weight_min": -110.8338851928711,
      "activations/layer5_attention_weight_max": 51.38115692138672,
      "activations/layer5_attention_weight_min": -62.51007843017578,
      "activations/layer6_attention_weight_max": 45.34775161743164,
      "activations/layer6_attention_weight_min": -48.598716735839844,
      "activations/layer7_attention_weight_max": 85.71099853515625,
      "activations/layer7_attention_weight_min": -86.05368041992188,
      "activations/layer8_attention_weight_max": 40.5794563293457,
      "activations/layer8_attention_weight_min": -40.0638427734375,
      "activations/layer9_attention_weight_max": 32.097679138183594,
      "activations/layer9_attention_weight_min": -31.91892433166504,
      "epoch": 11.13,
      "learning_rate": 7.903901515151515e-05,
      "loss": 2.7755,
      "step": 191500
    },
    {
      "activations/layer0_attention_weight_max": 14.67213249206543,
      "activations/layer0_attention_weight_min": -12.10938549041748,
      "activations/layer10_attention_weight_max": 40.296875,
      "activations/layer10_attention_weight_min": -33.968597412109375,
      "activations/layer11_attention_weight_max": 36.625579833984375,
      "activations/layer11_attention_weight_min": -34.227027893066406,
      "activations/layer12_attention_weight_max": 25.875459671020508,
      "activations/layer12_attention_weight_min": -25.62190055847168,
      "activations/layer13_attention_weight_max": 40.108619689941406,
      "activations/layer13_attention_weight_min": -35.300655364990234,
      "activations/layer14_attention_weight_max": 44.11061477661133,
      "activations/layer14_attention_weight_min": -38.60795211791992,
      "activations/layer15_attention_weight_max": 34.090423583984375,
      "activations/layer15_attention_weight_min": -31.9515380859375,
      "activations/layer16_attention_weight_max": 32.45402145385742,
      "activations/layer16_attention_weight_min": -29.909687042236328,
      "activations/layer17_attention_weight_max": 30.732975006103516,
      "activations/layer17_attention_weight_min": -27.121118545532227,
      "activations/layer18_attention_weight_max": 32.37097930908203,
      "activations/layer18_attention_weight_min": -22.992536544799805,
      "activations/layer19_attention_weight_max": 33.242740631103516,
      "activations/layer19_attention_weight_min": -31.089292526245117,
      "activations/layer1_attention_weight_max": 16.30986213684082,
      "activations/layer1_attention_weight_min": -15.264483451843262,
      "activations/layer20_attention_weight_max": 27.593564987182617,
      "activations/layer20_attention_weight_min": -25.068586349487305,
      "activations/layer21_attention_weight_max": 30.856321334838867,
      "activations/layer21_attention_weight_min": -24.017730712890625,
      "activations/layer22_attention_weight_max": 46.63754653930664,
      "activations/layer22_attention_weight_min": -29.117971420288086,
      "activations/layer23_attention_weight_max": 36.50925827026367,
      "activations/layer23_attention_weight_min": -25.653358459472656,
      "activations/layer2_attention_weight_max": 34.32972717285156,
      "activations/layer2_attention_weight_min": -33.22795104980469,
      "activations/layer3_attention_weight_max": 97.11967468261719,
      "activations/layer3_attention_weight_min": -98.28605651855469,
      "activations/layer4_attention_weight_max": 117.63211822509766,
      "activations/layer4_attention_weight_min": -118.09874725341797,
      "activations/layer5_attention_weight_max": 55.11897277832031,
      "activations/layer5_attention_weight_min": -65.17462158203125,
      "activations/layer6_attention_weight_max": 45.78803634643555,
      "activations/layer6_attention_weight_min": -47.24199295043945,
      "activations/layer7_attention_weight_max": 89.66960906982422,
      "activations/layer7_attention_weight_min": -90.36097717285156,
      "activations/layer8_attention_weight_max": 42.657859802246094,
      "activations/layer8_attention_weight_min": -41.63042449951172,
      "activations/layer9_attention_weight_max": 37.54098892211914,
      "activations/layer9_attention_weight_min": -33.797550201416016,
      "epoch": 11.13,
      "learning_rate": 7.902007575757575e-05,
      "loss": 2.7796,
      "step": 191550
    },
    {
      "activations/layer0_attention_weight_max": 15.2149019241333,
      "activations/layer0_attention_weight_min": -12.80075454711914,
      "activations/layer10_attention_weight_max": 32.910770416259766,
      "activations/layer10_attention_weight_min": -32.45763397216797,
      "activations/layer11_attention_weight_max": 35.5824089050293,
      "activations/layer11_attention_weight_min": -30.62579345703125,
      "activations/layer12_attention_weight_max": 25.06414031982422,
      "activations/layer12_attention_weight_min": -25.389179229736328,
      "activations/layer13_attention_weight_max": 38.791473388671875,
      "activations/layer13_attention_weight_min": -32.51882553100586,
      "activations/layer14_attention_weight_max": 43.96853256225586,
      "activations/layer14_attention_weight_min": -36.25221252441406,
      "activations/layer15_attention_weight_max": 34.14337158203125,
      "activations/layer15_attention_weight_min": -31.600494384765625,
      "activations/layer16_attention_weight_max": 29.95725440979004,
      "activations/layer16_attention_weight_min": -26.84587287902832,
      "activations/layer17_attention_weight_max": 33.008087158203125,
      "activations/layer17_attention_weight_min": -26.424026489257812,
      "activations/layer18_attention_weight_max": 30.176668167114258,
      "activations/layer18_attention_weight_min": -22.465673446655273,
      "activations/layer19_attention_weight_max": 34.40546417236328,
      "activations/layer19_attention_weight_min": -30.688642501831055,
      "activations/layer1_attention_weight_max": 16.5167293548584,
      "activations/layer1_attention_weight_min": -14.612537384033203,
      "activations/layer20_attention_weight_max": 28.31243324279785,
      "activations/layer20_attention_weight_min": -25.09479331970215,
      "activations/layer21_attention_weight_max": 26.637340545654297,
      "activations/layer21_attention_weight_min": -23.91326141357422,
      "activations/layer22_attention_weight_max": 43.208919525146484,
      "activations/layer22_attention_weight_min": -30.200904846191406,
      "activations/layer23_attention_weight_max": 35.90845489501953,
      "activations/layer23_attention_weight_min": -24.496496200561523,
      "activations/layer2_attention_weight_max": 32.651611328125,
      "activations/layer2_attention_weight_min": -31.970645904541016,
      "activations/layer3_attention_weight_max": 93.05760955810547,
      "activations/layer3_attention_weight_min": -95.8127212524414,
      "activations/layer4_attention_weight_max": 111.8410873413086,
      "activations/layer4_attention_weight_min": -108.08040618896484,
      "activations/layer5_attention_weight_max": 53.59965133666992,
      "activations/layer5_attention_weight_min": -60.39542770385742,
      "activations/layer6_attention_weight_max": 45.168697357177734,
      "activations/layer6_attention_weight_min": -44.702919006347656,
      "activations/layer7_attention_weight_max": 94.8943862915039,
      "activations/layer7_attention_weight_min": -87.703369140625,
      "activations/layer8_attention_weight_max": 41.474647521972656,
      "activations/layer8_attention_weight_min": -39.976314544677734,
      "activations/layer9_attention_weight_max": 35.41746139526367,
      "activations/layer9_attention_weight_min": -33.76824951171875,
      "epoch": 11.13,
      "learning_rate": 7.900113636363636e-05,
      "loss": 2.77,
      "step": 191600
    },
    {
      "activations/layer0_attention_weight_max": 15.005692481994629,
      "activations/layer0_attention_weight_min": -12.773263931274414,
      "activations/layer10_attention_weight_max": 30.694164276123047,
      "activations/layer10_attention_weight_min": -31.30620574951172,
      "activations/layer11_attention_weight_max": 29.832441329956055,
      "activations/layer11_attention_weight_min": -31.51165771484375,
      "activations/layer12_attention_weight_max": 30.345121383666992,
      "activations/layer12_attention_weight_min": -30.702714920043945,
      "activations/layer13_attention_weight_max": 41.086692810058594,
      "activations/layer13_attention_weight_min": -33.966556549072266,
      "activations/layer14_attention_weight_max": 49.2735481262207,
      "activations/layer14_attention_weight_min": -40.09699249267578,
      "activations/layer15_attention_weight_max": 40.397300720214844,
      "activations/layer15_attention_weight_min": -32.7006721496582,
      "activations/layer16_attention_weight_max": 30.792444229125977,
      "activations/layer16_attention_weight_min": -28.614194869995117,
      "activations/layer17_attention_weight_max": 30.67585563659668,
      "activations/layer17_attention_weight_min": -25.85601043701172,
      "activations/layer18_attention_weight_max": 31.76506996154785,
      "activations/layer18_attention_weight_min": -23.200536727905273,
      "activations/layer19_attention_weight_max": 35.958065032958984,
      "activations/layer19_attention_weight_min": -30.938684463500977,
      "activations/layer1_attention_weight_max": 18.232040405273438,
      "activations/layer1_attention_weight_min": -16.89285659790039,
      "activations/layer20_attention_weight_max": 30.6873722076416,
      "activations/layer20_attention_weight_min": -23.986845016479492,
      "activations/layer21_attention_weight_max": 28.87965965270996,
      "activations/layer21_attention_weight_min": -22.107942581176758,
      "activations/layer22_attention_weight_max": 46.915283203125,
      "activations/layer22_attention_weight_min": -28.98337173461914,
      "activations/layer23_attention_weight_max": 33.25975036621094,
      "activations/layer23_attention_weight_min": -23.60554313659668,
      "activations/layer2_attention_weight_max": 36.107845306396484,
      "activations/layer2_attention_weight_min": -35.59581756591797,
      "activations/layer3_attention_weight_max": 99.67346954345703,
      "activations/layer3_attention_weight_min": -98.11032104492188,
      "activations/layer4_attention_weight_max": 112.6500244140625,
      "activations/layer4_attention_weight_min": -112.1600570678711,
      "activations/layer5_attention_weight_max": 54.46825408935547,
      "activations/layer5_attention_weight_min": -60.929893493652344,
      "activations/layer6_attention_weight_max": 45.9352912902832,
      "activations/layer6_attention_weight_min": -47.00606918334961,
      "activations/layer7_attention_weight_max": 89.10478973388672,
      "activations/layer7_attention_weight_min": -87.2108154296875,
      "activations/layer8_attention_weight_max": 38.078365325927734,
      "activations/layer8_attention_weight_min": -39.37046432495117,
      "activations/layer9_attention_weight_max": 30.164875030517578,
      "activations/layer9_attention_weight_min": -31.64496421813965,
      "epoch": 11.14,
      "learning_rate": 7.898219696969697e-05,
      "loss": 2.7584,
      "step": 191650
    },
    {
      "activations/layer0_attention_weight_max": 15.52585220336914,
      "activations/layer0_attention_weight_min": -12.968361854553223,
      "activations/layer10_attention_weight_max": 31.787918090820312,
      "activations/layer10_attention_weight_min": -31.552793502807617,
      "activations/layer11_attention_weight_max": 33.32587814331055,
      "activations/layer11_attention_weight_min": -30.726530075073242,
      "activations/layer12_attention_weight_max": 23.684009552001953,
      "activations/layer12_attention_weight_min": -24.801008224487305,
      "activations/layer13_attention_weight_max": 36.352169036865234,
      "activations/layer13_attention_weight_min": -35.246734619140625,
      "activations/layer14_attention_weight_max": 43.289451599121094,
      "activations/layer14_attention_weight_min": -37.264469146728516,
      "activations/layer15_attention_weight_max": 34.026615142822266,
      "activations/layer15_attention_weight_min": -31.148435592651367,
      "activations/layer16_attention_weight_max": 29.23265266418457,
      "activations/layer16_attention_weight_min": -30.621891021728516,
      "activations/layer17_attention_weight_max": 28.739810943603516,
      "activations/layer17_attention_weight_min": -25.115867614746094,
      "activations/layer18_attention_weight_max": 29.42042350769043,
      "activations/layer18_attention_weight_min": -21.98790168762207,
      "activations/layer19_attention_weight_max": 33.3177375793457,
      "activations/layer19_attention_weight_min": -28.812986373901367,
      "activations/layer1_attention_weight_max": 18.172012329101562,
      "activations/layer1_attention_weight_min": -14.082837104797363,
      "activations/layer20_attention_weight_max": 30.852949142456055,
      "activations/layer20_attention_weight_min": -24.49931526184082,
      "activations/layer21_attention_weight_max": 28.946239471435547,
      "activations/layer21_attention_weight_min": -22.746475219726562,
      "activations/layer22_attention_weight_max": 43.921043395996094,
      "activations/layer22_attention_weight_min": -31.565895080566406,
      "activations/layer23_attention_weight_max": 31.297882080078125,
      "activations/layer23_attention_weight_min": -23.10657501220703,
      "activations/layer2_attention_weight_max": 33.27691650390625,
      "activations/layer2_attention_weight_min": -33.142616271972656,
      "activations/layer3_attention_weight_max": 95.49283599853516,
      "activations/layer3_attention_weight_min": -96.42645263671875,
      "activations/layer4_attention_weight_max": 110.7927017211914,
      "activations/layer4_attention_weight_min": -111.9350357055664,
      "activations/layer5_attention_weight_max": 52.70801544189453,
      "activations/layer5_attention_weight_min": -60.89110565185547,
      "activations/layer6_attention_weight_max": 43.79472351074219,
      "activations/layer6_attention_weight_min": -49.66005325317383,
      "activations/layer7_attention_weight_max": 89.68072509765625,
      "activations/layer7_attention_weight_min": -91.13607025146484,
      "activations/layer8_attention_weight_max": 39.800025939941406,
      "activations/layer8_attention_weight_min": -39.80067443847656,
      "activations/layer9_attention_weight_max": 33.58901596069336,
      "activations/layer9_attention_weight_min": -31.864246368408203,
      "epoch": 11.14,
      "learning_rate": 7.896325757575757e-05,
      "loss": 2.7625,
      "step": 191700
    },
    {
      "activations/layer0_attention_weight_max": 16.1096134185791,
      "activations/layer0_attention_weight_min": -11.777098655700684,
      "activations/layer10_attention_weight_max": 32.565181732177734,
      "activations/layer10_attention_weight_min": -33.923492431640625,
      "activations/layer11_attention_weight_max": 32.327064514160156,
      "activations/layer11_attention_weight_min": -32.3807487487793,
      "activations/layer12_attention_weight_max": 23.342960357666016,
      "activations/layer12_attention_weight_min": -24.97918701171875,
      "activations/layer13_attention_weight_max": 38.54349899291992,
      "activations/layer13_attention_weight_min": -34.67328643798828,
      "activations/layer14_attention_weight_max": 41.31501770019531,
      "activations/layer14_attention_weight_min": -37.07157897949219,
      "activations/layer15_attention_weight_max": 41.93838882446289,
      "activations/layer15_attention_weight_min": -32.27729034423828,
      "activations/layer16_attention_weight_max": 29.07819366455078,
      "activations/layer16_attention_weight_min": -27.82532501220703,
      "activations/layer17_attention_weight_max": 29.720123291015625,
      "activations/layer17_attention_weight_min": -25.195817947387695,
      "activations/layer18_attention_weight_max": 30.933494567871094,
      "activations/layer18_attention_weight_min": -24.357357025146484,
      "activations/layer19_attention_weight_max": 36.89638137817383,
      "activations/layer19_attention_weight_min": -30.457738876342773,
      "activations/layer1_attention_weight_max": 16.770503997802734,
      "activations/layer1_attention_weight_min": -15.724650382995605,
      "activations/layer20_attention_weight_max": 31.4418888092041,
      "activations/layer20_attention_weight_min": -24.613859176635742,
      "activations/layer21_attention_weight_max": 32.572513580322266,
      "activations/layer21_attention_weight_min": -23.5437068939209,
      "activations/layer22_attention_weight_max": 44.70771789550781,
      "activations/layer22_attention_weight_min": -31.48312759399414,
      "activations/layer23_attention_weight_max": 32.963966369628906,
      "activations/layer23_attention_weight_min": -24.38875389099121,
      "activations/layer2_attention_weight_max": 36.63538360595703,
      "activations/layer2_attention_weight_min": -37.353485107421875,
      "activations/layer3_attention_weight_max": 99.65682220458984,
      "activations/layer3_attention_weight_min": -102.6891098022461,
      "activations/layer4_attention_weight_max": 114.08998107910156,
      "activations/layer4_attention_weight_min": -119.17729949951172,
      "activations/layer5_attention_weight_max": 52.198638916015625,
      "activations/layer5_attention_weight_min": -65.28284454345703,
      "activations/layer6_attention_weight_max": 44.17323684692383,
      "activations/layer6_attention_weight_min": -48.950313568115234,
      "activations/layer7_attention_weight_max": 89.17977142333984,
      "activations/layer7_attention_weight_min": -88.2490463256836,
      "activations/layer8_attention_weight_max": 40.35082244873047,
      "activations/layer8_attention_weight_min": -40.69990539550781,
      "activations/layer9_attention_weight_max": 31.9800968170166,
      "activations/layer9_attention_weight_min": -32.86336898803711,
      "epoch": 11.14,
      "learning_rate": 7.894431818181818e-05,
      "loss": 2.7614,
      "step": 191750
    },
    {
      "activations/layer0_attention_weight_max": 16.62911605834961,
      "activations/layer0_attention_weight_min": -12.658340454101562,
      "activations/layer10_attention_weight_max": 30.96603012084961,
      "activations/layer10_attention_weight_min": -31.934823989868164,
      "activations/layer11_attention_weight_max": 30.861745834350586,
      "activations/layer11_attention_weight_min": -31.23467254638672,
      "activations/layer12_attention_weight_max": 26.160255432128906,
      "activations/layer12_attention_weight_min": -27.6572208404541,
      "activations/layer13_attention_weight_max": 37.67195129394531,
      "activations/layer13_attention_weight_min": -32.990867614746094,
      "activations/layer14_attention_weight_max": 48.96529006958008,
      "activations/layer14_attention_weight_min": -39.538848876953125,
      "activations/layer15_attention_weight_max": 36.7069206237793,
      "activations/layer15_attention_weight_min": -31.951139450073242,
      "activations/layer16_attention_weight_max": 32.48054885864258,
      "activations/layer16_attention_weight_min": -27.768003463745117,
      "activations/layer17_attention_weight_max": 32.413089752197266,
      "activations/layer17_attention_weight_min": -26.582632064819336,
      "activations/layer18_attention_weight_max": 30.995548248291016,
      "activations/layer18_attention_weight_min": -23.08329963684082,
      "activations/layer19_attention_weight_max": 34.663795471191406,
      "activations/layer19_attention_weight_min": -31.024906158447266,
      "activations/layer1_attention_weight_max": 16.970365524291992,
      "activations/layer1_attention_weight_min": -15.334573745727539,
      "activations/layer20_attention_weight_max": 31.37040901184082,
      "activations/layer20_attention_weight_min": -24.9797306060791,
      "activations/layer21_attention_weight_max": 33.094581604003906,
      "activations/layer21_attention_weight_min": -24.331161499023438,
      "activations/layer22_attention_weight_max": 43.744224548339844,
      "activations/layer22_attention_weight_min": -30.54054069519043,
      "activations/layer23_attention_weight_max": 34.28423309326172,
      "activations/layer23_attention_weight_min": -24.522518157958984,
      "activations/layer2_attention_weight_max": 31.434873580932617,
      "activations/layer2_attention_weight_min": -31.85749053955078,
      "activations/layer3_attention_weight_max": 91.52238464355469,
      "activations/layer3_attention_weight_min": -96.52594757080078,
      "activations/layer4_attention_weight_max": 110.39066314697266,
      "activations/layer4_attention_weight_min": -108.58208465576172,
      "activations/layer5_attention_weight_max": 54.15468978881836,
      "activations/layer5_attention_weight_min": -61.970882415771484,
      "activations/layer6_attention_weight_max": 45.72437286376953,
      "activations/layer6_attention_weight_min": -50.87504959106445,
      "activations/layer7_attention_weight_max": 95.76675415039062,
      "activations/layer7_attention_weight_min": -88.98390197753906,
      "activations/layer8_attention_weight_max": 38.30837631225586,
      "activations/layer8_attention_weight_min": -39.843074798583984,
      "activations/layer9_attention_weight_max": 31.12509536743164,
      "activations/layer9_attention_weight_min": -32.53468704223633,
      "epoch": 11.14,
      "learning_rate": 7.892537878787877e-05,
      "loss": 2.772,
      "step": 191800
    },
    {
      "activations/layer0_attention_weight_max": 15.127043724060059,
      "activations/layer0_attention_weight_min": -12.379064559936523,
      "activations/layer10_attention_weight_max": 30.095909118652344,
      "activations/layer10_attention_weight_min": -34.019412994384766,
      "activations/layer11_attention_weight_max": 31.879209518432617,
      "activations/layer11_attention_weight_min": -29.527442932128906,
      "activations/layer12_attention_weight_max": 22.05852508544922,
      "activations/layer12_attention_weight_min": -23.909204483032227,
      "activations/layer13_attention_weight_max": 36.4577751159668,
      "activations/layer13_attention_weight_min": -32.753231048583984,
      "activations/layer14_attention_weight_max": 48.41377258300781,
      "activations/layer14_attention_weight_min": -41.94540023803711,
      "activations/layer15_attention_weight_max": 32.93046951293945,
      "activations/layer15_attention_weight_min": -31.697031021118164,
      "activations/layer16_attention_weight_max": 28.196834564208984,
      "activations/layer16_attention_weight_min": -28.890514373779297,
      "activations/layer17_attention_weight_max": 31.941965103149414,
      "activations/layer17_attention_weight_min": -25.936464309692383,
      "activations/layer18_attention_weight_max": 31.425338745117188,
      "activations/layer18_attention_weight_min": -22.81899070739746,
      "activations/layer19_attention_weight_max": 35.126731872558594,
      "activations/layer19_attention_weight_min": -31.6390323638916,
      "activations/layer1_attention_weight_max": 16.88423728942871,
      "activations/layer1_attention_weight_min": -13.763068199157715,
      "activations/layer20_attention_weight_max": 29.93832015991211,
      "activations/layer20_attention_weight_min": -24.834318161010742,
      "activations/layer21_attention_weight_max": 29.422325134277344,
      "activations/layer21_attention_weight_min": -23.502182006835938,
      "activations/layer22_attention_weight_max": 47.2824592590332,
      "activations/layer22_attention_weight_min": -29.33702850341797,
      "activations/layer23_attention_weight_max": 35.21820831298828,
      "activations/layer23_attention_weight_min": -24.591537475585938,
      "activations/layer2_attention_weight_max": 33.55241012573242,
      "activations/layer2_attention_weight_min": -33.423118591308594,
      "activations/layer3_attention_weight_max": 93.43942260742188,
      "activations/layer3_attention_weight_min": -94.91209411621094,
      "activations/layer4_attention_weight_max": 111.47492980957031,
      "activations/layer4_attention_weight_min": -110.62713623046875,
      "activations/layer5_attention_weight_max": 51.9541130065918,
      "activations/layer5_attention_weight_min": -62.9710693359375,
      "activations/layer6_attention_weight_max": 44.36979293823242,
      "activations/layer6_attention_weight_min": -48.32802200317383,
      "activations/layer7_attention_weight_max": 87.04911041259766,
      "activations/layer7_attention_weight_min": -90.94512939453125,
      "activations/layer8_attention_weight_max": 39.066444396972656,
      "activations/layer8_attention_weight_min": -40.31354904174805,
      "activations/layer9_attention_weight_max": 31.079360961914062,
      "activations/layer9_attention_weight_min": -32.92536163330078,
      "epoch": 11.15,
      "learning_rate": 7.890643939393938e-05,
      "loss": 2.7844,
      "step": 191850
    },
    {
      "activations/layer0_attention_weight_max": 15.573439598083496,
      "activations/layer0_attention_weight_min": -13.041752815246582,
      "activations/layer10_attention_weight_max": 31.957406997680664,
      "activations/layer10_attention_weight_min": -33.562862396240234,
      "activations/layer11_attention_weight_max": 32.0137939453125,
      "activations/layer11_attention_weight_min": -31.51708221435547,
      "activations/layer12_attention_weight_max": 24.002344131469727,
      "activations/layer12_attention_weight_min": -24.728116989135742,
      "activations/layer13_attention_weight_max": 39.07804870605469,
      "activations/layer13_attention_weight_min": -32.55064392089844,
      "activations/layer14_attention_weight_max": 46.41756820678711,
      "activations/layer14_attention_weight_min": -44.12209701538086,
      "activations/layer15_attention_weight_max": 34.65703582763672,
      "activations/layer15_attention_weight_min": -31.568933486938477,
      "activations/layer16_attention_weight_max": 33.069149017333984,
      "activations/layer16_attention_weight_min": -30.366348266601562,
      "activations/layer17_attention_weight_max": 31.45071792602539,
      "activations/layer17_attention_weight_min": -24.65277671813965,
      "activations/layer18_attention_weight_max": 32.77923583984375,
      "activations/layer18_attention_weight_min": -25.511863708496094,
      "activations/layer19_attention_weight_max": 34.30254364013672,
      "activations/layer19_attention_weight_min": -30.363325119018555,
      "activations/layer1_attention_weight_max": 16.453691482543945,
      "activations/layer1_attention_weight_min": -14.462459564208984,
      "activations/layer20_attention_weight_max": 30.569358825683594,
      "activations/layer20_attention_weight_min": -25.092105865478516,
      "activations/layer21_attention_weight_max": 27.848691940307617,
      "activations/layer21_attention_weight_min": -23.700164794921875,
      "activations/layer22_attention_weight_max": 52.451904296875,
      "activations/layer22_attention_weight_min": -30.41660499572754,
      "activations/layer23_attention_weight_max": 34.07667922973633,
      "activations/layer23_attention_weight_min": -23.629718780517578,
      "activations/layer2_attention_weight_max": 33.09670639038086,
      "activations/layer2_attention_weight_min": -32.22669219970703,
      "activations/layer3_attention_weight_max": 93.36724090576172,
      "activations/layer3_attention_weight_min": -98.9393081665039,
      "activations/layer4_attention_weight_max": 112.1330337524414,
      "activations/layer4_attention_weight_min": -108.21646881103516,
      "activations/layer5_attention_weight_max": 51.675140380859375,
      "activations/layer5_attention_weight_min": -60.37028503417969,
      "activations/layer6_attention_weight_max": 43.501102447509766,
      "activations/layer6_attention_weight_min": -47.096778869628906,
      "activations/layer7_attention_weight_max": 84.0469741821289,
      "activations/layer7_attention_weight_min": -89.45053100585938,
      "activations/layer8_attention_weight_max": 40.285953521728516,
      "activations/layer8_attention_weight_min": -44.91127395629883,
      "activations/layer9_attention_weight_max": 31.938220977783203,
      "activations/layer9_attention_weight_min": -35.000213623046875,
      "epoch": 11.15,
      "learning_rate": 7.88875e-05,
      "loss": 2.7793,
      "step": 191900
    },
    {
      "activations/layer0_attention_weight_max": 15.074913024902344,
      "activations/layer0_attention_weight_min": -12.811197280883789,
      "activations/layer10_attention_weight_max": 36.70700454711914,
      "activations/layer10_attention_weight_min": -34.96269226074219,
      "activations/layer11_attention_weight_max": 37.47597885131836,
      "activations/layer11_attention_weight_min": -34.690956115722656,
      "activations/layer12_attention_weight_max": 27.24290657043457,
      "activations/layer12_attention_weight_min": -25.087730407714844,
      "activations/layer13_attention_weight_max": 45.764732360839844,
      "activations/layer13_attention_weight_min": -33.21464538574219,
      "activations/layer14_attention_weight_max": 42.43904113769531,
      "activations/layer14_attention_weight_min": -38.044029235839844,
      "activations/layer15_attention_weight_max": 43.17877197265625,
      "activations/layer15_attention_weight_min": -32.94707107543945,
      "activations/layer16_attention_weight_max": 30.96424102783203,
      "activations/layer16_attention_weight_min": -27.011953353881836,
      "activations/layer17_attention_weight_max": 30.60386848449707,
      "activations/layer17_attention_weight_min": -28.875770568847656,
      "activations/layer18_attention_weight_max": 31.292348861694336,
      "activations/layer18_attention_weight_min": -25.630542755126953,
      "activations/layer19_attention_weight_max": 35.60380172729492,
      "activations/layer19_attention_weight_min": -33.36028289794922,
      "activations/layer1_attention_weight_max": 17.101259231567383,
      "activations/layer1_attention_weight_min": -15.05360221862793,
      "activations/layer20_attention_weight_max": 30.038047790527344,
      "activations/layer20_attention_weight_min": -28.55192756652832,
      "activations/layer21_attention_weight_max": 28.651077270507812,
      "activations/layer21_attention_weight_min": -25.456533432006836,
      "activations/layer22_attention_weight_max": 42.95808029174805,
      "activations/layer22_attention_weight_min": -32.501338958740234,
      "activations/layer23_attention_weight_max": 36.99162673950195,
      "activations/layer23_attention_weight_min": -26.360830307006836,
      "activations/layer2_attention_weight_max": 36.306541442871094,
      "activations/layer2_attention_weight_min": -35.79135513305664,
      "activations/layer3_attention_weight_max": 99.26630401611328,
      "activations/layer3_attention_weight_min": -101.96255493164062,
      "activations/layer4_attention_weight_max": 116.55130767822266,
      "activations/layer4_attention_weight_min": -112.5698471069336,
      "activations/layer5_attention_weight_max": 53.9593505859375,
      "activations/layer5_attention_weight_min": -60.240386962890625,
      "activations/layer6_attention_weight_max": 45.41139602661133,
      "activations/layer6_attention_weight_min": -48.11429214477539,
      "activations/layer7_attention_weight_max": 103.5376968383789,
      "activations/layer7_attention_weight_min": -97.31415557861328,
      "activations/layer8_attention_weight_max": 45.57493209838867,
      "activations/layer8_attention_weight_min": -44.14540481567383,
      "activations/layer9_attention_weight_max": 38.18882369995117,
      "activations/layer9_attention_weight_min": -35.86387252807617,
      "epoch": 11.15,
      "learning_rate": 7.886856060606059e-05,
      "loss": 2.7719,
      "step": 191950
    },
    {
      "activations/layer0_attention_weight_max": 15.882561683654785,
      "activations/layer0_attention_weight_min": -12.448793411254883,
      "activations/layer10_attention_weight_max": 35.367164611816406,
      "activations/layer10_attention_weight_min": -33.81266403198242,
      "activations/layer11_attention_weight_max": 35.31074523925781,
      "activations/layer11_attention_weight_min": -33.31378173828125,
      "activations/layer12_attention_weight_max": 25.867856979370117,
      "activations/layer12_attention_weight_min": -26.27977752685547,
      "activations/layer13_attention_weight_max": 45.48979568481445,
      "activations/layer13_attention_weight_min": -35.340126037597656,
      "activations/layer14_attention_weight_max": 46.423587799072266,
      "activations/layer14_attention_weight_min": -36.62334442138672,
      "activations/layer15_attention_weight_max": 40.45749282836914,
      "activations/layer15_attention_weight_min": -34.717376708984375,
      "activations/layer16_attention_weight_max": 34.685970306396484,
      "activations/layer16_attention_weight_min": -29.121238708496094,
      "activations/layer17_attention_weight_max": 34.67334747314453,
      "activations/layer17_attention_weight_min": -27.979259490966797,
      "activations/layer18_attention_weight_max": 35.55295181274414,
      "activations/layer18_attention_weight_min": -26.7853946685791,
      "activations/layer19_attention_weight_max": 43.05997848510742,
      "activations/layer19_attention_weight_min": -33.41008377075195,
      "activations/layer1_attention_weight_max": 16.819684982299805,
      "activations/layer1_attention_weight_min": -16.099416732788086,
      "activations/layer20_attention_weight_max": 36.82478332519531,
      "activations/layer20_attention_weight_min": -25.514873504638672,
      "activations/layer21_attention_weight_max": 35.42467498779297,
      "activations/layer21_attention_weight_min": -24.729082107543945,
      "activations/layer22_attention_weight_max": 53.92584991455078,
      "activations/layer22_attention_weight_min": -32.082611083984375,
      "activations/layer23_attention_weight_max": 42.81373596191406,
      "activations/layer23_attention_weight_min": -23.75619125366211,
      "activations/layer2_attention_weight_max": 33.841407775878906,
      "activations/layer2_attention_weight_min": -35.903541564941406,
      "activations/layer3_attention_weight_max": 104.75874328613281,
      "activations/layer3_attention_weight_min": -102.23381805419922,
      "activations/layer4_attention_weight_max": 118.6145248413086,
      "activations/layer4_attention_weight_min": -114.28755950927734,
      "activations/layer5_attention_weight_max": 56.450374603271484,
      "activations/layer5_attention_weight_min": -63.72614288330078,
      "activations/layer6_attention_weight_max": 48.368751525878906,
      "activations/layer6_attention_weight_min": -51.71751022338867,
      "activations/layer7_attention_weight_max": 103.01579284667969,
      "activations/layer7_attention_weight_min": -92.68304443359375,
      "activations/layer8_attention_weight_max": 40.50885009765625,
      "activations/layer8_attention_weight_min": -42.87052536010742,
      "activations/layer9_attention_weight_max": 34.91041946411133,
      "activations/layer9_attention_weight_min": -35.697689056396484,
      "epoch": 11.16,
      "learning_rate": 7.88496212121212e-05,
      "loss": 2.7651,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.5587,
      "eval_samples_per_second": 501.712,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.5587,
      "eval_openwebtext_samples_per_second": 501.712,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 1.9949,
      "eval_wikitext_samples_per_second": 228.58,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5987,
      "eval_lambada_samples_per_second": 507.254,
      "step": 192000
    },
    {
      "activations/layer0_attention_weight_max": 15.7512845993042,
      "activations/layer0_attention_weight_min": -13.263773918151855,
      "activations/layer10_attention_weight_max": 36.00362777709961,
      "activations/layer10_attention_weight_min": -34.03786087036133,
      "activations/layer11_attention_weight_max": 35.25563049316406,
      "activations/layer11_attention_weight_min": -34.99643325805664,
      "activations/layer12_attention_weight_max": 27.237863540649414,
      "activations/layer12_attention_weight_min": -27.083459854125977,
      "activations/layer13_attention_weight_max": 44.21436309814453,
      "activations/layer13_attention_weight_min": -36.57633590698242,
      "activations/layer14_attention_weight_max": 50.73526382446289,
      "activations/layer14_attention_weight_min": -39.32240295410156,
      "activations/layer15_attention_weight_max": 44.07246780395508,
      "activations/layer15_attention_weight_min": -34.63634490966797,
      "activations/layer16_attention_weight_max": 35.913597106933594,
      "activations/layer16_attention_weight_min": -27.167360305786133,
      "activations/layer17_attention_weight_max": 41.57072830200195,
      "activations/layer17_attention_weight_min": -27.313589096069336,
      "activations/layer18_attention_weight_max": 35.96943283081055,
      "activations/layer18_attention_weight_min": -23.52029037475586,
      "activations/layer19_attention_weight_max": 37.89121627807617,
      "activations/layer19_attention_weight_min": -30.709707260131836,
      "activations/layer1_attention_weight_max": 17.927675247192383,
      "activations/layer1_attention_weight_min": -15.229700088500977,
      "activations/layer20_attention_weight_max": 29.733158111572266,
      "activations/layer20_attention_weight_min": -24.478092193603516,
      "activations/layer21_attention_weight_max": 29.431852340698242,
      "activations/layer21_attention_weight_min": -25.61040496826172,
      "activations/layer22_attention_weight_max": 42.679351806640625,
      "activations/layer22_attention_weight_min": -32.18954086303711,
      "activations/layer23_attention_weight_max": 37.90407943725586,
      "activations/layer23_attention_weight_min": -28.11993408203125,
      "activations/layer2_attention_weight_max": 33.49714660644531,
      "activations/layer2_attention_weight_min": -34.657875061035156,
      "activations/layer3_attention_weight_max": 101.0779037475586,
      "activations/layer3_attention_weight_min": -96.88238525390625,
      "activations/layer4_attention_weight_max": 118.8958740234375,
      "activations/layer4_attention_weight_min": -116.41228485107422,
      "activations/layer5_attention_weight_max": 56.58887481689453,
      "activations/layer5_attention_weight_min": -60.3416748046875,
      "activations/layer6_attention_weight_max": 46.02605056762695,
      "activations/layer6_attention_weight_min": -54.11713790893555,
      "activations/layer7_attention_weight_max": 97.4397964477539,
      "activations/layer7_attention_weight_min": -90.99783325195312,
      "activations/layer8_attention_weight_max": 43.275577545166016,
      "activations/layer8_attention_weight_min": -43.92557144165039,
      "activations/layer9_attention_weight_max": 36.3054084777832,
      "activations/layer9_attention_weight_min": -35.0981559753418,
      "epoch": 11.16,
      "learning_rate": 7.883068181818182e-05,
      "loss": 2.7658,
      "step": 192050
    },
    {
      "activations/layer0_attention_weight_max": 14.498382568359375,
      "activations/layer0_attention_weight_min": -13.360352516174316,
      "activations/layer10_attention_weight_max": 30.234359741210938,
      "activations/layer10_attention_weight_min": -34.2652473449707,
      "activations/layer11_attention_weight_max": 32.68769073486328,
      "activations/layer11_attention_weight_min": -32.39268493652344,
      "activations/layer12_attention_weight_max": 23.669708251953125,
      "activations/layer12_attention_weight_min": -25.684324264526367,
      "activations/layer13_attention_weight_max": 37.567405700683594,
      "activations/layer13_attention_weight_min": -35.38678741455078,
      "activations/layer14_attention_weight_max": 42.47597885131836,
      "activations/layer14_attention_weight_min": -40.08141326904297,
      "activations/layer15_attention_weight_max": 34.828857421875,
      "activations/layer15_attention_weight_min": -32.91504669189453,
      "activations/layer16_attention_weight_max": 28.324350357055664,
      "activations/layer16_attention_weight_min": -31.385746002197266,
      "activations/layer17_attention_weight_max": 30.75645637512207,
      "activations/layer17_attention_weight_min": -27.637187957763672,
      "activations/layer18_attention_weight_max": 31.7496280670166,
      "activations/layer18_attention_weight_min": -23.24074363708496,
      "activations/layer19_attention_weight_max": 32.34736251831055,
      "activations/layer19_attention_weight_min": -32.887298583984375,
      "activations/layer1_attention_weight_max": 16.26618194580078,
      "activations/layer1_attention_weight_min": -16.147403717041016,
      "activations/layer20_attention_weight_max": 30.03412437438965,
      "activations/layer20_attention_weight_min": -25.505813598632812,
      "activations/layer21_attention_weight_max": 28.720993041992188,
      "activations/layer21_attention_weight_min": -24.84914207458496,
      "activations/layer22_attention_weight_max": 45.27436065673828,
      "activations/layer22_attention_weight_min": -29.91684913635254,
      "activations/layer23_attention_weight_max": 35.63050842285156,
      "activations/layer23_attention_weight_min": -25.570480346679688,
      "activations/layer2_attention_weight_max": 32.69480514526367,
      "activations/layer2_attention_weight_min": -33.35073471069336,
      "activations/layer3_attention_weight_max": 93.74049377441406,
      "activations/layer3_attention_weight_min": -97.9369888305664,
      "activations/layer4_attention_weight_max": 117.57301330566406,
      "activations/layer4_attention_weight_min": -111.024169921875,
      "activations/layer5_attention_weight_max": 53.49414825439453,
      "activations/layer5_attention_weight_min": -63.13519287109375,
      "activations/layer6_attention_weight_max": 48.7484245300293,
      "activations/layer6_attention_weight_min": -48.75624084472656,
      "activations/layer7_attention_weight_max": 91.2823257446289,
      "activations/layer7_attention_weight_min": -90.60600280761719,
      "activations/layer8_attention_weight_max": 40.251495361328125,
      "activations/layer8_attention_weight_min": -39.98601150512695,
      "activations/layer9_attention_weight_max": 34.13483428955078,
      "activations/layer9_attention_weight_min": -32.596866607666016,
      "epoch": 11.16,
      "learning_rate": 7.881174242424242e-05,
      "loss": 2.7655,
      "step": 192100
    },
    {
      "activations/layer0_attention_weight_max": 16.02556610107422,
      "activations/layer0_attention_weight_min": -12.532560348510742,
      "activations/layer10_attention_weight_max": 34.1511344909668,
      "activations/layer10_attention_weight_min": -32.637481689453125,
      "activations/layer11_attention_weight_max": 33.30793762207031,
      "activations/layer11_attention_weight_min": -31.716276168823242,
      "activations/layer12_attention_weight_max": 31.744382858276367,
      "activations/layer12_attention_weight_min": -24.28567886352539,
      "activations/layer13_attention_weight_max": 34.517311096191406,
      "activations/layer13_attention_weight_min": -31.6965274810791,
      "activations/layer14_attention_weight_max": 40.67914962768555,
      "activations/layer14_attention_weight_min": -35.87420654296875,
      "activations/layer15_attention_weight_max": 31.929723739624023,
      "activations/layer15_attention_weight_min": -29.405683517456055,
      "activations/layer16_attention_weight_max": 31.794742584228516,
      "activations/layer16_attention_weight_min": -26.530412673950195,
      "activations/layer17_attention_weight_max": 29.043025970458984,
      "activations/layer17_attention_weight_min": -26.965688705444336,
      "activations/layer18_attention_weight_max": 31.333229064941406,
      "activations/layer18_attention_weight_min": -23.739051818847656,
      "activations/layer19_attention_weight_max": 34.787166595458984,
      "activations/layer19_attention_weight_min": -31.852354049682617,
      "activations/layer1_attention_weight_max": 16.445537567138672,
      "activations/layer1_attention_weight_min": -16.12383460998535,
      "activations/layer20_attention_weight_max": 28.70368003845215,
      "activations/layer20_attention_weight_min": -24.89250946044922,
      "activations/layer21_attention_weight_max": 29.05588722229004,
      "activations/layer21_attention_weight_min": -24.183353424072266,
      "activations/layer22_attention_weight_max": 42.54831314086914,
      "activations/layer22_attention_weight_min": -30.051668167114258,
      "activations/layer23_attention_weight_max": 33.90061950683594,
      "activations/layer23_attention_weight_min": -26.39071273803711,
      "activations/layer2_attention_weight_max": 33.9304084777832,
      "activations/layer2_attention_weight_min": -31.583213806152344,
      "activations/layer3_attention_weight_max": 95.84275817871094,
      "activations/layer3_attention_weight_min": -94.22930908203125,
      "activations/layer4_attention_weight_max": 111.73412322998047,
      "activations/layer4_attention_weight_min": -112.9466552734375,
      "activations/layer5_attention_weight_max": 52.70262145996094,
      "activations/layer5_attention_weight_min": -62.225807189941406,
      "activations/layer6_attention_weight_max": 44.775028228759766,
      "activations/layer6_attention_weight_min": -51.23348617553711,
      "activations/layer7_attention_weight_max": 95.82941436767578,
      "activations/layer7_attention_weight_min": -97.61479187011719,
      "activations/layer8_attention_weight_max": 42.23659896850586,
      "activations/layer8_attention_weight_min": -44.333255767822266,
      "activations/layer9_attention_weight_max": 33.042171478271484,
      "activations/layer9_attention_weight_min": -34.340606689453125,
      "epoch": 11.17,
      "learning_rate": 7.879280303030302e-05,
      "loss": 2.7796,
      "step": 192150
    },
    {
      "activations/layer0_attention_weight_max": 15.984512329101562,
      "activations/layer0_attention_weight_min": -13.559989929199219,
      "activations/layer10_attention_weight_max": 34.74897766113281,
      "activations/layer10_attention_weight_min": -35.081153869628906,
      "activations/layer11_attention_weight_max": 35.185935974121094,
      "activations/layer11_attention_weight_min": -32.71186065673828,
      "activations/layer12_attention_weight_max": 24.48999786376953,
      "activations/layer12_attention_weight_min": -25.901023864746094,
      "activations/layer13_attention_weight_max": 38.09540939331055,
      "activations/layer13_attention_weight_min": -33.43622970581055,
      "activations/layer14_attention_weight_max": 41.70061111450195,
      "activations/layer14_attention_weight_min": -37.46436309814453,
      "activations/layer15_attention_weight_max": 35.87876510620117,
      "activations/layer15_attention_weight_min": -30.160917282104492,
      "activations/layer16_attention_weight_max": 30.809907913208008,
      "activations/layer16_attention_weight_min": -26.318984985351562,
      "activations/layer17_attention_weight_max": 31.25213050842285,
      "activations/layer17_attention_weight_min": -24.34230613708496,
      "activations/layer18_attention_weight_max": 30.96337890625,
      "activations/layer18_attention_weight_min": -22.783998489379883,
      "activations/layer19_attention_weight_max": 35.15565872192383,
      "activations/layer19_attention_weight_min": -29.274572372436523,
      "activations/layer1_attention_weight_max": 16.32379150390625,
      "activations/layer1_attention_weight_min": -15.943826675415039,
      "activations/layer20_attention_weight_max": 30.803863525390625,
      "activations/layer20_attention_weight_min": -25.973051071166992,
      "activations/layer21_attention_weight_max": 29.426855087280273,
      "activations/layer21_attention_weight_min": -24.99994468688965,
      "activations/layer22_attention_weight_max": 44.2099609375,
      "activations/layer22_attention_weight_min": -30.04804229736328,
      "activations/layer23_attention_weight_max": 40.541419982910156,
      "activations/layer23_attention_weight_min": -24.457599639892578,
      "activations/layer2_attention_weight_max": 34.77103042602539,
      "activations/layer2_attention_weight_min": -34.18123245239258,
      "activations/layer3_attention_weight_max": 101.07855987548828,
      "activations/layer3_attention_weight_min": -107.5624771118164,
      "activations/layer4_attention_weight_max": 118.98479461669922,
      "activations/layer4_attention_weight_min": -118.73783111572266,
      "activations/layer5_attention_weight_max": 56.41280746459961,
      "activations/layer5_attention_weight_min": -61.73682403564453,
      "activations/layer6_attention_weight_max": 49.07929229736328,
      "activations/layer6_attention_weight_min": -48.741233825683594,
      "activations/layer7_attention_weight_max": 98.24624633789062,
      "activations/layer7_attention_weight_min": -92.93387603759766,
      "activations/layer8_attention_weight_max": 42.5951042175293,
      "activations/layer8_attention_weight_min": -44.65507888793945,
      "activations/layer9_attention_weight_max": 34.31846237182617,
      "activations/layer9_attention_weight_min": -36.27627944946289,
      "epoch": 11.17,
      "learning_rate": 7.877386363636364e-05,
      "loss": 2.787,
      "step": 192200
    },
    {
      "activations/layer0_attention_weight_max": 16.089826583862305,
      "activations/layer0_attention_weight_min": -13.864264488220215,
      "activations/layer10_attention_weight_max": 33.46556854248047,
      "activations/layer10_attention_weight_min": -32.78287124633789,
      "activations/layer11_attention_weight_max": 30.965444564819336,
      "activations/layer11_attention_weight_min": -31.277135848999023,
      "activations/layer12_attention_weight_max": 27.40449333190918,
      "activations/layer12_attention_weight_min": -25.406082153320312,
      "activations/layer13_attention_weight_max": 40.680259704589844,
      "activations/layer13_attention_weight_min": -35.11808776855469,
      "activations/layer14_attention_weight_max": 52.138912200927734,
      "activations/layer14_attention_weight_min": -40.55492401123047,
      "activations/layer15_attention_weight_max": 35.81464767456055,
      "activations/layer15_attention_weight_min": -33.19456481933594,
      "activations/layer16_attention_weight_max": 31.230676651000977,
      "activations/layer16_attention_weight_min": -28.664474487304688,
      "activations/layer17_attention_weight_max": 30.31268310546875,
      "activations/layer17_attention_weight_min": -25.6026611328125,
      "activations/layer18_attention_weight_max": 34.14228820800781,
      "activations/layer18_attention_weight_min": -24.937589645385742,
      "activations/layer19_attention_weight_max": 38.2083625793457,
      "activations/layer19_attention_weight_min": -31.600339889526367,
      "activations/layer1_attention_weight_max": 16.78089141845703,
      "activations/layer1_attention_weight_min": -15.321061134338379,
      "activations/layer20_attention_weight_max": 32.623069763183594,
      "activations/layer20_attention_weight_min": -26.54214096069336,
      "activations/layer21_attention_weight_max": 32.34259796142578,
      "activations/layer21_attention_weight_min": -25.71158218383789,
      "activations/layer22_attention_weight_max": 44.938934326171875,
      "activations/layer22_attention_weight_min": -31.18231201171875,
      "activations/layer23_attention_weight_max": 39.72662353515625,
      "activations/layer23_attention_weight_min": -26.267574310302734,
      "activations/layer2_attention_weight_max": 32.29542541503906,
      "activations/layer2_attention_weight_min": -31.60589599609375,
      "activations/layer3_attention_weight_max": 93.4456787109375,
      "activations/layer3_attention_weight_min": -94.69548034667969,
      "activations/layer4_attention_weight_max": 110.42189025878906,
      "activations/layer4_attention_weight_min": -112.15478515625,
      "activations/layer5_attention_weight_max": 51.96776580810547,
      "activations/layer5_attention_weight_min": -62.43754196166992,
      "activations/layer6_attention_weight_max": 45.22651672363281,
      "activations/layer6_attention_weight_min": -47.234676361083984,
      "activations/layer7_attention_weight_max": 93.65216064453125,
      "activations/layer7_attention_weight_min": -94.10809326171875,
      "activations/layer8_attention_weight_max": 41.10908508300781,
      "activations/layer8_attention_weight_min": -43.68802261352539,
      "activations/layer9_attention_weight_max": 34.054866790771484,
      "activations/layer9_attention_weight_min": -34.49313735961914,
      "epoch": 11.17,
      "learning_rate": 7.875530303030303e-05,
      "loss": 2.7803,
      "step": 192250
    },
    {
      "activations/layer0_attention_weight_max": 15.529906272888184,
      "activations/layer0_attention_weight_min": -11.990623474121094,
      "activations/layer10_attention_weight_max": 45.8978385925293,
      "activations/layer10_attention_weight_min": -40.92063522338867,
      "activations/layer11_attention_weight_max": 47.48650360107422,
      "activations/layer11_attention_weight_min": -42.17127227783203,
      "activations/layer12_attention_weight_max": 31.933626174926758,
      "activations/layer12_attention_weight_min": -29.926589965820312,
      "activations/layer13_attention_weight_max": 48.47977066040039,
      "activations/layer13_attention_weight_min": -37.991233825683594,
      "activations/layer14_attention_weight_max": 57.74309158325195,
      "activations/layer14_attention_weight_min": -42.9860954284668,
      "activations/layer15_attention_weight_max": 63.24126434326172,
      "activations/layer15_attention_weight_min": -37.843971252441406,
      "activations/layer16_attention_weight_max": 41.83603286743164,
      "activations/layer16_attention_weight_min": -31.453994750976562,
      "activations/layer17_attention_weight_max": 32.417938232421875,
      "activations/layer17_attention_weight_min": -31.262678146362305,
      "activations/layer18_attention_weight_max": 36.64914321899414,
      "activations/layer18_attention_weight_min": -27.20069694519043,
      "activations/layer19_attention_weight_max": 46.39229965209961,
      "activations/layer19_attention_weight_min": -37.39055252075195,
      "activations/layer1_attention_weight_max": 17.680875778198242,
      "activations/layer1_attention_weight_min": -14.501399993896484,
      "activations/layer20_attention_weight_max": 37.00782775878906,
      "activations/layer20_attention_weight_min": -27.939184188842773,
      "activations/layer21_attention_weight_max": 32.77436065673828,
      "activations/layer21_attention_weight_min": -24.661108016967773,
      "activations/layer22_attention_weight_max": 54.51071548461914,
      "activations/layer22_attention_weight_min": -30.43722152709961,
      "activations/layer23_attention_weight_max": 36.08772277832031,
      "activations/layer23_attention_weight_min": -28.028104782104492,
      "activations/layer2_attention_weight_max": 36.60934829711914,
      "activations/layer2_attention_weight_min": -36.80009460449219,
      "activations/layer3_attention_weight_max": 111.39248657226562,
      "activations/layer3_attention_weight_min": -108.4991226196289,
      "activations/layer4_attention_weight_max": 133.6755828857422,
      "activations/layer4_attention_weight_min": -123.17229461669922,
      "activations/layer5_attention_weight_max": 58.53062438964844,
      "activations/layer5_attention_weight_min": -63.63066101074219,
      "activations/layer6_attention_weight_max": 47.436180114746094,
      "activations/layer6_attention_weight_min": -49.08793640136719,
      "activations/layer7_attention_weight_max": 109.9300765991211,
      "activations/layer7_attention_weight_min": -105.19664764404297,
      "activations/layer8_attention_weight_max": 53.686187744140625,
      "activations/layer8_attention_weight_min": -51.55192947387695,
      "activations/layer9_attention_weight_max": 42.82882308959961,
      "activations/layer9_attention_weight_min": -40.149879455566406,
      "epoch": 11.17,
      "learning_rate": 7.873636363636363e-05,
      "loss": 2.7797,
      "step": 192300
    },
    {
      "activations/layer0_attention_weight_max": 16.105527877807617,
      "activations/layer0_attention_weight_min": -12.360552787780762,
      "activations/layer10_attention_weight_max": 33.87445831298828,
      "activations/layer10_attention_weight_min": -34.165225982666016,
      "activations/layer11_attention_weight_max": 33.25362014770508,
      "activations/layer11_attention_weight_min": -31.20254135131836,
      "activations/layer12_attention_weight_max": 27.030134201049805,
      "activations/layer12_attention_weight_min": -25.033159255981445,
      "activations/layer13_attention_weight_max": 39.01265335083008,
      "activations/layer13_attention_weight_min": -33.26032257080078,
      "activations/layer14_attention_weight_max": 47.524932861328125,
      "activations/layer14_attention_weight_min": -37.54207992553711,
      "activations/layer15_attention_weight_max": 36.66294860839844,
      "activations/layer15_attention_weight_min": -32.5653190612793,
      "activations/layer16_attention_weight_max": 34.59344482421875,
      "activations/layer16_attention_weight_min": -26.390005111694336,
      "activations/layer17_attention_weight_max": 31.951440811157227,
      "activations/layer17_attention_weight_min": -27.36101531982422,
      "activations/layer18_attention_weight_max": 30.984539031982422,
      "activations/layer18_attention_weight_min": -23.784738540649414,
      "activations/layer19_attention_weight_max": 34.99915313720703,
      "activations/layer19_attention_weight_min": -33.05170822143555,
      "activations/layer1_attention_weight_max": 16.315141677856445,
      "activations/layer1_attention_weight_min": -13.982138633728027,
      "activations/layer20_attention_weight_max": 30.10863494873047,
      "activations/layer20_attention_weight_min": -25.278886795043945,
      "activations/layer21_attention_weight_max": 29.230710983276367,
      "activations/layer21_attention_weight_min": -24.1804256439209,
      "activations/layer22_attention_weight_max": 45.69141387939453,
      "activations/layer22_attention_weight_min": -34.123844146728516,
      "activations/layer23_attention_weight_max": 39.131534576416016,
      "activations/layer23_attention_weight_min": -24.825599670410156,
      "activations/layer2_attention_weight_max": 33.93236541748047,
      "activations/layer2_attention_weight_min": -32.59313201904297,
      "activations/layer3_attention_weight_max": 100.15448760986328,
      "activations/layer3_attention_weight_min": -95.59770965576172,
      "activations/layer4_attention_weight_max": 116.46357727050781,
      "activations/layer4_attention_weight_min": -116.62503814697266,
      "activations/layer5_attention_weight_max": 54.10671615600586,
      "activations/layer5_attention_weight_min": -62.6816520690918,
      "activations/layer6_attention_weight_max": 44.574012756347656,
      "activations/layer6_attention_weight_min": -48.48915100097656,
      "activations/layer7_attention_weight_max": 86.01004028320312,
      "activations/layer7_attention_weight_min": -89.58163452148438,
      "activations/layer8_attention_weight_max": 41.80595397949219,
      "activations/layer8_attention_weight_min": -40.861053466796875,
      "activations/layer9_attention_weight_max": 34.66620635986328,
      "activations/layer9_attention_weight_min": -33.66343688964844,
      "epoch": 11.18,
      "learning_rate": 7.871742424242424e-05,
      "loss": 2.77,
      "step": 192350
    },
    {
      "activations/layer0_attention_weight_max": 15.22755241394043,
      "activations/layer0_attention_weight_min": -12.659565925598145,
      "activations/layer10_attention_weight_max": 32.827632904052734,
      "activations/layer10_attention_weight_min": -33.86127471923828,
      "activations/layer11_attention_weight_max": 31.442466735839844,
      "activations/layer11_attention_weight_min": -32.08270263671875,
      "activations/layer12_attention_weight_max": 24.872575759887695,
      "activations/layer12_attention_weight_min": -25.059162139892578,
      "activations/layer13_attention_weight_max": 38.65737533569336,
      "activations/layer13_attention_weight_min": -32.364723205566406,
      "activations/layer14_attention_weight_max": 41.98455810546875,
      "activations/layer14_attention_weight_min": -35.83636474609375,
      "activations/layer15_attention_weight_max": 36.42363357543945,
      "activations/layer15_attention_weight_min": -32.49746322631836,
      "activations/layer16_attention_weight_max": 31.020124435424805,
      "activations/layer16_attention_weight_min": -27.887413024902344,
      "activations/layer17_attention_weight_max": 29.780622482299805,
      "activations/layer17_attention_weight_min": -28.561574935913086,
      "activations/layer18_attention_weight_max": 31.19996452331543,
      "activations/layer18_attention_weight_min": -25.790618896484375,
      "activations/layer19_attention_weight_max": 34.979305267333984,
      "activations/layer19_attention_weight_min": -31.680130004882812,
      "activations/layer1_attention_weight_max": 16.92033576965332,
      "activations/layer1_attention_weight_min": -17.604341506958008,
      "activations/layer20_attention_weight_max": 31.9978084564209,
      "activations/layer20_attention_weight_min": -24.839982986450195,
      "activations/layer21_attention_weight_max": 32.00755310058594,
      "activations/layer21_attention_weight_min": -25.208337783813477,
      "activations/layer22_attention_weight_max": 48.25263977050781,
      "activations/layer22_attention_weight_min": -35.16656494140625,
      "activations/layer23_attention_weight_max": 37.59508514404297,
      "activations/layer23_attention_weight_min": -24.89192771911621,
      "activations/layer2_attention_weight_max": 33.237098693847656,
      "activations/layer2_attention_weight_min": -33.59032440185547,
      "activations/layer3_attention_weight_max": 97.75298309326172,
      "activations/layer3_attention_weight_min": -100.69766235351562,
      "activations/layer4_attention_weight_max": 116.76290130615234,
      "activations/layer4_attention_weight_min": -115.08099365234375,
      "activations/layer5_attention_weight_max": 56.01768493652344,
      "activations/layer5_attention_weight_min": -60.1242790222168,
      "activations/layer6_attention_weight_max": 47.377437591552734,
      "activations/layer6_attention_weight_min": -48.21782302856445,
      "activations/layer7_attention_weight_max": 89.49137115478516,
      "activations/layer7_attention_weight_min": -94.83795928955078,
      "activations/layer8_attention_weight_max": 39.24525833129883,
      "activations/layer8_attention_weight_min": -42.5580940246582,
      "activations/layer9_attention_weight_max": 33.0006217956543,
      "activations/layer9_attention_weight_min": -34.00554275512695,
      "epoch": 11.18,
      "learning_rate": 7.869848484848484e-05,
      "loss": 2.7856,
      "step": 192400
    },
    {
      "activations/layer0_attention_weight_max": 15.673662185668945,
      "activations/layer0_attention_weight_min": -12.795825004577637,
      "activations/layer10_attention_weight_max": 32.30864334106445,
      "activations/layer10_attention_weight_min": -31.771896362304688,
      "activations/layer11_attention_weight_max": 30.455852508544922,
      "activations/layer11_attention_weight_min": -29.38387107849121,
      "activations/layer12_attention_weight_max": 23.350147247314453,
      "activations/layer12_attention_weight_min": -24.186622619628906,
      "activations/layer13_attention_weight_max": 37.79044723510742,
      "activations/layer13_attention_weight_min": -32.888031005859375,
      "activations/layer14_attention_weight_max": 40.939781188964844,
      "activations/layer14_attention_weight_min": -35.74358367919922,
      "activations/layer15_attention_weight_max": 35.886905670166016,
      "activations/layer15_attention_weight_min": -30.376672744750977,
      "activations/layer16_attention_weight_max": 31.619104385375977,
      "activations/layer16_attention_weight_min": -29.42861557006836,
      "activations/layer17_attention_weight_max": 31.25214958190918,
      "activations/layer17_attention_weight_min": -26.298938751220703,
      "activations/layer18_attention_weight_max": 32.448150634765625,
      "activations/layer18_attention_weight_min": -22.424203872680664,
      "activations/layer19_attention_weight_max": 34.645172119140625,
      "activations/layer19_attention_weight_min": -29.636693954467773,
      "activations/layer1_attention_weight_max": 16.71044921875,
      "activations/layer1_attention_weight_min": -15.769773483276367,
      "activations/layer20_attention_weight_max": 29.44527816772461,
      "activations/layer20_attention_weight_min": -23.59008026123047,
      "activations/layer21_attention_weight_max": 30.844005584716797,
      "activations/layer21_attention_weight_min": -22.990575790405273,
      "activations/layer22_attention_weight_max": 44.75920486450195,
      "activations/layer22_attention_weight_min": -31.791284561157227,
      "activations/layer23_attention_weight_max": 32.35594177246094,
      "activations/layer23_attention_weight_min": -24.087032318115234,
      "activations/layer2_attention_weight_max": 33.56029510498047,
      "activations/layer2_attention_weight_min": -31.55145835876465,
      "activations/layer3_attention_weight_max": 95.65608215332031,
      "activations/layer3_attention_weight_min": -93.3993911743164,
      "activations/layer4_attention_weight_max": 114.3721923828125,
      "activations/layer4_attention_weight_min": -111.76651763916016,
      "activations/layer5_attention_weight_max": 50.93700408935547,
      "activations/layer5_attention_weight_min": -61.859527587890625,
      "activations/layer6_attention_weight_max": 46.14434051513672,
      "activations/layer6_attention_weight_min": -48.362037658691406,
      "activations/layer7_attention_weight_max": 87.06840515136719,
      "activations/layer7_attention_weight_min": -89.50279235839844,
      "activations/layer8_attention_weight_max": 38.92487716674805,
      "activations/layer8_attention_weight_min": -42.27027130126953,
      "activations/layer9_attention_weight_max": 31.959463119506836,
      "activations/layer9_attention_weight_min": -33.08087921142578,
      "epoch": 11.18,
      "learning_rate": 7.867954545454545e-05,
      "loss": 2.7788,
      "step": 192450
    },
    {
      "activations/layer0_attention_weight_max": 15.835027694702148,
      "activations/layer0_attention_weight_min": -12.342923164367676,
      "activations/layer10_attention_weight_max": 31.794376373291016,
      "activations/layer10_attention_weight_min": -32.87879943847656,
      "activations/layer11_attention_weight_max": 32.153568267822266,
      "activations/layer11_attention_weight_min": -31.726253509521484,
      "activations/layer12_attention_weight_max": 23.67034149169922,
      "activations/layer12_attention_weight_min": -27.126319885253906,
      "activations/layer13_attention_weight_max": 33.930965423583984,
      "activations/layer13_attention_weight_min": -31.025714874267578,
      "activations/layer14_attention_weight_max": 39.89352035522461,
      "activations/layer14_attention_weight_min": -36.11952590942383,
      "activations/layer15_attention_weight_max": 34.85110092163086,
      "activations/layer15_attention_weight_min": -30.417619705200195,
      "activations/layer16_attention_weight_max": 32.01127243041992,
      "activations/layer16_attention_weight_min": -26.36170768737793,
      "activations/layer17_attention_weight_max": 30.79779052734375,
      "activations/layer17_attention_weight_min": -24.076086044311523,
      "activations/layer18_attention_weight_max": 30.485939025878906,
      "activations/layer18_attention_weight_min": -23.901866912841797,
      "activations/layer19_attention_weight_max": 33.69313049316406,
      "activations/layer19_attention_weight_min": -30.3388614654541,
      "activations/layer1_attention_weight_max": 17.74590492248535,
      "activations/layer1_attention_weight_min": -16.866695404052734,
      "activations/layer20_attention_weight_max": 31.75527000427246,
      "activations/layer20_attention_weight_min": -24.082012176513672,
      "activations/layer21_attention_weight_max": 31.364105224609375,
      "activations/layer21_attention_weight_min": -22.896074295043945,
      "activations/layer22_attention_weight_max": 44.72161865234375,
      "activations/layer22_attention_weight_min": -28.160873413085938,
      "activations/layer23_attention_weight_max": 32.060890197753906,
      "activations/layer23_attention_weight_min": -24.22582244873047,
      "activations/layer2_attention_weight_max": 33.96269226074219,
      "activations/layer2_attention_weight_min": -32.681148529052734,
      "activations/layer3_attention_weight_max": 103.35626983642578,
      "activations/layer3_attention_weight_min": -97.91503143310547,
      "activations/layer4_attention_weight_max": 121.82454681396484,
      "activations/layer4_attention_weight_min": -113.5401611328125,
      "activations/layer5_attention_weight_max": 56.017303466796875,
      "activations/layer5_attention_weight_min": -60.15020751953125,
      "activations/layer6_attention_weight_max": 46.382266998291016,
      "activations/layer6_attention_weight_min": -48.41276550292969,
      "activations/layer7_attention_weight_max": 91.48479461669922,
      "activations/layer7_attention_weight_min": -85.25724029541016,
      "activations/layer8_attention_weight_max": 40.97054672241211,
      "activations/layer8_attention_weight_min": -39.15634536743164,
      "activations/layer9_attention_weight_max": 34.5126838684082,
      "activations/layer9_attention_weight_min": -31.412647247314453,
      "epoch": 11.19,
      "learning_rate": 7.866060606060606e-05,
      "loss": 2.7756,
      "step": 192500
    },
    {
      "activations/layer0_attention_weight_max": 16.83123207092285,
      "activations/layer0_attention_weight_min": -13.330674171447754,
      "activations/layer10_attention_weight_max": 32.37884521484375,
      "activations/layer10_attention_weight_min": -31.69338607788086,
      "activations/layer11_attention_weight_max": 34.42619323730469,
      "activations/layer11_attention_weight_min": -30.687732696533203,
      "activations/layer12_attention_weight_max": 23.726469039916992,
      "activations/layer12_attention_weight_min": -27.61388397216797,
      "activations/layer13_attention_weight_max": 36.23810958862305,
      "activations/layer13_attention_weight_min": -32.57012939453125,
      "activations/layer14_attention_weight_max": 43.624481201171875,
      "activations/layer14_attention_weight_min": -40.191497802734375,
      "activations/layer15_attention_weight_max": 33.56734848022461,
      "activations/layer15_attention_weight_min": -30.754959106445312,
      "activations/layer16_attention_weight_max": 32.83671569824219,
      "activations/layer16_attention_weight_min": -27.628610610961914,
      "activations/layer17_attention_weight_max": 29.76310157775879,
      "activations/layer17_attention_weight_min": -26.865888595581055,
      "activations/layer18_attention_weight_max": 30.946979522705078,
      "activations/layer18_attention_weight_min": -23.42404556274414,
      "activations/layer19_attention_weight_max": 32.91291427612305,
      "activations/layer19_attention_weight_min": -31.830535888671875,
      "activations/layer1_attention_weight_max": 16.31049346923828,
      "activations/layer1_attention_weight_min": -15.860885620117188,
      "activations/layer20_attention_weight_max": 28.436344146728516,
      "activations/layer20_attention_weight_min": -24.45587730407715,
      "activations/layer21_attention_weight_max": 25.725698471069336,
      "activations/layer21_attention_weight_min": -22.882156372070312,
      "activations/layer22_attention_weight_max": 39.50386428833008,
      "activations/layer22_attention_weight_min": -29.969358444213867,
      "activations/layer23_attention_weight_max": 34.280426025390625,
      "activations/layer23_attention_weight_min": -25.487892150878906,
      "activations/layer2_attention_weight_max": 35.34733200073242,
      "activations/layer2_attention_weight_min": -33.50553894042969,
      "activations/layer3_attention_weight_max": 98.421142578125,
      "activations/layer3_attention_weight_min": -101.30679321289062,
      "activations/layer4_attention_weight_max": 116.48502349853516,
      "activations/layer4_attention_weight_min": -113.1330337524414,
      "activations/layer5_attention_weight_max": 52.46273422241211,
      "activations/layer5_attention_weight_min": -62.51148223876953,
      "activations/layer6_attention_weight_max": 44.53681182861328,
      "activations/layer6_attention_weight_min": -47.40144729614258,
      "activations/layer7_attention_weight_max": 90.65505981445312,
      "activations/layer7_attention_weight_min": -89.95415496826172,
      "activations/layer8_attention_weight_max": 39.8563117980957,
      "activations/layer8_attention_weight_min": -40.37779998779297,
      "activations/layer9_attention_weight_max": 30.256366729736328,
      "activations/layer9_attention_weight_min": -32.626121520996094,
      "epoch": 11.19,
      "learning_rate": 7.864166666666665e-05,
      "loss": 2.7836,
      "step": 192550
    },
    {
      "activations/layer0_attention_weight_max": 15.247788429260254,
      "activations/layer0_attention_weight_min": -13.544578552246094,
      "activations/layer10_attention_weight_max": 31.191844940185547,
      "activations/layer10_attention_weight_min": -32.483760833740234,
      "activations/layer11_attention_weight_max": 33.304649353027344,
      "activations/layer11_attention_weight_min": -32.013763427734375,
      "activations/layer12_attention_weight_max": 24.03470230102539,
      "activations/layer12_attention_weight_min": -25.951099395751953,
      "activations/layer13_attention_weight_max": 40.497066497802734,
      "activations/layer13_attention_weight_min": -33.44135665893555,
      "activations/layer14_attention_weight_max": 46.15040588378906,
      "activations/layer14_attention_weight_min": -37.55684280395508,
      "activations/layer15_attention_weight_max": 34.93305206298828,
      "activations/layer15_attention_weight_min": -32.048927307128906,
      "activations/layer16_attention_weight_max": 28.077434539794922,
      "activations/layer16_attention_weight_min": -28.382761001586914,
      "activations/layer17_attention_weight_max": 30.442522048950195,
      "activations/layer17_attention_weight_min": -25.894424438476562,
      "activations/layer18_attention_weight_max": 28.145597457885742,
      "activations/layer18_attention_weight_min": -23.42631721496582,
      "activations/layer19_attention_weight_max": 31.767866134643555,
      "activations/layer19_attention_weight_min": -31.060117721557617,
      "activations/layer1_attention_weight_max": 16.2059268951416,
      "activations/layer1_attention_weight_min": -14.222593307495117,
      "activations/layer20_attention_weight_max": 28.534534454345703,
      "activations/layer20_attention_weight_min": -26.505582809448242,
      "activations/layer21_attention_weight_max": 31.605810165405273,
      "activations/layer21_attention_weight_min": -24.116954803466797,
      "activations/layer22_attention_weight_max": 40.217491149902344,
      "activations/layer22_attention_weight_min": -31.220930099487305,
      "activations/layer23_attention_weight_max": 32.98120880126953,
      "activations/layer23_attention_weight_min": -27.2407283782959,
      "activations/layer2_attention_weight_max": 32.372825622558594,
      "activations/layer2_attention_weight_min": -33.18504333496094,
      "activations/layer3_attention_weight_max": 97.26842498779297,
      "activations/layer3_attention_weight_min": -96.35618591308594,
      "activations/layer4_attention_weight_max": 110.54536437988281,
      "activations/layer4_attention_weight_min": -111.41462707519531,
      "activations/layer5_attention_weight_max": 51.631988525390625,
      "activations/layer5_attention_weight_min": -63.96275329589844,
      "activations/layer6_attention_weight_max": 42.51923370361328,
      "activations/layer6_attention_weight_min": -47.7358283996582,
      "activations/layer7_attention_weight_max": 89.1880874633789,
      "activations/layer7_attention_weight_min": -88.6275405883789,
      "activations/layer8_attention_weight_max": 38.867122650146484,
      "activations/layer8_attention_weight_min": -40.94499969482422,
      "activations/layer9_attention_weight_max": 32.73775863647461,
      "activations/layer9_attention_weight_min": -33.70895767211914,
      "epoch": 11.19,
      "learning_rate": 7.862272727272726e-05,
      "loss": 2.7714,
      "step": 192600
    },
    {
      "activations/layer0_attention_weight_max": 15.861907005310059,
      "activations/layer0_attention_weight_min": -12.16330623626709,
      "activations/layer10_attention_weight_max": 35.249671936035156,
      "activations/layer10_attention_weight_min": -33.060272216796875,
      "activations/layer11_attention_weight_max": 37.40038299560547,
      "activations/layer11_attention_weight_min": -33.251953125,
      "activations/layer12_attention_weight_max": 25.929025650024414,
      "activations/layer12_attention_weight_min": -27.221370697021484,
      "activations/layer13_attention_weight_max": 38.772457122802734,
      "activations/layer13_attention_weight_min": -35.05754470825195,
      "activations/layer14_attention_weight_max": 50.07328414916992,
      "activations/layer14_attention_weight_min": -38.5837516784668,
      "activations/layer15_attention_weight_max": 38.33143615722656,
      "activations/layer15_attention_weight_min": -32.03939437866211,
      "activations/layer16_attention_weight_max": 32.86587905883789,
      "activations/layer16_attention_weight_min": -28.591798782348633,
      "activations/layer17_attention_weight_max": 32.2542610168457,
      "activations/layer17_attention_weight_min": -24.5629825592041,
      "activations/layer18_attention_weight_max": 29.534629821777344,
      "activations/layer18_attention_weight_min": -23.389814376831055,
      "activations/layer19_attention_weight_max": 36.53504180908203,
      "activations/layer19_attention_weight_min": -35.37477111816406,
      "activations/layer1_attention_weight_max": 15.897233963012695,
      "activations/layer1_attention_weight_min": -14.895232200622559,
      "activations/layer20_attention_weight_max": 31.16998291015625,
      "activations/layer20_attention_weight_min": -26.12323570251465,
      "activations/layer21_attention_weight_max": 31.566129684448242,
      "activations/layer21_attention_weight_min": -24.46324920654297,
      "activations/layer22_attention_weight_max": 43.86796188354492,
      "activations/layer22_attention_weight_min": -32.0523567199707,
      "activations/layer23_attention_weight_max": 35.09013366699219,
      "activations/layer23_attention_weight_min": -27.870899200439453,
      "activations/layer2_attention_weight_max": 33.80208969116211,
      "activations/layer2_attention_weight_min": -31.8353271484375,
      "activations/layer3_attention_weight_max": 94.87085723876953,
      "activations/layer3_attention_weight_min": -96.0530014038086,
      "activations/layer4_attention_weight_max": 112.8021011352539,
      "activations/layer4_attention_weight_min": -110.02648162841797,
      "activations/layer5_attention_weight_max": 51.26903533935547,
      "activations/layer5_attention_weight_min": -60.819664001464844,
      "activations/layer6_attention_weight_max": 45.05848693847656,
      "activations/layer6_attention_weight_min": -48.10945129394531,
      "activations/layer7_attention_weight_max": 95.32572937011719,
      "activations/layer7_attention_weight_min": -89.92337799072266,
      "activations/layer8_attention_weight_max": 40.791770935058594,
      "activations/layer8_attention_weight_min": -41.64935302734375,
      "activations/layer9_attention_weight_max": 35.72926330566406,
      "activations/layer9_attention_weight_min": -34.051612854003906,
      "epoch": 11.19,
      "learning_rate": 7.860378787878788e-05,
      "loss": 2.7694,
      "step": 192650
    },
    {
      "activations/layer0_attention_weight_max": 14.971627235412598,
      "activations/layer0_attention_weight_min": -12.322102546691895,
      "activations/layer10_attention_weight_max": 33.72333908081055,
      "activations/layer10_attention_weight_min": -34.04791259765625,
      "activations/layer11_attention_weight_max": 36.75262451171875,
      "activations/layer11_attention_weight_min": -32.498878479003906,
      "activations/layer12_attention_weight_max": 25.368528366088867,
      "activations/layer12_attention_weight_min": -27.58094024658203,
      "activations/layer13_attention_weight_max": 41.70699691772461,
      "activations/layer13_attention_weight_min": -34.681495666503906,
      "activations/layer14_attention_weight_max": 40.72071838378906,
      "activations/layer14_attention_weight_min": -37.1308479309082,
      "activations/layer15_attention_weight_max": 35.79366683959961,
      "activations/layer15_attention_weight_min": -32.94350051879883,
      "activations/layer16_attention_weight_max": 30.242908477783203,
      "activations/layer16_attention_weight_min": -29.17222023010254,
      "activations/layer17_attention_weight_max": 30.444210052490234,
      "activations/layer17_attention_weight_min": -26.41531753540039,
      "activations/layer18_attention_weight_max": 29.67289924621582,
      "activations/layer18_attention_weight_min": -25.484004974365234,
      "activations/layer19_attention_weight_max": 34.25761032104492,
      "activations/layer19_attention_weight_min": -34.34005355834961,
      "activations/layer1_attention_weight_max": 16.203033447265625,
      "activations/layer1_attention_weight_min": -15.053293228149414,
      "activations/layer20_attention_weight_max": 30.188995361328125,
      "activations/layer20_attention_weight_min": -30.542490005493164,
      "activations/layer21_attention_weight_max": 32.08244323730469,
      "activations/layer21_attention_weight_min": -29.35980796813965,
      "activations/layer22_attention_weight_max": 49.58140563964844,
      "activations/layer22_attention_weight_min": -37.66464614868164,
      "activations/layer23_attention_weight_max": 34.250770568847656,
      "activations/layer23_attention_weight_min": -27.27081871032715,
      "activations/layer2_attention_weight_max": 34.25933074951172,
      "activations/layer2_attention_weight_min": -34.43815612792969,
      "activations/layer3_attention_weight_max": 103.13676452636719,
      "activations/layer3_attention_weight_min": -99.11347961425781,
      "activations/layer4_attention_weight_max": 123.83551025390625,
      "activations/layer4_attention_weight_min": -112.04591369628906,
      "activations/layer5_attention_weight_max": 54.91211700439453,
      "activations/layer5_attention_weight_min": -60.65367126464844,
      "activations/layer6_attention_weight_max": 45.21656799316406,
      "activations/layer6_attention_weight_min": -50.61941909790039,
      "activations/layer7_attention_weight_max": 90.7314224243164,
      "activations/layer7_attention_weight_min": -89.71361541748047,
      "activations/layer8_attention_weight_max": 43.76526641845703,
      "activations/layer8_attention_weight_min": -44.1635856628418,
      "activations/layer9_attention_weight_max": 34.55678939819336,
      "activations/layer9_attention_weight_min": -32.63926696777344,
      "epoch": 11.2,
      "learning_rate": 7.858484848484847e-05,
      "loss": 2.7723,
      "step": 192700
    },
    {
      "activations/layer0_attention_weight_max": 15.471582412719727,
      "activations/layer0_attention_weight_min": -11.797603607177734,
      "activations/layer10_attention_weight_max": 30.136734008789062,
      "activations/layer10_attention_weight_min": -31.68475914001465,
      "activations/layer11_attention_weight_max": 31.253156661987305,
      "activations/layer11_attention_weight_min": -31.636932373046875,
      "activations/layer12_attention_weight_max": 25.486854553222656,
      "activations/layer12_attention_weight_min": -24.862199783325195,
      "activations/layer13_attention_weight_max": 38.52324295043945,
      "activations/layer13_attention_weight_min": -34.4968376159668,
      "activations/layer14_attention_weight_max": 45.008811950683594,
      "activations/layer14_attention_weight_min": -40.75972366333008,
      "activations/layer15_attention_weight_max": 34.57533264160156,
      "activations/layer15_attention_weight_min": -32.83935546875,
      "activations/layer16_attention_weight_max": 32.37699508666992,
      "activations/layer16_attention_weight_min": -28.25019645690918,
      "activations/layer17_attention_weight_max": 30.666168212890625,
      "activations/layer17_attention_weight_min": -25.842336654663086,
      "activations/layer18_attention_weight_max": 33.57655715942383,
      "activations/layer18_attention_weight_min": -24.539295196533203,
      "activations/layer19_attention_weight_max": 35.20003128051758,
      "activations/layer19_attention_weight_min": -31.280807495117188,
      "activations/layer1_attention_weight_max": 16.697547912597656,
      "activations/layer1_attention_weight_min": -14.221155166625977,
      "activations/layer20_attention_weight_max": 30.7493839263916,
      "activations/layer20_attention_weight_min": -25.591489791870117,
      "activations/layer21_attention_weight_max": 31.397245407104492,
      "activations/layer21_attention_weight_min": -24.54074478149414,
      "activations/layer22_attention_weight_max": 43.21129608154297,
      "activations/layer22_attention_weight_min": -30.21867561340332,
      "activations/layer23_attention_weight_max": 33.7188606262207,
      "activations/layer23_attention_weight_min": -25.566503524780273,
      "activations/layer2_attention_weight_max": 33.093994140625,
      "activations/layer2_attention_weight_min": -32.72455978393555,
      "activations/layer3_attention_weight_max": 97.58289337158203,
      "activations/layer3_attention_weight_min": -97.94959259033203,
      "activations/layer4_attention_weight_max": 113.05523681640625,
      "activations/layer4_attention_weight_min": -115.05513000488281,
      "activations/layer5_attention_weight_max": 52.42905807495117,
      "activations/layer5_attention_weight_min": -64.18914031982422,
      "activations/layer6_attention_weight_max": 43.53846740722656,
      "activations/layer6_attention_weight_min": -50.21522903442383,
      "activations/layer7_attention_weight_max": 88.6142578125,
      "activations/layer7_attention_weight_min": -91.12031555175781,
      "activations/layer8_attention_weight_max": 36.92680740356445,
      "activations/layer8_attention_weight_min": -40.16170120239258,
      "activations/layer9_attention_weight_max": 30.542749404907227,
      "activations/layer9_attention_weight_min": -31.811582565307617,
      "epoch": 11.2,
      "learning_rate": 7.856590909090908e-05,
      "loss": 2.775,
      "step": 192750
    },
    {
      "activations/layer0_attention_weight_max": 16.203723907470703,
      "activations/layer0_attention_weight_min": -12.910959243774414,
      "activations/layer10_attention_weight_max": 31.131649017333984,
      "activations/layer10_attention_weight_min": -30.525615692138672,
      "activations/layer11_attention_weight_max": 29.63653564453125,
      "activations/layer11_attention_weight_min": -30.352516174316406,
      "activations/layer12_attention_weight_max": 23.363712310791016,
      "activations/layer12_attention_weight_min": -23.774736404418945,
      "activations/layer13_attention_weight_max": 36.12012481689453,
      "activations/layer13_attention_weight_min": -33.501319885253906,
      "activations/layer14_attention_weight_max": 39.661956787109375,
      "activations/layer14_attention_weight_min": -36.11505126953125,
      "activations/layer15_attention_weight_max": 34.79830551147461,
      "activations/layer15_attention_weight_min": -30.742876052856445,
      "activations/layer16_attention_weight_max": 29.59532356262207,
      "activations/layer16_attention_weight_min": -26.90004539489746,
      "activations/layer17_attention_weight_max": 30.44161033630371,
      "activations/layer17_attention_weight_min": -25.492143630981445,
      "activations/layer18_attention_weight_max": 29.498811721801758,
      "activations/layer18_attention_weight_min": -23.541292190551758,
      "activations/layer19_attention_weight_max": 30.60464859008789,
      "activations/layer19_attention_weight_min": -29.7540225982666,
      "activations/layer1_attention_weight_max": 16.037622451782227,
      "activations/layer1_attention_weight_min": -15.349442481994629,
      "activations/layer20_attention_weight_max": 28.35277557373047,
      "activations/layer20_attention_weight_min": -23.95401954650879,
      "activations/layer21_attention_weight_max": 29.385398864746094,
      "activations/layer21_attention_weight_min": -23.801250457763672,
      "activations/layer22_attention_weight_max": 40.27910232543945,
      "activations/layer22_attention_weight_min": -29.954540252685547,
      "activations/layer23_attention_weight_max": 34.01312255859375,
      "activations/layer23_attention_weight_min": -23.7735595703125,
      "activations/layer2_attention_weight_max": 33.620365142822266,
      "activations/layer2_attention_weight_min": -34.84992218017578,
      "activations/layer3_attention_weight_max": 98.96979522705078,
      "activations/layer3_attention_weight_min": -102.14837646484375,
      "activations/layer4_attention_weight_max": 113.37916564941406,
      "activations/layer4_attention_weight_min": -120.4797592163086,
      "activations/layer5_attention_weight_max": 54.637168884277344,
      "activations/layer5_attention_weight_min": -63.33824157714844,
      "activations/layer6_attention_weight_max": 44.64912796020508,
      "activations/layer6_attention_weight_min": -48.29466247558594,
      "activations/layer7_attention_weight_max": 82.92305755615234,
      "activations/layer7_attention_weight_min": -90.5846176147461,
      "activations/layer8_attention_weight_max": 39.752017974853516,
      "activations/layer8_attention_weight_min": -40.902530670166016,
      "activations/layer9_attention_weight_max": 32.363155364990234,
      "activations/layer9_attention_weight_min": -31.65140724182129,
      "epoch": 11.2,
      "learning_rate": 7.85469696969697e-05,
      "loss": 2.7689,
      "step": 192800
    },
    {
      "activations/layer0_attention_weight_max": 14.934261322021484,
      "activations/layer0_attention_weight_min": -11.860699653625488,
      "activations/layer10_attention_weight_max": 33.24324417114258,
      "activations/layer10_attention_weight_min": -34.18672180175781,
      "activations/layer11_attention_weight_max": 34.32897186279297,
      "activations/layer11_attention_weight_min": -33.612213134765625,
      "activations/layer12_attention_weight_max": 32.515235900878906,
      "activations/layer12_attention_weight_min": -25.317501068115234,
      "activations/layer13_attention_weight_max": 44.54359436035156,
      "activations/layer13_attention_weight_min": -35.37078094482422,
      "activations/layer14_attention_weight_max": 55.205989837646484,
      "activations/layer14_attention_weight_min": -40.5560417175293,
      "activations/layer15_attention_weight_max": 41.29704666137695,
      "activations/layer15_attention_weight_min": -35.673011779785156,
      "activations/layer16_attention_weight_max": 37.04842758178711,
      "activations/layer16_attention_weight_min": -28.81291389465332,
      "activations/layer17_attention_weight_max": 36.894996643066406,
      "activations/layer17_attention_weight_min": -26.853500366210938,
      "activations/layer18_attention_weight_max": 34.99361038208008,
      "activations/layer18_attention_weight_min": -24.18210792541504,
      "activations/layer19_attention_weight_max": 43.840538024902344,
      "activations/layer19_attention_weight_min": -33.25346755981445,
      "activations/layer1_attention_weight_max": 17.100645065307617,
      "activations/layer1_attention_weight_min": -14.79518985748291,
      "activations/layer20_attention_weight_max": 34.274078369140625,
      "activations/layer20_attention_weight_min": -25.349876403808594,
      "activations/layer21_attention_weight_max": 33.99198913574219,
      "activations/layer21_attention_weight_min": -24.45499038696289,
      "activations/layer22_attention_weight_max": 60.937278747558594,
      "activations/layer22_attention_weight_min": -31.06757354736328,
      "activations/layer23_attention_weight_max": 46.079586029052734,
      "activations/layer23_attention_weight_min": -24.352108001708984,
      "activations/layer2_attention_weight_max": 32.990196228027344,
      "activations/layer2_attention_weight_min": -33.38451385498047,
      "activations/layer3_attention_weight_max": 94.66178894042969,
      "activations/layer3_attention_weight_min": -100.44159698486328,
      "activations/layer4_attention_weight_max": 112.58080291748047,
      "activations/layer4_attention_weight_min": -110.99481201171875,
      "activations/layer5_attention_weight_max": 50.26744842529297,
      "activations/layer5_attention_weight_min": -62.568931579589844,
      "activations/layer6_attention_weight_max": 44.5649528503418,
      "activations/layer6_attention_weight_min": -48.43977355957031,
      "activations/layer7_attention_weight_max": 90.7826919555664,
      "activations/layer7_attention_weight_min": -88.89696502685547,
      "activations/layer8_attention_weight_max": 40.99581527709961,
      "activations/layer8_attention_weight_min": -43.17432403564453,
      "activations/layer9_attention_weight_max": 34.81053924560547,
      "activations/layer9_attention_weight_min": -33.98789596557617,
      "epoch": 11.21,
      "learning_rate": 7.85280303030303e-05,
      "loss": 2.771,
      "step": 192850
    },
    {
      "activations/layer0_attention_weight_max": 15.201579093933105,
      "activations/layer0_attention_weight_min": -11.51673412322998,
      "activations/layer10_attention_weight_max": 42.7143440246582,
      "activations/layer10_attention_weight_min": -37.43686294555664,
      "activations/layer11_attention_weight_max": 42.700164794921875,
      "activations/layer11_attention_weight_min": -37.221126556396484,
      "activations/layer12_attention_weight_max": 24.275510787963867,
      "activations/layer12_attention_weight_min": -30.03199577331543,
      "activations/layer13_attention_weight_max": 37.8027229309082,
      "activations/layer13_attention_weight_min": -32.93036651611328,
      "activations/layer14_attention_weight_max": 42.33072280883789,
      "activations/layer14_attention_weight_min": -37.08241653442383,
      "activations/layer15_attention_weight_max": 41.54563903808594,
      "activations/layer15_attention_weight_min": -34.56352233886719,
      "activations/layer16_attention_weight_max": 29.10396385192871,
      "activations/layer16_attention_weight_min": -27.044225692749023,
      "activations/layer17_attention_weight_max": 30.59515380859375,
      "activations/layer17_attention_weight_min": -23.829626083374023,
      "activations/layer18_attention_weight_max": 27.187381744384766,
      "activations/layer18_attention_weight_min": -21.650840759277344,
      "activations/layer19_attention_weight_max": 31.015222549438477,
      "activations/layer19_attention_weight_min": -28.8253173828125,
      "activations/layer1_attention_weight_max": 17.398754119873047,
      "activations/layer1_attention_weight_min": -14.362327575683594,
      "activations/layer20_attention_weight_max": 26.967424392700195,
      "activations/layer20_attention_weight_min": -22.404075622558594,
      "activations/layer21_attention_weight_max": 26.508907318115234,
      "activations/layer21_attention_weight_min": -22.17447853088379,
      "activations/layer22_attention_weight_max": 40.619102478027344,
      "activations/layer22_attention_weight_min": -29.071491241455078,
      "activations/layer23_attention_weight_max": 30.972272872924805,
      "activations/layer23_attention_weight_min": -24.038915634155273,
      "activations/layer2_attention_weight_max": 35.57970428466797,
      "activations/layer2_attention_weight_min": -34.22087478637695,
      "activations/layer3_attention_weight_max": 100.55477142333984,
      "activations/layer3_attention_weight_min": -100.88993835449219,
      "activations/layer4_attention_weight_max": 116.9966812133789,
      "activations/layer4_attention_weight_min": -114.10640716552734,
      "activations/layer5_attention_weight_max": 60.37950134277344,
      "activations/layer5_attention_weight_min": -64.41795349121094,
      "activations/layer6_attention_weight_max": 48.46699905395508,
      "activations/layer6_attention_weight_min": -48.49521255493164,
      "activations/layer7_attention_weight_max": 110.90823364257812,
      "activations/layer7_attention_weight_min": -102.02022552490234,
      "activations/layer8_attention_weight_max": 51.330421447753906,
      "activations/layer8_attention_weight_min": -45.54784393310547,
      "activations/layer9_attention_weight_max": 38.54010009765625,
      "activations/layer9_attention_weight_min": -38.44476318359375,
      "epoch": 11.21,
      "learning_rate": 7.85090909090909e-05,
      "loss": 2.7628,
      "step": 192900
    },
    {
      "activations/layer0_attention_weight_max": 15.41673469543457,
      "activations/layer0_attention_weight_min": -12.287179946899414,
      "activations/layer10_attention_weight_max": 35.99018859863281,
      "activations/layer10_attention_weight_min": -36.01411819458008,
      "activations/layer11_attention_weight_max": 35.14470672607422,
      "activations/layer11_attention_weight_min": -36.80474090576172,
      "activations/layer12_attention_weight_max": 25.71466827392578,
      "activations/layer12_attention_weight_min": -26.825809478759766,
      "activations/layer13_attention_weight_max": 40.39555740356445,
      "activations/layer13_attention_weight_min": -35.936397552490234,
      "activations/layer14_attention_weight_max": 41.73542785644531,
      "activations/layer14_attention_weight_min": -37.63813781738281,
      "activations/layer15_attention_weight_max": 36.710845947265625,
      "activations/layer15_attention_weight_min": -33.79429244995117,
      "activations/layer16_attention_weight_max": 33.436981201171875,
      "activations/layer16_attention_weight_min": -28.039539337158203,
      "activations/layer17_attention_weight_max": 30.492563247680664,
      "activations/layer17_attention_weight_min": -25.37920570373535,
      "activations/layer18_attention_weight_max": 30.732398986816406,
      "activations/layer18_attention_weight_min": -23.61568832397461,
      "activations/layer19_attention_weight_max": 38.43041229248047,
      "activations/layer19_attention_weight_min": -32.39486312866211,
      "activations/layer1_attention_weight_max": 16.1234188079834,
      "activations/layer1_attention_weight_min": -14.731640815734863,
      "activations/layer20_attention_weight_max": 31.659549713134766,
      "activations/layer20_attention_weight_min": -26.265087127685547,
      "activations/layer21_attention_weight_max": 31.10452651977539,
      "activations/layer21_attention_weight_min": -25.649600982666016,
      "activations/layer22_attention_weight_max": 51.07395935058594,
      "activations/layer22_attention_weight_min": -32.42552947998047,
      "activations/layer23_attention_weight_max": 41.843746185302734,
      "activations/layer23_attention_weight_min": -24.798254013061523,
      "activations/layer2_attention_weight_max": 32.97596740722656,
      "activations/layer2_attention_weight_min": -34.27825927734375,
      "activations/layer3_attention_weight_max": 98.45870971679688,
      "activations/layer3_attention_weight_min": -102.15467834472656,
      "activations/layer4_attention_weight_max": 117.3141860961914,
      "activations/layer4_attention_weight_min": -120.40614318847656,
      "activations/layer5_attention_weight_max": 54.547332763671875,
      "activations/layer5_attention_weight_min": -64.28581237792969,
      "activations/layer6_attention_weight_max": 47.17897415161133,
      "activations/layer6_attention_weight_min": -49.37892532348633,
      "activations/layer7_attention_weight_max": 99.66070556640625,
      "activations/layer7_attention_weight_min": -100.52813720703125,
      "activations/layer8_attention_weight_max": 44.73678970336914,
      "activations/layer8_attention_weight_min": -44.099266052246094,
      "activations/layer9_attention_weight_max": 36.65647506713867,
      "activations/layer9_attention_weight_min": -36.7357292175293,
      "epoch": 11.21,
      "learning_rate": 7.84901515151515e-05,
      "loss": 2.7705,
      "step": 192950
    },
    {
      "activations/layer0_attention_weight_max": 15.045610427856445,
      "activations/layer0_attention_weight_min": -12.760688781738281,
      "activations/layer10_attention_weight_max": 35.703033447265625,
      "activations/layer10_attention_weight_min": -34.27577590942383,
      "activations/layer11_attention_weight_max": 39.75772476196289,
      "activations/layer11_attention_weight_min": -32.558677673339844,
      "activations/layer12_attention_weight_max": 26.680679321289062,
      "activations/layer12_attention_weight_min": -26.009069442749023,
      "activations/layer13_attention_weight_max": 42.43719482421875,
      "activations/layer13_attention_weight_min": -33.4423828125,
      "activations/layer14_attention_weight_max": 46.92460632324219,
      "activations/layer14_attention_weight_min": -38.87614059448242,
      "activations/layer15_attention_weight_max": 39.18540573120117,
      "activations/layer15_attention_weight_min": -33.277259826660156,
      "activations/layer16_attention_weight_max": 33.215885162353516,
      "activations/layer16_attention_weight_min": -27.833126068115234,
      "activations/layer17_attention_weight_max": 33.06899642944336,
      "activations/layer17_attention_weight_min": -26.59220314025879,
      "activations/layer18_attention_weight_max": 34.25981903076172,
      "activations/layer18_attention_weight_min": -23.539852142333984,
      "activations/layer19_attention_weight_max": 38.0112190246582,
      "activations/layer19_attention_weight_min": -30.6538028717041,
      "activations/layer1_attention_weight_max": 17.31819725036621,
      "activations/layer1_attention_weight_min": -15.204944610595703,
      "activations/layer20_attention_weight_max": 30.080631256103516,
      "activations/layer20_attention_weight_min": -25.10546875,
      "activations/layer21_attention_weight_max": 31.69005584716797,
      "activations/layer21_attention_weight_min": -24.798568725585938,
      "activations/layer22_attention_weight_max": 51.66057205200195,
      "activations/layer22_attention_weight_min": -30.74431037902832,
      "activations/layer23_attention_weight_max": 37.06245422363281,
      "activations/layer23_attention_weight_min": -24.69820785522461,
      "activations/layer2_attention_weight_max": 33.700660705566406,
      "activations/layer2_attention_weight_min": -32.632293701171875,
      "activations/layer3_attention_weight_max": 97.18939971923828,
      "activations/layer3_attention_weight_min": -99.08625793457031,
      "activations/layer4_attention_weight_max": 118.95904541015625,
      "activations/layer4_attention_weight_min": -118.00691986083984,
      "activations/layer5_attention_weight_max": 54.20774841308594,
      "activations/layer5_attention_weight_min": -61.09298324584961,
      "activations/layer6_attention_weight_max": 44.90752410888672,
      "activations/layer6_attention_weight_min": -50.39504623413086,
      "activations/layer7_attention_weight_max": 92.76960754394531,
      "activations/layer7_attention_weight_min": -92.02395629882812,
      "activations/layer8_attention_weight_max": 41.796329498291016,
      "activations/layer8_attention_weight_min": -43.32415008544922,
      "activations/layer9_attention_weight_max": 36.08060836791992,
      "activations/layer9_attention_weight_min": -34.41686248779297,
      "epoch": 11.21,
      "learning_rate": 7.847121212121212e-05,
      "loss": 2.768,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.4752,
      "eval_samples_per_second": 506.655,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.4752,
      "eval_openwebtext_samples_per_second": 506.655,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 2.0153,
      "eval_wikitext_samples_per_second": 226.268,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_lambada_loss": 2.515625,
      "eval_lambada_ppl": 12.374340325455691,
      "eval_lambada_runtime": 9.6031,
      "eval_lambada_samples_per_second": 507.024,
      "step": 193000
    },
    {
      "activations/layer0_attention_weight_max": 15.295356750488281,
      "activations/layer0_attention_weight_min": -12.92951488494873,
      "activations/layer10_attention_weight_max": 32.63660430908203,
      "activations/layer10_attention_weight_min": -31.14000129699707,
      "activations/layer11_attention_weight_max": 32.83419418334961,
      "activations/layer11_attention_weight_min": -31.17469024658203,
      "activations/layer12_attention_weight_max": 28.547077178955078,
      "activations/layer12_attention_weight_min": -24.14673614501953,
      "activations/layer13_attention_weight_max": 40.98479080200195,
      "activations/layer13_attention_weight_min": -35.066070556640625,
      "activations/layer14_attention_weight_max": 48.582157135009766,
      "activations/layer14_attention_weight_min": -36.58805847167969,
      "activations/layer15_attention_weight_max": 36.50379943847656,
      "activations/layer15_attention_weight_min": -32.10879898071289,
      "activations/layer16_attention_weight_max": 34.09862518310547,
      "activations/layer16_attention_weight_min": -30.712356567382812,
      "activations/layer17_attention_weight_max": 31.436845779418945,
      "activations/layer17_attention_weight_min": -26.817739486694336,
      "activations/layer18_attention_weight_max": 31.684358596801758,
      "activations/layer18_attention_weight_min": -23.265501022338867,
      "activations/layer19_attention_weight_max": 37.16414260864258,
      "activations/layer19_attention_weight_min": -31.41152000427246,
      "activations/layer1_attention_weight_max": 16.76451873779297,
      "activations/layer1_attention_weight_min": -13.03215503692627,
      "activations/layer20_attention_weight_max": 32.76980972290039,
      "activations/layer20_attention_weight_min": -24.778274536132812,
      "activations/layer21_attention_weight_max": 32.71620178222656,
      "activations/layer21_attention_weight_min": -25.498538970947266,
      "activations/layer22_attention_weight_max": 48.57207489013672,
      "activations/layer22_attention_weight_min": -30.111116409301758,
      "activations/layer23_attention_weight_max": 39.537689208984375,
      "activations/layer23_attention_weight_min": -28.585464477539062,
      "activations/layer2_attention_weight_max": 30.494266510009766,
      "activations/layer2_attention_weight_min": -32.25061798095703,
      "activations/layer3_attention_weight_max": 90.90235900878906,
      "activations/layer3_attention_weight_min": -94.57481384277344,
      "activations/layer4_attention_weight_max": 106.5478744506836,
      "activations/layer4_attention_weight_min": -106.24842834472656,
      "activations/layer5_attention_weight_max": 51.23173141479492,
      "activations/layer5_attention_weight_min": -62.20713424682617,
      "activations/layer6_attention_weight_max": 42.31614303588867,
      "activations/layer6_attention_weight_min": -46.13765335083008,
      "activations/layer7_attention_weight_max": 86.0604476928711,
      "activations/layer7_attention_weight_min": -88.06407165527344,
      "activations/layer8_attention_weight_max": 38.14828109741211,
      "activations/layer8_attention_weight_min": -40.07835006713867,
      "activations/layer9_attention_weight_max": 31.868314743041992,
      "activations/layer9_attention_weight_min": -31.851736068725586,
      "epoch": 11.22,
      "learning_rate": 7.845227272727272e-05,
      "loss": 2.7516,
      "step": 193050
    },
    {
      "activations/layer0_attention_weight_max": 15.37083625793457,
      "activations/layer0_attention_weight_min": -12.025771141052246,
      "activations/layer10_attention_weight_max": 45.51518630981445,
      "activations/layer10_attention_weight_min": -41.996952056884766,
      "activations/layer11_attention_weight_max": 46.365333557128906,
      "activations/layer11_attention_weight_min": -42.41789245605469,
      "activations/layer12_attention_weight_max": 30.656761169433594,
      "activations/layer12_attention_weight_min": -26.871305465698242,
      "activations/layer13_attention_weight_max": 41.749053955078125,
      "activations/layer13_attention_weight_min": -33.11560821533203,
      "activations/layer14_attention_weight_max": 46.82302474975586,
      "activations/layer14_attention_weight_min": -36.62849044799805,
      "activations/layer15_attention_weight_max": 39.91531753540039,
      "activations/layer15_attention_weight_min": -32.274253845214844,
      "activations/layer16_attention_weight_max": 31.633556365966797,
      "activations/layer16_attention_weight_min": -27.008955001831055,
      "activations/layer17_attention_weight_max": 30.405059814453125,
      "activations/layer17_attention_weight_min": -26.292705535888672,
      "activations/layer18_attention_weight_max": 32.721805572509766,
      "activations/layer18_attention_weight_min": -23.76725959777832,
      "activations/layer19_attention_weight_max": 33.65449142456055,
      "activations/layer19_attention_weight_min": -29.585439682006836,
      "activations/layer1_attention_weight_max": 17.39311408996582,
      "activations/layer1_attention_weight_min": -14.86176872253418,
      "activations/layer20_attention_weight_max": 30.25429916381836,
      "activations/layer20_attention_weight_min": -24.372507095336914,
      "activations/layer21_attention_weight_max": 30.7913875579834,
      "activations/layer21_attention_weight_min": -23.62454605102539,
      "activations/layer22_attention_weight_max": 48.25518035888672,
      "activations/layer22_attention_weight_min": -30.1815185546875,
      "activations/layer23_attention_weight_max": 37.17116165161133,
      "activations/layer23_attention_weight_min": -26.902118682861328,
      "activations/layer2_attention_weight_max": 37.50682067871094,
      "activations/layer2_attention_weight_min": -39.82387161254883,
      "activations/layer3_attention_weight_max": 117.58840942382812,
      "activations/layer3_attention_weight_min": -119.82105255126953,
      "activations/layer4_attention_weight_max": 134.9220428466797,
      "activations/layer4_attention_weight_min": -126.12938690185547,
      "activations/layer5_attention_weight_max": 59.898345947265625,
      "activations/layer5_attention_weight_min": -62.63398361206055,
      "activations/layer6_attention_weight_max": 51.831443786621094,
      "activations/layer6_attention_weight_min": -48.63799285888672,
      "activations/layer7_attention_weight_max": 119.46294403076172,
      "activations/layer7_attention_weight_min": -103.82127380371094,
      "activations/layer8_attention_weight_max": 53.56733703613281,
      "activations/layer8_attention_weight_min": -48.45711135864258,
      "activations/layer9_attention_weight_max": 41.85112380981445,
      "activations/layer9_attention_weight_min": -41.09865951538086,
      "epoch": 11.22,
      "learning_rate": 7.843333333333332e-05,
      "loss": 2.7956,
      "step": 193100
    },
    {
      "activations/layer0_attention_weight_max": 15.496086120605469,
      "activations/layer0_attention_weight_min": -12.055886268615723,
      "activations/layer10_attention_weight_max": 33.869876861572266,
      "activations/layer10_attention_weight_min": -35.55107116699219,
      "activations/layer11_attention_weight_max": 32.434539794921875,
      "activations/layer11_attention_weight_min": -34.317909240722656,
      "activations/layer12_attention_weight_max": 24.189208984375,
      "activations/layer12_attention_weight_min": -24.546527862548828,
      "activations/layer13_attention_weight_max": 40.22100830078125,
      "activations/layer13_attention_weight_min": -33.788429260253906,
      "activations/layer14_attention_weight_max": 48.10902786254883,
      "activations/layer14_attention_weight_min": -39.55337905883789,
      "activations/layer15_attention_weight_max": 40.68717956542969,
      "activations/layer15_attention_weight_min": -32.9631462097168,
      "activations/layer16_attention_weight_max": 36.27333450317383,
      "activations/layer16_attention_weight_min": -27.293624877929688,
      "activations/layer17_attention_weight_max": 33.53658676147461,
      "activations/layer17_attention_weight_min": -27.07742691040039,
      "activations/layer18_attention_weight_max": 33.88086700439453,
      "activations/layer18_attention_weight_min": -24.795141220092773,
      "activations/layer19_attention_weight_max": 37.89657211303711,
      "activations/layer19_attention_weight_min": -32.600711822509766,
      "activations/layer1_attention_weight_max": 16.11086654663086,
      "activations/layer1_attention_weight_min": -14.263197898864746,
      "activations/layer20_attention_weight_max": 34.960018157958984,
      "activations/layer20_attention_weight_min": -26.842601776123047,
      "activations/layer21_attention_weight_max": 33.56891632080078,
      "activations/layer21_attention_weight_min": -25.97174644470215,
      "activations/layer22_attention_weight_max": 49.94418716430664,
      "activations/layer22_attention_weight_min": -31.612628936767578,
      "activations/layer23_attention_weight_max": 38.90361785888672,
      "activations/layer23_attention_weight_min": -25.387527465820312,
      "activations/layer2_attention_weight_max": 33.37894821166992,
      "activations/layer2_attention_weight_min": -34.31270217895508,
      "activations/layer3_attention_weight_max": 98.2474594116211,
      "activations/layer3_attention_weight_min": -101.68534088134766,
      "activations/layer4_attention_weight_max": 117.382568359375,
      "activations/layer4_attention_weight_min": -117.4914321899414,
      "activations/layer5_attention_weight_max": 55.81275939941406,
      "activations/layer5_attention_weight_min": -63.09298324584961,
      "activations/layer6_attention_weight_max": 46.86947250366211,
      "activations/layer6_attention_weight_min": -51.218692779541016,
      "activations/layer7_attention_weight_max": 91.29827117919922,
      "activations/layer7_attention_weight_min": -91.55054473876953,
      "activations/layer8_attention_weight_max": 40.08342742919922,
      "activations/layer8_attention_weight_min": -44.05886459350586,
      "activations/layer9_attention_weight_max": 34.88155746459961,
      "activations/layer9_attention_weight_min": -35.15230941772461,
      "epoch": 11.22,
      "learning_rate": 7.841439393939393e-05,
      "loss": 2.7832,
      "step": 193150
    },
    {
      "activations/layer0_attention_weight_max": 15.771198272705078,
      "activations/layer0_attention_weight_min": -11.905447959899902,
      "activations/layer10_attention_weight_max": 31.418996810913086,
      "activations/layer10_attention_weight_min": -30.192167282104492,
      "activations/layer11_attention_weight_max": 31.720970153808594,
      "activations/layer11_attention_weight_min": -30.021305084228516,
      "activations/layer12_attention_weight_max": 23.92882537841797,
      "activations/layer12_attention_weight_min": -24.16486358642578,
      "activations/layer13_attention_weight_max": 37.89997482299805,
      "activations/layer13_attention_weight_min": -33.952457427978516,
      "activations/layer14_attention_weight_max": 45.41224670410156,
      "activations/layer14_attention_weight_min": -35.92230987548828,
      "activations/layer15_attention_weight_max": 36.79868698120117,
      "activations/layer15_attention_weight_min": -31.85746955871582,
      "activations/layer16_attention_weight_max": 32.91322326660156,
      "activations/layer16_attention_weight_min": -26.89892578125,
      "activations/layer17_attention_weight_max": 30.89620590209961,
      "activations/layer17_attention_weight_min": -24.607799530029297,
      "activations/layer18_attention_weight_max": 29.1715087890625,
      "activations/layer18_attention_weight_min": -22.93158531188965,
      "activations/layer19_attention_weight_max": 32.14080810546875,
      "activations/layer19_attention_weight_min": -29.380765914916992,
      "activations/layer1_attention_weight_max": 16.704818725585938,
      "activations/layer1_attention_weight_min": -14.509851455688477,
      "activations/layer20_attention_weight_max": 26.965696334838867,
      "activations/layer20_attention_weight_min": -23.59281349182129,
      "activations/layer21_attention_weight_max": 26.577795028686523,
      "activations/layer21_attention_weight_min": -22.33024024963379,
      "activations/layer22_attention_weight_max": 40.28864288330078,
      "activations/layer22_attention_weight_min": -28.4352970123291,
      "activations/layer23_attention_weight_max": 37.33161926269531,
      "activations/layer23_attention_weight_min": -21.874279022216797,
      "activations/layer2_attention_weight_max": 33.698692321777344,
      "activations/layer2_attention_weight_min": -32.971153259277344,
      "activations/layer3_attention_weight_max": 96.54255676269531,
      "activations/layer3_attention_weight_min": -96.25749969482422,
      "activations/layer4_attention_weight_max": 113.89315032958984,
      "activations/layer4_attention_weight_min": -118.48744201660156,
      "activations/layer5_attention_weight_max": 52.54237365722656,
      "activations/layer5_attention_weight_min": -64.11992645263672,
      "activations/layer6_attention_weight_max": 44.56952667236328,
      "activations/layer6_attention_weight_min": -45.709861755371094,
      "activations/layer7_attention_weight_max": 88.98406982421875,
      "activations/layer7_attention_weight_min": -86.82843780517578,
      "activations/layer8_attention_weight_max": 39.021663665771484,
      "activations/layer8_attention_weight_min": -40.28129959106445,
      "activations/layer9_attention_weight_max": 32.91988754272461,
      "activations/layer9_attention_weight_min": -31.966306686401367,
      "epoch": 11.23,
      "learning_rate": 7.839545454545455e-05,
      "loss": 2.7676,
      "step": 193200
    },
    {
      "activations/layer0_attention_weight_max": 15.996570587158203,
      "activations/layer0_attention_weight_min": -12.257308006286621,
      "activations/layer10_attention_weight_max": 45.81758499145508,
      "activations/layer10_attention_weight_min": -40.081661224365234,
      "activations/layer11_attention_weight_max": 45.65073776245117,
      "activations/layer11_attention_weight_min": -41.90690612792969,
      "activations/layer12_attention_weight_max": 25.936933517456055,
      "activations/layer12_attention_weight_min": -28.032968521118164,
      "activations/layer13_attention_weight_max": 38.29553985595703,
      "activations/layer13_attention_weight_min": -36.2935905456543,
      "activations/layer14_attention_weight_max": 43.80821228027344,
      "activations/layer14_attention_weight_min": -36.986412048339844,
      "activations/layer15_attention_weight_max": 36.02555847167969,
      "activations/layer15_attention_weight_min": -29.769302368164062,
      "activations/layer16_attention_weight_max": 31.521448135375977,
      "activations/layer16_attention_weight_min": -27.169265747070312,
      "activations/layer17_attention_weight_max": 33.023582458496094,
      "activations/layer17_attention_weight_min": -24.61386489868164,
      "activations/layer18_attention_weight_max": 32.36054229736328,
      "activations/layer18_attention_weight_min": -22.71942710876465,
      "activations/layer19_attention_weight_max": 36.26838302612305,
      "activations/layer19_attention_weight_min": -29.971607208251953,
      "activations/layer1_attention_weight_max": 16.810094833374023,
      "activations/layer1_attention_weight_min": -16.62893295288086,
      "activations/layer20_attention_weight_max": 28.805309295654297,
      "activations/layer20_attention_weight_min": -22.64664649963379,
      "activations/layer21_attention_weight_max": 27.82169532775879,
      "activations/layer21_attention_weight_min": -22.51102638244629,
      "activations/layer22_attention_weight_max": 50.92722702026367,
      "activations/layer22_attention_weight_min": -30.566410064697266,
      "activations/layer23_attention_weight_max": 38.04500198364258,
      "activations/layer23_attention_weight_min": -23.96912384033203,
      "activations/layer2_attention_weight_max": 34.2381477355957,
      "activations/layer2_attention_weight_min": -33.69407272338867,
      "activations/layer3_attention_weight_max": 102.36467742919922,
      "activations/layer3_attention_weight_min": -101.32355499267578,
      "activations/layer4_attention_weight_max": 119.4446792602539,
      "activations/layer4_attention_weight_min": -116.20108795166016,
      "activations/layer5_attention_weight_max": 57.416133880615234,
      "activations/layer5_attention_weight_min": -62.65003967285156,
      "activations/layer6_attention_weight_max": 48.98259353637695,
      "activations/layer6_attention_weight_min": -52.94562530517578,
      "activations/layer7_attention_weight_max": 109.0212173461914,
      "activations/layer7_attention_weight_min": -93.59455871582031,
      "activations/layer8_attention_weight_max": 47.897823333740234,
      "activations/layer8_attention_weight_min": -43.10245132446289,
      "activations/layer9_attention_weight_max": 40.9854850769043,
      "activations/layer9_attention_weight_min": -39.58100891113281,
      "epoch": 11.23,
      "learning_rate": 7.837651515151514e-05,
      "loss": 2.7651,
      "step": 193250
    },
    {
      "activations/layer0_attention_weight_max": 16.139076232910156,
      "activations/layer0_attention_weight_min": -12.736215591430664,
      "activations/layer10_attention_weight_max": 33.20613479614258,
      "activations/layer10_attention_weight_min": -33.79076385498047,
      "activations/layer11_attention_weight_max": 32.49344253540039,
      "activations/layer11_attention_weight_min": -33.29441452026367,
      "activations/layer12_attention_weight_max": 29.93350601196289,
      "activations/layer12_attention_weight_min": -24.079607009887695,
      "activations/layer13_attention_weight_max": 47.75739669799805,
      "activations/layer13_attention_weight_min": -34.90279769897461,
      "activations/layer14_attention_weight_max": 43.56641387939453,
      "activations/layer14_attention_weight_min": -36.94504165649414,
      "activations/layer15_attention_weight_max": 42.46947479248047,
      "activations/layer15_attention_weight_min": -32.606300354003906,
      "activations/layer16_attention_weight_max": 30.13370704650879,
      "activations/layer16_attention_weight_min": -25.97873306274414,
      "activations/layer17_attention_weight_max": 29.6065616607666,
      "activations/layer17_attention_weight_min": -24.221553802490234,
      "activations/layer18_attention_weight_max": 31.135173797607422,
      "activations/layer18_attention_weight_min": -23.44047737121582,
      "activations/layer19_attention_weight_max": 33.376853942871094,
      "activations/layer19_attention_weight_min": -32.39841079711914,
      "activations/layer1_attention_weight_max": 16.82363510131836,
      "activations/layer1_attention_weight_min": -15.836779594421387,
      "activations/layer20_attention_weight_max": 31.27696990966797,
      "activations/layer20_attention_weight_min": -23.618120193481445,
      "activations/layer21_attention_weight_max": 30.259855270385742,
      "activations/layer21_attention_weight_min": -22.812299728393555,
      "activations/layer22_attention_weight_max": 41.75673294067383,
      "activations/layer22_attention_weight_min": -30.019813537597656,
      "activations/layer23_attention_weight_max": 34.551116943359375,
      "activations/layer23_attention_weight_min": -25.050642013549805,
      "activations/layer2_attention_weight_max": 34.21795654296875,
      "activations/layer2_attention_weight_min": -35.24722671508789,
      "activations/layer3_attention_weight_max": 99.97651672363281,
      "activations/layer3_attention_weight_min": -101.98216247558594,
      "activations/layer4_attention_weight_max": 110.88099670410156,
      "activations/layer4_attention_weight_min": -115.88276672363281,
      "activations/layer5_attention_weight_max": 51.31359100341797,
      "activations/layer5_attention_weight_min": -64.62080383300781,
      "activations/layer6_attention_weight_max": 45.27409362792969,
      "activations/layer6_attention_weight_min": -48.36521530151367,
      "activations/layer7_attention_weight_max": 85.42150115966797,
      "activations/layer7_attention_weight_min": -94.93706512451172,
      "activations/layer8_attention_weight_max": 39.79189682006836,
      "activations/layer8_attention_weight_min": -40.73998260498047,
      "activations/layer9_attention_weight_max": 32.7193717956543,
      "activations/layer9_attention_weight_min": -33.184722900390625,
      "epoch": 11.23,
      "learning_rate": 7.835757575757575e-05,
      "loss": 2.7912,
      "step": 193300
    },
    {
      "activations/layer0_attention_weight_max": 15.899250030517578,
      "activations/layer0_attention_weight_min": -13.0897216796875,
      "activations/layer10_attention_weight_max": 32.706878662109375,
      "activations/layer10_attention_weight_min": -32.38222122192383,
      "activations/layer11_attention_weight_max": 31.644466400146484,
      "activations/layer11_attention_weight_min": -30.383625030517578,
      "activations/layer12_attention_weight_max": 23.568161010742188,
      "activations/layer12_attention_weight_min": -28.231786727905273,
      "activations/layer13_attention_weight_max": 38.41204071044922,
      "activations/layer13_attention_weight_min": -33.03233337402344,
      "activations/layer14_attention_weight_max": 45.85272216796875,
      "activations/layer14_attention_weight_min": -37.71010971069336,
      "activations/layer15_attention_weight_max": 33.835975646972656,
      "activations/layer15_attention_weight_min": -31.411571502685547,
      "activations/layer16_attention_weight_max": 29.87217903137207,
      "activations/layer16_attention_weight_min": -27.571941375732422,
      "activations/layer17_attention_weight_max": 27.590288162231445,
      "activations/layer17_attention_weight_min": -26.533451080322266,
      "activations/layer18_attention_weight_max": 29.723896026611328,
      "activations/layer18_attention_weight_min": -23.401874542236328,
      "activations/layer19_attention_weight_max": 34.37636184692383,
      "activations/layer19_attention_weight_min": -29.53841209411621,
      "activations/layer1_attention_weight_max": 17.25594139099121,
      "activations/layer1_attention_weight_min": -14.28638744354248,
      "activations/layer20_attention_weight_max": 31.718093872070312,
      "activations/layer20_attention_weight_min": -24.309173583984375,
      "activations/layer21_attention_weight_max": 32.74397277832031,
      "activations/layer21_attention_weight_min": -23.429113388061523,
      "activations/layer22_attention_weight_max": 44.1263542175293,
      "activations/layer22_attention_weight_min": -29.787317276000977,
      "activations/layer23_attention_weight_max": 32.35559844970703,
      "activations/layer23_attention_weight_min": -23.29458236694336,
      "activations/layer2_attention_weight_max": 32.15967559814453,
      "activations/layer2_attention_weight_min": -32.370460510253906,
      "activations/layer3_attention_weight_max": 93.01667785644531,
      "activations/layer3_attention_weight_min": -93.07427978515625,
      "activations/layer4_attention_weight_max": 109.20061492919922,
      "activations/layer4_attention_weight_min": -109.68045806884766,
      "activations/layer5_attention_weight_max": 54.59341049194336,
      "activations/layer5_attention_weight_min": -61.38888168334961,
      "activations/layer6_attention_weight_max": 44.816688537597656,
      "activations/layer6_attention_weight_min": -47.30620574951172,
      "activations/layer7_attention_weight_max": 89.28313446044922,
      "activations/layer7_attention_weight_min": -86.82029724121094,
      "activations/layer8_attention_weight_max": 39.290985107421875,
      "activations/layer8_attention_weight_min": -41.31340789794922,
      "activations/layer9_attention_weight_max": 31.091344833374023,
      "activations/layer9_attention_weight_min": -34.623043060302734,
      "epoch": 11.23,
      "learning_rate": 7.833863636363637e-05,
      "loss": 2.7726,
      "step": 193350
    },
    {
      "activations/layer0_attention_weight_max": 14.622208595275879,
      "activations/layer0_attention_weight_min": -13.471238136291504,
      "activations/layer10_attention_weight_max": 33.54035949707031,
      "activations/layer10_attention_weight_min": -34.2574462890625,
      "activations/layer11_attention_weight_max": 33.47730255126953,
      "activations/layer11_attention_weight_min": -33.27992248535156,
      "activations/layer12_attention_weight_max": 29.89415740966797,
      "activations/layer12_attention_weight_min": -35.547203063964844,
      "activations/layer13_attention_weight_max": 36.811439514160156,
      "activations/layer13_attention_weight_min": -32.308128356933594,
      "activations/layer14_attention_weight_max": 43.56223678588867,
      "activations/layer14_attention_weight_min": -36.62693786621094,
      "activations/layer15_attention_weight_max": 35.1837043762207,
      "activations/layer15_attention_weight_min": -31.844154357910156,
      "activations/layer16_attention_weight_max": 35.297821044921875,
      "activations/layer16_attention_weight_min": -27.718185424804688,
      "activations/layer17_attention_weight_max": 32.62166213989258,
      "activations/layer17_attention_weight_min": -26.901376724243164,
      "activations/layer18_attention_weight_max": 29.561019897460938,
      "activations/layer18_attention_weight_min": -22.429851531982422,
      "activations/layer19_attention_weight_max": 40.845481872558594,
      "activations/layer19_attention_weight_min": -34.48643112182617,
      "activations/layer1_attention_weight_max": 16.41658592224121,
      "activations/layer1_attention_weight_min": -14.92748737335205,
      "activations/layer20_attention_weight_max": 30.54714584350586,
      "activations/layer20_attention_weight_min": -25.771257400512695,
      "activations/layer21_attention_weight_max": 28.22084617614746,
      "activations/layer21_attention_weight_min": -23.778701782226562,
      "activations/layer22_attention_weight_max": 41.11848449707031,
      "activations/layer22_attention_weight_min": -30.04005241394043,
      "activations/layer23_attention_weight_max": 33.72711944580078,
      "activations/layer23_attention_weight_min": -25.665184020996094,
      "activations/layer2_attention_weight_max": 33.605777740478516,
      "activations/layer2_attention_weight_min": -33.790042877197266,
      "activations/layer3_attention_weight_max": 96.8919677734375,
      "activations/layer3_attention_weight_min": -99.85799407958984,
      "activations/layer4_attention_weight_max": 114.9166259765625,
      "activations/layer4_attention_weight_min": -114.90771484375,
      "activations/layer5_attention_weight_max": 50.68977355957031,
      "activations/layer5_attention_weight_min": -62.45411682128906,
      "activations/layer6_attention_weight_max": 42.74985885620117,
      "activations/layer6_attention_weight_min": -48.41926574707031,
      "activations/layer7_attention_weight_max": 92.06472778320312,
      "activations/layer7_attention_weight_min": -89.5315933227539,
      "activations/layer8_attention_weight_max": 38.73531723022461,
      "activations/layer8_attention_weight_min": -42.01604461669922,
      "activations/layer9_attention_weight_max": 31.35024070739746,
      "activations/layer9_attention_weight_min": -31.356714248657227,
      "epoch": 11.24,
      "learning_rate": 7.831969696969696e-05,
      "loss": 2.7651,
      "step": 193400
    },
    {
      "activations/layer0_attention_weight_max": 15.015253067016602,
      "activations/layer0_attention_weight_min": -12.74363899230957,
      "activations/layer10_attention_weight_max": 40.442352294921875,
      "activations/layer10_attention_weight_min": -35.47954177856445,
      "activations/layer11_attention_weight_max": 42.12284469604492,
      "activations/layer11_attention_weight_min": -33.78901672363281,
      "activations/layer12_attention_weight_max": 32.6037712097168,
      "activations/layer12_attention_weight_min": -26.450239181518555,
      "activations/layer13_attention_weight_max": 42.5086784362793,
      "activations/layer13_attention_weight_min": -34.036163330078125,
      "activations/layer14_attention_weight_max": 44.274906158447266,
      "activations/layer14_attention_weight_min": -36.46866989135742,
      "activations/layer15_attention_weight_max": 40.748878479003906,
      "activations/layer15_attention_weight_min": -32.371944427490234,
      "activations/layer16_attention_weight_max": 32.311031341552734,
      "activations/layer16_attention_weight_min": -27.20916175842285,
      "activations/layer17_attention_weight_max": 34.351741790771484,
      "activations/layer17_attention_weight_min": -26.35662078857422,
      "activations/layer18_attention_weight_max": 33.8239631652832,
      "activations/layer18_attention_weight_min": -22.86043357849121,
      "activations/layer19_attention_weight_max": 34.154109954833984,
      "activations/layer19_attention_weight_min": -30.217361450195312,
      "activations/layer1_attention_weight_max": 16.729032516479492,
      "activations/layer1_attention_weight_min": -14.039067268371582,
      "activations/layer20_attention_weight_max": 29.868112564086914,
      "activations/layer20_attention_weight_min": -24.419828414916992,
      "activations/layer21_attention_weight_max": 30.414480209350586,
      "activations/layer21_attention_weight_min": -23.155216217041016,
      "activations/layer22_attention_weight_max": 45.60944366455078,
      "activations/layer22_attention_weight_min": -28.79072380065918,
      "activations/layer23_attention_weight_max": 35.192543029785156,
      "activations/layer23_attention_weight_min": -22.512283325195312,
      "activations/layer2_attention_weight_max": 35.88070297241211,
      "activations/layer2_attention_weight_min": -36.29170227050781,
      "activations/layer3_attention_weight_max": 102.25180053710938,
      "activations/layer3_attention_weight_min": -106.01681518554688,
      "activations/layer4_attention_weight_max": 122.911376953125,
      "activations/layer4_attention_weight_min": -126.8179702758789,
      "activations/layer5_attention_weight_max": 58.60566711425781,
      "activations/layer5_attention_weight_min": -66.6270751953125,
      "activations/layer6_attention_weight_max": 47.69139099121094,
      "activations/layer6_attention_weight_min": -48.75857925415039,
      "activations/layer7_attention_weight_max": 113.1633071899414,
      "activations/layer7_attention_weight_min": -93.4056167602539,
      "activations/layer8_attention_weight_max": 46.6697998046875,
      "activations/layer8_attention_weight_min": -45.30557632446289,
      "activations/layer9_attention_weight_max": 39.277828216552734,
      "activations/layer9_attention_weight_min": -36.366214752197266,
      "epoch": 11.24,
      "learning_rate": 7.830075757575757e-05,
      "loss": 2.7795,
      "step": 193450
    },
    {
      "activations/layer0_attention_weight_max": 15.530106544494629,
      "activations/layer0_attention_weight_min": -13.312466621398926,
      "activations/layer10_attention_weight_max": 33.7154541015625,
      "activations/layer10_attention_weight_min": -33.02839660644531,
      "activations/layer11_attention_weight_max": 35.88444900512695,
      "activations/layer11_attention_weight_min": -32.73019027709961,
      "activations/layer12_attention_weight_max": 25.636449813842773,
      "activations/layer12_attention_weight_min": -26.35851287841797,
      "activations/layer13_attention_weight_max": 42.655792236328125,
      "activations/layer13_attention_weight_min": -36.15289306640625,
      "activations/layer14_attention_weight_max": 53.01603698730469,
      "activations/layer14_attention_weight_min": -41.93229293823242,
      "activations/layer15_attention_weight_max": 37.805091857910156,
      "activations/layer15_attention_weight_min": -33.804622650146484,
      "activations/layer16_attention_weight_max": 33.06037902832031,
      "activations/layer16_attention_weight_min": -28.718965530395508,
      "activations/layer17_attention_weight_max": 31.44117546081543,
      "activations/layer17_attention_weight_min": -27.90540885925293,
      "activations/layer18_attention_weight_max": 29.990800857543945,
      "activations/layer18_attention_weight_min": -24.938491821289062,
      "activations/layer19_attention_weight_max": 35.90469741821289,
      "activations/layer19_attention_weight_min": -32.977317810058594,
      "activations/layer1_attention_weight_max": 15.821574211120605,
      "activations/layer1_attention_weight_min": -14.797443389892578,
      "activations/layer20_attention_weight_max": 29.76304817199707,
      "activations/layer20_attention_weight_min": -25.27655601501465,
      "activations/layer21_attention_weight_max": 28.012474060058594,
      "activations/layer21_attention_weight_min": -24.52733612060547,
      "activations/layer22_attention_weight_max": 43.11930847167969,
      "activations/layer22_attention_weight_min": -31.78107261657715,
      "activations/layer23_attention_weight_max": 34.166847229003906,
      "activations/layer23_attention_weight_min": -24.07448959350586,
      "activations/layer2_attention_weight_max": 33.76352310180664,
      "activations/layer2_attention_weight_min": -33.71790313720703,
      "activations/layer3_attention_weight_max": 100.13374328613281,
      "activations/layer3_attention_weight_min": -94.94661712646484,
      "activations/layer4_attention_weight_max": 115.56922912597656,
      "activations/layer4_attention_weight_min": -110.8553237915039,
      "activations/layer5_attention_weight_max": 56.084747314453125,
      "activations/layer5_attention_weight_min": -62.68842315673828,
      "activations/layer6_attention_weight_max": 47.61601257324219,
      "activations/layer6_attention_weight_min": -48.83064270019531,
      "activations/layer7_attention_weight_max": 97.15572357177734,
      "activations/layer7_attention_weight_min": -97.43104553222656,
      "activations/layer8_attention_weight_max": 42.56562042236328,
      "activations/layer8_attention_weight_min": -44.997222900390625,
      "activations/layer9_attention_weight_max": 33.99399948120117,
      "activations/layer9_attention_weight_min": -33.9491081237793,
      "epoch": 11.24,
      "learning_rate": 7.828219696969696e-05,
      "loss": 2.7934,
      "step": 193500
    },
    {
      "activations/layer0_attention_weight_max": 16.10723876953125,
      "activations/layer0_attention_weight_min": -12.313433647155762,
      "activations/layer10_attention_weight_max": 36.541717529296875,
      "activations/layer10_attention_weight_min": -35.34112548828125,
      "activations/layer11_attention_weight_max": 36.16995620727539,
      "activations/layer11_attention_weight_min": -35.125205993652344,
      "activations/layer12_attention_weight_max": 30.628536224365234,
      "activations/layer12_attention_weight_min": -26.581296920776367,
      "activations/layer13_attention_weight_max": 45.63921356201172,
      "activations/layer13_attention_weight_min": -36.21759033203125,
      "activations/layer14_attention_weight_max": 56.41597366333008,
      "activations/layer14_attention_weight_min": -41.42639923095703,
      "activations/layer15_attention_weight_max": 45.45956039428711,
      "activations/layer15_attention_weight_min": -35.51649856567383,
      "activations/layer16_attention_weight_max": 37.13123321533203,
      "activations/layer16_attention_weight_min": -27.701108932495117,
      "activations/layer17_attention_weight_max": 37.94807434082031,
      "activations/layer17_attention_weight_min": -29.35418128967285,
      "activations/layer18_attention_weight_max": 32.055572509765625,
      "activations/layer18_attention_weight_min": -26.91952896118164,
      "activations/layer19_attention_weight_max": 38.51597213745117,
      "activations/layer19_attention_weight_min": -31.91438102722168,
      "activations/layer1_attention_weight_max": 17.22378158569336,
      "activations/layer1_attention_weight_min": -15.18111801147461,
      "activations/layer20_attention_weight_max": 31.77985191345215,
      "activations/layer20_attention_weight_min": -24.993324279785156,
      "activations/layer21_attention_weight_max": 30.00236701965332,
      "activations/layer21_attention_weight_min": -25.943655014038086,
      "activations/layer22_attention_weight_max": 53.92732620239258,
      "activations/layer22_attention_weight_min": -33.54480743408203,
      "activations/layer23_attention_weight_max": 34.9043083190918,
      "activations/layer23_attention_weight_min": -26.35272789001465,
      "activations/layer2_attention_weight_max": 32.70134353637695,
      "activations/layer2_attention_weight_min": -32.673683166503906,
      "activations/layer3_attention_weight_max": 89.829833984375,
      "activations/layer3_attention_weight_min": -93.01878356933594,
      "activations/layer4_attention_weight_max": 110.0027084350586,
      "activations/layer4_attention_weight_min": -112.28779602050781,
      "activations/layer5_attention_weight_max": 51.89569091796875,
      "activations/layer5_attention_weight_min": -62.58367919921875,
      "activations/layer6_attention_weight_max": 45.696739196777344,
      "activations/layer6_attention_weight_min": -46.991004943847656,
      "activations/layer7_attention_weight_max": 92.1257553100586,
      "activations/layer7_attention_weight_min": -93.71128845214844,
      "activations/layer8_attention_weight_max": 44.945152282714844,
      "activations/layer8_attention_weight_min": -45.31694793701172,
      "activations/layer9_attention_weight_max": 37.961238861083984,
      "activations/layer9_attention_weight_min": -36.05121994018555,
      "epoch": 11.25,
      "learning_rate": 7.826325757575758e-05,
      "loss": 2.771,
      "step": 193550
    },
    {
      "activations/layer0_attention_weight_max": 15.609174728393555,
      "activations/layer0_attention_weight_min": -13.168088912963867,
      "activations/layer10_attention_weight_max": 46.21826171875,
      "activations/layer10_attention_weight_min": -41.74057388305664,
      "activations/layer11_attention_weight_max": 48.27073669433594,
      "activations/layer11_attention_weight_min": -43.78181076049805,
      "activations/layer12_attention_weight_max": 28.268089294433594,
      "activations/layer12_attention_weight_min": -29.668821334838867,
      "activations/layer13_attention_weight_max": 43.970680236816406,
      "activations/layer13_attention_weight_min": -38.17584991455078,
      "activations/layer14_attention_weight_max": 51.19053649902344,
      "activations/layer14_attention_weight_min": -38.840736389160156,
      "activations/layer15_attention_weight_max": 42.48582077026367,
      "activations/layer15_attention_weight_min": -35.04422378540039,
      "activations/layer16_attention_weight_max": 34.342796325683594,
      "activations/layer16_attention_weight_min": -28.10523223876953,
      "activations/layer17_attention_weight_max": 29.35173988342285,
      "activations/layer17_attention_weight_min": -26.841869354248047,
      "activations/layer18_attention_weight_max": 29.32184410095215,
      "activations/layer18_attention_weight_min": -24.45587158203125,
      "activations/layer19_attention_weight_max": 39.01163864135742,
      "activations/layer19_attention_weight_min": -31.342636108398438,
      "activations/layer1_attention_weight_max": 16.833528518676758,
      "activations/layer1_attention_weight_min": -15.313640594482422,
      "activations/layer20_attention_weight_max": 32.774208068847656,
      "activations/layer20_attention_weight_min": -26.386577606201172,
      "activations/layer21_attention_weight_max": 29.47758674621582,
      "activations/layer21_attention_weight_min": -23.892820358276367,
      "activations/layer22_attention_weight_max": 47.87100601196289,
      "activations/layer22_attention_weight_min": -30.679746627807617,
      "activations/layer23_attention_weight_max": 40.06084442138672,
      "activations/layer23_attention_weight_min": -24.460121154785156,
      "activations/layer2_attention_weight_max": 35.966102600097656,
      "activations/layer2_attention_weight_min": -38.11314392089844,
      "activations/layer3_attention_weight_max": 107.83329010009766,
      "activations/layer3_attention_weight_min": -111.32625579833984,
      "activations/layer4_attention_weight_max": 129.47802734375,
      "activations/layer4_attention_weight_min": -118.53507232666016,
      "activations/layer5_attention_weight_max": 65.63677978515625,
      "activations/layer5_attention_weight_min": -61.326263427734375,
      "activations/layer6_attention_weight_max": 52.79169464111328,
      "activations/layer6_attention_weight_min": -47.657413482666016,
      "activations/layer7_attention_weight_max": 115.1656723022461,
      "activations/layer7_attention_weight_min": -104.6136245727539,
      "activations/layer8_attention_weight_max": 52.98166275024414,
      "activations/layer8_attention_weight_min": -46.67034149169922,
      "activations/layer9_attention_weight_max": 42.96866989135742,
      "activations/layer9_attention_weight_min": -41.14326095581055,
      "epoch": 11.25,
      "learning_rate": 7.824431818181818e-05,
      "loss": 2.785,
      "step": 193600
    },
    {
      "activations/layer0_attention_weight_max": 15.450299263000488,
      "activations/layer0_attention_weight_min": -12.313224792480469,
      "activations/layer10_attention_weight_max": 32.479949951171875,
      "activations/layer10_attention_weight_min": -30.519567489624023,
      "activations/layer11_attention_weight_max": 32.183937072753906,
      "activations/layer11_attention_weight_min": -30.328388214111328,
      "activations/layer12_attention_weight_max": 24.496517181396484,
      "activations/layer12_attention_weight_min": -25.454748153686523,
      "activations/layer13_attention_weight_max": 35.79619598388672,
      "activations/layer13_attention_weight_min": -33.47030258178711,
      "activations/layer14_attention_weight_max": 43.70585250854492,
      "activations/layer14_attention_weight_min": -37.024383544921875,
      "activations/layer15_attention_weight_max": 33.81625747680664,
      "activations/layer15_attention_weight_min": -29.78915786743164,
      "activations/layer16_attention_weight_max": 28.527387619018555,
      "activations/layer16_attention_weight_min": -27.325223922729492,
      "activations/layer17_attention_weight_max": 32.175537109375,
      "activations/layer17_attention_weight_min": -25.44234275817871,
      "activations/layer18_attention_weight_max": 31.707481384277344,
      "activations/layer18_attention_weight_min": -23.877117156982422,
      "activations/layer19_attention_weight_max": 34.07503890991211,
      "activations/layer19_attention_weight_min": -32.206703186035156,
      "activations/layer1_attention_weight_max": 16.493608474731445,
      "activations/layer1_attention_weight_min": -15.276823043823242,
      "activations/layer20_attention_weight_max": 28.411165237426758,
      "activations/layer20_attention_weight_min": -25.149364471435547,
      "activations/layer21_attention_weight_max": 28.951969146728516,
      "activations/layer21_attention_weight_min": -25.6301326751709,
      "activations/layer22_attention_weight_max": 47.08109664916992,
      "activations/layer22_attention_weight_min": -27.663463592529297,
      "activations/layer23_attention_weight_max": 33.503746032714844,
      "activations/layer23_attention_weight_min": -24.838964462280273,
      "activations/layer2_attention_weight_max": 33.79631805419922,
      "activations/layer2_attention_weight_min": -34.27888488769531,
      "activations/layer3_attention_weight_max": 100.02251434326172,
      "activations/layer3_attention_weight_min": -97.16401672363281,
      "activations/layer4_attention_weight_max": 116.68636322021484,
      "activations/layer4_attention_weight_min": -111.12639617919922,
      "activations/layer5_attention_weight_max": 51.97008514404297,
      "activations/layer5_attention_weight_min": -60.219627380371094,
      "activations/layer6_attention_weight_max": 46.64147186279297,
      "activations/layer6_attention_weight_min": -47.37491226196289,
      "activations/layer7_attention_weight_max": 86.9909896850586,
      "activations/layer7_attention_weight_min": -89.9557876586914,
      "activations/layer8_attention_weight_max": 39.25147247314453,
      "activations/layer8_attention_weight_min": -40.62569046020508,
      "activations/layer9_attention_weight_max": 32.4220085144043,
      "activations/layer9_attention_weight_min": -31.24688148498535,
      "epoch": 11.25,
      "learning_rate": 7.822537878787878e-05,
      "loss": 2.766,
      "step": 193650
    },
    {
      "activations/layer0_attention_weight_max": 15.815532684326172,
      "activations/layer0_attention_weight_min": -12.408703804016113,
      "activations/layer10_attention_weight_max": 33.8790397644043,
      "activations/layer10_attention_weight_min": -33.308937072753906,
      "activations/layer11_attention_weight_max": 34.835548400878906,
      "activations/layer11_attention_weight_min": -31.460966110229492,
      "activations/layer12_attention_weight_max": 27.520191192626953,
      "activations/layer12_attention_weight_min": -26.45602798461914,
      "activations/layer13_attention_weight_max": 43.56483459472656,
      "activations/layer13_attention_weight_min": -34.99845886230469,
      "activations/layer14_attention_weight_max": 48.68365478515625,
      "activations/layer14_attention_weight_min": -38.981895446777344,
      "activations/layer15_attention_weight_max": 38.65673065185547,
      "activations/layer15_attention_weight_min": -33.081485748291016,
      "activations/layer16_attention_weight_max": 34.57640838623047,
      "activations/layer16_attention_weight_min": -28.151155471801758,
      "activations/layer17_attention_weight_max": 37.00566101074219,
      "activations/layer17_attention_weight_min": -26.246084213256836,
      "activations/layer18_attention_weight_max": 34.246307373046875,
      "activations/layer18_attention_weight_min": -24.1546573638916,
      "activations/layer19_attention_weight_max": 45.92695617675781,
      "activations/layer19_attention_weight_min": -31.990076065063477,
      "activations/layer1_attention_weight_max": 16.499622344970703,
      "activations/layer1_attention_weight_min": -14.441290855407715,
      "activations/layer20_attention_weight_max": 37.932090759277344,
      "activations/layer20_attention_weight_min": -25.594114303588867,
      "activations/layer21_attention_weight_max": 39.114234924316406,
      "activations/layer21_attention_weight_min": -23.803264617919922,
      "activations/layer22_attention_weight_max": 55.70448303222656,
      "activations/layer22_attention_weight_min": -29.19139289855957,
      "activations/layer23_attention_weight_max": 44.99896240234375,
      "activations/layer23_attention_weight_min": -23.916072845458984,
      "activations/layer2_attention_weight_max": 32.90086364746094,
      "activations/layer2_attention_weight_min": -33.48133850097656,
      "activations/layer3_attention_weight_max": 96.94890594482422,
      "activations/layer3_attention_weight_min": -96.86650848388672,
      "activations/layer4_attention_weight_max": 111.0441665649414,
      "activations/layer4_attention_weight_min": -114.80834197998047,
      "activations/layer5_attention_weight_max": 52.212852478027344,
      "activations/layer5_attention_weight_min": -64.81456756591797,
      "activations/layer6_attention_weight_max": 44.290531158447266,
      "activations/layer6_attention_weight_min": -48.30576705932617,
      "activations/layer7_attention_weight_max": 93.47274780273438,
      "activations/layer7_attention_weight_min": -89.8301010131836,
      "activations/layer8_attention_weight_max": 41.84260940551758,
      "activations/layer8_attention_weight_min": -43.05156326293945,
      "activations/layer9_attention_weight_max": 33.496952056884766,
      "activations/layer9_attention_weight_min": -33.84452438354492,
      "epoch": 11.26,
      "learning_rate": 7.820643939393938e-05,
      "loss": 2.7767,
      "step": 193700
    },
    {
      "activations/layer0_attention_weight_max": 16.06096076965332,
      "activations/layer0_attention_weight_min": -12.365631103515625,
      "activations/layer10_attention_weight_max": 31.361478805541992,
      "activations/layer10_attention_weight_min": -31.401945114135742,
      "activations/layer11_attention_weight_max": 31.99209976196289,
      "activations/layer11_attention_weight_min": -29.457931518554688,
      "activations/layer12_attention_weight_max": 25.36619758605957,
      "activations/layer12_attention_weight_min": -24.213693618774414,
      "activations/layer13_attention_weight_max": 39.40558624267578,
      "activations/layer13_attention_weight_min": -35.313140869140625,
      "activations/layer14_attention_weight_max": 43.46623611450195,
      "activations/layer14_attention_weight_min": -38.25240707397461,
      "activations/layer15_attention_weight_max": 34.507938385009766,
      "activations/layer15_attention_weight_min": -32.487571716308594,
      "activations/layer16_attention_weight_max": 30.55330467224121,
      "activations/layer16_attention_weight_min": -27.93524169921875,
      "activations/layer17_attention_weight_max": 30.95914649963379,
      "activations/layer17_attention_weight_min": -27.672378540039062,
      "activations/layer18_attention_weight_max": 33.846519470214844,
      "activations/layer18_attention_weight_min": -23.271814346313477,
      "activations/layer19_attention_weight_max": 37.54708480834961,
      "activations/layer19_attention_weight_min": -30.817020416259766,
      "activations/layer1_attention_weight_max": 16.716690063476562,
      "activations/layer1_attention_weight_min": -14.859682083129883,
      "activations/layer20_attention_weight_max": 28.817209243774414,
      "activations/layer20_attention_weight_min": -24.26042366027832,
      "activations/layer21_attention_weight_max": 27.507980346679688,
      "activations/layer21_attention_weight_min": -24.93223762512207,
      "activations/layer22_attention_weight_max": 43.53097152709961,
      "activations/layer22_attention_weight_min": -30.95279884338379,
      "activations/layer23_attention_weight_max": 33.12422561645508,
      "activations/layer23_attention_weight_min": -26.326946258544922,
      "activations/layer2_attention_weight_max": 32.74546813964844,
      "activations/layer2_attention_weight_min": -32.02412796020508,
      "activations/layer3_attention_weight_max": 93.4433822631836,
      "activations/layer3_attention_weight_min": -95.1917495727539,
      "activations/layer4_attention_weight_max": 111.64237213134766,
      "activations/layer4_attention_weight_min": -117.08955383300781,
      "activations/layer5_attention_weight_max": 52.2161979675293,
      "activations/layer5_attention_weight_min": -61.63557434082031,
      "activations/layer6_attention_weight_max": 44.497798919677734,
      "activations/layer6_attention_weight_min": -50.03383255004883,
      "activations/layer7_attention_weight_max": 90.0539779663086,
      "activations/layer7_attention_weight_min": -90.06945037841797,
      "activations/layer8_attention_weight_max": 39.2393913269043,
      "activations/layer8_attention_weight_min": -43.19315719604492,
      "activations/layer9_attention_weight_max": 32.36463165283203,
      "activations/layer9_attention_weight_min": -33.20991516113281,
      "epoch": 11.26,
      "learning_rate": 7.81875e-05,
      "loss": 2.7739,
      "step": 193750
    },
    {
      "activations/layer0_attention_weight_max": 15.31784725189209,
      "activations/layer0_attention_weight_min": -12.40473747253418,
      "activations/layer10_attention_weight_max": 30.562551498413086,
      "activations/layer10_attention_weight_min": -30.971012115478516,
      "activations/layer11_attention_weight_max": 31.183238983154297,
      "activations/layer11_attention_weight_min": -29.81393051147461,
      "activations/layer12_attention_weight_max": 23.683612823486328,
      "activations/layer12_attention_weight_min": -23.465927124023438,
      "activations/layer13_attention_weight_max": 42.207908630371094,
      "activations/layer13_attention_weight_min": -33.32022476196289,
      "activations/layer14_attention_weight_max": 42.675907135009766,
      "activations/layer14_attention_weight_min": -36.53321838378906,
      "activations/layer15_attention_weight_max": 36.867340087890625,
      "activations/layer15_attention_weight_min": -31.87588882446289,
      "activations/layer16_attention_weight_max": 31.38899803161621,
      "activations/layer16_attention_weight_min": -27.937847137451172,
      "activations/layer17_attention_weight_max": 31.039026260375977,
      "activations/layer17_attention_weight_min": -24.712656021118164,
      "activations/layer18_attention_weight_max": 30.85908317565918,
      "activations/layer18_attention_weight_min": -25.532615661621094,
      "activations/layer19_attention_weight_max": 33.76401901245117,
      "activations/layer19_attention_weight_min": -30.153995513916016,
      "activations/layer1_attention_weight_max": 16.39300537109375,
      "activations/layer1_attention_weight_min": -14.689193725585938,
      "activations/layer20_attention_weight_max": 31.33791732788086,
      "activations/layer20_attention_weight_min": -23.536046981811523,
      "activations/layer21_attention_weight_max": 32.83308029174805,
      "activations/layer21_attention_weight_min": -24.08019256591797,
      "activations/layer22_attention_weight_max": 48.665855407714844,
      "activations/layer22_attention_weight_min": -31.72587776184082,
      "activations/layer23_attention_weight_max": 39.33129119873047,
      "activations/layer23_attention_weight_min": -25.45857810974121,
      "activations/layer2_attention_weight_max": 32.631439208984375,
      "activations/layer2_attention_weight_min": -32.60342788696289,
      "activations/layer3_attention_weight_max": 94.1305160522461,
      "activations/layer3_attention_weight_min": -92.34967803955078,
      "activations/layer4_attention_weight_max": 112.91021728515625,
      "activations/layer4_attention_weight_min": -115.2046890258789,
      "activations/layer5_attention_weight_max": 50.113868713378906,
      "activations/layer5_attention_weight_min": -66.37354278564453,
      "activations/layer6_attention_weight_max": 45.40574264526367,
      "activations/layer6_attention_weight_min": -48.79765319824219,
      "activations/layer7_attention_weight_max": 85.6380386352539,
      "activations/layer7_attention_weight_min": -94.1530990600586,
      "activations/layer8_attention_weight_max": 38.77014923095703,
      "activations/layer8_attention_weight_min": -40.29973220825195,
      "activations/layer9_attention_weight_max": 32.04704666137695,
      "activations/layer9_attention_weight_min": -32.06748580932617,
      "epoch": 11.26,
      "learning_rate": 7.81685606060606e-05,
      "loss": 2.7881,
      "step": 193800
    },
    {
      "activations/layer0_attention_weight_max": 15.497693061828613,
      "activations/layer0_attention_weight_min": -12.46396255493164,
      "activations/layer10_attention_weight_max": 30.825387954711914,
      "activations/layer10_attention_weight_min": -32.507667541503906,
      "activations/layer11_attention_weight_max": 29.79956817626953,
      "activations/layer11_attention_weight_min": -29.52297592163086,
      "activations/layer12_attention_weight_max": 23.598690032958984,
      "activations/layer12_attention_weight_min": -22.82440757751465,
      "activations/layer13_attention_weight_max": 34.65868377685547,
      "activations/layer13_attention_weight_min": -31.970947265625,
      "activations/layer14_attention_weight_max": 39.23514175415039,
      "activations/layer14_attention_weight_min": -34.99055480957031,
      "activations/layer15_attention_weight_max": 33.430694580078125,
      "activations/layer15_attention_weight_min": -30.01314926147461,
      "activations/layer16_attention_weight_max": 28.787574768066406,
      "activations/layer16_attention_weight_min": -26.198089599609375,
      "activations/layer17_attention_weight_max": 32.902488708496094,
      "activations/layer17_attention_weight_min": -24.202219009399414,
      "activations/layer18_attention_weight_max": 28.788066864013672,
      "activations/layer18_attention_weight_min": -21.978404998779297,
      "activations/layer19_attention_weight_max": 35.70630645751953,
      "activations/layer19_attention_weight_min": -30.177993774414062,
      "activations/layer1_attention_weight_max": 16.801258087158203,
      "activations/layer1_attention_weight_min": -14.097877502441406,
      "activations/layer20_attention_weight_max": 28.245195388793945,
      "activations/layer20_attention_weight_min": -24.005340576171875,
      "activations/layer21_attention_weight_max": 27.952789306640625,
      "activations/layer21_attention_weight_min": -24.666826248168945,
      "activations/layer22_attention_weight_max": 40.49687194824219,
      "activations/layer22_attention_weight_min": -28.266605377197266,
      "activations/layer23_attention_weight_max": 34.77699279785156,
      "activations/layer23_attention_weight_min": -25.330554962158203,
      "activations/layer2_attention_weight_max": 33.65372085571289,
      "activations/layer2_attention_weight_min": -33.507781982421875,
      "activations/layer3_attention_weight_max": 95.21524047851562,
      "activations/layer3_attention_weight_min": -96.59259033203125,
      "activations/layer4_attention_weight_max": 110.52593231201172,
      "activations/layer4_attention_weight_min": -112.4430160522461,
      "activations/layer5_attention_weight_max": 51.273902893066406,
      "activations/layer5_attention_weight_min": -64.08304595947266,
      "activations/layer6_attention_weight_max": 43.2437858581543,
      "activations/layer6_attention_weight_min": -47.88790512084961,
      "activations/layer7_attention_weight_max": 91.60088348388672,
      "activations/layer7_attention_weight_min": -91.40373992919922,
      "activations/layer8_attention_weight_max": 37.6844596862793,
      "activations/layer8_attention_weight_min": -39.94148254394531,
      "activations/layer9_attention_weight_max": 30.40618324279785,
      "activations/layer9_attention_weight_min": -30.50348472595215,
      "epoch": 11.26,
      "learning_rate": 7.81496212121212e-05,
      "loss": 2.7867,
      "step": 193850
    },
    {
      "activations/layer0_attention_weight_max": 15.482892036437988,
      "activations/layer0_attention_weight_min": -12.22449016571045,
      "activations/layer10_attention_weight_max": 30.47667121887207,
      "activations/layer10_attention_weight_min": -31.02573585510254,
      "activations/layer11_attention_weight_max": 30.95558738708496,
      "activations/layer11_attention_weight_min": -29.38831901550293,
      "activations/layer12_attention_weight_max": 23.9776611328125,
      "activations/layer12_attention_weight_min": -23.726871490478516,
      "activations/layer13_attention_weight_max": 37.46834182739258,
      "activations/layer13_attention_weight_min": -34.110321044921875,
      "activations/layer14_attention_weight_max": 41.646644592285156,
      "activations/layer14_attention_weight_min": -34.660579681396484,
      "activations/layer15_attention_weight_max": 36.944854736328125,
      "activations/layer15_attention_weight_min": -31.69416618347168,
      "activations/layer16_attention_weight_max": 30.34491729736328,
      "activations/layer16_attention_weight_min": -28.53997230529785,
      "activations/layer17_attention_weight_max": 29.300302505493164,
      "activations/layer17_attention_weight_min": -25.449447631835938,
      "activations/layer18_attention_weight_max": 29.157018661499023,
      "activations/layer18_attention_weight_min": -25.024425506591797,
      "activations/layer19_attention_weight_max": 34.898895263671875,
      "activations/layer19_attention_weight_min": -28.568267822265625,
      "activations/layer1_attention_weight_max": 16.36881446838379,
      "activations/layer1_attention_weight_min": -13.691567420959473,
      "activations/layer20_attention_weight_max": 29.2898006439209,
      "activations/layer20_attention_weight_min": -24.116456985473633,
      "activations/layer21_attention_weight_max": 28.081268310546875,
      "activations/layer21_attention_weight_min": -24.20315933227539,
      "activations/layer22_attention_weight_max": 48.205875396728516,
      "activations/layer22_attention_weight_min": -28.728485107421875,
      "activations/layer23_attention_weight_max": 35.34614562988281,
      "activations/layer23_attention_weight_min": -25.533763885498047,
      "activations/layer2_attention_weight_max": 33.08127212524414,
      "activations/layer2_attention_weight_min": -32.6735725402832,
      "activations/layer3_attention_weight_max": 93.38985443115234,
      "activations/layer3_attention_weight_min": -94.65277099609375,
      "activations/layer4_attention_weight_max": 112.44654846191406,
      "activations/layer4_attention_weight_min": -111.55834197998047,
      "activations/layer5_attention_weight_max": 51.93011474609375,
      "activations/layer5_attention_weight_min": -63.918357849121094,
      "activations/layer6_attention_weight_max": 43.116329193115234,
      "activations/layer6_attention_weight_min": -45.391719818115234,
      "activations/layer7_attention_weight_max": 83.83101654052734,
      "activations/layer7_attention_weight_min": -84.97261047363281,
      "activations/layer8_attention_weight_max": 39.9921989440918,
      "activations/layer8_attention_weight_min": -37.20540237426758,
      "activations/layer9_attention_weight_max": 31.332656860351562,
      "activations/layer9_attention_weight_min": -30.568851470947266,
      "epoch": 11.27,
      "learning_rate": 7.813068181818181e-05,
      "loss": 2.7877,
      "step": 193900
    },
    {
      "activations/layer0_attention_weight_max": 16.085174560546875,
      "activations/layer0_attention_weight_min": -12.8050537109375,
      "activations/layer10_attention_weight_max": 32.569091796875,
      "activations/layer10_attention_weight_min": -32.690345764160156,
      "activations/layer11_attention_weight_max": 34.14115905761719,
      "activations/layer11_attention_weight_min": -33.814022064208984,
      "activations/layer12_attention_weight_max": 37.990596771240234,
      "activations/layer12_attention_weight_min": -24.87714385986328,
      "activations/layer13_attention_weight_max": 44.74901580810547,
      "activations/layer13_attention_weight_min": -35.869258880615234,
      "activations/layer14_attention_weight_max": 44.524845123291016,
      "activations/layer14_attention_weight_min": -39.51808166503906,
      "activations/layer15_attention_weight_max": 38.39152908325195,
      "activations/layer15_attention_weight_min": -32.60163879394531,
      "activations/layer16_attention_weight_max": 34.742042541503906,
      "activations/layer16_attention_weight_min": -27.033878326416016,
      "activations/layer17_attention_weight_max": 33.85096740722656,
      "activations/layer17_attention_weight_min": -26.385417938232422,
      "activations/layer18_attention_weight_max": 34.20845413208008,
      "activations/layer18_attention_weight_min": -24.807247161865234,
      "activations/layer19_attention_weight_max": 38.76980209350586,
      "activations/layer19_attention_weight_min": -30.8210506439209,
      "activations/layer1_attention_weight_max": 16.951322555541992,
      "activations/layer1_attention_weight_min": -16.85930633544922,
      "activations/layer20_attention_weight_max": 31.385923385620117,
      "activations/layer20_attention_weight_min": -25.25078773498535,
      "activations/layer21_attention_weight_max": 33.42226791381836,
      "activations/layer21_attention_weight_min": -25.221311569213867,
      "activations/layer22_attention_weight_max": 45.85064697265625,
      "activations/layer22_attention_weight_min": -32.66997528076172,
      "activations/layer23_attention_weight_max": 38.19708251953125,
      "activations/layer23_attention_weight_min": -27.456745147705078,
      "activations/layer2_attention_weight_max": 35.067867279052734,
      "activations/layer2_attention_weight_min": -33.657203674316406,
      "activations/layer3_attention_weight_max": 94.21607971191406,
      "activations/layer3_attention_weight_min": -98.48836517333984,
      "activations/layer4_attention_weight_max": 112.72908020019531,
      "activations/layer4_attention_weight_min": -112.00579071044922,
      "activations/layer5_attention_weight_max": 50.92611312866211,
      "activations/layer5_attention_weight_min": -63.76432418823242,
      "activations/layer6_attention_weight_max": 46.20467758178711,
      "activations/layer6_attention_weight_min": -47.796119689941406,
      "activations/layer7_attention_weight_max": 88.57524871826172,
      "activations/layer7_attention_weight_min": -87.60228729248047,
      "activations/layer8_attention_weight_max": 41.82651901245117,
      "activations/layer8_attention_weight_min": -45.38474655151367,
      "activations/layer9_attention_weight_max": 32.975181579589844,
      "activations/layer9_attention_weight_min": -35.323028564453125,
      "epoch": 11.27,
      "learning_rate": 7.811174242424243e-05,
      "loss": 2.7815,
      "step": 193950
    },
    {
      "activations/layer0_attention_weight_max": 16.030139923095703,
      "activations/layer0_attention_weight_min": -12.718317031860352,
      "activations/layer10_attention_weight_max": 34.43426513671875,
      "activations/layer10_attention_weight_min": -35.56150436401367,
      "activations/layer11_attention_weight_max": 34.47212600708008,
      "activations/layer11_attention_weight_min": -33.07062911987305,
      "activations/layer12_attention_weight_max": 26.40515899658203,
      "activations/layer12_attention_weight_min": -25.62141227722168,
      "activations/layer13_attention_weight_max": 42.5443229675293,
      "activations/layer13_attention_weight_min": -32.83771514892578,
      "activations/layer14_attention_weight_max": 43.20228576660156,
      "activations/layer14_attention_weight_min": -36.2962646484375,
      "activations/layer15_attention_weight_max": 34.93092727661133,
      "activations/layer15_attention_weight_min": -30.904647827148438,
      "activations/layer16_attention_weight_max": 30.137378692626953,
      "activations/layer16_attention_weight_min": -26.750099182128906,
      "activations/layer17_attention_weight_max": 33.52507781982422,
      "activations/layer17_attention_weight_min": -25.35391616821289,
      "activations/layer18_attention_weight_max": 29.456762313842773,
      "activations/layer18_attention_weight_min": -22.161800384521484,
      "activations/layer19_attention_weight_max": 37.50123977661133,
      "activations/layer19_attention_weight_min": -29.95945167541504,
      "activations/layer1_attention_weight_max": 16.797443389892578,
      "activations/layer1_attention_weight_min": -14.325587272644043,
      "activations/layer20_attention_weight_max": 31.929946899414062,
      "activations/layer20_attention_weight_min": -23.986278533935547,
      "activations/layer21_attention_weight_max": 30.742136001586914,
      "activations/layer21_attention_weight_min": -23.382932662963867,
      "activations/layer22_attention_weight_max": 56.10818862915039,
      "activations/layer22_attention_weight_min": -30.535884857177734,
      "activations/layer23_attention_weight_max": 40.83978271484375,
      "activations/layer23_attention_weight_min": -23.441326141357422,
      "activations/layer2_attention_weight_max": 33.99580764770508,
      "activations/layer2_attention_weight_min": -35.307456970214844,
      "activations/layer3_attention_weight_max": 102.24078369140625,
      "activations/layer3_attention_weight_min": -102.88928985595703,
      "activations/layer4_attention_weight_max": 120.21687316894531,
      "activations/layer4_attention_weight_min": -119.51656341552734,
      "activations/layer5_attention_weight_max": 53.92914962768555,
      "activations/layer5_attention_weight_min": -63.898460388183594,
      "activations/layer6_attention_weight_max": 44.9804573059082,
      "activations/layer6_attention_weight_min": -47.84139633178711,
      "activations/layer7_attention_weight_max": 97.70376586914062,
      "activations/layer7_attention_weight_min": -98.5892333984375,
      "activations/layer8_attention_weight_max": 41.79378128051758,
      "activations/layer8_attention_weight_min": -42.74739456176758,
      "activations/layer9_attention_weight_max": 34.73463439941406,
      "activations/layer9_attention_weight_min": -36.86198043823242,
      "epoch": 11.27,
      "learning_rate": 7.809280303030302e-05,
      "loss": 2.7765,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.4858,
      "eval_samples_per_second": 506.025,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.4858,
      "eval_openwebtext_samples_per_second": 506.025,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 2.0229,
      "eval_wikitext_samples_per_second": 225.422,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_lambada_loss": 2.572265625,
      "eval_lambada_ppl": 13.095460258581369,
      "eval_lambada_runtime": 9.5725,
      "eval_lambada_samples_per_second": 508.644,
      "step": 194000
    },
    {
      "activations/layer0_attention_weight_max": 16.0797119140625,
      "activations/layer0_attention_weight_min": -12.47205924987793,
      "activations/layer10_attention_weight_max": 36.142005920410156,
      "activations/layer10_attention_weight_min": -37.242279052734375,
      "activations/layer11_attention_weight_max": 36.38066101074219,
      "activations/layer11_attention_weight_min": -36.30601119995117,
      "activations/layer12_attention_weight_max": 24.76647186279297,
      "activations/layer12_attention_weight_min": -24.93703842163086,
      "activations/layer13_attention_weight_max": 35.93587875366211,
      "activations/layer13_attention_weight_min": -35.880615234375,
      "activations/layer14_attention_weight_max": 41.603179931640625,
      "activations/layer14_attention_weight_min": -36.966739654541016,
      "activations/layer15_attention_weight_max": 35.7049446105957,
      "activations/layer15_attention_weight_min": -30.632328033447266,
      "activations/layer16_attention_weight_max": 30.63245391845703,
      "activations/layer16_attention_weight_min": -26.62018585205078,
      "activations/layer17_attention_weight_max": 28.703081130981445,
      "activations/layer17_attention_weight_min": -24.925888061523438,
      "activations/layer18_attention_weight_max": 30.238548278808594,
      "activations/layer18_attention_weight_min": -21.96436309814453,
      "activations/layer19_attention_weight_max": 34.66481399536133,
      "activations/layer19_attention_weight_min": -28.89198875427246,
      "activations/layer1_attention_weight_max": 16.982669830322266,
      "activations/layer1_attention_weight_min": -15.189105033874512,
      "activations/layer20_attention_weight_max": 27.572038650512695,
      "activations/layer20_attention_weight_min": -24.48565673828125,
      "activations/layer21_attention_weight_max": 30.56915283203125,
      "activations/layer21_attention_weight_min": -21.43560791015625,
      "activations/layer22_attention_weight_max": 46.44294738769531,
      "activations/layer22_attention_weight_min": -28.20733642578125,
      "activations/layer23_attention_weight_max": 34.028446197509766,
      "activations/layer23_attention_weight_min": -22.388763427734375,
      "activations/layer2_attention_weight_max": 33.08241271972656,
      "activations/layer2_attention_weight_min": -33.40220642089844,
      "activations/layer3_attention_weight_max": 97.0271224975586,
      "activations/layer3_attention_weight_min": -98.47827911376953,
      "activations/layer4_attention_weight_max": 117.1507568359375,
      "activations/layer4_attention_weight_min": -113.36222076416016,
      "activations/layer5_attention_weight_max": 54.40769577026367,
      "activations/layer5_attention_weight_min": -62.45237350463867,
      "activations/layer6_attention_weight_max": 46.239112854003906,
      "activations/layer6_attention_weight_min": -49.425331115722656,
      "activations/layer7_attention_weight_max": 93.10189819335938,
      "activations/layer7_attention_weight_min": -93.26434326171875,
      "activations/layer8_attention_weight_max": 42.766719818115234,
      "activations/layer8_attention_weight_min": -46.974510192871094,
      "activations/layer9_attention_weight_max": 37.83561325073242,
      "activations/layer9_attention_weight_min": -36.66600799560547,
      "epoch": 11.28,
      "learning_rate": 7.807386363636363e-05,
      "loss": 2.7809,
      "step": 194050
    },
    {
      "activations/layer0_attention_weight_max": 16.993213653564453,
      "activations/layer0_attention_weight_min": -13.284375190734863,
      "activations/layer10_attention_weight_max": 35.52859115600586,
      "activations/layer10_attention_weight_min": -37.780025482177734,
      "activations/layer11_attention_weight_max": 33.93461227416992,
      "activations/layer11_attention_weight_min": -35.6527099609375,
      "activations/layer12_attention_weight_max": 32.142791748046875,
      "activations/layer12_attention_weight_min": -26.06949234008789,
      "activations/layer13_attention_weight_max": 35.99037170410156,
      "activations/layer13_attention_weight_min": -33.64197540283203,
      "activations/layer14_attention_weight_max": 42.842044830322266,
      "activations/layer14_attention_weight_min": -39.14859390258789,
      "activations/layer15_attention_weight_max": 36.5340461730957,
      "activations/layer15_attention_weight_min": -33.95594787597656,
      "activations/layer16_attention_weight_max": 32.013092041015625,
      "activations/layer16_attention_weight_min": -27.523088455200195,
      "activations/layer17_attention_weight_max": 30.64227294921875,
      "activations/layer17_attention_weight_min": -24.806941986083984,
      "activations/layer18_attention_weight_max": 30.903583526611328,
      "activations/layer18_attention_weight_min": -23.330368041992188,
      "activations/layer19_attention_weight_max": 33.576011657714844,
      "activations/layer19_attention_weight_min": -30.4466495513916,
      "activations/layer1_attention_weight_max": 16.35438346862793,
      "activations/layer1_attention_weight_min": -14.355586051940918,
      "activations/layer20_attention_weight_max": 28.389198303222656,
      "activations/layer20_attention_weight_min": -24.35580062866211,
      "activations/layer21_attention_weight_max": 29.313169479370117,
      "activations/layer21_attention_weight_min": -22.9952392578125,
      "activations/layer22_attention_weight_max": 42.9210205078125,
      "activations/layer22_attention_weight_min": -28.35500144958496,
      "activations/layer23_attention_weight_max": 37.2445068359375,
      "activations/layer23_attention_weight_min": -24.52669906616211,
      "activations/layer2_attention_weight_max": 34.58751678466797,
      "activations/layer2_attention_weight_min": -35.1656494140625,
      "activations/layer3_attention_weight_max": 101.731201171875,
      "activations/layer3_attention_weight_min": -106.49964141845703,
      "activations/layer4_attention_weight_max": 115.49029541015625,
      "activations/layer4_attention_weight_min": -117.49800872802734,
      "activations/layer5_attention_weight_max": 55.458309173583984,
      "activations/layer5_attention_weight_min": -65.47245788574219,
      "activations/layer6_attention_weight_max": 46.51441192626953,
      "activations/layer6_attention_weight_min": -52.193485260009766,
      "activations/layer7_attention_weight_max": 95.22663879394531,
      "activations/layer7_attention_weight_min": -96.24248504638672,
      "activations/layer8_attention_weight_max": 43.571876525878906,
      "activations/layer8_attention_weight_min": -45.19818115234375,
      "activations/layer9_attention_weight_max": 35.10097885131836,
      "activations/layer9_attention_weight_min": -35.90522384643555,
      "epoch": 11.28,
      "learning_rate": 7.805492424242425e-05,
      "loss": 2.7708,
      "step": 194100
    },
    {
      "activations/layer0_attention_weight_max": 16.21644401550293,
      "activations/layer0_attention_weight_min": -12.067532539367676,
      "activations/layer10_attention_weight_max": 31.63961410522461,
      "activations/layer10_attention_weight_min": -31.796092987060547,
      "activations/layer11_attention_weight_max": 29.86355209350586,
      "activations/layer11_attention_weight_min": -31.729705810546875,
      "activations/layer12_attention_weight_max": 26.737668991088867,
      "activations/layer12_attention_weight_min": -24.418994903564453,
      "activations/layer13_attention_weight_max": 40.13116455078125,
      "activations/layer13_attention_weight_min": -34.9059944152832,
      "activations/layer14_attention_weight_max": 43.58094787597656,
      "activations/layer14_attention_weight_min": -35.9324951171875,
      "activations/layer15_attention_weight_max": 34.03020095825195,
      "activations/layer15_attention_weight_min": -31.789649963378906,
      "activations/layer16_attention_weight_max": 30.83298683166504,
      "activations/layer16_attention_weight_min": -27.536123275756836,
      "activations/layer17_attention_weight_max": 29.50950813293457,
      "activations/layer17_attention_weight_min": -25.462610244750977,
      "activations/layer18_attention_weight_max": 30.394323348999023,
      "activations/layer18_attention_weight_min": -21.190799713134766,
      "activations/layer19_attention_weight_max": 36.531410217285156,
      "activations/layer19_attention_weight_min": -33.169925689697266,
      "activations/layer1_attention_weight_max": 16.95271110534668,
      "activations/layer1_attention_weight_min": -15.985589981079102,
      "activations/layer20_attention_weight_max": 28.22398567199707,
      "activations/layer20_attention_weight_min": -24.847936630249023,
      "activations/layer21_attention_weight_max": 29.8287353515625,
      "activations/layer21_attention_weight_min": -23.626001358032227,
      "activations/layer22_attention_weight_max": 41.4150390625,
      "activations/layer22_attention_weight_min": -28.362167358398438,
      "activations/layer23_attention_weight_max": 34.84953308105469,
      "activations/layer23_attention_weight_min": -25.83544158935547,
      "activations/layer2_attention_weight_max": 33.91669464111328,
      "activations/layer2_attention_weight_min": -32.64569854736328,
      "activations/layer3_attention_weight_max": 96.72930145263672,
      "activations/layer3_attention_weight_min": -95.45891571044922,
      "activations/layer4_attention_weight_max": 114.6692886352539,
      "activations/layer4_attention_weight_min": -111.18031311035156,
      "activations/layer5_attention_weight_max": 51.88343811035156,
      "activations/layer5_attention_weight_min": -60.93581771850586,
      "activations/layer6_attention_weight_max": 43.70382308959961,
      "activations/layer6_attention_weight_min": -46.416324615478516,
      "activations/layer7_attention_weight_max": 92.74845123291016,
      "activations/layer7_attention_weight_min": -90.60533905029297,
      "activations/layer8_attention_weight_max": 39.10909652709961,
      "activations/layer8_attention_weight_min": -43.24868392944336,
      "activations/layer9_attention_weight_max": 32.991817474365234,
      "activations/layer9_attention_weight_min": -34.330291748046875,
      "epoch": 11.28,
      "learning_rate": 7.803598484848484e-05,
      "loss": 2.7835,
      "step": 194150
    },
    {
      "activations/layer0_attention_weight_max": 15.408133506774902,
      "activations/layer0_attention_weight_min": -12.28832721710205,
      "activations/layer10_attention_weight_max": 34.17131423950195,
      "activations/layer10_attention_weight_min": -32.50575637817383,
      "activations/layer11_attention_weight_max": 30.244003295898438,
      "activations/layer11_attention_weight_min": -32.11983871459961,
      "activations/layer12_attention_weight_max": 23.349018096923828,
      "activations/layer12_attention_weight_min": -23.348403930664062,
      "activations/layer13_attention_weight_max": 35.10379409790039,
      "activations/layer13_attention_weight_min": -32.281776428222656,
      "activations/layer14_attention_weight_max": 40.14322280883789,
      "activations/layer14_attention_weight_min": -34.81422424316406,
      "activations/layer15_attention_weight_max": 32.326560974121094,
      "activations/layer15_attention_weight_min": -31.172574996948242,
      "activations/layer16_attention_weight_max": 30.4647216796875,
      "activations/layer16_attention_weight_min": -26.96767234802246,
      "activations/layer17_attention_weight_max": 30.55750274658203,
      "activations/layer17_attention_weight_min": -24.77897071838379,
      "activations/layer18_attention_weight_max": 32.644290924072266,
      "activations/layer18_attention_weight_min": -23.676406860351562,
      "activations/layer19_attention_weight_max": 35.43264389038086,
      "activations/layer19_attention_weight_min": -30.170211791992188,
      "activations/layer1_attention_weight_max": 15.918267250061035,
      "activations/layer1_attention_weight_min": -15.059671401977539,
      "activations/layer20_attention_weight_max": 31.580821990966797,
      "activations/layer20_attention_weight_min": -23.079069137573242,
      "activations/layer21_attention_weight_max": 28.28384017944336,
      "activations/layer21_attention_weight_min": -22.1403751373291,
      "activations/layer22_attention_weight_max": 40.28584289550781,
      "activations/layer22_attention_weight_min": -29.383441925048828,
      "activations/layer23_attention_weight_max": 34.45728302001953,
      "activations/layer23_attention_weight_min": -24.448537826538086,
      "activations/layer2_attention_weight_max": 35.99939727783203,
      "activations/layer2_attention_weight_min": -35.14073181152344,
      "activations/layer3_attention_weight_max": 97.80554962158203,
      "activations/layer3_attention_weight_min": -98.23497772216797,
      "activations/layer4_attention_weight_max": 114.91859436035156,
      "activations/layer4_attention_weight_min": -112.40874481201172,
      "activations/layer5_attention_weight_max": 51.30616760253906,
      "activations/layer5_attention_weight_min": -61.81415557861328,
      "activations/layer6_attention_weight_max": 42.315269470214844,
      "activations/layer6_attention_weight_min": -46.46989440917969,
      "activations/layer7_attention_weight_max": 90.05778503417969,
      "activations/layer7_attention_weight_min": -86.93619537353516,
      "activations/layer8_attention_weight_max": 40.05723571777344,
      "activations/layer8_attention_weight_min": -42.30775833129883,
      "activations/layer9_attention_weight_max": 34.075870513916016,
      "activations/layer9_attention_weight_min": -32.829811096191406,
      "epoch": 11.28,
      "learning_rate": 7.801704545454545e-05,
      "loss": 2.7754,
      "step": 194200
    },
    {
      "activations/layer0_attention_weight_max": 15.668766021728516,
      "activations/layer0_attention_weight_min": -12.966205596923828,
      "activations/layer10_attention_weight_max": 36.50814437866211,
      "activations/layer10_attention_weight_min": -32.08972930908203,
      "activations/layer11_attention_weight_max": 34.19026184082031,
      "activations/layer11_attention_weight_min": -32.578834533691406,
      "activations/layer12_attention_weight_max": 24.981359481811523,
      "activations/layer12_attention_weight_min": -25.490161895751953,
      "activations/layer13_attention_weight_max": 38.82802200317383,
      "activations/layer13_attention_weight_min": -34.39003372192383,
      "activations/layer14_attention_weight_max": 43.51664733886719,
      "activations/layer14_attention_weight_min": -37.588871002197266,
      "activations/layer15_attention_weight_max": 35.93187713623047,
      "activations/layer15_attention_weight_min": -34.87736511230469,
      "activations/layer16_attention_weight_max": 30.93897247314453,
      "activations/layer16_attention_weight_min": -29.594898223876953,
      "activations/layer17_attention_weight_max": 34.10261154174805,
      "activations/layer17_attention_weight_min": -27.312854766845703,
      "activations/layer18_attention_weight_max": 33.025848388671875,
      "activations/layer18_attention_weight_min": -27.739017486572266,
      "activations/layer19_attention_weight_max": 37.83307647705078,
      "activations/layer19_attention_weight_min": -35.36270523071289,
      "activations/layer1_attention_weight_max": 17.127614974975586,
      "activations/layer1_attention_weight_min": -16.87337303161621,
      "activations/layer20_attention_weight_max": 32.203468322753906,
      "activations/layer20_attention_weight_min": -29.758764266967773,
      "activations/layer21_attention_weight_max": 33.3674201965332,
      "activations/layer21_attention_weight_min": -26.93522834777832,
      "activations/layer22_attention_weight_max": 46.038909912109375,
      "activations/layer22_attention_weight_min": -35.804080963134766,
      "activations/layer23_attention_weight_max": 35.34789276123047,
      "activations/layer23_attention_weight_min": -26.070232391357422,
      "activations/layer2_attention_weight_max": 33.78181838989258,
      "activations/layer2_attention_weight_min": -33.7767333984375,
      "activations/layer3_attention_weight_max": 102.42029571533203,
      "activations/layer3_attention_weight_min": -100.4405746459961,
      "activations/layer4_attention_weight_max": 119.72747802734375,
      "activations/layer4_attention_weight_min": -116.86388397216797,
      "activations/layer5_attention_weight_max": 53.21141815185547,
      "activations/layer5_attention_weight_min": -63.5577507019043,
      "activations/layer6_attention_weight_max": 47.10240173339844,
      "activations/layer6_attention_weight_min": -49.808597564697266,
      "activations/layer7_attention_weight_max": 91.79400634765625,
      "activations/layer7_attention_weight_min": -90.09629821777344,
      "activations/layer8_attention_weight_max": 43.88700485229492,
      "activations/layer8_attention_weight_min": -44.36076354980469,
      "activations/layer9_attention_weight_max": 36.95779800415039,
      "activations/layer9_attention_weight_min": -34.14097213745117,
      "epoch": 11.29,
      "learning_rate": 7.799810606060605e-05,
      "loss": 2.7807,
      "step": 194250
    },
    {
      "activations/layer0_attention_weight_max": 15.735877990722656,
      "activations/layer0_attention_weight_min": -12.974749565124512,
      "activations/layer10_attention_weight_max": 34.51763153076172,
      "activations/layer10_attention_weight_min": -33.657081604003906,
      "activations/layer11_attention_weight_max": 33.184906005859375,
      "activations/layer11_attention_weight_min": -31.611141204833984,
      "activations/layer12_attention_weight_max": 26.922876358032227,
      "activations/layer12_attention_weight_min": -23.816373825073242,
      "activations/layer13_attention_weight_max": 42.668235778808594,
      "activations/layer13_attention_weight_min": -33.16461181640625,
      "activations/layer14_attention_weight_max": 48.23166275024414,
      "activations/layer14_attention_weight_min": -37.23835372924805,
      "activations/layer15_attention_weight_max": 37.63253402709961,
      "activations/layer15_attention_weight_min": -31.004587173461914,
      "activations/layer16_attention_weight_max": 30.729202270507812,
      "activations/layer16_attention_weight_min": -27.19490623474121,
      "activations/layer17_attention_weight_max": 32.94370651245117,
      "activations/layer17_attention_weight_min": -24.022220611572266,
      "activations/layer18_attention_weight_max": 30.432331085205078,
      "activations/layer18_attention_weight_min": -21.441707611083984,
      "activations/layer19_attention_weight_max": 36.80784225463867,
      "activations/layer19_attention_weight_min": -29.3326473236084,
      "activations/layer1_attention_weight_max": 16.73788833618164,
      "activations/layer1_attention_weight_min": -13.357868194580078,
      "activations/layer20_attention_weight_max": 29.970306396484375,
      "activations/layer20_attention_weight_min": -24.296710968017578,
      "activations/layer21_attention_weight_max": 28.08085823059082,
      "activations/layer21_attention_weight_min": -24.11760711669922,
      "activations/layer22_attention_weight_max": 48.35272216796875,
      "activations/layer22_attention_weight_min": -29.182695388793945,
      "activations/layer23_attention_weight_max": 34.11360168457031,
      "activations/layer23_attention_weight_min": -22.155553817749023,
      "activations/layer2_attention_weight_max": 32.97175979614258,
      "activations/layer2_attention_weight_min": -34.51539993286133,
      "activations/layer3_attention_weight_max": 97.86599731445312,
      "activations/layer3_attention_weight_min": -104.1767349243164,
      "activations/layer4_attention_weight_max": 115.30619049072266,
      "activations/layer4_attention_weight_min": -117.20299530029297,
      "activations/layer5_attention_weight_max": 53.73800277709961,
      "activations/layer5_attention_weight_min": -63.49449157714844,
      "activations/layer6_attention_weight_max": 45.58094024658203,
      "activations/layer6_attention_weight_min": -48.0882682800293,
      "activations/layer7_attention_weight_max": 93.09489440917969,
      "activations/layer7_attention_weight_min": -93.01073455810547,
      "activations/layer8_attention_weight_max": 39.981468200683594,
      "activations/layer8_attention_weight_min": -41.61769485473633,
      "activations/layer9_attention_weight_max": 33.62137222290039,
      "activations/layer9_attention_weight_min": -34.50465774536133,
      "epoch": 11.29,
      "learning_rate": 7.797916666666665e-05,
      "loss": 2.7755,
      "step": 194300
    },
    {
      "activations/layer0_attention_weight_max": 15.767546653747559,
      "activations/layer0_attention_weight_min": -12.653343200683594,
      "activations/layer10_attention_weight_max": 36.01405334472656,
      "activations/layer10_attention_weight_min": -36.14072036743164,
      "activations/layer11_attention_weight_max": 35.0685920715332,
      "activations/layer11_attention_weight_min": -32.398887634277344,
      "activations/layer12_attention_weight_max": 29.760658264160156,
      "activations/layer12_attention_weight_min": -26.86711311340332,
      "activations/layer13_attention_weight_max": 46.31638717651367,
      "activations/layer13_attention_weight_min": -34.65511703491211,
      "activations/layer14_attention_weight_max": 51.63172912597656,
      "activations/layer14_attention_weight_min": -39.37473678588867,
      "activations/layer15_attention_weight_max": 44.4943962097168,
      "activations/layer15_attention_weight_min": -33.420597076416016,
      "activations/layer16_attention_weight_max": 36.0474967956543,
      "activations/layer16_attention_weight_min": -28.415803909301758,
      "activations/layer17_attention_weight_max": 35.868499755859375,
      "activations/layer17_attention_weight_min": -27.643478393554688,
      "activations/layer18_attention_weight_max": 31.623483657836914,
      "activations/layer18_attention_weight_min": -23.367874145507812,
      "activations/layer19_attention_weight_max": 39.35824966430664,
      "activations/layer19_attention_weight_min": -32.492393493652344,
      "activations/layer1_attention_weight_max": 17.989059448242188,
      "activations/layer1_attention_weight_min": -15.196702003479004,
      "activations/layer20_attention_weight_max": 32.79801940917969,
      "activations/layer20_attention_weight_min": -25.943679809570312,
      "activations/layer21_attention_weight_max": 30.35672950744629,
      "activations/layer21_attention_weight_min": -24.78649139404297,
      "activations/layer22_attention_weight_max": 44.45026397705078,
      "activations/layer22_attention_weight_min": -31.66899299621582,
      "activations/layer23_attention_weight_max": 36.60282897949219,
      "activations/layer23_attention_weight_min": -24.14845848083496,
      "activations/layer2_attention_weight_max": 33.79010772705078,
      "activations/layer2_attention_weight_min": -34.05229187011719,
      "activations/layer3_attention_weight_max": 97.38721466064453,
      "activations/layer3_attention_weight_min": -100.64562225341797,
      "activations/layer4_attention_weight_max": 112.3397216796875,
      "activations/layer4_attention_weight_min": -111.39845275878906,
      "activations/layer5_attention_weight_max": 52.919227600097656,
      "activations/layer5_attention_weight_min": -61.334251403808594,
      "activations/layer6_attention_weight_max": 45.34761428833008,
      "activations/layer6_attention_weight_min": -49.00960922241211,
      "activations/layer7_attention_weight_max": 98.27847290039062,
      "activations/layer7_attention_weight_min": -96.29032897949219,
      "activations/layer8_attention_weight_max": 43.86163330078125,
      "activations/layer8_attention_weight_min": -45.55634307861328,
      "activations/layer9_attention_weight_max": 36.33250045776367,
      "activations/layer9_attention_weight_min": -36.477718353271484,
      "epoch": 11.29,
      "learning_rate": 7.796022727272727e-05,
      "loss": 2.7925,
      "step": 194350
    },
    {
      "activations/layer0_attention_weight_max": 15.96381950378418,
      "activations/layer0_attention_weight_min": -12.829985618591309,
      "activations/layer10_attention_weight_max": 33.310150146484375,
      "activations/layer10_attention_weight_min": -32.160888671875,
      "activations/layer11_attention_weight_max": 32.25330352783203,
      "activations/layer11_attention_weight_min": -31.488040924072266,
      "activations/layer12_attention_weight_max": 23.257047653198242,
      "activations/layer12_attention_weight_min": -23.46274185180664,
      "activations/layer13_attention_weight_max": 37.7559814453125,
      "activations/layer13_attention_weight_min": -32.85935592651367,
      "activations/layer14_attention_weight_max": 45.14173126220703,
      "activations/layer14_attention_weight_min": -38.74783706665039,
      "activations/layer15_attention_weight_max": 35.34988021850586,
      "activations/layer15_attention_weight_min": -32.11662673950195,
      "activations/layer16_attention_weight_max": 29.920387268066406,
      "activations/layer16_attention_weight_min": -27.704124450683594,
      "activations/layer17_attention_weight_max": 29.400094985961914,
      "activations/layer17_attention_weight_min": -26.5435791015625,
      "activations/layer18_attention_weight_max": 30.986928939819336,
      "activations/layer18_attention_weight_min": -24.002653121948242,
      "activations/layer19_attention_weight_max": 36.43289566040039,
      "activations/layer19_attention_weight_min": -33.137535095214844,
      "activations/layer1_attention_weight_max": 16.290674209594727,
      "activations/layer1_attention_weight_min": -15.363384246826172,
      "activations/layer20_attention_weight_max": 29.456640243530273,
      "activations/layer20_attention_weight_min": -25.36507797241211,
      "activations/layer21_attention_weight_max": 28.216934204101562,
      "activations/layer21_attention_weight_min": -24.37010955810547,
      "activations/layer22_attention_weight_max": 44.8499641418457,
      "activations/layer22_attention_weight_min": -30.96147918701172,
      "activations/layer23_attention_weight_max": 33.63148498535156,
      "activations/layer23_attention_weight_min": -27.078571319580078,
      "activations/layer2_attention_weight_max": 34.113121032714844,
      "activations/layer2_attention_weight_min": -33.99055099487305,
      "activations/layer3_attention_weight_max": 97.39916229248047,
      "activations/layer3_attention_weight_min": -103.78133392333984,
      "activations/layer4_attention_weight_max": 115.54561614990234,
      "activations/layer4_attention_weight_min": -115.14763641357422,
      "activations/layer5_attention_weight_max": 54.10455322265625,
      "activations/layer5_attention_weight_min": -63.58762741088867,
      "activations/layer6_attention_weight_max": 44.57624435424805,
      "activations/layer6_attention_weight_min": -47.89738082885742,
      "activations/layer7_attention_weight_max": 92.34298706054688,
      "activations/layer7_attention_weight_min": -86.61758422851562,
      "activations/layer8_attention_weight_max": 40.32600784301758,
      "activations/layer8_attention_weight_min": -39.62565994262695,
      "activations/layer9_attention_weight_max": 34.075965881347656,
      "activations/layer9_attention_weight_min": -33.3264274597168,
      "epoch": 11.3,
      "learning_rate": 7.794128787878787e-05,
      "loss": 2.7565,
      "step": 194400
    },
    {
      "activations/layer0_attention_weight_max": 16.187341690063477,
      "activations/layer0_attention_weight_min": -12.702149391174316,
      "activations/layer10_attention_weight_max": 30.23252296447754,
      "activations/layer10_attention_weight_min": -30.030771255493164,
      "activations/layer11_attention_weight_max": 28.993955612182617,
      "activations/layer11_attention_weight_min": -31.541288375854492,
      "activations/layer12_attention_weight_max": 22.813255310058594,
      "activations/layer12_attention_weight_min": -23.756149291992188,
      "activations/layer13_attention_weight_max": 39.4501838684082,
      "activations/layer13_attention_weight_min": -35.16639709472656,
      "activations/layer14_attention_weight_max": 43.92204666137695,
      "activations/layer14_attention_weight_min": -36.32284164428711,
      "activations/layer15_attention_weight_max": 34.73360824584961,
      "activations/layer15_attention_weight_min": -31.403676986694336,
      "activations/layer16_attention_weight_max": 31.5302734375,
      "activations/layer16_attention_weight_min": -26.487560272216797,
      "activations/layer17_attention_weight_max": 31.92820167541504,
      "activations/layer17_attention_weight_min": -24.8497257232666,
      "activations/layer18_attention_weight_max": 32.249473571777344,
      "activations/layer18_attention_weight_min": -22.704238891601562,
      "activations/layer19_attention_weight_max": 34.33312225341797,
      "activations/layer19_attention_weight_min": -31.8369083404541,
      "activations/layer1_attention_weight_max": 18.242042541503906,
      "activations/layer1_attention_weight_min": -14.867695808410645,
      "activations/layer20_attention_weight_max": 28.616546630859375,
      "activations/layer20_attention_weight_min": -24.852155685424805,
      "activations/layer21_attention_weight_max": 33.654701232910156,
      "activations/layer21_attention_weight_min": -24.164670944213867,
      "activations/layer22_attention_weight_max": 44.50055694580078,
      "activations/layer22_attention_weight_min": -29.166990280151367,
      "activations/layer23_attention_weight_max": 34.99356460571289,
      "activations/layer23_attention_weight_min": -23.859867095947266,
      "activations/layer2_attention_weight_max": 35.02564239501953,
      "activations/layer2_attention_weight_min": -33.75053024291992,
      "activations/layer3_attention_weight_max": 98.84925842285156,
      "activations/layer3_attention_weight_min": -96.98700714111328,
      "activations/layer4_attention_weight_max": 114.93110656738281,
      "activations/layer4_attention_weight_min": -116.20759582519531,
      "activations/layer5_attention_weight_max": 51.09620666503906,
      "activations/layer5_attention_weight_min": -63.30204772949219,
      "activations/layer6_attention_weight_max": 42.898963928222656,
      "activations/layer6_attention_weight_min": -46.06612014770508,
      "activations/layer7_attention_weight_max": 86.41577911376953,
      "activations/layer7_attention_weight_min": -87.99400329589844,
      "activations/layer8_attention_weight_max": 38.33771514892578,
      "activations/layer8_attention_weight_min": -38.45174026489258,
      "activations/layer9_attention_weight_max": 30.770978927612305,
      "activations/layer9_attention_weight_min": -31.7567138671875,
      "epoch": 11.3,
      "learning_rate": 7.792234848484849e-05,
      "loss": 2.7795,
      "step": 194450
    },
    {
      "activations/layer0_attention_weight_max": 16.34473991394043,
      "activations/layer0_attention_weight_min": -12.860342025756836,
      "activations/layer10_attention_weight_max": 42.29573059082031,
      "activations/layer10_attention_weight_min": -40.345062255859375,
      "activations/layer11_attention_weight_max": 42.60283660888672,
      "activations/layer11_attention_weight_min": -41.0024299621582,
      "activations/layer12_attention_weight_max": 27.713224411010742,
      "activations/layer12_attention_weight_min": -26.397775650024414,
      "activations/layer13_attention_weight_max": 42.8653450012207,
      "activations/layer13_attention_weight_min": -38.91203689575195,
      "activations/layer14_attention_weight_max": 50.23539352416992,
      "activations/layer14_attention_weight_min": -42.40987777709961,
      "activations/layer15_attention_weight_max": 37.58926773071289,
      "activations/layer15_attention_weight_min": -34.69930648803711,
      "activations/layer16_attention_weight_max": 33.08893585205078,
      "activations/layer16_attention_weight_min": -27.32954216003418,
      "activations/layer17_attention_weight_max": 32.509437561035156,
      "activations/layer17_attention_weight_min": -26.472003936767578,
      "activations/layer18_attention_weight_max": 33.73654556274414,
      "activations/layer18_attention_weight_min": -23.26878547668457,
      "activations/layer19_attention_weight_max": 36.39789581298828,
      "activations/layer19_attention_weight_min": -31.971887588500977,
      "activations/layer1_attention_weight_max": 17.324060440063477,
      "activations/layer1_attention_weight_min": -15.830041885375977,
      "activations/layer20_attention_weight_max": 33.02011489868164,
      "activations/layer20_attention_weight_min": -24.06498908996582,
      "activations/layer21_attention_weight_max": 33.34967041015625,
      "activations/layer21_attention_weight_min": -24.44894790649414,
      "activations/layer22_attention_weight_max": 55.69904327392578,
      "activations/layer22_attention_weight_min": -29.973772048950195,
      "activations/layer23_attention_weight_max": 38.28142547607422,
      "activations/layer23_attention_weight_min": -28.295997619628906,
      "activations/layer2_attention_weight_max": 35.26636505126953,
      "activations/layer2_attention_weight_min": -35.912437438964844,
      "activations/layer3_attention_weight_max": 98.70732879638672,
      "activations/layer3_attention_weight_min": -98.73786163330078,
      "activations/layer4_attention_weight_max": 116.63541412353516,
      "activations/layer4_attention_weight_min": -117.56953430175781,
      "activations/layer5_attention_weight_max": 51.939666748046875,
      "activations/layer5_attention_weight_min": -61.41835021972656,
      "activations/layer6_attention_weight_max": 46.372867584228516,
      "activations/layer6_attention_weight_min": -50.44736099243164,
      "activations/layer7_attention_weight_max": 106.86617279052734,
      "activations/layer7_attention_weight_min": -107.58287811279297,
      "activations/layer8_attention_weight_max": 51.0031623840332,
      "activations/layer8_attention_weight_min": -48.27683639526367,
      "activations/layer9_attention_weight_max": 42.30727767944336,
      "activations/layer9_attention_weight_min": -38.58187484741211,
      "epoch": 11.3,
      "learning_rate": 7.790340909090909e-05,
      "loss": 2.7795,
      "step": 194500
    },
    {
      "activations/layer0_attention_weight_max": 16.470134735107422,
      "activations/layer0_attention_weight_min": -11.807591438293457,
      "activations/layer10_attention_weight_max": 31.045146942138672,
      "activations/layer10_attention_weight_min": -31.69146728515625,
      "activations/layer11_attention_weight_max": 32.6186637878418,
      "activations/layer11_attention_weight_min": -34.48583984375,
      "activations/layer12_attention_weight_max": 23.90391731262207,
      "activations/layer12_attention_weight_min": -25.719820022583008,
      "activations/layer13_attention_weight_max": 38.15398406982422,
      "activations/layer13_attention_weight_min": -34.90339279174805,
      "activations/layer14_attention_weight_max": 45.73301696777344,
      "activations/layer14_attention_weight_min": -40.65251922607422,
      "activations/layer15_attention_weight_max": 35.983131408691406,
      "activations/layer15_attention_weight_min": -32.28431701660156,
      "activations/layer16_attention_weight_max": 30.511075973510742,
      "activations/layer16_attention_weight_min": -29.04812240600586,
      "activations/layer17_attention_weight_max": 31.652238845825195,
      "activations/layer17_attention_weight_min": -26.123294830322266,
      "activations/layer18_attention_weight_max": 30.79661750793457,
      "activations/layer18_attention_weight_min": -24.4295711517334,
      "activations/layer19_attention_weight_max": 36.62665557861328,
      "activations/layer19_attention_weight_min": -32.239261627197266,
      "activations/layer1_attention_weight_max": 15.957574844360352,
      "activations/layer1_attention_weight_min": -15.808140754699707,
      "activations/layer20_attention_weight_max": 28.6460018157959,
      "activations/layer20_attention_weight_min": -24.576932907104492,
      "activations/layer21_attention_weight_max": 29.317623138427734,
      "activations/layer21_attention_weight_min": -25.513383865356445,
      "activations/layer22_attention_weight_max": 44.872039794921875,
      "activations/layer22_attention_weight_min": -29.812252044677734,
      "activations/layer23_attention_weight_max": 34.752464294433594,
      "activations/layer23_attention_weight_min": -24.74102020263672,
      "activations/layer2_attention_weight_max": 34.266700744628906,
      "activations/layer2_attention_weight_min": -32.52296447753906,
      "activations/layer3_attention_weight_max": 101.2859878540039,
      "activations/layer3_attention_weight_min": -100.1680908203125,
      "activations/layer4_attention_weight_max": 120.30499267578125,
      "activations/layer4_attention_weight_min": -115.8863754272461,
      "activations/layer5_attention_weight_max": 54.35090637207031,
      "activations/layer5_attention_weight_min": -59.512847900390625,
      "activations/layer6_attention_weight_max": 45.68091583251953,
      "activations/layer6_attention_weight_min": -46.599666595458984,
      "activations/layer7_attention_weight_max": 87.94171142578125,
      "activations/layer7_attention_weight_min": -91.76559448242188,
      "activations/layer8_attention_weight_max": 39.96357345581055,
      "activations/layer8_attention_weight_min": -38.42622375488281,
      "activations/layer9_attention_weight_max": 31.198883056640625,
      "activations/layer9_attention_weight_min": -32.19419479370117,
      "epoch": 11.3,
      "learning_rate": 7.788446969696969e-05,
      "loss": 2.7693,
      "step": 194550
    },
    {
      "activations/layer0_attention_weight_max": 16.40277099609375,
      "activations/layer0_attention_weight_min": -12.67947006225586,
      "activations/layer10_attention_weight_max": 31.87030029296875,
      "activations/layer10_attention_weight_min": -32.46372604370117,
      "activations/layer11_attention_weight_max": 31.49616813659668,
      "activations/layer11_attention_weight_min": -31.945438385009766,
      "activations/layer12_attention_weight_max": 24.15259552001953,
      "activations/layer12_attention_weight_min": -23.819602966308594,
      "activations/layer13_attention_weight_max": 37.836483001708984,
      "activations/layer13_attention_weight_min": -34.3203125,
      "activations/layer14_attention_weight_max": 40.55105972290039,
      "activations/layer14_attention_weight_min": -38.480384826660156,
      "activations/layer15_attention_weight_max": 35.05156707763672,
      "activations/layer15_attention_weight_min": -32.09469985961914,
      "activations/layer16_attention_weight_max": 29.25040626525879,
      "activations/layer16_attention_weight_min": -27.13412857055664,
      "activations/layer17_attention_weight_max": 31.63648796081543,
      "activations/layer17_attention_weight_min": -25.089874267578125,
      "activations/layer18_attention_weight_max": 31.143606185913086,
      "activations/layer18_attention_weight_min": -21.561378479003906,
      "activations/layer19_attention_weight_max": 32.52579879760742,
      "activations/layer19_attention_weight_min": -29.832738876342773,
      "activations/layer1_attention_weight_max": 17.31360626220703,
      "activations/layer1_attention_weight_min": -15.44111442565918,
      "activations/layer20_attention_weight_max": 27.26479721069336,
      "activations/layer20_attention_weight_min": -24.471708297729492,
      "activations/layer21_attention_weight_max": 27.250211715698242,
      "activations/layer21_attention_weight_min": -23.6705379486084,
      "activations/layer22_attention_weight_max": 40.30939483642578,
      "activations/layer22_attention_weight_min": -30.15057373046875,
      "activations/layer23_attention_weight_max": 31.048437118530273,
      "activations/layer23_attention_weight_min": -27.39906883239746,
      "activations/layer2_attention_weight_max": 33.47444152832031,
      "activations/layer2_attention_weight_min": -33.704917907714844,
      "activations/layer3_attention_weight_max": 99.0708236694336,
      "activations/layer3_attention_weight_min": -100.4239730834961,
      "activations/layer4_attention_weight_max": 115.30609130859375,
      "activations/layer4_attention_weight_min": -115.2897720336914,
      "activations/layer5_attention_weight_max": 51.99200439453125,
      "activations/layer5_attention_weight_min": -60.97373580932617,
      "activations/layer6_attention_weight_max": 44.543277740478516,
      "activations/layer6_attention_weight_min": -48.447994232177734,
      "activations/layer7_attention_weight_max": 91.42819213867188,
      "activations/layer7_attention_weight_min": -85.1225814819336,
      "activations/layer8_attention_weight_max": 38.76567840576172,
      "activations/layer8_attention_weight_min": -39.894287109375,
      "activations/layer9_attention_weight_max": 31.886640548706055,
      "activations/layer9_attention_weight_min": -31.388498306274414,
      "epoch": 11.31,
      "learning_rate": 7.78655303030303e-05,
      "loss": 2.7765,
      "step": 194600
    },
    {
      "activations/layer0_attention_weight_max": 16.360177993774414,
      "activations/layer0_attention_weight_min": -12.715802192687988,
      "activations/layer10_attention_weight_max": 32.03487014770508,
      "activations/layer10_attention_weight_min": -31.172672271728516,
      "activations/layer11_attention_weight_max": 30.604673385620117,
      "activations/layer11_attention_weight_min": -30.061607360839844,
      "activations/layer12_attention_weight_max": 22.28970718383789,
      "activations/layer12_attention_weight_min": -26.020084381103516,
      "activations/layer13_attention_weight_max": 33.83081817626953,
      "activations/layer13_attention_weight_min": -34.536033630371094,
      "activations/layer14_attention_weight_max": 41.211509704589844,
      "activations/layer14_attention_weight_min": -37.45002365112305,
      "activations/layer15_attention_weight_max": 33.055904388427734,
      "activations/layer15_attention_weight_min": -28.799118041992188,
      "activations/layer16_attention_weight_max": 26.697490692138672,
      "activations/layer16_attention_weight_min": -26.871532440185547,
      "activations/layer17_attention_weight_max": 28.40503692626953,
      "activations/layer17_attention_weight_min": -25.301111221313477,
      "activations/layer18_attention_weight_max": 30.152128219604492,
      "activations/layer18_attention_weight_min": -25.207460403442383,
      "activations/layer19_attention_weight_max": 30.926115036010742,
      "activations/layer19_attention_weight_min": -29.449186325073242,
      "activations/layer1_attention_weight_max": 16.760602951049805,
      "activations/layer1_attention_weight_min": -15.454469680786133,
      "activations/layer20_attention_weight_max": 27.59727668762207,
      "activations/layer20_attention_weight_min": -24.69049072265625,
      "activations/layer21_attention_weight_max": 28.166576385498047,
      "activations/layer21_attention_weight_min": -22.432092666625977,
      "activations/layer22_attention_weight_max": 38.63196563720703,
      "activations/layer22_attention_weight_min": -30.52733612060547,
      "activations/layer23_attention_weight_max": 31.607379913330078,
      "activations/layer23_attention_weight_min": -23.7288818359375,
      "activations/layer2_attention_weight_max": 33.390892028808594,
      "activations/layer2_attention_weight_min": -34.23910140991211,
      "activations/layer3_attention_weight_max": 99.7144546508789,
      "activations/layer3_attention_weight_min": -105.81846618652344,
      "activations/layer4_attention_weight_max": 117.66246795654297,
      "activations/layer4_attention_weight_min": -118.41566467285156,
      "activations/layer5_attention_weight_max": 52.65443420410156,
      "activations/layer5_attention_weight_min": -63.251033782958984,
      "activations/layer6_attention_weight_max": 44.58867645263672,
      "activations/layer6_attention_weight_min": -48.19645690917969,
      "activations/layer7_attention_weight_max": 88.36808013916016,
      "activations/layer7_attention_weight_min": -89.20333862304688,
      "activations/layer8_attention_weight_max": 39.978843688964844,
      "activations/layer8_attention_weight_min": -42.23236083984375,
      "activations/layer9_attention_weight_max": 32.77853012084961,
      "activations/layer9_attention_weight_min": -32.562255859375,
      "epoch": 11.31,
      "learning_rate": 7.784659090909089e-05,
      "loss": 2.7812,
      "step": 194650
    },
    {
      "activations/layer0_attention_weight_max": 16.296661376953125,
      "activations/layer0_attention_weight_min": -12.562677383422852,
      "activations/layer10_attention_weight_max": 32.30289840698242,
      "activations/layer10_attention_weight_min": -31.757291793823242,
      "activations/layer11_attention_weight_max": 30.546092987060547,
      "activations/layer11_attention_weight_min": -30.844036102294922,
      "activations/layer12_attention_weight_max": 23.199186325073242,
      "activations/layer12_attention_weight_min": -27.095746994018555,
      "activations/layer13_attention_weight_max": 37.40313720703125,
      "activations/layer13_attention_weight_min": -31.620460510253906,
      "activations/layer14_attention_weight_max": 39.16093063354492,
      "activations/layer14_attention_weight_min": -35.16728210449219,
      "activations/layer15_attention_weight_max": 36.19142532348633,
      "activations/layer15_attention_weight_min": -29.34549331665039,
      "activations/layer16_attention_weight_max": 33.7417106628418,
      "activations/layer16_attention_weight_min": -26.40824317932129,
      "activations/layer17_attention_weight_max": 32.06038284301758,
      "activations/layer17_attention_weight_min": -25.25274085998535,
      "activations/layer18_attention_weight_max": 30.6394100189209,
      "activations/layer18_attention_weight_min": -22.19013214111328,
      "activations/layer19_attention_weight_max": 36.64917755126953,
      "activations/layer19_attention_weight_min": -29.346689224243164,
      "activations/layer1_attention_weight_max": 17.518962860107422,
      "activations/layer1_attention_weight_min": -16.273624420166016,
      "activations/layer20_attention_weight_max": 28.891441345214844,
      "activations/layer20_attention_weight_min": -24.89949607849121,
      "activations/layer21_attention_weight_max": 30.84085464477539,
      "activations/layer21_attention_weight_min": -23.74054527282715,
      "activations/layer22_attention_weight_max": 42.986595153808594,
      "activations/layer22_attention_weight_min": -29.605037689208984,
      "activations/layer23_attention_weight_max": 32.53466033935547,
      "activations/layer23_attention_weight_min": -21.919898986816406,
      "activations/layer2_attention_weight_max": 35.08137512207031,
      "activations/layer2_attention_weight_min": -32.779579162597656,
      "activations/layer3_attention_weight_max": 96.87879943847656,
      "activations/layer3_attention_weight_min": -97.50434112548828,
      "activations/layer4_attention_weight_max": 117.2549819946289,
      "activations/layer4_attention_weight_min": -120.86165618896484,
      "activations/layer5_attention_weight_max": 53.636287689208984,
      "activations/layer5_attention_weight_min": -60.806243896484375,
      "activations/layer6_attention_weight_max": 44.430233001708984,
      "activations/layer6_attention_weight_min": -48.29652786254883,
      "activations/layer7_attention_weight_max": 94.15904235839844,
      "activations/layer7_attention_weight_min": -94.06929016113281,
      "activations/layer8_attention_weight_max": 40.35430908203125,
      "activations/layer8_attention_weight_min": -41.67824935913086,
      "activations/layer9_attention_weight_max": 33.40034103393555,
      "activations/layer9_attention_weight_min": -32.72355270385742,
      "epoch": 11.31,
      "learning_rate": 7.78276515151515e-05,
      "loss": 2.757,
      "step": 194700
    },
    {
      "activations/layer0_attention_weight_max": 15.97614860534668,
      "activations/layer0_attention_weight_min": -12.84254264831543,
      "activations/layer10_attention_weight_max": 31.071117401123047,
      "activations/layer10_attention_weight_min": -32.05024337768555,
      "activations/layer11_attention_weight_max": 31.589767456054688,
      "activations/layer11_attention_weight_min": -32.90742492675781,
      "activations/layer12_attention_weight_max": 25.260498046875,
      "activations/layer12_attention_weight_min": -24.22514533996582,
      "activations/layer13_attention_weight_max": 38.02546310424805,
      "activations/layer13_attention_weight_min": -32.495819091796875,
      "activations/layer14_attention_weight_max": 43.76057815551758,
      "activations/layer14_attention_weight_min": -36.10187911987305,
      "activations/layer15_attention_weight_max": 35.02547836303711,
      "activations/layer15_attention_weight_min": -30.121776580810547,
      "activations/layer16_attention_weight_max": 30.21722412109375,
      "activations/layer16_attention_weight_min": -28.498655319213867,
      "activations/layer17_attention_weight_max": 32.81843948364258,
      "activations/layer17_attention_weight_min": -27.13568687438965,
      "activations/layer18_attention_weight_max": 33.046627044677734,
      "activations/layer18_attention_weight_min": -21.673063278198242,
      "activations/layer19_attention_weight_max": 34.527870178222656,
      "activations/layer19_attention_weight_min": -29.589237213134766,
      "activations/layer1_attention_weight_max": 17.745595932006836,
      "activations/layer1_attention_weight_min": -16.754520416259766,
      "activations/layer20_attention_weight_max": 29.249759674072266,
      "activations/layer20_attention_weight_min": -23.217208862304688,
      "activations/layer21_attention_weight_max": 30.34716033935547,
      "activations/layer21_attention_weight_min": -21.430660247802734,
      "activations/layer22_attention_weight_max": 47.14895248413086,
      "activations/layer22_attention_weight_min": -29.44365692138672,
      "activations/layer23_attention_weight_max": 37.206459045410156,
      "activations/layer23_attention_weight_min": -24.87242889404297,
      "activations/layer2_attention_weight_max": 32.5416374206543,
      "activations/layer2_attention_weight_min": -31.310516357421875,
      "activations/layer3_attention_weight_max": 96.12208557128906,
      "activations/layer3_attention_weight_min": -91.61266326904297,
      "activations/layer4_attention_weight_max": 111.4617919921875,
      "activations/layer4_attention_weight_min": -111.9041748046875,
      "activations/layer5_attention_weight_max": 51.653297424316406,
      "activations/layer5_attention_weight_min": -60.81736373901367,
      "activations/layer6_attention_weight_max": 43.499534606933594,
      "activations/layer6_attention_weight_min": -48.514713287353516,
      "activations/layer7_attention_weight_max": 83.6684341430664,
      "activations/layer7_attention_weight_min": -88.9423828125,
      "activations/layer8_attention_weight_max": 38.40132522583008,
      "activations/layer8_attention_weight_min": -42.139564514160156,
      "activations/layer9_attention_weight_max": 32.861045837402344,
      "activations/layer9_attention_weight_min": -33.581050872802734,
      "epoch": 11.32,
      "learning_rate": 7.780871212121212e-05,
      "loss": 2.775,
      "step": 194750
    },
    {
      "activations/layer0_attention_weight_max": 16.521263122558594,
      "activations/layer0_attention_weight_min": -12.292865753173828,
      "activations/layer10_attention_weight_max": 32.6047477722168,
      "activations/layer10_attention_weight_min": -31.457489013671875,
      "activations/layer11_attention_weight_max": 30.513233184814453,
      "activations/layer11_attention_weight_min": -29.710071563720703,
      "activations/layer12_attention_weight_max": 22.76297378540039,
      "activations/layer12_attention_weight_min": -24.370361328125,
      "activations/layer13_attention_weight_max": 37.16167449951172,
      "activations/layer13_attention_weight_min": -33.66513442993164,
      "activations/layer14_attention_weight_max": 42.11837387084961,
      "activations/layer14_attention_weight_min": -38.09835433959961,
      "activations/layer15_attention_weight_max": 33.74983215332031,
      "activations/layer15_attention_weight_min": -31.07634925842285,
      "activations/layer16_attention_weight_max": 28.596771240234375,
      "activations/layer16_attention_weight_min": -28.250389099121094,
      "activations/layer17_attention_weight_max": 31.941768646240234,
      "activations/layer17_attention_weight_min": -25.171483993530273,
      "activations/layer18_attention_weight_max": 30.473867416381836,
      "activations/layer18_attention_weight_min": -23.556238174438477,
      "activations/layer19_attention_weight_max": 32.48431396484375,
      "activations/layer19_attention_weight_min": -30.58870506286621,
      "activations/layer1_attention_weight_max": 18.73324203491211,
      "activations/layer1_attention_weight_min": -14.559698104858398,
      "activations/layer20_attention_weight_max": 30.222373962402344,
      "activations/layer20_attention_weight_min": -23.36674690246582,
      "activations/layer21_attention_weight_max": 29.29799461364746,
      "activations/layer21_attention_weight_min": -22.270776748657227,
      "activations/layer22_attention_weight_max": 44.187137603759766,
      "activations/layer22_attention_weight_min": -28.669172286987305,
      "activations/layer23_attention_weight_max": 33.75897979736328,
      "activations/layer23_attention_weight_min": -24.70233917236328,
      "activations/layer2_attention_weight_max": 33.68571472167969,
      "activations/layer2_attention_weight_min": -33.970420837402344,
      "activations/layer3_attention_weight_max": 96.30535125732422,
      "activations/layer3_attention_weight_min": -98.12934112548828,
      "activations/layer4_attention_weight_max": 120.38268280029297,
      "activations/layer4_attention_weight_min": -115.42317962646484,
      "activations/layer5_attention_weight_max": 54.14072799682617,
      "activations/layer5_attention_weight_min": -62.17845916748047,
      "activations/layer6_attention_weight_max": 44.99137878417969,
      "activations/layer6_attention_weight_min": -49.23590850830078,
      "activations/layer7_attention_weight_max": 88.45455932617188,
      "activations/layer7_attention_weight_min": -90.25377655029297,
      "activations/layer8_attention_weight_max": 37.74477005004883,
      "activations/layer8_attention_weight_min": -41.73003005981445,
      "activations/layer9_attention_weight_max": 33.48881149291992,
      "activations/layer9_attention_weight_min": -33.980224609375,
      "epoch": 11.32,
      "learning_rate": 7.778977272727271e-05,
      "loss": 2.7817,
      "step": 194800
    },
    {
      "activations/layer0_attention_weight_max": 15.435413360595703,
      "activations/layer0_attention_weight_min": -12.5372896194458,
      "activations/layer10_attention_weight_max": 32.181861877441406,
      "activations/layer10_attention_weight_min": -34.51947021484375,
      "activations/layer11_attention_weight_max": 33.40669631958008,
      "activations/layer11_attention_weight_min": -32.33853530883789,
      "activations/layer12_attention_weight_max": 23.90995216369629,
      "activations/layer12_attention_weight_min": -26.626176834106445,
      "activations/layer13_attention_weight_max": 37.66485595703125,
      "activations/layer13_attention_weight_min": -32.235740661621094,
      "activations/layer14_attention_weight_max": 47.92255783081055,
      "activations/layer14_attention_weight_min": -36.50642013549805,
      "activations/layer15_attention_weight_max": 37.28837585449219,
      "activations/layer15_attention_weight_min": -30.313873291015625,
      "activations/layer16_attention_weight_max": 30.5201416015625,
      "activations/layer16_attention_weight_min": -26.856679916381836,
      "activations/layer17_attention_weight_max": 33.61268997192383,
      "activations/layer17_attention_weight_min": -24.701797485351562,
      "activations/layer18_attention_weight_max": 30.744953155517578,
      "activations/layer18_attention_weight_min": -23.847951889038086,
      "activations/layer19_attention_weight_max": 36.26255798339844,
      "activations/layer19_attention_weight_min": -29.673635482788086,
      "activations/layer1_attention_weight_max": 16.563087463378906,
      "activations/layer1_attention_weight_min": -15.40058708190918,
      "activations/layer20_attention_weight_max": 31.044090270996094,
      "activations/layer20_attention_weight_min": -25.740629196166992,
      "activations/layer21_attention_weight_max": 31.90341567993164,
      "activations/layer21_attention_weight_min": -23.81317138671875,
      "activations/layer22_attention_weight_max": 48.87501525878906,
      "activations/layer22_attention_weight_min": -29.983036041259766,
      "activations/layer23_attention_weight_max": 38.043609619140625,
      "activations/layer23_attention_weight_min": -25.046239852905273,
      "activations/layer2_attention_weight_max": 33.32617950439453,
      "activations/layer2_attention_weight_min": -35.77572250366211,
      "activations/layer3_attention_weight_max": 94.5926742553711,
      "activations/layer3_attention_weight_min": -99.76181030273438,
      "activations/layer4_attention_weight_max": 112.97325897216797,
      "activations/layer4_attention_weight_min": -110.10213470458984,
      "activations/layer5_attention_weight_max": 52.34165573120117,
      "activations/layer5_attention_weight_min": -59.34150695800781,
      "activations/layer6_attention_weight_max": 43.24061584472656,
      "activations/layer6_attention_weight_min": -46.62779998779297,
      "activations/layer7_attention_weight_max": 83.39244079589844,
      "activations/layer7_attention_weight_min": -88.41249084472656,
      "activations/layer8_attention_weight_max": 38.2664794921875,
      "activations/layer8_attention_weight_min": -39.851905822753906,
      "activations/layer9_attention_weight_max": 32.94749069213867,
      "activations/layer9_attention_weight_min": -32.3754997253418,
      "epoch": 11.32,
      "learning_rate": 7.777083333333332e-05,
      "loss": 2.7718,
      "step": 194850
    },
    {
      "activations/layer0_attention_weight_max": 15.497536659240723,
      "activations/layer0_attention_weight_min": -12.451360702514648,
      "activations/layer10_attention_weight_max": 33.127777099609375,
      "activations/layer10_attention_weight_min": -32.500789642333984,
      "activations/layer11_attention_weight_max": 37.45940399169922,
      "activations/layer11_attention_weight_min": -33.452667236328125,
      "activations/layer12_attention_weight_max": 25.025394439697266,
      "activations/layer12_attention_weight_min": -26.08536720275879,
      "activations/layer13_attention_weight_max": 40.486053466796875,
      "activations/layer13_attention_weight_min": -35.92922592163086,
      "activations/layer14_attention_weight_max": 47.49931716918945,
      "activations/layer14_attention_weight_min": -40.799869537353516,
      "activations/layer15_attention_weight_max": 35.57388687133789,
      "activations/layer15_attention_weight_min": -31.447053909301758,
      "activations/layer16_attention_weight_max": 31.49810028076172,
      "activations/layer16_attention_weight_min": -28.670642852783203,
      "activations/layer17_attention_weight_max": 36.21013641357422,
      "activations/layer17_attention_weight_min": -26.709692001342773,
      "activations/layer18_attention_weight_max": 33.28203582763672,
      "activations/layer18_attention_weight_min": -23.228485107421875,
      "activations/layer19_attention_weight_max": 39.060951232910156,
      "activations/layer19_attention_weight_min": -32.611385345458984,
      "activations/layer1_attention_weight_max": 16.854652404785156,
      "activations/layer1_attention_weight_min": -14.67240047454834,
      "activations/layer20_attention_weight_max": 32.719573974609375,
      "activations/layer20_attention_weight_min": -25.29832649230957,
      "activations/layer21_attention_weight_max": 29.76790428161621,
      "activations/layer21_attention_weight_min": -24.46433448791504,
      "activations/layer22_attention_weight_max": 46.18202209472656,
      "activations/layer22_attention_weight_min": -30.981136322021484,
      "activations/layer23_attention_weight_max": 38.15095520019531,
      "activations/layer23_attention_weight_min": -25.559219360351562,
      "activations/layer2_attention_weight_max": 33.142311096191406,
      "activations/layer2_attention_weight_min": -32.28642654418945,
      "activations/layer3_attention_weight_max": 94.70398712158203,
      "activations/layer3_attention_weight_min": -93.55699157714844,
      "activations/layer4_attention_weight_max": 112.03839111328125,
      "activations/layer4_attention_weight_min": -112.73406982421875,
      "activations/layer5_attention_weight_max": 49.737213134765625,
      "activations/layer5_attention_weight_min": -60.03624725341797,
      "activations/layer6_attention_weight_max": 44.121864318847656,
      "activations/layer6_attention_weight_min": -49.04740905761719,
      "activations/layer7_attention_weight_max": 87.04688262939453,
      "activations/layer7_attention_weight_min": -87.56205749511719,
      "activations/layer8_attention_weight_max": 41.4643669128418,
      "activations/layer8_attention_weight_min": -42.036746978759766,
      "activations/layer9_attention_weight_max": 35.412776947021484,
      "activations/layer9_attention_weight_min": -33.3416633605957,
      "epoch": 11.32,
      "learning_rate": 7.775189393939394e-05,
      "loss": 2.7844,
      "step": 194900
    },
    {
      "activations/layer0_attention_weight_max": 15.675682067871094,
      "activations/layer0_attention_weight_min": -12.327211380004883,
      "activations/layer10_attention_weight_max": 32.275054931640625,
      "activations/layer10_attention_weight_min": -33.09831619262695,
      "activations/layer11_attention_weight_max": 33.30472946166992,
      "activations/layer11_attention_weight_min": -31.90216636657715,
      "activations/layer12_attention_weight_max": 23.3688907623291,
      "activations/layer12_attention_weight_min": -25.7211971282959,
      "activations/layer13_attention_weight_max": 37.767208099365234,
      "activations/layer13_attention_weight_min": -34.84222412109375,
      "activations/layer14_attention_weight_max": 42.42519760131836,
      "activations/layer14_attention_weight_min": -36.16431427001953,
      "activations/layer15_attention_weight_max": 34.616947174072266,
      "activations/layer15_attention_weight_min": -29.764318466186523,
      "activations/layer16_attention_weight_max": 30.421451568603516,
      "activations/layer16_attention_weight_min": -26.643007278442383,
      "activations/layer17_attention_weight_max": 31.268213272094727,
      "activations/layer17_attention_weight_min": -25.270259857177734,
      "activations/layer18_attention_weight_max": 32.824180603027344,
      "activations/layer18_attention_weight_min": -23.874170303344727,
      "activations/layer19_attention_weight_max": 32.662925720214844,
      "activations/layer19_attention_weight_min": -30.01747703552246,
      "activations/layer1_attention_weight_max": 17.26718521118164,
      "activations/layer1_attention_weight_min": -15.507292747497559,
      "activations/layer20_attention_weight_max": 29.36101531982422,
      "activations/layer20_attention_weight_min": -23.282011032104492,
      "activations/layer21_attention_weight_max": 26.96324348449707,
      "activations/layer21_attention_weight_min": -22.791393280029297,
      "activations/layer22_attention_weight_max": 43.78614807128906,
      "activations/layer22_attention_weight_min": -28.30760955810547,
      "activations/layer23_attention_weight_max": 32.54261016845703,
      "activations/layer23_attention_weight_min": -23.91464614868164,
      "activations/layer2_attention_weight_max": 32.72892761230469,
      "activations/layer2_attention_weight_min": -33.41547393798828,
      "activations/layer3_attention_weight_max": 99.79029083251953,
      "activations/layer3_attention_weight_min": -99.53609466552734,
      "activations/layer4_attention_weight_max": 114.644287109375,
      "activations/layer4_attention_weight_min": -119.04998779296875,
      "activations/layer5_attention_weight_max": 52.6519775390625,
      "activations/layer5_attention_weight_min": -62.83662414550781,
      "activations/layer6_attention_weight_max": 44.66083908081055,
      "activations/layer6_attention_weight_min": -48.025794982910156,
      "activations/layer7_attention_weight_max": 90.99296569824219,
      "activations/layer7_attention_weight_min": -97.74877166748047,
      "activations/layer8_attention_weight_max": 40.10014343261719,
      "activations/layer8_attention_weight_min": -42.184383392333984,
      "activations/layer9_attention_weight_max": 34.585262298583984,
      "activations/layer9_attention_weight_min": -33.94053268432617,
      "epoch": 11.33,
      "learning_rate": 7.773295454545454e-05,
      "loss": 2.7847,
      "step": 194950
    },
    {
      "activations/layer0_attention_weight_max": 15.352827072143555,
      "activations/layer0_attention_weight_min": -11.740330696105957,
      "activations/layer10_attention_weight_max": 31.737985610961914,
      "activations/layer10_attention_weight_min": -31.10698699951172,
      "activations/layer11_attention_weight_max": 31.752721786499023,
      "activations/layer11_attention_weight_min": -32.753883361816406,
      "activations/layer12_attention_weight_max": 22.631717681884766,
      "activations/layer12_attention_weight_min": -24.657194137573242,
      "activations/layer13_attention_weight_max": 36.127254486083984,
      "activations/layer13_attention_weight_min": -33.38216018676758,
      "activations/layer14_attention_weight_max": 38.05805206298828,
      "activations/layer14_attention_weight_min": -34.78766632080078,
      "activations/layer15_attention_weight_max": 33.56011962890625,
      "activations/layer15_attention_weight_min": -30.360074996948242,
      "activations/layer16_attention_weight_max": 27.405128479003906,
      "activations/layer16_attention_weight_min": -27.983314514160156,
      "activations/layer17_attention_weight_max": 31.403493881225586,
      "activations/layer17_attention_weight_min": -26.707645416259766,
      "activations/layer18_attention_weight_max": 27.475500106811523,
      "activations/layer18_attention_weight_min": -23.127607345581055,
      "activations/layer19_attention_weight_max": 30.281326293945312,
      "activations/layer19_attention_weight_min": -33.487545013427734,
      "activations/layer1_attention_weight_max": 16.646024703979492,
      "activations/layer1_attention_weight_min": -15.882035255432129,
      "activations/layer20_attention_weight_max": 28.809507369995117,
      "activations/layer20_attention_weight_min": -25.08543586730957,
      "activations/layer21_attention_weight_max": 25.65555763244629,
      "activations/layer21_attention_weight_min": -23.493816375732422,
      "activations/layer22_attention_weight_max": 38.417625427246094,
      "activations/layer22_attention_weight_min": -29.691635131835938,
      "activations/layer23_attention_weight_max": 29.570308685302734,
      "activations/layer23_attention_weight_min": -24.372920989990234,
      "activations/layer2_attention_weight_max": 32.90886306762695,
      "activations/layer2_attention_weight_min": -32.656673431396484,
      "activations/layer3_attention_weight_max": 98.4183120727539,
      "activations/layer3_attention_weight_min": -102.40813446044922,
      "activations/layer4_attention_weight_max": 112.1683120727539,
      "activations/layer4_attention_weight_min": -112.25654602050781,
      "activations/layer5_attention_weight_max": 52.389888763427734,
      "activations/layer5_attention_weight_min": -63.49474334716797,
      "activations/layer6_attention_weight_max": 45.964332580566406,
      "activations/layer6_attention_weight_min": -51.269283294677734,
      "activations/layer7_attention_weight_max": 88.93688201904297,
      "activations/layer7_attention_weight_min": -87.1701889038086,
      "activations/layer8_attention_weight_max": 37.81845474243164,
      "activations/layer8_attention_weight_min": -39.10445022583008,
      "activations/layer9_attention_weight_max": 31.3455867767334,
      "activations/layer9_attention_weight_min": -32.26426315307617,
      "epoch": 11.33,
      "learning_rate": 7.771401515151514e-05,
      "loss": 2.7763,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.5294,
      "eval_samples_per_second": 503.435,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.5294,
      "eval_openwebtext_samples_per_second": 503.435,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0298,
      "eval_wikitext_samples_per_second": 224.65,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.6449,
      "eval_lambada_samples_per_second": 504.826,
      "step": 195000
    },
    {
      "activations/layer0_attention_weight_max": 16.276247024536133,
      "activations/layer0_attention_weight_min": -12.272944450378418,
      "activations/layer10_attention_weight_max": 39.560752868652344,
      "activations/layer10_attention_weight_min": -34.48758316040039,
      "activations/layer11_attention_weight_max": 38.65761184692383,
      "activations/layer11_attention_weight_min": -34.8130989074707,
      "activations/layer12_attention_weight_max": 24.09927749633789,
      "activations/layer12_attention_weight_min": -25.449060440063477,
      "activations/layer13_attention_weight_max": 35.701236724853516,
      "activations/layer13_attention_weight_min": -35.301963806152344,
      "activations/layer14_attention_weight_max": 41.9995231628418,
      "activations/layer14_attention_weight_min": -37.935829162597656,
      "activations/layer15_attention_weight_max": 34.67518615722656,
      "activations/layer15_attention_weight_min": -31.85605812072754,
      "activations/layer16_attention_weight_max": 30.960493087768555,
      "activations/layer16_attention_weight_min": -27.541393280029297,
      "activations/layer17_attention_weight_max": 28.80843734741211,
      "activations/layer17_attention_weight_min": -26.73954200744629,
      "activations/layer18_attention_weight_max": 31.02153968811035,
      "activations/layer18_attention_weight_min": -24.751590728759766,
      "activations/layer19_attention_weight_max": 33.11082077026367,
      "activations/layer19_attention_weight_min": -30.70013999938965,
      "activations/layer1_attention_weight_max": 18.062637329101562,
      "activations/layer1_attention_weight_min": -14.514649391174316,
      "activations/layer20_attention_weight_max": 27.039216995239258,
      "activations/layer20_attention_weight_min": -24.655183792114258,
      "activations/layer21_attention_weight_max": 28.18668556213379,
      "activations/layer21_attention_weight_min": -24.27695655822754,
      "activations/layer22_attention_weight_max": 42.2566032409668,
      "activations/layer22_attention_weight_min": -31.37264060974121,
      "activations/layer23_attention_weight_max": 35.08241271972656,
      "activations/layer23_attention_weight_min": -24.94875144958496,
      "activations/layer2_attention_weight_max": 32.53736877441406,
      "activations/layer2_attention_weight_min": -34.09931945800781,
      "activations/layer3_attention_weight_max": 98.88041687011719,
      "activations/layer3_attention_weight_min": -95.60901641845703,
      "activations/layer4_attention_weight_max": 111.55252838134766,
      "activations/layer4_attention_weight_min": -113.05428314208984,
      "activations/layer5_attention_weight_max": 51.98643112182617,
      "activations/layer5_attention_weight_min": -61.84873580932617,
      "activations/layer6_attention_weight_max": 43.142269134521484,
      "activations/layer6_attention_weight_min": -48.09379577636719,
      "activations/layer7_attention_weight_max": 88.44628143310547,
      "activations/layer7_attention_weight_min": -92.66519927978516,
      "activations/layer8_attention_weight_max": 43.32108688354492,
      "activations/layer8_attention_weight_min": -39.82495880126953,
      "activations/layer9_attention_weight_max": 38.4412727355957,
      "activations/layer9_attention_weight_min": -33.07379913330078,
      "epoch": 11.33,
      "learning_rate": 7.769507575757576e-05,
      "loss": 2.7581,
      "step": 195050
    },
    {
      "activations/layer0_attention_weight_max": 15.31273078918457,
      "activations/layer0_attention_weight_min": -12.271674156188965,
      "activations/layer10_attention_weight_max": 31.060760498046875,
      "activations/layer10_attention_weight_min": -31.925872802734375,
      "activations/layer11_attention_weight_max": 31.815475463867188,
      "activations/layer11_attention_weight_min": -30.120168685913086,
      "activations/layer12_attention_weight_max": 25.865205764770508,
      "activations/layer12_attention_weight_min": -26.38245964050293,
      "activations/layer13_attention_weight_max": 38.242042541503906,
      "activations/layer13_attention_weight_min": -34.892051696777344,
      "activations/layer14_attention_weight_max": 46.44379425048828,
      "activations/layer14_attention_weight_min": -36.79452133178711,
      "activations/layer15_attention_weight_max": 39.1099967956543,
      "activations/layer15_attention_weight_min": -31.521442413330078,
      "activations/layer16_attention_weight_max": 32.15688705444336,
      "activations/layer16_attention_weight_min": -29.564090728759766,
      "activations/layer17_attention_weight_max": 35.48066711425781,
      "activations/layer17_attention_weight_min": -26.20029640197754,
      "activations/layer18_attention_weight_max": 33.281280517578125,
      "activations/layer18_attention_weight_min": -23.130966186523438,
      "activations/layer19_attention_weight_max": 33.54480743408203,
      "activations/layer19_attention_weight_min": -30.377750396728516,
      "activations/layer1_attention_weight_max": 16.154132843017578,
      "activations/layer1_attention_weight_min": -16.430828094482422,
      "activations/layer20_attention_weight_max": 27.935184478759766,
      "activations/layer20_attention_weight_min": -24.14325714111328,
      "activations/layer21_attention_weight_max": 28.644577026367188,
      "activations/layer21_attention_weight_min": -23.444889068603516,
      "activations/layer22_attention_weight_max": 47.00862503051758,
      "activations/layer22_attention_weight_min": -32.23930740356445,
      "activations/layer23_attention_weight_max": 33.58367156982422,
      "activations/layer23_attention_weight_min": -27.37680435180664,
      "activations/layer2_attention_weight_max": 34.52095031738281,
      "activations/layer2_attention_weight_min": -33.07862854003906,
      "activations/layer3_attention_weight_max": 97.73905181884766,
      "activations/layer3_attention_weight_min": -98.31100463867188,
      "activations/layer4_attention_weight_max": 115.18761444091797,
      "activations/layer4_attention_weight_min": -113.52784729003906,
      "activations/layer5_attention_weight_max": 54.08076095581055,
      "activations/layer5_attention_weight_min": -61.76936340332031,
      "activations/layer6_attention_weight_max": 45.8922004699707,
      "activations/layer6_attention_weight_min": -46.73625183105469,
      "activations/layer7_attention_weight_max": 102.50789642333984,
      "activations/layer7_attention_weight_min": -90.37246704101562,
      "activations/layer8_attention_weight_max": 40.499534606933594,
      "activations/layer8_attention_weight_min": -39.21186065673828,
      "activations/layer9_attention_weight_max": 31.714969635009766,
      "activations/layer9_attention_weight_min": -32.20164108276367,
      "epoch": 11.34,
      "learning_rate": 7.767613636363636e-05,
      "loss": 2.7627,
      "step": 195100
    },
    {
      "activations/layer0_attention_weight_max": 15.859552383422852,
      "activations/layer0_attention_weight_min": -12.134552001953125,
      "activations/layer10_attention_weight_max": 37.65834426879883,
      "activations/layer10_attention_weight_min": -34.89552688598633,
      "activations/layer11_attention_weight_max": 36.33479309082031,
      "activations/layer11_attention_weight_min": -34.9910774230957,
      "activations/layer12_attention_weight_max": 24.127180099487305,
      "activations/layer12_attention_weight_min": -29.641080856323242,
      "activations/layer13_attention_weight_max": 37.87776184082031,
      "activations/layer13_attention_weight_min": -33.86661148071289,
      "activations/layer14_attention_weight_max": 43.854652404785156,
      "activations/layer14_attention_weight_min": -41.64667510986328,
      "activations/layer15_attention_weight_max": 34.88345718383789,
      "activations/layer15_attention_weight_min": -33.52729034423828,
      "activations/layer16_attention_weight_max": 33.40719985961914,
      "activations/layer16_attention_weight_min": -28.11448097229004,
      "activations/layer17_attention_weight_max": 29.978017807006836,
      "activations/layer17_attention_weight_min": -26.47312355041504,
      "activations/layer18_attention_weight_max": 31.87883949279785,
      "activations/layer18_attention_weight_min": -23.788482666015625,
      "activations/layer19_attention_weight_max": 37.112300872802734,
      "activations/layer19_attention_weight_min": -34.35763931274414,
      "activations/layer1_attention_weight_max": 17.12126350402832,
      "activations/layer1_attention_weight_min": -15.979246139526367,
      "activations/layer20_attention_weight_max": 30.379894256591797,
      "activations/layer20_attention_weight_min": -25.069412231445312,
      "activations/layer21_attention_weight_max": 28.220029830932617,
      "activations/layer21_attention_weight_min": -24.55367088317871,
      "activations/layer22_attention_weight_max": 47.367794036865234,
      "activations/layer22_attention_weight_min": -29.415571212768555,
      "activations/layer23_attention_weight_max": 35.85703659057617,
      "activations/layer23_attention_weight_min": -25.374656677246094,
      "activations/layer2_attention_weight_max": 35.03287887573242,
      "activations/layer2_attention_weight_min": -37.533447265625,
      "activations/layer3_attention_weight_max": 99.99129486083984,
      "activations/layer3_attention_weight_min": -108.47293853759766,
      "activations/layer4_attention_weight_max": 114.87422943115234,
      "activations/layer4_attention_weight_min": -119.90614318847656,
      "activations/layer5_attention_weight_max": 53.180076599121094,
      "activations/layer5_attention_weight_min": -60.48188018798828,
      "activations/layer6_attention_weight_max": 46.782962799072266,
      "activations/layer6_attention_weight_min": -52.18873977661133,
      "activations/layer7_attention_weight_max": 103.662109375,
      "activations/layer7_attention_weight_min": -107.56332397460938,
      "activations/layer8_attention_weight_max": 45.37660598754883,
      "activations/layer8_attention_weight_min": -47.322574615478516,
      "activations/layer9_attention_weight_max": 37.492496490478516,
      "activations/layer9_attention_weight_min": -39.588558197021484,
      "epoch": 11.34,
      "learning_rate": 7.765719696969696e-05,
      "loss": 2.7873,
      "step": 195150
    },
    {
      "activations/layer0_attention_weight_max": 16.195327758789062,
      "activations/layer0_attention_weight_min": -11.225576400756836,
      "activations/layer10_attention_weight_max": 34.84968948364258,
      "activations/layer10_attention_weight_min": -31.984580993652344,
      "activations/layer11_attention_weight_max": 38.675537109375,
      "activations/layer11_attention_weight_min": -32.19880676269531,
      "activations/layer12_attention_weight_max": 22.8978271484375,
      "activations/layer12_attention_weight_min": -24.48204803466797,
      "activations/layer13_attention_weight_max": 34.30877685546875,
      "activations/layer13_attention_weight_min": -33.73027420043945,
      "activations/layer14_attention_weight_max": 47.63181686401367,
      "activations/layer14_attention_weight_min": -36.41819763183594,
      "activations/layer15_attention_weight_max": 36.73720932006836,
      "activations/layer15_attention_weight_min": -30.952634811401367,
      "activations/layer16_attention_weight_max": 32.77022933959961,
      "activations/layer16_attention_weight_min": -27.246103286743164,
      "activations/layer17_attention_weight_max": 32.75302505493164,
      "activations/layer17_attention_weight_min": -25.59315299987793,
      "activations/layer18_attention_weight_max": 29.448463439941406,
      "activations/layer18_attention_weight_min": -21.895177841186523,
      "activations/layer19_attention_weight_max": 35.243064880371094,
      "activations/layer19_attention_weight_min": -29.143491744995117,
      "activations/layer1_attention_weight_max": 16.566373825073242,
      "activations/layer1_attention_weight_min": -16.161287307739258,
      "activations/layer20_attention_weight_max": 28.9700984954834,
      "activations/layer20_attention_weight_min": -22.904497146606445,
      "activations/layer21_attention_weight_max": 27.543350219726562,
      "activations/layer21_attention_weight_min": -22.9830265045166,
      "activations/layer22_attention_weight_max": 47.343936920166016,
      "activations/layer22_attention_weight_min": -29.43944549560547,
      "activations/layer23_attention_weight_max": 37.353092193603516,
      "activations/layer23_attention_weight_min": -24.957420349121094,
      "activations/layer2_attention_weight_max": 38.024269104003906,
      "activations/layer2_attention_weight_min": -35.614830017089844,
      "activations/layer3_attention_weight_max": 105.13325500488281,
      "activations/layer3_attention_weight_min": -97.8417739868164,
      "activations/layer4_attention_weight_max": 112.60391998291016,
      "activations/layer4_attention_weight_min": -110.58332824707031,
      "activations/layer5_attention_weight_max": 56.50234603881836,
      "activations/layer5_attention_weight_min": -58.300636291503906,
      "activations/layer6_attention_weight_max": 43.068965911865234,
      "activations/layer6_attention_weight_min": -46.10810852050781,
      "activations/layer7_attention_weight_max": 101.00392150878906,
      "activations/layer7_attention_weight_min": -88.33302307128906,
      "activations/layer8_attention_weight_max": 40.737709045410156,
      "activations/layer8_attention_weight_min": -39.9538688659668,
      "activations/layer9_attention_weight_max": 32.583187103271484,
      "activations/layer9_attention_weight_min": -32.075782775878906,
      "epoch": 11.34,
      "learning_rate": 7.763825757575756e-05,
      "loss": 2.7794,
      "step": 195200
    },
    {
      "activations/layer0_attention_weight_max": 16.464115142822266,
      "activations/layer0_attention_weight_min": -12.656493186950684,
      "activations/layer10_attention_weight_max": 34.155982971191406,
      "activations/layer10_attention_weight_min": -32.426734924316406,
      "activations/layer11_attention_weight_max": 34.35149002075195,
      "activations/layer11_attention_weight_min": -31.910900115966797,
      "activations/layer12_attention_weight_max": 24.991744995117188,
      "activations/layer12_attention_weight_min": -25.95475959777832,
      "activations/layer13_attention_weight_max": 39.970115661621094,
      "activations/layer13_attention_weight_min": -37.00025177001953,
      "activations/layer14_attention_weight_max": 46.64956283569336,
      "activations/layer14_attention_weight_min": -41.48098373413086,
      "activations/layer15_attention_weight_max": 38.79573440551758,
      "activations/layer15_attention_weight_min": -32.276275634765625,
      "activations/layer16_attention_weight_max": 34.14084243774414,
      "activations/layer16_attention_weight_min": -28.82004737854004,
      "activations/layer17_attention_weight_max": 32.47626876831055,
      "activations/layer17_attention_weight_min": -24.70374870300293,
      "activations/layer18_attention_weight_max": 30.222993850708008,
      "activations/layer18_attention_weight_min": -23.068349838256836,
      "activations/layer19_attention_weight_max": 37.79804611206055,
      "activations/layer19_attention_weight_min": -32.43027877807617,
      "activations/layer1_attention_weight_max": 17.465816497802734,
      "activations/layer1_attention_weight_min": -15.684479713439941,
      "activations/layer20_attention_weight_max": 29.491182327270508,
      "activations/layer20_attention_weight_min": -25.106847763061523,
      "activations/layer21_attention_weight_max": 28.406448364257812,
      "activations/layer21_attention_weight_min": -24.328500747680664,
      "activations/layer22_attention_weight_max": 44.32966613769531,
      "activations/layer22_attention_weight_min": -28.94706916809082,
      "activations/layer23_attention_weight_max": 37.292030334472656,
      "activations/layer23_attention_weight_min": -24.62310028076172,
      "activations/layer2_attention_weight_max": 35.813751220703125,
      "activations/layer2_attention_weight_min": -35.175228118896484,
      "activations/layer3_attention_weight_max": 103.40335083007812,
      "activations/layer3_attention_weight_min": -102.55754089355469,
      "activations/layer4_attention_weight_max": 119.35774230957031,
      "activations/layer4_attention_weight_min": -117.5398178100586,
      "activations/layer5_attention_weight_max": 56.27415466308594,
      "activations/layer5_attention_weight_min": -64.0307846069336,
      "activations/layer6_attention_weight_max": 48.24015426635742,
      "activations/layer6_attention_weight_min": -50.32109451293945,
      "activations/layer7_attention_weight_max": 91.29673767089844,
      "activations/layer7_attention_weight_min": -89.00112915039062,
      "activations/layer8_attention_weight_max": 43.37346267700195,
      "activations/layer8_attention_weight_min": -41.07497787475586,
      "activations/layer9_attention_weight_max": 34.330902099609375,
      "activations/layer9_attention_weight_min": -32.646507263183594,
      "epoch": 11.35,
      "learning_rate": 7.761931818181818e-05,
      "loss": 2.7809,
      "step": 195250
    },
    {
      "activations/layer0_attention_weight_max": 16.327468872070312,
      "activations/layer0_attention_weight_min": -12.027074813842773,
      "activations/layer10_attention_weight_max": 32.94491958618164,
      "activations/layer10_attention_weight_min": -33.02911376953125,
      "activations/layer11_attention_weight_max": 33.410037994384766,
      "activations/layer11_attention_weight_min": -32.59415054321289,
      "activations/layer12_attention_weight_max": 27.515024185180664,
      "activations/layer12_attention_weight_min": -24.387954711914062,
      "activations/layer13_attention_weight_max": 40.32456970214844,
      "activations/layer13_attention_weight_min": -35.52488708496094,
      "activations/layer14_attention_weight_max": 46.30742263793945,
      "activations/layer14_attention_weight_min": -37.456886291503906,
      "activations/layer15_attention_weight_max": 37.55740737915039,
      "activations/layer15_attention_weight_min": -30.32748031616211,
      "activations/layer16_attention_weight_max": 30.754825592041016,
      "activations/layer16_attention_weight_min": -27.185766220092773,
      "activations/layer17_attention_weight_max": 30.42154884338379,
      "activations/layer17_attention_weight_min": -25.484146118164062,
      "activations/layer18_attention_weight_max": 30.41288948059082,
      "activations/layer18_attention_weight_min": -22.290695190429688,
      "activations/layer19_attention_weight_max": 32.62104415893555,
      "activations/layer19_attention_weight_min": -29.903451919555664,
      "activations/layer1_attention_weight_max": 17.61385726928711,
      "activations/layer1_attention_weight_min": -15.587288856506348,
      "activations/layer20_attention_weight_max": 26.13942527770996,
      "activations/layer20_attention_weight_min": -25.08095359802246,
      "activations/layer21_attention_weight_max": 27.25412940979004,
      "activations/layer21_attention_weight_min": -25.260862350463867,
      "activations/layer22_attention_weight_max": 46.662960052490234,
      "activations/layer22_attention_weight_min": -29.194984436035156,
      "activations/layer23_attention_weight_max": 33.41774368286133,
      "activations/layer23_attention_weight_min": -24.593151092529297,
      "activations/layer2_attention_weight_max": 34.40642547607422,
      "activations/layer2_attention_weight_min": -35.369239807128906,
      "activations/layer3_attention_weight_max": 97.48273468017578,
      "activations/layer3_attention_weight_min": -102.01469421386719,
      "activations/layer4_attention_weight_max": 120.8577651977539,
      "activations/layer4_attention_weight_min": -122.5728759765625,
      "activations/layer5_attention_weight_max": 54.9522705078125,
      "activations/layer5_attention_weight_min": -63.39426803588867,
      "activations/layer6_attention_weight_max": 46.97065353393555,
      "activations/layer6_attention_weight_min": -52.25600051879883,
      "activations/layer7_attention_weight_max": 92.94122314453125,
      "activations/layer7_attention_weight_min": -94.82149505615234,
      "activations/layer8_attention_weight_max": 42.38648223876953,
      "activations/layer8_attention_weight_min": -43.45058822631836,
      "activations/layer9_attention_weight_max": 33.30202102661133,
      "activations/layer9_attention_weight_min": -34.702884674072266,
      "epoch": 11.35,
      "learning_rate": 7.760037878787879e-05,
      "loss": 2.8015,
      "step": 195300
    },
    {
      "activations/layer0_attention_weight_max": 17.257497787475586,
      "activations/layer0_attention_weight_min": -13.25823974609375,
      "activations/layer10_attention_weight_max": 35.48872375488281,
      "activations/layer10_attention_weight_min": -34.410316467285156,
      "activations/layer11_attention_weight_max": 39.415985107421875,
      "activations/layer11_attention_weight_min": -32.10821533203125,
      "activations/layer12_attention_weight_max": 28.088598251342773,
      "activations/layer12_attention_weight_min": -25.42429542541504,
      "activations/layer13_attention_weight_max": 44.762718200683594,
      "activations/layer13_attention_weight_min": -36.14662551879883,
      "activations/layer14_attention_weight_max": 53.813602447509766,
      "activations/layer14_attention_weight_min": -41.177982330322266,
      "activations/layer15_attention_weight_max": 39.56975555419922,
      "activations/layer15_attention_weight_min": -31.767240524291992,
      "activations/layer16_attention_weight_max": 33.60200500488281,
      "activations/layer16_attention_weight_min": -27.317981719970703,
      "activations/layer17_attention_weight_max": 35.6443977355957,
      "activations/layer17_attention_weight_min": -26.82207489013672,
      "activations/layer18_attention_weight_max": 33.32170486450195,
      "activations/layer18_attention_weight_min": -23.9608154296875,
      "activations/layer19_attention_weight_max": 38.38247299194336,
      "activations/layer19_attention_weight_min": -31.025711059570312,
      "activations/layer1_attention_weight_max": 16.29831314086914,
      "activations/layer1_attention_weight_min": -15.365272521972656,
      "activations/layer20_attention_weight_max": 33.942955017089844,
      "activations/layer20_attention_weight_min": -25.862186431884766,
      "activations/layer21_attention_weight_max": 35.71892166137695,
      "activations/layer21_attention_weight_min": -23.446090698242188,
      "activations/layer22_attention_weight_max": 49.980125427246094,
      "activations/layer22_attention_weight_min": -30.06169319152832,
      "activations/layer23_attention_weight_max": 37.272193908691406,
      "activations/layer23_attention_weight_min": -23.234710693359375,
      "activations/layer2_attention_weight_max": 34.39436340332031,
      "activations/layer2_attention_weight_min": -33.26021194458008,
      "activations/layer3_attention_weight_max": 100.51810455322266,
      "activations/layer3_attention_weight_min": -97.06546783447266,
      "activations/layer4_attention_weight_max": 118.62691497802734,
      "activations/layer4_attention_weight_min": -114.88922119140625,
      "activations/layer5_attention_weight_max": 55.12310028076172,
      "activations/layer5_attention_weight_min": -59.92346954345703,
      "activations/layer6_attention_weight_max": 45.922786712646484,
      "activations/layer6_attention_weight_min": -46.96165084838867,
      "activations/layer7_attention_weight_max": 88.1605453491211,
      "activations/layer7_attention_weight_min": -91.57266998291016,
      "activations/layer8_attention_weight_max": 40.27862548828125,
      "activations/layer8_attention_weight_min": -41.546024322509766,
      "activations/layer9_attention_weight_max": 33.327144622802734,
      "activations/layer9_attention_weight_min": -35.2457160949707,
      "epoch": 11.35,
      "learning_rate": 7.758143939393938e-05,
      "loss": 2.781,
      "step": 195350
    },
    {
      "activations/layer0_attention_weight_max": 16.589004516601562,
      "activations/layer0_attention_weight_min": -12.89403247833252,
      "activations/layer10_attention_weight_max": 32.53485870361328,
      "activations/layer10_attention_weight_min": -35.658546447753906,
      "activations/layer11_attention_weight_max": 32.639503479003906,
      "activations/layer11_attention_weight_min": -35.987518310546875,
      "activations/layer12_attention_weight_max": 24.38488006591797,
      "activations/layer12_attention_weight_min": -26.558670043945312,
      "activations/layer13_attention_weight_max": 40.947242736816406,
      "activations/layer13_attention_weight_min": -34.37363815307617,
      "activations/layer14_attention_weight_max": 47.32236862182617,
      "activations/layer14_attention_weight_min": -38.437862396240234,
      "activations/layer15_attention_weight_max": 37.11845779418945,
      "activations/layer15_attention_weight_min": -31.590839385986328,
      "activations/layer16_attention_weight_max": 33.16465759277344,
      "activations/layer16_attention_weight_min": -26.29570198059082,
      "activations/layer17_attention_weight_max": 33.10771560668945,
      "activations/layer17_attention_weight_min": -24.30164909362793,
      "activations/layer18_attention_weight_max": 32.91432189941406,
      "activations/layer18_attention_weight_min": -24.346267700195312,
      "activations/layer19_attention_weight_max": 39.09638214111328,
      "activations/layer19_attention_weight_min": -30.207950592041016,
      "activations/layer1_attention_weight_max": 15.65404987335205,
      "activations/layer1_attention_weight_min": -14.163888931274414,
      "activations/layer20_attention_weight_max": 33.484779357910156,
      "activations/layer20_attention_weight_min": -24.41905975341797,
      "activations/layer21_attention_weight_max": 30.80904197692871,
      "activations/layer21_attention_weight_min": -24.592838287353516,
      "activations/layer22_attention_weight_max": 50.84500503540039,
      "activations/layer22_attention_weight_min": -30.79754638671875,
      "activations/layer23_attention_weight_max": 44.00326156616211,
      "activations/layer23_attention_weight_min": -26.170677185058594,
      "activations/layer2_attention_weight_max": 33.44547653198242,
      "activations/layer2_attention_weight_min": -34.08851623535156,
      "activations/layer3_attention_weight_max": 97.77485656738281,
      "activations/layer3_attention_weight_min": -99.49627685546875,
      "activations/layer4_attention_weight_max": 117.4228286743164,
      "activations/layer4_attention_weight_min": -122.26007080078125,
      "activations/layer5_attention_weight_max": 51.29560089111328,
      "activations/layer5_attention_weight_min": -62.97904586791992,
      "activations/layer6_attention_weight_max": 45.22275161743164,
      "activations/layer6_attention_weight_min": -48.73074722290039,
      "activations/layer7_attention_weight_max": 90.63198852539062,
      "activations/layer7_attention_weight_min": -92.8659439086914,
      "activations/layer8_attention_weight_max": 41.530818939208984,
      "activations/layer8_attention_weight_min": -45.73406982421875,
      "activations/layer9_attention_weight_max": 34.122005462646484,
      "activations/layer9_attention_weight_min": -36.4686279296875,
      "epoch": 11.35,
      "learning_rate": 7.75625e-05,
      "loss": 2.7918,
      "step": 195400
    },
    {
      "activations/layer0_attention_weight_max": 16.103620529174805,
      "activations/layer0_attention_weight_min": -12.316426277160645,
      "activations/layer10_attention_weight_max": 33.544403076171875,
      "activations/layer10_attention_weight_min": -31.813213348388672,
      "activations/layer11_attention_weight_max": 33.490577697753906,
      "activations/layer11_attention_weight_min": -31.612730026245117,
      "activations/layer12_attention_weight_max": 26.79062271118164,
      "activations/layer12_attention_weight_min": -25.254770278930664,
      "activations/layer13_attention_weight_max": 38.056785583496094,
      "activations/layer13_attention_weight_min": -34.55929183959961,
      "activations/layer14_attention_weight_max": 41.60511779785156,
      "activations/layer14_attention_weight_min": -37.30768966674805,
      "activations/layer15_attention_weight_max": 36.15290069580078,
      "activations/layer15_attention_weight_min": -32.43899917602539,
      "activations/layer16_attention_weight_max": 31.075708389282227,
      "activations/layer16_attention_weight_min": -30.075803756713867,
      "activations/layer17_attention_weight_max": 35.100093841552734,
      "activations/layer17_attention_weight_min": -27.530250549316406,
      "activations/layer18_attention_weight_max": 31.648460388183594,
      "activations/layer18_attention_weight_min": -25.217998504638672,
      "activations/layer19_attention_weight_max": 35.87293243408203,
      "activations/layer19_attention_weight_min": -31.31610679626465,
      "activations/layer1_attention_weight_max": 17.113004684448242,
      "activations/layer1_attention_weight_min": -15.471549987792969,
      "activations/layer20_attention_weight_max": 29.096906661987305,
      "activations/layer20_attention_weight_min": -25.013572692871094,
      "activations/layer21_attention_weight_max": 32.58661651611328,
      "activations/layer21_attention_weight_min": -24.342798233032227,
      "activations/layer22_attention_weight_max": 45.68680191040039,
      "activations/layer22_attention_weight_min": -30.51384162902832,
      "activations/layer23_attention_weight_max": 34.424137115478516,
      "activations/layer23_attention_weight_min": -25.435087203979492,
      "activations/layer2_attention_weight_max": 34.3979377746582,
      "activations/layer2_attention_weight_min": -33.44352722167969,
      "activations/layer3_attention_weight_max": 99.80525970458984,
      "activations/layer3_attention_weight_min": -102.07939147949219,
      "activations/layer4_attention_weight_max": 117.69734191894531,
      "activations/layer4_attention_weight_min": -115.26090240478516,
      "activations/layer5_attention_weight_max": 54.668853759765625,
      "activations/layer5_attention_weight_min": -64.30752563476562,
      "activations/layer6_attention_weight_max": 45.60430145263672,
      "activations/layer6_attention_weight_min": -50.05244445800781,
      "activations/layer7_attention_weight_max": 91.53630065917969,
      "activations/layer7_attention_weight_min": -89.05561065673828,
      "activations/layer8_attention_weight_max": 40.57566833496094,
      "activations/layer8_attention_weight_min": -39.95834732055664,
      "activations/layer9_attention_weight_max": 33.461578369140625,
      "activations/layer9_attention_weight_min": -32.81135177612305,
      "epoch": 11.36,
      "learning_rate": 7.754356060606061e-05,
      "loss": 2.7613,
      "step": 195450
    },
    {
      "activations/layer0_attention_weight_max": 16.15068244934082,
      "activations/layer0_attention_weight_min": -13.368739128112793,
      "activations/layer10_attention_weight_max": 37.6380500793457,
      "activations/layer10_attention_weight_min": -35.332279205322266,
      "activations/layer11_attention_weight_max": 36.44505310058594,
      "activations/layer11_attention_weight_min": -36.00029373168945,
      "activations/layer12_attention_weight_max": 23.309431076049805,
      "activations/layer12_attention_weight_min": -40.33997344970703,
      "activations/layer13_attention_weight_max": 38.880088806152344,
      "activations/layer13_attention_weight_min": -34.09537887573242,
      "activations/layer14_attention_weight_max": 37.88557815551758,
      "activations/layer14_attention_weight_min": -36.85134506225586,
      "activations/layer15_attention_weight_max": 33.088871002197266,
      "activations/layer15_attention_weight_min": -31.49611473083496,
      "activations/layer16_attention_weight_max": 27.625717163085938,
      "activations/layer16_attention_weight_min": -27.233505249023438,
      "activations/layer17_attention_weight_max": 31.36979866027832,
      "activations/layer17_attention_weight_min": -25.238391876220703,
      "activations/layer18_attention_weight_max": 34.76435470581055,
      "activations/layer18_attention_weight_min": -24.01581573486328,
      "activations/layer19_attention_weight_max": 32.61405563354492,
      "activations/layer19_attention_weight_min": -31.6813907623291,
      "activations/layer1_attention_weight_max": 17.098934173583984,
      "activations/layer1_attention_weight_min": -14.485471725463867,
      "activations/layer20_attention_weight_max": 27.143455505371094,
      "activations/layer20_attention_weight_min": -24.379880905151367,
      "activations/layer21_attention_weight_max": 30.965099334716797,
      "activations/layer21_attention_weight_min": -22.489667892456055,
      "activations/layer22_attention_weight_max": 38.745567321777344,
      "activations/layer22_attention_weight_min": -31.60752296447754,
      "activations/layer23_attention_weight_max": 31.8813533782959,
      "activations/layer23_attention_weight_min": -27.337810516357422,
      "activations/layer2_attention_weight_max": 33.906044006347656,
      "activations/layer2_attention_weight_min": -34.848838806152344,
      "activations/layer3_attention_weight_max": 94.4023208618164,
      "activations/layer3_attention_weight_min": -100.5929183959961,
      "activations/layer4_attention_weight_max": 110.03563690185547,
      "activations/layer4_attention_weight_min": -110.26661682128906,
      "activations/layer5_attention_weight_max": 55.18921661376953,
      "activations/layer5_attention_weight_min": -66.30378723144531,
      "activations/layer6_attention_weight_max": 44.39949035644531,
      "activations/layer6_attention_weight_min": -46.801082611083984,
      "activations/layer7_attention_weight_max": 97.90301513671875,
      "activations/layer7_attention_weight_min": -92.99830627441406,
      "activations/layer8_attention_weight_max": 43.025665283203125,
      "activations/layer8_attention_weight_min": -40.39371109008789,
      "activations/layer9_attention_weight_max": 33.30836486816406,
      "activations/layer9_attention_weight_min": -32.836849212646484,
      "epoch": 11.36,
      "learning_rate": 7.75246212121212e-05,
      "loss": 2.7611,
      "step": 195500
    },
    {
      "activations/layer0_attention_weight_max": 15.391031265258789,
      "activations/layer0_attention_weight_min": -12.053519248962402,
      "activations/layer10_attention_weight_max": 34.55136489868164,
      "activations/layer10_attention_weight_min": -32.106964111328125,
      "activations/layer11_attention_weight_max": 34.119075775146484,
      "activations/layer11_attention_weight_min": -30.169889450073242,
      "activations/layer12_attention_weight_max": 26.546920776367188,
      "activations/layer12_attention_weight_min": -36.80023193359375,
      "activations/layer13_attention_weight_max": 40.497589111328125,
      "activations/layer13_attention_weight_min": -32.867408752441406,
      "activations/layer14_attention_weight_max": 44.92459487915039,
      "activations/layer14_attention_weight_min": -37.02501678466797,
      "activations/layer15_attention_weight_max": 37.2190055847168,
      "activations/layer15_attention_weight_min": -31.006484985351562,
      "activations/layer16_attention_weight_max": 32.00541305541992,
      "activations/layer16_attention_weight_min": -26.65121841430664,
      "activations/layer17_attention_weight_max": 31.68248176574707,
      "activations/layer17_attention_weight_min": -26.05544662475586,
      "activations/layer18_attention_weight_max": 33.57392501831055,
      "activations/layer18_attention_weight_min": -23.579320907592773,
      "activations/layer19_attention_weight_max": 30.959623336791992,
      "activations/layer19_attention_weight_min": -31.539295196533203,
      "activations/layer1_attention_weight_max": 16.497900009155273,
      "activations/layer1_attention_weight_min": -14.553424835205078,
      "activations/layer20_attention_weight_max": 27.798709869384766,
      "activations/layer20_attention_weight_min": -26.449216842651367,
      "activations/layer21_attention_weight_max": 26.524105072021484,
      "activations/layer21_attention_weight_min": -23.352066040039062,
      "activations/layer22_attention_weight_max": 40.899593353271484,
      "activations/layer22_attention_weight_min": -33.15254592895508,
      "activations/layer23_attention_weight_max": 31.55874252319336,
      "activations/layer23_attention_weight_min": -29.983657836914062,
      "activations/layer2_attention_weight_max": 35.168701171875,
      "activations/layer2_attention_weight_min": -32.32422637939453,
      "activations/layer3_attention_weight_max": 94.75032043457031,
      "activations/layer3_attention_weight_min": -95.34538269042969,
      "activations/layer4_attention_weight_max": 113.60078430175781,
      "activations/layer4_attention_weight_min": -112.7027359008789,
      "activations/layer5_attention_weight_max": 51.34963607788086,
      "activations/layer5_attention_weight_min": -63.56415939331055,
      "activations/layer6_attention_weight_max": 43.68268585205078,
      "activations/layer6_attention_weight_min": -47.665103912353516,
      "activations/layer7_attention_weight_max": 86.01336669921875,
      "activations/layer7_attention_weight_min": -92.18279266357422,
      "activations/layer8_attention_weight_max": 39.595481872558594,
      "activations/layer8_attention_weight_min": -39.25151824951172,
      "activations/layer9_attention_weight_max": 32.7545280456543,
      "activations/layer9_attention_weight_min": -31.5853271484375,
      "epoch": 11.36,
      "learning_rate": 7.750568181818181e-05,
      "loss": 2.7766,
      "step": 195550
    },
    {
      "activations/layer0_attention_weight_max": 16.60563850402832,
      "activations/layer0_attention_weight_min": -11.843212127685547,
      "activations/layer10_attention_weight_max": 30.596641540527344,
      "activations/layer10_attention_weight_min": -31.679800033569336,
      "activations/layer11_attention_weight_max": 31.927762985229492,
      "activations/layer11_attention_weight_min": -30.181968688964844,
      "activations/layer12_attention_weight_max": 28.878202438354492,
      "activations/layer12_attention_weight_min": -26.602737426757812,
      "activations/layer13_attention_weight_max": 41.947906494140625,
      "activations/layer13_attention_weight_min": -33.53449630737305,
      "activations/layer14_attention_weight_max": 44.53557205200195,
      "activations/layer14_attention_weight_min": -36.949039459228516,
      "activations/layer15_attention_weight_max": 39.74919128417969,
      "activations/layer15_attention_weight_min": -34.005367279052734,
      "activations/layer16_attention_weight_max": 34.544063568115234,
      "activations/layer16_attention_weight_min": -27.9984073638916,
      "activations/layer17_attention_weight_max": 34.58231735229492,
      "activations/layer17_attention_weight_min": -25.215368270874023,
      "activations/layer18_attention_weight_max": 35.097591400146484,
      "activations/layer18_attention_weight_min": -22.617752075195312,
      "activations/layer19_attention_weight_max": 41.71426773071289,
      "activations/layer19_attention_weight_min": -31.046070098876953,
      "activations/layer1_attention_weight_max": 16.88808250427246,
      "activations/layer1_attention_weight_min": -14.587688446044922,
      "activations/layer20_attention_weight_max": 32.108375549316406,
      "activations/layer20_attention_weight_min": -25.139209747314453,
      "activations/layer21_attention_weight_max": 33.0022087097168,
      "activations/layer21_attention_weight_min": -23.421072006225586,
      "activations/layer22_attention_weight_max": 45.489540100097656,
      "activations/layer22_attention_weight_min": -32.977962493896484,
      "activations/layer23_attention_weight_max": 39.84929656982422,
      "activations/layer23_attention_weight_min": -25.405353546142578,
      "activations/layer2_attention_weight_max": 34.715728759765625,
      "activations/layer2_attention_weight_min": -33.27651596069336,
      "activations/layer3_attention_weight_max": 97.45106506347656,
      "activations/layer3_attention_weight_min": -98.37055206298828,
      "activations/layer4_attention_weight_max": 117.1944351196289,
      "activations/layer4_attention_weight_min": -117.20211029052734,
      "activations/layer5_attention_weight_max": 53.406494140625,
      "activations/layer5_attention_weight_min": -63.39419174194336,
      "activations/layer6_attention_weight_max": 47.23820877075195,
      "activations/layer6_attention_weight_min": -50.62607192993164,
      "activations/layer7_attention_weight_max": 93.07907104492188,
      "activations/layer7_attention_weight_min": -90.9410171508789,
      "activations/layer8_attention_weight_max": 39.69989013671875,
      "activations/layer8_attention_weight_min": -41.193607330322266,
      "activations/layer9_attention_weight_max": 32.45664596557617,
      "activations/layer9_attention_weight_min": -33.005760192871094,
      "epoch": 11.37,
      "learning_rate": 7.748674242424241e-05,
      "loss": 2.7762,
      "step": 195600
    },
    {
      "activations/layer0_attention_weight_max": 15.738208770751953,
      "activations/layer0_attention_weight_min": -13.671799659729004,
      "activations/layer10_attention_weight_max": 29.766874313354492,
      "activations/layer10_attention_weight_min": -30.339101791381836,
      "activations/layer11_attention_weight_max": 32.564727783203125,
      "activations/layer11_attention_weight_min": -30.40591049194336,
      "activations/layer12_attention_weight_max": 24.16031265258789,
      "activations/layer12_attention_weight_min": -25.297012329101562,
      "activations/layer13_attention_weight_max": 35.46811294555664,
      "activations/layer13_attention_weight_min": -32.20356750488281,
      "activations/layer14_attention_weight_max": 45.459007263183594,
      "activations/layer14_attention_weight_min": -36.152828216552734,
      "activations/layer15_attention_weight_max": 35.92390441894531,
      "activations/layer15_attention_weight_min": -31.659339904785156,
      "activations/layer16_attention_weight_max": 32.689544677734375,
      "activations/layer16_attention_weight_min": -27.77708625793457,
      "activations/layer17_attention_weight_max": 29.004615783691406,
      "activations/layer17_attention_weight_min": -24.83611297607422,
      "activations/layer18_attention_weight_max": 31.87176513671875,
      "activations/layer18_attention_weight_min": -21.95851707458496,
      "activations/layer19_attention_weight_max": 32.43867111206055,
      "activations/layer19_attention_weight_min": -29.6984920501709,
      "activations/layer1_attention_weight_max": 16.454753875732422,
      "activations/layer1_attention_weight_min": -17.072673797607422,
      "activations/layer20_attention_weight_max": 29.093509674072266,
      "activations/layer20_attention_weight_min": -23.259248733520508,
      "activations/layer21_attention_weight_max": 28.377309799194336,
      "activations/layer21_attention_weight_min": -23.649911880493164,
      "activations/layer22_attention_weight_max": 43.23426818847656,
      "activations/layer22_attention_weight_min": -29.83852767944336,
      "activations/layer23_attention_weight_max": 36.56212615966797,
      "activations/layer23_attention_weight_min": -23.53584861755371,
      "activations/layer2_attention_weight_max": 34.971126556396484,
      "activations/layer2_attention_weight_min": -35.399845123291016,
      "activations/layer3_attention_weight_max": 96.80125427246094,
      "activations/layer3_attention_weight_min": -99.54209899902344,
      "activations/layer4_attention_weight_max": 111.38317108154297,
      "activations/layer4_attention_weight_min": -116.0645751953125,
      "activations/layer5_attention_weight_max": 52.079490661621094,
      "activations/layer5_attention_weight_min": -61.42493438720703,
      "activations/layer6_attention_weight_max": 45.140174865722656,
      "activations/layer6_attention_weight_min": -47.466514587402344,
      "activations/layer7_attention_weight_max": 92.34648895263672,
      "activations/layer7_attention_weight_min": -88.06021881103516,
      "activations/layer8_attention_weight_max": 37.70437240600586,
      "activations/layer8_attention_weight_min": -38.02827835083008,
      "activations/layer9_attention_weight_max": 30.932416915893555,
      "activations/layer9_attention_weight_min": -31.9091739654541,
      "epoch": 11.37,
      "learning_rate": 7.746780303030302e-05,
      "loss": 2.7644,
      "step": 195650
    },
    {
      "activations/layer0_attention_weight_max": 15.71054458618164,
      "activations/layer0_attention_weight_min": -12.280131340026855,
      "activations/layer10_attention_weight_max": 33.77390670776367,
      "activations/layer10_attention_weight_min": -34.4962158203125,
      "activations/layer11_attention_weight_max": 33.79548645019531,
      "activations/layer11_attention_weight_min": -33.99269104003906,
      "activations/layer12_attention_weight_max": 24.39811134338379,
      "activations/layer12_attention_weight_min": -24.66379165649414,
      "activations/layer13_attention_weight_max": 40.79527282714844,
      "activations/layer13_attention_weight_min": -34.350704193115234,
      "activations/layer14_attention_weight_max": 44.7718391418457,
      "activations/layer14_attention_weight_min": -39.19442367553711,
      "activations/layer15_attention_weight_max": 40.186676025390625,
      "activations/layer15_attention_weight_min": -34.00699996948242,
      "activations/layer16_attention_weight_max": 31.504566192626953,
      "activations/layer16_attention_weight_min": -28.597209930419922,
      "activations/layer17_attention_weight_max": 36.15837097167969,
      "activations/layer17_attention_weight_min": -26.93894386291504,
      "activations/layer18_attention_weight_max": 32.398372650146484,
      "activations/layer18_attention_weight_min": -24.21326446533203,
      "activations/layer19_attention_weight_max": 36.342811584472656,
      "activations/layer19_attention_weight_min": -30.57146644592285,
      "activations/layer1_attention_weight_max": 16.811819076538086,
      "activations/layer1_attention_weight_min": -15.78454875946045,
      "activations/layer20_attention_weight_max": 31.045963287353516,
      "activations/layer20_attention_weight_min": -25.720745086669922,
      "activations/layer21_attention_weight_max": 30.427989959716797,
      "activations/layer21_attention_weight_min": -25.463260650634766,
      "activations/layer22_attention_weight_max": 42.24620819091797,
      "activations/layer22_attention_weight_min": -30.004316329956055,
      "activations/layer23_attention_weight_max": 34.207496643066406,
      "activations/layer23_attention_weight_min": -23.71613883972168,
      "activations/layer2_attention_weight_max": 33.69289016723633,
      "activations/layer2_attention_weight_min": -33.26188659667969,
      "activations/layer3_attention_weight_max": 102.3164291381836,
      "activations/layer3_attention_weight_min": -100.18775177001953,
      "activations/layer4_attention_weight_max": 119.11956024169922,
      "activations/layer4_attention_weight_min": -122.08776092529297,
      "activations/layer5_attention_weight_max": 55.230201721191406,
      "activations/layer5_attention_weight_min": -61.6965446472168,
      "activations/layer6_attention_weight_max": 47.26975631713867,
      "activations/layer6_attention_weight_min": -49.17238235473633,
      "activations/layer7_attention_weight_max": 91.21485137939453,
      "activations/layer7_attention_weight_min": -90.99600982666016,
      "activations/layer8_attention_weight_max": 40.43303298950195,
      "activations/layer8_attention_weight_min": -42.6561164855957,
      "activations/layer9_attention_weight_max": 34.439598083496094,
      "activations/layer9_attention_weight_min": -33.32515335083008,
      "epoch": 11.37,
      "learning_rate": 7.744886363636363e-05,
      "loss": 2.7693,
      "step": 195700
    },
    {
      "activations/layer0_attention_weight_max": 15.602975845336914,
      "activations/layer0_attention_weight_min": -11.995537757873535,
      "activations/layer10_attention_weight_max": 30.298004150390625,
      "activations/layer10_attention_weight_min": -30.315582275390625,
      "activations/layer11_attention_weight_max": 30.61742401123047,
      "activations/layer11_attention_weight_min": -29.960676193237305,
      "activations/layer12_attention_weight_max": 25.037702560424805,
      "activations/layer12_attention_weight_min": -25.84225082397461,
      "activations/layer13_attention_weight_max": 40.74892044067383,
      "activations/layer13_attention_weight_min": -35.401519775390625,
      "activations/layer14_attention_weight_max": 44.89999008178711,
      "activations/layer14_attention_weight_min": -38.073448181152344,
      "activations/layer15_attention_weight_max": 34.530765533447266,
      "activations/layer15_attention_weight_min": -32.2226676940918,
      "activations/layer16_attention_weight_max": 28.583003997802734,
      "activations/layer16_attention_weight_min": -28.072912216186523,
      "activations/layer17_attention_weight_max": 32.95510482788086,
      "activations/layer17_attention_weight_min": -25.759017944335938,
      "activations/layer18_attention_weight_max": 31.890892028808594,
      "activations/layer18_attention_weight_min": -22.913955688476562,
      "activations/layer19_attention_weight_max": 32.08434295654297,
      "activations/layer19_attention_weight_min": -29.932785034179688,
      "activations/layer1_attention_weight_max": 18.168981552124023,
      "activations/layer1_attention_weight_min": -17.322458267211914,
      "activations/layer20_attention_weight_max": 27.530900955200195,
      "activations/layer20_attention_weight_min": -23.741485595703125,
      "activations/layer21_attention_weight_max": 28.237186431884766,
      "activations/layer21_attention_weight_min": -23.572856903076172,
      "activations/layer22_attention_weight_max": 44.83782196044922,
      "activations/layer22_attention_weight_min": -29.089136123657227,
      "activations/layer23_attention_weight_max": 34.72376251220703,
      "activations/layer23_attention_weight_min": -27.049041748046875,
      "activations/layer2_attention_weight_max": 35.62329864501953,
      "activations/layer2_attention_weight_min": -33.90636444091797,
      "activations/layer3_attention_weight_max": 101.46033477783203,
      "activations/layer3_attention_weight_min": -100.7856674194336,
      "activations/layer4_attention_weight_max": 113.86164855957031,
      "activations/layer4_attention_weight_min": -114.4437026977539,
      "activations/layer5_attention_weight_max": 50.50786590576172,
      "activations/layer5_attention_weight_min": -65.53285217285156,
      "activations/layer6_attention_weight_max": 44.81751251220703,
      "activations/layer6_attention_weight_min": -50.55500030517578,
      "activations/layer7_attention_weight_max": 85.72151947021484,
      "activations/layer7_attention_weight_min": -88.44400787353516,
      "activations/layer8_attention_weight_max": 38.88884353637695,
      "activations/layer8_attention_weight_min": -40.60166549682617,
      "activations/layer9_attention_weight_max": 31.733633041381836,
      "activations/layer9_attention_weight_min": -31.37190055847168,
      "epoch": 11.37,
      "learning_rate": 7.742992424242423e-05,
      "loss": 2.763,
      "step": 195750
    },
    {
      "activations/layer0_attention_weight_max": 15.58188533782959,
      "activations/layer0_attention_weight_min": -12.054791450500488,
      "activations/layer10_attention_weight_max": 31.204391479492188,
      "activations/layer10_attention_weight_min": -32.340091705322266,
      "activations/layer11_attention_weight_max": 32.07695007324219,
      "activations/layer11_attention_weight_min": -32.6076774597168,
      "activations/layer12_attention_weight_max": 24.945146560668945,
      "activations/layer12_attention_weight_min": -24.319665908813477,
      "activations/layer13_attention_weight_max": 38.87807083129883,
      "activations/layer13_attention_weight_min": -33.00074005126953,
      "activations/layer14_attention_weight_max": 43.46865463256836,
      "activations/layer14_attention_weight_min": -38.055511474609375,
      "activations/layer15_attention_weight_max": 38.89152526855469,
      "activations/layer15_attention_weight_min": -31.296077728271484,
      "activations/layer16_attention_weight_max": 32.60038375854492,
      "activations/layer16_attention_weight_min": -27.730424880981445,
      "activations/layer17_attention_weight_max": 36.559871673583984,
      "activations/layer17_attention_weight_min": -26.478525161743164,
      "activations/layer18_attention_weight_max": 36.94198989868164,
      "activations/layer18_attention_weight_min": -24.623559951782227,
      "activations/layer19_attention_weight_max": 36.43549728393555,
      "activations/layer19_attention_weight_min": -31.125553131103516,
      "activations/layer1_attention_weight_max": 19.00703239440918,
      "activations/layer1_attention_weight_min": -17.317426681518555,
      "activations/layer20_attention_weight_max": 32.287315368652344,
      "activations/layer20_attention_weight_min": -24.54938316345215,
      "activations/layer21_attention_weight_max": 34.880645751953125,
      "activations/layer21_attention_weight_min": -26.311920166015625,
      "activations/layer22_attention_weight_max": 44.283321380615234,
      "activations/layer22_attention_weight_min": -29.897838592529297,
      "activations/layer23_attention_weight_max": 34.32779312133789,
      "activations/layer23_attention_weight_min": -25.32510757446289,
      "activations/layer2_attention_weight_max": 33.68696594238281,
      "activations/layer2_attention_weight_min": -33.182491302490234,
      "activations/layer3_attention_weight_max": 94.83123779296875,
      "activations/layer3_attention_weight_min": -100.04633331298828,
      "activations/layer4_attention_weight_max": 110.90441131591797,
      "activations/layer4_attention_weight_min": -114.615234375,
      "activations/layer5_attention_weight_max": 50.249664306640625,
      "activations/layer5_attention_weight_min": -63.74241638183594,
      "activations/layer6_attention_weight_max": 42.69636535644531,
      "activations/layer6_attention_weight_min": -47.680599212646484,
      "activations/layer7_attention_weight_max": 87.5546646118164,
      "activations/layer7_attention_weight_min": -90.98193359375,
      "activations/layer8_attention_weight_max": 38.64708709716797,
      "activations/layer8_attention_weight_min": -39.5931396484375,
      "activations/layer9_attention_weight_max": 32.26215744018555,
      "activations/layer9_attention_weight_min": -33.060062408447266,
      "epoch": 11.38,
      "learning_rate": 7.741098484848485e-05,
      "loss": 2.7605,
      "step": 195800
    },
    {
      "activations/layer0_attention_weight_max": 15.778960227966309,
      "activations/layer0_attention_weight_min": -12.555700302124023,
      "activations/layer10_attention_weight_max": 33.65978240966797,
      "activations/layer10_attention_weight_min": -31.568607330322266,
      "activations/layer11_attention_weight_max": 32.693809509277344,
      "activations/layer11_attention_weight_min": -29.476240158081055,
      "activations/layer12_attention_weight_max": 24.18231964111328,
      "activations/layer12_attention_weight_min": -24.77228546142578,
      "activations/layer13_attention_weight_max": 36.30192565917969,
      "activations/layer13_attention_weight_min": -33.532283782958984,
      "activations/layer14_attention_weight_max": 42.761741638183594,
      "activations/layer14_attention_weight_min": -36.034339904785156,
      "activations/layer15_attention_weight_max": 33.13498306274414,
      "activations/layer15_attention_weight_min": -30.610523223876953,
      "activations/layer16_attention_weight_max": 30.351579666137695,
      "activations/layer16_attention_weight_min": -27.574695587158203,
      "activations/layer17_attention_weight_max": 31.225215911865234,
      "activations/layer17_attention_weight_min": -25.728790283203125,
      "activations/layer18_attention_weight_max": 31.077848434448242,
      "activations/layer18_attention_weight_min": -25.672346115112305,
      "activations/layer19_attention_weight_max": 36.83095932006836,
      "activations/layer19_attention_weight_min": -32.565277099609375,
      "activations/layer1_attention_weight_max": 16.60465431213379,
      "activations/layer1_attention_weight_min": -15.978782653808594,
      "activations/layer20_attention_weight_max": 28.36394500732422,
      "activations/layer20_attention_weight_min": -25.73699378967285,
      "activations/layer21_attention_weight_max": 26.85319709777832,
      "activations/layer21_attention_weight_min": -25.473155975341797,
      "activations/layer22_attention_weight_max": 42.96929168701172,
      "activations/layer22_attention_weight_min": -32.98124313354492,
      "activations/layer23_attention_weight_max": 36.01537322998047,
      "activations/layer23_attention_weight_min": -27.879531860351562,
      "activations/layer2_attention_weight_max": 33.92902374267578,
      "activations/layer2_attention_weight_min": -33.40837860107422,
      "activations/layer3_attention_weight_max": 96.52803802490234,
      "activations/layer3_attention_weight_min": -98.00110626220703,
      "activations/layer4_attention_weight_max": 106.8882827758789,
      "activations/layer4_attention_weight_min": -112.97364807128906,
      "activations/layer5_attention_weight_max": 49.078651428222656,
      "activations/layer5_attention_weight_min": -63.74415588378906,
      "activations/layer6_attention_weight_max": 44.56746292114258,
      "activations/layer6_attention_weight_min": -48.312808990478516,
      "activations/layer7_attention_weight_max": 87.1104736328125,
      "activations/layer7_attention_weight_min": -88.07942199707031,
      "activations/layer8_attention_weight_max": 39.72267150878906,
      "activations/layer8_attention_weight_min": -38.1970329284668,
      "activations/layer9_attention_weight_max": 32.051513671875,
      "activations/layer9_attention_weight_min": -32.51893997192383,
      "epoch": 11.38,
      "learning_rate": 7.739204545454545e-05,
      "loss": 2.7886,
      "step": 195850
    },
    {
      "activations/layer0_attention_weight_max": 15.06999683380127,
      "activations/layer0_attention_weight_min": -12.94084358215332,
      "activations/layer10_attention_weight_max": 30.449234008789062,
      "activations/layer10_attention_weight_min": -32.8426628112793,
      "activations/layer11_attention_weight_max": 30.512847900390625,
      "activations/layer11_attention_weight_min": -29.354948043823242,
      "activations/layer12_attention_weight_max": 23.456119537353516,
      "activations/layer12_attention_weight_min": -24.21409034729004,
      "activations/layer13_attention_weight_max": 34.26447296142578,
      "activations/layer13_attention_weight_min": -31.507333755493164,
      "activations/layer14_attention_weight_max": 42.18227767944336,
      "activations/layer14_attention_weight_min": -35.168296813964844,
      "activations/layer15_attention_weight_max": 36.30813980102539,
      "activations/layer15_attention_weight_min": -32.076908111572266,
      "activations/layer16_attention_weight_max": 31.17729949951172,
      "activations/layer16_attention_weight_min": -27.472949981689453,
      "activations/layer17_attention_weight_max": 30.4853515625,
      "activations/layer17_attention_weight_min": -25.762271881103516,
      "activations/layer18_attention_weight_max": 29.229148864746094,
      "activations/layer18_attention_weight_min": -24.224056243896484,
      "activations/layer19_attention_weight_max": 34.47341537475586,
      "activations/layer19_attention_weight_min": -28.569303512573242,
      "activations/layer1_attention_weight_max": 15.676621437072754,
      "activations/layer1_attention_weight_min": -14.754145622253418,
      "activations/layer20_attention_weight_max": 31.148658752441406,
      "activations/layer20_attention_weight_min": -24.155492782592773,
      "activations/layer21_attention_weight_max": 29.898319244384766,
      "activations/layer21_attention_weight_min": -23.230764389038086,
      "activations/layer22_attention_weight_max": 44.28795623779297,
      "activations/layer22_attention_weight_min": -28.188385009765625,
      "activations/layer23_attention_weight_max": 35.52299499511719,
      "activations/layer23_attention_weight_min": -24.8758602142334,
      "activations/layer2_attention_weight_max": 34.196327209472656,
      "activations/layer2_attention_weight_min": -33.11839294433594,
      "activations/layer3_attention_weight_max": 98.24577331542969,
      "activations/layer3_attention_weight_min": -97.10169982910156,
      "activations/layer4_attention_weight_max": 112.37060546875,
      "activations/layer4_attention_weight_min": -117.13848876953125,
      "activations/layer5_attention_weight_max": 51.28106689453125,
      "activations/layer5_attention_weight_min": -64.18670654296875,
      "activations/layer6_attention_weight_max": 42.34210968017578,
      "activations/layer6_attention_weight_min": -46.909767150878906,
      "activations/layer7_attention_weight_max": 85.8381118774414,
      "activations/layer7_attention_weight_min": -94.18363952636719,
      "activations/layer8_attention_weight_max": 40.19221878051758,
      "activations/layer8_attention_weight_min": -39.978721618652344,
      "activations/layer9_attention_weight_max": 31.2568302154541,
      "activations/layer9_attention_weight_min": -33.52878189086914,
      "epoch": 11.38,
      "learning_rate": 7.737348484848484e-05,
      "loss": 2.7757,
      "step": 195900
    },
    {
      "activations/layer0_attention_weight_max": 15.627842903137207,
      "activations/layer0_attention_weight_min": -12.446280479431152,
      "activations/layer10_attention_weight_max": 32.090267181396484,
      "activations/layer10_attention_weight_min": -32.78590393066406,
      "activations/layer11_attention_weight_max": 31.266965866088867,
      "activations/layer11_attention_weight_min": -32.215782165527344,
      "activations/layer12_attention_weight_max": 23.48562240600586,
      "activations/layer12_attention_weight_min": -26.207029342651367,
      "activations/layer13_attention_weight_max": 35.65686798095703,
      "activations/layer13_attention_weight_min": -33.96845245361328,
      "activations/layer14_attention_weight_max": 43.99648666381836,
      "activations/layer14_attention_weight_min": -36.846153259277344,
      "activations/layer15_attention_weight_max": 34.72985076904297,
      "activations/layer15_attention_weight_min": -32.959720611572266,
      "activations/layer16_attention_weight_max": 27.961715698242188,
      "activations/layer16_attention_weight_min": -28.470849990844727,
      "activations/layer17_attention_weight_max": 30.1683292388916,
      "activations/layer17_attention_weight_min": -26.598093032836914,
      "activations/layer18_attention_weight_max": 30.932186126708984,
      "activations/layer18_attention_weight_min": -24.010902404785156,
      "activations/layer19_attention_weight_max": 32.5598030090332,
      "activations/layer19_attention_weight_min": -31.714807510375977,
      "activations/layer1_attention_weight_max": 16.429283142089844,
      "activations/layer1_attention_weight_min": -14.222886085510254,
      "activations/layer20_attention_weight_max": 27.060962677001953,
      "activations/layer20_attention_weight_min": -24.8719482421875,
      "activations/layer21_attention_weight_max": 28.39559555053711,
      "activations/layer21_attention_weight_min": -24.209165573120117,
      "activations/layer22_attention_weight_max": 38.301597595214844,
      "activations/layer22_attention_weight_min": -28.68453025817871,
      "activations/layer23_attention_weight_max": 33.227699279785156,
      "activations/layer23_attention_weight_min": -25.55551528930664,
      "activations/layer2_attention_weight_max": 33.4219856262207,
      "activations/layer2_attention_weight_min": -34.87028503417969,
      "activations/layer3_attention_weight_max": 101.31047821044922,
      "activations/layer3_attention_weight_min": -100.3851318359375,
      "activations/layer4_attention_weight_max": 114.20924377441406,
      "activations/layer4_attention_weight_min": -111.39422607421875,
      "activations/layer5_attention_weight_max": 56.53754806518555,
      "activations/layer5_attention_weight_min": -59.058982849121094,
      "activations/layer6_attention_weight_max": 44.50792694091797,
      "activations/layer6_attention_weight_min": -47.50440979003906,
      "activations/layer7_attention_weight_max": 88.9554214477539,
      "activations/layer7_attention_weight_min": -88.98252868652344,
      "activations/layer8_attention_weight_max": 40.1817741394043,
      "activations/layer8_attention_weight_min": -42.13420867919922,
      "activations/layer9_attention_weight_max": 33.67396545410156,
      "activations/layer9_attention_weight_min": -34.97985076904297,
      "epoch": 11.39,
      "learning_rate": 7.735454545454544e-05,
      "loss": 2.7778,
      "step": 195950
    },
    {
      "activations/layer0_attention_weight_max": 16.354738235473633,
      "activations/layer0_attention_weight_min": -12.49712085723877,
      "activations/layer10_attention_weight_max": 35.98582458496094,
      "activations/layer10_attention_weight_min": -34.73380661010742,
      "activations/layer11_attention_weight_max": 37.02094268798828,
      "activations/layer11_attention_weight_min": -37.33075714111328,
      "activations/layer12_attention_weight_max": 25.1362247467041,
      "activations/layer12_attention_weight_min": -26.918720245361328,
      "activations/layer13_attention_weight_max": 40.35254669189453,
      "activations/layer13_attention_weight_min": -33.384456634521484,
      "activations/layer14_attention_weight_max": 48.726478576660156,
      "activations/layer14_attention_weight_min": -41.70475769042969,
      "activations/layer15_attention_weight_max": 35.319793701171875,
      "activations/layer15_attention_weight_min": -30.787900924682617,
      "activations/layer16_attention_weight_max": 31.143630981445312,
      "activations/layer16_attention_weight_min": -27.096233367919922,
      "activations/layer17_attention_weight_max": 31.31962776184082,
      "activations/layer17_attention_weight_min": -25.79747772216797,
      "activations/layer18_attention_weight_max": 28.525821685791016,
      "activations/layer18_attention_weight_min": -23.974756240844727,
      "activations/layer19_attention_weight_max": 32.06642150878906,
      "activations/layer19_attention_weight_min": -29.904239654541016,
      "activations/layer1_attention_weight_max": 16.1584415435791,
      "activations/layer1_attention_weight_min": -15.286919593811035,
      "activations/layer20_attention_weight_max": 27.877506256103516,
      "activations/layer20_attention_weight_min": -24.666980743408203,
      "activations/layer21_attention_weight_max": 26.40550994873047,
      "activations/layer21_attention_weight_min": -22.381195068359375,
      "activations/layer22_attention_weight_max": 42.2698974609375,
      "activations/layer22_attention_weight_min": -28.708621978759766,
      "activations/layer23_attention_weight_max": 33.552452087402344,
      "activations/layer23_attention_weight_min": -24.693477630615234,
      "activations/layer2_attention_weight_max": 33.7471809387207,
      "activations/layer2_attention_weight_min": -33.090328216552734,
      "activations/layer3_attention_weight_max": 96.04692077636719,
      "activations/layer3_attention_weight_min": -94.77263641357422,
      "activations/layer4_attention_weight_max": 111.7386245727539,
      "activations/layer4_attention_weight_min": -113.08351135253906,
      "activations/layer5_attention_weight_max": 54.987060546875,
      "activations/layer5_attention_weight_min": -63.71144104003906,
      "activations/layer6_attention_weight_max": 43.44630432128906,
      "activations/layer6_attention_weight_min": -45.76302719116211,
      "activations/layer7_attention_weight_max": 93.61019134521484,
      "activations/layer7_attention_weight_min": -89.79496765136719,
      "activations/layer8_attention_weight_max": 42.17306137084961,
      "activations/layer8_attention_weight_min": -43.942909240722656,
      "activations/layer9_attention_weight_max": 36.523311614990234,
      "activations/layer9_attention_weight_min": -37.953269958496094,
      "epoch": 11.39,
      "learning_rate": 7.733560606060606e-05,
      "loss": 2.778,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.5786,
      "eval_samples_per_second": 500.546,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.5786,
      "eval_openwebtext_samples_per_second": 500.546,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0139,
      "eval_wikitext_samples_per_second": 226.432,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5982,
      "eval_lambada_samples_per_second": 507.284,
      "step": 196000
    },
    {
      "activations/layer0_attention_weight_max": 16.34703826904297,
      "activations/layer0_attention_weight_min": -11.693147659301758,
      "activations/layer10_attention_weight_max": 39.188568115234375,
      "activations/layer10_attention_weight_min": -33.40085983276367,
      "activations/layer11_attention_weight_max": 39.186676025390625,
      "activations/layer11_attention_weight_min": -33.46129608154297,
      "activations/layer12_attention_weight_max": 26.34450912475586,
      "activations/layer12_attention_weight_min": -26.692752838134766,
      "activations/layer13_attention_weight_max": 38.80413055419922,
      "activations/layer13_attention_weight_min": -35.75981140136719,
      "activations/layer14_attention_weight_max": 46.1768798828125,
      "activations/layer14_attention_weight_min": -37.483604431152344,
      "activations/layer15_attention_weight_max": 37.096702575683594,
      "activations/layer15_attention_weight_min": -32.11431884765625,
      "activations/layer16_attention_weight_max": 35.49448013305664,
      "activations/layer16_attention_weight_min": -27.16518211364746,
      "activations/layer17_attention_weight_max": 33.1162223815918,
      "activations/layer17_attention_weight_min": -26.400571823120117,
      "activations/layer18_attention_weight_max": 30.637914657592773,
      "activations/layer18_attention_weight_min": -21.67107582092285,
      "activations/layer19_attention_weight_max": 39.556480407714844,
      "activations/layer19_attention_weight_min": -30.462900161743164,
      "activations/layer1_attention_weight_max": 17.5128116607666,
      "activations/layer1_attention_weight_min": -15.892669677734375,
      "activations/layer20_attention_weight_max": 30.74819564819336,
      "activations/layer20_attention_weight_min": -26.933238983154297,
      "activations/layer21_attention_weight_max": 29.93896484375,
      "activations/layer21_attention_weight_min": -23.118520736694336,
      "activations/layer22_attention_weight_max": 47.81509780883789,
      "activations/layer22_attention_weight_min": -29.6945858001709,
      "activations/layer23_attention_weight_max": 35.45599365234375,
      "activations/layer23_attention_weight_min": -25.296422958374023,
      "activations/layer2_attention_weight_max": 36.35444641113281,
      "activations/layer2_attention_weight_min": -34.431724548339844,
      "activations/layer3_attention_weight_max": 103.75460052490234,
      "activations/layer3_attention_weight_min": -101.8595199584961,
      "activations/layer4_attention_weight_max": 113.54415130615234,
      "activations/layer4_attention_weight_min": -112.70552825927734,
      "activations/layer5_attention_weight_max": 53.22583770751953,
      "activations/layer5_attention_weight_min": -61.43571472167969,
      "activations/layer6_attention_weight_max": 45.69569778442383,
      "activations/layer6_attention_weight_min": -49.678958892822266,
      "activations/layer7_attention_weight_max": 95.00147247314453,
      "activations/layer7_attention_weight_min": -93.66432189941406,
      "activations/layer8_attention_weight_max": 42.39418411254883,
      "activations/layer8_attention_weight_min": -44.32905197143555,
      "activations/layer9_attention_weight_max": 35.5253791809082,
      "activations/layer9_attention_weight_min": -34.89168167114258,
      "epoch": 11.39,
      "learning_rate": 7.731666666666667e-05,
      "loss": 2.7744,
      "step": 196050
    },
    {
      "activations/layer0_attention_weight_max": 15.429184913635254,
      "activations/layer0_attention_weight_min": -12.093015670776367,
      "activations/layer10_attention_weight_max": 33.8803825378418,
      "activations/layer10_attention_weight_min": -33.79351806640625,
      "activations/layer11_attention_weight_max": 33.219234466552734,
      "activations/layer11_attention_weight_min": -32.800479888916016,
      "activations/layer12_attention_weight_max": 24.036487579345703,
      "activations/layer12_attention_weight_min": -25.34528160095215,
      "activations/layer13_attention_weight_max": 36.952632904052734,
      "activations/layer13_attention_weight_min": -35.1165885925293,
      "activations/layer14_attention_weight_max": 48.84361267089844,
      "activations/layer14_attention_weight_min": -43.925865173339844,
      "activations/layer15_attention_weight_max": 36.42170333862305,
      "activations/layer15_attention_weight_min": -32.21723937988281,
      "activations/layer16_attention_weight_max": 34.17258834838867,
      "activations/layer16_attention_weight_min": -28.111223220825195,
      "activations/layer17_attention_weight_max": 31.40250587463379,
      "activations/layer17_attention_weight_min": -26.5387020111084,
      "activations/layer18_attention_weight_max": 30.51270866394043,
      "activations/layer18_attention_weight_min": -22.87462043762207,
      "activations/layer19_attention_weight_max": 38.7230224609375,
      "activations/layer19_attention_weight_min": -29.57185173034668,
      "activations/layer1_attention_weight_max": 16.503864288330078,
      "activations/layer1_attention_weight_min": -15.446722030639648,
      "activations/layer20_attention_weight_max": 29.807109832763672,
      "activations/layer20_attention_weight_min": -24.39745330810547,
      "activations/layer21_attention_weight_max": 28.99892234802246,
      "activations/layer21_attention_weight_min": -23.407968521118164,
      "activations/layer22_attention_weight_max": 47.88779067993164,
      "activations/layer22_attention_weight_min": -31.043725967407227,
      "activations/layer23_attention_weight_max": 38.21056365966797,
      "activations/layer23_attention_weight_min": -24.488393783569336,
      "activations/layer2_attention_weight_max": 36.99723815917969,
      "activations/layer2_attention_weight_min": -36.299034118652344,
      "activations/layer3_attention_weight_max": 105.51768493652344,
      "activations/layer3_attention_weight_min": -104.9053955078125,
      "activations/layer4_attention_weight_max": 126.62963104248047,
      "activations/layer4_attention_weight_min": -126.28533935546875,
      "activations/layer5_attention_weight_max": 59.59208297729492,
      "activations/layer5_attention_weight_min": -67.67522430419922,
      "activations/layer6_attention_weight_max": 50.26833724975586,
      "activations/layer6_attention_weight_min": -55.77037048339844,
      "activations/layer7_attention_weight_max": 101.9269027709961,
      "activations/layer7_attention_weight_min": -93.6592788696289,
      "activations/layer8_attention_weight_max": 41.67412567138672,
      "activations/layer8_attention_weight_min": -44.4753303527832,
      "activations/layer9_attention_weight_max": 34.670040130615234,
      "activations/layer9_attention_weight_min": -35.79056167602539,
      "epoch": 11.39,
      "learning_rate": 7.729772727272726e-05,
      "loss": 2.7673,
      "step": 196100
    },
    {
      "activations/layer0_attention_weight_max": 15.468124389648438,
      "activations/layer0_attention_weight_min": -12.055898666381836,
      "activations/layer10_attention_weight_max": 38.10413360595703,
      "activations/layer10_attention_weight_min": -34.27251434326172,
      "activations/layer11_attention_weight_max": 40.0755729675293,
      "activations/layer11_attention_weight_min": -34.4708251953125,
      "activations/layer12_attention_weight_max": 25.99736213684082,
      "activations/layer12_attention_weight_min": -27.0263729095459,
      "activations/layer13_attention_weight_max": 40.559532165527344,
      "activations/layer13_attention_weight_min": -36.265201568603516,
      "activations/layer14_attention_weight_max": 55.50414276123047,
      "activations/layer14_attention_weight_min": -41.24235534667969,
      "activations/layer15_attention_weight_max": 36.18246078491211,
      "activations/layer15_attention_weight_min": -32.16960906982422,
      "activations/layer16_attention_weight_max": 30.57419776916504,
      "activations/layer16_attention_weight_min": -26.855607986450195,
      "activations/layer17_attention_weight_max": 34.740840911865234,
      "activations/layer17_attention_weight_min": -25.2662296295166,
      "activations/layer18_attention_weight_max": 31.343782424926758,
      "activations/layer18_attention_weight_min": -24.624004364013672,
      "activations/layer19_attention_weight_max": 34.00691604614258,
      "activations/layer19_attention_weight_min": -29.935791015625,
      "activations/layer1_attention_weight_max": 16.789247512817383,
      "activations/layer1_attention_weight_min": -14.609837532043457,
      "activations/layer20_attention_weight_max": 30.916879653930664,
      "activations/layer20_attention_weight_min": -24.772987365722656,
      "activations/layer21_attention_weight_max": 27.41805076599121,
      "activations/layer21_attention_weight_min": -22.928110122680664,
      "activations/layer22_attention_weight_max": 41.46866989135742,
      "activations/layer22_attention_weight_min": -29.913267135620117,
      "activations/layer23_attention_weight_max": 32.48268127441406,
      "activations/layer23_attention_weight_min": -25.41873550415039,
      "activations/layer2_attention_weight_max": 34.30200958251953,
      "activations/layer2_attention_weight_min": -33.730316162109375,
      "activations/layer3_attention_weight_max": 91.54345703125,
      "activations/layer3_attention_weight_min": -94.48544311523438,
      "activations/layer4_attention_weight_max": 109.60516357421875,
      "activations/layer4_attention_weight_min": -118.26619720458984,
      "activations/layer5_attention_weight_max": 52.755943298339844,
      "activations/layer5_attention_weight_min": -67.9391098022461,
      "activations/layer6_attention_weight_max": 43.78029251098633,
      "activations/layer6_attention_weight_min": -48.20702362060547,
      "activations/layer7_attention_weight_max": 97.69273376464844,
      "activations/layer7_attention_weight_min": -95.9459228515625,
      "activations/layer8_attention_weight_max": 43.42955780029297,
      "activations/layer8_attention_weight_min": -42.62141799926758,
      "activations/layer9_attention_weight_max": 34.82279586791992,
      "activations/layer9_attention_weight_min": -35.51418685913086,
      "epoch": 11.4,
      "learning_rate": 7.727916666666665e-05,
      "loss": 2.7738,
      "step": 196150
    },
    {
      "activations/layer0_attention_weight_max": 15.871081352233887,
      "activations/layer0_attention_weight_min": -12.313615798950195,
      "activations/layer10_attention_weight_max": 32.740753173828125,
      "activations/layer10_attention_weight_min": -31.90411949157715,
      "activations/layer11_attention_weight_max": 33.389095306396484,
      "activations/layer11_attention_weight_min": -32.139488220214844,
      "activations/layer12_attention_weight_max": 23.585840225219727,
      "activations/layer12_attention_weight_min": -24.60913848876953,
      "activations/layer13_attention_weight_max": 36.373207092285156,
      "activations/layer13_attention_weight_min": -33.963111877441406,
      "activations/layer14_attention_weight_max": 43.59638214111328,
      "activations/layer14_attention_weight_min": -37.118770599365234,
      "activations/layer15_attention_weight_max": 34.468997955322266,
      "activations/layer15_attention_weight_min": -33.007144927978516,
      "activations/layer16_attention_weight_max": 30.027345657348633,
      "activations/layer16_attention_weight_min": -26.85792350769043,
      "activations/layer17_attention_weight_max": 29.669458389282227,
      "activations/layer17_attention_weight_min": -26.102195739746094,
      "activations/layer18_attention_weight_max": 28.260562896728516,
      "activations/layer18_attention_weight_min": -23.947046279907227,
      "activations/layer19_attention_weight_max": 33.82003402709961,
      "activations/layer19_attention_weight_min": -31.11517906188965,
      "activations/layer1_attention_weight_max": 16.111555099487305,
      "activations/layer1_attention_weight_min": -17.047609329223633,
      "activations/layer20_attention_weight_max": 30.39894676208496,
      "activations/layer20_attention_weight_min": -24.743196487426758,
      "activations/layer21_attention_weight_max": 28.142192840576172,
      "activations/layer21_attention_weight_min": -24.82158851623535,
      "activations/layer22_attention_weight_max": 49.16844177246094,
      "activations/layer22_attention_weight_min": -30.330718994140625,
      "activations/layer23_attention_weight_max": 34.49482345581055,
      "activations/layer23_attention_weight_min": -25.347890853881836,
      "activations/layer2_attention_weight_max": 34.74562454223633,
      "activations/layer2_attention_weight_min": -34.46273422241211,
      "activations/layer3_attention_weight_max": 97.4012451171875,
      "activations/layer3_attention_weight_min": -98.53448486328125,
      "activations/layer4_attention_weight_max": 111.39276123046875,
      "activations/layer4_attention_weight_min": -115.48506164550781,
      "activations/layer5_attention_weight_max": 56.000022888183594,
      "activations/layer5_attention_weight_min": -63.10057830810547,
      "activations/layer6_attention_weight_max": 46.032352447509766,
      "activations/layer6_attention_weight_min": -47.80104064941406,
      "activations/layer7_attention_weight_max": 92.66613006591797,
      "activations/layer7_attention_weight_min": -91.51439666748047,
      "activations/layer8_attention_weight_max": 42.91998291015625,
      "activations/layer8_attention_weight_min": -41.101253509521484,
      "activations/layer9_attention_weight_max": 35.084205627441406,
      "activations/layer9_attention_weight_min": -34.71900939941406,
      "epoch": 11.4,
      "learning_rate": 7.726022727272727e-05,
      "loss": 2.7765,
      "step": 196200
    },
    {
      "activations/layer0_attention_weight_max": 15.425694465637207,
      "activations/layer0_attention_weight_min": -13.192591667175293,
      "activations/layer10_attention_weight_max": 31.771146774291992,
      "activations/layer10_attention_weight_min": -32.60737609863281,
      "activations/layer11_attention_weight_max": 30.06371307373047,
      "activations/layer11_attention_weight_min": -31.201200485229492,
      "activations/layer12_attention_weight_max": 24.856380462646484,
      "activations/layer12_attention_weight_min": -28.008033752441406,
      "activations/layer13_attention_weight_max": 37.828216552734375,
      "activations/layer13_attention_weight_min": -31.764591217041016,
      "activations/layer14_attention_weight_max": 44.68367385864258,
      "activations/layer14_attention_weight_min": -35.76485061645508,
      "activations/layer15_attention_weight_max": 35.705406188964844,
      "activations/layer15_attention_weight_min": -29.599260330200195,
      "activations/layer16_attention_weight_max": 27.68486213684082,
      "activations/layer16_attention_weight_min": -25.80794334411621,
      "activations/layer17_attention_weight_max": 32.06709289550781,
      "activations/layer17_attention_weight_min": -24.42777442932129,
      "activations/layer18_attention_weight_max": 29.557533264160156,
      "activations/layer18_attention_weight_min": -23.86125946044922,
      "activations/layer19_attention_weight_max": 31.969633102416992,
      "activations/layer19_attention_weight_min": -30.495874404907227,
      "activations/layer1_attention_weight_max": 16.151010513305664,
      "activations/layer1_attention_weight_min": -14.77720832824707,
      "activations/layer20_attention_weight_max": 27.94359016418457,
      "activations/layer20_attention_weight_min": -26.92645263671875,
      "activations/layer21_attention_weight_max": 27.89982032775879,
      "activations/layer21_attention_weight_min": -25.471084594726562,
      "activations/layer22_attention_weight_max": 38.244754791259766,
      "activations/layer22_attention_weight_min": -29.80133056640625,
      "activations/layer23_attention_weight_max": 33.38180160522461,
      "activations/layer23_attention_weight_min": -24.954923629760742,
      "activations/layer2_attention_weight_max": 32.59304428100586,
      "activations/layer2_attention_weight_min": -33.6008415222168,
      "activations/layer3_attention_weight_max": 92.22774505615234,
      "activations/layer3_attention_weight_min": -98.8669204711914,
      "activations/layer4_attention_weight_max": 107.38427734375,
      "activations/layer4_attention_weight_min": -109.80047607421875,
      "activations/layer5_attention_weight_max": 50.522857666015625,
      "activations/layer5_attention_weight_min": -60.55941390991211,
      "activations/layer6_attention_weight_max": 43.05712127685547,
      "activations/layer6_attention_weight_min": -45.27843475341797,
      "activations/layer7_attention_weight_max": 84.37274932861328,
      "activations/layer7_attention_weight_min": -83.75373077392578,
      "activations/layer8_attention_weight_max": 38.37294387817383,
      "activations/layer8_attention_weight_min": -39.11350631713867,
      "activations/layer9_attention_weight_max": 35.8096923828125,
      "activations/layer9_attention_weight_min": -33.93381881713867,
      "epoch": 11.4,
      "learning_rate": 7.724128787878788e-05,
      "loss": 2.7754,
      "step": 196250
    },
    {
      "activations/layer0_attention_weight_max": 15.039899826049805,
      "activations/layer0_attention_weight_min": -12.49553394317627,
      "activations/layer10_attention_weight_max": 33.71152114868164,
      "activations/layer10_attention_weight_min": -33.22332763671875,
      "activations/layer11_attention_weight_max": 35.68440246582031,
      "activations/layer11_attention_weight_min": -32.89775085449219,
      "activations/layer12_attention_weight_max": 30.468276977539062,
      "activations/layer12_attention_weight_min": -29.72831916809082,
      "activations/layer13_attention_weight_max": 44.123558044433594,
      "activations/layer13_attention_weight_min": -35.75236129760742,
      "activations/layer14_attention_weight_max": 55.56612777709961,
      "activations/layer14_attention_weight_min": -41.86361312866211,
      "activations/layer15_attention_weight_max": 41.77658462524414,
      "activations/layer15_attention_weight_min": -37.43084716796875,
      "activations/layer16_attention_weight_max": 36.60022735595703,
      "activations/layer16_attention_weight_min": -28.043909072875977,
      "activations/layer17_attention_weight_max": 38.2496223449707,
      "activations/layer17_attention_weight_min": -27.737539291381836,
      "activations/layer18_attention_weight_max": 32.18361282348633,
      "activations/layer18_attention_weight_min": -25.69777488708496,
      "activations/layer19_attention_weight_max": 45.826900482177734,
      "activations/layer19_attention_weight_min": -33.78789138793945,
      "activations/layer1_attention_weight_max": 17.831512451171875,
      "activations/layer1_attention_weight_min": -14.548724174499512,
      "activations/layer20_attention_weight_max": 35.490474700927734,
      "activations/layer20_attention_weight_min": -26.6275634765625,
      "activations/layer21_attention_weight_max": 32.4429817199707,
      "activations/layer21_attention_weight_min": -24.30112648010254,
      "activations/layer22_attention_weight_max": 48.96624755859375,
      "activations/layer22_attention_weight_min": -33.26028823852539,
      "activations/layer23_attention_weight_max": 39.40972137451172,
      "activations/layer23_attention_weight_min": -25.403118133544922,
      "activations/layer2_attention_weight_max": 33.780479431152344,
      "activations/layer2_attention_weight_min": -34.34415054321289,
      "activations/layer3_attention_weight_max": 98.65376281738281,
      "activations/layer3_attention_weight_min": -100.24211883544922,
      "activations/layer4_attention_weight_max": 112.62191009521484,
      "activations/layer4_attention_weight_min": -111.2705078125,
      "activations/layer5_attention_weight_max": 52.4047966003418,
      "activations/layer5_attention_weight_min": -63.785301208496094,
      "activations/layer6_attention_weight_max": 45.10289764404297,
      "activations/layer6_attention_weight_min": -48.17438888549805,
      "activations/layer7_attention_weight_max": 93.47915649414062,
      "activations/layer7_attention_weight_min": -95.1131591796875,
      "activations/layer8_attention_weight_max": 42.3028678894043,
      "activations/layer8_attention_weight_min": -41.99725341796875,
      "activations/layer9_attention_weight_max": 36.77421188354492,
      "activations/layer9_attention_weight_min": -34.26139450073242,
      "epoch": 11.41,
      "learning_rate": 7.722234848484847e-05,
      "loss": 2.7696,
      "step": 196300
    },
    {
      "activations/layer0_attention_weight_max": 15.216508865356445,
      "activations/layer0_attention_weight_min": -11.881448745727539,
      "activations/layer10_attention_weight_max": 29.634178161621094,
      "activations/layer10_attention_weight_min": -29.779287338256836,
      "activations/layer11_attention_weight_max": 29.239948272705078,
      "activations/layer11_attention_weight_min": -30.45362091064453,
      "activations/layer12_attention_weight_max": 22.457950592041016,
      "activations/layer12_attention_weight_min": -25.22449493408203,
      "activations/layer13_attention_weight_max": 35.74961471557617,
      "activations/layer13_attention_weight_min": -32.910301208496094,
      "activations/layer14_attention_weight_max": 37.86787414550781,
      "activations/layer14_attention_weight_min": -36.63820266723633,
      "activations/layer15_attention_weight_max": 32.228267669677734,
      "activations/layer15_attention_weight_min": -31.47147560119629,
      "activations/layer16_attention_weight_max": 29.774642944335938,
      "activations/layer16_attention_weight_min": -27.38751220703125,
      "activations/layer17_attention_weight_max": 28.048471450805664,
      "activations/layer17_attention_weight_min": -26.4144287109375,
      "activations/layer18_attention_weight_max": 32.48750305175781,
      "activations/layer18_attention_weight_min": -24.780019760131836,
      "activations/layer19_attention_weight_max": 34.76176071166992,
      "activations/layer19_attention_weight_min": -32.15574645996094,
      "activations/layer1_attention_weight_max": 16.169635772705078,
      "activations/layer1_attention_weight_min": -13.99134635925293,
      "activations/layer20_attention_weight_max": 28.648775100708008,
      "activations/layer20_attention_weight_min": -25.248291015625,
      "activations/layer21_attention_weight_max": 28.898693084716797,
      "activations/layer21_attention_weight_min": -23.85338592529297,
      "activations/layer22_attention_weight_max": 40.660400390625,
      "activations/layer22_attention_weight_min": -31.719215393066406,
      "activations/layer23_attention_weight_max": 40.267669677734375,
      "activations/layer23_attention_weight_min": -25.088722229003906,
      "activations/layer2_attention_weight_max": 33.70600509643555,
      "activations/layer2_attention_weight_min": -33.680870056152344,
      "activations/layer3_attention_weight_max": 95.03738403320312,
      "activations/layer3_attention_weight_min": -102.98947143554688,
      "activations/layer4_attention_weight_max": 109.84862518310547,
      "activations/layer4_attention_weight_min": -110.63358306884766,
      "activations/layer5_attention_weight_max": 49.264183044433594,
      "activations/layer5_attention_weight_min": -63.42920684814453,
      "activations/layer6_attention_weight_max": 43.05952453613281,
      "activations/layer6_attention_weight_min": -46.90595245361328,
      "activations/layer7_attention_weight_max": 86.69398498535156,
      "activations/layer7_attention_weight_min": -90.8157730102539,
      "activations/layer8_attention_weight_max": 40.40996551513672,
      "activations/layer8_attention_weight_min": -39.908477783203125,
      "activations/layer9_attention_weight_max": 32.33341598510742,
      "activations/layer9_attention_weight_min": -31.583770751953125,
      "epoch": 11.41,
      "learning_rate": 7.720340909090908e-05,
      "loss": 2.7648,
      "step": 196350
    },
    {
      "activations/layer0_attention_weight_max": 15.323238372802734,
      "activations/layer0_attention_weight_min": -12.182127952575684,
      "activations/layer10_attention_weight_max": 32.03963851928711,
      "activations/layer10_attention_weight_min": -31.134769439697266,
      "activations/layer11_attention_weight_max": 32.123512268066406,
      "activations/layer11_attention_weight_min": -31.07406997680664,
      "activations/layer12_attention_weight_max": 33.69514083862305,
      "activations/layer12_attention_weight_min": -35.99391555786133,
      "activations/layer13_attention_weight_max": 41.08942413330078,
      "activations/layer13_attention_weight_min": -36.998207092285156,
      "activations/layer14_attention_weight_max": 46.910423278808594,
      "activations/layer14_attention_weight_min": -39.66913604736328,
      "activations/layer15_attention_weight_max": 37.9018669128418,
      "activations/layer15_attention_weight_min": -33.704959869384766,
      "activations/layer16_attention_weight_max": 32.055294036865234,
      "activations/layer16_attention_weight_min": -29.44581413269043,
      "activations/layer17_attention_weight_max": 30.87723159790039,
      "activations/layer17_attention_weight_min": -28.01201057434082,
      "activations/layer18_attention_weight_max": 30.067279815673828,
      "activations/layer18_attention_weight_min": -23.823511123657227,
      "activations/layer19_attention_weight_max": 36.63312911987305,
      "activations/layer19_attention_weight_min": -31.853559494018555,
      "activations/layer1_attention_weight_max": 16.453092575073242,
      "activations/layer1_attention_weight_min": -14.77298355102539,
      "activations/layer20_attention_weight_max": 30.764108657836914,
      "activations/layer20_attention_weight_min": -25.027231216430664,
      "activations/layer21_attention_weight_max": 29.0919246673584,
      "activations/layer21_attention_weight_min": -23.94954490661621,
      "activations/layer22_attention_weight_max": 42.42390441894531,
      "activations/layer22_attention_weight_min": -32.415283203125,
      "activations/layer23_attention_weight_max": 36.11328887939453,
      "activations/layer23_attention_weight_min": -24.99586296081543,
      "activations/layer2_attention_weight_max": 34.58725357055664,
      "activations/layer2_attention_weight_min": -34.262393951416016,
      "activations/layer3_attention_weight_max": 96.72606658935547,
      "activations/layer3_attention_weight_min": -97.62271118164062,
      "activations/layer4_attention_weight_max": 111.13075256347656,
      "activations/layer4_attention_weight_min": -113.03597259521484,
      "activations/layer5_attention_weight_max": 53.77239990234375,
      "activations/layer5_attention_weight_min": -62.74270248413086,
      "activations/layer6_attention_weight_max": 43.74477005004883,
      "activations/layer6_attention_weight_min": -45.75901794433594,
      "activations/layer7_attention_weight_max": 88.24079895019531,
      "activations/layer7_attention_weight_min": -86.92051696777344,
      "activations/layer8_attention_weight_max": 38.83317947387695,
      "activations/layer8_attention_weight_min": -41.6484375,
      "activations/layer9_attention_weight_max": 32.54646301269531,
      "activations/layer9_attention_weight_min": -33.46761703491211,
      "epoch": 11.41,
      "learning_rate": 7.71844696969697e-05,
      "loss": 2.7842,
      "step": 196400
    },
    {
      "activations/layer0_attention_weight_max": 15.75455379486084,
      "activations/layer0_attention_weight_min": -12.716530799865723,
      "activations/layer10_attention_weight_max": 32.398963928222656,
      "activations/layer10_attention_weight_min": -34.953582763671875,
      "activations/layer11_attention_weight_max": 31.234432220458984,
      "activations/layer11_attention_weight_min": -33.46953582763672,
      "activations/layer12_attention_weight_max": 26.16295051574707,
      "activations/layer12_attention_weight_min": -26.055646896362305,
      "activations/layer13_attention_weight_max": 40.948387145996094,
      "activations/layer13_attention_weight_min": -34.88310623168945,
      "activations/layer14_attention_weight_max": 45.68559646606445,
      "activations/layer14_attention_weight_min": -39.91012954711914,
      "activations/layer15_attention_weight_max": 36.7020149230957,
      "activations/layer15_attention_weight_min": -31.31867027282715,
      "activations/layer16_attention_weight_max": 31.574663162231445,
      "activations/layer16_attention_weight_min": -26.81588363647461,
      "activations/layer17_attention_weight_max": 30.99643898010254,
      "activations/layer17_attention_weight_min": -28.6214599609375,
      "activations/layer18_attention_weight_max": 29.279193878173828,
      "activations/layer18_attention_weight_min": -25.389665603637695,
      "activations/layer19_attention_weight_max": 34.054264068603516,
      "activations/layer19_attention_weight_min": -30.155292510986328,
      "activations/layer1_attention_weight_max": 17.359912872314453,
      "activations/layer1_attention_weight_min": -16.190404891967773,
      "activations/layer20_attention_weight_max": 28.199323654174805,
      "activations/layer20_attention_weight_min": -24.553258895874023,
      "activations/layer21_attention_weight_max": 27.29766082763672,
      "activations/layer21_attention_weight_min": -24.219280242919922,
      "activations/layer22_attention_weight_max": 43.95331954956055,
      "activations/layer22_attention_weight_min": -31.282190322875977,
      "activations/layer23_attention_weight_max": 32.30696105957031,
      "activations/layer23_attention_weight_min": -28.249250411987305,
      "activations/layer2_attention_weight_max": 35.27394104003906,
      "activations/layer2_attention_weight_min": -34.06459426879883,
      "activations/layer3_attention_weight_max": 98.10649108886719,
      "activations/layer3_attention_weight_min": -101.92713165283203,
      "activations/layer4_attention_weight_max": 116.51611328125,
      "activations/layer4_attention_weight_min": -115.48589324951172,
      "activations/layer5_attention_weight_max": 53.822723388671875,
      "activations/layer5_attention_weight_min": -62.27387619018555,
      "activations/layer6_attention_weight_max": 44.60126495361328,
      "activations/layer6_attention_weight_min": -47.27737045288086,
      "activations/layer7_attention_weight_max": 91.86168670654297,
      "activations/layer7_attention_weight_min": -92.0530014038086,
      "activations/layer8_attention_weight_max": 41.5569953918457,
      "activations/layer8_attention_weight_min": -40.907569885253906,
      "activations/layer9_attention_weight_max": 31.43699073791504,
      "activations/layer9_attention_weight_min": -34.11981201171875,
      "epoch": 11.41,
      "learning_rate": 7.71655303030303e-05,
      "loss": 2.7739,
      "step": 196450
    },
    {
      "activations/layer0_attention_weight_max": 15.165410995483398,
      "activations/layer0_attention_weight_min": -13.027729988098145,
      "activations/layer10_attention_weight_max": 33.64692687988281,
      "activations/layer10_attention_weight_min": -32.764617919921875,
      "activations/layer11_attention_weight_max": 34.04138946533203,
      "activations/layer11_attention_weight_min": -31.6070556640625,
      "activations/layer12_attention_weight_max": 25.706802368164062,
      "activations/layer12_attention_weight_min": -24.278451919555664,
      "activations/layer13_attention_weight_max": 37.63507080078125,
      "activations/layer13_attention_weight_min": -34.42658615112305,
      "activations/layer14_attention_weight_max": 47.27756881713867,
      "activations/layer14_attention_weight_min": -36.50316619873047,
      "activations/layer15_attention_weight_max": 36.41492462158203,
      "activations/layer15_attention_weight_min": -32.76679611206055,
      "activations/layer16_attention_weight_max": 33.861907958984375,
      "activations/layer16_attention_weight_min": -27.785158157348633,
      "activations/layer17_attention_weight_max": 33.180538177490234,
      "activations/layer17_attention_weight_min": -25.658491134643555,
      "activations/layer18_attention_weight_max": 34.587120056152344,
      "activations/layer18_attention_weight_min": -22.700687408447266,
      "activations/layer19_attention_weight_max": 41.0826301574707,
      "activations/layer19_attention_weight_min": -30.431278228759766,
      "activations/layer1_attention_weight_max": 16.944231033325195,
      "activations/layer1_attention_weight_min": -16.017004013061523,
      "activations/layer20_attention_weight_max": 31.926223754882812,
      "activations/layer20_attention_weight_min": -24.519731521606445,
      "activations/layer21_attention_weight_max": 36.003692626953125,
      "activations/layer21_attention_weight_min": -24.579320907592773,
      "activations/layer22_attention_weight_max": 47.06253433227539,
      "activations/layer22_attention_weight_min": -30.08782196044922,
      "activations/layer23_attention_weight_max": 36.23307418823242,
      "activations/layer23_attention_weight_min": -24.985286712646484,
      "activations/layer2_attention_weight_max": 35.19035339355469,
      "activations/layer2_attention_weight_min": -35.3326416015625,
      "activations/layer3_attention_weight_max": 99.90474700927734,
      "activations/layer3_attention_weight_min": -101.22167205810547,
      "activations/layer4_attention_weight_max": 114.8106460571289,
      "activations/layer4_attention_weight_min": -118.37076568603516,
      "activations/layer5_attention_weight_max": 54.23662567138672,
      "activations/layer5_attention_weight_min": -63.514801025390625,
      "activations/layer6_attention_weight_max": 47.814292907714844,
      "activations/layer6_attention_weight_min": -48.260231018066406,
      "activations/layer7_attention_weight_max": 93.38143157958984,
      "activations/layer7_attention_weight_min": -88.97163391113281,
      "activations/layer8_attention_weight_max": 43.562400817871094,
      "activations/layer8_attention_weight_min": -40.57234191894531,
      "activations/layer9_attention_weight_max": 33.169246673583984,
      "activations/layer9_attention_weight_min": -33.536312103271484,
      "epoch": 11.42,
      "learning_rate": 7.71465909090909e-05,
      "loss": 2.7863,
      "step": 196500
    },
    {
      "activations/layer0_attention_weight_max": 15.745687484741211,
      "activations/layer0_attention_weight_min": -12.92615032196045,
      "activations/layer10_attention_weight_max": 33.866756439208984,
      "activations/layer10_attention_weight_min": -33.83935546875,
      "activations/layer11_attention_weight_max": 37.95451736450195,
      "activations/layer11_attention_weight_min": -32.85102081298828,
      "activations/layer12_attention_weight_max": 24.30990982055664,
      "activations/layer12_attention_weight_min": -24.62375831604004,
      "activations/layer13_attention_weight_max": 37.42753601074219,
      "activations/layer13_attention_weight_min": -33.91889953613281,
      "activations/layer14_attention_weight_max": 40.93734359741211,
      "activations/layer14_attention_weight_min": -35.303985595703125,
      "activations/layer15_attention_weight_max": 33.710819244384766,
      "activations/layer15_attention_weight_min": -30.46310043334961,
      "activations/layer16_attention_weight_max": 29.54121208190918,
      "activations/layer16_attention_weight_min": -27.985275268554688,
      "activations/layer17_attention_weight_max": 33.57157897949219,
      "activations/layer17_attention_weight_min": -24.786087036132812,
      "activations/layer18_attention_weight_max": 29.585954666137695,
      "activations/layer18_attention_weight_min": -23.354467391967773,
      "activations/layer19_attention_weight_max": 33.011146545410156,
      "activations/layer19_attention_weight_min": -29.98866844177246,
      "activations/layer1_attention_weight_max": 16.595428466796875,
      "activations/layer1_attention_weight_min": -15.784152030944824,
      "activations/layer20_attention_weight_max": 30.307411193847656,
      "activations/layer20_attention_weight_min": -24.665477752685547,
      "activations/layer21_attention_weight_max": 29.594860076904297,
      "activations/layer21_attention_weight_min": -24.0672664642334,
      "activations/layer22_attention_weight_max": 40.428863525390625,
      "activations/layer22_attention_weight_min": -28.7053279876709,
      "activations/layer23_attention_weight_max": 34.172821044921875,
      "activations/layer23_attention_weight_min": -22.885608673095703,
      "activations/layer2_attention_weight_max": 34.43585968017578,
      "activations/layer2_attention_weight_min": -33.62480163574219,
      "activations/layer3_attention_weight_max": 97.17771911621094,
      "activations/layer3_attention_weight_min": -95.4690170288086,
      "activations/layer4_attention_weight_max": 108.36726379394531,
      "activations/layer4_attention_weight_min": -110.24017333984375,
      "activations/layer5_attention_weight_max": 50.94272994995117,
      "activations/layer5_attention_weight_min": -61.298187255859375,
      "activations/layer6_attention_weight_max": 44.9805793762207,
      "activations/layer6_attention_weight_min": -46.39534378051758,
      "activations/layer7_attention_weight_max": 95.24403381347656,
      "activations/layer7_attention_weight_min": -90.52638244628906,
      "activations/layer8_attention_weight_max": 44.70644760131836,
      "activations/layer8_attention_weight_min": -42.003883361816406,
      "activations/layer9_attention_weight_max": 35.13385009765625,
      "activations/layer9_attention_weight_min": -35.59542465209961,
      "epoch": 11.42,
      "learning_rate": 7.71276515151515e-05,
      "loss": 2.7699,
      "step": 196550
    },
    {
      "activations/layer0_attention_weight_max": 15.951929092407227,
      "activations/layer0_attention_weight_min": -13.256692886352539,
      "activations/layer10_attention_weight_max": 42.226566314697266,
      "activations/layer10_attention_weight_min": -37.56981658935547,
      "activations/layer11_attention_weight_max": 39.593936920166016,
      "activations/layer11_attention_weight_min": -35.38543701171875,
      "activations/layer12_attention_weight_max": 31.24994468688965,
      "activations/layer12_attention_weight_min": -29.903465270996094,
      "activations/layer13_attention_weight_max": 38.99836349487305,
      "activations/layer13_attention_weight_min": -37.23332214355469,
      "activations/layer14_attention_weight_max": 46.4913330078125,
      "activations/layer14_attention_weight_min": -39.30990982055664,
      "activations/layer15_attention_weight_max": 35.0916748046875,
      "activations/layer15_attention_weight_min": -32.706207275390625,
      "activations/layer16_attention_weight_max": 30.91327667236328,
      "activations/layer16_attention_weight_min": -28.99015235900879,
      "activations/layer17_attention_weight_max": 33.82722091674805,
      "activations/layer17_attention_weight_min": -24.849491119384766,
      "activations/layer18_attention_weight_max": 33.533782958984375,
      "activations/layer18_attention_weight_min": -23.713417053222656,
      "activations/layer19_attention_weight_max": 35.939762115478516,
      "activations/layer19_attention_weight_min": -31.89944839477539,
      "activations/layer1_attention_weight_max": 18.24776268005371,
      "activations/layer1_attention_weight_min": -15.63687801361084,
      "activations/layer20_attention_weight_max": 27.669893264770508,
      "activations/layer20_attention_weight_min": -24.804170608520508,
      "activations/layer21_attention_weight_max": 27.644155502319336,
      "activations/layer21_attention_weight_min": -24.216045379638672,
      "activations/layer22_attention_weight_max": 42.70541000366211,
      "activations/layer22_attention_weight_min": -30.463064193725586,
      "activations/layer23_attention_weight_max": 33.35197448730469,
      "activations/layer23_attention_weight_min": -25.19815444946289,
      "activations/layer2_attention_weight_max": 33.71352767944336,
      "activations/layer2_attention_weight_min": -34.6682243347168,
      "activations/layer3_attention_weight_max": 102.2001953125,
      "activations/layer3_attention_weight_min": -95.93202209472656,
      "activations/layer4_attention_weight_max": 115.03919982910156,
      "activations/layer4_attention_weight_min": -110.2590560913086,
      "activations/layer5_attention_weight_max": 54.496883392333984,
      "activations/layer5_attention_weight_min": -61.29246139526367,
      "activations/layer6_attention_weight_max": 49.59500503540039,
      "activations/layer6_attention_weight_min": -46.17775344848633,
      "activations/layer7_attention_weight_max": 107.24378967285156,
      "activations/layer7_attention_weight_min": -94.75537109375,
      "activations/layer8_attention_weight_max": 50.83210754394531,
      "activations/layer8_attention_weight_min": -44.25166320800781,
      "activations/layer9_attention_weight_max": 40.25178909301758,
      "activations/layer9_attention_weight_min": -38.13246536254883,
      "epoch": 11.42,
      "learning_rate": 7.710871212121212e-05,
      "loss": 2.7886,
      "step": 196600
    },
    {
      "activations/layer0_attention_weight_max": 16.5208797454834,
      "activations/layer0_attention_weight_min": -12.309477806091309,
      "activations/layer10_attention_weight_max": 33.724342346191406,
      "activations/layer10_attention_weight_min": -36.1708984375,
      "activations/layer11_attention_weight_max": 33.443790435791016,
      "activations/layer11_attention_weight_min": -34.59454345703125,
      "activations/layer12_attention_weight_max": 25.384353637695312,
      "activations/layer12_attention_weight_min": -25.62303924560547,
      "activations/layer13_attention_weight_max": 46.25050735473633,
      "activations/layer13_attention_weight_min": -36.60591125488281,
      "activations/layer14_attention_weight_max": 46.18067169189453,
      "activations/layer14_attention_weight_min": -38.96672058105469,
      "activations/layer15_attention_weight_max": 37.68328094482422,
      "activations/layer15_attention_weight_min": -35.19879913330078,
      "activations/layer16_attention_weight_max": 31.23056983947754,
      "activations/layer16_attention_weight_min": -28.88993263244629,
      "activations/layer17_attention_weight_max": 31.070154190063477,
      "activations/layer17_attention_weight_min": -28.028926849365234,
      "activations/layer18_attention_weight_max": 29.691911697387695,
      "activations/layer18_attention_weight_min": -22.50468635559082,
      "activations/layer19_attention_weight_max": 38.8634147644043,
      "activations/layer19_attention_weight_min": -34.213584899902344,
      "activations/layer1_attention_weight_max": 18.726484298706055,
      "activations/layer1_attention_weight_min": -15.18020248413086,
      "activations/layer20_attention_weight_max": 28.313762664794922,
      "activations/layer20_attention_weight_min": -25.106157302856445,
      "activations/layer21_attention_weight_max": 26.994462966918945,
      "activations/layer21_attention_weight_min": -22.910554885864258,
      "activations/layer22_attention_weight_max": 46.33869171142578,
      "activations/layer22_attention_weight_min": -29.57064437866211,
      "activations/layer23_attention_weight_max": 38.88261413574219,
      "activations/layer23_attention_weight_min": -22.97738265991211,
      "activations/layer2_attention_weight_max": 34.47447967529297,
      "activations/layer2_attention_weight_min": -36.537315368652344,
      "activations/layer3_attention_weight_max": 98.29145812988281,
      "activations/layer3_attention_weight_min": -100.4129638671875,
      "activations/layer4_attention_weight_max": 118.03956604003906,
      "activations/layer4_attention_weight_min": -119.86295318603516,
      "activations/layer5_attention_weight_max": 52.118377685546875,
      "activations/layer5_attention_weight_min": -68.5128173828125,
      "activations/layer6_attention_weight_max": 45.28252029418945,
      "activations/layer6_attention_weight_min": -47.08539581298828,
      "activations/layer7_attention_weight_max": 93.36180114746094,
      "activations/layer7_attention_weight_min": -90.30364990234375,
      "activations/layer8_attention_weight_max": 40.15427780151367,
      "activations/layer8_attention_weight_min": -41.3575325012207,
      "activations/layer9_attention_weight_max": 33.435508728027344,
      "activations/layer9_attention_weight_min": -35.05766677856445,
      "epoch": 11.43,
      "learning_rate": 7.708977272727272e-05,
      "loss": 2.7825,
      "step": 196650
    },
    {
      "activations/layer0_attention_weight_max": 15.682093620300293,
      "activations/layer0_attention_weight_min": -13.18397045135498,
      "activations/layer10_attention_weight_max": 32.3375129699707,
      "activations/layer10_attention_weight_min": -32.534515380859375,
      "activations/layer11_attention_weight_max": 32.418609619140625,
      "activations/layer11_attention_weight_min": -32.66539764404297,
      "activations/layer12_attention_weight_max": 23.576364517211914,
      "activations/layer12_attention_weight_min": -24.05034828186035,
      "activations/layer13_attention_weight_max": 38.59520721435547,
      "activations/layer13_attention_weight_min": -33.41607666015625,
      "activations/layer14_attention_weight_max": 42.11109924316406,
      "activations/layer14_attention_weight_min": -37.15053176879883,
      "activations/layer15_attention_weight_max": 34.50102233886719,
      "activations/layer15_attention_weight_min": -29.6818904876709,
      "activations/layer16_attention_weight_max": 28.255403518676758,
      "activations/layer16_attention_weight_min": -26.840932846069336,
      "activations/layer17_attention_weight_max": 31.262563705444336,
      "activations/layer17_attention_weight_min": -25.812301635742188,
      "activations/layer18_attention_weight_max": 34.71476745605469,
      "activations/layer18_attention_weight_min": -22.259815216064453,
      "activations/layer19_attention_weight_max": 33.80461883544922,
      "activations/layer19_attention_weight_min": -31.060821533203125,
      "activations/layer1_attention_weight_max": 17.261676788330078,
      "activations/layer1_attention_weight_min": -14.726912498474121,
      "activations/layer20_attention_weight_max": 28.668895721435547,
      "activations/layer20_attention_weight_min": -24.251373291015625,
      "activations/layer21_attention_weight_max": 29.647335052490234,
      "activations/layer21_attention_weight_min": -22.923171997070312,
      "activations/layer22_attention_weight_max": 44.71610641479492,
      "activations/layer22_attention_weight_min": -30.03205108642578,
      "activations/layer23_attention_weight_max": 33.360496520996094,
      "activations/layer23_attention_weight_min": -23.43549919128418,
      "activations/layer2_attention_weight_max": 34.218231201171875,
      "activations/layer2_attention_weight_min": -33.433128356933594,
      "activations/layer3_attention_weight_max": 96.52094268798828,
      "activations/layer3_attention_weight_min": -97.04876708984375,
      "activations/layer4_attention_weight_max": 112.23384857177734,
      "activations/layer4_attention_weight_min": -114.1139907836914,
      "activations/layer5_attention_weight_max": 51.2620964050293,
      "activations/layer5_attention_weight_min": -63.319908142089844,
      "activations/layer6_attention_weight_max": 43.959102630615234,
      "activations/layer6_attention_weight_min": -47.21125793457031,
      "activations/layer7_attention_weight_max": 89.64263916015625,
      "activations/layer7_attention_weight_min": -89.95138549804688,
      "activations/layer8_attention_weight_max": 38.6172981262207,
      "activations/layer8_attention_weight_min": -42.44935607910156,
      "activations/layer9_attention_weight_max": 33.23279571533203,
      "activations/layer9_attention_weight_min": -32.646278381347656,
      "epoch": 11.43,
      "learning_rate": 7.707083333333332e-05,
      "loss": 2.7621,
      "step": 196700
    },
    {
      "activations/layer0_attention_weight_max": 15.326329231262207,
      "activations/layer0_attention_weight_min": -12.616697311401367,
      "activations/layer10_attention_weight_max": 35.542877197265625,
      "activations/layer10_attention_weight_min": -35.2578125,
      "activations/layer11_attention_weight_max": 34.589195251464844,
      "activations/layer11_attention_weight_min": -35.330322265625,
      "activations/layer12_attention_weight_max": 24.04435920715332,
      "activations/layer12_attention_weight_min": -26.71225357055664,
      "activations/layer13_attention_weight_max": 35.6417236328125,
      "activations/layer13_attention_weight_min": -37.05025863647461,
      "activations/layer14_attention_weight_max": 41.78741455078125,
      "activations/layer14_attention_weight_min": -40.44412612915039,
      "activations/layer15_attention_weight_max": 34.01521301269531,
      "activations/layer15_attention_weight_min": -33.49599075317383,
      "activations/layer16_attention_weight_max": 29.271141052246094,
      "activations/layer16_attention_weight_min": -32.07123565673828,
      "activations/layer17_attention_weight_max": 32.65486145019531,
      "activations/layer17_attention_weight_min": -26.833871841430664,
      "activations/layer18_attention_weight_max": 31.073820114135742,
      "activations/layer18_attention_weight_min": -22.40837860107422,
      "activations/layer19_attention_weight_max": 32.11882019042969,
      "activations/layer19_attention_weight_min": -30.719289779663086,
      "activations/layer1_attention_weight_max": 18.61408042907715,
      "activations/layer1_attention_weight_min": -16.853269577026367,
      "activations/layer20_attention_weight_max": 28.615726470947266,
      "activations/layer20_attention_weight_min": -25.57371711730957,
      "activations/layer21_attention_weight_max": 29.80023193359375,
      "activations/layer21_attention_weight_min": -24.32701301574707,
      "activations/layer22_attention_weight_max": 38.00196838378906,
      "activations/layer22_attention_weight_min": -29.22624397277832,
      "activations/layer23_attention_weight_max": 32.259315490722656,
      "activations/layer23_attention_weight_min": -24.62970542907715,
      "activations/layer2_attention_weight_max": 35.58155822753906,
      "activations/layer2_attention_weight_min": -34.59408187866211,
      "activations/layer3_attention_weight_max": 99.08187866210938,
      "activations/layer3_attention_weight_min": -103.40249633789062,
      "activations/layer4_attention_weight_max": 121.46522521972656,
      "activations/layer4_attention_weight_min": -118.83428192138672,
      "activations/layer5_attention_weight_max": 62.14535140991211,
      "activations/layer5_attention_weight_min": -68.77776336669922,
      "activations/layer6_attention_weight_max": 45.67500686645508,
      "activations/layer6_attention_weight_min": -49.1861686706543,
      "activations/layer7_attention_weight_max": 91.3027114868164,
      "activations/layer7_attention_weight_min": -92.08445739746094,
      "activations/layer8_attention_weight_max": 41.132102966308594,
      "activations/layer8_attention_weight_min": -42.10576248168945,
      "activations/layer9_attention_weight_max": 33.22574996948242,
      "activations/layer9_attention_weight_min": -34.008270263671875,
      "epoch": 11.43,
      "learning_rate": 7.705189393939394e-05,
      "loss": 2.751,
      "step": 196750
    },
    {
      "activations/layer0_attention_weight_max": 15.759504318237305,
      "activations/layer0_attention_weight_min": -12.500213623046875,
      "activations/layer10_attention_weight_max": 31.74799156188965,
      "activations/layer10_attention_weight_min": -32.77630615234375,
      "activations/layer11_attention_weight_max": 32.26383590698242,
      "activations/layer11_attention_weight_min": -33.75734329223633,
      "activations/layer12_attention_weight_max": 23.265052795410156,
      "activations/layer12_attention_weight_min": -28.063438415527344,
      "activations/layer13_attention_weight_max": 34.7729606628418,
      "activations/layer13_attention_weight_min": -33.24232482910156,
      "activations/layer14_attention_weight_max": 40.16462326049805,
      "activations/layer14_attention_weight_min": -39.03006362915039,
      "activations/layer15_attention_weight_max": 35.1756706237793,
      "activations/layer15_attention_weight_min": -30.863262176513672,
      "activations/layer16_attention_weight_max": 29.633684158325195,
      "activations/layer16_attention_weight_min": -26.27933120727539,
      "activations/layer17_attention_weight_max": 31.59818458557129,
      "activations/layer17_attention_weight_min": -26.25273895263672,
      "activations/layer18_attention_weight_max": 32.636627197265625,
      "activations/layer18_attention_weight_min": -27.308500289916992,
      "activations/layer19_attention_weight_max": 32.43191146850586,
      "activations/layer19_attention_weight_min": -28.709503173828125,
      "activations/layer1_attention_weight_max": 17.427688598632812,
      "activations/layer1_attention_weight_min": -15.870317459106445,
      "activations/layer20_attention_weight_max": 28.259187698364258,
      "activations/layer20_attention_weight_min": -23.484214782714844,
      "activations/layer21_attention_weight_max": 28.995994567871094,
      "activations/layer21_attention_weight_min": -23.767852783203125,
      "activations/layer22_attention_weight_max": 41.17534637451172,
      "activations/layer22_attention_weight_min": -29.025325775146484,
      "activations/layer23_attention_weight_max": 34.211509704589844,
      "activations/layer23_attention_weight_min": -24.643795013427734,
      "activations/layer2_attention_weight_max": 33.45039749145508,
      "activations/layer2_attention_weight_min": -34.32657241821289,
      "activations/layer3_attention_weight_max": 97.41009521484375,
      "activations/layer3_attention_weight_min": -100.25788116455078,
      "activations/layer4_attention_weight_max": 114.78717041015625,
      "activations/layer4_attention_weight_min": -119.57002258300781,
      "activations/layer5_attention_weight_max": 56.72954559326172,
      "activations/layer5_attention_weight_min": -63.51371383666992,
      "activations/layer6_attention_weight_max": 45.837337493896484,
      "activations/layer6_attention_weight_min": -49.84242630004883,
      "activations/layer7_attention_weight_max": 93.45500946044922,
      "activations/layer7_attention_weight_min": -99.51070404052734,
      "activations/layer8_attention_weight_max": 39.67145538330078,
      "activations/layer8_attention_weight_min": -42.24382781982422,
      "activations/layer9_attention_weight_max": 33.78388595581055,
      "activations/layer9_attention_weight_min": -36.99625015258789,
      "epoch": 11.44,
      "learning_rate": 7.703295454545455e-05,
      "loss": 2.7779,
      "step": 196800
    },
    {
      "activations/layer0_attention_weight_max": 15.196662902832031,
      "activations/layer0_attention_weight_min": -12.851890563964844,
      "activations/layer10_attention_weight_max": 32.86737060546875,
      "activations/layer10_attention_weight_min": -33.21690368652344,
      "activations/layer11_attention_weight_max": 32.510467529296875,
      "activations/layer11_attention_weight_min": -33.27560806274414,
      "activations/layer12_attention_weight_max": 27.482772827148438,
      "activations/layer12_attention_weight_min": -25.86896514892578,
      "activations/layer13_attention_weight_max": 39.212127685546875,
      "activations/layer13_attention_weight_min": -36.34937286376953,
      "activations/layer14_attention_weight_max": 50.539608001708984,
      "activations/layer14_attention_weight_min": -38.35601043701172,
      "activations/layer15_attention_weight_max": 37.516937255859375,
      "activations/layer15_attention_weight_min": -32.16520309448242,
      "activations/layer16_attention_weight_max": 30.815853118896484,
      "activations/layer16_attention_weight_min": -28.564701080322266,
      "activations/layer17_attention_weight_max": 31.904937744140625,
      "activations/layer17_attention_weight_min": -27.004013061523438,
      "activations/layer18_attention_weight_max": 29.782310485839844,
      "activations/layer18_attention_weight_min": -25.550838470458984,
      "activations/layer19_attention_weight_max": 35.311710357666016,
      "activations/layer19_attention_weight_min": -32.06792449951172,
      "activations/layer1_attention_weight_max": 16.034168243408203,
      "activations/layer1_attention_weight_min": -14.815874099731445,
      "activations/layer20_attention_weight_max": 28.167789459228516,
      "activations/layer20_attention_weight_min": -26.487401962280273,
      "activations/layer21_attention_weight_max": 26.78058433532715,
      "activations/layer21_attention_weight_min": -24.892282485961914,
      "activations/layer22_attention_weight_max": 46.25894546508789,
      "activations/layer22_attention_weight_min": -34.26804733276367,
      "activations/layer23_attention_weight_max": 34.297332763671875,
      "activations/layer23_attention_weight_min": -23.536346435546875,
      "activations/layer2_attention_weight_max": 33.835060119628906,
      "activations/layer2_attention_weight_min": -34.972564697265625,
      "activations/layer3_attention_weight_max": 100.02796173095703,
      "activations/layer3_attention_weight_min": -101.4382095336914,
      "activations/layer4_attention_weight_max": 112.84586334228516,
      "activations/layer4_attention_weight_min": -115.48961639404297,
      "activations/layer5_attention_weight_max": 53.30402755737305,
      "activations/layer5_attention_weight_min": -65.48997497558594,
      "activations/layer6_attention_weight_max": 44.03691101074219,
      "activations/layer6_attention_weight_min": -50.19495391845703,
      "activations/layer7_attention_weight_max": 91.78776550292969,
      "activations/layer7_attention_weight_min": -89.86476135253906,
      "activations/layer8_attention_weight_max": 39.51881408691406,
      "activations/layer8_attention_weight_min": -42.36555480957031,
      "activations/layer9_attention_weight_max": 34.9135627746582,
      "activations/layer9_attention_weight_min": -34.20182418823242,
      "epoch": 11.44,
      "learning_rate": 7.701401515151514e-05,
      "loss": 2.7897,
      "step": 196850
    },
    {
      "activations/layer0_attention_weight_max": 16.267515182495117,
      "activations/layer0_attention_weight_min": -12.46413516998291,
      "activations/layer10_attention_weight_max": 33.079795837402344,
      "activations/layer10_attention_weight_min": -30.761199951171875,
      "activations/layer11_attention_weight_max": 32.017295837402344,
      "activations/layer11_attention_weight_min": -30.914474487304688,
      "activations/layer12_attention_weight_max": 26.958921432495117,
      "activations/layer12_attention_weight_min": -26.66910743713379,
      "activations/layer13_attention_weight_max": 46.028011322021484,
      "activations/layer13_attention_weight_min": -35.03117752075195,
      "activations/layer14_attention_weight_max": 50.8681526184082,
      "activations/layer14_attention_weight_min": -38.9196891784668,
      "activations/layer15_attention_weight_max": 41.49128723144531,
      "activations/layer15_attention_weight_min": -34.81318664550781,
      "activations/layer16_attention_weight_max": 33.03658676147461,
      "activations/layer16_attention_weight_min": -28.673158645629883,
      "activations/layer17_attention_weight_max": 33.0361442565918,
      "activations/layer17_attention_weight_min": -26.846994400024414,
      "activations/layer18_attention_weight_max": 35.68474197387695,
      "activations/layer18_attention_weight_min": -24.163747787475586,
      "activations/layer19_attention_weight_max": 37.83148956298828,
      "activations/layer19_attention_weight_min": -30.40154457092285,
      "activations/layer1_attention_weight_max": 17.82501983642578,
      "activations/layer1_attention_weight_min": -15.165241241455078,
      "activations/layer20_attention_weight_max": 31.68646240234375,
      "activations/layer20_attention_weight_min": -24.79958152770996,
      "activations/layer21_attention_weight_max": 32.86738967895508,
      "activations/layer21_attention_weight_min": -23.422868728637695,
      "activations/layer22_attention_weight_max": 58.03926086425781,
      "activations/layer22_attention_weight_min": -30.25739288330078,
      "activations/layer23_attention_weight_max": 40.95098876953125,
      "activations/layer23_attention_weight_min": -26.564191818237305,
      "activations/layer2_attention_weight_max": 34.44830322265625,
      "activations/layer2_attention_weight_min": -33.39375305175781,
      "activations/layer3_attention_weight_max": 96.40558624267578,
      "activations/layer3_attention_weight_min": -97.60575103759766,
      "activations/layer4_attention_weight_max": 116.54060363769531,
      "activations/layer4_attention_weight_min": -114.19889068603516,
      "activations/layer5_attention_weight_max": 52.33396911621094,
      "activations/layer5_attention_weight_min": -64.83967590332031,
      "activations/layer6_attention_weight_max": 44.42429733276367,
      "activations/layer6_attention_weight_min": -47.88651657104492,
      "activations/layer7_attention_weight_max": 87.37334442138672,
      "activations/layer7_attention_weight_min": -89.10375213623047,
      "activations/layer8_attention_weight_max": 40.323455810546875,
      "activations/layer8_attention_weight_min": -41.05750274658203,
      "activations/layer9_attention_weight_max": 32.12316131591797,
      "activations/layer9_attention_weight_min": -32.216976165771484,
      "epoch": 11.44,
      "learning_rate": 7.699507575757575e-05,
      "loss": 2.7732,
      "step": 196900
    },
    {
      "activations/layer0_attention_weight_max": 16.727327346801758,
      "activations/layer0_attention_weight_min": -12.573116302490234,
      "activations/layer10_attention_weight_max": 34.46815872192383,
      "activations/layer10_attention_weight_min": -31.760690689086914,
      "activations/layer11_attention_weight_max": 33.092613220214844,
      "activations/layer11_attention_weight_min": -32.146728515625,
      "activations/layer12_attention_weight_max": 25.610023498535156,
      "activations/layer12_attention_weight_min": -25.333940505981445,
      "activations/layer13_attention_weight_max": 40.07647705078125,
      "activations/layer13_attention_weight_min": -34.33909225463867,
      "activations/layer14_attention_weight_max": 47.052024841308594,
      "activations/layer14_attention_weight_min": -38.328792572021484,
      "activations/layer15_attention_weight_max": 35.633663177490234,
      "activations/layer15_attention_weight_min": -31.91484260559082,
      "activations/layer16_attention_weight_max": 29.93069076538086,
      "activations/layer16_attention_weight_min": -26.832727432250977,
      "activations/layer17_attention_weight_max": 31.142499923706055,
      "activations/layer17_attention_weight_min": -25.456218719482422,
      "activations/layer18_attention_weight_max": 31.996929168701172,
      "activations/layer18_attention_weight_min": -27.709096908569336,
      "activations/layer19_attention_weight_max": 36.24140548706055,
      "activations/layer19_attention_weight_min": -29.796405792236328,
      "activations/layer1_attention_weight_max": 16.759323120117188,
      "activations/layer1_attention_weight_min": -14.307178497314453,
      "activations/layer20_attention_weight_max": 30.16707992553711,
      "activations/layer20_attention_weight_min": -23.7958927154541,
      "activations/layer21_attention_weight_max": 30.946678161621094,
      "activations/layer21_attention_weight_min": -25.62251853942871,
      "activations/layer22_attention_weight_max": 51.72983932495117,
      "activations/layer22_attention_weight_min": -30.07303810119629,
      "activations/layer23_attention_weight_max": 37.48525619506836,
      "activations/layer23_attention_weight_min": -27.658645629882812,
      "activations/layer2_attention_weight_max": 33.32344055175781,
      "activations/layer2_attention_weight_min": -31.98110580444336,
      "activations/layer3_attention_weight_max": 95.18890380859375,
      "activations/layer3_attention_weight_min": -93.63327026367188,
      "activations/layer4_attention_weight_max": 109.55956268310547,
      "activations/layer4_attention_weight_min": -110.86577606201172,
      "activations/layer5_attention_weight_max": 52.521324157714844,
      "activations/layer5_attention_weight_min": -59.98916244506836,
      "activations/layer6_attention_weight_max": 45.611961364746094,
      "activations/layer6_attention_weight_min": -49.332969665527344,
      "activations/layer7_attention_weight_max": 92.10761260986328,
      "activations/layer7_attention_weight_min": -94.51000213623047,
      "activations/layer8_attention_weight_max": 41.799495697021484,
      "activations/layer8_attention_weight_min": -42.15813446044922,
      "activations/layer9_attention_weight_max": 36.1452522277832,
      "activations/layer9_attention_weight_min": -37.76789093017578,
      "epoch": 11.44,
      "learning_rate": 7.697613636363637e-05,
      "loss": 2.7851,
      "step": 196950
    },
    {
      "activations/layer0_attention_weight_max": 16.123779296875,
      "activations/layer0_attention_weight_min": -13.24644660949707,
      "activations/layer10_attention_weight_max": 28.454132080078125,
      "activations/layer10_attention_weight_min": -28.66018295288086,
      "activations/layer11_attention_weight_max": 31.76148796081543,
      "activations/layer11_attention_weight_min": -28.709217071533203,
      "activations/layer12_attention_weight_max": 21.87104606628418,
      "activations/layer12_attention_weight_min": -23.93503189086914,
      "activations/layer13_attention_weight_max": 35.52635955810547,
      "activations/layer13_attention_weight_min": -34.63951873779297,
      "activations/layer14_attention_weight_max": 39.207542419433594,
      "activations/layer14_attention_weight_min": -36.57150650024414,
      "activations/layer15_attention_weight_max": 30.537063598632812,
      "activations/layer15_attention_weight_min": -31.151914596557617,
      "activations/layer16_attention_weight_max": 26.412858963012695,
      "activations/layer16_attention_weight_min": -26.967422485351562,
      "activations/layer17_attention_weight_max": 30.019874572753906,
      "activations/layer17_attention_weight_min": -25.44267463684082,
      "activations/layer18_attention_weight_max": 29.21194076538086,
      "activations/layer18_attention_weight_min": -23.393783569335938,
      "activations/layer19_attention_weight_max": 31.02369499206543,
      "activations/layer19_attention_weight_min": -30.939472198486328,
      "activations/layer1_attention_weight_max": 16.77716636657715,
      "activations/layer1_attention_weight_min": -13.586204528808594,
      "activations/layer20_attention_weight_max": 27.12786102294922,
      "activations/layer20_attention_weight_min": -25.5577335357666,
      "activations/layer21_attention_weight_max": 27.238534927368164,
      "activations/layer21_attention_weight_min": -23.701305389404297,
      "activations/layer22_attention_weight_max": 38.173240661621094,
      "activations/layer22_attention_weight_min": -30.17251205444336,
      "activations/layer23_attention_weight_max": 31.594341278076172,
      "activations/layer23_attention_weight_min": -26.190959930419922,
      "activations/layer2_attention_weight_max": 33.12532424926758,
      "activations/layer2_attention_weight_min": -34.158111572265625,
      "activations/layer3_attention_weight_max": 95.95207214355469,
      "activations/layer3_attention_weight_min": -94.98072052001953,
      "activations/layer4_attention_weight_max": 108.01268768310547,
      "activations/layer4_attention_weight_min": -107.16362762451172,
      "activations/layer5_attention_weight_max": 52.00341796875,
      "activations/layer5_attention_weight_min": -61.416744232177734,
      "activations/layer6_attention_weight_max": 45.367515563964844,
      "activations/layer6_attention_weight_min": -48.70759963989258,
      "activations/layer7_attention_weight_max": 85.6925048828125,
      "activations/layer7_attention_weight_min": -86.31056213378906,
      "activations/layer8_attention_weight_max": 42.29265213012695,
      "activations/layer8_attention_weight_min": -37.67876052856445,
      "activations/layer9_attention_weight_max": 29.002485275268555,
      "activations/layer9_attention_weight_min": -30.33259391784668,
      "epoch": 11.45,
      "learning_rate": 7.695719696969696e-05,
      "loss": 2.7739,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.5292,
      "eval_samples_per_second": 503.446,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.5292,
      "eval_openwebtext_samples_per_second": 503.446,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 2.0365,
      "eval_wikitext_samples_per_second": 223.908,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_lambada_loss": 2.486328125,
      "eval_lambada_ppl": 12.01706983184115,
      "eval_lambada_runtime": 9.6266,
      "eval_lambada_samples_per_second": 505.787,
      "step": 197000
    },
    {
      "activations/layer0_attention_weight_max": 15.316351890563965,
      "activations/layer0_attention_weight_min": -12.154733657836914,
      "activations/layer10_attention_weight_max": 40.67791748046875,
      "activations/layer10_attention_weight_min": -35.477317810058594,
      "activations/layer11_attention_weight_max": 37.52215576171875,
      "activations/layer11_attention_weight_min": -34.595741271972656,
      "activations/layer12_attention_weight_max": 26.146039962768555,
      "activations/layer12_attention_weight_min": -25.10749626159668,
      "activations/layer13_attention_weight_max": 40.01217269897461,
      "activations/layer13_attention_weight_min": -34.54967498779297,
      "activations/layer14_attention_weight_max": 49.00448226928711,
      "activations/layer14_attention_weight_min": -36.87370681762695,
      "activations/layer15_attention_weight_max": 37.51015853881836,
      "activations/layer15_attention_weight_min": -31.41766357421875,
      "activations/layer16_attention_weight_max": 31.96166229248047,
      "activations/layer16_attention_weight_min": -26.174049377441406,
      "activations/layer17_attention_weight_max": 32.252342224121094,
      "activations/layer17_attention_weight_min": -25.214218139648438,
      "activations/layer18_attention_weight_max": 29.22344398498535,
      "activations/layer18_attention_weight_min": -22.393007278442383,
      "activations/layer19_attention_weight_max": 38.75990295410156,
      "activations/layer19_attention_weight_min": -28.360631942749023,
      "activations/layer1_attention_weight_max": 16.038040161132812,
      "activations/layer1_attention_weight_min": -15.682701110839844,
      "activations/layer20_attention_weight_max": 32.76978302001953,
      "activations/layer20_attention_weight_min": -23.750871658325195,
      "activations/layer21_attention_weight_max": 29.1217098236084,
      "activations/layer21_attention_weight_min": -21.979944229125977,
      "activations/layer22_attention_weight_max": 43.80645751953125,
      "activations/layer22_attention_weight_min": -29.725435256958008,
      "activations/layer23_attention_weight_max": 39.888214111328125,
      "activations/layer23_attention_weight_min": -24.64325523376465,
      "activations/layer2_attention_weight_max": 34.149261474609375,
      "activations/layer2_attention_weight_min": -34.90977478027344,
      "activations/layer3_attention_weight_max": 99.45191955566406,
      "activations/layer3_attention_weight_min": -97.00128173828125,
      "activations/layer4_attention_weight_max": 118.3008804321289,
      "activations/layer4_attention_weight_min": -118.78177642822266,
      "activations/layer5_attention_weight_max": 54.089263916015625,
      "activations/layer5_attention_weight_min": -66.7346420288086,
      "activations/layer6_attention_weight_max": 48.14020919799805,
      "activations/layer6_attention_weight_min": -51.51543426513672,
      "activations/layer7_attention_weight_max": 112.35816192626953,
      "activations/layer7_attention_weight_min": -103.38732147216797,
      "activations/layer8_attention_weight_max": 48.935035705566406,
      "activations/layer8_attention_weight_min": -47.447181701660156,
      "activations/layer9_attention_weight_max": 40.44477844238281,
      "activations/layer9_attention_weight_min": -37.38921356201172,
      "epoch": 11.45,
      "learning_rate": 7.693825757575757e-05,
      "loss": 2.7795,
      "step": 197050
    },
    {
      "activations/layer0_attention_weight_max": 15.539155006408691,
      "activations/layer0_attention_weight_min": -12.865625381469727,
      "activations/layer10_attention_weight_max": 32.94976043701172,
      "activations/layer10_attention_weight_min": -31.713987350463867,
      "activations/layer11_attention_weight_max": 34.847625732421875,
      "activations/layer11_attention_weight_min": -30.796598434448242,
      "activations/layer12_attention_weight_max": 23.004098892211914,
      "activations/layer12_attention_weight_min": -25.52129554748535,
      "activations/layer13_attention_weight_max": 37.21010971069336,
      "activations/layer13_attention_weight_min": -33.84927749633789,
      "activations/layer14_attention_weight_max": 43.74315643310547,
      "activations/layer14_attention_weight_min": -38.00191116333008,
      "activations/layer15_attention_weight_max": 34.169715881347656,
      "activations/layer15_attention_weight_min": -31.695911407470703,
      "activations/layer16_attention_weight_max": 30.419124603271484,
      "activations/layer16_attention_weight_min": -28.965646743774414,
      "activations/layer17_attention_weight_max": 34.39506912231445,
      "activations/layer17_attention_weight_min": -24.721864700317383,
      "activations/layer18_attention_weight_max": 28.8798885345459,
      "activations/layer18_attention_weight_min": -22.633955001831055,
      "activations/layer19_attention_weight_max": 32.18756103515625,
      "activations/layer19_attention_weight_min": -31.845056533813477,
      "activations/layer1_attention_weight_max": 16.598909378051758,
      "activations/layer1_attention_weight_min": -16.187334060668945,
      "activations/layer20_attention_weight_max": 28.119096755981445,
      "activations/layer20_attention_weight_min": -26.045835494995117,
      "activations/layer21_attention_weight_max": 27.554155349731445,
      "activations/layer21_attention_weight_min": -24.093339920043945,
      "activations/layer22_attention_weight_max": 39.925201416015625,
      "activations/layer22_attention_weight_min": -29.56459617614746,
      "activations/layer23_attention_weight_max": 33.672698974609375,
      "activations/layer23_attention_weight_min": -25.23432159423828,
      "activations/layer2_attention_weight_max": 32.56346130371094,
      "activations/layer2_attention_weight_min": -33.55816650390625,
      "activations/layer3_attention_weight_max": 96.515869140625,
      "activations/layer3_attention_weight_min": -101.7679214477539,
      "activations/layer4_attention_weight_max": 111.03912353515625,
      "activations/layer4_attention_weight_min": -112.5692367553711,
      "activations/layer5_attention_weight_max": 52.312538146972656,
      "activations/layer5_attention_weight_min": -65.5554428100586,
      "activations/layer6_attention_weight_max": 44.569190979003906,
      "activations/layer6_attention_weight_min": -48.069725036621094,
      "activations/layer7_attention_weight_max": 94.55712127685547,
      "activations/layer7_attention_weight_min": -90.44087982177734,
      "activations/layer8_attention_weight_max": 38.6888427734375,
      "activations/layer8_attention_weight_min": -44.1755485534668,
      "activations/layer9_attention_weight_max": 34.17761993408203,
      "activations/layer9_attention_weight_min": -34.11823654174805,
      "epoch": 11.45,
      "learning_rate": 7.691931818181817e-05,
      "loss": 2.7885,
      "step": 197100
    },
    {
      "activations/layer0_attention_weight_max": 15.191166877746582,
      "activations/layer0_attention_weight_min": -12.678675651550293,
      "activations/layer10_attention_weight_max": 34.517852783203125,
      "activations/layer10_attention_weight_min": -34.650604248046875,
      "activations/layer11_attention_weight_max": 36.51702117919922,
      "activations/layer11_attention_weight_min": -34.484500885009766,
      "activations/layer12_attention_weight_max": 24.429004669189453,
      "activations/layer12_attention_weight_min": -24.646257400512695,
      "activations/layer13_attention_weight_max": 35.58528518676758,
      "activations/layer13_attention_weight_min": -33.91532516479492,
      "activations/layer14_attention_weight_max": 40.43436813354492,
      "activations/layer14_attention_weight_min": -34.79235076904297,
      "activations/layer15_attention_weight_max": 38.924163818359375,
      "activations/layer15_attention_weight_min": -33.45890808105469,
      "activations/layer16_attention_weight_max": 28.10285186767578,
      "activations/layer16_attention_weight_min": -26.95343017578125,
      "activations/layer17_attention_weight_max": 29.05643653869629,
      "activations/layer17_attention_weight_min": -25.086402893066406,
      "activations/layer18_attention_weight_max": 28.589754104614258,
      "activations/layer18_attention_weight_min": -22.971651077270508,
      "activations/layer19_attention_weight_max": 34.67177963256836,
      "activations/layer19_attention_weight_min": -30.945337295532227,
      "activations/layer1_attention_weight_max": 17.987106323242188,
      "activations/layer1_attention_weight_min": -15.902491569519043,
      "activations/layer20_attention_weight_max": 29.235700607299805,
      "activations/layer20_attention_weight_min": -26.484838485717773,
      "activations/layer21_attention_weight_max": 28.378067016601562,
      "activations/layer21_attention_weight_min": -24.768295288085938,
      "activations/layer22_attention_weight_max": 45.25321578979492,
      "activations/layer22_attention_weight_min": -28.093429565429688,
      "activations/layer23_attention_weight_max": 36.02362823486328,
      "activations/layer23_attention_weight_min": -24.035736083984375,
      "activations/layer2_attention_weight_max": 36.257564544677734,
      "activations/layer2_attention_weight_min": -33.157928466796875,
      "activations/layer3_attention_weight_max": 105.88258361816406,
      "activations/layer3_attention_weight_min": -105.20292663574219,
      "activations/layer4_attention_weight_max": 129.5861358642578,
      "activations/layer4_attention_weight_min": -125.46141815185547,
      "activations/layer5_attention_weight_max": 55.87618637084961,
      "activations/layer5_attention_weight_min": -64.78606414794922,
      "activations/layer6_attention_weight_max": 48.38936996459961,
      "activations/layer6_attention_weight_min": -52.1640510559082,
      "activations/layer7_attention_weight_max": 96.38758087158203,
      "activations/layer7_attention_weight_min": -95.49946594238281,
      "activations/layer8_attention_weight_max": 43.269351959228516,
      "activations/layer8_attention_weight_min": -44.58438491821289,
      "activations/layer9_attention_weight_max": 34.804622650146484,
      "activations/layer9_attention_weight_min": -34.388343811035156,
      "epoch": 11.46,
      "learning_rate": 7.690037878787877e-05,
      "loss": 2.7814,
      "step": 197150
    },
    {
      "activations/layer0_attention_weight_max": 15.798065185546875,
      "activations/layer0_attention_weight_min": -12.02956485748291,
      "activations/layer10_attention_weight_max": 32.1298713684082,
      "activations/layer10_attention_weight_min": -32.71113967895508,
      "activations/layer11_attention_weight_max": 31.7109375,
      "activations/layer11_attention_weight_min": -29.21059226989746,
      "activations/layer12_attention_weight_max": 25.081499099731445,
      "activations/layer12_attention_weight_min": -23.714120864868164,
      "activations/layer13_attention_weight_max": 37.31023025512695,
      "activations/layer13_attention_weight_min": -32.17650604248047,
      "activations/layer14_attention_weight_max": 44.715885162353516,
      "activations/layer14_attention_weight_min": -36.175384521484375,
      "activations/layer15_attention_weight_max": 35.37950134277344,
      "activations/layer15_attention_weight_min": -31.65120506286621,
      "activations/layer16_attention_weight_max": 31.416780471801758,
      "activations/layer16_attention_weight_min": -28.508007049560547,
      "activations/layer17_attention_weight_max": 30.406816482543945,
      "activations/layer17_attention_weight_min": -26.216777801513672,
      "activations/layer18_attention_weight_max": 32.61549377441406,
      "activations/layer18_attention_weight_min": -23.22585678100586,
      "activations/layer19_attention_weight_max": 34.861934661865234,
      "activations/layer19_attention_weight_min": -31.875852584838867,
      "activations/layer1_attention_weight_max": 18.07767677307129,
      "activations/layer1_attention_weight_min": -14.2423095703125,
      "activations/layer20_attention_weight_max": 27.164085388183594,
      "activations/layer20_attention_weight_min": -25.163118362426758,
      "activations/layer21_attention_weight_max": 28.652219772338867,
      "activations/layer21_attention_weight_min": -23.76800537109375,
      "activations/layer22_attention_weight_max": 43.64248275756836,
      "activations/layer22_attention_weight_min": -29.584646224975586,
      "activations/layer23_attention_weight_max": 34.885231018066406,
      "activations/layer23_attention_weight_min": -26.75444793701172,
      "activations/layer2_attention_weight_max": 31.464698791503906,
      "activations/layer2_attention_weight_min": -32.05768585205078,
      "activations/layer3_attention_weight_max": 91.2203140258789,
      "activations/layer3_attention_weight_min": -92.6368408203125,
      "activations/layer4_attention_weight_max": 107.62567138671875,
      "activations/layer4_attention_weight_min": -110.5820541381836,
      "activations/layer5_attention_weight_max": 51.3416862487793,
      "activations/layer5_attention_weight_min": -63.35413360595703,
      "activations/layer6_attention_weight_max": 41.86146926879883,
      "activations/layer6_attention_weight_min": -47.49827575683594,
      "activations/layer7_attention_weight_max": 86.12984466552734,
      "activations/layer7_attention_weight_min": -89.9460220336914,
      "activations/layer8_attention_weight_max": 37.554107666015625,
      "activations/layer8_attention_weight_min": -40.4630012512207,
      "activations/layer9_attention_weight_max": 31.61992835998535,
      "activations/layer9_attention_weight_min": -32.287715911865234,
      "epoch": 11.46,
      "learning_rate": 7.688143939393939e-05,
      "loss": 2.7832,
      "step": 197200
    },
    {
      "activations/layer0_attention_weight_max": 15.63703727722168,
      "activations/layer0_attention_weight_min": -12.746115684509277,
      "activations/layer10_attention_weight_max": 33.66654586791992,
      "activations/layer10_attention_weight_min": -31.03960418701172,
      "activations/layer11_attention_weight_max": 32.32356262207031,
      "activations/layer11_attention_weight_min": -30.98903465270996,
      "activations/layer12_attention_weight_max": 24.157428741455078,
      "activations/layer12_attention_weight_min": -24.342830657958984,
      "activations/layer13_attention_weight_max": 36.991783142089844,
      "activations/layer13_attention_weight_min": -33.963157653808594,
      "activations/layer14_attention_weight_max": 42.45716094970703,
      "activations/layer14_attention_weight_min": -35.130714416503906,
      "activations/layer15_attention_weight_max": 40.95819091796875,
      "activations/layer15_attention_weight_min": -31.046035766601562,
      "activations/layer16_attention_weight_max": 30.404170989990234,
      "activations/layer16_attention_weight_min": -26.985029220581055,
      "activations/layer17_attention_weight_max": 31.487802505493164,
      "activations/layer17_attention_weight_min": -26.21401023864746,
      "activations/layer18_attention_weight_max": 33.3156852722168,
      "activations/layer18_attention_weight_min": -24.90621566772461,
      "activations/layer19_attention_weight_max": 33.64653396606445,
      "activations/layer19_attention_weight_min": -33.154117584228516,
      "activations/layer1_attention_weight_max": 16.528827667236328,
      "activations/layer1_attention_weight_min": -15.099701881408691,
      "activations/layer20_attention_weight_max": 29.96477508544922,
      "activations/layer20_attention_weight_min": -26.321027755737305,
      "activations/layer21_attention_weight_max": 32.05329895019531,
      "activations/layer21_attention_weight_min": -24.71866798400879,
      "activations/layer22_attention_weight_max": 47.92045211791992,
      "activations/layer22_attention_weight_min": -34.447105407714844,
      "activations/layer23_attention_weight_max": 38.11564636230469,
      "activations/layer23_attention_weight_min": -25.62848472595215,
      "activations/layer2_attention_weight_max": 32.98197555541992,
      "activations/layer2_attention_weight_min": -32.46827697753906,
      "activations/layer3_attention_weight_max": 91.1185073852539,
      "activations/layer3_attention_weight_min": -92.00530242919922,
      "activations/layer4_attention_weight_max": 115.84930419921875,
      "activations/layer4_attention_weight_min": -110.81391906738281,
      "activations/layer5_attention_weight_max": 53.701072692871094,
      "activations/layer5_attention_weight_min": -66.05339050292969,
      "activations/layer6_attention_weight_max": 42.632720947265625,
      "activations/layer6_attention_weight_min": -48.07271194458008,
      "activations/layer7_attention_weight_max": 87.80432891845703,
      "activations/layer7_attention_weight_min": -89.39070892333984,
      "activations/layer8_attention_weight_max": 38.412052154541016,
      "activations/layer8_attention_weight_min": -41.026493072509766,
      "activations/layer9_attention_weight_max": 33.417930603027344,
      "activations/layer9_attention_weight_min": -32.29136276245117,
      "epoch": 11.46,
      "learning_rate": 7.686249999999999e-05,
      "loss": 2.7648,
      "step": 197250
    },
    {
      "activations/layer0_attention_weight_max": 16.391860961914062,
      "activations/layer0_attention_weight_min": -12.220475196838379,
      "activations/layer10_attention_weight_max": 32.92612838745117,
      "activations/layer10_attention_weight_min": -33.01071548461914,
      "activations/layer11_attention_weight_max": 33.152259826660156,
      "activations/layer11_attention_weight_min": -33.347084045410156,
      "activations/layer12_attention_weight_max": 25.47512435913086,
      "activations/layer12_attention_weight_min": -24.92807960510254,
      "activations/layer13_attention_weight_max": 38.89760208129883,
      "activations/layer13_attention_weight_min": -34.65519714355469,
      "activations/layer14_attention_weight_max": 45.306705474853516,
      "activations/layer14_attention_weight_min": -38.71710968017578,
      "activations/layer15_attention_weight_max": 39.081233978271484,
      "activations/layer15_attention_weight_min": -31.97880744934082,
      "activations/layer16_attention_weight_max": 31.185535430908203,
      "activations/layer16_attention_weight_min": -28.480852127075195,
      "activations/layer17_attention_weight_max": 31.83795166015625,
      "activations/layer17_attention_weight_min": -26.971534729003906,
      "activations/layer18_attention_weight_max": 35.349159240722656,
      "activations/layer18_attention_weight_min": -23.55195426940918,
      "activations/layer19_attention_weight_max": 36.16920852661133,
      "activations/layer19_attention_weight_min": -31.059614181518555,
      "activations/layer1_attention_weight_max": 17.081897735595703,
      "activations/layer1_attention_weight_min": -13.707551956176758,
      "activations/layer20_attention_weight_max": 30.374773025512695,
      "activations/layer20_attention_weight_min": -23.23624038696289,
      "activations/layer21_attention_weight_max": 27.327451705932617,
      "activations/layer21_attention_weight_min": -22.230819702148438,
      "activations/layer22_attention_weight_max": 44.488067626953125,
      "activations/layer22_attention_weight_min": -29.25086212158203,
      "activations/layer23_attention_weight_max": 36.309757232666016,
      "activations/layer23_attention_weight_min": -26.989538192749023,
      "activations/layer2_attention_weight_max": 33.37741470336914,
      "activations/layer2_attention_weight_min": -33.08900451660156,
      "activations/layer3_attention_weight_max": 97.25957489013672,
      "activations/layer3_attention_weight_min": -98.13212585449219,
      "activations/layer4_attention_weight_max": 117.45450592041016,
      "activations/layer4_attention_weight_min": -113.59566497802734,
      "activations/layer5_attention_weight_max": 52.16144943237305,
      "activations/layer5_attention_weight_min": -62.39075469970703,
      "activations/layer6_attention_weight_max": 45.89503479003906,
      "activations/layer6_attention_weight_min": -46.78907775878906,
      "activations/layer7_attention_weight_max": 92.7744369506836,
      "activations/layer7_attention_weight_min": -90.2841567993164,
      "activations/layer8_attention_weight_max": 42.36036682128906,
      "activations/layer8_attention_weight_min": -41.705894470214844,
      "activations/layer9_attention_weight_max": 37.03233337402344,
      "activations/layer9_attention_weight_min": -35.23476791381836,
      "epoch": 11.46,
      "learning_rate": 7.68435606060606e-05,
      "loss": 2.7832,
      "step": 197300
    },
    {
      "activations/layer0_attention_weight_max": 15.918254852294922,
      "activations/layer0_attention_weight_min": -12.775222778320312,
      "activations/layer10_attention_weight_max": 31.272497177124023,
      "activations/layer10_attention_weight_min": -31.2930965423584,
      "activations/layer11_attention_weight_max": 32.39634704589844,
      "activations/layer11_attention_weight_min": -32.63753128051758,
      "activations/layer12_attention_weight_max": 27.392839431762695,
      "activations/layer12_attention_weight_min": -24.581470489501953,
      "activations/layer13_attention_weight_max": 37.73690414428711,
      "activations/layer13_attention_weight_min": -33.05998229980469,
      "activations/layer14_attention_weight_max": 47.83864974975586,
      "activations/layer14_attention_weight_min": -36.46719741821289,
      "activations/layer15_attention_weight_max": 36.93938446044922,
      "activations/layer15_attention_weight_min": -33.2078857421875,
      "activations/layer16_attention_weight_max": 33.53644943237305,
      "activations/layer16_attention_weight_min": -28.60672950744629,
      "activations/layer17_attention_weight_max": 31.26102638244629,
      "activations/layer17_attention_weight_min": -26.439289093017578,
      "activations/layer18_attention_weight_max": 29.323284149169922,
      "activations/layer18_attention_weight_min": -22.949495315551758,
      "activations/layer19_attention_weight_max": 32.811527252197266,
      "activations/layer19_attention_weight_min": -30.865570068359375,
      "activations/layer1_attention_weight_max": 17.15998077392578,
      "activations/layer1_attention_weight_min": -17.168739318847656,
      "activations/layer20_attention_weight_max": 29.247886657714844,
      "activations/layer20_attention_weight_min": -26.079389572143555,
      "activations/layer21_attention_weight_max": 27.87579345703125,
      "activations/layer21_attention_weight_min": -23.6772518157959,
      "activations/layer22_attention_weight_max": 40.74663543701172,
      "activations/layer22_attention_weight_min": -30.64799690246582,
      "activations/layer23_attention_weight_max": 31.520009994506836,
      "activations/layer23_attention_weight_min": -22.77503204345703,
      "activations/layer2_attention_weight_max": 34.72041702270508,
      "activations/layer2_attention_weight_min": -32.429725646972656,
      "activations/layer3_attention_weight_max": 94.2293472290039,
      "activations/layer3_attention_weight_min": -97.71865844726562,
      "activations/layer4_attention_weight_max": 110.28861236572266,
      "activations/layer4_attention_weight_min": -116.0867691040039,
      "activations/layer5_attention_weight_max": 51.12091827392578,
      "activations/layer5_attention_weight_min": -61.31340789794922,
      "activations/layer6_attention_weight_max": 42.455345153808594,
      "activations/layer6_attention_weight_min": -44.97633361816406,
      "activations/layer7_attention_weight_max": 85.83904266357422,
      "activations/layer7_attention_weight_min": -85.5501708984375,
      "activations/layer8_attention_weight_max": 38.42934799194336,
      "activations/layer8_attention_weight_min": -39.57792663574219,
      "activations/layer9_attention_weight_max": 32.328792572021484,
      "activations/layer9_attention_weight_min": -31.72999382019043,
      "epoch": 11.47,
      "learning_rate": 7.682462121212121e-05,
      "loss": 2.7784,
      "step": 197350
    },
    {
      "activations/layer0_attention_weight_max": 15.806989669799805,
      "activations/layer0_attention_weight_min": -11.734692573547363,
      "activations/layer10_attention_weight_max": 34.94978332519531,
      "activations/layer10_attention_weight_min": -36.93254470825195,
      "activations/layer11_attention_weight_max": 31.527721405029297,
      "activations/layer11_attention_weight_min": -31.673404693603516,
      "activations/layer12_attention_weight_max": 29.63094711303711,
      "activations/layer12_attention_weight_min": -25.38372230529785,
      "activations/layer13_attention_weight_max": 38.95722198486328,
      "activations/layer13_attention_weight_min": -35.49032974243164,
      "activations/layer14_attention_weight_max": 52.28273010253906,
      "activations/layer14_attention_weight_min": -43.63028335571289,
      "activations/layer15_attention_weight_max": 37.08746337890625,
      "activations/layer15_attention_weight_min": -34.11916732788086,
      "activations/layer16_attention_weight_max": 32.37054443359375,
      "activations/layer16_attention_weight_min": -28.267333984375,
      "activations/layer17_attention_weight_max": 33.08795166015625,
      "activations/layer17_attention_weight_min": -25.235870361328125,
      "activations/layer18_attention_weight_max": 35.996829986572266,
      "activations/layer18_attention_weight_min": -25.55329704284668,
      "activations/layer19_attention_weight_max": 36.11976623535156,
      "activations/layer19_attention_weight_min": -32.788177490234375,
      "activations/layer1_attention_weight_max": 16.3555850982666,
      "activations/layer1_attention_weight_min": -14.260153770446777,
      "activations/layer20_attention_weight_max": 33.14276885986328,
      "activations/layer20_attention_weight_min": -28.25274658203125,
      "activations/layer21_attention_weight_max": 35.293701171875,
      "activations/layer21_attention_weight_min": -26.898386001586914,
      "activations/layer22_attention_weight_max": 50.18662643432617,
      "activations/layer22_attention_weight_min": -34.694496154785156,
      "activations/layer23_attention_weight_max": 36.16901779174805,
      "activations/layer23_attention_weight_min": -25.907306671142578,
      "activations/layer2_attention_weight_max": 34.275325775146484,
      "activations/layer2_attention_weight_min": -32.89320373535156,
      "activations/layer3_attention_weight_max": 96.04938507080078,
      "activations/layer3_attention_weight_min": -97.55597686767578,
      "activations/layer4_attention_weight_max": 113.3167953491211,
      "activations/layer4_attention_weight_min": -118.32130432128906,
      "activations/layer5_attention_weight_max": 49.30015182495117,
      "activations/layer5_attention_weight_min": -62.54088592529297,
      "activations/layer6_attention_weight_max": 43.33146286010742,
      "activations/layer6_attention_weight_min": -50.455875396728516,
      "activations/layer7_attention_weight_max": 89.31031799316406,
      "activations/layer7_attention_weight_min": -93.03924560546875,
      "activations/layer8_attention_weight_max": 37.62427520751953,
      "activations/layer8_attention_weight_min": -42.85478591918945,
      "activations/layer9_attention_weight_max": 34.991355895996094,
      "activations/layer9_attention_weight_min": -37.3143424987793,
      "epoch": 11.47,
      "learning_rate": 7.68060606060606e-05,
      "loss": 2.7837,
      "step": 197400
    },
    {
      "activations/layer0_attention_weight_max": 16.146602630615234,
      "activations/layer0_attention_weight_min": -13.128172874450684,
      "activations/layer10_attention_weight_max": 31.669591903686523,
      "activations/layer10_attention_weight_min": -32.61355972290039,
      "activations/layer11_attention_weight_max": 32.537109375,
      "activations/layer11_attention_weight_min": -32.34347915649414,
      "activations/layer12_attention_weight_max": 27.40164566040039,
      "activations/layer12_attention_weight_min": -25.144729614257812,
      "activations/layer13_attention_weight_max": 43.74806594848633,
      "activations/layer13_attention_weight_min": -33.64694595336914,
      "activations/layer14_attention_weight_max": 47.2197265625,
      "activations/layer14_attention_weight_min": -37.064598083496094,
      "activations/layer15_attention_weight_max": 37.67704772949219,
      "activations/layer15_attention_weight_min": -32.634037017822266,
      "activations/layer16_attention_weight_max": 31.437963485717773,
      "activations/layer16_attention_weight_min": -28.71769905090332,
      "activations/layer17_attention_weight_max": 33.62113571166992,
      "activations/layer17_attention_weight_min": -25.3363094329834,
      "activations/layer18_attention_weight_max": 33.63203048706055,
      "activations/layer18_attention_weight_min": -22.842483520507812,
      "activations/layer19_attention_weight_max": 37.13862228393555,
      "activations/layer19_attention_weight_min": -30.787412643432617,
      "activations/layer1_attention_weight_max": 16.366483688354492,
      "activations/layer1_attention_weight_min": -13.947978973388672,
      "activations/layer20_attention_weight_max": 32.634254455566406,
      "activations/layer20_attention_weight_min": -24.631975173950195,
      "activations/layer21_attention_weight_max": 33.186119079589844,
      "activations/layer21_attention_weight_min": -24.558635711669922,
      "activations/layer22_attention_weight_max": 51.79729080200195,
      "activations/layer22_attention_weight_min": -28.83966636657715,
      "activations/layer23_attention_weight_max": 38.92084503173828,
      "activations/layer23_attention_weight_min": -24.071578979492188,
      "activations/layer2_attention_weight_max": 31.52298355102539,
      "activations/layer2_attention_weight_min": -32.969783782958984,
      "activations/layer3_attention_weight_max": 90.29766845703125,
      "activations/layer3_attention_weight_min": -95.1885757446289,
      "activations/layer4_attention_weight_max": 107.9022216796875,
      "activations/layer4_attention_weight_min": -114.08955383300781,
      "activations/layer5_attention_weight_max": 49.79705810546875,
      "activations/layer5_attention_weight_min": -62.812129974365234,
      "activations/layer6_attention_weight_max": 41.502098083496094,
      "activations/layer6_attention_weight_min": -45.776798248291016,
      "activations/layer7_attention_weight_max": 84.78473663330078,
      "activations/layer7_attention_weight_min": -89.6922836303711,
      "activations/layer8_attention_weight_max": 38.60980224609375,
      "activations/layer8_attention_weight_min": -41.6196403503418,
      "activations/layer9_attention_weight_max": 32.69992446899414,
      "activations/layer9_attention_weight_min": -31.948225021362305,
      "epoch": 11.47,
      "learning_rate": 7.67871212121212e-05,
      "loss": 2.7601,
      "step": 197450
    },
    {
      "activations/layer0_attention_weight_max": 16.22593879699707,
      "activations/layer0_attention_weight_min": -12.38438892364502,
      "activations/layer10_attention_weight_max": 32.2945556640625,
      "activations/layer10_attention_weight_min": -32.568145751953125,
      "activations/layer11_attention_weight_max": 31.949783325195312,
      "activations/layer11_attention_weight_min": -31.85395622253418,
      "activations/layer12_attention_weight_max": 25.071500778198242,
      "activations/layer12_attention_weight_min": -34.83638381958008,
      "activations/layer13_attention_weight_max": 36.862056732177734,
      "activations/layer13_attention_weight_min": -33.58625411987305,
      "activations/layer14_attention_weight_max": 38.258262634277344,
      "activations/layer14_attention_weight_min": -35.74730682373047,
      "activations/layer15_attention_weight_max": 31.80939292907715,
      "activations/layer15_attention_weight_min": -30.888423919677734,
      "activations/layer16_attention_weight_max": 26.684309005737305,
      "activations/layer16_attention_weight_min": -27.84160614013672,
      "activations/layer17_attention_weight_max": 31.005781173706055,
      "activations/layer17_attention_weight_min": -25.95882225036621,
      "activations/layer18_attention_weight_max": 27.49506950378418,
      "activations/layer18_attention_weight_min": -21.751073837280273,
      "activations/layer19_attention_weight_max": 30.244184494018555,
      "activations/layer19_attention_weight_min": -31.215795516967773,
      "activations/layer1_attention_weight_max": 17.765031814575195,
      "activations/layer1_attention_weight_min": -15.712542533874512,
      "activations/layer20_attention_weight_max": 26.92681884765625,
      "activations/layer20_attention_weight_min": -24.95079231262207,
      "activations/layer21_attention_weight_max": 29.018150329589844,
      "activations/layer21_attention_weight_min": -24.828166961669922,
      "activations/layer22_attention_weight_max": 37.103790283203125,
      "activations/layer22_attention_weight_min": -30.213560104370117,
      "activations/layer23_attention_weight_max": 32.83037567138672,
      "activations/layer23_attention_weight_min": -24.242481231689453,
      "activations/layer2_attention_weight_max": 33.693687438964844,
      "activations/layer2_attention_weight_min": -33.40892791748047,
      "activations/layer3_attention_weight_max": 101.87003326416016,
      "activations/layer3_attention_weight_min": -102.2953872680664,
      "activations/layer4_attention_weight_max": 118.39497375488281,
      "activations/layer4_attention_weight_min": -116.6335678100586,
      "activations/layer5_attention_weight_max": 52.1438102722168,
      "activations/layer5_attention_weight_min": -62.96098327636719,
      "activations/layer6_attention_weight_max": 43.47117233276367,
      "activations/layer6_attention_weight_min": -48.310787200927734,
      "activations/layer7_attention_weight_max": 89.23967742919922,
      "activations/layer7_attention_weight_min": -90.23424530029297,
      "activations/layer8_attention_weight_max": 40.002445220947266,
      "activations/layer8_attention_weight_min": -40.264915466308594,
      "activations/layer9_attention_weight_max": 32.887237548828125,
      "activations/layer9_attention_weight_min": -32.86851501464844,
      "epoch": 11.48,
      "learning_rate": 7.676818181818182e-05,
      "loss": 2.787,
      "step": 197500
    },
    {
      "activations/layer0_attention_weight_max": 16.207918167114258,
      "activations/layer0_attention_weight_min": -13.438776969909668,
      "activations/layer10_attention_weight_max": 35.10908126831055,
      "activations/layer10_attention_weight_min": -33.716556549072266,
      "activations/layer11_attention_weight_max": 35.83091354370117,
      "activations/layer11_attention_weight_min": -32.58241271972656,
      "activations/layer12_attention_weight_max": 25.54454803466797,
      "activations/layer12_attention_weight_min": -28.95185089111328,
      "activations/layer13_attention_weight_max": 38.5225830078125,
      "activations/layer13_attention_weight_min": -31.531225204467773,
      "activations/layer14_attention_weight_max": 40.623199462890625,
      "activations/layer14_attention_weight_min": -35.94245529174805,
      "activations/layer15_attention_weight_max": 35.43900680541992,
      "activations/layer15_attention_weight_min": -31.300905227661133,
      "activations/layer16_attention_weight_max": 35.04924774169922,
      "activations/layer16_attention_weight_min": -25.491762161254883,
      "activations/layer17_attention_weight_max": 31.009302139282227,
      "activations/layer17_attention_weight_min": -23.97391700744629,
      "activations/layer18_attention_weight_max": 34.26892852783203,
      "activations/layer18_attention_weight_min": -23.66827392578125,
      "activations/layer19_attention_weight_max": 37.314796447753906,
      "activations/layer19_attention_weight_min": -30.54513931274414,
      "activations/layer1_attention_weight_max": 16.91214370727539,
      "activations/layer1_attention_weight_min": -14.472940444946289,
      "activations/layer20_attention_weight_max": 31.709148406982422,
      "activations/layer20_attention_weight_min": -24.787647247314453,
      "activations/layer21_attention_weight_max": 30.3585205078125,
      "activations/layer21_attention_weight_min": -24.292564392089844,
      "activations/layer22_attention_weight_max": 48.950965881347656,
      "activations/layer22_attention_weight_min": -29.23221778869629,
      "activations/layer23_attention_weight_max": 39.46879577636719,
      "activations/layer23_attention_weight_min": -23.444473266601562,
      "activations/layer2_attention_weight_max": 33.83286666870117,
      "activations/layer2_attention_weight_min": -33.33896255493164,
      "activations/layer3_attention_weight_max": 94.27406311035156,
      "activations/layer3_attention_weight_min": -95.91024780273438,
      "activations/layer4_attention_weight_max": 109.53466796875,
      "activations/layer4_attention_weight_min": -108.43370819091797,
      "activations/layer5_attention_weight_max": 48.990081787109375,
      "activations/layer5_attention_weight_min": -58.59893035888672,
      "activations/layer6_attention_weight_max": 41.62480926513672,
      "activations/layer6_attention_weight_min": -44.28594207763672,
      "activations/layer7_attention_weight_max": 85.06551361083984,
      "activations/layer7_attention_weight_min": -83.70085144042969,
      "activations/layer8_attention_weight_max": 38.1929817199707,
      "activations/layer8_attention_weight_min": -38.86690139770508,
      "activations/layer9_attention_weight_max": 33.1896858215332,
      "activations/layer9_attention_weight_min": -30.863849639892578,
      "epoch": 11.48,
      "learning_rate": 7.674924242424243e-05,
      "loss": 2.7738,
      "step": 197550
    },
    {
      "activations/layer0_attention_weight_max": 15.851945877075195,
      "activations/layer0_attention_weight_min": -12.31501293182373,
      "activations/layer10_attention_weight_max": 32.79008865356445,
      "activations/layer10_attention_weight_min": -33.20613098144531,
      "activations/layer11_attention_weight_max": 33.50548553466797,
      "activations/layer11_attention_weight_min": -34.20359802246094,
      "activations/layer12_attention_weight_max": 24.157560348510742,
      "activations/layer12_attention_weight_min": -29.997087478637695,
      "activations/layer13_attention_weight_max": 35.47848129272461,
      "activations/layer13_attention_weight_min": -32.85972213745117,
      "activations/layer14_attention_weight_max": 40.400882720947266,
      "activations/layer14_attention_weight_min": -34.47957992553711,
      "activations/layer15_attention_weight_max": 32.94011688232422,
      "activations/layer15_attention_weight_min": -30.261444091796875,
      "activations/layer16_attention_weight_max": 30.94026756286621,
      "activations/layer16_attention_weight_min": -26.347230911254883,
      "activations/layer17_attention_weight_max": 30.754005432128906,
      "activations/layer17_attention_weight_min": -24.286407470703125,
      "activations/layer18_attention_weight_max": 30.96265983581543,
      "activations/layer18_attention_weight_min": -22.355924606323242,
      "activations/layer19_attention_weight_max": 30.954349517822266,
      "activations/layer19_attention_weight_min": -29.978717803955078,
      "activations/layer1_attention_weight_max": 17.806896209716797,
      "activations/layer1_attention_weight_min": -16.15509605407715,
      "activations/layer20_attention_weight_max": 29.213850021362305,
      "activations/layer20_attention_weight_min": -23.25152587890625,
      "activations/layer21_attention_weight_max": 29.67519760131836,
      "activations/layer21_attention_weight_min": -23.443954467773438,
      "activations/layer22_attention_weight_max": 42.62176513671875,
      "activations/layer22_attention_weight_min": -31.24707794189453,
      "activations/layer23_attention_weight_max": 31.14733123779297,
      "activations/layer23_attention_weight_min": -25.246715545654297,
      "activations/layer2_attention_weight_max": 36.147640228271484,
      "activations/layer2_attention_weight_min": -32.94013214111328,
      "activations/layer3_attention_weight_max": 102.48281860351562,
      "activations/layer3_attention_weight_min": -96.92861938476562,
      "activations/layer4_attention_weight_max": 118.37493896484375,
      "activations/layer4_attention_weight_min": -114.6498031616211,
      "activations/layer5_attention_weight_max": 53.033241271972656,
      "activations/layer5_attention_weight_min": -63.07708740234375,
      "activations/layer6_attention_weight_max": 47.14113235473633,
      "activations/layer6_attention_weight_min": -46.04934310913086,
      "activations/layer7_attention_weight_max": 95.34379577636719,
      "activations/layer7_attention_weight_min": -89.2663345336914,
      "activations/layer8_attention_weight_max": 40.997901916503906,
      "activations/layer8_attention_weight_min": -41.504310607910156,
      "activations/layer9_attention_weight_max": 35.867027282714844,
      "activations/layer9_attention_weight_min": -33.89052963256836,
      "epoch": 11.48,
      "learning_rate": 7.673030303030302e-05,
      "loss": 2.8018,
      "step": 197600
    },
    {
      "activations/layer0_attention_weight_max": 15.413973808288574,
      "activations/layer0_attention_weight_min": -12.696221351623535,
      "activations/layer10_attention_weight_max": 33.80129623413086,
      "activations/layer10_attention_weight_min": -33.95048522949219,
      "activations/layer11_attention_weight_max": 34.52592468261719,
      "activations/layer11_attention_weight_min": -31.63780975341797,
      "activations/layer12_attention_weight_max": 25.154563903808594,
      "activations/layer12_attention_weight_min": -25.456193923950195,
      "activations/layer13_attention_weight_max": 38.15604782104492,
      "activations/layer13_attention_weight_min": -34.851417541503906,
      "activations/layer14_attention_weight_max": 49.78293228149414,
      "activations/layer14_attention_weight_min": -39.84855270385742,
      "activations/layer15_attention_weight_max": 35.510955810546875,
      "activations/layer15_attention_weight_min": -33.14798355102539,
      "activations/layer16_attention_weight_max": 31.46175193786621,
      "activations/layer16_attention_weight_min": -27.22332000732422,
      "activations/layer17_attention_weight_max": 32.906864166259766,
      "activations/layer17_attention_weight_min": -25.696382522583008,
      "activations/layer18_attention_weight_max": 32.396793365478516,
      "activations/layer18_attention_weight_min": -24.446744918823242,
      "activations/layer19_attention_weight_max": 36.27260971069336,
      "activations/layer19_attention_weight_min": -32.9387321472168,
      "activations/layer1_attention_weight_max": 18.005773544311523,
      "activations/layer1_attention_weight_min": -15.774897575378418,
      "activations/layer20_attention_weight_max": 29.036846160888672,
      "activations/layer20_attention_weight_min": -27.045331954956055,
      "activations/layer21_attention_weight_max": 29.101444244384766,
      "activations/layer21_attention_weight_min": -23.617630004882812,
      "activations/layer22_attention_weight_max": 42.67429733276367,
      "activations/layer22_attention_weight_min": -30.72290802001953,
      "activations/layer23_attention_weight_max": 35.33114242553711,
      "activations/layer23_attention_weight_min": -28.30433464050293,
      "activations/layer2_attention_weight_max": 33.23630905151367,
      "activations/layer2_attention_weight_min": -33.143775939941406,
      "activations/layer3_attention_weight_max": 96.96549224853516,
      "activations/layer3_attention_weight_min": -98.90499877929688,
      "activations/layer4_attention_weight_max": 116.03629302978516,
      "activations/layer4_attention_weight_min": -115.54912567138672,
      "activations/layer5_attention_weight_max": 52.551292419433594,
      "activations/layer5_attention_weight_min": -69.9161605834961,
      "activations/layer6_attention_weight_max": 46.75735855102539,
      "activations/layer6_attention_weight_min": -49.88127136230469,
      "activations/layer7_attention_weight_max": 92.40300750732422,
      "activations/layer7_attention_weight_min": -93.4560317993164,
      "activations/layer8_attention_weight_max": 40.309940338134766,
      "activations/layer8_attention_weight_min": -42.4754524230957,
      "activations/layer9_attention_weight_max": 34.759952545166016,
      "activations/layer9_attention_weight_min": -34.348514556884766,
      "epoch": 11.48,
      "learning_rate": 7.671136363636363e-05,
      "loss": 2.7858,
      "step": 197650
    },
    {
      "activations/layer0_attention_weight_max": 16.111783981323242,
      "activations/layer0_attention_weight_min": -12.165902137756348,
      "activations/layer10_attention_weight_max": 33.47718048095703,
      "activations/layer10_attention_weight_min": -33.50202178955078,
      "activations/layer11_attention_weight_max": 34.826969146728516,
      "activations/layer11_attention_weight_min": -33.300872802734375,
      "activations/layer12_attention_weight_max": 26.756799697875977,
      "activations/layer12_attention_weight_min": -29.215091705322266,
      "activations/layer13_attention_weight_max": 44.51021194458008,
      "activations/layer13_attention_weight_min": -35.774139404296875,
      "activations/layer14_attention_weight_max": 44.303043365478516,
      "activations/layer14_attention_weight_min": -38.862911224365234,
      "activations/layer15_attention_weight_max": 52.03490447998047,
      "activations/layer15_attention_weight_min": -36.569252014160156,
      "activations/layer16_attention_weight_max": 34.97587966918945,
      "activations/layer16_attention_weight_min": -30.291048049926758,
      "activations/layer17_attention_weight_max": 36.413963317871094,
      "activations/layer17_attention_weight_min": -28.93628692626953,
      "activations/layer18_attention_weight_max": 34.139190673828125,
      "activations/layer18_attention_weight_min": -22.76378059387207,
      "activations/layer19_attention_weight_max": 35.80098342895508,
      "activations/layer19_attention_weight_min": -31.028512954711914,
      "activations/layer1_attention_weight_max": 19.082353591918945,
      "activations/layer1_attention_weight_min": -16.505359649658203,
      "activations/layer20_attention_weight_max": 29.42987632751465,
      "activations/layer20_attention_weight_min": -24.167070388793945,
      "activations/layer21_attention_weight_max": 29.11351203918457,
      "activations/layer21_attention_weight_min": -23.61936378479004,
      "activations/layer22_attention_weight_max": 46.15372085571289,
      "activations/layer22_attention_weight_min": -30.07155418395996,
      "activations/layer23_attention_weight_max": 39.20766830444336,
      "activations/layer23_attention_weight_min": -24.39435577392578,
      "activations/layer2_attention_weight_max": 32.812767028808594,
      "activations/layer2_attention_weight_min": -33.94419479370117,
      "activations/layer3_attention_weight_max": 98.45843505859375,
      "activations/layer3_attention_weight_min": -99.65924835205078,
      "activations/layer4_attention_weight_max": 118.1783676147461,
      "activations/layer4_attention_weight_min": -116.70314025878906,
      "activations/layer5_attention_weight_max": 52.49129867553711,
      "activations/layer5_attention_weight_min": -65.54904174804688,
      "activations/layer6_attention_weight_max": 44.0054817199707,
      "activations/layer6_attention_weight_min": -49.70408248901367,
      "activations/layer7_attention_weight_max": 87.39961242675781,
      "activations/layer7_attention_weight_min": -91.19915008544922,
      "activations/layer8_attention_weight_max": 40.262962341308594,
      "activations/layer8_attention_weight_min": -40.439701080322266,
      "activations/layer9_attention_weight_max": 34.000972747802734,
      "activations/layer9_attention_weight_min": -32.60177993774414,
      "epoch": 11.49,
      "learning_rate": 7.669242424242422e-05,
      "loss": 2.7699,
      "step": 197700
    },
    {
      "activations/layer0_attention_weight_max": 15.416546821594238,
      "activations/layer0_attention_weight_min": -12.148905754089355,
      "activations/layer10_attention_weight_max": 32.64754104614258,
      "activations/layer10_attention_weight_min": -31.962358474731445,
      "activations/layer11_attention_weight_max": 33.50506591796875,
      "activations/layer11_attention_weight_min": -31.380401611328125,
      "activations/layer12_attention_weight_max": 28.828195571899414,
      "activations/layer12_attention_weight_min": -25.241174697875977,
      "activations/layer13_attention_weight_max": 41.33246994018555,
      "activations/layer13_attention_weight_min": -33.556705474853516,
      "activations/layer14_attention_weight_max": 46.1071891784668,
      "activations/layer14_attention_weight_min": -35.83899688720703,
      "activations/layer15_attention_weight_max": 35.23970413208008,
      "activations/layer15_attention_weight_min": -31.917631149291992,
      "activations/layer16_attention_weight_max": 33.87910461425781,
      "activations/layer16_attention_weight_min": -27.661739349365234,
      "activations/layer17_attention_weight_max": 31.893802642822266,
      "activations/layer17_attention_weight_min": -25.894636154174805,
      "activations/layer18_attention_weight_max": 29.830726623535156,
      "activations/layer18_attention_weight_min": -24.285747528076172,
      "activations/layer19_attention_weight_max": 38.56441879272461,
      "activations/layer19_attention_weight_min": -32.209041595458984,
      "activations/layer1_attention_weight_max": 16.876283645629883,
      "activations/layer1_attention_weight_min": -12.921414375305176,
      "activations/layer20_attention_weight_max": 35.17013931274414,
      "activations/layer20_attention_weight_min": -26.82967758178711,
      "activations/layer21_attention_weight_max": 29.512168884277344,
      "activations/layer21_attention_weight_min": -25.263212203979492,
      "activations/layer22_attention_weight_max": 47.50868225097656,
      "activations/layer22_attention_weight_min": -32.35785675048828,
      "activations/layer23_attention_weight_max": 35.24656677246094,
      "activations/layer23_attention_weight_min": -24.072498321533203,
      "activations/layer2_attention_weight_max": 36.02301025390625,
      "activations/layer2_attention_weight_min": -34.06549835205078,
      "activations/layer3_attention_weight_max": 98.14659881591797,
      "activations/layer3_attention_weight_min": -100.95862579345703,
      "activations/layer4_attention_weight_max": 113.40789031982422,
      "activations/layer4_attention_weight_min": -113.45738983154297,
      "activations/layer5_attention_weight_max": 54.77093505859375,
      "activations/layer5_attention_weight_min": -64.37775421142578,
      "activations/layer6_attention_weight_max": 45.975833892822266,
      "activations/layer6_attention_weight_min": -47.4060173034668,
      "activations/layer7_attention_weight_max": 90.6373062133789,
      "activations/layer7_attention_weight_min": -88.40994262695312,
      "activations/layer8_attention_weight_max": 40.04170227050781,
      "activations/layer8_attention_weight_min": -39.784141540527344,
      "activations/layer9_attention_weight_max": 32.789554595947266,
      "activations/layer9_attention_weight_min": -32.701656341552734,
      "epoch": 11.49,
      "learning_rate": 7.667348484848484e-05,
      "loss": 2.7696,
      "step": 197750
    },
    {
      "activations/layer0_attention_weight_max": 16.77541160583496,
      "activations/layer0_attention_weight_min": -12.048257827758789,
      "activations/layer10_attention_weight_max": 33.144412994384766,
      "activations/layer10_attention_weight_min": -33.2512092590332,
      "activations/layer11_attention_weight_max": 34.59878921508789,
      "activations/layer11_attention_weight_min": -31.320659637451172,
      "activations/layer12_attention_weight_max": 25.19232749938965,
      "activations/layer12_attention_weight_min": -25.825992584228516,
      "activations/layer13_attention_weight_max": 42.524593353271484,
      "activations/layer13_attention_weight_min": -33.340728759765625,
      "activations/layer14_attention_weight_max": 44.15507125854492,
      "activations/layer14_attention_weight_min": -36.45681381225586,
      "activations/layer15_attention_weight_max": 41.28001403808594,
      "activations/layer15_attention_weight_min": -32.7327766418457,
      "activations/layer16_attention_weight_max": 32.77384948730469,
      "activations/layer16_attention_weight_min": -28.95109748840332,
      "activations/layer17_attention_weight_max": 31.521257400512695,
      "activations/layer17_attention_weight_min": -25.875398635864258,
      "activations/layer18_attention_weight_max": 29.032079696655273,
      "activations/layer18_attention_weight_min": -23.91860580444336,
      "activations/layer19_attention_weight_max": 37.59320068359375,
      "activations/layer19_attention_weight_min": -30.217016220092773,
      "activations/layer1_attention_weight_max": 15.884669303894043,
      "activations/layer1_attention_weight_min": -14.78598690032959,
      "activations/layer20_attention_weight_max": 32.65437698364258,
      "activations/layer20_attention_weight_min": -24.23934555053711,
      "activations/layer21_attention_weight_max": 30.03485679626465,
      "activations/layer21_attention_weight_min": -23.506393432617188,
      "activations/layer22_attention_weight_max": 40.98918914794922,
      "activations/layer22_attention_weight_min": -28.835004806518555,
      "activations/layer23_attention_weight_max": 35.26054382324219,
      "activations/layer23_attention_weight_min": -24.020103454589844,
      "activations/layer2_attention_weight_max": 34.29819869995117,
      "activations/layer2_attention_weight_min": -32.81755828857422,
      "activations/layer3_attention_weight_max": 99.06800079345703,
      "activations/layer3_attention_weight_min": -96.9176025390625,
      "activations/layer4_attention_weight_max": 116.1785659790039,
      "activations/layer4_attention_weight_min": -112.4471435546875,
      "activations/layer5_attention_weight_max": 63.81339645385742,
      "activations/layer5_attention_weight_min": -63.075599670410156,
      "activations/layer6_attention_weight_max": 50.147945404052734,
      "activations/layer6_attention_weight_min": -49.096981048583984,
      "activations/layer7_attention_weight_max": 92.5677490234375,
      "activations/layer7_attention_weight_min": -89.82621765136719,
      "activations/layer8_attention_weight_max": 39.6143798828125,
      "activations/layer8_attention_weight_min": -40.5634651184082,
      "activations/layer9_attention_weight_max": 35.8675651550293,
      "activations/layer9_attention_weight_min": -32.826656341552734,
      "epoch": 11.49,
      "learning_rate": 7.665454545454545e-05,
      "loss": 2.7893,
      "step": 197800
    },
    {
      "activations/layer0_attention_weight_max": 15.79628849029541,
      "activations/layer0_attention_weight_min": -12.600131034851074,
      "activations/layer10_attention_weight_max": 35.300323486328125,
      "activations/layer10_attention_weight_min": -31.774139404296875,
      "activations/layer11_attention_weight_max": 30.997596740722656,
      "activations/layer11_attention_weight_min": -30.92970848083496,
      "activations/layer12_attention_weight_max": 23.73394012451172,
      "activations/layer12_attention_weight_min": -25.305387496948242,
      "activations/layer13_attention_weight_max": 37.55157470703125,
      "activations/layer13_attention_weight_min": -36.48303985595703,
      "activations/layer14_attention_weight_max": 40.1484260559082,
      "activations/layer14_attention_weight_min": -38.840415954589844,
      "activations/layer15_attention_weight_max": 38.14200973510742,
      "activations/layer15_attention_weight_min": -34.62924575805664,
      "activations/layer16_attention_weight_max": 31.00901985168457,
      "activations/layer16_attention_weight_min": -27.98846435546875,
      "activations/layer17_attention_weight_max": 37.310333251953125,
      "activations/layer17_attention_weight_min": -28.4279727935791,
      "activations/layer18_attention_weight_max": 34.220916748046875,
      "activations/layer18_attention_weight_min": -23.26190757751465,
      "activations/layer19_attention_weight_max": 34.039127349853516,
      "activations/layer19_attention_weight_min": -31.352645874023438,
      "activations/layer1_attention_weight_max": 18.037616729736328,
      "activations/layer1_attention_weight_min": -14.428479194641113,
      "activations/layer20_attention_weight_max": 29.428525924682617,
      "activations/layer20_attention_weight_min": -24.899023056030273,
      "activations/layer21_attention_weight_max": 31.54134750366211,
      "activations/layer21_attention_weight_min": -24.035198211669922,
      "activations/layer22_attention_weight_max": 44.21113586425781,
      "activations/layer22_attention_weight_min": -29.34013557434082,
      "activations/layer23_attention_weight_max": 33.628517150878906,
      "activations/layer23_attention_weight_min": -21.803050994873047,
      "activations/layer2_attention_weight_max": 32.8248291015625,
      "activations/layer2_attention_weight_min": -30.776561737060547,
      "activations/layer3_attention_weight_max": 98.77513885498047,
      "activations/layer3_attention_weight_min": -97.97723388671875,
      "activations/layer4_attention_weight_max": 113.60334777832031,
      "activations/layer4_attention_weight_min": -115.9552993774414,
      "activations/layer5_attention_weight_max": 57.81461715698242,
      "activations/layer5_attention_weight_min": -64.36717224121094,
      "activations/layer6_attention_weight_max": 46.821956634521484,
      "activations/layer6_attention_weight_min": -47.49197769165039,
      "activations/layer7_attention_weight_max": 93.36236572265625,
      "activations/layer7_attention_weight_min": -89.99102783203125,
      "activations/layer8_attention_weight_max": 41.85582733154297,
      "activations/layer8_attention_weight_min": -42.233821868896484,
      "activations/layer9_attention_weight_max": 35.04890060424805,
      "activations/layer9_attention_weight_min": -33.90684127807617,
      "epoch": 11.5,
      "learning_rate": 7.663560606060605e-05,
      "loss": 2.7761,
      "step": 197850
    },
    {
      "activations/layer0_attention_weight_max": 15.163507461547852,
      "activations/layer0_attention_weight_min": -12.880860328674316,
      "activations/layer10_attention_weight_max": 31.42548179626465,
      "activations/layer10_attention_weight_min": -31.63731575012207,
      "activations/layer11_attention_weight_max": 33.074012756347656,
      "activations/layer11_attention_weight_min": -32.19602966308594,
      "activations/layer12_attention_weight_max": 27.835315704345703,
      "activations/layer12_attention_weight_min": -26.30103302001953,
      "activations/layer13_attention_weight_max": 42.31022644042969,
      "activations/layer13_attention_weight_min": -33.14503479003906,
      "activations/layer14_attention_weight_max": 41.85380172729492,
      "activations/layer14_attention_weight_min": -35.89655685424805,
      "activations/layer15_attention_weight_max": 40.66737365722656,
      "activations/layer15_attention_weight_min": -31.79088592529297,
      "activations/layer16_attention_weight_max": 30.444372177124023,
      "activations/layer16_attention_weight_min": -28.218263626098633,
      "activations/layer17_attention_weight_max": 32.6840705871582,
      "activations/layer17_attention_weight_min": -26.019996643066406,
      "activations/layer18_attention_weight_max": 34.77934265136719,
      "activations/layer18_attention_weight_min": -23.62967300415039,
      "activations/layer19_attention_weight_max": 33.79114532470703,
      "activations/layer19_attention_weight_min": -31.2137451171875,
      "activations/layer1_attention_weight_max": 17.24983787536621,
      "activations/layer1_attention_weight_min": -13.94509506225586,
      "activations/layer20_attention_weight_max": 30.05626106262207,
      "activations/layer20_attention_weight_min": -24.51339340209961,
      "activations/layer21_attention_weight_max": 29.773183822631836,
      "activations/layer21_attention_weight_min": -24.637210845947266,
      "activations/layer22_attention_weight_max": 43.420867919921875,
      "activations/layer22_attention_weight_min": -29.58897590637207,
      "activations/layer23_attention_weight_max": 34.23374557495117,
      "activations/layer23_attention_weight_min": -25.803258895874023,
      "activations/layer2_attention_weight_max": 33.24265670776367,
      "activations/layer2_attention_weight_min": -32.146270751953125,
      "activations/layer3_attention_weight_max": 97.97184753417969,
      "activations/layer3_attention_weight_min": -99.08627319335938,
      "activations/layer4_attention_weight_max": 112.0208511352539,
      "activations/layer4_attention_weight_min": -122.34893035888672,
      "activations/layer5_attention_weight_max": 53.50411605834961,
      "activations/layer5_attention_weight_min": -62.88662338256836,
      "activations/layer6_attention_weight_max": 44.4220085144043,
      "activations/layer6_attention_weight_min": -49.341773986816406,
      "activations/layer7_attention_weight_max": 86.15092468261719,
      "activations/layer7_attention_weight_min": -90.8007583618164,
      "activations/layer8_attention_weight_max": 39.65420913696289,
      "activations/layer8_attention_weight_min": -43.96339416503906,
      "activations/layer9_attention_weight_max": 32.837158203125,
      "activations/layer9_attention_weight_min": -33.994789123535156,
      "epoch": 11.5,
      "learning_rate": 7.661666666666665e-05,
      "loss": 2.7935,
      "step": 197900
    },
    {
      "activations/layer0_attention_weight_max": 15.364707946777344,
      "activations/layer0_attention_weight_min": -12.460039138793945,
      "activations/layer10_attention_weight_max": 34.65262985229492,
      "activations/layer10_attention_weight_min": -35.08761215209961,
      "activations/layer11_attention_weight_max": 35.13555908203125,
      "activations/layer11_attention_weight_min": -37.16582107543945,
      "activations/layer12_attention_weight_max": 24.286884307861328,
      "activations/layer12_attention_weight_min": -28.355562210083008,
      "activations/layer13_attention_weight_max": 38.08430099487305,
      "activations/layer13_attention_weight_min": -33.501564025878906,
      "activations/layer14_attention_weight_max": 45.09807586669922,
      "activations/layer14_attention_weight_min": -35.98811340332031,
      "activations/layer15_attention_weight_max": 35.324485778808594,
      "activations/layer15_attention_weight_min": -33.57190704345703,
      "activations/layer16_attention_weight_max": 28.73944854736328,
      "activations/layer16_attention_weight_min": -27.7729434967041,
      "activations/layer17_attention_weight_max": 29.95585060119629,
      "activations/layer17_attention_weight_min": -24.33799934387207,
      "activations/layer18_attention_weight_max": 31.034584045410156,
      "activations/layer18_attention_weight_min": -22.464584350585938,
      "activations/layer19_attention_weight_max": 30.922271728515625,
      "activations/layer19_attention_weight_min": -31.143924713134766,
      "activations/layer1_attention_weight_max": 16.449642181396484,
      "activations/layer1_attention_weight_min": -15.176220893859863,
      "activations/layer20_attention_weight_max": 26.252214431762695,
      "activations/layer20_attention_weight_min": -26.206087112426758,
      "activations/layer21_attention_weight_max": 27.76883316040039,
      "activations/layer21_attention_weight_min": -23.99492073059082,
      "activations/layer22_attention_weight_max": 40.56380844116211,
      "activations/layer22_attention_weight_min": -29.53136444091797,
      "activations/layer23_attention_weight_max": 35.199851989746094,
      "activations/layer23_attention_weight_min": -24.086013793945312,
      "activations/layer2_attention_weight_max": 32.752227783203125,
      "activations/layer2_attention_weight_min": -34.371307373046875,
      "activations/layer3_attention_weight_max": 97.91071319580078,
      "activations/layer3_attention_weight_min": -100.52420043945312,
      "activations/layer4_attention_weight_max": 119.37129211425781,
      "activations/layer4_attention_weight_min": -118.92961883544922,
      "activations/layer5_attention_weight_max": 55.61831283569336,
      "activations/layer5_attention_weight_min": -64.07083129882812,
      "activations/layer6_attention_weight_max": 48.12010955810547,
      "activations/layer6_attention_weight_min": -50.2024040222168,
      "activations/layer7_attention_weight_max": 95.29589080810547,
      "activations/layer7_attention_weight_min": -97.94915008544922,
      "activations/layer8_attention_weight_max": 40.9922981262207,
      "activations/layer8_attention_weight_min": -42.89422607421875,
      "activations/layer9_attention_weight_max": 35.112274169921875,
      "activations/layer9_attention_weight_min": -35.54816818237305,
      "epoch": 11.5,
      "learning_rate": 7.659810606060605e-05,
      "loss": 2.7739,
      "step": 197950
    },
    {
      "activations/layer0_attention_weight_max": 15.18277645111084,
      "activations/layer0_attention_weight_min": -12.271703720092773,
      "activations/layer10_attention_weight_max": 32.336734771728516,
      "activations/layer10_attention_weight_min": -34.11212158203125,
      "activations/layer11_attention_weight_max": 31.976634979248047,
      "activations/layer11_attention_weight_min": -33.24431228637695,
      "activations/layer12_attention_weight_max": 35.06382369995117,
      "activations/layer12_attention_weight_min": -25.909786224365234,
      "activations/layer13_attention_weight_max": 41.264198303222656,
      "activations/layer13_attention_weight_min": -35.64637756347656,
      "activations/layer14_attention_weight_max": 47.4254264831543,
      "activations/layer14_attention_weight_min": -42.263816833496094,
      "activations/layer15_attention_weight_max": 36.115360260009766,
      "activations/layer15_attention_weight_min": -33.3737678527832,
      "activations/layer16_attention_weight_max": 30.494096755981445,
      "activations/layer16_attention_weight_min": -26.84979248046875,
      "activations/layer17_attention_weight_max": 31.71957015991211,
      "activations/layer17_attention_weight_min": -24.85610008239746,
      "activations/layer18_attention_weight_max": 32.351104736328125,
      "activations/layer18_attention_weight_min": -24.59245491027832,
      "activations/layer19_attention_weight_max": 34.260162353515625,
      "activations/layer19_attention_weight_min": -30.08859634399414,
      "activations/layer1_attention_weight_max": 15.965571403503418,
      "activations/layer1_attention_weight_min": -13.819580078125,
      "activations/layer20_attention_weight_max": 30.158790588378906,
      "activations/layer20_attention_weight_min": -21.935707092285156,
      "activations/layer21_attention_weight_max": 30.02919578552246,
      "activations/layer21_attention_weight_min": -22.181161880493164,
      "activations/layer22_attention_weight_max": 41.55451583862305,
      "activations/layer22_attention_weight_min": -28.299150466918945,
      "activations/layer23_attention_weight_max": 31.25704002380371,
      "activations/layer23_attention_weight_min": -26.427684783935547,
      "activations/layer2_attention_weight_max": 32.48387145996094,
      "activations/layer2_attention_weight_min": -32.40812683105469,
      "activations/layer3_attention_weight_max": 92.79582977294922,
      "activations/layer3_attention_weight_min": -99.77648162841797,
      "activations/layer4_attention_weight_max": 112.13384246826172,
      "activations/layer4_attention_weight_min": -119.29731750488281,
      "activations/layer5_attention_weight_max": 55.97381591796875,
      "activations/layer5_attention_weight_min": -65.8341064453125,
      "activations/layer6_attention_weight_max": 50.02737808227539,
      "activations/layer6_attention_weight_min": -48.49881362915039,
      "activations/layer7_attention_weight_max": 91.41790771484375,
      "activations/layer7_attention_weight_min": -94.79153442382812,
      "activations/layer8_attention_weight_max": 41.67210388183594,
      "activations/layer8_attention_weight_min": -42.95022201538086,
      "activations/layer9_attention_weight_max": 34.678497314453125,
      "activations/layer9_attention_weight_min": -35.54096221923828,
      "epoch": 11.5,
      "learning_rate": 7.657916666666666e-05,
      "loss": 2.7682,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5205,
      "eval_samples_per_second": 503.96,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5205,
      "eval_openwebtext_samples_per_second": 503.96,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 2.0907,
      "eval_wikitext_samples_per_second": 218.108,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_lambada_loss": 2.54296875,
      "eval_lambada_ppl": 12.717369675355013,
      "eval_lambada_runtime": 9.6595,
      "eval_lambada_samples_per_second": 504.062,
      "step": 198000
    },
    {
      "activations/layer0_attention_weight_max": 15.573853492736816,
      "activations/layer0_attention_weight_min": -12.595293045043945,
      "activations/layer10_attention_weight_max": 29.900047302246094,
      "activations/layer10_attention_weight_min": -31.40285873413086,
      "activations/layer11_attention_weight_max": 34.863136291503906,
      "activations/layer11_attention_weight_min": -31.590457916259766,
      "activations/layer12_attention_weight_max": 23.65310287475586,
      "activations/layer12_attention_weight_min": -24.582935333251953,
      "activations/layer13_attention_weight_max": 37.104652404785156,
      "activations/layer13_attention_weight_min": -34.52653121948242,
      "activations/layer14_attention_weight_max": 40.27729034423828,
      "activations/layer14_attention_weight_min": -36.691925048828125,
      "activations/layer15_attention_weight_max": 36.766910552978516,
      "activations/layer15_attention_weight_min": -32.11725997924805,
      "activations/layer16_attention_weight_max": 27.94114875793457,
      "activations/layer16_attention_weight_min": -28.194250106811523,
      "activations/layer17_attention_weight_max": 29.306447982788086,
      "activations/layer17_attention_weight_min": -24.331512451171875,
      "activations/layer18_attention_weight_max": 30.776689529418945,
      "activations/layer18_attention_weight_min": -23.53433609008789,
      "activations/layer19_attention_weight_max": 34.09169387817383,
      "activations/layer19_attention_weight_min": -29.003173828125,
      "activations/layer1_attention_weight_max": 17.463436126708984,
      "activations/layer1_attention_weight_min": -16.997167587280273,
      "activations/layer20_attention_weight_max": 28.567054748535156,
      "activations/layer20_attention_weight_min": -24.13578224182129,
      "activations/layer21_attention_weight_max": 26.482467651367188,
      "activations/layer21_attention_weight_min": -23.159521102905273,
      "activations/layer22_attention_weight_max": 38.520416259765625,
      "activations/layer22_attention_weight_min": -29.252620697021484,
      "activations/layer23_attention_weight_max": 29.537750244140625,
      "activations/layer23_attention_weight_min": -24.312007904052734,
      "activations/layer2_attention_weight_max": 32.624359130859375,
      "activations/layer2_attention_weight_min": -32.23232650756836,
      "activations/layer3_attention_weight_max": 95.4030532836914,
      "activations/layer3_attention_weight_min": -95.21887969970703,
      "activations/layer4_attention_weight_max": 111.21632385253906,
      "activations/layer4_attention_weight_min": -110.69464111328125,
      "activations/layer5_attention_weight_max": 50.8576774597168,
      "activations/layer5_attention_weight_min": -61.264068603515625,
      "activations/layer6_attention_weight_max": 43.51007843017578,
      "activations/layer6_attention_weight_min": -45.4709587097168,
      "activations/layer7_attention_weight_max": 87.66118621826172,
      "activations/layer7_attention_weight_min": -87.86611938476562,
      "activations/layer8_attention_weight_max": 37.782806396484375,
      "activations/layer8_attention_weight_min": -40.49003982543945,
      "activations/layer9_attention_weight_max": 32.49668884277344,
      "activations/layer9_attention_weight_min": -32.505714416503906,
      "epoch": 11.51,
      "learning_rate": 7.656022727272726e-05,
      "loss": 2.7928,
      "step": 198050
    },
    {
      "activations/layer0_attention_weight_max": 15.154399871826172,
      "activations/layer0_attention_weight_min": -12.70953369140625,
      "activations/layer10_attention_weight_max": 43.60356903076172,
      "activations/layer10_attention_weight_min": -37.10873794555664,
      "activations/layer11_attention_weight_max": 43.566246032714844,
      "activations/layer11_attention_weight_min": -35.96520233154297,
      "activations/layer12_attention_weight_max": 25.36603355407715,
      "activations/layer12_attention_weight_min": -27.34154510498047,
      "activations/layer13_attention_weight_max": 41.36948776245117,
      "activations/layer13_attention_weight_min": -35.377288818359375,
      "activations/layer14_attention_weight_max": 51.681373596191406,
      "activations/layer14_attention_weight_min": -39.70112609863281,
      "activations/layer15_attention_weight_max": 38.22189712524414,
      "activations/layer15_attention_weight_min": -34.06484603881836,
      "activations/layer16_attention_weight_max": 32.85215377807617,
      "activations/layer16_attention_weight_min": -28.023679733276367,
      "activations/layer17_attention_weight_max": 32.80667495727539,
      "activations/layer17_attention_weight_min": -26.019418716430664,
      "activations/layer18_attention_weight_max": 32.28417205810547,
      "activations/layer18_attention_weight_min": -22.68451499938965,
      "activations/layer19_attention_weight_max": 37.06766891479492,
      "activations/layer19_attention_weight_min": -32.217586517333984,
      "activations/layer1_attention_weight_max": 16.16269302368164,
      "activations/layer1_attention_weight_min": -14.58909797668457,
      "activations/layer20_attention_weight_max": 30.541698455810547,
      "activations/layer20_attention_weight_min": -25.002758026123047,
      "activations/layer21_attention_weight_max": 29.253190994262695,
      "activations/layer21_attention_weight_min": -24.76268768310547,
      "activations/layer22_attention_weight_max": 47.25358200073242,
      "activations/layer22_attention_weight_min": -31.484052658081055,
      "activations/layer23_attention_weight_max": 37.09442901611328,
      "activations/layer23_attention_weight_min": -26.387958526611328,
      "activations/layer2_attention_weight_max": 36.396484375,
      "activations/layer2_attention_weight_min": -36.99675750732422,
      "activations/layer3_attention_weight_max": 101.5504379272461,
      "activations/layer3_attention_weight_min": -105.97093963623047,
      "activations/layer4_attention_weight_max": 117.20057678222656,
      "activations/layer4_attention_weight_min": -120.60221099853516,
      "activations/layer5_attention_weight_max": 56.17981719970703,
      "activations/layer5_attention_weight_min": -64.41107177734375,
      "activations/layer6_attention_weight_max": 47.92416763305664,
      "activations/layer6_attention_weight_min": -50.45814514160156,
      "activations/layer7_attention_weight_max": 97.37855529785156,
      "activations/layer7_attention_weight_min": -97.71160125732422,
      "activations/layer8_attention_weight_max": 47.06122589111328,
      "activations/layer8_attention_weight_min": -44.4715461730957,
      "activations/layer9_attention_weight_max": 40.12892532348633,
      "activations/layer9_attention_weight_min": -36.291481018066406,
      "epoch": 11.51,
      "learning_rate": 7.654128787878788e-05,
      "loss": 2.779,
      "step": 198100
    },
    {
      "activations/layer0_attention_weight_max": 15.849448204040527,
      "activations/layer0_attention_weight_min": -12.337640762329102,
      "activations/layer10_attention_weight_max": 32.33793640136719,
      "activations/layer10_attention_weight_min": -32.038936614990234,
      "activations/layer11_attention_weight_max": 30.51319122314453,
      "activations/layer11_attention_weight_min": -30.550832748413086,
      "activations/layer12_attention_weight_max": 24.123735427856445,
      "activations/layer12_attention_weight_min": -23.775590896606445,
      "activations/layer13_attention_weight_max": 38.508819580078125,
      "activations/layer13_attention_weight_min": -32.57080841064453,
      "activations/layer14_attention_weight_max": 43.694034576416016,
      "activations/layer14_attention_weight_min": -38.68498229980469,
      "activations/layer15_attention_weight_max": 35.75752639770508,
      "activations/layer15_attention_weight_min": -30.68302345275879,
      "activations/layer16_attention_weight_max": 31.399059295654297,
      "activations/layer16_attention_weight_min": -26.91266632080078,
      "activations/layer17_attention_weight_max": 31.49603271484375,
      "activations/layer17_attention_weight_min": -25.121540069580078,
      "activations/layer18_attention_weight_max": 33.111637115478516,
      "activations/layer18_attention_weight_min": -23.68859100341797,
      "activations/layer19_attention_weight_max": 34.109527587890625,
      "activations/layer19_attention_weight_min": -32.306556701660156,
      "activations/layer1_attention_weight_max": 16.31245231628418,
      "activations/layer1_attention_weight_min": -15.263876914978027,
      "activations/layer20_attention_weight_max": 29.636323928833008,
      "activations/layer20_attention_weight_min": -23.540983200073242,
      "activations/layer21_attention_weight_max": 28.72300148010254,
      "activations/layer21_attention_weight_min": -23.976232528686523,
      "activations/layer22_attention_weight_max": 52.77886199951172,
      "activations/layer22_attention_weight_min": -33.01884841918945,
      "activations/layer23_attention_weight_max": 36.47467041015625,
      "activations/layer23_attention_weight_min": -25.12313461303711,
      "activations/layer2_attention_weight_max": 34.77541732788086,
      "activations/layer2_attention_weight_min": -33.18522262573242,
      "activations/layer3_attention_weight_max": 100.01008605957031,
      "activations/layer3_attention_weight_min": -97.99269104003906,
      "activations/layer4_attention_weight_max": 115.93707275390625,
      "activations/layer4_attention_weight_min": -116.6480484008789,
      "activations/layer5_attention_weight_max": 52.920021057128906,
      "activations/layer5_attention_weight_min": -61.499629974365234,
      "activations/layer6_attention_weight_max": 45.399723052978516,
      "activations/layer6_attention_weight_min": -48.15574645996094,
      "activations/layer7_attention_weight_max": 94.48218536376953,
      "activations/layer7_attention_weight_min": -89.33050537109375,
      "activations/layer8_attention_weight_max": 37.221981048583984,
      "activations/layer8_attention_weight_min": -39.15898513793945,
      "activations/layer9_attention_weight_max": 34.44381332397461,
      "activations/layer9_attention_weight_min": -33.18551254272461,
      "epoch": 11.51,
      "learning_rate": 7.652272727272727e-05,
      "loss": 2.7931,
      "step": 198150
    },
    {
      "activations/layer0_attention_weight_max": 15.185892105102539,
      "activations/layer0_attention_weight_min": -12.51652717590332,
      "activations/layer10_attention_weight_max": 32.56023025512695,
      "activations/layer10_attention_weight_min": -34.38520050048828,
      "activations/layer11_attention_weight_max": 30.936870574951172,
      "activations/layer11_attention_weight_min": -33.229949951171875,
      "activations/layer12_attention_weight_max": 23.862995147705078,
      "activations/layer12_attention_weight_min": -25.595239639282227,
      "activations/layer13_attention_weight_max": 38.39567184448242,
      "activations/layer13_attention_weight_min": -36.160911560058594,
      "activations/layer14_attention_weight_max": 41.995609283447266,
      "activations/layer14_attention_weight_min": -40.169456481933594,
      "activations/layer15_attention_weight_max": 33.998985290527344,
      "activations/layer15_attention_weight_min": -33.05024337768555,
      "activations/layer16_attention_weight_max": 32.4526481628418,
      "activations/layer16_attention_weight_min": -29.008892059326172,
      "activations/layer17_attention_weight_max": 31.13843536376953,
      "activations/layer17_attention_weight_min": -25.33025550842285,
      "activations/layer18_attention_weight_max": 31.52971076965332,
      "activations/layer18_attention_weight_min": -22.922258377075195,
      "activations/layer19_attention_weight_max": 33.863224029541016,
      "activations/layer19_attention_weight_min": -31.752338409423828,
      "activations/layer1_attention_weight_max": 15.927667617797852,
      "activations/layer1_attention_weight_min": -14.554278373718262,
      "activations/layer20_attention_weight_max": 31.016942977905273,
      "activations/layer20_attention_weight_min": -24.557449340820312,
      "activations/layer21_attention_weight_max": 30.566387176513672,
      "activations/layer21_attention_weight_min": -24.33116912841797,
      "activations/layer22_attention_weight_max": 43.51558303833008,
      "activations/layer22_attention_weight_min": -28.552209854125977,
      "activations/layer23_attention_weight_max": 38.064735412597656,
      "activations/layer23_attention_weight_min": -26.233348846435547,
      "activations/layer2_attention_weight_max": 33.39299011230469,
      "activations/layer2_attention_weight_min": -35.295860290527344,
      "activations/layer3_attention_weight_max": 94.6462631225586,
      "activations/layer3_attention_weight_min": -97.64480590820312,
      "activations/layer4_attention_weight_max": 108.87725830078125,
      "activations/layer4_attention_weight_min": -116.51922607421875,
      "activations/layer5_attention_weight_max": 51.78971862792969,
      "activations/layer5_attention_weight_min": -62.48015594482422,
      "activations/layer6_attention_weight_max": 45.06159973144531,
      "activations/layer6_attention_weight_min": -49.238426208496094,
      "activations/layer7_attention_weight_max": 96.3521728515625,
      "activations/layer7_attention_weight_min": -94.54457092285156,
      "activations/layer8_attention_weight_max": 40.208072662353516,
      "activations/layer8_attention_weight_min": -41.269508361816406,
      "activations/layer9_attention_weight_max": 33.70662307739258,
      "activations/layer9_attention_weight_min": -35.509517669677734,
      "epoch": 11.52,
      "learning_rate": 7.650378787878787e-05,
      "loss": 2.7868,
      "step": 198200
    },
    {
      "activations/layer0_attention_weight_max": 18.427080154418945,
      "activations/layer0_attention_weight_min": -12.467100143432617,
      "activations/layer10_attention_weight_max": 31.597606658935547,
      "activations/layer10_attention_weight_min": -32.764892578125,
      "activations/layer11_attention_weight_max": 32.98499298095703,
      "activations/layer11_attention_weight_min": -32.97393035888672,
      "activations/layer12_attention_weight_max": 25.327533721923828,
      "activations/layer12_attention_weight_min": -24.602453231811523,
      "activations/layer13_attention_weight_max": 39.25957489013672,
      "activations/layer13_attention_weight_min": -35.532989501953125,
      "activations/layer14_attention_weight_max": 47.96907424926758,
      "activations/layer14_attention_weight_min": -39.72463607788086,
      "activations/layer15_attention_weight_max": 37.122772216796875,
      "activations/layer15_attention_weight_min": -32.94261169433594,
      "activations/layer16_attention_weight_max": 30.828588485717773,
      "activations/layer16_attention_weight_min": -28.553482055664062,
      "activations/layer17_attention_weight_max": 33.86915969848633,
      "activations/layer17_attention_weight_min": -26.0396785736084,
      "activations/layer18_attention_weight_max": 31.925809860229492,
      "activations/layer18_attention_weight_min": -22.709617614746094,
      "activations/layer19_attention_weight_max": 34.830406188964844,
      "activations/layer19_attention_weight_min": -30.86725425720215,
      "activations/layer1_attention_weight_max": 16.629390716552734,
      "activations/layer1_attention_weight_min": -14.868556022644043,
      "activations/layer20_attention_weight_max": 28.676956176757812,
      "activations/layer20_attention_weight_min": -25.846839904785156,
      "activations/layer21_attention_weight_max": 27.8206844329834,
      "activations/layer21_attention_weight_min": -24.313186645507812,
      "activations/layer22_attention_weight_max": 41.37607955932617,
      "activations/layer22_attention_weight_min": -30.301044464111328,
      "activations/layer23_attention_weight_max": 33.86121368408203,
      "activations/layer23_attention_weight_min": -23.99847984313965,
      "activations/layer2_attention_weight_max": 31.820796966552734,
      "activations/layer2_attention_weight_min": -33.32293701171875,
      "activations/layer3_attention_weight_max": 94.43492126464844,
      "activations/layer3_attention_weight_min": -103.41759490966797,
      "activations/layer4_attention_weight_max": 111.06206512451172,
      "activations/layer4_attention_weight_min": -120.48802947998047,
      "activations/layer5_attention_weight_max": 50.69839096069336,
      "activations/layer5_attention_weight_min": -68.47372436523438,
      "activations/layer6_attention_weight_max": 45.35890579223633,
      "activations/layer6_attention_weight_min": -50.756168365478516,
      "activations/layer7_attention_weight_max": 91.00946044921875,
      "activations/layer7_attention_weight_min": -96.82720947265625,
      "activations/layer8_attention_weight_max": 40.10835266113281,
      "activations/layer8_attention_weight_min": -43.5931396484375,
      "activations/layer9_attention_weight_max": 35.0833854675293,
      "activations/layer9_attention_weight_min": -34.68387222290039,
      "epoch": 11.52,
      "learning_rate": 7.648484848484847e-05,
      "loss": 2.7997,
      "step": 198250
    },
    {
      "activations/layer0_attention_weight_max": 15.253854751586914,
      "activations/layer0_attention_weight_min": -13.291245460510254,
      "activations/layer10_attention_weight_max": 33.86920928955078,
      "activations/layer10_attention_weight_min": -33.35700225830078,
      "activations/layer11_attention_weight_max": 32.47879409790039,
      "activations/layer11_attention_weight_min": -31.572782516479492,
      "activations/layer12_attention_weight_max": 27.23632049560547,
      "activations/layer12_attention_weight_min": -25.3896484375,
      "activations/layer13_attention_weight_max": 42.72868728637695,
      "activations/layer13_attention_weight_min": -36.0789680480957,
      "activations/layer14_attention_weight_max": 48.049495697021484,
      "activations/layer14_attention_weight_min": -41.323028564453125,
      "activations/layer15_attention_weight_max": 38.65563201904297,
      "activations/layer15_attention_weight_min": -32.802703857421875,
      "activations/layer16_attention_weight_max": 33.502681732177734,
      "activations/layer16_attention_weight_min": -27.970232009887695,
      "activations/layer17_attention_weight_max": 31.40045166015625,
      "activations/layer17_attention_weight_min": -25.442134857177734,
      "activations/layer18_attention_weight_max": 29.690786361694336,
      "activations/layer18_attention_weight_min": -24.81669044494629,
      "activations/layer19_attention_weight_max": 35.03010177612305,
      "activations/layer19_attention_weight_min": -30.30511474609375,
      "activations/layer1_attention_weight_max": 17.22494888305664,
      "activations/layer1_attention_weight_min": -14.706446647644043,
      "activations/layer20_attention_weight_max": 27.18950653076172,
      "activations/layer20_attention_weight_min": -24.4411678314209,
      "activations/layer21_attention_weight_max": 27.332841873168945,
      "activations/layer21_attention_weight_min": -24.760387420654297,
      "activations/layer22_attention_weight_max": 47.71077346801758,
      "activations/layer22_attention_weight_min": -30.0808162689209,
      "activations/layer23_attention_weight_max": 34.52558135986328,
      "activations/layer23_attention_weight_min": -25.701709747314453,
      "activations/layer2_attention_weight_max": 31.789073944091797,
      "activations/layer2_attention_weight_min": -32.539161682128906,
      "activations/layer3_attention_weight_max": 91.9567642211914,
      "activations/layer3_attention_weight_min": -92.78768157958984,
      "activations/layer4_attention_weight_max": 114.874267578125,
      "activations/layer4_attention_weight_min": -115.74041748046875,
      "activations/layer5_attention_weight_max": 51.04761505126953,
      "activations/layer5_attention_weight_min": -62.497230529785156,
      "activations/layer6_attention_weight_max": 43.33200454711914,
      "activations/layer6_attention_weight_min": -52.672847747802734,
      "activations/layer7_attention_weight_max": 89.86770629882812,
      "activations/layer7_attention_weight_min": -97.73113250732422,
      "activations/layer8_attention_weight_max": 40.144832611083984,
      "activations/layer8_attention_weight_min": -42.91093444824219,
      "activations/layer9_attention_weight_max": 32.98125076293945,
      "activations/layer9_attention_weight_min": -33.45692443847656,
      "epoch": 11.52,
      "learning_rate": 7.646590909090909e-05,
      "loss": 2.7968,
      "step": 198300
    },
    {
      "activations/layer0_attention_weight_max": 15.847726821899414,
      "activations/layer0_attention_weight_min": -12.46931266784668,
      "activations/layer10_attention_weight_max": 30.220590591430664,
      "activations/layer10_attention_weight_min": -30.11128807067871,
      "activations/layer11_attention_weight_max": 31.168821334838867,
      "activations/layer11_attention_weight_min": -31.038785934448242,
      "activations/layer12_attention_weight_max": 22.28008460998535,
      "activations/layer12_attention_weight_min": -24.578304290771484,
      "activations/layer13_attention_weight_max": 36.732154846191406,
      "activations/layer13_attention_weight_min": -33.06288528442383,
      "activations/layer14_attention_weight_max": 39.55653762817383,
      "activations/layer14_attention_weight_min": -36.367305755615234,
      "activations/layer15_attention_weight_max": 33.561649322509766,
      "activations/layer15_attention_weight_min": -31.980884552001953,
      "activations/layer16_attention_weight_max": 27.26881217956543,
      "activations/layer16_attention_weight_min": -28.712509155273438,
      "activations/layer17_attention_weight_max": 28.99985694885254,
      "activations/layer17_attention_weight_min": -26.31355094909668,
      "activations/layer18_attention_weight_max": 29.012136459350586,
      "activations/layer18_attention_weight_min": -23.7766056060791,
      "activations/layer19_attention_weight_max": 30.11032485961914,
      "activations/layer19_attention_weight_min": -28.83536720275879,
      "activations/layer1_attention_weight_max": 16.885265350341797,
      "activations/layer1_attention_weight_min": -15.810535430908203,
      "activations/layer20_attention_weight_max": 25.487253189086914,
      "activations/layer20_attention_weight_min": -24.655925750732422,
      "activations/layer21_attention_weight_max": 25.277456283569336,
      "activations/layer21_attention_weight_min": -25.073789596557617,
      "activations/layer22_attention_weight_max": 37.275535583496094,
      "activations/layer22_attention_weight_min": -30.528934478759766,
      "activations/layer23_attention_weight_max": 30.48895835876465,
      "activations/layer23_attention_weight_min": -26.304479598999023,
      "activations/layer2_attention_weight_max": 33.29695129394531,
      "activations/layer2_attention_weight_min": -34.2125129699707,
      "activations/layer3_attention_weight_max": 97.10758972167969,
      "activations/layer3_attention_weight_min": -92.9709701538086,
      "activations/layer4_attention_weight_max": 110.73209381103516,
      "activations/layer4_attention_weight_min": -114.99382781982422,
      "activations/layer5_attention_weight_max": 49.299068450927734,
      "activations/layer5_attention_weight_min": -61.22667694091797,
      "activations/layer6_attention_weight_max": 42.36030197143555,
      "activations/layer6_attention_weight_min": -46.831398010253906,
      "activations/layer7_attention_weight_max": 84.05012512207031,
      "activations/layer7_attention_weight_min": -85.79923248291016,
      "activations/layer8_attention_weight_max": 35.960453033447266,
      "activations/layer8_attention_weight_min": -38.16069793701172,
      "activations/layer9_attention_weight_max": 31.480676651000977,
      "activations/layer9_attention_weight_min": -31.902393341064453,
      "epoch": 11.53,
      "learning_rate": 7.64469696969697e-05,
      "loss": 2.7581,
      "step": 198350
    },
    {
      "activations/layer0_attention_weight_max": 15.345087051391602,
      "activations/layer0_attention_weight_min": -13.099836349487305,
      "activations/layer10_attention_weight_max": 32.04286193847656,
      "activations/layer10_attention_weight_min": -31.149032592773438,
      "activations/layer11_attention_weight_max": 32.66756057739258,
      "activations/layer11_attention_weight_min": -33.02796173095703,
      "activations/layer12_attention_weight_max": 24.433246612548828,
      "activations/layer12_attention_weight_min": -26.262592315673828,
      "activations/layer13_attention_weight_max": 38.95765686035156,
      "activations/layer13_attention_weight_min": -37.654388427734375,
      "activations/layer14_attention_weight_max": 45.974205017089844,
      "activations/layer14_attention_weight_min": -39.228477478027344,
      "activations/layer15_attention_weight_max": 36.81826400756836,
      "activations/layer15_attention_weight_min": -34.08016586303711,
      "activations/layer16_attention_weight_max": 31.01154327392578,
      "activations/layer16_attention_weight_min": -32.24267578125,
      "activations/layer17_attention_weight_max": 30.98626708984375,
      "activations/layer17_attention_weight_min": -29.35915184020996,
      "activations/layer18_attention_weight_max": 34.35953140258789,
      "activations/layer18_attention_weight_min": -25.28278350830078,
      "activations/layer19_attention_weight_max": 33.50606155395508,
      "activations/layer19_attention_weight_min": -32.363460540771484,
      "activations/layer1_attention_weight_max": 17.055986404418945,
      "activations/layer1_attention_weight_min": -14.39150333404541,
      "activations/layer20_attention_weight_max": 30.039382934570312,
      "activations/layer20_attention_weight_min": -24.86391830444336,
      "activations/layer21_attention_weight_max": 28.967857360839844,
      "activations/layer21_attention_weight_min": -23.828372955322266,
      "activations/layer22_attention_weight_max": 43.79216766357422,
      "activations/layer22_attention_weight_min": -31.209985733032227,
      "activations/layer23_attention_weight_max": 32.46434783935547,
      "activations/layer23_attention_weight_min": -25.109052658081055,
      "activations/layer2_attention_weight_max": 32.333763122558594,
      "activations/layer2_attention_weight_min": -33.085227966308594,
      "activations/layer3_attention_weight_max": 90.58538818359375,
      "activations/layer3_attention_weight_min": -95.43466186523438,
      "activations/layer4_attention_weight_max": 110.54206848144531,
      "activations/layer4_attention_weight_min": -114.51704406738281,
      "activations/layer5_attention_weight_max": 55.1004638671875,
      "activations/layer5_attention_weight_min": -62.439720153808594,
      "activations/layer6_attention_weight_max": 43.31467819213867,
      "activations/layer6_attention_weight_min": -48.00638961791992,
      "activations/layer7_attention_weight_max": 88.29002380371094,
      "activations/layer7_attention_weight_min": -90.30485534667969,
      "activations/layer8_attention_weight_max": 41.0496711730957,
      "activations/layer8_attention_weight_min": -41.542236328125,
      "activations/layer9_attention_weight_max": 32.8486328125,
      "activations/layer9_attention_weight_min": -33.263023376464844,
      "epoch": 11.53,
      "learning_rate": 7.642803030303029e-05,
      "loss": 2.7774,
      "step": 198400
    },
    {
      "activations/layer0_attention_weight_max": 15.001628875732422,
      "activations/layer0_attention_weight_min": -12.53323745727539,
      "activations/layer10_attention_weight_max": 35.76043701171875,
      "activations/layer10_attention_weight_min": -33.27296829223633,
      "activations/layer11_attention_weight_max": 39.59209060668945,
      "activations/layer11_attention_weight_min": -33.92532730102539,
      "activations/layer12_attention_weight_max": 25.924528121948242,
      "activations/layer12_attention_weight_min": -24.103055953979492,
      "activations/layer13_attention_weight_max": 38.541927337646484,
      "activations/layer13_attention_weight_min": -33.075035095214844,
      "activations/layer14_attention_weight_max": 44.77611541748047,
      "activations/layer14_attention_weight_min": -37.91792678833008,
      "activations/layer15_attention_weight_max": 39.87146759033203,
      "activations/layer15_attention_weight_min": -33.9691276550293,
      "activations/layer16_attention_weight_max": 31.838666915893555,
      "activations/layer16_attention_weight_min": -31.545690536499023,
      "activations/layer17_attention_weight_max": 36.19503402709961,
      "activations/layer17_attention_weight_min": -28.73357582092285,
      "activations/layer18_attention_weight_max": 29.546527862548828,
      "activations/layer18_attention_weight_min": -22.507959365844727,
      "activations/layer19_attention_weight_max": 33.75993728637695,
      "activations/layer19_attention_weight_min": -30.0518798828125,
      "activations/layer1_attention_weight_max": 16.993581771850586,
      "activations/layer1_attention_weight_min": -13.813641548156738,
      "activations/layer20_attention_weight_max": 30.465442657470703,
      "activations/layer20_attention_weight_min": -23.73661231994629,
      "activations/layer21_attention_weight_max": 29.479862213134766,
      "activations/layer21_attention_weight_min": -24.58598518371582,
      "activations/layer22_attention_weight_max": 43.772274017333984,
      "activations/layer22_attention_weight_min": -27.72261619567871,
      "activations/layer23_attention_weight_max": 34.10918045043945,
      "activations/layer23_attention_weight_min": -24.169864654541016,
      "activations/layer2_attention_weight_max": 32.16087341308594,
      "activations/layer2_attention_weight_min": -33.32453536987305,
      "activations/layer3_attention_weight_max": 100.64472961425781,
      "activations/layer3_attention_weight_min": -98.54823303222656,
      "activations/layer4_attention_weight_max": 115.60137939453125,
      "activations/layer4_attention_weight_min": -119.3717269897461,
      "activations/layer5_attention_weight_max": 51.67525863647461,
      "activations/layer5_attention_weight_min": -63.1966667175293,
      "activations/layer6_attention_weight_max": 47.98603439331055,
      "activations/layer6_attention_weight_min": -48.278133392333984,
      "activations/layer7_attention_weight_max": 90.85199737548828,
      "activations/layer7_attention_weight_min": -92.00382995605469,
      "activations/layer8_attention_weight_max": 39.82465744018555,
      "activations/layer8_attention_weight_min": -41.23575973510742,
      "activations/layer9_attention_weight_max": 33.709373474121094,
      "activations/layer9_attention_weight_min": -32.26688003540039,
      "epoch": 11.53,
      "learning_rate": 7.64090909090909e-05,
      "loss": 2.7796,
      "step": 198450
    },
    {
      "activations/layer0_attention_weight_max": 14.749406814575195,
      "activations/layer0_attention_weight_min": -12.799474716186523,
      "activations/layer10_attention_weight_max": 33.171627044677734,
      "activations/layer10_attention_weight_min": -33.87542724609375,
      "activations/layer11_attention_weight_max": 35.403724670410156,
      "activations/layer11_attention_weight_min": -32.56761169433594,
      "activations/layer12_attention_weight_max": 24.26230239868164,
      "activations/layer12_attention_weight_min": -25.592323303222656,
      "activations/layer13_attention_weight_max": 36.726985931396484,
      "activations/layer13_attention_weight_min": -33.97265625,
      "activations/layer14_attention_weight_max": 41.43767547607422,
      "activations/layer14_attention_weight_min": -36.3720703125,
      "activations/layer15_attention_weight_max": 35.42097854614258,
      "activations/layer15_attention_weight_min": -31.115589141845703,
      "activations/layer16_attention_weight_max": 29.587814331054688,
      "activations/layer16_attention_weight_min": -27.34001922607422,
      "activations/layer17_attention_weight_max": 27.899301528930664,
      "activations/layer17_attention_weight_min": -25.42072296142578,
      "activations/layer18_attention_weight_max": 29.169921875,
      "activations/layer18_attention_weight_min": -22.785682678222656,
      "activations/layer19_attention_weight_max": 34.5460319519043,
      "activations/layer19_attention_weight_min": -29.433704376220703,
      "activations/layer1_attention_weight_max": 16.84793472290039,
      "activations/layer1_attention_weight_min": -15.53248119354248,
      "activations/layer20_attention_weight_max": 30.463151931762695,
      "activations/layer20_attention_weight_min": -25.730417251586914,
      "activations/layer21_attention_weight_max": 29.74753189086914,
      "activations/layer21_attention_weight_min": -23.44970703125,
      "activations/layer22_attention_weight_max": 40.960975646972656,
      "activations/layer22_attention_weight_min": -27.443878173828125,
      "activations/layer23_attention_weight_max": 32.22926330566406,
      "activations/layer23_attention_weight_min": -22.944875717163086,
      "activations/layer2_attention_weight_max": 34.13514709472656,
      "activations/layer2_attention_weight_min": -36.60911560058594,
      "activations/layer3_attention_weight_max": 95.81305694580078,
      "activations/layer3_attention_weight_min": -105.17633056640625,
      "activations/layer4_attention_weight_max": 110.13700103759766,
      "activations/layer4_attention_weight_min": -115.56011199951172,
      "activations/layer5_attention_weight_max": 51.306732177734375,
      "activations/layer5_attention_weight_min": -60.206886291503906,
      "activations/layer6_attention_weight_max": 44.78395462036133,
      "activations/layer6_attention_weight_min": -49.399131774902344,
      "activations/layer7_attention_weight_max": 87.66111755371094,
      "activations/layer7_attention_weight_min": -99.7662582397461,
      "activations/layer8_attention_weight_max": 40.57191848754883,
      "activations/layer8_attention_weight_min": -41.525943756103516,
      "activations/layer9_attention_weight_max": 34.74760055541992,
      "activations/layer9_attention_weight_min": -33.17609786987305,
      "epoch": 11.53,
      "learning_rate": 7.639015151515152e-05,
      "loss": 2.7842,
      "step": 198500
    },
    {
      "activations/layer0_attention_weight_max": 16.383686065673828,
      "activations/layer0_attention_weight_min": -12.59218692779541,
      "activations/layer10_attention_weight_max": 33.670318603515625,
      "activations/layer10_attention_weight_min": -35.546512603759766,
      "activations/layer11_attention_weight_max": 33.70578384399414,
      "activations/layer11_attention_weight_min": -35.36125564575195,
      "activations/layer12_attention_weight_max": 25.967594146728516,
      "activations/layer12_attention_weight_min": -26.66191291809082,
      "activations/layer13_attention_weight_max": 37.9111328125,
      "activations/layer13_attention_weight_min": -35.88932800292969,
      "activations/layer14_attention_weight_max": 47.69187545776367,
      "activations/layer14_attention_weight_min": -39.71165084838867,
      "activations/layer15_attention_weight_max": 35.829566955566406,
      "activations/layer15_attention_weight_min": -33.65856170654297,
      "activations/layer16_attention_weight_max": 33.68648910522461,
      "activations/layer16_attention_weight_min": -27.970903396606445,
      "activations/layer17_attention_weight_max": 31.14755630493164,
      "activations/layer17_attention_weight_min": -26.21982765197754,
      "activations/layer18_attention_weight_max": 29.954381942749023,
      "activations/layer18_attention_weight_min": -25.39500617980957,
      "activations/layer19_attention_weight_max": 38.45158004760742,
      "activations/layer19_attention_weight_min": -31.38425636291504,
      "activations/layer1_attention_weight_max": 16.519258499145508,
      "activations/layer1_attention_weight_min": -15.690694808959961,
      "activations/layer20_attention_weight_max": 29.836387634277344,
      "activations/layer20_attention_weight_min": -25.08803367614746,
      "activations/layer21_attention_weight_max": 28.44546890258789,
      "activations/layer21_attention_weight_min": -24.33661460876465,
      "activations/layer22_attention_weight_max": 44.763954162597656,
      "activations/layer22_attention_weight_min": -28.43675422668457,
      "activations/layer23_attention_weight_max": 37.30509948730469,
      "activations/layer23_attention_weight_min": -24.97524070739746,
      "activations/layer2_attention_weight_max": 36.26264190673828,
      "activations/layer2_attention_weight_min": -34.72459411621094,
      "activations/layer3_attention_weight_max": 101.87198638916016,
      "activations/layer3_attention_weight_min": -100.23884582519531,
      "activations/layer4_attention_weight_max": 123.23027801513672,
      "activations/layer4_attention_weight_min": -117.71569061279297,
      "activations/layer5_attention_weight_max": 55.94287872314453,
      "activations/layer5_attention_weight_min": -63.779720306396484,
      "activations/layer6_attention_weight_max": 48.35487747192383,
      "activations/layer6_attention_weight_min": -49.04804992675781,
      "activations/layer7_attention_weight_max": 93.7743911743164,
      "activations/layer7_attention_weight_min": -92.02586364746094,
      "activations/layer8_attention_weight_max": 41.33979797363281,
      "activations/layer8_attention_weight_min": -41.400726318359375,
      "activations/layer9_attention_weight_max": 35.0528678894043,
      "activations/layer9_attention_weight_min": -35.1917610168457,
      "epoch": 11.54,
      "learning_rate": 7.63712121212121e-05,
      "loss": 2.7769,
      "step": 198550
    },
    {
      "activations/layer0_attention_weight_max": 16.40708351135254,
      "activations/layer0_attention_weight_min": -12.047872543334961,
      "activations/layer10_attention_weight_max": 31.790157318115234,
      "activations/layer10_attention_weight_min": -31.317955017089844,
      "activations/layer11_attention_weight_max": 33.058834075927734,
      "activations/layer11_attention_weight_min": -31.167858123779297,
      "activations/layer12_attention_weight_max": 47.04806900024414,
      "activations/layer12_attention_weight_min": -26.458951950073242,
      "activations/layer13_attention_weight_max": 52.23918914794922,
      "activations/layer13_attention_weight_min": -32.74433898925781,
      "activations/layer14_attention_weight_max": 53.910152435302734,
      "activations/layer14_attention_weight_min": -38.715824127197266,
      "activations/layer15_attention_weight_max": 40.57771682739258,
      "activations/layer15_attention_weight_min": -34.15156173706055,
      "activations/layer16_attention_weight_max": 38.33548355102539,
      "activations/layer16_attention_weight_min": -30.705814361572266,
      "activations/layer17_attention_weight_max": 36.77401351928711,
      "activations/layer17_attention_weight_min": -26.682960510253906,
      "activations/layer18_attention_weight_max": 37.2340202331543,
      "activations/layer18_attention_weight_min": -24.99339485168457,
      "activations/layer19_attention_weight_max": 44.432552337646484,
      "activations/layer19_attention_weight_min": -31.178787231445312,
      "activations/layer1_attention_weight_max": 18.786867141723633,
      "activations/layer1_attention_weight_min": -16.443437576293945,
      "activations/layer20_attention_weight_max": 37.37260437011719,
      "activations/layer20_attention_weight_min": -23.50096893310547,
      "activations/layer21_attention_weight_max": 35.107879638671875,
      "activations/layer21_attention_weight_min": -22.830644607543945,
      "activations/layer22_attention_weight_max": 62.76615524291992,
      "activations/layer22_attention_weight_min": -32.23984909057617,
      "activations/layer23_attention_weight_max": 44.10011291503906,
      "activations/layer23_attention_weight_min": -25.27021598815918,
      "activations/layer2_attention_weight_max": 35.503047943115234,
      "activations/layer2_attention_weight_min": -34.18255615234375,
      "activations/layer3_attention_weight_max": 103.102783203125,
      "activations/layer3_attention_weight_min": -101.44771575927734,
      "activations/layer4_attention_weight_max": 112.53797912597656,
      "activations/layer4_attention_weight_min": -111.4404525756836,
      "activations/layer5_attention_weight_max": 52.701351165771484,
      "activations/layer5_attention_weight_min": -61.83984375,
      "activations/layer6_attention_weight_max": 47.81659698486328,
      "activations/layer6_attention_weight_min": -47.52048110961914,
      "activations/layer7_attention_weight_max": 87.08723449707031,
      "activations/layer7_attention_weight_min": -87.08087921142578,
      "activations/layer8_attention_weight_max": 39.43061065673828,
      "activations/layer8_attention_weight_min": -40.12800979614258,
      "activations/layer9_attention_weight_max": 31.478330612182617,
      "activations/layer9_attention_weight_min": -32.585079193115234,
      "epoch": 11.54,
      "learning_rate": 7.635227272727272e-05,
      "loss": 2.7769,
      "step": 198600
    },
    {
      "activations/layer0_attention_weight_max": 15.281168937683105,
      "activations/layer0_attention_weight_min": -12.075404167175293,
      "activations/layer10_attention_weight_max": 30.500478744506836,
      "activations/layer10_attention_weight_min": -31.078506469726562,
      "activations/layer11_attention_weight_max": 30.643522262573242,
      "activations/layer11_attention_weight_min": -31.360450744628906,
      "activations/layer12_attention_weight_max": 23.874515533447266,
      "activations/layer12_attention_weight_min": -25.16380500793457,
      "activations/layer13_attention_weight_max": 36.143829345703125,
      "activations/layer13_attention_weight_min": -33.12955856323242,
      "activations/layer14_attention_weight_max": 42.37696838378906,
      "activations/layer14_attention_weight_min": -35.576263427734375,
      "activations/layer15_attention_weight_max": 34.83454132080078,
      "activations/layer15_attention_weight_min": -31.03860092163086,
      "activations/layer16_attention_weight_max": 29.82609748840332,
      "activations/layer16_attention_weight_min": -26.97882652282715,
      "activations/layer17_attention_weight_max": 29.30116844177246,
      "activations/layer17_attention_weight_min": -24.480863571166992,
      "activations/layer18_attention_weight_max": 35.81420135498047,
      "activations/layer18_attention_weight_min": -21.69269371032715,
      "activations/layer19_attention_weight_max": 34.85410690307617,
      "activations/layer19_attention_weight_min": -31.437162399291992,
      "activations/layer1_attention_weight_max": 16.315603256225586,
      "activations/layer1_attention_weight_min": -14.745360374450684,
      "activations/layer20_attention_weight_max": 27.70506477355957,
      "activations/layer20_attention_weight_min": -24.794755935668945,
      "activations/layer21_attention_weight_max": 33.31376266479492,
      "activations/layer21_attention_weight_min": -24.465709686279297,
      "activations/layer22_attention_weight_max": 42.27397537231445,
      "activations/layer22_attention_weight_min": -31.48671531677246,
      "activations/layer23_attention_weight_max": 33.63706588745117,
      "activations/layer23_attention_weight_min": -25.536123275756836,
      "activations/layer2_attention_weight_max": 33.17804718017578,
      "activations/layer2_attention_weight_min": -32.8388671875,
      "activations/layer3_attention_weight_max": 97.24736785888672,
      "activations/layer3_attention_weight_min": -100.89051055908203,
      "activations/layer4_attention_weight_max": 114.1690444946289,
      "activations/layer4_attention_weight_min": -114.5337905883789,
      "activations/layer5_attention_weight_max": 53.343284606933594,
      "activations/layer5_attention_weight_min": -61.59380340576172,
      "activations/layer6_attention_weight_max": 44.75304412841797,
      "activations/layer6_attention_weight_min": -48.2307243347168,
      "activations/layer7_attention_weight_max": 88.91512298583984,
      "activations/layer7_attention_weight_min": -88.12924194335938,
      "activations/layer8_attention_weight_max": 39.72554397583008,
      "activations/layer8_attention_weight_min": -40.03596878051758,
      "activations/layer9_attention_weight_max": 30.691247940063477,
      "activations/layer9_attention_weight_min": -31.004653930664062,
      "epoch": 11.54,
      "learning_rate": 7.633333333333332e-05,
      "loss": 2.7856,
      "step": 198650
    },
    {
      "activations/layer0_attention_weight_max": 15.957581520080566,
      "activations/layer0_attention_weight_min": -12.009811401367188,
      "activations/layer10_attention_weight_max": 39.36025619506836,
      "activations/layer10_attention_weight_min": -33.52412796020508,
      "activations/layer11_attention_weight_max": 39.90039825439453,
      "activations/layer11_attention_weight_min": -35.22929000854492,
      "activations/layer12_attention_weight_max": 33.42166519165039,
      "activations/layer12_attention_weight_min": -26.588279724121094,
      "activations/layer13_attention_weight_max": 54.0125617980957,
      "activations/layer13_attention_weight_min": -34.70853805541992,
      "activations/layer14_attention_weight_max": 45.304996490478516,
      "activations/layer14_attention_weight_min": -38.5330924987793,
      "activations/layer15_attention_weight_max": 39.90728759765625,
      "activations/layer15_attention_weight_min": -34.23909378051758,
      "activations/layer16_attention_weight_max": 30.717378616333008,
      "activations/layer16_attention_weight_min": -34.044525146484375,
      "activations/layer17_attention_weight_max": 30.965723037719727,
      "activations/layer17_attention_weight_min": -26.196950912475586,
      "activations/layer18_attention_weight_max": 32.51996994018555,
      "activations/layer18_attention_weight_min": -24.44977569580078,
      "activations/layer19_attention_weight_max": 31.982624053955078,
      "activations/layer19_attention_weight_min": -32.50832748413086,
      "activations/layer1_attention_weight_max": 16.54069709777832,
      "activations/layer1_attention_weight_min": -15.720589637756348,
      "activations/layer20_attention_weight_max": 26.714794158935547,
      "activations/layer20_attention_weight_min": -25.69972801208496,
      "activations/layer21_attention_weight_max": 29.502426147460938,
      "activations/layer21_attention_weight_min": -23.710763931274414,
      "activations/layer22_attention_weight_max": 43.65813064575195,
      "activations/layer22_attention_weight_min": -30.849472045898438,
      "activations/layer23_attention_weight_max": 31.920021057128906,
      "activations/layer23_attention_weight_min": -25.423847198486328,
      "activations/layer2_attention_weight_max": 33.891746520996094,
      "activations/layer2_attention_weight_min": -34.244197845458984,
      "activations/layer3_attention_weight_max": 102.126220703125,
      "activations/layer3_attention_weight_min": -104.63439178466797,
      "activations/layer4_attention_weight_max": 113.29647064208984,
      "activations/layer4_attention_weight_min": -113.71650695800781,
      "activations/layer5_attention_weight_max": 54.4937744140625,
      "activations/layer5_attention_weight_min": -60.036293029785156,
      "activations/layer6_attention_weight_max": 44.41659927368164,
      "activations/layer6_attention_weight_min": -47.61841583251953,
      "activations/layer7_attention_weight_max": 90.40859985351562,
      "activations/layer7_attention_weight_min": -89.68000030517578,
      "activations/layer8_attention_weight_max": 40.430755615234375,
      "activations/layer8_attention_weight_min": -42.264366149902344,
      "activations/layer9_attention_weight_max": 37.34083938598633,
      "activations/layer9_attention_weight_min": -35.17289352416992,
      "epoch": 11.55,
      "learning_rate": 7.631439393939392e-05,
      "loss": 2.7548,
      "step": 198700
    },
    {
      "activations/layer0_attention_weight_max": 15.818315505981445,
      "activations/layer0_attention_weight_min": -12.339556694030762,
      "activations/layer10_attention_weight_max": 33.20710754394531,
      "activations/layer10_attention_weight_min": -34.359466552734375,
      "activations/layer11_attention_weight_max": 36.49374008178711,
      "activations/layer11_attention_weight_min": -33.57936096191406,
      "activations/layer12_attention_weight_max": 34.3709831237793,
      "activations/layer12_attention_weight_min": -27.01411247253418,
      "activations/layer13_attention_weight_max": 50.30270767211914,
      "activations/layer13_attention_weight_min": -36.026466369628906,
      "activations/layer14_attention_weight_max": 60.92599105834961,
      "activations/layer14_attention_weight_min": -41.54121780395508,
      "activations/layer15_attention_weight_max": 45.65644836425781,
      "activations/layer15_attention_weight_min": -34.60410690307617,
      "activations/layer16_attention_weight_max": 42.801551818847656,
      "activations/layer16_attention_weight_min": -30.276302337646484,
      "activations/layer17_attention_weight_max": 38.33473205566406,
      "activations/layer17_attention_weight_min": -27.645936965942383,
      "activations/layer18_attention_weight_max": 36.674400329589844,
      "activations/layer18_attention_weight_min": -24.448650360107422,
      "activations/layer19_attention_weight_max": 47.847347259521484,
      "activations/layer19_attention_weight_min": -32.860347747802734,
      "activations/layer1_attention_weight_max": 17.3906307220459,
      "activations/layer1_attention_weight_min": -15.479544639587402,
      "activations/layer20_attention_weight_max": 43.06383514404297,
      "activations/layer20_attention_weight_min": -25.58953857421875,
      "activations/layer21_attention_weight_max": 41.22432327270508,
      "activations/layer21_attention_weight_min": -25.506242752075195,
      "activations/layer22_attention_weight_max": 68.62667846679688,
      "activations/layer22_attention_weight_min": -32.343387603759766,
      "activations/layer23_attention_weight_max": 48.02082061767578,
      "activations/layer23_attention_weight_min": -25.522581100463867,
      "activations/layer2_attention_weight_max": 34.44873809814453,
      "activations/layer2_attention_weight_min": -34.369380950927734,
      "activations/layer3_attention_weight_max": 99.7899169921875,
      "activations/layer3_attention_weight_min": -104.19827270507812,
      "activations/layer4_attention_weight_max": 116.7115707397461,
      "activations/layer4_attention_weight_min": -119.69359588623047,
      "activations/layer5_attention_weight_max": 51.92076873779297,
      "activations/layer5_attention_weight_min": -63.024574279785156,
      "activations/layer6_attention_weight_max": 44.823543548583984,
      "activations/layer6_attention_weight_min": -47.42251205444336,
      "activations/layer7_attention_weight_max": 87.26725006103516,
      "activations/layer7_attention_weight_min": -96.91421508789062,
      "activations/layer8_attention_weight_max": 39.9796257019043,
      "activations/layer8_attention_weight_min": -45.044677734375,
      "activations/layer9_attention_weight_max": 33.604248046875,
      "activations/layer9_attention_weight_min": -36.11869430541992,
      "epoch": 11.55,
      "learning_rate": 7.629545454545454e-05,
      "loss": 2.783,
      "step": 198750
    },
    {
      "activations/layer0_attention_weight_max": 15.020857810974121,
      "activations/layer0_attention_weight_min": -12.810873031616211,
      "activations/layer10_attention_weight_max": 33.57223129272461,
      "activations/layer10_attention_weight_min": -32.55226516723633,
      "activations/layer11_attention_weight_max": 36.030540466308594,
      "activations/layer11_attention_weight_min": -32.88432312011719,
      "activations/layer12_attention_weight_max": 26.632360458374023,
      "activations/layer12_attention_weight_min": -26.4298038482666,
      "activations/layer13_attention_weight_max": 40.60307312011719,
      "activations/layer13_attention_weight_min": -36.514686584472656,
      "activations/layer14_attention_weight_max": 44.1616096496582,
      "activations/layer14_attention_weight_min": -38.633480072021484,
      "activations/layer15_attention_weight_max": 36.68353271484375,
      "activations/layer15_attention_weight_min": -34.780982971191406,
      "activations/layer16_attention_weight_max": 31.111413955688477,
      "activations/layer16_attention_weight_min": -29.67064666748047,
      "activations/layer17_attention_weight_max": 33.611480712890625,
      "activations/layer17_attention_weight_min": -26.94149398803711,
      "activations/layer18_attention_weight_max": 30.50897216796875,
      "activations/layer18_attention_weight_min": -26.02958869934082,
      "activations/layer19_attention_weight_max": 36.58344650268555,
      "activations/layer19_attention_weight_min": -31.385467529296875,
      "activations/layer1_attention_weight_max": 16.500398635864258,
      "activations/layer1_attention_weight_min": -15.21936321258545,
      "activations/layer20_attention_weight_max": 29.16441535949707,
      "activations/layer20_attention_weight_min": -25.09893035888672,
      "activations/layer21_attention_weight_max": 30.388015747070312,
      "activations/layer21_attention_weight_min": -24.95458221435547,
      "activations/layer22_attention_weight_max": 45.4334831237793,
      "activations/layer22_attention_weight_min": -32.28972244262695,
      "activations/layer23_attention_weight_max": 34.022727966308594,
      "activations/layer23_attention_weight_min": -27.140281677246094,
      "activations/layer2_attention_weight_max": 33.37881851196289,
      "activations/layer2_attention_weight_min": -32.48626708984375,
      "activations/layer3_attention_weight_max": 97.8587875366211,
      "activations/layer3_attention_weight_min": -98.7199935913086,
      "activations/layer4_attention_weight_max": 115.70027160644531,
      "activations/layer4_attention_weight_min": -116.36927795410156,
      "activations/layer5_attention_weight_max": 51.89997863769531,
      "activations/layer5_attention_weight_min": -63.861854553222656,
      "activations/layer6_attention_weight_max": 45.66459655761719,
      "activations/layer6_attention_weight_min": -48.69200897216797,
      "activations/layer7_attention_weight_max": 90.31684875488281,
      "activations/layer7_attention_weight_min": -91.58267974853516,
      "activations/layer8_attention_weight_max": 39.74359893798828,
      "activations/layer8_attention_weight_min": -41.747920989990234,
      "activations/layer9_attention_weight_max": 35.174015045166016,
      "activations/layer9_attention_weight_min": -34.15623474121094,
      "epoch": 11.55,
      "learning_rate": 7.627651515151514e-05,
      "loss": 2.7775,
      "step": 198800
    },
    {
      "activations/layer0_attention_weight_max": 16.454429626464844,
      "activations/layer0_attention_weight_min": -12.532824516296387,
      "activations/layer10_attention_weight_max": 32.252296447753906,
      "activations/layer10_attention_weight_min": -33.61651611328125,
      "activations/layer11_attention_weight_max": 31.30984878540039,
      "activations/layer11_attention_weight_min": -31.459684371948242,
      "activations/layer12_attention_weight_max": 27.611488342285156,
      "activations/layer12_attention_weight_min": -23.861106872558594,
      "activations/layer13_attention_weight_max": 34.719303131103516,
      "activations/layer13_attention_weight_min": -33.380828857421875,
      "activations/layer14_attention_weight_max": 38.61979293823242,
      "activations/layer14_attention_weight_min": -33.59641647338867,
      "activations/layer15_attention_weight_max": 31.67742919921875,
      "activations/layer15_attention_weight_min": -31.700849533081055,
      "activations/layer16_attention_weight_max": 27.38374900817871,
      "activations/layer16_attention_weight_min": -27.709518432617188,
      "activations/layer17_attention_weight_max": 28.816158294677734,
      "activations/layer17_attention_weight_min": -24.749814987182617,
      "activations/layer18_attention_weight_max": 28.7037410736084,
      "activations/layer18_attention_weight_min": -22.548185348510742,
      "activations/layer19_attention_weight_max": 33.310359954833984,
      "activations/layer19_attention_weight_min": -29.60017967224121,
      "activations/layer1_attention_weight_max": 17.190641403198242,
      "activations/layer1_attention_weight_min": -14.215579986572266,
      "activations/layer20_attention_weight_max": 26.260786056518555,
      "activations/layer20_attention_weight_min": -23.886343002319336,
      "activations/layer21_attention_weight_max": 27.99875831604004,
      "activations/layer21_attention_weight_min": -23.37361717224121,
      "activations/layer22_attention_weight_max": 41.489437103271484,
      "activations/layer22_attention_weight_min": -29.43899154663086,
      "activations/layer23_attention_weight_max": 36.060916900634766,
      "activations/layer23_attention_weight_min": -25.32191276550293,
      "activations/layer2_attention_weight_max": 33.85358810424805,
      "activations/layer2_attention_weight_min": -32.69104766845703,
      "activations/layer3_attention_weight_max": 98.90579986572266,
      "activations/layer3_attention_weight_min": -100.03519439697266,
      "activations/layer4_attention_weight_max": 117.4448471069336,
      "activations/layer4_attention_weight_min": -117.76824188232422,
      "activations/layer5_attention_weight_max": 51.362937927246094,
      "activations/layer5_attention_weight_min": -63.95697021484375,
      "activations/layer6_attention_weight_max": 41.50751495361328,
      "activations/layer6_attention_weight_min": -46.333011627197266,
      "activations/layer7_attention_weight_max": 89.094970703125,
      "activations/layer7_attention_weight_min": -86.5051040649414,
      "activations/layer8_attention_weight_max": 38.32670974731445,
      "activations/layer8_attention_weight_min": -40.1314697265625,
      "activations/layer9_attention_weight_max": 32.62325668334961,
      "activations/layer9_attention_weight_min": -32.92814254760742,
      "epoch": 11.55,
      "learning_rate": 7.625757575757576e-05,
      "loss": 2.7818,
      "step": 198850
    },
    {
      "activations/layer0_attention_weight_max": 16.126232147216797,
      "activations/layer0_attention_weight_min": -12.271065711975098,
      "activations/layer10_attention_weight_max": 32.596900939941406,
      "activations/layer10_attention_weight_min": -37.167274475097656,
      "activations/layer11_attention_weight_max": 33.003944396972656,
      "activations/layer11_attention_weight_min": -34.77406311035156,
      "activations/layer12_attention_weight_max": 23.909616470336914,
      "activations/layer12_attention_weight_min": -25.337865829467773,
      "activations/layer13_attention_weight_max": 34.687801361083984,
      "activations/layer13_attention_weight_min": -32.957889556884766,
      "activations/layer14_attention_weight_max": 42.33116912841797,
      "activations/layer14_attention_weight_min": -38.305057525634766,
      "activations/layer15_attention_weight_max": 39.064388275146484,
      "activations/layer15_attention_weight_min": -32.148643493652344,
      "activations/layer16_attention_weight_max": 29.809494018554688,
      "activations/layer16_attention_weight_min": -28.79499626159668,
      "activations/layer17_attention_weight_max": 28.734724044799805,
      "activations/layer17_attention_weight_min": -25.7215518951416,
      "activations/layer18_attention_weight_max": 30.001529693603516,
      "activations/layer18_attention_weight_min": -23.075206756591797,
      "activations/layer19_attention_weight_max": 34.420345306396484,
      "activations/layer19_attention_weight_min": -30.795413970947266,
      "activations/layer1_attention_weight_max": 16.475374221801758,
      "activations/layer1_attention_weight_min": -16.949037551879883,
      "activations/layer20_attention_weight_max": 29.879619598388672,
      "activations/layer20_attention_weight_min": -24.565505981445312,
      "activations/layer21_attention_weight_max": 29.273345947265625,
      "activations/layer21_attention_weight_min": -24.35556411743164,
      "activations/layer22_attention_weight_max": 41.3219108581543,
      "activations/layer22_attention_weight_min": -31.745601654052734,
      "activations/layer23_attention_weight_max": 41.68696594238281,
      "activations/layer23_attention_weight_min": -29.501670837402344,
      "activations/layer2_attention_weight_max": 36.288700103759766,
      "activations/layer2_attention_weight_min": -36.31901931762695,
      "activations/layer3_attention_weight_max": 101.45938110351562,
      "activations/layer3_attention_weight_min": -108.24158477783203,
      "activations/layer4_attention_weight_max": 116.94083404541016,
      "activations/layer4_attention_weight_min": -121.43348693847656,
      "activations/layer5_attention_weight_max": 55.311317443847656,
      "activations/layer5_attention_weight_min": -63.33478546142578,
      "activations/layer6_attention_weight_max": 45.35271453857422,
      "activations/layer6_attention_weight_min": -49.57503890991211,
      "activations/layer7_attention_weight_max": 92.87916564941406,
      "activations/layer7_attention_weight_min": -101.83985900878906,
      "activations/layer8_attention_weight_max": 42.54304504394531,
      "activations/layer8_attention_weight_min": -42.99254608154297,
      "activations/layer9_attention_weight_max": 33.17695999145508,
      "activations/layer9_attention_weight_min": -35.07832336425781,
      "epoch": 11.56,
      "learning_rate": 7.623863636363636e-05,
      "loss": 2.7731,
      "step": 198900
    },
    {
      "activations/layer0_attention_weight_max": 16.20285415649414,
      "activations/layer0_attention_weight_min": -12.559063911437988,
      "activations/layer10_attention_weight_max": 33.10359191894531,
      "activations/layer10_attention_weight_min": -33.90681838989258,
      "activations/layer11_attention_weight_max": 32.12214279174805,
      "activations/layer11_attention_weight_min": -32.970550537109375,
      "activations/layer12_attention_weight_max": 25.73843002319336,
      "activations/layer12_attention_weight_min": -27.084383010864258,
      "activations/layer13_attention_weight_max": 38.43110656738281,
      "activations/layer13_attention_weight_min": -35.843990325927734,
      "activations/layer14_attention_weight_max": 50.041744232177734,
      "activations/layer14_attention_weight_min": -40.94623947143555,
      "activations/layer15_attention_weight_max": 37.44816589355469,
      "activations/layer15_attention_weight_min": -33.595035552978516,
      "activations/layer16_attention_weight_max": 31.47106170654297,
      "activations/layer16_attention_weight_min": -28.274696350097656,
      "activations/layer17_attention_weight_max": 30.385986328125,
      "activations/layer17_attention_weight_min": -25.999696731567383,
      "activations/layer18_attention_weight_max": 29.902381896972656,
      "activations/layer18_attention_weight_min": -23.56745147705078,
      "activations/layer19_attention_weight_max": 35.78007888793945,
      "activations/layer19_attention_weight_min": -31.57623863220215,
      "activations/layer1_attention_weight_max": 17.25293731689453,
      "activations/layer1_attention_weight_min": -14.276638984680176,
      "activations/layer20_attention_weight_max": 30.067964553833008,
      "activations/layer20_attention_weight_min": -25.24233055114746,
      "activations/layer21_attention_weight_max": 28.921646118164062,
      "activations/layer21_attention_weight_min": -25.800859451293945,
      "activations/layer22_attention_weight_max": 42.2764778137207,
      "activations/layer22_attention_weight_min": -32.89331817626953,
      "activations/layer23_attention_weight_max": 34.837303161621094,
      "activations/layer23_attention_weight_min": -25.647804260253906,
      "activations/layer2_attention_weight_max": 33.287025451660156,
      "activations/layer2_attention_weight_min": -32.397621154785156,
      "activations/layer3_attention_weight_max": 96.0218505859375,
      "activations/layer3_attention_weight_min": -101.05513000488281,
      "activations/layer4_attention_weight_max": 115.7271499633789,
      "activations/layer4_attention_weight_min": -115.22298431396484,
      "activations/layer5_attention_weight_max": 53.31464385986328,
      "activations/layer5_attention_weight_min": -64.4109115600586,
      "activations/layer6_attention_weight_max": 45.406192779541016,
      "activations/layer6_attention_weight_min": -48.94546127319336,
      "activations/layer7_attention_weight_max": 88.31046295166016,
      "activations/layer7_attention_weight_min": -94.09334564208984,
      "activations/layer8_attention_weight_max": 40.42076110839844,
      "activations/layer8_attention_weight_min": -43.51121139526367,
      "activations/layer9_attention_weight_max": 33.031063079833984,
      "activations/layer9_attention_weight_min": -35.27569580078125,
      "epoch": 11.56,
      "learning_rate": 7.621969696969696e-05,
      "loss": 2.7786,
      "step": 198950
    },
    {
      "activations/layer0_attention_weight_max": 15.317791938781738,
      "activations/layer0_attention_weight_min": -12.02910041809082,
      "activations/layer10_attention_weight_max": 32.660770416259766,
      "activations/layer10_attention_weight_min": -33.482975006103516,
      "activations/layer11_attention_weight_max": 32.52061080932617,
      "activations/layer11_attention_weight_min": -32.74050521850586,
      "activations/layer12_attention_weight_max": 25.435956954956055,
      "activations/layer12_attention_weight_min": -25.280359268188477,
      "activations/layer13_attention_weight_max": 41.89924240112305,
      "activations/layer13_attention_weight_min": -34.37806701660156,
      "activations/layer14_attention_weight_max": 52.385162353515625,
      "activations/layer14_attention_weight_min": -35.99809646606445,
      "activations/layer15_attention_weight_max": 35.641239166259766,
      "activations/layer15_attention_weight_min": -32.81821823120117,
      "activations/layer16_attention_weight_max": 31.06446075439453,
      "activations/layer16_attention_weight_min": -28.966812133789062,
      "activations/layer17_attention_weight_max": 30.492462158203125,
      "activations/layer17_attention_weight_min": -27.537370681762695,
      "activations/layer18_attention_weight_max": 28.961095809936523,
      "activations/layer18_attention_weight_min": -24.51128578186035,
      "activations/layer19_attention_weight_max": 33.693870544433594,
      "activations/layer19_attention_weight_min": -33.74356460571289,
      "activations/layer1_attention_weight_max": 17.171892166137695,
      "activations/layer1_attention_weight_min": -15.682912826538086,
      "activations/layer20_attention_weight_max": 30.09355354309082,
      "activations/layer20_attention_weight_min": -26.24376106262207,
      "activations/layer21_attention_weight_max": 28.03881072998047,
      "activations/layer21_attention_weight_min": -24.453506469726562,
      "activations/layer22_attention_weight_max": 42.24629592895508,
      "activations/layer22_attention_weight_min": -30.849275588989258,
      "activations/layer23_attention_weight_max": 32.61302947998047,
      "activations/layer23_attention_weight_min": -25.90030288696289,
      "activations/layer2_attention_weight_max": 35.625728607177734,
      "activations/layer2_attention_weight_min": -33.819400787353516,
      "activations/layer3_attention_weight_max": 97.30616760253906,
      "activations/layer3_attention_weight_min": -101.65845489501953,
      "activations/layer4_attention_weight_max": 113.05299377441406,
      "activations/layer4_attention_weight_min": -111.3719253540039,
      "activations/layer5_attention_weight_max": 55.42893981933594,
      "activations/layer5_attention_weight_min": -62.008026123046875,
      "activations/layer6_attention_weight_max": 46.174068450927734,
      "activations/layer6_attention_weight_min": -49.360862731933594,
      "activations/layer7_attention_weight_max": 92.70683288574219,
      "activations/layer7_attention_weight_min": -90.47261047363281,
      "activations/layer8_attention_weight_max": 40.29894256591797,
      "activations/layer8_attention_weight_min": -40.95198440551758,
      "activations/layer9_attention_weight_max": 32.80416488647461,
      "activations/layer9_attention_weight_min": -34.284236907958984,
      "epoch": 11.56,
      "learning_rate": 7.620075757575757e-05,
      "loss": 2.7704,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5532,
      "eval_samples_per_second": 502.035,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5532,
      "eval_openwebtext_samples_per_second": 502.035,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 2.0296,
      "eval_wikitext_samples_per_second": 224.674,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_lambada_loss": 2.43359375,
      "eval_lambada_ppl": 11.399776500625777,
      "eval_lambada_runtime": 9.6325,
      "eval_lambada_samples_per_second": 505.476,
      "step": 199000
    },
    {
      "activations/layer0_attention_weight_max": 16.080894470214844,
      "activations/layer0_attention_weight_min": -12.71951675415039,
      "activations/layer10_attention_weight_max": 33.88780212402344,
      "activations/layer10_attention_weight_min": -32.16114807128906,
      "activations/layer11_attention_weight_max": 34.51921844482422,
      "activations/layer11_attention_weight_min": -30.7259521484375,
      "activations/layer12_attention_weight_max": 24.916584014892578,
      "activations/layer12_attention_weight_min": -25.825748443603516,
      "activations/layer13_attention_weight_max": 38.834102630615234,
      "activations/layer13_attention_weight_min": -34.50115203857422,
      "activations/layer14_attention_weight_max": 41.71580505371094,
      "activations/layer14_attention_weight_min": -36.249732971191406,
      "activations/layer15_attention_weight_max": 45.09257888793945,
      "activations/layer15_attention_weight_min": -33.9192008972168,
      "activations/layer16_attention_weight_max": 32.81832504272461,
      "activations/layer16_attention_weight_min": -28.58185577392578,
      "activations/layer17_attention_weight_max": 30.173967361450195,
      "activations/layer17_attention_weight_min": -25.983325958251953,
      "activations/layer18_attention_weight_max": 31.69599723815918,
      "activations/layer18_attention_weight_min": -23.285314559936523,
      "activations/layer19_attention_weight_max": 39.04923629760742,
      "activations/layer19_attention_weight_min": -30.608341217041016,
      "activations/layer1_attention_weight_max": 17.39328956604004,
      "activations/layer1_attention_weight_min": -15.610941886901855,
      "activations/layer20_attention_weight_max": 35.86870574951172,
      "activations/layer20_attention_weight_min": -23.839929580688477,
      "activations/layer21_attention_weight_max": 31.92193603515625,
      "activations/layer21_attention_weight_min": -23.2454891204834,
      "activations/layer22_attention_weight_max": 49.16602325439453,
      "activations/layer22_attention_weight_min": -30.31884002685547,
      "activations/layer23_attention_weight_max": 34.48517608642578,
      "activations/layer23_attention_weight_min": -22.415822982788086,
      "activations/layer2_attention_weight_max": 34.032344818115234,
      "activations/layer2_attention_weight_min": -34.58552169799805,
      "activations/layer3_attention_weight_max": 96.3507308959961,
      "activations/layer3_attention_weight_min": -99.9298324584961,
      "activations/layer4_attention_weight_max": 111.20855712890625,
      "activations/layer4_attention_weight_min": -118.76396942138672,
      "activations/layer5_attention_weight_max": 54.0318603515625,
      "activations/layer5_attention_weight_min": -60.417030334472656,
      "activations/layer6_attention_weight_max": 44.58152770996094,
      "activations/layer6_attention_weight_min": -46.805442810058594,
      "activations/layer7_attention_weight_max": 88.81452178955078,
      "activations/layer7_attention_weight_min": -92.1393051147461,
      "activations/layer8_attention_weight_max": 39.57321548461914,
      "activations/layer8_attention_weight_min": -40.0982551574707,
      "activations/layer9_attention_weight_max": 32.02295684814453,
      "activations/layer9_attention_weight_min": -32.58047103881836,
      "epoch": 11.57,
      "learning_rate": 7.618181818181819e-05,
      "loss": 2.7741,
      "step": 199050
    },
    {
      "activations/layer0_attention_weight_max": 15.903851509094238,
      "activations/layer0_attention_weight_min": -12.60354232788086,
      "activations/layer10_attention_weight_max": 34.852386474609375,
      "activations/layer10_attention_weight_min": -32.0274543762207,
      "activations/layer11_attention_weight_max": 34.26158905029297,
      "activations/layer11_attention_weight_min": -31.54909324645996,
      "activations/layer12_attention_weight_max": 24.31142807006836,
      "activations/layer12_attention_weight_min": -24.99987030029297,
      "activations/layer13_attention_weight_max": 37.86731719970703,
      "activations/layer13_attention_weight_min": -35.247249603271484,
      "activations/layer14_attention_weight_max": 41.27449035644531,
      "activations/layer14_attention_weight_min": -36.31813049316406,
      "activations/layer15_attention_weight_max": 35.659000396728516,
      "activations/layer15_attention_weight_min": -32.094261169433594,
      "activations/layer16_attention_weight_max": 29.23122215270996,
      "activations/layer16_attention_weight_min": -28.229875564575195,
      "activations/layer17_attention_weight_max": 30.73383903503418,
      "activations/layer17_attention_weight_min": -25.54559898376465,
      "activations/layer18_attention_weight_max": 32.30512237548828,
      "activations/layer18_attention_weight_min": -25.262311935424805,
      "activations/layer19_attention_weight_max": 33.546592712402344,
      "activations/layer19_attention_weight_min": -34.57334899902344,
      "activations/layer1_attention_weight_max": 15.4938325881958,
      "activations/layer1_attention_weight_min": -12.248819351196289,
      "activations/layer20_attention_weight_max": 29.116254806518555,
      "activations/layer20_attention_weight_min": -26.599943161010742,
      "activations/layer21_attention_weight_max": 28.289146423339844,
      "activations/layer21_attention_weight_min": -23.91770362854004,
      "activations/layer22_attention_weight_max": 42.716915130615234,
      "activations/layer22_attention_weight_min": -32.186363220214844,
      "activations/layer23_attention_weight_max": 34.15460205078125,
      "activations/layer23_attention_weight_min": -28.004688262939453,
      "activations/layer2_attention_weight_max": 30.915935516357422,
      "activations/layer2_attention_weight_min": -28.994613647460938,
      "activations/layer3_attention_weight_max": 91.00335693359375,
      "activations/layer3_attention_weight_min": -89.10094451904297,
      "activations/layer4_attention_weight_max": 119.21089172363281,
      "activations/layer4_attention_weight_min": -108.15705871582031,
      "activations/layer5_attention_weight_max": 55.984519958496094,
      "activations/layer5_attention_weight_min": -66.95333862304688,
      "activations/layer6_attention_weight_max": 46.16686248779297,
      "activations/layer6_attention_weight_min": -48.021358489990234,
      "activations/layer7_attention_weight_max": 96.09174346923828,
      "activations/layer7_attention_weight_min": -88.67670440673828,
      "activations/layer8_attention_weight_max": 41.99765396118164,
      "activations/layer8_attention_weight_min": -41.421260833740234,
      "activations/layer9_attention_weight_max": 35.20172119140625,
      "activations/layer9_attention_weight_min": -32.165794372558594,
      "epoch": 11.57,
      "learning_rate": 7.616287878787878e-05,
      "loss": 2.7827,
      "step": 199100
    },
    {
      "activations/layer0_attention_weight_max": 15.214764595031738,
      "activations/layer0_attention_weight_min": -12.067424774169922,
      "activations/layer10_attention_weight_max": 33.99650955200195,
      "activations/layer10_attention_weight_min": -33.841026306152344,
      "activations/layer11_attention_weight_max": 34.03269958496094,
      "activations/layer11_attention_weight_min": -33.38803482055664,
      "activations/layer12_attention_weight_max": 27.838375091552734,
      "activations/layer12_attention_weight_min": -27.152807235717773,
      "activations/layer13_attention_weight_max": 38.03186798095703,
      "activations/layer13_attention_weight_min": -35.622196197509766,
      "activations/layer14_attention_weight_max": 52.07107925415039,
      "activations/layer14_attention_weight_min": -41.92840576171875,
      "activations/layer15_attention_weight_max": 40.258811950683594,
      "activations/layer15_attention_weight_min": -34.7603874206543,
      "activations/layer16_attention_weight_max": 34.16038513183594,
      "activations/layer16_attention_weight_min": -27.252775192260742,
      "activations/layer17_attention_weight_max": 32.99953842163086,
      "activations/layer17_attention_weight_min": -24.95546531677246,
      "activations/layer18_attention_weight_max": 33.91199493408203,
      "activations/layer18_attention_weight_min": -24.009723663330078,
      "activations/layer19_attention_weight_max": 39.374935150146484,
      "activations/layer19_attention_weight_min": -31.29062843322754,
      "activations/layer1_attention_weight_max": 17.403423309326172,
      "activations/layer1_attention_weight_min": -13.924121856689453,
      "activations/layer20_attention_weight_max": 32.9506950378418,
      "activations/layer20_attention_weight_min": -23.41172981262207,
      "activations/layer21_attention_weight_max": 37.167640686035156,
      "activations/layer21_attention_weight_min": -23.911468505859375,
      "activations/layer22_attention_weight_max": 52.47988510131836,
      "activations/layer22_attention_weight_min": -29.057710647583008,
      "activations/layer23_attention_weight_max": 39.038326263427734,
      "activations/layer23_attention_weight_min": -25.482276916503906,
      "activations/layer2_attention_weight_max": 31.592453002929688,
      "activations/layer2_attention_weight_min": -31.107418060302734,
      "activations/layer3_attention_weight_max": 93.05985260009766,
      "activations/layer3_attention_weight_min": -98.44200134277344,
      "activations/layer4_attention_weight_max": 108.4588851928711,
      "activations/layer4_attention_weight_min": -115.1385498046875,
      "activations/layer5_attention_weight_max": 50.10773849487305,
      "activations/layer5_attention_weight_min": -62.94093322753906,
      "activations/layer6_attention_weight_max": 43.697410583496094,
      "activations/layer6_attention_weight_min": -47.42589569091797,
      "activations/layer7_attention_weight_max": 93.5195541381836,
      "activations/layer7_attention_weight_min": -91.93377685546875,
      "activations/layer8_attention_weight_max": 40.60357666015625,
      "activations/layer8_attention_weight_min": -42.40119934082031,
      "activations/layer9_attention_weight_max": 33.46854782104492,
      "activations/layer9_attention_weight_min": -34.29767990112305,
      "epoch": 11.57,
      "learning_rate": 7.614393939393939e-05,
      "loss": 2.7777,
      "step": 199150
    },
    {
      "activations/layer0_attention_weight_max": 16.477630615234375,
      "activations/layer0_attention_weight_min": -13.015525817871094,
      "activations/layer10_attention_weight_max": 32.27104949951172,
      "activations/layer10_attention_weight_min": -30.80668830871582,
      "activations/layer11_attention_weight_max": 28.62674331665039,
      "activations/layer11_attention_weight_min": -29.501296997070312,
      "activations/layer12_attention_weight_max": 23.524518966674805,
      "activations/layer12_attention_weight_min": -24.317777633666992,
      "activations/layer13_attention_weight_max": 37.32159423828125,
      "activations/layer13_attention_weight_min": -33.32583236694336,
      "activations/layer14_attention_weight_max": 42.51961135864258,
      "activations/layer14_attention_weight_min": -37.1848258972168,
      "activations/layer15_attention_weight_max": 36.79697799682617,
      "activations/layer15_attention_weight_min": -31.677310943603516,
      "activations/layer16_attention_weight_max": 29.789379119873047,
      "activations/layer16_attention_weight_min": -26.84678840637207,
      "activations/layer17_attention_weight_max": 33.9758186340332,
      "activations/layer17_attention_weight_min": -24.784778594970703,
      "activations/layer18_attention_weight_max": 32.113399505615234,
      "activations/layer18_attention_weight_min": -22.33065414428711,
      "activations/layer19_attention_weight_max": 32.50020980834961,
      "activations/layer19_attention_weight_min": -30.317096710205078,
      "activations/layer1_attention_weight_max": 16.309782028198242,
      "activations/layer1_attention_weight_min": -14.18271541595459,
      "activations/layer20_attention_weight_max": 26.942209243774414,
      "activations/layer20_attention_weight_min": -24.644807815551758,
      "activations/layer21_attention_weight_max": 28.461437225341797,
      "activations/layer21_attention_weight_min": -23.942527770996094,
      "activations/layer22_attention_weight_max": 43.38239669799805,
      "activations/layer22_attention_weight_min": -28.893369674682617,
      "activations/layer23_attention_weight_max": 34.317691802978516,
      "activations/layer23_attention_weight_min": -26.868824005126953,
      "activations/layer2_attention_weight_max": 30.929668426513672,
      "activations/layer2_attention_weight_min": -31.844717025756836,
      "activations/layer3_attention_weight_max": 93.09153747558594,
      "activations/layer3_attention_weight_min": -92.7447738647461,
      "activations/layer4_attention_weight_max": 112.57311248779297,
      "activations/layer4_attention_weight_min": -109.55953216552734,
      "activations/layer5_attention_weight_max": 49.07143783569336,
      "activations/layer5_attention_weight_min": -60.55794906616211,
      "activations/layer6_attention_weight_max": 45.274906158447266,
      "activations/layer6_attention_weight_min": -46.72172546386719,
      "activations/layer7_attention_weight_max": 87.59354400634766,
      "activations/layer7_attention_weight_min": -87.6660385131836,
      "activations/layer8_attention_weight_max": 41.62794876098633,
      "activations/layer8_attention_weight_min": -39.42044448852539,
      "activations/layer9_attention_weight_max": 31.1020565032959,
      "activations/layer9_attention_weight_min": -30.646320343017578,
      "epoch": 11.57,
      "learning_rate": 7.612499999999998e-05,
      "loss": 2.7907,
      "step": 199200
    },
    {
      "activations/layer0_attention_weight_max": 15.791703224182129,
      "activations/layer0_attention_weight_min": -12.038124084472656,
      "activations/layer10_attention_weight_max": 32.94832992553711,
      "activations/layer10_attention_weight_min": -32.51509475708008,
      "activations/layer11_attention_weight_max": 32.848384857177734,
      "activations/layer11_attention_weight_min": -31.413482666015625,
      "activations/layer12_attention_weight_max": 35.68144226074219,
      "activations/layer12_attention_weight_min": -32.98933029174805,
      "activations/layer13_attention_weight_max": 41.908958435058594,
      "activations/layer13_attention_weight_min": -33.847740173339844,
      "activations/layer14_attention_weight_max": 43.4912109375,
      "activations/layer14_attention_weight_min": -38.478946685791016,
      "activations/layer15_attention_weight_max": 37.542537689208984,
      "activations/layer15_attention_weight_min": -33.643802642822266,
      "activations/layer16_attention_weight_max": 32.106719970703125,
      "activations/layer16_attention_weight_min": -28.082782745361328,
      "activations/layer17_attention_weight_max": 32.44524383544922,
      "activations/layer17_attention_weight_min": -25.542360305786133,
      "activations/layer18_attention_weight_max": 31.696884155273438,
      "activations/layer18_attention_weight_min": -23.488937377929688,
      "activations/layer19_attention_weight_max": 38.36202621459961,
      "activations/layer19_attention_weight_min": -31.012714385986328,
      "activations/layer1_attention_weight_max": 16.168399810791016,
      "activations/layer1_attention_weight_min": -13.593032836914062,
      "activations/layer20_attention_weight_max": 30.037303924560547,
      "activations/layer20_attention_weight_min": -24.49272346496582,
      "activations/layer21_attention_weight_max": 31.034746170043945,
      "activations/layer21_attention_weight_min": -23.27086639404297,
      "activations/layer22_attention_weight_max": 47.28976821899414,
      "activations/layer22_attention_weight_min": -28.688737869262695,
      "activations/layer23_attention_weight_max": 36.81443786621094,
      "activations/layer23_attention_weight_min": -23.67074966430664,
      "activations/layer2_attention_weight_max": 31.252613067626953,
      "activations/layer2_attention_weight_min": -31.678380966186523,
      "activations/layer3_attention_weight_max": 93.0123291015625,
      "activations/layer3_attention_weight_min": -95.32405853271484,
      "activations/layer4_attention_weight_max": 107.46294403076172,
      "activations/layer4_attention_weight_min": -110.0603256225586,
      "activations/layer5_attention_weight_max": 58.042396545410156,
      "activations/layer5_attention_weight_min": -60.8961296081543,
      "activations/layer6_attention_weight_max": 42.0902099609375,
      "activations/layer6_attention_weight_min": -47.2507438659668,
      "activations/layer7_attention_weight_max": 96.34779357910156,
      "activations/layer7_attention_weight_min": -88.17692565917969,
      "activations/layer8_attention_weight_max": 40.0482177734375,
      "activations/layer8_attention_weight_min": -41.179840087890625,
      "activations/layer9_attention_weight_max": 31.549942016601562,
      "activations/layer9_attention_weight_min": -35.94771957397461,
      "epoch": 11.58,
      "learning_rate": 7.61060606060606e-05,
      "loss": 2.7789,
      "step": 199250
    },
    {
      "activations/layer0_attention_weight_max": 15.72608757019043,
      "activations/layer0_attention_weight_min": -12.073503494262695,
      "activations/layer10_attention_weight_max": 34.449344635009766,
      "activations/layer10_attention_weight_min": -32.244144439697266,
      "activations/layer11_attention_weight_max": 33.291786193847656,
      "activations/layer11_attention_weight_min": -33.207496643066406,
      "activations/layer12_attention_weight_max": 25.208656311035156,
      "activations/layer12_attention_weight_min": -24.33128547668457,
      "activations/layer13_attention_weight_max": 36.90167999267578,
      "activations/layer13_attention_weight_min": -35.327457427978516,
      "activations/layer14_attention_weight_max": 47.784759521484375,
      "activations/layer14_attention_weight_min": -41.79887771606445,
      "activations/layer15_attention_weight_max": 39.90364074707031,
      "activations/layer15_attention_weight_min": -32.773502349853516,
      "activations/layer16_attention_weight_max": 30.24468231201172,
      "activations/layer16_attention_weight_min": -28.14042854309082,
      "activations/layer17_attention_weight_max": 33.270450592041016,
      "activations/layer17_attention_weight_min": -24.808210372924805,
      "activations/layer18_attention_weight_max": 29.942237854003906,
      "activations/layer18_attention_weight_min": -22.9938907623291,
      "activations/layer19_attention_weight_max": 35.83586120605469,
      "activations/layer19_attention_weight_min": -32.204959869384766,
      "activations/layer1_attention_weight_max": 16.838088989257812,
      "activations/layer1_attention_weight_min": -14.31805419921875,
      "activations/layer20_attention_weight_max": 29.562469482421875,
      "activations/layer20_attention_weight_min": -24.914587020874023,
      "activations/layer21_attention_weight_max": 28.286911010742188,
      "activations/layer21_attention_weight_min": -24.00020408630371,
      "activations/layer22_attention_weight_max": 41.56938552856445,
      "activations/layer22_attention_weight_min": -28.43717384338379,
      "activations/layer23_attention_weight_max": 35.25492858886719,
      "activations/layer23_attention_weight_min": -22.869253158569336,
      "activations/layer2_attention_weight_max": 31.94674301147461,
      "activations/layer2_attention_weight_min": -33.13484191894531,
      "activations/layer3_attention_weight_max": 94.958251953125,
      "activations/layer3_attention_weight_min": -100.46601104736328,
      "activations/layer4_attention_weight_max": 111.0119857788086,
      "activations/layer4_attention_weight_min": -113.600830078125,
      "activations/layer5_attention_weight_max": 53.56523895263672,
      "activations/layer5_attention_weight_min": -65.38748168945312,
      "activations/layer6_attention_weight_max": 43.537498474121094,
      "activations/layer6_attention_weight_min": -48.01576232910156,
      "activations/layer7_attention_weight_max": 92.1983642578125,
      "activations/layer7_attention_weight_min": -96.3426513671875,
      "activations/layer8_attention_weight_max": 40.104591369628906,
      "activations/layer8_attention_weight_min": -44.49674606323242,
      "activations/layer9_attention_weight_max": 34.17716979980469,
      "activations/layer9_attention_weight_min": -33.06577682495117,
      "epoch": 11.58,
      "learning_rate": 7.608712121212121e-05,
      "loss": 2.7815,
      "step": 199300
    },
    {
      "activations/layer0_attention_weight_max": 16.143798828125,
      "activations/layer0_attention_weight_min": -12.56303596496582,
      "activations/layer10_attention_weight_max": 34.501930236816406,
      "activations/layer10_attention_weight_min": -35.2398567199707,
      "activations/layer11_attention_weight_max": 33.25505065917969,
      "activations/layer11_attention_weight_min": -35.396690368652344,
      "activations/layer12_attention_weight_max": 39.83607482910156,
      "activations/layer12_attention_weight_min": -30.207340240478516,
      "activations/layer13_attention_weight_max": 44.30795669555664,
      "activations/layer13_attention_weight_min": -35.23822021484375,
      "activations/layer14_attention_weight_max": 45.974796295166016,
      "activations/layer14_attention_weight_min": -42.41885757446289,
      "activations/layer15_attention_weight_max": 38.283241271972656,
      "activations/layer15_attention_weight_min": -35.172298431396484,
      "activations/layer16_attention_weight_max": 33.25992965698242,
      "activations/layer16_attention_weight_min": -30.700590133666992,
      "activations/layer17_attention_weight_max": 35.233154296875,
      "activations/layer17_attention_weight_min": -27.756572723388672,
      "activations/layer18_attention_weight_max": 35.17695236206055,
      "activations/layer18_attention_weight_min": -26.246681213378906,
      "activations/layer19_attention_weight_max": 38.646240234375,
      "activations/layer19_attention_weight_min": -33.47646713256836,
      "activations/layer1_attention_weight_max": 16.843412399291992,
      "activations/layer1_attention_weight_min": -14.383432388305664,
      "activations/layer20_attention_weight_max": 34.324378967285156,
      "activations/layer20_attention_weight_min": -24.768842697143555,
      "activations/layer21_attention_weight_max": 33.97879409790039,
      "activations/layer21_attention_weight_min": -24.52667236328125,
      "activations/layer22_attention_weight_max": 48.806678771972656,
      "activations/layer22_attention_weight_min": -30.386302947998047,
      "activations/layer23_attention_weight_max": 35.65391159057617,
      "activations/layer23_attention_weight_min": -25.275894165039062,
      "activations/layer2_attention_weight_max": 32.76179885864258,
      "activations/layer2_attention_weight_min": -32.68476867675781,
      "activations/layer3_attention_weight_max": 92.456787109375,
      "activations/layer3_attention_weight_min": -100.49015808105469,
      "activations/layer4_attention_weight_max": 112.4388198852539,
      "activations/layer4_attention_weight_min": -111.86927795410156,
      "activations/layer5_attention_weight_max": 51.53754425048828,
      "activations/layer5_attention_weight_min": -60.19822692871094,
      "activations/layer6_attention_weight_max": 47.39602279663086,
      "activations/layer6_attention_weight_min": -48.30455017089844,
      "activations/layer7_attention_weight_max": 94.1503677368164,
      "activations/layer7_attention_weight_min": -89.74716186523438,
      "activations/layer8_attention_weight_max": 40.940189361572266,
      "activations/layer8_attention_weight_min": -43.2222900390625,
      "activations/layer9_attention_weight_max": 34.19548797607422,
      "activations/layer9_attention_weight_min": -35.40275955200195,
      "epoch": 11.58,
      "learning_rate": 7.606818181818181e-05,
      "loss": 2.7764,
      "step": 199350
    },
    {
      "activations/layer0_attention_weight_max": 16.51909828186035,
      "activations/layer0_attention_weight_min": -12.35833740234375,
      "activations/layer10_attention_weight_max": 32.952178955078125,
      "activations/layer10_attention_weight_min": -32.29700469970703,
      "activations/layer11_attention_weight_max": 30.713573455810547,
      "activations/layer11_attention_weight_min": -30.997177124023438,
      "activations/layer12_attention_weight_max": 25.57465171813965,
      "activations/layer12_attention_weight_min": -24.73906898498535,
      "activations/layer13_attention_weight_max": 37.54772186279297,
      "activations/layer13_attention_weight_min": -33.31467819213867,
      "activations/layer14_attention_weight_max": 43.71185302734375,
      "activations/layer14_attention_weight_min": -38.162330627441406,
      "activations/layer15_attention_weight_max": 35.26202392578125,
      "activations/layer15_attention_weight_min": -32.87823486328125,
      "activations/layer16_attention_weight_max": 31.552257537841797,
      "activations/layer16_attention_weight_min": -28.949853897094727,
      "activations/layer17_attention_weight_max": 30.61137580871582,
      "activations/layer17_attention_weight_min": -27.37407875061035,
      "activations/layer18_attention_weight_max": 28.50732421875,
      "activations/layer18_attention_weight_min": -23.847980499267578,
      "activations/layer19_attention_weight_max": 31.932758331298828,
      "activations/layer19_attention_weight_min": -31.078332901000977,
      "activations/layer1_attention_weight_max": 16.800073623657227,
      "activations/layer1_attention_weight_min": -13.318563461303711,
      "activations/layer20_attention_weight_max": 28.577306747436523,
      "activations/layer20_attention_weight_min": -26.166141510009766,
      "activations/layer21_attention_weight_max": 28.192607879638672,
      "activations/layer21_attention_weight_min": -24.921140670776367,
      "activations/layer22_attention_weight_max": 42.367210388183594,
      "activations/layer22_attention_weight_min": -31.69005012512207,
      "activations/layer23_attention_weight_max": 32.59358596801758,
      "activations/layer23_attention_weight_min": -25.951797485351562,
      "activations/layer2_attention_weight_max": 31.880537033081055,
      "activations/layer2_attention_weight_min": -30.93838119506836,
      "activations/layer3_attention_weight_max": 91.78627014160156,
      "activations/layer3_attention_weight_min": -91.46257781982422,
      "activations/layer4_attention_weight_max": 109.86064910888672,
      "activations/layer4_attention_weight_min": -117.07167053222656,
      "activations/layer5_attention_weight_max": 51.25727462768555,
      "activations/layer5_attention_weight_min": -60.73885726928711,
      "activations/layer6_attention_weight_max": 46.129756927490234,
      "activations/layer6_attention_weight_min": -47.19780349731445,
      "activations/layer7_attention_weight_max": 85.33991241455078,
      "activations/layer7_attention_weight_min": -96.31775665283203,
      "activations/layer8_attention_weight_max": 39.41207504272461,
      "activations/layer8_attention_weight_min": -41.69008255004883,
      "activations/layer9_attention_weight_max": 32.96354293823242,
      "activations/layer9_attention_weight_min": -33.24303436279297,
      "epoch": 11.59,
      "learning_rate": 7.604924242424241e-05,
      "loss": 2.7746,
      "step": 199400
    },
    {
      "activations/layer0_attention_weight_max": 15.884801864624023,
      "activations/layer0_attention_weight_min": -12.840897560119629,
      "activations/layer10_attention_weight_max": 39.6890754699707,
      "activations/layer10_attention_weight_min": -36.265953063964844,
      "activations/layer11_attention_weight_max": 38.92547607421875,
      "activations/layer11_attention_weight_min": -36.027870178222656,
      "activations/layer12_attention_weight_max": 29.675960540771484,
      "activations/layer12_attention_weight_min": -27.07794952392578,
      "activations/layer13_attention_weight_max": 45.82467269897461,
      "activations/layer13_attention_weight_min": -38.67805099487305,
      "activations/layer14_attention_weight_max": 52.26810073852539,
      "activations/layer14_attention_weight_min": -41.70553970336914,
      "activations/layer15_attention_weight_max": 43.972660064697266,
      "activations/layer15_attention_weight_min": -35.89436340332031,
      "activations/layer16_attention_weight_max": 39.105281829833984,
      "activations/layer16_attention_weight_min": -31.19075584411621,
      "activations/layer17_attention_weight_max": 38.025699615478516,
      "activations/layer17_attention_weight_min": -26.229278564453125,
      "activations/layer18_attention_weight_max": 36.90789794921875,
      "activations/layer18_attention_weight_min": -23.266468048095703,
      "activations/layer19_attention_weight_max": 45.067256927490234,
      "activations/layer19_attention_weight_min": -31.588171005249023,
      "activations/layer1_attention_weight_max": 15.843701362609863,
      "activations/layer1_attention_weight_min": -15.035581588745117,
      "activations/layer20_attention_weight_max": 38.46487808227539,
      "activations/layer20_attention_weight_min": -23.88437843322754,
      "activations/layer21_attention_weight_max": 35.58384323120117,
      "activations/layer21_attention_weight_min": -23.61289405822754,
      "activations/layer22_attention_weight_max": 56.70266342163086,
      "activations/layer22_attention_weight_min": -29.086368560791016,
      "activations/layer23_attention_weight_max": 43.99128723144531,
      "activations/layer23_attention_weight_min": -27.46845245361328,
      "activations/layer2_attention_weight_max": 31.739696502685547,
      "activations/layer2_attention_weight_min": -30.731460571289062,
      "activations/layer3_attention_weight_max": 98.06145477294922,
      "activations/layer3_attention_weight_min": -97.12169647216797,
      "activations/layer4_attention_weight_max": 114.68922424316406,
      "activations/layer4_attention_weight_min": -112.83446502685547,
      "activations/layer5_attention_weight_max": 53.21824264526367,
      "activations/layer5_attention_weight_min": -62.54908752441406,
      "activations/layer6_attention_weight_max": 45.70939636230469,
      "activations/layer6_attention_weight_min": -48.050682067871094,
      "activations/layer7_attention_weight_max": 94.93804931640625,
      "activations/layer7_attention_weight_min": -94.04231262207031,
      "activations/layer8_attention_weight_max": 46.74853515625,
      "activations/layer8_attention_weight_min": -42.895408630371094,
      "activations/layer9_attention_weight_max": 41.08245849609375,
      "activations/layer9_attention_weight_min": -37.19380569458008,
      "epoch": 11.59,
      "learning_rate": 7.603030303030303e-05,
      "loss": 2.7742,
      "step": 199450
    },
    {
      "activations/layer0_attention_weight_max": 14.168194770812988,
      "activations/layer0_attention_weight_min": -13.185242652893066,
      "activations/layer10_attention_weight_max": 46.795345306396484,
      "activations/layer10_attention_weight_min": -45.59575653076172,
      "activations/layer11_attention_weight_max": 43.22457504272461,
      "activations/layer11_attention_weight_min": -46.96091842651367,
      "activations/layer12_attention_weight_max": 54.16301345825195,
      "activations/layer12_attention_weight_min": -29.281145095825195,
      "activations/layer13_attention_weight_max": 72.40125274658203,
      "activations/layer13_attention_weight_min": -36.6229362487793,
      "activations/layer14_attention_weight_max": 57.69843673706055,
      "activations/layer14_attention_weight_min": -38.80875015258789,
      "activations/layer15_attention_weight_max": 60.44294357299805,
      "activations/layer15_attention_weight_min": -34.561153411865234,
      "activations/layer16_attention_weight_max": 34.36445236206055,
      "activations/layer16_attention_weight_min": -28.93701171875,
      "activations/layer17_attention_weight_max": 29.65275764465332,
      "activations/layer17_attention_weight_min": -26.148893356323242,
      "activations/layer18_attention_weight_max": 32.08728790283203,
      "activations/layer18_attention_weight_min": -21.96670150756836,
      "activations/layer19_attention_weight_max": 35.80820846557617,
      "activations/layer19_attention_weight_min": -30.19297218322754,
      "activations/layer1_attention_weight_max": 16.88788604736328,
      "activations/layer1_attention_weight_min": -15.486579895019531,
      "activations/layer20_attention_weight_max": 29.873245239257812,
      "activations/layer20_attention_weight_min": -24.16193389892578,
      "activations/layer21_attention_weight_max": 29.230327606201172,
      "activations/layer21_attention_weight_min": -23.301456451416016,
      "activations/layer22_attention_weight_max": 38.750694274902344,
      "activations/layer22_attention_weight_min": -26.983680725097656,
      "activations/layer23_attention_weight_max": 34.11079788208008,
      "activations/layer23_attention_weight_min": -22.58868980407715,
      "activations/layer2_attention_weight_max": 36.29701232910156,
      "activations/layer2_attention_weight_min": -37.08500671386719,
      "activations/layer3_attention_weight_max": 105.609375,
      "activations/layer3_attention_weight_min": -110.18651580810547,
      "activations/layer4_attention_weight_max": 124.7897720336914,
      "activations/layer4_attention_weight_min": -123.38275146484375,
      "activations/layer5_attention_weight_max": 58.32950973510742,
      "activations/layer5_attention_weight_min": -65.02210235595703,
      "activations/layer6_attention_weight_max": 53.70716857910156,
      "activations/layer6_attention_weight_min": -54.553279876708984,
      "activations/layer7_attention_weight_max": 123.12004852294922,
      "activations/layer7_attention_weight_min": -119.57862091064453,
      "activations/layer8_attention_weight_max": 58.66263198852539,
      "activations/layer8_attention_weight_min": -57.95481491088867,
      "activations/layer9_attention_weight_max": 47.1214714050293,
      "activations/layer9_attention_weight_min": -46.20444869995117,
      "epoch": 11.59,
      "learning_rate": 7.601136363636363e-05,
      "loss": 2.7767,
      "step": 199500
    },
    {
      "activations/layer0_attention_weight_max": 15.627843856811523,
      "activations/layer0_attention_weight_min": -11.499062538146973,
      "activations/layer10_attention_weight_max": 35.34583282470703,
      "activations/layer10_attention_weight_min": -33.762939453125,
      "activations/layer11_attention_weight_max": 33.34406280517578,
      "activations/layer11_attention_weight_min": -33.183563232421875,
      "activations/layer12_attention_weight_max": 25.120498657226562,
      "activations/layer12_attention_weight_min": -26.30412483215332,
      "activations/layer13_attention_weight_max": 38.3808708190918,
      "activations/layer13_attention_weight_min": -33.719078063964844,
      "activations/layer14_attention_weight_max": 42.899864196777344,
      "activations/layer14_attention_weight_min": -38.368682861328125,
      "activations/layer15_attention_weight_max": 34.98085021972656,
      "activations/layer15_attention_weight_min": -31.91297149658203,
      "activations/layer16_attention_weight_max": 27.398637771606445,
      "activations/layer16_attention_weight_min": -28.85561752319336,
      "activations/layer17_attention_weight_max": 32.363101959228516,
      "activations/layer17_attention_weight_min": -25.801828384399414,
      "activations/layer18_attention_weight_max": 31.49073600769043,
      "activations/layer18_attention_weight_min": -22.54056167602539,
      "activations/layer19_attention_weight_max": 30.653738021850586,
      "activations/layer19_attention_weight_min": -31.9140567779541,
      "activations/layer1_attention_weight_max": 16.549488067626953,
      "activations/layer1_attention_weight_min": -14.53642463684082,
      "activations/layer20_attention_weight_max": 30.178743362426758,
      "activations/layer20_attention_weight_min": -25.367176055908203,
      "activations/layer21_attention_weight_max": 26.77181053161621,
      "activations/layer21_attention_weight_min": -24.318584442138672,
      "activations/layer22_attention_weight_max": 41.24749755859375,
      "activations/layer22_attention_weight_min": -27.84383773803711,
      "activations/layer23_attention_weight_max": 29.712139129638672,
      "activations/layer23_attention_weight_min": -23.97456932067871,
      "activations/layer2_attention_weight_max": 31.1662654876709,
      "activations/layer2_attention_weight_min": -31.487051010131836,
      "activations/layer3_attention_weight_max": 90.81304168701172,
      "activations/layer3_attention_weight_min": -92.50028228759766,
      "activations/layer4_attention_weight_max": 111.76580810546875,
      "activations/layer4_attention_weight_min": -114.96588134765625,
      "activations/layer5_attention_weight_max": 51.83259201049805,
      "activations/layer5_attention_weight_min": -60.84011459350586,
      "activations/layer6_attention_weight_max": 44.154884338378906,
      "activations/layer6_attention_weight_min": -49.37899398803711,
      "activations/layer7_attention_weight_max": 88.31996154785156,
      "activations/layer7_attention_weight_min": -90.6193618774414,
      "activations/layer8_attention_weight_max": 39.60741424560547,
      "activations/layer8_attention_weight_min": -42.83054733276367,
      "activations/layer9_attention_weight_max": 35.13246154785156,
      "activations/layer9_attention_weight_min": -34.174373626708984,
      "epoch": 11.59,
      "learning_rate": 7.599242424242423e-05,
      "loss": 2.7743,
      "step": 199550
    },
    {
      "activations/layer0_attention_weight_max": 15.632133483886719,
      "activations/layer0_attention_weight_min": -13.444697380065918,
      "activations/layer10_attention_weight_max": 35.93906784057617,
      "activations/layer10_attention_weight_min": -33.54939270019531,
      "activations/layer11_attention_weight_max": 37.7673225402832,
      "activations/layer11_attention_weight_min": -34.53680419921875,
      "activations/layer12_attention_weight_max": 39.49277877807617,
      "activations/layer12_attention_weight_min": -25.332658767700195,
      "activations/layer13_attention_weight_max": 48.25480270385742,
      "activations/layer13_attention_weight_min": -35.339603424072266,
      "activations/layer14_attention_weight_max": 49.32961654663086,
      "activations/layer14_attention_weight_min": -37.33150863647461,
      "activations/layer15_attention_weight_max": 55.195579528808594,
      "activations/layer15_attention_weight_min": -33.180397033691406,
      "activations/layer16_attention_weight_max": 41.00511169433594,
      "activations/layer16_attention_weight_min": -28.127729415893555,
      "activations/layer17_attention_weight_max": 31.38539695739746,
      "activations/layer17_attention_weight_min": -25.838754653930664,
      "activations/layer18_attention_weight_max": 34.04835510253906,
      "activations/layer18_attention_weight_min": -24.762237548828125,
      "activations/layer19_attention_weight_max": 39.388668060302734,
      "activations/layer19_attention_weight_min": -34.14397430419922,
      "activations/layer1_attention_weight_max": 16.500844955444336,
      "activations/layer1_attention_weight_min": -14.446290969848633,
      "activations/layer20_attention_weight_max": 34.236087799072266,
      "activations/layer20_attention_weight_min": -26.930463790893555,
      "activations/layer21_attention_weight_max": 33.39568328857422,
      "activations/layer21_attention_weight_min": -23.940099716186523,
      "activations/layer22_attention_weight_max": 49.50563049316406,
      "activations/layer22_attention_weight_min": -32.81230545043945,
      "activations/layer23_attention_weight_max": 39.95032501220703,
      "activations/layer23_attention_weight_min": -24.144302368164062,
      "activations/layer2_attention_weight_max": 34.56375503540039,
      "activations/layer2_attention_weight_min": -32.41130065917969,
      "activations/layer3_attention_weight_max": 93.96924591064453,
      "activations/layer3_attention_weight_min": -96.7925796508789,
      "activations/layer4_attention_weight_max": 108.96785736083984,
      "activations/layer4_attention_weight_min": -113.3356704711914,
      "activations/layer5_attention_weight_max": 52.72538757324219,
      "activations/layer5_attention_weight_min": -63.36237335205078,
      "activations/layer6_attention_weight_max": 45.212867736816406,
      "activations/layer6_attention_weight_min": -48.31433868408203,
      "activations/layer7_attention_weight_max": 90.87374877929688,
      "activations/layer7_attention_weight_min": -88.87635040283203,
      "activations/layer8_attention_weight_max": 43.47942352294922,
      "activations/layer8_attention_weight_min": -42.78339385986328,
      "activations/layer9_attention_weight_max": 36.968441009521484,
      "activations/layer9_attention_weight_min": -35.08487319946289,
      "epoch": 11.6,
      "learning_rate": 7.597348484848483e-05,
      "loss": 2.791,
      "step": 199600
    },
    {
      "activations/layer0_attention_weight_max": 15.324210166931152,
      "activations/layer0_attention_weight_min": -12.794548034667969,
      "activations/layer10_attention_weight_max": 31.615123748779297,
      "activations/layer10_attention_weight_min": -32.29928207397461,
      "activations/layer11_attention_weight_max": 30.543991088867188,
      "activations/layer11_attention_weight_min": -32.161766052246094,
      "activations/layer12_attention_weight_max": 36.230045318603516,
      "activations/layer12_attention_weight_min": -25.953388214111328,
      "activations/layer13_attention_weight_max": 43.12197494506836,
      "activations/layer13_attention_weight_min": -34.90229415893555,
      "activations/layer14_attention_weight_max": 40.3121452331543,
      "activations/layer14_attention_weight_min": -40.792118072509766,
      "activations/layer15_attention_weight_max": 33.6700325012207,
      "activations/layer15_attention_weight_min": -34.75138473510742,
      "activations/layer16_attention_weight_max": 29.1676082611084,
      "activations/layer16_attention_weight_min": -29.54379653930664,
      "activations/layer17_attention_weight_max": 29.431915283203125,
      "activations/layer17_attention_weight_min": -26.16872787475586,
      "activations/layer18_attention_weight_max": 28.78644371032715,
      "activations/layer18_attention_weight_min": -23.900144577026367,
      "activations/layer19_attention_weight_max": 32.36445236206055,
      "activations/layer19_attention_weight_min": -32.30582809448242,
      "activations/layer1_attention_weight_max": 16.16221046447754,
      "activations/layer1_attention_weight_min": -15.792165756225586,
      "activations/layer20_attention_weight_max": 31.038667678833008,
      "activations/layer20_attention_weight_min": -24.70985984802246,
      "activations/layer21_attention_weight_max": 31.077373504638672,
      "activations/layer21_attention_weight_min": -24.218765258789062,
      "activations/layer22_attention_weight_max": 41.945980072021484,
      "activations/layer22_attention_weight_min": -31.003496170043945,
      "activations/layer23_attention_weight_max": 32.10530090332031,
      "activations/layer23_attention_weight_min": -27.551450729370117,
      "activations/layer2_attention_weight_max": 32.77357482910156,
      "activations/layer2_attention_weight_min": -32.507999420166016,
      "activations/layer3_attention_weight_max": 93.99430084228516,
      "activations/layer3_attention_weight_min": -95.44213104248047,
      "activations/layer4_attention_weight_max": 110.79522705078125,
      "activations/layer4_attention_weight_min": -114.69337463378906,
      "activations/layer5_attention_weight_max": 51.061546325683594,
      "activations/layer5_attention_weight_min": -64.29637908935547,
      "activations/layer6_attention_weight_max": 45.129234313964844,
      "activations/layer6_attention_weight_min": -48.97922897338867,
      "activations/layer7_attention_weight_max": 90.13995361328125,
      "activations/layer7_attention_weight_min": -89.86620330810547,
      "activations/layer8_attention_weight_max": 42.120140075683594,
      "activations/layer8_attention_weight_min": -45.59075164794922,
      "activations/layer9_attention_weight_max": 32.55526351928711,
      "activations/layer9_attention_weight_min": -34.059608459472656,
      "epoch": 11.6,
      "learning_rate": 7.595454545454545e-05,
      "loss": 2.7678,
      "step": 199650
    },
    {
      "activations/layer0_attention_weight_max": 15.522658348083496,
      "activations/layer0_attention_weight_min": -12.050002098083496,
      "activations/layer10_attention_weight_max": 36.13977813720703,
      "activations/layer10_attention_weight_min": -36.9472541809082,
      "activations/layer11_attention_weight_max": 39.22755432128906,
      "activations/layer11_attention_weight_min": -34.8223991394043,
      "activations/layer12_attention_weight_max": 24.135759353637695,
      "activations/layer12_attention_weight_min": -26.32249641418457,
      "activations/layer13_attention_weight_max": 37.85710906982422,
      "activations/layer13_attention_weight_min": -34.28550338745117,
      "activations/layer14_attention_weight_max": 42.84857177734375,
      "activations/layer14_attention_weight_min": -37.404014587402344,
      "activations/layer15_attention_weight_max": 36.415771484375,
      "activations/layer15_attention_weight_min": -32.442623138427734,
      "activations/layer16_attention_weight_max": 30.139968872070312,
      "activations/layer16_attention_weight_min": -26.88578987121582,
      "activations/layer17_attention_weight_max": 33.28373718261719,
      "activations/layer17_attention_weight_min": -25.83348274230957,
      "activations/layer18_attention_weight_max": 33.3528938293457,
      "activations/layer18_attention_weight_min": -24.6933650970459,
      "activations/layer19_attention_weight_max": 36.144752502441406,
      "activations/layer19_attention_weight_min": -34.17851257324219,
      "activations/layer1_attention_weight_max": 16.206857681274414,
      "activations/layer1_attention_weight_min": -15.117505073547363,
      "activations/layer20_attention_weight_max": 29.68023681640625,
      "activations/layer20_attention_weight_min": -25.744932174682617,
      "activations/layer21_attention_weight_max": 29.841503143310547,
      "activations/layer21_attention_weight_min": -24.806264877319336,
      "activations/layer22_attention_weight_max": 48.01871109008789,
      "activations/layer22_attention_weight_min": -32.42306137084961,
      "activations/layer23_attention_weight_max": 35.14425277709961,
      "activations/layer23_attention_weight_min": -27.82802391052246,
      "activations/layer2_attention_weight_max": 34.256385803222656,
      "activations/layer2_attention_weight_min": -35.246063232421875,
      "activations/layer3_attention_weight_max": 96.74750518798828,
      "activations/layer3_attention_weight_min": -101.64913177490234,
      "activations/layer4_attention_weight_max": 114.26918029785156,
      "activations/layer4_attention_weight_min": -119.17340087890625,
      "activations/layer5_attention_weight_max": 57.13786315917969,
      "activations/layer5_attention_weight_min": -62.12852478027344,
      "activations/layer6_attention_weight_max": 45.89643096923828,
      "activations/layer6_attention_weight_min": -48.4437370300293,
      "activations/layer7_attention_weight_max": 94.78694915771484,
      "activations/layer7_attention_weight_min": -101.39090728759766,
      "activations/layer8_attention_weight_max": 42.27888107299805,
      "activations/layer8_attention_weight_min": -46.726924896240234,
      "activations/layer9_attention_weight_max": 34.40375900268555,
      "activations/layer9_attention_weight_min": -35.1431999206543,
      "epoch": 11.6,
      "learning_rate": 7.593560606060606e-05,
      "loss": 2.7718,
      "step": 199700
    },
    {
      "activations/layer0_attention_weight_max": 15.007304191589355,
      "activations/layer0_attention_weight_min": -11.756906509399414,
      "activations/layer10_attention_weight_max": 44.80866622924805,
      "activations/layer10_attention_weight_min": -40.44082260131836,
      "activations/layer11_attention_weight_max": 46.16184997558594,
      "activations/layer11_attention_weight_min": -40.43865203857422,
      "activations/layer12_attention_weight_max": 26.977399826049805,
      "activations/layer12_attention_weight_min": -24.953245162963867,
      "activations/layer13_attention_weight_max": 49.74205780029297,
      "activations/layer13_attention_weight_min": -36.28562545776367,
      "activations/layer14_attention_weight_max": 44.3303337097168,
      "activations/layer14_attention_weight_min": -38.48481369018555,
      "activations/layer15_attention_weight_max": 44.93215560913086,
      "activations/layer15_attention_weight_min": -33.54391098022461,
      "activations/layer16_attention_weight_max": 32.85947036743164,
      "activations/layer16_attention_weight_min": -28.373844146728516,
      "activations/layer17_attention_weight_max": 31.107873916625977,
      "activations/layer17_attention_weight_min": -26.638086318969727,
      "activations/layer18_attention_weight_max": 33.37467956542969,
      "activations/layer18_attention_weight_min": -23.927692413330078,
      "activations/layer19_attention_weight_max": 37.80392074584961,
      "activations/layer19_attention_weight_min": -32.129371643066406,
      "activations/layer1_attention_weight_max": 16.778104782104492,
      "activations/layer1_attention_weight_min": -14.937936782836914,
      "activations/layer20_attention_weight_max": 31.317813873291016,
      "activations/layer20_attention_weight_min": -23.986324310302734,
      "activations/layer21_attention_weight_max": 32.933067321777344,
      "activations/layer21_attention_weight_min": -24.712270736694336,
      "activations/layer22_attention_weight_max": 47.91119384765625,
      "activations/layer22_attention_weight_min": -30.298311233520508,
      "activations/layer23_attention_weight_max": 35.02018737792969,
      "activations/layer23_attention_weight_min": -24.522899627685547,
      "activations/layer2_attention_weight_max": 35.687862396240234,
      "activations/layer2_attention_weight_min": -33.63488006591797,
      "activations/layer3_attention_weight_max": 90.96306610107422,
      "activations/layer3_attention_weight_min": -92.57400512695312,
      "activations/layer4_attention_weight_max": 105.8943099975586,
      "activations/layer4_attention_weight_min": -101.45184326171875,
      "activations/layer5_attention_weight_max": 55.03794860839844,
      "activations/layer5_attention_weight_min": -62.07050323486328,
      "activations/layer6_attention_weight_max": 45.61994552612305,
      "activations/layer6_attention_weight_min": -45.61123275756836,
      "activations/layer7_attention_weight_max": 105.69047546386719,
      "activations/layer7_attention_weight_min": -96.16090393066406,
      "activations/layer8_attention_weight_max": 48.65370559692383,
      "activations/layer8_attention_weight_min": -44.90431213378906,
      "activations/layer9_attention_weight_max": 43.702266693115234,
      "activations/layer9_attention_weight_min": -41.02454376220703,
      "epoch": 11.61,
      "learning_rate": 7.591666666666665e-05,
      "loss": 2.7925,
      "step": 199750
    },
    {
      "activations/layer0_attention_weight_max": 15.106285095214844,
      "activations/layer0_attention_weight_min": -11.771990776062012,
      "activations/layer10_attention_weight_max": 37.2755126953125,
      "activations/layer10_attention_weight_min": -32.63765335083008,
      "activations/layer11_attention_weight_max": 34.500267028808594,
      "activations/layer11_attention_weight_min": -31.614459991455078,
      "activations/layer12_attention_weight_max": 25.961511611938477,
      "activations/layer12_attention_weight_min": -26.849180221557617,
      "activations/layer13_attention_weight_max": 42.647117614746094,
      "activations/layer13_attention_weight_min": -35.81150436401367,
      "activations/layer14_attention_weight_max": 41.74803161621094,
      "activations/layer14_attention_weight_min": -40.707489013671875,
      "activations/layer15_attention_weight_max": 39.20637512207031,
      "activations/layer15_attention_weight_min": -34.53731155395508,
      "activations/layer16_attention_weight_max": 33.213356018066406,
      "activations/layer16_attention_weight_min": -29.413433074951172,
      "activations/layer17_attention_weight_max": 31.416629791259766,
      "activations/layer17_attention_weight_min": -30.27699851989746,
      "activations/layer18_attention_weight_max": 32.807193756103516,
      "activations/layer18_attention_weight_min": -23.675899505615234,
      "activations/layer19_attention_weight_max": 39.61272430419922,
      "activations/layer19_attention_weight_min": -31.791645050048828,
      "activations/layer1_attention_weight_max": 16.079721450805664,
      "activations/layer1_attention_weight_min": -14.14867877960205,
      "activations/layer20_attention_weight_max": 32.95616149902344,
      "activations/layer20_attention_weight_min": -26.91399383544922,
      "activations/layer21_attention_weight_max": 35.300453186035156,
      "activations/layer21_attention_weight_min": -25.62456703186035,
      "activations/layer22_attention_weight_max": 48.28009796142578,
      "activations/layer22_attention_weight_min": -30.341463088989258,
      "activations/layer23_attention_weight_max": 37.692230224609375,
      "activations/layer23_attention_weight_min": -24.77728843688965,
      "activations/layer2_attention_weight_max": 32.94870376586914,
      "activations/layer2_attention_weight_min": -33.13240051269531,
      "activations/layer3_attention_weight_max": 93.99945831298828,
      "activations/layer3_attention_weight_min": -91.78015899658203,
      "activations/layer4_attention_weight_max": 113.05516815185547,
      "activations/layer4_attention_weight_min": -110.56412506103516,
      "activations/layer5_attention_weight_max": 50.64558029174805,
      "activations/layer5_attention_weight_min": -66.219970703125,
      "activations/layer6_attention_weight_max": 46.743797302246094,
      "activations/layer6_attention_weight_min": -49.30552673339844,
      "activations/layer7_attention_weight_max": 96.51840209960938,
      "activations/layer7_attention_weight_min": -98.0045166015625,
      "activations/layer8_attention_weight_max": 41.40242004394531,
      "activations/layer8_attention_weight_min": -43.97281265258789,
      "activations/layer9_attention_weight_max": 35.10985565185547,
      "activations/layer9_attention_weight_min": -32.98732376098633,
      "epoch": 11.61,
      "learning_rate": 7.589772727272727e-05,
      "loss": 2.7804,
      "step": 199800
    },
    {
      "activations/layer0_attention_weight_max": 16.126495361328125,
      "activations/layer0_attention_weight_min": -12.542401313781738,
      "activations/layer10_attention_weight_max": 32.43006896972656,
      "activations/layer10_attention_weight_min": -35.81348419189453,
      "activations/layer11_attention_weight_max": 30.60909080505371,
      "activations/layer11_attention_weight_min": -33.297645568847656,
      "activations/layer12_attention_weight_max": 34.69184494018555,
      "activations/layer12_attention_weight_min": -25.289810180664062,
      "activations/layer13_attention_weight_max": 37.20918655395508,
      "activations/layer13_attention_weight_min": -37.08403396606445,
      "activations/layer14_attention_weight_max": 39.82923126220703,
      "activations/layer14_attention_weight_min": -40.48137283325195,
      "activations/layer15_attention_weight_max": 33.80129623413086,
      "activations/layer15_attention_weight_min": -32.705196380615234,
      "activations/layer16_attention_weight_max": 28.38434600830078,
      "activations/layer16_attention_weight_min": -27.85930633544922,
      "activations/layer17_attention_weight_max": 27.363313674926758,
      "activations/layer17_attention_weight_min": -25.634765625,
      "activations/layer18_attention_weight_max": 29.487445831298828,
      "activations/layer18_attention_weight_min": -24.718828201293945,
      "activations/layer19_attention_weight_max": 30.001270294189453,
      "activations/layer19_attention_weight_min": -29.136518478393555,
      "activations/layer1_attention_weight_max": 16.73476791381836,
      "activations/layer1_attention_weight_min": -16.2794189453125,
      "activations/layer20_attention_weight_max": 27.9229736328125,
      "activations/layer20_attention_weight_min": -24.399211883544922,
      "activations/layer21_attention_weight_max": 24.982341766357422,
      "activations/layer21_attention_weight_min": -24.137481689453125,
      "activations/layer22_attention_weight_max": 39.855003356933594,
      "activations/layer22_attention_weight_min": -28.549768447875977,
      "activations/layer23_attention_weight_max": 29.445722579956055,
      "activations/layer23_attention_weight_min": -24.465383529663086,
      "activations/layer2_attention_weight_max": 33.08817672729492,
      "activations/layer2_attention_weight_min": -33.65555953979492,
      "activations/layer3_attention_weight_max": 93.89117431640625,
      "activations/layer3_attention_weight_min": -93.79889678955078,
      "activations/layer4_attention_weight_max": 111.72945404052734,
      "activations/layer4_attention_weight_min": -113.85743713378906,
      "activations/layer5_attention_weight_max": 53.55289840698242,
      "activations/layer5_attention_weight_min": -61.258880615234375,
      "activations/layer6_attention_weight_max": 43.90331268310547,
      "activations/layer6_attention_weight_min": -50.24282455444336,
      "activations/layer7_attention_weight_max": 86.9567642211914,
      "activations/layer7_attention_weight_min": -98.52520751953125,
      "activations/layer8_attention_weight_max": 40.13057327270508,
      "activations/layer8_attention_weight_min": -45.36655807495117,
      "activations/layer9_attention_weight_max": 33.878421783447266,
      "activations/layer9_attention_weight_min": -36.1866569519043,
      "epoch": 11.61,
      "learning_rate": 7.587878787878788e-05,
      "loss": 2.7895,
      "step": 199850
    },
    {
      "activations/layer0_attention_weight_max": 15.05615520477295,
      "activations/layer0_attention_weight_min": -12.233614921569824,
      "activations/layer10_attention_weight_max": 35.32508087158203,
      "activations/layer10_attention_weight_min": -32.873931884765625,
      "activations/layer11_attention_weight_max": 32.249961853027344,
      "activations/layer11_attention_weight_min": -32.86211395263672,
      "activations/layer12_attention_weight_max": 25.639184951782227,
      "activations/layer12_attention_weight_min": -24.910600662231445,
      "activations/layer13_attention_weight_max": 40.96324920654297,
      "activations/layer13_attention_weight_min": -33.95771789550781,
      "activations/layer14_attention_weight_max": 43.75825881958008,
      "activations/layer14_attention_weight_min": -37.67213439941406,
      "activations/layer15_attention_weight_max": 37.15971755981445,
      "activations/layer15_attention_weight_min": -34.145668029785156,
      "activations/layer16_attention_weight_max": 33.211097717285156,
      "activations/layer16_attention_weight_min": -28.548587799072266,
      "activations/layer17_attention_weight_max": 33.635650634765625,
      "activations/layer17_attention_weight_min": -26.482540130615234,
      "activations/layer18_attention_weight_max": 34.7563362121582,
      "activations/layer18_attention_weight_min": -25.056133270263672,
      "activations/layer19_attention_weight_max": 32.98854064941406,
      "activations/layer19_attention_weight_min": -33.5855598449707,
      "activations/layer1_attention_weight_max": 16.868274688720703,
      "activations/layer1_attention_weight_min": -13.778922080993652,
      "activations/layer20_attention_weight_max": 30.798643112182617,
      "activations/layer20_attention_weight_min": -25.39419174194336,
      "activations/layer21_attention_weight_max": 32.08038330078125,
      "activations/layer21_attention_weight_min": -23.675437927246094,
      "activations/layer22_attention_weight_max": 43.363189697265625,
      "activations/layer22_attention_weight_min": -29.518884658813477,
      "activations/layer23_attention_weight_max": 33.11097717285156,
      "activations/layer23_attention_weight_min": -24.59050750732422,
      "activations/layer2_attention_weight_max": 35.263954162597656,
      "activations/layer2_attention_weight_min": -33.3549690246582,
      "activations/layer3_attention_weight_max": 92.26583862304688,
      "activations/layer3_attention_weight_min": -93.79434204101562,
      "activations/layer4_attention_weight_max": 112.35323333740234,
      "activations/layer4_attention_weight_min": -110.26836395263672,
      "activations/layer5_attention_weight_max": 54.845863342285156,
      "activations/layer5_attention_weight_min": -62.413002014160156,
      "activations/layer6_attention_weight_max": 45.82819366455078,
      "activations/layer6_attention_weight_min": -49.11271667480469,
      "activations/layer7_attention_weight_max": 90.63297271728516,
      "activations/layer7_attention_weight_min": -90.75129699707031,
      "activations/layer8_attention_weight_max": 41.520225524902344,
      "activations/layer8_attention_weight_min": -43.682498931884766,
      "activations/layer9_attention_weight_max": 34.17898178100586,
      "activations/layer9_attention_weight_min": -33.84727478027344,
      "epoch": 11.62,
      "learning_rate": 7.585984848484847e-05,
      "loss": 2.7616,
      "step": 199900
    },
    {
      "activations/layer0_attention_weight_max": 15.754002571105957,
      "activations/layer0_attention_weight_min": -13.387700080871582,
      "activations/layer10_attention_weight_max": 35.6134033203125,
      "activations/layer10_attention_weight_min": -35.48743438720703,
      "activations/layer11_attention_weight_max": 35.42290115356445,
      "activations/layer11_attention_weight_min": -34.59469223022461,
      "activations/layer12_attention_weight_max": 26.583492279052734,
      "activations/layer12_attention_weight_min": -26.707300186157227,
      "activations/layer13_attention_weight_max": 39.332855224609375,
      "activations/layer13_attention_weight_min": -36.800270080566406,
      "activations/layer14_attention_weight_max": 47.950836181640625,
      "activations/layer14_attention_weight_min": -41.20159149169922,
      "activations/layer15_attention_weight_max": 36.484230041503906,
      "activations/layer15_attention_weight_min": -32.491065979003906,
      "activations/layer16_attention_weight_max": 31.128921508789062,
      "activations/layer16_attention_weight_min": -29.965898513793945,
      "activations/layer17_attention_weight_max": 34.05099868774414,
      "activations/layer17_attention_weight_min": -28.027931213378906,
      "activations/layer18_attention_weight_max": 31.13682746887207,
      "activations/layer18_attention_weight_min": -23.671432495117188,
      "activations/layer19_attention_weight_max": 33.33467483520508,
      "activations/layer19_attention_weight_min": -31.80037498474121,
      "activations/layer1_attention_weight_max": 18.778039932250977,
      "activations/layer1_attention_weight_min": -16.496747970581055,
      "activations/layer20_attention_weight_max": 29.759361267089844,
      "activations/layer20_attention_weight_min": -27.093387603759766,
      "activations/layer21_attention_weight_max": 32.540401458740234,
      "activations/layer21_attention_weight_min": -25.412521362304688,
      "activations/layer22_attention_weight_max": 44.311424255371094,
      "activations/layer22_attention_weight_min": -32.686737060546875,
      "activations/layer23_attention_weight_max": 35.029441833496094,
      "activations/layer23_attention_weight_min": -25.93218994140625,
      "activations/layer2_attention_weight_max": 34.093379974365234,
      "activations/layer2_attention_weight_min": -32.20161437988281,
      "activations/layer3_attention_weight_max": 97.01311492919922,
      "activations/layer3_attention_weight_min": -96.511474609375,
      "activations/layer4_attention_weight_max": 117.0294418334961,
      "activations/layer4_attention_weight_min": -119.57124328613281,
      "activations/layer5_attention_weight_max": 53.15452194213867,
      "activations/layer5_attention_weight_min": -63.523223876953125,
      "activations/layer6_attention_weight_max": 45.90732192993164,
      "activations/layer6_attention_weight_min": -51.50565719604492,
      "activations/layer7_attention_weight_max": 96.07319641113281,
      "activations/layer7_attention_weight_min": -96.1287612915039,
      "activations/layer8_attention_weight_max": 42.01704025268555,
      "activations/layer8_attention_weight_min": -44.716312408447266,
      "activations/layer9_attention_weight_max": 33.770774841308594,
      "activations/layer9_attention_weight_min": -35.94388961791992,
      "epoch": 11.62,
      "learning_rate": 7.584090909090908e-05,
      "loss": 2.779,
      "step": 199950
    },
    {
      "activations/layer0_attention_weight_max": 15.289817810058594,
      "activations/layer0_attention_weight_min": -11.675570487976074,
      "activations/layer10_attention_weight_max": 34.127052307128906,
      "activations/layer10_attention_weight_min": -34.90721893310547,
      "activations/layer11_attention_weight_max": 33.02773666381836,
      "activations/layer11_attention_weight_min": -35.453636169433594,
      "activations/layer12_attention_weight_max": 27.108781814575195,
      "activations/layer12_attention_weight_min": -25.150482177734375,
      "activations/layer13_attention_weight_max": 39.14948654174805,
      "activations/layer13_attention_weight_min": -33.99111557006836,
      "activations/layer14_attention_weight_max": 43.59152603149414,
      "activations/layer14_attention_weight_min": -38.527984619140625,
      "activations/layer15_attention_weight_max": 36.76387023925781,
      "activations/layer15_attention_weight_min": -33.47338104248047,
      "activations/layer16_attention_weight_max": 31.80120086669922,
      "activations/layer16_attention_weight_min": -28.28466033935547,
      "activations/layer17_attention_weight_max": 32.827144622802734,
      "activations/layer17_attention_weight_min": -27.357267379760742,
      "activations/layer18_attention_weight_max": 30.18516731262207,
      "activations/layer18_attention_weight_min": -23.99631690979004,
      "activations/layer19_attention_weight_max": 33.646602630615234,
      "activations/layer19_attention_weight_min": -31.327224731445312,
      "activations/layer1_attention_weight_max": 16.500072479248047,
      "activations/layer1_attention_weight_min": -14.602066040039062,
      "activations/layer20_attention_weight_max": 30.7419376373291,
      "activations/layer20_attention_weight_min": -26.55281639099121,
      "activations/layer21_attention_weight_max": 28.646141052246094,
      "activations/layer21_attention_weight_min": -23.951797485351562,
      "activations/layer22_attention_weight_max": 47.13481903076172,
      "activations/layer22_attention_weight_min": -29.52723503112793,
      "activations/layer23_attention_weight_max": 32.96707534790039,
      "activations/layer23_attention_weight_min": -25.409366607666016,
      "activations/layer2_attention_weight_max": 33.438907623291016,
      "activations/layer2_attention_weight_min": -31.173519134521484,
      "activations/layer3_attention_weight_max": 94.31272888183594,
      "activations/layer3_attention_weight_min": -94.21312713623047,
      "activations/layer4_attention_weight_max": 114.1574478149414,
      "activations/layer4_attention_weight_min": -119.09339904785156,
      "activations/layer5_attention_weight_max": 54.64558029174805,
      "activations/layer5_attention_weight_min": -65.18545532226562,
      "activations/layer6_attention_weight_max": 44.98804473876953,
      "activations/layer6_attention_weight_min": -48.724143981933594,
      "activations/layer7_attention_weight_max": 92.24044799804688,
      "activations/layer7_attention_weight_min": -89.97004699707031,
      "activations/layer8_attention_weight_max": 39.95166015625,
      "activations/layer8_attention_weight_min": -40.57139587402344,
      "activations/layer9_attention_weight_max": 34.6038818359375,
      "activations/layer9_attention_weight_min": -35.11021041870117,
      "epoch": 11.62,
      "learning_rate": 7.58219696969697e-05,
      "loss": 2.7815,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.5264,
      "eval_samples_per_second": 503.612,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.5264,
      "eval_openwebtext_samples_per_second": 503.612,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.022,
      "eval_wikitext_samples_per_second": 225.518,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_lambada_loss": 2.556640625,
      "eval_lambada_ppl": 12.892433965519283,
      "eval_lambada_runtime": 9.6459,
      "eval_lambada_samples_per_second": 504.772,
      "step": 200000
    },
    {
      "activations/layer0_attention_weight_max": 14.934995651245117,
      "activations/layer0_attention_weight_min": -11.535694122314453,
      "activations/layer10_attention_weight_max": 29.452054977416992,
      "activations/layer10_attention_weight_min": -30.715251922607422,
      "activations/layer11_attention_weight_max": 30.5892391204834,
      "activations/layer11_attention_weight_min": -30.203697204589844,
      "activations/layer12_attention_weight_max": 22.27212142944336,
      "activations/layer12_attention_weight_min": -25.083335876464844,
      "activations/layer13_attention_weight_max": 36.64484405517578,
      "activations/layer13_attention_weight_min": -33.62980270385742,
      "activations/layer14_attention_weight_max": 39.682960510253906,
      "activations/layer14_attention_weight_min": -34.86885452270508,
      "activations/layer15_attention_weight_max": 34.639652252197266,
      "activations/layer15_attention_weight_min": -29.7429141998291,
      "activations/layer16_attention_weight_max": 32.520286560058594,
      "activations/layer16_attention_weight_min": -28.040918350219727,
      "activations/layer17_attention_weight_max": 29.108022689819336,
      "activations/layer17_attention_weight_min": -25.666521072387695,
      "activations/layer18_attention_weight_max": 30.09135627746582,
      "activations/layer18_attention_weight_min": -23.073205947875977,
      "activations/layer19_attention_weight_max": 30.524343490600586,
      "activations/layer19_attention_weight_min": -29.39263916015625,
      "activations/layer1_attention_weight_max": 15.935698509216309,
      "activations/layer1_attention_weight_min": -15.90445327758789,
      "activations/layer20_attention_weight_max": 28.10307502746582,
      "activations/layer20_attention_weight_min": -26.113298416137695,
      "activations/layer21_attention_weight_max": 28.47416114807129,
      "activations/layer21_attention_weight_min": -23.151168823242188,
      "activations/layer22_attention_weight_max": 40.893157958984375,
      "activations/layer22_attention_weight_min": -29.123884201049805,
      "activations/layer23_attention_weight_max": 31.12824821472168,
      "activations/layer23_attention_weight_min": -25.70210838317871,
      "activations/layer2_attention_weight_max": 33.11815643310547,
      "activations/layer2_attention_weight_min": -32.33713912963867,
      "activations/layer3_attention_weight_max": 96.1766586303711,
      "activations/layer3_attention_weight_min": -94.39295196533203,
      "activations/layer4_attention_weight_max": 112.47896575927734,
      "activations/layer4_attention_weight_min": -112.64375305175781,
      "activations/layer5_attention_weight_max": 52.442649841308594,
      "activations/layer5_attention_weight_min": -61.78672790527344,
      "activations/layer6_attention_weight_max": 44.25578689575195,
      "activations/layer6_attention_weight_min": -47.08013153076172,
      "activations/layer7_attention_weight_max": 84.42552947998047,
      "activations/layer7_attention_weight_min": -88.2284164428711,
      "activations/layer8_attention_weight_max": 38.36806106567383,
      "activations/layer8_attention_weight_min": -40.111656188964844,
      "activations/layer9_attention_weight_max": 30.069040298461914,
      "activations/layer9_attention_weight_min": -30.2220401763916,
      "epoch": 11.62,
      "learning_rate": 7.580303030303029e-05,
      "loss": 2.775,
      "step": 200050
    },
    {
      "activations/layer0_attention_weight_max": 15.309974670410156,
      "activations/layer0_attention_weight_min": -11.286762237548828,
      "activations/layer10_attention_weight_max": 36.73854446411133,
      "activations/layer10_attention_weight_min": -36.33879470825195,
      "activations/layer11_attention_weight_max": 34.99028778076172,
      "activations/layer11_attention_weight_min": -35.02655792236328,
      "activations/layer12_attention_weight_max": 28.22394371032715,
      "activations/layer12_attention_weight_min": -25.815235137939453,
      "activations/layer13_attention_weight_max": 40.34092330932617,
      "activations/layer13_attention_weight_min": -34.85881805419922,
      "activations/layer14_attention_weight_max": 44.79706573486328,
      "activations/layer14_attention_weight_min": -40.390933990478516,
      "activations/layer15_attention_weight_max": 37.419525146484375,
      "activations/layer15_attention_weight_min": -34.77693557739258,
      "activations/layer16_attention_weight_max": 32.13040542602539,
      "activations/layer16_attention_weight_min": -29.108154296875,
      "activations/layer17_attention_weight_max": 34.250247955322266,
      "activations/layer17_attention_weight_min": -25.964197158813477,
      "activations/layer18_attention_weight_max": 32.215728759765625,
      "activations/layer18_attention_weight_min": -23.015880584716797,
      "activations/layer19_attention_weight_max": 35.250892639160156,
      "activations/layer19_attention_weight_min": -32.10162353515625,
      "activations/layer1_attention_weight_max": 16.629934310913086,
      "activations/layer1_attention_weight_min": -15.802063941955566,
      "activations/layer20_attention_weight_max": 30.437217712402344,
      "activations/layer20_attention_weight_min": -26.5806941986084,
      "activations/layer21_attention_weight_max": 29.87320327758789,
      "activations/layer21_attention_weight_min": -26.76675033569336,
      "activations/layer22_attention_weight_max": 48.04981231689453,
      "activations/layer22_attention_weight_min": -36.141380310058594,
      "activations/layer23_attention_weight_max": 36.611671447753906,
      "activations/layer23_attention_weight_min": -25.277456283569336,
      "activations/layer2_attention_weight_max": 32.61849594116211,
      "activations/layer2_attention_weight_min": -31.003957748413086,
      "activations/layer3_attention_weight_max": 94.1997299194336,
      "activations/layer3_attention_weight_min": -96.71607971191406,
      "activations/layer4_attention_weight_max": 113.26110076904297,
      "activations/layer4_attention_weight_min": -117.95519256591797,
      "activations/layer5_attention_weight_max": 55.105194091796875,
      "activations/layer5_attention_weight_min": -62.07344055175781,
      "activations/layer6_attention_weight_max": 46.517642974853516,
      "activations/layer6_attention_weight_min": -49.113521575927734,
      "activations/layer7_attention_weight_max": 91.10643005371094,
      "activations/layer7_attention_weight_min": -96.19487762451172,
      "activations/layer8_attention_weight_max": 44.20635986328125,
      "activations/layer8_attention_weight_min": -45.43503952026367,
      "activations/layer9_attention_weight_max": 36.21244430541992,
      "activations/layer9_attention_weight_min": -37.03462600708008,
      "epoch": 11.63,
      "learning_rate": 7.57840909090909e-05,
      "loss": 2.7745,
      "step": 200100
    },
    {
      "activations/layer0_attention_weight_max": 15.367853164672852,
      "activations/layer0_attention_weight_min": -11.109842300415039,
      "activations/layer10_attention_weight_max": 34.67664337158203,
      "activations/layer10_attention_weight_min": -34.28325653076172,
      "activations/layer11_attention_weight_max": 33.7406005859375,
      "activations/layer11_attention_weight_min": -32.430763244628906,
      "activations/layer12_attention_weight_max": 24.15349006652832,
      "activations/layer12_attention_weight_min": -24.2618408203125,
      "activations/layer13_attention_weight_max": 36.429710388183594,
      "activations/layer13_attention_weight_min": -31.736547470092773,
      "activations/layer14_attention_weight_max": 40.77119064331055,
      "activations/layer14_attention_weight_min": -35.07942199707031,
      "activations/layer15_attention_weight_max": 36.02385330200195,
      "activations/layer15_attention_weight_min": -30.729528427124023,
      "activations/layer16_attention_weight_max": 31.482404708862305,
      "activations/layer16_attention_weight_min": -26.16133689880371,
      "activations/layer17_attention_weight_max": 32.27500915527344,
      "activations/layer17_attention_weight_min": -26.781814575195312,
      "activations/layer18_attention_weight_max": 29.278732299804688,
      "activations/layer18_attention_weight_min": -24.76510238647461,
      "activations/layer19_attention_weight_max": 36.096744537353516,
      "activations/layer19_attention_weight_min": -29.51803970336914,
      "activations/layer1_attention_weight_max": 16.030521392822266,
      "activations/layer1_attention_weight_min": -15.069389343261719,
      "activations/layer20_attention_weight_max": 31.29210090637207,
      "activations/layer20_attention_weight_min": -25.49370002746582,
      "activations/layer21_attention_weight_max": 31.18489646911621,
      "activations/layer21_attention_weight_min": -24.435752868652344,
      "activations/layer22_attention_weight_max": 45.02222442626953,
      "activations/layer22_attention_weight_min": -30.92728042602539,
      "activations/layer23_attention_weight_max": 35.196197509765625,
      "activations/layer23_attention_weight_min": -23.320083618164062,
      "activations/layer2_attention_weight_max": 33.22412872314453,
      "activations/layer2_attention_weight_min": -33.88806915283203,
      "activations/layer3_attention_weight_max": 94.65226745605469,
      "activations/layer3_attention_weight_min": -95.33772277832031,
      "activations/layer4_attention_weight_max": 112.83184051513672,
      "activations/layer4_attention_weight_min": -117.96895599365234,
      "activations/layer5_attention_weight_max": 53.081398010253906,
      "activations/layer5_attention_weight_min": -64.62963104248047,
      "activations/layer6_attention_weight_max": 43.88090515136719,
      "activations/layer6_attention_weight_min": -47.920310974121094,
      "activations/layer7_attention_weight_max": 89.90569305419922,
      "activations/layer7_attention_weight_min": -91.82892608642578,
      "activations/layer8_attention_weight_max": 43.60518264770508,
      "activations/layer8_attention_weight_min": -42.98073196411133,
      "activations/layer9_attention_weight_max": 32.70615768432617,
      "activations/layer9_attention_weight_min": -33.571041107177734,
      "epoch": 11.63,
      "learning_rate": 7.57651515151515e-05,
      "loss": 2.7756,
      "step": 200150
    },
    {
      "activations/layer0_attention_weight_max": 15.631132125854492,
      "activations/layer0_attention_weight_min": -11.983949661254883,
      "activations/layer10_attention_weight_max": 40.44746398925781,
      "activations/layer10_attention_weight_min": -36.75994110107422,
      "activations/layer11_attention_weight_max": 41.43992614746094,
      "activations/layer11_attention_weight_min": -36.640743255615234,
      "activations/layer12_attention_weight_max": 28.561546325683594,
      "activations/layer12_attention_weight_min": -26.148948669433594,
      "activations/layer13_attention_weight_max": 45.03733825683594,
      "activations/layer13_attention_weight_min": -36.14131546020508,
      "activations/layer14_attention_weight_max": 49.67600631713867,
      "activations/layer14_attention_weight_min": -39.102989196777344,
      "activations/layer15_attention_weight_max": 42.368133544921875,
      "activations/layer15_attention_weight_min": -33.0544548034668,
      "activations/layer16_attention_weight_max": 33.95732879638672,
      "activations/layer16_attention_weight_min": -27.323867797851562,
      "activations/layer17_attention_weight_max": 36.33245849609375,
      "activations/layer17_attention_weight_min": -25.32637596130371,
      "activations/layer18_attention_weight_max": 31.443405151367188,
      "activations/layer18_attention_weight_min": -24.55632972717285,
      "activations/layer19_attention_weight_max": 37.23627853393555,
      "activations/layer19_attention_weight_min": -32.03956985473633,
      "activations/layer1_attention_weight_max": 17.116683959960938,
      "activations/layer1_attention_weight_min": -16.43242645263672,
      "activations/layer20_attention_weight_max": 32.739410400390625,
      "activations/layer20_attention_weight_min": -24.56968879699707,
      "activations/layer21_attention_weight_max": 31.268957138061523,
      "activations/layer21_attention_weight_min": -24.04710578918457,
      "activations/layer22_attention_weight_max": 49.86257553100586,
      "activations/layer22_attention_weight_min": -29.346324920654297,
      "activations/layer23_attention_weight_max": 36.4238395690918,
      "activations/layer23_attention_weight_min": -25.322011947631836,
      "activations/layer2_attention_weight_max": 34.19170379638672,
      "activations/layer2_attention_weight_min": -32.07685852050781,
      "activations/layer3_attention_weight_max": 99.98531341552734,
      "activations/layer3_attention_weight_min": -98.83277893066406,
      "activations/layer4_attention_weight_max": 119.3780746459961,
      "activations/layer4_attention_weight_min": -117.95562744140625,
      "activations/layer5_attention_weight_max": 55.79232406616211,
      "activations/layer5_attention_weight_min": -64.3010482788086,
      "activations/layer6_attention_weight_max": 48.66239547729492,
      "activations/layer6_attention_weight_min": -51.52399826049805,
      "activations/layer7_attention_weight_max": 98.01972198486328,
      "activations/layer7_attention_weight_min": -101.99443054199219,
      "activations/layer8_attention_weight_max": 44.53683853149414,
      "activations/layer8_attention_weight_min": -44.14642333984375,
      "activations/layer9_attention_weight_max": 41.05717086791992,
      "activations/layer9_attention_weight_min": -38.41721725463867,
      "epoch": 11.63,
      "learning_rate": 7.574621212121212e-05,
      "loss": 2.7746,
      "step": 200200
    },
    {
      "activations/layer0_attention_weight_max": 15.795333862304688,
      "activations/layer0_attention_weight_min": -12.575393676757812,
      "activations/layer10_attention_weight_max": 33.841331481933594,
      "activations/layer10_attention_weight_min": -34.64716339111328,
      "activations/layer11_attention_weight_max": 33.083404541015625,
      "activations/layer11_attention_weight_min": -32.033447265625,
      "activations/layer12_attention_weight_max": 24.27206802368164,
      "activations/layer12_attention_weight_min": -25.61441421508789,
      "activations/layer13_attention_weight_max": 38.38285827636719,
      "activations/layer13_attention_weight_min": -35.254512786865234,
      "activations/layer14_attention_weight_max": 45.02964401245117,
      "activations/layer14_attention_weight_min": -39.68735885620117,
      "activations/layer15_attention_weight_max": 37.91657638549805,
      "activations/layer15_attention_weight_min": -33.24676513671875,
      "activations/layer16_attention_weight_max": 31.695728302001953,
      "activations/layer16_attention_weight_min": -26.878618240356445,
      "activations/layer17_attention_weight_max": 31.36970329284668,
      "activations/layer17_attention_weight_min": -26.138967514038086,
      "activations/layer18_attention_weight_max": 33.378570556640625,
      "activations/layer18_attention_weight_min": -25.866939544677734,
      "activations/layer19_attention_weight_max": 40.23162078857422,
      "activations/layer19_attention_weight_min": -33.042598724365234,
      "activations/layer1_attention_weight_max": 16.39752769470215,
      "activations/layer1_attention_weight_min": -13.878913879394531,
      "activations/layer20_attention_weight_max": 33.35841751098633,
      "activations/layer20_attention_weight_min": -26.10640525817871,
      "activations/layer21_attention_weight_max": 32.22282791137695,
      "activations/layer21_attention_weight_min": -24.61387062072754,
      "activations/layer22_attention_weight_max": 48.94664001464844,
      "activations/layer22_attention_weight_min": -31.49750518798828,
      "activations/layer23_attention_weight_max": 41.42450714111328,
      "activations/layer23_attention_weight_min": -26.295528411865234,
      "activations/layer2_attention_weight_max": 31.863218307495117,
      "activations/layer2_attention_weight_min": -31.940702438354492,
      "activations/layer3_attention_weight_max": 93.68113708496094,
      "activations/layer3_attention_weight_min": -93.80426025390625,
      "activations/layer4_attention_weight_max": 111.6686782836914,
      "activations/layer4_attention_weight_min": -112.69815826416016,
      "activations/layer5_attention_weight_max": 53.22834014892578,
      "activations/layer5_attention_weight_min": -59.44877624511719,
      "activations/layer6_attention_weight_max": 44.3498420715332,
      "activations/layer6_attention_weight_min": -50.4788703918457,
      "activations/layer7_attention_weight_max": 87.3659896850586,
      "activations/layer7_attention_weight_min": -87.87693786621094,
      "activations/layer8_attention_weight_max": 41.14058303833008,
      "activations/layer8_attention_weight_min": -43.164573669433594,
      "activations/layer9_attention_weight_max": 34.52701950073242,
      "activations/layer9_attention_weight_min": -35.12781524658203,
      "epoch": 11.64,
      "learning_rate": 7.572727272727272e-05,
      "loss": 2.7705,
      "step": 200250
    },
    {
      "activations/layer0_attention_weight_max": 15.425288200378418,
      "activations/layer0_attention_weight_min": -11.882230758666992,
      "activations/layer10_attention_weight_max": 31.309219360351562,
      "activations/layer10_attention_weight_min": -32.05398941040039,
      "activations/layer11_attention_weight_max": 32.67515563964844,
      "activations/layer11_attention_weight_min": -34.43682098388672,
      "activations/layer12_attention_weight_max": 26.916650772094727,
      "activations/layer12_attention_weight_min": -24.55290985107422,
      "activations/layer13_attention_weight_max": 40.66416931152344,
      "activations/layer13_attention_weight_min": -34.95983123779297,
      "activations/layer14_attention_weight_max": 50.2519645690918,
      "activations/layer14_attention_weight_min": -41.94844055175781,
      "activations/layer15_attention_weight_max": 40.45838928222656,
      "activations/layer15_attention_weight_min": -32.3726806640625,
      "activations/layer16_attention_weight_max": 32.97140884399414,
      "activations/layer16_attention_weight_min": -28.280803680419922,
      "activations/layer17_attention_weight_max": 33.43083572387695,
      "activations/layer17_attention_weight_min": -24.48530387878418,
      "activations/layer18_attention_weight_max": 32.643402099609375,
      "activations/layer18_attention_weight_min": -22.811925888061523,
      "activations/layer19_attention_weight_max": 34.66569137573242,
      "activations/layer19_attention_weight_min": -31.173086166381836,
      "activations/layer1_attention_weight_max": 17.064319610595703,
      "activations/layer1_attention_weight_min": -14.928067207336426,
      "activations/layer20_attention_weight_max": 28.72503662109375,
      "activations/layer20_attention_weight_min": -26.896223068237305,
      "activations/layer21_attention_weight_max": 27.41410255432129,
      "activations/layer21_attention_weight_min": -23.162124633789062,
      "activations/layer22_attention_weight_max": 40.88396072387695,
      "activations/layer22_attention_weight_min": -29.005329132080078,
      "activations/layer23_attention_weight_max": 35.04541778564453,
      "activations/layer23_attention_weight_min": -23.850074768066406,
      "activations/layer2_attention_weight_max": 32.61106872558594,
      "activations/layer2_attention_weight_min": -31.643712997436523,
      "activations/layer3_attention_weight_max": 97.27491760253906,
      "activations/layer3_attention_weight_min": -95.91506958007812,
      "activations/layer4_attention_weight_max": 113.37789154052734,
      "activations/layer4_attention_weight_min": -112.63631439208984,
      "activations/layer5_attention_weight_max": 51.186248779296875,
      "activations/layer5_attention_weight_min": -63.791221618652344,
      "activations/layer6_attention_weight_max": 44.51898956298828,
      "activations/layer6_attention_weight_min": -49.20090866088867,
      "activations/layer7_attention_weight_max": 91.28561401367188,
      "activations/layer7_attention_weight_min": -93.7025375366211,
      "activations/layer8_attention_weight_max": 43.030582427978516,
      "activations/layer8_attention_weight_min": -41.66300582885742,
      "activations/layer9_attention_weight_max": 31.41141700744629,
      "activations/layer9_attention_weight_min": -32.368255615234375,
      "epoch": 11.64,
      "learning_rate": 7.570833333333332e-05,
      "loss": 2.782,
      "step": 200300
    },
    {
      "activations/layer0_attention_weight_max": 15.178654670715332,
      "activations/layer0_attention_weight_min": -11.619791030883789,
      "activations/layer10_attention_weight_max": 39.899444580078125,
      "activations/layer10_attention_weight_min": -34.20053482055664,
      "activations/layer11_attention_weight_max": 45.853736877441406,
      "activations/layer11_attention_weight_min": -35.764408111572266,
      "activations/layer12_attention_weight_max": 25.899518966674805,
      "activations/layer12_attention_weight_min": -32.81005096435547,
      "activations/layer13_attention_weight_max": 39.315284729003906,
      "activations/layer13_attention_weight_min": -38.05662155151367,
      "activations/layer14_attention_weight_max": 47.418766021728516,
      "activations/layer14_attention_weight_min": -47.9608268737793,
      "activations/layer15_attention_weight_max": 41.365848541259766,
      "activations/layer15_attention_weight_min": -36.584991455078125,
      "activations/layer16_attention_weight_max": 32.91579818725586,
      "activations/layer16_attention_weight_min": -28.13511848449707,
      "activations/layer17_attention_weight_max": 33.6313362121582,
      "activations/layer17_attention_weight_min": -29.124786376953125,
      "activations/layer18_attention_weight_max": 32.179595947265625,
      "activations/layer18_attention_weight_min": -23.748554229736328,
      "activations/layer19_attention_weight_max": 34.363304138183594,
      "activations/layer19_attention_weight_min": -33.07932662963867,
      "activations/layer1_attention_weight_max": 17.303367614746094,
      "activations/layer1_attention_weight_min": -13.827194213867188,
      "activations/layer20_attention_weight_max": 30.058120727539062,
      "activations/layer20_attention_weight_min": -25.201004028320312,
      "activations/layer21_attention_weight_max": 29.697715759277344,
      "activations/layer21_attention_weight_min": -24.512359619140625,
      "activations/layer22_attention_weight_max": 48.68431091308594,
      "activations/layer22_attention_weight_min": -31.56507110595703,
      "activations/layer23_attention_weight_max": 35.53192901611328,
      "activations/layer23_attention_weight_min": -27.132640838623047,
      "activations/layer2_attention_weight_max": 33.48925018310547,
      "activations/layer2_attention_weight_min": -31.64452362060547,
      "activations/layer3_attention_weight_max": 95.87660217285156,
      "activations/layer3_attention_weight_min": -98.01220703125,
      "activations/layer4_attention_weight_max": 114.07189178466797,
      "activations/layer4_attention_weight_min": -111.2689437866211,
      "activations/layer5_attention_weight_max": 51.94178771972656,
      "activations/layer5_attention_weight_min": -59.332176208496094,
      "activations/layer6_attention_weight_max": 44.70414733886719,
      "activations/layer6_attention_weight_min": -46.50767135620117,
      "activations/layer7_attention_weight_max": 102.70738220214844,
      "activations/layer7_attention_weight_min": -87.25836181640625,
      "activations/layer8_attention_weight_max": 41.334632873535156,
      "activations/layer8_attention_weight_min": -41.55921173095703,
      "activations/layer9_attention_weight_max": 33.711585998535156,
      "activations/layer9_attention_weight_min": -33.81528091430664,
      "epoch": 11.64,
      "learning_rate": 7.568939393939394e-05,
      "loss": 2.7742,
      "step": 200350
    },
    {
      "activations/layer0_attention_weight_max": 15.825676918029785,
      "activations/layer0_attention_weight_min": -11.316701889038086,
      "activations/layer10_attention_weight_max": 32.92640686035156,
      "activations/layer10_attention_weight_min": -36.06126403808594,
      "activations/layer11_attention_weight_max": 34.875762939453125,
      "activations/layer11_attention_weight_min": -32.37010192871094,
      "activations/layer12_attention_weight_max": 25.344602584838867,
      "activations/layer12_attention_weight_min": -29.57061195373535,
      "activations/layer13_attention_weight_max": 38.13290023803711,
      "activations/layer13_attention_weight_min": -36.128662109375,
      "activations/layer14_attention_weight_max": 44.7432861328125,
      "activations/layer14_attention_weight_min": -39.05637741088867,
      "activations/layer15_attention_weight_max": 36.311439514160156,
      "activations/layer15_attention_weight_min": -31.842021942138672,
      "activations/layer16_attention_weight_max": 30.320711135864258,
      "activations/layer16_attention_weight_min": -29.075374603271484,
      "activations/layer17_attention_weight_max": 33.44243621826172,
      "activations/layer17_attention_weight_min": -26.72588348388672,
      "activations/layer18_attention_weight_max": 33.87208557128906,
      "activations/layer18_attention_weight_min": -24.194719314575195,
      "activations/layer19_attention_weight_max": 37.5706672668457,
      "activations/layer19_attention_weight_min": -30.26388931274414,
      "activations/layer1_attention_weight_max": 17.1886043548584,
      "activations/layer1_attention_weight_min": -12.219542503356934,
      "activations/layer20_attention_weight_max": 30.703874588012695,
      "activations/layer20_attention_weight_min": -24.692399978637695,
      "activations/layer21_attention_weight_max": 29.02114486694336,
      "activations/layer21_attention_weight_min": -23.43955421447754,
      "activations/layer22_attention_weight_max": 49.19223403930664,
      "activations/layer22_attention_weight_min": -29.47004508972168,
      "activations/layer23_attention_weight_max": 39.2666015625,
      "activations/layer23_attention_weight_min": -25.875808715820312,
      "activations/layer2_attention_weight_max": 34.210975646972656,
      "activations/layer2_attention_weight_min": -33.15978240966797,
      "activations/layer3_attention_weight_max": 95.57374572753906,
      "activations/layer3_attention_weight_min": -98.60395050048828,
      "activations/layer4_attention_weight_max": 113.01466369628906,
      "activations/layer4_attention_weight_min": -114.4720230102539,
      "activations/layer5_attention_weight_max": 52.16979217529297,
      "activations/layer5_attention_weight_min": -63.5294303894043,
      "activations/layer6_attention_weight_max": 44.79161071777344,
      "activations/layer6_attention_weight_min": -51.63491439819336,
      "activations/layer7_attention_weight_max": 91.57148742675781,
      "activations/layer7_attention_weight_min": -95.07862854003906,
      "activations/layer8_attention_weight_max": 41.83260726928711,
      "activations/layer8_attention_weight_min": -41.18608474731445,
      "activations/layer9_attention_weight_max": 32.91413497924805,
      "activations/layer9_attention_weight_min": -34.69102096557617,
      "epoch": 11.64,
      "learning_rate": 7.567083333333333e-05,
      "loss": 2.788,
      "step": 200400
    },
    {
      "activations/layer0_attention_weight_max": 15.998924255371094,
      "activations/layer0_attention_weight_min": -11.972228050231934,
      "activations/layer10_attention_weight_max": 31.791234970092773,
      "activations/layer10_attention_weight_min": -31.164724349975586,
      "activations/layer11_attention_weight_max": 32.01895523071289,
      "activations/layer11_attention_weight_min": -29.64077377319336,
      "activations/layer12_attention_weight_max": 22.67034912109375,
      "activations/layer12_attention_weight_min": -25.986127853393555,
      "activations/layer13_attention_weight_max": 34.49893569946289,
      "activations/layer13_attention_weight_min": -34.528228759765625,
      "activations/layer14_attention_weight_max": 40.459651947021484,
      "activations/layer14_attention_weight_min": -35.96685028076172,
      "activations/layer15_attention_weight_max": 32.375816345214844,
      "activations/layer15_attention_weight_min": -31.6539306640625,
      "activations/layer16_attention_weight_max": 29.400318145751953,
      "activations/layer16_attention_weight_min": -29.089017868041992,
      "activations/layer17_attention_weight_max": 29.04722785949707,
      "activations/layer17_attention_weight_min": -24.975778579711914,
      "activations/layer18_attention_weight_max": 31.690326690673828,
      "activations/layer18_attention_weight_min": -23.7132568359375,
      "activations/layer19_attention_weight_max": 37.3784294128418,
      "activations/layer19_attention_weight_min": -31.946491241455078,
      "activations/layer1_attention_weight_max": 19.17160415649414,
      "activations/layer1_attention_weight_min": -17.28502655029297,
      "activations/layer20_attention_weight_max": 31.80306053161621,
      "activations/layer20_attention_weight_min": -25.725725173950195,
      "activations/layer21_attention_weight_max": 29.03891372680664,
      "activations/layer21_attention_weight_min": -24.555837631225586,
      "activations/layer22_attention_weight_max": 44.37454605102539,
      "activations/layer22_attention_weight_min": -30.00883674621582,
      "activations/layer23_attention_weight_max": 35.49273681640625,
      "activations/layer23_attention_weight_min": -26.618221282958984,
      "activations/layer2_attention_weight_max": 30.082645416259766,
      "activations/layer2_attention_weight_min": -31.69400405883789,
      "activations/layer3_attention_weight_max": 89.41522979736328,
      "activations/layer3_attention_weight_min": -93.96656799316406,
      "activations/layer4_attention_weight_max": 108.6699447631836,
      "activations/layer4_attention_weight_min": -106.80157470703125,
      "activations/layer5_attention_weight_max": 49.955013275146484,
      "activations/layer5_attention_weight_min": -65.7303466796875,
      "activations/layer6_attention_weight_max": 44.73857498168945,
      "activations/layer6_attention_weight_min": -47.71438980102539,
      "activations/layer7_attention_weight_max": 82.3488540649414,
      "activations/layer7_attention_weight_min": -84.91050720214844,
      "activations/layer8_attention_weight_max": 37.744659423828125,
      "activations/layer8_attention_weight_min": -42.58656692504883,
      "activations/layer9_attention_weight_max": 30.867551803588867,
      "activations/layer9_attention_weight_min": -33.021400451660156,
      "epoch": 11.65,
      "learning_rate": 7.565227272727272e-05,
      "loss": 2.7663,
      "step": 200450
    },
    {
      "activations/layer0_attention_weight_max": 15.558183670043945,
      "activations/layer0_attention_weight_min": -12.563552856445312,
      "activations/layer10_attention_weight_max": 32.8875732421875,
      "activations/layer10_attention_weight_min": -31.793907165527344,
      "activations/layer11_attention_weight_max": 33.797119140625,
      "activations/layer11_attention_weight_min": -32.94340515136719,
      "activations/layer12_attention_weight_max": 26.104856491088867,
      "activations/layer12_attention_weight_min": -24.57546615600586,
      "activations/layer13_attention_weight_max": 40.80385208129883,
      "activations/layer13_attention_weight_min": -32.757320404052734,
      "activations/layer14_attention_weight_max": 49.50498962402344,
      "activations/layer14_attention_weight_min": -36.9514045715332,
      "activations/layer15_attention_weight_max": 40.39952850341797,
      "activations/layer15_attention_weight_min": -31.99226951599121,
      "activations/layer16_attention_weight_max": 31.805269241333008,
      "activations/layer16_attention_weight_min": -26.805936813354492,
      "activations/layer17_attention_weight_max": 30.50275421142578,
      "activations/layer17_attention_weight_min": -25.150461196899414,
      "activations/layer18_attention_weight_max": 29.587316513061523,
      "activations/layer18_attention_weight_min": -23.61406135559082,
      "activations/layer19_attention_weight_max": 33.109642028808594,
      "activations/layer19_attention_weight_min": -32.70316696166992,
      "activations/layer1_attention_weight_max": 16.23828125,
      "activations/layer1_attention_weight_min": -12.325549125671387,
      "activations/layer20_attention_weight_max": 25.32134437561035,
      "activations/layer20_attention_weight_min": -25.178630828857422,
      "activations/layer21_attention_weight_max": 24.014068603515625,
      "activations/layer21_attention_weight_min": -25.090190887451172,
      "activations/layer22_attention_weight_max": 39.04648208618164,
      "activations/layer22_attention_weight_min": -32.48164367675781,
      "activations/layer23_attention_weight_max": 29.981544494628906,
      "activations/layer23_attention_weight_min": -27.074148178100586,
      "activations/layer2_attention_weight_max": 32.59547424316406,
      "activations/layer2_attention_weight_min": -32.01328659057617,
      "activations/layer3_attention_weight_max": 93.70843505859375,
      "activations/layer3_attention_weight_min": -96.46841430664062,
      "activations/layer4_attention_weight_max": 114.57916259765625,
      "activations/layer4_attention_weight_min": -112.2785873413086,
      "activations/layer5_attention_weight_max": 51.04340362548828,
      "activations/layer5_attention_weight_min": -59.916709899902344,
      "activations/layer6_attention_weight_max": 44.49319839477539,
      "activations/layer6_attention_weight_min": -46.55134963989258,
      "activations/layer7_attention_weight_max": 90.72191619873047,
      "activations/layer7_attention_weight_min": -94.26666259765625,
      "activations/layer8_attention_weight_max": 39.87187576293945,
      "activations/layer8_attention_weight_min": -41.90302276611328,
      "activations/layer9_attention_weight_max": 33.135345458984375,
      "activations/layer9_attention_weight_min": -32.802738189697266,
      "epoch": 11.65,
      "learning_rate": 7.563333333333333e-05,
      "loss": 2.8044,
      "step": 200500
    },
    {
      "activations/layer0_attention_weight_max": 14.862593650817871,
      "activations/layer0_attention_weight_min": -11.875481605529785,
      "activations/layer10_attention_weight_max": 36.6694221496582,
      "activations/layer10_attention_weight_min": -36.354026794433594,
      "activations/layer11_attention_weight_max": 34.042030334472656,
      "activations/layer11_attention_weight_min": -34.879642486572266,
      "activations/layer12_attention_weight_max": 29.492475509643555,
      "activations/layer12_attention_weight_min": -28.500446319580078,
      "activations/layer13_attention_weight_max": 44.9553108215332,
      "activations/layer13_attention_weight_min": -37.68653869628906,
      "activations/layer14_attention_weight_max": 52.01116943359375,
      "activations/layer14_attention_weight_min": -43.410545349121094,
      "activations/layer15_attention_weight_max": 39.57381057739258,
      "activations/layer15_attention_weight_min": -33.457542419433594,
      "activations/layer16_attention_weight_max": 33.18207931518555,
      "activations/layer16_attention_weight_min": -27.165569305419922,
      "activations/layer17_attention_weight_max": 34.547752380371094,
      "activations/layer17_attention_weight_min": -26.155038833618164,
      "activations/layer18_attention_weight_max": 33.72773361206055,
      "activations/layer18_attention_weight_min": -23.877552032470703,
      "activations/layer19_attention_weight_max": 40.465065002441406,
      "activations/layer19_attention_weight_min": -31.754871368408203,
      "activations/layer1_attention_weight_max": 16.549840927124023,
      "activations/layer1_attention_weight_min": -14.936570167541504,
      "activations/layer20_attention_weight_max": 32.78709030151367,
      "activations/layer20_attention_weight_min": -25.20768928527832,
      "activations/layer21_attention_weight_max": 33.95166778564453,
      "activations/layer21_attention_weight_min": -23.382902145385742,
      "activations/layer22_attention_weight_max": 49.12141418457031,
      "activations/layer22_attention_weight_min": -31.739501953125,
      "activations/layer23_attention_weight_max": 40.326358795166016,
      "activations/layer23_attention_weight_min": -29.39344024658203,
      "activations/layer2_attention_weight_max": 34.39529800415039,
      "activations/layer2_attention_weight_min": -33.120574951171875,
      "activations/layer3_attention_weight_max": 105.11564636230469,
      "activations/layer3_attention_weight_min": -101.79622650146484,
      "activations/layer4_attention_weight_max": 120.8645248413086,
      "activations/layer4_attention_weight_min": -118.88124084472656,
      "activations/layer5_attention_weight_max": 56.38279342651367,
      "activations/layer5_attention_weight_min": -63.821876525878906,
      "activations/layer6_attention_weight_max": 49.86909484863281,
      "activations/layer6_attention_weight_min": -51.548221588134766,
      "activations/layer7_attention_weight_max": 98.27155303955078,
      "activations/layer7_attention_weight_min": -98.29866790771484,
      "activations/layer8_attention_weight_max": 45.41560745239258,
      "activations/layer8_attention_weight_min": -43.915489196777344,
      "activations/layer9_attention_weight_max": 35.930782318115234,
      "activations/layer9_attention_weight_min": -36.24860382080078,
      "epoch": 11.65,
      "learning_rate": 7.561439393939392e-05,
      "loss": 2.7789,
      "step": 200550
    },
    {
      "activations/layer0_attention_weight_max": 15.358698844909668,
      "activations/layer0_attention_weight_min": -12.616522789001465,
      "activations/layer10_attention_weight_max": 37.03141784667969,
      "activations/layer10_attention_weight_min": -34.40887451171875,
      "activations/layer11_attention_weight_max": 34.75466537475586,
      "activations/layer11_attention_weight_min": -33.57870864868164,
      "activations/layer12_attention_weight_max": 26.13361930847168,
      "activations/layer12_attention_weight_min": -25.01251220703125,
      "activations/layer13_attention_weight_max": 41.099037170410156,
      "activations/layer13_attention_weight_min": -33.5622444152832,
      "activations/layer14_attention_weight_max": 46.66261291503906,
      "activations/layer14_attention_weight_min": -38.369842529296875,
      "activations/layer15_attention_weight_max": 35.8177604675293,
      "activations/layer15_attention_weight_min": -30.598041534423828,
      "activations/layer16_attention_weight_max": 33.06632614135742,
      "activations/layer16_attention_weight_min": -26.736074447631836,
      "activations/layer17_attention_weight_max": 32.83763885498047,
      "activations/layer17_attention_weight_min": -25.851673126220703,
      "activations/layer18_attention_weight_max": 32.241912841796875,
      "activations/layer18_attention_weight_min": -24.311885833740234,
      "activations/layer19_attention_weight_max": 37.82624816894531,
      "activations/layer19_attention_weight_min": -32.2322883605957,
      "activations/layer1_attention_weight_max": 17.313030242919922,
      "activations/layer1_attention_weight_min": -14.768534660339355,
      "activations/layer20_attention_weight_max": 30.287878036499023,
      "activations/layer20_attention_weight_min": -27.35055160522461,
      "activations/layer21_attention_weight_max": 31.326444625854492,
      "activations/layer21_attention_weight_min": -25.536277770996094,
      "activations/layer22_attention_weight_max": 54.45985412597656,
      "activations/layer22_attention_weight_min": -32.55255889892578,
      "activations/layer23_attention_weight_max": 36.50432205200195,
      "activations/layer23_attention_weight_min": -24.678455352783203,
      "activations/layer2_attention_weight_max": 33.45001220703125,
      "activations/layer2_attention_weight_min": -31.93492889404297,
      "activations/layer3_attention_weight_max": 99.28063201904297,
      "activations/layer3_attention_weight_min": -97.03984832763672,
      "activations/layer4_attention_weight_max": 113.53466033935547,
      "activations/layer4_attention_weight_min": -111.39824676513672,
      "activations/layer5_attention_weight_max": 55.38508605957031,
      "activations/layer5_attention_weight_min": -62.3418083190918,
      "activations/layer6_attention_weight_max": 45.7473258972168,
      "activations/layer6_attention_weight_min": -49.62612533569336,
      "activations/layer7_attention_weight_max": 89.93043518066406,
      "activations/layer7_attention_weight_min": -91.52798461914062,
      "activations/layer8_attention_weight_max": 43.15393829345703,
      "activations/layer8_attention_weight_min": -44.12531280517578,
      "activations/layer9_attention_weight_max": 37.77299118041992,
      "activations/layer9_attention_weight_min": -35.58323669433594,
      "epoch": 11.66,
      "learning_rate": 7.559545454545454e-05,
      "loss": 2.7815,
      "step": 200600
    },
    {
      "activations/layer0_attention_weight_max": 15.200760841369629,
      "activations/layer0_attention_weight_min": -12.044281005859375,
      "activations/layer10_attention_weight_max": 34.92951202392578,
      "activations/layer10_attention_weight_min": -32.843902587890625,
      "activations/layer11_attention_weight_max": 31.900312423706055,
      "activations/layer11_attention_weight_min": -31.750545501708984,
      "activations/layer12_attention_weight_max": 24.317813873291016,
      "activations/layer12_attention_weight_min": -24.966766357421875,
      "activations/layer13_attention_weight_max": 37.61428451538086,
      "activations/layer13_attention_weight_min": -33.99319076538086,
      "activations/layer14_attention_weight_max": 44.57787322998047,
      "activations/layer14_attention_weight_min": -39.37484359741211,
      "activations/layer15_attention_weight_max": 37.838871002197266,
      "activations/layer15_attention_weight_min": -31.89378547668457,
      "activations/layer16_attention_weight_max": 31.147457122802734,
      "activations/layer16_attention_weight_min": -27.456335067749023,
      "activations/layer17_attention_weight_max": 32.687721252441406,
      "activations/layer17_attention_weight_min": -24.23887062072754,
      "activations/layer18_attention_weight_max": 31.417964935302734,
      "activations/layer18_attention_weight_min": -22.60150909423828,
      "activations/layer19_attention_weight_max": 35.123294830322266,
      "activations/layer19_attention_weight_min": -30.58723258972168,
      "activations/layer1_attention_weight_max": 17.985397338867188,
      "activations/layer1_attention_weight_min": -15.686552047729492,
      "activations/layer20_attention_weight_max": 29.8122615814209,
      "activations/layer20_attention_weight_min": -23.97265625,
      "activations/layer21_attention_weight_max": 26.276676177978516,
      "activations/layer21_attention_weight_min": -22.60059928894043,
      "activations/layer22_attention_weight_max": 39.53487777709961,
      "activations/layer22_attention_weight_min": -29.27178192138672,
      "activations/layer23_attention_weight_max": 33.77581024169922,
      "activations/layer23_attention_weight_min": -23.452911376953125,
      "activations/layer2_attention_weight_max": 32.01160430908203,
      "activations/layer2_attention_weight_min": -32.388824462890625,
      "activations/layer3_attention_weight_max": 90.7978286743164,
      "activations/layer3_attention_weight_min": -94.31013488769531,
      "activations/layer4_attention_weight_max": 112.36315155029297,
      "activations/layer4_attention_weight_min": -114.13276672363281,
      "activations/layer5_attention_weight_max": 52.19065856933594,
      "activations/layer5_attention_weight_min": -64.51882934570312,
      "activations/layer6_attention_weight_max": 45.189754486083984,
      "activations/layer6_attention_weight_min": -49.44984817504883,
      "activations/layer7_attention_weight_max": 89.41755676269531,
      "activations/layer7_attention_weight_min": -89.66829681396484,
      "activations/layer8_attention_weight_max": 40.668148040771484,
      "activations/layer8_attention_weight_min": -40.81639099121094,
      "activations/layer9_attention_weight_max": 33.822479248046875,
      "activations/layer9_attention_weight_min": -33.183895111083984,
      "epoch": 11.66,
      "learning_rate": 7.557651515151515e-05,
      "loss": 2.7722,
      "step": 200650
    },
    {
      "activations/layer0_attention_weight_max": 14.631881713867188,
      "activations/layer0_attention_weight_min": -12.095663070678711,
      "activations/layer10_attention_weight_max": 31.711490631103516,
      "activations/layer10_attention_weight_min": -33.86314010620117,
      "activations/layer11_attention_weight_max": 32.647987365722656,
      "activations/layer11_attention_weight_min": -33.21287155151367,
      "activations/layer12_attention_weight_max": 22.7755184173584,
      "activations/layer12_attention_weight_min": -25.59064292907715,
      "activations/layer13_attention_weight_max": 35.84832000732422,
      "activations/layer13_attention_weight_min": -35.59076690673828,
      "activations/layer14_attention_weight_max": 38.12755584716797,
      "activations/layer14_attention_weight_min": -37.49668502807617,
      "activations/layer15_attention_weight_max": 35.27210235595703,
      "activations/layer15_attention_weight_min": -32.65453338623047,
      "activations/layer16_attention_weight_max": 30.123918533325195,
      "activations/layer16_attention_weight_min": -30.085004806518555,
      "activations/layer17_attention_weight_max": 31.712890625,
      "activations/layer17_attention_weight_min": -26.884265899658203,
      "activations/layer18_attention_weight_max": 27.05842399597168,
      "activations/layer18_attention_weight_min": -24.410860061645508,
      "activations/layer19_attention_weight_max": 29.213321685791016,
      "activations/layer19_attention_weight_min": -30.103750228881836,
      "activations/layer1_attention_weight_max": 16.25609588623047,
      "activations/layer1_attention_weight_min": -14.484101295471191,
      "activations/layer20_attention_weight_max": 25.511821746826172,
      "activations/layer20_attention_weight_min": -24.32583999633789,
      "activations/layer21_attention_weight_max": 25.055578231811523,
      "activations/layer21_attention_weight_min": -24.662151336669922,
      "activations/layer22_attention_weight_max": 38.12306594848633,
      "activations/layer22_attention_weight_min": -28.84819793701172,
      "activations/layer23_attention_weight_max": 31.24581527709961,
      "activations/layer23_attention_weight_min": -24.271991729736328,
      "activations/layer2_attention_weight_max": 32.96894073486328,
      "activations/layer2_attention_weight_min": -32.555416107177734,
      "activations/layer3_attention_weight_max": 96.32286071777344,
      "activations/layer3_attention_weight_min": -101.65760040283203,
      "activations/layer4_attention_weight_max": 112.59758758544922,
      "activations/layer4_attention_weight_min": -116.6968994140625,
      "activations/layer5_attention_weight_max": 54.220211029052734,
      "activations/layer5_attention_weight_min": -63.781551361083984,
      "activations/layer6_attention_weight_max": 44.175193786621094,
      "activations/layer6_attention_weight_min": -46.460487365722656,
      "activations/layer7_attention_weight_max": 87.00669860839844,
      "activations/layer7_attention_weight_min": -87.98825073242188,
      "activations/layer8_attention_weight_max": 43.13035583496094,
      "activations/layer8_attention_weight_min": -45.530662536621094,
      "activations/layer9_attention_weight_max": 32.64434814453125,
      "activations/layer9_attention_weight_min": -34.76606750488281,
      "epoch": 11.66,
      "learning_rate": 7.555757575757574e-05,
      "loss": 2.7758,
      "step": 200700
    },
    {
      "activations/layer0_attention_weight_max": 14.142330169677734,
      "activations/layer0_attention_weight_min": -12.24946117401123,
      "activations/layer10_attention_weight_max": 33.674476623535156,
      "activations/layer10_attention_weight_min": -35.060760498046875,
      "activations/layer11_attention_weight_max": 33.67906951904297,
      "activations/layer11_attention_weight_min": -33.170772552490234,
      "activations/layer12_attention_weight_max": 25.41152572631836,
      "activations/layer12_attention_weight_min": -27.095985412597656,
      "activations/layer13_attention_weight_max": 40.1038818359375,
      "activations/layer13_attention_weight_min": -35.61448287963867,
      "activations/layer14_attention_weight_max": 43.79334259033203,
      "activations/layer14_attention_weight_min": -40.82773971557617,
      "activations/layer15_attention_weight_max": 34.953372955322266,
      "activations/layer15_attention_weight_min": -33.3306884765625,
      "activations/layer16_attention_weight_max": 31.37889862060547,
      "activations/layer16_attention_weight_min": -28.151891708374023,
      "activations/layer17_attention_weight_max": 30.508291244506836,
      "activations/layer17_attention_weight_min": -27.47211456298828,
      "activations/layer18_attention_weight_max": 31.992712020874023,
      "activations/layer18_attention_weight_min": -25.510665893554688,
      "activations/layer19_attention_weight_max": 33.885650634765625,
      "activations/layer19_attention_weight_min": -31.0495662689209,
      "activations/layer1_attention_weight_max": 17.214305877685547,
      "activations/layer1_attention_weight_min": -16.24318504333496,
      "activations/layer20_attention_weight_max": 29.332124710083008,
      "activations/layer20_attention_weight_min": -27.150890350341797,
      "activations/layer21_attention_weight_max": 29.488718032836914,
      "activations/layer21_attention_weight_min": -26.512908935546875,
      "activations/layer22_attention_weight_max": 45.14143753051758,
      "activations/layer22_attention_weight_min": -33.32395553588867,
      "activations/layer23_attention_weight_max": 34.90393829345703,
      "activations/layer23_attention_weight_min": -27.62264633178711,
      "activations/layer2_attention_weight_max": 31.616561889648438,
      "activations/layer2_attention_weight_min": -30.58032989501953,
      "activations/layer3_attention_weight_max": 92.15852355957031,
      "activations/layer3_attention_weight_min": -89.7539291381836,
      "activations/layer4_attention_weight_max": 110.09111785888672,
      "activations/layer4_attention_weight_min": -108.9726791381836,
      "activations/layer5_attention_weight_max": 54.31383514404297,
      "activations/layer5_attention_weight_min": -61.918251037597656,
      "activations/layer6_attention_weight_max": 44.67215347290039,
      "activations/layer6_attention_weight_min": -47.602787017822266,
      "activations/layer7_attention_weight_max": 92.994140625,
      "activations/layer7_attention_weight_min": -96.46854400634766,
      "activations/layer8_attention_weight_max": 42.43131637573242,
      "activations/layer8_attention_weight_min": -44.211421966552734,
      "activations/layer9_attention_weight_max": 34.11881637573242,
      "activations/layer9_attention_weight_min": -34.31057357788086,
      "epoch": 11.66,
      "learning_rate": 7.553863636363635e-05,
      "loss": 2.7852,
      "step": 200750
    },
    {
      "activations/layer0_attention_weight_max": 14.971273422241211,
      "activations/layer0_attention_weight_min": -12.372527122497559,
      "activations/layer10_attention_weight_max": 29.973201751708984,
      "activations/layer10_attention_weight_min": -30.33779525756836,
      "activations/layer11_attention_weight_max": 30.32501220703125,
      "activations/layer11_attention_weight_min": -30.87933921813965,
      "activations/layer12_attention_weight_max": 23.925785064697266,
      "activations/layer12_attention_weight_min": -26.403642654418945,
      "activations/layer13_attention_weight_max": 36.20877456665039,
      "activations/layer13_attention_weight_min": -33.738792419433594,
      "activations/layer14_attention_weight_max": 41.341609954833984,
      "activations/layer14_attention_weight_min": -38.192359924316406,
      "activations/layer15_attention_weight_max": 34.12904739379883,
      "activations/layer15_attention_weight_min": -30.900470733642578,
      "activations/layer16_attention_weight_max": 29.96656608581543,
      "activations/layer16_attention_weight_min": -28.075496673583984,
      "activations/layer17_attention_weight_max": 30.838134765625,
      "activations/layer17_attention_weight_min": -25.744434356689453,
      "activations/layer18_attention_weight_max": 31.493436813354492,
      "activations/layer18_attention_weight_min": -22.141611099243164,
      "activations/layer19_attention_weight_max": 34.222984313964844,
      "activations/layer19_attention_weight_min": -31.83125114440918,
      "activations/layer1_attention_weight_max": 16.175865173339844,
      "activations/layer1_attention_weight_min": -13.838790893554688,
      "activations/layer20_attention_weight_max": 28.477632522583008,
      "activations/layer20_attention_weight_min": -24.3520565032959,
      "activations/layer21_attention_weight_max": 33.324161529541016,
      "activations/layer21_attention_weight_min": -23.72789192199707,
      "activations/layer22_attention_weight_max": 41.285011291503906,
      "activations/layer22_attention_weight_min": -30.135997772216797,
      "activations/layer23_attention_weight_max": 31.780874252319336,
      "activations/layer23_attention_weight_min": -25.633792877197266,
      "activations/layer2_attention_weight_max": 31.913955688476562,
      "activations/layer2_attention_weight_min": -31.28557586669922,
      "activations/layer3_attention_weight_max": 97.1175537109375,
      "activations/layer3_attention_weight_min": -95.59075927734375,
      "activations/layer4_attention_weight_max": 107.02162170410156,
      "activations/layer4_attention_weight_min": -111.0856704711914,
      "activations/layer5_attention_weight_max": 57.86571502685547,
      "activations/layer5_attention_weight_min": -61.51799011230469,
      "activations/layer6_attention_weight_max": 42.86514663696289,
      "activations/layer6_attention_weight_min": -48.32211685180664,
      "activations/layer7_attention_weight_max": 94.88397216796875,
      "activations/layer7_attention_weight_min": -87.02640533447266,
      "activations/layer8_attention_weight_max": 36.610626220703125,
      "activations/layer8_attention_weight_min": -40.3278694152832,
      "activations/layer9_attention_weight_max": 30.722293853759766,
      "activations/layer9_attention_weight_min": -32.102142333984375,
      "epoch": 11.67,
      "learning_rate": 7.551969696969697e-05,
      "loss": 2.7784,
      "step": 200800
    },
    {
      "activations/layer0_attention_weight_max": 15.196096420288086,
      "activations/layer0_attention_weight_min": -12.14255142211914,
      "activations/layer10_attention_weight_max": 36.86622619628906,
      "activations/layer10_attention_weight_min": -34.13407897949219,
      "activations/layer11_attention_weight_max": 33.607208251953125,
      "activations/layer11_attention_weight_min": -33.524330139160156,
      "activations/layer12_attention_weight_max": 25.39109992980957,
      "activations/layer12_attention_weight_min": -29.65224838256836,
      "activations/layer13_attention_weight_max": 39.317222595214844,
      "activations/layer13_attention_weight_min": -34.36030197143555,
      "activations/layer14_attention_weight_max": 43.948211669921875,
      "activations/layer14_attention_weight_min": -38.211883544921875,
      "activations/layer15_attention_weight_max": 37.47252655029297,
      "activations/layer15_attention_weight_min": -32.289581298828125,
      "activations/layer16_attention_weight_max": 31.34589385986328,
      "activations/layer16_attention_weight_min": -27.358720779418945,
      "activations/layer17_attention_weight_max": 34.88440704345703,
      "activations/layer17_attention_weight_min": -26.270084381103516,
      "activations/layer18_attention_weight_max": 31.905906677246094,
      "activations/layer18_attention_weight_min": -23.16229248046875,
      "activations/layer19_attention_weight_max": 35.187374114990234,
      "activations/layer19_attention_weight_min": -30.909494400024414,
      "activations/layer1_attention_weight_max": 16.745140075683594,
      "activations/layer1_attention_weight_min": -14.894647598266602,
      "activations/layer20_attention_weight_max": 34.349815368652344,
      "activations/layer20_attention_weight_min": -24.649553298950195,
      "activations/layer21_attention_weight_max": 31.628774642944336,
      "activations/layer21_attention_weight_min": -24.79326057434082,
      "activations/layer22_attention_weight_max": 45.22162628173828,
      "activations/layer22_attention_weight_min": -29.027240753173828,
      "activations/layer23_attention_weight_max": 34.265480041503906,
      "activations/layer23_attention_weight_min": -24.442296981811523,
      "activations/layer2_attention_weight_max": 35.881919860839844,
      "activations/layer2_attention_weight_min": -34.43389892578125,
      "activations/layer3_attention_weight_max": 99.04989624023438,
      "activations/layer3_attention_weight_min": -99.67070770263672,
      "activations/layer4_attention_weight_max": 119.69464111328125,
      "activations/layer4_attention_weight_min": -117.82630920410156,
      "activations/layer5_attention_weight_max": 54.110877990722656,
      "activations/layer5_attention_weight_min": -63.71517562866211,
      "activations/layer6_attention_weight_max": 45.769126892089844,
      "activations/layer6_attention_weight_min": -47.667503356933594,
      "activations/layer7_attention_weight_max": 89.97006225585938,
      "activations/layer7_attention_weight_min": -87.82347869873047,
      "activations/layer8_attention_weight_max": 43.37180709838867,
      "activations/layer8_attention_weight_min": -43.464717864990234,
      "activations/layer9_attention_weight_max": 33.8315544128418,
      "activations/layer9_attention_weight_min": -34.334556579589844,
      "epoch": 11.67,
      "learning_rate": 7.550075757575757e-05,
      "loss": 2.7792,
      "step": 200850
    },
    {
      "activations/layer0_attention_weight_max": 14.876274108886719,
      "activations/layer0_attention_weight_min": -12.164570808410645,
      "activations/layer10_attention_weight_max": 33.17359161376953,
      "activations/layer10_attention_weight_min": -32.06581115722656,
      "activations/layer11_attention_weight_max": 31.6241455078125,
      "activations/layer11_attention_weight_min": -31.10101890563965,
      "activations/layer12_attention_weight_max": 26.083707809448242,
      "activations/layer12_attention_weight_min": -23.760364532470703,
      "activations/layer13_attention_weight_max": 38.98116683959961,
      "activations/layer13_attention_weight_min": -33.520442962646484,
      "activations/layer14_attention_weight_max": 41.11809158325195,
      "activations/layer14_attention_weight_min": -35.7847785949707,
      "activations/layer15_attention_weight_max": 36.604225158691406,
      "activations/layer15_attention_weight_min": -30.379859924316406,
      "activations/layer16_attention_weight_max": 31.34185791015625,
      "activations/layer16_attention_weight_min": -28.7650203704834,
      "activations/layer17_attention_weight_max": 33.19278335571289,
      "activations/layer17_attention_weight_min": -26.005977630615234,
      "activations/layer18_attention_weight_max": 30.79986572265625,
      "activations/layer18_attention_weight_min": -21.7093563079834,
      "activations/layer19_attention_weight_max": 35.150936126708984,
      "activations/layer19_attention_weight_min": -30.55388832092285,
      "activations/layer1_attention_weight_max": 17.060466766357422,
      "activations/layer1_attention_weight_min": -13.129855155944824,
      "activations/layer20_attention_weight_max": 29.47771644592285,
      "activations/layer20_attention_weight_min": -23.560117721557617,
      "activations/layer21_attention_weight_max": 29.27127456665039,
      "activations/layer21_attention_weight_min": -22.302383422851562,
      "activations/layer22_attention_weight_max": 42.80375671386719,
      "activations/layer22_attention_weight_min": -27.641695022583008,
      "activations/layer23_attention_weight_max": 34.1139030456543,
      "activations/layer23_attention_weight_min": -24.197952270507812,
      "activations/layer2_attention_weight_max": 30.468854904174805,
      "activations/layer2_attention_weight_min": -31.53525161743164,
      "activations/layer3_attention_weight_max": 90.02699279785156,
      "activations/layer3_attention_weight_min": -91.47919464111328,
      "activations/layer4_attention_weight_max": 107.04215240478516,
      "activations/layer4_attention_weight_min": -111.64311981201172,
      "activations/layer5_attention_weight_max": 50.868446350097656,
      "activations/layer5_attention_weight_min": -61.75849151611328,
      "activations/layer6_attention_weight_max": 41.5490837097168,
      "activations/layer6_attention_weight_min": -48.91010284423828,
      "activations/layer7_attention_weight_max": 85.81837463378906,
      "activations/layer7_attention_weight_min": -86.62359619140625,
      "activations/layer8_attention_weight_max": 38.38642883300781,
      "activations/layer8_attention_weight_min": -41.944732666015625,
      "activations/layer9_attention_weight_max": 32.68830490112305,
      "activations/layer9_attention_weight_min": -32.595428466796875,
      "epoch": 11.67,
      "learning_rate": 7.548181818181817e-05,
      "loss": 2.79,
      "step": 200900
    },
    {
      "activations/layer0_attention_weight_max": 15.869237899780273,
      "activations/layer0_attention_weight_min": -12.10145378112793,
      "activations/layer10_attention_weight_max": 33.6551399230957,
      "activations/layer10_attention_weight_min": -33.41445541381836,
      "activations/layer11_attention_weight_max": 33.53160095214844,
      "activations/layer11_attention_weight_min": -31.862422943115234,
      "activations/layer12_attention_weight_max": 27.56585121154785,
      "activations/layer12_attention_weight_min": -25.932235717773438,
      "activations/layer13_attention_weight_max": 44.208839416503906,
      "activations/layer13_attention_weight_min": -36.198036193847656,
      "activations/layer14_attention_weight_max": 49.57082748413086,
      "activations/layer14_attention_weight_min": -38.0950813293457,
      "activations/layer15_attention_weight_max": 41.921199798583984,
      "activations/layer15_attention_weight_min": -32.56302261352539,
      "activations/layer16_attention_weight_max": 36.39082336425781,
      "activations/layer16_attention_weight_min": -27.892229080200195,
      "activations/layer17_attention_weight_max": 36.23301315307617,
      "activations/layer17_attention_weight_min": -27.66942596435547,
      "activations/layer18_attention_weight_max": 35.79609680175781,
      "activations/layer18_attention_weight_min": -24.800268173217773,
      "activations/layer19_attention_weight_max": 39.56244659423828,
      "activations/layer19_attention_weight_min": -30.324636459350586,
      "activations/layer1_attention_weight_max": 15.65651798248291,
      "activations/layer1_attention_weight_min": -14.10492992401123,
      "activations/layer20_attention_weight_max": 34.574832916259766,
      "activations/layer20_attention_weight_min": -24.72096824645996,
      "activations/layer21_attention_weight_max": 36.5811767578125,
      "activations/layer21_attention_weight_min": -24.194143295288086,
      "activations/layer22_attention_weight_max": 47.53010940551758,
      "activations/layer22_attention_weight_min": -28.954072952270508,
      "activations/layer23_attention_weight_max": 36.184837341308594,
      "activations/layer23_attention_weight_min": -23.806591033935547,
      "activations/layer2_attention_weight_max": 32.38813400268555,
      "activations/layer2_attention_weight_min": -33.479820251464844,
      "activations/layer3_attention_weight_max": 94.84778594970703,
      "activations/layer3_attention_weight_min": -99.0545654296875,
      "activations/layer4_attention_weight_max": 120.02544403076172,
      "activations/layer4_attention_weight_min": -120.00402069091797,
      "activations/layer5_attention_weight_max": 55.07469940185547,
      "activations/layer5_attention_weight_min": -62.604488372802734,
      "activations/layer6_attention_weight_max": 44.64358901977539,
      "activations/layer6_attention_weight_min": -47.587154388427734,
      "activations/layer7_attention_weight_max": 92.81343841552734,
      "activations/layer7_attention_weight_min": -87.12620544433594,
      "activations/layer8_attention_weight_max": 41.60116958618164,
      "activations/layer8_attention_weight_min": -41.09006881713867,
      "activations/layer9_attention_weight_max": 34.084774017333984,
      "activations/layer9_attention_weight_min": -35.705284118652344,
      "epoch": 11.68,
      "learning_rate": 7.546287878787879e-05,
      "loss": 2.7698,
      "step": 200950
    },
    {
      "activations/layer0_attention_weight_max": 15.248846054077148,
      "activations/layer0_attention_weight_min": -12.070155143737793,
      "activations/layer10_attention_weight_max": 31.671306610107422,
      "activations/layer10_attention_weight_min": -33.128360748291016,
      "activations/layer11_attention_weight_max": 32.0062255859375,
      "activations/layer11_attention_weight_min": -30.762176513671875,
      "activations/layer12_attention_weight_max": 24.08642578125,
      "activations/layer12_attention_weight_min": -24.088693618774414,
      "activations/layer13_attention_weight_max": 38.55427932739258,
      "activations/layer13_attention_weight_min": -33.35177230834961,
      "activations/layer14_attention_weight_max": 41.47366714477539,
      "activations/layer14_attention_weight_min": -37.302738189697266,
      "activations/layer15_attention_weight_max": 36.70695114135742,
      "activations/layer15_attention_weight_min": -30.936325073242188,
      "activations/layer16_attention_weight_max": 32.83755874633789,
      "activations/layer16_attention_weight_min": -28.076623916625977,
      "activations/layer17_attention_weight_max": 31.67679214477539,
      "activations/layer17_attention_weight_min": -25.705656051635742,
      "activations/layer18_attention_weight_max": 31.162357330322266,
      "activations/layer18_attention_weight_min": -22.189054489135742,
      "activations/layer19_attention_weight_max": 36.07418441772461,
      "activations/layer19_attention_weight_min": -32.76639938354492,
      "activations/layer1_attention_weight_max": 16.632078170776367,
      "activations/layer1_attention_weight_min": -13.750675201416016,
      "activations/layer20_attention_weight_max": 28.77608871459961,
      "activations/layer20_attention_weight_min": -25.204816818237305,
      "activations/layer21_attention_weight_max": 26.432193756103516,
      "activations/layer21_attention_weight_min": -23.732664108276367,
      "activations/layer22_attention_weight_max": 41.93922805786133,
      "activations/layer22_attention_weight_min": -27.732194900512695,
      "activations/layer23_attention_weight_max": 34.46943664550781,
      "activations/layer23_attention_weight_min": -24.23630142211914,
      "activations/layer2_attention_weight_max": 31.81255340576172,
      "activations/layer2_attention_weight_min": -32.13079833984375,
      "activations/layer3_attention_weight_max": 89.72515869140625,
      "activations/layer3_attention_weight_min": -96.83694458007812,
      "activations/layer4_attention_weight_max": 109.28997802734375,
      "activations/layer4_attention_weight_min": -114.19304656982422,
      "activations/layer5_attention_weight_max": 51.82170104980469,
      "activations/layer5_attention_weight_min": -61.887447357177734,
      "activations/layer6_attention_weight_max": 43.62223434448242,
      "activations/layer6_attention_weight_min": -47.449745178222656,
      "activations/layer7_attention_weight_max": 85.42385864257812,
      "activations/layer7_attention_weight_min": -91.369140625,
      "activations/layer8_attention_weight_max": 37.360233306884766,
      "activations/layer8_attention_weight_min": -41.348899841308594,
      "activations/layer9_attention_weight_max": 30.156091690063477,
      "activations/layer9_attention_weight_min": -33.73682403564453,
      "epoch": 11.68,
      "learning_rate": 7.544393939393939e-05,
      "loss": 2.7621,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.4216,
      "eval_samples_per_second": 509.877,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.4216,
      "eval_openwebtext_samples_per_second": 509.877,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 2.018,
      "eval_wikitext_samples_per_second": 225.961,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.597,
      "eval_lambada_samples_per_second": 507.347,
      "step": 201000
    },
    {
      "activations/layer0_attention_weight_max": 14.843162536621094,
      "activations/layer0_attention_weight_min": -12.32682991027832,
      "activations/layer10_attention_weight_max": 31.88742446899414,
      "activations/layer10_attention_weight_min": -31.50876808166504,
      "activations/layer11_attention_weight_max": 34.41501235961914,
      "activations/layer11_attention_weight_min": -32.33521270751953,
      "activations/layer12_attention_weight_max": 24.467206954956055,
      "activations/layer12_attention_weight_min": -25.96210479736328,
      "activations/layer13_attention_weight_max": 36.672279357910156,
      "activations/layer13_attention_weight_min": -34.32655334472656,
      "activations/layer14_attention_weight_max": 45.82410430908203,
      "activations/layer14_attention_weight_min": -39.065792083740234,
      "activations/layer15_attention_weight_max": 37.24538803100586,
      "activations/layer15_attention_weight_min": -34.082244873046875,
      "activations/layer16_attention_weight_max": 29.935441970825195,
      "activations/layer16_attention_weight_min": -28.7656307220459,
      "activations/layer17_attention_weight_max": 33.602474212646484,
      "activations/layer17_attention_weight_min": -25.900634765625,
      "activations/layer18_attention_weight_max": 30.834321975708008,
      "activations/layer18_attention_weight_min": -22.931507110595703,
      "activations/layer19_attention_weight_max": 33.906166076660156,
      "activations/layer19_attention_weight_min": -30.066268920898438,
      "activations/layer1_attention_weight_max": 18.707979202270508,
      "activations/layer1_attention_weight_min": -14.116004943847656,
      "activations/layer20_attention_weight_max": 29.989717483520508,
      "activations/layer20_attention_weight_min": -24.212438583374023,
      "activations/layer21_attention_weight_max": 29.5576171875,
      "activations/layer21_attention_weight_min": -24.937650680541992,
      "activations/layer22_attention_weight_max": 44.7198600769043,
      "activations/layer22_attention_weight_min": -29.8714599609375,
      "activations/layer23_attention_weight_max": 36.89601135253906,
      "activations/layer23_attention_weight_min": -25.567832946777344,
      "activations/layer2_attention_weight_max": 29.246688842773438,
      "activations/layer2_attention_weight_min": -31.547277450561523,
      "activations/layer3_attention_weight_max": 87.07579803466797,
      "activations/layer3_attention_weight_min": -94.53998565673828,
      "activations/layer4_attention_weight_max": 108.86539459228516,
      "activations/layer4_attention_weight_min": -114.87300109863281,
      "activations/layer5_attention_weight_max": 50.342315673828125,
      "activations/layer5_attention_weight_min": -60.4046630859375,
      "activations/layer6_attention_weight_max": 42.70083236694336,
      "activations/layer6_attention_weight_min": -47.04539489746094,
      "activations/layer7_attention_weight_max": 83.92547607421875,
      "activations/layer7_attention_weight_min": -85.16905212402344,
      "activations/layer8_attention_weight_max": 39.190948486328125,
      "activations/layer8_attention_weight_min": -39.59912872314453,
      "activations/layer9_attention_weight_max": 32.277008056640625,
      "activations/layer9_attention_weight_min": -32.83095932006836,
      "epoch": 11.68,
      "learning_rate": 7.542499999999999e-05,
      "loss": 2.7744,
      "step": 201050
    },
    {
      "activations/layer0_attention_weight_max": 15.250797271728516,
      "activations/layer0_attention_weight_min": -12.61557674407959,
      "activations/layer10_attention_weight_max": 31.298763275146484,
      "activations/layer10_attention_weight_min": -31.800216674804688,
      "activations/layer11_attention_weight_max": 31.97876739501953,
      "activations/layer11_attention_weight_min": -30.856130599975586,
      "activations/layer12_attention_weight_max": 24.893104553222656,
      "activations/layer12_attention_weight_min": -24.39413833618164,
      "activations/layer13_attention_weight_max": 40.12263488769531,
      "activations/layer13_attention_weight_min": -35.13606262207031,
      "activations/layer14_attention_weight_max": 43.548362731933594,
      "activations/layer14_attention_weight_min": -35.93526077270508,
      "activations/layer15_attention_weight_max": 36.47099685668945,
      "activations/layer15_attention_weight_min": -30.74981689453125,
      "activations/layer16_attention_weight_max": 33.09477233886719,
      "activations/layer16_attention_weight_min": -26.494606018066406,
      "activations/layer17_attention_weight_max": 31.85923194885254,
      "activations/layer17_attention_weight_min": -25.38199234008789,
      "activations/layer18_attention_weight_max": 32.14387130737305,
      "activations/layer18_attention_weight_min": -23.80816078186035,
      "activations/layer19_attention_weight_max": 38.670326232910156,
      "activations/layer19_attention_weight_min": -31.5237979888916,
      "activations/layer1_attention_weight_max": 16.733551025390625,
      "activations/layer1_attention_weight_min": -13.782492637634277,
      "activations/layer20_attention_weight_max": 31.508583068847656,
      "activations/layer20_attention_weight_min": -26.261245727539062,
      "activations/layer21_attention_weight_max": 29.01988983154297,
      "activations/layer21_attention_weight_min": -23.53509521484375,
      "activations/layer22_attention_weight_max": 49.355506896972656,
      "activations/layer22_attention_weight_min": -31.750017166137695,
      "activations/layer23_attention_weight_max": 37.41908264160156,
      "activations/layer23_attention_weight_min": -26.05268096923828,
      "activations/layer2_attention_weight_max": 28.792661666870117,
      "activations/layer2_attention_weight_min": -28.189146041870117,
      "activations/layer3_attention_weight_max": 93.37703704833984,
      "activations/layer3_attention_weight_min": -91.51811218261719,
      "activations/layer4_attention_weight_max": 116.14916229248047,
      "activations/layer4_attention_weight_min": -110.9985580444336,
      "activations/layer5_attention_weight_max": 51.29038619995117,
      "activations/layer5_attention_weight_min": -62.253990173339844,
      "activations/layer6_attention_weight_max": 43.48064041137695,
      "activations/layer6_attention_weight_min": -49.57141876220703,
      "activations/layer7_attention_weight_max": 86.27529907226562,
      "activations/layer7_attention_weight_min": -91.7923583984375,
      "activations/layer8_attention_weight_max": 38.64968490600586,
      "activations/layer8_attention_weight_min": -41.74554443359375,
      "activations/layer9_attention_weight_max": 33.3087272644043,
      "activations/layer9_attention_weight_min": -33.544795989990234,
      "epoch": 11.69,
      "learning_rate": 7.540606060606059e-05,
      "loss": 2.7844,
      "step": 201100
    },
    {
      "activations/layer0_attention_weight_max": 15.111919403076172,
      "activations/layer0_attention_weight_min": -11.846248626708984,
      "activations/layer10_attention_weight_max": 30.460994720458984,
      "activations/layer10_attention_weight_min": -32.24909591674805,
      "activations/layer11_attention_weight_max": 34.26261901855469,
      "activations/layer11_attention_weight_min": -31.70499038696289,
      "activations/layer12_attention_weight_max": 24.979223251342773,
      "activations/layer12_attention_weight_min": -25.614459991455078,
      "activations/layer13_attention_weight_max": 38.53248596191406,
      "activations/layer13_attention_weight_min": -35.93946075439453,
      "activations/layer14_attention_weight_max": 38.34518814086914,
      "activations/layer14_attention_weight_min": -36.803680419921875,
      "activations/layer15_attention_weight_max": 34.016292572021484,
      "activations/layer15_attention_weight_min": -31.136194229125977,
      "activations/layer16_attention_weight_max": 29.857608795166016,
      "activations/layer16_attention_weight_min": -28.13669776916504,
      "activations/layer17_attention_weight_max": 30.166385650634766,
      "activations/layer17_attention_weight_min": -25.795780181884766,
      "activations/layer18_attention_weight_max": 31.0323486328125,
      "activations/layer18_attention_weight_min": -25.292293548583984,
      "activations/layer19_attention_weight_max": 33.068843841552734,
      "activations/layer19_attention_weight_min": -32.76687240600586,
      "activations/layer1_attention_weight_max": 16.66606903076172,
      "activations/layer1_attention_weight_min": -14.90709400177002,
      "activations/layer20_attention_weight_max": 30.684680938720703,
      "activations/layer20_attention_weight_min": -25.949377059936523,
      "activations/layer21_attention_weight_max": 30.93882179260254,
      "activations/layer21_attention_weight_min": -26.768802642822266,
      "activations/layer22_attention_weight_max": 43.47358322143555,
      "activations/layer22_attention_weight_min": -30.536712646484375,
      "activations/layer23_attention_weight_max": 35.1460075378418,
      "activations/layer23_attention_weight_min": -24.410690307617188,
      "activations/layer2_attention_weight_max": 31.59749412536621,
      "activations/layer2_attention_weight_min": -30.26947593688965,
      "activations/layer3_attention_weight_max": 88.91925811767578,
      "activations/layer3_attention_weight_min": -91.8781967163086,
      "activations/layer4_attention_weight_max": 110.20360565185547,
      "activations/layer4_attention_weight_min": -108.9252700805664,
      "activations/layer5_attention_weight_max": 50.68398666381836,
      "activations/layer5_attention_weight_min": -64.09449768066406,
      "activations/layer6_attention_weight_max": 44.67365646362305,
      "activations/layer6_attention_weight_min": -46.538875579833984,
      "activations/layer7_attention_weight_max": 86.3818588256836,
      "activations/layer7_attention_weight_min": -86.6906509399414,
      "activations/layer8_attention_weight_max": 40.1508674621582,
      "activations/layer8_attention_weight_min": -40.57879638671875,
      "activations/layer9_attention_weight_max": 31.561920166015625,
      "activations/layer9_attention_weight_min": -32.79018783569336,
      "epoch": 11.69,
      "learning_rate": 7.53871212121212e-05,
      "loss": 2.7841,
      "step": 201150
    },
    {
      "activations/layer0_attention_weight_max": 15.488369941711426,
      "activations/layer0_attention_weight_min": -11.141913414001465,
      "activations/layer10_attention_weight_max": 38.851898193359375,
      "activations/layer10_attention_weight_min": -39.34636688232422,
      "activations/layer11_attention_weight_max": 37.446998596191406,
      "activations/layer11_attention_weight_min": -39.24431610107422,
      "activations/layer12_attention_weight_max": 25.985069274902344,
      "activations/layer12_attention_weight_min": -26.770626068115234,
      "activations/layer13_attention_weight_max": 39.2059211730957,
      "activations/layer13_attention_weight_min": -37.6223258972168,
      "activations/layer14_attention_weight_max": 49.230018615722656,
      "activations/layer14_attention_weight_min": -41.58270263671875,
      "activations/layer15_attention_weight_max": 37.136932373046875,
      "activations/layer15_attention_weight_min": -34.58597183227539,
      "activations/layer16_attention_weight_max": 31.877185821533203,
      "activations/layer16_attention_weight_min": -28.926719665527344,
      "activations/layer17_attention_weight_max": 31.84174346923828,
      "activations/layer17_attention_weight_min": -28.28365707397461,
      "activations/layer18_attention_weight_max": 31.959369659423828,
      "activations/layer18_attention_weight_min": -25.16679573059082,
      "activations/layer19_attention_weight_max": 37.029842376708984,
      "activations/layer19_attention_weight_min": -30.097509384155273,
      "activations/layer1_attention_weight_max": 17.14483642578125,
      "activations/layer1_attention_weight_min": -15.697800636291504,
      "activations/layer20_attention_weight_max": 30.463171005249023,
      "activations/layer20_attention_weight_min": -23.725200653076172,
      "activations/layer21_attention_weight_max": 31.050743103027344,
      "activations/layer21_attention_weight_min": -22.9636173248291,
      "activations/layer22_attention_weight_max": 45.84833526611328,
      "activations/layer22_attention_weight_min": -29.381513595581055,
      "activations/layer23_attention_weight_max": 35.87776184082031,
      "activations/layer23_attention_weight_min": -24.074935913085938,
      "activations/layer2_attention_weight_max": 31.82925796508789,
      "activations/layer2_attention_weight_min": -31.292768478393555,
      "activations/layer3_attention_weight_max": 88.06401824951172,
      "activations/layer3_attention_weight_min": -90.7410888671875,
      "activations/layer4_attention_weight_max": 111.01827239990234,
      "activations/layer4_attention_weight_min": -110.09896087646484,
      "activations/layer5_attention_weight_max": 51.0692138671875,
      "activations/layer5_attention_weight_min": -62.375831604003906,
      "activations/layer6_attention_weight_max": 45.01181411743164,
      "activations/layer6_attention_weight_min": -47.64731979370117,
      "activations/layer7_attention_weight_max": 95.17057800292969,
      "activations/layer7_attention_weight_min": -95.90234375,
      "activations/layer8_attention_weight_max": 46.83707046508789,
      "activations/layer8_attention_weight_min": -47.630165100097656,
      "activations/layer9_attention_weight_max": 37.78170394897461,
      "activations/layer9_attention_weight_min": -39.76874923706055,
      "epoch": 11.69,
      "learning_rate": 7.536818181818182e-05,
      "loss": 2.7676,
      "step": 201200
    },
    {
      "activations/layer0_attention_weight_max": 14.550638198852539,
      "activations/layer0_attention_weight_min": -12.774443626403809,
      "activations/layer10_attention_weight_max": 32.719051361083984,
      "activations/layer10_attention_weight_min": -33.24675369262695,
      "activations/layer11_attention_weight_max": 31.513715744018555,
      "activations/layer11_attention_weight_min": -32.560001373291016,
      "activations/layer12_attention_weight_max": 24.618982315063477,
      "activations/layer12_attention_weight_min": -24.934242248535156,
      "activations/layer13_attention_weight_max": 36.9954833984375,
      "activations/layer13_attention_weight_min": -34.454036712646484,
      "activations/layer14_attention_weight_max": 41.731353759765625,
      "activations/layer14_attention_weight_min": -34.80205535888672,
      "activations/layer15_attention_weight_max": 35.543582916259766,
      "activations/layer15_attention_weight_min": -31.50701332092285,
      "activations/layer16_attention_weight_max": 31.070877075195312,
      "activations/layer16_attention_weight_min": -29.65406608581543,
      "activations/layer17_attention_weight_max": 33.57209396362305,
      "activations/layer17_attention_weight_min": -28.258893966674805,
      "activations/layer18_attention_weight_max": 32.7120475769043,
      "activations/layer18_attention_weight_min": -24.250993728637695,
      "activations/layer19_attention_weight_max": 38.83356475830078,
      "activations/layer19_attention_weight_min": -32.244972229003906,
      "activations/layer1_attention_weight_max": 17.06389617919922,
      "activations/layer1_attention_weight_min": -15.091165542602539,
      "activations/layer20_attention_weight_max": 32.69668197631836,
      "activations/layer20_attention_weight_min": -24.28546142578125,
      "activations/layer21_attention_weight_max": 32.620304107666016,
      "activations/layer21_attention_weight_min": -25.151376724243164,
      "activations/layer22_attention_weight_max": 50.564544677734375,
      "activations/layer22_attention_weight_min": -31.50234603881836,
      "activations/layer23_attention_weight_max": 39.309356689453125,
      "activations/layer23_attention_weight_min": -25.156246185302734,
      "activations/layer2_attention_weight_max": 30.317794799804688,
      "activations/layer2_attention_weight_min": -29.970745086669922,
      "activations/layer3_attention_weight_max": 84.05438232421875,
      "activations/layer3_attention_weight_min": -82.42556762695312,
      "activations/layer4_attention_weight_max": 106.6771469116211,
      "activations/layer4_attention_weight_min": -99.4635009765625,
      "activations/layer5_attention_weight_max": 50.448394775390625,
      "activations/layer5_attention_weight_min": -63.67570495605469,
      "activations/layer6_attention_weight_max": 41.38119125366211,
      "activations/layer6_attention_weight_min": -44.4669075012207,
      "activations/layer7_attention_weight_max": 84.12931823730469,
      "activations/layer7_attention_weight_min": -83.1926040649414,
      "activations/layer8_attention_weight_max": 37.54539108276367,
      "activations/layer8_attention_weight_min": -40.36833572387695,
      "activations/layer9_attention_weight_max": 32.823177337646484,
      "activations/layer9_attention_weight_min": -33.13429260253906,
      "epoch": 11.69,
      "learning_rate": 7.534924242424241e-05,
      "loss": 2.7924,
      "step": 201250
    },
    {
      "activations/layer0_attention_weight_max": 15.395651817321777,
      "activations/layer0_attention_weight_min": -12.422667503356934,
      "activations/layer10_attention_weight_max": 40.83526611328125,
      "activations/layer10_attention_weight_min": -41.48930358886719,
      "activations/layer11_attention_weight_max": 38.56659698486328,
      "activations/layer11_attention_weight_min": -38.649776458740234,
      "activations/layer12_attention_weight_max": 25.244810104370117,
      "activations/layer12_attention_weight_min": -25.839876174926758,
      "activations/layer13_attention_weight_max": 35.5426025390625,
      "activations/layer13_attention_weight_min": -35.48549270629883,
      "activations/layer14_attention_weight_max": 39.6378288269043,
      "activations/layer14_attention_weight_min": -37.88969421386719,
      "activations/layer15_attention_weight_max": 35.88097381591797,
      "activations/layer15_attention_weight_min": -32.07863998413086,
      "activations/layer16_attention_weight_max": 29.59489631652832,
      "activations/layer16_attention_weight_min": -27.354337692260742,
      "activations/layer17_attention_weight_max": 30.373769760131836,
      "activations/layer17_attention_weight_min": -26.193374633789062,
      "activations/layer18_attention_weight_max": 31.6016902923584,
      "activations/layer18_attention_weight_min": -24.446796417236328,
      "activations/layer19_attention_weight_max": 31.65860939025879,
      "activations/layer19_attention_weight_min": -33.660484313964844,
      "activations/layer1_attention_weight_max": 17.49704933166504,
      "activations/layer1_attention_weight_min": -16.408355712890625,
      "activations/layer20_attention_weight_max": 30.163787841796875,
      "activations/layer20_attention_weight_min": -27.522300720214844,
      "activations/layer21_attention_weight_max": 31.264537811279297,
      "activations/layer21_attention_weight_min": -25.021255493164062,
      "activations/layer22_attention_weight_max": 42.90268325805664,
      "activations/layer22_attention_weight_min": -30.302175521850586,
      "activations/layer23_attention_weight_max": 33.588050842285156,
      "activations/layer23_attention_weight_min": -25.732879638671875,
      "activations/layer2_attention_weight_max": 31.981054306030273,
      "activations/layer2_attention_weight_min": -32.11134719848633,
      "activations/layer3_attention_weight_max": 96.91203308105469,
      "activations/layer3_attention_weight_min": -92.45683288574219,
      "activations/layer4_attention_weight_max": 110.38899993896484,
      "activations/layer4_attention_weight_min": -114.36751556396484,
      "activations/layer5_attention_weight_max": 51.97016525268555,
      "activations/layer5_attention_weight_min": -61.46202850341797,
      "activations/layer6_attention_weight_max": 47.62892532348633,
      "activations/layer6_attention_weight_min": -48.217281341552734,
      "activations/layer7_attention_weight_max": 98.98989868164062,
      "activations/layer7_attention_weight_min": -97.02913665771484,
      "activations/layer8_attention_weight_max": 48.773193359375,
      "activations/layer8_attention_weight_min": -49.27906799316406,
      "activations/layer9_attention_weight_max": 38.4776725769043,
      "activations/layer9_attention_weight_min": -40.688743591308594,
      "epoch": 11.7,
      "learning_rate": 7.533030303030302e-05,
      "loss": 2.7816,
      "step": 201300
    },
    {
      "activations/layer0_attention_weight_max": 15.138660430908203,
      "activations/layer0_attention_weight_min": -12.030494689941406,
      "activations/layer10_attention_weight_max": 30.95473289489746,
      "activations/layer10_attention_weight_min": -35.366188049316406,
      "activations/layer11_attention_weight_max": 31.450305938720703,
      "activations/layer11_attention_weight_min": -35.13190460205078,
      "activations/layer12_attention_weight_max": 22.542760848999023,
      "activations/layer12_attention_weight_min": -25.287033081054688,
      "activations/layer13_attention_weight_max": 36.80332565307617,
      "activations/layer13_attention_weight_min": -34.73999786376953,
      "activations/layer14_attention_weight_max": 41.79867172241211,
      "activations/layer14_attention_weight_min": -38.408172607421875,
      "activations/layer15_attention_weight_max": 39.48514938354492,
      "activations/layer15_attention_weight_min": -34.76731491088867,
      "activations/layer16_attention_weight_max": 32.48468780517578,
      "activations/layer16_attention_weight_min": -28.183244705200195,
      "activations/layer17_attention_weight_max": 32.777462005615234,
      "activations/layer17_attention_weight_min": -26.574888229370117,
      "activations/layer18_attention_weight_max": 31.403825759887695,
      "activations/layer18_attention_weight_min": -24.411638259887695,
      "activations/layer19_attention_weight_max": 32.567378997802734,
      "activations/layer19_attention_weight_min": -33.0374870300293,
      "activations/layer1_attention_weight_max": 15.59082317352295,
      "activations/layer1_attention_weight_min": -13.58414363861084,
      "activations/layer20_attention_weight_max": 29.28961753845215,
      "activations/layer20_attention_weight_min": -25.03656578063965,
      "activations/layer21_attention_weight_max": 30.102148056030273,
      "activations/layer21_attention_weight_min": -24.805265426635742,
      "activations/layer22_attention_weight_max": 45.911712646484375,
      "activations/layer22_attention_weight_min": -29.03731346130371,
      "activations/layer23_attention_weight_max": 32.07727813720703,
      "activations/layer23_attention_weight_min": -24.008541107177734,
      "activations/layer2_attention_weight_max": 31.04524803161621,
      "activations/layer2_attention_weight_min": -29.43698501586914,
      "activations/layer3_attention_weight_max": 85.84984588623047,
      "activations/layer3_attention_weight_min": -87.68347930908203,
      "activations/layer4_attention_weight_max": 105.25402069091797,
      "activations/layer4_attention_weight_min": -104.8706283569336,
      "activations/layer5_attention_weight_max": 51.75220489501953,
      "activations/layer5_attention_weight_min": -59.832862854003906,
      "activations/layer6_attention_weight_max": 44.23408508300781,
      "activations/layer6_attention_weight_min": -46.06630325317383,
      "activations/layer7_attention_weight_max": 86.90446472167969,
      "activations/layer7_attention_weight_min": -89.42618560791016,
      "activations/layer8_attention_weight_max": 39.839481353759766,
      "activations/layer8_attention_weight_min": -42.08881378173828,
      "activations/layer9_attention_weight_max": 31.823993682861328,
      "activations/layer9_attention_weight_min": -34.22556686401367,
      "epoch": 11.7,
      "learning_rate": 7.531136363636364e-05,
      "loss": 2.7692,
      "step": 201350
    },
    {
      "activations/layer0_attention_weight_max": 14.63280200958252,
      "activations/layer0_attention_weight_min": -13.122997283935547,
      "activations/layer10_attention_weight_max": 33.593021392822266,
      "activations/layer10_attention_weight_min": -33.78790283203125,
      "activations/layer11_attention_weight_max": 31.77703857421875,
      "activations/layer11_attention_weight_min": -33.927921295166016,
      "activations/layer12_attention_weight_max": 25.152755737304688,
      "activations/layer12_attention_weight_min": -28.02273178100586,
      "activations/layer13_attention_weight_max": 41.27789306640625,
      "activations/layer13_attention_weight_min": -33.498836517333984,
      "activations/layer14_attention_weight_max": 45.392616271972656,
      "activations/layer14_attention_weight_min": -34.73002243041992,
      "activations/layer15_attention_weight_max": 40.21095657348633,
      "activations/layer15_attention_weight_min": -32.406211853027344,
      "activations/layer16_attention_weight_max": 34.07810974121094,
      "activations/layer16_attention_weight_min": -28.84856414794922,
      "activations/layer17_attention_weight_max": 33.7862434387207,
      "activations/layer17_attention_weight_min": -25.4024600982666,
      "activations/layer18_attention_weight_max": 31.981075286865234,
      "activations/layer18_attention_weight_min": -22.14950180053711,
      "activations/layer19_attention_weight_max": 37.207237243652344,
      "activations/layer19_attention_weight_min": -30.438016891479492,
      "activations/layer1_attention_weight_max": 17.090791702270508,
      "activations/layer1_attention_weight_min": -13.363397598266602,
      "activations/layer20_attention_weight_max": 33.38254928588867,
      "activations/layer20_attention_weight_min": -23.593284606933594,
      "activations/layer21_attention_weight_max": 30.63006591796875,
      "activations/layer21_attention_weight_min": -23.230804443359375,
      "activations/layer22_attention_weight_max": 42.0897216796875,
      "activations/layer22_attention_weight_min": -29.538471221923828,
      "activations/layer23_attention_weight_max": 35.56909942626953,
      "activations/layer23_attention_weight_min": -27.19002342224121,
      "activations/layer2_attention_weight_max": 31.235092163085938,
      "activations/layer2_attention_weight_min": -31.108036041259766,
      "activations/layer3_attention_weight_max": 87.7379150390625,
      "activations/layer3_attention_weight_min": -88.17430877685547,
      "activations/layer4_attention_weight_max": 107.6581802368164,
      "activations/layer4_attention_weight_min": -110.95112609863281,
      "activations/layer5_attention_weight_max": 51.646087646484375,
      "activations/layer5_attention_weight_min": -61.79600143432617,
      "activations/layer6_attention_weight_max": 43.87316131591797,
      "activations/layer6_attention_weight_min": -49.77309036254883,
      "activations/layer7_attention_weight_max": 88.47569274902344,
      "activations/layer7_attention_weight_min": -92.2636947631836,
      "activations/layer8_attention_weight_max": 42.50509262084961,
      "activations/layer8_attention_weight_min": -42.449527740478516,
      "activations/layer9_attention_weight_max": 32.13172912597656,
      "activations/layer9_attention_weight_min": -34.04066848754883,
      "epoch": 11.7,
      "learning_rate": 7.529242424242423e-05,
      "loss": 2.7855,
      "step": 201400
    },
    {
      "activations/layer0_attention_weight_max": 14.644698143005371,
      "activations/layer0_attention_weight_min": -11.576257705688477,
      "activations/layer10_attention_weight_max": 33.43501663208008,
      "activations/layer10_attention_weight_min": -31.650182723999023,
      "activations/layer11_attention_weight_max": 33.431793212890625,
      "activations/layer11_attention_weight_min": -31.899076461791992,
      "activations/layer12_attention_weight_max": 25.287593841552734,
      "activations/layer12_attention_weight_min": -24.732641220092773,
      "activations/layer13_attention_weight_max": 38.62211608886719,
      "activations/layer13_attention_weight_min": -34.13496780395508,
      "activations/layer14_attention_weight_max": 40.191375732421875,
      "activations/layer14_attention_weight_min": -34.498680114746094,
      "activations/layer15_attention_weight_max": 36.56391906738281,
      "activations/layer15_attention_weight_min": -30.412473678588867,
      "activations/layer16_attention_weight_max": 29.847673416137695,
      "activations/layer16_attention_weight_min": -27.219011306762695,
      "activations/layer17_attention_weight_max": 30.48116111755371,
      "activations/layer17_attention_weight_min": -25.40890884399414,
      "activations/layer18_attention_weight_max": 30.914897918701172,
      "activations/layer18_attention_weight_min": -24.21599578857422,
      "activations/layer19_attention_weight_max": 33.91571807861328,
      "activations/layer19_attention_weight_min": -31.230514526367188,
      "activations/layer1_attention_weight_max": 16.593353271484375,
      "activations/layer1_attention_weight_min": -14.054583549499512,
      "activations/layer20_attention_weight_max": 32.28034210205078,
      "activations/layer20_attention_weight_min": -24.85884666442871,
      "activations/layer21_attention_weight_max": 31.082786560058594,
      "activations/layer21_attention_weight_min": -25.10745620727539,
      "activations/layer22_attention_weight_max": 47.93889236450195,
      "activations/layer22_attention_weight_min": -31.478004455566406,
      "activations/layer23_attention_weight_max": 42.27726745605469,
      "activations/layer23_attention_weight_min": -26.988460540771484,
      "activations/layer2_attention_weight_max": 28.80837059020996,
      "activations/layer2_attention_weight_min": -28.643054962158203,
      "activations/layer3_attention_weight_max": 86.23261260986328,
      "activations/layer3_attention_weight_min": -87.48715209960938,
      "activations/layer4_attention_weight_max": 103.00919342041016,
      "activations/layer4_attention_weight_min": -107.18585205078125,
      "activations/layer5_attention_weight_max": 48.997703552246094,
      "activations/layer5_attention_weight_min": -61.37663269042969,
      "activations/layer6_attention_weight_max": 41.1241455078125,
      "activations/layer6_attention_weight_min": -48.54499816894531,
      "activations/layer7_attention_weight_max": 88.64303588867188,
      "activations/layer7_attention_weight_min": -92.70818328857422,
      "activations/layer8_attention_weight_max": 39.16447067260742,
      "activations/layer8_attention_weight_min": -41.23635482788086,
      "activations/layer9_attention_weight_max": 35.5490837097168,
      "activations/layer9_attention_weight_min": -33.37925720214844,
      "epoch": 11.71,
      "learning_rate": 7.527348484848484e-05,
      "loss": 2.7861,
      "step": 201450
    },
    {
      "activations/layer0_attention_weight_max": 16.031463623046875,
      "activations/layer0_attention_weight_min": -12.118223190307617,
      "activations/layer10_attention_weight_max": 31.5194091796875,
      "activations/layer10_attention_weight_min": -35.010066986083984,
      "activations/layer11_attention_weight_max": 31.859676361083984,
      "activations/layer11_attention_weight_min": -32.18336486816406,
      "activations/layer12_attention_weight_max": 37.906063079833984,
      "activations/layer12_attention_weight_min": -27.038862228393555,
      "activations/layer13_attention_weight_max": 51.570369720458984,
      "activations/layer13_attention_weight_min": -36.402889251708984,
      "activations/layer14_attention_weight_max": 47.155067443847656,
      "activations/layer14_attention_weight_min": -40.496437072753906,
      "activations/layer15_attention_weight_max": 37.70758819580078,
      "activations/layer15_attention_weight_min": -31.5162353515625,
      "activations/layer16_attention_weight_max": 29.366512298583984,
      "activations/layer16_attention_weight_min": -27.953723907470703,
      "activations/layer17_attention_weight_max": 31.49650764465332,
      "activations/layer17_attention_weight_min": -25.18619728088379,
      "activations/layer18_attention_weight_max": 32.647769927978516,
      "activations/layer18_attention_weight_min": -22.3629093170166,
      "activations/layer19_attention_weight_max": 35.684539794921875,
      "activations/layer19_attention_weight_min": -31.145095825195312,
      "activations/layer1_attention_weight_max": 16.82291603088379,
      "activations/layer1_attention_weight_min": -13.936758041381836,
      "activations/layer20_attention_weight_max": 32.70030212402344,
      "activations/layer20_attention_weight_min": -25.10613250732422,
      "activations/layer21_attention_weight_max": 31.81838607788086,
      "activations/layer21_attention_weight_min": -24.345714569091797,
      "activations/layer22_attention_weight_max": 46.5136833190918,
      "activations/layer22_attention_weight_min": -29.206806182861328,
      "activations/layer23_attention_weight_max": 37.62377166748047,
      "activations/layer23_attention_weight_min": -23.44351577758789,
      "activations/layer2_attention_weight_max": 29.847736358642578,
      "activations/layer2_attention_weight_min": -28.991100311279297,
      "activations/layer3_attention_weight_max": 87.21208953857422,
      "activations/layer3_attention_weight_min": -89.29478454589844,
      "activations/layer4_attention_weight_max": 104.86588287353516,
      "activations/layer4_attention_weight_min": -106.65750885009766,
      "activations/layer5_attention_weight_max": 53.66498947143555,
      "activations/layer5_attention_weight_min": -64.18479919433594,
      "activations/layer6_attention_weight_max": 43.108055114746094,
      "activations/layer6_attention_weight_min": -47.76423645019531,
      "activations/layer7_attention_weight_max": 91.06639862060547,
      "activations/layer7_attention_weight_min": -85.0901870727539,
      "activations/layer8_attention_weight_max": 39.95745849609375,
      "activations/layer8_attention_weight_min": -41.758766174316406,
      "activations/layer9_attention_weight_max": 33.76076889038086,
      "activations/layer9_attention_weight_min": -35.50033187866211,
      "epoch": 11.71,
      "learning_rate": 7.525454545454546e-05,
      "loss": 2.7776,
      "step": 201500
    },
    {
      "activations/layer0_attention_weight_max": 14.318720817565918,
      "activations/layer0_attention_weight_min": -11.982908248901367,
      "activations/layer10_attention_weight_max": 49.46134948730469,
      "activations/layer10_attention_weight_min": -43.2402458190918,
      "activations/layer11_attention_weight_max": 49.097469329833984,
      "activations/layer11_attention_weight_min": -43.34122848510742,
      "activations/layer12_attention_weight_max": 28.187875747680664,
      "activations/layer12_attention_weight_min": -30.297029495239258,
      "activations/layer13_attention_weight_max": 39.433753967285156,
      "activations/layer13_attention_weight_min": -34.35128402709961,
      "activations/layer14_attention_weight_max": 40.498741149902344,
      "activations/layer14_attention_weight_min": -35.59738540649414,
      "activations/layer15_attention_weight_max": 35.52568817138672,
      "activations/layer15_attention_weight_min": -33.441383361816406,
      "activations/layer16_attention_weight_max": 34.0038948059082,
      "activations/layer16_attention_weight_min": -29.17729377746582,
      "activations/layer17_attention_weight_max": 32.74680709838867,
      "activations/layer17_attention_weight_min": -28.151859283447266,
      "activations/layer18_attention_weight_max": 30.060161590576172,
      "activations/layer18_attention_weight_min": -25.42153549194336,
      "activations/layer19_attention_weight_max": 35.22980880737305,
      "activations/layer19_attention_weight_min": -30.46756362915039,
      "activations/layer1_attention_weight_max": 16.681217193603516,
      "activations/layer1_attention_weight_min": -14.473888397216797,
      "activations/layer20_attention_weight_max": 28.855690002441406,
      "activations/layer20_attention_weight_min": -24.582212448120117,
      "activations/layer21_attention_weight_max": 25.674646377563477,
      "activations/layer21_attention_weight_min": -23.770103454589844,
      "activations/layer22_attention_weight_max": 43.257320404052734,
      "activations/layer22_attention_weight_min": -29.20895767211914,
      "activations/layer23_attention_weight_max": 36.76184844970703,
      "activations/layer23_attention_weight_min": -25.046445846557617,
      "activations/layer2_attention_weight_max": 32.908790588378906,
      "activations/layer2_attention_weight_min": -35.016578674316406,
      "activations/layer3_attention_weight_max": 98.8999252319336,
      "activations/layer3_attention_weight_min": -102.9049301147461,
      "activations/layer4_attention_weight_max": 119.3881607055664,
      "activations/layer4_attention_weight_min": -112.2126693725586,
      "activations/layer5_attention_weight_max": 57.655548095703125,
      "activations/layer5_attention_weight_min": -64.30448913574219,
      "activations/layer6_attention_weight_max": 47.66200637817383,
      "activations/layer6_attention_weight_min": -46.313716888427734,
      "activations/layer7_attention_weight_max": 113.17221069335938,
      "activations/layer7_attention_weight_min": -101.57058715820312,
      "activations/layer8_attention_weight_max": 54.24446487426758,
      "activations/layer8_attention_weight_min": -48.39023971557617,
      "activations/layer9_attention_weight_max": 44.4898567199707,
      "activations/layer9_attention_weight_min": -39.90034484863281,
      "epoch": 11.71,
      "learning_rate": 7.523560606060605e-05,
      "loss": 2.7689,
      "step": 201550
    },
    {
      "activations/layer0_attention_weight_max": 15.826108932495117,
      "activations/layer0_attention_weight_min": -12.258691787719727,
      "activations/layer10_attention_weight_max": 29.891740798950195,
      "activations/layer10_attention_weight_min": -31.111425399780273,
      "activations/layer11_attention_weight_max": 30.35302734375,
      "activations/layer11_attention_weight_min": -30.05054473876953,
      "activations/layer12_attention_weight_max": 39.6971321105957,
      "activations/layer12_attention_weight_min": -25.772018432617188,
      "activations/layer13_attention_weight_max": 47.66459274291992,
      "activations/layer13_attention_weight_min": -36.04497528076172,
      "activations/layer14_attention_weight_max": 43.11387634277344,
      "activations/layer14_attention_weight_min": -39.62210464477539,
      "activations/layer15_attention_weight_max": 39.669822692871094,
      "activations/layer15_attention_weight_min": -35.8375244140625,
      "activations/layer16_attention_weight_max": 32.62763595581055,
      "activations/layer16_attention_weight_min": -31.309322357177734,
      "activations/layer17_attention_weight_max": 31.227426528930664,
      "activations/layer17_attention_weight_min": -27.161710739135742,
      "activations/layer18_attention_weight_max": 32.87838363647461,
      "activations/layer18_attention_weight_min": -23.509456634521484,
      "activations/layer19_attention_weight_max": 36.96959686279297,
      "activations/layer19_attention_weight_min": -31.54850196838379,
      "activations/layer1_attention_weight_max": 16.888338088989258,
      "activations/layer1_attention_weight_min": -13.047684669494629,
      "activations/layer20_attention_weight_max": 33.67621994018555,
      "activations/layer20_attention_weight_min": -26.57259178161621,
      "activations/layer21_attention_weight_max": 32.68898391723633,
      "activations/layer21_attention_weight_min": -24.74599266052246,
      "activations/layer22_attention_weight_max": 48.15581130981445,
      "activations/layer22_attention_weight_min": -29.69304084777832,
      "activations/layer23_attention_weight_max": 35.29346466064453,
      "activations/layer23_attention_weight_min": -25.570783615112305,
      "activations/layer2_attention_weight_max": 30.240550994873047,
      "activations/layer2_attention_weight_min": -29.288724899291992,
      "activations/layer3_attention_weight_max": 86.88343048095703,
      "activations/layer3_attention_weight_min": -91.15453338623047,
      "activations/layer4_attention_weight_max": 101.45232391357422,
      "activations/layer4_attention_weight_min": -105.26856994628906,
      "activations/layer5_attention_weight_max": 49.860530853271484,
      "activations/layer5_attention_weight_min": -60.266136169433594,
      "activations/layer6_attention_weight_max": 44.87531280517578,
      "activations/layer6_attention_weight_min": -48.314762115478516,
      "activations/layer7_attention_weight_max": 92.68515014648438,
      "activations/layer7_attention_weight_min": -88.97319793701172,
      "activations/layer8_attention_weight_max": 38.0057373046875,
      "activations/layer8_attention_weight_min": -39.44073486328125,
      "activations/layer9_attention_weight_max": 31.324655532836914,
      "activations/layer9_attention_weight_min": -32.438961029052734,
      "epoch": 11.71,
      "learning_rate": 7.521666666666666e-05,
      "loss": 2.7922,
      "step": 201600
    },
    {
      "activations/layer0_attention_weight_max": 14.929783821105957,
      "activations/layer0_attention_weight_min": -11.606330871582031,
      "activations/layer10_attention_weight_max": 34.798255920410156,
      "activations/layer10_attention_weight_min": -33.490760803222656,
      "activations/layer11_attention_weight_max": 33.53847885131836,
      "activations/layer11_attention_weight_min": -31.497787475585938,
      "activations/layer12_attention_weight_max": 25.394672393798828,
      "activations/layer12_attention_weight_min": -25.089609146118164,
      "activations/layer13_attention_weight_max": 38.372352600097656,
      "activations/layer13_attention_weight_min": -34.47438049316406,
      "activations/layer14_attention_weight_max": 44.571510314941406,
      "activations/layer14_attention_weight_min": -36.82743453979492,
      "activations/layer15_attention_weight_max": 38.0732421875,
      "activations/layer15_attention_weight_min": -31.391199111938477,
      "activations/layer16_attention_weight_max": 33.33358383178711,
      "activations/layer16_attention_weight_min": -28.347265243530273,
      "activations/layer17_attention_weight_max": 30.57263946533203,
      "activations/layer17_attention_weight_min": -24.50869369506836,
      "activations/layer18_attention_weight_max": 33.819488525390625,
      "activations/layer18_attention_weight_min": -23.92412757873535,
      "activations/layer19_attention_weight_max": 34.98021697998047,
      "activations/layer19_attention_weight_min": -31.20102310180664,
      "activations/layer1_attention_weight_max": 17.095121383666992,
      "activations/layer1_attention_weight_min": -15.217936515808105,
      "activations/layer20_attention_weight_max": 32.30451583862305,
      "activations/layer20_attention_weight_min": -26.85365867614746,
      "activations/layer21_attention_weight_max": 31.366710662841797,
      "activations/layer21_attention_weight_min": -27.369150161743164,
      "activations/layer22_attention_weight_max": 49.32441711425781,
      "activations/layer22_attention_weight_min": -31.960508346557617,
      "activations/layer23_attention_weight_max": 35.254722595214844,
      "activations/layer23_attention_weight_min": -24.64952850341797,
      "activations/layer2_attention_weight_max": 32.46930694580078,
      "activations/layer2_attention_weight_min": -30.945377349853516,
      "activations/layer3_attention_weight_max": 91.19615173339844,
      "activations/layer3_attention_weight_min": -93.26524353027344,
      "activations/layer4_attention_weight_max": 111.7166748046875,
      "activations/layer4_attention_weight_min": -115.97847747802734,
      "activations/layer5_attention_weight_max": 56.75080108642578,
      "activations/layer5_attention_weight_min": -64.92103576660156,
      "activations/layer6_attention_weight_max": 44.369407653808594,
      "activations/layer6_attention_weight_min": -48.94777297973633,
      "activations/layer7_attention_weight_max": 89.7217788696289,
      "activations/layer7_attention_weight_min": -96.10430908203125,
      "activations/layer8_attention_weight_max": 41.629817962646484,
      "activations/layer8_attention_weight_min": -42.51750183105469,
      "activations/layer9_attention_weight_max": 34.54072189331055,
      "activations/layer9_attention_weight_min": -35.73838806152344,
      "epoch": 11.72,
      "learning_rate": 7.519772727272726e-05,
      "loss": 2.7545,
      "step": 201650
    },
    {
      "activations/layer0_attention_weight_max": 15.14554500579834,
      "activations/layer0_attention_weight_min": -11.90426254272461,
      "activations/layer10_attention_weight_max": 41.63652801513672,
      "activations/layer10_attention_weight_min": -39.29059600830078,
      "activations/layer11_attention_weight_max": 45.221553802490234,
      "activations/layer11_attention_weight_min": -40.80078125,
      "activations/layer12_attention_weight_max": 24.654356002807617,
      "activations/layer12_attention_weight_min": -28.928485870361328,
      "activations/layer13_attention_weight_max": 37.7037467956543,
      "activations/layer13_attention_weight_min": -34.241695404052734,
      "activations/layer14_attention_weight_max": 38.558345794677734,
      "activations/layer14_attention_weight_min": -34.86208724975586,
      "activations/layer15_attention_weight_max": 35.81555938720703,
      "activations/layer15_attention_weight_min": -32.12553405761719,
      "activations/layer16_attention_weight_max": 28.009716033935547,
      "activations/layer16_attention_weight_min": -26.642446517944336,
      "activations/layer17_attention_weight_max": 30.199134826660156,
      "activations/layer17_attention_weight_min": -27.41465187072754,
      "activations/layer18_attention_weight_max": 31.094022750854492,
      "activations/layer18_attention_weight_min": -24.11495590209961,
      "activations/layer19_attention_weight_max": 33.096923828125,
      "activations/layer19_attention_weight_min": -30.190536499023438,
      "activations/layer1_attention_weight_max": 16.625850677490234,
      "activations/layer1_attention_weight_min": -13.966856956481934,
      "activations/layer20_attention_weight_max": 29.478702545166016,
      "activations/layer20_attention_weight_min": -25.723299026489258,
      "activations/layer21_attention_weight_max": 31.253204345703125,
      "activations/layer21_attention_weight_min": -25.926755905151367,
      "activations/layer22_attention_weight_max": 44.07740020751953,
      "activations/layer22_attention_weight_min": -29.189607620239258,
      "activations/layer23_attention_weight_max": 35.46461486816406,
      "activations/layer23_attention_weight_min": -25.736814498901367,
      "activations/layer2_attention_weight_max": 33.682918548583984,
      "activations/layer2_attention_weight_min": -33.881065368652344,
      "activations/layer3_attention_weight_max": 89.02995300292969,
      "activations/layer3_attention_weight_min": -94.43640899658203,
      "activations/layer4_attention_weight_max": 110.23719787597656,
      "activations/layer4_attention_weight_min": -107.72245025634766,
      "activations/layer5_attention_weight_max": 51.96680450439453,
      "activations/layer5_attention_weight_min": -65.34408569335938,
      "activations/layer6_attention_weight_max": 44.16270446777344,
      "activations/layer6_attention_weight_min": -48.44322204589844,
      "activations/layer7_attention_weight_max": 109.69139099121094,
      "activations/layer7_attention_weight_min": -86.77278137207031,
      "activations/layer8_attention_weight_max": 48.227378845214844,
      "activations/layer8_attention_weight_min": -42.23723220825195,
      "activations/layer9_attention_weight_max": 36.93982696533203,
      "activations/layer9_attention_weight_min": -38.589962005615234,
      "epoch": 11.72,
      "learning_rate": 7.517878787878788e-05,
      "loss": 2.7781,
      "step": 201700
    },
    {
      "activations/layer0_attention_weight_max": 15.142377853393555,
      "activations/layer0_attention_weight_min": -11.468252182006836,
      "activations/layer10_attention_weight_max": 34.17888641357422,
      "activations/layer10_attention_weight_min": -33.190040588378906,
      "activations/layer11_attention_weight_max": 34.138099670410156,
      "activations/layer11_attention_weight_min": -31.887645721435547,
      "activations/layer12_attention_weight_max": 26.118101119995117,
      "activations/layer12_attention_weight_min": -27.25542640686035,
      "activations/layer13_attention_weight_max": 40.84695053100586,
      "activations/layer13_attention_weight_min": -36.14324951171875,
      "activations/layer14_attention_weight_max": 47.37712097167969,
      "activations/layer14_attention_weight_min": -37.96830368041992,
      "activations/layer15_attention_weight_max": 36.832672119140625,
      "activations/layer15_attention_weight_min": -33.188907623291016,
      "activations/layer16_attention_weight_max": 34.12620162963867,
      "activations/layer16_attention_weight_min": -30.477664947509766,
      "activations/layer17_attention_weight_max": 30.521759033203125,
      "activations/layer17_attention_weight_min": -28.724599838256836,
      "activations/layer18_attention_weight_max": 32.91470718383789,
      "activations/layer18_attention_weight_min": -24.944990158081055,
      "activations/layer19_attention_weight_max": 33.02297592163086,
      "activations/layer19_attention_weight_min": -32.87857437133789,
      "activations/layer1_attention_weight_max": 16.87166976928711,
      "activations/layer1_attention_weight_min": -14.995058059692383,
      "activations/layer20_attention_weight_max": 27.980241775512695,
      "activations/layer20_attention_weight_min": -26.990581512451172,
      "activations/layer21_attention_weight_max": 28.61311912536621,
      "activations/layer21_attention_weight_min": -26.312118530273438,
      "activations/layer22_attention_weight_max": 45.55421447753906,
      "activations/layer22_attention_weight_min": -35.16177749633789,
      "activations/layer23_attention_weight_max": 32.8323974609375,
      "activations/layer23_attention_weight_min": -29.77248764038086,
      "activations/layer2_attention_weight_max": 32.32774353027344,
      "activations/layer2_attention_weight_min": -31.2454891204834,
      "activations/layer3_attention_weight_max": 95.5523910522461,
      "activations/layer3_attention_weight_min": -93.57894897460938,
      "activations/layer4_attention_weight_max": 114.55472564697266,
      "activations/layer4_attention_weight_min": -113.84706115722656,
      "activations/layer5_attention_weight_max": 53.99970626831055,
      "activations/layer5_attention_weight_min": -65.0282974243164,
      "activations/layer6_attention_weight_max": 46.51044464111328,
      "activations/layer6_attention_weight_min": -49.57693099975586,
      "activations/layer7_attention_weight_max": 94.14009857177734,
      "activations/layer7_attention_weight_min": -95.04179382324219,
      "activations/layer8_attention_weight_max": 41.4002571105957,
      "activations/layer8_attention_weight_min": -43.0988883972168,
      "activations/layer9_attention_weight_max": 32.712181091308594,
      "activations/layer9_attention_weight_min": -35.50178527832031,
      "epoch": 11.72,
      "learning_rate": 7.515984848484848e-05,
      "loss": 2.7839,
      "step": 201750
    },
    {
      "activations/layer0_attention_weight_max": 16.105867385864258,
      "activations/layer0_attention_weight_min": -12.115520477294922,
      "activations/layer10_attention_weight_max": 36.185455322265625,
      "activations/layer10_attention_weight_min": -32.06671905517578,
      "activations/layer11_attention_weight_max": 39.86317443847656,
      "activations/layer11_attention_weight_min": -32.481903076171875,
      "activations/layer12_attention_weight_max": 28.057512283325195,
      "activations/layer12_attention_weight_min": -25.0959415435791,
      "activations/layer13_attention_weight_max": 44.56873321533203,
      "activations/layer13_attention_weight_min": -35.42402648925781,
      "activations/layer14_attention_weight_max": 52.139991760253906,
      "activations/layer14_attention_weight_min": -39.981056213378906,
      "activations/layer15_attention_weight_max": 44.81221008300781,
      "activations/layer15_attention_weight_min": -31.40553092956543,
      "activations/layer16_attention_weight_max": 37.112972259521484,
      "activations/layer16_attention_weight_min": -26.38283348083496,
      "activations/layer17_attention_weight_max": 33.59674835205078,
      "activations/layer17_attention_weight_min": -25.92353057861328,
      "activations/layer18_attention_weight_max": 34.905799865722656,
      "activations/layer18_attention_weight_min": -24.719301223754883,
      "activations/layer19_attention_weight_max": 38.90946578979492,
      "activations/layer19_attention_weight_min": -31.652456283569336,
      "activations/layer1_attention_weight_max": 17.097667694091797,
      "activations/layer1_attention_weight_min": -14.112536430358887,
      "activations/layer20_attention_weight_max": 32.39188003540039,
      "activations/layer20_attention_weight_min": -23.92523956298828,
      "activations/layer21_attention_weight_max": 29.84420394897461,
      "activations/layer21_attention_weight_min": -22.662296295166016,
      "activations/layer22_attention_weight_max": 54.171058654785156,
      "activations/layer22_attention_weight_min": -28.085386276245117,
      "activations/layer23_attention_weight_max": 36.78152847290039,
      "activations/layer23_attention_weight_min": -23.478561401367188,
      "activations/layer2_attention_weight_max": 33.098880767822266,
      "activations/layer2_attention_weight_min": -31.1445369720459,
      "activations/layer3_attention_weight_max": 98.96780395507812,
      "activations/layer3_attention_weight_min": -95.31764221191406,
      "activations/layer4_attention_weight_max": 114.43712615966797,
      "activations/layer4_attention_weight_min": -106.53653717041016,
      "activations/layer5_attention_weight_max": 52.01770782470703,
      "activations/layer5_attention_weight_min": -61.62489318847656,
      "activations/layer6_attention_weight_max": 45.24873352050781,
      "activations/layer6_attention_weight_min": -45.34944534301758,
      "activations/layer7_attention_weight_max": 89.65215301513672,
      "activations/layer7_attention_weight_min": -93.14331817626953,
      "activations/layer8_attention_weight_max": 46.00489807128906,
      "activations/layer8_attention_weight_min": -42.08763885498047,
      "activations/layer9_attention_weight_max": 35.75474548339844,
      "activations/layer9_attention_weight_min": -32.275394439697266,
      "epoch": 11.73,
      "learning_rate": 7.514090909090908e-05,
      "loss": 2.7853,
      "step": 201800
    },
    {
      "activations/layer0_attention_weight_max": 15.288179397583008,
      "activations/layer0_attention_weight_min": -11.609007835388184,
      "activations/layer10_attention_weight_max": 35.839271545410156,
      "activations/layer10_attention_weight_min": -34.761627197265625,
      "activations/layer11_attention_weight_max": 37.36155700683594,
      "activations/layer11_attention_weight_min": -34.56397247314453,
      "activations/layer12_attention_weight_max": 25.51972198486328,
      "activations/layer12_attention_weight_min": -31.16032600402832,
      "activations/layer13_attention_weight_max": 39.934654235839844,
      "activations/layer13_attention_weight_min": -32.54945755004883,
      "activations/layer14_attention_weight_max": 45.11124038696289,
      "activations/layer14_attention_weight_min": -39.18083953857422,
      "activations/layer15_attention_weight_max": 37.930702209472656,
      "activations/layer15_attention_weight_min": -33.18659973144531,
      "activations/layer16_attention_weight_max": 31.627307891845703,
      "activations/layer16_attention_weight_min": -26.86370849609375,
      "activations/layer17_attention_weight_max": 38.52215576171875,
      "activations/layer17_attention_weight_min": -26.422853469848633,
      "activations/layer18_attention_weight_max": 31.736467361450195,
      "activations/layer18_attention_weight_min": -23.53693199157715,
      "activations/layer19_attention_weight_max": 32.0499267578125,
      "activations/layer19_attention_weight_min": -29.02207374572754,
      "activations/layer1_attention_weight_max": 18.541393280029297,
      "activations/layer1_attention_weight_min": -15.679732322692871,
      "activations/layer20_attention_weight_max": 29.852243423461914,
      "activations/layer20_attention_weight_min": -24.142131805419922,
      "activations/layer21_attention_weight_max": 30.190011978149414,
      "activations/layer21_attention_weight_min": -23.124176025390625,
      "activations/layer22_attention_weight_max": 41.32087707519531,
      "activations/layer22_attention_weight_min": -29.608430862426758,
      "activations/layer23_attention_weight_max": 32.4530143737793,
      "activations/layer23_attention_weight_min": -24.23492431640625,
      "activations/layer2_attention_weight_max": 32.10541534423828,
      "activations/layer2_attention_weight_min": -31.877609252929688,
      "activations/layer3_attention_weight_max": 95.47784423828125,
      "activations/layer3_attention_weight_min": -94.66835021972656,
      "activations/layer4_attention_weight_max": 112.1318588256836,
      "activations/layer4_attention_weight_min": -113.2640380859375,
      "activations/layer5_attention_weight_max": 52.989662170410156,
      "activations/layer5_attention_weight_min": -61.29279327392578,
      "activations/layer6_attention_weight_max": 47.09975051879883,
      "activations/layer6_attention_weight_min": -49.71128845214844,
      "activations/layer7_attention_weight_max": 92.33259582519531,
      "activations/layer7_attention_weight_min": -95.6291732788086,
      "activations/layer8_attention_weight_max": 42.6318244934082,
      "activations/layer8_attention_weight_min": -44.69771194458008,
      "activations/layer9_attention_weight_max": 34.813716888427734,
      "activations/layer9_attention_weight_min": -35.15076446533203,
      "epoch": 11.73,
      "learning_rate": 7.51219696969697e-05,
      "loss": 2.771,
      "step": 201850
    },
    {
      "activations/layer0_attention_weight_max": 16.807880401611328,
      "activations/layer0_attention_weight_min": -11.962902069091797,
      "activations/layer10_attention_weight_max": 32.84880828857422,
      "activations/layer10_attention_weight_min": -33.76153564453125,
      "activations/layer11_attention_weight_max": 33.81733703613281,
      "activations/layer11_attention_weight_min": -33.43551254272461,
      "activations/layer12_attention_weight_max": 24.240264892578125,
      "activations/layer12_attention_weight_min": -24.755054473876953,
      "activations/layer13_attention_weight_max": 39.51402282714844,
      "activations/layer13_attention_weight_min": -34.646080017089844,
      "activations/layer14_attention_weight_max": 44.43992614746094,
      "activations/layer14_attention_weight_min": -40.29277038574219,
      "activations/layer15_attention_weight_max": 38.76301193237305,
      "activations/layer15_attention_weight_min": -33.83510971069336,
      "activations/layer16_attention_weight_max": 30.542753219604492,
      "activations/layer16_attention_weight_min": -27.66196632385254,
      "activations/layer17_attention_weight_max": 29.01358985900879,
      "activations/layer17_attention_weight_min": -24.841928482055664,
      "activations/layer18_attention_weight_max": 30.390525817871094,
      "activations/layer18_attention_weight_min": -22.327159881591797,
      "activations/layer19_attention_weight_max": 30.751523971557617,
      "activations/layer19_attention_weight_min": -30.421951293945312,
      "activations/layer1_attention_weight_max": 16.78534698486328,
      "activations/layer1_attention_weight_min": -14.144753456115723,
      "activations/layer20_attention_weight_max": 27.653867721557617,
      "activations/layer20_attention_weight_min": -23.74138641357422,
      "activations/layer21_attention_weight_max": 27.98295783996582,
      "activations/layer21_attention_weight_min": -23.305131912231445,
      "activations/layer22_attention_weight_max": 41.27846145629883,
      "activations/layer22_attention_weight_min": -30.04836082458496,
      "activations/layer23_attention_weight_max": 32.8709831237793,
      "activations/layer23_attention_weight_min": -21.967254638671875,
      "activations/layer2_attention_weight_max": 30.61347198486328,
      "activations/layer2_attention_weight_min": -31.31280517578125,
      "activations/layer3_attention_weight_max": 94.46113586425781,
      "activations/layer3_attention_weight_min": -95.15303802490234,
      "activations/layer4_attention_weight_max": 112.3735580444336,
      "activations/layer4_attention_weight_min": -118.06764221191406,
      "activations/layer5_attention_weight_max": 51.81737518310547,
      "activations/layer5_attention_weight_min": -64.33464050292969,
      "activations/layer6_attention_weight_max": 44.26865005493164,
      "activations/layer6_attention_weight_min": -47.512325286865234,
      "activations/layer7_attention_weight_max": 88.02750396728516,
      "activations/layer7_attention_weight_min": -90.17381286621094,
      "activations/layer8_attention_weight_max": 41.08073806762695,
      "activations/layer8_attention_weight_min": -41.89156723022461,
      "activations/layer9_attention_weight_max": 33.832786560058594,
      "activations/layer9_attention_weight_min": -33.91823959350586,
      "epoch": 11.73,
      "learning_rate": 7.510303030303031e-05,
      "loss": 2.7849,
      "step": 201900
    },
    {
      "activations/layer0_attention_weight_max": 16.49800682067871,
      "activations/layer0_attention_weight_min": -10.712261199951172,
      "activations/layer10_attention_weight_max": 32.19025421142578,
      "activations/layer10_attention_weight_min": -32.892276763916016,
      "activations/layer11_attention_weight_max": 33.73490905761719,
      "activations/layer11_attention_weight_min": -32.562191009521484,
      "activations/layer12_attention_weight_max": 27.244352340698242,
      "activations/layer12_attention_weight_min": -25.611772537231445,
      "activations/layer13_attention_weight_max": 41.47117233276367,
      "activations/layer13_attention_weight_min": -34.318077087402344,
      "activations/layer14_attention_weight_max": 47.67901611328125,
      "activations/layer14_attention_weight_min": -41.07942581176758,
      "activations/layer15_attention_weight_max": 36.77469253540039,
      "activations/layer15_attention_weight_min": -33.39966583251953,
      "activations/layer16_attention_weight_max": 31.12604522705078,
      "activations/layer16_attention_weight_min": -28.103748321533203,
      "activations/layer17_attention_weight_max": 31.6335391998291,
      "activations/layer17_attention_weight_min": -25.36544418334961,
      "activations/layer18_attention_weight_max": 33.828887939453125,
      "activations/layer18_attention_weight_min": -23.38914680480957,
      "activations/layer19_attention_weight_max": 38.26060485839844,
      "activations/layer19_attention_weight_min": -31.705785751342773,
      "activations/layer1_attention_weight_max": 16.04130744934082,
      "activations/layer1_attention_weight_min": -13.744524955749512,
      "activations/layer20_attention_weight_max": 35.39532470703125,
      "activations/layer20_attention_weight_min": -26.68309211730957,
      "activations/layer21_attention_weight_max": 35.1982421875,
      "activations/layer21_attention_weight_min": -24.760793685913086,
      "activations/layer22_attention_weight_max": 48.1385612487793,
      "activations/layer22_attention_weight_min": -31.239364624023438,
      "activations/layer23_attention_weight_max": 41.15619659423828,
      "activations/layer23_attention_weight_min": -25.182178497314453,
      "activations/layer2_attention_weight_max": 31.73775291442871,
      "activations/layer2_attention_weight_min": -31.061307907104492,
      "activations/layer3_attention_weight_max": 96.96427917480469,
      "activations/layer3_attention_weight_min": -94.78529357910156,
      "activations/layer4_attention_weight_max": 114.3584976196289,
      "activations/layer4_attention_weight_min": -115.63204193115234,
      "activations/layer5_attention_weight_max": 52.980857849121094,
      "activations/layer5_attention_weight_min": -61.53779220581055,
      "activations/layer6_attention_weight_max": 48.72883987426758,
      "activations/layer6_attention_weight_min": -47.32286834716797,
      "activations/layer7_attention_weight_max": 91.64141845703125,
      "activations/layer7_attention_weight_min": -89.41044616699219,
      "activations/layer8_attention_weight_max": 41.5398063659668,
      "activations/layer8_attention_weight_min": -43.88930130004883,
      "activations/layer9_attention_weight_max": 33.659629821777344,
      "activations/layer9_attention_weight_min": -32.996334075927734,
      "epoch": 11.73,
      "learning_rate": 7.50840909090909e-05,
      "loss": 2.7858,
      "step": 201950
    },
    {
      "activations/layer0_attention_weight_max": 16.890155792236328,
      "activations/layer0_attention_weight_min": -12.066726684570312,
      "activations/layer10_attention_weight_max": 31.37874412536621,
      "activations/layer10_attention_weight_min": -32.52019119262695,
      "activations/layer11_attention_weight_max": 32.499900817871094,
      "activations/layer11_attention_weight_min": -30.397878646850586,
      "activations/layer12_attention_weight_max": 23.462623596191406,
      "activations/layer12_attention_weight_min": -27.990507125854492,
      "activations/layer13_attention_weight_max": 39.05866622924805,
      "activations/layer13_attention_weight_min": -32.91442108154297,
      "activations/layer14_attention_weight_max": 38.95634078979492,
      "activations/layer14_attention_weight_min": -35.312461853027344,
      "activations/layer15_attention_weight_max": 33.7703971862793,
      "activations/layer15_attention_weight_min": -31.353639602661133,
      "activations/layer16_attention_weight_max": 30.142377853393555,
      "activations/layer16_attention_weight_min": -28.09527587890625,
      "activations/layer17_attention_weight_max": 31.330169677734375,
      "activations/layer17_attention_weight_min": -25.012887954711914,
      "activations/layer18_attention_weight_max": 32.33135223388672,
      "activations/layer18_attention_weight_min": -22.712860107421875,
      "activations/layer19_attention_weight_max": 37.62575912475586,
      "activations/layer19_attention_weight_min": -31.57722282409668,
      "activations/layer1_attention_weight_max": 17.466806411743164,
      "activations/layer1_attention_weight_min": -13.338255882263184,
      "activations/layer20_attention_weight_max": 28.64208984375,
      "activations/layer20_attention_weight_min": -25.90729331970215,
      "activations/layer21_attention_weight_max": 28.59235191345215,
      "activations/layer21_attention_weight_min": -24.972829818725586,
      "activations/layer22_attention_weight_max": 50.39854049682617,
      "activations/layer22_attention_weight_min": -30.993881225585938,
      "activations/layer23_attention_weight_max": 40.509132385253906,
      "activations/layer23_attention_weight_min": -25.232070922851562,
      "activations/layer2_attention_weight_max": 31.931926727294922,
      "activations/layer2_attention_weight_min": -31.32949447631836,
      "activations/layer3_attention_weight_max": 95.28466796875,
      "activations/layer3_attention_weight_min": -96.1631088256836,
      "activations/layer4_attention_weight_max": 114.8077163696289,
      "activations/layer4_attention_weight_min": -110.62458801269531,
      "activations/layer5_attention_weight_max": 53.49204635620117,
      "activations/layer5_attention_weight_min": -63.76723861694336,
      "activations/layer6_attention_weight_max": 43.75076675415039,
      "activations/layer6_attention_weight_min": -47.85886001586914,
      "activations/layer7_attention_weight_max": 87.13636779785156,
      "activations/layer7_attention_weight_min": -91.1492691040039,
      "activations/layer8_attention_weight_max": 39.15766143798828,
      "activations/layer8_attention_weight_min": -41.0837516784668,
      "activations/layer9_attention_weight_max": 34.62000274658203,
      "activations/layer9_attention_weight_min": -31.529905319213867,
      "epoch": 11.74,
      "learning_rate": 7.506515151515151e-05,
      "loss": 2.7756,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5662,
      "eval_samples_per_second": 501.271,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5662,
      "eval_openwebtext_samples_per_second": 501.271,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 2.0214,
      "eval_wikitext_samples_per_second": 225.588,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_lambada_loss": 2.46484375,
      "eval_lambada_ppl": 11.761644251483835,
      "eval_lambada_runtime": 9.6137,
      "eval_lambada_samples_per_second": 506.463,
      "step": 202000
    },
    {
      "activations/layer0_attention_weight_max": 15.890467643737793,
      "activations/layer0_attention_weight_min": -11.499163627624512,
      "activations/layer10_attention_weight_max": 30.140018463134766,
      "activations/layer10_attention_weight_min": -31.236276626586914,
      "activations/layer11_attention_weight_max": 31.18947982788086,
      "activations/layer11_attention_weight_min": -30.772193908691406,
      "activations/layer12_attention_weight_max": 23.14566993713379,
      "activations/layer12_attention_weight_min": -26.681297302246094,
      "activations/layer13_attention_weight_max": 36.70777893066406,
      "activations/layer13_attention_weight_min": -35.04066467285156,
      "activations/layer14_attention_weight_max": 40.6328010559082,
      "activations/layer14_attention_weight_min": -35.51932907104492,
      "activations/layer15_attention_weight_max": 33.77146530151367,
      "activations/layer15_attention_weight_min": -33.75663757324219,
      "activations/layer16_attention_weight_max": 30.63218879699707,
      "activations/layer16_attention_weight_min": -28.358963012695312,
      "activations/layer17_attention_weight_max": 30.65330696105957,
      "activations/layer17_attention_weight_min": -26.59202766418457,
      "activations/layer18_attention_weight_max": 31.267324447631836,
      "activations/layer18_attention_weight_min": -25.173738479614258,
      "activations/layer19_attention_weight_max": 32.80009841918945,
      "activations/layer19_attention_weight_min": -32.53646469116211,
      "activations/layer1_attention_weight_max": 15.940630912780762,
      "activations/layer1_attention_weight_min": -14.789793968200684,
      "activations/layer20_attention_weight_max": 29.4400634765625,
      "activations/layer20_attention_weight_min": -24.939979553222656,
      "activations/layer21_attention_weight_max": 29.59191131591797,
      "activations/layer21_attention_weight_min": -24.511064529418945,
      "activations/layer22_attention_weight_max": 46.17952346801758,
      "activations/layer22_attention_weight_min": -30.84494972229004,
      "activations/layer23_attention_weight_max": 32.500022888183594,
      "activations/layer23_attention_weight_min": -27.695083618164062,
      "activations/layer2_attention_weight_max": 33.091529846191406,
      "activations/layer2_attention_weight_min": -32.035789489746094,
      "activations/layer3_attention_weight_max": 94.28252410888672,
      "activations/layer3_attention_weight_min": -97.38861083984375,
      "activations/layer4_attention_weight_max": 111.5650863647461,
      "activations/layer4_attention_weight_min": -117.6086196899414,
      "activations/layer5_attention_weight_max": 51.31142807006836,
      "activations/layer5_attention_weight_min": -65.08560180664062,
      "activations/layer6_attention_weight_max": 43.37026596069336,
      "activations/layer6_attention_weight_min": -47.451847076416016,
      "activations/layer7_attention_weight_max": 85.2728042602539,
      "activations/layer7_attention_weight_min": -90.77969360351562,
      "activations/layer8_attention_weight_max": 37.31227493286133,
      "activations/layer8_attention_weight_min": -40.48738098144531,
      "activations/layer9_attention_weight_max": 30.594755172729492,
      "activations/layer9_attention_weight_min": -32.68354797363281,
      "epoch": 11.74,
      "learning_rate": 7.50462121212121e-05,
      "loss": 2.7832,
      "step": 202050
    },
    {
      "activations/layer0_attention_weight_max": 14.485108375549316,
      "activations/layer0_attention_weight_min": -12.249008178710938,
      "activations/layer10_attention_weight_max": 37.579315185546875,
      "activations/layer10_attention_weight_min": -34.49449920654297,
      "activations/layer11_attention_weight_max": 38.30370330810547,
      "activations/layer11_attention_weight_min": -32.57331085205078,
      "activations/layer12_attention_weight_max": 30.201236724853516,
      "activations/layer12_attention_weight_min": -25.164323806762695,
      "activations/layer13_attention_weight_max": 45.60456848144531,
      "activations/layer13_attention_weight_min": -34.639190673828125,
      "activations/layer14_attention_weight_max": 49.570960998535156,
      "activations/layer14_attention_weight_min": -36.742191314697266,
      "activations/layer15_attention_weight_max": 39.774967193603516,
      "activations/layer15_attention_weight_min": -33.04066848754883,
      "activations/layer16_attention_weight_max": 33.513519287109375,
      "activations/layer16_attention_weight_min": -29.751407623291016,
      "activations/layer17_attention_weight_max": 30.74672508239746,
      "activations/layer17_attention_weight_min": -25.757333755493164,
      "activations/layer18_attention_weight_max": 30.239919662475586,
      "activations/layer18_attention_weight_min": -25.13394546508789,
      "activations/layer19_attention_weight_max": 35.90251541137695,
      "activations/layer19_attention_weight_min": -29.115304946899414,
      "activations/layer1_attention_weight_max": 16.523757934570312,
      "activations/layer1_attention_weight_min": -15.198637008666992,
      "activations/layer20_attention_weight_max": 31.471960067749023,
      "activations/layer20_attention_weight_min": -24.157630920410156,
      "activations/layer21_attention_weight_max": 30.048254013061523,
      "activations/layer21_attention_weight_min": -24.347604751586914,
      "activations/layer22_attention_weight_max": 44.3553466796875,
      "activations/layer22_attention_weight_min": -31.206466674804688,
      "activations/layer23_attention_weight_max": 37.43503189086914,
      "activations/layer23_attention_weight_min": -24.704814910888672,
      "activations/layer2_attention_weight_max": 32.07069396972656,
      "activations/layer2_attention_weight_min": -32.26885223388672,
      "activations/layer3_attention_weight_max": 95.31055450439453,
      "activations/layer3_attention_weight_min": -98.35525512695312,
      "activations/layer4_attention_weight_max": 112.98779296875,
      "activations/layer4_attention_weight_min": -111.48844146728516,
      "activations/layer5_attention_weight_max": 51.50679016113281,
      "activations/layer5_attention_weight_min": -61.2496452331543,
      "activations/layer6_attention_weight_max": 44.39934158325195,
      "activations/layer6_attention_weight_min": -46.91537857055664,
      "activations/layer7_attention_weight_max": 89.44647979736328,
      "activations/layer7_attention_weight_min": -93.23826599121094,
      "activations/layer8_attention_weight_max": 47.48866653442383,
      "activations/layer8_attention_weight_min": -43.17201232910156,
      "activations/layer9_attention_weight_max": 36.94288635253906,
      "activations/layer9_attention_weight_min": -35.23774719238281,
      "epoch": 11.74,
      "learning_rate": 7.502727272727272e-05,
      "loss": 2.7885,
      "step": 202100
    },
    {
      "activations/layer0_attention_weight_max": 14.897825241088867,
      "activations/layer0_attention_weight_min": -11.588057518005371,
      "activations/layer10_attention_weight_max": 35.2993049621582,
      "activations/layer10_attention_weight_min": -32.98764419555664,
      "activations/layer11_attention_weight_max": 38.45524597167969,
      "activations/layer11_attention_weight_min": -36.4522590637207,
      "activations/layer12_attention_weight_max": 24.67510414123535,
      "activations/layer12_attention_weight_min": -27.94749641418457,
      "activations/layer13_attention_weight_max": 38.17152786254883,
      "activations/layer13_attention_weight_min": -33.56391143798828,
      "activations/layer14_attention_weight_max": 46.086029052734375,
      "activations/layer14_attention_weight_min": -37.32197189331055,
      "activations/layer15_attention_weight_max": 36.401371002197266,
      "activations/layer15_attention_weight_min": -35.501220703125,
      "activations/layer16_attention_weight_max": 33.09492111206055,
      "activations/layer16_attention_weight_min": -28.827119827270508,
      "activations/layer17_attention_weight_max": 30.859086990356445,
      "activations/layer17_attention_weight_min": -29.064111709594727,
      "activations/layer18_attention_weight_max": 30.9266300201416,
      "activations/layer18_attention_weight_min": -24.15642738342285,
      "activations/layer19_attention_weight_max": 34.08494567871094,
      "activations/layer19_attention_weight_min": -29.97233009338379,
      "activations/layer1_attention_weight_max": 17.651060104370117,
      "activations/layer1_attention_weight_min": -14.63050651550293,
      "activations/layer20_attention_weight_max": 28.853893280029297,
      "activations/layer20_attention_weight_min": -23.49759864807129,
      "activations/layer21_attention_weight_max": 30.725296020507812,
      "activations/layer21_attention_weight_min": -22.17619514465332,
      "activations/layer22_attention_weight_max": 46.35164260864258,
      "activations/layer22_attention_weight_min": -29.539201736450195,
      "activations/layer23_attention_weight_max": 33.35077667236328,
      "activations/layer23_attention_weight_min": -25.511709213256836,
      "activations/layer2_attention_weight_max": 33.73005676269531,
      "activations/layer2_attention_weight_min": -32.8543701171875,
      "activations/layer3_attention_weight_max": 96.3178482055664,
      "activations/layer3_attention_weight_min": -101.0512924194336,
      "activations/layer4_attention_weight_max": 113.613037109375,
      "activations/layer4_attention_weight_min": -113.9331283569336,
      "activations/layer5_attention_weight_max": 54.29460144042969,
      "activations/layer5_attention_weight_min": -61.25963592529297,
      "activations/layer6_attention_weight_max": 47.263118743896484,
      "activations/layer6_attention_weight_min": -46.72492599487305,
      "activations/layer7_attention_weight_max": 95.59454345703125,
      "activations/layer7_attention_weight_min": -91.18528747558594,
      "activations/layer8_attention_weight_max": 43.113319396972656,
      "activations/layer8_attention_weight_min": -44.38762664794922,
      "activations/layer9_attention_weight_max": 35.1380729675293,
      "activations/layer9_attention_weight_min": -35.25983810424805,
      "epoch": 11.75,
      "learning_rate": 7.500833333333333e-05,
      "loss": 2.7854,
      "step": 202150
    },
    {
      "activations/layer0_attention_weight_max": 15.198091506958008,
      "activations/layer0_attention_weight_min": -12.170498847961426,
      "activations/layer10_attention_weight_max": 33.16696548461914,
      "activations/layer10_attention_weight_min": -31.865169525146484,
      "activations/layer11_attention_weight_max": 33.83625793457031,
      "activations/layer11_attention_weight_min": -32.39878463745117,
      "activations/layer12_attention_weight_max": 26.255094528198242,
      "activations/layer12_attention_weight_min": -24.40188217163086,
      "activations/layer13_attention_weight_max": 37.303428649902344,
      "activations/layer13_attention_weight_min": -32.252925872802734,
      "activations/layer14_attention_weight_max": 46.31983184814453,
      "activations/layer14_attention_weight_min": -38.345619201660156,
      "activations/layer15_attention_weight_max": 34.99116897583008,
      "activations/layer15_attention_weight_min": -30.88614273071289,
      "activations/layer16_attention_weight_max": 32.94449234008789,
      "activations/layer16_attention_weight_min": -28.1866455078125,
      "activations/layer17_attention_weight_max": 34.63260269165039,
      "activations/layer17_attention_weight_min": -26.708471298217773,
      "activations/layer18_attention_weight_max": 32.832706451416016,
      "activations/layer18_attention_weight_min": -25.192272186279297,
      "activations/layer19_attention_weight_max": 39.96636199951172,
      "activations/layer19_attention_weight_min": -32.247108459472656,
      "activations/layer1_attention_weight_max": 17.355918884277344,
      "activations/layer1_attention_weight_min": -14.603242874145508,
      "activations/layer20_attention_weight_max": 31.731786727905273,
      "activations/layer20_attention_weight_min": -25.71038055419922,
      "activations/layer21_attention_weight_max": 29.929025650024414,
      "activations/layer21_attention_weight_min": -25.780179977416992,
      "activations/layer22_attention_weight_max": 52.503780364990234,
      "activations/layer22_attention_weight_min": -31.950288772583008,
      "activations/layer23_attention_weight_max": 38.98331069946289,
      "activations/layer23_attention_weight_min": -25.838237762451172,
      "activations/layer2_attention_weight_max": 31.831371307373047,
      "activations/layer2_attention_weight_min": -31.214900970458984,
      "activations/layer3_attention_weight_max": 94.34967041015625,
      "activations/layer3_attention_weight_min": -94.99801635742188,
      "activations/layer4_attention_weight_max": 108.92935943603516,
      "activations/layer4_attention_weight_min": -107.87794494628906,
      "activations/layer5_attention_weight_max": 49.119903564453125,
      "activations/layer5_attention_weight_min": -62.77294921875,
      "activations/layer6_attention_weight_max": 46.279537200927734,
      "activations/layer6_attention_weight_min": -48.228492736816406,
      "activations/layer7_attention_weight_max": 89.19173431396484,
      "activations/layer7_attention_weight_min": -90.94300079345703,
      "activations/layer8_attention_weight_max": 40.504302978515625,
      "activations/layer8_attention_weight_min": -43.41770553588867,
      "activations/layer9_attention_weight_max": 33.387630462646484,
      "activations/layer9_attention_weight_min": -33.3038444519043,
      "epoch": 11.75,
      "learning_rate": 7.498939393939393e-05,
      "loss": 2.7737,
      "step": 202200
    },
    {
      "activations/layer0_attention_weight_max": 15.938928604125977,
      "activations/layer0_attention_weight_min": -11.883699417114258,
      "activations/layer10_attention_weight_max": 31.57965087890625,
      "activations/layer10_attention_weight_min": -31.793737411499023,
      "activations/layer11_attention_weight_max": 29.981611251831055,
      "activations/layer11_attention_weight_min": -31.524229049682617,
      "activations/layer12_attention_weight_max": 24.65283966064453,
      "activations/layer12_attention_weight_min": -26.33055305480957,
      "activations/layer13_attention_weight_max": 36.377227783203125,
      "activations/layer13_attention_weight_min": -32.454471588134766,
      "activations/layer14_attention_weight_max": 39.297752380371094,
      "activations/layer14_attention_weight_min": -35.67063903808594,
      "activations/layer15_attention_weight_max": 32.781864166259766,
      "activations/layer15_attention_weight_min": -31.293424606323242,
      "activations/layer16_attention_weight_max": 28.17698860168457,
      "activations/layer16_attention_weight_min": -27.59280776977539,
      "activations/layer17_attention_weight_max": 30.828561782836914,
      "activations/layer17_attention_weight_min": -26.100065231323242,
      "activations/layer18_attention_weight_max": 31.073232650756836,
      "activations/layer18_attention_weight_min": -23.78600311279297,
      "activations/layer19_attention_weight_max": 35.21620559692383,
      "activations/layer19_attention_weight_min": -29.126707077026367,
      "activations/layer1_attention_weight_max": 16.531429290771484,
      "activations/layer1_attention_weight_min": -15.115076065063477,
      "activations/layer20_attention_weight_max": 32.88559341430664,
      "activations/layer20_attention_weight_min": -24.437786102294922,
      "activations/layer21_attention_weight_max": 32.15779113769531,
      "activations/layer21_attention_weight_min": -24.39958381652832,
      "activations/layer22_attention_weight_max": 51.44011688232422,
      "activations/layer22_attention_weight_min": -29.614105224609375,
      "activations/layer23_attention_weight_max": 36.26280212402344,
      "activations/layer23_attention_weight_min": -25.104717254638672,
      "activations/layer2_attention_weight_max": 31.686485290527344,
      "activations/layer2_attention_weight_min": -30.120849609375,
      "activations/layer3_attention_weight_max": 91.79600524902344,
      "activations/layer3_attention_weight_min": -91.46996307373047,
      "activations/layer4_attention_weight_max": 111.51371002197266,
      "activations/layer4_attention_weight_min": -107.1101303100586,
      "activations/layer5_attention_weight_max": 53.44999313354492,
      "activations/layer5_attention_weight_min": -59.71055221557617,
      "activations/layer6_attention_weight_max": 44.710941314697266,
      "activations/layer6_attention_weight_min": -45.46398162841797,
      "activations/layer7_attention_weight_max": 86.64501953125,
      "activations/layer7_attention_weight_min": -83.13383483886719,
      "activations/layer8_attention_weight_max": 41.02408981323242,
      "activations/layer8_attention_weight_min": -38.94252395629883,
      "activations/layer9_attention_weight_max": 30.484495162963867,
      "activations/layer9_attention_weight_min": -31.810300827026367,
      "epoch": 11.75,
      "learning_rate": 7.497045454545453e-05,
      "loss": 2.7797,
      "step": 202250
    },
    {
      "activations/layer0_attention_weight_max": 14.970980644226074,
      "activations/layer0_attention_weight_min": -11.487425804138184,
      "activations/layer10_attention_weight_max": 36.76148223876953,
      "activations/layer10_attention_weight_min": -35.07857131958008,
      "activations/layer11_attention_weight_max": 35.29154968261719,
      "activations/layer11_attention_weight_min": -31.364004135131836,
      "activations/layer12_attention_weight_max": 27.02536964416504,
      "activations/layer12_attention_weight_min": -26.52901840209961,
      "activations/layer13_attention_weight_max": 42.293006896972656,
      "activations/layer13_attention_weight_min": -35.741451263427734,
      "activations/layer14_attention_weight_max": 53.184898376464844,
      "activations/layer14_attention_weight_min": -42.775577545166016,
      "activations/layer15_attention_weight_max": 39.67315673828125,
      "activations/layer15_attention_weight_min": -34.24544906616211,
      "activations/layer16_attention_weight_max": 35.72607421875,
      "activations/layer16_attention_weight_min": -30.090864181518555,
      "activations/layer17_attention_weight_max": 36.33079147338867,
      "activations/layer17_attention_weight_min": -27.042165756225586,
      "activations/layer18_attention_weight_max": 35.58012008666992,
      "activations/layer18_attention_weight_min": -24.43033790588379,
      "activations/layer19_attention_weight_max": 42.032073974609375,
      "activations/layer19_attention_weight_min": -34.66998291015625,
      "activations/layer1_attention_weight_max": 16.302452087402344,
      "activations/layer1_attention_weight_min": -13.960749626159668,
      "activations/layer20_attention_weight_max": 38.665367126464844,
      "activations/layer20_attention_weight_min": -26.652698516845703,
      "activations/layer21_attention_weight_max": 35.84294509887695,
      "activations/layer21_attention_weight_min": -24.767210006713867,
      "activations/layer22_attention_weight_max": 57.49796676635742,
      "activations/layer22_attention_weight_min": -31.94770050048828,
      "activations/layer23_attention_weight_max": 40.88478469848633,
      "activations/layer23_attention_weight_min": -23.77475357055664,
      "activations/layer2_attention_weight_max": 32.81766128540039,
      "activations/layer2_attention_weight_min": -32.19340896606445,
      "activations/layer3_attention_weight_max": 91.59636688232422,
      "activations/layer3_attention_weight_min": -94.02445983886719,
      "activations/layer4_attention_weight_max": 111.18348693847656,
      "activations/layer4_attention_weight_min": -114.1237564086914,
      "activations/layer5_attention_weight_max": 51.2100830078125,
      "activations/layer5_attention_weight_min": -64.2039794921875,
      "activations/layer6_attention_weight_max": 44.948997497558594,
      "activations/layer6_attention_weight_min": -48.75835037231445,
      "activations/layer7_attention_weight_max": 93.22964477539062,
      "activations/layer7_attention_weight_min": -94.9346923828125,
      "activations/layer8_attention_weight_max": 43.333805084228516,
      "activations/layer8_attention_weight_min": -43.08095932006836,
      "activations/layer9_attention_weight_max": 34.77119827270508,
      "activations/layer9_attention_weight_min": -34.002593994140625,
      "epoch": 11.75,
      "learning_rate": 7.495151515151515e-05,
      "loss": 2.7831,
      "step": 202300
    },
    {
      "activations/layer0_attention_weight_max": 15.489707946777344,
      "activations/layer0_attention_weight_min": -11.856305122375488,
      "activations/layer10_attention_weight_max": 33.212623596191406,
      "activations/layer10_attention_weight_min": -33.63585662841797,
      "activations/layer11_attention_weight_max": 35.0269775390625,
      "activations/layer11_attention_weight_min": -37.63492965698242,
      "activations/layer12_attention_weight_max": 24.84707260131836,
      "activations/layer12_attention_weight_min": -25.99286651611328,
      "activations/layer13_attention_weight_max": 35.82345199584961,
      "activations/layer13_attention_weight_min": -35.96702575683594,
      "activations/layer14_attention_weight_max": 39.94892501831055,
      "activations/layer14_attention_weight_min": -37.88703918457031,
      "activations/layer15_attention_weight_max": 34.52388000488281,
      "activations/layer15_attention_weight_min": -33.18878936767578,
      "activations/layer16_attention_weight_max": 30.450435638427734,
      "activations/layer16_attention_weight_min": -27.989376068115234,
      "activations/layer17_attention_weight_max": 30.91341209411621,
      "activations/layer17_attention_weight_min": -25.772167205810547,
      "activations/layer18_attention_weight_max": 29.34765625,
      "activations/layer18_attention_weight_min": -23.430368423461914,
      "activations/layer19_attention_weight_max": 33.61725997924805,
      "activations/layer19_attention_weight_min": -31.667022705078125,
      "activations/layer1_attention_weight_max": 16.541940689086914,
      "activations/layer1_attention_weight_min": -13.109565734863281,
      "activations/layer20_attention_weight_max": 30.384885787963867,
      "activations/layer20_attention_weight_min": -26.082218170166016,
      "activations/layer21_attention_weight_max": 28.60638999938965,
      "activations/layer21_attention_weight_min": -24.24461555480957,
      "activations/layer22_attention_weight_max": 38.876163482666016,
      "activations/layer22_attention_weight_min": -28.89776039123535,
      "activations/layer23_attention_weight_max": 31.639013290405273,
      "activations/layer23_attention_weight_min": -24.42502212524414,
      "activations/layer2_attention_weight_max": 33.21553421020508,
      "activations/layer2_attention_weight_min": -33.01799011230469,
      "activations/layer3_attention_weight_max": 92.4507827758789,
      "activations/layer3_attention_weight_min": -96.32012176513672,
      "activations/layer4_attention_weight_max": 109.81977844238281,
      "activations/layer4_attention_weight_min": -112.44258880615234,
      "activations/layer5_attention_weight_max": 52.25558090209961,
      "activations/layer5_attention_weight_min": -62.30804443359375,
      "activations/layer6_attention_weight_max": 44.24100112915039,
      "activations/layer6_attention_weight_min": -47.46670913696289,
      "activations/layer7_attention_weight_max": 90.4367904663086,
      "activations/layer7_attention_weight_min": -90.79388427734375,
      "activations/layer8_attention_weight_max": 40.504554748535156,
      "activations/layer8_attention_weight_min": -42.55029296875,
      "activations/layer9_attention_weight_max": 32.3199577331543,
      "activations/layer9_attention_weight_min": -33.798274993896484,
      "epoch": 11.76,
      "learning_rate": 7.493257575757575e-05,
      "loss": 2.7718,
      "step": 202350
    },
    {
      "activations/layer0_attention_weight_max": 15.426145553588867,
      "activations/layer0_attention_weight_min": -11.91983699798584,
      "activations/layer10_attention_weight_max": 36.81943893432617,
      "activations/layer10_attention_weight_min": -32.06792068481445,
      "activations/layer11_attention_weight_max": 38.35768127441406,
      "activations/layer11_attention_weight_min": -33.539939880371094,
      "activations/layer12_attention_weight_max": 24.789573669433594,
      "activations/layer12_attention_weight_min": -26.864669799804688,
      "activations/layer13_attention_weight_max": 38.51250076293945,
      "activations/layer13_attention_weight_min": -34.85268783569336,
      "activations/layer14_attention_weight_max": 43.18069839477539,
      "activations/layer14_attention_weight_min": -39.277957916259766,
      "activations/layer15_attention_weight_max": 35.008121490478516,
      "activations/layer15_attention_weight_min": -32.8140869140625,
      "activations/layer16_attention_weight_max": 33.404449462890625,
      "activations/layer16_attention_weight_min": -27.369384765625,
      "activations/layer17_attention_weight_max": 35.93044662475586,
      "activations/layer17_attention_weight_min": -24.99234962463379,
      "activations/layer18_attention_weight_max": 33.72720718383789,
      "activations/layer18_attention_weight_min": -23.37701416015625,
      "activations/layer19_attention_weight_max": 36.96895980834961,
      "activations/layer19_attention_weight_min": -30.35660171508789,
      "activations/layer1_attention_weight_max": 15.807703018188477,
      "activations/layer1_attention_weight_min": -14.233319282531738,
      "activations/layer20_attention_weight_max": 34.24308776855469,
      "activations/layer20_attention_weight_min": -25.042644500732422,
      "activations/layer21_attention_weight_max": 31.413070678710938,
      "activations/layer21_attention_weight_min": -22.81487464904785,
      "activations/layer22_attention_weight_max": 45.814247131347656,
      "activations/layer22_attention_weight_min": -30.14201545715332,
      "activations/layer23_attention_weight_max": 35.89801788330078,
      "activations/layer23_attention_weight_min": -22.261764526367188,
      "activations/layer2_attention_weight_max": 34.676918029785156,
      "activations/layer2_attention_weight_min": -32.56504821777344,
      "activations/layer3_attention_weight_max": 94.14997863769531,
      "activations/layer3_attention_weight_min": -96.7270736694336,
      "activations/layer4_attention_weight_max": 110.2311019897461,
      "activations/layer4_attention_weight_min": -112.9643783569336,
      "activations/layer5_attention_weight_max": 51.31936264038086,
      "activations/layer5_attention_weight_min": -61.069820404052734,
      "activations/layer6_attention_weight_max": 44.858951568603516,
      "activations/layer6_attention_weight_min": -46.64686584472656,
      "activations/layer7_attention_weight_max": 93.04798889160156,
      "activations/layer7_attention_weight_min": -95.02078247070312,
      "activations/layer8_attention_weight_max": 42.65671157836914,
      "activations/layer8_attention_weight_min": -40.862144470214844,
      "activations/layer9_attention_weight_max": 35.36373519897461,
      "activations/layer9_attention_weight_min": -33.053890228271484,
      "epoch": 11.76,
      "learning_rate": 7.491363636363635e-05,
      "loss": 2.764,
      "step": 202400
    },
    {
      "activations/layer0_attention_weight_max": 15.884783744812012,
      "activations/layer0_attention_weight_min": -12.600415229797363,
      "activations/layer10_attention_weight_max": 32.048851013183594,
      "activations/layer10_attention_weight_min": -33.57713317871094,
      "activations/layer11_attention_weight_max": 33.62044906616211,
      "activations/layer11_attention_weight_min": -32.58232498168945,
      "activations/layer12_attention_weight_max": 24.759315490722656,
      "activations/layer12_attention_weight_min": -24.39005470275879,
      "activations/layer13_attention_weight_max": 38.591644287109375,
      "activations/layer13_attention_weight_min": -33.29863739013672,
      "activations/layer14_attention_weight_max": 43.33580780029297,
      "activations/layer14_attention_weight_min": -36.5423469543457,
      "activations/layer15_attention_weight_max": 36.749549865722656,
      "activations/layer15_attention_weight_min": -30.768688201904297,
      "activations/layer16_attention_weight_max": 30.684919357299805,
      "activations/layer16_attention_weight_min": -27.060256958007812,
      "activations/layer17_attention_weight_max": 33.44038772583008,
      "activations/layer17_attention_weight_min": -24.597564697265625,
      "activations/layer18_attention_weight_max": 31.10616683959961,
      "activations/layer18_attention_weight_min": -21.919565200805664,
      "activations/layer19_attention_weight_max": 37.2799186706543,
      "activations/layer19_attention_weight_min": -30.390918731689453,
      "activations/layer1_attention_weight_max": 16.8018856048584,
      "activations/layer1_attention_weight_min": -14.128168106079102,
      "activations/layer20_attention_weight_max": 28.50545883178711,
      "activations/layer20_attention_weight_min": -24.19849395751953,
      "activations/layer21_attention_weight_max": 30.139768600463867,
      "activations/layer21_attention_weight_min": -24.76858901977539,
      "activations/layer22_attention_weight_max": 42.27652359008789,
      "activations/layer22_attention_weight_min": -29.70604133605957,
      "activations/layer23_attention_weight_max": 34.5166015625,
      "activations/layer23_attention_weight_min": -23.58127212524414,
      "activations/layer2_attention_weight_max": 30.992002487182617,
      "activations/layer2_attention_weight_min": -30.995214462280273,
      "activations/layer3_attention_weight_max": 95.68614196777344,
      "activations/layer3_attention_weight_min": -93.914794921875,
      "activations/layer4_attention_weight_max": 111.82335662841797,
      "activations/layer4_attention_weight_min": -113.30049133300781,
      "activations/layer5_attention_weight_max": 54.669273376464844,
      "activations/layer5_attention_weight_min": -62.14665222167969,
      "activations/layer6_attention_weight_max": 45.55912780761719,
      "activations/layer6_attention_weight_min": -48.610023498535156,
      "activations/layer7_attention_weight_max": 88.08512115478516,
      "activations/layer7_attention_weight_min": -84.66346740722656,
      "activations/layer8_attention_weight_max": 40.391475677490234,
      "activations/layer8_attention_weight_min": -40.103965759277344,
      "activations/layer9_attention_weight_max": 31.97478675842285,
      "activations/layer9_attention_weight_min": -32.179500579833984,
      "epoch": 11.76,
      "learning_rate": 7.489469696969697e-05,
      "loss": 2.7716,
      "step": 202450
    },
    {
      "activations/layer0_attention_weight_max": 15.875368118286133,
      "activations/layer0_attention_weight_min": -12.414392471313477,
      "activations/layer10_attention_weight_max": 29.887914657592773,
      "activations/layer10_attention_weight_min": -29.404775619506836,
      "activations/layer11_attention_weight_max": 29.52980613708496,
      "activations/layer11_attention_weight_min": -27.94854736328125,
      "activations/layer12_attention_weight_max": 24.159610748291016,
      "activations/layer12_attention_weight_min": -24.953283309936523,
      "activations/layer13_attention_weight_max": 36.696815490722656,
      "activations/layer13_attention_weight_min": -31.577869415283203,
      "activations/layer14_attention_weight_max": 39.36255645751953,
      "activations/layer14_attention_weight_min": -37.51896667480469,
      "activations/layer15_attention_weight_max": 32.870262145996094,
      "activations/layer15_attention_weight_min": -31.28717803955078,
      "activations/layer16_attention_weight_max": 30.09522247314453,
      "activations/layer16_attention_weight_min": -28.130842208862305,
      "activations/layer17_attention_weight_max": 28.39510154724121,
      "activations/layer17_attention_weight_min": -25.66088104248047,
      "activations/layer18_attention_weight_max": 31.533084869384766,
      "activations/layer18_attention_weight_min": -24.406047821044922,
      "activations/layer19_attention_weight_max": 32.197540283203125,
      "activations/layer19_attention_weight_min": -33.63164520263672,
      "activations/layer1_attention_weight_max": 16.097671508789062,
      "activations/layer1_attention_weight_min": -13.706754684448242,
      "activations/layer20_attention_weight_max": 26.819244384765625,
      "activations/layer20_attention_weight_min": -26.352426528930664,
      "activations/layer21_attention_weight_max": 25.575389862060547,
      "activations/layer21_attention_weight_min": -24.914506912231445,
      "activations/layer22_attention_weight_max": 39.201904296875,
      "activations/layer22_attention_weight_min": -31.68603515625,
      "activations/layer23_attention_weight_max": 32.09990692138672,
      "activations/layer23_attention_weight_min": -26.92894744873047,
      "activations/layer2_attention_weight_max": 31.7698974609375,
      "activations/layer2_attention_weight_min": -31.521039962768555,
      "activations/layer3_attention_weight_max": 92.89215850830078,
      "activations/layer3_attention_weight_min": -99.58214569091797,
      "activations/layer4_attention_weight_max": 112.47551727294922,
      "activations/layer4_attention_weight_min": -115.78841400146484,
      "activations/layer5_attention_weight_max": 50.27070236206055,
      "activations/layer5_attention_weight_min": -58.519508361816406,
      "activations/layer6_attention_weight_max": 43.601497650146484,
      "activations/layer6_attention_weight_min": -47.079402923583984,
      "activations/layer7_attention_weight_max": 87.1969985961914,
      "activations/layer7_attention_weight_min": -90.0873031616211,
      "activations/layer8_attention_weight_max": 38.011287689208984,
      "activations/layer8_attention_weight_min": -38.825897216796875,
      "activations/layer9_attention_weight_max": 32.06641387939453,
      "activations/layer9_attention_weight_min": -30.9515380859375,
      "epoch": 11.77,
      "learning_rate": 7.487575757575757e-05,
      "loss": 2.7805,
      "step": 202500
    },
    {
      "activations/layer0_attention_weight_max": 14.94872760772705,
      "activations/layer0_attention_weight_min": -12.221470832824707,
      "activations/layer10_attention_weight_max": 33.06958770751953,
      "activations/layer10_attention_weight_min": -31.16146469116211,
      "activations/layer11_attention_weight_max": 32.702667236328125,
      "activations/layer11_attention_weight_min": -31.325048446655273,
      "activations/layer12_attention_weight_max": 25.586605072021484,
      "activations/layer12_attention_weight_min": -25.638587951660156,
      "activations/layer13_attention_weight_max": 39.580902099609375,
      "activations/layer13_attention_weight_min": -34.91447830200195,
      "activations/layer14_attention_weight_max": 45.28379440307617,
      "activations/layer14_attention_weight_min": -38.008724212646484,
      "activations/layer15_attention_weight_max": 37.72537612915039,
      "activations/layer15_attention_weight_min": -32.335758209228516,
      "activations/layer16_attention_weight_max": 32.68819046020508,
      "activations/layer16_attention_weight_min": -28.055709838867188,
      "activations/layer17_attention_weight_max": 31.57190704345703,
      "activations/layer17_attention_weight_min": -26.15322494506836,
      "activations/layer18_attention_weight_max": 33.02052307128906,
      "activations/layer18_attention_weight_min": -23.145586013793945,
      "activations/layer19_attention_weight_max": 33.37424850463867,
      "activations/layer19_attention_weight_min": -31.321990966796875,
      "activations/layer1_attention_weight_max": 16.26115608215332,
      "activations/layer1_attention_weight_min": -13.635523796081543,
      "activations/layer20_attention_weight_max": 29.749908447265625,
      "activations/layer20_attention_weight_min": -25.053359985351562,
      "activations/layer21_attention_weight_max": 30.009836196899414,
      "activations/layer21_attention_weight_min": -23.155471801757812,
      "activations/layer22_attention_weight_max": 42.84717559814453,
      "activations/layer22_attention_weight_min": -28.63787269592285,
      "activations/layer23_attention_weight_max": 31.946279525756836,
      "activations/layer23_attention_weight_min": -23.894760131835938,
      "activations/layer2_attention_weight_max": 31.88715934753418,
      "activations/layer2_attention_weight_min": -30.83722496032715,
      "activations/layer3_attention_weight_max": 92.67039489746094,
      "activations/layer3_attention_weight_min": -96.87543487548828,
      "activations/layer4_attention_weight_max": 110.13237762451172,
      "activations/layer4_attention_weight_min": -106.85493469238281,
      "activations/layer5_attention_weight_max": 51.997066497802734,
      "activations/layer5_attention_weight_min": -58.92226791381836,
      "activations/layer6_attention_weight_max": 43.53168487548828,
      "activations/layer6_attention_weight_min": -48.87714767456055,
      "activations/layer7_attention_weight_max": 86.1136703491211,
      "activations/layer7_attention_weight_min": -86.77703857421875,
      "activations/layer8_attention_weight_max": 41.04707717895508,
      "activations/layer8_attention_weight_min": -40.0197639465332,
      "activations/layer9_attention_weight_max": 32.66836166381836,
      "activations/layer9_attention_weight_min": -32.1395263671875,
      "epoch": 11.77,
      "learning_rate": 7.485681818181818e-05,
      "loss": 2.7756,
      "step": 202550
    },
    {
      "activations/layer0_attention_weight_max": 15.540038108825684,
      "activations/layer0_attention_weight_min": -11.54836368560791,
      "activations/layer10_attention_weight_max": 33.24318313598633,
      "activations/layer10_attention_weight_min": -32.98994827270508,
      "activations/layer11_attention_weight_max": 32.67306137084961,
      "activations/layer11_attention_weight_min": -33.487552642822266,
      "activations/layer12_attention_weight_max": 27.319421768188477,
      "activations/layer12_attention_weight_min": -24.877695083618164,
      "activations/layer13_attention_weight_max": 38.2482795715332,
      "activations/layer13_attention_weight_min": -33.95448303222656,
      "activations/layer14_attention_weight_max": 48.101959228515625,
      "activations/layer14_attention_weight_min": -39.24109649658203,
      "activations/layer15_attention_weight_max": 36.918827056884766,
      "activations/layer15_attention_weight_min": -33.58683776855469,
      "activations/layer16_attention_weight_max": 33.063331604003906,
      "activations/layer16_attention_weight_min": -28.896934509277344,
      "activations/layer17_attention_weight_max": 32.676204681396484,
      "activations/layer17_attention_weight_min": -25.883432388305664,
      "activations/layer18_attention_weight_max": 34.209259033203125,
      "activations/layer18_attention_weight_min": -24.017749786376953,
      "activations/layer19_attention_weight_max": 38.44718551635742,
      "activations/layer19_attention_weight_min": -30.7940616607666,
      "activations/layer1_attention_weight_max": 16.07213020324707,
      "activations/layer1_attention_weight_min": -14.877610206604004,
      "activations/layer20_attention_weight_max": 29.898780822753906,
      "activations/layer20_attention_weight_min": -25.741016387939453,
      "activations/layer21_attention_weight_max": 32.02263259887695,
      "activations/layer21_attention_weight_min": -24.443349838256836,
      "activations/layer22_attention_weight_max": 47.56654357910156,
      "activations/layer22_attention_weight_min": -28.928205490112305,
      "activations/layer23_attention_weight_max": 34.66822814941406,
      "activations/layer23_attention_weight_min": -25.753223419189453,
      "activations/layer2_attention_weight_max": 32.870277404785156,
      "activations/layer2_attention_weight_min": -32.940704345703125,
      "activations/layer3_attention_weight_max": 94.67530059814453,
      "activations/layer3_attention_weight_min": -96.31382751464844,
      "activations/layer4_attention_weight_max": 107.6448745727539,
      "activations/layer4_attention_weight_min": -112.28356170654297,
      "activations/layer5_attention_weight_max": 53.293785095214844,
      "activations/layer5_attention_weight_min": -60.85856628417969,
      "activations/layer6_attention_weight_max": 44.58220672607422,
      "activations/layer6_attention_weight_min": -47.93583679199219,
      "activations/layer7_attention_weight_max": 88.58081817626953,
      "activations/layer7_attention_weight_min": -88.23700714111328,
      "activations/layer8_attention_weight_max": 39.34614562988281,
      "activations/layer8_attention_weight_min": -40.02910232543945,
      "activations/layer9_attention_weight_max": 34.0838737487793,
      "activations/layer9_attention_weight_min": -34.08956527709961,
      "epoch": 11.77,
      "learning_rate": 7.483787878787879e-05,
      "loss": 2.7808,
      "step": 202600
    },
    {
      "activations/layer0_attention_weight_max": 16.633544921875,
      "activations/layer0_attention_weight_min": -13.127161979675293,
      "activations/layer10_attention_weight_max": 30.871915817260742,
      "activations/layer10_attention_weight_min": -31.031417846679688,
      "activations/layer11_attention_weight_max": 33.071876525878906,
      "activations/layer11_attention_weight_min": -31.517614364624023,
      "activations/layer12_attention_weight_max": 28.66345977783203,
      "activations/layer12_attention_weight_min": -24.63682746887207,
      "activations/layer13_attention_weight_max": 42.8065299987793,
      "activations/layer13_attention_weight_min": -33.666168212890625,
      "activations/layer14_attention_weight_max": 48.18242263793945,
      "activations/layer14_attention_weight_min": -36.596343994140625,
      "activations/layer15_attention_weight_max": 39.55198287963867,
      "activations/layer15_attention_weight_min": -31.983980178833008,
      "activations/layer16_attention_weight_max": 32.7812385559082,
      "activations/layer16_attention_weight_min": -27.55940055847168,
      "activations/layer17_attention_weight_max": 38.04660415649414,
      "activations/layer17_attention_weight_min": -25.940013885498047,
      "activations/layer18_attention_weight_max": 36.50102996826172,
      "activations/layer18_attention_weight_min": -22.755619049072266,
      "activations/layer19_attention_weight_max": 38.23299026489258,
      "activations/layer19_attention_weight_min": -30.79141616821289,
      "activations/layer1_attention_weight_max": 15.733419418334961,
      "activations/layer1_attention_weight_min": -14.185372352600098,
      "activations/layer20_attention_weight_max": 34.46411895751953,
      "activations/layer20_attention_weight_min": -25.321622848510742,
      "activations/layer21_attention_weight_max": 35.456520080566406,
      "activations/layer21_attention_weight_min": -23.25657081604004,
      "activations/layer22_attention_weight_max": 55.10108184814453,
      "activations/layer22_attention_weight_min": -29.61020851135254,
      "activations/layer23_attention_weight_max": 37.759490966796875,
      "activations/layer23_attention_weight_min": -24.56671142578125,
      "activations/layer2_attention_weight_max": 31.528255462646484,
      "activations/layer2_attention_weight_min": -30.99257469177246,
      "activations/layer3_attention_weight_max": 92.35452270507812,
      "activations/layer3_attention_weight_min": -94.79825592041016,
      "activations/layer4_attention_weight_max": 110.51537322998047,
      "activations/layer4_attention_weight_min": -112.11128234863281,
      "activations/layer5_attention_weight_max": 50.73542404174805,
      "activations/layer5_attention_weight_min": -65.26402282714844,
      "activations/layer6_attention_weight_max": 43.3121223449707,
      "activations/layer6_attention_weight_min": -47.522422790527344,
      "activations/layer7_attention_weight_max": 89.48794555664062,
      "activations/layer7_attention_weight_min": -91.19652557373047,
      "activations/layer8_attention_weight_max": 38.18821334838867,
      "activations/layer8_attention_weight_min": -40.56601333618164,
      "activations/layer9_attention_weight_max": 32.74812698364258,
      "activations/layer9_attention_weight_min": -32.520809173583984,
      "epoch": 11.78,
      "learning_rate": 7.481893939393939e-05,
      "loss": 2.7625,
      "step": 202650
    },
    {
      "activations/layer0_attention_weight_max": 15.924468040466309,
      "activations/layer0_attention_weight_min": -13.068696022033691,
      "activations/layer10_attention_weight_max": 35.98308181762695,
      "activations/layer10_attention_weight_min": -34.43376159667969,
      "activations/layer11_attention_weight_max": 36.5805549621582,
      "activations/layer11_attention_weight_min": -34.83772659301758,
      "activations/layer12_attention_weight_max": 40.22715377807617,
      "activations/layer12_attention_weight_min": -25.440874099731445,
      "activations/layer13_attention_weight_max": 41.551353454589844,
      "activations/layer13_attention_weight_min": -36.785396575927734,
      "activations/layer14_attention_weight_max": 46.14065933227539,
      "activations/layer14_attention_weight_min": -38.80570602416992,
      "activations/layer15_attention_weight_max": 39.46688461303711,
      "activations/layer15_attention_weight_min": -33.619510650634766,
      "activations/layer16_attention_weight_max": 32.8560676574707,
      "activations/layer16_attention_weight_min": -29.791475296020508,
      "activations/layer17_attention_weight_max": 31.32394027709961,
      "activations/layer17_attention_weight_min": -25.776784896850586,
      "activations/layer18_attention_weight_max": 30.842241287231445,
      "activations/layer18_attention_weight_min": -22.279237747192383,
      "activations/layer19_attention_weight_max": 35.041847229003906,
      "activations/layer19_attention_weight_min": -31.453922271728516,
      "activations/layer1_attention_weight_max": 15.735925674438477,
      "activations/layer1_attention_weight_min": -13.820223808288574,
      "activations/layer20_attention_weight_max": 32.64484405517578,
      "activations/layer20_attention_weight_min": -25.251935958862305,
      "activations/layer21_attention_weight_max": 30.661022186279297,
      "activations/layer21_attention_weight_min": -24.158315658569336,
      "activations/layer22_attention_weight_max": 48.006187438964844,
      "activations/layer22_attention_weight_min": -30.55983543395996,
      "activations/layer23_attention_weight_max": 35.986244201660156,
      "activations/layer23_attention_weight_min": -25.180282592773438,
      "activations/layer2_attention_weight_max": 33.80149459838867,
      "activations/layer2_attention_weight_min": -33.62035369873047,
      "activations/layer3_attention_weight_max": 97.78410339355469,
      "activations/layer3_attention_weight_min": -101.95327758789062,
      "activations/layer4_attention_weight_max": 113.54642486572266,
      "activations/layer4_attention_weight_min": -114.19013977050781,
      "activations/layer5_attention_weight_max": 56.63793182373047,
      "activations/layer5_attention_weight_min": -60.41251754760742,
      "activations/layer6_attention_weight_max": 47.46626281738281,
      "activations/layer6_attention_weight_min": -52.07896041870117,
      "activations/layer7_attention_weight_max": 100.49022674560547,
      "activations/layer7_attention_weight_min": -92.48666381835938,
      "activations/layer8_attention_weight_max": 44.39339065551758,
      "activations/layer8_attention_weight_min": -45.06155776977539,
      "activations/layer9_attention_weight_max": 38.293758392333984,
      "activations/layer9_attention_weight_min": -35.4133186340332,
      "epoch": 11.78,
      "learning_rate": 7.479999999999999e-05,
      "loss": 2.7838,
      "step": 202700
    },
    {
      "activations/layer0_attention_weight_max": 15.549460411071777,
      "activations/layer0_attention_weight_min": -11.702095985412598,
      "activations/layer10_attention_weight_max": 34.52510070800781,
      "activations/layer10_attention_weight_min": -34.96419143676758,
      "activations/layer11_attention_weight_max": 35.99806213378906,
      "activations/layer11_attention_weight_min": -33.913597106933594,
      "activations/layer12_attention_weight_max": 27.3773136138916,
      "activations/layer12_attention_weight_min": -27.55377769470215,
      "activations/layer13_attention_weight_max": 41.72821807861328,
      "activations/layer13_attention_weight_min": -37.49586868286133,
      "activations/layer14_attention_weight_max": 47.99525451660156,
      "activations/layer14_attention_weight_min": -39.953086853027344,
      "activations/layer15_attention_weight_max": 39.31079864501953,
      "activations/layer15_attention_weight_min": -32.578582763671875,
      "activations/layer16_attention_weight_max": 32.53702163696289,
      "activations/layer16_attention_weight_min": -27.57921600341797,
      "activations/layer17_attention_weight_max": 31.94436264038086,
      "activations/layer17_attention_weight_min": -25.770639419555664,
      "activations/layer18_attention_weight_max": 30.004365921020508,
      "activations/layer18_attention_weight_min": -23.23987579345703,
      "activations/layer19_attention_weight_max": 35.2717399597168,
      "activations/layer19_attention_weight_min": -32.76528549194336,
      "activations/layer1_attention_weight_max": 16.36603546142578,
      "activations/layer1_attention_weight_min": -15.530550003051758,
      "activations/layer20_attention_weight_max": 29.168006896972656,
      "activations/layer20_attention_weight_min": -23.71343231201172,
      "activations/layer21_attention_weight_max": 26.08133316040039,
      "activations/layer21_attention_weight_min": -22.41668128967285,
      "activations/layer22_attention_weight_max": 45.06989288330078,
      "activations/layer22_attention_weight_min": -28.931583404541016,
      "activations/layer23_attention_weight_max": 34.78243637084961,
      "activations/layer23_attention_weight_min": -25.679332733154297,
      "activations/layer2_attention_weight_max": 33.59960174560547,
      "activations/layer2_attention_weight_min": -32.52846908569336,
      "activations/layer3_attention_weight_max": 99.26686096191406,
      "activations/layer3_attention_weight_min": -99.24442291259766,
      "activations/layer4_attention_weight_max": 113.37760162353516,
      "activations/layer4_attention_weight_min": -114.89302825927734,
      "activations/layer5_attention_weight_max": 51.564117431640625,
      "activations/layer5_attention_weight_min": -57.86140060424805,
      "activations/layer6_attention_weight_max": 44.11913299560547,
      "activations/layer6_attention_weight_min": -47.64686584472656,
      "activations/layer7_attention_weight_max": 93.19218444824219,
      "activations/layer7_attention_weight_min": -94.57013702392578,
      "activations/layer8_attention_weight_max": 41.58624267578125,
      "activations/layer8_attention_weight_min": -42.95521545410156,
      "activations/layer9_attention_weight_max": 33.57700729370117,
      "activations/layer9_attention_weight_min": -34.375736236572266,
      "epoch": 11.78,
      "learning_rate": 7.47810606060606e-05,
      "loss": 2.7825,
      "step": 202750
    },
    {
      "activations/layer0_attention_weight_max": 15.461686134338379,
      "activations/layer0_attention_weight_min": -12.699006080627441,
      "activations/layer10_attention_weight_max": 30.70035171508789,
      "activations/layer10_attention_weight_min": -31.182035446166992,
      "activations/layer11_attention_weight_max": 31.26307487487793,
      "activations/layer11_attention_weight_min": -30.962440490722656,
      "activations/layer12_attention_weight_max": 23.084537506103516,
      "activations/layer12_attention_weight_min": -25.569108963012695,
      "activations/layer13_attention_weight_max": 35.33600997924805,
      "activations/layer13_attention_weight_min": -36.110382080078125,
      "activations/layer14_attention_weight_max": 37.74229431152344,
      "activations/layer14_attention_weight_min": -37.090675354003906,
      "activations/layer15_attention_weight_max": 34.357906341552734,
      "activations/layer15_attention_weight_min": -32.2581901550293,
      "activations/layer16_attention_weight_max": 28.240760803222656,
      "activations/layer16_attention_weight_min": -26.59868621826172,
      "activations/layer17_attention_weight_max": 29.929895401000977,
      "activations/layer17_attention_weight_min": -25.803632736206055,
      "activations/layer18_attention_weight_max": 31.689071655273438,
      "activations/layer18_attention_weight_min": -23.504690170288086,
      "activations/layer19_attention_weight_max": 32.60429000854492,
      "activations/layer19_attention_weight_min": -31.0615177154541,
      "activations/layer1_attention_weight_max": 16.980104446411133,
      "activations/layer1_attention_weight_min": -15.268264770507812,
      "activations/layer20_attention_weight_max": 28.456186294555664,
      "activations/layer20_attention_weight_min": -24.993135452270508,
      "activations/layer21_attention_weight_max": 30.32135581970215,
      "activations/layer21_attention_weight_min": -24.663291931152344,
      "activations/layer22_attention_weight_max": 41.33084487915039,
      "activations/layer22_attention_weight_min": -30.279130935668945,
      "activations/layer23_attention_weight_max": 35.92041778564453,
      "activations/layer23_attention_weight_min": -26.062252044677734,
      "activations/layer2_attention_weight_max": 32.43919372558594,
      "activations/layer2_attention_weight_min": -30.813434600830078,
      "activations/layer3_attention_weight_max": 92.82687377929688,
      "activations/layer3_attention_weight_min": -91.81199645996094,
      "activations/layer4_attention_weight_max": 109.24617767333984,
      "activations/layer4_attention_weight_min": -106.73091125488281,
      "activations/layer5_attention_weight_max": 50.05978012084961,
      "activations/layer5_attention_weight_min": -59.14127731323242,
      "activations/layer6_attention_weight_max": 43.356834411621094,
      "activations/layer6_attention_weight_min": -44.7775993347168,
      "activations/layer7_attention_weight_max": 86.93898010253906,
      "activations/layer7_attention_weight_min": -82.03792572021484,
      "activations/layer8_attention_weight_max": 39.23495864868164,
      "activations/layer8_attention_weight_min": -39.69984817504883,
      "activations/layer9_attention_weight_max": 32.487457275390625,
      "activations/layer9_attention_weight_min": -31.658594131469727,
      "epoch": 11.78,
      "learning_rate": 7.47621212121212e-05,
      "loss": 2.7829,
      "step": 202800
    },
    {
      "activations/layer0_attention_weight_max": 16.39499855041504,
      "activations/layer0_attention_weight_min": -11.832024574279785,
      "activations/layer10_attention_weight_max": 32.37250900268555,
      "activations/layer10_attention_weight_min": -31.803319931030273,
      "activations/layer11_attention_weight_max": 32.008548736572266,
      "activations/layer11_attention_weight_min": -31.19088363647461,
      "activations/layer12_attention_weight_max": 23.619415283203125,
      "activations/layer12_attention_weight_min": -24.79163360595703,
      "activations/layer13_attention_weight_max": 35.78282165527344,
      "activations/layer13_attention_weight_min": -33.056270599365234,
      "activations/layer14_attention_weight_max": 40.913795471191406,
      "activations/layer14_attention_weight_min": -36.57548904418945,
      "activations/layer15_attention_weight_max": 35.073760986328125,
      "activations/layer15_attention_weight_min": -31.955772399902344,
      "activations/layer16_attention_weight_max": 29.724855422973633,
      "activations/layer16_attention_weight_min": -27.555253982543945,
      "activations/layer17_attention_weight_max": 31.352502822875977,
      "activations/layer17_attention_weight_min": -26.722742080688477,
      "activations/layer18_attention_weight_max": 28.304391860961914,
      "activations/layer18_attention_weight_min": -23.44881820678711,
      "activations/layer19_attention_weight_max": 31.306604385375977,
      "activations/layer19_attention_weight_min": -33.52467727661133,
      "activations/layer1_attention_weight_max": 16.18302345275879,
      "activations/layer1_attention_weight_min": -14.33355712890625,
      "activations/layer20_attention_weight_max": 28.640220642089844,
      "activations/layer20_attention_weight_min": -27.291357040405273,
      "activations/layer21_attention_weight_max": 27.049314498901367,
      "activations/layer21_attention_weight_min": -25.92496681213379,
      "activations/layer22_attention_weight_max": 42.81595993041992,
      "activations/layer22_attention_weight_min": -33.1483039855957,
      "activations/layer23_attention_weight_max": 31.801658630371094,
      "activations/layer23_attention_weight_min": -25.28618621826172,
      "activations/layer2_attention_weight_max": 32.871185302734375,
      "activations/layer2_attention_weight_min": -31.7327938079834,
      "activations/layer3_attention_weight_max": 98.59820556640625,
      "activations/layer3_attention_weight_min": -99.25906372070312,
      "activations/layer4_attention_weight_max": 117.86348724365234,
      "activations/layer4_attention_weight_min": -117.8070297241211,
      "activations/layer5_attention_weight_max": 53.159908294677734,
      "activations/layer5_attention_weight_min": -63.692893981933594,
      "activations/layer6_attention_weight_max": 45.436824798583984,
      "activations/layer6_attention_weight_min": -50.282405853271484,
      "activations/layer7_attention_weight_max": 89.1882095336914,
      "activations/layer7_attention_weight_min": -94.14564514160156,
      "activations/layer8_attention_weight_max": 40.55085372924805,
      "activations/layer8_attention_weight_min": -42.13205337524414,
      "activations/layer9_attention_weight_max": 33.74317932128906,
      "activations/layer9_attention_weight_min": -33.51819610595703,
      "epoch": 11.79,
      "learning_rate": 7.47431818181818e-05,
      "loss": 2.7733,
      "step": 202850
    },
    {
      "activations/layer0_attention_weight_max": 16.446378707885742,
      "activations/layer0_attention_weight_min": -12.629573822021484,
      "activations/layer10_attention_weight_max": 31.13920021057129,
      "activations/layer10_attention_weight_min": -30.610519409179688,
      "activations/layer11_attention_weight_max": 29.767528533935547,
      "activations/layer11_attention_weight_min": -30.411479949951172,
      "activations/layer12_attention_weight_max": 24.992618560791016,
      "activations/layer12_attention_weight_min": -26.363866806030273,
      "activations/layer13_attention_weight_max": 39.49346923828125,
      "activations/layer13_attention_weight_min": -34.251033782958984,
      "activations/layer14_attention_weight_max": 45.02574157714844,
      "activations/layer14_attention_weight_min": -36.15107727050781,
      "activations/layer15_attention_weight_max": 37.372657775878906,
      "activations/layer15_attention_weight_min": -31.53715705871582,
      "activations/layer16_attention_weight_max": 34.51511764526367,
      "activations/layer16_attention_weight_min": -27.959041595458984,
      "activations/layer17_attention_weight_max": 35.67574691772461,
      "activations/layer17_attention_weight_min": -24.988107681274414,
      "activations/layer18_attention_weight_max": 32.843711853027344,
      "activations/layer18_attention_weight_min": -23.786649703979492,
      "activations/layer19_attention_weight_max": 39.408409118652344,
      "activations/layer19_attention_weight_min": -31.691497802734375,
      "activations/layer1_attention_weight_max": 16.23297882080078,
      "activations/layer1_attention_weight_min": -14.13004207611084,
      "activations/layer20_attention_weight_max": 33.304500579833984,
      "activations/layer20_attention_weight_min": -25.780967712402344,
      "activations/layer21_attention_weight_max": 30.9359188079834,
      "activations/layer21_attention_weight_min": -23.5123291015625,
      "activations/layer22_attention_weight_max": 51.75821304321289,
      "activations/layer22_attention_weight_min": -31.63044548034668,
      "activations/layer23_attention_weight_max": 36.11518859863281,
      "activations/layer23_attention_weight_min": -29.762771606445312,
      "activations/layer2_attention_weight_max": 31.202795028686523,
      "activations/layer2_attention_weight_min": -31.871952056884766,
      "activations/layer3_attention_weight_max": 93.39424896240234,
      "activations/layer3_attention_weight_min": -95.92401123046875,
      "activations/layer4_attention_weight_max": 109.19622802734375,
      "activations/layer4_attention_weight_min": -115.0499267578125,
      "activations/layer5_attention_weight_max": 50.129066467285156,
      "activations/layer5_attention_weight_min": -60.816551208496094,
      "activations/layer6_attention_weight_max": 41.57151412963867,
      "activations/layer6_attention_weight_min": -48.29926300048828,
      "activations/layer7_attention_weight_max": 84.94436645507812,
      "activations/layer7_attention_weight_min": -86.31130981445312,
      "activations/layer8_attention_weight_max": 36.74714279174805,
      "activations/layer8_attention_weight_min": -38.691036224365234,
      "activations/layer9_attention_weight_max": 30.55739402770996,
      "activations/layer9_attention_weight_min": -31.597095489501953,
      "epoch": 11.79,
      "learning_rate": 7.472424242424241e-05,
      "loss": 2.7799,
      "step": 202900
    },
    {
      "activations/layer0_attention_weight_max": 15.94925594329834,
      "activations/layer0_attention_weight_min": -11.723036766052246,
      "activations/layer10_attention_weight_max": 33.65414810180664,
      "activations/layer10_attention_weight_min": -32.348636627197266,
      "activations/layer11_attention_weight_max": 35.139015197753906,
      "activations/layer11_attention_weight_min": -33.14192581176758,
      "activations/layer12_attention_weight_max": 26.63087272644043,
      "activations/layer12_attention_weight_min": -28.100135803222656,
      "activations/layer13_attention_weight_max": 39.40233612060547,
      "activations/layer13_attention_weight_min": -33.36909103393555,
      "activations/layer14_attention_weight_max": 50.3978385925293,
      "activations/layer14_attention_weight_min": -37.247169494628906,
      "activations/layer15_attention_weight_max": 38.656471252441406,
      "activations/layer15_attention_weight_min": -32.4359130859375,
      "activations/layer16_attention_weight_max": 33.74409484863281,
      "activations/layer16_attention_weight_min": -28.768157958984375,
      "activations/layer17_attention_weight_max": 32.85576629638672,
      "activations/layer17_attention_weight_min": -26.215917587280273,
      "activations/layer18_attention_weight_max": 32.47176742553711,
      "activations/layer18_attention_weight_min": -23.795223236083984,
      "activations/layer19_attention_weight_max": 35.6721305847168,
      "activations/layer19_attention_weight_min": -33.38446044921875,
      "activations/layer1_attention_weight_max": 16.4301815032959,
      "activations/layer1_attention_weight_min": -15.147188186645508,
      "activations/layer20_attention_weight_max": 32.41774368286133,
      "activations/layer20_attention_weight_min": -27.37871551513672,
      "activations/layer21_attention_weight_max": 31.502670288085938,
      "activations/layer21_attention_weight_min": -24.427738189697266,
      "activations/layer22_attention_weight_max": 43.78373336791992,
      "activations/layer22_attention_weight_min": -32.53104019165039,
      "activations/layer23_attention_weight_max": 34.31309509277344,
      "activations/layer23_attention_weight_min": -25.24659538269043,
      "activations/layer2_attention_weight_max": 31.67446517944336,
      "activations/layer2_attention_weight_min": -31.162212371826172,
      "activations/layer3_attention_weight_max": 96.1123275756836,
      "activations/layer3_attention_weight_min": -95.4755630493164,
      "activations/layer4_attention_weight_max": 120.41548919677734,
      "activations/layer4_attention_weight_min": -114.7825927734375,
      "activations/layer5_attention_weight_max": 53.424720764160156,
      "activations/layer5_attention_weight_min": -66.9222412109375,
      "activations/layer6_attention_weight_max": 46.319358825683594,
      "activations/layer6_attention_weight_min": -45.89014434814453,
      "activations/layer7_attention_weight_max": 96.52310943603516,
      "activations/layer7_attention_weight_min": -91.58805847167969,
      "activations/layer8_attention_weight_max": 40.980186462402344,
      "activations/layer8_attention_weight_min": -41.74333953857422,
      "activations/layer9_attention_weight_max": 33.0233268737793,
      "activations/layer9_attention_weight_min": -33.18960952758789,
      "epoch": 11.79,
      "learning_rate": 7.470530303030302e-05,
      "loss": 2.7773,
      "step": 202950
    },
    {
      "activations/layer0_attention_weight_max": 16.142505645751953,
      "activations/layer0_attention_weight_min": -12.446674346923828,
      "activations/layer10_attention_weight_max": 31.539615631103516,
      "activations/layer10_attention_weight_min": -33.71322250366211,
      "activations/layer11_attention_weight_max": 32.719749450683594,
      "activations/layer11_attention_weight_min": -32.101402282714844,
      "activations/layer12_attention_weight_max": 25.918100357055664,
      "activations/layer12_attention_weight_min": -26.210742950439453,
      "activations/layer13_attention_weight_max": 38.0322151184082,
      "activations/layer13_attention_weight_min": -38.41445541381836,
      "activations/layer14_attention_weight_max": 47.34236526489258,
      "activations/layer14_attention_weight_min": -41.282711029052734,
      "activations/layer15_attention_weight_max": 36.14694595336914,
      "activations/layer15_attention_weight_min": -34.414119720458984,
      "activations/layer16_attention_weight_max": 34.98169708251953,
      "activations/layer16_attention_weight_min": -30.197463989257812,
      "activations/layer17_attention_weight_max": 33.007179260253906,
      "activations/layer17_attention_weight_min": -26.073694229125977,
      "activations/layer18_attention_weight_max": 31.33612632751465,
      "activations/layer18_attention_weight_min": -23.211563110351562,
      "activations/layer19_attention_weight_max": 39.56562423706055,
      "activations/layer19_attention_weight_min": -33.4466667175293,
      "activations/layer1_attention_weight_max": 16.34455680847168,
      "activations/layer1_attention_weight_min": -15.08949089050293,
      "activations/layer20_attention_weight_max": 34.56043243408203,
      "activations/layer20_attention_weight_min": -25.790443420410156,
      "activations/layer21_attention_weight_max": 30.960826873779297,
      "activations/layer21_attention_weight_min": -25.494192123413086,
      "activations/layer22_attention_weight_max": 51.447452545166016,
      "activations/layer22_attention_weight_min": -30.71076774597168,
      "activations/layer23_attention_weight_max": 42.03346252441406,
      "activations/layer23_attention_weight_min": -27.038663864135742,
      "activations/layer2_attention_weight_max": 33.9753532409668,
      "activations/layer2_attention_weight_min": -31.499488830566406,
      "activations/layer3_attention_weight_max": 97.09325408935547,
      "activations/layer3_attention_weight_min": -96.5113754272461,
      "activations/layer4_attention_weight_max": 112.67162322998047,
      "activations/layer4_attention_weight_min": -114.17835998535156,
      "activations/layer5_attention_weight_max": 50.55891036987305,
      "activations/layer5_attention_weight_min": -61.60009002685547,
      "activations/layer6_attention_weight_max": 42.8721809387207,
      "activations/layer6_attention_weight_min": -49.884681701660156,
      "activations/layer7_attention_weight_max": 87.32102966308594,
      "activations/layer7_attention_weight_min": -90.70277404785156,
      "activations/layer8_attention_weight_max": 38.94389343261719,
      "activations/layer8_attention_weight_min": -40.82185745239258,
      "activations/layer9_attention_weight_max": 32.783653259277344,
      "activations/layer9_attention_weight_min": -32.613407135009766,
      "epoch": 11.8,
      "learning_rate": 7.468636363636362e-05,
      "loss": 2.7615,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.586,
      "eval_samples_per_second": 500.119,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.586,
      "eval_openwebtext_samples_per_second": 500.119,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.0093,
      "eval_wikitext_samples_per_second": 226.948,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.641,
      "eval_lambada_samples_per_second": 505.031,
      "step": 203000
    },
    {
      "activations/layer0_attention_weight_max": 15.895079612731934,
      "activations/layer0_attention_weight_min": -11.619738578796387,
      "activations/layer10_attention_weight_max": 31.7965087890625,
      "activations/layer10_attention_weight_min": -33.12735366821289,
      "activations/layer11_attention_weight_max": 33.232421875,
      "activations/layer11_attention_weight_min": -36.20586013793945,
      "activations/layer12_attention_weight_max": 25.47710418701172,
      "activations/layer12_attention_weight_min": -23.973642349243164,
      "activations/layer13_attention_weight_max": 38.80262756347656,
      "activations/layer13_attention_weight_min": -33.2049560546875,
      "activations/layer14_attention_weight_max": 44.87378692626953,
      "activations/layer14_attention_weight_min": -39.69184112548828,
      "activations/layer15_attention_weight_max": 37.47280502319336,
      "activations/layer15_attention_weight_min": -33.82725524902344,
      "activations/layer16_attention_weight_max": 28.181371688842773,
      "activations/layer16_attention_weight_min": -28.163597106933594,
      "activations/layer17_attention_weight_max": 29.69852638244629,
      "activations/layer17_attention_weight_min": -26.49534034729004,
      "activations/layer18_attention_weight_max": 29.18963050842285,
      "activations/layer18_attention_weight_min": -24.526607513427734,
      "activations/layer19_attention_weight_max": 32.24850082397461,
      "activations/layer19_attention_weight_min": -31.819644927978516,
      "activations/layer1_attention_weight_max": 16.912057876586914,
      "activations/layer1_attention_weight_min": -14.974344253540039,
      "activations/layer20_attention_weight_max": 26.8155460357666,
      "activations/layer20_attention_weight_min": -25.672395706176758,
      "activations/layer21_attention_weight_max": 27.544143676757812,
      "activations/layer21_attention_weight_min": -25.76972007751465,
      "activations/layer22_attention_weight_max": 41.00090026855469,
      "activations/layer22_attention_weight_min": -30.60578727722168,
      "activations/layer23_attention_weight_max": 32.018211364746094,
      "activations/layer23_attention_weight_min": -27.119810104370117,
      "activations/layer2_attention_weight_max": 32.006622314453125,
      "activations/layer2_attention_weight_min": -30.947431564331055,
      "activations/layer3_attention_weight_max": 92.1855697631836,
      "activations/layer3_attention_weight_min": -92.03160858154297,
      "activations/layer4_attention_weight_max": 110.92937469482422,
      "activations/layer4_attention_weight_min": -111.4610366821289,
      "activations/layer5_attention_weight_max": 51.16649627685547,
      "activations/layer5_attention_weight_min": -59.51483154296875,
      "activations/layer6_attention_weight_max": 43.99943161010742,
      "activations/layer6_attention_weight_min": -46.627620697021484,
      "activations/layer7_attention_weight_max": 88.68755340576172,
      "activations/layer7_attention_weight_min": -85.86792755126953,
      "activations/layer8_attention_weight_max": 37.259552001953125,
      "activations/layer8_attention_weight_min": -41.5585823059082,
      "activations/layer9_attention_weight_max": 31.541641235351562,
      "activations/layer9_attention_weight_min": -34.537532806396484,
      "epoch": 11.8,
      "learning_rate": 7.466742424242424e-05,
      "loss": 2.7723,
      "step": 203050
    },
    {
      "activations/layer0_attention_weight_max": 15.635743141174316,
      "activations/layer0_attention_weight_min": -12.208636283874512,
      "activations/layer10_attention_weight_max": 40.28754425048828,
      "activations/layer10_attention_weight_min": -37.57828903198242,
      "activations/layer11_attention_weight_max": 42.46727752685547,
      "activations/layer11_attention_weight_min": -39.9326286315918,
      "activations/layer12_attention_weight_max": 24.499454498291016,
      "activations/layer12_attention_weight_min": -27.362218856811523,
      "activations/layer13_attention_weight_max": 38.7227897644043,
      "activations/layer13_attention_weight_min": -34.46818542480469,
      "activations/layer14_attention_weight_max": 42.66762924194336,
      "activations/layer14_attention_weight_min": -36.39579391479492,
      "activations/layer15_attention_weight_max": 36.48211669921875,
      "activations/layer15_attention_weight_min": -30.25501823425293,
      "activations/layer16_attention_weight_max": 31.16089630126953,
      "activations/layer16_attention_weight_min": -26.157146453857422,
      "activations/layer17_attention_weight_max": 32.91636657714844,
      "activations/layer17_attention_weight_min": -26.537302017211914,
      "activations/layer18_attention_weight_max": 31.612661361694336,
      "activations/layer18_attention_weight_min": -22.781539916992188,
      "activations/layer19_attention_weight_max": 30.60428810119629,
      "activations/layer19_attention_weight_min": -30.74893569946289,
      "activations/layer1_attention_weight_max": 17.397991180419922,
      "activations/layer1_attention_weight_min": -15.05040168762207,
      "activations/layer20_attention_weight_max": 29.070615768432617,
      "activations/layer20_attention_weight_min": -25.53753662109375,
      "activations/layer21_attention_weight_max": 26.648887634277344,
      "activations/layer21_attention_weight_min": -23.852977752685547,
      "activations/layer22_attention_weight_max": 41.00735855102539,
      "activations/layer22_attention_weight_min": -31.84567642211914,
      "activations/layer23_attention_weight_max": 35.81861877441406,
      "activations/layer23_attention_weight_min": -25.453441619873047,
      "activations/layer2_attention_weight_max": 37.02841567993164,
      "activations/layer2_attention_weight_min": -37.03620910644531,
      "activations/layer3_attention_weight_max": 96.96016693115234,
      "activations/layer3_attention_weight_min": -99.13568115234375,
      "activations/layer4_attention_weight_max": 116.61285400390625,
      "activations/layer4_attention_weight_min": -113.53189849853516,
      "activations/layer5_attention_weight_max": 49.65888595581055,
      "activations/layer5_attention_weight_min": -66.65494537353516,
      "activations/layer6_attention_weight_max": 42.47451400756836,
      "activations/layer6_attention_weight_min": -45.67641067504883,
      "activations/layer7_attention_weight_max": 110.17568969726562,
      "activations/layer7_attention_weight_min": -91.27921295166016,
      "activations/layer8_attention_weight_max": 47.97842788696289,
      "activations/layer8_attention_weight_min": -40.46472930908203,
      "activations/layer9_attention_weight_max": 37.471527099609375,
      "activations/layer9_attention_weight_min": -36.67257308959961,
      "epoch": 11.8,
      "learning_rate": 7.464848484848484e-05,
      "loss": 2.7738,
      "step": 203100
    },
    {
      "activations/layer0_attention_weight_max": 17.40962791442871,
      "activations/layer0_attention_weight_min": -11.257268905639648,
      "activations/layer10_attention_weight_max": 31.114585876464844,
      "activations/layer10_attention_weight_min": -32.4693489074707,
      "activations/layer11_attention_weight_max": 31.48845672607422,
      "activations/layer11_attention_weight_min": -30.873493194580078,
      "activations/layer12_attention_weight_max": 24.501792907714844,
      "activations/layer12_attention_weight_min": -25.12242889404297,
      "activations/layer13_attention_weight_max": 39.15754699707031,
      "activations/layer13_attention_weight_min": -34.089725494384766,
      "activations/layer14_attention_weight_max": 43.28689956665039,
      "activations/layer14_attention_weight_min": -36.724483489990234,
      "activations/layer15_attention_weight_max": 36.53094482421875,
      "activations/layer15_attention_weight_min": -31.45742416381836,
      "activations/layer16_attention_weight_max": 36.47825622558594,
      "activations/layer16_attention_weight_min": -29.34431266784668,
      "activations/layer17_attention_weight_max": 33.05668640136719,
      "activations/layer17_attention_weight_min": -25.908918380737305,
      "activations/layer18_attention_weight_max": 32.69548034667969,
      "activations/layer18_attention_weight_min": -24.025039672851562,
      "activations/layer19_attention_weight_max": 37.180450439453125,
      "activations/layer19_attention_weight_min": -33.097835540771484,
      "activations/layer1_attention_weight_max": 16.306615829467773,
      "activations/layer1_attention_weight_min": -14.927236557006836,
      "activations/layer20_attention_weight_max": 30.224952697753906,
      "activations/layer20_attention_weight_min": -28.10197639465332,
      "activations/layer21_attention_weight_max": 29.878299713134766,
      "activations/layer21_attention_weight_min": -25.83598518371582,
      "activations/layer22_attention_weight_max": 45.3859748840332,
      "activations/layer22_attention_weight_min": -32.114768981933594,
      "activations/layer23_attention_weight_max": 39.1498908996582,
      "activations/layer23_attention_weight_min": -28.813156127929688,
      "activations/layer2_attention_weight_max": 33.606224060058594,
      "activations/layer2_attention_weight_min": -32.03058624267578,
      "activations/layer3_attention_weight_max": 98.17313385009766,
      "activations/layer3_attention_weight_min": -100.36512756347656,
      "activations/layer4_attention_weight_max": 114.78519439697266,
      "activations/layer4_attention_weight_min": -115.55464935302734,
      "activations/layer5_attention_weight_max": 51.11638641357422,
      "activations/layer5_attention_weight_min": -60.051666259765625,
      "activations/layer6_attention_weight_max": 42.68965530395508,
      "activations/layer6_attention_weight_min": -47.53660583496094,
      "activations/layer7_attention_weight_max": 88.2353286743164,
      "activations/layer7_attention_weight_min": -87.96788024902344,
      "activations/layer8_attention_weight_max": 40.40184783935547,
      "activations/layer8_attention_weight_min": -41.19380569458008,
      "activations/layer9_attention_weight_max": 32.69948196411133,
      "activations/layer9_attention_weight_min": -34.077369689941406,
      "epoch": 11.8,
      "learning_rate": 7.462954545454546e-05,
      "loss": 2.7654,
      "step": 203150
    },
    {
      "activations/layer0_attention_weight_max": 16.602148056030273,
      "activations/layer0_attention_weight_min": -12.494309425354004,
      "activations/layer10_attention_weight_max": 33.05991744995117,
      "activations/layer10_attention_weight_min": -33.23003005981445,
      "activations/layer11_attention_weight_max": 32.009735107421875,
      "activations/layer11_attention_weight_min": -33.28295135498047,
      "activations/layer12_attention_weight_max": 26.80392837524414,
      "activations/layer12_attention_weight_min": -26.26441764831543,
      "activations/layer13_attention_weight_max": 43.79411315917969,
      "activations/layer13_attention_weight_min": -36.58759307861328,
      "activations/layer14_attention_weight_max": 46.39924621582031,
      "activations/layer14_attention_weight_min": -38.66950225830078,
      "activations/layer15_attention_weight_max": 38.500709533691406,
      "activations/layer15_attention_weight_min": -34.187564849853516,
      "activations/layer16_attention_weight_max": 32.547611236572266,
      "activations/layer16_attention_weight_min": -28.32493019104004,
      "activations/layer17_attention_weight_max": 32.78178787231445,
      "activations/layer17_attention_weight_min": -27.268892288208008,
      "activations/layer18_attention_weight_max": 30.531461715698242,
      "activations/layer18_attention_weight_min": -26.27362060546875,
      "activations/layer19_attention_weight_max": 36.243438720703125,
      "activations/layer19_attention_weight_min": -31.98127555847168,
      "activations/layer1_attention_weight_max": 16.410921096801758,
      "activations/layer1_attention_weight_min": -15.181087493896484,
      "activations/layer20_attention_weight_max": 31.384082794189453,
      "activations/layer20_attention_weight_min": -26.168689727783203,
      "activations/layer21_attention_weight_max": 29.016887664794922,
      "activations/layer21_attention_weight_min": -26.51224136352539,
      "activations/layer22_attention_weight_max": 46.842350006103516,
      "activations/layer22_attention_weight_min": -31.953548431396484,
      "activations/layer23_attention_weight_max": 33.94083023071289,
      "activations/layer23_attention_weight_min": -26.326828002929688,
      "activations/layer2_attention_weight_max": 33.58482360839844,
      "activations/layer2_attention_weight_min": -31.886503219604492,
      "activations/layer3_attention_weight_max": 98.46499633789062,
      "activations/layer3_attention_weight_min": -98.1928482055664,
      "activations/layer4_attention_weight_max": 117.9068374633789,
      "activations/layer4_attention_weight_min": -115.03812408447266,
      "activations/layer5_attention_weight_max": 54.525691986083984,
      "activations/layer5_attention_weight_min": -62.38650131225586,
      "activations/layer6_attention_weight_max": 46.62749481201172,
      "activations/layer6_attention_weight_min": -47.62802505493164,
      "activations/layer7_attention_weight_max": 93.41333770751953,
      "activations/layer7_attention_weight_min": -91.59981536865234,
      "activations/layer8_attention_weight_max": 42.74261474609375,
      "activations/layer8_attention_weight_min": -42.75679397583008,
      "activations/layer9_attention_weight_max": 33.26662826538086,
      "activations/layer9_attention_weight_min": -35.0893440246582,
      "epoch": 11.81,
      "learning_rate": 7.461060606060606e-05,
      "loss": 2.7921,
      "step": 203200
    },
    {
      "activations/layer0_attention_weight_max": 15.787052154541016,
      "activations/layer0_attention_weight_min": -12.443999290466309,
      "activations/layer10_attention_weight_max": 31.352310180664062,
      "activations/layer10_attention_weight_min": -32.67430877685547,
      "activations/layer11_attention_weight_max": 31.21437644958496,
      "activations/layer11_attention_weight_min": -33.07709503173828,
      "activations/layer12_attention_weight_max": 25.1262264251709,
      "activations/layer12_attention_weight_min": -25.174997329711914,
      "activations/layer13_attention_weight_max": 40.49859619140625,
      "activations/layer13_attention_weight_min": -36.03011703491211,
      "activations/layer14_attention_weight_max": 46.085792541503906,
      "activations/layer14_attention_weight_min": -39.012081146240234,
      "activations/layer15_attention_weight_max": 37.66191101074219,
      "activations/layer15_attention_weight_min": -33.29670333862305,
      "activations/layer16_attention_weight_max": 37.017486572265625,
      "activations/layer16_attention_weight_min": -27.744674682617188,
      "activations/layer17_attention_weight_max": 37.2041130065918,
      "activations/layer17_attention_weight_min": -25.98479652404785,
      "activations/layer18_attention_weight_max": 35.01667404174805,
      "activations/layer18_attention_weight_min": -23.967065811157227,
      "activations/layer19_attention_weight_max": 39.40068054199219,
      "activations/layer19_attention_weight_min": -31.820295333862305,
      "activations/layer1_attention_weight_max": 16.08867835998535,
      "activations/layer1_attention_weight_min": -14.532512664794922,
      "activations/layer20_attention_weight_max": 33.933250427246094,
      "activations/layer20_attention_weight_min": -24.890995025634766,
      "activations/layer21_attention_weight_max": 33.802345275878906,
      "activations/layer21_attention_weight_min": -24.23502540588379,
      "activations/layer22_attention_weight_max": 52.13928985595703,
      "activations/layer22_attention_weight_min": -30.81429672241211,
      "activations/layer23_attention_weight_max": 40.157012939453125,
      "activations/layer23_attention_weight_min": -25.221342086791992,
      "activations/layer2_attention_weight_max": 33.100555419921875,
      "activations/layer2_attention_weight_min": -31.28357696533203,
      "activations/layer3_attention_weight_max": 97.78365325927734,
      "activations/layer3_attention_weight_min": -97.56156158447266,
      "activations/layer4_attention_weight_max": 118.76622772216797,
      "activations/layer4_attention_weight_min": -112.3371810913086,
      "activations/layer5_attention_weight_max": 51.864898681640625,
      "activations/layer5_attention_weight_min": -62.01993942260742,
      "activations/layer6_attention_weight_max": 43.5557746887207,
      "activations/layer6_attention_weight_min": -48.566322326660156,
      "activations/layer7_attention_weight_max": 88.18617248535156,
      "activations/layer7_attention_weight_min": -89.50595092773438,
      "activations/layer8_attention_weight_max": 40.80232620239258,
      "activations/layer8_attention_weight_min": -41.235294342041016,
      "activations/layer9_attention_weight_max": 35.068809509277344,
      "activations/layer9_attention_weight_min": -34.8848991394043,
      "epoch": 11.81,
      "learning_rate": 7.459166666666666e-05,
      "loss": 2.7742,
      "step": 203250
    },
    {
      "activations/layer0_attention_weight_max": 15.167703628540039,
      "activations/layer0_attention_weight_min": -11.399923324584961,
      "activations/layer10_attention_weight_max": 34.35406494140625,
      "activations/layer10_attention_weight_min": -35.894798278808594,
      "activations/layer11_attention_weight_max": 32.97639083862305,
      "activations/layer11_attention_weight_min": -33.24116897583008,
      "activations/layer12_attention_weight_max": 26.863447189331055,
      "activations/layer12_attention_weight_min": -24.302433013916016,
      "activations/layer13_attention_weight_max": 41.46196746826172,
      "activations/layer13_attention_weight_min": -34.726131439208984,
      "activations/layer14_attention_weight_max": 46.664493560791016,
      "activations/layer14_attention_weight_min": -39.71953582763672,
      "activations/layer15_attention_weight_max": 38.80116653442383,
      "activations/layer15_attention_weight_min": -34.25431823730469,
      "activations/layer16_attention_weight_max": 33.218116760253906,
      "activations/layer16_attention_weight_min": -28.507709503173828,
      "activations/layer17_attention_weight_max": 33.02357864379883,
      "activations/layer17_attention_weight_min": -27.17471694946289,
      "activations/layer18_attention_weight_max": 30.869709014892578,
      "activations/layer18_attention_weight_min": -26.693422317504883,
      "activations/layer19_attention_weight_max": 37.71284484863281,
      "activations/layer19_attention_weight_min": -32.621280670166016,
      "activations/layer1_attention_weight_max": 16.569107055664062,
      "activations/layer1_attention_weight_min": -13.870134353637695,
      "activations/layer20_attention_weight_max": 34.04984664916992,
      "activations/layer20_attention_weight_min": -26.139869689941406,
      "activations/layer21_attention_weight_max": 29.437973022460938,
      "activations/layer21_attention_weight_min": -25.45140838623047,
      "activations/layer22_attention_weight_max": 44.79875564575195,
      "activations/layer22_attention_weight_min": -33.65775680541992,
      "activations/layer23_attention_weight_max": 34.693607330322266,
      "activations/layer23_attention_weight_min": -24.55999755859375,
      "activations/layer2_attention_weight_max": 33.908321380615234,
      "activations/layer2_attention_weight_min": -31.788654327392578,
      "activations/layer3_attention_weight_max": 101.09180450439453,
      "activations/layer3_attention_weight_min": -101.51850128173828,
      "activations/layer4_attention_weight_max": 118.87068176269531,
      "activations/layer4_attention_weight_min": -114.7563705444336,
      "activations/layer5_attention_weight_max": 52.231258392333984,
      "activations/layer5_attention_weight_min": -63.69758605957031,
      "activations/layer6_attention_weight_max": 45.13030242919922,
      "activations/layer6_attention_weight_min": -48.12954330444336,
      "activations/layer7_attention_weight_max": 93.97845458984375,
      "activations/layer7_attention_weight_min": -91.96855163574219,
      "activations/layer8_attention_weight_max": 39.81966018676758,
      "activations/layer8_attention_weight_min": -42.95662307739258,
      "activations/layer9_attention_weight_max": 35.85493087768555,
      "activations/layer9_attention_weight_min": -34.35692596435547,
      "epoch": 11.81,
      "learning_rate": 7.457272727272727e-05,
      "loss": 2.7976,
      "step": 203300
    },
    {
      "activations/layer0_attention_weight_max": 15.441852569580078,
      "activations/layer0_attention_weight_min": -12.10438060760498,
      "activations/layer10_attention_weight_max": 31.061613082885742,
      "activations/layer10_attention_weight_min": -32.29267120361328,
      "activations/layer11_attention_weight_max": 28.89117431640625,
      "activations/layer11_attention_weight_min": -30.848806381225586,
      "activations/layer12_attention_weight_max": 23.303546905517578,
      "activations/layer12_attention_weight_min": -25.494422912597656,
      "activations/layer13_attention_weight_max": 41.47569274902344,
      "activations/layer13_attention_weight_min": -35.65824890136719,
      "activations/layer14_attention_weight_max": 41.66152572631836,
      "activations/layer14_attention_weight_min": -38.613189697265625,
      "activations/layer15_attention_weight_max": 36.54648208618164,
      "activations/layer15_attention_weight_min": -32.532901763916016,
      "activations/layer16_attention_weight_max": 30.034381866455078,
      "activations/layer16_attention_weight_min": -28.74219512939453,
      "activations/layer17_attention_weight_max": 30.632301330566406,
      "activations/layer17_attention_weight_min": -25.898704528808594,
      "activations/layer18_attention_weight_max": 32.10717010498047,
      "activations/layer18_attention_weight_min": -24.37089729309082,
      "activations/layer19_attention_weight_max": 34.348846435546875,
      "activations/layer19_attention_weight_min": -31.175588607788086,
      "activations/layer1_attention_weight_max": 15.9906644821167,
      "activations/layer1_attention_weight_min": -14.450313568115234,
      "activations/layer20_attention_weight_max": 28.969985961914062,
      "activations/layer20_attention_weight_min": -24.98163414001465,
      "activations/layer21_attention_weight_max": 29.375810623168945,
      "activations/layer21_attention_weight_min": -23.613779067993164,
      "activations/layer22_attention_weight_max": 40.97536087036133,
      "activations/layer22_attention_weight_min": -30.60532569885254,
      "activations/layer23_attention_weight_max": 34.34787368774414,
      "activations/layer23_attention_weight_min": -25.92713165283203,
      "activations/layer2_attention_weight_max": 33.39807891845703,
      "activations/layer2_attention_weight_min": -33.11478805541992,
      "activations/layer3_attention_weight_max": 94.46659088134766,
      "activations/layer3_attention_weight_min": -102.69390106201172,
      "activations/layer4_attention_weight_max": 113.65340423583984,
      "activations/layer4_attention_weight_min": -115.9836654663086,
      "activations/layer5_attention_weight_max": 52.98200988769531,
      "activations/layer5_attention_weight_min": -62.947383880615234,
      "activations/layer6_attention_weight_max": 43.7762565612793,
      "activations/layer6_attention_weight_min": -48.73837661743164,
      "activations/layer7_attention_weight_max": 88.49927520751953,
      "activations/layer7_attention_weight_min": -93.94075012207031,
      "activations/layer8_attention_weight_max": 39.41268539428711,
      "activations/layer8_attention_weight_min": -41.81100082397461,
      "activations/layer9_attention_weight_max": 31.456912994384766,
      "activations/layer9_attention_weight_min": -35.02730941772461,
      "epoch": 11.82,
      "learning_rate": 7.455378787878788e-05,
      "loss": 2.7703,
      "step": 203350
    },
    {
      "activations/layer0_attention_weight_max": 16.10727310180664,
      "activations/layer0_attention_weight_min": -13.16884708404541,
      "activations/layer10_attention_weight_max": 34.39558792114258,
      "activations/layer10_attention_weight_min": -33.03211212158203,
      "activations/layer11_attention_weight_max": 34.97340393066406,
      "activations/layer11_attention_weight_min": -32.34144973754883,
      "activations/layer12_attention_weight_max": 24.835779190063477,
      "activations/layer12_attention_weight_min": -24.085983276367188,
      "activations/layer13_attention_weight_max": 41.19924545288086,
      "activations/layer13_attention_weight_min": -33.802059173583984,
      "activations/layer14_attention_weight_max": 46.70834732055664,
      "activations/layer14_attention_weight_min": -38.50370407104492,
      "activations/layer15_attention_weight_max": 37.16453170776367,
      "activations/layer15_attention_weight_min": -31.404062271118164,
      "activations/layer16_attention_weight_max": 31.361698150634766,
      "activations/layer16_attention_weight_min": -27.598674774169922,
      "activations/layer17_attention_weight_max": 32.992881774902344,
      "activations/layer17_attention_weight_min": -26.059349060058594,
      "activations/layer18_attention_weight_max": 33.33091354370117,
      "activations/layer18_attention_weight_min": -23.125015258789062,
      "activations/layer19_attention_weight_max": 34.64570236206055,
      "activations/layer19_attention_weight_min": -31.572952270507812,
      "activations/layer1_attention_weight_max": 16.319929122924805,
      "activations/layer1_attention_weight_min": -15.036267280578613,
      "activations/layer20_attention_weight_max": 28.693511962890625,
      "activations/layer20_attention_weight_min": -26.152864456176758,
      "activations/layer21_attention_weight_max": 27.154876708984375,
      "activations/layer21_attention_weight_min": -23.65172004699707,
      "activations/layer22_attention_weight_max": 47.057701110839844,
      "activations/layer22_attention_weight_min": -29.84903907775879,
      "activations/layer23_attention_weight_max": 36.29352951049805,
      "activations/layer23_attention_weight_min": -26.614538192749023,
      "activations/layer2_attention_weight_max": 33.19007873535156,
      "activations/layer2_attention_weight_min": -32.800315856933594,
      "activations/layer3_attention_weight_max": 99.0234146118164,
      "activations/layer3_attention_weight_min": -100.80987548828125,
      "activations/layer4_attention_weight_max": 115.51051330566406,
      "activations/layer4_attention_weight_min": -118.36323547363281,
      "activations/layer5_attention_weight_max": 52.98781967163086,
      "activations/layer5_attention_weight_min": -64.22521209716797,
      "activations/layer6_attention_weight_max": 45.88908004760742,
      "activations/layer6_attention_weight_min": -49.181949615478516,
      "activations/layer7_attention_weight_max": 94.85891723632812,
      "activations/layer7_attention_weight_min": -94.66790008544922,
      "activations/layer8_attention_weight_max": 41.04681396484375,
      "activations/layer8_attention_weight_min": -41.67777633666992,
      "activations/layer9_attention_weight_max": 34.39281463623047,
      "activations/layer9_attention_weight_min": -33.83068084716797,
      "epoch": 11.82,
      "learning_rate": 7.453484848484848e-05,
      "loss": 2.7675,
      "step": 203400
    },
    {
      "activations/layer0_attention_weight_max": 15.86767864227295,
      "activations/layer0_attention_weight_min": -11.571354866027832,
      "activations/layer10_attention_weight_max": 30.152360916137695,
      "activations/layer10_attention_weight_min": -32.75995635986328,
      "activations/layer11_attention_weight_max": 30.258651733398438,
      "activations/layer11_attention_weight_min": -31.10847282409668,
      "activations/layer12_attention_weight_max": 25.060482025146484,
      "activations/layer12_attention_weight_min": -25.13170051574707,
      "activations/layer13_attention_weight_max": 36.99294662475586,
      "activations/layer13_attention_weight_min": -34.66896057128906,
      "activations/layer14_attention_weight_max": 40.25934982299805,
      "activations/layer14_attention_weight_min": -37.72081756591797,
      "activations/layer15_attention_weight_max": 33.5504264831543,
      "activations/layer15_attention_weight_min": -32.35215377807617,
      "activations/layer16_attention_weight_max": 31.154495239257812,
      "activations/layer16_attention_weight_min": -27.092697143554688,
      "activations/layer17_attention_weight_max": 30.0887393951416,
      "activations/layer17_attention_weight_min": -26.673255920410156,
      "activations/layer18_attention_weight_max": 30.97978401184082,
      "activations/layer18_attention_weight_min": -24.15739631652832,
      "activations/layer19_attention_weight_max": 35.30934143066406,
      "activations/layer19_attention_weight_min": -30.937686920166016,
      "activations/layer1_attention_weight_max": 16.64269256591797,
      "activations/layer1_attention_weight_min": -15.773407936096191,
      "activations/layer20_attention_weight_max": 28.352073669433594,
      "activations/layer20_attention_weight_min": -24.69319725036621,
      "activations/layer21_attention_weight_max": 29.83231544494629,
      "activations/layer21_attention_weight_min": -25.693376541137695,
      "activations/layer22_attention_weight_max": 44.151458740234375,
      "activations/layer22_attention_weight_min": -30.463573455810547,
      "activations/layer23_attention_weight_max": 32.68206787109375,
      "activations/layer23_attention_weight_min": -24.303998947143555,
      "activations/layer2_attention_weight_max": 34.915470123291016,
      "activations/layer2_attention_weight_min": -33.0018424987793,
      "activations/layer3_attention_weight_max": 102.93991088867188,
      "activations/layer3_attention_weight_min": -99.35797882080078,
      "activations/layer4_attention_weight_max": 121.37483215332031,
      "activations/layer4_attention_weight_min": -115.32453918457031,
      "activations/layer5_attention_weight_max": 51.82752227783203,
      "activations/layer5_attention_weight_min": -58.96799850463867,
      "activations/layer6_attention_weight_max": 42.559017181396484,
      "activations/layer6_attention_weight_min": -46.700050354003906,
      "activations/layer7_attention_weight_max": 87.9372329711914,
      "activations/layer7_attention_weight_min": -91.91197204589844,
      "activations/layer8_attention_weight_max": 39.47377395629883,
      "activations/layer8_attention_weight_min": -40.2353630065918,
      "activations/layer9_attention_weight_max": 31.971755981445312,
      "activations/layer9_attention_weight_min": -33.26640701293945,
      "epoch": 11.82,
      "learning_rate": 7.451590909090908e-05,
      "loss": 2.7549,
      "step": 203450
    },
    {
      "activations/layer0_attention_weight_max": 15.114234924316406,
      "activations/layer0_attention_weight_min": -12.42155647277832,
      "activations/layer10_attention_weight_max": 32.95344924926758,
      "activations/layer10_attention_weight_min": -32.32879638671875,
      "activations/layer11_attention_weight_max": 36.063926696777344,
      "activations/layer11_attention_weight_min": -34.04519271850586,
      "activations/layer12_attention_weight_max": 25.58688735961914,
      "activations/layer12_attention_weight_min": -26.791440963745117,
      "activations/layer13_attention_weight_max": 42.31795120239258,
      "activations/layer13_attention_weight_min": -33.39335250854492,
      "activations/layer14_attention_weight_max": 44.81780242919922,
      "activations/layer14_attention_weight_min": -38.09699249267578,
      "activations/layer15_attention_weight_max": 42.22393798828125,
      "activations/layer15_attention_weight_min": -33.20563507080078,
      "activations/layer16_attention_weight_max": 33.816444396972656,
      "activations/layer16_attention_weight_min": -28.478450775146484,
      "activations/layer17_attention_weight_max": 32.16914367675781,
      "activations/layer17_attention_weight_min": -26.166793823242188,
      "activations/layer18_attention_weight_max": 29.396989822387695,
      "activations/layer18_attention_weight_min": -23.456026077270508,
      "activations/layer19_attention_weight_max": 39.60097122192383,
      "activations/layer19_attention_weight_min": -31.955692291259766,
      "activations/layer1_attention_weight_max": 17.60019302368164,
      "activations/layer1_attention_weight_min": -14.118705749511719,
      "activations/layer20_attention_weight_max": 30.68963050842285,
      "activations/layer20_attention_weight_min": -24.494468688964844,
      "activations/layer21_attention_weight_max": 29.206993103027344,
      "activations/layer21_attention_weight_min": -23.798728942871094,
      "activations/layer22_attention_weight_max": 39.707977294921875,
      "activations/layer22_attention_weight_min": -31.3065242767334,
      "activations/layer23_attention_weight_max": 32.92420196533203,
      "activations/layer23_attention_weight_min": -25.903470993041992,
      "activations/layer2_attention_weight_max": 32.973655700683594,
      "activations/layer2_attention_weight_min": -32.63681411743164,
      "activations/layer3_attention_weight_max": 99.53719329833984,
      "activations/layer3_attention_weight_min": -100.4120864868164,
      "activations/layer4_attention_weight_max": 117.96227264404297,
      "activations/layer4_attention_weight_min": -117.29154205322266,
      "activations/layer5_attention_weight_max": 58.83038330078125,
      "activations/layer5_attention_weight_min": -64.12018585205078,
      "activations/layer6_attention_weight_max": 45.35783004760742,
      "activations/layer6_attention_weight_min": -47.79948425292969,
      "activations/layer7_attention_weight_max": 93.76883697509766,
      "activations/layer7_attention_weight_min": -90.9156723022461,
      "activations/layer8_attention_weight_max": 39.54315948486328,
      "activations/layer8_attention_weight_min": -41.97022247314453,
      "activations/layer9_attention_weight_max": 31.6541748046875,
      "activations/layer9_attention_weight_min": -33.483795166015625,
      "epoch": 11.82,
      "learning_rate": 7.44969696969697e-05,
      "loss": 2.7753,
      "step": 203500
    },
    {
      "activations/layer0_attention_weight_max": 15.163737297058105,
      "activations/layer0_attention_weight_min": -11.512500762939453,
      "activations/layer10_attention_weight_max": 32.18311309814453,
      "activations/layer10_attention_weight_min": -33.65040588378906,
      "activations/layer11_attention_weight_max": 32.0498046875,
      "activations/layer11_attention_weight_min": -32.12211608886719,
      "activations/layer12_attention_weight_max": 23.778329849243164,
      "activations/layer12_attention_weight_min": -24.24021339416504,
      "activations/layer13_attention_weight_max": 38.71802520751953,
      "activations/layer13_attention_weight_min": -33.95414352416992,
      "activations/layer14_attention_weight_max": 43.75397872924805,
      "activations/layer14_attention_weight_min": -38.7758674621582,
      "activations/layer15_attention_weight_max": 35.07025146484375,
      "activations/layer15_attention_weight_min": -33.01713562011719,
      "activations/layer16_attention_weight_max": 28.565631866455078,
      "activations/layer16_attention_weight_min": -29.633718490600586,
      "activations/layer17_attention_weight_max": 29.97369384765625,
      "activations/layer17_attention_weight_min": -25.437707901000977,
      "activations/layer18_attention_weight_max": 30.285593032836914,
      "activations/layer18_attention_weight_min": -22.048534393310547,
      "activations/layer19_attention_weight_max": 33.77022933959961,
      "activations/layer19_attention_weight_min": -28.185876846313477,
      "activations/layer1_attention_weight_max": 16.684011459350586,
      "activations/layer1_attention_weight_min": -14.18319320678711,
      "activations/layer20_attention_weight_max": 27.706974029541016,
      "activations/layer20_attention_weight_min": -24.543546676635742,
      "activations/layer21_attention_weight_max": 28.19820785522461,
      "activations/layer21_attention_weight_min": -23.08670997619629,
      "activations/layer22_attention_weight_max": 42.74456787109375,
      "activations/layer22_attention_weight_min": -28.88044548034668,
      "activations/layer23_attention_weight_max": 30.79169273376465,
      "activations/layer23_attention_weight_min": -24.509130477905273,
      "activations/layer2_attention_weight_max": 33.88785171508789,
      "activations/layer2_attention_weight_min": -34.26565933227539,
      "activations/layer3_attention_weight_max": 103.59803771972656,
      "activations/layer3_attention_weight_min": -106.7659912109375,
      "activations/layer4_attention_weight_max": 116.67630767822266,
      "activations/layer4_attention_weight_min": -119.08025360107422,
      "activations/layer5_attention_weight_max": 53.61599349975586,
      "activations/layer5_attention_weight_min": -64.8304214477539,
      "activations/layer6_attention_weight_max": 48.23943328857422,
      "activations/layer6_attention_weight_min": -50.20192337036133,
      "activations/layer7_attention_weight_max": 99.73065185546875,
      "activations/layer7_attention_weight_min": -93.02111053466797,
      "activations/layer8_attention_weight_max": 44.93245315551758,
      "activations/layer8_attention_weight_min": -42.13683319091797,
      "activations/layer9_attention_weight_max": 33.77358627319336,
      "activations/layer9_attention_weight_min": -33.86428451538086,
      "epoch": 11.83,
      "learning_rate": 7.44780303030303e-05,
      "loss": 2.7824,
      "step": 203550
    },
    {
      "activations/layer0_attention_weight_max": 16.367355346679688,
      "activations/layer0_attention_weight_min": -12.317824363708496,
      "activations/layer10_attention_weight_max": 36.519989013671875,
      "activations/layer10_attention_weight_min": -32.301612854003906,
      "activations/layer11_attention_weight_max": 37.893558502197266,
      "activations/layer11_attention_weight_min": -32.79534912109375,
      "activations/layer12_attention_weight_max": 26.138141632080078,
      "activations/layer12_attention_weight_min": -27.40938949584961,
      "activations/layer13_attention_weight_max": 38.507293701171875,
      "activations/layer13_attention_weight_min": -32.70156478881836,
      "activations/layer14_attention_weight_max": 43.05278015136719,
      "activations/layer14_attention_weight_min": -36.636627197265625,
      "activations/layer15_attention_weight_max": 35.95719909667969,
      "activations/layer15_attention_weight_min": -31.306995391845703,
      "activations/layer16_attention_weight_max": 33.027008056640625,
      "activations/layer16_attention_weight_min": -26.59954261779785,
      "activations/layer17_attention_weight_max": 33.04130935668945,
      "activations/layer17_attention_weight_min": -25.2193660736084,
      "activations/layer18_attention_weight_max": 31.645479202270508,
      "activations/layer18_attention_weight_min": -23.149126052856445,
      "activations/layer19_attention_weight_max": 37.5969123840332,
      "activations/layer19_attention_weight_min": -29.233905792236328,
      "activations/layer1_attention_weight_max": 17.02777671813965,
      "activations/layer1_attention_weight_min": -18.630474090576172,
      "activations/layer20_attention_weight_max": 32.75893020629883,
      "activations/layer20_attention_weight_min": -23.031389236450195,
      "activations/layer21_attention_weight_max": 30.23628807067871,
      "activations/layer21_attention_weight_min": -23.62833595275879,
      "activations/layer22_attention_weight_max": 44.2867317199707,
      "activations/layer22_attention_weight_min": -28.53963279724121,
      "activations/layer23_attention_weight_max": 37.30747604370117,
      "activations/layer23_attention_weight_min": -24.927642822265625,
      "activations/layer2_attention_weight_max": 32.199256896972656,
      "activations/layer2_attention_weight_min": -33.13138198852539,
      "activations/layer3_attention_weight_max": 96.04536437988281,
      "activations/layer3_attention_weight_min": -98.91606903076172,
      "activations/layer4_attention_weight_max": 112.2362289428711,
      "activations/layer4_attention_weight_min": -113.89727020263672,
      "activations/layer5_attention_weight_max": 53.809288024902344,
      "activations/layer5_attention_weight_min": -58.0789909362793,
      "activations/layer6_attention_weight_max": 45.947776794433594,
      "activations/layer6_attention_weight_min": -47.49939727783203,
      "activations/layer7_attention_weight_max": 95.56710815429688,
      "activations/layer7_attention_weight_min": -92.32097625732422,
      "activations/layer8_attention_weight_max": 44.14952087402344,
      "activations/layer8_attention_weight_min": -41.69279861450195,
      "activations/layer9_attention_weight_max": 35.34387969970703,
      "activations/layer9_attention_weight_min": -33.18766403198242,
      "epoch": 11.83,
      "learning_rate": 7.44590909090909e-05,
      "loss": 2.7723,
      "step": 203600
    },
    {
      "activations/layer0_attention_weight_max": 15.488572120666504,
      "activations/layer0_attention_weight_min": -11.525577545166016,
      "activations/layer10_attention_weight_max": 30.687273025512695,
      "activations/layer10_attention_weight_min": -32.54829788208008,
      "activations/layer11_attention_weight_max": 30.701200485229492,
      "activations/layer11_attention_weight_min": -30.927734375,
      "activations/layer12_attention_weight_max": 27.240388870239258,
      "activations/layer12_attention_weight_min": -31.726438522338867,
      "activations/layer13_attention_weight_max": 39.88343811035156,
      "activations/layer13_attention_weight_min": -33.41954040527344,
      "activations/layer14_attention_weight_max": 42.598045349121094,
      "activations/layer14_attention_weight_min": -35.97819519042969,
      "activations/layer15_attention_weight_max": 38.64269256591797,
      "activations/layer15_attention_weight_min": -34.1134033203125,
      "activations/layer16_attention_weight_max": 31.705167770385742,
      "activations/layer16_attention_weight_min": -28.22998046875,
      "activations/layer17_attention_weight_max": 31.374441146850586,
      "activations/layer17_attention_weight_min": -26.35878562927246,
      "activations/layer18_attention_weight_max": 31.843130111694336,
      "activations/layer18_attention_weight_min": -23.661230087280273,
      "activations/layer19_attention_weight_max": 34.51648712158203,
      "activations/layer19_attention_weight_min": -32.99618148803711,
      "activations/layer1_attention_weight_max": 16.771345138549805,
      "activations/layer1_attention_weight_min": -16.026723861694336,
      "activations/layer20_attention_weight_max": 31.106510162353516,
      "activations/layer20_attention_weight_min": -24.204235076904297,
      "activations/layer21_attention_weight_max": 29.94706153869629,
      "activations/layer21_attention_weight_min": -23.90496063232422,
      "activations/layer22_attention_weight_max": 42.08662796020508,
      "activations/layer22_attention_weight_min": -29.196483612060547,
      "activations/layer23_attention_weight_max": 33.103519439697266,
      "activations/layer23_attention_weight_min": -23.251163482666016,
      "activations/layer2_attention_weight_max": 32.63890075683594,
      "activations/layer2_attention_weight_min": -32.732269287109375,
      "activations/layer3_attention_weight_max": 96.46316528320312,
      "activations/layer3_attention_weight_min": -99.2620849609375,
      "activations/layer4_attention_weight_max": 114.96739959716797,
      "activations/layer4_attention_weight_min": -109.1355209350586,
      "activations/layer5_attention_weight_max": 51.699951171875,
      "activations/layer5_attention_weight_min": -59.02275085449219,
      "activations/layer6_attention_weight_max": 44.62131881713867,
      "activations/layer6_attention_weight_min": -46.65436935424805,
      "activations/layer7_attention_weight_max": 86.42828369140625,
      "activations/layer7_attention_weight_min": -90.32923889160156,
      "activations/layer8_attention_weight_max": 36.88237380981445,
      "activations/layer8_attention_weight_min": -39.04826354980469,
      "activations/layer9_attention_weight_max": 30.918262481689453,
      "activations/layer9_attention_weight_min": -33.324493408203125,
      "epoch": 11.83,
      "learning_rate": 7.444015151515151e-05,
      "loss": 2.7654,
      "step": 203650
    },
    {
      "activations/layer0_attention_weight_max": 15.511492729187012,
      "activations/layer0_attention_weight_min": -11.713150978088379,
      "activations/layer10_attention_weight_max": 33.298255920410156,
      "activations/layer10_attention_weight_min": -33.962135314941406,
      "activations/layer11_attention_weight_max": 35.08867263793945,
      "activations/layer11_attention_weight_min": -32.765663146972656,
      "activations/layer12_attention_weight_max": 27.640769958496094,
      "activations/layer12_attention_weight_min": -27.94449234008789,
      "activations/layer13_attention_weight_max": 39.85917282104492,
      "activations/layer13_attention_weight_min": -36.46247100830078,
      "activations/layer14_attention_weight_max": 44.37288284301758,
      "activations/layer14_attention_weight_min": -36.691070556640625,
      "activations/layer15_attention_weight_max": 38.64509582519531,
      "activations/layer15_attention_weight_min": -32.609840393066406,
      "activations/layer16_attention_weight_max": 29.920366287231445,
      "activations/layer16_attention_weight_min": -29.613204956054688,
      "activations/layer17_attention_weight_max": 31.87622833251953,
      "activations/layer17_attention_weight_min": -26.79155921936035,
      "activations/layer18_attention_weight_max": 33.27298355102539,
      "activations/layer18_attention_weight_min": -23.763071060180664,
      "activations/layer19_attention_weight_max": 32.84429931640625,
      "activations/layer19_attention_weight_min": -32.23159408569336,
      "activations/layer1_attention_weight_max": 16.990007400512695,
      "activations/layer1_attention_weight_min": -15.268585205078125,
      "activations/layer20_attention_weight_max": 28.004131317138672,
      "activations/layer20_attention_weight_min": -24.328947067260742,
      "activations/layer21_attention_weight_max": 28.740997314453125,
      "activations/layer21_attention_weight_min": -23.32250213623047,
      "activations/layer22_attention_weight_max": 44.293697357177734,
      "activations/layer22_attention_weight_min": -31.454479217529297,
      "activations/layer23_attention_weight_max": 32.25755310058594,
      "activations/layer23_attention_weight_min": -27.127599716186523,
      "activations/layer2_attention_weight_max": 34.53886413574219,
      "activations/layer2_attention_weight_min": -32.904823303222656,
      "activations/layer3_attention_weight_max": 101.38531494140625,
      "activations/layer3_attention_weight_min": -97.19137573242188,
      "activations/layer4_attention_weight_max": 116.21099090576172,
      "activations/layer4_attention_weight_min": -115.3747329711914,
      "activations/layer5_attention_weight_max": 52.78045654296875,
      "activations/layer5_attention_weight_min": -63.48081588745117,
      "activations/layer6_attention_weight_max": 43.39155960083008,
      "activations/layer6_attention_weight_min": -47.25754165649414,
      "activations/layer7_attention_weight_max": 89.34944915771484,
      "activations/layer7_attention_weight_min": -91.67646789550781,
      "activations/layer8_attention_weight_max": 41.7591667175293,
      "activations/layer8_attention_weight_min": -41.98554992675781,
      "activations/layer9_attention_weight_max": 35.56779098510742,
      "activations/layer9_attention_weight_min": -34.25600814819336,
      "epoch": 11.84,
      "learning_rate": 7.442121212121211e-05,
      "loss": 2.7949,
      "step": 203700
    },
    {
      "activations/layer0_attention_weight_max": 15.46249008178711,
      "activations/layer0_attention_weight_min": -12.307173728942871,
      "activations/layer10_attention_weight_max": 34.00908279418945,
      "activations/layer10_attention_weight_min": -32.89894485473633,
      "activations/layer11_attention_weight_max": 34.37422180175781,
      "activations/layer11_attention_weight_min": -32.294227600097656,
      "activations/layer12_attention_weight_max": 28.64899444580078,
      "activations/layer12_attention_weight_min": -27.929988861083984,
      "activations/layer13_attention_weight_max": 42.63478469848633,
      "activations/layer13_attention_weight_min": -33.15742492675781,
      "activations/layer14_attention_weight_max": 45.19493865966797,
      "activations/layer14_attention_weight_min": -38.693145751953125,
      "activations/layer15_attention_weight_max": 40.24796676635742,
      "activations/layer15_attention_weight_min": -31.482751846313477,
      "activations/layer16_attention_weight_max": 32.875770568847656,
      "activations/layer16_attention_weight_min": -27.786916732788086,
      "activations/layer17_attention_weight_max": 33.651084899902344,
      "activations/layer17_attention_weight_min": -24.28909683227539,
      "activations/layer18_attention_weight_max": 34.73400115966797,
      "activations/layer18_attention_weight_min": -23.570266723632812,
      "activations/layer19_attention_weight_max": 38.34043502807617,
      "activations/layer19_attention_weight_min": -31.957273483276367,
      "activations/layer1_attention_weight_max": 16.527729034423828,
      "activations/layer1_attention_weight_min": -17.022615432739258,
      "activations/layer20_attention_weight_max": 32.76434326171875,
      "activations/layer20_attention_weight_min": -23.35125732421875,
      "activations/layer21_attention_weight_max": 31.004308700561523,
      "activations/layer21_attention_weight_min": -24.9670352935791,
      "activations/layer22_attention_weight_max": 49.001014709472656,
      "activations/layer22_attention_weight_min": -31.188325881958008,
      "activations/layer23_attention_weight_max": 34.74560546875,
      "activations/layer23_attention_weight_min": -24.889280319213867,
      "activations/layer2_attention_weight_max": 33.026832580566406,
      "activations/layer2_attention_weight_min": -33.42793655395508,
      "activations/layer3_attention_weight_max": 95.86615753173828,
      "activations/layer3_attention_weight_min": -98.92164611816406,
      "activations/layer4_attention_weight_max": 112.44058990478516,
      "activations/layer4_attention_weight_min": -113.37554931640625,
      "activations/layer5_attention_weight_max": 55.74132537841797,
      "activations/layer5_attention_weight_min": -59.52073287963867,
      "activations/layer6_attention_weight_max": 44.549129486083984,
      "activations/layer6_attention_weight_min": -46.99433135986328,
      "activations/layer7_attention_weight_max": 91.05738830566406,
      "activations/layer7_attention_weight_min": -90.30091094970703,
      "activations/layer8_attention_weight_max": 40.26588821411133,
      "activations/layer8_attention_weight_min": -41.91048049926758,
      "activations/layer9_attention_weight_max": 34.44171142578125,
      "activations/layer9_attention_weight_min": -33.00929260253906,
      "epoch": 11.84,
      "learning_rate": 7.440227272727271e-05,
      "loss": 2.7759,
      "step": 203750
    },
    {
      "activations/layer0_attention_weight_max": 15.975693702697754,
      "activations/layer0_attention_weight_min": -12.154817581176758,
      "activations/layer10_attention_weight_max": 30.915067672729492,
      "activations/layer10_attention_weight_min": -32.38970184326172,
      "activations/layer11_attention_weight_max": 31.03647232055664,
      "activations/layer11_attention_weight_min": -32.012821197509766,
      "activations/layer12_attention_weight_max": 24.09235382080078,
      "activations/layer12_attention_weight_min": -25.42327880859375,
      "activations/layer13_attention_weight_max": 38.090797424316406,
      "activations/layer13_attention_weight_min": -33.911865234375,
      "activations/layer14_attention_weight_max": 41.92094421386719,
      "activations/layer14_attention_weight_min": -39.51679611206055,
      "activations/layer15_attention_weight_max": 34.51890182495117,
      "activations/layer15_attention_weight_min": -30.098472595214844,
      "activations/layer16_attention_weight_max": 28.164709091186523,
      "activations/layer16_attention_weight_min": -26.734079360961914,
      "activations/layer17_attention_weight_max": 29.02115249633789,
      "activations/layer17_attention_weight_min": -26.110427856445312,
      "activations/layer18_attention_weight_max": 34.621360778808594,
      "activations/layer18_attention_weight_min": -26.15987205505371,
      "activations/layer19_attention_weight_max": 33.37252426147461,
      "activations/layer19_attention_weight_min": -31.269973754882812,
      "activations/layer1_attention_weight_max": 16.603851318359375,
      "activations/layer1_attention_weight_min": -15.108365058898926,
      "activations/layer20_attention_weight_max": 31.540937423706055,
      "activations/layer20_attention_weight_min": -27.086801528930664,
      "activations/layer21_attention_weight_max": 31.83909034729004,
      "activations/layer21_attention_weight_min": -24.802806854248047,
      "activations/layer22_attention_weight_max": 44.93882369995117,
      "activations/layer22_attention_weight_min": -31.29258918762207,
      "activations/layer23_attention_weight_max": 32.52463912963867,
      "activations/layer23_attention_weight_min": -25.23683738708496,
      "activations/layer2_attention_weight_max": 34.54527282714844,
      "activations/layer2_attention_weight_min": -34.528778076171875,
      "activations/layer3_attention_weight_max": 97.5811996459961,
      "activations/layer3_attention_weight_min": -99.97899627685547,
      "activations/layer4_attention_weight_max": 115.17180633544922,
      "activations/layer4_attention_weight_min": -113.98087310791016,
      "activations/layer5_attention_weight_max": 54.58425521850586,
      "activations/layer5_attention_weight_min": -62.660457611083984,
      "activations/layer6_attention_weight_max": 46.756107330322266,
      "activations/layer6_attention_weight_min": -48.08497619628906,
      "activations/layer7_attention_weight_max": 90.24397277832031,
      "activations/layer7_attention_weight_min": -90.0482177734375,
      "activations/layer8_attention_weight_max": 42.75111389160156,
      "activations/layer8_attention_weight_min": -44.15372085571289,
      "activations/layer9_attention_weight_max": 32.41110610961914,
      "activations/layer9_attention_weight_min": -32.489707946777344,
      "epoch": 11.84,
      "learning_rate": 7.438333333333333e-05,
      "loss": 2.7867,
      "step": 203800
    },
    {
      "activations/layer0_attention_weight_max": 16.23575782775879,
      "activations/layer0_attention_weight_min": -12.03024959564209,
      "activations/layer10_attention_weight_max": 31.641965866088867,
      "activations/layer10_attention_weight_min": -34.17589569091797,
      "activations/layer11_attention_weight_max": 33.87491226196289,
      "activations/layer11_attention_weight_min": -35.32971954345703,
      "activations/layer12_attention_weight_max": 25.341962814331055,
      "activations/layer12_attention_weight_min": -24.674156188964844,
      "activations/layer13_attention_weight_max": 38.68345642089844,
      "activations/layer13_attention_weight_min": -34.60938262939453,
      "activations/layer14_attention_weight_max": 44.2390022277832,
      "activations/layer14_attention_weight_min": -37.42830276489258,
      "activations/layer15_attention_weight_max": 35.86212158203125,
      "activations/layer15_attention_weight_min": -32.245872497558594,
      "activations/layer16_attention_weight_max": 29.999319076538086,
      "activations/layer16_attention_weight_min": -28.316360473632812,
      "activations/layer17_attention_weight_max": 30.55441665649414,
      "activations/layer17_attention_weight_min": -25.98655128479004,
      "activations/layer18_attention_weight_max": 31.681415557861328,
      "activations/layer18_attention_weight_min": -23.196054458618164,
      "activations/layer19_attention_weight_max": 32.06671142578125,
      "activations/layer19_attention_weight_min": -31.691328048706055,
      "activations/layer1_attention_weight_max": 15.962675094604492,
      "activations/layer1_attention_weight_min": -15.061017036437988,
      "activations/layer20_attention_weight_max": 28.905235290527344,
      "activations/layer20_attention_weight_min": -28.166210174560547,
      "activations/layer21_attention_weight_max": 27.188718795776367,
      "activations/layer21_attention_weight_min": -26.486534118652344,
      "activations/layer22_attention_weight_max": 45.42343521118164,
      "activations/layer22_attention_weight_min": -31.713281631469727,
      "activations/layer23_attention_weight_max": 34.78541946411133,
      "activations/layer23_attention_weight_min": -27.80042839050293,
      "activations/layer2_attention_weight_max": 34.72157287597656,
      "activations/layer2_attention_weight_min": -33.751708984375,
      "activations/layer3_attention_weight_max": 96.81654357910156,
      "activations/layer3_attention_weight_min": -100.26024627685547,
      "activations/layer4_attention_weight_max": 112.6248779296875,
      "activations/layer4_attention_weight_min": -112.02059173583984,
      "activations/layer5_attention_weight_max": 55.51670837402344,
      "activations/layer5_attention_weight_min": -60.360103607177734,
      "activations/layer6_attention_weight_max": 44.38620376586914,
      "activations/layer6_attention_weight_min": -46.23734664916992,
      "activations/layer7_attention_weight_max": 88.34982299804688,
      "activations/layer7_attention_weight_min": -96.40279388427734,
      "activations/layer8_attention_weight_max": 39.09146499633789,
      "activations/layer8_attention_weight_min": -40.20602035522461,
      "activations/layer9_attention_weight_max": 33.22903060913086,
      "activations/layer9_attention_weight_min": -32.23325729370117,
      "epoch": 11.84,
      "learning_rate": 7.436439393939393e-05,
      "loss": 2.7927,
      "step": 203850
    },
    {
      "activations/layer0_attention_weight_max": 15.879308700561523,
      "activations/layer0_attention_weight_min": -12.009926795959473,
      "activations/layer10_attention_weight_max": 33.116432189941406,
      "activations/layer10_attention_weight_min": -31.14828109741211,
      "activations/layer11_attention_weight_max": 34.021453857421875,
      "activations/layer11_attention_weight_min": -31.72205352783203,
      "activations/layer12_attention_weight_max": 23.6390323638916,
      "activations/layer12_attention_weight_min": -23.556324005126953,
      "activations/layer13_attention_weight_max": 35.52465057373047,
      "activations/layer13_attention_weight_min": -33.74444580078125,
      "activations/layer14_attention_weight_max": 40.735511779785156,
      "activations/layer14_attention_weight_min": -33.69805145263672,
      "activations/layer15_attention_weight_max": 34.02647399902344,
      "activations/layer15_attention_weight_min": -31.841983795166016,
      "activations/layer16_attention_weight_max": 30.24053955078125,
      "activations/layer16_attention_weight_min": -27.955280303955078,
      "activations/layer17_attention_weight_max": 29.905080795288086,
      "activations/layer17_attention_weight_min": -25.91653823852539,
      "activations/layer18_attention_weight_max": 28.682992935180664,
      "activations/layer18_attention_weight_min": -22.917802810668945,
      "activations/layer19_attention_weight_max": 31.856678009033203,
      "activations/layer19_attention_weight_min": -30.91928482055664,
      "activations/layer1_attention_weight_max": 16.258865356445312,
      "activations/layer1_attention_weight_min": -15.557194709777832,
      "activations/layer20_attention_weight_max": 30.98651123046875,
      "activations/layer20_attention_weight_min": -27.1591854095459,
      "activations/layer21_attention_weight_max": 26.02077293395996,
      "activations/layer21_attention_weight_min": -25.038833618164062,
      "activations/layer22_attention_weight_max": 41.4155387878418,
      "activations/layer22_attention_weight_min": -31.653841018676758,
      "activations/layer23_attention_weight_max": 32.530704498291016,
      "activations/layer23_attention_weight_min": -23.506786346435547,
      "activations/layer2_attention_weight_max": 32.836326599121094,
      "activations/layer2_attention_weight_min": -31.70168113708496,
      "activations/layer3_attention_weight_max": 95.82677459716797,
      "activations/layer3_attention_weight_min": -95.50448608398438,
      "activations/layer4_attention_weight_max": 111.52959442138672,
      "activations/layer4_attention_weight_min": -107.94107818603516,
      "activations/layer5_attention_weight_max": 51.417903900146484,
      "activations/layer5_attention_weight_min": -63.48109436035156,
      "activations/layer6_attention_weight_max": 42.83377456665039,
      "activations/layer6_attention_weight_min": -47.725345611572266,
      "activations/layer7_attention_weight_max": 90.91246032714844,
      "activations/layer7_attention_weight_min": -85.92958068847656,
      "activations/layer8_attention_weight_max": 39.14745330810547,
      "activations/layer8_attention_weight_min": -41.13113784790039,
      "activations/layer9_attention_weight_max": 32.667747497558594,
      "activations/layer9_attention_weight_min": -32.1093864440918,
      "epoch": 11.85,
      "learning_rate": 7.434545454545455e-05,
      "loss": 2.7832,
      "step": 203900
    },
    {
      "activations/layer0_attention_weight_max": 16.02761459350586,
      "activations/layer0_attention_weight_min": -11.909899711608887,
      "activations/layer10_attention_weight_max": 30.946502685546875,
      "activations/layer10_attention_weight_min": -32.632171630859375,
      "activations/layer11_attention_weight_max": 32.419105529785156,
      "activations/layer11_attention_weight_min": -31.03515625,
      "activations/layer12_attention_weight_max": 23.15475845336914,
      "activations/layer12_attention_weight_min": -24.797361373901367,
      "activations/layer13_attention_weight_max": 36.978206634521484,
      "activations/layer13_attention_weight_min": -33.60856246948242,
      "activations/layer14_attention_weight_max": 47.17285919189453,
      "activations/layer14_attention_weight_min": -36.04460525512695,
      "activations/layer15_attention_weight_max": 33.41732406616211,
      "activations/layer15_attention_weight_min": -30.39828872680664,
      "activations/layer16_attention_weight_max": 30.37184715270996,
      "activations/layer16_attention_weight_min": -26.697521209716797,
      "activations/layer17_attention_weight_max": 32.00920104980469,
      "activations/layer17_attention_weight_min": -24.73063087463379,
      "activations/layer18_attention_weight_max": 31.255043029785156,
      "activations/layer18_attention_weight_min": -23.67889404296875,
      "activations/layer19_attention_weight_max": 38.38423156738281,
      "activations/layer19_attention_weight_min": -30.600797653198242,
      "activations/layer1_attention_weight_max": 17.00864601135254,
      "activations/layer1_attention_weight_min": -16.20923614501953,
      "activations/layer20_attention_weight_max": 33.65797805786133,
      "activations/layer20_attention_weight_min": -23.270824432373047,
      "activations/layer21_attention_weight_max": 32.39736557006836,
      "activations/layer21_attention_weight_min": -23.426786422729492,
      "activations/layer22_attention_weight_max": 44.67002487182617,
      "activations/layer22_attention_weight_min": -28.69185447692871,
      "activations/layer23_attention_weight_max": 37.639442443847656,
      "activations/layer23_attention_weight_min": -24.020084381103516,
      "activations/layer2_attention_weight_max": 33.2626953125,
      "activations/layer2_attention_weight_min": -33.61307907104492,
      "activations/layer3_attention_weight_max": 97.1778793334961,
      "activations/layer3_attention_weight_min": -97.50090026855469,
      "activations/layer4_attention_weight_max": 113.92414855957031,
      "activations/layer4_attention_weight_min": -111.9135971069336,
      "activations/layer5_attention_weight_max": 53.545894622802734,
      "activations/layer5_attention_weight_min": -61.46826171875,
      "activations/layer6_attention_weight_max": 45.29710006713867,
      "activations/layer6_attention_weight_min": -45.95292282104492,
      "activations/layer7_attention_weight_max": 83.98443603515625,
      "activations/layer7_attention_weight_min": -89.1278305053711,
      "activations/layer8_attention_weight_max": 38.31416702270508,
      "activations/layer8_attention_weight_min": -41.856842041015625,
      "activations/layer9_attention_weight_max": 29.726276397705078,
      "activations/layer9_attention_weight_min": -31.649978637695312,
      "epoch": 11.85,
      "learning_rate": 7.432651515151515e-05,
      "loss": 2.7557,
      "step": 203950
    },
    {
      "activations/layer0_attention_weight_max": 16.920915603637695,
      "activations/layer0_attention_weight_min": -12.331319808959961,
      "activations/layer10_attention_weight_max": 31.12069320678711,
      "activations/layer10_attention_weight_min": -32.16520309448242,
      "activations/layer11_attention_weight_max": 31.373151779174805,
      "activations/layer11_attention_weight_min": -31.214561462402344,
      "activations/layer12_attention_weight_max": 25.38571548461914,
      "activations/layer12_attention_weight_min": -25.762317657470703,
      "activations/layer13_attention_weight_max": 38.70256805419922,
      "activations/layer13_attention_weight_min": -33.777671813964844,
      "activations/layer14_attention_weight_max": 43.33049011230469,
      "activations/layer14_attention_weight_min": -39.365169525146484,
      "activations/layer15_attention_weight_max": 34.211830139160156,
      "activations/layer15_attention_weight_min": -30.188322067260742,
      "activations/layer16_attention_weight_max": 27.81402015686035,
      "activations/layer16_attention_weight_min": -27.630159378051758,
      "activations/layer17_attention_weight_max": 31.88888931274414,
      "activations/layer17_attention_weight_min": -25.790029525756836,
      "activations/layer18_attention_weight_max": 30.713659286499023,
      "activations/layer18_attention_weight_min": -22.319067001342773,
      "activations/layer19_attention_weight_max": 32.4244499206543,
      "activations/layer19_attention_weight_min": -28.684219360351562,
      "activations/layer1_attention_weight_max": 16.517127990722656,
      "activations/layer1_attention_weight_min": -14.540861129760742,
      "activations/layer20_attention_weight_max": 28.983064651489258,
      "activations/layer20_attention_weight_min": -24.67525863647461,
      "activations/layer21_attention_weight_max": 30.754772186279297,
      "activations/layer21_attention_weight_min": -24.322357177734375,
      "activations/layer22_attention_weight_max": 41.37660217285156,
      "activations/layer22_attention_weight_min": -29.65755844116211,
      "activations/layer23_attention_weight_max": 32.191585540771484,
      "activations/layer23_attention_weight_min": -26.533926010131836,
      "activations/layer2_attention_weight_max": 32.2155647277832,
      "activations/layer2_attention_weight_min": -32.89997100830078,
      "activations/layer3_attention_weight_max": 97.59398651123047,
      "activations/layer3_attention_weight_min": -100.00354766845703,
      "activations/layer4_attention_weight_max": 111.67115783691406,
      "activations/layer4_attention_weight_min": -114.97100067138672,
      "activations/layer5_attention_weight_max": 50.600120544433594,
      "activations/layer5_attention_weight_min": -62.76372528076172,
      "activations/layer6_attention_weight_max": 44.893043518066406,
      "activations/layer6_attention_weight_min": -49.29009246826172,
      "activations/layer7_attention_weight_max": 88.45342254638672,
      "activations/layer7_attention_weight_min": -89.04794311523438,
      "activations/layer8_attention_weight_max": 39.0809211730957,
      "activations/layer8_attention_weight_min": -42.26213836669922,
      "activations/layer9_attention_weight_max": 32.63457489013672,
      "activations/layer9_attention_weight_min": -32.95457077026367,
      "epoch": 11.85,
      "learning_rate": 7.430757575757575e-05,
      "loss": 2.7807,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.5489,
      "eval_samples_per_second": 502.289,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.5489,
      "eval_openwebtext_samples_per_second": 502.289,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.016,
      "eval_wikitext_samples_per_second": 226.192,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_lambada_loss": 2.435546875,
      "eval_lambada_ppl": 11.422063446614425,
      "eval_lambada_runtime": 9.6327,
      "eval_lambada_samples_per_second": 505.467,
      "step": 204000
    },
    {
      "activations/layer0_attention_weight_max": 16.450931549072266,
      "activations/layer0_attention_weight_min": -11.817129135131836,
      "activations/layer10_attention_weight_max": 36.079463958740234,
      "activations/layer10_attention_weight_min": -36.965087890625,
      "activations/layer11_attention_weight_max": 36.683921813964844,
      "activations/layer11_attention_weight_min": -37.37480926513672,
      "activations/layer12_attention_weight_max": 25.82324981689453,
      "activations/layer12_attention_weight_min": -26.355955123901367,
      "activations/layer13_attention_weight_max": 39.746307373046875,
      "activations/layer13_attention_weight_min": -36.363800048828125,
      "activations/layer14_attention_weight_max": 45.13046646118164,
      "activations/layer14_attention_weight_min": -42.55474090576172,
      "activations/layer15_attention_weight_max": 36.773860931396484,
      "activations/layer15_attention_weight_min": -34.7092399597168,
      "activations/layer16_attention_weight_max": 30.555208206176758,
      "activations/layer16_attention_weight_min": -27.43936538696289,
      "activations/layer17_attention_weight_max": 32.9485969543457,
      "activations/layer17_attention_weight_min": -25.562034606933594,
      "activations/layer18_attention_weight_max": 30.636436462402344,
      "activations/layer18_attention_weight_min": -23.89540672302246,
      "activations/layer19_attention_weight_max": 41.903289794921875,
      "activations/layer19_attention_weight_min": -30.890487670898438,
      "activations/layer1_attention_weight_max": 17.682945251464844,
      "activations/layer1_attention_weight_min": -16.86931610107422,
      "activations/layer20_attention_weight_max": 33.00889587402344,
      "activations/layer20_attention_weight_min": -24.99689292907715,
      "activations/layer21_attention_weight_max": 31.908672332763672,
      "activations/layer21_attention_weight_min": -24.546655654907227,
      "activations/layer22_attention_weight_max": 47.90020751953125,
      "activations/layer22_attention_weight_min": -32.135459899902344,
      "activations/layer23_attention_weight_max": 38.07646179199219,
      "activations/layer23_attention_weight_min": -26.925077438354492,
      "activations/layer2_attention_weight_max": 35.773223876953125,
      "activations/layer2_attention_weight_min": -33.902000427246094,
      "activations/layer3_attention_weight_max": 100.60563659667969,
      "activations/layer3_attention_weight_min": -102.10858917236328,
      "activations/layer4_attention_weight_max": 117.88903045654297,
      "activations/layer4_attention_weight_min": -119.17535400390625,
      "activations/layer5_attention_weight_max": 53.567054748535156,
      "activations/layer5_attention_weight_min": -63.738868713378906,
      "activations/layer6_attention_weight_max": 45.206302642822266,
      "activations/layer6_attention_weight_min": -49.811153411865234,
      "activations/layer7_attention_weight_max": 93.2756118774414,
      "activations/layer7_attention_weight_min": -92.23667907714844,
      "activations/layer8_attention_weight_max": 43.5818977355957,
      "activations/layer8_attention_weight_min": -42.64115905761719,
      "activations/layer9_attention_weight_max": 37.463905334472656,
      "activations/layer9_attention_weight_min": -37.08399200439453,
      "epoch": 11.86,
      "learning_rate": 7.428863636363636e-05,
      "loss": 2.7694,
      "step": 204050
    },
    {
      "activations/layer0_attention_weight_max": 15.840545654296875,
      "activations/layer0_attention_weight_min": -12.11749267578125,
      "activations/layer10_attention_weight_max": 32.12325668334961,
      "activations/layer10_attention_weight_min": -30.59163475036621,
      "activations/layer11_attention_weight_max": 31.582387924194336,
      "activations/layer11_attention_weight_min": -28.966039657592773,
      "activations/layer12_attention_weight_max": 24.040027618408203,
      "activations/layer12_attention_weight_min": -25.00745391845703,
      "activations/layer13_attention_weight_max": 36.558101654052734,
      "activations/layer13_attention_weight_min": -34.48783874511719,
      "activations/layer14_attention_weight_max": 42.03331756591797,
      "activations/layer14_attention_weight_min": -36.26067352294922,
      "activations/layer15_attention_weight_max": 35.124900817871094,
      "activations/layer15_attention_weight_min": -33.72335433959961,
      "activations/layer16_attention_weight_max": 32.150638580322266,
      "activations/layer16_attention_weight_min": -28.636594772338867,
      "activations/layer17_attention_weight_max": 35.36962890625,
      "activations/layer17_attention_weight_min": -27.393871307373047,
      "activations/layer18_attention_weight_max": 33.039119720458984,
      "activations/layer18_attention_weight_min": -25.22946548461914,
      "activations/layer19_attention_weight_max": 36.881805419921875,
      "activations/layer19_attention_weight_min": -33.1527214050293,
      "activations/layer1_attention_weight_max": 16.625782012939453,
      "activations/layer1_attention_weight_min": -14.536343574523926,
      "activations/layer20_attention_weight_max": 32.19115447998047,
      "activations/layer20_attention_weight_min": -25.825803756713867,
      "activations/layer21_attention_weight_max": 29.387540817260742,
      "activations/layer21_attention_weight_min": -24.780590057373047,
      "activations/layer22_attention_weight_max": 43.286216735839844,
      "activations/layer22_attention_weight_min": -31.00737190246582,
      "activations/layer23_attention_weight_max": 34.898311614990234,
      "activations/layer23_attention_weight_min": -26.187061309814453,
      "activations/layer2_attention_weight_max": 36.41707992553711,
      "activations/layer2_attention_weight_min": -33.248497009277344,
      "activations/layer3_attention_weight_max": 104.1809310913086,
      "activations/layer3_attention_weight_min": -99.2963638305664,
      "activations/layer4_attention_weight_max": 113.0888900756836,
      "activations/layer4_attention_weight_min": -108.87593078613281,
      "activations/layer5_attention_weight_max": 53.33026885986328,
      "activations/layer5_attention_weight_min": -59.604488372802734,
      "activations/layer6_attention_weight_max": 46.1671257019043,
      "activations/layer6_attention_weight_min": -47.17861557006836,
      "activations/layer7_attention_weight_max": 87.41785430908203,
      "activations/layer7_attention_weight_min": -86.24373626708984,
      "activations/layer8_attention_weight_max": 40.38499450683594,
      "activations/layer8_attention_weight_min": -38.158897399902344,
      "activations/layer9_attention_weight_max": 30.98011589050293,
      "activations/layer9_attention_weight_min": -31.80191993713379,
      "epoch": 11.86,
      "learning_rate": 7.426969696969697e-05,
      "loss": 2.7925,
      "step": 204100
    },
    {
      "activations/layer0_attention_weight_max": 15.232016563415527,
      "activations/layer0_attention_weight_min": -12.38290786743164,
      "activations/layer10_attention_weight_max": 32.804195404052734,
      "activations/layer10_attention_weight_min": -31.69209861755371,
      "activations/layer11_attention_weight_max": 31.351520538330078,
      "activations/layer11_attention_weight_min": -32.32719421386719,
      "activations/layer12_attention_weight_max": 24.375892639160156,
      "activations/layer12_attention_weight_min": -25.186763763427734,
      "activations/layer13_attention_weight_max": 38.70905303955078,
      "activations/layer13_attention_weight_min": -32.775508880615234,
      "activations/layer14_attention_weight_max": 41.772308349609375,
      "activations/layer14_attention_weight_min": -37.59288024902344,
      "activations/layer15_attention_weight_max": 35.802467346191406,
      "activations/layer15_attention_weight_min": -31.92838478088379,
      "activations/layer16_attention_weight_max": 30.352493286132812,
      "activations/layer16_attention_weight_min": -27.612640380859375,
      "activations/layer17_attention_weight_max": 34.44472122192383,
      "activations/layer17_attention_weight_min": -27.045312881469727,
      "activations/layer18_attention_weight_max": 32.153568267822266,
      "activations/layer18_attention_weight_min": -28.185983657836914,
      "activations/layer19_attention_weight_max": 33.174415588378906,
      "activations/layer19_attention_weight_min": -32.261714935302734,
      "activations/layer1_attention_weight_max": 16.887794494628906,
      "activations/layer1_attention_weight_min": -14.530059814453125,
      "activations/layer20_attention_weight_max": 30.15672492980957,
      "activations/layer20_attention_weight_min": -23.873342514038086,
      "activations/layer21_attention_weight_max": 28.76627540588379,
      "activations/layer21_attention_weight_min": -21.821577072143555,
      "activations/layer22_attention_weight_max": 42.72665023803711,
      "activations/layer22_attention_weight_min": -28.641355514526367,
      "activations/layer23_attention_weight_max": 34.245059967041016,
      "activations/layer23_attention_weight_min": -23.38605499267578,
      "activations/layer2_attention_weight_max": 33.86582946777344,
      "activations/layer2_attention_weight_min": -32.21881866455078,
      "activations/layer3_attention_weight_max": 101.35918426513672,
      "activations/layer3_attention_weight_min": -96.60602569580078,
      "activations/layer4_attention_weight_max": 112.33760833740234,
      "activations/layer4_attention_weight_min": -113.91252899169922,
      "activations/layer5_attention_weight_max": 52.86528778076172,
      "activations/layer5_attention_weight_min": -60.50547409057617,
      "activations/layer6_attention_weight_max": 44.7299690246582,
      "activations/layer6_attention_weight_min": -48.31673812866211,
      "activations/layer7_attention_weight_max": 87.34013366699219,
      "activations/layer7_attention_weight_min": -88.26358032226562,
      "activations/layer8_attention_weight_max": 38.233272552490234,
      "activations/layer8_attention_weight_min": -39.064212799072266,
      "activations/layer9_attention_weight_max": 32.20629119873047,
      "activations/layer9_attention_weight_min": -33.60420608520508,
      "epoch": 11.86,
      "learning_rate": 7.425075757575757e-05,
      "loss": 2.7749,
      "step": 204150
    },
    {
      "activations/layer0_attention_weight_max": 15.69632339477539,
      "activations/layer0_attention_weight_min": -11.43162727355957,
      "activations/layer10_attention_weight_max": 35.54941940307617,
      "activations/layer10_attention_weight_min": -37.742828369140625,
      "activations/layer11_attention_weight_max": 35.14751052856445,
      "activations/layer11_attention_weight_min": -35.82931900024414,
      "activations/layer12_attention_weight_max": 27.56834602355957,
      "activations/layer12_attention_weight_min": -27.843448638916016,
      "activations/layer13_attention_weight_max": 43.648765563964844,
      "activations/layer13_attention_weight_min": -40.870704650878906,
      "activations/layer14_attention_weight_max": 43.40961456298828,
      "activations/layer14_attention_weight_min": -39.46154022216797,
      "activations/layer15_attention_weight_max": 41.29352569580078,
      "activations/layer15_attention_weight_min": -36.60275650024414,
      "activations/layer16_attention_weight_max": 31.467166900634766,
      "activations/layer16_attention_weight_min": -28.361181259155273,
      "activations/layer17_attention_weight_max": 35.56265640258789,
      "activations/layer17_attention_weight_min": -27.241979598999023,
      "activations/layer18_attention_weight_max": 33.792823791503906,
      "activations/layer18_attention_weight_min": -24.65023422241211,
      "activations/layer19_attention_weight_max": 38.68216323852539,
      "activations/layer19_attention_weight_min": -33.05392074584961,
      "activations/layer1_attention_weight_max": 16.881948471069336,
      "activations/layer1_attention_weight_min": -13.846580505371094,
      "activations/layer20_attention_weight_max": 32.129241943359375,
      "activations/layer20_attention_weight_min": -27.5389461517334,
      "activations/layer21_attention_weight_max": 33.07309341430664,
      "activations/layer21_attention_weight_min": -26.01865005493164,
      "activations/layer22_attention_weight_max": 45.71298599243164,
      "activations/layer22_attention_weight_min": -35.48811340332031,
      "activations/layer23_attention_weight_max": 38.52922821044922,
      "activations/layer23_attention_weight_min": -27.313169479370117,
      "activations/layer2_attention_weight_max": 33.07288360595703,
      "activations/layer2_attention_weight_min": -33.2528190612793,
      "activations/layer3_attention_weight_max": 96.15265655517578,
      "activations/layer3_attention_weight_min": -95.38848114013672,
      "activations/layer4_attention_weight_max": 112.4631576538086,
      "activations/layer4_attention_weight_min": -115.54393768310547,
      "activations/layer5_attention_weight_max": 53.41463851928711,
      "activations/layer5_attention_weight_min": -60.42103576660156,
      "activations/layer6_attention_weight_max": 45.66319274902344,
      "activations/layer6_attention_weight_min": -47.98979568481445,
      "activations/layer7_attention_weight_max": 95.64762115478516,
      "activations/layer7_attention_weight_min": -97.9520263671875,
      "activations/layer8_attention_weight_max": 42.54158401489258,
      "activations/layer8_attention_weight_min": -46.14651107788086,
      "activations/layer9_attention_weight_max": 35.75098419189453,
      "activations/layer9_attention_weight_min": -36.001033782958984,
      "epoch": 11.87,
      "learning_rate": 7.423181818181817e-05,
      "loss": 2.7646,
      "step": 204200
    },
    {
      "activations/layer0_attention_weight_max": 15.862159729003906,
      "activations/layer0_attention_weight_min": -12.58436107635498,
      "activations/layer10_attention_weight_max": 31.566301345825195,
      "activations/layer10_attention_weight_min": -32.585567474365234,
      "activations/layer11_attention_weight_max": 31.94464683532715,
      "activations/layer11_attention_weight_min": -31.53034019470215,
      "activations/layer12_attention_weight_max": 24.011432647705078,
      "activations/layer12_attention_weight_min": -25.56908416748047,
      "activations/layer13_attention_weight_max": 37.34926223754883,
      "activations/layer13_attention_weight_min": -37.14043045043945,
      "activations/layer14_attention_weight_max": 41.96333312988281,
      "activations/layer14_attention_weight_min": -38.263267517089844,
      "activations/layer15_attention_weight_max": 35.794654846191406,
      "activations/layer15_attention_weight_min": -32.34798049926758,
      "activations/layer16_attention_weight_max": 33.26332092285156,
      "activations/layer16_attention_weight_min": -29.235633850097656,
      "activations/layer17_attention_weight_max": 30.954341888427734,
      "activations/layer17_attention_weight_min": -26.73598861694336,
      "activations/layer18_attention_weight_max": 29.68679428100586,
      "activations/layer18_attention_weight_min": -24.456029891967773,
      "activations/layer19_attention_weight_max": 32.20338821411133,
      "activations/layer19_attention_weight_min": -31.385456085205078,
      "activations/layer1_attention_weight_max": 16.401140213012695,
      "activations/layer1_attention_weight_min": -15.064373016357422,
      "activations/layer20_attention_weight_max": 27.490249633789062,
      "activations/layer20_attention_weight_min": -25.86517333984375,
      "activations/layer21_attention_weight_max": 26.75876808166504,
      "activations/layer21_attention_weight_min": -24.27245330810547,
      "activations/layer22_attention_weight_max": 38.97453689575195,
      "activations/layer22_attention_weight_min": -29.11693572998047,
      "activations/layer23_attention_weight_max": 33.39130401611328,
      "activations/layer23_attention_weight_min": -24.772640228271484,
      "activations/layer2_attention_weight_max": 33.95555114746094,
      "activations/layer2_attention_weight_min": -33.64278793334961,
      "activations/layer3_attention_weight_max": 98.4345474243164,
      "activations/layer3_attention_weight_min": -98.80313873291016,
      "activations/layer4_attention_weight_max": 115.04280853271484,
      "activations/layer4_attention_weight_min": -116.88714599609375,
      "activations/layer5_attention_weight_max": 52.26460266113281,
      "activations/layer5_attention_weight_min": -63.95450973510742,
      "activations/layer6_attention_weight_max": 47.16386413574219,
      "activations/layer6_attention_weight_min": -47.20191192626953,
      "activations/layer7_attention_weight_max": 96.13775634765625,
      "activations/layer7_attention_weight_min": -95.34523010253906,
      "activations/layer8_attention_weight_max": 40.18084716796875,
      "activations/layer8_attention_weight_min": -44.62093734741211,
      "activations/layer9_attention_weight_max": 35.07965850830078,
      "activations/layer9_attention_weight_min": -35.05549240112305,
      "epoch": 11.87,
      "learning_rate": 7.421287878787878e-05,
      "loss": 2.7862,
      "step": 204250
    },
    {
      "activations/layer0_attention_weight_max": 15.878275871276855,
      "activations/layer0_attention_weight_min": -12.108255386352539,
      "activations/layer10_attention_weight_max": 35.53187942504883,
      "activations/layer10_attention_weight_min": -33.41350555419922,
      "activations/layer11_attention_weight_max": 37.83253860473633,
      "activations/layer11_attention_weight_min": -32.718021392822266,
      "activations/layer12_attention_weight_max": 25.591089248657227,
      "activations/layer12_attention_weight_min": -26.506160736083984,
      "activations/layer13_attention_weight_max": 39.11668014526367,
      "activations/layer13_attention_weight_min": -36.27367401123047,
      "activations/layer14_attention_weight_max": 46.258750915527344,
      "activations/layer14_attention_weight_min": -39.68677520751953,
      "activations/layer15_attention_weight_max": 41.671207427978516,
      "activations/layer15_attention_weight_min": -34.24216842651367,
      "activations/layer16_attention_weight_max": 40.09623718261719,
      "activations/layer16_attention_weight_min": -28.06562042236328,
      "activations/layer17_attention_weight_max": 37.85239791870117,
      "activations/layer17_attention_weight_min": -29.054691314697266,
      "activations/layer18_attention_weight_max": 34.71013259887695,
      "activations/layer18_attention_weight_min": -24.599262237548828,
      "activations/layer19_attention_weight_max": 39.882991790771484,
      "activations/layer19_attention_weight_min": -31.46055030822754,
      "activations/layer1_attention_weight_max": 17.703107833862305,
      "activations/layer1_attention_weight_min": -15.71202278137207,
      "activations/layer20_attention_weight_max": 36.1783561706543,
      "activations/layer20_attention_weight_min": -23.52859115600586,
      "activations/layer21_attention_weight_max": 31.03068733215332,
      "activations/layer21_attention_weight_min": -22.979032516479492,
      "activations/layer22_attention_weight_max": 46.26274871826172,
      "activations/layer22_attention_weight_min": -30.645214080810547,
      "activations/layer23_attention_weight_max": 34.56108856201172,
      "activations/layer23_attention_weight_min": -27.391883850097656,
      "activations/layer2_attention_weight_max": 34.12981414794922,
      "activations/layer2_attention_weight_min": -33.345977783203125,
      "activations/layer3_attention_weight_max": 98.3388442993164,
      "activations/layer3_attention_weight_min": -98.78870391845703,
      "activations/layer4_attention_weight_max": 114.8199234008789,
      "activations/layer4_attention_weight_min": -113.82494354248047,
      "activations/layer5_attention_weight_max": 50.91728973388672,
      "activations/layer5_attention_weight_min": -58.497283935546875,
      "activations/layer6_attention_weight_max": 44.33195114135742,
      "activations/layer6_attention_weight_min": -48.51816177368164,
      "activations/layer7_attention_weight_max": 90.65847778320312,
      "activations/layer7_attention_weight_min": -91.79158782958984,
      "activations/layer8_attention_weight_max": 42.08605194091797,
      "activations/layer8_attention_weight_min": -40.55537033081055,
      "activations/layer9_attention_weight_max": 33.742462158203125,
      "activations/layer9_attention_weight_min": -33.32479476928711,
      "epoch": 11.87,
      "learning_rate": 7.419393939393939e-05,
      "loss": 2.7834,
      "step": 204300
    },
    {
      "activations/layer0_attention_weight_max": 15.893884658813477,
      "activations/layer0_attention_weight_min": -11.996708869934082,
      "activations/layer10_attention_weight_max": 29.81659507751465,
      "activations/layer10_attention_weight_min": -32.004058837890625,
      "activations/layer11_attention_weight_max": 29.084598541259766,
      "activations/layer11_attention_weight_min": -31.324575424194336,
      "activations/layer12_attention_weight_max": 25.719552993774414,
      "activations/layer12_attention_weight_min": -25.155975341796875,
      "activations/layer13_attention_weight_max": 37.92686462402344,
      "activations/layer13_attention_weight_min": -34.319435119628906,
      "activations/layer14_attention_weight_max": 44.7159538269043,
      "activations/layer14_attention_weight_min": -37.43297576904297,
      "activations/layer15_attention_weight_max": 33.662330627441406,
      "activations/layer15_attention_weight_min": -34.05479049682617,
      "activations/layer16_attention_weight_max": 29.162443161010742,
      "activations/layer16_attention_weight_min": -28.35965919494629,
      "activations/layer17_attention_weight_max": 30.58897590637207,
      "activations/layer17_attention_weight_min": -26.573169708251953,
      "activations/layer18_attention_weight_max": 28.999643325805664,
      "activations/layer18_attention_weight_min": -24.214500427246094,
      "activations/layer19_attention_weight_max": 31.513412475585938,
      "activations/layer19_attention_weight_min": -31.747228622436523,
      "activations/layer1_attention_weight_max": 17.43912696838379,
      "activations/layer1_attention_weight_min": -15.332806587219238,
      "activations/layer20_attention_weight_max": 30.403169631958008,
      "activations/layer20_attention_weight_min": -25.09963035583496,
      "activations/layer21_attention_weight_max": 26.909177780151367,
      "activations/layer21_attention_weight_min": -24.275760650634766,
      "activations/layer22_attention_weight_max": 41.34517288208008,
      "activations/layer22_attention_weight_min": -31.14405059814453,
      "activations/layer23_attention_weight_max": 32.28968048095703,
      "activations/layer23_attention_weight_min": -23.8992977142334,
      "activations/layer2_attention_weight_max": 32.13579177856445,
      "activations/layer2_attention_weight_min": -31.758657455444336,
      "activations/layer3_attention_weight_max": 95.66416931152344,
      "activations/layer3_attention_weight_min": -97.82394409179688,
      "activations/layer4_attention_weight_max": 112.15312957763672,
      "activations/layer4_attention_weight_min": -113.9378662109375,
      "activations/layer5_attention_weight_max": 50.97920227050781,
      "activations/layer5_attention_weight_min": -61.886775970458984,
      "activations/layer6_attention_weight_max": 44.215248107910156,
      "activations/layer6_attention_weight_min": -46.79839324951172,
      "activations/layer7_attention_weight_max": 86.87541198730469,
      "activations/layer7_attention_weight_min": -89.9161148071289,
      "activations/layer8_attention_weight_max": 41.069908142089844,
      "activations/layer8_attention_weight_min": -45.064334869384766,
      "activations/layer9_attention_weight_max": 31.34250831604004,
      "activations/layer9_attention_weight_min": -34.544769287109375,
      "epoch": 11.87,
      "learning_rate": 7.417499999999999e-05,
      "loss": 2.7756,
      "step": 204350
    },
    {
      "activations/layer0_attention_weight_max": 16.756919860839844,
      "activations/layer0_attention_weight_min": -12.286006927490234,
      "activations/layer10_attention_weight_max": 35.159027099609375,
      "activations/layer10_attention_weight_min": -30.835575103759766,
      "activations/layer11_attention_weight_max": 32.24170684814453,
      "activations/layer11_attention_weight_min": -30.295734405517578,
      "activations/layer12_attention_weight_max": 22.397672653198242,
      "activations/layer12_attention_weight_min": -24.534706115722656,
      "activations/layer13_attention_weight_max": 34.410457611083984,
      "activations/layer13_attention_weight_min": -33.306053161621094,
      "activations/layer14_attention_weight_max": 39.337547302246094,
      "activations/layer14_attention_weight_min": -36.52781677246094,
      "activations/layer15_attention_weight_max": 33.96611785888672,
      "activations/layer15_attention_weight_min": -32.055145263671875,
      "activations/layer16_attention_weight_max": 30.522859573364258,
      "activations/layer16_attention_weight_min": -28.044010162353516,
      "activations/layer17_attention_weight_max": 29.64116859436035,
      "activations/layer17_attention_weight_min": -25.796829223632812,
      "activations/layer18_attention_weight_max": 29.815231323242188,
      "activations/layer18_attention_weight_min": -23.465946197509766,
      "activations/layer19_attention_weight_max": 31.279943466186523,
      "activations/layer19_attention_weight_min": -31.737913131713867,
      "activations/layer1_attention_weight_max": 17.072463989257812,
      "activations/layer1_attention_weight_min": -14.889363288879395,
      "activations/layer20_attention_weight_max": 27.25261878967285,
      "activations/layer20_attention_weight_min": -24.778270721435547,
      "activations/layer21_attention_weight_max": 26.83034896850586,
      "activations/layer21_attention_weight_min": -24.935319900512695,
      "activations/layer22_attention_weight_max": 36.893436431884766,
      "activations/layer22_attention_weight_min": -30.218990325927734,
      "activations/layer23_attention_weight_max": 28.702747344970703,
      "activations/layer23_attention_weight_min": -24.875883102416992,
      "activations/layer2_attention_weight_max": 34.33325958251953,
      "activations/layer2_attention_weight_min": -32.60607147216797,
      "activations/layer3_attention_weight_max": 99.75398254394531,
      "activations/layer3_attention_weight_min": -96.30252838134766,
      "activations/layer4_attention_weight_max": 112.1030502319336,
      "activations/layer4_attention_weight_min": -113.4519271850586,
      "activations/layer5_attention_weight_max": 51.7696533203125,
      "activations/layer5_attention_weight_min": -62.178009033203125,
      "activations/layer6_attention_weight_max": 46.20903015136719,
      "activations/layer6_attention_weight_min": -47.5875129699707,
      "activations/layer7_attention_weight_max": 90.14042663574219,
      "activations/layer7_attention_weight_min": -90.5031967163086,
      "activations/layer8_attention_weight_max": 41.066497802734375,
      "activations/layer8_attention_weight_min": -41.03547286987305,
      "activations/layer9_attention_weight_max": 32.12113952636719,
      "activations/layer9_attention_weight_min": -32.638675689697266,
      "epoch": 11.88,
      "learning_rate": 7.41560606060606e-05,
      "loss": 2.7762,
      "step": 204400
    },
    {
      "activations/layer0_attention_weight_max": 15.905532836914062,
      "activations/layer0_attention_weight_min": -12.781723022460938,
      "activations/layer10_attention_weight_max": 34.91943359375,
      "activations/layer10_attention_weight_min": -38.15513229370117,
      "activations/layer11_attention_weight_max": 34.683162689208984,
      "activations/layer11_attention_weight_min": -35.41755676269531,
      "activations/layer12_attention_weight_max": 26.833627700805664,
      "activations/layer12_attention_weight_min": -27.47345733642578,
      "activations/layer13_attention_weight_max": 37.953731536865234,
      "activations/layer13_attention_weight_min": -34.60143280029297,
      "activations/layer14_attention_weight_max": 46.83828353881836,
      "activations/layer14_attention_weight_min": -41.907169342041016,
      "activations/layer15_attention_weight_max": 36.74879455566406,
      "activations/layer15_attention_weight_min": -35.78651428222656,
      "activations/layer16_attention_weight_max": 30.955785751342773,
      "activations/layer16_attention_weight_min": -29.674070358276367,
      "activations/layer17_attention_weight_max": 31.97121810913086,
      "activations/layer17_attention_weight_min": -29.07242202758789,
      "activations/layer18_attention_weight_max": 33.278961181640625,
      "activations/layer18_attention_weight_min": -24.89242172241211,
      "activations/layer19_attention_weight_max": 39.239418029785156,
      "activations/layer19_attention_weight_min": -32.66769790649414,
      "activations/layer1_attention_weight_max": 17.518007278442383,
      "activations/layer1_attention_weight_min": -15.067353248596191,
      "activations/layer20_attention_weight_max": 33.8881721496582,
      "activations/layer20_attention_weight_min": -25.945728302001953,
      "activations/layer21_attention_weight_max": 32.48798370361328,
      "activations/layer21_attention_weight_min": -24.99954605102539,
      "activations/layer22_attention_weight_max": 46.6629524230957,
      "activations/layer22_attention_weight_min": -30.95620346069336,
      "activations/layer23_attention_weight_max": 37.66508483886719,
      "activations/layer23_attention_weight_min": -27.031776428222656,
      "activations/layer2_attention_weight_max": 33.08429718017578,
      "activations/layer2_attention_weight_min": -32.51301193237305,
      "activations/layer3_attention_weight_max": 95.03801727294922,
      "activations/layer3_attention_weight_min": -95.45938110351562,
      "activations/layer4_attention_weight_max": 107.56603240966797,
      "activations/layer4_attention_weight_min": -107.96480560302734,
      "activations/layer5_attention_weight_max": 51.14329528808594,
      "activations/layer5_attention_weight_min": -59.6676025390625,
      "activations/layer6_attention_weight_max": 44.48051071166992,
      "activations/layer6_attention_weight_min": -45.87458801269531,
      "activations/layer7_attention_weight_max": 95.20904541015625,
      "activations/layer7_attention_weight_min": -97.76146697998047,
      "activations/layer8_attention_weight_max": 44.01768493652344,
      "activations/layer8_attention_weight_min": -46.673194885253906,
      "activations/layer9_attention_weight_max": 36.688961029052734,
      "activations/layer9_attention_weight_min": -41.17985153198242,
      "epoch": 11.88,
      "learning_rate": 7.41371212121212e-05,
      "loss": 2.7797,
      "step": 204450
    },
    {
      "activations/layer0_attention_weight_max": 16.068557739257812,
      "activations/layer0_attention_weight_min": -12.199257850646973,
      "activations/layer10_attention_weight_max": 30.815471649169922,
      "activations/layer10_attention_weight_min": -31.518369674682617,
      "activations/layer11_attention_weight_max": 32.1099739074707,
      "activations/layer11_attention_weight_min": -32.27347183227539,
      "activations/layer12_attention_weight_max": 23.868497848510742,
      "activations/layer12_attention_weight_min": -24.646053314208984,
      "activations/layer13_attention_weight_max": 36.89109420776367,
      "activations/layer13_attention_weight_min": -35.25480651855469,
      "activations/layer14_attention_weight_max": 39.075801849365234,
      "activations/layer14_attention_weight_min": -39.582794189453125,
      "activations/layer15_attention_weight_max": 35.02208709716797,
      "activations/layer15_attention_weight_min": -33.298667907714844,
      "activations/layer16_attention_weight_max": 29.881256103515625,
      "activations/layer16_attention_weight_min": -27.39750099182129,
      "activations/layer17_attention_weight_max": 34.213592529296875,
      "activations/layer17_attention_weight_min": -25.829816818237305,
      "activations/layer18_attention_weight_max": 31.537338256835938,
      "activations/layer18_attention_weight_min": -23.770061492919922,
      "activations/layer19_attention_weight_max": 38.01272964477539,
      "activations/layer19_attention_weight_min": -29.80840492248535,
      "activations/layer1_attention_weight_max": 16.003870010375977,
      "activations/layer1_attention_weight_min": -13.654645919799805,
      "activations/layer20_attention_weight_max": 31.224199295043945,
      "activations/layer20_attention_weight_min": -24.186521530151367,
      "activations/layer21_attention_weight_max": 33.84663772583008,
      "activations/layer21_attention_weight_min": -23.760507583618164,
      "activations/layer22_attention_weight_max": 47.4240608215332,
      "activations/layer22_attention_weight_min": -30.296480178833008,
      "activations/layer23_attention_weight_max": 36.10932159423828,
      "activations/layer23_attention_weight_min": -24.858062744140625,
      "activations/layer2_attention_weight_max": 33.438087463378906,
      "activations/layer2_attention_weight_min": -33.231170654296875,
      "activations/layer3_attention_weight_max": 99.12446594238281,
      "activations/layer3_attention_weight_min": -101.84187316894531,
      "activations/layer4_attention_weight_max": 113.48026275634766,
      "activations/layer4_attention_weight_min": -111.16826629638672,
      "activations/layer5_attention_weight_max": 52.82658386230469,
      "activations/layer5_attention_weight_min": -62.77996826171875,
      "activations/layer6_attention_weight_max": 44.13192367553711,
      "activations/layer6_attention_weight_min": -48.399227142333984,
      "activations/layer7_attention_weight_max": 94.18653869628906,
      "activations/layer7_attention_weight_min": -93.18119812011719,
      "activations/layer8_attention_weight_max": 38.32868194580078,
      "activations/layer8_attention_weight_min": -40.394805908203125,
      "activations/layer9_attention_weight_max": 30.401920318603516,
      "activations/layer9_attention_weight_min": -32.69831466674805,
      "epoch": 11.88,
      "learning_rate": 7.411818181818182e-05,
      "loss": 2.7762,
      "step": 204500
    },
    {
      "activations/layer0_attention_weight_max": 15.541414260864258,
      "activations/layer0_attention_weight_min": -11.919485092163086,
      "activations/layer10_attention_weight_max": 32.37435531616211,
      "activations/layer10_attention_weight_min": -32.904052734375,
      "activations/layer11_attention_weight_max": 32.38258361816406,
      "activations/layer11_attention_weight_min": -33.44215774536133,
      "activations/layer12_attention_weight_max": 24.98125648498535,
      "activations/layer12_attention_weight_min": -25.898759841918945,
      "activations/layer13_attention_weight_max": 41.93817901611328,
      "activations/layer13_attention_weight_min": -34.14970397949219,
      "activations/layer14_attention_weight_max": 43.271339416503906,
      "activations/layer14_attention_weight_min": -38.53075408935547,
      "activations/layer15_attention_weight_max": 34.75044250488281,
      "activations/layer15_attention_weight_min": -32.3541259765625,
      "activations/layer16_attention_weight_max": 31.000926971435547,
      "activations/layer16_attention_weight_min": -26.66765022277832,
      "activations/layer17_attention_weight_max": 30.773441314697266,
      "activations/layer17_attention_weight_min": -26.6961612701416,
      "activations/layer18_attention_weight_max": 32.023170471191406,
      "activations/layer18_attention_weight_min": -24.763729095458984,
      "activations/layer19_attention_weight_max": 34.03019332885742,
      "activations/layer19_attention_weight_min": -33.032405853271484,
      "activations/layer1_attention_weight_max": 16.356403350830078,
      "activations/layer1_attention_weight_min": -15.738370895385742,
      "activations/layer20_attention_weight_max": 34.4778938293457,
      "activations/layer20_attention_weight_min": -27.5791072845459,
      "activations/layer21_attention_weight_max": 33.9048957824707,
      "activations/layer21_attention_weight_min": -25.55575942993164,
      "activations/layer22_attention_weight_max": 46.9315071105957,
      "activations/layer22_attention_weight_min": -33.29620361328125,
      "activations/layer23_attention_weight_max": 36.382843017578125,
      "activations/layer23_attention_weight_min": -24.376712799072266,
      "activations/layer2_attention_weight_max": 33.924320220947266,
      "activations/layer2_attention_weight_min": -33.90207290649414,
      "activations/layer3_attention_weight_max": 102.56990814208984,
      "activations/layer3_attention_weight_min": -98.87477111816406,
      "activations/layer4_attention_weight_max": 115.79727935791016,
      "activations/layer4_attention_weight_min": -119.92351531982422,
      "activations/layer5_attention_weight_max": 51.964229583740234,
      "activations/layer5_attention_weight_min": -59.87236022949219,
      "activations/layer6_attention_weight_max": 42.42845153808594,
      "activations/layer6_attention_weight_min": -47.737998962402344,
      "activations/layer7_attention_weight_max": 86.72122955322266,
      "activations/layer7_attention_weight_min": -92.56755065917969,
      "activations/layer8_attention_weight_max": 38.07208251953125,
      "activations/layer8_attention_weight_min": -41.3804817199707,
      "activations/layer9_attention_weight_max": 31.920705795288086,
      "activations/layer9_attention_weight_min": -34.15919494628906,
      "epoch": 11.89,
      "learning_rate": 7.409924242424242e-05,
      "loss": 2.7729,
      "step": 204550
    },
    {
      "activations/layer0_attention_weight_max": 16.536266326904297,
      "activations/layer0_attention_weight_min": -11.620372772216797,
      "activations/layer10_attention_weight_max": 30.603116989135742,
      "activations/layer10_attention_weight_min": -31.795286178588867,
      "activations/layer11_attention_weight_max": 34.27391815185547,
      "activations/layer11_attention_weight_min": -33.197322845458984,
      "activations/layer12_attention_weight_max": 25.042802810668945,
      "activations/layer12_attention_weight_min": -24.325101852416992,
      "activations/layer13_attention_weight_max": 39.8538932800293,
      "activations/layer13_attention_weight_min": -32.818565368652344,
      "activations/layer14_attention_weight_max": 43.104190826416016,
      "activations/layer14_attention_weight_min": -36.414852142333984,
      "activations/layer15_attention_weight_max": 37.564125061035156,
      "activations/layer15_attention_weight_min": -31.788414001464844,
      "activations/layer16_attention_weight_max": 32.75299835205078,
      "activations/layer16_attention_weight_min": -27.01811408996582,
      "activations/layer17_attention_weight_max": 31.274993896484375,
      "activations/layer17_attention_weight_min": -25.325586318969727,
      "activations/layer18_attention_weight_max": 33.61617660522461,
      "activations/layer18_attention_weight_min": -24.305803298950195,
      "activations/layer19_attention_weight_max": 35.70502471923828,
      "activations/layer19_attention_weight_min": -30.30768394470215,
      "activations/layer1_attention_weight_max": 17.088193893432617,
      "activations/layer1_attention_weight_min": -14.559829711914062,
      "activations/layer20_attention_weight_max": 31.35698127746582,
      "activations/layer20_attention_weight_min": -24.25832176208496,
      "activations/layer21_attention_weight_max": 30.904333114624023,
      "activations/layer21_attention_weight_min": -23.6509952545166,
      "activations/layer22_attention_weight_max": 49.63666534423828,
      "activations/layer22_attention_weight_min": -28.89737319946289,
      "activations/layer23_attention_weight_max": 37.53145217895508,
      "activations/layer23_attention_weight_min": -24.01439666748047,
      "activations/layer2_attention_weight_max": 34.01231002807617,
      "activations/layer2_attention_weight_min": -32.69218444824219,
      "activations/layer3_attention_weight_max": 98.7271499633789,
      "activations/layer3_attention_weight_min": -101.67863464355469,
      "activations/layer4_attention_weight_max": 114.71186828613281,
      "activations/layer4_attention_weight_min": -113.45870208740234,
      "activations/layer5_attention_weight_max": 54.34515380859375,
      "activations/layer5_attention_weight_min": -64.61090850830078,
      "activations/layer6_attention_weight_max": 42.16588592529297,
      "activations/layer6_attention_weight_min": -46.7293815612793,
      "activations/layer7_attention_weight_max": 87.1338882446289,
      "activations/layer7_attention_weight_min": -93.4186782836914,
      "activations/layer8_attention_weight_max": 37.42488098144531,
      "activations/layer8_attention_weight_min": -39.59293746948242,
      "activations/layer9_attention_weight_max": 32.394256591796875,
      "activations/layer9_attention_weight_min": -33.12696838378906,
      "epoch": 11.89,
      "learning_rate": 7.408030303030302e-05,
      "loss": 2.7797,
      "step": 204600
    },
    {
      "activations/layer0_attention_weight_max": 15.527143478393555,
      "activations/layer0_attention_weight_min": -12.531061172485352,
      "activations/layer10_attention_weight_max": 31.582244873046875,
      "activations/layer10_attention_weight_min": -31.270883560180664,
      "activations/layer11_attention_weight_max": 33.156890869140625,
      "activations/layer11_attention_weight_min": -32.41204833984375,
      "activations/layer12_attention_weight_max": 27.95052719116211,
      "activations/layer12_attention_weight_min": -25.20421600341797,
      "activations/layer13_attention_weight_max": 41.410491943359375,
      "activations/layer13_attention_weight_min": -35.177947998046875,
      "activations/layer14_attention_weight_max": 48.95193099975586,
      "activations/layer14_attention_weight_min": -38.286705017089844,
      "activations/layer15_attention_weight_max": 40.66035079956055,
      "activations/layer15_attention_weight_min": -32.56597900390625,
      "activations/layer16_attention_weight_max": 37.30035400390625,
      "activations/layer16_attention_weight_min": -27.013139724731445,
      "activations/layer17_attention_weight_max": 35.8348388671875,
      "activations/layer17_attention_weight_min": -25.99637794494629,
      "activations/layer18_attention_weight_max": 33.11903381347656,
      "activations/layer18_attention_weight_min": -22.517436981201172,
      "activations/layer19_attention_weight_max": 44.918479919433594,
      "activations/layer19_attention_weight_min": -31.26949119567871,
      "activations/layer1_attention_weight_max": 17.060848236083984,
      "activations/layer1_attention_weight_min": -14.073491096496582,
      "activations/layer20_attention_weight_max": 38.45843505859375,
      "activations/layer20_attention_weight_min": -23.790855407714844,
      "activations/layer21_attention_weight_max": 36.91034698486328,
      "activations/layer21_attention_weight_min": -22.963180541992188,
      "activations/layer22_attention_weight_max": 52.2427864074707,
      "activations/layer22_attention_weight_min": -27.63547134399414,
      "activations/layer23_attention_weight_max": 41.65125274658203,
      "activations/layer23_attention_weight_min": -25.58858871459961,
      "activations/layer2_attention_weight_max": 34.50199508666992,
      "activations/layer2_attention_weight_min": -33.38627243041992,
      "activations/layer3_attention_weight_max": 96.0433120727539,
      "activations/layer3_attention_weight_min": -94.91173553466797,
      "activations/layer4_attention_weight_max": 113.055419921875,
      "activations/layer4_attention_weight_min": -109.15106201171875,
      "activations/layer5_attention_weight_max": 51.318511962890625,
      "activations/layer5_attention_weight_min": -58.41749572753906,
      "activations/layer6_attention_weight_max": 43.738643646240234,
      "activations/layer6_attention_weight_min": -47.3646125793457,
      "activations/layer7_attention_weight_max": 86.771240234375,
      "activations/layer7_attention_weight_min": -88.19325256347656,
      "activations/layer8_attention_weight_max": 39.1985969543457,
      "activations/layer8_attention_weight_min": -38.21665954589844,
      "activations/layer9_attention_weight_max": 30.83318519592285,
      "activations/layer9_attention_weight_min": -30.516738891601562,
      "epoch": 11.89,
      "learning_rate": 7.406174242424243e-05,
      "loss": 2.7795,
      "step": 204650
    },
    {
      "activations/layer0_attention_weight_max": 15.96450424194336,
      "activations/layer0_attention_weight_min": -12.207154273986816,
      "activations/layer10_attention_weight_max": 32.4133415222168,
      "activations/layer10_attention_weight_min": -31.741727828979492,
      "activations/layer11_attention_weight_max": 31.814388275146484,
      "activations/layer11_attention_weight_min": -31.105304718017578,
      "activations/layer12_attention_weight_max": 24.898818969726562,
      "activations/layer12_attention_weight_min": -24.281476974487305,
      "activations/layer13_attention_weight_max": 35.64059066772461,
      "activations/layer13_attention_weight_min": -32.853492736816406,
      "activations/layer14_attention_weight_max": 41.7827262878418,
      "activations/layer14_attention_weight_min": -35.89508819580078,
      "activations/layer15_attention_weight_max": 34.42422866821289,
      "activations/layer15_attention_weight_min": -31.661352157592773,
      "activations/layer16_attention_weight_max": 29.853919982910156,
      "activations/layer16_attention_weight_min": -27.26072120666504,
      "activations/layer17_attention_weight_max": 31.634782791137695,
      "activations/layer17_attention_weight_min": -25.24374771118164,
      "activations/layer18_attention_weight_max": 30.601669311523438,
      "activations/layer18_attention_weight_min": -24.40777587890625,
      "activations/layer19_attention_weight_max": 32.961177825927734,
      "activations/layer19_attention_weight_min": -32.539207458496094,
      "activations/layer1_attention_weight_max": 15.93706226348877,
      "activations/layer1_attention_weight_min": -14.8594331741333,
      "activations/layer20_attention_weight_max": 27.745983123779297,
      "activations/layer20_attention_weight_min": -24.879749298095703,
      "activations/layer21_attention_weight_max": 27.156845092773438,
      "activations/layer21_attention_weight_min": -25.10953712463379,
      "activations/layer22_attention_weight_max": 43.05858612060547,
      "activations/layer22_attention_weight_min": -30.15907859802246,
      "activations/layer23_attention_weight_max": 33.11762237548828,
      "activations/layer23_attention_weight_min": -27.600248336791992,
      "activations/layer2_attention_weight_max": 33.15569305419922,
      "activations/layer2_attention_weight_min": -33.692665100097656,
      "activations/layer3_attention_weight_max": 97.1737289428711,
      "activations/layer3_attention_weight_min": -98.9605712890625,
      "activations/layer4_attention_weight_max": 113.58564758300781,
      "activations/layer4_attention_weight_min": -112.80965423583984,
      "activations/layer5_attention_weight_max": 51.44538879394531,
      "activations/layer5_attention_weight_min": -62.535240173339844,
      "activations/layer6_attention_weight_max": 42.654563903808594,
      "activations/layer6_attention_weight_min": -48.389339447021484,
      "activations/layer7_attention_weight_max": 87.1842269897461,
      "activations/layer7_attention_weight_min": -90.63692474365234,
      "activations/layer8_attention_weight_max": 41.05415344238281,
      "activations/layer8_attention_weight_min": -39.975013732910156,
      "activations/layer9_attention_weight_max": 31.594039916992188,
      "activations/layer9_attention_weight_min": -32.36659240722656,
      "epoch": 11.89,
      "learning_rate": 7.404280303030303e-05,
      "loss": 2.7914,
      "step": 204700
    },
    {
      "activations/layer0_attention_weight_max": 16.423484802246094,
      "activations/layer0_attention_weight_min": -12.33676815032959,
      "activations/layer10_attention_weight_max": 34.26459884643555,
      "activations/layer10_attention_weight_min": -33.92702102661133,
      "activations/layer11_attention_weight_max": 34.43098068237305,
      "activations/layer11_attention_weight_min": -32.915061950683594,
      "activations/layer12_attention_weight_max": 33.9949951171875,
      "activations/layer12_attention_weight_min": -25.55443000793457,
      "activations/layer13_attention_weight_max": 39.461177825927734,
      "activations/layer13_attention_weight_min": -34.937522888183594,
      "activations/layer14_attention_weight_max": 51.79437255859375,
      "activations/layer14_attention_weight_min": -42.63825225830078,
      "activations/layer15_attention_weight_max": 39.581905364990234,
      "activations/layer15_attention_weight_min": -36.22039031982422,
      "activations/layer16_attention_weight_max": 31.907621383666992,
      "activations/layer16_attention_weight_min": -28.934593200683594,
      "activations/layer17_attention_weight_max": 34.77082824707031,
      "activations/layer17_attention_weight_min": -27.321136474609375,
      "activations/layer18_attention_weight_max": 32.891963958740234,
      "activations/layer18_attention_weight_min": -22.66192626953125,
      "activations/layer19_attention_weight_max": 36.94490051269531,
      "activations/layer19_attention_weight_min": -31.186935424804688,
      "activations/layer1_attention_weight_max": 17.4725341796875,
      "activations/layer1_attention_weight_min": -15.05920696258545,
      "activations/layer20_attention_weight_max": 32.47604751586914,
      "activations/layer20_attention_weight_min": -25.035531997680664,
      "activations/layer21_attention_weight_max": 31.1849365234375,
      "activations/layer21_attention_weight_min": -23.161285400390625,
      "activations/layer22_attention_weight_max": 55.67351150512695,
      "activations/layer22_attention_weight_min": -30.666488647460938,
      "activations/layer23_attention_weight_max": 38.669986724853516,
      "activations/layer23_attention_weight_min": -26.010719299316406,
      "activations/layer2_attention_weight_max": 35.17589569091797,
      "activations/layer2_attention_weight_min": -33.47711944580078,
      "activations/layer3_attention_weight_max": 99.46532440185547,
      "activations/layer3_attention_weight_min": -100.91588592529297,
      "activations/layer4_attention_weight_max": 111.83387756347656,
      "activations/layer4_attention_weight_min": -116.1284408569336,
      "activations/layer5_attention_weight_max": 53.06243896484375,
      "activations/layer5_attention_weight_min": -62.670875549316406,
      "activations/layer6_attention_weight_max": 45.353538513183594,
      "activations/layer6_attention_weight_min": -50.182308197021484,
      "activations/layer7_attention_weight_max": 88.79290771484375,
      "activations/layer7_attention_weight_min": -101.19190216064453,
      "activations/layer8_attention_weight_max": 41.60796356201172,
      "activations/layer8_attention_weight_min": -45.3182258605957,
      "activations/layer9_attention_weight_max": 32.91617202758789,
      "activations/layer9_attention_weight_min": -34.45779037475586,
      "epoch": 11.9,
      "learning_rate": 7.402386363636363e-05,
      "loss": 2.7769,
      "step": 204750
    },
    {
      "activations/layer0_attention_weight_max": 16.41550636291504,
      "activations/layer0_attention_weight_min": -12.419708251953125,
      "activations/layer10_attention_weight_max": 32.4705924987793,
      "activations/layer10_attention_weight_min": -31.95714569091797,
      "activations/layer11_attention_weight_max": 35.748435974121094,
      "activations/layer11_attention_weight_min": -31.59316062927246,
      "activations/layer12_attention_weight_max": 20.953479766845703,
      "activations/layer12_attention_weight_min": -24.821151733398438,
      "activations/layer13_attention_weight_max": 37.00648498535156,
      "activations/layer13_attention_weight_min": -32.10169982910156,
      "activations/layer14_attention_weight_max": 41.7635612487793,
      "activations/layer14_attention_weight_min": -35.06033706665039,
      "activations/layer15_attention_weight_max": 32.91103744506836,
      "activations/layer15_attention_weight_min": -30.823429107666016,
      "activations/layer16_attention_weight_max": 27.820716857910156,
      "activations/layer16_attention_weight_min": -25.9600887298584,
      "activations/layer17_attention_weight_max": 29.55030059814453,
      "activations/layer17_attention_weight_min": -24.903684616088867,
      "activations/layer18_attention_weight_max": 30.740657806396484,
      "activations/layer18_attention_weight_min": -23.15711212158203,
      "activations/layer19_attention_weight_max": 32.72598648071289,
      "activations/layer19_attention_weight_min": -31.41627311706543,
      "activations/layer1_attention_weight_max": 17.826072692871094,
      "activations/layer1_attention_weight_min": -16.50838279724121,
      "activations/layer20_attention_weight_max": 25.7358341217041,
      "activations/layer20_attention_weight_min": -24.437360763549805,
      "activations/layer21_attention_weight_max": 23.613561630249023,
      "activations/layer21_attention_weight_min": -23.966751098632812,
      "activations/layer22_attention_weight_max": 43.43369674682617,
      "activations/layer22_attention_weight_min": -29.93446922302246,
      "activations/layer23_attention_weight_max": 31.535642623901367,
      "activations/layer23_attention_weight_min": -27.31829071044922,
      "activations/layer2_attention_weight_max": 33.341033935546875,
      "activations/layer2_attention_weight_min": -33.77743911743164,
      "activations/layer3_attention_weight_max": 103.82396697998047,
      "activations/layer3_attention_weight_min": -97.18260955810547,
      "activations/layer4_attention_weight_max": 113.3963623046875,
      "activations/layer4_attention_weight_min": -110.28531646728516,
      "activations/layer5_attention_weight_max": 50.93221664428711,
      "activations/layer5_attention_weight_min": -61.0894775390625,
      "activations/layer6_attention_weight_max": 44.201683044433594,
      "activations/layer6_attention_weight_min": -46.643226623535156,
      "activations/layer7_attention_weight_max": 96.61874389648438,
      "activations/layer7_attention_weight_min": -86.43321990966797,
      "activations/layer8_attention_weight_max": 39.26841735839844,
      "activations/layer8_attention_weight_min": -39.07163619995117,
      "activations/layer9_attention_weight_max": 32.100257873535156,
      "activations/layer9_attention_weight_min": -31.720609664916992,
      "epoch": 11.9,
      "learning_rate": 7.400530303030302e-05,
      "loss": 2.7801,
      "step": 204800
    },
    {
      "activations/layer0_attention_weight_max": 17.06197738647461,
      "activations/layer0_attention_weight_min": -12.881647109985352,
      "activations/layer10_attention_weight_max": 42.46973419189453,
      "activations/layer10_attention_weight_min": -39.11400604248047,
      "activations/layer11_attention_weight_max": 41.042972564697266,
      "activations/layer11_attention_weight_min": -36.14654541015625,
      "activations/layer12_attention_weight_max": 31.185365676879883,
      "activations/layer12_attention_weight_min": -28.278125762939453,
      "activations/layer13_attention_weight_max": 45.773109436035156,
      "activations/layer13_attention_weight_min": -39.20386505126953,
      "activations/layer14_attention_weight_max": 52.862789154052734,
      "activations/layer14_attention_weight_min": -44.04328536987305,
      "activations/layer15_attention_weight_max": 49.76459884643555,
      "activations/layer15_attention_weight_min": -36.647377014160156,
      "activations/layer16_attention_weight_max": 42.562007904052734,
      "activations/layer16_attention_weight_min": -27.20359992980957,
      "activations/layer17_attention_weight_max": 36.90826416015625,
      "activations/layer17_attention_weight_min": -25.6893310546875,
      "activations/layer18_attention_weight_max": 31.788578033447266,
      "activations/layer18_attention_weight_min": -23.931276321411133,
      "activations/layer19_attention_weight_max": 38.404605865478516,
      "activations/layer19_attention_weight_min": -31.372812271118164,
      "activations/layer1_attention_weight_max": 17.35480308532715,
      "activations/layer1_attention_weight_min": -15.666485786437988,
      "activations/layer20_attention_weight_max": 32.572792053222656,
      "activations/layer20_attention_weight_min": -25.563953399658203,
      "activations/layer21_attention_weight_max": 28.506284713745117,
      "activations/layer21_attention_weight_min": -23.630876541137695,
      "activations/layer22_attention_weight_max": 46.53191375732422,
      "activations/layer22_attention_weight_min": -30.740495681762695,
      "activations/layer23_attention_weight_max": 35.809200286865234,
      "activations/layer23_attention_weight_min": -24.31256675720215,
      "activations/layer2_attention_weight_max": 35.73210906982422,
      "activations/layer2_attention_weight_min": -34.7148551940918,
      "activations/layer3_attention_weight_max": 97.8214340209961,
      "activations/layer3_attention_weight_min": -96.74481964111328,
      "activations/layer4_attention_weight_max": 107.21346282958984,
      "activations/layer4_attention_weight_min": -108.52680969238281,
      "activations/layer5_attention_weight_max": 48.806854248046875,
      "activations/layer5_attention_weight_min": -56.93662643432617,
      "activations/layer6_attention_weight_max": 41.016963958740234,
      "activations/layer6_attention_weight_min": -47.13386154174805,
      "activations/layer7_attention_weight_max": 107.00515747070312,
      "activations/layer7_attention_weight_min": -90.40591430664062,
      "activations/layer8_attention_weight_max": 47.71877670288086,
      "activations/layer8_attention_weight_min": -44.88473129272461,
      "activations/layer9_attention_weight_max": 39.82307052612305,
      "activations/layer9_attention_weight_min": -39.75416564941406,
      "epoch": 11.9,
      "learning_rate": 7.398636363636363e-05,
      "loss": 2.7648,
      "step": 204850
    },
    {
      "activations/layer0_attention_weight_max": 16.295757293701172,
      "activations/layer0_attention_weight_min": -13.360578536987305,
      "activations/layer10_attention_weight_max": 31.07417869567871,
      "activations/layer10_attention_weight_min": -32.42481994628906,
      "activations/layer11_attention_weight_max": 31.391159057617188,
      "activations/layer11_attention_weight_min": -32.23918914794922,
      "activations/layer12_attention_weight_max": 25.869291305541992,
      "activations/layer12_attention_weight_min": -25.919065475463867,
      "activations/layer13_attention_weight_max": 38.198631286621094,
      "activations/layer13_attention_weight_min": -34.15890121459961,
      "activations/layer14_attention_weight_max": 43.78513717651367,
      "activations/layer14_attention_weight_min": -37.88697052001953,
      "activations/layer15_attention_weight_max": 34.2678108215332,
      "activations/layer15_attention_weight_min": -31.265748977661133,
      "activations/layer16_attention_weight_max": 30.668651580810547,
      "activations/layer16_attention_weight_min": -28.52553939819336,
      "activations/layer17_attention_weight_max": 31.510568618774414,
      "activations/layer17_attention_weight_min": -25.9720401763916,
      "activations/layer18_attention_weight_max": 28.636760711669922,
      "activations/layer18_attention_weight_min": -23.963808059692383,
      "activations/layer19_attention_weight_max": 33.86078643798828,
      "activations/layer19_attention_weight_min": -30.34922218322754,
      "activations/layer1_attention_weight_max": 17.400466918945312,
      "activations/layer1_attention_weight_min": -14.112979888916016,
      "activations/layer20_attention_weight_max": 29.167890548706055,
      "activations/layer20_attention_weight_min": -25.313854217529297,
      "activations/layer21_attention_weight_max": 30.699682235717773,
      "activations/layer21_attention_weight_min": -23.73788833618164,
      "activations/layer22_attention_weight_max": 43.67726135253906,
      "activations/layer22_attention_weight_min": -29.354082107543945,
      "activations/layer23_attention_weight_max": 36.79744338989258,
      "activations/layer23_attention_weight_min": -24.200878143310547,
      "activations/layer2_attention_weight_max": 34.426429748535156,
      "activations/layer2_attention_weight_min": -32.11103057861328,
      "activations/layer3_attention_weight_max": 95.65380096435547,
      "activations/layer3_attention_weight_min": -95.57130432128906,
      "activations/layer4_attention_weight_max": 113.9349365234375,
      "activations/layer4_attention_weight_min": -111.97545623779297,
      "activations/layer5_attention_weight_max": 51.41992950439453,
      "activations/layer5_attention_weight_min": -60.66795349121094,
      "activations/layer6_attention_weight_max": 45.60941696166992,
      "activations/layer6_attention_weight_min": -46.74930953979492,
      "activations/layer7_attention_weight_max": 91.21217346191406,
      "activations/layer7_attention_weight_min": -94.68941497802734,
      "activations/layer8_attention_weight_max": 41.04966354370117,
      "activations/layer8_attention_weight_min": -42.88117980957031,
      "activations/layer9_attention_weight_max": 31.288330078125,
      "activations/layer9_attention_weight_min": -32.724849700927734,
      "epoch": 11.91,
      "learning_rate": 7.396742424242424e-05,
      "loss": 2.7792,
      "step": 204900
    },
    {
      "activations/layer0_attention_weight_max": 16.187528610229492,
      "activations/layer0_attention_weight_min": -12.461440086364746,
      "activations/layer10_attention_weight_max": 32.67183303833008,
      "activations/layer10_attention_weight_min": -32.255775451660156,
      "activations/layer11_attention_weight_max": 34.927955627441406,
      "activations/layer11_attention_weight_min": -31.472394943237305,
      "activations/layer12_attention_weight_max": 25.46735191345215,
      "activations/layer12_attention_weight_min": -25.99628448486328,
      "activations/layer13_attention_weight_max": 38.661067962646484,
      "activations/layer13_attention_weight_min": -36.57819747924805,
      "activations/layer14_attention_weight_max": 43.41178894042969,
      "activations/layer14_attention_weight_min": -37.82980728149414,
      "activations/layer15_attention_weight_max": 37.818145751953125,
      "activations/layer15_attention_weight_min": -33.161399841308594,
      "activations/layer16_attention_weight_max": 31.404325485229492,
      "activations/layer16_attention_weight_min": -27.535472869873047,
      "activations/layer17_attention_weight_max": 31.51807975769043,
      "activations/layer17_attention_weight_min": -26.354551315307617,
      "activations/layer18_attention_weight_max": 33.025718688964844,
      "activations/layer18_attention_weight_min": -23.086156845092773,
      "activations/layer19_attention_weight_max": 35.79142379760742,
      "activations/layer19_attention_weight_min": -30.579893112182617,
      "activations/layer1_attention_weight_max": 16.151790618896484,
      "activations/layer1_attention_weight_min": -15.357457160949707,
      "activations/layer20_attention_weight_max": 29.284168243408203,
      "activations/layer20_attention_weight_min": -25.897808074951172,
      "activations/layer21_attention_weight_max": 32.368743896484375,
      "activations/layer21_attention_weight_min": -24.351224899291992,
      "activations/layer22_attention_weight_max": 42.206661224365234,
      "activations/layer22_attention_weight_min": -30.528926849365234,
      "activations/layer23_attention_weight_max": 36.93003463745117,
      "activations/layer23_attention_weight_min": -25.85377311706543,
      "activations/layer2_attention_weight_max": 32.69471740722656,
      "activations/layer2_attention_weight_min": -33.04424285888672,
      "activations/layer3_attention_weight_max": 94.10228729248047,
      "activations/layer3_attention_weight_min": -99.78360748291016,
      "activations/layer4_attention_weight_max": 114.80867004394531,
      "activations/layer4_attention_weight_min": -117.43621826171875,
      "activations/layer5_attention_weight_max": 50.40644454956055,
      "activations/layer5_attention_weight_min": -64.10820007324219,
      "activations/layer6_attention_weight_max": 43.73786926269531,
      "activations/layer6_attention_weight_min": -46.43339920043945,
      "activations/layer7_attention_weight_max": 88.77307891845703,
      "activations/layer7_attention_weight_min": -86.37348175048828,
      "activations/layer8_attention_weight_max": 42.94388198852539,
      "activations/layer8_attention_weight_min": -40.16195297241211,
      "activations/layer9_attention_weight_max": 33.58257293701172,
      "activations/layer9_attention_weight_min": -33.32331466674805,
      "epoch": 11.91,
      "learning_rate": 7.394848484848484e-05,
      "loss": 2.769,
      "step": 204950
    },
    {
      "activations/layer0_attention_weight_max": 15.658514976501465,
      "activations/layer0_attention_weight_min": -12.464676856994629,
      "activations/layer10_attention_weight_max": 31.954618453979492,
      "activations/layer10_attention_weight_min": -31.238462448120117,
      "activations/layer11_attention_weight_max": 34.728477478027344,
      "activations/layer11_attention_weight_min": -32.36113739013672,
      "activations/layer12_attention_weight_max": 27.899295806884766,
      "activations/layer12_attention_weight_min": -26.17829704284668,
      "activations/layer13_attention_weight_max": 40.884944915771484,
      "activations/layer13_attention_weight_min": -35.664180755615234,
      "activations/layer14_attention_weight_max": 47.200443267822266,
      "activations/layer14_attention_weight_min": -39.95750427246094,
      "activations/layer15_attention_weight_max": 38.54606246948242,
      "activations/layer15_attention_weight_min": -33.137691497802734,
      "activations/layer16_attention_weight_max": 30.422351837158203,
      "activations/layer16_attention_weight_min": -27.584796905517578,
      "activations/layer17_attention_weight_max": 31.54428482055664,
      "activations/layer17_attention_weight_min": -25.91826820373535,
      "activations/layer18_attention_weight_max": 29.7978458404541,
      "activations/layer18_attention_weight_min": -23.767852783203125,
      "activations/layer19_attention_weight_max": 32.38108444213867,
      "activations/layer19_attention_weight_min": -30.560190200805664,
      "activations/layer1_attention_weight_max": 16.79022979736328,
      "activations/layer1_attention_weight_min": -15.895683288574219,
      "activations/layer20_attention_weight_max": 28.921567916870117,
      "activations/layer20_attention_weight_min": -24.96199607849121,
      "activations/layer21_attention_weight_max": 29.103759765625,
      "activations/layer21_attention_weight_min": -23.677946090698242,
      "activations/layer22_attention_weight_max": 40.97029495239258,
      "activations/layer22_attention_weight_min": -29.83197021484375,
      "activations/layer23_attention_weight_max": 33.505470275878906,
      "activations/layer23_attention_weight_min": -26.495803833007812,
      "activations/layer2_attention_weight_max": 31.79604721069336,
      "activations/layer2_attention_weight_min": -33.96095275878906,
      "activations/layer3_attention_weight_max": 94.60492706298828,
      "activations/layer3_attention_weight_min": -95.30571746826172,
      "activations/layer4_attention_weight_max": 111.61429595947266,
      "activations/layer4_attention_weight_min": -114.01458740234375,
      "activations/layer5_attention_weight_max": 49.33721160888672,
      "activations/layer5_attention_weight_min": -60.81046676635742,
      "activations/layer6_attention_weight_max": 42.10834884643555,
      "activations/layer6_attention_weight_min": -47.90510940551758,
      "activations/layer7_attention_weight_max": 89.17154693603516,
      "activations/layer7_attention_weight_min": -85.17222595214844,
      "activations/layer8_attention_weight_max": 39.10623550415039,
      "activations/layer8_attention_weight_min": -39.29590606689453,
      "activations/layer9_attention_weight_max": 31.674161911010742,
      "activations/layer9_attention_weight_min": -33.2625732421875,
      "epoch": 11.91,
      "learning_rate": 7.392954545454544e-05,
      "loss": 2.7716,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.5481,
      "eval_samples_per_second": 502.334,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.5481,
      "eval_openwebtext_samples_per_second": 502.334,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.0464,
      "eval_wikitext_samples_per_second": 222.829,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.6904,
      "eval_lambada_samples_per_second": 502.456,
      "step": 205000
    },
    {
      "activations/layer0_attention_weight_max": 17.125255584716797,
      "activations/layer0_attention_weight_min": -12.29785442352295,
      "activations/layer10_attention_weight_max": 32.79278564453125,
      "activations/layer10_attention_weight_min": -32.094974517822266,
      "activations/layer11_attention_weight_max": 32.01699447631836,
      "activations/layer11_attention_weight_min": -32.831809997558594,
      "activations/layer12_attention_weight_max": 24.41134262084961,
      "activations/layer12_attention_weight_min": -26.337724685668945,
      "activations/layer13_attention_weight_max": 36.096641540527344,
      "activations/layer13_attention_weight_min": -32.913360595703125,
      "activations/layer14_attention_weight_max": 42.10276412963867,
      "activations/layer14_attention_weight_min": -38.37081527709961,
      "activations/layer15_attention_weight_max": 33.719764709472656,
      "activations/layer15_attention_weight_min": -32.07363510131836,
      "activations/layer16_attention_weight_max": 31.16742706298828,
      "activations/layer16_attention_weight_min": -28.821626663208008,
      "activations/layer17_attention_weight_max": 32.078983306884766,
      "activations/layer17_attention_weight_min": -25.359298706054688,
      "activations/layer18_attention_weight_max": 31.325063705444336,
      "activations/layer18_attention_weight_min": -23.554553985595703,
      "activations/layer19_attention_weight_max": 33.319149017333984,
      "activations/layer19_attention_weight_min": -31.039819717407227,
      "activations/layer1_attention_weight_max": 16.775239944458008,
      "activations/layer1_attention_weight_min": -15.431061744689941,
      "activations/layer20_attention_weight_max": 32.300350189208984,
      "activations/layer20_attention_weight_min": -26.857757568359375,
      "activations/layer21_attention_weight_max": 29.243797302246094,
      "activations/layer21_attention_weight_min": -25.516666412353516,
      "activations/layer22_attention_weight_max": 38.95711135864258,
      "activations/layer22_attention_weight_min": -30.33437728881836,
      "activations/layer23_attention_weight_max": 35.37396240234375,
      "activations/layer23_attention_weight_min": -24.03375244140625,
      "activations/layer2_attention_weight_max": 30.780582427978516,
      "activations/layer2_attention_weight_min": -32.39519500732422,
      "activations/layer3_attention_weight_max": 94.94278717041016,
      "activations/layer3_attention_weight_min": -96.38043975830078,
      "activations/layer4_attention_weight_max": 110.96038818359375,
      "activations/layer4_attention_weight_min": -115.42217254638672,
      "activations/layer5_attention_weight_max": 51.72590637207031,
      "activations/layer5_attention_weight_min": -62.51654052734375,
      "activations/layer6_attention_weight_max": 44.2174186706543,
      "activations/layer6_attention_weight_min": -49.74338912963867,
      "activations/layer7_attention_weight_max": 91.27008056640625,
      "activations/layer7_attention_weight_min": -91.49659729003906,
      "activations/layer8_attention_weight_max": 39.64019775390625,
      "activations/layer8_attention_weight_min": -42.48596954345703,
      "activations/layer9_attention_weight_max": 33.015926361083984,
      "activations/layer9_attention_weight_min": -33.594661712646484,
      "epoch": 11.91,
      "learning_rate": 7.391060606060605e-05,
      "loss": 2.7673,
      "step": 205050
    },
    {
      "activations/layer0_attention_weight_max": 16.033843994140625,
      "activations/layer0_attention_weight_min": -12.337922096252441,
      "activations/layer10_attention_weight_max": 33.0233039855957,
      "activations/layer10_attention_weight_min": -33.33293533325195,
      "activations/layer11_attention_weight_max": 35.005550384521484,
      "activations/layer11_attention_weight_min": -31.625341415405273,
      "activations/layer12_attention_weight_max": 26.444311141967773,
      "activations/layer12_attention_weight_min": -26.278194427490234,
      "activations/layer13_attention_weight_max": 40.02289581298828,
      "activations/layer13_attention_weight_min": -35.503028869628906,
      "activations/layer14_attention_weight_max": 44.77317810058594,
      "activations/layer14_attention_weight_min": -41.67727279663086,
      "activations/layer15_attention_weight_max": 40.378623962402344,
      "activations/layer15_attention_weight_min": -33.473236083984375,
      "activations/layer16_attention_weight_max": 33.84458541870117,
      "activations/layer16_attention_weight_min": -27.225177764892578,
      "activations/layer17_attention_weight_max": 38.74419021606445,
      "activations/layer17_attention_weight_min": -25.29680824279785,
      "activations/layer18_attention_weight_max": 34.275733947753906,
      "activations/layer18_attention_weight_min": -23.242910385131836,
      "activations/layer19_attention_weight_max": 37.1208381652832,
      "activations/layer19_attention_weight_min": -31.292144775390625,
      "activations/layer1_attention_weight_max": 17.037689208984375,
      "activations/layer1_attention_weight_min": -16.56827735900879,
      "activations/layer20_attention_weight_max": 34.501644134521484,
      "activations/layer20_attention_weight_min": -29.482864379882812,
      "activations/layer21_attention_weight_max": 33.437950134277344,
      "activations/layer21_attention_weight_min": -27.416290283203125,
      "activations/layer22_attention_weight_max": 45.355438232421875,
      "activations/layer22_attention_weight_min": -31.234161376953125,
      "activations/layer23_attention_weight_max": 35.03775405883789,
      "activations/layer23_attention_weight_min": -27.522186279296875,
      "activations/layer2_attention_weight_max": 33.04988479614258,
      "activations/layer2_attention_weight_min": -32.11525344848633,
      "activations/layer3_attention_weight_max": 96.99491119384766,
      "activations/layer3_attention_weight_min": -95.8604507446289,
      "activations/layer4_attention_weight_max": 112.17433166503906,
      "activations/layer4_attention_weight_min": -114.99320220947266,
      "activations/layer5_attention_weight_max": 53.03063201904297,
      "activations/layer5_attention_weight_min": -59.270782470703125,
      "activations/layer6_attention_weight_max": 45.25946807861328,
      "activations/layer6_attention_weight_min": -46.80418014526367,
      "activations/layer7_attention_weight_max": 90.5036392211914,
      "activations/layer7_attention_weight_min": -92.91446685791016,
      "activations/layer8_attention_weight_max": 41.06706619262695,
      "activations/layer8_attention_weight_min": -44.13124465942383,
      "activations/layer9_attention_weight_max": 33.34459686279297,
      "activations/layer9_attention_weight_min": -32.58919143676758,
      "epoch": 11.92,
      "learning_rate": 7.389166666666666e-05,
      "loss": 2.807,
      "step": 205100
    },
    {
      "activations/layer0_attention_weight_max": 16.10706329345703,
      "activations/layer0_attention_weight_min": -12.669295310974121,
      "activations/layer10_attention_weight_max": 32.34199523925781,
      "activations/layer10_attention_weight_min": -33.60136795043945,
      "activations/layer11_attention_weight_max": 33.80225372314453,
      "activations/layer11_attention_weight_min": -33.469085693359375,
      "activations/layer12_attention_weight_max": 24.01528549194336,
      "activations/layer12_attention_weight_min": -24.752946853637695,
      "activations/layer13_attention_weight_max": 35.505306243896484,
      "activations/layer13_attention_weight_min": -35.1597900390625,
      "activations/layer14_attention_weight_max": 42.270389556884766,
      "activations/layer14_attention_weight_min": -37.26192092895508,
      "activations/layer15_attention_weight_max": 36.37822723388672,
      "activations/layer15_attention_weight_min": -33.82514953613281,
      "activations/layer16_attention_weight_max": 31.35437774658203,
      "activations/layer16_attention_weight_min": -26.930492401123047,
      "activations/layer17_attention_weight_max": 30.63224983215332,
      "activations/layer17_attention_weight_min": -24.122314453125,
      "activations/layer18_attention_weight_max": 31.34408187866211,
      "activations/layer18_attention_weight_min": -23.550922393798828,
      "activations/layer19_attention_weight_max": 35.77155303955078,
      "activations/layer19_attention_weight_min": -30.359939575195312,
      "activations/layer1_attention_weight_max": 16.47333335876465,
      "activations/layer1_attention_weight_min": -14.960932731628418,
      "activations/layer20_attention_weight_max": 32.26136016845703,
      "activations/layer20_attention_weight_min": -25.597970962524414,
      "activations/layer21_attention_weight_max": 29.113542556762695,
      "activations/layer21_attention_weight_min": -24.197372436523438,
      "activations/layer22_attention_weight_max": 48.06064987182617,
      "activations/layer22_attention_weight_min": -29.9892635345459,
      "activations/layer23_attention_weight_max": 34.348854064941406,
      "activations/layer23_attention_weight_min": -25.082794189453125,
      "activations/layer2_attention_weight_max": 33.04332733154297,
      "activations/layer2_attention_weight_min": -33.05205154418945,
      "activations/layer3_attention_weight_max": 98.09125518798828,
      "activations/layer3_attention_weight_min": -98.1815185546875,
      "activations/layer4_attention_weight_max": 114.7762222290039,
      "activations/layer4_attention_weight_min": -114.87108612060547,
      "activations/layer5_attention_weight_max": 54.027557373046875,
      "activations/layer5_attention_weight_min": -63.03104782104492,
      "activations/layer6_attention_weight_max": 44.088130950927734,
      "activations/layer6_attention_weight_min": -49.64384460449219,
      "activations/layer7_attention_weight_max": 91.58831024169922,
      "activations/layer7_attention_weight_min": -92.89306640625,
      "activations/layer8_attention_weight_max": 41.167335510253906,
      "activations/layer8_attention_weight_min": -45.06436538696289,
      "activations/layer9_attention_weight_max": 35.34366226196289,
      "activations/layer9_attention_weight_min": -36.20488357543945,
      "epoch": 11.92,
      "learning_rate": 7.387272727272726e-05,
      "loss": 2.773,
      "step": 205150
    },
    {
      "activations/layer0_attention_weight_max": 16.5007381439209,
      "activations/layer0_attention_weight_min": -12.25937271118164,
      "activations/layer10_attention_weight_max": 31.385498046875,
      "activations/layer10_attention_weight_min": -32.43695068359375,
      "activations/layer11_attention_weight_max": 32.13214874267578,
      "activations/layer11_attention_weight_min": -34.419532775878906,
      "activations/layer12_attention_weight_max": 24.515825271606445,
      "activations/layer12_attention_weight_min": -26.95088005065918,
      "activations/layer13_attention_weight_max": 40.53077697753906,
      "activations/layer13_attention_weight_min": -33.37974548339844,
      "activations/layer14_attention_weight_max": 43.81422424316406,
      "activations/layer14_attention_weight_min": -35.65437698364258,
      "activations/layer15_attention_weight_max": 39.510719299316406,
      "activations/layer15_attention_weight_min": -33.244476318359375,
      "activations/layer16_attention_weight_max": 33.368385314941406,
      "activations/layer16_attention_weight_min": -27.956443786621094,
      "activations/layer17_attention_weight_max": 31.480356216430664,
      "activations/layer17_attention_weight_min": -27.57430648803711,
      "activations/layer18_attention_weight_max": 35.43008804321289,
      "activations/layer18_attention_weight_min": -26.101720809936523,
      "activations/layer19_attention_weight_max": 37.39723205566406,
      "activations/layer19_attention_weight_min": -34.23844528198242,
      "activations/layer1_attention_weight_max": 16.802772521972656,
      "activations/layer1_attention_weight_min": -15.070524215698242,
      "activations/layer20_attention_weight_max": 30.070499420166016,
      "activations/layer20_attention_weight_min": -27.062116622924805,
      "activations/layer21_attention_weight_max": 28.613595962524414,
      "activations/layer21_attention_weight_min": -25.135034561157227,
      "activations/layer22_attention_weight_max": 55.65656280517578,
      "activations/layer22_attention_weight_min": -31.58286476135254,
      "activations/layer23_attention_weight_max": 39.171775817871094,
      "activations/layer23_attention_weight_min": -27.317947387695312,
      "activations/layer2_attention_weight_max": 35.60293960571289,
      "activations/layer2_attention_weight_min": -33.860836029052734,
      "activations/layer3_attention_weight_max": 101.92579650878906,
      "activations/layer3_attention_weight_min": -95.82930755615234,
      "activations/layer4_attention_weight_max": 112.348876953125,
      "activations/layer4_attention_weight_min": -109.95328521728516,
      "activations/layer5_attention_weight_max": 55.24015426635742,
      "activations/layer5_attention_weight_min": -66.37586212158203,
      "activations/layer6_attention_weight_max": 44.887935638427734,
      "activations/layer6_attention_weight_min": -49.198055267333984,
      "activations/layer7_attention_weight_max": 87.52701568603516,
      "activations/layer7_attention_weight_min": -91.01737213134766,
      "activations/layer8_attention_weight_max": 41.20184326171875,
      "activations/layer8_attention_weight_min": -43.707340240478516,
      "activations/layer9_attention_weight_max": 31.083471298217773,
      "activations/layer9_attention_weight_min": -32.450355529785156,
      "epoch": 11.92,
      "learning_rate": 7.385378787878787e-05,
      "loss": 2.7859,
      "step": 205200
    },
    {
      "activations/layer0_attention_weight_max": 16.135663986206055,
      "activations/layer0_attention_weight_min": -11.874222755432129,
      "activations/layer10_attention_weight_max": 35.26935958862305,
      "activations/layer10_attention_weight_min": -36.51686096191406,
      "activations/layer11_attention_weight_max": 33.9434814453125,
      "activations/layer11_attention_weight_min": -35.19646072387695,
      "activations/layer12_attention_weight_max": 31.98158073425293,
      "activations/layer12_attention_weight_min": -25.667097091674805,
      "activations/layer13_attention_weight_max": 38.85942077636719,
      "activations/layer13_attention_weight_min": -35.24787902832031,
      "activations/layer14_attention_weight_max": 40.705604553222656,
      "activations/layer14_attention_weight_min": -38.60654067993164,
      "activations/layer15_attention_weight_max": 40.69029235839844,
      "activations/layer15_attention_weight_min": -32.98295211791992,
      "activations/layer16_attention_weight_max": 30.113834381103516,
      "activations/layer16_attention_weight_min": -28.626462936401367,
      "activations/layer17_attention_weight_max": 29.545379638671875,
      "activations/layer17_attention_weight_min": -25.407238006591797,
      "activations/layer18_attention_weight_max": 31.260787963867188,
      "activations/layer18_attention_weight_min": -22.726390838623047,
      "activations/layer19_attention_weight_max": 40.35231018066406,
      "activations/layer19_attention_weight_min": -29.309579849243164,
      "activations/layer1_attention_weight_max": 16.706384658813477,
      "activations/layer1_attention_weight_min": -16.735490798950195,
      "activations/layer20_attention_weight_max": 27.967578887939453,
      "activations/layer20_attention_weight_min": -24.731922149658203,
      "activations/layer21_attention_weight_max": 27.200469970703125,
      "activations/layer21_attention_weight_min": -23.107833862304688,
      "activations/layer22_attention_weight_max": 43.9039306640625,
      "activations/layer22_attention_weight_min": -29.887027740478516,
      "activations/layer23_attention_weight_max": 37.83061218261719,
      "activations/layer23_attention_weight_min": -24.617679595947266,
      "activations/layer2_attention_weight_max": 35.176761627197266,
      "activations/layer2_attention_weight_min": -35.984107971191406,
      "activations/layer3_attention_weight_max": 104.48155212402344,
      "activations/layer3_attention_weight_min": -109.06465911865234,
      "activations/layer4_attention_weight_max": 118.51261138916016,
      "activations/layer4_attention_weight_min": -116.55571746826172,
      "activations/layer5_attention_weight_max": 52.09016418457031,
      "activations/layer5_attention_weight_min": -61.869441986083984,
      "activations/layer6_attention_weight_max": 43.40624237060547,
      "activations/layer6_attention_weight_min": -46.669715881347656,
      "activations/layer7_attention_weight_max": 93.11036682128906,
      "activations/layer7_attention_weight_min": -88.2833251953125,
      "activations/layer8_attention_weight_max": 42.83773422241211,
      "activations/layer8_attention_weight_min": -42.231346130371094,
      "activations/layer9_attention_weight_max": 33.03843307495117,
      "activations/layer9_attention_weight_min": -34.42018508911133,
      "epoch": 11.93,
      "learning_rate": 7.383484848484847e-05,
      "loss": 2.7843,
      "step": 205250
    },
    {
      "activations/layer0_attention_weight_max": 15.318584442138672,
      "activations/layer0_attention_weight_min": -13.16510009765625,
      "activations/layer10_attention_weight_max": 31.52782440185547,
      "activations/layer10_attention_weight_min": -32.760398864746094,
      "activations/layer11_attention_weight_max": 31.6397705078125,
      "activations/layer11_attention_weight_min": -32.81160354614258,
      "activations/layer12_attention_weight_max": 23.99419593811035,
      "activations/layer12_attention_weight_min": -24.84794044494629,
      "activations/layer13_attention_weight_max": 37.02409744262695,
      "activations/layer13_attention_weight_min": -33.89237976074219,
      "activations/layer14_attention_weight_max": 40.03279495239258,
      "activations/layer14_attention_weight_min": -37.425533294677734,
      "activations/layer15_attention_weight_max": 37.10093307495117,
      "activations/layer15_attention_weight_min": -32.94636154174805,
      "activations/layer16_attention_weight_max": 29.23166275024414,
      "activations/layer16_attention_weight_min": -27.17281150817871,
      "activations/layer17_attention_weight_max": 32.44772720336914,
      "activations/layer17_attention_weight_min": -26.070186614990234,
      "activations/layer18_attention_weight_max": 33.550025939941406,
      "activations/layer18_attention_weight_min": -22.80777931213379,
      "activations/layer19_attention_weight_max": 33.571556091308594,
      "activations/layer19_attention_weight_min": -30.06558609008789,
      "activations/layer1_attention_weight_max": 16.61454963684082,
      "activations/layer1_attention_weight_min": -15.592184066772461,
      "activations/layer20_attention_weight_max": 31.497434616088867,
      "activations/layer20_attention_weight_min": -25.58779525756836,
      "activations/layer21_attention_weight_max": 28.314311981201172,
      "activations/layer21_attention_weight_min": -24.301677703857422,
      "activations/layer22_attention_weight_max": 39.58890151977539,
      "activations/layer22_attention_weight_min": -28.248403549194336,
      "activations/layer23_attention_weight_max": 30.53810691833496,
      "activations/layer23_attention_weight_min": -25.26112937927246,
      "activations/layer2_attention_weight_max": 34.874176025390625,
      "activations/layer2_attention_weight_min": -33.428916931152344,
      "activations/layer3_attention_weight_max": 100.2829360961914,
      "activations/layer3_attention_weight_min": -98.3319320678711,
      "activations/layer4_attention_weight_max": 118.5740737915039,
      "activations/layer4_attention_weight_min": -119.0970687866211,
      "activations/layer5_attention_weight_max": 55.811622619628906,
      "activations/layer5_attention_weight_min": -60.43630599975586,
      "activations/layer6_attention_weight_max": 47.591251373291016,
      "activations/layer6_attention_weight_min": -49.22592544555664,
      "activations/layer7_attention_weight_max": 96.78264617919922,
      "activations/layer7_attention_weight_min": -94.0353775024414,
      "activations/layer8_attention_weight_max": 41.74533462524414,
      "activations/layer8_attention_weight_min": -42.16835403442383,
      "activations/layer9_attention_weight_max": 33.88669967651367,
      "activations/layer9_attention_weight_min": -34.4405403137207,
      "epoch": 11.93,
      "learning_rate": 7.381590909090909e-05,
      "loss": 2.7989,
      "step": 205300
    },
    {
      "activations/layer0_attention_weight_max": 16.040414810180664,
      "activations/layer0_attention_weight_min": -12.680702209472656,
      "activations/layer10_attention_weight_max": 32.7324104309082,
      "activations/layer10_attention_weight_min": -31.96024513244629,
      "activations/layer11_attention_weight_max": 33.921688079833984,
      "activations/layer11_attention_weight_min": -31.359228134155273,
      "activations/layer12_attention_weight_max": 25.56812858581543,
      "activations/layer12_attention_weight_min": -25.75879669189453,
      "activations/layer13_attention_weight_max": 39.5891227722168,
      "activations/layer13_attention_weight_min": -34.333465576171875,
      "activations/layer14_attention_weight_max": 45.272132873535156,
      "activations/layer14_attention_weight_min": -36.79719924926758,
      "activations/layer15_attention_weight_max": 36.4130973815918,
      "activations/layer15_attention_weight_min": -33.35469436645508,
      "activations/layer16_attention_weight_max": 29.921550750732422,
      "activations/layer16_attention_weight_min": -27.587522506713867,
      "activations/layer17_attention_weight_max": 31.431726455688477,
      "activations/layer17_attention_weight_min": -25.633270263671875,
      "activations/layer18_attention_weight_max": 34.24339294433594,
      "activations/layer18_attention_weight_min": -24.112743377685547,
      "activations/layer19_attention_weight_max": 33.21382141113281,
      "activations/layer19_attention_weight_min": -29.552555084228516,
      "activations/layer1_attention_weight_max": 17.071781158447266,
      "activations/layer1_attention_weight_min": -16.40704917907715,
      "activations/layer20_attention_weight_max": 28.736360549926758,
      "activations/layer20_attention_weight_min": -25.318859100341797,
      "activations/layer21_attention_weight_max": 28.492876052856445,
      "activations/layer21_attention_weight_min": -23.399145126342773,
      "activations/layer22_attention_weight_max": 43.503929138183594,
      "activations/layer22_attention_weight_min": -30.871858596801758,
      "activations/layer23_attention_weight_max": 34.41386413574219,
      "activations/layer23_attention_weight_min": -25.6409912109375,
      "activations/layer2_attention_weight_max": 32.40081024169922,
      "activations/layer2_attention_weight_min": -32.371681213378906,
      "activations/layer3_attention_weight_max": 96.26795959472656,
      "activations/layer3_attention_weight_min": -96.86652374267578,
      "activations/layer4_attention_weight_max": 106.6545181274414,
      "activations/layer4_attention_weight_min": -114.3359375,
      "activations/layer5_attention_weight_max": 50.075538635253906,
      "activations/layer5_attention_weight_min": -61.72500228881836,
      "activations/layer6_attention_weight_max": 43.93531036376953,
      "activations/layer6_attention_weight_min": -47.552730560302734,
      "activations/layer7_attention_weight_max": 88.50244903564453,
      "activations/layer7_attention_weight_min": -92.83688354492188,
      "activations/layer8_attention_weight_max": 41.06511688232422,
      "activations/layer8_attention_weight_min": -41.972572326660156,
      "activations/layer9_attention_weight_max": 33.65354537963867,
      "activations/layer9_attention_weight_min": -32.553646087646484,
      "epoch": 11.93,
      "learning_rate": 7.379696969696969e-05,
      "loss": 2.7794,
      "step": 205350
    },
    {
      "activations/layer0_attention_weight_max": 15.532452583312988,
      "activations/layer0_attention_weight_min": -13.077154159545898,
      "activations/layer10_attention_weight_max": 36.73361587524414,
      "activations/layer10_attention_weight_min": -35.262962341308594,
      "activations/layer11_attention_weight_max": 37.50734329223633,
      "activations/layer11_attention_weight_min": -35.336814880371094,
      "activations/layer12_attention_weight_max": 25.675033569335938,
      "activations/layer12_attention_weight_min": -26.006080627441406,
      "activations/layer13_attention_weight_max": 41.033164978027344,
      "activations/layer13_attention_weight_min": -36.22109603881836,
      "activations/layer14_attention_weight_max": 44.37617874145508,
      "activations/layer14_attention_weight_min": -38.57718276977539,
      "activations/layer15_attention_weight_max": 35.082645416259766,
      "activations/layer15_attention_weight_min": -33.16607666015625,
      "activations/layer16_attention_weight_max": 28.20235252380371,
      "activations/layer16_attention_weight_min": -29.704544067382812,
      "activations/layer17_attention_weight_max": 32.679813385009766,
      "activations/layer17_attention_weight_min": -26.08373260498047,
      "activations/layer18_attention_weight_max": 31.512897491455078,
      "activations/layer18_attention_weight_min": -22.715721130371094,
      "activations/layer19_attention_weight_max": 34.0557861328125,
      "activations/layer19_attention_weight_min": -29.259382247924805,
      "activations/layer1_attention_weight_max": 16.130517959594727,
      "activations/layer1_attention_weight_min": -16.293350219726562,
      "activations/layer20_attention_weight_max": 28.383522033691406,
      "activations/layer20_attention_weight_min": -23.184293746948242,
      "activations/layer21_attention_weight_max": 27.848722457885742,
      "activations/layer21_attention_weight_min": -22.492246627807617,
      "activations/layer22_attention_weight_max": 41.32309341430664,
      "activations/layer22_attention_weight_min": -27.900863647460938,
      "activations/layer23_attention_weight_max": 33.60942840576172,
      "activations/layer23_attention_weight_min": -24.559425354003906,
      "activations/layer2_attention_weight_max": 33.33415222167969,
      "activations/layer2_attention_weight_min": -32.8619270324707,
      "activations/layer3_attention_weight_max": 100.91978454589844,
      "activations/layer3_attention_weight_min": -100.50206756591797,
      "activations/layer4_attention_weight_max": 119.7192153930664,
      "activations/layer4_attention_weight_min": -120.3176498413086,
      "activations/layer5_attention_weight_max": 53.69538116455078,
      "activations/layer5_attention_weight_min": -61.74172592163086,
      "activations/layer6_attention_weight_max": 45.58378219604492,
      "activations/layer6_attention_weight_min": -49.0977897644043,
      "activations/layer7_attention_weight_max": 94.82820892333984,
      "activations/layer7_attention_weight_min": -94.80716705322266,
      "activations/layer8_attention_weight_max": 42.903282165527344,
      "activations/layer8_attention_weight_min": -46.47275161743164,
      "activations/layer9_attention_weight_max": 38.7358512878418,
      "activations/layer9_attention_weight_min": -37.12118148803711,
      "epoch": 11.93,
      "learning_rate": 7.37780303030303e-05,
      "loss": 2.7689,
      "step": 205400
    },
    {
      "activations/layer0_attention_weight_max": 16.613475799560547,
      "activations/layer0_attention_weight_min": -12.200778007507324,
      "activations/layer10_attention_weight_max": 33.71281814575195,
      "activations/layer10_attention_weight_min": -32.96938705444336,
      "activations/layer11_attention_weight_max": 36.07072448730469,
      "activations/layer11_attention_weight_min": -29.86482810974121,
      "activations/layer12_attention_weight_max": 26.522262573242188,
      "activations/layer12_attention_weight_min": -24.099103927612305,
      "activations/layer13_attention_weight_max": 41.39259338378906,
      "activations/layer13_attention_weight_min": -32.18303298950195,
      "activations/layer14_attention_weight_max": 52.40962600708008,
      "activations/layer14_attention_weight_min": -38.806400299072266,
      "activations/layer15_attention_weight_max": 45.15019607543945,
      "activations/layer15_attention_weight_min": -34.85255813598633,
      "activations/layer16_attention_weight_max": 36.30329132080078,
      "activations/layer16_attention_weight_min": -29.005266189575195,
      "activations/layer17_attention_weight_max": 37.19954299926758,
      "activations/layer17_attention_weight_min": -26.032602310180664,
      "activations/layer18_attention_weight_max": 33.59272384643555,
      "activations/layer18_attention_weight_min": -23.401151657104492,
      "activations/layer19_attention_weight_max": 42.33795166015625,
      "activations/layer19_attention_weight_min": -30.393457412719727,
      "activations/layer1_attention_weight_max": 17.613454818725586,
      "activations/layer1_attention_weight_min": -14.371845245361328,
      "activations/layer20_attention_weight_max": 32.75111389160156,
      "activations/layer20_attention_weight_min": -22.788969039916992,
      "activations/layer21_attention_weight_max": 32.029518127441406,
      "activations/layer21_attention_weight_min": -23.921476364135742,
      "activations/layer22_attention_weight_max": 53.74577713012695,
      "activations/layer22_attention_weight_min": -29.81780242919922,
      "activations/layer23_attention_weight_max": 37.839820861816406,
      "activations/layer23_attention_weight_min": -24.335113525390625,
      "activations/layer2_attention_weight_max": 35.21010971069336,
      "activations/layer2_attention_weight_min": -34.50714874267578,
      "activations/layer3_attention_weight_max": 96.11732482910156,
      "activations/layer3_attention_weight_min": -95.22618865966797,
      "activations/layer4_attention_weight_max": 112.14837646484375,
      "activations/layer4_attention_weight_min": -111.2462158203125,
      "activations/layer5_attention_weight_max": 51.17664337158203,
      "activations/layer5_attention_weight_min": -61.28491973876953,
      "activations/layer6_attention_weight_max": 41.846412658691406,
      "activations/layer6_attention_weight_min": -46.14125442504883,
      "activations/layer7_attention_weight_max": 82.82434844970703,
      "activations/layer7_attention_weight_min": -87.04643249511719,
      "activations/layer8_attention_weight_max": 38.33723449707031,
      "activations/layer8_attention_weight_min": -41.182926177978516,
      "activations/layer9_attention_weight_max": 32.19500732421875,
      "activations/layer9_attention_weight_min": -31.327701568603516,
      "epoch": 11.94,
      "learning_rate": 7.375909090909091e-05,
      "loss": 2.7948,
      "step": 205450
    },
    {
      "activations/layer0_attention_weight_max": 16.44645118713379,
      "activations/layer0_attention_weight_min": -11.977151870727539,
      "activations/layer10_attention_weight_max": 31.851165771484375,
      "activations/layer10_attention_weight_min": -34.91342544555664,
      "activations/layer11_attention_weight_max": 32.76707077026367,
      "activations/layer11_attention_weight_min": -33.267208099365234,
      "activations/layer12_attention_weight_max": 23.809524536132812,
      "activations/layer12_attention_weight_min": -26.15646743774414,
      "activations/layer13_attention_weight_max": 38.48677062988281,
      "activations/layer13_attention_weight_min": -35.31736373901367,
      "activations/layer14_attention_weight_max": 40.355838775634766,
      "activations/layer14_attention_weight_min": -35.60844039916992,
      "activations/layer15_attention_weight_max": 34.05939483642578,
      "activations/layer15_attention_weight_min": -31.477643966674805,
      "activations/layer16_attention_weight_max": 28.67586898803711,
      "activations/layer16_attention_weight_min": -27.315004348754883,
      "activations/layer17_attention_weight_max": 34.21828842163086,
      "activations/layer17_attention_weight_min": -25.164701461791992,
      "activations/layer18_attention_weight_max": 29.47698402404785,
      "activations/layer18_attention_weight_min": -23.295734405517578,
      "activations/layer19_attention_weight_max": 33.145137786865234,
      "activations/layer19_attention_weight_min": -29.975723266601562,
      "activations/layer1_attention_weight_max": 16.066768646240234,
      "activations/layer1_attention_weight_min": -14.116148948669434,
      "activations/layer20_attention_weight_max": 28.498733520507812,
      "activations/layer20_attention_weight_min": -25.3472843170166,
      "activations/layer21_attention_weight_max": 28.44243621826172,
      "activations/layer21_attention_weight_min": -23.083406448364258,
      "activations/layer22_attention_weight_max": 44.77845764160156,
      "activations/layer22_attention_weight_min": -29.228349685668945,
      "activations/layer23_attention_weight_max": 34.92699432373047,
      "activations/layer23_attention_weight_min": -23.907859802246094,
      "activations/layer2_attention_weight_max": 34.08989334106445,
      "activations/layer2_attention_weight_min": -32.16303253173828,
      "activations/layer3_attention_weight_max": 95.61774444580078,
      "activations/layer3_attention_weight_min": -96.89857482910156,
      "activations/layer4_attention_weight_max": 107.61711883544922,
      "activations/layer4_attention_weight_min": -110.64759826660156,
      "activations/layer5_attention_weight_max": 50.78498840332031,
      "activations/layer5_attention_weight_min": -62.931575775146484,
      "activations/layer6_attention_weight_max": 44.069984436035156,
      "activations/layer6_attention_weight_min": -46.97324752807617,
      "activations/layer7_attention_weight_max": 89.51238250732422,
      "activations/layer7_attention_weight_min": -92.88363647460938,
      "activations/layer8_attention_weight_max": 38.59951400756836,
      "activations/layer8_attention_weight_min": -44.87188720703125,
      "activations/layer9_attention_weight_max": 32.26348114013672,
      "activations/layer9_attention_weight_min": -33.663700103759766,
      "epoch": 11.94,
      "learning_rate": 7.374015151515151e-05,
      "loss": 2.7702,
      "step": 205500
    },
    {
      "activations/layer0_attention_weight_max": 14.612584114074707,
      "activations/layer0_attention_weight_min": -12.606629371643066,
      "activations/layer10_attention_weight_max": 44.10331344604492,
      "activations/layer10_attention_weight_min": -45.086944580078125,
      "activations/layer11_attention_weight_max": 40.529563903808594,
      "activations/layer11_attention_weight_min": -45.5095329284668,
      "activations/layer12_attention_weight_max": 27.21886444091797,
      "activations/layer12_attention_weight_min": -31.133729934692383,
      "activations/layer13_attention_weight_max": 45.33483123779297,
      "activations/layer13_attention_weight_min": -42.582374572753906,
      "activations/layer14_attention_weight_max": 46.570068359375,
      "activations/layer14_attention_weight_min": -43.16569137573242,
      "activations/layer15_attention_weight_max": 42.38600540161133,
      "activations/layer15_attention_weight_min": -39.72197723388672,
      "activations/layer16_attention_weight_max": 30.54866600036621,
      "activations/layer16_attention_weight_min": -28.95301055908203,
      "activations/layer17_attention_weight_max": 33.80061340332031,
      "activations/layer17_attention_weight_min": -30.33489990234375,
      "activations/layer18_attention_weight_max": 29.2868595123291,
      "activations/layer18_attention_weight_min": -25.807790756225586,
      "activations/layer19_attention_weight_max": 34.570003509521484,
      "activations/layer19_attention_weight_min": -31.634017944335938,
      "activations/layer1_attention_weight_max": 17.891359329223633,
      "activations/layer1_attention_weight_min": -16.275314331054688,
      "activations/layer20_attention_weight_max": 27.76067543029785,
      "activations/layer20_attention_weight_min": -23.658971786499023,
      "activations/layer21_attention_weight_max": 29.424680709838867,
      "activations/layer21_attention_weight_min": -24.49251365661621,
      "activations/layer22_attention_weight_max": 39.867431640625,
      "activations/layer22_attention_weight_min": -29.227739334106445,
      "activations/layer23_attention_weight_max": 34.97310256958008,
      "activations/layer23_attention_weight_min": -23.791723251342773,
      "activations/layer2_attention_weight_max": 40.04151153564453,
      "activations/layer2_attention_weight_min": -41.21119689941406,
      "activations/layer3_attention_weight_max": 120.25894927978516,
      "activations/layer3_attention_weight_min": -116.33533477783203,
      "activations/layer4_attention_weight_max": 134.9304962158203,
      "activations/layer4_attention_weight_min": -134.4276123046875,
      "activations/layer5_attention_weight_max": 58.43755340576172,
      "activations/layer5_attention_weight_min": -62.20371627807617,
      "activations/layer6_attention_weight_max": 51.44059371948242,
      "activations/layer6_attention_weight_min": -51.55244445800781,
      "activations/layer7_attention_weight_max": 108.0712661743164,
      "activations/layer7_attention_weight_min": -108.3533935546875,
      "activations/layer8_attention_weight_max": 55.632328033447266,
      "activations/layer8_attention_weight_min": -57.85997772216797,
      "activations/layer9_attention_weight_max": 47.206695556640625,
      "activations/layer9_attention_weight_min": -46.30044937133789,
      "epoch": 11.94,
      "learning_rate": 7.372121212121211e-05,
      "loss": 2.7871,
      "step": 205550
    },
    {
      "activations/layer0_attention_weight_max": 16.509645462036133,
      "activations/layer0_attention_weight_min": -12.099796295166016,
      "activations/layer10_attention_weight_max": 35.74237823486328,
      "activations/layer10_attention_weight_min": -32.9211311340332,
      "activations/layer11_attention_weight_max": 40.10485076904297,
      "activations/layer11_attention_weight_min": -35.0675163269043,
      "activations/layer12_attention_weight_max": 24.69951629638672,
      "activations/layer12_attention_weight_min": -28.820823669433594,
      "activations/layer13_attention_weight_max": 44.194252014160156,
      "activations/layer13_attention_weight_min": -36.9110107421875,
      "activations/layer14_attention_weight_max": 44.438026428222656,
      "activations/layer14_attention_weight_min": -39.00403594970703,
      "activations/layer15_attention_weight_max": 45.80880355834961,
      "activations/layer15_attention_weight_min": -36.38243103027344,
      "activations/layer16_attention_weight_max": 31.00738525390625,
      "activations/layer16_attention_weight_min": -28.297542572021484,
      "activations/layer17_attention_weight_max": 30.822628021240234,
      "activations/layer17_attention_weight_min": -30.567386627197266,
      "activations/layer18_attention_weight_max": 30.231185913085938,
      "activations/layer18_attention_weight_min": -25.656234741210938,
      "activations/layer19_attention_weight_max": 34.42115020751953,
      "activations/layer19_attention_weight_min": -32.3381462097168,
      "activations/layer1_attention_weight_max": 17.675310134887695,
      "activations/layer1_attention_weight_min": -14.36136245727539,
      "activations/layer20_attention_weight_max": 29.904727935791016,
      "activations/layer20_attention_weight_min": -25.26951789855957,
      "activations/layer21_attention_weight_max": 25.53688621520996,
      "activations/layer21_attention_weight_min": -24.92704963684082,
      "activations/layer22_attention_weight_max": 38.86945343017578,
      "activations/layer22_attention_weight_min": -31.11155891418457,
      "activations/layer23_attention_weight_max": 30.83876609802246,
      "activations/layer23_attention_weight_min": -26.37457847595215,
      "activations/layer2_attention_weight_max": 33.41788101196289,
      "activations/layer2_attention_weight_min": -33.131080627441406,
      "activations/layer3_attention_weight_max": 97.5560073852539,
      "activations/layer3_attention_weight_min": -96.6308364868164,
      "activations/layer4_attention_weight_max": 117.6844711303711,
      "activations/layer4_attention_weight_min": -113.13056945800781,
      "activations/layer5_attention_weight_max": 54.11956787109375,
      "activations/layer5_attention_weight_min": -64.18953704833984,
      "activations/layer6_attention_weight_max": 46.89067840576172,
      "activations/layer6_attention_weight_min": -50.20686721801758,
      "activations/layer7_attention_weight_max": 92.83734893798828,
      "activations/layer7_attention_weight_min": -94.06820678710938,
      "activations/layer8_attention_weight_max": 43.47757339477539,
      "activations/layer8_attention_weight_min": -41.77027893066406,
      "activations/layer9_attention_weight_max": 33.97366714477539,
      "activations/layer9_attention_weight_min": -36.186580657958984,
      "epoch": 11.95,
      "learning_rate": 7.370227272727272e-05,
      "loss": 2.7755,
      "step": 205600
    },
    {
      "activations/layer0_attention_weight_max": 16.55286979675293,
      "activations/layer0_attention_weight_min": -13.183646202087402,
      "activations/layer10_attention_weight_max": 33.999183654785156,
      "activations/layer10_attention_weight_min": -30.636192321777344,
      "activations/layer11_attention_weight_max": 34.51030731201172,
      "activations/layer11_attention_weight_min": -32.218605041503906,
      "activations/layer12_attention_weight_max": 31.482898712158203,
      "activations/layer12_attention_weight_min": -26.267175674438477,
      "activations/layer13_attention_weight_max": 40.677913665771484,
      "activations/layer13_attention_weight_min": -34.87458419799805,
      "activations/layer14_attention_weight_max": 47.61994171142578,
      "activations/layer14_attention_weight_min": -37.594303131103516,
      "activations/layer15_attention_weight_max": 38.07695007324219,
      "activations/layer15_attention_weight_min": -32.73361587524414,
      "activations/layer16_attention_weight_max": 35.41257858276367,
      "activations/layer16_attention_weight_min": -26.61497688293457,
      "activations/layer17_attention_weight_max": 31.764747619628906,
      "activations/layer17_attention_weight_min": -26.555429458618164,
      "activations/layer18_attention_weight_max": 31.629051208496094,
      "activations/layer18_attention_weight_min": -22.570552825927734,
      "activations/layer19_attention_weight_max": 35.38663864135742,
      "activations/layer19_attention_weight_min": -29.827701568603516,
      "activations/layer1_attention_weight_max": 19.712162017822266,
      "activations/layer1_attention_weight_min": -17.16962242126465,
      "activations/layer20_attention_weight_max": 29.844032287597656,
      "activations/layer20_attention_weight_min": -23.31835174560547,
      "activations/layer21_attention_weight_max": 29.628143310546875,
      "activations/layer21_attention_weight_min": -23.196533203125,
      "activations/layer22_attention_weight_max": 44.55291748046875,
      "activations/layer22_attention_weight_min": -29.04667091369629,
      "activations/layer23_attention_weight_max": 33.42099380493164,
      "activations/layer23_attention_weight_min": -23.622623443603516,
      "activations/layer2_attention_weight_max": 36.84418869018555,
      "activations/layer2_attention_weight_min": -34.415306091308594,
      "activations/layer3_attention_weight_max": 97.16331481933594,
      "activations/layer3_attention_weight_min": -96.04002380371094,
      "activations/layer4_attention_weight_max": 111.88280487060547,
      "activations/layer4_attention_weight_min": -112.5821304321289,
      "activations/layer5_attention_weight_max": 51.818294525146484,
      "activations/layer5_attention_weight_min": -62.07014083862305,
      "activations/layer6_attention_weight_max": 46.50812911987305,
      "activations/layer6_attention_weight_min": -48.62487030029297,
      "activations/layer7_attention_weight_max": 90.85987091064453,
      "activations/layer7_attention_weight_min": -94.67656707763672,
      "activations/layer8_attention_weight_max": 41.108978271484375,
      "activations/layer8_attention_weight_min": -40.87586975097656,
      "activations/layer9_attention_weight_max": 34.07089614868164,
      "activations/layer9_attention_weight_min": -31.23978042602539,
      "epoch": 11.95,
      "learning_rate": 7.368333333333333e-05,
      "loss": 2.7838,
      "step": 205650
    },
    {
      "activations/layer0_attention_weight_max": 16.100486755371094,
      "activations/layer0_attention_weight_min": -11.40938949584961,
      "activations/layer10_attention_weight_max": 31.742435455322266,
      "activations/layer10_attention_weight_min": -34.76912307739258,
      "activations/layer11_attention_weight_max": 33.609039306640625,
      "activations/layer11_attention_weight_min": -34.2685432434082,
      "activations/layer12_attention_weight_max": 27.297334671020508,
      "activations/layer12_attention_weight_min": -25.041534423828125,
      "activations/layer13_attention_weight_max": 40.3524169921875,
      "activations/layer13_attention_weight_min": -34.77381896972656,
      "activations/layer14_attention_weight_max": 42.12765884399414,
      "activations/layer14_attention_weight_min": -36.199607849121094,
      "activations/layer15_attention_weight_max": 38.960391998291016,
      "activations/layer15_attention_weight_min": -33.32661819458008,
      "activations/layer16_attention_weight_max": 31.331787109375,
      "activations/layer16_attention_weight_min": -29.02041244506836,
      "activations/layer17_attention_weight_max": 32.53771209716797,
      "activations/layer17_attention_weight_min": -26.041479110717773,
      "activations/layer18_attention_weight_max": 30.30727767944336,
      "activations/layer18_attention_weight_min": -25.83371925354004,
      "activations/layer19_attention_weight_max": 34.371002197265625,
      "activations/layer19_attention_weight_min": -32.119728088378906,
      "activations/layer1_attention_weight_max": 17.656024932861328,
      "activations/layer1_attention_weight_min": -16.89563751220703,
      "activations/layer20_attention_weight_max": 30.829259872436523,
      "activations/layer20_attention_weight_min": -25.172183990478516,
      "activations/layer21_attention_weight_max": 30.155168533325195,
      "activations/layer21_attention_weight_min": -24.872331619262695,
      "activations/layer22_attention_weight_max": 45.54470443725586,
      "activations/layer22_attention_weight_min": -31.71727180480957,
      "activations/layer23_attention_weight_max": 34.85820007324219,
      "activations/layer23_attention_weight_min": -25.126523971557617,
      "activations/layer2_attention_weight_max": 33.441585540771484,
      "activations/layer2_attention_weight_min": -31.14447593688965,
      "activations/layer3_attention_weight_max": 95.59016418457031,
      "activations/layer3_attention_weight_min": -94.43599700927734,
      "activations/layer4_attention_weight_max": 115.39542388916016,
      "activations/layer4_attention_weight_min": -113.96758270263672,
      "activations/layer5_attention_weight_max": 51.748695373535156,
      "activations/layer5_attention_weight_min": -59.737770080566406,
      "activations/layer6_attention_weight_max": 44.319725036621094,
      "activations/layer6_attention_weight_min": -46.618751525878906,
      "activations/layer7_attention_weight_max": 94.9709701538086,
      "activations/layer7_attention_weight_min": -87.8099136352539,
      "activations/layer8_attention_weight_max": 41.491844177246094,
      "activations/layer8_attention_weight_min": -39.388607025146484,
      "activations/layer9_attention_weight_max": 32.123531341552734,
      "activations/layer9_attention_weight_min": -32.65341567993164,
      "epoch": 11.95,
      "learning_rate": 7.366439393939393e-05,
      "loss": 2.7729,
      "step": 205700
    },
    {
      "activations/layer0_attention_weight_max": 15.858946800231934,
      "activations/layer0_attention_weight_min": -11.817639350891113,
      "activations/layer10_attention_weight_max": 31.310503005981445,
      "activations/layer10_attention_weight_min": -31.452529907226562,
      "activations/layer11_attention_weight_max": 31.572551727294922,
      "activations/layer11_attention_weight_min": -32.284263610839844,
      "activations/layer12_attention_weight_max": 23.901248931884766,
      "activations/layer12_attention_weight_min": -24.711509704589844,
      "activations/layer13_attention_weight_max": 36.83910369873047,
      "activations/layer13_attention_weight_min": -33.40233612060547,
      "activations/layer14_attention_weight_max": 44.580101013183594,
      "activations/layer14_attention_weight_min": -38.86418914794922,
      "activations/layer15_attention_weight_max": 33.457130432128906,
      "activations/layer15_attention_weight_min": -31.62266731262207,
      "activations/layer16_attention_weight_max": 30.916934967041016,
      "activations/layer16_attention_weight_min": -28.821229934692383,
      "activations/layer17_attention_weight_max": 29.956079483032227,
      "activations/layer17_attention_weight_min": -26.131206512451172,
      "activations/layer18_attention_weight_max": 29.717552185058594,
      "activations/layer18_attention_weight_min": -24.399681091308594,
      "activations/layer19_attention_weight_max": 34.893699645996094,
      "activations/layer19_attention_weight_min": -33.5736198425293,
      "activations/layer1_attention_weight_max": 16.510244369506836,
      "activations/layer1_attention_weight_min": -14.666643142700195,
      "activations/layer20_attention_weight_max": 28.551042556762695,
      "activations/layer20_attention_weight_min": -25.726516723632812,
      "activations/layer21_attention_weight_max": 27.70058250427246,
      "activations/layer21_attention_weight_min": -23.810733795166016,
      "activations/layer22_attention_weight_max": 42.38032150268555,
      "activations/layer22_attention_weight_min": -30.262720108032227,
      "activations/layer23_attention_weight_max": 33.372642517089844,
      "activations/layer23_attention_weight_min": -25.28701400756836,
      "activations/layer2_attention_weight_max": 33.7276725769043,
      "activations/layer2_attention_weight_min": -33.25275421142578,
      "activations/layer3_attention_weight_max": 94.663330078125,
      "activations/layer3_attention_weight_min": -95.76457214355469,
      "activations/layer4_attention_weight_max": 108.41497039794922,
      "activations/layer4_attention_weight_min": -115.55633544921875,
      "activations/layer5_attention_weight_max": 48.733482360839844,
      "activations/layer5_attention_weight_min": -63.8530387878418,
      "activations/layer6_attention_weight_max": 44.42256164550781,
      "activations/layer6_attention_weight_min": -47.34074020385742,
      "activations/layer7_attention_weight_max": 93.94361877441406,
      "activations/layer7_attention_weight_min": -95.39154052734375,
      "activations/layer8_attention_weight_max": 42.785701751708984,
      "activations/layer8_attention_weight_min": -40.92434310913086,
      "activations/layer9_attention_weight_max": 33.229854583740234,
      "activations/layer9_attention_weight_min": -32.27383041381836,
      "epoch": 11.96,
      "learning_rate": 7.364583333333332e-05,
      "loss": 2.7852,
      "step": 205750
    },
    {
      "activations/layer0_attention_weight_max": 16.13261604309082,
      "activations/layer0_attention_weight_min": -12.281408309936523,
      "activations/layer10_attention_weight_max": 35.5460319519043,
      "activations/layer10_attention_weight_min": -33.3609504699707,
      "activations/layer11_attention_weight_max": 38.53265380859375,
      "activations/layer11_attention_weight_min": -33.01164627075195,
      "activations/layer12_attention_weight_max": 32.13663864135742,
      "activations/layer12_attention_weight_min": -26.468900680541992,
      "activations/layer13_attention_weight_max": 38.57867431640625,
      "activations/layer13_attention_weight_min": -35.33427047729492,
      "activations/layer14_attention_weight_max": 46.32036209106445,
      "activations/layer14_attention_weight_min": -39.32322311401367,
      "activations/layer15_attention_weight_max": 38.11503219604492,
      "activations/layer15_attention_weight_min": -34.657344818115234,
      "activations/layer16_attention_weight_max": 33.20932388305664,
      "activations/layer16_attention_weight_min": -28.712745666503906,
      "activations/layer17_attention_weight_max": 31.03294563293457,
      "activations/layer17_attention_weight_min": -26.66290855407715,
      "activations/layer18_attention_weight_max": 32.57645034790039,
      "activations/layer18_attention_weight_min": -23.16527557373047,
      "activations/layer19_attention_weight_max": 40.00457763671875,
      "activations/layer19_attention_weight_min": -30.14655876159668,
      "activations/layer1_attention_weight_max": 17.215496063232422,
      "activations/layer1_attention_weight_min": -15.433670997619629,
      "activations/layer20_attention_weight_max": 28.50079345703125,
      "activations/layer20_attention_weight_min": -24.710710525512695,
      "activations/layer21_attention_weight_max": 29.183835983276367,
      "activations/layer21_attention_weight_min": -23.26444435119629,
      "activations/layer22_attention_weight_max": 45.71589660644531,
      "activations/layer22_attention_weight_min": -28.870725631713867,
      "activations/layer23_attention_weight_max": 33.455440521240234,
      "activations/layer23_attention_weight_min": -24.90097427368164,
      "activations/layer2_attention_weight_max": 36.42171859741211,
      "activations/layer2_attention_weight_min": -34.52078628540039,
      "activations/layer3_attention_weight_max": 100.322509765625,
      "activations/layer3_attention_weight_min": -95.26810455322266,
      "activations/layer4_attention_weight_max": 111.85981750488281,
      "activations/layer4_attention_weight_min": -109.09197235107422,
      "activations/layer5_attention_weight_max": 52.43479919433594,
      "activations/layer5_attention_weight_min": -58.29635238647461,
      "activations/layer6_attention_weight_max": 47.00397491455078,
      "activations/layer6_attention_weight_min": -50.89098358154297,
      "activations/layer7_attention_weight_max": 95.1026840209961,
      "activations/layer7_attention_weight_min": -92.539794921875,
      "activations/layer8_attention_weight_max": 44.96944808959961,
      "activations/layer8_attention_weight_min": -43.58740234375,
      "activations/layer9_attention_weight_max": 34.961917877197266,
      "activations/layer9_attention_weight_min": -34.03371810913086,
      "epoch": 11.96,
      "learning_rate": 7.362689393939393e-05,
      "loss": 2.7831,
      "step": 205800
    },
    {
      "activations/layer0_attention_weight_max": 15.784332275390625,
      "activations/layer0_attention_weight_min": -13.333126068115234,
      "activations/layer10_attention_weight_max": 32.31732177734375,
      "activations/layer10_attention_weight_min": -33.038246154785156,
      "activations/layer11_attention_weight_max": 34.23837661743164,
      "activations/layer11_attention_weight_min": -32.519569396972656,
      "activations/layer12_attention_weight_max": 28.33359146118164,
      "activations/layer12_attention_weight_min": -25.458723068237305,
      "activations/layer13_attention_weight_max": 37.73603439331055,
      "activations/layer13_attention_weight_min": -34.66096496582031,
      "activations/layer14_attention_weight_max": 45.220367431640625,
      "activations/layer14_attention_weight_min": -39.458892822265625,
      "activations/layer15_attention_weight_max": 34.69866180419922,
      "activations/layer15_attention_weight_min": -32.419254302978516,
      "activations/layer16_attention_weight_max": 29.226173400878906,
      "activations/layer16_attention_weight_min": -28.15174674987793,
      "activations/layer17_attention_weight_max": 31.838708877563477,
      "activations/layer17_attention_weight_min": -25.741195678710938,
      "activations/layer18_attention_weight_max": 31.02456283569336,
      "activations/layer18_attention_weight_min": -21.88654136657715,
      "activations/layer19_attention_weight_max": 32.57044219970703,
      "activations/layer19_attention_weight_min": -30.249649047851562,
      "activations/layer1_attention_weight_max": 16.906686782836914,
      "activations/layer1_attention_weight_min": -14.036479949951172,
      "activations/layer20_attention_weight_max": 30.949844360351562,
      "activations/layer20_attention_weight_min": -24.97825813293457,
      "activations/layer21_attention_weight_max": 30.09147071838379,
      "activations/layer21_attention_weight_min": -23.444744110107422,
      "activations/layer22_attention_weight_max": 44.046165466308594,
      "activations/layer22_attention_weight_min": -27.8742733001709,
      "activations/layer23_attention_weight_max": 35.05783462524414,
      "activations/layer23_attention_weight_min": -25.409337997436523,
      "activations/layer2_attention_weight_max": 31.696346282958984,
      "activations/layer2_attention_weight_min": -33.857704162597656,
      "activations/layer3_attention_weight_max": 89.32415008544922,
      "activations/layer3_attention_weight_min": -93.4164810180664,
      "activations/layer4_attention_weight_max": 103.07106018066406,
      "activations/layer4_attention_weight_min": -104.1404037475586,
      "activations/layer5_attention_weight_max": 51.710567474365234,
      "activations/layer5_attention_weight_min": -60.451290130615234,
      "activations/layer6_attention_weight_max": 42.32014465332031,
      "activations/layer6_attention_weight_min": -45.56020736694336,
      "activations/layer7_attention_weight_max": 88.27873229980469,
      "activations/layer7_attention_weight_min": -92.4823989868164,
      "activations/layer8_attention_weight_max": 40.53093338012695,
      "activations/layer8_attention_weight_min": -42.39501190185547,
      "activations/layer9_attention_weight_max": 33.99457931518555,
      "activations/layer9_attention_weight_min": -34.40364074707031,
      "epoch": 11.96,
      "learning_rate": 7.360795454545454e-05,
      "loss": 2.7957,
      "step": 205850
    },
    {
      "activations/layer0_attention_weight_max": 15.94819164276123,
      "activations/layer0_attention_weight_min": -12.366923332214355,
      "activations/layer10_attention_weight_max": 40.92351150512695,
      "activations/layer10_attention_weight_min": -36.44319152832031,
      "activations/layer11_attention_weight_max": 48.26560974121094,
      "activations/layer11_attention_weight_min": -37.64635467529297,
      "activations/layer12_attention_weight_max": 31.792011260986328,
      "activations/layer12_attention_weight_min": -26.62058448791504,
      "activations/layer13_attention_weight_max": 44.851619720458984,
      "activations/layer13_attention_weight_min": -36.760406494140625,
      "activations/layer14_attention_weight_max": 54.079627990722656,
      "activations/layer14_attention_weight_min": -41.0990104675293,
      "activations/layer15_attention_weight_max": 41.26302719116211,
      "activations/layer15_attention_weight_min": -35.97339630126953,
      "activations/layer16_attention_weight_max": 33.27809143066406,
      "activations/layer16_attention_weight_min": -30.62779998779297,
      "activations/layer17_attention_weight_max": 33.014129638671875,
      "activations/layer17_attention_weight_min": -27.763484954833984,
      "activations/layer18_attention_weight_max": 32.15624237060547,
      "activations/layer18_attention_weight_min": -24.488887786865234,
      "activations/layer19_attention_weight_max": 39.33367919921875,
      "activations/layer19_attention_weight_min": -30.651304244995117,
      "activations/layer1_attention_weight_max": 17.18073081970215,
      "activations/layer1_attention_weight_min": -14.251054763793945,
      "activations/layer20_attention_weight_max": 31.058868408203125,
      "activations/layer20_attention_weight_min": -24.953712463378906,
      "activations/layer21_attention_weight_max": 29.563034057617188,
      "activations/layer21_attention_weight_min": -23.377182006835938,
      "activations/layer22_attention_weight_max": 44.84353256225586,
      "activations/layer22_attention_weight_min": -30.239810943603516,
      "activations/layer23_attention_weight_max": 35.654319763183594,
      "activations/layer23_attention_weight_min": -24.660221099853516,
      "activations/layer2_attention_weight_max": 33.82398223876953,
      "activations/layer2_attention_weight_min": -34.35193634033203,
      "activations/layer3_attention_weight_max": 91.42406463623047,
      "activations/layer3_attention_weight_min": -94.1459732055664,
      "activations/layer4_attention_weight_max": 111.97078704833984,
      "activations/layer4_attention_weight_min": -115.11812591552734,
      "activations/layer5_attention_weight_max": 54.410072326660156,
      "activations/layer5_attention_weight_min": -62.24268341064453,
      "activations/layer6_attention_weight_max": 47.81452560424805,
      "activations/layer6_attention_weight_min": -53.25527572631836,
      "activations/layer7_attention_weight_max": 100.33011627197266,
      "activations/layer7_attention_weight_min": -98.25220489501953,
      "activations/layer8_attention_weight_max": 48.24622344970703,
      "activations/layer8_attention_weight_min": -47.40782165527344,
      "activations/layer9_attention_weight_max": 41.68239974975586,
      "activations/layer9_attention_weight_min": -36.31235122680664,
      "epoch": 11.96,
      "learning_rate": 7.358901515151514e-05,
      "loss": 2.7798,
      "step": 205900
    },
    {
      "activations/layer0_attention_weight_max": 15.910853385925293,
      "activations/layer0_attention_weight_min": -11.640570640563965,
      "activations/layer10_attention_weight_max": 34.08159255981445,
      "activations/layer10_attention_weight_min": -31.285247802734375,
      "activations/layer11_attention_weight_max": 33.40011215209961,
      "activations/layer11_attention_weight_min": -30.73231315612793,
      "activations/layer12_attention_weight_max": 26.62305450439453,
      "activations/layer12_attention_weight_min": -26.268943786621094,
      "activations/layer13_attention_weight_max": 41.64229965209961,
      "activations/layer13_attention_weight_min": -32.51838302612305,
      "activations/layer14_attention_weight_max": 43.703025817871094,
      "activations/layer14_attention_weight_min": -37.48185348510742,
      "activations/layer15_attention_weight_max": 37.466407775878906,
      "activations/layer15_attention_weight_min": -30.85421371459961,
      "activations/layer16_attention_weight_max": 30.785388946533203,
      "activations/layer16_attention_weight_min": -26.634288787841797,
      "activations/layer17_attention_weight_max": 34.67558670043945,
      "activations/layer17_attention_weight_min": -25.05109214782715,
      "activations/layer18_attention_weight_max": 33.589881896972656,
      "activations/layer18_attention_weight_min": -24.005420684814453,
      "activations/layer19_attention_weight_max": 31.7846622467041,
      "activations/layer19_attention_weight_min": -30.247806549072266,
      "activations/layer1_attention_weight_max": 16.726306915283203,
      "activations/layer1_attention_weight_min": -14.462867736816406,
      "activations/layer20_attention_weight_max": 30.08433723449707,
      "activations/layer20_attention_weight_min": -24.68685531616211,
      "activations/layer21_attention_weight_max": 31.456052780151367,
      "activations/layer21_attention_weight_min": -24.533153533935547,
      "activations/layer22_attention_weight_max": 45.84185791015625,
      "activations/layer22_attention_weight_min": -28.28864097595215,
      "activations/layer23_attention_weight_max": 35.54490661621094,
      "activations/layer23_attention_weight_min": -25.476606369018555,
      "activations/layer2_attention_weight_max": 34.316184997558594,
      "activations/layer2_attention_weight_min": -32.92749786376953,
      "activations/layer3_attention_weight_max": 94.42237091064453,
      "activations/layer3_attention_weight_min": -93.31875610351562,
      "activations/layer4_attention_weight_max": 108.85042572021484,
      "activations/layer4_attention_weight_min": -110.35426330566406,
      "activations/layer5_attention_weight_max": 50.15426254272461,
      "activations/layer5_attention_weight_min": -62.00236511230469,
      "activations/layer6_attention_weight_max": 45.01543426513672,
      "activations/layer6_attention_weight_min": -45.26240539550781,
      "activations/layer7_attention_weight_max": 92.92984008789062,
      "activations/layer7_attention_weight_min": -91.77234649658203,
      "activations/layer8_attention_weight_max": 42.481868743896484,
      "activations/layer8_attention_weight_min": -40.941978454589844,
      "activations/layer9_attention_weight_max": 36.93463897705078,
      "activations/layer9_attention_weight_min": -32.86159896850586,
      "epoch": 11.97,
      "learning_rate": 7.357007575757575e-05,
      "loss": 2.7669,
      "step": 205950
    },
    {
      "activations/layer0_attention_weight_max": 15.592254638671875,
      "activations/layer0_attention_weight_min": -12.544888496398926,
      "activations/layer10_attention_weight_max": 33.2199821472168,
      "activations/layer10_attention_weight_min": -33.92036437988281,
      "activations/layer11_attention_weight_max": 32.374813079833984,
      "activations/layer11_attention_weight_min": -33.98141860961914,
      "activations/layer12_attention_weight_max": 26.633268356323242,
      "activations/layer12_attention_weight_min": -26.01178550720215,
      "activations/layer13_attention_weight_max": 38.521156311035156,
      "activations/layer13_attention_weight_min": -35.62502670288086,
      "activations/layer14_attention_weight_max": 49.814613342285156,
      "activations/layer14_attention_weight_min": -40.43144607543945,
      "activations/layer15_attention_weight_max": 38.79790496826172,
      "activations/layer15_attention_weight_min": -35.67638397216797,
      "activations/layer16_attention_weight_max": 32.122344970703125,
      "activations/layer16_attention_weight_min": -29.57145118713379,
      "activations/layer17_attention_weight_max": 33.801822662353516,
      "activations/layer17_attention_weight_min": -27.34786033630371,
      "activations/layer18_attention_weight_max": 35.059967041015625,
      "activations/layer18_attention_weight_min": -23.194284439086914,
      "activations/layer19_attention_weight_max": 34.41039276123047,
      "activations/layer19_attention_weight_min": -31.303281784057617,
      "activations/layer1_attention_weight_max": 16.635589599609375,
      "activations/layer1_attention_weight_min": -15.102446556091309,
      "activations/layer20_attention_weight_max": 30.354690551757812,
      "activations/layer20_attention_weight_min": -24.584659576416016,
      "activations/layer21_attention_weight_max": 30.668119430541992,
      "activations/layer21_attention_weight_min": -22.939014434814453,
      "activations/layer22_attention_weight_max": 44.04385757446289,
      "activations/layer22_attention_weight_min": -29.86502456665039,
      "activations/layer23_attention_weight_max": 33.65675354003906,
      "activations/layer23_attention_weight_min": -24.921138763427734,
      "activations/layer2_attention_weight_max": 33.74717712402344,
      "activations/layer2_attention_weight_min": -32.412940979003906,
      "activations/layer3_attention_weight_max": 94.28858184814453,
      "activations/layer3_attention_weight_min": -94.28089904785156,
      "activations/layer4_attention_weight_max": 114.0698471069336,
      "activations/layer4_attention_weight_min": -108.4190902709961,
      "activations/layer5_attention_weight_max": 52.571739196777344,
      "activations/layer5_attention_weight_min": -60.56168746948242,
      "activations/layer6_attention_weight_max": 44.48796081542969,
      "activations/layer6_attention_weight_min": -46.682884216308594,
      "activations/layer7_attention_weight_max": 91.85171508789062,
      "activations/layer7_attention_weight_min": -88.01221466064453,
      "activations/layer8_attention_weight_max": 42.008419036865234,
      "activations/layer8_attention_weight_min": -42.63343811035156,
      "activations/layer9_attention_weight_max": 34.058807373046875,
      "activations/layer9_attention_weight_min": -35.240943908691406,
      "epoch": 11.97,
      "learning_rate": 7.355113636363635e-05,
      "loss": 2.7705,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.4695,
      "eval_samples_per_second": 506.997,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.4695,
      "eval_openwebtext_samples_per_second": 506.997,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0598,
      "eval_wikitext_samples_per_second": 221.386,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.6213,
      "eval_lambada_samples_per_second": 506.065,
      "step": 206000
    },
    {
      "activations/layer0_attention_weight_max": 15.3789644241333,
      "activations/layer0_attention_weight_min": -12.296826362609863,
      "activations/layer10_attention_weight_max": 33.189483642578125,
      "activations/layer10_attention_weight_min": -33.871646881103516,
      "activations/layer11_attention_weight_max": 33.49250793457031,
      "activations/layer11_attention_weight_min": -33.02998352050781,
      "activations/layer12_attention_weight_max": 29.890169143676758,
      "activations/layer12_attention_weight_min": -26.87818145751953,
      "activations/layer13_attention_weight_max": 38.68269348144531,
      "activations/layer13_attention_weight_min": -35.201480865478516,
      "activations/layer14_attention_weight_max": 41.99216842651367,
      "activations/layer14_attention_weight_min": -41.635276794433594,
      "activations/layer15_attention_weight_max": 36.40147399902344,
      "activations/layer15_attention_weight_min": -35.13255310058594,
      "activations/layer16_attention_weight_max": 33.04286575317383,
      "activations/layer16_attention_weight_min": -30.392772674560547,
      "activations/layer17_attention_weight_max": 30.2703857421875,
      "activations/layer17_attention_weight_min": -28.91182518005371,
      "activations/layer18_attention_weight_max": 34.687644958496094,
      "activations/layer18_attention_weight_min": -23.881114959716797,
      "activations/layer19_attention_weight_max": 37.70533752441406,
      "activations/layer19_attention_weight_min": -30.12738609313965,
      "activations/layer1_attention_weight_max": 16.378753662109375,
      "activations/layer1_attention_weight_min": -15.005741119384766,
      "activations/layer20_attention_weight_max": 28.289836883544922,
      "activations/layer20_attention_weight_min": -24.378063201904297,
      "activations/layer21_attention_weight_max": 28.0592041015625,
      "activations/layer21_attention_weight_min": -24.03244972229004,
      "activations/layer22_attention_weight_max": 49.360198974609375,
      "activations/layer22_attention_weight_min": -30.102754592895508,
      "activations/layer23_attention_weight_max": 34.55362319946289,
      "activations/layer23_attention_weight_min": -24.969810485839844,
      "activations/layer2_attention_weight_max": 32.8382568359375,
      "activations/layer2_attention_weight_min": -33.432952880859375,
      "activations/layer3_attention_weight_max": 93.1182632446289,
      "activations/layer3_attention_weight_min": -93.36235809326172,
      "activations/layer4_attention_weight_max": 109.81917572021484,
      "activations/layer4_attention_weight_min": -112.32618713378906,
      "activations/layer5_attention_weight_max": 50.7689323425293,
      "activations/layer5_attention_weight_min": -62.257850646972656,
      "activations/layer6_attention_weight_max": 43.015830993652344,
      "activations/layer6_attention_weight_min": -47.2975959777832,
      "activations/layer7_attention_weight_max": 86.9637222290039,
      "activations/layer7_attention_weight_min": -87.0146713256836,
      "activations/layer8_attention_weight_max": 38.07939529418945,
      "activations/layer8_attention_weight_min": -40.120643615722656,
      "activations/layer9_attention_weight_max": 33.854248046875,
      "activations/layer9_attention_weight_min": -33.27226638793945,
      "epoch": 11.97,
      "learning_rate": 7.353219696969697e-05,
      "loss": 2.7633,
      "step": 206050
    },
    {
      "activations/layer0_attention_weight_max": 16.23575782775879,
      "activations/layer0_attention_weight_min": -12.470346450805664,
      "activations/layer10_attention_weight_max": 35.77781295776367,
      "activations/layer10_attention_weight_min": -33.01005172729492,
      "activations/layer11_attention_weight_max": 35.91208267211914,
      "activations/layer11_attention_weight_min": -32.29668045043945,
      "activations/layer12_attention_weight_max": 25.51742172241211,
      "activations/layer12_attention_weight_min": -26.37725067138672,
      "activations/layer13_attention_weight_max": 42.00877380371094,
      "activations/layer13_attention_weight_min": -36.32172393798828,
      "activations/layer14_attention_weight_max": 47.51643371582031,
      "activations/layer14_attention_weight_min": -39.64724349975586,
      "activations/layer15_attention_weight_max": 38.0513916015625,
      "activations/layer15_attention_weight_min": -34.570682525634766,
      "activations/layer16_attention_weight_max": 38.201656341552734,
      "activations/layer16_attention_weight_min": -28.238786697387695,
      "activations/layer17_attention_weight_max": 35.0744514465332,
      "activations/layer17_attention_weight_min": -26.63555335998535,
      "activations/layer18_attention_weight_max": 31.899858474731445,
      "activations/layer18_attention_weight_min": -23.116483688354492,
      "activations/layer19_attention_weight_max": 36.501529693603516,
      "activations/layer19_attention_weight_min": -30.241073608398438,
      "activations/layer1_attention_weight_max": 17.234432220458984,
      "activations/layer1_attention_weight_min": -15.032752990722656,
      "activations/layer20_attention_weight_max": 33.3625373840332,
      "activations/layer20_attention_weight_min": -25.6619930267334,
      "activations/layer21_attention_weight_max": 33.06642150878906,
      "activations/layer21_attention_weight_min": -24.308849334716797,
      "activations/layer22_attention_weight_max": 49.390350341796875,
      "activations/layer22_attention_weight_min": -29.287275314331055,
      "activations/layer23_attention_weight_max": 39.6652717590332,
      "activations/layer23_attention_weight_min": -25.531402587890625,
      "activations/layer2_attention_weight_max": 33.60274124145508,
      "activations/layer2_attention_weight_min": -32.754493713378906,
      "activations/layer3_attention_weight_max": 99.2822265625,
      "activations/layer3_attention_weight_min": -99.4749526977539,
      "activations/layer4_attention_weight_max": 118.62096405029297,
      "activations/layer4_attention_weight_min": -113.67927551269531,
      "activations/layer5_attention_weight_max": 55.093238830566406,
      "activations/layer5_attention_weight_min": -60.049278259277344,
      "activations/layer6_attention_weight_max": 48.16275405883789,
      "activations/layer6_attention_weight_min": -52.99235534667969,
      "activations/layer7_attention_weight_max": 99.70086669921875,
      "activations/layer7_attention_weight_min": -91.59058380126953,
      "activations/layer8_attention_weight_max": 44.42195510864258,
      "activations/layer8_attention_weight_min": -42.93859100341797,
      "activations/layer9_attention_weight_max": 35.87903594970703,
      "activations/layer9_attention_weight_min": -33.66441345214844,
      "epoch": 11.98,
      "learning_rate": 7.351325757575757e-05,
      "loss": 2.7792,
      "step": 206100
    },
    {
      "activations/layer0_attention_weight_max": 16.108652114868164,
      "activations/layer0_attention_weight_min": -12.718259811401367,
      "activations/layer10_attention_weight_max": 33.6856689453125,
      "activations/layer10_attention_weight_min": -35.627079010009766,
      "activations/layer11_attention_weight_max": 32.44679260253906,
      "activations/layer11_attention_weight_min": -31.996658325195312,
      "activations/layer12_attention_weight_max": 43.36801528930664,
      "activations/layer12_attention_weight_min": -27.429000854492188,
      "activations/layer13_attention_weight_max": 52.623695373535156,
      "activations/layer13_attention_weight_min": -36.23762512207031,
      "activations/layer14_attention_weight_max": 43.508609771728516,
      "activations/layer14_attention_weight_min": -38.163658142089844,
      "activations/layer15_attention_weight_max": 42.01321029663086,
      "activations/layer15_attention_weight_min": -36.73603057861328,
      "activations/layer16_attention_weight_max": 30.653032302856445,
      "activations/layer16_attention_weight_min": -27.102439880371094,
      "activations/layer17_attention_weight_max": 27.942405700683594,
      "activations/layer17_attention_weight_min": -26.277603149414062,
      "activations/layer18_attention_weight_max": 30.21917152404785,
      "activations/layer18_attention_weight_min": -22.581527709960938,
      "activations/layer19_attention_weight_max": 33.19615936279297,
      "activations/layer19_attention_weight_min": -30.811965942382812,
      "activations/layer1_attention_weight_max": 16.77834701538086,
      "activations/layer1_attention_weight_min": -16.140836715698242,
      "activations/layer20_attention_weight_max": 28.366058349609375,
      "activations/layer20_attention_weight_min": -24.094205856323242,
      "activations/layer21_attention_weight_max": 27.901042938232422,
      "activations/layer21_attention_weight_min": -22.640432357788086,
      "activations/layer22_attention_weight_max": 40.97972869873047,
      "activations/layer22_attention_weight_min": -28.908832550048828,
      "activations/layer23_attention_weight_max": 33.86762619018555,
      "activations/layer23_attention_weight_min": -24.33480453491211,
      "activations/layer2_attention_weight_max": 33.332054138183594,
      "activations/layer2_attention_weight_min": -32.08570861816406,
      "activations/layer3_attention_weight_max": 96.19994354248047,
      "activations/layer3_attention_weight_min": -94.37063598632812,
      "activations/layer4_attention_weight_max": 113.53887939453125,
      "activations/layer4_attention_weight_min": -110.29396057128906,
      "activations/layer5_attention_weight_max": 52.555694580078125,
      "activations/layer5_attention_weight_min": -59.21198272705078,
      "activations/layer6_attention_weight_max": 46.53712463378906,
      "activations/layer6_attention_weight_min": -47.17093276977539,
      "activations/layer7_attention_weight_max": 92.18378448486328,
      "activations/layer7_attention_weight_min": -89.56938934326172,
      "activations/layer8_attention_weight_max": 41.986045837402344,
      "activations/layer8_attention_weight_min": -43.420780181884766,
      "activations/layer9_attention_weight_max": 34.47819137573242,
      "activations/layer9_attention_weight_min": -36.50771713256836,
      "epoch": 11.98,
      "learning_rate": 7.349431818181818e-05,
      "loss": 2.7554,
      "step": 206150
    },
    {
      "activations/layer0_attention_weight_max": 16.07660675048828,
      "activations/layer0_attention_weight_min": -11.997512817382812,
      "activations/layer10_attention_weight_max": 30.64037322998047,
      "activations/layer10_attention_weight_min": -32.160064697265625,
      "activations/layer11_attention_weight_max": 30.484487533569336,
      "activations/layer11_attention_weight_min": -29.660018920898438,
      "activations/layer12_attention_weight_max": 22.952938079833984,
      "activations/layer12_attention_weight_min": -25.40966033935547,
      "activations/layer13_attention_weight_max": 35.58041000366211,
      "activations/layer13_attention_weight_min": -33.2844352722168,
      "activations/layer14_attention_weight_max": 42.866905212402344,
      "activations/layer14_attention_weight_min": -37.439517974853516,
      "activations/layer15_attention_weight_max": 34.857460021972656,
      "activations/layer15_attention_weight_min": -32.46253204345703,
      "activations/layer16_attention_weight_max": 30.195051193237305,
      "activations/layer16_attention_weight_min": -29.319772720336914,
      "activations/layer17_attention_weight_max": 29.96759033203125,
      "activations/layer17_attention_weight_min": -28.263843536376953,
      "activations/layer18_attention_weight_max": 29.17547035217285,
      "activations/layer18_attention_weight_min": -24.82305145263672,
      "activations/layer19_attention_weight_max": 31.54023551940918,
      "activations/layer19_attention_weight_min": -31.894351959228516,
      "activations/layer1_attention_weight_max": 16.90616798400879,
      "activations/layer1_attention_weight_min": -14.090118408203125,
      "activations/layer20_attention_weight_max": 29.382904052734375,
      "activations/layer20_attention_weight_min": -25.96734046936035,
      "activations/layer21_attention_weight_max": 27.1583309173584,
      "activations/layer21_attention_weight_min": -25.902860641479492,
      "activations/layer22_attention_weight_max": 43.02920150756836,
      "activations/layer22_attention_weight_min": -30.26914405822754,
      "activations/layer23_attention_weight_max": 31.65607452392578,
      "activations/layer23_attention_weight_min": -24.795591354370117,
      "activations/layer2_attention_weight_max": 31.440526962280273,
      "activations/layer2_attention_weight_min": -32.73540496826172,
      "activations/layer3_attention_weight_max": 91.33418273925781,
      "activations/layer3_attention_weight_min": -94.86157989501953,
      "activations/layer4_attention_weight_max": 110.4448471069336,
      "activations/layer4_attention_weight_min": -116.81062316894531,
      "activations/layer5_attention_weight_max": 50.620361328125,
      "activations/layer5_attention_weight_min": -61.716583251953125,
      "activations/layer6_attention_weight_max": 45.058311462402344,
      "activations/layer6_attention_weight_min": -46.14997482299805,
      "activations/layer7_attention_weight_max": 87.54932403564453,
      "activations/layer7_attention_weight_min": -91.82560729980469,
      "activations/layer8_attention_weight_max": 37.62419128417969,
      "activations/layer8_attention_weight_min": -40.635494232177734,
      "activations/layer9_attention_weight_max": 31.07777976989746,
      "activations/layer9_attention_weight_min": -32.04835510253906,
      "epoch": 11.98,
      "learning_rate": 7.347537878787879e-05,
      "loss": 2.7694,
      "step": 206200
    },
    {
      "activations/layer0_attention_weight_max": 15.271552085876465,
      "activations/layer0_attention_weight_min": -12.274770736694336,
      "activations/layer10_attention_weight_max": 31.92963218688965,
      "activations/layer10_attention_weight_min": -30.825881958007812,
      "activations/layer11_attention_weight_max": 31.83028221130371,
      "activations/layer11_attention_weight_min": -31.345951080322266,
      "activations/layer12_attention_weight_max": 25.97178077697754,
      "activations/layer12_attention_weight_min": -26.19391441345215,
      "activations/layer13_attention_weight_max": 42.06803894042969,
      "activations/layer13_attention_weight_min": -35.902015686035156,
      "activations/layer14_attention_weight_max": 46.7049560546875,
      "activations/layer14_attention_weight_min": -37.1165885925293,
      "activations/layer15_attention_weight_max": 36.77508544921875,
      "activations/layer15_attention_weight_min": -32.62977981567383,
      "activations/layer16_attention_weight_max": 32.61897277832031,
      "activations/layer16_attention_weight_min": -28.491378784179688,
      "activations/layer17_attention_weight_max": 32.281558990478516,
      "activations/layer17_attention_weight_min": -26.090356826782227,
      "activations/layer18_attention_weight_max": 33.611000061035156,
      "activations/layer18_attention_weight_min": -24.165775299072266,
      "activations/layer19_attention_weight_max": 35.57815170288086,
      "activations/layer19_attention_weight_min": -31.898818969726562,
      "activations/layer1_attention_weight_max": 15.697036743164062,
      "activations/layer1_attention_weight_min": -14.836711883544922,
      "activations/layer20_attention_weight_max": 31.31249237060547,
      "activations/layer20_attention_weight_min": -26.42738914489746,
      "activations/layer21_attention_weight_max": 29.7960262298584,
      "activations/layer21_attention_weight_min": -25.765844345092773,
      "activations/layer22_attention_weight_max": 51.041744232177734,
      "activations/layer22_attention_weight_min": -30.87952423095703,
      "activations/layer23_attention_weight_max": 35.233489990234375,
      "activations/layer23_attention_weight_min": -27.746158599853516,
      "activations/layer2_attention_weight_max": 32.476470947265625,
      "activations/layer2_attention_weight_min": -33.17575454711914,
      "activations/layer3_attention_weight_max": 93.2530288696289,
      "activations/layer3_attention_weight_min": -95.7090835571289,
      "activations/layer4_attention_weight_max": 114.65985107421875,
      "activations/layer4_attention_weight_min": -115.01006317138672,
      "activations/layer5_attention_weight_max": 52.543113708496094,
      "activations/layer5_attention_weight_min": -62.859432220458984,
      "activations/layer6_attention_weight_max": 46.772884368896484,
      "activations/layer6_attention_weight_min": -48.23429870605469,
      "activations/layer7_attention_weight_max": 91.02803039550781,
      "activations/layer7_attention_weight_min": -94.29933166503906,
      "activations/layer8_attention_weight_max": 39.533817291259766,
      "activations/layer8_attention_weight_min": -42.993370056152344,
      "activations/layer9_attention_weight_max": 33.90923309326172,
      "activations/layer9_attention_weight_min": -32.601192474365234,
      "epoch": 11.98,
      "learning_rate": 7.345643939393939e-05,
      "loss": 2.7806,
      "step": 206250
    },
    {
      "activations/layer0_attention_weight_max": 15.906451225280762,
      "activations/layer0_attention_weight_min": -12.017195701599121,
      "activations/layer10_attention_weight_max": 35.467796325683594,
      "activations/layer10_attention_weight_min": -31.54767417907715,
      "activations/layer11_attention_weight_max": 34.25483703613281,
      "activations/layer11_attention_weight_min": -31.214073181152344,
      "activations/layer12_attention_weight_max": 24.144975662231445,
      "activations/layer12_attention_weight_min": -26.335908889770508,
      "activations/layer13_attention_weight_max": 37.81277084350586,
      "activations/layer13_attention_weight_min": -32.723873138427734,
      "activations/layer14_attention_weight_max": 42.04228973388672,
      "activations/layer14_attention_weight_min": -37.34140396118164,
      "activations/layer15_attention_weight_max": 34.26691818237305,
      "activations/layer15_attention_weight_min": -32.37783432006836,
      "activations/layer16_attention_weight_max": 35.79636764526367,
      "activations/layer16_attention_weight_min": -28.135860443115234,
      "activations/layer17_attention_weight_max": 28.715288162231445,
      "activations/layer17_attention_weight_min": -25.336841583251953,
      "activations/layer18_attention_weight_max": 29.407211303710938,
      "activations/layer18_attention_weight_min": -26.38216781616211,
      "activations/layer19_attention_weight_max": 36.50844955444336,
      "activations/layer19_attention_weight_min": -31.233240127563477,
      "activations/layer1_attention_weight_max": 16.877256393432617,
      "activations/layer1_attention_weight_min": -14.349536895751953,
      "activations/layer20_attention_weight_max": 31.497591018676758,
      "activations/layer20_attention_weight_min": -26.21319007873535,
      "activations/layer21_attention_weight_max": 28.567832946777344,
      "activations/layer21_attention_weight_min": -24.52126121520996,
      "activations/layer22_attention_weight_max": 49.17192459106445,
      "activations/layer22_attention_weight_min": -30.4873104095459,
      "activations/layer23_attention_weight_max": 32.212947845458984,
      "activations/layer23_attention_weight_min": -23.914167404174805,
      "activations/layer2_attention_weight_max": 35.96682357788086,
      "activations/layer2_attention_weight_min": -33.845428466796875,
      "activations/layer3_attention_weight_max": 97.50914001464844,
      "activations/layer3_attention_weight_min": -98.98550415039062,
      "activations/layer4_attention_weight_max": 110.46475982666016,
      "activations/layer4_attention_weight_min": -112.36751556396484,
      "activations/layer5_attention_weight_max": 58.84337615966797,
      "activations/layer5_attention_weight_min": -60.7835693359375,
      "activations/layer6_attention_weight_max": 47.16801071166992,
      "activations/layer6_attention_weight_min": -45.92892074584961,
      "activations/layer7_attention_weight_max": 91.742431640625,
      "activations/layer7_attention_weight_min": -94.60081481933594,
      "activations/layer8_attention_weight_max": 41.001976013183594,
      "activations/layer8_attention_weight_min": -40.290977478027344,
      "activations/layer9_attention_weight_max": 35.32719421386719,
      "activations/layer9_attention_weight_min": -32.726585388183594,
      "epoch": 11.99,
      "learning_rate": 7.343749999999999e-05,
      "loss": 2.7614,
      "step": 206300
    },
    {
      "activations/layer0_attention_weight_max": 16.373069763183594,
      "activations/layer0_attention_weight_min": -12.310284614562988,
      "activations/layer10_attention_weight_max": 31.227142333984375,
      "activations/layer10_attention_weight_min": -31.307159423828125,
      "activations/layer11_attention_weight_max": 32.13256072998047,
      "activations/layer11_attention_weight_min": -31.22211456298828,
      "activations/layer12_attention_weight_max": 23.27413558959961,
      "activations/layer12_attention_weight_min": -24.956058502197266,
      "activations/layer13_attention_weight_max": 36.620113372802734,
      "activations/layer13_attention_weight_min": -34.479103088378906,
      "activations/layer14_attention_weight_max": 46.0120964050293,
      "activations/layer14_attention_weight_min": -38.293235778808594,
      "activations/layer15_attention_weight_max": 34.05984115600586,
      "activations/layer15_attention_weight_min": -31.9769229888916,
      "activations/layer16_attention_weight_max": 28.93644905090332,
      "activations/layer16_attention_weight_min": -29.614459991455078,
      "activations/layer17_attention_weight_max": 29.950761795043945,
      "activations/layer17_attention_weight_min": -25.23150062561035,
      "activations/layer18_attention_weight_max": 30.531667709350586,
      "activations/layer18_attention_weight_min": -22.99920654296875,
      "activations/layer19_attention_weight_max": 33.75402069091797,
      "activations/layer19_attention_weight_min": -32.42283630371094,
      "activations/layer1_attention_weight_max": 15.637462615966797,
      "activations/layer1_attention_weight_min": -14.794171333312988,
      "activations/layer20_attention_weight_max": 29.97264289855957,
      "activations/layer20_attention_weight_min": -27.160316467285156,
      "activations/layer21_attention_weight_max": 27.581205368041992,
      "activations/layer21_attention_weight_min": -26.1289005279541,
      "activations/layer22_attention_weight_max": 46.58644104003906,
      "activations/layer22_attention_weight_min": -31.15294075012207,
      "activations/layer23_attention_weight_max": 34.63687515258789,
      "activations/layer23_attention_weight_min": -24.38570785522461,
      "activations/layer2_attention_weight_max": 34.401729583740234,
      "activations/layer2_attention_weight_min": -32.18142318725586,
      "activations/layer3_attention_weight_max": 94.30839538574219,
      "activations/layer3_attention_weight_min": -93.55070495605469,
      "activations/layer4_attention_weight_max": 110.06558990478516,
      "activations/layer4_attention_weight_min": -109.7014389038086,
      "activations/layer5_attention_weight_max": 50.730384826660156,
      "activations/layer5_attention_weight_min": -59.11370086669922,
      "activations/layer6_attention_weight_max": 43.382266998291016,
      "activations/layer6_attention_weight_min": -47.16079330444336,
      "activations/layer7_attention_weight_max": 86.6374740600586,
      "activations/layer7_attention_weight_min": -89.19161224365234,
      "activations/layer8_attention_weight_max": 39.58022689819336,
      "activations/layer8_attention_weight_min": -40.73681640625,
      "activations/layer9_attention_weight_max": 32.890010833740234,
      "activations/layer9_attention_weight_min": -31.993961334228516,
      "epoch": 11.99,
      "learning_rate": 7.34185606060606e-05,
      "loss": 2.7789,
      "step": 206350
    },
    {
      "activations/layer0_attention_weight_max": 15.618526458740234,
      "activations/layer0_attention_weight_min": -12.83992862701416,
      "activations/layer10_attention_weight_max": 35.08914566040039,
      "activations/layer10_attention_weight_min": -32.97910690307617,
      "activations/layer11_attention_weight_max": 36.25815200805664,
      "activations/layer11_attention_weight_min": -33.750831604003906,
      "activations/layer12_attention_weight_max": 26.689531326293945,
      "activations/layer12_attention_weight_min": -25.049779891967773,
      "activations/layer13_attention_weight_max": 40.31934356689453,
      "activations/layer13_attention_weight_min": -35.56303024291992,
      "activations/layer14_attention_weight_max": 41.54438018798828,
      "activations/layer14_attention_weight_min": -36.329933166503906,
      "activations/layer15_attention_weight_max": 37.411258697509766,
      "activations/layer15_attention_weight_min": -33.14946365356445,
      "activations/layer16_attention_weight_max": 29.779504776000977,
      "activations/layer16_attention_weight_min": -27.82874298095703,
      "activations/layer17_attention_weight_max": 30.03525161743164,
      "activations/layer17_attention_weight_min": -26.429838180541992,
      "activations/layer18_attention_weight_max": 30.209623336791992,
      "activations/layer18_attention_weight_min": -23.272294998168945,
      "activations/layer19_attention_weight_max": 35.03920364379883,
      "activations/layer19_attention_weight_min": -30.83005714416504,
      "activations/layer1_attention_weight_max": 17.6314640045166,
      "activations/layer1_attention_weight_min": -14.674245834350586,
      "activations/layer20_attention_weight_max": 29.914377212524414,
      "activations/layer20_attention_weight_min": -25.817785263061523,
      "activations/layer21_attention_weight_max": 29.221126556396484,
      "activations/layer21_attention_weight_min": -26.178997039794922,
      "activations/layer22_attention_weight_max": 41.07198715209961,
      "activations/layer22_attention_weight_min": -29.696815490722656,
      "activations/layer23_attention_weight_max": 33.27783966064453,
      "activations/layer23_attention_weight_min": -26.142927169799805,
      "activations/layer2_attention_weight_max": 32.68855285644531,
      "activations/layer2_attention_weight_min": -33.03118896484375,
      "activations/layer3_attention_weight_max": 92.28028106689453,
      "activations/layer3_attention_weight_min": -92.402587890625,
      "activations/layer4_attention_weight_max": 111.6084213256836,
      "activations/layer4_attention_weight_min": -109.70201873779297,
      "activations/layer5_attention_weight_max": 49.05709457397461,
      "activations/layer5_attention_weight_min": -59.363059997558594,
      "activations/layer6_attention_weight_max": 42.61007308959961,
      "activations/layer6_attention_weight_min": -44.60722351074219,
      "activations/layer7_attention_weight_max": 88.93047332763672,
      "activations/layer7_attention_weight_min": -88.49717712402344,
      "activations/layer8_attention_weight_max": 41.03163528442383,
      "activations/layer8_attention_weight_min": -40.5471305847168,
      "activations/layer9_attention_weight_max": 35.37162399291992,
      "activations/layer9_attention_weight_min": -33.14313888549805,
      "epoch": 11.99,
      "learning_rate": 7.33996212121212e-05,
      "loss": 2.7681,
      "step": 206400
    },
    {
      "activations/layer0_attention_weight_max": 16.13213539123535,
      "activations/layer0_attention_weight_min": -11.801054954528809,
      "activations/layer10_attention_weight_max": 32.244972229003906,
      "activations/layer10_attention_weight_min": -36.392364501953125,
      "activations/layer11_attention_weight_max": 32.94937515258789,
      "activations/layer11_attention_weight_min": -34.35051727294922,
      "activations/layer12_attention_weight_max": 25.1314754486084,
      "activations/layer12_attention_weight_min": -25.12706756591797,
      "activations/layer13_attention_weight_max": 38.08037567138672,
      "activations/layer13_attention_weight_min": -33.34884262084961,
      "activations/layer14_attention_weight_max": 44.14192199707031,
      "activations/layer14_attention_weight_min": -35.86530685424805,
      "activations/layer15_attention_weight_max": 34.79022216796875,
      "activations/layer15_attention_weight_min": -31.80031967163086,
      "activations/layer16_attention_weight_max": 29.472824096679688,
      "activations/layer16_attention_weight_min": -27.895050048828125,
      "activations/layer17_attention_weight_max": 32.20264434814453,
      "activations/layer17_attention_weight_min": -27.45443344116211,
      "activations/layer18_attention_weight_max": 33.06867218017578,
      "activations/layer18_attention_weight_min": -27.1058292388916,
      "activations/layer19_attention_weight_max": 34.378135681152344,
      "activations/layer19_attention_weight_min": -31.70931053161621,
      "activations/layer1_attention_weight_max": 16.29410171508789,
      "activations/layer1_attention_weight_min": -15.800305366516113,
      "activations/layer20_attention_weight_max": 29.373247146606445,
      "activations/layer20_attention_weight_min": -25.474658966064453,
      "activations/layer21_attention_weight_max": 29.622989654541016,
      "activations/layer21_attention_weight_min": -26.040813446044922,
      "activations/layer22_attention_weight_max": 42.18217849731445,
      "activations/layer22_attention_weight_min": -30.589874267578125,
      "activations/layer23_attention_weight_max": 35.91642761230469,
      "activations/layer23_attention_weight_min": -26.87374496459961,
      "activations/layer2_attention_weight_max": 33.863494873046875,
      "activations/layer2_attention_weight_min": -33.843017578125,
      "activations/layer3_attention_weight_max": 97.0892562866211,
      "activations/layer3_attention_weight_min": -95.40550231933594,
      "activations/layer4_attention_weight_max": 117.5018310546875,
      "activations/layer4_attention_weight_min": -110.11637115478516,
      "activations/layer5_attention_weight_max": 53.70843505859375,
      "activations/layer5_attention_weight_min": -61.196407318115234,
      "activations/layer6_attention_weight_max": 45.940608978271484,
      "activations/layer6_attention_weight_min": -45.37797164916992,
      "activations/layer7_attention_weight_max": 94.40141296386719,
      "activations/layer7_attention_weight_min": -89.38945770263672,
      "activations/layer8_attention_weight_max": 41.98612594604492,
      "activations/layer8_attention_weight_min": -42.26675796508789,
      "activations/layer9_attention_weight_max": 32.52778244018555,
      "activations/layer9_attention_weight_min": -36.12277603149414,
      "epoch": 12.0,
      "learning_rate": 7.338068181818181e-05,
      "loss": 2.7793,
      "step": 206450
    },
    {
      "activations/layer0_attention_weight_max": 16.706632614135742,
      "activations/layer0_attention_weight_min": -12.24052619934082,
      "activations/layer10_attention_weight_max": 37.03373336791992,
      "activations/layer10_attention_weight_min": -32.45259094238281,
      "activations/layer11_attention_weight_max": 34.12721252441406,
      "activations/layer11_attention_weight_min": -33.511497497558594,
      "activations/layer12_attention_weight_max": 26.79491424560547,
      "activations/layer12_attention_weight_min": -24.9694766998291,
      "activations/layer13_attention_weight_max": 39.18248748779297,
      "activations/layer13_attention_weight_min": -33.75498962402344,
      "activations/layer14_attention_weight_max": 45.45591735839844,
      "activations/layer14_attention_weight_min": -39.73530578613281,
      "activations/layer15_attention_weight_max": 36.94251251220703,
      "activations/layer15_attention_weight_min": -31.227462768554688,
      "activations/layer16_attention_weight_max": 30.131114959716797,
      "activations/layer16_attention_weight_min": -28.2657527923584,
      "activations/layer17_attention_weight_max": 29.9290771484375,
      "activations/layer17_attention_weight_min": -24.9632568359375,
      "activations/layer18_attention_weight_max": 31.507057189941406,
      "activations/layer18_attention_weight_min": -23.951326370239258,
      "activations/layer19_attention_weight_max": 34.9207878112793,
      "activations/layer19_attention_weight_min": -30.00579833984375,
      "activations/layer1_attention_weight_max": 16.346235275268555,
      "activations/layer1_attention_weight_min": -15.045929908752441,
      "activations/layer20_attention_weight_max": 28.363346099853516,
      "activations/layer20_attention_weight_min": -24.1429500579834,
      "activations/layer21_attention_weight_max": 29.336265563964844,
      "activations/layer21_attention_weight_min": -22.752878189086914,
      "activations/layer22_attention_weight_max": 43.66404342651367,
      "activations/layer22_attention_weight_min": -30.076908111572266,
      "activations/layer23_attention_weight_max": 34.2114143371582,
      "activations/layer23_attention_weight_min": -24.4882869720459,
      "activations/layer2_attention_weight_max": 31.735687255859375,
      "activations/layer2_attention_weight_min": -31.051109313964844,
      "activations/layer3_attention_weight_max": 93.16110229492188,
      "activations/layer3_attention_weight_min": -94.21226501464844,
      "activations/layer4_attention_weight_max": 111.85205078125,
      "activations/layer4_attention_weight_min": -111.51189422607422,
      "activations/layer5_attention_weight_max": 51.583858489990234,
      "activations/layer5_attention_weight_min": -61.142250061035156,
      "activations/layer6_attention_weight_max": 45.23082733154297,
      "activations/layer6_attention_weight_min": -46.01755905151367,
      "activations/layer7_attention_weight_max": 93.13582611083984,
      "activations/layer7_attention_weight_min": -93.9433822631836,
      "activations/layer8_attention_weight_max": 39.68857955932617,
      "activations/layer8_attention_weight_min": -41.21391296386719,
      "activations/layer9_attention_weight_max": 33.73885726928711,
      "activations/layer9_attention_weight_min": -35.161582946777344,
      "epoch": 12.0,
      "learning_rate": 7.336174242424241e-05,
      "loss": 2.7694,
      "step": 206500
    },
    {
      "activations/layer0_attention_weight_max": 16.778846740722656,
      "activations/layer0_attention_weight_min": -11.882547378540039,
      "activations/layer10_attention_weight_max": 29.522966384887695,
      "activations/layer10_attention_weight_min": -32.436561584472656,
      "activations/layer11_attention_weight_max": 29.061412811279297,
      "activations/layer11_attention_weight_min": -29.640729904174805,
      "activations/layer12_attention_weight_max": 22.898832321166992,
      "activations/layer12_attention_weight_min": -24.663562774658203,
      "activations/layer13_attention_weight_max": 35.365394592285156,
      "activations/layer13_attention_weight_min": -33.097503662109375,
      "activations/layer14_attention_weight_max": 37.74879455566406,
      "activations/layer14_attention_weight_min": -34.710548400878906,
      "activations/layer15_attention_weight_max": 32.7725944519043,
      "activations/layer15_attention_weight_min": -31.686767578125,
      "activations/layer16_attention_weight_max": 29.02591323852539,
      "activations/layer16_attention_weight_min": -27.12944984436035,
      "activations/layer17_attention_weight_max": 32.188594818115234,
      "activations/layer17_attention_weight_min": -23.590641021728516,
      "activations/layer18_attention_weight_max": 30.821720123291016,
      "activations/layer18_attention_weight_min": -22.11450958251953,
      "activations/layer19_attention_weight_max": 28.916372299194336,
      "activations/layer19_attention_weight_min": -28.52585220336914,
      "activations/layer1_attention_weight_max": 16.553918838500977,
      "activations/layer1_attention_weight_min": -13.648055076599121,
      "activations/layer20_attention_weight_max": 26.42100715637207,
      "activations/layer20_attention_weight_min": -24.290786743164062,
      "activations/layer21_attention_weight_max": 28.544700622558594,
      "activations/layer21_attention_weight_min": -24.692747116088867,
      "activations/layer22_attention_weight_max": 41.834442138671875,
      "activations/layer22_attention_weight_min": -29.43390655517578,
      "activations/layer23_attention_weight_max": 30.44843292236328,
      "activations/layer23_attention_weight_min": -27.26004409790039,
      "activations/layer2_attention_weight_max": 33.61359405517578,
      "activations/layer2_attention_weight_min": -33.455257415771484,
      "activations/layer3_attention_weight_max": 97.89302825927734,
      "activations/layer3_attention_weight_min": -92.99590301513672,
      "activations/layer4_attention_weight_max": 107.977783203125,
      "activations/layer4_attention_weight_min": -107.3115463256836,
      "activations/layer5_attention_weight_max": 49.63344955444336,
      "activations/layer5_attention_weight_min": -60.29723358154297,
      "activations/layer6_attention_weight_max": 44.37626647949219,
      "activations/layer6_attention_weight_min": -45.13637161254883,
      "activations/layer7_attention_weight_max": 85.56770324707031,
      "activations/layer7_attention_weight_min": -86.41275024414062,
      "activations/layer8_attention_weight_max": 37.354888916015625,
      "activations/layer8_attention_weight_min": -41.02452087402344,
      "activations/layer9_attention_weight_max": 31.79144287109375,
      "activations/layer9_attention_weight_min": -31.227584838867188,
      "epoch": 12.0,
      "learning_rate": 7.334242424242423e-05,
      "loss": 2.8134,
      "step": 206550
    },
    {
      "activations/layer0_attention_weight_max": 16.427989959716797,
      "activations/layer0_attention_weight_min": -12.80713939666748,
      "activations/layer10_attention_weight_max": 30.8980770111084,
      "activations/layer10_attention_weight_min": -31.476957321166992,
      "activations/layer11_attention_weight_max": 30.887691497802734,
      "activations/layer11_attention_weight_min": -30.7606201171875,
      "activations/layer12_attention_weight_max": 26.899097442626953,
      "activations/layer12_attention_weight_min": -23.801225662231445,
      "activations/layer13_attention_weight_max": 41.21851348876953,
      "activations/layer13_attention_weight_min": -32.69031524658203,
      "activations/layer14_attention_weight_max": 42.76095199584961,
      "activations/layer14_attention_weight_min": -35.06401443481445,
      "activations/layer15_attention_weight_max": 37.44781494140625,
      "activations/layer15_attention_weight_min": -29.84938621520996,
      "activations/layer16_attention_weight_max": 32.23870849609375,
      "activations/layer16_attention_weight_min": -26.031583786010742,
      "activations/layer17_attention_weight_max": 30.762264251708984,
      "activations/layer17_attention_weight_min": -25.625091552734375,
      "activations/layer18_attention_weight_max": 31.661357879638672,
      "activations/layer18_attention_weight_min": -24.300289154052734,
      "activations/layer19_attention_weight_max": 35.0451774597168,
      "activations/layer19_attention_weight_min": -29.677602767944336,
      "activations/layer1_attention_weight_max": 16.8147029876709,
      "activations/layer1_attention_weight_min": -14.84167766571045,
      "activations/layer20_attention_weight_max": 29.32756805419922,
      "activations/layer20_attention_weight_min": -27.016721725463867,
      "activations/layer21_attention_weight_max": 29.623638153076172,
      "activations/layer21_attention_weight_min": -24.083402633666992,
      "activations/layer22_attention_weight_max": 42.64572525024414,
      "activations/layer22_attention_weight_min": -29.989004135131836,
      "activations/layer23_attention_weight_max": 33.085784912109375,
      "activations/layer23_attention_weight_min": -26.55101776123047,
      "activations/layer2_attention_weight_max": 32.51507568359375,
      "activations/layer2_attention_weight_min": -32.15601348876953,
      "activations/layer3_attention_weight_max": 95.06912231445312,
      "activations/layer3_attention_weight_min": -94.55390930175781,
      "activations/layer4_attention_weight_max": 106.34199523925781,
      "activations/layer4_attention_weight_min": -107.76287078857422,
      "activations/layer5_attention_weight_max": 50.785675048828125,
      "activations/layer5_attention_weight_min": -58.23591613769531,
      "activations/layer6_attention_weight_max": 41.693965911865234,
      "activations/layer6_attention_weight_min": -45.32500076293945,
      "activations/layer7_attention_weight_max": 82.69791412353516,
      "activations/layer7_attention_weight_min": -89.05160522460938,
      "activations/layer8_attention_weight_max": 36.57726287841797,
      "activations/layer8_attention_weight_min": -37.91801834106445,
      "activations/layer9_attention_weight_max": 31.55636978149414,
      "activations/layer9_attention_weight_min": -29.42698097229004,
      "epoch": 12.0,
      "learning_rate": 7.332348484848484e-05,
      "loss": 2.7531,
      "step": 206600
    },
    {
      "activations/layer0_attention_weight_max": 15.858390808105469,
      "activations/layer0_attention_weight_min": -13.047088623046875,
      "activations/layer10_attention_weight_max": 33.545249938964844,
      "activations/layer10_attention_weight_min": -32.15419006347656,
      "activations/layer11_attention_weight_max": 32.767417907714844,
      "activations/layer11_attention_weight_min": -31.37814712524414,
      "activations/layer12_attention_weight_max": 25.050764083862305,
      "activations/layer12_attention_weight_min": -27.51827621459961,
      "activations/layer13_attention_weight_max": 37.90861129760742,
      "activations/layer13_attention_weight_min": -34.742530822753906,
      "activations/layer14_attention_weight_max": 45.48805618286133,
      "activations/layer14_attention_weight_min": -42.773284912109375,
      "activations/layer15_attention_weight_max": 33.70377731323242,
      "activations/layer15_attention_weight_min": -32.39288330078125,
      "activations/layer16_attention_weight_max": 29.58106231689453,
      "activations/layer16_attention_weight_min": -29.963071823120117,
      "activations/layer17_attention_weight_max": 29.36020851135254,
      "activations/layer17_attention_weight_min": -24.110414505004883,
      "activations/layer18_attention_weight_max": 29.436540603637695,
      "activations/layer18_attention_weight_min": -22.53032684326172,
      "activations/layer19_attention_weight_max": 34.235164642333984,
      "activations/layer19_attention_weight_min": -30.920669555664062,
      "activations/layer1_attention_weight_max": 19.202083587646484,
      "activations/layer1_attention_weight_min": -16.8502254486084,
      "activations/layer20_attention_weight_max": 27.41594123840332,
      "activations/layer20_attention_weight_min": -23.67888641357422,
      "activations/layer21_attention_weight_max": 26.465713500976562,
      "activations/layer21_attention_weight_min": -23.71722412109375,
      "activations/layer22_attention_weight_max": 42.49669647216797,
      "activations/layer22_attention_weight_min": -31.788463592529297,
      "activations/layer23_attention_weight_max": 31.43250274658203,
      "activations/layer23_attention_weight_min": -24.06598663330078,
      "activations/layer2_attention_weight_max": 33.47269821166992,
      "activations/layer2_attention_weight_min": -31.603317260742188,
      "activations/layer3_attention_weight_max": 96.441162109375,
      "activations/layer3_attention_weight_min": -94.60755920410156,
      "activations/layer4_attention_weight_max": 115.16064453125,
      "activations/layer4_attention_weight_min": -109.99378967285156,
      "activations/layer5_attention_weight_max": 50.93013000488281,
      "activations/layer5_attention_weight_min": -66.16339874267578,
      "activations/layer6_attention_weight_max": 45.40514373779297,
      "activations/layer6_attention_weight_min": -48.350093841552734,
      "activations/layer7_attention_weight_max": 92.78749084472656,
      "activations/layer7_attention_weight_min": -89.45654296875,
      "activations/layer8_attention_weight_max": 40.58930969238281,
      "activations/layer8_attention_weight_min": -42.2228889465332,
      "activations/layer9_attention_weight_max": 34.26681137084961,
      "activations/layer9_attention_weight_min": -32.90021514892578,
      "epoch": 12.01,
      "learning_rate": 7.330454545454545e-05,
      "loss": 2.7713,
      "step": 206650
    },
    {
      "activations/layer0_attention_weight_max": 16.428848266601562,
      "activations/layer0_attention_weight_min": -12.32443904876709,
      "activations/layer10_attention_weight_max": 30.556718826293945,
      "activations/layer10_attention_weight_min": -30.356124877929688,
      "activations/layer11_attention_weight_max": 30.404958724975586,
      "activations/layer11_attention_weight_min": -30.220083236694336,
      "activations/layer12_attention_weight_max": 22.681848526000977,
      "activations/layer12_attention_weight_min": -23.93277931213379,
      "activations/layer13_attention_weight_max": 35.21919250488281,
      "activations/layer13_attention_weight_min": -31.68801498413086,
      "activations/layer14_attention_weight_max": 40.064605712890625,
      "activations/layer14_attention_weight_min": -34.593849182128906,
      "activations/layer15_attention_weight_max": 33.3790168762207,
      "activations/layer15_attention_weight_min": -32.46595764160156,
      "activations/layer16_attention_weight_max": 28.034679412841797,
      "activations/layer16_attention_weight_min": -26.21189308166504,
      "activations/layer17_attention_weight_max": 31.327863693237305,
      "activations/layer17_attention_weight_min": -24.651445388793945,
      "activations/layer18_attention_weight_max": 28.602779388427734,
      "activations/layer18_attention_weight_min": -21.58234405517578,
      "activations/layer19_attention_weight_max": 31.411691665649414,
      "activations/layer19_attention_weight_min": -29.866079330444336,
      "activations/layer1_attention_weight_max": 16.01273536682129,
      "activations/layer1_attention_weight_min": -14.610453605651855,
      "activations/layer20_attention_weight_max": 26.469791412353516,
      "activations/layer20_attention_weight_min": -23.760604858398438,
      "activations/layer21_attention_weight_max": 28.25617027282715,
      "activations/layer21_attention_weight_min": -22.464183807373047,
      "activations/layer22_attention_weight_max": 37.93841552734375,
      "activations/layer22_attention_weight_min": -29.27362060546875,
      "activations/layer23_attention_weight_max": 30.937421798706055,
      "activations/layer23_attention_weight_min": -24.471660614013672,
      "activations/layer2_attention_weight_max": 34.328575134277344,
      "activations/layer2_attention_weight_min": -33.871097564697266,
      "activations/layer3_attention_weight_max": 94.89405059814453,
      "activations/layer3_attention_weight_min": -97.82413482666016,
      "activations/layer4_attention_weight_max": 111.38140869140625,
      "activations/layer4_attention_weight_min": -109.28369140625,
      "activations/layer5_attention_weight_max": 48.64228439331055,
      "activations/layer5_attention_weight_min": -61.28095245361328,
      "activations/layer6_attention_weight_max": 42.934364318847656,
      "activations/layer6_attention_weight_min": -43.58384704589844,
      "activations/layer7_attention_weight_max": 85.9301528930664,
      "activations/layer7_attention_weight_min": -91.50853729248047,
      "activations/layer8_attention_weight_max": 38.329505920410156,
      "activations/layer8_attention_weight_min": -38.40303039550781,
      "activations/layer9_attention_weight_max": 29.971057891845703,
      "activations/layer9_attention_weight_min": -31.477590560913086,
      "epoch": 12.01,
      "learning_rate": 7.328560606060605e-05,
      "loss": 2.7568,
      "step": 206700
    },
    {
      "activations/layer0_attention_weight_max": 16.311233520507812,
      "activations/layer0_attention_weight_min": -12.313751220703125,
      "activations/layer10_attention_weight_max": 31.167715072631836,
      "activations/layer10_attention_weight_min": -33.40039825439453,
      "activations/layer11_attention_weight_max": 33.012611389160156,
      "activations/layer11_attention_weight_min": -32.29203414916992,
      "activations/layer12_attention_weight_max": 24.548049926757812,
      "activations/layer12_attention_weight_min": -25.450645446777344,
      "activations/layer13_attention_weight_max": 35.92539978027344,
      "activations/layer13_attention_weight_min": -32.09602355957031,
      "activations/layer14_attention_weight_max": 42.86616897583008,
      "activations/layer14_attention_weight_min": -35.729183197021484,
      "activations/layer15_attention_weight_max": 35.195804595947266,
      "activations/layer15_attention_weight_min": -33.68684387207031,
      "activations/layer16_attention_weight_max": 31.27172088623047,
      "activations/layer16_attention_weight_min": -28.069326400756836,
      "activations/layer17_attention_weight_max": 31.774011611938477,
      "activations/layer17_attention_weight_min": -27.813573837280273,
      "activations/layer18_attention_weight_max": 34.268280029296875,
      "activations/layer18_attention_weight_min": -26.870506286621094,
      "activations/layer19_attention_weight_max": 35.05615234375,
      "activations/layer19_attention_weight_min": -33.3339958190918,
      "activations/layer1_attention_weight_max": 16.12532615661621,
      "activations/layer1_attention_weight_min": -16.796001434326172,
      "activations/layer20_attention_weight_max": 30.956499099731445,
      "activations/layer20_attention_weight_min": -27.496482849121094,
      "activations/layer21_attention_weight_max": 30.327077865600586,
      "activations/layer21_attention_weight_min": -26.514616012573242,
      "activations/layer22_attention_weight_max": 47.83356475830078,
      "activations/layer22_attention_weight_min": -36.79564666748047,
      "activations/layer23_attention_weight_max": 38.45384979248047,
      "activations/layer23_attention_weight_min": -28.39410972595215,
      "activations/layer2_attention_weight_max": 33.8914794921875,
      "activations/layer2_attention_weight_min": -33.59950256347656,
      "activations/layer3_attention_weight_max": 92.21784210205078,
      "activations/layer3_attention_weight_min": -103.27791595458984,
      "activations/layer4_attention_weight_max": 110.84112548828125,
      "activations/layer4_attention_weight_min": -112.6813735961914,
      "activations/layer5_attention_weight_max": 49.61222457885742,
      "activations/layer5_attention_weight_min": -61.000999450683594,
      "activations/layer6_attention_weight_max": 42.75368881225586,
      "activations/layer6_attention_weight_min": -46.204349517822266,
      "activations/layer7_attention_weight_max": 85.50212097167969,
      "activations/layer7_attention_weight_min": -93.51620483398438,
      "activations/layer8_attention_weight_max": 40.04143524169922,
      "activations/layer8_attention_weight_min": -42.93082809448242,
      "activations/layer9_attention_weight_max": 31.368179321289062,
      "activations/layer9_attention_weight_min": -33.72050094604492,
      "epoch": 12.01,
      "learning_rate": 7.326666666666667e-05,
      "loss": 2.7582,
      "step": 206750
    },
    {
      "activations/layer0_attention_weight_max": 15.757655143737793,
      "activations/layer0_attention_weight_min": -12.375675201416016,
      "activations/layer10_attention_weight_max": 33.78142166137695,
      "activations/layer10_attention_weight_min": -32.39581298828125,
      "activations/layer11_attention_weight_max": 37.87278747558594,
      "activations/layer11_attention_weight_min": -33.971004486083984,
      "activations/layer12_attention_weight_max": 26.446428298950195,
      "activations/layer12_attention_weight_min": -27.08640480041504,
      "activations/layer13_attention_weight_max": 37.65603256225586,
      "activations/layer13_attention_weight_min": -36.50039291381836,
      "activations/layer14_attention_weight_max": 55.91893005371094,
      "activations/layer14_attention_weight_min": -40.50660705566406,
      "activations/layer15_attention_weight_max": 37.29771041870117,
      "activations/layer15_attention_weight_min": -34.6353645324707,
      "activations/layer16_attention_weight_max": 32.21127700805664,
      "activations/layer16_attention_weight_min": -29.347087860107422,
      "activations/layer17_attention_weight_max": 32.02817916870117,
      "activations/layer17_attention_weight_min": -26.68311882019043,
      "activations/layer18_attention_weight_max": 32.77336120605469,
      "activations/layer18_attention_weight_min": -23.673120498657227,
      "activations/layer19_attention_weight_max": 36.72074508666992,
      "activations/layer19_attention_weight_min": -33.49634552001953,
      "activations/layer1_attention_weight_max": 16.980669021606445,
      "activations/layer1_attention_weight_min": -14.758885383605957,
      "activations/layer20_attention_weight_max": 29.329038619995117,
      "activations/layer20_attention_weight_min": -29.447614669799805,
      "activations/layer21_attention_weight_max": 28.740848541259766,
      "activations/layer21_attention_weight_min": -25.34856605529785,
      "activations/layer22_attention_weight_max": 53.49057388305664,
      "activations/layer22_attention_weight_min": -34.208885192871094,
      "activations/layer23_attention_weight_max": 34.760101318359375,
      "activations/layer23_attention_weight_min": -28.540983200073242,
      "activations/layer2_attention_weight_max": 33.58775329589844,
      "activations/layer2_attention_weight_min": -31.614652633666992,
      "activations/layer3_attention_weight_max": 91.04878997802734,
      "activations/layer3_attention_weight_min": -95.3030014038086,
      "activations/layer4_attention_weight_max": 107.9586181640625,
      "activations/layer4_attention_weight_min": -109.64093017578125,
      "activations/layer5_attention_weight_max": 51.20249557495117,
      "activations/layer5_attention_weight_min": -59.14215087890625,
      "activations/layer6_attention_weight_max": 43.99971008300781,
      "activations/layer6_attention_weight_min": -45.756771087646484,
      "activations/layer7_attention_weight_max": 98.96699523925781,
      "activations/layer7_attention_weight_min": -88.45974731445312,
      "activations/layer8_attention_weight_max": 41.578147888183594,
      "activations/layer8_attention_weight_min": -42.261661529541016,
      "activations/layer9_attention_weight_max": 31.559045791625977,
      "activations/layer9_attention_weight_min": -31.913196563720703,
      "epoch": 12.02,
      "learning_rate": 7.324772727272727e-05,
      "loss": 2.7684,
      "step": 206800
    },
    {
      "activations/layer0_attention_weight_max": 16.314252853393555,
      "activations/layer0_attention_weight_min": -12.280170440673828,
      "activations/layer10_attention_weight_max": 34.632965087890625,
      "activations/layer10_attention_weight_min": -32.310081481933594,
      "activations/layer11_attention_weight_max": 33.14380645751953,
      "activations/layer11_attention_weight_min": -31.85254669189453,
      "activations/layer12_attention_weight_max": 26.905237197875977,
      "activations/layer12_attention_weight_min": -24.533349990844727,
      "activations/layer13_attention_weight_max": 40.774269104003906,
      "activations/layer13_attention_weight_min": -34.84189987182617,
      "activations/layer14_attention_weight_max": 44.11179733276367,
      "activations/layer14_attention_weight_min": -36.6232795715332,
      "activations/layer15_attention_weight_max": 38.44160842895508,
      "activations/layer15_attention_weight_min": -31.045228958129883,
      "activations/layer16_attention_weight_max": 34.32030487060547,
      "activations/layer16_attention_weight_min": -28.196651458740234,
      "activations/layer17_attention_weight_max": 32.54393005371094,
      "activations/layer17_attention_weight_min": -25.23667335510254,
      "activations/layer18_attention_weight_max": 33.09965133666992,
      "activations/layer18_attention_weight_min": -25.423553466796875,
      "activations/layer19_attention_weight_max": 37.27143478393555,
      "activations/layer19_attention_weight_min": -32.665000915527344,
      "activations/layer1_attention_weight_max": 17.990976333618164,
      "activations/layer1_attention_weight_min": -15.375082015991211,
      "activations/layer20_attention_weight_max": 32.16679382324219,
      "activations/layer20_attention_weight_min": -24.343109130859375,
      "activations/layer21_attention_weight_max": 30.976226806640625,
      "activations/layer21_attention_weight_min": -25.00346565246582,
      "activations/layer22_attention_weight_max": 46.02034378051758,
      "activations/layer22_attention_weight_min": -29.683198928833008,
      "activations/layer23_attention_weight_max": 41.77627944946289,
      "activations/layer23_attention_weight_min": -24.448352813720703,
      "activations/layer2_attention_weight_max": 32.07578659057617,
      "activations/layer2_attention_weight_min": -31.427885055541992,
      "activations/layer3_attention_weight_max": 94.26868438720703,
      "activations/layer3_attention_weight_min": -98.9229965209961,
      "activations/layer4_attention_weight_max": 110.04802703857422,
      "activations/layer4_attention_weight_min": -111.38410949707031,
      "activations/layer5_attention_weight_max": 54.326873779296875,
      "activations/layer5_attention_weight_min": -60.78321838378906,
      "activations/layer6_attention_weight_max": 42.61852264404297,
      "activations/layer6_attention_weight_min": -48.33053207397461,
      "activations/layer7_attention_weight_max": 92.64141082763672,
      "activations/layer7_attention_weight_min": -96.36595153808594,
      "activations/layer8_attention_weight_max": 43.509674072265625,
      "activations/layer8_attention_weight_min": -41.29692077636719,
      "activations/layer9_attention_weight_max": 33.55137252807617,
      "activations/layer9_attention_weight_min": -33.272090911865234,
      "epoch": 12.02,
      "learning_rate": 7.322878787878787e-05,
      "loss": 2.7637,
      "step": 206850
    },
    {
      "activations/layer0_attention_weight_max": 16.344160079956055,
      "activations/layer0_attention_weight_min": -12.038372993469238,
      "activations/layer10_attention_weight_max": 36.07571029663086,
      "activations/layer10_attention_weight_min": -33.45513916015625,
      "activations/layer11_attention_weight_max": 33.981319427490234,
      "activations/layer11_attention_weight_min": -34.92921829223633,
      "activations/layer12_attention_weight_max": 24.9945125579834,
      "activations/layer12_attention_weight_min": -24.62750816345215,
      "activations/layer13_attention_weight_max": 40.98265075683594,
      "activations/layer13_attention_weight_min": -33.155601501464844,
      "activations/layer14_attention_weight_max": 44.1668701171875,
      "activations/layer14_attention_weight_min": -39.06843948364258,
      "activations/layer15_attention_weight_max": 37.56804275512695,
      "activations/layer15_attention_weight_min": -32.73188781738281,
      "activations/layer16_attention_weight_max": 32.39485549926758,
      "activations/layer16_attention_weight_min": -28.558635711669922,
      "activations/layer17_attention_weight_max": 33.44091033935547,
      "activations/layer17_attention_weight_min": -25.93277359008789,
      "activations/layer18_attention_weight_max": 31.623336791992188,
      "activations/layer18_attention_weight_min": -22.897518157958984,
      "activations/layer19_attention_weight_max": 33.915771484375,
      "activations/layer19_attention_weight_min": -33.43888473510742,
      "activations/layer1_attention_weight_max": 16.899911880493164,
      "activations/layer1_attention_weight_min": -14.286447525024414,
      "activations/layer20_attention_weight_max": 28.747652053833008,
      "activations/layer20_attention_weight_min": -23.821317672729492,
      "activations/layer21_attention_weight_max": 27.843557357788086,
      "activations/layer21_attention_weight_min": -23.976970672607422,
      "activations/layer22_attention_weight_max": 45.19038772583008,
      "activations/layer22_attention_weight_min": -29.239730834960938,
      "activations/layer23_attention_weight_max": 33.61387634277344,
      "activations/layer23_attention_weight_min": -25.918529510498047,
      "activations/layer2_attention_weight_max": 29.94150161743164,
      "activations/layer2_attention_weight_min": -31.821147918701172,
      "activations/layer3_attention_weight_max": 90.72911834716797,
      "activations/layer3_attention_weight_min": -93.64198303222656,
      "activations/layer4_attention_weight_max": 108.82161712646484,
      "activations/layer4_attention_weight_min": -110.84081268310547,
      "activations/layer5_attention_weight_max": 50.02628707885742,
      "activations/layer5_attention_weight_min": -58.90700912475586,
      "activations/layer6_attention_weight_max": 45.68165969848633,
      "activations/layer6_attention_weight_min": -47.77336502075195,
      "activations/layer7_attention_weight_max": 92.93128204345703,
      "activations/layer7_attention_weight_min": -94.0069351196289,
      "activations/layer8_attention_weight_max": 39.861000061035156,
      "activations/layer8_attention_weight_min": -40.787445068359375,
      "activations/layer9_attention_weight_max": 35.424808502197266,
      "activations/layer9_attention_weight_min": -33.73902893066406,
      "epoch": 12.02,
      "learning_rate": 7.320984848484849e-05,
      "loss": 2.7701,
      "step": 206900
    },
    {
      "activations/layer0_attention_weight_max": 16.45186996459961,
      "activations/layer0_attention_weight_min": -11.27529525756836,
      "activations/layer10_attention_weight_max": 33.487545013427734,
      "activations/layer10_attention_weight_min": -33.06438064575195,
      "activations/layer11_attention_weight_max": 35.967529296875,
      "activations/layer11_attention_weight_min": -33.646018981933594,
      "activations/layer12_attention_weight_max": 26.132909774780273,
      "activations/layer12_attention_weight_min": -26.01970672607422,
      "activations/layer13_attention_weight_max": 39.55411911010742,
      "activations/layer13_attention_weight_min": -34.74463653564453,
      "activations/layer14_attention_weight_max": 47.5586051940918,
      "activations/layer14_attention_weight_min": -37.4687614440918,
      "activations/layer15_attention_weight_max": 34.893592834472656,
      "activations/layer15_attention_weight_min": -30.616275787353516,
      "activations/layer16_attention_weight_max": 31.21269416809082,
      "activations/layer16_attention_weight_min": -27.01054573059082,
      "activations/layer17_attention_weight_max": 35.496429443359375,
      "activations/layer17_attention_weight_min": -23.91092872619629,
      "activations/layer18_attention_weight_max": 31.916603088378906,
      "activations/layer18_attention_weight_min": -23.76460838317871,
      "activations/layer19_attention_weight_max": 36.17258834838867,
      "activations/layer19_attention_weight_min": -31.09004020690918,
      "activations/layer1_attention_weight_max": 16.446889877319336,
      "activations/layer1_attention_weight_min": -15.476503372192383,
      "activations/layer20_attention_weight_max": 31.379243850708008,
      "activations/layer20_attention_weight_min": -23.289566040039062,
      "activations/layer21_attention_weight_max": 30.263742446899414,
      "activations/layer21_attention_weight_min": -23.71209716796875,
      "activations/layer22_attention_weight_max": 50.218055725097656,
      "activations/layer22_attention_weight_min": -28.960556030273438,
      "activations/layer23_attention_weight_max": 39.20378112792969,
      "activations/layer23_attention_weight_min": -25.347728729248047,
      "activations/layer2_attention_weight_max": 36.3173828125,
      "activations/layer2_attention_weight_min": -32.53750228881836,
      "activations/layer3_attention_weight_max": 97.52486419677734,
      "activations/layer3_attention_weight_min": -94.27442932128906,
      "activations/layer4_attention_weight_max": 115.91065979003906,
      "activations/layer4_attention_weight_min": -112.76983642578125,
      "activations/layer5_attention_weight_max": 55.923370361328125,
      "activations/layer5_attention_weight_min": -64.30850219726562,
      "activations/layer6_attention_weight_max": 47.3144416809082,
      "activations/layer6_attention_weight_min": -48.367130279541016,
      "activations/layer7_attention_weight_max": 92.9548110961914,
      "activations/layer7_attention_weight_min": -96.14898681640625,
      "activations/layer8_attention_weight_max": 42.294105529785156,
      "activations/layer8_attention_weight_min": -42.4271240234375,
      "activations/layer9_attention_weight_max": 35.3100471496582,
      "activations/layer9_attention_weight_min": -34.157100677490234,
      "epoch": 12.02,
      "learning_rate": 7.319128787878788e-05,
      "loss": 2.7727,
      "step": 206950
    },
    {
      "activations/layer0_attention_weight_max": 15.719724655151367,
      "activations/layer0_attention_weight_min": -11.552330017089844,
      "activations/layer10_attention_weight_max": 33.81232452392578,
      "activations/layer10_attention_weight_min": -34.69630813598633,
      "activations/layer11_attention_weight_max": 33.512550354003906,
      "activations/layer11_attention_weight_min": -32.6233024597168,
      "activations/layer12_attention_weight_max": 27.955331802368164,
      "activations/layer12_attention_weight_min": -26.01915168762207,
      "activations/layer13_attention_weight_max": 44.34011459350586,
      "activations/layer13_attention_weight_min": -35.79121398925781,
      "activations/layer14_attention_weight_max": 56.30991744995117,
      "activations/layer14_attention_weight_min": -42.332847595214844,
      "activations/layer15_attention_weight_max": 42.4507942199707,
      "activations/layer15_attention_weight_min": -33.52098083496094,
      "activations/layer16_attention_weight_max": 37.537452697753906,
      "activations/layer16_attention_weight_min": -28.26365852355957,
      "activations/layer17_attention_weight_max": 37.96116638183594,
      "activations/layer17_attention_weight_min": -27.08443832397461,
      "activations/layer18_attention_weight_max": 32.49078369140625,
      "activations/layer18_attention_weight_min": -25.574905395507812,
      "activations/layer19_attention_weight_max": 42.425628662109375,
      "activations/layer19_attention_weight_min": -35.66926574707031,
      "activations/layer1_attention_weight_max": 16.556882858276367,
      "activations/layer1_attention_weight_min": -13.865893363952637,
      "activations/layer20_attention_weight_max": 33.47240447998047,
      "activations/layer20_attention_weight_min": -26.967876434326172,
      "activations/layer21_attention_weight_max": 33.744083404541016,
      "activations/layer21_attention_weight_min": -24.527690887451172,
      "activations/layer22_attention_weight_max": 51.91915512084961,
      "activations/layer22_attention_weight_min": -31.890352249145508,
      "activations/layer23_attention_weight_max": 39.51561737060547,
      "activations/layer23_attention_weight_min": -30.368255615234375,
      "activations/layer2_attention_weight_max": 33.3570556640625,
      "activations/layer2_attention_weight_min": -32.679718017578125,
      "activations/layer3_attention_weight_max": 96.0777359008789,
      "activations/layer3_attention_weight_min": -94.44757080078125,
      "activations/layer4_attention_weight_max": 115.20574188232422,
      "activations/layer4_attention_weight_min": -111.90484619140625,
      "activations/layer5_attention_weight_max": 51.47721862792969,
      "activations/layer5_attention_weight_min": -63.222755432128906,
      "activations/layer6_attention_weight_max": 45.97343063354492,
      "activations/layer6_attention_weight_min": -50.3335075378418,
      "activations/layer7_attention_weight_max": 93.98528289794922,
      "activations/layer7_attention_weight_min": -87.9268569946289,
      "activations/layer8_attention_weight_max": 41.43772506713867,
      "activations/layer8_attention_weight_min": -41.68082046508789,
      "activations/layer9_attention_weight_max": 32.82277297973633,
      "activations/layer9_attention_weight_min": -32.33041000366211,
      "epoch": 12.03,
      "learning_rate": 7.317234848484848e-05,
      "loss": 2.7751,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.5613,
      "eval_samples_per_second": 501.561,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.5613,
      "eval_openwebtext_samples_per_second": 501.561,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 2.0647,
      "eval_wikitext_samples_per_second": 220.86,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.6364,
      "eval_lambada_samples_per_second": 505.272,
      "step": 207000
    },
    {
      "activations/layer0_attention_weight_max": 14.837154388427734,
      "activations/layer0_attention_weight_min": -12.26390552520752,
      "activations/layer10_attention_weight_max": 34.00909423828125,
      "activations/layer10_attention_weight_min": -32.6185188293457,
      "activations/layer11_attention_weight_max": 33.465309143066406,
      "activations/layer11_attention_weight_min": -33.185081481933594,
      "activations/layer12_attention_weight_max": 32.58812713623047,
      "activations/layer12_attention_weight_min": -27.385578155517578,
      "activations/layer13_attention_weight_max": 44.04991149902344,
      "activations/layer13_attention_weight_min": -34.80186462402344,
      "activations/layer14_attention_weight_max": 53.13011169433594,
      "activations/layer14_attention_weight_min": -41.47501754760742,
      "activations/layer15_attention_weight_max": 41.242591857910156,
      "activations/layer15_attention_weight_min": -33.78654861450195,
      "activations/layer16_attention_weight_max": 34.86551284790039,
      "activations/layer16_attention_weight_min": -28.325761795043945,
      "activations/layer17_attention_weight_max": 35.29444885253906,
      "activations/layer17_attention_weight_min": -24.95713996887207,
      "activations/layer18_attention_weight_max": 36.46796798706055,
      "activations/layer18_attention_weight_min": -24.19032859802246,
      "activations/layer19_attention_weight_max": 40.73414993286133,
      "activations/layer19_attention_weight_min": -31.76338768005371,
      "activations/layer1_attention_weight_max": 17.113128662109375,
      "activations/layer1_attention_weight_min": -15.80819320678711,
      "activations/layer20_attention_weight_max": 34.274932861328125,
      "activations/layer20_attention_weight_min": -25.17214584350586,
      "activations/layer21_attention_weight_max": 34.104087829589844,
      "activations/layer21_attention_weight_min": -24.863237380981445,
      "activations/layer22_attention_weight_max": 58.43864822387695,
      "activations/layer22_attention_weight_min": -30.8656063079834,
      "activations/layer23_attention_weight_max": 41.15611267089844,
      "activations/layer23_attention_weight_min": -25.82343101501465,
      "activations/layer2_attention_weight_max": 33.78370666503906,
      "activations/layer2_attention_weight_min": -33.766845703125,
      "activations/layer3_attention_weight_max": 94.50479888916016,
      "activations/layer3_attention_weight_min": -101.79492950439453,
      "activations/layer4_attention_weight_max": 113.4058837890625,
      "activations/layer4_attention_weight_min": -115.17961883544922,
      "activations/layer5_attention_weight_max": 50.379600524902344,
      "activations/layer5_attention_weight_min": -61.97663497924805,
      "activations/layer6_attention_weight_max": 44.719459533691406,
      "activations/layer6_attention_weight_min": -47.103736877441406,
      "activations/layer7_attention_weight_max": 94.85504150390625,
      "activations/layer7_attention_weight_min": -93.31165313720703,
      "activations/layer8_attention_weight_max": 40.7496337890625,
      "activations/layer8_attention_weight_min": -42.24492263793945,
      "activations/layer9_attention_weight_max": 34.685176849365234,
      "activations/layer9_attention_weight_min": -35.51651382446289,
      "epoch": 12.03,
      "learning_rate": 7.315340909090908e-05,
      "loss": 2.7596,
      "step": 207050
    },
    {
      "activations/layer0_attention_weight_max": 15.613972663879395,
      "activations/layer0_attention_weight_min": -12.028470039367676,
      "activations/layer10_attention_weight_max": 31.00780487060547,
      "activations/layer10_attention_weight_min": -32.3390007019043,
      "activations/layer11_attention_weight_max": 33.87798309326172,
      "activations/layer11_attention_weight_min": -32.969268798828125,
      "activations/layer12_attention_weight_max": 25.527875900268555,
      "activations/layer12_attention_weight_min": -25.28656005859375,
      "activations/layer13_attention_weight_max": 36.46633529663086,
      "activations/layer13_attention_weight_min": -35.03005599975586,
      "activations/layer14_attention_weight_max": 44.19007110595703,
      "activations/layer14_attention_weight_min": -39.275177001953125,
      "activations/layer15_attention_weight_max": 36.204647064208984,
      "activations/layer15_attention_weight_min": -31.192848205566406,
      "activations/layer16_attention_weight_max": 36.342071533203125,
      "activations/layer16_attention_weight_min": -28.198625564575195,
      "activations/layer17_attention_weight_max": 33.79618453979492,
      "activations/layer17_attention_weight_min": -26.764816284179688,
      "activations/layer18_attention_weight_max": 32.73046112060547,
      "activations/layer18_attention_weight_min": -25.43610191345215,
      "activations/layer19_attention_weight_max": 38.864768981933594,
      "activations/layer19_attention_weight_min": -31.443008422851562,
      "activations/layer1_attention_weight_max": 16.90033721923828,
      "activations/layer1_attention_weight_min": -17.014259338378906,
      "activations/layer20_attention_weight_max": 30.41085433959961,
      "activations/layer20_attention_weight_min": -23.907318115234375,
      "activations/layer21_attention_weight_max": 28.025577545166016,
      "activations/layer21_attention_weight_min": -23.771764755249023,
      "activations/layer22_attention_weight_max": 41.6506233215332,
      "activations/layer22_attention_weight_min": -29.69454002380371,
      "activations/layer23_attention_weight_max": 34.89236068725586,
      "activations/layer23_attention_weight_min": -26.89082908630371,
      "activations/layer2_attention_weight_max": 35.38402557373047,
      "activations/layer2_attention_weight_min": -33.53551483154297,
      "activations/layer3_attention_weight_max": 99.51112365722656,
      "activations/layer3_attention_weight_min": -98.39120483398438,
      "activations/layer4_attention_weight_max": 114.43971252441406,
      "activations/layer4_attention_weight_min": -115.45946502685547,
      "activations/layer5_attention_weight_max": 52.41400909423828,
      "activations/layer5_attention_weight_min": -63.99827575683594,
      "activations/layer6_attention_weight_max": 44.11426544189453,
      "activations/layer6_attention_weight_min": -46.92328643798828,
      "activations/layer7_attention_weight_max": 102.02202606201172,
      "activations/layer7_attention_weight_min": -88.95636749267578,
      "activations/layer8_attention_weight_max": 41.34449005126953,
      "activations/layer8_attention_weight_min": -39.850093841552734,
      "activations/layer9_attention_weight_max": 33.25455856323242,
      "activations/layer9_attention_weight_min": -32.33032989501953,
      "epoch": 12.03,
      "learning_rate": 7.31344696969697e-05,
      "loss": 2.7694,
      "step": 207100
    },
    {
      "activations/layer0_attention_weight_max": 15.993378639221191,
      "activations/layer0_attention_weight_min": -12.219624519348145,
      "activations/layer10_attention_weight_max": 31.97028160095215,
      "activations/layer10_attention_weight_min": -31.13594627380371,
      "activations/layer11_attention_weight_max": 32.28004455566406,
      "activations/layer11_attention_weight_min": -32.10478591918945,
      "activations/layer12_attention_weight_max": 26.33823013305664,
      "activations/layer12_attention_weight_min": -26.259571075439453,
      "activations/layer13_attention_weight_max": 38.938636779785156,
      "activations/layer13_attention_weight_min": -35.039920806884766,
      "activations/layer14_attention_weight_max": 42.22801971435547,
      "activations/layer14_attention_weight_min": -35.44443130493164,
      "activations/layer15_attention_weight_max": 38.75584411621094,
      "activations/layer15_attention_weight_min": -31.87624740600586,
      "activations/layer16_attention_weight_max": 30.418119430541992,
      "activations/layer16_attention_weight_min": -29.024309158325195,
      "activations/layer17_attention_weight_max": 33.91248321533203,
      "activations/layer17_attention_weight_min": -25.213247299194336,
      "activations/layer18_attention_weight_max": 33.49782180786133,
      "activations/layer18_attention_weight_min": -24.104015350341797,
      "activations/layer19_attention_weight_max": 34.2833251953125,
      "activations/layer19_attention_weight_min": -31.562076568603516,
      "activations/layer1_attention_weight_max": 17.309701919555664,
      "activations/layer1_attention_weight_min": -15.051173210144043,
      "activations/layer20_attention_weight_max": 28.54867172241211,
      "activations/layer20_attention_weight_min": -27.65245819091797,
      "activations/layer21_attention_weight_max": 27.235807418823242,
      "activations/layer21_attention_weight_min": -24.657699584960938,
      "activations/layer22_attention_weight_max": 43.45023727416992,
      "activations/layer22_attention_weight_min": -31.337556838989258,
      "activations/layer23_attention_weight_max": 31.146665573120117,
      "activations/layer23_attention_weight_min": -25.337600708007812,
      "activations/layer2_attention_weight_max": 34.28347396850586,
      "activations/layer2_attention_weight_min": -30.817813873291016,
      "activations/layer3_attention_weight_max": 95.06034851074219,
      "activations/layer3_attention_weight_min": -92.20980072021484,
      "activations/layer4_attention_weight_max": 109.80799865722656,
      "activations/layer4_attention_weight_min": -107.15425109863281,
      "activations/layer5_attention_weight_max": 50.11354064941406,
      "activations/layer5_attention_weight_min": -61.24427795410156,
      "activations/layer6_attention_weight_max": 42.404632568359375,
      "activations/layer6_attention_weight_min": -46.04192352294922,
      "activations/layer7_attention_weight_max": 83.50408935546875,
      "activations/layer7_attention_weight_min": -90.35074615478516,
      "activations/layer8_attention_weight_max": 36.94151306152344,
      "activations/layer8_attention_weight_min": -40.38044357299805,
      "activations/layer9_attention_weight_max": 30.907575607299805,
      "activations/layer9_attention_weight_min": -33.082664489746094,
      "epoch": 12.04,
      "learning_rate": 7.31155303030303e-05,
      "loss": 2.7627,
      "step": 207150
    },
    {
      "activations/layer0_attention_weight_max": 15.656978607177734,
      "activations/layer0_attention_weight_min": -12.288186073303223,
      "activations/layer10_attention_weight_max": 37.37939453125,
      "activations/layer10_attention_weight_min": -32.73758316040039,
      "activations/layer11_attention_weight_max": 35.605003356933594,
      "activations/layer11_attention_weight_min": -31.397390365600586,
      "activations/layer12_attention_weight_max": 27.04898452758789,
      "activations/layer12_attention_weight_min": -34.12610626220703,
      "activations/layer13_attention_weight_max": 39.30607986450195,
      "activations/layer13_attention_weight_min": -33.10082244873047,
      "activations/layer14_attention_weight_max": 42.96826934814453,
      "activations/layer14_attention_weight_min": -36.8021354675293,
      "activations/layer15_attention_weight_max": 36.692283630371094,
      "activations/layer15_attention_weight_min": -31.1483154296875,
      "activations/layer16_attention_weight_max": 34.98820877075195,
      "activations/layer16_attention_weight_min": -27.80485725402832,
      "activations/layer17_attention_weight_max": 32.18190002441406,
      "activations/layer17_attention_weight_min": -25.858196258544922,
      "activations/layer18_attention_weight_max": 35.80723571777344,
      "activations/layer18_attention_weight_min": -24.09343719482422,
      "activations/layer19_attention_weight_max": 41.5474967956543,
      "activations/layer19_attention_weight_min": -30.25870132446289,
      "activations/layer1_attention_weight_max": 16.774812698364258,
      "activations/layer1_attention_weight_min": -14.452505111694336,
      "activations/layer20_attention_weight_max": 30.817607879638672,
      "activations/layer20_attention_weight_min": -23.397056579589844,
      "activations/layer21_attention_weight_max": 31.14739990234375,
      "activations/layer21_attention_weight_min": -24.696374893188477,
      "activations/layer22_attention_weight_max": 48.33969497680664,
      "activations/layer22_attention_weight_min": -30.01713752746582,
      "activations/layer23_attention_weight_max": 38.6818733215332,
      "activations/layer23_attention_weight_min": -26.232181549072266,
      "activations/layer2_attention_weight_max": 33.9225959777832,
      "activations/layer2_attention_weight_min": -33.462921142578125,
      "activations/layer3_attention_weight_max": 98.00078582763672,
      "activations/layer3_attention_weight_min": -99.30152130126953,
      "activations/layer4_attention_weight_max": 108.03364562988281,
      "activations/layer4_attention_weight_min": -116.0173568725586,
      "activations/layer5_attention_weight_max": 53.856040954589844,
      "activations/layer5_attention_weight_min": -64.09831237792969,
      "activations/layer6_attention_weight_max": 44.385562896728516,
      "activations/layer6_attention_weight_min": -45.29755783081055,
      "activations/layer7_attention_weight_max": 93.27523040771484,
      "activations/layer7_attention_weight_min": -87.1271743774414,
      "activations/layer8_attention_weight_max": 42.58636474609375,
      "activations/layer8_attention_weight_min": -39.25003433227539,
      "activations/layer9_attention_weight_max": 34.92216873168945,
      "activations/layer9_attention_weight_min": -36.262664794921875,
      "epoch": 12.04,
      "learning_rate": 7.30965909090909e-05,
      "loss": 2.7849,
      "step": 207200
    },
    {
      "activations/layer0_attention_weight_max": 14.98458194732666,
      "activations/layer0_attention_weight_min": -13.19990348815918,
      "activations/layer10_attention_weight_max": 34.13694763183594,
      "activations/layer10_attention_weight_min": -33.664947509765625,
      "activations/layer11_attention_weight_max": 36.022666931152344,
      "activations/layer11_attention_weight_min": -34.767330169677734,
      "activations/layer12_attention_weight_max": 29.630725860595703,
      "activations/layer12_attention_weight_min": -25.22309684753418,
      "activations/layer13_attention_weight_max": 40.03266143798828,
      "activations/layer13_attention_weight_min": -37.05555725097656,
      "activations/layer14_attention_weight_max": 44.8441276550293,
      "activations/layer14_attention_weight_min": -40.124210357666016,
      "activations/layer15_attention_weight_max": 36.552520751953125,
      "activations/layer15_attention_weight_min": -33.79273223876953,
      "activations/layer16_attention_weight_max": 32.48904037475586,
      "activations/layer16_attention_weight_min": -28.07951545715332,
      "activations/layer17_attention_weight_max": 32.04384231567383,
      "activations/layer17_attention_weight_min": -25.926366806030273,
      "activations/layer18_attention_weight_max": 29.300968170166016,
      "activations/layer18_attention_weight_min": -25.283233642578125,
      "activations/layer19_attention_weight_max": 34.84597396850586,
      "activations/layer19_attention_weight_min": -30.40060806274414,
      "activations/layer1_attention_weight_max": 16.097579956054688,
      "activations/layer1_attention_weight_min": -15.374580383300781,
      "activations/layer20_attention_weight_max": 29.798410415649414,
      "activations/layer20_attention_weight_min": -24.611244201660156,
      "activations/layer21_attention_weight_max": 27.5649471282959,
      "activations/layer21_attention_weight_min": -24.503887176513672,
      "activations/layer22_attention_weight_max": 45.44646072387695,
      "activations/layer22_attention_weight_min": -29.244802474975586,
      "activations/layer23_attention_weight_max": 36.70600128173828,
      "activations/layer23_attention_weight_min": -26.8884334564209,
      "activations/layer2_attention_weight_max": 33.76301193237305,
      "activations/layer2_attention_weight_min": -32.29678726196289,
      "activations/layer3_attention_weight_max": 96.31414031982422,
      "activations/layer3_attention_weight_min": -97.58077239990234,
      "activations/layer4_attention_weight_max": 112.8314208984375,
      "activations/layer4_attention_weight_min": -110.82866668701172,
      "activations/layer5_attention_weight_max": 50.70403289794922,
      "activations/layer5_attention_weight_min": -68.78034973144531,
      "activations/layer6_attention_weight_max": 44.39152526855469,
      "activations/layer6_attention_weight_min": -46.720462799072266,
      "activations/layer7_attention_weight_max": 93.8213119506836,
      "activations/layer7_attention_weight_min": -89.89486694335938,
      "activations/layer8_attention_weight_max": 40.376564025878906,
      "activations/layer8_attention_weight_min": -41.645912170410156,
      "activations/layer9_attention_weight_max": 32.678218841552734,
      "activations/layer9_attention_weight_min": -33.3583869934082,
      "epoch": 12.04,
      "learning_rate": 7.30776515151515e-05,
      "loss": 2.7682,
      "step": 207250
    },
    {
      "activations/layer0_attention_weight_max": 16.087604522705078,
      "activations/layer0_attention_weight_min": -11.821649551391602,
      "activations/layer10_attention_weight_max": 31.144054412841797,
      "activations/layer10_attention_weight_min": -32.66923904418945,
      "activations/layer11_attention_weight_max": 32.437950134277344,
      "activations/layer11_attention_weight_min": -32.758872985839844,
      "activations/layer12_attention_weight_max": 24.28948211669922,
      "activations/layer12_attention_weight_min": -27.59215545654297,
      "activations/layer13_attention_weight_max": 38.51764678955078,
      "activations/layer13_attention_weight_min": -33.166019439697266,
      "activations/layer14_attention_weight_max": 47.762535095214844,
      "activations/layer14_attention_weight_min": -37.33192825317383,
      "activations/layer15_attention_weight_max": 37.049072265625,
      "activations/layer15_attention_weight_min": -31.208303451538086,
      "activations/layer16_attention_weight_max": 32.725223541259766,
      "activations/layer16_attention_weight_min": -27.86305046081543,
      "activations/layer17_attention_weight_max": 32.177127838134766,
      "activations/layer17_attention_weight_min": -27.107303619384766,
      "activations/layer18_attention_weight_max": 36.93565368652344,
      "activations/layer18_attention_weight_min": -24.637813568115234,
      "activations/layer19_attention_weight_max": 39.2115364074707,
      "activations/layer19_attention_weight_min": -31.610097885131836,
      "activations/layer1_attention_weight_max": 16.69711685180664,
      "activations/layer1_attention_weight_min": -14.450196266174316,
      "activations/layer20_attention_weight_max": 32.80409622192383,
      "activations/layer20_attention_weight_min": -25.882951736450195,
      "activations/layer21_attention_weight_max": 32.07795333862305,
      "activations/layer21_attention_weight_min": -25.237625122070312,
      "activations/layer22_attention_weight_max": 50.22184371948242,
      "activations/layer22_attention_weight_min": -32.462425231933594,
      "activations/layer23_attention_weight_max": 40.11284637451172,
      "activations/layer23_attention_weight_min": -25.171066284179688,
      "activations/layer2_attention_weight_max": 34.09785079956055,
      "activations/layer2_attention_weight_min": -33.60557556152344,
      "activations/layer3_attention_weight_max": 98.48013305664062,
      "activations/layer3_attention_weight_min": -98.99867248535156,
      "activations/layer4_attention_weight_max": 114.38044738769531,
      "activations/layer4_attention_weight_min": -113.7260971069336,
      "activations/layer5_attention_weight_max": 57.050750732421875,
      "activations/layer5_attention_weight_min": -62.6212158203125,
      "activations/layer6_attention_weight_max": 43.30061340332031,
      "activations/layer6_attention_weight_min": -48.07035446166992,
      "activations/layer7_attention_weight_max": 88.7889404296875,
      "activations/layer7_attention_weight_min": -85.90347290039062,
      "activations/layer8_attention_weight_max": 38.681392669677734,
      "activations/layer8_attention_weight_min": -41.81167984008789,
      "activations/layer9_attention_weight_max": 31.976850509643555,
      "activations/layer9_attention_weight_min": -35.10325241088867,
      "epoch": 12.05,
      "learning_rate": 7.305871212121211e-05,
      "loss": 2.7757,
      "step": 207300
    },
    {
      "activations/layer0_attention_weight_max": 18.398515701293945,
      "activations/layer0_attention_weight_min": -12.469070434570312,
      "activations/layer10_attention_weight_max": 29.17222023010254,
      "activations/layer10_attention_weight_min": -32.968441009521484,
      "activations/layer11_attention_weight_max": 30.37625503540039,
      "activations/layer11_attention_weight_min": -30.942251205444336,
      "activations/layer12_attention_weight_max": 23.882904052734375,
      "activations/layer12_attention_weight_min": -24.91705322265625,
      "activations/layer13_attention_weight_max": 34.62604904174805,
      "activations/layer13_attention_weight_min": -33.64939880371094,
      "activations/layer14_attention_weight_max": 41.397499084472656,
      "activations/layer14_attention_weight_min": -38.350624084472656,
      "activations/layer15_attention_weight_max": 34.71396255493164,
      "activations/layer15_attention_weight_min": -32.52626037597656,
      "activations/layer16_attention_weight_max": 31.229265213012695,
      "activations/layer16_attention_weight_min": -27.203153610229492,
      "activations/layer17_attention_weight_max": 30.128658294677734,
      "activations/layer17_attention_weight_min": -27.829769134521484,
      "activations/layer18_attention_weight_max": 28.243078231811523,
      "activations/layer18_attention_weight_min": -23.4509334564209,
      "activations/layer19_attention_weight_max": 31.425113677978516,
      "activations/layer19_attention_weight_min": -32.284996032714844,
      "activations/layer1_attention_weight_max": 16.663984298706055,
      "activations/layer1_attention_weight_min": -14.21611499786377,
      "activations/layer20_attention_weight_max": 26.685216903686523,
      "activations/layer20_attention_weight_min": -23.908952713012695,
      "activations/layer21_attention_weight_max": 30.1344051361084,
      "activations/layer21_attention_weight_min": -23.405773162841797,
      "activations/layer22_attention_weight_max": 44.776824951171875,
      "activations/layer22_attention_weight_min": -29.513654708862305,
      "activations/layer23_attention_weight_max": 31.579633712768555,
      "activations/layer23_attention_weight_min": -25.576229095458984,
      "activations/layer2_attention_weight_max": 34.209068298339844,
      "activations/layer2_attention_weight_min": -33.025856018066406,
      "activations/layer3_attention_weight_max": 97.50265502929688,
      "activations/layer3_attention_weight_min": -99.6825942993164,
      "activations/layer4_attention_weight_max": 114.80081939697266,
      "activations/layer4_attention_weight_min": -112.37141418457031,
      "activations/layer5_attention_weight_max": 49.80644989013672,
      "activations/layer5_attention_weight_min": -58.930442810058594,
      "activations/layer6_attention_weight_max": 44.465492248535156,
      "activations/layer6_attention_weight_min": -45.44205856323242,
      "activations/layer7_attention_weight_max": 83.13679504394531,
      "activations/layer7_attention_weight_min": -91.5571060180664,
      "activations/layer8_attention_weight_max": 38.068634033203125,
      "activations/layer8_attention_weight_min": -40.728275299072266,
      "activations/layer9_attention_weight_max": 30.744260787963867,
      "activations/layer9_attention_weight_min": -33.43391036987305,
      "epoch": 12.05,
      "learning_rate": 7.303977272727272e-05,
      "loss": 2.7668,
      "step": 207350
    },
    {
      "activations/layer0_attention_weight_max": 16.5567626953125,
      "activations/layer0_attention_weight_min": -12.945907592773438,
      "activations/layer10_attention_weight_max": 36.74414825439453,
      "activations/layer10_attention_weight_min": -35.44441604614258,
      "activations/layer11_attention_weight_max": 37.075050354003906,
      "activations/layer11_attention_weight_min": -36.05881881713867,
      "activations/layer12_attention_weight_max": 27.731693267822266,
      "activations/layer12_attention_weight_min": -26.488183975219727,
      "activations/layer13_attention_weight_max": 41.94438171386719,
      "activations/layer13_attention_weight_min": -34.917076110839844,
      "activations/layer14_attention_weight_max": 52.64199447631836,
      "activations/layer14_attention_weight_min": -40.422027587890625,
      "activations/layer15_attention_weight_max": 42.7916259765625,
      "activations/layer15_attention_weight_min": -32.20893096923828,
      "activations/layer16_attention_weight_max": 36.26022720336914,
      "activations/layer16_attention_weight_min": -28.767887115478516,
      "activations/layer17_attention_weight_max": 35.506980895996094,
      "activations/layer17_attention_weight_min": -26.4992618560791,
      "activations/layer18_attention_weight_max": 33.53904342651367,
      "activations/layer18_attention_weight_min": -23.353740692138672,
      "activations/layer19_attention_weight_max": 36.89274597167969,
      "activations/layer19_attention_weight_min": -30.2570743560791,
      "activations/layer1_attention_weight_max": 19.063034057617188,
      "activations/layer1_attention_weight_min": -16.780826568603516,
      "activations/layer20_attention_weight_max": 30.27232551574707,
      "activations/layer20_attention_weight_min": -25.384376525878906,
      "activations/layer21_attention_weight_max": 30.560272216796875,
      "activations/layer21_attention_weight_min": -24.211820602416992,
      "activations/layer22_attention_weight_max": 48.3797492980957,
      "activations/layer22_attention_weight_min": -31.550031661987305,
      "activations/layer23_attention_weight_max": 38.04781723022461,
      "activations/layer23_attention_weight_min": -24.6408748626709,
      "activations/layer2_attention_weight_max": 33.73641586303711,
      "activations/layer2_attention_weight_min": -32.49439239501953,
      "activations/layer3_attention_weight_max": 95.46134948730469,
      "activations/layer3_attention_weight_min": -94.90774536132812,
      "activations/layer4_attention_weight_max": 115.51509857177734,
      "activations/layer4_attention_weight_min": -113.7149429321289,
      "activations/layer5_attention_weight_max": 52.724281311035156,
      "activations/layer5_attention_weight_min": -58.58864974975586,
      "activations/layer6_attention_weight_max": 45.10625076293945,
      "activations/layer6_attention_weight_min": -47.510887145996094,
      "activations/layer7_attention_weight_max": 97.12574768066406,
      "activations/layer7_attention_weight_min": -90.75015258789062,
      "activations/layer8_attention_weight_max": 44.089210510253906,
      "activations/layer8_attention_weight_min": -42.19092559814453,
      "activations/layer9_attention_weight_max": 34.704036712646484,
      "activations/layer9_attention_weight_min": -35.072078704833984,
      "epoch": 12.05,
      "learning_rate": 7.302083333333333e-05,
      "loss": 2.7659,
      "step": 207400
    },
    {
      "activations/layer0_attention_weight_max": 15.599637031555176,
      "activations/layer0_attention_weight_min": -12.484776496887207,
      "activations/layer10_attention_weight_max": 36.6177864074707,
      "activations/layer10_attention_weight_min": -33.5449333190918,
      "activations/layer11_attention_weight_max": 35.509891510009766,
      "activations/layer11_attention_weight_min": -33.72880554199219,
      "activations/layer12_attention_weight_max": 26.069149017333984,
      "activations/layer12_attention_weight_min": -25.384723663330078,
      "activations/layer13_attention_weight_max": 42.79230880737305,
      "activations/layer13_attention_weight_min": -35.84975051879883,
      "activations/layer14_attention_weight_max": 41.76475143432617,
      "activations/layer14_attention_weight_min": -36.88224411010742,
      "activations/layer15_attention_weight_max": 41.79265594482422,
      "activations/layer15_attention_weight_min": -33.06028366088867,
      "activations/layer16_attention_weight_max": 31.207181930541992,
      "activations/layer16_attention_weight_min": -28.187776565551758,
      "activations/layer17_attention_weight_max": 30.541439056396484,
      "activations/layer17_attention_weight_min": -26.35013771057129,
      "activations/layer18_attention_weight_max": 29.585735321044922,
      "activations/layer18_attention_weight_min": -23.716758728027344,
      "activations/layer19_attention_weight_max": 40.80009460449219,
      "activations/layer19_attention_weight_min": -30.79471206665039,
      "activations/layer1_attention_weight_max": 17.1215763092041,
      "activations/layer1_attention_weight_min": -15.675305366516113,
      "activations/layer20_attention_weight_max": 29.761295318603516,
      "activations/layer20_attention_weight_min": -24.11113166809082,
      "activations/layer21_attention_weight_max": 30.00754165649414,
      "activations/layer21_attention_weight_min": -23.804149627685547,
      "activations/layer22_attention_weight_max": 41.9246826171875,
      "activations/layer22_attention_weight_min": -28.649274826049805,
      "activations/layer23_attention_weight_max": 37.1295166015625,
      "activations/layer23_attention_weight_min": -25.9101619720459,
      "activations/layer2_attention_weight_max": 34.222171783447266,
      "activations/layer2_attention_weight_min": -33.46290588378906,
      "activations/layer3_attention_weight_max": 96.56523895263672,
      "activations/layer3_attention_weight_min": -95.98857116699219,
      "activations/layer4_attention_weight_max": 112.87152099609375,
      "activations/layer4_attention_weight_min": -110.2909927368164,
      "activations/layer5_attention_weight_max": 53.41856002807617,
      "activations/layer5_attention_weight_min": -64.4547119140625,
      "activations/layer6_attention_weight_max": 44.86353302001953,
      "activations/layer6_attention_weight_min": -45.95442581176758,
      "activations/layer7_attention_weight_max": 89.72064971923828,
      "activations/layer7_attention_weight_min": -93.31878662109375,
      "activations/layer8_attention_weight_max": 40.95339584350586,
      "activations/layer8_attention_weight_min": -40.72584533691406,
      "activations/layer9_attention_weight_max": 37.0223503112793,
      "activations/layer9_attention_weight_min": -34.493892669677734,
      "epoch": 12.05,
      "learning_rate": 7.300189393939393e-05,
      "loss": 2.7662,
      "step": 207450
    },
    {
      "activations/layer0_attention_weight_max": 16.316715240478516,
      "activations/layer0_attention_weight_min": -11.851685523986816,
      "activations/layer10_attention_weight_max": 33.44912338256836,
      "activations/layer10_attention_weight_min": -31.763944625854492,
      "activations/layer11_attention_weight_max": 33.29960632324219,
      "activations/layer11_attention_weight_min": -32.78026580810547,
      "activations/layer12_attention_weight_max": 27.163557052612305,
      "activations/layer12_attention_weight_min": -26.322559356689453,
      "activations/layer13_attention_weight_max": 42.8725471496582,
      "activations/layer13_attention_weight_min": -37.65110778808594,
      "activations/layer14_attention_weight_max": 49.42786407470703,
      "activations/layer14_attention_weight_min": -38.71742630004883,
      "activations/layer15_attention_weight_max": 38.40353012084961,
      "activations/layer15_attention_weight_min": -33.4683837890625,
      "activations/layer16_attention_weight_max": 32.0529670715332,
      "activations/layer16_attention_weight_min": -28.778865814208984,
      "activations/layer17_attention_weight_max": 33.28036880493164,
      "activations/layer17_attention_weight_min": -25.236040115356445,
      "activations/layer18_attention_weight_max": 35.806922912597656,
      "activations/layer18_attention_weight_min": -23.493785858154297,
      "activations/layer19_attention_weight_max": 37.445247650146484,
      "activations/layer19_attention_weight_min": -30.380773544311523,
      "activations/layer1_attention_weight_max": 17.135297775268555,
      "activations/layer1_attention_weight_min": -16.223909378051758,
      "activations/layer20_attention_weight_max": 32.71736145019531,
      "activations/layer20_attention_weight_min": -26.546146392822266,
      "activations/layer21_attention_weight_max": 39.570404052734375,
      "activations/layer21_attention_weight_min": -24.143863677978516,
      "activations/layer22_attention_weight_max": 48.02983474731445,
      "activations/layer22_attention_weight_min": -31.209688186645508,
      "activations/layer23_attention_weight_max": 38.941795349121094,
      "activations/layer23_attention_weight_min": -25.84864616394043,
      "activations/layer2_attention_weight_max": 32.72315216064453,
      "activations/layer2_attention_weight_min": -32.00698471069336,
      "activations/layer3_attention_weight_max": 90.9842529296875,
      "activations/layer3_attention_weight_min": -93.4179458618164,
      "activations/layer4_attention_weight_max": 108.7183837890625,
      "activations/layer4_attention_weight_min": -107.3082504272461,
      "activations/layer5_attention_weight_max": 53.20234680175781,
      "activations/layer5_attention_weight_min": -58.997764587402344,
      "activations/layer6_attention_weight_max": 44.46410369873047,
      "activations/layer6_attention_weight_min": -46.96159362792969,
      "activations/layer7_attention_weight_max": 91.99784088134766,
      "activations/layer7_attention_weight_min": -84.9949722290039,
      "activations/layer8_attention_weight_max": 40.37895202636719,
      "activations/layer8_attention_weight_min": -40.40986251831055,
      "activations/layer9_attention_weight_max": 32.59518051147461,
      "activations/layer9_attention_weight_min": -31.581680297851562,
      "epoch": 12.06,
      "learning_rate": 7.298295454545455e-05,
      "loss": 2.7707,
      "step": 207500
    },
    {
      "activations/layer0_attention_weight_max": 15.765554428100586,
      "activations/layer0_attention_weight_min": -12.337993621826172,
      "activations/layer10_attention_weight_max": 36.48078536987305,
      "activations/layer10_attention_weight_min": -36.65119552612305,
      "activations/layer11_attention_weight_max": 39.47568893432617,
      "activations/layer11_attention_weight_min": -37.692138671875,
      "activations/layer12_attention_weight_max": 28.274805068969727,
      "activations/layer12_attention_weight_min": -27.43500518798828,
      "activations/layer13_attention_weight_max": 42.66826629638672,
      "activations/layer13_attention_weight_min": -35.4654655456543,
      "activations/layer14_attention_weight_max": 46.235355377197266,
      "activations/layer14_attention_weight_min": -39.4719123840332,
      "activations/layer15_attention_weight_max": 37.11902618408203,
      "activations/layer15_attention_weight_min": -32.112152099609375,
      "activations/layer16_attention_weight_max": 30.758024215698242,
      "activations/layer16_attention_weight_min": -27.73894500732422,
      "activations/layer17_attention_weight_max": 33.00362014770508,
      "activations/layer17_attention_weight_min": -25.391305923461914,
      "activations/layer18_attention_weight_max": 34.14394760131836,
      "activations/layer18_attention_weight_min": -22.584821701049805,
      "activations/layer19_attention_weight_max": 36.78244400024414,
      "activations/layer19_attention_weight_min": -30.053098678588867,
      "activations/layer1_attention_weight_max": 18.650592803955078,
      "activations/layer1_attention_weight_min": -16.139881134033203,
      "activations/layer20_attention_weight_max": 32.88261413574219,
      "activations/layer20_attention_weight_min": -24.124284744262695,
      "activations/layer21_attention_weight_max": 30.56774139404297,
      "activations/layer21_attention_weight_min": -24.018341064453125,
      "activations/layer22_attention_weight_max": 41.00743103027344,
      "activations/layer22_attention_weight_min": -28.845903396606445,
      "activations/layer23_attention_weight_max": 36.19071960449219,
      "activations/layer23_attention_weight_min": -25.161701202392578,
      "activations/layer2_attention_weight_max": 34.318450927734375,
      "activations/layer2_attention_weight_min": -34.244529724121094,
      "activations/layer3_attention_weight_max": 97.3388671875,
      "activations/layer3_attention_weight_min": -98.31664276123047,
      "activations/layer4_attention_weight_max": 113.76334381103516,
      "activations/layer4_attention_weight_min": -109.88939666748047,
      "activations/layer5_attention_weight_max": 61.17219161987305,
      "activations/layer5_attention_weight_min": -61.122310638427734,
      "activations/layer6_attention_weight_max": 44.22779846191406,
      "activations/layer6_attention_weight_min": -47.116905212402344,
      "activations/layer7_attention_weight_max": 93.26399993896484,
      "activations/layer7_attention_weight_min": -93.99481964111328,
      "activations/layer8_attention_weight_max": 43.29414749145508,
      "activations/layer8_attention_weight_min": -42.84037399291992,
      "activations/layer9_attention_weight_max": 35.81044006347656,
      "activations/layer9_attention_weight_min": -36.001224517822266,
      "epoch": 12.06,
      "learning_rate": 7.296401515151515e-05,
      "loss": 2.7714,
      "step": 207550
    },
    {
      "activations/layer0_attention_weight_max": 16.49461555480957,
      "activations/layer0_attention_weight_min": -11.886544227600098,
      "activations/layer10_attention_weight_max": 29.514263153076172,
      "activations/layer10_attention_weight_min": -31.04892921447754,
      "activations/layer11_attention_weight_max": 29.742528915405273,
      "activations/layer11_attention_weight_min": -30.475093841552734,
      "activations/layer12_attention_weight_max": 24.04271697998047,
      "activations/layer12_attention_weight_min": -23.712621688842773,
      "activations/layer13_attention_weight_max": 34.919578552246094,
      "activations/layer13_attention_weight_min": -30.949424743652344,
      "activations/layer14_attention_weight_max": 37.94019317626953,
      "activations/layer14_attention_weight_min": -33.4801025390625,
      "activations/layer15_attention_weight_max": 33.38655090332031,
      "activations/layer15_attention_weight_min": -29.981504440307617,
      "activations/layer16_attention_weight_max": 29.587940216064453,
      "activations/layer16_attention_weight_min": -27.129024505615234,
      "activations/layer17_attention_weight_max": 30.599374771118164,
      "activations/layer17_attention_weight_min": -24.253494262695312,
      "activations/layer18_attention_weight_max": 29.932748794555664,
      "activations/layer18_attention_weight_min": -23.76228904724121,
      "activations/layer19_attention_weight_max": 32.33256149291992,
      "activations/layer19_attention_weight_min": -29.399927139282227,
      "activations/layer1_attention_weight_max": 16.0035400390625,
      "activations/layer1_attention_weight_min": -14.374979972839355,
      "activations/layer20_attention_weight_max": 27.294219970703125,
      "activations/layer20_attention_weight_min": -25.593120574951172,
      "activations/layer21_attention_weight_max": 26.611318588256836,
      "activations/layer21_attention_weight_min": -24.915916442871094,
      "activations/layer22_attention_weight_max": 41.9614143371582,
      "activations/layer22_attention_weight_min": -32.058223724365234,
      "activations/layer23_attention_weight_max": 36.92107009887695,
      "activations/layer23_attention_weight_min": -24.654613494873047,
      "activations/layer2_attention_weight_max": 33.0691032409668,
      "activations/layer2_attention_weight_min": -34.174476623535156,
      "activations/layer3_attention_weight_max": 95.42305755615234,
      "activations/layer3_attention_weight_min": -94.49996948242188,
      "activations/layer4_attention_weight_max": 109.20394134521484,
      "activations/layer4_attention_weight_min": -107.4338150024414,
      "activations/layer5_attention_weight_max": 50.490867614746094,
      "activations/layer5_attention_weight_min": -57.8171501159668,
      "activations/layer6_attention_weight_max": 43.2744026184082,
      "activations/layer6_attention_weight_min": -46.8576545715332,
      "activations/layer7_attention_weight_max": 88.2203598022461,
      "activations/layer7_attention_weight_min": -88.77505493164062,
      "activations/layer8_attention_weight_max": 38.25468826293945,
      "activations/layer8_attention_weight_min": -42.72132873535156,
      "activations/layer9_attention_weight_max": 30.34208106994629,
      "activations/layer9_attention_weight_min": -30.941364288330078,
      "epoch": 12.06,
      "learning_rate": 7.294507575757575e-05,
      "loss": 2.7618,
      "step": 207600
    },
    {
      "activations/layer0_attention_weight_max": 16.45949363708496,
      "activations/layer0_attention_weight_min": -11.466544151306152,
      "activations/layer10_attention_weight_max": 35.230628967285156,
      "activations/layer10_attention_weight_min": -32.722713470458984,
      "activations/layer11_attention_weight_max": 32.9940299987793,
      "activations/layer11_attention_weight_min": -33.005977630615234,
      "activations/layer12_attention_weight_max": 24.2434024810791,
      "activations/layer12_attention_weight_min": -26.319416046142578,
      "activations/layer13_attention_weight_max": 37.86452102661133,
      "activations/layer13_attention_weight_min": -34.391754150390625,
      "activations/layer14_attention_weight_max": 41.437225341796875,
      "activations/layer14_attention_weight_min": -38.52976989746094,
      "activations/layer15_attention_weight_max": 36.005271911621094,
      "activations/layer15_attention_weight_min": -31.25527000427246,
      "activations/layer16_attention_weight_max": 29.7646427154541,
      "activations/layer16_attention_weight_min": -27.6843318939209,
      "activations/layer17_attention_weight_max": 32.162166595458984,
      "activations/layer17_attention_weight_min": -27.274362564086914,
      "activations/layer18_attention_weight_max": 30.496416091918945,
      "activations/layer18_attention_weight_min": -23.880266189575195,
      "activations/layer19_attention_weight_max": 33.795631408691406,
      "activations/layer19_attention_weight_min": -32.488155364990234,
      "activations/layer1_attention_weight_max": 16.2616024017334,
      "activations/layer1_attention_weight_min": -14.358528137207031,
      "activations/layer20_attention_weight_max": 31.263225555419922,
      "activations/layer20_attention_weight_min": -25.669475555419922,
      "activations/layer21_attention_weight_max": 28.4379940032959,
      "activations/layer21_attention_weight_min": -25.929019927978516,
      "activations/layer22_attention_weight_max": 43.33070373535156,
      "activations/layer22_attention_weight_min": -31.429712295532227,
      "activations/layer23_attention_weight_max": 36.4063720703125,
      "activations/layer23_attention_weight_min": -25.3502254486084,
      "activations/layer2_attention_weight_max": 38.058570861816406,
      "activations/layer2_attention_weight_min": -34.041038513183594,
      "activations/layer3_attention_weight_max": 99.59524536132812,
      "activations/layer3_attention_weight_min": -97.10372161865234,
      "activations/layer4_attention_weight_max": 111.28254699707031,
      "activations/layer4_attention_weight_min": -113.76954650878906,
      "activations/layer5_attention_weight_max": 52.133419036865234,
      "activations/layer5_attention_weight_min": -66.07672882080078,
      "activations/layer6_attention_weight_max": 48.183841705322266,
      "activations/layer6_attention_weight_min": -50.82674789428711,
      "activations/layer7_attention_weight_max": 93.43425750732422,
      "activations/layer7_attention_weight_min": -95.11579132080078,
      "activations/layer8_attention_weight_max": 41.474185943603516,
      "activations/layer8_attention_weight_min": -42.74895095825195,
      "activations/layer9_attention_weight_max": 35.92817687988281,
      "activations/layer9_attention_weight_min": -34.962013244628906,
      "epoch": 12.07,
      "learning_rate": 7.292613636363635e-05,
      "loss": 2.7662,
      "step": 207650
    },
    {
      "activations/layer0_attention_weight_max": 16.272300720214844,
      "activations/layer0_attention_weight_min": -12.163375854492188,
      "activations/layer10_attention_weight_max": 32.10322570800781,
      "activations/layer10_attention_weight_min": -32.876014709472656,
      "activations/layer11_attention_weight_max": 31.789146423339844,
      "activations/layer11_attention_weight_min": -33.0771598815918,
      "activations/layer12_attention_weight_max": 26.014711380004883,
      "activations/layer12_attention_weight_min": -25.94019889831543,
      "activations/layer13_attention_weight_max": 39.74591827392578,
      "activations/layer13_attention_weight_min": -35.90155029296875,
      "activations/layer14_attention_weight_max": 46.525936126708984,
      "activations/layer14_attention_weight_min": -37.92875671386719,
      "activations/layer15_attention_weight_max": 37.097816467285156,
      "activations/layer15_attention_weight_min": -31.060592651367188,
      "activations/layer16_attention_weight_max": 31.322240829467773,
      "activations/layer16_attention_weight_min": -27.93400764465332,
      "activations/layer17_attention_weight_max": 30.898889541625977,
      "activations/layer17_attention_weight_min": -25.989301681518555,
      "activations/layer18_attention_weight_max": 29.968599319458008,
      "activations/layer18_attention_weight_min": -24.415441513061523,
      "activations/layer19_attention_weight_max": 33.397132873535156,
      "activations/layer19_attention_weight_min": -30.319747924804688,
      "activations/layer1_attention_weight_max": 16.216970443725586,
      "activations/layer1_attention_weight_min": -13.569433212280273,
      "activations/layer20_attention_weight_max": 26.8051815032959,
      "activations/layer20_attention_weight_min": -25.37923812866211,
      "activations/layer21_attention_weight_max": 26.750350952148438,
      "activations/layer21_attention_weight_min": -24.45204734802246,
      "activations/layer22_attention_weight_max": 41.630184173583984,
      "activations/layer22_attention_weight_min": -29.76646614074707,
      "activations/layer23_attention_weight_max": 32.52740478515625,
      "activations/layer23_attention_weight_min": -25.409053802490234,
      "activations/layer2_attention_weight_max": 32.94945526123047,
      "activations/layer2_attention_weight_min": -32.1175422668457,
      "activations/layer3_attention_weight_max": 95.99756622314453,
      "activations/layer3_attention_weight_min": -95.5476303100586,
      "activations/layer4_attention_weight_max": 110.70326232910156,
      "activations/layer4_attention_weight_min": -110.45448303222656,
      "activations/layer5_attention_weight_max": 50.60105895996094,
      "activations/layer5_attention_weight_min": -62.66225814819336,
      "activations/layer6_attention_weight_max": 43.27421188354492,
      "activations/layer6_attention_weight_min": -46.09678649902344,
      "activations/layer7_attention_weight_max": 86.28507232666016,
      "activations/layer7_attention_weight_min": -88.39677429199219,
      "activations/layer8_attention_weight_max": 40.91516876220703,
      "activations/layer8_attention_weight_min": -44.4067268371582,
      "activations/layer9_attention_weight_max": 34.07223129272461,
      "activations/layer9_attention_weight_min": -34.07335662841797,
      "epoch": 12.07,
      "learning_rate": 7.290719696969697e-05,
      "loss": 2.7572,
      "step": 207700
    },
    {
      "activations/layer0_attention_weight_max": 16.140338897705078,
      "activations/layer0_attention_weight_min": -12.592291831970215,
      "activations/layer10_attention_weight_max": 35.2947998046875,
      "activations/layer10_attention_weight_min": -33.91585922241211,
      "activations/layer11_attention_weight_max": 35.919715881347656,
      "activations/layer11_attention_weight_min": -34.03106689453125,
      "activations/layer12_attention_weight_max": 25.313831329345703,
      "activations/layer12_attention_weight_min": -28.860231399536133,
      "activations/layer13_attention_weight_max": 39.813262939453125,
      "activations/layer13_attention_weight_min": -35.46320343017578,
      "activations/layer14_attention_weight_max": 49.34561538696289,
      "activations/layer14_attention_weight_min": -43.93256378173828,
      "activations/layer15_attention_weight_max": 35.90719985961914,
      "activations/layer15_attention_weight_min": -33.969451904296875,
      "activations/layer16_attention_weight_max": 32.08567810058594,
      "activations/layer16_attention_weight_min": -28.42365264892578,
      "activations/layer17_attention_weight_max": 31.097686767578125,
      "activations/layer17_attention_weight_min": -25.962377548217773,
      "activations/layer18_attention_weight_max": 29.20327377319336,
      "activations/layer18_attention_weight_min": -22.76059913635254,
      "activations/layer19_attention_weight_max": 34.168392181396484,
      "activations/layer19_attention_weight_min": -32.51862716674805,
      "activations/layer1_attention_weight_max": 16.127220153808594,
      "activations/layer1_attention_weight_min": -13.324458122253418,
      "activations/layer20_attention_weight_max": 28.253528594970703,
      "activations/layer20_attention_weight_min": -26.556926727294922,
      "activations/layer21_attention_weight_max": 27.26397705078125,
      "activations/layer21_attention_weight_min": -24.24017906188965,
      "activations/layer22_attention_weight_max": 52.74818420410156,
      "activations/layer22_attention_weight_min": -31.10422134399414,
      "activations/layer23_attention_weight_max": 33.970787048339844,
      "activations/layer23_attention_weight_min": -26.540027618408203,
      "activations/layer2_attention_weight_max": 34.85296630859375,
      "activations/layer2_attention_weight_min": -35.51868438720703,
      "activations/layer3_attention_weight_max": 99.66400146484375,
      "activations/layer3_attention_weight_min": -103.46969604492188,
      "activations/layer4_attention_weight_max": 114.1278305053711,
      "activations/layer4_attention_weight_min": -119.1651611328125,
      "activations/layer5_attention_weight_max": 55.476890563964844,
      "activations/layer5_attention_weight_min": -64.61007690429688,
      "activations/layer6_attention_weight_max": 44.373085021972656,
      "activations/layer6_attention_weight_min": -47.42783737182617,
      "activations/layer7_attention_weight_max": 93.35193634033203,
      "activations/layer7_attention_weight_min": -91.9967041015625,
      "activations/layer8_attention_weight_max": 42.53761672973633,
      "activations/layer8_attention_weight_min": -43.69811248779297,
      "activations/layer9_attention_weight_max": 35.2891960144043,
      "activations/layer9_attention_weight_min": -35.701786041259766,
      "epoch": 12.07,
      "learning_rate": 7.288825757575757e-05,
      "loss": 2.7701,
      "step": 207750
    },
    {
      "activations/layer0_attention_weight_max": 16.4008731842041,
      "activations/layer0_attention_weight_min": -11.401680946350098,
      "activations/layer10_attention_weight_max": 33.45193099975586,
      "activations/layer10_attention_weight_min": -32.982818603515625,
      "activations/layer11_attention_weight_max": 33.51312255859375,
      "activations/layer11_attention_weight_min": -31.980398178100586,
      "activations/layer12_attention_weight_max": 26.861642837524414,
      "activations/layer12_attention_weight_min": -25.16534423828125,
      "activations/layer13_attention_weight_max": 41.94100570678711,
      "activations/layer13_attention_weight_min": -34.18195343017578,
      "activations/layer14_attention_weight_max": 45.75078582763672,
      "activations/layer14_attention_weight_min": -38.988800048828125,
      "activations/layer15_attention_weight_max": 36.38900375366211,
      "activations/layer15_attention_weight_min": -33.127376556396484,
      "activations/layer16_attention_weight_max": 29.83408546447754,
      "activations/layer16_attention_weight_min": -26.892602920532227,
      "activations/layer17_attention_weight_max": 31.48523712158203,
      "activations/layer17_attention_weight_min": -25.448837280273438,
      "activations/layer18_attention_weight_max": 31.678789138793945,
      "activations/layer18_attention_weight_min": -22.612281799316406,
      "activations/layer19_attention_weight_max": 33.70448303222656,
      "activations/layer19_attention_weight_min": -31.804443359375,
      "activations/layer1_attention_weight_max": 16.48974609375,
      "activations/layer1_attention_weight_min": -14.11724853515625,
      "activations/layer20_attention_weight_max": 29.114728927612305,
      "activations/layer20_attention_weight_min": -26.258207321166992,
      "activations/layer21_attention_weight_max": 29.559722900390625,
      "activations/layer21_attention_weight_min": -22.682636260986328,
      "activations/layer22_attention_weight_max": 42.834346771240234,
      "activations/layer22_attention_weight_min": -30.146169662475586,
      "activations/layer23_attention_weight_max": 32.459373474121094,
      "activations/layer23_attention_weight_min": -22.98609161376953,
      "activations/layer2_attention_weight_max": 33.567325592041016,
      "activations/layer2_attention_weight_min": -32.48263168334961,
      "activations/layer3_attention_weight_max": 94.69071960449219,
      "activations/layer3_attention_weight_min": -96.48523712158203,
      "activations/layer4_attention_weight_max": 111.10931396484375,
      "activations/layer4_attention_weight_min": -110.79986572265625,
      "activations/layer5_attention_weight_max": 53.424659729003906,
      "activations/layer5_attention_weight_min": -61.37241744995117,
      "activations/layer6_attention_weight_max": 45.3786735534668,
      "activations/layer6_attention_weight_min": -48.33452606201172,
      "activations/layer7_attention_weight_max": 93.79144287109375,
      "activations/layer7_attention_weight_min": -90.3265380859375,
      "activations/layer8_attention_weight_max": 43.136558532714844,
      "activations/layer8_attention_weight_min": -42.31691360473633,
      "activations/layer9_attention_weight_max": 34.92246627807617,
      "activations/layer9_attention_weight_min": -33.073158264160156,
      "epoch": 12.07,
      "learning_rate": 7.286969696969696e-05,
      "loss": 2.7725,
      "step": 207800
    },
    {
      "activations/layer0_attention_weight_max": 15.589361190795898,
      "activations/layer0_attention_weight_min": -12.532581329345703,
      "activations/layer10_attention_weight_max": 33.51881790161133,
      "activations/layer10_attention_weight_min": -32.936405181884766,
      "activations/layer11_attention_weight_max": 33.214195251464844,
      "activations/layer11_attention_weight_min": -31.369739532470703,
      "activations/layer12_attention_weight_max": 24.946182250976562,
      "activations/layer12_attention_weight_min": -24.864587783813477,
      "activations/layer13_attention_weight_max": 40.445556640625,
      "activations/layer13_attention_weight_min": -33.20615768432617,
      "activations/layer14_attention_weight_max": 43.42182922363281,
      "activations/layer14_attention_weight_min": -35.13368225097656,
      "activations/layer15_attention_weight_max": 37.19114685058594,
      "activations/layer15_attention_weight_min": -33.22825241088867,
      "activations/layer16_attention_weight_max": 31.714689254760742,
      "activations/layer16_attention_weight_min": -29.11192512512207,
      "activations/layer17_attention_weight_max": 31.98333168029785,
      "activations/layer17_attention_weight_min": -25.689472198486328,
      "activations/layer18_attention_weight_max": 32.527828216552734,
      "activations/layer18_attention_weight_min": -23.470211029052734,
      "activations/layer19_attention_weight_max": 38.617000579833984,
      "activations/layer19_attention_weight_min": -31.751623153686523,
      "activations/layer1_attention_weight_max": 16.179990768432617,
      "activations/layer1_attention_weight_min": -15.327861785888672,
      "activations/layer20_attention_weight_max": 33.63487243652344,
      "activations/layer20_attention_weight_min": -26.68514633178711,
      "activations/layer21_attention_weight_max": 30.271244049072266,
      "activations/layer21_attention_weight_min": -25.432132720947266,
      "activations/layer22_attention_weight_max": 44.9769287109375,
      "activations/layer22_attention_weight_min": -31.87654685974121,
      "activations/layer23_attention_weight_max": 35.99755096435547,
      "activations/layer23_attention_weight_min": -24.89484405517578,
      "activations/layer2_attention_weight_max": 34.69584655761719,
      "activations/layer2_attention_weight_min": -32.35865783691406,
      "activations/layer3_attention_weight_max": 96.85062408447266,
      "activations/layer3_attention_weight_min": -95.95458221435547,
      "activations/layer4_attention_weight_max": 110.25936126708984,
      "activations/layer4_attention_weight_min": -109.37654113769531,
      "activations/layer5_attention_weight_max": 52.33312225341797,
      "activations/layer5_attention_weight_min": -59.43270492553711,
      "activations/layer6_attention_weight_max": 43.515071868896484,
      "activations/layer6_attention_weight_min": -48.55054473876953,
      "activations/layer7_attention_weight_max": 86.52338409423828,
      "activations/layer7_attention_weight_min": -91.06697082519531,
      "activations/layer8_attention_weight_max": 40.24277877807617,
      "activations/layer8_attention_weight_min": -40.040958404541016,
      "activations/layer9_attention_weight_max": 32.663116455078125,
      "activations/layer9_attention_weight_min": -33.00740051269531,
      "epoch": 12.08,
      "learning_rate": 7.285075757575757e-05,
      "loss": 2.7753,
      "step": 207850
    },
    {
      "activations/layer0_attention_weight_max": 15.977190017700195,
      "activations/layer0_attention_weight_min": -12.220476150512695,
      "activations/layer10_attention_weight_max": 31.362985610961914,
      "activations/layer10_attention_weight_min": -33.64671325683594,
      "activations/layer11_attention_weight_max": 31.58123016357422,
      "activations/layer11_attention_weight_min": -32.1856575012207,
      "activations/layer12_attention_weight_max": 26.51772689819336,
      "activations/layer12_attention_weight_min": -25.02049446105957,
      "activations/layer13_attention_weight_max": 38.79882049560547,
      "activations/layer13_attention_weight_min": -34.43281173706055,
      "activations/layer14_attention_weight_max": 42.00633239746094,
      "activations/layer14_attention_weight_min": -35.94609069824219,
      "activations/layer15_attention_weight_max": 38.72552490234375,
      "activations/layer15_attention_weight_min": -34.13182830810547,
      "activations/layer16_attention_weight_max": 31.674707412719727,
      "activations/layer16_attention_weight_min": -26.876379013061523,
      "activations/layer17_attention_weight_max": 36.879920959472656,
      "activations/layer17_attention_weight_min": -27.18430519104004,
      "activations/layer18_attention_weight_max": 32.38261032104492,
      "activations/layer18_attention_weight_min": -23.608144760131836,
      "activations/layer19_attention_weight_max": 40.45256423950195,
      "activations/layer19_attention_weight_min": -31.53595542907715,
      "activations/layer1_attention_weight_max": 17.28989028930664,
      "activations/layer1_attention_weight_min": -12.564250946044922,
      "activations/layer20_attention_weight_max": 32.44886016845703,
      "activations/layer20_attention_weight_min": -26.04051399230957,
      "activations/layer21_attention_weight_max": 33.2739372253418,
      "activations/layer21_attention_weight_min": -24.50960350036621,
      "activations/layer22_attention_weight_max": 49.04798889160156,
      "activations/layer22_attention_weight_min": -30.7279052734375,
      "activations/layer23_attention_weight_max": 34.648597717285156,
      "activations/layer23_attention_weight_min": -27.34521484375,
      "activations/layer2_attention_weight_max": 35.31665802001953,
      "activations/layer2_attention_weight_min": -32.81640625,
      "activations/layer3_attention_weight_max": 98.66386413574219,
      "activations/layer3_attention_weight_min": -96.4451904296875,
      "activations/layer4_attention_weight_max": 116.6036148071289,
      "activations/layer4_attention_weight_min": -117.00068664550781,
      "activations/layer5_attention_weight_max": 53.319114685058594,
      "activations/layer5_attention_weight_min": -64.27584075927734,
      "activations/layer6_attention_weight_max": 43.598838806152344,
      "activations/layer6_attention_weight_min": -46.70790100097656,
      "activations/layer7_attention_weight_max": 92.81301879882812,
      "activations/layer7_attention_weight_min": -90.8099365234375,
      "activations/layer8_attention_weight_max": 40.57039260864258,
      "activations/layer8_attention_weight_min": -39.43161392211914,
      "activations/layer9_attention_weight_max": 32.08365249633789,
      "activations/layer9_attention_weight_min": -33.1431999206543,
      "epoch": 12.08,
      "learning_rate": 7.283181818181818e-05,
      "loss": 2.7623,
      "step": 207900
    },
    {
      "activations/layer0_attention_weight_max": 16.282142639160156,
      "activations/layer0_attention_weight_min": -12.236573219299316,
      "activations/layer10_attention_weight_max": 31.25445556640625,
      "activations/layer10_attention_weight_min": -30.96095848083496,
      "activations/layer11_attention_weight_max": 31.748563766479492,
      "activations/layer11_attention_weight_min": -31.01856231689453,
      "activations/layer12_attention_weight_max": 26.614791870117188,
      "activations/layer12_attention_weight_min": -24.774173736572266,
      "activations/layer13_attention_weight_max": 37.98647689819336,
      "activations/layer13_attention_weight_min": -33.88874435424805,
      "activations/layer14_attention_weight_max": 43.02962875366211,
      "activations/layer14_attention_weight_min": -34.14360046386719,
      "activations/layer15_attention_weight_max": 35.023895263671875,
      "activations/layer15_attention_weight_min": -29.602603912353516,
      "activations/layer16_attention_weight_max": 31.99706268310547,
      "activations/layer16_attention_weight_min": -28.520999908447266,
      "activations/layer17_attention_weight_max": 30.99592399597168,
      "activations/layer17_attention_weight_min": -23.86962890625,
      "activations/layer18_attention_weight_max": 31.799142837524414,
      "activations/layer18_attention_weight_min": -21.649044036865234,
      "activations/layer19_attention_weight_max": 36.261924743652344,
      "activations/layer19_attention_weight_min": -27.008895874023438,
      "activations/layer1_attention_weight_max": 16.707805633544922,
      "activations/layer1_attention_weight_min": -14.216609954833984,
      "activations/layer20_attention_weight_max": 27.783721923828125,
      "activations/layer20_attention_weight_min": -22.80176544189453,
      "activations/layer21_attention_weight_max": 26.71653175354004,
      "activations/layer21_attention_weight_min": -21.8493709564209,
      "activations/layer22_attention_weight_max": 42.792354583740234,
      "activations/layer22_attention_weight_min": -27.01922607421875,
      "activations/layer23_attention_weight_max": 36.36640167236328,
      "activations/layer23_attention_weight_min": -24.563335418701172,
      "activations/layer2_attention_weight_max": 31.794471740722656,
      "activations/layer2_attention_weight_min": -31.863933563232422,
      "activations/layer3_attention_weight_max": 94.07762145996094,
      "activations/layer3_attention_weight_min": -100.05265045166016,
      "activations/layer4_attention_weight_max": 108.70134735107422,
      "activations/layer4_attention_weight_min": -114.0695571899414,
      "activations/layer5_attention_weight_max": 50.90880584716797,
      "activations/layer5_attention_weight_min": -62.1174430847168,
      "activations/layer6_attention_weight_max": 44.84818649291992,
      "activations/layer6_attention_weight_min": -47.97962951660156,
      "activations/layer7_attention_weight_max": 88.13993072509766,
      "activations/layer7_attention_weight_min": -90.6344223022461,
      "activations/layer8_attention_weight_max": 39.00736999511719,
      "activations/layer8_attention_weight_min": -41.79977035522461,
      "activations/layer9_attention_weight_max": 31.27699089050293,
      "activations/layer9_attention_weight_min": -32.6226921081543,
      "epoch": 12.08,
      "learning_rate": 7.281287878787878e-05,
      "loss": 2.7686,
      "step": 207950
    },
    {
      "activations/layer0_attention_weight_max": 15.627546310424805,
      "activations/layer0_attention_weight_min": -11.460244178771973,
      "activations/layer10_attention_weight_max": 31.9249267578125,
      "activations/layer10_attention_weight_min": -33.01263427734375,
      "activations/layer11_attention_weight_max": 32.779170989990234,
      "activations/layer11_attention_weight_min": -32.76752853393555,
      "activations/layer12_attention_weight_max": 25.14826774597168,
      "activations/layer12_attention_weight_min": -26.545530319213867,
      "activations/layer13_attention_weight_max": 43.435298919677734,
      "activations/layer13_attention_weight_min": -36.591957092285156,
      "activations/layer14_attention_weight_max": 46.80240249633789,
      "activations/layer14_attention_weight_min": -38.62453079223633,
      "activations/layer15_attention_weight_max": 36.40686798095703,
      "activations/layer15_attention_weight_min": -33.333709716796875,
      "activations/layer16_attention_weight_max": 31.732816696166992,
      "activations/layer16_attention_weight_min": -27.361408233642578,
      "activations/layer17_attention_weight_max": 31.83894157409668,
      "activations/layer17_attention_weight_min": -25.245882034301758,
      "activations/layer18_attention_weight_max": 30.177413940429688,
      "activations/layer18_attention_weight_min": -25.34636116027832,
      "activations/layer19_attention_weight_max": 32.59017562866211,
      "activations/layer19_attention_weight_min": -31.892292022705078,
      "activations/layer1_attention_weight_max": 16.735929489135742,
      "activations/layer1_attention_weight_min": -15.35245418548584,
      "activations/layer20_attention_weight_max": 28.6560115814209,
      "activations/layer20_attention_weight_min": -25.341867446899414,
      "activations/layer21_attention_weight_max": 28.76430892944336,
      "activations/layer21_attention_weight_min": -24.235809326171875,
      "activations/layer22_attention_weight_max": 43.54116439819336,
      "activations/layer22_attention_weight_min": -30.25787925720215,
      "activations/layer23_attention_weight_max": 33.14350891113281,
      "activations/layer23_attention_weight_min": -25.326818466186523,
      "activations/layer2_attention_weight_max": 33.43960952758789,
      "activations/layer2_attention_weight_min": -32.284053802490234,
      "activations/layer3_attention_weight_max": 95.27374267578125,
      "activations/layer3_attention_weight_min": -97.06487274169922,
      "activations/layer4_attention_weight_max": 113.97698211669922,
      "activations/layer4_attention_weight_min": -115.26778411865234,
      "activations/layer5_attention_weight_max": 54.903987884521484,
      "activations/layer5_attention_weight_min": -63.40098190307617,
      "activations/layer6_attention_weight_max": 43.98055648803711,
      "activations/layer6_attention_weight_min": -48.65347671508789,
      "activations/layer7_attention_weight_max": 92.15515899658203,
      "activations/layer7_attention_weight_min": -89.17610168457031,
      "activations/layer8_attention_weight_max": 38.668174743652344,
      "activations/layer8_attention_weight_min": -41.311378479003906,
      "activations/layer9_attention_weight_max": 33.020442962646484,
      "activations/layer9_attention_weight_min": -32.66820526123047,
      "epoch": 12.09,
      "learning_rate": 7.279393939393938e-05,
      "loss": 2.7724,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.5575,
      "eval_samples_per_second": 501.782,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.5575,
      "eval_openwebtext_samples_per_second": 501.782,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 2.0437,
      "eval_wikitext_samples_per_second": 223.123,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_lambada_loss": 2.490234375,
      "eval_lambada_ppl": 12.064103313334172,
      "eval_lambada_runtime": 9.6338,
      "eval_lambada_samples_per_second": 505.405,
      "step": 208000
    },
    {
      "activations/layer0_attention_weight_max": 16.525819778442383,
      "activations/layer0_attention_weight_min": -13.100485801696777,
      "activations/layer10_attention_weight_max": 32.586761474609375,
      "activations/layer10_attention_weight_min": -32.7773323059082,
      "activations/layer11_attention_weight_max": 31.147541046142578,
      "activations/layer11_attention_weight_min": -30.76791000366211,
      "activations/layer12_attention_weight_max": 24.906818389892578,
      "activations/layer12_attention_weight_min": -24.525569915771484,
      "activations/layer13_attention_weight_max": 41.26224899291992,
      "activations/layer13_attention_weight_min": -33.45594787597656,
      "activations/layer14_attention_weight_max": 39.513187408447266,
      "activations/layer14_attention_weight_min": -36.02437973022461,
      "activations/layer15_attention_weight_max": 47.06727981567383,
      "activations/layer15_attention_weight_min": -32.15776443481445,
      "activations/layer16_attention_weight_max": 39.32602310180664,
      "activations/layer16_attention_weight_min": -27.65352439880371,
      "activations/layer17_attention_weight_max": 35.10866165161133,
      "activations/layer17_attention_weight_min": -25.584745407104492,
      "activations/layer18_attention_weight_max": 33.447784423828125,
      "activations/layer18_attention_weight_min": -25.93423080444336,
      "activations/layer19_attention_weight_max": 33.451942443847656,
      "activations/layer19_attention_weight_min": -30.303754806518555,
      "activations/layer1_attention_weight_max": 17.391035079956055,
      "activations/layer1_attention_weight_min": -15.84188175201416,
      "activations/layer20_attention_weight_max": 28.505348205566406,
      "activations/layer20_attention_weight_min": -22.855247497558594,
      "activations/layer21_attention_weight_max": 28.553152084350586,
      "activations/layer21_attention_weight_min": -23.193920135498047,
      "activations/layer22_attention_weight_max": 42.24273681640625,
      "activations/layer22_attention_weight_min": -29.488143920898438,
      "activations/layer23_attention_weight_max": 31.391162872314453,
      "activations/layer23_attention_weight_min": -25.652992248535156,
      "activations/layer2_attention_weight_max": 33.397666931152344,
      "activations/layer2_attention_weight_min": -31.69278335571289,
      "activations/layer3_attention_weight_max": 95.07415771484375,
      "activations/layer3_attention_weight_min": -93.43426513671875,
      "activations/layer4_attention_weight_max": 111.30644989013672,
      "activations/layer4_attention_weight_min": -106.55632781982422,
      "activations/layer5_attention_weight_max": 51.87242889404297,
      "activations/layer5_attention_weight_min": -58.58333969116211,
      "activations/layer6_attention_weight_max": 44.339576721191406,
      "activations/layer6_attention_weight_min": -46.841712951660156,
      "activations/layer7_attention_weight_max": 87.02518463134766,
      "activations/layer7_attention_weight_min": -85.29794311523438,
      "activations/layer8_attention_weight_max": 39.67856979370117,
      "activations/layer8_attention_weight_min": -40.04801559448242,
      "activations/layer9_attention_weight_max": 30.843076705932617,
      "activations/layer9_attention_weight_min": -31.616241455078125,
      "epoch": 12.09,
      "learning_rate": 7.2775e-05,
      "loss": 2.771,
      "step": 208050
    },
    {
      "activations/layer0_attention_weight_max": 16.026477813720703,
      "activations/layer0_attention_weight_min": -13.379822731018066,
      "activations/layer10_attention_weight_max": 31.019468307495117,
      "activations/layer10_attention_weight_min": -35.67521286010742,
      "activations/layer11_attention_weight_max": 30.24587631225586,
      "activations/layer11_attention_weight_min": -32.51393127441406,
      "activations/layer12_attention_weight_max": 23.429340362548828,
      "activations/layer12_attention_weight_min": -24.412979125976562,
      "activations/layer13_attention_weight_max": 38.49485778808594,
      "activations/layer13_attention_weight_min": -32.79515075683594,
      "activations/layer14_attention_weight_max": 45.4117546081543,
      "activations/layer14_attention_weight_min": -34.88239669799805,
      "activations/layer15_attention_weight_max": 33.98768997192383,
      "activations/layer15_attention_weight_min": -33.1446647644043,
      "activations/layer16_attention_weight_max": 32.339569091796875,
      "activations/layer16_attention_weight_min": -29.42422866821289,
      "activations/layer17_attention_weight_max": 29.17890739440918,
      "activations/layer17_attention_weight_min": -26.69122886657715,
      "activations/layer18_attention_weight_max": 31.56889533996582,
      "activations/layer18_attention_weight_min": -25.50371742248535,
      "activations/layer19_attention_weight_max": 31.57745933532715,
      "activations/layer19_attention_weight_min": -30.69182777404785,
      "activations/layer1_attention_weight_max": 16.32851219177246,
      "activations/layer1_attention_weight_min": -14.416090965270996,
      "activations/layer20_attention_weight_max": 27.54439353942871,
      "activations/layer20_attention_weight_min": -25.639585494995117,
      "activations/layer21_attention_weight_max": 26.10124397277832,
      "activations/layer21_attention_weight_min": -24.44673728942871,
      "activations/layer22_attention_weight_max": 39.56913757324219,
      "activations/layer22_attention_weight_min": -30.211830139160156,
      "activations/layer23_attention_weight_max": 36.49749755859375,
      "activations/layer23_attention_weight_min": -24.018844604492188,
      "activations/layer2_attention_weight_max": 33.953704833984375,
      "activations/layer2_attention_weight_min": -31.657367706298828,
      "activations/layer3_attention_weight_max": 93.62468719482422,
      "activations/layer3_attention_weight_min": -94.37503814697266,
      "activations/layer4_attention_weight_max": 112.22029113769531,
      "activations/layer4_attention_weight_min": -109.56074523925781,
      "activations/layer5_attention_weight_max": 52.65386962890625,
      "activations/layer5_attention_weight_min": -59.02362060546875,
      "activations/layer6_attention_weight_max": 45.37928771972656,
      "activations/layer6_attention_weight_min": -44.78926086425781,
      "activations/layer7_attention_weight_max": 89.30833435058594,
      "activations/layer7_attention_weight_min": -88.9474105834961,
      "activations/layer8_attention_weight_max": 37.11123275756836,
      "activations/layer8_attention_weight_min": -39.107215881347656,
      "activations/layer9_attention_weight_max": 29.780899047851562,
      "activations/layer9_attention_weight_min": -33.4998779296875,
      "epoch": 12.09,
      "learning_rate": 7.27560606060606e-05,
      "loss": 2.7582,
      "step": 208100
    },
    {
      "activations/layer0_attention_weight_max": 16.64957618713379,
      "activations/layer0_attention_weight_min": -11.971222877502441,
      "activations/layer10_attention_weight_max": 35.1514892578125,
      "activations/layer10_attention_weight_min": -31.843379974365234,
      "activations/layer11_attention_weight_max": 33.28779602050781,
      "activations/layer11_attention_weight_min": -31.7165584564209,
      "activations/layer12_attention_weight_max": 26.152929306030273,
      "activations/layer12_attention_weight_min": -25.193992614746094,
      "activations/layer13_attention_weight_max": 39.2397575378418,
      "activations/layer13_attention_weight_min": -33.712677001953125,
      "activations/layer14_attention_weight_max": 49.2445068359375,
      "activations/layer14_attention_weight_min": -40.922706604003906,
      "activations/layer15_attention_weight_max": 38.810237884521484,
      "activations/layer15_attention_weight_min": -30.81116485595703,
      "activations/layer16_attention_weight_max": 31.18352508544922,
      "activations/layer16_attention_weight_min": -26.58099365234375,
      "activations/layer17_attention_weight_max": 34.428993225097656,
      "activations/layer17_attention_weight_min": -24.52883529663086,
      "activations/layer18_attention_weight_max": 32.01392364501953,
      "activations/layer18_attention_weight_min": -23.87423324584961,
      "activations/layer19_attention_weight_max": 33.308746337890625,
      "activations/layer19_attention_weight_min": -31.321674346923828,
      "activations/layer1_attention_weight_max": 15.935850143432617,
      "activations/layer1_attention_weight_min": -14.83553695678711,
      "activations/layer20_attention_weight_max": 26.174747467041016,
      "activations/layer20_attention_weight_min": -23.420690536499023,
      "activations/layer21_attention_weight_max": 26.41337776184082,
      "activations/layer21_attention_weight_min": -23.89683723449707,
      "activations/layer22_attention_weight_max": 43.348731994628906,
      "activations/layer22_attention_weight_min": -29.161468505859375,
      "activations/layer23_attention_weight_max": 35.05118179321289,
      "activations/layer23_attention_weight_min": -23.20574951171875,
      "activations/layer2_attention_weight_max": 33.03447341918945,
      "activations/layer2_attention_weight_min": -32.100311279296875,
      "activations/layer3_attention_weight_max": 94.79178619384766,
      "activations/layer3_attention_weight_min": -94.9380111694336,
      "activations/layer4_attention_weight_max": 116.1596908569336,
      "activations/layer4_attention_weight_min": -114.11981201171875,
      "activations/layer5_attention_weight_max": 52.55403137207031,
      "activations/layer5_attention_weight_min": -61.57013702392578,
      "activations/layer6_attention_weight_max": 46.02983474731445,
      "activations/layer6_attention_weight_min": -47.148250579833984,
      "activations/layer7_attention_weight_max": 91.14061737060547,
      "activations/layer7_attention_weight_min": -88.77716064453125,
      "activations/layer8_attention_weight_max": 41.292686462402344,
      "activations/layer8_attention_weight_min": -43.143341064453125,
      "activations/layer9_attention_weight_max": 34.465553283691406,
      "activations/layer9_attention_weight_min": -34.5020751953125,
      "epoch": 12.09,
      "learning_rate": 7.273712121212121e-05,
      "loss": 2.7593,
      "step": 208150
    },
    {
      "activations/layer0_attention_weight_max": 16.155792236328125,
      "activations/layer0_attention_weight_min": -11.648468017578125,
      "activations/layer10_attention_weight_max": 33.732887268066406,
      "activations/layer10_attention_weight_min": -33.418304443359375,
      "activations/layer11_attention_weight_max": 35.85716247558594,
      "activations/layer11_attention_weight_min": -33.73944854736328,
      "activations/layer12_attention_weight_max": 27.620113372802734,
      "activations/layer12_attention_weight_min": -27.965267181396484,
      "activations/layer13_attention_weight_max": 42.43243408203125,
      "activations/layer13_attention_weight_min": -37.709739685058594,
      "activations/layer14_attention_weight_max": 51.35650634765625,
      "activations/layer14_attention_weight_min": -40.422210693359375,
      "activations/layer15_attention_weight_max": 41.47020721435547,
      "activations/layer15_attention_weight_min": -38.799835205078125,
      "activations/layer16_attention_weight_max": 32.95301818847656,
      "activations/layer16_attention_weight_min": -30.147342681884766,
      "activations/layer17_attention_weight_max": 37.172096252441406,
      "activations/layer17_attention_weight_min": -28.62880516052246,
      "activations/layer18_attention_weight_max": 32.36090087890625,
      "activations/layer18_attention_weight_min": -24.633869171142578,
      "activations/layer19_attention_weight_max": 35.571998596191406,
      "activations/layer19_attention_weight_min": -30.984785079956055,
      "activations/layer1_attention_weight_max": 16.93498992919922,
      "activations/layer1_attention_weight_min": -13.747171401977539,
      "activations/layer20_attention_weight_max": 32.08320999145508,
      "activations/layer20_attention_weight_min": -26.977970123291016,
      "activations/layer21_attention_weight_max": 32.31927490234375,
      "activations/layer21_attention_weight_min": -24.196794509887695,
      "activations/layer22_attention_weight_max": 54.7413444519043,
      "activations/layer22_attention_weight_min": -30.55612564086914,
      "activations/layer23_attention_weight_max": 40.56700134277344,
      "activations/layer23_attention_weight_min": -25.69533348083496,
      "activations/layer2_attention_weight_max": 32.43906784057617,
      "activations/layer2_attention_weight_min": -33.120994567871094,
      "activations/layer3_attention_weight_max": 96.27274322509766,
      "activations/layer3_attention_weight_min": -99.03909301757812,
      "activations/layer4_attention_weight_max": 113.95088958740234,
      "activations/layer4_attention_weight_min": -115.76942443847656,
      "activations/layer5_attention_weight_max": 52.924468994140625,
      "activations/layer5_attention_weight_min": -61.4604606628418,
      "activations/layer6_attention_weight_max": 44.24479293823242,
      "activations/layer6_attention_weight_min": -46.64289474487305,
      "activations/layer7_attention_weight_max": 89.96011352539062,
      "activations/layer7_attention_weight_min": -92.24890899658203,
      "activations/layer8_attention_weight_max": 41.45924758911133,
      "activations/layer8_attention_weight_min": -43.606117248535156,
      "activations/layer9_attention_weight_max": 34.59147262573242,
      "activations/layer9_attention_weight_min": -34.51901626586914,
      "epoch": 12.1,
      "learning_rate": 7.271818181818181e-05,
      "loss": 2.7719,
      "step": 208200
    },
    {
      "activations/layer0_attention_weight_max": 15.139495849609375,
      "activations/layer0_attention_weight_min": -11.856127738952637,
      "activations/layer10_attention_weight_max": 38.62165069580078,
      "activations/layer10_attention_weight_min": -37.255821228027344,
      "activations/layer11_attention_weight_max": 40.7530403137207,
      "activations/layer11_attention_weight_min": -37.88817596435547,
      "activations/layer12_attention_weight_max": 41.665008544921875,
      "activations/layer12_attention_weight_min": -27.250873565673828,
      "activations/layer13_attention_weight_max": 58.699424743652344,
      "activations/layer13_attention_weight_min": -34.72334289550781,
      "activations/layer14_attention_weight_max": 52.30940628051758,
      "activations/layer14_attention_weight_min": -39.57988739013672,
      "activations/layer15_attention_weight_max": 58.33852005004883,
      "activations/layer15_attention_weight_min": -33.648765563964844,
      "activations/layer16_attention_weight_max": 38.01818084716797,
      "activations/layer16_attention_weight_min": -29.00758934020996,
      "activations/layer17_attention_weight_max": 32.21186065673828,
      "activations/layer17_attention_weight_min": -26.543701171875,
      "activations/layer18_attention_weight_max": 33.15642547607422,
      "activations/layer18_attention_weight_min": -24.6798095703125,
      "activations/layer19_attention_weight_max": 33.63986587524414,
      "activations/layer19_attention_weight_min": -32.60988998413086,
      "activations/layer1_attention_weight_max": 15.851140022277832,
      "activations/layer1_attention_weight_min": -14.215977668762207,
      "activations/layer20_attention_weight_max": 30.83429527282715,
      "activations/layer20_attention_weight_min": -25.796117782592773,
      "activations/layer21_attention_weight_max": 34.69640350341797,
      "activations/layer21_attention_weight_min": -24.926677703857422,
      "activations/layer22_attention_weight_max": 44.8478889465332,
      "activations/layer22_attention_weight_min": -33.210567474365234,
      "activations/layer23_attention_weight_max": 37.739830017089844,
      "activations/layer23_attention_weight_min": -24.871923446655273,
      "activations/layer2_attention_weight_max": 34.75574493408203,
      "activations/layer2_attention_weight_min": -33.55863571166992,
      "activations/layer3_attention_weight_max": 98.1137924194336,
      "activations/layer3_attention_weight_min": -100.19119262695312,
      "activations/layer4_attention_weight_max": 114.88268280029297,
      "activations/layer4_attention_weight_min": -117.28716278076172,
      "activations/layer5_attention_weight_max": 53.859092712402344,
      "activations/layer5_attention_weight_min": -64.70553588867188,
      "activations/layer6_attention_weight_max": 44.52663040161133,
      "activations/layer6_attention_weight_min": -50.0292854309082,
      "activations/layer7_attention_weight_max": 96.4892578125,
      "activations/layer7_attention_weight_min": -102.81352233886719,
      "activations/layer8_attention_weight_max": 41.67146301269531,
      "activations/layer8_attention_weight_min": -44.83808135986328,
      "activations/layer9_attention_weight_max": 35.50751876831055,
      "activations/layer9_attention_weight_min": -36.90140914916992,
      "epoch": 12.1,
      "learning_rate": 7.269924242424243e-05,
      "loss": 2.7451,
      "step": 208250
    },
    {
      "activations/layer0_attention_weight_max": 14.885331153869629,
      "activations/layer0_attention_weight_min": -12.30453872680664,
      "activations/layer10_attention_weight_max": 33.94926071166992,
      "activations/layer10_attention_weight_min": -35.33817672729492,
      "activations/layer11_attention_weight_max": 36.52800369262695,
      "activations/layer11_attention_weight_min": -35.110595703125,
      "activations/layer12_attention_weight_max": 28.868770599365234,
      "activations/layer12_attention_weight_min": -26.453954696655273,
      "activations/layer13_attention_weight_max": 40.88959503173828,
      "activations/layer13_attention_weight_min": -35.24738693237305,
      "activations/layer14_attention_weight_max": 45.677337646484375,
      "activations/layer14_attention_weight_min": -40.51664733886719,
      "activations/layer15_attention_weight_max": 39.30854797363281,
      "activations/layer15_attention_weight_min": -34.8102912902832,
      "activations/layer16_attention_weight_max": 31.556425094604492,
      "activations/layer16_attention_weight_min": -28.413253784179688,
      "activations/layer17_attention_weight_max": 31.005916595458984,
      "activations/layer17_attention_weight_min": -27.2291259765625,
      "activations/layer18_attention_weight_max": 30.81501579284668,
      "activations/layer18_attention_weight_min": -24.618581771850586,
      "activations/layer19_attention_weight_max": 40.27780532836914,
      "activations/layer19_attention_weight_min": -32.56404113769531,
      "activations/layer1_attention_weight_max": 16.723478317260742,
      "activations/layer1_attention_weight_min": -15.791403770446777,
      "activations/layer20_attention_weight_max": 30.407352447509766,
      "activations/layer20_attention_weight_min": -27.014135360717773,
      "activations/layer21_attention_weight_max": 30.22029685974121,
      "activations/layer21_attention_weight_min": -25.49893569946289,
      "activations/layer22_attention_weight_max": 47.739906311035156,
      "activations/layer22_attention_weight_min": -32.21769332885742,
      "activations/layer23_attention_weight_max": 33.51445770263672,
      "activations/layer23_attention_weight_min": -25.587453842163086,
      "activations/layer2_attention_weight_max": 33.91123962402344,
      "activations/layer2_attention_weight_min": -33.32793045043945,
      "activations/layer3_attention_weight_max": 102.11805725097656,
      "activations/layer3_attention_weight_min": -102.38792419433594,
      "activations/layer4_attention_weight_max": 116.84791564941406,
      "activations/layer4_attention_weight_min": -117.47962951660156,
      "activations/layer5_attention_weight_max": 54.654884338378906,
      "activations/layer5_attention_weight_min": -63.94301986694336,
      "activations/layer6_attention_weight_max": 48.41038513183594,
      "activations/layer6_attention_weight_min": -52.47184371948242,
      "activations/layer7_attention_weight_max": 101.12169647216797,
      "activations/layer7_attention_weight_min": -99.3067626953125,
      "activations/layer8_attention_weight_max": 43.66810989379883,
      "activations/layer8_attention_weight_min": -47.496063232421875,
      "activations/layer9_attention_weight_max": 36.83186721801758,
      "activations/layer9_attention_weight_min": -36.6906623840332,
      "epoch": 12.1,
      "learning_rate": 7.268030303030303e-05,
      "loss": 2.7497,
      "step": 208300
    },
    {
      "activations/layer0_attention_weight_max": 16.562782287597656,
      "activations/layer0_attention_weight_min": -12.330561637878418,
      "activations/layer10_attention_weight_max": 30.16185188293457,
      "activations/layer10_attention_weight_min": -31.5041561126709,
      "activations/layer11_attention_weight_max": 29.634458541870117,
      "activations/layer11_attention_weight_min": -30.653488159179688,
      "activations/layer12_attention_weight_max": 23.082406997680664,
      "activations/layer12_attention_weight_min": -24.987829208374023,
      "activations/layer13_attention_weight_max": 34.0848274230957,
      "activations/layer13_attention_weight_min": -33.81993103027344,
      "activations/layer14_attention_weight_max": 39.09449005126953,
      "activations/layer14_attention_weight_min": -34.92549133300781,
      "activations/layer15_attention_weight_max": 33.22540283203125,
      "activations/layer15_attention_weight_min": -31.549945831298828,
      "activations/layer16_attention_weight_max": 31.256681442260742,
      "activations/layer16_attention_weight_min": -29.442201614379883,
      "activations/layer17_attention_weight_max": 29.22230339050293,
      "activations/layer17_attention_weight_min": -27.833858489990234,
      "activations/layer18_attention_weight_max": 31.430423736572266,
      "activations/layer18_attention_weight_min": -22.658184051513672,
      "activations/layer19_attention_weight_max": 30.717472076416016,
      "activations/layer19_attention_weight_min": -31.633316040039062,
      "activations/layer1_attention_weight_max": 17.201797485351562,
      "activations/layer1_attention_weight_min": -13.7052583694458,
      "activations/layer20_attention_weight_max": 25.967382431030273,
      "activations/layer20_attention_weight_min": -23.61309051513672,
      "activations/layer21_attention_weight_max": 31.31534767150879,
      "activations/layer21_attention_weight_min": -23.34593391418457,
      "activations/layer22_attention_weight_max": 42.68035888671875,
      "activations/layer22_attention_weight_min": -29.68427085876465,
      "activations/layer23_attention_weight_max": 33.04045104980469,
      "activations/layer23_attention_weight_min": -24.534236907958984,
      "activations/layer2_attention_weight_max": 33.96098327636719,
      "activations/layer2_attention_weight_min": -33.214141845703125,
      "activations/layer3_attention_weight_max": 95.02207946777344,
      "activations/layer3_attention_weight_min": -96.3554916381836,
      "activations/layer4_attention_weight_max": 113.97574615478516,
      "activations/layer4_attention_weight_min": -116.24547576904297,
      "activations/layer5_attention_weight_max": 53.451473236083984,
      "activations/layer5_attention_weight_min": -60.81534957885742,
      "activations/layer6_attention_weight_max": 43.11369323730469,
      "activations/layer6_attention_weight_min": -47.15807342529297,
      "activations/layer7_attention_weight_max": 86.9689712524414,
      "activations/layer7_attention_weight_min": -89.21695709228516,
      "activations/layer8_attention_weight_max": 37.95048904418945,
      "activations/layer8_attention_weight_min": -41.17734909057617,
      "activations/layer9_attention_weight_max": 30.054916381835938,
      "activations/layer9_attention_weight_min": -31.867740631103516,
      "epoch": 12.11,
      "learning_rate": 7.266136363636363e-05,
      "loss": 2.7615,
      "step": 208350
    },
    {
      "activations/layer0_attention_weight_max": 16.3812313079834,
      "activations/layer0_attention_weight_min": -12.354903221130371,
      "activations/layer10_attention_weight_max": 31.902618408203125,
      "activations/layer10_attention_weight_min": -32.83028793334961,
      "activations/layer11_attention_weight_max": 30.681676864624023,
      "activations/layer11_attention_weight_min": -31.972768783569336,
      "activations/layer12_attention_weight_max": 23.981897354125977,
      "activations/layer12_attention_weight_min": -25.4341983795166,
      "activations/layer13_attention_weight_max": 37.03301239013672,
      "activations/layer13_attention_weight_min": -35.10879898071289,
      "activations/layer14_attention_weight_max": 44.34862518310547,
      "activations/layer14_attention_weight_min": -38.558250427246094,
      "activations/layer15_attention_weight_max": 33.84636688232422,
      "activations/layer15_attention_weight_min": -32.160518646240234,
      "activations/layer16_attention_weight_max": 28.84597396850586,
      "activations/layer16_attention_weight_min": -26.299985885620117,
      "activations/layer17_attention_weight_max": 30.342458724975586,
      "activations/layer17_attention_weight_min": -24.579545974731445,
      "activations/layer18_attention_weight_max": 33.17037582397461,
      "activations/layer18_attention_weight_min": -23.32759666442871,
      "activations/layer19_attention_weight_max": 32.40465545654297,
      "activations/layer19_attention_weight_min": -28.807373046875,
      "activations/layer1_attention_weight_max": 16.112815856933594,
      "activations/layer1_attention_weight_min": -15.04007625579834,
      "activations/layer20_attention_weight_max": 33.269283294677734,
      "activations/layer20_attention_weight_min": -24.296789169311523,
      "activations/layer21_attention_weight_max": 28.002334594726562,
      "activations/layer21_attention_weight_min": -23.40013885498047,
      "activations/layer22_attention_weight_max": 42.90518569946289,
      "activations/layer22_attention_weight_min": -28.711164474487305,
      "activations/layer23_attention_weight_max": 32.90728759765625,
      "activations/layer23_attention_weight_min": -25.876750946044922,
      "activations/layer2_attention_weight_max": 33.63963317871094,
      "activations/layer2_attention_weight_min": -32.7420654296875,
      "activations/layer3_attention_weight_max": 97.94931030273438,
      "activations/layer3_attention_weight_min": -97.56800079345703,
      "activations/layer4_attention_weight_max": 111.6259994506836,
      "activations/layer4_attention_weight_min": -111.19792938232422,
      "activations/layer5_attention_weight_max": 51.81519317626953,
      "activations/layer5_attention_weight_min": -59.73108673095703,
      "activations/layer6_attention_weight_max": 45.446510314941406,
      "activations/layer6_attention_weight_min": -48.0523567199707,
      "activations/layer7_attention_weight_max": 86.85316467285156,
      "activations/layer7_attention_weight_min": -93.03494262695312,
      "activations/layer8_attention_weight_max": 41.96565246582031,
      "activations/layer8_attention_weight_min": -44.506649017333984,
      "activations/layer9_attention_weight_max": 33.034698486328125,
      "activations/layer9_attention_weight_min": -32.99132537841797,
      "epoch": 12.11,
      "learning_rate": 7.264242424242423e-05,
      "loss": 2.7613,
      "step": 208400
    },
    {
      "activations/layer0_attention_weight_max": 16.257944107055664,
      "activations/layer0_attention_weight_min": -12.945839881896973,
      "activations/layer10_attention_weight_max": 34.986759185791016,
      "activations/layer10_attention_weight_min": -34.01884460449219,
      "activations/layer11_attention_weight_max": 33.304847717285156,
      "activations/layer11_attention_weight_min": -33.59344482421875,
      "activations/layer12_attention_weight_max": 29.491050720214844,
      "activations/layer12_attention_weight_min": -27.902183532714844,
      "activations/layer13_attention_weight_max": 46.88639831542969,
      "activations/layer13_attention_weight_min": -37.453285217285156,
      "activations/layer14_attention_weight_max": 47.709564208984375,
      "activations/layer14_attention_weight_min": -43.31136703491211,
      "activations/layer15_attention_weight_max": 40.79469680786133,
      "activations/layer15_attention_weight_min": -35.07205581665039,
      "activations/layer16_attention_weight_max": 31.133089065551758,
      "activations/layer16_attention_weight_min": -27.78346824645996,
      "activations/layer17_attention_weight_max": 33.15808868408203,
      "activations/layer17_attention_weight_min": -26.93671989440918,
      "activations/layer18_attention_weight_max": 35.643524169921875,
      "activations/layer18_attention_weight_min": -24.356121063232422,
      "activations/layer19_attention_weight_max": 37.47897720336914,
      "activations/layer19_attention_weight_min": -31.415008544921875,
      "activations/layer1_attention_weight_max": 16.318532943725586,
      "activations/layer1_attention_weight_min": -15.299433708190918,
      "activations/layer20_attention_weight_max": 33.11151885986328,
      "activations/layer20_attention_weight_min": -26.50761604309082,
      "activations/layer21_attention_weight_max": 31.19676971435547,
      "activations/layer21_attention_weight_min": -24.482574462890625,
      "activations/layer22_attention_weight_max": 49.78730010986328,
      "activations/layer22_attention_weight_min": -32.796382904052734,
      "activations/layer23_attention_weight_max": 41.75668716430664,
      "activations/layer23_attention_weight_min": -24.977691650390625,
      "activations/layer2_attention_weight_max": 34.21007537841797,
      "activations/layer2_attention_weight_min": -33.23088073730469,
      "activations/layer3_attention_weight_max": 98.1788330078125,
      "activations/layer3_attention_weight_min": -98.76395416259766,
      "activations/layer4_attention_weight_max": 114.09224700927734,
      "activations/layer4_attention_weight_min": -113.99261474609375,
      "activations/layer5_attention_weight_max": 52.14634323120117,
      "activations/layer5_attention_weight_min": -58.80891418457031,
      "activations/layer6_attention_weight_max": 44.26628494262695,
      "activations/layer6_attention_weight_min": -46.77379608154297,
      "activations/layer7_attention_weight_max": 86.94513702392578,
      "activations/layer7_attention_weight_min": -93.48981475830078,
      "activations/layer8_attention_weight_max": 42.05464553833008,
      "activations/layer8_attention_weight_min": -44.12202835083008,
      "activations/layer9_attention_weight_max": 32.90482711791992,
      "activations/layer9_attention_weight_min": -34.3160514831543,
      "epoch": 12.11,
      "learning_rate": 7.262348484848485e-05,
      "loss": 2.7643,
      "step": 208450
    },
    {
      "activations/layer0_attention_weight_max": 16.313819885253906,
      "activations/layer0_attention_weight_min": -12.670958518981934,
      "activations/layer10_attention_weight_max": 33.611488342285156,
      "activations/layer10_attention_weight_min": -32.86191177368164,
      "activations/layer11_attention_weight_max": 33.08625793457031,
      "activations/layer11_attention_weight_min": -31.092130661010742,
      "activations/layer12_attention_weight_max": 23.56086540222168,
      "activations/layer12_attention_weight_min": -28.223102569580078,
      "activations/layer13_attention_weight_max": 34.23712921142578,
      "activations/layer13_attention_weight_min": -35.29386901855469,
      "activations/layer14_attention_weight_max": 39.02996063232422,
      "activations/layer14_attention_weight_min": -36.47186279296875,
      "activations/layer15_attention_weight_max": 34.78166580200195,
      "activations/layer15_attention_weight_min": -30.486892700195312,
      "activations/layer16_attention_weight_max": 32.73902893066406,
      "activations/layer16_attention_weight_min": -26.649776458740234,
      "activations/layer17_attention_weight_max": 28.264371871948242,
      "activations/layer17_attention_weight_min": -25.103628158569336,
      "activations/layer18_attention_weight_max": 27.99724006652832,
      "activations/layer18_attention_weight_min": -23.464412689208984,
      "activations/layer19_attention_weight_max": 30.560941696166992,
      "activations/layer19_attention_weight_min": -30.743337631225586,
      "activations/layer1_attention_weight_max": 16.36515235900879,
      "activations/layer1_attention_weight_min": -15.169968605041504,
      "activations/layer20_attention_weight_max": 28.009227752685547,
      "activations/layer20_attention_weight_min": -26.323013305664062,
      "activations/layer21_attention_weight_max": 28.318111419677734,
      "activations/layer21_attention_weight_min": -25.126977920532227,
      "activations/layer22_attention_weight_max": 39.37763977050781,
      "activations/layer22_attention_weight_min": -29.25565528869629,
      "activations/layer23_attention_weight_max": 28.159284591674805,
      "activations/layer23_attention_weight_min": -24.9627742767334,
      "activations/layer2_attention_weight_max": 34.86406707763672,
      "activations/layer2_attention_weight_min": -33.330482482910156,
      "activations/layer3_attention_weight_max": 96.9507827758789,
      "activations/layer3_attention_weight_min": -98.72821807861328,
      "activations/layer4_attention_weight_max": 111.0590591430664,
      "activations/layer4_attention_weight_min": -114.68004608154297,
      "activations/layer5_attention_weight_max": 51.01007080078125,
      "activations/layer5_attention_weight_min": -60.39140319824219,
      "activations/layer6_attention_weight_max": 45.70343780517578,
      "activations/layer6_attention_weight_min": -46.0793342590332,
      "activations/layer7_attention_weight_max": 90.28107452392578,
      "activations/layer7_attention_weight_min": -89.36136627197266,
      "activations/layer8_attention_weight_max": 40.32857131958008,
      "activations/layer8_attention_weight_min": -41.02256393432617,
      "activations/layer9_attention_weight_max": 33.752777099609375,
      "activations/layer9_attention_weight_min": -32.87945556640625,
      "epoch": 12.12,
      "learning_rate": 7.260454545454545e-05,
      "loss": 2.7522,
      "step": 208500
    },
    {
      "activations/layer0_attention_weight_max": 16.411968231201172,
      "activations/layer0_attention_weight_min": -13.025209426879883,
      "activations/layer10_attention_weight_max": 39.175537109375,
      "activations/layer10_attention_weight_min": -38.42045211791992,
      "activations/layer11_attention_weight_max": 40.735382080078125,
      "activations/layer11_attention_weight_min": -35.67417907714844,
      "activations/layer12_attention_weight_max": 30.61599349975586,
      "activations/layer12_attention_weight_min": -26.24996566772461,
      "activations/layer13_attention_weight_max": 46.732913970947266,
      "activations/layer13_attention_weight_min": -34.97256088256836,
      "activations/layer14_attention_weight_max": 52.66788864135742,
      "activations/layer14_attention_weight_min": -40.18568420410156,
      "activations/layer15_attention_weight_max": 44.03364944458008,
      "activations/layer15_attention_weight_min": -34.5068244934082,
      "activations/layer16_attention_weight_max": 35.33877182006836,
      "activations/layer16_attention_weight_min": -28.379817962646484,
      "activations/layer17_attention_weight_max": 33.35994338989258,
      "activations/layer17_attention_weight_min": -26.170318603515625,
      "activations/layer18_attention_weight_max": 34.910438537597656,
      "activations/layer18_attention_weight_min": -24.44920539855957,
      "activations/layer19_attention_weight_max": 39.76243209838867,
      "activations/layer19_attention_weight_min": -32.496822357177734,
      "activations/layer1_attention_weight_max": 16.594390869140625,
      "activations/layer1_attention_weight_min": -15.35225772857666,
      "activations/layer20_attention_weight_max": 31.57421112060547,
      "activations/layer20_attention_weight_min": -26.558439254760742,
      "activations/layer21_attention_weight_max": 34.0257453918457,
      "activations/layer21_attention_weight_min": -24.737823486328125,
      "activations/layer22_attention_weight_max": 49.35527801513672,
      "activations/layer22_attention_weight_min": -29.422407150268555,
      "activations/layer23_attention_weight_max": 35.34675216674805,
      "activations/layer23_attention_weight_min": -24.443241119384766,
      "activations/layer2_attention_weight_max": 34.97774124145508,
      "activations/layer2_attention_weight_min": -35.68192672729492,
      "activations/layer3_attention_weight_max": 102.4045181274414,
      "activations/layer3_attention_weight_min": -101.67743682861328,
      "activations/layer4_attention_weight_max": 114.8456039428711,
      "activations/layer4_attention_weight_min": -117.60271453857422,
      "activations/layer5_attention_weight_max": 52.92747116088867,
      "activations/layer5_attention_weight_min": -65.3233642578125,
      "activations/layer6_attention_weight_max": 46.830238342285156,
      "activations/layer6_attention_weight_min": -49.382930755615234,
      "activations/layer7_attention_weight_max": 98.63815307617188,
      "activations/layer7_attention_weight_min": -101.91483306884766,
      "activations/layer8_attention_weight_max": 48.00413513183594,
      "activations/layer8_attention_weight_min": -47.58134460449219,
      "activations/layer9_attention_weight_max": 40.815364837646484,
      "activations/layer9_attention_weight_min": -38.61811065673828,
      "epoch": 12.12,
      "learning_rate": 7.258560606060605e-05,
      "loss": 2.7557,
      "step": 208550
    },
    {
      "activations/layer0_attention_weight_max": 16.47800064086914,
      "activations/layer0_attention_weight_min": -11.801901817321777,
      "activations/layer10_attention_weight_max": 32.18238067626953,
      "activations/layer10_attention_weight_min": -34.103759765625,
      "activations/layer11_attention_weight_max": 35.11070251464844,
      "activations/layer11_attention_weight_min": -34.399864196777344,
      "activations/layer12_attention_weight_max": 24.41205596923828,
      "activations/layer12_attention_weight_min": -25.74242401123047,
      "activations/layer13_attention_weight_max": 38.55245590209961,
      "activations/layer13_attention_weight_min": -34.538204193115234,
      "activations/layer14_attention_weight_max": 42.79997253417969,
      "activations/layer14_attention_weight_min": -36.79291915893555,
      "activations/layer15_attention_weight_max": 37.67616271972656,
      "activations/layer15_attention_weight_min": -31.727991104125977,
      "activations/layer16_attention_weight_max": 32.5051383972168,
      "activations/layer16_attention_weight_min": -27.269556045532227,
      "activations/layer17_attention_weight_max": 35.57832717895508,
      "activations/layer17_attention_weight_min": -26.792627334594727,
      "activations/layer18_attention_weight_max": 35.303680419921875,
      "activations/layer18_attention_weight_min": -24.375873565673828,
      "activations/layer19_attention_weight_max": 39.29094696044922,
      "activations/layer19_attention_weight_min": -30.953027725219727,
      "activations/layer1_attention_weight_max": 16.86764144897461,
      "activations/layer1_attention_weight_min": -16.808324813842773,
      "activations/layer20_attention_weight_max": 33.09269714355469,
      "activations/layer20_attention_weight_min": -25.890775680541992,
      "activations/layer21_attention_weight_max": 33.46342468261719,
      "activations/layer21_attention_weight_min": -24.80232810974121,
      "activations/layer22_attention_weight_max": 47.68050003051758,
      "activations/layer22_attention_weight_min": -30.421161651611328,
      "activations/layer23_attention_weight_max": 39.59286117553711,
      "activations/layer23_attention_weight_min": -26.066238403320312,
      "activations/layer2_attention_weight_max": 34.741485595703125,
      "activations/layer2_attention_weight_min": -35.43065643310547,
      "activations/layer3_attention_weight_max": 105.54962921142578,
      "activations/layer3_attention_weight_min": -109.18115997314453,
      "activations/layer4_attention_weight_max": 122.20841217041016,
      "activations/layer4_attention_weight_min": -124.09748840332031,
      "activations/layer5_attention_weight_max": 56.56589889526367,
      "activations/layer5_attention_weight_min": -64.90211486816406,
      "activations/layer6_attention_weight_max": 46.98680877685547,
      "activations/layer6_attention_weight_min": -51.78633117675781,
      "activations/layer7_attention_weight_max": 90.87541961669922,
      "activations/layer7_attention_weight_min": -92.16828918457031,
      "activations/layer8_attention_weight_max": 41.34474182128906,
      "activations/layer8_attention_weight_min": -43.73878479003906,
      "activations/layer9_attention_weight_max": 34.210350036621094,
      "activations/layer9_attention_weight_min": -33.80813217163086,
      "epoch": 12.12,
      "learning_rate": 7.256666666666665e-05,
      "loss": 2.7573,
      "step": 208600
    },
    {
      "activations/layer0_attention_weight_max": 15.535002708435059,
      "activations/layer0_attention_weight_min": -11.373345375061035,
      "activations/layer10_attention_weight_max": 33.36090850830078,
      "activations/layer10_attention_weight_min": -37.0299072265625,
      "activations/layer11_attention_weight_max": 33.442138671875,
      "activations/layer11_attention_weight_min": -33.60116195678711,
      "activations/layer12_attention_weight_max": 28.85432243347168,
      "activations/layer12_attention_weight_min": -28.32719612121582,
      "activations/layer13_attention_weight_max": 42.425716400146484,
      "activations/layer13_attention_weight_min": -35.673282623291016,
      "activations/layer14_attention_weight_max": 49.615638732910156,
      "activations/layer14_attention_weight_min": -37.30708312988281,
      "activations/layer15_attention_weight_max": 39.140403747558594,
      "activations/layer15_attention_weight_min": -36.92716598510742,
      "activations/layer16_attention_weight_max": 35.0611572265625,
      "activations/layer16_attention_weight_min": -28.648664474487305,
      "activations/layer17_attention_weight_max": 40.227970123291016,
      "activations/layer17_attention_weight_min": -27.480712890625,
      "activations/layer18_attention_weight_max": 30.925704956054688,
      "activations/layer18_attention_weight_min": -24.72636604309082,
      "activations/layer19_attention_weight_max": 34.67335891723633,
      "activations/layer19_attention_weight_min": -30.38848876953125,
      "activations/layer1_attention_weight_max": 16.152631759643555,
      "activations/layer1_attention_weight_min": -14.953181266784668,
      "activations/layer20_attention_weight_max": 32.78373336791992,
      "activations/layer20_attention_weight_min": -25.216768264770508,
      "activations/layer21_attention_weight_max": 31.14886474609375,
      "activations/layer21_attention_weight_min": -24.363712310791016,
      "activations/layer22_attention_weight_max": 47.96417999267578,
      "activations/layer22_attention_weight_min": -31.879718780517578,
      "activations/layer23_attention_weight_max": 33.94798278808594,
      "activations/layer23_attention_weight_min": -28.314617156982422,
      "activations/layer2_attention_weight_max": 33.021942138671875,
      "activations/layer2_attention_weight_min": -34.208091735839844,
      "activations/layer3_attention_weight_max": 99.11447143554688,
      "activations/layer3_attention_weight_min": -98.55150604248047,
      "activations/layer4_attention_weight_max": 118.44451141357422,
      "activations/layer4_attention_weight_min": -116.4066390991211,
      "activations/layer5_attention_weight_max": 52.482177734375,
      "activations/layer5_attention_weight_min": -62.78181457519531,
      "activations/layer6_attention_weight_max": 44.69666290283203,
      "activations/layer6_attention_weight_min": -48.414066314697266,
      "activations/layer7_attention_weight_max": 86.68425750732422,
      "activations/layer7_attention_weight_min": -98.97207641601562,
      "activations/layer8_attention_weight_max": 40.04233169555664,
      "activations/layer8_attention_weight_min": -45.625736236572266,
      "activations/layer9_attention_weight_max": 33.135684967041016,
      "activations/layer9_attention_weight_min": -35.71369552612305,
      "epoch": 12.12,
      "learning_rate": 7.254772727272727e-05,
      "loss": 2.7446,
      "step": 208650
    },
    {
      "activations/layer0_attention_weight_max": 16.58640480041504,
      "activations/layer0_attention_weight_min": -12.617304801940918,
      "activations/layer10_attention_weight_max": 35.413177490234375,
      "activations/layer10_attention_weight_min": -33.33841323852539,
      "activations/layer11_attention_weight_max": 34.93610382080078,
      "activations/layer11_attention_weight_min": -32.8025016784668,
      "activations/layer12_attention_weight_max": 26.76384925842285,
      "activations/layer12_attention_weight_min": -25.48316764831543,
      "activations/layer13_attention_weight_max": 38.005958557128906,
      "activations/layer13_attention_weight_min": -35.79304504394531,
      "activations/layer14_attention_weight_max": 38.5468635559082,
      "activations/layer14_attention_weight_min": -36.314476013183594,
      "activations/layer15_attention_weight_max": 36.616241455078125,
      "activations/layer15_attention_weight_min": -33.28813552856445,
      "activations/layer16_attention_weight_max": 29.961091995239258,
      "activations/layer16_attention_weight_min": -27.704301834106445,
      "activations/layer17_attention_weight_max": 33.72270965576172,
      "activations/layer17_attention_weight_min": -28.360788345336914,
      "activations/layer18_attention_weight_max": 29.80031967163086,
      "activations/layer18_attention_weight_min": -21.735755920410156,
      "activations/layer19_attention_weight_max": 35.06745529174805,
      "activations/layer19_attention_weight_min": -30.193872451782227,
      "activations/layer1_attention_weight_max": 16.268770217895508,
      "activations/layer1_attention_weight_min": -14.005426406860352,
      "activations/layer20_attention_weight_max": 29.506336212158203,
      "activations/layer20_attention_weight_min": -24.722545623779297,
      "activations/layer21_attention_weight_max": 28.765254974365234,
      "activations/layer21_attention_weight_min": -24.731664657592773,
      "activations/layer22_attention_weight_max": 39.168338775634766,
      "activations/layer22_attention_weight_min": -28.75884246826172,
      "activations/layer23_attention_weight_max": 32.02146911621094,
      "activations/layer23_attention_weight_min": -23.160667419433594,
      "activations/layer2_attention_weight_max": 33.85161590576172,
      "activations/layer2_attention_weight_min": -33.61290740966797,
      "activations/layer3_attention_weight_max": 95.67023468017578,
      "activations/layer3_attention_weight_min": -100.90608978271484,
      "activations/layer4_attention_weight_max": 111.0650405883789,
      "activations/layer4_attention_weight_min": -116.3284912109375,
      "activations/layer5_attention_weight_max": 51.47156524658203,
      "activations/layer5_attention_weight_min": -59.47340393066406,
      "activations/layer6_attention_weight_max": 41.68917465209961,
      "activations/layer6_attention_weight_min": -44.87698745727539,
      "activations/layer7_attention_weight_max": 99.43635559082031,
      "activations/layer7_attention_weight_min": -84.12604522705078,
      "activations/layer8_attention_weight_max": 42.769649505615234,
      "activations/layer8_attention_weight_min": -43.256980895996094,
      "activations/layer9_attention_weight_max": 32.73006820678711,
      "activations/layer9_attention_weight_min": -33.48197555541992,
      "epoch": 12.13,
      "learning_rate": 7.252878787878787e-05,
      "loss": 2.755,
      "step": 208700
    },
    {
      "activations/layer0_attention_weight_max": 15.951642990112305,
      "activations/layer0_attention_weight_min": -11.729711532592773,
      "activations/layer10_attention_weight_max": 37.416229248046875,
      "activations/layer10_attention_weight_min": -35.96287536621094,
      "activations/layer11_attention_weight_max": 37.3715705871582,
      "activations/layer11_attention_weight_min": -34.677696228027344,
      "activations/layer12_attention_weight_max": 26.009645462036133,
      "activations/layer12_attention_weight_min": -24.699329376220703,
      "activations/layer13_attention_weight_max": 41.07253646850586,
      "activations/layer13_attention_weight_min": -35.75609588623047,
      "activations/layer14_attention_weight_max": 44.575462341308594,
      "activations/layer14_attention_weight_min": -38.80269241333008,
      "activations/layer15_attention_weight_max": 41.57959747314453,
      "activations/layer15_attention_weight_min": -32.39113235473633,
      "activations/layer16_attention_weight_max": 32.50172805786133,
      "activations/layer16_attention_weight_min": -26.657257080078125,
      "activations/layer17_attention_weight_max": 32.55764389038086,
      "activations/layer17_attention_weight_min": -26.541688919067383,
      "activations/layer18_attention_weight_max": 31.673683166503906,
      "activations/layer18_attention_weight_min": -23.360776901245117,
      "activations/layer19_attention_weight_max": 40.59931564331055,
      "activations/layer19_attention_weight_min": -32.09366989135742,
      "activations/layer1_attention_weight_max": 17.56785011291504,
      "activations/layer1_attention_weight_min": -14.092039108276367,
      "activations/layer20_attention_weight_max": 35.48760986328125,
      "activations/layer20_attention_weight_min": -25.446693420410156,
      "activations/layer21_attention_weight_max": 35.74922561645508,
      "activations/layer21_attention_weight_min": -24.389263153076172,
      "activations/layer22_attention_weight_max": 49.59915542602539,
      "activations/layer22_attention_weight_min": -32.13859939575195,
      "activations/layer23_attention_weight_max": 42.12577819824219,
      "activations/layer23_attention_weight_min": -25.727582931518555,
      "activations/layer2_attention_weight_max": 35.03706359863281,
      "activations/layer2_attention_weight_min": -33.168617248535156,
      "activations/layer3_attention_weight_max": 97.0425796508789,
      "activations/layer3_attention_weight_min": -100.84842681884766,
      "activations/layer4_attention_weight_max": 115.0362319946289,
      "activations/layer4_attention_weight_min": -116.34059143066406,
      "activations/layer5_attention_weight_max": 51.44590759277344,
      "activations/layer5_attention_weight_min": -65.5380859375,
      "activations/layer6_attention_weight_max": 44.04399490356445,
      "activations/layer6_attention_weight_min": -46.89073181152344,
      "activations/layer7_attention_weight_max": 93.89063262939453,
      "activations/layer7_attention_weight_min": -90.60977935791016,
      "activations/layer8_attention_weight_max": 44.239044189453125,
      "activations/layer8_attention_weight_min": -41.01547622680664,
      "activations/layer9_attention_weight_max": 35.450557708740234,
      "activations/layer9_attention_weight_min": -35.43571090698242,
      "epoch": 12.13,
      "learning_rate": 7.250984848484848e-05,
      "loss": 2.7546,
      "step": 208750
    },
    {
      "activations/layer0_attention_weight_max": 15.80012035369873,
      "activations/layer0_attention_weight_min": -12.672255516052246,
      "activations/layer10_attention_weight_max": 32.55638885498047,
      "activations/layer10_attention_weight_min": -31.14903450012207,
      "activations/layer11_attention_weight_max": 35.88618469238281,
      "activations/layer11_attention_weight_min": -31.800750732421875,
      "activations/layer12_attention_weight_max": 23.322668075561523,
      "activations/layer12_attention_weight_min": -27.334314346313477,
      "activations/layer13_attention_weight_max": 38.69489288330078,
      "activations/layer13_attention_weight_min": -34.61308288574219,
      "activations/layer14_attention_weight_max": 43.79255676269531,
      "activations/layer14_attention_weight_min": -37.31375503540039,
      "activations/layer15_attention_weight_max": 35.681739807128906,
      "activations/layer15_attention_weight_min": -33.86417007446289,
      "activations/layer16_attention_weight_max": 30.178789138793945,
      "activations/layer16_attention_weight_min": -27.745590209960938,
      "activations/layer17_attention_weight_max": 31.2117919921875,
      "activations/layer17_attention_weight_min": -26.27124786376953,
      "activations/layer18_attention_weight_max": 30.48187828063965,
      "activations/layer18_attention_weight_min": -22.104066848754883,
      "activations/layer19_attention_weight_max": 33.53483200073242,
      "activations/layer19_attention_weight_min": -32.251434326171875,
      "activations/layer1_attention_weight_max": 17.29800796508789,
      "activations/layer1_attention_weight_min": -14.34434986114502,
      "activations/layer20_attention_weight_max": 31.57884979248047,
      "activations/layer20_attention_weight_min": -24.926877975463867,
      "activations/layer21_attention_weight_max": 30.08261489868164,
      "activations/layer21_attention_weight_min": -23.386783599853516,
      "activations/layer22_attention_weight_max": 43.457176208496094,
      "activations/layer22_attention_weight_min": -29.57063102722168,
      "activations/layer23_attention_weight_max": 35.1303596496582,
      "activations/layer23_attention_weight_min": -24.64059829711914,
      "activations/layer2_attention_weight_max": 36.995235443115234,
      "activations/layer2_attention_weight_min": -32.8931884765625,
      "activations/layer3_attention_weight_max": 102.72750854492188,
      "activations/layer3_attention_weight_min": -103.25273895263672,
      "activations/layer4_attention_weight_max": 112.5213623046875,
      "activations/layer4_attention_weight_min": -118.19927978515625,
      "activations/layer5_attention_weight_max": 53.08300018310547,
      "activations/layer5_attention_weight_min": -61.642417907714844,
      "activations/layer6_attention_weight_max": 44.71070098876953,
      "activations/layer6_attention_weight_min": -46.56273651123047,
      "activations/layer7_attention_weight_max": 88.06720733642578,
      "activations/layer7_attention_weight_min": -85.14595794677734,
      "activations/layer8_attention_weight_max": 39.25367736816406,
      "activations/layer8_attention_weight_min": -39.927860260009766,
      "activations/layer9_attention_weight_max": 31.76166343688965,
      "activations/layer9_attention_weight_min": -32.721744537353516,
      "epoch": 12.13,
      "learning_rate": 7.249090909090908e-05,
      "loss": 2.755,
      "step": 208800
    },
    {
      "activations/layer0_attention_weight_max": 16.0670108795166,
      "activations/layer0_attention_weight_min": -12.732500076293945,
      "activations/layer10_attention_weight_max": 30.504955291748047,
      "activations/layer10_attention_weight_min": -31.1729793548584,
      "activations/layer11_attention_weight_max": 31.021854400634766,
      "activations/layer11_attention_weight_min": -30.22980499267578,
      "activations/layer12_attention_weight_max": 24.275632858276367,
      "activations/layer12_attention_weight_min": -26.666324615478516,
      "activations/layer13_attention_weight_max": 37.17873764038086,
      "activations/layer13_attention_weight_min": -34.14445495605469,
      "activations/layer14_attention_weight_max": 41.229759216308594,
      "activations/layer14_attention_weight_min": -35.41738510131836,
      "activations/layer15_attention_weight_max": 38.5035514831543,
      "activations/layer15_attention_weight_min": -30.86026382446289,
      "activations/layer16_attention_weight_max": 31.66393280029297,
      "activations/layer16_attention_weight_min": -26.58005142211914,
      "activations/layer17_attention_weight_max": 30.585304260253906,
      "activations/layer17_attention_weight_min": -24.860746383666992,
      "activations/layer18_attention_weight_max": 31.2415828704834,
      "activations/layer18_attention_weight_min": -25.359106063842773,
      "activations/layer19_attention_weight_max": 32.80559158325195,
      "activations/layer19_attention_weight_min": -33.49853515625,
      "activations/layer1_attention_weight_max": 16.704561233520508,
      "activations/layer1_attention_weight_min": -14.338908195495605,
      "activations/layer20_attention_weight_max": 29.328540802001953,
      "activations/layer20_attention_weight_min": -26.241670608520508,
      "activations/layer21_attention_weight_max": 27.600820541381836,
      "activations/layer21_attention_weight_min": -25.109922409057617,
      "activations/layer22_attention_weight_max": 41.10666275024414,
      "activations/layer22_attention_weight_min": -32.14517593383789,
      "activations/layer23_attention_weight_max": 33.721920013427734,
      "activations/layer23_attention_weight_min": -25.48965072631836,
      "activations/layer2_attention_weight_max": 32.244384765625,
      "activations/layer2_attention_weight_min": -32.341346740722656,
      "activations/layer3_attention_weight_max": 95.35186767578125,
      "activations/layer3_attention_weight_min": -96.34583282470703,
      "activations/layer4_attention_weight_max": 110.11103820800781,
      "activations/layer4_attention_weight_min": -115.55294036865234,
      "activations/layer5_attention_weight_max": 51.16462707519531,
      "activations/layer5_attention_weight_min": -59.58271408081055,
      "activations/layer6_attention_weight_max": 46.20083236694336,
      "activations/layer6_attention_weight_min": -48.190223693847656,
      "activations/layer7_attention_weight_max": 89.95773315429688,
      "activations/layer7_attention_weight_min": -96.45854187011719,
      "activations/layer8_attention_weight_max": 40.04315185546875,
      "activations/layer8_attention_weight_min": -42.0792350769043,
      "activations/layer9_attention_weight_max": 31.574298858642578,
      "activations/layer9_attention_weight_min": -31.774490356445312,
      "epoch": 12.14,
      "learning_rate": 7.24719696969697e-05,
      "loss": 2.7499,
      "step": 208850
    },
    {
      "activations/layer0_attention_weight_max": 17.026805877685547,
      "activations/layer0_attention_weight_min": -12.043246269226074,
      "activations/layer10_attention_weight_max": 32.278602600097656,
      "activations/layer10_attention_weight_min": -32.51886749267578,
      "activations/layer11_attention_weight_max": 32.15487289428711,
      "activations/layer11_attention_weight_min": -34.2978515625,
      "activations/layer12_attention_weight_max": 26.320154190063477,
      "activations/layer12_attention_weight_min": -27.663414001464844,
      "activations/layer13_attention_weight_max": 36.87831497192383,
      "activations/layer13_attention_weight_min": -33.659549713134766,
      "activations/layer14_attention_weight_max": 39.93485641479492,
      "activations/layer14_attention_weight_min": -35.34177017211914,
      "activations/layer15_attention_weight_max": 36.671913146972656,
      "activations/layer15_attention_weight_min": -32.48867416381836,
      "activations/layer16_attention_weight_max": 30.216312408447266,
      "activations/layer16_attention_weight_min": -28.202865600585938,
      "activations/layer17_attention_weight_max": 33.1170654296875,
      "activations/layer17_attention_weight_min": -28.86993408203125,
      "activations/layer18_attention_weight_max": 33.82061767578125,
      "activations/layer18_attention_weight_min": -24.44279670715332,
      "activations/layer19_attention_weight_max": 35.817630767822266,
      "activations/layer19_attention_weight_min": -32.24062728881836,
      "activations/layer1_attention_weight_max": 16.026525497436523,
      "activations/layer1_attention_weight_min": -14.39548110961914,
      "activations/layer20_attention_weight_max": 27.502849578857422,
      "activations/layer20_attention_weight_min": -26.782878875732422,
      "activations/layer21_attention_weight_max": 27.947795867919922,
      "activations/layer21_attention_weight_min": -26.012998580932617,
      "activations/layer22_attention_weight_max": 42.84036636352539,
      "activations/layer22_attention_weight_min": -31.09206199645996,
      "activations/layer23_attention_weight_max": 35.480003356933594,
      "activations/layer23_attention_weight_min": -24.767349243164062,
      "activations/layer2_attention_weight_max": 34.29052734375,
      "activations/layer2_attention_weight_min": -32.287906646728516,
      "activations/layer3_attention_weight_max": 96.38612365722656,
      "activations/layer3_attention_weight_min": -97.31901550292969,
      "activations/layer4_attention_weight_max": 110.15553283691406,
      "activations/layer4_attention_weight_min": -112.01971435546875,
      "activations/layer5_attention_weight_max": 49.00689697265625,
      "activations/layer5_attention_weight_min": -62.13129806518555,
      "activations/layer6_attention_weight_max": 43.794761657714844,
      "activations/layer6_attention_weight_min": -47.665340423583984,
      "activations/layer7_attention_weight_max": 90.13253784179688,
      "activations/layer7_attention_weight_min": -88.02189636230469,
      "activations/layer8_attention_weight_max": 39.9627571105957,
      "activations/layer8_attention_weight_min": -42.749656677246094,
      "activations/layer9_attention_weight_max": 34.99453353881836,
      "activations/layer9_attention_weight_min": -34.785465240478516,
      "epoch": 12.14,
      "learning_rate": 7.24530303030303e-05,
      "loss": 2.7767,
      "step": 208900
    },
    {
      "activations/layer0_attention_weight_max": 15.898694038391113,
      "activations/layer0_attention_weight_min": -11.65138053894043,
      "activations/layer10_attention_weight_max": 31.390178680419922,
      "activations/layer10_attention_weight_min": -32.61687088012695,
      "activations/layer11_attention_weight_max": 32.453067779541016,
      "activations/layer11_attention_weight_min": -31.95221519470215,
      "activations/layer12_attention_weight_max": 24.40228271484375,
      "activations/layer12_attention_weight_min": -28.17154884338379,
      "activations/layer13_attention_weight_max": 41.97857666015625,
      "activations/layer13_attention_weight_min": -34.254737854003906,
      "activations/layer14_attention_weight_max": 49.058502197265625,
      "activations/layer14_attention_weight_min": -38.10976791381836,
      "activations/layer15_attention_weight_max": 37.38383483886719,
      "activations/layer15_attention_weight_min": -32.93388366699219,
      "activations/layer16_attention_weight_max": 29.09149932861328,
      "activations/layer16_attention_weight_min": -26.09321403503418,
      "activations/layer17_attention_weight_max": 30.157772064208984,
      "activations/layer17_attention_weight_min": -23.74397087097168,
      "activations/layer18_attention_weight_max": 28.920310974121094,
      "activations/layer18_attention_weight_min": -22.128713607788086,
      "activations/layer19_attention_weight_max": 33.02431869506836,
      "activations/layer19_attention_weight_min": -32.39195251464844,
      "activations/layer1_attention_weight_max": 17.049339294433594,
      "activations/layer1_attention_weight_min": -15.815582275390625,
      "activations/layer20_attention_weight_max": 29.742687225341797,
      "activations/layer20_attention_weight_min": -23.084476470947266,
      "activations/layer21_attention_weight_max": 27.52035140991211,
      "activations/layer21_attention_weight_min": -23.420629501342773,
      "activations/layer22_attention_weight_max": 38.97142028808594,
      "activations/layer22_attention_weight_min": -27.767074584960938,
      "activations/layer23_attention_weight_max": 34.50005340576172,
      "activations/layer23_attention_weight_min": -23.493392944335938,
      "activations/layer2_attention_weight_max": 32.68074417114258,
      "activations/layer2_attention_weight_min": -32.93887710571289,
      "activations/layer3_attention_weight_max": 91.91671752929688,
      "activations/layer3_attention_weight_min": -93.88816833496094,
      "activations/layer4_attention_weight_max": 109.32099914550781,
      "activations/layer4_attention_weight_min": -113.42363739013672,
      "activations/layer5_attention_weight_max": 49.086883544921875,
      "activations/layer5_attention_weight_min": -58.823692321777344,
      "activations/layer6_attention_weight_max": 43.36912155151367,
      "activations/layer6_attention_weight_min": -47.58635330200195,
      "activations/layer7_attention_weight_max": 91.39179229736328,
      "activations/layer7_attention_weight_min": -92.36003112792969,
      "activations/layer8_attention_weight_max": 38.25004196166992,
      "activations/layer8_attention_weight_min": -40.62837600708008,
      "activations/layer9_attention_weight_max": 32.729427337646484,
      "activations/layer9_attention_weight_min": -33.22298049926758,
      "epoch": 12.14,
      "learning_rate": 7.24340909090909e-05,
      "loss": 2.7696,
      "step": 208950
    },
    {
      "activations/layer0_attention_weight_max": 16.438684463500977,
      "activations/layer0_attention_weight_min": -12.171460151672363,
      "activations/layer10_attention_weight_max": 31.77651023864746,
      "activations/layer10_attention_weight_min": -31.622209548950195,
      "activations/layer11_attention_weight_max": 34.06196212768555,
      "activations/layer11_attention_weight_min": -31.312026977539062,
      "activations/layer12_attention_weight_max": 24.74717903137207,
      "activations/layer12_attention_weight_min": -26.136768341064453,
      "activations/layer13_attention_weight_max": 36.85292053222656,
      "activations/layer13_attention_weight_min": -33.95587921142578,
      "activations/layer14_attention_weight_max": 44.200439453125,
      "activations/layer14_attention_weight_min": -37.39516067504883,
      "activations/layer15_attention_weight_max": 35.48036193847656,
      "activations/layer15_attention_weight_min": -32.13446807861328,
      "activations/layer16_attention_weight_max": 28.149293899536133,
      "activations/layer16_attention_weight_min": -30.0831356048584,
      "activations/layer17_attention_weight_max": 28.940568923950195,
      "activations/layer17_attention_weight_min": -24.06156349182129,
      "activations/layer18_attention_weight_max": 30.27818489074707,
      "activations/layer18_attention_weight_min": -21.5762882232666,
      "activations/layer19_attention_weight_max": 32.862098693847656,
      "activations/layer19_attention_weight_min": -30.132566452026367,
      "activations/layer1_attention_weight_max": 15.786391258239746,
      "activations/layer1_attention_weight_min": -13.380736351013184,
      "activations/layer20_attention_weight_max": 28.683420181274414,
      "activations/layer20_attention_weight_min": -24.101917266845703,
      "activations/layer21_attention_weight_max": 27.828704833984375,
      "activations/layer21_attention_weight_min": -23.915634155273438,
      "activations/layer22_attention_weight_max": 41.353904724121094,
      "activations/layer22_attention_weight_min": -28.20368766784668,
      "activations/layer23_attention_weight_max": 33.06890869140625,
      "activations/layer23_attention_weight_min": -24.511625289916992,
      "activations/layer2_attention_weight_max": 35.477195739746094,
      "activations/layer2_attention_weight_min": -35.32756042480469,
      "activations/layer3_attention_weight_max": 100.18346405029297,
      "activations/layer3_attention_weight_min": -98.77677917480469,
      "activations/layer4_attention_weight_max": 111.24263000488281,
      "activations/layer4_attention_weight_min": -112.79228973388672,
      "activations/layer5_attention_weight_max": 52.92301559448242,
      "activations/layer5_attention_weight_min": -63.119903564453125,
      "activations/layer6_attention_weight_max": 43.772464752197266,
      "activations/layer6_attention_weight_min": -47.30811309814453,
      "activations/layer7_attention_weight_max": 95.34878540039062,
      "activations/layer7_attention_weight_min": -90.22726440429688,
      "activations/layer8_attention_weight_max": 40.50733184814453,
      "activations/layer8_attention_weight_min": -41.09553146362305,
      "activations/layer9_attention_weight_max": 32.70124053955078,
      "activations/layer9_attention_weight_min": -31.287744522094727,
      "epoch": 12.14,
      "learning_rate": 7.241515151515152e-05,
      "loss": 2.7668,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_loss": 2.71875,
      "eval_runtime": 8.5521,
      "eval_samples_per_second": 502.097,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.5521,
      "eval_openwebtext_samples_per_second": 502.097,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9181,
      "eval_wikitext_samples_per_second": 237.738,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_lambada_loss": 2.423828125,
      "eval_lambada_ppl": 11.288992376773315,
      "eval_lambada_runtime": 9.5188,
      "eval_lambada_samples_per_second": 511.512,
      "step": 209000
    },
    {
      "activations/layer0_attention_weight_max": 16.101348876953125,
      "activations/layer0_attention_weight_min": -11.894805908203125,
      "activations/layer10_attention_weight_max": 31.5181941986084,
      "activations/layer10_attention_weight_min": -31.690902709960938,
      "activations/layer11_attention_weight_max": 29.788143157958984,
      "activations/layer11_attention_weight_min": -31.264196395874023,
      "activations/layer12_attention_weight_max": 23.070987701416016,
      "activations/layer12_attention_weight_min": -24.73611068725586,
      "activations/layer13_attention_weight_max": 35.16606903076172,
      "activations/layer13_attention_weight_min": -34.34953308105469,
      "activations/layer14_attention_weight_max": 43.463157653808594,
      "activations/layer14_attention_weight_min": -44.20429229736328,
      "activations/layer15_attention_weight_max": 32.146583557128906,
      "activations/layer15_attention_weight_min": -33.940486907958984,
      "activations/layer16_attention_weight_max": 29.35907745361328,
      "activations/layer16_attention_weight_min": -29.036632537841797,
      "activations/layer17_attention_weight_max": 28.728254318237305,
      "activations/layer17_attention_weight_min": -27.128541946411133,
      "activations/layer18_attention_weight_max": 31.609182357788086,
      "activations/layer18_attention_weight_min": -26.10817527770996,
      "activations/layer19_attention_weight_max": 33.35845184326172,
      "activations/layer19_attention_weight_min": -31.578824996948242,
      "activations/layer1_attention_weight_max": 16.447296142578125,
      "activations/layer1_attention_weight_min": -15.06553840637207,
      "activations/layer20_attention_weight_max": 28.468467712402344,
      "activations/layer20_attention_weight_min": -26.21450424194336,
      "activations/layer21_attention_weight_max": 28.095067977905273,
      "activations/layer21_attention_weight_min": -25.532718658447266,
      "activations/layer22_attention_weight_max": 42.5189208984375,
      "activations/layer22_attention_weight_min": -30.727176666259766,
      "activations/layer23_attention_weight_max": 33.0225715637207,
      "activations/layer23_attention_weight_min": -26.94463348388672,
      "activations/layer2_attention_weight_max": 33.310455322265625,
      "activations/layer2_attention_weight_min": -33.20518493652344,
      "activations/layer3_attention_weight_max": 95.95442199707031,
      "activations/layer3_attention_weight_min": -97.45665740966797,
      "activations/layer4_attention_weight_max": 111.34454345703125,
      "activations/layer4_attention_weight_min": -110.48299407958984,
      "activations/layer5_attention_weight_max": 50.722408294677734,
      "activations/layer5_attention_weight_min": -65.79171752929688,
      "activations/layer6_attention_weight_max": 43.41176986694336,
      "activations/layer6_attention_weight_min": -47.56065368652344,
      "activations/layer7_attention_weight_max": 87.34503936767578,
      "activations/layer7_attention_weight_min": -86.6889419555664,
      "activations/layer8_attention_weight_max": 38.900997161865234,
      "activations/layer8_attention_weight_min": -38.865272521972656,
      "activations/layer9_attention_weight_max": 31.87233543395996,
      "activations/layer9_attention_weight_min": -32.266414642333984,
      "epoch": 12.15,
      "learning_rate": 7.239621212121212e-05,
      "loss": 2.7521,
      "step": 209050
    },
    {
      "activations/layer0_attention_weight_max": 16.262775421142578,
      "activations/layer0_attention_weight_min": -11.91046142578125,
      "activations/layer10_attention_weight_max": 32.85332489013672,
      "activations/layer10_attention_weight_min": -34.3179817199707,
      "activations/layer11_attention_weight_max": 31.89992904663086,
      "activations/layer11_attention_weight_min": -32.602752685546875,
      "activations/layer12_attention_weight_max": 24.892004013061523,
      "activations/layer12_attention_weight_min": -28.03953742980957,
      "activations/layer13_attention_weight_max": 37.410255432128906,
      "activations/layer13_attention_weight_min": -34.1020622253418,
      "activations/layer14_attention_weight_max": 44.06568145751953,
      "activations/layer14_attention_weight_min": -37.16382598876953,
      "activations/layer15_attention_weight_max": 37.03450012207031,
      "activations/layer15_attention_weight_min": -32.400489807128906,
      "activations/layer16_attention_weight_max": 31.385820388793945,
      "activations/layer16_attention_weight_min": -28.376556396484375,
      "activations/layer17_attention_weight_max": 30.450124740600586,
      "activations/layer17_attention_weight_min": -26.675912857055664,
      "activations/layer18_attention_weight_max": 29.562414169311523,
      "activations/layer18_attention_weight_min": -22.412513732910156,
      "activations/layer19_attention_weight_max": 35.25947570800781,
      "activations/layer19_attention_weight_min": -31.208541870117188,
      "activations/layer1_attention_weight_max": 16.51425552368164,
      "activations/layer1_attention_weight_min": -13.688009262084961,
      "activations/layer20_attention_weight_max": 29.955322265625,
      "activations/layer20_attention_weight_min": -24.623123168945312,
      "activations/layer21_attention_weight_max": 31.0257568359375,
      "activations/layer21_attention_weight_min": -23.754867553710938,
      "activations/layer22_attention_weight_max": 45.794105529785156,
      "activations/layer22_attention_weight_min": -31.85655403137207,
      "activations/layer23_attention_weight_max": 34.377838134765625,
      "activations/layer23_attention_weight_min": -26.90748405456543,
      "activations/layer2_attention_weight_max": 31.803672790527344,
      "activations/layer2_attention_weight_min": -33.05265808105469,
      "activations/layer3_attention_weight_max": 95.24424743652344,
      "activations/layer3_attention_weight_min": -99.93693542480469,
      "activations/layer4_attention_weight_max": 114.0845947265625,
      "activations/layer4_attention_weight_min": -112.64552307128906,
      "activations/layer5_attention_weight_max": 51.88188934326172,
      "activations/layer5_attention_weight_min": -59.434104919433594,
      "activations/layer6_attention_weight_max": 47.58592224121094,
      "activations/layer6_attention_weight_min": -48.082828521728516,
      "activations/layer7_attention_weight_max": 93.39833068847656,
      "activations/layer7_attention_weight_min": -89.2667465209961,
      "activations/layer8_attention_weight_max": 43.77803421020508,
      "activations/layer8_attention_weight_min": -42.84126281738281,
      "activations/layer9_attention_weight_max": 32.444114685058594,
      "activations/layer9_attention_weight_min": -34.426612854003906,
      "epoch": 12.15,
      "learning_rate": 7.237727272727272e-05,
      "loss": 2.7481,
      "step": 209100
    },
    {
      "activations/layer0_attention_weight_max": 15.783796310424805,
      "activations/layer0_attention_weight_min": -12.529738426208496,
      "activations/layer10_attention_weight_max": 30.74908447265625,
      "activations/layer10_attention_weight_min": -32.395912170410156,
      "activations/layer11_attention_weight_max": 31.45697593688965,
      "activations/layer11_attention_weight_min": -32.30168914794922,
      "activations/layer12_attention_weight_max": 25.319425582885742,
      "activations/layer12_attention_weight_min": -26.215402603149414,
      "activations/layer13_attention_weight_max": 36.420188903808594,
      "activations/layer13_attention_weight_min": -31.715595245361328,
      "activations/layer14_attention_weight_max": 39.67219161987305,
      "activations/layer14_attention_weight_min": -34.23325729370117,
      "activations/layer15_attention_weight_max": 33.48097229003906,
      "activations/layer15_attention_weight_min": -31.149166107177734,
      "activations/layer16_attention_weight_max": 30.523027420043945,
      "activations/layer16_attention_weight_min": -25.63677406311035,
      "activations/layer17_attention_weight_max": 31.15747833251953,
      "activations/layer17_attention_weight_min": -24.659854888916016,
      "activations/layer18_attention_weight_max": 29.767292022705078,
      "activations/layer18_attention_weight_min": -24.31528663635254,
      "activations/layer19_attention_weight_max": 36.26264953613281,
      "activations/layer19_attention_weight_min": -31.391122817993164,
      "activations/layer1_attention_weight_max": 15.359410285949707,
      "activations/layer1_attention_weight_min": -13.376689910888672,
      "activations/layer20_attention_weight_max": 33.32345199584961,
      "activations/layer20_attention_weight_min": -26.631961822509766,
      "activations/layer21_attention_weight_max": 31.554893493652344,
      "activations/layer21_attention_weight_min": -23.726903915405273,
      "activations/layer22_attention_weight_max": 39.564796447753906,
      "activations/layer22_attention_weight_min": -31.551599502563477,
      "activations/layer23_attention_weight_max": 32.660003662109375,
      "activations/layer23_attention_weight_min": -26.380826950073242,
      "activations/layer2_attention_weight_max": 33.76566696166992,
      "activations/layer2_attention_weight_min": -33.97288513183594,
      "activations/layer3_attention_weight_max": 97.40763092041016,
      "activations/layer3_attention_weight_min": -98.57838439941406,
      "activations/layer4_attention_weight_max": 110.55266571044922,
      "activations/layer4_attention_weight_min": -109.2273178100586,
      "activations/layer5_attention_weight_max": 49.52445602416992,
      "activations/layer5_attention_weight_min": -67.00243377685547,
      "activations/layer6_attention_weight_max": 42.50530242919922,
      "activations/layer6_attention_weight_min": -46.13804244995117,
      "activations/layer7_attention_weight_max": 88.63511657714844,
      "activations/layer7_attention_weight_min": -88.0860366821289,
      "activations/layer8_attention_weight_max": 43.057037353515625,
      "activations/layer8_attention_weight_min": -40.097537994384766,
      "activations/layer9_attention_weight_max": 29.897207260131836,
      "activations/layer9_attention_weight_min": -31.627790451049805,
      "epoch": 12.15,
      "learning_rate": 7.235833333333332e-05,
      "loss": 2.758,
      "step": 209150
    },
    {
      "activations/layer0_attention_weight_max": 15.596606254577637,
      "activations/layer0_attention_weight_min": -12.799854278564453,
      "activations/layer10_attention_weight_max": 33.61817169189453,
      "activations/layer10_attention_weight_min": -35.48707962036133,
      "activations/layer11_attention_weight_max": 34.026084899902344,
      "activations/layer11_attention_weight_min": -35.79426574707031,
      "activations/layer12_attention_weight_max": 30.64240837097168,
      "activations/layer12_attention_weight_min": -24.393653869628906,
      "activations/layer13_attention_weight_max": 47.42945098876953,
      "activations/layer13_attention_weight_min": -34.282875061035156,
      "activations/layer14_attention_weight_max": 43.218631744384766,
      "activations/layer14_attention_weight_min": -37.76176452636719,
      "activations/layer15_attention_weight_max": 41.09230422973633,
      "activations/layer15_attention_weight_min": -32.122989654541016,
      "activations/layer16_attention_weight_max": 33.56068420410156,
      "activations/layer16_attention_weight_min": -26.2480525970459,
      "activations/layer17_attention_weight_max": 29.113290786743164,
      "activations/layer17_attention_weight_min": -25.174938201904297,
      "activations/layer18_attention_weight_max": 29.81599235534668,
      "activations/layer18_attention_weight_min": -25.751317977905273,
      "activations/layer19_attention_weight_max": 37.55821990966797,
      "activations/layer19_attention_weight_min": -30.574325561523438,
      "activations/layer1_attention_weight_max": 16.922117233276367,
      "activations/layer1_attention_weight_min": -15.175204277038574,
      "activations/layer20_attention_weight_max": 29.041597366333008,
      "activations/layer20_attention_weight_min": -24.418174743652344,
      "activations/layer21_attention_weight_max": 27.434635162353516,
      "activations/layer21_attention_weight_min": -23.139951705932617,
      "activations/layer22_attention_weight_max": 46.93387222290039,
      "activations/layer22_attention_weight_min": -32.42134094238281,
      "activations/layer23_attention_weight_max": 31.805944442749023,
      "activations/layer23_attention_weight_min": -23.824077606201172,
      "activations/layer2_attention_weight_max": 33.18756866455078,
      "activations/layer2_attention_weight_min": -35.47736358642578,
      "activations/layer3_attention_weight_max": 94.49966430664062,
      "activations/layer3_attention_weight_min": -105.45945739746094,
      "activations/layer4_attention_weight_max": 112.80860900878906,
      "activations/layer4_attention_weight_min": -118.05663299560547,
      "activations/layer5_attention_weight_max": 52.333858489990234,
      "activations/layer5_attention_weight_min": -62.487728118896484,
      "activations/layer6_attention_weight_max": 44.96078872680664,
      "activations/layer6_attention_weight_min": -49.39828872680664,
      "activations/layer7_attention_weight_max": 91.75355529785156,
      "activations/layer7_attention_weight_min": -91.23479461669922,
      "activations/layer8_attention_weight_max": 40.73311233520508,
      "activations/layer8_attention_weight_min": -41.24641036987305,
      "activations/layer9_attention_weight_max": 33.888118743896484,
      "activations/layer9_attention_weight_min": -33.3021240234375,
      "epoch": 12.16,
      "learning_rate": 7.233939393939394e-05,
      "loss": 2.7653,
      "step": 209200
    },
    {
      "activations/layer0_attention_weight_max": 16.315027236938477,
      "activations/layer0_attention_weight_min": -11.329937934875488,
      "activations/layer10_attention_weight_max": 30.99796485900879,
      "activations/layer10_attention_weight_min": -30.34210205078125,
      "activations/layer11_attention_weight_max": 31.35931396484375,
      "activations/layer11_attention_weight_min": -29.291934967041016,
      "activations/layer12_attention_weight_max": 23.505014419555664,
      "activations/layer12_attention_weight_min": -26.153549194335938,
      "activations/layer13_attention_weight_max": 35.60301208496094,
      "activations/layer13_attention_weight_min": -33.95659637451172,
      "activations/layer14_attention_weight_max": 37.319122314453125,
      "activations/layer14_attention_weight_min": -35.656272888183594,
      "activations/layer15_attention_weight_max": 33.2324104309082,
      "activations/layer15_attention_weight_min": -29.526378631591797,
      "activations/layer16_attention_weight_max": 27.856447219848633,
      "activations/layer16_attention_weight_min": -26.74824333190918,
      "activations/layer17_attention_weight_max": 32.53985595703125,
      "activations/layer17_attention_weight_min": -24.3503360748291,
      "activations/layer18_attention_weight_max": 30.825090408325195,
      "activations/layer18_attention_weight_min": -25.573307037353516,
      "activations/layer19_attention_weight_max": 32.959144592285156,
      "activations/layer19_attention_weight_min": -30.665719985961914,
      "activations/layer1_attention_weight_max": 16.21854591369629,
      "activations/layer1_attention_weight_min": -14.5042724609375,
      "activations/layer20_attention_weight_max": 28.6508731842041,
      "activations/layer20_attention_weight_min": -25.464509963989258,
      "activations/layer21_attention_weight_max": 28.062217712402344,
      "activations/layer21_attention_weight_min": -25.41924476623535,
      "activations/layer22_attention_weight_max": 38.42549133300781,
      "activations/layer22_attention_weight_min": -28.721784591674805,
      "activations/layer23_attention_weight_max": 30.25490379333496,
      "activations/layer23_attention_weight_min": -25.342815399169922,
      "activations/layer2_attention_weight_max": 32.32062911987305,
      "activations/layer2_attention_weight_min": -32.61216354370117,
      "activations/layer3_attention_weight_max": 92.76748657226562,
      "activations/layer3_attention_weight_min": -93.35067749023438,
      "activations/layer4_attention_weight_max": 108.52965545654297,
      "activations/layer4_attention_weight_min": -111.33391571044922,
      "activations/layer5_attention_weight_max": 50.941558837890625,
      "activations/layer5_attention_weight_min": -61.064186096191406,
      "activations/layer6_attention_weight_max": 41.957027435302734,
      "activations/layer6_attention_weight_min": -46.51316833496094,
      "activations/layer7_attention_weight_max": 90.62085723876953,
      "activations/layer7_attention_weight_min": -89.72164916992188,
      "activations/layer8_attention_weight_max": 37.913063049316406,
      "activations/layer8_attention_weight_min": -39.22557830810547,
      "activations/layer9_attention_weight_max": 31.517431259155273,
      "activations/layer9_attention_weight_min": -32.220455169677734,
      "epoch": 12.16,
      "learning_rate": 7.232045454545454e-05,
      "loss": 2.7629,
      "step": 209250
    },
    {
      "activations/layer0_attention_weight_max": 15.237825393676758,
      "activations/layer0_attention_weight_min": -11.824479103088379,
      "activations/layer10_attention_weight_max": 34.51124572753906,
      "activations/layer10_attention_weight_min": -31.19997215270996,
      "activations/layer11_attention_weight_max": 31.961807250976562,
      "activations/layer11_attention_weight_min": -31.931554794311523,
      "activations/layer12_attention_weight_max": 26.72874641418457,
      "activations/layer12_attention_weight_min": -27.117115020751953,
      "activations/layer13_attention_weight_max": 42.48005676269531,
      "activations/layer13_attention_weight_min": -33.745540618896484,
      "activations/layer14_attention_weight_max": 46.58591079711914,
      "activations/layer14_attention_weight_min": -36.170860290527344,
      "activations/layer15_attention_weight_max": 36.88257598876953,
      "activations/layer15_attention_weight_min": -32.098697662353516,
      "activations/layer16_attention_weight_max": 31.807449340820312,
      "activations/layer16_attention_weight_min": -25.646137237548828,
      "activations/layer17_attention_weight_max": 33.92196273803711,
      "activations/layer17_attention_weight_min": -26.41720962524414,
      "activations/layer18_attention_weight_max": 32.80937957763672,
      "activations/layer18_attention_weight_min": -25.371553421020508,
      "activations/layer19_attention_weight_max": 33.364479064941406,
      "activations/layer19_attention_weight_min": -30.282400131225586,
      "activations/layer1_attention_weight_max": 16.271331787109375,
      "activations/layer1_attention_weight_min": -13.386762619018555,
      "activations/layer20_attention_weight_max": 29.707639694213867,
      "activations/layer20_attention_weight_min": -26.200450897216797,
      "activations/layer21_attention_weight_max": 28.25759506225586,
      "activations/layer21_attention_weight_min": -26.116313934326172,
      "activations/layer22_attention_weight_max": 48.45573806762695,
      "activations/layer22_attention_weight_min": -33.96719741821289,
      "activations/layer23_attention_weight_max": 38.69548416137695,
      "activations/layer23_attention_weight_min": -27.1556396484375,
      "activations/layer2_attention_weight_max": 31.321640014648438,
      "activations/layer2_attention_weight_min": -32.24386978149414,
      "activations/layer3_attention_weight_max": 94.11538696289062,
      "activations/layer3_attention_weight_min": -94.86705780029297,
      "activations/layer4_attention_weight_max": 111.34938049316406,
      "activations/layer4_attention_weight_min": -108.94770812988281,
      "activations/layer5_attention_weight_max": 51.87103271484375,
      "activations/layer5_attention_weight_min": -59.033077239990234,
      "activations/layer6_attention_weight_max": 44.29328536987305,
      "activations/layer6_attention_weight_min": -44.654930114746094,
      "activations/layer7_attention_weight_max": 86.08008575439453,
      "activations/layer7_attention_weight_min": -83.69564056396484,
      "activations/layer8_attention_weight_max": 39.05217361450195,
      "activations/layer8_attention_weight_min": -39.07791519165039,
      "activations/layer9_attention_weight_max": 32.645320892333984,
      "activations/layer9_attention_weight_min": -32.58045959472656,
      "epoch": 12.16,
      "learning_rate": 7.230151515151514e-05,
      "loss": 2.7589,
      "step": 209300
    },
    {
      "activations/layer0_attention_weight_max": 16.53495216369629,
      "activations/layer0_attention_weight_min": -12.859338760375977,
      "activations/layer10_attention_weight_max": 31.436031341552734,
      "activations/layer10_attention_weight_min": -30.852392196655273,
      "activations/layer11_attention_weight_max": 32.717872619628906,
      "activations/layer11_attention_weight_min": -33.29423522949219,
      "activations/layer12_attention_weight_max": 26.082843780517578,
      "activations/layer12_attention_weight_min": -25.675798416137695,
      "activations/layer13_attention_weight_max": 39.64101028442383,
      "activations/layer13_attention_weight_min": -32.9254264831543,
      "activations/layer14_attention_weight_max": 45.80810546875,
      "activations/layer14_attention_weight_min": -35.99351501464844,
      "activations/layer15_attention_weight_max": 35.7774543762207,
      "activations/layer15_attention_weight_min": -31.510046005249023,
      "activations/layer16_attention_weight_max": 30.313785552978516,
      "activations/layer16_attention_weight_min": -25.999605178833008,
      "activations/layer17_attention_weight_max": 33.3021354675293,
      "activations/layer17_attention_weight_min": -27.551273345947266,
      "activations/layer18_attention_weight_max": 33.44365692138672,
      "activations/layer18_attention_weight_min": -22.992244720458984,
      "activations/layer19_attention_weight_max": 34.0691032409668,
      "activations/layer19_attention_weight_min": -30.882328033447266,
      "activations/layer1_attention_weight_max": 15.576946258544922,
      "activations/layer1_attention_weight_min": -12.819636344909668,
      "activations/layer20_attention_weight_max": 32.91447830200195,
      "activations/layer20_attention_weight_min": -24.810640335083008,
      "activations/layer21_attention_weight_max": 31.3466854095459,
      "activations/layer21_attention_weight_min": -25.786666870117188,
      "activations/layer22_attention_weight_max": 48.13804626464844,
      "activations/layer22_attention_weight_min": -30.295108795166016,
      "activations/layer23_attention_weight_max": 35.23571014404297,
      "activations/layer23_attention_weight_min": -24.3059139251709,
      "activations/layer2_attention_weight_max": 30.20090103149414,
      "activations/layer2_attention_weight_min": -29.107765197753906,
      "activations/layer3_attention_weight_max": 91.69512939453125,
      "activations/layer3_attention_weight_min": -91.36707305908203,
      "activations/layer4_attention_weight_max": 115.43959045410156,
      "activations/layer4_attention_weight_min": -113.29937744140625,
      "activations/layer5_attention_weight_max": 53.7180290222168,
      "activations/layer5_attention_weight_min": -61.791839599609375,
      "activations/layer6_attention_weight_max": 43.82695770263672,
      "activations/layer6_attention_weight_min": -49.648460388183594,
      "activations/layer7_attention_weight_max": 90.83648681640625,
      "activations/layer7_attention_weight_min": -93.07037353515625,
      "activations/layer8_attention_weight_max": 39.41273880004883,
      "activations/layer8_attention_weight_min": -39.92558670043945,
      "activations/layer9_attention_weight_max": 32.578067779541016,
      "activations/layer9_attention_weight_min": -31.996747970581055,
      "epoch": 12.16,
      "learning_rate": 7.228257575757574e-05,
      "loss": 2.76,
      "step": 209350
    },
    {
      "activations/layer0_attention_weight_max": 16.465312957763672,
      "activations/layer0_attention_weight_min": -12.507834434509277,
      "activations/layer10_attention_weight_max": 29.988920211791992,
      "activations/layer10_attention_weight_min": -31.419673919677734,
      "activations/layer11_attention_weight_max": 31.810644149780273,
      "activations/layer11_attention_weight_min": -30.51795196533203,
      "activations/layer12_attention_weight_max": 22.525304794311523,
      "activations/layer12_attention_weight_min": -23.75640869140625,
      "activations/layer13_attention_weight_max": 34.08170700073242,
      "activations/layer13_attention_weight_min": -33.400840759277344,
      "activations/layer14_attention_weight_max": 38.87714767456055,
      "activations/layer14_attention_weight_min": -35.101158142089844,
      "activations/layer15_attention_weight_max": 34.55348205566406,
      "activations/layer15_attention_weight_min": -31.59811782836914,
      "activations/layer16_attention_weight_max": 30.619726181030273,
      "activations/layer16_attention_weight_min": -26.848617553710938,
      "activations/layer17_attention_weight_max": 35.085636138916016,
      "activations/layer17_attention_weight_min": -24.878110885620117,
      "activations/layer18_attention_weight_max": 33.985626220703125,
      "activations/layer18_attention_weight_min": -23.7625675201416,
      "activations/layer19_attention_weight_max": 36.390682220458984,
      "activations/layer19_attention_weight_min": -32.940853118896484,
      "activations/layer1_attention_weight_max": 17.261634826660156,
      "activations/layer1_attention_weight_min": -14.218273162841797,
      "activations/layer20_attention_weight_max": 35.72660827636719,
      "activations/layer20_attention_weight_min": -24.597017288208008,
      "activations/layer21_attention_weight_max": 33.7868537902832,
      "activations/layer21_attention_weight_min": -23.834518432617188,
      "activations/layer22_attention_weight_max": 44.95756912231445,
      "activations/layer22_attention_weight_min": -31.234169006347656,
      "activations/layer23_attention_weight_max": 34.92730712890625,
      "activations/layer23_attention_weight_min": -27.0756893157959,
      "activations/layer2_attention_weight_max": 32.72698974609375,
      "activations/layer2_attention_weight_min": -31.805700302124023,
      "activations/layer3_attention_weight_max": 97.11554718017578,
      "activations/layer3_attention_weight_min": -93.83387756347656,
      "activations/layer4_attention_weight_max": 115.23663330078125,
      "activations/layer4_attention_weight_min": -109.69213104248047,
      "activations/layer5_attention_weight_max": 55.038909912109375,
      "activations/layer5_attention_weight_min": -66.01300048828125,
      "activations/layer6_attention_weight_max": 45.19845962524414,
      "activations/layer6_attention_weight_min": -46.10553741455078,
      "activations/layer7_attention_weight_max": 88.92659759521484,
      "activations/layer7_attention_weight_min": -86.8268814086914,
      "activations/layer8_attention_weight_max": 39.099056243896484,
      "activations/layer8_attention_weight_min": -38.34635543823242,
      "activations/layer9_attention_weight_max": 30.943029403686523,
      "activations/layer9_attention_weight_min": -31.540618896484375,
      "epoch": 12.17,
      "learning_rate": 7.226363636363636e-05,
      "loss": 2.7672,
      "step": 209400
    },
    {
      "activations/layer0_attention_weight_max": 16.16891098022461,
      "activations/layer0_attention_weight_min": -12.68524169921875,
      "activations/layer10_attention_weight_max": 32.36745834350586,
      "activations/layer10_attention_weight_min": -30.53043556213379,
      "activations/layer11_attention_weight_max": 31.590190887451172,
      "activations/layer11_attention_weight_min": -30.68928337097168,
      "activations/layer12_attention_weight_max": 25.360326766967773,
      "activations/layer12_attention_weight_min": -23.92820167541504,
      "activations/layer13_attention_weight_max": 37.38069534301758,
      "activations/layer13_attention_weight_min": -33.060142517089844,
      "activations/layer14_attention_weight_max": 46.9329719543457,
      "activations/layer14_attention_weight_min": -38.183074951171875,
      "activations/layer15_attention_weight_max": 35.66335678100586,
      "activations/layer15_attention_weight_min": -32.960487365722656,
      "activations/layer16_attention_weight_max": 31.96227264404297,
      "activations/layer16_attention_weight_min": -26.60753631591797,
      "activations/layer17_attention_weight_max": 31.28019905090332,
      "activations/layer17_attention_weight_min": -26.988933563232422,
      "activations/layer18_attention_weight_max": 28.578060150146484,
      "activations/layer18_attention_weight_min": -22.499282836914062,
      "activations/layer19_attention_weight_max": 33.823978424072266,
      "activations/layer19_attention_weight_min": -29.112756729125977,
      "activations/layer1_attention_weight_max": 15.722100257873535,
      "activations/layer1_attention_weight_min": -14.109918594360352,
      "activations/layer20_attention_weight_max": 30.779699325561523,
      "activations/layer20_attention_weight_min": -23.954322814941406,
      "activations/layer21_attention_weight_max": 28.876327514648438,
      "activations/layer21_attention_weight_min": -22.54939079284668,
      "activations/layer22_attention_weight_max": 41.85781478881836,
      "activations/layer22_attention_weight_min": -30.98490333557129,
      "activations/layer23_attention_weight_max": 33.80989074707031,
      "activations/layer23_attention_weight_min": -23.717788696289062,
      "activations/layer2_attention_weight_max": 31.904075622558594,
      "activations/layer2_attention_weight_min": -31.856176376342773,
      "activations/layer3_attention_weight_max": 95.38349914550781,
      "activations/layer3_attention_weight_min": -96.44364166259766,
      "activations/layer4_attention_weight_max": 110.19339752197266,
      "activations/layer4_attention_weight_min": -108.39449310302734,
      "activations/layer5_attention_weight_max": 49.10689926147461,
      "activations/layer5_attention_weight_min": -63.22032928466797,
      "activations/layer6_attention_weight_max": 42.24818420410156,
      "activations/layer6_attention_weight_min": -43.01140594482422,
      "activations/layer7_attention_weight_max": 87.6247787475586,
      "activations/layer7_attention_weight_min": -83.7505111694336,
      "activations/layer8_attention_weight_max": 38.3032112121582,
      "activations/layer8_attention_weight_min": -39.76210403442383,
      "activations/layer9_attention_weight_max": 35.46165084838867,
      "activations/layer9_attention_weight_min": -31.49788475036621,
      "epoch": 12.17,
      "learning_rate": 7.224469696969696e-05,
      "loss": 2.7452,
      "step": 209450
    },
    {
      "activations/layer0_attention_weight_max": 15.489989280700684,
      "activations/layer0_attention_weight_min": -12.551453590393066,
      "activations/layer10_attention_weight_max": 34.552978515625,
      "activations/layer10_attention_weight_min": -34.7076416015625,
      "activations/layer11_attention_weight_max": 35.81195831298828,
      "activations/layer11_attention_weight_min": -35.444602966308594,
      "activations/layer12_attention_weight_max": 26.87012481689453,
      "activations/layer12_attention_weight_min": -26.74793243408203,
      "activations/layer13_attention_weight_max": 43.650882720947266,
      "activations/layer13_attention_weight_min": -36.731807708740234,
      "activations/layer14_attention_weight_max": 49.93299865722656,
      "activations/layer14_attention_weight_min": -44.11975860595703,
      "activations/layer15_attention_weight_max": 40.53985595703125,
      "activations/layer15_attention_weight_min": -33.88195037841797,
      "activations/layer16_attention_weight_max": 36.295997619628906,
      "activations/layer16_attention_weight_min": -29.60838508605957,
      "activations/layer17_attention_weight_max": 34.077083587646484,
      "activations/layer17_attention_weight_min": -27.89063835144043,
      "activations/layer18_attention_weight_max": 33.490901947021484,
      "activations/layer18_attention_weight_min": -24.398134231567383,
      "activations/layer19_attention_weight_max": 39.24494552612305,
      "activations/layer19_attention_weight_min": -33.51066589355469,
      "activations/layer1_attention_weight_max": 16.344112396240234,
      "activations/layer1_attention_weight_min": -13.677323341369629,
      "activations/layer20_attention_weight_max": 29.70903968811035,
      "activations/layer20_attention_weight_min": -29.552467346191406,
      "activations/layer21_attention_weight_max": 29.540456771850586,
      "activations/layer21_attention_weight_min": -26.54568099975586,
      "activations/layer22_attention_weight_max": 46.09458923339844,
      "activations/layer22_attention_weight_min": -34.46491622924805,
      "activations/layer23_attention_weight_max": 35.9727897644043,
      "activations/layer23_attention_weight_min": -27.859390258789062,
      "activations/layer2_attention_weight_max": 32.62124252319336,
      "activations/layer2_attention_weight_min": -31.125713348388672,
      "activations/layer3_attention_weight_max": 93.26289367675781,
      "activations/layer3_attention_weight_min": -94.91169738769531,
      "activations/layer4_attention_weight_max": 114.45549774169922,
      "activations/layer4_attention_weight_min": -112.31572723388672,
      "activations/layer5_attention_weight_max": 52.76484298706055,
      "activations/layer5_attention_weight_min": -63.252376556396484,
      "activations/layer6_attention_weight_max": 43.50297546386719,
      "activations/layer6_attention_weight_min": -48.2028694152832,
      "activations/layer7_attention_weight_max": 94.02391052246094,
      "activations/layer7_attention_weight_min": -91.62097930908203,
      "activations/layer8_attention_weight_max": 40.82535934448242,
      "activations/layer8_attention_weight_min": -43.377166748046875,
      "activations/layer9_attention_weight_max": 34.94739532470703,
      "activations/layer9_attention_weight_min": -33.98988723754883,
      "epoch": 12.17,
      "learning_rate": 7.222575757575757e-05,
      "loss": 2.7722,
      "step": 209500
    },
    {
      "activations/layer0_attention_weight_max": 15.513590812683105,
      "activations/layer0_attention_weight_min": -11.90134334564209,
      "activations/layer10_attention_weight_max": 30.345251083374023,
      "activations/layer10_attention_weight_min": -31.13319969177246,
      "activations/layer11_attention_weight_max": 29.612071990966797,
      "activations/layer11_attention_weight_min": -30.946393966674805,
      "activations/layer12_attention_weight_max": 26.94680404663086,
      "activations/layer12_attention_weight_min": -23.930744171142578,
      "activations/layer13_attention_weight_max": 39.405235290527344,
      "activations/layer13_attention_weight_min": -32.2177734375,
      "activations/layer14_attention_weight_max": 45.50349044799805,
      "activations/layer14_attention_weight_min": -33.858009338378906,
      "activations/layer15_attention_weight_max": 39.63352966308594,
      "activations/layer15_attention_weight_min": -31.73496437072754,
      "activations/layer16_attention_weight_max": 32.437652587890625,
      "activations/layer16_attention_weight_min": -28.863000869750977,
      "activations/layer17_attention_weight_max": 34.7855110168457,
      "activations/layer17_attention_weight_min": -26.754301071166992,
      "activations/layer18_attention_weight_max": 36.04269027709961,
      "activations/layer18_attention_weight_min": -25.36415672302246,
      "activations/layer19_attention_weight_max": 38.9024772644043,
      "activations/layer19_attention_weight_min": -34.74565124511719,
      "activations/layer1_attention_weight_max": 16.019567489624023,
      "activations/layer1_attention_weight_min": -14.117210388183594,
      "activations/layer20_attention_weight_max": 32.89949035644531,
      "activations/layer20_attention_weight_min": -25.599842071533203,
      "activations/layer21_attention_weight_max": 30.650707244873047,
      "activations/layer21_attention_weight_min": -25.75579833984375,
      "activations/layer22_attention_weight_max": 43.9035758972168,
      "activations/layer22_attention_weight_min": -31.00519561767578,
      "activations/layer23_attention_weight_max": 34.423561096191406,
      "activations/layer23_attention_weight_min": -25.99828338623047,
      "activations/layer2_attention_weight_max": 34.71916198730469,
      "activations/layer2_attention_weight_min": -32.759246826171875,
      "activations/layer3_attention_weight_max": 91.02013397216797,
      "activations/layer3_attention_weight_min": -97.78349304199219,
      "activations/layer4_attention_weight_max": 107.16043853759766,
      "activations/layer4_attention_weight_min": -109.4725341796875,
      "activations/layer5_attention_weight_max": 51.99176025390625,
      "activations/layer5_attention_weight_min": -61.663936614990234,
      "activations/layer6_attention_weight_max": 45.4041633605957,
      "activations/layer6_attention_weight_min": -46.23099136352539,
      "activations/layer7_attention_weight_max": 85.77825164794922,
      "activations/layer7_attention_weight_min": -85.91981506347656,
      "activations/layer8_attention_weight_max": 38.708595275878906,
      "activations/layer8_attention_weight_min": -39.918861389160156,
      "activations/layer9_attention_weight_max": 30.81129264831543,
      "activations/layer9_attention_weight_min": -33.5297737121582,
      "epoch": 12.18,
      "learning_rate": 7.220681818181817e-05,
      "loss": 2.7689,
      "step": 209550
    },
    {
      "activations/layer0_attention_weight_max": 15.564777374267578,
      "activations/layer0_attention_weight_min": -11.670364379882812,
      "activations/layer10_attention_weight_max": 35.362972259521484,
      "activations/layer10_attention_weight_min": -35.5180778503418,
      "activations/layer11_attention_weight_max": 37.17433166503906,
      "activations/layer11_attention_weight_min": -36.28207015991211,
      "activations/layer12_attention_weight_max": 26.549503326416016,
      "activations/layer12_attention_weight_min": -25.01719093322754,
      "activations/layer13_attention_weight_max": 39.77482986450195,
      "activations/layer13_attention_weight_min": -32.5391960144043,
      "activations/layer14_attention_weight_max": 45.97154998779297,
      "activations/layer14_attention_weight_min": -37.43696594238281,
      "activations/layer15_attention_weight_max": 42.57040786743164,
      "activations/layer15_attention_weight_min": -34.654396057128906,
      "activations/layer16_attention_weight_max": 34.965362548828125,
      "activations/layer16_attention_weight_min": -27.778247833251953,
      "activations/layer17_attention_weight_max": 32.87112808227539,
      "activations/layer17_attention_weight_min": -26.079368591308594,
      "activations/layer18_attention_weight_max": 34.139007568359375,
      "activations/layer18_attention_weight_min": -22.700336456298828,
      "activations/layer19_attention_weight_max": 43.941104888916016,
      "activations/layer19_attention_weight_min": -31.42885398864746,
      "activations/layer1_attention_weight_max": 17.155935287475586,
      "activations/layer1_attention_weight_min": -14.337040901184082,
      "activations/layer20_attention_weight_max": 36.4488639831543,
      "activations/layer20_attention_weight_min": -25.51460838317871,
      "activations/layer21_attention_weight_max": 35.91188049316406,
      "activations/layer21_attention_weight_min": -23.74432373046875,
      "activations/layer22_attention_weight_max": 50.08488845825195,
      "activations/layer22_attention_weight_min": -29.489139556884766,
      "activations/layer23_attention_weight_max": 39.48668670654297,
      "activations/layer23_attention_weight_min": -24.922393798828125,
      "activations/layer2_attention_weight_max": 35.222930908203125,
      "activations/layer2_attention_weight_min": -34.03042984008789,
      "activations/layer3_attention_weight_max": 94.73308563232422,
      "activations/layer3_attention_weight_min": -97.42020416259766,
      "activations/layer4_attention_weight_max": 111.15864562988281,
      "activations/layer4_attention_weight_min": -114.68216705322266,
      "activations/layer5_attention_weight_max": 50.604209899902344,
      "activations/layer5_attention_weight_min": -62.395912170410156,
      "activations/layer6_attention_weight_max": 44.01237487792969,
      "activations/layer6_attention_weight_min": -47.880123138427734,
      "activations/layer7_attention_weight_max": 98.72506713867188,
      "activations/layer7_attention_weight_min": -97.03450775146484,
      "activations/layer8_attention_weight_max": 45.612674713134766,
      "activations/layer8_attention_weight_min": -47.30317306518555,
      "activations/layer9_attention_weight_max": 34.421546936035156,
      "activations/layer9_attention_weight_min": -37.73891830444336,
      "epoch": 12.18,
      "learning_rate": 7.218787878787879e-05,
      "loss": 2.7631,
      "step": 209600
    },
    {
      "activations/layer0_attention_weight_max": 15.740276336669922,
      "activations/layer0_attention_weight_min": -12.198349952697754,
      "activations/layer10_attention_weight_max": 30.97609519958496,
      "activations/layer10_attention_weight_min": -30.801876068115234,
      "activations/layer11_attention_weight_max": 30.407882690429688,
      "activations/layer11_attention_weight_min": -31.00037384033203,
      "activations/layer12_attention_weight_max": 25.545427322387695,
      "activations/layer12_attention_weight_min": -24.023351669311523,
      "activations/layer13_attention_weight_max": 39.129573822021484,
      "activations/layer13_attention_weight_min": -33.98995590209961,
      "activations/layer14_attention_weight_max": 48.26046371459961,
      "activations/layer14_attention_weight_min": -35.18788146972656,
      "activations/layer15_attention_weight_max": 38.66719055175781,
      "activations/layer15_attention_weight_min": -32.13274383544922,
      "activations/layer16_attention_weight_max": 33.605709075927734,
      "activations/layer16_attention_weight_min": -27.63191032409668,
      "activations/layer17_attention_weight_max": 33.43327713012695,
      "activations/layer17_attention_weight_min": -26.097389221191406,
      "activations/layer18_attention_weight_max": 33.935272216796875,
      "activations/layer18_attention_weight_min": -24.669034957885742,
      "activations/layer19_attention_weight_max": 39.450645446777344,
      "activations/layer19_attention_weight_min": -32.505489349365234,
      "activations/layer1_attention_weight_max": 16.474246978759766,
      "activations/layer1_attention_weight_min": -15.511661529541016,
      "activations/layer20_attention_weight_max": 32.958187103271484,
      "activations/layer20_attention_weight_min": -26.14519500732422,
      "activations/layer21_attention_weight_max": 30.543020248413086,
      "activations/layer21_attention_weight_min": -26.76625633239746,
      "activations/layer22_attention_weight_max": 49.80190658569336,
      "activations/layer22_attention_weight_min": -33.63096618652344,
      "activations/layer23_attention_weight_max": 40.68398666381836,
      "activations/layer23_attention_weight_min": -24.05559539794922,
      "activations/layer2_attention_weight_max": 31.99629020690918,
      "activations/layer2_attention_weight_min": -31.771007537841797,
      "activations/layer3_attention_weight_max": 92.17560577392578,
      "activations/layer3_attention_weight_min": -92.61552429199219,
      "activations/layer4_attention_weight_max": 108.29244232177734,
      "activations/layer4_attention_weight_min": -107.92573547363281,
      "activations/layer5_attention_weight_max": 53.11725997924805,
      "activations/layer5_attention_weight_min": -60.27216720581055,
      "activations/layer6_attention_weight_max": 42.23876190185547,
      "activations/layer6_attention_weight_min": -47.479270935058594,
      "activations/layer7_attention_weight_max": 83.98409271240234,
      "activations/layer7_attention_weight_min": -88.88565826416016,
      "activations/layer8_attention_weight_max": 38.33354568481445,
      "activations/layer8_attention_weight_min": -40.449546813964844,
      "activations/layer9_attention_weight_max": 30.345407485961914,
      "activations/layer9_attention_weight_min": -31.928274154663086,
      "epoch": 12.18,
      "learning_rate": 7.216893939393939e-05,
      "loss": 2.7641,
      "step": 209650
    },
    {
      "activations/layer0_attention_weight_max": 15.671462059020996,
      "activations/layer0_attention_weight_min": -12.181662559509277,
      "activations/layer10_attention_weight_max": 31.715576171875,
      "activations/layer10_attention_weight_min": -30.60245132446289,
      "activations/layer11_attention_weight_max": 31.940868377685547,
      "activations/layer11_attention_weight_min": -30.602108001708984,
      "activations/layer12_attention_weight_max": 24.069202423095703,
      "activations/layer12_attention_weight_min": -24.677330017089844,
      "activations/layer13_attention_weight_max": 36.859527587890625,
      "activations/layer13_attention_weight_min": -32.60195541381836,
      "activations/layer14_attention_weight_max": 48.66888427734375,
      "activations/layer14_attention_weight_min": -35.19095993041992,
      "activations/layer15_attention_weight_max": 34.47663497924805,
      "activations/layer15_attention_weight_min": -33.02513122558594,
      "activations/layer16_attention_weight_max": 29.88155174255371,
      "activations/layer16_attention_weight_min": -28.78862190246582,
      "activations/layer17_attention_weight_max": 29.36018180847168,
      "activations/layer17_attention_weight_min": -26.17789649963379,
      "activations/layer18_attention_weight_max": 31.778352737426758,
      "activations/layer18_attention_weight_min": -23.908971786499023,
      "activations/layer19_attention_weight_max": 32.21590042114258,
      "activations/layer19_attention_weight_min": -30.34663963317871,
      "activations/layer1_attention_weight_max": 16.862905502319336,
      "activations/layer1_attention_weight_min": -16.013723373413086,
      "activations/layer20_attention_weight_max": 28.667312622070312,
      "activations/layer20_attention_weight_min": -24.9293155670166,
      "activations/layer21_attention_weight_max": 27.462383270263672,
      "activations/layer21_attention_weight_min": -23.595420837402344,
      "activations/layer22_attention_weight_max": 41.78932189941406,
      "activations/layer22_attention_weight_min": -29.91545867919922,
      "activations/layer23_attention_weight_max": 33.97863006591797,
      "activations/layer23_attention_weight_min": -25.34882164001465,
      "activations/layer2_attention_weight_max": 32.10016632080078,
      "activations/layer2_attention_weight_min": -30.760601043701172,
      "activations/layer3_attention_weight_max": 92.72577667236328,
      "activations/layer3_attention_weight_min": -91.29532623291016,
      "activations/layer4_attention_weight_max": 109.86930084228516,
      "activations/layer4_attention_weight_min": -109.0986099243164,
      "activations/layer5_attention_weight_max": 54.15572738647461,
      "activations/layer5_attention_weight_min": -62.806243896484375,
      "activations/layer6_attention_weight_max": 45.412620544433594,
      "activations/layer6_attention_weight_min": -45.97437286376953,
      "activations/layer7_attention_weight_max": 89.87078857421875,
      "activations/layer7_attention_weight_min": -91.75464630126953,
      "activations/layer8_attention_weight_max": 42.153953552246094,
      "activations/layer8_attention_weight_min": -42.33908462524414,
      "activations/layer9_attention_weight_max": 32.976802825927734,
      "activations/layer9_attention_weight_min": -31.900808334350586,
      "epoch": 12.18,
      "learning_rate": 7.214999999999999e-05,
      "loss": 2.7417,
      "step": 209700
    },
    {
      "activations/layer0_attention_weight_max": 15.425854682922363,
      "activations/layer0_attention_weight_min": -12.949936866760254,
      "activations/layer10_attention_weight_max": 37.416465759277344,
      "activations/layer10_attention_weight_min": -37.91554641723633,
      "activations/layer11_attention_weight_max": 39.169559478759766,
      "activations/layer11_attention_weight_min": -39.37070083618164,
      "activations/layer12_attention_weight_max": 26.60102653503418,
      "activations/layer12_attention_weight_min": -28.37650489807129,
      "activations/layer13_attention_weight_max": 36.49867248535156,
      "activations/layer13_attention_weight_min": -38.043426513671875,
      "activations/layer14_attention_weight_max": 45.221702575683594,
      "activations/layer14_attention_weight_min": -39.19022750854492,
      "activations/layer15_attention_weight_max": 38.062992095947266,
      "activations/layer15_attention_weight_min": -35.79787063598633,
      "activations/layer16_attention_weight_max": 31.888505935668945,
      "activations/layer16_attention_weight_min": -26.873973846435547,
      "activations/layer17_attention_weight_max": 31.377174377441406,
      "activations/layer17_attention_weight_min": -27.397621154785156,
      "activations/layer18_attention_weight_max": 30.48871421813965,
      "activations/layer18_attention_weight_min": -24.073232650756836,
      "activations/layer19_attention_weight_max": 35.609962463378906,
      "activations/layer19_attention_weight_min": -31.504291534423828,
      "activations/layer1_attention_weight_max": 17.54647445678711,
      "activations/layer1_attention_weight_min": -14.249510765075684,
      "activations/layer20_attention_weight_max": 29.655975341796875,
      "activations/layer20_attention_weight_min": -25.74324607849121,
      "activations/layer21_attention_weight_max": 34.22355270385742,
      "activations/layer21_attention_weight_min": -23.834768295288086,
      "activations/layer22_attention_weight_max": 43.47615051269531,
      "activations/layer22_attention_weight_min": -30.90143585205078,
      "activations/layer23_attention_weight_max": 34.691532135009766,
      "activations/layer23_attention_weight_min": -25.459930419921875,
      "activations/layer2_attention_weight_max": 34.5223503112793,
      "activations/layer2_attention_weight_min": -34.947975158691406,
      "activations/layer3_attention_weight_max": 100.72196960449219,
      "activations/layer3_attention_weight_min": -102.18555450439453,
      "activations/layer4_attention_weight_max": 118.33612060546875,
      "activations/layer4_attention_weight_min": -114.39015197753906,
      "activations/layer5_attention_weight_max": 54.69354248046875,
      "activations/layer5_attention_weight_min": -61.15950012207031,
      "activations/layer6_attention_weight_max": 45.2392692565918,
      "activations/layer6_attention_weight_min": -52.210819244384766,
      "activations/layer7_attention_weight_max": 97.46234130859375,
      "activations/layer7_attention_weight_min": -102.6307601928711,
      "activations/layer8_attention_weight_max": 46.30125427246094,
      "activations/layer8_attention_weight_min": -44.85348129272461,
      "activations/layer9_attention_weight_max": 37.551597595214844,
      "activations/layer9_attention_weight_min": -36.233821868896484,
      "epoch": 12.19,
      "learning_rate": 7.213106060606061e-05,
      "loss": 2.7579,
      "step": 209750
    },
    {
      "activations/layer0_attention_weight_max": 16.344955444335938,
      "activations/layer0_attention_weight_min": -12.642630577087402,
      "activations/layer10_attention_weight_max": 34.293697357177734,
      "activations/layer10_attention_weight_min": -34.1013069152832,
      "activations/layer11_attention_weight_max": 34.33570098876953,
      "activations/layer11_attention_weight_min": -32.984413146972656,
      "activations/layer12_attention_weight_max": 26.26003646850586,
      "activations/layer12_attention_weight_min": -25.387449264526367,
      "activations/layer13_attention_weight_max": 40.30699157714844,
      "activations/layer13_attention_weight_min": -35.304141998291016,
      "activations/layer14_attention_weight_max": 44.621726989746094,
      "activations/layer14_attention_weight_min": -40.68791961669922,
      "activations/layer15_attention_weight_max": 37.554927825927734,
      "activations/layer15_attention_weight_min": -34.16141891479492,
      "activations/layer16_attention_weight_max": 32.12516784667969,
      "activations/layer16_attention_weight_min": -28.218650817871094,
      "activations/layer17_attention_weight_max": 30.44204330444336,
      "activations/layer17_attention_weight_min": -27.903438568115234,
      "activations/layer18_attention_weight_max": 34.92116165161133,
      "activations/layer18_attention_weight_min": -23.653522491455078,
      "activations/layer19_attention_weight_max": 33.855899810791016,
      "activations/layer19_attention_weight_min": -31.245580673217773,
      "activations/layer1_attention_weight_max": 16.496034622192383,
      "activations/layer1_attention_weight_min": -13.70004940032959,
      "activations/layer20_attention_weight_max": 29.571372985839844,
      "activations/layer20_attention_weight_min": -26.32969856262207,
      "activations/layer21_attention_weight_max": 28.733123779296875,
      "activations/layer21_attention_weight_min": -24.50638771057129,
      "activations/layer22_attention_weight_max": 44.26263427734375,
      "activations/layer22_attention_weight_min": -29.428749084472656,
      "activations/layer23_attention_weight_max": 36.12763977050781,
      "activations/layer23_attention_weight_min": -27.96190643310547,
      "activations/layer2_attention_weight_max": 33.366973876953125,
      "activations/layer2_attention_weight_min": -33.204429626464844,
      "activations/layer3_attention_weight_max": 95.13572692871094,
      "activations/layer3_attention_weight_min": -96.62169647216797,
      "activations/layer4_attention_weight_max": 111.95568084716797,
      "activations/layer4_attention_weight_min": -113.78956604003906,
      "activations/layer5_attention_weight_max": 52.85559844970703,
      "activations/layer5_attention_weight_min": -60.46324920654297,
      "activations/layer6_attention_weight_max": 45.82599639892578,
      "activations/layer6_attention_weight_min": -51.92145538330078,
      "activations/layer7_attention_weight_max": 96.09284973144531,
      "activations/layer7_attention_weight_min": -99.30138397216797,
      "activations/layer8_attention_weight_max": 43.658935546875,
      "activations/layer8_attention_weight_min": -45.1671028137207,
      "activations/layer9_attention_weight_max": 34.06958770751953,
      "activations/layer9_attention_weight_min": -36.255428314208984,
      "epoch": 12.19,
      "learning_rate": 7.211212121212121e-05,
      "loss": 2.7687,
      "step": 209800
    },
    {
      "activations/layer0_attention_weight_max": 16.07193374633789,
      "activations/layer0_attention_weight_min": -11.846898078918457,
      "activations/layer10_attention_weight_max": 35.032527923583984,
      "activations/layer10_attention_weight_min": -32.07472610473633,
      "activations/layer11_attention_weight_max": 33.714683532714844,
      "activations/layer11_attention_weight_min": -30.737945556640625,
      "activations/layer12_attention_weight_max": 24.07032585144043,
      "activations/layer12_attention_weight_min": -24.0565128326416,
      "activations/layer13_attention_weight_max": 38.0699348449707,
      "activations/layer13_attention_weight_min": -31.961626052856445,
      "activations/layer14_attention_weight_max": 42.397125244140625,
      "activations/layer14_attention_weight_min": -35.89958190917969,
      "activations/layer15_attention_weight_max": 36.47066116333008,
      "activations/layer15_attention_weight_min": -31.794965744018555,
      "activations/layer16_attention_weight_max": 30.78182029724121,
      "activations/layer16_attention_weight_min": -28.136781692504883,
      "activations/layer17_attention_weight_max": 33.9746208190918,
      "activations/layer17_attention_weight_min": -25.12149429321289,
      "activations/layer18_attention_weight_max": 32.25525665283203,
      "activations/layer18_attention_weight_min": -23.945438385009766,
      "activations/layer19_attention_weight_max": 36.11597442626953,
      "activations/layer19_attention_weight_min": -30.668954849243164,
      "activations/layer1_attention_weight_max": 15.705446243286133,
      "activations/layer1_attention_weight_min": -14.526288032531738,
      "activations/layer20_attention_weight_max": 30.801895141601562,
      "activations/layer20_attention_weight_min": -25.021135330200195,
      "activations/layer21_attention_weight_max": 30.39409637451172,
      "activations/layer21_attention_weight_min": -24.467971801757812,
      "activations/layer22_attention_weight_max": 44.67879104614258,
      "activations/layer22_attention_weight_min": -28.83314323425293,
      "activations/layer23_attention_weight_max": 36.02189254760742,
      "activations/layer23_attention_weight_min": -24.908498764038086,
      "activations/layer2_attention_weight_max": 31.91779899597168,
      "activations/layer2_attention_weight_min": -30.676481246948242,
      "activations/layer3_attention_weight_max": 89.00980377197266,
      "activations/layer3_attention_weight_min": -92.86445617675781,
      "activations/layer4_attention_weight_max": 111.18167877197266,
      "activations/layer4_attention_weight_min": -113.86415100097656,
      "activations/layer5_attention_weight_max": 54.04000473022461,
      "activations/layer5_attention_weight_min": -62.51948547363281,
      "activations/layer6_attention_weight_max": 43.81510925292969,
      "activations/layer6_attention_weight_min": -46.4672966003418,
      "activations/layer7_attention_weight_max": 89.51399230957031,
      "activations/layer7_attention_weight_min": -90.8805923461914,
      "activations/layer8_attention_weight_max": 41.870147705078125,
      "activations/layer8_attention_weight_min": -41.85213088989258,
      "activations/layer9_attention_weight_max": 35.1230583190918,
      "activations/layer9_attention_weight_min": -33.321041107177734,
      "epoch": 12.19,
      "learning_rate": 7.209318181818181e-05,
      "loss": 2.7805,
      "step": 209850
    },
    {
      "activations/layer0_attention_weight_max": 15.652729988098145,
      "activations/layer0_attention_weight_min": -12.718324661254883,
      "activations/layer10_attention_weight_max": 31.933481216430664,
      "activations/layer10_attention_weight_min": -32.865447998046875,
      "activations/layer11_attention_weight_max": 30.869619369506836,
      "activations/layer11_attention_weight_min": -33.63921356201172,
      "activations/layer12_attention_weight_max": 24.628677368164062,
      "activations/layer12_attention_weight_min": -26.92872428894043,
      "activations/layer13_attention_weight_max": 38.26424026489258,
      "activations/layer13_attention_weight_min": -33.1639518737793,
      "activations/layer14_attention_weight_max": 44.67098617553711,
      "activations/layer14_attention_weight_min": -35.445716857910156,
      "activations/layer15_attention_weight_max": 34.44585037231445,
      "activations/layer15_attention_weight_min": -29.774389266967773,
      "activations/layer16_attention_weight_max": 30.918087005615234,
      "activations/layer16_attention_weight_min": -26.586938858032227,
      "activations/layer17_attention_weight_max": 29.75944709777832,
      "activations/layer17_attention_weight_min": -24.810823440551758,
      "activations/layer18_attention_weight_max": 30.373714447021484,
      "activations/layer18_attention_weight_min": -24.5244140625,
      "activations/layer19_attention_weight_max": 30.7366943359375,
      "activations/layer19_attention_weight_min": -31.528226852416992,
      "activations/layer1_attention_weight_max": 15.741812705993652,
      "activations/layer1_attention_weight_min": -14.00831127166748,
      "activations/layer20_attention_weight_max": 28.13656234741211,
      "activations/layer20_attention_weight_min": -24.882539749145508,
      "activations/layer21_attention_weight_max": 29.867990493774414,
      "activations/layer21_attention_weight_min": -26.71187973022461,
      "activations/layer22_attention_weight_max": 40.08929443359375,
      "activations/layer22_attention_weight_min": -31.8503475189209,
      "activations/layer23_attention_weight_max": 32.96049880981445,
      "activations/layer23_attention_weight_min": -25.75727653503418,
      "activations/layer2_attention_weight_max": 32.74302291870117,
      "activations/layer2_attention_weight_min": -32.6259651184082,
      "activations/layer3_attention_weight_max": 92.47584533691406,
      "activations/layer3_attention_weight_min": -99.38787841796875,
      "activations/layer4_attention_weight_max": 110.35597229003906,
      "activations/layer4_attention_weight_min": -109.36132049560547,
      "activations/layer5_attention_weight_max": 49.64392852783203,
      "activations/layer5_attention_weight_min": -63.11766052246094,
      "activations/layer6_attention_weight_max": 43.9055061340332,
      "activations/layer6_attention_weight_min": -49.362789154052734,
      "activations/layer7_attention_weight_max": 95.75526428222656,
      "activations/layer7_attention_weight_min": -92.75486755371094,
      "activations/layer8_attention_weight_max": 40.65425491333008,
      "activations/layer8_attention_weight_min": -42.72537612915039,
      "activations/layer9_attention_weight_max": 31.45389747619629,
      "activations/layer9_attention_weight_min": -32.70022964477539,
      "epoch": 12.2,
      "learning_rate": 7.207424242424241e-05,
      "loss": 2.7627,
      "step": 209900
    },
    {
      "activations/layer0_attention_weight_max": 16.794620513916016,
      "activations/layer0_attention_weight_min": -12.786812782287598,
      "activations/layer10_attention_weight_max": 36.158790588378906,
      "activations/layer10_attention_weight_min": -33.359039306640625,
      "activations/layer11_attention_weight_max": 32.58259201049805,
      "activations/layer11_attention_weight_min": -32.6590576171875,
      "activations/layer12_attention_weight_max": 25.752840042114258,
      "activations/layer12_attention_weight_min": -24.622983932495117,
      "activations/layer13_attention_weight_max": 41.38813400268555,
      "activations/layer13_attention_weight_min": -34.1859130859375,
      "activations/layer14_attention_weight_max": 43.77156448364258,
      "activations/layer14_attention_weight_min": -36.414344787597656,
      "activations/layer15_attention_weight_max": 35.594764709472656,
      "activations/layer15_attention_weight_min": -32.106998443603516,
      "activations/layer16_attention_weight_max": 31.56032371520996,
      "activations/layer16_attention_weight_min": -27.161964416503906,
      "activations/layer17_attention_weight_max": 34.18547439575195,
      "activations/layer17_attention_weight_min": -26.335050582885742,
      "activations/layer18_attention_weight_max": 32.30805587768555,
      "activations/layer18_attention_weight_min": -22.911657333374023,
      "activations/layer19_attention_weight_max": 34.028907775878906,
      "activations/layer19_attention_weight_min": -28.8790340423584,
      "activations/layer1_attention_weight_max": 16.457313537597656,
      "activations/layer1_attention_weight_min": -14.705435752868652,
      "activations/layer20_attention_weight_max": 30.70697593688965,
      "activations/layer20_attention_weight_min": -24.637922286987305,
      "activations/layer21_attention_weight_max": 29.172733306884766,
      "activations/layer21_attention_weight_min": -24.124588012695312,
      "activations/layer22_attention_weight_max": 45.59559631347656,
      "activations/layer22_attention_weight_min": -29.18515396118164,
      "activations/layer23_attention_weight_max": 33.426780700683594,
      "activations/layer23_attention_weight_min": -25.447834014892578,
      "activations/layer2_attention_weight_max": 30.985301971435547,
      "activations/layer2_attention_weight_min": -31.235729217529297,
      "activations/layer3_attention_weight_max": 91.16413116455078,
      "activations/layer3_attention_weight_min": -92.79205322265625,
      "activations/layer4_attention_weight_max": 109.49055480957031,
      "activations/layer4_attention_weight_min": -108.47989654541016,
      "activations/layer5_attention_weight_max": 52.605072021484375,
      "activations/layer5_attention_weight_min": -60.20256805419922,
      "activations/layer6_attention_weight_max": 42.718196868896484,
      "activations/layer6_attention_weight_min": -46.70113754272461,
      "activations/layer7_attention_weight_max": 88.21257781982422,
      "activations/layer7_attention_weight_min": -92.27638244628906,
      "activations/layer8_attention_weight_max": 40.81561279296875,
      "activations/layer8_attention_weight_min": -42.46261215209961,
      "activations/layer9_attention_weight_max": 34.34226608276367,
      "activations/layer9_attention_weight_min": -33.920162200927734,
      "epoch": 12.2,
      "learning_rate": 7.205530303030303e-05,
      "loss": 2.7575,
      "step": 209950
    },
    {
      "activations/layer0_attention_weight_max": 15.504100799560547,
      "activations/layer0_attention_weight_min": -12.541069030761719,
      "activations/layer10_attention_weight_max": 35.33366775512695,
      "activations/layer10_attention_weight_min": -32.23768997192383,
      "activations/layer11_attention_weight_max": 33.03608703613281,
      "activations/layer11_attention_weight_min": -32.841758728027344,
      "activations/layer12_attention_weight_max": 25.858667373657227,
      "activations/layer12_attention_weight_min": -26.243120193481445,
      "activations/layer13_attention_weight_max": 38.69245147705078,
      "activations/layer13_attention_weight_min": -37.72711181640625,
      "activations/layer14_attention_weight_max": 45.86346435546875,
      "activations/layer14_attention_weight_min": -41.92137908935547,
      "activations/layer15_attention_weight_max": 36.847896575927734,
      "activations/layer15_attention_weight_min": -32.88909149169922,
      "activations/layer16_attention_weight_max": 34.86954116821289,
      "activations/layer16_attention_weight_min": -28.58232307434082,
      "activations/layer17_attention_weight_max": 30.98556900024414,
      "activations/layer17_attention_weight_min": -25.524707794189453,
      "activations/layer18_attention_weight_max": 32.64564514160156,
      "activations/layer18_attention_weight_min": -23.724763870239258,
      "activations/layer19_attention_weight_max": 35.000572204589844,
      "activations/layer19_attention_weight_min": -30.48383903503418,
      "activations/layer1_attention_weight_max": 15.780529975891113,
      "activations/layer1_attention_weight_min": -14.647031784057617,
      "activations/layer20_attention_weight_max": 30.693275451660156,
      "activations/layer20_attention_weight_min": -23.84244155883789,
      "activations/layer21_attention_weight_max": 29.0009765625,
      "activations/layer21_attention_weight_min": -25.253524780273438,
      "activations/layer22_attention_weight_max": 46.720794677734375,
      "activations/layer22_attention_weight_min": -30.823368072509766,
      "activations/layer23_attention_weight_max": 35.529624938964844,
      "activations/layer23_attention_weight_min": -24.631345748901367,
      "activations/layer2_attention_weight_max": 30.953819274902344,
      "activations/layer2_attention_weight_min": -30.998899459838867,
      "activations/layer3_attention_weight_max": 92.27649688720703,
      "activations/layer3_attention_weight_min": -93.06419372558594,
      "activations/layer4_attention_weight_max": 105.88228607177734,
      "activations/layer4_attention_weight_min": -106.95906066894531,
      "activations/layer5_attention_weight_max": 50.924190521240234,
      "activations/layer5_attention_weight_min": -65.2384033203125,
      "activations/layer6_attention_weight_max": 41.72182846069336,
      "activations/layer6_attention_weight_min": -48.386192321777344,
      "activations/layer7_attention_weight_max": 89.39398956298828,
      "activations/layer7_attention_weight_min": -89.95960235595703,
      "activations/layer8_attention_weight_max": 38.6280403137207,
      "activations/layer8_attention_weight_min": -41.75053405761719,
      "activations/layer9_attention_weight_max": 32.6706657409668,
      "activations/layer9_attention_weight_min": -34.34988021850586,
      "epoch": 12.2,
      "learning_rate": 7.203636363636363e-05,
      "loss": 2.7719,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4424,
      "eval_samples_per_second": 508.626,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4424,
      "eval_openwebtext_samples_per_second": 508.626,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9288,
      "eval_wikitext_samples_per_second": 236.42,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_lambada_loss": 2.470703125,
      "eval_lambada_ppl": 11.830762432692795,
      "eval_lambada_runtime": 9.4996,
      "eval_lambada_samples_per_second": 512.546,
      "step": 210000
    },
    {
      "activations/layer0_attention_weight_max": 16.155574798583984,
      "activations/layer0_attention_weight_min": -12.938312530517578,
      "activations/layer10_attention_weight_max": 34.304656982421875,
      "activations/layer10_attention_weight_min": -32.85295867919922,
      "activations/layer11_attention_weight_max": 33.14894104003906,
      "activations/layer11_attention_weight_min": -33.46393585205078,
      "activations/layer12_attention_weight_max": 24.971023559570312,
      "activations/layer12_attention_weight_min": -25.67690658569336,
      "activations/layer13_attention_weight_max": 41.356929779052734,
      "activations/layer13_attention_weight_min": -34.21318817138672,
      "activations/layer14_attention_weight_max": 44.52208709716797,
      "activations/layer14_attention_weight_min": -40.061100006103516,
      "activations/layer15_attention_weight_max": 37.095909118652344,
      "activations/layer15_attention_weight_min": -31.74905776977539,
      "activations/layer16_attention_weight_max": 32.337432861328125,
      "activations/layer16_attention_weight_min": -28.19877815246582,
      "activations/layer17_attention_weight_max": 33.223079681396484,
      "activations/layer17_attention_weight_min": -25.65662956237793,
      "activations/layer18_attention_weight_max": 33.22284698486328,
      "activations/layer18_attention_weight_min": -22.44152069091797,
      "activations/layer19_attention_weight_max": 37.36388397216797,
      "activations/layer19_attention_weight_min": -30.93463706970215,
      "activations/layer1_attention_weight_max": 16.619150161743164,
      "activations/layer1_attention_weight_min": -14.54192066192627,
      "activations/layer20_attention_weight_max": 31.24251365661621,
      "activations/layer20_attention_weight_min": -24.943668365478516,
      "activations/layer21_attention_weight_max": 34.017486572265625,
      "activations/layer21_attention_weight_min": -23.595500946044922,
      "activations/layer22_attention_weight_max": 44.182369232177734,
      "activations/layer22_attention_weight_min": -28.956045150756836,
      "activations/layer23_attention_weight_max": 38.79209518432617,
      "activations/layer23_attention_weight_min": -22.678550720214844,
      "activations/layer2_attention_weight_max": 31.578474044799805,
      "activations/layer2_attention_weight_min": -31.531597137451172,
      "activations/layer3_attention_weight_max": 90.17744445800781,
      "activations/layer3_attention_weight_min": -93.32784271240234,
      "activations/layer4_attention_weight_max": 106.86543273925781,
      "activations/layer4_attention_weight_min": -110.3409423828125,
      "activations/layer5_attention_weight_max": 52.837493896484375,
      "activations/layer5_attention_weight_min": -63.6721305847168,
      "activations/layer6_attention_weight_max": 46.57501983642578,
      "activations/layer6_attention_weight_min": -50.16708755493164,
      "activations/layer7_attention_weight_max": 90.3154525756836,
      "activations/layer7_attention_weight_min": -93.64054107666016,
      "activations/layer8_attention_weight_max": 42.1084098815918,
      "activations/layer8_attention_weight_min": -43.3049430847168,
      "activations/layer9_attention_weight_max": 33.83657455444336,
      "activations/layer9_attention_weight_min": -36.285728454589844,
      "epoch": 12.21,
      "learning_rate": 7.201742424242423e-05,
      "loss": 2.7642,
      "step": 210050
    },
    {
      "activations/layer0_attention_weight_max": 15.775260925292969,
      "activations/layer0_attention_weight_min": -12.056134223937988,
      "activations/layer10_attention_weight_max": 33.23969268798828,
      "activations/layer10_attention_weight_min": -31.281768798828125,
      "activations/layer11_attention_weight_max": 33.734375,
      "activations/layer11_attention_weight_min": -31.371152877807617,
      "activations/layer12_attention_weight_max": 36.57932662963867,
      "activations/layer12_attention_weight_min": -26.179624557495117,
      "activations/layer13_attention_weight_max": 47.38358688354492,
      "activations/layer13_attention_weight_min": -33.92815399169922,
      "activations/layer14_attention_weight_max": 43.187686920166016,
      "activations/layer14_attention_weight_min": -37.23881530761719,
      "activations/layer15_attention_weight_max": 37.51753234863281,
      "activations/layer15_attention_weight_min": -33.53461837768555,
      "activations/layer16_attention_weight_max": 32.96870422363281,
      "activations/layer16_attention_weight_min": -28.755582809448242,
      "activations/layer17_attention_weight_max": 34.2861328125,
      "activations/layer17_attention_weight_min": -25.549549102783203,
      "activations/layer18_attention_weight_max": 36.32057189941406,
      "activations/layer18_attention_weight_min": -25.898391723632812,
      "activations/layer19_attention_weight_max": 35.97623062133789,
      "activations/layer19_attention_weight_min": -30.677770614624023,
      "activations/layer1_attention_weight_max": 16.337663650512695,
      "activations/layer1_attention_weight_min": -15.665701866149902,
      "activations/layer20_attention_weight_max": 30.165733337402344,
      "activations/layer20_attention_weight_min": -24.064550399780273,
      "activations/layer21_attention_weight_max": 28.910995483398438,
      "activations/layer21_attention_weight_min": -23.390365600585938,
      "activations/layer22_attention_weight_max": 44.43482208251953,
      "activations/layer22_attention_weight_min": -29.726234436035156,
      "activations/layer23_attention_weight_max": 39.308311462402344,
      "activations/layer23_attention_weight_min": -25.61758804321289,
      "activations/layer2_attention_weight_max": 32.11151123046875,
      "activations/layer2_attention_weight_min": -33.13641357421875,
      "activations/layer3_attention_weight_max": 93.68445587158203,
      "activations/layer3_attention_weight_min": -94.6491470336914,
      "activations/layer4_attention_weight_max": 111.13350677490234,
      "activations/layer4_attention_weight_min": -115.08082580566406,
      "activations/layer5_attention_weight_max": 53.283966064453125,
      "activations/layer5_attention_weight_min": -62.23981475830078,
      "activations/layer6_attention_weight_max": 43.59611511230469,
      "activations/layer6_attention_weight_min": -46.84444046020508,
      "activations/layer7_attention_weight_max": 85.28479766845703,
      "activations/layer7_attention_weight_min": -86.78665924072266,
      "activations/layer8_attention_weight_max": 37.091739654541016,
      "activations/layer8_attention_weight_min": -38.36275100708008,
      "activations/layer9_attention_weight_max": 30.509078979492188,
      "activations/layer9_attention_weight_min": -31.687061309814453,
      "epoch": 12.21,
      "learning_rate": 7.199848484848484e-05,
      "loss": 2.757,
      "step": 210100
    },
    {
      "activations/layer0_attention_weight_max": 15.870694160461426,
      "activations/layer0_attention_weight_min": -12.8597993850708,
      "activations/layer10_attention_weight_max": 51.2670783996582,
      "activations/layer10_attention_weight_min": -46.17499923706055,
      "activations/layer11_attention_weight_max": 51.55329513549805,
      "activations/layer11_attention_weight_min": -49.02016830444336,
      "activations/layer12_attention_weight_max": 35.821224212646484,
      "activations/layer12_attention_weight_min": -31.671703338623047,
      "activations/layer13_attention_weight_max": 61.64470672607422,
      "activations/layer13_attention_weight_min": -43.17365264892578,
      "activations/layer14_attention_weight_max": 63.670291900634766,
      "activations/layer14_attention_weight_min": -46.922698974609375,
      "activations/layer15_attention_weight_max": 71.6407241821289,
      "activations/layer15_attention_weight_min": -40.87510681152344,
      "activations/layer16_attention_weight_max": 38.822505950927734,
      "activations/layer16_attention_weight_min": -29.882356643676758,
      "activations/layer17_attention_weight_max": 30.644649505615234,
      "activations/layer17_attention_weight_min": -29.15907859802246,
      "activations/layer18_attention_weight_max": 30.004207611083984,
      "activations/layer18_attention_weight_min": -25.092859268188477,
      "activations/layer19_attention_weight_max": 38.86469268798828,
      "activations/layer19_attention_weight_min": -30.010055541992188,
      "activations/layer1_attention_weight_max": 17.155914306640625,
      "activations/layer1_attention_weight_min": -14.581875801086426,
      "activations/layer20_attention_weight_max": 31.93422508239746,
      "activations/layer20_attention_weight_min": -23.212303161621094,
      "activations/layer21_attention_weight_max": 26.995590209960938,
      "activations/layer21_attention_weight_min": -23.927194595336914,
      "activations/layer22_attention_weight_max": 50.08391189575195,
      "activations/layer22_attention_weight_min": -30.996463775634766,
      "activations/layer23_attention_weight_max": 36.45735168457031,
      "activations/layer23_attention_weight_min": -25.67566680908203,
      "activations/layer2_attention_weight_max": 34.614444732666016,
      "activations/layer2_attention_weight_min": -32.62449645996094,
      "activations/layer3_attention_weight_max": 101.4717025756836,
      "activations/layer3_attention_weight_min": -100.04924011230469,
      "activations/layer4_attention_weight_max": 114.21756744384766,
      "activations/layer4_attention_weight_min": -112.3406753540039,
      "activations/layer5_attention_weight_max": 56.625152587890625,
      "activations/layer5_attention_weight_min": -58.57482147216797,
      "activations/layer6_attention_weight_max": 51.36845016479492,
      "activations/layer6_attention_weight_min": -48.87581253051758,
      "activations/layer7_attention_weight_max": 116.32110595703125,
      "activations/layer7_attention_weight_min": -105.82271575927734,
      "activations/layer8_attention_weight_max": 55.205299377441406,
      "activations/layer8_attention_weight_min": -51.52733612060547,
      "activations/layer9_attention_weight_max": 49.1443977355957,
      "activations/layer9_attention_weight_min": -46.15958786010742,
      "epoch": 12.21,
      "learning_rate": 7.197954545454545e-05,
      "loss": 2.7773,
      "step": 210150
    },
    {
      "activations/layer0_attention_weight_max": 16.197690963745117,
      "activations/layer0_attention_weight_min": -13.24599552154541,
      "activations/layer10_attention_weight_max": 33.684791564941406,
      "activations/layer10_attention_weight_min": -32.141204833984375,
      "activations/layer11_attention_weight_max": 31.665184020996094,
      "activations/layer11_attention_weight_min": -32.062721252441406,
      "activations/layer12_attention_weight_max": 24.361242294311523,
      "activations/layer12_attention_weight_min": -25.18035316467285,
      "activations/layer13_attention_weight_max": 40.02949905395508,
      "activations/layer13_attention_weight_min": -33.964847564697266,
      "activations/layer14_attention_weight_max": 43.8723030090332,
      "activations/layer14_attention_weight_min": -37.0437126159668,
      "activations/layer15_attention_weight_max": 39.04833984375,
      "activations/layer15_attention_weight_min": -32.41340637207031,
      "activations/layer16_attention_weight_max": 32.332820892333984,
      "activations/layer16_attention_weight_min": -28.80756950378418,
      "activations/layer17_attention_weight_max": 32.42787551879883,
      "activations/layer17_attention_weight_min": -26.20517921447754,
      "activations/layer18_attention_weight_max": 29.733421325683594,
      "activations/layer18_attention_weight_min": -26.261764526367188,
      "activations/layer19_attention_weight_max": 37.219520568847656,
      "activations/layer19_attention_weight_min": -32.00312042236328,
      "activations/layer1_attention_weight_max": 15.995194435119629,
      "activations/layer1_attention_weight_min": -13.693273544311523,
      "activations/layer20_attention_weight_max": 28.031633377075195,
      "activations/layer20_attention_weight_min": -25.14638328552246,
      "activations/layer21_attention_weight_max": 27.48622703552246,
      "activations/layer21_attention_weight_min": -25.369029998779297,
      "activations/layer22_attention_weight_max": 45.97649002075195,
      "activations/layer22_attention_weight_min": -30.75552749633789,
      "activations/layer23_attention_weight_max": 34.554168701171875,
      "activations/layer23_attention_weight_min": -23.696439743041992,
      "activations/layer2_attention_weight_max": 33.24086380004883,
      "activations/layer2_attention_weight_min": -30.244909286499023,
      "activations/layer3_attention_weight_max": 92.0399169921875,
      "activations/layer3_attention_weight_min": -92.08732604980469,
      "activations/layer4_attention_weight_max": 106.69435119628906,
      "activations/layer4_attention_weight_min": -106.66865539550781,
      "activations/layer5_attention_weight_max": 49.87921142578125,
      "activations/layer5_attention_weight_min": -60.0830192565918,
      "activations/layer6_attention_weight_max": 44.646339416503906,
      "activations/layer6_attention_weight_min": -49.61008834838867,
      "activations/layer7_attention_weight_max": 90.147216796875,
      "activations/layer7_attention_weight_min": -85.0265121459961,
      "activations/layer8_attention_weight_max": 38.3990364074707,
      "activations/layer8_attention_weight_min": -40.51054763793945,
      "activations/layer9_attention_weight_max": 31.551786422729492,
      "activations/layer9_attention_weight_min": -33.2755241394043,
      "epoch": 12.21,
      "learning_rate": 7.196060606060606e-05,
      "loss": 2.7725,
      "step": 210200
    },
    {
      "activations/layer0_attention_weight_max": 15.785791397094727,
      "activations/layer0_attention_weight_min": -12.605599403381348,
      "activations/layer10_attention_weight_max": 32.43729019165039,
      "activations/layer10_attention_weight_min": -35.18278121948242,
      "activations/layer11_attention_weight_max": 31.781848907470703,
      "activations/layer11_attention_weight_min": -36.40254592895508,
      "activations/layer12_attention_weight_max": 27.049272537231445,
      "activations/layer12_attention_weight_min": -25.832765579223633,
      "activations/layer13_attention_weight_max": 37.641292572021484,
      "activations/layer13_attention_weight_min": -35.67549133300781,
      "activations/layer14_attention_weight_max": 46.015647888183594,
      "activations/layer14_attention_weight_min": -37.65929412841797,
      "activations/layer15_attention_weight_max": 37.90099334716797,
      "activations/layer15_attention_weight_min": -33.89717102050781,
      "activations/layer16_attention_weight_max": 36.026336669921875,
      "activations/layer16_attention_weight_min": -27.658754348754883,
      "activations/layer17_attention_weight_max": 34.0594482421875,
      "activations/layer17_attention_weight_min": -26.435510635375977,
      "activations/layer18_attention_weight_max": 33.07318115234375,
      "activations/layer18_attention_weight_min": -23.542892456054688,
      "activations/layer19_attention_weight_max": 34.313899993896484,
      "activations/layer19_attention_weight_min": -30.885089874267578,
      "activations/layer1_attention_weight_max": 16.59008026123047,
      "activations/layer1_attention_weight_min": -13.936056137084961,
      "activations/layer20_attention_weight_max": 33.601009368896484,
      "activations/layer20_attention_weight_min": -24.790082931518555,
      "activations/layer21_attention_weight_max": 31.153942108154297,
      "activations/layer21_attention_weight_min": -23.994232177734375,
      "activations/layer22_attention_weight_max": 45.06294631958008,
      "activations/layer22_attention_weight_min": -31.583724975585938,
      "activations/layer23_attention_weight_max": 32.26706314086914,
      "activations/layer23_attention_weight_min": -23.253524780273438,
      "activations/layer2_attention_weight_max": 30.909391403198242,
      "activations/layer2_attention_weight_min": -29.86099624633789,
      "activations/layer3_attention_weight_max": 90.29137420654297,
      "activations/layer3_attention_weight_min": -91.02096557617188,
      "activations/layer4_attention_weight_max": 106.36517333984375,
      "activations/layer4_attention_weight_min": -109.26519775390625,
      "activations/layer5_attention_weight_max": 51.307090759277344,
      "activations/layer5_attention_weight_min": -64.07943725585938,
      "activations/layer6_attention_weight_max": 46.4565315246582,
      "activations/layer6_attention_weight_min": -47.178428649902344,
      "activations/layer7_attention_weight_max": 94.99710083007812,
      "activations/layer7_attention_weight_min": -89.04772186279297,
      "activations/layer8_attention_weight_max": 38.93330764770508,
      "activations/layer8_attention_weight_min": -41.64570999145508,
      "activations/layer9_attention_weight_max": 32.40351104736328,
      "activations/layer9_attention_weight_min": -34.524261474609375,
      "epoch": 12.22,
      "learning_rate": 7.194166666666666e-05,
      "loss": 2.7955,
      "step": 210250
    },
    {
      "activations/layer0_attention_weight_max": 16.434814453125,
      "activations/layer0_attention_weight_min": -12.288490295410156,
      "activations/layer10_attention_weight_max": 45.77132034301758,
      "activations/layer10_attention_weight_min": -39.765655517578125,
      "activations/layer11_attention_weight_max": 47.89277648925781,
      "activations/layer11_attention_weight_min": -41.77606201171875,
      "activations/layer12_attention_weight_max": 29.19459342956543,
      "activations/layer12_attention_weight_min": -27.204036712646484,
      "activations/layer13_attention_weight_max": 41.089210510253906,
      "activations/layer13_attention_weight_min": -37.29243469238281,
      "activations/layer14_attention_weight_max": 44.09885787963867,
      "activations/layer14_attention_weight_min": -41.80675506591797,
      "activations/layer15_attention_weight_max": 43.1895751953125,
      "activations/layer15_attention_weight_min": -36.580665588378906,
      "activations/layer16_attention_weight_max": 33.006080627441406,
      "activations/layer16_attention_weight_min": -29.135332107543945,
      "activations/layer17_attention_weight_max": 33.0876350402832,
      "activations/layer17_attention_weight_min": -26.87587547302246,
      "activations/layer18_attention_weight_max": 33.17405319213867,
      "activations/layer18_attention_weight_min": -23.949283599853516,
      "activations/layer19_attention_weight_max": 36.521663665771484,
      "activations/layer19_attention_weight_min": -31.885480880737305,
      "activations/layer1_attention_weight_max": 16.776094436645508,
      "activations/layer1_attention_weight_min": -15.701582908630371,
      "activations/layer20_attention_weight_max": 33.102813720703125,
      "activations/layer20_attention_weight_min": -25.778886795043945,
      "activations/layer21_attention_weight_max": 29.879009246826172,
      "activations/layer21_attention_weight_min": -23.264928817749023,
      "activations/layer22_attention_weight_max": 42.91117477416992,
      "activations/layer22_attention_weight_min": -29.6049861907959,
      "activations/layer23_attention_weight_max": 36.43102264404297,
      "activations/layer23_attention_weight_min": -24.61677360534668,
      "activations/layer2_attention_weight_max": 34.40610885620117,
      "activations/layer2_attention_weight_min": -32.05555725097656,
      "activations/layer3_attention_weight_max": 98.60599517822266,
      "activations/layer3_attention_weight_min": -99.05776977539062,
      "activations/layer4_attention_weight_max": 118.64912414550781,
      "activations/layer4_attention_weight_min": -113.14546966552734,
      "activations/layer5_attention_weight_max": 54.86349105834961,
      "activations/layer5_attention_weight_min": -63.280941009521484,
      "activations/layer6_attention_weight_max": 50.30574417114258,
      "activations/layer6_attention_weight_min": -48.242225646972656,
      "activations/layer7_attention_weight_max": 109.56001281738281,
      "activations/layer7_attention_weight_min": -98.7995376586914,
      "activations/layer8_attention_weight_max": 51.29457092285156,
      "activations/layer8_attention_weight_min": -47.895572662353516,
      "activations/layer9_attention_weight_max": 41.35234069824219,
      "activations/layer9_attention_weight_min": -39.87751770019531,
      "epoch": 12.22,
      "learning_rate": 7.192272727272726e-05,
      "loss": 2.7796,
      "step": 210300
    },
    {
      "activations/layer0_attention_weight_max": 15.771885871887207,
      "activations/layer0_attention_weight_min": -12.135117530822754,
      "activations/layer10_attention_weight_max": 33.4504280090332,
      "activations/layer10_attention_weight_min": -35.312435150146484,
      "activations/layer11_attention_weight_max": 33.314605712890625,
      "activations/layer11_attention_weight_min": -35.30461502075195,
      "activations/layer12_attention_weight_max": 24.384347915649414,
      "activations/layer12_attention_weight_min": -26.760902404785156,
      "activations/layer13_attention_weight_max": 38.27934646606445,
      "activations/layer13_attention_weight_min": -35.20417404174805,
      "activations/layer14_attention_weight_max": 46.61696243286133,
      "activations/layer14_attention_weight_min": -41.001365661621094,
      "activations/layer15_attention_weight_max": 35.89590072631836,
      "activations/layer15_attention_weight_min": -35.072166442871094,
      "activations/layer16_attention_weight_max": 30.568994522094727,
      "activations/layer16_attention_weight_min": -29.075069427490234,
      "activations/layer17_attention_weight_max": 32.07383728027344,
      "activations/layer17_attention_weight_min": -27.020601272583008,
      "activations/layer18_attention_weight_max": 35.10372543334961,
      "activations/layer18_attention_weight_min": -23.886898040771484,
      "activations/layer19_attention_weight_max": 40.32807922363281,
      "activations/layer19_attention_weight_min": -31.64996337890625,
      "activations/layer1_attention_weight_max": 17.339599609375,
      "activations/layer1_attention_weight_min": -14.310617446899414,
      "activations/layer20_attention_weight_max": 35.605403900146484,
      "activations/layer20_attention_weight_min": -24.757583618164062,
      "activations/layer21_attention_weight_max": 34.37854766845703,
      "activations/layer21_attention_weight_min": -23.153444290161133,
      "activations/layer22_attention_weight_max": 48.99710464477539,
      "activations/layer22_attention_weight_min": -30.49742889404297,
      "activations/layer23_attention_weight_max": 33.92485809326172,
      "activations/layer23_attention_weight_min": -24.04320526123047,
      "activations/layer2_attention_weight_max": 31.97127342224121,
      "activations/layer2_attention_weight_min": -33.501258850097656,
      "activations/layer3_attention_weight_max": 96.40592193603516,
      "activations/layer3_attention_weight_min": -97.1496353149414,
      "activations/layer4_attention_weight_max": 112.2095718383789,
      "activations/layer4_attention_weight_min": -113.50789642333984,
      "activations/layer5_attention_weight_max": 53.48918914794922,
      "activations/layer5_attention_weight_min": -63.57847213745117,
      "activations/layer6_attention_weight_max": 45.68721389770508,
      "activations/layer6_attention_weight_min": -49.3557243347168,
      "activations/layer7_attention_weight_max": 94.47842407226562,
      "activations/layer7_attention_weight_min": -94.79146575927734,
      "activations/layer8_attention_weight_max": 41.47488021850586,
      "activations/layer8_attention_weight_min": -45.75437927246094,
      "activations/layer9_attention_weight_max": 35.31351852416992,
      "activations/layer9_attention_weight_min": -37.44011688232422,
      "epoch": 12.22,
      "learning_rate": 7.190378787878788e-05,
      "loss": 2.7495,
      "step": 210350
    },
    {
      "activations/layer0_attention_weight_max": 16.477664947509766,
      "activations/layer0_attention_weight_min": -12.124407768249512,
      "activations/layer10_attention_weight_max": 31.96593475341797,
      "activations/layer10_attention_weight_min": -30.530229568481445,
      "activations/layer11_attention_weight_max": 34.93277359008789,
      "activations/layer11_attention_weight_min": -30.342792510986328,
      "activations/layer12_attention_weight_max": 24.29351234436035,
      "activations/layer12_attention_weight_min": -26.825336456298828,
      "activations/layer13_attention_weight_max": 37.15957260131836,
      "activations/layer13_attention_weight_min": -36.81847381591797,
      "activations/layer14_attention_weight_max": 40.232872009277344,
      "activations/layer14_attention_weight_min": -37.80055618286133,
      "activations/layer15_attention_weight_max": 36.426944732666016,
      "activations/layer15_attention_weight_min": -32.16619873046875,
      "activations/layer16_attention_weight_max": 31.75299835205078,
      "activations/layer16_attention_weight_min": -28.62870979309082,
      "activations/layer17_attention_weight_max": 28.74797248840332,
      "activations/layer17_attention_weight_min": -24.718639373779297,
      "activations/layer18_attention_weight_max": 28.11236000061035,
      "activations/layer18_attention_weight_min": -23.33746910095215,
      "activations/layer19_attention_weight_max": 31.66495132446289,
      "activations/layer19_attention_weight_min": -32.09383010864258,
      "activations/layer1_attention_weight_max": 16.936864852905273,
      "activations/layer1_attention_weight_min": -14.460699081420898,
      "activations/layer20_attention_weight_max": 29.72655487060547,
      "activations/layer20_attention_weight_min": -25.7347412109375,
      "activations/layer21_attention_weight_max": 29.369585037231445,
      "activations/layer21_attention_weight_min": -23.876506805419922,
      "activations/layer22_attention_weight_max": 44.729515075683594,
      "activations/layer22_attention_weight_min": -30.956954956054688,
      "activations/layer23_attention_weight_max": 34.8402214050293,
      "activations/layer23_attention_weight_min": -27.485618591308594,
      "activations/layer2_attention_weight_max": 33.45411682128906,
      "activations/layer2_attention_weight_min": -33.375885009765625,
      "activations/layer3_attention_weight_max": 99.31574249267578,
      "activations/layer3_attention_weight_min": -101.84371185302734,
      "activations/layer4_attention_weight_max": 118.71392059326172,
      "activations/layer4_attention_weight_min": -117.73943328857422,
      "activations/layer5_attention_weight_max": 51.922943115234375,
      "activations/layer5_attention_weight_min": -60.8491325378418,
      "activations/layer6_attention_weight_max": 43.62931823730469,
      "activations/layer6_attention_weight_min": -46.345787048339844,
      "activations/layer7_attention_weight_max": 89.98384094238281,
      "activations/layer7_attention_weight_min": -87.59156799316406,
      "activations/layer8_attention_weight_max": 39.591331481933594,
      "activations/layer8_attention_weight_min": -40.179996490478516,
      "activations/layer9_attention_weight_max": 31.162328720092773,
      "activations/layer9_attention_weight_min": -31.8554744720459,
      "epoch": 12.23,
      "learning_rate": 7.188484848484848e-05,
      "loss": 2.783,
      "step": 210400
    },
    {
      "activations/layer0_attention_weight_max": 15.927082061767578,
      "activations/layer0_attention_weight_min": -11.699087142944336,
      "activations/layer10_attention_weight_max": 32.736480712890625,
      "activations/layer10_attention_weight_min": -31.497472763061523,
      "activations/layer11_attention_weight_max": 33.74486541748047,
      "activations/layer11_attention_weight_min": -32.369346618652344,
      "activations/layer12_attention_weight_max": 27.051021575927734,
      "activations/layer12_attention_weight_min": -26.456689834594727,
      "activations/layer13_attention_weight_max": 39.786102294921875,
      "activations/layer13_attention_weight_min": -37.11436080932617,
      "activations/layer14_attention_weight_max": 46.481422424316406,
      "activations/layer14_attention_weight_min": -40.935455322265625,
      "activations/layer15_attention_weight_max": 38.459381103515625,
      "activations/layer15_attention_weight_min": -36.07889938354492,
      "activations/layer16_attention_weight_max": 32.82322692871094,
      "activations/layer16_attention_weight_min": -28.575098037719727,
      "activations/layer17_attention_weight_max": 36.802490234375,
      "activations/layer17_attention_weight_min": -25.131494522094727,
      "activations/layer18_attention_weight_max": 33.271324157714844,
      "activations/layer18_attention_weight_min": -23.6599178314209,
      "activations/layer19_attention_weight_max": 36.15971374511719,
      "activations/layer19_attention_weight_min": -32.295162200927734,
      "activations/layer1_attention_weight_max": 16.783336639404297,
      "activations/layer1_attention_weight_min": -14.179797172546387,
      "activations/layer20_attention_weight_max": 29.761844635009766,
      "activations/layer20_attention_weight_min": -24.621070861816406,
      "activations/layer21_attention_weight_max": 31.903873443603516,
      "activations/layer21_attention_weight_min": -24.0883846282959,
      "activations/layer22_attention_weight_max": 47.50276184082031,
      "activations/layer22_attention_weight_min": -28.994504928588867,
      "activations/layer23_attention_weight_max": 38.0894889831543,
      "activations/layer23_attention_weight_min": -23.945556640625,
      "activations/layer2_attention_weight_max": 31.18893814086914,
      "activations/layer2_attention_weight_min": -30.19318962097168,
      "activations/layer3_attention_weight_max": 91.98040008544922,
      "activations/layer3_attention_weight_min": -92.89954376220703,
      "activations/layer4_attention_weight_max": 106.04439544677734,
      "activations/layer4_attention_weight_min": -107.66169738769531,
      "activations/layer5_attention_weight_max": 51.64507293701172,
      "activations/layer5_attention_weight_min": -60.59564971923828,
      "activations/layer6_attention_weight_max": 42.566795349121094,
      "activations/layer6_attention_weight_min": -50.889034271240234,
      "activations/layer7_attention_weight_max": 88.41862487792969,
      "activations/layer7_attention_weight_min": -96.28956604003906,
      "activations/layer8_attention_weight_max": 38.57561492919922,
      "activations/layer8_attention_weight_min": -40.071624755859375,
      "activations/layer9_attention_weight_max": 32.897762298583984,
      "activations/layer9_attention_weight_min": -31.773603439331055,
      "epoch": 12.23,
      "learning_rate": 7.186590909090908e-05,
      "loss": 2.7671,
      "step": 210450
    },
    {
      "activations/layer0_attention_weight_max": 15.973278999328613,
      "activations/layer0_attention_weight_min": -11.934237480163574,
      "activations/layer10_attention_weight_max": 31.81697654724121,
      "activations/layer10_attention_weight_min": -31.618703842163086,
      "activations/layer11_attention_weight_max": 33.71976089477539,
      "activations/layer11_attention_weight_min": -32.187721252441406,
      "activations/layer12_attention_weight_max": 24.787004470825195,
      "activations/layer12_attention_weight_min": -32.05067443847656,
      "activations/layer13_attention_weight_max": 38.58348083496094,
      "activations/layer13_attention_weight_min": -33.64334487915039,
      "activations/layer14_attention_weight_max": 42.57160186767578,
      "activations/layer14_attention_weight_min": -35.45042419433594,
      "activations/layer15_attention_weight_max": 36.40657043457031,
      "activations/layer15_attention_weight_min": -33.222171783447266,
      "activations/layer16_attention_weight_max": 29.30684471130371,
      "activations/layer16_attention_weight_min": -28.084476470947266,
      "activations/layer17_attention_weight_max": 31.06817054748535,
      "activations/layer17_attention_weight_min": -25.18244743347168,
      "activations/layer18_attention_weight_max": 34.646854400634766,
      "activations/layer18_attention_weight_min": -23.063982009887695,
      "activations/layer19_attention_weight_max": 36.88621139526367,
      "activations/layer19_attention_weight_min": -31.098880767822266,
      "activations/layer1_attention_weight_max": 17.274686813354492,
      "activations/layer1_attention_weight_min": -14.513279914855957,
      "activations/layer20_attention_weight_max": 29.9976806640625,
      "activations/layer20_attention_weight_min": -27.003032684326172,
      "activations/layer21_attention_weight_max": 27.506858825683594,
      "activations/layer21_attention_weight_min": -22.440399169921875,
      "activations/layer22_attention_weight_max": 43.783267974853516,
      "activations/layer22_attention_weight_min": -31.083940505981445,
      "activations/layer23_attention_weight_max": 35.77207946777344,
      "activations/layer23_attention_weight_min": -24.463773727416992,
      "activations/layer2_attention_weight_max": 32.977081298828125,
      "activations/layer2_attention_weight_min": -33.051902770996094,
      "activations/layer3_attention_weight_max": 95.22171783447266,
      "activations/layer3_attention_weight_min": -93.49526977539062,
      "activations/layer4_attention_weight_max": 112.73503875732422,
      "activations/layer4_attention_weight_min": -106.04920959472656,
      "activations/layer5_attention_weight_max": 51.71751022338867,
      "activations/layer5_attention_weight_min": -59.355735778808594,
      "activations/layer6_attention_weight_max": 43.85826110839844,
      "activations/layer6_attention_weight_min": -48.041141510009766,
      "activations/layer7_attention_weight_max": 103.8141098022461,
      "activations/layer7_attention_weight_min": -91.81439971923828,
      "activations/layer8_attention_weight_max": 38.5881233215332,
      "activations/layer8_attention_weight_min": -41.41112518310547,
      "activations/layer9_attention_weight_max": 33.806358337402344,
      "activations/layer9_attention_weight_min": -38.09508514404297,
      "epoch": 12.23,
      "learning_rate": 7.18469696969697e-05,
      "loss": 2.7751,
      "step": 210500
    },
    {
      "activations/layer0_attention_weight_max": 16.014894485473633,
      "activations/layer0_attention_weight_min": -12.1141357421875,
      "activations/layer10_attention_weight_max": 30.92296028137207,
      "activations/layer10_attention_weight_min": -31.762237548828125,
      "activations/layer11_attention_weight_max": 32.81968688964844,
      "activations/layer11_attention_weight_min": -30.60411834716797,
      "activations/layer12_attention_weight_max": 22.967302322387695,
      "activations/layer12_attention_weight_min": -26.79033851623535,
      "activations/layer13_attention_weight_max": 37.68206024169922,
      "activations/layer13_attention_weight_min": -35.929161071777344,
      "activations/layer14_attention_weight_max": 41.67279052734375,
      "activations/layer14_attention_weight_min": -38.59471893310547,
      "activations/layer15_attention_weight_max": 33.485267639160156,
      "activations/layer15_attention_weight_min": -32.824859619140625,
      "activations/layer16_attention_weight_max": 28.54722023010254,
      "activations/layer16_attention_weight_min": -28.725812911987305,
      "activations/layer17_attention_weight_max": 28.552988052368164,
      "activations/layer17_attention_weight_min": -25.363744735717773,
      "activations/layer18_attention_weight_max": 30.425493240356445,
      "activations/layer18_attention_weight_min": -22.93115234375,
      "activations/layer19_attention_weight_max": 33.43937301635742,
      "activations/layer19_attention_weight_min": -30.715518951416016,
      "activations/layer1_attention_weight_max": 16.085182189941406,
      "activations/layer1_attention_weight_min": -14.308441162109375,
      "activations/layer20_attention_weight_max": 31.44464683532715,
      "activations/layer20_attention_weight_min": -25.05929183959961,
      "activations/layer21_attention_weight_max": 31.850444793701172,
      "activations/layer21_attention_weight_min": -24.219329833984375,
      "activations/layer22_attention_weight_max": 43.6845588684082,
      "activations/layer22_attention_weight_min": -30.81209945678711,
      "activations/layer23_attention_weight_max": 36.333030700683594,
      "activations/layer23_attention_weight_min": -28.469266891479492,
      "activations/layer2_attention_weight_max": 30.714927673339844,
      "activations/layer2_attention_weight_min": -31.919479370117188,
      "activations/layer3_attention_weight_max": 92.4229507446289,
      "activations/layer3_attention_weight_min": -92.54810333251953,
      "activations/layer4_attention_weight_max": 111.67411041259766,
      "activations/layer4_attention_weight_min": -114.74305725097656,
      "activations/layer5_attention_weight_max": 55.8674430847168,
      "activations/layer5_attention_weight_min": -61.29462432861328,
      "activations/layer6_attention_weight_max": 43.214942932128906,
      "activations/layer6_attention_weight_min": -45.9251823425293,
      "activations/layer7_attention_weight_max": 86.09373474121094,
      "activations/layer7_attention_weight_min": -85.75090789794922,
      "activations/layer8_attention_weight_max": 39.04853439331055,
      "activations/layer8_attention_weight_min": -39.178627014160156,
      "activations/layer9_attention_weight_max": 29.772912979125977,
      "activations/layer9_attention_weight_min": -31.655664443969727,
      "epoch": 12.23,
      "learning_rate": 7.18280303030303e-05,
      "loss": 2.7599,
      "step": 210550
    },
    {
      "activations/layer0_attention_weight_max": 14.358642578125,
      "activations/layer0_attention_weight_min": -12.528968811035156,
      "activations/layer10_attention_weight_max": 33.32841873168945,
      "activations/layer10_attention_weight_min": -33.11362075805664,
      "activations/layer11_attention_weight_max": 32.76117706298828,
      "activations/layer11_attention_weight_min": -32.70484161376953,
      "activations/layer12_attention_weight_max": 27.040836334228516,
      "activations/layer12_attention_weight_min": -26.248558044433594,
      "activations/layer13_attention_weight_max": 40.417747497558594,
      "activations/layer13_attention_weight_min": -34.70728302001953,
      "activations/layer14_attention_weight_max": 45.97614669799805,
      "activations/layer14_attention_weight_min": -36.96177291870117,
      "activations/layer15_attention_weight_max": 36.05411911010742,
      "activations/layer15_attention_weight_min": -32.0209846496582,
      "activations/layer16_attention_weight_max": 30.934314727783203,
      "activations/layer16_attention_weight_min": -26.836393356323242,
      "activations/layer17_attention_weight_max": 32.772499084472656,
      "activations/layer17_attention_weight_min": -24.984785079956055,
      "activations/layer18_attention_weight_max": 33.61528778076172,
      "activations/layer18_attention_weight_min": -22.949256896972656,
      "activations/layer19_attention_weight_max": 33.4764289855957,
      "activations/layer19_attention_weight_min": -29.190052032470703,
      "activations/layer1_attention_weight_max": 15.949783325195312,
      "activations/layer1_attention_weight_min": -14.189711570739746,
      "activations/layer20_attention_weight_max": 31.5028076171875,
      "activations/layer20_attention_weight_min": -23.06926155090332,
      "activations/layer21_attention_weight_max": 31.637121200561523,
      "activations/layer21_attention_weight_min": -24.67123794555664,
      "activations/layer22_attention_weight_max": 44.190879821777344,
      "activations/layer22_attention_weight_min": -29.23876953125,
      "activations/layer23_attention_weight_max": 35.08235549926758,
      "activations/layer23_attention_weight_min": -23.801029205322266,
      "activations/layer2_attention_weight_max": 31.888965606689453,
      "activations/layer2_attention_weight_min": -31.373310089111328,
      "activations/layer3_attention_weight_max": 93.3675308227539,
      "activations/layer3_attention_weight_min": -98.99755859375,
      "activations/layer4_attention_weight_max": 107.27385711669922,
      "activations/layer4_attention_weight_min": -110.3518295288086,
      "activations/layer5_attention_weight_max": 49.824859619140625,
      "activations/layer5_attention_weight_min": -62.746437072753906,
      "activations/layer6_attention_weight_max": 44.17264938354492,
      "activations/layer6_attention_weight_min": -49.0202751159668,
      "activations/layer7_attention_weight_max": 94.39006805419922,
      "activations/layer7_attention_weight_min": -93.5434799194336,
      "activations/layer8_attention_weight_max": 40.2775764465332,
      "activations/layer8_attention_weight_min": -41.49263381958008,
      "activations/layer9_attention_weight_max": 34.7923698425293,
      "activations/layer9_attention_weight_min": -34.35874557495117,
      "epoch": 12.24,
      "learning_rate": 7.18090909090909e-05,
      "loss": 2.771,
      "step": 210600
    },
    {
      "activations/layer0_attention_weight_max": 16.14331817626953,
      "activations/layer0_attention_weight_min": -13.020354270935059,
      "activations/layer10_attention_weight_max": 31.447444915771484,
      "activations/layer10_attention_weight_min": -31.28884506225586,
      "activations/layer11_attention_weight_max": 36.03936004638672,
      "activations/layer11_attention_weight_min": -31.803983688354492,
      "activations/layer12_attention_weight_max": 25.397977828979492,
      "activations/layer12_attention_weight_min": -23.946165084838867,
      "activations/layer13_attention_weight_max": 37.847042083740234,
      "activations/layer13_attention_weight_min": -33.80393600463867,
      "activations/layer14_attention_weight_max": 41.49602127075195,
      "activations/layer14_attention_weight_min": -36.158050537109375,
      "activations/layer15_attention_weight_max": 36.6818962097168,
      "activations/layer15_attention_weight_min": -33.32892608642578,
      "activations/layer16_attention_weight_max": 30.038827896118164,
      "activations/layer16_attention_weight_min": -27.230308532714844,
      "activations/layer17_attention_weight_max": 30.47817611694336,
      "activations/layer17_attention_weight_min": -25.775304794311523,
      "activations/layer18_attention_weight_max": 30.293739318847656,
      "activations/layer18_attention_weight_min": -24.135534286499023,
      "activations/layer19_attention_weight_max": 36.475669860839844,
      "activations/layer19_attention_weight_min": -31.805891036987305,
      "activations/layer1_attention_weight_max": 16.090476989746094,
      "activations/layer1_attention_weight_min": -14.849357604980469,
      "activations/layer20_attention_weight_max": 30.09402084350586,
      "activations/layer20_attention_weight_min": -25.744606018066406,
      "activations/layer21_attention_weight_max": 27.98966407775879,
      "activations/layer21_attention_weight_min": -26.126909255981445,
      "activations/layer22_attention_weight_max": 46.762882232666016,
      "activations/layer22_attention_weight_min": -30.9239501953125,
      "activations/layer23_attention_weight_max": 34.19944763183594,
      "activations/layer23_attention_weight_min": -25.60196304321289,
      "activations/layer2_attention_weight_max": 30.291723251342773,
      "activations/layer2_attention_weight_min": -32.67836380004883,
      "activations/layer3_attention_weight_max": 92.62545013427734,
      "activations/layer3_attention_weight_min": -93.35049438476562,
      "activations/layer4_attention_weight_max": 106.07242584228516,
      "activations/layer4_attention_weight_min": -109.45924377441406,
      "activations/layer5_attention_weight_max": 51.45497131347656,
      "activations/layer5_attention_weight_min": -61.43226623535156,
      "activations/layer6_attention_weight_max": 42.3592414855957,
      "activations/layer6_attention_weight_min": -45.38638687133789,
      "activations/layer7_attention_weight_max": 85.00517272949219,
      "activations/layer7_attention_weight_min": -84.59515380859375,
      "activations/layer8_attention_weight_max": 37.950016021728516,
      "activations/layer8_attention_weight_min": -39.24964141845703,
      "activations/layer9_attention_weight_max": 31.557050704956055,
      "activations/layer9_attention_weight_min": -31.684619903564453,
      "epoch": 12.24,
      "learning_rate": 7.17901515151515e-05,
      "loss": 2.7835,
      "step": 210650
    },
    {
      "activations/layer0_attention_weight_max": 15.969219207763672,
      "activations/layer0_attention_weight_min": -12.015518188476562,
      "activations/layer10_attention_weight_max": 33.332271575927734,
      "activations/layer10_attention_weight_min": -31.966611862182617,
      "activations/layer11_attention_weight_max": 34.278297424316406,
      "activations/layer11_attention_weight_min": -31.493896484375,
      "activations/layer12_attention_weight_max": 26.200321197509766,
      "activations/layer12_attention_weight_min": -24.373273849487305,
      "activations/layer13_attention_weight_max": 38.65011215209961,
      "activations/layer13_attention_weight_min": -33.68258285522461,
      "activations/layer14_attention_weight_max": 46.54135513305664,
      "activations/layer14_attention_weight_min": -37.49774932861328,
      "activations/layer15_attention_weight_max": 34.13772964477539,
      "activations/layer15_attention_weight_min": -31.520732879638672,
      "activations/layer16_attention_weight_max": 33.98539352416992,
      "activations/layer16_attention_weight_min": -28.103437423706055,
      "activations/layer17_attention_weight_max": 30.839521408081055,
      "activations/layer17_attention_weight_min": -25.763975143432617,
      "activations/layer18_attention_weight_max": 29.14271354675293,
      "activations/layer18_attention_weight_min": -25.07058334350586,
      "activations/layer19_attention_weight_max": 33.94388198852539,
      "activations/layer19_attention_weight_min": -31.994619369506836,
      "activations/layer1_attention_weight_max": 17.083354949951172,
      "activations/layer1_attention_weight_min": -15.210104942321777,
      "activations/layer20_attention_weight_max": 29.018835067749023,
      "activations/layer20_attention_weight_min": -25.52863121032715,
      "activations/layer21_attention_weight_max": 27.001663208007812,
      "activations/layer21_attention_weight_min": -25.312786102294922,
      "activations/layer22_attention_weight_max": 38.657188415527344,
      "activations/layer22_attention_weight_min": -30.274433135986328,
      "activations/layer23_attention_weight_max": 33.07093048095703,
      "activations/layer23_attention_weight_min": -25.294227600097656,
      "activations/layer2_attention_weight_max": 32.838340759277344,
      "activations/layer2_attention_weight_min": -35.34769821166992,
      "activations/layer3_attention_weight_max": 100.267333984375,
      "activations/layer3_attention_weight_min": -97.67533111572266,
      "activations/layer4_attention_weight_max": 114.2729263305664,
      "activations/layer4_attention_weight_min": -113.25660705566406,
      "activations/layer5_attention_weight_max": 52.675201416015625,
      "activations/layer5_attention_weight_min": -61.996543884277344,
      "activations/layer6_attention_weight_max": 46.123905181884766,
      "activations/layer6_attention_weight_min": -47.40055465698242,
      "activations/layer7_attention_weight_max": 90.15328216552734,
      "activations/layer7_attention_weight_min": -92.39358520507812,
      "activations/layer8_attention_weight_max": 44.54399108886719,
      "activations/layer8_attention_weight_min": -43.14943313598633,
      "activations/layer9_attention_weight_max": 32.104312896728516,
      "activations/layer9_attention_weight_min": -32.56523132324219,
      "epoch": 12.24,
      "learning_rate": 7.177121212121212e-05,
      "loss": 2.7712,
      "step": 210700
    },
    {
      "activations/layer0_attention_weight_max": 16.217493057250977,
      "activations/layer0_attention_weight_min": -12.157352447509766,
      "activations/layer10_attention_weight_max": 32.333702087402344,
      "activations/layer10_attention_weight_min": -31.915817260742188,
      "activations/layer11_attention_weight_max": 31.05525779724121,
      "activations/layer11_attention_weight_min": -31.640867233276367,
      "activations/layer12_attention_weight_max": 23.463579177856445,
      "activations/layer12_attention_weight_min": -25.380950927734375,
      "activations/layer13_attention_weight_max": 36.29655075073242,
      "activations/layer13_attention_weight_min": -34.26068115234375,
      "activations/layer14_attention_weight_max": 40.9570198059082,
      "activations/layer14_attention_weight_min": -36.39691162109375,
      "activations/layer15_attention_weight_max": 37.63789749145508,
      "activations/layer15_attention_weight_min": -32.528602600097656,
      "activations/layer16_attention_weight_max": 32.61577606201172,
      "activations/layer16_attention_weight_min": -28.627891540527344,
      "activations/layer17_attention_weight_max": 33.91389083862305,
      "activations/layer17_attention_weight_min": -27.041126251220703,
      "activations/layer18_attention_weight_max": 35.3614387512207,
      "activations/layer18_attention_weight_min": -22.853694915771484,
      "activations/layer19_attention_weight_max": 36.36922836303711,
      "activations/layer19_attention_weight_min": -34.42397689819336,
      "activations/layer1_attention_weight_max": 16.084945678710938,
      "activations/layer1_attention_weight_min": -14.697288513183594,
      "activations/layer20_attention_weight_max": 28.196361541748047,
      "activations/layer20_attention_weight_min": -24.3044490814209,
      "activations/layer21_attention_weight_max": 26.905405044555664,
      "activations/layer21_attention_weight_min": -24.041881561279297,
      "activations/layer22_attention_weight_max": 38.2415771484375,
      "activations/layer22_attention_weight_min": -30.45198631286621,
      "activations/layer23_attention_weight_max": 35.65799331665039,
      "activations/layer23_attention_weight_min": -25.588842391967773,
      "activations/layer2_attention_weight_max": 32.85457992553711,
      "activations/layer2_attention_weight_min": -31.664688110351562,
      "activations/layer3_attention_weight_max": 91.76814270019531,
      "activations/layer3_attention_weight_min": -94.52062225341797,
      "activations/layer4_attention_weight_max": 109.53765869140625,
      "activations/layer4_attention_weight_min": -109.91222381591797,
      "activations/layer5_attention_weight_max": 52.44798278808594,
      "activations/layer5_attention_weight_min": -61.397216796875,
      "activations/layer6_attention_weight_max": 42.929237365722656,
      "activations/layer6_attention_weight_min": -46.205299377441406,
      "activations/layer7_attention_weight_max": 89.40718078613281,
      "activations/layer7_attention_weight_min": -86.5110855102539,
      "activations/layer8_attention_weight_max": 39.02005386352539,
      "activations/layer8_attention_weight_min": -39.5950813293457,
      "activations/layer9_attention_weight_max": 33.215553283691406,
      "activations/layer9_attention_weight_min": -33.26188278198242,
      "epoch": 12.25,
      "learning_rate": 7.175227272727272e-05,
      "loss": 2.7807,
      "step": 210750
    },
    {
      "activations/layer0_attention_weight_max": 16.671768188476562,
      "activations/layer0_attention_weight_min": -11.997443199157715,
      "activations/layer10_attention_weight_max": 31.76706314086914,
      "activations/layer10_attention_weight_min": -31.918052673339844,
      "activations/layer11_attention_weight_max": 30.918546676635742,
      "activations/layer11_attention_weight_min": -31.36986541748047,
      "activations/layer12_attention_weight_max": 25.409204483032227,
      "activations/layer12_attention_weight_min": -24.71975326538086,
      "activations/layer13_attention_weight_max": 40.45835876464844,
      "activations/layer13_attention_weight_min": -33.25897216796875,
      "activations/layer14_attention_weight_max": 44.37525939941406,
      "activations/layer14_attention_weight_min": -35.96187973022461,
      "activations/layer15_attention_weight_max": 37.50471878051758,
      "activations/layer15_attention_weight_min": -31.12139892578125,
      "activations/layer16_attention_weight_max": 31.136856079101562,
      "activations/layer16_attention_weight_min": -30.574954986572266,
      "activations/layer17_attention_weight_max": 29.658437728881836,
      "activations/layer17_attention_weight_min": -25.2539119720459,
      "activations/layer18_attention_weight_max": 30.68696403503418,
      "activations/layer18_attention_weight_min": -23.95503807067871,
      "activations/layer19_attention_weight_max": 36.484432220458984,
      "activations/layer19_attention_weight_min": -32.045597076416016,
      "activations/layer1_attention_weight_max": 16.79500961303711,
      "activations/layer1_attention_weight_min": -14.582271575927734,
      "activations/layer20_attention_weight_max": 32.05758285522461,
      "activations/layer20_attention_weight_min": -24.6931209564209,
      "activations/layer21_attention_weight_max": 28.773040771484375,
      "activations/layer21_attention_weight_min": -24.487089157104492,
      "activations/layer22_attention_weight_max": 46.58094787597656,
      "activations/layer22_attention_weight_min": -31.421039581298828,
      "activations/layer23_attention_weight_max": 34.64441680908203,
      "activations/layer23_attention_weight_min": -26.230144500732422,
      "activations/layer2_attention_weight_max": 32.92755126953125,
      "activations/layer2_attention_weight_min": -31.46091079711914,
      "activations/layer3_attention_weight_max": 95.43830871582031,
      "activations/layer3_attention_weight_min": -93.97540283203125,
      "activations/layer4_attention_weight_max": 108.49907684326172,
      "activations/layer4_attention_weight_min": -113.3600845336914,
      "activations/layer5_attention_weight_max": 52.66494369506836,
      "activations/layer5_attention_weight_min": -61.63370895385742,
      "activations/layer6_attention_weight_max": 43.431270599365234,
      "activations/layer6_attention_weight_min": -48.00704574584961,
      "activations/layer7_attention_weight_max": 89.94233703613281,
      "activations/layer7_attention_weight_min": -90.53713989257812,
      "activations/layer8_attention_weight_max": 39.85731887817383,
      "activations/layer8_attention_weight_min": -39.68385696411133,
      "activations/layer9_attention_weight_max": 30.84531593322754,
      "activations/layer9_attention_weight_min": -31.650495529174805,
      "epoch": 12.25,
      "learning_rate": 7.173333333333332e-05,
      "loss": 2.7699,
      "step": 210800
    },
    {
      "activations/layer0_attention_weight_max": 16.271564483642578,
      "activations/layer0_attention_weight_min": -12.276845932006836,
      "activations/layer10_attention_weight_max": 32.78973388671875,
      "activations/layer10_attention_weight_min": -30.32152557373047,
      "activations/layer11_attention_weight_max": 31.07190704345703,
      "activations/layer11_attention_weight_min": -30.563945770263672,
      "activations/layer12_attention_weight_max": 24.020877838134766,
      "activations/layer12_attention_weight_min": -25.18354034423828,
      "activations/layer13_attention_weight_max": 37.82365798950195,
      "activations/layer13_attention_weight_min": -34.93815612792969,
      "activations/layer14_attention_weight_max": 47.40362548828125,
      "activations/layer14_attention_weight_min": -38.33231735229492,
      "activations/layer15_attention_weight_max": 34.123409271240234,
      "activations/layer15_attention_weight_min": -31.158580780029297,
      "activations/layer16_attention_weight_max": 28.592594146728516,
      "activations/layer16_attention_weight_min": -27.2423038482666,
      "activations/layer17_attention_weight_max": 29.26225471496582,
      "activations/layer17_attention_weight_min": -24.698848724365234,
      "activations/layer18_attention_weight_max": 31.107561111450195,
      "activations/layer18_attention_weight_min": -23.748655319213867,
      "activations/layer19_attention_weight_max": 38.59868240356445,
      "activations/layer19_attention_weight_min": -29.14850425720215,
      "activations/layer1_attention_weight_max": 15.883684158325195,
      "activations/layer1_attention_weight_min": -15.067089080810547,
      "activations/layer20_attention_weight_max": 30.04706382751465,
      "activations/layer20_attention_weight_min": -23.888025283813477,
      "activations/layer21_attention_weight_max": 28.184484481811523,
      "activations/layer21_attention_weight_min": -22.59456443786621,
      "activations/layer22_attention_weight_max": 40.59029769897461,
      "activations/layer22_attention_weight_min": -28.488510131835938,
      "activations/layer23_attention_weight_max": 35.919677734375,
      "activations/layer23_attention_weight_min": -25.210599899291992,
      "activations/layer2_attention_weight_max": 33.79050827026367,
      "activations/layer2_attention_weight_min": -33.50701904296875,
      "activations/layer3_attention_weight_max": 98.87543487548828,
      "activations/layer3_attention_weight_min": -98.30567169189453,
      "activations/layer4_attention_weight_max": 109.49666595458984,
      "activations/layer4_attention_weight_min": -110.6815414428711,
      "activations/layer5_attention_weight_max": 51.94075012207031,
      "activations/layer5_attention_weight_min": -62.27533721923828,
      "activations/layer6_attention_weight_max": 43.8689079284668,
      "activations/layer6_attention_weight_min": -45.35013961791992,
      "activations/layer7_attention_weight_max": 94.29949951171875,
      "activations/layer7_attention_weight_min": -84.43107604980469,
      "activations/layer8_attention_weight_max": 39.71230697631836,
      "activations/layer8_attention_weight_min": -40.60198974609375,
      "activations/layer9_attention_weight_max": 32.05067443847656,
      "activations/layer9_attention_weight_min": -32.623008728027344,
      "epoch": 12.25,
      "learning_rate": 7.171477272727272e-05,
      "loss": 2.757,
      "step": 210850
    },
    {
      "activations/layer0_attention_weight_max": 16.221851348876953,
      "activations/layer0_attention_weight_min": -12.693123817443848,
      "activations/layer10_attention_weight_max": 31.868331909179688,
      "activations/layer10_attention_weight_min": -33.82707977294922,
      "activations/layer11_attention_weight_max": 30.79430389404297,
      "activations/layer11_attention_weight_min": -32.549888610839844,
      "activations/layer12_attention_weight_max": 23.05371856689453,
      "activations/layer12_attention_weight_min": -24.066421508789062,
      "activations/layer13_attention_weight_max": 35.64337158203125,
      "activations/layer13_attention_weight_min": -33.316471099853516,
      "activations/layer14_attention_weight_max": 38.45267105102539,
      "activations/layer14_attention_weight_min": -36.7530632019043,
      "activations/layer15_attention_weight_max": 32.93089294433594,
      "activations/layer15_attention_weight_min": -31.942914962768555,
      "activations/layer16_attention_weight_max": 30.449474334716797,
      "activations/layer16_attention_weight_min": -27.87262535095215,
      "activations/layer17_attention_weight_max": 28.433618545532227,
      "activations/layer17_attention_weight_min": -25.07117462158203,
      "activations/layer18_attention_weight_max": 29.309053421020508,
      "activations/layer18_attention_weight_min": -24.5737361907959,
      "activations/layer19_attention_weight_max": 31.440673828125,
      "activations/layer19_attention_weight_min": -32.66609191894531,
      "activations/layer1_attention_weight_max": 15.500944137573242,
      "activations/layer1_attention_weight_min": -13.76032829284668,
      "activations/layer20_attention_weight_max": 27.461746215820312,
      "activations/layer20_attention_weight_min": -26.69524383544922,
      "activations/layer21_attention_weight_max": 29.649978637695312,
      "activations/layer21_attention_weight_min": -24.378080368041992,
      "activations/layer22_attention_weight_max": 41.05348205566406,
      "activations/layer22_attention_weight_min": -30.508468627929688,
      "activations/layer23_attention_weight_max": 32.324275970458984,
      "activations/layer23_attention_weight_min": -25.99956512451172,
      "activations/layer2_attention_weight_max": 32.716941833496094,
      "activations/layer2_attention_weight_min": -31.22225570678711,
      "activations/layer3_attention_weight_max": 92.39408111572266,
      "activations/layer3_attention_weight_min": -96.9122314453125,
      "activations/layer4_attention_weight_max": 108.34329986572266,
      "activations/layer4_attention_weight_min": -105.08290100097656,
      "activations/layer5_attention_weight_max": 48.70915222167969,
      "activations/layer5_attention_weight_min": -61.34377670288086,
      "activations/layer6_attention_weight_max": 44.47439193725586,
      "activations/layer6_attention_weight_min": -49.14069366455078,
      "activations/layer7_attention_weight_max": 86.58512115478516,
      "activations/layer7_attention_weight_min": -89.97941589355469,
      "activations/layer8_attention_weight_max": 37.801143646240234,
      "activations/layer8_attention_weight_min": -40.72142791748047,
      "activations/layer9_attention_weight_max": 30.46860694885254,
      "activations/layer9_attention_weight_min": -33.43128967285156,
      "epoch": 12.25,
      "learning_rate": 7.169583333333333e-05,
      "loss": 2.7678,
      "step": 210900
    },
    {
      "activations/layer0_attention_weight_max": 14.970191955566406,
      "activations/layer0_attention_weight_min": -12.893351554870605,
      "activations/layer10_attention_weight_max": 33.06160354614258,
      "activations/layer10_attention_weight_min": -32.917030334472656,
      "activations/layer11_attention_weight_max": 35.455142974853516,
      "activations/layer11_attention_weight_min": -33.356109619140625,
      "activations/layer12_attention_weight_max": 25.96031951904297,
      "activations/layer12_attention_weight_min": -27.015338897705078,
      "activations/layer13_attention_weight_max": 40.34412384033203,
      "activations/layer13_attention_weight_min": -35.39147186279297,
      "activations/layer14_attention_weight_max": 46.97108459472656,
      "activations/layer14_attention_weight_min": -40.184913635253906,
      "activations/layer15_attention_weight_max": 38.851402282714844,
      "activations/layer15_attention_weight_min": -34.162872314453125,
      "activations/layer16_attention_weight_max": 32.84732437133789,
      "activations/layer16_attention_weight_min": -28.342805862426758,
      "activations/layer17_attention_weight_max": 32.664329528808594,
      "activations/layer17_attention_weight_min": -26.10774803161621,
      "activations/layer18_attention_weight_max": 33.44621276855469,
      "activations/layer18_attention_weight_min": -23.966493606567383,
      "activations/layer19_attention_weight_max": 39.784847259521484,
      "activations/layer19_attention_weight_min": -33.6982536315918,
      "activations/layer1_attention_weight_max": 16.01688575744629,
      "activations/layer1_attention_weight_min": -14.444573402404785,
      "activations/layer20_attention_weight_max": 32.80577850341797,
      "activations/layer20_attention_weight_min": -26.43444061279297,
      "activations/layer21_attention_weight_max": 31.355697631835938,
      "activations/layer21_attention_weight_min": -24.92650032043457,
      "activations/layer22_attention_weight_max": 50.11470413208008,
      "activations/layer22_attention_weight_min": -31.22138786315918,
      "activations/layer23_attention_weight_max": 36.80183410644531,
      "activations/layer23_attention_weight_min": -26.341747283935547,
      "activations/layer2_attention_weight_max": 33.23481750488281,
      "activations/layer2_attention_weight_min": -33.04426574707031,
      "activations/layer3_attention_weight_max": 96.11000061035156,
      "activations/layer3_attention_weight_min": -99.01214599609375,
      "activations/layer4_attention_weight_max": 110.9647216796875,
      "activations/layer4_attention_weight_min": -113.27044677734375,
      "activations/layer5_attention_weight_max": 51.68769073486328,
      "activations/layer5_attention_weight_min": -60.60737228393555,
      "activations/layer6_attention_weight_max": 44.33765411376953,
      "activations/layer6_attention_weight_min": -50.19273376464844,
      "activations/layer7_attention_weight_max": 88.4624252319336,
      "activations/layer7_attention_weight_min": -95.05320739746094,
      "activations/layer8_attention_weight_max": 42.06083297729492,
      "activations/layer8_attention_weight_min": -42.93503189086914,
      "activations/layer9_attention_weight_max": 36.141841888427734,
      "activations/layer9_attention_weight_min": -33.06193923950195,
      "epoch": 12.26,
      "learning_rate": 7.167689393939394e-05,
      "loss": 2.7389,
      "step": 210950
    },
    {
      "activations/layer0_attention_weight_max": 14.903407096862793,
      "activations/layer0_attention_weight_min": -12.084354400634766,
      "activations/layer10_attention_weight_max": 44.41370391845703,
      "activations/layer10_attention_weight_min": -40.356422424316406,
      "activations/layer11_attention_weight_max": 43.97454833984375,
      "activations/layer11_attention_weight_min": -43.694976806640625,
      "activations/layer12_attention_weight_max": 26.301597595214844,
      "activations/layer12_attention_weight_min": -30.00110626220703,
      "activations/layer13_attention_weight_max": 36.62842559814453,
      "activations/layer13_attention_weight_min": -32.23641586303711,
      "activations/layer14_attention_weight_max": 42.94629669189453,
      "activations/layer14_attention_weight_min": -38.457237243652344,
      "activations/layer15_attention_weight_max": 36.533935546875,
      "activations/layer15_attention_weight_min": -33.06959915161133,
      "activations/layer16_attention_weight_max": 29.657983779907227,
      "activations/layer16_attention_weight_min": -29.776933670043945,
      "activations/layer17_attention_weight_max": 31.425506591796875,
      "activations/layer17_attention_weight_min": -26.130260467529297,
      "activations/layer18_attention_weight_max": 31.77000617980957,
      "activations/layer18_attention_weight_min": -22.768753051757812,
      "activations/layer19_attention_weight_max": 33.62516403198242,
      "activations/layer19_attention_weight_min": -31.723596572875977,
      "activations/layer1_attention_weight_max": 16.51102066040039,
      "activations/layer1_attention_weight_min": -14.867334365844727,
      "activations/layer20_attention_weight_max": 27.00078010559082,
      "activations/layer20_attention_weight_min": -26.512611389160156,
      "activations/layer21_attention_weight_max": 30.393407821655273,
      "activations/layer21_attention_weight_min": -26.48781394958496,
      "activations/layer22_attention_weight_max": 45.32106018066406,
      "activations/layer22_attention_weight_min": -32.7243537902832,
      "activations/layer23_attention_weight_max": 31.55767059326172,
      "activations/layer23_attention_weight_min": -25.21900177001953,
      "activations/layer2_attention_weight_max": 34.39667892456055,
      "activations/layer2_attention_weight_min": -35.91655731201172,
      "activations/layer3_attention_weight_max": 99.67853546142578,
      "activations/layer3_attention_weight_min": -101.21691131591797,
      "activations/layer4_attention_weight_max": 121.71697235107422,
      "activations/layer4_attention_weight_min": -116.47236633300781,
      "activations/layer5_attention_weight_max": 61.38064193725586,
      "activations/layer5_attention_weight_min": -64.39990234375,
      "activations/layer6_attention_weight_max": 51.58604431152344,
      "activations/layer6_attention_weight_min": -50.6147575378418,
      "activations/layer7_attention_weight_max": 112.67730712890625,
      "activations/layer7_attention_weight_min": -106.00362396240234,
      "activations/layer8_attention_weight_max": 54.77128601074219,
      "activations/layer8_attention_weight_min": -47.7088508605957,
      "activations/layer9_attention_weight_max": 39.66061019897461,
      "activations/layer9_attention_weight_min": -40.23107147216797,
      "epoch": 12.26,
      "learning_rate": 7.165795454545454e-05,
      "loss": 2.766,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4265,
      "eval_samples_per_second": 509.583,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4265,
      "eval_openwebtext_samples_per_second": 509.583,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9347,
      "eval_wikitext_samples_per_second": 235.699,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.5211,
      "eval_lambada_samples_per_second": 511.39,
      "step": 211000
    },
    {
      "activations/layer0_attention_weight_max": 15.095996856689453,
      "activations/layer0_attention_weight_min": -11.87448501586914,
      "activations/layer10_attention_weight_max": 35.8182373046875,
      "activations/layer10_attention_weight_min": -33.738494873046875,
      "activations/layer11_attention_weight_max": 34.5904426574707,
      "activations/layer11_attention_weight_min": -34.185096740722656,
      "activations/layer12_attention_weight_max": 32.900489807128906,
      "activations/layer12_attention_weight_min": -25.8912410736084,
      "activations/layer13_attention_weight_max": 47.21158981323242,
      "activations/layer13_attention_weight_min": -36.122459411621094,
      "activations/layer14_attention_weight_max": 46.60007858276367,
      "activations/layer14_attention_weight_min": -40.7287483215332,
      "activations/layer15_attention_weight_max": 41.48465347290039,
      "activations/layer15_attention_weight_min": -32.345829010009766,
      "activations/layer16_attention_weight_max": 36.211421966552734,
      "activations/layer16_attention_weight_min": -28.41718864440918,
      "activations/layer17_attention_weight_max": 41.97474670410156,
      "activations/layer17_attention_weight_min": -26.565908432006836,
      "activations/layer18_attention_weight_max": 36.87279510498047,
      "activations/layer18_attention_weight_min": -25.48351287841797,
      "activations/layer19_attention_weight_max": 35.27067947387695,
      "activations/layer19_attention_weight_min": -31.632047653198242,
      "activations/layer1_attention_weight_max": 15.669516563415527,
      "activations/layer1_attention_weight_min": -13.838496208190918,
      "activations/layer20_attention_weight_max": 33.14973831176758,
      "activations/layer20_attention_weight_min": -26.41707992553711,
      "activations/layer21_attention_weight_max": 30.496061325073242,
      "activations/layer21_attention_weight_min": -26.37558937072754,
      "activations/layer22_attention_weight_max": 51.111473083496094,
      "activations/layer22_attention_weight_min": -32.88153839111328,
      "activations/layer23_attention_weight_max": 36.817230224609375,
      "activations/layer23_attention_weight_min": -27.72616958618164,
      "activations/layer2_attention_weight_max": 31.54547119140625,
      "activations/layer2_attention_weight_min": -32.669334411621094,
      "activations/layer3_attention_weight_max": 92.67662811279297,
      "activations/layer3_attention_weight_min": -98.9586181640625,
      "activations/layer4_attention_weight_max": 113.99627685546875,
      "activations/layer4_attention_weight_min": -112.29345703125,
      "activations/layer5_attention_weight_max": 50.40492248535156,
      "activations/layer5_attention_weight_min": -59.4775505065918,
      "activations/layer6_attention_weight_max": 45.56685256958008,
      "activations/layer6_attention_weight_min": -47.975303649902344,
      "activations/layer7_attention_weight_max": 92.21334075927734,
      "activations/layer7_attention_weight_min": -87.95890808105469,
      "activations/layer8_attention_weight_max": 39.66054153442383,
      "activations/layer8_attention_weight_min": -41.72971725463867,
      "activations/layer9_attention_weight_max": 35.617919921875,
      "activations/layer9_attention_weight_min": -33.15793228149414,
      "epoch": 12.26,
      "learning_rate": 7.163901515151514e-05,
      "loss": 2.7551,
      "step": 211050
    },
    {
      "activations/layer0_attention_weight_max": 15.905619621276855,
      "activations/layer0_attention_weight_min": -12.833841323852539,
      "activations/layer10_attention_weight_max": 30.965200424194336,
      "activations/layer10_attention_weight_min": -31.304302215576172,
      "activations/layer11_attention_weight_max": 30.72966766357422,
      "activations/layer11_attention_weight_min": -32.381004333496094,
      "activations/layer12_attention_weight_max": 24.396203994750977,
      "activations/layer12_attention_weight_min": -23.980159759521484,
      "activations/layer13_attention_weight_max": 38.940670013427734,
      "activations/layer13_attention_weight_min": -34.18766403198242,
      "activations/layer14_attention_weight_max": 44.52336502075195,
      "activations/layer14_attention_weight_min": -36.575042724609375,
      "activations/layer15_attention_weight_max": 37.49571990966797,
      "activations/layer15_attention_weight_min": -32.476322174072266,
      "activations/layer16_attention_weight_max": 32.06581115722656,
      "activations/layer16_attention_weight_min": -28.964826583862305,
      "activations/layer17_attention_weight_max": 36.09671401977539,
      "activations/layer17_attention_weight_min": -24.79417610168457,
      "activations/layer18_attention_weight_max": 29.7509708404541,
      "activations/layer18_attention_weight_min": -23.492597579956055,
      "activations/layer19_attention_weight_max": 33.26386260986328,
      "activations/layer19_attention_weight_min": -30.443151473999023,
      "activations/layer1_attention_weight_max": 16.350650787353516,
      "activations/layer1_attention_weight_min": -15.89311408996582,
      "activations/layer20_attention_weight_max": 28.719703674316406,
      "activations/layer20_attention_weight_min": -24.694406509399414,
      "activations/layer21_attention_weight_max": 28.90720558166504,
      "activations/layer21_attention_weight_min": -24.273069381713867,
      "activations/layer22_attention_weight_max": 42.43858337402344,
      "activations/layer22_attention_weight_min": -30.02635955810547,
      "activations/layer23_attention_weight_max": 34.68962860107422,
      "activations/layer23_attention_weight_min": -27.598068237304688,
      "activations/layer2_attention_weight_max": 33.106807708740234,
      "activations/layer2_attention_weight_min": -32.9311408996582,
      "activations/layer3_attention_weight_max": 99.05787658691406,
      "activations/layer3_attention_weight_min": -94.55562591552734,
      "activations/layer4_attention_weight_max": 113.5911865234375,
      "activations/layer4_attention_weight_min": -116.0025634765625,
      "activations/layer5_attention_weight_max": 50.115169525146484,
      "activations/layer5_attention_weight_min": -61.30872344970703,
      "activations/layer6_attention_weight_max": 44.488258361816406,
      "activations/layer6_attention_weight_min": -48.58012008666992,
      "activations/layer7_attention_weight_max": 92.01947784423828,
      "activations/layer7_attention_weight_min": -90.50507354736328,
      "activations/layer8_attention_weight_max": 39.93207550048828,
      "activations/layer8_attention_weight_min": -41.63454818725586,
      "activations/layer9_attention_weight_max": 29.721906661987305,
      "activations/layer9_attention_weight_min": -31.153915405273438,
      "epoch": 12.27,
      "learning_rate": 7.162007575757576e-05,
      "loss": 2.7752,
      "step": 211100
    },
    {
      "activations/layer0_attention_weight_max": 16.696514129638672,
      "activations/layer0_attention_weight_min": -12.357458114624023,
      "activations/layer10_attention_weight_max": 30.203262329101562,
      "activations/layer10_attention_weight_min": -30.696569442749023,
      "activations/layer11_attention_weight_max": 29.83290672302246,
      "activations/layer11_attention_weight_min": -28.954631805419922,
      "activations/layer12_attention_weight_max": 23.118528366088867,
      "activations/layer12_attention_weight_min": -24.310096740722656,
      "activations/layer13_attention_weight_max": 37.62196350097656,
      "activations/layer13_attention_weight_min": -31.503543853759766,
      "activations/layer14_attention_weight_max": 41.983787536621094,
      "activations/layer14_attention_weight_min": -35.902584075927734,
      "activations/layer15_attention_weight_max": 34.10891342163086,
      "activations/layer15_attention_weight_min": -30.66761589050293,
      "activations/layer16_attention_weight_max": 31.570341110229492,
      "activations/layer16_attention_weight_min": -27.10370445251465,
      "activations/layer17_attention_weight_max": 30.97663688659668,
      "activations/layer17_attention_weight_min": -24.945497512817383,
      "activations/layer18_attention_weight_max": 29.785724639892578,
      "activations/layer18_attention_weight_min": -22.525609970092773,
      "activations/layer19_attention_weight_max": 34.03815841674805,
      "activations/layer19_attention_weight_min": -29.934091567993164,
      "activations/layer1_attention_weight_max": 16.640087127685547,
      "activations/layer1_attention_weight_min": -14.924179077148438,
      "activations/layer20_attention_weight_max": 27.559423446655273,
      "activations/layer20_attention_weight_min": -26.340734481811523,
      "activations/layer21_attention_weight_max": 26.119976043701172,
      "activations/layer21_attention_weight_min": -23.7021541595459,
      "activations/layer22_attention_weight_max": 39.83338928222656,
      "activations/layer22_attention_weight_min": -29.563602447509766,
      "activations/layer23_attention_weight_max": 34.62104034423828,
      "activations/layer23_attention_weight_min": -24.123586654663086,
      "activations/layer2_attention_weight_max": 32.84196090698242,
      "activations/layer2_attention_weight_min": -30.94955825805664,
      "activations/layer3_attention_weight_max": 96.09467315673828,
      "activations/layer3_attention_weight_min": -95.186767578125,
      "activations/layer4_attention_weight_max": 105.48158264160156,
      "activations/layer4_attention_weight_min": -109.0876235961914,
      "activations/layer5_attention_weight_max": 48.81169891357422,
      "activations/layer5_attention_weight_min": -59.9047966003418,
      "activations/layer6_attention_weight_max": 44.058956146240234,
      "activations/layer6_attention_weight_min": -46.205650329589844,
      "activations/layer7_attention_weight_max": 85.68836975097656,
      "activations/layer7_attention_weight_min": -87.63812255859375,
      "activations/layer8_attention_weight_max": 39.56247329711914,
      "activations/layer8_attention_weight_min": -38.674434661865234,
      "activations/layer9_attention_weight_max": 31.403610229492188,
      "activations/layer9_attention_weight_min": -31.41956901550293,
      "epoch": 12.27,
      "learning_rate": 7.160113636363636e-05,
      "loss": 2.7744,
      "step": 211150
    },
    {
      "activations/layer0_attention_weight_max": 16.13439178466797,
      "activations/layer0_attention_weight_min": -12.19101333618164,
      "activations/layer10_attention_weight_max": 33.097747802734375,
      "activations/layer10_attention_weight_min": -32.34706115722656,
      "activations/layer11_attention_weight_max": 32.832115173339844,
      "activations/layer11_attention_weight_min": -32.58866882324219,
      "activations/layer12_attention_weight_max": 25.798072814941406,
      "activations/layer12_attention_weight_min": -26.687171936035156,
      "activations/layer13_attention_weight_max": 39.81538391113281,
      "activations/layer13_attention_weight_min": -34.73565673828125,
      "activations/layer14_attention_weight_max": 44.58450698852539,
      "activations/layer14_attention_weight_min": -37.46200180053711,
      "activations/layer15_attention_weight_max": 34.75301742553711,
      "activations/layer15_attention_weight_min": -34.92502212524414,
      "activations/layer16_attention_weight_max": 30.494457244873047,
      "activations/layer16_attention_weight_min": -29.927326202392578,
      "activations/layer17_attention_weight_max": 31.27402687072754,
      "activations/layer17_attention_weight_min": -28.998870849609375,
      "activations/layer18_attention_weight_max": 30.160207748413086,
      "activations/layer18_attention_weight_min": -27.346403121948242,
      "activations/layer19_attention_weight_max": 34.27364730834961,
      "activations/layer19_attention_weight_min": -32.715476989746094,
      "activations/layer1_attention_weight_max": 16.196887969970703,
      "activations/layer1_attention_weight_min": -15.499942779541016,
      "activations/layer20_attention_weight_max": 29.596593856811523,
      "activations/layer20_attention_weight_min": -26.45693016052246,
      "activations/layer21_attention_weight_max": 30.16884422302246,
      "activations/layer21_attention_weight_min": -25.14068031311035,
      "activations/layer22_attention_weight_max": 44.93826675415039,
      "activations/layer22_attention_weight_min": -33.3780632019043,
      "activations/layer23_attention_weight_max": 34.55897903442383,
      "activations/layer23_attention_weight_min": -28.58563804626465,
      "activations/layer2_attention_weight_max": 32.493682861328125,
      "activations/layer2_attention_weight_min": -33.00595474243164,
      "activations/layer3_attention_weight_max": 96.50108337402344,
      "activations/layer3_attention_weight_min": -98.46372985839844,
      "activations/layer4_attention_weight_max": 111.95104217529297,
      "activations/layer4_attention_weight_min": -118.35749816894531,
      "activations/layer5_attention_weight_max": 55.0196647644043,
      "activations/layer5_attention_weight_min": -62.15979766845703,
      "activations/layer6_attention_weight_max": 44.319969177246094,
      "activations/layer6_attention_weight_min": -50.80569076538086,
      "activations/layer7_attention_weight_max": 89.83802032470703,
      "activations/layer7_attention_weight_min": -95.37287139892578,
      "activations/layer8_attention_weight_max": 40.422000885009766,
      "activations/layer8_attention_weight_min": -42.112648010253906,
      "activations/layer9_attention_weight_max": 32.40443420410156,
      "activations/layer9_attention_weight_min": -33.98588180541992,
      "epoch": 12.27,
      "learning_rate": 7.158219696969696e-05,
      "loss": 2.7727,
      "step": 211200
    },
    {
      "activations/layer0_attention_weight_max": 15.864493370056152,
      "activations/layer0_attention_weight_min": -12.270220756530762,
      "activations/layer10_attention_weight_max": 33.718475341796875,
      "activations/layer10_attention_weight_min": -32.7401237487793,
      "activations/layer11_attention_weight_max": 33.49655532836914,
      "activations/layer11_attention_weight_min": -31.35986328125,
      "activations/layer12_attention_weight_max": 26.708993911743164,
      "activations/layer12_attention_weight_min": -24.95138168334961,
      "activations/layer13_attention_weight_max": 39.338069915771484,
      "activations/layer13_attention_weight_min": -35.48859405517578,
      "activations/layer14_attention_weight_max": 45.020286560058594,
      "activations/layer14_attention_weight_min": -38.904273986816406,
      "activations/layer15_attention_weight_max": 39.73642349243164,
      "activations/layer15_attention_weight_min": -33.22669982910156,
      "activations/layer16_attention_weight_max": 34.70759582519531,
      "activations/layer16_attention_weight_min": -27.7686767578125,
      "activations/layer17_attention_weight_max": 30.29099464416504,
      "activations/layer17_attention_weight_min": -25.134817123413086,
      "activations/layer18_attention_weight_max": 31.234472274780273,
      "activations/layer18_attention_weight_min": -22.560802459716797,
      "activations/layer19_attention_weight_max": 33.1325798034668,
      "activations/layer19_attention_weight_min": -29.93525505065918,
      "activations/layer1_attention_weight_max": 16.296018600463867,
      "activations/layer1_attention_weight_min": -14.691168785095215,
      "activations/layer20_attention_weight_max": 28.743162155151367,
      "activations/layer20_attention_weight_min": -24.264436721801758,
      "activations/layer21_attention_weight_max": 27.163028717041016,
      "activations/layer21_attention_weight_min": -22.5219669342041,
      "activations/layer22_attention_weight_max": 40.586910247802734,
      "activations/layer22_attention_weight_min": -27.43953514099121,
      "activations/layer23_attention_weight_max": 31.6633358001709,
      "activations/layer23_attention_weight_min": -26.615888595581055,
      "activations/layer2_attention_weight_max": 32.63616943359375,
      "activations/layer2_attention_weight_min": -32.10871887207031,
      "activations/layer3_attention_weight_max": 98.03532409667969,
      "activations/layer3_attention_weight_min": -96.63958740234375,
      "activations/layer4_attention_weight_max": 112.18111419677734,
      "activations/layer4_attention_weight_min": -112.40477752685547,
      "activations/layer5_attention_weight_max": 54.45429229736328,
      "activations/layer5_attention_weight_min": -60.29446029663086,
      "activations/layer6_attention_weight_max": 43.382652282714844,
      "activations/layer6_attention_weight_min": -48.95079803466797,
      "activations/layer7_attention_weight_max": 91.6709976196289,
      "activations/layer7_attention_weight_min": -97.10971069335938,
      "activations/layer8_attention_weight_max": 37.827457427978516,
      "activations/layer8_attention_weight_min": -41.914756774902344,
      "activations/layer9_attention_weight_max": 31.860837936401367,
      "activations/layer9_attention_weight_min": -34.85865783691406,
      "epoch": 12.27,
      "learning_rate": 7.156325757575756e-05,
      "loss": 2.7648,
      "step": 211250
    },
    {
      "activations/layer0_attention_weight_max": 17.012218475341797,
      "activations/layer0_attention_weight_min": -12.24757194519043,
      "activations/layer10_attention_weight_max": 33.347328186035156,
      "activations/layer10_attention_weight_min": -34.69300842285156,
      "activations/layer11_attention_weight_max": 34.776084899902344,
      "activations/layer11_attention_weight_min": -32.44310760498047,
      "activations/layer12_attention_weight_max": 33.83617401123047,
      "activations/layer12_attention_weight_min": -25.73966407775879,
      "activations/layer13_attention_weight_max": 47.15745544433594,
      "activations/layer13_attention_weight_min": -34.50481414794922,
      "activations/layer14_attention_weight_max": 53.292232513427734,
      "activations/layer14_attention_weight_min": -39.65541458129883,
      "activations/layer15_attention_weight_max": 43.943153381347656,
      "activations/layer15_attention_weight_min": -33.99681091308594,
      "activations/layer16_attention_weight_max": 35.9323844909668,
      "activations/layer16_attention_weight_min": -27.131895065307617,
      "activations/layer17_attention_weight_max": 34.004112243652344,
      "activations/layer17_attention_weight_min": -26.960721969604492,
      "activations/layer18_attention_weight_max": 38.277000427246094,
      "activations/layer18_attention_weight_min": -24.623512268066406,
      "activations/layer19_attention_weight_max": 43.593360900878906,
      "activations/layer19_attention_weight_min": -31.772546768188477,
      "activations/layer1_attention_weight_max": 16.8113956451416,
      "activations/layer1_attention_weight_min": -16.698488235473633,
      "activations/layer20_attention_weight_max": 34.15531539916992,
      "activations/layer20_attention_weight_min": -26.01141929626465,
      "activations/layer21_attention_weight_max": 34.46921920776367,
      "activations/layer21_attention_weight_min": -24.626285552978516,
      "activations/layer22_attention_weight_max": 60.497650146484375,
      "activations/layer22_attention_weight_min": -32.56492614746094,
      "activations/layer23_attention_weight_max": 43.66987228393555,
      "activations/layer23_attention_weight_min": -26.955350875854492,
      "activations/layer2_attention_weight_max": 34.14822006225586,
      "activations/layer2_attention_weight_min": -32.952388763427734,
      "activations/layer3_attention_weight_max": 97.99748229980469,
      "activations/layer3_attention_weight_min": -100.73466491699219,
      "activations/layer4_attention_weight_max": 108.9261703491211,
      "activations/layer4_attention_weight_min": -109.10870361328125,
      "activations/layer5_attention_weight_max": 52.654457092285156,
      "activations/layer5_attention_weight_min": -63.89213562011719,
      "activations/layer6_attention_weight_max": 45.54859161376953,
      "activations/layer6_attention_weight_min": -51.23902893066406,
      "activations/layer7_attention_weight_max": 88.53003692626953,
      "activations/layer7_attention_weight_min": -94.72950744628906,
      "activations/layer8_attention_weight_max": 42.205284118652344,
      "activations/layer8_attention_weight_min": -43.3629035949707,
      "activations/layer9_attention_weight_max": 34.6308708190918,
      "activations/layer9_attention_weight_min": -35.50751876831055,
      "epoch": 12.28,
      "learning_rate": 7.154431818181818e-05,
      "loss": 2.7635,
      "step": 211300
    },
    {
      "activations/layer0_attention_weight_max": 16.138572692871094,
      "activations/layer0_attention_weight_min": -11.623065948486328,
      "activations/layer10_attention_weight_max": 31.47036361694336,
      "activations/layer10_attention_weight_min": -32.20294952392578,
      "activations/layer11_attention_weight_max": 31.671600341796875,
      "activations/layer11_attention_weight_min": -31.05304718017578,
      "activations/layer12_attention_weight_max": 26.803911209106445,
      "activations/layer12_attention_weight_min": -25.32001495361328,
      "activations/layer13_attention_weight_max": 40.30937957763672,
      "activations/layer13_attention_weight_min": -35.26789093017578,
      "activations/layer14_attention_weight_max": 40.802284240722656,
      "activations/layer14_attention_weight_min": -36.81710433959961,
      "activations/layer15_attention_weight_max": 34.25688171386719,
      "activations/layer15_attention_weight_min": -31.650117874145508,
      "activations/layer16_attention_weight_max": 29.677448272705078,
      "activations/layer16_attention_weight_min": -26.25482940673828,
      "activations/layer17_attention_weight_max": 31.89968490600586,
      "activations/layer17_attention_weight_min": -24.114492416381836,
      "activations/layer18_attention_weight_max": 31.816261291503906,
      "activations/layer18_attention_weight_min": -22.875062942504883,
      "activations/layer19_attention_weight_max": 36.40525817871094,
      "activations/layer19_attention_weight_min": -29.03763198852539,
      "activations/layer1_attention_weight_max": 18.208232879638672,
      "activations/layer1_attention_weight_min": -15.067172050476074,
      "activations/layer20_attention_weight_max": 31.660381317138672,
      "activations/layer20_attention_weight_min": -23.788253784179688,
      "activations/layer21_attention_weight_max": 28.37552833557129,
      "activations/layer21_attention_weight_min": -23.266061782836914,
      "activations/layer22_attention_weight_max": 42.81254959106445,
      "activations/layer22_attention_weight_min": -28.152299880981445,
      "activations/layer23_attention_weight_max": 36.07933044433594,
      "activations/layer23_attention_weight_min": -23.51234245300293,
      "activations/layer2_attention_weight_max": 33.81325149536133,
      "activations/layer2_attention_weight_min": -33.94532012939453,
      "activations/layer3_attention_weight_max": 94.90604400634766,
      "activations/layer3_attention_weight_min": -95.62355041503906,
      "activations/layer4_attention_weight_max": 109.1458740234375,
      "activations/layer4_attention_weight_min": -112.09908294677734,
      "activations/layer5_attention_weight_max": 48.95782470703125,
      "activations/layer5_attention_weight_min": -64.58814239501953,
      "activations/layer6_attention_weight_max": 42.79218292236328,
      "activations/layer6_attention_weight_min": -49.21464538574219,
      "activations/layer7_attention_weight_max": 85.91606140136719,
      "activations/layer7_attention_weight_min": -96.02897644042969,
      "activations/layer8_attention_weight_max": 42.913719177246094,
      "activations/layer8_attention_weight_min": -40.45159912109375,
      "activations/layer9_attention_weight_max": 34.00146484375,
      "activations/layer9_attention_weight_min": -32.60585403442383,
      "epoch": 12.28,
      "learning_rate": 7.152537878787878e-05,
      "loss": 2.7722,
      "step": 211350
    },
    {
      "activations/layer0_attention_weight_max": 16.154714584350586,
      "activations/layer0_attention_weight_min": -12.083999633789062,
      "activations/layer10_attention_weight_max": 35.37759017944336,
      "activations/layer10_attention_weight_min": -35.6513557434082,
      "activations/layer11_attention_weight_max": 35.61119842529297,
      "activations/layer11_attention_weight_min": -37.82036209106445,
      "activations/layer12_attention_weight_max": 26.210359573364258,
      "activations/layer12_attention_weight_min": -26.726625442504883,
      "activations/layer13_attention_weight_max": 42.921165466308594,
      "activations/layer13_attention_weight_min": -35.38112258911133,
      "activations/layer14_attention_weight_max": 44.32780075073242,
      "activations/layer14_attention_weight_min": -37.58176040649414,
      "activations/layer15_attention_weight_max": 38.07896041870117,
      "activations/layer15_attention_weight_min": -31.37902069091797,
      "activations/layer16_attention_weight_max": 28.13250160217285,
      "activations/layer16_attention_weight_min": -28.548967361450195,
      "activations/layer17_attention_weight_max": 32.424713134765625,
      "activations/layer17_attention_weight_min": -25.98784828186035,
      "activations/layer18_attention_weight_max": 31.60783576965332,
      "activations/layer18_attention_weight_min": -22.583208084106445,
      "activations/layer19_attention_weight_max": 38.52006149291992,
      "activations/layer19_attention_weight_min": -29.921323776245117,
      "activations/layer1_attention_weight_max": 18.277807235717773,
      "activations/layer1_attention_weight_min": -15.936699867248535,
      "activations/layer20_attention_weight_max": 29.54027557373047,
      "activations/layer20_attention_weight_min": -26.025060653686523,
      "activations/layer21_attention_weight_max": 29.164751052856445,
      "activations/layer21_attention_weight_min": -24.806020736694336,
      "activations/layer22_attention_weight_max": 41.394012451171875,
      "activations/layer22_attention_weight_min": -30.089929580688477,
      "activations/layer23_attention_weight_max": 33.11461639404297,
      "activations/layer23_attention_weight_min": -25.996017456054688,
      "activations/layer2_attention_weight_max": 34.592281341552734,
      "activations/layer2_attention_weight_min": -34.445152282714844,
      "activations/layer3_attention_weight_max": 98.8053970336914,
      "activations/layer3_attention_weight_min": -105.38672637939453,
      "activations/layer4_attention_weight_max": 115.42569732666016,
      "activations/layer4_attention_weight_min": -119.67288970947266,
      "activations/layer5_attention_weight_max": 50.906585693359375,
      "activations/layer5_attention_weight_min": -65.81092834472656,
      "activations/layer6_attention_weight_max": 45.98518753051758,
      "activations/layer6_attention_weight_min": -48.5881233215332,
      "activations/layer7_attention_weight_max": 91.30486297607422,
      "activations/layer7_attention_weight_min": -93.08319854736328,
      "activations/layer8_attention_weight_max": 43.99631118774414,
      "activations/layer8_attention_weight_min": -47.376102447509766,
      "activations/layer9_attention_weight_max": 34.70245361328125,
      "activations/layer9_attention_weight_min": -36.077457427978516,
      "epoch": 12.28,
      "learning_rate": 7.150643939393938e-05,
      "loss": 2.7792,
      "step": 211400
    },
    {
      "activations/layer0_attention_weight_max": 15.379040718078613,
      "activations/layer0_attention_weight_min": -11.794355392456055,
      "activations/layer10_attention_weight_max": 36.21295166015625,
      "activations/layer10_attention_weight_min": -32.49137496948242,
      "activations/layer11_attention_weight_max": 32.47963333129883,
      "activations/layer11_attention_weight_min": -32.36979293823242,
      "activations/layer12_attention_weight_max": 24.83899688720703,
      "activations/layer12_attention_weight_min": -26.10265350341797,
      "activations/layer13_attention_weight_max": 39.36644744873047,
      "activations/layer13_attention_weight_min": -35.51750183105469,
      "activations/layer14_attention_weight_max": 42.64266586303711,
      "activations/layer14_attention_weight_min": -37.996456146240234,
      "activations/layer15_attention_weight_max": 39.157108306884766,
      "activations/layer15_attention_weight_min": -33.36408615112305,
      "activations/layer16_attention_weight_max": 30.99205207824707,
      "activations/layer16_attention_weight_min": -28.034709930419922,
      "activations/layer17_attention_weight_max": 36.430477142333984,
      "activations/layer17_attention_weight_min": -26.67698860168457,
      "activations/layer18_attention_weight_max": 31.413991928100586,
      "activations/layer18_attention_weight_min": -23.65424346923828,
      "activations/layer19_attention_weight_max": 37.350929260253906,
      "activations/layer19_attention_weight_min": -31.1861572265625,
      "activations/layer1_attention_weight_max": 17.053415298461914,
      "activations/layer1_attention_weight_min": -15.571377754211426,
      "activations/layer20_attention_weight_max": 31.12950325012207,
      "activations/layer20_attention_weight_min": -25.54759979248047,
      "activations/layer21_attention_weight_max": 32.66834259033203,
      "activations/layer21_attention_weight_min": -24.524791717529297,
      "activations/layer22_attention_weight_max": 45.71011734008789,
      "activations/layer22_attention_weight_min": -30.077945709228516,
      "activations/layer23_attention_weight_max": 38.65043640136719,
      "activations/layer23_attention_weight_min": -25.048269271850586,
      "activations/layer2_attention_weight_max": 32.76482391357422,
      "activations/layer2_attention_weight_min": -31.262832641601562,
      "activations/layer3_attention_weight_max": 95.96381378173828,
      "activations/layer3_attention_weight_min": -94.51295471191406,
      "activations/layer4_attention_weight_max": 112.7987289428711,
      "activations/layer4_attention_weight_min": -114.56778717041016,
      "activations/layer5_attention_weight_max": 56.7156982421875,
      "activations/layer5_attention_weight_min": -63.9967155456543,
      "activations/layer6_attention_weight_max": 46.72956848144531,
      "activations/layer6_attention_weight_min": -49.025917053222656,
      "activations/layer7_attention_weight_max": 98.91857147216797,
      "activations/layer7_attention_weight_min": -93.22174835205078,
      "activations/layer8_attention_weight_max": 42.32879638671875,
      "activations/layer8_attention_weight_min": -40.02544021606445,
      "activations/layer9_attention_weight_max": 34.85573959350586,
      "activations/layer9_attention_weight_min": -33.005619049072266,
      "epoch": 12.29,
      "learning_rate": 7.14875e-05,
      "loss": 2.774,
      "step": 211450
    },
    {
      "activations/layer0_attention_weight_max": 16.452945709228516,
      "activations/layer0_attention_weight_min": -11.66850757598877,
      "activations/layer10_attention_weight_max": 31.651811599731445,
      "activations/layer10_attention_weight_min": -34.06538772583008,
      "activations/layer11_attention_weight_max": 33.71714782714844,
      "activations/layer11_attention_weight_min": -32.24167251586914,
      "activations/layer12_attention_weight_max": 36.750274658203125,
      "activations/layer12_attention_weight_min": -26.25133514404297,
      "activations/layer13_attention_weight_max": 46.79633331298828,
      "activations/layer13_attention_weight_min": -39.48539352416992,
      "activations/layer14_attention_weight_max": 43.93941116333008,
      "activations/layer14_attention_weight_min": -40.969173431396484,
      "activations/layer15_attention_weight_max": 40.464599609375,
      "activations/layer15_attention_weight_min": -34.74192810058594,
      "activations/layer16_attention_weight_max": 31.217199325561523,
      "activations/layer16_attention_weight_min": -28.988779067993164,
      "activations/layer17_attention_weight_max": 30.8060245513916,
      "activations/layer17_attention_weight_min": -26.27424430847168,
      "activations/layer18_attention_weight_max": 30.973491668701172,
      "activations/layer18_attention_weight_min": -23.8200626373291,
      "activations/layer19_attention_weight_max": 38.977027893066406,
      "activations/layer19_attention_weight_min": -34.10874557495117,
      "activations/layer1_attention_weight_max": 16.736677169799805,
      "activations/layer1_attention_weight_min": -14.780354499816895,
      "activations/layer20_attention_weight_max": 34.09489822387695,
      "activations/layer20_attention_weight_min": -24.516212463378906,
      "activations/layer21_attention_weight_max": 33.72452163696289,
      "activations/layer21_attention_weight_min": -23.70208740234375,
      "activations/layer22_attention_weight_max": 42.69712448120117,
      "activations/layer22_attention_weight_min": -28.694198608398438,
      "activations/layer23_attention_weight_max": 34.553916931152344,
      "activations/layer23_attention_weight_min": -23.733524322509766,
      "activations/layer2_attention_weight_max": 33.50946807861328,
      "activations/layer2_attention_weight_min": -33.29545593261719,
      "activations/layer3_attention_weight_max": 96.96318817138672,
      "activations/layer3_attention_weight_min": -95.03971099853516,
      "activations/layer4_attention_weight_max": 111.71881103515625,
      "activations/layer4_attention_weight_min": -108.3075942993164,
      "activations/layer5_attention_weight_max": 51.60499572753906,
      "activations/layer5_attention_weight_min": -60.40992736816406,
      "activations/layer6_attention_weight_max": 45.51657485961914,
      "activations/layer6_attention_weight_min": -48.137821197509766,
      "activations/layer7_attention_weight_max": 87.79508209228516,
      "activations/layer7_attention_weight_min": -89.92478942871094,
      "activations/layer8_attention_weight_max": 40.031673431396484,
      "activations/layer8_attention_weight_min": -43.274627685546875,
      "activations/layer9_attention_weight_max": 32.92526626586914,
      "activations/layer9_attention_weight_min": -34.595130920410156,
      "epoch": 12.29,
      "learning_rate": 7.14685606060606e-05,
      "loss": 2.77,
      "step": 211500
    },
    {
      "activations/layer0_attention_weight_max": 16.83303451538086,
      "activations/layer0_attention_weight_min": -11.673229217529297,
      "activations/layer10_attention_weight_max": 32.842620849609375,
      "activations/layer10_attention_weight_min": -32.557281494140625,
      "activations/layer11_attention_weight_max": 33.57886505126953,
      "activations/layer11_attention_weight_min": -32.89226150512695,
      "activations/layer12_attention_weight_max": 25.6174259185791,
      "activations/layer12_attention_weight_min": -27.854469299316406,
      "activations/layer13_attention_weight_max": 41.701778411865234,
      "activations/layer13_attention_weight_min": -34.798973083496094,
      "activations/layer14_attention_weight_max": 47.9228401184082,
      "activations/layer14_attention_weight_min": -37.11343765258789,
      "activations/layer15_attention_weight_max": 36.679847717285156,
      "activations/layer15_attention_weight_min": -33.09569549560547,
      "activations/layer16_attention_weight_max": 32.06906509399414,
      "activations/layer16_attention_weight_min": -26.230175018310547,
      "activations/layer17_attention_weight_max": 33.792259216308594,
      "activations/layer17_attention_weight_min": -25.643953323364258,
      "activations/layer18_attention_weight_max": 30.733928680419922,
      "activations/layer18_attention_weight_min": -24.798473358154297,
      "activations/layer19_attention_weight_max": 37.25474166870117,
      "activations/layer19_attention_weight_min": -31.097410202026367,
      "activations/layer1_attention_weight_max": 16.544992446899414,
      "activations/layer1_attention_weight_min": -15.035499572753906,
      "activations/layer20_attention_weight_max": 33.07782745361328,
      "activations/layer20_attention_weight_min": -25.10625457763672,
      "activations/layer21_attention_weight_max": 32.772010803222656,
      "activations/layer21_attention_weight_min": -24.802818298339844,
      "activations/layer22_attention_weight_max": 46.71052551269531,
      "activations/layer22_attention_weight_min": -31.813690185546875,
      "activations/layer23_attention_weight_max": 37.274593353271484,
      "activations/layer23_attention_weight_min": -26.589012145996094,
      "activations/layer2_attention_weight_max": 32.40659713745117,
      "activations/layer2_attention_weight_min": -33.534881591796875,
      "activations/layer3_attention_weight_max": 95.2923583984375,
      "activations/layer3_attention_weight_min": -98.30648803710938,
      "activations/layer4_attention_weight_max": 108.3604965209961,
      "activations/layer4_attention_weight_min": -109.58809661865234,
      "activations/layer5_attention_weight_max": 52.21958923339844,
      "activations/layer5_attention_weight_min": -58.73209762573242,
      "activations/layer6_attention_weight_max": 43.30318069458008,
      "activations/layer6_attention_weight_min": -47.884769439697266,
      "activations/layer7_attention_weight_max": 86.75082397460938,
      "activations/layer7_attention_weight_min": -88.37174987792969,
      "activations/layer8_attention_weight_max": 37.67177200317383,
      "activations/layer8_attention_weight_min": -40.276187896728516,
      "activations/layer9_attention_weight_max": 31.80042839050293,
      "activations/layer9_attention_weight_min": -32.875789642333984,
      "epoch": 12.29,
      "learning_rate": 7.14496212121212e-05,
      "loss": 2.7563,
      "step": 211550
    },
    {
      "activations/layer0_attention_weight_max": 15.737605094909668,
      "activations/layer0_attention_weight_min": -11.739282608032227,
      "activations/layer10_attention_weight_max": 36.21413040161133,
      "activations/layer10_attention_weight_min": -35.02473068237305,
      "activations/layer11_attention_weight_max": 35.26197052001953,
      "activations/layer11_attention_weight_min": -34.31032943725586,
      "activations/layer12_attention_weight_max": 26.54269027709961,
      "activations/layer12_attention_weight_min": -24.635581970214844,
      "activations/layer13_attention_weight_max": 42.1971435546875,
      "activations/layer13_attention_weight_min": -33.695308685302734,
      "activations/layer14_attention_weight_max": 44.93768310546875,
      "activations/layer14_attention_weight_min": -38.465248107910156,
      "activations/layer15_attention_weight_max": 35.82475280761719,
      "activations/layer15_attention_weight_min": -33.0931282043457,
      "activations/layer16_attention_weight_max": 31.698665618896484,
      "activations/layer16_attention_weight_min": -28.310955047607422,
      "activations/layer17_attention_weight_max": 32.460819244384766,
      "activations/layer17_attention_weight_min": -26.427867889404297,
      "activations/layer18_attention_weight_max": 33.68972396850586,
      "activations/layer18_attention_weight_min": -23.93519401550293,
      "activations/layer19_attention_weight_max": 39.473411560058594,
      "activations/layer19_attention_weight_min": -30.669309616088867,
      "activations/layer1_attention_weight_max": 16.149866104125977,
      "activations/layer1_attention_weight_min": -14.627554893493652,
      "activations/layer20_attention_weight_max": 30.763547897338867,
      "activations/layer20_attention_weight_min": -25.35887336730957,
      "activations/layer21_attention_weight_max": 27.054704666137695,
      "activations/layer21_attention_weight_min": -24.44322395324707,
      "activations/layer22_attention_weight_max": 47.321475982666016,
      "activations/layer22_attention_weight_min": -31.593008041381836,
      "activations/layer23_attention_weight_max": 36.339324951171875,
      "activations/layer23_attention_weight_min": -23.884496688842773,
      "activations/layer2_attention_weight_max": 31.782567977905273,
      "activations/layer2_attention_weight_min": -31.519737243652344,
      "activations/layer3_attention_weight_max": 99.09219360351562,
      "activations/layer3_attention_weight_min": -95.40196228027344,
      "activations/layer4_attention_weight_max": 116.1631851196289,
      "activations/layer4_attention_weight_min": -109.70169830322266,
      "activations/layer5_attention_weight_max": 54.37190246582031,
      "activations/layer5_attention_weight_min": -62.68482971191406,
      "activations/layer6_attention_weight_max": 47.901573181152344,
      "activations/layer6_attention_weight_min": -48.119224548339844,
      "activations/layer7_attention_weight_max": 95.43429565429688,
      "activations/layer7_attention_weight_min": -97.1542739868164,
      "activations/layer8_attention_weight_max": 44.340484619140625,
      "activations/layer8_attention_weight_min": -43.8226432800293,
      "activations/layer9_attention_weight_max": 35.76621627807617,
      "activations/layer9_attention_weight_min": -34.933494567871094,
      "epoch": 12.3,
      "learning_rate": 7.143068181818181e-05,
      "loss": 2.7811,
      "step": 211600
    },
    {
      "activations/layer0_attention_weight_max": 15.147490501403809,
      "activations/layer0_attention_weight_min": -11.758524894714355,
      "activations/layer10_attention_weight_max": 31.6490535736084,
      "activations/layer10_attention_weight_min": -33.22688674926758,
      "activations/layer11_attention_weight_max": 31.98016357421875,
      "activations/layer11_attention_weight_min": -31.14069175720215,
      "activations/layer12_attention_weight_max": 24.905277252197266,
      "activations/layer12_attention_weight_min": -27.004182815551758,
      "activations/layer13_attention_weight_max": 39.86155700683594,
      "activations/layer13_attention_weight_min": -34.72654724121094,
      "activations/layer14_attention_weight_max": 44.280250549316406,
      "activations/layer14_attention_weight_min": -38.610755920410156,
      "activations/layer15_attention_weight_max": 39.358314514160156,
      "activations/layer15_attention_weight_min": -34.24169921875,
      "activations/layer16_attention_weight_max": 33.813053131103516,
      "activations/layer16_attention_weight_min": -27.89002799987793,
      "activations/layer17_attention_weight_max": 37.67412185668945,
      "activations/layer17_attention_weight_min": -25.670087814331055,
      "activations/layer18_attention_weight_max": 40.94566345214844,
      "activations/layer18_attention_weight_min": -24.20461082458496,
      "activations/layer19_attention_weight_max": 40.123531341552734,
      "activations/layer19_attention_weight_min": -31.592557907104492,
      "activations/layer1_attention_weight_max": 17.097278594970703,
      "activations/layer1_attention_weight_min": -15.413409233093262,
      "activations/layer20_attention_weight_max": 34.43122100830078,
      "activations/layer20_attention_weight_min": -24.51067543029785,
      "activations/layer21_attention_weight_max": 33.32543182373047,
      "activations/layer21_attention_weight_min": -23.837200164794922,
      "activations/layer22_attention_weight_max": 46.86524200439453,
      "activations/layer22_attention_weight_min": -32.20228576660156,
      "activations/layer23_attention_weight_max": 39.002532958984375,
      "activations/layer23_attention_weight_min": -26.10671043395996,
      "activations/layer2_attention_weight_max": 32.239593505859375,
      "activations/layer2_attention_weight_min": -32.84934997558594,
      "activations/layer3_attention_weight_max": 100.79589080810547,
      "activations/layer3_attention_weight_min": -100.54679107666016,
      "activations/layer4_attention_weight_max": 120.22127532958984,
      "activations/layer4_attention_weight_min": -118.08900451660156,
      "activations/layer5_attention_weight_max": 54.83123779296875,
      "activations/layer5_attention_weight_min": -64.08218383789062,
      "activations/layer6_attention_weight_max": 45.939022064208984,
      "activations/layer6_attention_weight_min": -50.91648864746094,
      "activations/layer7_attention_weight_max": 95.08097076416016,
      "activations/layer7_attention_weight_min": -87.79331970214844,
      "activations/layer8_attention_weight_max": 40.56599426269531,
      "activations/layer8_attention_weight_min": -42.66423034667969,
      "activations/layer9_attention_weight_max": 32.71421813964844,
      "activations/layer9_attention_weight_min": -32.67009735107422,
      "epoch": 12.3,
      "learning_rate": 7.141174242424242e-05,
      "loss": 2.7933,
      "step": 211650
    },
    {
      "activations/layer0_attention_weight_max": 16.84463882446289,
      "activations/layer0_attention_weight_min": -12.183701515197754,
      "activations/layer10_attention_weight_max": 33.52743911743164,
      "activations/layer10_attention_weight_min": -32.40610885620117,
      "activations/layer11_attention_weight_max": 32.89430236816406,
      "activations/layer11_attention_weight_min": -32.03615188598633,
      "activations/layer12_attention_weight_max": 25.277189254760742,
      "activations/layer12_attention_weight_min": -25.850601196289062,
      "activations/layer13_attention_weight_max": 43.00825500488281,
      "activations/layer13_attention_weight_min": -36.803131103515625,
      "activations/layer14_attention_weight_max": 43.767578125,
      "activations/layer14_attention_weight_min": -37.69580841064453,
      "activations/layer15_attention_weight_max": 40.58610534667969,
      "activations/layer15_attention_weight_min": -32.341209411621094,
      "activations/layer16_attention_weight_max": 33.825927734375,
      "activations/layer16_attention_weight_min": -26.81380844116211,
      "activations/layer17_attention_weight_max": 33.15194320678711,
      "activations/layer17_attention_weight_min": -25.19203758239746,
      "activations/layer18_attention_weight_max": 31.16395378112793,
      "activations/layer18_attention_weight_min": -23.179065704345703,
      "activations/layer19_attention_weight_max": 32.77225875854492,
      "activations/layer19_attention_weight_min": -30.066267013549805,
      "activations/layer1_attention_weight_max": 16.659969329833984,
      "activations/layer1_attention_weight_min": -14.293661117553711,
      "activations/layer20_attention_weight_max": 27.98915672302246,
      "activations/layer20_attention_weight_min": -26.833938598632812,
      "activations/layer21_attention_weight_max": 28.083627700805664,
      "activations/layer21_attention_weight_min": -24.2298526763916,
      "activations/layer22_attention_weight_max": 40.30641174316406,
      "activations/layer22_attention_weight_min": -30.16796112060547,
      "activations/layer23_attention_weight_max": 31.612363815307617,
      "activations/layer23_attention_weight_min": -24.12468719482422,
      "activations/layer2_attention_weight_max": 33.48735809326172,
      "activations/layer2_attention_weight_min": -33.15701675415039,
      "activations/layer3_attention_weight_max": 94.34663391113281,
      "activations/layer3_attention_weight_min": -97.88201904296875,
      "activations/layer4_attention_weight_max": 108.27745819091797,
      "activations/layer4_attention_weight_min": -110.80889129638672,
      "activations/layer5_attention_weight_max": 51.51687240600586,
      "activations/layer5_attention_weight_min": -62.550048828125,
      "activations/layer6_attention_weight_max": 44.5484504699707,
      "activations/layer6_attention_weight_min": -49.24210739135742,
      "activations/layer7_attention_weight_max": 90.98992919921875,
      "activations/layer7_attention_weight_min": -92.21932220458984,
      "activations/layer8_attention_weight_max": 39.065513610839844,
      "activations/layer8_attention_weight_min": -40.43479537963867,
      "activations/layer9_attention_weight_max": 33.24235534667969,
      "activations/layer9_attention_weight_min": -32.88416290283203,
      "epoch": 12.3,
      "learning_rate": 7.139280303030303e-05,
      "loss": 2.7589,
      "step": 211700
    },
    {
      "activations/layer0_attention_weight_max": 16.41109848022461,
      "activations/layer0_attention_weight_min": -11.911345481872559,
      "activations/layer10_attention_weight_max": 33.498172760009766,
      "activations/layer10_attention_weight_min": -32.488800048828125,
      "activations/layer11_attention_weight_max": 34.63825988769531,
      "activations/layer11_attention_weight_min": -31.830686569213867,
      "activations/layer12_attention_weight_max": 28.253894805908203,
      "activations/layer12_attention_weight_min": -25.23241424560547,
      "activations/layer13_attention_weight_max": 42.09326934814453,
      "activations/layer13_attention_weight_min": -32.692787170410156,
      "activations/layer14_attention_weight_max": 50.049320220947266,
      "activations/layer14_attention_weight_min": -36.85907745361328,
      "activations/layer15_attention_weight_max": 37.72588348388672,
      "activations/layer15_attention_weight_min": -31.540931701660156,
      "activations/layer16_attention_weight_max": 30.866559982299805,
      "activations/layer16_attention_weight_min": -29.612350463867188,
      "activations/layer17_attention_weight_max": 31.42293357849121,
      "activations/layer17_attention_weight_min": -25.506681442260742,
      "activations/layer18_attention_weight_max": 35.86274337768555,
      "activations/layer18_attention_weight_min": -26.174560546875,
      "activations/layer19_attention_weight_max": 35.865787506103516,
      "activations/layer19_attention_weight_min": -32.05562973022461,
      "activations/layer1_attention_weight_max": 19.78750991821289,
      "activations/layer1_attention_weight_min": -19.52048110961914,
      "activations/layer20_attention_weight_max": 33.69180679321289,
      "activations/layer20_attention_weight_min": -25.66563606262207,
      "activations/layer21_attention_weight_max": 32.61506271362305,
      "activations/layer21_attention_weight_min": -24.896163940429688,
      "activations/layer22_attention_weight_max": 46.045013427734375,
      "activations/layer22_attention_weight_min": -30.592784881591797,
      "activations/layer23_attention_weight_max": 36.78714370727539,
      "activations/layer23_attention_weight_min": -26.926259994506836,
      "activations/layer2_attention_weight_max": 32.66606903076172,
      "activations/layer2_attention_weight_min": -33.36900329589844,
      "activations/layer3_attention_weight_max": 95.53438568115234,
      "activations/layer3_attention_weight_min": -95.58160400390625,
      "activations/layer4_attention_weight_max": 111.6146011352539,
      "activations/layer4_attention_weight_min": -112.8492202758789,
      "activations/layer5_attention_weight_max": 53.337799072265625,
      "activations/layer5_attention_weight_min": -60.039093017578125,
      "activations/layer6_attention_weight_max": 46.848899841308594,
      "activations/layer6_attention_weight_min": -51.71147918701172,
      "activations/layer7_attention_weight_max": 98.79547119140625,
      "activations/layer7_attention_weight_min": -96.17430114746094,
      "activations/layer8_attention_weight_max": 40.25366973876953,
      "activations/layer8_attention_weight_min": -45.35188674926758,
      "activations/layer9_attention_weight_max": 33.48107147216797,
      "activations/layer9_attention_weight_min": -34.02474594116211,
      "epoch": 12.3,
      "learning_rate": 7.137386363636363e-05,
      "loss": 2.7646,
      "step": 211750
    },
    {
      "activations/layer0_attention_weight_max": 16.394411087036133,
      "activations/layer0_attention_weight_min": -11.970074653625488,
      "activations/layer10_attention_weight_max": 33.50037384033203,
      "activations/layer10_attention_weight_min": -32.576087951660156,
      "activations/layer11_attention_weight_max": 35.14239501953125,
      "activations/layer11_attention_weight_min": -33.58831024169922,
      "activations/layer12_attention_weight_max": 25.480497360229492,
      "activations/layer12_attention_weight_min": -24.605548858642578,
      "activations/layer13_attention_weight_max": 36.49250411987305,
      "activations/layer13_attention_weight_min": -34.06809997558594,
      "activations/layer14_attention_weight_max": 41.77864456176758,
      "activations/layer14_attention_weight_min": -37.902462005615234,
      "activations/layer15_attention_weight_max": 34.35403060913086,
      "activations/layer15_attention_weight_min": -32.24205780029297,
      "activations/layer16_attention_weight_max": 30.246387481689453,
      "activations/layer16_attention_weight_min": -27.797393798828125,
      "activations/layer17_attention_weight_max": 31.219791412353516,
      "activations/layer17_attention_weight_min": -27.095460891723633,
      "activations/layer18_attention_weight_max": 32.42914581298828,
      "activations/layer18_attention_weight_min": -24.29404067993164,
      "activations/layer19_attention_weight_max": 35.51606750488281,
      "activations/layer19_attention_weight_min": -31.857213973999023,
      "activations/layer1_attention_weight_max": 15.949871063232422,
      "activations/layer1_attention_weight_min": -15.170104026794434,
      "activations/layer20_attention_weight_max": 30.732261657714844,
      "activations/layer20_attention_weight_min": -25.1578311920166,
      "activations/layer21_attention_weight_max": 30.328386306762695,
      "activations/layer21_attention_weight_min": -24.64301109313965,
      "activations/layer22_attention_weight_max": 48.100677490234375,
      "activations/layer22_attention_weight_min": -32.37129211425781,
      "activations/layer23_attention_weight_max": 38.247894287109375,
      "activations/layer23_attention_weight_min": -27.159801483154297,
      "activations/layer2_attention_weight_max": 33.644805908203125,
      "activations/layer2_attention_weight_min": -33.65448760986328,
      "activations/layer3_attention_weight_max": 95.64212036132812,
      "activations/layer3_attention_weight_min": -96.90355682373047,
      "activations/layer4_attention_weight_max": 110.59869384765625,
      "activations/layer4_attention_weight_min": -111.69091033935547,
      "activations/layer5_attention_weight_max": 50.248748779296875,
      "activations/layer5_attention_weight_min": -60.1457633972168,
      "activations/layer6_attention_weight_max": 44.5562629699707,
      "activations/layer6_attention_weight_min": -46.980342864990234,
      "activations/layer7_attention_weight_max": 87.82330322265625,
      "activations/layer7_attention_weight_min": -88.37596893310547,
      "activations/layer8_attention_weight_max": 38.94672775268555,
      "activations/layer8_attention_weight_min": -40.9573860168457,
      "activations/layer9_attention_weight_max": 32.457149505615234,
      "activations/layer9_attention_weight_min": -33.22163009643555,
      "epoch": 12.31,
      "learning_rate": 7.135492424242423e-05,
      "loss": 2.7893,
      "step": 211800
    },
    {
      "activations/layer0_attention_weight_max": 16.320714950561523,
      "activations/layer0_attention_weight_min": -11.953271865844727,
      "activations/layer10_attention_weight_max": 32.516929626464844,
      "activations/layer10_attention_weight_min": -33.978424072265625,
      "activations/layer11_attention_weight_max": 33.74726486206055,
      "activations/layer11_attention_weight_min": -33.120140075683594,
      "activations/layer12_attention_weight_max": 33.57320022583008,
      "activations/layer12_attention_weight_min": -26.274330139160156,
      "activations/layer13_attention_weight_max": 39.9063720703125,
      "activations/layer13_attention_weight_min": -34.04219055175781,
      "activations/layer14_attention_weight_max": 43.61854553222656,
      "activations/layer14_attention_weight_min": -36.57088088989258,
      "activations/layer15_attention_weight_max": 37.927520751953125,
      "activations/layer15_attention_weight_min": -33.323402404785156,
      "activations/layer16_attention_weight_max": 29.65900993347168,
      "activations/layer16_attention_weight_min": -27.979000091552734,
      "activations/layer17_attention_weight_max": 30.3851318359375,
      "activations/layer17_attention_weight_min": -25.970928192138672,
      "activations/layer18_attention_weight_max": 30.98077392578125,
      "activations/layer18_attention_weight_min": -22.776844024658203,
      "activations/layer19_attention_weight_max": 33.2638053894043,
      "activations/layer19_attention_weight_min": -29.554550170898438,
      "activations/layer1_attention_weight_max": 17.035245895385742,
      "activations/layer1_attention_weight_min": -16.32744789123535,
      "activations/layer20_attention_weight_max": 29.361328125,
      "activations/layer20_attention_weight_min": -23.143985748291016,
      "activations/layer21_attention_weight_max": 29.5456600189209,
      "activations/layer21_attention_weight_min": -23.57352066040039,
      "activations/layer22_attention_weight_max": 44.55283737182617,
      "activations/layer22_attention_weight_min": -30.4521541595459,
      "activations/layer23_attention_weight_max": 34.00633239746094,
      "activations/layer23_attention_weight_min": -25.00855827331543,
      "activations/layer2_attention_weight_max": 33.64419937133789,
      "activations/layer2_attention_weight_min": -33.17469787597656,
      "activations/layer3_attention_weight_max": 97.74061584472656,
      "activations/layer3_attention_weight_min": -98.34590148925781,
      "activations/layer4_attention_weight_max": 112.33308410644531,
      "activations/layer4_attention_weight_min": -114.94808197021484,
      "activations/layer5_attention_weight_max": 51.45719909667969,
      "activations/layer5_attention_weight_min": -60.481319427490234,
      "activations/layer6_attention_weight_max": 45.90278244018555,
      "activations/layer6_attention_weight_min": -47.73503875732422,
      "activations/layer7_attention_weight_max": 91.85752868652344,
      "activations/layer7_attention_weight_min": -89.42794799804688,
      "activations/layer8_attention_weight_max": 40.968841552734375,
      "activations/layer8_attention_weight_min": -42.58926773071289,
      "activations/layer9_attention_weight_max": 34.614967346191406,
      "activations/layer9_attention_weight_min": -34.34910202026367,
      "epoch": 12.31,
      "learning_rate": 7.133598484848485e-05,
      "loss": 2.7715,
      "step": 211850
    },
    {
      "activations/layer0_attention_weight_max": 15.581003189086914,
      "activations/layer0_attention_weight_min": -11.740429878234863,
      "activations/layer10_attention_weight_max": 34.292171478271484,
      "activations/layer10_attention_weight_min": -32.0205078125,
      "activations/layer11_attention_weight_max": 34.66230392456055,
      "activations/layer11_attention_weight_min": -32.48345184326172,
      "activations/layer12_attention_weight_max": 26.46289825439453,
      "activations/layer12_attention_weight_min": -25.852418899536133,
      "activations/layer13_attention_weight_max": 40.79398727416992,
      "activations/layer13_attention_weight_min": -36.02555847167969,
      "activations/layer14_attention_weight_max": 41.99052429199219,
      "activations/layer14_attention_weight_min": -36.953426361083984,
      "activations/layer15_attention_weight_max": 35.72795104980469,
      "activations/layer15_attention_weight_min": -33.804134368896484,
      "activations/layer16_attention_weight_max": 29.918542861938477,
      "activations/layer16_attention_weight_min": -30.16935920715332,
      "activations/layer17_attention_weight_max": 31.359195709228516,
      "activations/layer17_attention_weight_min": -26.02030372619629,
      "activations/layer18_attention_weight_max": 30.197650909423828,
      "activations/layer18_attention_weight_min": -25.645755767822266,
      "activations/layer19_attention_weight_max": 34.33216857910156,
      "activations/layer19_attention_weight_min": -30.860326766967773,
      "activations/layer1_attention_weight_max": 17.441965103149414,
      "activations/layer1_attention_weight_min": -16.850614547729492,
      "activations/layer20_attention_weight_max": 28.492589950561523,
      "activations/layer20_attention_weight_min": -25.808021545410156,
      "activations/layer21_attention_weight_max": 29.794246673583984,
      "activations/layer21_attention_weight_min": -25.34871482849121,
      "activations/layer22_attention_weight_max": 46.12617492675781,
      "activations/layer22_attention_weight_min": -31.65403938293457,
      "activations/layer23_attention_weight_max": 35.866004943847656,
      "activations/layer23_attention_weight_min": -26.535564422607422,
      "activations/layer2_attention_weight_max": 33.730743408203125,
      "activations/layer2_attention_weight_min": -32.786033630371094,
      "activations/layer3_attention_weight_max": 102.79093170166016,
      "activations/layer3_attention_weight_min": -104.5437240600586,
      "activations/layer4_attention_weight_max": 111.21871185302734,
      "activations/layer4_attention_weight_min": -117.4758071899414,
      "activations/layer5_attention_weight_max": 51.35053634643555,
      "activations/layer5_attention_weight_min": -61.22682189941406,
      "activations/layer6_attention_weight_max": 44.90571212768555,
      "activations/layer6_attention_weight_min": -47.35816955566406,
      "activations/layer7_attention_weight_max": 91.5672836303711,
      "activations/layer7_attention_weight_min": -95.08655548095703,
      "activations/layer8_attention_weight_max": 41.0479736328125,
      "activations/layer8_attention_weight_min": -41.01846694946289,
      "activations/layer9_attention_weight_max": 34.28684997558594,
      "activations/layer9_attention_weight_min": -31.636417388916016,
      "epoch": 12.31,
      "learning_rate": 7.131704545454545e-05,
      "loss": 2.7778,
      "step": 211900
    },
    {
      "activations/layer0_attention_weight_max": 15.662187576293945,
      "activations/layer0_attention_weight_min": -12.01929759979248,
      "activations/layer10_attention_weight_max": 32.76390075683594,
      "activations/layer10_attention_weight_min": -33.68347930908203,
      "activations/layer11_attention_weight_max": 31.94989585876465,
      "activations/layer11_attention_weight_min": -31.212629318237305,
      "activations/layer12_attention_weight_max": 23.932212829589844,
      "activations/layer12_attention_weight_min": -24.14555549621582,
      "activations/layer13_attention_weight_max": 35.312801361083984,
      "activations/layer13_attention_weight_min": -34.05961608886719,
      "activations/layer14_attention_weight_max": 38.9435920715332,
      "activations/layer14_attention_weight_min": -37.116661071777344,
      "activations/layer15_attention_weight_max": 34.82553482055664,
      "activations/layer15_attention_weight_min": -30.53203773498535,
      "activations/layer16_attention_weight_max": 29.12864875793457,
      "activations/layer16_attention_weight_min": -28.154932022094727,
      "activations/layer17_attention_weight_max": 31.83585548400879,
      "activations/layer17_attention_weight_min": -25.45820426940918,
      "activations/layer18_attention_weight_max": 32.50323486328125,
      "activations/layer18_attention_weight_min": -24.598371505737305,
      "activations/layer19_attention_weight_max": 33.956031799316406,
      "activations/layer19_attention_weight_min": -29.899215698242188,
      "activations/layer1_attention_weight_max": 18.183080673217773,
      "activations/layer1_attention_weight_min": -18.176424026489258,
      "activations/layer20_attention_weight_max": 28.4964599609375,
      "activations/layer20_attention_weight_min": -24.044042587280273,
      "activations/layer21_attention_weight_max": 31.076915740966797,
      "activations/layer21_attention_weight_min": -24.040639877319336,
      "activations/layer22_attention_weight_max": 46.01465606689453,
      "activations/layer22_attention_weight_min": -30.370195388793945,
      "activations/layer23_attention_weight_max": 37.31285095214844,
      "activations/layer23_attention_weight_min": -25.082305908203125,
      "activations/layer2_attention_weight_max": 33.475379943847656,
      "activations/layer2_attention_weight_min": -32.737857818603516,
      "activations/layer3_attention_weight_max": 94.24227905273438,
      "activations/layer3_attention_weight_min": -95.88382720947266,
      "activations/layer4_attention_weight_max": 113.99895477294922,
      "activations/layer4_attention_weight_min": -117.73943328857422,
      "activations/layer5_attention_weight_max": 50.60026168823242,
      "activations/layer5_attention_weight_min": -60.22303771972656,
      "activations/layer6_attention_weight_max": 43.69231033325195,
      "activations/layer6_attention_weight_min": -49.24116516113281,
      "activations/layer7_attention_weight_max": 89.69495391845703,
      "activations/layer7_attention_weight_min": -95.6015853881836,
      "activations/layer8_attention_weight_max": 40.61586380004883,
      "activations/layer8_attention_weight_min": -42.28141784667969,
      "activations/layer9_attention_weight_max": 32.76780319213867,
      "activations/layer9_attention_weight_min": -34.47567367553711,
      "epoch": 12.32,
      "learning_rate": 7.129810606060605e-05,
      "loss": 2.7662,
      "step": 211950
    },
    {
      "activations/layer0_attention_weight_max": 16.26123809814453,
      "activations/layer0_attention_weight_min": -11.99190902709961,
      "activations/layer10_attention_weight_max": 33.965797424316406,
      "activations/layer10_attention_weight_min": -35.238037109375,
      "activations/layer11_attention_weight_max": 34.96992874145508,
      "activations/layer11_attention_weight_min": -37.66941833496094,
      "activations/layer12_attention_weight_max": 26.216251373291016,
      "activations/layer12_attention_weight_min": -26.320772171020508,
      "activations/layer13_attention_weight_max": 38.44253158569336,
      "activations/layer13_attention_weight_min": -35.88759994506836,
      "activations/layer14_attention_weight_max": 43.390113830566406,
      "activations/layer14_attention_weight_min": -38.97648620605469,
      "activations/layer15_attention_weight_max": 37.77900695800781,
      "activations/layer15_attention_weight_min": -34.355751037597656,
      "activations/layer16_attention_weight_max": 28.780996322631836,
      "activations/layer16_attention_weight_min": -28.127546310424805,
      "activations/layer17_attention_weight_max": 32.37612533569336,
      "activations/layer17_attention_weight_min": -26.441783905029297,
      "activations/layer18_attention_weight_max": 31.96806526184082,
      "activations/layer18_attention_weight_min": -23.9139461517334,
      "activations/layer19_attention_weight_max": 36.25624465942383,
      "activations/layer19_attention_weight_min": -30.978076934814453,
      "activations/layer1_attention_weight_max": 16.00204849243164,
      "activations/layer1_attention_weight_min": -15.307159423828125,
      "activations/layer20_attention_weight_max": 33.760982513427734,
      "activations/layer20_attention_weight_min": -25.60972785949707,
      "activations/layer21_attention_weight_max": 31.006866455078125,
      "activations/layer21_attention_weight_min": -26.114782333374023,
      "activations/layer22_attention_weight_max": 50.19276428222656,
      "activations/layer22_attention_weight_min": -31.51993751525879,
      "activations/layer23_attention_weight_max": 37.206016540527344,
      "activations/layer23_attention_weight_min": -24.87921142578125,
      "activations/layer2_attention_weight_max": 31.796463012695312,
      "activations/layer2_attention_weight_min": -32.27485656738281,
      "activations/layer3_attention_weight_max": 96.79159545898438,
      "activations/layer3_attention_weight_min": -95.94623565673828,
      "activations/layer4_attention_weight_max": 112.4677963256836,
      "activations/layer4_attention_weight_min": -113.0851821899414,
      "activations/layer5_attention_weight_max": 53.58898162841797,
      "activations/layer5_attention_weight_min": -62.873104095458984,
      "activations/layer6_attention_weight_max": 45.15608215332031,
      "activations/layer6_attention_weight_min": -49.238182067871094,
      "activations/layer7_attention_weight_max": 90.02659606933594,
      "activations/layer7_attention_weight_min": -98.18643951416016,
      "activations/layer8_attention_weight_max": 41.01100158691406,
      "activations/layer8_attention_weight_min": -42.45517349243164,
      "activations/layer9_attention_weight_max": 35.24082946777344,
      "activations/layer9_attention_weight_min": -34.19976043701172,
      "epoch": 12.32,
      "learning_rate": 7.127916666666665e-05,
      "loss": 2.7752,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4788,
      "eval_samples_per_second": 506.438,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4788,
      "eval_openwebtext_samples_per_second": 506.438,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9486,
      "eval_wikitext_samples_per_second": 234.01,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_lambada_loss": 2.49609375,
      "eval_lambada_ppl": 12.134998917818166,
      "eval_lambada_runtime": 9.5086,
      "eval_lambada_samples_per_second": 512.063,
      "step": 212000
    },
    {
      "activations/layer0_attention_weight_max": 16.005577087402344,
      "activations/layer0_attention_weight_min": -12.45090389251709,
      "activations/layer10_attention_weight_max": 39.78778076171875,
      "activations/layer10_attention_weight_min": -38.9315071105957,
      "activations/layer11_attention_weight_max": 41.34021759033203,
      "activations/layer11_attention_weight_min": -40.486366271972656,
      "activations/layer12_attention_weight_max": 27.89275550842285,
      "activations/layer12_attention_weight_min": -27.17776870727539,
      "activations/layer13_attention_weight_max": 42.189109802246094,
      "activations/layer13_attention_weight_min": -33.717323303222656,
      "activations/layer14_attention_weight_max": 49.912662506103516,
      "activations/layer14_attention_weight_min": -39.29551315307617,
      "activations/layer15_attention_weight_max": 42.74679946899414,
      "activations/layer15_attention_weight_min": -34.351646423339844,
      "activations/layer16_attention_weight_max": 38.991241455078125,
      "activations/layer16_attention_weight_min": -31.133731842041016,
      "activations/layer17_attention_weight_max": 39.07115173339844,
      "activations/layer17_attention_weight_min": -31.46387481689453,
      "activations/layer18_attention_weight_max": 35.17508316040039,
      "activations/layer18_attention_weight_min": -26.917949676513672,
      "activations/layer19_attention_weight_max": 38.71710205078125,
      "activations/layer19_attention_weight_min": -33.62382125854492,
      "activations/layer1_attention_weight_max": 16.76734733581543,
      "activations/layer1_attention_weight_min": -15.852694511413574,
      "activations/layer20_attention_weight_max": 33.11832046508789,
      "activations/layer20_attention_weight_min": -26.839569091796875,
      "activations/layer21_attention_weight_max": 30.575138092041016,
      "activations/layer21_attention_weight_min": -24.463895797729492,
      "activations/layer22_attention_weight_max": 50.89381408691406,
      "activations/layer22_attention_weight_min": -33.63174057006836,
      "activations/layer23_attention_weight_max": 36.09090042114258,
      "activations/layer23_attention_weight_min": -26.992435455322266,
      "activations/layer2_attention_weight_max": 39.12510681152344,
      "activations/layer2_attention_weight_min": -37.1242790222168,
      "activations/layer3_attention_weight_max": 106.73991394042969,
      "activations/layer3_attention_weight_min": -106.46379089355469,
      "activations/layer4_attention_weight_max": 122.34545135498047,
      "activations/layer4_attention_weight_min": -113.22563934326172,
      "activations/layer5_attention_weight_max": 57.72251892089844,
      "activations/layer5_attention_weight_min": -60.45490264892578,
      "activations/layer6_attention_weight_max": 45.966224670410156,
      "activations/layer6_attention_weight_min": -47.52298355102539,
      "activations/layer7_attention_weight_max": 105.50817108154297,
      "activations/layer7_attention_weight_min": -98.57540130615234,
      "activations/layer8_attention_weight_max": 47.82228469848633,
      "activations/layer8_attention_weight_min": -42.43516540527344,
      "activations/layer9_attention_weight_max": 37.761070251464844,
      "activations/layer9_attention_weight_min": -37.2746696472168,
      "epoch": 12.32,
      "learning_rate": 7.126022727272727e-05,
      "loss": 2.774,
      "step": 212050
    },
    {
      "activations/layer0_attention_weight_max": 16.119140625,
      "activations/layer0_attention_weight_min": -12.41545581817627,
      "activations/layer10_attention_weight_max": 34.555625915527344,
      "activations/layer10_attention_weight_min": -33.27669906616211,
      "activations/layer11_attention_weight_max": 33.847373962402344,
      "activations/layer11_attention_weight_min": -32.677616119384766,
      "activations/layer12_attention_weight_max": 25.693782806396484,
      "activations/layer12_attention_weight_min": -25.463163375854492,
      "activations/layer13_attention_weight_max": 42.9636344909668,
      "activations/layer13_attention_weight_min": -34.640621185302734,
      "activations/layer14_attention_weight_max": 45.28069305419922,
      "activations/layer14_attention_weight_min": -39.86026382446289,
      "activations/layer15_attention_weight_max": 37.697914123535156,
      "activations/layer15_attention_weight_min": -34.465065002441406,
      "activations/layer16_attention_weight_max": 33.05692672729492,
      "activations/layer16_attention_weight_min": -29.14671516418457,
      "activations/layer17_attention_weight_max": 33.658145904541016,
      "activations/layer17_attention_weight_min": -25.618558883666992,
      "activations/layer18_attention_weight_max": 32.82862854003906,
      "activations/layer18_attention_weight_min": -23.20435333251953,
      "activations/layer19_attention_weight_max": 36.85390090942383,
      "activations/layer19_attention_weight_min": -32.05397033691406,
      "activations/layer1_attention_weight_max": 15.494956970214844,
      "activations/layer1_attention_weight_min": -13.753862380981445,
      "activations/layer20_attention_weight_max": 31.31377410888672,
      "activations/layer20_attention_weight_min": -26.808902740478516,
      "activations/layer21_attention_weight_max": 30.956436157226562,
      "activations/layer21_attention_weight_min": -24.273563385009766,
      "activations/layer22_attention_weight_max": 47.56005859375,
      "activations/layer22_attention_weight_min": -32.241050720214844,
      "activations/layer23_attention_weight_max": 36.73157501220703,
      "activations/layer23_attention_weight_min": -28.71333885192871,
      "activations/layer2_attention_weight_max": 33.747459411621094,
      "activations/layer2_attention_weight_min": -32.892112731933594,
      "activations/layer3_attention_weight_max": 100.28140258789062,
      "activations/layer3_attention_weight_min": -100.71857452392578,
      "activations/layer4_attention_weight_max": 121.82805633544922,
      "activations/layer4_attention_weight_min": -118.59081268310547,
      "activations/layer5_attention_weight_max": 55.76728820800781,
      "activations/layer5_attention_weight_min": -62.99909591674805,
      "activations/layer6_attention_weight_max": 47.62294387817383,
      "activations/layer6_attention_weight_min": -51.28716278076172,
      "activations/layer7_attention_weight_max": 94.83329010009766,
      "activations/layer7_attention_weight_min": -93.53314208984375,
      "activations/layer8_attention_weight_max": 41.75384521484375,
      "activations/layer8_attention_weight_min": -41.392147064208984,
      "activations/layer9_attention_weight_max": 33.91741943359375,
      "activations/layer9_attention_weight_min": -33.949256896972656,
      "epoch": 12.32,
      "learning_rate": 7.124166666666666e-05,
      "loss": 2.7789,
      "step": 212100
    },
    {
      "activations/layer0_attention_weight_max": 14.481362342834473,
      "activations/layer0_attention_weight_min": -12.260429382324219,
      "activations/layer10_attention_weight_max": 36.08300018310547,
      "activations/layer10_attention_weight_min": -33.78462600708008,
      "activations/layer11_attention_weight_max": 34.04749298095703,
      "activations/layer11_attention_weight_min": -33.44748306274414,
      "activations/layer12_attention_weight_max": 26.077110290527344,
      "activations/layer12_attention_weight_min": -26.197811126708984,
      "activations/layer13_attention_weight_max": 40.0487174987793,
      "activations/layer13_attention_weight_min": -35.84685134887695,
      "activations/layer14_attention_weight_max": 47.111385345458984,
      "activations/layer14_attention_weight_min": -38.73453903198242,
      "activations/layer15_attention_weight_max": 37.58512496948242,
      "activations/layer15_attention_weight_min": -30.785062789916992,
      "activations/layer16_attention_weight_max": 32.678489685058594,
      "activations/layer16_attention_weight_min": -29.159456253051758,
      "activations/layer17_attention_weight_max": 32.99488830566406,
      "activations/layer17_attention_weight_min": -24.808439254760742,
      "activations/layer18_attention_weight_max": 30.427936553955078,
      "activations/layer18_attention_weight_min": -22.54686164855957,
      "activations/layer19_attention_weight_max": 35.99878692626953,
      "activations/layer19_attention_weight_min": -29.101083755493164,
      "activations/layer1_attention_weight_max": 15.369342803955078,
      "activations/layer1_attention_weight_min": -15.206542015075684,
      "activations/layer20_attention_weight_max": 31.281892776489258,
      "activations/layer20_attention_weight_min": -25.613203048706055,
      "activations/layer21_attention_weight_max": 31.114782333374023,
      "activations/layer21_attention_weight_min": -24.561649322509766,
      "activations/layer22_attention_weight_max": 44.78676986694336,
      "activations/layer22_attention_weight_min": -29.380634307861328,
      "activations/layer23_attention_weight_max": 36.087562561035156,
      "activations/layer23_attention_weight_min": -24.48506736755371,
      "activations/layer2_attention_weight_max": 35.107032775878906,
      "activations/layer2_attention_weight_min": -33.58586883544922,
      "activations/layer3_attention_weight_max": 96.53958892822266,
      "activations/layer3_attention_weight_min": -98.76802062988281,
      "activations/layer4_attention_weight_max": 113.64190673828125,
      "activations/layer4_attention_weight_min": -114.0825424194336,
      "activations/layer5_attention_weight_max": 52.380516052246094,
      "activations/layer5_attention_weight_min": -60.84748077392578,
      "activations/layer6_attention_weight_max": 46.68804168701172,
      "activations/layer6_attention_weight_min": -49.808841705322266,
      "activations/layer7_attention_weight_max": 97.50313568115234,
      "activations/layer7_attention_weight_min": -98.36254119873047,
      "activations/layer8_attention_weight_max": 43.52785873413086,
      "activations/layer8_attention_weight_min": -44.2093391418457,
      "activations/layer9_attention_weight_max": 35.116546630859375,
      "activations/layer9_attention_weight_min": -34.272457122802734,
      "epoch": 12.33,
      "learning_rate": 7.122272727272726e-05,
      "loss": 2.7669,
      "step": 212150
    },
    {
      "activations/layer0_attention_weight_max": 15.867354393005371,
      "activations/layer0_attention_weight_min": -13.173354148864746,
      "activations/layer10_attention_weight_max": 33.42837142944336,
      "activations/layer10_attention_weight_min": -32.67729949951172,
      "activations/layer11_attention_weight_max": 32.08082580566406,
      "activations/layer11_attention_weight_min": -31.36517906188965,
      "activations/layer12_attention_weight_max": 25.815526962280273,
      "activations/layer12_attention_weight_min": -25.836387634277344,
      "activations/layer13_attention_weight_max": 39.517189025878906,
      "activations/layer13_attention_weight_min": -32.753265380859375,
      "activations/layer14_attention_weight_max": 42.84087371826172,
      "activations/layer14_attention_weight_min": -37.19786071777344,
      "activations/layer15_attention_weight_max": 34.191131591796875,
      "activations/layer15_attention_weight_min": -30.150691986083984,
      "activations/layer16_attention_weight_max": 28.460975646972656,
      "activations/layer16_attention_weight_min": -27.537521362304688,
      "activations/layer17_attention_weight_max": 31.675622940063477,
      "activations/layer17_attention_weight_min": -24.87127113342285,
      "activations/layer18_attention_weight_max": 31.323518753051758,
      "activations/layer18_attention_weight_min": -22.95139503479004,
      "activations/layer19_attention_weight_max": 33.5402946472168,
      "activations/layer19_attention_weight_min": -34.05015563964844,
      "activations/layer1_attention_weight_max": 18.43877601623535,
      "activations/layer1_attention_weight_min": -15.091772079467773,
      "activations/layer20_attention_weight_max": 32.73079299926758,
      "activations/layer20_attention_weight_min": -28.164188385009766,
      "activations/layer21_attention_weight_max": 31.841819763183594,
      "activations/layer21_attention_weight_min": -27.385345458984375,
      "activations/layer22_attention_weight_max": 46.974205017089844,
      "activations/layer22_attention_weight_min": -32.98389434814453,
      "activations/layer23_attention_weight_max": 34.49916076660156,
      "activations/layer23_attention_weight_min": -27.604446411132812,
      "activations/layer2_attention_weight_max": 33.28801727294922,
      "activations/layer2_attention_weight_min": -33.72224426269531,
      "activations/layer3_attention_weight_max": 96.31527709960938,
      "activations/layer3_attention_weight_min": -98.10103607177734,
      "activations/layer4_attention_weight_max": 110.62855529785156,
      "activations/layer4_attention_weight_min": -110.92411041259766,
      "activations/layer5_attention_weight_max": 50.647979736328125,
      "activations/layer5_attention_weight_min": -60.471832275390625,
      "activations/layer6_attention_weight_max": 44.962547302246094,
      "activations/layer6_attention_weight_min": -49.470909118652344,
      "activations/layer7_attention_weight_max": 89.28038024902344,
      "activations/layer7_attention_weight_min": -90.44049835205078,
      "activations/layer8_attention_weight_max": 39.1965217590332,
      "activations/layer8_attention_weight_min": -41.21809387207031,
      "activations/layer9_attention_weight_max": 32.86067199707031,
      "activations/layer9_attention_weight_min": -33.4024543762207,
      "epoch": 12.33,
      "learning_rate": 7.120378787878786e-05,
      "loss": 2.7679,
      "step": 212200
    },
    {
      "activations/layer0_attention_weight_max": 16.447216033935547,
      "activations/layer0_attention_weight_min": -11.599241256713867,
      "activations/layer10_attention_weight_max": 32.930335998535156,
      "activations/layer10_attention_weight_min": -33.442161560058594,
      "activations/layer11_attention_weight_max": 33.72489929199219,
      "activations/layer11_attention_weight_min": -34.89173889160156,
      "activations/layer12_attention_weight_max": 26.279848098754883,
      "activations/layer12_attention_weight_min": -25.220657348632812,
      "activations/layer13_attention_weight_max": 36.509918212890625,
      "activations/layer13_attention_weight_min": -33.565948486328125,
      "activations/layer14_attention_weight_max": 40.479427337646484,
      "activations/layer14_attention_weight_min": -36.490169525146484,
      "activations/layer15_attention_weight_max": 35.50122833251953,
      "activations/layer15_attention_weight_min": -33.213104248046875,
      "activations/layer16_attention_weight_max": 29.759601593017578,
      "activations/layer16_attention_weight_min": -29.53521728515625,
      "activations/layer17_attention_weight_max": 34.542572021484375,
      "activations/layer17_attention_weight_min": -26.734817504882812,
      "activations/layer18_attention_weight_max": 35.650245666503906,
      "activations/layer18_attention_weight_min": -23.301136016845703,
      "activations/layer19_attention_weight_max": 35.265132904052734,
      "activations/layer19_attention_weight_min": -29.443389892578125,
      "activations/layer1_attention_weight_max": 16.658536911010742,
      "activations/layer1_attention_weight_min": -14.557927131652832,
      "activations/layer20_attention_weight_max": 31.3747501373291,
      "activations/layer20_attention_weight_min": -23.172405242919922,
      "activations/layer21_attention_weight_max": 32.319793701171875,
      "activations/layer21_attention_weight_min": -22.318607330322266,
      "activations/layer22_attention_weight_max": 46.14668655395508,
      "activations/layer22_attention_weight_min": -30.033493041992188,
      "activations/layer23_attention_weight_max": 34.8151741027832,
      "activations/layer23_attention_weight_min": -26.046703338623047,
      "activations/layer2_attention_weight_max": 35.093284606933594,
      "activations/layer2_attention_weight_min": -33.14503479003906,
      "activations/layer3_attention_weight_max": 102.41771697998047,
      "activations/layer3_attention_weight_min": -97.8789291381836,
      "activations/layer4_attention_weight_max": 110.85228729248047,
      "activations/layer4_attention_weight_min": -107.51934814453125,
      "activations/layer5_attention_weight_max": 52.131038665771484,
      "activations/layer5_attention_weight_min": -62.72901153564453,
      "activations/layer6_attention_weight_max": 43.306888580322266,
      "activations/layer6_attention_weight_min": -49.5341911315918,
      "activations/layer7_attention_weight_max": 99.31854248046875,
      "activations/layer7_attention_weight_min": -95.6323013305664,
      "activations/layer8_attention_weight_max": 43.37139892578125,
      "activations/layer8_attention_weight_min": -42.16966247558594,
      "activations/layer9_attention_weight_max": 31.040571212768555,
      "activations/layer9_attention_weight_min": -34.07776641845703,
      "epoch": 12.33,
      "learning_rate": 7.118484848484848e-05,
      "loss": 2.7635,
      "step": 212250
    },
    {
      "activations/layer0_attention_weight_max": 16.29892921447754,
      "activations/layer0_attention_weight_min": -12.528261184692383,
      "activations/layer10_attention_weight_max": 35.42212677001953,
      "activations/layer10_attention_weight_min": -36.23670196533203,
      "activations/layer11_attention_weight_max": 34.311092376708984,
      "activations/layer11_attention_weight_min": -35.1110954284668,
      "activations/layer12_attention_weight_max": 27.939510345458984,
      "activations/layer12_attention_weight_min": -26.670495986938477,
      "activations/layer13_attention_weight_max": 41.732215881347656,
      "activations/layer13_attention_weight_min": -37.1972770690918,
      "activations/layer14_attention_weight_max": 46.776065826416016,
      "activations/layer14_attention_weight_min": -39.43485641479492,
      "activations/layer15_attention_weight_max": 39.33020782470703,
      "activations/layer15_attention_weight_min": -31.623920440673828,
      "activations/layer16_attention_weight_max": 32.87057113647461,
      "activations/layer16_attention_weight_min": -26.574710845947266,
      "activations/layer17_attention_weight_max": 35.28071975708008,
      "activations/layer17_attention_weight_min": -26.182281494140625,
      "activations/layer18_attention_weight_max": 34.90330505371094,
      "activations/layer18_attention_weight_min": -26.02597999572754,
      "activations/layer19_attention_weight_max": 33.240966796875,
      "activations/layer19_attention_weight_min": -29.374774932861328,
      "activations/layer1_attention_weight_max": 17.962379455566406,
      "activations/layer1_attention_weight_min": -16.935043334960938,
      "activations/layer20_attention_weight_max": 31.090147018432617,
      "activations/layer20_attention_weight_min": -25.521944046020508,
      "activations/layer21_attention_weight_max": 29.78057098388672,
      "activations/layer21_attention_weight_min": -25.678258895874023,
      "activations/layer22_attention_weight_max": 44.7794303894043,
      "activations/layer22_attention_weight_min": -30.614776611328125,
      "activations/layer23_attention_weight_max": 35.18980407714844,
      "activations/layer23_attention_weight_min": -26.789520263671875,
      "activations/layer2_attention_weight_max": 33.59306335449219,
      "activations/layer2_attention_weight_min": -32.149879455566406,
      "activations/layer3_attention_weight_max": 96.87175750732422,
      "activations/layer3_attention_weight_min": -94.09626770019531,
      "activations/layer4_attention_weight_max": 113.02117156982422,
      "activations/layer4_attention_weight_min": -112.84223937988281,
      "activations/layer5_attention_weight_max": 51.64152145385742,
      "activations/layer5_attention_weight_min": -60.48517990112305,
      "activations/layer6_attention_weight_max": 45.77943801879883,
      "activations/layer6_attention_weight_min": -50.94546127319336,
      "activations/layer7_attention_weight_max": 89.14637756347656,
      "activations/layer7_attention_weight_min": -98.4456558227539,
      "activations/layer8_attention_weight_max": 44.1747932434082,
      "activations/layer8_attention_weight_min": -47.70949172973633,
      "activations/layer9_attention_weight_max": 36.89936828613281,
      "activations/layer9_attention_weight_min": -37.263423919677734,
      "epoch": 12.34,
      "learning_rate": 7.116590909090908e-05,
      "loss": 2.7752,
      "step": 212300
    },
    {
      "activations/layer0_attention_weight_max": 15.973204612731934,
      "activations/layer0_attention_weight_min": -12.212607383728027,
      "activations/layer10_attention_weight_max": 34.63725280761719,
      "activations/layer10_attention_weight_min": -35.5743293762207,
      "activations/layer11_attention_weight_max": 32.96086883544922,
      "activations/layer11_attention_weight_min": -32.89630126953125,
      "activations/layer12_attention_weight_max": 26.27424430847168,
      "activations/layer12_attention_weight_min": -24.800575256347656,
      "activations/layer13_attention_weight_max": 38.35247039794922,
      "activations/layer13_attention_weight_min": -34.5921745300293,
      "activations/layer14_attention_weight_max": 44.23443603515625,
      "activations/layer14_attention_weight_min": -40.27750778198242,
      "activations/layer15_attention_weight_max": 35.477500915527344,
      "activations/layer15_attention_weight_min": -30.851715087890625,
      "activations/layer16_attention_weight_max": 29.902009963989258,
      "activations/layer16_attention_weight_min": -26.40941047668457,
      "activations/layer17_attention_weight_max": 30.70418357849121,
      "activations/layer17_attention_weight_min": -24.771066665649414,
      "activations/layer18_attention_weight_max": 30.97380256652832,
      "activations/layer18_attention_weight_min": -24.10152816772461,
      "activations/layer19_attention_weight_max": 34.98061752319336,
      "activations/layer19_attention_weight_min": -31.09754753112793,
      "activations/layer1_attention_weight_max": 16.328487396240234,
      "activations/layer1_attention_weight_min": -14.998144149780273,
      "activations/layer20_attention_weight_max": 29.787572860717773,
      "activations/layer20_attention_weight_min": -23.292741775512695,
      "activations/layer21_attention_weight_max": 29.758556365966797,
      "activations/layer21_attention_weight_min": -23.832645416259766,
      "activations/layer22_attention_weight_max": 48.104854583740234,
      "activations/layer22_attention_weight_min": -29.74818992614746,
      "activations/layer23_attention_weight_max": 36.054481506347656,
      "activations/layer23_attention_weight_min": -25.59356689453125,
      "activations/layer2_attention_weight_max": 34.32756805419922,
      "activations/layer2_attention_weight_min": -34.0791015625,
      "activations/layer3_attention_weight_max": 99.33584594726562,
      "activations/layer3_attention_weight_min": -99.48464965820312,
      "activations/layer4_attention_weight_max": 115.327880859375,
      "activations/layer4_attention_weight_min": -111.96285247802734,
      "activations/layer5_attention_weight_max": 52.03811264038086,
      "activations/layer5_attention_weight_min": -59.36589813232422,
      "activations/layer6_attention_weight_max": 47.96306228637695,
      "activations/layer6_attention_weight_min": -49.37282180786133,
      "activations/layer7_attention_weight_max": 91.04884338378906,
      "activations/layer7_attention_weight_min": -91.8472671508789,
      "activations/layer8_attention_weight_max": 40.99565505981445,
      "activations/layer8_attention_weight_min": -42.765228271484375,
      "activations/layer9_attention_weight_max": 36.63117218017578,
      "activations/layer9_attention_weight_min": -35.69588851928711,
      "epoch": 12.34,
      "learning_rate": 7.11469696969697e-05,
      "loss": 2.7765,
      "step": 212350
    },
    {
      "activations/layer0_attention_weight_max": 15.906477928161621,
      "activations/layer0_attention_weight_min": -12.215727806091309,
      "activations/layer10_attention_weight_max": 34.04463577270508,
      "activations/layer10_attention_weight_min": -34.805782318115234,
      "activations/layer11_attention_weight_max": 32.05471420288086,
      "activations/layer11_attention_weight_min": -32.234352111816406,
      "activations/layer12_attention_weight_max": 34.39775466918945,
      "activations/layer12_attention_weight_min": -33.981510162353516,
      "activations/layer13_attention_weight_max": 43.73142623901367,
      "activations/layer13_attention_weight_min": -34.49629592895508,
      "activations/layer14_attention_weight_max": 44.14250946044922,
      "activations/layer14_attention_weight_min": -36.33597183227539,
      "activations/layer15_attention_weight_max": 43.70843505859375,
      "activations/layer15_attention_weight_min": -35.15612030029297,
      "activations/layer16_attention_weight_max": 33.06462478637695,
      "activations/layer16_attention_weight_min": -29.546478271484375,
      "activations/layer17_attention_weight_max": 35.18851852416992,
      "activations/layer17_attention_weight_min": -25.738420486450195,
      "activations/layer18_attention_weight_max": 31.983417510986328,
      "activations/layer18_attention_weight_min": -24.110740661621094,
      "activations/layer19_attention_weight_max": 37.2132682800293,
      "activations/layer19_attention_weight_min": -30.554672241210938,
      "activations/layer1_attention_weight_max": 17.731454849243164,
      "activations/layer1_attention_weight_min": -14.714044570922852,
      "activations/layer20_attention_weight_max": 30.47255516052246,
      "activations/layer20_attention_weight_min": -23.972002029418945,
      "activations/layer21_attention_weight_max": 30.35335350036621,
      "activations/layer21_attention_weight_min": -23.955785751342773,
      "activations/layer22_attention_weight_max": 49.563865661621094,
      "activations/layer22_attention_weight_min": -30.34691047668457,
      "activations/layer23_attention_weight_max": 37.530826568603516,
      "activations/layer23_attention_weight_min": -23.887882232666016,
      "activations/layer2_attention_weight_max": 33.04398727416992,
      "activations/layer2_attention_weight_min": -34.82146072387695,
      "activations/layer3_attention_weight_max": 100.96891021728516,
      "activations/layer3_attention_weight_min": -101.33296203613281,
      "activations/layer4_attention_weight_max": 117.4635009765625,
      "activations/layer4_attention_weight_min": -116.0562744140625,
      "activations/layer5_attention_weight_max": 55.59613800048828,
      "activations/layer5_attention_weight_min": -66.76182556152344,
      "activations/layer6_attention_weight_max": 44.87221908569336,
      "activations/layer6_attention_weight_min": -48.43299102783203,
      "activations/layer7_attention_weight_max": 93.69793701171875,
      "activations/layer7_attention_weight_min": -93.45173645019531,
      "activations/layer8_attention_weight_max": 42.730628967285156,
      "activations/layer8_attention_weight_min": -43.88208770751953,
      "activations/layer9_attention_weight_max": 33.74106979370117,
      "activations/layer9_attention_weight_min": -38.925899505615234,
      "epoch": 12.34,
      "learning_rate": 7.112840909090908e-05,
      "loss": 2.7825,
      "step": 212400
    },
    {
      "activations/layer0_attention_weight_max": 15.1233549118042,
      "activations/layer0_attention_weight_min": -11.894057273864746,
      "activations/layer10_attention_weight_max": 32.141761779785156,
      "activations/layer10_attention_weight_min": -31.924755096435547,
      "activations/layer11_attention_weight_max": 33.09002685546875,
      "activations/layer11_attention_weight_min": -31.3701171875,
      "activations/layer12_attention_weight_max": 26.002626419067383,
      "activations/layer12_attention_weight_min": -25.819063186645508,
      "activations/layer13_attention_weight_max": 39.83628845214844,
      "activations/layer13_attention_weight_min": -35.83053207397461,
      "activations/layer14_attention_weight_max": 46.98169708251953,
      "activations/layer14_attention_weight_min": -37.55942916870117,
      "activations/layer15_attention_weight_max": 36.35026550292969,
      "activations/layer15_attention_weight_min": -33.182498931884766,
      "activations/layer16_attention_weight_max": 30.304231643676758,
      "activations/layer16_attention_weight_min": -28.05369758605957,
      "activations/layer17_attention_weight_max": 30.334707260131836,
      "activations/layer17_attention_weight_min": -25.95191192626953,
      "activations/layer18_attention_weight_max": 30.714927673339844,
      "activations/layer18_attention_weight_min": -24.654436111450195,
      "activations/layer19_attention_weight_max": 34.58329391479492,
      "activations/layer19_attention_weight_min": -30.61572265625,
      "activations/layer1_attention_weight_max": 16.7034969329834,
      "activations/layer1_attention_weight_min": -14.933049201965332,
      "activations/layer20_attention_weight_max": 27.265716552734375,
      "activations/layer20_attention_weight_min": -25.131309509277344,
      "activations/layer21_attention_weight_max": 27.418331146240234,
      "activations/layer21_attention_weight_min": -24.010704040527344,
      "activations/layer22_attention_weight_max": 48.12879180908203,
      "activations/layer22_attention_weight_min": -30.705341339111328,
      "activations/layer23_attention_weight_max": 36.27030944824219,
      "activations/layer23_attention_weight_min": -25.134044647216797,
      "activations/layer2_attention_weight_max": 33.69697570800781,
      "activations/layer2_attention_weight_min": -35.8258056640625,
      "activations/layer3_attention_weight_max": 94.71055603027344,
      "activations/layer3_attention_weight_min": -100.64289855957031,
      "activations/layer4_attention_weight_max": 104.48514556884766,
      "activations/layer4_attention_weight_min": -109.22029113769531,
      "activations/layer5_attention_weight_max": 52.90134811401367,
      "activations/layer5_attention_weight_min": -57.26760482788086,
      "activations/layer6_attention_weight_max": 43.89296340942383,
      "activations/layer6_attention_weight_min": -46.51637268066406,
      "activations/layer7_attention_weight_max": 93.56733703613281,
      "activations/layer7_attention_weight_min": -87.70077514648438,
      "activations/layer8_attention_weight_max": 41.00258255004883,
      "activations/layer8_attention_weight_min": -41.10277557373047,
      "activations/layer9_attention_weight_max": 34.153564453125,
      "activations/layer9_attention_weight_min": -31.996301651000977,
      "epoch": 12.34,
      "learning_rate": 7.110946969696969e-05,
      "loss": 2.7615,
      "step": 212450
    },
    {
      "activations/layer0_attention_weight_max": 16.350482940673828,
      "activations/layer0_attention_weight_min": -12.076643943786621,
      "activations/layer10_attention_weight_max": 31.914457321166992,
      "activations/layer10_attention_weight_min": -30.83815574645996,
      "activations/layer11_attention_weight_max": 32.51356887817383,
      "activations/layer11_attention_weight_min": -31.271833419799805,
      "activations/layer12_attention_weight_max": 23.03730010986328,
      "activations/layer12_attention_weight_min": -26.14972686767578,
      "activations/layer13_attention_weight_max": 35.923583984375,
      "activations/layer13_attention_weight_min": -32.045166015625,
      "activations/layer14_attention_weight_max": 40.863155364990234,
      "activations/layer14_attention_weight_min": -34.19164276123047,
      "activations/layer15_attention_weight_max": 37.42356491088867,
      "activations/layer15_attention_weight_min": -31.63544273376465,
      "activations/layer16_attention_weight_max": 31.668176651000977,
      "activations/layer16_attention_weight_min": -28.66355323791504,
      "activations/layer17_attention_weight_max": 30.801565170288086,
      "activations/layer17_attention_weight_min": -27.033470153808594,
      "activations/layer18_attention_weight_max": 33.9940299987793,
      "activations/layer18_attention_weight_min": -26.40420913696289,
      "activations/layer19_attention_weight_max": 33.43729782104492,
      "activations/layer19_attention_weight_min": -30.694040298461914,
      "activations/layer1_attention_weight_max": 15.884974479675293,
      "activations/layer1_attention_weight_min": -14.478008270263672,
      "activations/layer20_attention_weight_max": 26.61309051513672,
      "activations/layer20_attention_weight_min": -26.601024627685547,
      "activations/layer21_attention_weight_max": 28.64894676208496,
      "activations/layer21_attention_weight_min": -24.824552536010742,
      "activations/layer22_attention_weight_max": 52.0150032043457,
      "activations/layer22_attention_weight_min": -30.69571304321289,
      "activations/layer23_attention_weight_max": 37.85407257080078,
      "activations/layer23_attention_weight_min": -26.95973777770996,
      "activations/layer2_attention_weight_max": 34.16423797607422,
      "activations/layer2_attention_weight_min": -32.067745208740234,
      "activations/layer3_attention_weight_max": 93.84699249267578,
      "activations/layer3_attention_weight_min": -92.64468383789062,
      "activations/layer4_attention_weight_max": 109.42420959472656,
      "activations/layer4_attention_weight_min": -106.12568664550781,
      "activations/layer5_attention_weight_max": 51.52330017089844,
      "activations/layer5_attention_weight_min": -62.52218246459961,
      "activations/layer6_attention_weight_max": 45.46437454223633,
      "activations/layer6_attention_weight_min": -45.08085250854492,
      "activations/layer7_attention_weight_max": 85.558349609375,
      "activations/layer7_attention_weight_min": -89.31462097167969,
      "activations/layer8_attention_weight_max": 37.234737396240234,
      "activations/layer8_attention_weight_min": -39.28559112548828,
      "activations/layer9_attention_weight_max": 31.198942184448242,
      "activations/layer9_attention_weight_min": -31.800256729125977,
      "epoch": 12.35,
      "learning_rate": 7.10905303030303e-05,
      "loss": 2.7634,
      "step": 212500
    },
    {
      "activations/layer0_attention_weight_max": 16.13365364074707,
      "activations/layer0_attention_weight_min": -11.804665565490723,
      "activations/layer10_attention_weight_max": 32.695960998535156,
      "activations/layer10_attention_weight_min": -31.559667587280273,
      "activations/layer11_attention_weight_max": 36.653018951416016,
      "activations/layer11_attention_weight_min": -30.725194931030273,
      "activations/layer12_attention_weight_max": 40.22006607055664,
      "activations/layer12_attention_weight_min": -25.994604110717773,
      "activations/layer13_attention_weight_max": 54.33781814575195,
      "activations/layer13_attention_weight_min": -34.78422546386719,
      "activations/layer14_attention_weight_max": 41.83911895751953,
      "activations/layer14_attention_weight_min": -37.286338806152344,
      "activations/layer15_attention_weight_max": 46.029808044433594,
      "activations/layer15_attention_weight_min": -32.568397521972656,
      "activations/layer16_attention_weight_max": 29.6956729888916,
      "activations/layer16_attention_weight_min": -27.108051300048828,
      "activations/layer17_attention_weight_max": 30.031612396240234,
      "activations/layer17_attention_weight_min": -24.715003967285156,
      "activations/layer18_attention_weight_max": 30.84031867980957,
      "activations/layer18_attention_weight_min": -25.016021728515625,
      "activations/layer19_attention_weight_max": 32.23008728027344,
      "activations/layer19_attention_weight_min": -31.59663963317871,
      "activations/layer1_attention_weight_max": 17.296401977539062,
      "activations/layer1_attention_weight_min": -16.51279067993164,
      "activations/layer20_attention_weight_max": 29.75698471069336,
      "activations/layer20_attention_weight_min": -26.16884422302246,
      "activations/layer21_attention_weight_max": 28.982316970825195,
      "activations/layer21_attention_weight_min": -26.432729721069336,
      "activations/layer22_attention_weight_max": 41.64002227783203,
      "activations/layer22_attention_weight_min": -31.66127586364746,
      "activations/layer23_attention_weight_max": 34.708045959472656,
      "activations/layer23_attention_weight_min": -25.4985408782959,
      "activations/layer2_attention_weight_max": 34.504615783691406,
      "activations/layer2_attention_weight_min": -32.190185546875,
      "activations/layer3_attention_weight_max": 98.57621765136719,
      "activations/layer3_attention_weight_min": -94.20941162109375,
      "activations/layer4_attention_weight_max": 107.66156005859375,
      "activations/layer4_attention_weight_min": -108.53016662597656,
      "activations/layer5_attention_weight_max": 49.041866302490234,
      "activations/layer5_attention_weight_min": -59.42837142944336,
      "activations/layer6_attention_weight_max": 43.18642807006836,
      "activations/layer6_attention_weight_min": -46.81560134887695,
      "activations/layer7_attention_weight_max": 93.7575454711914,
      "activations/layer7_attention_weight_min": -87.8359603881836,
      "activations/layer8_attention_weight_max": 39.24998092651367,
      "activations/layer8_attention_weight_min": -41.96919250488281,
      "activations/layer9_attention_weight_max": 33.14682388305664,
      "activations/layer9_attention_weight_min": -31.77646827697754,
      "epoch": 12.35,
      "learning_rate": 7.10715909090909e-05,
      "loss": 2.7711,
      "step": 212550
    },
    {
      "activations/layer0_attention_weight_max": 16.280012130737305,
      "activations/layer0_attention_weight_min": -12.5014066696167,
      "activations/layer10_attention_weight_max": 39.013484954833984,
      "activations/layer10_attention_weight_min": -35.22159194946289,
      "activations/layer11_attention_weight_max": 41.16343688964844,
      "activations/layer11_attention_weight_min": -38.18946075439453,
      "activations/layer12_attention_weight_max": 32.00946807861328,
      "activations/layer12_attention_weight_min": -30.192623138427734,
      "activations/layer13_attention_weight_max": 49.03009796142578,
      "activations/layer13_attention_weight_min": -38.11916732788086,
      "activations/layer14_attention_weight_max": 66.58690643310547,
      "activations/layer14_attention_weight_min": -44.84870147705078,
      "activations/layer15_attention_weight_max": 49.45823287963867,
      "activations/layer15_attention_weight_min": -36.94445037841797,
      "activations/layer16_attention_weight_max": 37.01127243041992,
      "activations/layer16_attention_weight_min": -29.1578369140625,
      "activations/layer17_attention_weight_max": 36.27455139160156,
      "activations/layer17_attention_weight_min": -27.43915557861328,
      "activations/layer18_attention_weight_max": 34.76813507080078,
      "activations/layer18_attention_weight_min": -24.75740623474121,
      "activations/layer19_attention_weight_max": 39.20136260986328,
      "activations/layer19_attention_weight_min": -33.82915496826172,
      "activations/layer1_attention_weight_max": 16.689228057861328,
      "activations/layer1_attention_weight_min": -14.444246292114258,
      "activations/layer20_attention_weight_max": 33.84903335571289,
      "activations/layer20_attention_weight_min": -26.602195739746094,
      "activations/layer21_attention_weight_max": 29.954648971557617,
      "activations/layer21_attention_weight_min": -25.07941246032715,
      "activations/layer22_attention_weight_max": 50.21146011352539,
      "activations/layer22_attention_weight_min": -31.807777404785156,
      "activations/layer23_attention_weight_max": 37.50458526611328,
      "activations/layer23_attention_weight_min": -25.783241271972656,
      "activations/layer2_attention_weight_max": 33.8856201171875,
      "activations/layer2_attention_weight_min": -34.00743865966797,
      "activations/layer3_attention_weight_max": 93.47579956054688,
      "activations/layer3_attention_weight_min": -90.66303253173828,
      "activations/layer4_attention_weight_max": 110.7839126586914,
      "activations/layer4_attention_weight_min": -108.56715393066406,
      "activations/layer5_attention_weight_max": 52.46522521972656,
      "activations/layer5_attention_weight_min": -63.40296936035156,
      "activations/layer6_attention_weight_max": 43.96711730957031,
      "activations/layer6_attention_weight_min": -46.9911994934082,
      "activations/layer7_attention_weight_max": 103.13542175292969,
      "activations/layer7_attention_weight_min": -91.14691925048828,
      "activations/layer8_attention_weight_max": 44.62556838989258,
      "activations/layer8_attention_weight_min": -42.16337966918945,
      "activations/layer9_attention_weight_max": 39.22495651245117,
      "activations/layer9_attention_weight_min": -36.62989807128906,
      "epoch": 12.35,
      "learning_rate": 7.105265151515152e-05,
      "loss": 2.778,
      "step": 212600
    },
    {
      "activations/layer0_attention_weight_max": 16.469667434692383,
      "activations/layer0_attention_weight_min": -12.293197631835938,
      "activations/layer10_attention_weight_max": 30.13785743713379,
      "activations/layer10_attention_weight_min": -30.26549530029297,
      "activations/layer11_attention_weight_max": 31.5053768157959,
      "activations/layer11_attention_weight_min": -30.24085235595703,
      "activations/layer12_attention_weight_max": 23.27578353881836,
      "activations/layer12_attention_weight_min": -25.24612045288086,
      "activations/layer13_attention_weight_max": 35.057044982910156,
      "activations/layer13_attention_weight_min": -36.230587005615234,
      "activations/layer14_attention_weight_max": 44.686893463134766,
      "activations/layer14_attention_weight_min": -36.61796188354492,
      "activations/layer15_attention_weight_max": 34.560829162597656,
      "activations/layer15_attention_weight_min": -31.016672134399414,
      "activations/layer16_attention_weight_max": 32.26839065551758,
      "activations/layer16_attention_weight_min": -28.19017791748047,
      "activations/layer17_attention_weight_max": 28.33156967163086,
      "activations/layer17_attention_weight_min": -26.040151596069336,
      "activations/layer18_attention_weight_max": 30.504854202270508,
      "activations/layer18_attention_weight_min": -27.008459091186523,
      "activations/layer19_attention_weight_max": 33.475189208984375,
      "activations/layer19_attention_weight_min": -32.1010856628418,
      "activations/layer1_attention_weight_max": 16.368797302246094,
      "activations/layer1_attention_weight_min": -14.050332069396973,
      "activations/layer20_attention_weight_max": 27.231172561645508,
      "activations/layer20_attention_weight_min": -27.192092895507812,
      "activations/layer21_attention_weight_max": 27.7620849609375,
      "activations/layer21_attention_weight_min": -24.66472625732422,
      "activations/layer22_attention_weight_max": 43.00312805175781,
      "activations/layer22_attention_weight_min": -30.682723999023438,
      "activations/layer23_attention_weight_max": 33.315879821777344,
      "activations/layer23_attention_weight_min": -26.098968505859375,
      "activations/layer2_attention_weight_max": 31.944377899169922,
      "activations/layer2_attention_weight_min": -31.136547088623047,
      "activations/layer3_attention_weight_max": 90.77959442138672,
      "activations/layer3_attention_weight_min": -88.64539337158203,
      "activations/layer4_attention_weight_max": 104.6356430053711,
      "activations/layer4_attention_weight_min": -103.94490814208984,
      "activations/layer5_attention_weight_max": 48.77661895751953,
      "activations/layer5_attention_weight_min": -58.051937103271484,
      "activations/layer6_attention_weight_max": 40.70680236816406,
      "activations/layer6_attention_weight_min": -45.54316711425781,
      "activations/layer7_attention_weight_max": 83.79896545410156,
      "activations/layer7_attention_weight_min": -86.72960662841797,
      "activations/layer8_attention_weight_max": 37.53006362915039,
      "activations/layer8_attention_weight_min": -38.63351058959961,
      "activations/layer9_attention_weight_max": 30.751840591430664,
      "activations/layer9_attention_weight_min": -30.110595703125,
      "epoch": 12.36,
      "learning_rate": 7.103371212121212e-05,
      "loss": 2.763,
      "step": 212650
    },
    {
      "activations/layer0_attention_weight_max": 16.69846534729004,
      "activations/layer0_attention_weight_min": -11.834402084350586,
      "activations/layer10_attention_weight_max": 28.911602020263672,
      "activations/layer10_attention_weight_min": -31.471677780151367,
      "activations/layer11_attention_weight_max": 29.75295639038086,
      "activations/layer11_attention_weight_min": -32.45045471191406,
      "activations/layer12_attention_weight_max": 24.1273136138916,
      "activations/layer12_attention_weight_min": -24.991867065429688,
      "activations/layer13_attention_weight_max": 36.56264877319336,
      "activations/layer13_attention_weight_min": -33.22069549560547,
      "activations/layer14_attention_weight_max": 42.64497756958008,
      "activations/layer14_attention_weight_min": -38.847347259521484,
      "activations/layer15_attention_weight_max": 34.94001388549805,
      "activations/layer15_attention_weight_min": -31.47198486328125,
      "activations/layer16_attention_weight_max": 33.80086898803711,
      "activations/layer16_attention_weight_min": -27.997051239013672,
      "activations/layer17_attention_weight_max": 28.5339298248291,
      "activations/layer17_attention_weight_min": -24.369522094726562,
      "activations/layer18_attention_weight_max": 30.113862991333008,
      "activations/layer18_attention_weight_min": -25.313810348510742,
      "activations/layer19_attention_weight_max": 37.54421615600586,
      "activations/layer19_attention_weight_min": -31.94346046447754,
      "activations/layer1_attention_weight_max": 15.971708297729492,
      "activations/layer1_attention_weight_min": -15.463235855102539,
      "activations/layer20_attention_weight_max": 31.34050750732422,
      "activations/layer20_attention_weight_min": -24.12586784362793,
      "activations/layer21_attention_weight_max": 27.702041625976562,
      "activations/layer21_attention_weight_min": -22.707138061523438,
      "activations/layer22_attention_weight_max": 38.995750427246094,
      "activations/layer22_attention_weight_min": -28.329580307006836,
      "activations/layer23_attention_weight_max": 30.337718963623047,
      "activations/layer23_attention_weight_min": -22.955997467041016,
      "activations/layer2_attention_weight_max": 31.917516708374023,
      "activations/layer2_attention_weight_min": -31.63860321044922,
      "activations/layer3_attention_weight_max": 93.78046417236328,
      "activations/layer3_attention_weight_min": -90.47938537597656,
      "activations/layer4_attention_weight_max": 108.12108612060547,
      "activations/layer4_attention_weight_min": -110.07341766357422,
      "activations/layer5_attention_weight_max": 49.30051803588867,
      "activations/layer5_attention_weight_min": -61.620845794677734,
      "activations/layer6_attention_weight_max": 43.26523971557617,
      "activations/layer6_attention_weight_min": -48.26595687866211,
      "activations/layer7_attention_weight_max": 90.4326400756836,
      "activations/layer7_attention_weight_min": -89.152587890625,
      "activations/layer8_attention_weight_max": 40.62489700317383,
      "activations/layer8_attention_weight_min": -42.01466751098633,
      "activations/layer9_attention_weight_max": 31.585086822509766,
      "activations/layer9_attention_weight_min": -32.72417068481445,
      "epoch": 12.36,
      "learning_rate": 7.101477272727272e-05,
      "loss": 2.7829,
      "step": 212700
    },
    {
      "activations/layer0_attention_weight_max": 16.180397033691406,
      "activations/layer0_attention_weight_min": -12.133880615234375,
      "activations/layer10_attention_weight_max": 31.316129684448242,
      "activations/layer10_attention_weight_min": -30.158889770507812,
      "activations/layer11_attention_weight_max": 32.766876220703125,
      "activations/layer11_attention_weight_min": -30.295883178710938,
      "activations/layer12_attention_weight_max": 27.732999801635742,
      "activations/layer12_attention_weight_min": -24.199377059936523,
      "activations/layer13_attention_weight_max": 34.75991439819336,
      "activations/layer13_attention_weight_min": -33.04606628417969,
      "activations/layer14_attention_weight_max": 39.95304489135742,
      "activations/layer14_attention_weight_min": -36.2006721496582,
      "activations/layer15_attention_weight_max": 37.794273376464844,
      "activations/layer15_attention_weight_min": -32.780303955078125,
      "activations/layer16_attention_weight_max": 29.93896484375,
      "activations/layer16_attention_weight_min": -28.170427322387695,
      "activations/layer17_attention_weight_max": 31.31451416015625,
      "activations/layer17_attention_weight_min": -25.039731979370117,
      "activations/layer18_attention_weight_max": 29.521610260009766,
      "activations/layer18_attention_weight_min": -23.124916076660156,
      "activations/layer19_attention_weight_max": 34.08344650268555,
      "activations/layer19_attention_weight_min": -30.240909576416016,
      "activations/layer1_attention_weight_max": 16.328805923461914,
      "activations/layer1_attention_weight_min": -12.94459342956543,
      "activations/layer20_attention_weight_max": 29.07781219482422,
      "activations/layer20_attention_weight_min": -23.636014938354492,
      "activations/layer21_attention_weight_max": 26.699565887451172,
      "activations/layer21_attention_weight_min": -22.748971939086914,
      "activations/layer22_attention_weight_max": 42.43464279174805,
      "activations/layer22_attention_weight_min": -31.12008285522461,
      "activations/layer23_attention_weight_max": 36.5352668762207,
      "activations/layer23_attention_weight_min": -25.586997985839844,
      "activations/layer2_attention_weight_max": 29.603282928466797,
      "activations/layer2_attention_weight_min": -29.177001953125,
      "activations/layer3_attention_weight_max": 89.764404296875,
      "activations/layer3_attention_weight_min": -87.08467864990234,
      "activations/layer4_attention_weight_max": 111.4436264038086,
      "activations/layer4_attention_weight_min": -102.23653411865234,
      "activations/layer5_attention_weight_max": 51.035850524902344,
      "activations/layer5_attention_weight_min": -57.584197998046875,
      "activations/layer6_attention_weight_max": 44.114803314208984,
      "activations/layer6_attention_weight_min": -44.00428771972656,
      "activations/layer7_attention_weight_max": 85.25794982910156,
      "activations/layer7_attention_weight_min": -80.80448913574219,
      "activations/layer8_attention_weight_max": 39.65205001831055,
      "activations/layer8_attention_weight_min": -36.64625549316406,
      "activations/layer9_attention_weight_max": 33.27025604248047,
      "activations/layer9_attention_weight_min": -30.447256088256836,
      "epoch": 12.36,
      "learning_rate": 7.099583333333332e-05,
      "loss": 2.7712,
      "step": 212750
    },
    {
      "activations/layer0_attention_weight_max": 15.241634368896484,
      "activations/layer0_attention_weight_min": -12.835296630859375,
      "activations/layer10_attention_weight_max": 32.723487854003906,
      "activations/layer10_attention_weight_min": -33.87141799926758,
      "activations/layer11_attention_weight_max": 34.448116302490234,
      "activations/layer11_attention_weight_min": -32.65534973144531,
      "activations/layer12_attention_weight_max": 23.471054077148438,
      "activations/layer12_attention_weight_min": -24.622737884521484,
      "activations/layer13_attention_weight_max": 34.443275451660156,
      "activations/layer13_attention_weight_min": -33.19831085205078,
      "activations/layer14_attention_weight_max": 37.96622085571289,
      "activations/layer14_attention_weight_min": -36.81678771972656,
      "activations/layer15_attention_weight_max": 33.69447708129883,
      "activations/layer15_attention_weight_min": -31.254718780517578,
      "activations/layer16_attention_weight_max": 33.938846588134766,
      "activations/layer16_attention_weight_min": -28.537700653076172,
      "activations/layer17_attention_weight_max": 32.570743560791016,
      "activations/layer17_attention_weight_min": -25.081701278686523,
      "activations/layer18_attention_weight_max": 31.866989135742188,
      "activations/layer18_attention_weight_min": -23.301931381225586,
      "activations/layer19_attention_weight_max": 35.3001708984375,
      "activations/layer19_attention_weight_min": -30.205724716186523,
      "activations/layer1_attention_weight_max": 16.877164840698242,
      "activations/layer1_attention_weight_min": -13.295202255249023,
      "activations/layer20_attention_weight_max": 32.117530822753906,
      "activations/layer20_attention_weight_min": -24.102663040161133,
      "activations/layer21_attention_weight_max": 29.567276000976562,
      "activations/layer21_attention_weight_min": -23.371780395507812,
      "activations/layer22_attention_weight_max": 42.75677490234375,
      "activations/layer22_attention_weight_min": -29.928701400756836,
      "activations/layer23_attention_weight_max": 35.386619567871094,
      "activations/layer23_attention_weight_min": -30.03030014038086,
      "activations/layer2_attention_weight_max": 31.57924461364746,
      "activations/layer2_attention_weight_min": -31.225942611694336,
      "activations/layer3_attention_weight_max": 92.98517608642578,
      "activations/layer3_attention_weight_min": -97.9236831665039,
      "activations/layer4_attention_weight_max": 110.546142578125,
      "activations/layer4_attention_weight_min": -110.76457977294922,
      "activations/layer5_attention_weight_max": 50.157981872558594,
      "activations/layer5_attention_weight_min": -60.862030029296875,
      "activations/layer6_attention_weight_max": 43.47231674194336,
      "activations/layer6_attention_weight_min": -46.295780181884766,
      "activations/layer7_attention_weight_max": 89.9979019165039,
      "activations/layer7_attention_weight_min": -90.50991821289062,
      "activations/layer8_attention_weight_max": 43.38486099243164,
      "activations/layer8_attention_weight_min": -39.85569763183594,
      "activations/layer9_attention_weight_max": 33.559837341308594,
      "activations/layer9_attention_weight_min": -33.76347732543945,
      "epoch": 12.36,
      "learning_rate": 7.097689393939394e-05,
      "loss": 2.7949,
      "step": 212800
    },
    {
      "activations/layer0_attention_weight_max": 16.222515106201172,
      "activations/layer0_attention_weight_min": -12.616220474243164,
      "activations/layer10_attention_weight_max": 34.65626907348633,
      "activations/layer10_attention_weight_min": -34.5279541015625,
      "activations/layer11_attention_weight_max": 34.237918853759766,
      "activations/layer11_attention_weight_min": -32.0035514831543,
      "activations/layer12_attention_weight_max": 25.655698776245117,
      "activations/layer12_attention_weight_min": -25.477251052856445,
      "activations/layer13_attention_weight_max": 38.3764533996582,
      "activations/layer13_attention_weight_min": -34.771575927734375,
      "activations/layer14_attention_weight_max": 42.736656188964844,
      "activations/layer14_attention_weight_min": -36.46031951904297,
      "activations/layer15_attention_weight_max": 37.50973892211914,
      "activations/layer15_attention_weight_min": -31.37645721435547,
      "activations/layer16_attention_weight_max": 33.89322280883789,
      "activations/layer16_attention_weight_min": -27.441923141479492,
      "activations/layer17_attention_weight_max": 32.657779693603516,
      "activations/layer17_attention_weight_min": -25.383533477783203,
      "activations/layer18_attention_weight_max": 31.387104034423828,
      "activations/layer18_attention_weight_min": -23.177310943603516,
      "activations/layer19_attention_weight_max": 38.36962890625,
      "activations/layer19_attention_weight_min": -29.695669174194336,
      "activations/layer1_attention_weight_max": 16.646696090698242,
      "activations/layer1_attention_weight_min": -14.266355514526367,
      "activations/layer20_attention_weight_max": 30.315799713134766,
      "activations/layer20_attention_weight_min": -27.332569122314453,
      "activations/layer21_attention_weight_max": 31.194534301757812,
      "activations/layer21_attention_weight_min": -26.37514877319336,
      "activations/layer22_attention_weight_max": 54.13852310180664,
      "activations/layer22_attention_weight_min": -31.779186248779297,
      "activations/layer23_attention_weight_max": 40.17233657836914,
      "activations/layer23_attention_weight_min": -25.713171005249023,
      "activations/layer2_attention_weight_max": 32.70286560058594,
      "activations/layer2_attention_weight_min": -29.711393356323242,
      "activations/layer3_attention_weight_max": 90.74785614013672,
      "activations/layer3_attention_weight_min": -86.7095718383789,
      "activations/layer4_attention_weight_max": 106.8431396484375,
      "activations/layer4_attention_weight_min": -103.17516326904297,
      "activations/layer5_attention_weight_max": 50.424861907958984,
      "activations/layer5_attention_weight_min": -65.5711669921875,
      "activations/layer6_attention_weight_max": 44.38808059692383,
      "activations/layer6_attention_weight_min": -46.511451721191406,
      "activations/layer7_attention_weight_max": 89.52216339111328,
      "activations/layer7_attention_weight_min": -86.39917755126953,
      "activations/layer8_attention_weight_max": 42.04671096801758,
      "activations/layer8_attention_weight_min": -42.49675369262695,
      "activations/layer9_attention_weight_max": 34.540775299072266,
      "activations/layer9_attention_weight_min": -34.41019058227539,
      "epoch": 12.37,
      "learning_rate": 7.095795454545454e-05,
      "loss": 2.7802,
      "step": 212850
    },
    {
      "activations/layer0_attention_weight_max": 16.61517906188965,
      "activations/layer0_attention_weight_min": -12.441650390625,
      "activations/layer10_attention_weight_max": 30.91118621826172,
      "activations/layer10_attention_weight_min": -33.28069305419922,
      "activations/layer11_attention_weight_max": 32.31005096435547,
      "activations/layer11_attention_weight_min": -31.79113006591797,
      "activations/layer12_attention_weight_max": 23.56949806213379,
      "activations/layer12_attention_weight_min": -24.69640350341797,
      "activations/layer13_attention_weight_max": 36.58074951171875,
      "activations/layer13_attention_weight_min": -32.02287673950195,
      "activations/layer14_attention_weight_max": 41.6231803894043,
      "activations/layer14_attention_weight_min": -36.47846984863281,
      "activations/layer15_attention_weight_max": 35.240360260009766,
      "activations/layer15_attention_weight_min": -32.392364501953125,
      "activations/layer16_attention_weight_max": 29.45454216003418,
      "activations/layer16_attention_weight_min": -28.405580520629883,
      "activations/layer17_attention_weight_max": 28.665340423583984,
      "activations/layer17_attention_weight_min": -24.49040985107422,
      "activations/layer18_attention_weight_max": 28.389102935791016,
      "activations/layer18_attention_weight_min": -22.385826110839844,
      "activations/layer19_attention_weight_max": 31.372329711914062,
      "activations/layer19_attention_weight_min": -29.448505401611328,
      "activations/layer1_attention_weight_max": 17.476594924926758,
      "activations/layer1_attention_weight_min": -13.555603981018066,
      "activations/layer20_attention_weight_max": 26.690288543701172,
      "activations/layer20_attention_weight_min": -23.08481216430664,
      "activations/layer21_attention_weight_max": 25.826824188232422,
      "activations/layer21_attention_weight_min": -21.407323837280273,
      "activations/layer22_attention_weight_max": 37.10232925415039,
      "activations/layer22_attention_weight_min": -26.36787986755371,
      "activations/layer23_attention_weight_max": 30.338720321655273,
      "activations/layer23_attention_weight_min": -22.528676986694336,
      "activations/layer2_attention_weight_max": 33.258548736572266,
      "activations/layer2_attention_weight_min": -31.585250854492188,
      "activations/layer3_attention_weight_max": 97.35082244873047,
      "activations/layer3_attention_weight_min": -96.01055908203125,
      "activations/layer4_attention_weight_max": 113.1919174194336,
      "activations/layer4_attention_weight_min": -109.4288330078125,
      "activations/layer5_attention_weight_max": 50.94248962402344,
      "activations/layer5_attention_weight_min": -60.39057159423828,
      "activations/layer6_attention_weight_max": 45.45481491088867,
      "activations/layer6_attention_weight_min": -47.1223258972168,
      "activations/layer7_attention_weight_max": 91.70362854003906,
      "activations/layer7_attention_weight_min": -98.54145812988281,
      "activations/layer8_attention_weight_max": 40.55382537841797,
      "activations/layer8_attention_weight_min": -44.13632583618164,
      "activations/layer9_attention_weight_max": 31.565134048461914,
      "activations/layer9_attention_weight_min": -34.39780807495117,
      "epoch": 12.37,
      "learning_rate": 7.093901515151514e-05,
      "loss": 2.7797,
      "step": 212900
    },
    {
      "activations/layer0_attention_weight_max": 15.374166488647461,
      "activations/layer0_attention_weight_min": -12.554652214050293,
      "activations/layer10_attention_weight_max": 31.87795639038086,
      "activations/layer10_attention_weight_min": -33.24641418457031,
      "activations/layer11_attention_weight_max": 30.457448959350586,
      "activations/layer11_attention_weight_min": -33.21052932739258,
      "activations/layer12_attention_weight_max": 24.172401428222656,
      "activations/layer12_attention_weight_min": -25.56310272216797,
      "activations/layer13_attention_weight_max": 36.984092712402344,
      "activations/layer13_attention_weight_min": -36.192989349365234,
      "activations/layer14_attention_weight_max": 41.07362365722656,
      "activations/layer14_attention_weight_min": -40.54648971557617,
      "activations/layer15_attention_weight_max": 37.59412384033203,
      "activations/layer15_attention_weight_min": -32.303043365478516,
      "activations/layer16_attention_weight_max": 31.32016944885254,
      "activations/layer16_attention_weight_min": -29.030359268188477,
      "activations/layer17_attention_weight_max": 33.38907241821289,
      "activations/layer17_attention_weight_min": -25.069494247436523,
      "activations/layer18_attention_weight_max": 35.06877517700195,
      "activations/layer18_attention_weight_min": -23.394411087036133,
      "activations/layer19_attention_weight_max": 38.21221923828125,
      "activations/layer19_attention_weight_min": -30.2540283203125,
      "activations/layer1_attention_weight_max": 16.43967056274414,
      "activations/layer1_attention_weight_min": -14.544679641723633,
      "activations/layer20_attention_weight_max": 34.23871612548828,
      "activations/layer20_attention_weight_min": -25.579668045043945,
      "activations/layer21_attention_weight_max": 36.729042053222656,
      "activations/layer21_attention_weight_min": -24.5506649017334,
      "activations/layer22_attention_weight_max": 47.098167419433594,
      "activations/layer22_attention_weight_min": -31.57456398010254,
      "activations/layer23_attention_weight_max": 40.3516845703125,
      "activations/layer23_attention_weight_min": -25.56822967529297,
      "activations/layer2_attention_weight_max": 33.07545471191406,
      "activations/layer2_attention_weight_min": -30.96367645263672,
      "activations/layer3_attention_weight_max": 91.00740051269531,
      "activations/layer3_attention_weight_min": -90.65473175048828,
      "activations/layer4_attention_weight_max": 106.95903778076172,
      "activations/layer4_attention_weight_min": -105.27082824707031,
      "activations/layer5_attention_weight_max": 49.98613739013672,
      "activations/layer5_attention_weight_min": -59.94525146484375,
      "activations/layer6_attention_weight_max": 42.50728988647461,
      "activations/layer6_attention_weight_min": -46.21284484863281,
      "activations/layer7_attention_weight_max": 85.39492797851562,
      "activations/layer7_attention_weight_min": -87.87683868408203,
      "activations/layer8_attention_weight_max": 40.100059509277344,
      "activations/layer8_attention_weight_min": -40.259578704833984,
      "activations/layer9_attention_weight_max": 30.63797950744629,
      "activations/layer9_attention_weight_min": -32.29519271850586,
      "epoch": 12.37,
      "learning_rate": 7.092007575757574e-05,
      "loss": 2.759,
      "step": 212950
    },
    {
      "activations/layer0_attention_weight_max": 15.927400588989258,
      "activations/layer0_attention_weight_min": -12.125130653381348,
      "activations/layer10_attention_weight_max": 46.797603607177734,
      "activations/layer10_attention_weight_min": -42.33317947387695,
      "activations/layer11_attention_weight_max": 46.642147064208984,
      "activations/layer11_attention_weight_min": -44.84830856323242,
      "activations/layer12_attention_weight_max": 35.958492279052734,
      "activations/layer12_attention_weight_min": -30.265575408935547,
      "activations/layer13_attention_weight_max": 60.94664001464844,
      "activations/layer13_attention_weight_min": -43.06367874145508,
      "activations/layer14_attention_weight_max": 67.8792724609375,
      "activations/layer14_attention_weight_min": -50.00490188598633,
      "activations/layer15_attention_weight_max": 68.30397033691406,
      "activations/layer15_attention_weight_min": -43.97726058959961,
      "activations/layer16_attention_weight_max": 42.01686477661133,
      "activations/layer16_attention_weight_min": -30.887332916259766,
      "activations/layer17_attention_weight_max": 38.70798110961914,
      "activations/layer17_attention_weight_min": -31.748722076416016,
      "activations/layer18_attention_weight_max": 34.84025955200195,
      "activations/layer18_attention_weight_min": -25.914318084716797,
      "activations/layer19_attention_weight_max": 39.42621994018555,
      "activations/layer19_attention_weight_min": -35.638790130615234,
      "activations/layer1_attention_weight_max": 16.80548095703125,
      "activations/layer1_attention_weight_min": -15.483551979064941,
      "activations/layer20_attention_weight_max": 33.2513427734375,
      "activations/layer20_attention_weight_min": -25.704740524291992,
      "activations/layer21_attention_weight_max": 31.974990844726562,
      "activations/layer21_attention_weight_min": -24.3460750579834,
      "activations/layer22_attention_weight_max": 54.4178466796875,
      "activations/layer22_attention_weight_min": -32.4199104309082,
      "activations/layer23_attention_weight_max": 42.28579330444336,
      "activations/layer23_attention_weight_min": -25.96923065185547,
      "activations/layer2_attention_weight_max": 34.986148834228516,
      "activations/layer2_attention_weight_min": -32.91231155395508,
      "activations/layer3_attention_weight_max": 92.39683532714844,
      "activations/layer3_attention_weight_min": -99.1552505493164,
      "activations/layer4_attention_weight_max": 109.2018814086914,
      "activations/layer4_attention_weight_min": -108.860595703125,
      "activations/layer5_attention_weight_max": 52.76127243041992,
      "activations/layer5_attention_weight_min": -58.736473083496094,
      "activations/layer6_attention_weight_max": 48.48179244995117,
      "activations/layer6_attention_weight_min": -51.831993103027344,
      "activations/layer7_attention_weight_max": 108.57390594482422,
      "activations/layer7_attention_weight_min": -102.01268768310547,
      "activations/layer8_attention_weight_max": 50.81764221191406,
      "activations/layer8_attention_weight_min": -46.05512237548828,
      "activations/layer9_attention_weight_max": 44.15360641479492,
      "activations/layer9_attention_weight_min": -39.214599609375,
      "epoch": 12.38,
      "learning_rate": 7.090113636363636e-05,
      "loss": 2.7602,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4318,
      "eval_samples_per_second": 509.264,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4318,
      "eval_openwebtext_samples_per_second": 509.264,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9407,
      "eval_wikitext_samples_per_second": 234.967,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.5112,
      "eval_lambada_samples_per_second": 511.921,
      "step": 213000
    },
    {
      "activations/layer0_attention_weight_max": 15.620504379272461,
      "activations/layer0_attention_weight_min": -12.394301414489746,
      "activations/layer10_attention_weight_max": 38.63319778442383,
      "activations/layer10_attention_weight_min": -33.99052429199219,
      "activations/layer11_attention_weight_max": 35.30855941772461,
      "activations/layer11_attention_weight_min": -33.39348220825195,
      "activations/layer12_attention_weight_max": 26.438350677490234,
      "activations/layer12_attention_weight_min": -25.20996856689453,
      "activations/layer13_attention_weight_max": 42.19572830200195,
      "activations/layer13_attention_weight_min": -35.19021224975586,
      "activations/layer14_attention_weight_max": 45.4719352722168,
      "activations/layer14_attention_weight_min": -43.20540237426758,
      "activations/layer15_attention_weight_max": 36.62629318237305,
      "activations/layer15_attention_weight_min": -34.56856918334961,
      "activations/layer16_attention_weight_max": 31.957962036132812,
      "activations/layer16_attention_weight_min": -28.069028854370117,
      "activations/layer17_attention_weight_max": 31.824432373046875,
      "activations/layer17_attention_weight_min": -27.23709487915039,
      "activations/layer18_attention_weight_max": 30.957443237304688,
      "activations/layer18_attention_weight_min": -22.985572814941406,
      "activations/layer19_attention_weight_max": 38.0254020690918,
      "activations/layer19_attention_weight_min": -30.885242462158203,
      "activations/layer1_attention_weight_max": 15.753003120422363,
      "activations/layer1_attention_weight_min": -13.545320510864258,
      "activations/layer20_attention_weight_max": 30.488584518432617,
      "activations/layer20_attention_weight_min": -27.359312057495117,
      "activations/layer21_attention_weight_max": 28.73074722290039,
      "activations/layer21_attention_weight_min": -25.241209030151367,
      "activations/layer22_attention_weight_max": 43.92176818847656,
      "activations/layer22_attention_weight_min": -29.746036529541016,
      "activations/layer23_attention_weight_max": 35.051700592041016,
      "activations/layer23_attention_weight_min": -24.28093719482422,
      "activations/layer2_attention_weight_max": 31.39092254638672,
      "activations/layer2_attention_weight_min": -31.052753448486328,
      "activations/layer3_attention_weight_max": 93.18931579589844,
      "activations/layer3_attention_weight_min": -92.61356353759766,
      "activations/layer4_attention_weight_max": 112.59774017333984,
      "activations/layer4_attention_weight_min": -109.52546691894531,
      "activations/layer5_attention_weight_max": 54.85430908203125,
      "activations/layer5_attention_weight_min": -61.427669525146484,
      "activations/layer6_attention_weight_max": 43.62473678588867,
      "activations/layer6_attention_weight_min": -47.95808410644531,
      "activations/layer7_attention_weight_max": 91.66595458984375,
      "activations/layer7_attention_weight_min": -95.45471954345703,
      "activations/layer8_attention_weight_max": 43.82994842529297,
      "activations/layer8_attention_weight_min": -41.244956970214844,
      "activations/layer9_attention_weight_max": 36.10068893432617,
      "activations/layer9_attention_weight_min": -33.68845748901367,
      "epoch": 12.38,
      "learning_rate": 7.088219696969696e-05,
      "loss": 2.7683,
      "step": 213050
    },
    {
      "activations/layer0_attention_weight_max": 16.007091522216797,
      "activations/layer0_attention_weight_min": -12.31550121307373,
      "activations/layer10_attention_weight_max": 30.413869857788086,
      "activations/layer10_attention_weight_min": -30.20549201965332,
      "activations/layer11_attention_weight_max": 31.521596908569336,
      "activations/layer11_attention_weight_min": -30.63779067993164,
      "activations/layer12_attention_weight_max": 25.287803649902344,
      "activations/layer12_attention_weight_min": -24.042551040649414,
      "activations/layer13_attention_weight_max": 38.88966751098633,
      "activations/layer13_attention_weight_min": -34.19758224487305,
      "activations/layer14_attention_weight_max": 42.41305160522461,
      "activations/layer14_attention_weight_min": -38.58774948120117,
      "activations/layer15_attention_weight_max": 39.020904541015625,
      "activations/layer15_attention_weight_min": -32.83637237548828,
      "activations/layer16_attention_weight_max": 30.262622833251953,
      "activations/layer16_attention_weight_min": -27.1649169921875,
      "activations/layer17_attention_weight_max": 33.06443405151367,
      "activations/layer17_attention_weight_min": -25.787899017333984,
      "activations/layer18_attention_weight_max": 33.648529052734375,
      "activations/layer18_attention_weight_min": -26.478511810302734,
      "activations/layer19_attention_weight_max": 34.5142707824707,
      "activations/layer19_attention_weight_min": -30.643285751342773,
      "activations/layer1_attention_weight_max": 16.708444595336914,
      "activations/layer1_attention_weight_min": -12.846761703491211,
      "activations/layer20_attention_weight_max": 28.815311431884766,
      "activations/layer20_attention_weight_min": -26.628921508789062,
      "activations/layer21_attention_weight_max": 28.42904281616211,
      "activations/layer21_attention_weight_min": -26.181547164916992,
      "activations/layer22_attention_weight_max": 41.2104377746582,
      "activations/layer22_attention_weight_min": -34.01923370361328,
      "activations/layer23_attention_weight_max": 34.50244903564453,
      "activations/layer23_attention_weight_min": -26.61883544921875,
      "activations/layer2_attention_weight_max": 30.84712028503418,
      "activations/layer2_attention_weight_min": -30.09554100036621,
      "activations/layer3_attention_weight_max": 87.96902465820312,
      "activations/layer3_attention_weight_min": -89.03154754638672,
      "activations/layer4_attention_weight_max": 103.36366271972656,
      "activations/layer4_attention_weight_min": -105.82720184326172,
      "activations/layer5_attention_weight_max": 49.86408996582031,
      "activations/layer5_attention_weight_min": -60.516754150390625,
      "activations/layer6_attention_weight_max": 42.49342346191406,
      "activations/layer6_attention_weight_min": -46.08970642089844,
      "activations/layer7_attention_weight_max": 85.72489929199219,
      "activations/layer7_attention_weight_min": -86.89777374267578,
      "activations/layer8_attention_weight_max": 38.41164016723633,
      "activations/layer8_attention_weight_min": -43.18896484375,
      "activations/layer9_attention_weight_max": 31.805408477783203,
      "activations/layer9_attention_weight_min": -32.410335540771484,
      "epoch": 12.38,
      "learning_rate": 7.086325757575757e-05,
      "loss": 2.7704,
      "step": 213100
    },
    {
      "activations/layer0_attention_weight_max": 15.474542617797852,
      "activations/layer0_attention_weight_min": -12.877657890319824,
      "activations/layer10_attention_weight_max": 30.644012451171875,
      "activations/layer10_attention_weight_min": -31.10465431213379,
      "activations/layer11_attention_weight_max": 30.267723083496094,
      "activations/layer11_attention_weight_min": -31.574745178222656,
      "activations/layer12_attention_weight_max": 25.2110538482666,
      "activations/layer12_attention_weight_min": -24.708786010742188,
      "activations/layer13_attention_weight_max": 39.8621940612793,
      "activations/layer13_attention_weight_min": -32.77470016479492,
      "activations/layer14_attention_weight_max": 40.976104736328125,
      "activations/layer14_attention_weight_min": -33.62168502807617,
      "activations/layer15_attention_weight_max": 37.419395446777344,
      "activations/layer15_attention_weight_min": -33.49937057495117,
      "activations/layer16_attention_weight_max": 29.57158088684082,
      "activations/layer16_attention_weight_min": -29.470605850219727,
      "activations/layer17_attention_weight_max": 32.74642562866211,
      "activations/layer17_attention_weight_min": -28.562116622924805,
      "activations/layer18_attention_weight_max": 30.321664810180664,
      "activations/layer18_attention_weight_min": -27.282140731811523,
      "activations/layer19_attention_weight_max": 31.042612075805664,
      "activations/layer19_attention_weight_min": -32.63276672363281,
      "activations/layer1_attention_weight_max": 16.356842041015625,
      "activations/layer1_attention_weight_min": -13.273575782775879,
      "activations/layer20_attention_weight_max": 29.065521240234375,
      "activations/layer20_attention_weight_min": -26.085033416748047,
      "activations/layer21_attention_weight_max": 29.374454498291016,
      "activations/layer21_attention_weight_min": -25.734952926635742,
      "activations/layer22_attention_weight_max": 41.494873046875,
      "activations/layer22_attention_weight_min": -33.5140380859375,
      "activations/layer23_attention_weight_max": 31.556350708007812,
      "activations/layer23_attention_weight_min": -28.9903564453125,
      "activations/layer2_attention_weight_max": 32.01593017578125,
      "activations/layer2_attention_weight_min": -31.026533126831055,
      "activations/layer3_attention_weight_max": 92.03709411621094,
      "activations/layer3_attention_weight_min": -91.95246887207031,
      "activations/layer4_attention_weight_max": 111.90510559082031,
      "activations/layer4_attention_weight_min": -106.04046630859375,
      "activations/layer5_attention_weight_max": 49.890560150146484,
      "activations/layer5_attention_weight_min": -63.650787353515625,
      "activations/layer6_attention_weight_max": 45.083717346191406,
      "activations/layer6_attention_weight_min": -47.801902770996094,
      "activations/layer7_attention_weight_max": 87.04711151123047,
      "activations/layer7_attention_weight_min": -91.0490493774414,
      "activations/layer8_attention_weight_max": 37.98770523071289,
      "activations/layer8_attention_weight_min": -41.4557991027832,
      "activations/layer9_attention_weight_max": 31.259931564331055,
      "activations/layer9_attention_weight_min": -31.369333267211914,
      "epoch": 12.39,
      "learning_rate": 7.084431818181817e-05,
      "loss": 2.7696,
      "step": 213150
    },
    {
      "activations/layer0_attention_weight_max": 16.50922966003418,
      "activations/layer0_attention_weight_min": -12.796562194824219,
      "activations/layer10_attention_weight_max": 32.41345977783203,
      "activations/layer10_attention_weight_min": -32.11641311645508,
      "activations/layer11_attention_weight_max": 33.85690689086914,
      "activations/layer11_attention_weight_min": -32.430442810058594,
      "activations/layer12_attention_weight_max": 25.089841842651367,
      "activations/layer12_attention_weight_min": -24.854129791259766,
      "activations/layer13_attention_weight_max": 38.005611419677734,
      "activations/layer13_attention_weight_min": -33.70960998535156,
      "activations/layer14_attention_weight_max": 42.520599365234375,
      "activations/layer14_attention_weight_min": -37.768585205078125,
      "activations/layer15_attention_weight_max": 34.687129974365234,
      "activations/layer15_attention_weight_min": -32.532203674316406,
      "activations/layer16_attention_weight_max": 29.252683639526367,
      "activations/layer16_attention_weight_min": -27.585620880126953,
      "activations/layer17_attention_weight_max": 29.159753799438477,
      "activations/layer17_attention_weight_min": -24.441692352294922,
      "activations/layer18_attention_weight_max": 28.852195739746094,
      "activations/layer18_attention_weight_min": -22.558429718017578,
      "activations/layer19_attention_weight_max": 35.82302474975586,
      "activations/layer19_attention_weight_min": -31.306100845336914,
      "activations/layer1_attention_weight_max": 16.257732391357422,
      "activations/layer1_attention_weight_min": -13.34265422821045,
      "activations/layer20_attention_weight_max": 30.049869537353516,
      "activations/layer20_attention_weight_min": -25.135465621948242,
      "activations/layer21_attention_weight_max": 29.91830062866211,
      "activations/layer21_attention_weight_min": -24.653854370117188,
      "activations/layer22_attention_weight_max": 42.744197845458984,
      "activations/layer22_attention_weight_min": -30.127561569213867,
      "activations/layer23_attention_weight_max": 34.86650466918945,
      "activations/layer23_attention_weight_min": -25.25994110107422,
      "activations/layer2_attention_weight_max": 32.384429931640625,
      "activations/layer2_attention_weight_min": -30.9986629486084,
      "activations/layer3_attention_weight_max": 92.79006958007812,
      "activations/layer3_attention_weight_min": -94.92367553710938,
      "activations/layer4_attention_weight_max": 108.0871810913086,
      "activations/layer4_attention_weight_min": -109.6121826171875,
      "activations/layer5_attention_weight_max": 52.02212142944336,
      "activations/layer5_attention_weight_min": -60.22478485107422,
      "activations/layer6_attention_weight_max": 44.67564010620117,
      "activations/layer6_attention_weight_min": -47.2928352355957,
      "activations/layer7_attention_weight_max": 94.25594329833984,
      "activations/layer7_attention_weight_min": -91.41073608398438,
      "activations/layer8_attention_weight_max": 42.98762512207031,
      "activations/layer8_attention_weight_min": -42.071136474609375,
      "activations/layer9_attention_weight_max": 34.27253341674805,
      "activations/layer9_attention_weight_min": -35.66187286376953,
      "epoch": 12.39,
      "learning_rate": 7.082537878787879e-05,
      "loss": 2.7446,
      "step": 213200
    },
    {
      "activations/layer0_attention_weight_max": 15.60550594329834,
      "activations/layer0_attention_weight_min": -11.922416687011719,
      "activations/layer10_attention_weight_max": 32.09547805786133,
      "activations/layer10_attention_weight_min": -31.459121704101562,
      "activations/layer11_attention_weight_max": 31.088130950927734,
      "activations/layer11_attention_weight_min": -31.21126937866211,
      "activations/layer12_attention_weight_max": 25.828083038330078,
      "activations/layer12_attention_weight_min": -25.440876007080078,
      "activations/layer13_attention_weight_max": 35.141319274902344,
      "activations/layer13_attention_weight_min": -32.3997688293457,
      "activations/layer14_attention_weight_max": 40.06938552856445,
      "activations/layer14_attention_weight_min": -34.94293975830078,
      "activations/layer15_attention_weight_max": 35.72348403930664,
      "activations/layer15_attention_weight_min": -31.850576400756836,
      "activations/layer16_attention_weight_max": 30.38407325744629,
      "activations/layer16_attention_weight_min": -27.79486083984375,
      "activations/layer17_attention_weight_max": 28.894805908203125,
      "activations/layer17_attention_weight_min": -25.65876579284668,
      "activations/layer18_attention_weight_max": 29.957616806030273,
      "activations/layer18_attention_weight_min": -25.46769905090332,
      "activations/layer19_attention_weight_max": 30.864852905273438,
      "activations/layer19_attention_weight_min": -31.133499145507812,
      "activations/layer1_attention_weight_max": 17.05129623413086,
      "activations/layer1_attention_weight_min": -14.544288635253906,
      "activations/layer20_attention_weight_max": 28.144113540649414,
      "activations/layer20_attention_weight_min": -25.126630783081055,
      "activations/layer21_attention_weight_max": 29.001066207885742,
      "activations/layer21_attention_weight_min": -24.25260353088379,
      "activations/layer22_attention_weight_max": 41.07290267944336,
      "activations/layer22_attention_weight_min": -32.58393859863281,
      "activations/layer23_attention_weight_max": 31.908668518066406,
      "activations/layer23_attention_weight_min": -24.89632797241211,
      "activations/layer2_attention_weight_max": 31.820796966552734,
      "activations/layer2_attention_weight_min": -33.152950286865234,
      "activations/layer3_attention_weight_max": 90.61412048339844,
      "activations/layer3_attention_weight_min": -96.77535247802734,
      "activations/layer4_attention_weight_max": 105.81294250488281,
      "activations/layer4_attention_weight_min": -110.96036529541016,
      "activations/layer5_attention_weight_max": 48.72682189941406,
      "activations/layer5_attention_weight_min": -58.18170928955078,
      "activations/layer6_attention_weight_max": 43.13319396972656,
      "activations/layer6_attention_weight_min": -45.9122200012207,
      "activations/layer7_attention_weight_max": 85.92498779296875,
      "activations/layer7_attention_weight_min": -89.6410140991211,
      "activations/layer8_attention_weight_max": 37.99591064453125,
      "activations/layer8_attention_weight_min": -38.82442092895508,
      "activations/layer9_attention_weight_max": 31.031875610351562,
      "activations/layer9_attention_weight_min": -32.33687973022461,
      "epoch": 12.39,
      "learning_rate": 7.080643939393939e-05,
      "loss": 2.7528,
      "step": 213250
    },
    {
      "activations/layer0_attention_weight_max": 15.619871139526367,
      "activations/layer0_attention_weight_min": -12.31429386138916,
      "activations/layer10_attention_weight_max": 32.75986099243164,
      "activations/layer10_attention_weight_min": -34.457489013671875,
      "activations/layer11_attention_weight_max": 32.70152282714844,
      "activations/layer11_attention_weight_min": -34.7573356628418,
      "activations/layer12_attention_weight_max": 32.88304901123047,
      "activations/layer12_attention_weight_min": -27.02336883544922,
      "activations/layer13_attention_weight_max": 43.68777084350586,
      "activations/layer13_attention_weight_min": -36.94788360595703,
      "activations/layer14_attention_weight_max": 43.508209228515625,
      "activations/layer14_attention_weight_min": -41.60334014892578,
      "activations/layer15_attention_weight_max": 38.329044342041016,
      "activations/layer15_attention_weight_min": -36.78398513793945,
      "activations/layer16_attention_weight_max": 31.349178314208984,
      "activations/layer16_attention_weight_min": -30.37656593322754,
      "activations/layer17_attention_weight_max": 29.967323303222656,
      "activations/layer17_attention_weight_min": -27.67214012145996,
      "activations/layer18_attention_weight_max": 32.488975524902344,
      "activations/layer18_attention_weight_min": -26.03103256225586,
      "activations/layer19_attention_weight_max": 37.316593170166016,
      "activations/layer19_attention_weight_min": -33.517578125,
      "activations/layer1_attention_weight_max": 16.363082885742188,
      "activations/layer1_attention_weight_min": -13.538516998291016,
      "activations/layer20_attention_weight_max": 30.2547607421875,
      "activations/layer20_attention_weight_min": -26.52077293395996,
      "activations/layer21_attention_weight_max": 29.14801025390625,
      "activations/layer21_attention_weight_min": -24.76946449279785,
      "activations/layer22_attention_weight_max": 43.64500045776367,
      "activations/layer22_attention_weight_min": -32.1114387512207,
      "activations/layer23_attention_weight_max": 34.883460998535156,
      "activations/layer23_attention_weight_min": -26.362890243530273,
      "activations/layer2_attention_weight_max": 30.46583366394043,
      "activations/layer2_attention_weight_min": -30.471799850463867,
      "activations/layer3_attention_weight_max": 89.01432037353516,
      "activations/layer3_attention_weight_min": -92.99981689453125,
      "activations/layer4_attention_weight_max": 108.24427795410156,
      "activations/layer4_attention_weight_min": -111.93133544921875,
      "activations/layer5_attention_weight_max": 51.349395751953125,
      "activations/layer5_attention_weight_min": -61.87417221069336,
      "activations/layer6_attention_weight_max": 43.70500183105469,
      "activations/layer6_attention_weight_min": -45.988895416259766,
      "activations/layer7_attention_weight_max": 84.96176147460938,
      "activations/layer7_attention_weight_min": -88.17393493652344,
      "activations/layer8_attention_weight_max": 37.80961990356445,
      "activations/layer8_attention_weight_min": -41.4261360168457,
      "activations/layer9_attention_weight_max": 30.992643356323242,
      "activations/layer9_attention_weight_min": -33.07720947265625,
      "epoch": 12.39,
      "learning_rate": 7.078749999999999e-05,
      "loss": 2.7682,
      "step": 213300
    },
    {
      "activations/layer0_attention_weight_max": 15.280133247375488,
      "activations/layer0_attention_weight_min": -11.850226402282715,
      "activations/layer10_attention_weight_max": 32.933692932128906,
      "activations/layer10_attention_weight_min": -30.562742233276367,
      "activations/layer11_attention_weight_max": 32.10865020751953,
      "activations/layer11_attention_weight_min": -30.0183162689209,
      "activations/layer12_attention_weight_max": 28.389034271240234,
      "activations/layer12_attention_weight_min": -26.11968231201172,
      "activations/layer13_attention_weight_max": 39.079742431640625,
      "activations/layer13_attention_weight_min": -34.326759338378906,
      "activations/layer14_attention_weight_max": 41.99618911743164,
      "activations/layer14_attention_weight_min": -37.44394302368164,
      "activations/layer15_attention_weight_max": 38.73527908325195,
      "activations/layer15_attention_weight_min": -31.984012603759766,
      "activations/layer16_attention_weight_max": 33.54060363769531,
      "activations/layer16_attention_weight_min": -27.297544479370117,
      "activations/layer17_attention_weight_max": 31.647573471069336,
      "activations/layer17_attention_weight_min": -24.148630142211914,
      "activations/layer18_attention_weight_max": 30.77411460876465,
      "activations/layer18_attention_weight_min": -24.399154663085938,
      "activations/layer19_attention_weight_max": 33.5616455078125,
      "activations/layer19_attention_weight_min": -30.112165451049805,
      "activations/layer1_attention_weight_max": 17.274473190307617,
      "activations/layer1_attention_weight_min": -13.325396537780762,
      "activations/layer20_attention_weight_max": 27.885940551757812,
      "activations/layer20_attention_weight_min": -23.652055740356445,
      "activations/layer21_attention_weight_max": 28.08840560913086,
      "activations/layer21_attention_weight_min": -23.823440551757812,
      "activations/layer22_attention_weight_max": 41.63022232055664,
      "activations/layer22_attention_weight_min": -28.356985092163086,
      "activations/layer23_attention_weight_max": 34.850502014160156,
      "activations/layer23_attention_weight_min": -24.063690185546875,
      "activations/layer2_attention_weight_max": 31.228885650634766,
      "activations/layer2_attention_weight_min": -30.56340789794922,
      "activations/layer3_attention_weight_max": 92.36954498291016,
      "activations/layer3_attention_weight_min": -92.42204284667969,
      "activations/layer4_attention_weight_max": 108.7894058227539,
      "activations/layer4_attention_weight_min": -109.2667236328125,
      "activations/layer5_attention_weight_max": 51.51017761230469,
      "activations/layer5_attention_weight_min": -62.380821228027344,
      "activations/layer6_attention_weight_max": 42.86008071899414,
      "activations/layer6_attention_weight_min": -49.37171936035156,
      "activations/layer7_attention_weight_max": 92.155517578125,
      "activations/layer7_attention_weight_min": -93.86578369140625,
      "activations/layer8_attention_weight_max": 37.976829528808594,
      "activations/layer8_attention_weight_min": -40.771522521972656,
      "activations/layer9_attention_weight_max": 31.55742835998535,
      "activations/layer9_attention_weight_min": -33.711402893066406,
      "epoch": 12.4,
      "learning_rate": 7.076856060606061e-05,
      "loss": 2.7826,
      "step": 213350
    },
    {
      "activations/layer0_attention_weight_max": 15.967140197753906,
      "activations/layer0_attention_weight_min": -12.101706504821777,
      "activations/layer10_attention_weight_max": 32.751731872558594,
      "activations/layer10_attention_weight_min": -32.797203063964844,
      "activations/layer11_attention_weight_max": 31.97454071044922,
      "activations/layer11_attention_weight_min": -31.104055404663086,
      "activations/layer12_attention_weight_max": 23.61785125732422,
      "activations/layer12_attention_weight_min": -26.866579055786133,
      "activations/layer13_attention_weight_max": 37.58362579345703,
      "activations/layer13_attention_weight_min": -34.63977813720703,
      "activations/layer14_attention_weight_max": 38.75565719604492,
      "activations/layer14_attention_weight_min": -37.457584381103516,
      "activations/layer15_attention_weight_max": 35.8735237121582,
      "activations/layer15_attention_weight_min": -35.51398849487305,
      "activations/layer16_attention_weight_max": 29.13717269897461,
      "activations/layer16_attention_weight_min": -28.28288459777832,
      "activations/layer17_attention_weight_max": 28.464752197265625,
      "activations/layer17_attention_weight_min": -26.33529281616211,
      "activations/layer18_attention_weight_max": 32.974430084228516,
      "activations/layer18_attention_weight_min": -24.202035903930664,
      "activations/layer19_attention_weight_max": 33.81554412841797,
      "activations/layer19_attention_weight_min": -32.175270080566406,
      "activations/layer1_attention_weight_max": 15.778849601745605,
      "activations/layer1_attention_weight_min": -13.968026161193848,
      "activations/layer20_attention_weight_max": 29.40462875366211,
      "activations/layer20_attention_weight_min": -26.253087997436523,
      "activations/layer21_attention_weight_max": 27.87028694152832,
      "activations/layer21_attention_weight_min": -25.858379364013672,
      "activations/layer22_attention_weight_max": 42.1868782043457,
      "activations/layer22_attention_weight_min": -32.009944915771484,
      "activations/layer23_attention_weight_max": 36.09757995605469,
      "activations/layer23_attention_weight_min": -25.560123443603516,
      "activations/layer2_attention_weight_max": 30.837528228759766,
      "activations/layer2_attention_weight_min": -30.341344833374023,
      "activations/layer3_attention_weight_max": 90.5462875366211,
      "activations/layer3_attention_weight_min": -91.95465087890625,
      "activations/layer4_attention_weight_max": 104.54903411865234,
      "activations/layer4_attention_weight_min": -105.67174530029297,
      "activations/layer5_attention_weight_max": 51.87095642089844,
      "activations/layer5_attention_weight_min": -59.023956298828125,
      "activations/layer6_attention_weight_max": 42.575439453125,
      "activations/layer6_attention_weight_min": -48.38347625732422,
      "activations/layer7_attention_weight_max": 89.96798706054688,
      "activations/layer7_attention_weight_min": -89.00366973876953,
      "activations/layer8_attention_weight_max": 40.434513092041016,
      "activations/layer8_attention_weight_min": -40.10708999633789,
      "activations/layer9_attention_weight_max": 31.901905059814453,
      "activations/layer9_attention_weight_min": -33.46733093261719,
      "epoch": 12.4,
      "learning_rate": 7.074962121212121e-05,
      "loss": 2.7687,
      "step": 213400
    },
    {
      "activations/layer0_attention_weight_max": 15.409873008728027,
      "activations/layer0_attention_weight_min": -11.739265441894531,
      "activations/layer10_attention_weight_max": 32.09496307373047,
      "activations/layer10_attention_weight_min": -32.57239532470703,
      "activations/layer11_attention_weight_max": 30.6494197845459,
      "activations/layer11_attention_weight_min": -32.69481658935547,
      "activations/layer12_attention_weight_max": 40.966426849365234,
      "activations/layer12_attention_weight_min": -25.67337417602539,
      "activations/layer13_attention_weight_max": 46.387508392333984,
      "activations/layer13_attention_weight_min": -31.892868041992188,
      "activations/layer14_attention_weight_max": 46.2397346496582,
      "activations/layer14_attention_weight_min": -36.716285705566406,
      "activations/layer15_attention_weight_max": 37.12934112548828,
      "activations/layer15_attention_weight_min": -31.64122772216797,
      "activations/layer16_attention_weight_max": 30.692455291748047,
      "activations/layer16_attention_weight_min": -26.864343643188477,
      "activations/layer17_attention_weight_max": 30.568721771240234,
      "activations/layer17_attention_weight_min": -25.007532119750977,
      "activations/layer18_attention_weight_max": 31.477970123291016,
      "activations/layer18_attention_weight_min": -24.911237716674805,
      "activations/layer19_attention_weight_max": 31.520444869995117,
      "activations/layer19_attention_weight_min": -31.553115844726562,
      "activations/layer1_attention_weight_max": 16.21135902404785,
      "activations/layer1_attention_weight_min": -13.316499710083008,
      "activations/layer20_attention_weight_max": 27.87994384765625,
      "activations/layer20_attention_weight_min": -25.313322067260742,
      "activations/layer21_attention_weight_max": 29.487228393554688,
      "activations/layer21_attention_weight_min": -24.160913467407227,
      "activations/layer22_attention_weight_max": 39.904884338378906,
      "activations/layer22_attention_weight_min": -29.992584228515625,
      "activations/layer23_attention_weight_max": 33.457000732421875,
      "activations/layer23_attention_weight_min": -23.983022689819336,
      "activations/layer2_attention_weight_max": 31.040264129638672,
      "activations/layer2_attention_weight_min": -31.638015747070312,
      "activations/layer3_attention_weight_max": 90.03641510009766,
      "activations/layer3_attention_weight_min": -96.35991668701172,
      "activations/layer4_attention_weight_max": 105.97270965576172,
      "activations/layer4_attention_weight_min": -103.65380859375,
      "activations/layer5_attention_weight_max": 49.25455856323242,
      "activations/layer5_attention_weight_min": -58.35354232788086,
      "activations/layer6_attention_weight_max": 41.41040802001953,
      "activations/layer6_attention_weight_min": -47.0632438659668,
      "activations/layer7_attention_weight_max": 85.78528594970703,
      "activations/layer7_attention_weight_min": -91.63896179199219,
      "activations/layer8_attention_weight_max": 37.52792739868164,
      "activations/layer8_attention_weight_min": -40.58760452270508,
      "activations/layer9_attention_weight_max": 31.185592651367188,
      "activations/layer9_attention_weight_min": -34.943607330322266,
      "epoch": 12.4,
      "learning_rate": 7.073068181818181e-05,
      "loss": 2.7684,
      "step": 213450
    },
    {
      "activations/layer0_attention_weight_max": 15.989641189575195,
      "activations/layer0_attention_weight_min": -11.38106632232666,
      "activations/layer10_attention_weight_max": 33.10602569580078,
      "activations/layer10_attention_weight_min": -31.501798629760742,
      "activations/layer11_attention_weight_max": 33.661373138427734,
      "activations/layer11_attention_weight_min": -30.362010955810547,
      "activations/layer12_attention_weight_max": 25.8382511138916,
      "activations/layer12_attention_weight_min": -25.861984252929688,
      "activations/layer13_attention_weight_max": 43.635902404785156,
      "activations/layer13_attention_weight_min": -34.076988220214844,
      "activations/layer14_attention_weight_max": 49.806331634521484,
      "activations/layer14_attention_weight_min": -36.20228958129883,
      "activations/layer15_attention_weight_max": 39.13553237915039,
      "activations/layer15_attention_weight_min": -32.52483367919922,
      "activations/layer16_attention_weight_max": 33.864070892333984,
      "activations/layer16_attention_weight_min": -28.886886596679688,
      "activations/layer17_attention_weight_max": 35.1441764831543,
      "activations/layer17_attention_weight_min": -25.20221710205078,
      "activations/layer18_attention_weight_max": 32.47716522216797,
      "activations/layer18_attention_weight_min": -26.042407989501953,
      "activations/layer19_attention_weight_max": 39.238155364990234,
      "activations/layer19_attention_weight_min": -33.098262786865234,
      "activations/layer1_attention_weight_max": 16.56264305114746,
      "activations/layer1_attention_weight_min": -12.932868003845215,
      "activations/layer20_attention_weight_max": 34.87738800048828,
      "activations/layer20_attention_weight_min": -26.547508239746094,
      "activations/layer21_attention_weight_max": 33.72714614868164,
      "activations/layer21_attention_weight_min": -25.00470542907715,
      "activations/layer22_attention_weight_max": 53.65782165527344,
      "activations/layer22_attention_weight_min": -34.263668060302734,
      "activations/layer23_attention_weight_max": 40.058380126953125,
      "activations/layer23_attention_weight_min": -26.708755493164062,
      "activations/layer2_attention_weight_max": 30.231712341308594,
      "activations/layer2_attention_weight_min": -30.77100944519043,
      "activations/layer3_attention_weight_max": 90.2275390625,
      "activations/layer3_attention_weight_min": -91.41130065917969,
      "activations/layer4_attention_weight_max": 106.5946273803711,
      "activations/layer4_attention_weight_min": -108.76802825927734,
      "activations/layer5_attention_weight_max": 50.99462890625,
      "activations/layer5_attention_weight_min": -56.4525146484375,
      "activations/layer6_attention_weight_max": 42.83708190917969,
      "activations/layer6_attention_weight_min": -47.044822692871094,
      "activations/layer7_attention_weight_max": 86.65593719482422,
      "activations/layer7_attention_weight_min": -85.99994659423828,
      "activations/layer8_attention_weight_max": 39.62660217285156,
      "activations/layer8_attention_weight_min": -41.727596282958984,
      "activations/layer9_attention_weight_max": 31.866989135742188,
      "activations/layer9_attention_weight_min": -32.869815826416016,
      "epoch": 12.41,
      "learning_rate": 7.071174242424241e-05,
      "loss": 2.769,
      "step": 213500
    },
    {
      "activations/layer0_attention_weight_max": 15.333891868591309,
      "activations/layer0_attention_weight_min": -11.920805931091309,
      "activations/layer10_attention_weight_max": 30.890422821044922,
      "activations/layer10_attention_weight_min": -32.52751541137695,
      "activations/layer11_attention_weight_max": 31.498380661010742,
      "activations/layer11_attention_weight_min": -32.168190002441406,
      "activations/layer12_attention_weight_max": 23.430673599243164,
      "activations/layer12_attention_weight_min": -24.678482055664062,
      "activations/layer13_attention_weight_max": 38.0713005065918,
      "activations/layer13_attention_weight_min": -34.919158935546875,
      "activations/layer14_attention_weight_max": 41.52935028076172,
      "activations/layer14_attention_weight_min": -37.80646896362305,
      "activations/layer15_attention_weight_max": 33.50084686279297,
      "activations/layer15_attention_weight_min": -32.36311721801758,
      "activations/layer16_attention_weight_max": 29.852907180786133,
      "activations/layer16_attention_weight_min": -26.961933135986328,
      "activations/layer17_attention_weight_max": 29.400537490844727,
      "activations/layer17_attention_weight_min": -25.43690299987793,
      "activations/layer18_attention_weight_max": 32.01743698120117,
      "activations/layer18_attention_weight_min": -23.695087432861328,
      "activations/layer19_attention_weight_max": 32.808685302734375,
      "activations/layer19_attention_weight_min": -31.25777244567871,
      "activations/layer1_attention_weight_max": 16.044443130493164,
      "activations/layer1_attention_weight_min": -14.447394371032715,
      "activations/layer20_attention_weight_max": 31.08867073059082,
      "activations/layer20_attention_weight_min": -23.700336456298828,
      "activations/layer21_attention_weight_max": 30.210947036743164,
      "activations/layer21_attention_weight_min": -22.916967391967773,
      "activations/layer22_attention_weight_max": 43.02519607543945,
      "activations/layer22_attention_weight_min": -29.675312042236328,
      "activations/layer23_attention_weight_max": 32.53105163574219,
      "activations/layer23_attention_weight_min": -24.55646324157715,
      "activations/layer2_attention_weight_max": 32.237510681152344,
      "activations/layer2_attention_weight_min": -31.577190399169922,
      "activations/layer3_attention_weight_max": 93.3787612915039,
      "activations/layer3_attention_weight_min": -95.87090301513672,
      "activations/layer4_attention_weight_max": 111.45198822021484,
      "activations/layer4_attention_weight_min": -111.51753234863281,
      "activations/layer5_attention_weight_max": 50.08257293701172,
      "activations/layer5_attention_weight_min": -57.27053451538086,
      "activations/layer6_attention_weight_max": 45.7309684753418,
      "activations/layer6_attention_weight_min": -46.882083892822266,
      "activations/layer7_attention_weight_max": 91.85107421875,
      "activations/layer7_attention_weight_min": -85.40229034423828,
      "activations/layer8_attention_weight_max": 39.286128997802734,
      "activations/layer8_attention_weight_min": -39.32411575317383,
      "activations/layer9_attention_weight_max": 31.122303009033203,
      "activations/layer9_attention_weight_min": -31.908416748046875,
      "epoch": 12.41,
      "learning_rate": 7.069280303030303e-05,
      "loss": 2.7729,
      "step": 213550
    },
    {
      "activations/layer0_attention_weight_max": 15.355548858642578,
      "activations/layer0_attention_weight_min": -11.81680679321289,
      "activations/layer10_attention_weight_max": 31.010282516479492,
      "activations/layer10_attention_weight_min": -32.18121337890625,
      "activations/layer11_attention_weight_max": 29.252269744873047,
      "activations/layer11_attention_weight_min": -32.30225372314453,
      "activations/layer12_attention_weight_max": 23.14689064025879,
      "activations/layer12_attention_weight_min": -24.248310089111328,
      "activations/layer13_attention_weight_max": 35.810569763183594,
      "activations/layer13_attention_weight_min": -33.67140579223633,
      "activations/layer14_attention_weight_max": 41.45979690551758,
      "activations/layer14_attention_weight_min": -34.49081039428711,
      "activations/layer15_attention_weight_max": 33.56533432006836,
      "activations/layer15_attention_weight_min": -32.5496826171875,
      "activations/layer16_attention_weight_max": 30.93299102783203,
      "activations/layer16_attention_weight_min": -26.962894439697266,
      "activations/layer17_attention_weight_max": 33.665245056152344,
      "activations/layer17_attention_weight_min": -27.029916763305664,
      "activations/layer18_attention_weight_max": 32.37216567993164,
      "activations/layer18_attention_weight_min": -27.391338348388672,
      "activations/layer19_attention_weight_max": 35.30550003051758,
      "activations/layer19_attention_weight_min": -31.46367835998535,
      "activations/layer1_attention_weight_max": 15.941895484924316,
      "activations/layer1_attention_weight_min": -15.152114868164062,
      "activations/layer20_attention_weight_max": 30.835803985595703,
      "activations/layer20_attention_weight_min": -28.150949478149414,
      "activations/layer21_attention_weight_max": 29.040042877197266,
      "activations/layer21_attention_weight_min": -24.94414710998535,
      "activations/layer22_attention_weight_max": 45.27449035644531,
      "activations/layer22_attention_weight_min": -30.04235076904297,
      "activations/layer23_attention_weight_max": 39.56645202636719,
      "activations/layer23_attention_weight_min": -28.234573364257812,
      "activations/layer2_attention_weight_max": 30.214462280273438,
      "activations/layer2_attention_weight_min": -30.844585418701172,
      "activations/layer3_attention_weight_max": 90.98164367675781,
      "activations/layer3_attention_weight_min": -94.12529754638672,
      "activations/layer4_attention_weight_max": 106.04918670654297,
      "activations/layer4_attention_weight_min": -111.16033935546875,
      "activations/layer5_attention_weight_max": 50.87599182128906,
      "activations/layer5_attention_weight_min": -59.050750732421875,
      "activations/layer6_attention_weight_max": 44.486793518066406,
      "activations/layer6_attention_weight_min": -47.4262580871582,
      "activations/layer7_attention_weight_max": 86.50434875488281,
      "activations/layer7_attention_weight_min": -88.31586456298828,
      "activations/layer8_attention_weight_max": 38.28260803222656,
      "activations/layer8_attention_weight_min": -39.40801239013672,
      "activations/layer9_attention_weight_max": 29.952611923217773,
      "activations/layer9_attention_weight_min": -31.568511962890625,
      "epoch": 12.41,
      "learning_rate": 7.067386363636363e-05,
      "loss": 2.7776,
      "step": 213600
    },
    {
      "activations/layer0_attention_weight_max": 15.642125129699707,
      "activations/layer0_attention_weight_min": -12.295714378356934,
      "activations/layer10_attention_weight_max": 43.1983528137207,
      "activations/layer10_attention_weight_min": -40.70973587036133,
      "activations/layer11_attention_weight_max": 43.99348068237305,
      "activations/layer11_attention_weight_min": -41.68151092529297,
      "activations/layer12_attention_weight_max": 31.161300659179688,
      "activations/layer12_attention_weight_min": -27.51410675048828,
      "activations/layer13_attention_weight_max": 48.68601989746094,
      "activations/layer13_attention_weight_min": -38.19224166870117,
      "activations/layer14_attention_weight_max": 60.83644104003906,
      "activations/layer14_attention_weight_min": -44.151588439941406,
      "activations/layer15_attention_weight_max": 55.96856689453125,
      "activations/layer15_attention_weight_min": -36.21455383300781,
      "activations/layer16_attention_weight_max": 37.36678695678711,
      "activations/layer16_attention_weight_min": -28.664657592773438,
      "activations/layer17_attention_weight_max": 35.27302932739258,
      "activations/layer17_attention_weight_min": -27.759626388549805,
      "activations/layer18_attention_weight_max": 34.47152328491211,
      "activations/layer18_attention_weight_min": -24.356176376342773,
      "activations/layer19_attention_weight_max": 45.77178192138672,
      "activations/layer19_attention_weight_min": -33.85752487182617,
      "activations/layer1_attention_weight_max": 16.88798713684082,
      "activations/layer1_attention_weight_min": -14.181282997131348,
      "activations/layer20_attention_weight_max": 36.52217102050781,
      "activations/layer20_attention_weight_min": -25.07207489013672,
      "activations/layer21_attention_weight_max": 33.544105529785156,
      "activations/layer21_attention_weight_min": -24.005876541137695,
      "activations/layer22_attention_weight_max": 50.7179069519043,
      "activations/layer22_attention_weight_min": -31.474037170410156,
      "activations/layer23_attention_weight_max": 36.181671142578125,
      "activations/layer23_attention_weight_min": -25.476806640625,
      "activations/layer2_attention_weight_max": 35.80159378051758,
      "activations/layer2_attention_weight_min": -33.162940979003906,
      "activations/layer3_attention_weight_max": 99.74324798583984,
      "activations/layer3_attention_weight_min": -98.81721496582031,
      "activations/layer4_attention_weight_max": 112.99251556396484,
      "activations/layer4_attention_weight_min": -110.56788635253906,
      "activations/layer5_attention_weight_max": 53.81043243408203,
      "activations/layer5_attention_weight_min": -59.121788024902344,
      "activations/layer6_attention_weight_max": 46.60951232910156,
      "activations/layer6_attention_weight_min": -45.88428497314453,
      "activations/layer7_attention_weight_max": 104.56236267089844,
      "activations/layer7_attention_weight_min": -92.50084686279297,
      "activations/layer8_attention_weight_max": 49.67710876464844,
      "activations/layer8_attention_weight_min": -45.92293167114258,
      "activations/layer9_attention_weight_max": 42.98332977294922,
      "activations/layer9_attention_weight_min": -39.39519500732422,
      "epoch": 12.41,
      "learning_rate": 7.065492424242423e-05,
      "loss": 2.7708,
      "step": 213650
    },
    {
      "activations/layer0_attention_weight_max": 16.494403839111328,
      "activations/layer0_attention_weight_min": -12.970741271972656,
      "activations/layer10_attention_weight_max": 31.380754470825195,
      "activations/layer10_attention_weight_min": -31.99069595336914,
      "activations/layer11_attention_weight_max": 29.64728355407715,
      "activations/layer11_attention_weight_min": -31.33637809753418,
      "activations/layer12_attention_weight_max": 25.417259216308594,
      "activations/layer12_attention_weight_min": -25.711177825927734,
      "activations/layer13_attention_weight_max": 38.22606658935547,
      "activations/layer13_attention_weight_min": -34.162349700927734,
      "activations/layer14_attention_weight_max": 46.86106872558594,
      "activations/layer14_attention_weight_min": -37.883583068847656,
      "activations/layer15_attention_weight_max": 37.61514663696289,
      "activations/layer15_attention_weight_min": -32.947452545166016,
      "activations/layer16_attention_weight_max": 32.338043212890625,
      "activations/layer16_attention_weight_min": -26.845277786254883,
      "activations/layer17_attention_weight_max": 31.547767639160156,
      "activations/layer17_attention_weight_min": -24.07220458984375,
      "activations/layer18_attention_weight_max": 29.607276916503906,
      "activations/layer18_attention_weight_min": -23.097148895263672,
      "activations/layer19_attention_weight_max": 36.00228500366211,
      "activations/layer19_attention_weight_min": -31.94632339477539,
      "activations/layer1_attention_weight_max": 17.03356170654297,
      "activations/layer1_attention_weight_min": -16.39430809020996,
      "activations/layer20_attention_weight_max": 28.65521812438965,
      "activations/layer20_attention_weight_min": -25.25084686279297,
      "activations/layer21_attention_weight_max": 27.602933883666992,
      "activations/layer21_attention_weight_min": -23.4306697845459,
      "activations/layer22_attention_weight_max": 40.13157653808594,
      "activations/layer22_attention_weight_min": -31.87675666809082,
      "activations/layer23_attention_weight_max": 32.969573974609375,
      "activations/layer23_attention_weight_min": -26.603788375854492,
      "activations/layer2_attention_weight_max": 31.673587799072266,
      "activations/layer2_attention_weight_min": -31.480281829833984,
      "activations/layer3_attention_weight_max": 91.92813110351562,
      "activations/layer3_attention_weight_min": -91.32279205322266,
      "activations/layer4_attention_weight_max": 105.42513275146484,
      "activations/layer4_attention_weight_min": -105.46025848388672,
      "activations/layer5_attention_weight_max": 48.69244384765625,
      "activations/layer5_attention_weight_min": -58.525123596191406,
      "activations/layer6_attention_weight_max": 43.943111419677734,
      "activations/layer6_attention_weight_min": -45.01093673706055,
      "activations/layer7_attention_weight_max": 87.77196502685547,
      "activations/layer7_attention_weight_min": -89.69954681396484,
      "activations/layer8_attention_weight_max": 37.259063720703125,
      "activations/layer8_attention_weight_min": -39.68574142456055,
      "activations/layer9_attention_weight_max": 32.06926727294922,
      "activations/layer9_attention_weight_min": -31.57672119140625,
      "epoch": 12.42,
      "learning_rate": 7.063598484848485e-05,
      "loss": 2.7744,
      "step": 213700
    },
    {
      "activations/layer0_attention_weight_max": 15.865760803222656,
      "activations/layer0_attention_weight_min": -11.183755874633789,
      "activations/layer10_attention_weight_max": 32.0877571105957,
      "activations/layer10_attention_weight_min": -33.706756591796875,
      "activations/layer11_attention_weight_max": 30.283111572265625,
      "activations/layer11_attention_weight_min": -33.46312713623047,
      "activations/layer12_attention_weight_max": 25.049793243408203,
      "activations/layer12_attention_weight_min": -27.076623916625977,
      "activations/layer13_attention_weight_max": 37.20866012573242,
      "activations/layer13_attention_weight_min": -35.13844299316406,
      "activations/layer14_attention_weight_max": 40.98283767700195,
      "activations/layer14_attention_weight_min": -37.66456985473633,
      "activations/layer15_attention_weight_max": 35.768192291259766,
      "activations/layer15_attention_weight_min": -33.57189178466797,
      "activations/layer16_attention_weight_max": 32.46625518798828,
      "activations/layer16_attention_weight_min": -27.97163963317871,
      "activations/layer17_attention_weight_max": 30.10126495361328,
      "activations/layer17_attention_weight_min": -26.102275848388672,
      "activations/layer18_attention_weight_max": 26.735403060913086,
      "activations/layer18_attention_weight_min": -23.039329528808594,
      "activations/layer19_attention_weight_max": 31.059982299804688,
      "activations/layer19_attention_weight_min": -30.2633113861084,
      "activations/layer1_attention_weight_max": 15.944622993469238,
      "activations/layer1_attention_weight_min": -14.229979515075684,
      "activations/layer20_attention_weight_max": 28.920146942138672,
      "activations/layer20_attention_weight_min": -24.304759979248047,
      "activations/layer21_attention_weight_max": 26.433841705322266,
      "activations/layer21_attention_weight_min": -23.563474655151367,
      "activations/layer22_attention_weight_max": 39.33638381958008,
      "activations/layer22_attention_weight_min": -31.308650970458984,
      "activations/layer23_attention_weight_max": 31.254390716552734,
      "activations/layer23_attention_weight_min": -25.244993209838867,
      "activations/layer2_attention_weight_max": 28.785823822021484,
      "activations/layer2_attention_weight_min": -31.56165313720703,
      "activations/layer3_attention_weight_max": 90.79965209960938,
      "activations/layer3_attention_weight_min": -92.9495849609375,
      "activations/layer4_attention_weight_max": 111.19404602050781,
      "activations/layer4_attention_weight_min": -110.78861236572266,
      "activations/layer5_attention_weight_max": 53.42031478881836,
      "activations/layer5_attention_weight_min": -65.44276428222656,
      "activations/layer6_attention_weight_max": 45.38666915893555,
      "activations/layer6_attention_weight_min": -49.121192932128906,
      "activations/layer7_attention_weight_max": 91.69743347167969,
      "activations/layer7_attention_weight_min": -94.98137664794922,
      "activations/layer8_attention_weight_max": 40.020751953125,
      "activations/layer8_attention_weight_min": -43.09805679321289,
      "activations/layer9_attention_weight_max": 31.61988639831543,
      "activations/layer9_attention_weight_min": -34.01467514038086,
      "epoch": 12.42,
      "learning_rate": 7.061704545454545e-05,
      "loss": 2.7744,
      "step": 213750
    },
    {
      "activations/layer0_attention_weight_max": 15.957658767700195,
      "activations/layer0_attention_weight_min": -12.346446990966797,
      "activations/layer10_attention_weight_max": 35.412353515625,
      "activations/layer10_attention_weight_min": -35.95532989501953,
      "activations/layer11_attention_weight_max": 34.24679946899414,
      "activations/layer11_attention_weight_min": -32.339019775390625,
      "activations/layer12_attention_weight_max": 37.4123420715332,
      "activations/layer12_attention_weight_min": -26.3960018157959,
      "activations/layer13_attention_weight_max": 47.09654235839844,
      "activations/layer13_attention_weight_min": -35.9251708984375,
      "activations/layer14_attention_weight_max": 45.104251861572266,
      "activations/layer14_attention_weight_min": -38.38319396972656,
      "activations/layer15_attention_weight_max": 33.92005157470703,
      "activations/layer15_attention_weight_min": -34.07277297973633,
      "activations/layer16_attention_weight_max": 30.493806838989258,
      "activations/layer16_attention_weight_min": -28.798809051513672,
      "activations/layer17_attention_weight_max": 32.662349700927734,
      "activations/layer17_attention_weight_min": -26.290443420410156,
      "activations/layer18_attention_weight_max": 31.34743881225586,
      "activations/layer18_attention_weight_min": -23.74590492248535,
      "activations/layer19_attention_weight_max": 32.646949768066406,
      "activations/layer19_attention_weight_min": -31.8070011138916,
      "activations/layer1_attention_weight_max": 16.3037166595459,
      "activations/layer1_attention_weight_min": -14.389124870300293,
      "activations/layer20_attention_weight_max": 30.429710388183594,
      "activations/layer20_attention_weight_min": -25.416019439697266,
      "activations/layer21_attention_weight_max": 29.063180923461914,
      "activations/layer21_attention_weight_min": -23.330936431884766,
      "activations/layer22_attention_weight_max": 40.64006042480469,
      "activations/layer22_attention_weight_min": -30.03495216369629,
      "activations/layer23_attention_weight_max": 32.456275939941406,
      "activations/layer23_attention_weight_min": -26.058998107910156,
      "activations/layer2_attention_weight_max": 33.649471282958984,
      "activations/layer2_attention_weight_min": -33.9730110168457,
      "activations/layer3_attention_weight_max": 93.47042846679688,
      "activations/layer3_attention_weight_min": -94.71829986572266,
      "activations/layer4_attention_weight_max": 111.83489227294922,
      "activations/layer4_attention_weight_min": -112.5494155883789,
      "activations/layer5_attention_weight_max": 53.515228271484375,
      "activations/layer5_attention_weight_min": -66.3428726196289,
      "activations/layer6_attention_weight_max": 44.5947151184082,
      "activations/layer6_attention_weight_min": -48.66560363769531,
      "activations/layer7_attention_weight_max": 98.1560287475586,
      "activations/layer7_attention_weight_min": -90.58891296386719,
      "activations/layer8_attention_weight_max": 44.63386917114258,
      "activations/layer8_attention_weight_min": -42.3488655090332,
      "activations/layer9_attention_weight_max": 33.61589813232422,
      "activations/layer9_attention_weight_min": -34.483734130859375,
      "epoch": 12.42,
      "learning_rate": 7.059810606060605e-05,
      "loss": 2.7709,
      "step": 213800
    },
    {
      "activations/layer0_attention_weight_max": 14.91763687133789,
      "activations/layer0_attention_weight_min": -12.381789207458496,
      "activations/layer10_attention_weight_max": 31.448312759399414,
      "activations/layer10_attention_weight_min": -34.14246368408203,
      "activations/layer11_attention_weight_max": 29.10816764831543,
      "activations/layer11_attention_weight_min": -32.24203872680664,
      "activations/layer12_attention_weight_max": 23.5009822845459,
      "activations/layer12_attention_weight_min": -24.61420440673828,
      "activations/layer13_attention_weight_max": 38.08529281616211,
      "activations/layer13_attention_weight_min": -34.065303802490234,
      "activations/layer14_attention_weight_max": 39.74221420288086,
      "activations/layer14_attention_weight_min": -36.53866958618164,
      "activations/layer15_attention_weight_max": 36.98715591430664,
      "activations/layer15_attention_weight_min": -31.73125457763672,
      "activations/layer16_attention_weight_max": 29.993175506591797,
      "activations/layer16_attention_weight_min": -26.896604537963867,
      "activations/layer17_attention_weight_max": 31.267908096313477,
      "activations/layer17_attention_weight_min": -24.830093383789062,
      "activations/layer18_attention_weight_max": 31.082744598388672,
      "activations/layer18_attention_weight_min": -23.07027816772461,
      "activations/layer19_attention_weight_max": 33.98621368408203,
      "activations/layer19_attention_weight_min": -29.85784339904785,
      "activations/layer1_attention_weight_max": 17.04206085205078,
      "activations/layer1_attention_weight_min": -15.272622108459473,
      "activations/layer20_attention_weight_max": 31.607433319091797,
      "activations/layer20_attention_weight_min": -26.4731388092041,
      "activations/layer21_attention_weight_max": 30.352413177490234,
      "activations/layer21_attention_weight_min": -23.95000457763672,
      "activations/layer22_attention_weight_max": 44.645957946777344,
      "activations/layer22_attention_weight_min": -28.82882308959961,
      "activations/layer23_attention_weight_max": 37.55615997314453,
      "activations/layer23_attention_weight_min": -24.28020477294922,
      "activations/layer2_attention_weight_max": 31.026832580566406,
      "activations/layer2_attention_weight_min": -31.197370529174805,
      "activations/layer3_attention_weight_max": 93.76105499267578,
      "activations/layer3_attention_weight_min": -95.65131378173828,
      "activations/layer4_attention_weight_max": 115.85420989990234,
      "activations/layer4_attention_weight_min": -114.18599700927734,
      "activations/layer5_attention_weight_max": 51.99029541015625,
      "activations/layer5_attention_weight_min": -64.44068145751953,
      "activations/layer6_attention_weight_max": 45.88369369506836,
      "activations/layer6_attention_weight_min": -50.12169647216797,
      "activations/layer7_attention_weight_max": 96.0768814086914,
      "activations/layer7_attention_weight_min": -97.032958984375,
      "activations/layer8_attention_weight_max": 39.430850982666016,
      "activations/layer8_attention_weight_min": -45.22075653076172,
      "activations/layer9_attention_weight_max": 31.339719772338867,
      "activations/layer9_attention_weight_min": -34.53287887573242,
      "epoch": 12.43,
      "learning_rate": 7.057916666666666e-05,
      "loss": 2.7769,
      "step": 213850
    },
    {
      "activations/layer0_attention_weight_max": 16.657974243164062,
      "activations/layer0_attention_weight_min": -12.485689163208008,
      "activations/layer10_attention_weight_max": 31.66072654724121,
      "activations/layer10_attention_weight_min": -33.423927307128906,
      "activations/layer11_attention_weight_max": 32.00730895996094,
      "activations/layer11_attention_weight_min": -33.13780975341797,
      "activations/layer12_attention_weight_max": 42.86309814453125,
      "activations/layer12_attention_weight_min": -26.040592193603516,
      "activations/layer13_attention_weight_max": 48.86814498901367,
      "activations/layer13_attention_weight_min": -36.50126647949219,
      "activations/layer14_attention_weight_max": 43.244293212890625,
      "activations/layer14_attention_weight_min": -38.300941467285156,
      "activations/layer15_attention_weight_max": 36.25525665283203,
      "activations/layer15_attention_weight_min": -33.59980773925781,
      "activations/layer16_attention_weight_max": 32.65720748901367,
      "activations/layer16_attention_weight_min": -28.572921752929688,
      "activations/layer17_attention_weight_max": 31.125038146972656,
      "activations/layer17_attention_weight_min": -26.627548217773438,
      "activations/layer18_attention_weight_max": 32.62803649902344,
      "activations/layer18_attention_weight_min": -23.702104568481445,
      "activations/layer19_attention_weight_max": 36.507057189941406,
      "activations/layer19_attention_weight_min": -32.133140563964844,
      "activations/layer1_attention_weight_max": 16.535358428955078,
      "activations/layer1_attention_weight_min": -14.12491512298584,
      "activations/layer20_attention_weight_max": 30.0129337310791,
      "activations/layer20_attention_weight_min": -24.193767547607422,
      "activations/layer21_attention_weight_max": 28.59160614013672,
      "activations/layer21_attention_weight_min": -22.999176025390625,
      "activations/layer22_attention_weight_max": 39.99958038330078,
      "activations/layer22_attention_weight_min": -28.49162483215332,
      "activations/layer23_attention_weight_max": 31.40868377685547,
      "activations/layer23_attention_weight_min": -23.85451316833496,
      "activations/layer2_attention_weight_max": 31.88210678100586,
      "activations/layer2_attention_weight_min": -29.640247344970703,
      "activations/layer3_attention_weight_max": 93.37628936767578,
      "activations/layer3_attention_weight_min": -92.32905578613281,
      "activations/layer4_attention_weight_max": 116.0926284790039,
      "activations/layer4_attention_weight_min": -110.56974029541016,
      "activations/layer5_attention_weight_max": 52.97151184082031,
      "activations/layer5_attention_weight_min": -62.6836051940918,
      "activations/layer6_attention_weight_max": 46.190616607666016,
      "activations/layer6_attention_weight_min": -49.79094696044922,
      "activations/layer7_attention_weight_max": 94.60608673095703,
      "activations/layer7_attention_weight_min": -96.38646697998047,
      "activations/layer8_attention_weight_max": 41.66190719604492,
      "activations/layer8_attention_weight_min": -42.145362854003906,
      "activations/layer9_attention_weight_max": 32.35964584350586,
      "activations/layer9_attention_weight_min": -33.23968505859375,
      "epoch": 12.43,
      "learning_rate": 7.056060606060605e-05,
      "loss": 2.7667,
      "step": 213900
    },
    {
      "activations/layer0_attention_weight_max": 14.58293628692627,
      "activations/layer0_attention_weight_min": -13.12956428527832,
      "activations/layer10_attention_weight_max": 33.51605987548828,
      "activations/layer10_attention_weight_min": -33.16973876953125,
      "activations/layer11_attention_weight_max": 37.42757797241211,
      "activations/layer11_attention_weight_min": -32.764713287353516,
      "activations/layer12_attention_weight_max": 34.865299224853516,
      "activations/layer12_attention_weight_min": -24.986310958862305,
      "activations/layer13_attention_weight_max": 46.06993103027344,
      "activations/layer13_attention_weight_min": -35.16611099243164,
      "activations/layer14_attention_weight_max": 40.61192321777344,
      "activations/layer14_attention_weight_min": -39.38042449951172,
      "activations/layer15_attention_weight_max": 36.87678527832031,
      "activations/layer15_attention_weight_min": -33.24300003051758,
      "activations/layer16_attention_weight_max": 29.565107345581055,
      "activations/layer16_attention_weight_min": -30.195463180541992,
      "activations/layer17_attention_weight_max": 29.83102798461914,
      "activations/layer17_attention_weight_min": -26.609907150268555,
      "activations/layer18_attention_weight_max": 30.337881088256836,
      "activations/layer18_attention_weight_min": -24.886293411254883,
      "activations/layer19_attention_weight_max": 35.490177154541016,
      "activations/layer19_attention_weight_min": -32.040523529052734,
      "activations/layer1_attention_weight_max": 16.056222915649414,
      "activations/layer1_attention_weight_min": -15.23553466796875,
      "activations/layer20_attention_weight_max": 27.365842819213867,
      "activations/layer20_attention_weight_min": -27.80293846130371,
      "activations/layer21_attention_weight_max": 26.102848052978516,
      "activations/layer21_attention_weight_min": -25.743896484375,
      "activations/layer22_attention_weight_max": 44.01155090332031,
      "activations/layer22_attention_weight_min": -32.23587417602539,
      "activations/layer23_attention_weight_max": 32.838401794433594,
      "activations/layer23_attention_weight_min": -25.281009674072266,
      "activations/layer2_attention_weight_max": 34.84971237182617,
      "activations/layer2_attention_weight_min": -32.021209716796875,
      "activations/layer3_attention_weight_max": 92.68124389648438,
      "activations/layer3_attention_weight_min": -93.25166320800781,
      "activations/layer4_attention_weight_max": 111.22599029541016,
      "activations/layer4_attention_weight_min": -106.97377014160156,
      "activations/layer5_attention_weight_max": 52.55445098876953,
      "activations/layer5_attention_weight_min": -63.60425567626953,
      "activations/layer6_attention_weight_max": 45.41010284423828,
      "activations/layer6_attention_weight_min": -47.71355438232422,
      "activations/layer7_attention_weight_max": 91.60633850097656,
      "activations/layer7_attention_weight_min": -90.46598815917969,
      "activations/layer8_attention_weight_max": 43.473854064941406,
      "activations/layer8_attention_weight_min": -41.36735916137695,
      "activations/layer9_attention_weight_max": 33.096805572509766,
      "activations/layer9_attention_weight_min": -31.840301513671875,
      "epoch": 12.43,
      "learning_rate": 7.054204545454544e-05,
      "loss": 2.7695,
      "step": 213950
    },
    {
      "activations/layer0_attention_weight_max": 16.32396125793457,
      "activations/layer0_attention_weight_min": -12.237606048583984,
      "activations/layer10_attention_weight_max": 43.90364074707031,
      "activations/layer10_attention_weight_min": -35.26341247558594,
      "activations/layer11_attention_weight_max": 44.15560531616211,
      "activations/layer11_attention_weight_min": -36.81273651123047,
      "activations/layer12_attention_weight_max": 28.761625289916992,
      "activations/layer12_attention_weight_min": -25.19499969482422,
      "activations/layer13_attention_weight_max": 45.788516998291016,
      "activations/layer13_attention_weight_min": -37.53286361694336,
      "activations/layer14_attention_weight_max": 46.25282287597656,
      "activations/layer14_attention_weight_min": -41.51984786987305,
      "activations/layer15_attention_weight_max": 34.71082305908203,
      "activations/layer15_attention_weight_min": -33.298004150390625,
      "activations/layer16_attention_weight_max": 32.50174331665039,
      "activations/layer16_attention_weight_min": -29.368148803710938,
      "activations/layer17_attention_weight_max": 27.704858779907227,
      "activations/layer17_attention_weight_min": -25.862886428833008,
      "activations/layer18_attention_weight_max": 33.14958190917969,
      "activations/layer18_attention_weight_min": -23.28992462158203,
      "activations/layer19_attention_weight_max": 37.5617561340332,
      "activations/layer19_attention_weight_min": -33.89773178100586,
      "activations/layer1_attention_weight_max": 19.730907440185547,
      "activations/layer1_attention_weight_min": -14.395929336547852,
      "activations/layer20_attention_weight_max": 29.647478103637695,
      "activations/layer20_attention_weight_min": -25.779155731201172,
      "activations/layer21_attention_weight_max": 31.99945068359375,
      "activations/layer21_attention_weight_min": -23.83872413635254,
      "activations/layer22_attention_weight_max": 48.22725296020508,
      "activations/layer22_attention_weight_min": -32.300872802734375,
      "activations/layer23_attention_weight_max": 36.13323974609375,
      "activations/layer23_attention_weight_min": -26.044767379760742,
      "activations/layer2_attention_weight_max": 32.929439544677734,
      "activations/layer2_attention_weight_min": -30.707630157470703,
      "activations/layer3_attention_weight_max": 92.813232421875,
      "activations/layer3_attention_weight_min": -96.01270294189453,
      "activations/layer4_attention_weight_max": 107.77965545654297,
      "activations/layer4_attention_weight_min": -107.8759536743164,
      "activations/layer5_attention_weight_max": 52.9888916015625,
      "activations/layer5_attention_weight_min": -59.25310516357422,
      "activations/layer6_attention_weight_max": 43.537899017333984,
      "activations/layer6_attention_weight_min": -46.45619583129883,
      "activations/layer7_attention_weight_max": 90.7242202758789,
      "activations/layer7_attention_weight_min": -90.67660522460938,
      "activations/layer8_attention_weight_max": 43.01166534423828,
      "activations/layer8_attention_weight_min": -41.76679229736328,
      "activations/layer9_attention_weight_max": 34.747291564941406,
      "activations/layer9_attention_weight_min": -35.98064422607422,
      "epoch": 12.43,
      "learning_rate": 7.052310606060606e-05,
      "loss": 2.776,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4541,
      "eval_samples_per_second": 507.921,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4541,
      "eval_openwebtext_samples_per_second": 507.921,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9445,
      "eval_wikitext_samples_per_second": 234.505,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_lambada_loss": 2.5234375,
      "eval_lambada_ppl": 12.471393479493909,
      "eval_lambada_runtime": 9.496,
      "eval_lambada_samples_per_second": 512.744,
      "step": 214000
    },
    {
      "activations/layer0_attention_weight_max": 15.527494430541992,
      "activations/layer0_attention_weight_min": -11.904279708862305,
      "activations/layer10_attention_weight_max": 29.314714431762695,
      "activations/layer10_attention_weight_min": -31.73344612121582,
      "activations/layer11_attention_weight_max": 30.380979537963867,
      "activations/layer11_attention_weight_min": -31.447948455810547,
      "activations/layer12_attention_weight_max": 25.56169891357422,
      "activations/layer12_attention_weight_min": -24.87557601928711,
      "activations/layer13_attention_weight_max": 36.08448791503906,
      "activations/layer13_attention_weight_min": -34.02204895019531,
      "activations/layer14_attention_weight_max": 46.05306625366211,
      "activations/layer14_attention_weight_min": -39.85017395019531,
      "activations/layer15_attention_weight_max": 32.78759765625,
      "activations/layer15_attention_weight_min": -32.98698806762695,
      "activations/layer16_attention_weight_max": 28.367305755615234,
      "activations/layer16_attention_weight_min": -27.727149963378906,
      "activations/layer17_attention_weight_max": 28.55978775024414,
      "activations/layer17_attention_weight_min": -25.122787475585938,
      "activations/layer18_attention_weight_max": 28.406898498535156,
      "activations/layer18_attention_weight_min": -24.825458526611328,
      "activations/layer19_attention_weight_max": 34.202735900878906,
      "activations/layer19_attention_weight_min": -31.47564125061035,
      "activations/layer1_attention_weight_max": 17.350360870361328,
      "activations/layer1_attention_weight_min": -13.200714111328125,
      "activations/layer20_attention_weight_max": 27.176799774169922,
      "activations/layer20_attention_weight_min": -24.760169982910156,
      "activations/layer21_attention_weight_max": 26.56955909729004,
      "activations/layer21_attention_weight_min": -23.709341049194336,
      "activations/layer22_attention_weight_max": 38.259700775146484,
      "activations/layer22_attention_weight_min": -30.045055389404297,
      "activations/layer23_attention_weight_max": 32.07544708251953,
      "activations/layer23_attention_weight_min": -24.856090545654297,
      "activations/layer2_attention_weight_max": 28.992008209228516,
      "activations/layer2_attention_weight_min": -29.393659591674805,
      "activations/layer3_attention_weight_max": 85.66673278808594,
      "activations/layer3_attention_weight_min": -87.22386169433594,
      "activations/layer4_attention_weight_max": 108.6182861328125,
      "activations/layer4_attention_weight_min": -104.08846282958984,
      "activations/layer5_attention_weight_max": 50.70140838623047,
      "activations/layer5_attention_weight_min": -59.51044464111328,
      "activations/layer6_attention_weight_max": 44.43482971191406,
      "activations/layer6_attention_weight_min": -47.41450500488281,
      "activations/layer7_attention_weight_max": 86.66664123535156,
      "activations/layer7_attention_weight_min": -91.00233459472656,
      "activations/layer8_attention_weight_max": 39.6797981262207,
      "activations/layer8_attention_weight_min": -40.750877380371094,
      "activations/layer9_attention_weight_max": 30.927581787109375,
      "activations/layer9_attention_weight_min": -31.250446319580078,
      "epoch": 12.44,
      "learning_rate": 7.050416666666666e-05,
      "loss": 2.7695,
      "step": 214050
    },
    {
      "activations/layer0_attention_weight_max": 15.623372077941895,
      "activations/layer0_attention_weight_min": -12.254626274108887,
      "activations/layer10_attention_weight_max": 34.952552795410156,
      "activations/layer10_attention_weight_min": -34.706199645996094,
      "activations/layer11_attention_weight_max": 35.54431915283203,
      "activations/layer11_attention_weight_min": -34.67095947265625,
      "activations/layer12_attention_weight_max": 25.359777450561523,
      "activations/layer12_attention_weight_min": -24.766162872314453,
      "activations/layer13_attention_weight_max": 41.45317077636719,
      "activations/layer13_attention_weight_min": -35.2797966003418,
      "activations/layer14_attention_weight_max": 42.93888854980469,
      "activations/layer14_attention_weight_min": -36.19649124145508,
      "activations/layer15_attention_weight_max": 36.421714782714844,
      "activations/layer15_attention_weight_min": -33.0274543762207,
      "activations/layer16_attention_weight_max": 35.31238555908203,
      "activations/layer16_attention_weight_min": -28.29460334777832,
      "activations/layer17_attention_weight_max": 30.75994300842285,
      "activations/layer17_attention_weight_min": -25.11668586730957,
      "activations/layer18_attention_weight_max": 31.549396514892578,
      "activations/layer18_attention_weight_min": -25.61770248413086,
      "activations/layer19_attention_weight_max": 34.64338684082031,
      "activations/layer19_attention_weight_min": -32.974483489990234,
      "activations/layer1_attention_weight_max": 16.541667938232422,
      "activations/layer1_attention_weight_min": -13.014517784118652,
      "activations/layer20_attention_weight_max": 27.72933578491211,
      "activations/layer20_attention_weight_min": -25.911376953125,
      "activations/layer21_attention_weight_max": 28.204740524291992,
      "activations/layer21_attention_weight_min": -23.374313354492188,
      "activations/layer22_attention_weight_max": 43.38206100463867,
      "activations/layer22_attention_weight_min": -28.825056076049805,
      "activations/layer23_attention_weight_max": 33.44621276855469,
      "activations/layer23_attention_weight_min": -23.148765563964844,
      "activations/layer2_attention_weight_max": 29.75710678100586,
      "activations/layer2_attention_weight_min": -29.83629608154297,
      "activations/layer3_attention_weight_max": 91.45571899414062,
      "activations/layer3_attention_weight_min": -93.19412231445312,
      "activations/layer4_attention_weight_max": 109.54174041748047,
      "activations/layer4_attention_weight_min": -112.98992919921875,
      "activations/layer5_attention_weight_max": 55.1388053894043,
      "activations/layer5_attention_weight_min": -63.77728271484375,
      "activations/layer6_attention_weight_max": 44.79660415649414,
      "activations/layer6_attention_weight_min": -49.90758514404297,
      "activations/layer7_attention_weight_max": 95.20123291015625,
      "activations/layer7_attention_weight_min": -98.35371398925781,
      "activations/layer8_attention_weight_max": 41.370338439941406,
      "activations/layer8_attention_weight_min": -43.04839324951172,
      "activations/layer9_attention_weight_max": 35.34040069580078,
      "activations/layer9_attention_weight_min": -36.31131362915039,
      "epoch": 12.44,
      "learning_rate": 7.048522727272726e-05,
      "loss": 2.7747,
      "step": 214100
    },
    {
      "activations/layer0_attention_weight_max": 15.105416297912598,
      "activations/layer0_attention_weight_min": -11.401029586791992,
      "activations/layer10_attention_weight_max": 34.5263671875,
      "activations/layer10_attention_weight_min": -32.25392150878906,
      "activations/layer11_attention_weight_max": 35.27340316772461,
      "activations/layer11_attention_weight_min": -31.910932540893555,
      "activations/layer12_attention_weight_max": 26.06669807434082,
      "activations/layer12_attention_weight_min": -26.840164184570312,
      "activations/layer13_attention_weight_max": 40.291927337646484,
      "activations/layer13_attention_weight_min": -36.45082092285156,
      "activations/layer14_attention_weight_max": 39.60841369628906,
      "activations/layer14_attention_weight_min": -36.57243728637695,
      "activations/layer15_attention_weight_max": 35.841976165771484,
      "activations/layer15_attention_weight_min": -30.8387393951416,
      "activations/layer16_attention_weight_max": 29.782930374145508,
      "activations/layer16_attention_weight_min": -28.5482120513916,
      "activations/layer17_attention_weight_max": 31.48795509338379,
      "activations/layer17_attention_weight_min": -26.086362838745117,
      "activations/layer18_attention_weight_max": 29.995647430419922,
      "activations/layer18_attention_weight_min": -22.677366256713867,
      "activations/layer19_attention_weight_max": 33.08097839355469,
      "activations/layer19_attention_weight_min": -30.695117950439453,
      "activations/layer1_attention_weight_max": 17.408123016357422,
      "activations/layer1_attention_weight_min": -14.383831977844238,
      "activations/layer20_attention_weight_max": 26.724613189697266,
      "activations/layer20_attention_weight_min": -24.03191566467285,
      "activations/layer21_attention_weight_max": 28.490182876586914,
      "activations/layer21_attention_weight_min": -23.787755966186523,
      "activations/layer22_attention_weight_max": 37.60291290283203,
      "activations/layer22_attention_weight_min": -29.000776290893555,
      "activations/layer23_attention_weight_max": 31.864059448242188,
      "activations/layer23_attention_weight_min": -22.74477767944336,
      "activations/layer2_attention_weight_max": 30.402881622314453,
      "activations/layer2_attention_weight_min": -30.506752014160156,
      "activations/layer3_attention_weight_max": 86.71232604980469,
      "activations/layer3_attention_weight_min": -90.77478790283203,
      "activations/layer4_attention_weight_max": 104.1769790649414,
      "activations/layer4_attention_weight_min": -104.5293960571289,
      "activations/layer5_attention_weight_max": 47.32073974609375,
      "activations/layer5_attention_weight_min": -63.28657150268555,
      "activations/layer6_attention_weight_max": 42.149444580078125,
      "activations/layer6_attention_weight_min": -45.25603103637695,
      "activations/layer7_attention_weight_max": 87.70794677734375,
      "activations/layer7_attention_weight_min": -85.48636627197266,
      "activations/layer8_attention_weight_max": 37.65212631225586,
      "activations/layer8_attention_weight_min": -39.16497039794922,
      "activations/layer9_attention_weight_max": 32.60032653808594,
      "activations/layer9_attention_weight_min": -33.04909896850586,
      "epoch": 12.44,
      "learning_rate": 7.046628787878788e-05,
      "loss": 2.77,
      "step": 214150
    },
    {
      "activations/layer0_attention_weight_max": 16.140188217163086,
      "activations/layer0_attention_weight_min": -11.473052024841309,
      "activations/layer10_attention_weight_max": 33.03693389892578,
      "activations/layer10_attention_weight_min": -34.99494171142578,
      "activations/layer11_attention_weight_max": 33.992549896240234,
      "activations/layer11_attention_weight_min": -33.0029411315918,
      "activations/layer12_attention_weight_max": 23.73681640625,
      "activations/layer12_attention_weight_min": -25.93412208557129,
      "activations/layer13_attention_weight_max": 38.095458984375,
      "activations/layer13_attention_weight_min": -33.91944885253906,
      "activations/layer14_attention_weight_max": 43.49536895751953,
      "activations/layer14_attention_weight_min": -37.04161071777344,
      "activations/layer15_attention_weight_max": 35.877899169921875,
      "activations/layer15_attention_weight_min": -32.75875473022461,
      "activations/layer16_attention_weight_max": 31.025882720947266,
      "activations/layer16_attention_weight_min": -27.81656265258789,
      "activations/layer17_attention_weight_max": 32.12012481689453,
      "activations/layer17_attention_weight_min": -26.261104583740234,
      "activations/layer18_attention_weight_max": 30.393709182739258,
      "activations/layer18_attention_weight_min": -27.930259704589844,
      "activations/layer19_attention_weight_max": 42.32706832885742,
      "activations/layer19_attention_weight_min": -31.800073623657227,
      "activations/layer1_attention_weight_max": 17.1556339263916,
      "activations/layer1_attention_weight_min": -14.167642593383789,
      "activations/layer20_attention_weight_max": 30.39488983154297,
      "activations/layer20_attention_weight_min": -26.48503875732422,
      "activations/layer21_attention_weight_max": 32.8612060546875,
      "activations/layer21_attention_weight_min": -25.2790470123291,
      "activations/layer22_attention_weight_max": 48.545936584472656,
      "activations/layer22_attention_weight_min": -35.3929443359375,
      "activations/layer23_attention_weight_max": 39.216697692871094,
      "activations/layer23_attention_weight_min": -28.97536849975586,
      "activations/layer2_attention_weight_max": 29.076416015625,
      "activations/layer2_attention_weight_min": -30.51338768005371,
      "activations/layer3_attention_weight_max": 91.0592269897461,
      "activations/layer3_attention_weight_min": -89.28155517578125,
      "activations/layer4_attention_weight_max": 103.8891372680664,
      "activations/layer4_attention_weight_min": -108.00025939941406,
      "activations/layer5_attention_weight_max": 47.97561264038086,
      "activations/layer5_attention_weight_min": -62.13961410522461,
      "activations/layer6_attention_weight_max": 45.24701690673828,
      "activations/layer6_attention_weight_min": -49.39777374267578,
      "activations/layer7_attention_weight_max": 91.57378387451172,
      "activations/layer7_attention_weight_min": -93.63899230957031,
      "activations/layer8_attention_weight_max": 40.03078079223633,
      "activations/layer8_attention_weight_min": -43.94597244262695,
      "activations/layer9_attention_weight_max": 31.16132926940918,
      "activations/layer9_attention_weight_min": -34.78950500488281,
      "epoch": 12.45,
      "learning_rate": 7.044734848484848e-05,
      "loss": 2.7747,
      "step": 214200
    },
    {
      "activations/layer0_attention_weight_max": 15.615768432617188,
      "activations/layer0_attention_weight_min": -12.83141803741455,
      "activations/layer10_attention_weight_max": 32.710243225097656,
      "activations/layer10_attention_weight_min": -34.125404357910156,
      "activations/layer11_attention_weight_max": 33.02947998046875,
      "activations/layer11_attention_weight_min": -35.19951629638672,
      "activations/layer12_attention_weight_max": 24.98065948486328,
      "activations/layer12_attention_weight_min": -24.793222427368164,
      "activations/layer13_attention_weight_max": 37.340633392333984,
      "activations/layer13_attention_weight_min": -33.95583724975586,
      "activations/layer14_attention_weight_max": 42.145729064941406,
      "activations/layer14_attention_weight_min": -37.284934997558594,
      "activations/layer15_attention_weight_max": 35.898128509521484,
      "activations/layer15_attention_weight_min": -34.76095199584961,
      "activations/layer16_attention_weight_max": 29.09575080871582,
      "activations/layer16_attention_weight_min": -29.33536720275879,
      "activations/layer17_attention_weight_max": 29.936594009399414,
      "activations/layer17_attention_weight_min": -27.478599548339844,
      "activations/layer18_attention_weight_max": 32.37989807128906,
      "activations/layer18_attention_weight_min": -23.008975982666016,
      "activations/layer19_attention_weight_max": 31.98213768005371,
      "activations/layer19_attention_weight_min": -30.1253719329834,
      "activations/layer1_attention_weight_max": 17.0618839263916,
      "activations/layer1_attention_weight_min": -15.546663284301758,
      "activations/layer20_attention_weight_max": 29.362876892089844,
      "activations/layer20_attention_weight_min": -24.190826416015625,
      "activations/layer21_attention_weight_max": 29.5440731048584,
      "activations/layer21_attention_weight_min": -23.04509925842285,
      "activations/layer22_attention_weight_max": 48.452674865722656,
      "activations/layer22_attention_weight_min": -28.690250396728516,
      "activations/layer23_attention_weight_max": 35.063018798828125,
      "activations/layer23_attention_weight_min": -24.641475677490234,
      "activations/layer2_attention_weight_max": 30.340377807617188,
      "activations/layer2_attention_weight_min": -30.70461082458496,
      "activations/layer3_attention_weight_max": 85.75736999511719,
      "activations/layer3_attention_weight_min": -91.48173522949219,
      "activations/layer4_attention_weight_max": 109.07208251953125,
      "activations/layer4_attention_weight_min": -112.44435119628906,
      "activations/layer5_attention_weight_max": 50.539512634277344,
      "activations/layer5_attention_weight_min": -60.44645690917969,
      "activations/layer6_attention_weight_max": 44.31205749511719,
      "activations/layer6_attention_weight_min": -46.47245788574219,
      "activations/layer7_attention_weight_max": 83.85017395019531,
      "activations/layer7_attention_weight_min": -89.52179718017578,
      "activations/layer8_attention_weight_max": 40.25319290161133,
      "activations/layer8_attention_weight_min": -41.62068557739258,
      "activations/layer9_attention_weight_max": 35.30170440673828,
      "activations/layer9_attention_weight_min": -34.502498626708984,
      "epoch": 12.45,
      "learning_rate": 7.042840909090908e-05,
      "loss": 2.7655,
      "step": 214250
    },
    {
      "activations/layer0_attention_weight_max": 15.856667518615723,
      "activations/layer0_attention_weight_min": -12.474384307861328,
      "activations/layer10_attention_weight_max": 37.96659851074219,
      "activations/layer10_attention_weight_min": -37.153602600097656,
      "activations/layer11_attention_weight_max": 39.23207092285156,
      "activations/layer11_attention_weight_min": -36.4970703125,
      "activations/layer12_attention_weight_max": 25.69225311279297,
      "activations/layer12_attention_weight_min": -27.62833023071289,
      "activations/layer13_attention_weight_max": 41.61466979980469,
      "activations/layer13_attention_weight_min": -40.0646858215332,
      "activations/layer14_attention_weight_max": 40.89494705200195,
      "activations/layer14_attention_weight_min": -41.49465560913086,
      "activations/layer15_attention_weight_max": 34.89985275268555,
      "activations/layer15_attention_weight_min": -37.370697021484375,
      "activations/layer16_attention_weight_max": 28.31562614440918,
      "activations/layer16_attention_weight_min": -28.626983642578125,
      "activations/layer17_attention_weight_max": 27.66421890258789,
      "activations/layer17_attention_weight_min": -28.05328369140625,
      "activations/layer18_attention_weight_max": 28.565841674804688,
      "activations/layer18_attention_weight_min": -25.090559005737305,
      "activations/layer19_attention_weight_max": 30.492568969726562,
      "activations/layer19_attention_weight_min": -31.270017623901367,
      "activations/layer1_attention_weight_max": 16.208707809448242,
      "activations/layer1_attention_weight_min": -16.312002182006836,
      "activations/layer20_attention_weight_max": 29.129976272583008,
      "activations/layer20_attention_weight_min": -24.532752990722656,
      "activations/layer21_attention_weight_max": 27.19571876525879,
      "activations/layer21_attention_weight_min": -23.505172729492188,
      "activations/layer22_attention_weight_max": 40.15433883666992,
      "activations/layer22_attention_weight_min": -28.966428756713867,
      "activations/layer23_attention_weight_max": 32.58330154418945,
      "activations/layer23_attention_weight_min": -25.790775299072266,
      "activations/layer2_attention_weight_max": 32.11550521850586,
      "activations/layer2_attention_weight_min": -30.016033172607422,
      "activations/layer3_attention_weight_max": 96.75817108154297,
      "activations/layer3_attention_weight_min": -100.96329498291016,
      "activations/layer4_attention_weight_max": 116.97283172607422,
      "activations/layer4_attention_weight_min": -119.50078582763672,
      "activations/layer5_attention_weight_max": 50.42138671875,
      "activations/layer5_attention_weight_min": -59.864803314208984,
      "activations/layer6_attention_weight_max": 47.14760971069336,
      "activations/layer6_attention_weight_min": -47.16366958618164,
      "activations/layer7_attention_weight_max": 105.3504867553711,
      "activations/layer7_attention_weight_min": -95.8633041381836,
      "activations/layer8_attention_weight_max": 48.774696350097656,
      "activations/layer8_attention_weight_min": -45.95456314086914,
      "activations/layer9_attention_weight_max": 39.37322235107422,
      "activations/layer9_attention_weight_min": -37.86263656616211,
      "epoch": 12.45,
      "learning_rate": 7.04094696969697e-05,
      "loss": 2.7941,
      "step": 214300
    },
    {
      "activations/layer0_attention_weight_max": 16.41924285888672,
      "activations/layer0_attention_weight_min": -12.403912544250488,
      "activations/layer10_attention_weight_max": 32.63582229614258,
      "activations/layer10_attention_weight_min": -33.47468948364258,
      "activations/layer11_attention_weight_max": 32.53823471069336,
      "activations/layer11_attention_weight_min": -32.73955535888672,
      "activations/layer12_attention_weight_max": 24.267011642456055,
      "activations/layer12_attention_weight_min": -26.28619956970215,
      "activations/layer13_attention_weight_max": 37.05915451049805,
      "activations/layer13_attention_weight_min": -33.81982421875,
      "activations/layer14_attention_weight_max": 44.74186706542969,
      "activations/layer14_attention_weight_min": -41.06046676635742,
      "activations/layer15_attention_weight_max": 34.10950469970703,
      "activations/layer15_attention_weight_min": -32.99658203125,
      "activations/layer16_attention_weight_max": 29.070077896118164,
      "activations/layer16_attention_weight_min": -27.6966495513916,
      "activations/layer17_attention_weight_max": 28.46006965637207,
      "activations/layer17_attention_weight_min": -25.60003089904785,
      "activations/layer18_attention_weight_max": 27.724882125854492,
      "activations/layer18_attention_weight_min": -23.209957122802734,
      "activations/layer19_attention_weight_max": 29.038110733032227,
      "activations/layer19_attention_weight_min": -29.428558349609375,
      "activations/layer1_attention_weight_max": 16.657440185546875,
      "activations/layer1_attention_weight_min": -15.24443244934082,
      "activations/layer20_attention_weight_max": 27.05670166015625,
      "activations/layer20_attention_weight_min": -24.339733123779297,
      "activations/layer21_attention_weight_max": 25.512252807617188,
      "activations/layer21_attention_weight_min": -24.61099624633789,
      "activations/layer22_attention_weight_max": 36.387081146240234,
      "activations/layer22_attention_weight_min": -30.715749740600586,
      "activations/layer23_attention_weight_max": 30.73073959350586,
      "activations/layer23_attention_weight_min": -25.4168701171875,
      "activations/layer2_attention_weight_max": 29.324180603027344,
      "activations/layer2_attention_weight_min": -29.116308212280273,
      "activations/layer3_attention_weight_max": 82.89236450195312,
      "activations/layer3_attention_weight_min": -86.66824340820312,
      "activations/layer4_attention_weight_max": 104.378662109375,
      "activations/layer4_attention_weight_min": -105.548828125,
      "activations/layer5_attention_weight_max": 49.338157653808594,
      "activations/layer5_attention_weight_min": -58.18142318725586,
      "activations/layer6_attention_weight_max": 45.73044204711914,
      "activations/layer6_attention_weight_min": -48.23039245605469,
      "activations/layer7_attention_weight_max": 89.20096588134766,
      "activations/layer7_attention_weight_min": -91.80126953125,
      "activations/layer8_attention_weight_max": 40.87154006958008,
      "activations/layer8_attention_weight_min": -42.703426361083984,
      "activations/layer9_attention_weight_max": 31.61408805847168,
      "activations/layer9_attention_weight_min": -33.645137786865234,
      "epoch": 12.45,
      "learning_rate": 7.03905303030303e-05,
      "loss": 2.7602,
      "step": 214350
    },
    {
      "activations/layer0_attention_weight_max": 14.712404251098633,
      "activations/layer0_attention_weight_min": -12.158957481384277,
      "activations/layer10_attention_weight_max": 32.945045471191406,
      "activations/layer10_attention_weight_min": -30.83053970336914,
      "activations/layer11_attention_weight_max": 32.5075798034668,
      "activations/layer11_attention_weight_min": -30.706396102905273,
      "activations/layer12_attention_weight_max": 25.45891761779785,
      "activations/layer12_attention_weight_min": -24.920242309570312,
      "activations/layer13_attention_weight_max": 37.94133758544922,
      "activations/layer13_attention_weight_min": -32.825931549072266,
      "activations/layer14_attention_weight_max": 41.77594757080078,
      "activations/layer14_attention_weight_min": -36.341583251953125,
      "activations/layer15_attention_weight_max": 36.133113861083984,
      "activations/layer15_attention_weight_min": -33.240482330322266,
      "activations/layer16_attention_weight_max": 32.8034553527832,
      "activations/layer16_attention_weight_min": -28.47752571105957,
      "activations/layer17_attention_weight_max": 33.002685546875,
      "activations/layer17_attention_weight_min": -26.889110565185547,
      "activations/layer18_attention_weight_max": 35.29705047607422,
      "activations/layer18_attention_weight_min": -27.970190048217773,
      "activations/layer19_attention_weight_max": 34.59547805786133,
      "activations/layer19_attention_weight_min": -32.5813102722168,
      "activations/layer1_attention_weight_max": 16.150802612304688,
      "activations/layer1_attention_weight_min": -15.94845962524414,
      "activations/layer20_attention_weight_max": 29.678298950195312,
      "activations/layer20_attention_weight_min": -25.460819244384766,
      "activations/layer21_attention_weight_max": 30.698949813842773,
      "activations/layer21_attention_weight_min": -25.197519302368164,
      "activations/layer22_attention_weight_max": 45.53152847290039,
      "activations/layer22_attention_weight_min": -32.66337966918945,
      "activations/layer23_attention_weight_max": 37.69834899902344,
      "activations/layer23_attention_weight_min": -27.589431762695312,
      "activations/layer2_attention_weight_max": 31.132854461669922,
      "activations/layer2_attention_weight_min": -28.946125030517578,
      "activations/layer3_attention_weight_max": 94.7708969116211,
      "activations/layer3_attention_weight_min": -93.57526397705078,
      "activations/layer4_attention_weight_max": 110.5423812866211,
      "activations/layer4_attention_weight_min": -115.35401916503906,
      "activations/layer5_attention_weight_max": 54.22927474975586,
      "activations/layer5_attention_weight_min": -65.9495849609375,
      "activations/layer6_attention_weight_max": 46.96638870239258,
      "activations/layer6_attention_weight_min": -49.054954528808594,
      "activations/layer7_attention_weight_max": 93.09630584716797,
      "activations/layer7_attention_weight_min": -88.88866424560547,
      "activations/layer8_attention_weight_max": 41.51457977294922,
      "activations/layer8_attention_weight_min": -40.481571197509766,
      "activations/layer9_attention_weight_max": 34.27109146118164,
      "activations/layer9_attention_weight_min": -32.242794036865234,
      "epoch": 12.46,
      "learning_rate": 7.03715909090909e-05,
      "loss": 2.7707,
      "step": 214400
    },
    {
      "activations/layer0_attention_weight_max": 14.608113288879395,
      "activations/layer0_attention_weight_min": -11.825282096862793,
      "activations/layer10_attention_weight_max": 34.194427490234375,
      "activations/layer10_attention_weight_min": -33.309173583984375,
      "activations/layer11_attention_weight_max": 34.12694549560547,
      "activations/layer11_attention_weight_min": -34.562416076660156,
      "activations/layer12_attention_weight_max": 27.33619499206543,
      "activations/layer12_attention_weight_min": -24.648698806762695,
      "activations/layer13_attention_weight_max": 39.17772674560547,
      "activations/layer13_attention_weight_min": -35.030147552490234,
      "activations/layer14_attention_weight_max": 46.46148681640625,
      "activations/layer14_attention_weight_min": -37.24126052856445,
      "activations/layer15_attention_weight_max": 36.320247650146484,
      "activations/layer15_attention_weight_min": -34.251285552978516,
      "activations/layer16_attention_weight_max": 30.283422470092773,
      "activations/layer16_attention_weight_min": -28.052202224731445,
      "activations/layer17_attention_weight_max": 32.41122055053711,
      "activations/layer17_attention_weight_min": -25.431730270385742,
      "activations/layer18_attention_weight_max": 31.662071228027344,
      "activations/layer18_attention_weight_min": -22.907981872558594,
      "activations/layer19_attention_weight_max": 36.04794692993164,
      "activations/layer19_attention_weight_min": -32.64362716674805,
      "activations/layer1_attention_weight_max": 15.763800621032715,
      "activations/layer1_attention_weight_min": -14.416655540466309,
      "activations/layer20_attention_weight_max": 29.862077713012695,
      "activations/layer20_attention_weight_min": -26.107824325561523,
      "activations/layer21_attention_weight_max": 30.947935104370117,
      "activations/layer21_attention_weight_min": -25.01556968688965,
      "activations/layer22_attention_weight_max": 49.50344467163086,
      "activations/layer22_attention_weight_min": -31.893795013427734,
      "activations/layer23_attention_weight_max": 37.51561737060547,
      "activations/layer23_attention_weight_min": -27.170635223388672,
      "activations/layer2_attention_weight_max": 31.317562103271484,
      "activations/layer2_attention_weight_min": -32.88319396972656,
      "activations/layer3_attention_weight_max": 91.09481811523438,
      "activations/layer3_attention_weight_min": -96.29639434814453,
      "activations/layer4_attention_weight_max": 107.58223724365234,
      "activations/layer4_attention_weight_min": -113.26446533203125,
      "activations/layer5_attention_weight_max": 50.85676574707031,
      "activations/layer5_attention_weight_min": -65.60108947753906,
      "activations/layer6_attention_weight_max": 45.79541778564453,
      "activations/layer6_attention_weight_min": -49.55168914794922,
      "activations/layer7_attention_weight_max": 89.676025390625,
      "activations/layer7_attention_weight_min": -96.52299499511719,
      "activations/layer8_attention_weight_max": 40.618988037109375,
      "activations/layer8_attention_weight_min": -42.75153732299805,
      "activations/layer9_attention_weight_max": 34.15034866333008,
      "activations/layer9_attention_weight_min": -34.245845794677734,
      "epoch": 12.46,
      "learning_rate": 7.03526515151515e-05,
      "loss": 2.7769,
      "step": 214450
    },
    {
      "activations/layer0_attention_weight_max": 15.458833694458008,
      "activations/layer0_attention_weight_min": -12.79548168182373,
      "activations/layer10_attention_weight_max": 32.6797981262207,
      "activations/layer10_attention_weight_min": -33.02339553833008,
      "activations/layer11_attention_weight_max": 32.56255340576172,
      "activations/layer11_attention_weight_min": -31.181716918945312,
      "activations/layer12_attention_weight_max": 24.409921646118164,
      "activations/layer12_attention_weight_min": -25.9256649017334,
      "activations/layer13_attention_weight_max": 40.116905212402344,
      "activations/layer13_attention_weight_min": -35.15692901611328,
      "activations/layer14_attention_weight_max": 40.40806579589844,
      "activations/layer14_attention_weight_min": -35.66658020019531,
      "activations/layer15_attention_weight_max": 35.949466705322266,
      "activations/layer15_attention_weight_min": -31.947948455810547,
      "activations/layer16_attention_weight_max": 29.97911834716797,
      "activations/layer16_attention_weight_min": -26.43105697631836,
      "activations/layer17_attention_weight_max": 35.58037185668945,
      "activations/layer17_attention_weight_min": -24.936866760253906,
      "activations/layer18_attention_weight_max": 33.081600189208984,
      "activations/layer18_attention_weight_min": -24.235944747924805,
      "activations/layer19_attention_weight_max": 36.34331512451172,
      "activations/layer19_attention_weight_min": -31.171140670776367,
      "activations/layer1_attention_weight_max": 17.073530197143555,
      "activations/layer1_attention_weight_min": -15.006782531738281,
      "activations/layer20_attention_weight_max": 31.70781135559082,
      "activations/layer20_attention_weight_min": -25.235441207885742,
      "activations/layer21_attention_weight_max": 31.041641235351562,
      "activations/layer21_attention_weight_min": -24.600908279418945,
      "activations/layer22_attention_weight_max": 42.320472717285156,
      "activations/layer22_attention_weight_min": -29.20237159729004,
      "activations/layer23_attention_weight_max": 34.68115234375,
      "activations/layer23_attention_weight_min": -24.326007843017578,
      "activations/layer2_attention_weight_max": 30.556961059570312,
      "activations/layer2_attention_weight_min": -29.740947723388672,
      "activations/layer3_attention_weight_max": 88.95175170898438,
      "activations/layer3_attention_weight_min": -91.4613265991211,
      "activations/layer4_attention_weight_max": 101.8468017578125,
      "activations/layer4_attention_weight_min": -105.9614028930664,
      "activations/layer5_attention_weight_max": 48.22404479980469,
      "activations/layer5_attention_weight_min": -61.492210388183594,
      "activations/layer6_attention_weight_max": 42.785499572753906,
      "activations/layer6_attention_weight_min": -46.72126007080078,
      "activations/layer7_attention_weight_max": 88.7864761352539,
      "activations/layer7_attention_weight_min": -85.81395721435547,
      "activations/layer8_attention_weight_max": 39.310691833496094,
      "activations/layer8_attention_weight_min": -41.8321647644043,
      "activations/layer9_attention_weight_max": 33.98515701293945,
      "activations/layer9_attention_weight_min": -34.209144592285156,
      "epoch": 12.46,
      "learning_rate": 7.033371212121212e-05,
      "loss": 2.777,
      "step": 214500
    },
    {
      "activations/layer0_attention_weight_max": 15.808998107910156,
      "activations/layer0_attention_weight_min": -12.850071907043457,
      "activations/layer10_attention_weight_max": 31.253543853759766,
      "activations/layer10_attention_weight_min": -32.34315490722656,
      "activations/layer11_attention_weight_max": 31.541946411132812,
      "activations/layer11_attention_weight_min": -31.631309509277344,
      "activations/layer12_attention_weight_max": 22.91960906982422,
      "activations/layer12_attention_weight_min": -25.72996711730957,
      "activations/layer13_attention_weight_max": 35.3193244934082,
      "activations/layer13_attention_weight_min": -35.406551361083984,
      "activations/layer14_attention_weight_max": 38.56669235229492,
      "activations/layer14_attention_weight_min": -38.89352035522461,
      "activations/layer15_attention_weight_max": 32.24589157104492,
      "activations/layer15_attention_weight_min": -32.13899612426758,
      "activations/layer16_attention_weight_max": 28.696943283081055,
      "activations/layer16_attention_weight_min": -29.216821670532227,
      "activations/layer17_attention_weight_max": 27.410226821899414,
      "activations/layer17_attention_weight_min": -25.67148208618164,
      "activations/layer18_attention_weight_max": 30.043485641479492,
      "activations/layer18_attention_weight_min": -26.53411102294922,
      "activations/layer19_attention_weight_max": 30.558429718017578,
      "activations/layer19_attention_weight_min": -31.4837589263916,
      "activations/layer1_attention_weight_max": 16.216758728027344,
      "activations/layer1_attention_weight_min": -14.90420913696289,
      "activations/layer20_attention_weight_max": 27.629350662231445,
      "activations/layer20_attention_weight_min": -25.505985260009766,
      "activations/layer21_attention_weight_max": 27.593948364257812,
      "activations/layer21_attention_weight_min": -25.04744529724121,
      "activations/layer22_attention_weight_max": 42.7236328125,
      "activations/layer22_attention_weight_min": -29.434480667114258,
      "activations/layer23_attention_weight_max": 33.246280670166016,
      "activations/layer23_attention_weight_min": -25.11016082763672,
      "activations/layer2_attention_weight_max": 29.59328842163086,
      "activations/layer2_attention_weight_min": -29.180561065673828,
      "activations/layer3_attention_weight_max": 88.39176177978516,
      "activations/layer3_attention_weight_min": -89.73506927490234,
      "activations/layer4_attention_weight_max": 107.79967498779297,
      "activations/layer4_attention_weight_min": -109.47625732421875,
      "activations/layer5_attention_weight_max": 51.53437805175781,
      "activations/layer5_attention_weight_min": -63.88790512084961,
      "activations/layer6_attention_weight_max": 45.435855865478516,
      "activations/layer6_attention_weight_min": -48.52006530761719,
      "activations/layer7_attention_weight_max": 87.6781005859375,
      "activations/layer7_attention_weight_min": -87.62886810302734,
      "activations/layer8_attention_weight_max": 37.18312072753906,
      "activations/layer8_attention_weight_min": -39.51383590698242,
      "activations/layer9_attention_weight_max": 31.634490966796875,
      "activations/layer9_attention_weight_min": -32.760009765625,
      "epoch": 12.47,
      "learning_rate": 7.031477272727272e-05,
      "loss": 2.7681,
      "step": 214550
    },
    {
      "activations/layer0_attention_weight_max": 15.512393951416016,
      "activations/layer0_attention_weight_min": -11.873146057128906,
      "activations/layer10_attention_weight_max": 32.94293975830078,
      "activations/layer10_attention_weight_min": -31.431779861450195,
      "activations/layer11_attention_weight_max": 32.63547134399414,
      "activations/layer11_attention_weight_min": -30.77986717224121,
      "activations/layer12_attention_weight_max": 25.602134704589844,
      "activations/layer12_attention_weight_min": -25.307472229003906,
      "activations/layer13_attention_weight_max": 38.78448486328125,
      "activations/layer13_attention_weight_min": -33.71037292480469,
      "activations/layer14_attention_weight_max": 44.21230697631836,
      "activations/layer14_attention_weight_min": -39.02248001098633,
      "activations/layer15_attention_weight_max": 36.888423919677734,
      "activations/layer15_attention_weight_min": -33.044395446777344,
      "activations/layer16_attention_weight_max": 29.228689193725586,
      "activations/layer16_attention_weight_min": -27.67392921447754,
      "activations/layer17_attention_weight_max": 30.158905029296875,
      "activations/layer17_attention_weight_min": -25.655506134033203,
      "activations/layer18_attention_weight_max": 31.703231811523438,
      "activations/layer18_attention_weight_min": -24.566741943359375,
      "activations/layer19_attention_weight_max": 33.82173538208008,
      "activations/layer19_attention_weight_min": -30.746347427368164,
      "activations/layer1_attention_weight_max": 17.07794189453125,
      "activations/layer1_attention_weight_min": -14.503381729125977,
      "activations/layer20_attention_weight_max": 30.395029067993164,
      "activations/layer20_attention_weight_min": -25.549339294433594,
      "activations/layer21_attention_weight_max": 29.05763816833496,
      "activations/layer21_attention_weight_min": -25.031009674072266,
      "activations/layer22_attention_weight_max": 39.8760871887207,
      "activations/layer22_attention_weight_min": -31.944843292236328,
      "activations/layer23_attention_weight_max": 33.725250244140625,
      "activations/layer23_attention_weight_min": -23.848499298095703,
      "activations/layer2_attention_weight_max": 32.238609313964844,
      "activations/layer2_attention_weight_min": -30.563796997070312,
      "activations/layer3_attention_weight_max": 93.32559967041016,
      "activations/layer3_attention_weight_min": -93.07037353515625,
      "activations/layer4_attention_weight_max": 112.21703338623047,
      "activations/layer4_attention_weight_min": -112.56610107421875,
      "activations/layer5_attention_weight_max": 56.77519989013672,
      "activations/layer5_attention_weight_min": -62.208248138427734,
      "activations/layer6_attention_weight_max": 45.50278091430664,
      "activations/layer6_attention_weight_min": -49.66926956176758,
      "activations/layer7_attention_weight_max": 95.64795684814453,
      "activations/layer7_attention_weight_min": -88.22380828857422,
      "activations/layer8_attention_weight_max": 40.18593978881836,
      "activations/layer8_attention_weight_min": -39.85322189331055,
      "activations/layer9_attention_weight_max": 35.312503814697266,
      "activations/layer9_attention_weight_min": -31.98188591003418,
      "epoch": 12.47,
      "learning_rate": 7.029583333333333e-05,
      "loss": 2.771,
      "step": 214600
    },
    {
      "activations/layer0_attention_weight_max": 16.315200805664062,
      "activations/layer0_attention_weight_min": -11.767568588256836,
      "activations/layer10_attention_weight_max": 28.99151611328125,
      "activations/layer10_attention_weight_min": -30.98771095275879,
      "activations/layer11_attention_weight_max": 30.942611694335938,
      "activations/layer11_attention_weight_min": -31.824935913085938,
      "activations/layer12_attention_weight_max": 23.694969177246094,
      "activations/layer12_attention_weight_min": -25.139856338500977,
      "activations/layer13_attention_weight_max": 36.58346176147461,
      "activations/layer13_attention_weight_min": -35.55268096923828,
      "activations/layer14_attention_weight_max": 39.47518539428711,
      "activations/layer14_attention_weight_min": -36.3740234375,
      "activations/layer15_attention_weight_max": 35.30875778198242,
      "activations/layer15_attention_weight_min": -34.06570816040039,
      "activations/layer16_attention_weight_max": 29.736114501953125,
      "activations/layer16_attention_weight_min": -27.613656997680664,
      "activations/layer17_attention_weight_max": 32.06501770019531,
      "activations/layer17_attention_weight_min": -27.333786010742188,
      "activations/layer18_attention_weight_max": 32.00109100341797,
      "activations/layer18_attention_weight_min": -25.33094024658203,
      "activations/layer19_attention_weight_max": 37.35348129272461,
      "activations/layer19_attention_weight_min": -31.655323028564453,
      "activations/layer1_attention_weight_max": 15.508562088012695,
      "activations/layer1_attention_weight_min": -16.076658248901367,
      "activations/layer20_attention_weight_max": 32.892356872558594,
      "activations/layer20_attention_weight_min": -26.058256149291992,
      "activations/layer21_attention_weight_max": 32.12005615234375,
      "activations/layer21_attention_weight_min": -24.93039894104004,
      "activations/layer22_attention_weight_max": 46.38752746582031,
      "activations/layer22_attention_weight_min": -32.316139221191406,
      "activations/layer23_attention_weight_max": 39.049461364746094,
      "activations/layer23_attention_weight_min": -25.071937561035156,
      "activations/layer2_attention_weight_max": 32.160438537597656,
      "activations/layer2_attention_weight_min": -30.4024715423584,
      "activations/layer3_attention_weight_max": 86.98182678222656,
      "activations/layer3_attention_weight_min": -90.30452728271484,
      "activations/layer4_attention_weight_max": 105.82503509521484,
      "activations/layer4_attention_weight_min": -109.94548797607422,
      "activations/layer5_attention_weight_max": 48.72848129272461,
      "activations/layer5_attention_weight_min": -61.28530502319336,
      "activations/layer6_attention_weight_max": 41.03060531616211,
      "activations/layer6_attention_weight_min": -46.56653594970703,
      "activations/layer7_attention_weight_max": 83.52289581298828,
      "activations/layer7_attention_weight_min": -83.65321350097656,
      "activations/layer8_attention_weight_max": 37.4891242980957,
      "activations/layer8_attention_weight_min": -38.502010345458984,
      "activations/layer9_attention_weight_max": 29.119672775268555,
      "activations/layer9_attention_weight_min": -30.609647750854492,
      "epoch": 12.47,
      "learning_rate": 7.027689393939393e-05,
      "loss": 2.7649,
      "step": 214650
    },
    {
      "activations/layer0_attention_weight_max": 15.763470649719238,
      "activations/layer0_attention_weight_min": -11.816643714904785,
      "activations/layer10_attention_weight_max": 32.33861541748047,
      "activations/layer10_attention_weight_min": -33.70155715942383,
      "activations/layer11_attention_weight_max": 33.7765007019043,
      "activations/layer11_attention_weight_min": -32.653621673583984,
      "activations/layer12_attention_weight_max": 25.93811798095703,
      "activations/layer12_attention_weight_min": -25.609006881713867,
      "activations/layer13_attention_weight_max": 44.0329704284668,
      "activations/layer13_attention_weight_min": -34.545310974121094,
      "activations/layer14_attention_weight_max": 45.3143310546875,
      "activations/layer14_attention_weight_min": -38.4959831237793,
      "activations/layer15_attention_weight_max": 37.97638702392578,
      "activations/layer15_attention_weight_min": -33.952816009521484,
      "activations/layer16_attention_weight_max": 33.766536712646484,
      "activations/layer16_attention_weight_min": -28.25439453125,
      "activations/layer17_attention_weight_max": 33.975765228271484,
      "activations/layer17_attention_weight_min": -25.220375061035156,
      "activations/layer18_attention_weight_max": 35.28404998779297,
      "activations/layer18_attention_weight_min": -24.535675048828125,
      "activations/layer19_attention_weight_max": 38.11698913574219,
      "activations/layer19_attention_weight_min": -33.89924240112305,
      "activations/layer1_attention_weight_max": 15.903059005737305,
      "activations/layer1_attention_weight_min": -14.8082275390625,
      "activations/layer20_attention_weight_max": 34.90620422363281,
      "activations/layer20_attention_weight_min": -25.77687644958496,
      "activations/layer21_attention_weight_max": 32.30610275268555,
      "activations/layer21_attention_weight_min": -23.423315048217773,
      "activations/layer22_attention_weight_max": 54.459049224853516,
      "activations/layer22_attention_weight_min": -31.589921951293945,
      "activations/layer23_attention_weight_max": 39.597816467285156,
      "activations/layer23_attention_weight_min": -24.461261749267578,
      "activations/layer2_attention_weight_max": 30.567365646362305,
      "activations/layer2_attention_weight_min": -29.569997787475586,
      "activations/layer3_attention_weight_max": 91.1563720703125,
      "activations/layer3_attention_weight_min": -89.53638458251953,
      "activations/layer4_attention_weight_max": 114.64686584472656,
      "activations/layer4_attention_weight_min": -107.44927978515625,
      "activations/layer5_attention_weight_max": 53.407405853271484,
      "activations/layer5_attention_weight_min": -58.879737854003906,
      "activations/layer6_attention_weight_max": 44.69672393798828,
      "activations/layer6_attention_weight_min": -47.7542839050293,
      "activations/layer7_attention_weight_max": 94.89631652832031,
      "activations/layer7_attention_weight_min": -96.5272216796875,
      "activations/layer8_attention_weight_max": 41.37693786621094,
      "activations/layer8_attention_weight_min": -41.78998947143555,
      "activations/layer9_attention_weight_max": 34.348690032958984,
      "activations/layer9_attention_weight_min": -34.79096984863281,
      "epoch": 12.48,
      "learning_rate": 7.025795454545455e-05,
      "loss": 2.7881,
      "step": 214700
    },
    {
      "activations/layer0_attention_weight_max": 15.15585994720459,
      "activations/layer0_attention_weight_min": -12.236385345458984,
      "activations/layer10_attention_weight_max": 37.9412841796875,
      "activations/layer10_attention_weight_min": -33.89955520629883,
      "activations/layer11_attention_weight_max": 36.53289031982422,
      "activations/layer11_attention_weight_min": -34.722198486328125,
      "activations/layer12_attention_weight_max": 38.629364013671875,
      "activations/layer12_attention_weight_min": -25.191513061523438,
      "activations/layer13_attention_weight_max": 48.99825668334961,
      "activations/layer13_attention_weight_min": -34.58992004394531,
      "activations/layer14_attention_weight_max": 44.81925964355469,
      "activations/layer14_attention_weight_min": -38.72440719604492,
      "activations/layer15_attention_weight_max": 39.061187744140625,
      "activations/layer15_attention_weight_min": -35.031280517578125,
      "activations/layer16_attention_weight_max": 29.959224700927734,
      "activations/layer16_attention_weight_min": -26.7712345123291,
      "activations/layer17_attention_weight_max": 33.64778518676758,
      "activations/layer17_attention_weight_min": -26.218223571777344,
      "activations/layer18_attention_weight_max": 30.05829429626465,
      "activations/layer18_attention_weight_min": -24.139881134033203,
      "activations/layer19_attention_weight_max": 34.001033782958984,
      "activations/layer19_attention_weight_min": -31.894474029541016,
      "activations/layer1_attention_weight_max": 17.443540573120117,
      "activations/layer1_attention_weight_min": -14.05229663848877,
      "activations/layer20_attention_weight_max": 28.01936912536621,
      "activations/layer20_attention_weight_min": -26.31207275390625,
      "activations/layer21_attention_weight_max": 29.13958740234375,
      "activations/layer21_attention_weight_min": -24.873666763305664,
      "activations/layer22_attention_weight_max": 43.20524215698242,
      "activations/layer22_attention_weight_min": -31.76220703125,
      "activations/layer23_attention_weight_max": 32.88398742675781,
      "activations/layer23_attention_weight_min": -24.359960556030273,
      "activations/layer2_attention_weight_max": 31.259674072265625,
      "activations/layer2_attention_weight_min": -31.431135177612305,
      "activations/layer3_attention_weight_max": 85.75514221191406,
      "activations/layer3_attention_weight_min": -91.10731506347656,
      "activations/layer4_attention_weight_max": 103.54815673828125,
      "activations/layer4_attention_weight_min": -109.0086441040039,
      "activations/layer5_attention_weight_max": 48.84824752807617,
      "activations/layer5_attention_weight_min": -59.220970153808594,
      "activations/layer6_attention_weight_max": 42.24280548095703,
      "activations/layer6_attention_weight_min": -47.52082443237305,
      "activations/layer7_attention_weight_max": 93.35388946533203,
      "activations/layer7_attention_weight_min": -88.63848114013672,
      "activations/layer8_attention_weight_max": 44.78718948364258,
      "activations/layer8_attention_weight_min": -40.79530334472656,
      "activations/layer9_attention_weight_max": 36.182613372802734,
      "activations/layer9_attention_weight_min": -34.77439498901367,
      "epoch": 12.48,
      "learning_rate": 7.023901515151515e-05,
      "loss": 2.7796,
      "step": 214750
    },
    {
      "activations/layer0_attention_weight_max": 15.618231773376465,
      "activations/layer0_attention_weight_min": -12.5094633102417,
      "activations/layer10_attention_weight_max": 31.467824935913086,
      "activations/layer10_attention_weight_min": -32.64187240600586,
      "activations/layer11_attention_weight_max": 32.09099578857422,
      "activations/layer11_attention_weight_min": -31.58877944946289,
      "activations/layer12_attention_weight_max": 24.17902374267578,
      "activations/layer12_attention_weight_min": -24.59111785888672,
      "activations/layer13_attention_weight_max": 37.15120315551758,
      "activations/layer13_attention_weight_min": -34.65070724487305,
      "activations/layer14_attention_weight_max": 47.45197296142578,
      "activations/layer14_attention_weight_min": -39.7672233581543,
      "activations/layer15_attention_weight_max": 36.10456466674805,
      "activations/layer15_attention_weight_min": -32.02482986450195,
      "activations/layer16_attention_weight_max": 31.75850486755371,
      "activations/layer16_attention_weight_min": -27.324024200439453,
      "activations/layer17_attention_weight_max": 32.71950912475586,
      "activations/layer17_attention_weight_min": -26.504724502563477,
      "activations/layer18_attention_weight_max": 33.8277587890625,
      "activations/layer18_attention_weight_min": -26.003183364868164,
      "activations/layer19_attention_weight_max": 35.718624114990234,
      "activations/layer19_attention_weight_min": -31.256942749023438,
      "activations/layer1_attention_weight_max": 17.53680419921875,
      "activations/layer1_attention_weight_min": -15.010026931762695,
      "activations/layer20_attention_weight_max": 30.297683715820312,
      "activations/layer20_attention_weight_min": -27.5155086517334,
      "activations/layer21_attention_weight_max": 28.830686569213867,
      "activations/layer21_attention_weight_min": -26.357358932495117,
      "activations/layer22_attention_weight_max": 40.807498931884766,
      "activations/layer22_attention_weight_min": -33.53004455566406,
      "activations/layer23_attention_weight_max": 32.10588073730469,
      "activations/layer23_attention_weight_min": -28.397586822509766,
      "activations/layer2_attention_weight_max": 28.921255111694336,
      "activations/layer2_attention_weight_min": -30.78143882751465,
      "activations/layer3_attention_weight_max": 87.6792221069336,
      "activations/layer3_attention_weight_min": -88.69943237304688,
      "activations/layer4_attention_weight_max": 109.31941986083984,
      "activations/layer4_attention_weight_min": -109.13386535644531,
      "activations/layer5_attention_weight_max": 49.40083312988281,
      "activations/layer5_attention_weight_min": -64.08512115478516,
      "activations/layer6_attention_weight_max": 45.44765090942383,
      "activations/layer6_attention_weight_min": -45.970176696777344,
      "activations/layer7_attention_weight_max": 90.4388198852539,
      "activations/layer7_attention_weight_min": -88.94903564453125,
      "activations/layer8_attention_weight_max": 40.416160583496094,
      "activations/layer8_attention_weight_min": -40.92860412597656,
      "activations/layer9_attention_weight_max": 30.940176010131836,
      "activations/layer9_attention_weight_min": -32.851436614990234,
      "epoch": 12.48,
      "learning_rate": 7.022007575757575e-05,
      "loss": 2.7823,
      "step": 214800
    },
    {
      "activations/layer0_attention_weight_max": 16.315689086914062,
      "activations/layer0_attention_weight_min": -12.818002700805664,
      "activations/layer10_attention_weight_max": 31.302352905273438,
      "activations/layer10_attention_weight_min": -29.628753662109375,
      "activations/layer11_attention_weight_max": 31.521512985229492,
      "activations/layer11_attention_weight_min": -30.192485809326172,
      "activations/layer12_attention_weight_max": 22.888671875,
      "activations/layer12_attention_weight_min": -24.562580108642578,
      "activations/layer13_attention_weight_max": 34.14177703857422,
      "activations/layer13_attention_weight_min": -32.80192565917969,
      "activations/layer14_attention_weight_max": 35.810115814208984,
      "activations/layer14_attention_weight_min": -36.89326095581055,
      "activations/layer15_attention_weight_max": 33.26796340942383,
      "activations/layer15_attention_weight_min": -30.220325469970703,
      "activations/layer16_attention_weight_max": 28.11052703857422,
      "activations/layer16_attention_weight_min": -27.293787002563477,
      "activations/layer17_attention_weight_max": 31.10698890686035,
      "activations/layer17_attention_weight_min": -25.466657638549805,
      "activations/layer18_attention_weight_max": 29.771408081054688,
      "activations/layer18_attention_weight_min": -24.036434173583984,
      "activations/layer19_attention_weight_max": 31.123844146728516,
      "activations/layer19_attention_weight_min": -31.45550537109375,
      "activations/layer1_attention_weight_max": 16.185033798217773,
      "activations/layer1_attention_weight_min": -15.105877876281738,
      "activations/layer20_attention_weight_max": 31.647092819213867,
      "activations/layer20_attention_weight_min": -26.712730407714844,
      "activations/layer21_attention_weight_max": 25.846893310546875,
      "activations/layer21_attention_weight_min": -25.65654182434082,
      "activations/layer22_attention_weight_max": 39.047603607177734,
      "activations/layer22_attention_weight_min": -32.20273208618164,
      "activations/layer23_attention_weight_max": 33.82550811767578,
      "activations/layer23_attention_weight_min": -25.602951049804688,
      "activations/layer2_attention_weight_max": 30.669105529785156,
      "activations/layer2_attention_weight_min": -29.43169403076172,
      "activations/layer3_attention_weight_max": 89.48623657226562,
      "activations/layer3_attention_weight_min": -92.89857482910156,
      "activations/layer4_attention_weight_max": 106.99764251708984,
      "activations/layer4_attention_weight_min": -105.4011459350586,
      "activations/layer5_attention_weight_max": 50.819488525390625,
      "activations/layer5_attention_weight_min": -61.361595153808594,
      "activations/layer6_attention_weight_max": 43.62662124633789,
      "activations/layer6_attention_weight_min": -46.19593811035156,
      "activations/layer7_attention_weight_max": 83.97967529296875,
      "activations/layer7_attention_weight_min": -92.11775207519531,
      "activations/layer8_attention_weight_max": 39.62533187866211,
      "activations/layer8_attention_weight_min": -40.93243408203125,
      "activations/layer9_attention_weight_max": 31.967254638671875,
      "activations/layer9_attention_weight_min": -31.497634887695312,
      "epoch": 12.48,
      "learning_rate": 7.020113636363635e-05,
      "loss": 2.7572,
      "step": 214850
    },
    {
      "activations/layer0_attention_weight_max": 14.947388648986816,
      "activations/layer0_attention_weight_min": -12.760453224182129,
      "activations/layer10_attention_weight_max": 31.928325653076172,
      "activations/layer10_attention_weight_min": -31.56626319885254,
      "activations/layer11_attention_weight_max": 34.18062210083008,
      "activations/layer11_attention_weight_min": -31.90355682373047,
      "activations/layer12_attention_weight_max": 25.62236213684082,
      "activations/layer12_attention_weight_min": -25.00943946838379,
      "activations/layer13_attention_weight_max": 36.80012130737305,
      "activations/layer13_attention_weight_min": -33.672157287597656,
      "activations/layer14_attention_weight_max": 44.81401824951172,
      "activations/layer14_attention_weight_min": -36.01933670043945,
      "activations/layer15_attention_weight_max": 38.50722122192383,
      "activations/layer15_attention_weight_min": -32.33530807495117,
      "activations/layer16_attention_weight_max": 33.747371673583984,
      "activations/layer16_attention_weight_min": -28.390827178955078,
      "activations/layer17_attention_weight_max": 31.7973575592041,
      "activations/layer17_attention_weight_min": -27.31571388244629,
      "activations/layer18_attention_weight_max": 35.400962829589844,
      "activations/layer18_attention_weight_min": -24.3741397857666,
      "activations/layer19_attention_weight_max": 34.550045013427734,
      "activations/layer19_attention_weight_min": -31.662195205688477,
      "activations/layer1_attention_weight_max": 16.44941520690918,
      "activations/layer1_attention_weight_min": -13.795324325561523,
      "activations/layer20_attention_weight_max": 29.34723472595215,
      "activations/layer20_attention_weight_min": -25.96380043029785,
      "activations/layer21_attention_weight_max": 29.364818572998047,
      "activations/layer21_attention_weight_min": -25.30021095275879,
      "activations/layer22_attention_weight_max": 50.61811828613281,
      "activations/layer22_attention_weight_min": -30.916446685791016,
      "activations/layer23_attention_weight_max": 36.243797302246094,
      "activations/layer23_attention_weight_min": -24.076688766479492,
      "activations/layer2_attention_weight_max": 31.899534225463867,
      "activations/layer2_attention_weight_min": -31.49285888671875,
      "activations/layer3_attention_weight_max": 90.56333923339844,
      "activations/layer3_attention_weight_min": -92.67668914794922,
      "activations/layer4_attention_weight_max": 108.350341796875,
      "activations/layer4_attention_weight_min": -109.72115325927734,
      "activations/layer5_attention_weight_max": 50.74512481689453,
      "activations/layer5_attention_weight_min": -63.20127868652344,
      "activations/layer6_attention_weight_max": 44.80215835571289,
      "activations/layer6_attention_weight_min": -47.07265853881836,
      "activations/layer7_attention_weight_max": 87.50940704345703,
      "activations/layer7_attention_weight_min": -84.71771240234375,
      "activations/layer8_attention_weight_max": 39.34018325805664,
      "activations/layer8_attention_weight_min": -39.19290542602539,
      "activations/layer9_attention_weight_max": 32.06715774536133,
      "activations/layer9_attention_weight_min": -32.753719329833984,
      "epoch": 12.49,
      "learning_rate": 7.018219696969697e-05,
      "loss": 2.7838,
      "step": 214900
    },
    {
      "activations/layer0_attention_weight_max": 15.275251388549805,
      "activations/layer0_attention_weight_min": -11.684038162231445,
      "activations/layer10_attention_weight_max": 30.618267059326172,
      "activations/layer10_attention_weight_min": -31.503982543945312,
      "activations/layer11_attention_weight_max": 30.451831817626953,
      "activations/layer11_attention_weight_min": -30.436370849609375,
      "activations/layer12_attention_weight_max": 24.770347595214844,
      "activations/layer12_attention_weight_min": -26.014957427978516,
      "activations/layer13_attention_weight_max": 36.92958450317383,
      "activations/layer13_attention_weight_min": -35.67270278930664,
      "activations/layer14_attention_weight_max": 41.7821044921875,
      "activations/layer14_attention_weight_min": -40.713706970214844,
      "activations/layer15_attention_weight_max": 32.981868743896484,
      "activations/layer15_attention_weight_min": -32.78731155395508,
      "activations/layer16_attention_weight_max": 28.667261123657227,
      "activations/layer16_attention_weight_min": -27.54806137084961,
      "activations/layer17_attention_weight_max": 30.90587043762207,
      "activations/layer17_attention_weight_min": -25.0548038482666,
      "activations/layer18_attention_weight_max": 31.103336334228516,
      "activations/layer18_attention_weight_min": -23.147253036499023,
      "activations/layer19_attention_weight_max": 31.046802520751953,
      "activations/layer19_attention_weight_min": -30.42707633972168,
      "activations/layer1_attention_weight_max": 16.080951690673828,
      "activations/layer1_attention_weight_min": -14.815193176269531,
      "activations/layer20_attention_weight_max": 28.333993911743164,
      "activations/layer20_attention_weight_min": -24.6964054107666,
      "activations/layer21_attention_weight_max": 26.2847957611084,
      "activations/layer21_attention_weight_min": -24.32621955871582,
      "activations/layer22_attention_weight_max": 38.442405700683594,
      "activations/layer22_attention_weight_min": -28.976215362548828,
      "activations/layer23_attention_weight_max": 32.290489196777344,
      "activations/layer23_attention_weight_min": -24.47786521911621,
      "activations/layer2_attention_weight_max": 31.16132926940918,
      "activations/layer2_attention_weight_min": -31.142011642456055,
      "activations/layer3_attention_weight_max": 86.90125274658203,
      "activations/layer3_attention_weight_min": -88.29402160644531,
      "activations/layer4_attention_weight_max": 107.22566986083984,
      "activations/layer4_attention_weight_min": -105.01555633544922,
      "activations/layer5_attention_weight_max": 48.3984489440918,
      "activations/layer5_attention_weight_min": -58.41755676269531,
      "activations/layer6_attention_weight_max": 44.8415641784668,
      "activations/layer6_attention_weight_min": -48.48770523071289,
      "activations/layer7_attention_weight_max": 90.28729248046875,
      "activations/layer7_attention_weight_min": -86.89401245117188,
      "activations/layer8_attention_weight_max": 38.4870491027832,
      "activations/layer8_attention_weight_min": -40.426815032958984,
      "activations/layer9_attention_weight_max": 30.59995460510254,
      "activations/layer9_attention_weight_min": -32.32245635986328,
      "epoch": 12.49,
      "learning_rate": 7.016325757575757e-05,
      "loss": 2.77,
      "step": 214950
    },
    {
      "activations/layer0_attention_weight_max": 15.008683204650879,
      "activations/layer0_attention_weight_min": -11.70212173461914,
      "activations/layer10_attention_weight_max": 31.230472564697266,
      "activations/layer10_attention_weight_min": -32.6596794128418,
      "activations/layer11_attention_weight_max": 29.76581382751465,
      "activations/layer11_attention_weight_min": -31.193340301513672,
      "activations/layer12_attention_weight_max": 24.15239715576172,
      "activations/layer12_attention_weight_min": -24.649723052978516,
      "activations/layer13_attention_weight_max": 36.06412887573242,
      "activations/layer13_attention_weight_min": -33.28568649291992,
      "activations/layer14_attention_weight_max": 41.549842834472656,
      "activations/layer14_attention_weight_min": -39.02899932861328,
      "activations/layer15_attention_weight_max": 38.01647186279297,
      "activations/layer15_attention_weight_min": -33.234683990478516,
      "activations/layer16_attention_weight_max": 29.485910415649414,
      "activations/layer16_attention_weight_min": -28.046972274780273,
      "activations/layer17_attention_weight_max": 30.5230655670166,
      "activations/layer17_attention_weight_min": -25.332006454467773,
      "activations/layer18_attention_weight_max": 32.91962814331055,
      "activations/layer18_attention_weight_min": -24.640111923217773,
      "activations/layer19_attention_weight_max": 34.81833267211914,
      "activations/layer19_attention_weight_min": -31.01486587524414,
      "activations/layer1_attention_weight_max": 16.000085830688477,
      "activations/layer1_attention_weight_min": -13.993597030639648,
      "activations/layer20_attention_weight_max": 34.89018630981445,
      "activations/layer20_attention_weight_min": -24.684268951416016,
      "activations/layer21_attention_weight_max": 31.91483497619629,
      "activations/layer21_attention_weight_min": -24.60528564453125,
      "activations/layer22_attention_weight_max": 43.2512321472168,
      "activations/layer22_attention_weight_min": -30.709590911865234,
      "activations/layer23_attention_weight_max": 34.875789642333984,
      "activations/layer23_attention_weight_min": -26.41395378112793,
      "activations/layer2_attention_weight_max": 32.20014190673828,
      "activations/layer2_attention_weight_min": -32.70532989501953,
      "activations/layer3_attention_weight_max": 88.43197631835938,
      "activations/layer3_attention_weight_min": -91.2212142944336,
      "activations/layer4_attention_weight_max": 111.46129608154297,
      "activations/layer4_attention_weight_min": -109.24615478515625,
      "activations/layer5_attention_weight_max": 50.25565719604492,
      "activations/layer5_attention_weight_min": -61.761993408203125,
      "activations/layer6_attention_weight_max": 45.82419204711914,
      "activations/layer6_attention_weight_min": -47.618099212646484,
      "activations/layer7_attention_weight_max": 93.46041107177734,
      "activations/layer7_attention_weight_min": -91.48860931396484,
      "activations/layer8_attention_weight_max": 38.34919357299805,
      "activations/layer8_attention_weight_min": -40.19452667236328,
      "activations/layer9_attention_weight_max": 31.519454956054688,
      "activations/layer9_attention_weight_min": -33.03150939941406,
      "epoch": 12.49,
      "learning_rate": 7.014431818181817e-05,
      "loss": 2.7841,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.429,
      "eval_samples_per_second": 509.434,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.429,
      "eval_openwebtext_samples_per_second": 509.434,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9383,
      "eval_wikitext_samples_per_second": 235.252,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_lambada_loss": 2.484375,
      "eval_lambada_ppl": 11.993621898152476,
      "eval_lambada_runtime": 9.5218,
      "eval_lambada_samples_per_second": 511.353,
      "step": 215000
    },
    {
      "activations/layer0_attention_weight_max": 15.495723724365234,
      "activations/layer0_attention_weight_min": -12.18917465209961,
      "activations/layer10_attention_weight_max": 50.29631423950195,
      "activations/layer10_attention_weight_min": -46.942413330078125,
      "activations/layer11_attention_weight_max": 54.012245178222656,
      "activations/layer11_attention_weight_min": -52.160362243652344,
      "activations/layer12_attention_weight_max": 55.58281707763672,
      "activations/layer12_attention_weight_min": -37.89583206176758,
      "activations/layer13_attention_weight_max": 79.21829986572266,
      "activations/layer13_attention_weight_min": -53.08854675292969,
      "activations/layer14_attention_weight_max": 78.22180938720703,
      "activations/layer14_attention_weight_min": -63.583290100097656,
      "activations/layer15_attention_weight_max": 72.28605651855469,
      "activations/layer15_attention_weight_min": -57.585914611816406,
      "activations/layer16_attention_weight_max": 50.648311614990234,
      "activations/layer16_attention_weight_min": -35.3250846862793,
      "activations/layer17_attention_weight_max": 40.418113708496094,
      "activations/layer17_attention_weight_min": -36.19816970825195,
      "activations/layer18_attention_weight_max": 35.767086029052734,
      "activations/layer18_attention_weight_min": -29.68868637084961,
      "activations/layer19_attention_weight_max": 42.424530029296875,
      "activations/layer19_attention_weight_min": -34.90005111694336,
      "activations/layer1_attention_weight_max": 18.83163070678711,
      "activations/layer1_attention_weight_min": -14.105958938598633,
      "activations/layer20_attention_weight_max": 37.29355239868164,
      "activations/layer20_attention_weight_min": -25.80508041381836,
      "activations/layer21_attention_weight_max": 34.512359619140625,
      "activations/layer21_attention_weight_min": -27.853342056274414,
      "activations/layer22_attention_weight_max": 54.05702209472656,
      "activations/layer22_attention_weight_min": -38.79952621459961,
      "activations/layer23_attention_weight_max": 39.97827911376953,
      "activations/layer23_attention_weight_min": -26.862403869628906,
      "activations/layer2_attention_weight_max": 30.75986099243164,
      "activations/layer2_attention_weight_min": -31.290306091308594,
      "activations/layer3_attention_weight_max": 91.32042694091797,
      "activations/layer3_attention_weight_min": -91.41310119628906,
      "activations/layer4_attention_weight_max": 113.79528045654297,
      "activations/layer4_attention_weight_min": -113.36415100097656,
      "activations/layer5_attention_weight_max": 53.89825439453125,
      "activations/layer5_attention_weight_min": -65.70346069335938,
      "activations/layer6_attention_weight_max": 47.5289421081543,
      "activations/layer6_attention_weight_min": -48.48629379272461,
      "activations/layer7_attention_weight_max": 111.0442886352539,
      "activations/layer7_attention_weight_min": -107.16922760009766,
      "activations/layer8_attention_weight_max": 52.7364387512207,
      "activations/layer8_attention_weight_min": -52.71617889404297,
      "activations/layer9_attention_weight_max": 48.502098083496094,
      "activations/layer9_attention_weight_min": -45.63480758666992,
      "epoch": 12.5,
      "learning_rate": 7.012537878787877e-05,
      "loss": 2.7649,
      "step": 215050
    },
    {
      "activations/layer0_attention_weight_max": 15.272592544555664,
      "activations/layer0_attention_weight_min": -11.685064315795898,
      "activations/layer10_attention_weight_max": 32.66183090209961,
      "activations/layer10_attention_weight_min": -31.385290145874023,
      "activations/layer11_attention_weight_max": 33.2406120300293,
      "activations/layer11_attention_weight_min": -31.480815887451172,
      "activations/layer12_attention_weight_max": 28.24188804626465,
      "activations/layer12_attention_weight_min": -24.37459945678711,
      "activations/layer13_attention_weight_max": 42.36488723754883,
      "activations/layer13_attention_weight_min": -33.66085433959961,
      "activations/layer14_attention_weight_max": 46.69419860839844,
      "activations/layer14_attention_weight_min": -36.31945037841797,
      "activations/layer15_attention_weight_max": 41.48821258544922,
      "activations/layer15_attention_weight_min": -33.90386199951172,
      "activations/layer16_attention_weight_max": 32.911155700683594,
      "activations/layer16_attention_weight_min": -27.874282836914062,
      "activations/layer17_attention_weight_max": 32.524330139160156,
      "activations/layer17_attention_weight_min": -24.71088981628418,
      "activations/layer18_attention_weight_max": 33.71526336669922,
      "activations/layer18_attention_weight_min": -22.673696517944336,
      "activations/layer19_attention_weight_max": 42.09210205078125,
      "activations/layer19_attention_weight_min": -30.654754638671875,
      "activations/layer1_attention_weight_max": 17.0651912689209,
      "activations/layer1_attention_weight_min": -16.527563095092773,
      "activations/layer20_attention_weight_max": 32.628475189208984,
      "activations/layer20_attention_weight_min": -24.44212532043457,
      "activations/layer21_attention_weight_max": 29.903554916381836,
      "activations/layer21_attention_weight_min": -23.032567977905273,
      "activations/layer22_attention_weight_max": 50.67708969116211,
      "activations/layer22_attention_weight_min": -30.877845764160156,
      "activations/layer23_attention_weight_max": 41.71481704711914,
      "activations/layer23_attention_weight_min": -27.70940399169922,
      "activations/layer2_attention_weight_max": 30.463804244995117,
      "activations/layer2_attention_weight_min": -30.74881362915039,
      "activations/layer3_attention_weight_max": 90.48086547851562,
      "activations/layer3_attention_weight_min": -90.65861511230469,
      "activations/layer4_attention_weight_max": 107.61565399169922,
      "activations/layer4_attention_weight_min": -109.99556732177734,
      "activations/layer5_attention_weight_max": 51.026145935058594,
      "activations/layer5_attention_weight_min": -59.663536071777344,
      "activations/layer6_attention_weight_max": 42.738460540771484,
      "activations/layer6_attention_weight_min": -48.45491409301758,
      "activations/layer7_attention_weight_max": 88.66604614257812,
      "activations/layer7_attention_weight_min": -88.6200942993164,
      "activations/layer8_attention_weight_max": 42.00816345214844,
      "activations/layer8_attention_weight_min": -41.9846305847168,
      "activations/layer9_attention_weight_max": 32.251888275146484,
      "activations/layer9_attention_weight_min": -30.985431671142578,
      "epoch": 12.5,
      "learning_rate": 7.010643939393939e-05,
      "loss": 2.7748,
      "step": 215100
    },
    {
      "activations/layer0_attention_weight_max": 15.643085479736328,
      "activations/layer0_attention_weight_min": -11.828256607055664,
      "activations/layer10_attention_weight_max": 32.64856719970703,
      "activations/layer10_attention_weight_min": -33.0723876953125,
      "activations/layer11_attention_weight_max": 32.77873992919922,
      "activations/layer11_attention_weight_min": -33.47177505493164,
      "activations/layer12_attention_weight_max": 26.203079223632812,
      "activations/layer12_attention_weight_min": -27.987506866455078,
      "activations/layer13_attention_weight_max": 42.293601989746094,
      "activations/layer13_attention_weight_min": -34.459991455078125,
      "activations/layer14_attention_weight_max": 46.59525680541992,
      "activations/layer14_attention_weight_min": -38.284767150878906,
      "activations/layer15_attention_weight_max": 37.70378875732422,
      "activations/layer15_attention_weight_min": -32.33317184448242,
      "activations/layer16_attention_weight_max": 30.08551788330078,
      "activations/layer16_attention_weight_min": -27.053325653076172,
      "activations/layer17_attention_weight_max": 30.96045684814453,
      "activations/layer17_attention_weight_min": -24.904199600219727,
      "activations/layer18_attention_weight_max": 31.577909469604492,
      "activations/layer18_attention_weight_min": -24.14992904663086,
      "activations/layer19_attention_weight_max": 36.5803108215332,
      "activations/layer19_attention_weight_min": -32.32075881958008,
      "activations/layer1_attention_weight_max": 16.46398162841797,
      "activations/layer1_attention_weight_min": -15.912604331970215,
      "activations/layer20_attention_weight_max": 30.439390182495117,
      "activations/layer20_attention_weight_min": -25.296123504638672,
      "activations/layer21_attention_weight_max": 30.873088836669922,
      "activations/layer21_attention_weight_min": -24.833532333374023,
      "activations/layer22_attention_weight_max": 44.817298889160156,
      "activations/layer22_attention_weight_min": -29.709978103637695,
      "activations/layer23_attention_weight_max": 34.07151412963867,
      "activations/layer23_attention_weight_min": -24.7465877532959,
      "activations/layer2_attention_weight_max": 30.958480834960938,
      "activations/layer2_attention_weight_min": -30.690887451171875,
      "activations/layer3_attention_weight_max": 91.08280181884766,
      "activations/layer3_attention_weight_min": -96.06105041503906,
      "activations/layer4_attention_weight_max": 109.96614837646484,
      "activations/layer4_attention_weight_min": -116.3492202758789,
      "activations/layer5_attention_weight_max": 53.28229904174805,
      "activations/layer5_attention_weight_min": -63.21885299682617,
      "activations/layer6_attention_weight_max": 44.14990997314453,
      "activations/layer6_attention_weight_min": -49.0772590637207,
      "activations/layer7_attention_weight_max": 89.20388793945312,
      "activations/layer7_attention_weight_min": -90.08328247070312,
      "activations/layer8_attention_weight_max": 41.16353225708008,
      "activations/layer8_attention_weight_min": -44.077308654785156,
      "activations/layer9_attention_weight_max": 33.817684173583984,
      "activations/layer9_attention_weight_min": -33.051185607910156,
      "epoch": 12.5,
      "learning_rate": 7.008749999999999e-05,
      "loss": 2.7612,
      "step": 215150
    },
    {
      "activations/layer0_attention_weight_max": 15.642242431640625,
      "activations/layer0_attention_weight_min": -11.626703262329102,
      "activations/layer10_attention_weight_max": 38.82003402709961,
      "activations/layer10_attention_weight_min": -33.807861328125,
      "activations/layer11_attention_weight_max": 39.964969635009766,
      "activations/layer11_attention_weight_min": -34.421783447265625,
      "activations/layer12_attention_weight_max": 29.82697296142578,
      "activations/layer12_attention_weight_min": -25.992164611816406,
      "activations/layer13_attention_weight_max": 42.747188568115234,
      "activations/layer13_attention_weight_min": -34.76070785522461,
      "activations/layer14_attention_weight_max": 43.97883987426758,
      "activations/layer14_attention_weight_min": -37.750457763671875,
      "activations/layer15_attention_weight_max": 37.83613967895508,
      "activations/layer15_attention_weight_min": -32.46255111694336,
      "activations/layer16_attention_weight_max": 33.00888442993164,
      "activations/layer16_attention_weight_min": -26.35618019104004,
      "activations/layer17_attention_weight_max": 31.5870361328125,
      "activations/layer17_attention_weight_min": -25.35405921936035,
      "activations/layer18_attention_weight_max": 30.2049560546875,
      "activations/layer18_attention_weight_min": -23.685209274291992,
      "activations/layer19_attention_weight_max": 35.117698669433594,
      "activations/layer19_attention_weight_min": -31.636632919311523,
      "activations/layer1_attention_weight_max": 16.125282287597656,
      "activations/layer1_attention_weight_min": -14.568415641784668,
      "activations/layer20_attention_weight_max": 30.35674476623535,
      "activations/layer20_attention_weight_min": -29.185134887695312,
      "activations/layer21_attention_weight_max": 29.78622817993164,
      "activations/layer21_attention_weight_min": -25.00798988342285,
      "activations/layer22_attention_weight_max": 43.194847106933594,
      "activations/layer22_attention_weight_min": -30.09769058227539,
      "activations/layer23_attention_weight_max": 32.781490325927734,
      "activations/layer23_attention_weight_min": -24.44746208190918,
      "activations/layer2_attention_weight_max": 32.89586639404297,
      "activations/layer2_attention_weight_min": -32.99259948730469,
      "activations/layer3_attention_weight_max": 91.20343780517578,
      "activations/layer3_attention_weight_min": -95.63525390625,
      "activations/layer4_attention_weight_max": 109.18065643310547,
      "activations/layer4_attention_weight_min": -106.06758117675781,
      "activations/layer5_attention_weight_max": 51.75480270385742,
      "activations/layer5_attention_weight_min": -57.602718353271484,
      "activations/layer6_attention_weight_max": 44.16883087158203,
      "activations/layer6_attention_weight_min": -47.54590606689453,
      "activations/layer7_attention_weight_max": 100.3695068359375,
      "activations/layer7_attention_weight_min": -94.6512222290039,
      "activations/layer8_attention_weight_max": 45.22270584106445,
      "activations/layer8_attention_weight_min": -42.473472595214844,
      "activations/layer9_attention_weight_max": 36.848812103271484,
      "activations/layer9_attention_weight_min": -35.06166458129883,
      "epoch": 12.5,
      "learning_rate": 7.00685606060606e-05,
      "loss": 2.776,
      "step": 215200
    },
    {
      "activations/layer0_attention_weight_max": 15.621752738952637,
      "activations/layer0_attention_weight_min": -12.044426918029785,
      "activations/layer10_attention_weight_max": 31.167678833007812,
      "activations/layer10_attention_weight_min": -31.68435287475586,
      "activations/layer11_attention_weight_max": 32.4891242980957,
      "activations/layer11_attention_weight_min": -30.164701461791992,
      "activations/layer12_attention_weight_max": 22.88599967956543,
      "activations/layer12_attention_weight_min": -24.271331787109375,
      "activations/layer13_attention_weight_max": 35.630767822265625,
      "activations/layer13_attention_weight_min": -32.39152908325195,
      "activations/layer14_attention_weight_max": 38.762550354003906,
      "activations/layer14_attention_weight_min": -35.56444549560547,
      "activations/layer15_attention_weight_max": 34.57829284667969,
      "activations/layer15_attention_weight_min": -34.16398239135742,
      "activations/layer16_attention_weight_max": 30.13153839111328,
      "activations/layer16_attention_weight_min": -29.7359676361084,
      "activations/layer17_attention_weight_max": 30.77376365661621,
      "activations/layer17_attention_weight_min": -25.494842529296875,
      "activations/layer18_attention_weight_max": 29.739519119262695,
      "activations/layer18_attention_weight_min": -24.22757911682129,
      "activations/layer19_attention_weight_max": 35.255123138427734,
      "activations/layer19_attention_weight_min": -31.89181900024414,
      "activations/layer1_attention_weight_max": 16.35446548461914,
      "activations/layer1_attention_weight_min": -14.299707412719727,
      "activations/layer20_attention_weight_max": 27.696975708007812,
      "activations/layer20_attention_weight_min": -26.243263244628906,
      "activations/layer21_attention_weight_max": 26.93172836303711,
      "activations/layer21_attention_weight_min": -27.73727035522461,
      "activations/layer22_attention_weight_max": 41.73078155517578,
      "activations/layer22_attention_weight_min": -32.40812301635742,
      "activations/layer23_attention_weight_max": 36.50168991088867,
      "activations/layer23_attention_weight_min": -25.031484603881836,
      "activations/layer2_attention_weight_max": 34.60550308227539,
      "activations/layer2_attention_weight_min": -31.363445281982422,
      "activations/layer3_attention_weight_max": 90.35690307617188,
      "activations/layer3_attention_weight_min": -94.19491577148438,
      "activations/layer4_attention_weight_max": 105.9705810546875,
      "activations/layer4_attention_weight_min": -105.72386932373047,
      "activations/layer5_attention_weight_max": 54.23085021972656,
      "activations/layer5_attention_weight_min": -61.9354248046875,
      "activations/layer6_attention_weight_max": 42.99579620361328,
      "activations/layer6_attention_weight_min": -44.94294738769531,
      "activations/layer7_attention_weight_max": 86.97382354736328,
      "activations/layer7_attention_weight_min": -85.91165924072266,
      "activations/layer8_attention_weight_max": 37.86243438720703,
      "activations/layer8_attention_weight_min": -39.137451171875,
      "activations/layer9_attention_weight_max": 30.46869468688965,
      "activations/layer9_attention_weight_min": -31.60626220703125,
      "epoch": 12.51,
      "learning_rate": 7.00496212121212e-05,
      "loss": 2.7647,
      "step": 215250
    },
    {
      "activations/layer0_attention_weight_max": 16.039112091064453,
      "activations/layer0_attention_weight_min": -11.748373985290527,
      "activations/layer10_attention_weight_max": 34.39985275268555,
      "activations/layer10_attention_weight_min": -33.91144943237305,
      "activations/layer11_attention_weight_max": 32.052101135253906,
      "activations/layer11_attention_weight_min": -32.62807846069336,
      "activations/layer12_attention_weight_max": 24.014183044433594,
      "activations/layer12_attention_weight_min": -25.418197631835938,
      "activations/layer13_attention_weight_max": 37.7441291809082,
      "activations/layer13_attention_weight_min": -33.31283950805664,
      "activations/layer14_attention_weight_max": 42.046146392822266,
      "activations/layer14_attention_weight_min": -36.648162841796875,
      "activations/layer15_attention_weight_max": 35.03202438354492,
      "activations/layer15_attention_weight_min": -31.483583450317383,
      "activations/layer16_attention_weight_max": 32.886962890625,
      "activations/layer16_attention_weight_min": -27.61456298828125,
      "activations/layer17_attention_weight_max": 31.852895736694336,
      "activations/layer17_attention_weight_min": -25.359941482543945,
      "activations/layer18_attention_weight_max": 29.803874969482422,
      "activations/layer18_attention_weight_min": -23.502899169921875,
      "activations/layer19_attention_weight_max": 40.3659553527832,
      "activations/layer19_attention_weight_min": -30.776077270507812,
      "activations/layer1_attention_weight_max": 16.109195709228516,
      "activations/layer1_attention_weight_min": -13.487887382507324,
      "activations/layer20_attention_weight_max": 31.323402404785156,
      "activations/layer20_attention_weight_min": -24.40190315246582,
      "activations/layer21_attention_weight_max": 29.405107498168945,
      "activations/layer21_attention_weight_min": -24.01235008239746,
      "activations/layer22_attention_weight_max": 42.82425308227539,
      "activations/layer22_attention_weight_min": -28.781803131103516,
      "activations/layer23_attention_weight_max": 33.04373550415039,
      "activations/layer23_attention_weight_min": -22.597198486328125,
      "activations/layer2_attention_weight_max": 32.30424499511719,
      "activations/layer2_attention_weight_min": -31.20505142211914,
      "activations/layer3_attention_weight_max": 95.15003204345703,
      "activations/layer3_attention_weight_min": -94.31491088867188,
      "activations/layer4_attention_weight_max": 113.89347076416016,
      "activations/layer4_attention_weight_min": -112.47826385498047,
      "activations/layer5_attention_weight_max": 52.42327117919922,
      "activations/layer5_attention_weight_min": -60.24658966064453,
      "activations/layer6_attention_weight_max": 42.80988693237305,
      "activations/layer6_attention_weight_min": -47.90413284301758,
      "activations/layer7_attention_weight_max": 90.43536376953125,
      "activations/layer7_attention_weight_min": -89.48953247070312,
      "activations/layer8_attention_weight_max": 41.97129821777344,
      "activations/layer8_attention_weight_min": -41.738807678222656,
      "activations/layer9_attention_weight_max": 33.36884689331055,
      "activations/layer9_attention_weight_min": -34.312660217285156,
      "epoch": 12.51,
      "learning_rate": 7.003068181818181e-05,
      "loss": 2.7632,
      "step": 215300
    },
    {
      "activations/layer0_attention_weight_max": 16.210506439208984,
      "activations/layer0_attention_weight_min": -12.842521667480469,
      "activations/layer10_attention_weight_max": 31.408763885498047,
      "activations/layer10_attention_weight_min": -32.236305236816406,
      "activations/layer11_attention_weight_max": 31.236921310424805,
      "activations/layer11_attention_weight_min": -31.90953826904297,
      "activations/layer12_attention_weight_max": 23.94364356994629,
      "activations/layer12_attention_weight_min": -24.660865783691406,
      "activations/layer13_attention_weight_max": 39.075923919677734,
      "activations/layer13_attention_weight_min": -32.805419921875,
      "activations/layer14_attention_weight_max": 38.503658294677734,
      "activations/layer14_attention_weight_min": -36.301063537597656,
      "activations/layer15_attention_weight_max": 34.13404083251953,
      "activations/layer15_attention_weight_min": -33.53565979003906,
      "activations/layer16_attention_weight_max": 28.615751266479492,
      "activations/layer16_attention_weight_min": -28.053302764892578,
      "activations/layer17_attention_weight_max": 31.1013126373291,
      "activations/layer17_attention_weight_min": -26.37957191467285,
      "activations/layer18_attention_weight_max": 30.376554489135742,
      "activations/layer18_attention_weight_min": -24.207866668701172,
      "activations/layer19_attention_weight_max": 30.75516128540039,
      "activations/layer19_attention_weight_min": -31.228778839111328,
      "activations/layer1_attention_weight_max": 15.889309883117676,
      "activations/layer1_attention_weight_min": -14.52063274383545,
      "activations/layer20_attention_weight_max": 27.2917423248291,
      "activations/layer20_attention_weight_min": -24.700218200683594,
      "activations/layer21_attention_weight_max": 29.431644439697266,
      "activations/layer21_attention_weight_min": -25.53692626953125,
      "activations/layer22_attention_weight_max": 41.30067443847656,
      "activations/layer22_attention_weight_min": -29.290498733520508,
      "activations/layer23_attention_weight_max": 33.14540481567383,
      "activations/layer23_attention_weight_min": -25.193775177001953,
      "activations/layer2_attention_weight_max": 31.955032348632812,
      "activations/layer2_attention_weight_min": -30.257352828979492,
      "activations/layer3_attention_weight_max": 91.94970703125,
      "activations/layer3_attention_weight_min": -94.08303833007812,
      "activations/layer4_attention_weight_max": 110.8946304321289,
      "activations/layer4_attention_weight_min": -110.83513641357422,
      "activations/layer5_attention_weight_max": 49.99178695678711,
      "activations/layer5_attention_weight_min": -63.80994415283203,
      "activations/layer6_attention_weight_max": 44.59126281738281,
      "activations/layer6_attention_weight_min": -47.99213409423828,
      "activations/layer7_attention_weight_max": 89.47512817382812,
      "activations/layer7_attention_weight_min": -86.2480239868164,
      "activations/layer8_attention_weight_max": 39.24768829345703,
      "activations/layer8_attention_weight_min": -40.14482879638672,
      "activations/layer9_attention_weight_max": 31.317533493041992,
      "activations/layer9_attention_weight_min": -31.356536865234375,
      "epoch": 12.51,
      "learning_rate": 7.001174242424242e-05,
      "loss": 2.7685,
      "step": 215350
    },
    {
      "activations/layer0_attention_weight_max": 15.69776725769043,
      "activations/layer0_attention_weight_min": -12.187893867492676,
      "activations/layer10_attention_weight_max": 32.66659164428711,
      "activations/layer10_attention_weight_min": -33.582984924316406,
      "activations/layer11_attention_weight_max": 33.669029235839844,
      "activations/layer11_attention_weight_min": -34.77770233154297,
      "activations/layer12_attention_weight_max": 26.41470718383789,
      "activations/layer12_attention_weight_min": -25.49802017211914,
      "activations/layer13_attention_weight_max": 38.92024612426758,
      "activations/layer13_attention_weight_min": -35.11494064331055,
      "activations/layer14_attention_weight_max": 42.86550521850586,
      "activations/layer14_attention_weight_min": -39.52265167236328,
      "activations/layer15_attention_weight_max": 37.584266662597656,
      "activations/layer15_attention_weight_min": -35.17687225341797,
      "activations/layer16_attention_weight_max": 33.23776626586914,
      "activations/layer16_attention_weight_min": -28.348373413085938,
      "activations/layer17_attention_weight_max": 32.46506881713867,
      "activations/layer17_attention_weight_min": -26.880281448364258,
      "activations/layer18_attention_weight_max": 30.785213470458984,
      "activations/layer18_attention_weight_min": -23.257991790771484,
      "activations/layer19_attention_weight_max": 35.45378112792969,
      "activations/layer19_attention_weight_min": -30.109848022460938,
      "activations/layer1_attention_weight_max": 15.981176376342773,
      "activations/layer1_attention_weight_min": -14.546817779541016,
      "activations/layer20_attention_weight_max": 32.19223403930664,
      "activations/layer20_attention_weight_min": -25.923643112182617,
      "activations/layer21_attention_weight_max": 31.36600112915039,
      "activations/layer21_attention_weight_min": -23.75225067138672,
      "activations/layer22_attention_weight_max": 46.81377029418945,
      "activations/layer22_attention_weight_min": -29.864608764648438,
      "activations/layer23_attention_weight_max": 38.576168060302734,
      "activations/layer23_attention_weight_min": -27.10049057006836,
      "activations/layer2_attention_weight_max": 32.363014221191406,
      "activations/layer2_attention_weight_min": -31.12430191040039,
      "activations/layer3_attention_weight_max": 95.54505157470703,
      "activations/layer3_attention_weight_min": -94.47875213623047,
      "activations/layer4_attention_weight_max": 113.5706558227539,
      "activations/layer4_attention_weight_min": -112.13871002197266,
      "activations/layer5_attention_weight_max": 50.75462341308594,
      "activations/layer5_attention_weight_min": -62.14786911010742,
      "activations/layer6_attention_weight_max": 45.680240631103516,
      "activations/layer6_attention_weight_min": -47.69389724731445,
      "activations/layer7_attention_weight_max": 90.29896545410156,
      "activations/layer7_attention_weight_min": -88.1009292602539,
      "activations/layer8_attention_weight_max": 39.66683578491211,
      "activations/layer8_attention_weight_min": -41.2680549621582,
      "activations/layer9_attention_weight_max": 31.709211349487305,
      "activations/layer9_attention_weight_min": -34.23409652709961,
      "epoch": 12.52,
      "learning_rate": 6.999280303030302e-05,
      "loss": 2.7788,
      "step": 215400
    },
    {
      "activations/layer0_attention_weight_max": 15.594770431518555,
      "activations/layer0_attention_weight_min": -12.065998077392578,
      "activations/layer10_attention_weight_max": 31.74739646911621,
      "activations/layer10_attention_weight_min": -31.22336769104004,
      "activations/layer11_attention_weight_max": 32.78837203979492,
      "activations/layer11_attention_weight_min": -30.054777145385742,
      "activations/layer12_attention_weight_max": 24.94037437438965,
      "activations/layer12_attention_weight_min": -24.968351364135742,
      "activations/layer13_attention_weight_max": 39.48632049560547,
      "activations/layer13_attention_weight_min": -34.24223327636719,
      "activations/layer14_attention_weight_max": 43.48247146606445,
      "activations/layer14_attention_weight_min": -38.16010665893555,
      "activations/layer15_attention_weight_max": 34.99755096435547,
      "activations/layer15_attention_weight_min": -33.63652801513672,
      "activations/layer16_attention_weight_max": 29.40254783630371,
      "activations/layer16_attention_weight_min": -26.732194900512695,
      "activations/layer17_attention_weight_max": 29.254236221313477,
      "activations/layer17_attention_weight_min": -25.366806030273438,
      "activations/layer18_attention_weight_max": 29.488744735717773,
      "activations/layer18_attention_weight_min": -25.602293014526367,
      "activations/layer19_attention_weight_max": 33.64521408081055,
      "activations/layer19_attention_weight_min": -30.931915283203125,
      "activations/layer1_attention_weight_max": 16.387136459350586,
      "activations/layer1_attention_weight_min": -15.128662109375,
      "activations/layer20_attention_weight_max": 28.065135955810547,
      "activations/layer20_attention_weight_min": -26.61751937866211,
      "activations/layer21_attention_weight_max": 26.96391487121582,
      "activations/layer21_attention_weight_min": -24.614713668823242,
      "activations/layer22_attention_weight_max": 43.29848861694336,
      "activations/layer22_attention_weight_min": -30.665889739990234,
      "activations/layer23_attention_weight_max": 30.847431182861328,
      "activations/layer23_attention_weight_min": -26.104663848876953,
      "activations/layer2_attention_weight_max": 31.711009979248047,
      "activations/layer2_attention_weight_min": -30.328960418701172,
      "activations/layer3_attention_weight_max": 95.11676788330078,
      "activations/layer3_attention_weight_min": -88.5541763305664,
      "activations/layer4_attention_weight_max": 111.14226531982422,
      "activations/layer4_attention_weight_min": -106.32057189941406,
      "activations/layer5_attention_weight_max": 48.429603576660156,
      "activations/layer5_attention_weight_min": -59.5116081237793,
      "activations/layer6_attention_weight_max": 45.241859436035156,
      "activations/layer6_attention_weight_min": -46.33308029174805,
      "activations/layer7_attention_weight_max": 83.6978759765625,
      "activations/layer7_attention_weight_min": -83.6291732788086,
      "activations/layer8_attention_weight_max": 38.631752014160156,
      "activations/layer8_attention_weight_min": -39.77162551879883,
      "activations/layer9_attention_weight_max": 30.84473991394043,
      "activations/layer9_attention_weight_min": -31.028356552124023,
      "epoch": 12.52,
      "learning_rate": 6.997386363636364e-05,
      "loss": 2.7485,
      "step": 215450
    },
    {
      "activations/layer0_attention_weight_max": 16.746015548706055,
      "activations/layer0_attention_weight_min": -13.000353813171387,
      "activations/layer10_attention_weight_max": 29.441984176635742,
      "activations/layer10_attention_weight_min": -31.415111541748047,
      "activations/layer11_attention_weight_max": 32.009605407714844,
      "activations/layer11_attention_weight_min": -31.05202293395996,
      "activations/layer12_attention_weight_max": 23.633689880371094,
      "activations/layer12_attention_weight_min": -25.749286651611328,
      "activations/layer13_attention_weight_max": 36.8922004699707,
      "activations/layer13_attention_weight_min": -35.21017074584961,
      "activations/layer14_attention_weight_max": 41.12721252441406,
      "activations/layer14_attention_weight_min": -35.80475616455078,
      "activations/layer15_attention_weight_max": 38.007808685302734,
      "activations/layer15_attention_weight_min": -32.16596984863281,
      "activations/layer16_attention_weight_max": 29.964622497558594,
      "activations/layer16_attention_weight_min": -26.88843536376953,
      "activations/layer17_attention_weight_max": 27.91567611694336,
      "activations/layer17_attention_weight_min": -25.722272872924805,
      "activations/layer18_attention_weight_max": 30.480810165405273,
      "activations/layer18_attention_weight_min": -23.589839935302734,
      "activations/layer19_attention_weight_max": 32.47423553466797,
      "activations/layer19_attention_weight_min": -33.119625091552734,
      "activations/layer1_attention_weight_max": 15.619119644165039,
      "activations/layer1_attention_weight_min": -14.563240051269531,
      "activations/layer20_attention_weight_max": 30.273107528686523,
      "activations/layer20_attention_weight_min": -24.926889419555664,
      "activations/layer21_attention_weight_max": 28.215471267700195,
      "activations/layer21_attention_weight_min": -25.850994110107422,
      "activations/layer22_attention_weight_max": 40.402061462402344,
      "activations/layer22_attention_weight_min": -29.666597366333008,
      "activations/layer23_attention_weight_max": 31.795515060424805,
      "activations/layer23_attention_weight_min": -23.98271942138672,
      "activations/layer2_attention_weight_max": 29.596134185791016,
      "activations/layer2_attention_weight_min": -30.60072135925293,
      "activations/layer3_attention_weight_max": 90.3021469116211,
      "activations/layer3_attention_weight_min": -96.47944641113281,
      "activations/layer4_attention_weight_max": 104.75904846191406,
      "activations/layer4_attention_weight_min": -108.41773986816406,
      "activations/layer5_attention_weight_max": 51.18877410888672,
      "activations/layer5_attention_weight_min": -59.9766960144043,
      "activations/layer6_attention_weight_max": 42.1866455078125,
      "activations/layer6_attention_weight_min": -48.13005447387695,
      "activations/layer7_attention_weight_max": 85.92842102050781,
      "activations/layer7_attention_weight_min": -85.48455810546875,
      "activations/layer8_attention_weight_max": 40.45759963989258,
      "activations/layer8_attention_weight_min": -40.17169189453125,
      "activations/layer9_attention_weight_max": 28.7427921295166,
      "activations/layer9_attention_weight_min": -30.95368003845215,
      "epoch": 12.52,
      "learning_rate": 6.995492424242424e-05,
      "loss": 2.7751,
      "step": 215500
    },
    {
      "activations/layer0_attention_weight_max": 15.821942329406738,
      "activations/layer0_attention_weight_min": -12.101314544677734,
      "activations/layer10_attention_weight_max": 30.7965087890625,
      "activations/layer10_attention_weight_min": -31.44145393371582,
      "activations/layer11_attention_weight_max": 33.884674072265625,
      "activations/layer11_attention_weight_min": -31.447566986083984,
      "activations/layer12_attention_weight_max": 32.846805572509766,
      "activations/layer12_attention_weight_min": -26.17641830444336,
      "activations/layer13_attention_weight_max": 36.83745193481445,
      "activations/layer13_attention_weight_min": -34.348655700683594,
      "activations/layer14_attention_weight_max": 42.30466842651367,
      "activations/layer14_attention_weight_min": -41.802764892578125,
      "activations/layer15_attention_weight_max": 33.71861267089844,
      "activations/layer15_attention_weight_min": -32.22391128540039,
      "activations/layer16_attention_weight_max": 29.876089096069336,
      "activations/layer16_attention_weight_min": -28.09347152709961,
      "activations/layer17_attention_weight_max": 30.00503921508789,
      "activations/layer17_attention_weight_min": -25.389041900634766,
      "activations/layer18_attention_weight_max": 31.56138801574707,
      "activations/layer18_attention_weight_min": -23.441884994506836,
      "activations/layer19_attention_weight_max": 32.083621978759766,
      "activations/layer19_attention_weight_min": -31.6624813079834,
      "activations/layer1_attention_weight_max": 16.673946380615234,
      "activations/layer1_attention_weight_min": -14.042280197143555,
      "activations/layer20_attention_weight_max": 29.264991760253906,
      "activations/layer20_attention_weight_min": -25.104063034057617,
      "activations/layer21_attention_weight_max": 29.250885009765625,
      "activations/layer21_attention_weight_min": -23.5218448638916,
      "activations/layer22_attention_weight_max": 44.139408111572266,
      "activations/layer22_attention_weight_min": -30.657928466796875,
      "activations/layer23_attention_weight_max": 32.581878662109375,
      "activations/layer23_attention_weight_min": -24.77069091796875,
      "activations/layer2_attention_weight_max": 30.777599334716797,
      "activations/layer2_attention_weight_min": -31.007747650146484,
      "activations/layer3_attention_weight_max": 89.03147888183594,
      "activations/layer3_attention_weight_min": -95.1050796508789,
      "activations/layer4_attention_weight_max": 107.47467041015625,
      "activations/layer4_attention_weight_min": -110.68248748779297,
      "activations/layer5_attention_weight_max": 50.00699996948242,
      "activations/layer5_attention_weight_min": -67.33261108398438,
      "activations/layer6_attention_weight_max": 41.62528991699219,
      "activations/layer6_attention_weight_min": -45.71936798095703,
      "activations/layer7_attention_weight_max": 86.0178451538086,
      "activations/layer7_attention_weight_min": -84.71023559570312,
      "activations/layer8_attention_weight_max": 37.35316467285156,
      "activations/layer8_attention_weight_min": -38.61150360107422,
      "activations/layer9_attention_weight_max": 30.87139892578125,
      "activations/layer9_attention_weight_min": -29.991613388061523,
      "epoch": 12.52,
      "learning_rate": 6.993636363636363e-05,
      "loss": 2.7729,
      "step": 215550
    },
    {
      "activations/layer0_attention_weight_max": 16.73371124267578,
      "activations/layer0_attention_weight_min": -11.551277160644531,
      "activations/layer10_attention_weight_max": 33.386741638183594,
      "activations/layer10_attention_weight_min": -33.155487060546875,
      "activations/layer11_attention_weight_max": 34.29222106933594,
      "activations/layer11_attention_weight_min": -32.14680480957031,
      "activations/layer12_attention_weight_max": 24.890405654907227,
      "activations/layer12_attention_weight_min": -26.32334327697754,
      "activations/layer13_attention_weight_max": 36.31153869628906,
      "activations/layer13_attention_weight_min": -33.09944152832031,
      "activations/layer14_attention_weight_max": 39.88481140136719,
      "activations/layer14_attention_weight_min": -35.343292236328125,
      "activations/layer15_attention_weight_max": 37.1637077331543,
      "activations/layer15_attention_weight_min": -32.61046600341797,
      "activations/layer16_attention_weight_max": 30.975662231445312,
      "activations/layer16_attention_weight_min": -29.589916229248047,
      "activations/layer17_attention_weight_max": 31.227136611938477,
      "activations/layer17_attention_weight_min": -25.142526626586914,
      "activations/layer18_attention_weight_max": 29.269084930419922,
      "activations/layer18_attention_weight_min": -23.285982131958008,
      "activations/layer19_attention_weight_max": 31.914060592651367,
      "activations/layer19_attention_weight_min": -32.115413665771484,
      "activations/layer1_attention_weight_max": 16.688867568969727,
      "activations/layer1_attention_weight_min": -14.469741821289062,
      "activations/layer20_attention_weight_max": 26.906715393066406,
      "activations/layer20_attention_weight_min": -25.122602462768555,
      "activations/layer21_attention_weight_max": 26.62038803100586,
      "activations/layer21_attention_weight_min": -24.168521881103516,
      "activations/layer22_attention_weight_max": 43.721275329589844,
      "activations/layer22_attention_weight_min": -29.449304580688477,
      "activations/layer23_attention_weight_max": 30.734989166259766,
      "activations/layer23_attention_weight_min": -25.180723190307617,
      "activations/layer2_attention_weight_max": 34.27135467529297,
      "activations/layer2_attention_weight_min": -34.37369155883789,
      "activations/layer3_attention_weight_max": 104.00277709960938,
      "activations/layer3_attention_weight_min": -103.56786346435547,
      "activations/layer4_attention_weight_max": 118.75374603271484,
      "activations/layer4_attention_weight_min": -117.33397674560547,
      "activations/layer5_attention_weight_max": 53.77130126953125,
      "activations/layer5_attention_weight_min": -58.04508972167969,
      "activations/layer6_attention_weight_max": 43.92806625366211,
      "activations/layer6_attention_weight_min": -49.57875442504883,
      "activations/layer7_attention_weight_max": 88.84520721435547,
      "activations/layer7_attention_weight_min": -96.21709442138672,
      "activations/layer8_attention_weight_max": 40.75387954711914,
      "activations/layer8_attention_weight_min": -43.27271270751953,
      "activations/layer9_attention_weight_max": 34.1905517578125,
      "activations/layer9_attention_weight_min": -34.30404281616211,
      "epoch": 12.53,
      "learning_rate": 6.991742424242423e-05,
      "loss": 2.7658,
      "step": 215600
    },
    {
      "activations/layer0_attention_weight_max": 15.38202953338623,
      "activations/layer0_attention_weight_min": -11.774520874023438,
      "activations/layer10_attention_weight_max": 38.87451171875,
      "activations/layer10_attention_weight_min": -36.2300910949707,
      "activations/layer11_attention_weight_max": 40.74589538574219,
      "activations/layer11_attention_weight_min": -37.47039031982422,
      "activations/layer12_attention_weight_max": 27.03856086730957,
      "activations/layer12_attention_weight_min": -25.92280387878418,
      "activations/layer13_attention_weight_max": 39.469783782958984,
      "activations/layer13_attention_weight_min": -35.192928314208984,
      "activations/layer14_attention_weight_max": 44.8267936706543,
      "activations/layer14_attention_weight_min": -40.33060836791992,
      "activations/layer15_attention_weight_max": 36.84178924560547,
      "activations/layer15_attention_weight_min": -33.65751647949219,
      "activations/layer16_attention_weight_max": 29.6864070892334,
      "activations/layer16_attention_weight_min": -28.963911056518555,
      "activations/layer17_attention_weight_max": 29.41891860961914,
      "activations/layer17_attention_weight_min": -25.565000534057617,
      "activations/layer18_attention_weight_max": 29.0219783782959,
      "activations/layer18_attention_weight_min": -24.1475887298584,
      "activations/layer19_attention_weight_max": 38.8402214050293,
      "activations/layer19_attention_weight_min": -30.73733901977539,
      "activations/layer1_attention_weight_max": 16.700986862182617,
      "activations/layer1_attention_weight_min": -14.259358406066895,
      "activations/layer20_attention_weight_max": 31.090923309326172,
      "activations/layer20_attention_weight_min": -27.155593872070312,
      "activations/layer21_attention_weight_max": 30.480003356933594,
      "activations/layer21_attention_weight_min": -24.236820220947266,
      "activations/layer22_attention_weight_max": 46.79996871948242,
      "activations/layer22_attention_weight_min": -29.77216911315918,
      "activations/layer23_attention_weight_max": 35.696598052978516,
      "activations/layer23_attention_weight_min": -24.61223602294922,
      "activations/layer2_attention_weight_max": 30.503477096557617,
      "activations/layer2_attention_weight_min": -32.737831115722656,
      "activations/layer3_attention_weight_max": 91.09921264648438,
      "activations/layer3_attention_weight_min": -101.38643646240234,
      "activations/layer4_attention_weight_max": 108.47371673583984,
      "activations/layer4_attention_weight_min": -111.5688247680664,
      "activations/layer5_attention_weight_max": 52.96337890625,
      "activations/layer5_attention_weight_min": -58.47805404663086,
      "activations/layer6_attention_weight_max": 46.07765197753906,
      "activations/layer6_attention_weight_min": -47.24658966064453,
      "activations/layer7_attention_weight_max": 95.78922271728516,
      "activations/layer7_attention_weight_min": -94.1459732055664,
      "activations/layer8_attention_weight_max": 45.944366455078125,
      "activations/layer8_attention_weight_min": -43.38379669189453,
      "activations/layer9_attention_weight_max": 38.55532455444336,
      "activations/layer9_attention_weight_min": -35.50575256347656,
      "epoch": 12.53,
      "learning_rate": 6.989848484848485e-05,
      "loss": 2.7735,
      "step": 215650
    },
    {
      "activations/layer0_attention_weight_max": 15.167344093322754,
      "activations/layer0_attention_weight_min": -11.90645694732666,
      "activations/layer10_attention_weight_max": 35.627784729003906,
      "activations/layer10_attention_weight_min": -35.177520751953125,
      "activations/layer11_attention_weight_max": 37.1496467590332,
      "activations/layer11_attention_weight_min": -35.336524963378906,
      "activations/layer12_attention_weight_max": 27.157777786254883,
      "activations/layer12_attention_weight_min": -28.715877532958984,
      "activations/layer13_attention_weight_max": 41.5504150390625,
      "activations/layer13_attention_weight_min": -35.74717712402344,
      "activations/layer14_attention_weight_max": 46.49269485473633,
      "activations/layer14_attention_weight_min": -41.875709533691406,
      "activations/layer15_attention_weight_max": 43.08076095581055,
      "activations/layer15_attention_weight_min": -35.58539581298828,
      "activations/layer16_attention_weight_max": 36.564109802246094,
      "activations/layer16_attention_weight_min": -29.640275955200195,
      "activations/layer17_attention_weight_max": 35.814048767089844,
      "activations/layer17_attention_weight_min": -27.999595642089844,
      "activations/layer18_attention_weight_max": 38.17028045654297,
      "activations/layer18_attention_weight_min": -25.920177459716797,
      "activations/layer19_attention_weight_max": 46.47721481323242,
      "activations/layer19_attention_weight_min": -32.59455490112305,
      "activations/layer1_attention_weight_max": 16.1428165435791,
      "activations/layer1_attention_weight_min": -14.73870849609375,
      "activations/layer20_attention_weight_max": 35.87763595581055,
      "activations/layer20_attention_weight_min": -23.870637893676758,
      "activations/layer21_attention_weight_max": 36.152225494384766,
      "activations/layer21_attention_weight_min": -26.074216842651367,
      "activations/layer22_attention_weight_max": 55.9382209777832,
      "activations/layer22_attention_weight_min": -30.3919734954834,
      "activations/layer23_attention_weight_max": 43.33851623535156,
      "activations/layer23_attention_weight_min": -26.769668579101562,
      "activations/layer2_attention_weight_max": 30.763425827026367,
      "activations/layer2_attention_weight_min": -30.908023834228516,
      "activations/layer3_attention_weight_max": 91.08370971679688,
      "activations/layer3_attention_weight_min": -94.8321533203125,
      "activations/layer4_attention_weight_max": 112.6413345336914,
      "activations/layer4_attention_weight_min": -111.4512939453125,
      "activations/layer5_attention_weight_max": 50.805110931396484,
      "activations/layer5_attention_weight_min": -60.97712707519531,
      "activations/layer6_attention_weight_max": 45.01607131958008,
      "activations/layer6_attention_weight_min": -47.25222396850586,
      "activations/layer7_attention_weight_max": 92.81409454345703,
      "activations/layer7_attention_weight_min": -91.84437561035156,
      "activations/layer8_attention_weight_max": 41.98262405395508,
      "activations/layer8_attention_weight_min": -44.640708923339844,
      "activations/layer9_attention_weight_max": 36.43514633178711,
      "activations/layer9_attention_weight_min": -34.87266159057617,
      "epoch": 12.53,
      "learning_rate": 6.987954545454545e-05,
      "loss": 2.7594,
      "step": 215700
    },
    {
      "activations/layer0_attention_weight_max": 14.47287654876709,
      "activations/layer0_attention_weight_min": -11.69035530090332,
      "activations/layer10_attention_weight_max": 34.76424789428711,
      "activations/layer10_attention_weight_min": -34.277931213378906,
      "activations/layer11_attention_weight_max": 33.16790771484375,
      "activations/layer11_attention_weight_min": -33.08202362060547,
      "activations/layer12_attention_weight_max": 26.96062469482422,
      "activations/layer12_attention_weight_min": -26.63028335571289,
      "activations/layer13_attention_weight_max": 41.493804931640625,
      "activations/layer13_attention_weight_min": -35.58738327026367,
      "activations/layer14_attention_weight_max": 44.946598052978516,
      "activations/layer14_attention_weight_min": -38.983856201171875,
      "activations/layer15_attention_weight_max": 35.384490966796875,
      "activations/layer15_attention_weight_min": -34.10126495361328,
      "activations/layer16_attention_weight_max": 28.59906768798828,
      "activations/layer16_attention_weight_min": -30.30303955078125,
      "activations/layer17_attention_weight_max": 33.87556076049805,
      "activations/layer17_attention_weight_min": -29.662532806396484,
      "activations/layer18_attention_weight_max": 29.890235900878906,
      "activations/layer18_attention_weight_min": -25.252399444580078,
      "activations/layer19_attention_weight_max": 36.12575912475586,
      "activations/layer19_attention_weight_min": -33.972965240478516,
      "activations/layer1_attention_weight_max": 16.288347244262695,
      "activations/layer1_attention_weight_min": -15.316380500793457,
      "activations/layer20_attention_weight_max": 33.41325759887695,
      "activations/layer20_attention_weight_min": -28.454931259155273,
      "activations/layer21_attention_weight_max": 32.97652053833008,
      "activations/layer21_attention_weight_min": -28.49662208557129,
      "activations/layer22_attention_weight_max": 46.92375564575195,
      "activations/layer22_attention_weight_min": -32.434288024902344,
      "activations/layer23_attention_weight_max": 38.65327835083008,
      "activations/layer23_attention_weight_min": -24.21688461303711,
      "activations/layer2_attention_weight_max": 31.80994987487793,
      "activations/layer2_attention_weight_min": -31.40761375427246,
      "activations/layer3_attention_weight_max": 95.82347869873047,
      "activations/layer3_attention_weight_min": -98.21678924560547,
      "activations/layer4_attention_weight_max": 114.54036712646484,
      "activations/layer4_attention_weight_min": -116.474609375,
      "activations/layer5_attention_weight_max": 54.15010070800781,
      "activations/layer5_attention_weight_min": -63.368553161621094,
      "activations/layer6_attention_weight_max": 45.87822341918945,
      "activations/layer6_attention_weight_min": -49.893253326416016,
      "activations/layer7_attention_weight_max": 90.81266784667969,
      "activations/layer7_attention_weight_min": -93.69328308105469,
      "activations/layer8_attention_weight_max": 42.0211181640625,
      "activations/layer8_attention_weight_min": -44.08118438720703,
      "activations/layer9_attention_weight_max": 33.89689254760742,
      "activations/layer9_attention_weight_min": -33.73683166503906,
      "epoch": 12.54,
      "learning_rate": 6.986060606060605e-05,
      "loss": 2.7807,
      "step": 215750
    },
    {
      "activations/layer0_attention_weight_max": 15.378887176513672,
      "activations/layer0_attention_weight_min": -12.365877151489258,
      "activations/layer10_attention_weight_max": 36.27815246582031,
      "activations/layer10_attention_weight_min": -37.318485260009766,
      "activations/layer11_attention_weight_max": 36.72418212890625,
      "activations/layer11_attention_weight_min": -33.94829559326172,
      "activations/layer12_attention_weight_max": 27.80056381225586,
      "activations/layer12_attention_weight_min": -26.098403930664062,
      "activations/layer13_attention_weight_max": 42.82585144042969,
      "activations/layer13_attention_weight_min": -36.32872009277344,
      "activations/layer14_attention_weight_max": 46.53055953979492,
      "activations/layer14_attention_weight_min": -40.521488189697266,
      "activations/layer15_attention_weight_max": 52.73611068725586,
      "activations/layer15_attention_weight_min": -38.74673843383789,
      "activations/layer16_attention_weight_max": 33.45514678955078,
      "activations/layer16_attention_weight_min": -28.703996658325195,
      "activations/layer17_attention_weight_max": 36.078731536865234,
      "activations/layer17_attention_weight_min": -25.966686248779297,
      "activations/layer18_attention_weight_max": 30.921545028686523,
      "activations/layer18_attention_weight_min": -25.09672737121582,
      "activations/layer19_attention_weight_max": 31.695480346679688,
      "activations/layer19_attention_weight_min": -30.39019203186035,
      "activations/layer1_attention_weight_max": 16.706579208374023,
      "activations/layer1_attention_weight_min": -16.35725975036621,
      "activations/layer20_attention_weight_max": 28.71586799621582,
      "activations/layer20_attention_weight_min": -28.070816040039062,
      "activations/layer21_attention_weight_max": 28.58761978149414,
      "activations/layer21_attention_weight_min": -26.153301239013672,
      "activations/layer22_attention_weight_max": 41.452781677246094,
      "activations/layer22_attention_weight_min": -29.09650421142578,
      "activations/layer23_attention_weight_max": 32.57902526855469,
      "activations/layer23_attention_weight_min": -27.49611473083496,
      "activations/layer2_attention_weight_max": 31.05333709716797,
      "activations/layer2_attention_weight_min": -31.10804557800293,
      "activations/layer3_attention_weight_max": 96.3612289428711,
      "activations/layer3_attention_weight_min": -96.49369049072266,
      "activations/layer4_attention_weight_max": 112.7839126586914,
      "activations/layer4_attention_weight_min": -116.44872283935547,
      "activations/layer5_attention_weight_max": 50.38296127319336,
      "activations/layer5_attention_weight_min": -61.39105224609375,
      "activations/layer6_attention_weight_max": 45.907535552978516,
      "activations/layer6_attention_weight_min": -52.53620910644531,
      "activations/layer7_attention_weight_max": 94.2890396118164,
      "activations/layer7_attention_weight_min": -106.92330932617188,
      "activations/layer8_attention_weight_max": 41.918582916259766,
      "activations/layer8_attention_weight_min": -46.22909164428711,
      "activations/layer9_attention_weight_max": 36.6871452331543,
      "activations/layer9_attention_weight_min": -37.34722137451172,
      "epoch": 12.54,
      "learning_rate": 6.984166666666665e-05,
      "loss": 2.7603,
      "step": 215800
    },
    {
      "activations/layer0_attention_weight_max": 16.309953689575195,
      "activations/layer0_attention_weight_min": -12.24312973022461,
      "activations/layer10_attention_weight_max": 33.015289306640625,
      "activations/layer10_attention_weight_min": -32.37127685546875,
      "activations/layer11_attention_weight_max": 33.216217041015625,
      "activations/layer11_attention_weight_min": -31.18453025817871,
      "activations/layer12_attention_weight_max": 27.1148624420166,
      "activations/layer12_attention_weight_min": -26.6986026763916,
      "activations/layer13_attention_weight_max": 38.58839797973633,
      "activations/layer13_attention_weight_min": -37.29775619506836,
      "activations/layer14_attention_weight_max": 45.448760986328125,
      "activations/layer14_attention_weight_min": -41.775230407714844,
      "activations/layer15_attention_weight_max": 41.08785629272461,
      "activations/layer15_attention_weight_min": -32.54372024536133,
      "activations/layer16_attention_weight_max": 31.696619033813477,
      "activations/layer16_attention_weight_min": -28.183866500854492,
      "activations/layer17_attention_weight_max": 31.39742660522461,
      "activations/layer17_attention_weight_min": -26.754724502563477,
      "activations/layer18_attention_weight_max": 33.13777160644531,
      "activations/layer18_attention_weight_min": -25.906930923461914,
      "activations/layer19_attention_weight_max": 40.314300537109375,
      "activations/layer19_attention_weight_min": -31.687177658081055,
      "activations/layer1_attention_weight_max": 15.905503273010254,
      "activations/layer1_attention_weight_min": -14.2066650390625,
      "activations/layer20_attention_weight_max": 32.00080490112305,
      "activations/layer20_attention_weight_min": -25.3863582611084,
      "activations/layer21_attention_weight_max": 33.457916259765625,
      "activations/layer21_attention_weight_min": -23.866849899291992,
      "activations/layer22_attention_weight_max": 48.67325973510742,
      "activations/layer22_attention_weight_min": -31.51338005065918,
      "activations/layer23_attention_weight_max": 36.88330841064453,
      "activations/layer23_attention_weight_min": -24.35774040222168,
      "activations/layer2_attention_weight_max": 31.37774658203125,
      "activations/layer2_attention_weight_min": -31.239910125732422,
      "activations/layer3_attention_weight_max": 91.51031494140625,
      "activations/layer3_attention_weight_min": -94.02672576904297,
      "activations/layer4_attention_weight_max": 106.61915588378906,
      "activations/layer4_attention_weight_min": -113.43971252441406,
      "activations/layer5_attention_weight_max": 50.51277160644531,
      "activations/layer5_attention_weight_min": -62.46232223510742,
      "activations/layer6_attention_weight_max": 41.766204833984375,
      "activations/layer6_attention_weight_min": -48.52056121826172,
      "activations/layer7_attention_weight_max": 83.89777374267578,
      "activations/layer7_attention_weight_min": -89.83915710449219,
      "activations/layer8_attention_weight_max": 37.918540954589844,
      "activations/layer8_attention_weight_min": -39.686275482177734,
      "activations/layer9_attention_weight_max": 31.758316040039062,
      "activations/layer9_attention_weight_min": -31.661142349243164,
      "epoch": 12.54,
      "learning_rate": 6.982272727272727e-05,
      "loss": 2.7698,
      "step": 215850
    },
    {
      "activations/layer0_attention_weight_max": 15.996132850646973,
      "activations/layer0_attention_weight_min": -12.358922004699707,
      "activations/layer10_attention_weight_max": 30.11163330078125,
      "activations/layer10_attention_weight_min": -31.641881942749023,
      "activations/layer11_attention_weight_max": 29.622648239135742,
      "activations/layer11_attention_weight_min": -29.59381103515625,
      "activations/layer12_attention_weight_max": 21.85982894897461,
      "activations/layer12_attention_weight_min": -25.114591598510742,
      "activations/layer13_attention_weight_max": 33.823612213134766,
      "activations/layer13_attention_weight_min": -34.66170120239258,
      "activations/layer14_attention_weight_max": 37.72773742675781,
      "activations/layer14_attention_weight_min": -37.583900451660156,
      "activations/layer15_attention_weight_max": 32.69220733642578,
      "activations/layer15_attention_weight_min": -30.69092559814453,
      "activations/layer16_attention_weight_max": 27.85197639465332,
      "activations/layer16_attention_weight_min": -26.974929809570312,
      "activations/layer17_attention_weight_max": 28.891265869140625,
      "activations/layer17_attention_weight_min": -23.746728897094727,
      "activations/layer18_attention_weight_max": 28.48356819152832,
      "activations/layer18_attention_weight_min": -22.952194213867188,
      "activations/layer19_attention_weight_max": 30.06215476989746,
      "activations/layer19_attention_weight_min": -27.93817138671875,
      "activations/layer1_attention_weight_max": 16.703174591064453,
      "activations/layer1_attention_weight_min": -15.733860969543457,
      "activations/layer20_attention_weight_max": 25.97771644592285,
      "activations/layer20_attention_weight_min": -23.868364334106445,
      "activations/layer21_attention_weight_max": 29.052047729492188,
      "activations/layer21_attention_weight_min": -25.068435668945312,
      "activations/layer22_attention_weight_max": 35.6051139831543,
      "activations/layer22_attention_weight_min": -29.347412109375,
      "activations/layer23_attention_weight_max": 27.945205688476562,
      "activations/layer23_attention_weight_min": -23.61203384399414,
      "activations/layer2_attention_weight_max": 28.92180633544922,
      "activations/layer2_attention_weight_min": -28.892080307006836,
      "activations/layer3_attention_weight_max": 86.21824645996094,
      "activations/layer3_attention_weight_min": -89.17265319824219,
      "activations/layer4_attention_weight_max": 105.45714569091797,
      "activations/layer4_attention_weight_min": -104.40596008300781,
      "activations/layer5_attention_weight_max": 50.88469696044922,
      "activations/layer5_attention_weight_min": -59.259239196777344,
      "activations/layer6_attention_weight_max": 44.303184509277344,
      "activations/layer6_attention_weight_min": -45.96929168701172,
      "activations/layer7_attention_weight_max": 94.44364929199219,
      "activations/layer7_attention_weight_min": -85.81175231933594,
      "activations/layer8_attention_weight_max": 38.76730728149414,
      "activations/layer8_attention_weight_min": -39.92258071899414,
      "activations/layer9_attention_weight_max": 31.20878028869629,
      "activations/layer9_attention_weight_min": -31.119516372680664,
      "epoch": 12.55,
      "learning_rate": 6.980378787878787e-05,
      "loss": 2.7792,
      "step": 215900
    },
    {
      "activations/layer0_attention_weight_max": 14.955410957336426,
      "activations/layer0_attention_weight_min": -12.303629875183105,
      "activations/layer10_attention_weight_max": 37.20362091064453,
      "activations/layer10_attention_weight_min": -39.599300384521484,
      "activations/layer11_attention_weight_max": 34.547828674316406,
      "activations/layer11_attention_weight_min": -35.80684280395508,
      "activations/layer12_attention_weight_max": 25.737810134887695,
      "activations/layer12_attention_weight_min": -26.99042510986328,
      "activations/layer13_attention_weight_max": 41.310768127441406,
      "activations/layer13_attention_weight_min": -36.01774215698242,
      "activations/layer14_attention_weight_max": 46.13513946533203,
      "activations/layer14_attention_weight_min": -39.2050895690918,
      "activations/layer15_attention_weight_max": 39.26458740234375,
      "activations/layer15_attention_weight_min": -35.82974624633789,
      "activations/layer16_attention_weight_max": 33.31732177734375,
      "activations/layer16_attention_weight_min": -29.188579559326172,
      "activations/layer17_attention_weight_max": 32.740299224853516,
      "activations/layer17_attention_weight_min": -27.642478942871094,
      "activations/layer18_attention_weight_max": 36.1840934753418,
      "activations/layer18_attention_weight_min": -26.405200958251953,
      "activations/layer19_attention_weight_max": 43.12713623046875,
      "activations/layer19_attention_weight_min": -33.568878173828125,
      "activations/layer1_attention_weight_max": 16.6306209564209,
      "activations/layer1_attention_weight_min": -15.750596046447754,
      "activations/layer20_attention_weight_max": 32.82561492919922,
      "activations/layer20_attention_weight_min": -27.488874435424805,
      "activations/layer21_attention_weight_max": 31.759151458740234,
      "activations/layer21_attention_weight_min": -27.37852668762207,
      "activations/layer22_attention_weight_max": 51.59830093383789,
      "activations/layer22_attention_weight_min": -33.54991912841797,
      "activations/layer23_attention_weight_max": 38.63230895996094,
      "activations/layer23_attention_weight_min": -26.63717269897461,
      "activations/layer2_attention_weight_max": 32.75765609741211,
      "activations/layer2_attention_weight_min": -34.26294708251953,
      "activations/layer3_attention_weight_max": 100.34117126464844,
      "activations/layer3_attention_weight_min": -102.88536834716797,
      "activations/layer4_attention_weight_max": 116.6167984008789,
      "activations/layer4_attention_weight_min": -118.44417572021484,
      "activations/layer5_attention_weight_max": 51.03242492675781,
      "activations/layer5_attention_weight_min": -61.15717697143555,
      "activations/layer6_attention_weight_max": 45.84987258911133,
      "activations/layer6_attention_weight_min": -47.32411575317383,
      "activations/layer7_attention_weight_max": 91.5889892578125,
      "activations/layer7_attention_weight_min": -92.94881439208984,
      "activations/layer8_attention_weight_max": 43.14495086669922,
      "activations/layer8_attention_weight_min": -43.871395111083984,
      "activations/layer9_attention_weight_max": 36.834449768066406,
      "activations/layer9_attention_weight_min": -37.9452018737793,
      "epoch": 12.55,
      "learning_rate": 6.978522727272726e-05,
      "loss": 2.7694,
      "step": 215950
    },
    {
      "activations/layer0_attention_weight_max": 14.542631149291992,
      "activations/layer0_attention_weight_min": -12.52943229675293,
      "activations/layer10_attention_weight_max": 37.31654739379883,
      "activations/layer10_attention_weight_min": -33.07630157470703,
      "activations/layer11_attention_weight_max": 38.523414611816406,
      "activations/layer11_attention_weight_min": -33.093971252441406,
      "activations/layer12_attention_weight_max": 32.72815704345703,
      "activations/layer12_attention_weight_min": -25.254268646240234,
      "activations/layer13_attention_weight_max": 38.71967315673828,
      "activations/layer13_attention_weight_min": -34.144622802734375,
      "activations/layer14_attention_weight_max": 45.53342819213867,
      "activations/layer14_attention_weight_min": -37.77705764770508,
      "activations/layer15_attention_weight_max": 38.626319885253906,
      "activations/layer15_attention_weight_min": -34.95943069458008,
      "activations/layer16_attention_weight_max": 29.48128890991211,
      "activations/layer16_attention_weight_min": -28.492984771728516,
      "activations/layer17_attention_weight_max": 30.15248680114746,
      "activations/layer17_attention_weight_min": -26.482389450073242,
      "activations/layer18_attention_weight_max": 29.982421875,
      "activations/layer18_attention_weight_min": -23.233036041259766,
      "activations/layer19_attention_weight_max": 32.10498809814453,
      "activations/layer19_attention_weight_min": -29.8305606842041,
      "activations/layer1_attention_weight_max": 16.21215057373047,
      "activations/layer1_attention_weight_min": -13.881561279296875,
      "activations/layer20_attention_weight_max": 27.43922996520996,
      "activations/layer20_attention_weight_min": -23.988914489746094,
      "activations/layer21_attention_weight_max": 33.74959945678711,
      "activations/layer21_attention_weight_min": -23.540952682495117,
      "activations/layer22_attention_weight_max": 42.0225715637207,
      "activations/layer22_attention_weight_min": -30.507038116455078,
      "activations/layer23_attention_weight_max": 32.06585693359375,
      "activations/layer23_attention_weight_min": -24.360214233398438,
      "activations/layer2_attention_weight_max": 29.338857650756836,
      "activations/layer2_attention_weight_min": -28.86323356628418,
      "activations/layer3_attention_weight_max": 87.83826446533203,
      "activations/layer3_attention_weight_min": -89.40141296386719,
      "activations/layer4_attention_weight_max": 104.9149398803711,
      "activations/layer4_attention_weight_min": -106.55049133300781,
      "activations/layer5_attention_weight_max": 52.91559600830078,
      "activations/layer5_attention_weight_min": -60.33967208862305,
      "activations/layer6_attention_weight_max": 43.22922897338867,
      "activations/layer6_attention_weight_min": -47.66728973388672,
      "activations/layer7_attention_weight_max": 89.56483459472656,
      "activations/layer7_attention_weight_min": -89.8831787109375,
      "activations/layer8_attention_weight_max": 40.5064811706543,
      "activations/layer8_attention_weight_min": -40.224910736083984,
      "activations/layer9_attention_weight_max": 34.54340744018555,
      "activations/layer9_attention_weight_min": -34.45512008666992,
      "epoch": 12.55,
      "learning_rate": 6.976628787878787e-05,
      "loss": 2.7485,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4683,
      "eval_samples_per_second": 507.069,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4683,
      "eval_openwebtext_samples_per_second": 507.069,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9205,
      "eval_wikitext_samples_per_second": 237.435,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_lambada_loss": 2.58203125,
      "eval_lambada_ppl": 13.223972090529857,
      "eval_lambada_runtime": 9.5099,
      "eval_lambada_samples_per_second": 511.992,
      "step": 216000
    },
    {
      "activations/layer0_attention_weight_max": 15.61196231842041,
      "activations/layer0_attention_weight_min": -12.033617973327637,
      "activations/layer10_attention_weight_max": 36.468650817871094,
      "activations/layer10_attention_weight_min": -35.15898513793945,
      "activations/layer11_attention_weight_max": 35.254676818847656,
      "activations/layer11_attention_weight_min": -35.070289611816406,
      "activations/layer12_attention_weight_max": 28.32318687438965,
      "activations/layer12_attention_weight_min": -26.62619400024414,
      "activations/layer13_attention_weight_max": 41.00333786010742,
      "activations/layer13_attention_weight_min": -35.283233642578125,
      "activations/layer14_attention_weight_max": 43.02143096923828,
      "activations/layer14_attention_weight_min": -38.360652923583984,
      "activations/layer15_attention_weight_max": 39.90863037109375,
      "activations/layer15_attention_weight_min": -33.58441925048828,
      "activations/layer16_attention_weight_max": 30.732372283935547,
      "activations/layer16_attention_weight_min": -29.960098266601562,
      "activations/layer17_attention_weight_max": 32.07658767700195,
      "activations/layer17_attention_weight_min": -26.183279037475586,
      "activations/layer18_attention_weight_max": 29.074115753173828,
      "activations/layer18_attention_weight_min": -24.498144149780273,
      "activations/layer19_attention_weight_max": 35.27762222290039,
      "activations/layer19_attention_weight_min": -31.39188575744629,
      "activations/layer1_attention_weight_max": 18.176546096801758,
      "activations/layer1_attention_weight_min": -14.031338691711426,
      "activations/layer20_attention_weight_max": 30.799766540527344,
      "activations/layer20_attention_weight_min": -24.794843673706055,
      "activations/layer21_attention_weight_max": 26.81035614013672,
      "activations/layer21_attention_weight_min": -24.73616600036621,
      "activations/layer22_attention_weight_max": 43.72420883178711,
      "activations/layer22_attention_weight_min": -30.656641006469727,
      "activations/layer23_attention_weight_max": 34.69329833984375,
      "activations/layer23_attention_weight_min": -27.484424591064453,
      "activations/layer2_attention_weight_max": 30.85150718688965,
      "activations/layer2_attention_weight_min": -32.99250030517578,
      "activations/layer3_attention_weight_max": 89.11852264404297,
      "activations/layer3_attention_weight_min": -96.04241943359375,
      "activations/layer4_attention_weight_max": 106.2008056640625,
      "activations/layer4_attention_weight_min": -114.51216125488281,
      "activations/layer5_attention_weight_max": 49.8388786315918,
      "activations/layer5_attention_weight_min": -62.307918548583984,
      "activations/layer6_attention_weight_max": 44.584651947021484,
      "activations/layer6_attention_weight_min": -49.048011779785156,
      "activations/layer7_attention_weight_max": 92.87202453613281,
      "activations/layer7_attention_weight_min": -93.08431243896484,
      "activations/layer8_attention_weight_max": 42.639644622802734,
      "activations/layer8_attention_weight_min": -42.05658721923828,
      "activations/layer9_attention_weight_max": 38.08648681640625,
      "activations/layer9_attention_weight_min": -36.023624420166016,
      "epoch": 12.55,
      "learning_rate": 6.974734848484848e-05,
      "loss": 2.7746,
      "step": 216050
    },
    {
      "activations/layer0_attention_weight_max": 15.888218879699707,
      "activations/layer0_attention_weight_min": -12.172684669494629,
      "activations/layer10_attention_weight_max": 34.08412551879883,
      "activations/layer10_attention_weight_min": -33.17818069458008,
      "activations/layer11_attention_weight_max": 32.570709228515625,
      "activations/layer11_attention_weight_min": -33.47777557373047,
      "activations/layer12_attention_weight_max": 25.286760330200195,
      "activations/layer12_attention_weight_min": -25.092971801757812,
      "activations/layer13_attention_weight_max": 38.28303146362305,
      "activations/layer13_attention_weight_min": -35.20823287963867,
      "activations/layer14_attention_weight_max": 43.500953674316406,
      "activations/layer14_attention_weight_min": -39.05464553833008,
      "activations/layer15_attention_weight_max": 36.52207946777344,
      "activations/layer15_attention_weight_min": -34.39448547363281,
      "activations/layer16_attention_weight_max": 32.38322067260742,
      "activations/layer16_attention_weight_min": -27.123641967773438,
      "activations/layer17_attention_weight_max": 30.112606048583984,
      "activations/layer17_attention_weight_min": -25.475177764892578,
      "activations/layer18_attention_weight_max": 29.608983993530273,
      "activations/layer18_attention_weight_min": -23.1748046875,
      "activations/layer19_attention_weight_max": 33.305320739746094,
      "activations/layer19_attention_weight_min": -29.99472427368164,
      "activations/layer1_attention_weight_max": 16.31475067138672,
      "activations/layer1_attention_weight_min": -16.20806312561035,
      "activations/layer20_attention_weight_max": 30.540746688842773,
      "activations/layer20_attention_weight_min": -27.122318267822266,
      "activations/layer21_attention_weight_max": 28.32608985900879,
      "activations/layer21_attention_weight_min": -24.241539001464844,
      "activations/layer22_attention_weight_max": 39.47733688354492,
      "activations/layer22_attention_weight_min": -28.845829010009766,
      "activations/layer23_attention_weight_max": 30.693349838256836,
      "activations/layer23_attention_weight_min": -26.760906219482422,
      "activations/layer2_attention_weight_max": 32.227455139160156,
      "activations/layer2_attention_weight_min": -34.47735595703125,
      "activations/layer3_attention_weight_max": 94.17071533203125,
      "activations/layer3_attention_weight_min": -92.70097351074219,
      "activations/layer4_attention_weight_max": 112.4873275756836,
      "activations/layer4_attention_weight_min": -112.93377685546875,
      "activations/layer5_attention_weight_max": 50.270050048828125,
      "activations/layer5_attention_weight_min": -64.40453338623047,
      "activations/layer6_attention_weight_max": 45.704185485839844,
      "activations/layer6_attention_weight_min": -51.74131774902344,
      "activations/layer7_attention_weight_max": 91.74622344970703,
      "activations/layer7_attention_weight_min": -97.4800796508789,
      "activations/layer8_attention_weight_max": 39.93024826049805,
      "activations/layer8_attention_weight_min": -42.571475982666016,
      "activations/layer9_attention_weight_max": 32.70179748535156,
      "activations/layer9_attention_weight_min": -35.11921691894531,
      "epoch": 12.56,
      "learning_rate": 6.972840909090909e-05,
      "loss": 2.7597,
      "step": 216100
    },
    {
      "activations/layer0_attention_weight_max": 15.542625427246094,
      "activations/layer0_attention_weight_min": -12.47486400604248,
      "activations/layer10_attention_weight_max": 34.13652420043945,
      "activations/layer10_attention_weight_min": -31.3197078704834,
      "activations/layer11_attention_weight_max": 32.57292556762695,
      "activations/layer11_attention_weight_min": -31.949459075927734,
      "activations/layer12_attention_weight_max": 25.08546257019043,
      "activations/layer12_attention_weight_min": -25.115806579589844,
      "activations/layer13_attention_weight_max": 38.119361877441406,
      "activations/layer13_attention_weight_min": -33.044464111328125,
      "activations/layer14_attention_weight_max": 42.361698150634766,
      "activations/layer14_attention_weight_min": -36.504852294921875,
      "activations/layer15_attention_weight_max": 38.471885681152344,
      "activations/layer15_attention_weight_min": -34.02486801147461,
      "activations/layer16_attention_weight_max": 30.56614875793457,
      "activations/layer16_attention_weight_min": -28.2995662689209,
      "activations/layer17_attention_weight_max": 33.028751373291016,
      "activations/layer17_attention_weight_min": -26.3183536529541,
      "activations/layer18_attention_weight_max": 35.03651428222656,
      "activations/layer18_attention_weight_min": -24.639991760253906,
      "activations/layer19_attention_weight_max": 34.67739486694336,
      "activations/layer19_attention_weight_min": -31.2243709564209,
      "activations/layer1_attention_weight_max": 16.743234634399414,
      "activations/layer1_attention_weight_min": -14.371152877807617,
      "activations/layer20_attention_weight_max": 29.796857833862305,
      "activations/layer20_attention_weight_min": -25.683238983154297,
      "activations/layer21_attention_weight_max": 29.378368377685547,
      "activations/layer21_attention_weight_min": -24.94071388244629,
      "activations/layer22_attention_weight_max": 45.520694732666016,
      "activations/layer22_attention_weight_min": -29.61357879638672,
      "activations/layer23_attention_weight_max": 33.642417907714844,
      "activations/layer23_attention_weight_min": -25.642993927001953,
      "activations/layer2_attention_weight_max": 33.514892578125,
      "activations/layer2_attention_weight_min": -31.087202072143555,
      "activations/layer3_attention_weight_max": 91.37515258789062,
      "activations/layer3_attention_weight_min": -95.06137084960938,
      "activations/layer4_attention_weight_max": 109.931884765625,
      "activations/layer4_attention_weight_min": -110.48734283447266,
      "activations/layer5_attention_weight_max": 52.39594650268555,
      "activations/layer5_attention_weight_min": -64.87081909179688,
      "activations/layer6_attention_weight_max": 46.21760559082031,
      "activations/layer6_attention_weight_min": -49.80256271362305,
      "activations/layer7_attention_weight_max": 89.87936401367188,
      "activations/layer7_attention_weight_min": -90.79244995117188,
      "activations/layer8_attention_weight_max": 40.373348236083984,
      "activations/layer8_attention_weight_min": -40.02460861206055,
      "activations/layer9_attention_weight_max": 32.204246520996094,
      "activations/layer9_attention_weight_min": -33.03718185424805,
      "epoch": 12.56,
      "learning_rate": 6.970946969696969e-05,
      "loss": 2.7675,
      "step": 216150
    },
    {
      "activations/layer0_attention_weight_max": 14.956494331359863,
      "activations/layer0_attention_weight_min": -11.978919982910156,
      "activations/layer10_attention_weight_max": 31.76396942138672,
      "activations/layer10_attention_weight_min": -33.00095748901367,
      "activations/layer11_attention_weight_max": 31.244375228881836,
      "activations/layer11_attention_weight_min": -31.63827896118164,
      "activations/layer12_attention_weight_max": 22.768095016479492,
      "activations/layer12_attention_weight_min": -24.204191207885742,
      "activations/layer13_attention_weight_max": 36.050453186035156,
      "activations/layer13_attention_weight_min": -32.50048828125,
      "activations/layer14_attention_weight_max": 39.18624496459961,
      "activations/layer14_attention_weight_min": -35.5322151184082,
      "activations/layer15_attention_weight_max": 35.5051155090332,
      "activations/layer15_attention_weight_min": -33.47496032714844,
      "activations/layer16_attention_weight_max": 28.87895393371582,
      "activations/layer16_attention_weight_min": -27.413394927978516,
      "activations/layer17_attention_weight_max": 31.398109436035156,
      "activations/layer17_attention_weight_min": -25.774669647216797,
      "activations/layer18_attention_weight_max": 32.97863006591797,
      "activations/layer18_attention_weight_min": -25.78040885925293,
      "activations/layer19_attention_weight_max": 32.48204803466797,
      "activations/layer19_attention_weight_min": -31.749832153320312,
      "activations/layer1_attention_weight_max": 16.201072692871094,
      "activations/layer1_attention_weight_min": -14.133569717407227,
      "activations/layer20_attention_weight_max": 30.060075759887695,
      "activations/layer20_attention_weight_min": -25.674541473388672,
      "activations/layer21_attention_weight_max": 30.601591110229492,
      "activations/layer21_attention_weight_min": -25.9864501953125,
      "activations/layer22_attention_weight_max": 46.72576141357422,
      "activations/layer22_attention_weight_min": -34.60126495361328,
      "activations/layer23_attention_weight_max": 35.02901077270508,
      "activations/layer23_attention_weight_min": -26.170696258544922,
      "activations/layer2_attention_weight_max": 31.93031120300293,
      "activations/layer2_attention_weight_min": -32.40290451049805,
      "activations/layer3_attention_weight_max": 92.42346954345703,
      "activations/layer3_attention_weight_min": -95.8154296875,
      "activations/layer4_attention_weight_max": 111.4435806274414,
      "activations/layer4_attention_weight_min": -114.85332489013672,
      "activations/layer5_attention_weight_max": 50.777278900146484,
      "activations/layer5_attention_weight_min": -63.86750793457031,
      "activations/layer6_attention_weight_max": 43.53559112548828,
      "activations/layer6_attention_weight_min": -50.45917510986328,
      "activations/layer7_attention_weight_max": 89.23162841796875,
      "activations/layer7_attention_weight_min": -93.75112915039062,
      "activations/layer8_attention_weight_max": 40.390716552734375,
      "activations/layer8_attention_weight_min": -40.790279388427734,
      "activations/layer9_attention_weight_max": 31.106897354125977,
      "activations/layer9_attention_weight_min": -32.96878433227539,
      "epoch": 12.56,
      "learning_rate": 6.96905303030303e-05,
      "loss": 2.7818,
      "step": 216200
    },
    {
      "activations/layer0_attention_weight_max": 15.486045837402344,
      "activations/layer0_attention_weight_min": -13.20164680480957,
      "activations/layer10_attention_weight_max": 33.32381057739258,
      "activations/layer10_attention_weight_min": -33.058223724365234,
      "activations/layer11_attention_weight_max": 34.60289764404297,
      "activations/layer11_attention_weight_min": -31.302410125732422,
      "activations/layer12_attention_weight_max": 23.070449829101562,
      "activations/layer12_attention_weight_min": -24.256669998168945,
      "activations/layer13_attention_weight_max": 35.21568298339844,
      "activations/layer13_attention_weight_min": -33.13026428222656,
      "activations/layer14_attention_weight_max": 40.08867263793945,
      "activations/layer14_attention_weight_min": -38.54826354980469,
      "activations/layer15_attention_weight_max": 34.961299896240234,
      "activations/layer15_attention_weight_min": -32.326751708984375,
      "activations/layer16_attention_weight_max": 28.631608963012695,
      "activations/layer16_attention_weight_min": -28.026371002197266,
      "activations/layer17_attention_weight_max": 28.517107009887695,
      "activations/layer17_attention_weight_min": -24.799915313720703,
      "activations/layer18_attention_weight_max": 28.243144989013672,
      "activations/layer18_attention_weight_min": -23.432226181030273,
      "activations/layer19_attention_weight_max": 33.40339279174805,
      "activations/layer19_attention_weight_min": -29.812055587768555,
      "activations/layer1_attention_weight_max": 16.99622917175293,
      "activations/layer1_attention_weight_min": -14.23222541809082,
      "activations/layer20_attention_weight_max": 28.37435531616211,
      "activations/layer20_attention_weight_min": -26.212646484375,
      "activations/layer21_attention_weight_max": 25.623022079467773,
      "activations/layer21_attention_weight_min": -23.848846435546875,
      "activations/layer22_attention_weight_max": 42.872314453125,
      "activations/layer22_attention_weight_min": -29.63138771057129,
      "activations/layer23_attention_weight_max": 30.908729553222656,
      "activations/layer23_attention_weight_min": -24.36695671081543,
      "activations/layer2_attention_weight_max": 31.45223045349121,
      "activations/layer2_attention_weight_min": -32.016910552978516,
      "activations/layer3_attention_weight_max": 93.23651885986328,
      "activations/layer3_attention_weight_min": -98.4569091796875,
      "activations/layer4_attention_weight_max": 111.66609191894531,
      "activations/layer4_attention_weight_min": -114.228759765625,
      "activations/layer5_attention_weight_max": 50.87775802612305,
      "activations/layer5_attention_weight_min": -61.234291076660156,
      "activations/layer6_attention_weight_max": 44.61756134033203,
      "activations/layer6_attention_weight_min": -49.517147064208984,
      "activations/layer7_attention_weight_max": 94.58216857910156,
      "activations/layer7_attention_weight_min": -90.45722198486328,
      "activations/layer8_attention_weight_max": 40.76980209350586,
      "activations/layer8_attention_weight_min": -42.00416564941406,
      "activations/layer9_attention_weight_max": 35.43830108642578,
      "activations/layer9_attention_weight_min": -35.08773422241211,
      "epoch": 12.57,
      "learning_rate": 6.967159090909091e-05,
      "loss": 2.7697,
      "step": 216250
    },
    {
      "activations/layer0_attention_weight_max": 14.706766128540039,
      "activations/layer0_attention_weight_min": -10.99090576171875,
      "activations/layer10_attention_weight_max": 33.948516845703125,
      "activations/layer10_attention_weight_min": -32.59553527832031,
      "activations/layer11_attention_weight_max": 33.628318786621094,
      "activations/layer11_attention_weight_min": -31.837783813476562,
      "activations/layer12_attention_weight_max": 26.897642135620117,
      "activations/layer12_attention_weight_min": -26.423051834106445,
      "activations/layer13_attention_weight_max": 38.99677276611328,
      "activations/layer13_attention_weight_min": -34.457237243652344,
      "activations/layer14_attention_weight_max": 43.63972473144531,
      "activations/layer14_attention_weight_min": -41.32762145996094,
      "activations/layer15_attention_weight_max": 34.64326858520508,
      "activations/layer15_attention_weight_min": -34.715553283691406,
      "activations/layer16_attention_weight_max": 31.174776077270508,
      "activations/layer16_attention_weight_min": -27.626110076904297,
      "activations/layer17_attention_weight_max": 32.15686798095703,
      "activations/layer17_attention_weight_min": -25.958370208740234,
      "activations/layer18_attention_weight_max": 30.26127815246582,
      "activations/layer18_attention_weight_min": -23.199758529663086,
      "activations/layer19_attention_weight_max": 34.01264953613281,
      "activations/layer19_attention_weight_min": -31.878202438354492,
      "activations/layer1_attention_weight_max": 16.884233474731445,
      "activations/layer1_attention_weight_min": -15.024296760559082,
      "activations/layer20_attention_weight_max": 28.771055221557617,
      "activations/layer20_attention_weight_min": -25.857646942138672,
      "activations/layer21_attention_weight_max": 29.041790008544922,
      "activations/layer21_attention_weight_min": -24.63213539123535,
      "activations/layer22_attention_weight_max": 42.6356086730957,
      "activations/layer22_attention_weight_min": -30.40522575378418,
      "activations/layer23_attention_weight_max": 33.47004318237305,
      "activations/layer23_attention_weight_min": -24.31622886657715,
      "activations/layer2_attention_weight_max": 31.61880111694336,
      "activations/layer2_attention_weight_min": -29.830875396728516,
      "activations/layer3_attention_weight_max": 88.23406219482422,
      "activations/layer3_attention_weight_min": -90.19664764404297,
      "activations/layer4_attention_weight_max": 107.84541320800781,
      "activations/layer4_attention_weight_min": -106.29756164550781,
      "activations/layer5_attention_weight_max": 50.3740234375,
      "activations/layer5_attention_weight_min": -62.58419418334961,
      "activations/layer6_attention_weight_max": 43.473514556884766,
      "activations/layer6_attention_weight_min": -46.91056442260742,
      "activations/layer7_attention_weight_max": 91.47029113769531,
      "activations/layer7_attention_weight_min": -91.3008804321289,
      "activations/layer8_attention_weight_max": 43.345645904541016,
      "activations/layer8_attention_weight_min": -41.0969123840332,
      "activations/layer9_attention_weight_max": 34.61725616455078,
      "activations/layer9_attention_weight_min": -33.37894821166992,
      "epoch": 12.57,
      "learning_rate": 6.965265151515151e-05,
      "loss": 2.774,
      "step": 216300
    },
    {
      "activations/layer0_attention_weight_max": 16.027278900146484,
      "activations/layer0_attention_weight_min": -11.972576141357422,
      "activations/layer10_attention_weight_max": 48.78626251220703,
      "activations/layer10_attention_weight_min": -48.54337692260742,
      "activations/layer11_attention_weight_max": 49.264095306396484,
      "activations/layer11_attention_weight_min": -46.28681945800781,
      "activations/layer12_attention_weight_max": 35.39988327026367,
      "activations/layer12_attention_weight_min": -29.8952693939209,
      "activations/layer13_attention_weight_max": 54.3765754699707,
      "activations/layer13_attention_weight_min": -41.66402816772461,
      "activations/layer14_attention_weight_max": 70.8698959350586,
      "activations/layer14_attention_weight_min": -51.01786422729492,
      "activations/layer15_attention_weight_max": 58.29526138305664,
      "activations/layer15_attention_weight_min": -44.6829719543457,
      "activations/layer16_attention_weight_max": 46.95966339111328,
      "activations/layer16_attention_weight_min": -33.139366149902344,
      "activations/layer17_attention_weight_max": 42.07073211669922,
      "activations/layer17_attention_weight_min": -32.290218353271484,
      "activations/layer18_attention_weight_max": 36.40061569213867,
      "activations/layer18_attention_weight_min": -25.213640213012695,
      "activations/layer19_attention_weight_max": 39.93419647216797,
      "activations/layer19_attention_weight_min": -31.648324966430664,
      "activations/layer1_attention_weight_max": 17.77805519104004,
      "activations/layer1_attention_weight_min": -14.812944412231445,
      "activations/layer20_attention_weight_max": 33.69398498535156,
      "activations/layer20_attention_weight_min": -27.00301170349121,
      "activations/layer21_attention_weight_max": 33.29450988769531,
      "activations/layer21_attention_weight_min": -26.8289852142334,
      "activations/layer22_attention_weight_max": 61.09597396850586,
      "activations/layer22_attention_weight_min": -32.985965728759766,
      "activations/layer23_attention_weight_max": 42.30061340332031,
      "activations/layer23_attention_weight_min": -26.51068115234375,
      "activations/layer2_attention_weight_max": 35.614952087402344,
      "activations/layer2_attention_weight_min": -33.618446350097656,
      "activations/layer3_attention_weight_max": 98.71106719970703,
      "activations/layer3_attention_weight_min": -100.35697174072266,
      "activations/layer4_attention_weight_max": 120.9108657836914,
      "activations/layer4_attention_weight_min": -116.01885223388672,
      "activations/layer5_attention_weight_max": 56.926979064941406,
      "activations/layer5_attention_weight_min": -60.55005645751953,
      "activations/layer6_attention_weight_max": 51.84025192260742,
      "activations/layer6_attention_weight_min": -51.20774841308594,
      "activations/layer7_attention_weight_max": 115.8576431274414,
      "activations/layer7_attention_weight_min": -112.69297790527344,
      "activations/layer8_attention_weight_max": 53.44727325439453,
      "activations/layer8_attention_weight_min": -55.138587951660156,
      "activations/layer9_attention_weight_max": 48.75349807739258,
      "activations/layer9_attention_weight_min": -47.04324722290039,
      "epoch": 12.57,
      "learning_rate": 6.963371212121211e-05,
      "loss": 2.7697,
      "step": 216350
    },
    {
      "activations/layer0_attention_weight_max": 15.818222045898438,
      "activations/layer0_attention_weight_min": -12.198530197143555,
      "activations/layer10_attention_weight_max": 37.41921615600586,
      "activations/layer10_attention_weight_min": -35.48379898071289,
      "activations/layer11_attention_weight_max": 37.26207733154297,
      "activations/layer11_attention_weight_min": -35.17679214477539,
      "activations/layer12_attention_weight_max": 24.47959327697754,
      "activations/layer12_attention_weight_min": -30.442089080810547,
      "activations/layer13_attention_weight_max": 38.179931640625,
      "activations/layer13_attention_weight_min": -34.04757308959961,
      "activations/layer14_attention_weight_max": 42.17966842651367,
      "activations/layer14_attention_weight_min": -38.85686111450195,
      "activations/layer15_attention_weight_max": 38.73443603515625,
      "activations/layer15_attention_weight_min": -33.55561065673828,
      "activations/layer16_attention_weight_max": 28.426883697509766,
      "activations/layer16_attention_weight_min": -27.112333297729492,
      "activations/layer17_attention_weight_max": 29.45071792602539,
      "activations/layer17_attention_weight_min": -26.407289505004883,
      "activations/layer18_attention_weight_max": 30.875263214111328,
      "activations/layer18_attention_weight_min": -23.30894660949707,
      "activations/layer19_attention_weight_max": 33.83302307128906,
      "activations/layer19_attention_weight_min": -33.21977615356445,
      "activations/layer1_attention_weight_max": 16.257308959960938,
      "activations/layer1_attention_weight_min": -13.600764274597168,
      "activations/layer20_attention_weight_max": 31.531259536743164,
      "activations/layer20_attention_weight_min": -26.63080406188965,
      "activations/layer21_attention_weight_max": 30.60807991027832,
      "activations/layer21_attention_weight_min": -25.42893409729004,
      "activations/layer22_attention_weight_max": 44.43952178955078,
      "activations/layer22_attention_weight_min": -30.1950740814209,
      "activations/layer23_attention_weight_max": 33.32567596435547,
      "activations/layer23_attention_weight_min": -24.43460464477539,
      "activations/layer2_attention_weight_max": 30.632139205932617,
      "activations/layer2_attention_weight_min": -30.51624298095703,
      "activations/layer3_attention_weight_max": 89.27751922607422,
      "activations/layer3_attention_weight_min": -93.4625015258789,
      "activations/layer4_attention_weight_max": 108.95455169677734,
      "activations/layer4_attention_weight_min": -110.14714813232422,
      "activations/layer5_attention_weight_max": 54.07130432128906,
      "activations/layer5_attention_weight_min": -62.44501495361328,
      "activations/layer6_attention_weight_max": 43.12336730957031,
      "activations/layer6_attention_weight_min": -50.10681915283203,
      "activations/layer7_attention_weight_max": 92.6218490600586,
      "activations/layer7_attention_weight_min": -97.1333236694336,
      "activations/layer8_attention_weight_max": 43.212486267089844,
      "activations/layer8_attention_weight_min": -42.003318786621094,
      "activations/layer9_attention_weight_max": 33.1684684753418,
      "activations/layer9_attention_weight_min": -34.801918029785156,
      "epoch": 12.57,
      "learning_rate": 6.961477272727273e-05,
      "loss": 2.7732,
      "step": 216400
    },
    {
      "activations/layer0_attention_weight_max": 15.788749694824219,
      "activations/layer0_attention_weight_min": -13.023602485656738,
      "activations/layer10_attention_weight_max": 31.843746185302734,
      "activations/layer10_attention_weight_min": -31.315874099731445,
      "activations/layer11_attention_weight_max": 33.45391082763672,
      "activations/layer11_attention_weight_min": -33.52263641357422,
      "activations/layer12_attention_weight_max": 27.4995059967041,
      "activations/layer12_attention_weight_min": -26.315982818603516,
      "activations/layer13_attention_weight_max": 41.21875,
      "activations/layer13_attention_weight_min": -34.63143539428711,
      "activations/layer14_attention_weight_max": 42.73621368408203,
      "activations/layer14_attention_weight_min": -39.388954162597656,
      "activations/layer15_attention_weight_max": 37.61417007446289,
      "activations/layer15_attention_weight_min": -34.920738220214844,
      "activations/layer16_attention_weight_max": 30.941144943237305,
      "activations/layer16_attention_weight_min": -28.24894142150879,
      "activations/layer17_attention_weight_max": 33.07581329345703,
      "activations/layer17_attention_weight_min": -26.610179901123047,
      "activations/layer18_attention_weight_max": 31.19936752319336,
      "activations/layer18_attention_weight_min": -23.485164642333984,
      "activations/layer19_attention_weight_max": 36.026397705078125,
      "activations/layer19_attention_weight_min": -31.06170654296875,
      "activations/layer1_attention_weight_max": 18.074256896972656,
      "activations/layer1_attention_weight_min": -18.921287536621094,
      "activations/layer20_attention_weight_max": 31.767343521118164,
      "activations/layer20_attention_weight_min": -25.37455940246582,
      "activations/layer21_attention_weight_max": 28.508861541748047,
      "activations/layer21_attention_weight_min": -24.164413452148438,
      "activations/layer22_attention_weight_max": 46.9466552734375,
      "activations/layer22_attention_weight_min": -31.210487365722656,
      "activations/layer23_attention_weight_max": 36.22565460205078,
      "activations/layer23_attention_weight_min": -26.8100643157959,
      "activations/layer2_attention_weight_max": 31.645841598510742,
      "activations/layer2_attention_weight_min": -29.616153717041016,
      "activations/layer3_attention_weight_max": 87.9184341430664,
      "activations/layer3_attention_weight_min": -88.3430404663086,
      "activations/layer4_attention_weight_max": 106.33541107177734,
      "activations/layer4_attention_weight_min": -107.85597229003906,
      "activations/layer5_attention_weight_max": 51.99315643310547,
      "activations/layer5_attention_weight_min": -60.599342346191406,
      "activations/layer6_attention_weight_max": 44.13608932495117,
      "activations/layer6_attention_weight_min": -47.37815856933594,
      "activations/layer7_attention_weight_max": 91.40484619140625,
      "activations/layer7_attention_weight_min": -87.0057373046875,
      "activations/layer8_attention_weight_max": 39.20893096923828,
      "activations/layer8_attention_weight_min": -40.64562225341797,
      "activations/layer9_attention_weight_max": 31.34315299987793,
      "activations/layer9_attention_weight_min": -33.32737350463867,
      "epoch": 12.58,
      "learning_rate": 6.959583333333333e-05,
      "loss": 2.7804,
      "step": 216450
    },
    {
      "activations/layer0_attention_weight_max": 15.86312198638916,
      "activations/layer0_attention_weight_min": -11.783493995666504,
      "activations/layer10_attention_weight_max": 30.643789291381836,
      "activations/layer10_attention_weight_min": -30.7486572265625,
      "activations/layer11_attention_weight_max": 29.514530181884766,
      "activations/layer11_attention_weight_min": -29.727981567382812,
      "activations/layer12_attention_weight_max": 25.13314437866211,
      "activations/layer12_attention_weight_min": -25.632776260375977,
      "activations/layer13_attention_weight_max": 36.56281661987305,
      "activations/layer13_attention_weight_min": -31.413658142089844,
      "activations/layer14_attention_weight_max": 41.484375,
      "activations/layer14_attention_weight_min": -35.132930755615234,
      "activations/layer15_attention_weight_max": 34.79652786254883,
      "activations/layer15_attention_weight_min": -31.841211318969727,
      "activations/layer16_attention_weight_max": 29.835163116455078,
      "activations/layer16_attention_weight_min": -28.779783248901367,
      "activations/layer17_attention_weight_max": 30.2376708984375,
      "activations/layer17_attention_weight_min": -25.587308883666992,
      "activations/layer18_attention_weight_max": 31.203941345214844,
      "activations/layer18_attention_weight_min": -23.729232788085938,
      "activations/layer19_attention_weight_max": 34.12559127807617,
      "activations/layer19_attention_weight_min": -31.438520431518555,
      "activations/layer1_attention_weight_max": 16.882383346557617,
      "activations/layer1_attention_weight_min": -14.38918685913086,
      "activations/layer20_attention_weight_max": 31.17611312866211,
      "activations/layer20_attention_weight_min": -25.936302185058594,
      "activations/layer21_attention_weight_max": 31.802352905273438,
      "activations/layer21_attention_weight_min": -25.061279296875,
      "activations/layer22_attention_weight_max": 43.5067253112793,
      "activations/layer22_attention_weight_min": -30.476980209350586,
      "activations/layer23_attention_weight_max": 38.1992073059082,
      "activations/layer23_attention_weight_min": -26.53107452392578,
      "activations/layer2_attention_weight_max": 30.730154037475586,
      "activations/layer2_attention_weight_min": -31.000354766845703,
      "activations/layer3_attention_weight_max": 93.57889556884766,
      "activations/layer3_attention_weight_min": -93.14724731445312,
      "activations/layer4_attention_weight_max": 114.2016372680664,
      "activations/layer4_attention_weight_min": -115.89153289794922,
      "activations/layer5_attention_weight_max": 51.94208526611328,
      "activations/layer5_attention_weight_min": -61.827110290527344,
      "activations/layer6_attention_weight_max": 44.96088409423828,
      "activations/layer6_attention_weight_min": -46.204429626464844,
      "activations/layer7_attention_weight_max": 88.13294982910156,
      "activations/layer7_attention_weight_min": -82.36326599121094,
      "activations/layer8_attention_weight_max": 39.55504608154297,
      "activations/layer8_attention_weight_min": -40.81149673461914,
      "activations/layer9_attention_weight_max": 33.796173095703125,
      "activations/layer9_attention_weight_min": -31.83707618713379,
      "epoch": 12.58,
      "learning_rate": 6.957689393939393e-05,
      "loss": 2.7779,
      "step": 216500
    },
    {
      "activations/layer0_attention_weight_max": 16.986648559570312,
      "activations/layer0_attention_weight_min": -12.315401077270508,
      "activations/layer10_attention_weight_max": 32.31633377075195,
      "activations/layer10_attention_weight_min": -31.714691162109375,
      "activations/layer11_attention_weight_max": 30.63075828552246,
      "activations/layer11_attention_weight_min": -33.17863082885742,
      "activations/layer12_attention_weight_max": 23.653743743896484,
      "activations/layer12_attention_weight_min": -25.515600204467773,
      "activations/layer13_attention_weight_max": 36.026268005371094,
      "activations/layer13_attention_weight_min": -35.92591857910156,
      "activations/layer14_attention_weight_max": 38.647857666015625,
      "activations/layer14_attention_weight_min": -35.1678466796875,
      "activations/layer15_attention_weight_max": 35.407413482666016,
      "activations/layer15_attention_weight_min": -32.180992126464844,
      "activations/layer16_attention_weight_max": 30.539073944091797,
      "activations/layer16_attention_weight_min": -29.754289627075195,
      "activations/layer17_attention_weight_max": 30.66901969909668,
      "activations/layer17_attention_weight_min": -25.933130264282227,
      "activations/layer18_attention_weight_max": 32.002742767333984,
      "activations/layer18_attention_weight_min": -27.155614852905273,
      "activations/layer19_attention_weight_max": 34.972877502441406,
      "activations/layer19_attention_weight_min": -34.25822067260742,
      "activations/layer1_attention_weight_max": 16.555177688598633,
      "activations/layer1_attention_weight_min": -14.769815444946289,
      "activations/layer20_attention_weight_max": 28.89793586730957,
      "activations/layer20_attention_weight_min": -28.155445098876953,
      "activations/layer21_attention_weight_max": 28.3221492767334,
      "activations/layer21_attention_weight_min": -25.739158630371094,
      "activations/layer22_attention_weight_max": 40.440269470214844,
      "activations/layer22_attention_weight_min": -33.12097930908203,
      "activations/layer23_attention_weight_max": 36.99422836303711,
      "activations/layer23_attention_weight_min": -27.394012451171875,
      "activations/layer2_attention_weight_max": 32.56986618041992,
      "activations/layer2_attention_weight_min": -30.815902709960938,
      "activations/layer3_attention_weight_max": 91.02735137939453,
      "activations/layer3_attention_weight_min": -92.34779357910156,
      "activations/layer4_attention_weight_max": 105.94293212890625,
      "activations/layer4_attention_weight_min": -108.45096588134766,
      "activations/layer5_attention_weight_max": 49.226806640625,
      "activations/layer5_attention_weight_min": -56.83069610595703,
      "activations/layer6_attention_weight_max": 46.041690826416016,
      "activations/layer6_attention_weight_min": -46.73523712158203,
      "activations/layer7_attention_weight_max": 89.5855712890625,
      "activations/layer7_attention_weight_min": -88.97579956054688,
      "activations/layer8_attention_weight_max": 38.56100845336914,
      "activations/layer8_attention_weight_min": -41.93096923828125,
      "activations/layer9_attention_weight_max": 32.1842041015625,
      "activations/layer9_attention_weight_min": -31.557722091674805,
      "epoch": 12.58,
      "learning_rate": 6.955795454545453e-05,
      "loss": 2.7814,
      "step": 216550
    },
    {
      "activations/layer0_attention_weight_max": 15.274933815002441,
      "activations/layer0_attention_weight_min": -11.963281631469727,
      "activations/layer10_attention_weight_max": 32.80963897705078,
      "activations/layer10_attention_weight_min": -33.48080825805664,
      "activations/layer11_attention_weight_max": 34.728546142578125,
      "activations/layer11_attention_weight_min": -33.92820739746094,
      "activations/layer12_attention_weight_max": 34.04808044433594,
      "activations/layer12_attention_weight_min": -27.01963996887207,
      "activations/layer13_attention_weight_max": 42.635589599609375,
      "activations/layer13_attention_weight_min": -37.34835433959961,
      "activations/layer14_attention_weight_max": 44.67363739013672,
      "activations/layer14_attention_weight_min": -42.573028564453125,
      "activations/layer15_attention_weight_max": 37.409271240234375,
      "activations/layer15_attention_weight_min": -37.20131301879883,
      "activations/layer16_attention_weight_max": 31.92076301574707,
      "activations/layer16_attention_weight_min": -29.51548957824707,
      "activations/layer17_attention_weight_max": 31.314945220947266,
      "activations/layer17_attention_weight_min": -28.04583168029785,
      "activations/layer18_attention_weight_max": 30.223726272583008,
      "activations/layer18_attention_weight_min": -24.094623565673828,
      "activations/layer19_attention_weight_max": 33.37787628173828,
      "activations/layer19_attention_weight_min": -34.25758743286133,
      "activations/layer1_attention_weight_max": 15.816963195800781,
      "activations/layer1_attention_weight_min": -14.879809379577637,
      "activations/layer20_attention_weight_max": 29.687808990478516,
      "activations/layer20_attention_weight_min": -28.087080001831055,
      "activations/layer21_attention_weight_max": 27.805694580078125,
      "activations/layer21_attention_weight_min": -26.867034912109375,
      "activations/layer22_attention_weight_max": 41.482215881347656,
      "activations/layer22_attention_weight_min": -33.415565490722656,
      "activations/layer23_attention_weight_max": 33.973182678222656,
      "activations/layer23_attention_weight_min": -26.405948638916016,
      "activations/layer2_attention_weight_max": 30.69244384765625,
      "activations/layer2_attention_weight_min": -30.559688568115234,
      "activations/layer3_attention_weight_max": 89.6982421875,
      "activations/layer3_attention_weight_min": -90.37359619140625,
      "activations/layer4_attention_weight_max": 107.41450500488281,
      "activations/layer4_attention_weight_min": -107.1399917602539,
      "activations/layer5_attention_weight_max": 48.692787170410156,
      "activations/layer5_attention_weight_min": -60.14179992675781,
      "activations/layer6_attention_weight_max": 43.37444305419922,
      "activations/layer6_attention_weight_min": -48.10264587402344,
      "activations/layer7_attention_weight_max": 95.278076171875,
      "activations/layer7_attention_weight_min": -91.43798828125,
      "activations/layer8_attention_weight_max": 42.043949127197266,
      "activations/layer8_attention_weight_min": -44.12828826904297,
      "activations/layer9_attention_weight_max": 34.225502014160156,
      "activations/layer9_attention_weight_min": -34.411556243896484,
      "epoch": 12.59,
      "learning_rate": 6.953901515151515e-05,
      "loss": 2.758,
      "step": 216600
    },
    {
      "activations/layer0_attention_weight_max": 14.898839950561523,
      "activations/layer0_attention_weight_min": -11.954944610595703,
      "activations/layer10_attention_weight_max": 31.84830093383789,
      "activations/layer10_attention_weight_min": -34.970218658447266,
      "activations/layer11_attention_weight_max": 31.906238555908203,
      "activations/layer11_attention_weight_min": -32.992774963378906,
      "activations/layer12_attention_weight_max": 25.260839462280273,
      "activations/layer12_attention_weight_min": -25.363365173339844,
      "activations/layer13_attention_weight_max": 38.383724212646484,
      "activations/layer13_attention_weight_min": -33.972259521484375,
      "activations/layer14_attention_weight_max": 41.049686431884766,
      "activations/layer14_attention_weight_min": -35.83354187011719,
      "activations/layer15_attention_weight_max": 35.62437057495117,
      "activations/layer15_attention_weight_min": -33.11166763305664,
      "activations/layer16_attention_weight_max": 32.51903533935547,
      "activations/layer16_attention_weight_min": -27.511150360107422,
      "activations/layer17_attention_weight_max": 33.343902587890625,
      "activations/layer17_attention_weight_min": -27.354549407958984,
      "activations/layer18_attention_weight_max": 30.980913162231445,
      "activations/layer18_attention_weight_min": -23.315593719482422,
      "activations/layer19_attention_weight_max": 33.98782730102539,
      "activations/layer19_attention_weight_min": -31.349782943725586,
      "activations/layer1_attention_weight_max": 16.267929077148438,
      "activations/layer1_attention_weight_min": -13.093008041381836,
      "activations/layer20_attention_weight_max": 28.629261016845703,
      "activations/layer20_attention_weight_min": -26.81365966796875,
      "activations/layer21_attention_weight_max": 30.177587509155273,
      "activations/layer21_attention_weight_min": -24.536771774291992,
      "activations/layer22_attention_weight_max": 45.34556198120117,
      "activations/layer22_attention_weight_min": -30.71483612060547,
      "activations/layer23_attention_weight_max": 32.567386627197266,
      "activations/layer23_attention_weight_min": -25.34186553955078,
      "activations/layer2_attention_weight_max": 30.456907272338867,
      "activations/layer2_attention_weight_min": -32.13008499145508,
      "activations/layer3_attention_weight_max": 90.43334197998047,
      "activations/layer3_attention_weight_min": -93.22318267822266,
      "activations/layer4_attention_weight_max": 110.04603576660156,
      "activations/layer4_attention_weight_min": -116.49358367919922,
      "activations/layer5_attention_weight_max": 51.981910705566406,
      "activations/layer5_attention_weight_min": -65.62932586669922,
      "activations/layer6_attention_weight_max": 45.88645553588867,
      "activations/layer6_attention_weight_min": -50.748634338378906,
      "activations/layer7_attention_weight_max": 91.81352996826172,
      "activations/layer7_attention_weight_min": -94.57239532470703,
      "activations/layer8_attention_weight_max": 39.51325607299805,
      "activations/layer8_attention_weight_min": -42.31206130981445,
      "activations/layer9_attention_weight_max": 32.95399856567383,
      "activations/layer9_attention_weight_min": -33.89110565185547,
      "epoch": 12.59,
      "learning_rate": 6.952007575757575e-05,
      "loss": 2.7557,
      "step": 216650
    },
    {
      "activations/layer0_attention_weight_max": 15.092101097106934,
      "activations/layer0_attention_weight_min": -12.185254096984863,
      "activations/layer10_attention_weight_max": 34.98648452758789,
      "activations/layer10_attention_weight_min": -32.04231643676758,
      "activations/layer11_attention_weight_max": 34.11853790283203,
      "activations/layer11_attention_weight_min": -31.20989990234375,
      "activations/layer12_attention_weight_max": 27.19647979736328,
      "activations/layer12_attention_weight_min": -26.081945419311523,
      "activations/layer13_attention_weight_max": 39.09474182128906,
      "activations/layer13_attention_weight_min": -35.94133377075195,
      "activations/layer14_attention_weight_max": 41.813907623291016,
      "activations/layer14_attention_weight_min": -36.778560638427734,
      "activations/layer15_attention_weight_max": 38.0040397644043,
      "activations/layer15_attention_weight_min": -34.686767578125,
      "activations/layer16_attention_weight_max": 31.330371856689453,
      "activations/layer16_attention_weight_min": -27.87692642211914,
      "activations/layer17_attention_weight_max": 34.984798431396484,
      "activations/layer17_attention_weight_min": -26.44854736328125,
      "activations/layer18_attention_weight_max": 31.970684051513672,
      "activations/layer18_attention_weight_min": -24.249710083007812,
      "activations/layer19_attention_weight_max": 34.600318908691406,
      "activations/layer19_attention_weight_min": -30.9212589263916,
      "activations/layer1_attention_weight_max": 17.146827697753906,
      "activations/layer1_attention_weight_min": -14.320199012756348,
      "activations/layer20_attention_weight_max": 28.967748641967773,
      "activations/layer20_attention_weight_min": -24.687171936035156,
      "activations/layer21_attention_weight_max": 28.4630184173584,
      "activations/layer21_attention_weight_min": -23.93080711364746,
      "activations/layer22_attention_weight_max": 43.01602554321289,
      "activations/layer22_attention_weight_min": -27.344635009765625,
      "activations/layer23_attention_weight_max": 34.404361724853516,
      "activations/layer23_attention_weight_min": -24.170499801635742,
      "activations/layer2_attention_weight_max": 29.93153953552246,
      "activations/layer2_attention_weight_min": -29.104084014892578,
      "activations/layer3_attention_weight_max": 88.34906005859375,
      "activations/layer3_attention_weight_min": -88.4136962890625,
      "activations/layer4_attention_weight_max": 107.18086242675781,
      "activations/layer4_attention_weight_min": -107.9488754272461,
      "activations/layer5_attention_weight_max": 51.090362548828125,
      "activations/layer5_attention_weight_min": -60.70532989501953,
      "activations/layer6_attention_weight_max": 44.90208435058594,
      "activations/layer6_attention_weight_min": -48.72985076904297,
      "activations/layer7_attention_weight_max": 90.0650405883789,
      "activations/layer7_attention_weight_min": -89.94525146484375,
      "activations/layer8_attention_weight_max": 42.102752685546875,
      "activations/layer8_attention_weight_min": -42.46562576293945,
      "activations/layer9_attention_weight_max": 34.674686431884766,
      "activations/layer9_attention_weight_min": -33.23550033569336,
      "epoch": 12.59,
      "learning_rate": 6.950113636363635e-05,
      "loss": 2.7855,
      "step": 216700
    },
    {
      "activations/layer0_attention_weight_max": 15.714275360107422,
      "activations/layer0_attention_weight_min": -11.654735565185547,
      "activations/layer10_attention_weight_max": 31.5142879486084,
      "activations/layer10_attention_weight_min": -32.69456100463867,
      "activations/layer11_attention_weight_max": 31.171340942382812,
      "activations/layer11_attention_weight_min": -32.62179183959961,
      "activations/layer12_attention_weight_max": 25.36358070373535,
      "activations/layer12_attention_weight_min": -26.030118942260742,
      "activations/layer13_attention_weight_max": 38.51231002807617,
      "activations/layer13_attention_weight_min": -36.09031677246094,
      "activations/layer14_attention_weight_max": 46.1107063293457,
      "activations/layer14_attention_weight_min": -37.355918884277344,
      "activations/layer15_attention_weight_max": 37.545772552490234,
      "activations/layer15_attention_weight_min": -34.47541046142578,
      "activations/layer16_attention_weight_max": 32.04184341430664,
      "activations/layer16_attention_weight_min": -29.21753692626953,
      "activations/layer17_attention_weight_max": 31.310115814208984,
      "activations/layer17_attention_weight_min": -28.510894775390625,
      "activations/layer18_attention_weight_max": 32.50450897216797,
      "activations/layer18_attention_weight_min": -26.007770538330078,
      "activations/layer19_attention_weight_max": 40.997764587402344,
      "activations/layer19_attention_weight_min": -34.20473098754883,
      "activations/layer1_attention_weight_max": 16.815462112426758,
      "activations/layer1_attention_weight_min": -13.540216445922852,
      "activations/layer20_attention_weight_max": 31.98395347595215,
      "activations/layer20_attention_weight_min": -27.181427001953125,
      "activations/layer21_attention_weight_max": 29.223913192749023,
      "activations/layer21_attention_weight_min": -25.963485717773438,
      "activations/layer22_attention_weight_max": 44.538516998291016,
      "activations/layer22_attention_weight_min": -30.77399253845215,
      "activations/layer23_attention_weight_max": 35.33264923095703,
      "activations/layer23_attention_weight_min": -24.049833297729492,
      "activations/layer2_attention_weight_max": 32.216888427734375,
      "activations/layer2_attention_weight_min": -31.855390548706055,
      "activations/layer3_attention_weight_max": 93.25569915771484,
      "activations/layer3_attention_weight_min": -95.07941436767578,
      "activations/layer4_attention_weight_max": 108.15422821044922,
      "activations/layer4_attention_weight_min": -110.54058074951172,
      "activations/layer5_attention_weight_max": 54.18145751953125,
      "activations/layer5_attention_weight_min": -61.084224700927734,
      "activations/layer6_attention_weight_max": 43.580345153808594,
      "activations/layer6_attention_weight_min": -47.312984466552734,
      "activations/layer7_attention_weight_max": 88.61422729492188,
      "activations/layer7_attention_weight_min": -90.65760803222656,
      "activations/layer8_attention_weight_max": 39.533042907714844,
      "activations/layer8_attention_weight_min": -43.00548553466797,
      "activations/layer9_attention_weight_max": 31.104047775268555,
      "activations/layer9_attention_weight_min": -32.597660064697266,
      "epoch": 12.59,
      "learning_rate": 6.948219696969696e-05,
      "loss": 2.7662,
      "step": 216750
    },
    {
      "activations/layer0_attention_weight_max": 16.653947830200195,
      "activations/layer0_attention_weight_min": -11.35207748413086,
      "activations/layer10_attention_weight_max": 32.92100143432617,
      "activations/layer10_attention_weight_min": -31.974395751953125,
      "activations/layer11_attention_weight_max": 33.08141326904297,
      "activations/layer11_attention_weight_min": -32.1422004699707,
      "activations/layer12_attention_weight_max": 25.716670989990234,
      "activations/layer12_attention_weight_min": -25.322296142578125,
      "activations/layer13_attention_weight_max": 38.84624481201172,
      "activations/layer13_attention_weight_min": -35.79637145996094,
      "activations/layer14_attention_weight_max": 42.63428497314453,
      "activations/layer14_attention_weight_min": -36.32112121582031,
      "activations/layer15_attention_weight_max": 35.62640380859375,
      "activations/layer15_attention_weight_min": -33.46953201293945,
      "activations/layer16_attention_weight_max": 30.884220123291016,
      "activations/layer16_attention_weight_min": -28.05064582824707,
      "activations/layer17_attention_weight_max": 30.133087158203125,
      "activations/layer17_attention_weight_min": -25.749691009521484,
      "activations/layer18_attention_weight_max": 30.22284507751465,
      "activations/layer18_attention_weight_min": -23.037092208862305,
      "activations/layer19_attention_weight_max": 36.06832504272461,
      "activations/layer19_attention_weight_min": -30.235858917236328,
      "activations/layer1_attention_weight_max": 15.778740882873535,
      "activations/layer1_attention_weight_min": -14.83547306060791,
      "activations/layer20_attention_weight_max": 31.839237213134766,
      "activations/layer20_attention_weight_min": -24.521089553833008,
      "activations/layer21_attention_weight_max": 31.437286376953125,
      "activations/layer21_attention_weight_min": -23.184099197387695,
      "activations/layer22_attention_weight_max": 41.21621322631836,
      "activations/layer22_attention_weight_min": -30.107778549194336,
      "activations/layer23_attention_weight_max": 35.44305419921875,
      "activations/layer23_attention_weight_min": -26.783843994140625,
      "activations/layer2_attention_weight_max": 30.2907772064209,
      "activations/layer2_attention_weight_min": -31.64505386352539,
      "activations/layer3_attention_weight_max": 89.43708801269531,
      "activations/layer3_attention_weight_min": -88.92852020263672,
      "activations/layer4_attention_weight_max": 108.48663330078125,
      "activations/layer4_attention_weight_min": -107.87816619873047,
      "activations/layer5_attention_weight_max": 49.595603942871094,
      "activations/layer5_attention_weight_min": -59.60439682006836,
      "activations/layer6_attention_weight_max": 43.87193298339844,
      "activations/layer6_attention_weight_min": -48.41394805908203,
      "activations/layer7_attention_weight_max": 90.09681701660156,
      "activations/layer7_attention_weight_min": -85.48467254638672,
      "activations/layer8_attention_weight_max": 39.331878662109375,
      "activations/layer8_attention_weight_min": -40.58708953857422,
      "activations/layer9_attention_weight_max": 31.32685661315918,
      "activations/layer9_attention_weight_min": -31.961240768432617,
      "epoch": 12.6,
      "learning_rate": 6.946325757575757e-05,
      "loss": 2.7713,
      "step": 216800
    },
    {
      "activations/layer0_attention_weight_max": 15.699109077453613,
      "activations/layer0_attention_weight_min": -12.080957412719727,
      "activations/layer10_attention_weight_max": 32.53034210205078,
      "activations/layer10_attention_weight_min": -32.05584716796875,
      "activations/layer11_attention_weight_max": 32.502471923828125,
      "activations/layer11_attention_weight_min": -31.324031829833984,
      "activations/layer12_attention_weight_max": 30.267738342285156,
      "activations/layer12_attention_weight_min": -25.286535263061523,
      "activations/layer13_attention_weight_max": 46.540443420410156,
      "activations/layer13_attention_weight_min": -36.682273864746094,
      "activations/layer14_attention_weight_max": 53.28725814819336,
      "activations/layer14_attention_weight_min": -38.32725524902344,
      "activations/layer15_attention_weight_max": 45.7354621887207,
      "activations/layer15_attention_weight_min": -32.90316390991211,
      "activations/layer16_attention_weight_max": 36.35348892211914,
      "activations/layer16_attention_weight_min": -26.882888793945312,
      "activations/layer17_attention_weight_max": 37.24921798706055,
      "activations/layer17_attention_weight_min": -25.585771560668945,
      "activations/layer18_attention_weight_max": 33.3965950012207,
      "activations/layer18_attention_weight_min": -23.869491577148438,
      "activations/layer19_attention_weight_max": 37.45462417602539,
      "activations/layer19_attention_weight_min": -31.403043746948242,
      "activations/layer1_attention_weight_max": 16.36043357849121,
      "activations/layer1_attention_weight_min": -13.514932632446289,
      "activations/layer20_attention_weight_max": 31.2567081451416,
      "activations/layer20_attention_weight_min": -24.05573081970215,
      "activations/layer21_attention_weight_max": 29.988317489624023,
      "activations/layer21_attention_weight_min": -23.699316024780273,
      "activations/layer22_attention_weight_max": 50.1989631652832,
      "activations/layer22_attention_weight_min": -29.863590240478516,
      "activations/layer23_attention_weight_max": 33.720481872558594,
      "activations/layer23_attention_weight_min": -23.751392364501953,
      "activations/layer2_attention_weight_max": 30.55708885192871,
      "activations/layer2_attention_weight_min": -29.268726348876953,
      "activations/layer3_attention_weight_max": 87.30738067626953,
      "activations/layer3_attention_weight_min": -90.1053695678711,
      "activations/layer4_attention_weight_max": 106.90555572509766,
      "activations/layer4_attention_weight_min": -105.32318115234375,
      "activations/layer5_attention_weight_max": 51.43070602416992,
      "activations/layer5_attention_weight_min": -61.41930389404297,
      "activations/layer6_attention_weight_max": 45.56325912475586,
      "activations/layer6_attention_weight_min": -46.46775817871094,
      "activations/layer7_attention_weight_max": 94.52034759521484,
      "activations/layer7_attention_weight_min": -89.30541229248047,
      "activations/layer8_attention_weight_max": 38.81350326538086,
      "activations/layer8_attention_weight_min": -39.623252868652344,
      "activations/layer9_attention_weight_max": 30.68695068359375,
      "activations/layer9_attention_weight_min": -31.622364044189453,
      "epoch": 12.6,
      "learning_rate": 6.944431818181818e-05,
      "loss": 2.7656,
      "step": 216850
    },
    {
      "activations/layer0_attention_weight_max": 15.933409690856934,
      "activations/layer0_attention_weight_min": -11.535650253295898,
      "activations/layer10_attention_weight_max": 32.7371826171875,
      "activations/layer10_attention_weight_min": -32.60087585449219,
      "activations/layer11_attention_weight_max": 34.74516296386719,
      "activations/layer11_attention_weight_min": -32.36000061035156,
      "activations/layer12_attention_weight_max": 29.822715759277344,
      "activations/layer12_attention_weight_min": -25.396196365356445,
      "activations/layer13_attention_weight_max": 49.90782165527344,
      "activations/layer13_attention_weight_min": -36.6004753112793,
      "activations/layer14_attention_weight_max": 53.334007263183594,
      "activations/layer14_attention_weight_min": -37.90785598754883,
      "activations/layer15_attention_weight_max": 50.94406509399414,
      "activations/layer15_attention_weight_min": -34.4786491394043,
      "activations/layer16_attention_weight_max": 36.66530990600586,
      "activations/layer16_attention_weight_min": -31.49440574645996,
      "activations/layer17_attention_weight_max": 35.23988342285156,
      "activations/layer17_attention_weight_min": -26.53592872619629,
      "activations/layer18_attention_weight_max": 41.709266662597656,
      "activations/layer18_attention_weight_min": -25.0730037689209,
      "activations/layer19_attention_weight_max": 44.819190979003906,
      "activations/layer19_attention_weight_min": -32.39094924926758,
      "activations/layer1_attention_weight_max": 15.922870635986328,
      "activations/layer1_attention_weight_min": -14.860969543457031,
      "activations/layer20_attention_weight_max": 36.50114059448242,
      "activations/layer20_attention_weight_min": -24.321718215942383,
      "activations/layer21_attention_weight_max": 35.195674896240234,
      "activations/layer21_attention_weight_min": -24.73598289489746,
      "activations/layer22_attention_weight_max": 55.661224365234375,
      "activations/layer22_attention_weight_min": -33.004249572753906,
      "activations/layer23_attention_weight_max": 41.592735290527344,
      "activations/layer23_attention_weight_min": -25.208677291870117,
      "activations/layer2_attention_weight_max": 33.567405700683594,
      "activations/layer2_attention_weight_min": -30.913997650146484,
      "activations/layer3_attention_weight_max": 95.3791275024414,
      "activations/layer3_attention_weight_min": -93.24925231933594,
      "activations/layer4_attention_weight_max": 111.5714111328125,
      "activations/layer4_attention_weight_min": -108.70033264160156,
      "activations/layer5_attention_weight_max": 49.76344299316406,
      "activations/layer5_attention_weight_min": -58.73121643066406,
      "activations/layer6_attention_weight_max": 46.336299896240234,
      "activations/layer6_attention_weight_min": -48.726253509521484,
      "activations/layer7_attention_weight_max": 86.16918182373047,
      "activations/layer7_attention_weight_min": -96.63307189941406,
      "activations/layer8_attention_weight_max": 42.623924255371094,
      "activations/layer8_attention_weight_min": -43.21812438964844,
      "activations/layer9_attention_weight_max": 32.104427337646484,
      "activations/layer9_attention_weight_min": -34.98464584350586,
      "epoch": 12.6,
      "learning_rate": 6.942537878787878e-05,
      "loss": 2.797,
      "step": 216900
    },
    {
      "activations/layer0_attention_weight_max": 16.528594970703125,
      "activations/layer0_attention_weight_min": -12.902240753173828,
      "activations/layer10_attention_weight_max": 34.73117446899414,
      "activations/layer10_attention_weight_min": -32.913021087646484,
      "activations/layer11_attention_weight_max": 37.72682189941406,
      "activations/layer11_attention_weight_min": -32.85026550292969,
      "activations/layer12_attention_weight_max": 23.934730529785156,
      "activations/layer12_attention_weight_min": -33.102264404296875,
      "activations/layer13_attention_weight_max": 40.04758834838867,
      "activations/layer13_attention_weight_min": -32.703853607177734,
      "activations/layer14_attention_weight_max": 40.38718032836914,
      "activations/layer14_attention_weight_min": -36.800960540771484,
      "activations/layer15_attention_weight_max": 41.67636489868164,
      "activations/layer15_attention_weight_min": -32.300479888916016,
      "activations/layer16_attention_weight_max": 30.855106353759766,
      "activations/layer16_attention_weight_min": -28.247879028320312,
      "activations/layer17_attention_weight_max": 31.741403579711914,
      "activations/layer17_attention_weight_min": -25.779529571533203,
      "activations/layer18_attention_weight_max": 28.917387008666992,
      "activations/layer18_attention_weight_min": -24.458364486694336,
      "activations/layer19_attention_weight_max": 32.278812408447266,
      "activations/layer19_attention_weight_min": -31.479406356811523,
      "activations/layer1_attention_weight_max": 16.254596710205078,
      "activations/layer1_attention_weight_min": -14.815143585205078,
      "activations/layer20_attention_weight_max": 27.958372116088867,
      "activations/layer20_attention_weight_min": -24.259220123291016,
      "activations/layer21_attention_weight_max": 27.847753524780273,
      "activations/layer21_attention_weight_min": -22.439781188964844,
      "activations/layer22_attention_weight_max": 42.23896789550781,
      "activations/layer22_attention_weight_min": -30.047225952148438,
      "activations/layer23_attention_weight_max": 37.576751708984375,
      "activations/layer23_attention_weight_min": -24.69282341003418,
      "activations/layer2_attention_weight_max": 31.70745086669922,
      "activations/layer2_attention_weight_min": -32.180702209472656,
      "activations/layer3_attention_weight_max": 96.17002868652344,
      "activations/layer3_attention_weight_min": -92.8698959350586,
      "activations/layer4_attention_weight_max": 108.09600067138672,
      "activations/layer4_attention_weight_min": -108.1710433959961,
      "activations/layer5_attention_weight_max": 52.146385192871094,
      "activations/layer5_attention_weight_min": -60.781272888183594,
      "activations/layer6_attention_weight_max": 40.1374397277832,
      "activations/layer6_attention_weight_min": -43.90483856201172,
      "activations/layer7_attention_weight_max": 91.48469543457031,
      "activations/layer7_attention_weight_min": -87.3158187866211,
      "activations/layer8_attention_weight_max": 40.29074478149414,
      "activations/layer8_attention_weight_min": -40.6641960144043,
      "activations/layer9_attention_weight_max": 31.74371910095215,
      "activations/layer9_attention_weight_min": -32.4075813293457,
      "epoch": 12.61,
      "learning_rate": 6.94064393939394e-05,
      "loss": 2.7748,
      "step": 216950
    },
    {
      "activations/layer0_attention_weight_max": 16.24620819091797,
      "activations/layer0_attention_weight_min": -12.074665069580078,
      "activations/layer10_attention_weight_max": 31.26351547241211,
      "activations/layer10_attention_weight_min": -30.57927703857422,
      "activations/layer11_attention_weight_max": 29.4776611328125,
      "activations/layer11_attention_weight_min": -30.433509826660156,
      "activations/layer12_attention_weight_max": 24.305437088012695,
      "activations/layer12_attention_weight_min": -24.1276798248291,
      "activations/layer13_attention_weight_max": 35.854522705078125,
      "activations/layer13_attention_weight_min": -31.910354614257812,
      "activations/layer14_attention_weight_max": 38.827693939208984,
      "activations/layer14_attention_weight_min": -35.41027069091797,
      "activations/layer15_attention_weight_max": 35.01367950439453,
      "activations/layer15_attention_weight_min": -31.0869140625,
      "activations/layer16_attention_weight_max": 29.282512664794922,
      "activations/layer16_attention_weight_min": -27.66204071044922,
      "activations/layer17_attention_weight_max": 29.82387351989746,
      "activations/layer17_attention_weight_min": -25.06172752380371,
      "activations/layer18_attention_weight_max": 31.700382232666016,
      "activations/layer18_attention_weight_min": -23.069040298461914,
      "activations/layer19_attention_weight_max": 32.55930709838867,
      "activations/layer19_attention_weight_min": -31.85711097717285,
      "activations/layer1_attention_weight_max": 16.032228469848633,
      "activations/layer1_attention_weight_min": -13.319280624389648,
      "activations/layer20_attention_weight_max": 28.5953311920166,
      "activations/layer20_attention_weight_min": -24.584033966064453,
      "activations/layer21_attention_weight_max": 27.460325241088867,
      "activations/layer21_attention_weight_min": -23.897741317749023,
      "activations/layer22_attention_weight_max": 38.65848922729492,
      "activations/layer22_attention_weight_min": -29.999000549316406,
      "activations/layer23_attention_weight_max": 31.62192153930664,
      "activations/layer23_attention_weight_min": -23.54328727722168,
      "activations/layer2_attention_weight_max": 28.57861328125,
      "activations/layer2_attention_weight_min": -29.022069931030273,
      "activations/layer3_attention_weight_max": 87.84203338623047,
      "activations/layer3_attention_weight_min": -89.17345428466797,
      "activations/layer4_attention_weight_max": 96.80870819091797,
      "activations/layer4_attention_weight_min": -101.45430755615234,
      "activations/layer5_attention_weight_max": 48.38523864746094,
      "activations/layer5_attention_weight_min": -58.91999053955078,
      "activations/layer6_attention_weight_max": 41.06502914428711,
      "activations/layer6_attention_weight_min": -47.30448913574219,
      "activations/layer7_attention_weight_max": 81.41171264648438,
      "activations/layer7_attention_weight_min": -85.72845458984375,
      "activations/layer8_attention_weight_max": 38.597408294677734,
      "activations/layer8_attention_weight_min": -41.983375549316406,
      "activations/layer9_attention_weight_max": 30.043670654296875,
      "activations/layer9_attention_weight_min": -31.516489028930664,
      "epoch": 12.61,
      "learning_rate": 6.93875e-05,
      "loss": 2.766,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4369,
      "eval_samples_per_second": 508.952,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4369,
      "eval_openwebtext_samples_per_second": 508.952,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9602,
      "eval_wikitext_samples_per_second": 232.628,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.534,
      "eval_lambada_samples_per_second": 510.697,
      "step": 217000
    },
    {
      "activations/layer0_attention_weight_max": 16.558443069458008,
      "activations/layer0_attention_weight_min": -12.547664642333984,
      "activations/layer10_attention_weight_max": 36.202911376953125,
      "activations/layer10_attention_weight_min": -34.54030990600586,
      "activations/layer11_attention_weight_max": 35.604366302490234,
      "activations/layer11_attention_weight_min": -33.40166091918945,
      "activations/layer12_attention_weight_max": 25.266557693481445,
      "activations/layer12_attention_weight_min": -25.142105102539062,
      "activations/layer13_attention_weight_max": 39.38871765136719,
      "activations/layer13_attention_weight_min": -35.813472747802734,
      "activations/layer14_attention_weight_max": 41.04695129394531,
      "activations/layer14_attention_weight_min": -39.496604919433594,
      "activations/layer15_attention_weight_max": 34.55678939819336,
      "activations/layer15_attention_weight_min": -32.916786193847656,
      "activations/layer16_attention_weight_max": 27.41853141784668,
      "activations/layer16_attention_weight_min": -28.90264320373535,
      "activations/layer17_attention_weight_max": 27.20349884033203,
      "activations/layer17_attention_weight_min": -26.6528377532959,
      "activations/layer18_attention_weight_max": 28.919836044311523,
      "activations/layer18_attention_weight_min": -26.50262451171875,
      "activations/layer19_attention_weight_max": 36.42180252075195,
      "activations/layer19_attention_weight_min": -32.910919189453125,
      "activations/layer1_attention_weight_max": 16.043014526367188,
      "activations/layer1_attention_weight_min": -14.07180404663086,
      "activations/layer20_attention_weight_max": 29.402244567871094,
      "activations/layer20_attention_weight_min": -25.8048038482666,
      "activations/layer21_attention_weight_max": 28.118419647216797,
      "activations/layer21_attention_weight_min": -26.04985237121582,
      "activations/layer22_attention_weight_max": 41.32841873168945,
      "activations/layer22_attention_weight_min": -31.117422103881836,
      "activations/layer23_attention_weight_max": 30.550228118896484,
      "activations/layer23_attention_weight_min": -24.500778198242188,
      "activations/layer2_attention_weight_max": 32.02202224731445,
      "activations/layer2_attention_weight_min": -30.560651779174805,
      "activations/layer3_attention_weight_max": 94.25955963134766,
      "activations/layer3_attention_weight_min": -93.43148040771484,
      "activations/layer4_attention_weight_max": 108.2582778930664,
      "activations/layer4_attention_weight_min": -109.72389221191406,
      "activations/layer5_attention_weight_max": 52.04256820678711,
      "activations/layer5_attention_weight_min": -59.93768310546875,
      "activations/layer6_attention_weight_max": 46.72435760498047,
      "activations/layer6_attention_weight_min": -49.024532318115234,
      "activations/layer7_attention_weight_max": 91.70060729980469,
      "activations/layer7_attention_weight_min": -85.77438354492188,
      "activations/layer8_attention_weight_max": 43.00191116333008,
      "activations/layer8_attention_weight_min": -41.69844055175781,
      "activations/layer9_attention_weight_max": 34.27433395385742,
      "activations/layer9_attention_weight_min": -33.98972702026367,
      "epoch": 12.61,
      "learning_rate": 6.93685606060606e-05,
      "loss": 2.7779,
      "step": 217050
    },
    {
      "activations/layer0_attention_weight_max": 15.289679527282715,
      "activations/layer0_attention_weight_min": -11.636723518371582,
      "activations/layer10_attention_weight_max": 31.86415672302246,
      "activations/layer10_attention_weight_min": -32.50725173950195,
      "activations/layer11_attention_weight_max": 31.64057731628418,
      "activations/layer11_attention_weight_min": -31.42897605895996,
      "activations/layer12_attention_weight_max": 26.178590774536133,
      "activations/layer12_attention_weight_min": -24.896772384643555,
      "activations/layer13_attention_weight_max": 38.450679779052734,
      "activations/layer13_attention_weight_min": -34.88922119140625,
      "activations/layer14_attention_weight_max": 46.41856002807617,
      "activations/layer14_attention_weight_min": -38.54142761230469,
      "activations/layer15_attention_weight_max": 37.04281234741211,
      "activations/layer15_attention_weight_min": -33.75865173339844,
      "activations/layer16_attention_weight_max": 29.961336135864258,
      "activations/layer16_attention_weight_min": -28.01664161682129,
      "activations/layer17_attention_weight_max": 30.87336540222168,
      "activations/layer17_attention_weight_min": -26.59340476989746,
      "activations/layer18_attention_weight_max": 31.345355987548828,
      "activations/layer18_attention_weight_min": -26.6560001373291,
      "activations/layer19_attention_weight_max": 35.39331817626953,
      "activations/layer19_attention_weight_min": -34.58512496948242,
      "activations/layer1_attention_weight_max": 16.171279907226562,
      "activations/layer1_attention_weight_min": -13.845260620117188,
      "activations/layer20_attention_weight_max": 29.606298446655273,
      "activations/layer20_attention_weight_min": -24.94063377380371,
      "activations/layer21_attention_weight_max": 28.912405014038086,
      "activations/layer21_attention_weight_min": -25.207378387451172,
      "activations/layer22_attention_weight_max": 45.61697769165039,
      "activations/layer22_attention_weight_min": -31.88134002685547,
      "activations/layer23_attention_weight_max": 38.66570281982422,
      "activations/layer23_attention_weight_min": -27.083683013916016,
      "activations/layer2_attention_weight_max": 30.333295822143555,
      "activations/layer2_attention_weight_min": -29.408843994140625,
      "activations/layer3_attention_weight_max": 89.35911560058594,
      "activations/layer3_attention_weight_min": -92.24971771240234,
      "activations/layer4_attention_weight_max": 110.1308364868164,
      "activations/layer4_attention_weight_min": -108.38945770263672,
      "activations/layer5_attention_weight_max": 50.94820785522461,
      "activations/layer5_attention_weight_min": -61.007652282714844,
      "activations/layer6_attention_weight_max": 45.66167449951172,
      "activations/layer6_attention_weight_min": -48.03758239746094,
      "activations/layer7_attention_weight_max": 90.8697280883789,
      "activations/layer7_attention_weight_min": -91.07411193847656,
      "activations/layer8_attention_weight_max": 39.5357780456543,
      "activations/layer8_attention_weight_min": -44.333839416503906,
      "activations/layer9_attention_weight_max": 32.126808166503906,
      "activations/layer9_attention_weight_min": -33.44229507446289,
      "epoch": 12.61,
      "learning_rate": 6.93496212121212e-05,
      "loss": 2.7792,
      "step": 217100
    },
    {
      "activations/layer0_attention_weight_max": 15.880709648132324,
      "activations/layer0_attention_weight_min": -12.019295692443848,
      "activations/layer10_attention_weight_max": 30.97879981994629,
      "activations/layer10_attention_weight_min": -32.61613845825195,
      "activations/layer11_attention_weight_max": 31.67464256286621,
      "activations/layer11_attention_weight_min": -32.213653564453125,
      "activations/layer12_attention_weight_max": 23.639301300048828,
      "activations/layer12_attention_weight_min": -25.243629455566406,
      "activations/layer13_attention_weight_max": 37.006221771240234,
      "activations/layer13_attention_weight_min": -32.875511169433594,
      "activations/layer14_attention_weight_max": 38.37959289550781,
      "activations/layer14_attention_weight_min": -36.48222732543945,
      "activations/layer15_attention_weight_max": 32.67635726928711,
      "activations/layer15_attention_weight_min": -33.772621154785156,
      "activations/layer16_attention_weight_max": 27.72450828552246,
      "activations/layer16_attention_weight_min": -29.05760383605957,
      "activations/layer17_attention_weight_max": 30.21504020690918,
      "activations/layer17_attention_weight_min": -25.878713607788086,
      "activations/layer18_attention_weight_max": 29.00648307800293,
      "activations/layer18_attention_weight_min": -25.568038940429688,
      "activations/layer19_attention_weight_max": 31.35104751586914,
      "activations/layer19_attention_weight_min": -31.782957077026367,
      "activations/layer1_attention_weight_max": 16.681623458862305,
      "activations/layer1_attention_weight_min": -14.686622619628906,
      "activations/layer20_attention_weight_max": 28.155303955078125,
      "activations/layer20_attention_weight_min": -25.875328063964844,
      "activations/layer21_attention_weight_max": 27.095937728881836,
      "activations/layer21_attention_weight_min": -26.238737106323242,
      "activations/layer22_attention_weight_max": 39.257198333740234,
      "activations/layer22_attention_weight_min": -33.14864730834961,
      "activations/layer23_attention_weight_max": 32.58457946777344,
      "activations/layer23_attention_weight_min": -26.68914031982422,
      "activations/layer2_attention_weight_max": 31.631872177124023,
      "activations/layer2_attention_weight_min": -31.495901107788086,
      "activations/layer3_attention_weight_max": 92.77537536621094,
      "activations/layer3_attention_weight_min": -94.08744049072266,
      "activations/layer4_attention_weight_max": 110.30767059326172,
      "activations/layer4_attention_weight_min": -114.85643768310547,
      "activations/layer5_attention_weight_max": 52.568504333496094,
      "activations/layer5_attention_weight_min": -62.37522888183594,
      "activations/layer6_attention_weight_max": 42.46467208862305,
      "activations/layer6_attention_weight_min": -46.37675857543945,
      "activations/layer7_attention_weight_max": 87.44822692871094,
      "activations/layer7_attention_weight_min": -88.91510009765625,
      "activations/layer8_attention_weight_max": 39.4912109375,
      "activations/layer8_attention_weight_min": -40.03516387939453,
      "activations/layer9_attention_weight_max": 31.365224838256836,
      "activations/layer9_attention_weight_min": -33.24159622192383,
      "epoch": 12.62,
      "learning_rate": 6.933068181818182e-05,
      "loss": 2.7703,
      "step": 217150
    },
    {
      "activations/layer0_attention_weight_max": 14.83613109588623,
      "activations/layer0_attention_weight_min": -11.081298828125,
      "activations/layer10_attention_weight_max": 33.92354965209961,
      "activations/layer10_attention_weight_min": -33.56431198120117,
      "activations/layer11_attention_weight_max": 33.42036056518555,
      "activations/layer11_attention_weight_min": -31.989410400390625,
      "activations/layer12_attention_weight_max": 23.405826568603516,
      "activations/layer12_attention_weight_min": -24.918596267700195,
      "activations/layer13_attention_weight_max": 36.611305236816406,
      "activations/layer13_attention_weight_min": -35.647701263427734,
      "activations/layer14_attention_weight_max": 47.279598236083984,
      "activations/layer14_attention_weight_min": -36.960506439208984,
      "activations/layer15_attention_weight_max": 35.911598205566406,
      "activations/layer15_attention_weight_min": -31.854084014892578,
      "activations/layer16_attention_weight_max": 31.079055786132812,
      "activations/layer16_attention_weight_min": -27.201353073120117,
      "activations/layer17_attention_weight_max": 31.91485595703125,
      "activations/layer17_attention_weight_min": -25.559741973876953,
      "activations/layer18_attention_weight_max": 33.611305236816406,
      "activations/layer18_attention_weight_min": -24.057334899902344,
      "activations/layer19_attention_weight_max": 34.06080627441406,
      "activations/layer19_attention_weight_min": -32.62293243408203,
      "activations/layer1_attention_weight_max": 16.346996307373047,
      "activations/layer1_attention_weight_min": -14.318150520324707,
      "activations/layer20_attention_weight_max": 27.64341926574707,
      "activations/layer20_attention_weight_min": -24.84734535217285,
      "activations/layer21_attention_weight_max": 28.144134521484375,
      "activations/layer21_attention_weight_min": -25.033227920532227,
      "activations/layer22_attention_weight_max": 45.58810806274414,
      "activations/layer22_attention_weight_min": -31.6785945892334,
      "activations/layer23_attention_weight_max": 33.87725067138672,
      "activations/layer23_attention_weight_min": -24.69624137878418,
      "activations/layer2_attention_weight_max": 32.85954284667969,
      "activations/layer2_attention_weight_min": -30.848114013671875,
      "activations/layer3_attention_weight_max": 92.28518676757812,
      "activations/layer3_attention_weight_min": -90.90821838378906,
      "activations/layer4_attention_weight_max": 109.23023986816406,
      "activations/layer4_attention_weight_min": -110.50716400146484,
      "activations/layer5_attention_weight_max": 52.09580612182617,
      "activations/layer5_attention_weight_min": -61.216407775878906,
      "activations/layer6_attention_weight_max": 45.42750549316406,
      "activations/layer6_attention_weight_min": -47.120296478271484,
      "activations/layer7_attention_weight_max": 90.58598327636719,
      "activations/layer7_attention_weight_min": -93.48749542236328,
      "activations/layer8_attention_weight_max": 41.46065139770508,
      "activations/layer8_attention_weight_min": -44.0283317565918,
      "activations/layer9_attention_weight_max": 33.3416862487793,
      "activations/layer9_attention_weight_min": -33.45064926147461,
      "epoch": 12.62,
      "learning_rate": 6.931174242424242e-05,
      "loss": 2.7611,
      "step": 217200
    },
    {
      "activations/layer0_attention_weight_max": 15.644085884094238,
      "activations/layer0_attention_weight_min": -11.01012134552002,
      "activations/layer10_attention_weight_max": 31.367259979248047,
      "activations/layer10_attention_weight_min": -31.386240005493164,
      "activations/layer11_attention_weight_max": 35.531898498535156,
      "activations/layer11_attention_weight_min": -31.180530548095703,
      "activations/layer12_attention_weight_max": 26.475570678710938,
      "activations/layer12_attention_weight_min": -25.488300323486328,
      "activations/layer13_attention_weight_max": 43.06925582885742,
      "activations/layer13_attention_weight_min": -35.274375915527344,
      "activations/layer14_attention_weight_max": 47.19227600097656,
      "activations/layer14_attention_weight_min": -39.52082443237305,
      "activations/layer15_attention_weight_max": 41.77043533325195,
      "activations/layer15_attention_weight_min": -37.06875991821289,
      "activations/layer16_attention_weight_max": 33.38536834716797,
      "activations/layer16_attention_weight_min": -27.90427017211914,
      "activations/layer17_attention_weight_max": 32.77658462524414,
      "activations/layer17_attention_weight_min": -25.982635498046875,
      "activations/layer18_attention_weight_max": 36.519046783447266,
      "activations/layer18_attention_weight_min": -25.373510360717773,
      "activations/layer19_attention_weight_max": 41.4920539855957,
      "activations/layer19_attention_weight_min": -32.68905258178711,
      "activations/layer1_attention_weight_max": 16.105836868286133,
      "activations/layer1_attention_weight_min": -14.343265533447266,
      "activations/layer20_attention_weight_max": 33.34195327758789,
      "activations/layer20_attention_weight_min": -26.76848030090332,
      "activations/layer21_attention_weight_max": 32.8439826965332,
      "activations/layer21_attention_weight_min": -28.367319107055664,
      "activations/layer22_attention_weight_max": 54.080875396728516,
      "activations/layer22_attention_weight_min": -36.25369644165039,
      "activations/layer23_attention_weight_max": 38.50593566894531,
      "activations/layer23_attention_weight_min": -27.622297286987305,
      "activations/layer2_attention_weight_max": 32.27470779418945,
      "activations/layer2_attention_weight_min": -30.821125030517578,
      "activations/layer3_attention_weight_max": 91.9556884765625,
      "activations/layer3_attention_weight_min": -91.3716049194336,
      "activations/layer4_attention_weight_max": 111.1373519897461,
      "activations/layer4_attention_weight_min": -113.87755584716797,
      "activations/layer5_attention_weight_max": 52.051300048828125,
      "activations/layer5_attention_weight_min": -62.138336181640625,
      "activations/layer6_attention_weight_max": 45.89334487915039,
      "activations/layer6_attention_weight_min": -51.27686309814453,
      "activations/layer7_attention_weight_max": 90.7521743774414,
      "activations/layer7_attention_weight_min": -91.8498306274414,
      "activations/layer8_attention_weight_max": 40.521209716796875,
      "activations/layer8_attention_weight_min": -42.945125579833984,
      "activations/layer9_attention_weight_max": 33.89213180541992,
      "activations/layer9_attention_weight_min": -33.71851348876953,
      "epoch": 12.62,
      "learning_rate": 6.929280303030302e-05,
      "loss": 2.76,
      "step": 217250
    },
    {
      "activations/layer0_attention_weight_max": 16.499004364013672,
      "activations/layer0_attention_weight_min": -11.105058670043945,
      "activations/layer10_attention_weight_max": 33.91205596923828,
      "activations/layer10_attention_weight_min": -32.794586181640625,
      "activations/layer11_attention_weight_max": 34.40580749511719,
      "activations/layer11_attention_weight_min": -33.745452880859375,
      "activations/layer12_attention_weight_max": 27.09784507751465,
      "activations/layer12_attention_weight_min": -27.16939926147461,
      "activations/layer13_attention_weight_max": 36.79619598388672,
      "activations/layer13_attention_weight_min": -33.306419372558594,
      "activations/layer14_attention_weight_max": 42.18215560913086,
      "activations/layer14_attention_weight_min": -35.805049896240234,
      "activations/layer15_attention_weight_max": 34.58324432373047,
      "activations/layer15_attention_weight_min": -33.660987854003906,
      "activations/layer16_attention_weight_max": 30.940027236938477,
      "activations/layer16_attention_weight_min": -28.145042419433594,
      "activations/layer17_attention_weight_max": 28.51972770690918,
      "activations/layer17_attention_weight_min": -25.172082901000977,
      "activations/layer18_attention_weight_max": 30.638277053833008,
      "activations/layer18_attention_weight_min": -23.713167190551758,
      "activations/layer19_attention_weight_max": 36.10859298706055,
      "activations/layer19_attention_weight_min": -32.02561950683594,
      "activations/layer1_attention_weight_max": 17.545015335083008,
      "activations/layer1_attention_weight_min": -13.985487937927246,
      "activations/layer20_attention_weight_max": 28.92425537109375,
      "activations/layer20_attention_weight_min": -24.17057991027832,
      "activations/layer21_attention_weight_max": 29.08025360107422,
      "activations/layer21_attention_weight_min": -23.8935489654541,
      "activations/layer22_attention_weight_max": 44.74054718017578,
      "activations/layer22_attention_weight_min": -33.3311653137207,
      "activations/layer23_attention_weight_max": 35.30651092529297,
      "activations/layer23_attention_weight_min": -25.64096450805664,
      "activations/layer2_attention_weight_max": 35.03961944580078,
      "activations/layer2_attention_weight_min": -32.083003997802734,
      "activations/layer3_attention_weight_max": 96.0423355102539,
      "activations/layer3_attention_weight_min": -92.54326629638672,
      "activations/layer4_attention_weight_max": 110.11467742919922,
      "activations/layer4_attention_weight_min": -108.08043670654297,
      "activations/layer5_attention_weight_max": 53.090431213378906,
      "activations/layer5_attention_weight_min": -61.33427047729492,
      "activations/layer6_attention_weight_max": 43.88084411621094,
      "activations/layer6_attention_weight_min": -44.65437698364258,
      "activations/layer7_attention_weight_max": 92.2906723022461,
      "activations/layer7_attention_weight_min": -87.8572998046875,
      "activations/layer8_attention_weight_max": 40.358367919921875,
      "activations/layer8_attention_weight_min": -42.2012825012207,
      "activations/layer9_attention_weight_max": 33.23678207397461,
      "activations/layer9_attention_weight_min": -31.993850708007812,
      "epoch": 12.63,
      "learning_rate": 6.927386363636362e-05,
      "loss": 2.7732,
      "step": 217300
    },
    {
      "activations/layer0_attention_weight_max": 16.220924377441406,
      "activations/layer0_attention_weight_min": -12.602742195129395,
      "activations/layer10_attention_weight_max": 33.46422576904297,
      "activations/layer10_attention_weight_min": -32.16845703125,
      "activations/layer11_attention_weight_max": 33.281524658203125,
      "activations/layer11_attention_weight_min": -33.067142486572266,
      "activations/layer12_attention_weight_max": 27.184310913085938,
      "activations/layer12_attention_weight_min": -25.82246208190918,
      "activations/layer13_attention_weight_max": 44.65514373779297,
      "activations/layer13_attention_weight_min": -35.56125259399414,
      "activations/layer14_attention_weight_max": 45.368614196777344,
      "activations/layer14_attention_weight_min": -38.92562484741211,
      "activations/layer15_attention_weight_max": 39.6722526550293,
      "activations/layer15_attention_weight_min": -32.85474395751953,
      "activations/layer16_attention_weight_max": 31.988231658935547,
      "activations/layer16_attention_weight_min": -29.034257888793945,
      "activations/layer17_attention_weight_max": 32.88063430786133,
      "activations/layer17_attention_weight_min": -25.713865280151367,
      "activations/layer18_attention_weight_max": 32.15107345581055,
      "activations/layer18_attention_weight_min": -25.131135940551758,
      "activations/layer19_attention_weight_max": 36.79400634765625,
      "activations/layer19_attention_weight_min": -31.674875259399414,
      "activations/layer1_attention_weight_max": 18.38913917541504,
      "activations/layer1_attention_weight_min": -15.775825500488281,
      "activations/layer20_attention_weight_max": 29.107738494873047,
      "activations/layer20_attention_weight_min": -26.457866668701172,
      "activations/layer21_attention_weight_max": 32.4744873046875,
      "activations/layer21_attention_weight_min": -24.593332290649414,
      "activations/layer22_attention_weight_max": 44.071319580078125,
      "activations/layer22_attention_weight_min": -29.63727569580078,
      "activations/layer23_attention_weight_max": 35.54872131347656,
      "activations/layer23_attention_weight_min": -22.40707778930664,
      "activations/layer2_attention_weight_max": 31.8634090423584,
      "activations/layer2_attention_weight_min": -31.23299789428711,
      "activations/layer3_attention_weight_max": 90.95516204833984,
      "activations/layer3_attention_weight_min": -93.4970932006836,
      "activations/layer4_attention_weight_max": 112.41082000732422,
      "activations/layer4_attention_weight_min": -110.83869934082031,
      "activations/layer5_attention_weight_max": 52.38128662109375,
      "activations/layer5_attention_weight_min": -67.18063354492188,
      "activations/layer6_attention_weight_max": 45.04771423339844,
      "activations/layer6_attention_weight_min": -48.43955612182617,
      "activations/layer7_attention_weight_max": 91.9978256225586,
      "activations/layer7_attention_weight_min": -91.0269775390625,
      "activations/layer8_attention_weight_max": 38.7819938659668,
      "activations/layer8_attention_weight_min": -43.15866470336914,
      "activations/layer9_attention_weight_max": 33.91340637207031,
      "activations/layer9_attention_weight_min": -33.755245208740234,
      "epoch": 12.63,
      "learning_rate": 6.925492424242424e-05,
      "loss": 2.7655,
      "step": 217350
    },
    {
      "activations/layer0_attention_weight_max": 15.16450023651123,
      "activations/layer0_attention_weight_min": -12.347701072692871,
      "activations/layer10_attention_weight_max": 31.78507423400879,
      "activations/layer10_attention_weight_min": -33.748626708984375,
      "activations/layer11_attention_weight_max": 31.054914474487305,
      "activations/layer11_attention_weight_min": -33.32749557495117,
      "activations/layer12_attention_weight_max": 26.194887161254883,
      "activations/layer12_attention_weight_min": -26.928529739379883,
      "activations/layer13_attention_weight_max": 34.637107849121094,
      "activations/layer13_attention_weight_min": -36.10203170776367,
      "activations/layer14_attention_weight_max": 40.97166061401367,
      "activations/layer14_attention_weight_min": -37.70381546020508,
      "activations/layer15_attention_weight_max": 35.8846549987793,
      "activations/layer15_attention_weight_min": -33.0213508605957,
      "activations/layer16_attention_weight_max": 31.191722869873047,
      "activations/layer16_attention_weight_min": -28.0968074798584,
      "activations/layer17_attention_weight_max": 31.580474853515625,
      "activations/layer17_attention_weight_min": -25.900293350219727,
      "activations/layer18_attention_weight_max": 28.02155876159668,
      "activations/layer18_attention_weight_min": -28.730867385864258,
      "activations/layer19_attention_weight_max": 33.60440444946289,
      "activations/layer19_attention_weight_min": -30.47393226623535,
      "activations/layer1_attention_weight_max": 17.35931968688965,
      "activations/layer1_attention_weight_min": -15.635995864868164,
      "activations/layer20_attention_weight_max": 28.085437774658203,
      "activations/layer20_attention_weight_min": -25.478591918945312,
      "activations/layer21_attention_weight_max": 28.72286033630371,
      "activations/layer21_attention_weight_min": -25.43739891052246,
      "activations/layer22_attention_weight_max": 47.740257263183594,
      "activations/layer22_attention_weight_min": -30.804134368896484,
      "activations/layer23_attention_weight_max": 33.44451904296875,
      "activations/layer23_attention_weight_min": -28.124595642089844,
      "activations/layer2_attention_weight_max": 33.269287109375,
      "activations/layer2_attention_weight_min": -31.203033447265625,
      "activations/layer3_attention_weight_max": 94.04447174072266,
      "activations/layer3_attention_weight_min": -95.40999603271484,
      "activations/layer4_attention_weight_max": 111.56428527832031,
      "activations/layer4_attention_weight_min": -111.4575424194336,
      "activations/layer5_attention_weight_max": 52.01169967651367,
      "activations/layer5_attention_weight_min": -62.96384811401367,
      "activations/layer6_attention_weight_max": 43.58851623535156,
      "activations/layer6_attention_weight_min": -48.04180145263672,
      "activations/layer7_attention_weight_max": 89.1080093383789,
      "activations/layer7_attention_weight_min": -92.22643280029297,
      "activations/layer8_attention_weight_max": 39.90712356567383,
      "activations/layer8_attention_weight_min": -42.5639533996582,
      "activations/layer9_attention_weight_max": 31.188989639282227,
      "activations/layer9_attention_weight_min": -32.64276885986328,
      "epoch": 12.63,
      "learning_rate": 6.923598484848484e-05,
      "loss": 2.7758,
      "step": 217400
    },
    {
      "activations/layer0_attention_weight_max": 15.295327186584473,
      "activations/layer0_attention_weight_min": -11.900649070739746,
      "activations/layer10_attention_weight_max": 33.043033599853516,
      "activations/layer10_attention_weight_min": -31.647287368774414,
      "activations/layer11_attention_weight_max": 30.42653465270996,
      "activations/layer11_attention_weight_min": -30.675373077392578,
      "activations/layer12_attention_weight_max": 23.805198669433594,
      "activations/layer12_attention_weight_min": -24.70037078857422,
      "activations/layer13_attention_weight_max": 35.980648040771484,
      "activations/layer13_attention_weight_min": -34.270748138427734,
      "activations/layer14_attention_weight_max": 41.602333068847656,
      "activations/layer14_attention_weight_min": -38.53419494628906,
      "activations/layer15_attention_weight_max": 33.33354949951172,
      "activations/layer15_attention_weight_min": -32.74287414550781,
      "activations/layer16_attention_weight_max": 31.435359954833984,
      "activations/layer16_attention_weight_min": -28.18193817138672,
      "activations/layer17_attention_weight_max": 28.619014739990234,
      "activations/layer17_attention_weight_min": -25.74732780456543,
      "activations/layer18_attention_weight_max": 28.669273376464844,
      "activations/layer18_attention_weight_min": -25.423974990844727,
      "activations/layer19_attention_weight_max": 29.628265380859375,
      "activations/layer19_attention_weight_min": -30.68488883972168,
      "activations/layer1_attention_weight_max": 15.869296073913574,
      "activations/layer1_attention_weight_min": -15.419285774230957,
      "activations/layer20_attention_weight_max": 25.072240829467773,
      "activations/layer20_attention_weight_min": -26.72646141052246,
      "activations/layer21_attention_weight_max": 28.985490798950195,
      "activations/layer21_attention_weight_min": -25.44561195373535,
      "activations/layer22_attention_weight_max": 44.4350471496582,
      "activations/layer22_attention_weight_min": -29.129215240478516,
      "activations/layer23_attention_weight_max": 34.98349380493164,
      "activations/layer23_attention_weight_min": -25.52901268005371,
      "activations/layer2_attention_weight_max": 31.229270935058594,
      "activations/layer2_attention_weight_min": -31.882122039794922,
      "activations/layer3_attention_weight_max": 87.85765075683594,
      "activations/layer3_attention_weight_min": -90.03264617919922,
      "activations/layer4_attention_weight_max": 108.99464416503906,
      "activations/layer4_attention_weight_min": -109.67537689208984,
      "activations/layer5_attention_weight_max": 50.446651458740234,
      "activations/layer5_attention_weight_min": -61.884090423583984,
      "activations/layer6_attention_weight_max": 44.904109954833984,
      "activations/layer6_attention_weight_min": -48.59841537475586,
      "activations/layer7_attention_weight_max": 86.56011199951172,
      "activations/layer7_attention_weight_min": -88.83818054199219,
      "activations/layer8_attention_weight_max": 41.47343444824219,
      "activations/layer8_attention_weight_min": -42.91968536376953,
      "activations/layer9_attention_weight_max": 32.12678909301758,
      "activations/layer9_attention_weight_min": -33.760169982910156,
      "epoch": 12.64,
      "learning_rate": 6.921704545454545e-05,
      "loss": 2.7934,
      "step": 217450
    },
    {
      "activations/layer0_attention_weight_max": 14.882539749145508,
      "activations/layer0_attention_weight_min": -13.319924354553223,
      "activations/layer10_attention_weight_max": 34.91415786743164,
      "activations/layer10_attention_weight_min": -33.69779586791992,
      "activations/layer11_attention_weight_max": 34.887786865234375,
      "activations/layer11_attention_weight_min": -33.13774871826172,
      "activations/layer12_attention_weight_max": 29.445613861083984,
      "activations/layer12_attention_weight_min": -25.527591705322266,
      "activations/layer13_attention_weight_max": 42.627410888671875,
      "activations/layer13_attention_weight_min": -34.841461181640625,
      "activations/layer14_attention_weight_max": 45.85550308227539,
      "activations/layer14_attention_weight_min": -39.912845611572266,
      "activations/layer15_attention_weight_max": 39.537601470947266,
      "activations/layer15_attention_weight_min": -31.81267547607422,
      "activations/layer16_attention_weight_max": 32.23607635498047,
      "activations/layer16_attention_weight_min": -27.155908584594727,
      "activations/layer17_attention_weight_max": 30.80219841003418,
      "activations/layer17_attention_weight_min": -26.184267044067383,
      "activations/layer18_attention_weight_max": 31.396995544433594,
      "activations/layer18_attention_weight_min": -23.15192413330078,
      "activations/layer19_attention_weight_max": 39.8214225769043,
      "activations/layer19_attention_weight_min": -31.1701602935791,
      "activations/layer1_attention_weight_max": 16.30202293395996,
      "activations/layer1_attention_weight_min": -14.194031715393066,
      "activations/layer20_attention_weight_max": 30.59099006652832,
      "activations/layer20_attention_weight_min": -25.267587661743164,
      "activations/layer21_attention_weight_max": 32.063873291015625,
      "activations/layer21_attention_weight_min": -24.51491355895996,
      "activations/layer22_attention_weight_max": 48.5576171875,
      "activations/layer22_attention_weight_min": -31.258344650268555,
      "activations/layer23_attention_weight_max": 39.733680725097656,
      "activations/layer23_attention_weight_min": -27.27212905883789,
      "activations/layer2_attention_weight_max": 30.51233673095703,
      "activations/layer2_attention_weight_min": -31.749488830566406,
      "activations/layer3_attention_weight_max": 89.92274475097656,
      "activations/layer3_attention_weight_min": -89.6829833984375,
      "activations/layer4_attention_weight_max": 111.25257873535156,
      "activations/layer4_attention_weight_min": -113.78089904785156,
      "activations/layer5_attention_weight_max": 52.85407257080078,
      "activations/layer5_attention_weight_min": -62.891231536865234,
      "activations/layer6_attention_weight_max": 46.262996673583984,
      "activations/layer6_attention_weight_min": -49.482051849365234,
      "activations/layer7_attention_weight_max": 92.60426330566406,
      "activations/layer7_attention_weight_min": -91.40206146240234,
      "activations/layer8_attention_weight_max": 41.18510055541992,
      "activations/layer8_attention_weight_min": -43.38500213623047,
      "activations/layer9_attention_weight_max": 35.30396270751953,
      "activations/layer9_attention_weight_min": -34.7601318359375,
      "epoch": 12.64,
      "learning_rate": 6.919810606060605e-05,
      "loss": 2.786,
      "step": 217500
    },
    {
      "activations/layer0_attention_weight_max": 15.1640625,
      "activations/layer0_attention_weight_min": -12.051749229431152,
      "activations/layer10_attention_weight_max": 32.15217971801758,
      "activations/layer10_attention_weight_min": -30.461942672729492,
      "activations/layer11_attention_weight_max": 31.985050201416016,
      "activations/layer11_attention_weight_min": -30.957372665405273,
      "activations/layer12_attention_weight_max": 24.151992797851562,
      "activations/layer12_attention_weight_min": -25.245859146118164,
      "activations/layer13_attention_weight_max": 38.26744842529297,
      "activations/layer13_attention_weight_min": -33.23392868041992,
      "activations/layer14_attention_weight_max": 44.015724182128906,
      "activations/layer14_attention_weight_min": -37.27680206298828,
      "activations/layer15_attention_weight_max": 35.133087158203125,
      "activations/layer15_attention_weight_min": -32.845191955566406,
      "activations/layer16_attention_weight_max": 27.769058227539062,
      "activations/layer16_attention_weight_min": -27.598777770996094,
      "activations/layer17_attention_weight_max": 31.910593032836914,
      "activations/layer17_attention_weight_min": -27.019311904907227,
      "activations/layer18_attention_weight_max": 30.709903717041016,
      "activations/layer18_attention_weight_min": -25.172826766967773,
      "activations/layer19_attention_weight_max": 35.965030670166016,
      "activations/layer19_attention_weight_min": -31.996768951416016,
      "activations/layer1_attention_weight_max": 16.5809326171875,
      "activations/layer1_attention_weight_min": -16.237123489379883,
      "activations/layer20_attention_weight_max": 30.869844436645508,
      "activations/layer20_attention_weight_min": -25.625009536743164,
      "activations/layer21_attention_weight_max": 28.884824752807617,
      "activations/layer21_attention_weight_min": -25.157108306884766,
      "activations/layer22_attention_weight_max": 45.05096435546875,
      "activations/layer22_attention_weight_min": -33.43830490112305,
      "activations/layer23_attention_weight_max": 33.87043380737305,
      "activations/layer23_attention_weight_min": -25.68671226501465,
      "activations/layer2_attention_weight_max": 31.729900360107422,
      "activations/layer2_attention_weight_min": -30.015743255615234,
      "activations/layer3_attention_weight_max": 94.85979461669922,
      "activations/layer3_attention_weight_min": -90.15799713134766,
      "activations/layer4_attention_weight_max": 114.9343490600586,
      "activations/layer4_attention_weight_min": -107.15299224853516,
      "activations/layer5_attention_weight_max": 52.410011291503906,
      "activations/layer5_attention_weight_min": -62.0281982421875,
      "activations/layer6_attention_weight_max": 43.023685455322266,
      "activations/layer6_attention_weight_min": -44.84498596191406,
      "activations/layer7_attention_weight_max": 84.63432312011719,
      "activations/layer7_attention_weight_min": -84.79768371582031,
      "activations/layer8_attention_weight_max": 37.97773361206055,
      "activations/layer8_attention_weight_min": -40.083892822265625,
      "activations/layer9_attention_weight_max": 31.177865982055664,
      "activations/layer9_attention_weight_min": -31.95724868774414,
      "epoch": 12.64,
      "learning_rate": 6.917916666666667e-05,
      "loss": 2.7596,
      "step": 217550
    },
    {
      "activations/layer0_attention_weight_max": 16.576183319091797,
      "activations/layer0_attention_weight_min": -11.965414047241211,
      "activations/layer10_attention_weight_max": 33.39170837402344,
      "activations/layer10_attention_weight_min": -35.26368713378906,
      "activations/layer11_attention_weight_max": 33.906639099121094,
      "activations/layer11_attention_weight_min": -33.8270263671875,
      "activations/layer12_attention_weight_max": 25.763059616088867,
      "activations/layer12_attention_weight_min": -25.816816329956055,
      "activations/layer13_attention_weight_max": 39.81757736206055,
      "activations/layer13_attention_weight_min": -35.106689453125,
      "activations/layer14_attention_weight_max": 44.34587097167969,
      "activations/layer14_attention_weight_min": -37.662147521972656,
      "activations/layer15_attention_weight_max": 35.05637741088867,
      "activations/layer15_attention_weight_min": -32.595462799072266,
      "activations/layer16_attention_weight_max": 30.704504013061523,
      "activations/layer16_attention_weight_min": -28.867082595825195,
      "activations/layer17_attention_weight_max": 32.136112213134766,
      "activations/layer17_attention_weight_min": -27.687896728515625,
      "activations/layer18_attention_weight_max": 31.045833587646484,
      "activations/layer18_attention_weight_min": -27.215734481811523,
      "activations/layer19_attention_weight_max": 36.974090576171875,
      "activations/layer19_attention_weight_min": -30.350439071655273,
      "activations/layer1_attention_weight_max": 15.977688789367676,
      "activations/layer1_attention_weight_min": -15.857114791870117,
      "activations/layer20_attention_weight_max": 32.46102523803711,
      "activations/layer20_attention_weight_min": -24.697904586791992,
      "activations/layer21_attention_weight_max": 28.697795867919922,
      "activations/layer21_attention_weight_min": -25.27532958984375,
      "activations/layer22_attention_weight_max": 44.40444564819336,
      "activations/layer22_attention_weight_min": -31.75510025024414,
      "activations/layer23_attention_weight_max": 37.73283004760742,
      "activations/layer23_attention_weight_min": -26.162128448486328,
      "activations/layer2_attention_weight_max": 30.499853134155273,
      "activations/layer2_attention_weight_min": -29.497913360595703,
      "activations/layer3_attention_weight_max": 87.00990295410156,
      "activations/layer3_attention_weight_min": -90.58307647705078,
      "activations/layer4_attention_weight_max": 106.71752166748047,
      "activations/layer4_attention_weight_min": -106.83951568603516,
      "activations/layer5_attention_weight_max": 51.24626922607422,
      "activations/layer5_attention_weight_min": -62.635799407958984,
      "activations/layer6_attention_weight_max": 43.40660095214844,
      "activations/layer6_attention_weight_min": -47.92508316040039,
      "activations/layer7_attention_weight_max": 91.77820587158203,
      "activations/layer7_attention_weight_min": -100.79241180419922,
      "activations/layer8_attention_weight_max": 42.05351257324219,
      "activations/layer8_attention_weight_min": -44.92751693725586,
      "activations/layer9_attention_weight_max": 33.12875747680664,
      "activations/layer9_attention_weight_min": -35.400115966796875,
      "epoch": 12.64,
      "learning_rate": 6.916022727272727e-05,
      "loss": 2.7645,
      "step": 217600
    },
    {
      "activations/layer0_attention_weight_max": 15.090446472167969,
      "activations/layer0_attention_weight_min": -12.12592601776123,
      "activations/layer10_attention_weight_max": 34.08090591430664,
      "activations/layer10_attention_weight_min": -32.9554328918457,
      "activations/layer11_attention_weight_max": 36.041160583496094,
      "activations/layer11_attention_weight_min": -32.67456817626953,
      "activations/layer12_attention_weight_max": 23.84698486328125,
      "activations/layer12_attention_weight_min": -25.544998168945312,
      "activations/layer13_attention_weight_max": 36.6233024597168,
      "activations/layer13_attention_weight_min": -32.4935188293457,
      "activations/layer14_attention_weight_max": 38.97596740722656,
      "activations/layer14_attention_weight_min": -35.122406005859375,
      "activations/layer15_attention_weight_max": 33.53521728515625,
      "activations/layer15_attention_weight_min": -30.899211883544922,
      "activations/layer16_attention_weight_max": 28.771255493164062,
      "activations/layer16_attention_weight_min": -27.56646156311035,
      "activations/layer17_attention_weight_max": 28.105609893798828,
      "activations/layer17_attention_weight_min": -24.086275100708008,
      "activations/layer18_attention_weight_max": 29.6556396484375,
      "activations/layer18_attention_weight_min": -22.95412254333496,
      "activations/layer19_attention_weight_max": 31.629913330078125,
      "activations/layer19_attention_weight_min": -30.235172271728516,
      "activations/layer1_attention_weight_max": 16.157886505126953,
      "activations/layer1_attention_weight_min": -14.565948486328125,
      "activations/layer20_attention_weight_max": 27.988962173461914,
      "activations/layer20_attention_weight_min": -23.277198791503906,
      "activations/layer21_attention_weight_max": 27.935440063476562,
      "activations/layer21_attention_weight_min": -23.850481033325195,
      "activations/layer22_attention_weight_max": 42.42512512207031,
      "activations/layer22_attention_weight_min": -29.218557357788086,
      "activations/layer23_attention_weight_max": 31.99773406982422,
      "activations/layer23_attention_weight_min": -24.742650985717773,
      "activations/layer2_attention_weight_max": 32.32114028930664,
      "activations/layer2_attention_weight_min": -31.327266693115234,
      "activations/layer3_attention_weight_max": 96.56595611572266,
      "activations/layer3_attention_weight_min": -95.98412322998047,
      "activations/layer4_attention_weight_max": 112.53041076660156,
      "activations/layer4_attention_weight_min": -114.33580780029297,
      "activations/layer5_attention_weight_max": 52.576045989990234,
      "activations/layer5_attention_weight_min": -62.47844696044922,
      "activations/layer6_attention_weight_max": 46.09122848510742,
      "activations/layer6_attention_weight_min": -46.307228088378906,
      "activations/layer7_attention_weight_max": 92.47427368164062,
      "activations/layer7_attention_weight_min": -91.30314636230469,
      "activations/layer8_attention_weight_max": 42.50092697143555,
      "activations/layer8_attention_weight_min": -42.096065521240234,
      "activations/layer9_attention_weight_max": 32.86802291870117,
      "activations/layer9_attention_weight_min": -33.01919174194336,
      "epoch": 12.65,
      "learning_rate": 6.914128787878787e-05,
      "loss": 2.7658,
      "step": 217650
    },
    {
      "activations/layer0_attention_weight_max": 15.984053611755371,
      "activations/layer0_attention_weight_min": -11.79472541809082,
      "activations/layer10_attention_weight_max": 41.84169006347656,
      "activations/layer10_attention_weight_min": -39.31378173828125,
      "activations/layer11_attention_weight_max": 41.31475067138672,
      "activations/layer11_attention_weight_min": -39.61726379394531,
      "activations/layer12_attention_weight_max": 36.1881217956543,
      "activations/layer12_attention_weight_min": -27.99375343322754,
      "activations/layer13_attention_weight_max": 43.87157440185547,
      "activations/layer13_attention_weight_min": -40.10569381713867,
      "activations/layer14_attention_weight_max": 46.78037643432617,
      "activations/layer14_attention_weight_min": -39.15591049194336,
      "activations/layer15_attention_weight_max": 39.8950309753418,
      "activations/layer15_attention_weight_min": -33.42801284790039,
      "activations/layer16_attention_weight_max": 34.06090545654297,
      "activations/layer16_attention_weight_min": -28.117691040039062,
      "activations/layer17_attention_weight_max": 33.925758361816406,
      "activations/layer17_attention_weight_min": -25.58194923400879,
      "activations/layer18_attention_weight_max": 34.16544723510742,
      "activations/layer18_attention_weight_min": -24.242000579833984,
      "activations/layer19_attention_weight_max": 38.57992172241211,
      "activations/layer19_attention_weight_min": -31.64983558654785,
      "activations/layer1_attention_weight_max": 16.563907623291016,
      "activations/layer1_attention_weight_min": -16.537012100219727,
      "activations/layer20_attention_weight_max": 29.668317794799805,
      "activations/layer20_attention_weight_min": -24.746923446655273,
      "activations/layer21_attention_weight_max": 29.656347274780273,
      "activations/layer21_attention_weight_min": -23.424516677856445,
      "activations/layer22_attention_weight_max": 47.64408493041992,
      "activations/layer22_attention_weight_min": -29.68081283569336,
      "activations/layer23_attention_weight_max": 38.144439697265625,
      "activations/layer23_attention_weight_min": -25.225086212158203,
      "activations/layer2_attention_weight_max": 34.424591064453125,
      "activations/layer2_attention_weight_min": -34.57823181152344,
      "activations/layer3_attention_weight_max": 104.88667297363281,
      "activations/layer3_attention_weight_min": -107.1803207397461,
      "activations/layer4_attention_weight_max": 119.78435516357422,
      "activations/layer4_attention_weight_min": -120.30301666259766,
      "activations/layer5_attention_weight_max": 54.08094787597656,
      "activations/layer5_attention_weight_min": -60.358917236328125,
      "activations/layer6_attention_weight_max": 47.57411193847656,
      "activations/layer6_attention_weight_min": -51.77501678466797,
      "activations/layer7_attention_weight_max": 102.925537109375,
      "activations/layer7_attention_weight_min": -100.72823333740234,
      "activations/layer8_attention_weight_max": 48.53445053100586,
      "activations/layer8_attention_weight_min": -47.85188674926758,
      "activations/layer9_attention_weight_max": 39.691402435302734,
      "activations/layer9_attention_weight_min": -40.22307586669922,
      "epoch": 12.65,
      "learning_rate": 6.912234848484847e-05,
      "loss": 2.7636,
      "step": 217700
    },
    {
      "activations/layer0_attention_weight_max": 16.000768661499023,
      "activations/layer0_attention_weight_min": -11.806299209594727,
      "activations/layer10_attention_weight_max": 32.69563674926758,
      "activations/layer10_attention_weight_min": -33.869258880615234,
      "activations/layer11_attention_weight_max": 33.30173873901367,
      "activations/layer11_attention_weight_min": -32.679779052734375,
      "activations/layer12_attention_weight_max": 24.203018188476562,
      "activations/layer12_attention_weight_min": -24.34056854248047,
      "activations/layer13_attention_weight_max": 37.22086715698242,
      "activations/layer13_attention_weight_min": -33.45903396606445,
      "activations/layer14_attention_weight_max": 39.58299255371094,
      "activations/layer14_attention_weight_min": -36.56889343261719,
      "activations/layer15_attention_weight_max": 37.39414978027344,
      "activations/layer15_attention_weight_min": -30.810834884643555,
      "activations/layer16_attention_weight_max": 30.12995147705078,
      "activations/layer16_attention_weight_min": -27.264997482299805,
      "activations/layer17_attention_weight_max": 29.31793212890625,
      "activations/layer17_attention_weight_min": -25.960359573364258,
      "activations/layer18_attention_weight_max": 29.46405601501465,
      "activations/layer18_attention_weight_min": -23.80543327331543,
      "activations/layer19_attention_weight_max": 33.222476959228516,
      "activations/layer19_attention_weight_min": -31.6480655670166,
      "activations/layer1_attention_weight_max": 16.37055778503418,
      "activations/layer1_attention_weight_min": -14.876568794250488,
      "activations/layer20_attention_weight_max": 28.614200592041016,
      "activations/layer20_attention_weight_min": -25.20750617980957,
      "activations/layer21_attention_weight_max": 31.390897750854492,
      "activations/layer21_attention_weight_min": -24.310388565063477,
      "activations/layer22_attention_weight_max": 40.50890350341797,
      "activations/layer22_attention_weight_min": -32.54589080810547,
      "activations/layer23_attention_weight_max": 33.423431396484375,
      "activations/layer23_attention_weight_min": -23.149375915527344,
      "activations/layer2_attention_weight_max": 30.559675216674805,
      "activations/layer2_attention_weight_min": -30.058639526367188,
      "activations/layer3_attention_weight_max": 91.10545349121094,
      "activations/layer3_attention_weight_min": -92.6539535522461,
      "activations/layer4_attention_weight_max": 110.55370330810547,
      "activations/layer4_attention_weight_min": -106.3132095336914,
      "activations/layer5_attention_weight_max": 48.822669982910156,
      "activations/layer5_attention_weight_min": -61.462677001953125,
      "activations/layer6_attention_weight_max": 44.63668441772461,
      "activations/layer6_attention_weight_min": -45.569541931152344,
      "activations/layer7_attention_weight_max": 88.64155578613281,
      "activations/layer7_attention_weight_min": -88.0005874633789,
      "activations/layer8_attention_weight_max": 38.97843933105469,
      "activations/layer8_attention_weight_min": -42.35576248168945,
      "activations/layer9_attention_weight_max": 34.07343673706055,
      "activations/layer9_attention_weight_min": -35.03513717651367,
      "epoch": 12.65,
      "learning_rate": 6.910340909090909e-05,
      "loss": 2.7644,
      "step": 217750
    },
    {
      "activations/layer0_attention_weight_max": 16.16116714477539,
      "activations/layer0_attention_weight_min": -12.431757926940918,
      "activations/layer10_attention_weight_max": 33.050838470458984,
      "activations/layer10_attention_weight_min": -33.73683166503906,
      "activations/layer11_attention_weight_max": 32.737918853759766,
      "activations/layer11_attention_weight_min": -32.82658767700195,
      "activations/layer12_attention_weight_max": 25.289316177368164,
      "activations/layer12_attention_weight_min": -26.076765060424805,
      "activations/layer13_attention_weight_max": 38.33483123779297,
      "activations/layer13_attention_weight_min": -35.31350326538086,
      "activations/layer14_attention_weight_max": 40.19655227661133,
      "activations/layer14_attention_weight_min": -37.219276428222656,
      "activations/layer15_attention_weight_max": 37.90248107910156,
      "activations/layer15_attention_weight_min": -33.95549774169922,
      "activations/layer16_attention_weight_max": 32.382972717285156,
      "activations/layer16_attention_weight_min": -28.226604461669922,
      "activations/layer17_attention_weight_max": 38.91579818725586,
      "activations/layer17_attention_weight_min": -25.777868270874023,
      "activations/layer18_attention_weight_max": 32.368934631347656,
      "activations/layer18_attention_weight_min": -24.47798728942871,
      "activations/layer19_attention_weight_max": 34.72269058227539,
      "activations/layer19_attention_weight_min": -30.647079467773438,
      "activations/layer1_attention_weight_max": 15.43606948852539,
      "activations/layer1_attention_weight_min": -14.07709789276123,
      "activations/layer20_attention_weight_max": 25.899784088134766,
      "activations/layer20_attention_weight_min": -24.78325843811035,
      "activations/layer21_attention_weight_max": 29.50533103942871,
      "activations/layer21_attention_weight_min": -27.116044998168945,
      "activations/layer22_attention_weight_max": 44.05125045776367,
      "activations/layer22_attention_weight_min": -29.58076286315918,
      "activations/layer23_attention_weight_max": 33.9158935546875,
      "activations/layer23_attention_weight_min": -24.706771850585938,
      "activations/layer2_attention_weight_max": 30.419200897216797,
      "activations/layer2_attention_weight_min": -30.77387809753418,
      "activations/layer3_attention_weight_max": 92.2264404296875,
      "activations/layer3_attention_weight_min": -88.58534240722656,
      "activations/layer4_attention_weight_max": 110.52091217041016,
      "activations/layer4_attention_weight_min": -107.2996597290039,
      "activations/layer5_attention_weight_max": 49.5439567565918,
      "activations/layer5_attention_weight_min": -60.35221481323242,
      "activations/layer6_attention_weight_max": 42.090782165527344,
      "activations/layer6_attention_weight_min": -45.47429656982422,
      "activations/layer7_attention_weight_max": 86.9281005859375,
      "activations/layer7_attention_weight_min": -85.7062759399414,
      "activations/layer8_attention_weight_max": 39.280311584472656,
      "activations/layer8_attention_weight_min": -39.43568801879883,
      "activations/layer9_attention_weight_max": 31.913801193237305,
      "activations/layer9_attention_weight_min": -33.40711975097656,
      "epoch": 12.66,
      "learning_rate": 6.908446969696969e-05,
      "loss": 2.7731,
      "step": 217800
    },
    {
      "activations/layer0_attention_weight_max": 14.729734420776367,
      "activations/layer0_attention_weight_min": -11.666341781616211,
      "activations/layer10_attention_weight_max": 35.02033615112305,
      "activations/layer10_attention_weight_min": -34.06557083129883,
      "activations/layer11_attention_weight_max": 32.720855712890625,
      "activations/layer11_attention_weight_min": -33.844642639160156,
      "activations/layer12_attention_weight_max": 30.321285247802734,
      "activations/layer12_attention_weight_min": -26.100013732910156,
      "activations/layer13_attention_weight_max": 39.42788314819336,
      "activations/layer13_attention_weight_min": -35.4246940612793,
      "activations/layer14_attention_weight_max": 43.29425811767578,
      "activations/layer14_attention_weight_min": -40.12925720214844,
      "activations/layer15_attention_weight_max": 36.810386657714844,
      "activations/layer15_attention_weight_min": -32.84528350830078,
      "activations/layer16_attention_weight_max": 33.885013580322266,
      "activations/layer16_attention_weight_min": -28.060317993164062,
      "activations/layer17_attention_weight_max": 31.02018165588379,
      "activations/layer17_attention_weight_min": -26.73093032836914,
      "activations/layer18_attention_weight_max": 31.37566566467285,
      "activations/layer18_attention_weight_min": -23.324724197387695,
      "activations/layer19_attention_weight_max": 39.902034759521484,
      "activations/layer19_attention_weight_min": -30.29511070251465,
      "activations/layer1_attention_weight_max": 15.689886093139648,
      "activations/layer1_attention_weight_min": -16.323509216308594,
      "activations/layer20_attention_weight_max": 31.986936569213867,
      "activations/layer20_attention_weight_min": -25.21159553527832,
      "activations/layer21_attention_weight_max": 29.4448299407959,
      "activations/layer21_attention_weight_min": -23.949748992919922,
      "activations/layer22_attention_weight_max": 42.42967224121094,
      "activations/layer22_attention_weight_min": -29.243183135986328,
      "activations/layer23_attention_weight_max": 37.977752685546875,
      "activations/layer23_attention_weight_min": -24.418819427490234,
      "activations/layer2_attention_weight_max": 29.457561492919922,
      "activations/layer2_attention_weight_min": -30.24115753173828,
      "activations/layer3_attention_weight_max": 93.99603271484375,
      "activations/layer3_attention_weight_min": -94.54086303710938,
      "activations/layer4_attention_weight_max": 109.7896499633789,
      "activations/layer4_attention_weight_min": -112.7278060913086,
      "activations/layer5_attention_weight_max": 50.340206146240234,
      "activations/layer5_attention_weight_min": -62.182586669921875,
      "activations/layer6_attention_weight_max": 47.1641960144043,
      "activations/layer6_attention_weight_min": -53.25347137451172,
      "activations/layer7_attention_weight_max": 91.77037811279297,
      "activations/layer7_attention_weight_min": -95.62318420410156,
      "activations/layer8_attention_weight_max": 40.83290481567383,
      "activations/layer8_attention_weight_min": -45.20475387573242,
      "activations/layer9_attention_weight_max": 33.58750915527344,
      "activations/layer9_attention_weight_min": -34.89902877807617,
      "epoch": 12.66,
      "learning_rate": 6.906553030303029e-05,
      "loss": 2.7593,
      "step": 217850
    },
    {
      "activations/layer0_attention_weight_max": 15.998181343078613,
      "activations/layer0_attention_weight_min": -11.427184104919434,
      "activations/layer10_attention_weight_max": 30.978769302368164,
      "activations/layer10_attention_weight_min": -31.827518463134766,
      "activations/layer11_attention_weight_max": 31.35546875,
      "activations/layer11_attention_weight_min": -31.71222686767578,
      "activations/layer12_attention_weight_max": 23.730350494384766,
      "activations/layer12_attention_weight_min": -25.693017959594727,
      "activations/layer13_attention_weight_max": 39.59331130981445,
      "activations/layer13_attention_weight_min": -33.74336242675781,
      "activations/layer14_attention_weight_max": 41.26582717895508,
      "activations/layer14_attention_weight_min": -38.601463317871094,
      "activations/layer15_attention_weight_max": 36.962982177734375,
      "activations/layer15_attention_weight_min": -30.801471710205078,
      "activations/layer16_attention_weight_max": 28.564254760742188,
      "activations/layer16_attention_weight_min": -26.907215118408203,
      "activations/layer17_attention_weight_max": 33.432838439941406,
      "activations/layer17_attention_weight_min": -26.45677375793457,
      "activations/layer18_attention_weight_max": 29.14008140563965,
      "activations/layer18_attention_weight_min": -23.354339599609375,
      "activations/layer19_attention_weight_max": 30.659656524658203,
      "activations/layer19_attention_weight_min": -33.302650451660156,
      "activations/layer1_attention_weight_max": 15.921221733093262,
      "activations/layer1_attention_weight_min": -15.325870513916016,
      "activations/layer20_attention_weight_max": 26.854774475097656,
      "activations/layer20_attention_weight_min": -25.83782386779785,
      "activations/layer21_attention_weight_max": 26.1096134185791,
      "activations/layer21_attention_weight_min": -24.046464920043945,
      "activations/layer22_attention_weight_max": 39.097686767578125,
      "activations/layer22_attention_weight_min": -32.28939437866211,
      "activations/layer23_attention_weight_max": 33.714481353759766,
      "activations/layer23_attention_weight_min": -25.071617126464844,
      "activations/layer2_attention_weight_max": 31.89099884033203,
      "activations/layer2_attention_weight_min": -32.15003204345703,
      "activations/layer3_attention_weight_max": 87.86186981201172,
      "activations/layer3_attention_weight_min": -89.40177154541016,
      "activations/layer4_attention_weight_max": 108.00447082519531,
      "activations/layer4_attention_weight_min": -111.58296203613281,
      "activations/layer5_attention_weight_max": 47.69542694091797,
      "activations/layer5_attention_weight_min": -61.524105072021484,
      "activations/layer6_attention_weight_max": 41.99724197387695,
      "activations/layer6_attention_weight_min": -47.55100631713867,
      "activations/layer7_attention_weight_max": 91.88797760009766,
      "activations/layer7_attention_weight_min": -85.86223602294922,
      "activations/layer8_attention_weight_max": 38.36528015136719,
      "activations/layer8_attention_weight_min": -38.76285934448242,
      "activations/layer9_attention_weight_max": 31.626707077026367,
      "activations/layer9_attention_weight_min": -31.58640480041504,
      "epoch": 12.66,
      "learning_rate": 6.904659090909091e-05,
      "loss": 2.764,
      "step": 217900
    },
    {
      "activations/layer0_attention_weight_max": 16.540250778198242,
      "activations/layer0_attention_weight_min": -11.55249309539795,
      "activations/layer10_attention_weight_max": 33.435523986816406,
      "activations/layer10_attention_weight_min": -34.226287841796875,
      "activations/layer11_attention_weight_max": 32.423583984375,
      "activations/layer11_attention_weight_min": -34.50151443481445,
      "activations/layer12_attention_weight_max": 24.722665786743164,
      "activations/layer12_attention_weight_min": -25.50310707092285,
      "activations/layer13_attention_weight_max": 37.57866668701172,
      "activations/layer13_attention_weight_min": -33.27766799926758,
      "activations/layer14_attention_weight_max": 41.923057556152344,
      "activations/layer14_attention_weight_min": -35.41130065917969,
      "activations/layer15_attention_weight_max": 35.573822021484375,
      "activations/layer15_attention_weight_min": -31.313684463500977,
      "activations/layer16_attention_weight_max": 32.13909149169922,
      "activations/layer16_attention_weight_min": -27.189563751220703,
      "activations/layer17_attention_weight_max": 30.131521224975586,
      "activations/layer17_attention_weight_min": -25.673925399780273,
      "activations/layer18_attention_weight_max": 31.066139221191406,
      "activations/layer18_attention_weight_min": -23.0607967376709,
      "activations/layer19_attention_weight_max": 33.351375579833984,
      "activations/layer19_attention_weight_min": -30.468168258666992,
      "activations/layer1_attention_weight_max": 16.19835090637207,
      "activations/layer1_attention_weight_min": -15.332250595092773,
      "activations/layer20_attention_weight_max": 26.638315200805664,
      "activations/layer20_attention_weight_min": -24.06399917602539,
      "activations/layer21_attention_weight_max": 26.792938232421875,
      "activations/layer21_attention_weight_min": -23.98930549621582,
      "activations/layer22_attention_weight_max": 41.021053314208984,
      "activations/layer22_attention_weight_min": -29.348928451538086,
      "activations/layer23_attention_weight_max": 31.6268310546875,
      "activations/layer23_attention_weight_min": -25.979736328125,
      "activations/layer2_attention_weight_max": 31.51301383972168,
      "activations/layer2_attention_weight_min": -32.75916290283203,
      "activations/layer3_attention_weight_max": 89.7699203491211,
      "activations/layer3_attention_weight_min": -93.17202758789062,
      "activations/layer4_attention_weight_max": 110.56754302978516,
      "activations/layer4_attention_weight_min": -114.45755767822266,
      "activations/layer5_attention_weight_max": 53.086830139160156,
      "activations/layer5_attention_weight_min": -65.27754974365234,
      "activations/layer6_attention_weight_max": 46.414344787597656,
      "activations/layer6_attention_weight_min": -50.3610954284668,
      "activations/layer7_attention_weight_max": 90.31770324707031,
      "activations/layer7_attention_weight_min": -101.25830078125,
      "activations/layer8_attention_weight_max": 40.35357666015625,
      "activations/layer8_attention_weight_min": -43.201637268066406,
      "activations/layer9_attention_weight_max": 34.339420318603516,
      "activations/layer9_attention_weight_min": -34.72573471069336,
      "epoch": 12.66,
      "learning_rate": 6.902765151515151e-05,
      "loss": 2.7675,
      "step": 217950
    },
    {
      "activations/layer0_attention_weight_max": 15.121837615966797,
      "activations/layer0_attention_weight_min": -13.525556564331055,
      "activations/layer10_attention_weight_max": 31.98912811279297,
      "activations/layer10_attention_weight_min": -32.196109771728516,
      "activations/layer11_attention_weight_max": 31.105220794677734,
      "activations/layer11_attention_weight_min": -31.938657760620117,
      "activations/layer12_attention_weight_max": 24.908771514892578,
      "activations/layer12_attention_weight_min": -26.058076858520508,
      "activations/layer13_attention_weight_max": 39.57081604003906,
      "activations/layer13_attention_weight_min": -34.19489669799805,
      "activations/layer14_attention_weight_max": 40.36399841308594,
      "activations/layer14_attention_weight_min": -36.27692413330078,
      "activations/layer15_attention_weight_max": 35.951419830322266,
      "activations/layer15_attention_weight_min": -31.780168533325195,
      "activations/layer16_attention_weight_max": 29.62934112548828,
      "activations/layer16_attention_weight_min": -27.637296676635742,
      "activations/layer17_attention_weight_max": 30.003225326538086,
      "activations/layer17_attention_weight_min": -24.480207443237305,
      "activations/layer18_attention_weight_max": 31.289133071899414,
      "activations/layer18_attention_weight_min": -22.888111114501953,
      "activations/layer19_attention_weight_max": 33.752235412597656,
      "activations/layer19_attention_weight_min": -29.975683212280273,
      "activations/layer1_attention_weight_max": 16.893938064575195,
      "activations/layer1_attention_weight_min": -16.718730926513672,
      "activations/layer20_attention_weight_max": 26.319272994995117,
      "activations/layer20_attention_weight_min": -24.598676681518555,
      "activations/layer21_attention_weight_max": 25.8826904296875,
      "activations/layer21_attention_weight_min": -23.542064666748047,
      "activations/layer22_attention_weight_max": 42.47981643676758,
      "activations/layer22_attention_weight_min": -30.338470458984375,
      "activations/layer23_attention_weight_max": 34.11777114868164,
      "activations/layer23_attention_weight_min": -25.281597137451172,
      "activations/layer2_attention_weight_max": 31.105365753173828,
      "activations/layer2_attention_weight_min": -30.88249969482422,
      "activations/layer3_attention_weight_max": 91.54983520507812,
      "activations/layer3_attention_weight_min": -95.1405258178711,
      "activations/layer4_attention_weight_max": 106.09737396240234,
      "activations/layer4_attention_weight_min": -109.95659637451172,
      "activations/layer5_attention_weight_max": 51.489280700683594,
      "activations/layer5_attention_weight_min": -57.82024383544922,
      "activations/layer6_attention_weight_max": 42.26378631591797,
      "activations/layer6_attention_weight_min": -45.96792221069336,
      "activations/layer7_attention_weight_max": 87.3686752319336,
      "activations/layer7_attention_weight_min": -89.64723205566406,
      "activations/layer8_attention_weight_max": 38.636070251464844,
      "activations/layer8_attention_weight_min": -39.327579498291016,
      "activations/layer9_attention_weight_max": 32.496925354003906,
      "activations/layer9_attention_weight_min": -32.85602951049805,
      "epoch": 12.67,
      "learning_rate": 6.900871212121211e-05,
      "loss": 2.763,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.45,
      "eval_samples_per_second": 508.163,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.45,
      "eval_openwebtext_samples_per_second": 508.163,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 1.9619,
      "eval_wikitext_samples_per_second": 232.424,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_lambada_loss": 2.39453125,
      "eval_lambada_ppl": 10.963057921054194,
      "eval_lambada_runtime": 9.5071,
      "eval_lambada_samples_per_second": 512.142,
      "step": 218000
    },
    {
      "activations/layer0_attention_weight_max": 16.706806182861328,
      "activations/layer0_attention_weight_min": -11.810391426086426,
      "activations/layer10_attention_weight_max": 31.543949127197266,
      "activations/layer10_attention_weight_min": -32.229339599609375,
      "activations/layer11_attention_weight_max": 31.11580467224121,
      "activations/layer11_attention_weight_min": -30.177017211914062,
      "activations/layer12_attention_weight_max": 29.42664909362793,
      "activations/layer12_attention_weight_min": -24.787662506103516,
      "activations/layer13_attention_weight_max": 40.593807220458984,
      "activations/layer13_attention_weight_min": -33.07658386230469,
      "activations/layer14_attention_weight_max": 43.03288650512695,
      "activations/layer14_attention_weight_min": -35.639522552490234,
      "activations/layer15_attention_weight_max": 35.21078109741211,
      "activations/layer15_attention_weight_min": -30.936918258666992,
      "activations/layer16_attention_weight_max": 32.80312728881836,
      "activations/layer16_attention_weight_min": -26.83036231994629,
      "activations/layer17_attention_weight_max": 31.51445960998535,
      "activations/layer17_attention_weight_min": -25.20035171508789,
      "activations/layer18_attention_weight_max": 32.027061462402344,
      "activations/layer18_attention_weight_min": -23.99439239501953,
      "activations/layer19_attention_weight_max": 37.25053787231445,
      "activations/layer19_attention_weight_min": -34.722190856933594,
      "activations/layer1_attention_weight_max": 17.07086944580078,
      "activations/layer1_attention_weight_min": -16.92608642578125,
      "activations/layer20_attention_weight_max": 29.572410583496094,
      "activations/layer20_attention_weight_min": -25.505165100097656,
      "activations/layer21_attention_weight_max": 29.35531234741211,
      "activations/layer21_attention_weight_min": -24.92091941833496,
      "activations/layer22_attention_weight_max": 41.86783218383789,
      "activations/layer22_attention_weight_min": -30.784591674804688,
      "activations/layer23_attention_weight_max": 32.57459259033203,
      "activations/layer23_attention_weight_min": -24.65389633178711,
      "activations/layer2_attention_weight_max": 31.538002014160156,
      "activations/layer2_attention_weight_min": -31.028911590576172,
      "activations/layer3_attention_weight_max": 88.83997344970703,
      "activations/layer3_attention_weight_min": -90.6604995727539,
      "activations/layer4_attention_weight_max": 106.47674560546875,
      "activations/layer4_attention_weight_min": -107.77616119384766,
      "activations/layer5_attention_weight_max": 52.518035888671875,
      "activations/layer5_attention_weight_min": -61.286529541015625,
      "activations/layer6_attention_weight_max": 46.07606887817383,
      "activations/layer6_attention_weight_min": -47.44619369506836,
      "activations/layer7_attention_weight_max": 91.0460205078125,
      "activations/layer7_attention_weight_min": -87.42308807373047,
      "activations/layer8_attention_weight_max": 39.19578552246094,
      "activations/layer8_attention_weight_min": -41.64051055908203,
      "activations/layer9_attention_weight_max": 32.03010940551758,
      "activations/layer9_attention_weight_min": -32.223045349121094,
      "epoch": 12.67,
      "learning_rate": 6.898977272727271e-05,
      "loss": 2.7866,
      "step": 218050
    },
    {
      "activations/layer0_attention_weight_max": 15.958451271057129,
      "activations/layer0_attention_weight_min": -11.289549827575684,
      "activations/layer10_attention_weight_max": 34.88496017456055,
      "activations/layer10_attention_weight_min": -32.53525924682617,
      "activations/layer11_attention_weight_max": 33.14311599731445,
      "activations/layer11_attention_weight_min": -31.317073822021484,
      "activations/layer12_attention_weight_max": 23.828386306762695,
      "activations/layer12_attention_weight_min": -27.380842208862305,
      "activations/layer13_attention_weight_max": 39.2207145690918,
      "activations/layer13_attention_weight_min": -36.05291748046875,
      "activations/layer14_attention_weight_max": 38.90541458129883,
      "activations/layer14_attention_weight_min": -40.436187744140625,
      "activations/layer15_attention_weight_max": 36.54473114013672,
      "activations/layer15_attention_weight_min": -37.83085250854492,
      "activations/layer16_attention_weight_max": 31.426103591918945,
      "activations/layer16_attention_weight_min": -30.77197265625,
      "activations/layer17_attention_weight_max": 31.742212295532227,
      "activations/layer17_attention_weight_min": -26.870380401611328,
      "activations/layer18_attention_weight_max": 29.67584228515625,
      "activations/layer18_attention_weight_min": -23.288850784301758,
      "activations/layer19_attention_weight_max": 37.49038314819336,
      "activations/layer19_attention_weight_min": -30.494338989257812,
      "activations/layer1_attention_weight_max": 15.963955879211426,
      "activations/layer1_attention_weight_min": -14.658659934997559,
      "activations/layer20_attention_weight_max": 27.20290184020996,
      "activations/layer20_attention_weight_min": -24.64341926574707,
      "activations/layer21_attention_weight_max": 28.117069244384766,
      "activations/layer21_attention_weight_min": -24.640623092651367,
      "activations/layer22_attention_weight_max": 44.21571350097656,
      "activations/layer22_attention_weight_min": -30.386507034301758,
      "activations/layer23_attention_weight_max": 35.79703140258789,
      "activations/layer23_attention_weight_min": -25.436904907226562,
      "activations/layer2_attention_weight_max": 30.374343872070312,
      "activations/layer2_attention_weight_min": -31.045223236083984,
      "activations/layer3_attention_weight_max": 89.48355865478516,
      "activations/layer3_attention_weight_min": -96.9426498413086,
      "activations/layer4_attention_weight_max": 108.3331069946289,
      "activations/layer4_attention_weight_min": -111.97760009765625,
      "activations/layer5_attention_weight_max": 52.199913024902344,
      "activations/layer5_attention_weight_min": -61.865989685058594,
      "activations/layer6_attention_weight_max": 42.780941009521484,
      "activations/layer6_attention_weight_min": -46.998252868652344,
      "activations/layer7_attention_weight_max": 89.2366714477539,
      "activations/layer7_attention_weight_min": -90.46792602539062,
      "activations/layer8_attention_weight_max": 40.74235534667969,
      "activations/layer8_attention_weight_min": -39.936641693115234,
      "activations/layer9_attention_weight_max": 33.60067367553711,
      "activations/layer9_attention_weight_min": -34.23108673095703,
      "epoch": 12.67,
      "learning_rate": 6.897083333333333e-05,
      "loss": 2.7751,
      "step": 218100
    },
    {
      "activations/layer0_attention_weight_max": 16.157215118408203,
      "activations/layer0_attention_weight_min": -12.149163246154785,
      "activations/layer10_attention_weight_max": 32.99646759033203,
      "activations/layer10_attention_weight_min": -32.84349822998047,
      "activations/layer11_attention_weight_max": 32.782676696777344,
      "activations/layer11_attention_weight_min": -32.02308654785156,
      "activations/layer12_attention_weight_max": 24.882099151611328,
      "activations/layer12_attention_weight_min": -25.8147029876709,
      "activations/layer13_attention_weight_max": 39.09724044799805,
      "activations/layer13_attention_weight_min": -34.05606460571289,
      "activations/layer14_attention_weight_max": 48.152366638183594,
      "activations/layer14_attention_weight_min": -38.45075607299805,
      "activations/layer15_attention_weight_max": 40.94188690185547,
      "activations/layer15_attention_weight_min": -34.16959762573242,
      "activations/layer16_attention_weight_max": 30.814952850341797,
      "activations/layer16_attention_weight_min": -27.022544860839844,
      "activations/layer17_attention_weight_max": 35.2391242980957,
      "activations/layer17_attention_weight_min": -25.280122756958008,
      "activations/layer18_attention_weight_max": 33.89642333984375,
      "activations/layer18_attention_weight_min": -24.599943161010742,
      "activations/layer19_attention_weight_max": 35.52735900878906,
      "activations/layer19_attention_weight_min": -30.021244049072266,
      "activations/layer1_attention_weight_max": 15.96581745147705,
      "activations/layer1_attention_weight_min": -13.69616985321045,
      "activations/layer20_attention_weight_max": 28.344282150268555,
      "activations/layer20_attention_weight_min": -24.088651657104492,
      "activations/layer21_attention_weight_max": 27.041044235229492,
      "activations/layer21_attention_weight_min": -23.106782913208008,
      "activations/layer22_attention_weight_max": 38.53842544555664,
      "activations/layer22_attention_weight_min": -28.653772354125977,
      "activations/layer23_attention_weight_max": 32.11585235595703,
      "activations/layer23_attention_weight_min": -25.729772567749023,
      "activations/layer2_attention_weight_max": 30.81141471862793,
      "activations/layer2_attention_weight_min": -31.553314208984375,
      "activations/layer3_attention_weight_max": 96.61708068847656,
      "activations/layer3_attention_weight_min": -95.14763641357422,
      "activations/layer4_attention_weight_max": 112.23540496826172,
      "activations/layer4_attention_weight_min": -111.76647186279297,
      "activations/layer5_attention_weight_max": 53.38983154296875,
      "activations/layer5_attention_weight_min": -65.85530090332031,
      "activations/layer6_attention_weight_max": 43.612667083740234,
      "activations/layer6_attention_weight_min": -47.817867279052734,
      "activations/layer7_attention_weight_max": 88.08827209472656,
      "activations/layer7_attention_weight_min": -90.34217834472656,
      "activations/layer8_attention_weight_max": 40.16516876220703,
      "activations/layer8_attention_weight_min": -43.750362396240234,
      "activations/layer9_attention_weight_max": 35.507747650146484,
      "activations/layer9_attention_weight_min": -35.46774673461914,
      "epoch": 12.68,
      "learning_rate": 6.895189393939393e-05,
      "loss": 2.7792,
      "step": 218150
    },
    {
      "activations/layer0_attention_weight_max": 16.08913230895996,
      "activations/layer0_attention_weight_min": -13.047006607055664,
      "activations/layer10_attention_weight_max": 36.07917785644531,
      "activations/layer10_attention_weight_min": -35.59521484375,
      "activations/layer11_attention_weight_max": 38.15571594238281,
      "activations/layer11_attention_weight_min": -35.704811096191406,
      "activations/layer12_attention_weight_max": 27.913619995117188,
      "activations/layer12_attention_weight_min": -26.54465675354004,
      "activations/layer13_attention_weight_max": 41.92026901245117,
      "activations/layer13_attention_weight_min": -37.090450286865234,
      "activations/layer14_attention_weight_max": 50.556793212890625,
      "activations/layer14_attention_weight_min": -44.512596130371094,
      "activations/layer15_attention_weight_max": 39.29589080810547,
      "activations/layer15_attention_weight_min": -34.477054595947266,
      "activations/layer16_attention_weight_max": 33.60921859741211,
      "activations/layer16_attention_weight_min": -27.652393341064453,
      "activations/layer17_attention_weight_max": 34.11615753173828,
      "activations/layer17_attention_weight_min": -28.226205825805664,
      "activations/layer18_attention_weight_max": 35.445465087890625,
      "activations/layer18_attention_weight_min": -24.855323791503906,
      "activations/layer19_attention_weight_max": 36.41214370727539,
      "activations/layer19_attention_weight_min": -32.251976013183594,
      "activations/layer1_attention_weight_max": 16.37067413330078,
      "activations/layer1_attention_weight_min": -14.921548843383789,
      "activations/layer20_attention_weight_max": 34.60426712036133,
      "activations/layer20_attention_weight_min": -27.34160614013672,
      "activations/layer21_attention_weight_max": 32.230262756347656,
      "activations/layer21_attention_weight_min": -27.208810806274414,
      "activations/layer22_attention_weight_max": 48.65166091918945,
      "activations/layer22_attention_weight_min": -33.1502571105957,
      "activations/layer23_attention_weight_max": 38.208740234375,
      "activations/layer23_attention_weight_min": -27.686071395874023,
      "activations/layer2_attention_weight_max": 33.27699279785156,
      "activations/layer2_attention_weight_min": -30.78089141845703,
      "activations/layer3_attention_weight_max": 94.53270721435547,
      "activations/layer3_attention_weight_min": -94.55928802490234,
      "activations/layer4_attention_weight_max": 115.62391662597656,
      "activations/layer4_attention_weight_min": -112.40099334716797,
      "activations/layer5_attention_weight_max": 53.30210494995117,
      "activations/layer5_attention_weight_min": -60.65470504760742,
      "activations/layer6_attention_weight_max": 47.092567443847656,
      "activations/layer6_attention_weight_min": -51.94070816040039,
      "activations/layer7_attention_weight_max": 94.32905578613281,
      "activations/layer7_attention_weight_min": -101.31336975097656,
      "activations/layer8_attention_weight_max": 42.58133316040039,
      "activations/layer8_attention_weight_min": -42.941165924072266,
      "activations/layer9_attention_weight_max": 36.5145263671875,
      "activations/layer9_attention_weight_min": -34.70306396484375,
      "epoch": 12.68,
      "learning_rate": 6.893295454545454e-05,
      "loss": 2.7754,
      "step": 218200
    },
    {
      "activations/layer0_attention_weight_max": 14.585660934448242,
      "activations/layer0_attention_weight_min": -12.592729568481445,
      "activations/layer10_attention_weight_max": 35.11565017700195,
      "activations/layer10_attention_weight_min": -31.238290786743164,
      "activations/layer11_attention_weight_max": 34.06379699707031,
      "activations/layer11_attention_weight_min": -31.60870361328125,
      "activations/layer12_attention_weight_max": 32.545127868652344,
      "activations/layer12_attention_weight_min": -24.49578285217285,
      "activations/layer13_attention_weight_max": 39.63721466064453,
      "activations/layer13_attention_weight_min": -34.50050735473633,
      "activations/layer14_attention_weight_max": 48.587642669677734,
      "activations/layer14_attention_weight_min": -36.36793518066406,
      "activations/layer15_attention_weight_max": 37.0175895690918,
      "activations/layer15_attention_weight_min": -31.195749282836914,
      "activations/layer16_attention_weight_max": 33.71562957763672,
      "activations/layer16_attention_weight_min": -28.26984977722168,
      "activations/layer17_attention_weight_max": 32.835792541503906,
      "activations/layer17_attention_weight_min": -25.43777084350586,
      "activations/layer18_attention_weight_max": 32.73839569091797,
      "activations/layer18_attention_weight_min": -24.55162811279297,
      "activations/layer19_attention_weight_max": 34.395992279052734,
      "activations/layer19_attention_weight_min": -30.140304565429688,
      "activations/layer1_attention_weight_max": 15.701953887939453,
      "activations/layer1_attention_weight_min": -14.364227294921875,
      "activations/layer20_attention_weight_max": 32.37301254272461,
      "activations/layer20_attention_weight_min": -25.657737731933594,
      "activations/layer21_attention_weight_max": 31.424510955810547,
      "activations/layer21_attention_weight_min": -23.399328231811523,
      "activations/layer22_attention_weight_max": 45.58823013305664,
      "activations/layer22_attention_weight_min": -30.127824783325195,
      "activations/layer23_attention_weight_max": 35.33721160888672,
      "activations/layer23_attention_weight_min": -24.957164764404297,
      "activations/layer2_attention_weight_max": 30.50091552734375,
      "activations/layer2_attention_weight_min": -29.96455955505371,
      "activations/layer3_attention_weight_max": 88.28438568115234,
      "activations/layer3_attention_weight_min": -89.14346313476562,
      "activations/layer4_attention_weight_max": 107.5535659790039,
      "activations/layer4_attention_weight_min": -109.21881103515625,
      "activations/layer5_attention_weight_max": 49.44269561767578,
      "activations/layer5_attention_weight_min": -61.55420684814453,
      "activations/layer6_attention_weight_max": 42.00324249267578,
      "activations/layer6_attention_weight_min": -46.341732025146484,
      "activations/layer7_attention_weight_max": 88.25875091552734,
      "activations/layer7_attention_weight_min": -89.61831665039062,
      "activations/layer8_attention_weight_max": 39.38097381591797,
      "activations/layer8_attention_weight_min": -40.17271423339844,
      "activations/layer9_attention_weight_max": 33.55406951904297,
      "activations/layer9_attention_weight_min": -33.502254486083984,
      "epoch": 12.68,
      "learning_rate": 6.891401515151515e-05,
      "loss": 2.7614,
      "step": 218250
    },
    {
      "activations/layer0_attention_weight_max": 15.349597930908203,
      "activations/layer0_attention_weight_min": -11.892056465148926,
      "activations/layer10_attention_weight_max": 34.534393310546875,
      "activations/layer10_attention_weight_min": -33.878936767578125,
      "activations/layer11_attention_weight_max": 36.48906326293945,
      "activations/layer11_attention_weight_min": -33.32475280761719,
      "activations/layer12_attention_weight_max": 24.05645751953125,
      "activations/layer12_attention_weight_min": -25.77674674987793,
      "activations/layer13_attention_weight_max": 38.65876770019531,
      "activations/layer13_attention_weight_min": -35.10341262817383,
      "activations/layer14_attention_weight_max": 43.61023712158203,
      "activations/layer14_attention_weight_min": -37.568172454833984,
      "activations/layer15_attention_weight_max": 35.728939056396484,
      "activations/layer15_attention_weight_min": -32.875858306884766,
      "activations/layer16_attention_weight_max": 32.24246597290039,
      "activations/layer16_attention_weight_min": -29.86931037902832,
      "activations/layer17_attention_weight_max": 32.02907180786133,
      "activations/layer17_attention_weight_min": -24.93364143371582,
      "activations/layer18_attention_weight_max": 30.4664363861084,
      "activations/layer18_attention_weight_min": -26.518449783325195,
      "activations/layer19_attention_weight_max": 36.00543212890625,
      "activations/layer19_attention_weight_min": -31.09181022644043,
      "activations/layer1_attention_weight_max": 16.031187057495117,
      "activations/layer1_attention_weight_min": -14.191549301147461,
      "activations/layer20_attention_weight_max": 30.22920799255371,
      "activations/layer20_attention_weight_min": -24.558204650878906,
      "activations/layer21_attention_weight_max": 29.27398681640625,
      "activations/layer21_attention_weight_min": -24.413293838500977,
      "activations/layer22_attention_weight_max": 45.025108337402344,
      "activations/layer22_attention_weight_min": -30.636568069458008,
      "activations/layer23_attention_weight_max": 34.99351119995117,
      "activations/layer23_attention_weight_min": -25.207616806030273,
      "activations/layer2_attention_weight_max": 31.11470603942871,
      "activations/layer2_attention_weight_min": -30.71495819091797,
      "activations/layer3_attention_weight_max": 90.04509735107422,
      "activations/layer3_attention_weight_min": -92.85765075683594,
      "activations/layer4_attention_weight_max": 108.3523178100586,
      "activations/layer4_attention_weight_min": -108.80810546875,
      "activations/layer5_attention_weight_max": 49.45030975341797,
      "activations/layer5_attention_weight_min": -60.93523406982422,
      "activations/layer6_attention_weight_max": 43.85689163208008,
      "activations/layer6_attention_weight_min": -50.42417907714844,
      "activations/layer7_attention_weight_max": 86.92123413085938,
      "activations/layer7_attention_weight_min": -92.80610656738281,
      "activations/layer8_attention_weight_max": 39.72450637817383,
      "activations/layer8_attention_weight_min": -42.91255187988281,
      "activations/layer9_attention_weight_max": 31.05953025817871,
      "activations/layer9_attention_weight_min": -33.535858154296875,
      "epoch": 12.68,
      "learning_rate": 6.889507575757576e-05,
      "loss": 2.792,
      "step": 218300
    },
    {
      "activations/layer0_attention_weight_max": 15.586416244506836,
      "activations/layer0_attention_weight_min": -10.951515197753906,
      "activations/layer10_attention_weight_max": 32.955322265625,
      "activations/layer10_attention_weight_min": -32.32571029663086,
      "activations/layer11_attention_weight_max": 34.69572830200195,
      "activations/layer11_attention_weight_min": -31.950651168823242,
      "activations/layer12_attention_weight_max": 26.712833404541016,
      "activations/layer12_attention_weight_min": -25.06707763671875,
      "activations/layer13_attention_weight_max": 38.476463317871094,
      "activations/layer13_attention_weight_min": -35.04921340942383,
      "activations/layer14_attention_weight_max": 45.60567855834961,
      "activations/layer14_attention_weight_min": -41.039180755615234,
      "activations/layer15_attention_weight_max": 36.208946228027344,
      "activations/layer15_attention_weight_min": -33.13671875,
      "activations/layer16_attention_weight_max": 28.85993194580078,
      "activations/layer16_attention_weight_min": -29.58487319946289,
      "activations/layer17_attention_weight_max": 29.820329666137695,
      "activations/layer17_attention_weight_min": -25.36741065979004,
      "activations/layer18_attention_weight_max": 33.75546646118164,
      "activations/layer18_attention_weight_min": -24.759740829467773,
      "activations/layer19_attention_weight_max": 34.0824089050293,
      "activations/layer19_attention_weight_min": -31.163944244384766,
      "activations/layer1_attention_weight_max": 15.35049057006836,
      "activations/layer1_attention_weight_min": -15.154975891113281,
      "activations/layer20_attention_weight_max": 29.226654052734375,
      "activations/layer20_attention_weight_min": -25.913122177124023,
      "activations/layer21_attention_weight_max": 30.139219284057617,
      "activations/layer21_attention_weight_min": -25.083486557006836,
      "activations/layer22_attention_weight_max": 43.64480209350586,
      "activations/layer22_attention_weight_min": -30.04629135131836,
      "activations/layer23_attention_weight_max": 34.04549789428711,
      "activations/layer23_attention_weight_min": -25.949460983276367,
      "activations/layer2_attention_weight_max": 34.45634078979492,
      "activations/layer2_attention_weight_min": -31.62238883972168,
      "activations/layer3_attention_weight_max": 96.94566345214844,
      "activations/layer3_attention_weight_min": -94.66510009765625,
      "activations/layer4_attention_weight_max": 108.57991027832031,
      "activations/layer4_attention_weight_min": -109.22064208984375,
      "activations/layer5_attention_weight_max": 54.2172966003418,
      "activations/layer5_attention_weight_min": -59.40360641479492,
      "activations/layer6_attention_weight_max": 41.776641845703125,
      "activations/layer6_attention_weight_min": -47.559608459472656,
      "activations/layer7_attention_weight_max": 88.56157684326172,
      "activations/layer7_attention_weight_min": -86.38243865966797,
      "activations/layer8_attention_weight_max": 39.77017593383789,
      "activations/layer8_attention_weight_min": -44.185394287109375,
      "activations/layer9_attention_weight_max": 32.19736862182617,
      "activations/layer9_attention_weight_min": -32.665870666503906,
      "epoch": 12.69,
      "learning_rate": 6.887613636363636e-05,
      "loss": 2.7757,
      "step": 218350
    },
    {
      "activations/layer0_attention_weight_max": 15.440264701843262,
      "activations/layer0_attention_weight_min": -11.999463081359863,
      "activations/layer10_attention_weight_max": 30.428373336791992,
      "activations/layer10_attention_weight_min": -33.239253997802734,
      "activations/layer11_attention_weight_max": 31.08869171142578,
      "activations/layer11_attention_weight_min": -32.062835693359375,
      "activations/layer12_attention_weight_max": 23.410654067993164,
      "activations/layer12_attention_weight_min": -28.194494247436523,
      "activations/layer13_attention_weight_max": 36.32058334350586,
      "activations/layer13_attention_weight_min": -34.19300842285156,
      "activations/layer14_attention_weight_max": 39.90559387207031,
      "activations/layer14_attention_weight_min": -35.91115188598633,
      "activations/layer15_attention_weight_max": 36.95344924926758,
      "activations/layer15_attention_weight_min": -31.219514846801758,
      "activations/layer16_attention_weight_max": 29.151357650756836,
      "activations/layer16_attention_weight_min": -27.396249771118164,
      "activations/layer17_attention_weight_max": 29.842594146728516,
      "activations/layer17_attention_weight_min": -27.199512481689453,
      "activations/layer18_attention_weight_max": 28.700639724731445,
      "activations/layer18_attention_weight_min": -25.904996871948242,
      "activations/layer19_attention_weight_max": 33.92436599731445,
      "activations/layer19_attention_weight_min": -31.84212303161621,
      "activations/layer1_attention_weight_max": 18.274282455444336,
      "activations/layer1_attention_weight_min": -19.151124954223633,
      "activations/layer20_attention_weight_max": 27.334178924560547,
      "activations/layer20_attention_weight_min": -26.096302032470703,
      "activations/layer21_attention_weight_max": 25.88072967529297,
      "activations/layer21_attention_weight_min": -24.904130935668945,
      "activations/layer22_attention_weight_max": 39.50644302368164,
      "activations/layer22_attention_weight_min": -33.28857421875,
      "activations/layer23_attention_weight_max": 34.3607177734375,
      "activations/layer23_attention_weight_min": -27.10877799987793,
      "activations/layer2_attention_weight_max": 31.274124145507812,
      "activations/layer2_attention_weight_min": -28.38235855102539,
      "activations/layer3_attention_weight_max": 87.93695068359375,
      "activations/layer3_attention_weight_min": -87.50250244140625,
      "activations/layer4_attention_weight_max": 108.22755432128906,
      "activations/layer4_attention_weight_min": -106.88226318359375,
      "activations/layer5_attention_weight_max": 51.96466064453125,
      "activations/layer5_attention_weight_min": -61.277366638183594,
      "activations/layer6_attention_weight_max": 42.88804244995117,
      "activations/layer6_attention_weight_min": -46.65138244628906,
      "activations/layer7_attention_weight_max": 90.47618103027344,
      "activations/layer7_attention_weight_min": -90.45136260986328,
      "activations/layer8_attention_weight_max": 39.42765808105469,
      "activations/layer8_attention_weight_min": -42.18246841430664,
      "activations/layer9_attention_weight_max": 30.430570602416992,
      "activations/layer9_attention_weight_min": -35.20608139038086,
      "epoch": 12.69,
      "learning_rate": 6.885719696969696e-05,
      "loss": 2.7765,
      "step": 218400
    },
    {
      "activations/layer0_attention_weight_max": 16.119159698486328,
      "activations/layer0_attention_weight_min": -12.347064971923828,
      "activations/layer10_attention_weight_max": 31.742733001708984,
      "activations/layer10_attention_weight_min": -32.71380615234375,
      "activations/layer11_attention_weight_max": 32.72906494140625,
      "activations/layer11_attention_weight_min": -33.02250671386719,
      "activations/layer12_attention_weight_max": 24.269834518432617,
      "activations/layer12_attention_weight_min": -26.485183715820312,
      "activations/layer13_attention_weight_max": 37.45322799682617,
      "activations/layer13_attention_weight_min": -36.438716888427734,
      "activations/layer14_attention_weight_max": 42.17283248901367,
      "activations/layer14_attention_weight_min": -39.80269241333008,
      "activations/layer15_attention_weight_max": 36.87359619140625,
      "activations/layer15_attention_weight_min": -35.12260818481445,
      "activations/layer16_attention_weight_max": 33.599266052246094,
      "activations/layer16_attention_weight_min": -29.306785583496094,
      "activations/layer17_attention_weight_max": 34.21769714355469,
      "activations/layer17_attention_weight_min": -27.068721771240234,
      "activations/layer18_attention_weight_max": 32.79795455932617,
      "activations/layer18_attention_weight_min": -24.446775436401367,
      "activations/layer19_attention_weight_max": 35.93050765991211,
      "activations/layer19_attention_weight_min": -31.833524703979492,
      "activations/layer1_attention_weight_max": 16.549457550048828,
      "activations/layer1_attention_weight_min": -15.088647842407227,
      "activations/layer20_attention_weight_max": 27.970922470092773,
      "activations/layer20_attention_weight_min": -25.027658462524414,
      "activations/layer21_attention_weight_max": 28.01158905029297,
      "activations/layer21_attention_weight_min": -23.65862274169922,
      "activations/layer22_attention_weight_max": 44.274742126464844,
      "activations/layer22_attention_weight_min": -29.226749420166016,
      "activations/layer23_attention_weight_max": 35.70530700683594,
      "activations/layer23_attention_weight_min": -25.015708923339844,
      "activations/layer2_attention_weight_max": 30.47502899169922,
      "activations/layer2_attention_weight_min": -30.53634262084961,
      "activations/layer3_attention_weight_max": 89.39208221435547,
      "activations/layer3_attention_weight_min": -91.6197738647461,
      "activations/layer4_attention_weight_max": 109.49031829833984,
      "activations/layer4_attention_weight_min": -111.4910888671875,
      "activations/layer5_attention_weight_max": 52.79458236694336,
      "activations/layer5_attention_weight_min": -63.695465087890625,
      "activations/layer6_attention_weight_max": 44.668521881103516,
      "activations/layer6_attention_weight_min": -48.4078254699707,
      "activations/layer7_attention_weight_max": 84.24995422363281,
      "activations/layer7_attention_weight_min": -90.881591796875,
      "activations/layer8_attention_weight_max": 38.308631896972656,
      "activations/layer8_attention_weight_min": -41.217628479003906,
      "activations/layer9_attention_weight_max": 30.770139694213867,
      "activations/layer9_attention_weight_min": -32.98978042602539,
      "epoch": 12.69,
      "learning_rate": 6.883825757575756e-05,
      "loss": 2.7778,
      "step": 218450
    },
    {
      "activations/layer0_attention_weight_max": 15.976119995117188,
      "activations/layer0_attention_weight_min": -12.309435844421387,
      "activations/layer10_attention_weight_max": 32.726280212402344,
      "activations/layer10_attention_weight_min": -33.20858383178711,
      "activations/layer11_attention_weight_max": 32.05457305908203,
      "activations/layer11_attention_weight_min": -33.69239807128906,
      "activations/layer12_attention_weight_max": 26.155776977539062,
      "activations/layer12_attention_weight_min": -26.321226119995117,
      "activations/layer13_attention_weight_max": 39.92692947387695,
      "activations/layer13_attention_weight_min": -37.086036682128906,
      "activations/layer14_attention_weight_max": 46.347232818603516,
      "activations/layer14_attention_weight_min": -39.3766975402832,
      "activations/layer15_attention_weight_max": 36.4868278503418,
      "activations/layer15_attention_weight_min": -34.45542907714844,
      "activations/layer16_attention_weight_max": 31.695528030395508,
      "activations/layer16_attention_weight_min": -27.478225708007812,
      "activations/layer17_attention_weight_max": 33.81953430175781,
      "activations/layer17_attention_weight_min": -25.273296356201172,
      "activations/layer18_attention_weight_max": 35.97166061401367,
      "activations/layer18_attention_weight_min": -26.009817123413086,
      "activations/layer19_attention_weight_max": 35.61878967285156,
      "activations/layer19_attention_weight_min": -31.056976318359375,
      "activations/layer1_attention_weight_max": 16.31324577331543,
      "activations/layer1_attention_weight_min": -14.92386531829834,
      "activations/layer20_attention_weight_max": 31.169084548950195,
      "activations/layer20_attention_weight_min": -25.326128005981445,
      "activations/layer21_attention_weight_max": 28.78978729248047,
      "activations/layer21_attention_weight_min": -23.969446182250977,
      "activations/layer22_attention_weight_max": 49.06045913696289,
      "activations/layer22_attention_weight_min": -31.367158889770508,
      "activations/layer23_attention_weight_max": 39.594886779785156,
      "activations/layer23_attention_weight_min": -25.42201042175293,
      "activations/layer2_attention_weight_max": 29.970840454101562,
      "activations/layer2_attention_weight_min": -29.564512252807617,
      "activations/layer3_attention_weight_max": 85.12538146972656,
      "activations/layer3_attention_weight_min": -88.38622283935547,
      "activations/layer4_attention_weight_max": 103.01507568359375,
      "activations/layer4_attention_weight_min": -107.5152359008789,
      "activations/layer5_attention_weight_max": 48.69111251831055,
      "activations/layer5_attention_weight_min": -60.37119674682617,
      "activations/layer6_attention_weight_max": 45.60539245605469,
      "activations/layer6_attention_weight_min": -47.90974044799805,
      "activations/layer7_attention_weight_max": 87.51609802246094,
      "activations/layer7_attention_weight_min": -88.22685241699219,
      "activations/layer8_attention_weight_max": 39.48684310913086,
      "activations/layer8_attention_weight_min": -43.74858474731445,
      "activations/layer9_attention_weight_max": 32.421409606933594,
      "activations/layer9_attention_weight_min": -36.42148208618164,
      "epoch": 12.7,
      "learning_rate": 6.881931818181818e-05,
      "loss": 2.7741,
      "step": 218500
    },
    {
      "activations/layer0_attention_weight_max": 15.376548767089844,
      "activations/layer0_attention_weight_min": -12.344032287597656,
      "activations/layer10_attention_weight_max": 34.27323532104492,
      "activations/layer10_attention_weight_min": -32.3080940246582,
      "activations/layer11_attention_weight_max": 32.552181243896484,
      "activations/layer11_attention_weight_min": -33.29664993286133,
      "activations/layer12_attention_weight_max": 24.623817443847656,
      "activations/layer12_attention_weight_min": -25.421993255615234,
      "activations/layer13_attention_weight_max": 40.65980911254883,
      "activations/layer13_attention_weight_min": -33.99008560180664,
      "activations/layer14_attention_weight_max": 42.05308151245117,
      "activations/layer14_attention_weight_min": -37.17499923706055,
      "activations/layer15_attention_weight_max": 34.502559661865234,
      "activations/layer15_attention_weight_min": -31.828731536865234,
      "activations/layer16_attention_weight_max": 29.402767181396484,
      "activations/layer16_attention_weight_min": -26.558015823364258,
      "activations/layer17_attention_weight_max": 31.77857780456543,
      "activations/layer17_attention_weight_min": -26.449920654296875,
      "activations/layer18_attention_weight_max": 29.491065979003906,
      "activations/layer18_attention_weight_min": -23.961271286010742,
      "activations/layer19_attention_weight_max": 31.672571182250977,
      "activations/layer19_attention_weight_min": -31.867345809936523,
      "activations/layer1_attention_weight_max": 17.116870880126953,
      "activations/layer1_attention_weight_min": -14.041952133178711,
      "activations/layer20_attention_weight_max": 28.612133026123047,
      "activations/layer20_attention_weight_min": -24.83236312866211,
      "activations/layer21_attention_weight_max": 26.073570251464844,
      "activations/layer21_attention_weight_min": -24.71135902404785,
      "activations/layer22_attention_weight_max": 41.26252365112305,
      "activations/layer22_attention_weight_min": -29.71963882446289,
      "activations/layer23_attention_weight_max": 34.955078125,
      "activations/layer23_attention_weight_min": -24.430234909057617,
      "activations/layer2_attention_weight_max": 30.574630737304688,
      "activations/layer2_attention_weight_min": -30.306394577026367,
      "activations/layer3_attention_weight_max": 89.10785675048828,
      "activations/layer3_attention_weight_min": -93.7427978515625,
      "activations/layer4_attention_weight_max": 106.577392578125,
      "activations/layer4_attention_weight_min": -108.48493957519531,
      "activations/layer5_attention_weight_max": 52.26294708251953,
      "activations/layer5_attention_weight_min": -63.56245040893555,
      "activations/layer6_attention_weight_max": 44.79453659057617,
      "activations/layer6_attention_weight_min": -48.8062629699707,
      "activations/layer7_attention_weight_max": 90.60325622558594,
      "activations/layer7_attention_weight_min": -92.47196197509766,
      "activations/layer8_attention_weight_max": 41.574893951416016,
      "activations/layer8_attention_weight_min": -42.96091079711914,
      "activations/layer9_attention_weight_max": 33.34293746948242,
      "activations/layer9_attention_weight_min": -35.63193130493164,
      "epoch": 12.7,
      "learning_rate": 6.880037878787878e-05,
      "loss": 2.7998,
      "step": 218550
    },
    {
      "activations/layer0_attention_weight_max": 16.337430953979492,
      "activations/layer0_attention_weight_min": -11.857705116271973,
      "activations/layer10_attention_weight_max": 37.493560791015625,
      "activations/layer10_attention_weight_min": -35.52445983886719,
      "activations/layer11_attention_weight_max": 38.0055046081543,
      "activations/layer11_attention_weight_min": -35.3992805480957,
      "activations/layer12_attention_weight_max": 24.95804214477539,
      "activations/layer12_attention_weight_min": -24.683069229125977,
      "activations/layer13_attention_weight_max": 37.26639938354492,
      "activations/layer13_attention_weight_min": -34.794612884521484,
      "activations/layer14_attention_weight_max": 42.13875198364258,
      "activations/layer14_attention_weight_min": -40.81869888305664,
      "activations/layer15_attention_weight_max": 34.91834259033203,
      "activations/layer15_attention_weight_min": -35.790035247802734,
      "activations/layer16_attention_weight_max": 28.632034301757812,
      "activations/layer16_attention_weight_min": -28.816320419311523,
      "activations/layer17_attention_weight_max": 28.50657081604004,
      "activations/layer17_attention_weight_min": -26.004545211791992,
      "activations/layer18_attention_weight_max": 29.881113052368164,
      "activations/layer18_attention_weight_min": -24.65693473815918,
      "activations/layer19_attention_weight_max": 31.99820327758789,
      "activations/layer19_attention_weight_min": -30.550535202026367,
      "activations/layer1_attention_weight_max": 17.252483367919922,
      "activations/layer1_attention_weight_min": -15.768842697143555,
      "activations/layer20_attention_weight_max": 25.387157440185547,
      "activations/layer20_attention_weight_min": -24.440258026123047,
      "activations/layer21_attention_weight_max": 25.663461685180664,
      "activations/layer21_attention_weight_min": -23.585996627807617,
      "activations/layer22_attention_weight_max": 36.8796272277832,
      "activations/layer22_attention_weight_min": -31.20132827758789,
      "activations/layer23_attention_weight_max": 30.79286766052246,
      "activations/layer23_attention_weight_min": -23.984371185302734,
      "activations/layer2_attention_weight_max": 33.604305267333984,
      "activations/layer2_attention_weight_min": -32.588958740234375,
      "activations/layer3_attention_weight_max": 99.3955307006836,
      "activations/layer3_attention_weight_min": -98.0661849975586,
      "activations/layer4_attention_weight_max": 114.86083221435547,
      "activations/layer4_attention_weight_min": -109.4964599609375,
      "activations/layer5_attention_weight_max": 53.286529541015625,
      "activations/layer5_attention_weight_min": -61.818443298339844,
      "activations/layer6_attention_weight_max": 46.75442886352539,
      "activations/layer6_attention_weight_min": -48.88495635986328,
      "activations/layer7_attention_weight_max": 93.64412689208984,
      "activations/layer7_attention_weight_min": -96.4261703491211,
      "activations/layer8_attention_weight_max": 44.762481689453125,
      "activations/layer8_attention_weight_min": -45.13638687133789,
      "activations/layer9_attention_weight_max": 35.480472564697266,
      "activations/layer9_attention_weight_min": -34.903865814208984,
      "epoch": 12.7,
      "learning_rate": 6.878143939393938e-05,
      "loss": 2.7559,
      "step": 218600
    },
    {
      "activations/layer0_attention_weight_max": 15.346774101257324,
      "activations/layer0_attention_weight_min": -11.177754402160645,
      "activations/layer10_attention_weight_max": 32.672298431396484,
      "activations/layer10_attention_weight_min": -30.053539276123047,
      "activations/layer11_attention_weight_max": 33.53194808959961,
      "activations/layer11_attention_weight_min": -30.694900512695312,
      "activations/layer12_attention_weight_max": 26.14287567138672,
      "activations/layer12_attention_weight_min": -24.688674926757812,
      "activations/layer13_attention_weight_max": 39.892696380615234,
      "activations/layer13_attention_weight_min": -32.753257751464844,
      "activations/layer14_attention_weight_max": 42.960208892822266,
      "activations/layer14_attention_weight_min": -35.52579116821289,
      "activations/layer15_attention_weight_max": 37.18400573730469,
      "activations/layer15_attention_weight_min": -29.740875244140625,
      "activations/layer16_attention_weight_max": 34.18044662475586,
      "activations/layer16_attention_weight_min": -26.33849334716797,
      "activations/layer17_attention_weight_max": 30.55497932434082,
      "activations/layer17_attention_weight_min": -26.064157485961914,
      "activations/layer18_attention_weight_max": 33.257755279541016,
      "activations/layer18_attention_weight_min": -24.508087158203125,
      "activations/layer19_attention_weight_max": 38.722408294677734,
      "activations/layer19_attention_weight_min": -29.60658836364746,
      "activations/layer1_attention_weight_max": 16.890836715698242,
      "activations/layer1_attention_weight_min": -13.827730178833008,
      "activations/layer20_attention_weight_max": 30.242897033691406,
      "activations/layer20_attention_weight_min": -22.835569381713867,
      "activations/layer21_attention_weight_max": 31.308635711669922,
      "activations/layer21_attention_weight_min": -24.028223037719727,
      "activations/layer22_attention_weight_max": 43.66765213012695,
      "activations/layer22_attention_weight_min": -29.11473274230957,
      "activations/layer23_attention_weight_max": 35.745304107666016,
      "activations/layer23_attention_weight_min": -23.60871696472168,
      "activations/layer2_attention_weight_max": 32.8720703125,
      "activations/layer2_attention_weight_min": -30.62459945678711,
      "activations/layer3_attention_weight_max": 91.69139099121094,
      "activations/layer3_attention_weight_min": -94.80364227294922,
      "activations/layer4_attention_weight_max": 104.26322174072266,
      "activations/layer4_attention_weight_min": -107.89501953125,
      "activations/layer5_attention_weight_max": 51.60645294189453,
      "activations/layer5_attention_weight_min": -59.1798210144043,
      "activations/layer6_attention_weight_max": 41.084774017333984,
      "activations/layer6_attention_weight_min": -46.19876480102539,
      "activations/layer7_attention_weight_max": 92.02952575683594,
      "activations/layer7_attention_weight_min": -87.11576080322266,
      "activations/layer8_attention_weight_max": 40.04318618774414,
      "activations/layer8_attention_weight_min": -39.86874008178711,
      "activations/layer9_attention_weight_max": 31.972333908081055,
      "activations/layer9_attention_weight_min": -33.44364547729492,
      "epoch": 12.7,
      "learning_rate": 6.876249999999998e-05,
      "loss": 2.7711,
      "step": 218650
    },
    {
      "activations/layer0_attention_weight_max": 16.245264053344727,
      "activations/layer0_attention_weight_min": -12.3026704788208,
      "activations/layer10_attention_weight_max": 29.660934448242188,
      "activations/layer10_attention_weight_min": -30.46249771118164,
      "activations/layer11_attention_weight_max": 30.48016929626465,
      "activations/layer11_attention_weight_min": -30.128646850585938,
      "activations/layer12_attention_weight_max": 24.781463623046875,
      "activations/layer12_attention_weight_min": -23.2196102142334,
      "activations/layer13_attention_weight_max": 38.81427001953125,
      "activations/layer13_attention_weight_min": -32.348567962646484,
      "activations/layer14_attention_weight_max": 46.01471710205078,
      "activations/layer14_attention_weight_min": -35.74603271484375,
      "activations/layer15_attention_weight_max": 33.835052490234375,
      "activations/layer15_attention_weight_min": -31.46851348876953,
      "activations/layer16_attention_weight_max": 29.179763793945312,
      "activations/layer16_attention_weight_min": -27.367839813232422,
      "activations/layer17_attention_weight_max": 29.43318748474121,
      "activations/layer17_attention_weight_min": -25.81583595275879,
      "activations/layer18_attention_weight_max": 29.322622299194336,
      "activations/layer18_attention_weight_min": -23.889781951904297,
      "activations/layer19_attention_weight_max": 37.48149108886719,
      "activations/layer19_attention_weight_min": -31.64963722229004,
      "activations/layer1_attention_weight_max": 16.60978126525879,
      "activations/layer1_attention_weight_min": -14.49721622467041,
      "activations/layer20_attention_weight_max": 28.539031982421875,
      "activations/layer20_attention_weight_min": -27.2047119140625,
      "activations/layer21_attention_weight_max": 28.23948097229004,
      "activations/layer21_attention_weight_min": -25.873327255249023,
      "activations/layer22_attention_weight_max": 44.6504020690918,
      "activations/layer22_attention_weight_min": -29.227014541625977,
      "activations/layer23_attention_weight_max": 33.230186462402344,
      "activations/layer23_attention_weight_min": -24.21932601928711,
      "activations/layer2_attention_weight_max": 31.054363250732422,
      "activations/layer2_attention_weight_min": -31.350753784179688,
      "activations/layer3_attention_weight_max": 90.9153060913086,
      "activations/layer3_attention_weight_min": -96.18605041503906,
      "activations/layer4_attention_weight_max": 112.65770721435547,
      "activations/layer4_attention_weight_min": -113.37898254394531,
      "activations/layer5_attention_weight_max": 51.70277404785156,
      "activations/layer5_attention_weight_min": -59.62888717651367,
      "activations/layer6_attention_weight_max": 42.171897888183594,
      "activations/layer6_attention_weight_min": -46.44672775268555,
      "activations/layer7_attention_weight_max": 81.23695373535156,
      "activations/layer7_attention_weight_min": -86.81592559814453,
      "activations/layer8_attention_weight_max": 37.90825271606445,
      "activations/layer8_attention_weight_min": -39.27932357788086,
      "activations/layer9_attention_weight_max": 29.30706214904785,
      "activations/layer9_attention_weight_min": -31.883737564086914,
      "epoch": 12.71,
      "learning_rate": 6.87435606060606e-05,
      "loss": 2.7537,
      "step": 218700
    },
    {
      "activations/layer0_attention_weight_max": 15.979216575622559,
      "activations/layer0_attention_weight_min": -11.780255317687988,
      "activations/layer10_attention_weight_max": 34.01227569580078,
      "activations/layer10_attention_weight_min": -34.6180534362793,
      "activations/layer11_attention_weight_max": 35.205360412597656,
      "activations/layer11_attention_weight_min": -32.79167175292969,
      "activations/layer12_attention_weight_max": 38.22637176513672,
      "activations/layer12_attention_weight_min": -25.194211959838867,
      "activations/layer13_attention_weight_max": 45.01688003540039,
      "activations/layer13_attention_weight_min": -33.50202941894531,
      "activations/layer14_attention_weight_max": 44.49261474609375,
      "activations/layer14_attention_weight_min": -36.97114181518555,
      "activations/layer15_attention_weight_max": 42.01481628417969,
      "activations/layer15_attention_weight_min": -32.72021484375,
      "activations/layer16_attention_weight_max": 30.939908981323242,
      "activations/layer16_attention_weight_min": -28.860055923461914,
      "activations/layer17_attention_weight_max": 31.794015884399414,
      "activations/layer17_attention_weight_min": -24.9406681060791,
      "activations/layer18_attention_weight_max": 31.976497650146484,
      "activations/layer18_attention_weight_min": -23.646995544433594,
      "activations/layer19_attention_weight_max": 34.404415130615234,
      "activations/layer19_attention_weight_min": -31.44929313659668,
      "activations/layer1_attention_weight_max": 15.890572547912598,
      "activations/layer1_attention_weight_min": -14.34924602508545,
      "activations/layer20_attention_weight_max": 30.227094650268555,
      "activations/layer20_attention_weight_min": -29.387102127075195,
      "activations/layer21_attention_weight_max": 30.794925689697266,
      "activations/layer21_attention_weight_min": -25.18633270263672,
      "activations/layer22_attention_weight_max": 42.76227569580078,
      "activations/layer22_attention_weight_min": -33.54017639160156,
      "activations/layer23_attention_weight_max": 32.91122817993164,
      "activations/layer23_attention_weight_min": -25.761154174804688,
      "activations/layer2_attention_weight_max": 31.763431549072266,
      "activations/layer2_attention_weight_min": -30.923274993896484,
      "activations/layer3_attention_weight_max": 90.00161743164062,
      "activations/layer3_attention_weight_min": -89.87667846679688,
      "activations/layer4_attention_weight_max": 106.4514389038086,
      "activations/layer4_attention_weight_min": -105.74177551269531,
      "activations/layer5_attention_weight_max": 52.01818084716797,
      "activations/layer5_attention_weight_min": -57.715858459472656,
      "activations/layer6_attention_weight_max": 43.83469009399414,
      "activations/layer6_attention_weight_min": -45.80970764160156,
      "activations/layer7_attention_weight_max": 88.2479476928711,
      "activations/layer7_attention_weight_min": -87.08657836914062,
      "activations/layer8_attention_weight_max": 37.977012634277344,
      "activations/layer8_attention_weight_min": -41.002376556396484,
      "activations/layer9_attention_weight_max": 32.61243438720703,
      "activations/layer9_attention_weight_min": -35.84043884277344,
      "epoch": 12.71,
      "learning_rate": 6.87246212121212e-05,
      "loss": 2.7692,
      "step": 218750
    },
    {
      "activations/layer0_attention_weight_max": 16.66832733154297,
      "activations/layer0_attention_weight_min": -12.625070571899414,
      "activations/layer10_attention_weight_max": 30.17467498779297,
      "activations/layer10_attention_weight_min": -31.71898651123047,
      "activations/layer11_attention_weight_max": 32.61542510986328,
      "activations/layer11_attention_weight_min": -31.22429656982422,
      "activations/layer12_attention_weight_max": 23.07615852355957,
      "activations/layer12_attention_weight_min": -24.532175064086914,
      "activations/layer13_attention_weight_max": 35.94586944580078,
      "activations/layer13_attention_weight_min": -32.58158874511719,
      "activations/layer14_attention_weight_max": 39.67134475708008,
      "activations/layer14_attention_weight_min": -35.84858322143555,
      "activations/layer15_attention_weight_max": 36.80349349975586,
      "activations/layer15_attention_weight_min": -32.8408203125,
      "activations/layer16_attention_weight_max": 30.379968643188477,
      "activations/layer16_attention_weight_min": -27.299524307250977,
      "activations/layer17_attention_weight_max": 30.863346099853516,
      "activations/layer17_attention_weight_min": -26.073139190673828,
      "activations/layer18_attention_weight_max": 29.3569278717041,
      "activations/layer18_attention_weight_min": -25.553878784179688,
      "activations/layer19_attention_weight_max": 34.05690383911133,
      "activations/layer19_attention_weight_min": -32.252967834472656,
      "activations/layer1_attention_weight_max": 17.432647705078125,
      "activations/layer1_attention_weight_min": -16.214275360107422,
      "activations/layer20_attention_weight_max": 33.09973907470703,
      "activations/layer20_attention_weight_min": -27.768381118774414,
      "activations/layer21_attention_weight_max": 32.44620895385742,
      "activations/layer21_attention_weight_min": -25.80179214477539,
      "activations/layer22_attention_weight_max": 47.511199951171875,
      "activations/layer22_attention_weight_min": -30.67142677307129,
      "activations/layer23_attention_weight_max": 37.027034759521484,
      "activations/layer23_attention_weight_min": -25.10333251953125,
      "activations/layer2_attention_weight_max": 32.11683654785156,
      "activations/layer2_attention_weight_min": -31.72882843017578,
      "activations/layer3_attention_weight_max": 89.35094451904297,
      "activations/layer3_attention_weight_min": -89.97667694091797,
      "activations/layer4_attention_weight_max": 109.9994125366211,
      "activations/layer4_attention_weight_min": -107.51777648925781,
      "activations/layer5_attention_weight_max": 51.826080322265625,
      "activations/layer5_attention_weight_min": -61.439327239990234,
      "activations/layer6_attention_weight_max": 43.674434661865234,
      "activations/layer6_attention_weight_min": -49.91657257080078,
      "activations/layer7_attention_weight_max": 87.55802917480469,
      "activations/layer7_attention_weight_min": -85.01698303222656,
      "activations/layer8_attention_weight_max": 38.30889892578125,
      "activations/layer8_attention_weight_min": -40.56378173828125,
      "activations/layer9_attention_weight_max": 31.01645851135254,
      "activations/layer9_attention_weight_min": -32.15924835205078,
      "epoch": 12.71,
      "learning_rate": 6.870568181818182e-05,
      "loss": 2.7563,
      "step": 218800
    },
    {
      "activations/layer0_attention_weight_max": 16.7387752532959,
      "activations/layer0_attention_weight_min": -11.269440650939941,
      "activations/layer10_attention_weight_max": 34.01094436645508,
      "activations/layer10_attention_weight_min": -36.303287506103516,
      "activations/layer11_attention_weight_max": 32.60552978515625,
      "activations/layer11_attention_weight_min": -34.11363983154297,
      "activations/layer12_attention_weight_max": 25.383071899414062,
      "activations/layer12_attention_weight_min": -25.673913955688477,
      "activations/layer13_attention_weight_max": 38.247276306152344,
      "activations/layer13_attention_weight_min": -35.387535095214844,
      "activations/layer14_attention_weight_max": 43.42750549316406,
      "activations/layer14_attention_weight_min": -41.02233123779297,
      "activations/layer15_attention_weight_max": 37.19593811035156,
      "activations/layer15_attention_weight_min": -34.22113800048828,
      "activations/layer16_attention_weight_max": 30.335262298583984,
      "activations/layer16_attention_weight_min": -29.287425994873047,
      "activations/layer17_attention_weight_max": 34.57376480102539,
      "activations/layer17_attention_weight_min": -26.220476150512695,
      "activations/layer18_attention_weight_max": 33.67414093017578,
      "activations/layer18_attention_weight_min": -23.59317398071289,
      "activations/layer19_attention_weight_max": 39.950416564941406,
      "activations/layer19_attention_weight_min": -31.390064239501953,
      "activations/layer1_attention_weight_max": 16.14924430847168,
      "activations/layer1_attention_weight_min": -15.648785591125488,
      "activations/layer20_attention_weight_max": 32.3151969909668,
      "activations/layer20_attention_weight_min": -24.020000457763672,
      "activations/layer21_attention_weight_max": 31.310026168823242,
      "activations/layer21_attention_weight_min": -25.253093719482422,
      "activations/layer22_attention_weight_max": 47.12673568725586,
      "activations/layer22_attention_weight_min": -30.79191780090332,
      "activations/layer23_attention_weight_max": 39.17658996582031,
      "activations/layer23_attention_weight_min": -26.413944244384766,
      "activations/layer2_attention_weight_max": 31.290576934814453,
      "activations/layer2_attention_weight_min": -30.713245391845703,
      "activations/layer3_attention_weight_max": 91.510009765625,
      "activations/layer3_attention_weight_min": -95.84146881103516,
      "activations/layer4_attention_weight_max": 111.3197021484375,
      "activations/layer4_attention_weight_min": -111.8620376586914,
      "activations/layer5_attention_weight_max": 49.965301513671875,
      "activations/layer5_attention_weight_min": -60.28353500366211,
      "activations/layer6_attention_weight_max": 43.8942756652832,
      "activations/layer6_attention_weight_min": -49.93614959716797,
      "activations/layer7_attention_weight_max": 87.66992950439453,
      "activations/layer7_attention_weight_min": -92.49414825439453,
      "activations/layer8_attention_weight_max": 40.82273483276367,
      "activations/layer8_attention_weight_min": -42.452423095703125,
      "activations/layer9_attention_weight_max": 33.24065399169922,
      "activations/layer9_attention_weight_min": -34.021331787109375,
      "epoch": 12.72,
      "learning_rate": 6.868674242424242e-05,
      "loss": 2.7543,
      "step": 218850
    },
    {
      "activations/layer0_attention_weight_max": 15.2218017578125,
      "activations/layer0_attention_weight_min": -11.678468704223633,
      "activations/layer10_attention_weight_max": 35.52811813354492,
      "activations/layer10_attention_weight_min": -33.93840408325195,
      "activations/layer11_attention_weight_max": 35.061309814453125,
      "activations/layer11_attention_weight_min": -32.838218688964844,
      "activations/layer12_attention_weight_max": 26.403095245361328,
      "activations/layer12_attention_weight_min": -25.861066818237305,
      "activations/layer13_attention_weight_max": 40.803993225097656,
      "activations/layer13_attention_weight_min": -35.16064453125,
      "activations/layer14_attention_weight_max": 43.61138916015625,
      "activations/layer14_attention_weight_min": -37.23301315307617,
      "activations/layer15_attention_weight_max": 36.60259246826172,
      "activations/layer15_attention_weight_min": -33.442691802978516,
      "activations/layer16_attention_weight_max": 29.493824005126953,
      "activations/layer16_attention_weight_min": -27.285348892211914,
      "activations/layer17_attention_weight_max": 29.132991790771484,
      "activations/layer17_attention_weight_min": -25.289695739746094,
      "activations/layer18_attention_weight_max": 29.704622268676758,
      "activations/layer18_attention_weight_min": -23.83298683166504,
      "activations/layer19_attention_weight_max": 34.31403732299805,
      "activations/layer19_attention_weight_min": -30.930082321166992,
      "activations/layer1_attention_weight_max": 16.539175033569336,
      "activations/layer1_attention_weight_min": -15.088502883911133,
      "activations/layer20_attention_weight_max": 26.808382034301758,
      "activations/layer20_attention_weight_min": -25.213600158691406,
      "activations/layer21_attention_weight_max": 27.753665924072266,
      "activations/layer21_attention_weight_min": -25.459463119506836,
      "activations/layer22_attention_weight_max": 40.58453369140625,
      "activations/layer22_attention_weight_min": -30.837854385375977,
      "activations/layer23_attention_weight_max": 31.37636375427246,
      "activations/layer23_attention_weight_min": -26.031583786010742,
      "activations/layer2_attention_weight_max": 34.108036041259766,
      "activations/layer2_attention_weight_min": -31.642852783203125,
      "activations/layer3_attention_weight_max": 99.09085083007812,
      "activations/layer3_attention_weight_min": -97.10675811767578,
      "activations/layer4_attention_weight_max": 120.21761322021484,
      "activations/layer4_attention_weight_min": -113.2179183959961,
      "activations/layer5_attention_weight_max": 57.75463104248047,
      "activations/layer5_attention_weight_min": -64.677001953125,
      "activations/layer6_attention_weight_max": 45.82760238647461,
      "activations/layer6_attention_weight_min": -49.18996810913086,
      "activations/layer7_attention_weight_max": 98.74945831298828,
      "activations/layer7_attention_weight_min": -99.6275405883789,
      "activations/layer8_attention_weight_max": 45.33443069458008,
      "activations/layer8_attention_weight_min": -45.547035217285156,
      "activations/layer9_attention_weight_max": 36.24330520629883,
      "activations/layer9_attention_weight_min": -35.45195007324219,
      "epoch": 12.72,
      "learning_rate": 6.866780303030303e-05,
      "loss": 2.7733,
      "step": 218900
    },
    {
      "activations/layer0_attention_weight_max": 16.38629913330078,
      "activations/layer0_attention_weight_min": -11.904936790466309,
      "activations/layer10_attention_weight_max": 31.77294158935547,
      "activations/layer10_attention_weight_min": -34.353431701660156,
      "activations/layer11_attention_weight_max": 32.998748779296875,
      "activations/layer11_attention_weight_min": -34.92516326904297,
      "activations/layer12_attention_weight_max": 27.108211517333984,
      "activations/layer12_attention_weight_min": -25.532201766967773,
      "activations/layer13_attention_weight_max": 40.29595947265625,
      "activations/layer13_attention_weight_min": -38.028289794921875,
      "activations/layer14_attention_weight_max": 52.09661102294922,
      "activations/layer14_attention_weight_min": -41.665863037109375,
      "activations/layer15_attention_weight_max": 41.987144470214844,
      "activations/layer15_attention_weight_min": -34.711551666259766,
      "activations/layer16_attention_weight_max": 37.000709533691406,
      "activations/layer16_attention_weight_min": -28.30022621154785,
      "activations/layer17_attention_weight_max": 33.43577575683594,
      "activations/layer17_attention_weight_min": -25.00019645690918,
      "activations/layer18_attention_weight_max": 32.835044860839844,
      "activations/layer18_attention_weight_min": -24.305971145629883,
      "activations/layer19_attention_weight_max": 39.5360107421875,
      "activations/layer19_attention_weight_min": -31.107419967651367,
      "activations/layer1_attention_weight_max": 15.98116397857666,
      "activations/layer1_attention_weight_min": -15.068416595458984,
      "activations/layer20_attention_weight_max": 34.42295455932617,
      "activations/layer20_attention_weight_min": -24.916364669799805,
      "activations/layer21_attention_weight_max": 31.759408950805664,
      "activations/layer21_attention_weight_min": -25.344707489013672,
      "activations/layer22_attention_weight_max": 51.08784866333008,
      "activations/layer22_attention_weight_min": -30.081132888793945,
      "activations/layer23_attention_weight_max": 38.77510070800781,
      "activations/layer23_attention_weight_min": -26.503459930419922,
      "activations/layer2_attention_weight_max": 33.20233154296875,
      "activations/layer2_attention_weight_min": -30.35818099975586,
      "activations/layer3_attention_weight_max": 94.65421295166016,
      "activations/layer3_attention_weight_min": -91.67584228515625,
      "activations/layer4_attention_weight_max": 108.02489471435547,
      "activations/layer4_attention_weight_min": -116.14253997802734,
      "activations/layer5_attention_weight_max": 55.677459716796875,
      "activations/layer5_attention_weight_min": -63.88671112060547,
      "activations/layer6_attention_weight_max": 46.618587493896484,
      "activations/layer6_attention_weight_min": -48.01107406616211,
      "activations/layer7_attention_weight_max": 89.87962341308594,
      "activations/layer7_attention_weight_min": -93.08232879638672,
      "activations/layer8_attention_weight_max": 42.296749114990234,
      "activations/layer8_attention_weight_min": -43.31281280517578,
      "activations/layer9_attention_weight_max": 32.12528991699219,
      "activations/layer9_attention_weight_min": -33.381317138671875,
      "epoch": 12.72,
      "learning_rate": 6.864886363636363e-05,
      "loss": 2.7545,
      "step": 218950
    },
    {
      "activations/layer0_attention_weight_max": 16.050186157226562,
      "activations/layer0_attention_weight_min": -11.551782608032227,
      "activations/layer10_attention_weight_max": 33.514198303222656,
      "activations/layer10_attention_weight_min": -31.074092864990234,
      "activations/layer11_attention_weight_max": 32.163761138916016,
      "activations/layer11_attention_weight_min": -31.796184539794922,
      "activations/layer12_attention_weight_max": 25.64738655090332,
      "activations/layer12_attention_weight_min": -25.078704833984375,
      "activations/layer13_attention_weight_max": 41.302085876464844,
      "activations/layer13_attention_weight_min": -34.82529830932617,
      "activations/layer14_attention_weight_max": 44.79130554199219,
      "activations/layer14_attention_weight_min": -38.70595169067383,
      "activations/layer15_attention_weight_max": 41.48508071899414,
      "activations/layer15_attention_weight_min": -32.40394973754883,
      "activations/layer16_attention_weight_max": 32.623714447021484,
      "activations/layer16_attention_weight_min": -30.962011337280273,
      "activations/layer17_attention_weight_max": 30.332557678222656,
      "activations/layer17_attention_weight_min": -25.71805763244629,
      "activations/layer18_attention_weight_max": 34.216304779052734,
      "activations/layer18_attention_weight_min": -23.892932891845703,
      "activations/layer19_attention_weight_max": 32.92690658569336,
      "activations/layer19_attention_weight_min": -29.706283569335938,
      "activations/layer1_attention_weight_max": 16.36837387084961,
      "activations/layer1_attention_weight_min": -15.168585777282715,
      "activations/layer20_attention_weight_max": 28.254270553588867,
      "activations/layer20_attention_weight_min": -25.99074363708496,
      "activations/layer21_attention_weight_max": 29.745880126953125,
      "activations/layer21_attention_weight_min": -24.081680297851562,
      "activations/layer22_attention_weight_max": 41.34264373779297,
      "activations/layer22_attention_weight_min": -31.026683807373047,
      "activations/layer23_attention_weight_max": 32.57291030883789,
      "activations/layer23_attention_weight_min": -25.325109481811523,
      "activations/layer2_attention_weight_max": 31.86067008972168,
      "activations/layer2_attention_weight_min": -31.390056610107422,
      "activations/layer3_attention_weight_max": 89.87667083740234,
      "activations/layer3_attention_weight_min": -89.6381607055664,
      "activations/layer4_attention_weight_max": 106.8822021484375,
      "activations/layer4_attention_weight_min": -108.7883529663086,
      "activations/layer5_attention_weight_max": 50.263484954833984,
      "activations/layer5_attention_weight_min": -62.072601318359375,
      "activations/layer6_attention_weight_max": 43.50690841674805,
      "activations/layer6_attention_weight_min": -45.087947845458984,
      "activations/layer7_attention_weight_max": 90.74988555908203,
      "activations/layer7_attention_weight_min": -88.78205108642578,
      "activations/layer8_attention_weight_max": 40.73823165893555,
      "activations/layer8_attention_weight_min": -42.160133361816406,
      "activations/layer9_attention_weight_max": 33.12133026123047,
      "activations/layer9_attention_weight_min": -35.27060317993164,
      "epoch": 12.73,
      "learning_rate": 6.862992424242424e-05,
      "loss": 2.7662,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4284,
      "eval_samples_per_second": 509.468,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4284,
      "eval_openwebtext_samples_per_second": 509.468,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9279,
      "eval_wikitext_samples_per_second": 236.529,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.5071,
      "eval_lambada_samples_per_second": 512.142,
      "step": 219000
    },
    {
      "activations/layer0_attention_weight_max": 15.538440704345703,
      "activations/layer0_attention_weight_min": -11.640484809875488,
      "activations/layer10_attention_weight_max": 36.116519927978516,
      "activations/layer10_attention_weight_min": -34.90538787841797,
      "activations/layer11_attention_weight_max": 35.14565658569336,
      "activations/layer11_attention_weight_min": -34.155426025390625,
      "activations/layer12_attention_weight_max": 26.826818466186523,
      "activations/layer12_attention_weight_min": -26.590967178344727,
      "activations/layer13_attention_weight_max": 41.96775817871094,
      "activations/layer13_attention_weight_min": -36.142147064208984,
      "activations/layer14_attention_weight_max": 50.457515716552734,
      "activations/layer14_attention_weight_min": -40.487632751464844,
      "activations/layer15_attention_weight_max": 37.6947135925293,
      "activations/layer15_attention_weight_min": -34.620269775390625,
      "activations/layer16_attention_weight_max": 30.360319137573242,
      "activations/layer16_attention_weight_min": -27.961353302001953,
      "activations/layer17_attention_weight_max": 31.91525650024414,
      "activations/layer17_attention_weight_min": -26.734786987304688,
      "activations/layer18_attention_weight_max": 31.347551345825195,
      "activations/layer18_attention_weight_min": -26.018497467041016,
      "activations/layer19_attention_weight_max": 36.97764205932617,
      "activations/layer19_attention_weight_min": -31.9947566986084,
      "activations/layer1_attention_weight_max": 16.414186477661133,
      "activations/layer1_attention_weight_min": -17.71141815185547,
      "activations/layer20_attention_weight_max": 29.586536407470703,
      "activations/layer20_attention_weight_min": -26.108257293701172,
      "activations/layer21_attention_weight_max": 28.67864227294922,
      "activations/layer21_attention_weight_min": -24.88173484802246,
      "activations/layer22_attention_weight_max": 45.44465637207031,
      "activations/layer22_attention_weight_min": -30.680221557617188,
      "activations/layer23_attention_weight_max": 36.56615447998047,
      "activations/layer23_attention_weight_min": -26.235279083251953,
      "activations/layer2_attention_weight_max": 33.249168395996094,
      "activations/layer2_attention_weight_min": -32.21385955810547,
      "activations/layer3_attention_weight_max": 94.17234802246094,
      "activations/layer3_attention_weight_min": -94.06847381591797,
      "activations/layer4_attention_weight_max": 115.4681625366211,
      "activations/layer4_attention_weight_min": -113.82246398925781,
      "activations/layer5_attention_weight_max": 52.87408447265625,
      "activations/layer5_attention_weight_min": -60.62921905517578,
      "activations/layer6_attention_weight_max": 43.84926223754883,
      "activations/layer6_attention_weight_min": -49.37040328979492,
      "activations/layer7_attention_weight_max": 94.8172378540039,
      "activations/layer7_attention_weight_min": -93.3597183227539,
      "activations/layer8_attention_weight_max": 43.458953857421875,
      "activations/layer8_attention_weight_min": -46.01340103149414,
      "activations/layer9_attention_weight_max": 35.932579040527344,
      "activations/layer9_attention_weight_min": -35.05155563354492,
      "epoch": 12.73,
      "learning_rate": 6.861098484848485e-05,
      "loss": 2.7779,
      "step": 219050
    },
    {
      "activations/layer0_attention_weight_max": 15.411530494689941,
      "activations/layer0_attention_weight_min": -11.81901741027832,
      "activations/layer10_attention_weight_max": 30.681406021118164,
      "activations/layer10_attention_weight_min": -33.031219482421875,
      "activations/layer11_attention_weight_max": 33.184715270996094,
      "activations/layer11_attention_weight_min": -31.898448944091797,
      "activations/layer12_attention_weight_max": 25.60106086730957,
      "activations/layer12_attention_weight_min": -25.841751098632812,
      "activations/layer13_attention_weight_max": 34.233211517333984,
      "activations/layer13_attention_weight_min": -35.14070129394531,
      "activations/layer14_attention_weight_max": 38.93052673339844,
      "activations/layer14_attention_weight_min": -37.521018981933594,
      "activations/layer15_attention_weight_max": 33.2979850769043,
      "activations/layer15_attention_weight_min": -33.1528205871582,
      "activations/layer16_attention_weight_max": 29.448389053344727,
      "activations/layer16_attention_weight_min": -28.553442001342773,
      "activations/layer17_attention_weight_max": 29.99472427368164,
      "activations/layer17_attention_weight_min": -26.83838653564453,
      "activations/layer18_attention_weight_max": 28.926006317138672,
      "activations/layer18_attention_weight_min": -25.364015579223633,
      "activations/layer19_attention_weight_max": 32.48404312133789,
      "activations/layer19_attention_weight_min": -30.41497230529785,
      "activations/layer1_attention_weight_max": 15.322525024414062,
      "activations/layer1_attention_weight_min": -13.186470985412598,
      "activations/layer20_attention_weight_max": 26.23079490661621,
      "activations/layer20_attention_weight_min": -25.518348693847656,
      "activations/layer21_attention_weight_max": 25.927993774414062,
      "activations/layer21_attention_weight_min": -24.423925399780273,
      "activations/layer22_attention_weight_max": 39.15928268432617,
      "activations/layer22_attention_weight_min": -34.43654251098633,
      "activations/layer23_attention_weight_max": 32.81278610229492,
      "activations/layer23_attention_weight_min": -28.963306427001953,
      "activations/layer2_attention_weight_max": 33.88872528076172,
      "activations/layer2_attention_weight_min": -31.450572967529297,
      "activations/layer3_attention_weight_max": 96.47616577148438,
      "activations/layer3_attention_weight_min": -96.36517333984375,
      "activations/layer4_attention_weight_max": 110.7788314819336,
      "activations/layer4_attention_weight_min": -113.4538803100586,
      "activations/layer5_attention_weight_max": 50.64701843261719,
      "activations/layer5_attention_weight_min": -61.08985900878906,
      "activations/layer6_attention_weight_max": 46.33785629272461,
      "activations/layer6_attention_weight_min": -47.82035446166992,
      "activations/layer7_attention_weight_max": 88.66082000732422,
      "activations/layer7_attention_weight_min": -92.13536071777344,
      "activations/layer8_attention_weight_max": 39.46900177001953,
      "activations/layer8_attention_weight_min": -40.92453384399414,
      "activations/layer9_attention_weight_max": 31.26080894470215,
      "activations/layer9_attention_weight_min": -32.137577056884766,
      "epoch": 12.73,
      "learning_rate": 6.859204545454545e-05,
      "loss": 2.7791,
      "step": 219100
    },
    {
      "activations/layer0_attention_weight_max": 15.212822914123535,
      "activations/layer0_attention_weight_min": -12.347553253173828,
      "activations/layer10_attention_weight_max": 29.741146087646484,
      "activations/layer10_attention_weight_min": -29.703081130981445,
      "activations/layer11_attention_weight_max": 30.6984920501709,
      "activations/layer11_attention_weight_min": -31.15250015258789,
      "activations/layer12_attention_weight_max": 24.709442138671875,
      "activations/layer12_attention_weight_min": -29.867050170898438,
      "activations/layer13_attention_weight_max": 34.90168762207031,
      "activations/layer13_attention_weight_min": -31.07863998413086,
      "activations/layer14_attention_weight_max": 39.10396194458008,
      "activations/layer14_attention_weight_min": -34.75788497924805,
      "activations/layer15_attention_weight_max": 33.64308166503906,
      "activations/layer15_attention_weight_min": -31.437023162841797,
      "activations/layer16_attention_weight_max": 27.227460861206055,
      "activations/layer16_attention_weight_min": -26.368425369262695,
      "activations/layer17_attention_weight_max": 30.34695053100586,
      "activations/layer17_attention_weight_min": -25.86109733581543,
      "activations/layer18_attention_weight_max": 32.524810791015625,
      "activations/layer18_attention_weight_min": -27.55699348449707,
      "activations/layer19_attention_weight_max": 32.90785217285156,
      "activations/layer19_attention_weight_min": -31.249618530273438,
      "activations/layer1_attention_weight_max": 18.092376708984375,
      "activations/layer1_attention_weight_min": -15.505731582641602,
      "activations/layer20_attention_weight_max": 28.168071746826172,
      "activations/layer20_attention_weight_min": -24.122770309448242,
      "activations/layer21_attention_weight_max": 31.021780014038086,
      "activations/layer21_attention_weight_min": -24.53240394592285,
      "activations/layer22_attention_weight_max": 42.92249298095703,
      "activations/layer22_attention_weight_min": -30.91469955444336,
      "activations/layer23_attention_weight_max": 30.825401306152344,
      "activations/layer23_attention_weight_min": -27.8055419921875,
      "activations/layer2_attention_weight_max": 32.35209274291992,
      "activations/layer2_attention_weight_min": -31.792457580566406,
      "activations/layer3_attention_weight_max": 87.16686248779297,
      "activations/layer3_attention_weight_min": -93.3836669921875,
      "activations/layer4_attention_weight_max": 103.27699279785156,
      "activations/layer4_attention_weight_min": -103.26859283447266,
      "activations/layer5_attention_weight_max": 49.69678497314453,
      "activations/layer5_attention_weight_min": -60.770751953125,
      "activations/layer6_attention_weight_max": 44.2126579284668,
      "activations/layer6_attention_weight_min": -42.77006149291992,
      "activations/layer7_attention_weight_max": 85.84345245361328,
      "activations/layer7_attention_weight_min": -83.24530792236328,
      "activations/layer8_attention_weight_max": 37.87986755371094,
      "activations/layer8_attention_weight_min": -37.2801399230957,
      "activations/layer9_attention_weight_max": 31.135238647460938,
      "activations/layer9_attention_weight_min": -30.176321029663086,
      "epoch": 12.73,
      "learning_rate": 6.857310606060605e-05,
      "loss": 2.7759,
      "step": 219150
    },
    {
      "activations/layer0_attention_weight_max": 16.121807098388672,
      "activations/layer0_attention_weight_min": -11.375997543334961,
      "activations/layer10_attention_weight_max": 32.252254486083984,
      "activations/layer10_attention_weight_min": -30.639774322509766,
      "activations/layer11_attention_weight_max": 29.786596298217773,
      "activations/layer11_attention_weight_min": -30.112852096557617,
      "activations/layer12_attention_weight_max": 23.456684112548828,
      "activations/layer12_attention_weight_min": -24.992528915405273,
      "activations/layer13_attention_weight_max": 35.06679153442383,
      "activations/layer13_attention_weight_min": -33.70909881591797,
      "activations/layer14_attention_weight_max": 38.10072708129883,
      "activations/layer14_attention_weight_min": -38.17711639404297,
      "activations/layer15_attention_weight_max": 34.55123519897461,
      "activations/layer15_attention_weight_min": -31.88414764404297,
      "activations/layer16_attention_weight_max": 31.565343856811523,
      "activations/layer16_attention_weight_min": -29.484573364257812,
      "activations/layer17_attention_weight_max": 30.244754791259766,
      "activations/layer17_attention_weight_min": -25.39622688293457,
      "activations/layer18_attention_weight_max": 33.59054946899414,
      "activations/layer18_attention_weight_min": -24.987323760986328,
      "activations/layer19_attention_weight_max": 35.701629638671875,
      "activations/layer19_attention_weight_min": -32.42582321166992,
      "activations/layer1_attention_weight_max": 17.45781135559082,
      "activations/layer1_attention_weight_min": -14.580887794494629,
      "activations/layer20_attention_weight_max": 29.59526252746582,
      "activations/layer20_attention_weight_min": -26.729413986206055,
      "activations/layer21_attention_weight_max": 29.19878387451172,
      "activations/layer21_attention_weight_min": -25.759763717651367,
      "activations/layer22_attention_weight_max": 42.82450866699219,
      "activations/layer22_attention_weight_min": -31.064939498901367,
      "activations/layer23_attention_weight_max": 32.86846923828125,
      "activations/layer23_attention_weight_min": -27.63779067993164,
      "activations/layer2_attention_weight_max": 34.71595764160156,
      "activations/layer2_attention_weight_min": -33.28924560546875,
      "activations/layer3_attention_weight_max": 94.5332260131836,
      "activations/layer3_attention_weight_min": -96.80813598632812,
      "activations/layer4_attention_weight_max": 103.44154357910156,
      "activations/layer4_attention_weight_min": -108.3486557006836,
      "activations/layer5_attention_weight_max": 48.26433563232422,
      "activations/layer5_attention_weight_min": -58.55829620361328,
      "activations/layer6_attention_weight_max": 40.781829833984375,
      "activations/layer6_attention_weight_min": -44.575077056884766,
      "activations/layer7_attention_weight_max": 84.50818634033203,
      "activations/layer7_attention_weight_min": -82.24977111816406,
      "activations/layer8_attention_weight_max": 38.6898193359375,
      "activations/layer8_attention_weight_min": -39.897071838378906,
      "activations/layer9_attention_weight_max": 29.74146842956543,
      "activations/layer9_attention_weight_min": -31.409854888916016,
      "epoch": 12.74,
      "learning_rate": 6.855416666666665e-05,
      "loss": 2.7657,
      "step": 219200
    },
    {
      "activations/layer0_attention_weight_max": 16.40501594543457,
      "activations/layer0_attention_weight_min": -12.079879760742188,
      "activations/layer10_attention_weight_max": 33.72382354736328,
      "activations/layer10_attention_weight_min": -32.553993225097656,
      "activations/layer11_attention_weight_max": 33.8799934387207,
      "activations/layer11_attention_weight_min": -32.478790283203125,
      "activations/layer12_attention_weight_max": 26.911537170410156,
      "activations/layer12_attention_weight_min": -26.807100296020508,
      "activations/layer13_attention_weight_max": 43.20652770996094,
      "activations/layer13_attention_weight_min": -34.46892547607422,
      "activations/layer14_attention_weight_max": 49.2474365234375,
      "activations/layer14_attention_weight_min": -38.97464370727539,
      "activations/layer15_attention_weight_max": 38.04439926147461,
      "activations/layer15_attention_weight_min": -33.12810134887695,
      "activations/layer16_attention_weight_max": 32.78733825683594,
      "activations/layer16_attention_weight_min": -29.592500686645508,
      "activations/layer17_attention_weight_max": 33.3961181640625,
      "activations/layer17_attention_weight_min": -27.0814266204834,
      "activations/layer18_attention_weight_max": 33.67059326171875,
      "activations/layer18_attention_weight_min": -22.820070266723633,
      "activations/layer19_attention_weight_max": 34.041805267333984,
      "activations/layer19_attention_weight_min": -34.21435546875,
      "activations/layer1_attention_weight_max": 15.904525756835938,
      "activations/layer1_attention_weight_min": -14.914302825927734,
      "activations/layer20_attention_weight_max": 30.953800201416016,
      "activations/layer20_attention_weight_min": -23.463390350341797,
      "activations/layer21_attention_weight_max": 30.578323364257812,
      "activations/layer21_attention_weight_min": -22.807764053344727,
      "activations/layer22_attention_weight_max": 53.12437438964844,
      "activations/layer22_attention_weight_min": -29.27915382385254,
      "activations/layer23_attention_weight_max": 32.73386764526367,
      "activations/layer23_attention_weight_min": -23.555517196655273,
      "activations/layer2_attention_weight_max": 37.410911560058594,
      "activations/layer2_attention_weight_min": -33.372188568115234,
      "activations/layer3_attention_weight_max": 96.51901245117188,
      "activations/layer3_attention_weight_min": -97.2136459350586,
      "activations/layer4_attention_weight_max": 111.93709564208984,
      "activations/layer4_attention_weight_min": -114.68562316894531,
      "activations/layer5_attention_weight_max": 51.86017990112305,
      "activations/layer5_attention_weight_min": -59.91551971435547,
      "activations/layer6_attention_weight_max": 45.87102127075195,
      "activations/layer6_attention_weight_min": -48.919857025146484,
      "activations/layer7_attention_weight_max": 90.07281494140625,
      "activations/layer7_attention_weight_min": -93.73014068603516,
      "activations/layer8_attention_weight_max": 39.98651885986328,
      "activations/layer8_attention_weight_min": -41.52177810668945,
      "activations/layer9_attention_weight_max": 33.2204475402832,
      "activations/layer9_attention_weight_min": -35.39358139038086,
      "epoch": 12.74,
      "learning_rate": 6.853522727272727e-05,
      "loss": 2.7642,
      "step": 219250
    },
    {
      "activations/layer0_attention_weight_max": 16.584848403930664,
      "activations/layer0_attention_weight_min": -11.482827186584473,
      "activations/layer10_attention_weight_max": 32.3870849609375,
      "activations/layer10_attention_weight_min": -33.30207443237305,
      "activations/layer11_attention_weight_max": 33.14973831176758,
      "activations/layer11_attention_weight_min": -33.32863235473633,
      "activations/layer12_attention_weight_max": 25.356155395507812,
      "activations/layer12_attention_weight_min": -25.51575469970703,
      "activations/layer13_attention_weight_max": 39.003517150878906,
      "activations/layer13_attention_weight_min": -35.008975982666016,
      "activations/layer14_attention_weight_max": 41.9271354675293,
      "activations/layer14_attention_weight_min": -36.55632781982422,
      "activations/layer15_attention_weight_max": 36.941524505615234,
      "activations/layer15_attention_weight_min": -32.89319610595703,
      "activations/layer16_attention_weight_max": 29.764371871948242,
      "activations/layer16_attention_weight_min": -29.72791290283203,
      "activations/layer17_attention_weight_max": 34.54072189331055,
      "activations/layer17_attention_weight_min": -26.18961524963379,
      "activations/layer18_attention_weight_max": 31.400712966918945,
      "activations/layer18_attention_weight_min": -25.733814239501953,
      "activations/layer19_attention_weight_max": 33.36327362060547,
      "activations/layer19_attention_weight_min": -31.90390968322754,
      "activations/layer1_attention_weight_max": 16.443517684936523,
      "activations/layer1_attention_weight_min": -14.476125717163086,
      "activations/layer20_attention_weight_max": 28.38654136657715,
      "activations/layer20_attention_weight_min": -26.148216247558594,
      "activations/layer21_attention_weight_max": 28.123516082763672,
      "activations/layer21_attention_weight_min": -25.4063663482666,
      "activations/layer22_attention_weight_max": 48.719669342041016,
      "activations/layer22_attention_weight_min": -32.491825103759766,
      "activations/layer23_attention_weight_max": 38.69914627075195,
      "activations/layer23_attention_weight_min": -27.830368041992188,
      "activations/layer2_attention_weight_max": 32.840492248535156,
      "activations/layer2_attention_weight_min": -32.0185546875,
      "activations/layer3_attention_weight_max": 92.28121185302734,
      "activations/layer3_attention_weight_min": -94.85567474365234,
      "activations/layer4_attention_weight_max": 107.80604553222656,
      "activations/layer4_attention_weight_min": -111.41090393066406,
      "activations/layer5_attention_weight_max": 50.703125,
      "activations/layer5_attention_weight_min": -58.73133087158203,
      "activations/layer6_attention_weight_max": 44.3448600769043,
      "activations/layer6_attention_weight_min": -45.393741607666016,
      "activations/layer7_attention_weight_max": 90.244140625,
      "activations/layer7_attention_weight_min": -86.1015625,
      "activations/layer8_attention_weight_max": 40.02693557739258,
      "activations/layer8_attention_weight_min": -41.183319091796875,
      "activations/layer9_attention_weight_max": 33.31099319458008,
      "activations/layer9_attention_weight_min": -32.88056182861328,
      "epoch": 12.74,
      "learning_rate": 6.851628787878787e-05,
      "loss": 2.7796,
      "step": 219300
    },
    {
      "activations/layer0_attention_weight_max": 16.23638916015625,
      "activations/layer0_attention_weight_min": -11.113527297973633,
      "activations/layer10_attention_weight_max": 34.624725341796875,
      "activations/layer10_attention_weight_min": -34.30854415893555,
      "activations/layer11_attention_weight_max": 34.520721435546875,
      "activations/layer11_attention_weight_min": -35.01982498168945,
      "activations/layer12_attention_weight_max": 26.664878845214844,
      "activations/layer12_attention_weight_min": -26.803529739379883,
      "activations/layer13_attention_weight_max": 39.977500915527344,
      "activations/layer13_attention_weight_min": -35.34059524536133,
      "activations/layer14_attention_weight_max": 43.3339958190918,
      "activations/layer14_attention_weight_min": -37.78281021118164,
      "activations/layer15_attention_weight_max": 39.31753158569336,
      "activations/layer15_attention_weight_min": -33.45803451538086,
      "activations/layer16_attention_weight_max": 31.439640045166016,
      "activations/layer16_attention_weight_min": -28.748363494873047,
      "activations/layer17_attention_weight_max": 36.491939544677734,
      "activations/layer17_attention_weight_min": -25.81334686279297,
      "activations/layer18_attention_weight_max": 34.48457717895508,
      "activations/layer18_attention_weight_min": -23.076601028442383,
      "activations/layer19_attention_weight_max": 38.630062103271484,
      "activations/layer19_attention_weight_min": -30.774877548217773,
      "activations/layer1_attention_weight_max": 15.978795051574707,
      "activations/layer1_attention_weight_min": -14.238326072692871,
      "activations/layer20_attention_weight_max": 35.885833740234375,
      "activations/layer20_attention_weight_min": -24.965486526489258,
      "activations/layer21_attention_weight_max": 33.3640251159668,
      "activations/layer21_attention_weight_min": -24.94316291809082,
      "activations/layer22_attention_weight_max": 51.21760559082031,
      "activations/layer22_attention_weight_min": -29.262062072753906,
      "activations/layer23_attention_weight_max": 38.820068359375,
      "activations/layer23_attention_weight_min": -24.46053695678711,
      "activations/layer2_attention_weight_max": 32.114559173583984,
      "activations/layer2_attention_weight_min": -30.905372619628906,
      "activations/layer3_attention_weight_max": 95.45399475097656,
      "activations/layer3_attention_weight_min": -94.45088958740234,
      "activations/layer4_attention_weight_max": 110.50714111328125,
      "activations/layer4_attention_weight_min": -108.18538665771484,
      "activations/layer5_attention_weight_max": 51.542694091796875,
      "activations/layer5_attention_weight_min": -59.81562042236328,
      "activations/layer6_attention_weight_max": 42.90852737426758,
      "activations/layer6_attention_weight_min": -44.7215461730957,
      "activations/layer7_attention_weight_max": 87.1233901977539,
      "activations/layer7_attention_weight_min": -89.01522064208984,
      "activations/layer8_attention_weight_max": 41.32511520385742,
      "activations/layer8_attention_weight_min": -40.59550857543945,
      "activations/layer9_attention_weight_max": 36.07887649536133,
      "activations/layer9_attention_weight_min": -33.82353210449219,
      "epoch": 12.75,
      "learning_rate": 6.849734848484847e-05,
      "loss": 2.759,
      "step": 219350
    },
    {
      "activations/layer0_attention_weight_max": 15.874370574951172,
      "activations/layer0_attention_weight_min": -12.337509155273438,
      "activations/layer10_attention_weight_max": 33.91248321533203,
      "activations/layer10_attention_weight_min": -35.375732421875,
      "activations/layer11_attention_weight_max": 36.26047897338867,
      "activations/layer11_attention_weight_min": -35.99117660522461,
      "activations/layer12_attention_weight_max": 27.197660446166992,
      "activations/layer12_attention_weight_min": -26.539125442504883,
      "activations/layer13_attention_weight_max": 40.487998962402344,
      "activations/layer13_attention_weight_min": -35.095176696777344,
      "activations/layer14_attention_weight_max": 44.07082748413086,
      "activations/layer14_attention_weight_min": -39.72496032714844,
      "activations/layer15_attention_weight_max": 37.5150146484375,
      "activations/layer15_attention_weight_min": -35.27275466918945,
      "activations/layer16_attention_weight_max": 30.205793380737305,
      "activations/layer16_attention_weight_min": -27.33251190185547,
      "activations/layer17_attention_weight_max": 31.810760498046875,
      "activations/layer17_attention_weight_min": -25.035940170288086,
      "activations/layer18_attention_weight_max": 33.21515655517578,
      "activations/layer18_attention_weight_min": -24.730581283569336,
      "activations/layer19_attention_weight_max": 34.01246643066406,
      "activations/layer19_attention_weight_min": -30.85077476501465,
      "activations/layer1_attention_weight_max": 16.70557403564453,
      "activations/layer1_attention_weight_min": -15.166630744934082,
      "activations/layer20_attention_weight_max": 28.859872817993164,
      "activations/layer20_attention_weight_min": -25.302412033081055,
      "activations/layer21_attention_weight_max": 26.277782440185547,
      "activations/layer21_attention_weight_min": -24.482763290405273,
      "activations/layer22_attention_weight_max": 42.986106872558594,
      "activations/layer22_attention_weight_min": -31.539546966552734,
      "activations/layer23_attention_weight_max": 35.512794494628906,
      "activations/layer23_attention_weight_min": -26.357669830322266,
      "activations/layer2_attention_weight_max": 36.3978271484375,
      "activations/layer2_attention_weight_min": -33.76738739013672,
      "activations/layer3_attention_weight_max": 101.23038482666016,
      "activations/layer3_attention_weight_min": -102.40621185302734,
      "activations/layer4_attention_weight_max": 113.7100830078125,
      "activations/layer4_attention_weight_min": -119.80709075927734,
      "activations/layer5_attention_weight_max": 52.34332275390625,
      "activations/layer5_attention_weight_min": -64.74688720703125,
      "activations/layer6_attention_weight_max": 45.27284622192383,
      "activations/layer6_attention_weight_min": -47.32386016845703,
      "activations/layer7_attention_weight_max": 89.6644058227539,
      "activations/layer7_attention_weight_min": -93.60258483886719,
      "activations/layer8_attention_weight_max": 41.236568450927734,
      "activations/layer8_attention_weight_min": -47.63956832885742,
      "activations/layer9_attention_weight_max": 35.256072998046875,
      "activations/layer9_attention_weight_min": -36.070343017578125,
      "epoch": 12.75,
      "learning_rate": 6.847840909090907e-05,
      "loss": 2.7732,
      "step": 219400
    },
    {
      "activations/layer0_attention_weight_max": 16.751880645751953,
      "activations/layer0_attention_weight_min": -11.549749374389648,
      "activations/layer10_attention_weight_max": 33.68498611450195,
      "activations/layer10_attention_weight_min": -33.865272521972656,
      "activations/layer11_attention_weight_max": 31.523757934570312,
      "activations/layer11_attention_weight_min": -34.13407516479492,
      "activations/layer12_attention_weight_max": 24.710166931152344,
      "activations/layer12_attention_weight_min": -26.22862434387207,
      "activations/layer13_attention_weight_max": 36.92649841308594,
      "activations/layer13_attention_weight_min": -35.88255310058594,
      "activations/layer14_attention_weight_max": 41.248985290527344,
      "activations/layer14_attention_weight_min": -41.800865173339844,
      "activations/layer15_attention_weight_max": 37.064788818359375,
      "activations/layer15_attention_weight_min": -35.61235809326172,
      "activations/layer16_attention_weight_max": 31.988794326782227,
      "activations/layer16_attention_weight_min": -30.231006622314453,
      "activations/layer17_attention_weight_max": 31.42329978942871,
      "activations/layer17_attention_weight_min": -28.375844955444336,
      "activations/layer18_attention_weight_max": 31.06500244140625,
      "activations/layer18_attention_weight_min": -28.084203720092773,
      "activations/layer19_attention_weight_max": 37.278587341308594,
      "activations/layer19_attention_weight_min": -34.55570983886719,
      "activations/layer1_attention_weight_max": 16.295940399169922,
      "activations/layer1_attention_weight_min": -14.80772590637207,
      "activations/layer20_attention_weight_max": 28.565046310424805,
      "activations/layer20_attention_weight_min": -27.248271942138672,
      "activations/layer21_attention_weight_max": 27.49186134338379,
      "activations/layer21_attention_weight_min": -27.05155372619629,
      "activations/layer22_attention_weight_max": 46.3943977355957,
      "activations/layer22_attention_weight_min": -36.851619720458984,
      "activations/layer23_attention_weight_max": 36.398406982421875,
      "activations/layer23_attention_weight_min": -34.442867279052734,
      "activations/layer2_attention_weight_max": 29.965057373046875,
      "activations/layer2_attention_weight_min": -31.908035278320312,
      "activations/layer3_attention_weight_max": 94.72212982177734,
      "activations/layer3_attention_weight_min": -98.4947509765625,
      "activations/layer4_attention_weight_max": 108.04852294921875,
      "activations/layer4_attention_weight_min": -110.63825988769531,
      "activations/layer5_attention_weight_max": 50.23645782470703,
      "activations/layer5_attention_weight_min": -61.00332260131836,
      "activations/layer6_attention_weight_max": 41.60313034057617,
      "activations/layer6_attention_weight_min": -45.07341766357422,
      "activations/layer7_attention_weight_max": 91.67035675048828,
      "activations/layer7_attention_weight_min": -91.76414489746094,
      "activations/layer8_attention_weight_max": 39.52531814575195,
      "activations/layer8_attention_weight_min": -39.59138107299805,
      "activations/layer9_attention_weight_max": 32.056453704833984,
      "activations/layer9_attention_weight_min": -31.87786865234375,
      "epoch": 12.75,
      "learning_rate": 6.845946969696969e-05,
      "loss": 2.7707,
      "step": 219450
    },
    {
      "activations/layer0_attention_weight_max": 15.992436408996582,
      "activations/layer0_attention_weight_min": -11.7872953414917,
      "activations/layer10_attention_weight_max": 35.311195373535156,
      "activations/layer10_attention_weight_min": -36.13141632080078,
      "activations/layer11_attention_weight_max": 33.05650329589844,
      "activations/layer11_attention_weight_min": -34.49000549316406,
      "activations/layer12_attention_weight_max": 23.82386016845703,
      "activations/layer12_attention_weight_min": -25.93804168701172,
      "activations/layer13_attention_weight_max": 36.87843322753906,
      "activations/layer13_attention_weight_min": -35.81645965576172,
      "activations/layer14_attention_weight_max": 41.8476448059082,
      "activations/layer14_attention_weight_min": -42.94001007080078,
      "activations/layer15_attention_weight_max": 36.51530838012695,
      "activations/layer15_attention_weight_min": -35.881988525390625,
      "activations/layer16_attention_weight_max": 30.28141212463379,
      "activations/layer16_attention_weight_min": -29.268762588500977,
      "activations/layer17_attention_weight_max": 30.311248779296875,
      "activations/layer17_attention_weight_min": -27.908588409423828,
      "activations/layer18_attention_weight_max": 30.87301254272461,
      "activations/layer18_attention_weight_min": -24.60525894165039,
      "activations/layer19_attention_weight_max": 32.5825309753418,
      "activations/layer19_attention_weight_min": -32.7699089050293,
      "activations/layer1_attention_weight_max": 16.329355239868164,
      "activations/layer1_attention_weight_min": -14.42910099029541,
      "activations/layer20_attention_weight_max": 30.35076141357422,
      "activations/layer20_attention_weight_min": -27.78483772277832,
      "activations/layer21_attention_weight_max": 28.69428062438965,
      "activations/layer21_attention_weight_min": -25.012853622436523,
      "activations/layer22_attention_weight_max": 42.31706619262695,
      "activations/layer22_attention_weight_min": -33.70780563354492,
      "activations/layer23_attention_weight_max": 35.643577575683594,
      "activations/layer23_attention_weight_min": -26.981643676757812,
      "activations/layer2_attention_weight_max": 32.69685363769531,
      "activations/layer2_attention_weight_min": -32.50312805175781,
      "activations/layer3_attention_weight_max": 94.5424575805664,
      "activations/layer3_attention_weight_min": -99.02558898925781,
      "activations/layer4_attention_weight_max": 109.1203842163086,
      "activations/layer4_attention_weight_min": -111.40260314941406,
      "activations/layer5_attention_weight_max": 52.336273193359375,
      "activations/layer5_attention_weight_min": -61.61515808105469,
      "activations/layer6_attention_weight_max": 43.39755630493164,
      "activations/layer6_attention_weight_min": -47.40328598022461,
      "activations/layer7_attention_weight_max": 89.91608428955078,
      "activations/layer7_attention_weight_min": -101.45003509521484,
      "activations/layer8_attention_weight_max": 44.42283630371094,
      "activations/layer8_attention_weight_min": -44.083717346191406,
      "activations/layer9_attention_weight_max": 34.3482780456543,
      "activations/layer9_attention_weight_min": -37.08751678466797,
      "epoch": 12.75,
      "learning_rate": 6.844053030303029e-05,
      "loss": 2.7728,
      "step": 219500
    },
    {
      "activations/layer0_attention_weight_max": 15.774502754211426,
      "activations/layer0_attention_weight_min": -11.661561965942383,
      "activations/layer10_attention_weight_max": 35.89183044433594,
      "activations/layer10_attention_weight_min": -35.22760009765625,
      "activations/layer11_attention_weight_max": 35.14373779296875,
      "activations/layer11_attention_weight_min": -35.61087417602539,
      "activations/layer12_attention_weight_max": 29.032333374023438,
      "activations/layer12_attention_weight_min": -25.9776668548584,
      "activations/layer13_attention_weight_max": 45.22903060913086,
      "activations/layer13_attention_weight_min": -33.59920883178711,
      "activations/layer14_attention_weight_max": 45.73213195800781,
      "activations/layer14_attention_weight_min": -36.425758361816406,
      "activations/layer15_attention_weight_max": 38.94119644165039,
      "activations/layer15_attention_weight_min": -31.761079788208008,
      "activations/layer16_attention_weight_max": 31.066360473632812,
      "activations/layer16_attention_weight_min": -27.06157112121582,
      "activations/layer17_attention_weight_max": 31.478939056396484,
      "activations/layer17_attention_weight_min": -25.914026260375977,
      "activations/layer18_attention_weight_max": 32.496665954589844,
      "activations/layer18_attention_weight_min": -26.692996978759766,
      "activations/layer19_attention_weight_max": 36.9073371887207,
      "activations/layer19_attention_weight_min": -29.77046775817871,
      "activations/layer1_attention_weight_max": 15.49698257446289,
      "activations/layer1_attention_weight_min": -15.345574378967285,
      "activations/layer20_attention_weight_max": 28.962539672851562,
      "activations/layer20_attention_weight_min": -24.739118576049805,
      "activations/layer21_attention_weight_max": 29.50694465637207,
      "activations/layer21_attention_weight_min": -24.81467628479004,
      "activations/layer22_attention_weight_max": 49.514549255371094,
      "activations/layer22_attention_weight_min": -31.803314208984375,
      "activations/layer23_attention_weight_max": 38.54859161376953,
      "activations/layer23_attention_weight_min": -25.169584274291992,
      "activations/layer2_attention_weight_max": 31.697059631347656,
      "activations/layer2_attention_weight_min": -31.825607299804688,
      "activations/layer3_attention_weight_max": 91.81097412109375,
      "activations/layer3_attention_weight_min": -92.45965576171875,
      "activations/layer4_attention_weight_max": 107.69152069091797,
      "activations/layer4_attention_weight_min": -105.6635513305664,
      "activations/layer5_attention_weight_max": 49.685821533203125,
      "activations/layer5_attention_weight_min": -60.54893112182617,
      "activations/layer6_attention_weight_max": 42.91447830200195,
      "activations/layer6_attention_weight_min": -46.37923049926758,
      "activations/layer7_attention_weight_max": 91.6515884399414,
      "activations/layer7_attention_weight_min": -92.82736206054688,
      "activations/layer8_attention_weight_max": 43.146156311035156,
      "activations/layer8_attention_weight_min": -43.243446350097656,
      "activations/layer9_attention_weight_max": 34.54448699951172,
      "activations/layer9_attention_weight_min": -34.672325134277344,
      "epoch": 12.76,
      "learning_rate": 6.84215909090909e-05,
      "loss": 2.7636,
      "step": 219550
    },
    {
      "activations/layer0_attention_weight_max": 16.3685302734375,
      "activations/layer0_attention_weight_min": -12.160335540771484,
      "activations/layer10_attention_weight_max": 43.82355880737305,
      "activations/layer10_attention_weight_min": -42.24924850463867,
      "activations/layer11_attention_weight_max": 44.591453552246094,
      "activations/layer11_attention_weight_min": -45.17666244506836,
      "activations/layer12_attention_weight_max": 30.868541717529297,
      "activations/layer12_attention_weight_min": -28.6915225982666,
      "activations/layer13_attention_weight_max": 44.309200286865234,
      "activations/layer13_attention_weight_min": -38.75746536254883,
      "activations/layer14_attention_weight_max": 50.51162338256836,
      "activations/layer14_attention_weight_min": -45.41257095336914,
      "activations/layer15_attention_weight_max": 52.066368103027344,
      "activations/layer15_attention_weight_min": -39.81501770019531,
      "activations/layer16_attention_weight_max": 32.7877197265625,
      "activations/layer16_attention_weight_min": -28.020965576171875,
      "activations/layer17_attention_weight_max": 28.2247257232666,
      "activations/layer17_attention_weight_min": -28.623491287231445,
      "activations/layer18_attention_weight_max": 29.878700256347656,
      "activations/layer18_attention_weight_min": -24.719654083251953,
      "activations/layer19_attention_weight_max": 35.22835922241211,
      "activations/layer19_attention_weight_min": -29.331195831298828,
      "activations/layer1_attention_weight_max": 15.385843276977539,
      "activations/layer1_attention_weight_min": -16.337501525878906,
      "activations/layer20_attention_weight_max": 29.702730178833008,
      "activations/layer20_attention_weight_min": -22.77888298034668,
      "activations/layer21_attention_weight_max": 27.553409576416016,
      "activations/layer21_attention_weight_min": -23.95159912109375,
      "activations/layer22_attention_weight_max": 46.09205627441406,
      "activations/layer22_attention_weight_min": -31.022708892822266,
      "activations/layer23_attention_weight_max": 30.42743492126465,
      "activations/layer23_attention_weight_min": -27.862659454345703,
      "activations/layer2_attention_weight_max": 37.346336364746094,
      "activations/layer2_attention_weight_min": -34.976036071777344,
      "activations/layer3_attention_weight_max": 104.31100463867188,
      "activations/layer3_attention_weight_min": -103.80221557617188,
      "activations/layer4_attention_weight_max": 119.8218002319336,
      "activations/layer4_attention_weight_min": -113.00231170654297,
      "activations/layer5_attention_weight_max": 54.72004318237305,
      "activations/layer5_attention_weight_min": -62.74815368652344,
      "activations/layer6_attention_weight_max": 45.70100784301758,
      "activations/layer6_attention_weight_min": -47.64394760131836,
      "activations/layer7_attention_weight_max": 104.00628662109375,
      "activations/layer7_attention_weight_min": -96.6871109008789,
      "activations/layer8_attention_weight_max": 48.50776672363281,
      "activations/layer8_attention_weight_min": -46.033294677734375,
      "activations/layer9_attention_weight_max": 41.1838493347168,
      "activations/layer9_attention_weight_min": -41.60497283935547,
      "epoch": 12.76,
      "learning_rate": 6.840265151515151e-05,
      "loss": 2.7907,
      "step": 219600
    },
    {
      "activations/layer0_attention_weight_max": 15.524269104003906,
      "activations/layer0_attention_weight_min": -12.203033447265625,
      "activations/layer10_attention_weight_max": 33.243186950683594,
      "activations/layer10_attention_weight_min": -32.57630157470703,
      "activations/layer11_attention_weight_max": 31.705869674682617,
      "activations/layer11_attention_weight_min": -31.687734603881836,
      "activations/layer12_attention_weight_max": 24.075374603271484,
      "activations/layer12_attention_weight_min": -25.58653450012207,
      "activations/layer13_attention_weight_max": 37.19691848754883,
      "activations/layer13_attention_weight_min": -35.0408935546875,
      "activations/layer14_attention_weight_max": 39.5266227722168,
      "activations/layer14_attention_weight_min": -36.01194381713867,
      "activations/layer15_attention_weight_max": 33.901493072509766,
      "activations/layer15_attention_weight_min": -32.09663772583008,
      "activations/layer16_attention_weight_max": 28.614606857299805,
      "activations/layer16_attention_weight_min": -27.74895668029785,
      "activations/layer17_attention_weight_max": 30.208234786987305,
      "activations/layer17_attention_weight_min": -27.241867065429688,
      "activations/layer18_attention_weight_max": 29.313405990600586,
      "activations/layer18_attention_weight_min": -24.633264541625977,
      "activations/layer19_attention_weight_max": 33.44108200073242,
      "activations/layer19_attention_weight_min": -31.95932960510254,
      "activations/layer1_attention_weight_max": 16.533267974853516,
      "activations/layer1_attention_weight_min": -15.000356674194336,
      "activations/layer20_attention_weight_max": 28.589876174926758,
      "activations/layer20_attention_weight_min": -27.06524658203125,
      "activations/layer21_attention_weight_max": 27.309804916381836,
      "activations/layer21_attention_weight_min": -25.970396041870117,
      "activations/layer22_attention_weight_max": 41.31981658935547,
      "activations/layer22_attention_weight_min": -30.14790916442871,
      "activations/layer23_attention_weight_max": 34.66374206542969,
      "activations/layer23_attention_weight_min": -26.053442001342773,
      "activations/layer2_attention_weight_max": 32.539554595947266,
      "activations/layer2_attention_weight_min": -31.57752799987793,
      "activations/layer3_attention_weight_max": 95.74392700195312,
      "activations/layer3_attention_weight_min": -95.22709655761719,
      "activations/layer4_attention_weight_max": 107.81988525390625,
      "activations/layer4_attention_weight_min": -113.13215637207031,
      "activations/layer5_attention_weight_max": 51.34339904785156,
      "activations/layer5_attention_weight_min": -58.72710418701172,
      "activations/layer6_attention_weight_max": 45.64457702636719,
      "activations/layer6_attention_weight_min": -46.363948822021484,
      "activations/layer7_attention_weight_max": 87.5193099975586,
      "activations/layer7_attention_weight_min": -87.77406311035156,
      "activations/layer8_attention_weight_max": 41.65779113769531,
      "activations/layer8_attention_weight_min": -42.58429718017578,
      "activations/layer9_attention_weight_max": 34.9857292175293,
      "activations/layer9_attention_weight_min": -32.95528793334961,
      "epoch": 12.76,
      "learning_rate": 6.838371212121212e-05,
      "loss": 2.7739,
      "step": 219650
    },
    {
      "activations/layer0_attention_weight_max": 15.277024269104004,
      "activations/layer0_attention_weight_min": -11.602130889892578,
      "activations/layer10_attention_weight_max": 32.753013610839844,
      "activations/layer10_attention_weight_min": -35.183597564697266,
      "activations/layer11_attention_weight_max": 34.32977294921875,
      "activations/layer11_attention_weight_min": -32.789581298828125,
      "activations/layer12_attention_weight_max": 27.974361419677734,
      "activations/layer12_attention_weight_min": -26.143091201782227,
      "activations/layer13_attention_weight_max": 44.1982307434082,
      "activations/layer13_attention_weight_min": -36.7414665222168,
      "activations/layer14_attention_weight_max": 45.05787658691406,
      "activations/layer14_attention_weight_min": -39.04646682739258,
      "activations/layer15_attention_weight_max": 45.11985778808594,
      "activations/layer15_attention_weight_min": -33.41798400878906,
      "activations/layer16_attention_weight_max": 29.806943893432617,
      "activations/layer16_attention_weight_min": -28.60685920715332,
      "activations/layer17_attention_weight_max": 31.10179328918457,
      "activations/layer17_attention_weight_min": -26.59518814086914,
      "activations/layer18_attention_weight_max": 30.583927154541016,
      "activations/layer18_attention_weight_min": -26.13473129272461,
      "activations/layer19_attention_weight_max": 35.85447311401367,
      "activations/layer19_attention_weight_min": -32.6168327331543,
      "activations/layer1_attention_weight_max": 16.447002410888672,
      "activations/layer1_attention_weight_min": -15.585609436035156,
      "activations/layer20_attention_weight_max": 31.0755558013916,
      "activations/layer20_attention_weight_min": -29.017833709716797,
      "activations/layer21_attention_weight_max": 32.72163772583008,
      "activations/layer21_attention_weight_min": -25.709850311279297,
      "activations/layer22_attention_weight_max": 49.1007080078125,
      "activations/layer22_attention_weight_min": -30.64850425720215,
      "activations/layer23_attention_weight_max": 34.26348114013672,
      "activations/layer23_attention_weight_min": -26.658966064453125,
      "activations/layer2_attention_weight_max": 32.237884521484375,
      "activations/layer2_attention_weight_min": -31.391929626464844,
      "activations/layer3_attention_weight_max": 93.73434448242188,
      "activations/layer3_attention_weight_min": -95.09010314941406,
      "activations/layer4_attention_weight_max": 108.59212493896484,
      "activations/layer4_attention_weight_min": -108.35530853271484,
      "activations/layer5_attention_weight_max": 53.34013366699219,
      "activations/layer5_attention_weight_min": -59.63690948486328,
      "activations/layer6_attention_weight_max": 44.787391662597656,
      "activations/layer6_attention_weight_min": -46.88542938232422,
      "activations/layer7_attention_weight_max": 95.27671813964844,
      "activations/layer7_attention_weight_min": -88.13416290283203,
      "activations/layer8_attention_weight_max": 40.55086135864258,
      "activations/layer8_attention_weight_min": -44.153690338134766,
      "activations/layer9_attention_weight_max": 34.77328872680664,
      "activations/layer9_attention_weight_min": -33.72158432006836,
      "epoch": 12.77,
      "learning_rate": 6.836477272727272e-05,
      "loss": 2.7716,
      "step": 219700
    },
    {
      "activations/layer0_attention_weight_max": 14.912845611572266,
      "activations/layer0_attention_weight_min": -11.4328031539917,
      "activations/layer10_attention_weight_max": 31.00090789794922,
      "activations/layer10_attention_weight_min": -33.3206901550293,
      "activations/layer11_attention_weight_max": 31.816001892089844,
      "activations/layer11_attention_weight_min": -33.05900955200195,
      "activations/layer12_attention_weight_max": 23.607601165771484,
      "activations/layer12_attention_weight_min": -24.81620216369629,
      "activations/layer13_attention_weight_max": 37.28036117553711,
      "activations/layer13_attention_weight_min": -33.943233489990234,
      "activations/layer14_attention_weight_max": 45.83824157714844,
      "activations/layer14_attention_weight_min": -38.11912536621094,
      "activations/layer15_attention_weight_max": 36.1921272277832,
      "activations/layer15_attention_weight_min": -31.860244750976562,
      "activations/layer16_attention_weight_max": 30.11870574951172,
      "activations/layer16_attention_weight_min": -26.996707916259766,
      "activations/layer17_attention_weight_max": 32.31781005859375,
      "activations/layer17_attention_weight_min": -26.371538162231445,
      "activations/layer18_attention_weight_max": 37.41074752807617,
      "activations/layer18_attention_weight_min": -23.653200149536133,
      "activations/layer19_attention_weight_max": 36.6997184753418,
      "activations/layer19_attention_weight_min": -33.08759689331055,
      "activations/layer1_attention_weight_max": 16.652462005615234,
      "activations/layer1_attention_weight_min": -14.328313827514648,
      "activations/layer20_attention_weight_max": 32.757869720458984,
      "activations/layer20_attention_weight_min": -26.597867965698242,
      "activations/layer21_attention_weight_max": 29.38700294494629,
      "activations/layer21_attention_weight_min": -25.279766082763672,
      "activations/layer22_attention_weight_max": 49.09473419189453,
      "activations/layer22_attention_weight_min": -32.21412658691406,
      "activations/layer23_attention_weight_max": 38.10942077636719,
      "activations/layer23_attention_weight_min": -28.60219955444336,
      "activations/layer2_attention_weight_max": 32.49835968017578,
      "activations/layer2_attention_weight_min": -32.635372161865234,
      "activations/layer3_attention_weight_max": 95.18123626708984,
      "activations/layer3_attention_weight_min": -96.56315612792969,
      "activations/layer4_attention_weight_max": 111.10662078857422,
      "activations/layer4_attention_weight_min": -113.9616928100586,
      "activations/layer5_attention_weight_max": 50.6656608581543,
      "activations/layer5_attention_weight_min": -63.18663024902344,
      "activations/layer6_attention_weight_max": 43.0379524230957,
      "activations/layer6_attention_weight_min": -50.212711334228516,
      "activations/layer7_attention_weight_max": 85.58749389648438,
      "activations/layer7_attention_weight_min": -98.72071838378906,
      "activations/layer8_attention_weight_max": 37.70672607421875,
      "activations/layer8_attention_weight_min": -40.10665512084961,
      "activations/layer9_attention_weight_max": 30.760847091674805,
      "activations/layer9_attention_weight_min": -33.14928436279297,
      "epoch": 12.77,
      "learning_rate": 6.834583333333333e-05,
      "loss": 2.774,
      "step": 219750
    },
    {
      "activations/layer0_attention_weight_max": 16.370424270629883,
      "activations/layer0_attention_weight_min": -11.682355880737305,
      "activations/layer10_attention_weight_max": 31.663253784179688,
      "activations/layer10_attention_weight_min": -32.02276611328125,
      "activations/layer11_attention_weight_max": 32.90938186645508,
      "activations/layer11_attention_weight_min": -31.49437713623047,
      "activations/layer12_attention_weight_max": 25.136022567749023,
      "activations/layer12_attention_weight_min": -26.00299835205078,
      "activations/layer13_attention_weight_max": 37.47117614746094,
      "activations/layer13_attention_weight_min": -36.93251419067383,
      "activations/layer14_attention_weight_max": 42.64829635620117,
      "activations/layer14_attention_weight_min": -40.176029205322266,
      "activations/layer15_attention_weight_max": 36.57381057739258,
      "activations/layer15_attention_weight_min": -32.50935363769531,
      "activations/layer16_attention_weight_max": 28.79094886779785,
      "activations/layer16_attention_weight_min": -27.388925552368164,
      "activations/layer17_attention_weight_max": 31.19577980041504,
      "activations/layer17_attention_weight_min": -25.9385986328125,
      "activations/layer18_attention_weight_max": 32.25334548950195,
      "activations/layer18_attention_weight_min": -26.01706886291504,
      "activations/layer19_attention_weight_max": 33.35213851928711,
      "activations/layer19_attention_weight_min": -31.535486221313477,
      "activations/layer1_attention_weight_max": 16.14933967590332,
      "activations/layer1_attention_weight_min": -15.06339168548584,
      "activations/layer20_attention_weight_max": 28.443214416503906,
      "activations/layer20_attention_weight_min": -26.897844314575195,
      "activations/layer21_attention_weight_max": 29.50517463684082,
      "activations/layer21_attention_weight_min": -24.528892517089844,
      "activations/layer22_attention_weight_max": 42.93247985839844,
      "activations/layer22_attention_weight_min": -30.179380416870117,
      "activations/layer23_attention_weight_max": 34.79594421386719,
      "activations/layer23_attention_weight_min": -26.91085433959961,
      "activations/layer2_attention_weight_max": 32.062564849853516,
      "activations/layer2_attention_weight_min": -31.383377075195312,
      "activations/layer3_attention_weight_max": 92.29615020751953,
      "activations/layer3_attention_weight_min": -94.72917175292969,
      "activations/layer4_attention_weight_max": 108.9188003540039,
      "activations/layer4_attention_weight_min": -111.71492004394531,
      "activations/layer5_attention_weight_max": 49.70912170410156,
      "activations/layer5_attention_weight_min": -60.101409912109375,
      "activations/layer6_attention_weight_max": 43.80449295043945,
      "activations/layer6_attention_weight_min": -49.088497161865234,
      "activations/layer7_attention_weight_max": 85.0470962524414,
      "activations/layer7_attention_weight_min": -93.20052337646484,
      "activations/layer8_attention_weight_max": 41.60262680053711,
      "activations/layer8_attention_weight_min": -45.20398712158203,
      "activations/layer9_attention_weight_max": 32.19334411621094,
      "activations/layer9_attention_weight_min": -32.12467956542969,
      "epoch": 12.77,
      "learning_rate": 6.832689393939394e-05,
      "loss": 2.7614,
      "step": 219800
    },
    {
      "activations/layer0_attention_weight_max": 15.979852676391602,
      "activations/layer0_attention_weight_min": -11.936079025268555,
      "activations/layer10_attention_weight_max": 34.71361541748047,
      "activations/layer10_attention_weight_min": -35.56951904296875,
      "activations/layer11_attention_weight_max": 34.950592041015625,
      "activations/layer11_attention_weight_min": -33.95782470703125,
      "activations/layer12_attention_weight_max": 27.83816909790039,
      "activations/layer12_attention_weight_min": -26.897550582885742,
      "activations/layer13_attention_weight_max": 42.09983825683594,
      "activations/layer13_attention_weight_min": -34.55720138549805,
      "activations/layer14_attention_weight_max": 45.7420768737793,
      "activations/layer14_attention_weight_min": -37.45134353637695,
      "activations/layer15_attention_weight_max": 38.304962158203125,
      "activations/layer15_attention_weight_min": -33.35616683959961,
      "activations/layer16_attention_weight_max": 32.10569381713867,
      "activations/layer16_attention_weight_min": -26.60142707824707,
      "activations/layer17_attention_weight_max": 33.98927307128906,
      "activations/layer17_attention_weight_min": -26.1645450592041,
      "activations/layer18_attention_weight_max": 32.8493537902832,
      "activations/layer18_attention_weight_min": -23.748632431030273,
      "activations/layer19_attention_weight_max": 34.456661224365234,
      "activations/layer19_attention_weight_min": -29.686368942260742,
      "activations/layer1_attention_weight_max": 17.020835876464844,
      "activations/layer1_attention_weight_min": -14.808030128479004,
      "activations/layer20_attention_weight_max": 32.976722717285156,
      "activations/layer20_attention_weight_min": -25.277482986450195,
      "activations/layer21_attention_weight_max": 32.25218200683594,
      "activations/layer21_attention_weight_min": -25.5560245513916,
      "activations/layer22_attention_weight_max": 47.37995147705078,
      "activations/layer22_attention_weight_min": -29.965377807617188,
      "activations/layer23_attention_weight_max": 34.617286682128906,
      "activations/layer23_attention_weight_min": -25.944042205810547,
      "activations/layer2_attention_weight_max": 32.55094909667969,
      "activations/layer2_attention_weight_min": -32.93435287475586,
      "activations/layer3_attention_weight_max": 93.90612030029297,
      "activations/layer3_attention_weight_min": -95.55056762695312,
      "activations/layer4_attention_weight_max": 114.13390350341797,
      "activations/layer4_attention_weight_min": -116.1115493774414,
      "activations/layer5_attention_weight_max": 51.92609405517578,
      "activations/layer5_attention_weight_min": -63.64025115966797,
      "activations/layer6_attention_weight_max": 44.737918853759766,
      "activations/layer6_attention_weight_min": -46.28731155395508,
      "activations/layer7_attention_weight_max": 92.4342269897461,
      "activations/layer7_attention_weight_min": -91.5537338256836,
      "activations/layer8_attention_weight_max": 41.740665435791016,
      "activations/layer8_attention_weight_min": -43.38555145263672,
      "activations/layer9_attention_weight_max": 34.99592971801758,
      "activations/layer9_attention_weight_min": -35.74067306518555,
      "epoch": 12.77,
      "learning_rate": 6.830795454545454e-05,
      "loss": 2.7595,
      "step": 219850
    },
    {
      "activations/layer0_attention_weight_max": 15.734188079833984,
      "activations/layer0_attention_weight_min": -11.69986629486084,
      "activations/layer10_attention_weight_max": 31.056711196899414,
      "activations/layer10_attention_weight_min": -31.192840576171875,
      "activations/layer11_attention_weight_max": 31.630756378173828,
      "activations/layer11_attention_weight_min": -30.5084228515625,
      "activations/layer12_attention_weight_max": 23.602798461914062,
      "activations/layer12_attention_weight_min": -25.21601104736328,
      "activations/layer13_attention_weight_max": 34.560569763183594,
      "activations/layer13_attention_weight_min": -33.07110595703125,
      "activations/layer14_attention_weight_max": 39.6606330871582,
      "activations/layer14_attention_weight_min": -37.76047134399414,
      "activations/layer15_attention_weight_max": 35.03664779663086,
      "activations/layer15_attention_weight_min": -31.603595733642578,
      "activations/layer16_attention_weight_max": 28.87205696105957,
      "activations/layer16_attention_weight_min": -27.782669067382812,
      "activations/layer17_attention_weight_max": 29.39333152770996,
      "activations/layer17_attention_weight_min": -24.133956909179688,
      "activations/layer18_attention_weight_max": 29.112337112426758,
      "activations/layer18_attention_weight_min": -22.857873916625977,
      "activations/layer19_attention_weight_max": 32.4649543762207,
      "activations/layer19_attention_weight_min": -32.25175094604492,
      "activations/layer1_attention_weight_max": 15.94189167022705,
      "activations/layer1_attention_weight_min": -13.902373313903809,
      "activations/layer20_attention_weight_max": 29.132781982421875,
      "activations/layer20_attention_weight_min": -27.147350311279297,
      "activations/layer21_attention_weight_max": 29.07121467590332,
      "activations/layer21_attention_weight_min": -25.046714782714844,
      "activations/layer22_attention_weight_max": 44.03361129760742,
      "activations/layer22_attention_weight_min": -29.72093391418457,
      "activations/layer23_attention_weight_max": 33.49237060546875,
      "activations/layer23_attention_weight_min": -23.198598861694336,
      "activations/layer2_attention_weight_max": 31.80672264099121,
      "activations/layer2_attention_weight_min": -30.824756622314453,
      "activations/layer3_attention_weight_max": 91.64138793945312,
      "activations/layer3_attention_weight_min": -90.5081558227539,
      "activations/layer4_attention_weight_max": 111.66426086425781,
      "activations/layer4_attention_weight_min": -108.02965545654297,
      "activations/layer5_attention_weight_max": 48.83977508544922,
      "activations/layer5_attention_weight_min": -57.787471771240234,
      "activations/layer6_attention_weight_max": 40.786277770996094,
      "activations/layer6_attention_weight_min": -46.416053771972656,
      "activations/layer7_attention_weight_max": 86.09176635742188,
      "activations/layer7_attention_weight_min": -88.54907989501953,
      "activations/layer8_attention_weight_max": 38.32820129394531,
      "activations/layer8_attention_weight_min": -38.899330139160156,
      "activations/layer9_attention_weight_max": 30.45369529724121,
      "activations/layer9_attention_weight_min": -32.29267883300781,
      "epoch": 12.78,
      "learning_rate": 6.828901515151514e-05,
      "loss": 2.78,
      "step": 219900
    },
    {
      "activations/layer0_attention_weight_max": 15.992508888244629,
      "activations/layer0_attention_weight_min": -11.990424156188965,
      "activations/layer10_attention_weight_max": 34.04914474487305,
      "activations/layer10_attention_weight_min": -32.81316375732422,
      "activations/layer11_attention_weight_max": 34.57110595703125,
      "activations/layer11_attention_weight_min": -31.318105697631836,
      "activations/layer12_attention_weight_max": 24.859214782714844,
      "activations/layer12_attention_weight_min": -24.736568450927734,
      "activations/layer13_attention_weight_max": 36.90386199951172,
      "activations/layer13_attention_weight_min": -35.311859130859375,
      "activations/layer14_attention_weight_max": 43.7303352355957,
      "activations/layer14_attention_weight_min": -36.123077392578125,
      "activations/layer15_attention_weight_max": 35.109092712402344,
      "activations/layer15_attention_weight_min": -32.116912841796875,
      "activations/layer16_attention_weight_max": 32.424232482910156,
      "activations/layer16_attention_weight_min": -27.068330764770508,
      "activations/layer17_attention_weight_max": 29.26253890991211,
      "activations/layer17_attention_weight_min": -26.96026039123535,
      "activations/layer18_attention_weight_max": 33.12055587768555,
      "activations/layer18_attention_weight_min": -26.676372528076172,
      "activations/layer19_attention_weight_max": 32.87260437011719,
      "activations/layer19_attention_weight_min": -32.06588363647461,
      "activations/layer1_attention_weight_max": 16.58487319946289,
      "activations/layer1_attention_weight_min": -16.04578399658203,
      "activations/layer20_attention_weight_max": 29.285816192626953,
      "activations/layer20_attention_weight_min": -26.25337791442871,
      "activations/layer21_attention_weight_max": 28.833526611328125,
      "activations/layer21_attention_weight_min": -25.59893035888672,
      "activations/layer22_attention_weight_max": 38.689090728759766,
      "activations/layer22_attention_weight_min": -33.4300422668457,
      "activations/layer23_attention_weight_max": 31.590896606445312,
      "activations/layer23_attention_weight_min": -27.52728271484375,
      "activations/layer2_attention_weight_max": 31.378414154052734,
      "activations/layer2_attention_weight_min": -31.43323516845703,
      "activations/layer3_attention_weight_max": 95.4777603149414,
      "activations/layer3_attention_weight_min": -97.40670776367188,
      "activations/layer4_attention_weight_max": 107.81717681884766,
      "activations/layer4_attention_weight_min": -106.0228500366211,
      "activations/layer5_attention_weight_max": 51.77186584472656,
      "activations/layer5_attention_weight_min": -60.57343673706055,
      "activations/layer6_attention_weight_max": 46.64453887939453,
      "activations/layer6_attention_weight_min": -46.78642654418945,
      "activations/layer7_attention_weight_max": 95.88117218017578,
      "activations/layer7_attention_weight_min": -94.18679809570312,
      "activations/layer8_attention_weight_max": 42.89426803588867,
      "activations/layer8_attention_weight_min": -42.27815628051758,
      "activations/layer9_attention_weight_max": 36.19976806640625,
      "activations/layer9_attention_weight_min": -33.30534362792969,
      "epoch": 12.78,
      "learning_rate": 6.827007575757574e-05,
      "loss": 2.7724,
      "step": 219950
    },
    {
      "activations/layer0_attention_weight_max": 15.573009490966797,
      "activations/layer0_attention_weight_min": -11.806696891784668,
      "activations/layer10_attention_weight_max": 35.698272705078125,
      "activations/layer10_attention_weight_min": -34.28424835205078,
      "activations/layer11_attention_weight_max": 33.556785583496094,
      "activations/layer11_attention_weight_min": -35.69718933105469,
      "activations/layer12_attention_weight_max": 26.46517562866211,
      "activations/layer12_attention_weight_min": -25.98801040649414,
      "activations/layer13_attention_weight_max": 42.82315444946289,
      "activations/layer13_attention_weight_min": -36.30646514892578,
      "activations/layer14_attention_weight_max": 49.8753662109375,
      "activations/layer14_attention_weight_min": -39.9071159362793,
      "activations/layer15_attention_weight_max": 41.31999206542969,
      "activations/layer15_attention_weight_min": -35.12799072265625,
      "activations/layer16_attention_weight_max": 35.16745376586914,
      "activations/layer16_attention_weight_min": -29.804149627685547,
      "activations/layer17_attention_weight_max": 32.249046325683594,
      "activations/layer17_attention_weight_min": -26.230791091918945,
      "activations/layer18_attention_weight_max": 31.452312469482422,
      "activations/layer18_attention_weight_min": -23.417747497558594,
      "activations/layer19_attention_weight_max": 37.78620529174805,
      "activations/layer19_attention_weight_min": -31.792407989501953,
      "activations/layer1_attention_weight_max": 16.137428283691406,
      "activations/layer1_attention_weight_min": -16.856136322021484,
      "activations/layer20_attention_weight_max": 31.950698852539062,
      "activations/layer20_attention_weight_min": -24.78880500793457,
      "activations/layer21_attention_weight_max": 33.6806640625,
      "activations/layer21_attention_weight_min": -24.892013549804688,
      "activations/layer22_attention_weight_max": 46.76675796508789,
      "activations/layer22_attention_weight_min": -30.92802619934082,
      "activations/layer23_attention_weight_max": 35.20479202270508,
      "activations/layer23_attention_weight_min": -24.338420867919922,
      "activations/layer2_attention_weight_max": 31.95345115661621,
      "activations/layer2_attention_weight_min": -31.538206100463867,
      "activations/layer3_attention_weight_max": 97.43209075927734,
      "activations/layer3_attention_weight_min": -97.80419158935547,
      "activations/layer4_attention_weight_max": 110.74466705322266,
      "activations/layer4_attention_weight_min": -114.1364517211914,
      "activations/layer5_attention_weight_max": 51.70557403564453,
      "activations/layer5_attention_weight_min": -62.144447326660156,
      "activations/layer6_attention_weight_max": 44.75518035888672,
      "activations/layer6_attention_weight_min": -46.11927795410156,
      "activations/layer7_attention_weight_max": 90.75850677490234,
      "activations/layer7_attention_weight_min": -93.59833526611328,
      "activations/layer8_attention_weight_max": 40.94861602783203,
      "activations/layer8_attention_weight_min": -41.626617431640625,
      "activations/layer9_attention_weight_max": 35.4481315612793,
      "activations/layer9_attention_weight_min": -33.23061752319336,
      "epoch": 12.78,
      "learning_rate": 6.825113636363636e-05,
      "loss": 2.7761,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.4309,
      "eval_samples_per_second": 509.318,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.4309,
      "eval_openwebtext_samples_per_second": 509.318,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.9284,
      "eval_wikitext_samples_per_second": 236.463,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_lambada_loss": 2.51171875,
      "eval_lambada_ppl": 12.326097344475713,
      "eval_lambada_runtime": 9.5203,
      "eval_lambada_samples_per_second": 511.432,
      "step": 220000
    },
    {
      "activations/layer0_attention_weight_max": 16.39291000366211,
      "activations/layer0_attention_weight_min": -11.845992088317871,
      "activations/layer10_attention_weight_max": 30.56033706665039,
      "activations/layer10_attention_weight_min": -32.05354690551758,
      "activations/layer11_attention_weight_max": 30.45431900024414,
      "activations/layer11_attention_weight_min": -31.02935028076172,
      "activations/layer12_attention_weight_max": 23.674503326416016,
      "activations/layer12_attention_weight_min": -25.379770278930664,
      "activations/layer13_attention_weight_max": 37.320858001708984,
      "activations/layer13_attention_weight_min": -35.3316764831543,
      "activations/layer14_attention_weight_max": 42.1259880065918,
      "activations/layer14_attention_weight_min": -40.627532958984375,
      "activations/layer15_attention_weight_max": 35.862545013427734,
      "activations/layer15_attention_weight_min": -33.1019172668457,
      "activations/layer16_attention_weight_max": 29.2149658203125,
      "activations/layer16_attention_weight_min": -28.53921890258789,
      "activations/layer17_attention_weight_max": 30.70598030090332,
      "activations/layer17_attention_weight_min": -25.53347396850586,
      "activations/layer18_attention_weight_max": 33.37398147583008,
      "activations/layer18_attention_weight_min": -22.685461044311523,
      "activations/layer19_attention_weight_max": 34.04405975341797,
      "activations/layer19_attention_weight_min": -28.67996597290039,
      "activations/layer1_attention_weight_max": 16.66222381591797,
      "activations/layer1_attention_weight_min": -14.454817771911621,
      "activations/layer20_attention_weight_max": 27.87465476989746,
      "activations/layer20_attention_weight_min": -22.210664749145508,
      "activations/layer21_attention_weight_max": 30.33367156982422,
      "activations/layer21_attention_weight_min": -22.322998046875,
      "activations/layer22_attention_weight_max": 39.344810485839844,
      "activations/layer22_attention_weight_min": -29.146686553955078,
      "activations/layer23_attention_weight_max": 32.25391387939453,
      "activations/layer23_attention_weight_min": -24.71731185913086,
      "activations/layer2_attention_weight_max": 31.945228576660156,
      "activations/layer2_attention_weight_min": -31.844799041748047,
      "activations/layer3_attention_weight_max": 93.05176544189453,
      "activations/layer3_attention_weight_min": -91.87408447265625,
      "activations/layer4_attention_weight_max": 106.03913879394531,
      "activations/layer4_attention_weight_min": -107.40740966796875,
      "activations/layer5_attention_weight_max": 49.04689025878906,
      "activations/layer5_attention_weight_min": -64.22410583496094,
      "activations/layer6_attention_weight_max": 43.08310317993164,
      "activations/layer6_attention_weight_min": -47.014915466308594,
      "activations/layer7_attention_weight_max": 87.27885437011719,
      "activations/layer7_attention_weight_min": -86.6915512084961,
      "activations/layer8_attention_weight_max": 37.51588439941406,
      "activations/layer8_attention_weight_min": -40.97775650024414,
      "activations/layer9_attention_weight_max": 30.59720230102539,
      "activations/layer9_attention_weight_min": -31.9493465423584,
      "epoch": 12.79,
      "learning_rate": 6.823219696969696e-05,
      "loss": 2.7509,
      "step": 220050
    },
    {
      "activations/layer0_attention_weight_max": 15.62761116027832,
      "activations/layer0_attention_weight_min": -11.888565063476562,
      "activations/layer10_attention_weight_max": 33.156219482421875,
      "activations/layer10_attention_weight_min": -32.691890716552734,
      "activations/layer11_attention_weight_max": 32.727455139160156,
      "activations/layer11_attention_weight_min": -32.639076232910156,
      "activations/layer12_attention_weight_max": 26.20665168762207,
      "activations/layer12_attention_weight_min": -24.323944091796875,
      "activations/layer13_attention_weight_max": 39.1929931640625,
      "activations/layer13_attention_weight_min": -34.138580322265625,
      "activations/layer14_attention_weight_max": 45.11677551269531,
      "activations/layer14_attention_weight_min": -38.65929412841797,
      "activations/layer15_attention_weight_max": 35.59183883666992,
      "activations/layer15_attention_weight_min": -31.251800537109375,
      "activations/layer16_attention_weight_max": 29.29725456237793,
      "activations/layer16_attention_weight_min": -27.73426055908203,
      "activations/layer17_attention_weight_max": 32.01725387573242,
      "activations/layer17_attention_weight_min": -24.82175636291504,
      "activations/layer18_attention_weight_max": 32.962379455566406,
      "activations/layer18_attention_weight_min": -24.602493286132812,
      "activations/layer19_attention_weight_max": 34.50798416137695,
      "activations/layer19_attention_weight_min": -31.150663375854492,
      "activations/layer1_attention_weight_max": 16.17340850830078,
      "activations/layer1_attention_weight_min": -14.963705062866211,
      "activations/layer20_attention_weight_max": 30.49138641357422,
      "activations/layer20_attention_weight_min": -24.09400177001953,
      "activations/layer21_attention_weight_max": 31.602354049682617,
      "activations/layer21_attention_weight_min": -24.51361656188965,
      "activations/layer22_attention_weight_max": 43.74365234375,
      "activations/layer22_attention_weight_min": -29.01761817932129,
      "activations/layer23_attention_weight_max": 35.226409912109375,
      "activations/layer23_attention_weight_min": -24.149303436279297,
      "activations/layer2_attention_weight_max": 34.37499237060547,
      "activations/layer2_attention_weight_min": -35.71917724609375,
      "activations/layer3_attention_weight_max": 99.25678253173828,
      "activations/layer3_attention_weight_min": -102.76505279541016,
      "activations/layer4_attention_weight_max": 111.61480712890625,
      "activations/layer4_attention_weight_min": -114.2529525756836,
      "activations/layer5_attention_weight_max": 54.460533142089844,
      "activations/layer5_attention_weight_min": -61.78205108642578,
      "activations/layer6_attention_weight_max": 45.54334259033203,
      "activations/layer6_attention_weight_min": -48.00062942504883,
      "activations/layer7_attention_weight_max": 91.2945327758789,
      "activations/layer7_attention_weight_min": -89.86988067626953,
      "activations/layer8_attention_weight_max": 41.61540603637695,
      "activations/layer8_attention_weight_min": -42.9649658203125,
      "activations/layer9_attention_weight_max": 32.77155685424805,
      "activations/layer9_attention_weight_min": -33.024898529052734,
      "epoch": 12.79,
      "learning_rate": 6.821325757575756e-05,
      "loss": 2.759,
      "step": 220100
    },
    {
      "activations/layer0_attention_weight_max": 15.570426940917969,
      "activations/layer0_attention_weight_min": -11.591741561889648,
      "activations/layer10_attention_weight_max": 31.337688446044922,
      "activations/layer10_attention_weight_min": -30.759048461914062,
      "activations/layer11_attention_weight_max": 31.878803253173828,
      "activations/layer11_attention_weight_min": -31.956579208374023,
      "activations/layer12_attention_weight_max": 24.61273956298828,
      "activations/layer12_attention_weight_min": -24.787372589111328,
      "activations/layer13_attention_weight_max": 39.83248519897461,
      "activations/layer13_attention_weight_min": -35.973140716552734,
      "activations/layer14_attention_weight_max": 46.74570846557617,
      "activations/layer14_attention_weight_min": -41.80233383178711,
      "activations/layer15_attention_weight_max": 40.86493682861328,
      "activations/layer15_attention_weight_min": -34.7348747253418,
      "activations/layer16_attention_weight_max": 33.58565902709961,
      "activations/layer16_attention_weight_min": -29.547290802001953,
      "activations/layer17_attention_weight_max": 35.8163948059082,
      "activations/layer17_attention_weight_min": -27.308109283447266,
      "activations/layer18_attention_weight_max": 33.831443786621094,
      "activations/layer18_attention_weight_min": -23.509967803955078,
      "activations/layer19_attention_weight_max": 38.475852966308594,
      "activations/layer19_attention_weight_min": -31.90848731994629,
      "activations/layer1_attention_weight_max": 16.242273330688477,
      "activations/layer1_attention_weight_min": -16.335472106933594,
      "activations/layer20_attention_weight_max": 32.4494514465332,
      "activations/layer20_attention_weight_min": -27.679719924926758,
      "activations/layer21_attention_weight_max": 29.71817398071289,
      "activations/layer21_attention_weight_min": -25.402563095092773,
      "activations/layer22_attention_weight_max": 48.557559967041016,
      "activations/layer22_attention_weight_min": -34.02283477783203,
      "activations/layer23_attention_weight_max": 39.16047286987305,
      "activations/layer23_attention_weight_min": -26.25409698486328,
      "activations/layer2_attention_weight_max": 33.611289978027344,
      "activations/layer2_attention_weight_min": -32.564823150634766,
      "activations/layer3_attention_weight_max": 97.97663116455078,
      "activations/layer3_attention_weight_min": -94.7636489868164,
      "activations/layer4_attention_weight_max": 111.6825942993164,
      "activations/layer4_attention_weight_min": -109.51094055175781,
      "activations/layer5_attention_weight_max": 49.556915283203125,
      "activations/layer5_attention_weight_min": -59.65360641479492,
      "activations/layer6_attention_weight_max": 43.27603530883789,
      "activations/layer6_attention_weight_min": -46.356773376464844,
      "activations/layer7_attention_weight_max": 86.29228973388672,
      "activations/layer7_attention_weight_min": -90.10568237304688,
      "activations/layer8_attention_weight_max": 38.09027862548828,
      "activations/layer8_attention_weight_min": -39.904476165771484,
      "activations/layer9_attention_weight_max": 31.855804443359375,
      "activations/layer9_attention_weight_min": -33.2463493347168,
      "epoch": 12.79,
      "learning_rate": 6.819431818181818e-05,
      "loss": 2.7555,
      "step": 220150
    },
    {
      "activations/layer0_attention_weight_max": 15.844623565673828,
      "activations/layer0_attention_weight_min": -11.704967498779297,
      "activations/layer10_attention_weight_max": 33.34473419189453,
      "activations/layer10_attention_weight_min": -33.533077239990234,
      "activations/layer11_attention_weight_max": 31.711868286132812,
      "activations/layer11_attention_weight_min": -31.9080867767334,
      "activations/layer12_attention_weight_max": 25.754520416259766,
      "activations/layer12_attention_weight_min": -25.156997680664062,
      "activations/layer13_attention_weight_max": 41.552608489990234,
      "activations/layer13_attention_weight_min": -35.197303771972656,
      "activations/layer14_attention_weight_max": 46.19038772583008,
      "activations/layer14_attention_weight_min": -37.47254180908203,
      "activations/layer15_attention_weight_max": 35.87051010131836,
      "activations/layer15_attention_weight_min": -31.29607582092285,
      "activations/layer16_attention_weight_max": 30.295543670654297,
      "activations/layer16_attention_weight_min": -27.13851547241211,
      "activations/layer17_attention_weight_max": 29.80022621154785,
      "activations/layer17_attention_weight_min": -23.86674690246582,
      "activations/layer18_attention_weight_max": 30.338109970092773,
      "activations/layer18_attention_weight_min": -23.56170654296875,
      "activations/layer19_attention_weight_max": 34.70725631713867,
      "activations/layer19_attention_weight_min": -30.206771850585938,
      "activations/layer1_attention_weight_max": 15.678882598876953,
      "activations/layer1_attention_weight_min": -14.650622367858887,
      "activations/layer20_attention_weight_max": 30.13490867614746,
      "activations/layer20_attention_weight_min": -23.5981502532959,
      "activations/layer21_attention_weight_max": 30.696292877197266,
      "activations/layer21_attention_weight_min": -24.57071304321289,
      "activations/layer22_attention_weight_max": 43.57643127441406,
      "activations/layer22_attention_weight_min": -31.10982322692871,
      "activations/layer23_attention_weight_max": 32.727256774902344,
      "activations/layer23_attention_weight_min": -28.606670379638672,
      "activations/layer2_attention_weight_max": 32.109867095947266,
      "activations/layer2_attention_weight_min": -34.73073959350586,
      "activations/layer3_attention_weight_max": 97.09246826171875,
      "activations/layer3_attention_weight_min": -105.77533721923828,
      "activations/layer4_attention_weight_max": 110.78951263427734,
      "activations/layer4_attention_weight_min": -111.6660385131836,
      "activations/layer5_attention_weight_max": 54.62245178222656,
      "activations/layer5_attention_weight_min": -60.6029052734375,
      "activations/layer6_attention_weight_max": 43.445884704589844,
      "activations/layer6_attention_weight_min": -45.76405715942383,
      "activations/layer7_attention_weight_max": 89.31206512451172,
      "activations/layer7_attention_weight_min": -92.0458755493164,
      "activations/layer8_attention_weight_max": 39.367740631103516,
      "activations/layer8_attention_weight_min": -40.30839920043945,
      "activations/layer9_attention_weight_max": 31.54250144958496,
      "activations/layer9_attention_weight_min": -33.222537994384766,
      "epoch": 12.79,
      "learning_rate": 6.817537878787878e-05,
      "loss": 2.7678,
      "step": 220200
    },
    {
      "activations/layer0_attention_weight_max": 16.679128646850586,
      "activations/layer0_attention_weight_min": -12.049445152282715,
      "activations/layer10_attention_weight_max": 33.07749557495117,
      "activations/layer10_attention_weight_min": -31.703929901123047,
      "activations/layer11_attention_weight_max": 32.541202545166016,
      "activations/layer11_attention_weight_min": -31.571035385131836,
      "activations/layer12_attention_weight_max": 24.65277099609375,
      "activations/layer12_attention_weight_min": -24.053409576416016,
      "activations/layer13_attention_weight_max": 38.83495330810547,
      "activations/layer13_attention_weight_min": -32.16393280029297,
      "activations/layer14_attention_weight_max": 48.898983001708984,
      "activations/layer14_attention_weight_min": -34.16275405883789,
      "activations/layer15_attention_weight_max": 34.56856918334961,
      "activations/layer15_attention_weight_min": -31.810638427734375,
      "activations/layer16_attention_weight_max": 28.9022159576416,
      "activations/layer16_attention_weight_min": -27.15220832824707,
      "activations/layer17_attention_weight_max": 31.70985984802246,
      "activations/layer17_attention_weight_min": -25.94981575012207,
      "activations/layer18_attention_weight_max": 29.46320152282715,
      "activations/layer18_attention_weight_min": -23.76160430908203,
      "activations/layer19_attention_weight_max": 31.253644943237305,
      "activations/layer19_attention_weight_min": -33.722591400146484,
      "activations/layer1_attention_weight_max": 15.779001235961914,
      "activations/layer1_attention_weight_min": -15.558309555053711,
      "activations/layer20_attention_weight_max": 28.920930862426758,
      "activations/layer20_attention_weight_min": -25.422622680664062,
      "activations/layer21_attention_weight_max": 27.421123504638672,
      "activations/layer21_attention_weight_min": -23.50213050842285,
      "activations/layer22_attention_weight_max": 45.00579071044922,
      "activations/layer22_attention_weight_min": -30.64338493347168,
      "activations/layer23_attention_weight_max": 32.377105712890625,
      "activations/layer23_attention_weight_min": -24.698665618896484,
      "activations/layer2_attention_weight_max": 32.73362350463867,
      "activations/layer2_attention_weight_min": -33.00149917602539,
      "activations/layer3_attention_weight_max": 97.98066711425781,
      "activations/layer3_attention_weight_min": -99.40530395507812,
      "activations/layer4_attention_weight_max": 114.89985656738281,
      "activations/layer4_attention_weight_min": -115.96114349365234,
      "activations/layer5_attention_weight_max": 52.06229782104492,
      "activations/layer5_attention_weight_min": -62.914649963378906,
      "activations/layer6_attention_weight_max": 44.96127700805664,
      "activations/layer6_attention_weight_min": -47.9121208190918,
      "activations/layer7_attention_weight_max": 87.76214599609375,
      "activations/layer7_attention_weight_min": -97.75648498535156,
      "activations/layer8_attention_weight_max": 39.99079513549805,
      "activations/layer8_attention_weight_min": -42.21940994262695,
      "activations/layer9_attention_weight_max": 32.95306396484375,
      "activations/layer9_attention_weight_min": -33.64142608642578,
      "epoch": 12.8,
      "learning_rate": 6.81564393939394e-05,
      "loss": 2.7654,
      "step": 220250
    },
    {
      "activations/layer0_attention_weight_max": 15.942667007446289,
      "activations/layer0_attention_weight_min": -11.473487854003906,
      "activations/layer10_attention_weight_max": 37.207855224609375,
      "activations/layer10_attention_weight_min": -33.81220245361328,
      "activations/layer11_attention_weight_max": 38.72389221191406,
      "activations/layer11_attention_weight_min": -35.63792419433594,
      "activations/layer12_attention_weight_max": 30.408817291259766,
      "activations/layer12_attention_weight_min": -26.936006546020508,
      "activations/layer13_attention_weight_max": 47.124244689941406,
      "activations/layer13_attention_weight_min": -37.27867126464844,
      "activations/layer14_attention_weight_max": 53.84694290161133,
      "activations/layer14_attention_weight_min": -40.64600372314453,
      "activations/layer15_attention_weight_max": 52.717227935791016,
      "activations/layer15_attention_weight_min": -36.601951599121094,
      "activations/layer16_attention_weight_max": 35.845394134521484,
      "activations/layer16_attention_weight_min": -27.907751083374023,
      "activations/layer17_attention_weight_max": 37.749847412109375,
      "activations/layer17_attention_weight_min": -27.51453971862793,
      "activations/layer18_attention_weight_max": 32.38800811767578,
      "activations/layer18_attention_weight_min": -24.618608474731445,
      "activations/layer19_attention_weight_max": 36.53084945678711,
      "activations/layer19_attention_weight_min": -33.13182830810547,
      "activations/layer1_attention_weight_max": 17.180438995361328,
      "activations/layer1_attention_weight_min": -14.800018310546875,
      "activations/layer20_attention_weight_max": 33.54810333251953,
      "activations/layer20_attention_weight_min": -24.167247772216797,
      "activations/layer21_attention_weight_max": 33.38218307495117,
      "activations/layer21_attention_weight_min": -23.0854549407959,
      "activations/layer22_attention_weight_max": 47.74992752075195,
      "activations/layer22_attention_weight_min": -29.360267639160156,
      "activations/layer23_attention_weight_max": 35.920860290527344,
      "activations/layer23_attention_weight_min": -24.19359588623047,
      "activations/layer2_attention_weight_max": 36.599037170410156,
      "activations/layer2_attention_weight_min": -34.28290557861328,
      "activations/layer3_attention_weight_max": 100.78435516357422,
      "activations/layer3_attention_weight_min": -97.92347717285156,
      "activations/layer4_attention_weight_max": 114.4849624633789,
      "activations/layer4_attention_weight_min": -111.27860260009766,
      "activations/layer5_attention_weight_max": 54.58843231201172,
      "activations/layer5_attention_weight_min": -59.206687927246094,
      "activations/layer6_attention_weight_max": 45.89601516723633,
      "activations/layer6_attention_weight_min": -47.109169006347656,
      "activations/layer7_attention_weight_max": 93.45830535888672,
      "activations/layer7_attention_weight_min": -91.19519805908203,
      "activations/layer8_attention_weight_max": 42.13577651977539,
      "activations/layer8_attention_weight_min": -42.85194396972656,
      "activations/layer9_attention_weight_max": 34.88677978515625,
      "activations/layer9_attention_weight_min": -33.825218200683594,
      "epoch": 12.8,
      "learning_rate": 6.81375e-05,
      "loss": 2.7676,
      "step": 220300
    },
    {
      "activations/layer0_attention_weight_max": 17.03656005859375,
      "activations/layer0_attention_weight_min": -12.042621612548828,
      "activations/layer10_attention_weight_max": 34.911407470703125,
      "activations/layer10_attention_weight_min": -32.525367736816406,
      "activations/layer11_attention_weight_max": 33.71674346923828,
      "activations/layer11_attention_weight_min": -32.11100769042969,
      "activations/layer12_attention_weight_max": 26.903303146362305,
      "activations/layer12_attention_weight_min": -26.36048698425293,
      "activations/layer13_attention_weight_max": 41.14386749267578,
      "activations/layer13_attention_weight_min": -37.471370697021484,
      "activations/layer14_attention_weight_max": 44.632537841796875,
      "activations/layer14_attention_weight_min": -37.11897277832031,
      "activations/layer15_attention_weight_max": 38.917171478271484,
      "activations/layer15_attention_weight_min": -33.060665130615234,
      "activations/layer16_attention_weight_max": 34.86004638671875,
      "activations/layer16_attention_weight_min": -27.260427474975586,
      "activations/layer17_attention_weight_max": 30.60249900817871,
      "activations/layer17_attention_weight_min": -26.512866973876953,
      "activations/layer18_attention_weight_max": 31.268817901611328,
      "activations/layer18_attention_weight_min": -24.332761764526367,
      "activations/layer19_attention_weight_max": 34.95132827758789,
      "activations/layer19_attention_weight_min": -31.773773193359375,
      "activations/layer1_attention_weight_max": 16.651050567626953,
      "activations/layer1_attention_weight_min": -16.413467407226562,
      "activations/layer20_attention_weight_max": 28.598480224609375,
      "activations/layer20_attention_weight_min": -25.03917694091797,
      "activations/layer21_attention_weight_max": 27.487619400024414,
      "activations/layer21_attention_weight_min": -24.030797958374023,
      "activations/layer22_attention_weight_max": 38.71454620361328,
      "activations/layer22_attention_weight_min": -30.71229362487793,
      "activations/layer23_attention_weight_max": 32.026023864746094,
      "activations/layer23_attention_weight_min": -25.589702606201172,
      "activations/layer2_attention_weight_max": 32.68431091308594,
      "activations/layer2_attention_weight_min": -33.769081115722656,
      "activations/layer3_attention_weight_max": 96.10210418701172,
      "activations/layer3_attention_weight_min": -98.47453308105469,
      "activations/layer4_attention_weight_max": 113.6950454711914,
      "activations/layer4_attention_weight_min": -112.2317123413086,
      "activations/layer5_attention_weight_max": 53.52894592285156,
      "activations/layer5_attention_weight_min": -60.525489807128906,
      "activations/layer6_attention_weight_max": 44.22666549682617,
      "activations/layer6_attention_weight_min": -50.4238395690918,
      "activations/layer7_attention_weight_max": 92.52206420898438,
      "activations/layer7_attention_weight_min": -103.82550048828125,
      "activations/layer8_attention_weight_max": 40.96449661254883,
      "activations/layer8_attention_weight_min": -41.652923583984375,
      "activations/layer9_attention_weight_max": 33.27878189086914,
      "activations/layer9_attention_weight_min": -33.569766998291016,
      "epoch": 12.8,
      "learning_rate": 6.81185606060606e-05,
      "loss": 2.7733,
      "step": 220350
    },
    {
      "activations/layer0_attention_weight_max": 16.034160614013672,
      "activations/layer0_attention_weight_min": -11.438552856445312,
      "activations/layer10_attention_weight_max": 33.646732330322266,
      "activations/layer10_attention_weight_min": -34.86256790161133,
      "activations/layer11_attention_weight_max": 33.31944274902344,
      "activations/layer11_attention_weight_min": -32.88035583496094,
      "activations/layer12_attention_weight_max": 27.116336822509766,
      "activations/layer12_attention_weight_min": -27.095924377441406,
      "activations/layer13_attention_weight_max": 42.46444320678711,
      "activations/layer13_attention_weight_min": -38.00971984863281,
      "activations/layer14_attention_weight_max": 48.93003845214844,
      "activations/layer14_attention_weight_min": -42.250736236572266,
      "activations/layer15_attention_weight_max": 38.40818405151367,
      "activations/layer15_attention_weight_min": -34.949405670166016,
      "activations/layer16_attention_weight_max": 30.933568954467773,
      "activations/layer16_attention_weight_min": -28.927711486816406,
      "activations/layer17_attention_weight_max": 34.659603118896484,
      "activations/layer17_attention_weight_min": -25.588607788085938,
      "activations/layer18_attention_weight_max": 33.60660171508789,
      "activations/layer18_attention_weight_min": -23.21110725402832,
      "activations/layer19_attention_weight_max": 39.275543212890625,
      "activations/layer19_attention_weight_min": -30.55385398864746,
      "activations/layer1_attention_weight_max": 15.761957168579102,
      "activations/layer1_attention_weight_min": -14.281250953674316,
      "activations/layer20_attention_weight_max": 34.99369812011719,
      "activations/layer20_attention_weight_min": -25.528268814086914,
      "activations/layer21_attention_weight_max": 31.779552459716797,
      "activations/layer21_attention_weight_min": -25.308862686157227,
      "activations/layer22_attention_weight_max": 58.22993850708008,
      "activations/layer22_attention_weight_min": -31.706832885742188,
      "activations/layer23_attention_weight_max": 37.9201774597168,
      "activations/layer23_attention_weight_min": -23.561742782592773,
      "activations/layer2_attention_weight_max": 32.08926773071289,
      "activations/layer2_attention_weight_min": -34.00473403930664,
      "activations/layer3_attention_weight_max": 93.79228973388672,
      "activations/layer3_attention_weight_min": -98.23212432861328,
      "activations/layer4_attention_weight_max": 107.71410369873047,
      "activations/layer4_attention_weight_min": -109.01191711425781,
      "activations/layer5_attention_weight_max": 50.85075378417969,
      "activations/layer5_attention_weight_min": -58.083396911621094,
      "activations/layer6_attention_weight_max": 43.01955795288086,
      "activations/layer6_attention_weight_min": -44.23472213745117,
      "activations/layer7_attention_weight_max": 90.06024932861328,
      "activations/layer7_attention_weight_min": -86.68780517578125,
      "activations/layer8_attention_weight_max": 39.70086669921875,
      "activations/layer8_attention_weight_min": -41.15188217163086,
      "activations/layer9_attention_weight_max": 32.082977294921875,
      "activations/layer9_attention_weight_min": -33.800235748291016,
      "epoch": 12.81,
      "learning_rate": 6.809962121212121e-05,
      "loss": 2.7631,
      "step": 220400
    },
    {
      "activations/layer0_attention_weight_max": 15.622653007507324,
      "activations/layer0_attention_weight_min": -11.484051704406738,
      "activations/layer10_attention_weight_max": 34.21348571777344,
      "activations/layer10_attention_weight_min": -32.0378303527832,
      "activations/layer11_attention_weight_max": 32.33251953125,
      "activations/layer11_attention_weight_min": -31.675994873046875,
      "activations/layer12_attention_weight_max": 26.781726837158203,
      "activations/layer12_attention_weight_min": -24.621341705322266,
      "activations/layer13_attention_weight_max": 43.97187805175781,
      "activations/layer13_attention_weight_min": -33.32429122924805,
      "activations/layer14_attention_weight_max": 53.674007415771484,
      "activations/layer14_attention_weight_min": -38.63703155517578,
      "activations/layer15_attention_weight_max": 44.1789665222168,
      "activations/layer15_attention_weight_min": -33.434757232666016,
      "activations/layer16_attention_weight_max": 32.91202163696289,
      "activations/layer16_attention_weight_min": -28.745759963989258,
      "activations/layer17_attention_weight_max": 32.86748504638672,
      "activations/layer17_attention_weight_min": -28.153470993041992,
      "activations/layer18_attention_weight_max": 34.01716995239258,
      "activations/layer18_attention_weight_min": -24.8582706451416,
      "activations/layer19_attention_weight_max": 35.2578010559082,
      "activations/layer19_attention_weight_min": -31.587615966796875,
      "activations/layer1_attention_weight_max": 16.28369140625,
      "activations/layer1_attention_weight_min": -15.188100814819336,
      "activations/layer20_attention_weight_max": 30.532493591308594,
      "activations/layer20_attention_weight_min": -24.9311580657959,
      "activations/layer21_attention_weight_max": 31.3731746673584,
      "activations/layer21_attention_weight_min": -25.157541275024414,
      "activations/layer22_attention_weight_max": 46.04117202758789,
      "activations/layer22_attention_weight_min": -33.14426803588867,
      "activations/layer23_attention_weight_max": 37.147056579589844,
      "activations/layer23_attention_weight_min": -27.60811996459961,
      "activations/layer2_attention_weight_max": 31.230789184570312,
      "activations/layer2_attention_weight_min": -31.192367553710938,
      "activations/layer3_attention_weight_max": 95.35539245605469,
      "activations/layer3_attention_weight_min": -96.32099151611328,
      "activations/layer4_attention_weight_max": 110.4404067993164,
      "activations/layer4_attention_weight_min": -114.8152847290039,
      "activations/layer5_attention_weight_max": 50.322120666503906,
      "activations/layer5_attention_weight_min": -62.96613693237305,
      "activations/layer6_attention_weight_max": 46.19363784790039,
      "activations/layer6_attention_weight_min": -48.00043869018555,
      "activations/layer7_attention_weight_max": 92.67645263671875,
      "activations/layer7_attention_weight_min": -93.53288269042969,
      "activations/layer8_attention_weight_max": 41.90658187866211,
      "activations/layer8_attention_weight_min": -40.66311264038086,
      "activations/layer9_attention_weight_max": 33.93623352050781,
      "activations/layer9_attention_weight_min": -33.13889694213867,
      "epoch": 12.81,
      "learning_rate": 6.808068181818181e-05,
      "loss": 2.7827,
      "step": 220450
    },
    {
      "activations/layer0_attention_weight_max": 15.971260070800781,
      "activations/layer0_attention_weight_min": -11.28646469116211,
      "activations/layer10_attention_weight_max": 36.65341567993164,
      "activations/layer10_attention_weight_min": -32.891441345214844,
      "activations/layer11_attention_weight_max": 39.63496398925781,
      "activations/layer11_attention_weight_min": -36.004913330078125,
      "activations/layer12_attention_weight_max": 25.441858291625977,
      "activations/layer12_attention_weight_min": -26.402462005615234,
      "activations/layer13_attention_weight_max": 40.01034164428711,
      "activations/layer13_attention_weight_min": -33.906349182128906,
      "activations/layer14_attention_weight_max": 48.11250305175781,
      "activations/layer14_attention_weight_min": -35.85493087768555,
      "activations/layer15_attention_weight_max": 36.54585266113281,
      "activations/layer15_attention_weight_min": -33.44546890258789,
      "activations/layer16_attention_weight_max": 31.731840133666992,
      "activations/layer16_attention_weight_min": -28.492481231689453,
      "activations/layer17_attention_weight_max": 29.779523849487305,
      "activations/layer17_attention_weight_min": -26.34084129333496,
      "activations/layer18_attention_weight_max": 32.9359130859375,
      "activations/layer18_attention_weight_min": -24.749757766723633,
      "activations/layer19_attention_weight_max": 40.2171745300293,
      "activations/layer19_attention_weight_min": -33.312068939208984,
      "activations/layer1_attention_weight_max": 15.93952465057373,
      "activations/layer1_attention_weight_min": -14.981865882873535,
      "activations/layer20_attention_weight_max": 31.22597885131836,
      "activations/layer20_attention_weight_min": -26.061655044555664,
      "activations/layer21_attention_weight_max": 29.373807907104492,
      "activations/layer21_attention_weight_min": -25.880338668823242,
      "activations/layer22_attention_weight_max": 46.79591369628906,
      "activations/layer22_attention_weight_min": -31.812089920043945,
      "activations/layer23_attention_weight_max": 38.206298828125,
      "activations/layer23_attention_weight_min": -25.197322845458984,
      "activations/layer2_attention_weight_max": 33.04838562011719,
      "activations/layer2_attention_weight_min": -32.69581985473633,
      "activations/layer3_attention_weight_max": 95.2925796508789,
      "activations/layer3_attention_weight_min": -99.04737091064453,
      "activations/layer4_attention_weight_max": 113.70051574707031,
      "activations/layer4_attention_weight_min": -113.5416259765625,
      "activations/layer5_attention_weight_max": 58.429378509521484,
      "activations/layer5_attention_weight_min": -60.470455169677734,
      "activations/layer6_attention_weight_max": 40.869384765625,
      "activations/layer6_attention_weight_min": -44.188865661621094,
      "activations/layer7_attention_weight_max": 99.0385513305664,
      "activations/layer7_attention_weight_min": -86.63604736328125,
      "activations/layer8_attention_weight_max": 43.24819564819336,
      "activations/layer8_attention_weight_min": -40.2757453918457,
      "activations/layer9_attention_weight_max": 32.66546630859375,
      "activations/layer9_attention_weight_min": -32.93561935424805,
      "epoch": 12.81,
      "learning_rate": 6.806174242424242e-05,
      "loss": 2.7647,
      "step": 220500
    },
    {
      "activations/layer0_attention_weight_max": 16.55963134765625,
      "activations/layer0_attention_weight_min": -11.64820384979248,
      "activations/layer10_attention_weight_max": 31.87217140197754,
      "activations/layer10_attention_weight_min": -33.799747467041016,
      "activations/layer11_attention_weight_max": 31.30782127380371,
      "activations/layer11_attention_weight_min": -33.35023880004883,
      "activations/layer12_attention_weight_max": 23.965532302856445,
      "activations/layer12_attention_weight_min": -26.57259750366211,
      "activations/layer13_attention_weight_max": 38.06886672973633,
      "activations/layer13_attention_weight_min": -34.59419250488281,
      "activations/layer14_attention_weight_max": 42.86552429199219,
      "activations/layer14_attention_weight_min": -41.10239028930664,
      "activations/layer15_attention_weight_max": 36.33036804199219,
      "activations/layer15_attention_weight_min": -32.46895980834961,
      "activations/layer16_attention_weight_max": 35.122413635253906,
      "activations/layer16_attention_weight_min": -30.410282135009766,
      "activations/layer17_attention_weight_max": 30.15079116821289,
      "activations/layer17_attention_weight_min": -25.556350708007812,
      "activations/layer18_attention_weight_max": 29.5323486328125,
      "activations/layer18_attention_weight_min": -24.592010498046875,
      "activations/layer19_attention_weight_max": 33.631744384765625,
      "activations/layer19_attention_weight_min": -30.007898330688477,
      "activations/layer1_attention_weight_max": 16.797042846679688,
      "activations/layer1_attention_weight_min": -14.915205001831055,
      "activations/layer20_attention_weight_max": 28.278337478637695,
      "activations/layer20_attention_weight_min": -25.298057556152344,
      "activations/layer21_attention_weight_max": 27.52397918701172,
      "activations/layer21_attention_weight_min": -24.966398239135742,
      "activations/layer22_attention_weight_max": 43.86627197265625,
      "activations/layer22_attention_weight_min": -31.96267318725586,
      "activations/layer23_attention_weight_max": 34.203304290771484,
      "activations/layer23_attention_weight_min": -29.858413696289062,
      "activations/layer2_attention_weight_max": 32.82060241699219,
      "activations/layer2_attention_weight_min": -32.383235931396484,
      "activations/layer3_attention_weight_max": 96.52056121826172,
      "activations/layer3_attention_weight_min": -100.64014434814453,
      "activations/layer4_attention_weight_max": 109.90803527832031,
      "activations/layer4_attention_weight_min": -111.56253814697266,
      "activations/layer5_attention_weight_max": 53.706886291503906,
      "activations/layer5_attention_weight_min": -64.35997772216797,
      "activations/layer6_attention_weight_max": 43.89616394042969,
      "activations/layer6_attention_weight_min": -45.663246154785156,
      "activations/layer7_attention_weight_max": 86.11475372314453,
      "activations/layer7_attention_weight_min": -90.63597869873047,
      "activations/layer8_attention_weight_max": 38.713783264160156,
      "activations/layer8_attention_weight_min": -39.561153411865234,
      "activations/layer9_attention_weight_max": 32.72077941894531,
      "activations/layer9_attention_weight_min": -34.76741027832031,
      "epoch": 12.82,
      "learning_rate": 6.804280303030303e-05,
      "loss": 2.7714,
      "step": 220550
    },
    {
      "activations/layer0_attention_weight_max": 15.988786697387695,
      "activations/layer0_attention_weight_min": -11.546499252319336,
      "activations/layer10_attention_weight_max": 30.600988388061523,
      "activations/layer10_attention_weight_min": -31.532821655273438,
      "activations/layer11_attention_weight_max": 32.15789031982422,
      "activations/layer11_attention_weight_min": -31.515586853027344,
      "activations/layer12_attention_weight_max": 25.819461822509766,
      "activations/layer12_attention_weight_min": -24.76573371887207,
      "activations/layer13_attention_weight_max": 37.00609588623047,
      "activations/layer13_attention_weight_min": -31.99129867553711,
      "activations/layer14_attention_weight_max": 40.47026062011719,
      "activations/layer14_attention_weight_min": -34.048583984375,
      "activations/layer15_attention_weight_max": 33.16272735595703,
      "activations/layer15_attention_weight_min": -32.934967041015625,
      "activations/layer16_attention_weight_max": 28.257606506347656,
      "activations/layer16_attention_weight_min": -26.63361358642578,
      "activations/layer17_attention_weight_max": 29.353675842285156,
      "activations/layer17_attention_weight_min": -25.40072250366211,
      "activations/layer18_attention_weight_max": 31.856464385986328,
      "activations/layer18_attention_weight_min": -24.82156753540039,
      "activations/layer19_attention_weight_max": 31.68035316467285,
      "activations/layer19_attention_weight_min": -30.596670150756836,
      "activations/layer1_attention_weight_max": 15.867953300476074,
      "activations/layer1_attention_weight_min": -15.656739234924316,
      "activations/layer20_attention_weight_max": 28.68731117248535,
      "activations/layer20_attention_weight_min": -25.275644302368164,
      "activations/layer21_attention_weight_max": 28.43851661682129,
      "activations/layer21_attention_weight_min": -25.050344467163086,
      "activations/layer22_attention_weight_max": 38.79961013793945,
      "activations/layer22_attention_weight_min": -29.753145217895508,
      "activations/layer23_attention_weight_max": 32.365543365478516,
      "activations/layer23_attention_weight_min": -22.96359634399414,
      "activations/layer2_attention_weight_max": 33.219879150390625,
      "activations/layer2_attention_weight_min": -33.16761779785156,
      "activations/layer3_attention_weight_max": 95.58326721191406,
      "activations/layer3_attention_weight_min": -101.1019058227539,
      "activations/layer4_attention_weight_max": 109.9422607421875,
      "activations/layer4_attention_weight_min": -107.58849334716797,
      "activations/layer5_attention_weight_max": 50.466880798339844,
      "activations/layer5_attention_weight_min": -64.78828430175781,
      "activations/layer6_attention_weight_max": 43.55131530761719,
      "activations/layer6_attention_weight_min": -45.61564636230469,
      "activations/layer7_attention_weight_max": 91.44603729248047,
      "activations/layer7_attention_weight_min": -90.74160766601562,
      "activations/layer8_attention_weight_max": 39.11066818237305,
      "activations/layer8_attention_weight_min": -40.664520263671875,
      "activations/layer9_attention_weight_max": 31.058034896850586,
      "activations/layer9_attention_weight_min": -32.13697052001953,
      "epoch": 12.82,
      "learning_rate": 6.802386363636363e-05,
      "loss": 2.769,
      "step": 220600
    },
    {
      "activations/layer0_attention_weight_max": 16.031230926513672,
      "activations/layer0_attention_weight_min": -11.076981544494629,
      "activations/layer10_attention_weight_max": 30.95492172241211,
      "activations/layer10_attention_weight_min": -34.70642852783203,
      "activations/layer11_attention_weight_max": 31.84551239013672,
      "activations/layer11_attention_weight_min": -33.08226776123047,
      "activations/layer12_attention_weight_max": 25.124359130859375,
      "activations/layer12_attention_weight_min": -25.33074188232422,
      "activations/layer13_attention_weight_max": 38.205780029296875,
      "activations/layer13_attention_weight_min": -33.99303436279297,
      "activations/layer14_attention_weight_max": 43.87736129760742,
      "activations/layer14_attention_weight_min": -36.79488754272461,
      "activations/layer15_attention_weight_max": 40.20865249633789,
      "activations/layer15_attention_weight_min": -32.81361389160156,
      "activations/layer16_attention_weight_max": 29.15367317199707,
      "activations/layer16_attention_weight_min": -28.77372932434082,
      "activations/layer17_attention_weight_max": 29.630130767822266,
      "activations/layer17_attention_weight_min": -27.735414505004883,
      "activations/layer18_attention_weight_max": 31.930362701416016,
      "activations/layer18_attention_weight_min": -23.56838035583496,
      "activations/layer19_attention_weight_max": 35.841278076171875,
      "activations/layer19_attention_weight_min": -32.95754623413086,
      "activations/layer1_attention_weight_max": 16.352386474609375,
      "activations/layer1_attention_weight_min": -13.933972358703613,
      "activations/layer20_attention_weight_max": 31.805267333984375,
      "activations/layer20_attention_weight_min": -24.769298553466797,
      "activations/layer21_attention_weight_max": 31.07720184326172,
      "activations/layer21_attention_weight_min": -25.544902801513672,
      "activations/layer22_attention_weight_max": 44.74777603149414,
      "activations/layer22_attention_weight_min": -32.0057258605957,
      "activations/layer23_attention_weight_max": 38.29567337036133,
      "activations/layer23_attention_weight_min": -30.666065216064453,
      "activations/layer2_attention_weight_max": 30.87451934814453,
      "activations/layer2_attention_weight_min": -30.524486541748047,
      "activations/layer3_attention_weight_max": 92.46148681640625,
      "activations/layer3_attention_weight_min": -93.3346939086914,
      "activations/layer4_attention_weight_max": 107.34453582763672,
      "activations/layer4_attention_weight_min": -108.03277587890625,
      "activations/layer5_attention_weight_max": 50.22724914550781,
      "activations/layer5_attention_weight_min": -59.764739990234375,
      "activations/layer6_attention_weight_max": 44.988521575927734,
      "activations/layer6_attention_weight_min": -47.71638870239258,
      "activations/layer7_attention_weight_max": 91.14491271972656,
      "activations/layer7_attention_weight_min": -97.03423309326172,
      "activations/layer8_attention_weight_max": 38.721920013427734,
      "activations/layer8_attention_weight_min": -40.98059844970703,
      "activations/layer9_attention_weight_max": 31.410831451416016,
      "activations/layer9_attention_weight_min": -33.652435302734375,
      "epoch": 12.82,
      "learning_rate": 6.800492424242423e-05,
      "loss": 2.7679,
      "step": 220650
    },
    {
      "activations/layer0_attention_weight_max": 16.147552490234375,
      "activations/layer0_attention_weight_min": -13.157293319702148,
      "activations/layer10_attention_weight_max": 31.85392189025879,
      "activations/layer10_attention_weight_min": -32.817779541015625,
      "activations/layer11_attention_weight_max": 34.43958282470703,
      "activations/layer11_attention_weight_min": -34.17749786376953,
      "activations/layer12_attention_weight_max": 24.752643585205078,
      "activations/layer12_attention_weight_min": -26.47747230529785,
      "activations/layer13_attention_weight_max": 36.200218200683594,
      "activations/layer13_attention_weight_min": -34.9326286315918,
      "activations/layer14_attention_weight_max": 37.485469818115234,
      "activations/layer14_attention_weight_min": -37.15364456176758,
      "activations/layer15_attention_weight_max": 34.38703536987305,
      "activations/layer15_attention_weight_min": -32.278865814208984,
      "activations/layer16_attention_weight_max": 27.0498046875,
      "activations/layer16_attention_weight_min": -26.866634368896484,
      "activations/layer17_attention_weight_max": 28.308046340942383,
      "activations/layer17_attention_weight_min": -25.337669372558594,
      "activations/layer18_attention_weight_max": 30.4925594329834,
      "activations/layer18_attention_weight_min": -24.131078720092773,
      "activations/layer19_attention_weight_max": 31.225332260131836,
      "activations/layer19_attention_weight_min": -32.72144317626953,
      "activations/layer1_attention_weight_max": 16.36093521118164,
      "activations/layer1_attention_weight_min": -14.987080574035645,
      "activations/layer20_attention_weight_max": 27.001115798950195,
      "activations/layer20_attention_weight_min": -26.73943519592285,
      "activations/layer21_attention_weight_max": 27.567874908447266,
      "activations/layer21_attention_weight_min": -25.947893142700195,
      "activations/layer22_attention_weight_max": 40.43986129760742,
      "activations/layer22_attention_weight_min": -31.019922256469727,
      "activations/layer23_attention_weight_max": 33.755733489990234,
      "activations/layer23_attention_weight_min": -24.707799911499023,
      "activations/layer2_attention_weight_max": 32.07878875732422,
      "activations/layer2_attention_weight_min": -32.57965850830078,
      "activations/layer3_attention_weight_max": 94.15060424804688,
      "activations/layer3_attention_weight_min": -96.55458068847656,
      "activations/layer4_attention_weight_max": 109.03814697265625,
      "activations/layer4_attention_weight_min": -112.12127685546875,
      "activations/layer5_attention_weight_max": 55.331947326660156,
      "activations/layer5_attention_weight_min": -62.43976974487305,
      "activations/layer6_attention_weight_max": 45.6400260925293,
      "activations/layer6_attention_weight_min": -47.05577087402344,
      "activations/layer7_attention_weight_max": 91.7606430053711,
      "activations/layer7_attention_weight_min": -93.77236938476562,
      "activations/layer8_attention_weight_max": 41.24118423461914,
      "activations/layer8_attention_weight_min": -43.03920364379883,
      "activations/layer9_attention_weight_max": 33.48298263549805,
      "activations/layer9_attention_weight_min": -35.0914192199707,
      "epoch": 12.82,
      "learning_rate": 6.798636363636362e-05,
      "loss": 2.7482,
      "step": 220700
    },
    {
      "activations/layer0_attention_weight_max": 15.20484733581543,
      "activations/layer0_attention_weight_min": -12.195514678955078,
      "activations/layer10_attention_weight_max": 32.434776306152344,
      "activations/layer10_attention_weight_min": -30.821823120117188,
      "activations/layer11_attention_weight_max": 34.032691955566406,
      "activations/layer11_attention_weight_min": -31.125356674194336,
      "activations/layer12_attention_weight_max": 25.50069808959961,
      "activations/layer12_attention_weight_min": -25.18844985961914,
      "activations/layer13_attention_weight_max": 38.60874557495117,
      "activations/layer13_attention_weight_min": -35.7712516784668,
      "activations/layer14_attention_weight_max": 43.120277404785156,
      "activations/layer14_attention_weight_min": -37.00741958618164,
      "activations/layer15_attention_weight_max": 36.92195510864258,
      "activations/layer15_attention_weight_min": -33.824092864990234,
      "activations/layer16_attention_weight_max": 29.466249465942383,
      "activations/layer16_attention_weight_min": -27.10032081604004,
      "activations/layer17_attention_weight_max": 32.016334533691406,
      "activations/layer17_attention_weight_min": -24.997085571289062,
      "activations/layer18_attention_weight_max": 31.736806869506836,
      "activations/layer18_attention_weight_min": -23.902360916137695,
      "activations/layer19_attention_weight_max": 31.787424087524414,
      "activations/layer19_attention_weight_min": -30.867399215698242,
      "activations/layer1_attention_weight_max": 16.20196533203125,
      "activations/layer1_attention_weight_min": -14.965003967285156,
      "activations/layer20_attention_weight_max": 27.6428279876709,
      "activations/layer20_attention_weight_min": -24.372087478637695,
      "activations/layer21_attention_weight_max": 27.804149627685547,
      "activations/layer21_attention_weight_min": -25.08500862121582,
      "activations/layer22_attention_weight_max": 40.64877700805664,
      "activations/layer22_attention_weight_min": -29.72482681274414,
      "activations/layer23_attention_weight_max": 33.87186050415039,
      "activations/layer23_attention_weight_min": -23.999059677124023,
      "activations/layer2_attention_weight_max": 34.55525207519531,
      "activations/layer2_attention_weight_min": -33.427825927734375,
      "activations/layer3_attention_weight_max": 96.59015655517578,
      "activations/layer3_attention_weight_min": -98.2940444946289,
      "activations/layer4_attention_weight_max": 106.10809326171875,
      "activations/layer4_attention_weight_min": -112.62862396240234,
      "activations/layer5_attention_weight_max": 52.44106674194336,
      "activations/layer5_attention_weight_min": -59.04218673706055,
      "activations/layer6_attention_weight_max": 46.82832717895508,
      "activations/layer6_attention_weight_min": -48.12683868408203,
      "activations/layer7_attention_weight_max": 90.15015411376953,
      "activations/layer7_attention_weight_min": -88.27397155761719,
      "activations/layer8_attention_weight_max": 39.46221160888672,
      "activations/layer8_attention_weight_min": -40.674560546875,
      "activations/layer9_attention_weight_max": 31.113845825195312,
      "activations/layer9_attention_weight_min": -31.57191276550293,
      "epoch": 12.83,
      "learning_rate": 6.796742424242424e-05,
      "loss": 2.7772,
      "step": 220750
    },
    {
      "activations/layer0_attention_weight_max": 17.09122085571289,
      "activations/layer0_attention_weight_min": -11.67102336883545,
      "activations/layer10_attention_weight_max": 31.047813415527344,
      "activations/layer10_attention_weight_min": -33.39356231689453,
      "activations/layer11_attention_weight_max": 34.029396057128906,
      "activations/layer11_attention_weight_min": -33.2234992980957,
      "activations/layer12_attention_weight_max": 25.66468620300293,
      "activations/layer12_attention_weight_min": -31.64265251159668,
      "activations/layer13_attention_weight_max": 37.37016677856445,
      "activations/layer13_attention_weight_min": -36.795265197753906,
      "activations/layer14_attention_weight_max": 44.96066665649414,
      "activations/layer14_attention_weight_min": -39.61548614501953,
      "activations/layer15_attention_weight_max": 43.05925750732422,
      "activations/layer15_attention_weight_min": -33.3483772277832,
      "activations/layer16_attention_weight_max": 30.469877243041992,
      "activations/layer16_attention_weight_min": -28.72300910949707,
      "activations/layer17_attention_weight_max": 30.405473709106445,
      "activations/layer17_attention_weight_min": -24.850723266601562,
      "activations/layer18_attention_weight_max": 32.04990005493164,
      "activations/layer18_attention_weight_min": -25.4980525970459,
      "activations/layer19_attention_weight_max": 36.76605987548828,
      "activations/layer19_attention_weight_min": -32.482643127441406,
      "activations/layer1_attention_weight_max": 16.356658935546875,
      "activations/layer1_attention_weight_min": -14.560151100158691,
      "activations/layer20_attention_weight_max": 29.68780517578125,
      "activations/layer20_attention_weight_min": -25.637784957885742,
      "activations/layer21_attention_weight_max": 28.255992889404297,
      "activations/layer21_attention_weight_min": -22.59596824645996,
      "activations/layer22_attention_weight_max": 41.72430419921875,
      "activations/layer22_attention_weight_min": -31.917953491210938,
      "activations/layer23_attention_weight_max": 33.66166687011719,
      "activations/layer23_attention_weight_min": -24.690731048583984,
      "activations/layer2_attention_weight_max": 33.599456787109375,
      "activations/layer2_attention_weight_min": -35.522525787353516,
      "activations/layer3_attention_weight_max": 99.8027114868164,
      "activations/layer3_attention_weight_min": -98.8338394165039,
      "activations/layer4_attention_weight_max": 106.17760467529297,
      "activations/layer4_attention_weight_min": -106.10271453857422,
      "activations/layer5_attention_weight_max": 52.17644500732422,
      "activations/layer5_attention_weight_min": -62.428306579589844,
      "activations/layer6_attention_weight_max": 43.93012619018555,
      "activations/layer6_attention_weight_min": -47.9122200012207,
      "activations/layer7_attention_weight_max": 83.71881103515625,
      "activations/layer7_attention_weight_min": -87.92788696289062,
      "activations/layer8_attention_weight_max": 41.272979736328125,
      "activations/layer8_attention_weight_min": -43.34111022949219,
      "activations/layer9_attention_weight_max": 30.359434127807617,
      "activations/layer9_attention_weight_min": -34.49312210083008,
      "epoch": 12.83,
      "learning_rate": 6.794848484848484e-05,
      "loss": 2.7727,
      "step": 220800
    },
    {
      "activations/layer0_attention_weight_max": 17.080049514770508,
      "activations/layer0_attention_weight_min": -11.881522178649902,
      "activations/layer10_attention_weight_max": 38.118980407714844,
      "activations/layer10_attention_weight_min": -37.56884765625,
      "activations/layer11_attention_weight_max": 39.20117950439453,
      "activations/layer11_attention_weight_min": -37.85922622680664,
      "activations/layer12_attention_weight_max": 25.55718231201172,
      "activations/layer12_attention_weight_min": -27.956466674804688,
      "activations/layer13_attention_weight_max": 38.237274169921875,
      "activations/layer13_attention_weight_min": -35.146080017089844,
      "activations/layer14_attention_weight_max": 44.044410705566406,
      "activations/layer14_attention_weight_min": -38.757286071777344,
      "activations/layer15_attention_weight_max": 35.03481674194336,
      "activations/layer15_attention_weight_min": -34.167274475097656,
      "activations/layer16_attention_weight_max": 32.28474807739258,
      "activations/layer16_attention_weight_min": -26.66347885131836,
      "activations/layer17_attention_weight_max": 28.093639373779297,
      "activations/layer17_attention_weight_min": -26.242258071899414,
      "activations/layer18_attention_weight_max": 30.484012603759766,
      "activations/layer18_attention_weight_min": -23.300809860229492,
      "activations/layer19_attention_weight_max": 30.567625045776367,
      "activations/layer19_attention_weight_min": -28.96909523010254,
      "activations/layer1_attention_weight_max": 16.63277816772461,
      "activations/layer1_attention_weight_min": -15.062464714050293,
      "activations/layer20_attention_weight_max": 26.847476959228516,
      "activations/layer20_attention_weight_min": -24.164024353027344,
      "activations/layer21_attention_weight_max": 27.62302017211914,
      "activations/layer21_attention_weight_min": -23.207265853881836,
      "activations/layer22_attention_weight_max": 39.37074661254883,
      "activations/layer22_attention_weight_min": -28.395673751831055,
      "activations/layer23_attention_weight_max": 32.14703369140625,
      "activations/layer23_attention_weight_min": -23.738920211791992,
      "activations/layer2_attention_weight_max": 36.08177185058594,
      "activations/layer2_attention_weight_min": -33.8803825378418,
      "activations/layer3_attention_weight_max": 99.5179672241211,
      "activations/layer3_attention_weight_min": -98.31840515136719,
      "activations/layer4_attention_weight_max": 110.19525146484375,
      "activations/layer4_attention_weight_min": -108.47476959228516,
      "activations/layer5_attention_weight_max": 50.658592224121094,
      "activations/layer5_attention_weight_min": -60.27074432373047,
      "activations/layer6_attention_weight_max": 45.15610885620117,
      "activations/layer6_attention_weight_min": -46.31591033935547,
      "activations/layer7_attention_weight_max": 95.37037658691406,
      "activations/layer7_attention_weight_min": -92.85458374023438,
      "activations/layer8_attention_weight_max": 44.859169006347656,
      "activations/layer8_attention_weight_min": -42.52669143676758,
      "activations/layer9_attention_weight_max": 36.31208419799805,
      "activations/layer9_attention_weight_min": -35.52691650390625,
      "epoch": 12.83,
      "learning_rate": 6.792954545454544e-05,
      "loss": 2.7701,
      "step": 220850
    },
    {
      "activations/layer0_attention_weight_max": 15.9805269241333,
      "activations/layer0_attention_weight_min": -11.863276481628418,
      "activations/layer10_attention_weight_max": 32.96466064453125,
      "activations/layer10_attention_weight_min": -32.18824768066406,
      "activations/layer11_attention_weight_max": 31.920520782470703,
      "activations/layer11_attention_weight_min": -31.720518112182617,
      "activations/layer12_attention_weight_max": 26.699642181396484,
      "activations/layer12_attention_weight_min": -24.42500114440918,
      "activations/layer13_attention_weight_max": 37.19028854370117,
      "activations/layer13_attention_weight_min": -32.06606674194336,
      "activations/layer14_attention_weight_max": 41.11716842651367,
      "activations/layer14_attention_weight_min": -38.75394821166992,
      "activations/layer15_attention_weight_max": 35.50166320800781,
      "activations/layer15_attention_weight_min": -31.195131301879883,
      "activations/layer16_attention_weight_max": 32.51437759399414,
      "activations/layer16_attention_weight_min": -26.684789657592773,
      "activations/layer17_attention_weight_max": 31.116207122802734,
      "activations/layer17_attention_weight_min": -25.581087112426758,
      "activations/layer18_attention_weight_max": 29.64311408996582,
      "activations/layer18_attention_weight_min": -23.34018325805664,
      "activations/layer19_attention_weight_max": 37.97428512573242,
      "activations/layer19_attention_weight_min": -31.314207077026367,
      "activations/layer1_attention_weight_max": 17.35602378845215,
      "activations/layer1_attention_weight_min": -15.71619701385498,
      "activations/layer20_attention_weight_max": 36.228477478027344,
      "activations/layer20_attention_weight_min": -25.62777328491211,
      "activations/layer21_attention_weight_max": 31.754085540771484,
      "activations/layer21_attention_weight_min": -24.001712799072266,
      "activations/layer22_attention_weight_max": 44.155479431152344,
      "activations/layer22_attention_weight_min": -29.06357192993164,
      "activations/layer23_attention_weight_max": 34.99394989013672,
      "activations/layer23_attention_weight_min": -26.639890670776367,
      "activations/layer2_attention_weight_max": 33.46062088012695,
      "activations/layer2_attention_weight_min": -32.86418914794922,
      "activations/layer3_attention_weight_max": 101.43953704833984,
      "activations/layer3_attention_weight_min": -96.36434936523438,
      "activations/layer4_attention_weight_max": 110.0689697265625,
      "activations/layer4_attention_weight_min": -109.51820373535156,
      "activations/layer5_attention_weight_max": 54.60907745361328,
      "activations/layer5_attention_weight_min": -67.29916381835938,
      "activations/layer6_attention_weight_max": 47.586143493652344,
      "activations/layer6_attention_weight_min": -46.51285171508789,
      "activations/layer7_attention_weight_max": 87.76602935791016,
      "activations/layer7_attention_weight_min": -91.95842742919922,
      "activations/layer8_attention_weight_max": 39.61954879760742,
      "activations/layer8_attention_weight_min": -43.29793167114258,
      "activations/layer9_attention_weight_max": 32.3272819519043,
      "activations/layer9_attention_weight_min": -35.676883697509766,
      "epoch": 12.84,
      "learning_rate": 6.791098484848483e-05,
      "loss": 2.7573,
      "step": 220900
    },
    {
      "activations/layer0_attention_weight_max": 16.424468994140625,
      "activations/layer0_attention_weight_min": -12.502033233642578,
      "activations/layer10_attention_weight_max": 33.05659866333008,
      "activations/layer10_attention_weight_min": -31.620145797729492,
      "activations/layer11_attention_weight_max": 33.42316436767578,
      "activations/layer11_attention_weight_min": -31.575210571289062,
      "activations/layer12_attention_weight_max": 22.925003051757812,
      "activations/layer12_attention_weight_min": -24.462440490722656,
      "activations/layer13_attention_weight_max": 34.82257843017578,
      "activations/layer13_attention_weight_min": -31.607540130615234,
      "activations/layer14_attention_weight_max": 36.141632080078125,
      "activations/layer14_attention_weight_min": -34.9876594543457,
      "activations/layer15_attention_weight_max": 35.0001220703125,
      "activations/layer15_attention_weight_min": -32.590938568115234,
      "activations/layer16_attention_weight_max": 27.22286033630371,
      "activations/layer16_attention_weight_min": -29.1621150970459,
      "activations/layer17_attention_weight_max": 29.56193733215332,
      "activations/layer17_attention_weight_min": -24.73165512084961,
      "activations/layer18_attention_weight_max": 29.092477798461914,
      "activations/layer18_attention_weight_min": -23.57748031616211,
      "activations/layer19_attention_weight_max": 28.797534942626953,
      "activations/layer19_attention_weight_min": -32.152008056640625,
      "activations/layer1_attention_weight_max": 16.06987190246582,
      "activations/layer1_attention_weight_min": -14.880350112915039,
      "activations/layer20_attention_weight_max": 26.278783798217773,
      "activations/layer20_attention_weight_min": -25.985227584838867,
      "activations/layer21_attention_weight_max": 28.181211471557617,
      "activations/layer21_attention_weight_min": -24.34075927734375,
      "activations/layer22_attention_weight_max": 39.99756622314453,
      "activations/layer22_attention_weight_min": -31.163190841674805,
      "activations/layer23_attention_weight_max": 31.07834243774414,
      "activations/layer23_attention_weight_min": -25.05514144897461,
      "activations/layer2_attention_weight_max": 33.967613220214844,
      "activations/layer2_attention_weight_min": -34.319358825683594,
      "activations/layer3_attention_weight_max": 97.62744140625,
      "activations/layer3_attention_weight_min": -96.77584838867188,
      "activations/layer4_attention_weight_max": 114.46394348144531,
      "activations/layer4_attention_weight_min": -108.56261444091797,
      "activations/layer5_attention_weight_max": 53.80547332763672,
      "activations/layer5_attention_weight_min": -60.3260612487793,
      "activations/layer6_attention_weight_max": 44.4944953918457,
      "activations/layer6_attention_weight_min": -45.902427673339844,
      "activations/layer7_attention_weight_max": 86.53946685791016,
      "activations/layer7_attention_weight_min": -87.37911987304688,
      "activations/layer8_attention_weight_max": 39.798866271972656,
      "activations/layer8_attention_weight_min": -40.3802375793457,
      "activations/layer9_attention_weight_max": 31.59683609008789,
      "activations/layer9_attention_weight_min": -32.80158233642578,
      "epoch": 12.84,
      "learning_rate": 6.789204545454545e-05,
      "loss": 2.7598,
      "step": 220950
    },
    {
      "activations/layer0_attention_weight_max": 15.37441349029541,
      "activations/layer0_attention_weight_min": -11.82458782196045,
      "activations/layer10_attention_weight_max": 35.67045593261719,
      "activations/layer10_attention_weight_min": -33.90421676635742,
      "activations/layer11_attention_weight_max": 35.30665588378906,
      "activations/layer11_attention_weight_min": -33.337703704833984,
      "activations/layer12_attention_weight_max": 24.51258087158203,
      "activations/layer12_attention_weight_min": -28.172929763793945,
      "activations/layer13_attention_weight_max": 35.29328155517578,
      "activations/layer13_attention_weight_min": -33.26696014404297,
      "activations/layer14_attention_weight_max": 39.71059799194336,
      "activations/layer14_attention_weight_min": -36.55864715576172,
      "activations/layer15_attention_weight_max": 33.96729278564453,
      "activations/layer15_attention_weight_min": -31.724863052368164,
      "activations/layer16_attention_weight_max": 27.816692352294922,
      "activations/layer16_attention_weight_min": -28.079631805419922,
      "activations/layer17_attention_weight_max": 29.99900245666504,
      "activations/layer17_attention_weight_min": -24.700098037719727,
      "activations/layer18_attention_weight_max": 29.903226852416992,
      "activations/layer18_attention_weight_min": -23.033794403076172,
      "activations/layer19_attention_weight_max": 32.57344055175781,
      "activations/layer19_attention_weight_min": -29.130430221557617,
      "activations/layer1_attention_weight_max": 16.130680084228516,
      "activations/layer1_attention_weight_min": -14.678380966186523,
      "activations/layer20_attention_weight_max": 29.545320510864258,
      "activations/layer20_attention_weight_min": -22.684507369995117,
      "activations/layer21_attention_weight_max": 26.797677993774414,
      "activations/layer21_attention_weight_min": -22.511560440063477,
      "activations/layer22_attention_weight_max": 41.23355484008789,
      "activations/layer22_attention_weight_min": -28.67153549194336,
      "activations/layer23_attention_weight_max": 36.2778205871582,
      "activations/layer23_attention_weight_min": -24.875324249267578,
      "activations/layer2_attention_weight_max": 35.44108200073242,
      "activations/layer2_attention_weight_min": -33.93061828613281,
      "activations/layer3_attention_weight_max": 97.51370239257812,
      "activations/layer3_attention_weight_min": -101.02664184570312,
      "activations/layer4_attention_weight_max": 108.30574035644531,
      "activations/layer4_attention_weight_min": -108.30919647216797,
      "activations/layer5_attention_weight_max": 53.840003967285156,
      "activations/layer5_attention_weight_min": -60.353614807128906,
      "activations/layer6_attention_weight_max": 42.55521774291992,
      "activations/layer6_attention_weight_min": -44.74522018432617,
      "activations/layer7_attention_weight_max": 91.05223846435547,
      "activations/layer7_attention_weight_min": -89.28337860107422,
      "activations/layer8_attention_weight_max": 41.51472854614258,
      "activations/layer8_attention_weight_min": -43.034488677978516,
      "activations/layer9_attention_weight_max": 35.07133865356445,
      "activations/layer9_attention_weight_min": -33.306640625,
      "epoch": 12.84,
      "learning_rate": 6.787310606060605e-05,
      "loss": 2.7556,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.449,
      "eval_samples_per_second": 508.227,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.449,
      "eval_openwebtext_samples_per_second": 508.227,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9403,
      "eval_wikitext_samples_per_second": 235.015,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_lambada_loss": 2.53125,
      "eval_lambada_ppl": 12.569207830853442,
      "eval_lambada_runtime": 9.4962,
      "eval_lambada_samples_per_second": 512.731,
      "step": 221000
    },
    {
      "activations/layer0_attention_weight_max": 16.390233993530273,
      "activations/layer0_attention_weight_min": -11.551471710205078,
      "activations/layer10_attention_weight_max": 39.7120361328125,
      "activations/layer10_attention_weight_min": -39.58796691894531,
      "activations/layer11_attention_weight_max": 38.90099334716797,
      "activations/layer11_attention_weight_min": -38.92530822753906,
      "activations/layer12_attention_weight_max": 26.847614288330078,
      "activations/layer12_attention_weight_min": -27.953638076782227,
      "activations/layer13_attention_weight_max": 39.5959358215332,
      "activations/layer13_attention_weight_min": -36.384700775146484,
      "activations/layer14_attention_weight_max": 43.03997039794922,
      "activations/layer14_attention_weight_min": -39.551876068115234,
      "activations/layer15_attention_weight_max": 37.14025115966797,
      "activations/layer15_attention_weight_min": -34.64726638793945,
      "activations/layer16_attention_weight_max": 31.680753707885742,
      "activations/layer16_attention_weight_min": -30.041133880615234,
      "activations/layer17_attention_weight_max": 31.4403018951416,
      "activations/layer17_attention_weight_min": -26.725421905517578,
      "activations/layer18_attention_weight_max": 31.073665618896484,
      "activations/layer18_attention_weight_min": -23.87122344970703,
      "activations/layer19_attention_weight_max": 37.85866928100586,
      "activations/layer19_attention_weight_min": -31.32028579711914,
      "activations/layer1_attention_weight_max": 20.31597328186035,
      "activations/layer1_attention_weight_min": -18.726457595825195,
      "activations/layer20_attention_weight_max": 30.52946662902832,
      "activations/layer20_attention_weight_min": -25.473819732666016,
      "activations/layer21_attention_weight_max": 33.79340362548828,
      "activations/layer21_attention_weight_min": -26.847124099731445,
      "activations/layer22_attention_weight_max": 42.13882064819336,
      "activations/layer22_attention_weight_min": -32.6555290222168,
      "activations/layer23_attention_weight_max": 33.792598724365234,
      "activations/layer23_attention_weight_min": -27.408933639526367,
      "activations/layer2_attention_weight_max": 35.29562759399414,
      "activations/layer2_attention_weight_min": -34.536460876464844,
      "activations/layer3_attention_weight_max": 99.5372085571289,
      "activations/layer3_attention_weight_min": -102.67988586425781,
      "activations/layer4_attention_weight_max": 114.22645568847656,
      "activations/layer4_attention_weight_min": -118.61406707763672,
      "activations/layer5_attention_weight_max": 56.324195861816406,
      "activations/layer5_attention_weight_min": -60.365379333496094,
      "activations/layer6_attention_weight_max": 44.746070861816406,
      "activations/layer6_attention_weight_min": -45.25283432006836,
      "activations/layer7_attention_weight_max": 100.45722198486328,
      "activations/layer7_attention_weight_min": -98.69365692138672,
      "activations/layer8_attention_weight_max": 45.95107650756836,
      "activations/layer8_attention_weight_min": -45.959747314453125,
      "activations/layer9_attention_weight_max": 37.53377914428711,
      "activations/layer9_attention_weight_min": -36.49454879760742,
      "epoch": 12.84,
      "learning_rate": 6.785416666666666e-05,
      "loss": 2.7669,
      "step": 221050
    },
    {
      "activations/layer0_attention_weight_max": 16.680004119873047,
      "activations/layer0_attention_weight_min": -11.659771919250488,
      "activations/layer10_attention_weight_max": 31.32662582397461,
      "activations/layer10_attention_weight_min": -32.09540557861328,
      "activations/layer11_attention_weight_max": 30.90018081665039,
      "activations/layer11_attention_weight_min": -32.308265686035156,
      "activations/layer12_attention_weight_max": 29.825014114379883,
      "activations/layer12_attention_weight_min": -24.92555046081543,
      "activations/layer13_attention_weight_max": 40.1869010925293,
      "activations/layer13_attention_weight_min": -32.69340515136719,
      "activations/layer14_attention_weight_max": 43.0355224609375,
      "activations/layer14_attention_weight_min": -35.96641540527344,
      "activations/layer15_attention_weight_max": 36.00996398925781,
      "activations/layer15_attention_weight_min": -32.721832275390625,
      "activations/layer16_attention_weight_max": 29.584720611572266,
      "activations/layer16_attention_weight_min": -28.064584732055664,
      "activations/layer17_attention_weight_max": 31.18792724609375,
      "activations/layer17_attention_weight_min": -25.585845947265625,
      "activations/layer18_attention_weight_max": 30.645109176635742,
      "activations/layer18_attention_weight_min": -22.978418350219727,
      "activations/layer19_attention_weight_max": 30.28740882873535,
      "activations/layer19_attention_weight_min": -31.070219039916992,
      "activations/layer1_attention_weight_max": 16.272916793823242,
      "activations/layer1_attention_weight_min": -15.219048500061035,
      "activations/layer20_attention_weight_max": 26.2205810546875,
      "activations/layer20_attention_weight_min": -24.103853225708008,
      "activations/layer21_attention_weight_max": 26.746891021728516,
      "activations/layer21_attention_weight_min": -24.10556983947754,
      "activations/layer22_attention_weight_max": 43.21947479248047,
      "activations/layer22_attention_weight_min": -29.39122772216797,
      "activations/layer23_attention_weight_max": 32.16437530517578,
      "activations/layer23_attention_weight_min": -26.545331954956055,
      "activations/layer2_attention_weight_max": 33.052005767822266,
      "activations/layer2_attention_weight_min": -32.73375701904297,
      "activations/layer3_attention_weight_max": 96.66678619384766,
      "activations/layer3_attention_weight_min": -100.26532745361328,
      "activations/layer4_attention_weight_max": 106.3471450805664,
      "activations/layer4_attention_weight_min": -110.19246673583984,
      "activations/layer5_attention_weight_max": 51.04063415527344,
      "activations/layer5_attention_weight_min": -60.41506576538086,
      "activations/layer6_attention_weight_max": 45.15888595581055,
      "activations/layer6_attention_weight_min": -46.86699295043945,
      "activations/layer7_attention_weight_max": 87.53441619873047,
      "activations/layer7_attention_weight_min": -88.38414001464844,
      "activations/layer8_attention_weight_max": 38.244781494140625,
      "activations/layer8_attention_weight_min": -40.2440299987793,
      "activations/layer9_attention_weight_max": 31.705007553100586,
      "activations/layer9_attention_weight_min": -33.65578842163086,
      "epoch": 12.85,
      "learning_rate": 6.783522727272727e-05,
      "loss": 2.7724,
      "step": 221100
    },
    {
      "activations/layer0_attention_weight_max": 15.311393737792969,
      "activations/layer0_attention_weight_min": -12.05386734008789,
      "activations/layer10_attention_weight_max": 33.779293060302734,
      "activations/layer10_attention_weight_min": -33.65407180786133,
      "activations/layer11_attention_weight_max": 34.21438980102539,
      "activations/layer11_attention_weight_min": -33.37644577026367,
      "activations/layer12_attention_weight_max": 36.334110260009766,
      "activations/layer12_attention_weight_min": -27.122560501098633,
      "activations/layer13_attention_weight_max": 47.80811309814453,
      "activations/layer13_attention_weight_min": -35.52073669433594,
      "activations/layer14_attention_weight_max": 48.04847717285156,
      "activations/layer14_attention_weight_min": -37.66577911376953,
      "activations/layer15_attention_weight_max": 41.55625915527344,
      "activations/layer15_attention_weight_min": -33.269569396972656,
      "activations/layer16_attention_weight_max": 34.442543029785156,
      "activations/layer16_attention_weight_min": -27.19727325439453,
      "activations/layer17_attention_weight_max": 35.38503646850586,
      "activations/layer17_attention_weight_min": -27.579334259033203,
      "activations/layer18_attention_weight_max": 36.251834869384766,
      "activations/layer18_attention_weight_min": -26.431066513061523,
      "activations/layer19_attention_weight_max": 38.324256896972656,
      "activations/layer19_attention_weight_min": -29.135831832885742,
      "activations/layer1_attention_weight_max": 16.746105194091797,
      "activations/layer1_attention_weight_min": -14.493268966674805,
      "activations/layer20_attention_weight_max": 34.27576446533203,
      "activations/layer20_attention_weight_min": -25.008216857910156,
      "activations/layer21_attention_weight_max": 33.48091506958008,
      "activations/layer21_attention_weight_min": -23.155824661254883,
      "activations/layer22_attention_weight_max": 45.49493408203125,
      "activations/layer22_attention_weight_min": -28.46276092529297,
      "activations/layer23_attention_weight_max": 34.06319046020508,
      "activations/layer23_attention_weight_min": -26.21737289428711,
      "activations/layer2_attention_weight_max": 32.70414352416992,
      "activations/layer2_attention_weight_min": -34.68379592895508,
      "activations/layer3_attention_weight_max": 97.66085052490234,
      "activations/layer3_attention_weight_min": -98.10480499267578,
      "activations/layer4_attention_weight_max": 109.84223937988281,
      "activations/layer4_attention_weight_min": -109.40472412109375,
      "activations/layer5_attention_weight_max": 52.37795639038086,
      "activations/layer5_attention_weight_min": -62.41703796386719,
      "activations/layer6_attention_weight_max": 41.43600082397461,
      "activations/layer6_attention_weight_min": -47.208614349365234,
      "activations/layer7_attention_weight_max": 89.61782836914062,
      "activations/layer7_attention_weight_min": -90.45431518554688,
      "activations/layer8_attention_weight_max": 39.93535232543945,
      "activations/layer8_attention_weight_min": -41.63473892211914,
      "activations/layer9_attention_weight_max": 34.808746337890625,
      "activations/layer9_attention_weight_min": -36.031471252441406,
      "epoch": 12.85,
      "learning_rate": 6.781628787878788e-05,
      "loss": 2.7643,
      "step": 221150
    },
    {
      "activations/layer0_attention_weight_max": 16.77583885192871,
      "activations/layer0_attention_weight_min": -11.9624662399292,
      "activations/layer10_attention_weight_max": 38.4681282043457,
      "activations/layer10_attention_weight_min": -36.0381965637207,
      "activations/layer11_attention_weight_max": 37.74015808105469,
      "activations/layer11_attention_weight_min": -35.85981750488281,
      "activations/layer12_attention_weight_max": 27.76663589477539,
      "activations/layer12_attention_weight_min": -27.145729064941406,
      "activations/layer13_attention_weight_max": 41.17742919921875,
      "activations/layer13_attention_weight_min": -35.463069915771484,
      "activations/layer14_attention_weight_max": 45.56029510498047,
      "activations/layer14_attention_weight_min": -41.6818733215332,
      "activations/layer15_attention_weight_max": 35.97990417480469,
      "activations/layer15_attention_weight_min": -33.92363739013672,
      "activations/layer16_attention_weight_max": 30.67332649230957,
      "activations/layer16_attention_weight_min": -27.143545150756836,
      "activations/layer17_attention_weight_max": 29.52619743347168,
      "activations/layer17_attention_weight_min": -25.47538185119629,
      "activations/layer18_attention_weight_max": 28.457271575927734,
      "activations/layer18_attention_weight_min": -25.520244598388672,
      "activations/layer19_attention_weight_max": 35.95992660522461,
      "activations/layer19_attention_weight_min": -30.488367080688477,
      "activations/layer1_attention_weight_max": 16.524057388305664,
      "activations/layer1_attention_weight_min": -16.024478912353516,
      "activations/layer20_attention_weight_max": 29.727691650390625,
      "activations/layer20_attention_weight_min": -25.664226531982422,
      "activations/layer21_attention_weight_max": 25.61905860900879,
      "activations/layer21_attention_weight_min": -25.543914794921875,
      "activations/layer22_attention_weight_max": 40.756370544433594,
      "activations/layer22_attention_weight_min": -32.21360397338867,
      "activations/layer23_attention_weight_max": 34.46931076049805,
      "activations/layer23_attention_weight_min": -25.37613296508789,
      "activations/layer2_attention_weight_max": 33.27824020385742,
      "activations/layer2_attention_weight_min": -33.45656967163086,
      "activations/layer3_attention_weight_max": 99.962158203125,
      "activations/layer3_attention_weight_min": -104.04884338378906,
      "activations/layer4_attention_weight_max": 117.6161117553711,
      "activations/layer4_attention_weight_min": -120.34881591796875,
      "activations/layer5_attention_weight_max": 54.33729553222656,
      "activations/layer5_attention_weight_min": -61.664241790771484,
      "activations/layer6_attention_weight_max": 48.30868148803711,
      "activations/layer6_attention_weight_min": -49.83370590209961,
      "activations/layer7_attention_weight_max": 99.0826416015625,
      "activations/layer7_attention_weight_min": -101.66239166259766,
      "activations/layer8_attention_weight_max": 44.53133010864258,
      "activations/layer8_attention_weight_min": -46.70862579345703,
      "activations/layer9_attention_weight_max": 35.452396392822266,
      "activations/layer9_attention_weight_min": -36.63256072998047,
      "epoch": 12.85,
      "learning_rate": 6.779734848484848e-05,
      "loss": 2.7647,
      "step": 221200
    },
    {
      "activations/layer0_attention_weight_max": 16.127525329589844,
      "activations/layer0_attention_weight_min": -11.659128189086914,
      "activations/layer10_attention_weight_max": 30.713743209838867,
      "activations/layer10_attention_weight_min": -31.507980346679688,
      "activations/layer11_attention_weight_max": 30.704084396362305,
      "activations/layer11_attention_weight_min": -31.244075775146484,
      "activations/layer12_attention_weight_max": 25.81723976135254,
      "activations/layer12_attention_weight_min": -24.83592987060547,
      "activations/layer13_attention_weight_max": 36.92436599731445,
      "activations/layer13_attention_weight_min": -32.485469818115234,
      "activations/layer14_attention_weight_max": 42.74409866333008,
      "activations/layer14_attention_weight_min": -37.62184143066406,
      "activations/layer15_attention_weight_max": 35.47637939453125,
      "activations/layer15_attention_weight_min": -30.332244873046875,
      "activations/layer16_attention_weight_max": 30.172388076782227,
      "activations/layer16_attention_weight_min": -27.38067626953125,
      "activations/layer17_attention_weight_max": 30.895105361938477,
      "activations/layer17_attention_weight_min": -26.191619873046875,
      "activations/layer18_attention_weight_max": 30.885311126708984,
      "activations/layer18_attention_weight_min": -24.533641815185547,
      "activations/layer19_attention_weight_max": 32.44634246826172,
      "activations/layer19_attention_weight_min": -29.913679122924805,
      "activations/layer1_attention_weight_max": 16.367778778076172,
      "activations/layer1_attention_weight_min": -16.917415618896484,
      "activations/layer20_attention_weight_max": 28.686681747436523,
      "activations/layer20_attention_weight_min": -25.31416893005371,
      "activations/layer21_attention_weight_max": 28.271709442138672,
      "activations/layer21_attention_weight_min": -25.99951171875,
      "activations/layer22_attention_weight_max": 41.30825424194336,
      "activations/layer22_attention_weight_min": -32.5014762878418,
      "activations/layer23_attention_weight_max": 35.02804183959961,
      "activations/layer23_attention_weight_min": -26.475921630859375,
      "activations/layer2_attention_weight_max": 33.308868408203125,
      "activations/layer2_attention_weight_min": -33.9720458984375,
      "activations/layer3_attention_weight_max": 97.0296401977539,
      "activations/layer3_attention_weight_min": -106.49381256103516,
      "activations/layer4_attention_weight_max": 113.53852844238281,
      "activations/layer4_attention_weight_min": -114.2367172241211,
      "activations/layer5_attention_weight_max": 55.07096862792969,
      "activations/layer5_attention_weight_min": -60.302032470703125,
      "activations/layer6_attention_weight_max": 44.294429779052734,
      "activations/layer6_attention_weight_min": -49.11783981323242,
      "activations/layer7_attention_weight_max": 89.76841735839844,
      "activations/layer7_attention_weight_min": -92.5401611328125,
      "activations/layer8_attention_weight_max": 40.81294250488281,
      "activations/layer8_attention_weight_min": -41.82342529296875,
      "activations/layer9_attention_weight_max": 30.560714721679688,
      "activations/layer9_attention_weight_min": -32.980159759521484,
      "epoch": 12.86,
      "learning_rate": 6.777840909090908e-05,
      "loss": 2.7805,
      "step": 221250
    },
    {
      "activations/layer0_attention_weight_max": 16.199996948242188,
      "activations/layer0_attention_weight_min": -11.822053909301758,
      "activations/layer10_attention_weight_max": 33.6385612487793,
      "activations/layer10_attention_weight_min": -34.53266906738281,
      "activations/layer11_attention_weight_max": 32.99293518066406,
      "activations/layer11_attention_weight_min": -32.85344696044922,
      "activations/layer12_attention_weight_max": 27.371461868286133,
      "activations/layer12_attention_weight_min": -25.39780616760254,
      "activations/layer13_attention_weight_max": 43.74428939819336,
      "activations/layer13_attention_weight_min": -34.81349563598633,
      "activations/layer14_attention_weight_max": 43.99917984008789,
      "activations/layer14_attention_weight_min": -36.87337875366211,
      "activations/layer15_attention_weight_max": 38.81098175048828,
      "activations/layer15_attention_weight_min": -34.18478775024414,
      "activations/layer16_attention_weight_max": 30.93418312072754,
      "activations/layer16_attention_weight_min": -28.52463722229004,
      "activations/layer17_attention_weight_max": 33.42443084716797,
      "activations/layer17_attention_weight_min": -26.601848602294922,
      "activations/layer18_attention_weight_max": 31.222524642944336,
      "activations/layer18_attention_weight_min": -24.377851486206055,
      "activations/layer19_attention_weight_max": 33.6766242980957,
      "activations/layer19_attention_weight_min": -29.515539169311523,
      "activations/layer1_attention_weight_max": 17.851459503173828,
      "activations/layer1_attention_weight_min": -14.744969367980957,
      "activations/layer20_attention_weight_max": 27.99920654296875,
      "activations/layer20_attention_weight_min": -24.257970809936523,
      "activations/layer21_attention_weight_max": 29.76134490966797,
      "activations/layer21_attention_weight_min": -24.329906463623047,
      "activations/layer22_attention_weight_max": 44.26772689819336,
      "activations/layer22_attention_weight_min": -30.964611053466797,
      "activations/layer23_attention_weight_max": 35.448204040527344,
      "activations/layer23_attention_weight_min": -26.067222595214844,
      "activations/layer2_attention_weight_max": 33.970245361328125,
      "activations/layer2_attention_weight_min": -34.36998748779297,
      "activations/layer3_attention_weight_max": 99.41936492919922,
      "activations/layer3_attention_weight_min": -100.85218048095703,
      "activations/layer4_attention_weight_max": 115.77099609375,
      "activations/layer4_attention_weight_min": -116.12446594238281,
      "activations/layer5_attention_weight_max": 56.31797790527344,
      "activations/layer5_attention_weight_min": -65.9937515258789,
      "activations/layer6_attention_weight_max": 45.83479690551758,
      "activations/layer6_attention_weight_min": -47.883750915527344,
      "activations/layer7_attention_weight_max": 93.3796157836914,
      "activations/layer7_attention_weight_min": -96.92326354980469,
      "activations/layer8_attention_weight_max": 43.79731750488281,
      "activations/layer8_attention_weight_min": -41.47210693359375,
      "activations/layer9_attention_weight_max": 33.18320083618164,
      "activations/layer9_attention_weight_min": -34.45931625366211,
      "epoch": 12.86,
      "learning_rate": 6.775946969696969e-05,
      "loss": 2.7767,
      "step": 221300
    },
    {
      "activations/layer0_attention_weight_max": 15.811532974243164,
      "activations/layer0_attention_weight_min": -11.326729774475098,
      "activations/layer10_attention_weight_max": 36.09598922729492,
      "activations/layer10_attention_weight_min": -33.92244338989258,
      "activations/layer11_attention_weight_max": 37.40147018432617,
      "activations/layer11_attention_weight_min": -33.18535614013672,
      "activations/layer12_attention_weight_max": 24.248497009277344,
      "activations/layer12_attention_weight_min": -23.83197784423828,
      "activations/layer13_attention_weight_max": 37.54437255859375,
      "activations/layer13_attention_weight_min": -34.06658172607422,
      "activations/layer14_attention_weight_max": 41.69870376586914,
      "activations/layer14_attention_weight_min": -37.33440017700195,
      "activations/layer15_attention_weight_max": 36.826622009277344,
      "activations/layer15_attention_weight_min": -32.10161590576172,
      "activations/layer16_attention_weight_max": 29.981769561767578,
      "activations/layer16_attention_weight_min": -26.606666564941406,
      "activations/layer17_attention_weight_max": 31.13205337524414,
      "activations/layer17_attention_weight_min": -25.935686111450195,
      "activations/layer18_attention_weight_max": 34.070655822753906,
      "activations/layer18_attention_weight_min": -23.931909561157227,
      "activations/layer19_attention_weight_max": 33.148616790771484,
      "activations/layer19_attention_weight_min": -29.71864891052246,
      "activations/layer1_attention_weight_max": 15.671905517578125,
      "activations/layer1_attention_weight_min": -15.036351203918457,
      "activations/layer20_attention_weight_max": 29.78900718688965,
      "activations/layer20_attention_weight_min": -24.238378524780273,
      "activations/layer21_attention_weight_max": 28.20736312866211,
      "activations/layer21_attention_weight_min": -24.391992568969727,
      "activations/layer22_attention_weight_max": 42.86466979980469,
      "activations/layer22_attention_weight_min": -29.2271728515625,
      "activations/layer23_attention_weight_max": 34.930702209472656,
      "activations/layer23_attention_weight_min": -25.06830596923828,
      "activations/layer2_attention_weight_max": 33.013633728027344,
      "activations/layer2_attention_weight_min": -32.82123947143555,
      "activations/layer3_attention_weight_max": 98.01316833496094,
      "activations/layer3_attention_weight_min": -95.27339172363281,
      "activations/layer4_attention_weight_max": 109.21733856201172,
      "activations/layer4_attention_weight_min": -108.74068450927734,
      "activations/layer5_attention_weight_max": 50.93705749511719,
      "activations/layer5_attention_weight_min": -60.631195068359375,
      "activations/layer6_attention_weight_max": 42.564353942871094,
      "activations/layer6_attention_weight_min": -45.1729621887207,
      "activations/layer7_attention_weight_max": 94.20629119873047,
      "activations/layer7_attention_weight_min": -91.92466735839844,
      "activations/layer8_attention_weight_max": 41.72468185424805,
      "activations/layer8_attention_weight_min": -42.36577224731445,
      "activations/layer9_attention_weight_max": 33.43589401245117,
      "activations/layer9_attention_weight_min": -33.0932731628418,
      "epoch": 12.86,
      "learning_rate": 6.77405303030303e-05,
      "loss": 2.7511,
      "step": 221350
    },
    {
      "activations/layer0_attention_weight_max": 16.031391143798828,
      "activations/layer0_attention_weight_min": -12.540953636169434,
      "activations/layer10_attention_weight_max": 32.551517486572266,
      "activations/layer10_attention_weight_min": -32.874542236328125,
      "activations/layer11_attention_weight_max": 33.55633544921875,
      "activations/layer11_attention_weight_min": -31.97592544555664,
      "activations/layer12_attention_weight_max": 25.761194229125977,
      "activations/layer12_attention_weight_min": -24.425739288330078,
      "activations/layer13_attention_weight_max": 39.382720947265625,
      "activations/layer13_attention_weight_min": -31.469684600830078,
      "activations/layer14_attention_weight_max": 39.45100021362305,
      "activations/layer14_attention_weight_min": -33.05350112915039,
      "activations/layer15_attention_weight_max": 38.147438049316406,
      "activations/layer15_attention_weight_min": -31.05462646484375,
      "activations/layer16_attention_weight_max": 32.02397155761719,
      "activations/layer16_attention_weight_min": -25.956987380981445,
      "activations/layer17_attention_weight_max": 32.69359588623047,
      "activations/layer17_attention_weight_min": -25.139280319213867,
      "activations/layer18_attention_weight_max": 32.304115295410156,
      "activations/layer18_attention_weight_min": -23.5266170501709,
      "activations/layer19_attention_weight_max": 32.866458892822266,
      "activations/layer19_attention_weight_min": -31.10890769958496,
      "activations/layer1_attention_weight_max": 16.651681900024414,
      "activations/layer1_attention_weight_min": -15.958642959594727,
      "activations/layer20_attention_weight_max": 30.696672439575195,
      "activations/layer20_attention_weight_min": -25.92745018005371,
      "activations/layer21_attention_weight_max": 29.020381927490234,
      "activations/layer21_attention_weight_min": -22.811767578125,
      "activations/layer22_attention_weight_max": 40.8300666809082,
      "activations/layer22_attention_weight_min": -29.950822830200195,
      "activations/layer23_attention_weight_max": 33.87530517578125,
      "activations/layer23_attention_weight_min": -25.237506866455078,
      "activations/layer2_attention_weight_max": 31.920970916748047,
      "activations/layer2_attention_weight_min": -31.929662704467773,
      "activations/layer3_attention_weight_max": 94.71379089355469,
      "activations/layer3_attention_weight_min": -94.09162139892578,
      "activations/layer4_attention_weight_max": 112.692626953125,
      "activations/layer4_attention_weight_min": -111.89054107666016,
      "activations/layer5_attention_weight_max": 55.87548065185547,
      "activations/layer5_attention_weight_min": -60.600093841552734,
      "activations/layer6_attention_weight_max": 44.15895462036133,
      "activations/layer6_attention_weight_min": -46.88709259033203,
      "activations/layer7_attention_weight_max": 90.33344268798828,
      "activations/layer7_attention_weight_min": -88.94685363769531,
      "activations/layer8_attention_weight_max": 38.16815948486328,
      "activations/layer8_attention_weight_min": -41.0972785949707,
      "activations/layer9_attention_weight_max": 30.763662338256836,
      "activations/layer9_attention_weight_min": -34.857635498046875,
      "epoch": 12.86,
      "learning_rate": 6.77215909090909e-05,
      "loss": 2.7663,
      "step": 221400
    },
    {
      "activations/layer0_attention_weight_max": 15.823586463928223,
      "activations/layer0_attention_weight_min": -11.864521980285645,
      "activations/layer10_attention_weight_max": 30.813108444213867,
      "activations/layer10_attention_weight_min": -31.4814510345459,
      "activations/layer11_attention_weight_max": 32.64459228515625,
      "activations/layer11_attention_weight_min": -33.02976608276367,
      "activations/layer12_attention_weight_max": 24.19830322265625,
      "activations/layer12_attention_weight_min": -25.648317337036133,
      "activations/layer13_attention_weight_max": 37.922080993652344,
      "activations/layer13_attention_weight_min": -33.45463562011719,
      "activations/layer14_attention_weight_max": 42.11178970336914,
      "activations/layer14_attention_weight_min": -36.56396484375,
      "activations/layer15_attention_weight_max": 36.787208557128906,
      "activations/layer15_attention_weight_min": -31.11850929260254,
      "activations/layer16_attention_weight_max": 30.08247184753418,
      "activations/layer16_attention_weight_min": -26.495012283325195,
      "activations/layer17_attention_weight_max": 33.130027770996094,
      "activations/layer17_attention_weight_min": -25.09071922302246,
      "activations/layer18_attention_weight_max": 34.765079498291016,
      "activations/layer18_attention_weight_min": -25.084014892578125,
      "activations/layer19_attention_weight_max": 35.2680778503418,
      "activations/layer19_attention_weight_min": -32.0028190612793,
      "activations/layer1_attention_weight_max": 16.241939544677734,
      "activations/layer1_attention_weight_min": -14.64428424835205,
      "activations/layer20_attention_weight_max": 30.789560317993164,
      "activations/layer20_attention_weight_min": -24.292190551757812,
      "activations/layer21_attention_weight_max": 30.209148406982422,
      "activations/layer21_attention_weight_min": -23.817838668823242,
      "activations/layer22_attention_weight_max": 41.919429779052734,
      "activations/layer22_attention_weight_min": -30.43264389038086,
      "activations/layer23_attention_weight_max": 37.44059371948242,
      "activations/layer23_attention_weight_min": -23.314434051513672,
      "activations/layer2_attention_weight_max": 31.46613121032715,
      "activations/layer2_attention_weight_min": -30.62232208251953,
      "activations/layer3_attention_weight_max": 90.98281860351562,
      "activations/layer3_attention_weight_min": -92.46662139892578,
      "activations/layer4_attention_weight_max": 102.6549301147461,
      "activations/layer4_attention_weight_min": -105.510986328125,
      "activations/layer5_attention_weight_max": 47.720298767089844,
      "activations/layer5_attention_weight_min": -59.052303314208984,
      "activations/layer6_attention_weight_max": 41.13066101074219,
      "activations/layer6_attention_weight_min": -43.196712493896484,
      "activations/layer7_attention_weight_max": 83.27923583984375,
      "activations/layer7_attention_weight_min": -83.41801452636719,
      "activations/layer8_attention_weight_max": 39.92858123779297,
      "activations/layer8_attention_weight_min": -37.88727951049805,
      "activations/layer9_attention_weight_max": 31.527475357055664,
      "activations/layer9_attention_weight_min": -30.84711265563965,
      "epoch": 12.87,
      "learning_rate": 6.77026515151515e-05,
      "loss": 2.7639,
      "step": 221450
    },
    {
      "activations/layer0_attention_weight_max": 16.253257751464844,
      "activations/layer0_attention_weight_min": -12.08203411102295,
      "activations/layer10_attention_weight_max": 40.50937271118164,
      "activations/layer10_attention_weight_min": -38.183719635009766,
      "activations/layer11_attention_weight_max": 43.61417770385742,
      "activations/layer11_attention_weight_min": -37.58458709716797,
      "activations/layer12_attention_weight_max": 29.113265991210938,
      "activations/layer12_attention_weight_min": -27.331119537353516,
      "activations/layer13_attention_weight_max": 46.56245040893555,
      "activations/layer13_attention_weight_min": -38.03564453125,
      "activations/layer14_attention_weight_max": 49.53567886352539,
      "activations/layer14_attention_weight_min": -41.14978790283203,
      "activations/layer15_attention_weight_max": 39.5614013671875,
      "activations/layer15_attention_weight_min": -34.890987396240234,
      "activations/layer16_attention_weight_max": 34.2147102355957,
      "activations/layer16_attention_weight_min": -27.907806396484375,
      "activations/layer17_attention_weight_max": 30.10898780822754,
      "activations/layer17_attention_weight_min": -26.632505416870117,
      "activations/layer18_attention_weight_max": 31.871986389160156,
      "activations/layer18_attention_weight_min": -23.72534942626953,
      "activations/layer19_attention_weight_max": 34.55986404418945,
      "activations/layer19_attention_weight_min": -32.72034454345703,
      "activations/layer1_attention_weight_max": 15.989592552185059,
      "activations/layer1_attention_weight_min": -14.791118621826172,
      "activations/layer20_attention_weight_max": 31.84869384765625,
      "activations/layer20_attention_weight_min": -26.608434677124023,
      "activations/layer21_attention_weight_max": 33.18634796142578,
      "activations/layer21_attention_weight_min": -26.240501403808594,
      "activations/layer22_attention_weight_max": 50.88751220703125,
      "activations/layer22_attention_weight_min": -31.50919532775879,
      "activations/layer23_attention_weight_max": 36.795814514160156,
      "activations/layer23_attention_weight_min": -25.88591766357422,
      "activations/layer2_attention_weight_max": 36.14794921875,
      "activations/layer2_attention_weight_min": -33.609092712402344,
      "activations/layer3_attention_weight_max": 103.71648406982422,
      "activations/layer3_attention_weight_min": -103.75199127197266,
      "activations/layer4_attention_weight_max": 114.77816009521484,
      "activations/layer4_attention_weight_min": -113.27202606201172,
      "activations/layer5_attention_weight_max": 52.011775970458984,
      "activations/layer5_attention_weight_min": -61.98661422729492,
      "activations/layer6_attention_weight_max": 47.09524917602539,
      "activations/layer6_attention_weight_min": -47.477108001708984,
      "activations/layer7_attention_weight_max": 99.27106475830078,
      "activations/layer7_attention_weight_min": -98.35568237304688,
      "activations/layer8_attention_weight_max": 46.4799919128418,
      "activations/layer8_attention_weight_min": -44.29706573486328,
      "activations/layer9_attention_weight_max": 39.37948989868164,
      "activations/layer9_attention_weight_min": -34.7580451965332,
      "epoch": 12.87,
      "learning_rate": 6.768371212121212e-05,
      "loss": 2.7694,
      "step": 221500
    },
    {
      "activations/layer0_attention_weight_max": 16.07162094116211,
      "activations/layer0_attention_weight_min": -11.957277297973633,
      "activations/layer10_attention_weight_max": 32.94557189941406,
      "activations/layer10_attention_weight_min": -33.59788131713867,
      "activations/layer11_attention_weight_max": 34.2537956237793,
      "activations/layer11_attention_weight_min": -34.38718032836914,
      "activations/layer12_attention_weight_max": 24.736188888549805,
      "activations/layer12_attention_weight_min": -24.583759307861328,
      "activations/layer13_attention_weight_max": 35.43052291870117,
      "activations/layer13_attention_weight_min": -34.68001937866211,
      "activations/layer14_attention_weight_max": 42.5068359375,
      "activations/layer14_attention_weight_min": -38.23069381713867,
      "activations/layer15_attention_weight_max": 35.016441345214844,
      "activations/layer15_attention_weight_min": -32.616146087646484,
      "activations/layer16_attention_weight_max": 31.141077041625977,
      "activations/layer16_attention_weight_min": -29.091936111450195,
      "activations/layer17_attention_weight_max": 33.097965240478516,
      "activations/layer17_attention_weight_min": -25.98320770263672,
      "activations/layer18_attention_weight_max": 34.73902130126953,
      "activations/layer18_attention_weight_min": -24.236955642700195,
      "activations/layer19_attention_weight_max": 37.953407287597656,
      "activations/layer19_attention_weight_min": -30.132761001586914,
      "activations/layer1_attention_weight_max": 16.277029037475586,
      "activations/layer1_attention_weight_min": -14.167211532592773,
      "activations/layer20_attention_weight_max": 33.491798400878906,
      "activations/layer20_attention_weight_min": -24.079734802246094,
      "activations/layer21_attention_weight_max": 32.25550842285156,
      "activations/layer21_attention_weight_min": -24.143705368041992,
      "activations/layer22_attention_weight_max": 48.521575927734375,
      "activations/layer22_attention_weight_min": -31.282773971557617,
      "activations/layer23_attention_weight_max": 40.17847442626953,
      "activations/layer23_attention_weight_min": -26.062366485595703,
      "activations/layer2_attention_weight_max": 31.17147445678711,
      "activations/layer2_attention_weight_min": -31.04463005065918,
      "activations/layer3_attention_weight_max": 94.19911193847656,
      "activations/layer3_attention_weight_min": -92.35999298095703,
      "activations/layer4_attention_weight_max": 106.968017578125,
      "activations/layer4_attention_weight_min": -103.6958999633789,
      "activations/layer5_attention_weight_max": 51.77522277832031,
      "activations/layer5_attention_weight_min": -61.69822311401367,
      "activations/layer6_attention_weight_max": 43.72862243652344,
      "activations/layer6_attention_weight_min": -48.68852233886719,
      "activations/layer7_attention_weight_max": 91.64751434326172,
      "activations/layer7_attention_weight_min": -90.8902587890625,
      "activations/layer8_attention_weight_max": 40.365047454833984,
      "activations/layer8_attention_weight_min": -41.762603759765625,
      "activations/layer9_attention_weight_max": 32.20002365112305,
      "activations/layer9_attention_weight_min": -33.31156921386719,
      "epoch": 12.87,
      "learning_rate": 6.766477272727272e-05,
      "loss": 2.7963,
      "step": 221550
    },
    {
      "activations/layer0_attention_weight_max": 15.175859451293945,
      "activations/layer0_attention_weight_min": -12.555153846740723,
      "activations/layer10_attention_weight_max": 34.20570755004883,
      "activations/layer10_attention_weight_min": -32.192474365234375,
      "activations/layer11_attention_weight_max": 32.002166748046875,
      "activations/layer11_attention_weight_min": -31.96649932861328,
      "activations/layer12_attention_weight_max": 25.83297348022461,
      "activations/layer12_attention_weight_min": -25.13483238220215,
      "activations/layer13_attention_weight_max": 39.55730438232422,
      "activations/layer13_attention_weight_min": -33.90024948120117,
      "activations/layer14_attention_weight_max": 45.36183547973633,
      "activations/layer14_attention_weight_min": -37.77867889404297,
      "activations/layer15_attention_weight_max": 37.845951080322266,
      "activations/layer15_attention_weight_min": -33.675540924072266,
      "activations/layer16_attention_weight_max": 32.390708923339844,
      "activations/layer16_attention_weight_min": -28.43939208984375,
      "activations/layer17_attention_weight_max": 32.9755744934082,
      "activations/layer17_attention_weight_min": -24.90167236328125,
      "activations/layer18_attention_weight_max": 35.47576904296875,
      "activations/layer18_attention_weight_min": -25.352590560913086,
      "activations/layer19_attention_weight_max": 38.28825759887695,
      "activations/layer19_attention_weight_min": -32.69782638549805,
      "activations/layer1_attention_weight_max": 16.081995010375977,
      "activations/layer1_attention_weight_min": -14.546157836914062,
      "activations/layer20_attention_weight_max": 31.645767211914062,
      "activations/layer20_attention_weight_min": -25.938987731933594,
      "activations/layer21_attention_weight_max": 31.06463623046875,
      "activations/layer21_attention_weight_min": -24.54661750793457,
      "activations/layer22_attention_weight_max": 46.61269760131836,
      "activations/layer22_attention_weight_min": -32.80765914916992,
      "activations/layer23_attention_weight_max": 36.79643249511719,
      "activations/layer23_attention_weight_min": -28.87171745300293,
      "activations/layer2_attention_weight_max": 31.685087203979492,
      "activations/layer2_attention_weight_min": -31.856460571289062,
      "activations/layer3_attention_weight_max": 96.57759857177734,
      "activations/layer3_attention_weight_min": -101.33026885986328,
      "activations/layer4_attention_weight_max": 108.3701171875,
      "activations/layer4_attention_weight_min": -108.8637924194336,
      "activations/layer5_attention_weight_max": 51.864559173583984,
      "activations/layer5_attention_weight_min": -59.925628662109375,
      "activations/layer6_attention_weight_max": 42.24272155761719,
      "activations/layer6_attention_weight_min": -45.228824615478516,
      "activations/layer7_attention_weight_max": 88.34428405761719,
      "activations/layer7_attention_weight_min": -85.80828094482422,
      "activations/layer8_attention_weight_max": 38.68275833129883,
      "activations/layer8_attention_weight_min": -41.90327453613281,
      "activations/layer9_attention_weight_max": 33.70486068725586,
      "activations/layer9_attention_weight_min": -33.723487854003906,
      "epoch": 12.88,
      "learning_rate": 6.764583333333332e-05,
      "loss": 2.7738,
      "step": 221600
    },
    {
      "activations/layer0_attention_weight_max": 16.56403350830078,
      "activations/layer0_attention_weight_min": -11.949295043945312,
      "activations/layer10_attention_weight_max": 32.082069396972656,
      "activations/layer10_attention_weight_min": -33.101200103759766,
      "activations/layer11_attention_weight_max": 31.496129989624023,
      "activations/layer11_attention_weight_min": -32.11419677734375,
      "activations/layer12_attention_weight_max": 25.742843627929688,
      "activations/layer12_attention_weight_min": -26.427011489868164,
      "activations/layer13_attention_weight_max": 38.6710319519043,
      "activations/layer13_attention_weight_min": -34.83452606201172,
      "activations/layer14_attention_weight_max": 42.042816162109375,
      "activations/layer14_attention_weight_min": -37.762794494628906,
      "activations/layer15_attention_weight_max": 35.17915344238281,
      "activations/layer15_attention_weight_min": -32.170230865478516,
      "activations/layer16_attention_weight_max": 29.268491744995117,
      "activations/layer16_attention_weight_min": -27.76470375061035,
      "activations/layer17_attention_weight_max": 33.655914306640625,
      "activations/layer17_attention_weight_min": -27.712921142578125,
      "activations/layer18_attention_weight_max": 33.47605514526367,
      "activations/layer18_attention_weight_min": -26.630125045776367,
      "activations/layer19_attention_weight_max": 39.95405578613281,
      "activations/layer19_attention_weight_min": -31.763051986694336,
      "activations/layer1_attention_weight_max": 16.573448181152344,
      "activations/layer1_attention_weight_min": -15.634572982788086,
      "activations/layer20_attention_weight_max": 30.065534591674805,
      "activations/layer20_attention_weight_min": -25.461711883544922,
      "activations/layer21_attention_weight_max": 29.436155319213867,
      "activations/layer21_attention_weight_min": -23.94119644165039,
      "activations/layer22_attention_weight_max": 49.38002014160156,
      "activations/layer22_attention_weight_min": -33.36624526977539,
      "activations/layer23_attention_weight_max": 43.030418395996094,
      "activations/layer23_attention_weight_min": -26.96058464050293,
      "activations/layer2_attention_weight_max": 32.49068069458008,
      "activations/layer2_attention_weight_min": -33.19124984741211,
      "activations/layer3_attention_weight_max": 95.3713607788086,
      "activations/layer3_attention_weight_min": -94.7024917602539,
      "activations/layer4_attention_weight_max": 108.32118225097656,
      "activations/layer4_attention_weight_min": -110.28495025634766,
      "activations/layer5_attention_weight_max": 49.8984489440918,
      "activations/layer5_attention_weight_min": -61.21900177001953,
      "activations/layer6_attention_weight_max": 44.24671173095703,
      "activations/layer6_attention_weight_min": -45.13669967651367,
      "activations/layer7_attention_weight_max": 94.08045959472656,
      "activations/layer7_attention_weight_min": -93.19547271728516,
      "activations/layer8_attention_weight_max": 38.11786651611328,
      "activations/layer8_attention_weight_min": -40.01826858520508,
      "activations/layer9_attention_weight_max": 31.25632667541504,
      "activations/layer9_attention_weight_min": -34.674190521240234,
      "epoch": 12.88,
      "learning_rate": 6.762689393939394e-05,
      "loss": 2.7844,
      "step": 221650
    },
    {
      "activations/layer0_attention_weight_max": 16.24248504638672,
      "activations/layer0_attention_weight_min": -11.6361083984375,
      "activations/layer10_attention_weight_max": 34.21340560913086,
      "activations/layer10_attention_weight_min": -34.57316970825195,
      "activations/layer11_attention_weight_max": 32.06970977783203,
      "activations/layer11_attention_weight_min": -33.46470642089844,
      "activations/layer12_attention_weight_max": 25.742515563964844,
      "activations/layer12_attention_weight_min": -26.836450576782227,
      "activations/layer13_attention_weight_max": 38.87651443481445,
      "activations/layer13_attention_weight_min": -36.15985107421875,
      "activations/layer14_attention_weight_max": 41.916778564453125,
      "activations/layer14_attention_weight_min": -38.097801208496094,
      "activations/layer15_attention_weight_max": 37.53339385986328,
      "activations/layer15_attention_weight_min": -35.63441848754883,
      "activations/layer16_attention_weight_max": 30.706327438354492,
      "activations/layer16_attention_weight_min": -30.976533889770508,
      "activations/layer17_attention_weight_max": 32.47023010253906,
      "activations/layer17_attention_weight_min": -28.187591552734375,
      "activations/layer18_attention_weight_max": 28.971454620361328,
      "activations/layer18_attention_weight_min": -25.27129364013672,
      "activations/layer19_attention_weight_max": 34.16804504394531,
      "activations/layer19_attention_weight_min": -30.332883834838867,
      "activations/layer1_attention_weight_max": 16.188615798950195,
      "activations/layer1_attention_weight_min": -14.655753135681152,
      "activations/layer20_attention_weight_max": 32.40074920654297,
      "activations/layer20_attention_weight_min": -24.004606246948242,
      "activations/layer21_attention_weight_max": 31.59928321838379,
      "activations/layer21_attention_weight_min": -25.2874755859375,
      "activations/layer22_attention_weight_max": 42.97050094604492,
      "activations/layer22_attention_weight_min": -29.59305763244629,
      "activations/layer23_attention_weight_max": 34.30696487426758,
      "activations/layer23_attention_weight_min": -25.269264221191406,
      "activations/layer2_attention_weight_max": 33.168190002441406,
      "activations/layer2_attention_weight_min": -33.05973434448242,
      "activations/layer3_attention_weight_max": 95.97000885009766,
      "activations/layer3_attention_weight_min": -96.14970397949219,
      "activations/layer4_attention_weight_max": 112.35506439208984,
      "activations/layer4_attention_weight_min": -113.51497650146484,
      "activations/layer5_attention_weight_max": 49.35606002807617,
      "activations/layer5_attention_weight_min": -60.83850860595703,
      "activations/layer6_attention_weight_max": 45.42048263549805,
      "activations/layer6_attention_weight_min": -47.132606506347656,
      "activations/layer7_attention_weight_max": 96.96687316894531,
      "activations/layer7_attention_weight_min": -90.19805145263672,
      "activations/layer8_attention_weight_max": 42.18134689331055,
      "activations/layer8_attention_weight_min": -42.38156509399414,
      "activations/layer9_attention_weight_max": 34.331600189208984,
      "activations/layer9_attention_weight_min": -34.466400146484375,
      "epoch": 12.88,
      "learning_rate": 6.760795454545454e-05,
      "loss": 2.7697,
      "step": 221700
    },
    {
      "activations/layer0_attention_weight_max": 16.020912170410156,
      "activations/layer0_attention_weight_min": -11.90893268585205,
      "activations/layer10_attention_weight_max": 37.4494743347168,
      "activations/layer10_attention_weight_min": -37.869544982910156,
      "activations/layer11_attention_weight_max": 36.042335510253906,
      "activations/layer11_attention_weight_min": -35.202911376953125,
      "activations/layer12_attention_weight_max": 28.401691436767578,
      "activations/layer12_attention_weight_min": -26.825363159179688,
      "activations/layer13_attention_weight_max": 45.33509826660156,
      "activations/layer13_attention_weight_min": -37.42696762084961,
      "activations/layer14_attention_weight_max": 47.76978302001953,
      "activations/layer14_attention_weight_min": -42.68012237548828,
      "activations/layer15_attention_weight_max": 39.613346099853516,
      "activations/layer15_attention_weight_min": -33.42461013793945,
      "activations/layer16_attention_weight_max": 33.03965759277344,
      "activations/layer16_attention_weight_min": -29.53435516357422,
      "activations/layer17_attention_weight_max": 35.17131423950195,
      "activations/layer17_attention_weight_min": -26.037857055664062,
      "activations/layer18_attention_weight_max": 32.378074645996094,
      "activations/layer18_attention_weight_min": -25.52226448059082,
      "activations/layer19_attention_weight_max": 34.88032150268555,
      "activations/layer19_attention_weight_min": -31.966506958007812,
      "activations/layer1_attention_weight_max": 16.08995819091797,
      "activations/layer1_attention_weight_min": -14.867271423339844,
      "activations/layer20_attention_weight_max": 30.941993713378906,
      "activations/layer20_attention_weight_min": -27.877904891967773,
      "activations/layer21_attention_weight_max": 28.637365341186523,
      "activations/layer21_attention_weight_min": -25.213485717773438,
      "activations/layer22_attention_weight_max": 45.97751235961914,
      "activations/layer22_attention_weight_min": -31.91666603088379,
      "activations/layer23_attention_weight_max": 35.793495178222656,
      "activations/layer23_attention_weight_min": -26.360151290893555,
      "activations/layer2_attention_weight_max": 32.08534622192383,
      "activations/layer2_attention_weight_min": -33.90974426269531,
      "activations/layer3_attention_weight_max": 95.4894027709961,
      "activations/layer3_attention_weight_min": -101.27392578125,
      "activations/layer4_attention_weight_max": 106.295654296875,
      "activations/layer4_attention_weight_min": -114.35479736328125,
      "activations/layer5_attention_weight_max": 52.213134765625,
      "activations/layer5_attention_weight_min": -66.69317626953125,
      "activations/layer6_attention_weight_max": 43.38918685913086,
      "activations/layer6_attention_weight_min": -48.72245407104492,
      "activations/layer7_attention_weight_max": 93.68849182128906,
      "activations/layer7_attention_weight_min": -97.0395736694336,
      "activations/layer8_attention_weight_max": 41.498538970947266,
      "activations/layer8_attention_weight_min": -43.28813552856445,
      "activations/layer9_attention_weight_max": 33.79503631591797,
      "activations/layer9_attention_weight_min": -36.17112350463867,
      "epoch": 12.88,
      "learning_rate": 6.758901515151515e-05,
      "loss": 2.7699,
      "step": 221750
    },
    {
      "activations/layer0_attention_weight_max": 16.022811889648438,
      "activations/layer0_attention_weight_min": -11.440963745117188,
      "activations/layer10_attention_weight_max": 44.5902099609375,
      "activations/layer10_attention_weight_min": -37.97841262817383,
      "activations/layer11_attention_weight_max": 46.7038688659668,
      "activations/layer11_attention_weight_min": -40.37744140625,
      "activations/layer12_attention_weight_max": 38.70087814331055,
      "activations/layer12_attention_weight_min": -31.676511764526367,
      "activations/layer13_attention_weight_max": 60.149566650390625,
      "activations/layer13_attention_weight_min": -41.83047103881836,
      "activations/layer14_attention_weight_max": 69.22117614746094,
      "activations/layer14_attention_weight_min": -43.825382232666016,
      "activations/layer15_attention_weight_max": 56.77476501464844,
      "activations/layer15_attention_weight_min": -41.65666580200195,
      "activations/layer16_attention_weight_max": 49.79209518432617,
      "activations/layer16_attention_weight_min": -31.372957229614258,
      "activations/layer17_attention_weight_max": 49.60359573364258,
      "activations/layer17_attention_weight_min": -32.219566345214844,
      "activations/layer18_attention_weight_max": 41.571659088134766,
      "activations/layer18_attention_weight_min": -28.792762756347656,
      "activations/layer19_attention_weight_max": 50.20644760131836,
      "activations/layer19_attention_weight_min": -33.18718719482422,
      "activations/layer1_attention_weight_max": 16.09901237487793,
      "activations/layer1_attention_weight_min": -15.2797212600708,
      "activations/layer20_attention_weight_max": 39.76053237915039,
      "activations/layer20_attention_weight_min": -26.142532348632812,
      "activations/layer21_attention_weight_max": 38.42800521850586,
      "activations/layer21_attention_weight_min": -25.581787109375,
      "activations/layer22_attention_weight_max": 63.38750076293945,
      "activations/layer22_attention_weight_min": -34.24516677856445,
      "activations/layer23_attention_weight_max": 44.45635223388672,
      "activations/layer23_attention_weight_min": -27.040767669677734,
      "activations/layer2_attention_weight_max": 35.64780807495117,
      "activations/layer2_attention_weight_min": -35.275726318359375,
      "activations/layer3_attention_weight_max": 102.85579681396484,
      "activations/layer3_attention_weight_min": -103.84864807128906,
      "activations/layer4_attention_weight_max": 120.42671203613281,
      "activations/layer4_attention_weight_min": -117.96864318847656,
      "activations/layer5_attention_weight_max": 59.2254753112793,
      "activations/layer5_attention_weight_min": -62.59856414794922,
      "activations/layer6_attention_weight_max": 49.57626724243164,
      "activations/layer6_attention_weight_min": -51.30233383178711,
      "activations/layer7_attention_weight_max": 109.75708770751953,
      "activations/layer7_attention_weight_min": -99.16903686523438,
      "activations/layer8_attention_weight_max": 50.161048889160156,
      "activations/layer8_attention_weight_min": -45.47297286987305,
      "activations/layer9_attention_weight_max": 44.03322982788086,
      "activations/layer9_attention_weight_min": -40.20331954956055,
      "epoch": 12.89,
      "learning_rate": 6.757007575757576e-05,
      "loss": 2.7713,
      "step": 221800
    },
    {
      "activations/layer0_attention_weight_max": 15.334329605102539,
      "activations/layer0_attention_weight_min": -11.815868377685547,
      "activations/layer10_attention_weight_max": 37.33396911621094,
      "activations/layer10_attention_weight_min": -34.467506408691406,
      "activations/layer11_attention_weight_max": 38.7315673828125,
      "activations/layer11_attention_weight_min": -35.92041015625,
      "activations/layer12_attention_weight_max": 26.732851028442383,
      "activations/layer12_attention_weight_min": -25.39270782470703,
      "activations/layer13_attention_weight_max": 41.921730041503906,
      "activations/layer13_attention_weight_min": -35.70033264160156,
      "activations/layer14_attention_weight_max": 50.39789962768555,
      "activations/layer14_attention_weight_min": -39.87324905395508,
      "activations/layer15_attention_weight_max": 49.68507766723633,
      "activations/layer15_attention_weight_min": -34.54869842529297,
      "activations/layer16_attention_weight_max": 36.386566162109375,
      "activations/layer16_attention_weight_min": -28.6207332611084,
      "activations/layer17_attention_weight_max": 31.73802375793457,
      "activations/layer17_attention_weight_min": -25.5629825592041,
      "activations/layer18_attention_weight_max": 36.92975997924805,
      "activations/layer18_attention_weight_min": -23.192203521728516,
      "activations/layer19_attention_weight_max": 37.867515563964844,
      "activations/layer19_attention_weight_min": -31.249984741210938,
      "activations/layer1_attention_weight_max": 17.636388778686523,
      "activations/layer1_attention_weight_min": -15.892854690551758,
      "activations/layer20_attention_weight_max": 31.510129928588867,
      "activations/layer20_attention_weight_min": -25.13265037536621,
      "activations/layer21_attention_weight_max": 33.44868850708008,
      "activations/layer21_attention_weight_min": -25.40349578857422,
      "activations/layer22_attention_weight_max": 45.314781188964844,
      "activations/layer22_attention_weight_min": -27.86004638671875,
      "activations/layer23_attention_weight_max": 37.48261260986328,
      "activations/layer23_attention_weight_min": -25.09585952758789,
      "activations/layer2_attention_weight_max": 34.495811462402344,
      "activations/layer2_attention_weight_min": -33.945594787597656,
      "activations/layer3_attention_weight_max": 97.05438995361328,
      "activations/layer3_attention_weight_min": -98.01181030273438,
      "activations/layer4_attention_weight_max": 107.10176086425781,
      "activations/layer4_attention_weight_min": -106.82637786865234,
      "activations/layer5_attention_weight_max": 53.47901153564453,
      "activations/layer5_attention_weight_min": -59.30382537841797,
      "activations/layer6_attention_weight_max": 43.07454299926758,
      "activations/layer6_attention_weight_min": -46.941978454589844,
      "activations/layer7_attention_weight_max": 92.33356475830078,
      "activations/layer7_attention_weight_min": -89.13365936279297,
      "activations/layer8_attention_weight_max": 40.544857025146484,
      "activations/layer8_attention_weight_min": -40.044612884521484,
      "activations/layer9_attention_weight_max": 33.54752731323242,
      "activations/layer9_attention_weight_min": -32.35546875,
      "epoch": 12.89,
      "learning_rate": 6.755113636363636e-05,
      "loss": 2.7776,
      "step": 221850
    },
    {
      "activations/layer0_attention_weight_max": 17.740663528442383,
      "activations/layer0_attention_weight_min": -11.806695938110352,
      "activations/layer10_attention_weight_max": 33.371883392333984,
      "activations/layer10_attention_weight_min": -33.94570541381836,
      "activations/layer11_attention_weight_max": 33.49055480957031,
      "activations/layer11_attention_weight_min": -33.964202880859375,
      "activations/layer12_attention_weight_max": 27.82358741760254,
      "activations/layer12_attention_weight_min": -25.842018127441406,
      "activations/layer13_attention_weight_max": 43.38444137573242,
      "activations/layer13_attention_weight_min": -38.13386154174805,
      "activations/layer14_attention_weight_max": 49.9724235534668,
      "activations/layer14_attention_weight_min": -37.59358596801758,
      "activations/layer15_attention_weight_max": 41.400917053222656,
      "activations/layer15_attention_weight_min": -35.23147201538086,
      "activations/layer16_attention_weight_max": 34.499244689941406,
      "activations/layer16_attention_weight_min": -30.850940704345703,
      "activations/layer17_attention_weight_max": 36.34558868408203,
      "activations/layer17_attention_weight_min": -28.09012794494629,
      "activations/layer18_attention_weight_max": 35.929718017578125,
      "activations/layer18_attention_weight_min": -24.977699279785156,
      "activations/layer19_attention_weight_max": 39.416786193847656,
      "activations/layer19_attention_weight_min": -36.36928939819336,
      "activations/layer1_attention_weight_max": 15.616336822509766,
      "activations/layer1_attention_weight_min": -15.272000312805176,
      "activations/layer20_attention_weight_max": 34.52000427246094,
      "activations/layer20_attention_weight_min": -27.989871978759766,
      "activations/layer21_attention_weight_max": 32.5630989074707,
      "activations/layer21_attention_weight_min": -27.07564353942871,
      "activations/layer22_attention_weight_max": 50.23868179321289,
      "activations/layer22_attention_weight_min": -31.104413986206055,
      "activations/layer23_attention_weight_max": 39.06785202026367,
      "activations/layer23_attention_weight_min": -24.14423370361328,
      "activations/layer2_attention_weight_max": 35.160945892333984,
      "activations/layer2_attention_weight_min": -34.112571716308594,
      "activations/layer3_attention_weight_max": 101.44309997558594,
      "activations/layer3_attention_weight_min": -103.77015686035156,
      "activations/layer4_attention_weight_max": 111.38079071044922,
      "activations/layer4_attention_weight_min": -111.26643371582031,
      "activations/layer5_attention_weight_max": 51.479000091552734,
      "activations/layer5_attention_weight_min": -64.25065612792969,
      "activations/layer6_attention_weight_max": 42.757564544677734,
      "activations/layer6_attention_weight_min": -48.69120788574219,
      "activations/layer7_attention_weight_max": 93.31227111816406,
      "activations/layer7_attention_weight_min": -101.55885314941406,
      "activations/layer8_attention_weight_max": 40.92488479614258,
      "activations/layer8_attention_weight_min": -47.43995666503906,
      "activations/layer9_attention_weight_max": 34.43359375,
      "activations/layer9_attention_weight_min": -35.53010177612305,
      "epoch": 12.89,
      "learning_rate": 6.753219696969697e-05,
      "loss": 2.7709,
      "step": 221900
    },
    {
      "activations/layer0_attention_weight_max": 16.572481155395508,
      "activations/layer0_attention_weight_min": -12.45850658416748,
      "activations/layer10_attention_weight_max": 33.60033416748047,
      "activations/layer10_attention_weight_min": -30.984262466430664,
      "activations/layer11_attention_weight_max": 32.51624298095703,
      "activations/layer11_attention_weight_min": -31.05996322631836,
      "activations/layer12_attention_weight_max": 23.33104705810547,
      "activations/layer12_attention_weight_min": -26.290037155151367,
      "activations/layer13_attention_weight_max": 35.14910125732422,
      "activations/layer13_attention_weight_min": -35.65076446533203,
      "activations/layer14_attention_weight_max": 39.947269439697266,
      "activations/layer14_attention_weight_min": -36.53040313720703,
      "activations/layer15_attention_weight_max": 33.779151916503906,
      "activations/layer15_attention_weight_min": -32.310054779052734,
      "activations/layer16_attention_weight_max": 27.354692459106445,
      "activations/layer16_attention_weight_min": -27.314409255981445,
      "activations/layer17_attention_weight_max": 27.041229248046875,
      "activations/layer17_attention_weight_min": -25.16342544555664,
      "activations/layer18_attention_weight_max": 27.145845413208008,
      "activations/layer18_attention_weight_min": -23.964061737060547,
      "activations/layer19_attention_weight_max": 30.873214721679688,
      "activations/layer19_attention_weight_min": -30.40691566467285,
      "activations/layer1_attention_weight_max": 16.250547409057617,
      "activations/layer1_attention_weight_min": -14.147796630859375,
      "activations/layer20_attention_weight_max": 27.6533145904541,
      "activations/layer20_attention_weight_min": -24.344430923461914,
      "activations/layer21_attention_weight_max": 28.10540008544922,
      "activations/layer21_attention_weight_min": -23.729415893554688,
      "activations/layer22_attention_weight_max": 43.14816665649414,
      "activations/layer22_attention_weight_min": -30.12667465209961,
      "activations/layer23_attention_weight_max": 34.75324249267578,
      "activations/layer23_attention_weight_min": -26.136329650878906,
      "activations/layer2_attention_weight_max": 33.012516021728516,
      "activations/layer2_attention_weight_min": -33.53336715698242,
      "activations/layer3_attention_weight_max": 96.90959167480469,
      "activations/layer3_attention_weight_min": -97.63597106933594,
      "activations/layer4_attention_weight_max": 107.52587890625,
      "activations/layer4_attention_weight_min": -109.73296356201172,
      "activations/layer5_attention_weight_max": 54.883174896240234,
      "activations/layer5_attention_weight_min": -62.77168273925781,
      "activations/layer6_attention_weight_max": 42.87213897705078,
      "activations/layer6_attention_weight_min": -45.84805679321289,
      "activations/layer7_attention_weight_max": 87.80236053466797,
      "activations/layer7_attention_weight_min": -95.55465698242188,
      "activations/layer8_attention_weight_max": 40.3344612121582,
      "activations/layer8_attention_weight_min": -40.54137420654297,
      "activations/layer9_attention_weight_max": 36.380828857421875,
      "activations/layer9_attention_weight_min": -33.44965362548828,
      "epoch": 12.9,
      "learning_rate": 6.751325757575757e-05,
      "loss": 2.7644,
      "step": 221950
    },
    {
      "activations/layer0_attention_weight_max": 15.985710144042969,
      "activations/layer0_attention_weight_min": -12.359962463378906,
      "activations/layer10_attention_weight_max": 33.58639907836914,
      "activations/layer10_attention_weight_min": -33.645633697509766,
      "activations/layer11_attention_weight_max": 34.902671813964844,
      "activations/layer11_attention_weight_min": -33.858245849609375,
      "activations/layer12_attention_weight_max": 32.68657684326172,
      "activations/layer12_attention_weight_min": -25.00695037841797,
      "activations/layer13_attention_weight_max": 38.45991134643555,
      "activations/layer13_attention_weight_min": -33.753692626953125,
      "activations/layer14_attention_weight_max": 44.1253547668457,
      "activations/layer14_attention_weight_min": -37.90562438964844,
      "activations/layer15_attention_weight_max": 36.74168395996094,
      "activations/layer15_attention_weight_min": -34.122474670410156,
      "activations/layer16_attention_weight_max": 30.74009895324707,
      "activations/layer16_attention_weight_min": -29.367328643798828,
      "activations/layer17_attention_weight_max": 38.58605194091797,
      "activations/layer17_attention_weight_min": -26.360286712646484,
      "activations/layer18_attention_weight_max": 34.112186431884766,
      "activations/layer18_attention_weight_min": -25.571603775024414,
      "activations/layer19_attention_weight_max": 37.69663619995117,
      "activations/layer19_attention_weight_min": -33.042484283447266,
      "activations/layer1_attention_weight_max": 17.814464569091797,
      "activations/layer1_attention_weight_min": -16.866024017333984,
      "activations/layer20_attention_weight_max": 31.348142623901367,
      "activations/layer20_attention_weight_min": -25.853281021118164,
      "activations/layer21_attention_weight_max": 32.73189163208008,
      "activations/layer21_attention_weight_min": -24.660587310791016,
      "activations/layer22_attention_weight_max": 43.865577697753906,
      "activations/layer22_attention_weight_min": -31.074861526489258,
      "activations/layer23_attention_weight_max": 33.20559310913086,
      "activations/layer23_attention_weight_min": -25.002349853515625,
      "activations/layer2_attention_weight_max": 32.96434783935547,
      "activations/layer2_attention_weight_min": -34.74110412597656,
      "activations/layer3_attention_weight_max": 100.93915557861328,
      "activations/layer3_attention_weight_min": -102.4767837524414,
      "activations/layer4_attention_weight_max": 108.64192962646484,
      "activations/layer4_attention_weight_min": -112.055908203125,
      "activations/layer5_attention_weight_max": 50.57575607299805,
      "activations/layer5_attention_weight_min": -63.937068939208984,
      "activations/layer6_attention_weight_max": 44.73771286010742,
      "activations/layer6_attention_weight_min": -47.75133514404297,
      "activations/layer7_attention_weight_max": 90.88233947753906,
      "activations/layer7_attention_weight_min": -91.416259765625,
      "activations/layer8_attention_weight_max": 39.163726806640625,
      "activations/layer8_attention_weight_min": -41.94594192504883,
      "activations/layer9_attention_weight_max": 33.35105514526367,
      "activations/layer9_attention_weight_min": -34.404685974121094,
      "epoch": 12.9,
      "learning_rate": 6.749431818181817e-05,
      "loss": 2.7687,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4524,
      "eval_samples_per_second": 508.024,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4524,
      "eval_openwebtext_samples_per_second": 508.024,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9266,
      "eval_wikitext_samples_per_second": 236.683,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_lambada_loss": 2.546875,
      "eval_lambada_ppl": 12.76714405293921,
      "eval_lambada_runtime": 9.5202,
      "eval_lambada_samples_per_second": 511.438,
      "step": 222000
    },
    {
      "activations/layer0_attention_weight_max": 16.89640235900879,
      "activations/layer0_attention_weight_min": -11.368375778198242,
      "activations/layer10_attention_weight_max": 29.050260543823242,
      "activations/layer10_attention_weight_min": -31.363094329833984,
      "activations/layer11_attention_weight_max": 29.942066192626953,
      "activations/layer11_attention_weight_min": -31.726783752441406,
      "activations/layer12_attention_weight_max": 23.96636199951172,
      "activations/layer12_attention_weight_min": -24.4143009185791,
      "activations/layer13_attention_weight_max": 34.36518478393555,
      "activations/layer13_attention_weight_min": -32.809871673583984,
      "activations/layer14_attention_weight_max": 40.01030349731445,
      "activations/layer14_attention_weight_min": -37.212764739990234,
      "activations/layer15_attention_weight_max": 33.740570068359375,
      "activations/layer15_attention_weight_min": -32.63169479370117,
      "activations/layer16_attention_weight_max": 30.594636917114258,
      "activations/layer16_attention_weight_min": -26.328353881835938,
      "activations/layer17_attention_weight_max": 28.11895179748535,
      "activations/layer17_attention_weight_min": -24.46588706970215,
      "activations/layer18_attention_weight_max": 28.105947494506836,
      "activations/layer18_attention_weight_min": -24.704994201660156,
      "activations/layer19_attention_weight_max": 30.496444702148438,
      "activations/layer19_attention_weight_min": -29.630828857421875,
      "activations/layer1_attention_weight_max": 15.66555118560791,
      "activations/layer1_attention_weight_min": -17.442981719970703,
      "activations/layer20_attention_weight_max": 26.967649459838867,
      "activations/layer20_attention_weight_min": -24.73229217529297,
      "activations/layer21_attention_weight_max": 27.316312789916992,
      "activations/layer21_attention_weight_min": -23.750797271728516,
      "activations/layer22_attention_weight_max": 41.819637298583984,
      "activations/layer22_attention_weight_min": -31.10409164428711,
      "activations/layer23_attention_weight_max": 29.675107955932617,
      "activations/layer23_attention_weight_min": -26.379711151123047,
      "activations/layer2_attention_weight_max": 33.028106689453125,
      "activations/layer2_attention_weight_min": -32.916770935058594,
      "activations/layer3_attention_weight_max": 98.83602905273438,
      "activations/layer3_attention_weight_min": -99.06233215332031,
      "activations/layer4_attention_weight_max": 110.81424713134766,
      "activations/layer4_attention_weight_min": -105.26949310302734,
      "activations/layer5_attention_weight_max": 50.34458541870117,
      "activations/layer5_attention_weight_min": -62.91714096069336,
      "activations/layer6_attention_weight_max": 43.571353912353516,
      "activations/layer6_attention_weight_min": -43.516517639160156,
      "activations/layer7_attention_weight_max": 89.34349060058594,
      "activations/layer7_attention_weight_min": -85.10746765136719,
      "activations/layer8_attention_weight_max": 38.299034118652344,
      "activations/layer8_attention_weight_min": -38.22676467895508,
      "activations/layer9_attention_weight_max": 28.5178165435791,
      "activations/layer9_attention_weight_min": -31.25181007385254,
      "epoch": 12.9,
      "learning_rate": 6.747537878787878e-05,
      "loss": 2.7466,
      "step": 222050
    },
    {
      "activations/layer0_attention_weight_max": 16.838037490844727,
      "activations/layer0_attention_weight_min": -12.31987476348877,
      "activations/layer10_attention_weight_max": 33.308799743652344,
      "activations/layer10_attention_weight_min": -33.84622573852539,
      "activations/layer11_attention_weight_max": 34.23945999145508,
      "activations/layer11_attention_weight_min": -34.55868148803711,
      "activations/layer12_attention_weight_max": 27.646947860717773,
      "activations/layer12_attention_weight_min": -26.319690704345703,
      "activations/layer13_attention_weight_max": 42.73546600341797,
      "activations/layer13_attention_weight_min": -37.30900192260742,
      "activations/layer14_attention_weight_max": 47.72422409057617,
      "activations/layer14_attention_weight_min": -40.64640426635742,
      "activations/layer15_attention_weight_max": 44.254058837890625,
      "activations/layer15_attention_weight_min": -33.8712043762207,
      "activations/layer16_attention_weight_max": 35.4433479309082,
      "activations/layer16_attention_weight_min": -29.00873565673828,
      "activations/layer17_attention_weight_max": 33.53911209106445,
      "activations/layer17_attention_weight_min": -27.32056999206543,
      "activations/layer18_attention_weight_max": 34.22576904296875,
      "activations/layer18_attention_weight_min": -23.807275772094727,
      "activations/layer19_attention_weight_max": 36.92998123168945,
      "activations/layer19_attention_weight_min": -31.418867111206055,
      "activations/layer1_attention_weight_max": 16.52952003479004,
      "activations/layer1_attention_weight_min": -16.49313735961914,
      "activations/layer20_attention_weight_max": 34.36792755126953,
      "activations/layer20_attention_weight_min": -25.701353073120117,
      "activations/layer21_attention_weight_max": 31.80385398864746,
      "activations/layer21_attention_weight_min": -24.054189682006836,
      "activations/layer22_attention_weight_max": 53.923213958740234,
      "activations/layer22_attention_weight_min": -32.650535583496094,
      "activations/layer23_attention_weight_max": 39.26538848876953,
      "activations/layer23_attention_weight_min": -27.91598892211914,
      "activations/layer2_attention_weight_max": 34.07996368408203,
      "activations/layer2_attention_weight_min": -34.63930892944336,
      "activations/layer3_attention_weight_max": 99.5755615234375,
      "activations/layer3_attention_weight_min": -101.83708190917969,
      "activations/layer4_attention_weight_max": 116.5723876953125,
      "activations/layer4_attention_weight_min": -112.36882781982422,
      "activations/layer5_attention_weight_max": 54.98495101928711,
      "activations/layer5_attention_weight_min": -66.09800720214844,
      "activations/layer6_attention_weight_max": 44.78561782836914,
      "activations/layer6_attention_weight_min": -49.92197036743164,
      "activations/layer7_attention_weight_max": 91.69249725341797,
      "activations/layer7_attention_weight_min": -97.412109375,
      "activations/layer8_attention_weight_max": 41.16942596435547,
      "activations/layer8_attention_weight_min": -44.0407600402832,
      "activations/layer9_attention_weight_max": 33.1766242980957,
      "activations/layer9_attention_weight_min": -34.92707443237305,
      "epoch": 12.91,
      "learning_rate": 6.745643939393939e-05,
      "loss": 2.7616,
      "step": 222100
    },
    {
      "activations/layer0_attention_weight_max": 16.128185272216797,
      "activations/layer0_attention_weight_min": -11.559599876403809,
      "activations/layer10_attention_weight_max": 37.45228958129883,
      "activations/layer10_attention_weight_min": -38.218997955322266,
      "activations/layer11_attention_weight_max": 36.121910095214844,
      "activations/layer11_attention_weight_min": -35.81558609008789,
      "activations/layer12_attention_weight_max": 26.716232299804688,
      "activations/layer12_attention_weight_min": -25.72563934326172,
      "activations/layer13_attention_weight_max": 40.85847091674805,
      "activations/layer13_attention_weight_min": -35.924049377441406,
      "activations/layer14_attention_weight_max": 40.48712921142578,
      "activations/layer14_attention_weight_min": -38.1176643371582,
      "activations/layer15_attention_weight_max": 38.465606689453125,
      "activations/layer15_attention_weight_min": -33.30939865112305,
      "activations/layer16_attention_weight_max": 31.002058029174805,
      "activations/layer16_attention_weight_min": -27.67714500427246,
      "activations/layer17_attention_weight_max": 31.890954971313477,
      "activations/layer17_attention_weight_min": -25.725305557250977,
      "activations/layer18_attention_weight_max": 30.312583923339844,
      "activations/layer18_attention_weight_min": -24.81688117980957,
      "activations/layer19_attention_weight_max": 34.10842514038086,
      "activations/layer19_attention_weight_min": -30.200958251953125,
      "activations/layer1_attention_weight_max": 17.233598709106445,
      "activations/layer1_attention_weight_min": -15.111953735351562,
      "activations/layer20_attention_weight_max": 29.8966064453125,
      "activations/layer20_attention_weight_min": -24.616838455200195,
      "activations/layer21_attention_weight_max": 29.7520751953125,
      "activations/layer21_attention_weight_min": -23.648948669433594,
      "activations/layer22_attention_weight_max": 44.348411560058594,
      "activations/layer22_attention_weight_min": -31.671201705932617,
      "activations/layer23_attention_weight_max": 32.55828094482422,
      "activations/layer23_attention_weight_min": -23.98624038696289,
      "activations/layer2_attention_weight_max": 34.62971496582031,
      "activations/layer2_attention_weight_min": -33.364219665527344,
      "activations/layer3_attention_weight_max": 105.92440032958984,
      "activations/layer3_attention_weight_min": -106.72975158691406,
      "activations/layer4_attention_weight_max": 117.74507141113281,
      "activations/layer4_attention_weight_min": -116.53160858154297,
      "activations/layer5_attention_weight_max": 56.20701599121094,
      "activations/layer5_attention_weight_min": -61.5258903503418,
      "activations/layer6_attention_weight_max": 48.278133392333984,
      "activations/layer6_attention_weight_min": -49.72919845581055,
      "activations/layer7_attention_weight_max": 101.1902847290039,
      "activations/layer7_attention_weight_min": -105.85138702392578,
      "activations/layer8_attention_weight_max": 48.4608039855957,
      "activations/layer8_attention_weight_min": -50.11807632446289,
      "activations/layer9_attention_weight_max": 37.69841384887695,
      "activations/layer9_attention_weight_min": -36.955772399902344,
      "epoch": 12.91,
      "learning_rate": 6.743749999999999e-05,
      "loss": 2.7666,
      "step": 222150
    },
    {
      "activations/layer0_attention_weight_max": 14.973407745361328,
      "activations/layer0_attention_weight_min": -11.58279037475586,
      "activations/layer10_attention_weight_max": 30.567901611328125,
      "activations/layer10_attention_weight_min": -34.16276931762695,
      "activations/layer11_attention_weight_max": 31.81000328063965,
      "activations/layer11_attention_weight_min": -34.03876876831055,
      "activations/layer12_attention_weight_max": 29.601537704467773,
      "activations/layer12_attention_weight_min": -26.341819763183594,
      "activations/layer13_attention_weight_max": 41.201698303222656,
      "activations/layer13_attention_weight_min": -34.07757568359375,
      "activations/layer14_attention_weight_max": 42.10621643066406,
      "activations/layer14_attention_weight_min": -37.235267639160156,
      "activations/layer15_attention_weight_max": 37.08226776123047,
      "activations/layer15_attention_weight_min": -33.10905075073242,
      "activations/layer16_attention_weight_max": 33.20418930053711,
      "activations/layer16_attention_weight_min": -27.581520080566406,
      "activations/layer17_attention_weight_max": 30.29954719543457,
      "activations/layer17_attention_weight_min": -25.51030158996582,
      "activations/layer18_attention_weight_max": 31.585674285888672,
      "activations/layer18_attention_weight_min": -23.874391555786133,
      "activations/layer19_attention_weight_max": 35.608001708984375,
      "activations/layer19_attention_weight_min": -31.69180679321289,
      "activations/layer1_attention_weight_max": 16.271934509277344,
      "activations/layer1_attention_weight_min": -14.553275108337402,
      "activations/layer20_attention_weight_max": 31.268043518066406,
      "activations/layer20_attention_weight_min": -24.9509334564209,
      "activations/layer21_attention_weight_max": 27.7605037689209,
      "activations/layer21_attention_weight_min": -23.229717254638672,
      "activations/layer22_attention_weight_max": 43.388816833496094,
      "activations/layer22_attention_weight_min": -30.972829818725586,
      "activations/layer23_attention_weight_max": 30.656707763671875,
      "activations/layer23_attention_weight_min": -25.441328048706055,
      "activations/layer2_attention_weight_max": 32.85565185546875,
      "activations/layer2_attention_weight_min": -33.31731414794922,
      "activations/layer3_attention_weight_max": 96.94529724121094,
      "activations/layer3_attention_weight_min": -101.76294708251953,
      "activations/layer4_attention_weight_max": 108.51171112060547,
      "activations/layer4_attention_weight_min": -108.07305908203125,
      "activations/layer5_attention_weight_max": 51.80377960205078,
      "activations/layer5_attention_weight_min": -60.48286056518555,
      "activations/layer6_attention_weight_max": 41.62223434448242,
      "activations/layer6_attention_weight_min": -46.57864761352539,
      "activations/layer7_attention_weight_max": 88.44778442382812,
      "activations/layer7_attention_weight_min": -90.31005096435547,
      "activations/layer8_attention_weight_max": 38.398704528808594,
      "activations/layer8_attention_weight_min": -41.82834243774414,
      "activations/layer9_attention_weight_max": 30.617528915405273,
      "activations/layer9_attention_weight_min": -34.11629867553711,
      "epoch": 12.91,
      "learning_rate": 6.74185606060606e-05,
      "loss": 2.7601,
      "step": 222200
    },
    {
      "activations/layer0_attention_weight_max": 15.90241527557373,
      "activations/layer0_attention_weight_min": -12.112714767456055,
      "activations/layer10_attention_weight_max": 31.281246185302734,
      "activations/layer10_attention_weight_min": -34.31760787963867,
      "activations/layer11_attention_weight_max": 32.208919525146484,
      "activations/layer11_attention_weight_min": -31.18246841430664,
      "activations/layer12_attention_weight_max": 23.527257919311523,
      "activations/layer12_attention_weight_min": -25.53978157043457,
      "activations/layer13_attention_weight_max": 36.593711853027344,
      "activations/layer13_attention_weight_min": -33.01720428466797,
      "activations/layer14_attention_weight_max": 41.81201171875,
      "activations/layer14_attention_weight_min": -35.41288757324219,
      "activations/layer15_attention_weight_max": 35.083404541015625,
      "activations/layer15_attention_weight_min": -31.06278419494629,
      "activations/layer16_attention_weight_max": 31.146623611450195,
      "activations/layer16_attention_weight_min": -26.614276885986328,
      "activations/layer17_attention_weight_max": 31.31932258605957,
      "activations/layer17_attention_weight_min": -25.562786102294922,
      "activations/layer18_attention_weight_max": 30.067222595214844,
      "activations/layer18_attention_weight_min": -23.819068908691406,
      "activations/layer19_attention_weight_max": 32.16755294799805,
      "activations/layer19_attention_weight_min": -29.128631591796875,
      "activations/layer1_attention_weight_max": 15.781444549560547,
      "activations/layer1_attention_weight_min": -13.063196182250977,
      "activations/layer20_attention_weight_max": 27.500560760498047,
      "activations/layer20_attention_weight_min": -26.9421329498291,
      "activations/layer21_attention_weight_max": 29.856353759765625,
      "activations/layer21_attention_weight_min": -25.093090057373047,
      "activations/layer22_attention_weight_max": 42.930965423583984,
      "activations/layer22_attention_weight_min": -29.084972381591797,
      "activations/layer23_attention_weight_max": 33.017250061035156,
      "activations/layer23_attention_weight_min": -28.9962158203125,
      "activations/layer2_attention_weight_max": 30.835124969482422,
      "activations/layer2_attention_weight_min": -31.02132225036621,
      "activations/layer3_attention_weight_max": 92.4826889038086,
      "activations/layer3_attention_weight_min": -96.2417984008789,
      "activations/layer4_attention_weight_max": 108.28768157958984,
      "activations/layer4_attention_weight_min": -108.69148254394531,
      "activations/layer5_attention_weight_max": 57.154624938964844,
      "activations/layer5_attention_weight_min": -62.83168029785156,
      "activations/layer6_attention_weight_max": 43.517616271972656,
      "activations/layer6_attention_weight_min": -46.79037857055664,
      "activations/layer7_attention_weight_max": 87.53836059570312,
      "activations/layer7_attention_weight_min": -90.79667663574219,
      "activations/layer8_attention_weight_max": 39.28632354736328,
      "activations/layer8_attention_weight_min": -40.54045486450195,
      "activations/layer9_attention_weight_max": 31.49825096130371,
      "activations/layer9_attention_weight_min": -32.9182243347168,
      "epoch": 12.91,
      "learning_rate": 6.739962121212121e-05,
      "loss": 2.7779,
      "step": 222250
    },
    {
      "activations/layer0_attention_weight_max": 16.153213500976562,
      "activations/layer0_attention_weight_min": -11.9729642868042,
      "activations/layer10_attention_weight_max": 35.25069808959961,
      "activations/layer10_attention_weight_min": -32.25468063354492,
      "activations/layer11_attention_weight_max": 37.51033020019531,
      "activations/layer11_attention_weight_min": -34.10997009277344,
      "activations/layer12_attention_weight_max": 31.657150268554688,
      "activations/layer12_attention_weight_min": -25.340572357177734,
      "activations/layer13_attention_weight_max": 52.74620056152344,
      "activations/layer13_attention_weight_min": -33.308189392089844,
      "activations/layer14_attention_weight_max": 49.228057861328125,
      "activations/layer14_attention_weight_min": -38.30042266845703,
      "activations/layer15_attention_weight_max": 41.028106689453125,
      "activations/layer15_attention_weight_min": -32.188533782958984,
      "activations/layer16_attention_weight_max": 31.641393661499023,
      "activations/layer16_attention_weight_min": -27.568450927734375,
      "activations/layer17_attention_weight_max": 30.14678955078125,
      "activations/layer17_attention_weight_min": -25.933813095092773,
      "activations/layer18_attention_weight_max": 31.35362434387207,
      "activations/layer18_attention_weight_min": -25.05880355834961,
      "activations/layer19_attention_weight_max": 34.091468811035156,
      "activations/layer19_attention_weight_min": -32.770931243896484,
      "activations/layer1_attention_weight_max": 16.46836280822754,
      "activations/layer1_attention_weight_min": -15.082088470458984,
      "activations/layer20_attention_weight_max": 29.395572662353516,
      "activations/layer20_attention_weight_min": -26.105762481689453,
      "activations/layer21_attention_weight_max": 26.309104919433594,
      "activations/layer21_attention_weight_min": -25.102628707885742,
      "activations/layer22_attention_weight_max": 47.35784912109375,
      "activations/layer22_attention_weight_min": -32.40159606933594,
      "activations/layer23_attention_weight_max": 36.55919647216797,
      "activations/layer23_attention_weight_min": -27.49571990966797,
      "activations/layer2_attention_weight_max": 31.482221603393555,
      "activations/layer2_attention_weight_min": -32.49651336669922,
      "activations/layer3_attention_weight_max": 92.0365982055664,
      "activations/layer3_attention_weight_min": -96.06705474853516,
      "activations/layer4_attention_weight_max": 104.95891571044922,
      "activations/layer4_attention_weight_min": -102.55110168457031,
      "activations/layer5_attention_weight_max": 49.011993408203125,
      "activations/layer5_attention_weight_min": -58.72539520263672,
      "activations/layer6_attention_weight_max": 42.680809020996094,
      "activations/layer6_attention_weight_min": -44.77576446533203,
      "activations/layer7_attention_weight_max": 90.35102081298828,
      "activations/layer7_attention_weight_min": -89.25199890136719,
      "activations/layer8_attention_weight_max": 41.93963623046875,
      "activations/layer8_attention_weight_min": -45.11250305175781,
      "activations/layer9_attention_weight_max": 37.127742767333984,
      "activations/layer9_attention_weight_min": -33.68190002441406,
      "epoch": 12.92,
      "learning_rate": 6.738068181818181e-05,
      "loss": 2.7728,
      "step": 222300
    },
    {
      "activations/layer0_attention_weight_max": 15.494370460510254,
      "activations/layer0_attention_weight_min": -11.391541481018066,
      "activations/layer10_attention_weight_max": 33.366920471191406,
      "activations/layer10_attention_weight_min": -35.8836555480957,
      "activations/layer11_attention_weight_max": 33.14910888671875,
      "activations/layer11_attention_weight_min": -37.23851013183594,
      "activations/layer12_attention_weight_max": 24.534955978393555,
      "activations/layer12_attention_weight_min": -25.540958404541016,
      "activations/layer13_attention_weight_max": 42.93278121948242,
      "activations/layer13_attention_weight_min": -36.18975830078125,
      "activations/layer14_attention_weight_max": 44.158023834228516,
      "activations/layer14_attention_weight_min": -40.286922454833984,
      "activations/layer15_attention_weight_max": 37.18914794921875,
      "activations/layer15_attention_weight_min": -35.082027435302734,
      "activations/layer16_attention_weight_max": 31.510168075561523,
      "activations/layer16_attention_weight_min": -28.54922103881836,
      "activations/layer17_attention_weight_max": 29.17642593383789,
      "activations/layer17_attention_weight_min": -27.109149932861328,
      "activations/layer18_attention_weight_max": 30.9970703125,
      "activations/layer18_attention_weight_min": -25.80501365661621,
      "activations/layer19_attention_weight_max": 32.98348617553711,
      "activations/layer19_attention_weight_min": -30.9855899810791,
      "activations/layer1_attention_weight_max": 16.505470275878906,
      "activations/layer1_attention_weight_min": -14.030862808227539,
      "activations/layer20_attention_weight_max": 28.49492645263672,
      "activations/layer20_attention_weight_min": -25.038785934448242,
      "activations/layer21_attention_weight_max": 29.778087615966797,
      "activations/layer21_attention_weight_min": -24.33904266357422,
      "activations/layer22_attention_weight_max": 43.87137222290039,
      "activations/layer22_attention_weight_min": -32.79763412475586,
      "activations/layer23_attention_weight_max": 37.984100341796875,
      "activations/layer23_attention_weight_min": -27.03407096862793,
      "activations/layer2_attention_weight_max": 31.838939666748047,
      "activations/layer2_attention_weight_min": -32.257938385009766,
      "activations/layer3_attention_weight_max": 94.67058563232422,
      "activations/layer3_attention_weight_min": -96.12644958496094,
      "activations/layer4_attention_weight_max": 113.33717346191406,
      "activations/layer4_attention_weight_min": -111.2115707397461,
      "activations/layer5_attention_weight_max": 48.757545471191406,
      "activations/layer5_attention_weight_min": -61.81156539916992,
      "activations/layer6_attention_weight_max": 44.32975387573242,
      "activations/layer6_attention_weight_min": -49.08955001831055,
      "activations/layer7_attention_weight_max": 89.48114013671875,
      "activations/layer7_attention_weight_min": -93.37323760986328,
      "activations/layer8_attention_weight_max": 40.1663818359375,
      "activations/layer8_attention_weight_min": -42.39045333862305,
      "activations/layer9_attention_weight_max": 32.55317306518555,
      "activations/layer9_attention_weight_min": -34.97357940673828,
      "epoch": 12.92,
      "learning_rate": 6.736174242424241e-05,
      "loss": 2.7689,
      "step": 222350
    },
    {
      "activations/layer0_attention_weight_max": 15.371162414550781,
      "activations/layer0_attention_weight_min": -12.62514877319336,
      "activations/layer10_attention_weight_max": 31.942264556884766,
      "activations/layer10_attention_weight_min": -33.19727325439453,
      "activations/layer11_attention_weight_max": 33.488059997558594,
      "activations/layer11_attention_weight_min": -31.338729858398438,
      "activations/layer12_attention_weight_max": 25.708234786987305,
      "activations/layer12_attention_weight_min": -25.40652847290039,
      "activations/layer13_attention_weight_max": 41.58525466918945,
      "activations/layer13_attention_weight_min": -34.89128494262695,
      "activations/layer14_attention_weight_max": 41.38970184326172,
      "activations/layer14_attention_weight_min": -36.49458694458008,
      "activations/layer15_attention_weight_max": 38.697261810302734,
      "activations/layer15_attention_weight_min": -33.96147155761719,
      "activations/layer16_attention_weight_max": 31.51723289489746,
      "activations/layer16_attention_weight_min": -26.352733612060547,
      "activations/layer17_attention_weight_max": 31.94440460205078,
      "activations/layer17_attention_weight_min": -25.414066314697266,
      "activations/layer18_attention_weight_max": 30.81712532043457,
      "activations/layer18_attention_weight_min": -23.702409744262695,
      "activations/layer19_attention_weight_max": 34.920249938964844,
      "activations/layer19_attention_weight_min": -31.38993263244629,
      "activations/layer1_attention_weight_max": 16.388362884521484,
      "activations/layer1_attention_weight_min": -15.122281074523926,
      "activations/layer20_attention_weight_max": 32.65773391723633,
      "activations/layer20_attention_weight_min": -24.107667922973633,
      "activations/layer21_attention_weight_max": 31.632198333740234,
      "activations/layer21_attention_weight_min": -23.638214111328125,
      "activations/layer22_attention_weight_max": 53.88214111328125,
      "activations/layer22_attention_weight_min": -30.070005416870117,
      "activations/layer23_attention_weight_max": 36.42088317871094,
      "activations/layer23_attention_weight_min": -24.932188034057617,
      "activations/layer2_attention_weight_max": 33.41615676879883,
      "activations/layer2_attention_weight_min": -33.24522399902344,
      "activations/layer3_attention_weight_max": 98.75074768066406,
      "activations/layer3_attention_weight_min": -100.8191146850586,
      "activations/layer4_attention_weight_max": 104.2201919555664,
      "activations/layer4_attention_weight_min": -110.76690673828125,
      "activations/layer5_attention_weight_max": 53.35442352294922,
      "activations/layer5_attention_weight_min": -62.755271911621094,
      "activations/layer6_attention_weight_max": 45.202030181884766,
      "activations/layer6_attention_weight_min": -47.184627532958984,
      "activations/layer7_attention_weight_max": 85.93123626708984,
      "activations/layer7_attention_weight_min": -94.01986694335938,
      "activations/layer8_attention_weight_max": 39.5590934753418,
      "activations/layer8_attention_weight_min": -42.02838134765625,
      "activations/layer9_attention_weight_max": 32.82686996459961,
      "activations/layer9_attention_weight_min": -34.001747131347656,
      "epoch": 12.92,
      "learning_rate": 6.734280303030303e-05,
      "loss": 2.767,
      "step": 222400
    },
    {
      "activations/layer0_attention_weight_max": 15.849007606506348,
      "activations/layer0_attention_weight_min": -12.053828239440918,
      "activations/layer10_attention_weight_max": 39.1799201965332,
      "activations/layer10_attention_weight_min": -39.05119705200195,
      "activations/layer11_attention_weight_max": 39.070072174072266,
      "activations/layer11_attention_weight_min": -39.10697937011719,
      "activations/layer12_attention_weight_max": 26.629148483276367,
      "activations/layer12_attention_weight_min": -27.199647903442383,
      "activations/layer13_attention_weight_max": 42.50992965698242,
      "activations/layer13_attention_weight_min": -38.141387939453125,
      "activations/layer14_attention_weight_max": 47.74580001831055,
      "activations/layer14_attention_weight_min": -41.46135711669922,
      "activations/layer15_attention_weight_max": 46.807159423828125,
      "activations/layer15_attention_weight_min": -37.13417434692383,
      "activations/layer16_attention_weight_max": 30.047527313232422,
      "activations/layer16_attention_weight_min": -29.329483032226562,
      "activations/layer17_attention_weight_max": 30.67510414123535,
      "activations/layer17_attention_weight_min": -27.819522857666016,
      "activations/layer18_attention_weight_max": 31.29157829284668,
      "activations/layer18_attention_weight_min": -25.097492218017578,
      "activations/layer19_attention_weight_max": 30.622802734375,
      "activations/layer19_attention_weight_min": -34.14033889770508,
      "activations/layer1_attention_weight_max": 16.134361267089844,
      "activations/layer1_attention_weight_min": -14.33460521697998,
      "activations/layer20_attention_weight_max": 27.222639083862305,
      "activations/layer20_attention_weight_min": -24.88188362121582,
      "activations/layer21_attention_weight_max": 26.731224060058594,
      "activations/layer21_attention_weight_min": -24.044614791870117,
      "activations/layer22_attention_weight_max": 46.60549545288086,
      "activations/layer22_attention_weight_min": -29.93382453918457,
      "activations/layer23_attention_weight_max": 34.0684928894043,
      "activations/layer23_attention_weight_min": -25.97694969177246,
      "activations/layer2_attention_weight_max": 35.53175354003906,
      "activations/layer2_attention_weight_min": -37.13749694824219,
      "activations/layer3_attention_weight_max": 103.52540588378906,
      "activations/layer3_attention_weight_min": -108.08545684814453,
      "activations/layer4_attention_weight_max": 117.7200698852539,
      "activations/layer4_attention_weight_min": -114.63109588623047,
      "activations/layer5_attention_weight_max": 56.76020812988281,
      "activations/layer5_attention_weight_min": -62.10404968261719,
      "activations/layer6_attention_weight_max": 46.79261016845703,
      "activations/layer6_attention_weight_min": -47.264339447021484,
      "activations/layer7_attention_weight_max": 97.13551330566406,
      "activations/layer7_attention_weight_min": -91.8047866821289,
      "activations/layer8_attention_weight_max": 44.29499816894531,
      "activations/layer8_attention_weight_min": -43.00406265258789,
      "activations/layer9_attention_weight_max": 38.29058837890625,
      "activations/layer9_attention_weight_min": -37.60355758666992,
      "epoch": 12.93,
      "learning_rate": 6.732386363636363e-05,
      "loss": 2.7623,
      "step": 222450
    },
    {
      "activations/layer0_attention_weight_max": 15.636234283447266,
      "activations/layer0_attention_weight_min": -12.001644134521484,
      "activations/layer10_attention_weight_max": 34.5732421875,
      "activations/layer10_attention_weight_min": -34.329402923583984,
      "activations/layer11_attention_weight_max": 32.645965576171875,
      "activations/layer11_attention_weight_min": -34.07170104980469,
      "activations/layer12_attention_weight_max": 26.334091186523438,
      "activations/layer12_attention_weight_min": -26.39702606201172,
      "activations/layer13_attention_weight_max": 42.69036102294922,
      "activations/layer13_attention_weight_min": -35.85428237915039,
      "activations/layer14_attention_weight_max": 47.71834945678711,
      "activations/layer14_attention_weight_min": -39.342506408691406,
      "activations/layer15_attention_weight_max": 39.564064025878906,
      "activations/layer15_attention_weight_min": -34.654537200927734,
      "activations/layer16_attention_weight_max": 32.57258605957031,
      "activations/layer16_attention_weight_min": -30.494112014770508,
      "activations/layer17_attention_weight_max": 30.642887115478516,
      "activations/layer17_attention_weight_min": -25.15082359313965,
      "activations/layer18_attention_weight_max": 32.8883171081543,
      "activations/layer18_attention_weight_min": -23.824462890625,
      "activations/layer19_attention_weight_max": 35.26795959472656,
      "activations/layer19_attention_weight_min": -33.01890182495117,
      "activations/layer1_attention_weight_max": 16.175485610961914,
      "activations/layer1_attention_weight_min": -14.164239883422852,
      "activations/layer20_attention_weight_max": 27.06924057006836,
      "activations/layer20_attention_weight_min": -25.456663131713867,
      "activations/layer21_attention_weight_max": 29.728240966796875,
      "activations/layer21_attention_weight_min": -24.830406188964844,
      "activations/layer22_attention_weight_max": 46.12845230102539,
      "activations/layer22_attention_weight_min": -31.306766510009766,
      "activations/layer23_attention_weight_max": 33.223148345947266,
      "activations/layer23_attention_weight_min": -26.19441032409668,
      "activations/layer2_attention_weight_max": 38.34080123901367,
      "activations/layer2_attention_weight_min": -37.458858489990234,
      "activations/layer3_attention_weight_max": 107.7201156616211,
      "activations/layer3_attention_weight_min": -102.4085922241211,
      "activations/layer4_attention_weight_max": 111.01055145263672,
      "activations/layer4_attention_weight_min": -107.7242660522461,
      "activations/layer5_attention_weight_max": 54.130611419677734,
      "activations/layer5_attention_weight_min": -63.73518371582031,
      "activations/layer6_attention_weight_max": 46.90742492675781,
      "activations/layer6_attention_weight_min": -48.57828140258789,
      "activations/layer7_attention_weight_max": 103.79753112792969,
      "activations/layer7_attention_weight_min": -92.74362182617188,
      "activations/layer8_attention_weight_max": 43.28411102294922,
      "activations/layer8_attention_weight_min": -42.71696472167969,
      "activations/layer9_attention_weight_max": 33.467529296875,
      "activations/layer9_attention_weight_min": -39.10813522338867,
      "epoch": 12.93,
      "learning_rate": 6.730492424242424e-05,
      "loss": 2.7742,
      "step": 222500
    },
    {
      "activations/layer0_attention_weight_max": 16.003705978393555,
      "activations/layer0_attention_weight_min": -11.47581672668457,
      "activations/layer10_attention_weight_max": 35.425018310546875,
      "activations/layer10_attention_weight_min": -33.58647537231445,
      "activations/layer11_attention_weight_max": 34.76906967163086,
      "activations/layer11_attention_weight_min": -32.74901580810547,
      "activations/layer12_attention_weight_max": 27.735401153564453,
      "activations/layer12_attention_weight_min": -24.8834228515625,
      "activations/layer13_attention_weight_max": 43.23817443847656,
      "activations/layer13_attention_weight_min": -37.46897506713867,
      "activations/layer14_attention_weight_max": 45.13490295410156,
      "activations/layer14_attention_weight_min": -39.36408233642578,
      "activations/layer15_attention_weight_max": 41.60750961303711,
      "activations/layer15_attention_weight_min": -33.06434631347656,
      "activations/layer16_attention_weight_max": 33.87184143066406,
      "activations/layer16_attention_weight_min": -27.702302932739258,
      "activations/layer17_attention_weight_max": 33.147682189941406,
      "activations/layer17_attention_weight_min": -26.14141082763672,
      "activations/layer18_attention_weight_max": 34.12342071533203,
      "activations/layer18_attention_weight_min": -22.94563102722168,
      "activations/layer19_attention_weight_max": 34.167198181152344,
      "activations/layer19_attention_weight_min": -32.84249496459961,
      "activations/layer1_attention_weight_max": 16.75038719177246,
      "activations/layer1_attention_weight_min": -15.518911361694336,
      "activations/layer20_attention_weight_max": 31.086557388305664,
      "activations/layer20_attention_weight_min": -25.91758918762207,
      "activations/layer21_attention_weight_max": 29.444438934326172,
      "activations/layer21_attention_weight_min": -24.007888793945312,
      "activations/layer22_attention_weight_max": 45.32820129394531,
      "activations/layer22_attention_weight_min": -29.39919662475586,
      "activations/layer23_attention_weight_max": 35.39766311645508,
      "activations/layer23_attention_weight_min": -24.657299041748047,
      "activations/layer2_attention_weight_max": 31.97311019897461,
      "activations/layer2_attention_weight_min": -32.12537384033203,
      "activations/layer3_attention_weight_max": 98.83162689208984,
      "activations/layer3_attention_weight_min": -95.25308227539062,
      "activations/layer4_attention_weight_max": 109.650390625,
      "activations/layer4_attention_weight_min": -110.894287109375,
      "activations/layer5_attention_weight_max": 52.22486877441406,
      "activations/layer5_attention_weight_min": -66.03795623779297,
      "activations/layer6_attention_weight_max": 45.33856964111328,
      "activations/layer6_attention_weight_min": -46.06837463378906,
      "activations/layer7_attention_weight_max": 92.5949935913086,
      "activations/layer7_attention_weight_min": -88.95391845703125,
      "activations/layer8_attention_weight_max": 41.928627014160156,
      "activations/layer8_attention_weight_min": -41.59182357788086,
      "activations/layer9_attention_weight_max": 34.404296875,
      "activations/layer9_attention_weight_min": -33.5924186706543,
      "epoch": 12.93,
      "learning_rate": 6.728598484848485e-05,
      "loss": 2.7749,
      "step": 222550
    },
    {
      "activations/layer0_attention_weight_max": 16.30714988708496,
      "activations/layer0_attention_weight_min": -12.462775230407715,
      "activations/layer10_attention_weight_max": 33.618408203125,
      "activations/layer10_attention_weight_min": -32.2592887878418,
      "activations/layer11_attention_weight_max": 31.172887802124023,
      "activations/layer11_attention_weight_min": -33.17943572998047,
      "activations/layer12_attention_weight_max": 23.668880462646484,
      "activations/layer12_attention_weight_min": -25.35053062438965,
      "activations/layer13_attention_weight_max": 37.18913269042969,
      "activations/layer13_attention_weight_min": -33.07362747192383,
      "activations/layer14_attention_weight_max": 42.05560302734375,
      "activations/layer14_attention_weight_min": -36.24176788330078,
      "activations/layer15_attention_weight_max": 33.047119140625,
      "activations/layer15_attention_weight_min": -30.851455688476562,
      "activations/layer16_attention_weight_max": 29.488819122314453,
      "activations/layer16_attention_weight_min": -27.686382293701172,
      "activations/layer17_attention_weight_max": 28.408611297607422,
      "activations/layer17_attention_weight_min": -23.911556243896484,
      "activations/layer18_attention_weight_max": 29.293001174926758,
      "activations/layer18_attention_weight_min": -23.564529418945312,
      "activations/layer19_attention_weight_max": 32.04500961303711,
      "activations/layer19_attention_weight_min": -30.197097778320312,
      "activations/layer1_attention_weight_max": 16.223392486572266,
      "activations/layer1_attention_weight_min": -15.077041625976562,
      "activations/layer20_attention_weight_max": 26.256505966186523,
      "activations/layer20_attention_weight_min": -24.5405216217041,
      "activations/layer21_attention_weight_max": 26.73345184326172,
      "activations/layer21_attention_weight_min": -23.831052780151367,
      "activations/layer22_attention_weight_max": 41.106163024902344,
      "activations/layer22_attention_weight_min": -29.00588035583496,
      "activations/layer23_attention_weight_max": 32.621910095214844,
      "activations/layer23_attention_weight_min": -24.756839752197266,
      "activations/layer2_attention_weight_max": 35.31342697143555,
      "activations/layer2_attention_weight_min": -34.45543670654297,
      "activations/layer3_attention_weight_max": 103.71528625488281,
      "activations/layer3_attention_weight_min": -98.81886291503906,
      "activations/layer4_attention_weight_max": 111.79718017578125,
      "activations/layer4_attention_weight_min": -112.75931549072266,
      "activations/layer5_attention_weight_max": 52.17805480957031,
      "activations/layer5_attention_weight_min": -58.920074462890625,
      "activations/layer6_attention_weight_max": 44.952144622802734,
      "activations/layer6_attention_weight_min": -46.68631362915039,
      "activations/layer7_attention_weight_max": 91.04914855957031,
      "activations/layer7_attention_weight_min": -88.82512664794922,
      "activations/layer8_attention_weight_max": 40.63124465942383,
      "activations/layer8_attention_weight_min": -40.8441276550293,
      "activations/layer9_attention_weight_max": 33.2929573059082,
      "activations/layer9_attention_weight_min": -33.52788162231445,
      "epoch": 12.93,
      "learning_rate": 6.726704545454545e-05,
      "loss": 2.7933,
      "step": 222600
    },
    {
      "activations/layer0_attention_weight_max": 15.958026885986328,
      "activations/layer0_attention_weight_min": -12.330876350402832,
      "activations/layer10_attention_weight_max": 32.07347106933594,
      "activations/layer10_attention_weight_min": -33.090087890625,
      "activations/layer11_attention_weight_max": 31.35811996459961,
      "activations/layer11_attention_weight_min": -31.293479919433594,
      "activations/layer12_attention_weight_max": 24.52191162109375,
      "activations/layer12_attention_weight_min": -25.231107711791992,
      "activations/layer13_attention_weight_max": 38.34660339355469,
      "activations/layer13_attention_weight_min": -32.69330596923828,
      "activations/layer14_attention_weight_max": 41.94523620605469,
      "activations/layer14_attention_weight_min": -38.59510803222656,
      "activations/layer15_attention_weight_max": 34.1179313659668,
      "activations/layer15_attention_weight_min": -33.55583953857422,
      "activations/layer16_attention_weight_max": 27.21102523803711,
      "activations/layer16_attention_weight_min": -26.858226776123047,
      "activations/layer17_attention_weight_max": 30.58176040649414,
      "activations/layer17_attention_weight_min": -25.52826499938965,
      "activations/layer18_attention_weight_max": 30.67314338684082,
      "activations/layer18_attention_weight_min": -24.737871170043945,
      "activations/layer19_attention_weight_max": 35.05004119873047,
      "activations/layer19_attention_weight_min": -32.43757247924805,
      "activations/layer1_attention_weight_max": 16.237409591674805,
      "activations/layer1_attention_weight_min": -14.982978820800781,
      "activations/layer20_attention_weight_max": 32.8190803527832,
      "activations/layer20_attention_weight_min": -25.61144256591797,
      "activations/layer21_attention_weight_max": 28.3988037109375,
      "activations/layer21_attention_weight_min": -24.91597557067871,
      "activations/layer22_attention_weight_max": 43.45623779296875,
      "activations/layer22_attention_weight_min": -31.68171501159668,
      "activations/layer23_attention_weight_max": 32.350128173828125,
      "activations/layer23_attention_weight_min": -25.796730041503906,
      "activations/layer2_attention_weight_max": 32.107154846191406,
      "activations/layer2_attention_weight_min": -32.17224884033203,
      "activations/layer3_attention_weight_max": 96.00259399414062,
      "activations/layer3_attention_weight_min": -96.47773742675781,
      "activations/layer4_attention_weight_max": 105.4033432006836,
      "activations/layer4_attention_weight_min": -111.07696533203125,
      "activations/layer5_attention_weight_max": 49.94747543334961,
      "activations/layer5_attention_weight_min": -60.10544967651367,
      "activations/layer6_attention_weight_max": 41.32700729370117,
      "activations/layer6_attention_weight_min": -45.020408630371094,
      "activations/layer7_attention_weight_max": 87.06320190429688,
      "activations/layer7_attention_weight_min": -86.71080780029297,
      "activations/layer8_attention_weight_max": 39.47715759277344,
      "activations/layer8_attention_weight_min": -42.16002655029297,
      "activations/layer9_attention_weight_max": 31.8882999420166,
      "activations/layer9_attention_weight_min": -34.623348236083984,
      "epoch": 12.94,
      "learning_rate": 6.724810606060606e-05,
      "loss": 2.7852,
      "step": 222650
    },
    {
      "activations/layer0_attention_weight_max": 16.525930404663086,
      "activations/layer0_attention_weight_min": -12.274422645568848,
      "activations/layer10_attention_weight_max": 32.682640075683594,
      "activations/layer10_attention_weight_min": -32.79150390625,
      "activations/layer11_attention_weight_max": 34.9470100402832,
      "activations/layer11_attention_weight_min": -32.09870147705078,
      "activations/layer12_attention_weight_max": 27.216129302978516,
      "activations/layer12_attention_weight_min": -27.94686508178711,
      "activations/layer13_attention_weight_max": 43.17322540283203,
      "activations/layer13_attention_weight_min": -34.886558532714844,
      "activations/layer14_attention_weight_max": 41.623172760009766,
      "activations/layer14_attention_weight_min": -37.013431549072266,
      "activations/layer15_attention_weight_max": 38.048057556152344,
      "activations/layer15_attention_weight_min": -32.734989166259766,
      "activations/layer16_attention_weight_max": 30.035219192504883,
      "activations/layer16_attention_weight_min": -28.116220474243164,
      "activations/layer17_attention_weight_max": 29.61893653869629,
      "activations/layer17_attention_weight_min": -25.590721130371094,
      "activations/layer18_attention_weight_max": 30.845947265625,
      "activations/layer18_attention_weight_min": -23.01827621459961,
      "activations/layer19_attention_weight_max": 31.710132598876953,
      "activations/layer19_attention_weight_min": -29.79270362854004,
      "activations/layer1_attention_weight_max": 16.23495864868164,
      "activations/layer1_attention_weight_min": -14.836455345153809,
      "activations/layer20_attention_weight_max": 28.704927444458008,
      "activations/layer20_attention_weight_min": -23.746431350708008,
      "activations/layer21_attention_weight_max": 28.441631317138672,
      "activations/layer21_attention_weight_min": -22.35761260986328,
      "activations/layer22_attention_weight_max": 41.829185485839844,
      "activations/layer22_attention_weight_min": -29.01954460144043,
      "activations/layer23_attention_weight_max": 31.851341247558594,
      "activations/layer23_attention_weight_min": -26.443166732788086,
      "activations/layer2_attention_weight_max": 30.9812068939209,
      "activations/layer2_attention_weight_min": -32.66606521606445,
      "activations/layer3_attention_weight_max": 93.18991088867188,
      "activations/layer3_attention_weight_min": -94.22344970703125,
      "activations/layer4_attention_weight_max": 104.31197357177734,
      "activations/layer4_attention_weight_min": -103.65382385253906,
      "activations/layer5_attention_weight_max": 48.22539520263672,
      "activations/layer5_attention_weight_min": -61.48814392089844,
      "activations/layer6_attention_weight_max": 42.78426742553711,
      "activations/layer6_attention_weight_min": -47.480899810791016,
      "activations/layer7_attention_weight_max": 89.80382537841797,
      "activations/layer7_attention_weight_min": -85.83074951171875,
      "activations/layer8_attention_weight_max": 39.32341384887695,
      "activations/layer8_attention_weight_min": -39.53165054321289,
      "activations/layer9_attention_weight_max": 31.05800437927246,
      "activations/layer9_attention_weight_min": -32.003662109375,
      "epoch": 12.94,
      "learning_rate": 6.722916666666666e-05,
      "loss": 2.773,
      "step": 222700
    },
    {
      "activations/layer0_attention_weight_max": 15.442811012268066,
      "activations/layer0_attention_weight_min": -12.453315734863281,
      "activations/layer10_attention_weight_max": 31.155784606933594,
      "activations/layer10_attention_weight_min": -32.75684356689453,
      "activations/layer11_attention_weight_max": 31.177997589111328,
      "activations/layer11_attention_weight_min": -31.060094833374023,
      "activations/layer12_attention_weight_max": 24.577707290649414,
      "activations/layer12_attention_weight_min": -25.307199478149414,
      "activations/layer13_attention_weight_max": 37.59184265136719,
      "activations/layer13_attention_weight_min": -32.919654846191406,
      "activations/layer14_attention_weight_max": 45.42675018310547,
      "activations/layer14_attention_weight_min": -35.30624771118164,
      "activations/layer15_attention_weight_max": 35.71260070800781,
      "activations/layer15_attention_weight_min": -32.25507736206055,
      "activations/layer16_attention_weight_max": 29.53466796875,
      "activations/layer16_attention_weight_min": -27.37379264831543,
      "activations/layer17_attention_weight_max": 32.7241096496582,
      "activations/layer17_attention_weight_min": -25.77286720275879,
      "activations/layer18_attention_weight_max": 34.9201545715332,
      "activations/layer18_attention_weight_min": -24.97239875793457,
      "activations/layer19_attention_weight_max": 34.437496185302734,
      "activations/layer19_attention_weight_min": -31.037490844726562,
      "activations/layer1_attention_weight_max": 16.582921981811523,
      "activations/layer1_attention_weight_min": -14.903190612792969,
      "activations/layer20_attention_weight_max": 29.682432174682617,
      "activations/layer20_attention_weight_min": -25.609657287597656,
      "activations/layer21_attention_weight_max": 28.76479148864746,
      "activations/layer21_attention_weight_min": -24.553781509399414,
      "activations/layer22_attention_weight_max": 40.018043518066406,
      "activations/layer22_attention_weight_min": -31.225982666015625,
      "activations/layer23_attention_weight_max": 34.584896087646484,
      "activations/layer23_attention_weight_min": -29.60797691345215,
      "activations/layer2_attention_weight_max": 32.2561149597168,
      "activations/layer2_attention_weight_min": -31.257640838623047,
      "activations/layer3_attention_weight_max": 90.2367172241211,
      "activations/layer3_attention_weight_min": -93.19647979736328,
      "activations/layer4_attention_weight_max": 106.64251708984375,
      "activations/layer4_attention_weight_min": -109.93553924560547,
      "activations/layer5_attention_weight_max": 50.0442008972168,
      "activations/layer5_attention_weight_min": -60.87861251831055,
      "activations/layer6_attention_weight_max": 41.61293411254883,
      "activations/layer6_attention_weight_min": -44.86869812011719,
      "activations/layer7_attention_weight_max": 85.85115051269531,
      "activations/layer7_attention_weight_min": -87.39740753173828,
      "activations/layer8_attention_weight_max": 38.179195404052734,
      "activations/layer8_attention_weight_min": -39.649776458740234,
      "activations/layer9_attention_weight_max": 30.233097076416016,
      "activations/layer9_attention_weight_min": -31.695402145385742,
      "epoch": 12.94,
      "learning_rate": 6.721022727272726e-05,
      "loss": 2.7646,
      "step": 222750
    },
    {
      "activations/layer0_attention_weight_max": 16.27251625061035,
      "activations/layer0_attention_weight_min": -12.131884574890137,
      "activations/layer10_attention_weight_max": 31.720867156982422,
      "activations/layer10_attention_weight_min": -32.275657653808594,
      "activations/layer11_attention_weight_max": 33.05503845214844,
      "activations/layer11_attention_weight_min": -34.14387893676758,
      "activations/layer12_attention_weight_max": 24.64289665222168,
      "activations/layer12_attention_weight_min": -23.923734664916992,
      "activations/layer13_attention_weight_max": 41.11211395263672,
      "activations/layer13_attention_weight_min": -33.121891021728516,
      "activations/layer14_attention_weight_max": 43.088470458984375,
      "activations/layer14_attention_weight_min": -37.5525016784668,
      "activations/layer15_attention_weight_max": 37.458927154541016,
      "activations/layer15_attention_weight_min": -31.342838287353516,
      "activations/layer16_attention_weight_max": 32.29487609863281,
      "activations/layer16_attention_weight_min": -28.791706085205078,
      "activations/layer17_attention_weight_max": 32.23312759399414,
      "activations/layer17_attention_weight_min": -26.506380081176758,
      "activations/layer18_attention_weight_max": 29.97414779663086,
      "activations/layer18_attention_weight_min": -23.837982177734375,
      "activations/layer19_attention_weight_max": 36.23775863647461,
      "activations/layer19_attention_weight_min": -32.986900329589844,
      "activations/layer1_attention_weight_max": 16.59532356262207,
      "activations/layer1_attention_weight_min": -15.657469749450684,
      "activations/layer20_attention_weight_max": 33.13642883300781,
      "activations/layer20_attention_weight_min": -25.854337692260742,
      "activations/layer21_attention_weight_max": 31.022287368774414,
      "activations/layer21_attention_weight_min": -24.303123474121094,
      "activations/layer22_attention_weight_max": 45.532222747802734,
      "activations/layer22_attention_weight_min": -29.74915313720703,
      "activations/layer23_attention_weight_max": 33.613704681396484,
      "activations/layer23_attention_weight_min": -24.749256134033203,
      "activations/layer2_attention_weight_max": 32.94990158081055,
      "activations/layer2_attention_weight_min": -34.055274963378906,
      "activations/layer3_attention_weight_max": 99.5604019165039,
      "activations/layer3_attention_weight_min": -102.57798767089844,
      "activations/layer4_attention_weight_max": 113.54371643066406,
      "activations/layer4_attention_weight_min": -112.5048828125,
      "activations/layer5_attention_weight_max": 52.40562057495117,
      "activations/layer5_attention_weight_min": -65.4249038696289,
      "activations/layer6_attention_weight_max": 44.94142532348633,
      "activations/layer6_attention_weight_min": -47.34327697753906,
      "activations/layer7_attention_weight_max": 93.22747039794922,
      "activations/layer7_attention_weight_min": -91.3899154663086,
      "activations/layer8_attention_weight_max": 41.457115173339844,
      "activations/layer8_attention_weight_min": -41.00395965576172,
      "activations/layer9_attention_weight_max": 32.53629684448242,
      "activations/layer9_attention_weight_min": -32.37265396118164,
      "epoch": 12.95,
      "learning_rate": 6.719128787878787e-05,
      "loss": 2.7581,
      "step": 222800
    },
    {
      "activations/layer0_attention_weight_max": 15.781564712524414,
      "activations/layer0_attention_weight_min": -12.336048126220703,
      "activations/layer10_attention_weight_max": 31.50360679626465,
      "activations/layer10_attention_weight_min": -33.2187614440918,
      "activations/layer11_attention_weight_max": 33.399559020996094,
      "activations/layer11_attention_weight_min": -32.66244125366211,
      "activations/layer12_attention_weight_max": 25.448652267456055,
      "activations/layer12_attention_weight_min": -25.251998901367188,
      "activations/layer13_attention_weight_max": 39.954437255859375,
      "activations/layer13_attention_weight_min": -34.48635482788086,
      "activations/layer14_attention_weight_max": 49.721961975097656,
      "activations/layer14_attention_weight_min": -37.54814529418945,
      "activations/layer15_attention_weight_max": 37.22566223144531,
      "activations/layer15_attention_weight_min": -34.078670501708984,
      "activations/layer16_attention_weight_max": 32.21611785888672,
      "activations/layer16_attention_weight_min": -26.310638427734375,
      "activations/layer17_attention_weight_max": 31.74679946899414,
      "activations/layer17_attention_weight_min": -26.37959861755371,
      "activations/layer18_attention_weight_max": 34.1251106262207,
      "activations/layer18_attention_weight_min": -23.158504486083984,
      "activations/layer19_attention_weight_max": 34.908023834228516,
      "activations/layer19_attention_weight_min": -30.414112091064453,
      "activations/layer1_attention_weight_max": 16.860389709472656,
      "activations/layer1_attention_weight_min": -17.05823516845703,
      "activations/layer20_attention_weight_max": 29.575048446655273,
      "activations/layer20_attention_weight_min": -25.163646697998047,
      "activations/layer21_attention_weight_max": 29.537246704101562,
      "activations/layer21_attention_weight_min": -25.05757713317871,
      "activations/layer22_attention_weight_max": 43.58316421508789,
      "activations/layer22_attention_weight_min": -30.82143783569336,
      "activations/layer23_attention_weight_max": 35.16156768798828,
      "activations/layer23_attention_weight_min": -25.30583953857422,
      "activations/layer2_attention_weight_max": 32.8454704284668,
      "activations/layer2_attention_weight_min": -34.69013595581055,
      "activations/layer3_attention_weight_max": 97.14000701904297,
      "activations/layer3_attention_weight_min": -101.64561462402344,
      "activations/layer4_attention_weight_max": 111.5722427368164,
      "activations/layer4_attention_weight_min": -107.79011535644531,
      "activations/layer5_attention_weight_max": 52.558799743652344,
      "activations/layer5_attention_weight_min": -63.52388000488281,
      "activations/layer6_attention_weight_max": 42.29920959472656,
      "activations/layer6_attention_weight_min": -45.476951599121094,
      "activations/layer7_attention_weight_max": 84.72320556640625,
      "activations/layer7_attention_weight_min": -84.35536193847656,
      "activations/layer8_attention_weight_max": 41.160194396972656,
      "activations/layer8_attention_weight_min": -41.09910583496094,
      "activations/layer9_attention_weight_max": 31.88059425354004,
      "activations/layer9_attention_weight_min": -34.189151763916016,
      "epoch": 12.95,
      "learning_rate": 6.717234848484848e-05,
      "loss": 2.7735,
      "step": 222850
    },
    {
      "activations/layer0_attention_weight_max": 15.928266525268555,
      "activations/layer0_attention_weight_min": -12.14224624633789,
      "activations/layer10_attention_weight_max": 37.37334060668945,
      "activations/layer10_attention_weight_min": -33.99729919433594,
      "activations/layer11_attention_weight_max": 36.329856872558594,
      "activations/layer11_attention_weight_min": -35.40849685668945,
      "activations/layer12_attention_weight_max": 32.04218292236328,
      "activations/layer12_attention_weight_min": -26.258548736572266,
      "activations/layer13_attention_weight_max": 48.87474822998047,
      "activations/layer13_attention_weight_min": -35.25178527832031,
      "activations/layer14_attention_weight_max": 46.22062683105469,
      "activations/layer14_attention_weight_min": -37.16695785522461,
      "activations/layer15_attention_weight_max": 41.98408508300781,
      "activations/layer15_attention_weight_min": -33.674049377441406,
      "activations/layer16_attention_weight_max": 35.35309982299805,
      "activations/layer16_attention_weight_min": -28.03265953063965,
      "activations/layer17_attention_weight_max": 38.924713134765625,
      "activations/layer17_attention_weight_min": -26.90871810913086,
      "activations/layer18_attention_weight_max": 32.359954833984375,
      "activations/layer18_attention_weight_min": -24.72802734375,
      "activations/layer19_attention_weight_max": 39.83931350708008,
      "activations/layer19_attention_weight_min": -31.99884033203125,
      "activations/layer1_attention_weight_max": 15.813628196716309,
      "activations/layer1_attention_weight_min": -14.716785430908203,
      "activations/layer20_attention_weight_max": 36.871116638183594,
      "activations/layer20_attention_weight_min": -25.326030731201172,
      "activations/layer21_attention_weight_max": 35.836280822753906,
      "activations/layer21_attention_weight_min": -24.670516967773438,
      "activations/layer22_attention_weight_max": 49.67938995361328,
      "activations/layer22_attention_weight_min": -30.629549026489258,
      "activations/layer23_attention_weight_max": 40.06843948364258,
      "activations/layer23_attention_weight_min": -24.340648651123047,
      "activations/layer2_attention_weight_max": 32.261619567871094,
      "activations/layer2_attention_weight_min": -32.91448974609375,
      "activations/layer3_attention_weight_max": 97.9716567993164,
      "activations/layer3_attention_weight_min": -101.11915588378906,
      "activations/layer4_attention_weight_max": 111.86565399169922,
      "activations/layer4_attention_weight_min": -106.51871490478516,
      "activations/layer5_attention_weight_max": 50.676815032958984,
      "activations/layer5_attention_weight_min": -62.46240997314453,
      "activations/layer6_attention_weight_max": 45.992671966552734,
      "activations/layer6_attention_weight_min": -46.81214904785156,
      "activations/layer7_attention_weight_max": 94.10091400146484,
      "activations/layer7_attention_weight_min": -89.98282623291016,
      "activations/layer8_attention_weight_max": 42.43843460083008,
      "activations/layer8_attention_weight_min": -41.51905059814453,
      "activations/layer9_attention_weight_max": 34.86799240112305,
      "activations/layer9_attention_weight_min": -34.11126708984375,
      "epoch": 12.95,
      "learning_rate": 6.715340909090908e-05,
      "loss": 2.7739,
      "step": 222900
    },
    {
      "activations/layer0_attention_weight_max": 15.809282302856445,
      "activations/layer0_attention_weight_min": -10.843612670898438,
      "activations/layer10_attention_weight_max": 31.678836822509766,
      "activations/layer10_attention_weight_min": -34.980857849121094,
      "activations/layer11_attention_weight_max": 32.66744613647461,
      "activations/layer11_attention_weight_min": -31.191768646240234,
      "activations/layer12_attention_weight_max": 25.144832611083984,
      "activations/layer12_attention_weight_min": -25.214237213134766,
      "activations/layer13_attention_weight_max": 37.6134033203125,
      "activations/layer13_attention_weight_min": -35.62553405761719,
      "activations/layer14_attention_weight_max": 43.42599868774414,
      "activations/layer14_attention_weight_min": -38.739585876464844,
      "activations/layer15_attention_weight_max": 36.906558990478516,
      "activations/layer15_attention_weight_min": -32.239810943603516,
      "activations/layer16_attention_weight_max": 33.64470672607422,
      "activations/layer16_attention_weight_min": -26.979286193847656,
      "activations/layer17_attention_weight_max": 33.68254470825195,
      "activations/layer17_attention_weight_min": -24.37221908569336,
      "activations/layer18_attention_weight_max": 33.447513580322266,
      "activations/layer18_attention_weight_min": -23.160478591918945,
      "activations/layer19_attention_weight_max": 38.20076370239258,
      "activations/layer19_attention_weight_min": -31.325897216796875,
      "activations/layer1_attention_weight_max": 16.35355567932129,
      "activations/layer1_attention_weight_min": -13.751039505004883,
      "activations/layer20_attention_weight_max": 32.47212219238281,
      "activations/layer20_attention_weight_min": -24.907236099243164,
      "activations/layer21_attention_weight_max": 30.9509334564209,
      "activations/layer21_attention_weight_min": -24.347885131835938,
      "activations/layer22_attention_weight_max": 44.836669921875,
      "activations/layer22_attention_weight_min": -30.523788452148438,
      "activations/layer23_attention_weight_max": 37.62165069580078,
      "activations/layer23_attention_weight_min": -24.891857147216797,
      "activations/layer2_attention_weight_max": 34.01995849609375,
      "activations/layer2_attention_weight_min": -32.67341613769531,
      "activations/layer3_attention_weight_max": 101.21283721923828,
      "activations/layer3_attention_weight_min": -102.24222564697266,
      "activations/layer4_attention_weight_max": 108.94146728515625,
      "activations/layer4_attention_weight_min": -110.26393127441406,
      "activations/layer5_attention_weight_max": 52.08936309814453,
      "activations/layer5_attention_weight_min": -63.243377685546875,
      "activations/layer6_attention_weight_max": 44.287967681884766,
      "activations/layer6_attention_weight_min": -48.28409194946289,
      "activations/layer7_attention_weight_max": 96.30846405029297,
      "activations/layer7_attention_weight_min": -95.69478607177734,
      "activations/layer8_attention_weight_max": 41.80321502685547,
      "activations/layer8_attention_weight_min": -41.57849884033203,
      "activations/layer9_attention_weight_max": 33.734886169433594,
      "activations/layer9_attention_weight_min": -34.717342376708984,
      "epoch": 12.95,
      "learning_rate": 6.713446969696968e-05,
      "loss": 2.7468,
      "step": 222950
    },
    {
      "activations/layer0_attention_weight_max": 15.826289176940918,
      "activations/layer0_attention_weight_min": -12.141692161560059,
      "activations/layer10_attention_weight_max": 37.69876480102539,
      "activations/layer10_attention_weight_min": -33.89900588989258,
      "activations/layer11_attention_weight_max": 39.833702087402344,
      "activations/layer11_attention_weight_min": -37.87503433227539,
      "activations/layer12_attention_weight_max": 26.61809730529785,
      "activations/layer12_attention_weight_min": -24.989524841308594,
      "activations/layer13_attention_weight_max": 42.63336181640625,
      "activations/layer13_attention_weight_min": -33.13943099975586,
      "activations/layer14_attention_weight_max": 47.82892990112305,
      "activations/layer14_attention_weight_min": -37.32228088378906,
      "activations/layer15_attention_weight_max": 43.0335578918457,
      "activations/layer15_attention_weight_min": -33.41845703125,
      "activations/layer16_attention_weight_max": 34.747528076171875,
      "activations/layer16_attention_weight_min": -27.452348709106445,
      "activations/layer17_attention_weight_max": 34.86467361450195,
      "activations/layer17_attention_weight_min": -29.089155197143555,
      "activations/layer18_attention_weight_max": 30.676151275634766,
      "activations/layer18_attention_weight_min": -24.93450927734375,
      "activations/layer19_attention_weight_max": 40.52092742919922,
      "activations/layer19_attention_weight_min": -30.996591567993164,
      "activations/layer1_attention_weight_max": 17.602279663085938,
      "activations/layer1_attention_weight_min": -17.602352142333984,
      "activations/layer20_attention_weight_max": 35.083778381347656,
      "activations/layer20_attention_weight_min": -25.525440216064453,
      "activations/layer21_attention_weight_max": 34.83426284790039,
      "activations/layer21_attention_weight_min": -24.573379516601562,
      "activations/layer22_attention_weight_max": 57.39802932739258,
      "activations/layer22_attention_weight_min": -33.32201385498047,
      "activations/layer23_attention_weight_max": 41.05372619628906,
      "activations/layer23_attention_weight_min": -24.836212158203125,
      "activations/layer2_attention_weight_max": 36.962799072265625,
      "activations/layer2_attention_weight_min": -35.2975959777832,
      "activations/layer3_attention_weight_max": 106.99725341796875,
      "activations/layer3_attention_weight_min": -104.97413635253906,
      "activations/layer4_attention_weight_max": 114.25440979003906,
      "activations/layer4_attention_weight_min": -109.9190673828125,
      "activations/layer5_attention_weight_max": 53.86396026611328,
      "activations/layer5_attention_weight_min": -60.478755950927734,
      "activations/layer6_attention_weight_max": 45.269100189208984,
      "activations/layer6_attention_weight_min": -46.92793273925781,
      "activations/layer7_attention_weight_max": 100.29338073730469,
      "activations/layer7_attention_weight_min": -96.70980072021484,
      "activations/layer8_attention_weight_max": 46.426448822021484,
      "activations/layer8_attention_weight_min": -42.61244201660156,
      "activations/layer9_attention_weight_max": 37.062843322753906,
      "activations/layer9_attention_weight_min": -36.47850036621094,
      "epoch": 12.96,
      "learning_rate": 6.71155303030303e-05,
      "loss": 2.7523,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4345,
      "eval_samples_per_second": 509.097,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4345,
      "eval_openwebtext_samples_per_second": 509.097,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9338,
      "eval_wikitext_samples_per_second": 235.8,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_lambada_loss": 2.44921875,
      "eval_lambada_ppl": 11.57929685891098,
      "eval_lambada_runtime": 9.5002,
      "eval_lambada_samples_per_second": 512.515,
      "step": 223000
    },
    {
      "activations/layer0_attention_weight_max": 16.151817321777344,
      "activations/layer0_attention_weight_min": -12.100269317626953,
      "activations/layer10_attention_weight_max": 38.25127029418945,
      "activations/layer10_attention_weight_min": -38.66387939453125,
      "activations/layer11_attention_weight_max": 38.23280715942383,
      "activations/layer11_attention_weight_min": -36.5045166015625,
      "activations/layer12_attention_weight_max": 28.68474006652832,
      "activations/layer12_attention_weight_min": -28.91065788269043,
      "activations/layer13_attention_weight_max": 46.151004791259766,
      "activations/layer13_attention_weight_min": -37.64360427856445,
      "activations/layer14_attention_weight_max": 50.50544738769531,
      "activations/layer14_attention_weight_min": -43.86198425292969,
      "activations/layer15_attention_weight_max": 39.57738494873047,
      "activations/layer15_attention_weight_min": -37.46831512451172,
      "activations/layer16_attention_weight_max": 32.71207046508789,
      "activations/layer16_attention_weight_min": -28.718883514404297,
      "activations/layer17_attention_weight_max": 33.10343933105469,
      "activations/layer17_attention_weight_min": -26.193071365356445,
      "activations/layer18_attention_weight_max": 30.395286560058594,
      "activations/layer18_attention_weight_min": -23.95490837097168,
      "activations/layer19_attention_weight_max": 36.511104583740234,
      "activations/layer19_attention_weight_min": -31.11039161682129,
      "activations/layer1_attention_weight_max": 16.694561004638672,
      "activations/layer1_attention_weight_min": -13.7267427444458,
      "activations/layer20_attention_weight_max": 29.94136619567871,
      "activations/layer20_attention_weight_min": -25.516550064086914,
      "activations/layer21_attention_weight_max": 28.63936996459961,
      "activations/layer21_attention_weight_min": -24.455886840820312,
      "activations/layer22_attention_weight_max": 42.827083587646484,
      "activations/layer22_attention_weight_min": -31.731260299682617,
      "activations/layer23_attention_weight_max": 34.26348114013672,
      "activations/layer23_attention_weight_min": -23.99321746826172,
      "activations/layer2_attention_weight_max": 31.442161560058594,
      "activations/layer2_attention_weight_min": -31.865066528320312,
      "activations/layer3_attention_weight_max": 99.14962768554688,
      "activations/layer3_attention_weight_min": -99.14810180664062,
      "activations/layer4_attention_weight_max": 114.85070037841797,
      "activations/layer4_attention_weight_min": -115.33196258544922,
      "activations/layer5_attention_weight_max": 53.33757019042969,
      "activations/layer5_attention_weight_min": -65.37481689453125,
      "activations/layer6_attention_weight_max": 49.42655944824219,
      "activations/layer6_attention_weight_min": -50.97526550292969,
      "activations/layer7_attention_weight_max": 99.49765014648438,
      "activations/layer7_attention_weight_min": -101.43000793457031,
      "activations/layer8_attention_weight_max": 45.00600051879883,
      "activations/layer8_attention_weight_min": -47.3241081237793,
      "activations/layer9_attention_weight_max": 38.873531341552734,
      "activations/layer9_attention_weight_min": -39.12242126464844,
      "epoch": 12.96,
      "learning_rate": 6.70965909090909e-05,
      "loss": 2.781,
      "step": 223050
    },
    {
      "activations/layer0_attention_weight_max": 15.75143814086914,
      "activations/layer0_attention_weight_min": -11.647409439086914,
      "activations/layer10_attention_weight_max": 29.906198501586914,
      "activations/layer10_attention_weight_min": -31.808223724365234,
      "activations/layer11_attention_weight_max": 32.04338073730469,
      "activations/layer11_attention_weight_min": -31.31305503845215,
      "activations/layer12_attention_weight_max": 23.815845489501953,
      "activations/layer12_attention_weight_min": -24.960086822509766,
      "activations/layer13_attention_weight_max": 35.93972396850586,
      "activations/layer13_attention_weight_min": -32.56492233276367,
      "activations/layer14_attention_weight_max": 44.97529983520508,
      "activations/layer14_attention_weight_min": -36.97525405883789,
      "activations/layer15_attention_weight_max": 37.13547134399414,
      "activations/layer15_attention_weight_min": -31.97956085205078,
      "activations/layer16_attention_weight_max": 30.536405563354492,
      "activations/layer16_attention_weight_min": -26.522035598754883,
      "activations/layer17_attention_weight_max": 30.32740592956543,
      "activations/layer17_attention_weight_min": -24.217967987060547,
      "activations/layer18_attention_weight_max": 30.150236129760742,
      "activations/layer18_attention_weight_min": -22.447059631347656,
      "activations/layer19_attention_weight_max": 36.17921829223633,
      "activations/layer19_attention_weight_min": -31.259185791015625,
      "activations/layer1_attention_weight_max": 17.59779930114746,
      "activations/layer1_attention_weight_min": -14.862342834472656,
      "activations/layer20_attention_weight_max": 30.25442123413086,
      "activations/layer20_attention_weight_min": -24.27695655822754,
      "activations/layer21_attention_weight_max": 27.927352905273438,
      "activations/layer21_attention_weight_min": -25.592357635498047,
      "activations/layer22_attention_weight_max": 45.39830017089844,
      "activations/layer22_attention_weight_min": -30.12738609313965,
      "activations/layer23_attention_weight_max": 33.614784240722656,
      "activations/layer23_attention_weight_min": -27.269813537597656,
      "activations/layer2_attention_weight_max": 31.090137481689453,
      "activations/layer2_attention_weight_min": -31.494152069091797,
      "activations/layer3_attention_weight_max": 93.76023864746094,
      "activations/layer3_attention_weight_min": -94.87124633789062,
      "activations/layer4_attention_weight_max": 105.8390121459961,
      "activations/layer4_attention_weight_min": -108.84228515625,
      "activations/layer5_attention_weight_max": 50.17307662963867,
      "activations/layer5_attention_weight_min": -63.05510711669922,
      "activations/layer6_attention_weight_max": 42.71909713745117,
      "activations/layer6_attention_weight_min": -47.913002014160156,
      "activations/layer7_attention_weight_max": 84.18889617919922,
      "activations/layer7_attention_weight_min": -91.39957427978516,
      "activations/layer8_attention_weight_max": 37.34767532348633,
      "activations/layer8_attention_weight_min": -43.95524978637695,
      "activations/layer9_attention_weight_max": 31.59503173828125,
      "activations/layer9_attention_weight_min": -33.73670196533203,
      "epoch": 12.96,
      "learning_rate": 6.707765151515152e-05,
      "loss": 2.7707,
      "step": 223100
    },
    {
      "activations/layer0_attention_weight_max": 16.331445693969727,
      "activations/layer0_attention_weight_min": -12.324991226196289,
      "activations/layer10_attention_weight_max": 31.834171295166016,
      "activations/layer10_attention_weight_min": -33.196475982666016,
      "activations/layer11_attention_weight_max": 32.169464111328125,
      "activations/layer11_attention_weight_min": -33.93750762939453,
      "activations/layer12_attention_weight_max": 25.41868019104004,
      "activations/layer12_attention_weight_min": -26.837831497192383,
      "activations/layer13_attention_weight_max": 39.10413360595703,
      "activations/layer13_attention_weight_min": -37.46673583984375,
      "activations/layer14_attention_weight_max": 40.16062927246094,
      "activations/layer14_attention_weight_min": -37.44705581665039,
      "activations/layer15_attention_weight_max": 36.00835418701172,
      "activations/layer15_attention_weight_min": -32.65299987792969,
      "activations/layer16_attention_weight_max": 29.394546508789062,
      "activations/layer16_attention_weight_min": -29.383190155029297,
      "activations/layer17_attention_weight_max": 29.73377227783203,
      "activations/layer17_attention_weight_min": -26.29621124267578,
      "activations/layer18_attention_weight_max": 31.44987678527832,
      "activations/layer18_attention_weight_min": -24.648359298706055,
      "activations/layer19_attention_weight_max": 34.175724029541016,
      "activations/layer19_attention_weight_min": -31.541433334350586,
      "activations/layer1_attention_weight_max": 16.470008850097656,
      "activations/layer1_attention_weight_min": -14.637618064880371,
      "activations/layer20_attention_weight_max": 28.609556198120117,
      "activations/layer20_attention_weight_min": -24.69591522216797,
      "activations/layer21_attention_weight_max": 29.28316879272461,
      "activations/layer21_attention_weight_min": -23.89607810974121,
      "activations/layer22_attention_weight_max": 40.85920715332031,
      "activations/layer22_attention_weight_min": -31.30116081237793,
      "activations/layer23_attention_weight_max": 31.29021453857422,
      "activations/layer23_attention_weight_min": -26.779722213745117,
      "activations/layer2_attention_weight_max": 32.36328887939453,
      "activations/layer2_attention_weight_min": -34.70246887207031,
      "activations/layer3_attention_weight_max": 100.53833770751953,
      "activations/layer3_attention_weight_min": -103.5622787475586,
      "activations/layer4_attention_weight_max": 110.31319427490234,
      "activations/layer4_attention_weight_min": -112.9588394165039,
      "activations/layer5_attention_weight_max": 54.950897216796875,
      "activations/layer5_attention_weight_min": -64.42915344238281,
      "activations/layer6_attention_weight_max": 44.26033020019531,
      "activations/layer6_attention_weight_min": -46.24589157104492,
      "activations/layer7_attention_weight_max": 88.80522918701172,
      "activations/layer7_attention_weight_min": -88.29639434814453,
      "activations/layer8_attention_weight_max": 39.64072799682617,
      "activations/layer8_attention_weight_min": -41.75026321411133,
      "activations/layer9_attention_weight_max": 32.28324890136719,
      "activations/layer9_attention_weight_min": -33.96563720703125,
      "epoch": 12.97,
      "learning_rate": 6.705871212121212e-05,
      "loss": 2.7606,
      "step": 223150
    },
    {
      "activations/layer0_attention_weight_max": 15.386275291442871,
      "activations/layer0_attention_weight_min": -11.560365676879883,
      "activations/layer10_attention_weight_max": 33.510780334472656,
      "activations/layer10_attention_weight_min": -37.198753356933594,
      "activations/layer11_attention_weight_max": 33.6793098449707,
      "activations/layer11_attention_weight_min": -37.415504455566406,
      "activations/layer12_attention_weight_max": 29.314834594726562,
      "activations/layer12_attention_weight_min": -25.787948608398438,
      "activations/layer13_attention_weight_max": 46.61945724487305,
      "activations/layer13_attention_weight_min": -37.37583923339844,
      "activations/layer14_attention_weight_max": 56.16695785522461,
      "activations/layer14_attention_weight_min": -40.27766036987305,
      "activations/layer15_attention_weight_max": 41.88694763183594,
      "activations/layer15_attention_weight_min": -32.90422821044922,
      "activations/layer16_attention_weight_max": 35.6335334777832,
      "activations/layer16_attention_weight_min": -27.304122924804688,
      "activations/layer17_attention_weight_max": 39.62681198120117,
      "activations/layer17_attention_weight_min": -26.284204483032227,
      "activations/layer18_attention_weight_max": 37.87919616699219,
      "activations/layer18_attention_weight_min": -25.21244239807129,
      "activations/layer19_attention_weight_max": 45.2074089050293,
      "activations/layer19_attention_weight_min": -32.47343063354492,
      "activations/layer1_attention_weight_max": 16.11603355407715,
      "activations/layer1_attention_weight_min": -16.353864669799805,
      "activations/layer20_attention_weight_max": 40.589908599853516,
      "activations/layer20_attention_weight_min": -26.37050437927246,
      "activations/layer21_attention_weight_max": 37.4815788269043,
      "activations/layer21_attention_weight_min": -25.790788650512695,
      "activations/layer22_attention_weight_max": 64.31495666503906,
      "activations/layer22_attention_weight_min": -34.440452575683594,
      "activations/layer23_attention_weight_max": 42.68690490722656,
      "activations/layer23_attention_weight_min": -28.32045555114746,
      "activations/layer2_attention_weight_max": 34.7333984375,
      "activations/layer2_attention_weight_min": -34.609371185302734,
      "activations/layer3_attention_weight_max": 97.08635711669922,
      "activations/layer3_attention_weight_min": -99.99003601074219,
      "activations/layer4_attention_weight_max": 112.72450256347656,
      "activations/layer4_attention_weight_min": -117.4835433959961,
      "activations/layer5_attention_weight_max": 56.39514923095703,
      "activations/layer5_attention_weight_min": -60.69608688354492,
      "activations/layer6_attention_weight_max": 46.89717102050781,
      "activations/layer6_attention_weight_min": -48.52231216430664,
      "activations/layer7_attention_weight_max": 96.26493835449219,
      "activations/layer7_attention_weight_min": -95.12147521972656,
      "activations/layer8_attention_weight_max": 43.04196548461914,
      "activations/layer8_attention_weight_min": -47.22099685668945,
      "activations/layer9_attention_weight_max": 33.67655944824219,
      "activations/layer9_attention_weight_min": -39.164859771728516,
      "epoch": 12.97,
      "learning_rate": 6.703977272727272e-05,
      "loss": 2.7624,
      "step": 223200
    },
    {
      "activations/layer0_attention_weight_max": 15.966588973999023,
      "activations/layer0_attention_weight_min": -11.658408164978027,
      "activations/layer10_attention_weight_max": 33.11488342285156,
      "activations/layer10_attention_weight_min": -33.690975189208984,
      "activations/layer11_attention_weight_max": 30.811803817749023,
      "activations/layer11_attention_weight_min": -32.10404586791992,
      "activations/layer12_attention_weight_max": 28.498836517333984,
      "activations/layer12_attention_weight_min": -26.50774574279785,
      "activations/layer13_attention_weight_max": 43.01056671142578,
      "activations/layer13_attention_weight_min": -36.1634635925293,
      "activations/layer14_attention_weight_max": 50.80398941040039,
      "activations/layer14_attention_weight_min": -39.232643127441406,
      "activations/layer15_attention_weight_max": 42.495853424072266,
      "activations/layer15_attention_weight_min": -33.82278823852539,
      "activations/layer16_attention_weight_max": 36.39259338378906,
      "activations/layer16_attention_weight_min": -29.14079475402832,
      "activations/layer17_attention_weight_max": 34.391117095947266,
      "activations/layer17_attention_weight_min": -26.331260681152344,
      "activations/layer18_attention_weight_max": 35.319053649902344,
      "activations/layer18_attention_weight_min": -25.40104103088379,
      "activations/layer19_attention_weight_max": 39.97034454345703,
      "activations/layer19_attention_weight_min": -31.473037719726562,
      "activations/layer1_attention_weight_max": 16.84052085876465,
      "activations/layer1_attention_weight_min": -13.635452270507812,
      "activations/layer20_attention_weight_max": 36.240413665771484,
      "activations/layer20_attention_weight_min": -25.434635162353516,
      "activations/layer21_attention_weight_max": 37.25924301147461,
      "activations/layer21_attention_weight_min": -25.229787826538086,
      "activations/layer22_attention_weight_max": 52.63828659057617,
      "activations/layer22_attention_weight_min": -31.2359676361084,
      "activations/layer23_attention_weight_max": 39.94920349121094,
      "activations/layer23_attention_weight_min": -24.746246337890625,
      "activations/layer2_attention_weight_max": 30.899730682373047,
      "activations/layer2_attention_weight_min": -33.24574661254883,
      "activations/layer3_attention_weight_max": 91.43111419677734,
      "activations/layer3_attention_weight_min": -96.17929077148438,
      "activations/layer4_attention_weight_max": 105.84722900390625,
      "activations/layer4_attention_weight_min": -112.3475341796875,
      "activations/layer5_attention_weight_max": 50.500938415527344,
      "activations/layer5_attention_weight_min": -61.941341400146484,
      "activations/layer6_attention_weight_max": 43.57478713989258,
      "activations/layer6_attention_weight_min": -48.82900619506836,
      "activations/layer7_attention_weight_max": 91.59673309326172,
      "activations/layer7_attention_weight_min": -95.47337341308594,
      "activations/layer8_attention_weight_max": 38.455726623535156,
      "activations/layer8_attention_weight_min": -42.725345611572266,
      "activations/layer9_attention_weight_max": 33.499332427978516,
      "activations/layer9_attention_weight_min": -34.31328201293945,
      "epoch": 12.97,
      "learning_rate": 6.702121212121212e-05,
      "loss": 2.779,
      "step": 223250
    },
    {
      "activations/layer0_attention_weight_max": 16.5115966796875,
      "activations/layer0_attention_weight_min": -11.906044006347656,
      "activations/layer10_attention_weight_max": 39.217498779296875,
      "activations/layer10_attention_weight_min": -38.84872817993164,
      "activations/layer11_attention_weight_max": 38.73931121826172,
      "activations/layer11_attention_weight_min": -37.37762451171875,
      "activations/layer12_attention_weight_max": 37.714019775390625,
      "activations/layer12_attention_weight_min": -29.655622482299805,
      "activations/layer13_attention_weight_max": 60.379905700683594,
      "activations/layer13_attention_weight_min": -39.843360900878906,
      "activations/layer14_attention_weight_max": 56.52179718017578,
      "activations/layer14_attention_weight_min": -40.633270263671875,
      "activations/layer15_attention_weight_max": 57.56681442260742,
      "activations/layer15_attention_weight_min": -38.7965087890625,
      "activations/layer16_attention_weight_max": 38.75758361816406,
      "activations/layer16_attention_weight_min": -30.712739944458008,
      "activations/layer17_attention_weight_max": 35.71061325073242,
      "activations/layer17_attention_weight_min": -28.23982810974121,
      "activations/layer18_attention_weight_max": 36.62376022338867,
      "activations/layer18_attention_weight_min": -29.885234832763672,
      "activations/layer19_attention_weight_max": 40.05033493041992,
      "activations/layer19_attention_weight_min": -33.72819137573242,
      "activations/layer1_attention_weight_max": 16.541481018066406,
      "activations/layer1_attention_weight_min": -14.7775297164917,
      "activations/layer20_attention_weight_max": 35.114898681640625,
      "activations/layer20_attention_weight_min": -28.042757034301758,
      "activations/layer21_attention_weight_max": 34.915931701660156,
      "activations/layer21_attention_weight_min": -27.618675231933594,
      "activations/layer22_attention_weight_max": 50.68721389770508,
      "activations/layer22_attention_weight_min": -34.17073440551758,
      "activations/layer23_attention_weight_max": 35.965476989746094,
      "activations/layer23_attention_weight_min": -25.227848052978516,
      "activations/layer2_attention_weight_max": 32.59698486328125,
      "activations/layer2_attention_weight_min": -32.843505859375,
      "activations/layer3_attention_weight_max": 94.77412414550781,
      "activations/layer3_attention_weight_min": -98.59644317626953,
      "activations/layer4_attention_weight_max": 106.59586334228516,
      "activations/layer4_attention_weight_min": -109.69830322265625,
      "activations/layer5_attention_weight_max": 50.66850280761719,
      "activations/layer5_attention_weight_min": -60.88386535644531,
      "activations/layer6_attention_weight_max": 44.84932327270508,
      "activations/layer6_attention_weight_min": -47.17496871948242,
      "activations/layer7_attention_weight_max": 101.87648010253906,
      "activations/layer7_attention_weight_min": -92.20651245117188,
      "activations/layer8_attention_weight_max": 45.97119140625,
      "activations/layer8_attention_weight_min": -46.50950622558594,
      "activations/layer9_attention_weight_max": 38.66249465942383,
      "activations/layer9_attention_weight_min": -38.499061584472656,
      "epoch": 12.97,
      "learning_rate": 6.700227272727272e-05,
      "loss": 2.7609,
      "step": 223300
    },
    {
      "activations/layer0_attention_weight_max": 14.742764472961426,
      "activations/layer0_attention_weight_min": -11.503499031066895,
      "activations/layer10_attention_weight_max": 31.071794509887695,
      "activations/layer10_attention_weight_min": -32.8594856262207,
      "activations/layer11_attention_weight_max": 31.319272994995117,
      "activations/layer11_attention_weight_min": -31.688884735107422,
      "activations/layer12_attention_weight_max": 25.0653018951416,
      "activations/layer12_attention_weight_min": -25.163999557495117,
      "activations/layer13_attention_weight_max": 38.336151123046875,
      "activations/layer13_attention_weight_min": -34.21768569946289,
      "activations/layer14_attention_weight_max": 41.96351623535156,
      "activations/layer14_attention_weight_min": -37.76419448852539,
      "activations/layer15_attention_weight_max": 37.43989944458008,
      "activations/layer15_attention_weight_min": -32.6546630859375,
      "activations/layer16_attention_weight_max": 30.97607421875,
      "activations/layer16_attention_weight_min": -28.07865333557129,
      "activations/layer17_attention_weight_max": 31.351125717163086,
      "activations/layer17_attention_weight_min": -26.41534996032715,
      "activations/layer18_attention_weight_max": 32.76158905029297,
      "activations/layer18_attention_weight_min": -22.39967918395996,
      "activations/layer19_attention_weight_max": 33.540199279785156,
      "activations/layer19_attention_weight_min": -30.31803321838379,
      "activations/layer1_attention_weight_max": 16.737743377685547,
      "activations/layer1_attention_weight_min": -14.033016204833984,
      "activations/layer20_attention_weight_max": 28.753007888793945,
      "activations/layer20_attention_weight_min": -24.502500534057617,
      "activations/layer21_attention_weight_max": 29.19741439819336,
      "activations/layer21_attention_weight_min": -23.3265323638916,
      "activations/layer22_attention_weight_max": 46.473724365234375,
      "activations/layer22_attention_weight_min": -29.536380767822266,
      "activations/layer23_attention_weight_max": 35.49192810058594,
      "activations/layer23_attention_weight_min": -25.188222885131836,
      "activations/layer2_attention_weight_max": 32.635292053222656,
      "activations/layer2_attention_weight_min": -32.45338821411133,
      "activations/layer3_attention_weight_max": 96.07997131347656,
      "activations/layer3_attention_weight_min": -95.74723815917969,
      "activations/layer4_attention_weight_max": 114.15593719482422,
      "activations/layer4_attention_weight_min": -114.72940826416016,
      "activations/layer5_attention_weight_max": 55.67670822143555,
      "activations/layer5_attention_weight_min": -65.04859924316406,
      "activations/layer6_attention_weight_max": 45.62626647949219,
      "activations/layer6_attention_weight_min": -49.61003875732422,
      "activations/layer7_attention_weight_max": 92.14273071289062,
      "activations/layer7_attention_weight_min": -96.11534881591797,
      "activations/layer8_attention_weight_max": 40.3976936340332,
      "activations/layer8_attention_weight_min": -42.58953094482422,
      "activations/layer9_attention_weight_max": 32.16669464111328,
      "activations/layer9_attention_weight_min": -34.19547653198242,
      "epoch": 12.98,
      "learning_rate": 6.698371212121212e-05,
      "loss": 2.7742,
      "step": 223350
    },
    {
      "activations/layer0_attention_weight_max": 15.991909980773926,
      "activations/layer0_attention_weight_min": -11.442771911621094,
      "activations/layer10_attention_weight_max": 32.158058166503906,
      "activations/layer10_attention_weight_min": -33.410091400146484,
      "activations/layer11_attention_weight_max": 32.86182403564453,
      "activations/layer11_attention_weight_min": -33.50550842285156,
      "activations/layer12_attention_weight_max": 26.806476593017578,
      "activations/layer12_attention_weight_min": -24.659324645996094,
      "activations/layer13_attention_weight_max": 41.059814453125,
      "activations/layer13_attention_weight_min": -33.672767639160156,
      "activations/layer14_attention_weight_max": 54.114662170410156,
      "activations/layer14_attention_weight_min": -38.42733383178711,
      "activations/layer15_attention_weight_max": 39.98965835571289,
      "activations/layer15_attention_weight_min": -31.705123901367188,
      "activations/layer16_attention_weight_max": 32.84801483154297,
      "activations/layer16_attention_weight_min": -27.446399688720703,
      "activations/layer17_attention_weight_max": 34.714332580566406,
      "activations/layer17_attention_weight_min": -25.308095932006836,
      "activations/layer18_attention_weight_max": 37.609519958496094,
      "activations/layer18_attention_weight_min": -24.930294036865234,
      "activations/layer19_attention_weight_max": 39.13513946533203,
      "activations/layer19_attention_weight_min": -32.55950927734375,
      "activations/layer1_attention_weight_max": 17.140398025512695,
      "activations/layer1_attention_weight_min": -15.055145263671875,
      "activations/layer20_attention_weight_max": 33.27914047241211,
      "activations/layer20_attention_weight_min": -25.873722076416016,
      "activations/layer21_attention_weight_max": 31.739004135131836,
      "activations/layer21_attention_weight_min": -25.426626205444336,
      "activations/layer22_attention_weight_max": 53.6918830871582,
      "activations/layer22_attention_weight_min": -33.616207122802734,
      "activations/layer23_attention_weight_max": 36.75635528564453,
      "activations/layer23_attention_weight_min": -25.215087890625,
      "activations/layer2_attention_weight_max": 31.62837791442871,
      "activations/layer2_attention_weight_min": -32.11661148071289,
      "activations/layer3_attention_weight_max": 94.52014923095703,
      "activations/layer3_attention_weight_min": -101.399169921875,
      "activations/layer4_attention_weight_max": 109.08056640625,
      "activations/layer4_attention_weight_min": -109.38021087646484,
      "activations/layer5_attention_weight_max": 53.88338851928711,
      "activations/layer5_attention_weight_min": -64.49080657958984,
      "activations/layer6_attention_weight_max": 44.58660125732422,
      "activations/layer6_attention_weight_min": -47.70270919799805,
      "activations/layer7_attention_weight_max": 91.7103042602539,
      "activations/layer7_attention_weight_min": -89.62222290039062,
      "activations/layer8_attention_weight_max": 41.26185607910156,
      "activations/layer8_attention_weight_min": -44.63539123535156,
      "activations/layer9_attention_weight_max": 33.63691711425781,
      "activations/layer9_attention_weight_min": -36.01580047607422,
      "epoch": 12.98,
      "learning_rate": 6.696477272727273e-05,
      "loss": 2.7855,
      "step": 223400
    },
    {
      "activations/layer0_attention_weight_max": 15.774224281311035,
      "activations/layer0_attention_weight_min": -11.376425743103027,
      "activations/layer10_attention_weight_max": 31.339513778686523,
      "activations/layer10_attention_weight_min": -33.06058120727539,
      "activations/layer11_attention_weight_max": 31.083412170410156,
      "activations/layer11_attention_weight_min": -31.484859466552734,
      "activations/layer12_attention_weight_max": 23.357694625854492,
      "activations/layer12_attention_weight_min": -24.92137336730957,
      "activations/layer13_attention_weight_max": 37.049530029296875,
      "activations/layer13_attention_weight_min": -32.71909713745117,
      "activations/layer14_attention_weight_max": 40.095603942871094,
      "activations/layer14_attention_weight_min": -37.802120208740234,
      "activations/layer15_attention_weight_max": 35.29405975341797,
      "activations/layer15_attention_weight_min": -32.11825942993164,
      "activations/layer16_attention_weight_max": 31.82732391357422,
      "activations/layer16_attention_weight_min": -27.470373153686523,
      "activations/layer17_attention_weight_max": 30.88296890258789,
      "activations/layer17_attention_weight_min": -27.417993545532227,
      "activations/layer18_attention_weight_max": 34.1435661315918,
      "activations/layer18_attention_weight_min": -23.301664352416992,
      "activations/layer19_attention_weight_max": 37.76603317260742,
      "activations/layer19_attention_weight_min": -32.86225891113281,
      "activations/layer1_attention_weight_max": 15.904825210571289,
      "activations/layer1_attention_weight_min": -14.204532623291016,
      "activations/layer20_attention_weight_max": 33.936641693115234,
      "activations/layer20_attention_weight_min": -26.047412872314453,
      "activations/layer21_attention_weight_max": 31.941587448120117,
      "activations/layer21_attention_weight_min": -25.316179275512695,
      "activations/layer22_attention_weight_max": 41.258445739746094,
      "activations/layer22_attention_weight_min": -30.180051803588867,
      "activations/layer23_attention_weight_max": 37.84170150756836,
      "activations/layer23_attention_weight_min": -24.288963317871094,
      "activations/layer2_attention_weight_max": 31.6463623046875,
      "activations/layer2_attention_weight_min": -32.504905700683594,
      "activations/layer3_attention_weight_max": 91.38878631591797,
      "activations/layer3_attention_weight_min": -97.63227081298828,
      "activations/layer4_attention_weight_max": 107.61891174316406,
      "activations/layer4_attention_weight_min": -107.340576171875,
      "activations/layer5_attention_weight_max": 53.29724884033203,
      "activations/layer5_attention_weight_min": -62.173091888427734,
      "activations/layer6_attention_weight_max": 42.24311828613281,
      "activations/layer6_attention_weight_min": -46.79560089111328,
      "activations/layer7_attention_weight_max": 92.3135757446289,
      "activations/layer7_attention_weight_min": -94.29560089111328,
      "activations/layer8_attention_weight_max": 41.23103332519531,
      "activations/layer8_attention_weight_min": -41.54423904418945,
      "activations/layer9_attention_weight_max": 33.7020378112793,
      "activations/layer9_attention_weight_min": -33.149330139160156,
      "epoch": 12.98,
      "learning_rate": 6.694583333333333e-05,
      "loss": 2.7823,
      "step": 223450
    },
    {
      "activations/layer0_attention_weight_max": 15.549495697021484,
      "activations/layer0_attention_weight_min": -12.503805160522461,
      "activations/layer10_attention_weight_max": 33.0147590637207,
      "activations/layer10_attention_weight_min": -33.151824951171875,
      "activations/layer11_attention_weight_max": 34.25505065917969,
      "activations/layer11_attention_weight_min": -33.735450744628906,
      "activations/layer12_attention_weight_max": 28.461774826049805,
      "activations/layer12_attention_weight_min": -26.64219093322754,
      "activations/layer13_attention_weight_max": 43.70549011230469,
      "activations/layer13_attention_weight_min": -35.262901306152344,
      "activations/layer14_attention_weight_max": 48.73820114135742,
      "activations/layer14_attention_weight_min": -39.57280349731445,
      "activations/layer15_attention_weight_max": 41.568511962890625,
      "activations/layer15_attention_weight_min": -36.40363693237305,
      "activations/layer16_attention_weight_max": 31.949111938476562,
      "activations/layer16_attention_weight_min": -30.87725067138672,
      "activations/layer17_attention_weight_max": 33.1611213684082,
      "activations/layer17_attention_weight_min": -26.620126724243164,
      "activations/layer18_attention_weight_max": 30.837739944458008,
      "activations/layer18_attention_weight_min": -25.688373565673828,
      "activations/layer19_attention_weight_max": 34.59548568725586,
      "activations/layer19_attention_weight_min": -30.088224411010742,
      "activations/layer1_attention_weight_max": 16.581300735473633,
      "activations/layer1_attention_weight_min": -15.613035202026367,
      "activations/layer20_attention_weight_max": 29.28826141357422,
      "activations/layer20_attention_weight_min": -24.144512176513672,
      "activations/layer21_attention_weight_max": 29.786888122558594,
      "activations/layer21_attention_weight_min": -25.594560623168945,
      "activations/layer22_attention_weight_max": 45.6640739440918,
      "activations/layer22_attention_weight_min": -31.59768295288086,
      "activations/layer23_attention_weight_max": 35.121891021728516,
      "activations/layer23_attention_weight_min": -25.45122528076172,
      "activations/layer2_attention_weight_max": 34.526466369628906,
      "activations/layer2_attention_weight_min": -33.6832275390625,
      "activations/layer3_attention_weight_max": 99.65666198730469,
      "activations/layer3_attention_weight_min": -101.22541809082031,
      "activations/layer4_attention_weight_max": 110.67134857177734,
      "activations/layer4_attention_weight_min": -111.7239990234375,
      "activations/layer5_attention_weight_max": 52.006080627441406,
      "activations/layer5_attention_weight_min": -63.904624938964844,
      "activations/layer6_attention_weight_max": 44.30398178100586,
      "activations/layer6_attention_weight_min": -48.91117858886719,
      "activations/layer7_attention_weight_max": 93.50277709960938,
      "activations/layer7_attention_weight_min": -96.8014907836914,
      "activations/layer8_attention_weight_max": 42.94712829589844,
      "activations/layer8_attention_weight_min": -44.02616882324219,
      "activations/layer9_attention_weight_max": 33.81694030761719,
      "activations/layer9_attention_weight_min": -33.29853820800781,
      "epoch": 12.99,
      "learning_rate": 6.692689393939393e-05,
      "loss": 2.76,
      "step": 223500
    },
    {
      "activations/layer0_attention_weight_max": 15.751408576965332,
      "activations/layer0_attention_weight_min": -11.465241432189941,
      "activations/layer10_attention_weight_max": 36.29903793334961,
      "activations/layer10_attention_weight_min": -32.570831298828125,
      "activations/layer11_attention_weight_max": 36.199214935302734,
      "activations/layer11_attention_weight_min": -32.170989990234375,
      "activations/layer12_attention_weight_max": 28.075071334838867,
      "activations/layer12_attention_weight_min": -25.083648681640625,
      "activations/layer13_attention_weight_max": 41.579341888427734,
      "activations/layer13_attention_weight_min": -34.13813018798828,
      "activations/layer14_attention_weight_max": 47.33539962768555,
      "activations/layer14_attention_weight_min": -36.23421096801758,
      "activations/layer15_attention_weight_max": 41.58452224731445,
      "activations/layer15_attention_weight_min": -33.5524787902832,
      "activations/layer16_attention_weight_max": 35.533687591552734,
      "activations/layer16_attention_weight_min": -28.95356559753418,
      "activations/layer17_attention_weight_max": 34.97653579711914,
      "activations/layer17_attention_weight_min": -27.78420066833496,
      "activations/layer18_attention_weight_max": 34.460487365722656,
      "activations/layer18_attention_weight_min": -27.092510223388672,
      "activations/layer19_attention_weight_max": 36.153926849365234,
      "activations/layer19_attention_weight_min": -31.96366310119629,
      "activations/layer1_attention_weight_max": 17.071945190429688,
      "activations/layer1_attention_weight_min": -15.72769546508789,
      "activations/layer20_attention_weight_max": 31.716533660888672,
      "activations/layer20_attention_weight_min": -27.236242294311523,
      "activations/layer21_attention_weight_max": 28.939279556274414,
      "activations/layer21_attention_weight_min": -26.69930648803711,
      "activations/layer22_attention_weight_max": 45.77128982543945,
      "activations/layer22_attention_weight_min": -33.15235900878906,
      "activations/layer23_attention_weight_max": 39.43693923950195,
      "activations/layer23_attention_weight_min": -26.79473876953125,
      "activations/layer2_attention_weight_max": 33.524864196777344,
      "activations/layer2_attention_weight_min": -32.33311462402344,
      "activations/layer3_attention_weight_max": 100.43255615234375,
      "activations/layer3_attention_weight_min": -99.64897918701172,
      "activations/layer4_attention_weight_max": 114.20208740234375,
      "activations/layer4_attention_weight_min": -113.09708404541016,
      "activations/layer5_attention_weight_max": 53.699241638183594,
      "activations/layer5_attention_weight_min": -66.74567413330078,
      "activations/layer6_attention_weight_max": 46.02198791503906,
      "activations/layer6_attention_weight_min": -47.17146682739258,
      "activations/layer7_attention_weight_max": 92.86215209960938,
      "activations/layer7_attention_weight_min": -91.98238372802734,
      "activations/layer8_attention_weight_max": 45.268775939941406,
      "activations/layer8_attention_weight_min": -42.83187484741211,
      "activations/layer9_attention_weight_max": 36.400882720947266,
      "activations/layer9_attention_weight_min": -33.242862701416016,
      "epoch": 12.99,
      "learning_rate": 6.690795454545454e-05,
      "loss": 2.7763,
      "step": 223550
    },
    {
      "activations/layer0_attention_weight_max": 14.817171096801758,
      "activations/layer0_attention_weight_min": -11.747152328491211,
      "activations/layer10_attention_weight_max": 32.341270446777344,
      "activations/layer10_attention_weight_min": -36.452598571777344,
      "activations/layer11_attention_weight_max": 31.530345916748047,
      "activations/layer11_attention_weight_min": -34.48939514160156,
      "activations/layer12_attention_weight_max": 32.693016052246094,
      "activations/layer12_attention_weight_min": -25.635038375854492,
      "activations/layer13_attention_weight_max": 40.60966491699219,
      "activations/layer13_attention_weight_min": -34.452449798583984,
      "activations/layer14_attention_weight_max": 41.51823043823242,
      "activations/layer14_attention_weight_min": -37.56412887573242,
      "activations/layer15_attention_weight_max": 39.937744140625,
      "activations/layer15_attention_weight_min": -33.53104782104492,
      "activations/layer16_attention_weight_max": 29.56855010986328,
      "activations/layer16_attention_weight_min": -27.544979095458984,
      "activations/layer17_attention_weight_max": 31.363460540771484,
      "activations/layer17_attention_weight_min": -27.04099464416504,
      "activations/layer18_attention_weight_max": 32.21728515625,
      "activations/layer18_attention_weight_min": -23.33922576904297,
      "activations/layer19_attention_weight_max": 38.46458435058594,
      "activations/layer19_attention_weight_min": -30.798477172851562,
      "activations/layer1_attention_weight_max": 17.04974937438965,
      "activations/layer1_attention_weight_min": -15.68027400970459,
      "activations/layer20_attention_weight_max": 29.58474349975586,
      "activations/layer20_attention_weight_min": -27.934940338134766,
      "activations/layer21_attention_weight_max": 29.02341079711914,
      "activations/layer21_attention_weight_min": -23.783489227294922,
      "activations/layer22_attention_weight_max": 47.068695068359375,
      "activations/layer22_attention_weight_min": -31.06513786315918,
      "activations/layer23_attention_weight_max": 35.77650451660156,
      "activations/layer23_attention_weight_min": -25.708946228027344,
      "activations/layer2_attention_weight_max": 33.01654815673828,
      "activations/layer2_attention_weight_min": -32.11189270019531,
      "activations/layer3_attention_weight_max": 96.62931823730469,
      "activations/layer3_attention_weight_min": -97.04374694824219,
      "activations/layer4_attention_weight_max": 112.15886688232422,
      "activations/layer4_attention_weight_min": -111.2649154663086,
      "activations/layer5_attention_weight_max": 54.095726013183594,
      "activations/layer5_attention_weight_min": -61.040802001953125,
      "activations/layer6_attention_weight_max": 42.61399841308594,
      "activations/layer6_attention_weight_min": -46.02244186401367,
      "activations/layer7_attention_weight_max": 89.06737518310547,
      "activations/layer7_attention_weight_min": -87.82464599609375,
      "activations/layer8_attention_weight_max": 39.235050201416016,
      "activations/layer8_attention_weight_min": -44.77205276489258,
      "activations/layer9_attention_weight_max": 31.481992721557617,
      "activations/layer9_attention_weight_min": -35.02317810058594,
      "epoch": 12.99,
      "learning_rate": 6.688901515151515e-05,
      "loss": 2.7756,
      "step": 223600
    },
    {
      "activations/layer0_attention_weight_max": 15.920928955078125,
      "activations/layer0_attention_weight_min": -11.921844482421875,
      "activations/layer10_attention_weight_max": 33.23366928100586,
      "activations/layer10_attention_weight_min": -31.09507942199707,
      "activations/layer11_attention_weight_max": 31.887035369873047,
      "activations/layer11_attention_weight_min": -31.785518646240234,
      "activations/layer12_attention_weight_max": 23.42246437072754,
      "activations/layer12_attention_weight_min": -25.695772171020508,
      "activations/layer13_attention_weight_max": 35.506004333496094,
      "activations/layer13_attention_weight_min": -32.33917999267578,
      "activations/layer14_attention_weight_max": 39.88985061645508,
      "activations/layer14_attention_weight_min": -34.727413177490234,
      "activations/layer15_attention_weight_max": 33.1422004699707,
      "activations/layer15_attention_weight_min": -31.637670516967773,
      "activations/layer16_attention_weight_max": 30.648448944091797,
      "activations/layer16_attention_weight_min": -26.335023880004883,
      "activations/layer17_attention_weight_max": 29.906423568725586,
      "activations/layer17_attention_weight_min": -26.187108993530273,
      "activations/layer18_attention_weight_max": 33.15481948852539,
      "activations/layer18_attention_weight_min": -23.54157066345215,
      "activations/layer19_attention_weight_max": 32.81235122680664,
      "activations/layer19_attention_weight_min": -30.78937339782715,
      "activations/layer1_attention_weight_max": 17.553884506225586,
      "activations/layer1_attention_weight_min": -16.259431838989258,
      "activations/layer20_attention_weight_max": 28.18260955810547,
      "activations/layer20_attention_weight_min": -26.976394653320312,
      "activations/layer21_attention_weight_max": 28.795690536499023,
      "activations/layer21_attention_weight_min": -25.683263778686523,
      "activations/layer22_attention_weight_max": 41.63665771484375,
      "activations/layer22_attention_weight_min": -32.0435676574707,
      "activations/layer23_attention_weight_max": 34.276145935058594,
      "activations/layer23_attention_weight_min": -24.725223541259766,
      "activations/layer2_attention_weight_max": 32.876808166503906,
      "activations/layer2_attention_weight_min": -31.384166717529297,
      "activations/layer3_attention_weight_max": 96.5168685913086,
      "activations/layer3_attention_weight_min": -94.86791229248047,
      "activations/layer4_attention_weight_max": 106.59134674072266,
      "activations/layer4_attention_weight_min": -113.81898498535156,
      "activations/layer5_attention_weight_max": 52.825347900390625,
      "activations/layer5_attention_weight_min": -61.97028350830078,
      "activations/layer6_attention_weight_max": 44.085140228271484,
      "activations/layer6_attention_weight_min": -47.731449127197266,
      "activations/layer7_attention_weight_max": 89.35777282714844,
      "activations/layer7_attention_weight_min": -92.81260681152344,
      "activations/layer8_attention_weight_max": 43.290523529052734,
      "activations/layer8_attention_weight_min": -41.02296829223633,
      "activations/layer9_attention_weight_max": 34.00636291503906,
      "activations/layer9_attention_weight_min": -33.84206771850586,
      "epoch": 13.0,
      "learning_rate": 6.687007575757575e-05,
      "loss": 2.7536,
      "step": 223650
    },
    {
      "activations/layer0_attention_weight_max": 16.264759063720703,
      "activations/layer0_attention_weight_min": -12.311271667480469,
      "activations/layer10_attention_weight_max": 36.62408447265625,
      "activations/layer10_attention_weight_min": -34.496559143066406,
      "activations/layer11_attention_weight_max": 35.623077392578125,
      "activations/layer11_attention_weight_min": -34.01683044433594,
      "activations/layer12_attention_weight_max": 26.82036781311035,
      "activations/layer12_attention_weight_min": -25.444900512695312,
      "activations/layer13_attention_weight_max": 42.64860916137695,
      "activations/layer13_attention_weight_min": -36.69835662841797,
      "activations/layer14_attention_weight_max": 49.32057571411133,
      "activations/layer14_attention_weight_min": -39.346351623535156,
      "activations/layer15_attention_weight_max": 43.53096008300781,
      "activations/layer15_attention_weight_min": -34.74775695800781,
      "activations/layer16_attention_weight_max": 34.17422866821289,
      "activations/layer16_attention_weight_min": -28.337677001953125,
      "activations/layer17_attention_weight_max": 35.89836502075195,
      "activations/layer17_attention_weight_min": -27.55965805053711,
      "activations/layer18_attention_weight_max": 33.56667709350586,
      "activations/layer18_attention_weight_min": -25.303279876708984,
      "activations/layer19_attention_weight_max": 38.04120635986328,
      "activations/layer19_attention_weight_min": -33.27519989013672,
      "activations/layer1_attention_weight_max": 16.49323844909668,
      "activations/layer1_attention_weight_min": -14.563559532165527,
      "activations/layer20_attention_weight_max": 33.68431854248047,
      "activations/layer20_attention_weight_min": -27.063377380371094,
      "activations/layer21_attention_weight_max": 36.79615783691406,
      "activations/layer21_attention_weight_min": -25.21797752380371,
      "activations/layer22_attention_weight_max": 48.93568420410156,
      "activations/layer22_attention_weight_min": -31.772260665893555,
      "activations/layer23_attention_weight_max": 36.576629638671875,
      "activations/layer23_attention_weight_min": -25.223812103271484,
      "activations/layer2_attention_weight_max": 33.767791748046875,
      "activations/layer2_attention_weight_min": -35.684364318847656,
      "activations/layer3_attention_weight_max": 97.97882080078125,
      "activations/layer3_attention_weight_min": -101.96648406982422,
      "activations/layer4_attention_weight_max": 109.41948699951172,
      "activations/layer4_attention_weight_min": -118.52296447753906,
      "activations/layer5_attention_weight_max": 51.855613708496094,
      "activations/layer5_attention_weight_min": -62.61753463745117,
      "activations/layer6_attention_weight_max": 47.77564239501953,
      "activations/layer6_attention_weight_min": -47.14888000488281,
      "activations/layer7_attention_weight_max": 92.6174545288086,
      "activations/layer7_attention_weight_min": -93.62107849121094,
      "activations/layer8_attention_weight_max": 43.29519271850586,
      "activations/layer8_attention_weight_min": -47.51633071899414,
      "activations/layer9_attention_weight_max": 37.71290969848633,
      "activations/layer9_attention_weight_min": -37.63779067993164,
      "epoch": 13.0,
      "learning_rate": 6.685113636363635e-05,
      "loss": 2.7786,
      "step": 223700
    },
    {
      "activations/layer0_attention_weight_max": 15.938642501831055,
      "activations/layer0_attention_weight_min": -11.590826034545898,
      "activations/layer10_attention_weight_max": 36.36127471923828,
      "activations/layer10_attention_weight_min": -35.43950271606445,
      "activations/layer11_attention_weight_max": 38.4510383605957,
      "activations/layer11_attention_weight_min": -33.952945709228516,
      "activations/layer12_attention_weight_max": 26.667415618896484,
      "activations/layer12_attention_weight_min": -26.23322868347168,
      "activations/layer13_attention_weight_max": 40.784881591796875,
      "activations/layer13_attention_weight_min": -35.54388427734375,
      "activations/layer14_attention_weight_max": 45.55962371826172,
      "activations/layer14_attention_weight_min": -38.448429107666016,
      "activations/layer15_attention_weight_max": 38.378170013427734,
      "activations/layer15_attention_weight_min": -33.441036224365234,
      "activations/layer16_attention_weight_max": 33.56139373779297,
      "activations/layer16_attention_weight_min": -28.87714958190918,
      "activations/layer17_attention_weight_max": 31.52176284790039,
      "activations/layer17_attention_weight_min": -25.807430267333984,
      "activations/layer18_attention_weight_max": 32.139984130859375,
      "activations/layer18_attention_weight_min": -23.077556610107422,
      "activations/layer19_attention_weight_max": 43.700469970703125,
      "activations/layer19_attention_weight_min": -30.193017959594727,
      "activations/layer1_attention_weight_max": 18.921485900878906,
      "activations/layer1_attention_weight_min": -15.572531700134277,
      "activations/layer20_attention_weight_max": 39.31359100341797,
      "activations/layer20_attention_weight_min": -25.2096004486084,
      "activations/layer21_attention_weight_max": 35.64064407348633,
      "activations/layer21_attention_weight_min": -23.323135375976562,
      "activations/layer22_attention_weight_max": 46.42488479614258,
      "activations/layer22_attention_weight_min": -30.94405174255371,
      "activations/layer23_attention_weight_max": 38.99235534667969,
      "activations/layer23_attention_weight_min": -24.13724136352539,
      "activations/layer2_attention_weight_max": 34.16849136352539,
      "activations/layer2_attention_weight_min": -33.56099319458008,
      "activations/layer3_attention_weight_max": 95.61283111572266,
      "activations/layer3_attention_weight_min": -101.11238098144531,
      "activations/layer4_attention_weight_max": 110.46858978271484,
      "activations/layer4_attention_weight_min": -113.82735443115234,
      "activations/layer5_attention_weight_max": 50.973751068115234,
      "activations/layer5_attention_weight_min": -62.93128967285156,
      "activations/layer6_attention_weight_max": 44.2700080871582,
      "activations/layer6_attention_weight_min": -48.53645706176758,
      "activations/layer7_attention_weight_max": 90.14208984375,
      "activations/layer7_attention_weight_min": -100.9441909790039,
      "activations/layer8_attention_weight_max": 41.44803237915039,
      "activations/layer8_attention_weight_min": -42.335838317871094,
      "activations/layer9_attention_weight_max": 32.58857727050781,
      "activations/layer9_attention_weight_min": -34.526451110839844,
      "epoch": 13.0,
      "learning_rate": 6.683219696969695e-05,
      "loss": 2.8155,
      "step": 223750
    },
    {
      "activations/layer0_attention_weight_max": 15.697074890136719,
      "activations/layer0_attention_weight_min": -11.757243156433105,
      "activations/layer10_attention_weight_max": 32.44193649291992,
      "activations/layer10_attention_weight_min": -35.012577056884766,
      "activations/layer11_attention_weight_max": 31.890478134155273,
      "activations/layer11_attention_weight_min": -35.46412658691406,
      "activations/layer12_attention_weight_max": 26.655445098876953,
      "activations/layer12_attention_weight_min": -27.982051849365234,
      "activations/layer13_attention_weight_max": 43.900386810302734,
      "activations/layer13_attention_weight_min": -36.91688537597656,
      "activations/layer14_attention_weight_max": 50.035186767578125,
      "activations/layer14_attention_weight_min": -41.61151885986328,
      "activations/layer15_attention_weight_max": 39.68974685668945,
      "activations/layer15_attention_weight_min": -36.5838737487793,
      "activations/layer16_attention_weight_max": 35.102352142333984,
      "activations/layer16_attention_weight_min": -29.48416519165039,
      "activations/layer17_attention_weight_max": 32.71187973022461,
      "activations/layer17_attention_weight_min": -27.303186416625977,
      "activations/layer18_attention_weight_max": 33.071746826171875,
      "activations/layer18_attention_weight_min": -25.105144500732422,
      "activations/layer19_attention_weight_max": 38.152000427246094,
      "activations/layer19_attention_weight_min": -31.7724552154541,
      "activations/layer1_attention_weight_max": 17.176952362060547,
      "activations/layer1_attention_weight_min": -14.343104362487793,
      "activations/layer20_attention_weight_max": 33.97232437133789,
      "activations/layer20_attention_weight_min": -28.126724243164062,
      "activations/layer21_attention_weight_max": 33.85316848754883,
      "activations/layer21_attention_weight_min": -25.409019470214844,
      "activations/layer22_attention_weight_max": 51.33243179321289,
      "activations/layer22_attention_weight_min": -30.654998779296875,
      "activations/layer23_attention_weight_max": 41.74268341064453,
      "activations/layer23_attention_weight_min": -24.715312957763672,
      "activations/layer2_attention_weight_max": 33.55660629272461,
      "activations/layer2_attention_weight_min": -33.05796813964844,
      "activations/layer3_attention_weight_max": 97.00321960449219,
      "activations/layer3_attention_weight_min": -98.96516418457031,
      "activations/layer4_attention_weight_max": 113.08123016357422,
      "activations/layer4_attention_weight_min": -112.169921875,
      "activations/layer5_attention_weight_max": 54.871742248535156,
      "activations/layer5_attention_weight_min": -62.799720764160156,
      "activations/layer6_attention_weight_max": 47.86253356933594,
      "activations/layer6_attention_weight_min": -46.128536224365234,
      "activations/layer7_attention_weight_max": 95.89835357666016,
      "activations/layer7_attention_weight_min": -94.81591796875,
      "activations/layer8_attention_weight_max": 40.09440612792969,
      "activations/layer8_attention_weight_min": -40.92238998413086,
      "activations/layer9_attention_weight_max": 33.28606414794922,
      "activations/layer9_attention_weight_min": -33.893798828125,
      "epoch": 13.0,
      "learning_rate": 6.681325757575757e-05,
      "loss": 2.7665,
      "step": 223800
    },
    {
      "activations/layer0_attention_weight_max": 15.194571495056152,
      "activations/layer0_attention_weight_min": -12.173355102539062,
      "activations/layer10_attention_weight_max": 31.840726852416992,
      "activations/layer10_attention_weight_min": -33.473876953125,
      "activations/layer11_attention_weight_max": 32.942970275878906,
      "activations/layer11_attention_weight_min": -32.720298767089844,
      "activations/layer12_attention_weight_max": 25.66965103149414,
      "activations/layer12_attention_weight_min": -24.337072372436523,
      "activations/layer13_attention_weight_max": 38.215850830078125,
      "activations/layer13_attention_weight_min": -33.68669509887695,
      "activations/layer14_attention_weight_max": 42.160926818847656,
      "activations/layer14_attention_weight_min": -36.309349060058594,
      "activations/layer15_attention_weight_max": 36.82872772216797,
      "activations/layer15_attention_weight_min": -33.206729888916016,
      "activations/layer16_attention_weight_max": 30.393774032592773,
      "activations/layer16_attention_weight_min": -27.07908821105957,
      "activations/layer17_attention_weight_max": 31.51998519897461,
      "activations/layer17_attention_weight_min": -26.503110885620117,
      "activations/layer18_attention_weight_max": 31.8076229095459,
      "activations/layer18_attention_weight_min": -27.285202026367188,
      "activations/layer19_attention_weight_max": 35.04670333862305,
      "activations/layer19_attention_weight_min": -30.684951782226562,
      "activations/layer1_attention_weight_max": 15.54312515258789,
      "activations/layer1_attention_weight_min": -15.262819290161133,
      "activations/layer20_attention_weight_max": 31.97930908203125,
      "activations/layer20_attention_weight_min": -25.66225242614746,
      "activations/layer21_attention_weight_max": 32.83835220336914,
      "activations/layer21_attention_weight_min": -23.70441246032715,
      "activations/layer22_attention_weight_max": 44.530967712402344,
      "activations/layer22_attention_weight_min": -31.2932186126709,
      "activations/layer23_attention_weight_max": 34.02861785888672,
      "activations/layer23_attention_weight_min": -26.39984703063965,
      "activations/layer2_attention_weight_max": 33.490211486816406,
      "activations/layer2_attention_weight_min": -33.15104293823242,
      "activations/layer3_attention_weight_max": 96.3030776977539,
      "activations/layer3_attention_weight_min": -96.32077026367188,
      "activations/layer4_attention_weight_max": 110.76656341552734,
      "activations/layer4_attention_weight_min": -113.68155670166016,
      "activations/layer5_attention_weight_max": 52.237876892089844,
      "activations/layer5_attention_weight_min": -59.82778549194336,
      "activations/layer6_attention_weight_max": 44.2731819152832,
      "activations/layer6_attention_weight_min": -49.6860237121582,
      "activations/layer7_attention_weight_max": 88.40550231933594,
      "activations/layer7_attention_weight_min": -94.14002227783203,
      "activations/layer8_attention_weight_max": 39.51947021484375,
      "activations/layer8_attention_weight_min": -43.65501022338867,
      "activations/layer9_attention_weight_max": 31.328937530517578,
      "activations/layer9_attention_weight_min": -33.2633171081543,
      "epoch": 13.01,
      "learning_rate": 6.679431818181817e-05,
      "loss": 2.7634,
      "step": 223850
    },
    {
      "activations/layer0_attention_weight_max": 16.674549102783203,
      "activations/layer0_attention_weight_min": -12.051552772521973,
      "activations/layer10_attention_weight_max": 32.232627868652344,
      "activations/layer10_attention_weight_min": -34.36786651611328,
      "activations/layer11_attention_weight_max": 30.490127563476562,
      "activations/layer11_attention_weight_min": -32.24817657470703,
      "activations/layer12_attention_weight_max": 25.107547760009766,
      "activations/layer12_attention_weight_min": -26.963401794433594,
      "activations/layer13_attention_weight_max": 37.56584548950195,
      "activations/layer13_attention_weight_min": -33.9726448059082,
      "activations/layer14_attention_weight_max": 41.27133560180664,
      "activations/layer14_attention_weight_min": -38.82697677612305,
      "activations/layer15_attention_weight_max": 37.900360107421875,
      "activations/layer15_attention_weight_min": -32.41091537475586,
      "activations/layer16_attention_weight_max": 31.98995018005371,
      "activations/layer16_attention_weight_min": -27.843671798706055,
      "activations/layer17_attention_weight_max": 31.366785049438477,
      "activations/layer17_attention_weight_min": -26.805652618408203,
      "activations/layer18_attention_weight_max": 30.826566696166992,
      "activations/layer18_attention_weight_min": -25.69403648376465,
      "activations/layer19_attention_weight_max": 33.660980224609375,
      "activations/layer19_attention_weight_min": -30.834096908569336,
      "activations/layer1_attention_weight_max": 17.88604164123535,
      "activations/layer1_attention_weight_min": -15.166011810302734,
      "activations/layer20_attention_weight_max": 30.09257698059082,
      "activations/layer20_attention_weight_min": -26.10127830505371,
      "activations/layer21_attention_weight_max": 30.066753387451172,
      "activations/layer21_attention_weight_min": -24.710222244262695,
      "activations/layer22_attention_weight_max": 45.48095703125,
      "activations/layer22_attention_weight_min": -30.35657501220703,
      "activations/layer23_attention_weight_max": 34.37327575683594,
      "activations/layer23_attention_weight_min": -24.514545440673828,
      "activations/layer2_attention_weight_max": 33.461727142333984,
      "activations/layer2_attention_weight_min": -33.05278396606445,
      "activations/layer3_attention_weight_max": 95.34599304199219,
      "activations/layer3_attention_weight_min": -96.28639221191406,
      "activations/layer4_attention_weight_max": 108.59976959228516,
      "activations/layer4_attention_weight_min": -109.5703125,
      "activations/layer5_attention_weight_max": 52.103187561035156,
      "activations/layer5_attention_weight_min": -58.437744140625,
      "activations/layer6_attention_weight_max": 42.60835647583008,
      "activations/layer6_attention_weight_min": -47.09602737426758,
      "activations/layer7_attention_weight_max": 87.37738037109375,
      "activations/layer7_attention_weight_min": -88.70027160644531,
      "activations/layer8_attention_weight_max": 37.46573257446289,
      "activations/layer8_attention_weight_min": -41.76884841918945,
      "activations/layer9_attention_weight_max": 31.31507682800293,
      "activations/layer9_attention_weight_min": -32.04249572753906,
      "epoch": 13.01,
      "learning_rate": 6.677537878787879e-05,
      "loss": 2.7393,
      "step": 223900
    },
    {
      "activations/layer0_attention_weight_max": 16.416475296020508,
      "activations/layer0_attention_weight_min": -12.969694137573242,
      "activations/layer10_attention_weight_max": 32.1127815246582,
      "activations/layer10_attention_weight_min": -33.215492248535156,
      "activations/layer11_attention_weight_max": 31.767133712768555,
      "activations/layer11_attention_weight_min": -32.33156204223633,
      "activations/layer12_attention_weight_max": 24.85065269470215,
      "activations/layer12_attention_weight_min": -25.49022102355957,
      "activations/layer13_attention_weight_max": 37.66538619995117,
      "activations/layer13_attention_weight_min": -33.49345016479492,
      "activations/layer14_attention_weight_max": 46.807456970214844,
      "activations/layer14_attention_weight_min": -38.46649932861328,
      "activations/layer15_attention_weight_max": 37.19670104980469,
      "activations/layer15_attention_weight_min": -34.94786834716797,
      "activations/layer16_attention_weight_max": 31.62023162841797,
      "activations/layer16_attention_weight_min": -28.79655647277832,
      "activations/layer17_attention_weight_max": 30.188419342041016,
      "activations/layer17_attention_weight_min": -26.960662841796875,
      "activations/layer18_attention_weight_max": 30.07249641418457,
      "activations/layer18_attention_weight_min": -22.75899314880371,
      "activations/layer19_attention_weight_max": 41.19115447998047,
      "activations/layer19_attention_weight_min": -33.757080078125,
      "activations/layer1_attention_weight_max": 16.812585830688477,
      "activations/layer1_attention_weight_min": -15.244702339172363,
      "activations/layer20_attention_weight_max": 33.03068542480469,
      "activations/layer20_attention_weight_min": -27.320728302001953,
      "activations/layer21_attention_weight_max": 33.82124710083008,
      "activations/layer21_attention_weight_min": -26.090219497680664,
      "activations/layer22_attention_weight_max": 61.691131591796875,
      "activations/layer22_attention_weight_min": -34.32301330566406,
      "activations/layer23_attention_weight_max": 42.5416374206543,
      "activations/layer23_attention_weight_min": -26.091001510620117,
      "activations/layer2_attention_weight_max": 32.477108001708984,
      "activations/layer2_attention_weight_min": -31.40639305114746,
      "activations/layer3_attention_weight_max": 97.30140686035156,
      "activations/layer3_attention_weight_min": -98.08616638183594,
      "activations/layer4_attention_weight_max": 110.7501220703125,
      "activations/layer4_attention_weight_min": -108.7056884765625,
      "activations/layer5_attention_weight_max": 52.50885009765625,
      "activations/layer5_attention_weight_min": -63.45162582397461,
      "activations/layer6_attention_weight_max": 46.66572952270508,
      "activations/layer6_attention_weight_min": -48.60831832885742,
      "activations/layer7_attention_weight_max": 89.47396850585938,
      "activations/layer7_attention_weight_min": -90.58528137207031,
      "activations/layer8_attention_weight_max": 39.828025817871094,
      "activations/layer8_attention_weight_min": -42.66792297363281,
      "activations/layer9_attention_weight_max": 32.2547721862793,
      "activations/layer9_attention_weight_min": -31.98407554626465,
      "epoch": 13.01,
      "learning_rate": 6.675643939393939e-05,
      "loss": 2.7678,
      "step": 223950
    },
    {
      "activations/layer0_attention_weight_max": 16.09180450439453,
      "activations/layer0_attention_weight_min": -11.85715389251709,
      "activations/layer10_attention_weight_max": 31.425329208374023,
      "activations/layer10_attention_weight_min": -31.532363891601562,
      "activations/layer11_attention_weight_max": 30.70107650756836,
      "activations/layer11_attention_weight_min": -32.4340705871582,
      "activations/layer12_attention_weight_max": 24.20340347290039,
      "activations/layer12_attention_weight_min": -24.67511749267578,
      "activations/layer13_attention_weight_max": 39.70512771606445,
      "activations/layer13_attention_weight_min": -33.99455261230469,
      "activations/layer14_attention_weight_max": 42.39016342163086,
      "activations/layer14_attention_weight_min": -37.482749938964844,
      "activations/layer15_attention_weight_max": 34.66839599609375,
      "activations/layer15_attention_weight_min": -33.594242095947266,
      "activations/layer16_attention_weight_max": 31.266368865966797,
      "activations/layer16_attention_weight_min": -27.011714935302734,
      "activations/layer17_attention_weight_max": 37.30442428588867,
      "activations/layer17_attention_weight_min": -24.411596298217773,
      "activations/layer18_attention_weight_max": 35.99013137817383,
      "activations/layer18_attention_weight_min": -26.425161361694336,
      "activations/layer19_attention_weight_max": 36.24803161621094,
      "activations/layer19_attention_weight_min": -30.58064079284668,
      "activations/layer1_attention_weight_max": 16.367666244506836,
      "activations/layer1_attention_weight_min": -14.95090389251709,
      "activations/layer20_attention_weight_max": 28.603771209716797,
      "activations/layer20_attention_weight_min": -23.776451110839844,
      "activations/layer21_attention_weight_max": 33.98927688598633,
      "activations/layer21_attention_weight_min": -24.11687469482422,
      "activations/layer22_attention_weight_max": 43.35517883300781,
      "activations/layer22_attention_weight_min": -29.976579666137695,
      "activations/layer23_attention_weight_max": 34.51274871826172,
      "activations/layer23_attention_weight_min": -24.28700828552246,
      "activations/layer2_attention_weight_max": 33.389060974121094,
      "activations/layer2_attention_weight_min": -33.48332595825195,
      "activations/layer3_attention_weight_max": 92.0079116821289,
      "activations/layer3_attention_weight_min": -96.0020523071289,
      "activations/layer4_attention_weight_max": 105.97334289550781,
      "activations/layer4_attention_weight_min": -109.88774871826172,
      "activations/layer5_attention_weight_max": 48.70558166503906,
      "activations/layer5_attention_weight_min": -62.22312927246094,
      "activations/layer6_attention_weight_max": 44.062721252441406,
      "activations/layer6_attention_weight_min": -47.572261810302734,
      "activations/layer7_attention_weight_max": 88.49152374267578,
      "activations/layer7_attention_weight_min": -90.54680633544922,
      "activations/layer8_attention_weight_max": 39.249149322509766,
      "activations/layer8_attention_weight_min": -40.603843688964844,
      "activations/layer9_attention_weight_max": 32.00442886352539,
      "activations/layer9_attention_weight_min": -32.788692474365234,
      "epoch": 13.02,
      "learning_rate": 6.67375e-05,
      "loss": 2.7621,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4806,
      "eval_samples_per_second": 506.333,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4806,
      "eval_openwebtext_samples_per_second": 506.333,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9309,
      "eval_wikitext_samples_per_second": 236.159,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_lambada_loss": 2.501953125,
      "eval_lambada_ppl": 12.206311145618837,
      "eval_lambada_runtime": 9.5451,
      "eval_lambada_samples_per_second": 510.103,
      "step": 224000
    },
    {
      "activations/layer0_attention_weight_max": 15.6668062210083,
      "activations/layer0_attention_weight_min": -11.623311042785645,
      "activations/layer10_attention_weight_max": 31.09893798828125,
      "activations/layer10_attention_weight_min": -32.47104263305664,
      "activations/layer11_attention_weight_max": 30.761978149414062,
      "activations/layer11_attention_weight_min": -32.114837646484375,
      "activations/layer12_attention_weight_max": 27.680667877197266,
      "activations/layer12_attention_weight_min": -25.55302619934082,
      "activations/layer13_attention_weight_max": 38.22608184814453,
      "activations/layer13_attention_weight_min": -34.441062927246094,
      "activations/layer14_attention_weight_max": 41.83366012573242,
      "activations/layer14_attention_weight_min": -37.04481506347656,
      "activations/layer15_attention_weight_max": 37.52404022216797,
      "activations/layer15_attention_weight_min": -31.965675354003906,
      "activations/layer16_attention_weight_max": 31.332408905029297,
      "activations/layer16_attention_weight_min": -29.36693000793457,
      "activations/layer17_attention_weight_max": 29.110605239868164,
      "activations/layer17_attention_weight_min": -24.763019561767578,
      "activations/layer18_attention_weight_max": 30.407621383666992,
      "activations/layer18_attention_weight_min": -25.51093101501465,
      "activations/layer19_attention_weight_max": 33.87743377685547,
      "activations/layer19_attention_weight_min": -28.731414794921875,
      "activations/layer1_attention_weight_max": 16.535587310791016,
      "activations/layer1_attention_weight_min": -13.573419570922852,
      "activations/layer20_attention_weight_max": 26.619911193847656,
      "activations/layer20_attention_weight_min": -24.21393394470215,
      "activations/layer21_attention_weight_max": 25.8295841217041,
      "activations/layer21_attention_weight_min": -23.001853942871094,
      "activations/layer22_attention_weight_max": 36.789024353027344,
      "activations/layer22_attention_weight_min": -28.43782615661621,
      "activations/layer23_attention_weight_max": 32.11935043334961,
      "activations/layer23_attention_weight_min": -22.84523582458496,
      "activations/layer2_attention_weight_max": 31.996631622314453,
      "activations/layer2_attention_weight_min": -32.892574310302734,
      "activations/layer3_attention_weight_max": 90.71549224853516,
      "activations/layer3_attention_weight_min": -97.40092468261719,
      "activations/layer4_attention_weight_max": 104.40633392333984,
      "activations/layer4_attention_weight_min": -106.23787689208984,
      "activations/layer5_attention_weight_max": 52.35742950439453,
      "activations/layer5_attention_weight_min": -59.4688835144043,
      "activations/layer6_attention_weight_max": 41.28275680541992,
      "activations/layer6_attention_weight_min": -43.671302795410156,
      "activations/layer7_attention_weight_max": 87.12075805664062,
      "activations/layer7_attention_weight_min": -86.26535034179688,
      "activations/layer8_attention_weight_max": 38.90169143676758,
      "activations/layer8_attention_weight_min": -39.380287170410156,
      "activations/layer9_attention_weight_max": 30.719924926757812,
      "activations/layer9_attention_weight_min": -32.13904571533203,
      "epoch": 13.02,
      "learning_rate": 6.67185606060606e-05,
      "loss": 2.7541,
      "step": 224050
    },
    {
      "activations/layer0_attention_weight_max": 15.794184684753418,
      "activations/layer0_attention_weight_min": -11.98212718963623,
      "activations/layer10_attention_weight_max": 32.6467170715332,
      "activations/layer10_attention_weight_min": -33.19373321533203,
      "activations/layer11_attention_weight_max": 31.410091400146484,
      "activations/layer11_attention_weight_min": -31.14004135131836,
      "activations/layer12_attention_weight_max": 26.296920776367188,
      "activations/layer12_attention_weight_min": -27.217180252075195,
      "activations/layer13_attention_weight_max": 37.462135314941406,
      "activations/layer13_attention_weight_min": -34.63837432861328,
      "activations/layer14_attention_weight_max": 45.21133804321289,
      "activations/layer14_attention_weight_min": -37.74260330200195,
      "activations/layer15_attention_weight_max": 37.134300231933594,
      "activations/layer15_attention_weight_min": -32.86629867553711,
      "activations/layer16_attention_weight_max": 29.217586517333984,
      "activations/layer16_attention_weight_min": -27.258800506591797,
      "activations/layer17_attention_weight_max": 32.58286666870117,
      "activations/layer17_attention_weight_min": -25.592247009277344,
      "activations/layer18_attention_weight_max": 32.236419677734375,
      "activations/layer18_attention_weight_min": -24.937522888183594,
      "activations/layer19_attention_weight_max": 35.92274856567383,
      "activations/layer19_attention_weight_min": -31.57663345336914,
      "activations/layer1_attention_weight_max": 16.086084365844727,
      "activations/layer1_attention_weight_min": -15.275466918945312,
      "activations/layer20_attention_weight_max": 29.38970184326172,
      "activations/layer20_attention_weight_min": -25.507423400878906,
      "activations/layer21_attention_weight_max": 28.36283302307129,
      "activations/layer21_attention_weight_min": -23.9166316986084,
      "activations/layer22_attention_weight_max": 42.71994400024414,
      "activations/layer22_attention_weight_min": -31.473142623901367,
      "activations/layer23_attention_weight_max": 34.47159957885742,
      "activations/layer23_attention_weight_min": -27.745222091674805,
      "activations/layer2_attention_weight_max": 33.31785202026367,
      "activations/layer2_attention_weight_min": -34.438438415527344,
      "activations/layer3_attention_weight_max": 96.3448715209961,
      "activations/layer3_attention_weight_min": -94.35505676269531,
      "activations/layer4_attention_weight_max": 114.24321746826172,
      "activations/layer4_attention_weight_min": -109.49543762207031,
      "activations/layer5_attention_weight_max": 51.9581413269043,
      "activations/layer5_attention_weight_min": -61.97034454345703,
      "activations/layer6_attention_weight_max": 43.71316909790039,
      "activations/layer6_attention_weight_min": -45.66193771362305,
      "activations/layer7_attention_weight_max": 96.68466186523438,
      "activations/layer7_attention_weight_min": -91.77579498291016,
      "activations/layer8_attention_weight_max": 40.1890869140625,
      "activations/layer8_attention_weight_min": -41.90985870361328,
      "activations/layer9_attention_weight_max": 32.4996337890625,
      "activations/layer9_attention_weight_min": -33.95858383178711,
      "epoch": 13.02,
      "learning_rate": 6.66996212121212e-05,
      "loss": 2.7556,
      "step": 224100
    },
    {
      "activations/layer0_attention_weight_max": 15.833319664001465,
      "activations/layer0_attention_weight_min": -12.041520118713379,
      "activations/layer10_attention_weight_max": 40.76609802246094,
      "activations/layer10_attention_weight_min": -35.09162902832031,
      "activations/layer11_attention_weight_max": 42.71782684326172,
      "activations/layer11_attention_weight_min": -37.01713562011719,
      "activations/layer12_attention_weight_max": 33.528839111328125,
      "activations/layer12_attention_weight_min": -27.62165069580078,
      "activations/layer13_attention_weight_max": 58.8393669128418,
      "activations/layer13_attention_weight_min": -39.53081130981445,
      "activations/layer14_attention_weight_max": 69.8781509399414,
      "activations/layer14_attention_weight_min": -45.87883758544922,
      "activations/layer15_attention_weight_max": 68.60169982910156,
      "activations/layer15_attention_weight_min": -40.52989196777344,
      "activations/layer16_attention_weight_max": 41.57275390625,
      "activations/layer16_attention_weight_min": -28.438074111938477,
      "activations/layer17_attention_weight_max": 36.11199188232422,
      "activations/layer17_attention_weight_min": -26.51603126525879,
      "activations/layer18_attention_weight_max": 35.942298889160156,
      "activations/layer18_attention_weight_min": -23.680143356323242,
      "activations/layer19_attention_weight_max": 40.64280319213867,
      "activations/layer19_attention_weight_min": -33.18547439575195,
      "activations/layer1_attention_weight_max": 18.77533531188965,
      "activations/layer1_attention_weight_min": -15.310497283935547,
      "activations/layer20_attention_weight_max": 33.961727142333984,
      "activations/layer20_attention_weight_min": -27.33501434326172,
      "activations/layer21_attention_weight_max": 31.618364334106445,
      "activations/layer21_attention_weight_min": -26.32077407836914,
      "activations/layer22_attention_weight_max": 49.653045654296875,
      "activations/layer22_attention_weight_min": -31.67764663696289,
      "activations/layer23_attention_weight_max": 40.682342529296875,
      "activations/layer23_attention_weight_min": -27.780033111572266,
      "activations/layer2_attention_weight_max": 35.360694885253906,
      "activations/layer2_attention_weight_min": -33.51411819458008,
      "activations/layer3_attention_weight_max": 97.71198272705078,
      "activations/layer3_attention_weight_min": -98.47817993164062,
      "activations/layer4_attention_weight_max": 113.76533508300781,
      "activations/layer4_attention_weight_min": -111.52909088134766,
      "activations/layer5_attention_weight_max": 61.34320831298828,
      "activations/layer5_attention_weight_min": -60.90481185913086,
      "activations/layer6_attention_weight_max": 50.949134826660156,
      "activations/layer6_attention_weight_min": -45.15629196166992,
      "activations/layer7_attention_weight_max": 108.7848892211914,
      "activations/layer7_attention_weight_min": -92.70527648925781,
      "activations/layer8_attention_weight_max": 50.437007904052734,
      "activations/layer8_attention_weight_min": -43.19965362548828,
      "activations/layer9_attention_weight_max": 42.2306022644043,
      "activations/layer9_attention_weight_min": -36.305145263671875,
      "epoch": 13.02,
      "learning_rate": 6.668068181818182e-05,
      "loss": 2.7613,
      "step": 224150
    },
    {
      "activations/layer0_attention_weight_max": 16.173294067382812,
      "activations/layer0_attention_weight_min": -12.397887229919434,
      "activations/layer10_attention_weight_max": 31.237899780273438,
      "activations/layer10_attention_weight_min": -30.83983039855957,
      "activations/layer11_attention_weight_max": 30.456336975097656,
      "activations/layer11_attention_weight_min": -30.248912811279297,
      "activations/layer12_attention_weight_max": 24.517606735229492,
      "activations/layer12_attention_weight_min": -24.666479110717773,
      "activations/layer13_attention_weight_max": 36.648406982421875,
      "activations/layer13_attention_weight_min": -34.631256103515625,
      "activations/layer14_attention_weight_max": 42.65585708618164,
      "activations/layer14_attention_weight_min": -36.11763381958008,
      "activations/layer15_attention_weight_max": 35.93442916870117,
      "activations/layer15_attention_weight_min": -32.05843734741211,
      "activations/layer16_attention_weight_max": 30.438098907470703,
      "activations/layer16_attention_weight_min": -26.90064811706543,
      "activations/layer17_attention_weight_max": 31.67230987548828,
      "activations/layer17_attention_weight_min": -25.057947158813477,
      "activations/layer18_attention_weight_max": 32.593502044677734,
      "activations/layer18_attention_weight_min": -24.99357795715332,
      "activations/layer19_attention_weight_max": 32.17916488647461,
      "activations/layer19_attention_weight_min": -30.694320678710938,
      "activations/layer1_attention_weight_max": 16.195859909057617,
      "activations/layer1_attention_weight_min": -14.614555358886719,
      "activations/layer20_attention_weight_max": 26.763032913208008,
      "activations/layer20_attention_weight_min": -26.319040298461914,
      "activations/layer21_attention_weight_max": 27.454442977905273,
      "activations/layer21_attention_weight_min": -25.04270362854004,
      "activations/layer22_attention_weight_max": 46.31155776977539,
      "activations/layer22_attention_weight_min": -33.16718292236328,
      "activations/layer23_attention_weight_max": 34.470863342285156,
      "activations/layer23_attention_weight_min": -24.490081787109375,
      "activations/layer2_attention_weight_max": 37.09571075439453,
      "activations/layer2_attention_weight_min": -33.36608123779297,
      "activations/layer3_attention_weight_max": 98.78561401367188,
      "activations/layer3_attention_weight_min": -97.21887969970703,
      "activations/layer4_attention_weight_max": 109.47982025146484,
      "activations/layer4_attention_weight_min": -111.46791076660156,
      "activations/layer5_attention_weight_max": 52.85299301147461,
      "activations/layer5_attention_weight_min": -61.32292175292969,
      "activations/layer6_attention_weight_max": 45.506797790527344,
      "activations/layer6_attention_weight_min": -47.833457946777344,
      "activations/layer7_attention_weight_max": 90.21160888671875,
      "activations/layer7_attention_weight_min": -92.10092163085938,
      "activations/layer8_attention_weight_max": 42.52540588378906,
      "activations/layer8_attention_weight_min": -43.54486846923828,
      "activations/layer9_attention_weight_max": 31.875207901000977,
      "activations/layer9_attention_weight_min": -32.43442153930664,
      "epoch": 13.03,
      "learning_rate": 6.666174242424242e-05,
      "loss": 2.766,
      "step": 224200
    },
    {
      "activations/layer0_attention_weight_max": 16.288360595703125,
      "activations/layer0_attention_weight_min": -12.455768585205078,
      "activations/layer10_attention_weight_max": 29.371641159057617,
      "activations/layer10_attention_weight_min": -30.749176025390625,
      "activations/layer11_attention_weight_max": 30.05124282836914,
      "activations/layer11_attention_weight_min": -31.086196899414062,
      "activations/layer12_attention_weight_max": 28.453353881835938,
      "activations/layer12_attention_weight_min": -24.679264068603516,
      "activations/layer13_attention_weight_max": 36.791107177734375,
      "activations/layer13_attention_weight_min": -32.4100456237793,
      "activations/layer14_attention_weight_max": 39.03388595581055,
      "activations/layer14_attention_weight_min": -34.498390197753906,
      "activations/layer15_attention_weight_max": 37.08402633666992,
      "activations/layer15_attention_weight_min": -31.919578552246094,
      "activations/layer16_attention_weight_max": 30.124040603637695,
      "activations/layer16_attention_weight_min": -26.476640701293945,
      "activations/layer17_attention_weight_max": 28.415077209472656,
      "activations/layer17_attention_weight_min": -24.76079750061035,
      "activations/layer18_attention_weight_max": 31.220069885253906,
      "activations/layer18_attention_weight_min": -23.058809280395508,
      "activations/layer19_attention_weight_max": 33.91755676269531,
      "activations/layer19_attention_weight_min": -28.69232749938965,
      "activations/layer1_attention_weight_max": 17.644392013549805,
      "activations/layer1_attention_weight_min": -15.701077461242676,
      "activations/layer20_attention_weight_max": 29.608108520507812,
      "activations/layer20_attention_weight_min": -23.724308013916016,
      "activations/layer21_attention_weight_max": 29.448692321777344,
      "activations/layer21_attention_weight_min": -23.290029525756836,
      "activations/layer22_attention_weight_max": 38.56603240966797,
      "activations/layer22_attention_weight_min": -27.879703521728516,
      "activations/layer23_attention_weight_max": 31.023168563842773,
      "activations/layer23_attention_weight_min": -23.505435943603516,
      "activations/layer2_attention_weight_max": 33.596431732177734,
      "activations/layer2_attention_weight_min": -32.79918670654297,
      "activations/layer3_attention_weight_max": 96.00257110595703,
      "activations/layer3_attention_weight_min": -94.88239288330078,
      "activations/layer4_attention_weight_max": 105.80645751953125,
      "activations/layer4_attention_weight_min": -107.40435028076172,
      "activations/layer5_attention_weight_max": 51.00820541381836,
      "activations/layer5_attention_weight_min": -61.191402435302734,
      "activations/layer6_attention_weight_max": 43.79554748535156,
      "activations/layer6_attention_weight_min": -45.0599250793457,
      "activations/layer7_attention_weight_max": 84.8337631225586,
      "activations/layer7_attention_weight_min": -87.87499237060547,
      "activations/layer8_attention_weight_max": 36.3149528503418,
      "activations/layer8_attention_weight_min": -38.533939361572266,
      "activations/layer9_attention_weight_max": 29.285390853881836,
      "activations/layer9_attention_weight_min": -31.952627182006836,
      "epoch": 13.03,
      "learning_rate": 6.664280303030302e-05,
      "loss": 2.7693,
      "step": 224250
    },
    {
      "activations/layer0_attention_weight_max": 15.090866088867188,
      "activations/layer0_attention_weight_min": -12.510869026184082,
      "activations/layer10_attention_weight_max": 30.43662452697754,
      "activations/layer10_attention_weight_min": -32.408050537109375,
      "activations/layer11_attention_weight_max": 31.147016525268555,
      "activations/layer11_attention_weight_min": -31.23543930053711,
      "activations/layer12_attention_weight_max": 24.92218017578125,
      "activations/layer12_attention_weight_min": -25.309972763061523,
      "activations/layer13_attention_weight_max": 36.96902084350586,
      "activations/layer13_attention_weight_min": -34.71269226074219,
      "activations/layer14_attention_weight_max": 40.49245834350586,
      "activations/layer14_attention_weight_min": -36.61959457397461,
      "activations/layer15_attention_weight_max": 36.03642272949219,
      "activations/layer15_attention_weight_min": -34.08636474609375,
      "activations/layer16_attention_weight_max": 30.252840042114258,
      "activations/layer16_attention_weight_min": -28.660385131835938,
      "activations/layer17_attention_weight_max": 32.91960144042969,
      "activations/layer17_attention_weight_min": -26.9578914642334,
      "activations/layer18_attention_weight_max": 30.777273178100586,
      "activations/layer18_attention_weight_min": -25.729196548461914,
      "activations/layer19_attention_weight_max": 33.41457748413086,
      "activations/layer19_attention_weight_min": -30.331552505493164,
      "activations/layer1_attention_weight_max": 16.400070190429688,
      "activations/layer1_attention_weight_min": -14.859326362609863,
      "activations/layer20_attention_weight_max": 30.6229248046875,
      "activations/layer20_attention_weight_min": -25.82668113708496,
      "activations/layer21_attention_weight_max": 30.12852668762207,
      "activations/layer21_attention_weight_min": -25.26384735107422,
      "activations/layer22_attention_weight_max": 41.80474853515625,
      "activations/layer22_attention_weight_min": -30.897857666015625,
      "activations/layer23_attention_weight_max": 35.499427795410156,
      "activations/layer23_attention_weight_min": -25.620901107788086,
      "activations/layer2_attention_weight_max": 31.016677856445312,
      "activations/layer2_attention_weight_min": -32.69104766845703,
      "activations/layer3_attention_weight_max": 98.00621032714844,
      "activations/layer3_attention_weight_min": -98.80623626708984,
      "activations/layer4_attention_weight_max": 109.22602844238281,
      "activations/layer4_attention_weight_min": -114.64852142333984,
      "activations/layer5_attention_weight_max": 50.884765625,
      "activations/layer5_attention_weight_min": -63.20478820800781,
      "activations/layer6_attention_weight_max": 43.78089141845703,
      "activations/layer6_attention_weight_min": -45.05952072143555,
      "activations/layer7_attention_weight_max": 84.82364654541016,
      "activations/layer7_attention_weight_min": -85.52236938476562,
      "activations/layer8_attention_weight_max": 38.0539436340332,
      "activations/layer8_attention_weight_min": -38.7935791015625,
      "activations/layer9_attention_weight_max": 30.459054946899414,
      "activations/layer9_attention_weight_min": -31.47943115234375,
      "epoch": 13.03,
      "learning_rate": 6.662386363636363e-05,
      "loss": 2.7419,
      "step": 224300
    },
    {
      "activations/layer0_attention_weight_max": 16.655668258666992,
      "activations/layer0_attention_weight_min": -12.471015930175781,
      "activations/layer10_attention_weight_max": 32.890377044677734,
      "activations/layer10_attention_weight_min": -33.234153747558594,
      "activations/layer11_attention_weight_max": 32.58238983154297,
      "activations/layer11_attention_weight_min": -32.22809600830078,
      "activations/layer12_attention_weight_max": 24.160205841064453,
      "activations/layer12_attention_weight_min": -26.13361930847168,
      "activations/layer13_attention_weight_max": 39.26900100708008,
      "activations/layer13_attention_weight_min": -33.93090057373047,
      "activations/layer14_attention_weight_max": 44.214256286621094,
      "activations/layer14_attention_weight_min": -38.673545837402344,
      "activations/layer15_attention_weight_max": 36.65835189819336,
      "activations/layer15_attention_weight_min": -31.986778259277344,
      "activations/layer16_attention_weight_max": 29.59914207458496,
      "activations/layer16_attention_weight_min": -28.43480110168457,
      "activations/layer17_attention_weight_max": 30.080244064331055,
      "activations/layer17_attention_weight_min": -26.37775230407715,
      "activations/layer18_attention_weight_max": 31.751432418823242,
      "activations/layer18_attention_weight_min": -23.241172790527344,
      "activations/layer19_attention_weight_max": 35.5762825012207,
      "activations/layer19_attention_weight_min": -31.326501846313477,
      "activations/layer1_attention_weight_max": 17.44843292236328,
      "activations/layer1_attention_weight_min": -16.04078483581543,
      "activations/layer20_attention_weight_max": 30.092931747436523,
      "activations/layer20_attention_weight_min": -24.051862716674805,
      "activations/layer21_attention_weight_max": 33.539676666259766,
      "activations/layer21_attention_weight_min": -23.09953498840332,
      "activations/layer22_attention_weight_max": 45.67729568481445,
      "activations/layer22_attention_weight_min": -28.918487548828125,
      "activations/layer23_attention_weight_max": 35.12471008300781,
      "activations/layer23_attention_weight_min": -24.527019500732422,
      "activations/layer2_attention_weight_max": 33.86264419555664,
      "activations/layer2_attention_weight_min": -32.373931884765625,
      "activations/layer3_attention_weight_max": 97.48896789550781,
      "activations/layer3_attention_weight_min": -100.9801254272461,
      "activations/layer4_attention_weight_max": 111.1471176147461,
      "activations/layer4_attention_weight_min": -112.29070281982422,
      "activations/layer5_attention_weight_max": 53.02397918701172,
      "activations/layer5_attention_weight_min": -61.37731170654297,
      "activations/layer6_attention_weight_max": 45.103214263916016,
      "activations/layer6_attention_weight_min": -47.62196350097656,
      "activations/layer7_attention_weight_max": 96.61526489257812,
      "activations/layer7_attention_weight_min": -96.97747039794922,
      "activations/layer8_attention_weight_max": 45.78754425048828,
      "activations/layer8_attention_weight_min": -43.424495697021484,
      "activations/layer9_attention_weight_max": 34.16501235961914,
      "activations/layer9_attention_weight_min": -34.82794952392578,
      "epoch": 13.04,
      "learning_rate": 6.660492424242424e-05,
      "loss": 2.7564,
      "step": 224350
    },
    {
      "activations/layer0_attention_weight_max": 15.736518859863281,
      "activations/layer0_attention_weight_min": -12.028743743896484,
      "activations/layer10_attention_weight_max": 34.1492805480957,
      "activations/layer10_attention_weight_min": -35.2628059387207,
      "activations/layer11_attention_weight_max": 34.027462005615234,
      "activations/layer11_attention_weight_min": -33.63740539550781,
      "activations/layer12_attention_weight_max": 26.408105850219727,
      "activations/layer12_attention_weight_min": -24.952573776245117,
      "activations/layer13_attention_weight_max": 39.92549133300781,
      "activations/layer13_attention_weight_min": -34.23403549194336,
      "activations/layer14_attention_weight_max": 45.23079299926758,
      "activations/layer14_attention_weight_min": -38.57544708251953,
      "activations/layer15_attention_weight_max": 37.49060821533203,
      "activations/layer15_attention_weight_min": -34.9282341003418,
      "activations/layer16_attention_weight_max": 32.081809997558594,
      "activations/layer16_attention_weight_min": -27.93400764465332,
      "activations/layer17_attention_weight_max": 30.98105812072754,
      "activations/layer17_attention_weight_min": -26.555360794067383,
      "activations/layer18_attention_weight_max": 35.547935485839844,
      "activations/layer18_attention_weight_min": -23.627634048461914,
      "activations/layer19_attention_weight_max": 34.93129348754883,
      "activations/layer19_attention_weight_min": -30.76356315612793,
      "activations/layer1_attention_weight_max": 15.648445129394531,
      "activations/layer1_attention_weight_min": -15.23549747467041,
      "activations/layer20_attention_weight_max": 30.554447174072266,
      "activations/layer20_attention_weight_min": -23.5184383392334,
      "activations/layer21_attention_weight_max": 30.0587215423584,
      "activations/layer21_attention_weight_min": -22.0577392578125,
      "activations/layer22_attention_weight_max": 41.76033020019531,
      "activations/layer22_attention_weight_min": -29.77393341064453,
      "activations/layer23_attention_weight_max": 34.86212158203125,
      "activations/layer23_attention_weight_min": -25.220836639404297,
      "activations/layer2_attention_weight_max": 35.310699462890625,
      "activations/layer2_attention_weight_min": -32.99913787841797,
      "activations/layer3_attention_weight_max": 96.17970275878906,
      "activations/layer3_attention_weight_min": -95.05711364746094,
      "activations/layer4_attention_weight_max": 109.8803939819336,
      "activations/layer4_attention_weight_min": -108.16474914550781,
      "activations/layer5_attention_weight_max": 51.39824676513672,
      "activations/layer5_attention_weight_min": -58.54580307006836,
      "activations/layer6_attention_weight_max": 42.843955993652344,
      "activations/layer6_attention_weight_min": -44.02188491821289,
      "activations/layer7_attention_weight_max": 87.51628112792969,
      "activations/layer7_attention_weight_min": -89.1561050415039,
      "activations/layer8_attention_weight_max": 41.05511474609375,
      "activations/layer8_attention_weight_min": -42.971561431884766,
      "activations/layer9_attention_weight_max": 35.332698822021484,
      "activations/layer9_attention_weight_min": -35.960994720458984,
      "epoch": 13.04,
      "learning_rate": 6.658598484848484e-05,
      "loss": 2.7672,
      "step": 224400
    },
    {
      "activations/layer0_attention_weight_max": 15.84618854522705,
      "activations/layer0_attention_weight_min": -12.28481388092041,
      "activations/layer10_attention_weight_max": 34.39207458496094,
      "activations/layer10_attention_weight_min": -32.07485580444336,
      "activations/layer11_attention_weight_max": 31.868587493896484,
      "activations/layer11_attention_weight_min": -31.371341705322266,
      "activations/layer12_attention_weight_max": 26.20745277404785,
      "activations/layer12_attention_weight_min": -25.302669525146484,
      "activations/layer13_attention_weight_max": 39.699989318847656,
      "activations/layer13_attention_weight_min": -34.461883544921875,
      "activations/layer14_attention_weight_max": 42.63764572143555,
      "activations/layer14_attention_weight_min": -36.84316635131836,
      "activations/layer15_attention_weight_max": 37.924842834472656,
      "activations/layer15_attention_weight_min": -34.54830551147461,
      "activations/layer16_attention_weight_max": 31.104719161987305,
      "activations/layer16_attention_weight_min": -28.42599868774414,
      "activations/layer17_attention_weight_max": 31.29221534729004,
      "activations/layer17_attention_weight_min": -30.073974609375,
      "activations/layer18_attention_weight_max": 32.41513442993164,
      "activations/layer18_attention_weight_min": -25.878183364868164,
      "activations/layer19_attention_weight_max": 34.86625289916992,
      "activations/layer19_attention_weight_min": -33.334922790527344,
      "activations/layer1_attention_weight_max": 16.88759422302246,
      "activations/layer1_attention_weight_min": -14.528148651123047,
      "activations/layer20_attention_weight_max": 31.005687713623047,
      "activations/layer20_attention_weight_min": -24.90721893310547,
      "activations/layer21_attention_weight_max": 31.69871711730957,
      "activations/layer21_attention_weight_min": -25.843551635742188,
      "activations/layer22_attention_weight_max": 44.63238525390625,
      "activations/layer22_attention_weight_min": -31.614431381225586,
      "activations/layer23_attention_weight_max": 32.11393737792969,
      "activations/layer23_attention_weight_min": -25.274227142333984,
      "activations/layer2_attention_weight_max": 32.338050842285156,
      "activations/layer2_attention_weight_min": -32.65856170654297,
      "activations/layer3_attention_weight_max": 92.16388702392578,
      "activations/layer3_attention_weight_min": -101.54039764404297,
      "activations/layer4_attention_weight_max": 107.8373794555664,
      "activations/layer4_attention_weight_min": -115.2984848022461,
      "activations/layer5_attention_weight_max": 49.56678771972656,
      "activations/layer5_attention_weight_min": -61.37860870361328,
      "activations/layer6_attention_weight_max": 42.744224548339844,
      "activations/layer6_attention_weight_min": -46.69584655761719,
      "activations/layer7_attention_weight_max": 88.02014923095703,
      "activations/layer7_attention_weight_min": -90.87635040283203,
      "activations/layer8_attention_weight_max": 39.18832015991211,
      "activations/layer8_attention_weight_min": -41.233882904052734,
      "activations/layer9_attention_weight_max": 32.51963806152344,
      "activations/layer9_attention_weight_min": -33.22077560424805,
      "epoch": 13.04,
      "learning_rate": 6.656704545454544e-05,
      "loss": 2.7807,
      "step": 224450
    },
    {
      "activations/layer0_attention_weight_max": 16.21490478515625,
      "activations/layer0_attention_weight_min": -12.46042537689209,
      "activations/layer10_attention_weight_max": 31.773117065429688,
      "activations/layer10_attention_weight_min": -32.40407180786133,
      "activations/layer11_attention_weight_max": 31.20832061767578,
      "activations/layer11_attention_weight_min": -31.02022361755371,
      "activations/layer12_attention_weight_max": 25.37991714477539,
      "activations/layer12_attention_weight_min": -24.236270904541016,
      "activations/layer13_attention_weight_max": 39.261592864990234,
      "activations/layer13_attention_weight_min": -32.35776901245117,
      "activations/layer14_attention_weight_max": 42.52442169189453,
      "activations/layer14_attention_weight_min": -37.82327651977539,
      "activations/layer15_attention_weight_max": 35.955902099609375,
      "activations/layer15_attention_weight_min": -32.18985366821289,
      "activations/layer16_attention_weight_max": 30.1260986328125,
      "activations/layer16_attention_weight_min": -27.790658950805664,
      "activations/layer17_attention_weight_max": 30.24485969543457,
      "activations/layer17_attention_weight_min": -26.223243713378906,
      "activations/layer18_attention_weight_max": 32.73674774169922,
      "activations/layer18_attention_weight_min": -24.069948196411133,
      "activations/layer19_attention_weight_max": 33.2888298034668,
      "activations/layer19_attention_weight_min": -29.72140884399414,
      "activations/layer1_attention_weight_max": 17.1640682220459,
      "activations/layer1_attention_weight_min": -15.913061141967773,
      "activations/layer20_attention_weight_max": 29.72846031188965,
      "activations/layer20_attention_weight_min": -24.770357131958008,
      "activations/layer21_attention_weight_max": 30.019222259521484,
      "activations/layer21_attention_weight_min": -23.4976863861084,
      "activations/layer22_attention_weight_max": 45.448577880859375,
      "activations/layer22_attention_weight_min": -30.071788787841797,
      "activations/layer23_attention_weight_max": 33.698974609375,
      "activations/layer23_attention_weight_min": -25.94001007080078,
      "activations/layer2_attention_weight_max": 33.538299560546875,
      "activations/layer2_attention_weight_min": -31.887149810791016,
      "activations/layer3_attention_weight_max": 94.76312255859375,
      "activations/layer3_attention_weight_min": -94.9368896484375,
      "activations/layer4_attention_weight_max": 107.5464096069336,
      "activations/layer4_attention_weight_min": -109.12711334228516,
      "activations/layer5_attention_weight_max": 51.3304443359375,
      "activations/layer5_attention_weight_min": -65.6478271484375,
      "activations/layer6_attention_weight_max": 43.06296920776367,
      "activations/layer6_attention_weight_min": -46.178340911865234,
      "activations/layer7_attention_weight_max": 88.95506286621094,
      "activations/layer7_attention_weight_min": -94.11561584472656,
      "activations/layer8_attention_weight_max": 39.50514221191406,
      "activations/layer8_attention_weight_min": -42.34574890136719,
      "activations/layer9_attention_weight_max": 33.2354850769043,
      "activations/layer9_attention_weight_min": -35.708763122558594,
      "epoch": 13.04,
      "learning_rate": 6.654810606060606e-05,
      "loss": 2.7692,
      "step": 224500
    },
    {
      "activations/layer0_attention_weight_max": 18.0656681060791,
      "activations/layer0_attention_weight_min": -11.154854774475098,
      "activations/layer10_attention_weight_max": 36.38262176513672,
      "activations/layer10_attention_weight_min": -32.809181213378906,
      "activations/layer11_attention_weight_max": 38.760948181152344,
      "activations/layer11_attention_weight_min": -35.83008575439453,
      "activations/layer12_attention_weight_max": 26.923446655273438,
      "activations/layer12_attention_weight_min": -27.37797737121582,
      "activations/layer13_attention_weight_max": 41.48340606689453,
      "activations/layer13_attention_weight_min": -34.716835021972656,
      "activations/layer14_attention_weight_max": 47.99763870239258,
      "activations/layer14_attention_weight_min": -37.982391357421875,
      "activations/layer15_attention_weight_max": 37.6290283203125,
      "activations/layer15_attention_weight_min": -32.177642822265625,
      "activations/layer16_attention_weight_max": 30.818323135375977,
      "activations/layer16_attention_weight_min": -27.551097869873047,
      "activations/layer17_attention_weight_max": 33.31740188598633,
      "activations/layer17_attention_weight_min": -26.686420440673828,
      "activations/layer18_attention_weight_max": 30.28253173828125,
      "activations/layer18_attention_weight_min": -26.549074172973633,
      "activations/layer19_attention_weight_max": 33.891448974609375,
      "activations/layer19_attention_weight_min": -29.853734970092773,
      "activations/layer1_attention_weight_max": 17.207849502563477,
      "activations/layer1_attention_weight_min": -14.195439338684082,
      "activations/layer20_attention_weight_max": 29.27598762512207,
      "activations/layer20_attention_weight_min": -25.13055992126465,
      "activations/layer21_attention_weight_max": 26.808311462402344,
      "activations/layer21_attention_weight_min": -24.7645206451416,
      "activations/layer22_attention_weight_max": 45.238853454589844,
      "activations/layer22_attention_weight_min": -29.729660034179688,
      "activations/layer23_attention_weight_max": 34.10468292236328,
      "activations/layer23_attention_weight_min": -25.06653594970703,
      "activations/layer2_attention_weight_max": 32.154212951660156,
      "activations/layer2_attention_weight_min": -31.608280181884766,
      "activations/layer3_attention_weight_max": 97.84046173095703,
      "activations/layer3_attention_weight_min": -99.10511016845703,
      "activations/layer4_attention_weight_max": 112.68989562988281,
      "activations/layer4_attention_weight_min": -117.38215637207031,
      "activations/layer5_attention_weight_max": 52.33811950683594,
      "activations/layer5_attention_weight_min": -61.070655822753906,
      "activations/layer6_attention_weight_max": 44.7218132019043,
      "activations/layer6_attention_weight_min": -47.67911148071289,
      "activations/layer7_attention_weight_max": 94.11396026611328,
      "activations/layer7_attention_weight_min": -95.72523498535156,
      "activations/layer8_attention_weight_max": 41.2080078125,
      "activations/layer8_attention_weight_min": -41.915470123291016,
      "activations/layer9_attention_weight_max": 36.00761413574219,
      "activations/layer9_attention_weight_min": -34.874507904052734,
      "epoch": 13.05,
      "learning_rate": 6.652916666666666e-05,
      "loss": 2.7563,
      "step": 224550
    },
    {
      "activations/layer0_attention_weight_max": 16.2260799407959,
      "activations/layer0_attention_weight_min": -11.62974739074707,
      "activations/layer10_attention_weight_max": 40.24238586425781,
      "activations/layer10_attention_weight_min": -36.54018020629883,
      "activations/layer11_attention_weight_max": 43.286434173583984,
      "activations/layer11_attention_weight_min": -38.946449279785156,
      "activations/layer12_attention_weight_max": 34.34617233276367,
      "activations/layer12_attention_weight_min": -27.180164337158203,
      "activations/layer13_attention_weight_max": 54.65426254272461,
      "activations/layer13_attention_weight_min": -32.72970199584961,
      "activations/layer14_attention_weight_max": 55.75113296508789,
      "activations/layer14_attention_weight_min": -40.0311164855957,
      "activations/layer15_attention_weight_max": 64.24295806884766,
      "activations/layer15_attention_weight_min": -35.37028121948242,
      "activations/layer16_attention_weight_max": 45.52547836303711,
      "activations/layer16_attention_weight_min": -26.34891128540039,
      "activations/layer17_attention_weight_max": 30.95087432861328,
      "activations/layer17_attention_weight_min": -27.7562255859375,
      "activations/layer18_attention_weight_max": 26.034835815429688,
      "activations/layer18_attention_weight_min": -23.842872619628906,
      "activations/layer19_attention_weight_max": 33.729854583740234,
      "activations/layer19_attention_weight_min": -28.84225845336914,
      "activations/layer1_attention_weight_max": 16.86739730834961,
      "activations/layer1_attention_weight_min": -14.27389144897461,
      "activations/layer20_attention_weight_max": 27.949954986572266,
      "activations/layer20_attention_weight_min": -23.223344802856445,
      "activations/layer21_attention_weight_max": 28.087675094604492,
      "activations/layer21_attention_weight_min": -22.192472457885742,
      "activations/layer22_attention_weight_max": 41.85786437988281,
      "activations/layer22_attention_weight_min": -28.52793312072754,
      "activations/layer23_attention_weight_max": 33.73549270629883,
      "activations/layer23_attention_weight_min": -23.785457611083984,
      "activations/layer2_attention_weight_max": 35.112281799316406,
      "activations/layer2_attention_weight_min": -32.998497009277344,
      "activations/layer3_attention_weight_max": 97.85189819335938,
      "activations/layer3_attention_weight_min": -97.73924255371094,
      "activations/layer4_attention_weight_max": 110.55693817138672,
      "activations/layer4_attention_weight_min": -109.15361785888672,
      "activations/layer5_attention_weight_max": 56.94405746459961,
      "activations/layer5_attention_weight_min": -59.43170166015625,
      "activations/layer6_attention_weight_max": 44.847679138183594,
      "activations/layer6_attention_weight_min": -47.89851379394531,
      "activations/layer7_attention_weight_max": 105.35128021240234,
      "activations/layer7_attention_weight_min": -98.46465301513672,
      "activations/layer8_attention_weight_max": 46.61213684082031,
      "activations/layer8_attention_weight_min": -44.05403518676758,
      "activations/layer9_attention_weight_max": 36.5778923034668,
      "activations/layer9_attention_weight_min": -35.8001594543457,
      "epoch": 13.05,
      "learning_rate": 6.651060606060605e-05,
      "loss": 2.752,
      "step": 224600
    },
    {
      "activations/layer0_attention_weight_max": 16.344039916992188,
      "activations/layer0_attention_weight_min": -12.623283386230469,
      "activations/layer10_attention_weight_max": 31.67654037475586,
      "activations/layer10_attention_weight_min": -31.624664306640625,
      "activations/layer11_attention_weight_max": 30.701202392578125,
      "activations/layer11_attention_weight_min": -31.12915802001953,
      "activations/layer12_attention_weight_max": 25.126407623291016,
      "activations/layer12_attention_weight_min": -25.39396095275879,
      "activations/layer13_attention_weight_max": 37.62400436401367,
      "activations/layer13_attention_weight_min": -33.90989685058594,
      "activations/layer14_attention_weight_max": 39.99440383911133,
      "activations/layer14_attention_weight_min": -35.598960876464844,
      "activations/layer15_attention_weight_max": 33.87587356567383,
      "activations/layer15_attention_weight_min": -32.499656677246094,
      "activations/layer16_attention_weight_max": 30.69803810119629,
      "activations/layer16_attention_weight_min": -29.34879493713379,
      "activations/layer17_attention_weight_max": 29.545427322387695,
      "activations/layer17_attention_weight_min": -25.28318214416504,
      "activations/layer18_attention_weight_max": 29.879940032958984,
      "activations/layer18_attention_weight_min": -23.648334503173828,
      "activations/layer19_attention_weight_max": 34.41999435424805,
      "activations/layer19_attention_weight_min": -30.035959243774414,
      "activations/layer1_attention_weight_max": 15.732596397399902,
      "activations/layer1_attention_weight_min": -16.062637329101562,
      "activations/layer20_attention_weight_max": 26.731693267822266,
      "activations/layer20_attention_weight_min": -24.611520767211914,
      "activations/layer21_attention_weight_max": 26.362533569335938,
      "activations/layer21_attention_weight_min": -23.647735595703125,
      "activations/layer22_attention_weight_max": 39.42921447753906,
      "activations/layer22_attention_weight_min": -28.655834197998047,
      "activations/layer23_attention_weight_max": 34.946895599365234,
      "activations/layer23_attention_weight_min": -25.466514587402344,
      "activations/layer2_attention_weight_max": 32.7303466796875,
      "activations/layer2_attention_weight_min": -32.51255416870117,
      "activations/layer3_attention_weight_max": 95.20344543457031,
      "activations/layer3_attention_weight_min": -98.11756896972656,
      "activations/layer4_attention_weight_max": 109.2931137084961,
      "activations/layer4_attention_weight_min": -111.3663101196289,
      "activations/layer5_attention_weight_max": 50.490989685058594,
      "activations/layer5_attention_weight_min": -62.64237976074219,
      "activations/layer6_attention_weight_max": 43.30180358886719,
      "activations/layer6_attention_weight_min": -45.77127456665039,
      "activations/layer7_attention_weight_max": 91.39840698242188,
      "activations/layer7_attention_weight_min": -90.08260345458984,
      "activations/layer8_attention_weight_max": 40.82106399536133,
      "activations/layer8_attention_weight_min": -42.87034606933594,
      "activations/layer9_attention_weight_max": 30.215255737304688,
      "activations/layer9_attention_weight_min": -31.869388580322266,
      "epoch": 13.05,
      "learning_rate": 6.649166666666667e-05,
      "loss": 2.7492,
      "step": 224650
    },
    {
      "activations/layer0_attention_weight_max": 16.65256118774414,
      "activations/layer0_attention_weight_min": -11.655325889587402,
      "activations/layer10_attention_weight_max": 32.20233154296875,
      "activations/layer10_attention_weight_min": -33.12994384765625,
      "activations/layer11_attention_weight_max": 32.654510498046875,
      "activations/layer11_attention_weight_min": -31.735023498535156,
      "activations/layer12_attention_weight_max": 23.93124771118164,
      "activations/layer12_attention_weight_min": -25.544029235839844,
      "activations/layer13_attention_weight_max": 36.35153579711914,
      "activations/layer13_attention_weight_min": -35.914710998535156,
      "activations/layer14_attention_weight_max": 39.27090072631836,
      "activations/layer14_attention_weight_min": -37.62489700317383,
      "activations/layer15_attention_weight_max": 34.014713287353516,
      "activations/layer15_attention_weight_min": -32.883888244628906,
      "activations/layer16_attention_weight_max": 27.806175231933594,
      "activations/layer16_attention_weight_min": -26.574905395507812,
      "activations/layer17_attention_weight_max": 29.50689125061035,
      "activations/layer17_attention_weight_min": -26.04155731201172,
      "activations/layer18_attention_weight_max": 30.990516662597656,
      "activations/layer18_attention_weight_min": -24.80780601501465,
      "activations/layer19_attention_weight_max": 35.36431884765625,
      "activations/layer19_attention_weight_min": -30.357711791992188,
      "activations/layer1_attention_weight_max": 17.136484146118164,
      "activations/layer1_attention_weight_min": -15.74239444732666,
      "activations/layer20_attention_weight_max": 28.2381649017334,
      "activations/layer20_attention_weight_min": -24.96211814880371,
      "activations/layer21_attention_weight_max": 29.538238525390625,
      "activations/layer21_attention_weight_min": -24.662256240844727,
      "activations/layer22_attention_weight_max": 44.02565002441406,
      "activations/layer22_attention_weight_min": -30.635236740112305,
      "activations/layer23_attention_weight_max": 35.09700012207031,
      "activations/layer23_attention_weight_min": -24.637863159179688,
      "activations/layer2_attention_weight_max": 32.267269134521484,
      "activations/layer2_attention_weight_min": -33.00531005859375,
      "activations/layer3_attention_weight_max": 100.11176300048828,
      "activations/layer3_attention_weight_min": -97.2320556640625,
      "activations/layer4_attention_weight_max": 111.07405853271484,
      "activations/layer4_attention_weight_min": -110.62239837646484,
      "activations/layer5_attention_weight_max": 51.61687469482422,
      "activations/layer5_attention_weight_min": -62.318763732910156,
      "activations/layer6_attention_weight_max": 43.66252517700195,
      "activations/layer6_attention_weight_min": -46.17372131347656,
      "activations/layer7_attention_weight_max": 85.52384185791016,
      "activations/layer7_attention_weight_min": -86.13980865478516,
      "activations/layer8_attention_weight_max": 39.46756362915039,
      "activations/layer8_attention_weight_min": -41.36812210083008,
      "activations/layer9_attention_weight_max": 31.23883628845215,
      "activations/layer9_attention_weight_min": -31.845706939697266,
      "epoch": 13.06,
      "learning_rate": 6.647272727272727e-05,
      "loss": 2.7562,
      "step": 224700
    },
    {
      "activations/layer0_attention_weight_max": 15.794511795043945,
      "activations/layer0_attention_weight_min": -12.202235221862793,
      "activations/layer10_attention_weight_max": 33.2698860168457,
      "activations/layer10_attention_weight_min": -31.574460983276367,
      "activations/layer11_attention_weight_max": 30.99135398864746,
      "activations/layer11_attention_weight_min": -32.35392761230469,
      "activations/layer12_attention_weight_max": 26.830915451049805,
      "activations/layer12_attention_weight_min": -28.004844665527344,
      "activations/layer13_attention_weight_max": 42.30315017700195,
      "activations/layer13_attention_weight_min": -35.476558685302734,
      "activations/layer14_attention_weight_max": 43.6095085144043,
      "activations/layer14_attention_weight_min": -37.26088333129883,
      "activations/layer15_attention_weight_max": 36.39179992675781,
      "activations/layer15_attention_weight_min": -33.551414489746094,
      "activations/layer16_attention_weight_max": 29.539278030395508,
      "activations/layer16_attention_weight_min": -28.290878295898438,
      "activations/layer17_attention_weight_max": 31.363428115844727,
      "activations/layer17_attention_weight_min": -26.82052993774414,
      "activations/layer18_attention_weight_max": 30.646310806274414,
      "activations/layer18_attention_weight_min": -24.425113677978516,
      "activations/layer19_attention_weight_max": 32.796424865722656,
      "activations/layer19_attention_weight_min": -30.677875518798828,
      "activations/layer1_attention_weight_max": 17.399169921875,
      "activations/layer1_attention_weight_min": -14.280495643615723,
      "activations/layer20_attention_weight_max": 27.289278030395508,
      "activations/layer20_attention_weight_min": -23.165910720825195,
      "activations/layer21_attention_weight_max": 25.33891487121582,
      "activations/layer21_attention_weight_min": -22.684261322021484,
      "activations/layer22_attention_weight_max": 47.886783599853516,
      "activations/layer22_attention_weight_min": -29.389436721801758,
      "activations/layer23_attention_weight_max": 31.10018539428711,
      "activations/layer23_attention_weight_min": -24.670982360839844,
      "activations/layer2_attention_weight_max": 33.97509002685547,
      "activations/layer2_attention_weight_min": -34.07443618774414,
      "activations/layer3_attention_weight_max": 98.018310546875,
      "activations/layer3_attention_weight_min": -101.7082748413086,
      "activations/layer4_attention_weight_max": 113.24690246582031,
      "activations/layer4_attention_weight_min": -111.92579650878906,
      "activations/layer5_attention_weight_max": 52.85740280151367,
      "activations/layer5_attention_weight_min": -61.80192184448242,
      "activations/layer6_attention_weight_max": 43.88450241088867,
      "activations/layer6_attention_weight_min": -46.65410614013672,
      "activations/layer7_attention_weight_max": 90.76676940917969,
      "activations/layer7_attention_weight_min": -103.59004974365234,
      "activations/layer8_attention_weight_max": 40.66435623168945,
      "activations/layer8_attention_weight_min": -40.65149688720703,
      "activations/layer9_attention_weight_max": 33.039608001708984,
      "activations/layer9_attention_weight_min": -33.34275436401367,
      "epoch": 13.06,
      "learning_rate": 6.645378787878788e-05,
      "loss": 2.7659,
      "step": 224750
    },
    {
      "activations/layer0_attention_weight_max": 15.10647964477539,
      "activations/layer0_attention_weight_min": -12.3292818069458,
      "activations/layer10_attention_weight_max": 34.84633255004883,
      "activations/layer10_attention_weight_min": -34.08274841308594,
      "activations/layer11_attention_weight_max": 35.56187438964844,
      "activations/layer11_attention_weight_min": -33.359580993652344,
      "activations/layer12_attention_weight_max": 27.33146858215332,
      "activations/layer12_attention_weight_min": -25.51193618774414,
      "activations/layer13_attention_weight_max": 42.725120544433594,
      "activations/layer13_attention_weight_min": -36.61955642700195,
      "activations/layer14_attention_weight_max": 48.38764953613281,
      "activations/layer14_attention_weight_min": -38.46037673950195,
      "activations/layer15_attention_weight_max": 39.70034408569336,
      "activations/layer15_attention_weight_min": -34.023136138916016,
      "activations/layer16_attention_weight_max": 32.45051193237305,
      "activations/layer16_attention_weight_min": -29.29898452758789,
      "activations/layer17_attention_weight_max": 30.23822784423828,
      "activations/layer17_attention_weight_min": -25.143484115600586,
      "activations/layer18_attention_weight_max": 31.826026916503906,
      "activations/layer18_attention_weight_min": -23.553789138793945,
      "activations/layer19_attention_weight_max": 36.13291549682617,
      "activations/layer19_attention_weight_min": -30.10748291015625,
      "activations/layer1_attention_weight_max": 16.427228927612305,
      "activations/layer1_attention_weight_min": -17.327436447143555,
      "activations/layer20_attention_weight_max": 30.19508934020996,
      "activations/layer20_attention_weight_min": -24.10446548461914,
      "activations/layer21_attention_weight_max": 30.396249771118164,
      "activations/layer21_attention_weight_min": -25.328624725341797,
      "activations/layer22_attention_weight_max": 42.964481353759766,
      "activations/layer22_attention_weight_min": -29.456533432006836,
      "activations/layer23_attention_weight_max": 34.057586669921875,
      "activations/layer23_attention_weight_min": -24.309913635253906,
      "activations/layer2_attention_weight_max": 32.16011047363281,
      "activations/layer2_attention_weight_min": -31.43860626220703,
      "activations/layer3_attention_weight_max": 94.73170471191406,
      "activations/layer3_attention_weight_min": -98.41419219970703,
      "activations/layer4_attention_weight_max": 106.6897964477539,
      "activations/layer4_attention_weight_min": -111.36327362060547,
      "activations/layer5_attention_weight_max": 51.779083251953125,
      "activations/layer5_attention_weight_min": -62.349910736083984,
      "activations/layer6_attention_weight_max": 42.578834533691406,
      "activations/layer6_attention_weight_min": -47.08580017089844,
      "activations/layer7_attention_weight_max": 94.32581329345703,
      "activations/layer7_attention_weight_min": -94.26114654541016,
      "activations/layer8_attention_weight_max": 41.77561569213867,
      "activations/layer8_attention_weight_min": -41.18727493286133,
      "activations/layer9_attention_weight_max": 35.17300796508789,
      "activations/layer9_attention_weight_min": -32.76521301269531,
      "epoch": 13.06,
      "learning_rate": 6.643484848484848e-05,
      "loss": 2.7415,
      "step": 224800
    },
    {
      "activations/layer0_attention_weight_max": 15.215099334716797,
      "activations/layer0_attention_weight_min": -11.154295921325684,
      "activations/layer10_attention_weight_max": 29.881877899169922,
      "activations/layer10_attention_weight_min": -32.88178253173828,
      "activations/layer11_attention_weight_max": 32.485408782958984,
      "activations/layer11_attention_weight_min": -30.97234344482422,
      "activations/layer12_attention_weight_max": 23.810380935668945,
      "activations/layer12_attention_weight_min": -24.211700439453125,
      "activations/layer13_attention_weight_max": 36.82133483886719,
      "activations/layer13_attention_weight_min": -34.63749694824219,
      "activations/layer14_attention_weight_max": 44.60289001464844,
      "activations/layer14_attention_weight_min": -38.72618103027344,
      "activations/layer15_attention_weight_max": 35.45549774169922,
      "activations/layer15_attention_weight_min": -32.926979064941406,
      "activations/layer16_attention_weight_max": 33.14480209350586,
      "activations/layer16_attention_weight_min": -29.72685432434082,
      "activations/layer17_attention_weight_max": 32.45806884765625,
      "activations/layer17_attention_weight_min": -24.813501358032227,
      "activations/layer18_attention_weight_max": 31.33057975769043,
      "activations/layer18_attention_weight_min": -25.1722354888916,
      "activations/layer19_attention_weight_max": 36.79685592651367,
      "activations/layer19_attention_weight_min": -30.814645767211914,
      "activations/layer1_attention_weight_max": 15.900188446044922,
      "activations/layer1_attention_weight_min": -14.255309104919434,
      "activations/layer20_attention_weight_max": 31.615415573120117,
      "activations/layer20_attention_weight_min": -26.22001838684082,
      "activations/layer21_attention_weight_max": 32.972137451171875,
      "activations/layer21_attention_weight_min": -25.16072654724121,
      "activations/layer22_attention_weight_max": 48.86328887939453,
      "activations/layer22_attention_weight_min": -31.48166847229004,
      "activations/layer23_attention_weight_max": 36.877220153808594,
      "activations/layer23_attention_weight_min": -23.093456268310547,
      "activations/layer2_attention_weight_max": 34.34172821044922,
      "activations/layer2_attention_weight_min": -32.212745666503906,
      "activations/layer3_attention_weight_max": 93.49446105957031,
      "activations/layer3_attention_weight_min": -93.24434661865234,
      "activations/layer4_attention_weight_max": 106.7494888305664,
      "activations/layer4_attention_weight_min": -109.26301574707031,
      "activations/layer5_attention_weight_max": 48.73579406738281,
      "activations/layer5_attention_weight_min": -60.75794982910156,
      "activations/layer6_attention_weight_max": 43.83312225341797,
      "activations/layer6_attention_weight_min": -46.08393478393555,
      "activations/layer7_attention_weight_max": 84.632080078125,
      "activations/layer7_attention_weight_min": -85.36744689941406,
      "activations/layer8_attention_weight_max": 38.49953079223633,
      "activations/layer8_attention_weight_min": -41.196571350097656,
      "activations/layer9_attention_weight_max": 31.770233154296875,
      "activations/layer9_attention_weight_min": -31.664169311523438,
      "epoch": 13.07,
      "learning_rate": 6.641590909090909e-05,
      "loss": 2.7505,
      "step": 224850
    },
    {
      "activations/layer0_attention_weight_max": 16.104440689086914,
      "activations/layer0_attention_weight_min": -11.869549751281738,
      "activations/layer10_attention_weight_max": 30.20668601989746,
      "activations/layer10_attention_weight_min": -32.29278564453125,
      "activations/layer11_attention_weight_max": 30.940109252929688,
      "activations/layer11_attention_weight_min": -33.376712799072266,
      "activations/layer12_attention_weight_max": 24.880786895751953,
      "activations/layer12_attention_weight_min": -24.869522094726562,
      "activations/layer13_attention_weight_max": 38.159034729003906,
      "activations/layer13_attention_weight_min": -33.62042236328125,
      "activations/layer14_attention_weight_max": 39.805110931396484,
      "activations/layer14_attention_weight_min": -35.79167556762695,
      "activations/layer15_attention_weight_max": 37.16640853881836,
      "activations/layer15_attention_weight_min": -34.07301330566406,
      "activations/layer16_attention_weight_max": 29.468616485595703,
      "activations/layer16_attention_weight_min": -26.72933578491211,
      "activations/layer17_attention_weight_max": 30.91432762145996,
      "activations/layer17_attention_weight_min": -25.585416793823242,
      "activations/layer18_attention_weight_max": 30.121540069580078,
      "activations/layer18_attention_weight_min": -24.776166915893555,
      "activations/layer19_attention_weight_max": 33.32750701904297,
      "activations/layer19_attention_weight_min": -30.281274795532227,
      "activations/layer1_attention_weight_max": 16.755308151245117,
      "activations/layer1_attention_weight_min": -14.754950523376465,
      "activations/layer20_attention_weight_max": 27.2615909576416,
      "activations/layer20_attention_weight_min": -25.29793357849121,
      "activations/layer21_attention_weight_max": 27.48801612854004,
      "activations/layer21_attention_weight_min": -24.240482330322266,
      "activations/layer22_attention_weight_max": 40.27570343017578,
      "activations/layer22_attention_weight_min": -29.084571838378906,
      "activations/layer23_attention_weight_max": 32.55632781982422,
      "activations/layer23_attention_weight_min": -25.988739013671875,
      "activations/layer2_attention_weight_max": 34.32071304321289,
      "activations/layer2_attention_weight_min": -33.34051513671875,
      "activations/layer3_attention_weight_max": 97.84003448486328,
      "activations/layer3_attention_weight_min": -96.38362884521484,
      "activations/layer4_attention_weight_max": 113.2996826171875,
      "activations/layer4_attention_weight_min": -114.87560272216797,
      "activations/layer5_attention_weight_max": 50.49662780761719,
      "activations/layer5_attention_weight_min": -60.4033203125,
      "activations/layer6_attention_weight_max": 43.58477020263672,
      "activations/layer6_attention_weight_min": -46.62272644042969,
      "activations/layer7_attention_weight_max": 91.28074645996094,
      "activations/layer7_attention_weight_min": -92.85603332519531,
      "activations/layer8_attention_weight_max": 39.54867935180664,
      "activations/layer8_attention_weight_min": -41.585960388183594,
      "activations/layer9_attention_weight_max": 31.05146598815918,
      "activations/layer9_attention_weight_min": -33.692474365234375,
      "epoch": 13.07,
      "learning_rate": 6.639696969696969e-05,
      "loss": 2.7567,
      "step": 224900
    },
    {
      "activations/layer0_attention_weight_max": 15.617164611816406,
      "activations/layer0_attention_weight_min": -11.83607292175293,
      "activations/layer10_attention_weight_max": 31.0419979095459,
      "activations/layer10_attention_weight_min": -33.24788284301758,
      "activations/layer11_attention_weight_max": 31.170927047729492,
      "activations/layer11_attention_weight_min": -32.810115814208984,
      "activations/layer12_attention_weight_max": 24.27892303466797,
      "activations/layer12_attention_weight_min": -28.24724769592285,
      "activations/layer13_attention_weight_max": 35.73087692260742,
      "activations/layer13_attention_weight_min": -35.136253356933594,
      "activations/layer14_attention_weight_max": 42.12150192260742,
      "activations/layer14_attention_weight_min": -39.223567962646484,
      "activations/layer15_attention_weight_max": 34.39542770385742,
      "activations/layer15_attention_weight_min": -34.32896041870117,
      "activations/layer16_attention_weight_max": 30.441532135009766,
      "activations/layer16_attention_weight_min": -27.83038330078125,
      "activations/layer17_attention_weight_max": 31.064355850219727,
      "activations/layer17_attention_weight_min": -24.930259704589844,
      "activations/layer18_attention_weight_max": 29.41688346862793,
      "activations/layer18_attention_weight_min": -23.272357940673828,
      "activations/layer19_attention_weight_max": 33.677223205566406,
      "activations/layer19_attention_weight_min": -30.945449829101562,
      "activations/layer1_attention_weight_max": 16.210596084594727,
      "activations/layer1_attention_weight_min": -14.725318908691406,
      "activations/layer20_attention_weight_max": 28.654638290405273,
      "activations/layer20_attention_weight_min": -24.488719940185547,
      "activations/layer21_attention_weight_max": 25.86358642578125,
      "activations/layer21_attention_weight_min": -24.504558563232422,
      "activations/layer22_attention_weight_max": 40.65994644165039,
      "activations/layer22_attention_weight_min": -30.212509155273438,
      "activations/layer23_attention_weight_max": 33.74256134033203,
      "activations/layer23_attention_weight_min": -26.872045516967773,
      "activations/layer2_attention_weight_max": 35.542640686035156,
      "activations/layer2_attention_weight_min": -33.55137252807617,
      "activations/layer3_attention_weight_max": 102.51968383789062,
      "activations/layer3_attention_weight_min": -94.88098907470703,
      "activations/layer4_attention_weight_max": 112.78729248046875,
      "activations/layer4_attention_weight_min": -108.10516357421875,
      "activations/layer5_attention_weight_max": 53.164703369140625,
      "activations/layer5_attention_weight_min": -61.93185043334961,
      "activations/layer6_attention_weight_max": 41.79475402832031,
      "activations/layer6_attention_weight_min": -44.81827926635742,
      "activations/layer7_attention_weight_max": 89.4498519897461,
      "activations/layer7_attention_weight_min": -94.13404846191406,
      "activations/layer8_attention_weight_max": 39.01040267944336,
      "activations/layer8_attention_weight_min": -40.24152374267578,
      "activations/layer9_attention_weight_max": 30.373519897460938,
      "activations/layer9_attention_weight_min": -32.91154098510742,
      "epoch": 13.07,
      "learning_rate": 6.63780303030303e-05,
      "loss": 2.7614,
      "step": 224950
    },
    {
      "activations/layer0_attention_weight_max": 15.18581485748291,
      "activations/layer0_attention_weight_min": -11.794408798217773,
      "activations/layer10_attention_weight_max": 31.642431259155273,
      "activations/layer10_attention_weight_min": -31.98378562927246,
      "activations/layer11_attention_weight_max": 30.334930419921875,
      "activations/layer11_attention_weight_min": -31.409156799316406,
      "activations/layer12_attention_weight_max": 24.691688537597656,
      "activations/layer12_attention_weight_min": -25.45234489440918,
      "activations/layer13_attention_weight_max": 37.90919876098633,
      "activations/layer13_attention_weight_min": -35.55218505859375,
      "activations/layer14_attention_weight_max": 40.97568130493164,
      "activations/layer14_attention_weight_min": -35.75315475463867,
      "activations/layer15_attention_weight_max": 35.33872985839844,
      "activations/layer15_attention_weight_min": -36.06260299682617,
      "activations/layer16_attention_weight_max": 30.204069137573242,
      "activations/layer16_attention_weight_min": -30.02674102783203,
      "activations/layer17_attention_weight_max": 31.570159912109375,
      "activations/layer17_attention_weight_min": -26.036800384521484,
      "activations/layer18_attention_weight_max": 31.239492416381836,
      "activations/layer18_attention_weight_min": -23.436664581298828,
      "activations/layer19_attention_weight_max": 34.68494415283203,
      "activations/layer19_attention_weight_min": -32.3637580871582,
      "activations/layer1_attention_weight_max": 16.98229217529297,
      "activations/layer1_attention_weight_min": -16.139009475708008,
      "activations/layer20_attention_weight_max": 29.448219299316406,
      "activations/layer20_attention_weight_min": -26.002988815307617,
      "activations/layer21_attention_weight_max": 29.982030868530273,
      "activations/layer21_attention_weight_min": -24.300992965698242,
      "activations/layer22_attention_weight_max": 44.68797302246094,
      "activations/layer22_attention_weight_min": -31.841054916381836,
      "activations/layer23_attention_weight_max": 35.39633560180664,
      "activations/layer23_attention_weight_min": -25.916767120361328,
      "activations/layer2_attention_weight_max": 33.065006256103516,
      "activations/layer2_attention_weight_min": -32.66497802734375,
      "activations/layer3_attention_weight_max": 94.71902465820312,
      "activations/layer3_attention_weight_min": -98.46293640136719,
      "activations/layer4_attention_weight_max": 107.0427017211914,
      "activations/layer4_attention_weight_min": -112.97784423828125,
      "activations/layer5_attention_weight_max": 48.51760482788086,
      "activations/layer5_attention_weight_min": -62.29261016845703,
      "activations/layer6_attention_weight_max": 42.67906188964844,
      "activations/layer6_attention_weight_min": -44.90291213989258,
      "activations/layer7_attention_weight_max": 88.04461669921875,
      "activations/layer7_attention_weight_min": -90.27113342285156,
      "activations/layer8_attention_weight_max": 38.2177619934082,
      "activations/layer8_attention_weight_min": -38.83732986450195,
      "activations/layer9_attention_weight_max": 31.876773834228516,
      "activations/layer9_attention_weight_min": -31.50772476196289,
      "epoch": 13.07,
      "learning_rate": 6.63590909090909e-05,
      "loss": 2.7496,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4736,
      "eval_samples_per_second": 506.748,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4736,
      "eval_openwebtext_samples_per_second": 506.748,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.954,
      "eval_wikitext_samples_per_second": 233.363,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_lambada_loss": 2.4765625,
      "eval_lambada_ppl": 11.900286792057758,
      "eval_lambada_runtime": 9.565,
      "eval_lambada_samples_per_second": 509.044,
      "step": 225000
    },
    {
      "activations/layer0_attention_weight_max": 15.531902313232422,
      "activations/layer0_attention_weight_min": -12.198688507080078,
      "activations/layer10_attention_weight_max": 37.79931640625,
      "activations/layer10_attention_weight_min": -35.13603591918945,
      "activations/layer11_attention_weight_max": 39.68035125732422,
      "activations/layer11_attention_weight_min": -34.88432312011719,
      "activations/layer12_attention_weight_max": 29.060941696166992,
      "activations/layer12_attention_weight_min": -27.184810638427734,
      "activations/layer13_attention_weight_max": 45.341590881347656,
      "activations/layer13_attention_weight_min": -37.2874641418457,
      "activations/layer14_attention_weight_max": 53.14507293701172,
      "activations/layer14_attention_weight_min": -43.09107971191406,
      "activations/layer15_attention_weight_max": 39.39479446411133,
      "activations/layer15_attention_weight_min": -35.57149887084961,
      "activations/layer16_attention_weight_max": 33.455322265625,
      "activations/layer16_attention_weight_min": -28.246986389160156,
      "activations/layer17_attention_weight_max": 33.90194320678711,
      "activations/layer17_attention_weight_min": -26.25511360168457,
      "activations/layer18_attention_weight_max": 34.23553466796875,
      "activations/layer18_attention_weight_min": -25.794734954833984,
      "activations/layer19_attention_weight_max": 42.021522521972656,
      "activations/layer19_attention_weight_min": -33.76484298706055,
      "activations/layer1_attention_weight_max": 16.201539993286133,
      "activations/layer1_attention_weight_min": -14.385991096496582,
      "activations/layer20_attention_weight_max": 37.3113899230957,
      "activations/layer20_attention_weight_min": -26.238595962524414,
      "activations/layer21_attention_weight_max": 31.796611785888672,
      "activations/layer21_attention_weight_min": -25.695322036743164,
      "activations/layer22_attention_weight_max": 52.642478942871094,
      "activations/layer22_attention_weight_min": -32.30940628051758,
      "activations/layer23_attention_weight_max": 41.98869323730469,
      "activations/layer23_attention_weight_min": -24.968902587890625,
      "activations/layer2_attention_weight_max": 34.63451385498047,
      "activations/layer2_attention_weight_min": -32.9150276184082,
      "activations/layer3_attention_weight_max": 100.4503402709961,
      "activations/layer3_attention_weight_min": -103.86719512939453,
      "activations/layer4_attention_weight_max": 113.96337890625,
      "activations/layer4_attention_weight_min": -116.08397674560547,
      "activations/layer5_attention_weight_max": 52.22506332397461,
      "activations/layer5_attention_weight_min": -64.60823822021484,
      "activations/layer6_attention_weight_max": 45.93936538696289,
      "activations/layer6_attention_weight_min": -50.084144592285156,
      "activations/layer7_attention_weight_max": 96.41927337646484,
      "activations/layer7_attention_weight_min": -99.4404067993164,
      "activations/layer8_attention_weight_max": 42.869693756103516,
      "activations/layer8_attention_weight_min": -46.19605255126953,
      "activations/layer9_attention_weight_max": 35.930137634277344,
      "activations/layer9_attention_weight_min": -35.793033599853516,
      "epoch": 13.08,
      "learning_rate": 6.63401515151515e-05,
      "loss": 2.7773,
      "step": 225050
    },
    {
      "activations/layer0_attention_weight_max": 15.490966796875,
      "activations/layer0_attention_weight_min": -11.62948226928711,
      "activations/layer10_attention_weight_max": 31.32157325744629,
      "activations/layer10_attention_weight_min": -31.94008445739746,
      "activations/layer11_attention_weight_max": 33.463924407958984,
      "activations/layer11_attention_weight_min": -30.8358154296875,
      "activations/layer12_attention_weight_max": 24.435495376586914,
      "activations/layer12_attention_weight_min": -25.760581970214844,
      "activations/layer13_attention_weight_max": 38.896583557128906,
      "activations/layer13_attention_weight_min": -33.74584197998047,
      "activations/layer14_attention_weight_max": 44.6259651184082,
      "activations/layer14_attention_weight_min": -36.3820915222168,
      "activations/layer15_attention_weight_max": 38.201942443847656,
      "activations/layer15_attention_weight_min": -32.61109924316406,
      "activations/layer16_attention_weight_max": 32.79943084716797,
      "activations/layer16_attention_weight_min": -29.070478439331055,
      "activations/layer17_attention_weight_max": 34.03101348876953,
      "activations/layer17_attention_weight_min": -25.953948974609375,
      "activations/layer18_attention_weight_max": 35.06884002685547,
      "activations/layer18_attention_weight_min": -22.77553939819336,
      "activations/layer19_attention_weight_max": 35.32291030883789,
      "activations/layer19_attention_weight_min": -31.207534790039062,
      "activations/layer1_attention_weight_max": 16.775737762451172,
      "activations/layer1_attention_weight_min": -15.698221206665039,
      "activations/layer20_attention_weight_max": 28.335691452026367,
      "activations/layer20_attention_weight_min": -25.1396484375,
      "activations/layer21_attention_weight_max": 30.332700729370117,
      "activations/layer21_attention_weight_min": -25.613662719726562,
      "activations/layer22_attention_weight_max": 45.556724548339844,
      "activations/layer22_attention_weight_min": -31.511512756347656,
      "activations/layer23_attention_weight_max": 35.75284957885742,
      "activations/layer23_attention_weight_min": -26.66718864440918,
      "activations/layer2_attention_weight_max": 33.644126892089844,
      "activations/layer2_attention_weight_min": -32.393516540527344,
      "activations/layer3_attention_weight_max": 93.32551574707031,
      "activations/layer3_attention_weight_min": -95.26168823242188,
      "activations/layer4_attention_weight_max": 111.6868667602539,
      "activations/layer4_attention_weight_min": -109.16658782958984,
      "activations/layer5_attention_weight_max": 53.50726318359375,
      "activations/layer5_attention_weight_min": -58.20275115966797,
      "activations/layer6_attention_weight_max": 43.99762725830078,
      "activations/layer6_attention_weight_min": -46.45302963256836,
      "activations/layer7_attention_weight_max": 83.30262756347656,
      "activations/layer7_attention_weight_min": -83.9968490600586,
      "activations/layer8_attention_weight_max": 40.08548355102539,
      "activations/layer8_attention_weight_min": -40.89624786376953,
      "activations/layer9_attention_weight_max": 30.70883560180664,
      "activations/layer9_attention_weight_min": -33.309715270996094,
      "epoch": 13.08,
      "learning_rate": 6.632121212121212e-05,
      "loss": 2.7587,
      "step": 225100
    },
    {
      "activations/layer0_attention_weight_max": 15.970925331115723,
      "activations/layer0_attention_weight_min": -12.440411567687988,
      "activations/layer10_attention_weight_max": 30.235435485839844,
      "activations/layer10_attention_weight_min": -33.42286682128906,
      "activations/layer11_attention_weight_max": 30.244253158569336,
      "activations/layer11_attention_weight_min": -31.806163787841797,
      "activations/layer12_attention_weight_max": 25.07308578491211,
      "activations/layer12_attention_weight_min": -24.258371353149414,
      "activations/layer13_attention_weight_max": 38.03053665161133,
      "activations/layer13_attention_weight_min": -34.85413360595703,
      "activations/layer14_attention_weight_max": 43.3160285949707,
      "activations/layer14_attention_weight_min": -39.03255081176758,
      "activations/layer15_attention_weight_max": 37.555763244628906,
      "activations/layer15_attention_weight_min": -32.96870040893555,
      "activations/layer16_attention_weight_max": 35.958091735839844,
      "activations/layer16_attention_weight_min": -27.697046279907227,
      "activations/layer17_attention_weight_max": 30.326684951782227,
      "activations/layer17_attention_weight_min": -26.64202117919922,
      "activations/layer18_attention_weight_max": 30.605960845947266,
      "activations/layer18_attention_weight_min": -27.7561092376709,
      "activations/layer19_attention_weight_max": 38.43474578857422,
      "activations/layer19_attention_weight_min": -31.80668067932129,
      "activations/layer1_attention_weight_max": 16.575056076049805,
      "activations/layer1_attention_weight_min": -14.44286060333252,
      "activations/layer20_attention_weight_max": 30.81264305114746,
      "activations/layer20_attention_weight_min": -26.474285125732422,
      "activations/layer21_attention_weight_max": 29.86050796508789,
      "activations/layer21_attention_weight_min": -25.027124404907227,
      "activations/layer22_attention_weight_max": 46.076480865478516,
      "activations/layer22_attention_weight_min": -30.249528884887695,
      "activations/layer23_attention_weight_max": 35.47915267944336,
      "activations/layer23_attention_weight_min": -24.733577728271484,
      "activations/layer2_attention_weight_max": 30.82118034362793,
      "activations/layer2_attention_weight_min": -31.994171142578125,
      "activations/layer3_attention_weight_max": 95.95896911621094,
      "activations/layer3_attention_weight_min": -96.73733520507812,
      "activations/layer4_attention_weight_max": 109.37625885009766,
      "activations/layer4_attention_weight_min": -108.7281494140625,
      "activations/layer5_attention_weight_max": 55.811424255371094,
      "activations/layer5_attention_weight_min": -60.772705078125,
      "activations/layer6_attention_weight_max": 45.70427322387695,
      "activations/layer6_attention_weight_min": -46.330684661865234,
      "activations/layer7_attention_weight_max": 86.58905029296875,
      "activations/layer7_attention_weight_min": -90.12649536132812,
      "activations/layer8_attention_weight_max": 39.52241897583008,
      "activations/layer8_attention_weight_min": -40.36267852783203,
      "activations/layer9_attention_weight_max": 30.464658737182617,
      "activations/layer9_attention_weight_min": -32.59829330444336,
      "epoch": 13.08,
      "learning_rate": 6.630227272727272e-05,
      "loss": 2.7465,
      "step": 225150
    },
    {
      "activations/layer0_attention_weight_max": 16.189327239990234,
      "activations/layer0_attention_weight_min": -12.320514678955078,
      "activations/layer10_attention_weight_max": 35.0509033203125,
      "activations/layer10_attention_weight_min": -36.129638671875,
      "activations/layer11_attention_weight_max": 35.77006530761719,
      "activations/layer11_attention_weight_min": -34.34382629394531,
      "activations/layer12_attention_weight_max": 27.00650978088379,
      "activations/layer12_attention_weight_min": -25.93389320373535,
      "activations/layer13_attention_weight_max": 40.25788116455078,
      "activations/layer13_attention_weight_min": -38.402854919433594,
      "activations/layer14_attention_weight_max": 46.26726531982422,
      "activations/layer14_attention_weight_min": -39.7132453918457,
      "activations/layer15_attention_weight_max": 39.34073257446289,
      "activations/layer15_attention_weight_min": -34.512123107910156,
      "activations/layer16_attention_weight_max": 32.62897491455078,
      "activations/layer16_attention_weight_min": -27.241159439086914,
      "activations/layer17_attention_weight_max": 33.894020080566406,
      "activations/layer17_attention_weight_min": -25.450138092041016,
      "activations/layer18_attention_weight_max": 31.2618350982666,
      "activations/layer18_attention_weight_min": -24.7215633392334,
      "activations/layer19_attention_weight_max": 36.33407211303711,
      "activations/layer19_attention_weight_min": -33.22763442993164,
      "activations/layer1_attention_weight_max": 16.200387954711914,
      "activations/layer1_attention_weight_min": -15.362390518188477,
      "activations/layer20_attention_weight_max": 32.2906494140625,
      "activations/layer20_attention_weight_min": -24.916423797607422,
      "activations/layer21_attention_weight_max": 30.035831451416016,
      "activations/layer21_attention_weight_min": -24.699169158935547,
      "activations/layer22_attention_weight_max": 44.0793342590332,
      "activations/layer22_attention_weight_min": -32.474891662597656,
      "activations/layer23_attention_weight_max": 36.08138656616211,
      "activations/layer23_attention_weight_min": -25.530851364135742,
      "activations/layer2_attention_weight_max": 33.66684341430664,
      "activations/layer2_attention_weight_min": -32.69192123413086,
      "activations/layer3_attention_weight_max": 98.45330047607422,
      "activations/layer3_attention_weight_min": -98.54745483398438,
      "activations/layer4_attention_weight_max": 112.25483703613281,
      "activations/layer4_attention_weight_min": -113.9906005859375,
      "activations/layer5_attention_weight_max": 52.75341033935547,
      "activations/layer5_attention_weight_min": -64.07743072509766,
      "activations/layer6_attention_weight_max": 44.259403228759766,
      "activations/layer6_attention_weight_min": -48.684452056884766,
      "activations/layer7_attention_weight_max": 91.48851776123047,
      "activations/layer7_attention_weight_min": -94.92574310302734,
      "activations/layer8_attention_weight_max": 42.2459602355957,
      "activations/layer8_attention_weight_min": -43.48257827758789,
      "activations/layer9_attention_weight_max": 34.487911224365234,
      "activations/layer9_attention_weight_min": -35.363826751708984,
      "epoch": 13.09,
      "learning_rate": 6.628333333333332e-05,
      "loss": 2.764,
      "step": 225200
    },
    {
      "activations/layer0_attention_weight_max": 16.61254119873047,
      "activations/layer0_attention_weight_min": -12.509588241577148,
      "activations/layer10_attention_weight_max": 30.366939544677734,
      "activations/layer10_attention_weight_min": -31.978960037231445,
      "activations/layer11_attention_weight_max": 31.612133026123047,
      "activations/layer11_attention_weight_min": -32.035308837890625,
      "activations/layer12_attention_weight_max": 24.19508171081543,
      "activations/layer12_attention_weight_min": -26.01797866821289,
      "activations/layer13_attention_weight_max": 36.06486511230469,
      "activations/layer13_attention_weight_min": -34.36517333984375,
      "activations/layer14_attention_weight_max": 42.242088317871094,
      "activations/layer14_attention_weight_min": -37.12144088745117,
      "activations/layer15_attention_weight_max": 33.942604064941406,
      "activations/layer15_attention_weight_min": -33.37627029418945,
      "activations/layer16_attention_weight_max": 27.83380699157715,
      "activations/layer16_attention_weight_min": -28.70728874206543,
      "activations/layer17_attention_weight_max": 28.186302185058594,
      "activations/layer17_attention_weight_min": -25.781654357910156,
      "activations/layer18_attention_weight_max": 28.598304748535156,
      "activations/layer18_attention_weight_min": -24.06984519958496,
      "activations/layer19_attention_weight_max": 29.347875595092773,
      "activations/layer19_attention_weight_min": -30.363880157470703,
      "activations/layer1_attention_weight_max": 15.662254333496094,
      "activations/layer1_attention_weight_min": -14.21392822265625,
      "activations/layer20_attention_weight_max": 28.4337215423584,
      "activations/layer20_attention_weight_min": -25.132505416870117,
      "activations/layer21_attention_weight_max": 26.723997116088867,
      "activations/layer21_attention_weight_min": -23.949596405029297,
      "activations/layer22_attention_weight_max": 41.58924865722656,
      "activations/layer22_attention_weight_min": -29.583602905273438,
      "activations/layer23_attention_weight_max": 30.826309204101562,
      "activations/layer23_attention_weight_min": -25.605838775634766,
      "activations/layer2_attention_weight_max": 32.4749755859375,
      "activations/layer2_attention_weight_min": -32.65392303466797,
      "activations/layer3_attention_weight_max": 94.74028778076172,
      "activations/layer3_attention_weight_min": -97.8406982421875,
      "activations/layer4_attention_weight_max": 113.28742980957031,
      "activations/layer4_attention_weight_min": -107.5110092163086,
      "activations/layer5_attention_weight_max": 50.7618408203125,
      "activations/layer5_attention_weight_min": -60.063987731933594,
      "activations/layer6_attention_weight_max": 42.13410186767578,
      "activations/layer6_attention_weight_min": -46.5900993347168,
      "activations/layer7_attention_weight_max": 87.05896759033203,
      "activations/layer7_attention_weight_min": -87.1976089477539,
      "activations/layer8_attention_weight_max": 40.28197479248047,
      "activations/layer8_attention_weight_min": -42.96337890625,
      "activations/layer9_attention_weight_max": 31.64009666442871,
      "activations/layer9_attention_weight_min": -31.47052574157715,
      "epoch": 13.09,
      "learning_rate": 6.626439393939394e-05,
      "loss": 2.768,
      "step": 225250
    },
    {
      "activations/layer0_attention_weight_max": 16.71635627746582,
      "activations/layer0_attention_weight_min": -12.2128267288208,
      "activations/layer10_attention_weight_max": 34.0955924987793,
      "activations/layer10_attention_weight_min": -33.39137649536133,
      "activations/layer11_attention_weight_max": 33.94940948486328,
      "activations/layer11_attention_weight_min": -33.125492095947266,
      "activations/layer12_attention_weight_max": 38.47793960571289,
      "activations/layer12_attention_weight_min": -27.654924392700195,
      "activations/layer13_attention_weight_max": 46.0512809753418,
      "activations/layer13_attention_weight_min": -34.95320129394531,
      "activations/layer14_attention_weight_max": 40.91924285888672,
      "activations/layer14_attention_weight_min": -38.632530212402344,
      "activations/layer15_attention_weight_max": 39.45690155029297,
      "activations/layer15_attention_weight_min": -32.91495132446289,
      "activations/layer16_attention_weight_max": 31.1141357421875,
      "activations/layer16_attention_weight_min": -27.892610549926758,
      "activations/layer17_attention_weight_max": 32.08882522583008,
      "activations/layer17_attention_weight_min": -25.154165267944336,
      "activations/layer18_attention_weight_max": 35.39375686645508,
      "activations/layer18_attention_weight_min": -23.770734786987305,
      "activations/layer19_attention_weight_max": 36.07917022705078,
      "activations/layer19_attention_weight_min": -32.4176025390625,
      "activations/layer1_attention_weight_max": 18.19522476196289,
      "activations/layer1_attention_weight_min": -17.593339920043945,
      "activations/layer20_attention_weight_max": 32.52788162231445,
      "activations/layer20_attention_weight_min": -27.053749084472656,
      "activations/layer21_attention_weight_max": 35.15876388549805,
      "activations/layer21_attention_weight_min": -25.275827407836914,
      "activations/layer22_attention_weight_max": 44.67731475830078,
      "activations/layer22_attention_weight_min": -31.188995361328125,
      "activations/layer23_attention_weight_max": 35.09774398803711,
      "activations/layer23_attention_weight_min": -24.265064239501953,
      "activations/layer2_attention_weight_max": 34.456085205078125,
      "activations/layer2_attention_weight_min": -34.612098693847656,
      "activations/layer3_attention_weight_max": 95.52543640136719,
      "activations/layer3_attention_weight_min": -100.15335845947266,
      "activations/layer4_attention_weight_max": 109.61290740966797,
      "activations/layer4_attention_weight_min": -110.97173309326172,
      "activations/layer5_attention_weight_max": 52.76620101928711,
      "activations/layer5_attention_weight_min": -63.917503356933594,
      "activations/layer6_attention_weight_max": 41.85354232788086,
      "activations/layer6_attention_weight_min": -45.91301345825195,
      "activations/layer7_attention_weight_max": 88.74103546142578,
      "activations/layer7_attention_weight_min": -88.77742767333984,
      "activations/layer8_attention_weight_max": 39.94143295288086,
      "activations/layer8_attention_weight_min": -42.25161361694336,
      "activations/layer9_attention_weight_max": 33.275936126708984,
      "activations/layer9_attention_weight_min": -35.00961685180664,
      "epoch": 13.09,
      "learning_rate": 6.624545454545454e-05,
      "loss": 2.7574,
      "step": 225300
    },
    {
      "activations/layer0_attention_weight_max": 17.334060668945312,
      "activations/layer0_attention_weight_min": -12.109724998474121,
      "activations/layer10_attention_weight_max": 34.43966293334961,
      "activations/layer10_attention_weight_min": -35.94453811645508,
      "activations/layer11_attention_weight_max": 32.39839172363281,
      "activations/layer11_attention_weight_min": -32.922054290771484,
      "activations/layer12_attention_weight_max": 25.604774475097656,
      "activations/layer12_attention_weight_min": -26.986061096191406,
      "activations/layer13_attention_weight_max": 40.17058181762695,
      "activations/layer13_attention_weight_min": -38.26460266113281,
      "activations/layer14_attention_weight_max": 42.21173858642578,
      "activations/layer14_attention_weight_min": -40.68537902832031,
      "activations/layer15_attention_weight_max": 34.753353118896484,
      "activations/layer15_attention_weight_min": -33.346527099609375,
      "activations/layer16_attention_weight_max": 29.63344955444336,
      "activations/layer16_attention_weight_min": -30.664648056030273,
      "activations/layer17_attention_weight_max": 31.43099021911621,
      "activations/layer17_attention_weight_min": -29.313859939575195,
      "activations/layer18_attention_weight_max": 32.20069122314453,
      "activations/layer18_attention_weight_min": -26.6900691986084,
      "activations/layer19_attention_weight_max": 32.10834503173828,
      "activations/layer19_attention_weight_min": -31.13755226135254,
      "activations/layer1_attention_weight_max": 16.44342803955078,
      "activations/layer1_attention_weight_min": -14.476692199707031,
      "activations/layer20_attention_weight_max": 29.987110137939453,
      "activations/layer20_attention_weight_min": -25.3951473236084,
      "activations/layer21_attention_weight_max": 31.267471313476562,
      "activations/layer21_attention_weight_min": -24.185129165649414,
      "activations/layer22_attention_weight_max": 43.59209442138672,
      "activations/layer22_attention_weight_min": -30.18804931640625,
      "activations/layer23_attention_weight_max": 34.93759536743164,
      "activations/layer23_attention_weight_min": -26.291763305664062,
      "activations/layer2_attention_weight_max": 32.6031379699707,
      "activations/layer2_attention_weight_min": -33.078094482421875,
      "activations/layer3_attention_weight_max": 95.69962310791016,
      "activations/layer3_attention_weight_min": -98.05072021484375,
      "activations/layer4_attention_weight_max": 108.83719635009766,
      "activations/layer4_attention_weight_min": -111.13862609863281,
      "activations/layer5_attention_weight_max": 49.743797302246094,
      "activations/layer5_attention_weight_min": -61.6529541015625,
      "activations/layer6_attention_weight_max": 43.25284194946289,
      "activations/layer6_attention_weight_min": -47.36589813232422,
      "activations/layer7_attention_weight_max": 89.72799682617188,
      "activations/layer7_attention_weight_min": -99.175048828125,
      "activations/layer8_attention_weight_max": 40.57039260864258,
      "activations/layer8_attention_weight_min": -43.60710144042969,
      "activations/layer9_attention_weight_max": 33.984466552734375,
      "activations/layer9_attention_weight_min": -34.483402252197266,
      "epoch": 13.09,
      "learning_rate": 6.622651515151514e-05,
      "loss": 2.7693,
      "step": 225350
    },
    {
      "activations/layer0_attention_weight_max": 15.501056671142578,
      "activations/layer0_attention_weight_min": -12.34324836730957,
      "activations/layer10_attention_weight_max": 32.47346115112305,
      "activations/layer10_attention_weight_min": -34.68363571166992,
      "activations/layer11_attention_weight_max": 33.639644622802734,
      "activations/layer11_attention_weight_min": -33.46821212768555,
      "activations/layer12_attention_weight_max": 26.89557647705078,
      "activations/layer12_attention_weight_min": -25.176706314086914,
      "activations/layer13_attention_weight_max": 40.885337829589844,
      "activations/layer13_attention_weight_min": -36.49067687988281,
      "activations/layer14_attention_weight_max": 42.59496307373047,
      "activations/layer14_attention_weight_min": -36.04148864746094,
      "activations/layer15_attention_weight_max": 38.594085693359375,
      "activations/layer15_attention_weight_min": -31.901132583618164,
      "activations/layer16_attention_weight_max": 32.098785400390625,
      "activations/layer16_attention_weight_min": -28.359766006469727,
      "activations/layer17_attention_weight_max": 34.7580451965332,
      "activations/layer17_attention_weight_min": -25.398210525512695,
      "activations/layer18_attention_weight_max": 35.68870162963867,
      "activations/layer18_attention_weight_min": -22.96790313720703,
      "activations/layer19_attention_weight_max": 36.70687484741211,
      "activations/layer19_attention_weight_min": -31.983407974243164,
      "activations/layer1_attention_weight_max": 16.785419464111328,
      "activations/layer1_attention_weight_min": -14.974896430969238,
      "activations/layer20_attention_weight_max": 30.528947830200195,
      "activations/layer20_attention_weight_min": -24.36888313293457,
      "activations/layer21_attention_weight_max": 30.04213523864746,
      "activations/layer21_attention_weight_min": -24.50151824951172,
      "activations/layer22_attention_weight_max": 42.56291961669922,
      "activations/layer22_attention_weight_min": -30.81863784790039,
      "activations/layer23_attention_weight_max": 36.04632568359375,
      "activations/layer23_attention_weight_min": -26.701663970947266,
      "activations/layer2_attention_weight_max": 34.18083953857422,
      "activations/layer2_attention_weight_min": -35.185935974121094,
      "activations/layer3_attention_weight_max": 104.45043182373047,
      "activations/layer3_attention_weight_min": -101.63612365722656,
      "activations/layer4_attention_weight_max": 113.8607406616211,
      "activations/layer4_attention_weight_min": -118.92980194091797,
      "activations/layer5_attention_weight_max": 53.31382369995117,
      "activations/layer5_attention_weight_min": -61.740386962890625,
      "activations/layer6_attention_weight_max": 43.320533752441406,
      "activations/layer6_attention_weight_min": -50.28145217895508,
      "activations/layer7_attention_weight_max": 92.05962371826172,
      "activations/layer7_attention_weight_min": -93.7577133178711,
      "activations/layer8_attention_weight_max": 42.17942810058594,
      "activations/layer8_attention_weight_min": -44.5634880065918,
      "activations/layer9_attention_weight_max": 34.25745391845703,
      "activations/layer9_attention_weight_min": -35.87299346923828,
      "epoch": 13.1,
      "learning_rate": 6.620757575757576e-05,
      "loss": 2.7445,
      "step": 225400
    },
    {
      "activations/layer0_attention_weight_max": 15.596461296081543,
      "activations/layer0_attention_weight_min": -11.765400886535645,
      "activations/layer10_attention_weight_max": 32.12731170654297,
      "activations/layer10_attention_weight_min": -34.41341018676758,
      "activations/layer11_attention_weight_max": 34.22817611694336,
      "activations/layer11_attention_weight_min": -33.75235366821289,
      "activations/layer12_attention_weight_max": 22.777021408081055,
      "activations/layer12_attention_weight_min": -23.99929428100586,
      "activations/layer13_attention_weight_max": 35.195457458496094,
      "activations/layer13_attention_weight_min": -34.0339469909668,
      "activations/layer14_attention_weight_max": 37.734283447265625,
      "activations/layer14_attention_weight_min": -36.727840423583984,
      "activations/layer15_attention_weight_max": 35.6623649597168,
      "activations/layer15_attention_weight_min": -33.6221809387207,
      "activations/layer16_attention_weight_max": 27.563241958618164,
      "activations/layer16_attention_weight_min": -28.180822372436523,
      "activations/layer17_attention_weight_max": 28.898414611816406,
      "activations/layer17_attention_weight_min": -25.712615966796875,
      "activations/layer18_attention_weight_max": 29.399600982666016,
      "activations/layer18_attention_weight_min": -22.99294662475586,
      "activations/layer19_attention_weight_max": 32.191612243652344,
      "activations/layer19_attention_weight_min": -28.786727905273438,
      "activations/layer1_attention_weight_max": 15.954296112060547,
      "activations/layer1_attention_weight_min": -17.826648712158203,
      "activations/layer20_attention_weight_max": 27.738264083862305,
      "activations/layer20_attention_weight_min": -23.798381805419922,
      "activations/layer21_attention_weight_max": 28.036529541015625,
      "activations/layer21_attention_weight_min": -24.830455780029297,
      "activations/layer22_attention_weight_max": 44.737571716308594,
      "activations/layer22_attention_weight_min": -31.164472579956055,
      "activations/layer23_attention_weight_max": 33.5890007019043,
      "activations/layer23_attention_weight_min": -23.591014862060547,
      "activations/layer2_attention_weight_max": 34.70213317871094,
      "activations/layer2_attention_weight_min": -34.52751922607422,
      "activations/layer3_attention_weight_max": 95.24502563476562,
      "activations/layer3_attention_weight_min": -97.89209747314453,
      "activations/layer4_attention_weight_max": 106.21150970458984,
      "activations/layer4_attention_weight_min": -104.94288635253906,
      "activations/layer5_attention_weight_max": 50.925445556640625,
      "activations/layer5_attention_weight_min": -60.89576721191406,
      "activations/layer6_attention_weight_max": 42.03908157348633,
      "activations/layer6_attention_weight_min": -45.22697830200195,
      "activations/layer7_attention_weight_max": 95.54100799560547,
      "activations/layer7_attention_weight_min": -93.61123657226562,
      "activations/layer8_attention_weight_max": 39.053829193115234,
      "activations/layer8_attention_weight_min": -42.98622512817383,
      "activations/layer9_attention_weight_max": 34.359859466552734,
      "activations/layer9_attention_weight_min": -32.73023223876953,
      "epoch": 13.1,
      "learning_rate": 6.618863636363636e-05,
      "loss": 2.776,
      "step": 225450
    },
    {
      "activations/layer0_attention_weight_max": 15.996458053588867,
      "activations/layer0_attention_weight_min": -12.22293472290039,
      "activations/layer10_attention_weight_max": 30.235708236694336,
      "activations/layer10_attention_weight_min": -31.39333724975586,
      "activations/layer11_attention_weight_max": 29.510982513427734,
      "activations/layer11_attention_weight_min": -31.192276000976562,
      "activations/layer12_attention_weight_max": 22.880859375,
      "activations/layer12_attention_weight_min": -23.647336959838867,
      "activations/layer13_attention_weight_max": 34.052268981933594,
      "activations/layer13_attention_weight_min": -33.09950256347656,
      "activations/layer14_attention_weight_max": 39.973880767822266,
      "activations/layer14_attention_weight_min": -35.20505142211914,
      "activations/layer15_attention_weight_max": 35.67375564575195,
      "activations/layer15_attention_weight_min": -32.7507438659668,
      "activations/layer16_attention_weight_max": 28.079570770263672,
      "activations/layer16_attention_weight_min": -27.32243537902832,
      "activations/layer17_attention_weight_max": 32.08436584472656,
      "activations/layer17_attention_weight_min": -24.362781524658203,
      "activations/layer18_attention_weight_max": 29.93124008178711,
      "activations/layer18_attention_weight_min": -23.003820419311523,
      "activations/layer19_attention_weight_max": 30.384069442749023,
      "activations/layer19_attention_weight_min": -30.27125358581543,
      "activations/layer1_attention_weight_max": 16.12571907043457,
      "activations/layer1_attention_weight_min": -14.527694702148438,
      "activations/layer20_attention_weight_max": 28.447185516357422,
      "activations/layer20_attention_weight_min": -26.60161590576172,
      "activations/layer21_attention_weight_max": 28.1655330657959,
      "activations/layer21_attention_weight_min": -26.251628875732422,
      "activations/layer22_attention_weight_max": 39.92822265625,
      "activations/layer22_attention_weight_min": -32.142662048339844,
      "activations/layer23_attention_weight_max": 30.984573364257812,
      "activations/layer23_attention_weight_min": -23.994159698486328,
      "activations/layer2_attention_weight_max": 32.49660873413086,
      "activations/layer2_attention_weight_min": -32.044456481933594,
      "activations/layer3_attention_weight_max": 95.75007629394531,
      "activations/layer3_attention_weight_min": -99.13671112060547,
      "activations/layer4_attention_weight_max": 108.74608612060547,
      "activations/layer4_attention_weight_min": -113.26497650146484,
      "activations/layer5_attention_weight_max": 49.46031188964844,
      "activations/layer5_attention_weight_min": -62.87832260131836,
      "activations/layer6_attention_weight_max": 42.449058532714844,
      "activations/layer6_attention_weight_min": -47.73100662231445,
      "activations/layer7_attention_weight_max": 88.8729248046875,
      "activations/layer7_attention_weight_min": -90.01191711425781,
      "activations/layer8_attention_weight_max": 37.00189971923828,
      "activations/layer8_attention_weight_min": -39.64634704589844,
      "activations/layer9_attention_weight_max": 30.620624542236328,
      "activations/layer9_attention_weight_min": -31.33782958984375,
      "epoch": 13.1,
      "learning_rate": 6.616969696969697e-05,
      "loss": 2.7729,
      "step": 225500
    },
    {
      "activations/layer0_attention_weight_max": 15.963691711425781,
      "activations/layer0_attention_weight_min": -11.570297241210938,
      "activations/layer10_attention_weight_max": 30.074398040771484,
      "activations/layer10_attention_weight_min": -31.481298446655273,
      "activations/layer11_attention_weight_max": 29.484027862548828,
      "activations/layer11_attention_weight_min": -31.351917266845703,
      "activations/layer12_attention_weight_max": 23.705947875976562,
      "activations/layer12_attention_weight_min": -24.826190948486328,
      "activations/layer13_attention_weight_max": 34.962501525878906,
      "activations/layer13_attention_weight_min": -33.489097595214844,
      "activations/layer14_attention_weight_max": 40.776893615722656,
      "activations/layer14_attention_weight_min": -37.59269332885742,
      "activations/layer15_attention_weight_max": 35.28174591064453,
      "activations/layer15_attention_weight_min": -35.5309944152832,
      "activations/layer16_attention_weight_max": 27.042768478393555,
      "activations/layer16_attention_weight_min": -28.42927360534668,
      "activations/layer17_attention_weight_max": 30.125244140625,
      "activations/layer17_attention_weight_min": -27.174219131469727,
      "activations/layer18_attention_weight_max": 29.16010856628418,
      "activations/layer18_attention_weight_min": -25.78324317932129,
      "activations/layer19_attention_weight_max": 32.54283142089844,
      "activations/layer19_attention_weight_min": -32.65958786010742,
      "activations/layer1_attention_weight_max": 17.539609909057617,
      "activations/layer1_attention_weight_min": -16.135377883911133,
      "activations/layer20_attention_weight_max": 29.520904541015625,
      "activations/layer20_attention_weight_min": -26.828588485717773,
      "activations/layer21_attention_weight_max": 30.197710037231445,
      "activations/layer21_attention_weight_min": -24.3215389251709,
      "activations/layer22_attention_weight_max": 40.10730743408203,
      "activations/layer22_attention_weight_min": -30.84941864013672,
      "activations/layer23_attention_weight_max": 33.8587646484375,
      "activations/layer23_attention_weight_min": -25.49816131591797,
      "activations/layer2_attention_weight_max": 31.175899505615234,
      "activations/layer2_attention_weight_min": -31.968555450439453,
      "activations/layer3_attention_weight_max": 95.0356674194336,
      "activations/layer3_attention_weight_min": -96.241943359375,
      "activations/layer4_attention_weight_max": 109.1745834350586,
      "activations/layer4_attention_weight_min": -112.45259094238281,
      "activations/layer5_attention_weight_max": 54.92097091674805,
      "activations/layer5_attention_weight_min": -66.5010986328125,
      "activations/layer6_attention_weight_max": 42.497711181640625,
      "activations/layer6_attention_weight_min": -46.72931671142578,
      "activations/layer7_attention_weight_max": 86.46530151367188,
      "activations/layer7_attention_weight_min": -90.42610168457031,
      "activations/layer8_attention_weight_max": 37.312713623046875,
      "activations/layer8_attention_weight_min": -40.77504348754883,
      "activations/layer9_attention_weight_max": 29.149744033813477,
      "activations/layer9_attention_weight_min": -32.01716232299805,
      "epoch": 13.11,
      "learning_rate": 6.615075757575757e-05,
      "loss": 2.7744,
      "step": 225550
    },
    {
      "activations/layer0_attention_weight_max": 15.468379020690918,
      "activations/layer0_attention_weight_min": -11.843474388122559,
      "activations/layer10_attention_weight_max": 33.78266143798828,
      "activations/layer10_attention_weight_min": -32.95146179199219,
      "activations/layer11_attention_weight_max": 33.41838455200195,
      "activations/layer11_attention_weight_min": -31.655868530273438,
      "activations/layer12_attention_weight_max": 24.72210121154785,
      "activations/layer12_attention_weight_min": -25.715133666992188,
      "activations/layer13_attention_weight_max": 37.0377082824707,
      "activations/layer13_attention_weight_min": -36.41771697998047,
      "activations/layer14_attention_weight_max": 40.893924713134766,
      "activations/layer14_attention_weight_min": -39.039276123046875,
      "activations/layer15_attention_weight_max": 36.47783660888672,
      "activations/layer15_attention_weight_min": -36.2913818359375,
      "activations/layer16_attention_weight_max": 33.082130432128906,
      "activations/layer16_attention_weight_min": -29.52886962890625,
      "activations/layer17_attention_weight_max": 34.590171813964844,
      "activations/layer17_attention_weight_min": -27.242645263671875,
      "activations/layer18_attention_weight_max": 33.98632049560547,
      "activations/layer18_attention_weight_min": -25.76542854309082,
      "activations/layer19_attention_weight_max": 32.01746368408203,
      "activations/layer19_attention_weight_min": -34.54026412963867,
      "activations/layer1_attention_weight_max": 15.933201789855957,
      "activations/layer1_attention_weight_min": -15.680638313293457,
      "activations/layer20_attention_weight_max": 28.371952056884766,
      "activations/layer20_attention_weight_min": -29.419292449951172,
      "activations/layer21_attention_weight_max": 29.767385482788086,
      "activations/layer21_attention_weight_min": -25.920970916748047,
      "activations/layer22_attention_weight_max": 44.19070816040039,
      "activations/layer22_attention_weight_min": -30.592126846313477,
      "activations/layer23_attention_weight_max": 37.32904052734375,
      "activations/layer23_attention_weight_min": -27.765378952026367,
      "activations/layer2_attention_weight_max": 31.668258666992188,
      "activations/layer2_attention_weight_min": -32.14704132080078,
      "activations/layer3_attention_weight_max": 96.02128601074219,
      "activations/layer3_attention_weight_min": -96.46773529052734,
      "activations/layer4_attention_weight_max": 113.40282440185547,
      "activations/layer4_attention_weight_min": -114.9229507446289,
      "activations/layer5_attention_weight_max": 49.73579025268555,
      "activations/layer5_attention_weight_min": -64.98148345947266,
      "activations/layer6_attention_weight_max": 43.49197769165039,
      "activations/layer6_attention_weight_min": -47.92247772216797,
      "activations/layer7_attention_weight_max": 90.50713348388672,
      "activations/layer7_attention_weight_min": -91.70584869384766,
      "activations/layer8_attention_weight_max": 40.98012161254883,
      "activations/layer8_attention_weight_min": -41.82991409301758,
      "activations/layer9_attention_weight_max": 32.9981803894043,
      "activations/layer9_attention_weight_min": -34.037567138671875,
      "epoch": 13.11,
      "learning_rate": 6.613219696969696e-05,
      "loss": 2.7694,
      "step": 225600
    },
    {
      "activations/layer0_attention_weight_max": 16.63623046875,
      "activations/layer0_attention_weight_min": -12.065325736999512,
      "activations/layer10_attention_weight_max": 32.8948974609375,
      "activations/layer10_attention_weight_min": -33.24739456176758,
      "activations/layer11_attention_weight_max": 32.13124465942383,
      "activations/layer11_attention_weight_min": -32.841705322265625,
      "activations/layer12_attention_weight_max": 26.214059829711914,
      "activations/layer12_attention_weight_min": -26.012788772583008,
      "activations/layer13_attention_weight_max": 41.809730529785156,
      "activations/layer13_attention_weight_min": -35.587913513183594,
      "activations/layer14_attention_weight_max": 50.4833869934082,
      "activations/layer14_attention_weight_min": -36.73188018798828,
      "activations/layer15_attention_weight_max": 41.1378059387207,
      "activations/layer15_attention_weight_min": -33.322166442871094,
      "activations/layer16_attention_weight_max": 36.40057373046875,
      "activations/layer16_attention_weight_min": -28.231964111328125,
      "activations/layer17_attention_weight_max": 32.26955795288086,
      "activations/layer17_attention_weight_min": -27.65150260925293,
      "activations/layer18_attention_weight_max": 32.94461441040039,
      "activations/layer18_attention_weight_min": -24.22024154663086,
      "activations/layer19_attention_weight_max": 41.04512023925781,
      "activations/layer19_attention_weight_min": -30.54227638244629,
      "activations/layer1_attention_weight_max": 17.652496337890625,
      "activations/layer1_attention_weight_min": -14.36547565460205,
      "activations/layer20_attention_weight_max": 31.06786346435547,
      "activations/layer20_attention_weight_min": -26.626447677612305,
      "activations/layer21_attention_weight_max": 28.35869026184082,
      "activations/layer21_attention_weight_min": -27.15871238708496,
      "activations/layer22_attention_weight_max": 47.17854690551758,
      "activations/layer22_attention_weight_min": -29.800703048706055,
      "activations/layer23_attention_weight_max": 35.14570617675781,
      "activations/layer23_attention_weight_min": -25.9771728515625,
      "activations/layer2_attention_weight_max": 30.71230697631836,
      "activations/layer2_attention_weight_min": -32.15215301513672,
      "activations/layer3_attention_weight_max": 91.87973022460938,
      "activations/layer3_attention_weight_min": -92.044189453125,
      "activations/layer4_attention_weight_max": 107.74337005615234,
      "activations/layer4_attention_weight_min": -112.3066635131836,
      "activations/layer5_attention_weight_max": 50.29254150390625,
      "activations/layer5_attention_weight_min": -64.47687530517578,
      "activations/layer6_attention_weight_max": 45.87630844116211,
      "activations/layer6_attention_weight_min": -49.193763732910156,
      "activations/layer7_attention_weight_max": 90.2756576538086,
      "activations/layer7_attention_weight_min": -96.5195083618164,
      "activations/layer8_attention_weight_max": 38.89265823364258,
      "activations/layer8_attention_weight_min": -43.09550476074219,
      "activations/layer9_attention_weight_max": 33.64459228515625,
      "activations/layer9_attention_weight_min": -34.136451721191406,
      "epoch": 13.11,
      "learning_rate": 6.611325757575757e-05,
      "loss": 2.7598,
      "step": 225650
    },
    {
      "activations/layer0_attention_weight_max": 15.952181816101074,
      "activations/layer0_attention_weight_min": -12.213394165039062,
      "activations/layer10_attention_weight_max": 31.680673599243164,
      "activations/layer10_attention_weight_min": -33.19709396362305,
      "activations/layer11_attention_weight_max": 31.8702392578125,
      "activations/layer11_attention_weight_min": -33.58210754394531,
      "activations/layer12_attention_weight_max": 25.148101806640625,
      "activations/layer12_attention_weight_min": -25.759340286254883,
      "activations/layer13_attention_weight_max": 39.08936309814453,
      "activations/layer13_attention_weight_min": -37.1680908203125,
      "activations/layer14_attention_weight_max": 42.14453125,
      "activations/layer14_attention_weight_min": -38.42399978637695,
      "activations/layer15_attention_weight_max": 36.97352981567383,
      "activations/layer15_attention_weight_min": -33.959205627441406,
      "activations/layer16_attention_weight_max": 32.860191345214844,
      "activations/layer16_attention_weight_min": -28.267759323120117,
      "activations/layer17_attention_weight_max": 30.199106216430664,
      "activations/layer17_attention_weight_min": -24.942358016967773,
      "activations/layer18_attention_weight_max": 29.97021484375,
      "activations/layer18_attention_weight_min": -24.680463790893555,
      "activations/layer19_attention_weight_max": 35.76378631591797,
      "activations/layer19_attention_weight_min": -29.53473472595215,
      "activations/layer1_attention_weight_max": 16.148731231689453,
      "activations/layer1_attention_weight_min": -15.873079299926758,
      "activations/layer20_attention_weight_max": 32.245296478271484,
      "activations/layer20_attention_weight_min": -26.42433738708496,
      "activations/layer21_attention_weight_max": 30.370738983154297,
      "activations/layer21_attention_weight_min": -26.102128982543945,
      "activations/layer22_attention_weight_max": 41.76918411254883,
      "activations/layer22_attention_weight_min": -30.344026565551758,
      "activations/layer23_attention_weight_max": 35.66808319091797,
      "activations/layer23_attention_weight_min": -26.461524963378906,
      "activations/layer2_attention_weight_max": 34.73430633544922,
      "activations/layer2_attention_weight_min": -31.7128963470459,
      "activations/layer3_attention_weight_max": 93.266845703125,
      "activations/layer3_attention_weight_min": -90.5019302368164,
      "activations/layer4_attention_weight_max": 106.17567443847656,
      "activations/layer4_attention_weight_min": -106.13996887207031,
      "activations/layer5_attention_weight_max": 52.55532455444336,
      "activations/layer5_attention_weight_min": -60.50891876220703,
      "activations/layer6_attention_weight_max": 43.85396194458008,
      "activations/layer6_attention_weight_min": -46.90681457519531,
      "activations/layer7_attention_weight_max": 85.79481506347656,
      "activations/layer7_attention_weight_min": -94.6104507446289,
      "activations/layer8_attention_weight_max": 39.04745101928711,
      "activations/layer8_attention_weight_min": -42.91714096069336,
      "activations/layer9_attention_weight_max": 32.13186264038086,
      "activations/layer9_attention_weight_min": -34.6713752746582,
      "epoch": 13.11,
      "learning_rate": 6.609431818181818e-05,
      "loss": 2.7667,
      "step": 225700
    },
    {
      "activations/layer0_attention_weight_max": 15.889944076538086,
      "activations/layer0_attention_weight_min": -12.21822452545166,
      "activations/layer10_attention_weight_max": 36.75703811645508,
      "activations/layer10_attention_weight_min": -35.7900390625,
      "activations/layer11_attention_weight_max": 33.963645935058594,
      "activations/layer11_attention_weight_min": -34.26433181762695,
      "activations/layer12_attention_weight_max": 26.162508010864258,
      "activations/layer12_attention_weight_min": -25.55870819091797,
      "activations/layer13_attention_weight_max": 36.954673767089844,
      "activations/layer13_attention_weight_min": -34.078956604003906,
      "activations/layer14_attention_weight_max": 44.83893966674805,
      "activations/layer14_attention_weight_min": -37.354759216308594,
      "activations/layer15_attention_weight_max": 35.78921127319336,
      "activations/layer15_attention_weight_min": -32.11819839477539,
      "activations/layer16_attention_weight_max": 32.96429443359375,
      "activations/layer16_attention_weight_min": -28.08272361755371,
      "activations/layer17_attention_weight_max": 32.85982131958008,
      "activations/layer17_attention_weight_min": -26.03420639038086,
      "activations/layer18_attention_weight_max": 29.430641174316406,
      "activations/layer18_attention_weight_min": -24.138362884521484,
      "activations/layer19_attention_weight_max": 35.66669845581055,
      "activations/layer19_attention_weight_min": -31.750030517578125,
      "activations/layer1_attention_weight_max": 15.781106948852539,
      "activations/layer1_attention_weight_min": -15.494441032409668,
      "activations/layer20_attention_weight_max": 32.61328887939453,
      "activations/layer20_attention_weight_min": -28.147621154785156,
      "activations/layer21_attention_weight_max": 31.259239196777344,
      "activations/layer21_attention_weight_min": -25.958892822265625,
      "activations/layer22_attention_weight_max": 50.21709442138672,
      "activations/layer22_attention_weight_min": -30.56427001953125,
      "activations/layer23_attention_weight_max": 36.28251266479492,
      "activations/layer23_attention_weight_min": -24.973630905151367,
      "activations/layer2_attention_weight_max": 32.91004943847656,
      "activations/layer2_attention_weight_min": -32.148353576660156,
      "activations/layer3_attention_weight_max": 100.00946807861328,
      "activations/layer3_attention_weight_min": -97.49835205078125,
      "activations/layer4_attention_weight_max": 116.3087387084961,
      "activations/layer4_attention_weight_min": -114.20863342285156,
      "activations/layer5_attention_weight_max": 52.42713165283203,
      "activations/layer5_attention_weight_min": -59.742210388183594,
      "activations/layer6_attention_weight_max": 44.883934020996094,
      "activations/layer6_attention_weight_min": -48.49568176269531,
      "activations/layer7_attention_weight_max": 94.22398376464844,
      "activations/layer7_attention_weight_min": -94.79173278808594,
      "activations/layer8_attention_weight_max": 44.19907760620117,
      "activations/layer8_attention_weight_min": -47.28038787841797,
      "activations/layer9_attention_weight_max": 34.797847747802734,
      "activations/layer9_attention_weight_min": -35.48428726196289,
      "epoch": 13.12,
      "learning_rate": 6.607537878787878e-05,
      "loss": 2.7484,
      "step": 225750
    },
    {
      "activations/layer0_attention_weight_max": 16.163291931152344,
      "activations/layer0_attention_weight_min": -12.519012451171875,
      "activations/layer10_attention_weight_max": 32.75732421875,
      "activations/layer10_attention_weight_min": -34.143943786621094,
      "activations/layer11_attention_weight_max": 35.692909240722656,
      "activations/layer11_attention_weight_min": -33.82884979248047,
      "activations/layer12_attention_weight_max": 25.70372200012207,
      "activations/layer12_attention_weight_min": -27.137861251831055,
      "activations/layer13_attention_weight_max": 43.18170928955078,
      "activations/layer13_attention_weight_min": -37.84095001220703,
      "activations/layer14_attention_weight_max": 42.563541412353516,
      "activations/layer14_attention_weight_min": -37.091102600097656,
      "activations/layer15_attention_weight_max": 37.380130767822266,
      "activations/layer15_attention_weight_min": -35.757972717285156,
      "activations/layer16_attention_weight_max": 33.538673400878906,
      "activations/layer16_attention_weight_min": -27.521318435668945,
      "activations/layer17_attention_weight_max": 29.61029815673828,
      "activations/layer17_attention_weight_min": -26.910104751586914,
      "activations/layer18_attention_weight_max": 29.253698348999023,
      "activations/layer18_attention_weight_min": -24.528011322021484,
      "activations/layer19_attention_weight_max": 34.4105110168457,
      "activations/layer19_attention_weight_min": -31.59456443786621,
      "activations/layer1_attention_weight_max": 16.40349578857422,
      "activations/layer1_attention_weight_min": -14.066076278686523,
      "activations/layer20_attention_weight_max": 30.184978485107422,
      "activations/layer20_attention_weight_min": -27.907424926757812,
      "activations/layer21_attention_weight_max": 30.592504501342773,
      "activations/layer21_attention_weight_min": -28.32456398010254,
      "activations/layer22_attention_weight_max": 41.0040168762207,
      "activations/layer22_attention_weight_min": -30.195241928100586,
      "activations/layer23_attention_weight_max": 31.856643676757812,
      "activations/layer23_attention_weight_min": -23.900833129882812,
      "activations/layer2_attention_weight_max": 32.113014221191406,
      "activations/layer2_attention_weight_min": -31.411848068237305,
      "activations/layer3_attention_weight_max": 90.7021255493164,
      "activations/layer3_attention_weight_min": -93.81482696533203,
      "activations/layer4_attention_weight_max": 104.20980072021484,
      "activations/layer4_attention_weight_min": -113.39366912841797,
      "activations/layer5_attention_weight_max": 51.818443298339844,
      "activations/layer5_attention_weight_min": -63.25794982910156,
      "activations/layer6_attention_weight_max": 42.56965255737305,
      "activations/layer6_attention_weight_min": -44.87559127807617,
      "activations/layer7_attention_weight_max": 88.13646697998047,
      "activations/layer7_attention_weight_min": -90.63156127929688,
      "activations/layer8_attention_weight_max": 39.03062057495117,
      "activations/layer8_attention_weight_min": -41.39885330200195,
      "activations/layer9_attention_weight_max": 32.973052978515625,
      "activations/layer9_attention_weight_min": -34.72564697265625,
      "epoch": 13.12,
      "learning_rate": 6.605643939393938e-05,
      "loss": 2.7582,
      "step": 225800
    },
    {
      "activations/layer0_attention_weight_max": 15.54654598236084,
      "activations/layer0_attention_weight_min": -12.553293228149414,
      "activations/layer10_attention_weight_max": 33.13098907470703,
      "activations/layer10_attention_weight_min": -32.972999572753906,
      "activations/layer11_attention_weight_max": 33.78712463378906,
      "activations/layer11_attention_weight_min": -32.18386459350586,
      "activations/layer12_attention_weight_max": 26.04096794128418,
      "activations/layer12_attention_weight_min": -27.115554809570312,
      "activations/layer13_attention_weight_max": 38.84114456176758,
      "activations/layer13_attention_weight_min": -37.25071716308594,
      "activations/layer14_attention_weight_max": 42.125587463378906,
      "activations/layer14_attention_weight_min": -39.78709411621094,
      "activations/layer15_attention_weight_max": 38.83070373535156,
      "activations/layer15_attention_weight_min": -35.5473747253418,
      "activations/layer16_attention_weight_max": 31.96942138671875,
      "activations/layer16_attention_weight_min": -28.520214080810547,
      "activations/layer17_attention_weight_max": 31.04709243774414,
      "activations/layer17_attention_weight_min": -25.906558990478516,
      "activations/layer18_attention_weight_max": 34.787113189697266,
      "activations/layer18_attention_weight_min": -25.0412654876709,
      "activations/layer19_attention_weight_max": 32.091644287109375,
      "activations/layer19_attention_weight_min": -31.794391632080078,
      "activations/layer1_attention_weight_max": 16.513805389404297,
      "activations/layer1_attention_weight_min": -13.58846378326416,
      "activations/layer20_attention_weight_max": 28.500608444213867,
      "activations/layer20_attention_weight_min": -26.393054962158203,
      "activations/layer21_attention_weight_max": 27.548673629760742,
      "activations/layer21_attention_weight_min": -25.510894775390625,
      "activations/layer22_attention_weight_max": 42.005733489990234,
      "activations/layer22_attention_weight_min": -31.407678604125977,
      "activations/layer23_attention_weight_max": 31.82327651977539,
      "activations/layer23_attention_weight_min": -26.656177520751953,
      "activations/layer2_attention_weight_max": 33.52161407470703,
      "activations/layer2_attention_weight_min": -31.29277801513672,
      "activations/layer3_attention_weight_max": 95.54652404785156,
      "activations/layer3_attention_weight_min": -94.96701049804688,
      "activations/layer4_attention_weight_max": 108.37989807128906,
      "activations/layer4_attention_weight_min": -111.98841857910156,
      "activations/layer5_attention_weight_max": 49.5349235534668,
      "activations/layer5_attention_weight_min": -61.86051940917969,
      "activations/layer6_attention_weight_max": 43.06084442138672,
      "activations/layer6_attention_weight_min": -45.558006286621094,
      "activations/layer7_attention_weight_max": 92.18318939208984,
      "activations/layer7_attention_weight_min": -92.98236083984375,
      "activations/layer8_attention_weight_max": 38.49612808227539,
      "activations/layer8_attention_weight_min": -41.58952713012695,
      "activations/layer9_attention_weight_max": 31.74346351623535,
      "activations/layer9_attention_weight_min": -34.28804016113281,
      "epoch": 13.12,
      "learning_rate": 6.603749999999999e-05,
      "loss": 2.7552,
      "step": 225850
    },
    {
      "activations/layer0_attention_weight_max": 15.666203498840332,
      "activations/layer0_attention_weight_min": -11.952560424804688,
      "activations/layer10_attention_weight_max": 32.15961456298828,
      "activations/layer10_attention_weight_min": -32.82636260986328,
      "activations/layer11_attention_weight_max": 32.209014892578125,
      "activations/layer11_attention_weight_min": -33.02991485595703,
      "activations/layer12_attention_weight_max": 25.827302932739258,
      "activations/layer12_attention_weight_min": -25.052997589111328,
      "activations/layer13_attention_weight_max": 38.944976806640625,
      "activations/layer13_attention_weight_min": -34.01427459716797,
      "activations/layer14_attention_weight_max": 40.64790725708008,
      "activations/layer14_attention_weight_min": -36.02470397949219,
      "activations/layer15_attention_weight_max": 36.83662796020508,
      "activations/layer15_attention_weight_min": -31.812847137451172,
      "activations/layer16_attention_weight_max": 29.40294075012207,
      "activations/layer16_attention_weight_min": -28.906051635742188,
      "activations/layer17_attention_weight_max": 30.850582122802734,
      "activations/layer17_attention_weight_min": -25.98448944091797,
      "activations/layer18_attention_weight_max": 31.633150100708008,
      "activations/layer18_attention_weight_min": -22.465816497802734,
      "activations/layer19_attention_weight_max": 36.46643829345703,
      "activations/layer19_attention_weight_min": -30.79633140563965,
      "activations/layer1_attention_weight_max": 16.946361541748047,
      "activations/layer1_attention_weight_min": -14.226483345031738,
      "activations/layer20_attention_weight_max": 28.2393741607666,
      "activations/layer20_attention_weight_min": -25.670005798339844,
      "activations/layer21_attention_weight_max": 27.720266342163086,
      "activations/layer21_attention_weight_min": -24.520647048950195,
      "activations/layer22_attention_weight_max": 41.95191192626953,
      "activations/layer22_attention_weight_min": -31.954423904418945,
      "activations/layer23_attention_weight_max": 33.560848236083984,
      "activations/layer23_attention_weight_min": -24.22994613647461,
      "activations/layer2_attention_weight_max": 32.010009765625,
      "activations/layer2_attention_weight_min": -31.14376449584961,
      "activations/layer3_attention_weight_max": 93.99858093261719,
      "activations/layer3_attention_weight_min": -95.34414672851562,
      "activations/layer4_attention_weight_max": 106.46114349365234,
      "activations/layer4_attention_weight_min": -109.44600677490234,
      "activations/layer5_attention_weight_max": 49.716224670410156,
      "activations/layer5_attention_weight_min": -63.06562423706055,
      "activations/layer6_attention_weight_max": 46.2833366394043,
      "activations/layer6_attention_weight_min": -46.00251007080078,
      "activations/layer7_attention_weight_max": 88.25298309326172,
      "activations/layer7_attention_weight_min": -95.3691635131836,
      "activations/layer8_attention_weight_max": 39.18621826171875,
      "activations/layer8_attention_weight_min": -41.21263122558594,
      "activations/layer9_attention_weight_max": 33.515357971191406,
      "activations/layer9_attention_weight_min": -32.67138671875,
      "epoch": 13.13,
      "learning_rate": 6.60185606060606e-05,
      "loss": 2.7731,
      "step": 225900
    },
    {
      "activations/layer0_attention_weight_max": 14.817752838134766,
      "activations/layer0_attention_weight_min": -12.433588027954102,
      "activations/layer10_attention_weight_max": 38.17194366455078,
      "activations/layer10_attention_weight_min": -34.76406478881836,
      "activations/layer11_attention_weight_max": 35.072265625,
      "activations/layer11_attention_weight_min": -33.17955780029297,
      "activations/layer12_attention_weight_max": 27.85307502746582,
      "activations/layer12_attention_weight_min": -26.198657989501953,
      "activations/layer13_attention_weight_max": 40.6967658996582,
      "activations/layer13_attention_weight_min": -35.087608337402344,
      "activations/layer14_attention_weight_max": 45.1768798828125,
      "activations/layer14_attention_weight_min": -37.439292907714844,
      "activations/layer15_attention_weight_max": 37.90087890625,
      "activations/layer15_attention_weight_min": -33.4162712097168,
      "activations/layer16_attention_weight_max": 30.729646682739258,
      "activations/layer16_attention_weight_min": -26.671630859375,
      "activations/layer17_attention_weight_max": 30.938724517822266,
      "activations/layer17_attention_weight_min": -26.64977264404297,
      "activations/layer18_attention_weight_max": 34.1233024597168,
      "activations/layer18_attention_weight_min": -27.023061752319336,
      "activations/layer19_attention_weight_max": 35.44349670410156,
      "activations/layer19_attention_weight_min": -30.6991024017334,
      "activations/layer1_attention_weight_max": 16.977664947509766,
      "activations/layer1_attention_weight_min": -16.93954849243164,
      "activations/layer20_attention_weight_max": 31.638647079467773,
      "activations/layer20_attention_weight_min": -25.64373207092285,
      "activations/layer21_attention_weight_max": 31.405168533325195,
      "activations/layer21_attention_weight_min": -24.940990447998047,
      "activations/layer22_attention_weight_max": 47.30743408203125,
      "activations/layer22_attention_weight_min": -30.43160057067871,
      "activations/layer23_attention_weight_max": 37.11466979980469,
      "activations/layer23_attention_weight_min": -25.318225860595703,
      "activations/layer2_attention_weight_max": 33.297462463378906,
      "activations/layer2_attention_weight_min": -33.69520950317383,
      "activations/layer3_attention_weight_max": 98.0687255859375,
      "activations/layer3_attention_weight_min": -101.84832000732422,
      "activations/layer4_attention_weight_max": 110.46583557128906,
      "activations/layer4_attention_weight_min": -112.22476959228516,
      "activations/layer5_attention_weight_max": 55.25360870361328,
      "activations/layer5_attention_weight_min": -62.79344940185547,
      "activations/layer6_attention_weight_max": 44.977230072021484,
      "activations/layer6_attention_weight_min": -48.39704895019531,
      "activations/layer7_attention_weight_max": 94.38336181640625,
      "activations/layer7_attention_weight_min": -92.48789978027344,
      "activations/layer8_attention_weight_max": 42.20197677612305,
      "activations/layer8_attention_weight_min": -45.59334182739258,
      "activations/layer9_attention_weight_max": 37.04541015625,
      "activations/layer9_attention_weight_min": -36.16249465942383,
      "epoch": 13.13,
      "learning_rate": 6.59996212121212e-05,
      "loss": 2.7632,
      "step": 225950
    },
    {
      "activations/layer0_attention_weight_max": 15.004549980163574,
      "activations/layer0_attention_weight_min": -12.420899391174316,
      "activations/layer10_attention_weight_max": 29.543258666992188,
      "activations/layer10_attention_weight_min": -31.622148513793945,
      "activations/layer11_attention_weight_max": 30.69196891784668,
      "activations/layer11_attention_weight_min": -32.5669059753418,
      "activations/layer12_attention_weight_max": 24.745819091796875,
      "activations/layer12_attention_weight_min": -25.154525756835938,
      "activations/layer13_attention_weight_max": 39.351287841796875,
      "activations/layer13_attention_weight_min": -32.93634033203125,
      "activations/layer14_attention_weight_max": 41.762332916259766,
      "activations/layer14_attention_weight_min": -35.62947463989258,
      "activations/layer15_attention_weight_max": 36.894962310791016,
      "activations/layer15_attention_weight_min": -32.61661148071289,
      "activations/layer16_attention_weight_max": 30.060848236083984,
      "activations/layer16_attention_weight_min": -27.004167556762695,
      "activations/layer17_attention_weight_max": 30.21626091003418,
      "activations/layer17_attention_weight_min": -24.903379440307617,
      "activations/layer18_attention_weight_max": 33.22153854370117,
      "activations/layer18_attention_weight_min": -24.18842315673828,
      "activations/layer19_attention_weight_max": 33.45344161987305,
      "activations/layer19_attention_weight_min": -30.261341094970703,
      "activations/layer1_attention_weight_max": 16.199310302734375,
      "activations/layer1_attention_weight_min": -14.264298439025879,
      "activations/layer20_attention_weight_max": 32.29407501220703,
      "activations/layer20_attention_weight_min": -23.20319366455078,
      "activations/layer21_attention_weight_max": 33.144351959228516,
      "activations/layer21_attention_weight_min": -23.29515838623047,
      "activations/layer22_attention_weight_max": 44.54480743408203,
      "activations/layer22_attention_weight_min": -29.98709487915039,
      "activations/layer23_attention_weight_max": 33.204376220703125,
      "activations/layer23_attention_weight_min": -23.627056121826172,
      "activations/layer2_attention_weight_max": 32.07048797607422,
      "activations/layer2_attention_weight_min": -31.685184478759766,
      "activations/layer3_attention_weight_max": 94.09912109375,
      "activations/layer3_attention_weight_min": -94.2375717163086,
      "activations/layer4_attention_weight_max": 108.1969223022461,
      "activations/layer4_attention_weight_min": -108.5378646850586,
      "activations/layer5_attention_weight_max": 49.04521942138672,
      "activations/layer5_attention_weight_min": -63.96105194091797,
      "activations/layer6_attention_weight_max": 41.41502380371094,
      "activations/layer6_attention_weight_min": -43.83742904663086,
      "activations/layer7_attention_weight_max": 87.86557006835938,
      "activations/layer7_attention_weight_min": -87.31635284423828,
      "activations/layer8_attention_weight_max": 36.98896026611328,
      "activations/layer8_attention_weight_min": -38.69978713989258,
      "activations/layer9_attention_weight_max": 30.8360652923584,
      "activations/layer9_attention_weight_min": -32.55789566040039,
      "epoch": 13.13,
      "learning_rate": 6.59806818181818e-05,
      "loss": 2.7653,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4882,
      "eval_samples_per_second": 505.879,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4882,
      "eval_openwebtext_samples_per_second": 505.879,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9328,
      "eval_wikitext_samples_per_second": 235.926,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_lambada_loss": 2.517578125,
      "eval_lambada_ppl": 12.398532576458457,
      "eval_lambada_runtime": 9.5306,
      "eval_lambada_samples_per_second": 510.88,
      "step": 226000
    },
    {
      "activations/layer0_attention_weight_max": 16.099788665771484,
      "activations/layer0_attention_weight_min": -12.276717185974121,
      "activations/layer10_attention_weight_max": 35.07299041748047,
      "activations/layer10_attention_weight_min": -34.80709457397461,
      "activations/layer11_attention_weight_max": 35.45221710205078,
      "activations/layer11_attention_weight_min": -32.47692108154297,
      "activations/layer12_attention_weight_max": 27.51949691772461,
      "activations/layer12_attention_weight_min": -26.11363410949707,
      "activations/layer13_attention_weight_max": 43.65867614746094,
      "activations/layer13_attention_weight_min": -37.91004180908203,
      "activations/layer14_attention_weight_max": 46.12705612182617,
      "activations/layer14_attention_weight_min": -40.63668441772461,
      "activations/layer15_attention_weight_max": 38.82249069213867,
      "activations/layer15_attention_weight_min": -33.922340393066406,
      "activations/layer16_attention_weight_max": 33.10322189331055,
      "activations/layer16_attention_weight_min": -29.198930740356445,
      "activations/layer17_attention_weight_max": 36.41620635986328,
      "activations/layer17_attention_weight_min": -27.453540802001953,
      "activations/layer18_attention_weight_max": 33.05724334716797,
      "activations/layer18_attention_weight_min": -24.31853485107422,
      "activations/layer19_attention_weight_max": 38.054134368896484,
      "activations/layer19_attention_weight_min": -32.94926452636719,
      "activations/layer1_attention_weight_max": 16.504758834838867,
      "activations/layer1_attention_weight_min": -14.846959114074707,
      "activations/layer20_attention_weight_max": 31.210569381713867,
      "activations/layer20_attention_weight_min": -27.61260986328125,
      "activations/layer21_attention_weight_max": 29.61185073852539,
      "activations/layer21_attention_weight_min": -27.10638427734375,
      "activations/layer22_attention_weight_max": 49.75608444213867,
      "activations/layer22_attention_weight_min": -34.38519287109375,
      "activations/layer23_attention_weight_max": 37.37207794189453,
      "activations/layer23_attention_weight_min": -26.734336853027344,
      "activations/layer2_attention_weight_max": 34.06687545776367,
      "activations/layer2_attention_weight_min": -32.03688049316406,
      "activations/layer3_attention_weight_max": 96.57839965820312,
      "activations/layer3_attention_weight_min": -99.82577514648438,
      "activations/layer4_attention_weight_max": 112.5855484008789,
      "activations/layer4_attention_weight_min": -114.2967529296875,
      "activations/layer5_attention_weight_max": 55.38589096069336,
      "activations/layer5_attention_weight_min": -62.955657958984375,
      "activations/layer6_attention_weight_max": 45.00299835205078,
      "activations/layer6_attention_weight_min": -50.88528060913086,
      "activations/layer7_attention_weight_max": 93.48014068603516,
      "activations/layer7_attention_weight_min": -96.55509185791016,
      "activations/layer8_attention_weight_max": 41.81834030151367,
      "activations/layer8_attention_weight_min": -42.55760955810547,
      "activations/layer9_attention_weight_max": 34.301387786865234,
      "activations/layer9_attention_weight_min": -35.65491485595703,
      "epoch": 13.13,
      "learning_rate": 6.596174242424242e-05,
      "loss": 2.7677,
      "step": 226050
    },
    {
      "activations/layer0_attention_weight_max": 15.513160705566406,
      "activations/layer0_attention_weight_min": -12.538102149963379,
      "activations/layer10_attention_weight_max": 33.670406341552734,
      "activations/layer10_attention_weight_min": -32.516571044921875,
      "activations/layer11_attention_weight_max": 31.811429977416992,
      "activations/layer11_attention_weight_min": -32.20256805419922,
      "activations/layer12_attention_weight_max": 25.1566104888916,
      "activations/layer12_attention_weight_min": -25.564939498901367,
      "activations/layer13_attention_weight_max": 40.474884033203125,
      "activations/layer13_attention_weight_min": -33.63302993774414,
      "activations/layer14_attention_weight_max": 45.25309753417969,
      "activations/layer14_attention_weight_min": -39.52574920654297,
      "activations/layer15_attention_weight_max": 36.64204788208008,
      "activations/layer15_attention_weight_min": -34.036956787109375,
      "activations/layer16_attention_weight_max": 32.872703552246094,
      "activations/layer16_attention_weight_min": -28.043376922607422,
      "activations/layer17_attention_weight_max": 34.32937240600586,
      "activations/layer17_attention_weight_min": -28.945228576660156,
      "activations/layer18_attention_weight_max": 31.196701049804688,
      "activations/layer18_attention_weight_min": -25.1528263092041,
      "activations/layer19_attention_weight_max": 34.87850570678711,
      "activations/layer19_attention_weight_min": -31.628952026367188,
      "activations/layer1_attention_weight_max": 16.423261642456055,
      "activations/layer1_attention_weight_min": -14.572516441345215,
      "activations/layer20_attention_weight_max": 32.56200408935547,
      "activations/layer20_attention_weight_min": -27.73625946044922,
      "activations/layer21_attention_weight_max": 30.197832107543945,
      "activations/layer21_attention_weight_min": -26.36168098449707,
      "activations/layer22_attention_weight_max": 42.60651397705078,
      "activations/layer22_attention_weight_min": -30.940969467163086,
      "activations/layer23_attention_weight_max": 32.68480682373047,
      "activations/layer23_attention_weight_min": -26.02273178100586,
      "activations/layer2_attention_weight_max": 32.19180679321289,
      "activations/layer2_attention_weight_min": -32.65850067138672,
      "activations/layer3_attention_weight_max": 96.59879302978516,
      "activations/layer3_attention_weight_min": -97.91734313964844,
      "activations/layer4_attention_weight_max": 109.58833312988281,
      "activations/layer4_attention_weight_min": -106.57811737060547,
      "activations/layer5_attention_weight_max": 51.0920524597168,
      "activations/layer5_attention_weight_min": -61.48004913330078,
      "activations/layer6_attention_weight_max": 45.61695861816406,
      "activations/layer6_attention_weight_min": -49.03730392456055,
      "activations/layer7_attention_weight_max": 89.17906951904297,
      "activations/layer7_attention_weight_min": -99.32321166992188,
      "activations/layer8_attention_weight_max": 40.18650436401367,
      "activations/layer8_attention_weight_min": -41.72703170776367,
      "activations/layer9_attention_weight_max": 33.071834564208984,
      "activations/layer9_attention_weight_min": -32.866878509521484,
      "epoch": 13.14,
      "learning_rate": 6.594280303030302e-05,
      "loss": 2.7572,
      "step": 226100
    },
    {
      "activations/layer0_attention_weight_max": 15.56697940826416,
      "activations/layer0_attention_weight_min": -12.060355186462402,
      "activations/layer10_attention_weight_max": 31.92746925354004,
      "activations/layer10_attention_weight_min": -32.057891845703125,
      "activations/layer11_attention_weight_max": 32.87896728515625,
      "activations/layer11_attention_weight_min": -31.83027458190918,
      "activations/layer12_attention_weight_max": 25.751590728759766,
      "activations/layer12_attention_weight_min": -25.77361297607422,
      "activations/layer13_attention_weight_max": 37.17033767700195,
      "activations/layer13_attention_weight_min": -34.3266487121582,
      "activations/layer14_attention_weight_max": 42.27672576904297,
      "activations/layer14_attention_weight_min": -38.03164291381836,
      "activations/layer15_attention_weight_max": 37.39326095581055,
      "activations/layer15_attention_weight_min": -35.20528030395508,
      "activations/layer16_attention_weight_max": 29.99959945678711,
      "activations/layer16_attention_weight_min": -30.81669044494629,
      "activations/layer17_attention_weight_max": 33.985565185546875,
      "activations/layer17_attention_weight_min": -29.318756103515625,
      "activations/layer18_attention_weight_max": 31.369186401367188,
      "activations/layer18_attention_weight_min": -26.172727584838867,
      "activations/layer19_attention_weight_max": 37.72952651977539,
      "activations/layer19_attention_weight_min": -32.46556854248047,
      "activations/layer1_attention_weight_max": 16.046382904052734,
      "activations/layer1_attention_weight_min": -14.7327241897583,
      "activations/layer20_attention_weight_max": 31.619888305664062,
      "activations/layer20_attention_weight_min": -27.51406478881836,
      "activations/layer21_attention_weight_max": 32.542442321777344,
      "activations/layer21_attention_weight_min": -26.14763069152832,
      "activations/layer22_attention_weight_max": 44.731239318847656,
      "activations/layer22_attention_weight_min": -33.44786071777344,
      "activations/layer23_attention_weight_max": 36.84205627441406,
      "activations/layer23_attention_weight_min": -25.36170196533203,
      "activations/layer2_attention_weight_max": 32.98674774169922,
      "activations/layer2_attention_weight_min": -33.810142517089844,
      "activations/layer3_attention_weight_max": 96.52104949951172,
      "activations/layer3_attention_weight_min": -95.39002227783203,
      "activations/layer4_attention_weight_max": 107.58763885498047,
      "activations/layer4_attention_weight_min": -111.40278625488281,
      "activations/layer5_attention_weight_max": 52.371116638183594,
      "activations/layer5_attention_weight_min": -66.58716583251953,
      "activations/layer6_attention_weight_max": 43.64716720581055,
      "activations/layer6_attention_weight_min": -46.024314880371094,
      "activations/layer7_attention_weight_max": 92.9612045288086,
      "activations/layer7_attention_weight_min": -89.79342651367188,
      "activations/layer8_attention_weight_max": 38.54798889160156,
      "activations/layer8_attention_weight_min": -41.55960464477539,
      "activations/layer9_attention_weight_max": 31.877521514892578,
      "activations/layer9_attention_weight_min": -34.86007308959961,
      "epoch": 13.14,
      "learning_rate": 6.592386363636364e-05,
      "loss": 2.7653,
      "step": 226150
    },
    {
      "activations/layer0_attention_weight_max": 16.079988479614258,
      "activations/layer0_attention_weight_min": -11.664338111877441,
      "activations/layer10_attention_weight_max": 32.460906982421875,
      "activations/layer10_attention_weight_min": -33.61745071411133,
      "activations/layer11_attention_weight_max": 31.505027770996094,
      "activations/layer11_attention_weight_min": -31.8558406829834,
      "activations/layer12_attention_weight_max": 25.10227394104004,
      "activations/layer12_attention_weight_min": -25.131980895996094,
      "activations/layer13_attention_weight_max": 40.49150848388672,
      "activations/layer13_attention_weight_min": -35.113582611083984,
      "activations/layer14_attention_weight_max": 41.93319320678711,
      "activations/layer14_attention_weight_min": -36.97557830810547,
      "activations/layer15_attention_weight_max": 36.59822082519531,
      "activations/layer15_attention_weight_min": -34.503902435302734,
      "activations/layer16_attention_weight_max": 30.75135040283203,
      "activations/layer16_attention_weight_min": -27.596561431884766,
      "activations/layer17_attention_weight_max": 29.884525299072266,
      "activations/layer17_attention_weight_min": -26.574159622192383,
      "activations/layer18_attention_weight_max": 29.79627799987793,
      "activations/layer18_attention_weight_min": -23.7290096282959,
      "activations/layer19_attention_weight_max": 33.179325103759766,
      "activations/layer19_attention_weight_min": -31.38179588317871,
      "activations/layer1_attention_weight_max": 16.09918975830078,
      "activations/layer1_attention_weight_min": -15.707708358764648,
      "activations/layer20_attention_weight_max": 27.590986251831055,
      "activations/layer20_attention_weight_min": -25.300968170166016,
      "activations/layer21_attention_weight_max": 29.08707046508789,
      "activations/layer21_attention_weight_min": -25.461437225341797,
      "activations/layer22_attention_weight_max": 40.23368453979492,
      "activations/layer22_attention_weight_min": -31.43989372253418,
      "activations/layer23_attention_weight_max": 34.27752685546875,
      "activations/layer23_attention_weight_min": -24.757125854492188,
      "activations/layer2_attention_weight_max": 30.943862915039062,
      "activations/layer2_attention_weight_min": -30.86969566345215,
      "activations/layer3_attention_weight_max": 93.19286346435547,
      "activations/layer3_attention_weight_min": -94.84449768066406,
      "activations/layer4_attention_weight_max": 106.76885986328125,
      "activations/layer4_attention_weight_min": -108.2054443359375,
      "activations/layer5_attention_weight_max": 50.91259765625,
      "activations/layer5_attention_weight_min": -64.1305923461914,
      "activations/layer6_attention_weight_max": 44.5070915222168,
      "activations/layer6_attention_weight_min": -46.9271354675293,
      "activations/layer7_attention_weight_max": 85.86750793457031,
      "activations/layer7_attention_weight_min": -88.49375915527344,
      "activations/layer8_attention_weight_max": 39.380428314208984,
      "activations/layer8_attention_weight_min": -42.08655548095703,
      "activations/layer9_attention_weight_max": 33.81096267700195,
      "activations/layer9_attention_weight_min": -35.03099060058594,
      "epoch": 13.14,
      "learning_rate": 6.590492424242424e-05,
      "loss": 2.748,
      "step": 226200
    },
    {
      "activations/layer0_attention_weight_max": 16.333133697509766,
      "activations/layer0_attention_weight_min": -12.63859748840332,
      "activations/layer10_attention_weight_max": 34.17723846435547,
      "activations/layer10_attention_weight_min": -35.5105094909668,
      "activations/layer11_attention_weight_max": 33.14006423950195,
      "activations/layer11_attention_weight_min": -33.71107864379883,
      "activations/layer12_attention_weight_max": 34.32372283935547,
      "activations/layer12_attention_weight_min": -28.415878295898438,
      "activations/layer13_attention_weight_max": 45.25639343261719,
      "activations/layer13_attention_weight_min": -35.24089813232422,
      "activations/layer14_attention_weight_max": 46.77116775512695,
      "activations/layer14_attention_weight_min": -40.557952880859375,
      "activations/layer15_attention_weight_max": 38.065738677978516,
      "activations/layer15_attention_weight_min": -35.63845443725586,
      "activations/layer16_attention_weight_max": 35.367462158203125,
      "activations/layer16_attention_weight_min": -27.644405364990234,
      "activations/layer17_attention_weight_max": 30.81772232055664,
      "activations/layer17_attention_weight_min": -26.381174087524414,
      "activations/layer18_attention_weight_max": 32.71791458129883,
      "activations/layer18_attention_weight_min": -23.73019027709961,
      "activations/layer19_attention_weight_max": 33.15946578979492,
      "activations/layer19_attention_weight_min": -34.67975616455078,
      "activations/layer1_attention_weight_max": 15.994094848632812,
      "activations/layer1_attention_weight_min": -16.228498458862305,
      "activations/layer20_attention_weight_max": 28.3562068939209,
      "activations/layer20_attention_weight_min": -26.893321990966797,
      "activations/layer21_attention_weight_max": 29.193511962890625,
      "activations/layer21_attention_weight_min": -24.628976821899414,
      "activations/layer22_attention_weight_max": 42.49658966064453,
      "activations/layer22_attention_weight_min": -33.866455078125,
      "activations/layer23_attention_weight_max": 33.2750244140625,
      "activations/layer23_attention_weight_min": -24.368751525878906,
      "activations/layer2_attention_weight_max": 32.268798828125,
      "activations/layer2_attention_weight_min": -32.40973663330078,
      "activations/layer3_attention_weight_max": 96.86003112792969,
      "activations/layer3_attention_weight_min": -99.69407653808594,
      "activations/layer4_attention_weight_max": 110.45735168457031,
      "activations/layer4_attention_weight_min": -109.77125549316406,
      "activations/layer5_attention_weight_max": 52.96461486816406,
      "activations/layer5_attention_weight_min": -61.41036605834961,
      "activations/layer6_attention_weight_max": 45.23590850830078,
      "activations/layer6_attention_weight_min": -46.825199127197266,
      "activations/layer7_attention_weight_max": 93.0425033569336,
      "activations/layer7_attention_weight_min": -92.47903442382812,
      "activations/layer8_attention_weight_max": 40.1900520324707,
      "activations/layer8_attention_weight_min": -44.0654411315918,
      "activations/layer9_attention_weight_max": 33.278160095214844,
      "activations/layer9_attention_weight_min": -34.9709587097168,
      "epoch": 13.15,
      "learning_rate": 6.588598484848485e-05,
      "loss": 2.7706,
      "step": 226250
    },
    {
      "activations/layer0_attention_weight_max": 15.416089057922363,
      "activations/layer0_attention_weight_min": -11.727898597717285,
      "activations/layer10_attention_weight_max": 34.66600036621094,
      "activations/layer10_attention_weight_min": -33.75065994262695,
      "activations/layer11_attention_weight_max": 32.823692321777344,
      "activations/layer11_attention_weight_min": -33.61543273925781,
      "activations/layer12_attention_weight_max": 27.393003463745117,
      "activations/layer12_attention_weight_min": -26.449464797973633,
      "activations/layer13_attention_weight_max": 42.624725341796875,
      "activations/layer13_attention_weight_min": -34.971885681152344,
      "activations/layer14_attention_weight_max": 45.37454605102539,
      "activations/layer14_attention_weight_min": -37.12959289550781,
      "activations/layer15_attention_weight_max": 39.3748664855957,
      "activations/layer15_attention_weight_min": -33.44654083251953,
      "activations/layer16_attention_weight_max": 29.703126907348633,
      "activations/layer16_attention_weight_min": -27.70819091796875,
      "activations/layer17_attention_weight_max": 29.432838439941406,
      "activations/layer17_attention_weight_min": -26.033214569091797,
      "activations/layer18_attention_weight_max": 32.265968322753906,
      "activations/layer18_attention_weight_min": -26.027233123779297,
      "activations/layer19_attention_weight_max": 36.387474060058594,
      "activations/layer19_attention_weight_min": -31.84602928161621,
      "activations/layer1_attention_weight_max": 16.237268447875977,
      "activations/layer1_attention_weight_min": -15.794292449951172,
      "activations/layer20_attention_weight_max": 31.500947952270508,
      "activations/layer20_attention_weight_min": -27.7020263671875,
      "activations/layer21_attention_weight_max": 32.113197326660156,
      "activations/layer21_attention_weight_min": -26.2763729095459,
      "activations/layer22_attention_weight_max": 41.63076400756836,
      "activations/layer22_attention_weight_min": -34.188323974609375,
      "activations/layer23_attention_weight_max": 35.118621826171875,
      "activations/layer23_attention_weight_min": -26.837797164916992,
      "activations/layer2_attention_weight_max": 32.10989761352539,
      "activations/layer2_attention_weight_min": -33.38380432128906,
      "activations/layer3_attention_weight_max": 99.19549560546875,
      "activations/layer3_attention_weight_min": -98.80030059814453,
      "activations/layer4_attention_weight_max": 108.46258544921875,
      "activations/layer4_attention_weight_min": -112.717529296875,
      "activations/layer5_attention_weight_max": 51.44695281982422,
      "activations/layer5_attention_weight_min": -60.11904525756836,
      "activations/layer6_attention_weight_max": 44.773536682128906,
      "activations/layer6_attention_weight_min": -45.82057571411133,
      "activations/layer7_attention_weight_max": 95.37776947021484,
      "activations/layer7_attention_weight_min": -92.37682342529297,
      "activations/layer8_attention_weight_max": 40.76478958129883,
      "activations/layer8_attention_weight_min": -43.87069320678711,
      "activations/layer9_attention_weight_max": 34.7799186706543,
      "activations/layer9_attention_weight_min": -34.79818344116211,
      "epoch": 13.15,
      "learning_rate": 6.586704545454545e-05,
      "loss": 2.7784,
      "step": 226300
    },
    {
      "activations/layer0_attention_weight_max": 14.680380821228027,
      "activations/layer0_attention_weight_min": -11.807063102722168,
      "activations/layer10_attention_weight_max": 32.708038330078125,
      "activations/layer10_attention_weight_min": -33.10711669921875,
      "activations/layer11_attention_weight_max": 32.81452941894531,
      "activations/layer11_attention_weight_min": -32.930519104003906,
      "activations/layer12_attention_weight_max": 26.37242317199707,
      "activations/layer12_attention_weight_min": -26.38454818725586,
      "activations/layer13_attention_weight_max": 42.60252380371094,
      "activations/layer13_attention_weight_min": -34.00318145751953,
      "activations/layer14_attention_weight_max": 46.17632293701172,
      "activations/layer14_attention_weight_min": -36.106712341308594,
      "activations/layer15_attention_weight_max": 40.22188949584961,
      "activations/layer15_attention_weight_min": -34.944862365722656,
      "activations/layer16_attention_weight_max": 33.96149826049805,
      "activations/layer16_attention_weight_min": -28.844907760620117,
      "activations/layer17_attention_weight_max": 34.75196838378906,
      "activations/layer17_attention_weight_min": -26.88090705871582,
      "activations/layer18_attention_weight_max": 36.42341995239258,
      "activations/layer18_attention_weight_min": -24.63614273071289,
      "activations/layer19_attention_weight_max": 38.76957702636719,
      "activations/layer19_attention_weight_min": -31.494321823120117,
      "activations/layer1_attention_weight_max": 16.111427307128906,
      "activations/layer1_attention_weight_min": -14.389020919799805,
      "activations/layer20_attention_weight_max": 33.733028411865234,
      "activations/layer20_attention_weight_min": -23.616458892822266,
      "activations/layer21_attention_weight_max": 32.863956451416016,
      "activations/layer21_attention_weight_min": -23.828195571899414,
      "activations/layer22_attention_weight_max": 47.882568359375,
      "activations/layer22_attention_weight_min": -30.37750816345215,
      "activations/layer23_attention_weight_max": 36.4446907043457,
      "activations/layer23_attention_weight_min": -26.47418975830078,
      "activations/layer2_attention_weight_max": 32.64985656738281,
      "activations/layer2_attention_weight_min": -34.644081115722656,
      "activations/layer3_attention_weight_max": 98.46749877929688,
      "activations/layer3_attention_weight_min": -99.6909408569336,
      "activations/layer4_attention_weight_max": 114.80208587646484,
      "activations/layer4_attention_weight_min": -116.2876205444336,
      "activations/layer5_attention_weight_max": 53.24358367919922,
      "activations/layer5_attention_weight_min": -62.31343078613281,
      "activations/layer6_attention_weight_max": 44.19973373413086,
      "activations/layer6_attention_weight_min": -45.830013275146484,
      "activations/layer7_attention_weight_max": 90.10063171386719,
      "activations/layer7_attention_weight_min": -93.54906463623047,
      "activations/layer8_attention_weight_max": 39.62116622924805,
      "activations/layer8_attention_weight_min": -41.34930419921875,
      "activations/layer9_attention_weight_max": 32.794029235839844,
      "activations/layer9_attention_weight_min": -32.49978256225586,
      "epoch": 13.15,
      "learning_rate": 6.584810606060605e-05,
      "loss": 2.7603,
      "step": 226350
    },
    {
      "activations/layer0_attention_weight_max": 14.866667747497559,
      "activations/layer0_attention_weight_min": -12.693325996398926,
      "activations/layer10_attention_weight_max": 32.03867721557617,
      "activations/layer10_attention_weight_min": -34.82861328125,
      "activations/layer11_attention_weight_max": 32.72254943847656,
      "activations/layer11_attention_weight_min": -34.07780075073242,
      "activations/layer12_attention_weight_max": 27.239154815673828,
      "activations/layer12_attention_weight_min": -25.729969024658203,
      "activations/layer13_attention_weight_max": 39.456172943115234,
      "activations/layer13_attention_weight_min": -33.765830993652344,
      "activations/layer14_attention_weight_max": 46.94137191772461,
      "activations/layer14_attention_weight_min": -36.72737121582031,
      "activations/layer15_attention_weight_max": 37.30744934082031,
      "activations/layer15_attention_weight_min": -32.78298568725586,
      "activations/layer16_attention_weight_max": 31.875442504882812,
      "activations/layer16_attention_weight_min": -28.277118682861328,
      "activations/layer17_attention_weight_max": 33.0432014465332,
      "activations/layer17_attention_weight_min": -26.814599990844727,
      "activations/layer18_attention_weight_max": 30.23883056640625,
      "activations/layer18_attention_weight_min": -23.114784240722656,
      "activations/layer19_attention_weight_max": 33.78032302856445,
      "activations/layer19_attention_weight_min": -31.300739288330078,
      "activations/layer1_attention_weight_max": 16.60276985168457,
      "activations/layer1_attention_weight_min": -14.836402893066406,
      "activations/layer20_attention_weight_max": 30.745134353637695,
      "activations/layer20_attention_weight_min": -26.028724670410156,
      "activations/layer21_attention_weight_max": 33.59391403198242,
      "activations/layer21_attention_weight_min": -24.123703002929688,
      "activations/layer22_attention_weight_max": 45.7149543762207,
      "activations/layer22_attention_weight_min": -29.703441619873047,
      "activations/layer23_attention_weight_max": 33.710693359375,
      "activations/layer23_attention_weight_min": -24.84450912475586,
      "activations/layer2_attention_weight_max": 31.990339279174805,
      "activations/layer2_attention_weight_min": -33.313114166259766,
      "activations/layer3_attention_weight_max": 94.82757568359375,
      "activations/layer3_attention_weight_min": -98.02376556396484,
      "activations/layer4_attention_weight_max": 109.15364837646484,
      "activations/layer4_attention_weight_min": -110.93790435791016,
      "activations/layer5_attention_weight_max": 51.14967346191406,
      "activations/layer5_attention_weight_min": -60.8713493347168,
      "activations/layer6_attention_weight_max": 45.32960510253906,
      "activations/layer6_attention_weight_min": -46.89133071899414,
      "activations/layer7_attention_weight_max": 94.65157318115234,
      "activations/layer7_attention_weight_min": -88.33131408691406,
      "activations/layer8_attention_weight_max": 39.73948669433594,
      "activations/layer8_attention_weight_min": -40.096405029296875,
      "activations/layer9_attention_weight_max": 32.67227554321289,
      "activations/layer9_attention_weight_min": -33.271942138671875,
      "epoch": 13.16,
      "learning_rate": 6.582916666666666e-05,
      "loss": 2.7628,
      "step": 226400
    },
    {
      "activations/layer0_attention_weight_max": 15.054752349853516,
      "activations/layer0_attention_weight_min": -11.696647644042969,
      "activations/layer10_attention_weight_max": 37.01487731933594,
      "activations/layer10_attention_weight_min": -35.193267822265625,
      "activations/layer11_attention_weight_max": 38.2943229675293,
      "activations/layer11_attention_weight_min": -34.48973846435547,
      "activations/layer12_attention_weight_max": 27.94486427307129,
      "activations/layer12_attention_weight_min": -25.758670806884766,
      "activations/layer13_attention_weight_max": 42.885887145996094,
      "activations/layer13_attention_weight_min": -34.28655242919922,
      "activations/layer14_attention_weight_max": 43.91638946533203,
      "activations/layer14_attention_weight_min": -37.61043930053711,
      "activations/layer15_attention_weight_max": 40.47760772705078,
      "activations/layer15_attention_weight_min": -34.106319427490234,
      "activations/layer16_attention_weight_max": 31.54006004333496,
      "activations/layer16_attention_weight_min": -29.849205017089844,
      "activations/layer17_attention_weight_max": 33.327613830566406,
      "activations/layer17_attention_weight_min": -25.66886329650879,
      "activations/layer18_attention_weight_max": 30.435277938842773,
      "activations/layer18_attention_weight_min": -23.191619873046875,
      "activations/layer19_attention_weight_max": 37.984615325927734,
      "activations/layer19_attention_weight_min": -31.308120727539062,
      "activations/layer1_attention_weight_max": 17.41269302368164,
      "activations/layer1_attention_weight_min": -13.941471099853516,
      "activations/layer20_attention_weight_max": 30.556987762451172,
      "activations/layer20_attention_weight_min": -24.99489402770996,
      "activations/layer21_attention_weight_max": 30.36032485961914,
      "activations/layer21_attention_weight_min": -24.607467651367188,
      "activations/layer22_attention_weight_max": 45.77056121826172,
      "activations/layer22_attention_weight_min": -30.798810958862305,
      "activations/layer23_attention_weight_max": 31.487873077392578,
      "activations/layer23_attention_weight_min": -25.94583511352539,
      "activations/layer2_attention_weight_max": 33.11613464355469,
      "activations/layer2_attention_weight_min": -33.09238052368164,
      "activations/layer3_attention_weight_max": 91.74781036376953,
      "activations/layer3_attention_weight_min": -93.46210479736328,
      "activations/layer4_attention_weight_max": 105.55390930175781,
      "activations/layer4_attention_weight_min": -113.7029800415039,
      "activations/layer5_attention_weight_max": 52.38361358642578,
      "activations/layer5_attention_weight_min": -63.5909538269043,
      "activations/layer6_attention_weight_max": 43.48457336425781,
      "activations/layer6_attention_weight_min": -48.11764907836914,
      "activations/layer7_attention_weight_max": 95.4131851196289,
      "activations/layer7_attention_weight_min": -93.4364242553711,
      "activations/layer8_attention_weight_max": 43.18547439575195,
      "activations/layer8_attention_weight_min": -44.52443313598633,
      "activations/layer9_attention_weight_max": 36.32319641113281,
      "activations/layer9_attention_weight_min": -35.33097839355469,
      "epoch": 13.16,
      "learning_rate": 6.581022727272727e-05,
      "loss": 2.7725,
      "step": 226450
    },
    {
      "activations/layer0_attention_weight_max": 15.740325927734375,
      "activations/layer0_attention_weight_min": -12.844802856445312,
      "activations/layer10_attention_weight_max": 36.28990936279297,
      "activations/layer10_attention_weight_min": -37.4200325012207,
      "activations/layer11_attention_weight_max": 35.59209442138672,
      "activations/layer11_attention_weight_min": -37.18998718261719,
      "activations/layer12_attention_weight_max": 29.566919326782227,
      "activations/layer12_attention_weight_min": -27.316431045532227,
      "activations/layer13_attention_weight_max": 43.35970687866211,
      "activations/layer13_attention_weight_min": -37.44596862792969,
      "activations/layer14_attention_weight_max": 45.80027770996094,
      "activations/layer14_attention_weight_min": -42.884193420410156,
      "activations/layer15_attention_weight_max": 43.18028259277344,
      "activations/layer15_attention_weight_min": -35.3196907043457,
      "activations/layer16_attention_weight_max": 30.03023910522461,
      "activations/layer16_attention_weight_min": -27.592025756835938,
      "activations/layer17_attention_weight_max": 29.88829231262207,
      "activations/layer17_attention_weight_min": -27.239582061767578,
      "activations/layer18_attention_weight_max": 32.61651611328125,
      "activations/layer18_attention_weight_min": -24.855337142944336,
      "activations/layer19_attention_weight_max": 35.53328323364258,
      "activations/layer19_attention_weight_min": -31.4719295501709,
      "activations/layer1_attention_weight_max": 16.87586784362793,
      "activations/layer1_attention_weight_min": -15.434396743774414,
      "activations/layer20_attention_weight_max": 30.878938674926758,
      "activations/layer20_attention_weight_min": -24.50674057006836,
      "activations/layer21_attention_weight_max": 29.880050659179688,
      "activations/layer21_attention_weight_min": -25.212505340576172,
      "activations/layer22_attention_weight_max": 44.81887435913086,
      "activations/layer22_attention_weight_min": -31.595815658569336,
      "activations/layer23_attention_weight_max": 33.36316680908203,
      "activations/layer23_attention_weight_min": -24.80794906616211,
      "activations/layer2_attention_weight_max": 34.32762145996094,
      "activations/layer2_attention_weight_min": -33.45130157470703,
      "activations/layer3_attention_weight_max": 96.90127563476562,
      "activations/layer3_attention_weight_min": -101.13274383544922,
      "activations/layer4_attention_weight_max": 114.14344787597656,
      "activations/layer4_attention_weight_min": -114.11305236816406,
      "activations/layer5_attention_weight_max": 51.992401123046875,
      "activations/layer5_attention_weight_min": -60.40443420410156,
      "activations/layer6_attention_weight_max": 45.87785720825195,
      "activations/layer6_attention_weight_min": -46.62445831298828,
      "activations/layer7_attention_weight_max": 93.67913818359375,
      "activations/layer7_attention_weight_min": -92.27628326416016,
      "activations/layer8_attention_weight_max": 41.13054275512695,
      "activations/layer8_attention_weight_min": -43.05137634277344,
      "activations/layer9_attention_weight_max": 38.02914047241211,
      "activations/layer9_attention_weight_min": -38.3288688659668,
      "epoch": 13.16,
      "learning_rate": 6.579128787878787e-05,
      "loss": 2.7452,
      "step": 226500
    },
    {
      "activations/layer0_attention_weight_max": 14.780774116516113,
      "activations/layer0_attention_weight_min": -11.541627883911133,
      "activations/layer10_attention_weight_max": 37.22265625,
      "activations/layer10_attention_weight_min": -35.264076232910156,
      "activations/layer11_attention_weight_max": 35.8551025390625,
      "activations/layer11_attention_weight_min": -35.57057189941406,
      "activations/layer12_attention_weight_max": 43.29927444458008,
      "activations/layer12_attention_weight_min": -31.75872230529785,
      "activations/layer13_attention_weight_max": 63.084110260009766,
      "activations/layer13_attention_weight_min": -37.56217956542969,
      "activations/layer14_attention_weight_max": 55.06262969970703,
      "activations/layer14_attention_weight_min": -43.0333251953125,
      "activations/layer15_attention_weight_max": 61.075809478759766,
      "activations/layer15_attention_weight_min": -38.510318756103516,
      "activations/layer16_attention_weight_max": 32.402347564697266,
      "activations/layer16_attention_weight_min": -27.9168643951416,
      "activations/layer17_attention_weight_max": 31.530799865722656,
      "activations/layer17_attention_weight_min": -26.667753219604492,
      "activations/layer18_attention_weight_max": 31.65887451171875,
      "activations/layer18_attention_weight_min": -24.756547927856445,
      "activations/layer19_attention_weight_max": 36.28672409057617,
      "activations/layer19_attention_weight_min": -30.835988998413086,
      "activations/layer1_attention_weight_max": 16.468366622924805,
      "activations/layer1_attention_weight_min": -14.046331405639648,
      "activations/layer20_attention_weight_max": 29.12871742248535,
      "activations/layer20_attention_weight_min": -26.35823631286621,
      "activations/layer21_attention_weight_max": 29.772689819335938,
      "activations/layer21_attention_weight_min": -26.846227645874023,
      "activations/layer22_attention_weight_max": 45.09272003173828,
      "activations/layer22_attention_weight_min": -32.607852935791016,
      "activations/layer23_attention_weight_max": 35.50974655151367,
      "activations/layer23_attention_weight_min": -24.475858688354492,
      "activations/layer2_attention_weight_max": 32.58531951904297,
      "activations/layer2_attention_weight_min": -34.677154541015625,
      "activations/layer3_attention_weight_max": 99.95887756347656,
      "activations/layer3_attention_weight_min": -100.58132934570312,
      "activations/layer4_attention_weight_max": 110.6796875,
      "activations/layer4_attention_weight_min": -111.0410385131836,
      "activations/layer5_attention_weight_max": 52.05526351928711,
      "activations/layer5_attention_weight_min": -61.150054931640625,
      "activations/layer6_attention_weight_max": 43.438350677490234,
      "activations/layer6_attention_weight_min": -46.62525177001953,
      "activations/layer7_attention_weight_max": 92.59906768798828,
      "activations/layer7_attention_weight_min": -88.28107452392578,
      "activations/layer8_attention_weight_max": 42.221458435058594,
      "activations/layer8_attention_weight_min": -41.79219436645508,
      "activations/layer9_attention_weight_max": 34.06090545654297,
      "activations/layer9_attention_weight_min": -35.854820251464844,
      "epoch": 13.16,
      "learning_rate": 6.577234848484847e-05,
      "loss": 2.7588,
      "step": 226550
    },
    {
      "activations/layer0_attention_weight_max": 15.339797019958496,
      "activations/layer0_attention_weight_min": -12.573932647705078,
      "activations/layer10_attention_weight_max": 33.16106033325195,
      "activations/layer10_attention_weight_min": -34.99458312988281,
      "activations/layer11_attention_weight_max": 33.66942596435547,
      "activations/layer11_attention_weight_min": -33.84180450439453,
      "activations/layer12_attention_weight_max": 25.90678596496582,
      "activations/layer12_attention_weight_min": -25.34303092956543,
      "activations/layer13_attention_weight_max": 39.31038284301758,
      "activations/layer13_attention_weight_min": -35.16900634765625,
      "activations/layer14_attention_weight_max": 45.44871139526367,
      "activations/layer14_attention_weight_min": -37.702728271484375,
      "activations/layer15_attention_weight_max": 36.15311813354492,
      "activations/layer15_attention_weight_min": -33.756080627441406,
      "activations/layer16_attention_weight_max": 30.299352645874023,
      "activations/layer16_attention_weight_min": -28.60527992248535,
      "activations/layer17_attention_weight_max": 30.3740177154541,
      "activations/layer17_attention_weight_min": -27.282121658325195,
      "activations/layer18_attention_weight_max": 31.873872756958008,
      "activations/layer18_attention_weight_min": -25.69115447998047,
      "activations/layer19_attention_weight_max": 35.905860900878906,
      "activations/layer19_attention_weight_min": -30.7874813079834,
      "activations/layer1_attention_weight_max": 16.378623962402344,
      "activations/layer1_attention_weight_min": -15.255148887634277,
      "activations/layer20_attention_weight_max": 30.7546443939209,
      "activations/layer20_attention_weight_min": -25.174148559570312,
      "activations/layer21_attention_weight_max": 31.072402954101562,
      "activations/layer21_attention_weight_min": -24.975759506225586,
      "activations/layer22_attention_weight_max": 51.706119537353516,
      "activations/layer22_attention_weight_min": -32.1710319519043,
      "activations/layer23_attention_weight_max": 39.68943405151367,
      "activations/layer23_attention_weight_min": -26.653461456298828,
      "activations/layer2_attention_weight_max": 33.71062469482422,
      "activations/layer2_attention_weight_min": -32.6437873840332,
      "activations/layer3_attention_weight_max": 96.66844940185547,
      "activations/layer3_attention_weight_min": -99.04155731201172,
      "activations/layer4_attention_weight_max": 114.0439453125,
      "activations/layer4_attention_weight_min": -112.72054290771484,
      "activations/layer5_attention_weight_max": 51.16442108154297,
      "activations/layer5_attention_weight_min": -64.33954620361328,
      "activations/layer6_attention_weight_max": 43.3803596496582,
      "activations/layer6_attention_weight_min": -45.488487243652344,
      "activations/layer7_attention_weight_max": 91.34473419189453,
      "activations/layer7_attention_weight_min": -93.01559448242188,
      "activations/layer8_attention_weight_max": 39.17271041870117,
      "activations/layer8_attention_weight_min": -41.655094146728516,
      "activations/layer9_attention_weight_max": 34.4061164855957,
      "activations/layer9_attention_weight_min": -33.69303894042969,
      "epoch": 13.17,
      "learning_rate": 6.575340909090908e-05,
      "loss": 2.7504,
      "step": 226600
    },
    {
      "activations/layer0_attention_weight_max": 14.586100578308105,
      "activations/layer0_attention_weight_min": -12.594454765319824,
      "activations/layer10_attention_weight_max": 33.188472747802734,
      "activations/layer10_attention_weight_min": -34.116207122802734,
      "activations/layer11_attention_weight_max": 33.342552185058594,
      "activations/layer11_attention_weight_min": -33.36054992675781,
      "activations/layer12_attention_weight_max": 24.784238815307617,
      "activations/layer12_attention_weight_min": -26.692039489746094,
      "activations/layer13_attention_weight_max": 36.935951232910156,
      "activations/layer13_attention_weight_min": -37.29261016845703,
      "activations/layer14_attention_weight_max": 44.43670654296875,
      "activations/layer14_attention_weight_min": -40.63542175292969,
      "activations/layer15_attention_weight_max": 34.39503860473633,
      "activations/layer15_attention_weight_min": -35.7072868347168,
      "activations/layer16_attention_weight_max": 30.087404251098633,
      "activations/layer16_attention_weight_min": -28.369722366333008,
      "activations/layer17_attention_weight_max": 30.03151512145996,
      "activations/layer17_attention_weight_min": -27.945642471313477,
      "activations/layer18_attention_weight_max": 30.949810028076172,
      "activations/layer18_attention_weight_min": -25.246746063232422,
      "activations/layer19_attention_weight_max": 34.164615631103516,
      "activations/layer19_attention_weight_min": -30.97303581237793,
      "activations/layer1_attention_weight_max": 16.62016487121582,
      "activations/layer1_attention_weight_min": -15.761075019836426,
      "activations/layer20_attention_weight_max": 28.656625747680664,
      "activations/layer20_attention_weight_min": -25.041332244873047,
      "activations/layer21_attention_weight_max": 29.07159423828125,
      "activations/layer21_attention_weight_min": -23.49941062927246,
      "activations/layer22_attention_weight_max": 42.27705383300781,
      "activations/layer22_attention_weight_min": -29.801654815673828,
      "activations/layer23_attention_weight_max": 31.42366600036621,
      "activations/layer23_attention_weight_min": -27.059097290039062,
      "activations/layer2_attention_weight_max": 33.94367218017578,
      "activations/layer2_attention_weight_min": -32.38740921020508,
      "activations/layer3_attention_weight_max": 101.10869598388672,
      "activations/layer3_attention_weight_min": -98.79452514648438,
      "activations/layer4_attention_weight_max": 116.12843322753906,
      "activations/layer4_attention_weight_min": -115.6922836303711,
      "activations/layer5_attention_weight_max": 57.110939025878906,
      "activations/layer5_attention_weight_min": -62.248695373535156,
      "activations/layer6_attention_weight_max": 46.727779388427734,
      "activations/layer6_attention_weight_min": -47.10658645629883,
      "activations/layer7_attention_weight_max": 88.55350494384766,
      "activations/layer7_attention_weight_min": -91.42545318603516,
      "activations/layer8_attention_weight_max": 40.25920867919922,
      "activations/layer8_attention_weight_min": -41.20347595214844,
      "activations/layer9_attention_weight_max": 33.788108825683594,
      "activations/layer9_attention_weight_min": -32.50898742675781,
      "epoch": 13.17,
      "learning_rate": 6.573446969696969e-05,
      "loss": 2.7468,
      "step": 226650
    },
    {
      "activations/layer0_attention_weight_max": 16.15181541442871,
      "activations/layer0_attention_weight_min": -11.846382141113281,
      "activations/layer10_attention_weight_max": 32.282413482666016,
      "activations/layer10_attention_weight_min": -33.348976135253906,
      "activations/layer11_attention_weight_max": 34.746700286865234,
      "activations/layer11_attention_weight_min": -32.423892974853516,
      "activations/layer12_attention_weight_max": 24.9492244720459,
      "activations/layer12_attention_weight_min": -25.17613410949707,
      "activations/layer13_attention_weight_max": 37.13549041748047,
      "activations/layer13_attention_weight_min": -33.44258499145508,
      "activations/layer14_attention_weight_max": 42.275390625,
      "activations/layer14_attention_weight_min": -36.128257751464844,
      "activations/layer15_attention_weight_max": 36.145816802978516,
      "activations/layer15_attention_weight_min": -33.81282424926758,
      "activations/layer16_attention_weight_max": 29.51768684387207,
      "activations/layer16_attention_weight_min": -30.880264282226562,
      "activations/layer17_attention_weight_max": 30.460309982299805,
      "activations/layer17_attention_weight_min": -30.22396469116211,
      "activations/layer18_attention_weight_max": 29.86994743347168,
      "activations/layer18_attention_weight_min": -27.007488250732422,
      "activations/layer19_attention_weight_max": 38.7602653503418,
      "activations/layer19_attention_weight_min": -32.081565856933594,
      "activations/layer1_attention_weight_max": 16.066509246826172,
      "activations/layer1_attention_weight_min": -13.68614387512207,
      "activations/layer20_attention_weight_max": 28.344482421875,
      "activations/layer20_attention_weight_min": -26.953630447387695,
      "activations/layer21_attention_weight_max": 30.081008911132812,
      "activations/layer21_attention_weight_min": -24.641714096069336,
      "activations/layer22_attention_weight_max": 41.73255920410156,
      "activations/layer22_attention_weight_min": -31.21700668334961,
      "activations/layer23_attention_weight_max": 33.45258331298828,
      "activations/layer23_attention_weight_min": -26.52993392944336,
      "activations/layer2_attention_weight_max": 32.952762603759766,
      "activations/layer2_attention_weight_min": -32.710235595703125,
      "activations/layer3_attention_weight_max": 96.15890502929688,
      "activations/layer3_attention_weight_min": -99.87921142578125,
      "activations/layer4_attention_weight_max": 111.95356750488281,
      "activations/layer4_attention_weight_min": -115.16112518310547,
      "activations/layer5_attention_weight_max": 50.56785583496094,
      "activations/layer5_attention_weight_min": -61.554969787597656,
      "activations/layer6_attention_weight_max": 43.91347885131836,
      "activations/layer6_attention_weight_min": -47.4636344909668,
      "activations/layer7_attention_weight_max": 87.1923828125,
      "activations/layer7_attention_weight_min": -90.61050415039062,
      "activations/layer8_attention_weight_max": 41.661956787109375,
      "activations/layer8_attention_weight_min": -42.039276123046875,
      "activations/layer9_attention_weight_max": 33.04423141479492,
      "activations/layer9_attention_weight_min": -33.686588287353516,
      "epoch": 13.17,
      "learning_rate": 6.571553030303029e-05,
      "loss": 2.7877,
      "step": 226700
    },
    {
      "activations/layer0_attention_weight_max": 15.19655704498291,
      "activations/layer0_attention_weight_min": -12.568726539611816,
      "activations/layer10_attention_weight_max": 35.72615432739258,
      "activations/layer10_attention_weight_min": -32.905887603759766,
      "activations/layer11_attention_weight_max": 35.45051574707031,
      "activations/layer11_attention_weight_min": -32.55031967163086,
      "activations/layer12_attention_weight_max": 26.277128219604492,
      "activations/layer12_attention_weight_min": -28.576623916625977,
      "activations/layer13_attention_weight_max": 40.350196838378906,
      "activations/layer13_attention_weight_min": -34.89153289794922,
      "activations/layer14_attention_weight_max": 46.62605285644531,
      "activations/layer14_attention_weight_min": -39.63682174682617,
      "activations/layer15_attention_weight_max": 37.4599494934082,
      "activations/layer15_attention_weight_min": -35.5167236328125,
      "activations/layer16_attention_weight_max": 32.704864501953125,
      "activations/layer16_attention_weight_min": -28.927047729492188,
      "activations/layer17_attention_weight_max": 30.8605899810791,
      "activations/layer17_attention_weight_min": -28.06219482421875,
      "activations/layer18_attention_weight_max": 28.77104377746582,
      "activations/layer18_attention_weight_min": -25.380001068115234,
      "activations/layer19_attention_weight_max": 35.3067512512207,
      "activations/layer19_attention_weight_min": -31.901798248291016,
      "activations/layer1_attention_weight_max": 16.74114227294922,
      "activations/layer1_attention_weight_min": -15.694604873657227,
      "activations/layer20_attention_weight_max": 28.715110778808594,
      "activations/layer20_attention_weight_min": -24.8198299407959,
      "activations/layer21_attention_weight_max": 30.31113052368164,
      "activations/layer21_attention_weight_min": -25.224742889404297,
      "activations/layer22_attention_weight_max": 39.46419906616211,
      "activations/layer22_attention_weight_min": -29.63157844543457,
      "activations/layer23_attention_weight_max": 32.78739547729492,
      "activations/layer23_attention_weight_min": -26.349811553955078,
      "activations/layer2_attention_weight_max": 35.960174560546875,
      "activations/layer2_attention_weight_min": -33.171836853027344,
      "activations/layer3_attention_weight_max": 98.92293548583984,
      "activations/layer3_attention_weight_min": -98.09089660644531,
      "activations/layer4_attention_weight_max": 110.61848449707031,
      "activations/layer4_attention_weight_min": -108.64581298828125,
      "activations/layer5_attention_weight_max": 50.388580322265625,
      "activations/layer5_attention_weight_min": -62.809608459472656,
      "activations/layer6_attention_weight_max": 46.49054718017578,
      "activations/layer6_attention_weight_min": -48.50986862182617,
      "activations/layer7_attention_weight_max": 96.6900634765625,
      "activations/layer7_attention_weight_min": -91.34187316894531,
      "activations/layer8_attention_weight_max": 41.06303787231445,
      "activations/layer8_attention_weight_min": -43.39324951171875,
      "activations/layer9_attention_weight_max": 33.35893249511719,
      "activations/layer9_attention_weight_min": -34.14976119995117,
      "epoch": 13.18,
      "learning_rate": 6.569659090909091e-05,
      "loss": 2.7623,
      "step": 226750
    },
    {
      "activations/layer0_attention_weight_max": 15.399788856506348,
      "activations/layer0_attention_weight_min": -12.567760467529297,
      "activations/layer10_attention_weight_max": 31.136577606201172,
      "activations/layer10_attention_weight_min": -33.9901123046875,
      "activations/layer11_attention_weight_max": 31.493154525756836,
      "activations/layer11_attention_weight_min": -31.825809478759766,
      "activations/layer12_attention_weight_max": 23.52904510498047,
      "activations/layer12_attention_weight_min": -25.43460464477539,
      "activations/layer13_attention_weight_max": 35.49569320678711,
      "activations/layer13_attention_weight_min": -35.701656341552734,
      "activations/layer14_attention_weight_max": 39.55381774902344,
      "activations/layer14_attention_weight_min": -37.12871170043945,
      "activations/layer15_attention_weight_max": 32.8652458190918,
      "activations/layer15_attention_weight_min": -31.877714157104492,
      "activations/layer16_attention_weight_max": 30.748672485351562,
      "activations/layer16_attention_weight_min": -27.723636627197266,
      "activations/layer17_attention_weight_max": 30.779537200927734,
      "activations/layer17_attention_weight_min": -24.247371673583984,
      "activations/layer18_attention_weight_max": 31.85719108581543,
      "activations/layer18_attention_weight_min": -24.3735408782959,
      "activations/layer19_attention_weight_max": 36.644317626953125,
      "activations/layer19_attention_weight_min": -31.51047134399414,
      "activations/layer1_attention_weight_max": 16.559120178222656,
      "activations/layer1_attention_weight_min": -15.186363220214844,
      "activations/layer20_attention_weight_max": 30.74724578857422,
      "activations/layer20_attention_weight_min": -25.269914627075195,
      "activations/layer21_attention_weight_max": 31.75156593322754,
      "activations/layer21_attention_weight_min": -25.41788101196289,
      "activations/layer22_attention_weight_max": 39.548561096191406,
      "activations/layer22_attention_weight_min": -28.63857650756836,
      "activations/layer23_attention_weight_max": 34.4351692199707,
      "activations/layer23_attention_weight_min": -24.54452133178711,
      "activations/layer2_attention_weight_max": 32.35895538330078,
      "activations/layer2_attention_weight_min": -31.907682418823242,
      "activations/layer3_attention_weight_max": 93.66921997070312,
      "activations/layer3_attention_weight_min": -98.3611831665039,
      "activations/layer4_attention_weight_max": 108.82452392578125,
      "activations/layer4_attention_weight_min": -107.81524658203125,
      "activations/layer5_attention_weight_max": 53.03791427612305,
      "activations/layer5_attention_weight_min": -63.112728118896484,
      "activations/layer6_attention_weight_max": 43.45060348510742,
      "activations/layer6_attention_weight_min": -47.551536560058594,
      "activations/layer7_attention_weight_max": 89.06922912597656,
      "activations/layer7_attention_weight_min": -95.3973159790039,
      "activations/layer8_attention_weight_max": 38.26476287841797,
      "activations/layer8_attention_weight_min": -41.53519058227539,
      "activations/layer9_attention_weight_max": 31.00760841369629,
      "activations/layer9_attention_weight_min": -33.860267639160156,
      "epoch": 13.18,
      "learning_rate": 6.567765151515151e-05,
      "loss": 2.775,
      "step": 226800
    },
    {
      "activations/layer0_attention_weight_max": 14.650362968444824,
      "activations/layer0_attention_weight_min": -11.630460739135742,
      "activations/layer10_attention_weight_max": 50.10626220703125,
      "activations/layer10_attention_weight_min": -49.111263275146484,
      "activations/layer11_attention_weight_max": 50.2266960144043,
      "activations/layer11_attention_weight_min": -48.792930603027344,
      "activations/layer12_attention_weight_max": 37.99468994140625,
      "activations/layer12_attention_weight_min": -31.88766860961914,
      "activations/layer13_attention_weight_max": 57.96699142456055,
      "activations/layer13_attention_weight_min": -44.977298736572266,
      "activations/layer14_attention_weight_max": 66.43376922607422,
      "activations/layer14_attention_weight_min": -52.66215133666992,
      "activations/layer15_attention_weight_max": 65.8362808227539,
      "activations/layer15_attention_weight_min": -47.09906005859375,
      "activations/layer16_attention_weight_max": 42.7079963684082,
      "activations/layer16_attention_weight_min": -32.42184829711914,
      "activations/layer17_attention_weight_max": 35.105445861816406,
      "activations/layer17_attention_weight_min": -33.373050689697266,
      "activations/layer18_attention_weight_max": 31.11138916015625,
      "activations/layer18_attention_weight_min": -28.118627548217773,
      "activations/layer19_attention_weight_max": 39.95956802368164,
      "activations/layer19_attention_weight_min": -33.87827682495117,
      "activations/layer1_attention_weight_max": 16.784515380859375,
      "activations/layer1_attention_weight_min": -15.177994728088379,
      "activations/layer20_attention_weight_max": 31.403173446655273,
      "activations/layer20_attention_weight_min": -25.78371238708496,
      "activations/layer21_attention_weight_max": 27.41392707824707,
      "activations/layer21_attention_weight_min": -26.852874755859375,
      "activations/layer22_attention_weight_max": 45.71302032470703,
      "activations/layer22_attention_weight_min": -33.41421890258789,
      "activations/layer23_attention_weight_max": 33.402557373046875,
      "activations/layer23_attention_weight_min": -26.316444396972656,
      "activations/layer2_attention_weight_max": 36.38032913208008,
      "activations/layer2_attention_weight_min": -36.08596420288086,
      "activations/layer3_attention_weight_max": 109.24948120117188,
      "activations/layer3_attention_weight_min": -109.89376068115234,
      "activations/layer4_attention_weight_max": 120.75804901123047,
      "activations/layer4_attention_weight_min": -117.15016174316406,
      "activations/layer5_attention_weight_max": 56.30365753173828,
      "activations/layer5_attention_weight_min": -66.20370483398438,
      "activations/layer6_attention_weight_max": 49.881038665771484,
      "activations/layer6_attention_weight_min": -48.33840560913086,
      "activations/layer7_attention_weight_max": 115.12371826171875,
      "activations/layer7_attention_weight_min": -110.25220489501953,
      "activations/layer8_attention_weight_max": 53.709835052490234,
      "activations/layer8_attention_weight_min": -53.51022720336914,
      "activations/layer9_attention_weight_max": 47.03845977783203,
      "activations/layer9_attention_weight_min": -46.658443450927734,
      "epoch": 13.18,
      "learning_rate": 6.565871212121212e-05,
      "loss": 2.7646,
      "step": 226850
    },
    {
      "activations/layer0_attention_weight_max": 14.987627983093262,
      "activations/layer0_attention_weight_min": -12.527275085449219,
      "activations/layer10_attention_weight_max": 32.85382843017578,
      "activations/layer10_attention_weight_min": -32.31572723388672,
      "activations/layer11_attention_weight_max": 32.0851936340332,
      "activations/layer11_attention_weight_min": -34.198062896728516,
      "activations/layer12_attention_weight_max": 34.24127197265625,
      "activations/layer12_attention_weight_min": -27.22121810913086,
      "activations/layer13_attention_weight_max": 42.78834915161133,
      "activations/layer13_attention_weight_min": -35.18344497680664,
      "activations/layer14_attention_weight_max": 44.39943313598633,
      "activations/layer14_attention_weight_min": -38.599693298339844,
      "activations/layer15_attention_weight_max": 39.274288177490234,
      "activations/layer15_attention_weight_min": -33.741363525390625,
      "activations/layer16_attention_weight_max": 33.55695343017578,
      "activations/layer16_attention_weight_min": -28.983842849731445,
      "activations/layer17_attention_weight_max": 30.777557373046875,
      "activations/layer17_attention_weight_min": -26.17818832397461,
      "activations/layer18_attention_weight_max": 33.70234298706055,
      "activations/layer18_attention_weight_min": -25.012195587158203,
      "activations/layer19_attention_weight_max": 37.35444259643555,
      "activations/layer19_attention_weight_min": -31.465768814086914,
      "activations/layer1_attention_weight_max": 16.516681671142578,
      "activations/layer1_attention_weight_min": -13.852739334106445,
      "activations/layer20_attention_weight_max": 37.91867446899414,
      "activations/layer20_attention_weight_min": -25.863059997558594,
      "activations/layer21_attention_weight_max": 34.399078369140625,
      "activations/layer21_attention_weight_min": -23.46409797668457,
      "activations/layer22_attention_weight_max": 44.213157653808594,
      "activations/layer22_attention_weight_min": -30.626556396484375,
      "activations/layer23_attention_weight_max": 36.78424072265625,
      "activations/layer23_attention_weight_min": -25.2503719329834,
      "activations/layer2_attention_weight_max": 32.48773956298828,
      "activations/layer2_attention_weight_min": -32.78230285644531,
      "activations/layer3_attention_weight_max": 95.52230834960938,
      "activations/layer3_attention_weight_min": -98.6009521484375,
      "activations/layer4_attention_weight_max": 108.30379486083984,
      "activations/layer4_attention_weight_min": -109.27977752685547,
      "activations/layer5_attention_weight_max": 51.731807708740234,
      "activations/layer5_attention_weight_min": -62.642433166503906,
      "activations/layer6_attention_weight_max": 44.38663101196289,
      "activations/layer6_attention_weight_min": -46.859493255615234,
      "activations/layer7_attention_weight_max": 84.44761657714844,
      "activations/layer7_attention_weight_min": -88.83533477783203,
      "activations/layer8_attention_weight_max": 41.508209228515625,
      "activations/layer8_attention_weight_min": -42.2104377746582,
      "activations/layer9_attention_weight_max": 34.08623123168945,
      "activations/layer9_attention_weight_min": -34.057247161865234,
      "epoch": 13.18,
      "learning_rate": 6.563977272727273e-05,
      "loss": 2.7636,
      "step": 226900
    },
    {
      "activations/layer0_attention_weight_max": 16.183631896972656,
      "activations/layer0_attention_weight_min": -11.307337760925293,
      "activations/layer10_attention_weight_max": 40.56206130981445,
      "activations/layer10_attention_weight_min": -35.6912841796875,
      "activations/layer11_attention_weight_max": 41.63608169555664,
      "activations/layer11_attention_weight_min": -36.87432098388672,
      "activations/layer12_attention_weight_max": 24.858640670776367,
      "activations/layer12_attention_weight_min": -26.056392669677734,
      "activations/layer13_attention_weight_max": 38.9716796875,
      "activations/layer13_attention_weight_min": -34.41147994995117,
      "activations/layer14_attention_weight_max": 44.07073211669922,
      "activations/layer14_attention_weight_min": -34.493446350097656,
      "activations/layer15_attention_weight_max": 36.64258575439453,
      "activations/layer15_attention_weight_min": -32.909759521484375,
      "activations/layer16_attention_weight_max": 31.006059646606445,
      "activations/layer16_attention_weight_min": -26.449071884155273,
      "activations/layer17_attention_weight_max": 31.1978759765625,
      "activations/layer17_attention_weight_min": -25.6987247467041,
      "activations/layer18_attention_weight_max": 30.927682876586914,
      "activations/layer18_attention_weight_min": -25.078344345092773,
      "activations/layer19_attention_weight_max": 36.49490737915039,
      "activations/layer19_attention_weight_min": -30.557758331298828,
      "activations/layer1_attention_weight_max": 16.716054916381836,
      "activations/layer1_attention_weight_min": -14.139436721801758,
      "activations/layer20_attention_weight_max": 32.11479187011719,
      "activations/layer20_attention_weight_min": -26.572298049926758,
      "activations/layer21_attention_weight_max": 31.7618408203125,
      "activations/layer21_attention_weight_min": -25.47125244140625,
      "activations/layer22_attention_weight_max": 47.67993927001953,
      "activations/layer22_attention_weight_min": -32.25494384765625,
      "activations/layer23_attention_weight_max": 37.490108489990234,
      "activations/layer23_attention_weight_min": -26.43019676208496,
      "activations/layer2_attention_weight_max": 33.128013610839844,
      "activations/layer2_attention_weight_min": -32.72994613647461,
      "activations/layer3_attention_weight_max": 100.34712219238281,
      "activations/layer3_attention_weight_min": -101.16854095458984,
      "activations/layer4_attention_weight_max": 113.8934326171875,
      "activations/layer4_attention_weight_min": -113.1357192993164,
      "activations/layer5_attention_weight_max": 57.31640625,
      "activations/layer5_attention_weight_min": -61.828857421875,
      "activations/layer6_attention_weight_max": 46.94123077392578,
      "activations/layer6_attention_weight_min": -49.50660705566406,
      "activations/layer7_attention_weight_max": 102.73674774169922,
      "activations/layer7_attention_weight_min": -95.06800842285156,
      "activations/layer8_attention_weight_max": 48.03998947143555,
      "activations/layer8_attention_weight_min": -44.103248596191406,
      "activations/layer9_attention_weight_max": 38.610023498535156,
      "activations/layer9_attention_weight_min": -35.51060104370117,
      "epoch": 13.19,
      "learning_rate": 6.562083333333333e-05,
      "loss": 2.7755,
      "step": 226950
    },
    {
      "activations/layer0_attention_weight_max": 15.428528785705566,
      "activations/layer0_attention_weight_min": -11.95251178741455,
      "activations/layer10_attention_weight_max": 33.86830520629883,
      "activations/layer10_attention_weight_min": -31.824241638183594,
      "activations/layer11_attention_weight_max": 32.600162506103516,
      "activations/layer11_attention_weight_min": -31.972156524658203,
      "activations/layer12_attention_weight_max": 23.882543563842773,
      "activations/layer12_attention_weight_min": -25.985740661621094,
      "activations/layer13_attention_weight_max": 36.81515121459961,
      "activations/layer13_attention_weight_min": -34.03403854370117,
      "activations/layer14_attention_weight_max": 39.80158233642578,
      "activations/layer14_attention_weight_min": -36.715579986572266,
      "activations/layer15_attention_weight_max": 35.319366455078125,
      "activations/layer15_attention_weight_min": -32.215065002441406,
      "activations/layer16_attention_weight_max": 28.99701690673828,
      "activations/layer16_attention_weight_min": -27.84206199645996,
      "activations/layer17_attention_weight_max": 29.161190032958984,
      "activations/layer17_attention_weight_min": -26.738178253173828,
      "activations/layer18_attention_weight_max": 30.439918518066406,
      "activations/layer18_attention_weight_min": -25.857330322265625,
      "activations/layer19_attention_weight_max": 35.38152313232422,
      "activations/layer19_attention_weight_min": -32.10997009277344,
      "activations/layer1_attention_weight_max": 16.209943771362305,
      "activations/layer1_attention_weight_min": -14.6236572265625,
      "activations/layer20_attention_weight_max": 32.68523406982422,
      "activations/layer20_attention_weight_min": -25.755313873291016,
      "activations/layer21_attention_weight_max": 31.94432258605957,
      "activations/layer21_attention_weight_min": -24.56523323059082,
      "activations/layer22_attention_weight_max": 42.007720947265625,
      "activations/layer22_attention_weight_min": -31.89137077331543,
      "activations/layer23_attention_weight_max": 36.415828704833984,
      "activations/layer23_attention_weight_min": -26.25897216796875,
      "activations/layer2_attention_weight_max": 36.21643829345703,
      "activations/layer2_attention_weight_min": -33.51432800292969,
      "activations/layer3_attention_weight_max": 99.46748352050781,
      "activations/layer3_attention_weight_min": -97.38562774658203,
      "activations/layer4_attention_weight_max": 110.55207824707031,
      "activations/layer4_attention_weight_min": -109.99574279785156,
      "activations/layer5_attention_weight_max": 52.72683334350586,
      "activations/layer5_attention_weight_min": -60.28046798706055,
      "activations/layer6_attention_weight_max": 42.6316032409668,
      "activations/layer6_attention_weight_min": -45.047584533691406,
      "activations/layer7_attention_weight_max": 89.68280029296875,
      "activations/layer7_attention_weight_min": -91.17742919921875,
      "activations/layer8_attention_weight_max": 40.34751510620117,
      "activations/layer8_attention_weight_min": -40.28064727783203,
      "activations/layer9_attention_weight_max": 32.706024169921875,
      "activations/layer9_attention_weight_min": -31.883468627929688,
      "epoch": 13.19,
      "learning_rate": 6.560189393939394e-05,
      "loss": 2.7422,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_loss": 2.71875,
      "eval_runtime": 8.5234,
      "eval_samples_per_second": 503.79,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.5234,
      "eval_openwebtext_samples_per_second": 503.79,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9311,
      "eval_wikitext_samples_per_second": 236.134,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_lambada_loss": 2.44140625,
      "eval_lambada_ppl": 11.489186055851967,
      "eval_lambada_runtime": 10.5789,
      "eval_lambada_samples_per_second": 460.256,
      "step": 227000
    },
    {
      "activations/layer0_attention_weight_max": 15.705723762512207,
      "activations/layer0_attention_weight_min": -12.113351821899414,
      "activations/layer10_attention_weight_max": 31.72593879699707,
      "activations/layer10_attention_weight_min": -32.550418853759766,
      "activations/layer11_attention_weight_max": 34.64189147949219,
      "activations/layer11_attention_weight_min": -33.73514938354492,
      "activations/layer12_attention_weight_max": 24.430021286010742,
      "activations/layer12_attention_weight_min": -30.447996139526367,
      "activations/layer13_attention_weight_max": 36.93006134033203,
      "activations/layer13_attention_weight_min": -34.383758544921875,
      "activations/layer14_attention_weight_max": 39.4561653137207,
      "activations/layer14_attention_weight_min": -36.91350173950195,
      "activations/layer15_attention_weight_max": 35.503334045410156,
      "activations/layer15_attention_weight_min": -34.42171859741211,
      "activations/layer16_attention_weight_max": 28.96762466430664,
      "activations/layer16_attention_weight_min": -29.78497886657715,
      "activations/layer17_attention_weight_max": 30.484079360961914,
      "activations/layer17_attention_weight_min": -28.251619338989258,
      "activations/layer18_attention_weight_max": 33.081504821777344,
      "activations/layer18_attention_weight_min": -25.820009231567383,
      "activations/layer19_attention_weight_max": 35.382083892822266,
      "activations/layer19_attention_weight_min": -32.49502182006836,
      "activations/layer1_attention_weight_max": 15.791668891906738,
      "activations/layer1_attention_weight_min": -14.983784675598145,
      "activations/layer20_attention_weight_max": 32.480995178222656,
      "activations/layer20_attention_weight_min": -26.364768981933594,
      "activations/layer21_attention_weight_max": 32.62123107910156,
      "activations/layer21_attention_weight_min": -24.000818252563477,
      "activations/layer22_attention_weight_max": 43.73200225830078,
      "activations/layer22_attention_weight_min": -30.81535530090332,
      "activations/layer23_attention_weight_max": 38.41991424560547,
      "activations/layer23_attention_weight_min": -27.025455474853516,
      "activations/layer2_attention_weight_max": 34.73282241821289,
      "activations/layer2_attention_weight_min": -33.671112060546875,
      "activations/layer3_attention_weight_max": 96.3572006225586,
      "activations/layer3_attention_weight_min": -98.51802825927734,
      "activations/layer4_attention_weight_max": 106.26179504394531,
      "activations/layer4_attention_weight_min": -112.1794662475586,
      "activations/layer5_attention_weight_max": 52.67726135253906,
      "activations/layer5_attention_weight_min": -63.4379997253418,
      "activations/layer6_attention_weight_max": 41.76290512084961,
      "activations/layer6_attention_weight_min": -48.88741683959961,
      "activations/layer7_attention_weight_max": 84.3547134399414,
      "activations/layer7_attention_weight_min": -91.60477447509766,
      "activations/layer8_attention_weight_max": 38.80337142944336,
      "activations/layer8_attention_weight_min": -42.55744552612305,
      "activations/layer9_attention_weight_max": 33.02003479003906,
      "activations/layer9_attention_weight_min": -34.256954193115234,
      "epoch": 13.19,
      "learning_rate": 6.558295454545454e-05,
      "loss": 2.7703,
      "step": 227050
    },
    {
      "activations/layer0_attention_weight_max": 16.15572166442871,
      "activations/layer0_attention_weight_min": -12.04246711730957,
      "activations/layer10_attention_weight_max": 32.62889099121094,
      "activations/layer10_attention_weight_min": -34.25331115722656,
      "activations/layer11_attention_weight_max": 31.906238555908203,
      "activations/layer11_attention_weight_min": -32.634246826171875,
      "activations/layer12_attention_weight_max": 24.511091232299805,
      "activations/layer12_attention_weight_min": -24.663164138793945,
      "activations/layer13_attention_weight_max": 36.17511749267578,
      "activations/layer13_attention_weight_min": -35.890174865722656,
      "activations/layer14_attention_weight_max": 39.48729705810547,
      "activations/layer14_attention_weight_min": -37.84165573120117,
      "activations/layer15_attention_weight_max": 34.72346878051758,
      "activations/layer15_attention_weight_min": -32.51579666137695,
      "activations/layer16_attention_weight_max": 27.50233268737793,
      "activations/layer16_attention_weight_min": -27.38710594177246,
      "activations/layer17_attention_weight_max": 28.711389541625977,
      "activations/layer17_attention_weight_min": -25.26293182373047,
      "activations/layer18_attention_weight_max": 30.61778450012207,
      "activations/layer18_attention_weight_min": -23.72861671447754,
      "activations/layer19_attention_weight_max": 30.860395431518555,
      "activations/layer19_attention_weight_min": -31.258604049682617,
      "activations/layer1_attention_weight_max": 15.788488388061523,
      "activations/layer1_attention_weight_min": -14.103546142578125,
      "activations/layer20_attention_weight_max": 27.03814697265625,
      "activations/layer20_attention_weight_min": -26.36678123474121,
      "activations/layer21_attention_weight_max": 27.834190368652344,
      "activations/layer21_attention_weight_min": -24.870817184448242,
      "activations/layer22_attention_weight_max": 44.49709701538086,
      "activations/layer22_attention_weight_min": -34.16614532470703,
      "activations/layer23_attention_weight_max": 33.008056640625,
      "activations/layer23_attention_weight_min": -27.301786422729492,
      "activations/layer2_attention_weight_max": 31.2004451751709,
      "activations/layer2_attention_weight_min": -31.735998153686523,
      "activations/layer3_attention_weight_max": 94.974853515625,
      "activations/layer3_attention_weight_min": -94.99622344970703,
      "activations/layer4_attention_weight_max": 105.67769622802734,
      "activations/layer4_attention_weight_min": -110.2564926147461,
      "activations/layer5_attention_weight_max": 49.83518981933594,
      "activations/layer5_attention_weight_min": -61.4632568359375,
      "activations/layer6_attention_weight_max": 44.01811218261719,
      "activations/layer6_attention_weight_min": -45.31874465942383,
      "activations/layer7_attention_weight_max": 87.03424835205078,
      "activations/layer7_attention_weight_min": -89.57395935058594,
      "activations/layer8_attention_weight_max": 39.40796661376953,
      "activations/layer8_attention_weight_min": -40.960365295410156,
      "activations/layer9_attention_weight_max": 31.5671443939209,
      "activations/layer9_attention_weight_min": -35.064266204833984,
      "epoch": 13.2,
      "learning_rate": 6.556401515151514e-05,
      "loss": 2.7644,
      "step": 227100
    },
    {
      "activations/layer0_attention_weight_max": 15.945677757263184,
      "activations/layer0_attention_weight_min": -12.890844345092773,
      "activations/layer10_attention_weight_max": 33.12221145629883,
      "activations/layer10_attention_weight_min": -34.48426055908203,
      "activations/layer11_attention_weight_max": 32.96937561035156,
      "activations/layer11_attention_weight_min": -34.05194091796875,
      "activations/layer12_attention_weight_max": 24.653703689575195,
      "activations/layer12_attention_weight_min": -26.96453285217285,
      "activations/layer13_attention_weight_max": 36.51750183105469,
      "activations/layer13_attention_weight_min": -38.40044021606445,
      "activations/layer14_attention_weight_max": 43.77748107910156,
      "activations/layer14_attention_weight_min": -42.754398345947266,
      "activations/layer15_attention_weight_max": 36.78468322753906,
      "activations/layer15_attention_weight_min": -35.056766510009766,
      "activations/layer16_attention_weight_max": 31.572933197021484,
      "activations/layer16_attention_weight_min": -29.64352798461914,
      "activations/layer17_attention_weight_max": 30.872446060180664,
      "activations/layer17_attention_weight_min": -26.2404727935791,
      "activations/layer18_attention_weight_max": 31.485368728637695,
      "activations/layer18_attention_weight_min": -23.930503845214844,
      "activations/layer19_attention_weight_max": 33.36381149291992,
      "activations/layer19_attention_weight_min": -31.099685668945312,
      "activations/layer1_attention_weight_max": 16.95232582092285,
      "activations/layer1_attention_weight_min": -14.799847602844238,
      "activations/layer20_attention_weight_max": 29.810678482055664,
      "activations/layer20_attention_weight_min": -24.473859786987305,
      "activations/layer21_attention_weight_max": 29.112497329711914,
      "activations/layer21_attention_weight_min": -25.2032413482666,
      "activations/layer22_attention_weight_max": 45.33295440673828,
      "activations/layer22_attention_weight_min": -31.23079490661621,
      "activations/layer23_attention_weight_max": 34.53034973144531,
      "activations/layer23_attention_weight_min": -24.304275512695312,
      "activations/layer2_attention_weight_max": 34.88214874267578,
      "activations/layer2_attention_weight_min": -35.86610794067383,
      "activations/layer3_attention_weight_max": 98.21983337402344,
      "activations/layer3_attention_weight_min": -102.5450210571289,
      "activations/layer4_attention_weight_max": 111.4798355102539,
      "activations/layer4_attention_weight_min": -117.49236297607422,
      "activations/layer5_attention_weight_max": 50.532222747802734,
      "activations/layer5_attention_weight_min": -61.38977813720703,
      "activations/layer6_attention_weight_max": 48.050697326660156,
      "activations/layer6_attention_weight_min": -48.991519927978516,
      "activations/layer7_attention_weight_max": 88.00164794921875,
      "activations/layer7_attention_weight_min": -99.69554901123047,
      "activations/layer8_attention_weight_max": 42.3795051574707,
      "activations/layer8_attention_weight_min": -44.93474197387695,
      "activations/layer9_attention_weight_max": 34.31800842285156,
      "activations/layer9_attention_weight_min": -34.290523529052734,
      "epoch": 13.2,
      "learning_rate": 6.554507575757575e-05,
      "loss": 2.7809,
      "step": 227150
    },
    {
      "activations/layer0_attention_weight_max": 15.947805404663086,
      "activations/layer0_attention_weight_min": -12.636871337890625,
      "activations/layer10_attention_weight_max": 36.451148986816406,
      "activations/layer10_attention_weight_min": -34.47304153442383,
      "activations/layer11_attention_weight_max": 38.80987548828125,
      "activations/layer11_attention_weight_min": -36.29376220703125,
      "activations/layer12_attention_weight_max": 26.723209381103516,
      "activations/layer12_attention_weight_min": -24.63280487060547,
      "activations/layer13_attention_weight_max": 38.26036834716797,
      "activations/layer13_attention_weight_min": -31.940082550048828,
      "activations/layer14_attention_weight_max": 41.51945495605469,
      "activations/layer14_attention_weight_min": -35.4429817199707,
      "activations/layer15_attention_weight_max": 34.52295684814453,
      "activations/layer15_attention_weight_min": -31.190340042114258,
      "activations/layer16_attention_weight_max": 27.672639846801758,
      "activations/layer16_attention_weight_min": -27.862699508666992,
      "activations/layer17_attention_weight_max": 28.45292091369629,
      "activations/layer17_attention_weight_min": -25.19236946105957,
      "activations/layer18_attention_weight_max": 29.262210845947266,
      "activations/layer18_attention_weight_min": -25.598081588745117,
      "activations/layer19_attention_weight_max": 34.860740661621094,
      "activations/layer19_attention_weight_min": -29.643545150756836,
      "activations/layer1_attention_weight_max": 16.27096176147461,
      "activations/layer1_attention_weight_min": -14.007840156555176,
      "activations/layer20_attention_weight_max": 25.409284591674805,
      "activations/layer20_attention_weight_min": -23.416879653930664,
      "activations/layer21_attention_weight_max": 26.445341110229492,
      "activations/layer21_attention_weight_min": -23.169870376586914,
      "activations/layer22_attention_weight_max": 37.40996170043945,
      "activations/layer22_attention_weight_min": -28.554183959960938,
      "activations/layer23_attention_weight_max": 30.113510131835938,
      "activations/layer23_attention_weight_min": -24.158300399780273,
      "activations/layer2_attention_weight_max": 31.912567138671875,
      "activations/layer2_attention_weight_min": -32.91565704345703,
      "activations/layer3_attention_weight_max": 94.16191864013672,
      "activations/layer3_attention_weight_min": -97.63963317871094,
      "activations/layer4_attention_weight_max": 105.36894989013672,
      "activations/layer4_attention_weight_min": -105.93409729003906,
      "activations/layer5_attention_weight_max": 53.86126708984375,
      "activations/layer5_attention_weight_min": -58.65414047241211,
      "activations/layer6_attention_weight_max": 41.49455261230469,
      "activations/layer6_attention_weight_min": -45.35890197753906,
      "activations/layer7_attention_weight_max": 97.93870544433594,
      "activations/layer7_attention_weight_min": -89.39214324951172,
      "activations/layer8_attention_weight_max": 42.172183990478516,
      "activations/layer8_attention_weight_min": -42.463035583496094,
      "activations/layer9_attention_weight_max": 35.89134216308594,
      "activations/layer9_attention_weight_min": -35.502567291259766,
      "epoch": 13.2,
      "learning_rate": 6.552613636363636e-05,
      "loss": 2.7424,
      "step": 227200
    },
    {
      "activations/layer0_attention_weight_max": 15.564624786376953,
      "activations/layer0_attention_weight_min": -12.29977798461914,
      "activations/layer10_attention_weight_max": 30.686620712280273,
      "activations/layer10_attention_weight_min": -32.201141357421875,
      "activations/layer11_attention_weight_max": 31.028736114501953,
      "activations/layer11_attention_weight_min": -31.89763641357422,
      "activations/layer12_attention_weight_max": 24.749059677124023,
      "activations/layer12_attention_weight_min": -24.274805068969727,
      "activations/layer13_attention_weight_max": 35.617034912109375,
      "activations/layer13_attention_weight_min": -33.92323303222656,
      "activations/layer14_attention_weight_max": 40.48491287231445,
      "activations/layer14_attention_weight_min": -39.14304733276367,
      "activations/layer15_attention_weight_max": 35.17625427246094,
      "activations/layer15_attention_weight_min": -31.79732894897461,
      "activations/layer16_attention_weight_max": 29.166961669921875,
      "activations/layer16_attention_weight_min": -28.140554428100586,
      "activations/layer17_attention_weight_max": 35.81673049926758,
      "activations/layer17_attention_weight_min": -24.72876739501953,
      "activations/layer18_attention_weight_max": 30.591527938842773,
      "activations/layer18_attention_weight_min": -25.989011764526367,
      "activations/layer19_attention_weight_max": 34.64198303222656,
      "activations/layer19_attention_weight_min": -29.425140380859375,
      "activations/layer1_attention_weight_max": 15.678510665893555,
      "activations/layer1_attention_weight_min": -14.30872631072998,
      "activations/layer20_attention_weight_max": 28.411083221435547,
      "activations/layer20_attention_weight_min": -25.019834518432617,
      "activations/layer21_attention_weight_max": 28.454965591430664,
      "activations/layer21_attention_weight_min": -23.38006019592285,
      "activations/layer22_attention_weight_max": 43.51576232910156,
      "activations/layer22_attention_weight_min": -29.45928192138672,
      "activations/layer23_attention_weight_max": 33.42939376831055,
      "activations/layer23_attention_weight_min": -25.448814392089844,
      "activations/layer2_attention_weight_max": 33.3184814453125,
      "activations/layer2_attention_weight_min": -33.3816032409668,
      "activations/layer3_attention_weight_max": 98.48271942138672,
      "activations/layer3_attention_weight_min": -99.27088928222656,
      "activations/layer4_attention_weight_max": 106.53472137451172,
      "activations/layer4_attention_weight_min": -113.16340637207031,
      "activations/layer5_attention_weight_max": 49.14949035644531,
      "activations/layer5_attention_weight_min": -62.56578063964844,
      "activations/layer6_attention_weight_max": 43.26597213745117,
      "activations/layer6_attention_weight_min": -47.26913833618164,
      "activations/layer7_attention_weight_max": 86.04853820800781,
      "activations/layer7_attention_weight_min": -86.77874755859375,
      "activations/layer8_attention_weight_max": 38.20475387573242,
      "activations/layer8_attention_weight_min": -42.11976623535156,
      "activations/layer9_attention_weight_max": 30.57931900024414,
      "activations/layer9_attention_weight_min": -31.387725830078125,
      "epoch": 13.2,
      "learning_rate": 6.550719696969696e-05,
      "loss": 2.7754,
      "step": 227250
    },
    {
      "activations/layer0_attention_weight_max": 16.022235870361328,
      "activations/layer0_attention_weight_min": -12.288966178894043,
      "activations/layer10_attention_weight_max": 30.338665008544922,
      "activations/layer10_attention_weight_min": -32.1285400390625,
      "activations/layer11_attention_weight_max": 30.493785858154297,
      "activations/layer11_attention_weight_min": -31.318883895874023,
      "activations/layer12_attention_weight_max": 25.257709503173828,
      "activations/layer12_attention_weight_min": -25.00149154663086,
      "activations/layer13_attention_weight_max": 37.315914154052734,
      "activations/layer13_attention_weight_min": -33.38765335083008,
      "activations/layer14_attention_weight_max": 43.765411376953125,
      "activations/layer14_attention_weight_min": -36.79546356201172,
      "activations/layer15_attention_weight_max": 35.06362533569336,
      "activations/layer15_attention_weight_min": -31.755203247070312,
      "activations/layer16_attention_weight_max": 30.305761337280273,
      "activations/layer16_attention_weight_min": -26.621469497680664,
      "activations/layer17_attention_weight_max": 29.44697380065918,
      "activations/layer17_attention_weight_min": -24.709495544433594,
      "activations/layer18_attention_weight_max": 31.798995971679688,
      "activations/layer18_attention_weight_min": -24.646530151367188,
      "activations/layer19_attention_weight_max": 33.993221282958984,
      "activations/layer19_attention_weight_min": -31.063552856445312,
      "activations/layer1_attention_weight_max": 16.35931396484375,
      "activations/layer1_attention_weight_min": -15.234288215637207,
      "activations/layer20_attention_weight_max": 27.721588134765625,
      "activations/layer20_attention_weight_min": -24.49358558654785,
      "activations/layer21_attention_weight_max": 25.56110191345215,
      "activations/layer21_attention_weight_min": -24.190444946289062,
      "activations/layer22_attention_weight_max": 42.6068115234375,
      "activations/layer22_attention_weight_min": -30.76326560974121,
      "activations/layer23_attention_weight_max": 29.149166107177734,
      "activations/layer23_attention_weight_min": -25.801559448242188,
      "activations/layer2_attention_weight_max": 33.33582305908203,
      "activations/layer2_attention_weight_min": -33.73855972290039,
      "activations/layer3_attention_weight_max": 95.96092987060547,
      "activations/layer3_attention_weight_min": -99.03376770019531,
      "activations/layer4_attention_weight_max": 107.98114013671875,
      "activations/layer4_attention_weight_min": -114.1811294555664,
      "activations/layer5_attention_weight_max": 50.579368591308594,
      "activations/layer5_attention_weight_min": -62.098846435546875,
      "activations/layer6_attention_weight_max": 42.05376434326172,
      "activations/layer6_attention_weight_min": -46.255332946777344,
      "activations/layer7_attention_weight_max": 85.0324935913086,
      "activations/layer7_attention_weight_min": -89.22370147705078,
      "activations/layer8_attention_weight_max": 37.46131134033203,
      "activations/layer8_attention_weight_min": -42.105506896972656,
      "activations/layer9_attention_weight_max": 30.342193603515625,
      "activations/layer9_attention_weight_min": -31.47780418395996,
      "epoch": 13.21,
      "learning_rate": 6.548825757575756e-05,
      "loss": 2.7389,
      "step": 227300
    },
    {
      "activations/layer0_attention_weight_max": 17.789630889892578,
      "activations/layer0_attention_weight_min": -12.142488479614258,
      "activations/layer10_attention_weight_max": 32.533447265625,
      "activations/layer10_attention_weight_min": -34.168212890625,
      "activations/layer11_attention_weight_max": 33.74924087524414,
      "activations/layer11_attention_weight_min": -31.813392639160156,
      "activations/layer12_attention_weight_max": 24.66937255859375,
      "activations/layer12_attention_weight_min": -25.53006935119629,
      "activations/layer13_attention_weight_max": 38.68504333496094,
      "activations/layer13_attention_weight_min": -34.712982177734375,
      "activations/layer14_attention_weight_max": 42.64924621582031,
      "activations/layer14_attention_weight_min": -36.02416229248047,
      "activations/layer15_attention_weight_max": 35.478843688964844,
      "activations/layer15_attention_weight_min": -33.39988708496094,
      "activations/layer16_attention_weight_max": 30.09018898010254,
      "activations/layer16_attention_weight_min": -27.873348236083984,
      "activations/layer17_attention_weight_max": 33.2273063659668,
      "activations/layer17_attention_weight_min": -26.2919864654541,
      "activations/layer18_attention_weight_max": 31.320402145385742,
      "activations/layer18_attention_weight_min": -24.9744930267334,
      "activations/layer19_attention_weight_max": 31.380163192749023,
      "activations/layer19_attention_weight_min": -31.224454879760742,
      "activations/layer1_attention_weight_max": 16.991313934326172,
      "activations/layer1_attention_weight_min": -16.204689025878906,
      "activations/layer20_attention_weight_max": 30.001340866088867,
      "activations/layer20_attention_weight_min": -24.220911026000977,
      "activations/layer21_attention_weight_max": 28.150192260742188,
      "activations/layer21_attention_weight_min": -23.809738159179688,
      "activations/layer22_attention_weight_max": 40.07594299316406,
      "activations/layer22_attention_weight_min": -30.06602668762207,
      "activations/layer23_attention_weight_max": 32.95064163208008,
      "activations/layer23_attention_weight_min": -25.013126373291016,
      "activations/layer2_attention_weight_max": 33.85997772216797,
      "activations/layer2_attention_weight_min": -34.50994873046875,
      "activations/layer3_attention_weight_max": 98.50346374511719,
      "activations/layer3_attention_weight_min": -97.41477966308594,
      "activations/layer4_attention_weight_max": 110.20738983154297,
      "activations/layer4_attention_weight_min": -110.55134582519531,
      "activations/layer5_attention_weight_max": 52.17504119873047,
      "activations/layer5_attention_weight_min": -61.85892868041992,
      "activations/layer6_attention_weight_max": 46.662227630615234,
      "activations/layer6_attention_weight_min": -48.9168586730957,
      "activations/layer7_attention_weight_max": 90.30467987060547,
      "activations/layer7_attention_weight_min": -92.58311462402344,
      "activations/layer8_attention_weight_max": 40.5132942199707,
      "activations/layer8_attention_weight_min": -45.53386306762695,
      "activations/layer9_attention_weight_max": 32.27904510498047,
      "activations/layer9_attention_weight_min": -33.678802490234375,
      "epoch": 13.21,
      "learning_rate": 6.546931818181818e-05,
      "loss": 2.7694,
      "step": 227350
    },
    {
      "activations/layer0_attention_weight_max": 15.745524406433105,
      "activations/layer0_attention_weight_min": -12.311227798461914,
      "activations/layer10_attention_weight_max": 36.26667022705078,
      "activations/layer10_attention_weight_min": -36.21105194091797,
      "activations/layer11_attention_weight_max": 34.174739837646484,
      "activations/layer11_attention_weight_min": -34.46498107910156,
      "activations/layer12_attention_weight_max": 27.89079475402832,
      "activations/layer12_attention_weight_min": -26.518518447875977,
      "activations/layer13_attention_weight_max": 42.39232635498047,
      "activations/layer13_attention_weight_min": -36.6191520690918,
      "activations/layer14_attention_weight_max": 47.53120803833008,
      "activations/layer14_attention_weight_min": -39.317501068115234,
      "activations/layer15_attention_weight_max": 41.436161041259766,
      "activations/layer15_attention_weight_min": -35.46146011352539,
      "activations/layer16_attention_weight_max": 31.103199005126953,
      "activations/layer16_attention_weight_min": -29.758956909179688,
      "activations/layer17_attention_weight_max": 31.15188980102539,
      "activations/layer17_attention_weight_min": -26.04216766357422,
      "activations/layer18_attention_weight_max": 32.015201568603516,
      "activations/layer18_attention_weight_min": -24.097171783447266,
      "activations/layer19_attention_weight_max": 39.154808044433594,
      "activations/layer19_attention_weight_min": -31.667692184448242,
      "activations/layer1_attention_weight_max": 15.44648551940918,
      "activations/layer1_attention_weight_min": -13.666448593139648,
      "activations/layer20_attention_weight_max": 31.942821502685547,
      "activations/layer20_attention_weight_min": -25.10093879699707,
      "activations/layer21_attention_weight_max": 28.990482330322266,
      "activations/layer21_attention_weight_min": -23.861209869384766,
      "activations/layer22_attention_weight_max": 48.118690490722656,
      "activations/layer22_attention_weight_min": -30.675537109375,
      "activations/layer23_attention_weight_max": 36.46282958984375,
      "activations/layer23_attention_weight_min": -25.73019027709961,
      "activations/layer2_attention_weight_max": 32.90895462036133,
      "activations/layer2_attention_weight_min": -33.78498458862305,
      "activations/layer3_attention_weight_max": 102.6295394897461,
      "activations/layer3_attention_weight_min": -99.16095733642578,
      "activations/layer4_attention_weight_max": 116.21843719482422,
      "activations/layer4_attention_weight_min": -112.79422760009766,
      "activations/layer5_attention_weight_max": 54.94031524658203,
      "activations/layer5_attention_weight_min": -61.57408905029297,
      "activations/layer6_attention_weight_max": 46.42007827758789,
      "activations/layer6_attention_weight_min": -46.691165924072266,
      "activations/layer7_attention_weight_max": 97.25021362304688,
      "activations/layer7_attention_weight_min": -92.61703491210938,
      "activations/layer8_attention_weight_max": 42.53239822387695,
      "activations/layer8_attention_weight_min": -43.28774642944336,
      "activations/layer9_attention_weight_max": 34.07011032104492,
      "activations/layer9_attention_weight_min": -35.93853759765625,
      "epoch": 13.21,
      "learning_rate": 6.545037878787878e-05,
      "loss": 2.7584,
      "step": 227400
    },
    {
      "activations/layer0_attention_weight_max": 16.348604202270508,
      "activations/layer0_attention_weight_min": -12.663386344909668,
      "activations/layer10_attention_weight_max": 32.034908294677734,
      "activations/layer10_attention_weight_min": -32.746116638183594,
      "activations/layer11_attention_weight_max": 33.210113525390625,
      "activations/layer11_attention_weight_min": -33.02313232421875,
      "activations/layer12_attention_weight_max": 26.770566940307617,
      "activations/layer12_attention_weight_min": -25.275487899780273,
      "activations/layer13_attention_weight_max": 41.57850646972656,
      "activations/layer13_attention_weight_min": -34.84130859375,
      "activations/layer14_attention_weight_max": 47.16764831542969,
      "activations/layer14_attention_weight_min": -36.79349136352539,
      "activations/layer15_attention_weight_max": 39.31689453125,
      "activations/layer15_attention_weight_min": -34.5073127746582,
      "activations/layer16_attention_weight_max": 32.97996139526367,
      "activations/layer16_attention_weight_min": -28.21953773498535,
      "activations/layer17_attention_weight_max": 31.897518157958984,
      "activations/layer17_attention_weight_min": -26.162935256958008,
      "activations/layer18_attention_weight_max": 39.5402946472168,
      "activations/layer18_attention_weight_min": -23.98177719116211,
      "activations/layer19_attention_weight_max": 40.2966423034668,
      "activations/layer19_attention_weight_min": -32.173927307128906,
      "activations/layer1_attention_weight_max": 15.652247428894043,
      "activations/layer1_attention_weight_min": -15.23410415649414,
      "activations/layer20_attention_weight_max": 31.280235290527344,
      "activations/layer20_attention_weight_min": -25.596250534057617,
      "activations/layer21_attention_weight_max": 31.47293472290039,
      "activations/layer21_attention_weight_min": -24.237417221069336,
      "activations/layer22_attention_weight_max": 48.465572357177734,
      "activations/layer22_attention_weight_min": -30.320478439331055,
      "activations/layer23_attention_weight_max": 37.15020751953125,
      "activations/layer23_attention_weight_min": -25.929473876953125,
      "activations/layer2_attention_weight_max": 32.96119689941406,
      "activations/layer2_attention_weight_min": -32.851402282714844,
      "activations/layer3_attention_weight_max": 94.7158203125,
      "activations/layer3_attention_weight_min": -100.0264892578125,
      "activations/layer4_attention_weight_max": 108.98439025878906,
      "activations/layer4_attention_weight_min": -110.58330535888672,
      "activations/layer5_attention_weight_max": 53.7451171875,
      "activations/layer5_attention_weight_min": -60.93867111206055,
      "activations/layer6_attention_weight_max": 43.708065032958984,
      "activations/layer6_attention_weight_min": -48.62856674194336,
      "activations/layer7_attention_weight_max": 86.34978485107422,
      "activations/layer7_attention_weight_min": -94.51991271972656,
      "activations/layer8_attention_weight_max": 39.57139587402344,
      "activations/layer8_attention_weight_min": -43.66187286376953,
      "activations/layer9_attention_weight_max": 32.5473518371582,
      "activations/layer9_attention_weight_min": -33.81843185424805,
      "epoch": 13.22,
      "learning_rate": 6.543143939393938e-05,
      "loss": 2.7471,
      "step": 227450
    },
    {
      "activations/layer0_attention_weight_max": 16.94063377380371,
      "activations/layer0_attention_weight_min": -12.646594047546387,
      "activations/layer10_attention_weight_max": 32.096160888671875,
      "activations/layer10_attention_weight_min": -33.724082946777344,
      "activations/layer11_attention_weight_max": 30.873363494873047,
      "activations/layer11_attention_weight_min": -31.763324737548828,
      "activations/layer12_attention_weight_max": 26.828693389892578,
      "activations/layer12_attention_weight_min": -24.542713165283203,
      "activations/layer13_attention_weight_max": 38.936988830566406,
      "activations/layer13_attention_weight_min": -33.78499984741211,
      "activations/layer14_attention_weight_max": 44.44115447998047,
      "activations/layer14_attention_weight_min": -36.443092346191406,
      "activations/layer15_attention_weight_max": 38.348388671875,
      "activations/layer15_attention_weight_min": -34.185020446777344,
      "activations/layer16_attention_weight_max": 32.71049118041992,
      "activations/layer16_attention_weight_min": -29.318622589111328,
      "activations/layer17_attention_weight_max": 31.808069229125977,
      "activations/layer17_attention_weight_min": -26.04421615600586,
      "activations/layer18_attention_weight_max": 35.87933349609375,
      "activations/layer18_attention_weight_min": -24.526721954345703,
      "activations/layer19_attention_weight_max": 34.27676010131836,
      "activations/layer19_attention_weight_min": -31.07672119140625,
      "activations/layer1_attention_weight_max": 15.177550315856934,
      "activations/layer1_attention_weight_min": -13.802486419677734,
      "activations/layer20_attention_weight_max": 29.12824058532715,
      "activations/layer20_attention_weight_min": -27.410539627075195,
      "activations/layer21_attention_weight_max": 29.5802001953125,
      "activations/layer21_attention_weight_min": -24.903772354125977,
      "activations/layer22_attention_weight_max": 43.60240936279297,
      "activations/layer22_attention_weight_min": -32.165836334228516,
      "activations/layer23_attention_weight_max": 38.016807556152344,
      "activations/layer23_attention_weight_min": -24.127395629882812,
      "activations/layer2_attention_weight_max": 30.029556274414062,
      "activations/layer2_attention_weight_min": -31.742246627807617,
      "activations/layer3_attention_weight_max": 89.80828094482422,
      "activations/layer3_attention_weight_min": -95.37739562988281,
      "activations/layer4_attention_weight_max": 104.70930480957031,
      "activations/layer4_attention_weight_min": -107.38629150390625,
      "activations/layer5_attention_weight_max": 49.76697540283203,
      "activations/layer5_attention_weight_min": -61.26630401611328,
      "activations/layer6_attention_weight_max": 41.94292449951172,
      "activations/layer6_attention_weight_min": -48.31526565551758,
      "activations/layer7_attention_weight_max": 87.76709747314453,
      "activations/layer7_attention_weight_min": -94.04833984375,
      "activations/layer8_attention_weight_max": 38.69203567504883,
      "activations/layer8_attention_weight_min": -42.026283264160156,
      "activations/layer9_attention_weight_max": 33.96693801879883,
      "activations/layer9_attention_weight_min": -34.68693161010742,
      "epoch": 13.22,
      "learning_rate": 6.54125e-05,
      "loss": 2.7419,
      "step": 227500
    },
    {
      "activations/layer0_attention_weight_max": 18.182832717895508,
      "activations/layer0_attention_weight_min": -12.678898811340332,
      "activations/layer10_attention_weight_max": 30.675731658935547,
      "activations/layer10_attention_weight_min": -32.53990936279297,
      "activations/layer11_attention_weight_max": 31.22760581970215,
      "activations/layer11_attention_weight_min": -32.51394271850586,
      "activations/layer12_attention_weight_max": 26.477697372436523,
      "activations/layer12_attention_weight_min": -26.165912628173828,
      "activations/layer13_attention_weight_max": 39.848915100097656,
      "activations/layer13_attention_weight_min": -32.81002426147461,
      "activations/layer14_attention_weight_max": 49.137271881103516,
      "activations/layer14_attention_weight_min": -36.1578483581543,
      "activations/layer15_attention_weight_max": 39.516258239746094,
      "activations/layer15_attention_weight_min": -32.71390151977539,
      "activations/layer16_attention_weight_max": 38.009464263916016,
      "activations/layer16_attention_weight_min": -28.187297821044922,
      "activations/layer17_attention_weight_max": 32.08347702026367,
      "activations/layer17_attention_weight_min": -25.456541061401367,
      "activations/layer18_attention_weight_max": 30.376995086669922,
      "activations/layer18_attention_weight_min": -22.99644660949707,
      "activations/layer19_attention_weight_max": 39.25809097290039,
      "activations/layer19_attention_weight_min": -34.97136688232422,
      "activations/layer1_attention_weight_max": 16.203847885131836,
      "activations/layer1_attention_weight_min": -14.811551094055176,
      "activations/layer20_attention_weight_max": 32.95354461669922,
      "activations/layer20_attention_weight_min": -26.688631057739258,
      "activations/layer21_attention_weight_max": 35.959407806396484,
      "activations/layer21_attention_weight_min": -25.351160049438477,
      "activations/layer22_attention_weight_max": 43.98843765258789,
      "activations/layer22_attention_weight_min": -32.77620315551758,
      "activations/layer23_attention_weight_max": 40.15087890625,
      "activations/layer23_attention_weight_min": -27.573789596557617,
      "activations/layer2_attention_weight_max": 31.34095573425293,
      "activations/layer2_attention_weight_min": -31.795169830322266,
      "activations/layer3_attention_weight_max": 91.7018814086914,
      "activations/layer3_attention_weight_min": -97.37630462646484,
      "activations/layer4_attention_weight_max": 106.41633605957031,
      "activations/layer4_attention_weight_min": -107.4524154663086,
      "activations/layer5_attention_weight_max": 51.19248580932617,
      "activations/layer5_attention_weight_min": -62.16546630859375,
      "activations/layer6_attention_weight_max": 42.698909759521484,
      "activations/layer6_attention_weight_min": -46.53126907348633,
      "activations/layer7_attention_weight_max": 88.25404357910156,
      "activations/layer7_attention_weight_min": -91.34749603271484,
      "activations/layer8_attention_weight_max": 39.85127258300781,
      "activations/layer8_attention_weight_min": -43.7004280090332,
      "activations/layer9_attention_weight_max": 30.335535049438477,
      "activations/layer9_attention_weight_min": -32.50190353393555,
      "epoch": 13.22,
      "learning_rate": 6.53935606060606e-05,
      "loss": 2.763,
      "step": 227550
    },
    {
      "activations/layer0_attention_weight_max": 15.57201099395752,
      "activations/layer0_attention_weight_min": -12.600897789001465,
      "activations/layer10_attention_weight_max": 31.94442367553711,
      "activations/layer10_attention_weight_min": -32.25248336791992,
      "activations/layer11_attention_weight_max": 32.089256286621094,
      "activations/layer11_attention_weight_min": -31.162824630737305,
      "activations/layer12_attention_weight_max": 25.374313354492188,
      "activations/layer12_attention_weight_min": -24.57284927368164,
      "activations/layer13_attention_weight_max": 39.62496566772461,
      "activations/layer13_attention_weight_min": -31.217952728271484,
      "activations/layer14_attention_weight_max": 39.080997467041016,
      "activations/layer14_attention_weight_min": -34.1934928894043,
      "activations/layer15_attention_weight_max": 36.11363983154297,
      "activations/layer15_attention_weight_min": -32.30156707763672,
      "activations/layer16_attention_weight_max": 30.85032081604004,
      "activations/layer16_attention_weight_min": -27.47834014892578,
      "activations/layer17_attention_weight_max": 32.41670608520508,
      "activations/layer17_attention_weight_min": -25.337596893310547,
      "activations/layer18_attention_weight_max": 32.25665283203125,
      "activations/layer18_attention_weight_min": -23.28310203552246,
      "activations/layer19_attention_weight_max": 32.07743835449219,
      "activations/layer19_attention_weight_min": -31.71649169921875,
      "activations/layer1_attention_weight_max": 16.80725860595703,
      "activations/layer1_attention_weight_min": -13.316222190856934,
      "activations/layer20_attention_weight_max": 28.52220916748047,
      "activations/layer20_attention_weight_min": -23.168479919433594,
      "activations/layer21_attention_weight_max": 28.242626190185547,
      "activations/layer21_attention_weight_min": -22.907695770263672,
      "activations/layer22_attention_weight_max": 48.351505279541016,
      "activations/layer22_attention_weight_min": -29.043231964111328,
      "activations/layer23_attention_weight_max": 34.96564865112305,
      "activations/layer23_attention_weight_min": -24.04975700378418,
      "activations/layer2_attention_weight_max": 30.850990295410156,
      "activations/layer2_attention_weight_min": -31.74085807800293,
      "activations/layer3_attention_weight_max": 95.44959259033203,
      "activations/layer3_attention_weight_min": -97.46351623535156,
      "activations/layer4_attention_weight_max": 108.40631103515625,
      "activations/layer4_attention_weight_min": -114.35820770263672,
      "activations/layer5_attention_weight_max": 52.6126708984375,
      "activations/layer5_attention_weight_min": -61.94903564453125,
      "activations/layer6_attention_weight_max": 44.64042663574219,
      "activations/layer6_attention_weight_min": -48.431846618652344,
      "activations/layer7_attention_weight_max": 93.12252044677734,
      "activations/layer7_attention_weight_min": -97.60295104980469,
      "activations/layer8_attention_weight_max": 40.15645217895508,
      "activations/layer8_attention_weight_min": -42.08234786987305,
      "activations/layer9_attention_weight_max": 32.922908782958984,
      "activations/layer9_attention_weight_min": -32.91399002075195,
      "epoch": 13.22,
      "learning_rate": 6.537462121212121e-05,
      "loss": 2.7578,
      "step": 227600
    },
    {
      "activations/layer0_attention_weight_max": 15.84078311920166,
      "activations/layer0_attention_weight_min": -12.616425514221191,
      "activations/layer10_attention_weight_max": 32.607032775878906,
      "activations/layer10_attention_weight_min": -34.641754150390625,
      "activations/layer11_attention_weight_max": 34.67822265625,
      "activations/layer11_attention_weight_min": -32.81565475463867,
      "activations/layer12_attention_weight_max": 27.738168716430664,
      "activations/layer12_attention_weight_min": -25.96479034423828,
      "activations/layer13_attention_weight_max": 40.501155853271484,
      "activations/layer13_attention_weight_min": -35.470706939697266,
      "activations/layer14_attention_weight_max": 46.9388542175293,
      "activations/layer14_attention_weight_min": -37.5566291809082,
      "activations/layer15_attention_weight_max": 38.230430603027344,
      "activations/layer15_attention_weight_min": -31.941221237182617,
      "activations/layer16_attention_weight_max": 31.798782348632812,
      "activations/layer16_attention_weight_min": -27.942350387573242,
      "activations/layer17_attention_weight_max": 32.228179931640625,
      "activations/layer17_attention_weight_min": -24.558835983276367,
      "activations/layer18_attention_weight_max": 33.43321990966797,
      "activations/layer18_attention_weight_min": -26.09137535095215,
      "activations/layer19_attention_weight_max": 42.181636810302734,
      "activations/layer19_attention_weight_min": -34.14338684082031,
      "activations/layer1_attention_weight_max": 15.987343788146973,
      "activations/layer1_attention_weight_min": -15.509414672851562,
      "activations/layer20_attention_weight_max": 35.005279541015625,
      "activations/layer20_attention_weight_min": -24.831937789916992,
      "activations/layer21_attention_weight_max": 34.874305725097656,
      "activations/layer21_attention_weight_min": -25.062267303466797,
      "activations/layer22_attention_weight_max": 47.61282730102539,
      "activations/layer22_attention_weight_min": -30.752614974975586,
      "activations/layer23_attention_weight_max": 36.383338928222656,
      "activations/layer23_attention_weight_min": -26.137266159057617,
      "activations/layer2_attention_weight_max": 33.35881423950195,
      "activations/layer2_attention_weight_min": -32.52799987792969,
      "activations/layer3_attention_weight_max": 98.223388671875,
      "activations/layer3_attention_weight_min": -95.92958068847656,
      "activations/layer4_attention_weight_max": 113.2324447631836,
      "activations/layer4_attention_weight_min": -110.37242889404297,
      "activations/layer5_attention_weight_max": 54.242366790771484,
      "activations/layer5_attention_weight_min": -61.0716667175293,
      "activations/layer6_attention_weight_max": 46.46025085449219,
      "activations/layer6_attention_weight_min": -48.676979064941406,
      "activations/layer7_attention_weight_max": 95.66070556640625,
      "activations/layer7_attention_weight_min": -94.13104248046875,
      "activations/layer8_attention_weight_max": 45.62803268432617,
      "activations/layer8_attention_weight_min": -45.671913146972656,
      "activations/layer9_attention_weight_max": 34.075496673583984,
      "activations/layer9_attention_weight_min": -36.379417419433594,
      "epoch": 13.23,
      "learning_rate": 6.535568181818182e-05,
      "loss": 2.7681,
      "step": 227650
    },
    {
      "activations/layer0_attention_weight_max": 15.523694038391113,
      "activations/layer0_attention_weight_min": -12.32453727722168,
      "activations/layer10_attention_weight_max": 37.182167053222656,
      "activations/layer10_attention_weight_min": -35.455345153808594,
      "activations/layer11_attention_weight_max": 37.85246658325195,
      "activations/layer11_attention_weight_min": -36.51971435546875,
      "activations/layer12_attention_weight_max": 26.136110305786133,
      "activations/layer12_attention_weight_min": -26.057315826416016,
      "activations/layer13_attention_weight_max": 38.674095153808594,
      "activations/layer13_attention_weight_min": -37.60163116455078,
      "activations/layer14_attention_weight_max": 42.63447570800781,
      "activations/layer14_attention_weight_min": -38.14613723754883,
      "activations/layer15_attention_weight_max": 36.44756317138672,
      "activations/layer15_attention_weight_min": -32.49479675292969,
      "activations/layer16_attention_weight_max": 29.331708908081055,
      "activations/layer16_attention_weight_min": -26.1486759185791,
      "activations/layer17_attention_weight_max": 31.402984619140625,
      "activations/layer17_attention_weight_min": -25.04144287109375,
      "activations/layer18_attention_weight_max": 30.431331634521484,
      "activations/layer18_attention_weight_min": -22.91345977783203,
      "activations/layer19_attention_weight_max": 37.05182647705078,
      "activations/layer19_attention_weight_min": -32.17145538330078,
      "activations/layer1_attention_weight_max": 15.766167640686035,
      "activations/layer1_attention_weight_min": -15.745777130126953,
      "activations/layer20_attention_weight_max": 30.020051956176758,
      "activations/layer20_attention_weight_min": -24.638017654418945,
      "activations/layer21_attention_weight_max": 29.905532836914062,
      "activations/layer21_attention_weight_min": -25.30303382873535,
      "activations/layer22_attention_weight_max": 51.06646728515625,
      "activations/layer22_attention_weight_min": -31.65056610107422,
      "activations/layer23_attention_weight_max": 36.84166717529297,
      "activations/layer23_attention_weight_min": -26.326711654663086,
      "activations/layer2_attention_weight_max": 33.0389404296875,
      "activations/layer2_attention_weight_min": -33.6626091003418,
      "activations/layer3_attention_weight_max": 97.59505462646484,
      "activations/layer3_attention_weight_min": -100.94956970214844,
      "activations/layer4_attention_weight_max": 112.40412902832031,
      "activations/layer4_attention_weight_min": -110.16278076171875,
      "activations/layer5_attention_weight_max": 53.6146240234375,
      "activations/layer5_attention_weight_min": -60.13001251220703,
      "activations/layer6_attention_weight_max": 47.12309265136719,
      "activations/layer6_attention_weight_min": -48.7090950012207,
      "activations/layer7_attention_weight_max": 98.06095886230469,
      "activations/layer7_attention_weight_min": -101.8881607055664,
      "activations/layer8_attention_weight_max": 44.76677322387695,
      "activations/layer8_attention_weight_min": -45.243499755859375,
      "activations/layer9_attention_weight_max": 35.99851989746094,
      "activations/layer9_attention_weight_min": -35.997310638427734,
      "epoch": 13.23,
      "learning_rate": 6.533674242424242e-05,
      "loss": 2.7638,
      "step": 227700
    },
    {
      "activations/layer0_attention_weight_max": 16.920345306396484,
      "activations/layer0_attention_weight_min": -11.87780475616455,
      "activations/layer10_attention_weight_max": 33.414703369140625,
      "activations/layer10_attention_weight_min": -32.517372131347656,
      "activations/layer11_attention_weight_max": 31.937572479248047,
      "activations/layer11_attention_weight_min": -32.55037307739258,
      "activations/layer12_attention_weight_max": 27.877458572387695,
      "activations/layer12_attention_weight_min": -25.82151985168457,
      "activations/layer13_attention_weight_max": 41.63639831542969,
      "activations/layer13_attention_weight_min": -34.2488899230957,
      "activations/layer14_attention_weight_max": 46.64046096801758,
      "activations/layer14_attention_weight_min": -38.54985046386719,
      "activations/layer15_attention_weight_max": 41.80646896362305,
      "activations/layer15_attention_weight_min": -33.03230667114258,
      "activations/layer16_attention_weight_max": 33.6138801574707,
      "activations/layer16_attention_weight_min": -28.913957595825195,
      "activations/layer17_attention_weight_max": 32.603458404541016,
      "activations/layer17_attention_weight_min": -25.64642906188965,
      "activations/layer18_attention_weight_max": 34.95528030395508,
      "activations/layer18_attention_weight_min": -25.254959106445312,
      "activations/layer19_attention_weight_max": 40.95600509643555,
      "activations/layer19_attention_weight_min": -31.592700958251953,
      "activations/layer1_attention_weight_max": 16.567527770996094,
      "activations/layer1_attention_weight_min": -15.571460723876953,
      "activations/layer20_attention_weight_max": 36.452903747558594,
      "activations/layer20_attention_weight_min": -24.6788272857666,
      "activations/layer21_attention_weight_max": 32.180564880371094,
      "activations/layer21_attention_weight_min": -24.771989822387695,
      "activations/layer22_attention_weight_max": 50.53620910644531,
      "activations/layer22_attention_weight_min": -30.42957878112793,
      "activations/layer23_attention_weight_max": 42.05717468261719,
      "activations/layer23_attention_weight_min": -26.271223068237305,
      "activations/layer2_attention_weight_max": 33.96232223510742,
      "activations/layer2_attention_weight_min": -33.21430969238281,
      "activations/layer3_attention_weight_max": 95.88372802734375,
      "activations/layer3_attention_weight_min": -101.71896362304688,
      "activations/layer4_attention_weight_max": 114.45812225341797,
      "activations/layer4_attention_weight_min": -114.63912200927734,
      "activations/layer5_attention_weight_max": 52.22166442871094,
      "activations/layer5_attention_weight_min": -65.862060546875,
      "activations/layer6_attention_weight_max": 44.561161041259766,
      "activations/layer6_attention_weight_min": -50.609378814697266,
      "activations/layer7_attention_weight_max": 89.87625885009766,
      "activations/layer7_attention_weight_min": -94.63195037841797,
      "activations/layer8_attention_weight_max": 40.67250442504883,
      "activations/layer8_attention_weight_min": -42.72472381591797,
      "activations/layer9_attention_weight_max": 32.10171127319336,
      "activations/layer9_attention_weight_min": -33.06956481933594,
      "epoch": 13.23,
      "learning_rate": 6.531780303030303e-05,
      "loss": 2.7509,
      "step": 227750
    },
    {
      "activations/layer0_attention_weight_max": 16.176158905029297,
      "activations/layer0_attention_weight_min": -11.917088508605957,
      "activations/layer10_attention_weight_max": 33.20530700683594,
      "activations/layer10_attention_weight_min": -35.542293548583984,
      "activations/layer11_attention_weight_max": 34.543617248535156,
      "activations/layer11_attention_weight_min": -33.8409309387207,
      "activations/layer12_attention_weight_max": 27.151615142822266,
      "activations/layer12_attention_weight_min": -28.166032791137695,
      "activations/layer13_attention_weight_max": 43.64234161376953,
      "activations/layer13_attention_weight_min": -37.61528396606445,
      "activations/layer14_attention_weight_max": 48.29743957519531,
      "activations/layer14_attention_weight_min": -42.5810661315918,
      "activations/layer15_attention_weight_max": 41.04948806762695,
      "activations/layer15_attention_weight_min": -37.63603591918945,
      "activations/layer16_attention_weight_max": 32.42422103881836,
      "activations/layer16_attention_weight_min": -28.7086181640625,
      "activations/layer17_attention_weight_max": 31.93772315979004,
      "activations/layer17_attention_weight_min": -27.742900848388672,
      "activations/layer18_attention_weight_max": 31.898160934448242,
      "activations/layer18_attention_weight_min": -22.768163681030273,
      "activations/layer19_attention_weight_max": 34.60103225708008,
      "activations/layer19_attention_weight_min": -30.999588012695312,
      "activations/layer1_attention_weight_max": 15.873076438903809,
      "activations/layer1_attention_weight_min": -14.278882026672363,
      "activations/layer20_attention_weight_max": 31.058006286621094,
      "activations/layer20_attention_weight_min": -25.915376663208008,
      "activations/layer21_attention_weight_max": 32.062583923339844,
      "activations/layer21_attention_weight_min": -24.142532348632812,
      "activations/layer22_attention_weight_max": 46.37727737426758,
      "activations/layer22_attention_weight_min": -28.93186378479004,
      "activations/layer23_attention_weight_max": 36.325340270996094,
      "activations/layer23_attention_weight_min": -24.419986724853516,
      "activations/layer2_attention_weight_max": 32.182838439941406,
      "activations/layer2_attention_weight_min": -32.698577880859375,
      "activations/layer3_attention_weight_max": 96.54100036621094,
      "activations/layer3_attention_weight_min": -97.9716796875,
      "activations/layer4_attention_weight_max": 112.89283752441406,
      "activations/layer4_attention_weight_min": -110.24608612060547,
      "activations/layer5_attention_weight_max": 51.98093795776367,
      "activations/layer5_attention_weight_min": -62.713836669921875,
      "activations/layer6_attention_weight_max": 45.82809066772461,
      "activations/layer6_attention_weight_min": -47.83061218261719,
      "activations/layer7_attention_weight_max": 90.93814849853516,
      "activations/layer7_attention_weight_min": -93.22509765625,
      "activations/layer8_attention_weight_max": 40.50835418701172,
      "activations/layer8_attention_weight_min": -43.20137405395508,
      "activations/layer9_attention_weight_max": 34.428260803222656,
      "activations/layer9_attention_weight_min": -34.4781494140625,
      "epoch": 13.24,
      "learning_rate": 6.529886363636363e-05,
      "loss": 2.7872,
      "step": 227800
    },
    {
      "activations/layer0_attention_weight_max": 15.280692100524902,
      "activations/layer0_attention_weight_min": -11.95496940612793,
      "activations/layer10_attention_weight_max": 32.744537353515625,
      "activations/layer10_attention_weight_min": -34.03208541870117,
      "activations/layer11_attention_weight_max": 33.893280029296875,
      "activations/layer11_attention_weight_min": -32.19445037841797,
      "activations/layer12_attention_weight_max": 26.757007598876953,
      "activations/layer12_attention_weight_min": -25.431421279907227,
      "activations/layer13_attention_weight_max": 41.118797302246094,
      "activations/layer13_attention_weight_min": -35.352088928222656,
      "activations/layer14_attention_weight_max": 47.35484313964844,
      "activations/layer14_attention_weight_min": -40.43331527709961,
      "activations/layer15_attention_weight_max": 41.054847717285156,
      "activations/layer15_attention_weight_min": -33.775733947753906,
      "activations/layer16_attention_weight_max": 35.33648681640625,
      "activations/layer16_attention_weight_min": -28.758617401123047,
      "activations/layer17_attention_weight_max": 33.32185363769531,
      "activations/layer17_attention_weight_min": -25.431821823120117,
      "activations/layer18_attention_weight_max": 31.645883560180664,
      "activations/layer18_attention_weight_min": -24.101289749145508,
      "activations/layer19_attention_weight_max": 34.679866790771484,
      "activations/layer19_attention_weight_min": -29.99556541442871,
      "activations/layer1_attention_weight_max": 16.609817504882812,
      "activations/layer1_attention_weight_min": -15.586771965026855,
      "activations/layer20_attention_weight_max": 30.15065574645996,
      "activations/layer20_attention_weight_min": -26.350847244262695,
      "activations/layer21_attention_weight_max": 30.49395751953125,
      "activations/layer21_attention_weight_min": -24.169889450073242,
      "activations/layer22_attention_weight_max": 44.33584213256836,
      "activations/layer22_attention_weight_min": -31.56092643737793,
      "activations/layer23_attention_weight_max": 35.35279083251953,
      "activations/layer23_attention_weight_min": -26.765148162841797,
      "activations/layer2_attention_weight_max": 34.347557067871094,
      "activations/layer2_attention_weight_min": -31.20220184326172,
      "activations/layer3_attention_weight_max": 99.454345703125,
      "activations/layer3_attention_weight_min": -96.73341369628906,
      "activations/layer4_attention_weight_max": 108.7835922241211,
      "activations/layer4_attention_weight_min": -108.16197967529297,
      "activations/layer5_attention_weight_max": 52.090423583984375,
      "activations/layer5_attention_weight_min": -61.94035339355469,
      "activations/layer6_attention_weight_max": 46.8023567199707,
      "activations/layer6_attention_weight_min": -47.32876205444336,
      "activations/layer7_attention_weight_max": 95.78825378417969,
      "activations/layer7_attention_weight_min": -86.31755828857422,
      "activations/layer8_attention_weight_max": 42.28607177734375,
      "activations/layer8_attention_weight_min": -43.637351989746094,
      "activations/layer9_attention_weight_max": 33.36716842651367,
      "activations/layer9_attention_weight_min": -32.75625228881836,
      "epoch": 13.24,
      "learning_rate": 6.527992424242424e-05,
      "loss": 2.7342,
      "step": 227850
    },
    {
      "activations/layer0_attention_weight_max": 15.703357696533203,
      "activations/layer0_attention_weight_min": -11.721020698547363,
      "activations/layer10_attention_weight_max": 32.809547424316406,
      "activations/layer10_attention_weight_min": -31.915794372558594,
      "activations/layer11_attention_weight_max": 30.929443359375,
      "activations/layer11_attention_weight_min": -30.871234893798828,
      "activations/layer12_attention_weight_max": 23.258909225463867,
      "activations/layer12_attention_weight_min": -26.238121032714844,
      "activations/layer13_attention_weight_max": 35.94401168823242,
      "activations/layer13_attention_weight_min": -34.944679260253906,
      "activations/layer14_attention_weight_max": 38.564945220947266,
      "activations/layer14_attention_weight_min": -36.77399826049805,
      "activations/layer15_attention_weight_max": 35.003753662109375,
      "activations/layer15_attention_weight_min": -31.005216598510742,
      "activations/layer16_attention_weight_max": 29.340288162231445,
      "activations/layer16_attention_weight_min": -27.773286819458008,
      "activations/layer17_attention_weight_max": 26.98373031616211,
      "activations/layer17_attention_weight_min": -25.078914642333984,
      "activations/layer18_attention_weight_max": 29.059650421142578,
      "activations/layer18_attention_weight_min": -24.88884735107422,
      "activations/layer19_attention_weight_max": 32.751888275146484,
      "activations/layer19_attention_weight_min": -31.89700698852539,
      "activations/layer1_attention_weight_max": 16.563621520996094,
      "activations/layer1_attention_weight_min": -13.92129135131836,
      "activations/layer20_attention_weight_max": 26.020450592041016,
      "activations/layer20_attention_weight_min": -26.836217880249023,
      "activations/layer21_attention_weight_max": 25.95902442932129,
      "activations/layer21_attention_weight_min": -26.377073287963867,
      "activations/layer22_attention_weight_max": 41.259117126464844,
      "activations/layer22_attention_weight_min": -34.02854537963867,
      "activations/layer23_attention_weight_max": 30.6956787109375,
      "activations/layer23_attention_weight_min": -28.305496215820312,
      "activations/layer2_attention_weight_max": 31.404830932617188,
      "activations/layer2_attention_weight_min": -31.82149314880371,
      "activations/layer3_attention_weight_max": 95.38568878173828,
      "activations/layer3_attention_weight_min": -95.12832641601562,
      "activations/layer4_attention_weight_max": 105.97191619873047,
      "activations/layer4_attention_weight_min": -109.7623519897461,
      "activations/layer5_attention_weight_max": 52.2327880859375,
      "activations/layer5_attention_weight_min": -62.470001220703125,
      "activations/layer6_attention_weight_max": 43.67213821411133,
      "activations/layer6_attention_weight_min": -46.51123809814453,
      "activations/layer7_attention_weight_max": 87.48517608642578,
      "activations/layer7_attention_weight_min": -92.11099243164062,
      "activations/layer8_attention_weight_max": 40.0890998840332,
      "activations/layer8_attention_weight_min": -41.005638122558594,
      "activations/layer9_attention_weight_max": 31.529672622680664,
      "activations/layer9_attention_weight_min": -32.93140411376953,
      "epoch": 13.24,
      "learning_rate": 6.526098484848484e-05,
      "loss": 2.7724,
      "step": 227900
    },
    {
      "activations/layer0_attention_weight_max": 15.84289836883545,
      "activations/layer0_attention_weight_min": -12.033400535583496,
      "activations/layer10_attention_weight_max": 33.481143951416016,
      "activations/layer10_attention_weight_min": -34.444984436035156,
      "activations/layer11_attention_weight_max": 31.163043975830078,
      "activations/layer11_attention_weight_min": -32.46710205078125,
      "activations/layer12_attention_weight_max": 24.615829467773438,
      "activations/layer12_attention_weight_min": -24.423917770385742,
      "activations/layer13_attention_weight_max": 38.72209167480469,
      "activations/layer13_attention_weight_min": -35.04568862915039,
      "activations/layer14_attention_weight_max": 42.15888595581055,
      "activations/layer14_attention_weight_min": -38.536128997802734,
      "activations/layer15_attention_weight_max": 34.07707977294922,
      "activations/layer15_attention_weight_min": -32.18536376953125,
      "activations/layer16_attention_weight_max": 28.177072525024414,
      "activations/layer16_attention_weight_min": -28.548276901245117,
      "activations/layer17_attention_weight_max": 30.38199806213379,
      "activations/layer17_attention_weight_min": -26.349361419677734,
      "activations/layer18_attention_weight_max": 29.738065719604492,
      "activations/layer18_attention_weight_min": -22.60712242126465,
      "activations/layer19_attention_weight_max": 36.11030197143555,
      "activations/layer19_attention_weight_min": -30.70941162109375,
      "activations/layer1_attention_weight_max": 16.446760177612305,
      "activations/layer1_attention_weight_min": -15.38105583190918,
      "activations/layer20_attention_weight_max": 31.201519012451172,
      "activations/layer20_attention_weight_min": -24.508556365966797,
      "activations/layer21_attention_weight_max": 34.63431930541992,
      "activations/layer21_attention_weight_min": -24.347192764282227,
      "activations/layer22_attention_weight_max": 43.09443664550781,
      "activations/layer22_attention_weight_min": -32.22157287597656,
      "activations/layer23_attention_weight_max": 32.81143569946289,
      "activations/layer23_attention_weight_min": -24.696279525756836,
      "activations/layer2_attention_weight_max": 32.01695251464844,
      "activations/layer2_attention_weight_min": -32.134674072265625,
      "activations/layer3_attention_weight_max": 97.22852325439453,
      "activations/layer3_attention_weight_min": -95.57861328125,
      "activations/layer4_attention_weight_max": 109.99322509765625,
      "activations/layer4_attention_weight_min": -110.16278076171875,
      "activations/layer5_attention_weight_max": 52.21119689941406,
      "activations/layer5_attention_weight_min": -62.031673431396484,
      "activations/layer6_attention_weight_max": 42.749515533447266,
      "activations/layer6_attention_weight_min": -46.85664749145508,
      "activations/layer7_attention_weight_max": 88.5247802734375,
      "activations/layer7_attention_weight_min": -91.7378921508789,
      "activations/layer8_attention_weight_max": 44.85355758666992,
      "activations/layer8_attention_weight_min": -42.4576301574707,
      "activations/layer9_attention_weight_max": 33.2121467590332,
      "activations/layer9_attention_weight_min": -34.517539978027344,
      "epoch": 13.25,
      "learning_rate": 6.524204545454545e-05,
      "loss": 2.7623,
      "step": 227950
    },
    {
      "activations/layer0_attention_weight_max": 15.83648681640625,
      "activations/layer0_attention_weight_min": -11.751258850097656,
      "activations/layer10_attention_weight_max": 31.83451271057129,
      "activations/layer10_attention_weight_min": -32.90523147583008,
      "activations/layer11_attention_weight_max": 32.56733322143555,
      "activations/layer11_attention_weight_min": -31.468711853027344,
      "activations/layer12_attention_weight_max": 25.41118812561035,
      "activations/layer12_attention_weight_min": -26.349157333374023,
      "activations/layer13_attention_weight_max": 41.211631774902344,
      "activations/layer13_attention_weight_min": -35.89453125,
      "activations/layer14_attention_weight_max": 43.49973678588867,
      "activations/layer14_attention_weight_min": -40.07046890258789,
      "activations/layer15_attention_weight_max": 39.27513885498047,
      "activations/layer15_attention_weight_min": -35.290462493896484,
      "activations/layer16_attention_weight_max": 32.72468948364258,
      "activations/layer16_attention_weight_min": -28.262592315673828,
      "activations/layer17_attention_weight_max": 29.93021583557129,
      "activations/layer17_attention_weight_min": -26.33708953857422,
      "activations/layer18_attention_weight_max": 29.30093002319336,
      "activations/layer18_attention_weight_min": -25.345443725585938,
      "activations/layer19_attention_weight_max": 36.050533294677734,
      "activations/layer19_attention_weight_min": -33.205963134765625,
      "activations/layer1_attention_weight_max": 16.89957618713379,
      "activations/layer1_attention_weight_min": -15.379440307617188,
      "activations/layer20_attention_weight_max": 28.387229919433594,
      "activations/layer20_attention_weight_min": -24.810306549072266,
      "activations/layer21_attention_weight_max": 28.08755874633789,
      "activations/layer21_attention_weight_min": -23.331619262695312,
      "activations/layer22_attention_weight_max": 42.7358283996582,
      "activations/layer22_attention_weight_min": -30.747270584106445,
      "activations/layer23_attention_weight_max": 37.496612548828125,
      "activations/layer23_attention_weight_min": -26.37415885925293,
      "activations/layer2_attention_weight_max": 32.14524841308594,
      "activations/layer2_attention_weight_min": -33.453834533691406,
      "activations/layer3_attention_weight_max": 97.62306213378906,
      "activations/layer3_attention_weight_min": -102.10685729980469,
      "activations/layer4_attention_weight_max": 110.62786865234375,
      "activations/layer4_attention_weight_min": -112.83562469482422,
      "activations/layer5_attention_weight_max": 51.905494689941406,
      "activations/layer5_attention_weight_min": -60.42371368408203,
      "activations/layer6_attention_weight_max": 42.84974670410156,
      "activations/layer6_attention_weight_min": -47.483219146728516,
      "activations/layer7_attention_weight_max": 86.5534439086914,
      "activations/layer7_attention_weight_min": -87.80998992919922,
      "activations/layer8_attention_weight_max": 38.647613525390625,
      "activations/layer8_attention_weight_min": -39.55925369262695,
      "activations/layer9_attention_weight_max": 31.867624282836914,
      "activations/layer9_attention_weight_min": -31.898162841796875,
      "epoch": 13.25,
      "learning_rate": 6.522310606060605e-05,
      "loss": 2.7678,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4473,
      "eval_samples_per_second": 508.329,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4473,
      "eval_openwebtext_samples_per_second": 508.329,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 1.9567,
      "eval_wikitext_samples_per_second": 233.045,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_lambada_loss": 2.533203125,
      "eval_lambada_ppl": 12.593781054375055,
      "eval_lambada_runtime": 9.5739,
      "eval_lambada_samples_per_second": 508.572,
      "step": 228000
    },
    {
      "activations/layer0_attention_weight_max": 15.427958488464355,
      "activations/layer0_attention_weight_min": -12.415040969848633,
      "activations/layer10_attention_weight_max": 33.84000778198242,
      "activations/layer10_attention_weight_min": -31.98166847229004,
      "activations/layer11_attention_weight_max": 32.96266174316406,
      "activations/layer11_attention_weight_min": -31.630367279052734,
      "activations/layer12_attention_weight_max": 26.13290023803711,
      "activations/layer12_attention_weight_min": -25.229522705078125,
      "activations/layer13_attention_weight_max": 37.59145736694336,
      "activations/layer13_attention_weight_min": -34.285030364990234,
      "activations/layer14_attention_weight_max": 41.64887619018555,
      "activations/layer14_attention_weight_min": -40.62416076660156,
      "activations/layer15_attention_weight_max": 37.94980239868164,
      "activations/layer15_attention_weight_min": -32.275726318359375,
      "activations/layer16_attention_weight_max": 28.47770881652832,
      "activations/layer16_attention_weight_min": -29.35746955871582,
      "activations/layer17_attention_weight_max": 30.83980941772461,
      "activations/layer17_attention_weight_min": -26.10947608947754,
      "activations/layer18_attention_weight_max": 30.44904899597168,
      "activations/layer18_attention_weight_min": -24.239049911499023,
      "activations/layer19_attention_weight_max": 32.771549224853516,
      "activations/layer19_attention_weight_min": -30.598663330078125,
      "activations/layer1_attention_weight_max": 16.02635383605957,
      "activations/layer1_attention_weight_min": -15.532469749450684,
      "activations/layer20_attention_weight_max": 27.04119300842285,
      "activations/layer20_attention_weight_min": -26.043306350708008,
      "activations/layer21_attention_weight_max": 27.59079933166504,
      "activations/layer21_attention_weight_min": -24.506174087524414,
      "activations/layer22_attention_weight_max": 41.248069763183594,
      "activations/layer22_attention_weight_min": -31.87925148010254,
      "activations/layer23_attention_weight_max": 31.702131271362305,
      "activations/layer23_attention_weight_min": -25.387374877929688,
      "activations/layer2_attention_weight_max": 32.76389694213867,
      "activations/layer2_attention_weight_min": -32.092716217041016,
      "activations/layer3_attention_weight_max": 99.45993041992188,
      "activations/layer3_attention_weight_min": -99.99272918701172,
      "activations/layer4_attention_weight_max": 117.10652160644531,
      "activations/layer4_attention_weight_min": -117.0237808227539,
      "activations/layer5_attention_weight_max": 51.13143539428711,
      "activations/layer5_attention_weight_min": -62.216163635253906,
      "activations/layer6_attention_weight_max": 43.4709358215332,
      "activations/layer6_attention_weight_min": -46.4432258605957,
      "activations/layer7_attention_weight_max": 89.86770629882812,
      "activations/layer7_attention_weight_min": -92.5713119506836,
      "activations/layer8_attention_weight_max": 39.199806213378906,
      "activations/layer8_attention_weight_min": -41.80263900756836,
      "activations/layer9_attention_weight_max": 32.6956901550293,
      "activations/layer9_attention_weight_min": -33.50197982788086,
      "epoch": 13.25,
      "learning_rate": 6.520416666666665e-05,
      "loss": 2.7619,
      "step": 228050
    },
    {
      "activations/layer0_attention_weight_max": 16.122997283935547,
      "activations/layer0_attention_weight_min": -11.494802474975586,
      "activations/layer10_attention_weight_max": 34.455142974853516,
      "activations/layer10_attention_weight_min": -32.93618392944336,
      "activations/layer11_attention_weight_max": 33.2990608215332,
      "activations/layer11_attention_weight_min": -33.002784729003906,
      "activations/layer12_attention_weight_max": 24.829618453979492,
      "activations/layer12_attention_weight_min": -24.58523941040039,
      "activations/layer13_attention_weight_max": 36.97443389892578,
      "activations/layer13_attention_weight_min": -32.375831604003906,
      "activations/layer14_attention_weight_max": 40.43695831298828,
      "activations/layer14_attention_weight_min": -38.15660858154297,
      "activations/layer15_attention_weight_max": 37.13843536376953,
      "activations/layer15_attention_weight_min": -31.34341049194336,
      "activations/layer16_attention_weight_max": 28.093902587890625,
      "activations/layer16_attention_weight_min": -25.703075408935547,
      "activations/layer17_attention_weight_max": 28.364091873168945,
      "activations/layer17_attention_weight_min": -25.10350227355957,
      "activations/layer18_attention_weight_max": 30.29034423828125,
      "activations/layer18_attention_weight_min": -23.644107818603516,
      "activations/layer19_attention_weight_max": 34.040889739990234,
      "activations/layer19_attention_weight_min": -29.64737319946289,
      "activations/layer1_attention_weight_max": 17.127540588378906,
      "activations/layer1_attention_weight_min": -14.623958587646484,
      "activations/layer20_attention_weight_max": 27.615650177001953,
      "activations/layer20_attention_weight_min": -25.491493225097656,
      "activations/layer21_attention_weight_max": 31.065275192260742,
      "activations/layer21_attention_weight_min": -24.713151931762695,
      "activations/layer22_attention_weight_max": 45.906410217285156,
      "activations/layer22_attention_weight_min": -30.57171630859375,
      "activations/layer23_attention_weight_max": 32.146488189697266,
      "activations/layer23_attention_weight_min": -25.371206283569336,
      "activations/layer2_attention_weight_max": 33.636863708496094,
      "activations/layer2_attention_weight_min": -32.61029052734375,
      "activations/layer3_attention_weight_max": 99.86225128173828,
      "activations/layer3_attention_weight_min": -100.0958023071289,
      "activations/layer4_attention_weight_max": 109.66934967041016,
      "activations/layer4_attention_weight_min": -108.93534088134766,
      "activations/layer5_attention_weight_max": 55.72791290283203,
      "activations/layer5_attention_weight_min": -60.20942687988281,
      "activations/layer6_attention_weight_max": 42.744163513183594,
      "activations/layer6_attention_weight_min": -45.709632873535156,
      "activations/layer7_attention_weight_max": 90.43565368652344,
      "activations/layer7_attention_weight_min": -90.64970397949219,
      "activations/layer8_attention_weight_max": 40.801513671875,
      "activations/layer8_attention_weight_min": -41.84212112426758,
      "activations/layer9_attention_weight_max": 32.5865364074707,
      "activations/layer9_attention_weight_min": -33.26837921142578,
      "epoch": 13.25,
      "learning_rate": 6.518522727272727e-05,
      "loss": 2.7547,
      "step": 228100
    },
    {
      "activations/layer0_attention_weight_max": 15.259661674499512,
      "activations/layer0_attention_weight_min": -12.61098861694336,
      "activations/layer10_attention_weight_max": 31.67665672302246,
      "activations/layer10_attention_weight_min": -32.44960403442383,
      "activations/layer11_attention_weight_max": 31.585479736328125,
      "activations/layer11_attention_weight_min": -31.375545501708984,
      "activations/layer12_attention_weight_max": 23.7130069732666,
      "activations/layer12_attention_weight_min": -24.794565200805664,
      "activations/layer13_attention_weight_max": 37.373138427734375,
      "activations/layer13_attention_weight_min": -34.87944793701172,
      "activations/layer14_attention_weight_max": 44.08750534057617,
      "activations/layer14_attention_weight_min": -38.42388153076172,
      "activations/layer15_attention_weight_max": 38.82343673706055,
      "activations/layer15_attention_weight_min": -33.702362060546875,
      "activations/layer16_attention_weight_max": 30.9073486328125,
      "activations/layer16_attention_weight_min": -27.419050216674805,
      "activations/layer17_attention_weight_max": 30.803340911865234,
      "activations/layer17_attention_weight_min": -25.422731399536133,
      "activations/layer18_attention_weight_max": 30.417049407958984,
      "activations/layer18_attention_weight_min": -22.53688621520996,
      "activations/layer19_attention_weight_max": 34.7242546081543,
      "activations/layer19_attention_weight_min": -32.118953704833984,
      "activations/layer1_attention_weight_max": 16.794921875,
      "activations/layer1_attention_weight_min": -15.065949440002441,
      "activations/layer20_attention_weight_max": 33.28286361694336,
      "activations/layer20_attention_weight_min": -24.995786666870117,
      "activations/layer21_attention_weight_max": 37.07185745239258,
      "activations/layer21_attention_weight_min": -23.43036460876465,
      "activations/layer22_attention_weight_max": 47.52330017089844,
      "activations/layer22_attention_weight_min": -30.807186126708984,
      "activations/layer23_attention_weight_max": 33.12108612060547,
      "activations/layer23_attention_weight_min": -28.16269302368164,
      "activations/layer2_attention_weight_max": 31.417715072631836,
      "activations/layer2_attention_weight_min": -32.69845962524414,
      "activations/layer3_attention_weight_max": 96.10977172851562,
      "activations/layer3_attention_weight_min": -96.10401153564453,
      "activations/layer4_attention_weight_max": 110.36573791503906,
      "activations/layer4_attention_weight_min": -115.05047607421875,
      "activations/layer5_attention_weight_max": 52.72959899902344,
      "activations/layer5_attention_weight_min": -62.79499816894531,
      "activations/layer6_attention_weight_max": 43.86415481567383,
      "activations/layer6_attention_weight_min": -47.71122360229492,
      "activations/layer7_attention_weight_max": 90.54395294189453,
      "activations/layer7_attention_weight_min": -91.43719482421875,
      "activations/layer8_attention_weight_max": 39.17538833618164,
      "activations/layer8_attention_weight_min": -40.7612190246582,
      "activations/layer9_attention_weight_max": 30.58349609375,
      "activations/layer9_attention_weight_min": -31.53020668029785,
      "epoch": 13.26,
      "learning_rate": 6.516628787878787e-05,
      "loss": 2.763,
      "step": 228150
    },
    {
      "activations/layer0_attention_weight_max": 16.06171417236328,
      "activations/layer0_attention_weight_min": -12.590777397155762,
      "activations/layer10_attention_weight_max": 33.77766036987305,
      "activations/layer10_attention_weight_min": -32.26347732543945,
      "activations/layer11_attention_weight_max": 35.57567596435547,
      "activations/layer11_attention_weight_min": -31.764822006225586,
      "activations/layer12_attention_weight_max": 24.98122787475586,
      "activations/layer12_attention_weight_min": -25.40624237060547,
      "activations/layer13_attention_weight_max": 38.550506591796875,
      "activations/layer13_attention_weight_min": -33.923828125,
      "activations/layer14_attention_weight_max": 41.78662109375,
      "activations/layer14_attention_weight_min": -37.74632263183594,
      "activations/layer15_attention_weight_max": 36.27306365966797,
      "activations/layer15_attention_weight_min": -34.04152297973633,
      "activations/layer16_attention_weight_max": 30.94026756286621,
      "activations/layer16_attention_weight_min": -30.209062576293945,
      "activations/layer17_attention_weight_max": 31.95576286315918,
      "activations/layer17_attention_weight_min": -28.391389846801758,
      "activations/layer18_attention_weight_max": 29.093332290649414,
      "activations/layer18_attention_weight_min": -25.917348861694336,
      "activations/layer19_attention_weight_max": 34.83779525756836,
      "activations/layer19_attention_weight_min": -37.312503814697266,
      "activations/layer1_attention_weight_max": 16.362747192382812,
      "activations/layer1_attention_weight_min": -15.476194381713867,
      "activations/layer20_attention_weight_max": 26.666114807128906,
      "activations/layer20_attention_weight_min": -29.025543212890625,
      "activations/layer21_attention_weight_max": 27.415103912353516,
      "activations/layer21_attention_weight_min": -28.509056091308594,
      "activations/layer22_attention_weight_max": 38.910823822021484,
      "activations/layer22_attention_weight_min": -33.804779052734375,
      "activations/layer23_attention_weight_max": 33.44070816040039,
      "activations/layer23_attention_weight_min": -26.32714080810547,
      "activations/layer2_attention_weight_max": 32.7524299621582,
      "activations/layer2_attention_weight_min": -32.3473014831543,
      "activations/layer3_attention_weight_max": 96.84534454345703,
      "activations/layer3_attention_weight_min": -98.68116760253906,
      "activations/layer4_attention_weight_max": 110.42274475097656,
      "activations/layer4_attention_weight_min": -109.65826416015625,
      "activations/layer5_attention_weight_max": 52.39144515991211,
      "activations/layer5_attention_weight_min": -60.101192474365234,
      "activations/layer6_attention_weight_max": 44.84996795654297,
      "activations/layer6_attention_weight_min": -47.30745315551758,
      "activations/layer7_attention_weight_max": 86.99555206298828,
      "activations/layer7_attention_weight_min": -88.84955596923828,
      "activations/layer8_attention_weight_max": 38.854270935058594,
      "activations/layer8_attention_weight_min": -43.80705642700195,
      "activations/layer9_attention_weight_max": 32.492244720458984,
      "activations/layer9_attention_weight_min": -31.842435836791992,
      "epoch": 13.26,
      "learning_rate": 6.514734848484849e-05,
      "loss": 2.7683,
      "step": 228200
    },
    {
      "activations/layer0_attention_weight_max": 15.720159530639648,
      "activations/layer0_attention_weight_min": -12.53264331817627,
      "activations/layer10_attention_weight_max": 31.70535659790039,
      "activations/layer10_attention_weight_min": -30.32549285888672,
      "activations/layer11_attention_weight_max": 30.471771240234375,
      "activations/layer11_attention_weight_min": -31.4027156829834,
      "activations/layer12_attention_weight_max": 24.60742950439453,
      "activations/layer12_attention_weight_min": -25.53812026977539,
      "activations/layer13_attention_weight_max": 35.667236328125,
      "activations/layer13_attention_weight_min": -35.18699264526367,
      "activations/layer14_attention_weight_max": 40.24081802368164,
      "activations/layer14_attention_weight_min": -38.3450813293457,
      "activations/layer15_attention_weight_max": 34.77127456665039,
      "activations/layer15_attention_weight_min": -32.002174377441406,
      "activations/layer16_attention_weight_max": 28.01622200012207,
      "activations/layer16_attention_weight_min": -29.249181747436523,
      "activations/layer17_attention_weight_max": 30.118066787719727,
      "activations/layer17_attention_weight_min": -25.024911880493164,
      "activations/layer18_attention_weight_max": 32.66568374633789,
      "activations/layer18_attention_weight_min": -23.508520126342773,
      "activations/layer19_attention_weight_max": 33.05708694458008,
      "activations/layer19_attention_weight_min": -31.213708877563477,
      "activations/layer1_attention_weight_max": 17.058916091918945,
      "activations/layer1_attention_weight_min": -15.446303367614746,
      "activations/layer20_attention_weight_max": 27.738494873046875,
      "activations/layer20_attention_weight_min": -24.358478546142578,
      "activations/layer21_attention_weight_max": 29.682281494140625,
      "activations/layer21_attention_weight_min": -23.381622314453125,
      "activations/layer22_attention_weight_max": 40.97928237915039,
      "activations/layer22_attention_weight_min": -30.100854873657227,
      "activations/layer23_attention_weight_max": 36.111717224121094,
      "activations/layer23_attention_weight_min": -27.461240768432617,
      "activations/layer2_attention_weight_max": 33.66372299194336,
      "activations/layer2_attention_weight_min": -31.566600799560547,
      "activations/layer3_attention_weight_max": 94.9577865600586,
      "activations/layer3_attention_weight_min": -92.38917541503906,
      "activations/layer4_attention_weight_max": 104.62125396728516,
      "activations/layer4_attention_weight_min": -106.33015441894531,
      "activations/layer5_attention_weight_max": 51.0588493347168,
      "activations/layer5_attention_weight_min": -60.03296661376953,
      "activations/layer6_attention_weight_max": 43.72264099121094,
      "activations/layer6_attention_weight_min": -46.04867935180664,
      "activations/layer7_attention_weight_max": 87.15782928466797,
      "activations/layer7_attention_weight_min": -86.10224914550781,
      "activations/layer8_attention_weight_max": 39.967891693115234,
      "activations/layer8_attention_weight_min": -42.13193893432617,
      "activations/layer9_attention_weight_max": 30.50785255432129,
      "activations/layer9_attention_weight_min": -32.14669418334961,
      "epoch": 13.26,
      "learning_rate": 6.512840909090909e-05,
      "loss": 2.768,
      "step": 228250
    },
    {
      "activations/layer0_attention_weight_max": 15.822210311889648,
      "activations/layer0_attention_weight_min": -11.876372337341309,
      "activations/layer10_attention_weight_max": 31.218765258789062,
      "activations/layer10_attention_weight_min": -31.6761474609375,
      "activations/layer11_attention_weight_max": 31.147930145263672,
      "activations/layer11_attention_weight_min": -30.40351104736328,
      "activations/layer12_attention_weight_max": 34.74935531616211,
      "activations/layer12_attention_weight_min": -25.5530948638916,
      "activations/layer13_attention_weight_max": 41.343711853027344,
      "activations/layer13_attention_weight_min": -34.95905685424805,
      "activations/layer14_attention_weight_max": 42.167388916015625,
      "activations/layer14_attention_weight_min": -38.715599060058594,
      "activations/layer15_attention_weight_max": 40.851863861083984,
      "activations/layer15_attention_weight_min": -34.70138931274414,
      "activations/layer16_attention_weight_max": 30.691463470458984,
      "activations/layer16_attention_weight_min": -28.162296295166016,
      "activations/layer17_attention_weight_max": 31.53953742980957,
      "activations/layer17_attention_weight_min": -25.245759963989258,
      "activations/layer18_attention_weight_max": 30.28740882873535,
      "activations/layer18_attention_weight_min": -24.596969604492188,
      "activations/layer19_attention_weight_max": 33.1259651184082,
      "activations/layer19_attention_weight_min": -29.594030380249023,
      "activations/layer1_attention_weight_max": 16.729167938232422,
      "activations/layer1_attention_weight_min": -16.653539657592773,
      "activations/layer20_attention_weight_max": 31.264476776123047,
      "activations/layer20_attention_weight_min": -24.15989875793457,
      "activations/layer21_attention_weight_max": 30.952117919921875,
      "activations/layer21_attention_weight_min": -24.854991912841797,
      "activations/layer22_attention_weight_max": 39.6235237121582,
      "activations/layer22_attention_weight_min": -29.196069717407227,
      "activations/layer23_attention_weight_max": 32.9625129699707,
      "activations/layer23_attention_weight_min": -24.09130096435547,
      "activations/layer2_attention_weight_max": 30.21670150756836,
      "activations/layer2_attention_weight_min": -30.32762908935547,
      "activations/layer3_attention_weight_max": 84.04938507080078,
      "activations/layer3_attention_weight_min": -90.15511322021484,
      "activations/layer4_attention_weight_max": 99.80204772949219,
      "activations/layer4_attention_weight_min": -102.20297241210938,
      "activations/layer5_attention_weight_max": 48.368282318115234,
      "activations/layer5_attention_weight_min": -58.4013671875,
      "activations/layer6_attention_weight_max": 40.93154525756836,
      "activations/layer6_attention_weight_min": -42.789127349853516,
      "activations/layer7_attention_weight_max": 87.70541381835938,
      "activations/layer7_attention_weight_min": -86.86277770996094,
      "activations/layer8_attention_weight_max": 37.11537170410156,
      "activations/layer8_attention_weight_min": -38.61030960083008,
      "activations/layer9_attention_weight_max": 30.851953506469727,
      "activations/layer9_attention_weight_min": -30.91986083984375,
      "epoch": 13.27,
      "learning_rate": 6.510946969696969e-05,
      "loss": 2.7611,
      "step": 228300
    },
    {
      "activations/layer0_attention_weight_max": 15.26047134399414,
      "activations/layer0_attention_weight_min": -11.90259838104248,
      "activations/layer10_attention_weight_max": 30.59776496887207,
      "activations/layer10_attention_weight_min": -32.453697204589844,
      "activations/layer11_attention_weight_max": 31.593944549560547,
      "activations/layer11_attention_weight_min": -30.93545150756836,
      "activations/layer12_attention_weight_max": 25.39579200744629,
      "activations/layer12_attention_weight_min": -26.22199821472168,
      "activations/layer13_attention_weight_max": 39.29692077636719,
      "activations/layer13_attention_weight_min": -35.71631622314453,
      "activations/layer14_attention_weight_max": 46.12908172607422,
      "activations/layer14_attention_weight_min": -36.9549446105957,
      "activations/layer15_attention_weight_max": 39.76724624633789,
      "activations/layer15_attention_weight_min": -32.37736129760742,
      "activations/layer16_attention_weight_max": 30.27004051208496,
      "activations/layer16_attention_weight_min": -28.03692626953125,
      "activations/layer17_attention_weight_max": 32.78944778442383,
      "activations/layer17_attention_weight_min": -26.44329833984375,
      "activations/layer18_attention_weight_max": 33.41877746582031,
      "activations/layer18_attention_weight_min": -24.863283157348633,
      "activations/layer19_attention_weight_max": 37.731075286865234,
      "activations/layer19_attention_weight_min": -34.076934814453125,
      "activations/layer1_attention_weight_max": 16.15945816040039,
      "activations/layer1_attention_weight_min": -15.143284797668457,
      "activations/layer20_attention_weight_max": 30.714677810668945,
      "activations/layer20_attention_weight_min": -26.61985969543457,
      "activations/layer21_attention_weight_max": 31.035736083984375,
      "activations/layer21_attention_weight_min": -27.347990036010742,
      "activations/layer22_attention_weight_max": 43.41089630126953,
      "activations/layer22_attention_weight_min": -32.592201232910156,
      "activations/layer23_attention_weight_max": 35.82614517211914,
      "activations/layer23_attention_weight_min": -27.799633026123047,
      "activations/layer2_attention_weight_max": 31.115942001342773,
      "activations/layer2_attention_weight_min": -32.34132385253906,
      "activations/layer3_attention_weight_max": 93.20591735839844,
      "activations/layer3_attention_weight_min": -93.4759521484375,
      "activations/layer4_attention_weight_max": 107.11016082763672,
      "activations/layer4_attention_weight_min": -106.58060455322266,
      "activations/layer5_attention_weight_max": 51.65436553955078,
      "activations/layer5_attention_weight_min": -60.298362731933594,
      "activations/layer6_attention_weight_max": 45.39943313598633,
      "activations/layer6_attention_weight_min": -46.33740997314453,
      "activations/layer7_attention_weight_max": 88.32774353027344,
      "activations/layer7_attention_weight_min": -88.40885925292969,
      "activations/layer8_attention_weight_max": 38.76407241821289,
      "activations/layer8_attention_weight_min": -38.7592658996582,
      "activations/layer9_attention_weight_max": 31.51150894165039,
      "activations/layer9_attention_weight_min": -31.631567001342773,
      "epoch": 13.27,
      "learning_rate": 6.50905303030303e-05,
      "loss": 2.7496,
      "step": 228350
    },
    {
      "activations/layer0_attention_weight_max": 16.33763313293457,
      "activations/layer0_attention_weight_min": -12.70205307006836,
      "activations/layer10_attention_weight_max": 33.66040802001953,
      "activations/layer10_attention_weight_min": -33.23686599731445,
      "activations/layer11_attention_weight_max": 31.86996078491211,
      "activations/layer11_attention_weight_min": -31.371320724487305,
      "activations/layer12_attention_weight_max": 25.40996742248535,
      "activations/layer12_attention_weight_min": -30.152591705322266,
      "activations/layer13_attention_weight_max": 37.06401443481445,
      "activations/layer13_attention_weight_min": -38.389705657958984,
      "activations/layer14_attention_weight_max": 46.27710723876953,
      "activations/layer14_attention_weight_min": -42.39741134643555,
      "activations/layer15_attention_weight_max": 36.67692565917969,
      "activations/layer15_attention_weight_min": -32.3653450012207,
      "activations/layer16_attention_weight_max": 28.215394973754883,
      "activations/layer16_attention_weight_min": -28.856399536132812,
      "activations/layer17_attention_weight_max": 28.88911247253418,
      "activations/layer17_attention_weight_min": -26.124292373657227,
      "activations/layer18_attention_weight_max": 29.95326042175293,
      "activations/layer18_attention_weight_min": -23.89848518371582,
      "activations/layer19_attention_weight_max": 31.10109519958496,
      "activations/layer19_attention_weight_min": -31.64651870727539,
      "activations/layer1_attention_weight_max": 16.674072265625,
      "activations/layer1_attention_weight_min": -15.079768180847168,
      "activations/layer20_attention_weight_max": 25.972421646118164,
      "activations/layer20_attention_weight_min": -24.140892028808594,
      "activations/layer21_attention_weight_max": 25.30223846435547,
      "activations/layer21_attention_weight_min": -23.918277740478516,
      "activations/layer22_attention_weight_max": 43.82646560668945,
      "activations/layer22_attention_weight_min": -29.53333282470703,
      "activations/layer23_attention_weight_max": 28.515209197998047,
      "activations/layer23_attention_weight_min": -26.451892852783203,
      "activations/layer2_attention_weight_max": 33.12067413330078,
      "activations/layer2_attention_weight_min": -33.020931243896484,
      "activations/layer3_attention_weight_max": 97.80957794189453,
      "activations/layer3_attention_weight_min": -95.60536193847656,
      "activations/layer4_attention_weight_max": 106.96101379394531,
      "activations/layer4_attention_weight_min": -106.9955062866211,
      "activations/layer5_attention_weight_max": 51.80420684814453,
      "activations/layer5_attention_weight_min": -60.90035629272461,
      "activations/layer6_attention_weight_max": 45.815650939941406,
      "activations/layer6_attention_weight_min": -47.044273376464844,
      "activations/layer7_attention_weight_max": 95.12855529785156,
      "activations/layer7_attention_weight_min": -89.3698959350586,
      "activations/layer8_attention_weight_max": 40.871089935302734,
      "activations/layer8_attention_weight_min": -40.14433288574219,
      "activations/layer9_attention_weight_max": 32.31813430786133,
      "activations/layer9_attention_weight_min": -33.65165328979492,
      "epoch": 13.27,
      "learning_rate": 6.50715909090909e-05,
      "loss": 2.7573,
      "step": 228400
    },
    {
      "activations/layer0_attention_weight_max": 15.956698417663574,
      "activations/layer0_attention_weight_min": -13.53683090209961,
      "activations/layer10_attention_weight_max": 31.28359031677246,
      "activations/layer10_attention_weight_min": -32.539554595947266,
      "activations/layer11_attention_weight_max": 32.02842712402344,
      "activations/layer11_attention_weight_min": -31.2535400390625,
      "activations/layer12_attention_weight_max": 24.889453887939453,
      "activations/layer12_attention_weight_min": -25.922380447387695,
      "activations/layer13_attention_weight_max": 35.8499755859375,
      "activations/layer13_attention_weight_min": -34.71883010864258,
      "activations/layer14_attention_weight_max": 43.58749008178711,
      "activations/layer14_attention_weight_min": -38.41523742675781,
      "activations/layer15_attention_weight_max": 36.41221618652344,
      "activations/layer15_attention_weight_min": -33.770530700683594,
      "activations/layer16_attention_weight_max": 29.73358154296875,
      "activations/layer16_attention_weight_min": -27.771142959594727,
      "activations/layer17_attention_weight_max": 32.761268615722656,
      "activations/layer17_attention_weight_min": -26.381771087646484,
      "activations/layer18_attention_weight_max": 32.98009490966797,
      "activations/layer18_attention_weight_min": -24.739540100097656,
      "activations/layer19_attention_weight_max": 33.42332458496094,
      "activations/layer19_attention_weight_min": -32.08534622192383,
      "activations/layer1_attention_weight_max": 15.471383094787598,
      "activations/layer1_attention_weight_min": -14.021364212036133,
      "activations/layer20_attention_weight_max": 26.8928279876709,
      "activations/layer20_attention_weight_min": -24.77523422241211,
      "activations/layer21_attention_weight_max": 28.481876373291016,
      "activations/layer21_attention_weight_min": -23.49237060546875,
      "activations/layer22_attention_weight_max": 39.43630599975586,
      "activations/layer22_attention_weight_min": -28.81084632873535,
      "activations/layer23_attention_weight_max": 32.95457458496094,
      "activations/layer23_attention_weight_min": -26.466278076171875,
      "activations/layer2_attention_weight_max": 31.022647857666016,
      "activations/layer2_attention_weight_min": -31.66802978515625,
      "activations/layer3_attention_weight_max": 93.6697006225586,
      "activations/layer3_attention_weight_min": -95.52938842773438,
      "activations/layer4_attention_weight_max": 107.26434326171875,
      "activations/layer4_attention_weight_min": -115.14745330810547,
      "activations/layer5_attention_weight_max": 52.577423095703125,
      "activations/layer5_attention_weight_min": -61.389827728271484,
      "activations/layer6_attention_weight_max": 43.05111312866211,
      "activations/layer6_attention_weight_min": -45.60175323486328,
      "activations/layer7_attention_weight_max": 89.43447875976562,
      "activations/layer7_attention_weight_min": -91.75774383544922,
      "activations/layer8_attention_weight_max": 41.71278762817383,
      "activations/layer8_attention_weight_min": -38.858238220214844,
      "activations/layer9_attention_weight_max": 30.870227813720703,
      "activations/layer9_attention_weight_min": -31.512338638305664,
      "epoch": 13.27,
      "learning_rate": 6.505265151515151e-05,
      "loss": 2.7588,
      "step": 228450
    },
    {
      "activations/layer0_attention_weight_max": 14.969754219055176,
      "activations/layer0_attention_weight_min": -12.291868209838867,
      "activations/layer10_attention_weight_max": 37.67862319946289,
      "activations/layer10_attention_weight_min": -32.75999450683594,
      "activations/layer11_attention_weight_max": 39.26084899902344,
      "activations/layer11_attention_weight_min": -34.91938400268555,
      "activations/layer12_attention_weight_max": 25.44284439086914,
      "activations/layer12_attention_weight_min": -25.988693237304688,
      "activations/layer13_attention_weight_max": 38.316864013671875,
      "activations/layer13_attention_weight_min": -33.44010543823242,
      "activations/layer14_attention_weight_max": 43.01609420776367,
      "activations/layer14_attention_weight_min": -36.83804702758789,
      "activations/layer15_attention_weight_max": 35.384647369384766,
      "activations/layer15_attention_weight_min": -33.01506805419922,
      "activations/layer16_attention_weight_max": 31.700210571289062,
      "activations/layer16_attention_weight_min": -26.355520248413086,
      "activations/layer17_attention_weight_max": 29.437891006469727,
      "activations/layer17_attention_weight_min": -26.913007736206055,
      "activations/layer18_attention_weight_max": 29.3137264251709,
      "activations/layer18_attention_weight_min": -24.43769073486328,
      "activations/layer19_attention_weight_max": 33.43336868286133,
      "activations/layer19_attention_weight_min": -31.510889053344727,
      "activations/layer1_attention_weight_max": 17.253244400024414,
      "activations/layer1_attention_weight_min": -14.702032089233398,
      "activations/layer20_attention_weight_max": 28.76778793334961,
      "activations/layer20_attention_weight_min": -24.81597328186035,
      "activations/layer21_attention_weight_max": 29.433364868164062,
      "activations/layer21_attention_weight_min": -25.335678100585938,
      "activations/layer22_attention_weight_max": 50.358734130859375,
      "activations/layer22_attention_weight_min": -30.130979537963867,
      "activations/layer23_attention_weight_max": 35.0406608581543,
      "activations/layer23_attention_weight_min": -24.0764217376709,
      "activations/layer2_attention_weight_max": 34.043182373046875,
      "activations/layer2_attention_weight_min": -33.59400177001953,
      "activations/layer3_attention_weight_max": 102.45545196533203,
      "activations/layer3_attention_weight_min": -102.31108093261719,
      "activations/layer4_attention_weight_max": 109.43755340576172,
      "activations/layer4_attention_weight_min": -107.42581939697266,
      "activations/layer5_attention_weight_max": 51.75741195678711,
      "activations/layer5_attention_weight_min": -59.603790283203125,
      "activations/layer6_attention_weight_max": 43.139766693115234,
      "activations/layer6_attention_weight_min": -43.783409118652344,
      "activations/layer7_attention_weight_max": 91.65316009521484,
      "activations/layer7_attention_weight_min": -89.83100891113281,
      "activations/layer8_attention_weight_max": 43.36650848388672,
      "activations/layer8_attention_weight_min": -41.57276153564453,
      "activations/layer9_attention_weight_max": 33.46334457397461,
      "activations/layer9_attention_weight_min": -33.12808609008789,
      "epoch": 13.28,
      "learning_rate": 6.503371212121211e-05,
      "loss": 2.764,
      "step": 228500
    },
    {
      "activations/layer0_attention_weight_max": 15.103429794311523,
      "activations/layer0_attention_weight_min": -13.723779678344727,
      "activations/layer10_attention_weight_max": 31.336925506591797,
      "activations/layer10_attention_weight_min": -32.893959045410156,
      "activations/layer11_attention_weight_max": 33.27726745605469,
      "activations/layer11_attention_weight_min": -33.041568756103516,
      "activations/layer12_attention_weight_max": 29.41122055053711,
      "activations/layer12_attention_weight_min": -24.92975616455078,
      "activations/layer13_attention_weight_max": 46.217960357666016,
      "activations/layer13_attention_weight_min": -34.74565124511719,
      "activations/layer14_attention_weight_max": 48.65373229980469,
      "activations/layer14_attention_weight_min": -36.68185806274414,
      "activations/layer15_attention_weight_max": 39.55303192138672,
      "activations/layer15_attention_weight_min": -31.456920623779297,
      "activations/layer16_attention_weight_max": 33.62947463989258,
      "activations/layer16_attention_weight_min": -27.368595123291016,
      "activations/layer17_attention_weight_max": 32.43901443481445,
      "activations/layer17_attention_weight_min": -24.49775505065918,
      "activations/layer18_attention_weight_max": 35.57978439331055,
      "activations/layer18_attention_weight_min": -25.055757522583008,
      "activations/layer19_attention_weight_max": 38.2899284362793,
      "activations/layer19_attention_weight_min": -29.269296646118164,
      "activations/layer1_attention_weight_max": 16.35641860961914,
      "activations/layer1_attention_weight_min": -15.198149681091309,
      "activations/layer20_attention_weight_max": 31.36288070678711,
      "activations/layer20_attention_weight_min": -23.68610191345215,
      "activations/layer21_attention_weight_max": 31.9996337890625,
      "activations/layer21_attention_weight_min": -23.715166091918945,
      "activations/layer22_attention_weight_max": 49.71580505371094,
      "activations/layer22_attention_weight_min": -30.41776466369629,
      "activations/layer23_attention_weight_max": 41.585411071777344,
      "activations/layer23_attention_weight_min": -24.45958709716797,
      "activations/layer2_attention_weight_max": 33.170413970947266,
      "activations/layer2_attention_weight_min": -32.353363037109375,
      "activations/layer3_attention_weight_max": 98.95829772949219,
      "activations/layer3_attention_weight_min": -100.9380111694336,
      "activations/layer4_attention_weight_max": 107.21590423583984,
      "activations/layer4_attention_weight_min": -110.41336822509766,
      "activations/layer5_attention_weight_max": 52.1318359375,
      "activations/layer5_attention_weight_min": -65.29010009765625,
      "activations/layer6_attention_weight_max": 46.83525848388672,
      "activations/layer6_attention_weight_min": -47.54644012451172,
      "activations/layer7_attention_weight_max": 94.73020935058594,
      "activations/layer7_attention_weight_min": -97.2510986328125,
      "activations/layer8_attention_weight_max": 42.38218688964844,
      "activations/layer8_attention_weight_min": -43.23714828491211,
      "activations/layer9_attention_weight_max": 33.66921615600586,
      "activations/layer9_attention_weight_min": -34.81704330444336,
      "epoch": 13.28,
      "learning_rate": 6.501477272727272e-05,
      "loss": 2.759,
      "step": 228550
    },
    {
      "activations/layer0_attention_weight_max": 15.173964500427246,
      "activations/layer0_attention_weight_min": -12.189275741577148,
      "activations/layer10_attention_weight_max": 31.628538131713867,
      "activations/layer10_attention_weight_min": -32.98859405517578,
      "activations/layer11_attention_weight_max": 31.13272476196289,
      "activations/layer11_attention_weight_min": -33.144439697265625,
      "activations/layer12_attention_weight_max": 25.623023986816406,
      "activations/layer12_attention_weight_min": -25.599369049072266,
      "activations/layer13_attention_weight_max": 37.840606689453125,
      "activations/layer13_attention_weight_min": -33.94776916503906,
      "activations/layer14_attention_weight_max": 45.75409698486328,
      "activations/layer14_attention_weight_min": -37.01728439331055,
      "activations/layer15_attention_weight_max": 37.88364028930664,
      "activations/layer15_attention_weight_min": -34.05015182495117,
      "activations/layer16_attention_weight_max": 31.720096588134766,
      "activations/layer16_attention_weight_min": -29.63604164123535,
      "activations/layer17_attention_weight_max": 30.33747100830078,
      "activations/layer17_attention_weight_min": -25.57742691040039,
      "activations/layer18_attention_weight_max": 31.238834381103516,
      "activations/layer18_attention_weight_min": -24.43925666809082,
      "activations/layer19_attention_weight_max": 34.680702209472656,
      "activations/layer19_attention_weight_min": -32.0139045715332,
      "activations/layer1_attention_weight_max": 16.60250473022461,
      "activations/layer1_attention_weight_min": -15.280412673950195,
      "activations/layer20_attention_weight_max": 29.667156219482422,
      "activations/layer20_attention_weight_min": -24.785770416259766,
      "activations/layer21_attention_weight_max": 28.861881256103516,
      "activations/layer21_attention_weight_min": -24.034271240234375,
      "activations/layer22_attention_weight_max": 43.59999465942383,
      "activations/layer22_attention_weight_min": -32.948909759521484,
      "activations/layer23_attention_weight_max": 33.70915985107422,
      "activations/layer23_attention_weight_min": -24.63034439086914,
      "activations/layer2_attention_weight_max": 32.62669372558594,
      "activations/layer2_attention_weight_min": -32.681488037109375,
      "activations/layer3_attention_weight_max": 95.6883544921875,
      "activations/layer3_attention_weight_min": -97.26473236083984,
      "activations/layer4_attention_weight_max": 109.13648986816406,
      "activations/layer4_attention_weight_min": -111.39305877685547,
      "activations/layer5_attention_weight_max": 49.69439697265625,
      "activations/layer5_attention_weight_min": -62.07505798339844,
      "activations/layer6_attention_weight_max": 45.121131896972656,
      "activations/layer6_attention_weight_min": -45.97583770751953,
      "activations/layer7_attention_weight_max": 89.26023864746094,
      "activations/layer7_attention_weight_min": -92.08505249023438,
      "activations/layer8_attention_weight_max": 38.79609298706055,
      "activations/layer8_attention_weight_min": -41.51371765136719,
      "activations/layer9_attention_weight_max": 31.783777236938477,
      "activations/layer9_attention_weight_min": -32.811588287353516,
      "epoch": 13.28,
      "learning_rate": 6.499583333333333e-05,
      "loss": 2.7557,
      "step": 228600
    },
    {
      "activations/layer0_attention_weight_max": 16.901498794555664,
      "activations/layer0_attention_weight_min": -12.428325653076172,
      "activations/layer10_attention_weight_max": 32.68735885620117,
      "activations/layer10_attention_weight_min": -32.287086486816406,
      "activations/layer11_attention_weight_max": 33.422637939453125,
      "activations/layer11_attention_weight_min": -33.029903411865234,
      "activations/layer12_attention_weight_max": 25.88982391357422,
      "activations/layer12_attention_weight_min": -27.039283752441406,
      "activations/layer13_attention_weight_max": 37.53942108154297,
      "activations/layer13_attention_weight_min": -33.281219482421875,
      "activations/layer14_attention_weight_max": 41.612754821777344,
      "activations/layer14_attention_weight_min": -37.10627365112305,
      "activations/layer15_attention_weight_max": 37.6716423034668,
      "activations/layer15_attention_weight_min": -31.774093627929688,
      "activations/layer16_attention_weight_max": 32.27858352661133,
      "activations/layer16_attention_weight_min": -28.70964241027832,
      "activations/layer17_attention_weight_max": 31.424856185913086,
      "activations/layer17_attention_weight_min": -26.865886688232422,
      "activations/layer18_attention_weight_max": 33.37513732910156,
      "activations/layer18_attention_weight_min": -23.542766571044922,
      "activations/layer19_attention_weight_max": 39.34587478637695,
      "activations/layer19_attention_weight_min": -31.291763305664062,
      "activations/layer1_attention_weight_max": 16.602737426757812,
      "activations/layer1_attention_weight_min": -14.396673202514648,
      "activations/layer20_attention_weight_max": 32.15788269042969,
      "activations/layer20_attention_weight_min": -25.15194320678711,
      "activations/layer21_attention_weight_max": 30.388769149780273,
      "activations/layer21_attention_weight_min": -24.894676208496094,
      "activations/layer22_attention_weight_max": 47.22216796875,
      "activations/layer22_attention_weight_min": -31.420583724975586,
      "activations/layer23_attention_weight_max": 37.05607986450195,
      "activations/layer23_attention_weight_min": -25.57368278503418,
      "activations/layer2_attention_weight_max": 33.6087532043457,
      "activations/layer2_attention_weight_min": -32.469295501708984,
      "activations/layer3_attention_weight_max": 98.72649383544922,
      "activations/layer3_attention_weight_min": -97.86283874511719,
      "activations/layer4_attention_weight_max": 110.6410140991211,
      "activations/layer4_attention_weight_min": -108.42579650878906,
      "activations/layer5_attention_weight_max": 51.40186309814453,
      "activations/layer5_attention_weight_min": -59.60091781616211,
      "activations/layer6_attention_weight_max": 46.11445999145508,
      "activations/layer6_attention_weight_min": -48.357303619384766,
      "activations/layer7_attention_weight_max": 90.62604522705078,
      "activations/layer7_attention_weight_min": -93.19629669189453,
      "activations/layer8_attention_weight_max": 41.60041809082031,
      "activations/layer8_attention_weight_min": -42.49827575683594,
      "activations/layer9_attention_weight_max": 33.859317779541016,
      "activations/layer9_attention_weight_min": -34.2021598815918,
      "epoch": 13.29,
      "learning_rate": 6.497689393939393e-05,
      "loss": 2.7532,
      "step": 228650
    },
    {
      "activations/layer0_attention_weight_max": 16.384281158447266,
      "activations/layer0_attention_weight_min": -12.146524429321289,
      "activations/layer10_attention_weight_max": 33.895477294921875,
      "activations/layer10_attention_weight_min": -33.45567321777344,
      "activations/layer11_attention_weight_max": 36.07378387451172,
      "activations/layer11_attention_weight_min": -33.10701370239258,
      "activations/layer12_attention_weight_max": 25.000064849853516,
      "activations/layer12_attention_weight_min": -25.95808219909668,
      "activations/layer13_attention_weight_max": 37.78801345825195,
      "activations/layer13_attention_weight_min": -33.90740966796875,
      "activations/layer14_attention_weight_max": 43.498878479003906,
      "activations/layer14_attention_weight_min": -37.29955291748047,
      "activations/layer15_attention_weight_max": 39.70082092285156,
      "activations/layer15_attention_weight_min": -34.7989501953125,
      "activations/layer16_attention_weight_max": 30.38805389404297,
      "activations/layer16_attention_weight_min": -29.651851654052734,
      "activations/layer17_attention_weight_max": 30.43699836730957,
      "activations/layer17_attention_weight_min": -26.43509864807129,
      "activations/layer18_attention_weight_max": 30.358205795288086,
      "activations/layer18_attention_weight_min": -23.211868286132812,
      "activations/layer19_attention_weight_max": 34.424137115478516,
      "activations/layer19_attention_weight_min": -31.289560317993164,
      "activations/layer1_attention_weight_max": 18.254545211791992,
      "activations/layer1_attention_weight_min": -15.608111381530762,
      "activations/layer20_attention_weight_max": 30.505985260009766,
      "activations/layer20_attention_weight_min": -25.28218650817871,
      "activations/layer21_attention_weight_max": 28.262386322021484,
      "activations/layer21_attention_weight_min": -26.697738647460938,
      "activations/layer22_attention_weight_max": 43.624088287353516,
      "activations/layer22_attention_weight_min": -28.771379470825195,
      "activations/layer23_attention_weight_max": 33.50904846191406,
      "activations/layer23_attention_weight_min": -23.060333251953125,
      "activations/layer2_attention_weight_max": 32.98345184326172,
      "activations/layer2_attention_weight_min": -35.51327896118164,
      "activations/layer3_attention_weight_max": 100.63462829589844,
      "activations/layer3_attention_weight_min": -99.19697570800781,
      "activations/layer4_attention_weight_max": 108.652587890625,
      "activations/layer4_attention_weight_min": -110.92150115966797,
      "activations/layer5_attention_weight_max": 50.97176742553711,
      "activations/layer5_attention_weight_min": -59.70101547241211,
      "activations/layer6_attention_weight_max": 45.63001251220703,
      "activations/layer6_attention_weight_min": -47.09975814819336,
      "activations/layer7_attention_weight_max": 89.06462097167969,
      "activations/layer7_attention_weight_min": -88.6359634399414,
      "activations/layer8_attention_weight_max": 42.5350456237793,
      "activations/layer8_attention_weight_min": -41.5065803527832,
      "activations/layer9_attention_weight_max": 33.689918518066406,
      "activations/layer9_attention_weight_min": -32.33501052856445,
      "epoch": 13.29,
      "learning_rate": 6.495795454545454e-05,
      "loss": 2.7579,
      "step": 228700
    },
    {
      "activations/layer0_attention_weight_max": 16.042909622192383,
      "activations/layer0_attention_weight_min": -12.835640907287598,
      "activations/layer10_attention_weight_max": 32.06671142578125,
      "activations/layer10_attention_weight_min": -32.390960693359375,
      "activations/layer11_attention_weight_max": 31.918785095214844,
      "activations/layer11_attention_weight_min": -31.341642379760742,
      "activations/layer12_attention_weight_max": 25.660629272460938,
      "activations/layer12_attention_weight_min": -25.692304611206055,
      "activations/layer13_attention_weight_max": 40.53547286987305,
      "activations/layer13_attention_weight_min": -35.10454177856445,
      "activations/layer14_attention_weight_max": 42.51811981201172,
      "activations/layer14_attention_weight_min": -39.36574172973633,
      "activations/layer15_attention_weight_max": 35.495235443115234,
      "activations/layer15_attention_weight_min": -32.30229187011719,
      "activations/layer16_attention_weight_max": 27.578109741210938,
      "activations/layer16_attention_weight_min": -26.528095245361328,
      "activations/layer17_attention_weight_max": 30.62411880493164,
      "activations/layer17_attention_weight_min": -25.246971130371094,
      "activations/layer18_attention_weight_max": 30.563072204589844,
      "activations/layer18_attention_weight_min": -24.613059997558594,
      "activations/layer19_attention_weight_max": 29.298831939697266,
      "activations/layer19_attention_weight_min": -31.17184829711914,
      "activations/layer1_attention_weight_max": 16.04602813720703,
      "activations/layer1_attention_weight_min": -14.97895622253418,
      "activations/layer20_attention_weight_max": 26.60846519470215,
      "activations/layer20_attention_weight_min": -23.633010864257812,
      "activations/layer21_attention_weight_max": 25.236684799194336,
      "activations/layer21_attention_weight_min": -23.54701805114746,
      "activations/layer22_attention_weight_max": 40.8082275390625,
      "activations/layer22_attention_weight_min": -29.282304763793945,
      "activations/layer23_attention_weight_max": 29.301082611083984,
      "activations/layer23_attention_weight_min": -24.933364868164062,
      "activations/layer2_attention_weight_max": 33.26161193847656,
      "activations/layer2_attention_weight_min": -33.50743103027344,
      "activations/layer3_attention_weight_max": 101.00830841064453,
      "activations/layer3_attention_weight_min": -105.46668243408203,
      "activations/layer4_attention_weight_max": 111.84211730957031,
      "activations/layer4_attention_weight_min": -114.1525650024414,
      "activations/layer5_attention_weight_max": 53.13578796386719,
      "activations/layer5_attention_weight_min": -62.559471130371094,
      "activations/layer6_attention_weight_max": 43.21768569946289,
      "activations/layer6_attention_weight_min": -46.43159484863281,
      "activations/layer7_attention_weight_max": 87.23068237304688,
      "activations/layer7_attention_weight_min": -91.68843841552734,
      "activations/layer8_attention_weight_max": 39.063255310058594,
      "activations/layer8_attention_weight_min": -41.82514953613281,
      "activations/layer9_attention_weight_max": 31.806177139282227,
      "activations/layer9_attention_weight_min": -31.724781036376953,
      "epoch": 13.29,
      "learning_rate": 6.493901515151514e-05,
      "loss": 2.7586,
      "step": 228750
    },
    {
      "activations/layer0_attention_weight_max": 15.728167533874512,
      "activations/layer0_attention_weight_min": -12.007946014404297,
      "activations/layer10_attention_weight_max": 32.89508819580078,
      "activations/layer10_attention_weight_min": -34.50275802612305,
      "activations/layer11_attention_weight_max": 32.32273864746094,
      "activations/layer11_attention_weight_min": -33.28814697265625,
      "activations/layer12_attention_weight_max": 24.93133544921875,
      "activations/layer12_attention_weight_min": -25.70993423461914,
      "activations/layer13_attention_weight_max": 37.675315856933594,
      "activations/layer13_attention_weight_min": -35.93440246582031,
      "activations/layer14_attention_weight_max": 47.02573013305664,
      "activations/layer14_attention_weight_min": -37.73152160644531,
      "activations/layer15_attention_weight_max": 38.83454513549805,
      "activations/layer15_attention_weight_min": -33.32770919799805,
      "activations/layer16_attention_weight_max": 31.351696014404297,
      "activations/layer16_attention_weight_min": -27.228824615478516,
      "activations/layer17_attention_weight_max": 32.92521286010742,
      "activations/layer17_attention_weight_min": -25.219423294067383,
      "activations/layer18_attention_weight_max": 33.95237731933594,
      "activations/layer18_attention_weight_min": -23.86644744873047,
      "activations/layer19_attention_weight_max": 36.8070182800293,
      "activations/layer19_attention_weight_min": -31.690540313720703,
      "activations/layer1_attention_weight_max": 15.965333938598633,
      "activations/layer1_attention_weight_min": -14.663468360900879,
      "activations/layer20_attention_weight_max": 30.566505432128906,
      "activations/layer20_attention_weight_min": -26.898714065551758,
      "activations/layer21_attention_weight_max": 28.81635093688965,
      "activations/layer21_attention_weight_min": -24.62090301513672,
      "activations/layer22_attention_weight_max": 47.62102127075195,
      "activations/layer22_attention_weight_min": -31.524154663085938,
      "activations/layer23_attention_weight_max": 36.76957321166992,
      "activations/layer23_attention_weight_min": -24.94611167907715,
      "activations/layer2_attention_weight_max": 32.91783905029297,
      "activations/layer2_attention_weight_min": -34.446807861328125,
      "activations/layer3_attention_weight_max": 97.09544372558594,
      "activations/layer3_attention_weight_min": -101.6004638671875,
      "activations/layer4_attention_weight_max": 108.96556854248047,
      "activations/layer4_attention_weight_min": -111.69255828857422,
      "activations/layer5_attention_weight_max": 51.55503845214844,
      "activations/layer5_attention_weight_min": -63.971107482910156,
      "activations/layer6_attention_weight_max": 44.97019958496094,
      "activations/layer6_attention_weight_min": -48.10147476196289,
      "activations/layer7_attention_weight_max": 90.49920654296875,
      "activations/layer7_attention_weight_min": -88.98722076416016,
      "activations/layer8_attention_weight_max": 40.72649383544922,
      "activations/layer8_attention_weight_min": -42.39447784423828,
      "activations/layer9_attention_weight_max": 34.0618782043457,
      "activations/layer9_attention_weight_min": -34.00106430053711,
      "epoch": 13.29,
      "learning_rate": 6.492007575757574e-05,
      "loss": 2.7398,
      "step": 228800
    },
    {
      "activations/layer0_attention_weight_max": 16.273483276367188,
      "activations/layer0_attention_weight_min": -12.220541000366211,
      "activations/layer10_attention_weight_max": 32.04290771484375,
      "activations/layer10_attention_weight_min": -33.211814880371094,
      "activations/layer11_attention_weight_max": 31.884201049804688,
      "activations/layer11_attention_weight_min": -32.80441665649414,
      "activations/layer12_attention_weight_max": 23.84770393371582,
      "activations/layer12_attention_weight_min": -25.229047775268555,
      "activations/layer13_attention_weight_max": 35.67825698852539,
      "activations/layer13_attention_weight_min": -33.39007568359375,
      "activations/layer14_attention_weight_max": 39.22329330444336,
      "activations/layer14_attention_weight_min": -35.727752685546875,
      "activations/layer15_attention_weight_max": 33.62677764892578,
      "activations/layer15_attention_weight_min": -31.36224365234375,
      "activations/layer16_attention_weight_max": 27.598299026489258,
      "activations/layer16_attention_weight_min": -26.802946090698242,
      "activations/layer17_attention_weight_max": 30.87826919555664,
      "activations/layer17_attention_weight_min": -26.408754348754883,
      "activations/layer18_attention_weight_max": 30.381446838378906,
      "activations/layer18_attention_weight_min": -25.4401912689209,
      "activations/layer19_attention_weight_max": 32.549827575683594,
      "activations/layer19_attention_weight_min": -29.443586349487305,
      "activations/layer1_attention_weight_max": 16.278539657592773,
      "activations/layer1_attention_weight_min": -14.847558975219727,
      "activations/layer20_attention_weight_max": 28.359472274780273,
      "activations/layer20_attention_weight_min": -26.4151554107666,
      "activations/layer21_attention_weight_max": 29.43099594116211,
      "activations/layer21_attention_weight_min": -24.774301528930664,
      "activations/layer22_attention_weight_max": 41.02534866333008,
      "activations/layer22_attention_weight_min": -30.186561584472656,
      "activations/layer23_attention_weight_max": 30.664398193359375,
      "activations/layer23_attention_weight_min": -24.563186645507812,
      "activations/layer2_attention_weight_max": 31.707700729370117,
      "activations/layer2_attention_weight_min": -32.12153625488281,
      "activations/layer3_attention_weight_max": 97.76956939697266,
      "activations/layer3_attention_weight_min": -95.88319396972656,
      "activations/layer4_attention_weight_max": 111.03682708740234,
      "activations/layer4_attention_weight_min": -111.20824432373047,
      "activations/layer5_attention_weight_max": 53.05424499511719,
      "activations/layer5_attention_weight_min": -62.216705322265625,
      "activations/layer6_attention_weight_max": 45.943687438964844,
      "activations/layer6_attention_weight_min": -45.91756057739258,
      "activations/layer7_attention_weight_max": 91.20370483398438,
      "activations/layer7_attention_weight_min": -92.7336654663086,
      "activations/layer8_attention_weight_max": 41.3115119934082,
      "activations/layer8_attention_weight_min": -41.78249740600586,
      "activations/layer9_attention_weight_max": 31.061447143554688,
      "activations/layer9_attention_weight_min": -33.01774215698242,
      "epoch": 13.3,
      "learning_rate": 6.490113636363636e-05,
      "loss": 2.7844,
      "step": 228850
    },
    {
      "activations/layer0_attention_weight_max": 15.598532676696777,
      "activations/layer0_attention_weight_min": -11.637083053588867,
      "activations/layer10_attention_weight_max": 32.85740661621094,
      "activations/layer10_attention_weight_min": -31.97422981262207,
      "activations/layer11_attention_weight_max": 33.87007141113281,
      "activations/layer11_attention_weight_min": -32.45872497558594,
      "activations/layer12_attention_weight_max": 26.289230346679688,
      "activations/layer12_attention_weight_min": -23.818647384643555,
      "activations/layer13_attention_weight_max": 37.22990798950195,
      "activations/layer13_attention_weight_min": -33.51728439331055,
      "activations/layer14_attention_weight_max": 45.318851470947266,
      "activations/layer14_attention_weight_min": -35.47990417480469,
      "activations/layer15_attention_weight_max": 38.26641082763672,
      "activations/layer15_attention_weight_min": -33.813629150390625,
      "activations/layer16_attention_weight_max": 33.70928192138672,
      "activations/layer16_attention_weight_min": -25.76532745361328,
      "activations/layer17_attention_weight_max": 31.02098274230957,
      "activations/layer17_attention_weight_min": -25.670122146606445,
      "activations/layer18_attention_weight_max": 32.87827682495117,
      "activations/layer18_attention_weight_min": -24.576377868652344,
      "activations/layer19_attention_weight_max": 42.26443099975586,
      "activations/layer19_attention_weight_min": -30.396146774291992,
      "activations/layer1_attention_weight_max": 17.71590805053711,
      "activations/layer1_attention_weight_min": -14.5470552444458,
      "activations/layer20_attention_weight_max": 33.33732604980469,
      "activations/layer20_attention_weight_min": -25.9792537689209,
      "activations/layer21_attention_weight_max": 32.62764358520508,
      "activations/layer21_attention_weight_min": -23.849449157714844,
      "activations/layer22_attention_weight_max": 49.906681060791016,
      "activations/layer22_attention_weight_min": -30.463838577270508,
      "activations/layer23_attention_weight_max": 39.196044921875,
      "activations/layer23_attention_weight_min": -29.140335083007812,
      "activations/layer2_attention_weight_max": 31.811038970947266,
      "activations/layer2_attention_weight_min": -34.708614349365234,
      "activations/layer3_attention_weight_max": 99.60850524902344,
      "activations/layer3_attention_weight_min": -105.03901672363281,
      "activations/layer4_attention_weight_max": 105.6865463256836,
      "activations/layer4_attention_weight_min": -111.7094497680664,
      "activations/layer5_attention_weight_max": 54.85449981689453,
      "activations/layer5_attention_weight_min": -62.7393798828125,
      "activations/layer6_attention_weight_max": 44.358638763427734,
      "activations/layer6_attention_weight_min": -46.29429244995117,
      "activations/layer7_attention_weight_max": 91.15989685058594,
      "activations/layer7_attention_weight_min": -91.11603546142578,
      "activations/layer8_attention_weight_max": 39.586341857910156,
      "activations/layer8_attention_weight_min": -44.56996536254883,
      "activations/layer9_attention_weight_max": 31.004491806030273,
      "activations/layer9_attention_weight_min": -33.539466857910156,
      "epoch": 13.3,
      "learning_rate": 6.488219696969696e-05,
      "loss": 2.7671,
      "step": 228900
    },
    {
      "activations/layer0_attention_weight_max": 16.9328670501709,
      "activations/layer0_attention_weight_min": -12.181449890136719,
      "activations/layer10_attention_weight_max": 33.012847900390625,
      "activations/layer10_attention_weight_min": -32.65178680419922,
      "activations/layer11_attention_weight_max": 34.14228057861328,
      "activations/layer11_attention_weight_min": -32.997314453125,
      "activations/layer12_attention_weight_max": 38.10276412963867,
      "activations/layer12_attention_weight_min": -26.412960052490234,
      "activations/layer13_attention_weight_max": 45.199459075927734,
      "activations/layer13_attention_weight_min": -34.422306060791016,
      "activations/layer14_attention_weight_max": 42.914451599121094,
      "activations/layer14_attention_weight_min": -39.33944320678711,
      "activations/layer15_attention_weight_max": 36.946632385253906,
      "activations/layer15_attention_weight_min": -34.011253356933594,
      "activations/layer16_attention_weight_max": 30.21170997619629,
      "activations/layer16_attention_weight_min": -29.477005004882812,
      "activations/layer17_attention_weight_max": 32.25188446044922,
      "activations/layer17_attention_weight_min": -29.52605628967285,
      "activations/layer18_attention_weight_max": 30.72457504272461,
      "activations/layer18_attention_weight_min": -25.854066848754883,
      "activations/layer19_attention_weight_max": 33.89633560180664,
      "activations/layer19_attention_weight_min": -34.29134750366211,
      "activations/layer1_attention_weight_max": 17.228384017944336,
      "activations/layer1_attention_weight_min": -13.820106506347656,
      "activations/layer20_attention_weight_max": 29.01286506652832,
      "activations/layer20_attention_weight_min": -26.2171688079834,
      "activations/layer21_attention_weight_max": 26.45475196838379,
      "activations/layer21_attention_weight_min": -25.36564064025879,
      "activations/layer22_attention_weight_max": 46.48439025878906,
      "activations/layer22_attention_weight_min": -32.98104476928711,
      "activations/layer23_attention_weight_max": 36.301856994628906,
      "activations/layer23_attention_weight_min": -25.24871063232422,
      "activations/layer2_attention_weight_max": 33.25226593017578,
      "activations/layer2_attention_weight_min": -32.6452522277832,
      "activations/layer3_attention_weight_max": 100.04967498779297,
      "activations/layer3_attention_weight_min": -100.18421936035156,
      "activations/layer4_attention_weight_max": 111.39257049560547,
      "activations/layer4_attention_weight_min": -115.78321838378906,
      "activations/layer5_attention_weight_max": 54.32528305053711,
      "activations/layer5_attention_weight_min": -60.80909729003906,
      "activations/layer6_attention_weight_max": 45.41105651855469,
      "activations/layer6_attention_weight_min": -47.72567367553711,
      "activations/layer7_attention_weight_max": 87.45926666259766,
      "activations/layer7_attention_weight_min": -93.25750732421875,
      "activations/layer8_attention_weight_max": 43.04387664794922,
      "activations/layer8_attention_weight_min": -41.00620651245117,
      "activations/layer9_attention_weight_max": 32.25056076049805,
      "activations/layer9_attention_weight_min": -32.797237396240234,
      "epoch": 13.3,
      "learning_rate": 6.486325757575758e-05,
      "loss": 2.7575,
      "step": 228950
    },
    {
      "activations/layer0_attention_weight_max": 14.771876335144043,
      "activations/layer0_attention_weight_min": -13.355233192443848,
      "activations/layer10_attention_weight_max": 33.704261779785156,
      "activations/layer10_attention_weight_min": -32.71857452392578,
      "activations/layer11_attention_weight_max": 33.871368408203125,
      "activations/layer11_attention_weight_min": -33.940589904785156,
      "activations/layer12_attention_weight_max": 28.13797950744629,
      "activations/layer12_attention_weight_min": -25.90818214416504,
      "activations/layer13_attention_weight_max": 39.178489685058594,
      "activations/layer13_attention_weight_min": -36.53364562988281,
      "activations/layer14_attention_weight_max": 40.87010192871094,
      "activations/layer14_attention_weight_min": -38.8852653503418,
      "activations/layer15_attention_weight_max": 35.748756408691406,
      "activations/layer15_attention_weight_min": -35.89140701293945,
      "activations/layer16_attention_weight_max": 30.84269142150879,
      "activations/layer16_attention_weight_min": -26.94617462158203,
      "activations/layer17_attention_weight_max": 31.393314361572266,
      "activations/layer17_attention_weight_min": -26.582361221313477,
      "activations/layer18_attention_weight_max": 32.58710479736328,
      "activations/layer18_attention_weight_min": -22.231674194335938,
      "activations/layer19_attention_weight_max": 36.65009689331055,
      "activations/layer19_attention_weight_min": -28.92911148071289,
      "activations/layer1_attention_weight_max": 16.98283576965332,
      "activations/layer1_attention_weight_min": -14.996609687805176,
      "activations/layer20_attention_weight_max": 30.089998245239258,
      "activations/layer20_attention_weight_min": -24.424253463745117,
      "activations/layer21_attention_weight_max": 29.340545654296875,
      "activations/layer21_attention_weight_min": -24.203048706054688,
      "activations/layer22_attention_weight_max": 43.052650451660156,
      "activations/layer22_attention_weight_min": -29.135759353637695,
      "activations/layer23_attention_weight_max": 33.6453971862793,
      "activations/layer23_attention_weight_min": -24.31246566772461,
      "activations/layer2_attention_weight_max": 32.401371002197266,
      "activations/layer2_attention_weight_min": -32.81301498413086,
      "activations/layer3_attention_weight_max": 98.45440673828125,
      "activations/layer3_attention_weight_min": -99.26412200927734,
      "activations/layer4_attention_weight_max": 112.77640533447266,
      "activations/layer4_attention_weight_min": -110.97071838378906,
      "activations/layer5_attention_weight_max": 52.73640441894531,
      "activations/layer5_attention_weight_min": -62.630306243896484,
      "activations/layer6_attention_weight_max": 44.320377349853516,
      "activations/layer6_attention_weight_min": -47.0922737121582,
      "activations/layer7_attention_weight_max": 92.51605987548828,
      "activations/layer7_attention_weight_min": -94.84474182128906,
      "activations/layer8_attention_weight_max": 41.26686096191406,
      "activations/layer8_attention_weight_min": -43.17497253417969,
      "activations/layer9_attention_weight_max": 35.97454833984375,
      "activations/layer9_attention_weight_min": -33.860137939453125,
      "epoch": 13.31,
      "learning_rate": 6.484431818181818e-05,
      "loss": 2.7837,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_loss": 2.71875,
      "eval_runtime": 8.456,
      "eval_samples_per_second": 507.803,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.456,
      "eval_openwebtext_samples_per_second": 507.803,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9447,
      "eval_wikitext_samples_per_second": 234.48,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.51,
      "eval_lambada_samples_per_second": 511.986,
      "step": 229000
    },
    {
      "activations/layer0_attention_weight_max": 16.737428665161133,
      "activations/layer0_attention_weight_min": -12.26444149017334,
      "activations/layer10_attention_weight_max": 32.9987678527832,
      "activations/layer10_attention_weight_min": -34.256866455078125,
      "activations/layer11_attention_weight_max": 33.35386657714844,
      "activations/layer11_attention_weight_min": -34.651771545410156,
      "activations/layer12_attention_weight_max": 26.62955093383789,
      "activations/layer12_attention_weight_min": -26.077672958374023,
      "activations/layer13_attention_weight_max": 41.07722854614258,
      "activations/layer13_attention_weight_min": -35.41503143310547,
      "activations/layer14_attention_weight_max": 43.97618865966797,
      "activations/layer14_attention_weight_min": -38.24814987182617,
      "activations/layer15_attention_weight_max": 39.15723419189453,
      "activations/layer15_attention_weight_min": -33.96574783325195,
      "activations/layer16_attention_weight_max": 33.17144775390625,
      "activations/layer16_attention_weight_min": -29.409215927124023,
      "activations/layer17_attention_weight_max": 32.94190216064453,
      "activations/layer17_attention_weight_min": -27.347835540771484,
      "activations/layer18_attention_weight_max": 32.18423080444336,
      "activations/layer18_attention_weight_min": -26.296417236328125,
      "activations/layer19_attention_weight_max": 33.77478790283203,
      "activations/layer19_attention_weight_min": -32.24741744995117,
      "activations/layer1_attention_weight_max": 18.459697723388672,
      "activations/layer1_attention_weight_min": -15.474325180053711,
      "activations/layer20_attention_weight_max": 30.40755844116211,
      "activations/layer20_attention_weight_min": -27.676063537597656,
      "activations/layer21_attention_weight_max": 31.411941528320312,
      "activations/layer21_attention_weight_min": -26.32626724243164,
      "activations/layer22_attention_weight_max": 48.04917526245117,
      "activations/layer22_attention_weight_min": -33.427852630615234,
      "activations/layer23_attention_weight_max": 34.977169036865234,
      "activations/layer23_attention_weight_min": -26.449260711669922,
      "activations/layer2_attention_weight_max": 34.383506774902344,
      "activations/layer2_attention_weight_min": -36.61445617675781,
      "activations/layer3_attention_weight_max": 104.3855972290039,
      "activations/layer3_attention_weight_min": -106.23334503173828,
      "activations/layer4_attention_weight_max": 113.51686096191406,
      "activations/layer4_attention_weight_min": -114.9601058959961,
      "activations/layer5_attention_weight_max": 54.82013702392578,
      "activations/layer5_attention_weight_min": -63.213706970214844,
      "activations/layer6_attention_weight_max": 43.08892059326172,
      "activations/layer6_attention_weight_min": -45.96985626220703,
      "activations/layer7_attention_weight_max": 90.22021484375,
      "activations/layer7_attention_weight_min": -88.87491607666016,
      "activations/layer8_attention_weight_max": 40.45476150512695,
      "activations/layer8_attention_weight_min": -41.512184143066406,
      "activations/layer9_attention_weight_max": 33.855690002441406,
      "activations/layer9_attention_weight_min": -32.50926208496094,
      "epoch": 13.31,
      "learning_rate": 6.482537878787878e-05,
      "loss": 2.7711,
      "step": 229050
    },
    {
      "activations/layer0_attention_weight_max": 15.534040451049805,
      "activations/layer0_attention_weight_min": -12.020837783813477,
      "activations/layer10_attention_weight_max": 32.107269287109375,
      "activations/layer10_attention_weight_min": -32.755523681640625,
      "activations/layer11_attention_weight_max": 32.129783630371094,
      "activations/layer11_attention_weight_min": -32.69902038574219,
      "activations/layer12_attention_weight_max": 24.09644889831543,
      "activations/layer12_attention_weight_min": -24.713193893432617,
      "activations/layer13_attention_weight_max": 37.76953125,
      "activations/layer13_attention_weight_min": -33.08849334716797,
      "activations/layer14_attention_weight_max": 41.003692626953125,
      "activations/layer14_attention_weight_min": -37.08494186401367,
      "activations/layer15_attention_weight_max": 38.19818115234375,
      "activations/layer15_attention_weight_min": -32.80917739868164,
      "activations/layer16_attention_weight_max": 30.646743774414062,
      "activations/layer16_attention_weight_min": -27.767759323120117,
      "activations/layer17_attention_weight_max": 32.540382385253906,
      "activations/layer17_attention_weight_min": -25.61638832092285,
      "activations/layer18_attention_weight_max": 31.176677703857422,
      "activations/layer18_attention_weight_min": -25.05617904663086,
      "activations/layer19_attention_weight_max": 35.935577392578125,
      "activations/layer19_attention_weight_min": -30.61529541015625,
      "activations/layer1_attention_weight_max": 16.738527297973633,
      "activations/layer1_attention_weight_min": -13.766511917114258,
      "activations/layer20_attention_weight_max": 31.007354736328125,
      "activations/layer20_attention_weight_min": -25.390775680541992,
      "activations/layer21_attention_weight_max": 30.300390243530273,
      "activations/layer21_attention_weight_min": -24.924131393432617,
      "activations/layer22_attention_weight_max": 48.55168533325195,
      "activations/layer22_attention_weight_min": -29.85582733154297,
      "activations/layer23_attention_weight_max": 36.205535888671875,
      "activations/layer23_attention_weight_min": -23.566104888916016,
      "activations/layer2_attention_weight_max": 32.73442077636719,
      "activations/layer2_attention_weight_min": -33.8042106628418,
      "activations/layer3_attention_weight_max": 95.89374542236328,
      "activations/layer3_attention_weight_min": -102.60855102539062,
      "activations/layer4_attention_weight_max": 112.587890625,
      "activations/layer4_attention_weight_min": -111.33113861083984,
      "activations/layer5_attention_weight_max": 54.291114807128906,
      "activations/layer5_attention_weight_min": -64.94141387939453,
      "activations/layer6_attention_weight_max": 44.291080474853516,
      "activations/layer6_attention_weight_min": -46.603328704833984,
      "activations/layer7_attention_weight_max": 92.6710205078125,
      "activations/layer7_attention_weight_min": -88.10159301757812,
      "activations/layer8_attention_weight_max": 39.91303634643555,
      "activations/layer8_attention_weight_min": -40.989593505859375,
      "activations/layer9_attention_weight_max": 33.836151123046875,
      "activations/layer9_attention_weight_min": -32.69243240356445,
      "epoch": 13.31,
      "learning_rate": 6.48064393939394e-05,
      "loss": 2.7677,
      "step": 229100
    },
    {
      "activations/layer0_attention_weight_max": 15.3597993850708,
      "activations/layer0_attention_weight_min": -12.929681777954102,
      "activations/layer10_attention_weight_max": 32.92512130737305,
      "activations/layer10_attention_weight_min": -31.83136558532715,
      "activations/layer11_attention_weight_max": 33.42045211791992,
      "activations/layer11_attention_weight_min": -31.80445098876953,
      "activations/layer12_attention_weight_max": 27.517086029052734,
      "activations/layer12_attention_weight_min": -24.975854873657227,
      "activations/layer13_attention_weight_max": 41.00929641723633,
      "activations/layer13_attention_weight_min": -35.80324935913086,
      "activations/layer14_attention_weight_max": 44.24850082397461,
      "activations/layer14_attention_weight_min": -36.1312141418457,
      "activations/layer15_attention_weight_max": 37.890316009521484,
      "activations/layer15_attention_weight_min": -35.5540657043457,
      "activations/layer16_attention_weight_max": 29.87041664123535,
      "activations/layer16_attention_weight_min": -28.685827255249023,
      "activations/layer17_attention_weight_max": 30.44854736328125,
      "activations/layer17_attention_weight_min": -26.518199920654297,
      "activations/layer18_attention_weight_max": 31.796432495117188,
      "activations/layer18_attention_weight_min": -24.503150939941406,
      "activations/layer19_attention_weight_max": 34.30129623413086,
      "activations/layer19_attention_weight_min": -33.96809005737305,
      "activations/layer1_attention_weight_max": 15.852561950683594,
      "activations/layer1_attention_weight_min": -15.525298118591309,
      "activations/layer20_attention_weight_max": 33.83526611328125,
      "activations/layer20_attention_weight_min": -27.97063636779785,
      "activations/layer21_attention_weight_max": 29.51019287109375,
      "activations/layer21_attention_weight_min": -27.171958923339844,
      "activations/layer22_attention_weight_max": 49.32740020751953,
      "activations/layer22_attention_weight_min": -33.47883605957031,
      "activations/layer23_attention_weight_max": 36.87638473510742,
      "activations/layer23_attention_weight_min": -28.26485252380371,
      "activations/layer2_attention_weight_max": 34.087074279785156,
      "activations/layer2_attention_weight_min": -34.898773193359375,
      "activations/layer3_attention_weight_max": 101.05230712890625,
      "activations/layer3_attention_weight_min": -102.52973175048828,
      "activations/layer4_attention_weight_max": 111.52156066894531,
      "activations/layer4_attention_weight_min": -113.94156646728516,
      "activations/layer5_attention_weight_max": 52.78839874267578,
      "activations/layer5_attention_weight_min": -60.22385787963867,
      "activations/layer6_attention_weight_max": 46.53103256225586,
      "activations/layer6_attention_weight_min": -48.75331497192383,
      "activations/layer7_attention_weight_max": 95.69803619384766,
      "activations/layer7_attention_weight_min": -93.94750213623047,
      "activations/layer8_attention_weight_max": 41.49581527709961,
      "activations/layer8_attention_weight_min": -42.42041015625,
      "activations/layer9_attention_weight_max": 33.75354766845703,
      "activations/layer9_attention_weight_min": -32.81504440307617,
      "epoch": 13.31,
      "learning_rate": 6.47875e-05,
      "loss": 2.7665,
      "step": 229150
    },
    {
      "activations/layer0_attention_weight_max": 16.005664825439453,
      "activations/layer0_attention_weight_min": -11.637080192565918,
      "activations/layer10_attention_weight_max": 48.683197021484375,
      "activations/layer10_attention_weight_min": -44.167266845703125,
      "activations/layer11_attention_weight_max": 51.049156188964844,
      "activations/layer11_attention_weight_min": -48.74050521850586,
      "activations/layer12_attention_weight_max": 50.20647430419922,
      "activations/layer12_attention_weight_min": -33.19645309448242,
      "activations/layer13_attention_weight_max": 73.7175521850586,
      "activations/layer13_attention_weight_min": -44.76078796386719,
      "activations/layer14_attention_weight_max": 67.50553131103516,
      "activations/layer14_attention_weight_min": -53.84345245361328,
      "activations/layer15_attention_weight_max": 64.79390716552734,
      "activations/layer15_attention_weight_min": -50.8085823059082,
      "activations/layer16_attention_weight_max": 42.83274841308594,
      "activations/layer16_attention_weight_min": -33.64649963378906,
      "activations/layer17_attention_weight_max": 39.797889709472656,
      "activations/layer17_attention_weight_min": -34.360958099365234,
      "activations/layer18_attention_weight_max": 31.170452117919922,
      "activations/layer18_attention_weight_min": -30.353256225585938,
      "activations/layer19_attention_weight_max": 37.93854904174805,
      "activations/layer19_attention_weight_min": -33.133392333984375,
      "activations/layer1_attention_weight_max": 16.733844757080078,
      "activations/layer1_attention_weight_min": -13.77793025970459,
      "activations/layer20_attention_weight_max": 30.177574157714844,
      "activations/layer20_attention_weight_min": -25.96495819091797,
      "activations/layer21_attention_weight_max": 28.441814422607422,
      "activations/layer21_attention_weight_min": -28.051156997680664,
      "activations/layer22_attention_weight_max": 41.65701675415039,
      "activations/layer22_attention_weight_min": -32.80039596557617,
      "activations/layer23_attention_weight_max": 33.7421875,
      "activations/layer23_attention_weight_min": -25.283885955810547,
      "activations/layer2_attention_weight_max": 33.23588943481445,
      "activations/layer2_attention_weight_min": -33.22355651855469,
      "activations/layer3_attention_weight_max": 95.34663391113281,
      "activations/layer3_attention_weight_min": -97.88734436035156,
      "activations/layer4_attention_weight_max": 115.82930755615234,
      "activations/layer4_attention_weight_min": -105.56378173828125,
      "activations/layer5_attention_weight_max": 58.585838317871094,
      "activations/layer5_attention_weight_min": -59.40809631347656,
      "activations/layer6_attention_weight_max": 47.733726501464844,
      "activations/layer6_attention_weight_min": -45.94716262817383,
      "activations/layer7_attention_weight_max": 115.84849548339844,
      "activations/layer7_attention_weight_min": -97.57665252685547,
      "activations/layer8_attention_weight_max": 54.81531524658203,
      "activations/layer8_attention_weight_min": -49.65327072143555,
      "activations/layer9_attention_weight_max": 44.657875061035156,
      "activations/layer9_attention_weight_min": -42.84996032714844,
      "epoch": 13.32,
      "learning_rate": 6.47685606060606e-05,
      "loss": 2.7429,
      "step": 229200
    },
    {
      "activations/layer0_attention_weight_max": 15.930383682250977,
      "activations/layer0_attention_weight_min": -12.131584167480469,
      "activations/layer10_attention_weight_max": 35.60243606567383,
      "activations/layer10_attention_weight_min": -35.606449127197266,
      "activations/layer11_attention_weight_max": 37.46650314331055,
      "activations/layer11_attention_weight_min": -35.95204544067383,
      "activations/layer12_attention_weight_max": 26.552629470825195,
      "activations/layer12_attention_weight_min": -26.415634155273438,
      "activations/layer13_attention_weight_max": 39.53605651855469,
      "activations/layer13_attention_weight_min": -35.39809799194336,
      "activations/layer14_attention_weight_max": 50.53009796142578,
      "activations/layer14_attention_weight_min": -41.57038116455078,
      "activations/layer15_attention_weight_max": 39.68800354003906,
      "activations/layer15_attention_weight_min": -34.525386810302734,
      "activations/layer16_attention_weight_max": 30.357357025146484,
      "activations/layer16_attention_weight_min": -27.468008041381836,
      "activations/layer17_attention_weight_max": 32.322635650634766,
      "activations/layer17_attention_weight_min": -25.583560943603516,
      "activations/layer18_attention_weight_max": 33.156436920166016,
      "activations/layer18_attention_weight_min": -23.998788833618164,
      "activations/layer19_attention_weight_max": 36.76072311401367,
      "activations/layer19_attention_weight_min": -33.16180419921875,
      "activations/layer1_attention_weight_max": 17.652006149291992,
      "activations/layer1_attention_weight_min": -16.049531936645508,
      "activations/layer20_attention_weight_max": 30.290687561035156,
      "activations/layer20_attention_weight_min": -26.28905487060547,
      "activations/layer21_attention_weight_max": 30.057249069213867,
      "activations/layer21_attention_weight_min": -26.441768646240234,
      "activations/layer22_attention_weight_max": 45.97060775756836,
      "activations/layer22_attention_weight_min": -31.760623931884766,
      "activations/layer23_attention_weight_max": 37.152496337890625,
      "activations/layer23_attention_weight_min": -24.879638671875,
      "activations/layer2_attention_weight_max": 33.69725799560547,
      "activations/layer2_attention_weight_min": -35.26373291015625,
      "activations/layer3_attention_weight_max": 105.25605773925781,
      "activations/layer3_attention_weight_min": -110.5244369506836,
      "activations/layer4_attention_weight_max": 112.7943115234375,
      "activations/layer4_attention_weight_min": -114.82647705078125,
      "activations/layer5_attention_weight_max": 54.45746612548828,
      "activations/layer5_attention_weight_min": -62.651397705078125,
      "activations/layer6_attention_weight_max": 47.03274917602539,
      "activations/layer6_attention_weight_min": -52.86454391479492,
      "activations/layer7_attention_weight_max": 99.73097229003906,
      "activations/layer7_attention_weight_min": -99.60801696777344,
      "activations/layer8_attention_weight_max": 42.73774719238281,
      "activations/layer8_attention_weight_min": -43.969825744628906,
      "activations/layer9_attention_weight_max": 35.331844329833984,
      "activations/layer9_attention_weight_min": -36.075408935546875,
      "epoch": 13.32,
      "learning_rate": 6.47496212121212e-05,
      "loss": 2.7708,
      "step": 229250
    },
    {
      "activations/layer0_attention_weight_max": 16.849924087524414,
      "activations/layer0_attention_weight_min": -11.586575508117676,
      "activations/layer10_attention_weight_max": 31.69301414489746,
      "activations/layer10_attention_weight_min": -32.12171173095703,
      "activations/layer11_attention_weight_max": 32.231502532958984,
      "activations/layer11_attention_weight_min": -31.706222534179688,
      "activations/layer12_attention_weight_max": 27.66750717163086,
      "activations/layer12_attention_weight_min": -27.180465698242188,
      "activations/layer13_attention_weight_max": 40.58998489379883,
      "activations/layer13_attention_weight_min": -36.18151092529297,
      "activations/layer14_attention_weight_max": 44.76613998413086,
      "activations/layer14_attention_weight_min": -37.995418548583984,
      "activations/layer15_attention_weight_max": 35.83267593383789,
      "activations/layer15_attention_weight_min": -33.45698165893555,
      "activations/layer16_attention_weight_max": 29.12200164794922,
      "activations/layer16_attention_weight_min": -27.268856048583984,
      "activations/layer17_attention_weight_max": 30.45273208618164,
      "activations/layer17_attention_weight_min": -25.33233070373535,
      "activations/layer18_attention_weight_max": 28.629009246826172,
      "activations/layer18_attention_weight_min": -24.07037925720215,
      "activations/layer19_attention_weight_max": 35.441951751708984,
      "activations/layer19_attention_weight_min": -29.718420028686523,
      "activations/layer1_attention_weight_max": 16.216896057128906,
      "activations/layer1_attention_weight_min": -14.988852500915527,
      "activations/layer20_attention_weight_max": 32.079830169677734,
      "activations/layer20_attention_weight_min": -25.338327407836914,
      "activations/layer21_attention_weight_max": 31.978633880615234,
      "activations/layer21_attention_weight_min": -25.74371337890625,
      "activations/layer22_attention_weight_max": 43.085105895996094,
      "activations/layer22_attention_weight_min": -29.82868194580078,
      "activations/layer23_attention_weight_max": 33.026771545410156,
      "activations/layer23_attention_weight_min": -23.94891357421875,
      "activations/layer2_attention_weight_max": 33.30621337890625,
      "activations/layer2_attention_weight_min": -32.99738311767578,
      "activations/layer3_attention_weight_max": 98.0187759399414,
      "activations/layer3_attention_weight_min": -99.0751953125,
      "activations/layer4_attention_weight_max": 108.63716888427734,
      "activations/layer4_attention_weight_min": -111.11153411865234,
      "activations/layer5_attention_weight_max": 49.553401947021484,
      "activations/layer5_attention_weight_min": -62.90188217163086,
      "activations/layer6_attention_weight_max": 44.77168655395508,
      "activations/layer6_attention_weight_min": -45.10553741455078,
      "activations/layer7_attention_weight_max": 99.24715423583984,
      "activations/layer7_attention_weight_min": -89.46465301513672,
      "activations/layer8_attention_weight_max": 40.2333869934082,
      "activations/layer8_attention_weight_min": -40.5026741027832,
      "activations/layer9_attention_weight_max": 32.03064727783203,
      "activations/layer9_attention_weight_min": -33.30852127075195,
      "epoch": 13.32,
      "learning_rate": 6.473068181818181e-05,
      "loss": 2.7867,
      "step": 229300
    },
    {
      "activations/layer0_attention_weight_max": 16.765827178955078,
      "activations/layer0_attention_weight_min": -11.907646179199219,
      "activations/layer10_attention_weight_max": 31.90256118774414,
      "activations/layer10_attention_weight_min": -32.623836517333984,
      "activations/layer11_attention_weight_max": 33.810638427734375,
      "activations/layer11_attention_weight_min": -33.46826171875,
      "activations/layer12_attention_weight_max": 30.248455047607422,
      "activations/layer12_attention_weight_min": -27.10361671447754,
      "activations/layer13_attention_weight_max": 38.35664749145508,
      "activations/layer13_attention_weight_min": -33.66252899169922,
      "activations/layer14_attention_weight_max": 44.12465286254883,
      "activations/layer14_attention_weight_min": -39.19226837158203,
      "activations/layer15_attention_weight_max": 39.8699836730957,
      "activations/layer15_attention_weight_min": -34.23713684082031,
      "activations/layer16_attention_weight_max": 30.268115997314453,
      "activations/layer16_attention_weight_min": -28.459484100341797,
      "activations/layer17_attention_weight_max": 30.153656005859375,
      "activations/layer17_attention_weight_min": -28.082094192504883,
      "activations/layer18_attention_weight_max": 33.193817138671875,
      "activations/layer18_attention_weight_min": -28.156328201293945,
      "activations/layer19_attention_weight_max": 31.983240127563477,
      "activations/layer19_attention_weight_min": -29.661691665649414,
      "activations/layer1_attention_weight_max": 16.21061134338379,
      "activations/layer1_attention_weight_min": -15.054756164550781,
      "activations/layer20_attention_weight_max": 29.68756103515625,
      "activations/layer20_attention_weight_min": -24.70355224609375,
      "activations/layer21_attention_weight_max": 32.330848693847656,
      "activations/layer21_attention_weight_min": -23.723562240600586,
      "activations/layer22_attention_weight_max": 43.1253547668457,
      "activations/layer22_attention_weight_min": -29.32868766784668,
      "activations/layer23_attention_weight_max": 35.724674224853516,
      "activations/layer23_attention_weight_min": -23.88623809814453,
      "activations/layer2_attention_weight_max": 35.761383056640625,
      "activations/layer2_attention_weight_min": -33.03525924682617,
      "activations/layer3_attention_weight_max": 101.69635772705078,
      "activations/layer3_attention_weight_min": -99.1980972290039,
      "activations/layer4_attention_weight_max": 112.72468566894531,
      "activations/layer4_attention_weight_min": -112.44283294677734,
      "activations/layer5_attention_weight_max": 55.98175048828125,
      "activations/layer5_attention_weight_min": -62.4712028503418,
      "activations/layer6_attention_weight_max": 43.68437194824219,
      "activations/layer6_attention_weight_min": -49.64866256713867,
      "activations/layer7_attention_weight_max": 91.35413360595703,
      "activations/layer7_attention_weight_min": -89.4425048828125,
      "activations/layer8_attention_weight_max": 40.09733963012695,
      "activations/layer8_attention_weight_min": -40.04820251464844,
      "activations/layer9_attention_weight_max": 31.437185287475586,
      "activations/layer9_attention_weight_min": -31.84683609008789,
      "epoch": 13.33,
      "learning_rate": 6.471174242424242e-05,
      "loss": 2.7633,
      "step": 229350
    },
    {
      "activations/layer0_attention_weight_max": 15.8897123336792,
      "activations/layer0_attention_weight_min": -11.482272148132324,
      "activations/layer10_attention_weight_max": 36.53586959838867,
      "activations/layer10_attention_weight_min": -36.20369338989258,
      "activations/layer11_attention_weight_max": 37.12735366821289,
      "activations/layer11_attention_weight_min": -37.12035369873047,
      "activations/layer12_attention_weight_max": 25.516508102416992,
      "activations/layer12_attention_weight_min": -25.708606719970703,
      "activations/layer13_attention_weight_max": 39.91619110107422,
      "activations/layer13_attention_weight_min": -34.44017791748047,
      "activations/layer14_attention_weight_max": 42.056800842285156,
      "activations/layer14_attention_weight_min": -38.54692459106445,
      "activations/layer15_attention_weight_max": 40.58027267456055,
      "activations/layer15_attention_weight_min": -34.24903106689453,
      "activations/layer16_attention_weight_max": 29.56912612915039,
      "activations/layer16_attention_weight_min": -26.62722396850586,
      "activations/layer17_attention_weight_max": 30.48298454284668,
      "activations/layer17_attention_weight_min": -24.559900283813477,
      "activations/layer18_attention_weight_max": 31.50995445251465,
      "activations/layer18_attention_weight_min": -24.691587448120117,
      "activations/layer19_attention_weight_max": 32.182838439941406,
      "activations/layer19_attention_weight_min": -30.663095474243164,
      "activations/layer1_attention_weight_max": 16.39116096496582,
      "activations/layer1_attention_weight_min": -14.593551635742188,
      "activations/layer20_attention_weight_max": 29.694766998291016,
      "activations/layer20_attention_weight_min": -25.10735321044922,
      "activations/layer21_attention_weight_max": 29.937686920166016,
      "activations/layer21_attention_weight_min": -24.148996353149414,
      "activations/layer22_attention_weight_max": 44.23543930053711,
      "activations/layer22_attention_weight_min": -30.00568389892578,
      "activations/layer23_attention_weight_max": 34.144535064697266,
      "activations/layer23_attention_weight_min": -23.468807220458984,
      "activations/layer2_attention_weight_max": 35.820194244384766,
      "activations/layer2_attention_weight_min": -35.49857711791992,
      "activations/layer3_attention_weight_max": 102.49262237548828,
      "activations/layer3_attention_weight_min": -108.07699584960938,
      "activations/layer4_attention_weight_max": 111.22295379638672,
      "activations/layer4_attention_weight_min": -113.73856353759766,
      "activations/layer5_attention_weight_max": 53.628570556640625,
      "activations/layer5_attention_weight_min": -63.83221435546875,
      "activations/layer6_attention_weight_max": 43.656558990478516,
      "activations/layer6_attention_weight_min": -48.26713180541992,
      "activations/layer7_attention_weight_max": 94.353271484375,
      "activations/layer7_attention_weight_min": -99.86345672607422,
      "activations/layer8_attention_weight_max": 40.872581481933594,
      "activations/layer8_attention_weight_min": -44.51400375366211,
      "activations/layer9_attention_weight_max": 35.81119918823242,
      "activations/layer9_attention_weight_min": -36.119991302490234,
      "epoch": 13.33,
      "learning_rate": 6.469280303030302e-05,
      "loss": 2.753,
      "step": 229400
    },
    {
      "activations/layer0_attention_weight_max": 16.48618507385254,
      "activations/layer0_attention_weight_min": -12.401445388793945,
      "activations/layer10_attention_weight_max": 34.64890670776367,
      "activations/layer10_attention_weight_min": -34.26348876953125,
      "activations/layer11_attention_weight_max": 34.00635528564453,
      "activations/layer11_attention_weight_min": -33.97343826293945,
      "activations/layer12_attention_weight_max": 27.182086944580078,
      "activations/layer12_attention_weight_min": -26.14900016784668,
      "activations/layer13_attention_weight_max": 40.076904296875,
      "activations/layer13_attention_weight_min": -35.599266052246094,
      "activations/layer14_attention_weight_max": 46.9091911315918,
      "activations/layer14_attention_weight_min": -38.67697525024414,
      "activations/layer15_attention_weight_max": 42.23954772949219,
      "activations/layer15_attention_weight_min": -32.730804443359375,
      "activations/layer16_attention_weight_max": 36.29093551635742,
      "activations/layer16_attention_weight_min": -29.49088478088379,
      "activations/layer17_attention_weight_max": 38.80109405517578,
      "activations/layer17_attention_weight_min": -27.459667205810547,
      "activations/layer18_attention_weight_max": 34.352108001708984,
      "activations/layer18_attention_weight_min": -24.09554672241211,
      "activations/layer19_attention_weight_max": 39.04609298706055,
      "activations/layer19_attention_weight_min": -34.018829345703125,
      "activations/layer1_attention_weight_max": 16.269487380981445,
      "activations/layer1_attention_weight_min": -14.910599708557129,
      "activations/layer20_attention_weight_max": 33.91350555419922,
      "activations/layer20_attention_weight_min": -26.020505905151367,
      "activations/layer21_attention_weight_max": 32.55507278442383,
      "activations/layer21_attention_weight_min": -25.914871215820312,
      "activations/layer22_attention_weight_max": 50.282737731933594,
      "activations/layer22_attention_weight_min": -31.51392364501953,
      "activations/layer23_attention_weight_max": 36.979061126708984,
      "activations/layer23_attention_weight_min": -25.72989273071289,
      "activations/layer2_attention_weight_max": 34.85320281982422,
      "activations/layer2_attention_weight_min": -35.64917755126953,
      "activations/layer3_attention_weight_max": 101.9711685180664,
      "activations/layer3_attention_weight_min": -102.29229736328125,
      "activations/layer4_attention_weight_max": 115.18513488769531,
      "activations/layer4_attention_weight_min": -113.76094055175781,
      "activations/layer5_attention_weight_max": 51.943382263183594,
      "activations/layer5_attention_weight_min": -63.783050537109375,
      "activations/layer6_attention_weight_max": 45.38663101196289,
      "activations/layer6_attention_weight_min": -51.01123809814453,
      "activations/layer7_attention_weight_max": 97.76943969726562,
      "activations/layer7_attention_weight_min": -96.16504669189453,
      "activations/layer8_attention_weight_max": 41.80046463012695,
      "activations/layer8_attention_weight_min": -46.18130874633789,
      "activations/layer9_attention_weight_max": 32.59686279296875,
      "activations/layer9_attention_weight_min": -34.75947189331055,
      "epoch": 13.33,
      "learning_rate": 6.467386363636363e-05,
      "loss": 2.7713,
      "step": 229450
    },
    {
      "activations/layer0_attention_weight_max": 16.350540161132812,
      "activations/layer0_attention_weight_min": -12.583772659301758,
      "activations/layer10_attention_weight_max": 33.365108489990234,
      "activations/layer10_attention_weight_min": -34.14867401123047,
      "activations/layer11_attention_weight_max": 33.78718566894531,
      "activations/layer11_attention_weight_min": -35.06838607788086,
      "activations/layer12_attention_weight_max": 26.35538101196289,
      "activations/layer12_attention_weight_min": -25.119226455688477,
      "activations/layer13_attention_weight_max": 39.52509689331055,
      "activations/layer13_attention_weight_min": -34.706485748291016,
      "activations/layer14_attention_weight_max": 45.02061462402344,
      "activations/layer14_attention_weight_min": -38.563629150390625,
      "activations/layer15_attention_weight_max": 36.57535171508789,
      "activations/layer15_attention_weight_min": -32.86066818237305,
      "activations/layer16_attention_weight_max": 30.85052490234375,
      "activations/layer16_attention_weight_min": -27.637529373168945,
      "activations/layer17_attention_weight_max": 32.893184661865234,
      "activations/layer17_attention_weight_min": -27.068817138671875,
      "activations/layer18_attention_weight_max": 31.939682006835938,
      "activations/layer18_attention_weight_min": -25.587472915649414,
      "activations/layer19_attention_weight_max": 32.79116439819336,
      "activations/layer19_attention_weight_min": -33.715579986572266,
      "activations/layer1_attention_weight_max": 16.4282169342041,
      "activations/layer1_attention_weight_min": -16.091644287109375,
      "activations/layer20_attention_weight_max": 29.6147403717041,
      "activations/layer20_attention_weight_min": -26.648120880126953,
      "activations/layer21_attention_weight_max": 29.209014892578125,
      "activations/layer21_attention_weight_min": -26.00847053527832,
      "activations/layer22_attention_weight_max": 46.332115173339844,
      "activations/layer22_attention_weight_min": -33.05742263793945,
      "activations/layer23_attention_weight_max": 33.85535430908203,
      "activations/layer23_attention_weight_min": -27.502355575561523,
      "activations/layer2_attention_weight_max": 32.96303939819336,
      "activations/layer2_attention_weight_min": -33.08405303955078,
      "activations/layer3_attention_weight_max": 96.48944091796875,
      "activations/layer3_attention_weight_min": -101.7119140625,
      "activations/layer4_attention_weight_max": 110.09625244140625,
      "activations/layer4_attention_weight_min": -111.42533111572266,
      "activations/layer5_attention_weight_max": 50.45490646362305,
      "activations/layer5_attention_weight_min": -61.87058639526367,
      "activations/layer6_attention_weight_max": 44.400997161865234,
      "activations/layer6_attention_weight_min": -48.139217376708984,
      "activations/layer7_attention_weight_max": 93.11116790771484,
      "activations/layer7_attention_weight_min": -99.65209197998047,
      "activations/layer8_attention_weight_max": 43.01640701293945,
      "activations/layer8_attention_weight_min": -42.787818908691406,
      "activations/layer9_attention_weight_max": 34.20587921142578,
      "activations/layer9_attention_weight_min": -32.86408615112305,
      "epoch": 13.34,
      "learning_rate": 6.465492424242423e-05,
      "loss": 2.7723,
      "step": 229500
    },
    {
      "activations/layer0_attention_weight_max": 15.235567092895508,
      "activations/layer0_attention_weight_min": -11.985737800598145,
      "activations/layer10_attention_weight_max": 31.56622314453125,
      "activations/layer10_attention_weight_min": -31.011648178100586,
      "activations/layer11_attention_weight_max": 31.158775329589844,
      "activations/layer11_attention_weight_min": -31.790273666381836,
      "activations/layer12_attention_weight_max": 24.780521392822266,
      "activations/layer12_attention_weight_min": -25.085107803344727,
      "activations/layer13_attention_weight_max": 38.88282775878906,
      "activations/layer13_attention_weight_min": -34.76202392578125,
      "activations/layer14_attention_weight_max": 43.29706954956055,
      "activations/layer14_attention_weight_min": -37.729862213134766,
      "activations/layer15_attention_weight_max": 37.01643753051758,
      "activations/layer15_attention_weight_min": -34.16671371459961,
      "activations/layer16_attention_weight_max": 29.657182693481445,
      "activations/layer16_attention_weight_min": -27.472978591918945,
      "activations/layer17_attention_weight_max": 31.378339767456055,
      "activations/layer17_attention_weight_min": -26.114267349243164,
      "activations/layer18_attention_weight_max": 30.46957015991211,
      "activations/layer18_attention_weight_min": -24.241376876831055,
      "activations/layer19_attention_weight_max": 30.792150497436523,
      "activations/layer19_attention_weight_min": -29.49755859375,
      "activations/layer1_attention_weight_max": 16.714752197265625,
      "activations/layer1_attention_weight_min": -13.77569580078125,
      "activations/layer20_attention_weight_max": 29.966625213623047,
      "activations/layer20_attention_weight_min": -25.572484970092773,
      "activations/layer21_attention_weight_max": 29.01759910583496,
      "activations/layer21_attention_weight_min": -26.165714263916016,
      "activations/layer22_attention_weight_max": 40.88562774658203,
      "activations/layer22_attention_weight_min": -30.937118530273438,
      "activations/layer23_attention_weight_max": 32.58642578125,
      "activations/layer23_attention_weight_min": -27.116634368896484,
      "activations/layer2_attention_weight_max": 32.81791305541992,
      "activations/layer2_attention_weight_min": -34.63663101196289,
      "activations/layer3_attention_weight_max": 92.42339324951172,
      "activations/layer3_attention_weight_min": -96.66802215576172,
      "activations/layer4_attention_weight_max": 105.60042572021484,
      "activations/layer4_attention_weight_min": -110.45796966552734,
      "activations/layer5_attention_weight_max": 53.06075668334961,
      "activations/layer5_attention_weight_min": -65.64083862304688,
      "activations/layer6_attention_weight_max": 41.801300048828125,
      "activations/layer6_attention_weight_min": -44.341800689697266,
      "activations/layer7_attention_weight_max": 84.48806762695312,
      "activations/layer7_attention_weight_min": -85.43413543701172,
      "activations/layer8_attention_weight_max": 37.15699768066406,
      "activations/layer8_attention_weight_min": -38.22087478637695,
      "activations/layer9_attention_weight_max": 29.877490997314453,
      "activations/layer9_attention_weight_min": -30.867816925048828,
      "epoch": 13.34,
      "learning_rate": 6.463598484848485e-05,
      "loss": 2.7526,
      "step": 229550
    },
    {
      "activations/layer0_attention_weight_max": 16.362350463867188,
      "activations/layer0_attention_weight_min": -11.71133804321289,
      "activations/layer10_attention_weight_max": 31.676401138305664,
      "activations/layer10_attention_weight_min": -32.4017219543457,
      "activations/layer11_attention_weight_max": 30.929431915283203,
      "activations/layer11_attention_weight_min": -32.34125900268555,
      "activations/layer12_attention_weight_max": 25.762113571166992,
      "activations/layer12_attention_weight_min": -24.818283081054688,
      "activations/layer13_attention_weight_max": 40.19599914550781,
      "activations/layer13_attention_weight_min": -33.26233673095703,
      "activations/layer14_attention_weight_max": 42.71109390258789,
      "activations/layer14_attention_weight_min": -37.70560836791992,
      "activations/layer15_attention_weight_max": 39.247623443603516,
      "activations/layer15_attention_weight_min": -32.70829391479492,
      "activations/layer16_attention_weight_max": 32.388877868652344,
      "activations/layer16_attention_weight_min": -27.877805709838867,
      "activations/layer17_attention_weight_max": 31.81167221069336,
      "activations/layer17_attention_weight_min": -26.538999557495117,
      "activations/layer18_attention_weight_max": 31.21102523803711,
      "activations/layer18_attention_weight_min": -25.055513381958008,
      "activations/layer19_attention_weight_max": 36.55213165283203,
      "activations/layer19_attention_weight_min": -30.7320613861084,
      "activations/layer1_attention_weight_max": 15.88698673248291,
      "activations/layer1_attention_weight_min": -14.217490196228027,
      "activations/layer20_attention_weight_max": 31.483720779418945,
      "activations/layer20_attention_weight_min": -25.12099266052246,
      "activations/layer21_attention_weight_max": 30.130468368530273,
      "activations/layer21_attention_weight_min": -24.679418563842773,
      "activations/layer22_attention_weight_max": 48.8771858215332,
      "activations/layer22_attention_weight_min": -30.611408233642578,
      "activations/layer23_attention_weight_max": 35.09973907470703,
      "activations/layer23_attention_weight_min": -24.957290649414062,
      "activations/layer2_attention_weight_max": 33.639793395996094,
      "activations/layer2_attention_weight_min": -34.27347183227539,
      "activations/layer3_attention_weight_max": 99.64473724365234,
      "activations/layer3_attention_weight_min": -100.84842681884766,
      "activations/layer4_attention_weight_max": 108.48436737060547,
      "activations/layer4_attention_weight_min": -110.6537094116211,
      "activations/layer5_attention_weight_max": 53.980125427246094,
      "activations/layer5_attention_weight_min": -60.24135971069336,
      "activations/layer6_attention_weight_max": 43.477394104003906,
      "activations/layer6_attention_weight_min": -46.993839263916016,
      "activations/layer7_attention_weight_max": 91.98126220703125,
      "activations/layer7_attention_weight_min": -91.31205749511719,
      "activations/layer8_attention_weight_max": 39.166744232177734,
      "activations/layer8_attention_weight_min": -42.15061569213867,
      "activations/layer9_attention_weight_max": 33.525630950927734,
      "activations/layer9_attention_weight_min": -32.18973922729492,
      "epoch": 13.34,
      "learning_rate": 6.461704545454545e-05,
      "loss": 2.7602,
      "step": 229600
    },
    {
      "activations/layer0_attention_weight_max": 16.455455780029297,
      "activations/layer0_attention_weight_min": -13.041913986206055,
      "activations/layer10_attention_weight_max": 35.000431060791016,
      "activations/layer10_attention_weight_min": -32.36833572387695,
      "activations/layer11_attention_weight_max": 35.51005554199219,
      "activations/layer11_attention_weight_min": -33.325008392333984,
      "activations/layer12_attention_weight_max": 26.484655380249023,
      "activations/layer12_attention_weight_min": -25.098445892333984,
      "activations/layer13_attention_weight_max": 41.05995178222656,
      "activations/layer13_attention_weight_min": -33.8098030090332,
      "activations/layer14_attention_weight_max": 45.85191345214844,
      "activations/layer14_attention_weight_min": -36.99504089355469,
      "activations/layer15_attention_weight_max": 37.999847412109375,
      "activations/layer15_attention_weight_min": -34.57746505737305,
      "activations/layer16_attention_weight_max": 30.099714279174805,
      "activations/layer16_attention_weight_min": -27.067617416381836,
      "activations/layer17_attention_weight_max": 31.547292709350586,
      "activations/layer17_attention_weight_min": -25.004806518554688,
      "activations/layer18_attention_weight_max": 31.365610122680664,
      "activations/layer18_attention_weight_min": -24.656570434570312,
      "activations/layer19_attention_weight_max": 34.01845932006836,
      "activations/layer19_attention_weight_min": -30.23860740661621,
      "activations/layer1_attention_weight_max": 17.57640266418457,
      "activations/layer1_attention_weight_min": -15.408276557922363,
      "activations/layer20_attention_weight_max": 28.4763126373291,
      "activations/layer20_attention_weight_min": -24.148786544799805,
      "activations/layer21_attention_weight_max": 31.64133644104004,
      "activations/layer21_attention_weight_min": -23.279563903808594,
      "activations/layer22_attention_weight_max": 37.851436614990234,
      "activations/layer22_attention_weight_min": -30.819013595581055,
      "activations/layer23_attention_weight_max": 33.150634765625,
      "activations/layer23_attention_weight_min": -24.430028915405273,
      "activations/layer2_attention_weight_max": 35.39949417114258,
      "activations/layer2_attention_weight_min": -33.44361114501953,
      "activations/layer3_attention_weight_max": 98.1178970336914,
      "activations/layer3_attention_weight_min": -101.94906616210938,
      "activations/layer4_attention_weight_max": 111.53548431396484,
      "activations/layer4_attention_weight_min": -112.78887176513672,
      "activations/layer5_attention_weight_max": 53.737030029296875,
      "activations/layer5_attention_weight_min": -65.5653305053711,
      "activations/layer6_attention_weight_max": 43.82685089111328,
      "activations/layer6_attention_weight_min": -46.29801559448242,
      "activations/layer7_attention_weight_max": 96.15583801269531,
      "activations/layer7_attention_weight_min": -89.9736328125,
      "activations/layer8_attention_weight_max": 44.48439407348633,
      "activations/layer8_attention_weight_min": -40.60267639160156,
      "activations/layer9_attention_weight_max": 32.20792770385742,
      "activations/layer9_attention_weight_min": -34.022193908691406,
      "epoch": 13.34,
      "learning_rate": 6.459810606060607e-05,
      "loss": 2.7355,
      "step": 229650
    },
    {
      "activations/layer0_attention_weight_max": 16.300397872924805,
      "activations/layer0_attention_weight_min": -12.276863098144531,
      "activations/layer10_attention_weight_max": 32.79090881347656,
      "activations/layer10_attention_weight_min": -31.13328742980957,
      "activations/layer11_attention_weight_max": 33.390289306640625,
      "activations/layer11_attention_weight_min": -30.53439712524414,
      "activations/layer12_attention_weight_max": 25.623926162719727,
      "activations/layer12_attention_weight_min": -24.558759689331055,
      "activations/layer13_attention_weight_max": 36.34318161010742,
      "activations/layer13_attention_weight_min": -33.1608772277832,
      "activations/layer14_attention_weight_max": 38.75141906738281,
      "activations/layer14_attention_weight_min": -36.440521240234375,
      "activations/layer15_attention_weight_max": 34.00837326049805,
      "activations/layer15_attention_weight_min": -32.810020446777344,
      "activations/layer16_attention_weight_max": 31.31598663330078,
      "activations/layer16_attention_weight_min": -29.901071548461914,
      "activations/layer17_attention_weight_max": 28.60381507873535,
      "activations/layer17_attention_weight_min": -24.897167205810547,
      "activations/layer18_attention_weight_max": 29.209318161010742,
      "activations/layer18_attention_weight_min": -22.884685516357422,
      "activations/layer19_attention_weight_max": 32.441654205322266,
      "activations/layer19_attention_weight_min": -30.95269775390625,
      "activations/layer1_attention_weight_max": 16.5361385345459,
      "activations/layer1_attention_weight_min": -13.939390182495117,
      "activations/layer20_attention_weight_max": 28.376148223876953,
      "activations/layer20_attention_weight_min": -25.630468368530273,
      "activations/layer21_attention_weight_max": 26.912839889526367,
      "activations/layer21_attention_weight_min": -24.291067123413086,
      "activations/layer22_attention_weight_max": 45.514957427978516,
      "activations/layer22_attention_weight_min": -28.255939483642578,
      "activations/layer23_attention_weight_max": 31.933469772338867,
      "activations/layer23_attention_weight_min": -23.922080993652344,
      "activations/layer2_attention_weight_max": 33.63256072998047,
      "activations/layer2_attention_weight_min": -34.19760513305664,
      "activations/layer3_attention_weight_max": 100.3338394165039,
      "activations/layer3_attention_weight_min": -100.28784942626953,
      "activations/layer4_attention_weight_max": 112.79766845703125,
      "activations/layer4_attention_weight_min": -111.9844970703125,
      "activations/layer5_attention_weight_max": 52.27192687988281,
      "activations/layer5_attention_weight_min": -61.36820983886719,
      "activations/layer6_attention_weight_max": 42.399757385253906,
      "activations/layer6_attention_weight_min": -46.69660568237305,
      "activations/layer7_attention_weight_max": 88.25835418701172,
      "activations/layer7_attention_weight_min": -87.86214447021484,
      "activations/layer8_attention_weight_max": 39.20271682739258,
      "activations/layer8_attention_weight_min": -42.550682067871094,
      "activations/layer9_attention_weight_max": 32.06686782836914,
      "activations/layer9_attention_weight_min": -31.61050796508789,
      "epoch": 13.35,
      "learning_rate": 6.457916666666667e-05,
      "loss": 2.7811,
      "step": 229700
    },
    {
      "activations/layer0_attention_weight_max": 15.625450134277344,
      "activations/layer0_attention_weight_min": -12.566703796386719,
      "activations/layer10_attention_weight_max": 34.76215362548828,
      "activations/layer10_attention_weight_min": -34.20740509033203,
      "activations/layer11_attention_weight_max": 34.45207214355469,
      "activations/layer11_attention_weight_min": -32.362098693847656,
      "activations/layer12_attention_weight_max": 24.0837459564209,
      "activations/layer12_attention_weight_min": -25.5151424407959,
      "activations/layer13_attention_weight_max": 37.94396209716797,
      "activations/layer13_attention_weight_min": -37.29756164550781,
      "activations/layer14_attention_weight_max": 42.7507438659668,
      "activations/layer14_attention_weight_min": -40.076900482177734,
      "activations/layer15_attention_weight_max": 35.46205139160156,
      "activations/layer15_attention_weight_min": -34.81871795654297,
      "activations/layer16_attention_weight_max": 30.346830368041992,
      "activations/layer16_attention_weight_min": -27.32819938659668,
      "activations/layer17_attention_weight_max": 30.14940071105957,
      "activations/layer17_attention_weight_min": -25.26959800720215,
      "activations/layer18_attention_weight_max": 30.850900650024414,
      "activations/layer18_attention_weight_min": -24.325040817260742,
      "activations/layer19_attention_weight_max": 33.76960754394531,
      "activations/layer19_attention_weight_min": -34.45586013793945,
      "activations/layer1_attention_weight_max": 18.24427032470703,
      "activations/layer1_attention_weight_min": -16.71210479736328,
      "activations/layer20_attention_weight_max": 30.562381744384766,
      "activations/layer20_attention_weight_min": -28.4952392578125,
      "activations/layer21_attention_weight_max": 30.493667602539062,
      "activations/layer21_attention_weight_min": -25.365806579589844,
      "activations/layer22_attention_weight_max": 43.39848709106445,
      "activations/layer22_attention_weight_min": -32.102989196777344,
      "activations/layer23_attention_weight_max": 36.037559509277344,
      "activations/layer23_attention_weight_min": -26.44762420654297,
      "activations/layer2_attention_weight_max": 34.37274169921875,
      "activations/layer2_attention_weight_min": -33.79692077636719,
      "activations/layer3_attention_weight_max": 98.71488952636719,
      "activations/layer3_attention_weight_min": -102.48839569091797,
      "activations/layer4_attention_weight_max": 109.6246337890625,
      "activations/layer4_attention_weight_min": -112.62081146240234,
      "activations/layer5_attention_weight_max": 53.39948272705078,
      "activations/layer5_attention_weight_min": -62.06267547607422,
      "activations/layer6_attention_weight_max": 44.142940521240234,
      "activations/layer6_attention_weight_min": -46.24415969848633,
      "activations/layer7_attention_weight_max": 86.71319580078125,
      "activations/layer7_attention_weight_min": -96.24663543701172,
      "activations/layer8_attention_weight_max": 42.518218994140625,
      "activations/layer8_attention_weight_min": -43.94165802001953,
      "activations/layer9_attention_weight_max": 35.70127868652344,
      "activations/layer9_attention_weight_min": -34.92729187011719,
      "epoch": 13.35,
      "learning_rate": 6.456060606060606e-05,
      "loss": 2.761,
      "step": 229750
    },
    {
      "activations/layer0_attention_weight_max": 16.26753807067871,
      "activations/layer0_attention_weight_min": -12.434450149536133,
      "activations/layer10_attention_weight_max": 32.2637939453125,
      "activations/layer10_attention_weight_min": -31.70941162109375,
      "activations/layer11_attention_weight_max": 31.0006103515625,
      "activations/layer11_attention_weight_min": -32.632049560546875,
      "activations/layer12_attention_weight_max": 24.856393814086914,
      "activations/layer12_attention_weight_min": -25.620540618896484,
      "activations/layer13_attention_weight_max": 40.404693603515625,
      "activations/layer13_attention_weight_min": -34.24142074584961,
      "activations/layer14_attention_weight_max": 43.74417495727539,
      "activations/layer14_attention_weight_min": -35.22711944580078,
      "activations/layer15_attention_weight_max": 38.55446243286133,
      "activations/layer15_attention_weight_min": -33.42023468017578,
      "activations/layer16_attention_weight_max": 35.48637008666992,
      "activations/layer16_attention_weight_min": -28.07231330871582,
      "activations/layer17_attention_weight_max": 31.24941635131836,
      "activations/layer17_attention_weight_min": -28.301034927368164,
      "activations/layer18_attention_weight_max": 36.49812698364258,
      "activations/layer18_attention_weight_min": -23.480026245117188,
      "activations/layer19_attention_weight_max": 39.19358444213867,
      "activations/layer19_attention_weight_min": -30.059329986572266,
      "activations/layer1_attention_weight_max": 16.478384017944336,
      "activations/layer1_attention_weight_min": -15.409337997436523,
      "activations/layer20_attention_weight_max": 33.23157501220703,
      "activations/layer20_attention_weight_min": -25.157630920410156,
      "activations/layer21_attention_weight_max": 36.676998138427734,
      "activations/layer21_attention_weight_min": -24.099498748779297,
      "activations/layer22_attention_weight_max": 44.588623046875,
      "activations/layer22_attention_weight_min": -30.04170036315918,
      "activations/layer23_attention_weight_max": 36.85276794433594,
      "activations/layer23_attention_weight_min": -25.827484130859375,
      "activations/layer2_attention_weight_max": 31.58540916442871,
      "activations/layer2_attention_weight_min": -33.73527526855469,
      "activations/layer3_attention_weight_max": 97.30167388916016,
      "activations/layer3_attention_weight_min": -102.36244201660156,
      "activations/layer4_attention_weight_max": 110.34461975097656,
      "activations/layer4_attention_weight_min": -111.26204681396484,
      "activations/layer5_attention_weight_max": 53.57781219482422,
      "activations/layer5_attention_weight_min": -63.586029052734375,
      "activations/layer6_attention_weight_max": 40.863494873046875,
      "activations/layer6_attention_weight_min": -45.89564514160156,
      "activations/layer7_attention_weight_max": 87.76367950439453,
      "activations/layer7_attention_weight_min": -92.44470977783203,
      "activations/layer8_attention_weight_max": 38.755287170410156,
      "activations/layer8_attention_weight_min": -43.61686325073242,
      "activations/layer9_attention_weight_max": 31.756057739257812,
      "activations/layer9_attention_weight_min": -32.901729583740234,
      "epoch": 13.35,
      "learning_rate": 6.454166666666666e-05,
      "loss": 2.7658,
      "step": 229800
    },
    {
      "activations/layer0_attention_weight_max": 16.085325241088867,
      "activations/layer0_attention_weight_min": -12.165804862976074,
      "activations/layer10_attention_weight_max": 35.285221099853516,
      "activations/layer10_attention_weight_min": -37.0350227355957,
      "activations/layer11_attention_weight_max": 34.604393005371094,
      "activations/layer11_attention_weight_min": -37.921714782714844,
      "activations/layer12_attention_weight_max": 27.68889617919922,
      "activations/layer12_attention_weight_min": -27.098451614379883,
      "activations/layer13_attention_weight_max": 40.42366409301758,
      "activations/layer13_attention_weight_min": -36.88423156738281,
      "activations/layer14_attention_weight_max": 43.584434509277344,
      "activations/layer14_attention_weight_min": -40.76655578613281,
      "activations/layer15_attention_weight_max": 37.94633483886719,
      "activations/layer15_attention_weight_min": -35.74443054199219,
      "activations/layer16_attention_weight_max": 29.836225509643555,
      "activations/layer16_attention_weight_min": -28.54340934753418,
      "activations/layer17_attention_weight_max": 32.02157974243164,
      "activations/layer17_attention_weight_min": -27.202362060546875,
      "activations/layer18_attention_weight_max": 31.379253387451172,
      "activations/layer18_attention_weight_min": -24.629735946655273,
      "activations/layer19_attention_weight_max": 35.119564056396484,
      "activations/layer19_attention_weight_min": -31.4538631439209,
      "activations/layer1_attention_weight_max": 16.314289093017578,
      "activations/layer1_attention_weight_min": -14.2025728225708,
      "activations/layer20_attention_weight_max": 32.45713424682617,
      "activations/layer20_attention_weight_min": -26.2667179107666,
      "activations/layer21_attention_weight_max": 31.413103103637695,
      "activations/layer21_attention_weight_min": -27.027400970458984,
      "activations/layer22_attention_weight_max": 43.32741928100586,
      "activations/layer22_attention_weight_min": -30.866207122802734,
      "activations/layer23_attention_weight_max": 35.046531677246094,
      "activations/layer23_attention_weight_min": -25.130849838256836,
      "activations/layer2_attention_weight_max": 32.00397491455078,
      "activations/layer2_attention_weight_min": -33.75080108642578,
      "activations/layer3_attention_weight_max": 96.75975799560547,
      "activations/layer3_attention_weight_min": -99.37982177734375,
      "activations/layer4_attention_weight_max": 113.2251968383789,
      "activations/layer4_attention_weight_min": -117.6771240234375,
      "activations/layer5_attention_weight_max": 53.65572738647461,
      "activations/layer5_attention_weight_min": -62.897666931152344,
      "activations/layer6_attention_weight_max": 46.75429916381836,
      "activations/layer6_attention_weight_min": -52.046226501464844,
      "activations/layer7_attention_weight_max": 100.0262680053711,
      "activations/layer7_attention_weight_min": -105.39764404296875,
      "activations/layer8_attention_weight_max": 44.5338020324707,
      "activations/layer8_attention_weight_min": -47.26405334472656,
      "activations/layer9_attention_weight_max": 35.783992767333984,
      "activations/layer9_attention_weight_min": -37.040069580078125,
      "epoch": 13.36,
      "learning_rate": 6.452272727272727e-05,
      "loss": 2.7752,
      "step": 229850
    },
    {
      "activations/layer0_attention_weight_max": 16.208881378173828,
      "activations/layer0_attention_weight_min": -11.782000541687012,
      "activations/layer10_attention_weight_max": 38.002349853515625,
      "activations/layer10_attention_weight_min": -32.5645751953125,
      "activations/layer11_attention_weight_max": 38.01953125,
      "activations/layer11_attention_weight_min": -33.7698860168457,
      "activations/layer12_attention_weight_max": 40.386016845703125,
      "activations/layer12_attention_weight_min": -25.857898712158203,
      "activations/layer13_attention_weight_max": 51.273616790771484,
      "activations/layer13_attention_weight_min": -33.569393157958984,
      "activations/layer14_attention_weight_max": 50.430789947509766,
      "activations/layer14_attention_weight_min": -41.76773452758789,
      "activations/layer15_attention_weight_max": 46.63304138183594,
      "activations/layer15_attention_weight_min": -39.310325622558594,
      "activations/layer16_attention_weight_max": 30.986438751220703,
      "activations/layer16_attention_weight_min": -28.709877014160156,
      "activations/layer17_attention_weight_max": 34.63631057739258,
      "activations/layer17_attention_weight_min": -27.25242805480957,
      "activations/layer18_attention_weight_max": 33.40467834472656,
      "activations/layer18_attention_weight_min": -24.40369987487793,
      "activations/layer19_attention_weight_max": 35.47084045410156,
      "activations/layer19_attention_weight_min": -32.874568939208984,
      "activations/layer1_attention_weight_max": 18.279428482055664,
      "activations/layer1_attention_weight_min": -15.933116912841797,
      "activations/layer20_attention_weight_max": 29.069812774658203,
      "activations/layer20_attention_weight_min": -26.465946197509766,
      "activations/layer21_attention_weight_max": 29.84166717529297,
      "activations/layer21_attention_weight_min": -25.008481979370117,
      "activations/layer22_attention_weight_max": 47.71561813354492,
      "activations/layer22_attention_weight_min": -32.01116943359375,
      "activations/layer23_attention_weight_max": 36.1536865234375,
      "activations/layer23_attention_weight_min": -27.515554428100586,
      "activations/layer2_attention_weight_max": 34.43975830078125,
      "activations/layer2_attention_weight_min": -33.54975128173828,
      "activations/layer3_attention_weight_max": 100.97344970703125,
      "activations/layer3_attention_weight_min": -98.42597198486328,
      "activations/layer4_attention_weight_max": 110.86617279052734,
      "activations/layer4_attention_weight_min": -109.59871673583984,
      "activations/layer5_attention_weight_max": 49.8431396484375,
      "activations/layer5_attention_weight_min": -60.77030944824219,
      "activations/layer6_attention_weight_max": 42.81266784667969,
      "activations/layer6_attention_weight_min": -46.33693313598633,
      "activations/layer7_attention_weight_max": 98.89978790283203,
      "activations/layer7_attention_weight_min": -95.4225082397461,
      "activations/layer8_attention_weight_max": 45.73115539550781,
      "activations/layer8_attention_weight_min": -42.91753005981445,
      "activations/layer9_attention_weight_max": 37.315982818603516,
      "activations/layer9_attention_weight_min": -34.5190544128418,
      "epoch": 13.36,
      "learning_rate": 6.450378787878788e-05,
      "loss": 2.775,
      "step": 229900
    },
    {
      "activations/layer0_attention_weight_max": 15.68464469909668,
      "activations/layer0_attention_weight_min": -12.778523445129395,
      "activations/layer10_attention_weight_max": 29.880239486694336,
      "activations/layer10_attention_weight_min": -33.0,
      "activations/layer11_attention_weight_max": 29.621299743652344,
      "activations/layer11_attention_weight_min": -30.563888549804688,
      "activations/layer12_attention_weight_max": 23.971946716308594,
      "activations/layer12_attention_weight_min": -24.580726623535156,
      "activations/layer13_attention_weight_max": 38.124847412109375,
      "activations/layer13_attention_weight_min": -34.368125915527344,
      "activations/layer14_attention_weight_max": 44.529510498046875,
      "activations/layer14_attention_weight_min": -39.416893005371094,
      "activations/layer15_attention_weight_max": 36.23661804199219,
      "activations/layer15_attention_weight_min": -33.421661376953125,
      "activations/layer16_attention_weight_max": 28.483123779296875,
      "activations/layer16_attention_weight_min": -27.99813461303711,
      "activations/layer17_attention_weight_max": 30.17978858947754,
      "activations/layer17_attention_weight_min": -26.17642593383789,
      "activations/layer18_attention_weight_max": 31.907833099365234,
      "activations/layer18_attention_weight_min": -25.3690242767334,
      "activations/layer19_attention_weight_max": 33.10139465332031,
      "activations/layer19_attention_weight_min": -35.34769058227539,
      "activations/layer1_attention_weight_max": 16.116558074951172,
      "activations/layer1_attention_weight_min": -14.737486839294434,
      "activations/layer20_attention_weight_max": 31.580522537231445,
      "activations/layer20_attention_weight_min": -28.036548614501953,
      "activations/layer21_attention_weight_max": 28.30541229248047,
      "activations/layer21_attention_weight_min": -25.714401245117188,
      "activations/layer22_attention_weight_max": 39.66563415527344,
      "activations/layer22_attention_weight_min": -33.410301208496094,
      "activations/layer23_attention_weight_max": 31.0008602142334,
      "activations/layer23_attention_weight_min": -26.26091766357422,
      "activations/layer2_attention_weight_max": 32.46179962158203,
      "activations/layer2_attention_weight_min": -33.385765075683594,
      "activations/layer3_attention_weight_max": 94.82022094726562,
      "activations/layer3_attention_weight_min": -101.91177368164062,
      "activations/layer4_attention_weight_max": 106.83919525146484,
      "activations/layer4_attention_weight_min": -107.94384765625,
      "activations/layer5_attention_weight_max": 51.071685791015625,
      "activations/layer5_attention_weight_min": -60.31153869628906,
      "activations/layer6_attention_weight_max": 42.68580627441406,
      "activations/layer6_attention_weight_min": -45.38228988647461,
      "activations/layer7_attention_weight_max": 85.41470336914062,
      "activations/layer7_attention_weight_min": -85.64546203613281,
      "activations/layer8_attention_weight_max": 37.20895004272461,
      "activations/layer8_attention_weight_min": -41.543209075927734,
      "activations/layer9_attention_weight_max": 31.837366104125977,
      "activations/layer9_attention_weight_min": -32.448787689208984,
      "epoch": 13.36,
      "learning_rate": 6.448484848484848e-05,
      "loss": 2.7604,
      "step": 229950
    },
    {
      "activations/layer0_attention_weight_max": 15.863443374633789,
      "activations/layer0_attention_weight_min": -11.947067260742188,
      "activations/layer10_attention_weight_max": 31.175817489624023,
      "activations/layer10_attention_weight_min": -31.706090927124023,
      "activations/layer11_attention_weight_max": 31.874753952026367,
      "activations/layer11_attention_weight_min": -31.393688201904297,
      "activations/layer12_attention_weight_max": 25.323759078979492,
      "activations/layer12_attention_weight_min": -26.493412017822266,
      "activations/layer13_attention_weight_max": 38.82670211791992,
      "activations/layer13_attention_weight_min": -34.18171310424805,
      "activations/layer14_attention_weight_max": 44.74232864379883,
      "activations/layer14_attention_weight_min": -37.12331771850586,
      "activations/layer15_attention_weight_max": 38.904014587402344,
      "activations/layer15_attention_weight_min": -34.390533447265625,
      "activations/layer16_attention_weight_max": 33.00752258300781,
      "activations/layer16_attention_weight_min": -29.702775955200195,
      "activations/layer17_attention_weight_max": 36.392189025878906,
      "activations/layer17_attention_weight_min": -26.570383071899414,
      "activations/layer18_attention_weight_max": 34.10586166381836,
      "activations/layer18_attention_weight_min": -23.494483947753906,
      "activations/layer19_attention_weight_max": 35.124271392822266,
      "activations/layer19_attention_weight_min": -31.52290153503418,
      "activations/layer1_attention_weight_max": 16.49451446533203,
      "activations/layer1_attention_weight_min": -13.765397071838379,
      "activations/layer20_attention_weight_max": 33.234737396240234,
      "activations/layer20_attention_weight_min": -25.202129364013672,
      "activations/layer21_attention_weight_max": 31.413381576538086,
      "activations/layer21_attention_weight_min": -23.84482192993164,
      "activations/layer22_attention_weight_max": 47.78785705566406,
      "activations/layer22_attention_weight_min": -29.729055404663086,
      "activations/layer23_attention_weight_max": 35.45642852783203,
      "activations/layer23_attention_weight_min": -25.321849822998047,
      "activations/layer2_attention_weight_max": 34.00238037109375,
      "activations/layer2_attention_weight_min": -32.63819885253906,
      "activations/layer3_attention_weight_max": 96.38318634033203,
      "activations/layer3_attention_weight_min": -93.14936828613281,
      "activations/layer4_attention_weight_max": 108.14888763427734,
      "activations/layer4_attention_weight_min": -104.97955322265625,
      "activations/layer5_attention_weight_max": 53.434532165527344,
      "activations/layer5_attention_weight_min": -59.61411666870117,
      "activations/layer6_attention_weight_max": 42.63576126098633,
      "activations/layer6_attention_weight_min": -46.20740509033203,
      "activations/layer7_attention_weight_max": 93.0403060913086,
      "activations/layer7_attention_weight_min": -84.49759674072266,
      "activations/layer8_attention_weight_max": 38.96671676635742,
      "activations/layer8_attention_weight_min": -42.166324615478516,
      "activations/layer9_attention_weight_max": 30.70020866394043,
      "activations/layer9_attention_weight_min": -31.421430587768555,
      "epoch": 13.36,
      "learning_rate": 6.446590909090908e-05,
      "loss": 2.7661,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4719,
      "eval_samples_per_second": 506.854,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4719,
      "eval_openwebtext_samples_per_second": 506.854,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9661,
      "eval_wikitext_samples_per_second": 231.931,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_lambada_loss": 2.427734375,
      "eval_lambada_ppl": 11.333176243677455,
      "eval_lambada_runtime": 9.5447,
      "eval_lambada_samples_per_second": 510.125,
      "step": 230000
    },
    {
      "activations/layer0_attention_weight_max": 16.054048538208008,
      "activations/layer0_attention_weight_min": -12.134130477905273,
      "activations/layer10_attention_weight_max": 39.483219146728516,
      "activations/layer10_attention_weight_min": -36.15694808959961,
      "activations/layer11_attention_weight_max": 41.59264373779297,
      "activations/layer11_attention_weight_min": -40.27588653564453,
      "activations/layer12_attention_weight_max": 26.851945877075195,
      "activations/layer12_attention_weight_min": -28.471372604370117,
      "activations/layer13_attention_weight_max": 37.96158981323242,
      "activations/layer13_attention_weight_min": -36.10304260253906,
      "activations/layer14_attention_weight_max": 42.70281219482422,
      "activations/layer14_attention_weight_min": -38.972251892089844,
      "activations/layer15_attention_weight_max": 45.42071533203125,
      "activations/layer15_attention_weight_min": -35.90336227416992,
      "activations/layer16_attention_weight_max": 33.1112174987793,
      "activations/layer16_attention_weight_min": -27.275848388671875,
      "activations/layer17_attention_weight_max": 30.57342529296875,
      "activations/layer17_attention_weight_min": -28.396623611450195,
      "activations/layer18_attention_weight_max": 30.03632926940918,
      "activations/layer18_attention_weight_min": -26.817806243896484,
      "activations/layer19_attention_weight_max": 36.48625946044922,
      "activations/layer19_attention_weight_min": -32.418460845947266,
      "activations/layer1_attention_weight_max": 16.99004364013672,
      "activations/layer1_attention_weight_min": -15.791483879089355,
      "activations/layer20_attention_weight_max": 29.769535064697266,
      "activations/layer20_attention_weight_min": -24.19737434387207,
      "activations/layer21_attention_weight_max": 29.780460357666016,
      "activations/layer21_attention_weight_min": -24.792572021484375,
      "activations/layer22_attention_weight_max": 40.968055725097656,
      "activations/layer22_attention_weight_min": -30.882213592529297,
      "activations/layer23_attention_weight_max": 37.144256591796875,
      "activations/layer23_attention_weight_min": -25.588703155517578,
      "activations/layer2_attention_weight_max": 34.0672492980957,
      "activations/layer2_attention_weight_min": -36.31255340576172,
      "activations/layer3_attention_weight_max": 94.4463882446289,
      "activations/layer3_attention_weight_min": -101.4322509765625,
      "activations/layer4_attention_weight_max": 108.12300109863281,
      "activations/layer4_attention_weight_min": -108.98221588134766,
      "activations/layer5_attention_weight_max": 55.88619613647461,
      "activations/layer5_attention_weight_min": -57.397178649902344,
      "activations/layer6_attention_weight_max": 41.959224700927734,
      "activations/layer6_attention_weight_min": -44.23503875732422,
      "activations/layer7_attention_weight_max": 95.1448745727539,
      "activations/layer7_attention_weight_min": -90.55810546875,
      "activations/layer8_attention_weight_max": 43.07113265991211,
      "activations/layer8_attention_weight_min": -43.31791687011719,
      "activations/layer9_attention_weight_max": 34.076412200927734,
      "activations/layer9_attention_weight_min": -35.41389083862305,
      "epoch": 13.37,
      "learning_rate": 6.444734848484848e-05,
      "loss": 2.7644,
      "step": 230050
    },
    {
      "activations/layer0_attention_weight_max": 15.744413375854492,
      "activations/layer0_attention_weight_min": -14.505043029785156,
      "activations/layer10_attention_weight_max": 30.793350219726562,
      "activations/layer10_attention_weight_min": -32.803123474121094,
      "activations/layer11_attention_weight_max": 31.624980926513672,
      "activations/layer11_attention_weight_min": -30.952655792236328,
      "activations/layer12_attention_weight_max": 24.46196174621582,
      "activations/layer12_attention_weight_min": -25.94066619873047,
      "activations/layer13_attention_weight_max": 39.74044418334961,
      "activations/layer13_attention_weight_min": -35.56673049926758,
      "activations/layer14_attention_weight_max": 40.96992492675781,
      "activations/layer14_attention_weight_min": -37.83289337158203,
      "activations/layer15_attention_weight_max": 37.114967346191406,
      "activations/layer15_attention_weight_min": -32.55000305175781,
      "activations/layer16_attention_weight_max": 27.5540714263916,
      "activations/layer16_attention_weight_min": -27.92119598388672,
      "activations/layer17_attention_weight_max": 31.31452178955078,
      "activations/layer17_attention_weight_min": -25.96977996826172,
      "activations/layer18_attention_weight_max": 31.988683700561523,
      "activations/layer18_attention_weight_min": -25.935060501098633,
      "activations/layer19_attention_weight_max": 35.686641693115234,
      "activations/layer19_attention_weight_min": -31.315378189086914,
      "activations/layer1_attention_weight_max": 15.993573188781738,
      "activations/layer1_attention_weight_min": -14.970653533935547,
      "activations/layer20_attention_weight_max": 29.49049949645996,
      "activations/layer20_attention_weight_min": -24.88182830810547,
      "activations/layer21_attention_weight_max": 29.046056747436523,
      "activations/layer21_attention_weight_min": -24.750688552856445,
      "activations/layer22_attention_weight_max": 52.99497604370117,
      "activations/layer22_attention_weight_min": -31.41648292541504,
      "activations/layer23_attention_weight_max": 36.315040588378906,
      "activations/layer23_attention_weight_min": -24.82675552368164,
      "activations/layer2_attention_weight_max": 34.22801971435547,
      "activations/layer2_attention_weight_min": -33.335636138916016,
      "activations/layer3_attention_weight_max": 101.33689880371094,
      "activations/layer3_attention_weight_min": -104.50830078125,
      "activations/layer4_attention_weight_max": 114.67655181884766,
      "activations/layer4_attention_weight_min": -108.16259765625,
      "activations/layer5_attention_weight_max": 52.592071533203125,
      "activations/layer5_attention_weight_min": -60.03647994995117,
      "activations/layer6_attention_weight_max": 44.57851791381836,
      "activations/layer6_attention_weight_min": -47.137962341308594,
      "activations/layer7_attention_weight_max": 88.17440032958984,
      "activations/layer7_attention_weight_min": -92.17372131347656,
      "activations/layer8_attention_weight_max": 39.50099563598633,
      "activations/layer8_attention_weight_min": -42.221160888671875,
      "activations/layer9_attention_weight_max": 33.38582992553711,
      "activations/layer9_attention_weight_min": -33.22037887573242,
      "epoch": 13.37,
      "learning_rate": 6.442840909090908e-05,
      "loss": 2.7751,
      "step": 230100
    },
    {
      "activations/layer0_attention_weight_max": 16.2049503326416,
      "activations/layer0_attention_weight_min": -12.677508354187012,
      "activations/layer10_attention_weight_max": 35.17891311645508,
      "activations/layer10_attention_weight_min": -34.01697540283203,
      "activations/layer11_attention_weight_max": 33.818626403808594,
      "activations/layer11_attention_weight_min": -33.1950569152832,
      "activations/layer12_attention_weight_max": 26.550338745117188,
      "activations/layer12_attention_weight_min": -24.40723991394043,
      "activations/layer13_attention_weight_max": 36.780574798583984,
      "activations/layer13_attention_weight_min": -34.17863845825195,
      "activations/layer14_attention_weight_max": 40.54912567138672,
      "activations/layer14_attention_weight_min": -35.43907928466797,
      "activations/layer15_attention_weight_max": 34.0857048034668,
      "activations/layer15_attention_weight_min": -32.114933013916016,
      "activations/layer16_attention_weight_max": 29.0169734954834,
      "activations/layer16_attention_weight_min": -28.302526473999023,
      "activations/layer17_attention_weight_max": 31.51443862915039,
      "activations/layer17_attention_weight_min": -26.041629791259766,
      "activations/layer18_attention_weight_max": 30.411527633666992,
      "activations/layer18_attention_weight_min": -24.628652572631836,
      "activations/layer19_attention_weight_max": 34.182533264160156,
      "activations/layer19_attention_weight_min": -30.405370712280273,
      "activations/layer1_attention_weight_max": 16.516765594482422,
      "activations/layer1_attention_weight_min": -16.570459365844727,
      "activations/layer20_attention_weight_max": 27.437902450561523,
      "activations/layer20_attention_weight_min": -24.552452087402344,
      "activations/layer21_attention_weight_max": 28.136045455932617,
      "activations/layer21_attention_weight_min": -23.09619903564453,
      "activations/layer22_attention_weight_max": 40.550987243652344,
      "activations/layer22_attention_weight_min": -29.195724487304688,
      "activations/layer23_attention_weight_max": 34.022239685058594,
      "activations/layer23_attention_weight_min": -25.99355697631836,
      "activations/layer2_attention_weight_max": 34.74937438964844,
      "activations/layer2_attention_weight_min": -34.514713287353516,
      "activations/layer3_attention_weight_max": 100.58421325683594,
      "activations/layer3_attention_weight_min": -102.56734466552734,
      "activations/layer4_attention_weight_max": 106.73626708984375,
      "activations/layer4_attention_weight_min": -108.72977447509766,
      "activations/layer5_attention_weight_max": 51.78575897216797,
      "activations/layer5_attention_weight_min": -62.6621208190918,
      "activations/layer6_attention_weight_max": 43.03537368774414,
      "activations/layer6_attention_weight_min": -47.594139099121094,
      "activations/layer7_attention_weight_max": 88.56077575683594,
      "activations/layer7_attention_weight_min": -88.53014373779297,
      "activations/layer8_attention_weight_max": 40.423095703125,
      "activations/layer8_attention_weight_min": -40.54798126220703,
      "activations/layer9_attention_weight_max": 33.25828552246094,
      "activations/layer9_attention_weight_min": -32.970420837402344,
      "epoch": 13.37,
      "learning_rate": 6.440946969696969e-05,
      "loss": 2.7477,
      "step": 230150
    },
    {
      "activations/layer0_attention_weight_max": 16.142108917236328,
      "activations/layer0_attention_weight_min": -12.509406089782715,
      "activations/layer10_attention_weight_max": 33.8213005065918,
      "activations/layer10_attention_weight_min": -31.108678817749023,
      "activations/layer11_attention_weight_max": 32.325462341308594,
      "activations/layer11_attention_weight_min": -30.81853485107422,
      "activations/layer12_attention_weight_max": 24.30741310119629,
      "activations/layer12_attention_weight_min": -23.425878524780273,
      "activations/layer13_attention_weight_max": 35.91965103149414,
      "activations/layer13_attention_weight_min": -32.85746765136719,
      "activations/layer14_attention_weight_max": 36.57624816894531,
      "activations/layer14_attention_weight_min": -36.12520980834961,
      "activations/layer15_attention_weight_max": 32.9765625,
      "activations/layer15_attention_weight_min": -30.20256233215332,
      "activations/layer16_attention_weight_max": 29.391063690185547,
      "activations/layer16_attention_weight_min": -27.895404815673828,
      "activations/layer17_attention_weight_max": 28.643659591674805,
      "activations/layer17_attention_weight_min": -26.634695053100586,
      "activations/layer18_attention_weight_max": 29.994304656982422,
      "activations/layer18_attention_weight_min": -23.90105438232422,
      "activations/layer19_attention_weight_max": 30.91778564453125,
      "activations/layer19_attention_weight_min": -32.34653091430664,
      "activations/layer1_attention_weight_max": 16.119014739990234,
      "activations/layer1_attention_weight_min": -15.222233772277832,
      "activations/layer20_attention_weight_max": 26.093351364135742,
      "activations/layer20_attention_weight_min": -28.313390731811523,
      "activations/layer21_attention_weight_max": 26.881559371948242,
      "activations/layer21_attention_weight_min": -25.763530731201172,
      "activations/layer22_attention_weight_max": 39.43526840209961,
      "activations/layer22_attention_weight_min": -33.48887252807617,
      "activations/layer23_attention_weight_max": 34.731201171875,
      "activations/layer23_attention_weight_min": -30.17226791381836,
      "activations/layer2_attention_weight_max": 33.22697830200195,
      "activations/layer2_attention_weight_min": -34.63078308105469,
      "activations/layer3_attention_weight_max": 99.08760833740234,
      "activations/layer3_attention_weight_min": -101.0906982421875,
      "activations/layer4_attention_weight_max": 108.66515350341797,
      "activations/layer4_attention_weight_min": -108.46207427978516,
      "activations/layer5_attention_weight_max": 51.708465576171875,
      "activations/layer5_attention_weight_min": -58.921241760253906,
      "activations/layer6_attention_weight_max": 41.80967330932617,
      "activations/layer6_attention_weight_min": -45.87546157836914,
      "activations/layer7_attention_weight_max": 88.47054290771484,
      "activations/layer7_attention_weight_min": -88.7200698852539,
      "activations/layer8_attention_weight_max": 40.255680084228516,
      "activations/layer8_attention_weight_min": -39.841957092285156,
      "activations/layer9_attention_weight_max": 32.432952880859375,
      "activations/layer9_attention_weight_min": -33.95742416381836,
      "epoch": 13.38,
      "learning_rate": 6.439053030303029e-05,
      "loss": 2.7663,
      "step": 230200
    },
    {
      "activations/layer0_attention_weight_max": 16.08313751220703,
      "activations/layer0_attention_weight_min": -12.61310863494873,
      "activations/layer10_attention_weight_max": 32.9147834777832,
      "activations/layer10_attention_weight_min": -32.706642150878906,
      "activations/layer11_attention_weight_max": 31.383283615112305,
      "activations/layer11_attention_weight_min": -34.46283721923828,
      "activations/layer12_attention_weight_max": 25.44391441345215,
      "activations/layer12_attention_weight_min": -32.436683654785156,
      "activations/layer13_attention_weight_max": 40.12263488769531,
      "activations/layer13_attention_weight_min": -38.55986404418945,
      "activations/layer14_attention_weight_max": 38.176937103271484,
      "activations/layer14_attention_weight_min": -37.39848709106445,
      "activations/layer15_attention_weight_max": 39.8762092590332,
      "activations/layer15_attention_weight_min": -34.91143035888672,
      "activations/layer16_attention_weight_max": 29.079809188842773,
      "activations/layer16_attention_weight_min": -27.62155532836914,
      "activations/layer17_attention_weight_max": 28.96555519104004,
      "activations/layer17_attention_weight_min": -24.898048400878906,
      "activations/layer18_attention_weight_max": 29.760848999023438,
      "activations/layer18_attention_weight_min": -23.922943115234375,
      "activations/layer19_attention_weight_max": 33.20289611816406,
      "activations/layer19_attention_weight_min": -30.666645050048828,
      "activations/layer1_attention_weight_max": 16.3308162689209,
      "activations/layer1_attention_weight_min": -16.306669235229492,
      "activations/layer20_attention_weight_max": 26.4576416015625,
      "activations/layer20_attention_weight_min": -23.44802474975586,
      "activations/layer21_attention_weight_max": 26.412778854370117,
      "activations/layer21_attention_weight_min": -23.834392547607422,
      "activations/layer22_attention_weight_max": 34.7484016418457,
      "activations/layer22_attention_weight_min": -29.654144287109375,
      "activations/layer23_attention_weight_max": 28.938526153564453,
      "activations/layer23_attention_weight_min": -24.74923324584961,
      "activations/layer2_attention_weight_max": 32.95294952392578,
      "activations/layer2_attention_weight_min": -32.011512756347656,
      "activations/layer3_attention_weight_max": 96.33065032958984,
      "activations/layer3_attention_weight_min": -96.83708190917969,
      "activations/layer4_attention_weight_max": 110.26649475097656,
      "activations/layer4_attention_weight_min": -112.34319305419922,
      "activations/layer5_attention_weight_max": 55.449886322021484,
      "activations/layer5_attention_weight_min": -65.7333984375,
      "activations/layer6_attention_weight_max": 45.54367446899414,
      "activations/layer6_attention_weight_min": -48.54928970336914,
      "activations/layer7_attention_weight_max": 93.82840728759766,
      "activations/layer7_attention_weight_min": -105.29264068603516,
      "activations/layer8_attention_weight_max": 41.419673919677734,
      "activations/layer8_attention_weight_min": -43.91121292114258,
      "activations/layer9_attention_weight_max": 33.096370697021484,
      "activations/layer9_attention_weight_min": -33.915809631347656,
      "epoch": 13.38,
      "learning_rate": 6.43715909090909e-05,
      "loss": 2.7717,
      "step": 230250
    },
    {
      "activations/layer0_attention_weight_max": 16.048006057739258,
      "activations/layer0_attention_weight_min": -11.866127014160156,
      "activations/layer10_attention_weight_max": 31.85917091369629,
      "activations/layer10_attention_weight_min": -31.197147369384766,
      "activations/layer11_attention_weight_max": 32.3088493347168,
      "activations/layer11_attention_weight_min": -32.42331314086914,
      "activations/layer12_attention_weight_max": 27.655344009399414,
      "activations/layer12_attention_weight_min": -25.08104133605957,
      "activations/layer13_attention_weight_max": 41.401878356933594,
      "activations/layer13_attention_weight_min": -33.56220245361328,
      "activations/layer14_attention_weight_max": 45.75381088256836,
      "activations/layer14_attention_weight_min": -36.46811294555664,
      "activations/layer15_attention_weight_max": 36.93206024169922,
      "activations/layer15_attention_weight_min": -32.92418670654297,
      "activations/layer16_attention_weight_max": 30.434850692749023,
      "activations/layer16_attention_weight_min": -31.380908966064453,
      "activations/layer17_attention_weight_max": 32.628807067871094,
      "activations/layer17_attention_weight_min": -27.734933853149414,
      "activations/layer18_attention_weight_max": 29.9354305267334,
      "activations/layer18_attention_weight_min": -25.258106231689453,
      "activations/layer19_attention_weight_max": 35.32200241088867,
      "activations/layer19_attention_weight_min": -31.463977813720703,
      "activations/layer1_attention_weight_max": 16.31346321105957,
      "activations/layer1_attention_weight_min": -14.686933517456055,
      "activations/layer20_attention_weight_max": 28.181867599487305,
      "activations/layer20_attention_weight_min": -24.42622947692871,
      "activations/layer21_attention_weight_max": 27.27652359008789,
      "activations/layer21_attention_weight_min": -23.39035415649414,
      "activations/layer22_attention_weight_max": 43.45520782470703,
      "activations/layer22_attention_weight_min": -31.66786766052246,
      "activations/layer23_attention_weight_max": 35.36133575439453,
      "activations/layer23_attention_weight_min": -26.82611846923828,
      "activations/layer2_attention_weight_max": 35.21577835083008,
      "activations/layer2_attention_weight_min": -34.10947799682617,
      "activations/layer3_attention_weight_max": 103.61953735351562,
      "activations/layer3_attention_weight_min": -101.16030883789062,
      "activations/layer4_attention_weight_max": 116.18199920654297,
      "activations/layer4_attention_weight_min": -120.07655334472656,
      "activations/layer5_attention_weight_max": 52.50122833251953,
      "activations/layer5_attention_weight_min": -63.62113952636719,
      "activations/layer6_attention_weight_max": 46.22492980957031,
      "activations/layer6_attention_weight_min": -47.612857818603516,
      "activations/layer7_attention_weight_max": 87.00399017333984,
      "activations/layer7_attention_weight_min": -92.0639419555664,
      "activations/layer8_attention_weight_max": 40.32060241699219,
      "activations/layer8_attention_weight_min": -40.49113082885742,
      "activations/layer9_attention_weight_max": 33.42557144165039,
      "activations/layer9_attention_weight_min": -32.50971221923828,
      "epoch": 13.38,
      "learning_rate": 6.43526515151515e-05,
      "loss": 2.7582,
      "step": 230300
    },
    {
      "activations/layer0_attention_weight_max": 16.46585464477539,
      "activations/layer0_attention_weight_min": -13.144320487976074,
      "activations/layer10_attention_weight_max": 32.81687545776367,
      "activations/layer10_attention_weight_min": -30.799850463867188,
      "activations/layer11_attention_weight_max": 32.56952667236328,
      "activations/layer11_attention_weight_min": -32.43524169921875,
      "activations/layer12_attention_weight_max": 38.77488327026367,
      "activations/layer12_attention_weight_min": -25.658897399902344,
      "activations/layer13_attention_weight_max": 48.146324157714844,
      "activations/layer13_attention_weight_min": -34.65765380859375,
      "activations/layer14_attention_weight_max": 47.330726623535156,
      "activations/layer14_attention_weight_min": -37.62468338012695,
      "activations/layer15_attention_weight_max": 39.296836853027344,
      "activations/layer15_attention_weight_min": -33.611122131347656,
      "activations/layer16_attention_weight_max": 33.09819412231445,
      "activations/layer16_attention_weight_min": -29.86906623840332,
      "activations/layer17_attention_weight_max": 32.010250091552734,
      "activations/layer17_attention_weight_min": -26.69988250732422,
      "activations/layer18_attention_weight_max": 30.571491241455078,
      "activations/layer18_attention_weight_min": -23.868389129638672,
      "activations/layer19_attention_weight_max": 38.61931610107422,
      "activations/layer19_attention_weight_min": -32.06888961791992,
      "activations/layer1_attention_weight_max": 16.604246139526367,
      "activations/layer1_attention_weight_min": -14.900431632995605,
      "activations/layer20_attention_weight_max": 33.442909240722656,
      "activations/layer20_attention_weight_min": -28.040658950805664,
      "activations/layer21_attention_weight_max": 29.490962982177734,
      "activations/layer21_attention_weight_min": -26.17159080505371,
      "activations/layer22_attention_weight_max": 45.48904800415039,
      "activations/layer22_attention_weight_min": -30.587732315063477,
      "activations/layer23_attention_weight_max": 32.595916748046875,
      "activations/layer23_attention_weight_min": -26.544021606445312,
      "activations/layer2_attention_weight_max": 32.02572250366211,
      "activations/layer2_attention_weight_min": -32.175392150878906,
      "activations/layer3_attention_weight_max": 95.23552703857422,
      "activations/layer3_attention_weight_min": -97.98826599121094,
      "activations/layer4_attention_weight_max": 106.20311737060547,
      "activations/layer4_attention_weight_min": -110.41218566894531,
      "activations/layer5_attention_weight_max": 50.92646789550781,
      "activations/layer5_attention_weight_min": -60.113990783691406,
      "activations/layer6_attention_weight_max": 44.252254486083984,
      "activations/layer6_attention_weight_min": -45.197608947753906,
      "activations/layer7_attention_weight_max": 90.94190979003906,
      "activations/layer7_attention_weight_min": -84.92173767089844,
      "activations/layer8_attention_weight_max": 40.06864547729492,
      "activations/layer8_attention_weight_min": -39.50645065307617,
      "activations/layer9_attention_weight_max": 35.29378890991211,
      "activations/layer9_attention_weight_min": -32.244117736816406,
      "epoch": 13.38,
      "learning_rate": 6.433371212121212e-05,
      "loss": 2.763,
      "step": 230350
    },
    {
      "activations/layer0_attention_weight_max": 16.016111373901367,
      "activations/layer0_attention_weight_min": -12.649516105651855,
      "activations/layer10_attention_weight_max": 29.988182067871094,
      "activations/layer10_attention_weight_min": -31.723325729370117,
      "activations/layer11_attention_weight_max": 30.04536247253418,
      "activations/layer11_attention_weight_min": -30.46392822265625,
      "activations/layer12_attention_weight_max": 23.418243408203125,
      "activations/layer12_attention_weight_min": -25.025054931640625,
      "activations/layer13_attention_weight_max": 34.076820373535156,
      "activations/layer13_attention_weight_min": -32.80970764160156,
      "activations/layer14_attention_weight_max": 36.98433303833008,
      "activations/layer14_attention_weight_min": -36.7799186706543,
      "activations/layer15_attention_weight_max": 32.455081939697266,
      "activations/layer15_attention_weight_min": -32.112823486328125,
      "activations/layer16_attention_weight_max": 27.498271942138672,
      "activations/layer16_attention_weight_min": -28.00627326965332,
      "activations/layer17_attention_weight_max": 29.15805435180664,
      "activations/layer17_attention_weight_min": -25.23175048828125,
      "activations/layer18_attention_weight_max": 28.23406410217285,
      "activations/layer18_attention_weight_min": -24.100101470947266,
      "activations/layer19_attention_weight_max": 29.999425888061523,
      "activations/layer19_attention_weight_min": -31.42422866821289,
      "activations/layer1_attention_weight_max": 17.259714126586914,
      "activations/layer1_attention_weight_min": -16.18614959716797,
      "activations/layer20_attention_weight_max": 27.668190002441406,
      "activations/layer20_attention_weight_min": -25.684370040893555,
      "activations/layer21_attention_weight_max": 29.23751449584961,
      "activations/layer21_attention_weight_min": -24.979576110839844,
      "activations/layer22_attention_weight_max": 39.39811706542969,
      "activations/layer22_attention_weight_min": -30.485273361206055,
      "activations/layer23_attention_weight_max": 30.591596603393555,
      "activations/layer23_attention_weight_min": -25.224712371826172,
      "activations/layer2_attention_weight_max": 34.17393493652344,
      "activations/layer2_attention_weight_min": -34.118751525878906,
      "activations/layer3_attention_weight_max": 102.06060028076172,
      "activations/layer3_attention_weight_min": -101.86608123779297,
      "activations/layer4_attention_weight_max": 109.97216796875,
      "activations/layer4_attention_weight_min": -115.0276107788086,
      "activations/layer5_attention_weight_max": 52.90601348876953,
      "activations/layer5_attention_weight_min": -61.97895431518555,
      "activations/layer6_attention_weight_max": 42.625099182128906,
      "activations/layer6_attention_weight_min": -44.97364044189453,
      "activations/layer7_attention_weight_max": 88.3052978515625,
      "activations/layer7_attention_weight_min": -88.68756866455078,
      "activations/layer8_attention_weight_max": 41.643272399902344,
      "activations/layer8_attention_weight_min": -38.57447052001953,
      "activations/layer9_attention_weight_max": 30.96295738220215,
      "activations/layer9_attention_weight_min": -31.60370445251465,
      "epoch": 13.39,
      "learning_rate": 6.431477272727272e-05,
      "loss": 2.7588,
      "step": 230400
    },
    {
      "activations/layer0_attention_weight_max": 15.16025161743164,
      "activations/layer0_attention_weight_min": -12.925494194030762,
      "activations/layer10_attention_weight_max": 33.04583740234375,
      "activations/layer10_attention_weight_min": -34.18940734863281,
      "activations/layer11_attention_weight_max": 33.609676361083984,
      "activations/layer11_attention_weight_min": -34.84332275390625,
      "activations/layer12_attention_weight_max": 27.03678321838379,
      "activations/layer12_attention_weight_min": -25.35027503967285,
      "activations/layer13_attention_weight_max": 41.67814254760742,
      "activations/layer13_attention_weight_min": -34.55824661254883,
      "activations/layer14_attention_weight_max": 43.2520866394043,
      "activations/layer14_attention_weight_min": -37.28993606567383,
      "activations/layer15_attention_weight_max": 39.90205001831055,
      "activations/layer15_attention_weight_min": -34.30836868286133,
      "activations/layer16_attention_weight_max": 31.48244285583496,
      "activations/layer16_attention_weight_min": -29.565284729003906,
      "activations/layer17_attention_weight_max": 30.964345932006836,
      "activations/layer17_attention_weight_min": -26.40424156188965,
      "activations/layer18_attention_weight_max": 31.79157829284668,
      "activations/layer18_attention_weight_min": -25.46685028076172,
      "activations/layer19_attention_weight_max": 37.36875915527344,
      "activations/layer19_attention_weight_min": -31.92966651916504,
      "activations/layer1_attention_weight_max": 17.13770294189453,
      "activations/layer1_attention_weight_min": -15.484527587890625,
      "activations/layer20_attention_weight_max": 31.323463439941406,
      "activations/layer20_attention_weight_min": -26.63469123840332,
      "activations/layer21_attention_weight_max": 31.471111297607422,
      "activations/layer21_attention_weight_min": -26.143369674682617,
      "activations/layer22_attention_weight_max": 43.424373626708984,
      "activations/layer22_attention_weight_min": -31.115707397460938,
      "activations/layer23_attention_weight_max": 35.85634994506836,
      "activations/layer23_attention_weight_min": -27.666275024414062,
      "activations/layer2_attention_weight_max": 32.48848342895508,
      "activations/layer2_attention_weight_min": -33.690364837646484,
      "activations/layer3_attention_weight_max": 95.75430297851562,
      "activations/layer3_attention_weight_min": -103.592041015625,
      "activations/layer4_attention_weight_max": 107.47257232666016,
      "activations/layer4_attention_weight_min": -109.75239562988281,
      "activations/layer5_attention_weight_max": 51.580806732177734,
      "activations/layer5_attention_weight_min": -60.566429138183594,
      "activations/layer6_attention_weight_max": 43.54417037963867,
      "activations/layer6_attention_weight_min": -46.508445739746094,
      "activations/layer7_attention_weight_max": 90.83219146728516,
      "activations/layer7_attention_weight_min": -93.48147583007812,
      "activations/layer8_attention_weight_max": 40.275917053222656,
      "activations/layer8_attention_weight_min": -40.56300735473633,
      "activations/layer9_attention_weight_max": 33.573055267333984,
      "activations/layer9_attention_weight_min": -33.48072814941406,
      "epoch": 13.39,
      "learning_rate": 6.429583333333334e-05,
      "loss": 2.7687,
      "step": 230450
    },
    {
      "activations/layer0_attention_weight_max": 15.60867691040039,
      "activations/layer0_attention_weight_min": -13.272812843322754,
      "activations/layer10_attention_weight_max": 37.300994873046875,
      "activations/layer10_attention_weight_min": -34.9962043762207,
      "activations/layer11_attention_weight_max": 39.52939224243164,
      "activations/layer11_attention_weight_min": -33.265594482421875,
      "activations/layer12_attention_weight_max": 25.635475158691406,
      "activations/layer12_attention_weight_min": -25.897342681884766,
      "activations/layer13_attention_weight_max": 37.68906784057617,
      "activations/layer13_attention_weight_min": -34.47760772705078,
      "activations/layer14_attention_weight_max": 42.7608757019043,
      "activations/layer14_attention_weight_min": -39.23286056518555,
      "activations/layer15_attention_weight_max": 39.49805450439453,
      "activations/layer15_attention_weight_min": -35.3065185546875,
      "activations/layer16_attention_weight_max": 33.8190803527832,
      "activations/layer16_attention_weight_min": -26.866695404052734,
      "activations/layer17_attention_weight_max": 30.101449966430664,
      "activations/layer17_attention_weight_min": -25.8784122467041,
      "activations/layer18_attention_weight_max": 30.3372859954834,
      "activations/layer18_attention_weight_min": -23.680315017700195,
      "activations/layer19_attention_weight_max": 32.11610412597656,
      "activations/layer19_attention_weight_min": -30.50077247619629,
      "activations/layer1_attention_weight_max": 17.199499130249023,
      "activations/layer1_attention_weight_min": -16.509784698486328,
      "activations/layer20_attention_weight_max": 29.557222366333008,
      "activations/layer20_attention_weight_min": -24.22983741760254,
      "activations/layer21_attention_weight_max": 28.54772186279297,
      "activations/layer21_attention_weight_min": -24.603466033935547,
      "activations/layer22_attention_weight_max": 40.251163482666016,
      "activations/layer22_attention_weight_min": -29.74020767211914,
      "activations/layer23_attention_weight_max": 36.441314697265625,
      "activations/layer23_attention_weight_min": -25.044803619384766,
      "activations/layer2_attention_weight_max": 34.00764846801758,
      "activations/layer2_attention_weight_min": -34.839317321777344,
      "activations/layer3_attention_weight_max": 102.66840362548828,
      "activations/layer3_attention_weight_min": -101.01612091064453,
      "activations/layer4_attention_weight_max": 114.66157531738281,
      "activations/layer4_attention_weight_min": -109.65743255615234,
      "activations/layer5_attention_weight_max": 52.77058029174805,
      "activations/layer5_attention_weight_min": -60.17293930053711,
      "activations/layer6_attention_weight_max": 47.050209045410156,
      "activations/layer6_attention_weight_min": -45.56620788574219,
      "activations/layer7_attention_weight_max": 99.0958251953125,
      "activations/layer7_attention_weight_min": -87.66425323486328,
      "activations/layer8_attention_weight_max": 41.67625427246094,
      "activations/layer8_attention_weight_min": -42.19912338256836,
      "activations/layer9_attention_weight_max": 34.49799346923828,
      "activations/layer9_attention_weight_min": -33.82425308227539,
      "epoch": 13.39,
      "learning_rate": 6.427689393939394e-05,
      "loss": 2.7664,
      "step": 230500
    },
    {
      "activations/layer0_attention_weight_max": 15.900893211364746,
      "activations/layer0_attention_weight_min": -12.282136917114258,
      "activations/layer10_attention_weight_max": 35.58250427246094,
      "activations/layer10_attention_weight_min": -36.52130126953125,
      "activations/layer11_attention_weight_max": 35.47770309448242,
      "activations/layer11_attention_weight_min": -33.63311004638672,
      "activations/layer12_attention_weight_max": 29.964954376220703,
      "activations/layer12_attention_weight_min": -26.65201759338379,
      "activations/layer13_attention_weight_max": 47.76552963256836,
      "activations/layer13_attention_weight_min": -34.783424377441406,
      "activations/layer14_attention_weight_max": 57.26179504394531,
      "activations/layer14_attention_weight_min": -43.28719711303711,
      "activations/layer15_attention_weight_max": 46.430511474609375,
      "activations/layer15_attention_weight_min": -35.782958984375,
      "activations/layer16_attention_weight_max": 35.944908142089844,
      "activations/layer16_attention_weight_min": -28.867414474487305,
      "activations/layer17_attention_weight_max": 37.88044738769531,
      "activations/layer17_attention_weight_min": -27.768075942993164,
      "activations/layer18_attention_weight_max": 33.36630630493164,
      "activations/layer18_attention_weight_min": -26.387195587158203,
      "activations/layer19_attention_weight_max": 38.32062911987305,
      "activations/layer19_attention_weight_min": -33.321990966796875,
      "activations/layer1_attention_weight_max": 16.8089656829834,
      "activations/layer1_attention_weight_min": -15.68019962310791,
      "activations/layer20_attention_weight_max": 32.808719635009766,
      "activations/layer20_attention_weight_min": -26.81016731262207,
      "activations/layer21_attention_weight_max": 30.685264587402344,
      "activations/layer21_attention_weight_min": -25.517295837402344,
      "activations/layer22_attention_weight_max": 46.6832389831543,
      "activations/layer22_attention_weight_min": -33.962547302246094,
      "activations/layer23_attention_weight_max": 35.45062255859375,
      "activations/layer23_attention_weight_min": -26.084945678710938,
      "activations/layer2_attention_weight_max": 33.16966247558594,
      "activations/layer2_attention_weight_min": -36.06387710571289,
      "activations/layer3_attention_weight_max": 101.86343383789062,
      "activations/layer3_attention_weight_min": -104.83421325683594,
      "activations/layer4_attention_weight_max": 111.58574676513672,
      "activations/layer4_attention_weight_min": -113.45763397216797,
      "activations/layer5_attention_weight_max": 51.90193176269531,
      "activations/layer5_attention_weight_min": -59.78864669799805,
      "activations/layer6_attention_weight_max": 47.34779739379883,
      "activations/layer6_attention_weight_min": -48.79601287841797,
      "activations/layer7_attention_weight_max": 97.47689819335938,
      "activations/layer7_attention_weight_min": -98.60006713867188,
      "activations/layer8_attention_weight_max": 44.98618698120117,
      "activations/layer8_attention_weight_min": -45.41096115112305,
      "activations/layer9_attention_weight_max": 34.69533920288086,
      "activations/layer9_attention_weight_min": -34.740108489990234,
      "epoch": 13.4,
      "learning_rate": 6.425795454545454e-05,
      "loss": 2.7588,
      "step": 230550
    },
    {
      "activations/layer0_attention_weight_max": 16.26607894897461,
      "activations/layer0_attention_weight_min": -12.530375480651855,
      "activations/layer10_attention_weight_max": 32.824832916259766,
      "activations/layer10_attention_weight_min": -32.79329299926758,
      "activations/layer11_attention_weight_max": 32.32980728149414,
      "activations/layer11_attention_weight_min": -31.843727111816406,
      "activations/layer12_attention_weight_max": 25.43718147277832,
      "activations/layer12_attention_weight_min": -25.535728454589844,
      "activations/layer13_attention_weight_max": 39.446041107177734,
      "activations/layer13_attention_weight_min": -34.943180084228516,
      "activations/layer14_attention_weight_max": 43.93494415283203,
      "activations/layer14_attention_weight_min": -38.59170913696289,
      "activations/layer15_attention_weight_max": 35.71073532104492,
      "activations/layer15_attention_weight_min": -33.6077880859375,
      "activations/layer16_attention_weight_max": 29.210887908935547,
      "activations/layer16_attention_weight_min": -28.79726219177246,
      "activations/layer17_attention_weight_max": 34.67885208129883,
      "activations/layer17_attention_weight_min": -25.48927116394043,
      "activations/layer18_attention_weight_max": 32.9864387512207,
      "activations/layer18_attention_weight_min": -24.577165603637695,
      "activations/layer19_attention_weight_max": 34.67811965942383,
      "activations/layer19_attention_weight_min": -31.977783203125,
      "activations/layer1_attention_weight_max": 18.52313995361328,
      "activations/layer1_attention_weight_min": -15.786605834960938,
      "activations/layer20_attention_weight_max": 32.2581901550293,
      "activations/layer20_attention_weight_min": -26.09950828552246,
      "activations/layer21_attention_weight_max": 30.040136337280273,
      "activations/layer21_attention_weight_min": -25.931241989135742,
      "activations/layer22_attention_weight_max": 45.17073440551758,
      "activations/layer22_attention_weight_min": -31.773361206054688,
      "activations/layer23_attention_weight_max": 33.735137939453125,
      "activations/layer23_attention_weight_min": -25.26809310913086,
      "activations/layer2_attention_weight_max": 33.02946853637695,
      "activations/layer2_attention_weight_min": -32.51443862915039,
      "activations/layer3_attention_weight_max": 96.94681549072266,
      "activations/layer3_attention_weight_min": -96.23786163330078,
      "activations/layer4_attention_weight_max": 109.96282958984375,
      "activations/layer4_attention_weight_min": -113.08525848388672,
      "activations/layer5_attention_weight_max": 54.697120666503906,
      "activations/layer5_attention_weight_min": -63.323936462402344,
      "activations/layer6_attention_weight_max": 43.14655303955078,
      "activations/layer6_attention_weight_min": -46.21487808227539,
      "activations/layer7_attention_weight_max": 92.95512390136719,
      "activations/layer7_attention_weight_min": -89.96958923339844,
      "activations/layer8_attention_weight_max": 41.59134292602539,
      "activations/layer8_attention_weight_min": -42.966026306152344,
      "activations/layer9_attention_weight_max": 33.14567184448242,
      "activations/layer9_attention_weight_min": -34.25206756591797,
      "epoch": 13.4,
      "learning_rate": 6.423901515151515e-05,
      "loss": 2.7501,
      "step": 230600
    },
    {
      "activations/layer0_attention_weight_max": 15.293665885925293,
      "activations/layer0_attention_weight_min": -13.33594036102295,
      "activations/layer10_attention_weight_max": 34.38836669921875,
      "activations/layer10_attention_weight_min": -34.10306930541992,
      "activations/layer11_attention_weight_max": 38.19633102416992,
      "activations/layer11_attention_weight_min": -33.49007797241211,
      "activations/layer12_attention_weight_max": 29.215885162353516,
      "activations/layer12_attention_weight_min": -28.290433883666992,
      "activations/layer13_attention_weight_max": 43.98137664794922,
      "activations/layer13_attention_weight_min": -36.78145217895508,
      "activations/layer14_attention_weight_max": 51.89341354370117,
      "activations/layer14_attention_weight_min": -40.11199188232422,
      "activations/layer15_attention_weight_max": 44.54109191894531,
      "activations/layer15_attention_weight_min": -35.97718811035156,
      "activations/layer16_attention_weight_max": 34.38720703125,
      "activations/layer16_attention_weight_min": -30.18711280822754,
      "activations/layer17_attention_weight_max": 36.43545150756836,
      "activations/layer17_attention_weight_min": -30.37759017944336,
      "activations/layer18_attention_weight_max": 35.49924850463867,
      "activations/layer18_attention_weight_min": -25.99501609802246,
      "activations/layer19_attention_weight_max": 40.9223747253418,
      "activations/layer19_attention_weight_min": -35.252037048339844,
      "activations/layer1_attention_weight_max": 16.449848175048828,
      "activations/layer1_attention_weight_min": -15.368449211120605,
      "activations/layer20_attention_weight_max": 35.695465087890625,
      "activations/layer20_attention_weight_min": -28.500215530395508,
      "activations/layer21_attention_weight_max": 32.7759895324707,
      "activations/layer21_attention_weight_min": -26.326284408569336,
      "activations/layer22_attention_weight_max": 49.732025146484375,
      "activations/layer22_attention_weight_min": -32.41077423095703,
      "activations/layer23_attention_weight_max": 38.71295928955078,
      "activations/layer23_attention_weight_min": -25.50330352783203,
      "activations/layer2_attention_weight_max": 33.32453918457031,
      "activations/layer2_attention_weight_min": -35.07765197753906,
      "activations/layer3_attention_weight_max": 98.05374908447266,
      "activations/layer3_attention_weight_min": -100.03954315185547,
      "activations/layer4_attention_weight_max": 116.41259765625,
      "activations/layer4_attention_weight_min": -113.0871810913086,
      "activations/layer5_attention_weight_max": 53.52976989746094,
      "activations/layer5_attention_weight_min": -66.22494506835938,
      "activations/layer6_attention_weight_max": 45.03390121459961,
      "activations/layer6_attention_weight_min": -48.25453567504883,
      "activations/layer7_attention_weight_max": 93.82823944091797,
      "activations/layer7_attention_weight_min": -92.21283721923828,
      "activations/layer8_attention_weight_max": 41.0239143371582,
      "activations/layer8_attention_weight_min": -41.472408294677734,
      "activations/layer9_attention_weight_max": 34.28153991699219,
      "activations/layer9_attention_weight_min": -33.84493637084961,
      "epoch": 13.4,
      "learning_rate": 6.422007575757575e-05,
      "loss": 2.7646,
      "step": 230650
    },
    {
      "activations/layer0_attention_weight_max": 16.949724197387695,
      "activations/layer0_attention_weight_min": -12.37829303741455,
      "activations/layer10_attention_weight_max": 41.17066955566406,
      "activations/layer10_attention_weight_min": -35.68114471435547,
      "activations/layer11_attention_weight_max": 39.39466094970703,
      "activations/layer11_attention_weight_min": -37.76112747192383,
      "activations/layer12_attention_weight_max": 30.059925079345703,
      "activations/layer12_attention_weight_min": -28.05213737487793,
      "activations/layer13_attention_weight_max": 50.81447982788086,
      "activations/layer13_attention_weight_min": -38.70467758178711,
      "activations/layer14_attention_weight_max": 64.83718872070312,
      "activations/layer14_attention_weight_min": -46.240394592285156,
      "activations/layer15_attention_weight_max": 50.22783660888672,
      "activations/layer15_attention_weight_min": -38.470096588134766,
      "activations/layer16_attention_weight_max": 38.95118713378906,
      "activations/layer16_attention_weight_min": -27.692230224609375,
      "activations/layer17_attention_weight_max": 35.93107986450195,
      "activations/layer17_attention_weight_min": -27.80596351623535,
      "activations/layer18_attention_weight_max": 38.64780044555664,
      "activations/layer18_attention_weight_min": -25.380599975585938,
      "activations/layer19_attention_weight_max": 41.58182907104492,
      "activations/layer19_attention_weight_min": -32.227333068847656,
      "activations/layer1_attention_weight_max": 17.018444061279297,
      "activations/layer1_attention_weight_min": -15.57229995727539,
      "activations/layer20_attention_weight_max": 33.739620208740234,
      "activations/layer20_attention_weight_min": -29.6292781829834,
      "activations/layer21_attention_weight_max": 32.92997741699219,
      "activations/layer21_attention_weight_min": -26.7413387298584,
      "activations/layer22_attention_weight_max": 55.86800003051758,
      "activations/layer22_attention_weight_min": -34.544063568115234,
      "activations/layer23_attention_weight_max": 43.08146667480469,
      "activations/layer23_attention_weight_min": -24.192846298217773,
      "activations/layer2_attention_weight_max": 32.107913970947266,
      "activations/layer2_attention_weight_min": -33.40809631347656,
      "activations/layer3_attention_weight_max": 98.38495635986328,
      "activations/layer3_attention_weight_min": -98.36030578613281,
      "activations/layer4_attention_weight_max": 108.5375747680664,
      "activations/layer4_attention_weight_min": -113.02349853515625,
      "activations/layer5_attention_weight_max": 55.38829803466797,
      "activations/layer5_attention_weight_min": -61.779266357421875,
      "activations/layer6_attention_weight_max": 45.2814826965332,
      "activations/layer6_attention_weight_min": -46.17146682739258,
      "activations/layer7_attention_weight_max": 101.35345458984375,
      "activations/layer7_attention_weight_min": -97.41313934326172,
      "activations/layer8_attention_weight_max": 46.8106575012207,
      "activations/layer8_attention_weight_min": -44.60030746459961,
      "activations/layer9_attention_weight_max": 35.139705657958984,
      "activations/layer9_attention_weight_min": -36.34214782714844,
      "epoch": 13.4,
      "learning_rate": 6.420113636363636e-05,
      "loss": 2.7458,
      "step": 230700
    },
    {
      "activations/layer0_attention_weight_max": 17.36675453186035,
      "activations/layer0_attention_weight_min": -12.592839241027832,
      "activations/layer10_attention_weight_max": 36.41997146606445,
      "activations/layer10_attention_weight_min": -34.482147216796875,
      "activations/layer11_attention_weight_max": 36.24531555175781,
      "activations/layer11_attention_weight_min": -33.748313903808594,
      "activations/layer12_attention_weight_max": 27.847389221191406,
      "activations/layer12_attention_weight_min": -25.919551849365234,
      "activations/layer13_attention_weight_max": 43.159645080566406,
      "activations/layer13_attention_weight_min": -35.063106536865234,
      "activations/layer14_attention_weight_max": 43.28017044067383,
      "activations/layer14_attention_weight_min": -36.41960906982422,
      "activations/layer15_attention_weight_max": 39.6059455871582,
      "activations/layer15_attention_weight_min": -35.32830810546875,
      "activations/layer16_attention_weight_max": 32.50532913208008,
      "activations/layer16_attention_weight_min": -28.8428955078125,
      "activations/layer17_attention_weight_max": 34.47466278076172,
      "activations/layer17_attention_weight_min": -28.307815551757812,
      "activations/layer18_attention_weight_max": 34.7291259765625,
      "activations/layer18_attention_weight_min": -25.042621612548828,
      "activations/layer19_attention_weight_max": 38.88333511352539,
      "activations/layer19_attention_weight_min": -31.01725196838379,
      "activations/layer1_attention_weight_max": 16.413089752197266,
      "activations/layer1_attention_weight_min": -14.425738334655762,
      "activations/layer20_attention_weight_max": 33.02225112915039,
      "activations/layer20_attention_weight_min": -27.813730239868164,
      "activations/layer21_attention_weight_max": 35.134735107421875,
      "activations/layer21_attention_weight_min": -24.943614959716797,
      "activations/layer22_attention_weight_max": 48.516666412353516,
      "activations/layer22_attention_weight_min": -32.134239196777344,
      "activations/layer23_attention_weight_max": 38.05236053466797,
      "activations/layer23_attention_weight_min": -26.426288604736328,
      "activations/layer2_attention_weight_max": 35.41938781738281,
      "activations/layer2_attention_weight_min": -34.291351318359375,
      "activations/layer3_attention_weight_max": 105.59852600097656,
      "activations/layer3_attention_weight_min": -101.01854705810547,
      "activations/layer4_attention_weight_max": 117.11763763427734,
      "activations/layer4_attention_weight_min": -115.7628402709961,
      "activations/layer5_attention_weight_max": 53.78459930419922,
      "activations/layer5_attention_weight_min": -61.28138732910156,
      "activations/layer6_attention_weight_max": 46.072837829589844,
      "activations/layer6_attention_weight_min": -48.68657302856445,
      "activations/layer7_attention_weight_max": 98.71980285644531,
      "activations/layer7_attention_weight_min": -97.89696502685547,
      "activations/layer8_attention_weight_max": 44.42426300048828,
      "activations/layer8_attention_weight_min": -43.361209869384766,
      "activations/layer9_attention_weight_max": 35.92708969116211,
      "activations/layer9_attention_weight_min": -35.37388229370117,
      "epoch": 13.41,
      "learning_rate": 6.418219696969696e-05,
      "loss": 2.7678,
      "step": 230750
    },
    {
      "activations/layer0_attention_weight_max": 19.024187088012695,
      "activations/layer0_attention_weight_min": -12.885348320007324,
      "activations/layer10_attention_weight_max": 33.37421417236328,
      "activations/layer10_attention_weight_min": -34.68927001953125,
      "activations/layer11_attention_weight_max": 32.55867385864258,
      "activations/layer11_attention_weight_min": -32.84012222290039,
      "activations/layer12_attention_weight_max": 28.36109161376953,
      "activations/layer12_attention_weight_min": -29.766841888427734,
      "activations/layer13_attention_weight_max": 40.917701721191406,
      "activations/layer13_attention_weight_min": -37.62732696533203,
      "activations/layer14_attention_weight_max": 43.685447692871094,
      "activations/layer14_attention_weight_min": -38.93931579589844,
      "activations/layer15_attention_weight_max": 42.15011978149414,
      "activations/layer15_attention_weight_min": -32.665687561035156,
      "activations/layer16_attention_weight_max": 39.521297454833984,
      "activations/layer16_attention_weight_min": -26.797128677368164,
      "activations/layer17_attention_weight_max": 32.60661315917969,
      "activations/layer17_attention_weight_min": -24.519672393798828,
      "activations/layer18_attention_weight_max": 33.31889724731445,
      "activations/layer18_attention_weight_min": -25.666683197021484,
      "activations/layer19_attention_weight_max": 36.644187927246094,
      "activations/layer19_attention_weight_min": -30.128599166870117,
      "activations/layer1_attention_weight_max": 17.011350631713867,
      "activations/layer1_attention_weight_min": -14.596290588378906,
      "activations/layer20_attention_weight_max": 32.49054718017578,
      "activations/layer20_attention_weight_min": -25.736509323120117,
      "activations/layer21_attention_weight_max": 33.66975402832031,
      "activations/layer21_attention_weight_min": -24.534250259399414,
      "activations/layer22_attention_weight_max": 53.40886688232422,
      "activations/layer22_attention_weight_min": -31.04427146911621,
      "activations/layer23_attention_weight_max": 38.56941604614258,
      "activations/layer23_attention_weight_min": -26.175460815429688,
      "activations/layer2_attention_weight_max": 34.348182678222656,
      "activations/layer2_attention_weight_min": -34.02759552001953,
      "activations/layer3_attention_weight_max": 95.55425262451172,
      "activations/layer3_attention_weight_min": -98.25724029541016,
      "activations/layer4_attention_weight_max": 106.96053314208984,
      "activations/layer4_attention_weight_min": -108.50322723388672,
      "activations/layer5_attention_weight_max": 49.589759826660156,
      "activations/layer5_attention_weight_min": -60.53266525268555,
      "activations/layer6_attention_weight_max": 41.67775344848633,
      "activations/layer6_attention_weight_min": -47.376827239990234,
      "activations/layer7_attention_weight_max": 90.90338134765625,
      "activations/layer7_attention_weight_min": -92.28768920898438,
      "activations/layer8_attention_weight_max": 41.61730194091797,
      "activations/layer8_attention_weight_min": -43.097076416015625,
      "activations/layer9_attention_weight_max": 33.69596481323242,
      "activations/layer9_attention_weight_min": -34.07945251464844,
      "epoch": 13.41,
      "learning_rate": 6.416325757575757e-05,
      "loss": 2.7582,
      "step": 230800
    },
    {
      "activations/layer0_attention_weight_max": 16.553983688354492,
      "activations/layer0_attention_weight_min": -13.013429641723633,
      "activations/layer10_attention_weight_max": 32.58646011352539,
      "activations/layer10_attention_weight_min": -34.110931396484375,
      "activations/layer11_attention_weight_max": 32.26096725463867,
      "activations/layer11_attention_weight_min": -32.677825927734375,
      "activations/layer12_attention_weight_max": 26.478219985961914,
      "activations/layer12_attention_weight_min": -25.979854583740234,
      "activations/layer13_attention_weight_max": 40.29883575439453,
      "activations/layer13_attention_weight_min": -34.862911224365234,
      "activations/layer14_attention_weight_max": 46.17654037475586,
      "activations/layer14_attention_weight_min": -37.22957992553711,
      "activations/layer15_attention_weight_max": 38.39196014404297,
      "activations/layer15_attention_weight_min": -34.878902435302734,
      "activations/layer16_attention_weight_max": 32.96860122680664,
      "activations/layer16_attention_weight_min": -26.77604866027832,
      "activations/layer17_attention_weight_max": 34.74384689331055,
      "activations/layer17_attention_weight_min": -25.615110397338867,
      "activations/layer18_attention_weight_max": 32.04419708251953,
      "activations/layer18_attention_weight_min": -23.533910751342773,
      "activations/layer19_attention_weight_max": 36.42228698730469,
      "activations/layer19_attention_weight_min": -30.69284439086914,
      "activations/layer1_attention_weight_max": 16.80613136291504,
      "activations/layer1_attention_weight_min": -14.339131355285645,
      "activations/layer20_attention_weight_max": 30.054330825805664,
      "activations/layer20_attention_weight_min": -23.878740310668945,
      "activations/layer21_attention_weight_max": 28.90673065185547,
      "activations/layer21_attention_weight_min": -24.35521125793457,
      "activations/layer22_attention_weight_max": 52.60736083984375,
      "activations/layer22_attention_weight_min": -29.27752685546875,
      "activations/layer23_attention_weight_max": 34.8930778503418,
      "activations/layer23_attention_weight_min": -24.76422882080078,
      "activations/layer2_attention_weight_max": 32.94659423828125,
      "activations/layer2_attention_weight_min": -35.071502685546875,
      "activations/layer3_attention_weight_max": 98.2840805053711,
      "activations/layer3_attention_weight_min": -102.67721557617188,
      "activations/layer4_attention_weight_max": 114.3023681640625,
      "activations/layer4_attention_weight_min": -116.54027557373047,
      "activations/layer5_attention_weight_max": 52.30272674560547,
      "activations/layer5_attention_weight_min": -61.9688835144043,
      "activations/layer6_attention_weight_max": 45.12970733642578,
      "activations/layer6_attention_weight_min": -47.82289123535156,
      "activations/layer7_attention_weight_max": 92.05179595947266,
      "activations/layer7_attention_weight_min": -95.30793762207031,
      "activations/layer8_attention_weight_max": 40.51625442504883,
      "activations/layer8_attention_weight_min": -44.288185119628906,
      "activations/layer9_attention_weight_max": 33.422794342041016,
      "activations/layer9_attention_weight_min": -35.83297348022461,
      "epoch": 13.41,
      "learning_rate": 6.414431818181817e-05,
      "loss": 2.7554,
      "step": 230850
    },
    {
      "activations/layer0_attention_weight_max": 16.509485244750977,
      "activations/layer0_attention_weight_min": -12.070779800415039,
      "activations/layer10_attention_weight_max": 31.695615768432617,
      "activations/layer10_attention_weight_min": -33.3552131652832,
      "activations/layer11_attention_weight_max": 31.34033966064453,
      "activations/layer11_attention_weight_min": -33.83000946044922,
      "activations/layer12_attention_weight_max": 24.877609252929688,
      "activations/layer12_attention_weight_min": -25.19200325012207,
      "activations/layer13_attention_weight_max": 36.056121826171875,
      "activations/layer13_attention_weight_min": -36.607200622558594,
      "activations/layer14_attention_weight_max": 39.5504264831543,
      "activations/layer14_attention_weight_min": -37.340579986572266,
      "activations/layer15_attention_weight_max": 35.33588409423828,
      "activations/layer15_attention_weight_min": -32.919593811035156,
      "activations/layer16_attention_weight_max": 26.322664260864258,
      "activations/layer16_attention_weight_min": -28.59897232055664,
      "activations/layer17_attention_weight_max": 27.656044006347656,
      "activations/layer17_attention_weight_min": -26.919050216674805,
      "activations/layer18_attention_weight_max": 27.84650230407715,
      "activations/layer18_attention_weight_min": -23.57603645324707,
      "activations/layer19_attention_weight_max": 30.630401611328125,
      "activations/layer19_attention_weight_min": -31.69283103942871,
      "activations/layer1_attention_weight_max": 16.098609924316406,
      "activations/layer1_attention_weight_min": -15.617914199829102,
      "activations/layer20_attention_weight_max": 26.86468505859375,
      "activations/layer20_attention_weight_min": -24.963457107543945,
      "activations/layer21_attention_weight_max": 26.966033935546875,
      "activations/layer21_attention_weight_min": -23.97757339477539,
      "activations/layer22_attention_weight_max": 37.642757415771484,
      "activations/layer22_attention_weight_min": -30.91769790649414,
      "activations/layer23_attention_weight_max": 30.433238983154297,
      "activations/layer23_attention_weight_min": -25.726871490478516,
      "activations/layer2_attention_weight_max": 33.37042999267578,
      "activations/layer2_attention_weight_min": -33.788753509521484,
      "activations/layer3_attention_weight_max": 97.90921020507812,
      "activations/layer3_attention_weight_min": -98.31981658935547,
      "activations/layer4_attention_weight_max": 113.8442153930664,
      "activations/layer4_attention_weight_min": -111.1611328125,
      "activations/layer5_attention_weight_max": 53.71186065673828,
      "activations/layer5_attention_weight_min": -61.59428024291992,
      "activations/layer6_attention_weight_max": 43.67292022705078,
      "activations/layer6_attention_weight_min": -46.85187530517578,
      "activations/layer7_attention_weight_max": 88.40017700195312,
      "activations/layer7_attention_weight_min": -93.13026428222656,
      "activations/layer8_attention_weight_max": 38.47731399536133,
      "activations/layer8_attention_weight_min": -42.53255844116211,
      "activations/layer9_attention_weight_max": 31.695785522460938,
      "activations/layer9_attention_weight_min": -33.802452087402344,
      "epoch": 13.42,
      "learning_rate": 6.412537878787878e-05,
      "loss": 2.7773,
      "step": 230900
    },
    {
      "activations/layer0_attention_weight_max": 16.3243350982666,
      "activations/layer0_attention_weight_min": -12.396065711975098,
      "activations/layer10_attention_weight_max": 34.80541229248047,
      "activations/layer10_attention_weight_min": -32.152687072753906,
      "activations/layer11_attention_weight_max": 34.740333557128906,
      "activations/layer11_attention_weight_min": -32.60707473754883,
      "activations/layer12_attention_weight_max": 27.169904708862305,
      "activations/layer12_attention_weight_min": -25.064573287963867,
      "activations/layer13_attention_weight_max": 38.9024658203125,
      "activations/layer13_attention_weight_min": -33.8565673828125,
      "activations/layer14_attention_weight_max": 44.971771240234375,
      "activations/layer14_attention_weight_min": -37.961326599121094,
      "activations/layer15_attention_weight_max": 37.85173797607422,
      "activations/layer15_attention_weight_min": -31.83013916015625,
      "activations/layer16_attention_weight_max": 31.89654541015625,
      "activations/layer16_attention_weight_min": -28.775609970092773,
      "activations/layer17_attention_weight_max": 33.7486572265625,
      "activations/layer17_attention_weight_min": -25.60487174987793,
      "activations/layer18_attention_weight_max": 31.421438217163086,
      "activations/layer18_attention_weight_min": -23.092376708984375,
      "activations/layer19_attention_weight_max": 36.05498123168945,
      "activations/layer19_attention_weight_min": -29.538284301757812,
      "activations/layer1_attention_weight_max": 16.09247398376465,
      "activations/layer1_attention_weight_min": -14.5517578125,
      "activations/layer20_attention_weight_max": 29.4886474609375,
      "activations/layer20_attention_weight_min": -25.46166229248047,
      "activations/layer21_attention_weight_max": 29.505149841308594,
      "activations/layer21_attention_weight_min": -25.061168670654297,
      "activations/layer22_attention_weight_max": 45.26311111450195,
      "activations/layer22_attention_weight_min": -32.067440032958984,
      "activations/layer23_attention_weight_max": 37.12667465209961,
      "activations/layer23_attention_weight_min": -27.506576538085938,
      "activations/layer2_attention_weight_max": 33.012088775634766,
      "activations/layer2_attention_weight_min": -33.31854248046875,
      "activations/layer3_attention_weight_max": 95.75727081298828,
      "activations/layer3_attention_weight_min": -99.44461059570312,
      "activations/layer4_attention_weight_max": 109.76708984375,
      "activations/layer4_attention_weight_min": -106.0328598022461,
      "activations/layer5_attention_weight_max": 50.082984924316406,
      "activations/layer5_attention_weight_min": -61.30785369873047,
      "activations/layer6_attention_weight_max": 42.48229217529297,
      "activations/layer6_attention_weight_min": -44.706111907958984,
      "activations/layer7_attention_weight_max": 88.9745864868164,
      "activations/layer7_attention_weight_min": -85.40790557861328,
      "activations/layer8_attention_weight_max": 38.448570251464844,
      "activations/layer8_attention_weight_min": -39.68578338623047,
      "activations/layer9_attention_weight_max": 32.7763671875,
      "activations/layer9_attention_weight_min": -32.43384552001953,
      "epoch": 13.42,
      "learning_rate": 6.410643939393939e-05,
      "loss": 2.7503,
      "step": 230950
    },
    {
      "activations/layer0_attention_weight_max": 14.296372413635254,
      "activations/layer0_attention_weight_min": -12.387998580932617,
      "activations/layer10_attention_weight_max": 36.46100616455078,
      "activations/layer10_attention_weight_min": -35.082794189453125,
      "activations/layer11_attention_weight_max": 38.064353942871094,
      "activations/layer11_attention_weight_min": -34.385047912597656,
      "activations/layer12_attention_weight_max": 38.26966094970703,
      "activations/layer12_attention_weight_min": -27.85564613342285,
      "activations/layer13_attention_weight_max": 48.79306411743164,
      "activations/layer13_attention_weight_min": -37.64905548095703,
      "activations/layer14_attention_weight_max": 47.834930419921875,
      "activations/layer14_attention_weight_min": -41.624412536621094,
      "activations/layer15_attention_weight_max": 41.9409294128418,
      "activations/layer15_attention_weight_min": -35.17766189575195,
      "activations/layer16_attention_weight_max": 33.63642120361328,
      "activations/layer16_attention_weight_min": -28.189624786376953,
      "activations/layer17_attention_weight_max": 36.36104965209961,
      "activations/layer17_attention_weight_min": -26.405330657958984,
      "activations/layer18_attention_weight_max": 32.01777648925781,
      "activations/layer18_attention_weight_min": -26.28512191772461,
      "activations/layer19_attention_weight_max": 35.103485107421875,
      "activations/layer19_attention_weight_min": -32.095149993896484,
      "activations/layer1_attention_weight_max": 17.032363891601562,
      "activations/layer1_attention_weight_min": -15.279806137084961,
      "activations/layer20_attention_weight_max": 31.414794921875,
      "activations/layer20_attention_weight_min": -26.64098358154297,
      "activations/layer21_attention_weight_max": 32.422401428222656,
      "activations/layer21_attention_weight_min": -26.051115036010742,
      "activations/layer22_attention_weight_max": 53.229286193847656,
      "activations/layer22_attention_weight_min": -32.48810577392578,
      "activations/layer23_attention_weight_max": 36.5142707824707,
      "activations/layer23_attention_weight_min": -25.709693908691406,
      "activations/layer2_attention_weight_max": 35.42390060424805,
      "activations/layer2_attention_weight_min": -36.65690994262695,
      "activations/layer3_attention_weight_max": 104.95823669433594,
      "activations/layer3_attention_weight_min": -106.14869689941406,
      "activations/layer4_attention_weight_max": 122.20953369140625,
      "activations/layer4_attention_weight_min": -117.59553527832031,
      "activations/layer5_attention_weight_max": 55.94025421142578,
      "activations/layer5_attention_weight_min": -63.583656311035156,
      "activations/layer6_attention_weight_max": 47.671791076660156,
      "activations/layer6_attention_weight_min": -47.210994720458984,
      "activations/layer7_attention_weight_max": 99.30056762695312,
      "activations/layer7_attention_weight_min": -96.71047973632812,
      "activations/layer8_attention_weight_max": 43.91963577270508,
      "activations/layer8_attention_weight_min": -43.36508560180664,
      "activations/layer9_attention_weight_max": 37.9802131652832,
      "activations/layer9_attention_weight_min": -35.79873275756836,
      "epoch": 13.42,
      "learning_rate": 6.408749999999999e-05,
      "loss": 2.7737,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.441,
      "eval_samples_per_second": 508.71,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.441,
      "eval_openwebtext_samples_per_second": 508.71,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9275,
      "eval_wikitext_samples_per_second": 236.574,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_lambada_loss": 2.548828125,
      "eval_lambada_ppl": 12.792104248423877,
      "eval_lambada_runtime": 9.5227,
      "eval_lambada_samples_per_second": 511.303,
      "step": 231000
    },
    {
      "activations/layer0_attention_weight_max": 16.050413131713867,
      "activations/layer0_attention_weight_min": -11.97140884399414,
      "activations/layer10_attention_weight_max": 35.24629592895508,
      "activations/layer10_attention_weight_min": -34.821327209472656,
      "activations/layer11_attention_weight_max": 34.313133239746094,
      "activations/layer11_attention_weight_min": -34.01228332519531,
      "activations/layer12_attention_weight_max": 29.258785247802734,
      "activations/layer12_attention_weight_min": -26.98908042907715,
      "activations/layer13_attention_weight_max": 44.267696380615234,
      "activations/layer13_attention_weight_min": -37.89869689941406,
      "activations/layer14_attention_weight_max": 48.460052490234375,
      "activations/layer14_attention_weight_min": -41.045894622802734,
      "activations/layer15_attention_weight_max": 41.14567184448242,
      "activations/layer15_attention_weight_min": -34.52278518676758,
      "activations/layer16_attention_weight_max": 33.363304138183594,
      "activations/layer16_attention_weight_min": -27.914371490478516,
      "activations/layer17_attention_weight_max": 33.66272735595703,
      "activations/layer17_attention_weight_min": -27.18836784362793,
      "activations/layer18_attention_weight_max": 33.101409912109375,
      "activations/layer18_attention_weight_min": -23.468013763427734,
      "activations/layer19_attention_weight_max": 38.77213668823242,
      "activations/layer19_attention_weight_min": -29.871145248413086,
      "activations/layer1_attention_weight_max": 16.640188217163086,
      "activations/layer1_attention_weight_min": -14.163016319274902,
      "activations/layer20_attention_weight_max": 34.353946685791016,
      "activations/layer20_attention_weight_min": -24.03436851501465,
      "activations/layer21_attention_weight_max": 34.380279541015625,
      "activations/layer21_attention_weight_min": -24.134124755859375,
      "activations/layer22_attention_weight_max": 53.44510269165039,
      "activations/layer22_attention_weight_min": -30.35076904296875,
      "activations/layer23_attention_weight_max": 36.611656188964844,
      "activations/layer23_attention_weight_min": -24.888385772705078,
      "activations/layer2_attention_weight_max": 34.864471435546875,
      "activations/layer2_attention_weight_min": -35.12163543701172,
      "activations/layer3_attention_weight_max": 99.45681762695312,
      "activations/layer3_attention_weight_min": -102.04930114746094,
      "activations/layer4_attention_weight_max": 116.49371337890625,
      "activations/layer4_attention_weight_min": -117.1422348022461,
      "activations/layer5_attention_weight_max": 53.65465545654297,
      "activations/layer5_attention_weight_min": -62.25946044921875,
      "activations/layer6_attention_weight_max": 46.558475494384766,
      "activations/layer6_attention_weight_min": -50.346092224121094,
      "activations/layer7_attention_weight_max": 101.27702331542969,
      "activations/layer7_attention_weight_min": -100.07879638671875,
      "activations/layer8_attention_weight_max": 44.02242660522461,
      "activations/layer8_attention_weight_min": -44.64625549316406,
      "activations/layer9_attention_weight_max": 35.85773849487305,
      "activations/layer9_attention_weight_min": -35.330657958984375,
      "epoch": 13.43,
      "learning_rate": 6.406856060606061e-05,
      "loss": 2.7687,
      "step": 231050
    },
    {
      "activations/layer0_attention_weight_max": 16.62000274658203,
      "activations/layer0_attention_weight_min": -12.858558654785156,
      "activations/layer10_attention_weight_max": 37.01655960083008,
      "activations/layer10_attention_weight_min": -35.50157928466797,
      "activations/layer11_attention_weight_max": 39.041717529296875,
      "activations/layer11_attention_weight_min": -36.527469635009766,
      "activations/layer12_attention_weight_max": 29.867734909057617,
      "activations/layer12_attention_weight_min": -28.37425422668457,
      "activations/layer13_attention_weight_max": 44.86567687988281,
      "activations/layer13_attention_weight_min": -37.05916213989258,
      "activations/layer14_attention_weight_max": 55.27555847167969,
      "activations/layer14_attention_weight_min": -44.49102783203125,
      "activations/layer15_attention_weight_max": 50.11122131347656,
      "activations/layer15_attention_weight_min": -40.51188659667969,
      "activations/layer16_attention_weight_max": 39.32260513305664,
      "activations/layer16_attention_weight_min": -26.492753982543945,
      "activations/layer17_attention_weight_max": 38.72325897216797,
      "activations/layer17_attention_weight_min": -27.064661026000977,
      "activations/layer18_attention_weight_max": 35.815948486328125,
      "activations/layer18_attention_weight_min": -25.972179412841797,
      "activations/layer19_attention_weight_max": 42.10563278198242,
      "activations/layer19_attention_weight_min": -30.690458297729492,
      "activations/layer1_attention_weight_max": 16.959062576293945,
      "activations/layer1_attention_weight_min": -14.842695236206055,
      "activations/layer20_attention_weight_max": 33.31122589111328,
      "activations/layer20_attention_weight_min": -24.84316635131836,
      "activations/layer21_attention_weight_max": 31.45688247680664,
      "activations/layer21_attention_weight_min": -23.156618118286133,
      "activations/layer22_attention_weight_max": 46.71354293823242,
      "activations/layer22_attention_weight_min": -31.61344337463379,
      "activations/layer23_attention_weight_max": 34.26713180541992,
      "activations/layer23_attention_weight_min": -24.71588134765625,
      "activations/layer2_attention_weight_max": 39.14533996582031,
      "activations/layer2_attention_weight_min": -36.359718322753906,
      "activations/layer3_attention_weight_max": 101.93802642822266,
      "activations/layer3_attention_weight_min": -102.79607391357422,
      "activations/layer4_attention_weight_max": 111.72566223144531,
      "activations/layer4_attention_weight_min": -116.4081039428711,
      "activations/layer5_attention_weight_max": 52.523555755615234,
      "activations/layer5_attention_weight_min": -64.02967834472656,
      "activations/layer6_attention_weight_max": 42.53273010253906,
      "activations/layer6_attention_weight_min": -46.827720642089844,
      "activations/layer7_attention_weight_max": 89.9551010131836,
      "activations/layer7_attention_weight_min": -90.14663696289062,
      "activations/layer8_attention_weight_max": 41.5198860168457,
      "activations/layer8_attention_weight_min": -41.340431213378906,
      "activations/layer9_attention_weight_max": 33.827816009521484,
      "activations/layer9_attention_weight_min": -33.55965805053711,
      "epoch": 13.43,
      "learning_rate": 6.404962121212121e-05,
      "loss": 2.7788,
      "step": 231100
    },
    {
      "activations/layer0_attention_weight_max": 15.933455467224121,
      "activations/layer0_attention_weight_min": -13.050555229187012,
      "activations/layer10_attention_weight_max": 35.5899543762207,
      "activations/layer10_attention_weight_min": -34.19093704223633,
      "activations/layer11_attention_weight_max": 37.04711151123047,
      "activations/layer11_attention_weight_min": -36.52986526489258,
      "activations/layer12_attention_weight_max": 27.328208923339844,
      "activations/layer12_attention_weight_min": -27.040143966674805,
      "activations/layer13_attention_weight_max": 38.13112258911133,
      "activations/layer13_attention_weight_min": -34.1512565612793,
      "activations/layer14_attention_weight_max": 52.30278778076172,
      "activations/layer14_attention_weight_min": -36.74161148071289,
      "activations/layer15_attention_weight_max": 38.86119079589844,
      "activations/layer15_attention_weight_min": -35.85884094238281,
      "activations/layer16_attention_weight_max": 33.888511657714844,
      "activations/layer16_attention_weight_min": -29.698421478271484,
      "activations/layer17_attention_weight_max": 32.50129699707031,
      "activations/layer17_attention_weight_min": -29.49742317199707,
      "activations/layer18_attention_weight_max": 29.2468204498291,
      "activations/layer18_attention_weight_min": -28.350595474243164,
      "activations/layer19_attention_weight_max": 36.8731575012207,
      "activations/layer19_attention_weight_min": -32.01057434082031,
      "activations/layer1_attention_weight_max": 16.82217025756836,
      "activations/layer1_attention_weight_min": -16.06662940979004,
      "activations/layer20_attention_weight_max": 33.44412612915039,
      "activations/layer20_attention_weight_min": -27.46346092224121,
      "activations/layer21_attention_weight_max": 31.967729568481445,
      "activations/layer21_attention_weight_min": -25.9415340423584,
      "activations/layer22_attention_weight_max": 42.148834228515625,
      "activations/layer22_attention_weight_min": -33.8351936340332,
      "activations/layer23_attention_weight_max": 35.213539123535156,
      "activations/layer23_attention_weight_min": -26.04425621032715,
      "activations/layer2_attention_weight_max": 34.71849060058594,
      "activations/layer2_attention_weight_min": -35.52699661254883,
      "activations/layer3_attention_weight_max": 99.96768188476562,
      "activations/layer3_attention_weight_min": -102.0091323852539,
      "activations/layer4_attention_weight_max": 111.1785888671875,
      "activations/layer4_attention_weight_min": -112.09059143066406,
      "activations/layer5_attention_weight_max": 52.82099533081055,
      "activations/layer5_attention_weight_min": -60.90681457519531,
      "activations/layer6_attention_weight_max": 45.2565803527832,
      "activations/layer6_attention_weight_min": -46.99543380737305,
      "activations/layer7_attention_weight_max": 91.06671905517578,
      "activations/layer7_attention_weight_min": -99.83080291748047,
      "activations/layer8_attention_weight_max": 42.37217330932617,
      "activations/layer8_attention_weight_min": -43.4795036315918,
      "activations/layer9_attention_weight_max": 34.108062744140625,
      "activations/layer9_attention_weight_min": -35.84598922729492,
      "epoch": 13.43,
      "learning_rate": 6.403068181818181e-05,
      "loss": 2.7681,
      "step": 231150
    },
    {
      "activations/layer0_attention_weight_max": 15.116949081420898,
      "activations/layer0_attention_weight_min": -12.251518249511719,
      "activations/layer10_attention_weight_max": 37.67748260498047,
      "activations/layer10_attention_weight_min": -34.82556915283203,
      "activations/layer11_attention_weight_max": 38.39715576171875,
      "activations/layer11_attention_weight_min": -37.004249572753906,
      "activations/layer12_attention_weight_max": 28.477479934692383,
      "activations/layer12_attention_weight_min": -27.31661605834961,
      "activations/layer13_attention_weight_max": 37.89897155761719,
      "activations/layer13_attention_weight_min": -36.683074951171875,
      "activations/layer14_attention_weight_max": 43.85475158691406,
      "activations/layer14_attention_weight_min": -40.37773132324219,
      "activations/layer15_attention_weight_max": 40.55628204345703,
      "activations/layer15_attention_weight_min": -38.095855712890625,
      "activations/layer16_attention_weight_max": 33.08684539794922,
      "activations/layer16_attention_weight_min": -31.397798538208008,
      "activations/layer17_attention_weight_max": 29.70462989807129,
      "activations/layer17_attention_weight_min": -28.841508865356445,
      "activations/layer18_attention_weight_max": 30.629314422607422,
      "activations/layer18_attention_weight_min": -25.954898834228516,
      "activations/layer19_attention_weight_max": 39.073577880859375,
      "activations/layer19_attention_weight_min": -34.67499542236328,
      "activations/layer1_attention_weight_max": 17.0459041595459,
      "activations/layer1_attention_weight_min": -14.927032470703125,
      "activations/layer20_attention_weight_max": 31.348941802978516,
      "activations/layer20_attention_weight_min": -25.847875595092773,
      "activations/layer21_attention_weight_max": 30.798095703125,
      "activations/layer21_attention_weight_min": -24.477340698242188,
      "activations/layer22_attention_weight_max": 42.9969367980957,
      "activations/layer22_attention_weight_min": -31.87970733642578,
      "activations/layer23_attention_weight_max": 34.73382568359375,
      "activations/layer23_attention_weight_min": -27.250324249267578,
      "activations/layer2_attention_weight_max": 34.48033905029297,
      "activations/layer2_attention_weight_min": -33.976566314697266,
      "activations/layer3_attention_weight_max": 103.64110565185547,
      "activations/layer3_attention_weight_min": -105.17980194091797,
      "activations/layer4_attention_weight_max": 113.21046447753906,
      "activations/layer4_attention_weight_min": -111.73109436035156,
      "activations/layer5_attention_weight_max": 52.70146179199219,
      "activations/layer5_attention_weight_min": -65.19281768798828,
      "activations/layer6_attention_weight_max": 44.180625915527344,
      "activations/layer6_attention_weight_min": -48.13072967529297,
      "activations/layer7_attention_weight_max": 107.59263610839844,
      "activations/layer7_attention_weight_min": -95.75199127197266,
      "activations/layer8_attention_weight_max": 43.05284881591797,
      "activations/layer8_attention_weight_min": -43.77372360229492,
      "activations/layer9_attention_weight_max": 34.904937744140625,
      "activations/layer9_attention_weight_min": -35.073814392089844,
      "epoch": 13.43,
      "learning_rate": 6.401174242424243e-05,
      "loss": 2.7717,
      "step": 231200
    },
    {
      "activations/layer0_attention_weight_max": 15.548686027526855,
      "activations/layer0_attention_weight_min": -12.66390609741211,
      "activations/layer10_attention_weight_max": 32.9986572265625,
      "activations/layer10_attention_weight_min": -31.5794734954834,
      "activations/layer11_attention_weight_max": 35.028011322021484,
      "activations/layer11_attention_weight_min": -32.209991455078125,
      "activations/layer12_attention_weight_max": 25.56366729736328,
      "activations/layer12_attention_weight_min": -27.009035110473633,
      "activations/layer13_attention_weight_max": 41.09099197387695,
      "activations/layer13_attention_weight_min": -33.032012939453125,
      "activations/layer14_attention_weight_max": 45.64266586303711,
      "activations/layer14_attention_weight_min": -36.285552978515625,
      "activations/layer15_attention_weight_max": 41.04842758178711,
      "activations/layer15_attention_weight_min": -32.807247161865234,
      "activations/layer16_attention_weight_max": 29.76775360107422,
      "activations/layer16_attention_weight_min": -26.50461769104004,
      "activations/layer17_attention_weight_max": 30.34305763244629,
      "activations/layer17_attention_weight_min": -24.425962448120117,
      "activations/layer18_attention_weight_max": 30.582429885864258,
      "activations/layer18_attention_weight_min": -21.67253303527832,
      "activations/layer19_attention_weight_max": 34.952022552490234,
      "activations/layer19_attention_weight_min": -27.287479400634766,
      "activations/layer1_attention_weight_max": 17.510822296142578,
      "activations/layer1_attention_weight_min": -18.505672454833984,
      "activations/layer20_attention_weight_max": 32.12618637084961,
      "activations/layer20_attention_weight_min": -22.72657585144043,
      "activations/layer21_attention_weight_max": 32.13670349121094,
      "activations/layer21_attention_weight_min": -21.86731719970703,
      "activations/layer22_attention_weight_max": 39.48846435546875,
      "activations/layer22_attention_weight_min": -27.271183013916016,
      "activations/layer23_attention_weight_max": 32.42630386352539,
      "activations/layer23_attention_weight_min": -22.765460968017578,
      "activations/layer2_attention_weight_max": 34.415950775146484,
      "activations/layer2_attention_weight_min": -36.26605987548828,
      "activations/layer3_attention_weight_max": 101.1868667602539,
      "activations/layer3_attention_weight_min": -104.11093139648438,
      "activations/layer4_attention_weight_max": 109.2954330444336,
      "activations/layer4_attention_weight_min": -110.43636322021484,
      "activations/layer5_attention_weight_max": 52.273555755615234,
      "activations/layer5_attention_weight_min": -62.90435791015625,
      "activations/layer6_attention_weight_max": 44.26128387451172,
      "activations/layer6_attention_weight_min": -48.09116744995117,
      "activations/layer7_attention_weight_max": 90.2361068725586,
      "activations/layer7_attention_weight_min": -98.95553588867188,
      "activations/layer8_attention_weight_max": 42.17942810058594,
      "activations/layer8_attention_weight_min": -43.036415100097656,
      "activations/layer9_attention_weight_max": 34.473670959472656,
      "activations/layer9_attention_weight_min": -33.567787170410156,
      "epoch": 13.44,
      "learning_rate": 6.399280303030303e-05,
      "loss": 2.7495,
      "step": 231250
    },
    {
      "activations/layer0_attention_weight_max": 15.615056991577148,
      "activations/layer0_attention_weight_min": -11.912690162658691,
      "activations/layer10_attention_weight_max": 43.80612564086914,
      "activations/layer10_attention_weight_min": -43.30361557006836,
      "activations/layer11_attention_weight_max": 46.40050506591797,
      "activations/layer11_attention_weight_min": -43.383522033691406,
      "activations/layer12_attention_weight_max": 34.7109489440918,
      "activations/layer12_attention_weight_min": -27.171236038208008,
      "activations/layer13_attention_weight_max": 42.40509796142578,
      "activations/layer13_attention_weight_min": -34.86503601074219,
      "activations/layer14_attention_weight_max": 41.8883171081543,
      "activations/layer14_attention_weight_min": -36.33083724975586,
      "activations/layer15_attention_weight_max": 44.0093879699707,
      "activations/layer15_attention_weight_min": -35.42555618286133,
      "activations/layer16_attention_weight_max": 32.92680358886719,
      "activations/layer16_attention_weight_min": -28.61806297302246,
      "activations/layer17_attention_weight_max": 29.94866371154785,
      "activations/layer17_attention_weight_min": -26.187509536743164,
      "activations/layer18_attention_weight_max": 29.492652893066406,
      "activations/layer18_attention_weight_min": -23.22124671936035,
      "activations/layer19_attention_weight_max": 33.56667709350586,
      "activations/layer19_attention_weight_min": -31.01299476623535,
      "activations/layer1_attention_weight_max": 16.700510025024414,
      "activations/layer1_attention_weight_min": -14.087881088256836,
      "activations/layer20_attention_weight_max": 30.628883361816406,
      "activations/layer20_attention_weight_min": -26.21623992919922,
      "activations/layer21_attention_weight_max": 30.918691635131836,
      "activations/layer21_attention_weight_min": -24.091405868530273,
      "activations/layer22_attention_weight_max": 46.110435485839844,
      "activations/layer22_attention_weight_min": -29.00889778137207,
      "activations/layer23_attention_weight_max": 34.227516174316406,
      "activations/layer23_attention_weight_min": -24.28663444519043,
      "activations/layer2_attention_weight_max": 37.87315368652344,
      "activations/layer2_attention_weight_min": -35.385040283203125,
      "activations/layer3_attention_weight_max": 106.41165161132812,
      "activations/layer3_attention_weight_min": -107.96822357177734,
      "activations/layer4_attention_weight_max": 120.15157318115234,
      "activations/layer4_attention_weight_min": -112.556396484375,
      "activations/layer5_attention_weight_max": 62.30370330810547,
      "activations/layer5_attention_weight_min": -60.34974670410156,
      "activations/layer6_attention_weight_max": 55.1168327331543,
      "activations/layer6_attention_weight_min": -53.70675277709961,
      "activations/layer7_attention_weight_max": 119.53099060058594,
      "activations/layer7_attention_weight_min": -111.51495361328125,
      "activations/layer8_attention_weight_max": 55.49317169189453,
      "activations/layer8_attention_weight_min": -54.058135986328125,
      "activations/layer9_attention_weight_max": 44.74235153198242,
      "activations/layer9_attention_weight_min": -46.27956008911133,
      "epoch": 13.44,
      "learning_rate": 6.397386363636363e-05,
      "loss": 2.7612,
      "step": 231300
    },
    {
      "activations/layer0_attention_weight_max": 15.74460220336914,
      "activations/layer0_attention_weight_min": -12.607773780822754,
      "activations/layer10_attention_weight_max": 39.25886154174805,
      "activations/layer10_attention_weight_min": -34.10601806640625,
      "activations/layer11_attention_weight_max": 36.47351837158203,
      "activations/layer11_attention_weight_min": -32.932804107666016,
      "activations/layer12_attention_weight_max": 24.023868560791016,
      "activations/layer12_attention_weight_min": -25.526691436767578,
      "activations/layer13_attention_weight_max": 36.46272277832031,
      "activations/layer13_attention_weight_min": -34.58707809448242,
      "activations/layer14_attention_weight_max": 43.5595703125,
      "activations/layer14_attention_weight_min": -37.446624755859375,
      "activations/layer15_attention_weight_max": 36.182193756103516,
      "activations/layer15_attention_weight_min": -32.88716125488281,
      "activations/layer16_attention_weight_max": 31.722766876220703,
      "activations/layer16_attention_weight_min": -27.635448455810547,
      "activations/layer17_attention_weight_max": 31.006765365600586,
      "activations/layer17_attention_weight_min": -27.345142364501953,
      "activations/layer18_attention_weight_max": 29.699256896972656,
      "activations/layer18_attention_weight_min": -25.03984260559082,
      "activations/layer19_attention_weight_max": 36.85221481323242,
      "activations/layer19_attention_weight_min": -31.14752197265625,
      "activations/layer1_attention_weight_max": 17.384389877319336,
      "activations/layer1_attention_weight_min": -16.531749725341797,
      "activations/layer20_attention_weight_max": 29.75152015686035,
      "activations/layer20_attention_weight_min": -24.950117111206055,
      "activations/layer21_attention_weight_max": 31.641239166259766,
      "activations/layer21_attention_weight_min": -25.203298568725586,
      "activations/layer22_attention_weight_max": 41.60639953613281,
      "activations/layer22_attention_weight_min": -32.69917297363281,
      "activations/layer23_attention_weight_max": 31.84062957763672,
      "activations/layer23_attention_weight_min": -25.09839630126953,
      "activations/layer2_attention_weight_max": 36.683250427246094,
      "activations/layer2_attention_weight_min": -35.1170654296875,
      "activations/layer3_attention_weight_max": 107.60354614257812,
      "activations/layer3_attention_weight_min": -105.81773376464844,
      "activations/layer4_attention_weight_max": 124.10804748535156,
      "activations/layer4_attention_weight_min": -121.42994689941406,
      "activations/layer5_attention_weight_max": 61.02398681640625,
      "activations/layer5_attention_weight_min": -66.83802795410156,
      "activations/layer6_attention_weight_max": 49.84032440185547,
      "activations/layer6_attention_weight_min": -49.70387268066406,
      "activations/layer7_attention_weight_max": 98.96825408935547,
      "activations/layer7_attention_weight_min": -97.59746551513672,
      "activations/layer8_attention_weight_max": 45.20023727416992,
      "activations/layer8_attention_weight_min": -42.8554573059082,
      "activations/layer9_attention_weight_max": 35.60110092163086,
      "activations/layer9_attention_weight_min": -33.78571319580078,
      "epoch": 13.44,
      "learning_rate": 6.395492424242424e-05,
      "loss": 2.7676,
      "step": 231350
    },
    {
      "activations/layer0_attention_weight_max": 16.11646270751953,
      "activations/layer0_attention_weight_min": -12.457871437072754,
      "activations/layer10_attention_weight_max": 30.01919174194336,
      "activations/layer10_attention_weight_min": -31.314434051513672,
      "activations/layer11_attention_weight_max": 29.656230926513672,
      "activations/layer11_attention_weight_min": -30.442211151123047,
      "activations/layer12_attention_weight_max": 23.146032333374023,
      "activations/layer12_attention_weight_min": -24.319162368774414,
      "activations/layer13_attention_weight_max": 34.16428756713867,
      "activations/layer13_attention_weight_min": -34.29174041748047,
      "activations/layer14_attention_weight_max": 40.20600891113281,
      "activations/layer14_attention_weight_min": -35.4459114074707,
      "activations/layer15_attention_weight_max": 31.138935089111328,
      "activations/layer15_attention_weight_min": -32.4190673828125,
      "activations/layer16_attention_weight_max": 30.37317657470703,
      "activations/layer16_attention_weight_min": -27.80130386352539,
      "activations/layer17_attention_weight_max": 27.36028480529785,
      "activations/layer17_attention_weight_min": -25.977540969848633,
      "activations/layer18_attention_weight_max": 28.468847274780273,
      "activations/layer18_attention_weight_min": -22.991003036499023,
      "activations/layer19_attention_weight_max": 35.64255142211914,
      "activations/layer19_attention_weight_min": -28.632734298706055,
      "activations/layer1_attention_weight_max": 17.636388778686523,
      "activations/layer1_attention_weight_min": -15.686623573303223,
      "activations/layer20_attention_weight_max": 25.320690155029297,
      "activations/layer20_attention_weight_min": -23.931167602539062,
      "activations/layer21_attention_weight_max": 24.69115447998047,
      "activations/layer21_attention_weight_min": -23.07713508605957,
      "activations/layer22_attention_weight_max": 36.082645416259766,
      "activations/layer22_attention_weight_min": -28.00777816772461,
      "activations/layer23_attention_weight_max": 32.044097900390625,
      "activations/layer23_attention_weight_min": -25.16469955444336,
      "activations/layer2_attention_weight_max": 33.251953125,
      "activations/layer2_attention_weight_min": -33.94183349609375,
      "activations/layer3_attention_weight_max": 94.61784362792969,
      "activations/layer3_attention_weight_min": -98.11833953857422,
      "activations/layer4_attention_weight_max": 107.33904266357422,
      "activations/layer4_attention_weight_min": -107.69200897216797,
      "activations/layer5_attention_weight_max": 52.28243637084961,
      "activations/layer5_attention_weight_min": -64.0697250366211,
      "activations/layer6_attention_weight_max": 43.47061538696289,
      "activations/layer6_attention_weight_min": -44.84854507446289,
      "activations/layer7_attention_weight_max": 90.25836181640625,
      "activations/layer7_attention_weight_min": -87.08503723144531,
      "activations/layer8_attention_weight_max": 38.85825729370117,
      "activations/layer8_attention_weight_min": -39.0541877746582,
      "activations/layer9_attention_weight_max": 31.534961700439453,
      "activations/layer9_attention_weight_min": -30.12725830078125,
      "epoch": 13.45,
      "learning_rate": 6.393598484848485e-05,
      "loss": 2.7641,
      "step": 231400
    },
    {
      "activations/layer0_attention_weight_max": 15.025487899780273,
      "activations/layer0_attention_weight_min": -12.698738098144531,
      "activations/layer10_attention_weight_max": 31.878238677978516,
      "activations/layer10_attention_weight_min": -31.80900764465332,
      "activations/layer11_attention_weight_max": 31.298965454101562,
      "activations/layer11_attention_weight_min": -32.03950119018555,
      "activations/layer12_attention_weight_max": 41.32883834838867,
      "activations/layer12_attention_weight_min": -26.01780891418457,
      "activations/layer13_attention_weight_max": 44.25795364379883,
      "activations/layer13_attention_weight_min": -35.89527130126953,
      "activations/layer14_attention_weight_max": 41.83625793457031,
      "activations/layer14_attention_weight_min": -37.90154266357422,
      "activations/layer15_attention_weight_max": 36.22989273071289,
      "activations/layer15_attention_weight_min": -34.937217712402344,
      "activations/layer16_attention_weight_max": 30.32828712463379,
      "activations/layer16_attention_weight_min": -30.056602478027344,
      "activations/layer17_attention_weight_max": 30.322181701660156,
      "activations/layer17_attention_weight_min": -24.918560028076172,
      "activations/layer18_attention_weight_max": 31.393278121948242,
      "activations/layer18_attention_weight_min": -23.636802673339844,
      "activations/layer19_attention_weight_max": 33.58079147338867,
      "activations/layer19_attention_weight_min": -30.488981246948242,
      "activations/layer1_attention_weight_max": 16.783323287963867,
      "activations/layer1_attention_weight_min": -15.927141189575195,
      "activations/layer20_attention_weight_max": 28.537694931030273,
      "activations/layer20_attention_weight_min": -26.42889404296875,
      "activations/layer21_attention_weight_max": 29.9104061126709,
      "activations/layer21_attention_weight_min": -24.384057998657227,
      "activations/layer22_attention_weight_max": 39.51848220825195,
      "activations/layer22_attention_weight_min": -28.652219772338867,
      "activations/layer23_attention_weight_max": 40.09579086303711,
      "activations/layer23_attention_weight_min": -26.37765884399414,
      "activations/layer2_attention_weight_max": 34.09008026123047,
      "activations/layer2_attention_weight_min": -33.59586715698242,
      "activations/layer3_attention_weight_max": 98.90129852294922,
      "activations/layer3_attention_weight_min": -99.11164093017578,
      "activations/layer4_attention_weight_max": 111.92694091796875,
      "activations/layer4_attention_weight_min": -110.69780731201172,
      "activations/layer5_attention_weight_max": 52.273956298828125,
      "activations/layer5_attention_weight_min": -59.529415130615234,
      "activations/layer6_attention_weight_max": 45.51858139038086,
      "activations/layer6_attention_weight_min": -44.923362731933594,
      "activations/layer7_attention_weight_max": 82.74683380126953,
      "activations/layer7_attention_weight_min": -86.73282623291016,
      "activations/layer8_attention_weight_max": 38.466434478759766,
      "activations/layer8_attention_weight_min": -38.4444465637207,
      "activations/layer9_attention_weight_max": 32.10519027709961,
      "activations/layer9_attention_weight_min": -32.736759185791016,
      "epoch": 13.45,
      "learning_rate": 6.391704545454545e-05,
      "loss": 2.7445,
      "step": 231450
    },
    {
      "activations/layer0_attention_weight_max": 15.998392105102539,
      "activations/layer0_attention_weight_min": -12.980812072753906,
      "activations/layer10_attention_weight_max": 33.33736038208008,
      "activations/layer10_attention_weight_min": -32.75597381591797,
      "activations/layer11_attention_weight_max": 32.71746826171875,
      "activations/layer11_attention_weight_min": -31.66613006591797,
      "activations/layer12_attention_weight_max": 25.50662612915039,
      "activations/layer12_attention_weight_min": -25.618995666503906,
      "activations/layer13_attention_weight_max": 38.475364685058594,
      "activations/layer13_attention_weight_min": -34.74512481689453,
      "activations/layer14_attention_weight_max": 41.07502746582031,
      "activations/layer14_attention_weight_min": -35.884010314941406,
      "activations/layer15_attention_weight_max": 35.12535095214844,
      "activations/layer15_attention_weight_min": -32.79924392700195,
      "activations/layer16_attention_weight_max": 28.480302810668945,
      "activations/layer16_attention_weight_min": -28.38915252685547,
      "activations/layer17_attention_weight_max": 29.14974021911621,
      "activations/layer17_attention_weight_min": -25.668004989624023,
      "activations/layer18_attention_weight_max": 29.52474594116211,
      "activations/layer18_attention_weight_min": -26.96002197265625,
      "activations/layer19_attention_weight_max": 33.55171203613281,
      "activations/layer19_attention_weight_min": -31.806447982788086,
      "activations/layer1_attention_weight_max": 16.253870010375977,
      "activations/layer1_attention_weight_min": -14.900107383728027,
      "activations/layer20_attention_weight_max": 26.922515869140625,
      "activations/layer20_attention_weight_min": -26.324504852294922,
      "activations/layer21_attention_weight_max": 27.536630630493164,
      "activations/layer21_attention_weight_min": -24.706851959228516,
      "activations/layer22_attention_weight_max": 39.166236877441406,
      "activations/layer22_attention_weight_min": -32.7674446105957,
      "activations/layer23_attention_weight_max": 30.185150146484375,
      "activations/layer23_attention_weight_min": -26.862194061279297,
      "activations/layer2_attention_weight_max": 34.62187194824219,
      "activations/layer2_attention_weight_min": -34.54454803466797,
      "activations/layer3_attention_weight_max": 99.26918029785156,
      "activations/layer3_attention_weight_min": -100.65762329101562,
      "activations/layer4_attention_weight_max": 110.6010971069336,
      "activations/layer4_attention_weight_min": -110.3123779296875,
      "activations/layer5_attention_weight_max": 51.08705139160156,
      "activations/layer5_attention_weight_min": -60.393348693847656,
      "activations/layer6_attention_weight_max": 44.26387405395508,
      "activations/layer6_attention_weight_min": -47.30943298339844,
      "activations/layer7_attention_weight_max": 87.65419006347656,
      "activations/layer7_attention_weight_min": -88.59672546386719,
      "activations/layer8_attention_weight_max": 40.97017288208008,
      "activations/layer8_attention_weight_min": -43.188232421875,
      "activations/layer9_attention_weight_max": 32.86812210083008,
      "activations/layer9_attention_weight_min": -33.80276870727539,
      "epoch": 13.45,
      "learning_rate": 6.389810606060605e-05,
      "loss": 2.7595,
      "step": 231500
    },
    {
      "activations/layer0_attention_weight_max": 16.314456939697266,
      "activations/layer0_attention_weight_min": -12.372907638549805,
      "activations/layer10_attention_weight_max": 33.14249801635742,
      "activations/layer10_attention_weight_min": -32.7077522277832,
      "activations/layer11_attention_weight_max": 32.509300231933594,
      "activations/layer11_attention_weight_min": -31.328611373901367,
      "activations/layer12_attention_weight_max": 25.14076042175293,
      "activations/layer12_attention_weight_min": -25.636959075927734,
      "activations/layer13_attention_weight_max": 41.669761657714844,
      "activations/layer13_attention_weight_min": -35.44538116455078,
      "activations/layer14_attention_weight_max": 48.3022575378418,
      "activations/layer14_attention_weight_min": -41.054508209228516,
      "activations/layer15_attention_weight_max": 37.29230880737305,
      "activations/layer15_attention_weight_min": -33.6949577331543,
      "activations/layer16_attention_weight_max": 30.323192596435547,
      "activations/layer16_attention_weight_min": -32.130680084228516,
      "activations/layer17_attention_weight_max": 30.17098045349121,
      "activations/layer17_attention_weight_min": -29.963077545166016,
      "activations/layer18_attention_weight_max": 30.943449020385742,
      "activations/layer18_attention_weight_min": -27.282907485961914,
      "activations/layer19_attention_weight_max": 37.63524627685547,
      "activations/layer19_attention_weight_min": -32.04771041870117,
      "activations/layer1_attention_weight_max": 16.379470825195312,
      "activations/layer1_attention_weight_min": -15.28415298461914,
      "activations/layer20_attention_weight_max": 31.712993621826172,
      "activations/layer20_attention_weight_min": -25.971546173095703,
      "activations/layer21_attention_weight_max": 28.6682186126709,
      "activations/layer21_attention_weight_min": -24.988615036010742,
      "activations/layer22_attention_weight_max": 45.648868560791016,
      "activations/layer22_attention_weight_min": -31.84200668334961,
      "activations/layer23_attention_weight_max": 32.371849060058594,
      "activations/layer23_attention_weight_min": -23.8531551361084,
      "activations/layer2_attention_weight_max": 33.9547119140625,
      "activations/layer2_attention_weight_min": -34.77473449707031,
      "activations/layer3_attention_weight_max": 98.11998748779297,
      "activations/layer3_attention_weight_min": -102.6693115234375,
      "activations/layer4_attention_weight_max": 110.72218322753906,
      "activations/layer4_attention_weight_min": -117.85346984863281,
      "activations/layer5_attention_weight_max": 52.642086029052734,
      "activations/layer5_attention_weight_min": -63.65450668334961,
      "activations/layer6_attention_weight_max": 44.626834869384766,
      "activations/layer6_attention_weight_min": -49.06540298461914,
      "activations/layer7_attention_weight_max": 89.18488311767578,
      "activations/layer7_attention_weight_min": -90.40406799316406,
      "activations/layer8_attention_weight_max": 41.21171188354492,
      "activations/layer8_attention_weight_min": -42.88987350463867,
      "activations/layer9_attention_weight_max": 32.88675308227539,
      "activations/layer9_attention_weight_min": -34.164405822753906,
      "epoch": 13.45,
      "learning_rate": 6.387916666666666e-05,
      "loss": 2.7695,
      "step": 231550
    },
    {
      "activations/layer0_attention_weight_max": 16.07927131652832,
      "activations/layer0_attention_weight_min": -12.482946395874023,
      "activations/layer10_attention_weight_max": 40.20845031738281,
      "activations/layer10_attention_weight_min": -37.9312744140625,
      "activations/layer11_attention_weight_max": 39.952430725097656,
      "activations/layer11_attention_weight_min": -36.76543045043945,
      "activations/layer12_attention_weight_max": 28.25897979736328,
      "activations/layer12_attention_weight_min": -26.695314407348633,
      "activations/layer13_attention_weight_max": 41.95709228515625,
      "activations/layer13_attention_weight_min": -36.29613494873047,
      "activations/layer14_attention_weight_max": 48.94952392578125,
      "activations/layer14_attention_weight_min": -41.108821868896484,
      "activations/layer15_attention_weight_max": 40.14769744873047,
      "activations/layer15_attention_weight_min": -35.23139190673828,
      "activations/layer16_attention_weight_max": 36.29997253417969,
      "activations/layer16_attention_weight_min": -27.706104278564453,
      "activations/layer17_attention_weight_max": 32.198463439941406,
      "activations/layer17_attention_weight_min": -25.316476821899414,
      "activations/layer18_attention_weight_max": 32.504241943359375,
      "activations/layer18_attention_weight_min": -23.098188400268555,
      "activations/layer19_attention_weight_max": 39.526668548583984,
      "activations/layer19_attention_weight_min": -32.92231369018555,
      "activations/layer1_attention_weight_max": 16.195661544799805,
      "activations/layer1_attention_weight_min": -15.632787704467773,
      "activations/layer20_attention_weight_max": 33.332271575927734,
      "activations/layer20_attention_weight_min": -24.036375045776367,
      "activations/layer21_attention_weight_max": 31.338130950927734,
      "activations/layer21_attention_weight_min": -23.654399871826172,
      "activations/layer22_attention_weight_max": 47.29329299926758,
      "activations/layer22_attention_weight_min": -29.338146209716797,
      "activations/layer23_attention_weight_max": 36.85600280761719,
      "activations/layer23_attention_weight_min": -26.230655670166016,
      "activations/layer2_attention_weight_max": 34.64655685424805,
      "activations/layer2_attention_weight_min": -36.118080139160156,
      "activations/layer3_attention_weight_max": 106.92355346679688,
      "activations/layer3_attention_weight_min": -105.73894500732422,
      "activations/layer4_attention_weight_max": 118.85298919677734,
      "activations/layer4_attention_weight_min": -119.06404113769531,
      "activations/layer5_attention_weight_max": 55.12986755371094,
      "activations/layer5_attention_weight_min": -62.7325439453125,
      "activations/layer6_attention_weight_max": 48.975833892822266,
      "activations/layer6_attention_weight_min": -53.813255310058594,
      "activations/layer7_attention_weight_max": 104.99658203125,
      "activations/layer7_attention_weight_min": -106.16460418701172,
      "activations/layer8_attention_weight_max": 47.273475646972656,
      "activations/layer8_attention_weight_min": -49.60630798339844,
      "activations/layer9_attention_weight_max": 37.77558898925781,
      "activations/layer9_attention_weight_min": -37.095802307128906,
      "epoch": 13.46,
      "learning_rate": 6.386022727272726e-05,
      "loss": 2.7593,
      "step": 231600
    },
    {
      "activations/layer0_attention_weight_max": 16.557424545288086,
      "activations/layer0_attention_weight_min": -14.033285140991211,
      "activations/layer10_attention_weight_max": 36.20309066772461,
      "activations/layer10_attention_weight_min": -33.765968322753906,
      "activations/layer11_attention_weight_max": 36.422752380371094,
      "activations/layer11_attention_weight_min": -34.97785568237305,
      "activations/layer12_attention_weight_max": 25.901124954223633,
      "activations/layer12_attention_weight_min": -26.55571174621582,
      "activations/layer13_attention_weight_max": 39.93584442138672,
      "activations/layer13_attention_weight_min": -35.44935607910156,
      "activations/layer14_attention_weight_max": 42.22797775268555,
      "activations/layer14_attention_weight_min": -37.25566482543945,
      "activations/layer15_attention_weight_max": 36.45918273925781,
      "activations/layer15_attention_weight_min": -36.30766296386719,
      "activations/layer16_attention_weight_max": 35.259220123291016,
      "activations/layer16_attention_weight_min": -27.725954055786133,
      "activations/layer17_attention_weight_max": 29.4251651763916,
      "activations/layer17_attention_weight_min": -25.13056755065918,
      "activations/layer18_attention_weight_max": 30.904220581054688,
      "activations/layer18_attention_weight_min": -24.07219696044922,
      "activations/layer19_attention_weight_max": 40.70848083496094,
      "activations/layer19_attention_weight_min": -30.710718154907227,
      "activations/layer1_attention_weight_max": 16.530183792114258,
      "activations/layer1_attention_weight_min": -17.42220687866211,
      "activations/layer20_attention_weight_max": 30.21029281616211,
      "activations/layer20_attention_weight_min": -26.078744888305664,
      "activations/layer21_attention_weight_max": 30.56911849975586,
      "activations/layer21_attention_weight_min": -25.01041603088379,
      "activations/layer22_attention_weight_max": 47.6109504699707,
      "activations/layer22_attention_weight_min": -30.361312866210938,
      "activations/layer23_attention_weight_max": 38.37514114379883,
      "activations/layer23_attention_weight_min": -25.077499389648438,
      "activations/layer2_attention_weight_max": 33.377559661865234,
      "activations/layer2_attention_weight_min": -34.612327575683594,
      "activations/layer3_attention_weight_max": 99.13267517089844,
      "activations/layer3_attention_weight_min": -101.71977233886719,
      "activations/layer4_attention_weight_max": 110.74825286865234,
      "activations/layer4_attention_weight_min": -111.8572006225586,
      "activations/layer5_attention_weight_max": 53.7500114440918,
      "activations/layer5_attention_weight_min": -64.47418212890625,
      "activations/layer6_attention_weight_max": 42.79636001586914,
      "activations/layer6_attention_weight_min": -46.979373931884766,
      "activations/layer7_attention_weight_max": 88.0535888671875,
      "activations/layer7_attention_weight_min": -94.66425323486328,
      "activations/layer8_attention_weight_max": 40.508602142333984,
      "activations/layer8_attention_weight_min": -41.4229850769043,
      "activations/layer9_attention_weight_max": 36.081329345703125,
      "activations/layer9_attention_weight_min": -35.4370231628418,
      "epoch": 13.46,
      "learning_rate": 6.384128787878787e-05,
      "loss": 2.7769,
      "step": 231650
    },
    {
      "activations/layer0_attention_weight_max": 15.464597702026367,
      "activations/layer0_attention_weight_min": -12.487226486206055,
      "activations/layer10_attention_weight_max": 34.086055755615234,
      "activations/layer10_attention_weight_min": -31.85032844543457,
      "activations/layer11_attention_weight_max": 32.965938568115234,
      "activations/layer11_attention_weight_min": -30.724323272705078,
      "activations/layer12_attention_weight_max": 25.554590225219727,
      "activations/layer12_attention_weight_min": -25.13157844543457,
      "activations/layer13_attention_weight_max": 38.41805648803711,
      "activations/layer13_attention_weight_min": -32.309139251708984,
      "activations/layer14_attention_weight_max": 40.961822509765625,
      "activations/layer14_attention_weight_min": -36.70792770385742,
      "activations/layer15_attention_weight_max": 35.51588439941406,
      "activations/layer15_attention_weight_min": -31.54062843322754,
      "activations/layer16_attention_weight_max": 31.294788360595703,
      "activations/layer16_attention_weight_min": -29.36199951171875,
      "activations/layer17_attention_weight_max": 29.315467834472656,
      "activations/layer17_attention_weight_min": -24.999858856201172,
      "activations/layer18_attention_weight_max": 31.963430404663086,
      "activations/layer18_attention_weight_min": -25.130651473999023,
      "activations/layer19_attention_weight_max": 34.49970626831055,
      "activations/layer19_attention_weight_min": -33.05079650878906,
      "activations/layer1_attention_weight_max": 16.152362823486328,
      "activations/layer1_attention_weight_min": -14.602416038513184,
      "activations/layer20_attention_weight_max": 28.801912307739258,
      "activations/layer20_attention_weight_min": -25.31456756591797,
      "activations/layer21_attention_weight_max": 28.093801498413086,
      "activations/layer21_attention_weight_min": -25.314041137695312,
      "activations/layer22_attention_weight_max": 43.218101501464844,
      "activations/layer22_attention_weight_min": -30.422090530395508,
      "activations/layer23_attention_weight_max": 35.90238952636719,
      "activations/layer23_attention_weight_min": -23.00997543334961,
      "activations/layer2_attention_weight_max": 33.888893127441406,
      "activations/layer2_attention_weight_min": -34.199554443359375,
      "activations/layer3_attention_weight_max": 96.41570281982422,
      "activations/layer3_attention_weight_min": -101.90122985839844,
      "activations/layer4_attention_weight_max": 108.64330291748047,
      "activations/layer4_attention_weight_min": -110.81572723388672,
      "activations/layer5_attention_weight_max": 51.07682418823242,
      "activations/layer5_attention_weight_min": -60.46787643432617,
      "activations/layer6_attention_weight_max": 41.907081604003906,
      "activations/layer6_attention_weight_min": -44.22795867919922,
      "activations/layer7_attention_weight_max": 86.87389373779297,
      "activations/layer7_attention_weight_min": -86.94580078125,
      "activations/layer8_attention_weight_max": 39.16023635864258,
      "activations/layer8_attention_weight_min": -40.80450439453125,
      "activations/layer9_attention_weight_max": 31.96661949157715,
      "activations/layer9_attention_weight_min": -31.760923385620117,
      "epoch": 13.46,
      "learning_rate": 6.382234848484848e-05,
      "loss": 2.7641,
      "step": 231700
    },
    {
      "activations/layer0_attention_weight_max": 15.807549476623535,
      "activations/layer0_attention_weight_min": -12.848069190979004,
      "activations/layer10_attention_weight_max": 30.81268310546875,
      "activations/layer10_attention_weight_min": -31.16347885131836,
      "activations/layer11_attention_weight_max": 30.67202377319336,
      "activations/layer11_attention_weight_min": -31.389738082885742,
      "activations/layer12_attention_weight_max": 25.294958114624023,
      "activations/layer12_attention_weight_min": -25.729206085205078,
      "activations/layer13_attention_weight_max": 37.009864807128906,
      "activations/layer13_attention_weight_min": -34.161109924316406,
      "activations/layer14_attention_weight_max": 40.56940460205078,
      "activations/layer14_attention_weight_min": -36.79615783691406,
      "activations/layer15_attention_weight_max": 33.27364730834961,
      "activations/layer15_attention_weight_min": -32.719642639160156,
      "activations/layer16_attention_weight_max": 30.43863868713379,
      "activations/layer16_attention_weight_min": -30.819698333740234,
      "activations/layer17_attention_weight_max": 28.06915283203125,
      "activations/layer17_attention_weight_min": -27.709070205688477,
      "activations/layer18_attention_weight_max": 31.75739860534668,
      "activations/layer18_attention_weight_min": -24.506017684936523,
      "activations/layer19_attention_weight_max": 35.52604293823242,
      "activations/layer19_attention_weight_min": -30.032987594604492,
      "activations/layer1_attention_weight_max": 16.648591995239258,
      "activations/layer1_attention_weight_min": -14.775546073913574,
      "activations/layer20_attention_weight_max": 30.343902587890625,
      "activations/layer20_attention_weight_min": -25.45263671875,
      "activations/layer21_attention_weight_max": 30.935863494873047,
      "activations/layer21_attention_weight_min": -24.370010375976562,
      "activations/layer22_attention_weight_max": 40.99726104736328,
      "activations/layer22_attention_weight_min": -29.946796417236328,
      "activations/layer23_attention_weight_max": 31.558494567871094,
      "activations/layer23_attention_weight_min": -25.49053955078125,
      "activations/layer2_attention_weight_max": 33.40222930908203,
      "activations/layer2_attention_weight_min": -34.246673583984375,
      "activations/layer3_attention_weight_max": 96.39664459228516,
      "activations/layer3_attention_weight_min": -99.31459045410156,
      "activations/layer4_attention_weight_max": 110.56514739990234,
      "activations/layer4_attention_weight_min": -112.90673828125,
      "activations/layer5_attention_weight_max": 51.62750244140625,
      "activations/layer5_attention_weight_min": -68.12666320800781,
      "activations/layer6_attention_weight_max": 43.21510314941406,
      "activations/layer6_attention_weight_min": -48.08638000488281,
      "activations/layer7_attention_weight_max": 89.58794403076172,
      "activations/layer7_attention_weight_min": -93.87614440917969,
      "activations/layer8_attention_weight_max": 39.99281311035156,
      "activations/layer8_attention_weight_min": -40.266510009765625,
      "activations/layer9_attention_weight_max": 31.385391235351562,
      "activations/layer9_attention_weight_min": -32.26152038574219,
      "epoch": 13.47,
      "learning_rate": 6.380340909090908e-05,
      "loss": 2.7537,
      "step": 231750
    },
    {
      "activations/layer0_attention_weight_max": 16.336809158325195,
      "activations/layer0_attention_weight_min": -13.063578605651855,
      "activations/layer10_attention_weight_max": 31.851858139038086,
      "activations/layer10_attention_weight_min": -31.33713150024414,
      "activations/layer11_attention_weight_max": 32.49531555175781,
      "activations/layer11_attention_weight_min": -31.62842559814453,
      "activations/layer12_attention_weight_max": 33.42218780517578,
      "activations/layer12_attention_weight_min": -25.546199798583984,
      "activations/layer13_attention_weight_max": 42.52370834350586,
      "activations/layer13_attention_weight_min": -32.83013916015625,
      "activations/layer14_attention_weight_max": 43.205535888671875,
      "activations/layer14_attention_weight_min": -36.56749725341797,
      "activations/layer15_attention_weight_max": 38.802913665771484,
      "activations/layer15_attention_weight_min": -35.73906707763672,
      "activations/layer16_attention_weight_max": 32.125553131103516,
      "activations/layer16_attention_weight_min": -27.8592529296875,
      "activations/layer17_attention_weight_max": 32.66390609741211,
      "activations/layer17_attention_weight_min": -26.419727325439453,
      "activations/layer18_attention_weight_max": 33.80698776245117,
      "activations/layer18_attention_weight_min": -24.878480911254883,
      "activations/layer19_attention_weight_max": 38.47822952270508,
      "activations/layer19_attention_weight_min": -31.64483642578125,
      "activations/layer1_attention_weight_max": 16.172496795654297,
      "activations/layer1_attention_weight_min": -14.643346786499023,
      "activations/layer20_attention_weight_max": 32.47058868408203,
      "activations/layer20_attention_weight_min": -26.915752410888672,
      "activations/layer21_attention_weight_max": 29.69002914428711,
      "activations/layer21_attention_weight_min": -24.520305633544922,
      "activations/layer22_attention_weight_max": 49.82638168334961,
      "activations/layer22_attention_weight_min": -31.237804412841797,
      "activations/layer23_attention_weight_max": 32.81056594848633,
      "activations/layer23_attention_weight_min": -24.269041061401367,
      "activations/layer2_attention_weight_max": 34.245784759521484,
      "activations/layer2_attention_weight_min": -33.736175537109375,
      "activations/layer3_attention_weight_max": 100.50924682617188,
      "activations/layer3_attention_weight_min": -102.8991470336914,
      "activations/layer4_attention_weight_max": 111.04266357421875,
      "activations/layer4_attention_weight_min": -109.9287338256836,
      "activations/layer5_attention_weight_max": 54.78760528564453,
      "activations/layer5_attention_weight_min": -62.85111618041992,
      "activations/layer6_attention_weight_max": 44.1557731628418,
      "activations/layer6_attention_weight_min": -44.26168441772461,
      "activations/layer7_attention_weight_max": 87.50137329101562,
      "activations/layer7_attention_weight_min": -86.63803100585938,
      "activations/layer8_attention_weight_max": 39.602394104003906,
      "activations/layer8_attention_weight_min": -39.17299270629883,
      "activations/layer9_attention_weight_max": 30.9766788482666,
      "activations/layer9_attention_weight_min": -32.41545104980469,
      "epoch": 13.47,
      "learning_rate": 6.37844696969697e-05,
      "loss": 2.7487,
      "step": 231800
    },
    {
      "activations/layer0_attention_weight_max": 15.858800888061523,
      "activations/layer0_attention_weight_min": -12.510148048400879,
      "activations/layer10_attention_weight_max": 34.08687973022461,
      "activations/layer10_attention_weight_min": -31.43526840209961,
      "activations/layer11_attention_weight_max": 33.15522003173828,
      "activations/layer11_attention_weight_min": -32.75004196166992,
      "activations/layer12_attention_weight_max": 24.926063537597656,
      "activations/layer12_attention_weight_min": -25.05759620666504,
      "activations/layer13_attention_weight_max": 39.87627029418945,
      "activations/layer13_attention_weight_min": -35.96055221557617,
      "activations/layer14_attention_weight_max": 41.40631866455078,
      "activations/layer14_attention_weight_min": -36.065757751464844,
      "activations/layer15_attention_weight_max": 39.366058349609375,
      "activations/layer15_attention_weight_min": -33.113624572753906,
      "activations/layer16_attention_weight_max": 30.089330673217773,
      "activations/layer16_attention_weight_min": -27.199460983276367,
      "activations/layer17_attention_weight_max": 31.719953536987305,
      "activations/layer17_attention_weight_min": -25.754093170166016,
      "activations/layer18_attention_weight_max": 29.50844955444336,
      "activations/layer18_attention_weight_min": -25.786209106445312,
      "activations/layer19_attention_weight_max": 32.4846305847168,
      "activations/layer19_attention_weight_min": -31.052734375,
      "activations/layer1_attention_weight_max": 16.860300064086914,
      "activations/layer1_attention_weight_min": -15.581708908081055,
      "activations/layer20_attention_weight_max": 27.149803161621094,
      "activations/layer20_attention_weight_min": -25.2509708404541,
      "activations/layer21_attention_weight_max": 27.733781814575195,
      "activations/layer21_attention_weight_min": -25.721435546875,
      "activations/layer22_attention_weight_max": 39.44895553588867,
      "activations/layer22_attention_weight_min": -30.216346740722656,
      "activations/layer23_attention_weight_max": 30.846694946289062,
      "activations/layer23_attention_weight_min": -24.951171875,
      "activations/layer2_attention_weight_max": 33.34373092651367,
      "activations/layer2_attention_weight_min": -32.94917297363281,
      "activations/layer3_attention_weight_max": 101.36463165283203,
      "activations/layer3_attention_weight_min": -101.23357391357422,
      "activations/layer4_attention_weight_max": 111.54741668701172,
      "activations/layer4_attention_weight_min": -110.16402435302734,
      "activations/layer5_attention_weight_max": 51.977447509765625,
      "activations/layer5_attention_weight_min": -64.77394104003906,
      "activations/layer6_attention_weight_max": 43.572853088378906,
      "activations/layer6_attention_weight_min": -47.38932800292969,
      "activations/layer7_attention_weight_max": 89.39796447753906,
      "activations/layer7_attention_weight_min": -91.32626342773438,
      "activations/layer8_attention_weight_max": 44.2192268371582,
      "activations/layer8_attention_weight_min": -42.39064025878906,
      "activations/layer9_attention_weight_max": 32.19980239868164,
      "activations/layer9_attention_weight_min": -33.24387741088867,
      "epoch": 13.47,
      "learning_rate": 6.37655303030303e-05,
      "loss": 2.7544,
      "step": 231850
    },
    {
      "activations/layer0_attention_weight_max": 15.419978141784668,
      "activations/layer0_attention_weight_min": -12.748042106628418,
      "activations/layer10_attention_weight_max": 42.25773239135742,
      "activations/layer10_attention_weight_min": -38.80276107788086,
      "activations/layer11_attention_weight_max": 40.738224029541016,
      "activations/layer11_attention_weight_min": -37.960044860839844,
      "activations/layer12_attention_weight_max": 30.208921432495117,
      "activations/layer12_attention_weight_min": -28.58616828918457,
      "activations/layer13_attention_weight_max": 44.15122604370117,
      "activations/layer13_attention_weight_min": -37.63795471191406,
      "activations/layer14_attention_weight_max": 52.420616149902344,
      "activations/layer14_attention_weight_min": -42.750221252441406,
      "activations/layer15_attention_weight_max": 46.019046783447266,
      "activations/layer15_attention_weight_min": -36.78701400756836,
      "activations/layer16_attention_weight_max": 34.664588928222656,
      "activations/layer16_attention_weight_min": -28.904481887817383,
      "activations/layer17_attention_weight_max": 33.507537841796875,
      "activations/layer17_attention_weight_min": -27.988515853881836,
      "activations/layer18_attention_weight_max": 31.725322723388672,
      "activations/layer18_attention_weight_min": -27.301559448242188,
      "activations/layer19_attention_weight_max": 37.80524826049805,
      "activations/layer19_attention_weight_min": -30.750654220581055,
      "activations/layer1_attention_weight_max": 17.434049606323242,
      "activations/layer1_attention_weight_min": -14.566390991210938,
      "activations/layer20_attention_weight_max": 33.359657287597656,
      "activations/layer20_attention_weight_min": -23.752399444580078,
      "activations/layer21_attention_weight_max": 31.565610885620117,
      "activations/layer21_attention_weight_min": -25.19997215270996,
      "activations/layer22_attention_weight_max": 53.288272857666016,
      "activations/layer22_attention_weight_min": -30.472097396850586,
      "activations/layer23_attention_weight_max": 38.223846435546875,
      "activations/layer23_attention_weight_min": -24.23868179321289,
      "activations/layer2_attention_weight_max": 36.219215393066406,
      "activations/layer2_attention_weight_min": -35.315818786621094,
      "activations/layer3_attention_weight_max": 107.1817855834961,
      "activations/layer3_attention_weight_min": -105.08728790283203,
      "activations/layer4_attention_weight_max": 115.74031066894531,
      "activations/layer4_attention_weight_min": -114.20674896240234,
      "activations/layer5_attention_weight_max": 51.73114013671875,
      "activations/layer5_attention_weight_min": -62.7125244140625,
      "activations/layer6_attention_weight_max": 46.74972915649414,
      "activations/layer6_attention_weight_min": -47.76920700073242,
      "activations/layer7_attention_weight_max": 100.83070373535156,
      "activations/layer7_attention_weight_min": -95.3333969116211,
      "activations/layer8_attention_weight_max": 45.81216812133789,
      "activations/layer8_attention_weight_min": -44.7335319519043,
      "activations/layer9_attention_weight_max": 40.096256256103516,
      "activations/layer9_attention_weight_min": -38.94011688232422,
      "epoch": 13.47,
      "learning_rate": 6.37465909090909e-05,
      "loss": 2.7795,
      "step": 231900
    },
    {
      "activations/layer0_attention_weight_max": 14.631232261657715,
      "activations/layer0_attention_weight_min": -12.669957160949707,
      "activations/layer10_attention_weight_max": 42.17814254760742,
      "activations/layer10_attention_weight_min": -36.311851501464844,
      "activations/layer11_attention_weight_max": 46.645362854003906,
      "activations/layer11_attention_weight_min": -35.2271614074707,
      "activations/layer12_attention_weight_max": 32.583763122558594,
      "activations/layer12_attention_weight_min": -26.61825180053711,
      "activations/layer13_attention_weight_max": 51.5850944519043,
      "activations/layer13_attention_weight_min": -37.01801300048828,
      "activations/layer14_attention_weight_max": 53.88621139526367,
      "activations/layer14_attention_weight_min": -39.87173080444336,
      "activations/layer15_attention_weight_max": 54.616703033447266,
      "activations/layer15_attention_weight_min": -36.45647048950195,
      "activations/layer16_attention_weight_max": 36.21363067626953,
      "activations/layer16_attention_weight_min": -29.86119270324707,
      "activations/layer17_attention_weight_max": 32.7254638671875,
      "activations/layer17_attention_weight_min": -26.448801040649414,
      "activations/layer18_attention_weight_max": 31.113290786743164,
      "activations/layer18_attention_weight_min": -25.936521530151367,
      "activations/layer19_attention_weight_max": 35.11521911621094,
      "activations/layer19_attention_weight_min": -31.376672744750977,
      "activations/layer1_attention_weight_max": 18.59483528137207,
      "activations/layer1_attention_weight_min": -16.309810638427734,
      "activations/layer20_attention_weight_max": 32.62982177734375,
      "activations/layer20_attention_weight_min": -26.93880271911621,
      "activations/layer21_attention_weight_max": 31.718564987182617,
      "activations/layer21_attention_weight_min": -25.539514541625977,
      "activations/layer22_attention_weight_max": 43.646484375,
      "activations/layer22_attention_weight_min": -32.0136604309082,
      "activations/layer23_attention_weight_max": 33.08055877685547,
      "activations/layer23_attention_weight_min": -24.133949279785156,
      "activations/layer2_attention_weight_max": 35.53180694580078,
      "activations/layer2_attention_weight_min": -35.07263946533203,
      "activations/layer3_attention_weight_max": 105.32676696777344,
      "activations/layer3_attention_weight_min": -102.48226928710938,
      "activations/layer4_attention_weight_max": 118.74958801269531,
      "activations/layer4_attention_weight_min": -115.062744140625,
      "activations/layer5_attention_weight_max": 53.24601364135742,
      "activations/layer5_attention_weight_min": -63.85734176635742,
      "activations/layer6_attention_weight_max": 46.17430877685547,
      "activations/layer6_attention_weight_min": -48.39809799194336,
      "activations/layer7_attention_weight_max": 103.6002426147461,
      "activations/layer7_attention_weight_min": -94.32089233398438,
      "activations/layer8_attention_weight_max": 48.88595199584961,
      "activations/layer8_attention_weight_min": -44.48747634887695,
      "activations/layer9_attention_weight_max": 40.0118522644043,
      "activations/layer9_attention_weight_min": -35.50570297241211,
      "epoch": 13.48,
      "learning_rate": 6.372765151515152e-05,
      "loss": 2.747,
      "step": 231950
    },
    {
      "activations/layer0_attention_weight_max": 15.453415870666504,
      "activations/layer0_attention_weight_min": -12.663153648376465,
      "activations/layer10_attention_weight_max": 33.32065963745117,
      "activations/layer10_attention_weight_min": -34.169349670410156,
      "activations/layer11_attention_weight_max": 33.36604309082031,
      "activations/layer11_attention_weight_min": -32.48204040527344,
      "activations/layer12_attention_weight_max": 26.926143646240234,
      "activations/layer12_attention_weight_min": -26.133268356323242,
      "activations/layer13_attention_weight_max": 36.67519760131836,
      "activations/layer13_attention_weight_min": -34.16991424560547,
      "activations/layer14_attention_weight_max": 40.79639434814453,
      "activations/layer14_attention_weight_min": -37.96305465698242,
      "activations/layer15_attention_weight_max": 35.736934661865234,
      "activations/layer15_attention_weight_min": -32.95341491699219,
      "activations/layer16_attention_weight_max": 31.24501609802246,
      "activations/layer16_attention_weight_min": -27.060239791870117,
      "activations/layer17_attention_weight_max": 28.0872745513916,
      "activations/layer17_attention_weight_min": -26.678041458129883,
      "activations/layer18_attention_weight_max": 33.07561492919922,
      "activations/layer18_attention_weight_min": -25.045934677124023,
      "activations/layer19_attention_weight_max": 33.296913146972656,
      "activations/layer19_attention_weight_min": -32.9174690246582,
      "activations/layer1_attention_weight_max": 16.90778350830078,
      "activations/layer1_attention_weight_min": -15.894299507141113,
      "activations/layer20_attention_weight_max": 28.255859375,
      "activations/layer20_attention_weight_min": -25.491313934326172,
      "activations/layer21_attention_weight_max": 29.15285873413086,
      "activations/layer21_attention_weight_min": -23.545719146728516,
      "activations/layer22_attention_weight_max": 42.96246337890625,
      "activations/layer22_attention_weight_min": -31.520004272460938,
      "activations/layer23_attention_weight_max": 33.53777313232422,
      "activations/layer23_attention_weight_min": -26.567546844482422,
      "activations/layer2_attention_weight_max": 34.92686462402344,
      "activations/layer2_attention_weight_min": -33.754356384277344,
      "activations/layer3_attention_weight_max": 99.5008316040039,
      "activations/layer3_attention_weight_min": -99.21458435058594,
      "activations/layer4_attention_weight_max": 108.17742156982422,
      "activations/layer4_attention_weight_min": -112.5638198852539,
      "activations/layer5_attention_weight_max": 56.92529296875,
      "activations/layer5_attention_weight_min": -63.09920883178711,
      "activations/layer6_attention_weight_max": 43.600215911865234,
      "activations/layer6_attention_weight_min": -47.49839401245117,
      "activations/layer7_attention_weight_max": 88.56658172607422,
      "activations/layer7_attention_weight_min": -92.88710021972656,
      "activations/layer8_attention_weight_max": 42.60872268676758,
      "activations/layer8_attention_weight_min": -41.4782600402832,
      "activations/layer9_attention_weight_max": 35.75530242919922,
      "activations/layer9_attention_weight_min": -34.933258056640625,
      "epoch": 13.48,
      "learning_rate": 6.370871212121212e-05,
      "loss": 2.7615,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.5001,
      "eval_samples_per_second": 505.173,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.5001,
      "eval_openwebtext_samples_per_second": 505.173,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9483,
      "eval_wikitext_samples_per_second": 234.049,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_lambada_loss": 2.439453125,
      "eval_lambada_ppl": 11.46676813896006,
      "eval_lambada_runtime": 9.57,
      "eval_lambada_samples_per_second": 508.779,
      "step": 232000
    },
    {
      "activations/layer0_attention_weight_max": 16.422719955444336,
      "activations/layer0_attention_weight_min": -13.169327735900879,
      "activations/layer10_attention_weight_max": 34.0191650390625,
      "activations/layer10_attention_weight_min": -33.98529052734375,
      "activations/layer11_attention_weight_max": 33.945987701416016,
      "activations/layer11_attention_weight_min": -33.31377410888672,
      "activations/layer12_attention_weight_max": 25.12438201904297,
      "activations/layer12_attention_weight_min": -24.981746673583984,
      "activations/layer13_attention_weight_max": 39.293983459472656,
      "activations/layer13_attention_weight_min": -34.779624938964844,
      "activations/layer14_attention_weight_max": 42.24100875854492,
      "activations/layer14_attention_weight_min": -36.08778762817383,
      "activations/layer15_attention_weight_max": 36.94352722167969,
      "activations/layer15_attention_weight_min": -32.04420852661133,
      "activations/layer16_attention_weight_max": 28.828447341918945,
      "activations/layer16_attention_weight_min": -28.300121307373047,
      "activations/layer17_attention_weight_max": 34.03215026855469,
      "activations/layer17_attention_weight_min": -24.69335174560547,
      "activations/layer18_attention_weight_max": 33.42267990112305,
      "activations/layer18_attention_weight_min": -23.216732025146484,
      "activations/layer19_attention_weight_max": 31.81647300720215,
      "activations/layer19_attention_weight_min": -30.69370460510254,
      "activations/layer1_attention_weight_max": 17.257219314575195,
      "activations/layer1_attention_weight_min": -15.983790397644043,
      "activations/layer20_attention_weight_max": 27.35223388671875,
      "activations/layer20_attention_weight_min": -23.470306396484375,
      "activations/layer21_attention_weight_max": 26.52720832824707,
      "activations/layer21_attention_weight_min": -24.075408935546875,
      "activations/layer22_attention_weight_max": 39.626426696777344,
      "activations/layer22_attention_weight_min": -28.57048988342285,
      "activations/layer23_attention_weight_max": 31.642345428466797,
      "activations/layer23_attention_weight_min": -23.79091453552246,
      "activations/layer2_attention_weight_max": 35.49641418457031,
      "activations/layer2_attention_weight_min": -37.94641876220703,
      "activations/layer3_attention_weight_max": 104.49832153320312,
      "activations/layer3_attention_weight_min": -109.41019439697266,
      "activations/layer4_attention_weight_max": 117.25516510009766,
      "activations/layer4_attention_weight_min": -119.11016082763672,
      "activations/layer5_attention_weight_max": 54.63407897949219,
      "activations/layer5_attention_weight_min": -63.46822738647461,
      "activations/layer6_attention_weight_max": 45.54325866699219,
      "activations/layer6_attention_weight_min": -46.37246322631836,
      "activations/layer7_attention_weight_max": 93.3350601196289,
      "activations/layer7_attention_weight_min": -92.24531555175781,
      "activations/layer8_attention_weight_max": 42.86190414428711,
      "activations/layer8_attention_weight_min": -44.18024444580078,
      "activations/layer9_attention_weight_max": 35.69904708862305,
      "activations/layer9_attention_weight_min": -35.58832550048828,
      "epoch": 13.48,
      "learning_rate": 6.369015151515151e-05,
      "loss": 2.7709,
      "step": 232050
    },
    {
      "activations/layer0_attention_weight_max": 15.738887786865234,
      "activations/layer0_attention_weight_min": -12.773819923400879,
      "activations/layer10_attention_weight_max": 34.372249603271484,
      "activations/layer10_attention_weight_min": -32.396297454833984,
      "activations/layer11_attention_weight_max": 34.67937469482422,
      "activations/layer11_attention_weight_min": -32.93148422241211,
      "activations/layer12_attention_weight_max": 35.33414840698242,
      "activations/layer12_attention_weight_min": -25.36199378967285,
      "activations/layer13_attention_weight_max": 47.61137390136719,
      "activations/layer13_attention_weight_min": -35.211570739746094,
      "activations/layer14_attention_weight_max": 51.20902633666992,
      "activations/layer14_attention_weight_min": -35.831390380859375,
      "activations/layer15_attention_weight_max": 50.04069900512695,
      "activations/layer15_attention_weight_min": -35.35562515258789,
      "activations/layer16_attention_weight_max": 34.2857780456543,
      "activations/layer16_attention_weight_min": -27.716251373291016,
      "activations/layer17_attention_weight_max": 34.534812927246094,
      "activations/layer17_attention_weight_min": -24.804269790649414,
      "activations/layer18_attention_weight_max": 28.954059600830078,
      "activations/layer18_attention_weight_min": -23.774417877197266,
      "activations/layer19_attention_weight_max": 33.27196502685547,
      "activations/layer19_attention_weight_min": -29.899412155151367,
      "activations/layer1_attention_weight_max": 17.5506649017334,
      "activations/layer1_attention_weight_min": -16.511600494384766,
      "activations/layer20_attention_weight_max": 29.520069122314453,
      "activations/layer20_attention_weight_min": -23.92616844177246,
      "activations/layer21_attention_weight_max": 25.582834243774414,
      "activations/layer21_attention_weight_min": -24.055017471313477,
      "activations/layer22_attention_weight_max": 37.955772399902344,
      "activations/layer22_attention_weight_min": -29.0673885345459,
      "activations/layer23_attention_weight_max": 31.004920959472656,
      "activations/layer23_attention_weight_min": -24.884441375732422,
      "activations/layer2_attention_weight_max": 33.43698501586914,
      "activations/layer2_attention_weight_min": -32.65060043334961,
      "activations/layer3_attention_weight_max": 95.62037658691406,
      "activations/layer3_attention_weight_min": -101.71267700195312,
      "activations/layer4_attention_weight_max": 109.89131927490234,
      "activations/layer4_attention_weight_min": -107.77938079833984,
      "activations/layer5_attention_weight_max": 53.056480407714844,
      "activations/layer5_attention_weight_min": -60.2066650390625,
      "activations/layer6_attention_weight_max": 41.52758026123047,
      "activations/layer6_attention_weight_min": -43.77952575683594,
      "activations/layer7_attention_weight_max": 89.75335693359375,
      "activations/layer7_attention_weight_min": -84.03463745117188,
      "activations/layer8_attention_weight_max": 40.26156997680664,
      "activations/layer8_attention_weight_min": -41.83709716796875,
      "activations/layer9_attention_weight_max": 32.12883758544922,
      "activations/layer9_attention_weight_min": -32.44075012207031,
      "epoch": 13.49,
      "learning_rate": 6.367121212121211e-05,
      "loss": 2.7643,
      "step": 232100
    },
    {
      "activations/layer0_attention_weight_max": 16.433887481689453,
      "activations/layer0_attention_weight_min": -13.116954803466797,
      "activations/layer10_attention_weight_max": 31.173824310302734,
      "activations/layer10_attention_weight_min": -31.292936325073242,
      "activations/layer11_attention_weight_max": 33.15571212768555,
      "activations/layer11_attention_weight_min": -29.980817794799805,
      "activations/layer12_attention_weight_max": 25.477813720703125,
      "activations/layer12_attention_weight_min": -25.437057495117188,
      "activations/layer13_attention_weight_max": 41.95252227783203,
      "activations/layer13_attention_weight_min": -32.5550537109375,
      "activations/layer14_attention_weight_max": 40.02119064331055,
      "activations/layer14_attention_weight_min": -34.54922103881836,
      "activations/layer15_attention_weight_max": 39.32266616821289,
      "activations/layer15_attention_weight_min": -32.23015213012695,
      "activations/layer16_attention_weight_max": 31.43523406982422,
      "activations/layer16_attention_weight_min": -25.39337730407715,
      "activations/layer17_attention_weight_max": 31.51551628112793,
      "activations/layer17_attention_weight_min": -25.27842903137207,
      "activations/layer18_attention_weight_max": 33.30534362792969,
      "activations/layer18_attention_weight_min": -25.221696853637695,
      "activations/layer19_attention_weight_max": 31.66416358947754,
      "activations/layer19_attention_weight_min": -29.953336715698242,
      "activations/layer1_attention_weight_max": 16.44038963317871,
      "activations/layer1_attention_weight_min": -14.680807113647461,
      "activations/layer20_attention_weight_max": 29.697898864746094,
      "activations/layer20_attention_weight_min": -25.517221450805664,
      "activations/layer21_attention_weight_max": 28.392301559448242,
      "activations/layer21_attention_weight_min": -24.972240447998047,
      "activations/layer22_attention_weight_max": 42.47694396972656,
      "activations/layer22_attention_weight_min": -31.472883224487305,
      "activations/layer23_attention_weight_max": 30.30463981628418,
      "activations/layer23_attention_weight_min": -26.21095848083496,
      "activations/layer2_attention_weight_max": 34.353981018066406,
      "activations/layer2_attention_weight_min": -33.65705871582031,
      "activations/layer3_attention_weight_max": 96.27206420898438,
      "activations/layer3_attention_weight_min": -95.85482025146484,
      "activations/layer4_attention_weight_max": 108.80648040771484,
      "activations/layer4_attention_weight_min": -108.1458511352539,
      "activations/layer5_attention_weight_max": 49.81089782714844,
      "activations/layer5_attention_weight_min": -60.98210144042969,
      "activations/layer6_attention_weight_max": 42.57367706298828,
      "activations/layer6_attention_weight_min": -45.2214241027832,
      "activations/layer7_attention_weight_max": 84.18909454345703,
      "activations/layer7_attention_weight_min": -87.12825012207031,
      "activations/layer8_attention_weight_max": 37.336063385009766,
      "activations/layer8_attention_weight_min": -40.245338439941406,
      "activations/layer9_attention_weight_max": 31.015798568725586,
      "activations/layer9_attention_weight_min": -30.239578247070312,
      "epoch": 13.49,
      "learning_rate": 6.365227272727272e-05,
      "loss": 2.7489,
      "step": 232150
    },
    {
      "activations/layer0_attention_weight_max": 16.12675666809082,
      "activations/layer0_attention_weight_min": -12.703038215637207,
      "activations/layer10_attention_weight_max": 46.44928741455078,
      "activations/layer10_attention_weight_min": -41.37958908081055,
      "activations/layer11_attention_weight_max": 45.496238708496094,
      "activations/layer11_attention_weight_min": -40.27854537963867,
      "activations/layer12_attention_weight_max": 37.34523391723633,
      "activations/layer12_attention_weight_min": -27.82270050048828,
      "activations/layer13_attention_weight_max": 55.715946197509766,
      "activations/layer13_attention_weight_min": -36.01057434082031,
      "activations/layer14_attention_weight_max": 49.450443267822266,
      "activations/layer14_attention_weight_min": -40.10371398925781,
      "activations/layer15_attention_weight_max": 53.621368408203125,
      "activations/layer15_attention_weight_min": -38.62670135498047,
      "activations/layer16_attention_weight_max": 34.65982437133789,
      "activations/layer16_attention_weight_min": -26.81592559814453,
      "activations/layer17_attention_weight_max": 29.99497413635254,
      "activations/layer17_attention_weight_min": -25.969573974609375,
      "activations/layer18_attention_weight_max": 29.72933578491211,
      "activations/layer18_attention_weight_min": -24.74589729309082,
      "activations/layer19_attention_weight_max": 36.51607894897461,
      "activations/layer19_attention_weight_min": -31.5539493560791,
      "activations/layer1_attention_weight_max": 16.59473419189453,
      "activations/layer1_attention_weight_min": -14.978952407836914,
      "activations/layer20_attention_weight_max": 29.624494552612305,
      "activations/layer20_attention_weight_min": -25.776437759399414,
      "activations/layer21_attention_weight_max": 27.72119140625,
      "activations/layer21_attention_weight_min": -24.903831481933594,
      "activations/layer22_attention_weight_max": 41.36783981323242,
      "activations/layer22_attention_weight_min": -29.809492111206055,
      "activations/layer23_attention_weight_max": 35.00541687011719,
      "activations/layer23_attention_weight_min": -25.388662338256836,
      "activations/layer2_attention_weight_max": 38.76079559326172,
      "activations/layer2_attention_weight_min": -36.90462875366211,
      "activations/layer3_attention_weight_max": 113.66140747070312,
      "activations/layer3_attention_weight_min": -108.64701843261719,
      "activations/layer4_attention_weight_max": 131.2513885498047,
      "activations/layer4_attention_weight_min": -111.65361785888672,
      "activations/layer5_attention_weight_max": 58.58268737792969,
      "activations/layer5_attention_weight_min": -63.83445739746094,
      "activations/layer6_attention_weight_max": 46.06016159057617,
      "activations/layer6_attention_weight_min": -44.964054107666016,
      "activations/layer7_attention_weight_max": 114.81192016601562,
      "activations/layer7_attention_weight_min": -95.051025390625,
      "activations/layer8_attention_weight_max": 50.47700500488281,
      "activations/layer8_attention_weight_min": -47.48057556152344,
      "activations/layer9_attention_weight_max": 42.94929504394531,
      "activations/layer9_attention_weight_min": -40.66233444213867,
      "epoch": 13.49,
      "learning_rate": 6.363333333333333e-05,
      "loss": 2.7693,
      "step": 232200
    },
    {
      "activations/layer0_attention_weight_max": 15.16272258758545,
      "activations/layer0_attention_weight_min": -12.71715259552002,
      "activations/layer10_attention_weight_max": 32.33549118041992,
      "activations/layer10_attention_weight_min": -32.64592361450195,
      "activations/layer11_attention_weight_max": 31.815250396728516,
      "activations/layer11_attention_weight_min": -30.539907455444336,
      "activations/layer12_attention_weight_max": 26.341941833496094,
      "activations/layer12_attention_weight_min": -24.94513702392578,
      "activations/layer13_attention_weight_max": 39.060020446777344,
      "activations/layer13_attention_weight_min": -34.384159088134766,
      "activations/layer14_attention_weight_max": 43.63539505004883,
      "activations/layer14_attention_weight_min": -37.83733367919922,
      "activations/layer15_attention_weight_max": 37.0540885925293,
      "activations/layer15_attention_weight_min": -32.66101837158203,
      "activations/layer16_attention_weight_max": 30.707828521728516,
      "activations/layer16_attention_weight_min": -27.163509368896484,
      "activations/layer17_attention_weight_max": 31.475793838500977,
      "activations/layer17_attention_weight_min": -26.499841690063477,
      "activations/layer18_attention_weight_max": 32.689300537109375,
      "activations/layer18_attention_weight_min": -24.509746551513672,
      "activations/layer19_attention_weight_max": 36.78764343261719,
      "activations/layer19_attention_weight_min": -30.43680191040039,
      "activations/layer1_attention_weight_max": 17.043121337890625,
      "activations/layer1_attention_weight_min": -14.115983963012695,
      "activations/layer20_attention_weight_max": 33.82222366333008,
      "activations/layer20_attention_weight_min": -24.430797576904297,
      "activations/layer21_attention_weight_max": 32.353084564208984,
      "activations/layer21_attention_weight_min": -23.469810485839844,
      "activations/layer22_attention_weight_max": 46.45098114013672,
      "activations/layer22_attention_weight_min": -30.385114669799805,
      "activations/layer23_attention_weight_max": 35.08587646484375,
      "activations/layer23_attention_weight_min": -24.0618896484375,
      "activations/layer2_attention_weight_max": 32.3336296081543,
      "activations/layer2_attention_weight_min": -33.38663864135742,
      "activations/layer3_attention_weight_max": 95.07230377197266,
      "activations/layer3_attention_weight_min": -97.56389617919922,
      "activations/layer4_attention_weight_max": 108.4869155883789,
      "activations/layer4_attention_weight_min": -109.87837219238281,
      "activations/layer5_attention_weight_max": 50.66375732421875,
      "activations/layer5_attention_weight_min": -60.997127532958984,
      "activations/layer6_attention_weight_max": 43.66208267211914,
      "activations/layer6_attention_weight_min": -45.519412994384766,
      "activations/layer7_attention_weight_max": 90.9615249633789,
      "activations/layer7_attention_weight_min": -88.89397430419922,
      "activations/layer8_attention_weight_max": 38.93463134765625,
      "activations/layer8_attention_weight_min": -40.91497802734375,
      "activations/layer9_attention_weight_max": 32.905757904052734,
      "activations/layer9_attention_weight_min": -32.4262809753418,
      "epoch": 13.5,
      "learning_rate": 6.361439393939393e-05,
      "loss": 2.7565,
      "step": 232250
    },
    {
      "activations/layer0_attention_weight_max": 15.51210880279541,
      "activations/layer0_attention_weight_min": -12.379624366760254,
      "activations/layer10_attention_weight_max": 32.898529052734375,
      "activations/layer10_attention_weight_min": -32.47018814086914,
      "activations/layer11_attention_weight_max": 34.91783905029297,
      "activations/layer11_attention_weight_min": -33.67308807373047,
      "activations/layer12_attention_weight_max": 25.336078643798828,
      "activations/layer12_attention_weight_min": -27.094274520874023,
      "activations/layer13_attention_weight_max": 38.1439094543457,
      "activations/layer13_attention_weight_min": -37.05769729614258,
      "activations/layer14_attention_weight_max": 42.52247619628906,
      "activations/layer14_attention_weight_min": -39.3654899597168,
      "activations/layer15_attention_weight_max": 35.53667068481445,
      "activations/layer15_attention_weight_min": -32.79683303833008,
      "activations/layer16_attention_weight_max": 30.336206436157227,
      "activations/layer16_attention_weight_min": -27.1615047454834,
      "activations/layer17_attention_weight_max": 29.68532943725586,
      "activations/layer17_attention_weight_min": -25.43992805480957,
      "activations/layer18_attention_weight_max": 31.61650276184082,
      "activations/layer18_attention_weight_min": -23.334491729736328,
      "activations/layer19_attention_weight_max": 30.9464111328125,
      "activations/layer19_attention_weight_min": -30.31495475769043,
      "activations/layer1_attention_weight_max": 15.636201858520508,
      "activations/layer1_attention_weight_min": -13.667506217956543,
      "activations/layer20_attention_weight_max": 26.608964920043945,
      "activations/layer20_attention_weight_min": -24.4346923828125,
      "activations/layer21_attention_weight_max": 27.16092300415039,
      "activations/layer21_attention_weight_min": -23.972673416137695,
      "activations/layer22_attention_weight_max": 41.042686462402344,
      "activations/layer22_attention_weight_min": -29.541290283203125,
      "activations/layer23_attention_weight_max": 32.53547286987305,
      "activations/layer23_attention_weight_min": -24.947559356689453,
      "activations/layer2_attention_weight_max": 31.65194320678711,
      "activations/layer2_attention_weight_min": -32.85701370239258,
      "activations/layer3_attention_weight_max": 96.21763610839844,
      "activations/layer3_attention_weight_min": -100.28829956054688,
      "activations/layer4_attention_weight_max": 109.3034896850586,
      "activations/layer4_attention_weight_min": -111.28194427490234,
      "activations/layer5_attention_weight_max": 50.654930114746094,
      "activations/layer5_attention_weight_min": -64.1502914428711,
      "activations/layer6_attention_weight_max": 44.47019577026367,
      "activations/layer6_attention_weight_min": -48.07358169555664,
      "activations/layer7_attention_weight_max": 88.38738250732422,
      "activations/layer7_attention_weight_min": -97.35002136230469,
      "activations/layer8_attention_weight_max": 39.7841682434082,
      "activations/layer8_attention_weight_min": -43.061256408691406,
      "activations/layer9_attention_weight_max": 32.84019470214844,
      "activations/layer9_attention_weight_min": -31.81381607055664,
      "epoch": 13.5,
      "learning_rate": 6.359545454545454e-05,
      "loss": 2.7794,
      "step": 232300
    },
    {
      "activations/layer0_attention_weight_max": 16.32249641418457,
      "activations/layer0_attention_weight_min": -12.514623641967773,
      "activations/layer10_attention_weight_max": 30.976457595825195,
      "activations/layer10_attention_weight_min": -30.73617172241211,
      "activations/layer11_attention_weight_max": 31.272235870361328,
      "activations/layer11_attention_weight_min": -31.128835678100586,
      "activations/layer12_attention_weight_max": 24.449487686157227,
      "activations/layer12_attention_weight_min": -26.744726181030273,
      "activations/layer13_attention_weight_max": 37.16742706298828,
      "activations/layer13_attention_weight_min": -35.336761474609375,
      "activations/layer14_attention_weight_max": 43.5211181640625,
      "activations/layer14_attention_weight_min": -39.10908889770508,
      "activations/layer15_attention_weight_max": 36.40081787109375,
      "activations/layer15_attention_weight_min": -32.859153747558594,
      "activations/layer16_attention_weight_max": 30.80885887145996,
      "activations/layer16_attention_weight_min": -28.117929458618164,
      "activations/layer17_attention_weight_max": 30.947818756103516,
      "activations/layer17_attention_weight_min": -26.60985565185547,
      "activations/layer18_attention_weight_max": 32.802005767822266,
      "activations/layer18_attention_weight_min": -24.226970672607422,
      "activations/layer19_attention_weight_max": 32.270591735839844,
      "activations/layer19_attention_weight_min": -30.14145278930664,
      "activations/layer1_attention_weight_max": 16.450260162353516,
      "activations/layer1_attention_weight_min": -15.71522045135498,
      "activations/layer20_attention_weight_max": 28.416297912597656,
      "activations/layer20_attention_weight_min": -25.19186019897461,
      "activations/layer21_attention_weight_max": 28.22627830505371,
      "activations/layer21_attention_weight_min": -25.9271183013916,
      "activations/layer22_attention_weight_max": 42.54574203491211,
      "activations/layer22_attention_weight_min": -31.557296752929688,
      "activations/layer23_attention_weight_max": 34.89478302001953,
      "activations/layer23_attention_weight_min": -25.68628692626953,
      "activations/layer2_attention_weight_max": 33.29405212402344,
      "activations/layer2_attention_weight_min": -31.554462432861328,
      "activations/layer3_attention_weight_max": 98.1016845703125,
      "activations/layer3_attention_weight_min": -97.18682861328125,
      "activations/layer4_attention_weight_max": 109.4958724975586,
      "activations/layer4_attention_weight_min": -109.14697265625,
      "activations/layer5_attention_weight_max": 52.15442657470703,
      "activations/layer5_attention_weight_min": -63.086055755615234,
      "activations/layer6_attention_weight_max": 44.81415557861328,
      "activations/layer6_attention_weight_min": -45.646629333496094,
      "activations/layer7_attention_weight_max": 90.68665313720703,
      "activations/layer7_attention_weight_min": -91.2416000366211,
      "activations/layer8_attention_weight_max": 39.820884704589844,
      "activations/layer8_attention_weight_min": -39.18203353881836,
      "activations/layer9_attention_weight_max": 31.73310661315918,
      "activations/layer9_attention_weight_min": -31.286874771118164,
      "epoch": 13.5,
      "learning_rate": 6.357651515151514e-05,
      "loss": 2.7653,
      "step": 232350
    },
    {
      "activations/layer0_attention_weight_max": 16.177053451538086,
      "activations/layer0_attention_weight_min": -12.80974006652832,
      "activations/layer10_attention_weight_max": 37.49813461303711,
      "activations/layer10_attention_weight_min": -36.43499755859375,
      "activations/layer11_attention_weight_max": 36.05323791503906,
      "activations/layer11_attention_weight_min": -34.69440460205078,
      "activations/layer12_attention_weight_max": 25.740131378173828,
      "activations/layer12_attention_weight_min": -26.25019645690918,
      "activations/layer13_attention_weight_max": 38.41166687011719,
      "activations/layer13_attention_weight_min": -35.019447326660156,
      "activations/layer14_attention_weight_max": 40.072593688964844,
      "activations/layer14_attention_weight_min": -37.30055236816406,
      "activations/layer15_attention_weight_max": 34.1427116394043,
      "activations/layer15_attention_weight_min": -33.056766510009766,
      "activations/layer16_attention_weight_max": 28.22355842590332,
      "activations/layer16_attention_weight_min": -28.593936920166016,
      "activations/layer17_attention_weight_max": 28.696725845336914,
      "activations/layer17_attention_weight_min": -26.165719985961914,
      "activations/layer18_attention_weight_max": 27.538776397705078,
      "activations/layer18_attention_weight_min": -23.379220962524414,
      "activations/layer19_attention_weight_max": 32.27793502807617,
      "activations/layer19_attention_weight_min": -32.71381378173828,
      "activations/layer1_attention_weight_max": 17.238109588623047,
      "activations/layer1_attention_weight_min": -15.132436752319336,
      "activations/layer20_attention_weight_max": 28.7628116607666,
      "activations/layer20_attention_weight_min": -25.41569709777832,
      "activations/layer21_attention_weight_max": 30.072660446166992,
      "activations/layer21_attention_weight_min": -24.673282623291016,
      "activations/layer22_attention_weight_max": 36.787506103515625,
      "activations/layer22_attention_weight_min": -30.732717514038086,
      "activations/layer23_attention_weight_max": 31.022674560546875,
      "activations/layer23_attention_weight_min": -25.449642181396484,
      "activations/layer2_attention_weight_max": 35.72602844238281,
      "activations/layer2_attention_weight_min": -35.40380096435547,
      "activations/layer3_attention_weight_max": 99.53630828857422,
      "activations/layer3_attention_weight_min": -107.3127670288086,
      "activations/layer4_attention_weight_max": 109.34832763671875,
      "activations/layer4_attention_weight_min": -111.35780334472656,
      "activations/layer5_attention_weight_max": 54.28264617919922,
      "activations/layer5_attention_weight_min": -62.93378829956055,
      "activations/layer6_attention_weight_max": 45.612648010253906,
      "activations/layer6_attention_weight_min": -49.10987091064453,
      "activations/layer7_attention_weight_max": 90.79774475097656,
      "activations/layer7_attention_weight_min": -95.35767364501953,
      "activations/layer8_attention_weight_max": 42.621944427490234,
      "activations/layer8_attention_weight_min": -42.75551986694336,
      "activations/layer9_attention_weight_max": 34.8014030456543,
      "activations/layer9_attention_weight_min": -36.197147369384766,
      "epoch": 13.5,
      "learning_rate": 6.355757575757575e-05,
      "loss": 2.7538,
      "step": 232400
    },
    {
      "activations/layer0_attention_weight_max": 15.554304122924805,
      "activations/layer0_attention_weight_min": -11.809664726257324,
      "activations/layer10_attention_weight_max": 37.64849853515625,
      "activations/layer10_attention_weight_min": -36.964195251464844,
      "activations/layer11_attention_weight_max": 38.87060546875,
      "activations/layer11_attention_weight_min": -35.87050247192383,
      "activations/layer12_attention_weight_max": 28.47425079345703,
      "activations/layer12_attention_weight_min": -26.530057907104492,
      "activations/layer13_attention_weight_max": 45.43461608886719,
      "activations/layer13_attention_weight_min": -36.4847297668457,
      "activations/layer14_attention_weight_max": 50.9043083190918,
      "activations/layer14_attention_weight_min": -39.26649856567383,
      "activations/layer15_attention_weight_max": 46.559410095214844,
      "activations/layer15_attention_weight_min": -33.987911224365234,
      "activations/layer16_attention_weight_max": 34.75076675415039,
      "activations/layer16_attention_weight_min": -29.282869338989258,
      "activations/layer17_attention_weight_max": 37.9178466796875,
      "activations/layer17_attention_weight_min": -27.1048641204834,
      "activations/layer18_attention_weight_max": 38.42455291748047,
      "activations/layer18_attention_weight_min": -24.373445510864258,
      "activations/layer19_attention_weight_max": 42.38486862182617,
      "activations/layer19_attention_weight_min": -30.944448471069336,
      "activations/layer1_attention_weight_max": 16.646282196044922,
      "activations/layer1_attention_weight_min": -15.474854469299316,
      "activations/layer20_attention_weight_max": 33.021263122558594,
      "activations/layer20_attention_weight_min": -25.949384689331055,
      "activations/layer21_attention_weight_max": 31.804302215576172,
      "activations/layer21_attention_weight_min": -23.865507125854492,
      "activations/layer22_attention_weight_max": 53.9130859375,
      "activations/layer22_attention_weight_min": -30.732194900512695,
      "activations/layer23_attention_weight_max": 39.78931427001953,
      "activations/layer23_attention_weight_min": -24.86566162109375,
      "activations/layer2_attention_weight_max": 34.662315368652344,
      "activations/layer2_attention_weight_min": -34.25855255126953,
      "activations/layer3_attention_weight_max": 102.45093536376953,
      "activations/layer3_attention_weight_min": -100.75362396240234,
      "activations/layer4_attention_weight_max": 118.48797607421875,
      "activations/layer4_attention_weight_min": -116.33504486083984,
      "activations/layer5_attention_weight_max": 54.20756530761719,
      "activations/layer5_attention_weight_min": -65.241455078125,
      "activations/layer6_attention_weight_max": 47.118038177490234,
      "activations/layer6_attention_weight_min": -49.64015579223633,
      "activations/layer7_attention_weight_max": 95.27845001220703,
      "activations/layer7_attention_weight_min": -97.53258514404297,
      "activations/layer8_attention_weight_max": 46.24311828613281,
      "activations/layer8_attention_weight_min": -47.105865478515625,
      "activations/layer9_attention_weight_max": 36.93052291870117,
      "activations/layer9_attention_weight_min": -37.61026382446289,
      "epoch": 13.51,
      "learning_rate": 6.353863636363636e-05,
      "loss": 2.7972,
      "step": 232450
    },
    {
      "activations/layer0_attention_weight_max": 15.390323638916016,
      "activations/layer0_attention_weight_min": -12.930949211120605,
      "activations/layer10_attention_weight_max": 33.86177444458008,
      "activations/layer10_attention_weight_min": -31.70012664794922,
      "activations/layer11_attention_weight_max": 33.93402862548828,
      "activations/layer11_attention_weight_min": -31.30695915222168,
      "activations/layer12_attention_weight_max": 27.898292541503906,
      "activations/layer12_attention_weight_min": -25.948623657226562,
      "activations/layer13_attention_weight_max": 41.58449172973633,
      "activations/layer13_attention_weight_min": -34.74988555908203,
      "activations/layer14_attention_weight_max": 49.020530700683594,
      "activations/layer14_attention_weight_min": -38.674678802490234,
      "activations/layer15_attention_weight_max": 40.49176788330078,
      "activations/layer15_attention_weight_min": -35.123558044433594,
      "activations/layer16_attention_weight_max": 32.444026947021484,
      "activations/layer16_attention_weight_min": -27.845888137817383,
      "activations/layer17_attention_weight_max": 31.55131721496582,
      "activations/layer17_attention_weight_min": -26.623916625976562,
      "activations/layer18_attention_weight_max": 33.378910064697266,
      "activations/layer18_attention_weight_min": -24.328289031982422,
      "activations/layer19_attention_weight_max": 37.12183380126953,
      "activations/layer19_attention_weight_min": -30.289945602416992,
      "activations/layer1_attention_weight_max": 16.49539566040039,
      "activations/layer1_attention_weight_min": -14.157613754272461,
      "activations/layer20_attention_weight_max": 33.10248565673828,
      "activations/layer20_attention_weight_min": -24.9208984375,
      "activations/layer21_attention_weight_max": 32.832252502441406,
      "activations/layer21_attention_weight_min": -23.810293197631836,
      "activations/layer22_attention_weight_max": 45.76141357421875,
      "activations/layer22_attention_weight_min": -31.695804595947266,
      "activations/layer23_attention_weight_max": 34.07373046875,
      "activations/layer23_attention_weight_min": -26.580978393554688,
      "activations/layer2_attention_weight_max": 33.02384948730469,
      "activations/layer2_attention_weight_min": -32.11114501953125,
      "activations/layer3_attention_weight_max": 96.28998565673828,
      "activations/layer3_attention_weight_min": -99.47734069824219,
      "activations/layer4_attention_weight_max": 110.4775161743164,
      "activations/layer4_attention_weight_min": -117.04939270019531,
      "activations/layer5_attention_weight_max": 52.65203094482422,
      "activations/layer5_attention_weight_min": -65.0595474243164,
      "activations/layer6_attention_weight_max": 44.41213607788086,
      "activations/layer6_attention_weight_min": -48.56208038330078,
      "activations/layer7_attention_weight_max": 91.39945220947266,
      "activations/layer7_attention_weight_min": -96.296875,
      "activations/layer8_attention_weight_max": 40.47777557373047,
      "activations/layer8_attention_weight_min": -41.0489387512207,
      "activations/layer9_attention_weight_max": 32.87458038330078,
      "activations/layer9_attention_weight_min": -33.758663177490234,
      "epoch": 13.51,
      "learning_rate": 6.351969696969696e-05,
      "loss": 2.7478,
      "step": 232500
    },
    {
      "activations/layer0_attention_weight_max": 16.240013122558594,
      "activations/layer0_attention_weight_min": -12.988944053649902,
      "activations/layer10_attention_weight_max": 37.15788650512695,
      "activations/layer10_attention_weight_min": -34.44193649291992,
      "activations/layer11_attention_weight_max": 34.501251220703125,
      "activations/layer11_attention_weight_min": -34.36542510986328,
      "activations/layer12_attention_weight_max": 25.387008666992188,
      "activations/layer12_attention_weight_min": -25.78584098815918,
      "activations/layer13_attention_weight_max": 39.0996208190918,
      "activations/layer13_attention_weight_min": -34.319732666015625,
      "activations/layer14_attention_weight_max": 44.259033203125,
      "activations/layer14_attention_weight_min": -37.419803619384766,
      "activations/layer15_attention_weight_max": 37.67093276977539,
      "activations/layer15_attention_weight_min": -32.97501754760742,
      "activations/layer16_attention_weight_max": 30.244083404541016,
      "activations/layer16_attention_weight_min": -27.932226181030273,
      "activations/layer17_attention_weight_max": 30.26144790649414,
      "activations/layer17_attention_weight_min": -26.939311981201172,
      "activations/layer18_attention_weight_max": 30.68770980834961,
      "activations/layer18_attention_weight_min": -24.379179000854492,
      "activations/layer19_attention_weight_max": 32.254573822021484,
      "activations/layer19_attention_weight_min": -29.341068267822266,
      "activations/layer1_attention_weight_max": 16.07917022705078,
      "activations/layer1_attention_weight_min": -15.215716361999512,
      "activations/layer20_attention_weight_max": 28.939611434936523,
      "activations/layer20_attention_weight_min": -24.820045471191406,
      "activations/layer21_attention_weight_max": 27.18312644958496,
      "activations/layer21_attention_weight_min": -23.294309616088867,
      "activations/layer22_attention_weight_max": 45.48344039916992,
      "activations/layer22_attention_weight_min": -29.818729400634766,
      "activations/layer23_attention_weight_max": 33.424530029296875,
      "activations/layer23_attention_weight_min": -25.102142333984375,
      "activations/layer2_attention_weight_max": 33.91474151611328,
      "activations/layer2_attention_weight_min": -33.61988830566406,
      "activations/layer3_attention_weight_max": 97.36029815673828,
      "activations/layer3_attention_weight_min": -101.81389617919922,
      "activations/layer4_attention_weight_max": 111.05843353271484,
      "activations/layer4_attention_weight_min": -109.86732482910156,
      "activations/layer5_attention_weight_max": 51.699581146240234,
      "activations/layer5_attention_weight_min": -62.132415771484375,
      "activations/layer6_attention_weight_max": 45.849647521972656,
      "activations/layer6_attention_weight_min": -48.25496292114258,
      "activations/layer7_attention_weight_max": 92.00947570800781,
      "activations/layer7_attention_weight_min": -98.82776641845703,
      "activations/layer8_attention_weight_max": 42.69546890258789,
      "activations/layer8_attention_weight_min": -43.399051666259766,
      "activations/layer9_attention_weight_max": 33.24748992919922,
      "activations/layer9_attention_weight_min": -36.618160247802734,
      "epoch": 13.51,
      "learning_rate": 6.350075757575758e-05,
      "loss": 2.7681,
      "step": 232550
    },
    {
      "activations/layer0_attention_weight_max": 16.370901107788086,
      "activations/layer0_attention_weight_min": -12.719114303588867,
      "activations/layer10_attention_weight_max": 33.305694580078125,
      "activations/layer10_attention_weight_min": -34.68057632446289,
      "activations/layer11_attention_weight_max": 32.12855529785156,
      "activations/layer11_attention_weight_min": -33.41547393798828,
      "activations/layer12_attention_weight_max": 26.803035736083984,
      "activations/layer12_attention_weight_min": -25.718854904174805,
      "activations/layer13_attention_weight_max": 38.95874786376953,
      "activations/layer13_attention_weight_min": -34.79936981201172,
      "activations/layer14_attention_weight_max": 41.72623825073242,
      "activations/layer14_attention_weight_min": -37.25166702270508,
      "activations/layer15_attention_weight_max": 35.90079116821289,
      "activations/layer15_attention_weight_min": -31.951536178588867,
      "activations/layer16_attention_weight_max": 29.138601303100586,
      "activations/layer16_attention_weight_min": -28.85415267944336,
      "activations/layer17_attention_weight_max": 30.52976417541504,
      "activations/layer17_attention_weight_min": -26.182279586791992,
      "activations/layer18_attention_weight_max": 31.736440658569336,
      "activations/layer18_attention_weight_min": -24.06015968322754,
      "activations/layer19_attention_weight_max": 35.358154296875,
      "activations/layer19_attention_weight_min": -30.29302978515625,
      "activations/layer1_attention_weight_max": 16.307039260864258,
      "activations/layer1_attention_weight_min": -15.520798683166504,
      "activations/layer20_attention_weight_max": 29.443614959716797,
      "activations/layer20_attention_weight_min": -25.8050594329834,
      "activations/layer21_attention_weight_max": 30.961334228515625,
      "activations/layer21_attention_weight_min": -24.695119857788086,
      "activations/layer22_attention_weight_max": 41.897525787353516,
      "activations/layer22_attention_weight_min": -29.913360595703125,
      "activations/layer23_attention_weight_max": 32.342315673828125,
      "activations/layer23_attention_weight_min": -24.33277130126953,
      "activations/layer2_attention_weight_max": 34.606109619140625,
      "activations/layer2_attention_weight_min": -34.88767623901367,
      "activations/layer3_attention_weight_max": 101.32289123535156,
      "activations/layer3_attention_weight_min": -102.55406951904297,
      "activations/layer4_attention_weight_max": 114.68782806396484,
      "activations/layer4_attention_weight_min": -114.33294677734375,
      "activations/layer5_attention_weight_max": 54.54526901245117,
      "activations/layer5_attention_weight_min": -64.07373809814453,
      "activations/layer6_attention_weight_max": 46.59072494506836,
      "activations/layer6_attention_weight_min": -45.50450897216797,
      "activations/layer7_attention_weight_max": 91.24830627441406,
      "activations/layer7_attention_weight_min": -91.21881866455078,
      "activations/layer8_attention_weight_max": 41.027652740478516,
      "activations/layer8_attention_weight_min": -41.179161071777344,
      "activations/layer9_attention_weight_max": 33.53401565551758,
      "activations/layer9_attention_weight_min": -33.55984115600586,
      "epoch": 13.52,
      "learning_rate": 6.348219696969697e-05,
      "loss": 2.7571,
      "step": 232600
    },
    {
      "activations/layer0_attention_weight_max": 16.729063034057617,
      "activations/layer0_attention_weight_min": -11.963861465454102,
      "activations/layer10_attention_weight_max": 32.7226676940918,
      "activations/layer10_attention_weight_min": -34.65388870239258,
      "activations/layer11_attention_weight_max": 33.77734375,
      "activations/layer11_attention_weight_min": -32.6523323059082,
      "activations/layer12_attention_weight_max": 23.714109420776367,
      "activations/layer12_attention_weight_min": -26.540328979492188,
      "activations/layer13_attention_weight_max": 35.40168762207031,
      "activations/layer13_attention_weight_min": -36.45977020263672,
      "activations/layer14_attention_weight_max": 39.57451629638672,
      "activations/layer14_attention_weight_min": -37.892093658447266,
      "activations/layer15_attention_weight_max": 37.629783630371094,
      "activations/layer15_attention_weight_min": -33.217132568359375,
      "activations/layer16_attention_weight_max": 36.78102493286133,
      "activations/layer16_attention_weight_min": -28.803752899169922,
      "activations/layer17_attention_weight_max": 29.6672306060791,
      "activations/layer17_attention_weight_min": -26.870874404907227,
      "activations/layer18_attention_weight_max": 28.695945739746094,
      "activations/layer18_attention_weight_min": -24.118242263793945,
      "activations/layer19_attention_weight_max": 30.60853385925293,
      "activations/layer19_attention_weight_min": -32.16344451904297,
      "activations/layer1_attention_weight_max": 16.292325973510742,
      "activations/layer1_attention_weight_min": -14.51668643951416,
      "activations/layer20_attention_weight_max": 28.974468231201172,
      "activations/layer20_attention_weight_min": -25.72284507751465,
      "activations/layer21_attention_weight_max": 27.86964988708496,
      "activations/layer21_attention_weight_min": -25.531845092773438,
      "activations/layer22_attention_weight_max": 44.06968688964844,
      "activations/layer22_attention_weight_min": -31.780635833740234,
      "activations/layer23_attention_weight_max": 30.65699577331543,
      "activations/layer23_attention_weight_min": -25.382545471191406,
      "activations/layer2_attention_weight_max": 34.26127624511719,
      "activations/layer2_attention_weight_min": -34.05112838745117,
      "activations/layer3_attention_weight_max": 99.5260009765625,
      "activations/layer3_attention_weight_min": -102.77783966064453,
      "activations/layer4_attention_weight_max": 113.79768371582031,
      "activations/layer4_attention_weight_min": -119.19561767578125,
      "activations/layer5_attention_weight_max": 51.92710876464844,
      "activations/layer5_attention_weight_min": -68.81410217285156,
      "activations/layer6_attention_weight_max": 46.77617263793945,
      "activations/layer6_attention_weight_min": -50.020301818847656,
      "activations/layer7_attention_weight_max": 89.47322845458984,
      "activations/layer7_attention_weight_min": -96.7468032836914,
      "activations/layer8_attention_weight_max": 41.4742546081543,
      "activations/layer8_attention_weight_min": -43.90990447998047,
      "activations/layer9_attention_weight_max": 33.55635452270508,
      "activations/layer9_attention_weight_min": -35.23527145385742,
      "epoch": 13.52,
      "learning_rate": 6.346325757575757e-05,
      "loss": 2.7751,
      "step": 232650
    },
    {
      "activations/layer0_attention_weight_max": 17.397972106933594,
      "activations/layer0_attention_weight_min": -12.714346885681152,
      "activations/layer10_attention_weight_max": 33.04932403564453,
      "activations/layer10_attention_weight_min": -31.62692642211914,
      "activations/layer11_attention_weight_max": 34.431724548339844,
      "activations/layer11_attention_weight_min": -32.32908630371094,
      "activations/layer12_attention_weight_max": 24.83991813659668,
      "activations/layer12_attention_weight_min": -26.06303596496582,
      "activations/layer13_attention_weight_max": 37.92710876464844,
      "activations/layer13_attention_weight_min": -39.59229278564453,
      "activations/layer14_attention_weight_max": 42.11243438720703,
      "activations/layer14_attention_weight_min": -39.755584716796875,
      "activations/layer15_attention_weight_max": 38.12449264526367,
      "activations/layer15_attention_weight_min": -33.92918395996094,
      "activations/layer16_attention_weight_max": 31.15235137939453,
      "activations/layer16_attention_weight_min": -28.9401912689209,
      "activations/layer17_attention_weight_max": 32.3757209777832,
      "activations/layer17_attention_weight_min": -25.78374671936035,
      "activations/layer18_attention_weight_max": 32.113800048828125,
      "activations/layer18_attention_weight_min": -25.5063533782959,
      "activations/layer19_attention_weight_max": 33.71928024291992,
      "activations/layer19_attention_weight_min": -32.132301330566406,
      "activations/layer1_attention_weight_max": 16.556926727294922,
      "activations/layer1_attention_weight_min": -14.530810356140137,
      "activations/layer20_attention_weight_max": 32.38145446777344,
      "activations/layer20_attention_weight_min": -25.501419067382812,
      "activations/layer21_attention_weight_max": 32.22719192504883,
      "activations/layer21_attention_weight_min": -24.190378189086914,
      "activations/layer22_attention_weight_max": 41.89507293701172,
      "activations/layer22_attention_weight_min": -30.45293426513672,
      "activations/layer23_attention_weight_max": 34.30329895019531,
      "activations/layer23_attention_weight_min": -27.43798828125,
      "activations/layer2_attention_weight_max": 34.91578674316406,
      "activations/layer2_attention_weight_min": -35.91962814331055,
      "activations/layer3_attention_weight_max": 102.10765075683594,
      "activations/layer3_attention_weight_min": -108.38825225830078,
      "activations/layer4_attention_weight_max": 111.26822662353516,
      "activations/layer4_attention_weight_min": -121.06056213378906,
      "activations/layer5_attention_weight_max": 58.726654052734375,
      "activations/layer5_attention_weight_min": -66.19857788085938,
      "activations/layer6_attention_weight_max": 44.66456985473633,
      "activations/layer6_attention_weight_min": -46.82604217529297,
      "activations/layer7_attention_weight_max": 87.64202117919922,
      "activations/layer7_attention_weight_min": -90.85704803466797,
      "activations/layer8_attention_weight_max": 39.618003845214844,
      "activations/layer8_attention_weight_min": -40.502017974853516,
      "activations/layer9_attention_weight_max": 30.850122451782227,
      "activations/layer9_attention_weight_min": -31.093673706054688,
      "epoch": 13.52,
      "learning_rate": 6.344431818181818e-05,
      "loss": 2.7596,
      "step": 232700
    },
    {
      "activations/layer0_attention_weight_max": 16.745805740356445,
      "activations/layer0_attention_weight_min": -12.728636741638184,
      "activations/layer10_attention_weight_max": 34.869972229003906,
      "activations/layer10_attention_weight_min": -32.86968994140625,
      "activations/layer11_attention_weight_max": 34.344947814941406,
      "activations/layer11_attention_weight_min": -34.36677551269531,
      "activations/layer12_attention_weight_max": 25.479915618896484,
      "activations/layer12_attention_weight_min": -30.710651397705078,
      "activations/layer13_attention_weight_max": 37.404937744140625,
      "activations/layer13_attention_weight_min": -33.61320877075195,
      "activations/layer14_attention_weight_max": 40.17854309082031,
      "activations/layer14_attention_weight_min": -39.65700149536133,
      "activations/layer15_attention_weight_max": 35.0245246887207,
      "activations/layer15_attention_weight_min": -31.604745864868164,
      "activations/layer16_attention_weight_max": 29.540279388427734,
      "activations/layer16_attention_weight_min": -30.983125686645508,
      "activations/layer17_attention_weight_max": 32.034080505371094,
      "activations/layer17_attention_weight_min": -24.831035614013672,
      "activations/layer18_attention_weight_max": 31.77788734436035,
      "activations/layer18_attention_weight_min": -25.137670516967773,
      "activations/layer19_attention_weight_max": 36.13003158569336,
      "activations/layer19_attention_weight_min": -30.42190933227539,
      "activations/layer1_attention_weight_max": 17.380916595458984,
      "activations/layer1_attention_weight_min": -16.652894973754883,
      "activations/layer20_attention_weight_max": 27.935129165649414,
      "activations/layer20_attention_weight_min": -25.510082244873047,
      "activations/layer21_attention_weight_max": 26.982858657836914,
      "activations/layer21_attention_weight_min": -23.73587989807129,
      "activations/layer22_attention_weight_max": 39.543548583984375,
      "activations/layer22_attention_weight_min": -29.994464874267578,
      "activations/layer23_attention_weight_max": 32.58439254760742,
      "activations/layer23_attention_weight_min": -24.2844295501709,
      "activations/layer2_attention_weight_max": 36.3583984375,
      "activations/layer2_attention_weight_min": -32.525856018066406,
      "activations/layer3_attention_weight_max": 91.93881225585938,
      "activations/layer3_attention_weight_min": -98.9368896484375,
      "activations/layer4_attention_weight_max": 104.83162689208984,
      "activations/layer4_attention_weight_min": -106.72395324707031,
      "activations/layer5_attention_weight_max": 51.22878646850586,
      "activations/layer5_attention_weight_min": -61.607723236083984,
      "activations/layer6_attention_weight_max": 43.787906646728516,
      "activations/layer6_attention_weight_min": -47.740421295166016,
      "activations/layer7_attention_weight_max": 93.10736083984375,
      "activations/layer7_attention_weight_min": -91.43524169921875,
      "activations/layer8_attention_weight_max": 40.513248443603516,
      "activations/layer8_attention_weight_min": -45.53053283691406,
      "activations/layer9_attention_weight_max": 34.27274703979492,
      "activations/layer9_attention_weight_min": -35.504940032958984,
      "epoch": 13.52,
      "learning_rate": 6.342537878787879e-05,
      "loss": 2.7589,
      "step": 232750
    },
    {
      "activations/layer0_attention_weight_max": 17.317264556884766,
      "activations/layer0_attention_weight_min": -12.259982109069824,
      "activations/layer10_attention_weight_max": 34.28968811035156,
      "activations/layer10_attention_weight_min": -32.12849044799805,
      "activations/layer11_attention_weight_max": 32.27540588378906,
      "activations/layer11_attention_weight_min": -33.3405647277832,
      "activations/layer12_attention_weight_max": 25.530841827392578,
      "activations/layer12_attention_weight_min": -25.668054580688477,
      "activations/layer13_attention_weight_max": 37.087982177734375,
      "activations/layer13_attention_weight_min": -34.178035736083984,
      "activations/layer14_attention_weight_max": 40.91606903076172,
      "activations/layer14_attention_weight_min": -36.77825164794922,
      "activations/layer15_attention_weight_max": 36.372440338134766,
      "activations/layer15_attention_weight_min": -33.26034164428711,
      "activations/layer16_attention_weight_max": 31.103593826293945,
      "activations/layer16_attention_weight_min": -26.6911563873291,
      "activations/layer17_attention_weight_max": 30.169565200805664,
      "activations/layer17_attention_weight_min": -27.05234718322754,
      "activations/layer18_attention_weight_max": 32.59980392456055,
      "activations/layer18_attention_weight_min": -25.94576072692871,
      "activations/layer19_attention_weight_max": 34.83100509643555,
      "activations/layer19_attention_weight_min": -31.65484046936035,
      "activations/layer1_attention_weight_max": 16.37538719177246,
      "activations/layer1_attention_weight_min": -14.270303726196289,
      "activations/layer20_attention_weight_max": 30.220224380493164,
      "activations/layer20_attention_weight_min": -26.536272048950195,
      "activations/layer21_attention_weight_max": 28.88339614868164,
      "activations/layer21_attention_weight_min": -25.35173988342285,
      "activations/layer22_attention_weight_max": 42.97412109375,
      "activations/layer22_attention_weight_min": -32.03573989868164,
      "activations/layer23_attention_weight_max": 33.978729248046875,
      "activations/layer23_attention_weight_min": -26.57614517211914,
      "activations/layer2_attention_weight_max": 34.917049407958984,
      "activations/layer2_attention_weight_min": -33.843894958496094,
      "activations/layer3_attention_weight_max": 97.36479187011719,
      "activations/layer3_attention_weight_min": -99.24822235107422,
      "activations/layer4_attention_weight_max": 109.3487319946289,
      "activations/layer4_attention_weight_min": -113.1815185546875,
      "activations/layer5_attention_weight_max": 57.130706787109375,
      "activations/layer5_attention_weight_min": -61.535438537597656,
      "activations/layer6_attention_weight_max": 45.15843963623047,
      "activations/layer6_attention_weight_min": -47.88674545288086,
      "activations/layer7_attention_weight_max": 91.78857421875,
      "activations/layer7_attention_weight_min": -88.2238998413086,
      "activations/layer8_attention_weight_max": 40.485904693603516,
      "activations/layer8_attention_weight_min": -41.07121276855469,
      "activations/layer9_attention_weight_max": 33.628807067871094,
      "activations/layer9_attention_weight_min": -33.573333740234375,
      "epoch": 13.53,
      "learning_rate": 6.340643939393939e-05,
      "loss": 2.7716,
      "step": 232800
    },
    {
      "activations/layer0_attention_weight_max": 16.430776596069336,
      "activations/layer0_attention_weight_min": -12.505847930908203,
      "activations/layer10_attention_weight_max": 48.366119384765625,
      "activations/layer10_attention_weight_min": -49.037315368652344,
      "activations/layer11_attention_weight_max": 48.98092269897461,
      "activations/layer11_attention_weight_min": -48.827083587646484,
      "activations/layer12_attention_weight_max": 31.196699142456055,
      "activations/layer12_attention_weight_min": -32.372459411621094,
      "activations/layer13_attention_weight_max": 44.81779861450195,
      "activations/layer13_attention_weight_min": -44.37584686279297,
      "activations/layer14_attention_weight_max": 53.41885757446289,
      "activations/layer14_attention_weight_min": -52.88298797607422,
      "activations/layer15_attention_weight_max": 44.00969696044922,
      "activations/layer15_attention_weight_min": -43.402366638183594,
      "activations/layer16_attention_weight_max": 33.05085754394531,
      "activations/layer16_attention_weight_min": -28.81002426147461,
      "activations/layer17_attention_weight_max": 28.812530517578125,
      "activations/layer17_attention_weight_min": -27.969459533691406,
      "activations/layer18_attention_weight_max": 31.175540924072266,
      "activations/layer18_attention_weight_min": -23.422883987426758,
      "activations/layer19_attention_weight_max": 32.43663787841797,
      "activations/layer19_attention_weight_min": -31.026975631713867,
      "activations/layer1_attention_weight_max": 16.434762954711914,
      "activations/layer1_attention_weight_min": -15.575173377990723,
      "activations/layer20_attention_weight_max": 26.58568000793457,
      "activations/layer20_attention_weight_min": -24.420818328857422,
      "activations/layer21_attention_weight_max": 27.882993698120117,
      "activations/layer21_attention_weight_min": -23.0772647857666,
      "activations/layer22_attention_weight_max": 41.33658218383789,
      "activations/layer22_attention_weight_min": -31.155914306640625,
      "activations/layer23_attention_weight_max": 30.559017181396484,
      "activations/layer23_attention_weight_min": -25.2893123626709,
      "activations/layer2_attention_weight_max": 37.12596893310547,
      "activations/layer2_attention_weight_min": -34.03633117675781,
      "activations/layer3_attention_weight_max": 103.19760131835938,
      "activations/layer3_attention_weight_min": -105.10125732421875,
      "activations/layer4_attention_weight_max": 117.39570617675781,
      "activations/layer4_attention_weight_min": -113.38427734375,
      "activations/layer5_attention_weight_max": 58.840660095214844,
      "activations/layer5_attention_weight_min": -62.34403991699219,
      "activations/layer6_attention_weight_max": 50.6876335144043,
      "activations/layer6_attention_weight_min": -50.293636322021484,
      "activations/layer7_attention_weight_max": 120.5072250366211,
      "activations/layer7_attention_weight_min": -117.65512084960938,
      "activations/layer8_attention_weight_max": 56.92749786376953,
      "activations/layer8_attention_weight_min": -55.182823181152344,
      "activations/layer9_attention_weight_max": 45.70681381225586,
      "activations/layer9_attention_weight_min": -44.439449310302734,
      "epoch": 13.53,
      "learning_rate": 6.338749999999999e-05,
      "loss": 2.7567,
      "step": 232850
    },
    {
      "activations/layer0_attention_weight_max": 17.767656326293945,
      "activations/layer0_attention_weight_min": -12.98015022277832,
      "activations/layer10_attention_weight_max": 32.687225341796875,
      "activations/layer10_attention_weight_min": -31.22836685180664,
      "activations/layer11_attention_weight_max": 35.40534210205078,
      "activations/layer11_attention_weight_min": -32.1435546875,
      "activations/layer12_attention_weight_max": 29.868305206298828,
      "activations/layer12_attention_weight_min": -27.687021255493164,
      "activations/layer13_attention_weight_max": 37.90808868408203,
      "activations/layer13_attention_weight_min": -33.97166442871094,
      "activations/layer14_attention_weight_max": 41.10049057006836,
      "activations/layer14_attention_weight_min": -37.63159942626953,
      "activations/layer15_attention_weight_max": 36.2042236328125,
      "activations/layer15_attention_weight_min": -35.579280853271484,
      "activations/layer16_attention_weight_max": 29.470211029052734,
      "activations/layer16_attention_weight_min": -27.616445541381836,
      "activations/layer17_attention_weight_max": 28.753475189208984,
      "activations/layer17_attention_weight_min": -27.885995864868164,
      "activations/layer18_attention_weight_max": 31.438945770263672,
      "activations/layer18_attention_weight_min": -23.865755081176758,
      "activations/layer19_attention_weight_max": 32.472835540771484,
      "activations/layer19_attention_weight_min": -30.139999389648438,
      "activations/layer1_attention_weight_max": 16.379314422607422,
      "activations/layer1_attention_weight_min": -14.951689720153809,
      "activations/layer20_attention_weight_max": 26.51464080810547,
      "activations/layer20_attention_weight_min": -24.181936264038086,
      "activations/layer21_attention_weight_max": 28.363067626953125,
      "activations/layer21_attention_weight_min": -23.024822235107422,
      "activations/layer22_attention_weight_max": 38.75460433959961,
      "activations/layer22_attention_weight_min": -29.502763748168945,
      "activations/layer23_attention_weight_max": 34.072975158691406,
      "activations/layer23_attention_weight_min": -25.78102684020996,
      "activations/layer2_attention_weight_max": 35.432369232177734,
      "activations/layer2_attention_weight_min": -33.805973052978516,
      "activations/layer3_attention_weight_max": 104.45425415039062,
      "activations/layer3_attention_weight_min": -99.88302612304688,
      "activations/layer4_attention_weight_max": 117.92986297607422,
      "activations/layer4_attention_weight_min": -111.43206024169922,
      "activations/layer5_attention_weight_max": 55.159786224365234,
      "activations/layer5_attention_weight_min": -60.02842330932617,
      "activations/layer6_attention_weight_max": 44.73334884643555,
      "activations/layer6_attention_weight_min": -45.83164596557617,
      "activations/layer7_attention_weight_max": 93.11553955078125,
      "activations/layer7_attention_weight_min": -89.58432006835938,
      "activations/layer8_attention_weight_max": 40.949092864990234,
      "activations/layer8_attention_weight_min": -42.145076751708984,
      "activations/layer9_attention_weight_max": 32.85520553588867,
      "activations/layer9_attention_weight_min": -32.00177001953125,
      "epoch": 13.53,
      "learning_rate": 6.33685606060606e-05,
      "loss": 2.7485,
      "step": 232900
    },
    {
      "activations/layer0_attention_weight_max": 15.758554458618164,
      "activations/layer0_attention_weight_min": -12.590493202209473,
      "activations/layer10_attention_weight_max": 34.2610969543457,
      "activations/layer10_attention_weight_min": -32.7148323059082,
      "activations/layer11_attention_weight_max": 34.672950744628906,
      "activations/layer11_attention_weight_min": -32.572757720947266,
      "activations/layer12_attention_weight_max": 27.36190414428711,
      "activations/layer12_attention_weight_min": -26.149507522583008,
      "activations/layer13_attention_weight_max": 39.327308654785156,
      "activations/layer13_attention_weight_min": -33.44649124145508,
      "activations/layer14_attention_weight_max": 41.994789123535156,
      "activations/layer14_attention_weight_min": -36.21025848388672,
      "activations/layer15_attention_weight_max": 40.8675422668457,
      "activations/layer15_attention_weight_min": -32.592369079589844,
      "activations/layer16_attention_weight_max": 31.039030075073242,
      "activations/layer16_attention_weight_min": -26.71428680419922,
      "activations/layer17_attention_weight_max": 31.934106826782227,
      "activations/layer17_attention_weight_min": -26.10051727294922,
      "activations/layer18_attention_weight_max": 29.857154846191406,
      "activations/layer18_attention_weight_min": -23.679521560668945,
      "activations/layer19_attention_weight_max": 33.46297073364258,
      "activations/layer19_attention_weight_min": -31.549020767211914,
      "activations/layer1_attention_weight_max": 17.28797149658203,
      "activations/layer1_attention_weight_min": -14.932428359985352,
      "activations/layer20_attention_weight_max": 28.0963077545166,
      "activations/layer20_attention_weight_min": -25.771318435668945,
      "activations/layer21_attention_weight_max": 29.29050064086914,
      "activations/layer21_attention_weight_min": -25.257043838500977,
      "activations/layer22_attention_weight_max": 43.05560302734375,
      "activations/layer22_attention_weight_min": -30.05623435974121,
      "activations/layer23_attention_weight_max": 33.46430587768555,
      "activations/layer23_attention_weight_min": -27.154653549194336,
      "activations/layer2_attention_weight_max": 34.065887451171875,
      "activations/layer2_attention_weight_min": -33.636295318603516,
      "activations/layer3_attention_weight_max": 99.4542007446289,
      "activations/layer3_attention_weight_min": -97.45307159423828,
      "activations/layer4_attention_weight_max": 110.9598617553711,
      "activations/layer4_attention_weight_min": -109.90787506103516,
      "activations/layer5_attention_weight_max": 54.424373626708984,
      "activations/layer5_attention_weight_min": -60.630157470703125,
      "activations/layer6_attention_weight_max": 44.51512145996094,
      "activations/layer6_attention_weight_min": -46.378623962402344,
      "activations/layer7_attention_weight_max": 89.6746597290039,
      "activations/layer7_attention_weight_min": -86.95219421386719,
      "activations/layer8_attention_weight_max": 39.900630950927734,
      "activations/layer8_attention_weight_min": -41.85559844970703,
      "activations/layer9_attention_weight_max": 32.784446716308594,
      "activations/layer9_attention_weight_min": -34.32280349731445,
      "epoch": 13.54,
      "learning_rate": 6.33496212121212e-05,
      "loss": 2.7622,
      "step": 232950
    },
    {
      "activations/layer0_attention_weight_max": 16.259687423706055,
      "activations/layer0_attention_weight_min": -12.654787063598633,
      "activations/layer10_attention_weight_max": 34.10736846923828,
      "activations/layer10_attention_weight_min": -34.85333251953125,
      "activations/layer11_attention_weight_max": 32.405601501464844,
      "activations/layer11_attention_weight_min": -33.40587615966797,
      "activations/layer12_attention_weight_max": 25.5588436126709,
      "activations/layer12_attention_weight_min": -25.973493576049805,
      "activations/layer13_attention_weight_max": 37.7518196105957,
      "activations/layer13_attention_weight_min": -34.10152816772461,
      "activations/layer14_attention_weight_max": 41.69658660888672,
      "activations/layer14_attention_weight_min": -39.449337005615234,
      "activations/layer15_attention_weight_max": 36.8001594543457,
      "activations/layer15_attention_weight_min": -33.958580017089844,
      "activations/layer16_attention_weight_max": 28.761728286743164,
      "activations/layer16_attention_weight_min": -27.92158317565918,
      "activations/layer17_attention_weight_max": 31.957265853881836,
      "activations/layer17_attention_weight_min": -26.268795013427734,
      "activations/layer18_attention_weight_max": 32.433433532714844,
      "activations/layer18_attention_weight_min": -24.175121307373047,
      "activations/layer19_attention_weight_max": 35.196258544921875,
      "activations/layer19_attention_weight_min": -31.070138931274414,
      "activations/layer1_attention_weight_max": 16.32853889465332,
      "activations/layer1_attention_weight_min": -15.535462379455566,
      "activations/layer20_attention_weight_max": 27.652088165283203,
      "activations/layer20_attention_weight_min": -24.256925582885742,
      "activations/layer21_attention_weight_max": 27.02219581604004,
      "activations/layer21_attention_weight_min": -24.23276710510254,
      "activations/layer22_attention_weight_max": 47.28732681274414,
      "activations/layer22_attention_weight_min": -32.143699645996094,
      "activations/layer23_attention_weight_max": 34.71569061279297,
      "activations/layer23_attention_weight_min": -29.79742431640625,
      "activations/layer2_attention_weight_max": 33.192649841308594,
      "activations/layer2_attention_weight_min": -33.198455810546875,
      "activations/layer3_attention_weight_max": 99.94268798828125,
      "activations/layer3_attention_weight_min": -102.29857635498047,
      "activations/layer4_attention_weight_max": 111.4134750366211,
      "activations/layer4_attention_weight_min": -113.42162322998047,
      "activations/layer5_attention_weight_max": 54.21879577636719,
      "activations/layer5_attention_weight_min": -62.76945114135742,
      "activations/layer6_attention_weight_max": 43.3592643737793,
      "activations/layer6_attention_weight_min": -47.19112014770508,
      "activations/layer7_attention_weight_max": 89.00261688232422,
      "activations/layer7_attention_weight_min": -97.65655517578125,
      "activations/layer8_attention_weight_max": 40.84321975708008,
      "activations/layer8_attention_weight_min": -43.83851623535156,
      "activations/layer9_attention_weight_max": 33.1927604675293,
      "activations/layer9_attention_weight_min": -35.62202453613281,
      "epoch": 13.54,
      "learning_rate": 6.333068181818181e-05,
      "loss": 2.7656,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4509,
      "eval_samples_per_second": 508.113,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4509,
      "eval_openwebtext_samples_per_second": 508.113,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9345,
      "eval_wikitext_samples_per_second": 235.723,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_lambada_loss": 2.39453125,
      "eval_lambada_ppl": 10.963057921054194,
      "eval_lambada_runtime": 9.5576,
      "eval_lambada_samples_per_second": 509.435,
      "step": 233000
    },
    {
      "activations/layer0_attention_weight_max": 15.200508117675781,
      "activations/layer0_attention_weight_min": -12.667101860046387,
      "activations/layer10_attention_weight_max": 36.79973220825195,
      "activations/layer10_attention_weight_min": -33.804141998291016,
      "activations/layer11_attention_weight_max": 37.91282272338867,
      "activations/layer11_attention_weight_min": -33.26737594604492,
      "activations/layer12_attention_weight_max": 24.678638458251953,
      "activations/layer12_attention_weight_min": -24.928136825561523,
      "activations/layer13_attention_weight_max": 35.4213981628418,
      "activations/layer13_attention_weight_min": -34.10488510131836,
      "activations/layer14_attention_weight_max": 45.292213439941406,
      "activations/layer14_attention_weight_min": -38.607975006103516,
      "activations/layer15_attention_weight_max": 34.880550384521484,
      "activations/layer15_attention_weight_min": -35.234127044677734,
      "activations/layer16_attention_weight_max": 30.972591400146484,
      "activations/layer16_attention_weight_min": -27.048830032348633,
      "activations/layer17_attention_weight_max": 28.667213439941406,
      "activations/layer17_attention_weight_min": -25.792842864990234,
      "activations/layer18_attention_weight_max": 28.898611068725586,
      "activations/layer18_attention_weight_min": -23.469898223876953,
      "activations/layer19_attention_weight_max": 34.65330505371094,
      "activations/layer19_attention_weight_min": -32.13654327392578,
      "activations/layer1_attention_weight_max": 16.982112884521484,
      "activations/layer1_attention_weight_min": -17.050519943237305,
      "activations/layer20_attention_weight_max": 28.686227798461914,
      "activations/layer20_attention_weight_min": -25.473905563354492,
      "activations/layer21_attention_weight_max": 28.915563583374023,
      "activations/layer21_attention_weight_min": -25.753854751586914,
      "activations/layer22_attention_weight_max": 39.35930633544922,
      "activations/layer22_attention_weight_min": -29.896230697631836,
      "activations/layer23_attention_weight_max": 36.21338653564453,
      "activations/layer23_attention_weight_min": -24.075910568237305,
      "activations/layer2_attention_weight_max": 38.47711181640625,
      "activations/layer2_attention_weight_min": -34.22740173339844,
      "activations/layer3_attention_weight_max": 99.07923126220703,
      "activations/layer3_attention_weight_min": -100.96968078613281,
      "activations/layer4_attention_weight_max": 111.555908203125,
      "activations/layer4_attention_weight_min": -109.88358306884766,
      "activations/layer5_attention_weight_max": 53.10395431518555,
      "activations/layer5_attention_weight_min": -61.6867790222168,
      "activations/layer6_attention_weight_max": 44.152732849121094,
      "activations/layer6_attention_weight_min": -45.4097785949707,
      "activations/layer7_attention_weight_max": 103.3635025024414,
      "activations/layer7_attention_weight_min": -85.75834655761719,
      "activations/layer8_attention_weight_max": 43.98812484741211,
      "activations/layer8_attention_weight_min": -41.69131851196289,
      "activations/layer9_attention_weight_max": 34.63398742675781,
      "activations/layer9_attention_weight_min": -34.95999526977539,
      "epoch": 13.54,
      "learning_rate": 6.331174242424241e-05,
      "loss": 2.7612,
      "step": 233050
    },
    {
      "activations/layer0_attention_weight_max": 15.448119163513184,
      "activations/layer0_attention_weight_min": -13.099395751953125,
      "activations/layer10_attention_weight_max": 35.45261764526367,
      "activations/layer10_attention_weight_min": -33.88699722290039,
      "activations/layer11_attention_weight_max": 35.99017333984375,
      "activations/layer11_attention_weight_min": -35.33929443359375,
      "activations/layer12_attention_weight_max": 27.319595336914062,
      "activations/layer12_attention_weight_min": -26.96072769165039,
      "activations/layer13_attention_weight_max": 41.96583557128906,
      "activations/layer13_attention_weight_min": -38.386566162109375,
      "activations/layer14_attention_weight_max": 43.007328033447266,
      "activations/layer14_attention_weight_min": -39.53408432006836,
      "activations/layer15_attention_weight_max": 38.948394775390625,
      "activations/layer15_attention_weight_min": -36.55216598510742,
      "activations/layer16_attention_weight_max": 33.16526794433594,
      "activations/layer16_attention_weight_min": -30.32274627685547,
      "activations/layer17_attention_weight_max": 30.900020599365234,
      "activations/layer17_attention_weight_min": -26.237648010253906,
      "activations/layer18_attention_weight_max": 31.391952514648438,
      "activations/layer18_attention_weight_min": -24.44727897644043,
      "activations/layer19_attention_weight_max": 33.10176467895508,
      "activations/layer19_attention_weight_min": -30.589157104492188,
      "activations/layer1_attention_weight_max": 16.059728622436523,
      "activations/layer1_attention_weight_min": -14.054396629333496,
      "activations/layer20_attention_weight_max": 27.35128402709961,
      "activations/layer20_attention_weight_min": -24.59267807006836,
      "activations/layer21_attention_weight_max": 26.74546241760254,
      "activations/layer21_attention_weight_min": -24.426170349121094,
      "activations/layer22_attention_weight_max": 44.59030532836914,
      "activations/layer22_attention_weight_min": -30.860944747924805,
      "activations/layer23_attention_weight_max": 31.981021881103516,
      "activations/layer23_attention_weight_min": -26.803075790405273,
      "activations/layer2_attention_weight_max": 33.72344207763672,
      "activations/layer2_attention_weight_min": -34.268455505371094,
      "activations/layer3_attention_weight_max": 97.8446273803711,
      "activations/layer3_attention_weight_min": -102.0031967163086,
      "activations/layer4_attention_weight_max": 111.14620208740234,
      "activations/layer4_attention_weight_min": -116.05428314208984,
      "activations/layer5_attention_weight_max": 52.623199462890625,
      "activations/layer5_attention_weight_min": -65.20321655273438,
      "activations/layer6_attention_weight_max": 46.85603713989258,
      "activations/layer6_attention_weight_min": -48.77024841308594,
      "activations/layer7_attention_weight_max": 94.79973602294922,
      "activations/layer7_attention_weight_min": -98.67330932617188,
      "activations/layer8_attention_weight_max": 46.053443908691406,
      "activations/layer8_attention_weight_min": -44.24925994873047,
      "activations/layer9_attention_weight_max": 35.20146942138672,
      "activations/layer9_attention_weight_min": -35.38115692138672,
      "epoch": 13.54,
      "learning_rate": 6.329280303030302e-05,
      "loss": 2.7667,
      "step": 233100
    },
    {
      "activations/layer0_attention_weight_max": 16.582290649414062,
      "activations/layer0_attention_weight_min": -12.022550582885742,
      "activations/layer10_attention_weight_max": 31.894699096679688,
      "activations/layer10_attention_weight_min": -30.984281539916992,
      "activations/layer11_attention_weight_max": 32.10894775390625,
      "activations/layer11_attention_weight_min": -32.03022003173828,
      "activations/layer12_attention_weight_max": 24.3245849609375,
      "activations/layer12_attention_weight_min": -23.72276496887207,
      "activations/layer13_attention_weight_max": 37.581451416015625,
      "activations/layer13_attention_weight_min": -32.43144607543945,
      "activations/layer14_attention_weight_max": 41.19989776611328,
      "activations/layer14_attention_weight_min": -34.851558685302734,
      "activations/layer15_attention_weight_max": 36.646751403808594,
      "activations/layer15_attention_weight_min": -31.727937698364258,
      "activations/layer16_attention_weight_max": 31.554092407226562,
      "activations/layer16_attention_weight_min": -27.33063316345215,
      "activations/layer17_attention_weight_max": 34.90519714355469,
      "activations/layer17_attention_weight_min": -27.668298721313477,
      "activations/layer18_attention_weight_max": 31.390291213989258,
      "activations/layer18_attention_weight_min": -23.94219398498535,
      "activations/layer19_attention_weight_max": 33.843318939208984,
      "activations/layer19_attention_weight_min": -33.02238464355469,
      "activations/layer1_attention_weight_max": 16.475454330444336,
      "activations/layer1_attention_weight_min": -15.50910758972168,
      "activations/layer20_attention_weight_max": 27.39824867248535,
      "activations/layer20_attention_weight_min": -24.649015426635742,
      "activations/layer21_attention_weight_max": 28.778514862060547,
      "activations/layer21_attention_weight_min": -23.93840980529785,
      "activations/layer22_attention_weight_max": 39.85963821411133,
      "activations/layer22_attention_weight_min": -29.76795768737793,
      "activations/layer23_attention_weight_max": 37.32826614379883,
      "activations/layer23_attention_weight_min": -24.4222412109375,
      "activations/layer2_attention_weight_max": 36.22545623779297,
      "activations/layer2_attention_weight_min": -34.29099655151367,
      "activations/layer3_attention_weight_max": 94.10298919677734,
      "activations/layer3_attention_weight_min": -98.28619384765625,
      "activations/layer4_attention_weight_max": 108.71105194091797,
      "activations/layer4_attention_weight_min": -111.8506851196289,
      "activations/layer5_attention_weight_max": 49.67274475097656,
      "activations/layer5_attention_weight_min": -59.932708740234375,
      "activations/layer6_attention_weight_max": 43.08060073852539,
      "activations/layer6_attention_weight_min": -46.206180572509766,
      "activations/layer7_attention_weight_max": 85.80509185791016,
      "activations/layer7_attention_weight_min": -86.68125915527344,
      "activations/layer8_attention_weight_max": 38.228450775146484,
      "activations/layer8_attention_weight_min": -42.68714904785156,
      "activations/layer9_attention_weight_max": 31.896194458007812,
      "activations/layer9_attention_weight_min": -34.243709564208984,
      "epoch": 13.55,
      "learning_rate": 6.327386363636363e-05,
      "loss": 2.7705,
      "step": 233150
    },
    {
      "activations/layer0_attention_weight_max": 15.209554672241211,
      "activations/layer0_attention_weight_min": -12.202698707580566,
      "activations/layer10_attention_weight_max": 39.49216842651367,
      "activations/layer10_attention_weight_min": -38.26792526245117,
      "activations/layer11_attention_weight_max": 42.85432052612305,
      "activations/layer11_attention_weight_min": -40.00040054321289,
      "activations/layer12_attention_weight_max": 25.11315155029297,
      "activations/layer12_attention_weight_min": -28.780187606811523,
      "activations/layer13_attention_weight_max": 38.67982864379883,
      "activations/layer13_attention_weight_min": -36.79930877685547,
      "activations/layer14_attention_weight_max": 44.321754455566406,
      "activations/layer14_attention_weight_min": -37.512794494628906,
      "activations/layer15_attention_weight_max": 36.99871826171875,
      "activations/layer15_attention_weight_min": -35.36251449584961,
      "activations/layer16_attention_weight_max": 32.62934875488281,
      "activations/layer16_attention_weight_min": -29.58533477783203,
      "activations/layer17_attention_weight_max": 28.763235092163086,
      "activations/layer17_attention_weight_min": -27.758895874023438,
      "activations/layer18_attention_weight_max": 31.161470413208008,
      "activations/layer18_attention_weight_min": -25.0130672454834,
      "activations/layer19_attention_weight_max": 39.01121520996094,
      "activations/layer19_attention_weight_min": -31.580228805541992,
      "activations/layer1_attention_weight_max": 16.112409591674805,
      "activations/layer1_attention_weight_min": -14.423120498657227,
      "activations/layer20_attention_weight_max": 29.7042179107666,
      "activations/layer20_attention_weight_min": -25.572154998779297,
      "activations/layer21_attention_weight_max": 28.68375015258789,
      "activations/layer21_attention_weight_min": -24.706573486328125,
      "activations/layer22_attention_weight_max": 39.06072998046875,
      "activations/layer22_attention_weight_min": -31.39724349975586,
      "activations/layer23_attention_weight_max": 31.359943389892578,
      "activations/layer23_attention_weight_min": -24.518110275268555,
      "activations/layer2_attention_weight_max": 34.875328063964844,
      "activations/layer2_attention_weight_min": -34.227481842041016,
      "activations/layer3_attention_weight_max": 106.18142700195312,
      "activations/layer3_attention_weight_min": -105.31111907958984,
      "activations/layer4_attention_weight_max": 117.98661041259766,
      "activations/layer4_attention_weight_min": -117.3195571899414,
      "activations/layer5_attention_weight_max": 55.25472640991211,
      "activations/layer5_attention_weight_min": -61.83306121826172,
      "activations/layer6_attention_weight_max": 48.06132125854492,
      "activations/layer6_attention_weight_min": -47.45675277709961,
      "activations/layer7_attention_weight_max": 107.24311065673828,
      "activations/layer7_attention_weight_min": -106.501220703125,
      "activations/layer8_attention_weight_max": 49.71866989135742,
      "activations/layer8_attention_weight_min": -47.152626037597656,
      "activations/layer9_attention_weight_max": 39.3133544921875,
      "activations/layer9_attention_weight_min": -40.118831634521484,
      "epoch": 13.55,
      "learning_rate": 6.325492424242424e-05,
      "loss": 2.7779,
      "step": 233200
    },
    {
      "activations/layer0_attention_weight_max": 15.919841766357422,
      "activations/layer0_attention_weight_min": -12.499799728393555,
      "activations/layer10_attention_weight_max": 38.814517974853516,
      "activations/layer10_attention_weight_min": -37.0738525390625,
      "activations/layer11_attention_weight_max": 39.19752502441406,
      "activations/layer11_attention_weight_min": -35.61732482910156,
      "activations/layer12_attention_weight_max": 29.081056594848633,
      "activations/layer12_attention_weight_min": -26.5914249420166,
      "activations/layer13_attention_weight_max": 46.79828643798828,
      "activations/layer13_attention_weight_min": -36.626869201660156,
      "activations/layer14_attention_weight_max": 50.24062728881836,
      "activations/layer14_attention_weight_min": -39.38142395019531,
      "activations/layer15_attention_weight_max": 44.4826774597168,
      "activations/layer15_attention_weight_min": -37.147682189941406,
      "activations/layer16_attention_weight_max": 37.6915168762207,
      "activations/layer16_attention_weight_min": -30.952173233032227,
      "activations/layer17_attention_weight_max": 36.07395935058594,
      "activations/layer17_attention_weight_min": -27.38435935974121,
      "activations/layer18_attention_weight_max": 32.84182357788086,
      "activations/layer18_attention_weight_min": -25.661710739135742,
      "activations/layer19_attention_weight_max": 37.940711975097656,
      "activations/layer19_attention_weight_min": -32.950016021728516,
      "activations/layer1_attention_weight_max": 16.883485794067383,
      "activations/layer1_attention_weight_min": -15.52625846862793,
      "activations/layer20_attention_weight_max": 33.97494888305664,
      "activations/layer20_attention_weight_min": -27.50075340270996,
      "activations/layer21_attention_weight_max": 31.81442642211914,
      "activations/layer21_attention_weight_min": -25.3606014251709,
      "activations/layer22_attention_weight_max": 54.49388122558594,
      "activations/layer22_attention_weight_min": -34.30937576293945,
      "activations/layer23_attention_weight_max": 38.30643844604492,
      "activations/layer23_attention_weight_min": -25.978193283081055,
      "activations/layer2_attention_weight_max": 33.662498474121094,
      "activations/layer2_attention_weight_min": -34.73371887207031,
      "activations/layer3_attention_weight_max": 100.79350280761719,
      "activations/layer3_attention_weight_min": -102.47758483886719,
      "activations/layer4_attention_weight_max": 118.77823638916016,
      "activations/layer4_attention_weight_min": -114.47810363769531,
      "activations/layer5_attention_weight_max": 53.10721969604492,
      "activations/layer5_attention_weight_min": -59.12759017944336,
      "activations/layer6_attention_weight_max": 47.42099380493164,
      "activations/layer6_attention_weight_min": -48.97940444946289,
      "activations/layer7_attention_weight_max": 95.53079986572266,
      "activations/layer7_attention_weight_min": -93.35660552978516,
      "activations/layer8_attention_weight_max": 46.232086181640625,
      "activations/layer8_attention_weight_min": -48.82365798950195,
      "activations/layer9_attention_weight_max": 38.29286575317383,
      "activations/layer9_attention_weight_min": -38.294071197509766,
      "epoch": 13.55,
      "learning_rate": 6.323598484848484e-05,
      "loss": 2.7607,
      "step": 233250
    },
    {
      "activations/layer0_attention_weight_max": 16.077878952026367,
      "activations/layer0_attention_weight_min": -12.404621124267578,
      "activations/layer10_attention_weight_max": 33.79535675048828,
      "activations/layer10_attention_weight_min": -34.09202194213867,
      "activations/layer11_attention_weight_max": 36.104759216308594,
      "activations/layer11_attention_weight_min": -33.612300872802734,
      "activations/layer12_attention_weight_max": 24.97858238220215,
      "activations/layer12_attention_weight_min": -26.501436233520508,
      "activations/layer13_attention_weight_max": 38.89457702636719,
      "activations/layer13_attention_weight_min": -33.745269775390625,
      "activations/layer14_attention_weight_max": 42.21830749511719,
      "activations/layer14_attention_weight_min": -37.22289276123047,
      "activations/layer15_attention_weight_max": 36.67643737792969,
      "activations/layer15_attention_weight_min": -31.85940933227539,
      "activations/layer16_attention_weight_max": 30.816923141479492,
      "activations/layer16_attention_weight_min": -27.288734436035156,
      "activations/layer17_attention_weight_max": 33.24545669555664,
      "activations/layer17_attention_weight_min": -24.84714698791504,
      "activations/layer18_attention_weight_max": 30.478960037231445,
      "activations/layer18_attention_weight_min": -23.01243782043457,
      "activations/layer19_attention_weight_max": 35.33833312988281,
      "activations/layer19_attention_weight_min": -30.747648239135742,
      "activations/layer1_attention_weight_max": 17.216245651245117,
      "activations/layer1_attention_weight_min": -16.88652801513672,
      "activations/layer20_attention_weight_max": 30.890064239501953,
      "activations/layer20_attention_weight_min": -23.665550231933594,
      "activations/layer21_attention_weight_max": 30.07455825805664,
      "activations/layer21_attention_weight_min": -22.976682662963867,
      "activations/layer22_attention_weight_max": 46.62281036376953,
      "activations/layer22_attention_weight_min": -31.688283920288086,
      "activations/layer23_attention_weight_max": 33.581687927246094,
      "activations/layer23_attention_weight_min": -24.531118392944336,
      "activations/layer2_attention_weight_max": 35.08330535888672,
      "activations/layer2_attention_weight_min": -33.654380798339844,
      "activations/layer3_attention_weight_max": 104.9052505493164,
      "activations/layer3_attention_weight_min": -106.2558822631836,
      "activations/layer4_attention_weight_max": 115.47975158691406,
      "activations/layer4_attention_weight_min": -116.0760726928711,
      "activations/layer5_attention_weight_max": 57.246559143066406,
      "activations/layer5_attention_weight_min": -65.82215881347656,
      "activations/layer6_attention_weight_max": 46.01835250854492,
      "activations/layer6_attention_weight_min": -44.94777297973633,
      "activations/layer7_attention_weight_max": 94.08193969726562,
      "activations/layer7_attention_weight_min": -93.98487091064453,
      "activations/layer8_attention_weight_max": 41.69999313354492,
      "activations/layer8_attention_weight_min": -41.490989685058594,
      "activations/layer9_attention_weight_max": 34.368831634521484,
      "activations/layer9_attention_weight_min": -32.8939208984375,
      "epoch": 13.56,
      "learning_rate": 6.321704545454546e-05,
      "loss": 2.7559,
      "step": 233300
    },
    {
      "activations/layer0_attention_weight_max": 16.961021423339844,
      "activations/layer0_attention_weight_min": -12.305339813232422,
      "activations/layer10_attention_weight_max": 36.30349349975586,
      "activations/layer10_attention_weight_min": -35.178260803222656,
      "activations/layer11_attention_weight_max": 37.250389099121094,
      "activations/layer11_attention_weight_min": -34.9759635925293,
      "activations/layer12_attention_weight_max": 27.321842193603516,
      "activations/layer12_attention_weight_min": -27.12272834777832,
      "activations/layer13_attention_weight_max": 39.92924118041992,
      "activations/layer13_attention_weight_min": -37.95878219604492,
      "activations/layer14_attention_weight_max": 41.16561508178711,
      "activations/layer14_attention_weight_min": -38.520755767822266,
      "activations/layer15_attention_weight_max": 36.157493591308594,
      "activations/layer15_attention_weight_min": -33.41366958618164,
      "activations/layer16_attention_weight_max": 29.290210723876953,
      "activations/layer16_attention_weight_min": -28.33599281311035,
      "activations/layer17_attention_weight_max": 32.943416595458984,
      "activations/layer17_attention_weight_min": -26.94023895263672,
      "activations/layer18_attention_weight_max": 31.398971557617188,
      "activations/layer18_attention_weight_min": -24.835464477539062,
      "activations/layer19_attention_weight_max": 35.340450286865234,
      "activations/layer19_attention_weight_min": -31.860876083374023,
      "activations/layer1_attention_weight_max": 16.701791763305664,
      "activations/layer1_attention_weight_min": -14.698661804199219,
      "activations/layer20_attention_weight_max": 32.50959777832031,
      "activations/layer20_attention_weight_min": -26.639629364013672,
      "activations/layer21_attention_weight_max": 31.318241119384766,
      "activations/layer21_attention_weight_min": -25.73789405822754,
      "activations/layer22_attention_weight_max": 49.4705924987793,
      "activations/layer22_attention_weight_min": -31.9105224609375,
      "activations/layer23_attention_weight_max": 34.67656326293945,
      "activations/layer23_attention_weight_min": -26.507373809814453,
      "activations/layer2_attention_weight_max": 33.38441467285156,
      "activations/layer2_attention_weight_min": -33.58857345581055,
      "activations/layer3_attention_weight_max": 103.09919738769531,
      "activations/layer3_attention_weight_min": -101.60465240478516,
      "activations/layer4_attention_weight_max": 116.84175109863281,
      "activations/layer4_attention_weight_min": -116.74320220947266,
      "activations/layer5_attention_weight_max": 55.70259094238281,
      "activations/layer5_attention_weight_min": -63.65914535522461,
      "activations/layer6_attention_weight_max": 47.74248504638672,
      "activations/layer6_attention_weight_min": -47.94499206542969,
      "activations/layer7_attention_weight_max": 99.82145690917969,
      "activations/layer7_attention_weight_min": -100.77777862548828,
      "activations/layer8_attention_weight_max": 44.7492790222168,
      "activations/layer8_attention_weight_min": -45.67906188964844,
      "activations/layer9_attention_weight_max": 36.675357818603516,
      "activations/layer9_attention_weight_min": -35.541141510009766,
      "epoch": 13.56,
      "learning_rate": 6.319810606060606e-05,
      "loss": 2.7826,
      "step": 233350
    },
    {
      "activations/layer0_attention_weight_max": 16.803964614868164,
      "activations/layer0_attention_weight_min": -12.562257766723633,
      "activations/layer10_attention_weight_max": 33.061737060546875,
      "activations/layer10_attention_weight_min": -32.539154052734375,
      "activations/layer11_attention_weight_max": 32.384456634521484,
      "activations/layer11_attention_weight_min": -31.500110626220703,
      "activations/layer12_attention_weight_max": 24.721115112304688,
      "activations/layer12_attention_weight_min": -25.602949142456055,
      "activations/layer13_attention_weight_max": 36.286251068115234,
      "activations/layer13_attention_weight_min": -32.80622100830078,
      "activations/layer14_attention_weight_max": 42.531219482421875,
      "activations/layer14_attention_weight_min": -35.037025451660156,
      "activations/layer15_attention_weight_max": 35.8911247253418,
      "activations/layer15_attention_weight_min": -32.55073547363281,
      "activations/layer16_attention_weight_max": 29.377634048461914,
      "activations/layer16_attention_weight_min": -28.424650192260742,
      "activations/layer17_attention_weight_max": 30.104095458984375,
      "activations/layer17_attention_weight_min": -24.205163955688477,
      "activations/layer18_attention_weight_max": 28.573904037475586,
      "activations/layer18_attention_weight_min": -23.357303619384766,
      "activations/layer19_attention_weight_max": 32.14033126831055,
      "activations/layer19_attention_weight_min": -29.6358642578125,
      "activations/layer1_attention_weight_max": 17.092090606689453,
      "activations/layer1_attention_weight_min": -15.727710723876953,
      "activations/layer20_attention_weight_max": 27.214244842529297,
      "activations/layer20_attention_weight_min": -25.184724807739258,
      "activations/layer21_attention_weight_max": 27.087034225463867,
      "activations/layer21_attention_weight_min": -24.629802703857422,
      "activations/layer22_attention_weight_max": 37.469478607177734,
      "activations/layer22_attention_weight_min": -32.65702438354492,
      "activations/layer23_attention_weight_max": 32.25605392456055,
      "activations/layer23_attention_weight_min": -25.754432678222656,
      "activations/layer2_attention_weight_max": 33.50971603393555,
      "activations/layer2_attention_weight_min": -33.304725646972656,
      "activations/layer3_attention_weight_max": 96.18204498291016,
      "activations/layer3_attention_weight_min": -98.21810913085938,
      "activations/layer4_attention_weight_max": 108.17845916748047,
      "activations/layer4_attention_weight_min": -112.72139739990234,
      "activations/layer5_attention_weight_max": 54.13087463378906,
      "activations/layer5_attention_weight_min": -61.148406982421875,
      "activations/layer6_attention_weight_max": 44.03250503540039,
      "activations/layer6_attention_weight_min": -47.41379928588867,
      "activations/layer7_attention_weight_max": 85.31246185302734,
      "activations/layer7_attention_weight_min": -89.5445785522461,
      "activations/layer8_attention_weight_max": 38.6155891418457,
      "activations/layer8_attention_weight_min": -42.6787109375,
      "activations/layer9_attention_weight_max": 30.452550888061523,
      "activations/layer9_attention_weight_min": -33.578697204589844,
      "epoch": 13.56,
      "learning_rate": 6.317916666666666e-05,
      "loss": 2.7517,
      "step": 233400
    },
    {
      "activations/layer0_attention_weight_max": 16.545330047607422,
      "activations/layer0_attention_weight_min": -12.399101257324219,
      "activations/layer10_attention_weight_max": 34.560882568359375,
      "activations/layer10_attention_weight_min": -36.88397216796875,
      "activations/layer11_attention_weight_max": 35.617923736572266,
      "activations/layer11_attention_weight_min": -35.50726318359375,
      "activations/layer12_attention_weight_max": 25.9904842376709,
      "activations/layer12_attention_weight_min": -26.197893142700195,
      "activations/layer13_attention_weight_max": 39.191829681396484,
      "activations/layer13_attention_weight_min": -34.21657943725586,
      "activations/layer14_attention_weight_max": 42.522216796875,
      "activations/layer14_attention_weight_min": -37.23206329345703,
      "activations/layer15_attention_weight_max": 36.93095016479492,
      "activations/layer15_attention_weight_min": -32.646480560302734,
      "activations/layer16_attention_weight_max": 29.295677185058594,
      "activations/layer16_attention_weight_min": -30.876691818237305,
      "activations/layer17_attention_weight_max": 29.66736602783203,
      "activations/layer17_attention_weight_min": -25.775312423706055,
      "activations/layer18_attention_weight_max": 29.972326278686523,
      "activations/layer18_attention_weight_min": -23.47545623779297,
      "activations/layer19_attention_weight_max": 31.045007705688477,
      "activations/layer19_attention_weight_min": -28.553415298461914,
      "activations/layer1_attention_weight_max": 17.369482040405273,
      "activations/layer1_attention_weight_min": -15.829315185546875,
      "activations/layer20_attention_weight_max": 27.24689292907715,
      "activations/layer20_attention_weight_min": -23.79948616027832,
      "activations/layer21_attention_weight_max": 27.180498123168945,
      "activations/layer21_attention_weight_min": -23.094907760620117,
      "activations/layer22_attention_weight_max": 45.141815185546875,
      "activations/layer22_attention_weight_min": -29.622440338134766,
      "activations/layer23_attention_weight_max": 31.040847778320312,
      "activations/layer23_attention_weight_min": -24.865875244140625,
      "activations/layer2_attention_weight_max": 35.64348602294922,
      "activations/layer2_attention_weight_min": -35.943603515625,
      "activations/layer3_attention_weight_max": 98.78811645507812,
      "activations/layer3_attention_weight_min": -102.3236312866211,
      "activations/layer4_attention_weight_max": 113.22552490234375,
      "activations/layer4_attention_weight_min": -114.8711929321289,
      "activations/layer5_attention_weight_max": 54.36808395385742,
      "activations/layer5_attention_weight_min": -59.61449432373047,
      "activations/layer6_attention_weight_max": 46.278831481933594,
      "activations/layer6_attention_weight_min": -50.018821716308594,
      "activations/layer7_attention_weight_max": 94.02816772460938,
      "activations/layer7_attention_weight_min": -94.03158569335938,
      "activations/layer8_attention_weight_max": 44.060184478759766,
      "activations/layer8_attention_weight_min": -44.70448303222656,
      "activations/layer9_attention_weight_max": 34.82661056518555,
      "activations/layer9_attention_weight_min": -35.6960563659668,
      "epoch": 13.56,
      "learning_rate": 6.316022727272727e-05,
      "loss": 2.762,
      "step": 233450
    },
    {
      "activations/layer0_attention_weight_max": 15.18870735168457,
      "activations/layer0_attention_weight_min": -13.116021156311035,
      "activations/layer10_attention_weight_max": 32.29197692871094,
      "activations/layer10_attention_weight_min": -32.60610580444336,
      "activations/layer11_attention_weight_max": 32.08898162841797,
      "activations/layer11_attention_weight_min": -31.113643646240234,
      "activations/layer12_attention_weight_max": 25.386552810668945,
      "activations/layer12_attention_weight_min": -26.987062454223633,
      "activations/layer13_attention_weight_max": 37.00478744506836,
      "activations/layer13_attention_weight_min": -37.24055480957031,
      "activations/layer14_attention_weight_max": 42.2882194519043,
      "activations/layer14_attention_weight_min": -38.29043960571289,
      "activations/layer15_attention_weight_max": 36.70109176635742,
      "activations/layer15_attention_weight_min": -34.43928909301758,
      "activations/layer16_attention_weight_max": 30.96710205078125,
      "activations/layer16_attention_weight_min": -28.657859802246094,
      "activations/layer17_attention_weight_max": 31.534196853637695,
      "activations/layer17_attention_weight_min": -25.011056900024414,
      "activations/layer18_attention_weight_max": 31.53672981262207,
      "activations/layer18_attention_weight_min": -23.063190460205078,
      "activations/layer19_attention_weight_max": 33.51691436767578,
      "activations/layer19_attention_weight_min": -31.75922203063965,
      "activations/layer1_attention_weight_max": 17.525102615356445,
      "activations/layer1_attention_weight_min": -15.261487007141113,
      "activations/layer20_attention_weight_max": 30.487930297851562,
      "activations/layer20_attention_weight_min": -24.259553909301758,
      "activations/layer21_attention_weight_max": 29.469985961914062,
      "activations/layer21_attention_weight_min": -23.673290252685547,
      "activations/layer22_attention_weight_max": 37.849361419677734,
      "activations/layer22_attention_weight_min": -29.40094566345215,
      "activations/layer23_attention_weight_max": 32.32038116455078,
      "activations/layer23_attention_weight_min": -24.890607833862305,
      "activations/layer2_attention_weight_max": 37.797454833984375,
      "activations/layer2_attention_weight_min": -36.1634407043457,
      "activations/layer3_attention_weight_max": 102.73104095458984,
      "activations/layer3_attention_weight_min": -102.52204132080078,
      "activations/layer4_attention_weight_max": 111.01409149169922,
      "activations/layer4_attention_weight_min": -113.21380615234375,
      "activations/layer5_attention_weight_max": 52.55094528198242,
      "activations/layer5_attention_weight_min": -64.51419067382812,
      "activations/layer6_attention_weight_max": 43.16862487792969,
      "activations/layer6_attention_weight_min": -44.447750091552734,
      "activations/layer7_attention_weight_max": 90.02657318115234,
      "activations/layer7_attention_weight_min": -92.14830017089844,
      "activations/layer8_attention_weight_max": 41.229434967041016,
      "activations/layer8_attention_weight_min": -41.53573226928711,
      "activations/layer9_attention_weight_max": 29.94797134399414,
      "activations/layer9_attention_weight_min": -32.775146484375,
      "epoch": 13.57,
      "learning_rate": 6.314128787878788e-05,
      "loss": 2.7646,
      "step": 233500
    },
    {
      "activations/layer0_attention_weight_max": 16.627365112304688,
      "activations/layer0_attention_weight_min": -13.163980484008789,
      "activations/layer10_attention_weight_max": 32.2098388671875,
      "activations/layer10_attention_weight_min": -32.59297561645508,
      "activations/layer11_attention_weight_max": 32.24370574951172,
      "activations/layer11_attention_weight_min": -32.4088134765625,
      "activations/layer12_attention_weight_max": 41.81068420410156,
      "activations/layer12_attention_weight_min": -28.33700942993164,
      "activations/layer13_attention_weight_max": 54.175411224365234,
      "activations/layer13_attention_weight_min": -39.939693450927734,
      "activations/layer14_attention_weight_max": 47.003173828125,
      "activations/layer14_attention_weight_min": -39.77775192260742,
      "activations/layer15_attention_weight_max": 40.87844467163086,
      "activations/layer15_attention_weight_min": -35.46999740600586,
      "activations/layer16_attention_weight_max": 37.56774139404297,
      "activations/layer16_attention_weight_min": -28.377206802368164,
      "activations/layer17_attention_weight_max": 33.691650390625,
      "activations/layer17_attention_weight_min": -26.205162048339844,
      "activations/layer18_attention_weight_max": 31.815706253051758,
      "activations/layer18_attention_weight_min": -25.802507400512695,
      "activations/layer19_attention_weight_max": 35.413814544677734,
      "activations/layer19_attention_weight_min": -32.50083923339844,
      "activations/layer1_attention_weight_max": 16.73183822631836,
      "activations/layer1_attention_weight_min": -15.719648361206055,
      "activations/layer20_attention_weight_max": 32.89887237548828,
      "activations/layer20_attention_weight_min": -25.446514129638672,
      "activations/layer21_attention_weight_max": 31.84755516052246,
      "activations/layer21_attention_weight_min": -25.916015625,
      "activations/layer22_attention_weight_max": 39.231327056884766,
      "activations/layer22_attention_weight_min": -31.93970489501953,
      "activations/layer23_attention_weight_max": 32.17334747314453,
      "activations/layer23_attention_weight_min": -27.11037254333496,
      "activations/layer2_attention_weight_max": 34.44445037841797,
      "activations/layer2_attention_weight_min": -32.62725067138672,
      "activations/layer3_attention_weight_max": 98.81109619140625,
      "activations/layer3_attention_weight_min": -100.72887420654297,
      "activations/layer4_attention_weight_max": 108.14029693603516,
      "activations/layer4_attention_weight_min": -111.2166976928711,
      "activations/layer5_attention_weight_max": 51.18014144897461,
      "activations/layer5_attention_weight_min": -60.99993896484375,
      "activations/layer6_attention_weight_max": 42.681297302246094,
      "activations/layer6_attention_weight_min": -47.03044509887695,
      "activations/layer7_attention_weight_max": 90.50823211669922,
      "activations/layer7_attention_weight_min": -86.74079895019531,
      "activations/layer8_attention_weight_max": 38.7383918762207,
      "activations/layer8_attention_weight_min": -39.61691665649414,
      "activations/layer9_attention_weight_max": 31.041458129882812,
      "activations/layer9_attention_weight_min": -34.13950729370117,
      "epoch": 13.57,
      "learning_rate": 6.312234848484848e-05,
      "loss": 2.7643,
      "step": 233550
    },
    {
      "activations/layer0_attention_weight_max": 16.859695434570312,
      "activations/layer0_attention_weight_min": -12.421838760375977,
      "activations/layer10_attention_weight_max": 34.19937515258789,
      "activations/layer10_attention_weight_min": -36.92889404296875,
      "activations/layer11_attention_weight_max": 33.98604965209961,
      "activations/layer11_attention_weight_min": -35.2087287902832,
      "activations/layer12_attention_weight_max": 28.894472122192383,
      "activations/layer12_attention_weight_min": -26.43655776977539,
      "activations/layer13_attention_weight_max": 47.828975677490234,
      "activations/layer13_attention_weight_min": -35.53359603881836,
      "activations/layer14_attention_weight_max": 45.497032165527344,
      "activations/layer14_attention_weight_min": -38.32081604003906,
      "activations/layer15_attention_weight_max": 40.792724609375,
      "activations/layer15_attention_weight_min": -36.01588821411133,
      "activations/layer16_attention_weight_max": 31.608943939208984,
      "activations/layer16_attention_weight_min": -29.20421028137207,
      "activations/layer17_attention_weight_max": 30.222219467163086,
      "activations/layer17_attention_weight_min": -28.91050148010254,
      "activations/layer18_attention_weight_max": 29.464881896972656,
      "activations/layer18_attention_weight_min": -24.943462371826172,
      "activations/layer19_attention_weight_max": 38.559932708740234,
      "activations/layer19_attention_weight_min": -33.63027572631836,
      "activations/layer1_attention_weight_max": 19.158960342407227,
      "activations/layer1_attention_weight_min": -17.839387893676758,
      "activations/layer20_attention_weight_max": 32.75578308105469,
      "activations/layer20_attention_weight_min": -28.765159606933594,
      "activations/layer21_attention_weight_max": 32.38382339477539,
      "activations/layer21_attention_weight_min": -26.974533081054688,
      "activations/layer22_attention_weight_max": 40.776390075683594,
      "activations/layer22_attention_weight_min": -33.47193145751953,
      "activations/layer23_attention_weight_max": 35.179847717285156,
      "activations/layer23_attention_weight_min": -25.841800689697266,
      "activations/layer2_attention_weight_max": 33.691749572753906,
      "activations/layer2_attention_weight_min": -34.92954635620117,
      "activations/layer3_attention_weight_max": 102.37210845947266,
      "activations/layer3_attention_weight_min": -107.494873046875,
      "activations/layer4_attention_weight_max": 116.6731948852539,
      "activations/layer4_attention_weight_min": -119.0821304321289,
      "activations/layer5_attention_weight_max": 54.43848419189453,
      "activations/layer5_attention_weight_min": -67.84490203857422,
      "activations/layer6_attention_weight_max": 52.15656661987305,
      "activations/layer6_attention_weight_min": -53.33196258544922,
      "activations/layer7_attention_weight_max": 105.08253479003906,
      "activations/layer7_attention_weight_min": -110.792724609375,
      "activations/layer8_attention_weight_max": 43.7413444519043,
      "activations/layer8_attention_weight_min": -46.84455490112305,
      "activations/layer9_attention_weight_max": 32.386985778808594,
      "activations/layer9_attention_weight_min": -38.55434799194336,
      "epoch": 13.57,
      "learning_rate": 6.310378787878787e-05,
      "loss": 2.7751,
      "step": 233600
    },
    {
      "activations/layer0_attention_weight_max": 15.373466491699219,
      "activations/layer0_attention_weight_min": -12.415156364440918,
      "activations/layer10_attention_weight_max": 33.07715606689453,
      "activations/layer10_attention_weight_min": -33.07701873779297,
      "activations/layer11_attention_weight_max": 31.241226196289062,
      "activations/layer11_attention_weight_min": -32.98930358886719,
      "activations/layer12_attention_weight_max": 31.24871063232422,
      "activations/layer12_attention_weight_min": -26.073001861572266,
      "activations/layer13_attention_weight_max": 39.744529724121094,
      "activations/layer13_attention_weight_min": -34.33824157714844,
      "activations/layer14_attention_weight_max": 45.95533752441406,
      "activations/layer14_attention_weight_min": -37.902549743652344,
      "activations/layer15_attention_weight_max": 38.455780029296875,
      "activations/layer15_attention_weight_min": -33.716339111328125,
      "activations/layer16_attention_weight_max": 31.21019744873047,
      "activations/layer16_attention_weight_min": -28.1153564453125,
      "activations/layer17_attention_weight_max": 36.623226165771484,
      "activations/layer17_attention_weight_min": -26.743213653564453,
      "activations/layer18_attention_weight_max": 34.7921142578125,
      "activations/layer18_attention_weight_min": -24.573169708251953,
      "activations/layer19_attention_weight_max": 37.86920166015625,
      "activations/layer19_attention_weight_min": -32.87032699584961,
      "activations/layer1_attention_weight_max": 16.812911987304688,
      "activations/layer1_attention_weight_min": -15.01976203918457,
      "activations/layer20_attention_weight_max": 33.816200256347656,
      "activations/layer20_attention_weight_min": -25.604047775268555,
      "activations/layer21_attention_weight_max": 32.31304931640625,
      "activations/layer21_attention_weight_min": -24.83854866027832,
      "activations/layer22_attention_weight_max": 45.79944610595703,
      "activations/layer22_attention_weight_min": -29.79667854309082,
      "activations/layer23_attention_weight_max": 42.393035888671875,
      "activations/layer23_attention_weight_min": -26.8087158203125,
      "activations/layer2_attention_weight_max": 32.72175979614258,
      "activations/layer2_attention_weight_min": -34.1903076171875,
      "activations/layer3_attention_weight_max": 96.5224609375,
      "activations/layer3_attention_weight_min": -100.96830749511719,
      "activations/layer4_attention_weight_max": 105.59284973144531,
      "activations/layer4_attention_weight_min": -108.27989959716797,
      "activations/layer5_attention_weight_max": 50.87437438964844,
      "activations/layer5_attention_weight_min": -65.11550903320312,
      "activations/layer6_attention_weight_max": 41.50608444213867,
      "activations/layer6_attention_weight_min": -46.18672561645508,
      "activations/layer7_attention_weight_max": 92.77357482910156,
      "activations/layer7_attention_weight_min": -87.74546813964844,
      "activations/layer8_attention_weight_max": 37.06062316894531,
      "activations/layer8_attention_weight_min": -39.788238525390625,
      "activations/layer9_attention_weight_max": 31.555295944213867,
      "activations/layer9_attention_weight_min": -32.48339080810547,
      "epoch": 13.58,
      "learning_rate": 6.308484848484848e-05,
      "loss": 2.7587,
      "step": 233650
    },
    {
      "activations/layer0_attention_weight_max": 16.470354080200195,
      "activations/layer0_attention_weight_min": -12.768477439880371,
      "activations/layer10_attention_weight_max": 35.66607666015625,
      "activations/layer10_attention_weight_min": -32.11149978637695,
      "activations/layer11_attention_weight_max": 34.475730895996094,
      "activations/layer11_attention_weight_min": -32.50030517578125,
      "activations/layer12_attention_weight_max": 27.36417007446289,
      "activations/layer12_attention_weight_min": -25.820283889770508,
      "activations/layer13_attention_weight_max": 40.58364486694336,
      "activations/layer13_attention_weight_min": -34.156639099121094,
      "activations/layer14_attention_weight_max": 47.32961654663086,
      "activations/layer14_attention_weight_min": -35.991729736328125,
      "activations/layer15_attention_weight_max": 38.38711929321289,
      "activations/layer15_attention_weight_min": -37.016666412353516,
      "activations/layer16_attention_weight_max": 31.40886688232422,
      "activations/layer16_attention_weight_min": -28.900264739990234,
      "activations/layer17_attention_weight_max": 32.70079040527344,
      "activations/layer17_attention_weight_min": -26.39618492126465,
      "activations/layer18_attention_weight_max": 34.43950653076172,
      "activations/layer18_attention_weight_min": -23.968067169189453,
      "activations/layer19_attention_weight_max": 34.353248596191406,
      "activations/layer19_attention_weight_min": -33.60538864135742,
      "activations/layer1_attention_weight_max": 15.987154960632324,
      "activations/layer1_attention_weight_min": -14.535316467285156,
      "activations/layer20_attention_weight_max": 28.813058853149414,
      "activations/layer20_attention_weight_min": -27.439191818237305,
      "activations/layer21_attention_weight_max": 28.617584228515625,
      "activations/layer21_attention_weight_min": -25.70501708984375,
      "activations/layer22_attention_weight_max": 42.231353759765625,
      "activations/layer22_attention_weight_min": -30.408100128173828,
      "activations/layer23_attention_weight_max": 37.29438400268555,
      "activations/layer23_attention_weight_min": -27.644153594970703,
      "activations/layer2_attention_weight_max": 33.614593505859375,
      "activations/layer2_attention_weight_min": -32.80906677246094,
      "activations/layer3_attention_weight_max": 100.82606506347656,
      "activations/layer3_attention_weight_min": -98.49111938476562,
      "activations/layer4_attention_weight_max": 116.4199447631836,
      "activations/layer4_attention_weight_min": -113.67137908935547,
      "activations/layer5_attention_weight_max": 54.209800720214844,
      "activations/layer5_attention_weight_min": -63.569087982177734,
      "activations/layer6_attention_weight_max": 46.45387268066406,
      "activations/layer6_attention_weight_min": -48.828514099121094,
      "activations/layer7_attention_weight_max": 89.50975036621094,
      "activations/layer7_attention_weight_min": -89.29427337646484,
      "activations/layer8_attention_weight_max": 43.86794662475586,
      "activations/layer8_attention_weight_min": -42.50096893310547,
      "activations/layer9_attention_weight_max": 34.928077697753906,
      "activations/layer9_attention_weight_min": -33.44901657104492,
      "epoch": 13.58,
      "learning_rate": 6.306590909090909e-05,
      "loss": 2.7586,
      "step": 233700
    },
    {
      "activations/layer0_attention_weight_max": 16.867746353149414,
      "activations/layer0_attention_weight_min": -12.97565746307373,
      "activations/layer10_attention_weight_max": 41.12456130981445,
      "activations/layer10_attention_weight_min": -38.870479583740234,
      "activations/layer11_attention_weight_max": 41.868865966796875,
      "activations/layer11_attention_weight_min": -38.79820251464844,
      "activations/layer12_attention_weight_max": 34.51939010620117,
      "activations/layer12_attention_weight_min": -28.122817993164062,
      "activations/layer13_attention_weight_max": 52.33073425292969,
      "activations/layer13_attention_weight_min": -40.932830810546875,
      "activations/layer14_attention_weight_max": 62.20573806762695,
      "activations/layer14_attention_weight_min": -42.71318435668945,
      "activations/layer15_attention_weight_max": 57.42648696899414,
      "activations/layer15_attention_weight_min": -39.62497329711914,
      "activations/layer16_attention_weight_max": 37.2186164855957,
      "activations/layer16_attention_weight_min": -27.622556686401367,
      "activations/layer17_attention_weight_max": 35.5755500793457,
      "activations/layer17_attention_weight_min": -29.728214263916016,
      "activations/layer18_attention_weight_max": 32.79961013793945,
      "activations/layer18_attention_weight_min": -25.77910804748535,
      "activations/layer19_attention_weight_max": 35.829158782958984,
      "activations/layer19_attention_weight_min": -30.571020126342773,
      "activations/layer1_attention_weight_max": 17.923437118530273,
      "activations/layer1_attention_weight_min": -15.303126335144043,
      "activations/layer20_attention_weight_max": 31.867664337158203,
      "activations/layer20_attention_weight_min": -25.463367462158203,
      "activations/layer21_attention_weight_max": 29.9024658203125,
      "activations/layer21_attention_weight_min": -24.378032684326172,
      "activations/layer22_attention_weight_max": 48.39140701293945,
      "activations/layer22_attention_weight_min": -32.46381759643555,
      "activations/layer23_attention_weight_max": 40.68915557861328,
      "activations/layer23_attention_weight_min": -27.189495086669922,
      "activations/layer2_attention_weight_max": 34.75810241699219,
      "activations/layer2_attention_weight_min": -34.155052185058594,
      "activations/layer3_attention_weight_max": 98.33869171142578,
      "activations/layer3_attention_weight_min": -101.40039825439453,
      "activations/layer4_attention_weight_max": 113.03230285644531,
      "activations/layer4_attention_weight_min": -114.0553970336914,
      "activations/layer5_attention_weight_max": 53.29179382324219,
      "activations/layer5_attention_weight_min": -62.78838348388672,
      "activations/layer6_attention_weight_max": 47.13030242919922,
      "activations/layer6_attention_weight_min": -48.51155090332031,
      "activations/layer7_attention_weight_max": 103.41744232177734,
      "activations/layer7_attention_weight_min": -94.30714416503906,
      "activations/layer8_attention_weight_max": 49.168121337890625,
      "activations/layer8_attention_weight_min": -45.80427551269531,
      "activations/layer9_attention_weight_max": 43.573726654052734,
      "activations/layer9_attention_weight_min": -37.078365325927734,
      "epoch": 13.58,
      "learning_rate": 6.304696969696969e-05,
      "loss": 2.7743,
      "step": 233750
    },
    {
      "activations/layer0_attention_weight_max": 16.05756950378418,
      "activations/layer0_attention_weight_min": -12.233413696289062,
      "activations/layer10_attention_weight_max": 37.0582160949707,
      "activations/layer10_attention_weight_min": -37.92108154296875,
      "activations/layer11_attention_weight_max": 35.24987030029297,
      "activations/layer11_attention_weight_min": -36.17404556274414,
      "activations/layer12_attention_weight_max": 27.267047882080078,
      "activations/layer12_attention_weight_min": -27.997224807739258,
      "activations/layer13_attention_weight_max": 39.9152717590332,
      "activations/layer13_attention_weight_min": -39.503700256347656,
      "activations/layer14_attention_weight_max": 42.89484405517578,
      "activations/layer14_attention_weight_min": -43.8300666809082,
      "activations/layer15_attention_weight_max": 37.79264831542969,
      "activations/layer15_attention_weight_min": -37.672061920166016,
      "activations/layer16_attention_weight_max": 28.45356559753418,
      "activations/layer16_attention_weight_min": -29.3425350189209,
      "activations/layer17_attention_weight_max": 30.64205551147461,
      "activations/layer17_attention_weight_min": -27.49412727355957,
      "activations/layer18_attention_weight_max": 32.527191162109375,
      "activations/layer18_attention_weight_min": -25.57611656188965,
      "activations/layer19_attention_weight_max": 33.74502944946289,
      "activations/layer19_attention_weight_min": -31.53803062438965,
      "activations/layer1_attention_weight_max": 15.67190170288086,
      "activations/layer1_attention_weight_min": -15.830252647399902,
      "activations/layer20_attention_weight_max": 30.109119415283203,
      "activations/layer20_attention_weight_min": -28.707107543945312,
      "activations/layer21_attention_weight_max": 29.710813522338867,
      "activations/layer21_attention_weight_min": -27.0368709564209,
      "activations/layer22_attention_weight_max": 39.08103942871094,
      "activations/layer22_attention_weight_min": -30.690773010253906,
      "activations/layer23_attention_weight_max": 30.942405700683594,
      "activations/layer23_attention_weight_min": -25.925769805908203,
      "activations/layer2_attention_weight_max": 32.862937927246094,
      "activations/layer2_attention_weight_min": -32.765262603759766,
      "activations/layer3_attention_weight_max": 94.6209945678711,
      "activations/layer3_attention_weight_min": -98.47061157226562,
      "activations/layer4_attention_weight_max": 106.7956771850586,
      "activations/layer4_attention_weight_min": -108.78874969482422,
      "activations/layer5_attention_weight_max": 51.65870666503906,
      "activations/layer5_attention_weight_min": -63.986351013183594,
      "activations/layer6_attention_weight_max": 43.416080474853516,
      "activations/layer6_attention_weight_min": -47.54746627807617,
      "activations/layer7_attention_weight_max": 90.94400787353516,
      "activations/layer7_attention_weight_min": -96.89191436767578,
      "activations/layer8_attention_weight_max": 41.92074203491211,
      "activations/layer8_attention_weight_min": -46.68581771850586,
      "activations/layer9_attention_weight_max": 34.84550476074219,
      "activations/layer9_attention_weight_min": -36.8315544128418,
      "epoch": 13.59,
      "learning_rate": 6.302803030303029e-05,
      "loss": 2.7477,
      "step": 233800
    },
    {
      "activations/layer0_attention_weight_max": 15.109525680541992,
      "activations/layer0_attention_weight_min": -12.113992691040039,
      "activations/layer10_attention_weight_max": 33.6965446472168,
      "activations/layer10_attention_weight_min": -32.689186096191406,
      "activations/layer11_attention_weight_max": 34.9171142578125,
      "activations/layer11_attention_weight_min": -34.262245178222656,
      "activations/layer12_attention_weight_max": 26.489639282226562,
      "activations/layer12_attention_weight_min": -25.457902908325195,
      "activations/layer13_attention_weight_max": 37.94598388671875,
      "activations/layer13_attention_weight_min": -34.969154357910156,
      "activations/layer14_attention_weight_max": 46.53242874145508,
      "activations/layer14_attention_weight_min": -38.7943229675293,
      "activations/layer15_attention_weight_max": 37.04608154296875,
      "activations/layer15_attention_weight_min": -34.550907135009766,
      "activations/layer16_attention_weight_max": 31.626787185668945,
      "activations/layer16_attention_weight_min": -26.94556999206543,
      "activations/layer17_attention_weight_max": 29.73476219177246,
      "activations/layer17_attention_weight_min": -25.613819122314453,
      "activations/layer18_attention_weight_max": 33.23736572265625,
      "activations/layer18_attention_weight_min": -23.30294418334961,
      "activations/layer19_attention_weight_max": 35.62712097167969,
      "activations/layer19_attention_weight_min": -28.946578979492188,
      "activations/layer1_attention_weight_max": 16.85093116760254,
      "activations/layer1_attention_weight_min": -15.477033615112305,
      "activations/layer20_attention_weight_max": 29.04894256591797,
      "activations/layer20_attention_weight_min": -24.914756774902344,
      "activations/layer21_attention_weight_max": 30.77775001525879,
      "activations/layer21_attention_weight_min": -24.51888084411621,
      "activations/layer22_attention_weight_max": 44.87712478637695,
      "activations/layer22_attention_weight_min": -30.754268646240234,
      "activations/layer23_attention_weight_max": 36.54203414916992,
      "activations/layer23_attention_weight_min": -28.64175033569336,
      "activations/layer2_attention_weight_max": 34.86994934082031,
      "activations/layer2_attention_weight_min": -34.27040100097656,
      "activations/layer3_attention_weight_max": 104.74351501464844,
      "activations/layer3_attention_weight_min": -104.45609283447266,
      "activations/layer4_attention_weight_max": 116.16603088378906,
      "activations/layer4_attention_weight_min": -113.43852996826172,
      "activations/layer5_attention_weight_max": 55.32411193847656,
      "activations/layer5_attention_weight_min": -67.45232391357422,
      "activations/layer6_attention_weight_max": 44.871177673339844,
      "activations/layer6_attention_weight_min": -47.74574661254883,
      "activations/layer7_attention_weight_max": 98.38785552978516,
      "activations/layer7_attention_weight_min": -88.11884307861328,
      "activations/layer8_attention_weight_max": 40.82207107543945,
      "activations/layer8_attention_weight_min": -41.90840148925781,
      "activations/layer9_attention_weight_max": 33.41770935058594,
      "activations/layer9_attention_weight_min": -34.35954666137695,
      "epoch": 13.59,
      "learning_rate": 6.30090909090909e-05,
      "loss": 2.7775,
      "step": 233850
    },
    {
      "activations/layer0_attention_weight_max": 16.418743133544922,
      "activations/layer0_attention_weight_min": -11.928312301635742,
      "activations/layer10_attention_weight_max": 34.336761474609375,
      "activations/layer10_attention_weight_min": -34.0478401184082,
      "activations/layer11_attention_weight_max": 32.357879638671875,
      "activations/layer11_attention_weight_min": -33.34363555908203,
      "activations/layer12_attention_weight_max": 26.06983757019043,
      "activations/layer12_attention_weight_min": -27.910356521606445,
      "activations/layer13_attention_weight_max": 39.150997161865234,
      "activations/layer13_attention_weight_min": -38.43465042114258,
      "activations/layer14_attention_weight_max": 43.343719482421875,
      "activations/layer14_attention_weight_min": -42.65333557128906,
      "activations/layer15_attention_weight_max": 38.53995895385742,
      "activations/layer15_attention_weight_min": -34.485477447509766,
      "activations/layer16_attention_weight_max": 30.625381469726562,
      "activations/layer16_attention_weight_min": -27.357776641845703,
      "activations/layer17_attention_weight_max": 31.400827407836914,
      "activations/layer17_attention_weight_min": -24.827051162719727,
      "activations/layer18_attention_weight_max": 34.400840759277344,
      "activations/layer18_attention_weight_min": -23.206789016723633,
      "activations/layer19_attention_weight_max": 34.818626403808594,
      "activations/layer19_attention_weight_min": -31.93564796447754,
      "activations/layer1_attention_weight_max": 16.836387634277344,
      "activations/layer1_attention_weight_min": -15.287582397460938,
      "activations/layer20_attention_weight_max": 29.850372314453125,
      "activations/layer20_attention_weight_min": -25.2163028717041,
      "activations/layer21_attention_weight_max": 28.562711715698242,
      "activations/layer21_attention_weight_min": -23.713424682617188,
      "activations/layer22_attention_weight_max": 44.87614059448242,
      "activations/layer22_attention_weight_min": -29.540828704833984,
      "activations/layer23_attention_weight_max": 32.58938217163086,
      "activations/layer23_attention_weight_min": -26.19719696044922,
      "activations/layer2_attention_weight_max": 33.78749084472656,
      "activations/layer2_attention_weight_min": -34.22327423095703,
      "activations/layer3_attention_weight_max": 101.58553314208984,
      "activations/layer3_attention_weight_min": -102.65547180175781,
      "activations/layer4_attention_weight_max": 111.20706939697266,
      "activations/layer4_attention_weight_min": -110.96156311035156,
      "activations/layer5_attention_weight_max": 51.87891387939453,
      "activations/layer5_attention_weight_min": -61.53350067138672,
      "activations/layer6_attention_weight_max": 44.24290084838867,
      "activations/layer6_attention_weight_min": -49.1716194152832,
      "activations/layer7_attention_weight_max": 85.99036407470703,
      "activations/layer7_attention_weight_min": -91.61325073242188,
      "activations/layer8_attention_weight_max": 38.07082748413086,
      "activations/layer8_attention_weight_min": -39.50300979614258,
      "activations/layer9_attention_weight_max": 30.399667739868164,
      "activations/layer9_attention_weight_min": -32.472042083740234,
      "epoch": 13.59,
      "learning_rate": 6.29901515151515e-05,
      "loss": 2.7573,
      "step": 233900
    },
    {
      "activations/layer0_attention_weight_max": 16.928102493286133,
      "activations/layer0_attention_weight_min": -11.749444007873535,
      "activations/layer10_attention_weight_max": 33.78731918334961,
      "activations/layer10_attention_weight_min": -33.92034912109375,
      "activations/layer11_attention_weight_max": 33.78533935546875,
      "activations/layer11_attention_weight_min": -34.1504020690918,
      "activations/layer12_attention_weight_max": 36.95349884033203,
      "activations/layer12_attention_weight_min": -26.559288024902344,
      "activations/layer13_attention_weight_max": 43.43956756591797,
      "activations/layer13_attention_weight_min": -39.22446823120117,
      "activations/layer14_attention_weight_max": 47.007965087890625,
      "activations/layer14_attention_weight_min": -39.434303283691406,
      "activations/layer15_attention_weight_max": 42.11417770385742,
      "activations/layer15_attention_weight_min": -34.9528694152832,
      "activations/layer16_attention_weight_max": 31.9002685546875,
      "activations/layer16_attention_weight_min": -28.51629638671875,
      "activations/layer17_attention_weight_max": 30.09339141845703,
      "activations/layer17_attention_weight_min": -27.330854415893555,
      "activations/layer18_attention_weight_max": 31.237106323242188,
      "activations/layer18_attention_weight_min": -24.757814407348633,
      "activations/layer19_attention_weight_max": 33.241188049316406,
      "activations/layer19_attention_weight_min": -33.23627853393555,
      "activations/layer1_attention_weight_max": 16.356765747070312,
      "activations/layer1_attention_weight_min": -17.610929489135742,
      "activations/layer20_attention_weight_max": 30.896533966064453,
      "activations/layer20_attention_weight_min": -27.749855041503906,
      "activations/layer21_attention_weight_max": 32.62985610961914,
      "activations/layer21_attention_weight_min": -28.250804901123047,
      "activations/layer22_attention_weight_max": 44.6566162109375,
      "activations/layer22_attention_weight_min": -33.6068229675293,
      "activations/layer23_attention_weight_max": 34.01980972290039,
      "activations/layer23_attention_weight_min": -25.215757369995117,
      "activations/layer2_attention_weight_max": 32.945491790771484,
      "activations/layer2_attention_weight_min": -34.059417724609375,
      "activations/layer3_attention_weight_max": 96.7083740234375,
      "activations/layer3_attention_weight_min": -103.65838623046875,
      "activations/layer4_attention_weight_max": 110.6313705444336,
      "activations/layer4_attention_weight_min": -113.32476806640625,
      "activations/layer5_attention_weight_max": 54.0534782409668,
      "activations/layer5_attention_weight_min": -61.695491790771484,
      "activations/layer6_attention_weight_max": 43.664310455322266,
      "activations/layer6_attention_weight_min": -47.12985610961914,
      "activations/layer7_attention_weight_max": 91.6323013305664,
      "activations/layer7_attention_weight_min": -90.8389663696289,
      "activations/layer8_attention_weight_max": 39.310726165771484,
      "activations/layer8_attention_weight_min": -41.097591400146484,
      "activations/layer9_attention_weight_max": 32.9102897644043,
      "activations/layer9_attention_weight_min": -34.47250747680664,
      "epoch": 13.59,
      "learning_rate": 6.297121212121212e-05,
      "loss": 2.7662,
      "step": 233950
    },
    {
      "activations/layer0_attention_weight_max": 16.519466400146484,
      "activations/layer0_attention_weight_min": -12.745134353637695,
      "activations/layer10_attention_weight_max": 32.748992919921875,
      "activations/layer10_attention_weight_min": -31.947507858276367,
      "activations/layer11_attention_weight_max": 33.3750114440918,
      "activations/layer11_attention_weight_min": -32.74824523925781,
      "activations/layer12_attention_weight_max": 25.932504653930664,
      "activations/layer12_attention_weight_min": -24.798593521118164,
      "activations/layer13_attention_weight_max": 39.61371612548828,
      "activations/layer13_attention_weight_min": -33.82282257080078,
      "activations/layer14_attention_weight_max": 44.02874755859375,
      "activations/layer14_attention_weight_min": -40.82866287231445,
      "activations/layer15_attention_weight_max": 35.21137237548828,
      "activations/layer15_attention_weight_min": -33.55168914794922,
      "activations/layer16_attention_weight_max": 27.68284034729004,
      "activations/layer16_attention_weight_min": -30.340612411499023,
      "activations/layer17_attention_weight_max": 30.87027931213379,
      "activations/layer17_attention_weight_min": -25.106468200683594,
      "activations/layer18_attention_weight_max": 31.247032165527344,
      "activations/layer18_attention_weight_min": -23.84076690673828,
      "activations/layer19_attention_weight_max": 36.56698989868164,
      "activations/layer19_attention_weight_min": -31.43634033203125,
      "activations/layer1_attention_weight_max": 16.669273376464844,
      "activations/layer1_attention_weight_min": -14.785211563110352,
      "activations/layer20_attention_weight_max": 28.454517364501953,
      "activations/layer20_attention_weight_min": -25.11846160888672,
      "activations/layer21_attention_weight_max": 27.288236618041992,
      "activations/layer21_attention_weight_min": -23.783536911010742,
      "activations/layer22_attention_weight_max": 40.62598419189453,
      "activations/layer22_attention_weight_min": -30.20658302307129,
      "activations/layer23_attention_weight_max": 31.41364097595215,
      "activations/layer23_attention_weight_min": -26.261383056640625,
      "activations/layer2_attention_weight_max": 32.17658233642578,
      "activations/layer2_attention_weight_min": -31.820453643798828,
      "activations/layer3_attention_weight_max": 97.47643280029297,
      "activations/layer3_attention_weight_min": -95.82868194580078,
      "activations/layer4_attention_weight_max": 107.5484619140625,
      "activations/layer4_attention_weight_min": -113.0676040649414,
      "activations/layer5_attention_weight_max": 54.9196891784668,
      "activations/layer5_attention_weight_min": -62.0290641784668,
      "activations/layer6_attention_weight_max": 42.01921463012695,
      "activations/layer6_attention_weight_min": -47.679996490478516,
      "activations/layer7_attention_weight_max": 84.9873046875,
      "activations/layer7_attention_weight_min": -89.92845153808594,
      "activations/layer8_attention_weight_max": 39.50408172607422,
      "activations/layer8_attention_weight_min": -41.88457489013672,
      "activations/layer9_attention_weight_max": 30.852888107299805,
      "activations/layer9_attention_weight_min": -32.06668472290039,
      "epoch": 13.6,
      "learning_rate": 6.295227272727272e-05,
      "loss": 2.7579,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4971,
      "eval_samples_per_second": 505.352,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4971,
      "eval_openwebtext_samples_per_second": 505.352,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.9863,
      "eval_wikitext_samples_per_second": 229.573,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_lambada_loss": 2.41015625,
      "eval_lambada_ppl": 11.135700962413912,
      "eval_lambada_runtime": 9.5219,
      "eval_lambada_samples_per_second": 511.35,
      "step": 234000
    },
    {
      "activations/layer0_attention_weight_max": 15.451321601867676,
      "activations/layer0_attention_weight_min": -12.809922218322754,
      "activations/layer10_attention_weight_max": 36.09370803833008,
      "activations/layer10_attention_weight_min": -33.75163650512695,
      "activations/layer11_attention_weight_max": 36.34898376464844,
      "activations/layer11_attention_weight_min": -34.50862121582031,
      "activations/layer12_attention_weight_max": 28.26185417175293,
      "activations/layer12_attention_weight_min": -29.249792098999023,
      "activations/layer13_attention_weight_max": 42.24086380004883,
      "activations/layer13_attention_weight_min": -37.43294906616211,
      "activations/layer14_attention_weight_max": 56.447776794433594,
      "activations/layer14_attention_weight_min": -42.531253814697266,
      "activations/layer15_attention_weight_max": 42.114463806152344,
      "activations/layer15_attention_weight_min": -38.23844909667969,
      "activations/layer16_attention_weight_max": 38.553794860839844,
      "activations/layer16_attention_weight_min": -29.79414176940918,
      "activations/layer17_attention_weight_max": 31.10402488708496,
      "activations/layer17_attention_weight_min": -29.46920394897461,
      "activations/layer18_attention_weight_max": 32.47372055053711,
      "activations/layer18_attention_weight_min": -24.427701950073242,
      "activations/layer19_attention_weight_max": 37.968929290771484,
      "activations/layer19_attention_weight_min": -33.013851165771484,
      "activations/layer1_attention_weight_max": 16.556201934814453,
      "activations/layer1_attention_weight_min": -14.190081596374512,
      "activations/layer20_attention_weight_max": 31.149871826171875,
      "activations/layer20_attention_weight_min": -23.783580780029297,
      "activations/layer21_attention_weight_max": 31.23451805114746,
      "activations/layer21_attention_weight_min": -25.59429931640625,
      "activations/layer22_attention_weight_max": 45.73323059082031,
      "activations/layer22_attention_weight_min": -31.11861801147461,
      "activations/layer23_attention_weight_max": 37.005027770996094,
      "activations/layer23_attention_weight_min": -23.837696075439453,
      "activations/layer2_attention_weight_max": 34.32278060913086,
      "activations/layer2_attention_weight_min": -35.10210418701172,
      "activations/layer3_attention_weight_max": 98.94623565673828,
      "activations/layer3_attention_weight_min": -98.66183471679688,
      "activations/layer4_attention_weight_max": 110.14672088623047,
      "activations/layer4_attention_weight_min": -109.475341796875,
      "activations/layer5_attention_weight_max": 55.777854919433594,
      "activations/layer5_attention_weight_min": -63.85307312011719,
      "activations/layer6_attention_weight_max": 44.03755569458008,
      "activations/layer6_attention_weight_min": -46.04814529418945,
      "activations/layer7_attention_weight_max": 94.4780044555664,
      "activations/layer7_attention_weight_min": -92.87513732910156,
      "activations/layer8_attention_weight_max": 42.5973014831543,
      "activations/layer8_attention_weight_min": -41.21699142456055,
      "activations/layer9_attention_weight_max": 34.681373596191406,
      "activations/layer9_attention_weight_min": -35.26107406616211,
      "epoch": 13.6,
      "learning_rate": 6.293333333333334e-05,
      "loss": 2.7683,
      "step": 234050
    },
    {
      "activations/layer0_attention_weight_max": 15.35134506225586,
      "activations/layer0_attention_weight_min": -12.368407249450684,
      "activations/layer10_attention_weight_max": 33.43632888793945,
      "activations/layer10_attention_weight_min": -33.892181396484375,
      "activations/layer11_attention_weight_max": 34.52776336669922,
      "activations/layer11_attention_weight_min": -32.87025451660156,
      "activations/layer12_attention_weight_max": 25.820676803588867,
      "activations/layer12_attention_weight_min": -25.065406799316406,
      "activations/layer13_attention_weight_max": 39.734893798828125,
      "activations/layer13_attention_weight_min": -33.57929229736328,
      "activations/layer14_attention_weight_max": 41.13119888305664,
      "activations/layer14_attention_weight_min": -36.93386459350586,
      "activations/layer15_attention_weight_max": 37.91714096069336,
      "activations/layer15_attention_weight_min": -32.934417724609375,
      "activations/layer16_attention_weight_max": 29.267860412597656,
      "activations/layer16_attention_weight_min": -29.060428619384766,
      "activations/layer17_attention_weight_max": 32.80366516113281,
      "activations/layer17_attention_weight_min": -25.970298767089844,
      "activations/layer18_attention_weight_max": 34.81661605834961,
      "activations/layer18_attention_weight_min": -23.421125411987305,
      "activations/layer19_attention_weight_max": 36.245018005371094,
      "activations/layer19_attention_weight_min": -32.57673263549805,
      "activations/layer1_attention_weight_max": 16.887800216674805,
      "activations/layer1_attention_weight_min": -14.506952285766602,
      "activations/layer20_attention_weight_max": 31.73137092590332,
      "activations/layer20_attention_weight_min": -25.468936920166016,
      "activations/layer21_attention_weight_max": 31.693401336669922,
      "activations/layer21_attention_weight_min": -24.862085342407227,
      "activations/layer22_attention_weight_max": 46.24920654296875,
      "activations/layer22_attention_weight_min": -33.85689926147461,
      "activations/layer23_attention_weight_max": 32.88383483886719,
      "activations/layer23_attention_weight_min": -27.942480087280273,
      "activations/layer2_attention_weight_max": 34.752342224121094,
      "activations/layer2_attention_weight_min": -32.812435150146484,
      "activations/layer3_attention_weight_max": 101.06758880615234,
      "activations/layer3_attention_weight_min": -100.70001983642578,
      "activations/layer4_attention_weight_max": 113.71479797363281,
      "activations/layer4_attention_weight_min": -113.53145599365234,
      "activations/layer5_attention_weight_max": 53.2592887878418,
      "activations/layer5_attention_weight_min": -62.044334411621094,
      "activations/layer6_attention_weight_max": 47.01805114746094,
      "activations/layer6_attention_weight_min": -45.81914138793945,
      "activations/layer7_attention_weight_max": 90.31792449951172,
      "activations/layer7_attention_weight_min": -88.51280212402344,
      "activations/layer8_attention_weight_max": 41.004981994628906,
      "activations/layer8_attention_weight_min": -43.0934944152832,
      "activations/layer9_attention_weight_max": 32.06074523925781,
      "activations/layer9_attention_weight_min": -33.39048767089844,
      "epoch": 13.6,
      "learning_rate": 6.291439393939394e-05,
      "loss": 2.7689,
      "step": 234100
    },
    {
      "activations/layer0_attention_weight_max": 16.57832145690918,
      "activations/layer0_attention_weight_min": -11.338157653808594,
      "activations/layer10_attention_weight_max": 30.974864959716797,
      "activations/layer10_attention_weight_min": -34.30072784423828,
      "activations/layer11_attention_weight_max": 33.47583770751953,
      "activations/layer11_attention_weight_min": -33.948455810546875,
      "activations/layer12_attention_weight_max": 26.410070419311523,
      "activations/layer12_attention_weight_min": -25.477439880371094,
      "activations/layer13_attention_weight_max": 38.451904296875,
      "activations/layer13_attention_weight_min": -37.86259841918945,
      "activations/layer14_attention_weight_max": 47.78584289550781,
      "activations/layer14_attention_weight_min": -39.22555160522461,
      "activations/layer15_attention_weight_max": 40.549583435058594,
      "activations/layer15_attention_weight_min": -33.90379333496094,
      "activations/layer16_attention_weight_max": 35.99825668334961,
      "activations/layer16_attention_weight_min": -29.814638137817383,
      "activations/layer17_attention_weight_max": 33.677249908447266,
      "activations/layer17_attention_weight_min": -27.142385482788086,
      "activations/layer18_attention_weight_max": 33.35382843017578,
      "activations/layer18_attention_weight_min": -24.739093780517578,
      "activations/layer19_attention_weight_max": 40.272708892822266,
      "activations/layer19_attention_weight_min": -32.57970428466797,
      "activations/layer1_attention_weight_max": 16.877180099487305,
      "activations/layer1_attention_weight_min": -14.225186347961426,
      "activations/layer20_attention_weight_max": 32.99568557739258,
      "activations/layer20_attention_weight_min": -26.767784118652344,
      "activations/layer21_attention_weight_max": 37.42726135253906,
      "activations/layer21_attention_weight_min": -25.231956481933594,
      "activations/layer22_attention_weight_max": 51.04535675048828,
      "activations/layer22_attention_weight_min": -31.760297775268555,
      "activations/layer23_attention_weight_max": 40.47633361816406,
      "activations/layer23_attention_weight_min": -26.052528381347656,
      "activations/layer2_attention_weight_max": 33.77836608886719,
      "activations/layer2_attention_weight_min": -35.403141021728516,
      "activations/layer3_attention_weight_max": 94.44713592529297,
      "activations/layer3_attention_weight_min": -98.6132583618164,
      "activations/layer4_attention_weight_max": 106.15630340576172,
      "activations/layer4_attention_weight_min": -109.99723052978516,
      "activations/layer5_attention_weight_max": 52.610313415527344,
      "activations/layer5_attention_weight_min": -62.69672393798828,
      "activations/layer6_attention_weight_max": 44.81682205200195,
      "activations/layer6_attention_weight_min": -46.944393157958984,
      "activations/layer7_attention_weight_max": 89.07536315917969,
      "activations/layer7_attention_weight_min": -94.86656951904297,
      "activations/layer8_attention_weight_max": 41.8380126953125,
      "activations/layer8_attention_weight_min": -42.802284240722656,
      "activations/layer9_attention_weight_max": 32.361812591552734,
      "activations/layer9_attention_weight_min": -33.13540267944336,
      "epoch": 13.61,
      "learning_rate": 6.289545454545454e-05,
      "loss": 2.773,
      "step": 234150
    },
    {
      "activations/layer0_attention_weight_max": 17.318052291870117,
      "activations/layer0_attention_weight_min": -12.339245796203613,
      "activations/layer10_attention_weight_max": 31.142934799194336,
      "activations/layer10_attention_weight_min": -32.35674285888672,
      "activations/layer11_attention_weight_max": 31.074872970581055,
      "activations/layer11_attention_weight_min": -34.255069732666016,
      "activations/layer12_attention_weight_max": 24.258214950561523,
      "activations/layer12_attention_weight_min": -25.938737869262695,
      "activations/layer13_attention_weight_max": 36.80303955078125,
      "activations/layer13_attention_weight_min": -34.258480072021484,
      "activations/layer14_attention_weight_max": 39.350582122802734,
      "activations/layer14_attention_weight_min": -37.152679443359375,
      "activations/layer15_attention_weight_max": 35.286048889160156,
      "activations/layer15_attention_weight_min": -34.38084030151367,
      "activations/layer16_attention_weight_max": 30.683837890625,
      "activations/layer16_attention_weight_min": -28.548418045043945,
      "activations/layer17_attention_weight_max": 30.91094398498535,
      "activations/layer17_attention_weight_min": -27.085491180419922,
      "activations/layer18_attention_weight_max": 32.14729690551758,
      "activations/layer18_attention_weight_min": -26.262413024902344,
      "activations/layer19_attention_weight_max": 36.753902435302734,
      "activations/layer19_attention_weight_min": -33.43635177612305,
      "activations/layer1_attention_weight_max": 15.722633361816406,
      "activations/layer1_attention_weight_min": -13.913532257080078,
      "activations/layer20_attention_weight_max": 31.651628494262695,
      "activations/layer20_attention_weight_min": -25.568260192871094,
      "activations/layer21_attention_weight_max": 29.341123580932617,
      "activations/layer21_attention_weight_min": -25.12664222717285,
      "activations/layer22_attention_weight_max": 39.80008316040039,
      "activations/layer22_attention_weight_min": -31.99086570739746,
      "activations/layer23_attention_weight_max": 36.41815185546875,
      "activations/layer23_attention_weight_min": -26.654617309570312,
      "activations/layer2_attention_weight_max": 35.72329330444336,
      "activations/layer2_attention_weight_min": -33.28707504272461,
      "activations/layer3_attention_weight_max": 97.61126708984375,
      "activations/layer3_attention_weight_min": -99.80303192138672,
      "activations/layer4_attention_weight_max": 107.64464569091797,
      "activations/layer4_attention_weight_min": -113.76712799072266,
      "activations/layer5_attention_weight_max": 51.19548034667969,
      "activations/layer5_attention_weight_min": -62.03191375732422,
      "activations/layer6_attention_weight_max": 44.01072311401367,
      "activations/layer6_attention_weight_min": -48.84137725830078,
      "activations/layer7_attention_weight_max": 89.9748306274414,
      "activations/layer7_attention_weight_min": -94.9839096069336,
      "activations/layer8_attention_weight_max": 38.12028121948242,
      "activations/layer8_attention_weight_min": -43.4597053527832,
      "activations/layer9_attention_weight_max": 30.24199867248535,
      "activations/layer9_attention_weight_min": -32.351444244384766,
      "epoch": 13.61,
      "learning_rate": 6.287651515151515e-05,
      "loss": 2.763,
      "step": 234200
    },
    {
      "activations/layer0_attention_weight_max": 16.448585510253906,
      "activations/layer0_attention_weight_min": -12.934627532958984,
      "activations/layer10_attention_weight_max": 31.408432006835938,
      "activations/layer10_attention_weight_min": -33.34803771972656,
      "activations/layer11_attention_weight_max": 31.333127975463867,
      "activations/layer11_attention_weight_min": -32.21308898925781,
      "activations/layer12_attention_weight_max": 26.069150924682617,
      "activations/layer12_attention_weight_min": -26.12352752685547,
      "activations/layer13_attention_weight_max": 38.47197341918945,
      "activations/layer13_attention_weight_min": -35.83629608154297,
      "activations/layer14_attention_weight_max": 44.518280029296875,
      "activations/layer14_attention_weight_min": -37.71192932128906,
      "activations/layer15_attention_weight_max": 36.49087142944336,
      "activations/layer15_attention_weight_min": -33.51683807373047,
      "activations/layer16_attention_weight_max": 30.29059600830078,
      "activations/layer16_attention_weight_min": -29.290050506591797,
      "activations/layer17_attention_weight_max": 31.36773681640625,
      "activations/layer17_attention_weight_min": -25.74727439880371,
      "activations/layer18_attention_weight_max": 33.32179260253906,
      "activations/layer18_attention_weight_min": -24.036088943481445,
      "activations/layer19_attention_weight_max": 35.63186264038086,
      "activations/layer19_attention_weight_min": -32.588226318359375,
      "activations/layer1_attention_weight_max": 16.33446502685547,
      "activations/layer1_attention_weight_min": -14.741375923156738,
      "activations/layer20_attention_weight_max": 28.859161376953125,
      "activations/layer20_attention_weight_min": -25.069780349731445,
      "activations/layer21_attention_weight_max": 29.151397705078125,
      "activations/layer21_attention_weight_min": -25.396142959594727,
      "activations/layer22_attention_weight_max": 45.511199951171875,
      "activations/layer22_attention_weight_min": -31.667932510375977,
      "activations/layer23_attention_weight_max": 35.8216667175293,
      "activations/layer23_attention_weight_min": -26.513301849365234,
      "activations/layer2_attention_weight_max": 32.32158660888672,
      "activations/layer2_attention_weight_min": -32.8187141418457,
      "activations/layer3_attention_weight_max": 93.91510009765625,
      "activations/layer3_attention_weight_min": -92.77523040771484,
      "activations/layer4_attention_weight_max": 105.07491302490234,
      "activations/layer4_attention_weight_min": -107.95790100097656,
      "activations/layer5_attention_weight_max": 51.62101364135742,
      "activations/layer5_attention_weight_min": -65.23445129394531,
      "activations/layer6_attention_weight_max": 43.37202072143555,
      "activations/layer6_attention_weight_min": -44.886924743652344,
      "activations/layer7_attention_weight_max": 88.52676391601562,
      "activations/layer7_attention_weight_min": -92.11174011230469,
      "activations/layer8_attention_weight_max": 38.487369537353516,
      "activations/layer8_attention_weight_min": -39.282447814941406,
      "activations/layer9_attention_weight_max": 32.3917236328125,
      "activations/layer9_attention_weight_min": -32.289215087890625,
      "epoch": 13.61,
      "learning_rate": 6.285757575757576e-05,
      "loss": 2.7803,
      "step": 234250
    },
    {
      "activations/layer0_attention_weight_max": 16.405649185180664,
      "activations/layer0_attention_weight_min": -12.971683502197266,
      "activations/layer10_attention_weight_max": 31.73427391052246,
      "activations/layer10_attention_weight_min": -32.85380172729492,
      "activations/layer11_attention_weight_max": 32.359619140625,
      "activations/layer11_attention_weight_min": -30.83722496032715,
      "activations/layer12_attention_weight_max": 26.721847534179688,
      "activations/layer12_attention_weight_min": -26.005558013916016,
      "activations/layer13_attention_weight_max": 40.05232620239258,
      "activations/layer13_attention_weight_min": -36.24302291870117,
      "activations/layer14_attention_weight_max": 46.35446548461914,
      "activations/layer14_attention_weight_min": -41.125431060791016,
      "activations/layer15_attention_weight_max": 39.314842224121094,
      "activations/layer15_attention_weight_min": -33.440101623535156,
      "activations/layer16_attention_weight_max": 29.29411506652832,
      "activations/layer16_attention_weight_min": -27.92277717590332,
      "activations/layer17_attention_weight_max": 29.9365177154541,
      "activations/layer17_attention_weight_min": -26.751354217529297,
      "activations/layer18_attention_weight_max": 33.46640396118164,
      "activations/layer18_attention_weight_min": -22.958066940307617,
      "activations/layer19_attention_weight_max": 33.01687240600586,
      "activations/layer19_attention_weight_min": -32.46638107299805,
      "activations/layer1_attention_weight_max": 16.386049270629883,
      "activations/layer1_attention_weight_min": -14.058841705322266,
      "activations/layer20_attention_weight_max": 28.37432098388672,
      "activations/layer20_attention_weight_min": -24.589643478393555,
      "activations/layer21_attention_weight_max": 28.600278854370117,
      "activations/layer21_attention_weight_min": -25.18784523010254,
      "activations/layer22_attention_weight_max": 45.39408493041992,
      "activations/layer22_attention_weight_min": -32.90122985839844,
      "activations/layer23_attention_weight_max": 37.02873611450195,
      "activations/layer23_attention_weight_min": -27.45437240600586,
      "activations/layer2_attention_weight_max": 32.72466278076172,
      "activations/layer2_attention_weight_min": -31.45881462097168,
      "activations/layer3_attention_weight_max": 95.94303131103516,
      "activations/layer3_attention_weight_min": -98.4853744506836,
      "activations/layer4_attention_weight_max": 112.25715637207031,
      "activations/layer4_attention_weight_min": -114.1427993774414,
      "activations/layer5_attention_weight_max": 50.66262435913086,
      "activations/layer5_attention_weight_min": -62.426124572753906,
      "activations/layer6_attention_weight_max": 43.04634094238281,
      "activations/layer6_attention_weight_min": -46.120277404785156,
      "activations/layer7_attention_weight_max": 89.86415100097656,
      "activations/layer7_attention_weight_min": -91.67605590820312,
      "activations/layer8_attention_weight_max": 37.930503845214844,
      "activations/layer8_attention_weight_min": -40.31535720825195,
      "activations/layer9_attention_weight_max": 32.398353576660156,
      "activations/layer9_attention_weight_min": -32.17068099975586,
      "epoch": 13.61,
      "learning_rate": 6.283863636363636e-05,
      "loss": 2.7634,
      "step": 234300
    },
    {
      "activations/layer0_attention_weight_max": 16.117883682250977,
      "activations/layer0_attention_weight_min": -12.440969467163086,
      "activations/layer10_attention_weight_max": 33.013710021972656,
      "activations/layer10_attention_weight_min": -31.438674926757812,
      "activations/layer11_attention_weight_max": 34.76878356933594,
      "activations/layer11_attention_weight_min": -32.074195861816406,
      "activations/layer12_attention_weight_max": 27.364946365356445,
      "activations/layer12_attention_weight_min": -25.637435913085938,
      "activations/layer13_attention_weight_max": 39.86908721923828,
      "activations/layer13_attention_weight_min": -35.00019454956055,
      "activations/layer14_attention_weight_max": 44.52308654785156,
      "activations/layer14_attention_weight_min": -36.775264739990234,
      "activations/layer15_attention_weight_max": 43.08684539794922,
      "activations/layer15_attention_weight_min": -32.882511138916016,
      "activations/layer16_attention_weight_max": 30.430204391479492,
      "activations/layer16_attention_weight_min": -26.27977180480957,
      "activations/layer17_attention_weight_max": 30.78120994567871,
      "activations/layer17_attention_weight_min": -23.6209774017334,
      "activations/layer18_attention_weight_max": 31.219514846801758,
      "activations/layer18_attention_weight_min": -23.299924850463867,
      "activations/layer19_attention_weight_max": 35.463558197021484,
      "activations/layer19_attention_weight_min": -29.015304565429688,
      "activations/layer1_attention_weight_max": 17.122947692871094,
      "activations/layer1_attention_weight_min": -15.681649208068848,
      "activations/layer20_attention_weight_max": 31.980512619018555,
      "activations/layer20_attention_weight_min": -26.534217834472656,
      "activations/layer21_attention_weight_max": 30.990005493164062,
      "activations/layer21_attention_weight_min": -23.787546157836914,
      "activations/layer22_attention_weight_max": 50.859928131103516,
      "activations/layer22_attention_weight_min": -31.039588928222656,
      "activations/layer23_attention_weight_max": 34.81987762451172,
      "activations/layer23_attention_weight_min": -24.530027389526367,
      "activations/layer2_attention_weight_max": 34.218360900878906,
      "activations/layer2_attention_weight_min": -35.14016342163086,
      "activations/layer3_attention_weight_max": 97.38223266601562,
      "activations/layer3_attention_weight_min": -99.9021987915039,
      "activations/layer4_attention_weight_max": 114.74794006347656,
      "activations/layer4_attention_weight_min": -118.49565887451172,
      "activations/layer5_attention_weight_max": 54.963741302490234,
      "activations/layer5_attention_weight_min": -64.47758483886719,
      "activations/layer6_attention_weight_max": 43.57260513305664,
      "activations/layer6_attention_weight_min": -47.270931243896484,
      "activations/layer7_attention_weight_max": 92.79533386230469,
      "activations/layer7_attention_weight_min": -91.57003021240234,
      "activations/layer8_attention_weight_max": 39.22357940673828,
      "activations/layer8_attention_weight_min": -44.11692810058594,
      "activations/layer9_attention_weight_max": 31.5284423828125,
      "activations/layer9_attention_weight_min": -32.14113998413086,
      "epoch": 13.62,
      "learning_rate": 6.281969696969696e-05,
      "loss": 2.7577,
      "step": 234350
    },
    {
      "activations/layer0_attention_weight_max": 15.621840476989746,
      "activations/layer0_attention_weight_min": -12.58992862701416,
      "activations/layer10_attention_weight_max": 36.715423583984375,
      "activations/layer10_attention_weight_min": -34.58004379272461,
      "activations/layer11_attention_weight_max": 37.86549758911133,
      "activations/layer11_attention_weight_min": -34.503936767578125,
      "activations/layer12_attention_weight_max": 28.611732482910156,
      "activations/layer12_attention_weight_min": -27.18351173400879,
      "activations/layer13_attention_weight_max": 46.67790603637695,
      "activations/layer13_attention_weight_min": -38.80717086791992,
      "activations/layer14_attention_weight_max": 46.81797790527344,
      "activations/layer14_attention_weight_min": -41.3060417175293,
      "activations/layer15_attention_weight_max": 39.88209533691406,
      "activations/layer15_attention_weight_min": -35.03154373168945,
      "activations/layer16_attention_weight_max": 33.00231170654297,
      "activations/layer16_attention_weight_min": -30.9716796875,
      "activations/layer17_attention_weight_max": 32.19479751586914,
      "activations/layer17_attention_weight_min": -26.781116485595703,
      "activations/layer18_attention_weight_max": 37.11216354370117,
      "activations/layer18_attention_weight_min": -26.635459899902344,
      "activations/layer19_attention_weight_max": 40.579158782958984,
      "activations/layer19_attention_weight_min": -34.98093795776367,
      "activations/layer1_attention_weight_max": 16.153093338012695,
      "activations/layer1_attention_weight_min": -14.7579927444458,
      "activations/layer20_attention_weight_max": 34.627716064453125,
      "activations/layer20_attention_weight_min": -30.153079986572266,
      "activations/layer21_attention_weight_max": 32.00223159790039,
      "activations/layer21_attention_weight_min": -27.562175750732422,
      "activations/layer22_attention_weight_max": 52.25950622558594,
      "activations/layer22_attention_weight_min": -34.48643493652344,
      "activations/layer23_attention_weight_max": 42.741119384765625,
      "activations/layer23_attention_weight_min": -29.183063507080078,
      "activations/layer2_attention_weight_max": 33.024009704589844,
      "activations/layer2_attention_weight_min": -33.665199279785156,
      "activations/layer3_attention_weight_max": 97.97271728515625,
      "activations/layer3_attention_weight_min": -102.54359436035156,
      "activations/layer4_attention_weight_max": 109.65996551513672,
      "activations/layer4_attention_weight_min": -112.63846588134766,
      "activations/layer5_attention_weight_max": 51.6636962890625,
      "activations/layer5_attention_weight_min": -62.87466049194336,
      "activations/layer6_attention_weight_max": 45.41666030883789,
      "activations/layer6_attention_weight_min": -45.61165237426758,
      "activations/layer7_attention_weight_max": 90.26095581054688,
      "activations/layer7_attention_weight_min": -90.70232391357422,
      "activations/layer8_attention_weight_max": 42.44377517700195,
      "activations/layer8_attention_weight_min": -43.11482620239258,
      "activations/layer9_attention_weight_max": 36.0550651550293,
      "activations/layer9_attention_weight_min": -33.01943588256836,
      "epoch": 13.62,
      "learning_rate": 6.280075757575757e-05,
      "loss": 2.7788,
      "step": 234400
    },
    {
      "activations/layer0_attention_weight_max": 16.73402214050293,
      "activations/layer0_attention_weight_min": -12.841143608093262,
      "activations/layer10_attention_weight_max": 39.17394256591797,
      "activations/layer10_attention_weight_min": -33.42251205444336,
      "activations/layer11_attention_weight_max": 41.83160400390625,
      "activations/layer11_attention_weight_min": -35.59166717529297,
      "activations/layer12_attention_weight_max": 25.002613067626953,
      "activations/layer12_attention_weight_min": -28.543502807617188,
      "activations/layer13_attention_weight_max": 40.32331466674805,
      "activations/layer13_attention_weight_min": -35.77190399169922,
      "activations/layer14_attention_weight_max": 52.44676971435547,
      "activations/layer14_attention_weight_min": -39.08576202392578,
      "activations/layer15_attention_weight_max": 40.71316146850586,
      "activations/layer15_attention_weight_min": -35.24298095703125,
      "activations/layer16_attention_weight_max": 34.053794860839844,
      "activations/layer16_attention_weight_min": -28.41961097717285,
      "activations/layer17_attention_weight_max": 31.21657943725586,
      "activations/layer17_attention_weight_min": -29.74848175048828,
      "activations/layer18_attention_weight_max": 30.667648315429688,
      "activations/layer18_attention_weight_min": -27.67938804626465,
      "activations/layer19_attention_weight_max": 34.311405181884766,
      "activations/layer19_attention_weight_min": -33.932518005371094,
      "activations/layer1_attention_weight_max": 16.771778106689453,
      "activations/layer1_attention_weight_min": -16.074182510375977,
      "activations/layer20_attention_weight_max": 29.263050079345703,
      "activations/layer20_attention_weight_min": -27.745281219482422,
      "activations/layer21_attention_weight_max": 31.646543502807617,
      "activations/layer21_attention_weight_min": -28.218992233276367,
      "activations/layer22_attention_weight_max": 44.480472564697266,
      "activations/layer22_attention_weight_min": -35.26446533203125,
      "activations/layer23_attention_weight_max": 31.646743774414062,
      "activations/layer23_attention_weight_min": -28.263290405273438,
      "activations/layer2_attention_weight_max": 34.313995361328125,
      "activations/layer2_attention_weight_min": -32.68994903564453,
      "activations/layer3_attention_weight_max": 99.1917953491211,
      "activations/layer3_attention_weight_min": -98.6775894165039,
      "activations/layer4_attention_weight_max": 115.90616607666016,
      "activations/layer4_attention_weight_min": -107.6356430053711,
      "activations/layer5_attention_weight_max": 54.524803161621094,
      "activations/layer5_attention_weight_min": -62.07304382324219,
      "activations/layer6_attention_weight_max": 44.28773880004883,
      "activations/layer6_attention_weight_min": -45.65678405761719,
      "activations/layer7_attention_weight_max": 95.52490997314453,
      "activations/layer7_attention_weight_min": -91.93094635009766,
      "activations/layer8_attention_weight_max": 44.70530319213867,
      "activations/layer8_attention_weight_min": -41.47129440307617,
      "activations/layer9_attention_weight_max": 38.8921012878418,
      "activations/layer9_attention_weight_min": -37.43156433105469,
      "epoch": 13.62,
      "learning_rate": 6.278181818181818e-05,
      "loss": 2.7708,
      "step": 234450
    },
    {
      "activations/layer0_attention_weight_max": 16.18842887878418,
      "activations/layer0_attention_weight_min": -12.215544700622559,
      "activations/layer10_attention_weight_max": 32.08753204345703,
      "activations/layer10_attention_weight_min": -35.1524543762207,
      "activations/layer11_attention_weight_max": 33.459503173828125,
      "activations/layer11_attention_weight_min": -35.302001953125,
      "activations/layer12_attention_weight_max": 25.937231063842773,
      "activations/layer12_attention_weight_min": -25.96133041381836,
      "activations/layer13_attention_weight_max": 38.17628479003906,
      "activations/layer13_attention_weight_min": -35.722450256347656,
      "activations/layer14_attention_weight_max": 42.33994674682617,
      "activations/layer14_attention_weight_min": -40.124263763427734,
      "activations/layer15_attention_weight_max": 38.710514068603516,
      "activations/layer15_attention_weight_min": -36.35685348510742,
      "activations/layer16_attention_weight_max": 31.597814559936523,
      "activations/layer16_attention_weight_min": -26.898998260498047,
      "activations/layer17_attention_weight_max": 31.85649299621582,
      "activations/layer17_attention_weight_min": -26.566871643066406,
      "activations/layer18_attention_weight_max": 32.34308624267578,
      "activations/layer18_attention_weight_min": -25.514366149902344,
      "activations/layer19_attention_weight_max": 31.583295822143555,
      "activations/layer19_attention_weight_min": -33.439170837402344,
      "activations/layer1_attention_weight_max": 16.44312286376953,
      "activations/layer1_attention_weight_min": -16.22541046142578,
      "activations/layer20_attention_weight_max": 30.66644287109375,
      "activations/layer20_attention_weight_min": -24.48968505859375,
      "activations/layer21_attention_weight_max": 29.583824157714844,
      "activations/layer21_attention_weight_min": -23.13233757019043,
      "activations/layer22_attention_weight_max": 44.17914962768555,
      "activations/layer22_attention_weight_min": -31.61799430847168,
      "activations/layer23_attention_weight_max": 33.83729934692383,
      "activations/layer23_attention_weight_min": -26.991195678710938,
      "activations/layer2_attention_weight_max": 33.556095123291016,
      "activations/layer2_attention_weight_min": -35.25309753417969,
      "activations/layer3_attention_weight_max": 101.26199340820312,
      "activations/layer3_attention_weight_min": -100.42601013183594,
      "activations/layer4_attention_weight_max": 111.37226867675781,
      "activations/layer4_attention_weight_min": -108.31359100341797,
      "activations/layer5_attention_weight_max": 55.211090087890625,
      "activations/layer5_attention_weight_min": -61.622161865234375,
      "activations/layer6_attention_weight_max": 44.32716751098633,
      "activations/layer6_attention_weight_min": -46.8045768737793,
      "activations/layer7_attention_weight_max": 90.8333511352539,
      "activations/layer7_attention_weight_min": -90.00065612792969,
      "activations/layer8_attention_weight_max": 43.132568359375,
      "activations/layer8_attention_weight_min": -42.759220123291016,
      "activations/layer9_attention_weight_max": 32.86167526245117,
      "activations/layer9_attention_weight_min": -35.707977294921875,
      "epoch": 13.63,
      "learning_rate": 6.276287878787878e-05,
      "loss": 2.7488,
      "step": 234500
    },
    {
      "activations/layer0_attention_weight_max": 15.256645202636719,
      "activations/layer0_attention_weight_min": -12.537890434265137,
      "activations/layer10_attention_weight_max": 30.301864624023438,
      "activations/layer10_attention_weight_min": -30.28677749633789,
      "activations/layer11_attention_weight_max": 31.05037498474121,
      "activations/layer11_attention_weight_min": -32.05284881591797,
      "activations/layer12_attention_weight_max": 25.685115814208984,
      "activations/layer12_attention_weight_min": -23.667924880981445,
      "activations/layer13_attention_weight_max": 39.82638168334961,
      "activations/layer13_attention_weight_min": -33.38882827758789,
      "activations/layer14_attention_weight_max": 43.64524841308594,
      "activations/layer14_attention_weight_min": -36.62257385253906,
      "activations/layer15_attention_weight_max": 38.87795639038086,
      "activations/layer15_attention_weight_min": -33.66340255737305,
      "activations/layer16_attention_weight_max": 32.65665817260742,
      "activations/layer16_attention_weight_min": -27.633872985839844,
      "activations/layer17_attention_weight_max": 35.15376663208008,
      "activations/layer17_attention_weight_min": -25.6395206451416,
      "activations/layer18_attention_weight_max": 32.69268798828125,
      "activations/layer18_attention_weight_min": -23.998214721679688,
      "activations/layer19_attention_weight_max": 41.094295501708984,
      "activations/layer19_attention_weight_min": -31.804462432861328,
      "activations/layer1_attention_weight_max": 17.384326934814453,
      "activations/layer1_attention_weight_min": -14.815773010253906,
      "activations/layer20_attention_weight_max": 29.650365829467773,
      "activations/layer20_attention_weight_min": -25.033069610595703,
      "activations/layer21_attention_weight_max": 28.032970428466797,
      "activations/layer21_attention_weight_min": -25.139659881591797,
      "activations/layer22_attention_weight_max": 43.98048400878906,
      "activations/layer22_attention_weight_min": -30.935604095458984,
      "activations/layer23_attention_weight_max": 42.782100677490234,
      "activations/layer23_attention_weight_min": -24.704483032226562,
      "activations/layer2_attention_weight_max": 33.95988464355469,
      "activations/layer2_attention_weight_min": -31.1547908782959,
      "activations/layer3_attention_weight_max": 90.79979705810547,
      "activations/layer3_attention_weight_min": -92.03374481201172,
      "activations/layer4_attention_weight_max": 104.48955535888672,
      "activations/layer4_attention_weight_min": -106.29930877685547,
      "activations/layer5_attention_weight_max": 48.504638671875,
      "activations/layer5_attention_weight_min": -61.44863510131836,
      "activations/layer6_attention_weight_max": 42.15409851074219,
      "activations/layer6_attention_weight_min": -45.394229888916016,
      "activations/layer7_attention_weight_max": 86.65576934814453,
      "activations/layer7_attention_weight_min": -88.54985046386719,
      "activations/layer8_attention_weight_max": 36.253379821777344,
      "activations/layer8_attention_weight_min": -40.00171661376953,
      "activations/layer9_attention_weight_max": 28.769916534423828,
      "activations/layer9_attention_weight_min": -30.745859146118164,
      "epoch": 13.63,
      "learning_rate": 6.274393939393939e-05,
      "loss": 2.7562,
      "step": 234550
    },
    {
      "activations/layer0_attention_weight_max": 15.864384651184082,
      "activations/layer0_attention_weight_min": -12.538246154785156,
      "activations/layer10_attention_weight_max": 33.08977127075195,
      "activations/layer10_attention_weight_min": -32.57392120361328,
      "activations/layer11_attention_weight_max": 32.08055877685547,
      "activations/layer11_attention_weight_min": -31.739261627197266,
      "activations/layer12_attention_weight_max": 24.096736907958984,
      "activations/layer12_attention_weight_min": -25.1817626953125,
      "activations/layer13_attention_weight_max": 36.96977233886719,
      "activations/layer13_attention_weight_min": -34.292579650878906,
      "activations/layer14_attention_weight_max": 52.80289077758789,
      "activations/layer14_attention_weight_min": -37.76996612548828,
      "activations/layer15_attention_weight_max": 39.72935485839844,
      "activations/layer15_attention_weight_min": -32.40106964111328,
      "activations/layer16_attention_weight_max": 34.24422836303711,
      "activations/layer16_attention_weight_min": -28.229801177978516,
      "activations/layer17_attention_weight_max": 32.66910171508789,
      "activations/layer17_attention_weight_min": -26.247398376464844,
      "activations/layer18_attention_weight_max": 31.018632888793945,
      "activations/layer18_attention_weight_min": -26.45369529724121,
      "activations/layer19_attention_weight_max": 38.20143508911133,
      "activations/layer19_attention_weight_min": -31.511280059814453,
      "activations/layer1_attention_weight_max": 15.890013694763184,
      "activations/layer1_attention_weight_min": -15.3968505859375,
      "activations/layer20_attention_weight_max": 31.852569580078125,
      "activations/layer20_attention_weight_min": -26.63603401184082,
      "activations/layer21_attention_weight_max": 31.369474411010742,
      "activations/layer21_attention_weight_min": -25.845489501953125,
      "activations/layer22_attention_weight_max": 44.10840606689453,
      "activations/layer22_attention_weight_min": -29.72458267211914,
      "activations/layer23_attention_weight_max": 37.10588836669922,
      "activations/layer23_attention_weight_min": -26.728515625,
      "activations/layer2_attention_weight_max": 35.598480224609375,
      "activations/layer2_attention_weight_min": -34.33991241455078,
      "activations/layer3_attention_weight_max": 101.78694915771484,
      "activations/layer3_attention_weight_min": -103.712890625,
      "activations/layer4_attention_weight_max": 113.12401580810547,
      "activations/layer4_attention_weight_min": -111.29874420166016,
      "activations/layer5_attention_weight_max": 53.973487854003906,
      "activations/layer5_attention_weight_min": -62.64097213745117,
      "activations/layer6_attention_weight_max": 44.43104553222656,
      "activations/layer6_attention_weight_min": -48.75870132446289,
      "activations/layer7_attention_weight_max": 92.8034896850586,
      "activations/layer7_attention_weight_min": -94.9688491821289,
      "activations/layer8_attention_weight_max": 38.844234466552734,
      "activations/layer8_attention_weight_min": -40.571922302246094,
      "activations/layer9_attention_weight_max": 31.921878814697266,
      "activations/layer9_attention_weight_min": -32.55832290649414,
      "epoch": 13.63,
      "learning_rate": 6.272499999999999e-05,
      "loss": 2.771,
      "step": 234600
    },
    {
      "activations/layer0_attention_weight_max": 16.287006378173828,
      "activations/layer0_attention_weight_min": -12.326775550842285,
      "activations/layer10_attention_weight_max": 31.12736701965332,
      "activations/layer10_attention_weight_min": -33.369361877441406,
      "activations/layer11_attention_weight_max": 30.673660278320312,
      "activations/layer11_attention_weight_min": -30.82339859008789,
      "activations/layer12_attention_weight_max": 23.23753547668457,
      "activations/layer12_attention_weight_min": -24.08426856994629,
      "activations/layer13_attention_weight_max": 38.34767150878906,
      "activations/layer13_attention_weight_min": -33.34415817260742,
      "activations/layer14_attention_weight_max": 39.89706039428711,
      "activations/layer14_attention_weight_min": -35.236690521240234,
      "activations/layer15_attention_weight_max": 37.80093002319336,
      "activations/layer15_attention_weight_min": -32.77180480957031,
      "activations/layer16_attention_weight_max": 32.26982498168945,
      "activations/layer16_attention_weight_min": -26.753692626953125,
      "activations/layer17_attention_weight_max": 30.034265518188477,
      "activations/layer17_attention_weight_min": -24.277986526489258,
      "activations/layer18_attention_weight_max": 32.70195770263672,
      "activations/layer18_attention_weight_min": -22.324398040771484,
      "activations/layer19_attention_weight_max": 34.07215118408203,
      "activations/layer19_attention_weight_min": -30.167434692382812,
      "activations/layer1_attention_weight_max": 16.0920352935791,
      "activations/layer1_attention_weight_min": -15.693770408630371,
      "activations/layer20_attention_weight_max": 30.92853355407715,
      "activations/layer20_attention_weight_min": -25.464859008789062,
      "activations/layer21_attention_weight_max": 33.09379577636719,
      "activations/layer21_attention_weight_min": -23.45212173461914,
      "activations/layer22_attention_weight_max": 45.67326736450195,
      "activations/layer22_attention_weight_min": -29.416385650634766,
      "activations/layer23_attention_weight_max": 35.966060638427734,
      "activations/layer23_attention_weight_min": -24.382047653198242,
      "activations/layer2_attention_weight_max": 33.17717361450195,
      "activations/layer2_attention_weight_min": -33.16407012939453,
      "activations/layer3_attention_weight_max": 99.8453140258789,
      "activations/layer3_attention_weight_min": -99.36024475097656,
      "activations/layer4_attention_weight_max": 110.45448303222656,
      "activations/layer4_attention_weight_min": -106.92183685302734,
      "activations/layer5_attention_weight_max": 51.58844757080078,
      "activations/layer5_attention_weight_min": -63.77276611328125,
      "activations/layer6_attention_weight_max": 43.37972640991211,
      "activations/layer6_attention_weight_min": -45.31117630004883,
      "activations/layer7_attention_weight_max": 90.00531005859375,
      "activations/layer7_attention_weight_min": -90.9662094116211,
      "activations/layer8_attention_weight_max": 38.58617401123047,
      "activations/layer8_attention_weight_min": -41.69472122192383,
      "activations/layer9_attention_weight_max": 33.396728515625,
      "activations/layer9_attention_weight_min": -32.30007553100586,
      "epoch": 13.63,
      "learning_rate": 6.27060606060606e-05,
      "loss": 2.7628,
      "step": 234650
    },
    {
      "activations/layer0_attention_weight_max": 15.583486557006836,
      "activations/layer0_attention_weight_min": -13.289530754089355,
      "activations/layer10_attention_weight_max": 36.903167724609375,
      "activations/layer10_attention_weight_min": -34.888031005859375,
      "activations/layer11_attention_weight_max": 34.67637252807617,
      "activations/layer11_attention_weight_min": -32.546592712402344,
      "activations/layer12_attention_weight_max": 23.820877075195312,
      "activations/layer12_attention_weight_min": -25.340778350830078,
      "activations/layer13_attention_weight_max": 34.131996154785156,
      "activations/layer13_attention_weight_min": -32.31718444824219,
      "activations/layer14_attention_weight_max": 45.32685852050781,
      "activations/layer14_attention_weight_min": -36.17668533325195,
      "activations/layer15_attention_weight_max": 35.1318473815918,
      "activations/layer15_attention_weight_min": -31.76497459411621,
      "activations/layer16_attention_weight_max": 30.90160369873047,
      "activations/layer16_attention_weight_min": -26.856679916381836,
      "activations/layer17_attention_weight_max": 31.84308624267578,
      "activations/layer17_attention_weight_min": -26.005992889404297,
      "activations/layer18_attention_weight_max": 29.524572372436523,
      "activations/layer18_attention_weight_min": -23.71417236328125,
      "activations/layer19_attention_weight_max": 32.72184371948242,
      "activations/layer19_attention_weight_min": -32.540611267089844,
      "activations/layer1_attention_weight_max": 17.86962127685547,
      "activations/layer1_attention_weight_min": -17.839937210083008,
      "activations/layer20_attention_weight_max": 28.807132720947266,
      "activations/layer20_attention_weight_min": -26.141788482666016,
      "activations/layer21_attention_weight_max": 27.56724739074707,
      "activations/layer21_attention_weight_min": -26.476924896240234,
      "activations/layer22_attention_weight_max": 42.114097595214844,
      "activations/layer22_attention_weight_min": -32.674896240234375,
      "activations/layer23_attention_weight_max": 33.18352508544922,
      "activations/layer23_attention_weight_min": -25.15094757080078,
      "activations/layer2_attention_weight_max": 36.78528594970703,
      "activations/layer2_attention_weight_min": -32.83897018432617,
      "activations/layer3_attention_weight_max": 103.63129425048828,
      "activations/layer3_attention_weight_min": -98.32078552246094,
      "activations/layer4_attention_weight_max": 110.46309661865234,
      "activations/layer4_attention_weight_min": -105.7825927734375,
      "activations/layer5_attention_weight_max": 51.11600875854492,
      "activations/layer5_attention_weight_min": -62.099281311035156,
      "activations/layer6_attention_weight_max": 42.325279235839844,
      "activations/layer6_attention_weight_min": -46.30717086791992,
      "activations/layer7_attention_weight_max": 96.22675323486328,
      "activations/layer7_attention_weight_min": -87.63923645019531,
      "activations/layer8_attention_weight_max": 42.53106689453125,
      "activations/layer8_attention_weight_min": -41.65724563598633,
      "activations/layer9_attention_weight_max": 32.96718215942383,
      "activations/layer9_attention_weight_min": -33.99140930175781,
      "epoch": 13.64,
      "learning_rate": 6.268712121212121e-05,
      "loss": 2.7686,
      "step": 234700
    },
    {
      "activations/layer0_attention_weight_max": 16.400014877319336,
      "activations/layer0_attention_weight_min": -12.938982963562012,
      "activations/layer10_attention_weight_max": 29.04925537109375,
      "activations/layer10_attention_weight_min": -31.07354736328125,
      "activations/layer11_attention_weight_max": 29.811138153076172,
      "activations/layer11_attention_weight_min": -30.127988815307617,
      "activations/layer12_attention_weight_max": 22.83865737915039,
      "activations/layer12_attention_weight_min": -24.445735931396484,
      "activations/layer13_attention_weight_max": 34.343048095703125,
      "activations/layer13_attention_weight_min": -32.03807067871094,
      "activations/layer14_attention_weight_max": 37.344783782958984,
      "activations/layer14_attention_weight_min": -35.12400436401367,
      "activations/layer15_attention_weight_max": 32.411048889160156,
      "activations/layer15_attention_weight_min": -31.338151931762695,
      "activations/layer16_attention_weight_max": 27.792261123657227,
      "activations/layer16_attention_weight_min": -27.385984420776367,
      "activations/layer17_attention_weight_max": 25.81208038330078,
      "activations/layer17_attention_weight_min": -24.870763778686523,
      "activations/layer18_attention_weight_max": 31.997102737426758,
      "activations/layer18_attention_weight_min": -23.83836555480957,
      "activations/layer19_attention_weight_max": 28.40726089477539,
      "activations/layer19_attention_weight_min": -29.669513702392578,
      "activations/layer1_attention_weight_max": 16.16938591003418,
      "activations/layer1_attention_weight_min": -13.963205337524414,
      "activations/layer20_attention_weight_max": 24.33950424194336,
      "activations/layer20_attention_weight_min": -24.393260955810547,
      "activations/layer21_attention_weight_max": 23.727279663085938,
      "activations/layer21_attention_weight_min": -23.654611587524414,
      "activations/layer22_attention_weight_max": 37.2779426574707,
      "activations/layer22_attention_weight_min": -28.258413314819336,
      "activations/layer23_attention_weight_max": 30.092479705810547,
      "activations/layer23_attention_weight_min": -26.47957420349121,
      "activations/layer2_attention_weight_max": 32.574134826660156,
      "activations/layer2_attention_weight_min": -33.781150817871094,
      "activations/layer3_attention_weight_max": 97.80867004394531,
      "activations/layer3_attention_weight_min": -100.70787811279297,
      "activations/layer4_attention_weight_max": 110.1875,
      "activations/layer4_attention_weight_min": -110.49946594238281,
      "activations/layer5_attention_weight_max": 50.33482360839844,
      "activations/layer5_attention_weight_min": -65.2053451538086,
      "activations/layer6_attention_weight_max": 42.66018295288086,
      "activations/layer6_attention_weight_min": -45.370140075683594,
      "activations/layer7_attention_weight_max": 94.75267028808594,
      "activations/layer7_attention_weight_min": -91.9605712890625,
      "activations/layer8_attention_weight_max": 37.67275619506836,
      "activations/layer8_attention_weight_min": -39.70379638671875,
      "activations/layer9_attention_weight_max": 30.17445182800293,
      "activations/layer9_attention_weight_min": -31.604450225830078,
      "epoch": 13.64,
      "learning_rate": 6.266818181818181e-05,
      "loss": 2.7698,
      "step": 234750
    },
    {
      "activations/layer0_attention_weight_max": 15.32575511932373,
      "activations/layer0_attention_weight_min": -12.409339904785156,
      "activations/layer10_attention_weight_max": 32.99901580810547,
      "activations/layer10_attention_weight_min": -34.22052001953125,
      "activations/layer11_attention_weight_max": 35.66171646118164,
      "activations/layer11_attention_weight_min": -36.56354522705078,
      "activations/layer12_attention_weight_max": 24.806232452392578,
      "activations/layer12_attention_weight_min": -26.31069564819336,
      "activations/layer13_attention_weight_max": 37.4460334777832,
      "activations/layer13_attention_weight_min": -34.15614318847656,
      "activations/layer14_attention_weight_max": 40.488773345947266,
      "activations/layer14_attention_weight_min": -36.76877212524414,
      "activations/layer15_attention_weight_max": 34.901851654052734,
      "activations/layer15_attention_weight_min": -32.59353256225586,
      "activations/layer16_attention_weight_max": 30.042110443115234,
      "activations/layer16_attention_weight_min": -28.70998191833496,
      "activations/layer17_attention_weight_max": 29.53470802307129,
      "activations/layer17_attention_weight_min": -25.66684913635254,
      "activations/layer18_attention_weight_max": 30.199087142944336,
      "activations/layer18_attention_weight_min": -26.75948715209961,
      "activations/layer19_attention_weight_max": 34.07857131958008,
      "activations/layer19_attention_weight_min": -30.619359970092773,
      "activations/layer1_attention_weight_max": 17.040515899658203,
      "activations/layer1_attention_weight_min": -14.632111549377441,
      "activations/layer20_attention_weight_max": 29.455232620239258,
      "activations/layer20_attention_weight_min": -24.69757652282715,
      "activations/layer21_attention_weight_max": 26.69750213623047,
      "activations/layer21_attention_weight_min": -24.30510139465332,
      "activations/layer22_attention_weight_max": 39.2086296081543,
      "activations/layer22_attention_weight_min": -31.4324951171875,
      "activations/layer23_attention_weight_max": 33.78642654418945,
      "activations/layer23_attention_weight_min": -26.07815170288086,
      "activations/layer2_attention_weight_max": 32.540870666503906,
      "activations/layer2_attention_weight_min": -33.749332427978516,
      "activations/layer3_attention_weight_max": 98.4106216430664,
      "activations/layer3_attention_weight_min": -101.05850219726562,
      "activations/layer4_attention_weight_max": 106.8462142944336,
      "activations/layer4_attention_weight_min": -115.07714080810547,
      "activations/layer5_attention_weight_max": 50.21792221069336,
      "activations/layer5_attention_weight_min": -63.334590911865234,
      "activations/layer6_attention_weight_max": 46.77766418457031,
      "activations/layer6_attention_weight_min": -47.075889587402344,
      "activations/layer7_attention_weight_max": 93.44757080078125,
      "activations/layer7_attention_weight_min": -94.2967529296875,
      "activations/layer8_attention_weight_max": 40.86289978027344,
      "activations/layer8_attention_weight_min": -43.53337478637695,
      "activations/layer9_attention_weight_max": 33.12734603881836,
      "activations/layer9_attention_weight_min": -36.09541702270508,
      "epoch": 13.64,
      "learning_rate": 6.264924242424243e-05,
      "loss": 2.7647,
      "step": 234800
    },
    {
      "activations/layer0_attention_weight_max": 15.445387840270996,
      "activations/layer0_attention_weight_min": -12.513689994812012,
      "activations/layer10_attention_weight_max": 37.39374542236328,
      "activations/layer10_attention_weight_min": -35.31779479980469,
      "activations/layer11_attention_weight_max": 36.00193786621094,
      "activations/layer11_attention_weight_min": -33.90184020996094,
      "activations/layer12_attention_weight_max": 29.057512283325195,
      "activations/layer12_attention_weight_min": -26.28294563293457,
      "activations/layer13_attention_weight_max": 44.40935516357422,
      "activations/layer13_attention_weight_min": -36.472171783447266,
      "activations/layer14_attention_weight_max": 45.16477584838867,
      "activations/layer14_attention_weight_min": -42.1236686706543,
      "activations/layer15_attention_weight_max": 38.257083892822266,
      "activations/layer15_attention_weight_min": -36.17072296142578,
      "activations/layer16_attention_weight_max": 32.2568473815918,
      "activations/layer16_attention_weight_min": -29.168214797973633,
      "activations/layer17_attention_weight_max": 31.624881744384766,
      "activations/layer17_attention_weight_min": -28.185955047607422,
      "activations/layer18_attention_weight_max": 32.706851959228516,
      "activations/layer18_attention_weight_min": -24.932756423950195,
      "activations/layer19_attention_weight_max": 33.05908203125,
      "activations/layer19_attention_weight_min": -31.856903076171875,
      "activations/layer1_attention_weight_max": 17.385215759277344,
      "activations/layer1_attention_weight_min": -15.76411247253418,
      "activations/layer20_attention_weight_max": 28.2841796875,
      "activations/layer20_attention_weight_min": -24.891324996948242,
      "activations/layer21_attention_weight_max": 29.060989379882812,
      "activations/layer21_attention_weight_min": -23.109621047973633,
      "activations/layer22_attention_weight_max": 41.66912841796875,
      "activations/layer22_attention_weight_min": -31.771915435791016,
      "activations/layer23_attention_weight_max": 34.34465408325195,
      "activations/layer23_attention_weight_min": -24.473712921142578,
      "activations/layer2_attention_weight_max": 33.462074279785156,
      "activations/layer2_attention_weight_min": -34.070892333984375,
      "activations/layer3_attention_weight_max": 100.17915344238281,
      "activations/layer3_attention_weight_min": -100.8403091430664,
      "activations/layer4_attention_weight_max": 109.7933120727539,
      "activations/layer4_attention_weight_min": -117.8636703491211,
      "activations/layer5_attention_weight_max": 51.8013916015625,
      "activations/layer5_attention_weight_min": -66.31820678710938,
      "activations/layer6_attention_weight_max": 44.54912185668945,
      "activations/layer6_attention_weight_min": -48.02344512939453,
      "activations/layer7_attention_weight_max": 88.68333435058594,
      "activations/layer7_attention_weight_min": -93.26805114746094,
      "activations/layer8_attention_weight_max": 40.6169319152832,
      "activations/layer8_attention_weight_min": -42.27997589111328,
      "activations/layer9_attention_weight_max": 35.85688400268555,
      "activations/layer9_attention_weight_min": -35.940223693847656,
      "epoch": 13.65,
      "learning_rate": 6.263030303030303e-05,
      "loss": 2.7684,
      "step": 234850
    },
    {
      "activations/layer0_attention_weight_max": 15.439472198486328,
      "activations/layer0_attention_weight_min": -12.500328063964844,
      "activations/layer10_attention_weight_max": 33.02022171020508,
      "activations/layer10_attention_weight_min": -36.135162353515625,
      "activations/layer11_attention_weight_max": 33.65000534057617,
      "activations/layer11_attention_weight_min": -37.540794372558594,
      "activations/layer12_attention_weight_max": 25.695758819580078,
      "activations/layer12_attention_weight_min": -26.046541213989258,
      "activations/layer13_attention_weight_max": 37.557979583740234,
      "activations/layer13_attention_weight_min": -34.342193603515625,
      "activations/layer14_attention_weight_max": 43.587181091308594,
      "activations/layer14_attention_weight_min": -38.67282485961914,
      "activations/layer15_attention_weight_max": 36.283329010009766,
      "activations/layer15_attention_weight_min": -32.93550109863281,
      "activations/layer16_attention_weight_max": 29.544429779052734,
      "activations/layer16_attention_weight_min": -28.248083114624023,
      "activations/layer17_attention_weight_max": 32.737545013427734,
      "activations/layer17_attention_weight_min": -26.867395401000977,
      "activations/layer18_attention_weight_max": 31.9627685546875,
      "activations/layer18_attention_weight_min": -26.313289642333984,
      "activations/layer19_attention_weight_max": 34.089717864990234,
      "activations/layer19_attention_weight_min": -31.684293746948242,
      "activations/layer1_attention_weight_max": 17.0108642578125,
      "activations/layer1_attention_weight_min": -14.402288436889648,
      "activations/layer20_attention_weight_max": 29.714454650878906,
      "activations/layer20_attention_weight_min": -26.970640182495117,
      "activations/layer21_attention_weight_max": 29.103679656982422,
      "activations/layer21_attention_weight_min": -25.975610733032227,
      "activations/layer22_attention_weight_max": 44.98848342895508,
      "activations/layer22_attention_weight_min": -30.836164474487305,
      "activations/layer23_attention_weight_max": 34.855072021484375,
      "activations/layer23_attention_weight_min": -26.01416778564453,
      "activations/layer2_attention_weight_max": 35.150489807128906,
      "activations/layer2_attention_weight_min": -35.91436004638672,
      "activations/layer3_attention_weight_max": 101.35940551757812,
      "activations/layer3_attention_weight_min": -104.11193084716797,
      "activations/layer4_attention_weight_max": 114.3302230834961,
      "activations/layer4_attention_weight_min": -111.50933074951172,
      "activations/layer5_attention_weight_max": 51.9068603515625,
      "activations/layer5_attention_weight_min": -62.147029876708984,
      "activations/layer6_attention_weight_max": 48.248313903808594,
      "activations/layer6_attention_weight_min": -50.950748443603516,
      "activations/layer7_attention_weight_max": 95.38893127441406,
      "activations/layer7_attention_weight_min": -96.50899505615234,
      "activations/layer8_attention_weight_max": 41.431312561035156,
      "activations/layer8_attention_weight_min": -44.35661697387695,
      "activations/layer9_attention_weight_max": 34.19136428833008,
      "activations/layer9_attention_weight_min": -35.9055290222168,
      "epoch": 13.65,
      "learning_rate": 6.261136363636363e-05,
      "loss": 2.7479,
      "step": 234900
    },
    {
      "activations/layer0_attention_weight_max": 16.37041664123535,
      "activations/layer0_attention_weight_min": -12.24534797668457,
      "activations/layer10_attention_weight_max": 30.665218353271484,
      "activations/layer10_attention_weight_min": -32.171905517578125,
      "activations/layer11_attention_weight_max": 32.68587875366211,
      "activations/layer11_attention_weight_min": -32.692230224609375,
      "activations/layer12_attention_weight_max": 25.533016204833984,
      "activations/layer12_attention_weight_min": -26.01011848449707,
      "activations/layer13_attention_weight_max": 40.600921630859375,
      "activations/layer13_attention_weight_min": -34.51406478881836,
      "activations/layer14_attention_weight_max": 43.70369338989258,
      "activations/layer14_attention_weight_min": -37.01328659057617,
      "activations/layer15_attention_weight_max": 37.057613372802734,
      "activations/layer15_attention_weight_min": -32.408809661865234,
      "activations/layer16_attention_weight_max": 30.791826248168945,
      "activations/layer16_attention_weight_min": -27.385944366455078,
      "activations/layer17_attention_weight_max": 31.336669921875,
      "activations/layer17_attention_weight_min": -25.217275619506836,
      "activations/layer18_attention_weight_max": 30.94863510131836,
      "activations/layer18_attention_weight_min": -25.21682357788086,
      "activations/layer19_attention_weight_max": 33.734432220458984,
      "activations/layer19_attention_weight_min": -30.878036499023438,
      "activations/layer1_attention_weight_max": 18.173538208007812,
      "activations/layer1_attention_weight_min": -15.268714904785156,
      "activations/layer20_attention_weight_max": 28.27943992614746,
      "activations/layer20_attention_weight_min": -25.25126838684082,
      "activations/layer21_attention_weight_max": 26.41985321044922,
      "activations/layer21_attention_weight_min": -23.584609985351562,
      "activations/layer22_attention_weight_max": 42.59651184082031,
      "activations/layer22_attention_weight_min": -30.427385330200195,
      "activations/layer23_attention_weight_max": 33.59242630004883,
      "activations/layer23_attention_weight_min": -23.943891525268555,
      "activations/layer2_attention_weight_max": 32.8717155456543,
      "activations/layer2_attention_weight_min": -34.80208969116211,
      "activations/layer3_attention_weight_max": 98.076904296875,
      "activations/layer3_attention_weight_min": -104.79288482666016,
      "activations/layer4_attention_weight_max": 107.90931701660156,
      "activations/layer4_attention_weight_min": -111.2007827758789,
      "activations/layer5_attention_weight_max": 51.089881896972656,
      "activations/layer5_attention_weight_min": -60.531063079833984,
      "activations/layer6_attention_weight_max": 43.10041809082031,
      "activations/layer6_attention_weight_min": -47.19724655151367,
      "activations/layer7_attention_weight_max": 94.67623138427734,
      "activations/layer7_attention_weight_min": -92.80577850341797,
      "activations/layer8_attention_weight_max": 40.575897216796875,
      "activations/layer8_attention_weight_min": -41.862220764160156,
      "activations/layer9_attention_weight_max": 32.37647247314453,
      "activations/layer9_attention_weight_min": -33.48930358886719,
      "epoch": 13.65,
      "learning_rate": 6.259242424242423e-05,
      "loss": 2.7744,
      "step": 234950
    },
    {
      "activations/layer0_attention_weight_max": 16.029022216796875,
      "activations/layer0_attention_weight_min": -12.59839916229248,
      "activations/layer10_attention_weight_max": 32.30104446411133,
      "activations/layer10_attention_weight_min": -32.02599334716797,
      "activations/layer11_attention_weight_max": 32.729583740234375,
      "activations/layer11_attention_weight_min": -32.116905212402344,
      "activations/layer12_attention_weight_max": 24.450000762939453,
      "activations/layer12_attention_weight_min": -27.398067474365234,
      "activations/layer13_attention_weight_max": 38.474571228027344,
      "activations/layer13_attention_weight_min": -36.219520568847656,
      "activations/layer14_attention_weight_max": 41.1100959777832,
      "activations/layer14_attention_weight_min": -39.88068389892578,
      "activations/layer15_attention_weight_max": 36.59423065185547,
      "activations/layer15_attention_weight_min": -36.58260726928711,
      "activations/layer16_attention_weight_max": 30.740201950073242,
      "activations/layer16_attention_weight_min": -28.165576934814453,
      "activations/layer17_attention_weight_max": 31.10579490661621,
      "activations/layer17_attention_weight_min": -26.56882095336914,
      "activations/layer18_attention_weight_max": 30.843351364135742,
      "activations/layer18_attention_weight_min": -26.455745697021484,
      "activations/layer19_attention_weight_max": 31.56788444519043,
      "activations/layer19_attention_weight_min": -30.447607040405273,
      "activations/layer1_attention_weight_max": 17.568462371826172,
      "activations/layer1_attention_weight_min": -15.090746879577637,
      "activations/layer20_attention_weight_max": 28.491775512695312,
      "activations/layer20_attention_weight_min": -23.0933837890625,
      "activations/layer21_attention_weight_max": 27.151376724243164,
      "activations/layer21_attention_weight_min": -22.897066116333008,
      "activations/layer22_attention_weight_max": 40.45000076293945,
      "activations/layer22_attention_weight_min": -33.46788787841797,
      "activations/layer23_attention_weight_max": 29.038265228271484,
      "activations/layer23_attention_weight_min": -25.01093292236328,
      "activations/layer2_attention_weight_max": 33.875247955322266,
      "activations/layer2_attention_weight_min": -34.24112319946289,
      "activations/layer3_attention_weight_max": 99.2804946899414,
      "activations/layer3_attention_weight_min": -98.93892669677734,
      "activations/layer4_attention_weight_max": 106.4874267578125,
      "activations/layer4_attention_weight_min": -107.57484436035156,
      "activations/layer5_attention_weight_max": 49.72203826904297,
      "activations/layer5_attention_weight_min": -62.502010345458984,
      "activations/layer6_attention_weight_max": 41.3392333984375,
      "activations/layer6_attention_weight_min": -44.209800720214844,
      "activations/layer7_attention_weight_max": 87.03241729736328,
      "activations/layer7_attention_weight_min": -87.44371795654297,
      "activations/layer8_attention_weight_max": 38.407752990722656,
      "activations/layer8_attention_weight_min": -40.00204849243164,
      "activations/layer9_attention_weight_max": 32.967437744140625,
      "activations/layer9_attention_weight_min": -31.826852798461914,
      "epoch": 13.65,
      "learning_rate": 6.257348484848485e-05,
      "loss": 2.7637,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4715,
      "eval_samples_per_second": 506.877,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4715,
      "eval_openwebtext_samples_per_second": 506.877,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.9462,
      "eval_wikitext_samples_per_second": 234.303,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_lambada_loss": 2.421875,
      "eval_lambada_ppl": 11.26696508157019,
      "eval_lambada_runtime": 9.5467,
      "eval_lambada_samples_per_second": 510.017,
      "step": 235000
    },
    {
      "activations/layer0_attention_weight_max": 15.027512550354004,
      "activations/layer0_attention_weight_min": -13.212699890136719,
      "activations/layer10_attention_weight_max": 32.17523193359375,
      "activations/layer10_attention_weight_min": -32.13847351074219,
      "activations/layer11_attention_weight_max": 34.06795883178711,
      "activations/layer11_attention_weight_min": -32.3121223449707,
      "activations/layer12_attention_weight_max": 25.01943588256836,
      "activations/layer12_attention_weight_min": -24.230587005615234,
      "activations/layer13_attention_weight_max": 37.854637145996094,
      "activations/layer13_attention_weight_min": -33.64683151245117,
      "activations/layer14_attention_weight_max": 41.80651092529297,
      "activations/layer14_attention_weight_min": -36.125770568847656,
      "activations/layer15_attention_weight_max": 35.3825569152832,
      "activations/layer15_attention_weight_min": -32.049560546875,
      "activations/layer16_attention_weight_max": 33.67040252685547,
      "activations/layer16_attention_weight_min": -28.92667007446289,
      "activations/layer17_attention_weight_max": 30.587860107421875,
      "activations/layer17_attention_weight_min": -25.899200439453125,
      "activations/layer18_attention_weight_max": 30.008886337280273,
      "activations/layer18_attention_weight_min": -23.81118392944336,
      "activations/layer19_attention_weight_max": 32.86615753173828,
      "activations/layer19_attention_weight_min": -31.66943359375,
      "activations/layer1_attention_weight_max": 16.24130630493164,
      "activations/layer1_attention_weight_min": -14.82286262512207,
      "activations/layer20_attention_weight_max": 30.472333908081055,
      "activations/layer20_attention_weight_min": -26.143522262573242,
      "activations/layer21_attention_weight_max": 29.157323837280273,
      "activations/layer21_attention_weight_min": -25.805240631103516,
      "activations/layer22_attention_weight_max": 44.17234802246094,
      "activations/layer22_attention_weight_min": -33.49764633178711,
      "activations/layer23_attention_weight_max": 36.27991485595703,
      "activations/layer23_attention_weight_min": -27.07103729248047,
      "activations/layer2_attention_weight_max": 36.007598876953125,
      "activations/layer2_attention_weight_min": -35.82805633544922,
      "activations/layer3_attention_weight_max": 104.0708999633789,
      "activations/layer3_attention_weight_min": -103.18505859375,
      "activations/layer4_attention_weight_max": 114.16374969482422,
      "activations/layer4_attention_weight_min": -112.54994201660156,
      "activations/layer5_attention_weight_max": 51.89939498901367,
      "activations/layer5_attention_weight_min": -64.64836120605469,
      "activations/layer6_attention_weight_max": 45.32146072387695,
      "activations/layer6_attention_weight_min": -46.37302017211914,
      "activations/layer7_attention_weight_max": 88.73943328857422,
      "activations/layer7_attention_weight_min": -88.77176666259766,
      "activations/layer8_attention_weight_max": 39.628108978271484,
      "activations/layer8_attention_weight_min": -42.01103973388672,
      "activations/layer9_attention_weight_max": 31.925565719604492,
      "activations/layer9_attention_weight_min": -30.741565704345703,
      "epoch": 13.66,
      "learning_rate": 6.255454545454545e-05,
      "loss": 2.7495,
      "step": 235050
    },
    {
      "activations/layer0_attention_weight_max": 16.46034049987793,
      "activations/layer0_attention_weight_min": -12.47246265411377,
      "activations/layer10_attention_weight_max": 34.024600982666016,
      "activations/layer10_attention_weight_min": -31.99351692199707,
      "activations/layer11_attention_weight_max": 31.105480194091797,
      "activations/layer11_attention_weight_min": -30.864051818847656,
      "activations/layer12_attention_weight_max": 24.606489181518555,
      "activations/layer12_attention_weight_min": -25.06391716003418,
      "activations/layer13_attention_weight_max": 37.14313888549805,
      "activations/layer13_attention_weight_min": -34.93006896972656,
      "activations/layer14_attention_weight_max": 41.44248580932617,
      "activations/layer14_attention_weight_min": -38.1958122253418,
      "activations/layer15_attention_weight_max": 34.93186950683594,
      "activations/layer15_attention_weight_min": -33.63937759399414,
      "activations/layer16_attention_weight_max": 27.65495491027832,
      "activations/layer16_attention_weight_min": -29.242441177368164,
      "activations/layer17_attention_weight_max": 28.858802795410156,
      "activations/layer17_attention_weight_min": -25.525171279907227,
      "activations/layer18_attention_weight_max": 31.202306747436523,
      "activations/layer18_attention_weight_min": -26.67181968688965,
      "activations/layer19_attention_weight_max": 31.395797729492188,
      "activations/layer19_attention_weight_min": -31.056631088256836,
      "activations/layer1_attention_weight_max": 18.362842559814453,
      "activations/layer1_attention_weight_min": -15.370664596557617,
      "activations/layer20_attention_weight_max": 26.17705726623535,
      "activations/layer20_attention_weight_min": -25.097959518432617,
      "activations/layer21_attention_weight_max": 28.299680709838867,
      "activations/layer21_attention_weight_min": -24.354455947875977,
      "activations/layer22_attention_weight_max": 40.96052551269531,
      "activations/layer22_attention_weight_min": -31.15696907043457,
      "activations/layer23_attention_weight_max": 31.771833419799805,
      "activations/layer23_attention_weight_min": -26.000789642333984,
      "activations/layer2_attention_weight_max": 32.556663513183594,
      "activations/layer2_attention_weight_min": -32.96308898925781,
      "activations/layer3_attention_weight_max": 98.31565856933594,
      "activations/layer3_attention_weight_min": -97.30271911621094,
      "activations/layer4_attention_weight_max": 107.19445037841797,
      "activations/layer4_attention_weight_min": -107.48583221435547,
      "activations/layer5_attention_weight_max": 50.167518615722656,
      "activations/layer5_attention_weight_min": -64.3729248046875,
      "activations/layer6_attention_weight_max": 42.57963180541992,
      "activations/layer6_attention_weight_min": -45.347530364990234,
      "activations/layer7_attention_weight_max": 83.7544174194336,
      "activations/layer7_attention_weight_min": -83.69232177734375,
      "activations/layer8_attention_weight_max": 39.478424072265625,
      "activations/layer8_attention_weight_min": -40.569374084472656,
      "activations/layer9_attention_weight_max": 33.081298828125,
      "activations/layer9_attention_weight_min": -32.65497970581055,
      "epoch": 13.66,
      "learning_rate": 6.253560606060605e-05,
      "loss": 2.7701,
      "step": 235100
    },
    {
      "activations/layer0_attention_weight_max": 16.198467254638672,
      "activations/layer0_attention_weight_min": -12.874746322631836,
      "activations/layer10_attention_weight_max": 29.690231323242188,
      "activations/layer10_attention_weight_min": -30.399566650390625,
      "activations/layer11_attention_weight_max": 30.7515869140625,
      "activations/layer11_attention_weight_min": -29.976295471191406,
      "activations/layer12_attention_weight_max": 26.052078247070312,
      "activations/layer12_attention_weight_min": -25.168710708618164,
      "activations/layer13_attention_weight_max": 40.149635314941406,
      "activations/layer13_attention_weight_min": -33.48475646972656,
      "activations/layer14_attention_weight_max": 42.08359146118164,
      "activations/layer14_attention_weight_min": -35.71037292480469,
      "activations/layer15_attention_weight_max": 38.296409606933594,
      "activations/layer15_attention_weight_min": -32.38553237915039,
      "activations/layer16_attention_weight_max": 31.940935134887695,
      "activations/layer16_attention_weight_min": -26.358638763427734,
      "activations/layer17_attention_weight_max": 31.1168212890625,
      "activations/layer17_attention_weight_min": -24.140380859375,
      "activations/layer18_attention_weight_max": 32.519840240478516,
      "activations/layer18_attention_weight_min": -22.622825622558594,
      "activations/layer19_attention_weight_max": 35.22579574584961,
      "activations/layer19_attention_weight_min": -29.882169723510742,
      "activations/layer1_attention_weight_max": 16.333255767822266,
      "activations/layer1_attention_weight_min": -14.543444633483887,
      "activations/layer20_attention_weight_max": 30.803781509399414,
      "activations/layer20_attention_weight_min": -23.098772048950195,
      "activations/layer21_attention_weight_max": 32.85616683959961,
      "activations/layer21_attention_weight_min": -23.35514259338379,
      "activations/layer22_attention_weight_max": 40.37654113769531,
      "activations/layer22_attention_weight_min": -29.184751510620117,
      "activations/layer23_attention_weight_max": 31.5130672454834,
      "activations/layer23_attention_weight_min": -23.651962280273438,
      "activations/layer2_attention_weight_max": 33.54509353637695,
      "activations/layer2_attention_weight_min": -35.34019470214844,
      "activations/layer3_attention_weight_max": 97.7248764038086,
      "activations/layer3_attention_weight_min": -99.23273468017578,
      "activations/layer4_attention_weight_max": 107.99980926513672,
      "activations/layer4_attention_weight_min": -108.21388244628906,
      "activations/layer5_attention_weight_max": 49.97172164916992,
      "activations/layer5_attention_weight_min": -65.99224090576172,
      "activations/layer6_attention_weight_max": 41.16168975830078,
      "activations/layer6_attention_weight_min": -46.06517791748047,
      "activations/layer7_attention_weight_max": 88.32195281982422,
      "activations/layer7_attention_weight_min": -90.51927185058594,
      "activations/layer8_attention_weight_max": 38.58448791503906,
      "activations/layer8_attention_weight_min": -39.58224105834961,
      "activations/layer9_attention_weight_max": 29.957260131835938,
      "activations/layer9_attention_weight_min": -31.356653213500977,
      "epoch": 13.66,
      "learning_rate": 6.251666666666666e-05,
      "loss": 2.7646,
      "step": 235150
    },
    {
      "activations/layer0_attention_weight_max": 16.053272247314453,
      "activations/layer0_attention_weight_min": -12.102681159973145,
      "activations/layer10_attention_weight_max": 31.85784149169922,
      "activations/layer10_attention_weight_min": -33.098236083984375,
      "activations/layer11_attention_weight_max": 31.720043182373047,
      "activations/layer11_attention_weight_min": -33.99070739746094,
      "activations/layer12_attention_weight_max": 25.196619033813477,
      "activations/layer12_attention_weight_min": -25.94980812072754,
      "activations/layer13_attention_weight_max": 37.089996337890625,
      "activations/layer13_attention_weight_min": -33.904083251953125,
      "activations/layer14_attention_weight_max": 40.12411880493164,
      "activations/layer14_attention_weight_min": -37.98413848876953,
      "activations/layer15_attention_weight_max": 42.57000732421875,
      "activations/layer15_attention_weight_min": -32.610591888427734,
      "activations/layer16_attention_weight_max": 27.780227661132812,
      "activations/layer16_attention_weight_min": -27.672771453857422,
      "activations/layer17_attention_weight_max": 29.995635986328125,
      "activations/layer17_attention_weight_min": -26.289236068725586,
      "activations/layer18_attention_weight_max": 27.735307693481445,
      "activations/layer18_attention_weight_min": -25.61168098449707,
      "activations/layer19_attention_weight_max": 31.794998168945312,
      "activations/layer19_attention_weight_min": -31.04494285583496,
      "activations/layer1_attention_weight_max": 16.374332427978516,
      "activations/layer1_attention_weight_min": -13.988659858703613,
      "activations/layer20_attention_weight_max": 30.638399124145508,
      "activations/layer20_attention_weight_min": -27.637483596801758,
      "activations/layer21_attention_weight_max": 29.616994857788086,
      "activations/layer21_attention_weight_min": -26.24966812133789,
      "activations/layer22_attention_weight_max": 41.03253173828125,
      "activations/layer22_attention_weight_min": -32.83123779296875,
      "activations/layer23_attention_weight_max": 32.84965515136719,
      "activations/layer23_attention_weight_min": -27.175792694091797,
      "activations/layer2_attention_weight_max": 33.24783706665039,
      "activations/layer2_attention_weight_min": -33.88422775268555,
      "activations/layer3_attention_weight_max": 98.31314849853516,
      "activations/layer3_attention_weight_min": -98.75098419189453,
      "activations/layer4_attention_weight_max": 110.8665542602539,
      "activations/layer4_attention_weight_min": -105.64078521728516,
      "activations/layer5_attention_weight_max": 50.252723693847656,
      "activations/layer5_attention_weight_min": -60.35873031616211,
      "activations/layer6_attention_weight_max": 43.6884880065918,
      "activations/layer6_attention_weight_min": -45.65443801879883,
      "activations/layer7_attention_weight_max": 88.23326873779297,
      "activations/layer7_attention_weight_min": -88.39179992675781,
      "activations/layer8_attention_weight_max": 39.6245002746582,
      "activations/layer8_attention_weight_min": -39.56643295288086,
      "activations/layer9_attention_weight_max": 32.158077239990234,
      "activations/layer9_attention_weight_min": -34.17097091674805,
      "epoch": 13.67,
      "learning_rate": 6.249772727272727e-05,
      "loss": 2.7702,
      "step": 235200
    },
    {
      "activations/layer0_attention_weight_max": 15.52862548828125,
      "activations/layer0_attention_weight_min": -13.265876770019531,
      "activations/layer10_attention_weight_max": 33.734107971191406,
      "activations/layer10_attention_weight_min": -34.10345458984375,
      "activations/layer11_attention_weight_max": 34.57923126220703,
      "activations/layer11_attention_weight_min": -32.6435661315918,
      "activations/layer12_attention_weight_max": 26.974855422973633,
      "activations/layer12_attention_weight_min": -26.1179256439209,
      "activations/layer13_attention_weight_max": 40.771339416503906,
      "activations/layer13_attention_weight_min": -36.212276458740234,
      "activations/layer14_attention_weight_max": 48.66895294189453,
      "activations/layer14_attention_weight_min": -40.158042907714844,
      "activations/layer15_attention_weight_max": 40.50837707519531,
      "activations/layer15_attention_weight_min": -33.387969970703125,
      "activations/layer16_attention_weight_max": 30.806106567382812,
      "activations/layer16_attention_weight_min": -28.71820831298828,
      "activations/layer17_attention_weight_max": 32.847686767578125,
      "activations/layer17_attention_weight_min": -25.05757713317871,
      "activations/layer18_attention_weight_max": 33.75694274902344,
      "activations/layer18_attention_weight_min": -23.405057907104492,
      "activations/layer19_attention_weight_max": 36.0020751953125,
      "activations/layer19_attention_weight_min": -29.376663208007812,
      "activations/layer1_attention_weight_max": 16.82931900024414,
      "activations/layer1_attention_weight_min": -14.96822738647461,
      "activations/layer20_attention_weight_max": 29.118114471435547,
      "activations/layer20_attention_weight_min": -25.377290725708008,
      "activations/layer21_attention_weight_max": 32.03738784790039,
      "activations/layer21_attention_weight_min": -24.886289596557617,
      "activations/layer22_attention_weight_max": 46.705074310302734,
      "activations/layer22_attention_weight_min": -28.859222412109375,
      "activations/layer23_attention_weight_max": 36.25441360473633,
      "activations/layer23_attention_weight_min": -24.10182762145996,
      "activations/layer2_attention_weight_max": 33.92973327636719,
      "activations/layer2_attention_weight_min": -33.55765151977539,
      "activations/layer3_attention_weight_max": 99.92381286621094,
      "activations/layer3_attention_weight_min": -100.45464324951172,
      "activations/layer4_attention_weight_max": 110.20548248291016,
      "activations/layer4_attention_weight_min": -111.47967529296875,
      "activations/layer5_attention_weight_max": 50.95991134643555,
      "activations/layer5_attention_weight_min": -64.90110778808594,
      "activations/layer6_attention_weight_max": 43.44270706176758,
      "activations/layer6_attention_weight_min": -48.951595306396484,
      "activations/layer7_attention_weight_max": 90.49897766113281,
      "activations/layer7_attention_weight_min": -88.69584655761719,
      "activations/layer8_attention_weight_max": 41.08948516845703,
      "activations/layer8_attention_weight_min": -41.12834930419922,
      "activations/layer9_attention_weight_max": 32.43516159057617,
      "activations/layer9_attention_weight_min": -33.7396354675293,
      "epoch": 13.67,
      "learning_rate": 6.247916666666666e-05,
      "loss": 2.7708,
      "step": 235250
    },
    {
      "activations/layer0_attention_weight_max": 16.0924129486084,
      "activations/layer0_attention_weight_min": -12.805028915405273,
      "activations/layer10_attention_weight_max": 30.382858276367188,
      "activations/layer10_attention_weight_min": -32.46352005004883,
      "activations/layer11_attention_weight_max": 32.743980407714844,
      "activations/layer11_attention_weight_min": -33.48664474487305,
      "activations/layer12_attention_weight_max": 24.924537658691406,
      "activations/layer12_attention_weight_min": -23.37824249267578,
      "activations/layer13_attention_weight_max": 35.6422233581543,
      "activations/layer13_attention_weight_min": -32.907100677490234,
      "activations/layer14_attention_weight_max": 41.277740478515625,
      "activations/layer14_attention_weight_min": -35.10300827026367,
      "activations/layer15_attention_weight_max": 36.7506217956543,
      "activations/layer15_attention_weight_min": -32.69444274902344,
      "activations/layer16_attention_weight_max": 31.268692016601562,
      "activations/layer16_attention_weight_min": -29.482236862182617,
      "activations/layer17_attention_weight_max": 31.092370986938477,
      "activations/layer17_attention_weight_min": -25.844951629638672,
      "activations/layer18_attention_weight_max": 31.7724609375,
      "activations/layer18_attention_weight_min": -24.249595642089844,
      "activations/layer19_attention_weight_max": 35.34613800048828,
      "activations/layer19_attention_weight_min": -30.60987663269043,
      "activations/layer1_attention_weight_max": 16.499095916748047,
      "activations/layer1_attention_weight_min": -15.195661544799805,
      "activations/layer20_attention_weight_max": 31.88633918762207,
      "activations/layer20_attention_weight_min": -25.53493881225586,
      "activations/layer21_attention_weight_max": 32.187740325927734,
      "activations/layer21_attention_weight_min": -26.46048355102539,
      "activations/layer22_attention_weight_max": 47.37985610961914,
      "activations/layer22_attention_weight_min": -32.50297164916992,
      "activations/layer23_attention_weight_max": 35.778724670410156,
      "activations/layer23_attention_weight_min": -26.23680877685547,
      "activations/layer2_attention_weight_max": 36.729984283447266,
      "activations/layer2_attention_weight_min": -34.47120666503906,
      "activations/layer3_attention_weight_max": 102.66539764404297,
      "activations/layer3_attention_weight_min": -103.5869369506836,
      "activations/layer4_attention_weight_max": 113.7481918334961,
      "activations/layer4_attention_weight_min": -115.53043365478516,
      "activations/layer5_attention_weight_max": 51.402679443359375,
      "activations/layer5_attention_weight_min": -66.693359375,
      "activations/layer6_attention_weight_max": 44.58440017700195,
      "activations/layer6_attention_weight_min": -46.31629943847656,
      "activations/layer7_attention_weight_max": 90.23784637451172,
      "activations/layer7_attention_weight_min": -93.72212982177734,
      "activations/layer8_attention_weight_max": 43.611175537109375,
      "activations/layer8_attention_weight_min": -41.59758377075195,
      "activations/layer9_attention_weight_max": 30.67632484436035,
      "activations/layer9_attention_weight_min": -32.905731201171875,
      "epoch": 13.67,
      "learning_rate": 6.246022727272727e-05,
      "loss": 2.7756,
      "step": 235300
    },
    {
      "activations/layer0_attention_weight_max": 16.511877059936523,
      "activations/layer0_attention_weight_min": -13.303850173950195,
      "activations/layer10_attention_weight_max": 32.737789154052734,
      "activations/layer10_attention_weight_min": -33.950050354003906,
      "activations/layer11_attention_weight_max": 34.047340393066406,
      "activations/layer11_attention_weight_min": -31.89980697631836,
      "activations/layer12_attention_weight_max": 25.913597106933594,
      "activations/layer12_attention_weight_min": -24.071243286132812,
      "activations/layer13_attention_weight_max": 36.367156982421875,
      "activations/layer13_attention_weight_min": -34.359397888183594,
      "activations/layer14_attention_weight_max": 44.254642486572266,
      "activations/layer14_attention_weight_min": -38.52049255371094,
      "activations/layer15_attention_weight_max": 34.219970703125,
      "activations/layer15_attention_weight_min": -33.45516586303711,
      "activations/layer16_attention_weight_max": 30.494579315185547,
      "activations/layer16_attention_weight_min": -28.17568588256836,
      "activations/layer17_attention_weight_max": 28.641679763793945,
      "activations/layer17_attention_weight_min": -25.57895851135254,
      "activations/layer18_attention_weight_max": 29.697282791137695,
      "activations/layer18_attention_weight_min": -25.24041175842285,
      "activations/layer19_attention_weight_max": 36.2120475769043,
      "activations/layer19_attention_weight_min": -31.006925582885742,
      "activations/layer1_attention_weight_max": 16.846176147460938,
      "activations/layer1_attention_weight_min": -14.223590850830078,
      "activations/layer20_attention_weight_max": 31.203460693359375,
      "activations/layer20_attention_weight_min": -24.841012954711914,
      "activations/layer21_attention_weight_max": 29.571311950683594,
      "activations/layer21_attention_weight_min": -23.69588279724121,
      "activations/layer22_attention_weight_max": 40.4717903137207,
      "activations/layer22_attention_weight_min": -29.049320220947266,
      "activations/layer23_attention_weight_max": 38.72306823730469,
      "activations/layer23_attention_weight_min": -26.220787048339844,
      "activations/layer2_attention_weight_max": 34.3914794921875,
      "activations/layer2_attention_weight_min": -34.62151336669922,
      "activations/layer3_attention_weight_max": 98.36567687988281,
      "activations/layer3_attention_weight_min": -104.56961059570312,
      "activations/layer4_attention_weight_max": 106.33489990234375,
      "activations/layer4_attention_weight_min": -109.27046203613281,
      "activations/layer5_attention_weight_max": 50.96510314941406,
      "activations/layer5_attention_weight_min": -62.22056198120117,
      "activations/layer6_attention_weight_max": 43.44072723388672,
      "activations/layer6_attention_weight_min": -47.685245513916016,
      "activations/layer7_attention_weight_max": 88.24960327148438,
      "activations/layer7_attention_weight_min": -92.3683090209961,
      "activations/layer8_attention_weight_max": 37.8227653503418,
      "activations/layer8_attention_weight_min": -41.85567092895508,
      "activations/layer9_attention_weight_max": 31.170480728149414,
      "activations/layer9_attention_weight_min": -33.35346221923828,
      "epoch": 13.68,
      "learning_rate": 6.244128787878787e-05,
      "loss": 2.7614,
      "step": 235350
    },
    {
      "activations/layer0_attention_weight_max": 16.943687438964844,
      "activations/layer0_attention_weight_min": -13.093546867370605,
      "activations/layer10_attention_weight_max": 31.745573043823242,
      "activations/layer10_attention_weight_min": -32.750465393066406,
      "activations/layer11_attention_weight_max": 31.133960723876953,
      "activations/layer11_attention_weight_min": -30.802688598632812,
      "activations/layer12_attention_weight_max": 25.459794998168945,
      "activations/layer12_attention_weight_min": -25.432939529418945,
      "activations/layer13_attention_weight_max": 39.230010986328125,
      "activations/layer13_attention_weight_min": -34.59343719482422,
      "activations/layer14_attention_weight_max": 41.55705642700195,
      "activations/layer14_attention_weight_min": -37.04719161987305,
      "activations/layer15_attention_weight_max": 36.95873260498047,
      "activations/layer15_attention_weight_min": -34.886295318603516,
      "activations/layer16_attention_weight_max": 29.131946563720703,
      "activations/layer16_attention_weight_min": -27.320749282836914,
      "activations/layer17_attention_weight_max": 29.070444107055664,
      "activations/layer17_attention_weight_min": -25.195608139038086,
      "activations/layer18_attention_weight_max": 31.321809768676758,
      "activations/layer18_attention_weight_min": -24.05917739868164,
      "activations/layer19_attention_weight_max": 32.1250114440918,
      "activations/layer19_attention_weight_min": -31.421186447143555,
      "activations/layer1_attention_weight_max": 16.04853057861328,
      "activations/layer1_attention_weight_min": -15.507959365844727,
      "activations/layer20_attention_weight_max": 29.61481285095215,
      "activations/layer20_attention_weight_min": -25.743995666503906,
      "activations/layer21_attention_weight_max": 28.181760787963867,
      "activations/layer21_attention_weight_min": -24.239444732666016,
      "activations/layer22_attention_weight_max": 41.325435638427734,
      "activations/layer22_attention_weight_min": -32.27618408203125,
      "activations/layer23_attention_weight_max": 31.782997131347656,
      "activations/layer23_attention_weight_min": -26.087520599365234,
      "activations/layer2_attention_weight_max": 33.28172302246094,
      "activations/layer2_attention_weight_min": -33.68877029418945,
      "activations/layer3_attention_weight_max": 100.01068115234375,
      "activations/layer3_attention_weight_min": -100.34645080566406,
      "activations/layer4_attention_weight_max": 108.39904022216797,
      "activations/layer4_attention_weight_min": -109.3293685913086,
      "activations/layer5_attention_weight_max": 50.72624206542969,
      "activations/layer5_attention_weight_min": -60.69816589355469,
      "activations/layer6_attention_weight_max": 43.69020080566406,
      "activations/layer6_attention_weight_min": -45.74079895019531,
      "activations/layer7_attention_weight_max": 86.62110137939453,
      "activations/layer7_attention_weight_min": -87.68965148925781,
      "activations/layer8_attention_weight_max": 38.94744110107422,
      "activations/layer8_attention_weight_min": -42.75114822387695,
      "activations/layer9_attention_weight_max": 31.40850830078125,
      "activations/layer9_attention_weight_min": -35.45541000366211,
      "epoch": 13.68,
      "learning_rate": 6.242234848484847e-05,
      "loss": 2.7664,
      "step": 235400
    },
    {
      "activations/layer0_attention_weight_max": 16.619647979736328,
      "activations/layer0_attention_weight_min": -12.079813003540039,
      "activations/layer10_attention_weight_max": 32.23215103149414,
      "activations/layer10_attention_weight_min": -32.260650634765625,
      "activations/layer11_attention_weight_max": 32.121421813964844,
      "activations/layer11_attention_weight_min": -30.78377342224121,
      "activations/layer12_attention_weight_max": 26.838668823242188,
      "activations/layer12_attention_weight_min": -25.634946823120117,
      "activations/layer13_attention_weight_max": 39.77741622924805,
      "activations/layer13_attention_weight_min": -35.805301666259766,
      "activations/layer14_attention_weight_max": 43.193626403808594,
      "activations/layer14_attention_weight_min": -37.40944290161133,
      "activations/layer15_attention_weight_max": 37.8879508972168,
      "activations/layer15_attention_weight_min": -33.20940399169922,
      "activations/layer16_attention_weight_max": 29.601591110229492,
      "activations/layer16_attention_weight_min": -26.579254150390625,
      "activations/layer17_attention_weight_max": 30.8309268951416,
      "activations/layer17_attention_weight_min": -25.833324432373047,
      "activations/layer18_attention_weight_max": 31.18807601928711,
      "activations/layer18_attention_weight_min": -23.151473999023438,
      "activations/layer19_attention_weight_max": 33.55174255371094,
      "activations/layer19_attention_weight_min": -29.475141525268555,
      "activations/layer1_attention_weight_max": 16.49980354309082,
      "activations/layer1_attention_weight_min": -14.547966957092285,
      "activations/layer20_attention_weight_max": 28.71483039855957,
      "activations/layer20_attention_weight_min": -23.990215301513672,
      "activations/layer21_attention_weight_max": 28.05238914489746,
      "activations/layer21_attention_weight_min": -22.822673797607422,
      "activations/layer22_attention_weight_max": 46.23622131347656,
      "activations/layer22_attention_weight_min": -29.85286521911621,
      "activations/layer23_attention_weight_max": 34.744422912597656,
      "activations/layer23_attention_weight_min": -24.034934997558594,
      "activations/layer2_attention_weight_max": 34.96186065673828,
      "activations/layer2_attention_weight_min": -33.26601028442383,
      "activations/layer3_attention_weight_max": 97.52679443359375,
      "activations/layer3_attention_weight_min": -101.68223571777344,
      "activations/layer4_attention_weight_max": 110.2331771850586,
      "activations/layer4_attention_weight_min": -113.74163818359375,
      "activations/layer5_attention_weight_max": 52.84495162963867,
      "activations/layer5_attention_weight_min": -61.90293884277344,
      "activations/layer6_attention_weight_max": 43.72896194458008,
      "activations/layer6_attention_weight_min": -48.99251174926758,
      "activations/layer7_attention_weight_max": 89.57486724853516,
      "activations/layer7_attention_weight_min": -94.41702270507812,
      "activations/layer8_attention_weight_max": 37.99223709106445,
      "activations/layer8_attention_weight_min": -43.263702392578125,
      "activations/layer9_attention_weight_max": 31.10744285583496,
      "activations/layer9_attention_weight_min": -33.02970886230469,
      "epoch": 13.68,
      "learning_rate": 6.240340909090909e-05,
      "loss": 2.7585,
      "step": 235450
    },
    {
      "activations/layer0_attention_weight_max": 15.3797607421875,
      "activations/layer0_attention_weight_min": -12.610261917114258,
      "activations/layer10_attention_weight_max": 34.76665496826172,
      "activations/layer10_attention_weight_min": -34.407047271728516,
      "activations/layer11_attention_weight_max": 34.75728225708008,
      "activations/layer11_attention_weight_min": -32.66853332519531,
      "activations/layer12_attention_weight_max": 25.22433853149414,
      "activations/layer12_attention_weight_min": -24.17841339111328,
      "activations/layer13_attention_weight_max": 39.130332946777344,
      "activations/layer13_attention_weight_min": -35.84219741821289,
      "activations/layer14_attention_weight_max": 40.654380798339844,
      "activations/layer14_attention_weight_min": -36.87019348144531,
      "activations/layer15_attention_weight_max": 36.819923400878906,
      "activations/layer15_attention_weight_min": -34.508766174316406,
      "activations/layer16_attention_weight_max": 30.511837005615234,
      "activations/layer16_attention_weight_min": -28.370098114013672,
      "activations/layer17_attention_weight_max": 31.609453201293945,
      "activations/layer17_attention_weight_min": -25.816450119018555,
      "activations/layer18_attention_weight_max": 29.066926956176758,
      "activations/layer18_attention_weight_min": -24.64417839050293,
      "activations/layer19_attention_weight_max": 31.38336753845215,
      "activations/layer19_attention_weight_min": -31.118133544921875,
      "activations/layer1_attention_weight_max": 16.753421783447266,
      "activations/layer1_attention_weight_min": -16.890748977661133,
      "activations/layer20_attention_weight_max": 29.74403953552246,
      "activations/layer20_attention_weight_min": -25.74445152282715,
      "activations/layer21_attention_weight_max": 29.987571716308594,
      "activations/layer21_attention_weight_min": -24.219375610351562,
      "activations/layer22_attention_weight_max": 43.2397346496582,
      "activations/layer22_attention_weight_min": -30.282712936401367,
      "activations/layer23_attention_weight_max": 31.11886978149414,
      "activations/layer23_attention_weight_min": -26.04873275756836,
      "activations/layer2_attention_weight_max": 33.43240737915039,
      "activations/layer2_attention_weight_min": -33.16713333129883,
      "activations/layer3_attention_weight_max": 98.41797637939453,
      "activations/layer3_attention_weight_min": -97.41980743408203,
      "activations/layer4_attention_weight_max": 107.73723602294922,
      "activations/layer4_attention_weight_min": -107.32928466796875,
      "activations/layer5_attention_weight_max": 51.03716278076172,
      "activations/layer5_attention_weight_min": -62.1283073425293,
      "activations/layer6_attention_weight_max": 43.691219329833984,
      "activations/layer6_attention_weight_min": -44.398563385009766,
      "activations/layer7_attention_weight_max": 91.20587921142578,
      "activations/layer7_attention_weight_min": -90.5144271850586,
      "activations/layer8_attention_weight_max": 40.69575881958008,
      "activations/layer8_attention_weight_min": -41.87495422363281,
      "activations/layer9_attention_weight_max": 32.79827117919922,
      "activations/layer9_attention_weight_min": -32.816165924072266,
      "epoch": 13.68,
      "learning_rate": 6.238446969696969e-05,
      "loss": 2.765,
      "step": 235500
    },
    {
      "activations/layer0_attention_weight_max": 16.04677391052246,
      "activations/layer0_attention_weight_min": -12.335689544677734,
      "activations/layer10_attention_weight_max": 35.63526916503906,
      "activations/layer10_attention_weight_min": -33.77885055541992,
      "activations/layer11_attention_weight_max": 36.37467575073242,
      "activations/layer11_attention_weight_min": -34.83697509765625,
      "activations/layer12_attention_weight_max": 31.760316848754883,
      "activations/layer12_attention_weight_min": -26.060888290405273,
      "activations/layer13_attention_weight_max": 42.252769470214844,
      "activations/layer13_attention_weight_min": -33.821815490722656,
      "activations/layer14_attention_weight_max": 47.03907012939453,
      "activations/layer14_attention_weight_min": -39.73633575439453,
      "activations/layer15_attention_weight_max": 41.54000473022461,
      "activations/layer15_attention_weight_min": -35.122337341308594,
      "activations/layer16_attention_weight_max": 35.675411224365234,
      "activations/layer16_attention_weight_min": -27.406843185424805,
      "activations/layer17_attention_weight_max": 35.809837341308594,
      "activations/layer17_attention_weight_min": -26.693918228149414,
      "activations/layer18_attention_weight_max": 39.621517181396484,
      "activations/layer18_attention_weight_min": -24.02252769470215,
      "activations/layer19_attention_weight_max": 42.52458953857422,
      "activations/layer19_attention_weight_min": -30.620468139648438,
      "activations/layer1_attention_weight_max": 16.30050277709961,
      "activations/layer1_attention_weight_min": -15.416330337524414,
      "activations/layer20_attention_weight_max": 34.29497528076172,
      "activations/layer20_attention_weight_min": -24.00719451904297,
      "activations/layer21_attention_weight_max": 32.022705078125,
      "activations/layer21_attention_weight_min": -23.98468589782715,
      "activations/layer22_attention_weight_max": 52.30955505371094,
      "activations/layer22_attention_weight_min": -31.018653869628906,
      "activations/layer23_attention_weight_max": 39.295921325683594,
      "activations/layer23_attention_weight_min": -24.308982849121094,
      "activations/layer2_attention_weight_max": 33.12656784057617,
      "activations/layer2_attention_weight_min": -32.48766326904297,
      "activations/layer3_attention_weight_max": 98.41365814208984,
      "activations/layer3_attention_weight_min": -100.5558853149414,
      "activations/layer4_attention_weight_max": 111.28810119628906,
      "activations/layer4_attention_weight_min": -116.27345275878906,
      "activations/layer5_attention_weight_max": 53.319801330566406,
      "activations/layer5_attention_weight_min": -61.70028305053711,
      "activations/layer6_attention_weight_max": 42.5791015625,
      "activations/layer6_attention_weight_min": -45.76973342895508,
      "activations/layer7_attention_weight_max": 89.0897216796875,
      "activations/layer7_attention_weight_min": -91.09058380126953,
      "activations/layer8_attention_weight_max": 43.223182678222656,
      "activations/layer8_attention_weight_min": -43.14398956298828,
      "activations/layer9_attention_weight_max": 36.51567077636719,
      "activations/layer9_attention_weight_min": -34.153289794921875,
      "epoch": 13.69,
      "learning_rate": 6.23655303030303e-05,
      "loss": 2.7724,
      "step": 235550
    },
    {
      "activations/layer0_attention_weight_max": 15.396172523498535,
      "activations/layer0_attention_weight_min": -12.944510459899902,
      "activations/layer10_attention_weight_max": 32.30884552001953,
      "activations/layer10_attention_weight_min": -32.03997802734375,
      "activations/layer11_attention_weight_max": 33.22956848144531,
      "activations/layer11_attention_weight_min": -32.452972412109375,
      "activations/layer12_attention_weight_max": 26.585294723510742,
      "activations/layer12_attention_weight_min": -25.777652740478516,
      "activations/layer13_attention_weight_max": 39.0927848815918,
      "activations/layer13_attention_weight_min": -34.26024627685547,
      "activations/layer14_attention_weight_max": 43.60831069946289,
      "activations/layer14_attention_weight_min": -37.120269775390625,
      "activations/layer15_attention_weight_max": 38.012107849121094,
      "activations/layer15_attention_weight_min": -36.94942092895508,
      "activations/layer16_attention_weight_max": 29.982133865356445,
      "activations/layer16_attention_weight_min": -28.900585174560547,
      "activations/layer17_attention_weight_max": 32.412837982177734,
      "activations/layer17_attention_weight_min": -27.078197479248047,
      "activations/layer18_attention_weight_max": 34.143306732177734,
      "activations/layer18_attention_weight_min": -24.772371292114258,
      "activations/layer19_attention_weight_max": 34.691322326660156,
      "activations/layer19_attention_weight_min": -30.900083541870117,
      "activations/layer1_attention_weight_max": 15.929047584533691,
      "activations/layer1_attention_weight_min": -14.729582786560059,
      "activations/layer20_attention_weight_max": 27.50365447998047,
      "activations/layer20_attention_weight_min": -25.645057678222656,
      "activations/layer21_attention_weight_max": 27.14981460571289,
      "activations/layer21_attention_weight_min": -24.685869216918945,
      "activations/layer22_attention_weight_max": 43.18708419799805,
      "activations/layer22_attention_weight_min": -31.831329345703125,
      "activations/layer23_attention_weight_max": 33.122982025146484,
      "activations/layer23_attention_weight_min": -27.990142822265625,
      "activations/layer2_attention_weight_max": 35.00098419189453,
      "activations/layer2_attention_weight_min": -33.237342834472656,
      "activations/layer3_attention_weight_max": 97.2939224243164,
      "activations/layer3_attention_weight_min": -99.32958221435547,
      "activations/layer4_attention_weight_max": 108.47281646728516,
      "activations/layer4_attention_weight_min": -106.64208221435547,
      "activations/layer5_attention_weight_max": 54.579612731933594,
      "activations/layer5_attention_weight_min": -60.827354431152344,
      "activations/layer6_attention_weight_max": 42.90475082397461,
      "activations/layer6_attention_weight_min": -45.2353401184082,
      "activations/layer7_attention_weight_max": 90.60236358642578,
      "activations/layer7_attention_weight_min": -94.46373748779297,
      "activations/layer8_attention_weight_max": 39.87646484375,
      "activations/layer8_attention_weight_min": -41.26140213012695,
      "activations/layer9_attention_weight_max": 32.75918197631836,
      "activations/layer9_attention_weight_min": -33.56179428100586,
      "epoch": 13.69,
      "learning_rate": 6.234659090909091e-05,
      "loss": 2.759,
      "step": 235600
    },
    {
      "activations/layer0_attention_weight_max": 16.060426712036133,
      "activations/layer0_attention_weight_min": -12.17762565612793,
      "activations/layer10_attention_weight_max": 32.897911071777344,
      "activations/layer10_attention_weight_min": -34.68658447265625,
      "activations/layer11_attention_weight_max": 32.65860366821289,
      "activations/layer11_attention_weight_min": -34.21091079711914,
      "activations/layer12_attention_weight_max": 25.894737243652344,
      "activations/layer12_attention_weight_min": -26.749671936035156,
      "activations/layer13_attention_weight_max": 40.506072998046875,
      "activations/layer13_attention_weight_min": -36.80152130126953,
      "activations/layer14_attention_weight_max": 42.9152946472168,
      "activations/layer14_attention_weight_min": -38.52185821533203,
      "activations/layer15_attention_weight_max": 36.440391540527344,
      "activations/layer15_attention_weight_min": -34.91401290893555,
      "activations/layer16_attention_weight_max": 30.02251625061035,
      "activations/layer16_attention_weight_min": -29.455188751220703,
      "activations/layer17_attention_weight_max": 31.63212013244629,
      "activations/layer17_attention_weight_min": -25.919458389282227,
      "activations/layer18_attention_weight_max": 31.744770050048828,
      "activations/layer18_attention_weight_min": -24.008712768554688,
      "activations/layer19_attention_weight_max": 39.004146575927734,
      "activations/layer19_attention_weight_min": -32.96965789794922,
      "activations/layer1_attention_weight_max": 17.060523986816406,
      "activations/layer1_attention_weight_min": -15.20039176940918,
      "activations/layer20_attention_weight_max": 33.60248947143555,
      "activations/layer20_attention_weight_min": -27.7416934967041,
      "activations/layer21_attention_weight_max": 29.578426361083984,
      "activations/layer21_attention_weight_min": -24.83400535583496,
      "activations/layer22_attention_weight_max": 51.06351852416992,
      "activations/layer22_attention_weight_min": -31.10685157775879,
      "activations/layer23_attention_weight_max": 34.738807678222656,
      "activations/layer23_attention_weight_min": -25.022151947021484,
      "activations/layer2_attention_weight_max": 34.057228088378906,
      "activations/layer2_attention_weight_min": -35.10491943359375,
      "activations/layer3_attention_weight_max": 101.48677825927734,
      "activations/layer3_attention_weight_min": -102.0001220703125,
      "activations/layer4_attention_weight_max": 111.24852752685547,
      "activations/layer4_attention_weight_min": -112.31856536865234,
      "activations/layer5_attention_weight_max": 53.950706481933594,
      "activations/layer5_attention_weight_min": -62.76586151123047,
      "activations/layer6_attention_weight_max": 45.121986389160156,
      "activations/layer6_attention_weight_min": -47.43333053588867,
      "activations/layer7_attention_weight_max": 89.37162017822266,
      "activations/layer7_attention_weight_min": -92.3506088256836,
      "activations/layer8_attention_weight_max": 40.730438232421875,
      "activations/layer8_attention_weight_min": -44.85334014892578,
      "activations/layer9_attention_weight_max": 34.710994720458984,
      "activations/layer9_attention_weight_min": -35.03558349609375,
      "epoch": 13.69,
      "learning_rate": 6.232765151515151e-05,
      "loss": 2.7677,
      "step": 235650
    },
    {
      "activations/layer0_attention_weight_max": 16.75349235534668,
      "activations/layer0_attention_weight_min": -11.984217643737793,
      "activations/layer10_attention_weight_max": 38.5994873046875,
      "activations/layer10_attention_weight_min": -38.726470947265625,
      "activations/layer11_attention_weight_max": 40.41546630859375,
      "activations/layer11_attention_weight_min": -40.774818420410156,
      "activations/layer12_attention_weight_max": 27.220666885375977,
      "activations/layer12_attention_weight_min": -28.367441177368164,
      "activations/layer13_attention_weight_max": 38.76189422607422,
      "activations/layer13_attention_weight_min": -35.70543670654297,
      "activations/layer14_attention_weight_max": 41.83467102050781,
      "activations/layer14_attention_weight_min": -38.73186111450195,
      "activations/layer15_attention_weight_max": 36.34901428222656,
      "activations/layer15_attention_weight_min": -34.816524505615234,
      "activations/layer16_attention_weight_max": 31.586458206176758,
      "activations/layer16_attention_weight_min": -29.506372451782227,
      "activations/layer17_attention_weight_max": 32.72029113769531,
      "activations/layer17_attention_weight_min": -25.783363342285156,
      "activations/layer18_attention_weight_max": 29.44133186340332,
      "activations/layer18_attention_weight_min": -23.0443058013916,
      "activations/layer19_attention_weight_max": 34.00014114379883,
      "activations/layer19_attention_weight_min": -31.730905532836914,
      "activations/layer1_attention_weight_max": 16.62200355529785,
      "activations/layer1_attention_weight_min": -14.998459815979004,
      "activations/layer20_attention_weight_max": 29.332836151123047,
      "activations/layer20_attention_weight_min": -25.089588165283203,
      "activations/layer21_attention_weight_max": 26.760940551757812,
      "activations/layer21_attention_weight_min": -22.811809539794922,
      "activations/layer22_attention_weight_max": 42.976806640625,
      "activations/layer22_attention_weight_min": -28.95052719116211,
      "activations/layer23_attention_weight_max": 31.143226623535156,
      "activations/layer23_attention_weight_min": -22.293907165527344,
      "activations/layer2_attention_weight_max": 34.03414535522461,
      "activations/layer2_attention_weight_min": -35.714019775390625,
      "activations/layer3_attention_weight_max": 101.5999526977539,
      "activations/layer3_attention_weight_min": -101.07319641113281,
      "activations/layer4_attention_weight_max": 113.0023422241211,
      "activations/layer4_attention_weight_min": -111.96319580078125,
      "activations/layer5_attention_weight_max": 59.52085876464844,
      "activations/layer5_attention_weight_min": -63.43976593017578,
      "activations/layer6_attention_weight_max": 45.78203201293945,
      "activations/layer6_attention_weight_min": -46.66303253173828,
      "activations/layer7_attention_weight_max": 106.87451934814453,
      "activations/layer7_attention_weight_min": -93.5125961303711,
      "activations/layer8_attention_weight_max": 49.12248992919922,
      "activations/layer8_attention_weight_min": -45.03098678588867,
      "activations/layer9_attention_weight_max": 34.43314743041992,
      "activations/layer9_attention_weight_min": -35.76056671142578,
      "epoch": 13.7,
      "learning_rate": 6.230871212121211e-05,
      "loss": 2.77,
      "step": 235700
    },
    {
      "activations/layer0_attention_weight_max": 16.242258071899414,
      "activations/layer0_attention_weight_min": -12.659289360046387,
      "activations/layer10_attention_weight_max": 30.224227905273438,
      "activations/layer10_attention_weight_min": -32.96903610229492,
      "activations/layer11_attention_weight_max": 32.532283782958984,
      "activations/layer11_attention_weight_min": -31.931867599487305,
      "activations/layer12_attention_weight_max": 24.681312561035156,
      "activations/layer12_attention_weight_min": -25.545434951782227,
      "activations/layer13_attention_weight_max": 36.65801239013672,
      "activations/layer13_attention_weight_min": -33.702171325683594,
      "activations/layer14_attention_weight_max": 38.660003662109375,
      "activations/layer14_attention_weight_min": -36.97459030151367,
      "activations/layer15_attention_weight_max": 32.971656799316406,
      "activations/layer15_attention_weight_min": -32.111045837402344,
      "activations/layer16_attention_weight_max": 26.834474563598633,
      "activations/layer16_attention_weight_min": -28.71603775024414,
      "activations/layer17_attention_weight_max": 29.010009765625,
      "activations/layer17_attention_weight_min": -25.398008346557617,
      "activations/layer18_attention_weight_max": 28.828697204589844,
      "activations/layer18_attention_weight_min": -22.505168914794922,
      "activations/layer19_attention_weight_max": 33.39023971557617,
      "activations/layer19_attention_weight_min": -31.348804473876953,
      "activations/layer1_attention_weight_max": 17.17885398864746,
      "activations/layer1_attention_weight_min": -14.493330955505371,
      "activations/layer20_attention_weight_max": 27.10726547241211,
      "activations/layer20_attention_weight_min": -24.64256477355957,
      "activations/layer21_attention_weight_max": 27.353721618652344,
      "activations/layer21_attention_weight_min": -25.413291931152344,
      "activations/layer22_attention_weight_max": 39.62712478637695,
      "activations/layer22_attention_weight_min": -28.11125946044922,
      "activations/layer23_attention_weight_max": 28.813003540039062,
      "activations/layer23_attention_weight_min": -24.937063217163086,
      "activations/layer2_attention_weight_max": 32.40159606933594,
      "activations/layer2_attention_weight_min": -34.471309661865234,
      "activations/layer3_attention_weight_max": 99.48059844970703,
      "activations/layer3_attention_weight_min": -102.19673919677734,
      "activations/layer4_attention_weight_max": 109.03594207763672,
      "activations/layer4_attention_weight_min": -107.13907623291016,
      "activations/layer5_attention_weight_max": 50.12257385253906,
      "activations/layer5_attention_weight_min": -64.02325439453125,
      "activations/layer6_attention_weight_max": 45.067298889160156,
      "activations/layer6_attention_weight_min": -47.06594467163086,
      "activations/layer7_attention_weight_max": 86.78114318847656,
      "activations/layer7_attention_weight_min": -89.4398422241211,
      "activations/layer8_attention_weight_max": 40.219688415527344,
      "activations/layer8_attention_weight_min": -39.316383361816406,
      "activations/layer9_attention_weight_max": 31.713293075561523,
      "activations/layer9_attention_weight_min": -31.521711349487305,
      "epoch": 13.7,
      "learning_rate": 6.228977272727273e-05,
      "loss": 2.758,
      "step": 235750
    },
    {
      "activations/layer0_attention_weight_max": 16.14017105102539,
      "activations/layer0_attention_weight_min": -12.943270683288574,
      "activations/layer10_attention_weight_max": 33.310367584228516,
      "activations/layer10_attention_weight_min": -32.759429931640625,
      "activations/layer11_attention_weight_max": 32.18891143798828,
      "activations/layer11_attention_weight_min": -32.44035720825195,
      "activations/layer12_attention_weight_max": 24.843076705932617,
      "activations/layer12_attention_weight_min": -24.31680679321289,
      "activations/layer13_attention_weight_max": 37.713497161865234,
      "activations/layer13_attention_weight_min": -33.547332763671875,
      "activations/layer14_attention_weight_max": 45.980411529541016,
      "activations/layer14_attention_weight_min": -35.85073471069336,
      "activations/layer15_attention_weight_max": 35.314239501953125,
      "activations/layer15_attention_weight_min": -32.923587799072266,
      "activations/layer16_attention_weight_max": 32.55116653442383,
      "activations/layer16_attention_weight_min": -28.008352279663086,
      "activations/layer17_attention_weight_max": 31.689287185668945,
      "activations/layer17_attention_weight_min": -25.13644790649414,
      "activations/layer18_attention_weight_max": 33.41990661621094,
      "activations/layer18_attention_weight_min": -23.942432403564453,
      "activations/layer19_attention_weight_max": 36.945919036865234,
      "activations/layer19_attention_weight_min": -30.281797409057617,
      "activations/layer1_attention_weight_max": 16.137739181518555,
      "activations/layer1_attention_weight_min": -14.678156852722168,
      "activations/layer20_attention_weight_max": 30.81112289428711,
      "activations/layer20_attention_weight_min": -25.068761825561523,
      "activations/layer21_attention_weight_max": 30.05010223388672,
      "activations/layer21_attention_weight_min": -23.440994262695312,
      "activations/layer22_attention_weight_max": 41.51591873168945,
      "activations/layer22_attention_weight_min": -30.432716369628906,
      "activations/layer23_attention_weight_max": 37.8441276550293,
      "activations/layer23_attention_weight_min": -26.0275936126709,
      "activations/layer2_attention_weight_max": 33.865135192871094,
      "activations/layer2_attention_weight_min": -32.97087097167969,
      "activations/layer3_attention_weight_max": 101.72622680664062,
      "activations/layer3_attention_weight_min": -100.35738372802734,
      "activations/layer4_attention_weight_max": 114.77104949951172,
      "activations/layer4_attention_weight_min": -114.4195556640625,
      "activations/layer5_attention_weight_max": 53.51744842529297,
      "activations/layer5_attention_weight_min": -63.50812911987305,
      "activations/layer6_attention_weight_max": 43.4792366027832,
      "activations/layer6_attention_weight_min": -47.062679290771484,
      "activations/layer7_attention_weight_max": 96.83625793457031,
      "activations/layer7_attention_weight_min": -90.21442413330078,
      "activations/layer8_attention_weight_max": 41.055824279785156,
      "activations/layer8_attention_weight_min": -41.53321838378906,
      "activations/layer9_attention_weight_max": 33.546695709228516,
      "activations/layer9_attention_weight_min": -33.673458099365234,
      "epoch": 13.7,
      "learning_rate": 6.227083333333333e-05,
      "loss": 2.7635,
      "step": 235800
    },
    {
      "activations/layer0_attention_weight_max": 16.02299690246582,
      "activations/layer0_attention_weight_min": -12.598910331726074,
      "activations/layer10_attention_weight_max": 34.72715759277344,
      "activations/layer10_attention_weight_min": -31.765506744384766,
      "activations/layer11_attention_weight_max": 32.70364761352539,
      "activations/layer11_attention_weight_min": -31.513235092163086,
      "activations/layer12_attention_weight_max": 27.265953063964844,
      "activations/layer12_attention_weight_min": -25.240190505981445,
      "activations/layer13_attention_weight_max": 38.347347259521484,
      "activations/layer13_attention_weight_min": -35.04301452636719,
      "activations/layer14_attention_weight_max": 42.624908447265625,
      "activations/layer14_attention_weight_min": -37.35134506225586,
      "activations/layer15_attention_weight_max": 36.6961784362793,
      "activations/layer15_attention_weight_min": -32.58103561401367,
      "activations/layer16_attention_weight_max": 29.947446823120117,
      "activations/layer16_attention_weight_min": -28.02178382873535,
      "activations/layer17_attention_weight_max": 33.8365592956543,
      "activations/layer17_attention_weight_min": -25.458600997924805,
      "activations/layer18_attention_weight_max": 33.748382568359375,
      "activations/layer18_attention_weight_min": -23.572275161743164,
      "activations/layer19_attention_weight_max": 34.52465057373047,
      "activations/layer19_attention_weight_min": -29.851774215698242,
      "activations/layer1_attention_weight_max": 16.097822189331055,
      "activations/layer1_attention_weight_min": -14.016681671142578,
      "activations/layer20_attention_weight_max": 27.957569122314453,
      "activations/layer20_attention_weight_min": -23.35295295715332,
      "activations/layer21_attention_weight_max": 27.431169509887695,
      "activations/layer21_attention_weight_min": -24.099002838134766,
      "activations/layer22_attention_weight_max": 45.2450065612793,
      "activations/layer22_attention_weight_min": -29.86182403564453,
      "activations/layer23_attention_weight_max": 32.47418975830078,
      "activations/layer23_attention_weight_min": -25.56282615661621,
      "activations/layer2_attention_weight_max": 33.38997268676758,
      "activations/layer2_attention_weight_min": -34.991859436035156,
      "activations/layer3_attention_weight_max": 101.57455444335938,
      "activations/layer3_attention_weight_min": -99.12895202636719,
      "activations/layer4_attention_weight_max": 112.4041748046875,
      "activations/layer4_attention_weight_min": -115.45609283447266,
      "activations/layer5_attention_weight_max": 53.01612854003906,
      "activations/layer5_attention_weight_min": -63.104820251464844,
      "activations/layer6_attention_weight_max": 46.9927864074707,
      "activations/layer6_attention_weight_min": -46.51968002319336,
      "activations/layer7_attention_weight_max": 95.52873992919922,
      "activations/layer7_attention_weight_min": -93.0145492553711,
      "activations/layer8_attention_weight_max": 42.8118782043457,
      "activations/layer8_attention_weight_min": -40.90885925292969,
      "activations/layer9_attention_weight_max": 33.34644317626953,
      "activations/layer9_attention_weight_min": -33.558101654052734,
      "epoch": 13.7,
      "learning_rate": 6.225189393939393e-05,
      "loss": 2.7722,
      "step": 235850
    },
    {
      "activations/layer0_attention_weight_max": 16.173376083374023,
      "activations/layer0_attention_weight_min": -12.102335929870605,
      "activations/layer10_attention_weight_max": 32.56795120239258,
      "activations/layer10_attention_weight_min": -32.710487365722656,
      "activations/layer11_attention_weight_max": 32.54418182373047,
      "activations/layer11_attention_weight_min": -32.520835876464844,
      "activations/layer12_attention_weight_max": 25.71912384033203,
      "activations/layer12_attention_weight_min": -25.773962020874023,
      "activations/layer13_attention_weight_max": 40.20561981201172,
      "activations/layer13_attention_weight_min": -33.687591552734375,
      "activations/layer14_attention_weight_max": 43.083091735839844,
      "activations/layer14_attention_weight_min": -35.54892349243164,
      "activations/layer15_attention_weight_max": 37.3801155090332,
      "activations/layer15_attention_weight_min": -32.53697967529297,
      "activations/layer16_attention_weight_max": 30.470264434814453,
      "activations/layer16_attention_weight_min": -28.033039093017578,
      "activations/layer17_attention_weight_max": 32.966041564941406,
      "activations/layer17_attention_weight_min": -26.16364288330078,
      "activations/layer18_attention_weight_max": 31.997751235961914,
      "activations/layer18_attention_weight_min": -23.238908767700195,
      "activations/layer19_attention_weight_max": 36.89731979370117,
      "activations/layer19_attention_weight_min": -31.49434471130371,
      "activations/layer1_attention_weight_max": 16.624174118041992,
      "activations/layer1_attention_weight_min": -14.879064559936523,
      "activations/layer20_attention_weight_max": 30.450788497924805,
      "activations/layer20_attention_weight_min": -23.846311569213867,
      "activations/layer21_attention_weight_max": 29.35201072692871,
      "activations/layer21_attention_weight_min": -23.862211227416992,
      "activations/layer22_attention_weight_max": 44.0978889465332,
      "activations/layer22_attention_weight_min": -30.50893783569336,
      "activations/layer23_attention_weight_max": 39.9417610168457,
      "activations/layer23_attention_weight_min": -25.706188201904297,
      "activations/layer2_attention_weight_max": 33.059608459472656,
      "activations/layer2_attention_weight_min": -33.69232177734375,
      "activations/layer3_attention_weight_max": 97.10819244384766,
      "activations/layer3_attention_weight_min": -97.9942398071289,
      "activations/layer4_attention_weight_max": 109.07868957519531,
      "activations/layer4_attention_weight_min": -104.39730072021484,
      "activations/layer5_attention_weight_max": 50.58364486694336,
      "activations/layer5_attention_weight_min": -60.771209716796875,
      "activations/layer6_attention_weight_max": 42.55877685546875,
      "activations/layer6_attention_weight_min": -47.74040985107422,
      "activations/layer7_attention_weight_max": 90.4925308227539,
      "activations/layer7_attention_weight_min": -87.3625717163086,
      "activations/layer8_attention_weight_max": 37.98713302612305,
      "activations/layer8_attention_weight_min": -39.0949821472168,
      "activations/layer9_attention_weight_max": 31.839431762695312,
      "activations/layer9_attention_weight_min": -31.602752685546875,
      "epoch": 13.71,
      "learning_rate": 6.223295454545453e-05,
      "loss": 2.7615,
      "step": 235900
    },
    {
      "activations/layer0_attention_weight_max": 15.673561096191406,
      "activations/layer0_attention_weight_min": -12.750280380249023,
      "activations/layer10_attention_weight_max": 30.26766014099121,
      "activations/layer10_attention_weight_min": -31.14681625366211,
      "activations/layer11_attention_weight_max": 31.274585723876953,
      "activations/layer11_attention_weight_min": -29.940853118896484,
      "activations/layer12_attention_weight_max": 25.186973571777344,
      "activations/layer12_attention_weight_min": -24.707563400268555,
      "activations/layer13_attention_weight_max": 36.166107177734375,
      "activations/layer13_attention_weight_min": -32.66120910644531,
      "activations/layer14_attention_weight_max": 42.617576599121094,
      "activations/layer14_attention_weight_min": -34.97126007080078,
      "activations/layer15_attention_weight_max": 35.51670837402344,
      "activations/layer15_attention_weight_min": -31.66387939453125,
      "activations/layer16_attention_weight_max": 32.80721664428711,
      "activations/layer16_attention_weight_min": -29.406076431274414,
      "activations/layer17_attention_weight_max": 29.5876407623291,
      "activations/layer17_attention_weight_min": -24.753271102905273,
      "activations/layer18_attention_weight_max": 30.61968421936035,
      "activations/layer18_attention_weight_min": -24.73187255859375,
      "activations/layer19_attention_weight_max": 34.92698287963867,
      "activations/layer19_attention_weight_min": -30.42779541015625,
      "activations/layer1_attention_weight_max": 17.872087478637695,
      "activations/layer1_attention_weight_min": -16.690710067749023,
      "activations/layer20_attention_weight_max": 27.957372665405273,
      "activations/layer20_attention_weight_min": -23.790128707885742,
      "activations/layer21_attention_weight_max": 26.02797508239746,
      "activations/layer21_attention_weight_min": -23.96689796447754,
      "activations/layer22_attention_weight_max": 41.09672546386719,
      "activations/layer22_attention_weight_min": -28.55646324157715,
      "activations/layer23_attention_weight_max": 30.395687103271484,
      "activations/layer23_attention_weight_min": -23.594860076904297,
      "activations/layer2_attention_weight_max": 35.347965240478516,
      "activations/layer2_attention_weight_min": -33.69578552246094,
      "activations/layer3_attention_weight_max": 102.74124145507812,
      "activations/layer3_attention_weight_min": -102.14704895019531,
      "activations/layer4_attention_weight_max": 110.79117584228516,
      "activations/layer4_attention_weight_min": -112.03533172607422,
      "activations/layer5_attention_weight_max": 51.61703109741211,
      "activations/layer5_attention_weight_min": -61.55329132080078,
      "activations/layer6_attention_weight_max": 43.30727767944336,
      "activations/layer6_attention_weight_min": -47.18610763549805,
      "activations/layer7_attention_weight_max": 90.2101821899414,
      "activations/layer7_attention_weight_min": -91.94656372070312,
      "activations/layer8_attention_weight_max": 39.331729888916016,
      "activations/layer8_attention_weight_min": -41.090049743652344,
      "activations/layer9_attention_weight_max": 33.01276779174805,
      "activations/layer9_attention_weight_min": -31.87322425842285,
      "epoch": 13.71,
      "learning_rate": 6.221401515151514e-05,
      "loss": 2.7571,
      "step": 235950
    },
    {
      "activations/layer0_attention_weight_max": 17.479007720947266,
      "activations/layer0_attention_weight_min": -12.298090934753418,
      "activations/layer10_attention_weight_max": 35.0923957824707,
      "activations/layer10_attention_weight_min": -33.36025619506836,
      "activations/layer11_attention_weight_max": 35.66083908081055,
      "activations/layer11_attention_weight_min": -34.08660888671875,
      "activations/layer12_attention_weight_max": 29.459070205688477,
      "activations/layer12_attention_weight_min": -29.320924758911133,
      "activations/layer13_attention_weight_max": 42.44199752807617,
      "activations/layer13_attention_weight_min": -36.045143127441406,
      "activations/layer14_attention_weight_max": 45.186283111572266,
      "activations/layer14_attention_weight_min": -38.118263244628906,
      "activations/layer15_attention_weight_max": 40.202816009521484,
      "activations/layer15_attention_weight_min": -35.35681915283203,
      "activations/layer16_attention_weight_max": 33.87089157104492,
      "activations/layer16_attention_weight_min": -26.166942596435547,
      "activations/layer17_attention_weight_max": 32.55405807495117,
      "activations/layer17_attention_weight_min": -25.647354125976562,
      "activations/layer18_attention_weight_max": 33.74868392944336,
      "activations/layer18_attention_weight_min": -24.723703384399414,
      "activations/layer19_attention_weight_max": 40.212608337402344,
      "activations/layer19_attention_weight_min": -30.280954360961914,
      "activations/layer1_attention_weight_max": 17.112882614135742,
      "activations/layer1_attention_weight_min": -14.081411361694336,
      "activations/layer20_attention_weight_max": 33.542091369628906,
      "activations/layer20_attention_weight_min": -24.332860946655273,
      "activations/layer21_attention_weight_max": 29.21727180480957,
      "activations/layer21_attention_weight_min": -23.300439834594727,
      "activations/layer22_attention_weight_max": 47.874507904052734,
      "activations/layer22_attention_weight_min": -29.40973472595215,
      "activations/layer23_attention_weight_max": 39.69743347167969,
      "activations/layer23_attention_weight_min": -24.845355987548828,
      "activations/layer2_attention_weight_max": 36.06087875366211,
      "activations/layer2_attention_weight_min": -33.040870666503906,
      "activations/layer3_attention_weight_max": 99.8896713256836,
      "activations/layer3_attention_weight_min": -100.83238220214844,
      "activations/layer4_attention_weight_max": 110.62677001953125,
      "activations/layer4_attention_weight_min": -115.5010986328125,
      "activations/layer5_attention_weight_max": 51.25419616699219,
      "activations/layer5_attention_weight_min": -63.64406967163086,
      "activations/layer6_attention_weight_max": 43.459510803222656,
      "activations/layer6_attention_weight_min": -45.444435119628906,
      "activations/layer7_attention_weight_max": 94.62168884277344,
      "activations/layer7_attention_weight_min": -90.30528259277344,
      "activations/layer8_attention_weight_max": 40.87923049926758,
      "activations/layer8_attention_weight_min": -44.20952606201172,
      "activations/layer9_attention_weight_max": 36.246456146240234,
      "activations/layer9_attention_weight_min": -34.719417572021484,
      "epoch": 13.71,
      "learning_rate": 6.219507575757575e-05,
      "loss": 2.7662,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4866,
      "eval_samples_per_second": 505.971,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4866,
      "eval_openwebtext_samples_per_second": 505.971,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9443,
      "eval_wikitext_samples_per_second": 234.535,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_lambada_loss": 2.55859375,
      "eval_lambada_ppl": 12.917639106991658,
      "eval_lambada_runtime": 9.5249,
      "eval_lambada_samples_per_second": 511.185,
      "step": 236000
    },
    {
      "activations/layer0_attention_weight_max": 18.192949295043945,
      "activations/layer0_attention_weight_min": -12.398777961730957,
      "activations/layer10_attention_weight_max": 37.293724060058594,
      "activations/layer10_attention_weight_min": -34.51271438598633,
      "activations/layer11_attention_weight_max": 35.25422668457031,
      "activations/layer11_attention_weight_min": -35.47525405883789,
      "activations/layer12_attention_weight_max": 32.7213134765625,
      "activations/layer12_attention_weight_min": -27.180225372314453,
      "activations/layer13_attention_weight_max": 43.239158630371094,
      "activations/layer13_attention_weight_min": -38.8857536315918,
      "activations/layer14_attention_weight_max": 48.199520111083984,
      "activations/layer14_attention_weight_min": -38.71500015258789,
      "activations/layer15_attention_weight_max": 41.191184997558594,
      "activations/layer15_attention_weight_min": -35.06364440917969,
      "activations/layer16_attention_weight_max": 33.844932556152344,
      "activations/layer16_attention_weight_min": -28.105573654174805,
      "activations/layer17_attention_weight_max": 32.10966491699219,
      "activations/layer17_attention_weight_min": -27.577856063842773,
      "activations/layer18_attention_weight_max": 34.158851623535156,
      "activations/layer18_attention_weight_min": -25.05264663696289,
      "activations/layer19_attention_weight_max": 41.793331146240234,
      "activations/layer19_attention_weight_min": -30.25752830505371,
      "activations/layer1_attention_weight_max": 16.23969841003418,
      "activations/layer1_attention_weight_min": -15.122550010681152,
      "activations/layer20_attention_weight_max": 34.055057525634766,
      "activations/layer20_attention_weight_min": -23.870140075683594,
      "activations/layer21_attention_weight_max": 30.875112533569336,
      "activations/layer21_attention_weight_min": -23.51157569885254,
      "activations/layer22_attention_weight_max": 49.993255615234375,
      "activations/layer22_attention_weight_min": -30.79445457458496,
      "activations/layer23_attention_weight_max": 37.98160171508789,
      "activations/layer23_attention_weight_min": -25.438655853271484,
      "activations/layer2_attention_weight_max": 33.796424865722656,
      "activations/layer2_attention_weight_min": -34.12401580810547,
      "activations/layer3_attention_weight_max": 101.33599853515625,
      "activations/layer3_attention_weight_min": -98.75833129882812,
      "activations/layer4_attention_weight_max": 113.8387451171875,
      "activations/layer4_attention_weight_min": -113.33702850341797,
      "activations/layer5_attention_weight_max": 58.608680725097656,
      "activations/layer5_attention_weight_min": -62.92689514160156,
      "activations/layer6_attention_weight_max": 46.70509719848633,
      "activations/layer6_attention_weight_min": -47.81822204589844,
      "activations/layer7_attention_weight_max": 99.14110565185547,
      "activations/layer7_attention_weight_min": -98.4958267211914,
      "activations/layer8_attention_weight_max": 42.44656753540039,
      "activations/layer8_attention_weight_min": -45.18975830078125,
      "activations/layer9_attention_weight_max": 35.220272064208984,
      "activations/layer9_attention_weight_min": -35.285491943359375,
      "epoch": 13.72,
      "learning_rate": 6.217613636363636e-05,
      "loss": 2.7783,
      "step": 236050
    },
    {
      "activations/layer0_attention_weight_max": 15.629568099975586,
      "activations/layer0_attention_weight_min": -12.911853790283203,
      "activations/layer10_attention_weight_max": 31.96213150024414,
      "activations/layer10_attention_weight_min": -31.969167709350586,
      "activations/layer11_attention_weight_max": 31.643163681030273,
      "activations/layer11_attention_weight_min": -29.92205810546875,
      "activations/layer12_attention_weight_max": 25.62919807434082,
      "activations/layer12_attention_weight_min": -25.776107788085938,
      "activations/layer13_attention_weight_max": 41.1023063659668,
      "activations/layer13_attention_weight_min": -34.59649658203125,
      "activations/layer14_attention_weight_max": 43.73288345336914,
      "activations/layer14_attention_weight_min": -37.05493927001953,
      "activations/layer15_attention_weight_max": 38.856529235839844,
      "activations/layer15_attention_weight_min": -33.6041145324707,
      "activations/layer16_attention_weight_max": 35.7609977722168,
      "activations/layer16_attention_weight_min": -28.909278869628906,
      "activations/layer17_attention_weight_max": 34.730430603027344,
      "activations/layer17_attention_weight_min": -26.979639053344727,
      "activations/layer18_attention_weight_max": 34.75851058959961,
      "activations/layer18_attention_weight_min": -25.291786193847656,
      "activations/layer19_attention_weight_max": 44.3213996887207,
      "activations/layer19_attention_weight_min": -32.98812484741211,
      "activations/layer1_attention_weight_max": 15.848576545715332,
      "activations/layer1_attention_weight_min": -15.342368125915527,
      "activations/layer20_attention_weight_max": 37.038570404052734,
      "activations/layer20_attention_weight_min": -25.135757446289062,
      "activations/layer21_attention_weight_max": 36.36641311645508,
      "activations/layer21_attention_weight_min": -23.294601440429688,
      "activations/layer22_attention_weight_max": 53.53543472290039,
      "activations/layer22_attention_weight_min": -29.94797134399414,
      "activations/layer23_attention_weight_max": 40.43262481689453,
      "activations/layer23_attention_weight_min": -24.657941818237305,
      "activations/layer2_attention_weight_max": 33.884056091308594,
      "activations/layer2_attention_weight_min": -36.387184143066406,
      "activations/layer3_attention_weight_max": 98.44950103759766,
      "activations/layer3_attention_weight_min": -106.31780242919922,
      "activations/layer4_attention_weight_max": 111.5443344116211,
      "activations/layer4_attention_weight_min": -110.77827453613281,
      "activations/layer5_attention_weight_max": 56.70314025878906,
      "activations/layer5_attention_weight_min": -62.433738708496094,
      "activations/layer6_attention_weight_max": 43.81917953491211,
      "activations/layer6_attention_weight_min": -46.667030334472656,
      "activations/layer7_attention_weight_max": 91.03823852539062,
      "activations/layer7_attention_weight_min": -90.6346435546875,
      "activations/layer8_attention_weight_max": 40.27037048339844,
      "activations/layer8_attention_weight_min": -40.79383850097656,
      "activations/layer9_attention_weight_max": 32.47507095336914,
      "activations/layer9_attention_weight_min": -32.0015983581543,
      "epoch": 13.72,
      "learning_rate": 6.215719696969696e-05,
      "loss": 2.7704,
      "step": 236100
    },
    {
      "activations/layer0_attention_weight_max": 16.283700942993164,
      "activations/layer0_attention_weight_min": -13.264313697814941,
      "activations/layer10_attention_weight_max": 30.04574203491211,
      "activations/layer10_attention_weight_min": -31.190011978149414,
      "activations/layer11_attention_weight_max": 30.6767635345459,
      "activations/layer11_attention_weight_min": -30.09716796875,
      "activations/layer12_attention_weight_max": 26.360950469970703,
      "activations/layer12_attention_weight_min": -26.298219680786133,
      "activations/layer13_attention_weight_max": 39.357906341552734,
      "activations/layer13_attention_weight_min": -35.59740447998047,
      "activations/layer14_attention_weight_max": 43.99711227416992,
      "activations/layer14_attention_weight_min": -39.34062194824219,
      "activations/layer15_attention_weight_max": 36.78303527832031,
      "activations/layer15_attention_weight_min": -36.207908630371094,
      "activations/layer16_attention_weight_max": 32.264015197753906,
      "activations/layer16_attention_weight_min": -28.024993896484375,
      "activations/layer17_attention_weight_max": 33.980873107910156,
      "activations/layer17_attention_weight_min": -26.49215316772461,
      "activations/layer18_attention_weight_max": 33.49393081665039,
      "activations/layer18_attention_weight_min": -26.007707595825195,
      "activations/layer19_attention_weight_max": 35.76853942871094,
      "activations/layer19_attention_weight_min": -32.607540130615234,
      "activations/layer1_attention_weight_max": 16.749309539794922,
      "activations/layer1_attention_weight_min": -14.573809623718262,
      "activations/layer20_attention_weight_max": 31.361011505126953,
      "activations/layer20_attention_weight_min": -27.736608505249023,
      "activations/layer21_attention_weight_max": 27.72713279724121,
      "activations/layer21_attention_weight_min": -25.769800186157227,
      "activations/layer22_attention_weight_max": 44.3884162902832,
      "activations/layer22_attention_weight_min": -32.507991790771484,
      "activations/layer23_attention_weight_max": 33.203758239746094,
      "activations/layer23_attention_weight_min": -28.20146942138672,
      "activations/layer2_attention_weight_max": 31.82196807861328,
      "activations/layer2_attention_weight_min": -33.22123336791992,
      "activations/layer3_attention_weight_max": 100.51044464111328,
      "activations/layer3_attention_weight_min": -100.47381591796875,
      "activations/layer4_attention_weight_max": 105.77982330322266,
      "activations/layer4_attention_weight_min": -109.81019592285156,
      "activations/layer5_attention_weight_max": 49.4681510925293,
      "activations/layer5_attention_weight_min": -60.109066009521484,
      "activations/layer6_attention_weight_max": 41.37685012817383,
      "activations/layer6_attention_weight_min": -44.31895446777344,
      "activations/layer7_attention_weight_max": 82.28579711914062,
      "activations/layer7_attention_weight_min": -86.47399139404297,
      "activations/layer8_attention_weight_max": 36.396766662597656,
      "activations/layer8_attention_weight_min": -39.515018463134766,
      "activations/layer9_attention_weight_max": 29.519275665283203,
      "activations/layer9_attention_weight_min": -31.200284957885742,
      "epoch": 13.72,
      "learning_rate": 6.213825757575758e-05,
      "loss": 2.7569,
      "step": 236150
    },
    {
      "activations/layer0_attention_weight_max": 16.70438003540039,
      "activations/layer0_attention_weight_min": -12.75937271118164,
      "activations/layer10_attention_weight_max": 31.66964340209961,
      "activations/layer10_attention_weight_min": -31.977779388427734,
      "activations/layer11_attention_weight_max": 33.79693603515625,
      "activations/layer11_attention_weight_min": -32.07708740234375,
      "activations/layer12_attention_weight_max": 23.737552642822266,
      "activations/layer12_attention_weight_min": -24.82207489013672,
      "activations/layer13_attention_weight_max": 36.63664627075195,
      "activations/layer13_attention_weight_min": -34.16926193237305,
      "activations/layer14_attention_weight_max": 39.35661697387695,
      "activations/layer14_attention_weight_min": -38.73080825805664,
      "activations/layer15_attention_weight_max": 33.90856170654297,
      "activations/layer15_attention_weight_min": -33.709651947021484,
      "activations/layer16_attention_weight_max": 29.118547439575195,
      "activations/layer16_attention_weight_min": -28.665414810180664,
      "activations/layer17_attention_weight_max": 32.052947998046875,
      "activations/layer17_attention_weight_min": -27.660551071166992,
      "activations/layer18_attention_weight_max": 31.811016082763672,
      "activations/layer18_attention_weight_min": -27.063495635986328,
      "activations/layer19_attention_weight_max": 33.97050857543945,
      "activations/layer19_attention_weight_min": -34.102745056152344,
      "activations/layer1_attention_weight_max": 16.997617721557617,
      "activations/layer1_attention_weight_min": -14.385248184204102,
      "activations/layer20_attention_weight_max": 29.801889419555664,
      "activations/layer20_attention_weight_min": -28.668548583984375,
      "activations/layer21_attention_weight_max": 30.160568237304688,
      "activations/layer21_attention_weight_min": -27.230819702148438,
      "activations/layer22_attention_weight_max": 50.246212005615234,
      "activations/layer22_attention_weight_min": -35.02912902832031,
      "activations/layer23_attention_weight_max": 37.602848052978516,
      "activations/layer23_attention_weight_min": -26.873594284057617,
      "activations/layer2_attention_weight_max": 33.71335983276367,
      "activations/layer2_attention_weight_min": -34.961307525634766,
      "activations/layer3_attention_weight_max": 102.8501205444336,
      "activations/layer3_attention_weight_min": -103.1281509399414,
      "activations/layer4_attention_weight_max": 109.3406753540039,
      "activations/layer4_attention_weight_min": -114.5676498413086,
      "activations/layer5_attention_weight_max": 53.354576110839844,
      "activations/layer5_attention_weight_min": -62.04508972167969,
      "activations/layer6_attention_weight_max": 44.324668884277344,
      "activations/layer6_attention_weight_min": -45.84260177612305,
      "activations/layer7_attention_weight_max": 89.91472625732422,
      "activations/layer7_attention_weight_min": -93.77156829833984,
      "activations/layer8_attention_weight_max": 39.244407653808594,
      "activations/layer8_attention_weight_min": -42.76628494262695,
      "activations/layer9_attention_weight_max": 31.6055908203125,
      "activations/layer9_attention_weight_min": -32.808990478515625,
      "epoch": 13.72,
      "learning_rate": 6.211931818181818e-05,
      "loss": 2.7651,
      "step": 236200
    },
    {
      "activations/layer0_attention_weight_max": 15.579835891723633,
      "activations/layer0_attention_weight_min": -13.586255073547363,
      "activations/layer10_attention_weight_max": 33.97994613647461,
      "activations/layer10_attention_weight_min": -34.00055694580078,
      "activations/layer11_attention_weight_max": 31.65334701538086,
      "activations/layer11_attention_weight_min": -32.418357849121094,
      "activations/layer12_attention_weight_max": 26.716659545898438,
      "activations/layer12_attention_weight_min": -25.219486236572266,
      "activations/layer13_attention_weight_max": 38.22765350341797,
      "activations/layer13_attention_weight_min": -35.577423095703125,
      "activations/layer14_attention_weight_max": 42.69922637939453,
      "activations/layer14_attention_weight_min": -38.30202102661133,
      "activations/layer15_attention_weight_max": 35.263221740722656,
      "activations/layer15_attention_weight_min": -33.85068130493164,
      "activations/layer16_attention_weight_max": 29.991943359375,
      "activations/layer16_attention_weight_min": -29.59964942932129,
      "activations/layer17_attention_weight_max": 29.23682975769043,
      "activations/layer17_attention_weight_min": -24.982177734375,
      "activations/layer18_attention_weight_max": 32.90150451660156,
      "activations/layer18_attention_weight_min": -24.0463809967041,
      "activations/layer19_attention_weight_max": 32.617225646972656,
      "activations/layer19_attention_weight_min": -29.80875587463379,
      "activations/layer1_attention_weight_max": 16.54563331604004,
      "activations/layer1_attention_weight_min": -15.1417875289917,
      "activations/layer20_attention_weight_max": 29.108640670776367,
      "activations/layer20_attention_weight_min": -24.638748168945312,
      "activations/layer21_attention_weight_max": 29.539031982421875,
      "activations/layer21_attention_weight_min": -24.534263610839844,
      "activations/layer22_attention_weight_max": 42.26008987426758,
      "activations/layer22_attention_weight_min": -30.068071365356445,
      "activations/layer23_attention_weight_max": 31.567495346069336,
      "activations/layer23_attention_weight_min": -25.26527214050293,
      "activations/layer2_attention_weight_max": 34.78003692626953,
      "activations/layer2_attention_weight_min": -33.76770782470703,
      "activations/layer3_attention_weight_max": 104.65789031982422,
      "activations/layer3_attention_weight_min": -106.9261474609375,
      "activations/layer4_attention_weight_max": 114.98262786865234,
      "activations/layer4_attention_weight_min": -112.3024673461914,
      "activations/layer5_attention_weight_max": 53.341636657714844,
      "activations/layer5_attention_weight_min": -62.06864929199219,
      "activations/layer6_attention_weight_max": 45.13478469848633,
      "activations/layer6_attention_weight_min": -48.34688186645508,
      "activations/layer7_attention_weight_max": 92.37067413330078,
      "activations/layer7_attention_weight_min": -94.89116668701172,
      "activations/layer8_attention_weight_max": 41.06562423706055,
      "activations/layer8_attention_weight_min": -41.5988655090332,
      "activations/layer9_attention_weight_max": 33.16546630859375,
      "activations/layer9_attention_weight_min": -32.570838928222656,
      "epoch": 13.73,
      "learning_rate": 6.210037878787878e-05,
      "loss": 2.7613,
      "step": 236250
    },
    {
      "activations/layer0_attention_weight_max": 16.658641815185547,
      "activations/layer0_attention_weight_min": -13.388684272766113,
      "activations/layer10_attention_weight_max": 30.798912048339844,
      "activations/layer10_attention_weight_min": -32.158966064453125,
      "activations/layer11_attention_weight_max": 31.939727783203125,
      "activations/layer11_attention_weight_min": -32.01631546020508,
      "activations/layer12_attention_weight_max": 24.67951011657715,
      "activations/layer12_attention_weight_min": -25.16577911376953,
      "activations/layer13_attention_weight_max": 38.26823806762695,
      "activations/layer13_attention_weight_min": -36.45879364013672,
      "activations/layer14_attention_weight_max": 39.958526611328125,
      "activations/layer14_attention_weight_min": -35.58124542236328,
      "activations/layer15_attention_weight_max": 38.29901123046875,
      "activations/layer15_attention_weight_min": -33.27241897583008,
      "activations/layer16_attention_weight_max": 35.82994842529297,
      "activations/layer16_attention_weight_min": -28.683761596679688,
      "activations/layer17_attention_weight_max": 31.68378448486328,
      "activations/layer17_attention_weight_min": -25.57656478881836,
      "activations/layer18_attention_weight_max": 31.938819885253906,
      "activations/layer18_attention_weight_min": -25.485881805419922,
      "activations/layer19_attention_weight_max": 33.87092208862305,
      "activations/layer19_attention_weight_min": -31.87619972229004,
      "activations/layer1_attention_weight_max": 16.874250411987305,
      "activations/layer1_attention_weight_min": -15.486838340759277,
      "activations/layer20_attention_weight_max": 29.62477684020996,
      "activations/layer20_attention_weight_min": -26.559919357299805,
      "activations/layer21_attention_weight_max": 32.1408805847168,
      "activations/layer21_attention_weight_min": -24.58184242248535,
      "activations/layer22_attention_weight_max": 47.055015563964844,
      "activations/layer22_attention_weight_min": -29.705577850341797,
      "activations/layer23_attention_weight_max": 34.700279235839844,
      "activations/layer23_attention_weight_min": -25.598281860351562,
      "activations/layer2_attention_weight_max": 32.60836410522461,
      "activations/layer2_attention_weight_min": -33.21063995361328,
      "activations/layer3_attention_weight_max": 98.4127426147461,
      "activations/layer3_attention_weight_min": -99.81594848632812,
      "activations/layer4_attention_weight_max": 110.01470184326172,
      "activations/layer4_attention_weight_min": -109.77906799316406,
      "activations/layer5_attention_weight_max": 50.97565460205078,
      "activations/layer5_attention_weight_min": -62.97398376464844,
      "activations/layer6_attention_weight_max": 44.61405944824219,
      "activations/layer6_attention_weight_min": -47.7999153137207,
      "activations/layer7_attention_weight_max": 90.48664093017578,
      "activations/layer7_attention_weight_min": -90.05615997314453,
      "activations/layer8_attention_weight_max": 38.344852447509766,
      "activations/layer8_attention_weight_min": -39.840660095214844,
      "activations/layer9_attention_weight_max": 30.463748931884766,
      "activations/layer9_attention_weight_min": -31.25957679748535,
      "epoch": 13.73,
      "learning_rate": 6.20814393939394e-05,
      "loss": 2.7541,
      "step": 236300
    },
    {
      "activations/layer0_attention_weight_max": 15.293917655944824,
      "activations/layer0_attention_weight_min": -12.362445831298828,
      "activations/layer10_attention_weight_max": 32.3453369140625,
      "activations/layer10_attention_weight_min": -31.350555419921875,
      "activations/layer11_attention_weight_max": 30.591121673583984,
      "activations/layer11_attention_weight_min": -31.475399017333984,
      "activations/layer12_attention_weight_max": 25.42446517944336,
      "activations/layer12_attention_weight_min": -25.010337829589844,
      "activations/layer13_attention_weight_max": 37.925010681152344,
      "activations/layer13_attention_weight_min": -34.40073776245117,
      "activations/layer14_attention_weight_max": 39.74693298339844,
      "activations/layer14_attention_weight_min": -38.24067687988281,
      "activations/layer15_attention_weight_max": 37.11904525756836,
      "activations/layer15_attention_weight_min": -33.59185028076172,
      "activations/layer16_attention_weight_max": 28.90447235107422,
      "activations/layer16_attention_weight_min": -28.695165634155273,
      "activations/layer17_attention_weight_max": 29.617536544799805,
      "activations/layer17_attention_weight_min": -26.589582443237305,
      "activations/layer18_attention_weight_max": 30.58968734741211,
      "activations/layer18_attention_weight_min": -24.675752639770508,
      "activations/layer19_attention_weight_max": 35.861873626708984,
      "activations/layer19_attention_weight_min": -31.555509567260742,
      "activations/layer1_attention_weight_max": 16.690397262573242,
      "activations/layer1_attention_weight_min": -15.078117370605469,
      "activations/layer20_attention_weight_max": 27.964941024780273,
      "activations/layer20_attention_weight_min": -24.75469398498535,
      "activations/layer21_attention_weight_max": 29.244571685791016,
      "activations/layer21_attention_weight_min": -25.0324764251709,
      "activations/layer22_attention_weight_max": 40.446754455566406,
      "activations/layer22_attention_weight_min": -33.158287048339844,
      "activations/layer23_attention_weight_max": 34.828102111816406,
      "activations/layer23_attention_weight_min": -27.786632537841797,
      "activations/layer2_attention_weight_max": 34.14979934692383,
      "activations/layer2_attention_weight_min": -35.66778564453125,
      "activations/layer3_attention_weight_max": 100.34945678710938,
      "activations/layer3_attention_weight_min": -100.40452575683594,
      "activations/layer4_attention_weight_max": 114.21825408935547,
      "activations/layer4_attention_weight_min": -109.3808364868164,
      "activations/layer5_attention_weight_max": 52.21540069580078,
      "activations/layer5_attention_weight_min": -61.1756591796875,
      "activations/layer6_attention_weight_max": 45.59794616699219,
      "activations/layer6_attention_weight_min": -47.07255172729492,
      "activations/layer7_attention_weight_max": 92.2703857421875,
      "activations/layer7_attention_weight_min": -90.0016860961914,
      "activations/layer8_attention_weight_max": 39.0583610534668,
      "activations/layer8_attention_weight_min": -40.20587921142578,
      "activations/layer9_attention_weight_max": 31.840436935424805,
      "activations/layer9_attention_weight_min": -32.896480560302734,
      "epoch": 13.73,
      "learning_rate": 6.20625e-05,
      "loss": 2.7587,
      "step": 236350
    },
    {
      "activations/layer0_attention_weight_max": 15.922577857971191,
      "activations/layer0_attention_weight_min": -14.135400772094727,
      "activations/layer10_attention_weight_max": 34.53761672973633,
      "activations/layer10_attention_weight_min": -33.03512954711914,
      "activations/layer11_attention_weight_max": 36.00095748901367,
      "activations/layer11_attention_weight_min": -33.1849479675293,
      "activations/layer12_attention_weight_max": 26.536415100097656,
      "activations/layer12_attention_weight_min": -27.142295837402344,
      "activations/layer13_attention_weight_max": 40.36131286621094,
      "activations/layer13_attention_weight_min": -34.53179168701172,
      "activations/layer14_attention_weight_max": 43.29452133178711,
      "activations/layer14_attention_weight_min": -36.42008972167969,
      "activations/layer15_attention_weight_max": 37.03205108642578,
      "activations/layer15_attention_weight_min": -32.697566986083984,
      "activations/layer16_attention_weight_max": 30.349092483520508,
      "activations/layer16_attention_weight_min": -26.295976638793945,
      "activations/layer17_attention_weight_max": 30.758949279785156,
      "activations/layer17_attention_weight_min": -24.124797821044922,
      "activations/layer18_attention_weight_max": 31.012907028198242,
      "activations/layer18_attention_weight_min": -24.63980484008789,
      "activations/layer19_attention_weight_max": 33.428565979003906,
      "activations/layer19_attention_weight_min": -29.20932960510254,
      "activations/layer1_attention_weight_max": 16.842853546142578,
      "activations/layer1_attention_weight_min": -15.617990493774414,
      "activations/layer20_attention_weight_max": 28.750431060791016,
      "activations/layer20_attention_weight_min": -24.928905487060547,
      "activations/layer21_attention_weight_max": 29.646780014038086,
      "activations/layer21_attention_weight_min": -23.826799392700195,
      "activations/layer22_attention_weight_max": 42.46040344238281,
      "activations/layer22_attention_weight_min": -31.741554260253906,
      "activations/layer23_attention_weight_max": 33.62897491455078,
      "activations/layer23_attention_weight_min": -28.261154174804688,
      "activations/layer2_attention_weight_max": 36.06296157836914,
      "activations/layer2_attention_weight_min": -34.72541046142578,
      "activations/layer3_attention_weight_max": 110.20323181152344,
      "activations/layer3_attention_weight_min": -106.46126556396484,
      "activations/layer4_attention_weight_max": 119.62251281738281,
      "activations/layer4_attention_weight_min": -117.50206756591797,
      "activations/layer5_attention_weight_max": 55.39666748046875,
      "activations/layer5_attention_weight_min": -63.13176727294922,
      "activations/layer6_attention_weight_max": 50.84793472290039,
      "activations/layer6_attention_weight_min": -50.714969635009766,
      "activations/layer7_attention_weight_max": 93.61799621582031,
      "activations/layer7_attention_weight_min": -99.41217041015625,
      "activations/layer8_attention_weight_max": 43.5622444152832,
      "activations/layer8_attention_weight_min": -43.949546813964844,
      "activations/layer9_attention_weight_max": 35.56858444213867,
      "activations/layer9_attention_weight_min": -34.38204574584961,
      "epoch": 13.74,
      "learning_rate": 6.20435606060606e-05,
      "loss": 2.7642,
      "step": 236400
    },
    {
      "activations/layer0_attention_weight_max": 16.035476684570312,
      "activations/layer0_attention_weight_min": -12.574775695800781,
      "activations/layer10_attention_weight_max": 35.59281921386719,
      "activations/layer10_attention_weight_min": -33.2171516418457,
      "activations/layer11_attention_weight_max": 37.42407989501953,
      "activations/layer11_attention_weight_min": -35.08971405029297,
      "activations/layer12_attention_weight_max": 24.143747329711914,
      "activations/layer12_attention_weight_min": -30.252002716064453,
      "activations/layer13_attention_weight_max": 38.961090087890625,
      "activations/layer13_attention_weight_min": -32.813880920410156,
      "activations/layer14_attention_weight_max": 41.43632125854492,
      "activations/layer14_attention_weight_min": -38.70457458496094,
      "activations/layer15_attention_weight_max": 39.73711395263672,
      "activations/layer15_attention_weight_min": -33.1054573059082,
      "activations/layer16_attention_weight_max": 29.29816246032715,
      "activations/layer16_attention_weight_min": -27.96495819091797,
      "activations/layer17_attention_weight_max": 28.55425453186035,
      "activations/layer17_attention_weight_min": -26.387767791748047,
      "activations/layer18_attention_weight_max": 30.337383270263672,
      "activations/layer18_attention_weight_min": -24.610633850097656,
      "activations/layer19_attention_weight_max": 32.81257247924805,
      "activations/layer19_attention_weight_min": -30.270416259765625,
      "activations/layer1_attention_weight_max": 17.391803741455078,
      "activations/layer1_attention_weight_min": -15.886934280395508,
      "activations/layer20_attention_weight_max": 28.522356033325195,
      "activations/layer20_attention_weight_min": -24.79728889465332,
      "activations/layer21_attention_weight_max": 27.98440170288086,
      "activations/layer21_attention_weight_min": -24.243431091308594,
      "activations/layer22_attention_weight_max": 42.33952713012695,
      "activations/layer22_attention_weight_min": -31.13654327392578,
      "activations/layer23_attention_weight_max": 33.78150939941406,
      "activations/layer23_attention_weight_min": -25.998546600341797,
      "activations/layer2_attention_weight_max": 37.0078125,
      "activations/layer2_attention_weight_min": -36.147857666015625,
      "activations/layer3_attention_weight_max": 109.94831085205078,
      "activations/layer3_attention_weight_min": -108.57581329345703,
      "activations/layer4_attention_weight_max": 112.57866668701172,
      "activations/layer4_attention_weight_min": -112.40470123291016,
      "activations/layer5_attention_weight_max": 55.10163879394531,
      "activations/layer5_attention_weight_min": -61.165916442871094,
      "activations/layer6_attention_weight_max": 43.35102462768555,
      "activations/layer6_attention_weight_min": -45.73867416381836,
      "activations/layer7_attention_weight_max": 96.2054214477539,
      "activations/layer7_attention_weight_min": -91.298828125,
      "activations/layer8_attention_weight_max": 41.32573318481445,
      "activations/layer8_attention_weight_min": -42.82598876953125,
      "activations/layer9_attention_weight_max": 33.619346618652344,
      "activations/layer9_attention_weight_min": -34.177947998046875,
      "epoch": 13.74,
      "learning_rate": 6.20246212121212e-05,
      "loss": 2.7715,
      "step": 236450
    },
    {
      "activations/layer0_attention_weight_max": 15.760000228881836,
      "activations/layer0_attention_weight_min": -12.247493743896484,
      "activations/layer10_attention_weight_max": 36.55949020385742,
      "activations/layer10_attention_weight_min": -38.42549514770508,
      "activations/layer11_attention_weight_max": 40.31795120239258,
      "activations/layer11_attention_weight_min": -38.05363464355469,
      "activations/layer12_attention_weight_max": 26.629587173461914,
      "activations/layer12_attention_weight_min": -25.64044761657715,
      "activations/layer13_attention_weight_max": 38.59114074707031,
      "activations/layer13_attention_weight_min": -35.54299545288086,
      "activations/layer14_attention_weight_max": 44.71603012084961,
      "activations/layer14_attention_weight_min": -38.261444091796875,
      "activations/layer15_attention_weight_max": 41.62485122680664,
      "activations/layer15_attention_weight_min": -33.9030647277832,
      "activations/layer16_attention_weight_max": 32.58871841430664,
      "activations/layer16_attention_weight_min": -27.97797393798828,
      "activations/layer17_attention_weight_max": 32.1649284362793,
      "activations/layer17_attention_weight_min": -27.822525024414062,
      "activations/layer18_attention_weight_max": 34.249332427978516,
      "activations/layer18_attention_weight_min": -24.19839859008789,
      "activations/layer19_attention_weight_max": 36.338218688964844,
      "activations/layer19_attention_weight_min": -34.253265380859375,
      "activations/layer1_attention_weight_max": 16.20103645324707,
      "activations/layer1_attention_weight_min": -15.281122207641602,
      "activations/layer20_attention_weight_max": 31.021135330200195,
      "activations/layer20_attention_weight_min": -25.61247444152832,
      "activations/layer21_attention_weight_max": 30.360885620117188,
      "activations/layer21_attention_weight_min": -24.842947006225586,
      "activations/layer22_attention_weight_max": 44.283573150634766,
      "activations/layer22_attention_weight_min": -34.62177276611328,
      "activations/layer23_attention_weight_max": 37.076202392578125,
      "activations/layer23_attention_weight_min": -29.41168212890625,
      "activations/layer2_attention_weight_max": 35.9564323425293,
      "activations/layer2_attention_weight_min": -34.55250549316406,
      "activations/layer3_attention_weight_max": 102.56648254394531,
      "activations/layer3_attention_weight_min": -105.89034271240234,
      "activations/layer4_attention_weight_max": 114.7622299194336,
      "activations/layer4_attention_weight_min": -114.03602600097656,
      "activations/layer5_attention_weight_max": 52.308631896972656,
      "activations/layer5_attention_weight_min": -60.94163131713867,
      "activations/layer6_attention_weight_max": 44.00392150878906,
      "activations/layer6_attention_weight_min": -51.16682815551758,
      "activations/layer7_attention_weight_max": 94.27401733398438,
      "activations/layer7_attention_weight_min": -98.88448333740234,
      "activations/layer8_attention_weight_max": 43.37553405761719,
      "activations/layer8_attention_weight_min": -44.44031524658203,
      "activations/layer9_attention_weight_max": 34.032649993896484,
      "activations/layer9_attention_weight_min": -36.1661491394043,
      "epoch": 13.74,
      "learning_rate": 6.200568181818182e-05,
      "loss": 2.751,
      "step": 236500
    },
    {
      "activations/layer0_attention_weight_max": 15.879246711730957,
      "activations/layer0_attention_weight_min": -12.525495529174805,
      "activations/layer10_attention_weight_max": 32.08300018310547,
      "activations/layer10_attention_weight_min": -32.58980178833008,
      "activations/layer11_attention_weight_max": 31.992897033691406,
      "activations/layer11_attention_weight_min": -33.42931365966797,
      "activations/layer12_attention_weight_max": 28.150728225708008,
      "activations/layer12_attention_weight_min": -26.663265228271484,
      "activations/layer13_attention_weight_max": 41.497032165527344,
      "activations/layer13_attention_weight_min": -36.58987045288086,
      "activations/layer14_attention_weight_max": 46.14643478393555,
      "activations/layer14_attention_weight_min": -38.954071044921875,
      "activations/layer15_attention_weight_max": 37.573204040527344,
      "activations/layer15_attention_weight_min": -33.131683349609375,
      "activations/layer16_attention_weight_max": 30.0225830078125,
      "activations/layer16_attention_weight_min": -29.038522720336914,
      "activations/layer17_attention_weight_max": 32.92984390258789,
      "activations/layer17_attention_weight_min": -25.27572250366211,
      "activations/layer18_attention_weight_max": 30.441471099853516,
      "activations/layer18_attention_weight_min": -26.003286361694336,
      "activations/layer19_attention_weight_max": 35.69925308227539,
      "activations/layer19_attention_weight_min": -30.833730697631836,
      "activations/layer1_attention_weight_max": 16.398847579956055,
      "activations/layer1_attention_weight_min": -15.098128318786621,
      "activations/layer20_attention_weight_max": 28.654033660888672,
      "activations/layer20_attention_weight_min": -24.94146728515625,
      "activations/layer21_attention_weight_max": 32.05903244018555,
      "activations/layer21_attention_weight_min": -25.176992416381836,
      "activations/layer22_attention_weight_max": 43.95429611206055,
      "activations/layer22_attention_weight_min": -32.6845703125,
      "activations/layer23_attention_weight_max": 32.88624954223633,
      "activations/layer23_attention_weight_min": -25.597211837768555,
      "activations/layer2_attention_weight_max": 33.986026763916016,
      "activations/layer2_attention_weight_min": -34.087318420410156,
      "activations/layer3_attention_weight_max": 102.31752014160156,
      "activations/layer3_attention_weight_min": -104.07230377197266,
      "activations/layer4_attention_weight_max": 115.8001937866211,
      "activations/layer4_attention_weight_min": -111.3732681274414,
      "activations/layer5_attention_weight_max": 53.84759521484375,
      "activations/layer5_attention_weight_min": -64.06696319580078,
      "activations/layer6_attention_weight_max": 44.61157989501953,
      "activations/layer6_attention_weight_min": -47.153263092041016,
      "activations/layer7_attention_weight_max": 91.7641372680664,
      "activations/layer7_attention_weight_min": -91.22168731689453,
      "activations/layer8_attention_weight_max": 40.054908752441406,
      "activations/layer8_attention_weight_min": -41.5115966796875,
      "activations/layer9_attention_weight_max": 33.02181625366211,
      "activations/layer9_attention_weight_min": -33.61315155029297,
      "epoch": 13.74,
      "learning_rate": 6.198674242424242e-05,
      "loss": 2.7388,
      "step": 236550
    },
    {
      "activations/layer0_attention_weight_max": 16.70360565185547,
      "activations/layer0_attention_weight_min": -13.034114837646484,
      "activations/layer10_attention_weight_max": 30.732906341552734,
      "activations/layer10_attention_weight_min": -32.66347885131836,
      "activations/layer11_attention_weight_max": 32.32908630371094,
      "activations/layer11_attention_weight_min": -31.163421630859375,
      "activations/layer12_attention_weight_max": 26.35770606994629,
      "activations/layer12_attention_weight_min": -26.526676177978516,
      "activations/layer13_attention_weight_max": 40.51240539550781,
      "activations/layer13_attention_weight_min": -33.71836471557617,
      "activations/layer14_attention_weight_max": 45.16315841674805,
      "activations/layer14_attention_weight_min": -39.23623275756836,
      "activations/layer15_attention_weight_max": 43.2397346496582,
      "activations/layer15_attention_weight_min": -37.196327209472656,
      "activations/layer16_attention_weight_max": 36.4289665222168,
      "activations/layer16_attention_weight_min": -28.399045944213867,
      "activations/layer17_attention_weight_max": 33.03296661376953,
      "activations/layer17_attention_weight_min": -27.337621688842773,
      "activations/layer18_attention_weight_max": 32.39360046386719,
      "activations/layer18_attention_weight_min": -25.20383071899414,
      "activations/layer19_attention_weight_max": 41.480709075927734,
      "activations/layer19_attention_weight_min": -32.11229705810547,
      "activations/layer1_attention_weight_max": 17.85227394104004,
      "activations/layer1_attention_weight_min": -14.310445785522461,
      "activations/layer20_attention_weight_max": 31.457530975341797,
      "activations/layer20_attention_weight_min": -24.965288162231445,
      "activations/layer21_attention_weight_max": 29.537029266357422,
      "activations/layer21_attention_weight_min": -24.554710388183594,
      "activations/layer22_attention_weight_max": 46.13214874267578,
      "activations/layer22_attention_weight_min": -30.42085075378418,
      "activations/layer23_attention_weight_max": 37.235198974609375,
      "activations/layer23_attention_weight_min": -27.059125900268555,
      "activations/layer2_attention_weight_max": 34.26697540283203,
      "activations/layer2_attention_weight_min": -33.19941711425781,
      "activations/layer3_attention_weight_max": 101.61763763427734,
      "activations/layer3_attention_weight_min": -101.59102630615234,
      "activations/layer4_attention_weight_max": 118.18157958984375,
      "activations/layer4_attention_weight_min": -118.7569808959961,
      "activations/layer5_attention_weight_max": 51.65644454956055,
      "activations/layer5_attention_weight_min": -67.0862045288086,
      "activations/layer6_attention_weight_max": 47.67700958251953,
      "activations/layer6_attention_weight_min": -50.474430084228516,
      "activations/layer7_attention_weight_max": 90.74876403808594,
      "activations/layer7_attention_weight_min": -93.32052612304688,
      "activations/layer8_attention_weight_max": 38.36494827270508,
      "activations/layer8_attention_weight_min": -43.89743423461914,
      "activations/layer9_attention_weight_max": 31.062976837158203,
      "activations/layer9_attention_weight_min": -31.861209869384766,
      "epoch": 13.75,
      "learning_rate": 6.196780303030302e-05,
      "loss": 2.7665,
      "step": 236600
    },
    {
      "activations/layer0_attention_weight_max": 16.391521453857422,
      "activations/layer0_attention_weight_min": -12.575475692749023,
      "activations/layer10_attention_weight_max": 36.00732421875,
      "activations/layer10_attention_weight_min": -35.48686599731445,
      "activations/layer11_attention_weight_max": 37.46073913574219,
      "activations/layer11_attention_weight_min": -40.450538635253906,
      "activations/layer12_attention_weight_max": 37.24369430541992,
      "activations/layer12_attention_weight_min": -27.553627014160156,
      "activations/layer13_attention_weight_max": 49.85423278808594,
      "activations/layer13_attention_weight_min": -36.53804016113281,
      "activations/layer14_attention_weight_max": 48.929046630859375,
      "activations/layer14_attention_weight_min": -42.56016540527344,
      "activations/layer15_attention_weight_max": 46.08134460449219,
      "activations/layer15_attention_weight_min": -35.622440338134766,
      "activations/layer16_attention_weight_max": 32.62790298461914,
      "activations/layer16_attention_weight_min": -28.428956985473633,
      "activations/layer17_attention_weight_max": 33.62422561645508,
      "activations/layer17_attention_weight_min": -26.348783493041992,
      "activations/layer18_attention_weight_max": 32.3865852355957,
      "activations/layer18_attention_weight_min": -27.507266998291016,
      "activations/layer19_attention_weight_max": 41.57515335083008,
      "activations/layer19_attention_weight_min": -33.52646255493164,
      "activations/layer1_attention_weight_max": 16.691274642944336,
      "activations/layer1_attention_weight_min": -14.470008850097656,
      "activations/layer20_attention_weight_max": 34.8237419128418,
      "activations/layer20_attention_weight_min": -26.953718185424805,
      "activations/layer21_attention_weight_max": 29.848478317260742,
      "activations/layer21_attention_weight_min": -24.713096618652344,
      "activations/layer22_attention_weight_max": 43.77618408203125,
      "activations/layer22_attention_weight_min": -30.459774017333984,
      "activations/layer23_attention_weight_max": 34.054786682128906,
      "activations/layer23_attention_weight_min": -26.248798370361328,
      "activations/layer2_attention_weight_max": 34.12706756591797,
      "activations/layer2_attention_weight_min": -35.3284797668457,
      "activations/layer3_attention_weight_max": 103.55960083007812,
      "activations/layer3_attention_weight_min": -105.34942626953125,
      "activations/layer4_attention_weight_max": 113.9370346069336,
      "activations/layer4_attention_weight_min": -115.26148986816406,
      "activations/layer5_attention_weight_max": 53.77699279785156,
      "activations/layer5_attention_weight_min": -64.26890563964844,
      "activations/layer6_attention_weight_max": 45.602176666259766,
      "activations/layer6_attention_weight_min": -48.12748718261719,
      "activations/layer7_attention_weight_max": 93.0597915649414,
      "activations/layer7_attention_weight_min": -96.85147094726562,
      "activations/layer8_attention_weight_max": 43.12278366088867,
      "activations/layer8_attention_weight_min": -46.26226043701172,
      "activations/layer9_attention_weight_max": 36.64794921875,
      "activations/layer9_attention_weight_min": -38.340396881103516,
      "epoch": 13.75,
      "learning_rate": 6.194886363636363e-05,
      "loss": 2.7563,
      "step": 236650
    },
    {
      "activations/layer0_attention_weight_max": 16.725341796875,
      "activations/layer0_attention_weight_min": -12.673417091369629,
      "activations/layer10_attention_weight_max": 31.335205078125,
      "activations/layer10_attention_weight_min": -34.43996810913086,
      "activations/layer11_attention_weight_max": 33.205413818359375,
      "activations/layer11_attention_weight_min": -33.604957580566406,
      "activations/layer12_attention_weight_max": 28.278596878051758,
      "activations/layer12_attention_weight_min": -26.08879852294922,
      "activations/layer13_attention_weight_max": 46.28545379638672,
      "activations/layer13_attention_weight_min": -36.71345138549805,
      "activations/layer14_attention_weight_max": 50.78349304199219,
      "activations/layer14_attention_weight_min": -38.92354202270508,
      "activations/layer15_attention_weight_max": 39.25159454345703,
      "activations/layer15_attention_weight_min": -34.253273010253906,
      "activations/layer16_attention_weight_max": 35.34172058105469,
      "activations/layer16_attention_weight_min": -27.6226806640625,
      "activations/layer17_attention_weight_max": 33.38981628417969,
      "activations/layer17_attention_weight_min": -26.378992080688477,
      "activations/layer18_attention_weight_max": 32.52961349487305,
      "activations/layer18_attention_weight_min": -25.9349365234375,
      "activations/layer19_attention_weight_max": 35.00619125366211,
      "activations/layer19_attention_weight_min": -31.352197647094727,
      "activations/layer1_attention_weight_max": 16.92606544494629,
      "activations/layer1_attention_weight_min": -15.004097938537598,
      "activations/layer20_attention_weight_max": 31.615217208862305,
      "activations/layer20_attention_weight_min": -25.52852439880371,
      "activations/layer21_attention_weight_max": 32.72888946533203,
      "activations/layer21_attention_weight_min": -26.54519271850586,
      "activations/layer22_attention_weight_max": 42.64055252075195,
      "activations/layer22_attention_weight_min": -32.37852478027344,
      "activations/layer23_attention_weight_max": 35.67039489746094,
      "activations/layer23_attention_weight_min": -27.329601287841797,
      "activations/layer2_attention_weight_max": 34.108055114746094,
      "activations/layer2_attention_weight_min": -34.507110595703125,
      "activations/layer3_attention_weight_max": 102.87444305419922,
      "activations/layer3_attention_weight_min": -102.55313873291016,
      "activations/layer4_attention_weight_max": 113.4439926147461,
      "activations/layer4_attention_weight_min": -114.50685119628906,
      "activations/layer5_attention_weight_max": 54.1395378112793,
      "activations/layer5_attention_weight_min": -62.55424118041992,
      "activations/layer6_attention_weight_max": 48.03181838989258,
      "activations/layer6_attention_weight_min": -49.461551666259766,
      "activations/layer7_attention_weight_max": 102.84332275390625,
      "activations/layer7_attention_weight_min": -94.18424224853516,
      "activations/layer8_attention_weight_max": 41.43233871459961,
      "activations/layer8_attention_weight_min": -42.9029426574707,
      "activations/layer9_attention_weight_max": 35.5369987487793,
      "activations/layer9_attention_weight_min": -33.66332244873047,
      "epoch": 13.75,
      "learning_rate": 6.192992424242424e-05,
      "loss": 2.762,
      "step": 236700
    },
    {
      "activations/layer0_attention_weight_max": 16.708337783813477,
      "activations/layer0_attention_weight_min": -14.171448707580566,
      "activations/layer10_attention_weight_max": 34.29909896850586,
      "activations/layer10_attention_weight_min": -31.551450729370117,
      "activations/layer11_attention_weight_max": 33.81380081176758,
      "activations/layer11_attention_weight_min": -31.418994903564453,
      "activations/layer12_attention_weight_max": 25.78108024597168,
      "activations/layer12_attention_weight_min": -24.68000602722168,
      "activations/layer13_attention_weight_max": 38.91105270385742,
      "activations/layer13_attention_weight_min": -32.58747863769531,
      "activations/layer14_attention_weight_max": 41.74412155151367,
      "activations/layer14_attention_weight_min": -37.375205993652344,
      "activations/layer15_attention_weight_max": 37.91973114013672,
      "activations/layer15_attention_weight_min": -32.777767181396484,
      "activations/layer16_attention_weight_max": 30.351627349853516,
      "activations/layer16_attention_weight_min": -27.075912475585938,
      "activations/layer17_attention_weight_max": 30.00754737854004,
      "activations/layer17_attention_weight_min": -26.227920532226562,
      "activations/layer18_attention_weight_max": 32.54433059692383,
      "activations/layer18_attention_weight_min": -26.482009887695312,
      "activations/layer19_attention_weight_max": 35.90358352661133,
      "activations/layer19_attention_weight_min": -32.463016510009766,
      "activations/layer1_attention_weight_max": 16.831768035888672,
      "activations/layer1_attention_weight_min": -15.308160781860352,
      "activations/layer20_attention_weight_max": 29.40791893005371,
      "activations/layer20_attention_weight_min": -27.685405731201172,
      "activations/layer21_attention_weight_max": 28.988176345825195,
      "activations/layer21_attention_weight_min": -27.19441795349121,
      "activations/layer22_attention_weight_max": 41.58447265625,
      "activations/layer22_attention_weight_min": -33.92654800415039,
      "activations/layer23_attention_weight_max": 32.923099517822266,
      "activations/layer23_attention_weight_min": -28.326007843017578,
      "activations/layer2_attention_weight_max": 35.66621017456055,
      "activations/layer2_attention_weight_min": -35.07257080078125,
      "activations/layer3_attention_weight_max": 105.37349700927734,
      "activations/layer3_attention_weight_min": -103.89311981201172,
      "activations/layer4_attention_weight_max": 115.39318084716797,
      "activations/layer4_attention_weight_min": -119.3218765258789,
      "activations/layer5_attention_weight_max": 52.859535217285156,
      "activations/layer5_attention_weight_min": -65.12435913085938,
      "activations/layer6_attention_weight_max": 47.20387649536133,
      "activations/layer6_attention_weight_min": -51.53201675415039,
      "activations/layer7_attention_weight_max": 93.2725601196289,
      "activations/layer7_attention_weight_min": -98.3993911743164,
      "activations/layer8_attention_weight_max": 41.23567581176758,
      "activations/layer8_attention_weight_min": -41.45478057861328,
      "activations/layer9_attention_weight_max": 32.74613952636719,
      "activations/layer9_attention_weight_min": -32.14453887939453,
      "epoch": 13.76,
      "learning_rate": 6.191098484848484e-05,
      "loss": 2.7699,
      "step": 236750
    },
    {
      "activations/layer0_attention_weight_max": 16.630096435546875,
      "activations/layer0_attention_weight_min": -12.606480598449707,
      "activations/layer10_attention_weight_max": 35.014007568359375,
      "activations/layer10_attention_weight_min": -35.855690002441406,
      "activations/layer11_attention_weight_max": 39.5235481262207,
      "activations/layer11_attention_weight_min": -37.117496490478516,
      "activations/layer12_attention_weight_max": 32.34280014038086,
      "activations/layer12_attention_weight_min": -29.99364471435547,
      "activations/layer13_attention_weight_max": 51.43938064575195,
      "activations/layer13_attention_weight_min": -38.024864196777344,
      "activations/layer14_attention_weight_max": 58.97464370727539,
      "activations/layer14_attention_weight_min": -45.5147705078125,
      "activations/layer15_attention_weight_max": 47.832027435302734,
      "activations/layer15_attention_weight_min": -37.67613220214844,
      "activations/layer16_attention_weight_max": 40.986995697021484,
      "activations/layer16_attention_weight_min": -28.51407814025879,
      "activations/layer17_attention_weight_max": 33.94563674926758,
      "activations/layer17_attention_weight_min": -26.027956008911133,
      "activations/layer18_attention_weight_max": 34.727943420410156,
      "activations/layer18_attention_weight_min": -25.959304809570312,
      "activations/layer19_attention_weight_max": 42.76764678955078,
      "activations/layer19_attention_weight_min": -31.31965446472168,
      "activations/layer1_attention_weight_max": 17.305194854736328,
      "activations/layer1_attention_weight_min": -15.319704055786133,
      "activations/layer20_attention_weight_max": 34.517677307128906,
      "activations/layer20_attention_weight_min": -25.363868713378906,
      "activations/layer21_attention_weight_max": 36.313194274902344,
      "activations/layer21_attention_weight_min": -24.306188583374023,
      "activations/layer22_attention_weight_max": 53.2916374206543,
      "activations/layer22_attention_weight_min": -30.900632858276367,
      "activations/layer23_attention_weight_max": 40.020599365234375,
      "activations/layer23_attention_weight_min": -26.377531051635742,
      "activations/layer2_attention_weight_max": 34.905906677246094,
      "activations/layer2_attention_weight_min": -33.76494216918945,
      "activations/layer3_attention_weight_max": 102.06350708007812,
      "activations/layer3_attention_weight_min": -104.57037353515625,
      "activations/layer4_attention_weight_max": 111.85096740722656,
      "activations/layer4_attention_weight_min": -111.71429443359375,
      "activations/layer5_attention_weight_max": 52.42882537841797,
      "activations/layer5_attention_weight_min": -64.14253997802734,
      "activations/layer6_attention_weight_max": 43.53092575073242,
      "activations/layer6_attention_weight_min": -45.04584503173828,
      "activations/layer7_attention_weight_max": 98.26760864257812,
      "activations/layer7_attention_weight_min": -93.62901306152344,
      "activations/layer8_attention_weight_max": 41.502716064453125,
      "activations/layer8_attention_weight_min": -43.698490142822266,
      "activations/layer9_attention_weight_max": 33.72639083862305,
      "activations/layer9_attention_weight_min": -34.50455093383789,
      "epoch": 13.76,
      "learning_rate": 6.189204545454545e-05,
      "loss": 2.7782,
      "step": 236800
    },
    {
      "activations/layer0_attention_weight_max": 16.164823532104492,
      "activations/layer0_attention_weight_min": -12.506425857543945,
      "activations/layer10_attention_weight_max": 34.14997100830078,
      "activations/layer10_attention_weight_min": -34.077938079833984,
      "activations/layer11_attention_weight_max": 32.75032043457031,
      "activations/layer11_attention_weight_min": -32.42436599731445,
      "activations/layer12_attention_weight_max": 24.1415958404541,
      "activations/layer12_attention_weight_min": -26.43050765991211,
      "activations/layer13_attention_weight_max": 37.093406677246094,
      "activations/layer13_attention_weight_min": -34.69828796386719,
      "activations/layer14_attention_weight_max": 41.74958419799805,
      "activations/layer14_attention_weight_min": -36.26377487182617,
      "activations/layer15_attention_weight_max": 37.67389678955078,
      "activations/layer15_attention_weight_min": -32.85013961791992,
      "activations/layer16_attention_weight_max": 29.259294509887695,
      "activations/layer16_attention_weight_min": -28.57818603515625,
      "activations/layer17_attention_weight_max": 31.657840728759766,
      "activations/layer17_attention_weight_min": -27.963356018066406,
      "activations/layer18_attention_weight_max": 30.58824348449707,
      "activations/layer18_attention_weight_min": -25.37300682067871,
      "activations/layer19_attention_weight_max": 36.43679428100586,
      "activations/layer19_attention_weight_min": -29.827716827392578,
      "activations/layer1_attention_weight_max": 17.802703857421875,
      "activations/layer1_attention_weight_min": -15.579276084899902,
      "activations/layer20_attention_weight_max": 30.946033477783203,
      "activations/layer20_attention_weight_min": -24.883298873901367,
      "activations/layer21_attention_weight_max": 30.201852798461914,
      "activations/layer21_attention_weight_min": -24.97092628479004,
      "activations/layer22_attention_weight_max": 43.08481216430664,
      "activations/layer22_attention_weight_min": -31.385101318359375,
      "activations/layer23_attention_weight_max": 35.836753845214844,
      "activations/layer23_attention_weight_min": -24.40713119506836,
      "activations/layer2_attention_weight_max": 35.42935562133789,
      "activations/layer2_attention_weight_min": -36.53403091430664,
      "activations/layer3_attention_weight_max": 102.17108917236328,
      "activations/layer3_attention_weight_min": -107.12606811523438,
      "activations/layer4_attention_weight_max": 113.8971939086914,
      "activations/layer4_attention_weight_min": -116.16112518310547,
      "activations/layer5_attention_weight_max": 52.219573974609375,
      "activations/layer5_attention_weight_min": -67.36588287353516,
      "activations/layer6_attention_weight_max": 45.74354934692383,
      "activations/layer6_attention_weight_min": -50.34843444824219,
      "activations/layer7_attention_weight_max": 93.9925537109375,
      "activations/layer7_attention_weight_min": -98.86581420898438,
      "activations/layer8_attention_weight_max": 40.98677444458008,
      "activations/layer8_attention_weight_min": -43.84550857543945,
      "activations/layer9_attention_weight_max": 32.04999923706055,
      "activations/layer9_attention_weight_min": -33.88865280151367,
      "epoch": 13.76,
      "learning_rate": 6.187310606060605e-05,
      "loss": 2.7499,
      "step": 236850
    },
    {
      "activations/layer0_attention_weight_max": 17.076295852661133,
      "activations/layer0_attention_weight_min": -12.597841262817383,
      "activations/layer10_attention_weight_max": 37.76481246948242,
      "activations/layer10_attention_weight_min": -35.9666748046875,
      "activations/layer11_attention_weight_max": 38.14478302001953,
      "activations/layer11_attention_weight_min": -36.49903106689453,
      "activations/layer12_attention_weight_max": 26.28919219970703,
      "activations/layer12_attention_weight_min": -28.032926559448242,
      "activations/layer13_attention_weight_max": 37.38822555541992,
      "activations/layer13_attention_weight_min": -33.93830490112305,
      "activations/layer14_attention_weight_max": 40.241024017333984,
      "activations/layer14_attention_weight_min": -36.10145568847656,
      "activations/layer15_attention_weight_max": 37.79661178588867,
      "activations/layer15_attention_weight_min": -36.18000411987305,
      "activations/layer16_attention_weight_max": 31.00588035583496,
      "activations/layer16_attention_weight_min": -27.349035263061523,
      "activations/layer17_attention_weight_max": 29.44731330871582,
      "activations/layer17_attention_weight_min": -28.30632972717285,
      "activations/layer18_attention_weight_max": 31.24700355529785,
      "activations/layer18_attention_weight_min": -27.159006118774414,
      "activations/layer19_attention_weight_max": 32.441680908203125,
      "activations/layer19_attention_weight_min": -30.94425392150879,
      "activations/layer1_attention_weight_max": 17.444156646728516,
      "activations/layer1_attention_weight_min": -14.89935302734375,
      "activations/layer20_attention_weight_max": 26.420974731445312,
      "activations/layer20_attention_weight_min": -25.789810180664062,
      "activations/layer21_attention_weight_max": 25.806150436401367,
      "activations/layer21_attention_weight_min": -25.498199462890625,
      "activations/layer22_attention_weight_max": 38.63904571533203,
      "activations/layer22_attention_weight_min": -34.084861755371094,
      "activations/layer23_attention_weight_max": 29.96095085144043,
      "activations/layer23_attention_weight_min": -26.195295333862305,
      "activations/layer2_attention_weight_max": 34.43980407714844,
      "activations/layer2_attention_weight_min": -35.35014343261719,
      "activations/layer3_attention_weight_max": 104.12911224365234,
      "activations/layer3_attention_weight_min": -103.20069122314453,
      "activations/layer4_attention_weight_max": 107.83373260498047,
      "activations/layer4_attention_weight_min": -109.91316986083984,
      "activations/layer5_attention_weight_max": 50.28803634643555,
      "activations/layer5_attention_weight_min": -60.57748031616211,
      "activations/layer6_attention_weight_max": 41.54905700683594,
      "activations/layer6_attention_weight_min": -45.6953010559082,
      "activations/layer7_attention_weight_max": 94.98526763916016,
      "activations/layer7_attention_weight_min": -86.6829605102539,
      "activations/layer8_attention_weight_max": 43.51260757446289,
      "activations/layer8_attention_weight_min": -44.30801010131836,
      "activations/layer9_attention_weight_max": 36.637474060058594,
      "activations/layer9_attention_weight_min": -36.1576042175293,
      "epoch": 13.77,
      "learning_rate": 6.185416666666667e-05,
      "loss": 2.7645,
      "step": 236900
    },
    {
      "activations/layer0_attention_weight_max": 17.010164260864258,
      "activations/layer0_attention_weight_min": -12.739718437194824,
      "activations/layer10_attention_weight_max": 33.81884765625,
      "activations/layer10_attention_weight_min": -33.92463684082031,
      "activations/layer11_attention_weight_max": 33.98991394042969,
      "activations/layer11_attention_weight_min": -33.326271057128906,
      "activations/layer12_attention_weight_max": 26.744956970214844,
      "activations/layer12_attention_weight_min": -24.66582489013672,
      "activations/layer13_attention_weight_max": 41.40556335449219,
      "activations/layer13_attention_weight_min": -35.10736846923828,
      "activations/layer14_attention_weight_max": 46.121742248535156,
      "activations/layer14_attention_weight_min": -38.067604064941406,
      "activations/layer15_attention_weight_max": 41.3240966796875,
      "activations/layer15_attention_weight_min": -32.46165084838867,
      "activations/layer16_attention_weight_max": 36.177215576171875,
      "activations/layer16_attention_weight_min": -28.830482482910156,
      "activations/layer17_attention_weight_max": 30.97489356994629,
      "activations/layer17_attention_weight_min": -26.112550735473633,
      "activations/layer18_attention_weight_max": 30.61202621459961,
      "activations/layer18_attention_weight_min": -23.095857620239258,
      "activations/layer19_attention_weight_max": 33.33529281616211,
      "activations/layer19_attention_weight_min": -31.150379180908203,
      "activations/layer1_attention_weight_max": 16.85710906982422,
      "activations/layer1_attention_weight_min": -15.179698944091797,
      "activations/layer20_attention_weight_max": 31.419893264770508,
      "activations/layer20_attention_weight_min": -23.481361389160156,
      "activations/layer21_attention_weight_max": 29.113407135009766,
      "activations/layer21_attention_weight_min": -22.133880615234375,
      "activations/layer22_attention_weight_max": 48.73966598510742,
      "activations/layer22_attention_weight_min": -29.21233367919922,
      "activations/layer23_attention_weight_max": 33.84627914428711,
      "activations/layer23_attention_weight_min": -26.288097381591797,
      "activations/layer2_attention_weight_max": 34.97172927856445,
      "activations/layer2_attention_weight_min": -33.501834869384766,
      "activations/layer3_attention_weight_max": 95.75830078125,
      "activations/layer3_attention_weight_min": -100.69141387939453,
      "activations/layer4_attention_weight_max": 110.4589614868164,
      "activations/layer4_attention_weight_min": -113.1217269897461,
      "activations/layer5_attention_weight_max": 52.12087631225586,
      "activations/layer5_attention_weight_min": -63.076080322265625,
      "activations/layer6_attention_weight_max": 45.353614807128906,
      "activations/layer6_attention_weight_min": -46.44346618652344,
      "activations/layer7_attention_weight_max": 93.5246353149414,
      "activations/layer7_attention_weight_min": -89.18334197998047,
      "activations/layer8_attention_weight_max": 39.951927185058594,
      "activations/layer8_attention_weight_min": -40.69361114501953,
      "activations/layer9_attention_weight_max": 32.18484115600586,
      "activations/layer9_attention_weight_min": -34.73350143432617,
      "epoch": 13.77,
      "learning_rate": 6.183522727272727e-05,
      "loss": 2.7709,
      "step": 236950
    },
    {
      "activations/layer0_attention_weight_max": 16.177762985229492,
      "activations/layer0_attention_weight_min": -12.63381290435791,
      "activations/layer10_attention_weight_max": 34.8502082824707,
      "activations/layer10_attention_weight_min": -31.34332847595215,
      "activations/layer11_attention_weight_max": 33.31690216064453,
      "activations/layer11_attention_weight_min": -32.651397705078125,
      "activations/layer12_attention_weight_max": 25.02644157409668,
      "activations/layer12_attention_weight_min": -25.044721603393555,
      "activations/layer13_attention_weight_max": 38.649147033691406,
      "activations/layer13_attention_weight_min": -33.221214294433594,
      "activations/layer14_attention_weight_max": 46.96635818481445,
      "activations/layer14_attention_weight_min": -38.2012825012207,
      "activations/layer15_attention_weight_max": 38.76487350463867,
      "activations/layer15_attention_weight_min": -33.63330841064453,
      "activations/layer16_attention_weight_max": 33.202213287353516,
      "activations/layer16_attention_weight_min": -27.244857788085938,
      "activations/layer17_attention_weight_max": 32.05537796020508,
      "activations/layer17_attention_weight_min": -28.390249252319336,
      "activations/layer18_attention_weight_max": 30.93209457397461,
      "activations/layer18_attention_weight_min": -25.87710189819336,
      "activations/layer19_attention_weight_max": 37.392696380615234,
      "activations/layer19_attention_weight_min": -34.492794036865234,
      "activations/layer1_attention_weight_max": 16.66029930114746,
      "activations/layer1_attention_weight_min": -15.164002418518066,
      "activations/layer20_attention_weight_max": 34.011627197265625,
      "activations/layer20_attention_weight_min": -28.80596351623535,
      "activations/layer21_attention_weight_max": 30.145519256591797,
      "activations/layer21_attention_weight_min": -26.429851531982422,
      "activations/layer22_attention_weight_max": 43.91908264160156,
      "activations/layer22_attention_weight_min": -33.38957595825195,
      "activations/layer23_attention_weight_max": 35.63500213623047,
      "activations/layer23_attention_weight_min": -25.80364227294922,
      "activations/layer2_attention_weight_max": 35.41156768798828,
      "activations/layer2_attention_weight_min": -34.35582733154297,
      "activations/layer3_attention_weight_max": 101.71040344238281,
      "activations/layer3_attention_weight_min": -101.9337387084961,
      "activations/layer4_attention_weight_max": 110.68505096435547,
      "activations/layer4_attention_weight_min": -110.45333099365234,
      "activations/layer5_attention_weight_max": 56.40974044799805,
      "activations/layer5_attention_weight_min": -61.486297607421875,
      "activations/layer6_attention_weight_max": 44.743431091308594,
      "activations/layer6_attention_weight_min": -46.925838470458984,
      "activations/layer7_attention_weight_max": 101.02227783203125,
      "activations/layer7_attention_weight_min": -94.2890625,
      "activations/layer8_attention_weight_max": 42.15732192993164,
      "activations/layer8_attention_weight_min": -42.642578125,
      "activations/layer9_attention_weight_max": 35.07422637939453,
      "activations/layer9_attention_weight_min": -33.394161224365234,
      "epoch": 13.77,
      "learning_rate": 6.181628787878787e-05,
      "loss": 2.7445,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4294,
      "eval_samples_per_second": 509.408,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4294,
      "eval_openwebtext_samples_per_second": 509.408,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9785,
      "eval_wikitext_samples_per_second": 230.476,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_lambada_loss": 2.505859375,
      "eval_lambada_ppl": 12.254085296672228,
      "eval_lambada_runtime": 9.5404,
      "eval_lambada_samples_per_second": 510.358,
      "step": 237000
    },
    {
      "activations/layer0_attention_weight_max": 16.74468994140625,
      "activations/layer0_attention_weight_min": -12.576021194458008,
      "activations/layer10_attention_weight_max": 35.582069396972656,
      "activations/layer10_attention_weight_min": -34.6468505859375,
      "activations/layer11_attention_weight_max": 32.128231048583984,
      "activations/layer11_attention_weight_min": -31.288930892944336,
      "activations/layer12_attention_weight_max": 26.265756607055664,
      "activations/layer12_attention_weight_min": -25.706079483032227,
      "activations/layer13_attention_weight_max": 38.43654251098633,
      "activations/layer13_attention_weight_min": -37.21016311645508,
      "activations/layer14_attention_weight_max": 41.52034378051758,
      "activations/layer14_attention_weight_min": -39.064796447753906,
      "activations/layer15_attention_weight_max": 35.45878601074219,
      "activations/layer15_attention_weight_min": -35.437076568603516,
      "activations/layer16_attention_weight_max": 31.863927841186523,
      "activations/layer16_attention_weight_min": -33.03296661376953,
      "activations/layer17_attention_weight_max": 30.27910804748535,
      "activations/layer17_attention_weight_min": -26.160480499267578,
      "activations/layer18_attention_weight_max": 28.311002731323242,
      "activations/layer18_attention_weight_min": -25.043546676635742,
      "activations/layer19_attention_weight_max": 31.910614013671875,
      "activations/layer19_attention_weight_min": -31.519941329956055,
      "activations/layer1_attention_weight_max": 16.367773056030273,
      "activations/layer1_attention_weight_min": -14.477328300476074,
      "activations/layer20_attention_weight_max": 27.876596450805664,
      "activations/layer20_attention_weight_min": -25.67251205444336,
      "activations/layer21_attention_weight_max": 29.227514266967773,
      "activations/layer21_attention_weight_min": -24.502288818359375,
      "activations/layer22_attention_weight_max": 39.115257263183594,
      "activations/layer22_attention_weight_min": -28.815210342407227,
      "activations/layer23_attention_weight_max": 33.60232925415039,
      "activations/layer23_attention_weight_min": -24.431915283203125,
      "activations/layer2_attention_weight_max": 33.81878662109375,
      "activations/layer2_attention_weight_min": -33.680419921875,
      "activations/layer3_attention_weight_max": 98.83958435058594,
      "activations/layer3_attention_weight_min": -100.54084014892578,
      "activations/layer4_attention_weight_max": 112.63481903076172,
      "activations/layer4_attention_weight_min": -112.5057144165039,
      "activations/layer5_attention_weight_max": 54.476600646972656,
      "activations/layer5_attention_weight_min": -61.910972595214844,
      "activations/layer6_attention_weight_max": 44.012916564941406,
      "activations/layer6_attention_weight_min": -44.37278747558594,
      "activations/layer7_attention_weight_max": 93.13687133789062,
      "activations/layer7_attention_weight_min": -94.93921661376953,
      "activations/layer8_attention_weight_max": 39.83393096923828,
      "activations/layer8_attention_weight_min": -41.167320251464844,
      "activations/layer9_attention_weight_max": 34.193115234375,
      "activations/layer9_attention_weight_min": -34.83444595336914,
      "epoch": 13.77,
      "learning_rate": 6.179734848484849e-05,
      "loss": 2.7598,
      "step": 237050
    },
    {
      "activations/layer0_attention_weight_max": 16.405805587768555,
      "activations/layer0_attention_weight_min": -12.330501556396484,
      "activations/layer10_attention_weight_max": 36.71233367919922,
      "activations/layer10_attention_weight_min": -35.79905700683594,
      "activations/layer11_attention_weight_max": 37.17413330078125,
      "activations/layer11_attention_weight_min": -37.237335205078125,
      "activations/layer12_attention_weight_max": 26.59453010559082,
      "activations/layer12_attention_weight_min": -26.393095016479492,
      "activations/layer13_attention_weight_max": 40.683502197265625,
      "activations/layer13_attention_weight_min": -36.54880142211914,
      "activations/layer14_attention_weight_max": 50.21782302856445,
      "activations/layer14_attention_weight_min": -41.49714279174805,
      "activations/layer15_attention_weight_max": 37.18153381347656,
      "activations/layer15_attention_weight_min": -34.74739456176758,
      "activations/layer16_attention_weight_max": 33.54862594604492,
      "activations/layer16_attention_weight_min": -30.853713989257812,
      "activations/layer17_attention_weight_max": 34.93238830566406,
      "activations/layer17_attention_weight_min": -26.24668312072754,
      "activations/layer18_attention_weight_max": 33.85643768310547,
      "activations/layer18_attention_weight_min": -23.001222610473633,
      "activations/layer19_attention_weight_max": 38.5184211730957,
      "activations/layer19_attention_weight_min": -30.179046630859375,
      "activations/layer1_attention_weight_max": 17.486825942993164,
      "activations/layer1_attention_weight_min": -15.498796463012695,
      "activations/layer20_attention_weight_max": 31.213045120239258,
      "activations/layer20_attention_weight_min": -25.093093872070312,
      "activations/layer21_attention_weight_max": 33.903228759765625,
      "activations/layer21_attention_weight_min": -22.89493751525879,
      "activations/layer22_attention_weight_max": 43.07613754272461,
      "activations/layer22_attention_weight_min": -29.729917526245117,
      "activations/layer23_attention_weight_max": 37.73822021484375,
      "activations/layer23_attention_weight_min": -25.24493980407715,
      "activations/layer2_attention_weight_max": 35.56944274902344,
      "activations/layer2_attention_weight_min": -36.24607467651367,
      "activations/layer3_attention_weight_max": 103.94625854492188,
      "activations/layer3_attention_weight_min": -104.57234954833984,
      "activations/layer4_attention_weight_max": 117.13445281982422,
      "activations/layer4_attention_weight_min": -113.06117248535156,
      "activations/layer5_attention_weight_max": 58.00361633300781,
      "activations/layer5_attention_weight_min": -62.30190658569336,
      "activations/layer6_attention_weight_max": 47.836021423339844,
      "activations/layer6_attention_weight_min": -48.672977447509766,
      "activations/layer7_attention_weight_max": 97.54757690429688,
      "activations/layer7_attention_weight_min": -103.22188568115234,
      "activations/layer8_attention_weight_max": 45.2390022277832,
      "activations/layer8_attention_weight_min": -44.95321273803711,
      "activations/layer9_attention_weight_max": 35.43060302734375,
      "activations/layer9_attention_weight_min": -36.85772705078125,
      "epoch": 13.78,
      "learning_rate": 6.177878787878788e-05,
      "loss": 2.7524,
      "step": 237100
    },
    {
      "activations/layer0_attention_weight_max": 15.721105575561523,
      "activations/layer0_attention_weight_min": -12.82463550567627,
      "activations/layer10_attention_weight_max": 32.94565963745117,
      "activations/layer10_attention_weight_min": -33.46356201171875,
      "activations/layer11_attention_weight_max": 33.0832405090332,
      "activations/layer11_attention_weight_min": -33.5823974609375,
      "activations/layer12_attention_weight_max": 25.13288116455078,
      "activations/layer12_attention_weight_min": -27.426895141601562,
      "activations/layer13_attention_weight_max": 40.329891204833984,
      "activations/layer13_attention_weight_min": -36.97378158569336,
      "activations/layer14_attention_weight_max": 41.50970458984375,
      "activations/layer14_attention_weight_min": -38.66033172607422,
      "activations/layer15_attention_weight_max": 37.28879928588867,
      "activations/layer15_attention_weight_min": -33.09299850463867,
      "activations/layer16_attention_weight_max": 31.551740646362305,
      "activations/layer16_attention_weight_min": -29.49903106689453,
      "activations/layer17_attention_weight_max": 28.79397201538086,
      "activations/layer17_attention_weight_min": -27.310461044311523,
      "activations/layer18_attention_weight_max": 31.712499618530273,
      "activations/layer18_attention_weight_min": -24.882671356201172,
      "activations/layer19_attention_weight_max": 36.545528411865234,
      "activations/layer19_attention_weight_min": -31.280378341674805,
      "activations/layer1_attention_weight_max": 16.55217742919922,
      "activations/layer1_attention_weight_min": -15.549522399902344,
      "activations/layer20_attention_weight_max": 29.912792205810547,
      "activations/layer20_attention_weight_min": -25.403947830200195,
      "activations/layer21_attention_weight_max": 29.65477752685547,
      "activations/layer21_attention_weight_min": -24.398283004760742,
      "activations/layer22_attention_weight_max": 46.32220458984375,
      "activations/layer22_attention_weight_min": -29.166133880615234,
      "activations/layer23_attention_weight_max": 36.20174789428711,
      "activations/layer23_attention_weight_min": -26.85672378540039,
      "activations/layer2_attention_weight_max": 34.45759582519531,
      "activations/layer2_attention_weight_min": -33.34680938720703,
      "activations/layer3_attention_weight_max": 104.05303955078125,
      "activations/layer3_attention_weight_min": -102.61149597167969,
      "activations/layer4_attention_weight_max": 116.00182342529297,
      "activations/layer4_attention_weight_min": -112.42341613769531,
      "activations/layer5_attention_weight_max": 58.449867248535156,
      "activations/layer5_attention_weight_min": -63.37150192260742,
      "activations/layer6_attention_weight_max": 46.7988166809082,
      "activations/layer6_attention_weight_min": -47.955894470214844,
      "activations/layer7_attention_weight_max": 100.48912048339844,
      "activations/layer7_attention_weight_min": -97.69452667236328,
      "activations/layer8_attention_weight_max": 42.108848571777344,
      "activations/layer8_attention_weight_min": -44.04768753051758,
      "activations/layer9_attention_weight_max": 33.92781448364258,
      "activations/layer9_attention_weight_min": -34.51222229003906,
      "epoch": 13.78,
      "learning_rate": 6.176022727272727e-05,
      "loss": 2.7425,
      "step": 237150
    },
    {
      "activations/layer0_attention_weight_max": 16.549821853637695,
      "activations/layer0_attention_weight_min": -12.450100898742676,
      "activations/layer10_attention_weight_max": 35.015480041503906,
      "activations/layer10_attention_weight_min": -35.49005126953125,
      "activations/layer11_attention_weight_max": 34.15932083129883,
      "activations/layer11_attention_weight_min": -35.14139938354492,
      "activations/layer12_attention_weight_max": 28.15038299560547,
      "activations/layer12_attention_weight_min": -27.753938674926758,
      "activations/layer13_attention_weight_max": 39.844154357910156,
      "activations/layer13_attention_weight_min": -37.584346771240234,
      "activations/layer14_attention_weight_max": 42.78211975097656,
      "activations/layer14_attention_weight_min": -42.19877243041992,
      "activations/layer15_attention_weight_max": 39.22828674316406,
      "activations/layer15_attention_weight_min": -35.272857666015625,
      "activations/layer16_attention_weight_max": 31.17552375793457,
      "activations/layer16_attention_weight_min": -29.551237106323242,
      "activations/layer17_attention_weight_max": 30.89910888671875,
      "activations/layer17_attention_weight_min": -26.242136001586914,
      "activations/layer18_attention_weight_max": 30.96647834777832,
      "activations/layer18_attention_weight_min": -23.661447525024414,
      "activations/layer19_attention_weight_max": 33.6142692565918,
      "activations/layer19_attention_weight_min": -30.270511627197266,
      "activations/layer1_attention_weight_max": 18.23837661743164,
      "activations/layer1_attention_weight_min": -16.954238891601562,
      "activations/layer20_attention_weight_max": 27.98845100402832,
      "activations/layer20_attention_weight_min": -24.491111755371094,
      "activations/layer21_attention_weight_max": 27.901153564453125,
      "activations/layer21_attention_weight_min": -24.635177612304688,
      "activations/layer22_attention_weight_max": 42.78882598876953,
      "activations/layer22_attention_weight_min": -28.91605567932129,
      "activations/layer23_attention_weight_max": 35.02538299560547,
      "activations/layer23_attention_weight_min": -23.599842071533203,
      "activations/layer2_attention_weight_max": 35.20659637451172,
      "activations/layer2_attention_weight_min": -38.26297378540039,
      "activations/layer3_attention_weight_max": 104.03466796875,
      "activations/layer3_attention_weight_min": -114.90384674072266,
      "activations/layer4_attention_weight_max": 118.7138442993164,
      "activations/layer4_attention_weight_min": -120.6725082397461,
      "activations/layer5_attention_weight_max": 54.64183044433594,
      "activations/layer5_attention_weight_min": -64.22479248046875,
      "activations/layer6_attention_weight_max": 46.079383850097656,
      "activations/layer6_attention_weight_min": -49.88017272949219,
      "activations/layer7_attention_weight_max": 106.5219955444336,
      "activations/layer7_attention_weight_min": -100.56431579589844,
      "activations/layer8_attention_weight_max": 43.204437255859375,
      "activations/layer8_attention_weight_min": -47.82097244262695,
      "activations/layer9_attention_weight_max": 35.37858581542969,
      "activations/layer9_attention_weight_min": -37.07682418823242,
      "epoch": 13.78,
      "learning_rate": 6.174128787878787e-05,
      "loss": 2.739,
      "step": 237200
    },
    {
      "activations/layer0_attention_weight_max": 15.753911972045898,
      "activations/layer0_attention_weight_min": -12.671220779418945,
      "activations/layer10_attention_weight_max": 35.31416320800781,
      "activations/layer10_attention_weight_min": -35.09442901611328,
      "activations/layer11_attention_weight_max": 34.34394073486328,
      "activations/layer11_attention_weight_min": -36.97467803955078,
      "activations/layer12_attention_weight_max": 38.83355712890625,
      "activations/layer12_attention_weight_min": -26.533649444580078,
      "activations/layer13_attention_weight_max": 46.0675163269043,
      "activations/layer13_attention_weight_min": -36.09872055053711,
      "activations/layer14_attention_weight_max": 47.262733459472656,
      "activations/layer14_attention_weight_min": -37.90525817871094,
      "activations/layer15_attention_weight_max": 39.01560592651367,
      "activations/layer15_attention_weight_min": -33.94173812866211,
      "activations/layer16_attention_weight_max": 32.18827438354492,
      "activations/layer16_attention_weight_min": -28.08128547668457,
      "activations/layer17_attention_weight_max": 33.7709846496582,
      "activations/layer17_attention_weight_min": -26.779632568359375,
      "activations/layer18_attention_weight_max": 32.32463455200195,
      "activations/layer18_attention_weight_min": -24.13266944885254,
      "activations/layer19_attention_weight_max": 36.397762298583984,
      "activations/layer19_attention_weight_min": -33.83828353881836,
      "activations/layer1_attention_weight_max": 17.047773361206055,
      "activations/layer1_attention_weight_min": -15.00804328918457,
      "activations/layer20_attention_weight_max": 32.2532958984375,
      "activations/layer20_attention_weight_min": -27.374500274658203,
      "activations/layer21_attention_weight_max": 32.274452209472656,
      "activations/layer21_attention_weight_min": -25.781278610229492,
      "activations/layer22_attention_weight_max": 47.0405158996582,
      "activations/layer22_attention_weight_min": -32.87112808227539,
      "activations/layer23_attention_weight_max": 36.78370666503906,
      "activations/layer23_attention_weight_min": -28.399532318115234,
      "activations/layer2_attention_weight_max": 38.78322219848633,
      "activations/layer2_attention_weight_min": -37.82194519042969,
      "activations/layer3_attention_weight_max": 103.37725830078125,
      "activations/layer3_attention_weight_min": -105.08795166015625,
      "activations/layer4_attention_weight_max": 119.37700653076172,
      "activations/layer4_attention_weight_min": -116.9049301147461,
      "activations/layer5_attention_weight_max": 56.244667053222656,
      "activations/layer5_attention_weight_min": -62.88691329956055,
      "activations/layer6_attention_weight_max": 47.346317291259766,
      "activations/layer6_attention_weight_min": -48.121150970458984,
      "activations/layer7_attention_weight_max": 107.3765869140625,
      "activations/layer7_attention_weight_min": -93.58293151855469,
      "activations/layer8_attention_weight_max": 44.60488510131836,
      "activations/layer8_attention_weight_min": -43.831783294677734,
      "activations/layer9_attention_weight_max": 34.11315155029297,
      "activations/layer9_attention_weight_min": -34.728660583496094,
      "epoch": 13.79,
      "learning_rate": 6.172234848484848e-05,
      "loss": 2.7511,
      "step": 237250
    },
    {
      "activations/layer0_attention_weight_max": 16.10063362121582,
      "activations/layer0_attention_weight_min": -12.006260871887207,
      "activations/layer10_attention_weight_max": 41.62015914916992,
      "activations/layer10_attention_weight_min": -38.82600784301758,
      "activations/layer11_attention_weight_max": 39.92444610595703,
      "activations/layer11_attention_weight_min": -39.7533073425293,
      "activations/layer12_attention_weight_max": 30.77130699157715,
      "activations/layer12_attention_weight_min": -26.247568130493164,
      "activations/layer13_attention_weight_max": 45.0317268371582,
      "activations/layer13_attention_weight_min": -35.035709381103516,
      "activations/layer14_attention_weight_max": 46.74650573730469,
      "activations/layer14_attention_weight_min": -37.20475387573242,
      "activations/layer15_attention_weight_max": 46.60555648803711,
      "activations/layer15_attention_weight_min": -38.555416107177734,
      "activations/layer16_attention_weight_max": 34.25728988647461,
      "activations/layer16_attention_weight_min": -27.85320472717285,
      "activations/layer17_attention_weight_max": 29.790130615234375,
      "activations/layer17_attention_weight_min": -24.675058364868164,
      "activations/layer18_attention_weight_max": 29.911691665649414,
      "activations/layer18_attention_weight_min": -23.209672927856445,
      "activations/layer19_attention_weight_max": 35.72982406616211,
      "activations/layer19_attention_weight_min": -31.99763298034668,
      "activations/layer1_attention_weight_max": 18.260822296142578,
      "activations/layer1_attention_weight_min": -15.57436466217041,
      "activations/layer20_attention_weight_max": 29.505996704101562,
      "activations/layer20_attention_weight_min": -23.527116775512695,
      "activations/layer21_attention_weight_max": 28.147336959838867,
      "activations/layer21_attention_weight_min": -23.299524307250977,
      "activations/layer22_attention_weight_max": 41.36146926879883,
      "activations/layer22_attention_weight_min": -29.764144897460938,
      "activations/layer23_attention_weight_max": 34.226139068603516,
      "activations/layer23_attention_weight_min": -26.017208099365234,
      "activations/layer2_attention_weight_max": 35.38909149169922,
      "activations/layer2_attention_weight_min": -37.72560119628906,
      "activations/layer3_attention_weight_max": 105.1670913696289,
      "activations/layer3_attention_weight_min": -106.83224487304688,
      "activations/layer4_attention_weight_max": 118.09132385253906,
      "activations/layer4_attention_weight_min": -113.6341781616211,
      "activations/layer5_attention_weight_max": 53.797630310058594,
      "activations/layer5_attention_weight_min": -59.9710807800293,
      "activations/layer6_attention_weight_max": 43.77063751220703,
      "activations/layer6_attention_weight_min": -45.64046859741211,
      "activations/layer7_attention_weight_max": 105.20169830322266,
      "activations/layer7_attention_weight_min": -95.04647064208984,
      "activations/layer8_attention_weight_max": 46.50415802001953,
      "activations/layer8_attention_weight_min": -43.95790481567383,
      "activations/layer9_attention_weight_max": 38.236019134521484,
      "activations/layer9_attention_weight_min": -38.32396697998047,
      "epoch": 13.79,
      "learning_rate": 6.170340909090909e-05,
      "loss": 2.7573,
      "step": 237300
    },
    {
      "activations/layer0_attention_weight_max": 16.324926376342773,
      "activations/layer0_attention_weight_min": -12.274465560913086,
      "activations/layer10_attention_weight_max": 32.236019134521484,
      "activations/layer10_attention_weight_min": -34.13664245605469,
      "activations/layer11_attention_weight_max": 32.15013122558594,
      "activations/layer11_attention_weight_min": -31.604345321655273,
      "activations/layer12_attention_weight_max": 25.571537017822266,
      "activations/layer12_attention_weight_min": -27.341215133666992,
      "activations/layer13_attention_weight_max": 39.25439453125,
      "activations/layer13_attention_weight_min": -36.58546829223633,
      "activations/layer14_attention_weight_max": 42.41331481933594,
      "activations/layer14_attention_weight_min": -41.58163833618164,
      "activations/layer15_attention_weight_max": 41.79209899902344,
      "activations/layer15_attention_weight_min": -35.15425491333008,
      "activations/layer16_attention_weight_max": 30.232192993164062,
      "activations/layer16_attention_weight_min": -29.1947078704834,
      "activations/layer17_attention_weight_max": 30.632585525512695,
      "activations/layer17_attention_weight_min": -26.32081413269043,
      "activations/layer18_attention_weight_max": 32.141178131103516,
      "activations/layer18_attention_weight_min": -24.431615829467773,
      "activations/layer19_attention_weight_max": 43.530662536621094,
      "activations/layer19_attention_weight_min": -32.513267517089844,
      "activations/layer1_attention_weight_max": 17.546396255493164,
      "activations/layer1_attention_weight_min": -15.453448295593262,
      "activations/layer20_attention_weight_max": 36.71115493774414,
      "activations/layer20_attention_weight_min": -26.01827049255371,
      "activations/layer21_attention_weight_max": 36.006675720214844,
      "activations/layer21_attention_weight_min": -25.47292137145996,
      "activations/layer22_attention_weight_max": 48.61526870727539,
      "activations/layer22_attention_weight_min": -30.919952392578125,
      "activations/layer23_attention_weight_max": 36.17171859741211,
      "activations/layer23_attention_weight_min": -24.594144821166992,
      "activations/layer2_attention_weight_max": 35.0452766418457,
      "activations/layer2_attention_weight_min": -36.08203125,
      "activations/layer3_attention_weight_max": 103.52484893798828,
      "activations/layer3_attention_weight_min": -111.15252685546875,
      "activations/layer4_attention_weight_max": 117.68920135498047,
      "activations/layer4_attention_weight_min": -118.14452362060547,
      "activations/layer5_attention_weight_max": 56.89305877685547,
      "activations/layer5_attention_weight_min": -63.705284118652344,
      "activations/layer6_attention_weight_max": 45.03547286987305,
      "activations/layer6_attention_weight_min": -48.35421371459961,
      "activations/layer7_attention_weight_max": 92.36312866210938,
      "activations/layer7_attention_weight_min": -97.85877227783203,
      "activations/layer8_attention_weight_max": 39.472557067871094,
      "activations/layer8_attention_weight_min": -41.67345428466797,
      "activations/layer9_attention_weight_max": 31.30401039123535,
      "activations/layer9_attention_weight_min": -32.95957565307617,
      "epoch": 13.79,
      "learning_rate": 6.168446969696969e-05,
      "loss": 2.7579,
      "step": 237350
    },
    {
      "activations/layer0_attention_weight_max": 16.44713592529297,
      "activations/layer0_attention_weight_min": -12.934432029724121,
      "activations/layer10_attention_weight_max": 31.855627059936523,
      "activations/layer10_attention_weight_min": -31.77507781982422,
      "activations/layer11_attention_weight_max": 32.90980529785156,
      "activations/layer11_attention_weight_min": -31.014236450195312,
      "activations/layer12_attention_weight_max": 26.99091148376465,
      "activations/layer12_attention_weight_min": -24.033390045166016,
      "activations/layer13_attention_weight_max": 39.99983596801758,
      "activations/layer13_attention_weight_min": -33.38846969604492,
      "activations/layer14_attention_weight_max": 49.303653717041016,
      "activations/layer14_attention_weight_min": -38.295345306396484,
      "activations/layer15_attention_weight_max": 39.749324798583984,
      "activations/layer15_attention_weight_min": -33.046016693115234,
      "activations/layer16_attention_weight_max": 34.46646499633789,
      "activations/layer16_attention_weight_min": -29.67254638671875,
      "activations/layer17_attention_weight_max": 34.57939529418945,
      "activations/layer17_attention_weight_min": -25.861669540405273,
      "activations/layer18_attention_weight_max": 33.31926345825195,
      "activations/layer18_attention_weight_min": -25.04764747619629,
      "activations/layer19_attention_weight_max": 43.582054138183594,
      "activations/layer19_attention_weight_min": -32.0913200378418,
      "activations/layer1_attention_weight_max": 15.93780517578125,
      "activations/layer1_attention_weight_min": -14.917837142944336,
      "activations/layer20_attention_weight_max": 34.014007568359375,
      "activations/layer20_attention_weight_min": -27.0512752532959,
      "activations/layer21_attention_weight_max": 34.9955940246582,
      "activations/layer21_attention_weight_min": -25.235015869140625,
      "activations/layer22_attention_weight_max": 49.84554672241211,
      "activations/layer22_attention_weight_min": -32.325008392333984,
      "activations/layer23_attention_weight_max": 37.587158203125,
      "activations/layer23_attention_weight_min": -25.208087921142578,
      "activations/layer2_attention_weight_max": 35.120174407958984,
      "activations/layer2_attention_weight_min": -36.094425201416016,
      "activations/layer3_attention_weight_max": 100.24146270751953,
      "activations/layer3_attention_weight_min": -104.15750885009766,
      "activations/layer4_attention_weight_max": 111.64320373535156,
      "activations/layer4_attention_weight_min": -111.31462860107422,
      "activations/layer5_attention_weight_max": 53.575096130371094,
      "activations/layer5_attention_weight_min": -60.488609313964844,
      "activations/layer6_attention_weight_max": 43.74618148803711,
      "activations/layer6_attention_weight_min": -47.76151657104492,
      "activations/layer7_attention_weight_max": 95.5264663696289,
      "activations/layer7_attention_weight_min": -93.06355285644531,
      "activations/layer8_attention_weight_max": 41.154136657714844,
      "activations/layer8_attention_weight_min": -42.154144287109375,
      "activations/layer9_attention_weight_max": 33.11909866333008,
      "activations/layer9_attention_weight_min": -32.37728500366211,
      "epoch": 13.79,
      "learning_rate": 6.166553030303029e-05,
      "loss": 2.7447,
      "step": 237400
    },
    {
      "activations/layer0_attention_weight_max": 16.607078552246094,
      "activations/layer0_attention_weight_min": -13.301952362060547,
      "activations/layer10_attention_weight_max": 34.501861572265625,
      "activations/layer10_attention_weight_min": -33.055030822753906,
      "activations/layer11_attention_weight_max": 33.49501419067383,
      "activations/layer11_attention_weight_min": -32.57967758178711,
      "activations/layer12_attention_weight_max": 26.856998443603516,
      "activations/layer12_attention_weight_min": -25.707246780395508,
      "activations/layer13_attention_weight_max": 41.05666732788086,
      "activations/layer13_attention_weight_min": -36.60101318359375,
      "activations/layer14_attention_weight_max": 43.782569885253906,
      "activations/layer14_attention_weight_min": -38.01912307739258,
      "activations/layer15_attention_weight_max": 39.82090377807617,
      "activations/layer15_attention_weight_min": -36.222251892089844,
      "activations/layer16_attention_weight_max": 32.61366653442383,
      "activations/layer16_attention_weight_min": -28.622760772705078,
      "activations/layer17_attention_weight_max": 32.34926986694336,
      "activations/layer17_attention_weight_min": -28.6431884765625,
      "activations/layer18_attention_weight_max": 35.46134567260742,
      "activations/layer18_attention_weight_min": -26.860300064086914,
      "activations/layer19_attention_weight_max": 38.8277702331543,
      "activations/layer19_attention_weight_min": -31.414539337158203,
      "activations/layer1_attention_weight_max": 16.937538146972656,
      "activations/layer1_attention_weight_min": -14.654793739318848,
      "activations/layer20_attention_weight_max": 31.753026962280273,
      "activations/layer20_attention_weight_min": -25.51342010498047,
      "activations/layer21_attention_weight_max": 31.574142456054688,
      "activations/layer21_attention_weight_min": -24.589824676513672,
      "activations/layer22_attention_weight_max": 44.348236083984375,
      "activations/layer22_attention_weight_min": -33.54127502441406,
      "activations/layer23_attention_weight_max": 35.369903564453125,
      "activations/layer23_attention_weight_min": -26.304349899291992,
      "activations/layer2_attention_weight_max": 34.96303939819336,
      "activations/layer2_attention_weight_min": -36.05411911010742,
      "activations/layer3_attention_weight_max": 99.63301849365234,
      "activations/layer3_attention_weight_min": -102.45304870605469,
      "activations/layer4_attention_weight_max": 108.6120376586914,
      "activations/layer4_attention_weight_min": -114.7972640991211,
      "activations/layer5_attention_weight_max": 48.817867279052734,
      "activations/layer5_attention_weight_min": -64.37724304199219,
      "activations/layer6_attention_weight_max": 42.87104415893555,
      "activations/layer6_attention_weight_min": -46.14453887939453,
      "activations/layer7_attention_weight_max": 84.5373306274414,
      "activations/layer7_attention_weight_min": -85.61104583740234,
      "activations/layer8_attention_weight_max": 40.13542556762695,
      "activations/layer8_attention_weight_min": -40.86493682861328,
      "activations/layer9_attention_weight_max": 34.994598388671875,
      "activations/layer9_attention_weight_min": -32.80747604370117,
      "epoch": 13.8,
      "learning_rate": 6.16465909090909e-05,
      "loss": 2.7586,
      "step": 237450
    },
    {
      "activations/layer0_attention_weight_max": 16.232528686523438,
      "activations/layer0_attention_weight_min": -12.245321273803711,
      "activations/layer10_attention_weight_max": 34.811424255371094,
      "activations/layer10_attention_weight_min": -34.29408645629883,
      "activations/layer11_attention_weight_max": 35.723167419433594,
      "activations/layer11_attention_weight_min": -34.731239318847656,
      "activations/layer12_attention_weight_max": 25.285236358642578,
      "activations/layer12_attention_weight_min": -28.179309844970703,
      "activations/layer13_attention_weight_max": 40.746620178222656,
      "activations/layer13_attention_weight_min": -36.034912109375,
      "activations/layer14_attention_weight_max": 41.46144104003906,
      "activations/layer14_attention_weight_min": -38.12186813354492,
      "activations/layer15_attention_weight_max": 36.55897521972656,
      "activations/layer15_attention_weight_min": -33.80381393432617,
      "activations/layer16_attention_weight_max": 30.097875595092773,
      "activations/layer16_attention_weight_min": -28.430356979370117,
      "activations/layer17_attention_weight_max": 33.27323532104492,
      "activations/layer17_attention_weight_min": -29.72747802734375,
      "activations/layer18_attention_weight_max": 31.869380950927734,
      "activations/layer18_attention_weight_min": -25.375385284423828,
      "activations/layer19_attention_weight_max": 34.212459564208984,
      "activations/layer19_attention_weight_min": -31.574766159057617,
      "activations/layer1_attention_weight_max": 17.10896873474121,
      "activations/layer1_attention_weight_min": -14.274651527404785,
      "activations/layer20_attention_weight_max": 29.866853713989258,
      "activations/layer20_attention_weight_min": -25.201948165893555,
      "activations/layer21_attention_weight_max": 29.32565689086914,
      "activations/layer21_attention_weight_min": -24.8553409576416,
      "activations/layer22_attention_weight_max": 43.40634536743164,
      "activations/layer22_attention_weight_min": -29.41055679321289,
      "activations/layer23_attention_weight_max": 33.86991500854492,
      "activations/layer23_attention_weight_min": -24.08340072631836,
      "activations/layer2_attention_weight_max": 35.2080078125,
      "activations/layer2_attention_weight_min": -33.3409538269043,
      "activations/layer3_attention_weight_max": 104.5648193359375,
      "activations/layer3_attention_weight_min": -104.14655303955078,
      "activations/layer4_attention_weight_max": 114.17909240722656,
      "activations/layer4_attention_weight_min": -116.57325744628906,
      "activations/layer5_attention_weight_max": 53.08430099487305,
      "activations/layer5_attention_weight_min": -61.39839172363281,
      "activations/layer6_attention_weight_max": 44.43338394165039,
      "activations/layer6_attention_weight_min": -50.47931671142578,
      "activations/layer7_attention_weight_max": 90.61906433105469,
      "activations/layer7_attention_weight_min": -88.21775817871094,
      "activations/layer8_attention_weight_max": 41.55192947387695,
      "activations/layer8_attention_weight_min": -39.821868896484375,
      "activations/layer9_attention_weight_max": 34.558319091796875,
      "activations/layer9_attention_weight_min": -33.86609649658203,
      "epoch": 13.8,
      "learning_rate": 6.16276515151515e-05,
      "loss": 2.7509,
      "step": 237500
    },
    {
      "activations/layer0_attention_weight_max": 16.75379753112793,
      "activations/layer0_attention_weight_min": -12.753690719604492,
      "activations/layer10_attention_weight_max": 35.71674346923828,
      "activations/layer10_attention_weight_min": -35.57196044921875,
      "activations/layer11_attention_weight_max": 35.535308837890625,
      "activations/layer11_attention_weight_min": -34.480377197265625,
      "activations/layer12_attention_weight_max": 28.85747718811035,
      "activations/layer12_attention_weight_min": -27.304777145385742,
      "activations/layer13_attention_weight_max": 46.88624572753906,
      "activations/layer13_attention_weight_min": -36.90028762817383,
      "activations/layer14_attention_weight_max": 55.11793518066406,
      "activations/layer14_attention_weight_min": -39.14128875732422,
      "activations/layer15_attention_weight_max": 44.69950485229492,
      "activations/layer15_attention_weight_min": -38.118202209472656,
      "activations/layer16_attention_weight_max": 35.9886360168457,
      "activations/layer16_attention_weight_min": -28.652496337890625,
      "activations/layer17_attention_weight_max": 34.77082824707031,
      "activations/layer17_attention_weight_min": -30.644250869750977,
      "activations/layer18_attention_weight_max": 34.977569580078125,
      "activations/layer18_attention_weight_min": -29.22516441345215,
      "activations/layer19_attention_weight_max": 41.988590240478516,
      "activations/layer19_attention_weight_min": -34.27349090576172,
      "activations/layer1_attention_weight_max": 17.39271354675293,
      "activations/layer1_attention_weight_min": -13.78199577331543,
      "activations/layer20_attention_weight_max": 37.419246673583984,
      "activations/layer20_attention_weight_min": -27.398731231689453,
      "activations/layer21_attention_weight_max": 34.73442459106445,
      "activations/layer21_attention_weight_min": -27.481021881103516,
      "activations/layer22_attention_weight_max": 47.26699447631836,
      "activations/layer22_attention_weight_min": -34.56372833251953,
      "activations/layer23_attention_weight_max": 38.507598876953125,
      "activations/layer23_attention_weight_min": -27.419189453125,
      "activations/layer2_attention_weight_max": 32.88439178466797,
      "activations/layer2_attention_weight_min": -34.56789779663086,
      "activations/layer3_attention_weight_max": 99.69155883789062,
      "activations/layer3_attention_weight_min": -100.81201934814453,
      "activations/layer4_attention_weight_max": 115.46858978271484,
      "activations/layer4_attention_weight_min": -117.7824478149414,
      "activations/layer5_attention_weight_max": 53.29648971557617,
      "activations/layer5_attention_weight_min": -67.11743927001953,
      "activations/layer6_attention_weight_max": 46.846153259277344,
      "activations/layer6_attention_weight_min": -48.32950973510742,
      "activations/layer7_attention_weight_max": 98.9629898071289,
      "activations/layer7_attention_weight_min": -94.16593170166016,
      "activations/layer8_attention_weight_max": 43.90740203857422,
      "activations/layer8_attention_weight_min": -44.769981384277344,
      "activations/layer9_attention_weight_max": 37.3529052734375,
      "activations/layer9_attention_weight_min": -37.31199264526367,
      "epoch": 13.8,
      "learning_rate": 6.160871212121212e-05,
      "loss": 2.7652,
      "step": 237550
    },
    {
      "activations/layer0_attention_weight_max": 17.717374801635742,
      "activations/layer0_attention_weight_min": -12.354704856872559,
      "activations/layer10_attention_weight_max": 33.47947692871094,
      "activations/layer10_attention_weight_min": -35.6408576965332,
      "activations/layer11_attention_weight_max": 34.44327926635742,
      "activations/layer11_attention_weight_min": -34.12075424194336,
      "activations/layer12_attention_weight_max": 25.649309158325195,
      "activations/layer12_attention_weight_min": -27.73713493347168,
      "activations/layer13_attention_weight_max": 38.615562438964844,
      "activations/layer13_attention_weight_min": -36.27699279785156,
      "activations/layer14_attention_weight_max": 47.569766998291016,
      "activations/layer14_attention_weight_min": -39.62291717529297,
      "activations/layer15_attention_weight_max": 37.65946960449219,
      "activations/layer15_attention_weight_min": -35.518863677978516,
      "activations/layer16_attention_weight_max": 31.69586181640625,
      "activations/layer16_attention_weight_min": -29.292470932006836,
      "activations/layer17_attention_weight_max": 31.299306869506836,
      "activations/layer17_attention_weight_min": -27.54514503479004,
      "activations/layer18_attention_weight_max": 32.914398193359375,
      "activations/layer18_attention_weight_min": -28.204538345336914,
      "activations/layer19_attention_weight_max": 33.953983306884766,
      "activations/layer19_attention_weight_min": -30.96394157409668,
      "activations/layer1_attention_weight_max": 17.209321975708008,
      "activations/layer1_attention_weight_min": -15.017705917358398,
      "activations/layer20_attention_weight_max": 31.312511444091797,
      "activations/layer20_attention_weight_min": -26.0307674407959,
      "activations/layer21_attention_weight_max": 31.211734771728516,
      "activations/layer21_attention_weight_min": -25.14800453186035,
      "activations/layer22_attention_weight_max": 51.0337028503418,
      "activations/layer22_attention_weight_min": -28.718238830566406,
      "activations/layer23_attention_weight_max": 38.42546844482422,
      "activations/layer23_attention_weight_min": -25.436683654785156,
      "activations/layer2_attention_weight_max": 32.170501708984375,
      "activations/layer2_attention_weight_min": -33.18732452392578,
      "activations/layer3_attention_weight_max": 94.9593505859375,
      "activations/layer3_attention_weight_min": -96.58421325683594,
      "activations/layer4_attention_weight_max": 110.73085021972656,
      "activations/layer4_attention_weight_min": -107.77886962890625,
      "activations/layer5_attention_weight_max": 52.76337432861328,
      "activations/layer5_attention_weight_min": -61.228759765625,
      "activations/layer6_attention_weight_max": 46.97153091430664,
      "activations/layer6_attention_weight_min": -47.04672622680664,
      "activations/layer7_attention_weight_max": 93.14417266845703,
      "activations/layer7_attention_weight_min": -92.4589614868164,
      "activations/layer8_attention_weight_max": 40.17031478881836,
      "activations/layer8_attention_weight_min": -44.47111892700195,
      "activations/layer9_attention_weight_max": 35.44916915893555,
      "activations/layer9_attention_weight_min": -37.04563522338867,
      "epoch": 13.81,
      "learning_rate": 6.158977272727272e-05,
      "loss": 2.7453,
      "step": 237600
    },
    {
      "activations/layer0_attention_weight_max": 16.94292449951172,
      "activations/layer0_attention_weight_min": -12.889961242675781,
      "activations/layer10_attention_weight_max": 32.437294006347656,
      "activations/layer10_attention_weight_min": -32.762451171875,
      "activations/layer11_attention_weight_max": 31.23902702331543,
      "activations/layer11_attention_weight_min": -31.314390182495117,
      "activations/layer12_attention_weight_max": 28.83397102355957,
      "activations/layer12_attention_weight_min": -25.773603439331055,
      "activations/layer13_attention_weight_max": 39.7300910949707,
      "activations/layer13_attention_weight_min": -35.55162811279297,
      "activations/layer14_attention_weight_max": 50.004756927490234,
      "activations/layer14_attention_weight_min": -38.66136932373047,
      "activations/layer15_attention_weight_max": 47.84595489501953,
      "activations/layer15_attention_weight_min": -38.45839309692383,
      "activations/layer16_attention_weight_max": 37.15340042114258,
      "activations/layer16_attention_weight_min": -28.543596267700195,
      "activations/layer17_attention_weight_max": 33.135955810546875,
      "activations/layer17_attention_weight_min": -26.691059112548828,
      "activations/layer18_attention_weight_max": 33.1155891418457,
      "activations/layer18_attention_weight_min": -26.00777816772461,
      "activations/layer19_attention_weight_max": 45.69026184082031,
      "activations/layer19_attention_weight_min": -32.49099349975586,
      "activations/layer1_attention_weight_max": 16.557222366333008,
      "activations/layer1_attention_weight_min": -14.249295234680176,
      "activations/layer20_attention_weight_max": 40.656532287597656,
      "activations/layer20_attention_weight_min": -25.178089141845703,
      "activations/layer21_attention_weight_max": 35.35000228881836,
      "activations/layer21_attention_weight_min": -24.1961727142334,
      "activations/layer22_attention_weight_max": 46.21110153198242,
      "activations/layer22_attention_weight_min": -31.21162223815918,
      "activations/layer23_attention_weight_max": 36.505706787109375,
      "activations/layer23_attention_weight_min": -25.166410446166992,
      "activations/layer2_attention_weight_max": 35.43379592895508,
      "activations/layer2_attention_weight_min": -34.813507080078125,
      "activations/layer3_attention_weight_max": 100.42472076416016,
      "activations/layer3_attention_weight_min": -106.80904388427734,
      "activations/layer4_attention_weight_max": 116.39558410644531,
      "activations/layer4_attention_weight_min": -114.18843078613281,
      "activations/layer5_attention_weight_max": 50.76557922363281,
      "activations/layer5_attention_weight_min": -62.87310028076172,
      "activations/layer6_attention_weight_max": 44.442081451416016,
      "activations/layer6_attention_weight_min": -46.26194763183594,
      "activations/layer7_attention_weight_max": 92.2633285522461,
      "activations/layer7_attention_weight_min": -85.32022857666016,
      "activations/layer8_attention_weight_max": 38.49295425415039,
      "activations/layer8_attention_weight_min": -38.84170150756836,
      "activations/layer9_attention_weight_max": 31.855453491210938,
      "activations/layer9_attention_weight_min": -32.36567306518555,
      "epoch": 13.81,
      "learning_rate": 6.157083333333334e-05,
      "loss": 2.7493,
      "step": 237650
    },
    {
      "activations/layer0_attention_weight_max": 16.056116104125977,
      "activations/layer0_attention_weight_min": -12.423032760620117,
      "activations/layer10_attention_weight_max": 34.85208511352539,
      "activations/layer10_attention_weight_min": -36.44672393798828,
      "activations/layer11_attention_weight_max": 34.03795623779297,
      "activations/layer11_attention_weight_min": -34.93179702758789,
      "activations/layer12_attention_weight_max": 31.917207717895508,
      "activations/layer12_attention_weight_min": -27.816909790039062,
      "activations/layer13_attention_weight_max": 39.24320983886719,
      "activations/layer13_attention_weight_min": -38.98341751098633,
      "activations/layer14_attention_weight_max": 44.587833404541016,
      "activations/layer14_attention_weight_min": -39.18552780151367,
      "activations/layer15_attention_weight_max": 38.87733459472656,
      "activations/layer15_attention_weight_min": -35.78778839111328,
      "activations/layer16_attention_weight_max": 30.828508377075195,
      "activations/layer16_attention_weight_min": -27.96541404724121,
      "activations/layer17_attention_weight_max": 35.26240921020508,
      "activations/layer17_attention_weight_min": -29.68511390686035,
      "activations/layer18_attention_weight_max": 30.789213180541992,
      "activations/layer18_attention_weight_min": -26.67876434326172,
      "activations/layer19_attention_weight_max": 36.36838912963867,
      "activations/layer19_attention_weight_min": -32.658966064453125,
      "activations/layer1_attention_weight_max": 16.221359252929688,
      "activations/layer1_attention_weight_min": -16.11085319519043,
      "activations/layer20_attention_weight_max": 30.7351016998291,
      "activations/layer20_attention_weight_min": -26.3262996673584,
      "activations/layer21_attention_weight_max": 32.100284576416016,
      "activations/layer21_attention_weight_min": -25.226299285888672,
      "activations/layer22_attention_weight_max": 44.186607360839844,
      "activations/layer22_attention_weight_min": -33.045650482177734,
      "activations/layer23_attention_weight_max": 36.285728454589844,
      "activations/layer23_attention_weight_min": -28.15993309020996,
      "activations/layer2_attention_weight_max": 35.303340911865234,
      "activations/layer2_attention_weight_min": -34.6740608215332,
      "activations/layer3_attention_weight_max": 101.49598693847656,
      "activations/layer3_attention_weight_min": -103.18502807617188,
      "activations/layer4_attention_weight_max": 119.28167724609375,
      "activations/layer4_attention_weight_min": -117.75322723388672,
      "activations/layer5_attention_weight_max": 53.050376892089844,
      "activations/layer5_attention_weight_min": -63.43003463745117,
      "activations/layer6_attention_weight_max": 47.534324645996094,
      "activations/layer6_attention_weight_min": -50.70112991333008,
      "activations/layer7_attention_weight_max": 93.4200668334961,
      "activations/layer7_attention_weight_min": -97.99273681640625,
      "activations/layer8_attention_weight_max": 44.446533203125,
      "activations/layer8_attention_weight_min": -45.61403274536133,
      "activations/layer9_attention_weight_max": 35.74211120605469,
      "activations/layer9_attention_weight_min": -36.23977279663086,
      "epoch": 13.81,
      "learning_rate": 6.155189393939394e-05,
      "loss": 2.7647,
      "step": 237700
    },
    {
      "activations/layer0_attention_weight_max": 17.089441299438477,
      "activations/layer0_attention_weight_min": -12.389781951904297,
      "activations/layer10_attention_weight_max": 32.57977294921875,
      "activations/layer10_attention_weight_min": -32.82905197143555,
      "activations/layer11_attention_weight_max": 33.69028091430664,
      "activations/layer11_attention_weight_min": -31.888530731201172,
      "activations/layer12_attention_weight_max": 25.12077522277832,
      "activations/layer12_attention_weight_min": -26.39891242980957,
      "activations/layer13_attention_weight_max": 39.75923538208008,
      "activations/layer13_attention_weight_min": -35.63300704956055,
      "activations/layer14_attention_weight_max": 41.52272033691406,
      "activations/layer14_attention_weight_min": -36.89814758300781,
      "activations/layer15_attention_weight_max": 37.60041809082031,
      "activations/layer15_attention_weight_min": -33.61172103881836,
      "activations/layer16_attention_weight_max": 30.03446388244629,
      "activations/layer16_attention_weight_min": -27.538776397705078,
      "activations/layer17_attention_weight_max": 29.51173973083496,
      "activations/layer17_attention_weight_min": -26.70103645324707,
      "activations/layer18_attention_weight_max": 29.83277702331543,
      "activations/layer18_attention_weight_min": -25.017045974731445,
      "activations/layer19_attention_weight_max": 29.550586700439453,
      "activations/layer19_attention_weight_min": -30.91973876953125,
      "activations/layer1_attention_weight_max": 16.274106979370117,
      "activations/layer1_attention_weight_min": -15.32950210571289,
      "activations/layer20_attention_weight_max": 28.080820083618164,
      "activations/layer20_attention_weight_min": -27.06993293762207,
      "activations/layer21_attention_weight_max": 28.856597900390625,
      "activations/layer21_attention_weight_min": -23.89534568786621,
      "activations/layer22_attention_weight_max": 37.16477584838867,
      "activations/layer22_attention_weight_min": -30.005409240722656,
      "activations/layer23_attention_weight_max": 30.832387924194336,
      "activations/layer23_attention_weight_min": -28.10829734802246,
      "activations/layer2_attention_weight_max": 34.820098876953125,
      "activations/layer2_attention_weight_min": -34.6222038269043,
      "activations/layer3_attention_weight_max": 98.64431762695312,
      "activations/layer3_attention_weight_min": -101.03976440429688,
      "activations/layer4_attention_weight_max": 111.9399185180664,
      "activations/layer4_attention_weight_min": -116.34175872802734,
      "activations/layer5_attention_weight_max": 51.5307731628418,
      "activations/layer5_attention_weight_min": -63.29071807861328,
      "activations/layer6_attention_weight_max": 43.47291564941406,
      "activations/layer6_attention_weight_min": -48.066768646240234,
      "activations/layer7_attention_weight_max": 90.40459442138672,
      "activations/layer7_attention_weight_min": -90.1263198852539,
      "activations/layer8_attention_weight_max": 37.61952209472656,
      "activations/layer8_attention_weight_min": -39.520755767822266,
      "activations/layer9_attention_weight_max": 32.53837585449219,
      "activations/layer9_attention_weight_min": -31.57759666442871,
      "epoch": 13.81,
      "learning_rate": 6.153295454545454e-05,
      "loss": 2.7437,
      "step": 237750
    },
    {
      "activations/layer0_attention_weight_max": 15.4351224899292,
      "activations/layer0_attention_weight_min": -12.148886680603027,
      "activations/layer10_attention_weight_max": 32.988006591796875,
      "activations/layer10_attention_weight_min": -32.16961669921875,
      "activations/layer11_attention_weight_max": 32.6612548828125,
      "activations/layer11_attention_weight_min": -31.50528335571289,
      "activations/layer12_attention_weight_max": 27.963993072509766,
      "activations/layer12_attention_weight_min": -26.952014923095703,
      "activations/layer13_attention_weight_max": 43.6132698059082,
      "activations/layer13_attention_weight_min": -35.92360305786133,
      "activations/layer14_attention_weight_max": 44.52107238769531,
      "activations/layer14_attention_weight_min": -38.83158493041992,
      "activations/layer15_attention_weight_max": 39.1799201965332,
      "activations/layer15_attention_weight_min": -32.86509704589844,
      "activations/layer16_attention_weight_max": 33.6035041809082,
      "activations/layer16_attention_weight_min": -29.57244873046875,
      "activations/layer17_attention_weight_max": 31.98657989501953,
      "activations/layer17_attention_weight_min": -26.17484474182129,
      "activations/layer18_attention_weight_max": 32.84889221191406,
      "activations/layer18_attention_weight_min": -24.659515380859375,
      "activations/layer19_attention_weight_max": 36.77527618408203,
      "activations/layer19_attention_weight_min": -30.59711265563965,
      "activations/layer1_attention_weight_max": 16.071088790893555,
      "activations/layer1_attention_weight_min": -16.274658203125,
      "activations/layer20_attention_weight_max": 30.82666015625,
      "activations/layer20_attention_weight_min": -23.920225143432617,
      "activations/layer21_attention_weight_max": 30.65865707397461,
      "activations/layer21_attention_weight_min": -23.448144912719727,
      "activations/layer22_attention_weight_max": 48.0116081237793,
      "activations/layer22_attention_weight_min": -30.364643096923828,
      "activations/layer23_attention_weight_max": 36.275875091552734,
      "activations/layer23_attention_weight_min": -26.071561813354492,
      "activations/layer2_attention_weight_max": 35.79474639892578,
      "activations/layer2_attention_weight_min": -32.611061096191406,
      "activations/layer3_attention_weight_max": 100.48997497558594,
      "activations/layer3_attention_weight_min": -99.50728607177734,
      "activations/layer4_attention_weight_max": 112.84716033935547,
      "activations/layer4_attention_weight_min": -108.57972717285156,
      "activations/layer5_attention_weight_max": 53.24290084838867,
      "activations/layer5_attention_weight_min": -64.64583587646484,
      "activations/layer6_attention_weight_max": 43.18218231201172,
      "activations/layer6_attention_weight_min": -46.02875900268555,
      "activations/layer7_attention_weight_max": 95.49993133544922,
      "activations/layer7_attention_weight_min": -93.0516357421875,
      "activations/layer8_attention_weight_max": 40.84933090209961,
      "activations/layer8_attention_weight_min": -41.844791412353516,
      "activations/layer9_attention_weight_max": 33.32901382446289,
      "activations/layer9_attention_weight_min": -34.05160140991211,
      "epoch": 13.82,
      "learning_rate": 6.151401515151515e-05,
      "loss": 2.7572,
      "step": 237800
    },
    {
      "activations/layer0_attention_weight_max": 16.3797550201416,
      "activations/layer0_attention_weight_min": -12.849360466003418,
      "activations/layer10_attention_weight_max": 35.789764404296875,
      "activations/layer10_attention_weight_min": -34.84257888793945,
      "activations/layer11_attention_weight_max": 36.87223434448242,
      "activations/layer11_attention_weight_min": -34.384498596191406,
      "activations/layer12_attention_weight_max": 27.629074096679688,
      "activations/layer12_attention_weight_min": -26.15715980529785,
      "activations/layer13_attention_weight_max": 41.73193359375,
      "activations/layer13_attention_weight_min": -35.45225524902344,
      "activations/layer14_attention_weight_max": 47.17801284790039,
      "activations/layer14_attention_weight_min": -37.45121383666992,
      "activations/layer15_attention_weight_max": 38.51723861694336,
      "activations/layer15_attention_weight_min": -34.40155029296875,
      "activations/layer16_attention_weight_max": 30.14826202392578,
      "activations/layer16_attention_weight_min": -27.83010482788086,
      "activations/layer17_attention_weight_max": 30.57093048095703,
      "activations/layer17_attention_weight_min": -27.202104568481445,
      "activations/layer18_attention_weight_max": 31.778879165649414,
      "activations/layer18_attention_weight_min": -23.91952133178711,
      "activations/layer19_attention_weight_max": 33.713897705078125,
      "activations/layer19_attention_weight_min": -30.2656307220459,
      "activations/layer1_attention_weight_max": 16.746578216552734,
      "activations/layer1_attention_weight_min": -15.419973373413086,
      "activations/layer20_attention_weight_max": 29.538461685180664,
      "activations/layer20_attention_weight_min": -23.87974739074707,
      "activations/layer21_attention_weight_max": 29.63214111328125,
      "activations/layer21_attention_weight_min": -22.7867431640625,
      "activations/layer22_attention_weight_max": 40.57968521118164,
      "activations/layer22_attention_weight_min": -30.144962310791016,
      "activations/layer23_attention_weight_max": 33.91781997680664,
      "activations/layer23_attention_weight_min": -23.668739318847656,
      "activations/layer2_attention_weight_max": 34.918609619140625,
      "activations/layer2_attention_weight_min": -34.34904479980469,
      "activations/layer3_attention_weight_max": 100.7856674194336,
      "activations/layer3_attention_weight_min": -101.27098083496094,
      "activations/layer4_attention_weight_max": 113.3554916381836,
      "activations/layer4_attention_weight_min": -111.95361328125,
      "activations/layer5_attention_weight_max": 51.502105712890625,
      "activations/layer5_attention_weight_min": -69.02596282958984,
      "activations/layer6_attention_weight_max": 45.064720153808594,
      "activations/layer6_attention_weight_min": -48.699214935302734,
      "activations/layer7_attention_weight_max": 95.36239624023438,
      "activations/layer7_attention_weight_min": -98.67761993408203,
      "activations/layer8_attention_weight_max": 43.1123161315918,
      "activations/layer8_attention_weight_min": -43.9400749206543,
      "activations/layer9_attention_weight_max": 35.72958755493164,
      "activations/layer9_attention_weight_min": -35.9980354309082,
      "epoch": 13.82,
      "learning_rate": 6.149507575757576e-05,
      "loss": 2.7625,
      "step": 237850
    },
    {
      "activations/layer0_attention_weight_max": 15.01038646697998,
      "activations/layer0_attention_weight_min": -12.652050018310547,
      "activations/layer10_attention_weight_max": 36.30596923828125,
      "activations/layer10_attention_weight_min": -34.88008499145508,
      "activations/layer11_attention_weight_max": 35.78316116333008,
      "activations/layer11_attention_weight_min": -35.57233428955078,
      "activations/layer12_attention_weight_max": 38.27873992919922,
      "activations/layer12_attention_weight_min": -26.343111038208008,
      "activations/layer13_attention_weight_max": 42.007423400878906,
      "activations/layer13_attention_weight_min": -37.27275085449219,
      "activations/layer14_attention_weight_max": 46.05879211425781,
      "activations/layer14_attention_weight_min": -41.72106170654297,
      "activations/layer15_attention_weight_max": 38.35951614379883,
      "activations/layer15_attention_weight_min": -35.03321838378906,
      "activations/layer16_attention_weight_max": 31.95532989501953,
      "activations/layer16_attention_weight_min": -29.37396812438965,
      "activations/layer17_attention_weight_max": 34.99675369262695,
      "activations/layer17_attention_weight_min": -28.12754249572754,
      "activations/layer18_attention_weight_max": 33.162384033203125,
      "activations/layer18_attention_weight_min": -26.09360122680664,
      "activations/layer19_attention_weight_max": 39.28233337402344,
      "activations/layer19_attention_weight_min": -33.60880661010742,
      "activations/layer1_attention_weight_max": 16.818090438842773,
      "activations/layer1_attention_weight_min": -15.716450691223145,
      "activations/layer20_attention_weight_max": 35.34452438354492,
      "activations/layer20_attention_weight_min": -27.979711532592773,
      "activations/layer21_attention_weight_max": 35.75257873535156,
      "activations/layer21_attention_weight_min": -26.92100715637207,
      "activations/layer22_attention_weight_max": 46.80635070800781,
      "activations/layer22_attention_weight_min": -33.73414611816406,
      "activations/layer23_attention_weight_max": 35.939239501953125,
      "activations/layer23_attention_weight_min": -25.504741668701172,
      "activations/layer2_attention_weight_max": 33.09499740600586,
      "activations/layer2_attention_weight_min": -34.535423278808594,
      "activations/layer3_attention_weight_max": 102.23469543457031,
      "activations/layer3_attention_weight_min": -102.60672760009766,
      "activations/layer4_attention_weight_max": 120.24259185791016,
      "activations/layer4_attention_weight_min": -119.31632232666016,
      "activations/layer5_attention_weight_max": 53.00619125366211,
      "activations/layer5_attention_weight_min": -65.96260070800781,
      "activations/layer6_attention_weight_max": 47.56063461303711,
      "activations/layer6_attention_weight_min": -49.493675231933594,
      "activations/layer7_attention_weight_max": 95.91033935546875,
      "activations/layer7_attention_weight_min": -96.01924896240234,
      "activations/layer8_attention_weight_max": 41.53462219238281,
      "activations/layer8_attention_weight_min": -44.296321868896484,
      "activations/layer9_attention_weight_max": 34.2363395690918,
      "activations/layer9_attention_weight_min": -34.83296585083008,
      "epoch": 13.82,
      "learning_rate": 6.147613636363636e-05,
      "loss": 2.7488,
      "step": 237900
    },
    {
      "activations/layer0_attention_weight_max": 14.792627334594727,
      "activations/layer0_attention_weight_min": -12.705089569091797,
      "activations/layer10_attention_weight_max": 32.428531646728516,
      "activations/layer10_attention_weight_min": -32.63742446899414,
      "activations/layer11_attention_weight_max": 31.711318969726562,
      "activations/layer11_attention_weight_min": -32.06608200073242,
      "activations/layer12_attention_weight_max": 25.772005081176758,
      "activations/layer12_attention_weight_min": -25.331161499023438,
      "activations/layer13_attention_weight_max": 37.56666564941406,
      "activations/layer13_attention_weight_min": -35.087013244628906,
      "activations/layer14_attention_weight_max": 42.54670333862305,
      "activations/layer14_attention_weight_min": -36.135311126708984,
      "activations/layer15_attention_weight_max": 37.56281280517578,
      "activations/layer15_attention_weight_min": -33.89430236816406,
      "activations/layer16_attention_weight_max": 30.161808013916016,
      "activations/layer16_attention_weight_min": -27.455217361450195,
      "activations/layer17_attention_weight_max": 30.16566276550293,
      "activations/layer17_attention_weight_min": -25.625709533691406,
      "activations/layer18_attention_weight_max": 31.758224487304688,
      "activations/layer18_attention_weight_min": -23.302644729614258,
      "activations/layer19_attention_weight_max": 36.817691802978516,
      "activations/layer19_attention_weight_min": -30.151025772094727,
      "activations/layer1_attention_weight_max": 16.28714370727539,
      "activations/layer1_attention_weight_min": -14.862885475158691,
      "activations/layer20_attention_weight_max": 31.260257720947266,
      "activations/layer20_attention_weight_min": -24.626697540283203,
      "activations/layer21_attention_weight_max": 32.89563751220703,
      "activations/layer21_attention_weight_min": -25.282482147216797,
      "activations/layer22_attention_weight_max": 40.720855712890625,
      "activations/layer22_attention_weight_min": -31.755464553833008,
      "activations/layer23_attention_weight_max": 34.488494873046875,
      "activations/layer23_attention_weight_min": -29.349462509155273,
      "activations/layer2_attention_weight_max": 32.64350891113281,
      "activations/layer2_attention_weight_min": -33.51903533935547,
      "activations/layer3_attention_weight_max": 99.77635955810547,
      "activations/layer3_attention_weight_min": -102.23446655273438,
      "activations/layer4_attention_weight_max": 111.73973846435547,
      "activations/layer4_attention_weight_min": -111.15998840332031,
      "activations/layer5_attention_weight_max": 53.78010177612305,
      "activations/layer5_attention_weight_min": -65.3035888671875,
      "activations/layer6_attention_weight_max": 42.81351852416992,
      "activations/layer6_attention_weight_min": -44.9345817565918,
      "activations/layer7_attention_weight_max": 87.91407012939453,
      "activations/layer7_attention_weight_min": -87.79963684082031,
      "activations/layer8_attention_weight_max": 38.56118392944336,
      "activations/layer8_attention_weight_min": -39.79201126098633,
      "activations/layer9_attention_weight_max": 32.39352035522461,
      "activations/layer9_attention_weight_min": -32.41538619995117,
      "epoch": 13.83,
      "learning_rate": 6.145719696969696e-05,
      "loss": 2.7747,
      "step": 237950
    },
    {
      "activations/layer0_attention_weight_max": 15.792828559875488,
      "activations/layer0_attention_weight_min": -13.58345890045166,
      "activations/layer10_attention_weight_max": 34.65016174316406,
      "activations/layer10_attention_weight_min": -37.27891159057617,
      "activations/layer11_attention_weight_max": 36.27091979980469,
      "activations/layer11_attention_weight_min": -36.90589141845703,
      "activations/layer12_attention_weight_max": 31.041887283325195,
      "activations/layer12_attention_weight_min": -29.143491744995117,
      "activations/layer13_attention_weight_max": 42.299072265625,
      "activations/layer13_attention_weight_min": -38.577392578125,
      "activations/layer14_attention_weight_max": 47.4533576965332,
      "activations/layer14_attention_weight_min": -40.94565963745117,
      "activations/layer15_attention_weight_max": 41.81748962402344,
      "activations/layer15_attention_weight_min": -37.8451042175293,
      "activations/layer16_attention_weight_max": 34.01083755493164,
      "activations/layer16_attention_weight_min": -30.97062110900879,
      "activations/layer17_attention_weight_max": 31.93922233581543,
      "activations/layer17_attention_weight_min": -29.168460845947266,
      "activations/layer18_attention_weight_max": 29.88174057006836,
      "activations/layer18_attention_weight_min": -26.244441986083984,
      "activations/layer19_attention_weight_max": 34.23619842529297,
      "activations/layer19_attention_weight_min": -32.52793502807617,
      "activations/layer1_attention_weight_max": 16.586177825927734,
      "activations/layer1_attention_weight_min": -15.221604347229004,
      "activations/layer20_attention_weight_max": 30.358013153076172,
      "activations/layer20_attention_weight_min": -27.117185592651367,
      "activations/layer21_attention_weight_max": 28.84482765197754,
      "activations/layer21_attention_weight_min": -25.343515396118164,
      "activations/layer22_attention_weight_max": 41.279090881347656,
      "activations/layer22_attention_weight_min": -31.473392486572266,
      "activations/layer23_attention_weight_max": 33.02696228027344,
      "activations/layer23_attention_weight_min": -24.80249786376953,
      "activations/layer2_attention_weight_max": 33.75636291503906,
      "activations/layer2_attention_weight_min": -36.221275329589844,
      "activations/layer3_attention_weight_max": 101.25028991699219,
      "activations/layer3_attention_weight_min": -103.71281433105469,
      "activations/layer4_attention_weight_max": 112.82964324951172,
      "activations/layer4_attention_weight_min": -115.0112533569336,
      "activations/layer5_attention_weight_max": 52.63396453857422,
      "activations/layer5_attention_weight_min": -60.57528305053711,
      "activations/layer6_attention_weight_max": 44.47066879272461,
      "activations/layer6_attention_weight_min": -47.48965072631836,
      "activations/layer7_attention_weight_max": 95.71903991699219,
      "activations/layer7_attention_weight_min": -97.21068572998047,
      "activations/layer8_attention_weight_max": 42.62689971923828,
      "activations/layer8_attention_weight_min": -44.376861572265625,
      "activations/layer9_attention_weight_max": 32.85050964355469,
      "activations/layer9_attention_weight_min": -36.230159759521484,
      "epoch": 13.83,
      "learning_rate": 6.143825757575757e-05,
      "loss": 2.7465,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.8138,
      "eval_samples_per_second": 487.192,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.8138,
      "eval_openwebtext_samples_per_second": 487.192,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.289,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_lambada_loss": 2.42578125,
      "eval_lambada_ppl": 11.311062736078481,
      "eval_lambada_runtime": 9.5551,
      "eval_lambada_samples_per_second": 509.57,
      "step": 238000
    },
    {
      "activations/layer0_attention_weight_max": 15.294853210449219,
      "activations/layer0_attention_weight_min": -12.03907299041748,
      "activations/layer10_attention_weight_max": 34.664581298828125,
      "activations/layer10_attention_weight_min": -34.16822052001953,
      "activations/layer11_attention_weight_max": 33.41205596923828,
      "activations/layer11_attention_weight_min": -30.90119743347168,
      "activations/layer12_attention_weight_max": 25.52549171447754,
      "activations/layer12_attention_weight_min": -25.726036071777344,
      "activations/layer13_attention_weight_max": 40.28633117675781,
      "activations/layer13_attention_weight_min": -32.97481155395508,
      "activations/layer14_attention_weight_max": 42.46855545043945,
      "activations/layer14_attention_weight_min": -36.437355041503906,
      "activations/layer15_attention_weight_max": 38.87945556640625,
      "activations/layer15_attention_weight_min": -33.430633544921875,
      "activations/layer16_attention_weight_max": 29.0797061920166,
      "activations/layer16_attention_weight_min": -27.82682991027832,
      "activations/layer17_attention_weight_max": 30.098453521728516,
      "activations/layer17_attention_weight_min": -25.790103912353516,
      "activations/layer18_attention_weight_max": 29.6002140045166,
      "activations/layer18_attention_weight_min": -25.87084197998047,
      "activations/layer19_attention_weight_max": 31.39546012878418,
      "activations/layer19_attention_weight_min": -31.519758224487305,
      "activations/layer1_attention_weight_max": 16.811603546142578,
      "activations/layer1_attention_weight_min": -14.228619575500488,
      "activations/layer20_attention_weight_max": 28.683513641357422,
      "activations/layer20_attention_weight_min": -25.130531311035156,
      "activations/layer21_attention_weight_max": 27.124553680419922,
      "activations/layer21_attention_weight_min": -24.168546676635742,
      "activations/layer22_attention_weight_max": 38.345088958740234,
      "activations/layer22_attention_weight_min": -29.59776496887207,
      "activations/layer23_attention_weight_max": 30.60072135925293,
      "activations/layer23_attention_weight_min": -24.55921745300293,
      "activations/layer2_attention_weight_max": 34.99882507324219,
      "activations/layer2_attention_weight_min": -33.898292541503906,
      "activations/layer3_attention_weight_max": 103.58873748779297,
      "activations/layer3_attention_weight_min": -107.36421966552734,
      "activations/layer4_attention_weight_max": 115.34329986572266,
      "activations/layer4_attention_weight_min": -121.26643371582031,
      "activations/layer5_attention_weight_max": 55.26447296142578,
      "activations/layer5_attention_weight_min": -64.24847412109375,
      "activations/layer6_attention_weight_max": 48.71848678588867,
      "activations/layer6_attention_weight_min": -49.318946838378906,
      "activations/layer7_attention_weight_max": 100.11922454833984,
      "activations/layer7_attention_weight_min": -96.56462860107422,
      "activations/layer8_attention_weight_max": 45.20166778564453,
      "activations/layer8_attention_weight_min": -43.92524337768555,
      "activations/layer9_attention_weight_max": 34.44333267211914,
      "activations/layer9_attention_weight_min": -35.87863540649414,
      "epoch": 13.83,
      "learning_rate": 6.141931818181818e-05,
      "loss": 2.7735,
      "step": 238050
    },
    {
      "activations/layer0_attention_weight_max": 16.74616241455078,
      "activations/layer0_attention_weight_min": -13.374839782714844,
      "activations/layer10_attention_weight_max": 31.519237518310547,
      "activations/layer10_attention_weight_min": -32.9227180480957,
      "activations/layer11_attention_weight_max": 31.685935974121094,
      "activations/layer11_attention_weight_min": -32.13389587402344,
      "activations/layer12_attention_weight_max": 26.545181274414062,
      "activations/layer12_attention_weight_min": -27.068710327148438,
      "activations/layer13_attention_weight_max": 37.58467483520508,
      "activations/layer13_attention_weight_min": -37.99003982543945,
      "activations/layer14_attention_weight_max": 42.46512985229492,
      "activations/layer14_attention_weight_min": -41.80028533935547,
      "activations/layer15_attention_weight_max": 36.539669036865234,
      "activations/layer15_attention_weight_min": -35.130130767822266,
      "activations/layer16_attention_weight_max": 29.549320220947266,
      "activations/layer16_attention_weight_min": -29.29471778869629,
      "activations/layer17_attention_weight_max": 29.890573501586914,
      "activations/layer17_attention_weight_min": -25.51985740661621,
      "activations/layer18_attention_weight_max": 30.698261260986328,
      "activations/layer18_attention_weight_min": -23.471622467041016,
      "activations/layer19_attention_weight_max": 32.542423248291016,
      "activations/layer19_attention_weight_min": -30.483718872070312,
      "activations/layer1_attention_weight_max": 16.426918029785156,
      "activations/layer1_attention_weight_min": -17.119953155517578,
      "activations/layer20_attention_weight_max": 31.465248107910156,
      "activations/layer20_attention_weight_min": -25.148935317993164,
      "activations/layer21_attention_weight_max": 26.983346939086914,
      "activations/layer21_attention_weight_min": -25.19980812072754,
      "activations/layer22_attention_weight_max": 38.8197021484375,
      "activations/layer22_attention_weight_min": -28.08228302001953,
      "activations/layer23_attention_weight_max": 31.39737319946289,
      "activations/layer23_attention_weight_min": -25.181377410888672,
      "activations/layer2_attention_weight_max": 34.985931396484375,
      "activations/layer2_attention_weight_min": -34.96489715576172,
      "activations/layer3_attention_weight_max": 101.07933044433594,
      "activations/layer3_attention_weight_min": -103.4482650756836,
      "activations/layer4_attention_weight_max": 112.88685607910156,
      "activations/layer4_attention_weight_min": -118.3916244506836,
      "activations/layer5_attention_weight_max": 56.581321716308594,
      "activations/layer5_attention_weight_min": -61.41241455078125,
      "activations/layer6_attention_weight_max": 44.510337829589844,
      "activations/layer6_attention_weight_min": -47.2570686340332,
      "activations/layer7_attention_weight_max": 91.2833480834961,
      "activations/layer7_attention_weight_min": -94.88658142089844,
      "activations/layer8_attention_weight_max": 40.908599853515625,
      "activations/layer8_attention_weight_min": -42.72936248779297,
      "activations/layer9_attention_weight_max": 31.995046615600586,
      "activations/layer9_attention_weight_min": -33.21113967895508,
      "epoch": 13.83,
      "learning_rate": 6.140037878787878e-05,
      "loss": 2.7667,
      "step": 238100
    },
    {
      "activations/layer0_attention_weight_max": 16.447246551513672,
      "activations/layer0_attention_weight_min": -12.385570526123047,
      "activations/layer10_attention_weight_max": 31.33756446838379,
      "activations/layer10_attention_weight_min": -32.18961715698242,
      "activations/layer11_attention_weight_max": 34.241825103759766,
      "activations/layer11_attention_weight_min": -32.17620086669922,
      "activations/layer12_attention_weight_max": 27.22515296936035,
      "activations/layer12_attention_weight_min": -27.30552101135254,
      "activations/layer13_attention_weight_max": 37.648006439208984,
      "activations/layer13_attention_weight_min": -32.715885162353516,
      "activations/layer14_attention_weight_max": 39.040042877197266,
      "activations/layer14_attention_weight_min": -36.0814323425293,
      "activations/layer15_attention_weight_max": 36.86708068847656,
      "activations/layer15_attention_weight_min": -31.3685245513916,
      "activations/layer16_attention_weight_max": 33.217071533203125,
      "activations/layer16_attention_weight_min": -26.093006134033203,
      "activations/layer17_attention_weight_max": 33.60562515258789,
      "activations/layer17_attention_weight_min": -25.021997451782227,
      "activations/layer18_attention_weight_max": 35.473731994628906,
      "activations/layer18_attention_weight_min": -24.20505142211914,
      "activations/layer19_attention_weight_max": 34.7252311706543,
      "activations/layer19_attention_weight_min": -31.737640380859375,
      "activations/layer1_attention_weight_max": 18.371374130249023,
      "activations/layer1_attention_weight_min": -17.35987091064453,
      "activations/layer20_attention_weight_max": 30.268898010253906,
      "activations/layer20_attention_weight_min": -25.04893684387207,
      "activations/layer21_attention_weight_max": 28.626134872436523,
      "activations/layer21_attention_weight_min": -25.003843307495117,
      "activations/layer22_attention_weight_max": 40.57057571411133,
      "activations/layer22_attention_weight_min": -31.267759323120117,
      "activations/layer23_attention_weight_max": 33.16315460205078,
      "activations/layer23_attention_weight_min": -24.217479705810547,
      "activations/layer2_attention_weight_max": 33.94853973388672,
      "activations/layer2_attention_weight_min": -33.83180236816406,
      "activations/layer3_attention_weight_max": 99.95331573486328,
      "activations/layer3_attention_weight_min": -99.32093048095703,
      "activations/layer4_attention_weight_max": 113.38358306884766,
      "activations/layer4_attention_weight_min": -116.6114730834961,
      "activations/layer5_attention_weight_max": 53.12781524658203,
      "activations/layer5_attention_weight_min": -60.5814323425293,
      "activations/layer6_attention_weight_max": 44.132789611816406,
      "activations/layer6_attention_weight_min": -45.9207878112793,
      "activations/layer7_attention_weight_max": 91.87324523925781,
      "activations/layer7_attention_weight_min": -89.87342071533203,
      "activations/layer8_attention_weight_max": 40.189083099365234,
      "activations/layer8_attention_weight_min": -39.40266036987305,
      "activations/layer9_attention_weight_max": 33.47811508178711,
      "activations/layer9_attention_weight_min": -32.19575119018555,
      "epoch": 13.84,
      "learning_rate": 6.138143939393938e-05,
      "loss": 2.7546,
      "step": 238150
    },
    {
      "activations/layer0_attention_weight_max": 15.665703773498535,
      "activations/layer0_attention_weight_min": -13.408003807067871,
      "activations/layer10_attention_weight_max": 32.57787322998047,
      "activations/layer10_attention_weight_min": -33.49286651611328,
      "activations/layer11_attention_weight_max": 32.76202392578125,
      "activations/layer11_attention_weight_min": -32.48518371582031,
      "activations/layer12_attention_weight_max": 24.418521881103516,
      "activations/layer12_attention_weight_min": -26.290481567382812,
      "activations/layer13_attention_weight_max": 36.38957214355469,
      "activations/layer13_attention_weight_min": -35.33888244628906,
      "activations/layer14_attention_weight_max": 39.73945999145508,
      "activations/layer14_attention_weight_min": -37.22368621826172,
      "activations/layer15_attention_weight_max": 35.27635955810547,
      "activations/layer15_attention_weight_min": -32.17082595825195,
      "activations/layer16_attention_weight_max": 29.937721252441406,
      "activations/layer16_attention_weight_min": -26.92361068725586,
      "activations/layer17_attention_weight_max": 28.862192153930664,
      "activations/layer17_attention_weight_min": -25.08379554748535,
      "activations/layer18_attention_weight_max": 29.072383880615234,
      "activations/layer18_attention_weight_min": -23.98666000366211,
      "activations/layer19_attention_weight_max": 33.88398361206055,
      "activations/layer19_attention_weight_min": -29.408634185791016,
      "activations/layer1_attention_weight_max": 17.08571434020996,
      "activations/layer1_attention_weight_min": -15.01402759552002,
      "activations/layer20_attention_weight_max": 28.431438446044922,
      "activations/layer20_attention_weight_min": -22.726760864257812,
      "activations/layer21_attention_weight_max": 25.703079223632812,
      "activations/layer21_attention_weight_min": -22.3950252532959,
      "activations/layer22_attention_weight_max": 41.12497329711914,
      "activations/layer22_attention_weight_min": -27.496986389160156,
      "activations/layer23_attention_weight_max": 32.29281997680664,
      "activations/layer23_attention_weight_min": -25.386463165283203,
      "activations/layer2_attention_weight_max": 34.290626525878906,
      "activations/layer2_attention_weight_min": -33.613563537597656,
      "activations/layer3_attention_weight_max": 103.08676147460938,
      "activations/layer3_attention_weight_min": -103.05775451660156,
      "activations/layer4_attention_weight_max": 114.2979736328125,
      "activations/layer4_attention_weight_min": -113.54779052734375,
      "activations/layer5_attention_weight_max": 54.617671966552734,
      "activations/layer5_attention_weight_min": -62.766021728515625,
      "activations/layer6_attention_weight_max": 45.16004943847656,
      "activations/layer6_attention_weight_min": -47.52833938598633,
      "activations/layer7_attention_weight_max": 90.03990173339844,
      "activations/layer7_attention_weight_min": -96.42102813720703,
      "activations/layer8_attention_weight_max": 41.6865119934082,
      "activations/layer8_attention_weight_min": -45.145050048828125,
      "activations/layer9_attention_weight_max": 32.5915641784668,
      "activations/layer9_attention_weight_min": -33.69734573364258,
      "epoch": 13.84,
      "learning_rate": 6.13625e-05,
      "loss": 2.7543,
      "step": 238200
    },
    {
      "activations/layer0_attention_weight_max": 16.115480422973633,
      "activations/layer0_attention_weight_min": -12.180033683776855,
      "activations/layer10_attention_weight_max": 35.23402404785156,
      "activations/layer10_attention_weight_min": -32.298309326171875,
      "activations/layer11_attention_weight_max": 35.198265075683594,
      "activations/layer11_attention_weight_min": -33.15949249267578,
      "activations/layer12_attention_weight_max": 23.90578842163086,
      "activations/layer12_attention_weight_min": -24.03412437438965,
      "activations/layer13_attention_weight_max": 37.852169036865234,
      "activations/layer13_attention_weight_min": -33.124794006347656,
      "activations/layer14_attention_weight_max": 42.80910110473633,
      "activations/layer14_attention_weight_min": -37.88700866699219,
      "activations/layer15_attention_weight_max": 36.44974136352539,
      "activations/layer15_attention_weight_min": -32.895328521728516,
      "activations/layer16_attention_weight_max": 34.31435775756836,
      "activations/layer16_attention_weight_min": -30.658727645874023,
      "activations/layer17_attention_weight_max": 29.866119384765625,
      "activations/layer17_attention_weight_min": -26.42156410217285,
      "activations/layer18_attention_weight_max": 33.01543426513672,
      "activations/layer18_attention_weight_min": -24.998722076416016,
      "activations/layer19_attention_weight_max": 33.56016540527344,
      "activations/layer19_attention_weight_min": -32.1641960144043,
      "activations/layer1_attention_weight_max": 16.739883422851562,
      "activations/layer1_attention_weight_min": -15.217866897583008,
      "activations/layer20_attention_weight_max": 29.5244197845459,
      "activations/layer20_attention_weight_min": -27.266674041748047,
      "activations/layer21_attention_weight_max": 30.888051986694336,
      "activations/layer21_attention_weight_min": -26.412559509277344,
      "activations/layer22_attention_weight_max": 48.00496292114258,
      "activations/layer22_attention_weight_min": -32.25448989868164,
      "activations/layer23_attention_weight_max": 34.7158203125,
      "activations/layer23_attention_weight_min": -25.678741455078125,
      "activations/layer2_attention_weight_max": 35.120033264160156,
      "activations/layer2_attention_weight_min": -34.896339416503906,
      "activations/layer3_attention_weight_max": 105.8789291381836,
      "activations/layer3_attention_weight_min": -104.54119873046875,
      "activations/layer4_attention_weight_max": 117.19127655029297,
      "activations/layer4_attention_weight_min": -115.572021484375,
      "activations/layer5_attention_weight_max": 53.738609313964844,
      "activations/layer5_attention_weight_min": -63.42727279663086,
      "activations/layer6_attention_weight_max": 46.92858123779297,
      "activations/layer6_attention_weight_min": -49.77289581298828,
      "activations/layer7_attention_weight_max": 89.67214965820312,
      "activations/layer7_attention_weight_min": -91.69155883789062,
      "activations/layer8_attention_weight_max": 39.5543098449707,
      "activations/layer8_attention_weight_min": -40.235870361328125,
      "activations/layer9_attention_weight_max": 35.09649658203125,
      "activations/layer9_attention_weight_min": -33.675132751464844,
      "epoch": 13.84,
      "learning_rate": 6.13435606060606e-05,
      "loss": 2.749,
      "step": 238250
    },
    {
      "activations/layer0_attention_weight_max": 17.045818328857422,
      "activations/layer0_attention_weight_min": -13.16898250579834,
      "activations/layer10_attention_weight_max": 32.88637924194336,
      "activations/layer10_attention_weight_min": -33.73053741455078,
      "activations/layer11_attention_weight_max": 33.63054275512695,
      "activations/layer11_attention_weight_min": -32.48261642456055,
      "activations/layer12_attention_weight_max": 25.053802490234375,
      "activations/layer12_attention_weight_min": -27.423730850219727,
      "activations/layer13_attention_weight_max": 39.146141052246094,
      "activations/layer13_attention_weight_min": -33.67323303222656,
      "activations/layer14_attention_weight_max": 39.318946838378906,
      "activations/layer14_attention_weight_min": -36.14129638671875,
      "activations/layer15_attention_weight_max": 38.981021881103516,
      "activations/layer15_attention_weight_min": -33.0350456237793,
      "activations/layer16_attention_weight_max": 28.628408432006836,
      "activations/layer16_attention_weight_min": -27.049251556396484,
      "activations/layer17_attention_weight_max": 31.074026107788086,
      "activations/layer17_attention_weight_min": -27.01481056213379,
      "activations/layer18_attention_weight_max": 32.864158630371094,
      "activations/layer18_attention_weight_min": -24.375057220458984,
      "activations/layer19_attention_weight_max": 31.81659507751465,
      "activations/layer19_attention_weight_min": -30.958724975585938,
      "activations/layer1_attention_weight_max": 17.13373374938965,
      "activations/layer1_attention_weight_min": -17.524707794189453,
      "activations/layer20_attention_weight_max": 27.22002410888672,
      "activations/layer20_attention_weight_min": -26.867984771728516,
      "activations/layer21_attention_weight_max": 28.29262924194336,
      "activations/layer21_attention_weight_min": -23.647550582885742,
      "activations/layer22_attention_weight_max": 38.6271858215332,
      "activations/layer22_attention_weight_min": -28.046295166015625,
      "activations/layer23_attention_weight_max": 31.7386531829834,
      "activations/layer23_attention_weight_min": -25.35756492614746,
      "activations/layer2_attention_weight_max": 35.090797424316406,
      "activations/layer2_attention_weight_min": -32.52381896972656,
      "activations/layer3_attention_weight_max": 100.22004699707031,
      "activations/layer3_attention_weight_min": -102.1349105834961,
      "activations/layer4_attention_weight_max": 110.08379364013672,
      "activations/layer4_attention_weight_min": -114.65462493896484,
      "activations/layer5_attention_weight_max": 51.36318588256836,
      "activations/layer5_attention_weight_min": -63.023345947265625,
      "activations/layer6_attention_weight_max": 43.24501419067383,
      "activations/layer6_attention_weight_min": -46.07741928100586,
      "activations/layer7_attention_weight_max": 94.46459197998047,
      "activations/layer7_attention_weight_min": -89.6730728149414,
      "activations/layer8_attention_weight_max": 40.41282653808594,
      "activations/layer8_attention_weight_min": -42.184452056884766,
      "activations/layer9_attention_weight_max": 34.00870895385742,
      "activations/layer9_attention_weight_min": -33.10645294189453,
      "epoch": 13.85,
      "learning_rate": 6.132462121212121e-05,
      "loss": 2.7534,
      "step": 238300
    },
    {
      "activations/layer0_attention_weight_max": 16.09511375427246,
      "activations/layer0_attention_weight_min": -12.988245010375977,
      "activations/layer10_attention_weight_max": 30.044384002685547,
      "activations/layer10_attention_weight_min": -30.951366424560547,
      "activations/layer11_attention_weight_max": 32.58211135864258,
      "activations/layer11_attention_weight_min": -32.197593688964844,
      "activations/layer12_attention_weight_max": 29.90349006652832,
      "activations/layer12_attention_weight_min": -25.373531341552734,
      "activations/layer13_attention_weight_max": 41.28052520751953,
      "activations/layer13_attention_weight_min": -34.64952087402344,
      "activations/layer14_attention_weight_max": 45.91869354248047,
      "activations/layer14_attention_weight_min": -37.94325256347656,
      "activations/layer15_attention_weight_max": 38.06303405761719,
      "activations/layer15_attention_weight_min": -35.34855270385742,
      "activations/layer16_attention_weight_max": 30.35265350341797,
      "activations/layer16_attention_weight_min": -29.165843963623047,
      "activations/layer17_attention_weight_max": 30.65191078186035,
      "activations/layer17_attention_weight_min": -26.921953201293945,
      "activations/layer18_attention_weight_max": 30.711349487304688,
      "activations/layer18_attention_weight_min": -25.51783561706543,
      "activations/layer19_attention_weight_max": 34.634071350097656,
      "activations/layer19_attention_weight_min": -34.151859283447266,
      "activations/layer1_attention_weight_max": 16.20848274230957,
      "activations/layer1_attention_weight_min": -15.720541954040527,
      "activations/layer20_attention_weight_max": 33.29659652709961,
      "activations/layer20_attention_weight_min": -25.782621383666992,
      "activations/layer21_attention_weight_max": 32.58965301513672,
      "activations/layer21_attention_weight_min": -25.091434478759766,
      "activations/layer22_attention_weight_max": 44.56094741821289,
      "activations/layer22_attention_weight_min": -33.74734115600586,
      "activations/layer23_attention_weight_max": 33.36349868774414,
      "activations/layer23_attention_weight_min": -26.02979278564453,
      "activations/layer2_attention_weight_max": 32.6806640625,
      "activations/layer2_attention_weight_min": -33.213966369628906,
      "activations/layer3_attention_weight_max": 97.81556701660156,
      "activations/layer3_attention_weight_min": -98.39299774169922,
      "activations/layer4_attention_weight_max": 111.52717590332031,
      "activations/layer4_attention_weight_min": -114.69515228271484,
      "activations/layer5_attention_weight_max": 51.87638473510742,
      "activations/layer5_attention_weight_min": -60.727447509765625,
      "activations/layer6_attention_weight_max": 42.4288215637207,
      "activations/layer6_attention_weight_min": -46.28422164916992,
      "activations/layer7_attention_weight_max": 90.02202606201172,
      "activations/layer7_attention_weight_min": -85.7417984008789,
      "activations/layer8_attention_weight_max": 39.132835388183594,
      "activations/layer8_attention_weight_min": -42.10447311401367,
      "activations/layer9_attention_weight_max": 29.148508071899414,
      "activations/layer9_attention_weight_min": -32.9697151184082,
      "epoch": 13.85,
      "learning_rate": 6.130568181818181e-05,
      "loss": 2.746,
      "step": 238350
    },
    {
      "activations/layer0_attention_weight_max": 15.520611763000488,
      "activations/layer0_attention_weight_min": -12.886617660522461,
      "activations/layer10_attention_weight_max": 32.73051834106445,
      "activations/layer10_attention_weight_min": -32.42912673950195,
      "activations/layer11_attention_weight_max": 32.07497787475586,
      "activations/layer11_attention_weight_min": -32.02353286743164,
      "activations/layer12_attention_weight_max": 25.513935089111328,
      "activations/layer12_attention_weight_min": -25.172441482543945,
      "activations/layer13_attention_weight_max": 36.705440521240234,
      "activations/layer13_attention_weight_min": -34.43473434448242,
      "activations/layer14_attention_weight_max": 43.820743560791016,
      "activations/layer14_attention_weight_min": -36.794830322265625,
      "activations/layer15_attention_weight_max": 35.947811126708984,
      "activations/layer15_attention_weight_min": -32.02583694458008,
      "activations/layer16_attention_weight_max": 31.47486114501953,
      "activations/layer16_attention_weight_min": -28.30601692199707,
      "activations/layer17_attention_weight_max": 33.429283142089844,
      "activations/layer17_attention_weight_min": -26.04001235961914,
      "activations/layer18_attention_weight_max": 32.193519592285156,
      "activations/layer18_attention_weight_min": -25.001737594604492,
      "activations/layer19_attention_weight_max": 31.69013786315918,
      "activations/layer19_attention_weight_min": -30.951887130737305,
      "activations/layer1_attention_weight_max": 15.551280975341797,
      "activations/layer1_attention_weight_min": -14.964343070983887,
      "activations/layer20_attention_weight_max": 29.565141677856445,
      "activations/layer20_attention_weight_min": -26.013280868530273,
      "activations/layer21_attention_weight_max": 29.138504028320312,
      "activations/layer21_attention_weight_min": -24.856456756591797,
      "activations/layer22_attention_weight_max": 46.03792190551758,
      "activations/layer22_attention_weight_min": -32.29669189453125,
      "activations/layer23_attention_weight_max": 32.80745315551758,
      "activations/layer23_attention_weight_min": -26.088050842285156,
      "activations/layer2_attention_weight_max": 33.779449462890625,
      "activations/layer2_attention_weight_min": -34.15795135498047,
      "activations/layer3_attention_weight_max": 103.1446762084961,
      "activations/layer3_attention_weight_min": -100.58492279052734,
      "activations/layer4_attention_weight_max": 115.35139465332031,
      "activations/layer4_attention_weight_min": -113.29450988769531,
      "activations/layer5_attention_weight_max": 52.95083236694336,
      "activations/layer5_attention_weight_min": -64.32440948486328,
      "activations/layer6_attention_weight_max": 46.67454528808594,
      "activations/layer6_attention_weight_min": -48.000247955322266,
      "activations/layer7_attention_weight_max": 99.7304916381836,
      "activations/layer7_attention_weight_min": -93.05237579345703,
      "activations/layer8_attention_weight_max": 44.296241760253906,
      "activations/layer8_attention_weight_min": -43.44227981567383,
      "activations/layer9_attention_weight_max": 34.850154876708984,
      "activations/layer9_attention_weight_min": -34.49653625488281,
      "epoch": 13.85,
      "learning_rate": 6.128712121212122e-05,
      "loss": 2.7618,
      "step": 238400
    },
    {
      "activations/layer0_attention_weight_max": 16.9012393951416,
      "activations/layer0_attention_weight_min": -13.157593727111816,
      "activations/layer10_attention_weight_max": 34.904720306396484,
      "activations/layer10_attention_weight_min": -34.578102111816406,
      "activations/layer11_attention_weight_max": 37.37295913696289,
      "activations/layer11_attention_weight_min": -34.33369827270508,
      "activations/layer12_attention_weight_max": 27.326974868774414,
      "activations/layer12_attention_weight_min": -26.486127853393555,
      "activations/layer13_attention_weight_max": 37.51575469970703,
      "activations/layer13_attention_weight_min": -36.28720474243164,
      "activations/layer14_attention_weight_max": 41.091068267822266,
      "activations/layer14_attention_weight_min": -38.2329216003418,
      "activations/layer15_attention_weight_max": 38.94444274902344,
      "activations/layer15_attention_weight_min": -33.3150634765625,
      "activations/layer16_attention_weight_max": 29.494138717651367,
      "activations/layer16_attention_weight_min": -27.29850196838379,
      "activations/layer17_attention_weight_max": 30.39039421081543,
      "activations/layer17_attention_weight_min": -25.92612648010254,
      "activations/layer18_attention_weight_max": 28.535892486572266,
      "activations/layer18_attention_weight_min": -23.698516845703125,
      "activations/layer19_attention_weight_max": 30.496292114257812,
      "activations/layer19_attention_weight_min": -32.816158294677734,
      "activations/layer1_attention_weight_max": 16.06084442138672,
      "activations/layer1_attention_weight_min": -14.563854217529297,
      "activations/layer20_attention_weight_max": 27.817291259765625,
      "activations/layer20_attention_weight_min": -23.866085052490234,
      "activations/layer21_attention_weight_max": 27.29740333557129,
      "activations/layer21_attention_weight_min": -25.55417251586914,
      "activations/layer22_attention_weight_max": 38.935176849365234,
      "activations/layer22_attention_weight_min": -29.58916473388672,
      "activations/layer23_attention_weight_max": 31.922134399414062,
      "activations/layer23_attention_weight_min": -23.653091430664062,
      "activations/layer2_attention_weight_max": 32.67718505859375,
      "activations/layer2_attention_weight_min": -33.75663757324219,
      "activations/layer3_attention_weight_max": 98.84982299804688,
      "activations/layer3_attention_weight_min": -99.5094985961914,
      "activations/layer4_attention_weight_max": 109.0321273803711,
      "activations/layer4_attention_weight_min": -110.7883071899414,
      "activations/layer5_attention_weight_max": 59.382835388183594,
      "activations/layer5_attention_weight_min": -59.16415023803711,
      "activations/layer6_attention_weight_max": 44.166690826416016,
      "activations/layer6_attention_weight_min": -45.943565368652344,
      "activations/layer7_attention_weight_max": 86.25762176513672,
      "activations/layer7_attention_weight_min": -90.46359252929688,
      "activations/layer8_attention_weight_max": 41.78839874267578,
      "activations/layer8_attention_weight_min": -42.27312088012695,
      "activations/layer9_attention_weight_max": 32.64888381958008,
      "activations/layer9_attention_weight_min": -34.4302864074707,
      "epoch": 13.86,
      "learning_rate": 6.126818181818182e-05,
      "loss": 2.7791,
      "step": 238450
    },
    {
      "activations/layer0_attention_weight_max": 15.61627197265625,
      "activations/layer0_attention_weight_min": -12.314924240112305,
      "activations/layer10_attention_weight_max": 33.5458984375,
      "activations/layer10_attention_weight_min": -33.816322326660156,
      "activations/layer11_attention_weight_max": 35.630126953125,
      "activations/layer11_attention_weight_min": -33.0069694519043,
      "activations/layer12_attention_weight_max": 26.918764114379883,
      "activations/layer12_attention_weight_min": -26.40915298461914,
      "activations/layer13_attention_weight_max": 42.1424560546875,
      "activations/layer13_attention_weight_min": -35.073081970214844,
      "activations/layer14_attention_weight_max": 43.08515167236328,
      "activations/layer14_attention_weight_min": -38.25271987915039,
      "activations/layer15_attention_weight_max": 37.3527946472168,
      "activations/layer15_attention_weight_min": -35.27908706665039,
      "activations/layer16_attention_weight_max": 28.320159912109375,
      "activations/layer16_attention_weight_min": -26.48070526123047,
      "activations/layer17_attention_weight_max": 29.35736083984375,
      "activations/layer17_attention_weight_min": -26.39866065979004,
      "activations/layer18_attention_weight_max": 29.21613883972168,
      "activations/layer18_attention_weight_min": -24.25091552734375,
      "activations/layer19_attention_weight_max": 34.9794921875,
      "activations/layer19_attention_weight_min": -30.4168758392334,
      "activations/layer1_attention_weight_max": 17.394287109375,
      "activations/layer1_attention_weight_min": -13.6325101852417,
      "activations/layer20_attention_weight_max": 29.442344665527344,
      "activations/layer20_attention_weight_min": -26.621858596801758,
      "activations/layer21_attention_weight_max": 31.466018676757812,
      "activations/layer21_attention_weight_min": -24.5003604888916,
      "activations/layer22_attention_weight_max": 41.09163284301758,
      "activations/layer22_attention_weight_min": -29.848644256591797,
      "activations/layer23_attention_weight_max": 34.95224380493164,
      "activations/layer23_attention_weight_min": -27.09161376953125,
      "activations/layer2_attention_weight_max": 31.712303161621094,
      "activations/layer2_attention_weight_min": -30.929500579833984,
      "activations/layer3_attention_weight_max": 99.58708953857422,
      "activations/layer3_attention_weight_min": -99.2835922241211,
      "activations/layer4_attention_weight_max": 115.7291259765625,
      "activations/layer4_attention_weight_min": -114.64668273925781,
      "activations/layer5_attention_weight_max": 56.0804443359375,
      "activations/layer5_attention_weight_min": -63.72007751464844,
      "activations/layer6_attention_weight_max": 45.389469146728516,
      "activations/layer6_attention_weight_min": -47.37273406982422,
      "activations/layer7_attention_weight_max": 95.0745620727539,
      "activations/layer7_attention_weight_min": -99.4501724243164,
      "activations/layer8_attention_weight_max": 44.277435302734375,
      "activations/layer8_attention_weight_min": -44.816219329833984,
      "activations/layer9_attention_weight_max": 34.77830123901367,
      "activations/layer9_attention_weight_min": -34.98698806762695,
      "epoch": 13.86,
      "learning_rate": 6.124924242424242e-05,
      "loss": 2.7729,
      "step": 238500
    },
    {
      "activations/layer0_attention_weight_max": 15.826648712158203,
      "activations/layer0_attention_weight_min": -13.406036376953125,
      "activations/layer10_attention_weight_max": 36.1563606262207,
      "activations/layer10_attention_weight_min": -33.69536590576172,
      "activations/layer11_attention_weight_max": 33.97996139526367,
      "activations/layer11_attention_weight_min": -34.38728713989258,
      "activations/layer12_attention_weight_max": 24.933530807495117,
      "activations/layer12_attention_weight_min": -25.853357315063477,
      "activations/layer13_attention_weight_max": 39.74144744873047,
      "activations/layer13_attention_weight_min": -35.3493537902832,
      "activations/layer14_attention_weight_max": 45.36678695678711,
      "activations/layer14_attention_weight_min": -38.7520866394043,
      "activations/layer15_attention_weight_max": 36.492469787597656,
      "activations/layer15_attention_weight_min": -33.351173400878906,
      "activations/layer16_attention_weight_max": 30.190031051635742,
      "activations/layer16_attention_weight_min": -26.33728790283203,
      "activations/layer17_attention_weight_max": 28.363370895385742,
      "activations/layer17_attention_weight_min": -25.993698120117188,
      "activations/layer18_attention_weight_max": 29.903987884521484,
      "activations/layer18_attention_weight_min": -24.550203323364258,
      "activations/layer19_attention_weight_max": 32.147159576416016,
      "activations/layer19_attention_weight_min": -32.38539505004883,
      "activations/layer1_attention_weight_max": 17.147323608398438,
      "activations/layer1_attention_weight_min": -14.722228050231934,
      "activations/layer20_attention_weight_max": 29.183853149414062,
      "activations/layer20_attention_weight_min": -26.745832443237305,
      "activations/layer21_attention_weight_max": 29.717058181762695,
      "activations/layer21_attention_weight_min": -26.028091430664062,
      "activations/layer22_attention_weight_max": 42.57999801635742,
      "activations/layer22_attention_weight_min": -34.63139724731445,
      "activations/layer23_attention_weight_max": 34.60267639160156,
      "activations/layer23_attention_weight_min": -25.65680503845215,
      "activations/layer2_attention_weight_max": 32.98124694824219,
      "activations/layer2_attention_weight_min": -33.05078887939453,
      "activations/layer3_attention_weight_max": 94.34888458251953,
      "activations/layer3_attention_weight_min": -94.70040893554688,
      "activations/layer4_attention_weight_max": 111.056396484375,
      "activations/layer4_attention_weight_min": -113.22649383544922,
      "activations/layer5_attention_weight_max": 53.99071502685547,
      "activations/layer5_attention_weight_min": -63.14265441894531,
      "activations/layer6_attention_weight_max": 45.29075622558594,
      "activations/layer6_attention_weight_min": -46.735435485839844,
      "activations/layer7_attention_weight_max": 92.9199447631836,
      "activations/layer7_attention_weight_min": -92.82483673095703,
      "activations/layer8_attention_weight_max": 40.60259246826172,
      "activations/layer8_attention_weight_min": -42.94662094116211,
      "activations/layer9_attention_weight_max": 33.70174026489258,
      "activations/layer9_attention_weight_min": -33.66926193237305,
      "epoch": 13.86,
      "learning_rate": 6.123030303030302e-05,
      "loss": 2.7643,
      "step": 238550
    },
    {
      "activations/layer0_attention_weight_max": 16.139347076416016,
      "activations/layer0_attention_weight_min": -12.104715347290039,
      "activations/layer10_attention_weight_max": 31.866588592529297,
      "activations/layer10_attention_weight_min": -37.104469299316406,
      "activations/layer11_attention_weight_max": 32.84830093383789,
      "activations/layer11_attention_weight_min": -38.363441467285156,
      "activations/layer12_attention_weight_max": 26.01847267150879,
      "activations/layer12_attention_weight_min": -28.058820724487305,
      "activations/layer13_attention_weight_max": 38.61884307861328,
      "activations/layer13_attention_weight_min": -36.99998474121094,
      "activations/layer14_attention_weight_max": 43.23304748535156,
      "activations/layer14_attention_weight_min": -39.87187957763672,
      "activations/layer15_attention_weight_max": 38.42311096191406,
      "activations/layer15_attention_weight_min": -34.683895111083984,
      "activations/layer16_attention_weight_max": 30.879150390625,
      "activations/layer16_attention_weight_min": -29.501731872558594,
      "activations/layer17_attention_weight_max": 35.092933654785156,
      "activations/layer17_attention_weight_min": -29.10286521911621,
      "activations/layer18_attention_weight_max": 34.194217681884766,
      "activations/layer18_attention_weight_min": -25.226091384887695,
      "activations/layer19_attention_weight_max": 37.0512809753418,
      "activations/layer19_attention_weight_min": -31.292409896850586,
      "activations/layer1_attention_weight_max": 16.20695686340332,
      "activations/layer1_attention_weight_min": -15.280609130859375,
      "activations/layer20_attention_weight_max": 33.196983337402344,
      "activations/layer20_attention_weight_min": -26.372364044189453,
      "activations/layer21_attention_weight_max": 32.42573547363281,
      "activations/layer21_attention_weight_min": -26.27317237854004,
      "activations/layer22_attention_weight_max": 50.332366943359375,
      "activations/layer22_attention_weight_min": -34.08605194091797,
      "activations/layer23_attention_weight_max": 40.83914566040039,
      "activations/layer23_attention_weight_min": -28.791805267333984,
      "activations/layer2_attention_weight_max": 32.92266845703125,
      "activations/layer2_attention_weight_min": -34.318382263183594,
      "activations/layer3_attention_weight_max": 96.95221710205078,
      "activations/layer3_attention_weight_min": -98.78640747070312,
      "activations/layer4_attention_weight_max": 112.42833709716797,
      "activations/layer4_attention_weight_min": -109.28981018066406,
      "activations/layer5_attention_weight_max": 53.13175964355469,
      "activations/layer5_attention_weight_min": -64.7042236328125,
      "activations/layer6_attention_weight_max": 43.85896682739258,
      "activations/layer6_attention_weight_min": -47.72464370727539,
      "activations/layer7_attention_weight_max": 88.59928131103516,
      "activations/layer7_attention_weight_min": -87.44880676269531,
      "activations/layer8_attention_weight_max": 39.604305267333984,
      "activations/layer8_attention_weight_min": -44.74317932128906,
      "activations/layer9_attention_weight_max": 31.548009872436523,
      "activations/layer9_attention_weight_min": -35.024051666259766,
      "epoch": 13.86,
      "learning_rate": 6.121136363636364e-05,
      "loss": 2.7624,
      "step": 238600
    },
    {
      "activations/layer0_attention_weight_max": 15.607681274414062,
      "activations/layer0_attention_weight_min": -13.237695693969727,
      "activations/layer10_attention_weight_max": 34.42824172973633,
      "activations/layer10_attention_weight_min": -33.52077102661133,
      "activations/layer11_attention_weight_max": 33.65435791015625,
      "activations/layer11_attention_weight_min": -32.19990539550781,
      "activations/layer12_attention_weight_max": 25.955692291259766,
      "activations/layer12_attention_weight_min": -27.63711929321289,
      "activations/layer13_attention_weight_max": 39.51768493652344,
      "activations/layer13_attention_weight_min": -38.23357391357422,
      "activations/layer14_attention_weight_max": 46.6822395324707,
      "activations/layer14_attention_weight_min": -42.11925506591797,
      "activations/layer15_attention_weight_max": 37.88005828857422,
      "activations/layer15_attention_weight_min": -35.11991882324219,
      "activations/layer16_attention_weight_max": 30.071290969848633,
      "activations/layer16_attention_weight_min": -28.58303451538086,
      "activations/layer17_attention_weight_max": 32.714542388916016,
      "activations/layer17_attention_weight_min": -26.75603485107422,
      "activations/layer18_attention_weight_max": 34.082759857177734,
      "activations/layer18_attention_weight_min": -26.50387191772461,
      "activations/layer19_attention_weight_max": 34.865821838378906,
      "activations/layer19_attention_weight_min": -32.854103088378906,
      "activations/layer1_attention_weight_max": 17.075414657592773,
      "activations/layer1_attention_weight_min": -17.38523292541504,
      "activations/layer20_attention_weight_max": 31.292490005493164,
      "activations/layer20_attention_weight_min": -25.9556884765625,
      "activations/layer21_attention_weight_max": 29.819272994995117,
      "activations/layer21_attention_weight_min": -24.084768295288086,
      "activations/layer22_attention_weight_max": 46.389888763427734,
      "activations/layer22_attention_weight_min": -31.98972511291504,
      "activations/layer23_attention_weight_max": 34.517723083496094,
      "activations/layer23_attention_weight_min": -25.70177459716797,
      "activations/layer2_attention_weight_max": 33.34035110473633,
      "activations/layer2_attention_weight_min": -32.84679412841797,
      "activations/layer3_attention_weight_max": 100.68687438964844,
      "activations/layer3_attention_weight_min": -98.18186950683594,
      "activations/layer4_attention_weight_max": 108.49853515625,
      "activations/layer4_attention_weight_min": -107.28974914550781,
      "activations/layer5_attention_weight_max": 51.14751434326172,
      "activations/layer5_attention_weight_min": -65.84243774414062,
      "activations/layer6_attention_weight_max": 44.67539978027344,
      "activations/layer6_attention_weight_min": -46.36083221435547,
      "activations/layer7_attention_weight_max": 92.20477294921875,
      "activations/layer7_attention_weight_min": -90.04198455810547,
      "activations/layer8_attention_weight_max": 42.27545166015625,
      "activations/layer8_attention_weight_min": -43.43989181518555,
      "activations/layer9_attention_weight_max": 32.36051559448242,
      "activations/layer9_attention_weight_min": -33.312618255615234,
      "epoch": 13.87,
      "learning_rate": 6.119242424242424e-05,
      "loss": 2.7413,
      "step": 238650
    },
    {
      "activations/layer0_attention_weight_max": 16.126752853393555,
      "activations/layer0_attention_weight_min": -12.605680465698242,
      "activations/layer10_attention_weight_max": 34.63640594482422,
      "activations/layer10_attention_weight_min": -34.123924255371094,
      "activations/layer11_attention_weight_max": 35.27621078491211,
      "activations/layer11_attention_weight_min": -33.98724365234375,
      "activations/layer12_attention_weight_max": 25.857561111450195,
      "activations/layer12_attention_weight_min": -25.822389602661133,
      "activations/layer13_attention_weight_max": 37.76087951660156,
      "activations/layer13_attention_weight_min": -36.134822845458984,
      "activations/layer14_attention_weight_max": 44.150184631347656,
      "activations/layer14_attention_weight_min": -38.39714050292969,
      "activations/layer15_attention_weight_max": 37.983036041259766,
      "activations/layer15_attention_weight_min": -36.70396423339844,
      "activations/layer16_attention_weight_max": 29.486928939819336,
      "activations/layer16_attention_weight_min": -28.270957946777344,
      "activations/layer17_attention_weight_max": 30.338653564453125,
      "activations/layer17_attention_weight_min": -25.992759704589844,
      "activations/layer18_attention_weight_max": 30.85902214050293,
      "activations/layer18_attention_weight_min": -22.95703887939453,
      "activations/layer19_attention_weight_max": 34.741703033447266,
      "activations/layer19_attention_weight_min": -30.015518188476562,
      "activations/layer1_attention_weight_max": 16.959381103515625,
      "activations/layer1_attention_weight_min": -14.37736988067627,
      "activations/layer20_attention_weight_max": 31.338171005249023,
      "activations/layer20_attention_weight_min": -23.64972496032715,
      "activations/layer21_attention_weight_max": 30.446908950805664,
      "activations/layer21_attention_weight_min": -22.366479873657227,
      "activations/layer22_attention_weight_max": 44.767452239990234,
      "activations/layer22_attention_weight_min": -29.070405960083008,
      "activations/layer23_attention_weight_max": 38.38018798828125,
      "activations/layer23_attention_weight_min": -24.774932861328125,
      "activations/layer2_attention_weight_max": 33.00712203979492,
      "activations/layer2_attention_weight_min": -34.05266571044922,
      "activations/layer3_attention_weight_max": 103.26797485351562,
      "activations/layer3_attention_weight_min": -100.33930969238281,
      "activations/layer4_attention_weight_max": 112.7404556274414,
      "activations/layer4_attention_weight_min": -112.99641418457031,
      "activations/layer5_attention_weight_max": 55.63439178466797,
      "activations/layer5_attention_weight_min": -67.50426483154297,
      "activations/layer6_attention_weight_max": 47.21754455566406,
      "activations/layer6_attention_weight_min": -50.81692886352539,
      "activations/layer7_attention_weight_max": 93.392578125,
      "activations/layer7_attention_weight_min": -98.27117156982422,
      "activations/layer8_attention_weight_max": 44.03020095825195,
      "activations/layer8_attention_weight_min": -42.91197204589844,
      "activations/layer9_attention_weight_max": 35.297691345214844,
      "activations/layer9_attention_weight_min": -35.01499938964844,
      "epoch": 13.87,
      "learning_rate": 6.117348484848484e-05,
      "loss": 2.7753,
      "step": 238700
    },
    {
      "activations/layer0_attention_weight_max": 15.738630294799805,
      "activations/layer0_attention_weight_min": -12.062882423400879,
      "activations/layer10_attention_weight_max": 34.80268096923828,
      "activations/layer10_attention_weight_min": -35.40959930419922,
      "activations/layer11_attention_weight_max": 37.2205696105957,
      "activations/layer11_attention_weight_min": -37.934715270996094,
      "activations/layer12_attention_weight_max": 28.935104370117188,
      "activations/layer12_attention_weight_min": -29.853641510009766,
      "activations/layer13_attention_weight_max": 44.48298263549805,
      "activations/layer13_attention_weight_min": -43.14220428466797,
      "activations/layer14_attention_weight_max": 48.32315444946289,
      "activations/layer14_attention_weight_min": -45.91573715209961,
      "activations/layer15_attention_weight_max": 42.96076583862305,
      "activations/layer15_attention_weight_min": -41.33382034301758,
      "activations/layer16_attention_weight_max": 37.602237701416016,
      "activations/layer16_attention_weight_min": -29.595964431762695,
      "activations/layer17_attention_weight_max": 36.07194900512695,
      "activations/layer17_attention_weight_min": -26.4858341217041,
      "activations/layer18_attention_weight_max": 34.943153381347656,
      "activations/layer18_attention_weight_min": -25.845943450927734,
      "activations/layer19_attention_weight_max": 45.707366943359375,
      "activations/layer19_attention_weight_min": -33.44389343261719,
      "activations/layer1_attention_weight_max": 16.45206069946289,
      "activations/layer1_attention_weight_min": -14.523877143859863,
      "activations/layer20_attention_weight_max": 34.9469108581543,
      "activations/layer20_attention_weight_min": -26.25078773498535,
      "activations/layer21_attention_weight_max": 34.74095153808594,
      "activations/layer21_attention_weight_min": -27.041831970214844,
      "activations/layer22_attention_weight_max": 49.29684066772461,
      "activations/layer22_attention_weight_min": -32.27321243286133,
      "activations/layer23_attention_weight_max": 41.02499771118164,
      "activations/layer23_attention_weight_min": -25.868330001831055,
      "activations/layer2_attention_weight_max": 34.65727615356445,
      "activations/layer2_attention_weight_min": -33.682830810546875,
      "activations/layer3_attention_weight_max": 105.30878448486328,
      "activations/layer3_attention_weight_min": -97.86836242675781,
      "activations/layer4_attention_weight_max": 120.58089447021484,
      "activations/layer4_attention_weight_min": -109.91251373291016,
      "activations/layer5_attention_weight_max": 56.802303314208984,
      "activations/layer5_attention_weight_min": -64.81708526611328,
      "activations/layer6_attention_weight_max": 46.41816329956055,
      "activations/layer6_attention_weight_min": -46.01868438720703,
      "activations/layer7_attention_weight_max": 94.95360565185547,
      "activations/layer7_attention_weight_min": -93.49452209472656,
      "activations/layer8_attention_weight_max": 42.45989990234375,
      "activations/layer8_attention_weight_min": -42.4256477355957,
      "activations/layer9_attention_weight_max": 33.738243103027344,
      "activations/layer9_attention_weight_min": -35.19977569580078,
      "epoch": 13.87,
      "learning_rate": 6.115454545454545e-05,
      "loss": 2.7426,
      "step": 238750
    },
    {
      "activations/layer0_attention_weight_max": 15.412995338439941,
      "activations/layer0_attention_weight_min": -12.5742826461792,
      "activations/layer10_attention_weight_max": 31.781667709350586,
      "activations/layer10_attention_weight_min": -33.293643951416016,
      "activations/layer11_attention_weight_max": 30.774381637573242,
      "activations/layer11_attention_weight_min": -33.9638671875,
      "activations/layer12_attention_weight_max": 32.079856872558594,
      "activations/layer12_attention_weight_min": -28.087400436401367,
      "activations/layer13_attention_weight_max": 40.57740020751953,
      "activations/layer13_attention_weight_min": -39.92243957519531,
      "activations/layer14_attention_weight_max": 43.520591735839844,
      "activations/layer14_attention_weight_min": -43.980735778808594,
      "activations/layer15_attention_weight_max": 38.589080810546875,
      "activations/layer15_attention_weight_min": -36.55732727050781,
      "activations/layer16_attention_weight_max": 35.409454345703125,
      "activations/layer16_attention_weight_min": -29.264774322509766,
      "activations/layer17_attention_weight_max": 35.35243606567383,
      "activations/layer17_attention_weight_min": -26.442529678344727,
      "activations/layer18_attention_weight_max": 37.00816345214844,
      "activations/layer18_attention_weight_min": -25.15986442565918,
      "activations/layer19_attention_weight_max": 39.879581451416016,
      "activations/layer19_attention_weight_min": -31.924482345581055,
      "activations/layer1_attention_weight_max": 16.326553344726562,
      "activations/layer1_attention_weight_min": -14.122864723205566,
      "activations/layer20_attention_weight_max": 34.60207748413086,
      "activations/layer20_attention_weight_min": -26.456314086914062,
      "activations/layer21_attention_weight_max": 36.491031646728516,
      "activations/layer21_attention_weight_min": -24.99765396118164,
      "activations/layer22_attention_weight_max": 47.50584411621094,
      "activations/layer22_attention_weight_min": -32.812313079833984,
      "activations/layer23_attention_weight_max": 35.29225158691406,
      "activations/layer23_attention_weight_min": -29.114028930664062,
      "activations/layer2_attention_weight_max": 32.948917388916016,
      "activations/layer2_attention_weight_min": -32.47783660888672,
      "activations/layer3_attention_weight_max": 97.6864242553711,
      "activations/layer3_attention_weight_min": -95.52656555175781,
      "activations/layer4_attention_weight_max": 107.15677642822266,
      "activations/layer4_attention_weight_min": -111.67928314208984,
      "activations/layer5_attention_weight_max": 54.363826751708984,
      "activations/layer5_attention_weight_min": -64.14299011230469,
      "activations/layer6_attention_weight_max": 44.76144790649414,
      "activations/layer6_attention_weight_min": -48.43683624267578,
      "activations/layer7_attention_weight_max": 87.66095733642578,
      "activations/layer7_attention_weight_min": -94.07440185546875,
      "activations/layer8_attention_weight_max": 38.757118225097656,
      "activations/layer8_attention_weight_min": -42.05601501464844,
      "activations/layer9_attention_weight_max": 32.69240951538086,
      "activations/layer9_attention_weight_min": -33.4648323059082,
      "epoch": 13.88,
      "learning_rate": 6.113560606060606e-05,
      "loss": 2.7697,
      "step": 238800
    },
    {
      "activations/layer0_attention_weight_max": 14.962424278259277,
      "activations/layer0_attention_weight_min": -12.40892219543457,
      "activations/layer10_attention_weight_max": 35.705814361572266,
      "activations/layer10_attention_weight_min": -32.9272575378418,
      "activations/layer11_attention_weight_max": 33.503719329833984,
      "activations/layer11_attention_weight_min": -32.53455352783203,
      "activations/layer12_attention_weight_max": 32.876834869384766,
      "activations/layer12_attention_weight_min": -26.012296676635742,
      "activations/layer13_attention_weight_max": 41.37750244140625,
      "activations/layer13_attention_weight_min": -35.153076171875,
      "activations/layer14_attention_weight_max": 43.452239990234375,
      "activations/layer14_attention_weight_min": -39.18278503417969,
      "activations/layer15_attention_weight_max": 37.58213806152344,
      "activations/layer15_attention_weight_min": -33.104488372802734,
      "activations/layer16_attention_weight_max": 31.059158325195312,
      "activations/layer16_attention_weight_min": -27.95850372314453,
      "activations/layer17_attention_weight_max": 29.402902603149414,
      "activations/layer17_attention_weight_min": -24.978334426879883,
      "activations/layer18_attention_weight_max": 29.601055145263672,
      "activations/layer18_attention_weight_min": -24.97643280029297,
      "activations/layer19_attention_weight_max": 31.743183135986328,
      "activations/layer19_attention_weight_min": -29.51545524597168,
      "activations/layer1_attention_weight_max": 16.571271896362305,
      "activations/layer1_attention_weight_min": -16.84168243408203,
      "activations/layer20_attention_weight_max": 28.875682830810547,
      "activations/layer20_attention_weight_min": -26.406999588012695,
      "activations/layer21_attention_weight_max": 29.624509811401367,
      "activations/layer21_attention_weight_min": -24.250015258789062,
      "activations/layer22_attention_weight_max": 43.3028564453125,
      "activations/layer22_attention_weight_min": -28.866933822631836,
      "activations/layer23_attention_weight_max": 31.175092697143555,
      "activations/layer23_attention_weight_min": -25.620756149291992,
      "activations/layer2_attention_weight_max": 34.44121551513672,
      "activations/layer2_attention_weight_min": -33.389408111572266,
      "activations/layer3_attention_weight_max": 100.51754760742188,
      "activations/layer3_attention_weight_min": -98.17281341552734,
      "activations/layer4_attention_weight_max": 113.06090545654297,
      "activations/layer4_attention_weight_min": -112.0360107421875,
      "activations/layer5_attention_weight_max": 54.060848236083984,
      "activations/layer5_attention_weight_min": -61.623268127441406,
      "activations/layer6_attention_weight_max": 45.415122985839844,
      "activations/layer6_attention_weight_min": -48.84053421020508,
      "activations/layer7_attention_weight_max": 96.53390502929688,
      "activations/layer7_attention_weight_min": -90.38343811035156,
      "activations/layer8_attention_weight_max": 39.747806549072266,
      "activations/layer8_attention_weight_min": -40.985477447509766,
      "activations/layer9_attention_weight_max": 32.88312530517578,
      "activations/layer9_attention_weight_min": -33.519527435302734,
      "epoch": 13.88,
      "learning_rate": 6.111666666666666e-05,
      "loss": 2.7641,
      "step": 238850
    },
    {
      "activations/layer0_attention_weight_max": 16.65448760986328,
      "activations/layer0_attention_weight_min": -12.400208473205566,
      "activations/layer10_attention_weight_max": 40.301727294921875,
      "activations/layer10_attention_weight_min": -33.97945022583008,
      "activations/layer11_attention_weight_max": 36.3021240234375,
      "activations/layer11_attention_weight_min": -33.253135681152344,
      "activations/layer12_attention_weight_max": 27.34430694580078,
      "activations/layer12_attention_weight_min": -27.3400936126709,
      "activations/layer13_attention_weight_max": 39.83533477783203,
      "activations/layer13_attention_weight_min": -34.813682556152344,
      "activations/layer14_attention_weight_max": 44.4654655456543,
      "activations/layer14_attention_weight_min": -36.68775939941406,
      "activations/layer15_attention_weight_max": 38.75014114379883,
      "activations/layer15_attention_weight_min": -33.16907501220703,
      "activations/layer16_attention_weight_max": 30.130859375,
      "activations/layer16_attention_weight_min": -26.876426696777344,
      "activations/layer17_attention_weight_max": 28.66995620727539,
      "activations/layer17_attention_weight_min": -25.819265365600586,
      "activations/layer18_attention_weight_max": 31.77523422241211,
      "activations/layer18_attention_weight_min": -25.642107009887695,
      "activations/layer19_attention_weight_max": 31.254281997680664,
      "activations/layer19_attention_weight_min": -30.298126220703125,
      "activations/layer1_attention_weight_max": 17.504501342773438,
      "activations/layer1_attention_weight_min": -15.386308670043945,
      "activations/layer20_attention_weight_max": 28.974340438842773,
      "activations/layer20_attention_weight_min": -25.385459899902344,
      "activations/layer21_attention_weight_max": 31.16156578063965,
      "activations/layer21_attention_weight_min": -24.439495086669922,
      "activations/layer22_attention_weight_max": 41.52018737792969,
      "activations/layer22_attention_weight_min": -29.861469268798828,
      "activations/layer23_attention_weight_max": 33.47718811035156,
      "activations/layer23_attention_weight_min": -28.35312271118164,
      "activations/layer2_attention_weight_max": 34.41297149658203,
      "activations/layer2_attention_weight_min": -35.59208297729492,
      "activations/layer3_attention_weight_max": 100.98757934570312,
      "activations/layer3_attention_weight_min": -100.32331848144531,
      "activations/layer4_attention_weight_max": 113.83562469482422,
      "activations/layer4_attention_weight_min": -111.77156829833984,
      "activations/layer5_attention_weight_max": 54.4616813659668,
      "activations/layer5_attention_weight_min": -60.63690948486328,
      "activations/layer6_attention_weight_max": 45.58302688598633,
      "activations/layer6_attention_weight_min": -47.79990768432617,
      "activations/layer7_attention_weight_max": 95.91726684570312,
      "activations/layer7_attention_weight_min": -90.22991180419922,
      "activations/layer8_attention_weight_max": 46.45719909667969,
      "activations/layer8_attention_weight_min": -42.998329162597656,
      "activations/layer9_attention_weight_max": 37.37272644042969,
      "activations/layer9_attention_weight_min": -34.97700119018555,
      "epoch": 13.88,
      "learning_rate": 6.109772727272726e-05,
      "loss": 2.7689,
      "step": 238900
    },
    {
      "activations/layer0_attention_weight_max": 16.84947395324707,
      "activations/layer0_attention_weight_min": -12.398518562316895,
      "activations/layer10_attention_weight_max": 32.38850021362305,
      "activations/layer10_attention_weight_min": -33.94739532470703,
      "activations/layer11_attention_weight_max": 31.639556884765625,
      "activations/layer11_attention_weight_min": -31.78479766845703,
      "activations/layer12_attention_weight_max": 25.21531105041504,
      "activations/layer12_attention_weight_min": -24.690155029296875,
      "activations/layer13_attention_weight_max": 38.061424255371094,
      "activations/layer13_attention_weight_min": -34.3594856262207,
      "activations/layer14_attention_weight_max": 41.844608306884766,
      "activations/layer14_attention_weight_min": -38.08240509033203,
      "activations/layer15_attention_weight_max": 36.068023681640625,
      "activations/layer15_attention_weight_min": -31.939420700073242,
      "activations/layer16_attention_weight_max": 34.13956069946289,
      "activations/layer16_attention_weight_min": -28.6790714263916,
      "activations/layer17_attention_weight_max": 33.10160827636719,
      "activations/layer17_attention_weight_min": -25.401018142700195,
      "activations/layer18_attention_weight_max": 30.94219398498535,
      "activations/layer18_attention_weight_min": -24.980892181396484,
      "activations/layer19_attention_weight_max": 35.65998077392578,
      "activations/layer19_attention_weight_min": -30.616714477539062,
      "activations/layer1_attention_weight_max": 16.426528930664062,
      "activations/layer1_attention_weight_min": -14.016109466552734,
      "activations/layer20_attention_weight_max": 29.940034866333008,
      "activations/layer20_attention_weight_min": -23.90989875793457,
      "activations/layer21_attention_weight_max": 28.599273681640625,
      "activations/layer21_attention_weight_min": -23.016345977783203,
      "activations/layer22_attention_weight_max": 41.47076416015625,
      "activations/layer22_attention_weight_min": -31.83612823486328,
      "activations/layer23_attention_weight_max": 32.336814880371094,
      "activations/layer23_attention_weight_min": -26.045513153076172,
      "activations/layer2_attention_weight_max": 33.473880767822266,
      "activations/layer2_attention_weight_min": -33.47073745727539,
      "activations/layer3_attention_weight_max": 102.1042709350586,
      "activations/layer3_attention_weight_min": -99.60742950439453,
      "activations/layer4_attention_weight_max": 113.3812026977539,
      "activations/layer4_attention_weight_min": -112.9815673828125,
      "activations/layer5_attention_weight_max": 55.41064453125,
      "activations/layer5_attention_weight_min": -66.3751220703125,
      "activations/layer6_attention_weight_max": 44.142234802246094,
      "activations/layer6_attention_weight_min": -47.70830154418945,
      "activations/layer7_attention_weight_max": 91.76228332519531,
      "activations/layer7_attention_weight_min": -94.24657440185547,
      "activations/layer8_attention_weight_max": 40.645408630371094,
      "activations/layer8_attention_weight_min": -43.506256103515625,
      "activations/layer9_attention_weight_max": 32.11550521850586,
      "activations/layer9_attention_weight_min": -33.59873962402344,
      "epoch": 13.88,
      "learning_rate": 6.107878787878787e-05,
      "loss": 2.7678,
      "step": 238950
    },
    {
      "activations/layer0_attention_weight_max": 17.06083106994629,
      "activations/layer0_attention_weight_min": -12.091635704040527,
      "activations/layer10_attention_weight_max": 32.82011795043945,
      "activations/layer10_attention_weight_min": -35.295265197753906,
      "activations/layer11_attention_weight_max": 31.849117279052734,
      "activations/layer11_attention_weight_min": -34.607540130615234,
      "activations/layer12_attention_weight_max": 25.915922164916992,
      "activations/layer12_attention_weight_min": -27.02210807800293,
      "activations/layer13_attention_weight_max": 39.384429931640625,
      "activations/layer13_attention_weight_min": -34.992305755615234,
      "activations/layer14_attention_weight_max": 45.043540954589844,
      "activations/layer14_attention_weight_min": -36.19672775268555,
      "activations/layer15_attention_weight_max": 41.161319732666016,
      "activations/layer15_attention_weight_min": -32.780391693115234,
      "activations/layer16_attention_weight_max": 30.785490036010742,
      "activations/layer16_attention_weight_min": -26.960390090942383,
      "activations/layer17_attention_weight_max": 32.17546844482422,
      "activations/layer17_attention_weight_min": -26.904346466064453,
      "activations/layer18_attention_weight_max": 31.465694427490234,
      "activations/layer18_attention_weight_min": -24.292654037475586,
      "activations/layer19_attention_weight_max": 33.79042053222656,
      "activations/layer19_attention_weight_min": -30.591266632080078,
      "activations/layer1_attention_weight_max": 16.352331161499023,
      "activations/layer1_attention_weight_min": -14.123026847839355,
      "activations/layer20_attention_weight_max": 30.324073791503906,
      "activations/layer20_attention_weight_min": -25.976795196533203,
      "activations/layer21_attention_weight_max": 26.81004524230957,
      "activations/layer21_attention_weight_min": -24.546640396118164,
      "activations/layer22_attention_weight_max": 46.85978317260742,
      "activations/layer22_attention_weight_min": -31.688339233398438,
      "activations/layer23_attention_weight_max": 35.388404846191406,
      "activations/layer23_attention_weight_min": -27.241239547729492,
      "activations/layer2_attention_weight_max": 33.021484375,
      "activations/layer2_attention_weight_min": -34.505226135253906,
      "activations/layer3_attention_weight_max": 101.36048126220703,
      "activations/layer3_attention_weight_min": -103.69512939453125,
      "activations/layer4_attention_weight_max": 112.59305572509766,
      "activations/layer4_attention_weight_min": -113.7680892944336,
      "activations/layer5_attention_weight_max": 54.86759567260742,
      "activations/layer5_attention_weight_min": -62.71976852416992,
      "activations/layer6_attention_weight_max": 44.048484802246094,
      "activations/layer6_attention_weight_min": -49.37638854980469,
      "activations/layer7_attention_weight_max": 93.51288604736328,
      "activations/layer7_attention_weight_min": -90.47372436523438,
      "activations/layer8_attention_weight_max": 37.464691162109375,
      "activations/layer8_attention_weight_min": -42.60191345214844,
      "activations/layer9_attention_weight_max": 32.13219451904297,
      "activations/layer9_attention_weight_min": -34.20115661621094,
      "epoch": 13.89,
      "learning_rate": 6.105984848484847e-05,
      "loss": 2.7599,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4965,
      "eval_samples_per_second": 505.387,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4965,
      "eval_openwebtext_samples_per_second": 505.387,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9931,
      "eval_wikitext_samples_per_second": 228.792,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_lambada_loss": 2.44921875,
      "eval_lambada_ppl": 11.57929685891098,
      "eval_lambada_runtime": 9.6018,
      "eval_lambada_samples_per_second": 507.091,
      "step": 239000
    },
    {
      "activations/layer0_attention_weight_max": 16.469371795654297,
      "activations/layer0_attention_weight_min": -13.2257661819458,
      "activations/layer10_attention_weight_max": 34.925140380859375,
      "activations/layer10_attention_weight_min": -34.71314239501953,
      "activations/layer11_attention_weight_max": 33.685447692871094,
      "activations/layer11_attention_weight_min": -34.084903717041016,
      "activations/layer12_attention_weight_max": 24.99454116821289,
      "activations/layer12_attention_weight_min": -26.625022888183594,
      "activations/layer13_attention_weight_max": 38.61184310913086,
      "activations/layer13_attention_weight_min": -36.755279541015625,
      "activations/layer14_attention_weight_max": 38.38194274902344,
      "activations/layer14_attention_weight_min": -40.39777755737305,
      "activations/layer15_attention_weight_max": 35.572792053222656,
      "activations/layer15_attention_weight_min": -34.64979553222656,
      "activations/layer16_attention_weight_max": 27.32835578918457,
      "activations/layer16_attention_weight_min": -28.136219024658203,
      "activations/layer17_attention_weight_max": 27.805627822875977,
      "activations/layer17_attention_weight_min": -25.50188446044922,
      "activations/layer18_attention_weight_max": 28.24642562866211,
      "activations/layer18_attention_weight_min": -25.84989356994629,
      "activations/layer19_attention_weight_max": 31.081317901611328,
      "activations/layer19_attention_weight_min": -35.6741828918457,
      "activations/layer1_attention_weight_max": 16.601022720336914,
      "activations/layer1_attention_weight_min": -15.57783031463623,
      "activations/layer20_attention_weight_max": 25.050779342651367,
      "activations/layer20_attention_weight_min": -25.25597381591797,
      "activations/layer21_attention_weight_max": 24.778982162475586,
      "activations/layer21_attention_weight_min": -24.55450439453125,
      "activations/layer22_attention_weight_max": 38.65454864501953,
      "activations/layer22_attention_weight_min": -30.53467559814453,
      "activations/layer23_attention_weight_max": 31.021739959716797,
      "activations/layer23_attention_weight_min": -25.603933334350586,
      "activations/layer2_attention_weight_max": 34.61669921875,
      "activations/layer2_attention_weight_min": -34.84363555908203,
      "activations/layer3_attention_weight_max": 104.94497680664062,
      "activations/layer3_attention_weight_min": -105.35184478759766,
      "activations/layer4_attention_weight_max": 120.27362060546875,
      "activations/layer4_attention_weight_min": -117.87599182128906,
      "activations/layer5_attention_weight_max": 54.05854034423828,
      "activations/layer5_attention_weight_min": -63.5263671875,
      "activations/layer6_attention_weight_max": 45.88826370239258,
      "activations/layer6_attention_weight_min": -46.802101135253906,
      "activations/layer7_attention_weight_max": 92.87422943115234,
      "activations/layer7_attention_weight_min": -89.05077362060547,
      "activations/layer8_attention_weight_max": 42.07286071777344,
      "activations/layer8_attention_weight_min": -41.78219985961914,
      "activations/layer9_attention_weight_max": 34.37710952758789,
      "activations/layer9_attention_weight_min": -35.005924224853516,
      "epoch": 13.89,
      "learning_rate": 6.104090909090909e-05,
      "loss": 2.7623,
      "step": 239050
    },
    {
      "activations/layer0_attention_weight_max": 15.584030151367188,
      "activations/layer0_attention_weight_min": -12.779162406921387,
      "activations/layer10_attention_weight_max": 32.9499626159668,
      "activations/layer10_attention_weight_min": -32.774009704589844,
      "activations/layer11_attention_weight_max": 32.461692810058594,
      "activations/layer11_attention_weight_min": -32.0498161315918,
      "activations/layer12_attention_weight_max": 24.847652435302734,
      "activations/layer12_attention_weight_min": -26.887271881103516,
      "activations/layer13_attention_weight_max": 39.42853927612305,
      "activations/layer13_attention_weight_min": -33.125518798828125,
      "activations/layer14_attention_weight_max": 42.6418571472168,
      "activations/layer14_attention_weight_min": -35.44200897216797,
      "activations/layer15_attention_weight_max": 36.64206314086914,
      "activations/layer15_attention_weight_min": -31.170549392700195,
      "activations/layer16_attention_weight_max": 31.00510597229004,
      "activations/layer16_attention_weight_min": -26.811046600341797,
      "activations/layer17_attention_weight_max": 30.985748291015625,
      "activations/layer17_attention_weight_min": -25.24271583557129,
      "activations/layer18_attention_weight_max": 34.993255615234375,
      "activations/layer18_attention_weight_min": -23.37306785583496,
      "activations/layer19_attention_weight_max": 35.199771881103516,
      "activations/layer19_attention_weight_min": -30.297399520874023,
      "activations/layer1_attention_weight_max": 16.67466926574707,
      "activations/layer1_attention_weight_min": -14.47400951385498,
      "activations/layer20_attention_weight_max": 27.56313133239746,
      "activations/layer20_attention_weight_min": -24.380271911621094,
      "activations/layer21_attention_weight_max": 29.2037410736084,
      "activations/layer21_attention_weight_min": -23.996402740478516,
      "activations/layer22_attention_weight_max": 41.495079040527344,
      "activations/layer22_attention_weight_min": -31.929887771606445,
      "activations/layer23_attention_weight_max": 34.666019439697266,
      "activations/layer23_attention_weight_min": -25.693885803222656,
      "activations/layer2_attention_weight_max": 35.38257598876953,
      "activations/layer2_attention_weight_min": -33.602943420410156,
      "activations/layer3_attention_weight_max": 98.26158905029297,
      "activations/layer3_attention_weight_min": -97.0202407836914,
      "activations/layer4_attention_weight_max": 110.8232421875,
      "activations/layer4_attention_weight_min": -112.96065521240234,
      "activations/layer5_attention_weight_max": 51.05367660522461,
      "activations/layer5_attention_weight_min": -62.23258972167969,
      "activations/layer6_attention_weight_max": 43.19771194458008,
      "activations/layer6_attention_weight_min": -47.71487808227539,
      "activations/layer7_attention_weight_max": 88.89423370361328,
      "activations/layer7_attention_weight_min": -88.3804931640625,
      "activations/layer8_attention_weight_max": 38.77527618408203,
      "activations/layer8_attention_weight_min": -41.206573486328125,
      "activations/layer9_attention_weight_max": 33.16584396362305,
      "activations/layer9_attention_weight_min": -34.08317184448242,
      "epoch": 13.89,
      "learning_rate": 6.102196969696969e-05,
      "loss": 2.7518,
      "step": 239100
    },
    {
      "activations/layer0_attention_weight_max": 15.720755577087402,
      "activations/layer0_attention_weight_min": -12.908825874328613,
      "activations/layer10_attention_weight_max": 34.04802322387695,
      "activations/layer10_attention_weight_min": -33.4995002746582,
      "activations/layer11_attention_weight_max": 33.59685516357422,
      "activations/layer11_attention_weight_min": -34.213050842285156,
      "activations/layer12_attention_weight_max": 26.21572494506836,
      "activations/layer12_attention_weight_min": -25.089780807495117,
      "activations/layer13_attention_weight_max": 41.55009841918945,
      "activations/layer13_attention_weight_min": -35.597862243652344,
      "activations/layer14_attention_weight_max": 50.6179313659668,
      "activations/layer14_attention_weight_min": -44.849334716796875,
      "activations/layer15_attention_weight_max": 40.38417434692383,
      "activations/layer15_attention_weight_min": -34.449947357177734,
      "activations/layer16_attention_weight_max": 32.58076477050781,
      "activations/layer16_attention_weight_min": -27.47996711730957,
      "activations/layer17_attention_weight_max": 34.661861419677734,
      "activations/layer17_attention_weight_min": -25.970172882080078,
      "activations/layer18_attention_weight_max": 34.859493255615234,
      "activations/layer18_attention_weight_min": -24.752044677734375,
      "activations/layer19_attention_weight_max": 38.81421661376953,
      "activations/layer19_attention_weight_min": -35.697837829589844,
      "activations/layer1_attention_weight_max": 15.837080001831055,
      "activations/layer1_attention_weight_min": -13.090058326721191,
      "activations/layer20_attention_weight_max": 35.138607025146484,
      "activations/layer20_attention_weight_min": -28.240549087524414,
      "activations/layer21_attention_weight_max": 32.78960418701172,
      "activations/layer21_attention_weight_min": -27.173765182495117,
      "activations/layer22_attention_weight_max": 52.313026428222656,
      "activations/layer22_attention_weight_min": -35.04719543457031,
      "activations/layer23_attention_weight_max": 40.3751220703125,
      "activations/layer23_attention_weight_min": -26.921428680419922,
      "activations/layer2_attention_weight_max": 36.737022399902344,
      "activations/layer2_attention_weight_min": -33.38273239135742,
      "activations/layer3_attention_weight_max": 99.10760498046875,
      "activations/layer3_attention_weight_min": -98.75045776367188,
      "activations/layer4_attention_weight_max": 112.88629150390625,
      "activations/layer4_attention_weight_min": -114.7821273803711,
      "activations/layer5_attention_weight_max": 51.984004974365234,
      "activations/layer5_attention_weight_min": -67.95187377929688,
      "activations/layer6_attention_weight_max": 42.985172271728516,
      "activations/layer6_attention_weight_min": -47.81977462768555,
      "activations/layer7_attention_weight_max": 92.37665557861328,
      "activations/layer7_attention_weight_min": -103.10684204101562,
      "activations/layer8_attention_weight_max": 39.326908111572266,
      "activations/layer8_attention_weight_min": -42.55596923828125,
      "activations/layer9_attention_weight_max": 33.030120849609375,
      "activations/layer9_attention_weight_min": -33.970211029052734,
      "epoch": 13.9,
      "learning_rate": 6.10030303030303e-05,
      "loss": 2.7794,
      "step": 239150
    },
    {
      "activations/layer0_attention_weight_max": 17.023391723632812,
      "activations/layer0_attention_weight_min": -12.300775527954102,
      "activations/layer10_attention_weight_max": 33.55549621582031,
      "activations/layer10_attention_weight_min": -31.993663787841797,
      "activations/layer11_attention_weight_max": 35.684879302978516,
      "activations/layer11_attention_weight_min": -32.751243591308594,
      "activations/layer12_attention_weight_max": 26.034826278686523,
      "activations/layer12_attention_weight_min": -27.161157608032227,
      "activations/layer13_attention_weight_max": 40.24974060058594,
      "activations/layer13_attention_weight_min": -35.25450897216797,
      "activations/layer14_attention_weight_max": 42.975833892822266,
      "activations/layer14_attention_weight_min": -38.80512237548828,
      "activations/layer15_attention_weight_max": 36.0622444152832,
      "activations/layer15_attention_weight_min": -37.20309829711914,
      "activations/layer16_attention_weight_max": 28.625478744506836,
      "activations/layer16_attention_weight_min": -30.460681915283203,
      "activations/layer17_attention_weight_max": 31.49142837524414,
      "activations/layer17_attention_weight_min": -28.480560302734375,
      "activations/layer18_attention_weight_max": 31.896160125732422,
      "activations/layer18_attention_weight_min": -27.28363037109375,
      "activations/layer19_attention_weight_max": 34.335689544677734,
      "activations/layer19_attention_weight_min": -33.38562774658203,
      "activations/layer1_attention_weight_max": 16.9072322845459,
      "activations/layer1_attention_weight_min": -15.792299270629883,
      "activations/layer20_attention_weight_max": 28.900896072387695,
      "activations/layer20_attention_weight_min": -25.5455265045166,
      "activations/layer21_attention_weight_max": 29.432781219482422,
      "activations/layer21_attention_weight_min": -25.706918716430664,
      "activations/layer22_attention_weight_max": 46.35691833496094,
      "activations/layer22_attention_weight_min": -34.763511657714844,
      "activations/layer23_attention_weight_max": 34.587188720703125,
      "activations/layer23_attention_weight_min": -27.90786361694336,
      "activations/layer2_attention_weight_max": 36.145145416259766,
      "activations/layer2_attention_weight_min": -34.88633346557617,
      "activations/layer3_attention_weight_max": 101.48287963867188,
      "activations/layer3_attention_weight_min": -98.97258758544922,
      "activations/layer4_attention_weight_max": 111.60079193115234,
      "activations/layer4_attention_weight_min": -109.02103424072266,
      "activations/layer5_attention_weight_max": 51.47455978393555,
      "activations/layer5_attention_weight_min": -64.71389770507812,
      "activations/layer6_attention_weight_max": 43.70478820800781,
      "activations/layer6_attention_weight_min": -45.70268630981445,
      "activations/layer7_attention_weight_max": 91.81130981445312,
      "activations/layer7_attention_weight_min": -90.62336730957031,
      "activations/layer8_attention_weight_max": 42.746212005615234,
      "activations/layer8_attention_weight_min": -40.02163314819336,
      "activations/layer9_attention_weight_max": 32.87607192993164,
      "activations/layer9_attention_weight_min": -31.760562896728516,
      "epoch": 13.9,
      "learning_rate": 6.098409090909091e-05,
      "loss": 2.7554,
      "step": 239200
    },
    {
      "activations/layer0_attention_weight_max": 16.21630096435547,
      "activations/layer0_attention_weight_min": -12.399721145629883,
      "activations/layer10_attention_weight_max": 33.378936767578125,
      "activations/layer10_attention_weight_min": -33.926998138427734,
      "activations/layer11_attention_weight_max": 33.54757308959961,
      "activations/layer11_attention_weight_min": -35.14555358886719,
      "activations/layer12_attention_weight_max": 24.482093811035156,
      "activations/layer12_attention_weight_min": -26.0048770904541,
      "activations/layer13_attention_weight_max": 38.093196868896484,
      "activations/layer13_attention_weight_min": -35.36265182495117,
      "activations/layer14_attention_weight_max": 42.26426696777344,
      "activations/layer14_attention_weight_min": -37.366764068603516,
      "activations/layer15_attention_weight_max": 37.755943298339844,
      "activations/layer15_attention_weight_min": -33.19781494140625,
      "activations/layer16_attention_weight_max": 28.80702018737793,
      "activations/layer16_attention_weight_min": -28.15378761291504,
      "activations/layer17_attention_weight_max": 31.327531814575195,
      "activations/layer17_attention_weight_min": -25.31491470336914,
      "activations/layer18_attention_weight_max": 32.581626892089844,
      "activations/layer18_attention_weight_min": -23.80974006652832,
      "activations/layer19_attention_weight_max": 35.086830139160156,
      "activations/layer19_attention_weight_min": -30.71604347229004,
      "activations/layer1_attention_weight_max": 15.573371887207031,
      "activations/layer1_attention_weight_min": -14.032879829406738,
      "activations/layer20_attention_weight_max": 28.006792068481445,
      "activations/layer20_attention_weight_min": -25.016761779785156,
      "activations/layer21_attention_weight_max": 28.1134090423584,
      "activations/layer21_attention_weight_min": -24.34111785888672,
      "activations/layer22_attention_weight_max": 42.2603874206543,
      "activations/layer22_attention_weight_min": -32.29459762573242,
      "activations/layer23_attention_weight_max": 35.59099197387695,
      "activations/layer23_attention_weight_min": -26.698383331298828,
      "activations/layer2_attention_weight_max": 33.495086669921875,
      "activations/layer2_attention_weight_min": -33.46839141845703,
      "activations/layer3_attention_weight_max": 98.38186645507812,
      "activations/layer3_attention_weight_min": -95.74639129638672,
      "activations/layer4_attention_weight_max": 111.03716278076172,
      "activations/layer4_attention_weight_min": -109.78606414794922,
      "activations/layer5_attention_weight_max": 53.07700729370117,
      "activations/layer5_attention_weight_min": -65.58049011230469,
      "activations/layer6_attention_weight_max": 44.42617416381836,
      "activations/layer6_attention_weight_min": -49.26171112060547,
      "activations/layer7_attention_weight_max": 93.72229766845703,
      "activations/layer7_attention_weight_min": -95.13150787353516,
      "activations/layer8_attention_weight_max": 41.07044982910156,
      "activations/layer8_attention_weight_min": -41.964996337890625,
      "activations/layer9_attention_weight_max": 35.8862419128418,
      "activations/layer9_attention_weight_min": -35.39816665649414,
      "epoch": 13.9,
      "learning_rate": 6.09655303030303e-05,
      "loss": 2.758,
      "step": 239250
    },
    {
      "activations/layer0_attention_weight_max": 16.56426429748535,
      "activations/layer0_attention_weight_min": -12.430097579956055,
      "activations/layer10_attention_weight_max": 32.820526123046875,
      "activations/layer10_attention_weight_min": -31.30824851989746,
      "activations/layer11_attention_weight_max": 32.34291076660156,
      "activations/layer11_attention_weight_min": -32.74949645996094,
      "activations/layer12_attention_weight_max": 28.16696548461914,
      "activations/layer12_attention_weight_min": -25.855112075805664,
      "activations/layer13_attention_weight_max": 38.25826644897461,
      "activations/layer13_attention_weight_min": -35.64458465576172,
      "activations/layer14_attention_weight_max": 43.68203353881836,
      "activations/layer14_attention_weight_min": -42.080352783203125,
      "activations/layer15_attention_weight_max": 39.95543670654297,
      "activations/layer15_attention_weight_min": -33.967498779296875,
      "activations/layer16_attention_weight_max": 30.10502052307129,
      "activations/layer16_attention_weight_min": -28.70184898376465,
      "activations/layer17_attention_weight_max": 30.65627670288086,
      "activations/layer17_attention_weight_min": -26.689437866210938,
      "activations/layer18_attention_weight_max": 30.964378356933594,
      "activations/layer18_attention_weight_min": -23.375688552856445,
      "activations/layer19_attention_weight_max": 32.44523620605469,
      "activations/layer19_attention_weight_min": -30.659732818603516,
      "activations/layer1_attention_weight_max": 16.40222930908203,
      "activations/layer1_attention_weight_min": -15.505542755126953,
      "activations/layer20_attention_weight_max": 29.255064010620117,
      "activations/layer20_attention_weight_min": -24.073711395263672,
      "activations/layer21_attention_weight_max": 29.206634521484375,
      "activations/layer21_attention_weight_min": -24.42510223388672,
      "activations/layer22_attention_weight_max": 41.981163024902344,
      "activations/layer22_attention_weight_min": -33.11088180541992,
      "activations/layer23_attention_weight_max": 35.897769927978516,
      "activations/layer23_attention_weight_min": -26.71402359008789,
      "activations/layer2_attention_weight_max": 33.96222686767578,
      "activations/layer2_attention_weight_min": -32.86393737792969,
      "activations/layer3_attention_weight_max": 97.43601989746094,
      "activations/layer3_attention_weight_min": -99.47867584228516,
      "activations/layer4_attention_weight_max": 109.2608871459961,
      "activations/layer4_attention_weight_min": -110.85626220703125,
      "activations/layer5_attention_weight_max": 52.686973571777344,
      "activations/layer5_attention_weight_min": -61.83684539794922,
      "activations/layer6_attention_weight_max": 42.17168426513672,
      "activations/layer6_attention_weight_min": -43.741432189941406,
      "activations/layer7_attention_weight_max": 88.67820739746094,
      "activations/layer7_attention_weight_min": -85.13074493408203,
      "activations/layer8_attention_weight_max": 37.96128845214844,
      "activations/layer8_attention_weight_min": -38.57871627807617,
      "activations/layer9_attention_weight_max": 31.276365280151367,
      "activations/layer9_attention_weight_min": -32.59896469116211,
      "epoch": 13.9,
      "learning_rate": 6.09465909090909e-05,
      "loss": 2.7511,
      "step": 239300
    },
    {
      "activations/layer0_attention_weight_max": 16.64410972595215,
      "activations/layer0_attention_weight_min": -12.365339279174805,
      "activations/layer10_attention_weight_max": 30.90085792541504,
      "activations/layer10_attention_weight_min": -31.530094146728516,
      "activations/layer11_attention_weight_max": 31.463293075561523,
      "activations/layer11_attention_weight_min": -33.244300842285156,
      "activations/layer12_attention_weight_max": 25.993118286132812,
      "activations/layer12_attention_weight_min": -24.87500762939453,
      "activations/layer13_attention_weight_max": 38.72044372558594,
      "activations/layer13_attention_weight_min": -33.67377853393555,
      "activations/layer14_attention_weight_max": 40.37002182006836,
      "activations/layer14_attention_weight_min": -39.11438751220703,
      "activations/layer15_attention_weight_max": 33.960235595703125,
      "activations/layer15_attention_weight_min": -32.79792022705078,
      "activations/layer16_attention_weight_max": 31.328811645507812,
      "activations/layer16_attention_weight_min": -28.139080047607422,
      "activations/layer17_attention_weight_max": 28.60459327697754,
      "activations/layer17_attention_weight_min": -25.87430191040039,
      "activations/layer18_attention_weight_max": 31.858354568481445,
      "activations/layer18_attention_weight_min": -24.639150619506836,
      "activations/layer19_attention_weight_max": 38.29190444946289,
      "activations/layer19_attention_weight_min": -30.21185874938965,
      "activations/layer1_attention_weight_max": 16.84998321533203,
      "activations/layer1_attention_weight_min": -15.746674537658691,
      "activations/layer20_attention_weight_max": 31.379182815551758,
      "activations/layer20_attention_weight_min": -24.220640182495117,
      "activations/layer21_attention_weight_max": 29.333688735961914,
      "activations/layer21_attention_weight_min": -24.62694549560547,
      "activations/layer22_attention_weight_max": 47.256591796875,
      "activations/layer22_attention_weight_min": -29.14753532409668,
      "activations/layer23_attention_weight_max": 36.10089111328125,
      "activations/layer23_attention_weight_min": -24.431468963623047,
      "activations/layer2_attention_weight_max": 33.383323669433594,
      "activations/layer2_attention_weight_min": -31.72713279724121,
      "activations/layer3_attention_weight_max": 95.56954956054688,
      "activations/layer3_attention_weight_min": -99.55520629882812,
      "activations/layer4_attention_weight_max": 109.17481994628906,
      "activations/layer4_attention_weight_min": -109.07732391357422,
      "activations/layer5_attention_weight_max": 52.702598571777344,
      "activations/layer5_attention_weight_min": -66.78780364990234,
      "activations/layer6_attention_weight_max": 44.56276321411133,
      "activations/layer6_attention_weight_min": -45.43452072143555,
      "activations/layer7_attention_weight_max": 91.06234741210938,
      "activations/layer7_attention_weight_min": -101.41291046142578,
      "activations/layer8_attention_weight_max": 39.8419189453125,
      "activations/layer8_attention_weight_min": -43.392459869384766,
      "activations/layer9_attention_weight_max": 30.70393943786621,
      "activations/layer9_attention_weight_min": -31.632476806640625,
      "epoch": 13.91,
      "learning_rate": 6.0927651515151515e-05,
      "loss": 2.7615,
      "step": 239350
    },
    {
      "activations/layer0_attention_weight_max": 16.320432662963867,
      "activations/layer0_attention_weight_min": -13.699166297912598,
      "activations/layer10_attention_weight_max": 30.561918258666992,
      "activations/layer10_attention_weight_min": -31.722171783447266,
      "activations/layer11_attention_weight_max": 33.043304443359375,
      "activations/layer11_attention_weight_min": -32.500267028808594,
      "activations/layer12_attention_weight_max": 27.03256607055664,
      "activations/layer12_attention_weight_min": -25.855281829833984,
      "activations/layer13_attention_weight_max": 38.20262908935547,
      "activations/layer13_attention_weight_min": -33.65087127685547,
      "activations/layer14_attention_weight_max": 46.65713882446289,
      "activations/layer14_attention_weight_min": -35.15755844116211,
      "activations/layer15_attention_weight_max": 40.482994079589844,
      "activations/layer15_attention_weight_min": -34.879295349121094,
      "activations/layer16_attention_weight_max": 33.4846076965332,
      "activations/layer16_attention_weight_min": -29.749958038330078,
      "activations/layer17_attention_weight_max": 29.860544204711914,
      "activations/layer17_attention_weight_min": -25.888628005981445,
      "activations/layer18_attention_weight_max": 29.149822235107422,
      "activations/layer18_attention_weight_min": -24.804113388061523,
      "activations/layer19_attention_weight_max": 36.17302703857422,
      "activations/layer19_attention_weight_min": -31.705686569213867,
      "activations/layer1_attention_weight_max": 16.662353515625,
      "activations/layer1_attention_weight_min": -15.409516334533691,
      "activations/layer20_attention_weight_max": 29.69208335876465,
      "activations/layer20_attention_weight_min": -26.064708709716797,
      "activations/layer21_attention_weight_max": 29.18178367614746,
      "activations/layer21_attention_weight_min": -24.32193374633789,
      "activations/layer22_attention_weight_max": 42.705345153808594,
      "activations/layer22_attention_weight_min": -29.93410873413086,
      "activations/layer23_attention_weight_max": 34.093624114990234,
      "activations/layer23_attention_weight_min": -25.73226547241211,
      "activations/layer2_attention_weight_max": 33.82756042480469,
      "activations/layer2_attention_weight_min": -31.72371482849121,
      "activations/layer3_attention_weight_max": 97.48445892333984,
      "activations/layer3_attention_weight_min": -96.33208465576172,
      "activations/layer4_attention_weight_max": 110.10533905029297,
      "activations/layer4_attention_weight_min": -105.1057357788086,
      "activations/layer5_attention_weight_max": 55.32468795776367,
      "activations/layer5_attention_weight_min": -61.47390365600586,
      "activations/layer6_attention_weight_max": 45.69203186035156,
      "activations/layer6_attention_weight_min": -45.69548797607422,
      "activations/layer7_attention_weight_max": 86.69353485107422,
      "activations/layer7_attention_weight_min": -86.12931060791016,
      "activations/layer8_attention_weight_max": 39.36252212524414,
      "activations/layer8_attention_weight_min": -39.204158782958984,
      "activations/layer9_attention_weight_max": 30.132221221923828,
      "activations/layer9_attention_weight_min": -32.747169494628906,
      "epoch": 13.91,
      "learning_rate": 6.090871212121212e-05,
      "loss": 2.7672,
      "step": 239400
    },
    {
      "activations/layer0_attention_weight_max": 17.098752975463867,
      "activations/layer0_attention_weight_min": -13.551470756530762,
      "activations/layer10_attention_weight_max": 33.71588134765625,
      "activations/layer10_attention_weight_min": -32.800941467285156,
      "activations/layer11_attention_weight_max": 34.89179229736328,
      "activations/layer11_attention_weight_min": -32.79624938964844,
      "activations/layer12_attention_weight_max": 24.455707550048828,
      "activations/layer12_attention_weight_min": -25.19369888305664,
      "activations/layer13_attention_weight_max": 39.45310974121094,
      "activations/layer13_attention_weight_min": -32.73263168334961,
      "activations/layer14_attention_weight_max": 41.835418701171875,
      "activations/layer14_attention_weight_min": -37.49156188964844,
      "activations/layer15_attention_weight_max": 35.64509963989258,
      "activations/layer15_attention_weight_min": -31.848318099975586,
      "activations/layer16_attention_weight_max": 32.00169372558594,
      "activations/layer16_attention_weight_min": -26.458951950073242,
      "activations/layer17_attention_weight_max": 31.722137451171875,
      "activations/layer17_attention_weight_min": -25.3708553314209,
      "activations/layer18_attention_weight_max": 34.426231384277344,
      "activations/layer18_attention_weight_min": -25.011449813842773,
      "activations/layer19_attention_weight_max": 37.40478515625,
      "activations/layer19_attention_weight_min": -30.538183212280273,
      "activations/layer1_attention_weight_max": 15.759760856628418,
      "activations/layer1_attention_weight_min": -14.6725435256958,
      "activations/layer20_attention_weight_max": 29.912464141845703,
      "activations/layer20_attention_weight_min": -26.209238052368164,
      "activations/layer21_attention_weight_max": 30.585994720458984,
      "activations/layer21_attention_weight_min": -24.921953201293945,
      "activations/layer22_attention_weight_max": 44.96714401245117,
      "activations/layer22_attention_weight_min": -32.8472900390625,
      "activations/layer23_attention_weight_max": 34.43346405029297,
      "activations/layer23_attention_weight_min": -26.472126007080078,
      "activations/layer2_attention_weight_max": 33.59819030761719,
      "activations/layer2_attention_weight_min": -34.393959045410156,
      "activations/layer3_attention_weight_max": 99.93025970458984,
      "activations/layer3_attention_weight_min": -100.82038879394531,
      "activations/layer4_attention_weight_max": 110.6316146850586,
      "activations/layer4_attention_weight_min": -114.52276611328125,
      "activations/layer5_attention_weight_max": 52.690330505371094,
      "activations/layer5_attention_weight_min": -62.05538558959961,
      "activations/layer6_attention_weight_max": 43.27271270751953,
      "activations/layer6_attention_weight_min": -45.684940338134766,
      "activations/layer7_attention_weight_max": 89.76414489746094,
      "activations/layer7_attention_weight_min": -90.75468444824219,
      "activations/layer8_attention_weight_max": 41.92193603515625,
      "activations/layer8_attention_weight_min": -42.38872146606445,
      "activations/layer9_attention_weight_max": 32.5218391418457,
      "activations/layer9_attention_weight_min": -34.045753479003906,
      "epoch": 13.91,
      "learning_rate": 6.088977272727272e-05,
      "loss": 2.7517,
      "step": 239450
    },
    {
      "activations/layer0_attention_weight_max": 16.276212692260742,
      "activations/layer0_attention_weight_min": -12.653804779052734,
      "activations/layer10_attention_weight_max": 36.57424545288086,
      "activations/layer10_attention_weight_min": -38.465328216552734,
      "activations/layer11_attention_weight_max": 34.62559127807617,
      "activations/layer11_attention_weight_min": -35.6638069152832,
      "activations/layer12_attention_weight_max": 25.573068618774414,
      "activations/layer12_attention_weight_min": -26.287317276000977,
      "activations/layer13_attention_weight_max": 37.94320297241211,
      "activations/layer13_attention_weight_min": -35.268375396728516,
      "activations/layer14_attention_weight_max": 44.46405029296875,
      "activations/layer14_attention_weight_min": -38.052181243896484,
      "activations/layer15_attention_weight_max": 37.35158157348633,
      "activations/layer15_attention_weight_min": -35.25101089477539,
      "activations/layer16_attention_weight_max": 30.52672004699707,
      "activations/layer16_attention_weight_min": -27.284128189086914,
      "activations/layer17_attention_weight_max": 30.223731994628906,
      "activations/layer17_attention_weight_min": -26.693994522094727,
      "activations/layer18_attention_weight_max": 29.312536239624023,
      "activations/layer18_attention_weight_min": -24.271907806396484,
      "activations/layer19_attention_weight_max": 31.82958984375,
      "activations/layer19_attention_weight_min": -30.797836303710938,
      "activations/layer1_attention_weight_max": 16.33196449279785,
      "activations/layer1_attention_weight_min": -14.703354835510254,
      "activations/layer20_attention_weight_max": 27.552339553833008,
      "activations/layer20_attention_weight_min": -25.613540649414062,
      "activations/layer21_attention_weight_max": 28.96267318725586,
      "activations/layer21_attention_weight_min": -26.12920570373535,
      "activations/layer22_attention_weight_max": 41.253173828125,
      "activations/layer22_attention_weight_min": -30.74006462097168,
      "activations/layer23_attention_weight_max": 31.787946701049805,
      "activations/layer23_attention_weight_min": -23.978168487548828,
      "activations/layer2_attention_weight_max": 33.43857955932617,
      "activations/layer2_attention_weight_min": -34.794822692871094,
      "activations/layer3_attention_weight_max": 102.7112808227539,
      "activations/layer3_attention_weight_min": -104.75444030761719,
      "activations/layer4_attention_weight_max": 114.44610595703125,
      "activations/layer4_attention_weight_min": -111.56111907958984,
      "activations/layer5_attention_weight_max": 54.86567306518555,
      "activations/layer5_attention_weight_min": -65.63858795166016,
      "activations/layer6_attention_weight_max": 46.518978118896484,
      "activations/layer6_attention_weight_min": -45.427799224853516,
      "activations/layer7_attention_weight_max": 91.04365539550781,
      "activations/layer7_attention_weight_min": -95.84566497802734,
      "activations/layer8_attention_weight_max": 42.20817565917969,
      "activations/layer8_attention_weight_min": -44.338436126708984,
      "activations/layer9_attention_weight_max": 33.800254821777344,
      "activations/layer9_attention_weight_min": -34.702457427978516,
      "epoch": 13.92,
      "learning_rate": 6.087083333333333e-05,
      "loss": 2.7548,
      "step": 239500
    },
    {
      "activations/layer0_attention_weight_max": 15.96210765838623,
      "activations/layer0_attention_weight_min": -13.243074417114258,
      "activations/layer10_attention_weight_max": 31.8261775970459,
      "activations/layer10_attention_weight_min": -32.49238204956055,
      "activations/layer11_attention_weight_max": 31.079557418823242,
      "activations/layer11_attention_weight_min": -32.49973678588867,
      "activations/layer12_attention_weight_max": 23.995136260986328,
      "activations/layer12_attention_weight_min": -25.209247589111328,
      "activations/layer13_attention_weight_max": 39.29515838623047,
      "activations/layer13_attention_weight_min": -35.85139083862305,
      "activations/layer14_attention_weight_max": 42.3267822265625,
      "activations/layer14_attention_weight_min": -38.397300720214844,
      "activations/layer15_attention_weight_max": 34.737030029296875,
      "activations/layer15_attention_weight_min": -33.67948532104492,
      "activations/layer16_attention_weight_max": 28.41546630859375,
      "activations/layer16_attention_weight_min": -28.98972511291504,
      "activations/layer17_attention_weight_max": 29.569440841674805,
      "activations/layer17_attention_weight_min": -26.303115844726562,
      "activations/layer18_attention_weight_max": 32.62259292602539,
      "activations/layer18_attention_weight_min": -25.369915008544922,
      "activations/layer19_attention_weight_max": 34.69334030151367,
      "activations/layer19_attention_weight_min": -30.994277954101562,
      "activations/layer1_attention_weight_max": 16.67340660095215,
      "activations/layer1_attention_weight_min": -15.385869026184082,
      "activations/layer20_attention_weight_max": 32.161598205566406,
      "activations/layer20_attention_weight_min": -23.791288375854492,
      "activations/layer21_attention_weight_max": 33.46839904785156,
      "activations/layer21_attention_weight_min": -25.40147590637207,
      "activations/layer22_attention_weight_max": 40.665836334228516,
      "activations/layer22_attention_weight_min": -29.669349670410156,
      "activations/layer23_attention_weight_max": 34.46403503417969,
      "activations/layer23_attention_weight_min": -27.759031295776367,
      "activations/layer2_attention_weight_max": 34.85712432861328,
      "activations/layer2_attention_weight_min": -35.958648681640625,
      "activations/layer3_attention_weight_max": 101.09686279296875,
      "activations/layer3_attention_weight_min": -101.94522857666016,
      "activations/layer4_attention_weight_max": 115.45938873291016,
      "activations/layer4_attention_weight_min": -118.24016571044922,
      "activations/layer5_attention_weight_max": 51.8899040222168,
      "activations/layer5_attention_weight_min": -64.49462127685547,
      "activations/layer6_attention_weight_max": 44.18628692626953,
      "activations/layer6_attention_weight_min": -49.518104553222656,
      "activations/layer7_attention_weight_max": 95.95964813232422,
      "activations/layer7_attention_weight_min": -102.65821838378906,
      "activations/layer8_attention_weight_max": 39.40077209472656,
      "activations/layer8_attention_weight_min": -44.259971618652344,
      "activations/layer9_attention_weight_max": 31.87664794921875,
      "activations/layer9_attention_weight_min": -34.30366134643555,
      "epoch": 13.92,
      "learning_rate": 6.0851893939393935e-05,
      "loss": 2.7643,
      "step": 239550
    },
    {
      "activations/layer0_attention_weight_max": 16.288299560546875,
      "activations/layer0_attention_weight_min": -12.196845054626465,
      "activations/layer10_attention_weight_max": 33.55464553833008,
      "activations/layer10_attention_weight_min": -34.00360107421875,
      "activations/layer11_attention_weight_max": 33.5421028137207,
      "activations/layer11_attention_weight_min": -35.00621032714844,
      "activations/layer12_attention_weight_max": 30.134510040283203,
      "activations/layer12_attention_weight_min": -26.09246063232422,
      "activations/layer13_attention_weight_max": 43.933204650878906,
      "activations/layer13_attention_weight_min": -37.68975067138672,
      "activations/layer14_attention_weight_max": 44.95482635498047,
      "activations/layer14_attention_weight_min": -42.7448844909668,
      "activations/layer15_attention_weight_max": 41.477081298828125,
      "activations/layer15_attention_weight_min": -34.44270706176758,
      "activations/layer16_attention_weight_max": 32.487815856933594,
      "activations/layer16_attention_weight_min": -26.953125,
      "activations/layer17_attention_weight_max": 36.4239616394043,
      "activations/layer17_attention_weight_min": -26.491670608520508,
      "activations/layer18_attention_weight_max": 33.608802795410156,
      "activations/layer18_attention_weight_min": -24.869197845458984,
      "activations/layer19_attention_weight_max": 34.50466537475586,
      "activations/layer19_attention_weight_min": -32.04244613647461,
      "activations/layer1_attention_weight_max": 16.106033325195312,
      "activations/layer1_attention_weight_min": -14.530144691467285,
      "activations/layer20_attention_weight_max": 31.297529220581055,
      "activations/layer20_attention_weight_min": -25.308399200439453,
      "activations/layer21_attention_weight_max": 33.2689094543457,
      "activations/layer21_attention_weight_min": -23.8541316986084,
      "activations/layer22_attention_weight_max": 43.371559143066406,
      "activations/layer22_attention_weight_min": -32.31767654418945,
      "activations/layer23_attention_weight_max": 34.868919372558594,
      "activations/layer23_attention_weight_min": -26.769926071166992,
      "activations/layer2_attention_weight_max": 33.35804748535156,
      "activations/layer2_attention_weight_min": -33.11248779296875,
      "activations/layer3_attention_weight_max": 108.53626251220703,
      "activations/layer3_attention_weight_min": -106.46161651611328,
      "activations/layer4_attention_weight_max": 114.9261245727539,
      "activations/layer4_attention_weight_min": -118.83965301513672,
      "activations/layer5_attention_weight_max": 53.02363204956055,
      "activations/layer5_attention_weight_min": -62.93749237060547,
      "activations/layer6_attention_weight_max": 46.4925651550293,
      "activations/layer6_attention_weight_min": -48.93486404418945,
      "activations/layer7_attention_weight_max": 93.00707244873047,
      "activations/layer7_attention_weight_min": -101.71149444580078,
      "activations/layer8_attention_weight_max": 40.400447845458984,
      "activations/layer8_attention_weight_min": -43.46293258666992,
      "activations/layer9_attention_weight_max": 31.993576049804688,
      "activations/layer9_attention_weight_min": -33.65522384643555,
      "epoch": 13.92,
      "learning_rate": 6.083295454545454e-05,
      "loss": 2.7591,
      "step": 239600
    },
    {
      "activations/layer0_attention_weight_max": 16.142440795898438,
      "activations/layer0_attention_weight_min": -12.437630653381348,
      "activations/layer10_attention_weight_max": 37.9373893737793,
      "activations/layer10_attention_weight_min": -43.05438995361328,
      "activations/layer11_attention_weight_max": 40.25490188598633,
      "activations/layer11_attention_weight_min": -40.42864227294922,
      "activations/layer12_attention_weight_max": 27.068191528320312,
      "activations/layer12_attention_weight_min": -28.438806533813477,
      "activations/layer13_attention_weight_max": 38.82209777832031,
      "activations/layer13_attention_weight_min": -37.87728500366211,
      "activations/layer14_attention_weight_max": 45.982200622558594,
      "activations/layer14_attention_weight_min": -41.266414642333984,
      "activations/layer15_attention_weight_max": 38.94155502319336,
      "activations/layer15_attention_weight_min": -38.92259979248047,
      "activations/layer16_attention_weight_max": 31.68120574951172,
      "activations/layer16_attention_weight_min": -29.584598541259766,
      "activations/layer17_attention_weight_max": 32.34383773803711,
      "activations/layer17_attention_weight_min": -27.523141860961914,
      "activations/layer18_attention_weight_max": 30.588993072509766,
      "activations/layer18_attention_weight_min": -25.326229095458984,
      "activations/layer19_attention_weight_max": 32.936641693115234,
      "activations/layer19_attention_weight_min": -31.819528579711914,
      "activations/layer1_attention_weight_max": 16.45351791381836,
      "activations/layer1_attention_weight_min": -16.419532775878906,
      "activations/layer20_attention_weight_max": 26.135692596435547,
      "activations/layer20_attention_weight_min": -24.24977684020996,
      "activations/layer21_attention_weight_max": 28.53011131286621,
      "activations/layer21_attention_weight_min": -24.811925888061523,
      "activations/layer22_attention_weight_max": 44.30307388305664,
      "activations/layer22_attention_weight_min": -31.71600914001465,
      "activations/layer23_attention_weight_max": 35.70144271850586,
      "activations/layer23_attention_weight_min": -26.29999351501465,
      "activations/layer2_attention_weight_max": 33.49787902832031,
      "activations/layer2_attention_weight_min": -35.53663635253906,
      "activations/layer3_attention_weight_max": 99.70002746582031,
      "activations/layer3_attention_weight_min": -104.74163818359375,
      "activations/layer4_attention_weight_max": 115.98048400878906,
      "activations/layer4_attention_weight_min": -118.41339111328125,
      "activations/layer5_attention_weight_max": 51.34115219116211,
      "activations/layer5_attention_weight_min": -65.28370666503906,
      "activations/layer6_attention_weight_max": 46.175437927246094,
      "activations/layer6_attention_weight_min": -47.91460418701172,
      "activations/layer7_attention_weight_max": 95.10698699951172,
      "activations/layer7_attention_weight_min": -101.65354919433594,
      "activations/layer8_attention_weight_max": 44.28438186645508,
      "activations/layer8_attention_weight_min": -47.2723274230957,
      "activations/layer9_attention_weight_max": 35.25480270385742,
      "activations/layer9_attention_weight_min": -40.545440673828125,
      "epoch": 13.92,
      "learning_rate": 6.0814015151515145e-05,
      "loss": 2.7671,
      "step": 239650
    },
    {
      "activations/layer0_attention_weight_max": 15.22415828704834,
      "activations/layer0_attention_weight_min": -13.066431999206543,
      "activations/layer10_attention_weight_max": 32.204620361328125,
      "activations/layer10_attention_weight_min": -31.529739379882812,
      "activations/layer11_attention_weight_max": 33.69465637207031,
      "activations/layer11_attention_weight_min": -32.99802017211914,
      "activations/layer12_attention_weight_max": 28.246665954589844,
      "activations/layer12_attention_weight_min": -27.350399017333984,
      "activations/layer13_attention_weight_max": 40.889366149902344,
      "activations/layer13_attention_weight_min": -36.440650939941406,
      "activations/layer14_attention_weight_max": 51.442291259765625,
      "activations/layer14_attention_weight_min": -41.54084777832031,
      "activations/layer15_attention_weight_max": 39.28668212890625,
      "activations/layer15_attention_weight_min": -35.12258529663086,
      "activations/layer16_attention_weight_max": 32.51419448852539,
      "activations/layer16_attention_weight_min": -29.071481704711914,
      "activations/layer17_attention_weight_max": 33.54749298095703,
      "activations/layer17_attention_weight_min": -27.703237533569336,
      "activations/layer18_attention_weight_max": 33.3004150390625,
      "activations/layer18_attention_weight_min": -26.349504470825195,
      "activations/layer19_attention_weight_max": 39.49363327026367,
      "activations/layer19_attention_weight_min": -31.73980140686035,
      "activations/layer1_attention_weight_max": 16.660696029663086,
      "activations/layer1_attention_weight_min": -14.271238327026367,
      "activations/layer20_attention_weight_max": 33.05469512939453,
      "activations/layer20_attention_weight_min": -24.18752670288086,
      "activations/layer21_attention_weight_max": 34.57646560668945,
      "activations/layer21_attention_weight_min": -24.148399353027344,
      "activations/layer22_attention_weight_max": 54.03142547607422,
      "activations/layer22_attention_weight_min": -31.67072868347168,
      "activations/layer23_attention_weight_max": 34.383819580078125,
      "activations/layer23_attention_weight_min": -25.533262252807617,
      "activations/layer2_attention_weight_max": 33.364990234375,
      "activations/layer2_attention_weight_min": -35.241676330566406,
      "activations/layer3_attention_weight_max": 103.13317108154297,
      "activations/layer3_attention_weight_min": -99.97193908691406,
      "activations/layer4_attention_weight_max": 116.30960845947266,
      "activations/layer4_attention_weight_min": -118.9250717163086,
      "activations/layer5_attention_weight_max": 53.41691207885742,
      "activations/layer5_attention_weight_min": -62.071205139160156,
      "activations/layer6_attention_weight_max": 44.789302825927734,
      "activations/layer6_attention_weight_min": -48.03736877441406,
      "activations/layer7_attention_weight_max": 93.7511978149414,
      "activations/layer7_attention_weight_min": -96.01679229736328,
      "activations/layer8_attention_weight_max": 41.1800537109375,
      "activations/layer8_attention_weight_min": -43.81585693359375,
      "activations/layer9_attention_weight_max": 31.822330474853516,
      "activations/layer9_attention_weight_min": -32.80236053466797,
      "epoch": 13.93,
      "learning_rate": 6.0795454545454536e-05,
      "loss": 2.7543,
      "step": 239700
    },
    {
      "activations/layer0_attention_weight_max": 16.40088653564453,
      "activations/layer0_attention_weight_min": -12.63801097869873,
      "activations/layer10_attention_weight_max": 36.312103271484375,
      "activations/layer10_attention_weight_min": -34.955657958984375,
      "activations/layer11_attention_weight_max": 35.31962585449219,
      "activations/layer11_attention_weight_min": -33.745819091796875,
      "activations/layer12_attention_weight_max": 26.826234817504883,
      "activations/layer12_attention_weight_min": -25.96157455444336,
      "activations/layer13_attention_weight_max": 40.390377044677734,
      "activations/layer13_attention_weight_min": -35.01615524291992,
      "activations/layer14_attention_weight_max": 44.2978401184082,
      "activations/layer14_attention_weight_min": -36.87755584716797,
      "activations/layer15_attention_weight_max": 38.83452606201172,
      "activations/layer15_attention_weight_min": -33.1815071105957,
      "activations/layer16_attention_weight_max": 31.023818969726562,
      "activations/layer16_attention_weight_min": -27.598989486694336,
      "activations/layer17_attention_weight_max": 36.82748794555664,
      "activations/layer17_attention_weight_min": -27.43536949157715,
      "activations/layer18_attention_weight_max": 33.085140228271484,
      "activations/layer18_attention_weight_min": -25.687002182006836,
      "activations/layer19_attention_weight_max": 38.74906921386719,
      "activations/layer19_attention_weight_min": -29.91780662536621,
      "activations/layer1_attention_weight_max": 17.139005661010742,
      "activations/layer1_attention_weight_min": -15.023641586303711,
      "activations/layer20_attention_weight_max": 33.67738723754883,
      "activations/layer20_attention_weight_min": -25.08760643005371,
      "activations/layer21_attention_weight_max": 31.452342987060547,
      "activations/layer21_attention_weight_min": -23.301828384399414,
      "activations/layer22_attention_weight_max": 53.98600387573242,
      "activations/layer22_attention_weight_min": -29.881765365600586,
      "activations/layer23_attention_weight_max": 37.687255859375,
      "activations/layer23_attention_weight_min": -24.190502166748047,
      "activations/layer2_attention_weight_max": 33.774391174316406,
      "activations/layer2_attention_weight_min": -32.43741226196289,
      "activations/layer3_attention_weight_max": 97.68106079101562,
      "activations/layer3_attention_weight_min": -96.86799621582031,
      "activations/layer4_attention_weight_max": 107.83426666259766,
      "activations/layer4_attention_weight_min": -106.97773742675781,
      "activations/layer5_attention_weight_max": 54.51940155029297,
      "activations/layer5_attention_weight_min": -67.14015197753906,
      "activations/layer6_attention_weight_max": 45.30699157714844,
      "activations/layer6_attention_weight_min": -45.56991958618164,
      "activations/layer7_attention_weight_max": 96.55110931396484,
      "activations/layer7_attention_weight_min": -93.51160430908203,
      "activations/layer8_attention_weight_max": 40.987667083740234,
      "activations/layer8_attention_weight_min": -40.36442184448242,
      "activations/layer9_attention_weight_max": 36.67057418823242,
      "activations/layer9_attention_weight_min": -33.7287712097168,
      "epoch": 13.93,
      "learning_rate": 6.077651515151515e-05,
      "loss": 2.7543,
      "step": 239750
    },
    {
      "activations/layer0_attention_weight_max": 16.518712997436523,
      "activations/layer0_attention_weight_min": -13.426545143127441,
      "activations/layer10_attention_weight_max": 31.709152221679688,
      "activations/layer10_attention_weight_min": -31.107200622558594,
      "activations/layer11_attention_weight_max": 31.30821990966797,
      "activations/layer11_attention_weight_min": -31.52569580078125,
      "activations/layer12_attention_weight_max": 22.857797622680664,
      "activations/layer12_attention_weight_min": -23.884552001953125,
      "activations/layer13_attention_weight_max": 38.23981475830078,
      "activations/layer13_attention_weight_min": -34.49917221069336,
      "activations/layer14_attention_weight_max": 39.19218063354492,
      "activations/layer14_attention_weight_min": -35.60734558105469,
      "activations/layer15_attention_weight_max": 35.51328659057617,
      "activations/layer15_attention_weight_min": -32.795108795166016,
      "activations/layer16_attention_weight_max": 27.228778839111328,
      "activations/layer16_attention_weight_min": -29.14574432373047,
      "activations/layer17_attention_weight_max": 27.990989685058594,
      "activations/layer17_attention_weight_min": -27.613195419311523,
      "activations/layer18_attention_weight_max": 31.298995971679688,
      "activations/layer18_attention_weight_min": -24.660900115966797,
      "activations/layer19_attention_weight_max": 32.02290725708008,
      "activations/layer19_attention_weight_min": -30.275501251220703,
      "activations/layer1_attention_weight_max": 16.399202346801758,
      "activations/layer1_attention_weight_min": -14.513867378234863,
      "activations/layer20_attention_weight_max": 28.492551803588867,
      "activations/layer20_attention_weight_min": -23.988277435302734,
      "activations/layer21_attention_weight_max": 27.742361068725586,
      "activations/layer21_attention_weight_min": -24.66196060180664,
      "activations/layer22_attention_weight_max": 37.533382415771484,
      "activations/layer22_attention_weight_min": -31.60239028930664,
      "activations/layer23_attention_weight_max": 30.99677276611328,
      "activations/layer23_attention_weight_min": -26.350730895996094,
      "activations/layer2_attention_weight_max": 33.488529205322266,
      "activations/layer2_attention_weight_min": -33.46407699584961,
      "activations/layer3_attention_weight_max": 95.37649536132812,
      "activations/layer3_attention_weight_min": -97.97420501708984,
      "activations/layer4_attention_weight_max": 108.7755355834961,
      "activations/layer4_attention_weight_min": -111.6422348022461,
      "activations/layer5_attention_weight_max": 53.284385681152344,
      "activations/layer5_attention_weight_min": -66.16743469238281,
      "activations/layer6_attention_weight_max": 42.235633850097656,
      "activations/layer6_attention_weight_min": -48.09461975097656,
      "activations/layer7_attention_weight_max": 84.6368637084961,
      "activations/layer7_attention_weight_min": -91.51302337646484,
      "activations/layer8_attention_weight_max": 38.572872161865234,
      "activations/layer8_attention_weight_min": -41.18458938598633,
      "activations/layer9_attention_weight_max": 31.075504302978516,
      "activations/layer9_attention_weight_min": -30.99378776550293,
      "epoch": 13.93,
      "learning_rate": 6.075757575757575e-05,
      "loss": 2.7669,
      "step": 239800
    },
    {
      "activations/layer0_attention_weight_max": 16.7016544342041,
      "activations/layer0_attention_weight_min": -12.269594192504883,
      "activations/layer10_attention_weight_max": 35.051273345947266,
      "activations/layer10_attention_weight_min": -33.75886535644531,
      "activations/layer11_attention_weight_max": 33.03871154785156,
      "activations/layer11_attention_weight_min": -34.842288970947266,
      "activations/layer12_attention_weight_max": 28.535598754882812,
      "activations/layer12_attention_weight_min": -26.3414249420166,
      "activations/layer13_attention_weight_max": 43.43672180175781,
      "activations/layer13_attention_weight_min": -36.00059509277344,
      "activations/layer14_attention_weight_max": 52.57972717285156,
      "activations/layer14_attention_weight_min": -39.03159713745117,
      "activations/layer15_attention_weight_max": 48.02324295043945,
      "activations/layer15_attention_weight_min": -37.249847412109375,
      "activations/layer16_attention_weight_max": 34.80771255493164,
      "activations/layer16_attention_weight_min": -27.192655563354492,
      "activations/layer17_attention_weight_max": 32.8226432800293,
      "activations/layer17_attention_weight_min": -27.934072494506836,
      "activations/layer18_attention_weight_max": 36.61257553100586,
      "activations/layer18_attention_weight_min": -25.803560256958008,
      "activations/layer19_attention_weight_max": 42.820472717285156,
      "activations/layer19_attention_weight_min": -34.1308708190918,
      "activations/layer1_attention_weight_max": 16.015403747558594,
      "activations/layer1_attention_weight_min": -14.313409805297852,
      "activations/layer20_attention_weight_max": 38.83832550048828,
      "activations/layer20_attention_weight_min": -26.4998722076416,
      "activations/layer21_attention_weight_max": 34.391151428222656,
      "activations/layer21_attention_weight_min": -25.535797119140625,
      "activations/layer22_attention_weight_max": 47.79033279418945,
      "activations/layer22_attention_weight_min": -32.81547164916992,
      "activations/layer23_attention_weight_max": 38.88019943237305,
      "activations/layer23_attention_weight_min": -23.809728622436523,
      "activations/layer2_attention_weight_max": 33.710205078125,
      "activations/layer2_attention_weight_min": -31.891681671142578,
      "activations/layer3_attention_weight_max": 98.3740005493164,
      "activations/layer3_attention_weight_min": -98.4332046508789,
      "activations/layer4_attention_weight_max": 110.30620574951172,
      "activations/layer4_attention_weight_min": -111.58644104003906,
      "activations/layer5_attention_weight_max": 51.93608093261719,
      "activations/layer5_attention_weight_min": -61.74978256225586,
      "activations/layer6_attention_weight_max": 43.72596740722656,
      "activations/layer6_attention_weight_min": -50.178157806396484,
      "activations/layer7_attention_weight_max": 92.61315155029297,
      "activations/layer7_attention_weight_min": -90.35846710205078,
      "activations/layer8_attention_weight_max": 40.77894973754883,
      "activations/layer8_attention_weight_min": -43.87834548950195,
      "activations/layer9_attention_weight_max": 33.84891128540039,
      "activations/layer9_attention_weight_min": -32.65901565551758,
      "epoch": 13.94,
      "learning_rate": 6.0738636363636354e-05,
      "loss": 2.7636,
      "step": 239850
    },
    {
      "activations/layer0_attention_weight_max": 16.39505386352539,
      "activations/layer0_attention_weight_min": -12.388740539550781,
      "activations/layer10_attention_weight_max": 31.558347702026367,
      "activations/layer10_attention_weight_min": -31.95294189453125,
      "activations/layer11_attention_weight_max": 30.723814010620117,
      "activations/layer11_attention_weight_min": -31.42001724243164,
      "activations/layer12_attention_weight_max": 25.759700775146484,
      "activations/layer12_attention_weight_min": -24.60990333557129,
      "activations/layer13_attention_weight_max": 37.51756286621094,
      "activations/layer13_attention_weight_min": -33.940399169921875,
      "activations/layer14_attention_weight_max": 42.66783142089844,
      "activations/layer14_attention_weight_min": -38.365360260009766,
      "activations/layer15_attention_weight_max": 37.72172927856445,
      "activations/layer15_attention_weight_min": -34.26532745361328,
      "activations/layer16_attention_weight_max": 28.278825759887695,
      "activations/layer16_attention_weight_min": -29.197031021118164,
      "activations/layer17_attention_weight_max": 29.58675765991211,
      "activations/layer17_attention_weight_min": -25.668123245239258,
      "activations/layer18_attention_weight_max": 28.805992126464844,
      "activations/layer18_attention_weight_min": -24.56270980834961,
      "activations/layer19_attention_weight_max": 35.00246810913086,
      "activations/layer19_attention_weight_min": -30.67705726623535,
      "activations/layer1_attention_weight_max": 16.28120231628418,
      "activations/layer1_attention_weight_min": -14.961101531982422,
      "activations/layer20_attention_weight_max": 28.687593460083008,
      "activations/layer20_attention_weight_min": -25.8420467376709,
      "activations/layer21_attention_weight_max": 28.29574203491211,
      "activations/layer21_attention_weight_min": -23.616392135620117,
      "activations/layer22_attention_weight_max": 40.81610107421875,
      "activations/layer22_attention_weight_min": -29.883935928344727,
      "activations/layer23_attention_weight_max": 34.54733657836914,
      "activations/layer23_attention_weight_min": -26.09322738647461,
      "activations/layer2_attention_weight_max": 33.033355712890625,
      "activations/layer2_attention_weight_min": -32.60730743408203,
      "activations/layer3_attention_weight_max": 102.51160430908203,
      "activations/layer3_attention_weight_min": -98.86891174316406,
      "activations/layer4_attention_weight_max": 113.3599624633789,
      "activations/layer4_attention_weight_min": -113.00709533691406,
      "activations/layer5_attention_weight_max": 52.229331970214844,
      "activations/layer5_attention_weight_min": -62.184226989746094,
      "activations/layer6_attention_weight_max": 42.3848876953125,
      "activations/layer6_attention_weight_min": -46.64845657348633,
      "activations/layer7_attention_weight_max": 89.21366119384766,
      "activations/layer7_attention_weight_min": -90.6605224609375,
      "activations/layer8_attention_weight_max": 37.34634780883789,
      "activations/layer8_attention_weight_min": -39.10673904418945,
      "activations/layer9_attention_weight_max": 30.576635360717773,
      "activations/layer9_attention_weight_min": -31.793258666992188,
      "epoch": 13.94,
      "learning_rate": 6.071969696969697e-05,
      "loss": 2.7641,
      "step": 239900
    },
    {
      "activations/layer0_attention_weight_max": 16.54882049560547,
      "activations/layer0_attention_weight_min": -12.64405345916748,
      "activations/layer10_attention_weight_max": 33.9191780090332,
      "activations/layer10_attention_weight_min": -33.85247802734375,
      "activations/layer11_attention_weight_max": 35.643524169921875,
      "activations/layer11_attention_weight_min": -34.10516357421875,
      "activations/layer12_attention_weight_max": 25.051177978515625,
      "activations/layer12_attention_weight_min": -25.049762725830078,
      "activations/layer13_attention_weight_max": 39.46885681152344,
      "activations/layer13_attention_weight_min": -35.46279525756836,
      "activations/layer14_attention_weight_max": 41.73335266113281,
      "activations/layer14_attention_weight_min": -36.21599578857422,
      "activations/layer15_attention_weight_max": 36.58522033691406,
      "activations/layer15_attention_weight_min": -35.33930969238281,
      "activations/layer16_attention_weight_max": 30.769588470458984,
      "activations/layer16_attention_weight_min": -26.85392189025879,
      "activations/layer17_attention_weight_max": 31.14094352722168,
      "activations/layer17_attention_weight_min": -25.862009048461914,
      "activations/layer18_attention_weight_max": 30.484317779541016,
      "activations/layer18_attention_weight_min": -24.228734970092773,
      "activations/layer19_attention_weight_max": 33.30314254760742,
      "activations/layer19_attention_weight_min": -31.981740951538086,
      "activations/layer1_attention_weight_max": 16.152984619140625,
      "activations/layer1_attention_weight_min": -14.522579193115234,
      "activations/layer20_attention_weight_max": 30.683696746826172,
      "activations/layer20_attention_weight_min": -25.99419593811035,
      "activations/layer21_attention_weight_max": 30.358291625976562,
      "activations/layer21_attention_weight_min": -25.273643493652344,
      "activations/layer22_attention_weight_max": 44.20024871826172,
      "activations/layer22_attention_weight_min": -31.477468490600586,
      "activations/layer23_attention_weight_max": 31.249919891357422,
      "activations/layer23_attention_weight_min": -26.670515060424805,
      "activations/layer2_attention_weight_max": 33.664825439453125,
      "activations/layer2_attention_weight_min": -34.65143585205078,
      "activations/layer3_attention_weight_max": 96.36067199707031,
      "activations/layer3_attention_weight_min": -97.61557006835938,
      "activations/layer4_attention_weight_max": 105.29498291015625,
      "activations/layer4_attention_weight_min": -111.42247772216797,
      "activations/layer5_attention_weight_max": 52.693565368652344,
      "activations/layer5_attention_weight_min": -63.00643539428711,
      "activations/layer6_attention_weight_max": 43.151695251464844,
      "activations/layer6_attention_weight_min": -47.13667678833008,
      "activations/layer7_attention_weight_max": 91.39224243164062,
      "activations/layer7_attention_weight_min": -85.2762680053711,
      "activations/layer8_attention_weight_max": 39.59225845336914,
      "activations/layer8_attention_weight_min": -40.49657440185547,
      "activations/layer9_attention_weight_max": 31.839969635009766,
      "activations/layer9_attention_weight_min": -34.0745964050293,
      "epoch": 13.94,
      "learning_rate": 6.070075757575757e-05,
      "loss": 2.767,
      "step": 239950
    },
    {
      "activations/layer0_attention_weight_max": 16.26515007019043,
      "activations/layer0_attention_weight_min": -12.26699447631836,
      "activations/layer10_attention_weight_max": 33.928924560546875,
      "activations/layer10_attention_weight_min": -35.2965202331543,
      "activations/layer11_attention_weight_max": 33.25663757324219,
      "activations/layer11_attention_weight_min": -33.005409240722656,
      "activations/layer12_attention_weight_max": 25.841211318969727,
      "activations/layer12_attention_weight_min": -25.521942138671875,
      "activations/layer13_attention_weight_max": 39.48262023925781,
      "activations/layer13_attention_weight_min": -36.218021392822266,
      "activations/layer14_attention_weight_max": 41.93144226074219,
      "activations/layer14_attention_weight_min": -36.51897048950195,
      "activations/layer15_attention_weight_max": 35.340755462646484,
      "activations/layer15_attention_weight_min": -31.791053771972656,
      "activations/layer16_attention_weight_max": 30.758670806884766,
      "activations/layer16_attention_weight_min": -26.960805892944336,
      "activations/layer17_attention_weight_max": 29.374027252197266,
      "activations/layer17_attention_weight_min": -26.258756637573242,
      "activations/layer18_attention_weight_max": 31.636764526367188,
      "activations/layer18_attention_weight_min": -23.175048828125,
      "activations/layer19_attention_weight_max": 32.824825286865234,
      "activations/layer19_attention_weight_min": -29.066959381103516,
      "activations/layer1_attention_weight_max": 15.534934043884277,
      "activations/layer1_attention_weight_min": -15.01403522491455,
      "activations/layer20_attention_weight_max": 29.645917892456055,
      "activations/layer20_attention_weight_min": -24.239717483520508,
      "activations/layer21_attention_weight_max": 27.847503662109375,
      "activations/layer21_attention_weight_min": -23.104909896850586,
      "activations/layer22_attention_weight_max": 41.89715576171875,
      "activations/layer22_attention_weight_min": -28.503496170043945,
      "activations/layer23_attention_weight_max": 32.22241973876953,
      "activations/layer23_attention_weight_min": -24.672931671142578,
      "activations/layer2_attention_weight_max": 33.79388427734375,
      "activations/layer2_attention_weight_min": -33.95509338378906,
      "activations/layer3_attention_weight_max": 97.44502258300781,
      "activations/layer3_attention_weight_min": -96.68499755859375,
      "activations/layer4_attention_weight_max": 109.71516418457031,
      "activations/layer4_attention_weight_min": -112.25975799560547,
      "activations/layer5_attention_weight_max": 51.592430114746094,
      "activations/layer5_attention_weight_min": -62.24909591674805,
      "activations/layer6_attention_weight_max": 45.41452407836914,
      "activations/layer6_attention_weight_min": -49.25425338745117,
      "activations/layer7_attention_weight_max": 93.2779312133789,
      "activations/layer7_attention_weight_min": -90.68392181396484,
      "activations/layer8_attention_weight_max": 43.00990676879883,
      "activations/layer8_attention_weight_min": -43.785865783691406,
      "activations/layer9_attention_weight_max": 33.9558219909668,
      "activations/layer9_attention_weight_min": -36.104801177978516,
      "epoch": 13.95,
      "learning_rate": 6.068181818181818e-05,
      "loss": 2.7665,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.5161,
      "eval_samples_per_second": 504.221,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.5161,
      "eval_openwebtext_samples_per_second": 504.221,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9903,
      "eval_wikitext_samples_per_second": 229.112,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.5545,
      "eval_lambada_samples_per_second": 509.602,
      "step": 240000
    },
    {
      "activations/layer0_attention_weight_max": 16.293527603149414,
      "activations/layer0_attention_weight_min": -12.578505516052246,
      "activations/layer10_attention_weight_max": 32.987274169921875,
      "activations/layer10_attention_weight_min": -32.28598403930664,
      "activations/layer11_attention_weight_max": 34.10647201538086,
      "activations/layer11_attention_weight_min": -33.89921569824219,
      "activations/layer12_attention_weight_max": 26.32194709777832,
      "activations/layer12_attention_weight_min": -25.569881439208984,
      "activations/layer13_attention_weight_max": 39.36886215209961,
      "activations/layer13_attention_weight_min": -34.78559494018555,
      "activations/layer14_attention_weight_max": 46.804683685302734,
      "activations/layer14_attention_weight_min": -38.44361114501953,
      "activations/layer15_attention_weight_max": 37.516292572021484,
      "activations/layer15_attention_weight_min": -32.72425079345703,
      "activations/layer16_attention_weight_max": 30.840755462646484,
      "activations/layer16_attention_weight_min": -27.629247665405273,
      "activations/layer17_attention_weight_max": 32.297401428222656,
      "activations/layer17_attention_weight_min": -27.709243774414062,
      "activations/layer18_attention_weight_max": 31.191659927368164,
      "activations/layer18_attention_weight_min": -24.970144271850586,
      "activations/layer19_attention_weight_max": 34.24142074584961,
      "activations/layer19_attention_weight_min": -32.149288177490234,
      "activations/layer1_attention_weight_max": 16.61094856262207,
      "activations/layer1_attention_weight_min": -14.43558406829834,
      "activations/layer20_attention_weight_max": 32.28990173339844,
      "activations/layer20_attention_weight_min": -26.83197593688965,
      "activations/layer21_attention_weight_max": 30.393447875976562,
      "activations/layer21_attention_weight_min": -28.053768157958984,
      "activations/layer22_attention_weight_max": 41.8610954284668,
      "activations/layer22_attention_weight_min": -34.363487243652344,
      "activations/layer23_attention_weight_max": 36.276885986328125,
      "activations/layer23_attention_weight_min": -26.339908599853516,
      "activations/layer2_attention_weight_max": 34.44461441040039,
      "activations/layer2_attention_weight_min": -33.05507278442383,
      "activations/layer3_attention_weight_max": 97.06803131103516,
      "activations/layer3_attention_weight_min": -97.62488555908203,
      "activations/layer4_attention_weight_max": 108.42950439453125,
      "activations/layer4_attention_weight_min": -110.62699127197266,
      "activations/layer5_attention_weight_max": 51.736053466796875,
      "activations/layer5_attention_weight_min": -64.5935287475586,
      "activations/layer6_attention_weight_max": 46.053504943847656,
      "activations/layer6_attention_weight_min": -47.497806549072266,
      "activations/layer7_attention_weight_max": 91.38337707519531,
      "activations/layer7_attention_weight_min": -93.20463562011719,
      "activations/layer8_attention_weight_max": 42.66221237182617,
      "activations/layer8_attention_weight_min": -42.92493438720703,
      "activations/layer9_attention_weight_max": 33.92902755737305,
      "activations/layer9_attention_weight_min": -34.34781265258789,
      "epoch": 13.95,
      "learning_rate": 6.066287878787878e-05,
      "loss": 2.7633,
      "step": 240050
    },
    {
      "activations/layer0_attention_weight_max": 15.924040794372559,
      "activations/layer0_attention_weight_min": -11.975893020629883,
      "activations/layer10_attention_weight_max": 43.40642166137695,
      "activations/layer10_attention_weight_min": -40.6566162109375,
      "activations/layer11_attention_weight_max": 45.8617057800293,
      "activations/layer11_attention_weight_min": -41.73316192626953,
      "activations/layer12_attention_weight_max": 31.133377075195312,
      "activations/layer12_attention_weight_min": -28.990219116210938,
      "activations/layer13_attention_weight_max": 55.11264419555664,
      "activations/layer13_attention_weight_min": -38.81440734863281,
      "activations/layer14_attention_weight_max": 58.87605667114258,
      "activations/layer14_attention_weight_min": -43.827919006347656,
      "activations/layer15_attention_weight_max": 59.06593704223633,
      "activations/layer15_attention_weight_min": -42.69536209106445,
      "activations/layer16_attention_weight_max": 40.42338562011719,
      "activations/layer16_attention_weight_min": -29.880247116088867,
      "activations/layer17_attention_weight_max": 34.82771301269531,
      "activations/layer17_attention_weight_min": -31.750680923461914,
      "activations/layer18_attention_weight_max": 30.838180541992188,
      "activations/layer18_attention_weight_min": -25.875680923461914,
      "activations/layer19_attention_weight_max": 36.37723159790039,
      "activations/layer19_attention_weight_min": -29.5194034576416,
      "activations/layer1_attention_weight_max": 18.894960403442383,
      "activations/layer1_attention_weight_min": -18.312475204467773,
      "activations/layer20_attention_weight_max": 30.551183700561523,
      "activations/layer20_attention_weight_min": -23.37570571899414,
      "activations/layer21_attention_weight_max": 25.355703353881836,
      "activations/layer21_attention_weight_min": -23.34665870666504,
      "activations/layer22_attention_weight_max": 38.52149963378906,
      "activations/layer22_attention_weight_min": -28.997468948364258,
      "activations/layer23_attention_weight_max": 30.949222564697266,
      "activations/layer23_attention_weight_min": -25.918094635009766,
      "activations/layer2_attention_weight_max": 38.93391418457031,
      "activations/layer2_attention_weight_min": -36.55199432373047,
      "activations/layer3_attention_weight_max": 107.71446990966797,
      "activations/layer3_attention_weight_min": -108.41464233398438,
      "activations/layer4_attention_weight_max": 113.37556457519531,
      "activations/layer4_attention_weight_min": -110.59142303466797,
      "activations/layer5_attention_weight_max": 51.31144714355469,
      "activations/layer5_attention_weight_min": -63.195709228515625,
      "activations/layer6_attention_weight_max": 49.172420501708984,
      "activations/layer6_attention_weight_min": -48.23487091064453,
      "activations/layer7_attention_weight_max": 105.19371032714844,
      "activations/layer7_attention_weight_min": -99.56514739990234,
      "activations/layer8_attention_weight_max": 50.50862503051758,
      "activations/layer8_attention_weight_min": -49.138023376464844,
      "activations/layer9_attention_weight_max": 42.31425094604492,
      "activations/layer9_attention_weight_min": -41.447845458984375,
      "epoch": 13.95,
      "learning_rate": 6.0643939393939395e-05,
      "loss": 2.7581,
      "step": 240100
    },
    {
      "activations/layer0_attention_weight_max": 17.61341094970703,
      "activations/layer0_attention_weight_min": -12.565777778625488,
      "activations/layer10_attention_weight_max": 31.701269149780273,
      "activations/layer10_attention_weight_min": -31.077104568481445,
      "activations/layer11_attention_weight_max": 30.60391616821289,
      "activations/layer11_attention_weight_min": -31.17896842956543,
      "activations/layer12_attention_weight_max": 25.093734741210938,
      "activations/layer12_attention_weight_min": -26.05748748779297,
      "activations/layer13_attention_weight_max": 37.57937240600586,
      "activations/layer13_attention_weight_min": -34.183067321777344,
      "activations/layer14_attention_weight_max": 40.667022705078125,
      "activations/layer14_attention_weight_min": -36.0002326965332,
      "activations/layer15_attention_weight_max": 34.4157829284668,
      "activations/layer15_attention_weight_min": -31.613567352294922,
      "activations/layer16_attention_weight_max": 30.632583618164062,
      "activations/layer16_attention_weight_min": -27.991567611694336,
      "activations/layer17_attention_weight_max": 32.0966796875,
      "activations/layer17_attention_weight_min": -26.556489944458008,
      "activations/layer18_attention_weight_max": 31.527034759521484,
      "activations/layer18_attention_weight_min": -24.908987045288086,
      "activations/layer19_attention_weight_max": 35.64327621459961,
      "activations/layer19_attention_weight_min": -30.954389572143555,
      "activations/layer1_attention_weight_max": 15.933396339416504,
      "activations/layer1_attention_weight_min": -14.796521186828613,
      "activations/layer20_attention_weight_max": 30.375991821289062,
      "activations/layer20_attention_weight_min": -24.282028198242188,
      "activations/layer21_attention_weight_max": 28.901763916015625,
      "activations/layer21_attention_weight_min": -23.874473571777344,
      "activations/layer22_attention_weight_max": 43.21477508544922,
      "activations/layer22_attention_weight_min": -31.57776641845703,
      "activations/layer23_attention_weight_max": 32.82334899902344,
      "activations/layer23_attention_weight_min": -25.138219833374023,
      "activations/layer2_attention_weight_max": 30.583694458007812,
      "activations/layer2_attention_weight_min": -29.83479881286621,
      "activations/layer3_attention_weight_max": 93.847412109375,
      "activations/layer3_attention_weight_min": -93.38841247558594,
      "activations/layer4_attention_weight_max": 111.50467681884766,
      "activations/layer4_attention_weight_min": -109.97315979003906,
      "activations/layer5_attention_weight_max": 52.42255401611328,
      "activations/layer5_attention_weight_min": -61.02851486206055,
      "activations/layer6_attention_weight_max": 42.73616409301758,
      "activations/layer6_attention_weight_min": -46.5809211730957,
      "activations/layer7_attention_weight_max": 85.82687377929688,
      "activations/layer7_attention_weight_min": -94.78096008300781,
      "activations/layer8_attention_weight_max": 37.615142822265625,
      "activations/layer8_attention_weight_min": -42.42362976074219,
      "activations/layer9_attention_weight_max": 30.75433921813965,
      "activations/layer9_attention_weight_min": -31.4353084564209,
      "epoch": 13.95,
      "learning_rate": 6.0624999999999996e-05,
      "loss": 2.7642,
      "step": 240150
    },
    {
      "activations/layer0_attention_weight_max": 15.764249801635742,
      "activations/layer0_attention_weight_min": -12.468070030212402,
      "activations/layer10_attention_weight_max": 33.40650939941406,
      "activations/layer10_attention_weight_min": -34.74995422363281,
      "activations/layer11_attention_weight_max": 34.265907287597656,
      "activations/layer11_attention_weight_min": -32.84935760498047,
      "activations/layer12_attention_weight_max": 27.40595817565918,
      "activations/layer12_attention_weight_min": -25.8248233795166,
      "activations/layer13_attention_weight_max": 40.62892532348633,
      "activations/layer13_attention_weight_min": -36.56713104248047,
      "activations/layer14_attention_weight_max": 47.95122146606445,
      "activations/layer14_attention_weight_min": -40.666351318359375,
      "activations/layer15_attention_weight_max": 38.45082473754883,
      "activations/layer15_attention_weight_min": -37.202335357666016,
      "activations/layer16_attention_weight_max": 30.33234405517578,
      "activations/layer16_attention_weight_min": -28.623048782348633,
      "activations/layer17_attention_weight_max": 31.26543617248535,
      "activations/layer17_attention_weight_min": -26.215295791625977,
      "activations/layer18_attention_weight_max": 29.232017517089844,
      "activations/layer18_attention_weight_min": -25.1752986907959,
      "activations/layer19_attention_weight_max": 32.8126335144043,
      "activations/layer19_attention_weight_min": -34.001155853271484,
      "activations/layer1_attention_weight_max": 16.606840133666992,
      "activations/layer1_attention_weight_min": -13.694695472717285,
      "activations/layer20_attention_weight_max": 27.30276107788086,
      "activations/layer20_attention_weight_min": -28.317296981811523,
      "activations/layer21_attention_weight_max": 27.41714859008789,
      "activations/layer21_attention_weight_min": -24.519737243652344,
      "activations/layer22_attention_weight_max": 47.0974006652832,
      "activations/layer22_attention_weight_min": -31.4927978515625,
      "activations/layer23_attention_weight_max": 32.052608489990234,
      "activations/layer23_attention_weight_min": -24.84359359741211,
      "activations/layer2_attention_weight_max": 34.51655578613281,
      "activations/layer2_attention_weight_min": -35.08137512207031,
      "activations/layer3_attention_weight_max": 98.112060546875,
      "activations/layer3_attention_weight_min": -101.87176513671875,
      "activations/layer4_attention_weight_max": 111.045166015625,
      "activations/layer4_attention_weight_min": -113.88502502441406,
      "activations/layer5_attention_weight_max": 50.66278076171875,
      "activations/layer5_attention_weight_min": -64.78199768066406,
      "activations/layer6_attention_weight_max": 44.1472282409668,
      "activations/layer6_attention_weight_min": -48.112098693847656,
      "activations/layer7_attention_weight_max": 96.01837921142578,
      "activations/layer7_attention_weight_min": -96.61670684814453,
      "activations/layer8_attention_weight_max": 40.09383010864258,
      "activations/layer8_attention_weight_min": -43.05711364746094,
      "activations/layer9_attention_weight_max": 32.43790817260742,
      "activations/layer9_attention_weight_min": -34.27158737182617,
      "epoch": 13.96,
      "learning_rate": 6.06060606060606e-05,
      "loss": 2.7713,
      "step": 240200
    },
    {
      "activations/layer0_attention_weight_max": 16.499103546142578,
      "activations/layer0_attention_weight_min": -13.194714546203613,
      "activations/layer10_attention_weight_max": 31.208709716796875,
      "activations/layer10_attention_weight_min": -31.622318267822266,
      "activations/layer11_attention_weight_max": 31.783790588378906,
      "activations/layer11_attention_weight_min": -30.51515769958496,
      "activations/layer12_attention_weight_max": 22.61625862121582,
      "activations/layer12_attention_weight_min": -24.946794509887695,
      "activations/layer13_attention_weight_max": 34.358421325683594,
      "activations/layer13_attention_weight_min": -34.337215423583984,
      "activations/layer14_attention_weight_max": 36.835792541503906,
      "activations/layer14_attention_weight_min": -35.73065185546875,
      "activations/layer15_attention_weight_max": 33.06399154663086,
      "activations/layer15_attention_weight_min": -32.241798400878906,
      "activations/layer16_attention_weight_max": 27.090539932250977,
      "activations/layer16_attention_weight_min": -26.55461311340332,
      "activations/layer17_attention_weight_max": 30.906410217285156,
      "activations/layer17_attention_weight_min": -25.435850143432617,
      "activations/layer18_attention_weight_max": 27.863927841186523,
      "activations/layer18_attention_weight_min": -23.459123611450195,
      "activations/layer19_attention_weight_max": 31.772207260131836,
      "activations/layer19_attention_weight_min": -28.781742095947266,
      "activations/layer1_attention_weight_max": 15.516830444335938,
      "activations/layer1_attention_weight_min": -13.947094917297363,
      "activations/layer20_attention_weight_max": 25.437467575073242,
      "activations/layer20_attention_weight_min": -23.586811065673828,
      "activations/layer21_attention_weight_max": 26.544979095458984,
      "activations/layer21_attention_weight_min": -22.602205276489258,
      "activations/layer22_attention_weight_max": 39.35070037841797,
      "activations/layer22_attention_weight_min": -27.646970748901367,
      "activations/layer23_attention_weight_max": 31.75770378112793,
      "activations/layer23_attention_weight_min": -25.592758178710938,
      "activations/layer2_attention_weight_max": 31.98065185546875,
      "activations/layer2_attention_weight_min": -31.139358520507812,
      "activations/layer3_attention_weight_max": 93.6243667602539,
      "activations/layer3_attention_weight_min": -93.49793243408203,
      "activations/layer4_attention_weight_max": 108.3719711303711,
      "activations/layer4_attention_weight_min": -107.83068084716797,
      "activations/layer5_attention_weight_max": 52.676326751708984,
      "activations/layer5_attention_weight_min": -62.054317474365234,
      "activations/layer6_attention_weight_max": 44.72043228149414,
      "activations/layer6_attention_weight_min": -48.23013687133789,
      "activations/layer7_attention_weight_max": 90.06665802001953,
      "activations/layer7_attention_weight_min": -87.89134216308594,
      "activations/layer8_attention_weight_max": 38.94454574584961,
      "activations/layer8_attention_weight_min": -41.44684982299805,
      "activations/layer9_attention_weight_max": 31.347665786743164,
      "activations/layer9_attention_weight_min": -31.556705474853516,
      "epoch": 13.96,
      "learning_rate": 6.0587121212121206e-05,
      "loss": 2.7697,
      "step": 240250
    },
    {
      "activations/layer0_attention_weight_max": 14.902761459350586,
      "activations/layer0_attention_weight_min": -12.04059886932373,
      "activations/layer10_attention_weight_max": 33.411865234375,
      "activations/layer10_attention_weight_min": -36.77250671386719,
      "activations/layer11_attention_weight_max": 32.98516845703125,
      "activations/layer11_attention_weight_min": -35.54358673095703,
      "activations/layer12_attention_weight_max": 26.661787033081055,
      "activations/layer12_attention_weight_min": -26.50342559814453,
      "activations/layer13_attention_weight_max": 40.893409729003906,
      "activations/layer13_attention_weight_min": -35.16054153442383,
      "activations/layer14_attention_weight_max": 43.71733856201172,
      "activations/layer14_attention_weight_min": -39.29960632324219,
      "activations/layer15_attention_weight_max": 40.5880126953125,
      "activations/layer15_attention_weight_min": -35.022491455078125,
      "activations/layer16_attention_weight_max": 32.350860595703125,
      "activations/layer16_attention_weight_min": -30.571565628051758,
      "activations/layer17_attention_weight_max": 31.42399024963379,
      "activations/layer17_attention_weight_min": -26.642963409423828,
      "activations/layer18_attention_weight_max": 34.78361511230469,
      "activations/layer18_attention_weight_min": -25.6833438873291,
      "activations/layer19_attention_weight_max": 37.68304443359375,
      "activations/layer19_attention_weight_min": -30.970260620117188,
      "activations/layer1_attention_weight_max": 16.010026931762695,
      "activations/layer1_attention_weight_min": -13.355124473571777,
      "activations/layer20_attention_weight_max": 32.632530212402344,
      "activations/layer20_attention_weight_min": -24.62022590637207,
      "activations/layer21_attention_weight_max": 31.62340545654297,
      "activations/layer21_attention_weight_min": -24.190940856933594,
      "activations/layer22_attention_weight_max": 42.20508575439453,
      "activations/layer22_attention_weight_min": -32.11960220336914,
      "activations/layer23_attention_weight_max": 35.57392120361328,
      "activations/layer23_attention_weight_min": -27.695938110351562,
      "activations/layer2_attention_weight_max": 32.541748046875,
      "activations/layer2_attention_weight_min": -31.003448486328125,
      "activations/layer3_attention_weight_max": 91.26575469970703,
      "activations/layer3_attention_weight_min": -94.00345611572266,
      "activations/layer4_attention_weight_max": 106.90416717529297,
      "activations/layer4_attention_weight_min": -108.28018951416016,
      "activations/layer5_attention_weight_max": 52.41579818725586,
      "activations/layer5_attention_weight_min": -62.74364471435547,
      "activations/layer6_attention_weight_max": 42.31962203979492,
      "activations/layer6_attention_weight_min": -47.602256774902344,
      "activations/layer7_attention_weight_max": 89.2576675415039,
      "activations/layer7_attention_weight_min": -89.03849792480469,
      "activations/layer8_attention_weight_max": 40.35363006591797,
      "activations/layer8_attention_weight_min": -40.065696716308594,
      "activations/layer9_attention_weight_max": 33.24668502807617,
      "activations/layer9_attention_weight_min": -36.49559020996094,
      "epoch": 13.96,
      "learning_rate": 6.0568181818181814e-05,
      "loss": 2.769,
      "step": 240300
    },
    {
      "activations/layer0_attention_weight_max": 16.96463966369629,
      "activations/layer0_attention_weight_min": -12.49194049835205,
      "activations/layer10_attention_weight_max": 31.69266128540039,
      "activations/layer10_attention_weight_min": -35.38047790527344,
      "activations/layer11_attention_weight_max": 32.321815490722656,
      "activations/layer11_attention_weight_min": -35.62825012207031,
      "activations/layer12_attention_weight_max": 24.322736740112305,
      "activations/layer12_attention_weight_min": -25.125492095947266,
      "activations/layer13_attention_weight_max": 37.24855422973633,
      "activations/layer13_attention_weight_min": -33.7472038269043,
      "activations/layer14_attention_weight_max": 45.59348678588867,
      "activations/layer14_attention_weight_min": -37.71992492675781,
      "activations/layer15_attention_weight_max": 40.30525207519531,
      "activations/layer15_attention_weight_min": -36.71393966674805,
      "activations/layer16_attention_weight_max": 28.413057327270508,
      "activations/layer16_attention_weight_min": -27.702537536621094,
      "activations/layer17_attention_weight_max": 30.533512115478516,
      "activations/layer17_attention_weight_min": -27.764387130737305,
      "activations/layer18_attention_weight_max": 32.03441619873047,
      "activations/layer18_attention_weight_min": -25.605770111083984,
      "activations/layer19_attention_weight_max": 31.953012466430664,
      "activations/layer19_attention_weight_min": -33.57084274291992,
      "activations/layer1_attention_weight_max": 16.631202697753906,
      "activations/layer1_attention_weight_min": -13.49990463256836,
      "activations/layer20_attention_weight_max": 29.487104415893555,
      "activations/layer20_attention_weight_min": -26.792030334472656,
      "activations/layer21_attention_weight_max": 26.980663299560547,
      "activations/layer21_attention_weight_min": -30.897178649902344,
      "activations/layer22_attention_weight_max": 39.791988372802734,
      "activations/layer22_attention_weight_min": -31.146522521972656,
      "activations/layer23_attention_weight_max": 30.991348266601562,
      "activations/layer23_attention_weight_min": -26.420385360717773,
      "activations/layer2_attention_weight_max": 32.39739990234375,
      "activations/layer2_attention_weight_min": -31.78384780883789,
      "activations/layer3_attention_weight_max": 94.40673065185547,
      "activations/layer3_attention_weight_min": -97.22549438476562,
      "activations/layer4_attention_weight_max": 106.4168472290039,
      "activations/layer4_attention_weight_min": -113.471923828125,
      "activations/layer5_attention_weight_max": 48.71153259277344,
      "activations/layer5_attention_weight_min": -63.147430419921875,
      "activations/layer6_attention_weight_max": 42.092952728271484,
      "activations/layer6_attention_weight_min": -48.47825241088867,
      "activations/layer7_attention_weight_max": 85.16736602783203,
      "activations/layer7_attention_weight_min": -89.33031463623047,
      "activations/layer8_attention_weight_max": 38.267250061035156,
      "activations/layer8_attention_weight_min": -40.67234420776367,
      "activations/layer9_attention_weight_max": 30.494979858398438,
      "activations/layer9_attention_weight_min": -33.9526252746582,
      "epoch": 13.97,
      "learning_rate": 6.054924242424242e-05,
      "loss": 2.7698,
      "step": 240350
    },
    {
      "activations/layer0_attention_weight_max": 16.94652557373047,
      "activations/layer0_attention_weight_min": -12.849218368530273,
      "activations/layer10_attention_weight_max": 32.5212516784668,
      "activations/layer10_attention_weight_min": -33.1967658996582,
      "activations/layer11_attention_weight_max": 32.93256378173828,
      "activations/layer11_attention_weight_min": -32.817771911621094,
      "activations/layer12_attention_weight_max": 27.22532844543457,
      "activations/layer12_attention_weight_min": -26.662559509277344,
      "activations/layer13_attention_weight_max": 40.1639404296875,
      "activations/layer13_attention_weight_min": -35.22246170043945,
      "activations/layer14_attention_weight_max": 47.27089309692383,
      "activations/layer14_attention_weight_min": -37.96332931518555,
      "activations/layer15_attention_weight_max": 38.467529296875,
      "activations/layer15_attention_weight_min": -33.51239776611328,
      "activations/layer16_attention_weight_max": 29.77363395690918,
      "activations/layer16_attention_weight_min": -27.804712295532227,
      "activations/layer17_attention_weight_max": 31.60945701599121,
      "activations/layer17_attention_weight_min": -27.03934669494629,
      "activations/layer18_attention_weight_max": 29.827857971191406,
      "activations/layer18_attention_weight_min": -24.467893600463867,
      "activations/layer19_attention_weight_max": 31.81785011291504,
      "activations/layer19_attention_weight_min": -31.612234115600586,
      "activations/layer1_attention_weight_max": 16.281280517578125,
      "activations/layer1_attention_weight_min": -13.898307800292969,
      "activations/layer20_attention_weight_max": 30.4796142578125,
      "activations/layer20_attention_weight_min": -26.39191246032715,
      "activations/layer21_attention_weight_max": 31.178123474121094,
      "activations/layer21_attention_weight_min": -25.62502098083496,
      "activations/layer22_attention_weight_max": 42.66547775268555,
      "activations/layer22_attention_weight_min": -32.62803268432617,
      "activations/layer23_attention_weight_max": 33.7752571105957,
      "activations/layer23_attention_weight_min": -25.44731903076172,
      "activations/layer2_attention_weight_max": 33.84980010986328,
      "activations/layer2_attention_weight_min": -33.54400634765625,
      "activations/layer3_attention_weight_max": 93.04973602294922,
      "activations/layer3_attention_weight_min": -100.26084899902344,
      "activations/layer4_attention_weight_max": 106.97905731201172,
      "activations/layer4_attention_weight_min": -114.38152313232422,
      "activations/layer5_attention_weight_max": 52.47724533081055,
      "activations/layer5_attention_weight_min": -64.991943359375,
      "activations/layer6_attention_weight_max": 45.2865104675293,
      "activations/layer6_attention_weight_min": -50.63671112060547,
      "activations/layer7_attention_weight_max": 94.02296447753906,
      "activations/layer7_attention_weight_min": -102.98580932617188,
      "activations/layer8_attention_weight_max": 41.20771789550781,
      "activations/layer8_attention_weight_min": -47.363094329833984,
      "activations/layer9_attention_weight_max": 34.37007522583008,
      "activations/layer9_attention_weight_min": -33.624176025390625,
      "epoch": 13.97,
      "learning_rate": 6.0530303030303024e-05,
      "loss": 2.7777,
      "step": 240400
    },
    {
      "activations/layer0_attention_weight_max": 16.037717819213867,
      "activations/layer0_attention_weight_min": -12.226303100585938,
      "activations/layer10_attention_weight_max": 30.29758071899414,
      "activations/layer10_attention_weight_min": -30.59159278869629,
      "activations/layer11_attention_weight_max": 30.643497467041016,
      "activations/layer11_attention_weight_min": -31.557594299316406,
      "activations/layer12_attention_weight_max": 24.948484420776367,
      "activations/layer12_attention_weight_min": -25.75802993774414,
      "activations/layer13_attention_weight_max": 38.14697265625,
      "activations/layer13_attention_weight_min": -33.61827850341797,
      "activations/layer14_attention_weight_max": 39.46480941772461,
      "activations/layer14_attention_weight_min": -35.26580810546875,
      "activations/layer15_attention_weight_max": 34.19427490234375,
      "activations/layer15_attention_weight_min": -30.669750213623047,
      "activations/layer16_attention_weight_max": 27.58153533935547,
      "activations/layer16_attention_weight_min": -26.894287109375,
      "activations/layer17_attention_weight_max": 30.610464096069336,
      "activations/layer17_attention_weight_min": -24.956729888916016,
      "activations/layer18_attention_weight_max": 30.943735122680664,
      "activations/layer18_attention_weight_min": -23.68218994140625,
      "activations/layer19_attention_weight_max": 37.63490295410156,
      "activations/layer19_attention_weight_min": -30.799657821655273,
      "activations/layer1_attention_weight_max": 16.93787384033203,
      "activations/layer1_attention_weight_min": -14.921260833740234,
      "activations/layer20_attention_weight_max": 31.925939559936523,
      "activations/layer20_attention_weight_min": -24.531293869018555,
      "activations/layer21_attention_weight_max": 32.51185989379883,
      "activations/layer21_attention_weight_min": -26.495609283447266,
      "activations/layer22_attention_weight_max": 47.49063491821289,
      "activations/layer22_attention_weight_min": -31.30195426940918,
      "activations/layer23_attention_weight_max": 35.76390838623047,
      "activations/layer23_attention_weight_min": -25.031978607177734,
      "activations/layer2_attention_weight_max": 32.761268615722656,
      "activations/layer2_attention_weight_min": -33.63046646118164,
      "activations/layer3_attention_weight_max": 94.3858413696289,
      "activations/layer3_attention_weight_min": -99.0354995727539,
      "activations/layer4_attention_weight_max": 109.92436981201172,
      "activations/layer4_attention_weight_min": -111.62287902832031,
      "activations/layer5_attention_weight_max": 49.839534759521484,
      "activations/layer5_attention_weight_min": -63.191585540771484,
      "activations/layer6_attention_weight_max": 42.06416702270508,
      "activations/layer6_attention_weight_min": -46.88880920410156,
      "activations/layer7_attention_weight_max": 85.7337417602539,
      "activations/layer7_attention_weight_min": -85.77952575683594,
      "activations/layer8_attention_weight_max": 36.65290832519531,
      "activations/layer8_attention_weight_min": -43.24724197387695,
      "activations/layer9_attention_weight_max": 29.608503341674805,
      "activations/layer9_attention_weight_min": -32.598663330078125,
      "epoch": 13.97,
      "learning_rate": 6.0511363636363626e-05,
      "loss": 2.7595,
      "step": 240450
    },
    {
      "activations/layer0_attention_weight_max": 14.684646606445312,
      "activations/layer0_attention_weight_min": -12.756651878356934,
      "activations/layer10_attention_weight_max": 33.16509246826172,
      "activations/layer10_attention_weight_min": -32.606658935546875,
      "activations/layer11_attention_weight_max": 32.19084167480469,
      "activations/layer11_attention_weight_min": -33.05837631225586,
      "activations/layer12_attention_weight_max": 25.584716796875,
      "activations/layer12_attention_weight_min": -25.410537719726562,
      "activations/layer13_attention_weight_max": 41.13322830200195,
      "activations/layer13_attention_weight_min": -36.74700927734375,
      "activations/layer14_attention_weight_max": 42.889617919921875,
      "activations/layer14_attention_weight_min": -38.80017852783203,
      "activations/layer15_attention_weight_max": 37.475013732910156,
      "activations/layer15_attention_weight_min": -33.650047302246094,
      "activations/layer16_attention_weight_max": 33.9620475769043,
      "activations/layer16_attention_weight_min": -28.1785831451416,
      "activations/layer17_attention_weight_max": 31.897987365722656,
      "activations/layer17_attention_weight_min": -26.910871505737305,
      "activations/layer18_attention_weight_max": 33.81861114501953,
      "activations/layer18_attention_weight_min": -26.393503189086914,
      "activations/layer19_attention_weight_max": 35.07429122924805,
      "activations/layer19_attention_weight_min": -31.546735763549805,
      "activations/layer1_attention_weight_max": 16.25052261352539,
      "activations/layer1_attention_weight_min": -13.852640151977539,
      "activations/layer20_attention_weight_max": 30.195100784301758,
      "activations/layer20_attention_weight_min": -24.81698226928711,
      "activations/layer21_attention_weight_max": 32.016021728515625,
      "activations/layer21_attention_weight_min": -24.008987426757812,
      "activations/layer22_attention_weight_max": 46.96546936035156,
      "activations/layer22_attention_weight_min": -30.481029510498047,
      "activations/layer23_attention_weight_max": 35.04364776611328,
      "activations/layer23_attention_weight_min": -26.550336837768555,
      "activations/layer2_attention_weight_max": 32.299644470214844,
      "activations/layer2_attention_weight_min": -31.640640258789062,
      "activations/layer3_attention_weight_max": 91.54640197753906,
      "activations/layer3_attention_weight_min": -92.60706329345703,
      "activations/layer4_attention_weight_max": 105.10091400146484,
      "activations/layer4_attention_weight_min": -112.53829193115234,
      "activations/layer5_attention_weight_max": 49.92932891845703,
      "activations/layer5_attention_weight_min": -62.315040588378906,
      "activations/layer6_attention_weight_max": 42.50579071044922,
      "activations/layer6_attention_weight_min": -45.37124252319336,
      "activations/layer7_attention_weight_max": 86.91170501708984,
      "activations/layer7_attention_weight_min": -90.31940460205078,
      "activations/layer8_attention_weight_max": 40.505645751953125,
      "activations/layer8_attention_weight_min": -42.194305419921875,
      "activations/layer9_attention_weight_max": 31.597885131835938,
      "activations/layer9_attention_weight_min": -33.204044342041016,
      "epoch": 13.97,
      "learning_rate": 6.049242424242424e-05,
      "loss": 2.7612,
      "step": 240500
    },
    {
      "activations/layer0_attention_weight_max": 15.909281730651855,
      "activations/layer0_attention_weight_min": -12.532575607299805,
      "activations/layer10_attention_weight_max": 36.831058502197266,
      "activations/layer10_attention_weight_min": -33.21083450317383,
      "activations/layer11_attention_weight_max": 34.82801055908203,
      "activations/layer11_attention_weight_min": -34.332149505615234,
      "activations/layer12_attention_weight_max": 27.78894805908203,
      "activations/layer12_attention_weight_min": -26.191987991333008,
      "activations/layer13_attention_weight_max": 38.31786346435547,
      "activations/layer13_attention_weight_min": -34.89891815185547,
      "activations/layer14_attention_weight_max": 42.429832458496094,
      "activations/layer14_attention_weight_min": -37.23432540893555,
      "activations/layer15_attention_weight_max": 37.251895904541016,
      "activations/layer15_attention_weight_min": -34.17390060424805,
      "activations/layer16_attention_weight_max": 33.625003814697266,
      "activations/layer16_attention_weight_min": -28.152254104614258,
      "activations/layer17_attention_weight_max": 35.25788116455078,
      "activations/layer17_attention_weight_min": -26.394357681274414,
      "activations/layer18_attention_weight_max": 35.43558120727539,
      "activations/layer18_attention_weight_min": -24.05678939819336,
      "activations/layer19_attention_weight_max": 39.37724685668945,
      "activations/layer19_attention_weight_min": -31.39828872680664,
      "activations/layer1_attention_weight_max": 16.04048728942871,
      "activations/layer1_attention_weight_min": -13.392656326293945,
      "activations/layer20_attention_weight_max": 32.52106475830078,
      "activations/layer20_attention_weight_min": -26.134658813476562,
      "activations/layer21_attention_weight_max": 36.26089859008789,
      "activations/layer21_attention_weight_min": -25.0848331451416,
      "activations/layer22_attention_weight_max": 49.70872497558594,
      "activations/layer22_attention_weight_min": -30.854415893554688,
      "activations/layer23_attention_weight_max": 37.89118194580078,
      "activations/layer23_attention_weight_min": -26.044456481933594,
      "activations/layer2_attention_weight_max": 33.253108978271484,
      "activations/layer2_attention_weight_min": -32.76314163208008,
      "activations/layer3_attention_weight_max": 94.44490051269531,
      "activations/layer3_attention_weight_min": -94.47932434082031,
      "activations/layer4_attention_weight_max": 108.61134338378906,
      "activations/layer4_attention_weight_min": -112.41643524169922,
      "activations/layer5_attention_weight_max": 51.550079345703125,
      "activations/layer5_attention_weight_min": -65.74725341796875,
      "activations/layer6_attention_weight_max": 43.82112503051758,
      "activations/layer6_attention_weight_min": -47.68912887573242,
      "activations/layer7_attention_weight_max": 92.28688049316406,
      "activations/layer7_attention_weight_min": -89.34823608398438,
      "activations/layer8_attention_weight_max": 42.57895278930664,
      "activations/layer8_attention_weight_min": -40.9951057434082,
      "activations/layer9_attention_weight_max": 34.38935852050781,
      "activations/layer9_attention_weight_min": -34.8237419128418,
      "epoch": 13.98,
      "learning_rate": 6.047348484848484e-05,
      "loss": 2.7652,
      "step": 240550
    },
    {
      "activations/layer0_attention_weight_max": 16.639799118041992,
      "activations/layer0_attention_weight_min": -12.702693939208984,
      "activations/layer10_attention_weight_max": 30.77884292602539,
      "activations/layer10_attention_weight_min": -33.692176818847656,
      "activations/layer11_attention_weight_max": 32.61222839355469,
      "activations/layer11_attention_weight_min": -31.855558395385742,
      "activations/layer12_attention_weight_max": 25.34336280822754,
      "activations/layer12_attention_weight_min": -25.716209411621094,
      "activations/layer13_attention_weight_max": 37.55263137817383,
      "activations/layer13_attention_weight_min": -32.75550842285156,
      "activations/layer14_attention_weight_max": 39.26997375488281,
      "activations/layer14_attention_weight_min": -34.24781036376953,
      "activations/layer15_attention_weight_max": 33.86629867553711,
      "activations/layer15_attention_weight_min": -32.70311737060547,
      "activations/layer16_attention_weight_max": 31.130435943603516,
      "activations/layer16_attention_weight_min": -26.55072593688965,
      "activations/layer17_attention_weight_max": 28.8194580078125,
      "activations/layer17_attention_weight_min": -24.19388198852539,
      "activations/layer18_attention_weight_max": 31.504667282104492,
      "activations/layer18_attention_weight_min": -22.233659744262695,
      "activations/layer19_attention_weight_max": 33.64860916137695,
      "activations/layer19_attention_weight_min": -29.604629516601562,
      "activations/layer1_attention_weight_max": 16.48843765258789,
      "activations/layer1_attention_weight_min": -14.708492279052734,
      "activations/layer20_attention_weight_max": 30.82158851623535,
      "activations/layer20_attention_weight_min": -24.093881607055664,
      "activations/layer21_attention_weight_max": 28.192195892333984,
      "activations/layer21_attention_weight_min": -24.244421005249023,
      "activations/layer22_attention_weight_max": 42.78868103027344,
      "activations/layer22_attention_weight_min": -29.92942237854004,
      "activations/layer23_attention_weight_max": 36.07300567626953,
      "activations/layer23_attention_weight_min": -23.886642456054688,
      "activations/layer2_attention_weight_max": 31.400856018066406,
      "activations/layer2_attention_weight_min": -33.34039306640625,
      "activations/layer3_attention_weight_max": 92.00599670410156,
      "activations/layer3_attention_weight_min": -95.1531753540039,
      "activations/layer4_attention_weight_max": 102.39656829833984,
      "activations/layer4_attention_weight_min": -102.9442367553711,
      "activations/layer5_attention_weight_max": 51.00543212890625,
      "activations/layer5_attention_weight_min": -59.418479919433594,
      "activations/layer6_attention_weight_max": 42.73890686035156,
      "activations/layer6_attention_weight_min": -43.805580139160156,
      "activations/layer7_attention_weight_max": 92.06449890136719,
      "activations/layer7_attention_weight_min": -89.32223510742188,
      "activations/layer8_attention_weight_max": 38.58269119262695,
      "activations/layer8_attention_weight_min": -42.183353424072266,
      "activations/layer9_attention_weight_max": 30.644123077392578,
      "activations/layer9_attention_weight_min": -31.019392013549805,
      "epoch": 13.98,
      "learning_rate": 6.045454545454545e-05,
      "loss": 2.7765,
      "step": 240600
    },
    {
      "activations/layer0_attention_weight_max": 15.705090522766113,
      "activations/layer0_attention_weight_min": -12.525775909423828,
      "activations/layer10_attention_weight_max": 29.582353591918945,
      "activations/layer10_attention_weight_min": -29.60551643371582,
      "activations/layer11_attention_weight_max": 29.21572494506836,
      "activations/layer11_attention_weight_min": -29.233295440673828,
      "activations/layer12_attention_weight_max": 22.93256950378418,
      "activations/layer12_attention_weight_min": -25.123992919921875,
      "activations/layer13_attention_weight_max": 33.15287399291992,
      "activations/layer13_attention_weight_min": -31.239477157592773,
      "activations/layer14_attention_weight_max": 34.904273986816406,
      "activations/layer14_attention_weight_min": -34.52180862426758,
      "activations/layer15_attention_weight_max": 30.889307022094727,
      "activations/layer15_attention_weight_min": -30.012575149536133,
      "activations/layer16_attention_weight_max": 26.818246841430664,
      "activations/layer16_attention_weight_min": -27.288387298583984,
      "activations/layer17_attention_weight_max": 27.875614166259766,
      "activations/layer17_attention_weight_min": -24.584028244018555,
      "activations/layer18_attention_weight_max": 31.52987289428711,
      "activations/layer18_attention_weight_min": -23.553035736083984,
      "activations/layer19_attention_weight_max": 31.63567543029785,
      "activations/layer19_attention_weight_min": -30.008420944213867,
      "activations/layer1_attention_weight_max": 16.319324493408203,
      "activations/layer1_attention_weight_min": -14.93185043334961,
      "activations/layer20_attention_weight_max": 32.09340286254883,
      "activations/layer20_attention_weight_min": -23.596290588378906,
      "activations/layer21_attention_weight_max": 28.336814880371094,
      "activations/layer21_attention_weight_min": -22.852794647216797,
      "activations/layer22_attention_weight_max": 42.50567626953125,
      "activations/layer22_attention_weight_min": -27.972572326660156,
      "activations/layer23_attention_weight_max": 29.76318359375,
      "activations/layer23_attention_weight_min": -24.856107711791992,
      "activations/layer2_attention_weight_max": 32.63484191894531,
      "activations/layer2_attention_weight_min": -31.954280853271484,
      "activations/layer3_attention_weight_max": 98.94779968261719,
      "activations/layer3_attention_weight_min": -95.02806091308594,
      "activations/layer4_attention_weight_max": 106.07317352294922,
      "activations/layer4_attention_weight_min": -105.81495666503906,
      "activations/layer5_attention_weight_max": 51.44562911987305,
      "activations/layer5_attention_weight_min": -62.45355224609375,
      "activations/layer6_attention_weight_max": 46.35403060913086,
      "activations/layer6_attention_weight_min": -45.62944412231445,
      "activations/layer7_attention_weight_max": 88.07184600830078,
      "activations/layer7_attention_weight_min": -87.77322387695312,
      "activations/layer8_attention_weight_max": 36.706390380859375,
      "activations/layer8_attention_weight_min": -41.65929412841797,
      "activations/layer9_attention_weight_max": 29.148279190063477,
      "activations/layer9_attention_weight_min": -30.951078414916992,
      "epoch": 13.98,
      "learning_rate": 6.043560606060606e-05,
      "loss": 2.7606,
      "step": 240650
    },
    {
      "activations/layer0_attention_weight_max": 16.805675506591797,
      "activations/layer0_attention_weight_min": -12.752497673034668,
      "activations/layer10_attention_weight_max": 30.970468521118164,
      "activations/layer10_attention_weight_min": -33.06370544433594,
      "activations/layer11_attention_weight_max": 31.8707275390625,
      "activations/layer11_attention_weight_min": -32.03900146484375,
      "activations/layer12_attention_weight_max": 23.927587509155273,
      "activations/layer12_attention_weight_min": -26.136430740356445,
      "activations/layer13_attention_weight_max": 38.0357780456543,
      "activations/layer13_attention_weight_min": -34.76792907714844,
      "activations/layer14_attention_weight_max": 42.396270751953125,
      "activations/layer14_attention_weight_min": -36.77665710449219,
      "activations/layer15_attention_weight_max": 35.188697814941406,
      "activations/layer15_attention_weight_min": -35.67768096923828,
      "activations/layer16_attention_weight_max": 30.456249237060547,
      "activations/layer16_attention_weight_min": -31.45159149169922,
      "activations/layer17_attention_weight_max": 29.10726547241211,
      "activations/layer17_attention_weight_min": -27.11661148071289,
      "activations/layer18_attention_weight_max": 31.808391571044922,
      "activations/layer18_attention_weight_min": -23.8663330078125,
      "activations/layer19_attention_weight_max": 41.125789642333984,
      "activations/layer19_attention_weight_min": -32.66191101074219,
      "activations/layer1_attention_weight_max": 16.45128631591797,
      "activations/layer1_attention_weight_min": -14.515836715698242,
      "activations/layer20_attention_weight_max": 30.23931312561035,
      "activations/layer20_attention_weight_min": -24.855457305908203,
      "activations/layer21_attention_weight_max": 28.679338455200195,
      "activations/layer21_attention_weight_min": -24.522289276123047,
      "activations/layer22_attention_weight_max": 40.81956100463867,
      "activations/layer22_attention_weight_min": -32.06732940673828,
      "activations/layer23_attention_weight_max": 34.944034576416016,
      "activations/layer23_attention_weight_min": -25.66058921813965,
      "activations/layer2_attention_weight_max": 32.561275482177734,
      "activations/layer2_attention_weight_min": -32.56258773803711,
      "activations/layer3_attention_weight_max": 94.30253601074219,
      "activations/layer3_attention_weight_min": -96.40369415283203,
      "activations/layer4_attention_weight_max": 108.67225646972656,
      "activations/layer4_attention_weight_min": -113.56058502197266,
      "activations/layer5_attention_weight_max": 54.339622497558594,
      "activations/layer5_attention_weight_min": -61.44469451904297,
      "activations/layer6_attention_weight_max": 43.300941467285156,
      "activations/layer6_attention_weight_min": -45.02448272705078,
      "activations/layer7_attention_weight_max": 87.41230010986328,
      "activations/layer7_attention_weight_min": -92.74507141113281,
      "activations/layer8_attention_weight_max": 40.548397064208984,
      "activations/layer8_attention_weight_min": -41.13288497924805,
      "activations/layer9_attention_weight_max": 31.3479061126709,
      "activations/layer9_attention_weight_min": -32.553096771240234,
      "epoch": 13.99,
      "learning_rate": 6.041666666666666e-05,
      "loss": 2.7556,
      "step": 240700
    },
    {
      "activations/layer0_attention_weight_max": 15.548215866088867,
      "activations/layer0_attention_weight_min": -12.538586616516113,
      "activations/layer10_attention_weight_max": 34.059654235839844,
      "activations/layer10_attention_weight_min": -35.325782775878906,
      "activations/layer11_attention_weight_max": 33.882442474365234,
      "activations/layer11_attention_weight_min": -33.33509063720703,
      "activations/layer12_attention_weight_max": 26.576618194580078,
      "activations/layer12_attention_weight_min": -26.6441707611084,
      "activations/layer13_attention_weight_max": 39.09189987182617,
      "activations/layer13_attention_weight_min": -37.28276443481445,
      "activations/layer14_attention_weight_max": 51.86334991455078,
      "activations/layer14_attention_weight_min": -43.4492301940918,
      "activations/layer15_attention_weight_max": 37.1164665222168,
      "activations/layer15_attention_weight_min": -35.429718017578125,
      "activations/layer16_attention_weight_max": 29.846899032592773,
      "activations/layer16_attention_weight_min": -30.15089988708496,
      "activations/layer17_attention_weight_max": 31.618629455566406,
      "activations/layer17_attention_weight_min": -27.330432891845703,
      "activations/layer18_attention_weight_max": 33.758087158203125,
      "activations/layer18_attention_weight_min": -23.688037872314453,
      "activations/layer19_attention_weight_max": 33.37083435058594,
      "activations/layer19_attention_weight_min": -31.383779525756836,
      "activations/layer1_attention_weight_max": 16.411710739135742,
      "activations/layer1_attention_weight_min": -16.1727237701416,
      "activations/layer20_attention_weight_max": 30.357149124145508,
      "activations/layer20_attention_weight_min": -24.87093734741211,
      "activations/layer21_attention_weight_max": 29.851938247680664,
      "activations/layer21_attention_weight_min": -24.702404022216797,
      "activations/layer22_attention_weight_max": 46.707611083984375,
      "activations/layer22_attention_weight_min": -32.040191650390625,
      "activations/layer23_attention_weight_max": 35.19647216796875,
      "activations/layer23_attention_weight_min": -25.2294979095459,
      "activations/layer2_attention_weight_max": 33.44923400878906,
      "activations/layer2_attention_weight_min": -34.373291015625,
      "activations/layer3_attention_weight_max": 99.51524353027344,
      "activations/layer3_attention_weight_min": -98.71183776855469,
      "activations/layer4_attention_weight_max": 110.446044921875,
      "activations/layer4_attention_weight_min": -110.50091552734375,
      "activations/layer5_attention_weight_max": 50.28843688964844,
      "activations/layer5_attention_weight_min": -61.04462814331055,
      "activations/layer6_attention_weight_max": 42.88020706176758,
      "activations/layer6_attention_weight_min": -44.6374397277832,
      "activations/layer7_attention_weight_max": 91.84444427490234,
      "activations/layer7_attention_weight_min": -91.2148666381836,
      "activations/layer8_attention_weight_max": 42.14247131347656,
      "activations/layer8_attention_weight_min": -42.468963623046875,
      "activations/layer9_attention_weight_max": 34.51243591308594,
      "activations/layer9_attention_weight_min": -35.70044708251953,
      "epoch": 13.99,
      "learning_rate": 6.039772727272727e-05,
      "loss": 2.7591,
      "step": 240750
    },
    {
      "activations/layer0_attention_weight_max": 16.35333251953125,
      "activations/layer0_attention_weight_min": -11.945940971374512,
      "activations/layer10_attention_weight_max": 32.44097900390625,
      "activations/layer10_attention_weight_min": -31.987070083618164,
      "activations/layer11_attention_weight_max": 32.63022232055664,
      "activations/layer11_attention_weight_min": -31.733558654785156,
      "activations/layer12_attention_weight_max": 26.19396209716797,
      "activations/layer12_attention_weight_min": -25.546222686767578,
      "activations/layer13_attention_weight_max": 39.12606430053711,
      "activations/layer13_attention_weight_min": -35.046714782714844,
      "activations/layer14_attention_weight_max": 41.50171661376953,
      "activations/layer14_attention_weight_min": -38.0005989074707,
      "activations/layer15_attention_weight_max": 37.30466079711914,
      "activations/layer15_attention_weight_min": -33.598793029785156,
      "activations/layer16_attention_weight_max": 30.368038177490234,
      "activations/layer16_attention_weight_min": -28.011938095092773,
      "activations/layer17_attention_weight_max": 29.26422119140625,
      "activations/layer17_attention_weight_min": -27.682559967041016,
      "activations/layer18_attention_weight_max": 29.6529541015625,
      "activations/layer18_attention_weight_min": -24.83359146118164,
      "activations/layer19_attention_weight_max": 34.47605514526367,
      "activations/layer19_attention_weight_min": -31.092554092407227,
      "activations/layer1_attention_weight_max": 17.832433700561523,
      "activations/layer1_attention_weight_min": -13.049407005310059,
      "activations/layer20_attention_weight_max": 28.31814193725586,
      "activations/layer20_attention_weight_min": -24.66429328918457,
      "activations/layer21_attention_weight_max": 26.855533599853516,
      "activations/layer21_attention_weight_min": -24.786815643310547,
      "activations/layer22_attention_weight_max": 41.6639289855957,
      "activations/layer22_attention_weight_min": -33.171592712402344,
      "activations/layer23_attention_weight_max": 32.55815124511719,
      "activations/layer23_attention_weight_min": -27.282934188842773,
      "activations/layer2_attention_weight_max": 31.873023986816406,
      "activations/layer2_attention_weight_min": -30.768089294433594,
      "activations/layer3_attention_weight_max": 94.08322143554688,
      "activations/layer3_attention_weight_min": -92.92401123046875,
      "activations/layer4_attention_weight_max": 107.80281829833984,
      "activations/layer4_attention_weight_min": -112.42794036865234,
      "activations/layer5_attention_weight_max": 53.56367492675781,
      "activations/layer5_attention_weight_min": -65.89312744140625,
      "activations/layer6_attention_weight_max": 43.50188446044922,
      "activations/layer6_attention_weight_min": -45.68488311767578,
      "activations/layer7_attention_weight_max": 89.46376037597656,
      "activations/layer7_attention_weight_min": -90.5658950805664,
      "activations/layer8_attention_weight_max": 39.13658905029297,
      "activations/layer8_attention_weight_min": -43.701473236083984,
      "activations/layer9_attention_weight_max": 31.48085594177246,
      "activations/layer9_attention_weight_min": -34.29401779174805,
      "epoch": 13.99,
      "learning_rate": 6.037878787878787e-05,
      "loss": 2.7714,
      "step": 240800
    },
    {
      "activations/layer0_attention_weight_max": 16.421321868896484,
      "activations/layer0_attention_weight_min": -12.542808532714844,
      "activations/layer10_attention_weight_max": 32.858978271484375,
      "activations/layer10_attention_weight_min": -33.9952392578125,
      "activations/layer11_attention_weight_max": 32.69213104248047,
      "activations/layer11_attention_weight_min": -31.433658599853516,
      "activations/layer12_attention_weight_max": 26.180946350097656,
      "activations/layer12_attention_weight_min": -24.519323348999023,
      "activations/layer13_attention_weight_max": 38.671669006347656,
      "activations/layer13_attention_weight_min": -35.569271087646484,
      "activations/layer14_attention_weight_max": 41.714569091796875,
      "activations/layer14_attention_weight_min": -37.24795913696289,
      "activations/layer15_attention_weight_max": 36.49930191040039,
      "activations/layer15_attention_weight_min": -33.4785270690918,
      "activations/layer16_attention_weight_max": 29.978748321533203,
      "activations/layer16_attention_weight_min": -27.83186149597168,
      "activations/layer17_attention_weight_max": 30.07887840270996,
      "activations/layer17_attention_weight_min": -26.9868106842041,
      "activations/layer18_attention_weight_max": 33.136962890625,
      "activations/layer18_attention_weight_min": -25.296205520629883,
      "activations/layer19_attention_weight_max": 38.0947380065918,
      "activations/layer19_attention_weight_min": -30.94266128540039,
      "activations/layer1_attention_weight_max": 17.476381301879883,
      "activations/layer1_attention_weight_min": -14.97819995880127,
      "activations/layer20_attention_weight_max": 30.9617862701416,
      "activations/layer20_attention_weight_min": -24.730119705200195,
      "activations/layer21_attention_weight_max": 32.058719635009766,
      "activations/layer21_attention_weight_min": -24.192258834838867,
      "activations/layer22_attention_weight_max": 47.68486785888672,
      "activations/layer22_attention_weight_min": -31.450727462768555,
      "activations/layer23_attention_weight_max": 42.34658432006836,
      "activations/layer23_attention_weight_min": -25.499509811401367,
      "activations/layer2_attention_weight_max": 32.99104309082031,
      "activations/layer2_attention_weight_min": -32.29364776611328,
      "activations/layer3_attention_weight_max": 101.17031860351562,
      "activations/layer3_attention_weight_min": -98.86397552490234,
      "activations/layer4_attention_weight_max": 114.3517837524414,
      "activations/layer4_attention_weight_min": -114.75731658935547,
      "activations/layer5_attention_weight_max": 52.26629638671875,
      "activations/layer5_attention_weight_min": -65.09546661376953,
      "activations/layer6_attention_weight_max": 43.09303665161133,
      "activations/layer6_attention_weight_min": -48.69951629638672,
      "activations/layer7_attention_weight_max": 88.71026611328125,
      "activations/layer7_attention_weight_min": -92.97071075439453,
      "activations/layer8_attention_weight_max": 38.344078063964844,
      "activations/layer8_attention_weight_min": -42.47218704223633,
      "activations/layer9_attention_weight_max": 33.20623016357422,
      "activations/layer9_attention_weight_min": -34.72796630859375,
      "epoch": 13.99,
      "learning_rate": 6.0359848484848485e-05,
      "loss": 2.7472,
      "step": 240850
    },
    {
      "activations/layer0_attention_weight_max": 16.254356384277344,
      "activations/layer0_attention_weight_min": -12.533942222595215,
      "activations/layer10_attention_weight_max": 38.99517822265625,
      "activations/layer10_attention_weight_min": -35.108489990234375,
      "activations/layer11_attention_weight_max": 38.010963439941406,
      "activations/layer11_attention_weight_min": -36.11717224121094,
      "activations/layer12_attention_weight_max": 25.5213565826416,
      "activations/layer12_attention_weight_min": -27.200944900512695,
      "activations/layer13_attention_weight_max": 37.91621398925781,
      "activations/layer13_attention_weight_min": -38.1864128112793,
      "activations/layer14_attention_weight_max": 40.71965408325195,
      "activations/layer14_attention_weight_min": -37.98600769042969,
      "activations/layer15_attention_weight_max": 41.52263641357422,
      "activations/layer15_attention_weight_min": -34.50371170043945,
      "activations/layer16_attention_weight_max": 30.76676368713379,
      "activations/layer16_attention_weight_min": -27.394088745117188,
      "activations/layer17_attention_weight_max": 29.489967346191406,
      "activations/layer17_attention_weight_min": -26.902437210083008,
      "activations/layer18_attention_weight_max": 30.29024887084961,
      "activations/layer18_attention_weight_min": -26.5057315826416,
      "activations/layer19_attention_weight_max": 37.6154899597168,
      "activations/layer19_attention_weight_min": -29.685791015625,
      "activations/layer1_attention_weight_max": 16.507736206054688,
      "activations/layer1_attention_weight_min": -13.848302841186523,
      "activations/layer20_attention_weight_max": 31.086875915527344,
      "activations/layer20_attention_weight_min": -23.986186981201172,
      "activations/layer21_attention_weight_max": 29.469921112060547,
      "activations/layer21_attention_weight_min": -23.191055297851562,
      "activations/layer22_attention_weight_max": 41.6743049621582,
      "activations/layer22_attention_weight_min": -28.70833396911621,
      "activations/layer23_attention_weight_max": 36.007137298583984,
      "activations/layer23_attention_weight_min": -25.008581161499023,
      "activations/layer2_attention_weight_max": 33.450950622558594,
      "activations/layer2_attention_weight_min": -32.599185943603516,
      "activations/layer3_attention_weight_max": 94.48081970214844,
      "activations/layer3_attention_weight_min": -96.9284896850586,
      "activations/layer4_attention_weight_max": 108.16045379638672,
      "activations/layer4_attention_weight_min": -108.39373016357422,
      "activations/layer5_attention_weight_max": 52.45459747314453,
      "activations/layer5_attention_weight_min": -61.76493453979492,
      "activations/layer6_attention_weight_max": 47.62057876586914,
      "activations/layer6_attention_weight_min": -46.25326919555664,
      "activations/layer7_attention_weight_max": 98.06453704833984,
      "activations/layer7_attention_weight_min": -91.58854675292969,
      "activations/layer8_attention_weight_max": 46.8109016418457,
      "activations/layer8_attention_weight_min": -42.019996643066406,
      "activations/layer9_attention_weight_max": 40.81816864013672,
      "activations/layer9_attention_weight_min": -35.80110168457031,
      "epoch": 14.0,
      "learning_rate": 6.0340909090909087e-05,
      "loss": 2.7575,
      "step": 240900
    },
    {
      "activations/layer0_attention_weight_max": 16.923131942749023,
      "activations/layer0_attention_weight_min": -12.182961463928223,
      "activations/layer10_attention_weight_max": 33.02492904663086,
      "activations/layer10_attention_weight_min": -32.828025817871094,
      "activations/layer11_attention_weight_max": 33.110042572021484,
      "activations/layer11_attention_weight_min": -33.743125915527344,
      "activations/layer12_attention_weight_max": 24.96415138244629,
      "activations/layer12_attention_weight_min": -25.294687271118164,
      "activations/layer13_attention_weight_max": 38.159278869628906,
      "activations/layer13_attention_weight_min": -35.024742126464844,
      "activations/layer14_attention_weight_max": 40.5870246887207,
      "activations/layer14_attention_weight_min": -39.50436019897461,
      "activations/layer15_attention_weight_max": 35.379844665527344,
      "activations/layer15_attention_weight_min": -32.91286087036133,
      "activations/layer16_attention_weight_max": 30.571874618530273,
      "activations/layer16_attention_weight_min": -29.183677673339844,
      "activations/layer17_attention_weight_max": 30.284109115600586,
      "activations/layer17_attention_weight_min": -27.554115295410156,
      "activations/layer18_attention_weight_max": 33.07904052734375,
      "activations/layer18_attention_weight_min": -23.964128494262695,
      "activations/layer19_attention_weight_max": 35.17581558227539,
      "activations/layer19_attention_weight_min": -32.28569793701172,
      "activations/layer1_attention_weight_max": 16.199247360229492,
      "activations/layer1_attention_weight_min": -13.873635292053223,
      "activations/layer20_attention_weight_max": 30.79390525817871,
      "activations/layer20_attention_weight_min": -26.56260871887207,
      "activations/layer21_attention_weight_max": 31.030118942260742,
      "activations/layer21_attention_weight_min": -24.44339370727539,
      "activations/layer22_attention_weight_max": 43.89800262451172,
      "activations/layer22_attention_weight_min": -30.134281158447266,
      "activations/layer23_attention_weight_max": 37.98717498779297,
      "activations/layer23_attention_weight_min": -27.91156768798828,
      "activations/layer2_attention_weight_max": 32.681175231933594,
      "activations/layer2_attention_weight_min": -32.54761505126953,
      "activations/layer3_attention_weight_max": 94.92286682128906,
      "activations/layer3_attention_weight_min": -95.48080444335938,
      "activations/layer4_attention_weight_max": 110.85384368896484,
      "activations/layer4_attention_weight_min": -109.13392639160156,
      "activations/layer5_attention_weight_max": 52.88773727416992,
      "activations/layer5_attention_weight_min": -64.22036743164062,
      "activations/layer6_attention_weight_max": 44.45942687988281,
      "activations/layer6_attention_weight_min": -47.43050765991211,
      "activations/layer7_attention_weight_max": 92.09931182861328,
      "activations/layer7_attention_weight_min": -98.44322204589844,
      "activations/layer8_attention_weight_max": 40.6132698059082,
      "activations/layer8_attention_weight_min": -40.243106842041016,
      "activations/layer9_attention_weight_max": 31.05145263671875,
      "activations/layer9_attention_weight_min": -31.892988204956055,
      "epoch": 14.0,
      "learning_rate": 6.032196969696969e-05,
      "loss": 2.8189,
      "step": 240950
    },
    {
      "activations/layer0_attention_weight_max": 16.356733322143555,
      "activations/layer0_attention_weight_min": -12.992836952209473,
      "activations/layer10_attention_weight_max": 31.223541259765625,
      "activations/layer10_attention_weight_min": -32.404144287109375,
      "activations/layer11_attention_weight_max": 31.658294677734375,
      "activations/layer11_attention_weight_min": -32.125125885009766,
      "activations/layer12_attention_weight_max": 24.25190544128418,
      "activations/layer12_attention_weight_min": -24.486082077026367,
      "activations/layer13_attention_weight_max": 36.24001693725586,
      "activations/layer13_attention_weight_min": -33.131805419921875,
      "activations/layer14_attention_weight_max": 42.466129302978516,
      "activations/layer14_attention_weight_min": -34.56951904296875,
      "activations/layer15_attention_weight_max": 35.760982513427734,
      "activations/layer15_attention_weight_min": -31.7283878326416,
      "activations/layer16_attention_weight_max": 28.770404815673828,
      "activations/layer16_attention_weight_min": -29.551712036132812,
      "activations/layer17_attention_weight_max": 32.37957000732422,
      "activations/layer17_attention_weight_min": -24.290599822998047,
      "activations/layer18_attention_weight_max": 31.63253402709961,
      "activations/layer18_attention_weight_min": -22.766008377075195,
      "activations/layer19_attention_weight_max": 34.0573616027832,
      "activations/layer19_attention_weight_min": -31.554229736328125,
      "activations/layer1_attention_weight_max": 16.65232276916504,
      "activations/layer1_attention_weight_min": -13.883414268493652,
      "activations/layer20_attention_weight_max": 31.749332427978516,
      "activations/layer20_attention_weight_min": -25.160850524902344,
      "activations/layer21_attention_weight_max": 31.386857986450195,
      "activations/layer21_attention_weight_min": -23.66356658935547,
      "activations/layer22_attention_weight_max": 45.877044677734375,
      "activations/layer22_attention_weight_min": -29.63320541381836,
      "activations/layer23_attention_weight_max": 33.41360855102539,
      "activations/layer23_attention_weight_min": -22.660289764404297,
      "activations/layer2_attention_weight_max": 32.88786315917969,
      "activations/layer2_attention_weight_min": -31.62936782836914,
      "activations/layer3_attention_weight_max": 92.67359924316406,
      "activations/layer3_attention_weight_min": -95.8714370727539,
      "activations/layer4_attention_weight_max": 106.36396789550781,
      "activations/layer4_attention_weight_min": -113.11053466796875,
      "activations/layer5_attention_weight_max": 50.788108825683594,
      "activations/layer5_attention_weight_min": -60.61281967163086,
      "activations/layer6_attention_weight_max": 43.704532623291016,
      "activations/layer6_attention_weight_min": -48.08529281616211,
      "activations/layer7_attention_weight_max": 88.91226196289062,
      "activations/layer7_attention_weight_min": -90.08243560791016,
      "activations/layer8_attention_weight_max": 39.23280715942383,
      "activations/layer8_attention_weight_min": -40.41719055175781,
      "activations/layer9_attention_weight_max": 30.113391876220703,
      "activations/layer9_attention_weight_min": -32.669456481933594,
      "epoch": 14.0,
      "learning_rate": 6.0303030303030296e-05,
      "loss": 2.7519,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.5102,
      "eval_samples_per_second": 504.573,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.5102,
      "eval_openwebtext_samples_per_second": 504.573,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 2.0006,
      "eval_wikitext_samples_per_second": 227.928,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_lambada_loss": 2.4375,
      "eval_lambada_ppl": 11.444393964331121,
      "eval_lambada_runtime": 9.6457,
      "eval_lambada_samples_per_second": 504.784,
      "step": 241000
    },
    {
      "activations/layer0_attention_weight_max": 16.617191314697266,
      "activations/layer0_attention_weight_min": -12.526686668395996,
      "activations/layer10_attention_weight_max": 44.167423248291016,
      "activations/layer10_attention_weight_min": -43.606658935546875,
      "activations/layer11_attention_weight_max": 47.49656677246094,
      "activations/layer11_attention_weight_min": -47.09724044799805,
      "activations/layer12_attention_weight_max": 27.500411987304688,
      "activations/layer12_attention_weight_min": -28.257320404052734,
      "activations/layer13_attention_weight_max": 42.3393669128418,
      "activations/layer13_attention_weight_min": -37.27610397338867,
      "activations/layer14_attention_weight_max": 46.036338806152344,
      "activations/layer14_attention_weight_min": -37.871795654296875,
      "activations/layer15_attention_weight_max": 43.96351623535156,
      "activations/layer15_attention_weight_min": -40.45029067993164,
      "activations/layer16_attention_weight_max": 32.71870803833008,
      "activations/layer16_attention_weight_min": -28.76140022277832,
      "activations/layer17_attention_weight_max": 37.316322326660156,
      "activations/layer17_attention_weight_min": -30.552629470825195,
      "activations/layer18_attention_weight_max": 31.45684814453125,
      "activations/layer18_attention_weight_min": -25.88306999206543,
      "activations/layer19_attention_weight_max": 37.854007720947266,
      "activations/layer19_attention_weight_min": -32.38218307495117,
      "activations/layer1_attention_weight_max": 18.024290084838867,
      "activations/layer1_attention_weight_min": -14.014692306518555,
      "activations/layer20_attention_weight_max": 34.67435836791992,
      "activations/layer20_attention_weight_min": -26.508621215820312,
      "activations/layer21_attention_weight_max": 33.42456817626953,
      "activations/layer21_attention_weight_min": -24.90856170654297,
      "activations/layer22_attention_weight_max": 50.673702239990234,
      "activations/layer22_attention_weight_min": -33.05422592163086,
      "activations/layer23_attention_weight_max": 38.960304260253906,
      "activations/layer23_attention_weight_min": -26.003726959228516,
      "activations/layer2_attention_weight_max": 35.96449279785156,
      "activations/layer2_attention_weight_min": -37.42717361450195,
      "activations/layer3_attention_weight_max": 110.08584594726562,
      "activations/layer3_attention_weight_min": -111.4623794555664,
      "activations/layer4_attention_weight_max": 133.66064453125,
      "activations/layer4_attention_weight_min": -120.34471893310547,
      "activations/layer5_attention_weight_max": 67.8642349243164,
      "activations/layer5_attention_weight_min": -62.23513412475586,
      "activations/layer6_attention_weight_max": 50.83163833618164,
      "activations/layer6_attention_weight_min": -48.26481246948242,
      "activations/layer7_attention_weight_max": 119.29985809326172,
      "activations/layer7_attention_weight_min": -109.61705017089844,
      "activations/layer8_attention_weight_max": 54.80974197387695,
      "activations/layer8_attention_weight_min": -51.72697067260742,
      "activations/layer9_attention_weight_max": 41.35655212402344,
      "activations/layer9_attention_weight_min": -42.568172454833984,
      "epoch": 14.01,
      "learning_rate": 6.0284090909090905e-05,
      "loss": 2.7475,
      "step": 241050
    },
    {
      "activations/layer0_attention_weight_max": 17.351503372192383,
      "activations/layer0_attention_weight_min": -13.018852233886719,
      "activations/layer10_attention_weight_max": 35.837684631347656,
      "activations/layer10_attention_weight_min": -35.3463020324707,
      "activations/layer11_attention_weight_max": 31.13098907470703,
      "activations/layer11_attention_weight_min": -32.88665008544922,
      "activations/layer12_attention_weight_max": 25.994661331176758,
      "activations/layer12_attention_weight_min": -24.818050384521484,
      "activations/layer13_attention_weight_max": 36.824012756347656,
      "activations/layer13_attention_weight_min": -35.43684387207031,
      "activations/layer14_attention_weight_max": 40.30894088745117,
      "activations/layer14_attention_weight_min": -37.03849411010742,
      "activations/layer15_attention_weight_max": 34.775047302246094,
      "activations/layer15_attention_weight_min": -34.70938491821289,
      "activations/layer16_attention_weight_max": 28.13939094543457,
      "activations/layer16_attention_weight_min": -27.470951080322266,
      "activations/layer17_attention_weight_max": 31.36087989807129,
      "activations/layer17_attention_weight_min": -24.492624282836914,
      "activations/layer18_attention_weight_max": 32.73054122924805,
      "activations/layer18_attention_weight_min": -23.81100845336914,
      "activations/layer19_attention_weight_max": 33.77421569824219,
      "activations/layer19_attention_weight_min": -31.068256378173828,
      "activations/layer1_attention_weight_max": 15.708873748779297,
      "activations/layer1_attention_weight_min": -14.883462905883789,
      "activations/layer20_attention_weight_max": 28.55175018310547,
      "activations/layer20_attention_weight_min": -25.0402889251709,
      "activations/layer21_attention_weight_max": 29.281963348388672,
      "activations/layer21_attention_weight_min": -23.409832000732422,
      "activations/layer22_attention_weight_max": 45.93499755859375,
      "activations/layer22_attention_weight_min": -29.9718017578125,
      "activations/layer23_attention_weight_max": 34.087554931640625,
      "activations/layer23_attention_weight_min": -26.11848258972168,
      "activations/layer2_attention_weight_max": 33.313209533691406,
      "activations/layer2_attention_weight_min": -33.78797149658203,
      "activations/layer3_attention_weight_max": 95.65119171142578,
      "activations/layer3_attention_weight_min": -95.28327178955078,
      "activations/layer4_attention_weight_max": 110.34468078613281,
      "activations/layer4_attention_weight_min": -113.91429901123047,
      "activations/layer5_attention_weight_max": 53.18492126464844,
      "activations/layer5_attention_weight_min": -63.83590316772461,
      "activations/layer6_attention_weight_max": 47.20383834838867,
      "activations/layer6_attention_weight_min": -48.11698913574219,
      "activations/layer7_attention_weight_max": 89.33390808105469,
      "activations/layer7_attention_weight_min": -97.18875122070312,
      "activations/layer8_attention_weight_max": 41.3721809387207,
      "activations/layer8_attention_weight_min": -45.1778450012207,
      "activations/layer9_attention_weight_max": 34.1458854675293,
      "activations/layer9_attention_weight_min": -36.26683044433594,
      "epoch": 14.01,
      "learning_rate": 6.026515151515151e-05,
      "loss": 2.75,
      "step": 241100
    },
    {
      "activations/layer0_attention_weight_max": 17.02588653564453,
      "activations/layer0_attention_weight_min": -12.460988998413086,
      "activations/layer10_attention_weight_max": 33.26884841918945,
      "activations/layer10_attention_weight_min": -33.07904815673828,
      "activations/layer11_attention_weight_max": 33.44225311279297,
      "activations/layer11_attention_weight_min": -32.99393844604492,
      "activations/layer12_attention_weight_max": 24.471485137939453,
      "activations/layer12_attention_weight_min": -25.033178329467773,
      "activations/layer13_attention_weight_max": 41.00568389892578,
      "activations/layer13_attention_weight_min": -35.91429138183594,
      "activations/layer14_attention_weight_max": 45.33269500732422,
      "activations/layer14_attention_weight_min": -43.32366943359375,
      "activations/layer15_attention_weight_max": 36.74076461791992,
      "activations/layer15_attention_weight_min": -33.895423889160156,
      "activations/layer16_attention_weight_max": 32.5457763671875,
      "activations/layer16_attention_weight_min": -28.502588272094727,
      "activations/layer17_attention_weight_max": 31.109487533569336,
      "activations/layer17_attention_weight_min": -25.897130966186523,
      "activations/layer18_attention_weight_max": 38.5882568359375,
      "activations/layer18_attention_weight_min": -26.566574096679688,
      "activations/layer19_attention_weight_max": 37.7202033996582,
      "activations/layer19_attention_weight_min": -32.9771728515625,
      "activations/layer1_attention_weight_max": 16.659828186035156,
      "activations/layer1_attention_weight_min": -15.482630729675293,
      "activations/layer20_attention_weight_max": 32.89967727661133,
      "activations/layer20_attention_weight_min": -25.520278930664062,
      "activations/layer21_attention_weight_max": 29.717485427856445,
      "activations/layer21_attention_weight_min": -25.13500213623047,
      "activations/layer22_attention_weight_max": 49.33570098876953,
      "activations/layer22_attention_weight_min": -29.820207595825195,
      "activations/layer23_attention_weight_max": 35.10840606689453,
      "activations/layer23_attention_weight_min": -25.450193405151367,
      "activations/layer2_attention_weight_max": 33.33144760131836,
      "activations/layer2_attention_weight_min": -33.349796295166016,
      "activations/layer3_attention_weight_max": 102.23954010009766,
      "activations/layer3_attention_weight_min": -99.76964569091797,
      "activations/layer4_attention_weight_max": 111.17962646484375,
      "activations/layer4_attention_weight_min": -115.34500885009766,
      "activations/layer5_attention_weight_max": 53.45158004760742,
      "activations/layer5_attention_weight_min": -68.06486511230469,
      "activations/layer6_attention_weight_max": 46.490455627441406,
      "activations/layer6_attention_weight_min": -49.298458099365234,
      "activations/layer7_attention_weight_max": 96.22713470458984,
      "activations/layer7_attention_weight_min": -96.01924896240234,
      "activations/layer8_attention_weight_max": 41.46144485473633,
      "activations/layer8_attention_weight_min": -43.097320556640625,
      "activations/layer9_attention_weight_max": 34.26401138305664,
      "activations/layer9_attention_weight_min": -33.6368522644043,
      "epoch": 14.01,
      "learning_rate": 6.0246212121212114e-05,
      "loss": 2.7548,
      "step": 241150
    },
    {
      "activations/layer0_attention_weight_max": 17.004079818725586,
      "activations/layer0_attention_weight_min": -12.96035385131836,
      "activations/layer10_attention_weight_max": 34.341766357421875,
      "activations/layer10_attention_weight_min": -33.028961181640625,
      "activations/layer11_attention_weight_max": 33.329124450683594,
      "activations/layer11_attention_weight_min": -33.01559066772461,
      "activations/layer12_attention_weight_max": 25.044517517089844,
      "activations/layer12_attention_weight_min": -25.266599655151367,
      "activations/layer13_attention_weight_max": 39.61039733886719,
      "activations/layer13_attention_weight_min": -36.05577087402344,
      "activations/layer14_attention_weight_max": 41.72011184692383,
      "activations/layer14_attention_weight_min": -36.430171966552734,
      "activations/layer15_attention_weight_max": 36.34284591674805,
      "activations/layer15_attention_weight_min": -31.947389602661133,
      "activations/layer16_attention_weight_max": 29.305782318115234,
      "activations/layer16_attention_weight_min": -27.45909309387207,
      "activations/layer17_attention_weight_max": 30.37754249572754,
      "activations/layer17_attention_weight_min": -27.241519927978516,
      "activations/layer18_attention_weight_max": 30.3585205078125,
      "activations/layer18_attention_weight_min": -28.408992767333984,
      "activations/layer19_attention_weight_max": 34.96198654174805,
      "activations/layer19_attention_weight_min": -31.77504539489746,
      "activations/layer1_attention_weight_max": 16.22391700744629,
      "activations/layer1_attention_weight_min": -15.598599433898926,
      "activations/layer20_attention_weight_max": 31.5458927154541,
      "activations/layer20_attention_weight_min": -26.428855895996094,
      "activations/layer21_attention_weight_max": 28.352134704589844,
      "activations/layer21_attention_weight_min": -27.01561164855957,
      "activations/layer22_attention_weight_max": 43.826690673828125,
      "activations/layer22_attention_weight_min": -32.51405715942383,
      "activations/layer23_attention_weight_max": 33.21139144897461,
      "activations/layer23_attention_weight_min": -25.817699432373047,
      "activations/layer2_attention_weight_max": 33.33806610107422,
      "activations/layer2_attention_weight_min": -33.445533752441406,
      "activations/layer3_attention_weight_max": 98.95809173583984,
      "activations/layer3_attention_weight_min": -97.02510833740234,
      "activations/layer4_attention_weight_max": 115.62718963623047,
      "activations/layer4_attention_weight_min": -111.92156982421875,
      "activations/layer5_attention_weight_max": 53.07188415527344,
      "activations/layer5_attention_weight_min": -65.95228576660156,
      "activations/layer6_attention_weight_max": 43.57254409790039,
      "activations/layer6_attention_weight_min": -47.52237319946289,
      "activations/layer7_attention_weight_max": 93.07084655761719,
      "activations/layer7_attention_weight_min": -93.74092864990234,
      "activations/layer8_attention_weight_max": 40.12894058227539,
      "activations/layer8_attention_weight_min": -42.79585647583008,
      "activations/layer9_attention_weight_max": 33.5503044128418,
      "activations/layer9_attention_weight_min": -34.544795989990234,
      "epoch": 14.02,
      "learning_rate": 6.0227272727272716e-05,
      "loss": 2.7535,
      "step": 241200
    },
    {
      "activations/layer0_attention_weight_max": 15.112021446228027,
      "activations/layer0_attention_weight_min": -12.588894844055176,
      "activations/layer10_attention_weight_max": 33.86909484863281,
      "activations/layer10_attention_weight_min": -31.45615577697754,
      "activations/layer11_attention_weight_max": 33.85899353027344,
      "activations/layer11_attention_weight_min": -31.890968322753906,
      "activations/layer12_attention_weight_max": 26.301517486572266,
      "activations/layer12_attention_weight_min": -28.47622299194336,
      "activations/layer13_attention_weight_max": 43.390342712402344,
      "activations/layer13_attention_weight_min": -35.653072357177734,
      "activations/layer14_attention_weight_max": 44.307220458984375,
      "activations/layer14_attention_weight_min": -38.28114318847656,
      "activations/layer15_attention_weight_max": 40.08989334106445,
      "activations/layer15_attention_weight_min": -34.499847412109375,
      "activations/layer16_attention_weight_max": 31.794599533081055,
      "activations/layer16_attention_weight_min": -30.14410400390625,
      "activations/layer17_attention_weight_max": 32.053306579589844,
      "activations/layer17_attention_weight_min": -26.218305587768555,
      "activations/layer18_attention_weight_max": 32.00376510620117,
      "activations/layer18_attention_weight_min": -26.039098739624023,
      "activations/layer19_attention_weight_max": 35.75556564331055,
      "activations/layer19_attention_weight_min": -32.53573226928711,
      "activations/layer1_attention_weight_max": 16.46862030029297,
      "activations/layer1_attention_weight_min": -15.23263931274414,
      "activations/layer20_attention_weight_max": 30.22488021850586,
      "activations/layer20_attention_weight_min": -25.3862361907959,
      "activations/layer21_attention_weight_max": 32.362083435058594,
      "activations/layer21_attention_weight_min": -25.350017547607422,
      "activations/layer22_attention_weight_max": 43.03239440917969,
      "activations/layer22_attention_weight_min": -31.288801193237305,
      "activations/layer23_attention_weight_max": 34.73991775512695,
      "activations/layer23_attention_weight_min": -24.810619354248047,
      "activations/layer2_attention_weight_max": 33.15327453613281,
      "activations/layer2_attention_weight_min": -31.127880096435547,
      "activations/layer3_attention_weight_max": 94.28792572021484,
      "activations/layer3_attention_weight_min": -95.53504180908203,
      "activations/layer4_attention_weight_max": 111.0825424194336,
      "activations/layer4_attention_weight_min": -107.2152099609375,
      "activations/layer5_attention_weight_max": 50.481781005859375,
      "activations/layer5_attention_weight_min": -60.77924346923828,
      "activations/layer6_attention_weight_max": 41.69023513793945,
      "activations/layer6_attention_weight_min": -45.569828033447266,
      "activations/layer7_attention_weight_max": 88.11968994140625,
      "activations/layer7_attention_weight_min": -87.51773071289062,
      "activations/layer8_attention_weight_max": 40.18620300292969,
      "activations/layer8_attention_weight_min": -39.88166809082031,
      "activations/layer9_attention_weight_max": 34.74154281616211,
      "activations/layer9_attention_weight_min": -32.179439544677734,
      "epoch": 14.02,
      "learning_rate": 6.020833333333333e-05,
      "loss": 2.7476,
      "step": 241250
    },
    {
      "activations/layer0_attention_weight_max": 16.322416305541992,
      "activations/layer0_attention_weight_min": -13.01773452758789,
      "activations/layer10_attention_weight_max": 33.368900299072266,
      "activations/layer10_attention_weight_min": -32.14453887939453,
      "activations/layer11_attention_weight_max": 32.918514251708984,
      "activations/layer11_attention_weight_min": -32.71246337890625,
      "activations/layer12_attention_weight_max": 24.712120056152344,
      "activations/layer12_attention_weight_min": -26.757240295410156,
      "activations/layer13_attention_weight_max": 37.11855697631836,
      "activations/layer13_attention_weight_min": -36.12923049926758,
      "activations/layer14_attention_weight_max": 39.14528274536133,
      "activations/layer14_attention_weight_min": -37.429412841796875,
      "activations/layer15_attention_weight_max": 35.23444366455078,
      "activations/layer15_attention_weight_min": -34.99452590942383,
      "activations/layer16_attention_weight_max": 30.862199783325195,
      "activations/layer16_attention_weight_min": -30.79644775390625,
      "activations/layer17_attention_weight_max": 30.280120849609375,
      "activations/layer17_attention_weight_min": -25.845426559448242,
      "activations/layer18_attention_weight_max": 29.012075424194336,
      "activations/layer18_attention_weight_min": -23.440406799316406,
      "activations/layer19_attention_weight_max": 30.022138595581055,
      "activations/layer19_attention_weight_min": -29.824247360229492,
      "activations/layer1_attention_weight_max": 17.088878631591797,
      "activations/layer1_attention_weight_min": -15.844768524169922,
      "activations/layer20_attention_weight_max": 27.224626541137695,
      "activations/layer20_attention_weight_min": -23.726173400878906,
      "activations/layer21_attention_weight_max": 26.309783935546875,
      "activations/layer21_attention_weight_min": -25.017202377319336,
      "activations/layer22_attention_weight_max": 37.961341857910156,
      "activations/layer22_attention_weight_min": -31.109270095825195,
      "activations/layer23_attention_weight_max": 32.368568420410156,
      "activations/layer23_attention_weight_min": -22.446399688720703,
      "activations/layer2_attention_weight_max": 32.306373596191406,
      "activations/layer2_attention_weight_min": -31.349666595458984,
      "activations/layer3_attention_weight_max": 94.683837890625,
      "activations/layer3_attention_weight_min": -92.81303405761719,
      "activations/layer4_attention_weight_max": 101.24151611328125,
      "activations/layer4_attention_weight_min": -108.00748443603516,
      "activations/layer5_attention_weight_max": 51.77748107910156,
      "activations/layer5_attention_weight_min": -62.19016647338867,
      "activations/layer6_attention_weight_max": 43.58124542236328,
      "activations/layer6_attention_weight_min": -44.32843780517578,
      "activations/layer7_attention_weight_max": 86.72084045410156,
      "activations/layer7_attention_weight_min": -88.57095336914062,
      "activations/layer8_attention_weight_max": 38.87715530395508,
      "activations/layer8_attention_weight_min": -39.81270217895508,
      "activations/layer9_attention_weight_max": 31.07999038696289,
      "activations/layer9_attention_weight_min": -33.00397872924805,
      "epoch": 14.02,
      "learning_rate": 6.018939393939393e-05,
      "loss": 2.7385,
      "step": 241300
    },
    {
      "activations/layer0_attention_weight_max": 16.22799301147461,
      "activations/layer0_attention_weight_min": -13.291632652282715,
      "activations/layer10_attention_weight_max": 38.008941650390625,
      "activations/layer10_attention_weight_min": -36.760986328125,
      "activations/layer11_attention_weight_max": 39.94825744628906,
      "activations/layer11_attention_weight_min": -37.379852294921875,
      "activations/layer12_attention_weight_max": 27.590959548950195,
      "activations/layer12_attention_weight_min": -26.09929656982422,
      "activations/layer13_attention_weight_max": 42.58189010620117,
      "activations/layer13_attention_weight_min": -34.811771392822266,
      "activations/layer14_attention_weight_max": 46.107818603515625,
      "activations/layer14_attention_weight_min": -37.38084411621094,
      "activations/layer15_attention_weight_max": 41.84670639038086,
      "activations/layer15_attention_weight_min": -34.23318099975586,
      "activations/layer16_attention_weight_max": 32.1945686340332,
      "activations/layer16_attention_weight_min": -27.345836639404297,
      "activations/layer17_attention_weight_max": 32.68254470825195,
      "activations/layer17_attention_weight_min": -25.98784065246582,
      "activations/layer18_attention_weight_max": 33.29753875732422,
      "activations/layer18_attention_weight_min": -26.167455673217773,
      "activations/layer19_attention_weight_max": 35.150550842285156,
      "activations/layer19_attention_weight_min": -31.77034568786621,
      "activations/layer1_attention_weight_max": 16.22015953063965,
      "activations/layer1_attention_weight_min": -14.879586219787598,
      "activations/layer20_attention_weight_max": 29.793079376220703,
      "activations/layer20_attention_weight_min": -30.80849266052246,
      "activations/layer21_attention_weight_max": 30.68410873413086,
      "activations/layer21_attention_weight_min": -27.213109970092773,
      "activations/layer22_attention_weight_max": 43.74142074584961,
      "activations/layer22_attention_weight_min": -32.81079864501953,
      "activations/layer23_attention_weight_max": 33.531219482421875,
      "activations/layer23_attention_weight_min": -27.798534393310547,
      "activations/layer2_attention_weight_max": 34.509986877441406,
      "activations/layer2_attention_weight_min": -34.867584228515625,
      "activations/layer3_attention_weight_max": 103.07569885253906,
      "activations/layer3_attention_weight_min": -103.33500671386719,
      "activations/layer4_attention_weight_max": 116.87029266357422,
      "activations/layer4_attention_weight_min": -114.54902648925781,
      "activations/layer5_attention_weight_max": 54.669212341308594,
      "activations/layer5_attention_weight_min": -62.6261100769043,
      "activations/layer6_attention_weight_max": 45.49350357055664,
      "activations/layer6_attention_weight_min": -47.81616973876953,
      "activations/layer7_attention_weight_max": 94.28700256347656,
      "activations/layer7_attention_weight_min": -94.96732330322266,
      "activations/layer8_attention_weight_max": 42.59765625,
      "activations/layer8_attention_weight_min": -45.20601272583008,
      "activations/layer9_attention_weight_max": 36.61637496948242,
      "activations/layer9_attention_weight_min": -38.214813232421875,
      "epoch": 14.02,
      "learning_rate": 6.017045454545454e-05,
      "loss": 2.7481,
      "step": 241350
    },
    {
      "activations/layer0_attention_weight_max": 15.520055770874023,
      "activations/layer0_attention_weight_min": -13.15058422088623,
      "activations/layer10_attention_weight_max": 34.10187911987305,
      "activations/layer10_attention_weight_min": -33.51518249511719,
      "activations/layer11_attention_weight_max": 33.72888946533203,
      "activations/layer11_attention_weight_min": -33.05879592895508,
      "activations/layer12_attention_weight_max": 26.073673248291016,
      "activations/layer12_attention_weight_min": -24.431371688842773,
      "activations/layer13_attention_weight_max": 38.13737869262695,
      "activations/layer13_attention_weight_min": -33.07256317138672,
      "activations/layer14_attention_weight_max": 40.257415771484375,
      "activations/layer14_attention_weight_min": -37.41781234741211,
      "activations/layer15_attention_weight_max": 34.92619323730469,
      "activations/layer15_attention_weight_min": -31.232009887695312,
      "activations/layer16_attention_weight_max": 28.61652374267578,
      "activations/layer16_attention_weight_min": -27.565073013305664,
      "activations/layer17_attention_weight_max": 31.396339416503906,
      "activations/layer17_attention_weight_min": -24.974693298339844,
      "activations/layer18_attention_weight_max": 32.168670654296875,
      "activations/layer18_attention_weight_min": -22.08744239807129,
      "activations/layer19_attention_weight_max": 35.807430267333984,
      "activations/layer19_attention_weight_min": -28.39725685119629,
      "activations/layer1_attention_weight_max": 15.95292854309082,
      "activations/layer1_attention_weight_min": -14.62044620513916,
      "activations/layer20_attention_weight_max": 28.5251407623291,
      "activations/layer20_attention_weight_min": -23.985063552856445,
      "activations/layer21_attention_weight_max": 30.196434020996094,
      "activations/layer21_attention_weight_min": -23.625150680541992,
      "activations/layer22_attention_weight_max": 42.140960693359375,
      "activations/layer22_attention_weight_min": -29.713438034057617,
      "activations/layer23_attention_weight_max": 36.010536193847656,
      "activations/layer23_attention_weight_min": -23.240829467773438,
      "activations/layer2_attention_weight_max": 33.02275466918945,
      "activations/layer2_attention_weight_min": -32.29878616333008,
      "activations/layer3_attention_weight_max": 95.50502014160156,
      "activations/layer3_attention_weight_min": -97.72412872314453,
      "activations/layer4_attention_weight_max": 109.28084564208984,
      "activations/layer4_attention_weight_min": -108.03633117675781,
      "activations/layer5_attention_weight_max": 54.866416931152344,
      "activations/layer5_attention_weight_min": -64.59117126464844,
      "activations/layer6_attention_weight_max": 45.539363861083984,
      "activations/layer6_attention_weight_min": -45.23643493652344,
      "activations/layer7_attention_weight_max": 91.92870330810547,
      "activations/layer7_attention_weight_min": -92.58077239990234,
      "activations/layer8_attention_weight_max": 41.014442443847656,
      "activations/layer8_attention_weight_min": -41.915035247802734,
      "activations/layer9_attention_weight_max": 32.65060043334961,
      "activations/layer9_attention_weight_min": -32.50616455078125,
      "epoch": 14.03,
      "learning_rate": 6.015151515151514e-05,
      "loss": 2.7494,
      "step": 241400
    },
    {
      "activations/layer0_attention_weight_max": 16.099613189697266,
      "activations/layer0_attention_weight_min": -13.062727928161621,
      "activations/layer10_attention_weight_max": 31.385955810546875,
      "activations/layer10_attention_weight_min": -33.03558349609375,
      "activations/layer11_attention_weight_max": 31.221466064453125,
      "activations/layer11_attention_weight_min": -33.872066497802734,
      "activations/layer12_attention_weight_max": 25.266334533691406,
      "activations/layer12_attention_weight_min": -25.92814064025879,
      "activations/layer13_attention_weight_max": 39.48396682739258,
      "activations/layer13_attention_weight_min": -33.8674201965332,
      "activations/layer14_attention_weight_max": 43.80643081665039,
      "activations/layer14_attention_weight_min": -36.94014358520508,
      "activations/layer15_attention_weight_max": 41.81519317626953,
      "activations/layer15_attention_weight_min": -33.36017990112305,
      "activations/layer16_attention_weight_max": 31.910877227783203,
      "activations/layer16_attention_weight_min": -28.94637107849121,
      "activations/layer17_attention_weight_max": 35.65567398071289,
      "activations/layer17_attention_weight_min": -25.03322982788086,
      "activations/layer18_attention_weight_max": 32.27131271362305,
      "activations/layer18_attention_weight_min": -24.474515914916992,
      "activations/layer19_attention_weight_max": 37.8182487487793,
      "activations/layer19_attention_weight_min": -31.690977096557617,
      "activations/layer1_attention_weight_max": 15.930513381958008,
      "activations/layer1_attention_weight_min": -14.261161804199219,
      "activations/layer20_attention_weight_max": 34.734867095947266,
      "activations/layer20_attention_weight_min": -25.020584106445312,
      "activations/layer21_attention_weight_max": 30.83298110961914,
      "activations/layer21_attention_weight_min": -24.454504013061523,
      "activations/layer22_attention_weight_max": 45.651554107666016,
      "activations/layer22_attention_weight_min": -30.61810874938965,
      "activations/layer23_attention_weight_max": 36.032955169677734,
      "activations/layer23_attention_weight_min": -24.57982635498047,
      "activations/layer2_attention_weight_max": 33.03049850463867,
      "activations/layer2_attention_weight_min": -32.63441467285156,
      "activations/layer3_attention_weight_max": 94.36323547363281,
      "activations/layer3_attention_weight_min": -92.67887115478516,
      "activations/layer4_attention_weight_max": 104.48530578613281,
      "activations/layer4_attention_weight_min": -106.78889465332031,
      "activations/layer5_attention_weight_max": 52.40766906738281,
      "activations/layer5_attention_weight_min": -61.535545349121094,
      "activations/layer6_attention_weight_max": 43.38136672973633,
      "activations/layer6_attention_weight_min": -44.195701599121094,
      "activations/layer7_attention_weight_max": 87.6942138671875,
      "activations/layer7_attention_weight_min": -88.45704650878906,
      "activations/layer8_attention_weight_max": 40.870670318603516,
      "activations/layer8_attention_weight_min": -40.39582824707031,
      "activations/layer9_attention_weight_max": 31.285831451416016,
      "activations/layer9_attention_weight_min": -33.85097885131836,
      "epoch": 14.03,
      "learning_rate": 6.013257575757576e-05,
      "loss": 2.7466,
      "step": 241450
    },
    {
      "activations/layer0_attention_weight_max": 16.615827560424805,
      "activations/layer0_attention_weight_min": -12.962236404418945,
      "activations/layer10_attention_weight_max": 30.829021453857422,
      "activations/layer10_attention_weight_min": -32.868568420410156,
      "activations/layer11_attention_weight_max": 30.776716232299805,
      "activations/layer11_attention_weight_min": -32.06029510498047,
      "activations/layer12_attention_weight_max": 24.652950286865234,
      "activations/layer12_attention_weight_min": -24.820505142211914,
      "activations/layer13_attention_weight_max": 38.37654495239258,
      "activations/layer13_attention_weight_min": -33.78807067871094,
      "activations/layer14_attention_weight_max": 43.53255844116211,
      "activations/layer14_attention_weight_min": -35.593807220458984,
      "activations/layer15_attention_weight_max": 39.174957275390625,
      "activations/layer15_attention_weight_min": -34.00568771362305,
      "activations/layer16_attention_weight_max": 32.456825256347656,
      "activations/layer16_attention_weight_min": -29.52324104309082,
      "activations/layer17_attention_weight_max": 31.116933822631836,
      "activations/layer17_attention_weight_min": -28.358652114868164,
      "activations/layer18_attention_weight_max": 30.830312728881836,
      "activations/layer18_attention_weight_min": -25.72090721130371,
      "activations/layer19_attention_weight_max": 31.04518699645996,
      "activations/layer19_attention_weight_min": -31.6768856048584,
      "activations/layer1_attention_weight_max": 16.187416076660156,
      "activations/layer1_attention_weight_min": -14.672656059265137,
      "activations/layer20_attention_weight_max": 27.350669860839844,
      "activations/layer20_attention_weight_min": -23.85204315185547,
      "activations/layer21_attention_weight_max": 26.221574783325195,
      "activations/layer21_attention_weight_min": -23.660202026367188,
      "activations/layer22_attention_weight_max": 42.9185905456543,
      "activations/layer22_attention_weight_min": -31.861034393310547,
      "activations/layer23_attention_weight_max": 39.04186248779297,
      "activations/layer23_attention_weight_min": -26.68776512145996,
      "activations/layer2_attention_weight_max": 33.49091720581055,
      "activations/layer2_attention_weight_min": -33.10752487182617,
      "activations/layer3_attention_weight_max": 96.50785827636719,
      "activations/layer3_attention_weight_min": -96.87348937988281,
      "activations/layer4_attention_weight_max": 111.14137268066406,
      "activations/layer4_attention_weight_min": -111.2056655883789,
      "activations/layer5_attention_weight_max": 51.98484802246094,
      "activations/layer5_attention_weight_min": -59.83156967163086,
      "activations/layer6_attention_weight_max": 43.063926696777344,
      "activations/layer6_attention_weight_min": -47.52683639526367,
      "activations/layer7_attention_weight_max": 87.95838165283203,
      "activations/layer7_attention_weight_min": -94.42674255371094,
      "activations/layer8_attention_weight_max": 41.544551849365234,
      "activations/layer8_attention_weight_min": -46.44082260131836,
      "activations/layer9_attention_weight_max": 31.383834838867188,
      "activations/layer9_attention_weight_min": -33.872676849365234,
      "epoch": 14.03,
      "learning_rate": 6.011363636363636e-05,
      "loss": 2.7529,
      "step": 241500
    },
    {
      "activations/layer0_attention_weight_max": 16.12091064453125,
      "activations/layer0_attention_weight_min": -11.826325416564941,
      "activations/layer10_attention_weight_max": 34.366676330566406,
      "activations/layer10_attention_weight_min": -33.06869125366211,
      "activations/layer11_attention_weight_max": 34.31378173828125,
      "activations/layer11_attention_weight_min": -31.994230270385742,
      "activations/layer12_attention_weight_max": 26.844608306884766,
      "activations/layer12_attention_weight_min": -27.406436920166016,
      "activations/layer13_attention_weight_max": 44.46873474121094,
      "activations/layer13_attention_weight_min": -36.060218811035156,
      "activations/layer14_attention_weight_max": 48.34284210205078,
      "activations/layer14_attention_weight_min": -39.78730010986328,
      "activations/layer15_attention_weight_max": 38.74703598022461,
      "activations/layer15_attention_weight_min": -35.52164077758789,
      "activations/layer16_attention_weight_max": 31.320865631103516,
      "activations/layer16_attention_weight_min": -28.222944259643555,
      "activations/layer17_attention_weight_max": 29.784881591796875,
      "activations/layer17_attention_weight_min": -25.664072036743164,
      "activations/layer18_attention_weight_max": 34.70014190673828,
      "activations/layer18_attention_weight_min": -23.68168067932129,
      "activations/layer19_attention_weight_max": 34.32206344604492,
      "activations/layer19_attention_weight_min": -32.59463119506836,
      "activations/layer1_attention_weight_max": 16.581769943237305,
      "activations/layer1_attention_weight_min": -15.385355949401855,
      "activations/layer20_attention_weight_max": 29.630693435668945,
      "activations/layer20_attention_weight_min": -22.917675018310547,
      "activations/layer21_attention_weight_max": 28.638378143310547,
      "activations/layer21_attention_weight_min": -22.176620483398438,
      "activations/layer22_attention_weight_max": 43.72642135620117,
      "activations/layer22_attention_weight_min": -29.093902587890625,
      "activations/layer23_attention_weight_max": 35.692386627197266,
      "activations/layer23_attention_weight_min": -23.113645553588867,
      "activations/layer2_attention_weight_max": 33.5320930480957,
      "activations/layer2_attention_weight_min": -32.71013259887695,
      "activations/layer3_attention_weight_max": 97.51387023925781,
      "activations/layer3_attention_weight_min": -99.05952453613281,
      "activations/layer4_attention_weight_max": 108.5638427734375,
      "activations/layer4_attention_weight_min": -110.85478973388672,
      "activations/layer5_attention_weight_max": 50.22391891479492,
      "activations/layer5_attention_weight_min": -63.39069747924805,
      "activations/layer6_attention_weight_max": 44.18210983276367,
      "activations/layer6_attention_weight_min": -49.00364685058594,
      "activations/layer7_attention_weight_max": 91.73342895507812,
      "activations/layer7_attention_weight_min": -97.57086944580078,
      "activations/layer8_attention_weight_max": 40.69175338745117,
      "activations/layer8_attention_weight_min": -41.60850143432617,
      "activations/layer9_attention_weight_max": 35.320655822753906,
      "activations/layer9_attention_weight_min": -32.85286331176758,
      "epoch": 14.04,
      "learning_rate": 6.009469696969696e-05,
      "loss": 2.7538,
      "step": 241550
    },
    {
      "activations/layer0_attention_weight_max": 15.025400161743164,
      "activations/layer0_attention_weight_min": -12.579790115356445,
      "activations/layer10_attention_weight_max": 35.041107177734375,
      "activations/layer10_attention_weight_min": -33.985511779785156,
      "activations/layer11_attention_weight_max": 33.64293670654297,
      "activations/layer11_attention_weight_min": -33.02532196044922,
      "activations/layer12_attention_weight_max": 29.721757888793945,
      "activations/layer12_attention_weight_min": -24.712778091430664,
      "activations/layer13_attention_weight_max": 40.10297775268555,
      "activations/layer13_attention_weight_min": -34.327423095703125,
      "activations/layer14_attention_weight_max": 43.55338668823242,
      "activations/layer14_attention_weight_min": -39.362823486328125,
      "activations/layer15_attention_weight_max": 38.80992889404297,
      "activations/layer15_attention_weight_min": -34.25493240356445,
      "activations/layer16_attention_weight_max": 31.413785934448242,
      "activations/layer16_attention_weight_min": -28.331647872924805,
      "activations/layer17_attention_weight_max": 29.8077335357666,
      "activations/layer17_attention_weight_min": -26.96209144592285,
      "activations/layer18_attention_weight_max": 30.500993728637695,
      "activations/layer18_attention_weight_min": -22.80025863647461,
      "activations/layer19_attention_weight_max": 33.081172943115234,
      "activations/layer19_attention_weight_min": -29.6905574798584,
      "activations/layer1_attention_weight_max": 16.95716094970703,
      "activations/layer1_attention_weight_min": -14.359186172485352,
      "activations/layer20_attention_weight_max": 30.652481079101562,
      "activations/layer20_attention_weight_min": -22.7507381439209,
      "activations/layer21_attention_weight_max": 30.732406616210938,
      "activations/layer21_attention_weight_min": -22.63134765625,
      "activations/layer22_attention_weight_max": 43.31814956665039,
      "activations/layer22_attention_weight_min": -28.763410568237305,
      "activations/layer23_attention_weight_max": 33.03534698486328,
      "activations/layer23_attention_weight_min": -24.495407104492188,
      "activations/layer2_attention_weight_max": 34.36093521118164,
      "activations/layer2_attention_weight_min": -32.6247673034668,
      "activations/layer3_attention_weight_max": 95.25007629394531,
      "activations/layer3_attention_weight_min": -95.80432891845703,
      "activations/layer4_attention_weight_max": 106.30401611328125,
      "activations/layer4_attention_weight_min": -104.46402740478516,
      "activations/layer5_attention_weight_max": 50.98933029174805,
      "activations/layer5_attention_weight_min": -64.45475006103516,
      "activations/layer6_attention_weight_max": 45.023799896240234,
      "activations/layer6_attention_weight_min": -49.76221466064453,
      "activations/layer7_attention_weight_max": 92.9095687866211,
      "activations/layer7_attention_weight_min": -93.17827606201172,
      "activations/layer8_attention_weight_max": 42.49897766113281,
      "activations/layer8_attention_weight_min": -42.40398406982422,
      "activations/layer9_attention_weight_max": 32.869384765625,
      "activations/layer9_attention_weight_min": -34.694129943847656,
      "epoch": 14.04,
      "learning_rate": 6.0075757575757575e-05,
      "loss": 2.7677,
      "step": 241600
    },
    {
      "activations/layer0_attention_weight_max": 15.680317878723145,
      "activations/layer0_attention_weight_min": -12.527730941772461,
      "activations/layer10_attention_weight_max": 36.921627044677734,
      "activations/layer10_attention_weight_min": -35.648494720458984,
      "activations/layer11_attention_weight_max": 34.07563781738281,
      "activations/layer11_attention_weight_min": -35.64796829223633,
      "activations/layer12_attention_weight_max": 24.15580177307129,
      "activations/layer12_attention_weight_min": -25.42483901977539,
      "activations/layer13_attention_weight_max": 36.051448822021484,
      "activations/layer13_attention_weight_min": -34.535484313964844,
      "activations/layer14_attention_weight_max": 38.75711441040039,
      "activations/layer14_attention_weight_min": -37.295021057128906,
      "activations/layer15_attention_weight_max": 36.17502975463867,
      "activations/layer15_attention_weight_min": -35.70380401611328,
      "activations/layer16_attention_weight_max": 28.5672607421875,
      "activations/layer16_attention_weight_min": -28.624528884887695,
      "activations/layer17_attention_weight_max": 30.447120666503906,
      "activations/layer17_attention_weight_min": -28.086538314819336,
      "activations/layer18_attention_weight_max": 33.118534088134766,
      "activations/layer18_attention_weight_min": -25.118270874023438,
      "activations/layer19_attention_weight_max": 32.02876281738281,
      "activations/layer19_attention_weight_min": -30.395925521850586,
      "activations/layer1_attention_weight_max": 17.24111557006836,
      "activations/layer1_attention_weight_min": -13.698366165161133,
      "activations/layer20_attention_weight_max": 31.79155921936035,
      "activations/layer20_attention_weight_min": -25.21257972717285,
      "activations/layer21_attention_weight_max": 28.750015258789062,
      "activations/layer21_attention_weight_min": -24.11712074279785,
      "activations/layer22_attention_weight_max": 40.69340515136719,
      "activations/layer22_attention_weight_min": -30.827871322631836,
      "activations/layer23_attention_weight_max": 31.31597900390625,
      "activations/layer23_attention_weight_min": -26.43039321899414,
      "activations/layer2_attention_weight_max": 33.72480773925781,
      "activations/layer2_attention_weight_min": -32.59295654296875,
      "activations/layer3_attention_weight_max": 98.78894805908203,
      "activations/layer3_attention_weight_min": -97.0440673828125,
      "activations/layer4_attention_weight_max": 111.6040267944336,
      "activations/layer4_attention_weight_min": -110.54461669921875,
      "activations/layer5_attention_weight_max": 50.72704315185547,
      "activations/layer5_attention_weight_min": -64.62156677246094,
      "activations/layer6_attention_weight_max": 42.93910217285156,
      "activations/layer6_attention_weight_min": -46.2747688293457,
      "activations/layer7_attention_weight_max": 86.96244049072266,
      "activations/layer7_attention_weight_min": -91.9288101196289,
      "activations/layer8_attention_weight_max": 40.693145751953125,
      "activations/layer8_attention_weight_min": -42.65247344970703,
      "activations/layer9_attention_weight_max": 34.409645080566406,
      "activations/layer9_attention_weight_min": -35.285152435302734,
      "epoch": 14.04,
      "learning_rate": 6.005681818181818e-05,
      "loss": 2.75,
      "step": 241650
    },
    {
      "activations/layer0_attention_weight_max": 16.54695701599121,
      "activations/layer0_attention_weight_min": -12.511894226074219,
      "activations/layer10_attention_weight_max": 32.540069580078125,
      "activations/layer10_attention_weight_min": -32.461402893066406,
      "activations/layer11_attention_weight_max": 35.054561614990234,
      "activations/layer11_attention_weight_min": -31.944976806640625,
      "activations/layer12_attention_weight_max": 25.86994743347168,
      "activations/layer12_attention_weight_min": -24.78017807006836,
      "activations/layer13_attention_weight_max": 39.905357360839844,
      "activations/layer13_attention_weight_min": -33.94511795043945,
      "activations/layer14_attention_weight_max": 43.81385803222656,
      "activations/layer14_attention_weight_min": -38.62067794799805,
      "activations/layer15_attention_weight_max": 37.740196228027344,
      "activations/layer15_attention_weight_min": -31.94101333618164,
      "activations/layer16_attention_weight_max": 27.741748809814453,
      "activations/layer16_attention_weight_min": -28.201581954956055,
      "activations/layer17_attention_weight_max": 32.27607345581055,
      "activations/layer17_attention_weight_min": -25.82847023010254,
      "activations/layer18_attention_weight_max": 33.61894989013672,
      "activations/layer18_attention_weight_min": -23.37540626525879,
      "activations/layer19_attention_weight_max": 34.79503631591797,
      "activations/layer19_attention_weight_min": -29.729360580444336,
      "activations/layer1_attention_weight_max": 16.921894073486328,
      "activations/layer1_attention_weight_min": -14.785224914550781,
      "activations/layer20_attention_weight_max": 28.000389099121094,
      "activations/layer20_attention_weight_min": -24.1611385345459,
      "activations/layer21_attention_weight_max": 32.290245056152344,
      "activations/layer21_attention_weight_min": -23.321151733398438,
      "activations/layer22_attention_weight_max": 45.6241569519043,
      "activations/layer22_attention_weight_min": -28.41047477722168,
      "activations/layer23_attention_weight_max": 36.126617431640625,
      "activations/layer23_attention_weight_min": -23.495738983154297,
      "activations/layer2_attention_weight_max": 32.8157844543457,
      "activations/layer2_attention_weight_min": -33.50381088256836,
      "activations/layer3_attention_weight_max": 93.21763610839844,
      "activations/layer3_attention_weight_min": -96.8536148071289,
      "activations/layer4_attention_weight_max": 108.1955337524414,
      "activations/layer4_attention_weight_min": -106.57433319091797,
      "activations/layer5_attention_weight_max": 50.783260345458984,
      "activations/layer5_attention_weight_min": -62.182491302490234,
      "activations/layer6_attention_weight_max": 42.31158447265625,
      "activations/layer6_attention_weight_min": -45.317527770996094,
      "activations/layer7_attention_weight_max": 85.4503402709961,
      "activations/layer7_attention_weight_min": -92.64826202392578,
      "activations/layer8_attention_weight_max": 39.53372573852539,
      "activations/layer8_attention_weight_min": -43.029293060302734,
      "activations/layer9_attention_weight_max": 32.93950271606445,
      "activations/layer9_attention_weight_min": -33.4593620300293,
      "epoch": 14.04,
      "learning_rate": 6.0037878787878785e-05,
      "loss": 2.7616,
      "step": 241700
    },
    {
      "activations/layer0_attention_weight_max": 15.825220108032227,
      "activations/layer0_attention_weight_min": -12.692388534545898,
      "activations/layer10_attention_weight_max": 36.69586181640625,
      "activations/layer10_attention_weight_min": -35.524261474609375,
      "activations/layer11_attention_weight_max": 36.386497497558594,
      "activations/layer11_attention_weight_min": -33.09710693359375,
      "activations/layer12_attention_weight_max": 27.08327865600586,
      "activations/layer12_attention_weight_min": -27.496313095092773,
      "activations/layer13_attention_weight_max": 40.97261047363281,
      "activations/layer13_attention_weight_min": -36.846473693847656,
      "activations/layer14_attention_weight_max": 43.36155319213867,
      "activations/layer14_attention_weight_min": -36.4155387878418,
      "activations/layer15_attention_weight_max": 36.810386657714844,
      "activations/layer15_attention_weight_min": -34.34416580200195,
      "activations/layer16_attention_weight_max": 29.168498992919922,
      "activations/layer16_attention_weight_min": -26.59215545654297,
      "activations/layer17_attention_weight_max": 31.004547119140625,
      "activations/layer17_attention_weight_min": -25.747419357299805,
      "activations/layer18_attention_weight_max": 35.44159698486328,
      "activations/layer18_attention_weight_min": -22.469738006591797,
      "activations/layer19_attention_weight_max": 33.84456253051758,
      "activations/layer19_attention_weight_min": -31.55269432067871,
      "activations/layer1_attention_weight_max": 16.86566734313965,
      "activations/layer1_attention_weight_min": -15.970903396606445,
      "activations/layer20_attention_weight_max": 29.1080379486084,
      "activations/layer20_attention_weight_min": -25.536039352416992,
      "activations/layer21_attention_weight_max": 29.114803314208984,
      "activations/layer21_attention_weight_min": -23.79694175720215,
      "activations/layer22_attention_weight_max": 48.35730743408203,
      "activations/layer22_attention_weight_min": -28.1175479888916,
      "activations/layer23_attention_weight_max": 34.712093353271484,
      "activations/layer23_attention_weight_min": -23.093090057373047,
      "activations/layer2_attention_weight_max": 34.267181396484375,
      "activations/layer2_attention_weight_min": -34.81418228149414,
      "activations/layer3_attention_weight_max": 101.41279602050781,
      "activations/layer3_attention_weight_min": -101.70074462890625,
      "activations/layer4_attention_weight_max": 118.1415023803711,
      "activations/layer4_attention_weight_min": -114.80543518066406,
      "activations/layer5_attention_weight_max": 55.566436767578125,
      "activations/layer5_attention_weight_min": -62.551109313964844,
      "activations/layer6_attention_weight_max": 46.27208709716797,
      "activations/layer6_attention_weight_min": -47.9819221496582,
      "activations/layer7_attention_weight_max": 96.5037841796875,
      "activations/layer7_attention_weight_min": -96.60610961914062,
      "activations/layer8_attention_weight_max": 44.52964401245117,
      "activations/layer8_attention_weight_min": -43.23806381225586,
      "activations/layer9_attention_weight_max": 35.24583053588867,
      "activations/layer9_attention_weight_min": -34.004146575927734,
      "epoch": 14.05,
      "learning_rate": 6.0018939393939386e-05,
      "loss": 2.7588,
      "step": 241750
    },
    {
      "activations/layer0_attention_weight_max": 15.288580894470215,
      "activations/layer0_attention_weight_min": -13.225034713745117,
      "activations/layer10_attention_weight_max": 37.01538848876953,
      "activations/layer10_attention_weight_min": -34.01264572143555,
      "activations/layer11_attention_weight_max": 34.13227081298828,
      "activations/layer11_attention_weight_min": -34.53000259399414,
      "activations/layer12_attention_weight_max": 26.740495681762695,
      "activations/layer12_attention_weight_min": -26.12871742248535,
      "activations/layer13_attention_weight_max": 39.506858825683594,
      "activations/layer13_attention_weight_min": -36.11158752441406,
      "activations/layer14_attention_weight_max": 43.45262908935547,
      "activations/layer14_attention_weight_min": -38.7452507019043,
      "activations/layer15_attention_weight_max": 38.47315216064453,
      "activations/layer15_attention_weight_min": -34.4063606262207,
      "activations/layer16_attention_weight_max": 31.94873809814453,
      "activations/layer16_attention_weight_min": -28.90084457397461,
      "activations/layer17_attention_weight_max": 30.65972328186035,
      "activations/layer17_attention_weight_min": -26.04332160949707,
      "activations/layer18_attention_weight_max": 32.05038070678711,
      "activations/layer18_attention_weight_min": -23.950416564941406,
      "activations/layer19_attention_weight_max": 34.03156280517578,
      "activations/layer19_attention_weight_min": -30.52280616760254,
      "activations/layer1_attention_weight_max": 16.1644344329834,
      "activations/layer1_attention_weight_min": -14.215300559997559,
      "activations/layer20_attention_weight_max": 29.08110237121582,
      "activations/layer20_attention_weight_min": -24.771818161010742,
      "activations/layer21_attention_weight_max": 28.471946716308594,
      "activations/layer21_attention_weight_min": -23.545419692993164,
      "activations/layer22_attention_weight_max": 42.05344009399414,
      "activations/layer22_attention_weight_min": -31.14385986328125,
      "activations/layer23_attention_weight_max": 31.990955352783203,
      "activations/layer23_attention_weight_min": -26.14301109313965,
      "activations/layer2_attention_weight_max": 33.42213439941406,
      "activations/layer2_attention_weight_min": -33.71211242675781,
      "activations/layer3_attention_weight_max": 99.04826354980469,
      "activations/layer3_attention_weight_min": -96.16299438476562,
      "activations/layer4_attention_weight_max": 111.67561340332031,
      "activations/layer4_attention_weight_min": -107.90666961669922,
      "activations/layer5_attention_weight_max": 53.456398010253906,
      "activations/layer5_attention_weight_min": -62.18740463256836,
      "activations/layer6_attention_weight_max": 44.88905334472656,
      "activations/layer6_attention_weight_min": -48.55748748779297,
      "activations/layer7_attention_weight_max": 92.29485321044922,
      "activations/layer7_attention_weight_min": -94.5026626586914,
      "activations/layer8_attention_weight_max": 46.109947204589844,
      "activations/layer8_attention_weight_min": -42.9835090637207,
      "activations/layer9_attention_weight_max": 37.23680877685547,
      "activations/layer9_attention_weight_min": -35.13889694213867,
      "epoch": 14.05,
      "learning_rate": 5.9999999999999995e-05,
      "loss": 2.7397,
      "step": 241800
    },
    {
      "activations/layer0_attention_weight_max": 17.08736801147461,
      "activations/layer0_attention_weight_min": -12.489309310913086,
      "activations/layer10_attention_weight_max": 30.217947006225586,
      "activations/layer10_attention_weight_min": -32.597267150878906,
      "activations/layer11_attention_weight_max": 30.57575225830078,
      "activations/layer11_attention_weight_min": -31.62232780456543,
      "activations/layer12_attention_weight_max": 23.692829132080078,
      "activations/layer12_attention_weight_min": -24.53877067565918,
      "activations/layer13_attention_weight_max": 36.04841232299805,
      "activations/layer13_attention_weight_min": -31.606801986694336,
      "activations/layer14_attention_weight_max": 39.49357986450195,
      "activations/layer14_attention_weight_min": -34.54011154174805,
      "activations/layer15_attention_weight_max": 34.824424743652344,
      "activations/layer15_attention_weight_min": -31.300743103027344,
      "activations/layer16_attention_weight_max": 30.577991485595703,
      "activations/layer16_attention_weight_min": -27.145427703857422,
      "activations/layer17_attention_weight_max": 29.226091384887695,
      "activations/layer17_attention_weight_min": -25.003320693969727,
      "activations/layer18_attention_weight_max": 29.261465072631836,
      "activations/layer18_attention_weight_min": -22.98919677734375,
      "activations/layer19_attention_weight_max": 35.451717376708984,
      "activations/layer19_attention_weight_min": -30.71259880065918,
      "activations/layer1_attention_weight_max": 16.03628158569336,
      "activations/layer1_attention_weight_min": -15.97534465789795,
      "activations/layer20_attention_weight_max": 29.907127380371094,
      "activations/layer20_attention_weight_min": -24.041248321533203,
      "activations/layer21_attention_weight_max": 29.177684783935547,
      "activations/layer21_attention_weight_min": -23.184709548950195,
      "activations/layer22_attention_weight_max": 38.84465408325195,
      "activations/layer22_attention_weight_min": -30.07158660888672,
      "activations/layer23_attention_weight_max": 34.36088562011719,
      "activations/layer23_attention_weight_min": -24.360889434814453,
      "activations/layer2_attention_weight_max": 33.494354248046875,
      "activations/layer2_attention_weight_min": -32.94960021972656,
      "activations/layer3_attention_weight_max": 98.51004791259766,
      "activations/layer3_attention_weight_min": -94.48851776123047,
      "activations/layer4_attention_weight_max": 106.8181381225586,
      "activations/layer4_attention_weight_min": -107.44221496582031,
      "activations/layer5_attention_weight_max": 49.8831901550293,
      "activations/layer5_attention_weight_min": -59.90687561035156,
      "activations/layer6_attention_weight_max": 42.86288070678711,
      "activations/layer6_attention_weight_min": -47.45563888549805,
      "activations/layer7_attention_weight_max": 87.8895263671875,
      "activations/layer7_attention_weight_min": -91.42950439453125,
      "activations/layer8_attention_weight_max": 38.70684814453125,
      "activations/layer8_attention_weight_min": -40.39067077636719,
      "activations/layer9_attention_weight_max": 30.12611198425293,
      "activations/layer9_attention_weight_min": -32.668521881103516,
      "epoch": 14.05,
      "learning_rate": 5.99810606060606e-05,
      "loss": 2.7537,
      "step": 241850
    },
    {
      "activations/layer0_attention_weight_max": 16.340925216674805,
      "activations/layer0_attention_weight_min": -12.753177642822266,
      "activations/layer10_attention_weight_max": 32.7056770324707,
      "activations/layer10_attention_weight_min": -33.449501037597656,
      "activations/layer11_attention_weight_max": 34.707855224609375,
      "activations/layer11_attention_weight_min": -31.648845672607422,
      "activations/layer12_attention_weight_max": 27.06505012512207,
      "activations/layer12_attention_weight_min": -26.032691955566406,
      "activations/layer13_attention_weight_max": 41.1734504699707,
      "activations/layer13_attention_weight_min": -34.2536735534668,
      "activations/layer14_attention_weight_max": 47.380035400390625,
      "activations/layer14_attention_weight_min": -36.32119369506836,
      "activations/layer15_attention_weight_max": 41.124271392822266,
      "activations/layer15_attention_weight_min": -32.85188674926758,
      "activations/layer16_attention_weight_max": 34.636390686035156,
      "activations/layer16_attention_weight_min": -26.130023956298828,
      "activations/layer17_attention_weight_max": 30.22506332397461,
      "activations/layer17_attention_weight_min": -25.647565841674805,
      "activations/layer18_attention_weight_max": 31.514888763427734,
      "activations/layer18_attention_weight_min": -23.675954818725586,
      "activations/layer19_attention_weight_max": 34.42869186401367,
      "activations/layer19_attention_weight_min": -31.936643600463867,
      "activations/layer1_attention_weight_max": 15.625794410705566,
      "activations/layer1_attention_weight_min": -14.387436866760254,
      "activations/layer20_attention_weight_max": 29.02822494506836,
      "activations/layer20_attention_weight_min": -24.803672790527344,
      "activations/layer21_attention_weight_max": 29.053752899169922,
      "activations/layer21_attention_weight_min": -24.00119400024414,
      "activations/layer22_attention_weight_max": 44.80950164794922,
      "activations/layer22_attention_weight_min": -29.177318572998047,
      "activations/layer23_attention_weight_max": 39.230186462402344,
      "activations/layer23_attention_weight_min": -24.4233455657959,
      "activations/layer2_attention_weight_max": 32.91691207885742,
      "activations/layer2_attention_weight_min": -32.48772048950195,
      "activations/layer3_attention_weight_max": 94.46026611328125,
      "activations/layer3_attention_weight_min": -96.34765625,
      "activations/layer4_attention_weight_max": 105.20372772216797,
      "activations/layer4_attention_weight_min": -107.94866943359375,
      "activations/layer5_attention_weight_max": 49.943599700927734,
      "activations/layer5_attention_weight_min": -62.503292083740234,
      "activations/layer6_attention_weight_max": 43.992313385009766,
      "activations/layer6_attention_weight_min": -48.75043869018555,
      "activations/layer7_attention_weight_max": 91.65847778320312,
      "activations/layer7_attention_weight_min": -91.11666107177734,
      "activations/layer8_attention_weight_max": 41.79767990112305,
      "activations/layer8_attention_weight_min": -45.30596923828125,
      "activations/layer9_attention_weight_max": 34.514183044433594,
      "activations/layer9_attention_weight_min": -36.27022933959961,
      "epoch": 14.06,
      "learning_rate": 5.9962121212121205e-05,
      "loss": 2.7625,
      "step": 241900
    },
    {
      "activations/layer0_attention_weight_max": 16.644323348999023,
      "activations/layer0_attention_weight_min": -13.132431030273438,
      "activations/layer10_attention_weight_max": 33.34101867675781,
      "activations/layer10_attention_weight_min": -32.25971221923828,
      "activations/layer11_attention_weight_max": 33.88748550415039,
      "activations/layer11_attention_weight_min": -31.67604637145996,
      "activations/layer12_attention_weight_max": 27.02371597290039,
      "activations/layer12_attention_weight_min": -24.625089645385742,
      "activations/layer13_attention_weight_max": 37.479644775390625,
      "activations/layer13_attention_weight_min": -32.8368034362793,
      "activations/layer14_attention_weight_max": 38.272972106933594,
      "activations/layer14_attention_weight_min": -34.37380599975586,
      "activations/layer15_attention_weight_max": 37.34137725830078,
      "activations/layer15_attention_weight_min": -31.726118087768555,
      "activations/layer16_attention_weight_max": 30.025571823120117,
      "activations/layer16_attention_weight_min": -27.098989486694336,
      "activations/layer17_attention_weight_max": 29.78817367553711,
      "activations/layer17_attention_weight_min": -24.871610641479492,
      "activations/layer18_attention_weight_max": 31.757957458496094,
      "activations/layer18_attention_weight_min": -23.761770248413086,
      "activations/layer19_attention_weight_max": 34.46965408325195,
      "activations/layer19_attention_weight_min": -28.9334659576416,
      "activations/layer1_attention_weight_max": 16.60393524169922,
      "activations/layer1_attention_weight_min": -14.912811279296875,
      "activations/layer20_attention_weight_max": 28.679210662841797,
      "activations/layer20_attention_weight_min": -24.76329231262207,
      "activations/layer21_attention_weight_max": 29.833660125732422,
      "activations/layer21_attention_weight_min": -23.420129776000977,
      "activations/layer22_attention_weight_max": 41.660728454589844,
      "activations/layer22_attention_weight_min": -29.982465744018555,
      "activations/layer23_attention_weight_max": 32.61199951171875,
      "activations/layer23_attention_weight_min": -24.774890899658203,
      "activations/layer2_attention_weight_max": 32.41762924194336,
      "activations/layer2_attention_weight_min": -34.377952575683594,
      "activations/layer3_attention_weight_max": 96.15827178955078,
      "activations/layer3_attention_weight_min": -99.65645599365234,
      "activations/layer4_attention_weight_max": 112.22159576416016,
      "activations/layer4_attention_weight_min": -110.36299896240234,
      "activations/layer5_attention_weight_max": 51.9996337890625,
      "activations/layer5_attention_weight_min": -62.27613067626953,
      "activations/layer6_attention_weight_max": 44.2800178527832,
      "activations/layer6_attention_weight_min": -46.86015319824219,
      "activations/layer7_attention_weight_max": 93.84431457519531,
      "activations/layer7_attention_weight_min": -89.04126739501953,
      "activations/layer8_attention_weight_max": 41.95525360107422,
      "activations/layer8_attention_weight_min": -41.854881286621094,
      "activations/layer9_attention_weight_max": 34.489402770996094,
      "activations/layer9_attention_weight_min": -32.21289825439453,
      "epoch": 14.06,
      "learning_rate": 5.994318181818181e-05,
      "loss": 2.7411,
      "step": 241950
    },
    {
      "activations/layer0_attention_weight_max": 16.916860580444336,
      "activations/layer0_attention_weight_min": -12.241793632507324,
      "activations/layer10_attention_weight_max": 38.638572692871094,
      "activations/layer10_attention_weight_min": -34.047142028808594,
      "activations/layer11_attention_weight_max": 37.588714599609375,
      "activations/layer11_attention_weight_min": -34.03623580932617,
      "activations/layer12_attention_weight_max": 27.191734313964844,
      "activations/layer12_attention_weight_min": -26.76896095275879,
      "activations/layer13_attention_weight_max": 42.706912994384766,
      "activations/layer13_attention_weight_min": -32.61598205566406,
      "activations/layer14_attention_weight_max": 42.44568634033203,
      "activations/layer14_attention_weight_min": -34.63547897338867,
      "activations/layer15_attention_weight_max": 37.14462661743164,
      "activations/layer15_attention_weight_min": -34.31705093383789,
      "activations/layer16_attention_weight_max": 32.143428802490234,
      "activations/layer16_attention_weight_min": -26.513904571533203,
      "activations/layer17_attention_weight_max": 31.485132217407227,
      "activations/layer17_attention_weight_min": -27.16688346862793,
      "activations/layer18_attention_weight_max": 32.21574401855469,
      "activations/layer18_attention_weight_min": -25.3997859954834,
      "activations/layer19_attention_weight_max": 36.396080017089844,
      "activations/layer19_attention_weight_min": -32.799617767333984,
      "activations/layer1_attention_weight_max": 16.73274803161621,
      "activations/layer1_attention_weight_min": -14.54131031036377,
      "activations/layer20_attention_weight_max": 31.987253189086914,
      "activations/layer20_attention_weight_min": -24.590852737426758,
      "activations/layer21_attention_weight_max": 31.40309715270996,
      "activations/layer21_attention_weight_min": -24.65213394165039,
      "activations/layer22_attention_weight_max": 48.73570251464844,
      "activations/layer22_attention_weight_min": -30.29625701904297,
      "activations/layer23_attention_weight_max": 36.82242965698242,
      "activations/layer23_attention_weight_min": -26.255550384521484,
      "activations/layer2_attention_weight_max": 33.39111328125,
      "activations/layer2_attention_weight_min": -33.64694595336914,
      "activations/layer3_attention_weight_max": 101.28314208984375,
      "activations/layer3_attention_weight_min": -100.8736572265625,
      "activations/layer4_attention_weight_max": 115.64495086669922,
      "activations/layer4_attention_weight_min": -111.02113342285156,
      "activations/layer5_attention_weight_max": 54.08787155151367,
      "activations/layer5_attention_weight_min": -63.69187927246094,
      "activations/layer6_attention_weight_max": 43.47561264038086,
      "activations/layer6_attention_weight_min": -46.76283645629883,
      "activations/layer7_attention_weight_max": 91.36017608642578,
      "activations/layer7_attention_weight_min": -90.07828521728516,
      "activations/layer8_attention_weight_max": 43.22272491455078,
      "activations/layer8_attention_weight_min": -43.037845611572266,
      "activations/layer9_attention_weight_max": 37.65891647338867,
      "activations/layer9_attention_weight_min": -35.21872329711914,
      "epoch": 14.06,
      "learning_rate": 5.992424242424242e-05,
      "loss": 2.7589,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4987,
      "eval_samples_per_second": 505.251,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4987,
      "eval_openwebtext_samples_per_second": 505.251,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 2.0043,
      "eval_wikitext_samples_per_second": 227.513,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_lambada_loss": 2.396484375,
      "eval_lambada_ppl": 10.984491067549964,
      "eval_lambada_runtime": 9.5686,
      "eval_lambada_samples_per_second": 508.852,
      "step": 242000
    },
    {
      "activations/layer0_attention_weight_max": 16.472883224487305,
      "activations/layer0_attention_weight_min": -12.581308364868164,
      "activations/layer10_attention_weight_max": 35.177608489990234,
      "activations/layer10_attention_weight_min": -33.76873779296875,
      "activations/layer11_attention_weight_max": 34.24521255493164,
      "activations/layer11_attention_weight_min": -33.77140808105469,
      "activations/layer12_attention_weight_max": 24.01304054260254,
      "activations/layer12_attention_weight_min": -24.675498962402344,
      "activations/layer13_attention_weight_max": 36.39586639404297,
      "activations/layer13_attention_weight_min": -34.279640197753906,
      "activations/layer14_attention_weight_max": 38.928985595703125,
      "activations/layer14_attention_weight_min": -37.21940994262695,
      "activations/layer15_attention_weight_max": 36.222633361816406,
      "activations/layer15_attention_weight_min": -33.326316833496094,
      "activations/layer16_attention_weight_max": 27.609643936157227,
      "activations/layer16_attention_weight_min": -27.034791946411133,
      "activations/layer17_attention_weight_max": 29.37440299987793,
      "activations/layer17_attention_weight_min": -26.028831481933594,
      "activations/layer18_attention_weight_max": 31.83306884765625,
      "activations/layer18_attention_weight_min": -25.524633407592773,
      "activations/layer19_attention_weight_max": 34.0560302734375,
      "activations/layer19_attention_weight_min": -30.361173629760742,
      "activations/layer1_attention_weight_max": 16.3835391998291,
      "activations/layer1_attention_weight_min": -13.827122688293457,
      "activations/layer20_attention_weight_max": 29.073585510253906,
      "activations/layer20_attention_weight_min": -25.989118576049805,
      "activations/layer21_attention_weight_max": 29.87600326538086,
      "activations/layer21_attention_weight_min": -25.14558982849121,
      "activations/layer22_attention_weight_max": 38.118656158447266,
      "activations/layer22_attention_weight_min": -30.889036178588867,
      "activations/layer23_attention_weight_max": 33.316864013671875,
      "activations/layer23_attention_weight_min": -25.627391815185547,
      "activations/layer2_attention_weight_max": 36.59376525878906,
      "activations/layer2_attention_weight_min": -36.107723236083984,
      "activations/layer3_attention_weight_max": 96.96428680419922,
      "activations/layer3_attention_weight_min": -100.07595825195312,
      "activations/layer4_attention_weight_max": 110.33985900878906,
      "activations/layer4_attention_weight_min": -104.92974090576172,
      "activations/layer5_attention_weight_max": 50.347862243652344,
      "activations/layer5_attention_weight_min": -61.904998779296875,
      "activations/layer6_attention_weight_max": 41.95341110229492,
      "activations/layer6_attention_weight_min": -46.84146499633789,
      "activations/layer7_attention_weight_max": 90.72205352783203,
      "activations/layer7_attention_weight_min": -83.34998321533203,
      "activations/layer8_attention_weight_max": 40.898555755615234,
      "activations/layer8_attention_weight_min": -40.898292541503906,
      "activations/layer9_attention_weight_max": 30.335046768188477,
      "activations/layer9_attention_weight_min": -32.25170135498047,
      "epoch": 14.06,
      "learning_rate": 5.990530303030302e-05,
      "loss": 2.755,
      "step": 242050
    },
    {
      "activations/layer0_attention_weight_max": 16.177303314208984,
      "activations/layer0_attention_weight_min": -11.200234413146973,
      "activations/layer10_attention_weight_max": 30.214025497436523,
      "activations/layer10_attention_weight_min": -31.373966217041016,
      "activations/layer11_attention_weight_max": 31.38796615600586,
      "activations/layer11_attention_weight_min": -31.841693878173828,
      "activations/layer12_attention_weight_max": 24.687332153320312,
      "activations/layer12_attention_weight_min": -25.51805877685547,
      "activations/layer13_attention_weight_max": 35.98924255371094,
      "activations/layer13_attention_weight_min": -31.810319900512695,
      "activations/layer14_attention_weight_max": 42.657470703125,
      "activations/layer14_attention_weight_min": -36.03274154663086,
      "activations/layer15_attention_weight_max": 38.18470764160156,
      "activations/layer15_attention_weight_min": -31.470205307006836,
      "activations/layer16_attention_weight_max": 29.65209197998047,
      "activations/layer16_attention_weight_min": -25.838502883911133,
      "activations/layer17_attention_weight_max": 33.0293083190918,
      "activations/layer17_attention_weight_min": -24.146718978881836,
      "activations/layer18_attention_weight_max": 33.57503128051758,
      "activations/layer18_attention_weight_min": -22.37248992919922,
      "activations/layer19_attention_weight_max": 36.44393539428711,
      "activations/layer19_attention_weight_min": -28.75848960876465,
      "activations/layer1_attention_weight_max": 17.260953903198242,
      "activations/layer1_attention_weight_min": -14.267051696777344,
      "activations/layer20_attention_weight_max": 32.911617279052734,
      "activations/layer20_attention_weight_min": -23.164127349853516,
      "activations/layer21_attention_weight_max": 29.6160945892334,
      "activations/layer21_attention_weight_min": -22.869834899902344,
      "activations/layer22_attention_weight_max": 42.17294692993164,
      "activations/layer22_attention_weight_min": -31.252126693725586,
      "activations/layer23_attention_weight_max": 33.649993896484375,
      "activations/layer23_attention_weight_min": -21.97927474975586,
      "activations/layer2_attention_weight_max": 37.20905685424805,
      "activations/layer2_attention_weight_min": -34.754295349121094,
      "activations/layer3_attention_weight_max": 107.09098052978516,
      "activations/layer3_attention_weight_min": -107.36771392822266,
      "activations/layer4_attention_weight_max": 117.50885009765625,
      "activations/layer4_attention_weight_min": -113.07181549072266,
      "activations/layer5_attention_weight_max": 51.83024597167969,
      "activations/layer5_attention_weight_min": -59.08283996582031,
      "activations/layer6_attention_weight_max": 44.35570526123047,
      "activations/layer6_attention_weight_min": -44.37716293334961,
      "activations/layer7_attention_weight_max": 90.04855346679688,
      "activations/layer7_attention_weight_min": -88.03456115722656,
      "activations/layer8_attention_weight_max": 38.738773345947266,
      "activations/layer8_attention_weight_min": -39.436744689941406,
      "activations/layer9_attention_weight_max": 29.303396224975586,
      "activations/layer9_attention_weight_min": -31.61151123046875,
      "epoch": 14.07,
      "learning_rate": 5.988636363636363e-05,
      "loss": 2.7666,
      "step": 242100
    },
    {
      "activations/layer0_attention_weight_max": 16.16106414794922,
      "activations/layer0_attention_weight_min": -12.303488731384277,
      "activations/layer10_attention_weight_max": 37.3083610534668,
      "activations/layer10_attention_weight_min": -36.64768600463867,
      "activations/layer11_attention_weight_max": 37.57665252685547,
      "activations/layer11_attention_weight_min": -39.20539855957031,
      "activations/layer12_attention_weight_max": 27.148860931396484,
      "activations/layer12_attention_weight_min": -26.074705123901367,
      "activations/layer13_attention_weight_max": 39.159027099609375,
      "activations/layer13_attention_weight_min": -35.926658630371094,
      "activations/layer14_attention_weight_max": 42.50236129760742,
      "activations/layer14_attention_weight_min": -38.83070373535156,
      "activations/layer15_attention_weight_max": 37.29768371582031,
      "activations/layer15_attention_weight_min": -35.615474700927734,
      "activations/layer16_attention_weight_max": 31.55937385559082,
      "activations/layer16_attention_weight_min": -29.129131317138672,
      "activations/layer17_attention_weight_max": 31.30557632446289,
      "activations/layer17_attention_weight_min": -25.43338966369629,
      "activations/layer18_attention_weight_max": 31.13631820678711,
      "activations/layer18_attention_weight_min": -24.676109313964844,
      "activations/layer19_attention_weight_max": 34.724334716796875,
      "activations/layer19_attention_weight_min": -29.330780029296875,
      "activations/layer1_attention_weight_max": 17.53115463256836,
      "activations/layer1_attention_weight_min": -14.594345092773438,
      "activations/layer20_attention_weight_max": 29.01996421813965,
      "activations/layer20_attention_weight_min": -25.112993240356445,
      "activations/layer21_attention_weight_max": 27.810075759887695,
      "activations/layer21_attention_weight_min": -23.18916130065918,
      "activations/layer22_attention_weight_max": 39.785640716552734,
      "activations/layer22_attention_weight_min": -30.278656005859375,
      "activations/layer23_attention_weight_max": 35.30698776245117,
      "activations/layer23_attention_weight_min": -26.044578552246094,
      "activations/layer2_attention_weight_max": 35.04647445678711,
      "activations/layer2_attention_weight_min": -33.18695831298828,
      "activations/layer3_attention_weight_max": 94.56157684326172,
      "activations/layer3_attention_weight_min": -95.61009979248047,
      "activations/layer4_attention_weight_max": 106.69014739990234,
      "activations/layer4_attention_weight_min": -107.8177261352539,
      "activations/layer5_attention_weight_max": 51.19186019897461,
      "activations/layer5_attention_weight_min": -64.28783416748047,
      "activations/layer6_attention_weight_max": 47.012718200683594,
      "activations/layer6_attention_weight_min": -45.77483367919922,
      "activations/layer7_attention_weight_max": 96.02686309814453,
      "activations/layer7_attention_weight_min": -90.32474517822266,
      "activations/layer8_attention_weight_max": 46.6593132019043,
      "activations/layer8_attention_weight_min": -45.68135070800781,
      "activations/layer9_attention_weight_max": 37.680763244628906,
      "activations/layer9_attention_weight_min": -36.72087860107422,
      "epoch": 14.07,
      "learning_rate": 5.986742424242423e-05,
      "loss": 2.7448,
      "step": 242150
    },
    {
      "activations/layer0_attention_weight_max": 15.8419771194458,
      "activations/layer0_attention_weight_min": -13.446922302246094,
      "activations/layer10_attention_weight_max": 30.91194725036621,
      "activations/layer10_attention_weight_min": -31.921628952026367,
      "activations/layer11_attention_weight_max": 30.976390838623047,
      "activations/layer11_attention_weight_min": -31.29236602783203,
      "activations/layer12_attention_weight_max": 31.869001388549805,
      "activations/layer12_attention_weight_min": -26.208011627197266,
      "activations/layer13_attention_weight_max": 38.886962890625,
      "activations/layer13_attention_weight_min": -36.268943786621094,
      "activations/layer14_attention_weight_max": 41.83913040161133,
      "activations/layer14_attention_weight_min": -38.51789093017578,
      "activations/layer15_attention_weight_max": 37.57936096191406,
      "activations/layer15_attention_weight_min": -33.14809036254883,
      "activations/layer16_attention_weight_max": 31.738479614257812,
      "activations/layer16_attention_weight_min": -28.95194435119629,
      "activations/layer17_attention_weight_max": 32.04178237915039,
      "activations/layer17_attention_weight_min": -25.867992401123047,
      "activations/layer18_attention_weight_max": 33.814605712890625,
      "activations/layer18_attention_weight_min": -25.46707534790039,
      "activations/layer19_attention_weight_max": 38.005210876464844,
      "activations/layer19_attention_weight_min": -30.638181686401367,
      "activations/layer1_attention_weight_max": 15.927347183227539,
      "activations/layer1_attention_weight_min": -14.185147285461426,
      "activations/layer20_attention_weight_max": 31.751070022583008,
      "activations/layer20_attention_weight_min": -24.407804489135742,
      "activations/layer21_attention_weight_max": 29.34166717529297,
      "activations/layer21_attention_weight_min": -25.354482650756836,
      "activations/layer22_attention_weight_max": 48.01997375488281,
      "activations/layer22_attention_weight_min": -29.85375213623047,
      "activations/layer23_attention_weight_max": 35.86556625366211,
      "activations/layer23_attention_weight_min": -24.340965270996094,
      "activations/layer2_attention_weight_max": 33.282222747802734,
      "activations/layer2_attention_weight_min": -33.294769287109375,
      "activations/layer3_attention_weight_max": 94.38270568847656,
      "activations/layer3_attention_weight_min": -95.1387710571289,
      "activations/layer4_attention_weight_max": 108.17252349853516,
      "activations/layer4_attention_weight_min": -112.0379409790039,
      "activations/layer5_attention_weight_max": 50.101898193359375,
      "activations/layer5_attention_weight_min": -62.38914108276367,
      "activations/layer6_attention_weight_max": 46.96918869018555,
      "activations/layer6_attention_weight_min": -46.25649642944336,
      "activations/layer7_attention_weight_max": 90.01652526855469,
      "activations/layer7_attention_weight_min": -88.35052490234375,
      "activations/layer8_attention_weight_max": 41.962467193603516,
      "activations/layer8_attention_weight_min": -39.9279670715332,
      "activations/layer9_attention_weight_max": 31.080795288085938,
      "activations/layer9_attention_weight_min": -32.41484832763672,
      "epoch": 14.07,
      "learning_rate": 5.984848484848485e-05,
      "loss": 2.75,
      "step": 242200
    },
    {
      "activations/layer0_attention_weight_max": 16.151763916015625,
      "activations/layer0_attention_weight_min": -12.178210258483887,
      "activations/layer10_attention_weight_max": 34.589698791503906,
      "activations/layer10_attention_weight_min": -33.54141616821289,
      "activations/layer11_attention_weight_max": 38.80765914916992,
      "activations/layer11_attention_weight_min": -32.689579010009766,
      "activations/layer12_attention_weight_max": 28.44772720336914,
      "activations/layer12_attention_weight_min": -25.73668670654297,
      "activations/layer13_attention_weight_max": 45.43539047241211,
      "activations/layer13_attention_weight_min": -34.00516128540039,
      "activations/layer14_attention_weight_max": 49.8206672668457,
      "activations/layer14_attention_weight_min": -36.603519439697266,
      "activations/layer15_attention_weight_max": 46.139251708984375,
      "activations/layer15_attention_weight_min": -32.68979263305664,
      "activations/layer16_attention_weight_max": 34.50566482543945,
      "activations/layer16_attention_weight_min": -26.784692764282227,
      "activations/layer17_attention_weight_max": 36.949859619140625,
      "activations/layer17_attention_weight_min": -25.9472713470459,
      "activations/layer18_attention_weight_max": 33.88273620605469,
      "activations/layer18_attention_weight_min": -23.26848793029785,
      "activations/layer19_attention_weight_max": 35.67186737060547,
      "activations/layer19_attention_weight_min": -32.218772888183594,
      "activations/layer1_attention_weight_max": 15.338774681091309,
      "activations/layer1_attention_weight_min": -14.570638656616211,
      "activations/layer20_attention_weight_max": 33.10103988647461,
      "activations/layer20_attention_weight_min": -26.306739807128906,
      "activations/layer21_attention_weight_max": 30.64629364013672,
      "activations/layer21_attention_weight_min": -24.60106086730957,
      "activations/layer22_attention_weight_max": 44.85520553588867,
      "activations/layer22_attention_weight_min": -29.589921951293945,
      "activations/layer23_attention_weight_max": 35.6578254699707,
      "activations/layer23_attention_weight_min": -25.224029541015625,
      "activations/layer2_attention_weight_max": 33.17647171020508,
      "activations/layer2_attention_weight_min": -35.18916320800781,
      "activations/layer3_attention_weight_max": 98.92755889892578,
      "activations/layer3_attention_weight_min": -100.53777313232422,
      "activations/layer4_attention_weight_max": 113.70914459228516,
      "activations/layer4_attention_weight_min": -112.46988677978516,
      "activations/layer5_attention_weight_max": 52.73208236694336,
      "activations/layer5_attention_weight_min": -62.1139030456543,
      "activations/layer6_attention_weight_max": 45.39977264404297,
      "activations/layer6_attention_weight_min": -47.58198547363281,
      "activations/layer7_attention_weight_max": 94.11399841308594,
      "activations/layer7_attention_weight_min": -89.79559326171875,
      "activations/layer8_attention_weight_max": 44.69123458862305,
      "activations/layer8_attention_weight_min": -41.335723876953125,
      "activations/layer9_attention_weight_max": 36.061710357666016,
      "activations/layer9_attention_weight_min": -33.68735885620117,
      "epoch": 14.08,
      "learning_rate": 5.982954545454545e-05,
      "loss": 2.7472,
      "step": 242250
    },
    {
      "activations/layer0_attention_weight_max": 16.128992080688477,
      "activations/layer0_attention_weight_min": -12.634234428405762,
      "activations/layer10_attention_weight_max": 45.51702880859375,
      "activations/layer10_attention_weight_min": -44.5068473815918,
      "activations/layer11_attention_weight_max": 45.545372009277344,
      "activations/layer11_attention_weight_min": -45.081886291503906,
      "activations/layer12_attention_weight_max": 34.150413513183594,
      "activations/layer12_attention_weight_min": -30.445714950561523,
      "activations/layer13_attention_weight_max": 49.73103713989258,
      "activations/layer13_attention_weight_min": -40.183834075927734,
      "activations/layer14_attention_weight_max": 58.54255294799805,
      "activations/layer14_attention_weight_min": -45.243568420410156,
      "activations/layer15_attention_weight_max": 49.99274444580078,
      "activations/layer15_attention_weight_min": -40.98565673828125,
      "activations/layer16_attention_weight_max": 37.25032043457031,
      "activations/layer16_attention_weight_min": -26.8790283203125,
      "activations/layer17_attention_weight_max": 33.727264404296875,
      "activations/layer17_attention_weight_min": -26.760906219482422,
      "activations/layer18_attention_weight_max": 33.83366775512695,
      "activations/layer18_attention_weight_min": -24.237138748168945,
      "activations/layer19_attention_weight_max": 36.82106399536133,
      "activations/layer19_attention_weight_min": -31.369171142578125,
      "activations/layer1_attention_weight_max": 16.756223678588867,
      "activations/layer1_attention_weight_min": -15.827606201171875,
      "activations/layer20_attention_weight_max": 32.92662048339844,
      "activations/layer20_attention_weight_min": -24.246337890625,
      "activations/layer21_attention_weight_max": 31.558805465698242,
      "activations/layer21_attention_weight_min": -24.180465698242188,
      "activations/layer22_attention_weight_max": 48.35725784301758,
      "activations/layer22_attention_weight_min": -31.42405128479004,
      "activations/layer23_attention_weight_max": 37.28583526611328,
      "activations/layer23_attention_weight_min": -26.623672485351562,
      "activations/layer2_attention_weight_max": 38.26509094238281,
      "activations/layer2_attention_weight_min": -34.99528884887695,
      "activations/layer3_attention_weight_max": 108.42308044433594,
      "activations/layer3_attention_weight_min": -103.78388977050781,
      "activations/layer4_attention_weight_max": 117.18653869628906,
      "activations/layer4_attention_weight_min": -115.5399169921875,
      "activations/layer5_attention_weight_max": 53.8020133972168,
      "activations/layer5_attention_weight_min": -65.45425415039062,
      "activations/layer6_attention_weight_max": 52.65492630004883,
      "activations/layer6_attention_weight_min": -47.96991729736328,
      "activations/layer7_attention_weight_max": 123.95855712890625,
      "activations/layer7_attention_weight_min": -110.57377624511719,
      "activations/layer8_attention_weight_max": 59.00706100463867,
      "activations/layer8_attention_weight_min": -55.22785568237305,
      "activations/layer9_attention_weight_max": 46.691532135009766,
      "activations/layer9_attention_weight_min": -44.7139778137207,
      "epoch": 14.08,
      "learning_rate": 5.981060606060605e-05,
      "loss": 2.7608,
      "step": 242300
    },
    {
      "activations/layer0_attention_weight_max": 16.519289016723633,
      "activations/layer0_attention_weight_min": -12.689332962036133,
      "activations/layer10_attention_weight_max": 34.42939758300781,
      "activations/layer10_attention_weight_min": -32.61577224731445,
      "activations/layer11_attention_weight_max": 34.564971923828125,
      "activations/layer11_attention_weight_min": -33.55109405517578,
      "activations/layer12_attention_weight_max": 26.398151397705078,
      "activations/layer12_attention_weight_min": -25.67571449279785,
      "activations/layer13_attention_weight_max": 37.991127014160156,
      "activations/layer13_attention_weight_min": -32.351165771484375,
      "activations/layer14_attention_weight_max": 43.08332061767578,
      "activations/layer14_attention_weight_min": -34.793914794921875,
      "activations/layer15_attention_weight_max": 37.15943908691406,
      "activations/layer15_attention_weight_min": -33.075477600097656,
      "activations/layer16_attention_weight_max": 30.934585571289062,
      "activations/layer16_attention_weight_min": -25.924524307250977,
      "activations/layer17_attention_weight_max": 29.14483070373535,
      "activations/layer17_attention_weight_min": -24.172672271728516,
      "activations/layer18_attention_weight_max": 29.737628936767578,
      "activations/layer18_attention_weight_min": -24.555070877075195,
      "activations/layer19_attention_weight_max": 32.78743362426758,
      "activations/layer19_attention_weight_min": -30.006418228149414,
      "activations/layer1_attention_weight_max": 16.019275665283203,
      "activations/layer1_attention_weight_min": -15.17013931274414,
      "activations/layer20_attention_weight_max": 29.84422492980957,
      "activations/layer20_attention_weight_min": -23.617998123168945,
      "activations/layer21_attention_weight_max": 29.61068344116211,
      "activations/layer21_attention_weight_min": -23.050308227539062,
      "activations/layer22_attention_weight_max": 41.62009048461914,
      "activations/layer22_attention_weight_min": -29.526123046875,
      "activations/layer23_attention_weight_max": 33.42301940917969,
      "activations/layer23_attention_weight_min": -24.358671188354492,
      "activations/layer2_attention_weight_max": 35.23479461669922,
      "activations/layer2_attention_weight_min": -35.193695068359375,
      "activations/layer3_attention_weight_max": 101.11863708496094,
      "activations/layer3_attention_weight_min": -103.33600616455078,
      "activations/layer4_attention_weight_max": 116.20861053466797,
      "activations/layer4_attention_weight_min": -113.20794677734375,
      "activations/layer5_attention_weight_max": 53.399147033691406,
      "activations/layer5_attention_weight_min": -60.3874626159668,
      "activations/layer6_attention_weight_max": 48.53242111206055,
      "activations/layer6_attention_weight_min": -46.650386810302734,
      "activations/layer7_attention_weight_max": 98.55888366699219,
      "activations/layer7_attention_weight_min": -95.68553924560547,
      "activations/layer8_attention_weight_max": 40.723148345947266,
      "activations/layer8_attention_weight_min": -43.11838150024414,
      "activations/layer9_attention_weight_max": 34.00530242919922,
      "activations/layer9_attention_weight_min": -33.31551742553711,
      "epoch": 14.08,
      "learning_rate": 5.9791666666666665e-05,
      "loss": 2.7593,
      "step": 242350
    },
    {
      "activations/layer0_attention_weight_max": 15.660701751708984,
      "activations/layer0_attention_weight_min": -12.762660026550293,
      "activations/layer10_attention_weight_max": 33.833251953125,
      "activations/layer10_attention_weight_min": -33.985172271728516,
      "activations/layer11_attention_weight_max": 35.074989318847656,
      "activations/layer11_attention_weight_min": -33.08173370361328,
      "activations/layer12_attention_weight_max": 27.049564361572266,
      "activations/layer12_attention_weight_min": -26.902610778808594,
      "activations/layer13_attention_weight_max": 45.1978645324707,
      "activations/layer13_attention_weight_min": -36.077056884765625,
      "activations/layer14_attention_weight_max": 45.253448486328125,
      "activations/layer14_attention_weight_min": -39.29332733154297,
      "activations/layer15_attention_weight_max": 37.494590759277344,
      "activations/layer15_attention_weight_min": -34.32840347290039,
      "activations/layer16_attention_weight_max": 32.5192985534668,
      "activations/layer16_attention_weight_min": -26.417383193969727,
      "activations/layer17_attention_weight_max": 31.5170841217041,
      "activations/layer17_attention_weight_min": -25.75588035583496,
      "activations/layer18_attention_weight_max": 29.650249481201172,
      "activations/layer18_attention_weight_min": -24.101924896240234,
      "activations/layer19_attention_weight_max": 38.124732971191406,
      "activations/layer19_attention_weight_min": -29.678640365600586,
      "activations/layer1_attention_weight_max": 15.940535545349121,
      "activations/layer1_attention_weight_min": -14.827030181884766,
      "activations/layer20_attention_weight_max": 30.38665771484375,
      "activations/layer20_attention_weight_min": -23.13006591796875,
      "activations/layer21_attention_weight_max": 27.97739601135254,
      "activations/layer21_attention_weight_min": -22.407751083374023,
      "activations/layer22_attention_weight_max": 42.43635177612305,
      "activations/layer22_attention_weight_min": -28.677183151245117,
      "activations/layer23_attention_weight_max": 33.2922477722168,
      "activations/layer23_attention_weight_min": -22.740890502929688,
      "activations/layer2_attention_weight_max": 37.74403381347656,
      "activations/layer2_attention_weight_min": -35.153846740722656,
      "activations/layer3_attention_weight_max": 101.46331787109375,
      "activations/layer3_attention_weight_min": -104.43236541748047,
      "activations/layer4_attention_weight_max": 110.80891418457031,
      "activations/layer4_attention_weight_min": -111.71376037597656,
      "activations/layer5_attention_weight_max": 54.38862609863281,
      "activations/layer5_attention_weight_min": -61.29609298706055,
      "activations/layer6_attention_weight_max": 44.8554801940918,
      "activations/layer6_attention_weight_min": -47.08058547973633,
      "activations/layer7_attention_weight_max": 89.82530212402344,
      "activations/layer7_attention_weight_min": -92.78739929199219,
      "activations/layer8_attention_weight_max": 40.7479362487793,
      "activations/layer8_attention_weight_min": -42.299983978271484,
      "activations/layer9_attention_weight_max": 34.05239486694336,
      "activations/layer9_attention_weight_min": -33.323726654052734,
      "epoch": 14.08,
      "learning_rate": 5.977272727272727e-05,
      "loss": 2.7314,
      "step": 242400
    },
    {
      "activations/layer0_attention_weight_max": 15.654509544372559,
      "activations/layer0_attention_weight_min": -12.740313529968262,
      "activations/layer10_attention_weight_max": 34.01235580444336,
      "activations/layer10_attention_weight_min": -33.46537399291992,
      "activations/layer11_attention_weight_max": 34.011810302734375,
      "activations/layer11_attention_weight_min": -33.820377349853516,
      "activations/layer12_attention_weight_max": 26.51730728149414,
      "activations/layer12_attention_weight_min": -26.00050163269043,
      "activations/layer13_attention_weight_max": 39.33579635620117,
      "activations/layer13_attention_weight_min": -36.360687255859375,
      "activations/layer14_attention_weight_max": 42.23334503173828,
      "activations/layer14_attention_weight_min": -40.2829475402832,
      "activations/layer15_attention_weight_max": 37.62788009643555,
      "activations/layer15_attention_weight_min": -36.2508430480957,
      "activations/layer16_attention_weight_max": 31.28361701965332,
      "activations/layer16_attention_weight_min": -27.583057403564453,
      "activations/layer17_attention_weight_max": 32.02817916870117,
      "activations/layer17_attention_weight_min": -27.032217025756836,
      "activations/layer18_attention_weight_max": 30.181941986083984,
      "activations/layer18_attention_weight_min": -25.304073333740234,
      "activations/layer19_attention_weight_max": 34.34492111206055,
      "activations/layer19_attention_weight_min": -32.24833679199219,
      "activations/layer1_attention_weight_max": 16.1983642578125,
      "activations/layer1_attention_weight_min": -15.141079902648926,
      "activations/layer20_attention_weight_max": 30.946060180664062,
      "activations/layer20_attention_weight_min": -25.67137908935547,
      "activations/layer21_attention_weight_max": 32.181549072265625,
      "activations/layer21_attention_weight_min": -24.310606002807617,
      "activations/layer22_attention_weight_max": 46.3531494140625,
      "activations/layer22_attention_weight_min": -32.27872085571289,
      "activations/layer23_attention_weight_max": 37.47161102294922,
      "activations/layer23_attention_weight_min": -23.867712020874023,
      "activations/layer2_attention_weight_max": 34.49486541748047,
      "activations/layer2_attention_weight_min": -35.146034240722656,
      "activations/layer3_attention_weight_max": 102.93492126464844,
      "activations/layer3_attention_weight_min": -102.30792999267578,
      "activations/layer4_attention_weight_max": 115.06597137451172,
      "activations/layer4_attention_weight_min": -115.63529205322266,
      "activations/layer5_attention_weight_max": 53.65718078613281,
      "activations/layer5_attention_weight_min": -66.63323974609375,
      "activations/layer6_attention_weight_max": 46.33406066894531,
      "activations/layer6_attention_weight_min": -48.93803405761719,
      "activations/layer7_attention_weight_max": 96.45814514160156,
      "activations/layer7_attention_weight_min": -95.61101531982422,
      "activations/layer8_attention_weight_max": 41.43507766723633,
      "activations/layer8_attention_weight_min": -43.80160903930664,
      "activations/layer9_attention_weight_max": 32.917442321777344,
      "activations/layer9_attention_weight_min": -34.63478088378906,
      "epoch": 14.09,
      "learning_rate": 5.9753787878787875e-05,
      "loss": 2.7365,
      "step": 242450
    },
    {
      "activations/layer0_attention_weight_max": 15.432134628295898,
      "activations/layer0_attention_weight_min": -11.905620574951172,
      "activations/layer10_attention_weight_max": 32.15198516845703,
      "activations/layer10_attention_weight_min": -32.9941291809082,
      "activations/layer11_attention_weight_max": 31.388317108154297,
      "activations/layer11_attention_weight_min": -32.70146560668945,
      "activations/layer12_attention_weight_max": 25.390029907226562,
      "activations/layer12_attention_weight_min": -24.1813907623291,
      "activations/layer13_attention_weight_max": 39.65388107299805,
      "activations/layer13_attention_weight_min": -32.50751495361328,
      "activations/layer14_attention_weight_max": 39.449127197265625,
      "activations/layer14_attention_weight_min": -36.02388000488281,
      "activations/layer15_attention_weight_max": 35.258907318115234,
      "activations/layer15_attention_weight_min": -32.680965423583984,
      "activations/layer16_attention_weight_max": 28.399646759033203,
      "activations/layer16_attention_weight_min": -27.27543067932129,
      "activations/layer17_attention_weight_max": 29.32529640197754,
      "activations/layer17_attention_weight_min": -24.563657760620117,
      "activations/layer18_attention_weight_max": 33.48935317993164,
      "activations/layer18_attention_weight_min": -23.900171279907227,
      "activations/layer19_attention_weight_max": 32.551265716552734,
      "activations/layer19_attention_weight_min": -29.32683753967285,
      "activations/layer1_attention_weight_max": 15.997336387634277,
      "activations/layer1_attention_weight_min": -14.885263442993164,
      "activations/layer20_attention_weight_max": 28.39904022216797,
      "activations/layer20_attention_weight_min": -25.401872634887695,
      "activations/layer21_attention_weight_max": 28.910436630249023,
      "activations/layer21_attention_weight_min": -23.965911865234375,
      "activations/layer22_attention_weight_max": 40.364139556884766,
      "activations/layer22_attention_weight_min": -29.71674156188965,
      "activations/layer23_attention_weight_max": 32.766448974609375,
      "activations/layer23_attention_weight_min": -25.770734786987305,
      "activations/layer2_attention_weight_max": 33.23509979248047,
      "activations/layer2_attention_weight_min": -33.86351776123047,
      "activations/layer3_attention_weight_max": 100.87970733642578,
      "activations/layer3_attention_weight_min": -101.22327423095703,
      "activations/layer4_attention_weight_max": 113.94482421875,
      "activations/layer4_attention_weight_min": -106.76155853271484,
      "activations/layer5_attention_weight_max": 49.824440002441406,
      "activations/layer5_attention_weight_min": -61.113433837890625,
      "activations/layer6_attention_weight_max": 43.304405212402344,
      "activations/layer6_attention_weight_min": -46.559940338134766,
      "activations/layer7_attention_weight_max": 88.58275604248047,
      "activations/layer7_attention_weight_min": -86.24153137207031,
      "activations/layer8_attention_weight_max": 39.6939582824707,
      "activations/layer8_attention_weight_min": -39.39312744140625,
      "activations/layer9_attention_weight_max": 32.0217399597168,
      "activations/layer9_attention_weight_min": -32.48159408569336,
      "epoch": 14.09,
      "learning_rate": 5.973484848484848e-05,
      "loss": 2.7473,
      "step": 242500
    },
    {
      "activations/layer0_attention_weight_max": 16.37590980529785,
      "activations/layer0_attention_weight_min": -12.486337661743164,
      "activations/layer10_attention_weight_max": 31.854820251464844,
      "activations/layer10_attention_weight_min": -34.508323669433594,
      "activations/layer11_attention_weight_max": 31.7627010345459,
      "activations/layer11_attention_weight_min": -33.48110580444336,
      "activations/layer12_attention_weight_max": 26.81894874572754,
      "activations/layer12_attention_weight_min": -25.05406951904297,
      "activations/layer13_attention_weight_max": 39.183837890625,
      "activations/layer13_attention_weight_min": -34.37061309814453,
      "activations/layer14_attention_weight_max": 42.68659591674805,
      "activations/layer14_attention_weight_min": -37.74536895751953,
      "activations/layer15_attention_weight_max": 36.87823486328125,
      "activations/layer15_attention_weight_min": -36.06051254272461,
      "activations/layer16_attention_weight_max": 30.915979385375977,
      "activations/layer16_attention_weight_min": -27.05417823791504,
      "activations/layer17_attention_weight_max": 32.31474685668945,
      "activations/layer17_attention_weight_min": -26.77656364440918,
      "activations/layer18_attention_weight_max": 32.63608932495117,
      "activations/layer18_attention_weight_min": -25.715723037719727,
      "activations/layer19_attention_weight_max": 33.85918045043945,
      "activations/layer19_attention_weight_min": -33.273902893066406,
      "activations/layer1_attention_weight_max": 17.282073974609375,
      "activations/layer1_attention_weight_min": -14.58213996887207,
      "activations/layer20_attention_weight_max": 28.7673282623291,
      "activations/layer20_attention_weight_min": -28.8248348236084,
      "activations/layer21_attention_weight_max": 28.78355598449707,
      "activations/layer21_attention_weight_min": -26.1290283203125,
      "activations/layer22_attention_weight_max": 46.97156524658203,
      "activations/layer22_attention_weight_min": -32.17180252075195,
      "activations/layer23_attention_weight_max": 36.827091217041016,
      "activations/layer23_attention_weight_min": -26.566062927246094,
      "activations/layer2_attention_weight_max": 32.876792907714844,
      "activations/layer2_attention_weight_min": -33.04454040527344,
      "activations/layer3_attention_weight_max": 98.3296127319336,
      "activations/layer3_attention_weight_min": -98.82154846191406,
      "activations/layer4_attention_weight_max": 107.11663818359375,
      "activations/layer4_attention_weight_min": -109.39837646484375,
      "activations/layer5_attention_weight_max": 50.373409271240234,
      "activations/layer5_attention_weight_min": -61.19374084472656,
      "activations/layer6_attention_weight_max": 44.11623001098633,
      "activations/layer6_attention_weight_min": -47.38877487182617,
      "activations/layer7_attention_weight_max": 88.52713775634766,
      "activations/layer7_attention_weight_min": -91.1724853515625,
      "activations/layer8_attention_weight_max": 40.57280349731445,
      "activations/layer8_attention_weight_min": -40.466209411621094,
      "activations/layer9_attention_weight_max": 31.3342342376709,
      "activations/layer9_attention_weight_min": -33.26191329956055,
      "epoch": 14.09,
      "learning_rate": 5.971590909090909e-05,
      "loss": 2.7569,
      "step": 242550
    },
    {
      "activations/layer0_attention_weight_max": 16.206138610839844,
      "activations/layer0_attention_weight_min": -12.017337799072266,
      "activations/layer10_attention_weight_max": 37.84111785888672,
      "activations/layer10_attention_weight_min": -35.48625183105469,
      "activations/layer11_attention_weight_max": 36.19883728027344,
      "activations/layer11_attention_weight_min": -36.411643981933594,
      "activations/layer12_attention_weight_max": 25.42936134338379,
      "activations/layer12_attention_weight_min": -25.887414932250977,
      "activations/layer13_attention_weight_max": 42.63369369506836,
      "activations/layer13_attention_weight_min": -35.67898178100586,
      "activations/layer14_attention_weight_max": 43.4754638671875,
      "activations/layer14_attention_weight_min": -39.09854507446289,
      "activations/layer15_attention_weight_max": 35.80593490600586,
      "activations/layer15_attention_weight_min": -36.335689544677734,
      "activations/layer16_attention_weight_max": 28.53388786315918,
      "activations/layer16_attention_weight_min": -27.671789169311523,
      "activations/layer17_attention_weight_max": 27.92429542541504,
      "activations/layer17_attention_weight_min": -27.023210525512695,
      "activations/layer18_attention_weight_max": 29.377132415771484,
      "activations/layer18_attention_weight_min": -24.30056381225586,
      "activations/layer19_attention_weight_max": 34.43300247192383,
      "activations/layer19_attention_weight_min": -31.050317764282227,
      "activations/layer1_attention_weight_max": 16.787282943725586,
      "activations/layer1_attention_weight_min": -15.113126754760742,
      "activations/layer20_attention_weight_max": 28.70638656616211,
      "activations/layer20_attention_weight_min": -25.858320236206055,
      "activations/layer21_attention_weight_max": 29.17440414428711,
      "activations/layer21_attention_weight_min": -24.605077743530273,
      "activations/layer22_attention_weight_max": 39.7703857421875,
      "activations/layer22_attention_weight_min": -33.009273529052734,
      "activations/layer23_attention_weight_max": 33.38917541503906,
      "activations/layer23_attention_weight_min": -27.138092041015625,
      "activations/layer2_attention_weight_max": 33.319210052490234,
      "activations/layer2_attention_weight_min": -34.81710433959961,
      "activations/layer3_attention_weight_max": 97.8362808227539,
      "activations/layer3_attention_weight_min": -101.9514389038086,
      "activations/layer4_attention_weight_max": 108.34175872802734,
      "activations/layer4_attention_weight_min": -113.08473205566406,
      "activations/layer5_attention_weight_max": 51.578369140625,
      "activations/layer5_attention_weight_min": -63.26469039916992,
      "activations/layer6_attention_weight_max": 44.629276275634766,
      "activations/layer6_attention_weight_min": -47.39219665527344,
      "activations/layer7_attention_weight_max": 102.05117797851562,
      "activations/layer7_attention_weight_min": -94.79447937011719,
      "activations/layer8_attention_weight_max": 45.14695358276367,
      "activations/layer8_attention_weight_min": -46.3943977355957,
      "activations/layer9_attention_weight_max": 36.4261589050293,
      "activations/layer9_attention_weight_min": -36.1262321472168,
      "epoch": 14.1,
      "learning_rate": 5.969696969696969e-05,
      "loss": 2.745,
      "step": 242600
    },
    {
      "activations/layer0_attention_weight_max": 15.816389083862305,
      "activations/layer0_attention_weight_min": -12.208440780639648,
      "activations/layer10_attention_weight_max": 31.7463436126709,
      "activations/layer10_attention_weight_min": -32.9213981628418,
      "activations/layer11_attention_weight_max": 30.78261947631836,
      "activations/layer11_attention_weight_min": -31.22905731201172,
      "activations/layer12_attention_weight_max": 27.42970085144043,
      "activations/layer12_attention_weight_min": -25.10036849975586,
      "activations/layer13_attention_weight_max": 42.16801452636719,
      "activations/layer13_attention_weight_min": -35.001827239990234,
      "activations/layer14_attention_weight_max": 48.69941711425781,
      "activations/layer14_attention_weight_min": -37.1065559387207,
      "activations/layer15_attention_weight_max": 34.51532745361328,
      "activations/layer15_attention_weight_min": -33.69789505004883,
      "activations/layer16_attention_weight_max": 26.567508697509766,
      "activations/layer16_attention_weight_min": -28.194738388061523,
      "activations/layer17_attention_weight_max": 29.020633697509766,
      "activations/layer17_attention_weight_min": -25.81095314025879,
      "activations/layer18_attention_weight_max": 30.648151397705078,
      "activations/layer18_attention_weight_min": -27.370403289794922,
      "activations/layer19_attention_weight_max": 34.54621124267578,
      "activations/layer19_attention_weight_min": -32.08686065673828,
      "activations/layer1_attention_weight_max": 16.478378295898438,
      "activations/layer1_attention_weight_min": -16.71761703491211,
      "activations/layer20_attention_weight_max": 29.91119956970215,
      "activations/layer20_attention_weight_min": -26.525894165039062,
      "activations/layer21_attention_weight_max": 27.401018142700195,
      "activations/layer21_attention_weight_min": -25.842790603637695,
      "activations/layer22_attention_weight_max": 43.181129455566406,
      "activations/layer22_attention_weight_min": -33.484275817871094,
      "activations/layer23_attention_weight_max": 36.43634033203125,
      "activations/layer23_attention_weight_min": -27.228946685791016,
      "activations/layer2_attention_weight_max": 33.68021774291992,
      "activations/layer2_attention_weight_min": -34.60887145996094,
      "activations/layer3_attention_weight_max": 98.01812744140625,
      "activations/layer3_attention_weight_min": -104.83165740966797,
      "activations/layer4_attention_weight_max": 110.41755676269531,
      "activations/layer4_attention_weight_min": -112.18375396728516,
      "activations/layer5_attention_weight_max": 52.137264251708984,
      "activations/layer5_attention_weight_min": -59.01496505737305,
      "activations/layer6_attention_weight_max": 43.35160446166992,
      "activations/layer6_attention_weight_min": -47.79523468017578,
      "activations/layer7_attention_weight_max": 89.11531066894531,
      "activations/layer7_attention_weight_min": -93.23474884033203,
      "activations/layer8_attention_weight_max": 40.19860076904297,
      "activations/layer8_attention_weight_min": -44.300777435302734,
      "activations/layer9_attention_weight_max": 32.45138168334961,
      "activations/layer9_attention_weight_min": -35.0493278503418,
      "epoch": 14.1,
      "learning_rate": 5.9678030303030295e-05,
      "loss": 2.76,
      "step": 242650
    },
    {
      "activations/layer0_attention_weight_max": 15.232953071594238,
      "activations/layer0_attention_weight_min": -12.30355167388916,
      "activations/layer10_attention_weight_max": 32.097869873046875,
      "activations/layer10_attention_weight_min": -33.50019073486328,
      "activations/layer11_attention_weight_max": 32.52638244628906,
      "activations/layer11_attention_weight_min": -32.15898513793945,
      "activations/layer12_attention_weight_max": 24.63123321533203,
      "activations/layer12_attention_weight_min": -25.175308227539062,
      "activations/layer13_attention_weight_max": 36.921504974365234,
      "activations/layer13_attention_weight_min": -35.52513885498047,
      "activations/layer14_attention_weight_max": 40.452125549316406,
      "activations/layer14_attention_weight_min": -37.21587371826172,
      "activations/layer15_attention_weight_max": 36.011898040771484,
      "activations/layer15_attention_weight_min": -32.593658447265625,
      "activations/layer16_attention_weight_max": 28.125917434692383,
      "activations/layer16_attention_weight_min": -27.423696517944336,
      "activations/layer17_attention_weight_max": 29.0789794921875,
      "activations/layer17_attention_weight_min": -25.5491886138916,
      "activations/layer18_attention_weight_max": 30.150108337402344,
      "activations/layer18_attention_weight_min": -25.021053314208984,
      "activations/layer19_attention_weight_max": 33.76115417480469,
      "activations/layer19_attention_weight_min": -33.102821350097656,
      "activations/layer1_attention_weight_max": 15.913707733154297,
      "activations/layer1_attention_weight_min": -13.513215065002441,
      "activations/layer20_attention_weight_max": 28.427019119262695,
      "activations/layer20_attention_weight_min": -25.665292739868164,
      "activations/layer21_attention_weight_max": 29.97139549255371,
      "activations/layer21_attention_weight_min": -25.78938865661621,
      "activations/layer22_attention_weight_max": 42.704368591308594,
      "activations/layer22_attention_weight_min": -29.54442596435547,
      "activations/layer23_attention_weight_max": 34.628273010253906,
      "activations/layer23_attention_weight_min": -25.590381622314453,
      "activations/layer2_attention_weight_max": 33.8962287902832,
      "activations/layer2_attention_weight_min": -35.23630142211914,
      "activations/layer3_attention_weight_max": 97.96651458740234,
      "activations/layer3_attention_weight_min": -104.01495361328125,
      "activations/layer4_attention_weight_max": 108.779052734375,
      "activations/layer4_attention_weight_min": -112.14678955078125,
      "activations/layer5_attention_weight_max": 53.00091552734375,
      "activations/layer5_attention_weight_min": -65.27406311035156,
      "activations/layer6_attention_weight_max": 46.39197540283203,
      "activations/layer6_attention_weight_min": -47.359596252441406,
      "activations/layer7_attention_weight_max": 89.42189025878906,
      "activations/layer7_attention_weight_min": -95.7354507446289,
      "activations/layer8_attention_weight_max": 40.01480484008789,
      "activations/layer8_attention_weight_min": -41.96310043334961,
      "activations/layer9_attention_weight_max": 32.793758392333984,
      "activations/layer9_attention_weight_min": -33.84321594238281,
      "epoch": 14.1,
      "learning_rate": 5.96590909090909e-05,
      "loss": 2.7524,
      "step": 242700
    },
    {
      "activations/layer0_attention_weight_max": 15.099686622619629,
      "activations/layer0_attention_weight_min": -12.560498237609863,
      "activations/layer10_attention_weight_max": 31.75766372680664,
      "activations/layer10_attention_weight_min": -32.22425079345703,
      "activations/layer11_attention_weight_max": 33.040504455566406,
      "activations/layer11_attention_weight_min": -32.29815673828125,
      "activations/layer12_attention_weight_max": 24.55898666381836,
      "activations/layer12_attention_weight_min": -26.16889762878418,
      "activations/layer13_attention_weight_max": 37.27170944213867,
      "activations/layer13_attention_weight_min": -33.706993103027344,
      "activations/layer14_attention_weight_max": 40.37721633911133,
      "activations/layer14_attention_weight_min": -37.39033508300781,
      "activations/layer15_attention_weight_max": 34.789161682128906,
      "activations/layer15_attention_weight_min": -31.964000701904297,
      "activations/layer16_attention_weight_max": 27.72146987915039,
      "activations/layer16_attention_weight_min": -25.776865005493164,
      "activations/layer17_attention_weight_max": 29.94813346862793,
      "activations/layer17_attention_weight_min": -23.994884490966797,
      "activations/layer18_attention_weight_max": 31.89024543762207,
      "activations/layer18_attention_weight_min": -25.614660263061523,
      "activations/layer19_attention_weight_max": 31.583337783813477,
      "activations/layer19_attention_weight_min": -29.85618019104004,
      "activations/layer1_attention_weight_max": 15.790057182312012,
      "activations/layer1_attention_weight_min": -14.196636199951172,
      "activations/layer20_attention_weight_max": 27.320592880249023,
      "activations/layer20_attention_weight_min": -24.472728729248047,
      "activations/layer21_attention_weight_max": 29.795841217041016,
      "activations/layer21_attention_weight_min": -23.906713485717773,
      "activations/layer22_attention_weight_max": 43.17924118041992,
      "activations/layer22_attention_weight_min": -30.436201095581055,
      "activations/layer23_attention_weight_max": 30.96761131286621,
      "activations/layer23_attention_weight_min": -26.229202270507812,
      "activations/layer2_attention_weight_max": 32.8215446472168,
      "activations/layer2_attention_weight_min": -32.370018005371094,
      "activations/layer3_attention_weight_max": 94.61123657226562,
      "activations/layer3_attention_weight_min": -97.71566009521484,
      "activations/layer4_attention_weight_max": 106.12377166748047,
      "activations/layer4_attention_weight_min": -113.14649963378906,
      "activations/layer5_attention_weight_max": 48.49433135986328,
      "activations/layer5_attention_weight_min": -61.15926742553711,
      "activations/layer6_attention_weight_max": 43.18859100341797,
      "activations/layer6_attention_weight_min": -46.45509719848633,
      "activations/layer7_attention_weight_max": 87.69085693359375,
      "activations/layer7_attention_weight_min": -89.11016845703125,
      "activations/layer8_attention_weight_max": 38.700706481933594,
      "activations/layer8_attention_weight_min": -41.614891052246094,
      "activations/layer9_attention_weight_max": 31.98295021057129,
      "activations/layer9_attention_weight_min": -32.21256637573242,
      "epoch": 14.11,
      "learning_rate": 5.964015151515151e-05,
      "loss": 2.763,
      "step": 242750
    },
    {
      "activations/layer0_attention_weight_max": 15.906901359558105,
      "activations/layer0_attention_weight_min": -12.614677429199219,
      "activations/layer10_attention_weight_max": 32.93608856201172,
      "activations/layer10_attention_weight_min": -32.15306854248047,
      "activations/layer11_attention_weight_max": 32.088096618652344,
      "activations/layer11_attention_weight_min": -34.486751556396484,
      "activations/layer12_attention_weight_max": 26.096616744995117,
      "activations/layer12_attention_weight_min": -25.115243911743164,
      "activations/layer13_attention_weight_max": 35.848628997802734,
      "activations/layer13_attention_weight_min": -33.02771759033203,
      "activations/layer14_attention_weight_max": 39.17979049682617,
      "activations/layer14_attention_weight_min": -34.92563247680664,
      "activations/layer15_attention_weight_max": 33.42475891113281,
      "activations/layer15_attention_weight_min": -31.077272415161133,
      "activations/layer16_attention_weight_max": 29.732784271240234,
      "activations/layer16_attention_weight_min": -27.217588424682617,
      "activations/layer17_attention_weight_max": 30.581735610961914,
      "activations/layer17_attention_weight_min": -24.91855812072754,
      "activations/layer18_attention_weight_max": 30.574851989746094,
      "activations/layer18_attention_weight_min": -25.205217361450195,
      "activations/layer19_attention_weight_max": 32.43900680541992,
      "activations/layer19_attention_weight_min": -32.1678581237793,
      "activations/layer1_attention_weight_max": 16.566652297973633,
      "activations/layer1_attention_weight_min": -13.974674224853516,
      "activations/layer20_attention_weight_max": 29.7598934173584,
      "activations/layer20_attention_weight_min": -26.037935256958008,
      "activations/layer21_attention_weight_max": 29.1427001953125,
      "activations/layer21_attention_weight_min": -25.553312301635742,
      "activations/layer22_attention_weight_max": 40.012054443359375,
      "activations/layer22_attention_weight_min": -30.55267333984375,
      "activations/layer23_attention_weight_max": 31.357879638671875,
      "activations/layer23_attention_weight_min": -28.854963302612305,
      "activations/layer2_attention_weight_max": 31.935285568237305,
      "activations/layer2_attention_weight_min": -31.749141693115234,
      "activations/layer3_attention_weight_max": 97.19342041015625,
      "activations/layer3_attention_weight_min": -93.2592544555664,
      "activations/layer4_attention_weight_max": 109.54695892333984,
      "activations/layer4_attention_weight_min": -103.46829986572266,
      "activations/layer5_attention_weight_max": 51.689002990722656,
      "activations/layer5_attention_weight_min": -63.36357116699219,
      "activations/layer6_attention_weight_max": 43.01258087158203,
      "activations/layer6_attention_weight_min": -47.49989700317383,
      "activations/layer7_attention_weight_max": 91.06907653808594,
      "activations/layer7_attention_weight_min": -88.53697967529297,
      "activations/layer8_attention_weight_max": 41.19890213012695,
      "activations/layer8_attention_weight_min": -40.64059066772461,
      "activations/layer9_attention_weight_max": 30.955411911010742,
      "activations/layer9_attention_weight_min": -33.336429595947266,
      "epoch": 14.11,
      "learning_rate": 5.962121212121212e-05,
      "loss": 2.77,
      "step": 242800
    },
    {
      "activations/layer0_attention_weight_max": 16.12624168395996,
      "activations/layer0_attention_weight_min": -12.38143253326416,
      "activations/layer10_attention_weight_max": 31.81076431274414,
      "activations/layer10_attention_weight_min": -31.339536666870117,
      "activations/layer11_attention_weight_max": 31.690319061279297,
      "activations/layer11_attention_weight_min": -31.20758628845215,
      "activations/layer12_attention_weight_max": 25.635480880737305,
      "activations/layer12_attention_weight_min": -24.28243637084961,
      "activations/layer13_attention_weight_max": 38.455623626708984,
      "activations/layer13_attention_weight_min": -33.977027893066406,
      "activations/layer14_attention_weight_max": 45.04326248168945,
      "activations/layer14_attention_weight_min": -35.14146423339844,
      "activations/layer15_attention_weight_max": 36.94119644165039,
      "activations/layer15_attention_weight_min": -33.399253845214844,
      "activations/layer16_attention_weight_max": 30.660648345947266,
      "activations/layer16_attention_weight_min": -25.906978607177734,
      "activations/layer17_attention_weight_max": 31.04066276550293,
      "activations/layer17_attention_weight_min": -26.053020477294922,
      "activations/layer18_attention_weight_max": 32.1170539855957,
      "activations/layer18_attention_weight_min": -24.927221298217773,
      "activations/layer19_attention_weight_max": 33.294471740722656,
      "activations/layer19_attention_weight_min": -30.673795700073242,
      "activations/layer1_attention_weight_max": 16.279987335205078,
      "activations/layer1_attention_weight_min": -14.332714080810547,
      "activations/layer20_attention_weight_max": 30.234567642211914,
      "activations/layer20_attention_weight_min": -24.752357482910156,
      "activations/layer21_attention_weight_max": 30.599008560180664,
      "activations/layer21_attention_weight_min": -25.3228816986084,
      "activations/layer22_attention_weight_max": 41.56227111816406,
      "activations/layer22_attention_weight_min": -32.45766067504883,
      "activations/layer23_attention_weight_max": 33.219398498535156,
      "activations/layer23_attention_weight_min": -26.248075485229492,
      "activations/layer2_attention_weight_max": 33.515682220458984,
      "activations/layer2_attention_weight_min": -32.77589797973633,
      "activations/layer3_attention_weight_max": 97.87915802001953,
      "activations/layer3_attention_weight_min": -94.53482818603516,
      "activations/layer4_attention_weight_max": 110.05606842041016,
      "activations/layer4_attention_weight_min": -106.5941390991211,
      "activations/layer5_attention_weight_max": 51.874351501464844,
      "activations/layer5_attention_weight_min": -67.1593246459961,
      "activations/layer6_attention_weight_max": 46.52332305908203,
      "activations/layer6_attention_weight_min": -45.22962188720703,
      "activations/layer7_attention_weight_max": 91.27458190917969,
      "activations/layer7_attention_weight_min": -93.16600036621094,
      "activations/layer8_attention_weight_max": 38.09135818481445,
      "activations/layer8_attention_weight_min": -40.69239044189453,
      "activations/layer9_attention_weight_max": 30.897436141967773,
      "activations/layer9_attention_weight_min": -31.732025146484375,
      "epoch": 14.11,
      "learning_rate": 5.960227272727272e-05,
      "loss": 2.7411,
      "step": 242850
    },
    {
      "activations/layer0_attention_weight_max": 16.425180435180664,
      "activations/layer0_attention_weight_min": -11.66867733001709,
      "activations/layer10_attention_weight_max": 32.76679611206055,
      "activations/layer10_attention_weight_min": -30.6541805267334,
      "activations/layer11_attention_weight_max": 31.937915802001953,
      "activations/layer11_attention_weight_min": -31.63775634765625,
      "activations/layer12_attention_weight_max": 25.407211303710938,
      "activations/layer12_attention_weight_min": -25.324325561523438,
      "activations/layer13_attention_weight_max": 39.99378204345703,
      "activations/layer13_attention_weight_min": -32.80315399169922,
      "activations/layer14_attention_weight_max": 42.42061233520508,
      "activations/layer14_attention_weight_min": -37.90067672729492,
      "activations/layer15_attention_weight_max": 39.64979553222656,
      "activations/layer15_attention_weight_min": -34.20473098754883,
      "activations/layer16_attention_weight_max": 30.208471298217773,
      "activations/layer16_attention_weight_min": -26.980379104614258,
      "activations/layer17_attention_weight_max": 31.243465423583984,
      "activations/layer17_attention_weight_min": -24.853378295898438,
      "activations/layer18_attention_weight_max": 33.0203971862793,
      "activations/layer18_attention_weight_min": -23.329858779907227,
      "activations/layer19_attention_weight_max": 34.399715423583984,
      "activations/layer19_attention_weight_min": -29.294477462768555,
      "activations/layer1_attention_weight_max": 16.34299087524414,
      "activations/layer1_attention_weight_min": -14.517916679382324,
      "activations/layer20_attention_weight_max": 29.83283042907715,
      "activations/layer20_attention_weight_min": -28.212785720825195,
      "activations/layer21_attention_weight_max": 29.14240074157715,
      "activations/layer21_attention_weight_min": -25.48833465576172,
      "activations/layer22_attention_weight_max": 38.60806655883789,
      "activations/layer22_attention_weight_min": -27.940767288208008,
      "activations/layer23_attention_weight_max": 33.412841796875,
      "activations/layer23_attention_weight_min": -25.888404846191406,
      "activations/layer2_attention_weight_max": 31.31963348388672,
      "activations/layer2_attention_weight_min": -32.4163818359375,
      "activations/layer3_attention_weight_max": 90.685791015625,
      "activations/layer3_attention_weight_min": -91.7188720703125,
      "activations/layer4_attention_weight_max": 103.0254898071289,
      "activations/layer4_attention_weight_min": -106.6011734008789,
      "activations/layer5_attention_weight_max": 50.298763275146484,
      "activations/layer5_attention_weight_min": -62.516258239746094,
      "activations/layer6_attention_weight_max": 40.54181671142578,
      "activations/layer6_attention_weight_min": -43.96210861206055,
      "activations/layer7_attention_weight_max": 85.76012420654297,
      "activations/layer7_attention_weight_min": -86.77684020996094,
      "activations/layer8_attention_weight_max": 38.06584548950195,
      "activations/layer8_attention_weight_min": -40.851600646972656,
      "activations/layer9_attention_weight_max": 32.02527618408203,
      "activations/layer9_attention_weight_min": -32.06633758544922,
      "epoch": 14.11,
      "learning_rate": 5.958333333333332e-05,
      "loss": 2.7542,
      "step": 242900
    },
    {
      "activations/layer0_attention_weight_max": 15.61172866821289,
      "activations/layer0_attention_weight_min": -12.372537612915039,
      "activations/layer10_attention_weight_max": 31.12216567993164,
      "activations/layer10_attention_weight_min": -32.913185119628906,
      "activations/layer11_attention_weight_max": 32.27749252319336,
      "activations/layer11_attention_weight_min": -31.329574584960938,
      "activations/layer12_attention_weight_max": 30.319480895996094,
      "activations/layer12_attention_weight_min": -25.479122161865234,
      "activations/layer13_attention_weight_max": 37.94887161254883,
      "activations/layer13_attention_weight_min": -32.28718948364258,
      "activations/layer14_attention_weight_max": 42.581817626953125,
      "activations/layer14_attention_weight_min": -34.903099060058594,
      "activations/layer15_attention_weight_max": 33.79156494140625,
      "activations/layer15_attention_weight_min": -30.965524673461914,
      "activations/layer16_attention_weight_max": 30.667383193969727,
      "activations/layer16_attention_weight_min": -26.89545249938965,
      "activations/layer17_attention_weight_max": 27.62181282043457,
      "activations/layer17_attention_weight_min": -25.509546279907227,
      "activations/layer18_attention_weight_max": 30.619998931884766,
      "activations/layer18_attention_weight_min": -22.392398834228516,
      "activations/layer19_attention_weight_max": 32.363643646240234,
      "activations/layer19_attention_weight_min": -29.77972412109375,
      "activations/layer1_attention_weight_max": 15.750434875488281,
      "activations/layer1_attention_weight_min": -14.381442070007324,
      "activations/layer20_attention_weight_max": 29.64280891418457,
      "activations/layer20_attention_weight_min": -22.90814208984375,
      "activations/layer21_attention_weight_max": 29.55089569091797,
      "activations/layer21_attention_weight_min": -23.6160945892334,
      "activations/layer22_attention_weight_max": 38.94371795654297,
      "activations/layer22_attention_weight_min": -30.03731918334961,
      "activations/layer23_attention_weight_max": 32.596214294433594,
      "activations/layer23_attention_weight_min": -25.915090560913086,
      "activations/layer2_attention_weight_max": 34.41731643676758,
      "activations/layer2_attention_weight_min": -34.485374450683594,
      "activations/layer3_attention_weight_max": 99.18238830566406,
      "activations/layer3_attention_weight_min": -101.68951416015625,
      "activations/layer4_attention_weight_max": 109.71517181396484,
      "activations/layer4_attention_weight_min": -110.08995819091797,
      "activations/layer5_attention_weight_max": 51.42597579956055,
      "activations/layer5_attention_weight_min": -65.53831481933594,
      "activations/layer6_attention_weight_max": 43.992549896240234,
      "activations/layer6_attention_weight_min": -45.17702102661133,
      "activations/layer7_attention_weight_max": 91.31343078613281,
      "activations/layer7_attention_weight_min": -89.18500518798828,
      "activations/layer8_attention_weight_max": 38.8250617980957,
      "activations/layer8_attention_weight_min": -40.70369338989258,
      "activations/layer9_attention_weight_max": 31.2781982421875,
      "activations/layer9_attention_weight_min": -33.07900619506836,
      "epoch": 14.12,
      "learning_rate": 5.956439393939394e-05,
      "loss": 2.7551,
      "step": 242950
    },
    {
      "activations/layer0_attention_weight_max": 15.807869911193848,
      "activations/layer0_attention_weight_min": -12.23099136352539,
      "activations/layer10_attention_weight_max": 32.133636474609375,
      "activations/layer10_attention_weight_min": -35.174346923828125,
      "activations/layer11_attention_weight_max": 33.13531494140625,
      "activations/layer11_attention_weight_min": -33.948707580566406,
      "activations/layer12_attention_weight_max": 28.618741989135742,
      "activations/layer12_attention_weight_min": -25.790082931518555,
      "activations/layer13_attention_weight_max": 40.906978607177734,
      "activations/layer13_attention_weight_min": -33.6772346496582,
      "activations/layer14_attention_weight_max": 44.10761260986328,
      "activations/layer14_attention_weight_min": -39.53953170776367,
      "activations/layer15_attention_weight_max": 41.99967575073242,
      "activations/layer15_attention_weight_min": -35.762691497802734,
      "activations/layer16_attention_weight_max": 31.365129470825195,
      "activations/layer16_attention_weight_min": -27.075485229492188,
      "activations/layer17_attention_weight_max": 33.77052688598633,
      "activations/layer17_attention_weight_min": -26.40485191345215,
      "activations/layer18_attention_weight_max": 32.716705322265625,
      "activations/layer18_attention_weight_min": -24.182233810424805,
      "activations/layer19_attention_weight_max": 38.307464599609375,
      "activations/layer19_attention_weight_min": -31.63886070251465,
      "activations/layer1_attention_weight_max": 16.883298873901367,
      "activations/layer1_attention_weight_min": -14.018674850463867,
      "activations/layer20_attention_weight_max": 31.587095260620117,
      "activations/layer20_attention_weight_min": -26.130447387695312,
      "activations/layer21_attention_weight_max": 30.879629135131836,
      "activations/layer21_attention_weight_min": -25.137065887451172,
      "activations/layer22_attention_weight_max": 48.44251251220703,
      "activations/layer22_attention_weight_min": -33.20868682861328,
      "activations/layer23_attention_weight_max": 36.79458236694336,
      "activations/layer23_attention_weight_min": -28.850324630737305,
      "activations/layer2_attention_weight_max": 33.52768325805664,
      "activations/layer2_attention_weight_min": -33.13544845581055,
      "activations/layer3_attention_weight_max": 101.07396697998047,
      "activations/layer3_attention_weight_min": -98.05365753173828,
      "activations/layer4_attention_weight_max": 111.80047607421875,
      "activations/layer4_attention_weight_min": -109.71260070800781,
      "activations/layer5_attention_weight_max": 52.477413177490234,
      "activations/layer5_attention_weight_min": -63.61117172241211,
      "activations/layer6_attention_weight_max": 42.7869758605957,
      "activations/layer6_attention_weight_min": -45.71031951904297,
      "activations/layer7_attention_weight_max": 88.99589538574219,
      "activations/layer7_attention_weight_min": -89.26664733886719,
      "activations/layer8_attention_weight_max": 39.33140182495117,
      "activations/layer8_attention_weight_min": -41.08820343017578,
      "activations/layer9_attention_weight_max": 32.94243621826172,
      "activations/layer9_attention_weight_min": -34.447166442871094,
      "epoch": 14.12,
      "learning_rate": 5.954545454545454e-05,
      "loss": 2.7388,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.5313,
      "eval_samples_per_second": 503.324,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.5313,
      "eval_openwebtext_samples_per_second": 503.324,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 2.0172,
      "eval_wikitext_samples_per_second": 226.054,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_lambada_loss": 2.498046875,
      "eval_lambada_ppl": 12.158723248329283,
      "eval_lambada_runtime": 9.5713,
      "eval_lambada_samples_per_second": 508.709,
      "step": 243000
    },
    {
      "activations/layer0_attention_weight_max": 15.900397300720215,
      "activations/layer0_attention_weight_min": -12.696383476257324,
      "activations/layer10_attention_weight_max": 33.986175537109375,
      "activations/layer10_attention_weight_min": -32.81871795654297,
      "activations/layer11_attention_weight_max": 32.918277740478516,
      "activations/layer11_attention_weight_min": -31.783246994018555,
      "activations/layer12_attention_weight_max": 24.6502628326416,
      "activations/layer12_attention_weight_min": -24.792884826660156,
      "activations/layer13_attention_weight_max": 38.48795700073242,
      "activations/layer13_attention_weight_min": -34.98373794555664,
      "activations/layer14_attention_weight_max": 45.20796585083008,
      "activations/layer14_attention_weight_min": -35.841529846191406,
      "activations/layer15_attention_weight_max": 35.80373001098633,
      "activations/layer15_attention_weight_min": -33.32345199584961,
      "activations/layer16_attention_weight_max": 32.371368408203125,
      "activations/layer16_attention_weight_min": -28.489620208740234,
      "activations/layer17_attention_weight_max": 30.04364776611328,
      "activations/layer17_attention_weight_min": -25.048080444335938,
      "activations/layer18_attention_weight_max": 31.644258499145508,
      "activations/layer18_attention_weight_min": -25.033344268798828,
      "activations/layer19_attention_weight_max": 33.36181640625,
      "activations/layer19_attention_weight_min": -32.39028549194336,
      "activations/layer1_attention_weight_max": 18.414447784423828,
      "activations/layer1_attention_weight_min": -14.850950241088867,
      "activations/layer20_attention_weight_max": 29.41288948059082,
      "activations/layer20_attention_weight_min": -26.841962814331055,
      "activations/layer21_attention_weight_max": 28.743255615234375,
      "activations/layer21_attention_weight_min": -26.32083511352539,
      "activations/layer22_attention_weight_max": 39.40498733520508,
      "activations/layer22_attention_weight_min": -34.43759536743164,
      "activations/layer23_attention_weight_max": 36.27920150756836,
      "activations/layer23_attention_weight_min": -25.836776733398438,
      "activations/layer2_attention_weight_max": 33.94289016723633,
      "activations/layer2_attention_weight_min": -32.84516906738281,
      "activations/layer3_attention_weight_max": 101.49028015136719,
      "activations/layer3_attention_weight_min": -97.17530822753906,
      "activations/layer4_attention_weight_max": 114.27547454833984,
      "activations/layer4_attention_weight_min": -114.21612548828125,
      "activations/layer5_attention_weight_max": 52.965938568115234,
      "activations/layer5_attention_weight_min": -63.24937438964844,
      "activations/layer6_attention_weight_max": 45.3953971862793,
      "activations/layer6_attention_weight_min": -47.38993453979492,
      "activations/layer7_attention_weight_max": 97.99121856689453,
      "activations/layer7_attention_weight_min": -90.9104232788086,
      "activations/layer8_attention_weight_max": 42.040550231933594,
      "activations/layer8_attention_weight_min": -40.924171447753906,
      "activations/layer9_attention_weight_max": 36.244632720947266,
      "activations/layer9_attention_weight_min": -33.95210266113281,
      "epoch": 14.12,
      "learning_rate": 5.952651515151515e-05,
      "loss": 2.7512,
      "step": 243050
    },
    {
      "activations/layer0_attention_weight_max": 16.058706283569336,
      "activations/layer0_attention_weight_min": -12.05135440826416,
      "activations/layer10_attention_weight_max": 40.59329605102539,
      "activations/layer10_attention_weight_min": -38.38847732543945,
      "activations/layer11_attention_weight_max": 41.44565200805664,
      "activations/layer11_attention_weight_min": -38.08842468261719,
      "activations/layer12_attention_weight_max": 28.588943481445312,
      "activations/layer12_attention_weight_min": -27.033952713012695,
      "activations/layer13_attention_weight_max": 37.02309799194336,
      "activations/layer13_attention_weight_min": -36.56036376953125,
      "activations/layer14_attention_weight_max": 46.518253326416016,
      "activations/layer14_attention_weight_min": -40.25200271606445,
      "activations/layer15_attention_weight_max": 45.64653778076172,
      "activations/layer15_attention_weight_min": -41.45578384399414,
      "activations/layer16_attention_weight_max": 31.847135543823242,
      "activations/layer16_attention_weight_min": -29.7523193359375,
      "activations/layer17_attention_weight_max": 32.18016052246094,
      "activations/layer17_attention_weight_min": -27.09312629699707,
      "activations/layer18_attention_weight_max": 28.53387451171875,
      "activations/layer18_attention_weight_min": -22.95603370666504,
      "activations/layer19_attention_weight_max": 34.08895492553711,
      "activations/layer19_attention_weight_min": -28.95230484008789,
      "activations/layer1_attention_weight_max": 16.363481521606445,
      "activations/layer1_attention_weight_min": -15.114912033081055,
      "activations/layer20_attention_weight_max": 32.86640930175781,
      "activations/layer20_attention_weight_min": -23.529769897460938,
      "activations/layer21_attention_weight_max": 30.93068504333496,
      "activations/layer21_attention_weight_min": -23.30986976623535,
      "activations/layer22_attention_weight_max": 41.80720520019531,
      "activations/layer22_attention_weight_min": -28.98674774169922,
      "activations/layer23_attention_weight_max": 34.121585845947266,
      "activations/layer23_attention_weight_min": -24.376890182495117,
      "activations/layer2_attention_weight_max": 36.05760192871094,
      "activations/layer2_attention_weight_min": -36.155555725097656,
      "activations/layer3_attention_weight_max": 103.634521484375,
      "activations/layer3_attention_weight_min": -100.6122055053711,
      "activations/layer4_attention_weight_max": 109.73712158203125,
      "activations/layer4_attention_weight_min": -114.665771484375,
      "activations/layer5_attention_weight_max": 51.796180725097656,
      "activations/layer5_attention_weight_min": -61.948516845703125,
      "activations/layer6_attention_weight_max": 43.17811584472656,
      "activations/layer6_attention_weight_min": -45.624351501464844,
      "activations/layer7_attention_weight_max": 106.26956176757812,
      "activations/layer7_attention_weight_min": -97.9579086303711,
      "activations/layer8_attention_weight_max": 45.390380859375,
      "activations/layer8_attention_weight_min": -45.854881286621094,
      "activations/layer9_attention_weight_max": 38.04325485229492,
      "activations/layer9_attention_weight_min": -38.9178466796875,
      "epoch": 14.13,
      "learning_rate": 5.950757575757575e-05,
      "loss": 2.7567,
      "step": 243100
    },
    {
      "activations/layer0_attention_weight_max": 15.844497680664062,
      "activations/layer0_attention_weight_min": -11.956781387329102,
      "activations/layer10_attention_weight_max": 36.41858673095703,
      "activations/layer10_attention_weight_min": -35.069190979003906,
      "activations/layer11_attention_weight_max": 36.984405517578125,
      "activations/layer11_attention_weight_min": -36.589256286621094,
      "activations/layer12_attention_weight_max": 26.796907424926758,
      "activations/layer12_attention_weight_min": -28.098873138427734,
      "activations/layer13_attention_weight_max": 42.30454635620117,
      "activations/layer13_attention_weight_min": -35.73529052734375,
      "activations/layer14_attention_weight_max": 51.99300765991211,
      "activations/layer14_attention_weight_min": -45.391204833984375,
      "activations/layer15_attention_weight_max": 48.01551818847656,
      "activations/layer15_attention_weight_min": -37.88515090942383,
      "activations/layer16_attention_weight_max": 40.27861404418945,
      "activations/layer16_attention_weight_min": -28.041582107543945,
      "activations/layer17_attention_weight_max": 36.175682067871094,
      "activations/layer17_attention_weight_min": -29.98344612121582,
      "activations/layer18_attention_weight_max": 32.756752014160156,
      "activations/layer18_attention_weight_min": -26.464921951293945,
      "activations/layer19_attention_weight_max": 39.754276275634766,
      "activations/layer19_attention_weight_min": -34.04808044433594,
      "activations/layer1_attention_weight_max": 16.875398635864258,
      "activations/layer1_attention_weight_min": -13.294346809387207,
      "activations/layer20_attention_weight_max": 31.92385482788086,
      "activations/layer20_attention_weight_min": -23.931312561035156,
      "activations/layer21_attention_weight_max": 31.219980239868164,
      "activations/layer21_attention_weight_min": -23.24500846862793,
      "activations/layer22_attention_weight_max": 44.06194305419922,
      "activations/layer22_attention_weight_min": -33.978904724121094,
      "activations/layer23_attention_weight_max": 31.057348251342773,
      "activations/layer23_attention_weight_min": -25.394474029541016,
      "activations/layer2_attention_weight_max": 37.11567687988281,
      "activations/layer2_attention_weight_min": -39.418922424316406,
      "activations/layer3_attention_weight_max": 107.75603485107422,
      "activations/layer3_attention_weight_min": -110.46883392333984,
      "activations/layer4_attention_weight_max": 117.172119140625,
      "activations/layer4_attention_weight_min": -111.9287109375,
      "activations/layer5_attention_weight_max": 55.06136703491211,
      "activations/layer5_attention_weight_min": -62.634193420410156,
      "activations/layer6_attention_weight_max": 43.07852554321289,
      "activations/layer6_attention_weight_min": -43.69685363769531,
      "activations/layer7_attention_weight_max": 93.66988372802734,
      "activations/layer7_attention_weight_min": -86.97705841064453,
      "activations/layer8_attention_weight_max": 40.6148567199707,
      "activations/layer8_attention_weight_min": -40.6183967590332,
      "activations/layer9_attention_weight_max": 33.037776947021484,
      "activations/layer9_attention_weight_min": -33.98158645629883,
      "epoch": 14.13,
      "learning_rate": 5.9488636363636364e-05,
      "loss": 2.7579,
      "step": 243150
    },
    {
      "activations/layer0_attention_weight_max": 15.867072105407715,
      "activations/layer0_attention_weight_min": -12.806567192077637,
      "activations/layer10_attention_weight_max": 40.91004943847656,
      "activations/layer10_attention_weight_min": -39.03449249267578,
      "activations/layer11_attention_weight_max": 41.11021041870117,
      "activations/layer11_attention_weight_min": -37.64169692993164,
      "activations/layer12_attention_weight_max": 38.793392181396484,
      "activations/layer12_attention_weight_min": -28.196460723876953,
      "activations/layer13_attention_weight_max": 54.05002975463867,
      "activations/layer13_attention_weight_min": -38.32746505737305,
      "activations/layer14_attention_weight_max": 55.69451141357422,
      "activations/layer14_attention_weight_min": -43.82381820678711,
      "activations/layer15_attention_weight_max": 52.504024505615234,
      "activations/layer15_attention_weight_min": -38.24187088012695,
      "activations/layer16_attention_weight_max": 38.43375778198242,
      "activations/layer16_attention_weight_min": -29.74555206298828,
      "activations/layer17_attention_weight_max": 37.35630798339844,
      "activations/layer17_attention_weight_min": -29.04781723022461,
      "activations/layer18_attention_weight_max": 39.81507110595703,
      "activations/layer18_attention_weight_min": -26.11187744140625,
      "activations/layer19_attention_weight_max": 44.8433723449707,
      "activations/layer19_attention_weight_min": -32.166378021240234,
      "activations/layer1_attention_weight_max": 16.775651931762695,
      "activations/layer1_attention_weight_min": -14.095927238464355,
      "activations/layer20_attention_weight_max": 37.114784240722656,
      "activations/layer20_attention_weight_min": -24.8523006439209,
      "activations/layer21_attention_weight_max": 39.32725143432617,
      "activations/layer21_attention_weight_min": -25.260025024414062,
      "activations/layer22_attention_weight_max": 62.19828414916992,
      "activations/layer22_attention_weight_min": -31.583393096923828,
      "activations/layer23_attention_weight_max": 41.77622985839844,
      "activations/layer23_attention_weight_min": -24.894607543945312,
      "activations/layer2_attention_weight_max": 34.728580474853516,
      "activations/layer2_attention_weight_min": -34.99943542480469,
      "activations/layer3_attention_weight_max": 101.5650634765625,
      "activations/layer3_attention_weight_min": -100.44036865234375,
      "activations/layer4_attention_weight_max": 117.75148010253906,
      "activations/layer4_attention_weight_min": -114.18407440185547,
      "activations/layer5_attention_weight_max": 53.73712921142578,
      "activations/layer5_attention_weight_min": -62.01079559326172,
      "activations/layer6_attention_weight_max": 50.67821502685547,
      "activations/layer6_attention_weight_min": -49.35301208496094,
      "activations/layer7_attention_weight_max": 111.08415222167969,
      "activations/layer7_attention_weight_min": -104.17052459716797,
      "activations/layer8_attention_weight_max": 49.596107482910156,
      "activations/layer8_attention_weight_min": -48.41607666015625,
      "activations/layer9_attention_weight_max": 41.816436767578125,
      "activations/layer9_attention_weight_min": -40.0343132019043,
      "epoch": 14.13,
      "learning_rate": 5.9469696969696965e-05,
      "loss": 2.7427,
      "step": 243200
    },
    {
      "activations/layer0_attention_weight_max": 15.281546592712402,
      "activations/layer0_attention_weight_min": -12.70136833190918,
      "activations/layer10_attention_weight_max": 33.431884765625,
      "activations/layer10_attention_weight_min": -33.07035446166992,
      "activations/layer11_attention_weight_max": 32.573482513427734,
      "activations/layer11_attention_weight_min": -32.377864837646484,
      "activations/layer12_attention_weight_max": 24.799903869628906,
      "activations/layer12_attention_weight_min": -26.79110336303711,
      "activations/layer13_attention_weight_max": 38.76800537109375,
      "activations/layer13_attention_weight_min": -36.185543060302734,
      "activations/layer14_attention_weight_max": 46.433753967285156,
      "activations/layer14_attention_weight_min": -38.85789489746094,
      "activations/layer15_attention_weight_max": 35.658935546875,
      "activations/layer15_attention_weight_min": -35.363040924072266,
      "activations/layer16_attention_weight_max": 30.2005615234375,
      "activations/layer16_attention_weight_min": -31.061389923095703,
      "activations/layer17_attention_weight_max": 32.58360290527344,
      "activations/layer17_attention_weight_min": -28.890560150146484,
      "activations/layer18_attention_weight_max": 32.13962936401367,
      "activations/layer18_attention_weight_min": -26.90776824951172,
      "activations/layer19_attention_weight_max": 31.498031616210938,
      "activations/layer19_attention_weight_min": -30.05781364440918,
      "activations/layer1_attention_weight_max": 16.258115768432617,
      "activations/layer1_attention_weight_min": -13.94377326965332,
      "activations/layer20_attention_weight_max": 27.31377601623535,
      "activations/layer20_attention_weight_min": -24.950260162353516,
      "activations/layer21_attention_weight_max": 28.898677825927734,
      "activations/layer21_attention_weight_min": -25.430999755859375,
      "activations/layer22_attention_weight_max": 45.41579818725586,
      "activations/layer22_attention_weight_min": -31.29517364501953,
      "activations/layer23_attention_weight_max": 34.76152038574219,
      "activations/layer23_attention_weight_min": -26.274076461791992,
      "activations/layer2_attention_weight_max": 31.55343246459961,
      "activations/layer2_attention_weight_min": -31.17101287841797,
      "activations/layer3_attention_weight_max": 92.93231201171875,
      "activations/layer3_attention_weight_min": -96.39527893066406,
      "activations/layer4_attention_weight_max": 107.47899627685547,
      "activations/layer4_attention_weight_min": -109.80791473388672,
      "activations/layer5_attention_weight_max": 50.37451934814453,
      "activations/layer5_attention_weight_min": -63.66149139404297,
      "activations/layer6_attention_weight_max": 44.26749801635742,
      "activations/layer6_attention_weight_min": -49.121055603027344,
      "activations/layer7_attention_weight_max": 90.43636322021484,
      "activations/layer7_attention_weight_min": -94.28526306152344,
      "activations/layer8_attention_weight_max": 41.124168395996094,
      "activations/layer8_attention_weight_min": -45.514400482177734,
      "activations/layer9_attention_weight_max": 32.23874282836914,
      "activations/layer9_attention_weight_min": -33.464229583740234,
      "epoch": 14.13,
      "learning_rate": 5.945075757575757e-05,
      "loss": 2.7396,
      "step": 243250
    },
    {
      "activations/layer0_attention_weight_max": 16.278732299804688,
      "activations/layer0_attention_weight_min": -12.141950607299805,
      "activations/layer10_attention_weight_max": 35.475006103515625,
      "activations/layer10_attention_weight_min": -34.851768493652344,
      "activations/layer11_attention_weight_max": 36.883975982666016,
      "activations/layer11_attention_weight_min": -35.24533462524414,
      "activations/layer12_attention_weight_max": 25.965951919555664,
      "activations/layer12_attention_weight_min": -26.3778133392334,
      "activations/layer13_attention_weight_max": 39.6545295715332,
      "activations/layer13_attention_weight_min": -35.50608444213867,
      "activations/layer14_attention_weight_max": 42.370033264160156,
      "activations/layer14_attention_weight_min": -37.47425079345703,
      "activations/layer15_attention_weight_max": 39.74467086791992,
      "activations/layer15_attention_weight_min": -34.00520706176758,
      "activations/layer16_attention_weight_max": 31.20786476135254,
      "activations/layer16_attention_weight_min": -27.70632553100586,
      "activations/layer17_attention_weight_max": 32.54787063598633,
      "activations/layer17_attention_weight_min": -26.63506507873535,
      "activations/layer18_attention_weight_max": 35.466957092285156,
      "activations/layer18_attention_weight_min": -24.779823303222656,
      "activations/layer19_attention_weight_max": 37.98074722290039,
      "activations/layer19_attention_weight_min": -32.41122817993164,
      "activations/layer1_attention_weight_max": 16.223201751708984,
      "activations/layer1_attention_weight_min": -14.680262565612793,
      "activations/layer20_attention_weight_max": 32.475223541259766,
      "activations/layer20_attention_weight_min": -28.81981658935547,
      "activations/layer21_attention_weight_max": 31.048452377319336,
      "activations/layer21_attention_weight_min": -27.056150436401367,
      "activations/layer22_attention_weight_max": 45.31008529663086,
      "activations/layer22_attention_weight_min": -34.719783782958984,
      "activations/layer23_attention_weight_max": 37.31055450439453,
      "activations/layer23_attention_weight_min": -27.337871551513672,
      "activations/layer2_attention_weight_max": 33.13477325439453,
      "activations/layer2_attention_weight_min": -32.08223342895508,
      "activations/layer3_attention_weight_max": 100.1761474609375,
      "activations/layer3_attention_weight_min": -97.24655151367188,
      "activations/layer4_attention_weight_max": 114.09319305419922,
      "activations/layer4_attention_weight_min": -111.33773040771484,
      "activations/layer5_attention_weight_max": 55.23426055908203,
      "activations/layer5_attention_weight_min": -65.2753677368164,
      "activations/layer6_attention_weight_max": 44.85658264160156,
      "activations/layer6_attention_weight_min": -48.440223693847656,
      "activations/layer7_attention_weight_max": 94.14691162109375,
      "activations/layer7_attention_weight_min": -90.85539245605469,
      "activations/layer8_attention_weight_max": 43.28936767578125,
      "activations/layer8_attention_weight_min": -42.46159744262695,
      "activations/layer9_attention_weight_max": 33.707847595214844,
      "activations/layer9_attention_weight_min": -35.3509521484375,
      "epoch": 14.14,
      "learning_rate": 5.943181818181818e-05,
      "loss": 2.7672,
      "step": 243300
    },
    {
      "activations/layer0_attention_weight_max": 15.4617280960083,
      "activations/layer0_attention_weight_min": -12.697002410888672,
      "activations/layer10_attention_weight_max": 35.394405364990234,
      "activations/layer10_attention_weight_min": -32.70505905151367,
      "activations/layer11_attention_weight_max": 36.05693054199219,
      "activations/layer11_attention_weight_min": -33.28417205810547,
      "activations/layer12_attention_weight_max": 25.20822525024414,
      "activations/layer12_attention_weight_min": -32.191749572753906,
      "activations/layer13_attention_weight_max": 38.5071907043457,
      "activations/layer13_attention_weight_min": -33.34471130371094,
      "activations/layer14_attention_weight_max": 40.09861373901367,
      "activations/layer14_attention_weight_min": -38.34377670288086,
      "activations/layer15_attention_weight_max": 35.15105438232422,
      "activations/layer15_attention_weight_min": -32.92173767089844,
      "activations/layer16_attention_weight_max": 29.400680541992188,
      "activations/layer16_attention_weight_min": -26.018787384033203,
      "activations/layer17_attention_weight_max": 31.89518165588379,
      "activations/layer17_attention_weight_min": -24.413541793823242,
      "activations/layer18_attention_weight_max": 30.802936553955078,
      "activations/layer18_attention_weight_min": -22.40814971923828,
      "activations/layer19_attention_weight_max": 34.47955322265625,
      "activations/layer19_attention_weight_min": -31.695749282836914,
      "activations/layer1_attention_weight_max": 16.259048461914062,
      "activations/layer1_attention_weight_min": -14.663386344909668,
      "activations/layer20_attention_weight_max": 28.849529266357422,
      "activations/layer20_attention_weight_min": -25.869626998901367,
      "activations/layer21_attention_weight_max": 27.145782470703125,
      "activations/layer21_attention_weight_min": -24.121973037719727,
      "activations/layer22_attention_weight_max": 42.79787826538086,
      "activations/layer22_attention_weight_min": -30.63655662536621,
      "activations/layer23_attention_weight_max": 32.55330276489258,
      "activations/layer23_attention_weight_min": -25.357818603515625,
      "activations/layer2_attention_weight_max": 32.73822784423828,
      "activations/layer2_attention_weight_min": -32.52115249633789,
      "activations/layer3_attention_weight_max": 101.18653106689453,
      "activations/layer3_attention_weight_min": -97.6474380493164,
      "activations/layer4_attention_weight_max": 113.3310317993164,
      "activations/layer4_attention_weight_min": -113.13590240478516,
      "activations/layer5_attention_weight_max": 53.15732955932617,
      "activations/layer5_attention_weight_min": -62.68193054199219,
      "activations/layer6_attention_weight_max": 46.425235748291016,
      "activations/layer6_attention_weight_min": -46.0458869934082,
      "activations/layer7_attention_weight_max": 92.33068084716797,
      "activations/layer7_attention_weight_min": -94.47459411621094,
      "activations/layer8_attention_weight_max": 41.34956741333008,
      "activations/layer8_attention_weight_min": -43.863277435302734,
      "activations/layer9_attention_weight_max": 33.84804916381836,
      "activations/layer9_attention_weight_min": -33.30999755859375,
      "epoch": 14.14,
      "learning_rate": 5.941287878787878e-05,
      "loss": 2.7382,
      "step": 243350
    },
    {
      "activations/layer0_attention_weight_max": 16.043622970581055,
      "activations/layer0_attention_weight_min": -12.170208930969238,
      "activations/layer10_attention_weight_max": 34.1494140625,
      "activations/layer10_attention_weight_min": -35.55840301513672,
      "activations/layer11_attention_weight_max": 32.64331817626953,
      "activations/layer11_attention_weight_min": -33.953575134277344,
      "activations/layer12_attention_weight_max": 28.815881729125977,
      "activations/layer12_attention_weight_min": -25.891244888305664,
      "activations/layer13_attention_weight_max": 43.76802062988281,
      "activations/layer13_attention_weight_min": -34.85488510131836,
      "activations/layer14_attention_weight_max": 44.14935302734375,
      "activations/layer14_attention_weight_min": -36.7380256652832,
      "activations/layer15_attention_weight_max": 38.05119323730469,
      "activations/layer15_attention_weight_min": -33.867252349853516,
      "activations/layer16_attention_weight_max": 30.399364471435547,
      "activations/layer16_attention_weight_min": -28.034032821655273,
      "activations/layer17_attention_weight_max": 33.49665832519531,
      "activations/layer17_attention_weight_min": -26.368911743164062,
      "activations/layer18_attention_weight_max": 32.88377380371094,
      "activations/layer18_attention_weight_min": -24.392536163330078,
      "activations/layer19_attention_weight_max": 39.1833381652832,
      "activations/layer19_attention_weight_min": -31.0642032623291,
      "activations/layer1_attention_weight_max": 17.36033058166504,
      "activations/layer1_attention_weight_min": -15.7564697265625,
      "activations/layer20_attention_weight_max": 30.668100357055664,
      "activations/layer20_attention_weight_min": -24.999155044555664,
      "activations/layer21_attention_weight_max": 29.991544723510742,
      "activations/layer21_attention_weight_min": -24.34807777404785,
      "activations/layer22_attention_weight_max": 43.46170425415039,
      "activations/layer22_attention_weight_min": -31.549463272094727,
      "activations/layer23_attention_weight_max": 39.88964080810547,
      "activations/layer23_attention_weight_min": -26.967411041259766,
      "activations/layer2_attention_weight_max": 32.96904754638672,
      "activations/layer2_attention_weight_min": -33.431251525878906,
      "activations/layer3_attention_weight_max": 94.20842742919922,
      "activations/layer3_attention_weight_min": -100.88175964355469,
      "activations/layer4_attention_weight_max": 111.12772369384766,
      "activations/layer4_attention_weight_min": -111.77169036865234,
      "activations/layer5_attention_weight_max": 52.38188171386719,
      "activations/layer5_attention_weight_min": -62.700111389160156,
      "activations/layer6_attention_weight_max": 45.52253341674805,
      "activations/layer6_attention_weight_min": -48.94413375854492,
      "activations/layer7_attention_weight_max": 92.90543365478516,
      "activations/layer7_attention_weight_min": -99.09671020507812,
      "activations/layer8_attention_weight_max": 40.82081604003906,
      "activations/layer8_attention_weight_min": -44.06236267089844,
      "activations/layer9_attention_weight_max": 34.489376068115234,
      "activations/layer9_attention_weight_min": -34.57554244995117,
      "epoch": 14.14,
      "learning_rate": 5.9393939393939385e-05,
      "loss": 2.756,
      "step": 243400
    },
    {
      "activations/layer0_attention_weight_max": 16.12513542175293,
      "activations/layer0_attention_weight_min": -11.814736366271973,
      "activations/layer10_attention_weight_max": 34.31268310546875,
      "activations/layer10_attention_weight_min": -32.883216857910156,
      "activations/layer11_attention_weight_max": 35.959163665771484,
      "activations/layer11_attention_weight_min": -34.58984375,
      "activations/layer12_attention_weight_max": 31.665342330932617,
      "activations/layer12_attention_weight_min": -25.979751586914062,
      "activations/layer13_attention_weight_max": 37.199424743652344,
      "activations/layer13_attention_weight_min": -34.31983947753906,
      "activations/layer14_attention_weight_max": 40.50891876220703,
      "activations/layer14_attention_weight_min": -35.03602981567383,
      "activations/layer15_attention_weight_max": 35.3443717956543,
      "activations/layer15_attention_weight_min": -34.245025634765625,
      "activations/layer16_attention_weight_max": 31.180179595947266,
      "activations/layer16_attention_weight_min": -28.42751121520996,
      "activations/layer17_attention_weight_max": 31.176015853881836,
      "activations/layer17_attention_weight_min": -25.224700927734375,
      "activations/layer18_attention_weight_max": 30.612239837646484,
      "activations/layer18_attention_weight_min": -23.723173141479492,
      "activations/layer19_attention_weight_max": 33.46288299560547,
      "activations/layer19_attention_weight_min": -30.09124183654785,
      "activations/layer1_attention_weight_max": 16.552200317382812,
      "activations/layer1_attention_weight_min": -14.948755264282227,
      "activations/layer20_attention_weight_max": 29.582378387451172,
      "activations/layer20_attention_weight_min": -25.626874923706055,
      "activations/layer21_attention_weight_max": 31.022964477539062,
      "activations/layer21_attention_weight_min": -24.021896362304688,
      "activations/layer22_attention_weight_max": 44.89373016357422,
      "activations/layer22_attention_weight_min": -30.05670166015625,
      "activations/layer23_attention_weight_max": 32.760467529296875,
      "activations/layer23_attention_weight_min": -25.746061325073242,
      "activations/layer2_attention_weight_max": 32.98222732543945,
      "activations/layer2_attention_weight_min": -33.254302978515625,
      "activations/layer3_attention_weight_max": 96.05712127685547,
      "activations/layer3_attention_weight_min": -96.44393920898438,
      "activations/layer4_attention_weight_max": 109.53775787353516,
      "activations/layer4_attention_weight_min": -109.7630615234375,
      "activations/layer5_attention_weight_max": 51.76299285888672,
      "activations/layer5_attention_weight_min": -61.94428634643555,
      "activations/layer6_attention_weight_max": 44.244056701660156,
      "activations/layer6_attention_weight_min": -46.702213287353516,
      "activations/layer7_attention_weight_max": 90.5020523071289,
      "activations/layer7_attention_weight_min": -94.84512329101562,
      "activations/layer8_attention_weight_max": 41.41162109375,
      "activations/layer8_attention_weight_min": -41.28169631958008,
      "activations/layer9_attention_weight_max": 35.17624282836914,
      "activations/layer9_attention_weight_min": -34.68328857421875,
      "epoch": 14.15,
      "learning_rate": 5.937499999999999e-05,
      "loss": 2.7626,
      "step": 243450
    },
    {
      "activations/layer0_attention_weight_max": 16.32002067565918,
      "activations/layer0_attention_weight_min": -12.391802787780762,
      "activations/layer10_attention_weight_max": 36.379695892333984,
      "activations/layer10_attention_weight_min": -33.074092864990234,
      "activations/layer11_attention_weight_max": 35.565101623535156,
      "activations/layer11_attention_weight_min": -33.44949722290039,
      "activations/layer12_attention_weight_max": 30.181068420410156,
      "activations/layer12_attention_weight_min": -24.32935905456543,
      "activations/layer13_attention_weight_max": 46.621971130371094,
      "activations/layer13_attention_weight_min": -33.45574951171875,
      "activations/layer14_attention_weight_max": 43.43537139892578,
      "activations/layer14_attention_weight_min": -36.69817352294922,
      "activations/layer15_attention_weight_max": 38.891334533691406,
      "activations/layer15_attention_weight_min": -31.74234390258789,
      "activations/layer16_attention_weight_max": 29.779436111450195,
      "activations/layer16_attention_weight_min": -27.018564224243164,
      "activations/layer17_attention_weight_max": 30.52149200439453,
      "activations/layer17_attention_weight_min": -25.680816650390625,
      "activations/layer18_attention_weight_max": 35.1213493347168,
      "activations/layer18_attention_weight_min": -23.37587547302246,
      "activations/layer19_attention_weight_max": 33.38119125366211,
      "activations/layer19_attention_weight_min": -30.291549682617188,
      "activations/layer1_attention_weight_max": 17.199378967285156,
      "activations/layer1_attention_weight_min": -14.118749618530273,
      "activations/layer20_attention_weight_max": 28.33684539794922,
      "activations/layer20_attention_weight_min": -23.144559860229492,
      "activations/layer21_attention_weight_max": 26.969940185546875,
      "activations/layer21_attention_weight_min": -25.704923629760742,
      "activations/layer22_attention_weight_max": 43.178802490234375,
      "activations/layer22_attention_weight_min": -31.103994369506836,
      "activations/layer23_attention_weight_max": 31.315895080566406,
      "activations/layer23_attention_weight_min": -23.41609001159668,
      "activations/layer2_attention_weight_max": 34.31520080566406,
      "activations/layer2_attention_weight_min": -34.89374542236328,
      "activations/layer3_attention_weight_max": 98.58597564697266,
      "activations/layer3_attention_weight_min": -101.86052703857422,
      "activations/layer4_attention_weight_max": 112.59954071044922,
      "activations/layer4_attention_weight_min": -112.7354736328125,
      "activations/layer5_attention_weight_max": 53.80450439453125,
      "activations/layer5_attention_weight_min": -63.079559326171875,
      "activations/layer6_attention_weight_max": 45.3981819152832,
      "activations/layer6_attention_weight_min": -47.98805618286133,
      "activations/layer7_attention_weight_max": 94.4743423461914,
      "activations/layer7_attention_weight_min": -93.24607849121094,
      "activations/layer8_attention_weight_max": 40.47304153442383,
      "activations/layer8_attention_weight_min": -41.87196731567383,
      "activations/layer9_attention_weight_max": 32.662567138671875,
      "activations/layer9_attention_weight_min": -33.1995849609375,
      "epoch": 14.15,
      "learning_rate": 5.93560606060606e-05,
      "loss": 2.7649,
      "step": 243500
    },
    {
      "activations/layer0_attention_weight_max": 16.2259521484375,
      "activations/layer0_attention_weight_min": -12.291537284851074,
      "activations/layer10_attention_weight_max": 34.275909423828125,
      "activations/layer10_attention_weight_min": -34.117671966552734,
      "activations/layer11_attention_weight_max": 33.57196044921875,
      "activations/layer11_attention_weight_min": -32.86131286621094,
      "activations/layer12_attention_weight_max": 28.702119827270508,
      "activations/layer12_attention_weight_min": -28.141294479370117,
      "activations/layer13_attention_weight_max": 42.90599060058594,
      "activations/layer13_attention_weight_min": -40.08468246459961,
      "activations/layer14_attention_weight_max": 45.48753356933594,
      "activations/layer14_attention_weight_min": -42.287269592285156,
      "activations/layer15_attention_weight_max": 41.26359939575195,
      "activations/layer15_attention_weight_min": -37.66787338256836,
      "activations/layer16_attention_weight_max": 32.71092224121094,
      "activations/layer16_attention_weight_min": -28.094152450561523,
      "activations/layer17_attention_weight_max": 30.74666976928711,
      "activations/layer17_attention_weight_min": -27.86729621887207,
      "activations/layer18_attention_weight_max": 33.140785217285156,
      "activations/layer18_attention_weight_min": -24.665830612182617,
      "activations/layer19_attention_weight_max": 41.24656677246094,
      "activations/layer19_attention_weight_min": -33.163875579833984,
      "activations/layer1_attention_weight_max": 16.811071395874023,
      "activations/layer1_attention_weight_min": -13.966130256652832,
      "activations/layer20_attention_weight_max": 34.43422317504883,
      "activations/layer20_attention_weight_min": -27.08162498474121,
      "activations/layer21_attention_weight_max": 32.275115966796875,
      "activations/layer21_attention_weight_min": -26.365177154541016,
      "activations/layer22_attention_weight_max": 47.42556381225586,
      "activations/layer22_attention_weight_min": -33.13610076904297,
      "activations/layer23_attention_weight_max": 34.044960021972656,
      "activations/layer23_attention_weight_min": -26.38947868347168,
      "activations/layer2_attention_weight_max": 35.02064895629883,
      "activations/layer2_attention_weight_min": -31.926984786987305,
      "activations/layer3_attention_weight_max": 102.46990966796875,
      "activations/layer3_attention_weight_min": -98.18146514892578,
      "activations/layer4_attention_weight_max": 109.49781799316406,
      "activations/layer4_attention_weight_min": -114.48735809326172,
      "activations/layer5_attention_weight_max": 50.59053039550781,
      "activations/layer5_attention_weight_min": -63.136024475097656,
      "activations/layer6_attention_weight_max": 43.62822341918945,
      "activations/layer6_attention_weight_min": -46.22864532470703,
      "activations/layer7_attention_weight_max": 96.23129272460938,
      "activations/layer7_attention_weight_min": -97.64261627197266,
      "activations/layer8_attention_weight_max": 41.0483283996582,
      "activations/layer8_attention_weight_min": -44.34389114379883,
      "activations/layer9_attention_weight_max": 32.83804702758789,
      "activations/layer9_attention_weight_min": -33.955318450927734,
      "epoch": 14.15,
      "learning_rate": 5.933712121212121e-05,
      "loss": 2.7531,
      "step": 243550
    },
    {
      "activations/layer0_attention_weight_max": 15.63680648803711,
      "activations/layer0_attention_weight_min": -12.725666999816895,
      "activations/layer10_attention_weight_max": 34.001007080078125,
      "activations/layer10_attention_weight_min": -32.72905349731445,
      "activations/layer11_attention_weight_max": 33.107421875,
      "activations/layer11_attention_weight_min": -34.745521545410156,
      "activations/layer12_attention_weight_max": 28.000032424926758,
      "activations/layer12_attention_weight_min": -27.118770599365234,
      "activations/layer13_attention_weight_max": 42.82040023803711,
      "activations/layer13_attention_weight_min": -34.58376693725586,
      "activations/layer14_attention_weight_max": 45.918975830078125,
      "activations/layer14_attention_weight_min": -38.43905258178711,
      "activations/layer15_attention_weight_max": 40.14973449707031,
      "activations/layer15_attention_weight_min": -34.64168167114258,
      "activations/layer16_attention_weight_max": 31.362972259521484,
      "activations/layer16_attention_weight_min": -28.3709716796875,
      "activations/layer17_attention_weight_max": 32.83990478515625,
      "activations/layer17_attention_weight_min": -25.777402877807617,
      "activations/layer18_attention_weight_max": 35.729496002197266,
      "activations/layer18_attention_weight_min": -24.262821197509766,
      "activations/layer19_attention_weight_max": 36.48480987548828,
      "activations/layer19_attention_weight_min": -32.338253021240234,
      "activations/layer1_attention_weight_max": 16.22605323791504,
      "activations/layer1_attention_weight_min": -15.884407997131348,
      "activations/layer20_attention_weight_max": 33.00286865234375,
      "activations/layer20_attention_weight_min": -25.956457138061523,
      "activations/layer21_attention_weight_max": 32.54383087158203,
      "activations/layer21_attention_weight_min": -25.62261390686035,
      "activations/layer22_attention_weight_max": 45.8603630065918,
      "activations/layer22_attention_weight_min": -29.058074951171875,
      "activations/layer23_attention_weight_max": 35.07852554321289,
      "activations/layer23_attention_weight_min": -25.981334686279297,
      "activations/layer2_attention_weight_max": 33.504295349121094,
      "activations/layer2_attention_weight_min": -34.42826843261719,
      "activations/layer3_attention_weight_max": 100.00093078613281,
      "activations/layer3_attention_weight_min": -100.50228881835938,
      "activations/layer4_attention_weight_max": 113.10090637207031,
      "activations/layer4_attention_weight_min": -114.0426254272461,
      "activations/layer5_attention_weight_max": 52.89673614501953,
      "activations/layer5_attention_weight_min": -59.88196563720703,
      "activations/layer6_attention_weight_max": 45.84593200683594,
      "activations/layer6_attention_weight_min": -51.02315902709961,
      "activations/layer7_attention_weight_max": 99.04146575927734,
      "activations/layer7_attention_weight_min": -93.98573303222656,
      "activations/layer8_attention_weight_max": 41.950321197509766,
      "activations/layer8_attention_weight_min": -41.73927307128906,
      "activations/layer9_attention_weight_max": 33.743621826171875,
      "activations/layer9_attention_weight_min": -33.97650909423828,
      "epoch": 14.15,
      "learning_rate": 5.931818181818181e-05,
      "loss": 2.7496,
      "step": 243600
    },
    {
      "activations/layer0_attention_weight_max": 15.95135498046875,
      "activations/layer0_attention_weight_min": -12.716590881347656,
      "activations/layer10_attention_weight_max": 34.27045440673828,
      "activations/layer10_attention_weight_min": -34.297454833984375,
      "activations/layer11_attention_weight_max": 37.11744689941406,
      "activations/layer11_attention_weight_min": -34.84135437011719,
      "activations/layer12_attention_weight_max": 24.91259002685547,
      "activations/layer12_attention_weight_min": -24.710662841796875,
      "activations/layer13_attention_weight_max": 39.642818450927734,
      "activations/layer13_attention_weight_min": -32.51240158081055,
      "activations/layer14_attention_weight_max": 45.48455047607422,
      "activations/layer14_attention_weight_min": -35.4444694519043,
      "activations/layer15_attention_weight_max": 37.06521224975586,
      "activations/layer15_attention_weight_min": -32.3145637512207,
      "activations/layer16_attention_weight_max": 34.70348358154297,
      "activations/layer16_attention_weight_min": -26.013132095336914,
      "activations/layer17_attention_weight_max": 30.2315673828125,
      "activations/layer17_attention_weight_min": -24.730405807495117,
      "activations/layer18_attention_weight_max": 33.148284912109375,
      "activations/layer18_attention_weight_min": -23.899059295654297,
      "activations/layer19_attention_weight_max": 37.18646240234375,
      "activations/layer19_attention_weight_min": -31.586868286132812,
      "activations/layer1_attention_weight_max": 15.926913261413574,
      "activations/layer1_attention_weight_min": -14.544290542602539,
      "activations/layer20_attention_weight_max": 31.295207977294922,
      "activations/layer20_attention_weight_min": -25.403762817382812,
      "activations/layer21_attention_weight_max": 33.1258544921875,
      "activations/layer21_attention_weight_min": -26.07170295715332,
      "activations/layer22_attention_weight_max": 43.84864044189453,
      "activations/layer22_attention_weight_min": -30.303831100463867,
      "activations/layer23_attention_weight_max": 35.60322189331055,
      "activations/layer23_attention_weight_min": -24.608503341674805,
      "activations/layer2_attention_weight_max": 33.12825012207031,
      "activations/layer2_attention_weight_min": -33.71289825439453,
      "activations/layer3_attention_weight_max": 97.11134338378906,
      "activations/layer3_attention_weight_min": -98.1484146118164,
      "activations/layer4_attention_weight_max": 107.90936279296875,
      "activations/layer4_attention_weight_min": -106.37889862060547,
      "activations/layer5_attention_weight_max": 51.31736755371094,
      "activations/layer5_attention_weight_min": -59.126495361328125,
      "activations/layer6_attention_weight_max": 44.96446990966797,
      "activations/layer6_attention_weight_min": -46.712406158447266,
      "activations/layer7_attention_weight_max": 94.40907287597656,
      "activations/layer7_attention_weight_min": -89.46035766601562,
      "activations/layer8_attention_weight_max": 40.7791862487793,
      "activations/layer8_attention_weight_min": -44.8726692199707,
      "activations/layer9_attention_weight_max": 34.018009185791016,
      "activations/layer9_attention_weight_min": -34.23002243041992,
      "epoch": 14.16,
      "learning_rate": 5.929924242424241e-05,
      "loss": 2.7685,
      "step": 243650
    },
    {
      "activations/layer0_attention_weight_max": 15.838336944580078,
      "activations/layer0_attention_weight_min": -13.699644088745117,
      "activations/layer10_attention_weight_max": 30.353652954101562,
      "activations/layer10_attention_weight_min": -31.823421478271484,
      "activations/layer11_attention_weight_max": 29.601131439208984,
      "activations/layer11_attention_weight_min": -31.387805938720703,
      "activations/layer12_attention_weight_max": 24.078125,
      "activations/layer12_attention_weight_min": -25.687528610229492,
      "activations/layer13_attention_weight_max": 36.43358612060547,
      "activations/layer13_attention_weight_min": -32.410457611083984,
      "activations/layer14_attention_weight_max": 40.81953430175781,
      "activations/layer14_attention_weight_min": -37.668087005615234,
      "activations/layer15_attention_weight_max": 34.59227752685547,
      "activations/layer15_attention_weight_min": -31.987218856811523,
      "activations/layer16_attention_weight_max": 31.05646514892578,
      "activations/layer16_attention_weight_min": -30.958410263061523,
      "activations/layer17_attention_weight_max": 29.039485931396484,
      "activations/layer17_attention_weight_min": -25.85606575012207,
      "activations/layer18_attention_weight_max": 32.07002258300781,
      "activations/layer18_attention_weight_min": -24.797622680664062,
      "activations/layer19_attention_weight_max": 34.739654541015625,
      "activations/layer19_attention_weight_min": -30.907690048217773,
      "activations/layer1_attention_weight_max": 15.95703125,
      "activations/layer1_attention_weight_min": -14.875894546508789,
      "activations/layer20_attention_weight_max": 29.775859832763672,
      "activations/layer20_attention_weight_min": -27.55345344543457,
      "activations/layer21_attention_weight_max": 30.10776710510254,
      "activations/layer21_attention_weight_min": -23.909629821777344,
      "activations/layer22_attention_weight_max": 42.50812530517578,
      "activations/layer22_attention_weight_min": -32.10655975341797,
      "activations/layer23_attention_weight_max": 36.6440544128418,
      "activations/layer23_attention_weight_min": -29.100181579589844,
      "activations/layer2_attention_weight_max": 33.19183349609375,
      "activations/layer2_attention_weight_min": -33.5738525390625,
      "activations/layer3_attention_weight_max": 96.0311050415039,
      "activations/layer3_attention_weight_min": -96.33349609375,
      "activations/layer4_attention_weight_max": 105.7637939453125,
      "activations/layer4_attention_weight_min": -107.19792938232422,
      "activations/layer5_attention_weight_max": 52.531211853027344,
      "activations/layer5_attention_weight_min": -60.25718307495117,
      "activations/layer6_attention_weight_max": 45.85661697387695,
      "activations/layer6_attention_weight_min": -45.47590637207031,
      "activations/layer7_attention_weight_max": 91.37559509277344,
      "activations/layer7_attention_weight_min": -88.45539093017578,
      "activations/layer8_attention_weight_max": 39.729251861572266,
      "activations/layer8_attention_weight_min": -39.4011116027832,
      "activations/layer9_attention_weight_max": 31.000324249267578,
      "activations/layer9_attention_weight_min": -31.486835479736328,
      "epoch": 14.16,
      "learning_rate": 5.928030303030303e-05,
      "loss": 2.7692,
      "step": 243700
    },
    {
      "activations/layer0_attention_weight_max": 16.279827117919922,
      "activations/layer0_attention_weight_min": -12.343881607055664,
      "activations/layer10_attention_weight_max": 30.92804527282715,
      "activations/layer10_attention_weight_min": -33.096290588378906,
      "activations/layer11_attention_weight_max": 33.007259368896484,
      "activations/layer11_attention_weight_min": -32.625099182128906,
      "activations/layer12_attention_weight_max": 24.477649688720703,
      "activations/layer12_attention_weight_min": -24.436681747436523,
      "activations/layer13_attention_weight_max": 37.810176849365234,
      "activations/layer13_attention_weight_min": -34.59470748901367,
      "activations/layer14_attention_weight_max": 39.38426208496094,
      "activations/layer14_attention_weight_min": -35.17414855957031,
      "activations/layer15_attention_weight_max": 36.191524505615234,
      "activations/layer15_attention_weight_min": -32.56104278564453,
      "activations/layer16_attention_weight_max": 30.999780654907227,
      "activations/layer16_attention_weight_min": -27.220748901367188,
      "activations/layer17_attention_weight_max": 29.70526885986328,
      "activations/layer17_attention_weight_min": -23.6027774810791,
      "activations/layer18_attention_weight_max": 31.717815399169922,
      "activations/layer18_attention_weight_min": -24.31599998474121,
      "activations/layer19_attention_weight_max": 34.43868637084961,
      "activations/layer19_attention_weight_min": -30.16908836364746,
      "activations/layer1_attention_weight_max": 17.215953826904297,
      "activations/layer1_attention_weight_min": -14.772481918334961,
      "activations/layer20_attention_weight_max": 26.533414840698242,
      "activations/layer20_attention_weight_min": -26.205833435058594,
      "activations/layer21_attention_weight_max": 28.214279174804688,
      "activations/layer21_attention_weight_min": -24.4625301361084,
      "activations/layer22_attention_weight_max": 37.94878387451172,
      "activations/layer22_attention_weight_min": -33.18179702758789,
      "activations/layer23_attention_weight_max": 32.090484619140625,
      "activations/layer23_attention_weight_min": -26.61501693725586,
      "activations/layer2_attention_weight_max": 31.518157958984375,
      "activations/layer2_attention_weight_min": -32.389495849609375,
      "activations/layer3_attention_weight_max": 93.16155242919922,
      "activations/layer3_attention_weight_min": -95.73609924316406,
      "activations/layer4_attention_weight_max": 107.8049545288086,
      "activations/layer4_attention_weight_min": -103.67317962646484,
      "activations/layer5_attention_weight_max": 51.1483154296875,
      "activations/layer5_attention_weight_min": -60.44556427001953,
      "activations/layer6_attention_weight_max": 41.279048919677734,
      "activations/layer6_attention_weight_min": -45.57356262207031,
      "activations/layer7_attention_weight_max": 91.76543426513672,
      "activations/layer7_attention_weight_min": -89.17757415771484,
      "activations/layer8_attention_weight_max": 38.67974853515625,
      "activations/layer8_attention_weight_min": -39.982948303222656,
      "activations/layer9_attention_weight_max": 32.63656997680664,
      "activations/layer9_attention_weight_min": -34.179046630859375,
      "epoch": 14.16,
      "learning_rate": 5.926136363636363e-05,
      "loss": 2.7584,
      "step": 243750
    },
    {
      "activations/layer0_attention_weight_max": 16.31067657470703,
      "activations/layer0_attention_weight_min": -12.587435722351074,
      "activations/layer10_attention_weight_max": 31.014249801635742,
      "activations/layer10_attention_weight_min": -32.38446044921875,
      "activations/layer11_attention_weight_max": 30.70171356201172,
      "activations/layer11_attention_weight_min": -31.954120635986328,
      "activations/layer12_attention_weight_max": 25.928604125976562,
      "activations/layer12_attention_weight_min": -24.87122917175293,
      "activations/layer13_attention_weight_max": 37.99935531616211,
      "activations/layer13_attention_weight_min": -32.71998977661133,
      "activations/layer14_attention_weight_max": 44.46048355102539,
      "activations/layer14_attention_weight_min": -35.34838104248047,
      "activations/layer15_attention_weight_max": 37.70576095581055,
      "activations/layer15_attention_weight_min": -33.522953033447266,
      "activations/layer16_attention_weight_max": 29.47849464416504,
      "activations/layer16_attention_weight_min": -29.014968872070312,
      "activations/layer17_attention_weight_max": 31.31894874572754,
      "activations/layer17_attention_weight_min": -25.505388259887695,
      "activations/layer18_attention_weight_max": 32.20823669433594,
      "activations/layer18_attention_weight_min": -25.806657791137695,
      "activations/layer19_attention_weight_max": 34.991397857666016,
      "activations/layer19_attention_weight_min": -32.26243209838867,
      "activations/layer1_attention_weight_max": 16.391220092773438,
      "activations/layer1_attention_weight_min": -15.59455394744873,
      "activations/layer20_attention_weight_max": 28.731639862060547,
      "activations/layer20_attention_weight_min": -25.77716064453125,
      "activations/layer21_attention_weight_max": 28.27736473083496,
      "activations/layer21_attention_weight_min": -25.098373413085938,
      "activations/layer22_attention_weight_max": 42.40966796875,
      "activations/layer22_attention_weight_min": -31.46860694885254,
      "activations/layer23_attention_weight_max": 36.89897918701172,
      "activations/layer23_attention_weight_min": -26.824996948242188,
      "activations/layer2_attention_weight_max": 32.62171936035156,
      "activations/layer2_attention_weight_min": -33.85844421386719,
      "activations/layer3_attention_weight_max": 94.00777435302734,
      "activations/layer3_attention_weight_min": -100.35567474365234,
      "activations/layer4_attention_weight_max": 112.7007064819336,
      "activations/layer4_attention_weight_min": -110.94864654541016,
      "activations/layer5_attention_weight_max": 55.94157028198242,
      "activations/layer5_attention_weight_min": -61.85569763183594,
      "activations/layer6_attention_weight_max": 43.02077102661133,
      "activations/layer6_attention_weight_min": -46.020755767822266,
      "activations/layer7_attention_weight_max": 88.04743957519531,
      "activations/layer7_attention_weight_min": -91.91877746582031,
      "activations/layer8_attention_weight_max": 39.01372146606445,
      "activations/layer8_attention_weight_min": -40.1390380859375,
      "activations/layer9_attention_weight_max": 30.416549682617188,
      "activations/layer9_attention_weight_min": -33.28303146362305,
      "epoch": 14.17,
      "learning_rate": 5.924242424242424e-05,
      "loss": 2.7523,
      "step": 243800
    },
    {
      "activations/layer0_attention_weight_max": 16.61623191833496,
      "activations/layer0_attention_weight_min": -12.583345413208008,
      "activations/layer10_attention_weight_max": 30.707101821899414,
      "activations/layer10_attention_weight_min": -32.862266540527344,
      "activations/layer11_attention_weight_max": 30.88532066345215,
      "activations/layer11_attention_weight_min": -31.3502197265625,
      "activations/layer12_attention_weight_max": 24.632408142089844,
      "activations/layer12_attention_weight_min": -25.846969604492188,
      "activations/layer13_attention_weight_max": 37.524192810058594,
      "activations/layer13_attention_weight_min": -35.16764831542969,
      "activations/layer14_attention_weight_max": 45.49595642089844,
      "activations/layer14_attention_weight_min": -35.33317184448242,
      "activations/layer15_attention_weight_max": 38.90208053588867,
      "activations/layer15_attention_weight_min": -33.20167541503906,
      "activations/layer16_attention_weight_max": 32.956539154052734,
      "activations/layer16_attention_weight_min": -28.8508358001709,
      "activations/layer17_attention_weight_max": 34.5220832824707,
      "activations/layer17_attention_weight_min": -27.467529296875,
      "activations/layer18_attention_weight_max": 32.257572174072266,
      "activations/layer18_attention_weight_min": -23.805334091186523,
      "activations/layer19_attention_weight_max": 38.17329788208008,
      "activations/layer19_attention_weight_min": -30.62198257446289,
      "activations/layer1_attention_weight_max": 16.76292610168457,
      "activations/layer1_attention_weight_min": -13.976724624633789,
      "activations/layer20_attention_weight_max": 32.19076919555664,
      "activations/layer20_attention_weight_min": -24.194421768188477,
      "activations/layer21_attention_weight_max": 33.42738723754883,
      "activations/layer21_attention_weight_min": -23.074939727783203,
      "activations/layer22_attention_weight_max": 51.426021575927734,
      "activations/layer22_attention_weight_min": -29.348291397094727,
      "activations/layer23_attention_weight_max": 35.490028381347656,
      "activations/layer23_attention_weight_min": -25.49277114868164,
      "activations/layer2_attention_weight_max": 33.99800109863281,
      "activations/layer2_attention_weight_min": -32.14124298095703,
      "activations/layer3_attention_weight_max": 95.94634246826172,
      "activations/layer3_attention_weight_min": -97.31920623779297,
      "activations/layer4_attention_weight_max": 107.13309478759766,
      "activations/layer4_attention_weight_min": -109.05919647216797,
      "activations/layer5_attention_weight_max": 49.841041564941406,
      "activations/layer5_attention_weight_min": -64.02823638916016,
      "activations/layer6_attention_weight_max": 40.90412902832031,
      "activations/layer6_attention_weight_min": -43.562599182128906,
      "activations/layer7_attention_weight_max": 95.93115997314453,
      "activations/layer7_attention_weight_min": -95.2939224243164,
      "activations/layer8_attention_weight_max": 38.41312789916992,
      "activations/layer8_attention_weight_min": -43.432167053222656,
      "activations/layer9_attention_weight_max": 31.614171981811523,
      "activations/layer9_attention_weight_min": -32.701393127441406,
      "epoch": 14.17,
      "learning_rate": 5.922348484848484e-05,
      "loss": 2.7611,
      "step": 243850
    },
    {
      "activations/layer0_attention_weight_max": 16.076196670532227,
      "activations/layer0_attention_weight_min": -12.860535621643066,
      "activations/layer10_attention_weight_max": 31.601083755493164,
      "activations/layer10_attention_weight_min": -32.02941131591797,
      "activations/layer11_attention_weight_max": 32.86216735839844,
      "activations/layer11_attention_weight_min": -31.913997650146484,
      "activations/layer12_attention_weight_max": 26.74620246887207,
      "activations/layer12_attention_weight_min": -25.494400024414062,
      "activations/layer13_attention_weight_max": 41.34025955200195,
      "activations/layer13_attention_weight_min": -35.01287841796875,
      "activations/layer14_attention_weight_max": 43.380577087402344,
      "activations/layer14_attention_weight_min": -35.66539001464844,
      "activations/layer15_attention_weight_max": 37.529170989990234,
      "activations/layer15_attention_weight_min": -32.762569427490234,
      "activations/layer16_attention_weight_max": 31.723339080810547,
      "activations/layer16_attention_weight_min": -27.16672134399414,
      "activations/layer17_attention_weight_max": 31.532922744750977,
      "activations/layer17_attention_weight_min": -25.724367141723633,
      "activations/layer18_attention_weight_max": 31.64313316345215,
      "activations/layer18_attention_weight_min": -25.213979721069336,
      "activations/layer19_attention_weight_max": 43.098636627197266,
      "activations/layer19_attention_weight_min": -31.09676170349121,
      "activations/layer1_attention_weight_max": 15.762316703796387,
      "activations/layer1_attention_weight_min": -15.506994247436523,
      "activations/layer20_attention_weight_max": 35.115562438964844,
      "activations/layer20_attention_weight_min": -24.724746704101562,
      "activations/layer21_attention_weight_max": 34.94361877441406,
      "activations/layer21_attention_weight_min": -24.183311462402344,
      "activations/layer22_attention_weight_max": 46.007198333740234,
      "activations/layer22_attention_weight_min": -30.552339553833008,
      "activations/layer23_attention_weight_max": 35.36577224731445,
      "activations/layer23_attention_weight_min": -26.513553619384766,
      "activations/layer2_attention_weight_max": 33.045654296875,
      "activations/layer2_attention_weight_min": -32.957862854003906,
      "activations/layer3_attention_weight_max": 98.17733001708984,
      "activations/layer3_attention_weight_min": -101.24250030517578,
      "activations/layer4_attention_weight_max": 110.949462890625,
      "activations/layer4_attention_weight_min": -116.17010498046875,
      "activations/layer5_attention_weight_max": 54.921688079833984,
      "activations/layer5_attention_weight_min": -61.938499450683594,
      "activations/layer6_attention_weight_max": 44.28001022338867,
      "activations/layer6_attention_weight_min": -47.410953521728516,
      "activations/layer7_attention_weight_max": 90.24520111083984,
      "activations/layer7_attention_weight_min": -94.1094741821289,
      "activations/layer8_attention_weight_max": 37.89934158325195,
      "activations/layer8_attention_weight_min": -40.821754455566406,
      "activations/layer9_attention_weight_max": 31.14821434020996,
      "activations/layer9_attention_weight_min": -34.125396728515625,
      "epoch": 14.17,
      "learning_rate": 5.9204545454545454e-05,
      "loss": 2.7673,
      "step": 243900
    },
    {
      "activations/layer0_attention_weight_max": 15.552056312561035,
      "activations/layer0_attention_weight_min": -13.022078514099121,
      "activations/layer10_attention_weight_max": 33.329776763916016,
      "activations/layer10_attention_weight_min": -33.392303466796875,
      "activations/layer11_attention_weight_max": 33.608428955078125,
      "activations/layer11_attention_weight_min": -31.417312622070312,
      "activations/layer12_attention_weight_max": 26.348661422729492,
      "activations/layer12_attention_weight_min": -23.83784294128418,
      "activations/layer13_attention_weight_max": 41.06309127807617,
      "activations/layer13_attention_weight_min": -34.45590591430664,
      "activations/layer14_attention_weight_max": 42.20176315307617,
      "activations/layer14_attention_weight_min": -37.67131042480469,
      "activations/layer15_attention_weight_max": 40.29921340942383,
      "activations/layer15_attention_weight_min": -33.973602294921875,
      "activations/layer16_attention_weight_max": 31.349132537841797,
      "activations/layer16_attention_weight_min": -25.56620216369629,
      "activations/layer17_attention_weight_max": 31.241994857788086,
      "activations/layer17_attention_weight_min": -24.196712493896484,
      "activations/layer18_attention_weight_max": 36.18017578125,
      "activations/layer18_attention_weight_min": -24.46063804626465,
      "activations/layer19_attention_weight_max": 36.30667495727539,
      "activations/layer19_attention_weight_min": -30.560483932495117,
      "activations/layer1_attention_weight_max": 19.173763275146484,
      "activations/layer1_attention_weight_min": -15.44793701171875,
      "activations/layer20_attention_weight_max": 29.80437469482422,
      "activations/layer20_attention_weight_min": -23.364194869995117,
      "activations/layer21_attention_weight_max": 28.52531623840332,
      "activations/layer21_attention_weight_min": -23.954965591430664,
      "activations/layer22_attention_weight_max": 48.11035919189453,
      "activations/layer22_attention_weight_min": -30.76218605041504,
      "activations/layer23_attention_weight_max": 33.92804718017578,
      "activations/layer23_attention_weight_min": -24.538658142089844,
      "activations/layer2_attention_weight_max": 34.24692153930664,
      "activations/layer2_attention_weight_min": -33.788604736328125,
      "activations/layer3_attention_weight_max": 99.96715545654297,
      "activations/layer3_attention_weight_min": -97.95198059082031,
      "activations/layer4_attention_weight_max": 115.96065521240234,
      "activations/layer4_attention_weight_min": -112.09331512451172,
      "activations/layer5_attention_weight_max": 53.879844665527344,
      "activations/layer5_attention_weight_min": -63.01679992675781,
      "activations/layer6_attention_weight_max": 42.95914840698242,
      "activations/layer6_attention_weight_min": -45.13838577270508,
      "activations/layer7_attention_weight_max": 88.9962387084961,
      "activations/layer7_attention_weight_min": -88.2501449584961,
      "activations/layer8_attention_weight_max": 39.74610900878906,
      "activations/layer8_attention_weight_min": -42.58079528808594,
      "activations/layer9_attention_weight_max": 33.04081726074219,
      "activations/layer9_attention_weight_min": -31.851499557495117,
      "epoch": 14.17,
      "learning_rate": 5.9185606060606055e-05,
      "loss": 2.7586,
      "step": 243950
    },
    {
      "activations/layer0_attention_weight_max": 16.47751235961914,
      "activations/layer0_attention_weight_min": -12.323225975036621,
      "activations/layer10_attention_weight_max": 30.737314224243164,
      "activations/layer10_attention_weight_min": -31.19798469543457,
      "activations/layer11_attention_weight_max": 33.358856201171875,
      "activations/layer11_attention_weight_min": -32.40296936035156,
      "activations/layer12_attention_weight_max": 26.549230575561523,
      "activations/layer12_attention_weight_min": -25.076391220092773,
      "activations/layer13_attention_weight_max": 39.60885238647461,
      "activations/layer13_attention_weight_min": -36.56800842285156,
      "activations/layer14_attention_weight_max": 47.31863021850586,
      "activations/layer14_attention_weight_min": -38.00386047363281,
      "activations/layer15_attention_weight_max": 38.84083938598633,
      "activations/layer15_attention_weight_min": -31.807994842529297,
      "activations/layer16_attention_weight_max": 31.4312744140625,
      "activations/layer16_attention_weight_min": -28.298416137695312,
      "activations/layer17_attention_weight_max": 31.334848403930664,
      "activations/layer17_attention_weight_min": -25.240812301635742,
      "activations/layer18_attention_weight_max": 34.46747970581055,
      "activations/layer18_attention_weight_min": -23.11490821838379,
      "activations/layer19_attention_weight_max": 34.35255432128906,
      "activations/layer19_attention_weight_min": -30.82071876525879,
      "activations/layer1_attention_weight_max": 16.06168556213379,
      "activations/layer1_attention_weight_min": -14.084020614624023,
      "activations/layer20_attention_weight_max": 31.143430709838867,
      "activations/layer20_attention_weight_min": -25.33806800842285,
      "activations/layer21_attention_weight_max": 31.41684913635254,
      "activations/layer21_attention_weight_min": -25.734033584594727,
      "activations/layer22_attention_weight_max": 48.20332717895508,
      "activations/layer22_attention_weight_min": -30.432861328125,
      "activations/layer23_attention_weight_max": 34.07820129394531,
      "activations/layer23_attention_weight_min": -26.258007049560547,
      "activations/layer2_attention_weight_max": 31.611661911010742,
      "activations/layer2_attention_weight_min": -32.27765655517578,
      "activations/layer3_attention_weight_max": 96.31302642822266,
      "activations/layer3_attention_weight_min": -99.57923126220703,
      "activations/layer4_attention_weight_max": 107.51081848144531,
      "activations/layer4_attention_weight_min": -112.4923095703125,
      "activations/layer5_attention_weight_max": 55.98052978515625,
      "activations/layer5_attention_weight_min": -65.47616577148438,
      "activations/layer6_attention_weight_max": 43.03003692626953,
      "activations/layer6_attention_weight_min": -45.43734359741211,
      "activations/layer7_attention_weight_max": 88.84020233154297,
      "activations/layer7_attention_weight_min": -86.80786895751953,
      "activations/layer8_attention_weight_max": 38.44966506958008,
      "activations/layer8_attention_weight_min": -39.59305191040039,
      "activations/layer9_attention_weight_max": 31.791162490844727,
      "activations/layer9_attention_weight_min": -31.524185180664062,
      "epoch": 14.18,
      "learning_rate": 5.916666666666666e-05,
      "loss": 2.7429,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.5572,
      "eval_samples_per_second": 501.799,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.5572,
      "eval_openwebtext_samples_per_second": 501.799,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 2.0152,
      "eval_wikitext_samples_per_second": 226.278,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_lambada_loss": 2.53125,
      "eval_lambada_ppl": 12.569207830853442,
      "eval_lambada_runtime": 9.5876,
      "eval_lambada_samples_per_second": 507.846,
      "step": 244000
    },
    {
      "activations/layer0_attention_weight_max": 15.726167678833008,
      "activations/layer0_attention_weight_min": -13.208932876586914,
      "activations/layer10_attention_weight_max": 32.62515640258789,
      "activations/layer10_attention_weight_min": -32.02260971069336,
      "activations/layer11_attention_weight_max": 33.46638870239258,
      "activations/layer11_attention_weight_min": -32.326171875,
      "activations/layer12_attention_weight_max": 24.26984405517578,
      "activations/layer12_attention_weight_min": -25.185787200927734,
      "activations/layer13_attention_weight_max": 37.090938568115234,
      "activations/layer13_attention_weight_min": -33.25625991821289,
      "activations/layer14_attention_weight_max": 40.89082717895508,
      "activations/layer14_attention_weight_min": -34.381248474121094,
      "activations/layer15_attention_weight_max": 36.62907791137695,
      "activations/layer15_attention_weight_min": -31.803916931152344,
      "activations/layer16_attention_weight_max": 30.235305786132812,
      "activations/layer16_attention_weight_min": -25.797626495361328,
      "activations/layer17_attention_weight_max": 33.56832504272461,
      "activations/layer17_attention_weight_min": -25.52462387084961,
      "activations/layer18_attention_weight_max": 32.96677017211914,
      "activations/layer18_attention_weight_min": -23.13982391357422,
      "activations/layer19_attention_weight_max": 34.71812438964844,
      "activations/layer19_attention_weight_min": -31.703022003173828,
      "activations/layer1_attention_weight_max": 16.55482292175293,
      "activations/layer1_attention_weight_min": -14.600937843322754,
      "activations/layer20_attention_weight_max": 28.975996017456055,
      "activations/layer20_attention_weight_min": -23.543079376220703,
      "activations/layer21_attention_weight_max": 28.47934913635254,
      "activations/layer21_attention_weight_min": -23.74003028869629,
      "activations/layer22_attention_weight_max": 45.93428421020508,
      "activations/layer22_attention_weight_min": -30.26279640197754,
      "activations/layer23_attention_weight_max": 34.18442153930664,
      "activations/layer23_attention_weight_min": -24.11481475830078,
      "activations/layer2_attention_weight_max": 31.77864646911621,
      "activations/layer2_attention_weight_min": -32.45383071899414,
      "activations/layer3_attention_weight_max": 93.51113891601562,
      "activations/layer3_attention_weight_min": -96.08609008789062,
      "activations/layer4_attention_weight_max": 103.85709381103516,
      "activations/layer4_attention_weight_min": -106.83182525634766,
      "activations/layer5_attention_weight_max": 55.03910446166992,
      "activations/layer5_attention_weight_min": -60.449920654296875,
      "activations/layer6_attention_weight_max": 44.10980987548828,
      "activations/layer6_attention_weight_min": -44.52141189575195,
      "activations/layer7_attention_weight_max": 88.24478912353516,
      "activations/layer7_attention_weight_min": -86.59142303466797,
      "activations/layer8_attention_weight_max": 39.125343322753906,
      "activations/layer8_attention_weight_min": -39.57288360595703,
      "activations/layer9_attention_weight_max": 32.413795471191406,
      "activations/layer9_attention_weight_min": -33.11204528808594,
      "epoch": 14.18,
      "learning_rate": 5.914772727272727e-05,
      "loss": 2.7654,
      "step": 244050
    },
    {
      "activations/layer0_attention_weight_max": 15.777226448059082,
      "activations/layer0_attention_weight_min": -12.12781810760498,
      "activations/layer10_attention_weight_max": 32.47060775756836,
      "activations/layer10_attention_weight_min": -33.30582809448242,
      "activations/layer11_attention_weight_max": 33.98445129394531,
      "activations/layer11_attention_weight_min": -33.57353973388672,
      "activations/layer12_attention_weight_max": 26.12000274658203,
      "activations/layer12_attention_weight_min": -25.699485778808594,
      "activations/layer13_attention_weight_max": 39.45716857910156,
      "activations/layer13_attention_weight_min": -34.96571731567383,
      "activations/layer14_attention_weight_max": 44.63087463378906,
      "activations/layer14_attention_weight_min": -36.007652282714844,
      "activations/layer15_attention_weight_max": 37.479576110839844,
      "activations/layer15_attention_weight_min": -33.10384750366211,
      "activations/layer16_attention_weight_max": 29.148313522338867,
      "activations/layer16_attention_weight_min": -31.058439254760742,
      "activations/layer17_attention_weight_max": 31.025686264038086,
      "activations/layer17_attention_weight_min": -26.43779945373535,
      "activations/layer18_attention_weight_max": 31.519140243530273,
      "activations/layer18_attention_weight_min": -25.61028480529785,
      "activations/layer19_attention_weight_max": 31.7945613861084,
      "activations/layer19_attention_weight_min": -31.645328521728516,
      "activations/layer1_attention_weight_max": 15.601930618286133,
      "activations/layer1_attention_weight_min": -14.578350067138672,
      "activations/layer20_attention_weight_max": 27.100914001464844,
      "activations/layer20_attention_weight_min": -25.239883422851562,
      "activations/layer21_attention_weight_max": 27.095956802368164,
      "activations/layer21_attention_weight_min": -22.897319793701172,
      "activations/layer22_attention_weight_max": 46.619014739990234,
      "activations/layer22_attention_weight_min": -30.718278884887695,
      "activations/layer23_attention_weight_max": 35.63228988647461,
      "activations/layer23_attention_weight_min": -26.850065231323242,
      "activations/layer2_attention_weight_max": 31.62598419189453,
      "activations/layer2_attention_weight_min": -32.009857177734375,
      "activations/layer3_attention_weight_max": 97.54017639160156,
      "activations/layer3_attention_weight_min": -98.01582336425781,
      "activations/layer4_attention_weight_max": 111.94281005859375,
      "activations/layer4_attention_weight_min": -114.78800201416016,
      "activations/layer5_attention_weight_max": 53.344017028808594,
      "activations/layer5_attention_weight_min": -59.75648498535156,
      "activations/layer6_attention_weight_max": 44.27712631225586,
      "activations/layer6_attention_weight_min": -47.22133255004883,
      "activations/layer7_attention_weight_max": 87.03319549560547,
      "activations/layer7_attention_weight_min": -92.87068176269531,
      "activations/layer8_attention_weight_max": 39.8997917175293,
      "activations/layer8_attention_weight_min": -40.34404373168945,
      "activations/layer9_attention_weight_max": 32.018096923828125,
      "activations/layer9_attention_weight_min": -34.600406646728516,
      "epoch": 14.18,
      "learning_rate": 5.9128787878787874e-05,
      "loss": 2.7575,
      "step": 244100
    },
    {
      "activations/layer0_attention_weight_max": 16.05763053894043,
      "activations/layer0_attention_weight_min": -12.092595100402832,
      "activations/layer10_attention_weight_max": 29.89830780029297,
      "activations/layer10_attention_weight_min": -31.134288787841797,
      "activations/layer11_attention_weight_max": 30.258750915527344,
      "activations/layer11_attention_weight_min": -30.71691131591797,
      "activations/layer12_attention_weight_max": 26.88425064086914,
      "activations/layer12_attention_weight_min": -27.17889976501465,
      "activations/layer13_attention_weight_max": 38.10122299194336,
      "activations/layer13_attention_weight_min": -33.251136779785156,
      "activations/layer14_attention_weight_max": 42.8269157409668,
      "activations/layer14_attention_weight_min": -35.06421661376953,
      "activations/layer15_attention_weight_max": 37.58064270019531,
      "activations/layer15_attention_weight_min": -33.8554801940918,
      "activations/layer16_attention_weight_max": 34.352455139160156,
      "activations/layer16_attention_weight_min": -28.35837745666504,
      "activations/layer17_attention_weight_max": 27.597476959228516,
      "activations/layer17_attention_weight_min": -26.919641494750977,
      "activations/layer18_attention_weight_max": 30.362443923950195,
      "activations/layer18_attention_weight_min": -24.718929290771484,
      "activations/layer19_attention_weight_max": 34.21708297729492,
      "activations/layer19_attention_weight_min": -28.41654396057129,
      "activations/layer1_attention_weight_max": 17.034210205078125,
      "activations/layer1_attention_weight_min": -14.28463077545166,
      "activations/layer20_attention_weight_max": 28.252023696899414,
      "activations/layer20_attention_weight_min": -23.462568283081055,
      "activations/layer21_attention_weight_max": 27.3518123626709,
      "activations/layer21_attention_weight_min": -22.003387451171875,
      "activations/layer22_attention_weight_max": 42.495155334472656,
      "activations/layer22_attention_weight_min": -30.570283889770508,
      "activations/layer23_attention_weight_max": 32.610225677490234,
      "activations/layer23_attention_weight_min": -26.089828491210938,
      "activations/layer2_attention_weight_max": 31.663604736328125,
      "activations/layer2_attention_weight_min": -33.53165054321289,
      "activations/layer3_attention_weight_max": 93.49089813232422,
      "activations/layer3_attention_weight_min": -99.5680160522461,
      "activations/layer4_attention_weight_max": 105.0993423461914,
      "activations/layer4_attention_weight_min": -107.26155090332031,
      "activations/layer5_attention_weight_max": 53.88438415527344,
      "activations/layer5_attention_weight_min": -63.06248474121094,
      "activations/layer6_attention_weight_max": 41.687705993652344,
      "activations/layer6_attention_weight_min": -46.10012435913086,
      "activations/layer7_attention_weight_max": 93.99867248535156,
      "activations/layer7_attention_weight_min": -87.75785827636719,
      "activations/layer8_attention_weight_max": 37.567291259765625,
      "activations/layer8_attention_weight_min": -40.11146926879883,
      "activations/layer9_attention_weight_max": 30.76022720336914,
      "activations/layer9_attention_weight_min": -31.146564483642578,
      "epoch": 14.19,
      "learning_rate": 5.910984848484848e-05,
      "loss": 2.7573,
      "step": 244150
    },
    {
      "activations/layer0_attention_weight_max": 16.35650634765625,
      "activations/layer0_attention_weight_min": -13.083913803100586,
      "activations/layer10_attention_weight_max": 36.79107666015625,
      "activations/layer10_attention_weight_min": -35.014678955078125,
      "activations/layer11_attention_weight_max": 39.58254623413086,
      "activations/layer11_attention_weight_min": -36.970638275146484,
      "activations/layer12_attention_weight_max": 27.861312866210938,
      "activations/layer12_attention_weight_min": -30.134790420532227,
      "activations/layer13_attention_weight_max": 41.56981658935547,
      "activations/layer13_attention_weight_min": -38.57529830932617,
      "activations/layer14_attention_weight_max": 47.42490005493164,
      "activations/layer14_attention_weight_min": -40.371788024902344,
      "activations/layer15_attention_weight_max": 46.22468185424805,
      "activations/layer15_attention_weight_min": -37.94057846069336,
      "activations/layer16_attention_weight_max": 31.836669921875,
      "activations/layer16_attention_weight_min": -28.35451316833496,
      "activations/layer17_attention_weight_max": 29.522586822509766,
      "activations/layer17_attention_weight_min": -25.544042587280273,
      "activations/layer18_attention_weight_max": 33.40345764160156,
      "activations/layer18_attention_weight_min": -24.627185821533203,
      "activations/layer19_attention_weight_max": 36.23168182373047,
      "activations/layer19_attention_weight_min": -30.04363441467285,
      "activations/layer1_attention_weight_max": 16.061059951782227,
      "activations/layer1_attention_weight_min": -14.983281135559082,
      "activations/layer20_attention_weight_max": 31.952329635620117,
      "activations/layer20_attention_weight_min": -24.32889747619629,
      "activations/layer21_attention_weight_max": 34.49085235595703,
      "activations/layer21_attention_weight_min": -24.128713607788086,
      "activations/layer22_attention_weight_max": 45.896602630615234,
      "activations/layer22_attention_weight_min": -29.518478393554688,
      "activations/layer23_attention_weight_max": 36.956886291503906,
      "activations/layer23_attention_weight_min": -22.870452880859375,
      "activations/layer2_attention_weight_max": 35.900089263916016,
      "activations/layer2_attention_weight_min": -34.57356643676758,
      "activations/layer3_attention_weight_max": 102.80535125732422,
      "activations/layer3_attention_weight_min": -101.16364288330078,
      "activations/layer4_attention_weight_max": 119.53974914550781,
      "activations/layer4_attention_weight_min": -111.80391693115234,
      "activations/layer5_attention_weight_max": 54.95768737792969,
      "activations/layer5_attention_weight_min": -62.48405456542969,
      "activations/layer6_attention_weight_max": 49.560447692871094,
      "activations/layer6_attention_weight_min": -47.26186752319336,
      "activations/layer7_attention_weight_max": 100.24037170410156,
      "activations/layer7_attention_weight_min": -94.70633697509766,
      "activations/layer8_attention_weight_max": 45.55610656738281,
      "activations/layer8_attention_weight_min": -42.6098518371582,
      "activations/layer9_attention_weight_max": 36.335426330566406,
      "activations/layer9_attention_weight_min": -34.63962173461914,
      "epoch": 14.19,
      "learning_rate": 5.909090909090908e-05,
      "loss": 2.7622,
      "step": 244200
    },
    {
      "activations/layer0_attention_weight_max": 15.947012901306152,
      "activations/layer0_attention_weight_min": -12.640419006347656,
      "activations/layer10_attention_weight_max": 33.44464111328125,
      "activations/layer10_attention_weight_min": -31.843284606933594,
      "activations/layer11_attention_weight_max": 33.709632873535156,
      "activations/layer11_attention_weight_min": -31.86871910095215,
      "activations/layer12_attention_weight_max": 26.408695220947266,
      "activations/layer12_attention_weight_min": -28.26423454284668,
      "activations/layer13_attention_weight_max": 37.793907165527344,
      "activations/layer13_attention_weight_min": -35.225948333740234,
      "activations/layer14_attention_weight_max": 41.25948715209961,
      "activations/layer14_attention_weight_min": -36.52702713012695,
      "activations/layer15_attention_weight_max": 35.77122497558594,
      "activations/layer15_attention_weight_min": -33.953468322753906,
      "activations/layer16_attention_weight_max": 27.82488441467285,
      "activations/layer16_attention_weight_min": -26.863306045532227,
      "activations/layer17_attention_weight_max": 30.633867263793945,
      "activations/layer17_attention_weight_min": -23.987024307250977,
      "activations/layer18_attention_weight_max": 33.82745361328125,
      "activations/layer18_attention_weight_min": -23.031009674072266,
      "activations/layer19_attention_weight_max": 33.32151412963867,
      "activations/layer19_attention_weight_min": -31.33843421936035,
      "activations/layer1_attention_weight_max": 14.963949203491211,
      "activations/layer1_attention_weight_min": -14.298644065856934,
      "activations/layer20_attention_weight_max": 30.385969161987305,
      "activations/layer20_attention_weight_min": -25.340208053588867,
      "activations/layer21_attention_weight_max": 31.104637145996094,
      "activations/layer21_attention_weight_min": -23.357030868530273,
      "activations/layer22_attention_weight_max": 41.189727783203125,
      "activations/layer22_attention_weight_min": -30.64742660522461,
      "activations/layer23_attention_weight_max": 33.27075958251953,
      "activations/layer23_attention_weight_min": -26.771230697631836,
      "activations/layer2_attention_weight_max": 33.586456298828125,
      "activations/layer2_attention_weight_min": -33.68193817138672,
      "activations/layer3_attention_weight_max": 94.09453582763672,
      "activations/layer3_attention_weight_min": -95.9996337890625,
      "activations/layer4_attention_weight_max": 110.09615325927734,
      "activations/layer4_attention_weight_min": -112.7551498413086,
      "activations/layer5_attention_weight_max": 52.533443450927734,
      "activations/layer5_attention_weight_min": -61.11314392089844,
      "activations/layer6_attention_weight_max": 42.99427795410156,
      "activations/layer6_attention_weight_min": -47.37970733642578,
      "activations/layer7_attention_weight_max": 92.10040283203125,
      "activations/layer7_attention_weight_min": -96.27494812011719,
      "activations/layer8_attention_weight_max": 37.38209915161133,
      "activations/layer8_attention_weight_min": -40.5959587097168,
      "activations/layer9_attention_weight_max": 30.61347007751465,
      "activations/layer9_attention_weight_min": -32.861114501953125,
      "epoch": 14.19,
      "learning_rate": 5.90719696969697e-05,
      "loss": 2.7649,
      "step": 244250
    },
    {
      "activations/layer0_attention_weight_max": 15.169678688049316,
      "activations/layer0_attention_weight_min": -12.15298080444336,
      "activations/layer10_attention_weight_max": 33.166534423828125,
      "activations/layer10_attention_weight_min": -31.767070770263672,
      "activations/layer11_attention_weight_max": 33.33446502685547,
      "activations/layer11_attention_weight_min": -33.72269821166992,
      "activations/layer12_attention_weight_max": 26.79078483581543,
      "activations/layer12_attention_weight_min": -25.469852447509766,
      "activations/layer13_attention_weight_max": 40.74457931518555,
      "activations/layer13_attention_weight_min": -34.478519439697266,
      "activations/layer14_attention_weight_max": 47.83356475830078,
      "activations/layer14_attention_weight_min": -38.51094436645508,
      "activations/layer15_attention_weight_max": 43.224639892578125,
      "activations/layer15_attention_weight_min": -35.11386489868164,
      "activations/layer16_attention_weight_max": 30.509733200073242,
      "activations/layer16_attention_weight_min": -29.936683654785156,
      "activations/layer17_attention_weight_max": 34.64321517944336,
      "activations/layer17_attention_weight_min": -26.941068649291992,
      "activations/layer18_attention_weight_max": 31.351354598999023,
      "activations/layer18_attention_weight_min": -23.824249267578125,
      "activations/layer19_attention_weight_max": 34.37446212768555,
      "activations/layer19_attention_weight_min": -31.413700103759766,
      "activations/layer1_attention_weight_max": 16.552255630493164,
      "activations/layer1_attention_weight_min": -15.579916954040527,
      "activations/layer20_attention_weight_max": 30.272727966308594,
      "activations/layer20_attention_weight_min": -24.39348030090332,
      "activations/layer21_attention_weight_max": 28.37054443359375,
      "activations/layer21_attention_weight_min": -23.466081619262695,
      "activations/layer22_attention_weight_max": 48.696353912353516,
      "activations/layer22_attention_weight_min": -29.671552658081055,
      "activations/layer23_attention_weight_max": 35.8372802734375,
      "activations/layer23_attention_weight_min": -24.363449096679688,
      "activations/layer2_attention_weight_max": 33.04431915283203,
      "activations/layer2_attention_weight_min": -32.003570556640625,
      "activations/layer3_attention_weight_max": 97.3327407836914,
      "activations/layer3_attention_weight_min": -96.5169448852539,
      "activations/layer4_attention_weight_max": 110.8355712890625,
      "activations/layer4_attention_weight_min": -110.1537094116211,
      "activations/layer5_attention_weight_max": 51.87226486206055,
      "activations/layer5_attention_weight_min": -65.45478820800781,
      "activations/layer6_attention_weight_max": 43.457828521728516,
      "activations/layer6_attention_weight_min": -49.37934875488281,
      "activations/layer7_attention_weight_max": 89.06836700439453,
      "activations/layer7_attention_weight_min": -90.9834976196289,
      "activations/layer8_attention_weight_max": 40.205135345458984,
      "activations/layer8_attention_weight_min": -43.605873107910156,
      "activations/layer9_attention_weight_max": 35.23728561401367,
      "activations/layer9_attention_weight_min": -33.62278366088867,
      "epoch": 14.2,
      "learning_rate": 5.90530303030303e-05,
      "loss": 2.7597,
      "step": 244300
    },
    {
      "activations/layer0_attention_weight_max": 15.608144760131836,
      "activations/layer0_attention_weight_min": -12.137775421142578,
      "activations/layer10_attention_weight_max": 32.17719268798828,
      "activations/layer10_attention_weight_min": -31.7096004486084,
      "activations/layer11_attention_weight_max": 32.62920379638672,
      "activations/layer11_attention_weight_min": -34.07566833496094,
      "activations/layer12_attention_weight_max": 25.51059913635254,
      "activations/layer12_attention_weight_min": -24.657480239868164,
      "activations/layer13_attention_weight_max": 38.34266662597656,
      "activations/layer13_attention_weight_min": -32.485809326171875,
      "activations/layer14_attention_weight_max": 41.05769729614258,
      "activations/layer14_attention_weight_min": -37.98796081542969,
      "activations/layer15_attention_weight_max": 39.392391204833984,
      "activations/layer15_attention_weight_min": -34.35974884033203,
      "activations/layer16_attention_weight_max": 30.646533966064453,
      "activations/layer16_attention_weight_min": -27.302167892456055,
      "activations/layer17_attention_weight_max": 30.325876235961914,
      "activations/layer17_attention_weight_min": -25.65880012512207,
      "activations/layer18_attention_weight_max": 30.75603675842285,
      "activations/layer18_attention_weight_min": -26.734167098999023,
      "activations/layer19_attention_weight_max": 35.649173736572266,
      "activations/layer19_attention_weight_min": -30.95108985900879,
      "activations/layer1_attention_weight_max": 15.788374900817871,
      "activations/layer1_attention_weight_min": -15.098655700683594,
      "activations/layer20_attention_weight_max": 28.804466247558594,
      "activations/layer20_attention_weight_min": -25.726884841918945,
      "activations/layer21_attention_weight_max": 29.044490814208984,
      "activations/layer21_attention_weight_min": -25.325157165527344,
      "activations/layer22_attention_weight_max": 43.75229263305664,
      "activations/layer22_attention_weight_min": -31.887447357177734,
      "activations/layer23_attention_weight_max": 34.20259094238281,
      "activations/layer23_attention_weight_min": -25.500381469726562,
      "activations/layer2_attention_weight_max": 33.768646240234375,
      "activations/layer2_attention_weight_min": -34.60675811767578,
      "activations/layer3_attention_weight_max": 101.34693908691406,
      "activations/layer3_attention_weight_min": -98.98869323730469,
      "activations/layer4_attention_weight_max": 112.0973892211914,
      "activations/layer4_attention_weight_min": -112.92461395263672,
      "activations/layer5_attention_weight_max": 53.201995849609375,
      "activations/layer5_attention_weight_min": -62.41636657714844,
      "activations/layer6_attention_weight_max": 44.309425354003906,
      "activations/layer6_attention_weight_min": -48.56450653076172,
      "activations/layer7_attention_weight_max": 96.55269622802734,
      "activations/layer7_attention_weight_min": -88.25679016113281,
      "activations/layer8_attention_weight_max": 38.825923919677734,
      "activations/layer8_attention_weight_min": -42.994449615478516,
      "activations/layer9_attention_weight_max": 31.377592086791992,
      "activations/layer9_attention_weight_min": -33.56694412231445,
      "epoch": 14.2,
      "learning_rate": 5.90340909090909e-05,
      "loss": 2.7672,
      "step": 244350
    },
    {
      "activations/layer0_attention_weight_max": 17.309314727783203,
      "activations/layer0_attention_weight_min": -13.06362247467041,
      "activations/layer10_attention_weight_max": 32.97481155395508,
      "activations/layer10_attention_weight_min": -34.841243743896484,
      "activations/layer11_attention_weight_max": 31.642234802246094,
      "activations/layer11_attention_weight_min": -32.16876220703125,
      "activations/layer12_attention_weight_max": 26.205429077148438,
      "activations/layer12_attention_weight_min": -24.966278076171875,
      "activations/layer13_attention_weight_max": 39.15898895263672,
      "activations/layer13_attention_weight_min": -33.300899505615234,
      "activations/layer14_attention_weight_max": 41.921512603759766,
      "activations/layer14_attention_weight_min": -37.51216506958008,
      "activations/layer15_attention_weight_max": 37.638099670410156,
      "activations/layer15_attention_weight_min": -33.30466079711914,
      "activations/layer16_attention_weight_max": 30.230207443237305,
      "activations/layer16_attention_weight_min": -27.003358840942383,
      "activations/layer17_attention_weight_max": 34.35591506958008,
      "activations/layer17_attention_weight_min": -25.013586044311523,
      "activations/layer18_attention_weight_max": 34.00733947753906,
      "activations/layer18_attention_weight_min": -23.22787857055664,
      "activations/layer19_attention_weight_max": 34.48748779296875,
      "activations/layer19_attention_weight_min": -30.02174186706543,
      "activations/layer1_attention_weight_max": 16.000768661499023,
      "activations/layer1_attention_weight_min": -14.90252685546875,
      "activations/layer20_attention_weight_max": 30.248016357421875,
      "activations/layer20_attention_weight_min": -23.974332809448242,
      "activations/layer21_attention_weight_max": 28.380929946899414,
      "activations/layer21_attention_weight_min": -22.839792251586914,
      "activations/layer22_attention_weight_max": 42.08393478393555,
      "activations/layer22_attention_weight_min": -29.090362548828125,
      "activations/layer23_attention_weight_max": 33.05620574951172,
      "activations/layer23_attention_weight_min": -23.74086570739746,
      "activations/layer2_attention_weight_max": 34.3365364074707,
      "activations/layer2_attention_weight_min": -33.33894348144531,
      "activations/layer3_attention_weight_max": 99.43225860595703,
      "activations/layer3_attention_weight_min": -100.2016372680664,
      "activations/layer4_attention_weight_max": 112.98705291748047,
      "activations/layer4_attention_weight_min": -115.74317932128906,
      "activations/layer5_attention_weight_max": 51.323944091796875,
      "activations/layer5_attention_weight_min": -66.38925170898438,
      "activations/layer6_attention_weight_max": 46.12876892089844,
      "activations/layer6_attention_weight_min": -49.68911361694336,
      "activations/layer7_attention_weight_max": 92.97216796875,
      "activations/layer7_attention_weight_min": -94.78264617919922,
      "activations/layer8_attention_weight_max": 39.06159973144531,
      "activations/layer8_attention_weight_min": -41.90755081176758,
      "activations/layer9_attention_weight_max": 31.211965560913086,
      "activations/layer9_attention_weight_min": -33.46051788330078,
      "epoch": 14.2,
      "learning_rate": 5.901515151515151e-05,
      "loss": 2.7625,
      "step": 244400
    },
    {
      "activations/layer0_attention_weight_max": 16.20003890991211,
      "activations/layer0_attention_weight_min": -12.97077465057373,
      "activations/layer10_attention_weight_max": 31.441375732421875,
      "activations/layer10_attention_weight_min": -32.42218017578125,
      "activations/layer11_attention_weight_max": 33.425445556640625,
      "activations/layer11_attention_weight_min": -31.166397094726562,
      "activations/layer12_attention_weight_max": 25.17269515991211,
      "activations/layer12_attention_weight_min": -25.901445388793945,
      "activations/layer13_attention_weight_max": 39.535133361816406,
      "activations/layer13_attention_weight_min": -34.979957580566406,
      "activations/layer14_attention_weight_max": 42.12168502807617,
      "activations/layer14_attention_weight_min": -36.887210845947266,
      "activations/layer15_attention_weight_max": 37.74104309082031,
      "activations/layer15_attention_weight_min": -33.845184326171875,
      "activations/layer16_attention_weight_max": 29.660850524902344,
      "activations/layer16_attention_weight_min": -27.404695510864258,
      "activations/layer17_attention_weight_max": 32.9024543762207,
      "activations/layer17_attention_weight_min": -25.99418067932129,
      "activations/layer18_attention_weight_max": 28.855003356933594,
      "activations/layer18_attention_weight_min": -25.77600860595703,
      "activations/layer19_attention_weight_max": 32.868099212646484,
      "activations/layer19_attention_weight_min": -33.104740142822266,
      "activations/layer1_attention_weight_max": 17.086275100708008,
      "activations/layer1_attention_weight_min": -15.59342098236084,
      "activations/layer20_attention_weight_max": 29.54589080810547,
      "activations/layer20_attention_weight_min": -25.687816619873047,
      "activations/layer21_attention_weight_max": 26.4904842376709,
      "activations/layer21_attention_weight_min": -25.871885299682617,
      "activations/layer22_attention_weight_max": 40.40060806274414,
      "activations/layer22_attention_weight_min": -33.79997253417969,
      "activations/layer23_attention_weight_max": 33.896915435791016,
      "activations/layer23_attention_weight_min": -27.021987915039062,
      "activations/layer2_attention_weight_max": 33.38518142700195,
      "activations/layer2_attention_weight_min": -32.916988372802734,
      "activations/layer3_attention_weight_max": 93.2124252319336,
      "activations/layer3_attention_weight_min": -99.33474731445312,
      "activations/layer4_attention_weight_max": 108.39666748046875,
      "activations/layer4_attention_weight_min": -112.55572509765625,
      "activations/layer5_attention_weight_max": 51.90385055541992,
      "activations/layer5_attention_weight_min": -64.68754577636719,
      "activations/layer6_attention_weight_max": 43.14866638183594,
      "activations/layer6_attention_weight_min": -45.8107795715332,
      "activations/layer7_attention_weight_max": 89.60031127929688,
      "activations/layer7_attention_weight_min": -88.77078247070312,
      "activations/layer8_attention_weight_max": 40.947505950927734,
      "activations/layer8_attention_weight_min": -40.89595413208008,
      "activations/layer9_attention_weight_max": 30.978879928588867,
      "activations/layer9_attention_weight_min": -31.495258331298828,
      "epoch": 14.2,
      "learning_rate": 5.899621212121212e-05,
      "loss": 2.7656,
      "step": 244450
    },
    {
      "activations/layer0_attention_weight_max": 15.537023544311523,
      "activations/layer0_attention_weight_min": -14.2847261428833,
      "activations/layer10_attention_weight_max": 33.42362976074219,
      "activations/layer10_attention_weight_min": -34.16065216064453,
      "activations/layer11_attention_weight_max": 35.82210159301758,
      "activations/layer11_attention_weight_min": -36.566551208496094,
      "activations/layer12_attention_weight_max": 27.050432205200195,
      "activations/layer12_attention_weight_min": -28.300535202026367,
      "activations/layer13_attention_weight_max": 39.109657287597656,
      "activations/layer13_attention_weight_min": -36.26212692260742,
      "activations/layer14_attention_weight_max": 45.22950744628906,
      "activations/layer14_attention_weight_min": -37.90042495727539,
      "activations/layer15_attention_weight_max": 37.65470504760742,
      "activations/layer15_attention_weight_min": -37.7432975769043,
      "activations/layer16_attention_weight_max": 31.408748626708984,
      "activations/layer16_attention_weight_min": -28.20359230041504,
      "activations/layer17_attention_weight_max": 35.34588623046875,
      "activations/layer17_attention_weight_min": -26.478607177734375,
      "activations/layer18_attention_weight_max": 36.776973724365234,
      "activations/layer18_attention_weight_min": -25.225143432617188,
      "activations/layer19_attention_weight_max": 34.02562713623047,
      "activations/layer19_attention_weight_min": -30.863407135009766,
      "activations/layer1_attention_weight_max": 17.41446876525879,
      "activations/layer1_attention_weight_min": -15.091885566711426,
      "activations/layer20_attention_weight_max": 32.73996353149414,
      "activations/layer20_attention_weight_min": -24.814369201660156,
      "activations/layer21_attention_weight_max": 30.184616088867188,
      "activations/layer21_attention_weight_min": -23.9665470123291,
      "activations/layer22_attention_weight_max": 46.43449783325195,
      "activations/layer22_attention_weight_min": -30.567338943481445,
      "activations/layer23_attention_weight_max": 34.03996658325195,
      "activations/layer23_attention_weight_min": -26.35340690612793,
      "activations/layer2_attention_weight_max": 32.650047302246094,
      "activations/layer2_attention_weight_min": -34.88825225830078,
      "activations/layer3_attention_weight_max": 96.5234146118164,
      "activations/layer3_attention_weight_min": -100.7857894897461,
      "activations/layer4_attention_weight_max": 111.70399475097656,
      "activations/layer4_attention_weight_min": -111.50484466552734,
      "activations/layer5_attention_weight_max": 52.986907958984375,
      "activations/layer5_attention_weight_min": -61.936824798583984,
      "activations/layer6_attention_weight_max": 43.76293182373047,
      "activations/layer6_attention_weight_min": -48.425846099853516,
      "activations/layer7_attention_weight_max": 92.0957260131836,
      "activations/layer7_attention_weight_min": -93.65766906738281,
      "activations/layer8_attention_weight_max": 42.64296340942383,
      "activations/layer8_attention_weight_min": -42.27236557006836,
      "activations/layer9_attention_weight_max": 32.39674377441406,
      "activations/layer9_attention_weight_min": -36.006622314453125,
      "epoch": 14.21,
      "learning_rate": 5.8977272727272726e-05,
      "loss": 2.7497,
      "step": 244500
    },
    {
      "activations/layer0_attention_weight_max": 15.347434997558594,
      "activations/layer0_attention_weight_min": -13.960585594177246,
      "activations/layer10_attention_weight_max": 33.723445892333984,
      "activations/layer10_attention_weight_min": -33.53660583496094,
      "activations/layer11_attention_weight_max": 33.34613800048828,
      "activations/layer11_attention_weight_min": -34.183719635009766,
      "activations/layer12_attention_weight_max": 26.505922317504883,
      "activations/layer12_attention_weight_min": -25.747533798217773,
      "activations/layer13_attention_weight_max": 39.661895751953125,
      "activations/layer13_attention_weight_min": -37.71218490600586,
      "activations/layer14_attention_weight_max": 43.23811721801758,
      "activations/layer14_attention_weight_min": -38.69407272338867,
      "activations/layer15_attention_weight_max": 37.44193649291992,
      "activations/layer15_attention_weight_min": -33.82956314086914,
      "activations/layer16_attention_weight_max": 33.56107711791992,
      "activations/layer16_attention_weight_min": -28.07541847229004,
      "activations/layer17_attention_weight_max": 32.484588623046875,
      "activations/layer17_attention_weight_min": -25.608448028564453,
      "activations/layer18_attention_weight_max": 33.35289764404297,
      "activations/layer18_attention_weight_min": -24.47498893737793,
      "activations/layer19_attention_weight_max": 42.8693962097168,
      "activations/layer19_attention_weight_min": -31.851224899291992,
      "activations/layer1_attention_weight_max": 15.460341453552246,
      "activations/layer1_attention_weight_min": -12.796380043029785,
      "activations/layer20_attention_weight_max": 30.816057205200195,
      "activations/layer20_attention_weight_min": -24.444744110107422,
      "activations/layer21_attention_weight_max": 28.99542236328125,
      "activations/layer21_attention_weight_min": -24.1013126373291,
      "activations/layer22_attention_weight_max": 41.648094177246094,
      "activations/layer22_attention_weight_min": -29.7520751953125,
      "activations/layer23_attention_weight_max": 33.922950744628906,
      "activations/layer23_attention_weight_min": -23.552104949951172,
      "activations/layer2_attention_weight_max": 33.869964599609375,
      "activations/layer2_attention_weight_min": -32.073944091796875,
      "activations/layer3_attention_weight_max": 97.43500518798828,
      "activations/layer3_attention_weight_min": -100.96199035644531,
      "activations/layer4_attention_weight_max": 111.84105682373047,
      "activations/layer4_attention_weight_min": -110.17125701904297,
      "activations/layer5_attention_weight_max": 54.44098663330078,
      "activations/layer5_attention_weight_min": -63.076820373535156,
      "activations/layer6_attention_weight_max": 47.23015213012695,
      "activations/layer6_attention_weight_min": -47.806907653808594,
      "activations/layer7_attention_weight_max": 88.15876770019531,
      "activations/layer7_attention_weight_min": -95.21539306640625,
      "activations/layer8_attention_weight_max": 44.44279861450195,
      "activations/layer8_attention_weight_min": -43.239200592041016,
      "activations/layer9_attention_weight_max": 34.542415618896484,
      "activations/layer9_attention_weight_min": -36.7742805480957,
      "epoch": 14.21,
      "learning_rate": 5.895833333333333e-05,
      "loss": 2.772,
      "step": 244550
    },
    {
      "activations/layer0_attention_weight_max": 16.081119537353516,
      "activations/layer0_attention_weight_min": -12.548837661743164,
      "activations/layer10_attention_weight_max": 33.83774185180664,
      "activations/layer10_attention_weight_min": -33.182952880859375,
      "activations/layer11_attention_weight_max": 34.15454864501953,
      "activations/layer11_attention_weight_min": -34.963600158691406,
      "activations/layer12_attention_weight_max": 27.24815559387207,
      "activations/layer12_attention_weight_min": -26.889942169189453,
      "activations/layer13_attention_weight_max": 39.976318359375,
      "activations/layer13_attention_weight_min": -36.08001708984375,
      "activations/layer14_attention_weight_max": 43.46125411987305,
      "activations/layer14_attention_weight_min": -37.92194366455078,
      "activations/layer15_attention_weight_max": 37.655948638916016,
      "activations/layer15_attention_weight_min": -35.23765182495117,
      "activations/layer16_attention_weight_max": 29.332311630249023,
      "activations/layer16_attention_weight_min": -29.79149055480957,
      "activations/layer17_attention_weight_max": 30.14618682861328,
      "activations/layer17_attention_weight_min": -25.09958267211914,
      "activations/layer18_attention_weight_max": 31.909738540649414,
      "activations/layer18_attention_weight_min": -23.585073471069336,
      "activations/layer19_attention_weight_max": 31.443796157836914,
      "activations/layer19_attention_weight_min": -30.519067764282227,
      "activations/layer1_attention_weight_max": 16.518720626831055,
      "activations/layer1_attention_weight_min": -15.28159236907959,
      "activations/layer20_attention_weight_max": 28.69503402709961,
      "activations/layer20_attention_weight_min": -24.268104553222656,
      "activations/layer21_attention_weight_max": 29.55953025817871,
      "activations/layer21_attention_weight_min": -25.583969116210938,
      "activations/layer22_attention_weight_max": 40.96038818359375,
      "activations/layer22_attention_weight_min": -29.3564453125,
      "activations/layer23_attention_weight_max": 34.16813278198242,
      "activations/layer23_attention_weight_min": -23.819255828857422,
      "activations/layer2_attention_weight_max": 31.538877487182617,
      "activations/layer2_attention_weight_min": -31.136762619018555,
      "activations/layer3_attention_weight_max": 92.27578735351562,
      "activations/layer3_attention_weight_min": -92.59900665283203,
      "activations/layer4_attention_weight_max": 108.59838104248047,
      "activations/layer4_attention_weight_min": -112.85457611083984,
      "activations/layer5_attention_weight_max": 49.15298080444336,
      "activations/layer5_attention_weight_min": -60.0780029296875,
      "activations/layer6_attention_weight_max": 43.43339920043945,
      "activations/layer6_attention_weight_min": -45.778507232666016,
      "activations/layer7_attention_weight_max": 90.1669692993164,
      "activations/layer7_attention_weight_min": -92.8253402709961,
      "activations/layer8_attention_weight_max": 40.99599838256836,
      "activations/layer8_attention_weight_min": -42.488468170166016,
      "activations/layer9_attention_weight_max": 33.45005416870117,
      "activations/layer9_attention_weight_min": -33.345027923583984,
      "epoch": 14.21,
      "learning_rate": 5.893939393939393e-05,
      "loss": 2.7586,
      "step": 244600
    },
    {
      "activations/layer0_attention_weight_max": 15.518097877502441,
      "activations/layer0_attention_weight_min": -12.856340408325195,
      "activations/layer10_attention_weight_max": 32.62222671508789,
      "activations/layer10_attention_weight_min": -34.618690490722656,
      "activations/layer11_attention_weight_max": 31.355276107788086,
      "activations/layer11_attention_weight_min": -31.70050048828125,
      "activations/layer12_attention_weight_max": 26.08201026916504,
      "activations/layer12_attention_weight_min": -26.064308166503906,
      "activations/layer13_attention_weight_max": 40.213592529296875,
      "activations/layer13_attention_weight_min": -34.24134826660156,
      "activations/layer14_attention_weight_max": 42.18568420410156,
      "activations/layer14_attention_weight_min": -35.74836349487305,
      "activations/layer15_attention_weight_max": 40.157745361328125,
      "activations/layer15_attention_weight_min": -35.185813903808594,
      "activations/layer16_attention_weight_max": 30.112424850463867,
      "activations/layer16_attention_weight_min": -30.174083709716797,
      "activations/layer17_attention_weight_max": 33.92735290527344,
      "activations/layer17_attention_weight_min": -26.553457260131836,
      "activations/layer18_attention_weight_max": 32.094398498535156,
      "activations/layer18_attention_weight_min": -26.782756805419922,
      "activations/layer19_attention_weight_max": 33.473304748535156,
      "activations/layer19_attention_weight_min": -31.615921020507812,
      "activations/layer1_attention_weight_max": 15.356375694274902,
      "activations/layer1_attention_weight_min": -13.387289047241211,
      "activations/layer20_attention_weight_max": 31.845502853393555,
      "activations/layer20_attention_weight_min": -26.683303833007812,
      "activations/layer21_attention_weight_max": 28.524648666381836,
      "activations/layer21_attention_weight_min": -26.74514389038086,
      "activations/layer22_attention_weight_max": 42.98557662963867,
      "activations/layer22_attention_weight_min": -33.951255798339844,
      "activations/layer23_attention_weight_max": 34.470855712890625,
      "activations/layer23_attention_weight_min": -25.233917236328125,
      "activations/layer2_attention_weight_max": 33.58454513549805,
      "activations/layer2_attention_weight_min": -31.136920928955078,
      "activations/layer3_attention_weight_max": 91.37024688720703,
      "activations/layer3_attention_weight_min": -90.57057189941406,
      "activations/layer4_attention_weight_max": 109.5794448852539,
      "activations/layer4_attention_weight_min": -107.85652923583984,
      "activations/layer5_attention_weight_max": 54.2768669128418,
      "activations/layer5_attention_weight_min": -63.57765197753906,
      "activations/layer6_attention_weight_max": 41.91950988769531,
      "activations/layer6_attention_weight_min": -47.20113754272461,
      "activations/layer7_attention_weight_max": 91.02466583251953,
      "activations/layer7_attention_weight_min": -95.36212158203125,
      "activations/layer8_attention_weight_max": 41.2438850402832,
      "activations/layer8_attention_weight_min": -42.17580795288086,
      "activations/layer9_attention_weight_max": 32.09599304199219,
      "activations/layer9_attention_weight_min": -32.726314544677734,
      "epoch": 14.22,
      "learning_rate": 5.8920833333333333e-05,
      "loss": 2.7527,
      "step": 244650
    },
    {
      "activations/layer0_attention_weight_max": 16.207387924194336,
      "activations/layer0_attention_weight_min": -12.3013916015625,
      "activations/layer10_attention_weight_max": 31.967424392700195,
      "activations/layer10_attention_weight_min": -31.24909019470215,
      "activations/layer11_attention_weight_max": 32.51317596435547,
      "activations/layer11_attention_weight_min": -31.168968200683594,
      "activations/layer12_attention_weight_max": 25.076343536376953,
      "activations/layer12_attention_weight_min": -24.75384521484375,
      "activations/layer13_attention_weight_max": 37.77989959716797,
      "activations/layer13_attention_weight_min": -32.72296905517578,
      "activations/layer14_attention_weight_max": 39.67611312866211,
      "activations/layer14_attention_weight_min": -38.48897933959961,
      "activations/layer15_attention_weight_max": 35.17216491699219,
      "activations/layer15_attention_weight_min": -35.13219451904297,
      "activations/layer16_attention_weight_max": 29.408790588378906,
      "activations/layer16_attention_weight_min": -28.390275955200195,
      "activations/layer17_attention_weight_max": 32.69969177246094,
      "activations/layer17_attention_weight_min": -26.434886932373047,
      "activations/layer18_attention_weight_max": 30.7359676361084,
      "activations/layer18_attention_weight_min": -25.599456787109375,
      "activations/layer19_attention_weight_max": 33.29680252075195,
      "activations/layer19_attention_weight_min": -34.82487487792969,
      "activations/layer1_attention_weight_max": 18.276033401489258,
      "activations/layer1_attention_weight_min": -16.14464569091797,
      "activations/layer20_attention_weight_max": 30.228364944458008,
      "activations/layer20_attention_weight_min": -25.30801010131836,
      "activations/layer21_attention_weight_max": 30.97850227355957,
      "activations/layer21_attention_weight_min": -24.136417388916016,
      "activations/layer22_attention_weight_max": 38.942588806152344,
      "activations/layer22_attention_weight_min": -34.426918029785156,
      "activations/layer23_attention_weight_max": 32.828102111816406,
      "activations/layer23_attention_weight_min": -28.745262145996094,
      "activations/layer2_attention_weight_max": 33.76031494140625,
      "activations/layer2_attention_weight_min": -32.3446044921875,
      "activations/layer3_attention_weight_max": 94.62711334228516,
      "activations/layer3_attention_weight_min": -98.43226623535156,
      "activations/layer4_attention_weight_max": 108.94654846191406,
      "activations/layer4_attention_weight_min": -108.52446746826172,
      "activations/layer5_attention_weight_max": 49.16195297241211,
      "activations/layer5_attention_weight_min": -57.62527084350586,
      "activations/layer6_attention_weight_max": 43.02129364013672,
      "activations/layer6_attention_weight_min": -47.00706100463867,
      "activations/layer7_attention_weight_max": 88.87616729736328,
      "activations/layer7_attention_weight_min": -91.14432525634766,
      "activations/layer8_attention_weight_max": 39.450801849365234,
      "activations/layer8_attention_weight_min": -41.03262710571289,
      "activations/layer9_attention_weight_max": 31.66634178161621,
      "activations/layer9_attention_weight_min": -33.24959945678711,
      "epoch": 14.22,
      "learning_rate": 5.8901893939393935e-05,
      "loss": 2.7389,
      "step": 244700
    },
    {
      "activations/layer0_attention_weight_max": 16.069477081298828,
      "activations/layer0_attention_weight_min": -13.173460960388184,
      "activations/layer10_attention_weight_max": 32.15802001953125,
      "activations/layer10_attention_weight_min": -31.9434814453125,
      "activations/layer11_attention_weight_max": 31.53515625,
      "activations/layer11_attention_weight_min": -33.744049072265625,
      "activations/layer12_attention_weight_max": 24.811649322509766,
      "activations/layer12_attention_weight_min": -24.531776428222656,
      "activations/layer13_attention_weight_max": 37.140342712402344,
      "activations/layer13_attention_weight_min": -34.129844665527344,
      "activations/layer14_attention_weight_max": 45.46051025390625,
      "activations/layer14_attention_weight_min": -36.989227294921875,
      "activations/layer15_attention_weight_max": 37.406166076660156,
      "activations/layer15_attention_weight_min": -34.88679504394531,
      "activations/layer16_attention_weight_max": 31.464757919311523,
      "activations/layer16_attention_weight_min": -27.635953903198242,
      "activations/layer17_attention_weight_max": 30.094324111938477,
      "activations/layer17_attention_weight_min": -28.423980712890625,
      "activations/layer18_attention_weight_max": 29.560813903808594,
      "activations/layer18_attention_weight_min": -26.407527923583984,
      "activations/layer19_attention_weight_max": 32.09199905395508,
      "activations/layer19_attention_weight_min": -33.667991638183594,
      "activations/layer1_attention_weight_max": 16.201770782470703,
      "activations/layer1_attention_weight_min": -16.74555206298828,
      "activations/layer20_attention_weight_max": 28.092212677001953,
      "activations/layer20_attention_weight_min": -26.7318172454834,
      "activations/layer21_attention_weight_max": 26.626840591430664,
      "activations/layer21_attention_weight_min": -23.833572387695312,
      "activations/layer22_attention_weight_max": 44.05476379394531,
      "activations/layer22_attention_weight_min": -28.771747589111328,
      "activations/layer23_attention_weight_max": 33.2226676940918,
      "activations/layer23_attention_weight_min": -23.784923553466797,
      "activations/layer2_attention_weight_max": 32.066070556640625,
      "activations/layer2_attention_weight_min": -32.405487060546875,
      "activations/layer3_attention_weight_max": 92.80516052246094,
      "activations/layer3_attention_weight_min": -99.57295989990234,
      "activations/layer4_attention_weight_max": 103.311279296875,
      "activations/layer4_attention_weight_min": -105.81146240234375,
      "activations/layer5_attention_weight_max": 52.86057662963867,
      "activations/layer5_attention_weight_min": -62.161277770996094,
      "activations/layer6_attention_weight_max": 42.16259002685547,
      "activations/layer6_attention_weight_min": -44.607967376708984,
      "activations/layer7_attention_weight_max": 86.81135559082031,
      "activations/layer7_attention_weight_min": -89.53829193115234,
      "activations/layer8_attention_weight_max": 39.02045440673828,
      "activations/layer8_attention_weight_min": -41.64088821411133,
      "activations/layer9_attention_weight_max": 29.5004825592041,
      "activations/layer9_attention_weight_min": -31.82723045349121,
      "epoch": 14.22,
      "learning_rate": 5.8882954545454536e-05,
      "loss": 2.7519,
      "step": 244750
    },
    {
      "activations/layer0_attention_weight_max": 15.058050155639648,
      "activations/layer0_attention_weight_min": -11.515911102294922,
      "activations/layer10_attention_weight_max": 31.371688842773438,
      "activations/layer10_attention_weight_min": -29.690471649169922,
      "activations/layer11_attention_weight_max": 32.17827606201172,
      "activations/layer11_attention_weight_min": -29.395109176635742,
      "activations/layer12_attention_weight_max": 25.35395622253418,
      "activations/layer12_attention_weight_min": -27.361143112182617,
      "activations/layer13_attention_weight_max": 37.766658782958984,
      "activations/layer13_attention_weight_min": -33.60471725463867,
      "activations/layer14_attention_weight_max": 39.636573791503906,
      "activations/layer14_attention_weight_min": -36.08061981201172,
      "activations/layer15_attention_weight_max": 35.26112365722656,
      "activations/layer15_attention_weight_min": -32.716548919677734,
      "activations/layer16_attention_weight_max": 30.022916793823242,
      "activations/layer16_attention_weight_min": -27.20328140258789,
      "activations/layer17_attention_weight_max": 29.94219970703125,
      "activations/layer17_attention_weight_min": -25.421079635620117,
      "activations/layer18_attention_weight_max": 29.80091667175293,
      "activations/layer18_attention_weight_min": -24.148141860961914,
      "activations/layer19_attention_weight_max": 31.832077026367188,
      "activations/layer19_attention_weight_min": -31.942535400390625,
      "activations/layer1_attention_weight_max": 16.18988609313965,
      "activations/layer1_attention_weight_min": -14.183837890625,
      "activations/layer20_attention_weight_max": 28.140544891357422,
      "activations/layer20_attention_weight_min": -23.72992515563965,
      "activations/layer21_attention_weight_max": 28.133249282836914,
      "activations/layer21_attention_weight_min": -24.520244598388672,
      "activations/layer22_attention_weight_max": 38.4293098449707,
      "activations/layer22_attention_weight_min": -31.722530364990234,
      "activations/layer23_attention_weight_max": 31.302501678466797,
      "activations/layer23_attention_weight_min": -25.51593017578125,
      "activations/layer2_attention_weight_max": 33.73331832885742,
      "activations/layer2_attention_weight_min": -33.59019088745117,
      "activations/layer3_attention_weight_max": 96.97087860107422,
      "activations/layer3_attention_weight_min": -95.48623657226562,
      "activations/layer4_attention_weight_max": 107.1724853515625,
      "activations/layer4_attention_weight_min": -104.94901275634766,
      "activations/layer5_attention_weight_max": 54.662010192871094,
      "activations/layer5_attention_weight_min": -62.200035095214844,
      "activations/layer6_attention_weight_max": 42.775081634521484,
      "activations/layer6_attention_weight_min": -45.0909309387207,
      "activations/layer7_attention_weight_max": 96.84367370605469,
      "activations/layer7_attention_weight_min": -86.30873107910156,
      "activations/layer8_attention_weight_max": 41.60943603515625,
      "activations/layer8_attention_weight_min": -38.76347732543945,
      "activations/layer9_attention_weight_max": 30.401647567749023,
      "activations/layer9_attention_weight_min": -30.259174346923828,
      "epoch": 14.22,
      "learning_rate": 5.8864015151515145e-05,
      "loss": 2.7444,
      "step": 244800
    },
    {
      "activations/layer0_attention_weight_max": 16.199281692504883,
      "activations/layer0_attention_weight_min": -12.436627388000488,
      "activations/layer10_attention_weight_max": 31.17789077758789,
      "activations/layer10_attention_weight_min": -31.033267974853516,
      "activations/layer11_attention_weight_max": 30.699071884155273,
      "activations/layer11_attention_weight_min": -31.6815242767334,
      "activations/layer12_attention_weight_max": 23.861127853393555,
      "activations/layer12_attention_weight_min": -25.237199783325195,
      "activations/layer13_attention_weight_max": 40.64799499511719,
      "activations/layer13_attention_weight_min": -34.2241325378418,
      "activations/layer14_attention_weight_max": 38.71355056762695,
      "activations/layer14_attention_weight_min": -36.50039291381836,
      "activations/layer15_attention_weight_max": 35.2291145324707,
      "activations/layer15_attention_weight_min": -33.8213996887207,
      "activations/layer16_attention_weight_max": 26.57183837890625,
      "activations/layer16_attention_weight_min": -27.70496368408203,
      "activations/layer17_attention_weight_max": 29.04962158203125,
      "activations/layer17_attention_weight_min": -26.130765914916992,
      "activations/layer18_attention_weight_max": 33.0992546081543,
      "activations/layer18_attention_weight_min": -25.790882110595703,
      "activations/layer19_attention_weight_max": 30.622512817382812,
      "activations/layer19_attention_weight_min": -30.32382583618164,
      "activations/layer1_attention_weight_max": 16.434579849243164,
      "activations/layer1_attention_weight_min": -15.581409454345703,
      "activations/layer20_attention_weight_max": 28.8023738861084,
      "activations/layer20_attention_weight_min": -25.216978073120117,
      "activations/layer21_attention_weight_max": 28.483627319335938,
      "activations/layer21_attention_weight_min": -24.395463943481445,
      "activations/layer22_attention_weight_max": 40.63722229003906,
      "activations/layer22_attention_weight_min": -31.12234878540039,
      "activations/layer23_attention_weight_max": 32.80678939819336,
      "activations/layer23_attention_weight_min": -25.316038131713867,
      "activations/layer2_attention_weight_max": 30.328777313232422,
      "activations/layer2_attention_weight_min": -30.634307861328125,
      "activations/layer3_attention_weight_max": 90.27620697021484,
      "activations/layer3_attention_weight_min": -94.32923126220703,
      "activations/layer4_attention_weight_max": 102.51360321044922,
      "activations/layer4_attention_weight_min": -104.59764862060547,
      "activations/layer5_attention_weight_max": 52.70232391357422,
      "activations/layer5_attention_weight_min": -59.24291229248047,
      "activations/layer6_attention_weight_max": 42.710960388183594,
      "activations/layer6_attention_weight_min": -47.90443420410156,
      "activations/layer7_attention_weight_max": 90.5162582397461,
      "activations/layer7_attention_weight_min": -89.82723999023438,
      "activations/layer8_attention_weight_max": 38.90720748901367,
      "activations/layer8_attention_weight_min": -39.680511474609375,
      "activations/layer9_attention_weight_max": 30.393991470336914,
      "activations/layer9_attention_weight_min": -31.078357696533203,
      "epoch": 14.23,
      "learning_rate": 5.884507575757575e-05,
      "loss": 2.7503,
      "step": 244850
    },
    {
      "activations/layer0_attention_weight_max": 15.688090324401855,
      "activations/layer0_attention_weight_min": -13.727513313293457,
      "activations/layer10_attention_weight_max": 34.87706756591797,
      "activations/layer10_attention_weight_min": -32.740814208984375,
      "activations/layer11_attention_weight_max": 35.96233367919922,
      "activations/layer11_attention_weight_min": -34.023582458496094,
      "activations/layer12_attention_weight_max": 26.756567001342773,
      "activations/layer12_attention_weight_min": -24.564115524291992,
      "activations/layer13_attention_weight_max": 39.1431770324707,
      "activations/layer13_attention_weight_min": -33.324920654296875,
      "activations/layer14_attention_weight_max": 40.26863098144531,
      "activations/layer14_attention_weight_min": -34.95905685424805,
      "activations/layer15_attention_weight_max": 38.04447937011719,
      "activations/layer15_attention_weight_min": -30.878799438476562,
      "activations/layer16_attention_weight_max": 29.19507598876953,
      "activations/layer16_attention_weight_min": -27.76679801940918,
      "activations/layer17_attention_weight_max": 30.862735748291016,
      "activations/layer17_attention_weight_min": -27.450485229492188,
      "activations/layer18_attention_weight_max": 30.86456298828125,
      "activations/layer18_attention_weight_min": -23.807191848754883,
      "activations/layer19_attention_weight_max": 32.30883026123047,
      "activations/layer19_attention_weight_min": -31.948083877563477,
      "activations/layer1_attention_weight_max": 16.16210174560547,
      "activations/layer1_attention_weight_min": -13.78333854675293,
      "activations/layer20_attention_weight_max": 26.859806060791016,
      "activations/layer20_attention_weight_min": -25.108003616333008,
      "activations/layer21_attention_weight_max": 27.233884811401367,
      "activations/layer21_attention_weight_min": -25.401845932006836,
      "activations/layer22_attention_weight_max": 43.583438873291016,
      "activations/layer22_attention_weight_min": -31.63897705078125,
      "activations/layer23_attention_weight_max": 31.280975341796875,
      "activations/layer23_attention_weight_min": -25.28347396850586,
      "activations/layer2_attention_weight_max": 35.61528396606445,
      "activations/layer2_attention_weight_min": -32.23271942138672,
      "activations/layer3_attention_weight_max": 99.486328125,
      "activations/layer3_attention_weight_min": -98.32689666748047,
      "activations/layer4_attention_weight_max": 113.48114013671875,
      "activations/layer4_attention_weight_min": -114.58247375488281,
      "activations/layer5_attention_weight_max": 53.22526168823242,
      "activations/layer5_attention_weight_min": -64.34941864013672,
      "activations/layer6_attention_weight_max": 48.613609313964844,
      "activations/layer6_attention_weight_min": -48.59065246582031,
      "activations/layer7_attention_weight_max": 95.11188507080078,
      "activations/layer7_attention_weight_min": -99.94559478759766,
      "activations/layer8_attention_weight_max": 43.06463623046875,
      "activations/layer8_attention_weight_min": -43.53266143798828,
      "activations/layer9_attention_weight_max": 34.978939056396484,
      "activations/layer9_attention_weight_min": -33.685604095458984,
      "epoch": 14.23,
      "learning_rate": 5.882613636363636e-05,
      "loss": 2.767,
      "step": 244900
    },
    {
      "activations/layer0_attention_weight_max": 16.99211311340332,
      "activations/layer0_attention_weight_min": -13.105426788330078,
      "activations/layer10_attention_weight_max": 31.508548736572266,
      "activations/layer10_attention_weight_min": -30.676267623901367,
      "activations/layer11_attention_weight_max": 31.24555206298828,
      "activations/layer11_attention_weight_min": -33.06970977783203,
      "activations/layer12_attention_weight_max": 24.39545440673828,
      "activations/layer12_attention_weight_min": -26.304941177368164,
      "activations/layer13_attention_weight_max": 38.58662033081055,
      "activations/layer13_attention_weight_min": -34.63203430175781,
      "activations/layer14_attention_weight_max": 39.380271911621094,
      "activations/layer14_attention_weight_min": -41.24594497680664,
      "activations/layer15_attention_weight_max": 36.90869903564453,
      "activations/layer15_attention_weight_min": -35.304569244384766,
      "activations/layer16_attention_weight_max": 28.753759384155273,
      "activations/layer16_attention_weight_min": -28.1278018951416,
      "activations/layer17_attention_weight_max": 32.56037521362305,
      "activations/layer17_attention_weight_min": -26.03779411315918,
      "activations/layer18_attention_weight_max": 32.08572006225586,
      "activations/layer18_attention_weight_min": -25.73057746887207,
      "activations/layer19_attention_weight_max": 31.556142807006836,
      "activations/layer19_attention_weight_min": -31.17258644104004,
      "activations/layer1_attention_weight_max": 16.964996337890625,
      "activations/layer1_attention_weight_min": -15.999093055725098,
      "activations/layer20_attention_weight_max": 27.907608032226562,
      "activations/layer20_attention_weight_min": -24.941301345825195,
      "activations/layer21_attention_weight_max": 29.348539352416992,
      "activations/layer21_attention_weight_min": -24.087718963623047,
      "activations/layer22_attention_weight_max": 38.67569351196289,
      "activations/layer22_attention_weight_min": -30.400978088378906,
      "activations/layer23_attention_weight_max": 31.922073364257812,
      "activations/layer23_attention_weight_min": -25.157621383666992,
      "activations/layer2_attention_weight_max": 32.28695297241211,
      "activations/layer2_attention_weight_min": -32.05094909667969,
      "activations/layer3_attention_weight_max": 96.95232391357422,
      "activations/layer3_attention_weight_min": -95.7021484375,
      "activations/layer4_attention_weight_max": 107.18431854248047,
      "activations/layer4_attention_weight_min": -108.7637939453125,
      "activations/layer5_attention_weight_max": 52.19194030761719,
      "activations/layer5_attention_weight_min": -62.58349609375,
      "activations/layer6_attention_weight_max": 42.15607452392578,
      "activations/layer6_attention_weight_min": -44.154666900634766,
      "activations/layer7_attention_weight_max": 88.35092163085938,
      "activations/layer7_attention_weight_min": -87.81140899658203,
      "activations/layer8_attention_weight_max": 39.995323181152344,
      "activations/layer8_attention_weight_min": -40.41292190551758,
      "activations/layer9_attention_weight_max": 31.84061050415039,
      "activations/layer9_attention_weight_min": -32.0067253112793,
      "epoch": 14.23,
      "learning_rate": 5.880719696969696e-05,
      "loss": 2.7484,
      "step": 244950
    },
    {
      "activations/layer0_attention_weight_max": 14.707409858703613,
      "activations/layer0_attention_weight_min": -12.367791175842285,
      "activations/layer10_attention_weight_max": 31.67101287841797,
      "activations/layer10_attention_weight_min": -32.98358154296875,
      "activations/layer11_attention_weight_max": 30.648822784423828,
      "activations/layer11_attention_weight_min": -32.79121398925781,
      "activations/layer12_attention_weight_max": 24.023399353027344,
      "activations/layer12_attention_weight_min": -26.332136154174805,
      "activations/layer13_attention_weight_max": 38.618995666503906,
      "activations/layer13_attention_weight_min": -33.618690490722656,
      "activations/layer14_attention_weight_max": 41.83304977416992,
      "activations/layer14_attention_weight_min": -36.1333122253418,
      "activations/layer15_attention_weight_max": 39.66240692138672,
      "activations/layer15_attention_weight_min": -33.78742218017578,
      "activations/layer16_attention_weight_max": 31.54539680480957,
      "activations/layer16_attention_weight_min": -27.70789337158203,
      "activations/layer17_attention_weight_max": 30.28496742248535,
      "activations/layer17_attention_weight_min": -25.720754623413086,
      "activations/layer18_attention_weight_max": 34.172515869140625,
      "activations/layer18_attention_weight_min": -24.23690414428711,
      "activations/layer19_attention_weight_max": 37.15303421020508,
      "activations/layer19_attention_weight_min": -33.188480377197266,
      "activations/layer1_attention_weight_max": 16.030960083007812,
      "activations/layer1_attention_weight_min": -15.906054496765137,
      "activations/layer20_attention_weight_max": 31.647533416748047,
      "activations/layer20_attention_weight_min": -27.18573570251465,
      "activations/layer21_attention_weight_max": 30.09518051147461,
      "activations/layer21_attention_weight_min": -25.502666473388672,
      "activations/layer22_attention_weight_max": 41.7758674621582,
      "activations/layer22_attention_weight_min": -32.70256423950195,
      "activations/layer23_attention_weight_max": 33.22913360595703,
      "activations/layer23_attention_weight_min": -27.531103134155273,
      "activations/layer2_attention_weight_max": 34.35001754760742,
      "activations/layer2_attention_weight_min": -34.4034538269043,
      "activations/layer3_attention_weight_max": 96.06871032714844,
      "activations/layer3_attention_weight_min": -101.0794906616211,
      "activations/layer4_attention_weight_max": 110.33880615234375,
      "activations/layer4_attention_weight_min": -109.97557830810547,
      "activations/layer5_attention_weight_max": 50.427154541015625,
      "activations/layer5_attention_weight_min": -61.4299430847168,
      "activations/layer6_attention_weight_max": 43.65849685668945,
      "activations/layer6_attention_weight_min": -45.04191970825195,
      "activations/layer7_attention_weight_max": 87.489501953125,
      "activations/layer7_attention_weight_min": -89.84500885009766,
      "activations/layer8_attention_weight_max": 38.17348098754883,
      "activations/layer8_attention_weight_min": -40.09086608886719,
      "activations/layer9_attention_weight_max": 31.07339859008789,
      "activations/layer9_attention_weight_min": -33.34371566772461,
      "epoch": 14.24,
      "learning_rate": 5.878825757575758e-05,
      "loss": 2.7597,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.5001,
      "eval_samples_per_second": 505.169,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.5001,
      "eval_openwebtext_samples_per_second": 505.169,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9967,
      "eval_wikitext_samples_per_second": 228.376,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_lambada_loss": 2.431640625,
      "eval_lambada_ppl": 11.3775330413472,
      "eval_lambada_runtime": 9.5741,
      "eval_lambada_samples_per_second": 508.558,
      "step": 245000
    },
    {
      "activations/layer0_attention_weight_max": 15.541707038879395,
      "activations/layer0_attention_weight_min": -12.078433990478516,
      "activations/layer10_attention_weight_max": 36.17609405517578,
      "activations/layer10_attention_weight_min": -35.68619155883789,
      "activations/layer11_attention_weight_max": 36.61777114868164,
      "activations/layer11_attention_weight_min": -35.34693908691406,
      "activations/layer12_attention_weight_max": 27.90326499938965,
      "activations/layer12_attention_weight_min": -27.059558868408203,
      "activations/layer13_attention_weight_max": 43.700096130371094,
      "activations/layer13_attention_weight_min": -36.159420013427734,
      "activations/layer14_attention_weight_max": 48.90877914428711,
      "activations/layer14_attention_weight_min": -37.416873931884766,
      "activations/layer15_attention_weight_max": 40.333377838134766,
      "activations/layer15_attention_weight_min": -33.89061737060547,
      "activations/layer16_attention_weight_max": 31.849376678466797,
      "activations/layer16_attention_weight_min": -28.431726455688477,
      "activations/layer17_attention_weight_max": 31.22258186340332,
      "activations/layer17_attention_weight_min": -26.93047523498535,
      "activations/layer18_attention_weight_max": 32.63438034057617,
      "activations/layer18_attention_weight_min": -25.182973861694336,
      "activations/layer19_attention_weight_max": 36.10660934448242,
      "activations/layer19_attention_weight_min": -30.937223434448242,
      "activations/layer1_attention_weight_max": 16.367279052734375,
      "activations/layer1_attention_weight_min": -15.115205764770508,
      "activations/layer20_attention_weight_max": 28.069622039794922,
      "activations/layer20_attention_weight_min": -24.700149536132812,
      "activations/layer21_attention_weight_max": 27.28304672241211,
      "activations/layer21_attention_weight_min": -25.74163055419922,
      "activations/layer22_attention_weight_max": 41.25225830078125,
      "activations/layer22_attention_weight_min": -30.527551651000977,
      "activations/layer23_attention_weight_max": 34.74081802368164,
      "activations/layer23_attention_weight_min": -26.567649841308594,
      "activations/layer2_attention_weight_max": 35.370948791503906,
      "activations/layer2_attention_weight_min": -33.1619758605957,
      "activations/layer3_attention_weight_max": 95.73516845703125,
      "activations/layer3_attention_weight_min": -97.20303344726562,
      "activations/layer4_attention_weight_max": 112.87306213378906,
      "activations/layer4_attention_weight_min": -112.24727630615234,
      "activations/layer5_attention_weight_max": 55.92236328125,
      "activations/layer5_attention_weight_min": -62.82362365722656,
      "activations/layer6_attention_weight_max": 45.148555755615234,
      "activations/layer6_attention_weight_min": -48.01237106323242,
      "activations/layer7_attention_weight_max": 99.53202819824219,
      "activations/layer7_attention_weight_min": -101.69830322265625,
      "activations/layer8_attention_weight_max": 44.766597747802734,
      "activations/layer8_attention_weight_min": -44.22314453125,
      "activations/layer9_attention_weight_max": 34.70016098022461,
      "activations/layer9_attention_weight_min": -34.73142623901367,
      "epoch": 14.24,
      "learning_rate": 5.876931818181818e-05,
      "loss": 2.7603,
      "step": 245050
    },
    {
      "activations/layer0_attention_weight_max": 15.118453979492188,
      "activations/layer0_attention_weight_min": -13.75912094116211,
      "activations/layer10_attention_weight_max": 40.91533279418945,
      "activations/layer10_attention_weight_min": -38.458656311035156,
      "activations/layer11_attention_weight_max": 42.6348991394043,
      "activations/layer11_attention_weight_min": -37.149810791015625,
      "activations/layer12_attention_weight_max": 30.385059356689453,
      "activations/layer12_attention_weight_min": -25.623273849487305,
      "activations/layer13_attention_weight_max": 41.03098678588867,
      "activations/layer13_attention_weight_min": -34.3076171875,
      "activations/layer14_attention_weight_max": 48.21905517578125,
      "activations/layer14_attention_weight_min": -40.0156364440918,
      "activations/layer15_attention_weight_max": 43.62887954711914,
      "activations/layer15_attention_weight_min": -35.885013580322266,
      "activations/layer16_attention_weight_max": 35.872657775878906,
      "activations/layer16_attention_weight_min": -26.111051559448242,
      "activations/layer17_attention_weight_max": 32.86286926269531,
      "activations/layer17_attention_weight_min": -26.346277236938477,
      "activations/layer18_attention_weight_max": 31.78387451171875,
      "activations/layer18_attention_weight_min": -25.764019012451172,
      "activations/layer19_attention_weight_max": 35.4145622253418,
      "activations/layer19_attention_weight_min": -30.090957641601562,
      "activations/layer1_attention_weight_max": 17.542308807373047,
      "activations/layer1_attention_weight_min": -15.752193450927734,
      "activations/layer20_attention_weight_max": 30.465179443359375,
      "activations/layer20_attention_weight_min": -23.675283432006836,
      "activations/layer21_attention_weight_max": 29.409337997436523,
      "activations/layer21_attention_weight_min": -22.685531616210938,
      "activations/layer22_attention_weight_max": 45.120914459228516,
      "activations/layer22_attention_weight_min": -28.223588943481445,
      "activations/layer23_attention_weight_max": 30.51571273803711,
      "activations/layer23_attention_weight_min": -24.980915069580078,
      "activations/layer2_attention_weight_max": 36.74599838256836,
      "activations/layer2_attention_weight_min": -37.138099670410156,
      "activations/layer3_attention_weight_max": 106.69819641113281,
      "activations/layer3_attention_weight_min": -112.35720825195312,
      "activations/layer4_attention_weight_max": 116.58646392822266,
      "activations/layer4_attention_weight_min": -114.48213195800781,
      "activations/layer5_attention_weight_max": 56.11361312866211,
      "activations/layer5_attention_weight_min": -63.984153747558594,
      "activations/layer6_attention_weight_max": 45.585350036621094,
      "activations/layer6_attention_weight_min": -45.441810607910156,
      "activations/layer7_attention_weight_max": 102.1144790649414,
      "activations/layer7_attention_weight_min": -96.84224700927734,
      "activations/layer8_attention_weight_max": 48.26507568359375,
      "activations/layer8_attention_weight_min": -46.4066047668457,
      "activations/layer9_attention_weight_max": 38.58089065551758,
      "activations/layer9_attention_weight_min": -37.77219772338867,
      "epoch": 14.24,
      "learning_rate": 5.875037878787878e-05,
      "loss": 2.7433,
      "step": 245100
    },
    {
      "activations/layer0_attention_weight_max": 15.308013916015625,
      "activations/layer0_attention_weight_min": -12.202005386352539,
      "activations/layer10_attention_weight_max": 36.88096618652344,
      "activations/layer10_attention_weight_min": -34.15674591064453,
      "activations/layer11_attention_weight_max": 37.74193572998047,
      "activations/layer11_attention_weight_min": -34.25987243652344,
      "activations/layer12_attention_weight_max": 27.539051055908203,
      "activations/layer12_attention_weight_min": -26.445144653320312,
      "activations/layer13_attention_weight_max": 40.310211181640625,
      "activations/layer13_attention_weight_min": -36.44547653198242,
      "activations/layer14_attention_weight_max": 44.15241622924805,
      "activations/layer14_attention_weight_min": -36.39204406738281,
      "activations/layer15_attention_weight_max": 37.7625846862793,
      "activations/layer15_attention_weight_min": -32.55336380004883,
      "activations/layer16_attention_weight_max": 34.117252349853516,
      "activations/layer16_attention_weight_min": -27.18880844116211,
      "activations/layer17_attention_weight_max": 28.91130256652832,
      "activations/layer17_attention_weight_min": -27.062471389770508,
      "activations/layer18_attention_weight_max": 31.456090927124023,
      "activations/layer18_attention_weight_min": -22.625186920166016,
      "activations/layer19_attention_weight_max": 35.02054977416992,
      "activations/layer19_attention_weight_min": -30.80264663696289,
      "activations/layer1_attention_weight_max": 16.88454818725586,
      "activations/layer1_attention_weight_min": -13.200887680053711,
      "activations/layer20_attention_weight_max": 29.614368438720703,
      "activations/layer20_attention_weight_min": -24.64898109436035,
      "activations/layer21_attention_weight_max": 31.135692596435547,
      "activations/layer21_attention_weight_min": -23.556699752807617,
      "activations/layer22_attention_weight_max": 39.345157623291016,
      "activations/layer22_attention_weight_min": -29.594533920288086,
      "activations/layer23_attention_weight_max": 34.537227630615234,
      "activations/layer23_attention_weight_min": -24.149803161621094,
      "activations/layer2_attention_weight_max": 35.82494354248047,
      "activations/layer2_attention_weight_min": -35.23332214355469,
      "activations/layer3_attention_weight_max": 103.55036926269531,
      "activations/layer3_attention_weight_min": -109.20690155029297,
      "activations/layer4_attention_weight_max": 113.11030578613281,
      "activations/layer4_attention_weight_min": -112.0550308227539,
      "activations/layer5_attention_weight_max": 53.846309661865234,
      "activations/layer5_attention_weight_min": -64.30266571044922,
      "activations/layer6_attention_weight_max": 43.80073547363281,
      "activations/layer6_attention_weight_min": -45.05851745605469,
      "activations/layer7_attention_weight_max": 95.99579620361328,
      "activations/layer7_attention_weight_min": -94.99569702148438,
      "activations/layer8_attention_weight_max": 41.47194290161133,
      "activations/layer8_attention_weight_min": -42.34474563598633,
      "activations/layer9_attention_weight_max": 33.13938903808594,
      "activations/layer9_attention_weight_min": -33.28083038330078,
      "epoch": 14.24,
      "learning_rate": 5.873143939393939e-05,
      "loss": 2.7516,
      "step": 245150
    },
    {
      "activations/layer0_attention_weight_max": 15.911073684692383,
      "activations/layer0_attention_weight_min": -13.166792869567871,
      "activations/layer10_attention_weight_max": 31.38068199157715,
      "activations/layer10_attention_weight_min": -32.209739685058594,
      "activations/layer11_attention_weight_max": 32.52012252807617,
      "activations/layer11_attention_weight_min": -33.02031326293945,
      "activations/layer12_attention_weight_max": 26.283138275146484,
      "activations/layer12_attention_weight_min": -25.793371200561523,
      "activations/layer13_attention_weight_max": 37.407073974609375,
      "activations/layer13_attention_weight_min": -34.80910110473633,
      "activations/layer14_attention_weight_max": 41.43352127075195,
      "activations/layer14_attention_weight_min": -38.22028732299805,
      "activations/layer15_attention_weight_max": 36.18491744995117,
      "activations/layer15_attention_weight_min": -33.22735595703125,
      "activations/layer16_attention_weight_max": 28.355051040649414,
      "activations/layer16_attention_weight_min": -27.759817123413086,
      "activations/layer17_attention_weight_max": 29.787784576416016,
      "activations/layer17_attention_weight_min": -25.60478401184082,
      "activations/layer18_attention_weight_max": 31.689781188964844,
      "activations/layer18_attention_weight_min": -24.257291793823242,
      "activations/layer19_attention_weight_max": 38.046260833740234,
      "activations/layer19_attention_weight_min": -29.149808883666992,
      "activations/layer1_attention_weight_max": 16.040639877319336,
      "activations/layer1_attention_weight_min": -14.510998725891113,
      "activations/layer20_attention_weight_max": 30.378795623779297,
      "activations/layer20_attention_weight_min": -25.9350643157959,
      "activations/layer21_attention_weight_max": 30.548330307006836,
      "activations/layer21_attention_weight_min": -24.60624885559082,
      "activations/layer22_attention_weight_max": 42.758216857910156,
      "activations/layer22_attention_weight_min": -31.35152816772461,
      "activations/layer23_attention_weight_max": 36.091556549072266,
      "activations/layer23_attention_weight_min": -26.678030014038086,
      "activations/layer2_attention_weight_max": 33.42887878417969,
      "activations/layer2_attention_weight_min": -33.595706939697266,
      "activations/layer3_attention_weight_max": 91.29602813720703,
      "activations/layer3_attention_weight_min": -94.04243469238281,
      "activations/layer4_attention_weight_max": 106.0276107788086,
      "activations/layer4_attention_weight_min": -112.6093978881836,
      "activations/layer5_attention_weight_max": 49.369712829589844,
      "activations/layer5_attention_weight_min": -61.732200622558594,
      "activations/layer6_attention_weight_max": 43.69099807739258,
      "activations/layer6_attention_weight_min": -48.54460906982422,
      "activations/layer7_attention_weight_max": 88.19844055175781,
      "activations/layer7_attention_weight_min": -95.81800079345703,
      "activations/layer8_attention_weight_max": 39.5272102355957,
      "activations/layer8_attention_weight_min": -40.93775939941406,
      "activations/layer9_attention_weight_max": 30.98353385925293,
      "activations/layer9_attention_weight_min": -33.18708419799805,
      "epoch": 14.25,
      "learning_rate": 5.87125e-05,
      "loss": 2.7702,
      "step": 245200
    },
    {
      "activations/layer0_attention_weight_max": 15.498261451721191,
      "activations/layer0_attention_weight_min": -12.184608459472656,
      "activations/layer10_attention_weight_max": 28.931982040405273,
      "activations/layer10_attention_weight_min": -31.09788703918457,
      "activations/layer11_attention_weight_max": 30.001937866210938,
      "activations/layer11_attention_weight_min": -29.606552124023438,
      "activations/layer12_attention_weight_max": 24.30575180053711,
      "activations/layer12_attention_weight_min": -24.58679962158203,
      "activations/layer13_attention_weight_max": 35.57554244995117,
      "activations/layer13_attention_weight_min": -31.923492431640625,
      "activations/layer14_attention_weight_max": 42.26173782348633,
      "activations/layer14_attention_weight_min": -37.24274444580078,
      "activations/layer15_attention_weight_max": 35.54352951049805,
      "activations/layer15_attention_weight_min": -35.18053436279297,
      "activations/layer16_attention_weight_max": 30.21843719482422,
      "activations/layer16_attention_weight_min": -26.895442962646484,
      "activations/layer17_attention_weight_max": 31.271387100219727,
      "activations/layer17_attention_weight_min": -25.122102737426758,
      "activations/layer18_attention_weight_max": 31.103017807006836,
      "activations/layer18_attention_weight_min": -25.317745208740234,
      "activations/layer19_attention_weight_max": 38.13270568847656,
      "activations/layer19_attention_weight_min": -31.01473045349121,
      "activations/layer1_attention_weight_max": 15.713781356811523,
      "activations/layer1_attention_weight_min": -13.7108736038208,
      "activations/layer20_attention_weight_max": 33.740352630615234,
      "activations/layer20_attention_weight_min": -25.22557258605957,
      "activations/layer21_attention_weight_max": 34.61798095703125,
      "activations/layer21_attention_weight_min": -25.66606330871582,
      "activations/layer22_attention_weight_max": 50.0866584777832,
      "activations/layer22_attention_weight_min": -31.26632308959961,
      "activations/layer23_attention_weight_max": 39.51663589477539,
      "activations/layer23_attention_weight_min": -27.310123443603516,
      "activations/layer2_attention_weight_max": 29.304677963256836,
      "activations/layer2_attention_weight_min": -29.65736961364746,
      "activations/layer3_attention_weight_max": 90.63907623291016,
      "activations/layer3_attention_weight_min": -90.3873062133789,
      "activations/layer4_attention_weight_max": 104.89852142333984,
      "activations/layer4_attention_weight_min": -105.3197021484375,
      "activations/layer5_attention_weight_max": 52.5589599609375,
      "activations/layer5_attention_weight_min": -62.1783447265625,
      "activations/layer6_attention_weight_max": 42.973365783691406,
      "activations/layer6_attention_weight_min": -45.02089309692383,
      "activations/layer7_attention_weight_max": 90.50216674804688,
      "activations/layer7_attention_weight_min": -90.88442993164062,
      "activations/layer8_attention_weight_max": 40.195030212402344,
      "activations/layer8_attention_weight_min": -39.32687759399414,
      "activations/layer9_attention_weight_max": 28.48799705505371,
      "activations/layer9_attention_weight_min": -31.544713973999023,
      "epoch": 14.25,
      "learning_rate": 5.8693560606060606e-05,
      "loss": 2.7617,
      "step": 245250
    },
    {
      "activations/layer0_attention_weight_max": 14.34244155883789,
      "activations/layer0_attention_weight_min": -13.180632591247559,
      "activations/layer10_attention_weight_max": 31.524930953979492,
      "activations/layer10_attention_weight_min": -33.1281623840332,
      "activations/layer11_attention_weight_max": 31.452503204345703,
      "activations/layer11_attention_weight_min": -31.064109802246094,
      "activations/layer12_attention_weight_max": 25.256473541259766,
      "activations/layer12_attention_weight_min": -24.423892974853516,
      "activations/layer13_attention_weight_max": 38.44988250732422,
      "activations/layer13_attention_weight_min": -33.269222259521484,
      "activations/layer14_attention_weight_max": 44.66046142578125,
      "activations/layer14_attention_weight_min": -34.72795867919922,
      "activations/layer15_attention_weight_max": 36.750221252441406,
      "activations/layer15_attention_weight_min": -32.01591110229492,
      "activations/layer16_attention_weight_max": 33.72188186645508,
      "activations/layer16_attention_weight_min": -27.02958869934082,
      "activations/layer17_attention_weight_max": 30.859025955200195,
      "activations/layer17_attention_weight_min": -27.211156845092773,
      "activations/layer18_attention_weight_max": 32.52166748046875,
      "activations/layer18_attention_weight_min": -24.323129653930664,
      "activations/layer19_attention_weight_max": 32.8424186706543,
      "activations/layer19_attention_weight_min": -30.101022720336914,
      "activations/layer1_attention_weight_max": 18.784975051879883,
      "activations/layer1_attention_weight_min": -15.17862606048584,
      "activations/layer20_attention_weight_max": 26.32427978515625,
      "activations/layer20_attention_weight_min": -24.194719314575195,
      "activations/layer21_attention_weight_max": 28.603811264038086,
      "activations/layer21_attention_weight_min": -23.962127685546875,
      "activations/layer22_attention_weight_max": 45.741233825683594,
      "activations/layer22_attention_weight_min": -28.7097225189209,
      "activations/layer23_attention_weight_max": 34.79736328125,
      "activations/layer23_attention_weight_min": -25.39636993408203,
      "activations/layer2_attention_weight_max": 31.674091339111328,
      "activations/layer2_attention_weight_min": -30.644168853759766,
      "activations/layer3_attention_weight_max": 88.73733520507812,
      "activations/layer3_attention_weight_min": -88.05587005615234,
      "activations/layer4_attention_weight_max": 114.15840911865234,
      "activations/layer4_attention_weight_min": -111.95111083984375,
      "activations/layer5_attention_weight_max": 52.86143493652344,
      "activations/layer5_attention_weight_min": -64.43464660644531,
      "activations/layer6_attention_weight_max": 44.81692886352539,
      "activations/layer6_attention_weight_min": -46.76251983642578,
      "activations/layer7_attention_weight_max": 96.35699462890625,
      "activations/layer7_attention_weight_min": -93.83509063720703,
      "activations/layer8_attention_weight_max": 40.52052688598633,
      "activations/layer8_attention_weight_min": -39.361080169677734,
      "activations/layer9_attention_weight_max": 31.992889404296875,
      "activations/layer9_attention_weight_min": -31.977985382080078,
      "epoch": 14.25,
      "learning_rate": 5.8674999999999996e-05,
      "loss": 2.7544,
      "step": 245300
    },
    {
      "activations/layer0_attention_weight_max": 16.268291473388672,
      "activations/layer0_attention_weight_min": -13.4680814743042,
      "activations/layer10_attention_weight_max": 32.227210998535156,
      "activations/layer10_attention_weight_min": -33.60517501831055,
      "activations/layer11_attention_weight_max": 30.03639793395996,
      "activations/layer11_attention_weight_min": -31.65839195251465,
      "activations/layer12_attention_weight_max": 24.279218673706055,
      "activations/layer12_attention_weight_min": -26.1378116607666,
      "activations/layer13_attention_weight_max": 39.229270935058594,
      "activations/layer13_attention_weight_min": -34.818511962890625,
      "activations/layer14_attention_weight_max": 43.19038009643555,
      "activations/layer14_attention_weight_min": -37.705421447753906,
      "activations/layer15_attention_weight_max": 37.115108489990234,
      "activations/layer15_attention_weight_min": -33.73906326293945,
      "activations/layer16_attention_weight_max": 30.279504776000977,
      "activations/layer16_attention_weight_min": -28.90065574645996,
      "activations/layer17_attention_weight_max": 31.704301834106445,
      "activations/layer17_attention_weight_min": -26.25235939025879,
      "activations/layer18_attention_weight_max": 34.806949615478516,
      "activations/layer18_attention_weight_min": -24.85835075378418,
      "activations/layer19_attention_weight_max": 34.40619659423828,
      "activations/layer19_attention_weight_min": -30.20330810546875,
      "activations/layer1_attention_weight_max": 15.9605712890625,
      "activations/layer1_attention_weight_min": -15.831206321716309,
      "activations/layer20_attention_weight_max": 28.550865173339844,
      "activations/layer20_attention_weight_min": -25.123430252075195,
      "activations/layer21_attention_weight_max": 32.77198028564453,
      "activations/layer21_attention_weight_min": -23.422454833984375,
      "activations/layer22_attention_weight_max": 39.14241027832031,
      "activations/layer22_attention_weight_min": -29.2685604095459,
      "activations/layer23_attention_weight_max": 33.33066177368164,
      "activations/layer23_attention_weight_min": -24.712913513183594,
      "activations/layer2_attention_weight_max": 30.023258209228516,
      "activations/layer2_attention_weight_min": -29.870590209960938,
      "activations/layer3_attention_weight_max": 89.7337417602539,
      "activations/layer3_attention_weight_min": -88.39702606201172,
      "activations/layer4_attention_weight_max": 103.22112274169922,
      "activations/layer4_attention_weight_min": -105.70112609863281,
      "activations/layer5_attention_weight_max": 51.12556457519531,
      "activations/layer5_attention_weight_min": -60.31586837768555,
      "activations/layer6_attention_weight_max": 41.52445983886719,
      "activations/layer6_attention_weight_min": -47.01451110839844,
      "activations/layer7_attention_weight_max": 85.9655532836914,
      "activations/layer7_attention_weight_min": -87.76323699951172,
      "activations/layer8_attention_weight_max": 37.51721954345703,
      "activations/layer8_attention_weight_min": -42.071083068847656,
      "activations/layer9_attention_weight_max": 30.7890625,
      "activations/layer9_attention_weight_min": -34.087364196777344,
      "epoch": 14.26,
      "learning_rate": 5.86560606060606e-05,
      "loss": 2.748,
      "step": 245350
    },
    {
      "activations/layer0_attention_weight_max": 15.526681900024414,
      "activations/layer0_attention_weight_min": -13.383586883544922,
      "activations/layer10_attention_weight_max": 33.275489807128906,
      "activations/layer10_attention_weight_min": -33.617794036865234,
      "activations/layer11_attention_weight_max": 32.250885009765625,
      "activations/layer11_attention_weight_min": -32.582340240478516,
      "activations/layer12_attention_weight_max": 25.188873291015625,
      "activations/layer12_attention_weight_min": -27.468324661254883,
      "activations/layer13_attention_weight_max": 36.784629821777344,
      "activations/layer13_attention_weight_min": -37.015499114990234,
      "activations/layer14_attention_weight_max": 41.7469596862793,
      "activations/layer14_attention_weight_min": -36.804039001464844,
      "activations/layer15_attention_weight_max": 40.75320816040039,
      "activations/layer15_attention_weight_min": -34.535335540771484,
      "activations/layer16_attention_weight_max": 35.185550689697266,
      "activations/layer16_attention_weight_min": -26.244354248046875,
      "activations/layer17_attention_weight_max": 31.32763671875,
      "activations/layer17_attention_weight_min": -24.465822219848633,
      "activations/layer18_attention_weight_max": 29.40261459350586,
      "activations/layer18_attention_weight_min": -23.433788299560547,
      "activations/layer19_attention_weight_max": 35.508392333984375,
      "activations/layer19_attention_weight_min": -31.096670150756836,
      "activations/layer1_attention_weight_max": 17.252527236938477,
      "activations/layer1_attention_weight_min": -15.211512565612793,
      "activations/layer20_attention_weight_max": 34.107913970947266,
      "activations/layer20_attention_weight_min": -25.80055809020996,
      "activations/layer21_attention_weight_max": 30.106155395507812,
      "activations/layer21_attention_weight_min": -24.04277992248535,
      "activations/layer22_attention_weight_max": 42.438438415527344,
      "activations/layer22_attention_weight_min": -30.846792221069336,
      "activations/layer23_attention_weight_max": 32.82119369506836,
      "activations/layer23_attention_weight_min": -25.173818588256836,
      "activations/layer2_attention_weight_max": 31.313575744628906,
      "activations/layer2_attention_weight_min": -32.329933166503906,
      "activations/layer3_attention_weight_max": 93.28549194335938,
      "activations/layer3_attention_weight_min": -100.22138977050781,
      "activations/layer4_attention_weight_max": 110.7430191040039,
      "activations/layer4_attention_weight_min": -111.37638092041016,
      "activations/layer5_attention_weight_max": 51.115325927734375,
      "activations/layer5_attention_weight_min": -63.654170989990234,
      "activations/layer6_attention_weight_max": 43.66603469848633,
      "activations/layer6_attention_weight_min": -47.20222473144531,
      "activations/layer7_attention_weight_max": 91.24890899658203,
      "activations/layer7_attention_weight_min": -103.4264144897461,
      "activations/layer8_attention_weight_max": 43.36617660522461,
      "activations/layer8_attention_weight_min": -44.91518783569336,
      "activations/layer9_attention_weight_max": 31.547727584838867,
      "activations/layer9_attention_weight_min": -34.444278717041016,
      "epoch": 14.26,
      "learning_rate": 5.863712121212121e-05,
      "loss": 2.7567,
      "step": 245400
    },
    {
      "activations/layer0_attention_weight_max": 16.11142349243164,
      "activations/layer0_attention_weight_min": -12.92204475402832,
      "activations/layer10_attention_weight_max": 33.50562286376953,
      "activations/layer10_attention_weight_min": -32.466590881347656,
      "activations/layer11_attention_weight_max": 34.74186706542969,
      "activations/layer11_attention_weight_min": -32.98220443725586,
      "activations/layer12_attention_weight_max": 26.101457595825195,
      "activations/layer12_attention_weight_min": -26.187849044799805,
      "activations/layer13_attention_weight_max": 37.56010055541992,
      "activations/layer13_attention_weight_min": -34.5734977722168,
      "activations/layer14_attention_weight_max": 39.29042434692383,
      "activations/layer14_attention_weight_min": -34.94085693359375,
      "activations/layer15_attention_weight_max": 35.34779357910156,
      "activations/layer15_attention_weight_min": -33.84888458251953,
      "activations/layer16_attention_weight_max": 30.83989715576172,
      "activations/layer16_attention_weight_min": -28.125598907470703,
      "activations/layer17_attention_weight_max": 30.245866775512695,
      "activations/layer17_attention_weight_min": -25.70107078552246,
      "activations/layer18_attention_weight_max": 30.70003890991211,
      "activations/layer18_attention_weight_min": -24.00365447998047,
      "activations/layer19_attention_weight_max": 34.27294921875,
      "activations/layer19_attention_weight_min": -30.986127853393555,
      "activations/layer1_attention_weight_max": 16.49012565612793,
      "activations/layer1_attention_weight_min": -14.740410804748535,
      "activations/layer20_attention_weight_max": 28.94192123413086,
      "activations/layer20_attention_weight_min": -25.448156356811523,
      "activations/layer21_attention_weight_max": 29.008695602416992,
      "activations/layer21_attention_weight_min": -26.214168548583984,
      "activations/layer22_attention_weight_max": 44.07986831665039,
      "activations/layer22_attention_weight_min": -34.63731384277344,
      "activations/layer23_attention_weight_max": 34.26146697998047,
      "activations/layer23_attention_weight_min": -28.452194213867188,
      "activations/layer2_attention_weight_max": 30.981870651245117,
      "activations/layer2_attention_weight_min": -31.745628356933594,
      "activations/layer3_attention_weight_max": 91.33013916015625,
      "activations/layer3_attention_weight_min": -94.63365173339844,
      "activations/layer4_attention_weight_max": 106.48345947265625,
      "activations/layer4_attention_weight_min": -109.24593353271484,
      "activations/layer5_attention_weight_max": 50.405330657958984,
      "activations/layer5_attention_weight_min": -62.33213806152344,
      "activations/layer6_attention_weight_max": 42.9581184387207,
      "activations/layer6_attention_weight_min": -48.41566467285156,
      "activations/layer7_attention_weight_max": 92.02872467041016,
      "activations/layer7_attention_weight_min": -90.3582763671875,
      "activations/layer8_attention_weight_max": 39.65138626098633,
      "activations/layer8_attention_weight_min": -41.183467864990234,
      "activations/layer9_attention_weight_max": 32.723026275634766,
      "activations/layer9_attention_weight_min": -33.48841857910156,
      "epoch": 14.26,
      "learning_rate": 5.8618181818181814e-05,
      "loss": 2.7709,
      "step": 245450
    },
    {
      "activations/layer0_attention_weight_max": 15.71536922454834,
      "activations/layer0_attention_weight_min": -12.547589302062988,
      "activations/layer10_attention_weight_max": 32.9945182800293,
      "activations/layer10_attention_weight_min": -35.67417526245117,
      "activations/layer11_attention_weight_max": 30.59644317626953,
      "activations/layer11_attention_weight_min": -35.89573669433594,
      "activations/layer12_attention_weight_max": 25.597776412963867,
      "activations/layer12_attention_weight_min": -25.470930099487305,
      "activations/layer13_attention_weight_max": 41.575618743896484,
      "activations/layer13_attention_weight_min": -34.260032653808594,
      "activations/layer14_attention_weight_max": 42.07280349731445,
      "activations/layer14_attention_weight_min": -35.969600677490234,
      "activations/layer15_attention_weight_max": 37.16825485229492,
      "activations/layer15_attention_weight_min": -35.640201568603516,
      "activations/layer16_attention_weight_max": 31.612165451049805,
      "activations/layer16_attention_weight_min": -28.373699188232422,
      "activations/layer17_attention_weight_max": 32.55982208251953,
      "activations/layer17_attention_weight_min": -26.854482650756836,
      "activations/layer18_attention_weight_max": 31.435762405395508,
      "activations/layer18_attention_weight_min": -24.18541145324707,
      "activations/layer19_attention_weight_max": 31.429410934448242,
      "activations/layer19_attention_weight_min": -29.871458053588867,
      "activations/layer1_attention_weight_max": 16.1385555267334,
      "activations/layer1_attention_weight_min": -14.453534126281738,
      "activations/layer20_attention_weight_max": 27.124238967895508,
      "activations/layer20_attention_weight_min": -25.113927841186523,
      "activations/layer21_attention_weight_max": 26.31633186340332,
      "activations/layer21_attention_weight_min": -26.218473434448242,
      "activations/layer22_attention_weight_max": 38.538475036621094,
      "activations/layer22_attention_weight_min": -31.863969802856445,
      "activations/layer23_attention_weight_max": 32.06768798828125,
      "activations/layer23_attention_weight_min": -25.9075984954834,
      "activations/layer2_attention_weight_max": 30.91446304321289,
      "activations/layer2_attention_weight_min": -31.857948303222656,
      "activations/layer3_attention_weight_max": 98.49980926513672,
      "activations/layer3_attention_weight_min": -98.08961486816406,
      "activations/layer4_attention_weight_max": 112.47708892822266,
      "activations/layer4_attention_weight_min": -108.1893310546875,
      "activations/layer5_attention_weight_max": 52.47649383544922,
      "activations/layer5_attention_weight_min": -62.420013427734375,
      "activations/layer6_attention_weight_max": 46.401798248291016,
      "activations/layer6_attention_weight_min": -48.51926040649414,
      "activations/layer7_attention_weight_max": 93.8160400390625,
      "activations/layer7_attention_weight_min": -98.31818389892578,
      "activations/layer8_attention_weight_max": 42.17100524902344,
      "activations/layer8_attention_weight_min": -48.372013092041016,
      "activations/layer9_attention_weight_max": 31.314023971557617,
      "activations/layer9_attention_weight_min": -38.77713394165039,
      "epoch": 14.26,
      "learning_rate": 5.8599242424242416e-05,
      "loss": 2.7556,
      "step": 245500
    },
    {
      "activations/layer0_attention_weight_max": 15.841264724731445,
      "activations/layer0_attention_weight_min": -12.833077430725098,
      "activations/layer10_attention_weight_max": 30.27052879333496,
      "activations/layer10_attention_weight_min": -30.75606346130371,
      "activations/layer11_attention_weight_max": 32.85615921020508,
      "activations/layer11_attention_weight_min": -31.39315414428711,
      "activations/layer12_attention_weight_max": 27.819303512573242,
      "activations/layer12_attention_weight_min": -26.199586868286133,
      "activations/layer13_attention_weight_max": 41.31940460205078,
      "activations/layer13_attention_weight_min": -34.888771057128906,
      "activations/layer14_attention_weight_max": 48.35988235473633,
      "activations/layer14_attention_weight_min": -37.04407501220703,
      "activations/layer15_attention_weight_max": 41.32243347167969,
      "activations/layer15_attention_weight_min": -32.39432144165039,
      "activations/layer16_attention_weight_max": 34.80506134033203,
      "activations/layer16_attention_weight_min": -27.423524856567383,
      "activations/layer17_attention_weight_max": 33.84480667114258,
      "activations/layer17_attention_weight_min": -25.688695907592773,
      "activations/layer18_attention_weight_max": 35.1197395324707,
      "activations/layer18_attention_weight_min": -24.94235610961914,
      "activations/layer19_attention_weight_max": 36.13383102416992,
      "activations/layer19_attention_weight_min": -31.051376342773438,
      "activations/layer1_attention_weight_max": 17.305585861206055,
      "activations/layer1_attention_weight_min": -15.68089771270752,
      "activations/layer20_attention_weight_max": 29.329238891601562,
      "activations/layer20_attention_weight_min": -25.41106605529785,
      "activations/layer21_attention_weight_max": 35.03489685058594,
      "activations/layer21_attention_weight_min": -25.24252700805664,
      "activations/layer22_attention_weight_max": 42.10239791870117,
      "activations/layer22_attention_weight_min": -30.92540168762207,
      "activations/layer23_attention_weight_max": 38.57271194458008,
      "activations/layer23_attention_weight_min": -25.4412841796875,
      "activations/layer2_attention_weight_max": 33.26634979248047,
      "activations/layer2_attention_weight_min": -32.506473541259766,
      "activations/layer3_attention_weight_max": 97.05107879638672,
      "activations/layer3_attention_weight_min": -100.74844360351562,
      "activations/layer4_attention_weight_max": 111.79595184326172,
      "activations/layer4_attention_weight_min": -111.3238754272461,
      "activations/layer5_attention_weight_max": 53.236732482910156,
      "activations/layer5_attention_weight_min": -62.22765350341797,
      "activations/layer6_attention_weight_max": 43.510719299316406,
      "activations/layer6_attention_weight_min": -47.61128616333008,
      "activations/layer7_attention_weight_max": 88.66838073730469,
      "activations/layer7_attention_weight_min": -89.23188018798828,
      "activations/layer8_attention_weight_max": 39.073211669921875,
      "activations/layer8_attention_weight_min": -42.408878326416016,
      "activations/layer9_attention_weight_max": 29.974966049194336,
      "activations/layer9_attention_weight_min": -31.86384391784668,
      "epoch": 14.27,
      "learning_rate": 5.8580303030303024e-05,
      "loss": 2.7462,
      "step": 245550
    },
    {
      "activations/layer0_attention_weight_max": 15.850122451782227,
      "activations/layer0_attention_weight_min": -12.868321418762207,
      "activations/layer10_attention_weight_max": 31.17332649230957,
      "activations/layer10_attention_weight_min": -31.555715560913086,
      "activations/layer11_attention_weight_max": 31.957067489624023,
      "activations/layer11_attention_weight_min": -31.200363159179688,
      "activations/layer12_attention_weight_max": 23.78061294555664,
      "activations/layer12_attention_weight_min": -24.792673110961914,
      "activations/layer13_attention_weight_max": 37.38128662109375,
      "activations/layer13_attention_weight_min": -35.70149612426758,
      "activations/layer14_attention_weight_max": 38.44136047363281,
      "activations/layer14_attention_weight_min": -37.85736846923828,
      "activations/layer15_attention_weight_max": 34.988887786865234,
      "activations/layer15_attention_weight_min": -33.53569793701172,
      "activations/layer16_attention_weight_max": 31.77358055114746,
      "activations/layer16_attention_weight_min": -27.5986385345459,
      "activations/layer17_attention_weight_max": 29.249887466430664,
      "activations/layer17_attention_weight_min": -25.46794891357422,
      "activations/layer18_attention_weight_max": 30.01167106628418,
      "activations/layer18_attention_weight_min": -27.169206619262695,
      "activations/layer19_attention_weight_max": 32.230220794677734,
      "activations/layer19_attention_weight_min": -31.081695556640625,
      "activations/layer1_attention_weight_max": 15.66473388671875,
      "activations/layer1_attention_weight_min": -15.28500747680664,
      "activations/layer20_attention_weight_max": 28.932510375976562,
      "activations/layer20_attention_weight_min": -25.627147674560547,
      "activations/layer21_attention_weight_max": 29.120784759521484,
      "activations/layer21_attention_weight_min": -23.989120483398438,
      "activations/layer22_attention_weight_max": 38.09266662597656,
      "activations/layer22_attention_weight_min": -31.33361053466797,
      "activations/layer23_attention_weight_max": 33.20616149902344,
      "activations/layer23_attention_weight_min": -27.004837036132812,
      "activations/layer2_attention_weight_max": 33.10141372680664,
      "activations/layer2_attention_weight_min": -32.95526885986328,
      "activations/layer3_attention_weight_max": 96.86943817138672,
      "activations/layer3_attention_weight_min": -95.05841827392578,
      "activations/layer4_attention_weight_max": 105.7155990600586,
      "activations/layer4_attention_weight_min": -106.58672332763672,
      "activations/layer5_attention_weight_max": 50.87293243408203,
      "activations/layer5_attention_weight_min": -63.05113983154297,
      "activations/layer6_attention_weight_max": 42.721275329589844,
      "activations/layer6_attention_weight_min": -45.036930084228516,
      "activations/layer7_attention_weight_max": 81.9270248413086,
      "activations/layer7_attention_weight_min": -82.6134033203125,
      "activations/layer8_attention_weight_max": 37.78678512573242,
      "activations/layer8_attention_weight_min": -39.595394134521484,
      "activations/layer9_attention_weight_max": 32.09259033203125,
      "activations/layer9_attention_weight_min": -30.77189064025879,
      "epoch": 14.27,
      "learning_rate": 5.856136363636363e-05,
      "loss": 2.761,
      "step": 245600
    },
    {
      "activations/layer0_attention_weight_max": 15.473702430725098,
      "activations/layer0_attention_weight_min": -12.47649097442627,
      "activations/layer10_attention_weight_max": 32.82134246826172,
      "activations/layer10_attention_weight_min": -34.03754806518555,
      "activations/layer11_attention_weight_max": 32.390769958496094,
      "activations/layer11_attention_weight_min": -32.17050552368164,
      "activations/layer12_attention_weight_max": 25.22484588623047,
      "activations/layer12_attention_weight_min": -25.43943977355957,
      "activations/layer13_attention_weight_max": 39.31455993652344,
      "activations/layer13_attention_weight_min": -33.9239501953125,
      "activations/layer14_attention_weight_max": 40.79191207885742,
      "activations/layer14_attention_weight_min": -36.1513671875,
      "activations/layer15_attention_weight_max": 38.13313674926758,
      "activations/layer15_attention_weight_min": -33.21454620361328,
      "activations/layer16_attention_weight_max": 30.36619758605957,
      "activations/layer16_attention_weight_min": -27.507457733154297,
      "activations/layer17_attention_weight_max": 31.73651123046875,
      "activations/layer17_attention_weight_min": -24.398874282836914,
      "activations/layer18_attention_weight_max": 33.00027847290039,
      "activations/layer18_attention_weight_min": -24.34530258178711,
      "activations/layer19_attention_weight_max": 33.32122039794922,
      "activations/layer19_attention_weight_min": -32.90409469604492,
      "activations/layer1_attention_weight_max": 15.978731155395508,
      "activations/layer1_attention_weight_min": -14.21854019165039,
      "activations/layer20_attention_weight_max": 29.80137825012207,
      "activations/layer20_attention_weight_min": -26.332368850708008,
      "activations/layer21_attention_weight_max": 29.900291442871094,
      "activations/layer21_attention_weight_min": -25.70497703552246,
      "activations/layer22_attention_weight_max": 45.615440368652344,
      "activations/layer22_attention_weight_min": -30.858356475830078,
      "activations/layer23_attention_weight_max": 35.36521530151367,
      "activations/layer23_attention_weight_min": -29.187538146972656,
      "activations/layer2_attention_weight_max": 31.66836166381836,
      "activations/layer2_attention_weight_min": -32.21508026123047,
      "activations/layer3_attention_weight_max": 92.7553482055664,
      "activations/layer3_attention_weight_min": -96.37876892089844,
      "activations/layer4_attention_weight_max": 109.3303451538086,
      "activations/layer4_attention_weight_min": -106.321533203125,
      "activations/layer5_attention_weight_max": 52.099571228027344,
      "activations/layer5_attention_weight_min": -63.441524505615234,
      "activations/layer6_attention_weight_max": 43.534053802490234,
      "activations/layer6_attention_weight_min": -46.88479232788086,
      "activations/layer7_attention_weight_max": 89.23101043701172,
      "activations/layer7_attention_weight_min": -91.77404022216797,
      "activations/layer8_attention_weight_max": 39.4604377746582,
      "activations/layer8_attention_weight_min": -42.72205352783203,
      "activations/layer9_attention_weight_max": 31.157733917236328,
      "activations/layer9_attention_weight_min": -34.20414352416992,
      "epoch": 14.27,
      "learning_rate": 5.854242424242424e-05,
      "loss": 2.7438,
      "step": 245650
    },
    {
      "activations/layer0_attention_weight_max": 16.11599349975586,
      "activations/layer0_attention_weight_min": -13.022124290466309,
      "activations/layer10_attention_weight_max": 35.23378372192383,
      "activations/layer10_attention_weight_min": -34.134830474853516,
      "activations/layer11_attention_weight_max": 36.87223434448242,
      "activations/layer11_attention_weight_min": -35.55137634277344,
      "activations/layer12_attention_weight_max": 27.513498306274414,
      "activations/layer12_attention_weight_min": -25.250431060791016,
      "activations/layer13_attention_weight_max": 39.78443908691406,
      "activations/layer13_attention_weight_min": -33.68273162841797,
      "activations/layer14_attention_weight_max": 42.00678634643555,
      "activations/layer14_attention_weight_min": -34.71765899658203,
      "activations/layer15_attention_weight_max": 39.26186752319336,
      "activations/layer15_attention_weight_min": -32.28803634643555,
      "activations/layer16_attention_weight_max": 28.87640380859375,
      "activations/layer16_attention_weight_min": -27.762569427490234,
      "activations/layer17_attention_weight_max": 32.22947692871094,
      "activations/layer17_attention_weight_min": -23.881372451782227,
      "activations/layer18_attention_weight_max": 33.9876594543457,
      "activations/layer18_attention_weight_min": -23.608325958251953,
      "activations/layer19_attention_weight_max": 35.200218200683594,
      "activations/layer19_attention_weight_min": -29.024404525756836,
      "activations/layer1_attention_weight_max": 15.740666389465332,
      "activations/layer1_attention_weight_min": -15.034814834594727,
      "activations/layer20_attention_weight_max": 30.584739685058594,
      "activations/layer20_attention_weight_min": -23.50600814819336,
      "activations/layer21_attention_weight_max": 30.55270004272461,
      "activations/layer21_attention_weight_min": -24.87103271484375,
      "activations/layer22_attention_weight_max": 50.42029571533203,
      "activations/layer22_attention_weight_min": -32.35288619995117,
      "activations/layer23_attention_weight_max": 36.0286750793457,
      "activations/layer23_attention_weight_min": -24.563800811767578,
      "activations/layer2_attention_weight_max": 32.47279739379883,
      "activations/layer2_attention_weight_min": -32.47018051147461,
      "activations/layer3_attention_weight_max": 95.65629577636719,
      "activations/layer3_attention_weight_min": -96.78860473632812,
      "activations/layer4_attention_weight_max": 103.96443176269531,
      "activations/layer4_attention_weight_min": -104.80266571044922,
      "activations/layer5_attention_weight_max": 49.22494125366211,
      "activations/layer5_attention_weight_min": -62.236228942871094,
      "activations/layer6_attention_weight_max": 42.52699279785156,
      "activations/layer6_attention_weight_min": -43.819175720214844,
      "activations/layer7_attention_weight_max": 93.97557067871094,
      "activations/layer7_attention_weight_min": -94.004150390625,
      "activations/layer8_attention_weight_max": 44.222618103027344,
      "activations/layer8_attention_weight_min": -42.552181243896484,
      "activations/layer9_attention_weight_max": 34.550819396972656,
      "activations/layer9_attention_weight_min": -35.5284309387207,
      "epoch": 14.28,
      "learning_rate": 5.852348484848484e-05,
      "loss": 2.7443,
      "step": 245700
    },
    {
      "activations/layer0_attention_weight_max": 16.384925842285156,
      "activations/layer0_attention_weight_min": -13.074544906616211,
      "activations/layer10_attention_weight_max": 34.59355545043945,
      "activations/layer10_attention_weight_min": -35.97947311401367,
      "activations/layer11_attention_weight_max": 34.37580108642578,
      "activations/layer11_attention_weight_min": -34.682254791259766,
      "activations/layer12_attention_weight_max": 25.76969337463379,
      "activations/layer12_attention_weight_min": -26.684280395507812,
      "activations/layer13_attention_weight_max": 37.98545837402344,
      "activations/layer13_attention_weight_min": -35.61610794067383,
      "activations/layer14_attention_weight_max": 43.25883483886719,
      "activations/layer14_attention_weight_min": -36.04956817626953,
      "activations/layer15_attention_weight_max": 38.464447021484375,
      "activations/layer15_attention_weight_min": -32.07036590576172,
      "activations/layer16_attention_weight_max": 34.61098861694336,
      "activations/layer16_attention_weight_min": -27.6998233795166,
      "activations/layer17_attention_weight_max": 31.01491928100586,
      "activations/layer17_attention_weight_min": -24.87558364868164,
      "activations/layer18_attention_weight_max": 31.181503295898438,
      "activations/layer18_attention_weight_min": -24.1997013092041,
      "activations/layer19_attention_weight_max": 33.152645111083984,
      "activations/layer19_attention_weight_min": -30.663299560546875,
      "activations/layer1_attention_weight_max": 16.459627151489258,
      "activations/layer1_attention_weight_min": -18.822856903076172,
      "activations/layer20_attention_weight_max": 29.87581443786621,
      "activations/layer20_attention_weight_min": -24.712461471557617,
      "activations/layer21_attention_weight_max": 29.95262908935547,
      "activations/layer21_attention_weight_min": -24.800029754638672,
      "activations/layer22_attention_weight_max": 39.36920166015625,
      "activations/layer22_attention_weight_min": -29.81932258605957,
      "activations/layer23_attention_weight_max": 30.331769943237305,
      "activations/layer23_attention_weight_min": -23.74684715270996,
      "activations/layer2_attention_weight_max": 33.138607025146484,
      "activations/layer2_attention_weight_min": -31.622411727905273,
      "activations/layer3_attention_weight_max": 99.01004791259766,
      "activations/layer3_attention_weight_min": -98.28801727294922,
      "activations/layer4_attention_weight_max": 108.49018859863281,
      "activations/layer4_attention_weight_min": -113.50066375732422,
      "activations/layer5_attention_weight_max": 50.30879211425781,
      "activations/layer5_attention_weight_min": -64.30207061767578,
      "activations/layer6_attention_weight_max": 46.291561126708984,
      "activations/layer6_attention_weight_min": -48.11625671386719,
      "activations/layer7_attention_weight_max": 94.55602264404297,
      "activations/layer7_attention_weight_min": -93.0707015991211,
      "activations/layer8_attention_weight_max": 41.578189849853516,
      "activations/layer8_attention_weight_min": -43.013221740722656,
      "activations/layer9_attention_weight_max": 34.9830436706543,
      "activations/layer9_attention_weight_min": -34.82801055908203,
      "epoch": 14.28,
      "learning_rate": 5.8504545454545444e-05,
      "loss": 2.7568,
      "step": 245750
    },
    {
      "activations/layer0_attention_weight_max": 15.313884735107422,
      "activations/layer0_attention_weight_min": -12.164238929748535,
      "activations/layer10_attention_weight_max": 33.843406677246094,
      "activations/layer10_attention_weight_min": -35.7547721862793,
      "activations/layer11_attention_weight_max": 34.3048095703125,
      "activations/layer11_attention_weight_min": -37.12543487548828,
      "activations/layer12_attention_weight_max": 26.36490821838379,
      "activations/layer12_attention_weight_min": -24.990585327148438,
      "activations/layer13_attention_weight_max": 43.96078872680664,
      "activations/layer13_attention_weight_min": -34.320133209228516,
      "activations/layer14_attention_weight_max": 50.06894302368164,
      "activations/layer14_attention_weight_min": -38.010982513427734,
      "activations/layer15_attention_weight_max": 43.78696060180664,
      "activations/layer15_attention_weight_min": -33.46656799316406,
      "activations/layer16_attention_weight_max": 30.594572067260742,
      "activations/layer16_attention_weight_min": -27.505163192749023,
      "activations/layer17_attention_weight_max": 32.523372650146484,
      "activations/layer17_attention_weight_min": -24.92882537841797,
      "activations/layer18_attention_weight_max": 34.758628845214844,
      "activations/layer18_attention_weight_min": -25.5210018157959,
      "activations/layer19_attention_weight_max": 36.61310577392578,
      "activations/layer19_attention_weight_min": -31.72894859313965,
      "activations/layer1_attention_weight_max": 16.541139602661133,
      "activations/layer1_attention_weight_min": -14.857074737548828,
      "activations/layer20_attention_weight_max": 28.782243728637695,
      "activations/layer20_attention_weight_min": -26.374971389770508,
      "activations/layer21_attention_weight_max": 28.747344970703125,
      "activations/layer21_attention_weight_min": -23.971731185913086,
      "activations/layer22_attention_weight_max": 42.285057067871094,
      "activations/layer22_attention_weight_min": -31.688562393188477,
      "activations/layer23_attention_weight_max": 37.07393264770508,
      "activations/layer23_attention_weight_min": -25.278797149658203,
      "activations/layer2_attention_weight_max": 32.691864013671875,
      "activations/layer2_attention_weight_min": -32.36676788330078,
      "activations/layer3_attention_weight_max": 98.0040283203125,
      "activations/layer3_attention_weight_min": -98.70140838623047,
      "activations/layer4_attention_weight_max": 108.07779693603516,
      "activations/layer4_attention_weight_min": -109.18895721435547,
      "activations/layer5_attention_weight_max": 53.33864974975586,
      "activations/layer5_attention_weight_min": -61.455284118652344,
      "activations/layer6_attention_weight_max": 42.96937561035156,
      "activations/layer6_attention_weight_min": -45.295326232910156,
      "activations/layer7_attention_weight_max": 88.68500518798828,
      "activations/layer7_attention_weight_min": -92.81229400634766,
      "activations/layer8_attention_weight_max": 39.946998596191406,
      "activations/layer8_attention_weight_min": -42.18659210205078,
      "activations/layer9_attention_weight_max": 34.011959075927734,
      "activations/layer9_attention_weight_min": -35.70869064331055,
      "epoch": 14.28,
      "learning_rate": 5.848560606060606e-05,
      "loss": 2.7398,
      "step": 245800
    },
    {
      "activations/layer0_attention_weight_max": 15.777689933776855,
      "activations/layer0_attention_weight_min": -11.764681816101074,
      "activations/layer10_attention_weight_max": 31.894798278808594,
      "activations/layer10_attention_weight_min": -32.11902618408203,
      "activations/layer11_attention_weight_max": 32.42779541015625,
      "activations/layer11_attention_weight_min": -33.57078170776367,
      "activations/layer12_attention_weight_max": 26.15222930908203,
      "activations/layer12_attention_weight_min": -26.339548110961914,
      "activations/layer13_attention_weight_max": 38.052978515625,
      "activations/layer13_attention_weight_min": -36.60744094848633,
      "activations/layer14_attention_weight_max": 47.398345947265625,
      "activations/layer14_attention_weight_min": -38.7299919128418,
      "activations/layer15_attention_weight_max": 36.36677932739258,
      "activations/layer15_attention_weight_min": -35.11552810668945,
      "activations/layer16_attention_weight_max": 31.09630584716797,
      "activations/layer16_attention_weight_min": -28.288970947265625,
      "activations/layer17_attention_weight_max": 31.0740966796875,
      "activations/layer17_attention_weight_min": -26.720623016357422,
      "activations/layer18_attention_weight_max": 32.97138595581055,
      "activations/layer18_attention_weight_min": -26.762481689453125,
      "activations/layer19_attention_weight_max": 38.24607849121094,
      "activations/layer19_attention_weight_min": -32.61537170410156,
      "activations/layer1_attention_weight_max": 16.845624923706055,
      "activations/layer1_attention_weight_min": -15.352754592895508,
      "activations/layer20_attention_weight_max": 29.798839569091797,
      "activations/layer20_attention_weight_min": -25.354955673217773,
      "activations/layer21_attention_weight_max": 27.29884910583496,
      "activations/layer21_attention_weight_min": -25.489242553710938,
      "activations/layer22_attention_weight_max": 42.80459976196289,
      "activations/layer22_attention_weight_min": -36.82154083251953,
      "activations/layer23_attention_weight_max": 35.56711196899414,
      "activations/layer23_attention_weight_min": -26.59809684753418,
      "activations/layer2_attention_weight_max": 32.79041290283203,
      "activations/layer2_attention_weight_min": -32.059051513671875,
      "activations/layer3_attention_weight_max": 94.25615692138672,
      "activations/layer3_attention_weight_min": -92.91944122314453,
      "activations/layer4_attention_weight_max": 108.0584487915039,
      "activations/layer4_attention_weight_min": -111.8869857788086,
      "activations/layer5_attention_weight_max": 50.78733444213867,
      "activations/layer5_attention_weight_min": -60.06328582763672,
      "activations/layer6_attention_weight_max": 43.05132293701172,
      "activations/layer6_attention_weight_min": -44.61323165893555,
      "activations/layer7_attention_weight_max": 88.64041900634766,
      "activations/layer7_attention_weight_min": -90.3513412475586,
      "activations/layer8_attention_weight_max": 38.85988235473633,
      "activations/layer8_attention_weight_min": -40.25974655151367,
      "activations/layer9_attention_weight_max": 31.8472900390625,
      "activations/layer9_attention_weight_min": -31.130956649780273,
      "epoch": 14.29,
      "learning_rate": 5.846666666666666e-05,
      "loss": 2.7311,
      "step": 245850
    },
    {
      "activations/layer0_attention_weight_max": 15.89847469329834,
      "activations/layer0_attention_weight_min": -12.273539543151855,
      "activations/layer10_attention_weight_max": 33.712005615234375,
      "activations/layer10_attention_weight_min": -32.078521728515625,
      "activations/layer11_attention_weight_max": 34.79985046386719,
      "activations/layer11_attention_weight_min": -33.23806381225586,
      "activations/layer12_attention_weight_max": 25.003570556640625,
      "activations/layer12_attention_weight_min": -26.007226943969727,
      "activations/layer13_attention_weight_max": 38.81010055541992,
      "activations/layer13_attention_weight_min": -34.83899688720703,
      "activations/layer14_attention_weight_max": 44.4278450012207,
      "activations/layer14_attention_weight_min": -38.094425201416016,
      "activations/layer15_attention_weight_max": 39.79308319091797,
      "activations/layer15_attention_weight_min": -34.668914794921875,
      "activations/layer16_attention_weight_max": 30.52752685546875,
      "activations/layer16_attention_weight_min": -27.946931838989258,
      "activations/layer17_attention_weight_max": 28.731348037719727,
      "activations/layer17_attention_weight_min": -25.767240524291992,
      "activations/layer18_attention_weight_max": 29.462703704833984,
      "activations/layer18_attention_weight_min": -26.428314208984375,
      "activations/layer19_attention_weight_max": 31.90659523010254,
      "activations/layer19_attention_weight_min": -32.15680694580078,
      "activations/layer1_attention_weight_max": 16.640710830688477,
      "activations/layer1_attention_weight_min": -14.970701217651367,
      "activations/layer20_attention_weight_max": 26.9095516204834,
      "activations/layer20_attention_weight_min": -25.71794891357422,
      "activations/layer21_attention_weight_max": 25.91628074645996,
      "activations/layer21_attention_weight_min": -25.39977264404297,
      "activations/layer22_attention_weight_max": 41.49031448364258,
      "activations/layer22_attention_weight_min": -30.92681884765625,
      "activations/layer23_attention_weight_max": 31.3376522064209,
      "activations/layer23_attention_weight_min": -26.2325496673584,
      "activations/layer2_attention_weight_max": 31.28447723388672,
      "activations/layer2_attention_weight_min": -31.909854888916016,
      "activations/layer3_attention_weight_max": 96.96338653564453,
      "activations/layer3_attention_weight_min": -98.40396118164062,
      "activations/layer4_attention_weight_max": 108.97087860107422,
      "activations/layer4_attention_weight_min": -110.10892486572266,
      "activations/layer5_attention_weight_max": 49.07514572143555,
      "activations/layer5_attention_weight_min": -66.71949768066406,
      "activations/layer6_attention_weight_max": 44.725746154785156,
      "activations/layer6_attention_weight_min": -45.52922439575195,
      "activations/layer7_attention_weight_max": 92.00865173339844,
      "activations/layer7_attention_weight_min": -91.36077117919922,
      "activations/layer8_attention_weight_max": 41.9553337097168,
      "activations/layer8_attention_weight_min": -40.676780700683594,
      "activations/layer9_attention_weight_max": 32.878509521484375,
      "activations/layer9_attention_weight_min": -31.787511825561523,
      "epoch": 14.29,
      "learning_rate": 5.844772727272727e-05,
      "loss": 2.7391,
      "step": 245900
    },
    {
      "activations/layer0_attention_weight_max": 14.709994316101074,
      "activations/layer0_attention_weight_min": -12.44910717010498,
      "activations/layer10_attention_weight_max": 41.01628875732422,
      "activations/layer10_attention_weight_min": -37.65568161010742,
      "activations/layer11_attention_weight_max": 42.8253288269043,
      "activations/layer11_attention_weight_min": -36.374244689941406,
      "activations/layer12_attention_weight_max": 26.640344619750977,
      "activations/layer12_attention_weight_min": -25.825149536132812,
      "activations/layer13_attention_weight_max": 40.98932647705078,
      "activations/layer13_attention_weight_min": -34.533447265625,
      "activations/layer14_attention_weight_max": 45.801612854003906,
      "activations/layer14_attention_weight_min": -37.898094177246094,
      "activations/layer15_attention_weight_max": 39.956939697265625,
      "activations/layer15_attention_weight_min": -36.34999465942383,
      "activations/layer16_attention_weight_max": 34.978275299072266,
      "activations/layer16_attention_weight_min": -27.767412185668945,
      "activations/layer17_attention_weight_max": 33.71601867675781,
      "activations/layer17_attention_weight_min": -25.777931213378906,
      "activations/layer18_attention_weight_max": 30.97150230407715,
      "activations/layer18_attention_weight_min": -25.103483200073242,
      "activations/layer19_attention_weight_max": 38.581336975097656,
      "activations/layer19_attention_weight_min": -31.59644889831543,
      "activations/layer1_attention_weight_max": 18.31508445739746,
      "activations/layer1_attention_weight_min": -15.925456047058105,
      "activations/layer20_attention_weight_max": 31.11897850036621,
      "activations/layer20_attention_weight_min": -25.427255630493164,
      "activations/layer21_attention_weight_max": 32.5810432434082,
      "activations/layer21_attention_weight_min": -24.477733612060547,
      "activations/layer22_attention_weight_max": 45.857627868652344,
      "activations/layer22_attention_weight_min": -29.446138381958008,
      "activations/layer23_attention_weight_max": 33.119651794433594,
      "activations/layer23_attention_weight_min": -25.019451141357422,
      "activations/layer2_attention_weight_max": 37.04317092895508,
      "activations/layer2_attention_weight_min": -34.78169250488281,
      "activations/layer3_attention_weight_max": 107.15093231201172,
      "activations/layer3_attention_weight_min": -102.995849609375,
      "activations/layer4_attention_weight_max": 119.10839080810547,
      "activations/layer4_attention_weight_min": -114.52967834472656,
      "activations/layer5_attention_weight_max": 54.69270324707031,
      "activations/layer5_attention_weight_min": -63.08137130737305,
      "activations/layer6_attention_weight_max": 49.349388122558594,
      "activations/layer6_attention_weight_min": -50.49135208129883,
      "activations/layer7_attention_weight_max": 108.00507354736328,
      "activations/layer7_attention_weight_min": -104.49906921386719,
      "activations/layer8_attention_weight_max": 50.5330696105957,
      "activations/layer8_attention_weight_min": -48.186824798583984,
      "activations/layer9_attention_weight_max": 38.268131256103516,
      "activations/layer9_attention_weight_min": -37.485469818115234,
      "epoch": 14.29,
      "learning_rate": 5.842878787878788e-05,
      "loss": 2.7605,
      "step": 245950
    },
    {
      "activations/layer0_attention_weight_max": 15.756608009338379,
      "activations/layer0_attention_weight_min": -13.047094345092773,
      "activations/layer10_attention_weight_max": 35.8405647277832,
      "activations/layer10_attention_weight_min": -39.81282043457031,
      "activations/layer11_attention_weight_max": 36.29851150512695,
      "activations/layer11_attention_weight_min": -40.148284912109375,
      "activations/layer12_attention_weight_max": 28.68253517150879,
      "activations/layer12_attention_weight_min": -28.95922088623047,
      "activations/layer13_attention_weight_max": 38.85209274291992,
      "activations/layer13_attention_weight_min": -36.72057342529297,
      "activations/layer14_attention_weight_max": 48.43874740600586,
      "activations/layer14_attention_weight_min": -44.28363800048828,
      "activations/layer15_attention_weight_max": 39.89403533935547,
      "activations/layer15_attention_weight_min": -39.05278396606445,
      "activations/layer16_attention_weight_max": 28.662626266479492,
      "activations/layer16_attention_weight_min": -28.74801254272461,
      "activations/layer17_attention_weight_max": 31.20852279663086,
      "activations/layer17_attention_weight_min": -29.69011878967285,
      "activations/layer18_attention_weight_max": 28.96788215637207,
      "activations/layer18_attention_weight_min": -27.271236419677734,
      "activations/layer19_attention_weight_max": 32.131961822509766,
      "activations/layer19_attention_weight_min": -34.12624740600586,
      "activations/layer1_attention_weight_max": 16.681026458740234,
      "activations/layer1_attention_weight_min": -16.280288696289062,
      "activations/layer20_attention_weight_max": 26.453819274902344,
      "activations/layer20_attention_weight_min": -25.83578109741211,
      "activations/layer21_attention_weight_max": 27.735387802124023,
      "activations/layer21_attention_weight_min": -26.653715133666992,
      "activations/layer22_attention_weight_max": 43.82979965209961,
      "activations/layer22_attention_weight_min": -35.51715087890625,
      "activations/layer23_attention_weight_max": 34.43265151977539,
      "activations/layer23_attention_weight_min": -31.527233123779297,
      "activations/layer2_attention_weight_max": 33.653385162353516,
      "activations/layer2_attention_weight_min": -32.93882751464844,
      "activations/layer3_attention_weight_max": 98.70506286621094,
      "activations/layer3_attention_weight_min": -99.27460479736328,
      "activations/layer4_attention_weight_max": 110.311767578125,
      "activations/layer4_attention_weight_min": -116.01307678222656,
      "activations/layer5_attention_weight_max": 52.418514251708984,
      "activations/layer5_attention_weight_min": -64.37133026123047,
      "activations/layer6_attention_weight_max": 49.218406677246094,
      "activations/layer6_attention_weight_min": -51.636817932128906,
      "activations/layer7_attention_weight_max": 94.63224029541016,
      "activations/layer7_attention_weight_min": -108.42645263671875,
      "activations/layer8_attention_weight_max": 45.442161560058594,
      "activations/layer8_attention_weight_min": -50.50493240356445,
      "activations/layer9_attention_weight_max": 36.32265853881836,
      "activations/layer9_attention_weight_min": -40.395259857177734,
      "epoch": 14.29,
      "learning_rate": 5.840984848484848e-05,
      "loss": 2.7609,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4898,
      "eval_samples_per_second": 505.782,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4898,
      "eval_openwebtext_samples_per_second": 505.782,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9997,
      "eval_wikitext_samples_per_second": 228.036,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_lambada_loss": 2.458984375,
      "eval_lambada_ppl": 11.69292987543966,
      "eval_lambada_runtime": 9.6053,
      "eval_lambada_samples_per_second": 506.907,
      "step": 246000
    },
    {
      "activations/layer0_attention_weight_max": 16.74931526184082,
      "activations/layer0_attention_weight_min": -12.911211967468262,
      "activations/layer10_attention_weight_max": 36.662872314453125,
      "activations/layer10_attention_weight_min": -33.61280059814453,
      "activations/layer11_attention_weight_max": 34.419708251953125,
      "activations/layer11_attention_weight_min": -32.92372512817383,
      "activations/layer12_attention_weight_max": 26.45463752746582,
      "activations/layer12_attention_weight_min": -27.893638610839844,
      "activations/layer13_attention_weight_max": 38.18749237060547,
      "activations/layer13_attention_weight_min": -36.38442611694336,
      "activations/layer14_attention_weight_max": 42.1710205078125,
      "activations/layer14_attention_weight_min": -39.957862854003906,
      "activations/layer15_attention_weight_max": 35.31128692626953,
      "activations/layer15_attention_weight_min": -34.512367248535156,
      "activations/layer16_attention_weight_max": 34.231048583984375,
      "activations/layer16_attention_weight_min": -28.348590850830078,
      "activations/layer17_attention_weight_max": 30.8154296875,
      "activations/layer17_attention_weight_min": -25.857376098632812,
      "activations/layer18_attention_weight_max": 32.85963439941406,
      "activations/layer18_attention_weight_min": -24.871395111083984,
      "activations/layer19_attention_weight_max": 36.47782516479492,
      "activations/layer19_attention_weight_min": -31.358642578125,
      "activations/layer1_attention_weight_max": 16.229328155517578,
      "activations/layer1_attention_weight_min": -14.280492782592773,
      "activations/layer20_attention_weight_max": 31.382444381713867,
      "activations/layer20_attention_weight_min": -24.763832092285156,
      "activations/layer21_attention_weight_max": 32.39342498779297,
      "activations/layer21_attention_weight_min": -25.094161987304688,
      "activations/layer22_attention_weight_max": 41.83757781982422,
      "activations/layer22_attention_weight_min": -34.73158264160156,
      "activations/layer23_attention_weight_max": 33.88872528076172,
      "activations/layer23_attention_weight_min": -26.844411849975586,
      "activations/layer2_attention_weight_max": 32.86154556274414,
      "activations/layer2_attention_weight_min": -32.485286712646484,
      "activations/layer3_attention_weight_max": 97.21312713623047,
      "activations/layer3_attention_weight_min": -95.36807250976562,
      "activations/layer4_attention_weight_max": 109.73799133300781,
      "activations/layer4_attention_weight_min": -110.00406646728516,
      "activations/layer5_attention_weight_max": 52.21038818359375,
      "activations/layer5_attention_weight_min": -64.18963623046875,
      "activations/layer6_attention_weight_max": 45.000328063964844,
      "activations/layer6_attention_weight_min": -46.15959930419922,
      "activations/layer7_attention_weight_max": 89.95938873291016,
      "activations/layer7_attention_weight_min": -92.728515625,
      "activations/layer8_attention_weight_max": 41.686004638671875,
      "activations/layer8_attention_weight_min": -41.8612060546875,
      "activations/layer9_attention_weight_max": 32.22243118286133,
      "activations/layer9_attention_weight_min": -35.94486618041992,
      "epoch": 14.3,
      "learning_rate": 5.8390909090909087e-05,
      "loss": 2.7583,
      "step": 246050
    },
    {
      "activations/layer0_attention_weight_max": 15.340407371520996,
      "activations/layer0_attention_weight_min": -12.810637474060059,
      "activations/layer10_attention_weight_max": 30.60582733154297,
      "activations/layer10_attention_weight_min": -32.459163665771484,
      "activations/layer11_attention_weight_max": 29.79559898376465,
      "activations/layer11_attention_weight_min": -31.78585433959961,
      "activations/layer12_attention_weight_max": 26.282609939575195,
      "activations/layer12_attention_weight_min": -24.67861557006836,
      "activations/layer13_attention_weight_max": 37.504302978515625,
      "activations/layer13_attention_weight_min": -34.73680877685547,
      "activations/layer14_attention_weight_max": 44.47003173828125,
      "activations/layer14_attention_weight_min": -38.479217529296875,
      "activations/layer15_attention_weight_max": 37.13130187988281,
      "activations/layer15_attention_weight_min": -33.5338249206543,
      "activations/layer16_attention_weight_max": 31.28871726989746,
      "activations/layer16_attention_weight_min": -26.542402267456055,
      "activations/layer17_attention_weight_max": 30.53200340270996,
      "activations/layer17_attention_weight_min": -26.421945571899414,
      "activations/layer18_attention_weight_max": 31.14382553100586,
      "activations/layer18_attention_weight_min": -26.416635513305664,
      "activations/layer19_attention_weight_max": 35.853267669677734,
      "activations/layer19_attention_weight_min": -32.95229721069336,
      "activations/layer1_attention_weight_max": 16.719554901123047,
      "activations/layer1_attention_weight_min": -15.547506332397461,
      "activations/layer20_attention_weight_max": 32.01164245605469,
      "activations/layer20_attention_weight_min": -25.508438110351562,
      "activations/layer21_attention_weight_max": 32.82830810546875,
      "activations/layer21_attention_weight_min": -26.603567123413086,
      "activations/layer22_attention_weight_max": 42.06486892700195,
      "activations/layer22_attention_weight_min": -31.15998649597168,
      "activations/layer23_attention_weight_max": 34.91679763793945,
      "activations/layer23_attention_weight_min": -27.24066925048828,
      "activations/layer2_attention_weight_max": 32.24991226196289,
      "activations/layer2_attention_weight_min": -33.13509750366211,
      "activations/layer3_attention_weight_max": 94.98018646240234,
      "activations/layer3_attention_weight_min": -95.47875213623047,
      "activations/layer4_attention_weight_max": 109.8467025756836,
      "activations/layer4_attention_weight_min": -110.47429656982422,
      "activations/layer5_attention_weight_max": 54.14756393432617,
      "activations/layer5_attention_weight_min": -63.8546142578125,
      "activations/layer6_attention_weight_max": 43.19764709472656,
      "activations/layer6_attention_weight_min": -46.38979721069336,
      "activations/layer7_attention_weight_max": 89.54225158691406,
      "activations/layer7_attention_weight_min": -89.76282501220703,
      "activations/layer8_attention_weight_max": 40.51169204711914,
      "activations/layer8_attention_weight_min": -39.760284423828125,
      "activations/layer9_attention_weight_max": 29.498071670532227,
      "activations/layer9_attention_weight_min": -31.481176376342773,
      "epoch": 14.3,
      "learning_rate": 5.837196969696969e-05,
      "loss": 2.7473,
      "step": 246100
    },
    {
      "activations/layer0_attention_weight_max": 16.0274715423584,
      "activations/layer0_attention_weight_min": -12.472301483154297,
      "activations/layer10_attention_weight_max": 33.07281494140625,
      "activations/layer10_attention_weight_min": -33.107521057128906,
      "activations/layer11_attention_weight_max": 33.737693786621094,
      "activations/layer11_attention_weight_min": -31.921619415283203,
      "activations/layer12_attention_weight_max": 28.0603084564209,
      "activations/layer12_attention_weight_min": -28.519296646118164,
      "activations/layer13_attention_weight_max": 43.92592239379883,
      "activations/layer13_attention_weight_min": -34.220848083496094,
      "activations/layer14_attention_weight_max": 45.93370819091797,
      "activations/layer14_attention_weight_min": -37.146705627441406,
      "activations/layer15_attention_weight_max": 42.10431671142578,
      "activations/layer15_attention_weight_min": -34.24635696411133,
      "activations/layer16_attention_weight_max": 33.47458267211914,
      "activations/layer16_attention_weight_min": -27.102144241333008,
      "activations/layer17_attention_weight_max": 32.54069900512695,
      "activations/layer17_attention_weight_min": -27.14931869506836,
      "activations/layer18_attention_weight_max": 32.93439483642578,
      "activations/layer18_attention_weight_min": -24.045936584472656,
      "activations/layer19_attention_weight_max": 37.35917282104492,
      "activations/layer19_attention_weight_min": -32.518863677978516,
      "activations/layer1_attention_weight_max": 17.538354873657227,
      "activations/layer1_attention_weight_min": -15.941872596740723,
      "activations/layer20_attention_weight_max": 30.00111961364746,
      "activations/layer20_attention_weight_min": -25.50823211669922,
      "activations/layer21_attention_weight_max": 30.01226806640625,
      "activations/layer21_attention_weight_min": -25.623332977294922,
      "activations/layer22_attention_weight_max": 54.61882400512695,
      "activations/layer22_attention_weight_min": -31.155929565429688,
      "activations/layer23_attention_weight_max": 35.046566009521484,
      "activations/layer23_attention_weight_min": -24.611221313476562,
      "activations/layer2_attention_weight_max": 37.55091857910156,
      "activations/layer2_attention_weight_min": -35.034461975097656,
      "activations/layer3_attention_weight_max": 105.8215103149414,
      "activations/layer3_attention_weight_min": -101.24749755859375,
      "activations/layer4_attention_weight_max": 113.7182388305664,
      "activations/layer4_attention_weight_min": -114.85260009765625,
      "activations/layer5_attention_weight_max": 54.17687225341797,
      "activations/layer5_attention_weight_min": -61.872352600097656,
      "activations/layer6_attention_weight_max": 44.71031951904297,
      "activations/layer6_attention_weight_min": -47.37018966674805,
      "activations/layer7_attention_weight_max": 91.63055419921875,
      "activations/layer7_attention_weight_min": -91.13825225830078,
      "activations/layer8_attention_weight_max": 41.7686653137207,
      "activations/layer8_attention_weight_min": -41.61003494262695,
      "activations/layer9_attention_weight_max": 32.944427490234375,
      "activations/layer9_attention_weight_min": -33.23777770996094,
      "epoch": 14.3,
      "learning_rate": 5.83530303030303e-05,
      "loss": 2.7431,
      "step": 246150
    },
    {
      "activations/layer0_attention_weight_max": 16.008071899414062,
      "activations/layer0_attention_weight_min": -12.171466827392578,
      "activations/layer10_attention_weight_max": 32.3101806640625,
      "activations/layer10_attention_weight_min": -33.30059814453125,
      "activations/layer11_attention_weight_max": 31.831565856933594,
      "activations/layer11_attention_weight_min": -33.19917297363281,
      "activations/layer12_attention_weight_max": 24.97514533996582,
      "activations/layer12_attention_weight_min": -27.10416030883789,
      "activations/layer13_attention_weight_max": 37.80564880371094,
      "activations/layer13_attention_weight_min": -35.968746185302734,
      "activations/layer14_attention_weight_max": 41.35708999633789,
      "activations/layer14_attention_weight_min": -41.05757522583008,
      "activations/layer15_attention_weight_max": 37.63250732421875,
      "activations/layer15_attention_weight_min": -38.64795684814453,
      "activations/layer16_attention_weight_max": 29.395397186279297,
      "activations/layer16_attention_weight_min": -29.49915313720703,
      "activations/layer17_attention_weight_max": 31.049943923950195,
      "activations/layer17_attention_weight_min": -27.15304946899414,
      "activations/layer18_attention_weight_max": 30.339632034301758,
      "activations/layer18_attention_weight_min": -24.08750343322754,
      "activations/layer19_attention_weight_max": 33.186492919921875,
      "activations/layer19_attention_weight_min": -31.387100219726562,
      "activations/layer1_attention_weight_max": 16.857500076293945,
      "activations/layer1_attention_weight_min": -14.299057960510254,
      "activations/layer20_attention_weight_max": 29.3262939453125,
      "activations/layer20_attention_weight_min": -26.083934783935547,
      "activations/layer21_attention_weight_max": 25.86041259765625,
      "activations/layer21_attention_weight_min": -24.9997501373291,
      "activations/layer22_attention_weight_max": 38.0650634765625,
      "activations/layer22_attention_weight_min": -31.16229248046875,
      "activations/layer23_attention_weight_max": 31.838054656982422,
      "activations/layer23_attention_weight_min": -25.494266510009766,
      "activations/layer2_attention_weight_max": 31.964168548583984,
      "activations/layer2_attention_weight_min": -32.4742546081543,
      "activations/layer3_attention_weight_max": 99.08500671386719,
      "activations/layer3_attention_weight_min": -98.80071258544922,
      "activations/layer4_attention_weight_max": 116.37406158447266,
      "activations/layer4_attention_weight_min": -115.2436752319336,
      "activations/layer5_attention_weight_max": 52.50767135620117,
      "activations/layer5_attention_weight_min": -61.47267150878906,
      "activations/layer6_attention_weight_max": 45.0629768371582,
      "activations/layer6_attention_weight_min": -48.3431282043457,
      "activations/layer7_attention_weight_max": 90.39065551757812,
      "activations/layer7_attention_weight_min": -95.52997589111328,
      "activations/layer8_attention_weight_max": 43.15272903442383,
      "activations/layer8_attention_weight_min": -43.86509704589844,
      "activations/layer9_attention_weight_max": 31.68989372253418,
      "activations/layer9_attention_weight_min": -34.019569396972656,
      "epoch": 14.31,
      "learning_rate": 5.8334090909090905e-05,
      "loss": 2.7759,
      "step": 246200
    },
    {
      "activations/layer0_attention_weight_max": 16.31431770324707,
      "activations/layer0_attention_weight_min": -12.981491088867188,
      "activations/layer10_attention_weight_max": 32.74683380126953,
      "activations/layer10_attention_weight_min": -33.014183044433594,
      "activations/layer11_attention_weight_max": 36.24496078491211,
      "activations/layer11_attention_weight_min": -33.216793060302734,
      "activations/layer12_attention_weight_max": 25.725894927978516,
      "activations/layer12_attention_weight_min": -24.974384307861328,
      "activations/layer13_attention_weight_max": 41.2696647644043,
      "activations/layer13_attention_weight_min": -36.328514099121094,
      "activations/layer14_attention_weight_max": 47.23393630981445,
      "activations/layer14_attention_weight_min": -41.227149963378906,
      "activations/layer15_attention_weight_max": 42.087303161621094,
      "activations/layer15_attention_weight_min": -37.456607818603516,
      "activations/layer16_attention_weight_max": 34.95549774169922,
      "activations/layer16_attention_weight_min": -27.0673828125,
      "activations/layer17_attention_weight_max": 35.25093460083008,
      "activations/layer17_attention_weight_min": -26.19468116760254,
      "activations/layer18_attention_weight_max": 33.282527923583984,
      "activations/layer18_attention_weight_min": -28.434377670288086,
      "activations/layer19_attention_weight_max": 39.2574577331543,
      "activations/layer19_attention_weight_min": -31.195798873901367,
      "activations/layer1_attention_weight_max": 16.32527732849121,
      "activations/layer1_attention_weight_min": -14.901986122131348,
      "activations/layer20_attention_weight_max": 33.67528533935547,
      "activations/layer20_attention_weight_min": -24.868112564086914,
      "activations/layer21_attention_weight_max": 30.75554656982422,
      "activations/layer21_attention_weight_min": -24.50762176513672,
      "activations/layer22_attention_weight_max": 49.20591354370117,
      "activations/layer22_attention_weight_min": -33.919837951660156,
      "activations/layer23_attention_weight_max": 36.11076354980469,
      "activations/layer23_attention_weight_min": -27.255756378173828,
      "activations/layer2_attention_weight_max": 32.998802185058594,
      "activations/layer2_attention_weight_min": -33.620704650878906,
      "activations/layer3_attention_weight_max": 100.68234252929688,
      "activations/layer3_attention_weight_min": -99.98873901367188,
      "activations/layer4_attention_weight_max": 110.84478759765625,
      "activations/layer4_attention_weight_min": -109.43859100341797,
      "activations/layer5_attention_weight_max": 50.297908782958984,
      "activations/layer5_attention_weight_min": -64.23550415039062,
      "activations/layer6_attention_weight_max": 42.666500091552734,
      "activations/layer6_attention_weight_min": -45.76087188720703,
      "activations/layer7_attention_weight_max": 94.52974700927734,
      "activations/layer7_attention_weight_min": -84.13275146484375,
      "activations/layer8_attention_weight_max": 41.79768371582031,
      "activations/layer8_attention_weight_min": -41.864681243896484,
      "activations/layer9_attention_weight_max": 31.1387882232666,
      "activations/layer9_attention_weight_min": -32.594295501708984,
      "epoch": 14.31,
      "learning_rate": 5.8315151515151506e-05,
      "loss": 2.7598,
      "step": 246250
    },
    {
      "activations/layer0_attention_weight_max": 14.888010025024414,
      "activations/layer0_attention_weight_min": -12.365569114685059,
      "activations/layer10_attention_weight_max": 32.65770721435547,
      "activations/layer10_attention_weight_min": -34.35567855834961,
      "activations/layer11_attention_weight_max": 32.065643310546875,
      "activations/layer11_attention_weight_min": -33.508914947509766,
      "activations/layer12_attention_weight_max": 25.56586456298828,
      "activations/layer12_attention_weight_min": -25.97708511352539,
      "activations/layer13_attention_weight_max": 39.97340393066406,
      "activations/layer13_attention_weight_min": -37.02617263793945,
      "activations/layer14_attention_weight_max": 45.70747756958008,
      "activations/layer14_attention_weight_min": -39.15901565551758,
      "activations/layer15_attention_weight_max": 39.768436431884766,
      "activations/layer15_attention_weight_min": -32.95864486694336,
      "activations/layer16_attention_weight_max": 31.57459259033203,
      "activations/layer16_attention_weight_min": -29.21198272705078,
      "activations/layer17_attention_weight_max": 29.8789005279541,
      "activations/layer17_attention_weight_min": -25.727758407592773,
      "activations/layer18_attention_weight_max": 32.059234619140625,
      "activations/layer18_attention_weight_min": -24.648826599121094,
      "activations/layer19_attention_weight_max": 31.88372230529785,
      "activations/layer19_attention_weight_min": -31.76751136779785,
      "activations/layer1_attention_weight_max": 16.125789642333984,
      "activations/layer1_attention_weight_min": -15.294358253479004,
      "activations/layer20_attention_weight_max": 27.424455642700195,
      "activations/layer20_attention_weight_min": -25.108797073364258,
      "activations/layer21_attention_weight_max": 27.723526000976562,
      "activations/layer21_attention_weight_min": -24.55805015563965,
      "activations/layer22_attention_weight_max": 47.16121292114258,
      "activations/layer22_attention_weight_min": -31.311384201049805,
      "activations/layer23_attention_weight_max": 30.45096206665039,
      "activations/layer23_attention_weight_min": -24.929515838623047,
      "activations/layer2_attention_weight_max": 32.72197723388672,
      "activations/layer2_attention_weight_min": -33.815738677978516,
      "activations/layer3_attention_weight_max": 100.5640869140625,
      "activations/layer3_attention_weight_min": -99.64820098876953,
      "activations/layer4_attention_weight_max": 112.29119110107422,
      "activations/layer4_attention_weight_min": -112.9043960571289,
      "activations/layer5_attention_weight_max": 53.49602508544922,
      "activations/layer5_attention_weight_min": -64.91673278808594,
      "activations/layer6_attention_weight_max": 46.97539520263672,
      "activations/layer6_attention_weight_min": -52.64628601074219,
      "activations/layer7_attention_weight_max": 96.15132904052734,
      "activations/layer7_attention_weight_min": -95.85015869140625,
      "activations/layer8_attention_weight_max": 40.53550720214844,
      "activations/layer8_attention_weight_min": -43.00632095336914,
      "activations/layer9_attention_weight_max": 31.652685165405273,
      "activations/layer9_attention_weight_min": -33.50320053100586,
      "epoch": 14.31,
      "learning_rate": 5.8296212121212114e-05,
      "loss": 2.7407,
      "step": 246300
    },
    {
      "activations/layer0_attention_weight_max": 16.40361213684082,
      "activations/layer0_attention_weight_min": -12.06248664855957,
      "activations/layer10_attention_weight_max": 31.860044479370117,
      "activations/layer10_attention_weight_min": -32.75246047973633,
      "activations/layer11_attention_weight_max": 33.553646087646484,
      "activations/layer11_attention_weight_min": -32.68498229980469,
      "activations/layer12_attention_weight_max": 26.695100784301758,
      "activations/layer12_attention_weight_min": -26.253555297851562,
      "activations/layer13_attention_weight_max": 40.416770935058594,
      "activations/layer13_attention_weight_min": -36.38210678100586,
      "activations/layer14_attention_weight_max": 42.45747375488281,
      "activations/layer14_attention_weight_min": -38.11839294433594,
      "activations/layer15_attention_weight_max": 34.6432991027832,
      "activations/layer15_attention_weight_min": -33.924251556396484,
      "activations/layer16_attention_weight_max": 29.699419021606445,
      "activations/layer16_attention_weight_min": -28.40309715270996,
      "activations/layer17_attention_weight_max": 31.970863342285156,
      "activations/layer17_attention_weight_min": -27.540924072265625,
      "activations/layer18_attention_weight_max": 30.609392166137695,
      "activations/layer18_attention_weight_min": -25.977752685546875,
      "activations/layer19_attention_weight_max": 34.4313850402832,
      "activations/layer19_attention_weight_min": -31.879043579101562,
      "activations/layer1_attention_weight_max": 15.950640678405762,
      "activations/layer1_attention_weight_min": -15.995800971984863,
      "activations/layer20_attention_weight_max": 28.406003952026367,
      "activations/layer20_attention_weight_min": -26.86836814880371,
      "activations/layer21_attention_weight_max": 31.037343978881836,
      "activations/layer21_attention_weight_min": -26.22365951538086,
      "activations/layer22_attention_weight_max": 42.62702560424805,
      "activations/layer22_attention_weight_min": -31.457685470581055,
      "activations/layer23_attention_weight_max": 32.66679000854492,
      "activations/layer23_attention_weight_min": -25.448394775390625,
      "activations/layer2_attention_weight_max": 32.98339080810547,
      "activations/layer2_attention_weight_min": -33.25209045410156,
      "activations/layer3_attention_weight_max": 96.4695816040039,
      "activations/layer3_attention_weight_min": -95.33966064453125,
      "activations/layer4_attention_weight_max": 109.71915435791016,
      "activations/layer4_attention_weight_min": -106.7555160522461,
      "activations/layer5_attention_weight_max": 52.49430465698242,
      "activations/layer5_attention_weight_min": -62.557823181152344,
      "activations/layer6_attention_weight_max": 44.78867721557617,
      "activations/layer6_attention_weight_min": -48.969261169433594,
      "activations/layer7_attention_weight_max": 89.83724212646484,
      "activations/layer7_attention_weight_min": -87.5380630493164,
      "activations/layer8_attention_weight_max": 42.84981155395508,
      "activations/layer8_attention_weight_min": -40.81718444824219,
      "activations/layer9_attention_weight_max": 32.310264587402344,
      "activations/layer9_attention_weight_min": -32.49506378173828,
      "epoch": 14.31,
      "learning_rate": 5.827727272727272e-05,
      "loss": 2.7573,
      "step": 246350
    },
    {
      "activations/layer0_attention_weight_max": 15.91767406463623,
      "activations/layer0_attention_weight_min": -12.807443618774414,
      "activations/layer10_attention_weight_max": 31.47435188293457,
      "activations/layer10_attention_weight_min": -32.86209487915039,
      "activations/layer11_attention_weight_max": 32.18151092529297,
      "activations/layer11_attention_weight_min": -32.110816955566406,
      "activations/layer12_attention_weight_max": 24.793441772460938,
      "activations/layer12_attention_weight_min": -25.38202667236328,
      "activations/layer13_attention_weight_max": 38.88422393798828,
      "activations/layer13_attention_weight_min": -34.31101608276367,
      "activations/layer14_attention_weight_max": 47.368473052978516,
      "activations/layer14_attention_weight_min": -40.7890510559082,
      "activations/layer15_attention_weight_max": 37.50979995727539,
      "activations/layer15_attention_weight_min": -33.62333297729492,
      "activations/layer16_attention_weight_max": 30.091903686523438,
      "activations/layer16_attention_weight_min": -26.932472229003906,
      "activations/layer17_attention_weight_max": 29.922006607055664,
      "activations/layer17_attention_weight_min": -25.602319717407227,
      "activations/layer18_attention_weight_max": 30.63117790222168,
      "activations/layer18_attention_weight_min": -26.09935760498047,
      "activations/layer19_attention_weight_max": 36.65859603881836,
      "activations/layer19_attention_weight_min": -30.801624298095703,
      "activations/layer1_attention_weight_max": 16.711673736572266,
      "activations/layer1_attention_weight_min": -14.55309009552002,
      "activations/layer20_attention_weight_max": 30.923505783081055,
      "activations/layer20_attention_weight_min": -24.526470184326172,
      "activations/layer21_attention_weight_max": 30.791934967041016,
      "activations/layer21_attention_weight_min": -24.230175018310547,
      "activations/layer22_attention_weight_max": 44.64753723144531,
      "activations/layer22_attention_weight_min": -29.01662826538086,
      "activations/layer23_attention_weight_max": 36.42258834838867,
      "activations/layer23_attention_weight_min": -25.701169967651367,
      "activations/layer2_attention_weight_max": 33.57152557373047,
      "activations/layer2_attention_weight_min": -31.748104095458984,
      "activations/layer3_attention_weight_max": 97.83226776123047,
      "activations/layer3_attention_weight_min": -99.63619232177734,
      "activations/layer4_attention_weight_max": 109.65689849853516,
      "activations/layer4_attention_weight_min": -108.33773040771484,
      "activations/layer5_attention_weight_max": 55.836891174316406,
      "activations/layer5_attention_weight_min": -59.62370300292969,
      "activations/layer6_attention_weight_max": 45.17198944091797,
      "activations/layer6_attention_weight_min": -47.00251007080078,
      "activations/layer7_attention_weight_max": 92.0899887084961,
      "activations/layer7_attention_weight_min": -94.7667007446289,
      "activations/layer8_attention_weight_max": 38.240074157714844,
      "activations/layer8_attention_weight_min": -43.586097717285156,
      "activations/layer9_attention_weight_max": 31.288909912109375,
      "activations/layer9_attention_weight_min": -32.2114372253418,
      "epoch": 14.32,
      "learning_rate": 5.825833333333333e-05,
      "loss": 2.7665,
      "step": 246400
    },
    {
      "activations/layer0_attention_weight_max": 15.58035659790039,
      "activations/layer0_attention_weight_min": -12.091476440429688,
      "activations/layer10_attention_weight_max": 30.04905128479004,
      "activations/layer10_attention_weight_min": -33.39918899536133,
      "activations/layer11_attention_weight_max": 31.933490753173828,
      "activations/layer11_attention_weight_min": -33.01810073852539,
      "activations/layer12_attention_weight_max": 25.16749382019043,
      "activations/layer12_attention_weight_min": -26.673500061035156,
      "activations/layer13_attention_weight_max": 38.32564926147461,
      "activations/layer13_attention_weight_min": -35.42082977294922,
      "activations/layer14_attention_weight_max": 44.07173156738281,
      "activations/layer14_attention_weight_min": -39.68259811401367,
      "activations/layer15_attention_weight_max": 36.11724090576172,
      "activations/layer15_attention_weight_min": -34.10409164428711,
      "activations/layer16_attention_weight_max": 32.57292938232422,
      "activations/layer16_attention_weight_min": -28.87611961364746,
      "activations/layer17_attention_weight_max": 30.46563148498535,
      "activations/layer17_attention_weight_min": -27.349063873291016,
      "activations/layer18_attention_weight_max": 31.959997177124023,
      "activations/layer18_attention_weight_min": -25.616239547729492,
      "activations/layer19_attention_weight_max": 37.599609375,
      "activations/layer19_attention_weight_min": -31.426191329956055,
      "activations/layer1_attention_weight_max": 16.688720703125,
      "activations/layer1_attention_weight_min": -13.587130546569824,
      "activations/layer20_attention_weight_max": 29.32228660583496,
      "activations/layer20_attention_weight_min": -23.558134078979492,
      "activations/layer21_attention_weight_max": 29.894256591796875,
      "activations/layer21_attention_weight_min": -22.959592819213867,
      "activations/layer22_attention_weight_max": 44.1967658996582,
      "activations/layer22_attention_weight_min": -30.62465476989746,
      "activations/layer23_attention_weight_max": 35.47563552856445,
      "activations/layer23_attention_weight_min": -26.079593658447266,
      "activations/layer2_attention_weight_max": 31.087886810302734,
      "activations/layer2_attention_weight_min": -31.868833541870117,
      "activations/layer3_attention_weight_max": 92.64054870605469,
      "activations/layer3_attention_weight_min": -94.42525482177734,
      "activations/layer4_attention_weight_max": 105.6518325805664,
      "activations/layer4_attention_weight_min": -108.28173065185547,
      "activations/layer5_attention_weight_max": 51.686492919921875,
      "activations/layer5_attention_weight_min": -62.616798400878906,
      "activations/layer6_attention_weight_max": 43.0660285949707,
      "activations/layer6_attention_weight_min": -46.38557434082031,
      "activations/layer7_attention_weight_max": 88.77232360839844,
      "activations/layer7_attention_weight_min": -91.96760559082031,
      "activations/layer8_attention_weight_max": 39.49569320678711,
      "activations/layer8_attention_weight_min": -43.58095169067383,
      "activations/layer9_attention_weight_max": 30.825834274291992,
      "activations/layer9_attention_weight_min": -33.912662506103516,
      "epoch": 14.32,
      "learning_rate": 5.823939393939393e-05,
      "loss": 2.7697,
      "step": 246450
    },
    {
      "activations/layer0_attention_weight_max": 16.06279754638672,
      "activations/layer0_attention_weight_min": -11.669975280761719,
      "activations/layer10_attention_weight_max": 34.68364334106445,
      "activations/layer10_attention_weight_min": -37.006839752197266,
      "activations/layer11_attention_weight_max": 33.699302673339844,
      "activations/layer11_attention_weight_min": -33.34856414794922,
      "activations/layer12_attention_weight_max": 26.460996627807617,
      "activations/layer12_attention_weight_min": -26.556222915649414,
      "activations/layer13_attention_weight_max": 40.93377685546875,
      "activations/layer13_attention_weight_min": -36.50632858276367,
      "activations/layer14_attention_weight_max": 42.53878402709961,
      "activations/layer14_attention_weight_min": -38.4746208190918,
      "activations/layer15_attention_weight_max": 37.36418533325195,
      "activations/layer15_attention_weight_min": -34.16445541381836,
      "activations/layer16_attention_weight_max": 29.820972442626953,
      "activations/layer16_attention_weight_min": -28.051225662231445,
      "activations/layer17_attention_weight_max": 28.756439208984375,
      "activations/layer17_attention_weight_min": -26.839706420898438,
      "activations/layer18_attention_weight_max": 32.68958282470703,
      "activations/layer18_attention_weight_min": -25.321178436279297,
      "activations/layer19_attention_weight_max": 34.91508102416992,
      "activations/layer19_attention_weight_min": -32.117828369140625,
      "activations/layer1_attention_weight_max": 16.582542419433594,
      "activations/layer1_attention_weight_min": -16.77286720275879,
      "activations/layer20_attention_weight_max": 28.867170333862305,
      "activations/layer20_attention_weight_min": -24.971439361572266,
      "activations/layer21_attention_weight_max": 29.073497772216797,
      "activations/layer21_attention_weight_min": -26.707651138305664,
      "activations/layer22_attention_weight_max": 48.05097961425781,
      "activations/layer22_attention_weight_min": -32.46384048461914,
      "activations/layer23_attention_weight_max": 33.67535400390625,
      "activations/layer23_attention_weight_min": -25.519264221191406,
      "activations/layer2_attention_weight_max": 32.901981353759766,
      "activations/layer2_attention_weight_min": -32.109046936035156,
      "activations/layer3_attention_weight_max": 96.39226531982422,
      "activations/layer3_attention_weight_min": -99.82630157470703,
      "activations/layer4_attention_weight_max": 111.41124725341797,
      "activations/layer4_attention_weight_min": -113.49153137207031,
      "activations/layer5_attention_weight_max": 52.002159118652344,
      "activations/layer5_attention_weight_min": -62.69198226928711,
      "activations/layer6_attention_weight_max": 45.61433792114258,
      "activations/layer6_attention_weight_min": -48.288394927978516,
      "activations/layer7_attention_weight_max": 101.9496078491211,
      "activations/layer7_attention_weight_min": -97.88847351074219,
      "activations/layer8_attention_weight_max": 44.435752868652344,
      "activations/layer8_attention_weight_min": -44.17513656616211,
      "activations/layer9_attention_weight_max": 32.9547004699707,
      "activations/layer9_attention_weight_min": -36.01958465576172,
      "epoch": 14.32,
      "learning_rate": 5.8220454545454534e-05,
      "loss": 2.7576,
      "step": 246500
    },
    {
      "activations/layer0_attention_weight_max": 16.20661735534668,
      "activations/layer0_attention_weight_min": -12.26901912689209,
      "activations/layer10_attention_weight_max": 34.11198425292969,
      "activations/layer10_attention_weight_min": -33.15807342529297,
      "activations/layer11_attention_weight_max": 33.246864318847656,
      "activations/layer11_attention_weight_min": -32.7923469543457,
      "activations/layer12_attention_weight_max": 26.386917114257812,
      "activations/layer12_attention_weight_min": -25.690980911254883,
      "activations/layer13_attention_weight_max": 40.545867919921875,
      "activations/layer13_attention_weight_min": -33.10045623779297,
      "activations/layer14_attention_weight_max": 48.701011657714844,
      "activations/layer14_attention_weight_min": -37.61237335205078,
      "activations/layer15_attention_weight_max": 38.763118743896484,
      "activations/layer15_attention_weight_min": -33.84209442138672,
      "activations/layer16_attention_weight_max": 30.90294647216797,
      "activations/layer16_attention_weight_min": -28.155010223388672,
      "activations/layer17_attention_weight_max": 32.748226165771484,
      "activations/layer17_attention_weight_min": -26.897626876831055,
      "activations/layer18_attention_weight_max": 33.756500244140625,
      "activations/layer18_attention_weight_min": -27.41376304626465,
      "activations/layer19_attention_weight_max": 35.316280364990234,
      "activations/layer19_attention_weight_min": -32.72244644165039,
      "activations/layer1_attention_weight_max": 17.009754180908203,
      "activations/layer1_attention_weight_min": -15.607605934143066,
      "activations/layer20_attention_weight_max": 31.52903938293457,
      "activations/layer20_attention_weight_min": -26.369529724121094,
      "activations/layer21_attention_weight_max": 32.14139938354492,
      "activations/layer21_attention_weight_min": -25.133970260620117,
      "activations/layer22_attention_weight_max": 51.908973693847656,
      "activations/layer22_attention_weight_min": -32.824607849121094,
      "activations/layer23_attention_weight_max": 34.79335021972656,
      "activations/layer23_attention_weight_min": -25.397159576416016,
      "activations/layer2_attention_weight_max": 32.2678108215332,
      "activations/layer2_attention_weight_min": -32.41501235961914,
      "activations/layer3_attention_weight_max": 95.87081146240234,
      "activations/layer3_attention_weight_min": -97.76787567138672,
      "activations/layer4_attention_weight_max": 111.10186004638672,
      "activations/layer4_attention_weight_min": -110.8417739868164,
      "activations/layer5_attention_weight_max": 53.01282501220703,
      "activations/layer5_attention_weight_min": -60.069000244140625,
      "activations/layer6_attention_weight_max": 44.06238555908203,
      "activations/layer6_attention_weight_min": -46.64997100830078,
      "activations/layer7_attention_weight_max": 92.05567932128906,
      "activations/layer7_attention_weight_min": -93.85530853271484,
      "activations/layer8_attention_weight_max": 41.58634567260742,
      "activations/layer8_attention_weight_min": -42.891357421875,
      "activations/layer9_attention_weight_max": 36.69634246826172,
      "activations/layer9_attention_weight_min": -33.95086669921875,
      "epoch": 14.33,
      "learning_rate": 5.820151515151515e-05,
      "loss": 2.757,
      "step": 246550
    },
    {
      "activations/layer0_attention_weight_max": 15.857396125793457,
      "activations/layer0_attention_weight_min": -11.707523345947266,
      "activations/layer10_attention_weight_max": 36.546852111816406,
      "activations/layer10_attention_weight_min": -33.21256637573242,
      "activations/layer11_attention_weight_max": 33.696510314941406,
      "activations/layer11_attention_weight_min": -31.646503448486328,
      "activations/layer12_attention_weight_max": 26.97516441345215,
      "activations/layer12_attention_weight_min": -24.377948760986328,
      "activations/layer13_attention_weight_max": 37.50873565673828,
      "activations/layer13_attention_weight_min": -33.3390007019043,
      "activations/layer14_attention_weight_max": 41.23701477050781,
      "activations/layer14_attention_weight_min": -36.60426712036133,
      "activations/layer15_attention_weight_max": 36.930442810058594,
      "activations/layer15_attention_weight_min": -31.96669578552246,
      "activations/layer16_attention_weight_max": 32.09623718261719,
      "activations/layer16_attention_weight_min": -30.396240234375,
      "activations/layer17_attention_weight_max": 30.64946937561035,
      "activations/layer17_attention_weight_min": -29.19936180114746,
      "activations/layer18_attention_weight_max": 32.12836456298828,
      "activations/layer18_attention_weight_min": -26.76947021484375,
      "activations/layer19_attention_weight_max": 36.00423049926758,
      "activations/layer19_attention_weight_min": -34.595558166503906,
      "activations/layer1_attention_weight_max": 17.87469482421875,
      "activations/layer1_attention_weight_min": -14.472955703735352,
      "activations/layer20_attention_weight_max": 29.709043502807617,
      "activations/layer20_attention_weight_min": -26.291481018066406,
      "activations/layer21_attention_weight_max": 27.442317962646484,
      "activations/layer21_attention_weight_min": -23.785287857055664,
      "activations/layer22_attention_weight_max": 45.72454833984375,
      "activations/layer22_attention_weight_min": -31.698986053466797,
      "activations/layer23_attention_weight_max": 33.49787139892578,
      "activations/layer23_attention_weight_min": -24.95502281188965,
      "activations/layer2_attention_weight_max": 32.7766227722168,
      "activations/layer2_attention_weight_min": -32.71092224121094,
      "activations/layer3_attention_weight_max": 93.58139038085938,
      "activations/layer3_attention_weight_min": -95.47441101074219,
      "activations/layer4_attention_weight_max": 110.7319564819336,
      "activations/layer4_attention_weight_min": -112.69671630859375,
      "activations/layer5_attention_weight_max": 50.96929931640625,
      "activations/layer5_attention_weight_min": -60.8234977722168,
      "activations/layer6_attention_weight_max": 44.08574676513672,
      "activations/layer6_attention_weight_min": -46.371131896972656,
      "activations/layer7_attention_weight_max": 87.53993225097656,
      "activations/layer7_attention_weight_min": -90.48777770996094,
      "activations/layer8_attention_weight_max": 41.39584732055664,
      "activations/layer8_attention_weight_min": -42.079795837402344,
      "activations/layer9_attention_weight_max": 32.937034606933594,
      "activations/layer9_attention_weight_min": -33.36386489868164,
      "epoch": 14.33,
      "learning_rate": 5.818257575757575e-05,
      "loss": 2.7426,
      "step": 246600
    },
    {
      "activations/layer0_attention_weight_max": 16.146116256713867,
      "activations/layer0_attention_weight_min": -14.003255844116211,
      "activations/layer10_attention_weight_max": 34.735145568847656,
      "activations/layer10_attention_weight_min": -33.199256896972656,
      "activations/layer11_attention_weight_max": 35.54313278198242,
      "activations/layer11_attention_weight_min": -33.55903625488281,
      "activations/layer12_attention_weight_max": 27.850051879882812,
      "activations/layer12_attention_weight_min": -26.351037979125977,
      "activations/layer13_attention_weight_max": 41.90366744995117,
      "activations/layer13_attention_weight_min": -35.344512939453125,
      "activations/layer14_attention_weight_max": 45.73468017578125,
      "activations/layer14_attention_weight_min": -37.71890640258789,
      "activations/layer15_attention_weight_max": 38.85423278808594,
      "activations/layer15_attention_weight_min": -36.19049072265625,
      "activations/layer16_attention_weight_max": 31.56763458251953,
      "activations/layer16_attention_weight_min": -28.785343170166016,
      "activations/layer17_attention_weight_max": 32.24980545043945,
      "activations/layer17_attention_weight_min": -27.18912696838379,
      "activations/layer18_attention_weight_max": 33.83272171020508,
      "activations/layer18_attention_weight_min": -26.11753273010254,
      "activations/layer19_attention_weight_max": 37.3940544128418,
      "activations/layer19_attention_weight_min": -33.03341293334961,
      "activations/layer1_attention_weight_max": 17.001893997192383,
      "activations/layer1_attention_weight_min": -15.106355667114258,
      "activations/layer20_attention_weight_max": 33.11511993408203,
      "activations/layer20_attention_weight_min": -26.5373592376709,
      "activations/layer21_attention_weight_max": 31.221065521240234,
      "activations/layer21_attention_weight_min": -27.275920867919922,
      "activations/layer22_attention_weight_max": 47.53528594970703,
      "activations/layer22_attention_weight_min": -37.110050201416016,
      "activations/layer23_attention_weight_max": 36.87311553955078,
      "activations/layer23_attention_weight_min": -31.247100830078125,
      "activations/layer2_attention_weight_max": 32.66300964355469,
      "activations/layer2_attention_weight_min": -33.27294921875,
      "activations/layer3_attention_weight_max": 98.15564727783203,
      "activations/layer3_attention_weight_min": -100.46365356445312,
      "activations/layer4_attention_weight_max": 109.24006652832031,
      "activations/layer4_attention_weight_min": -113.99593353271484,
      "activations/layer5_attention_weight_max": 52.56345748901367,
      "activations/layer5_attention_weight_min": -62.97739791870117,
      "activations/layer6_attention_weight_max": 44.5001335144043,
      "activations/layer6_attention_weight_min": -46.43742752075195,
      "activations/layer7_attention_weight_max": 95.62149047851562,
      "activations/layer7_attention_weight_min": -91.07746124267578,
      "activations/layer8_attention_weight_max": 43.754112243652344,
      "activations/layer8_attention_weight_min": -45.15840148925781,
      "activations/layer9_attention_weight_max": 34.177490234375,
      "activations/layer9_attention_weight_min": -34.71891784667969,
      "epoch": 14.33,
      "learning_rate": 5.816363636363636e-05,
      "loss": 2.7524,
      "step": 246650
    },
    {
      "activations/layer0_attention_weight_max": 15.378955841064453,
      "activations/layer0_attention_weight_min": -11.767762184143066,
      "activations/layer10_attention_weight_max": 32.53651809692383,
      "activations/layer10_attention_weight_min": -36.74334716796875,
      "activations/layer11_attention_weight_max": 31.985870361328125,
      "activations/layer11_attention_weight_min": -33.26626968383789,
      "activations/layer12_attention_weight_max": 28.143756866455078,
      "activations/layer12_attention_weight_min": -25.507429122924805,
      "activations/layer13_attention_weight_max": 43.32044982910156,
      "activations/layer13_attention_weight_min": -35.20789337158203,
      "activations/layer14_attention_weight_max": 44.3360710144043,
      "activations/layer14_attention_weight_min": -39.419864654541016,
      "activations/layer15_attention_weight_max": 40.64502716064453,
      "activations/layer15_attention_weight_min": -35.577796936035156,
      "activations/layer16_attention_weight_max": 30.792875289916992,
      "activations/layer16_attention_weight_min": -31.1195068359375,
      "activations/layer17_attention_weight_max": 32.50054168701172,
      "activations/layer17_attention_weight_min": -27.628955841064453,
      "activations/layer18_attention_weight_max": 33.469017028808594,
      "activations/layer18_attention_weight_min": -26.609819412231445,
      "activations/layer19_attention_weight_max": 40.209476470947266,
      "activations/layer19_attention_weight_min": -35.6263313293457,
      "activations/layer1_attention_weight_max": 17.877655029296875,
      "activations/layer1_attention_weight_min": -15.372692108154297,
      "activations/layer20_attention_weight_max": 33.45167922973633,
      "activations/layer20_attention_weight_min": -27.189023971557617,
      "activations/layer21_attention_weight_max": 33.00392532348633,
      "activations/layer21_attention_weight_min": -28.30035400390625,
      "activations/layer22_attention_weight_max": 49.4044303894043,
      "activations/layer22_attention_weight_min": -34.248138427734375,
      "activations/layer23_attention_weight_max": 36.22419738769531,
      "activations/layer23_attention_weight_min": -27.138465881347656,
      "activations/layer2_attention_weight_max": 33.25849914550781,
      "activations/layer2_attention_weight_min": -34.47121810913086,
      "activations/layer3_attention_weight_max": 96.13938903808594,
      "activations/layer3_attention_weight_min": -99.8296127319336,
      "activations/layer4_attention_weight_max": 110.50455474853516,
      "activations/layer4_attention_weight_min": -114.3529281616211,
      "activations/layer5_attention_weight_max": 51.13739013671875,
      "activations/layer5_attention_weight_min": -62.43258285522461,
      "activations/layer6_attention_weight_max": 44.216732025146484,
      "activations/layer6_attention_weight_min": -48.17118835449219,
      "activations/layer7_attention_weight_max": 93.35350799560547,
      "activations/layer7_attention_weight_min": -92.70014953613281,
      "activations/layer8_attention_weight_max": 39.70181655883789,
      "activations/layer8_attention_weight_min": -45.41267776489258,
      "activations/layer9_attention_weight_max": 31.30768394470215,
      "activations/layer9_attention_weight_min": -36.56365966796875,
      "epoch": 14.33,
      "learning_rate": 5.814469696969696e-05,
      "loss": 2.7672,
      "step": 246700
    },
    {
      "activations/layer0_attention_weight_max": 16.20473861694336,
      "activations/layer0_attention_weight_min": -12.789226531982422,
      "activations/layer10_attention_weight_max": 32.703948974609375,
      "activations/layer10_attention_weight_min": -31.53464126586914,
      "activations/layer11_attention_weight_max": 32.651405334472656,
      "activations/layer11_attention_weight_min": -31.222110748291016,
      "activations/layer12_attention_weight_max": 25.50188636779785,
      "activations/layer12_attention_weight_min": -25.997268676757812,
      "activations/layer13_attention_weight_max": 37.03609085083008,
      "activations/layer13_attention_weight_min": -33.21775817871094,
      "activations/layer14_attention_weight_max": 42.45485305786133,
      "activations/layer14_attention_weight_min": -35.61543655395508,
      "activations/layer15_attention_weight_max": 38.91019058227539,
      "activations/layer15_attention_weight_min": -32.52959060668945,
      "activations/layer16_attention_weight_max": 28.910606384277344,
      "activations/layer16_attention_weight_min": -27.2098388671875,
      "activations/layer17_attention_weight_max": 29.49778175354004,
      "activations/layer17_attention_weight_min": -25.01380729675293,
      "activations/layer18_attention_weight_max": 30.51190185546875,
      "activations/layer18_attention_weight_min": -22.446125030517578,
      "activations/layer19_attention_weight_max": 39.11251449584961,
      "activations/layer19_attention_weight_min": -29.615375518798828,
      "activations/layer1_attention_weight_max": 16.98637580871582,
      "activations/layer1_attention_weight_min": -16.17561149597168,
      "activations/layer20_attention_weight_max": 32.075740814208984,
      "activations/layer20_attention_weight_min": -25.015413284301758,
      "activations/layer21_attention_weight_max": 30.244853973388672,
      "activations/layer21_attention_weight_min": -23.73332405090332,
      "activations/layer22_attention_weight_max": 43.588623046875,
      "activations/layer22_attention_weight_min": -30.954933166503906,
      "activations/layer23_attention_weight_max": 34.04828643798828,
      "activations/layer23_attention_weight_min": -24.066844940185547,
      "activations/layer2_attention_weight_max": 34.61339569091797,
      "activations/layer2_attention_weight_min": -33.37562561035156,
      "activations/layer3_attention_weight_max": 101.09307098388672,
      "activations/layer3_attention_weight_min": -99.79132843017578,
      "activations/layer4_attention_weight_max": 114.94771575927734,
      "activations/layer4_attention_weight_min": -109.25008392333984,
      "activations/layer5_attention_weight_max": 53.91912078857422,
      "activations/layer5_attention_weight_min": -64.19632720947266,
      "activations/layer6_attention_weight_max": 43.82746124267578,
      "activations/layer6_attention_weight_min": -44.02313995361328,
      "activations/layer7_attention_weight_max": 88.63402557373047,
      "activations/layer7_attention_weight_min": -88.98916625976562,
      "activations/layer8_attention_weight_max": 39.636985778808594,
      "activations/layer8_attention_weight_min": -40.82558059692383,
      "activations/layer9_attention_weight_max": 30.420791625976562,
      "activations/layer9_attention_weight_min": -31.11886978149414,
      "epoch": 14.34,
      "learning_rate": 5.8125757575757575e-05,
      "loss": 2.7579,
      "step": 246750
    },
    {
      "activations/layer0_attention_weight_max": 16.075843811035156,
      "activations/layer0_attention_weight_min": -12.49567985534668,
      "activations/layer10_attention_weight_max": 34.78397750854492,
      "activations/layer10_attention_weight_min": -35.514610290527344,
      "activations/layer11_attention_weight_max": 36.35021209716797,
      "activations/layer11_attention_weight_min": -37.02202224731445,
      "activations/layer12_attention_weight_max": 27.9155216217041,
      "activations/layer12_attention_weight_min": -27.23164176940918,
      "activations/layer13_attention_weight_max": 41.72901916503906,
      "activations/layer13_attention_weight_min": -36.978363037109375,
      "activations/layer14_attention_weight_max": 44.57406234741211,
      "activations/layer14_attention_weight_min": -39.67753219604492,
      "activations/layer15_attention_weight_max": 41.420780181884766,
      "activations/layer15_attention_weight_min": -37.41930389404297,
      "activations/layer16_attention_weight_max": 35.14042282104492,
      "activations/layer16_attention_weight_min": -28.216264724731445,
      "activations/layer17_attention_weight_max": 34.65807342529297,
      "activations/layer17_attention_weight_min": -27.01127815246582,
      "activations/layer18_attention_weight_max": 34.647491455078125,
      "activations/layer18_attention_weight_min": -25.76043701171875,
      "activations/layer19_attention_weight_max": 35.85306930541992,
      "activations/layer19_attention_weight_min": -31.0382137298584,
      "activations/layer1_attention_weight_max": 18.127552032470703,
      "activations/layer1_attention_weight_min": -15.283642768859863,
      "activations/layer20_attention_weight_max": 28.298534393310547,
      "activations/layer20_attention_weight_min": -24.486572265625,
      "activations/layer21_attention_weight_max": 30.83908462524414,
      "activations/layer21_attention_weight_min": -23.075586318969727,
      "activations/layer22_attention_weight_max": 40.53607940673828,
      "activations/layer22_attention_weight_min": -30.435565948486328,
      "activations/layer23_attention_weight_max": 33.2674446105957,
      "activations/layer23_attention_weight_min": -27.05777359008789,
      "activations/layer2_attention_weight_max": 34.190399169921875,
      "activations/layer2_attention_weight_min": -34.87953567504883,
      "activations/layer3_attention_weight_max": 105.58384704589844,
      "activations/layer3_attention_weight_min": -107.97760009765625,
      "activations/layer4_attention_weight_max": 119.59065246582031,
      "activations/layer4_attention_weight_min": -117.63768768310547,
      "activations/layer5_attention_weight_max": 53.594261169433594,
      "activations/layer5_attention_weight_min": -63.072021484375,
      "activations/layer6_attention_weight_max": 45.662513732910156,
      "activations/layer6_attention_weight_min": -48.372703552246094,
      "activations/layer7_attention_weight_max": 94.9083480834961,
      "activations/layer7_attention_weight_min": -99.1507339477539,
      "activations/layer8_attention_weight_max": 43.410797119140625,
      "activations/layer8_attention_weight_min": -43.36748123168945,
      "activations/layer9_attention_weight_max": 34.573734283447266,
      "activations/layer9_attention_weight_min": -34.40131378173828,
      "epoch": 14.34,
      "learning_rate": 5.810681818181818e-05,
      "loss": 2.7433,
      "step": 246800
    },
    {
      "activations/layer0_attention_weight_max": 16.078445434570312,
      "activations/layer0_attention_weight_min": -13.355798721313477,
      "activations/layer10_attention_weight_max": 32.45647430419922,
      "activations/layer10_attention_weight_min": -33.92384719848633,
      "activations/layer11_attention_weight_max": 32.59995651245117,
      "activations/layer11_attention_weight_min": -34.35746765136719,
      "activations/layer12_attention_weight_max": 25.4206600189209,
      "activations/layer12_attention_weight_min": -25.86217498779297,
      "activations/layer13_attention_weight_max": 39.07743453979492,
      "activations/layer13_attention_weight_min": -37.15888214111328,
      "activations/layer14_attention_weight_max": 41.33647537231445,
      "activations/layer14_attention_weight_min": -37.964717864990234,
      "activations/layer15_attention_weight_max": 36.704833984375,
      "activations/layer15_attention_weight_min": -34.40310287475586,
      "activations/layer16_attention_weight_max": 31.061023712158203,
      "activations/layer16_attention_weight_min": -29.05577278137207,
      "activations/layer17_attention_weight_max": 33.9111328125,
      "activations/layer17_attention_weight_min": -27.926870346069336,
      "activations/layer18_attention_weight_max": 33.27113723754883,
      "activations/layer18_attention_weight_min": -24.5639591217041,
      "activations/layer19_attention_weight_max": 33.808006286621094,
      "activations/layer19_attention_weight_min": -33.534629821777344,
      "activations/layer1_attention_weight_max": 16.87721061706543,
      "activations/layer1_attention_weight_min": -15.876789093017578,
      "activations/layer20_attention_weight_max": 30.67708969116211,
      "activations/layer20_attention_weight_min": -27.009069442749023,
      "activations/layer21_attention_weight_max": 32.65199661254883,
      "activations/layer21_attention_weight_min": -25.48748016357422,
      "activations/layer22_attention_weight_max": 45.37812042236328,
      "activations/layer22_attention_weight_min": -30.594758987426758,
      "activations/layer23_attention_weight_max": 32.36102294921875,
      "activations/layer23_attention_weight_min": -25.338001251220703,
      "activations/layer2_attention_weight_max": 30.82123565673828,
      "activations/layer2_attention_weight_min": -32.089141845703125,
      "activations/layer3_attention_weight_max": 97.30734252929688,
      "activations/layer3_attention_weight_min": -99.4725570678711,
      "activations/layer4_attention_weight_max": 112.32928466796875,
      "activations/layer4_attention_weight_min": -113.50432586669922,
      "activations/layer5_attention_weight_max": 52.13166809082031,
      "activations/layer5_attention_weight_min": -64.43187713623047,
      "activations/layer6_attention_weight_max": 44.64545440673828,
      "activations/layer6_attention_weight_min": -47.937843322753906,
      "activations/layer7_attention_weight_max": 95.09388732910156,
      "activations/layer7_attention_weight_min": -98.25448608398438,
      "activations/layer8_attention_weight_max": 41.06492233276367,
      "activations/layer8_attention_weight_min": -44.1290397644043,
      "activations/layer9_attention_weight_max": 32.400516510009766,
      "activations/layer9_attention_weight_min": -35.393009185791016,
      "epoch": 14.34,
      "learning_rate": 5.808787878787878e-05,
      "loss": 2.7527,
      "step": 246850
    },
    {
      "activations/layer0_attention_weight_max": 15.8810453414917,
      "activations/layer0_attention_weight_min": -13.636431694030762,
      "activations/layer10_attention_weight_max": 33.00672149658203,
      "activations/layer10_attention_weight_min": -33.7068977355957,
      "activations/layer11_attention_weight_max": 33.08015441894531,
      "activations/layer11_attention_weight_min": -33.2947998046875,
      "activations/layer12_attention_weight_max": 26.42744255065918,
      "activations/layer12_attention_weight_min": -25.326311111450195,
      "activations/layer13_attention_weight_max": 43.69135665893555,
      "activations/layer13_attention_weight_min": -35.5617790222168,
      "activations/layer14_attention_weight_max": 43.91886520385742,
      "activations/layer14_attention_weight_min": -36.510658264160156,
      "activations/layer15_attention_weight_max": 43.05778121948242,
      "activations/layer15_attention_weight_min": -35.30484390258789,
      "activations/layer16_attention_weight_max": 32.615909576416016,
      "activations/layer16_attention_weight_min": -28.54828643798828,
      "activations/layer17_attention_weight_max": 32.7020149230957,
      "activations/layer17_attention_weight_min": -26.177379608154297,
      "activations/layer18_attention_weight_max": 32.65113830566406,
      "activations/layer18_attention_weight_min": -24.865604400634766,
      "activations/layer19_attention_weight_max": 36.76900863647461,
      "activations/layer19_attention_weight_min": -33.19223403930664,
      "activations/layer1_attention_weight_max": 17.122419357299805,
      "activations/layer1_attention_weight_min": -16.277565002441406,
      "activations/layer20_attention_weight_max": 29.231170654296875,
      "activations/layer20_attention_weight_min": -24.668590545654297,
      "activations/layer21_attention_weight_max": 33.053157806396484,
      "activations/layer21_attention_weight_min": -23.77128028869629,
      "activations/layer22_attention_weight_max": 42.386741638183594,
      "activations/layer22_attention_weight_min": -31.517080307006836,
      "activations/layer23_attention_weight_max": 33.022377014160156,
      "activations/layer23_attention_weight_min": -28.312923431396484,
      "activations/layer2_attention_weight_max": 34.786643981933594,
      "activations/layer2_attention_weight_min": -34.035186767578125,
      "activations/layer3_attention_weight_max": 97.34780883789062,
      "activations/layer3_attention_weight_min": -97.05670928955078,
      "activations/layer4_attention_weight_max": 110.96369934082031,
      "activations/layer4_attention_weight_min": -111.11168670654297,
      "activations/layer5_attention_weight_max": 50.61367416381836,
      "activations/layer5_attention_weight_min": -61.756893157958984,
      "activations/layer6_attention_weight_max": 43.46043014526367,
      "activations/layer6_attention_weight_min": -46.696861267089844,
      "activations/layer7_attention_weight_max": 90.03905487060547,
      "activations/layer7_attention_weight_min": -86.96367645263672,
      "activations/layer8_attention_weight_max": 41.2764778137207,
      "activations/layer8_attention_weight_min": -42.508026123046875,
      "activations/layer9_attention_weight_max": 34.650691986083984,
      "activations/layer9_attention_weight_min": -34.30013656616211,
      "epoch": 14.35,
      "learning_rate": 5.806893939393939e-05,
      "loss": 2.7529,
      "step": 246900
    },
    {
      "activations/layer0_attention_weight_max": 15.536110877990723,
      "activations/layer0_attention_weight_min": -12.563667297363281,
      "activations/layer10_attention_weight_max": 32.19930648803711,
      "activations/layer10_attention_weight_min": -33.67598342895508,
      "activations/layer11_attention_weight_max": 32.75377655029297,
      "activations/layer11_attention_weight_min": -31.77605438232422,
      "activations/layer12_attention_weight_max": 25.23117446899414,
      "activations/layer12_attention_weight_min": -26.010395050048828,
      "activations/layer13_attention_weight_max": 36.11702346801758,
      "activations/layer13_attention_weight_min": -35.18688201904297,
      "activations/layer14_attention_weight_max": 41.967491149902344,
      "activations/layer14_attention_weight_min": -38.77863693237305,
      "activations/layer15_attention_weight_max": 38.346221923828125,
      "activations/layer15_attention_weight_min": -32.06399917602539,
      "activations/layer16_attention_weight_max": 31.75958251953125,
      "activations/layer16_attention_weight_min": -28.63651466369629,
      "activations/layer17_attention_weight_max": 32.68452835083008,
      "activations/layer17_attention_weight_min": -25.91379737854004,
      "activations/layer18_attention_weight_max": 33.11397933959961,
      "activations/layer18_attention_weight_min": -25.35474395751953,
      "activations/layer19_attention_weight_max": 34.262351989746094,
      "activations/layer19_attention_weight_min": -33.600868225097656,
      "activations/layer1_attention_weight_max": 16.577028274536133,
      "activations/layer1_attention_weight_min": -15.92723274230957,
      "activations/layer20_attention_weight_max": 28.823762893676758,
      "activations/layer20_attention_weight_min": -27.083044052124023,
      "activations/layer21_attention_weight_max": 29.370378494262695,
      "activations/layer21_attention_weight_min": -25.943334579467773,
      "activations/layer22_attention_weight_max": 44.390281677246094,
      "activations/layer22_attention_weight_min": -30.7801513671875,
      "activations/layer23_attention_weight_max": 31.39813804626465,
      "activations/layer23_attention_weight_min": -25.22456932067871,
      "activations/layer2_attention_weight_max": 33.32714080810547,
      "activations/layer2_attention_weight_min": -31.796016693115234,
      "activations/layer3_attention_weight_max": 95.6461181640625,
      "activations/layer3_attention_weight_min": -93.18798065185547,
      "activations/layer4_attention_weight_max": 105.70748138427734,
      "activations/layer4_attention_weight_min": -104.89019775390625,
      "activations/layer5_attention_weight_max": 49.503807067871094,
      "activations/layer5_attention_weight_min": -65.4416275024414,
      "activations/layer6_attention_weight_max": 43.69956588745117,
      "activations/layer6_attention_weight_min": -44.43455505371094,
      "activations/layer7_attention_weight_max": 84.5146484375,
      "activations/layer7_attention_weight_min": -87.0191421508789,
      "activations/layer8_attention_weight_max": 40.35137176513672,
      "activations/layer8_attention_weight_min": -40.283416748046875,
      "activations/layer9_attention_weight_max": 30.697559356689453,
      "activations/layer9_attention_weight_min": -32.178585052490234,
      "epoch": 14.35,
      "learning_rate": 5.8049999999999995e-05,
      "loss": 2.7466,
      "step": 246950
    },
    {
      "activations/layer0_attention_weight_max": 15.498095512390137,
      "activations/layer0_attention_weight_min": -12.956879615783691,
      "activations/layer10_attention_weight_max": 32.75529479980469,
      "activations/layer10_attention_weight_min": -33.013694763183594,
      "activations/layer11_attention_weight_max": 34.808956146240234,
      "activations/layer11_attention_weight_min": -32.60478210449219,
      "activations/layer12_attention_weight_max": 34.39080047607422,
      "activations/layer12_attention_weight_min": -26.13197135925293,
      "activations/layer13_attention_weight_max": 44.17789077758789,
      "activations/layer13_attention_weight_min": -35.07956314086914,
      "activations/layer14_attention_weight_max": 42.046241760253906,
      "activations/layer14_attention_weight_min": -36.369197845458984,
      "activations/layer15_attention_weight_max": 37.23915481567383,
      "activations/layer15_attention_weight_min": -33.634254455566406,
      "activations/layer16_attention_weight_max": 29.15439796447754,
      "activations/layer16_attention_weight_min": -28.087203979492188,
      "activations/layer17_attention_weight_max": 30.155385971069336,
      "activations/layer17_attention_weight_min": -26.083463668823242,
      "activations/layer18_attention_weight_max": 32.7612419128418,
      "activations/layer18_attention_weight_min": -25.369863510131836,
      "activations/layer19_attention_weight_max": 33.735687255859375,
      "activations/layer19_attention_weight_min": -32.56577682495117,
      "activations/layer1_attention_weight_max": 17.377532958984375,
      "activations/layer1_attention_weight_min": -16.083303451538086,
      "activations/layer20_attention_weight_max": 30.421953201293945,
      "activations/layer20_attention_weight_min": -24.767444610595703,
      "activations/layer21_attention_weight_max": 28.8170223236084,
      "activations/layer21_attention_weight_min": -25.13786506652832,
      "activations/layer22_attention_weight_max": 40.67919921875,
      "activations/layer22_attention_weight_min": -31.173154830932617,
      "activations/layer23_attention_weight_max": 36.17246627807617,
      "activations/layer23_attention_weight_min": -25.842527389526367,
      "activations/layer2_attention_weight_max": 31.905715942382812,
      "activations/layer2_attention_weight_min": -31.945903778076172,
      "activations/layer3_attention_weight_max": 93.991455078125,
      "activations/layer3_attention_weight_min": -93.65132141113281,
      "activations/layer4_attention_weight_max": 108.2870101928711,
      "activations/layer4_attention_weight_min": -109.1916275024414,
      "activations/layer5_attention_weight_max": 50.15019226074219,
      "activations/layer5_attention_weight_min": -62.61648178100586,
      "activations/layer6_attention_weight_max": 44.69070053100586,
      "activations/layer6_attention_weight_min": -44.98283767700195,
      "activations/layer7_attention_weight_max": 87.54751586914062,
      "activations/layer7_attention_weight_min": -88.12893676757812,
      "activations/layer8_attention_weight_max": 38.820457458496094,
      "activations/layer8_attention_weight_min": -41.26322937011719,
      "activations/layer9_attention_weight_max": 32.77477264404297,
      "activations/layer9_attention_weight_min": -34.115020751953125,
      "epoch": 14.35,
      "learning_rate": 5.80310606060606e-05,
      "loss": 2.7514,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5214,
      "eval_samples_per_second": 503.905,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5214,
      "eval_openwebtext_samples_per_second": 503.905,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 2.0018,
      "eval_wikitext_samples_per_second": 227.795,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_lambada_loss": 2.490234375,
      "eval_lambada_ppl": 12.064103313334172,
      "eval_lambada_runtime": 9.595,
      "eval_lambada_samples_per_second": 507.452,
      "step": 247000
    },
    {
      "activations/layer0_attention_weight_max": 15.609684944152832,
      "activations/layer0_attention_weight_min": -13.757553100585938,
      "activations/layer10_attention_weight_max": 31.31585693359375,
      "activations/layer10_attention_weight_min": -33.2401123046875,
      "activations/layer11_attention_weight_max": 32.18302917480469,
      "activations/layer11_attention_weight_min": -33.26618957519531,
      "activations/layer12_attention_weight_max": 28.597997665405273,
      "activations/layer12_attention_weight_min": -25.351030349731445,
      "activations/layer13_attention_weight_max": 41.46796417236328,
      "activations/layer13_attention_weight_min": -34.58991241455078,
      "activations/layer14_attention_weight_max": 42.512306213378906,
      "activations/layer14_attention_weight_min": -36.416133880615234,
      "activations/layer15_attention_weight_max": 37.91197967529297,
      "activations/layer15_attention_weight_min": -33.55397033691406,
      "activations/layer16_attention_weight_max": 29.774192810058594,
      "activations/layer16_attention_weight_min": -27.38610076904297,
      "activations/layer17_attention_weight_max": 29.742048263549805,
      "activations/layer17_attention_weight_min": -24.904172897338867,
      "activations/layer18_attention_weight_max": 30.2407169342041,
      "activations/layer18_attention_weight_min": -24.411529541015625,
      "activations/layer19_attention_weight_max": 31.16693687438965,
      "activations/layer19_attention_weight_min": -32.61568832397461,
      "activations/layer1_attention_weight_max": 16.439523696899414,
      "activations/layer1_attention_weight_min": -11.150656700134277,
      "activations/layer20_attention_weight_max": 28.30389976501465,
      "activations/layer20_attention_weight_min": -25.09004783630371,
      "activations/layer21_attention_weight_max": 27.88539695739746,
      "activations/layer21_attention_weight_min": -24.57707977294922,
      "activations/layer22_attention_weight_max": 40.58818435668945,
      "activations/layer22_attention_weight_min": -28.04405403137207,
      "activations/layer23_attention_weight_max": 31.456180572509766,
      "activations/layer23_attention_weight_min": -25.161178588867188,
      "activations/layer2_attention_weight_max": 29.83887481689453,
      "activations/layer2_attention_weight_min": -28.94545555114746,
      "activations/layer3_attention_weight_max": 85.68721008300781,
      "activations/layer3_attention_weight_min": -86.50110626220703,
      "activations/layer4_attention_weight_max": 99.18981170654297,
      "activations/layer4_attention_weight_min": -104.2279052734375,
      "activations/layer5_attention_weight_max": 47.37620162963867,
      "activations/layer5_attention_weight_min": -57.27172088623047,
      "activations/layer6_attention_weight_max": 42.66917037963867,
      "activations/layer6_attention_weight_min": -47.468421936035156,
      "activations/layer7_attention_weight_max": 86.89385223388672,
      "activations/layer7_attention_weight_min": -90.94950103759766,
      "activations/layer8_attention_weight_max": 38.94776916503906,
      "activations/layer8_attention_weight_min": -40.9348030090332,
      "activations/layer9_attention_weight_max": 31.387130737304688,
      "activations/layer9_attention_weight_min": -31.818098068237305,
      "epoch": 14.36,
      "learning_rate": 5.8012121212121205e-05,
      "loss": 2.7472,
      "step": 247050
    },
    {
      "activations/layer0_attention_weight_max": 15.430867195129395,
      "activations/layer0_attention_weight_min": -13.346741676330566,
      "activations/layer10_attention_weight_max": 30.486223220825195,
      "activations/layer10_attention_weight_min": -31.97917938232422,
      "activations/layer11_attention_weight_max": 30.57842254638672,
      "activations/layer11_attention_weight_min": -31.3255672454834,
      "activations/layer12_attention_weight_max": 22.865394592285156,
      "activations/layer12_attention_weight_min": -24.72894287109375,
      "activations/layer13_attention_weight_max": 35.12791442871094,
      "activations/layer13_attention_weight_min": -33.24861526489258,
      "activations/layer14_attention_weight_max": 36.85982131958008,
      "activations/layer14_attention_weight_min": -35.33399963378906,
      "activations/layer15_attention_weight_max": 32.49183654785156,
      "activations/layer15_attention_weight_min": -33.44919204711914,
      "activations/layer16_attention_weight_max": 26.75800323486328,
      "activations/layer16_attention_weight_min": -25.696672439575195,
      "activations/layer17_attention_weight_max": 31.1527099609375,
      "activations/layer17_attention_weight_min": -26.011085510253906,
      "activations/layer18_attention_weight_max": 27.67913818359375,
      "activations/layer18_attention_weight_min": -24.161287307739258,
      "activations/layer19_attention_weight_max": 29.508975982666016,
      "activations/layer19_attention_weight_min": -31.34329605102539,
      "activations/layer1_attention_weight_max": 16.906070709228516,
      "activations/layer1_attention_weight_min": -14.746315002441406,
      "activations/layer20_attention_weight_max": 28.229995727539062,
      "activations/layer20_attention_weight_min": -26.283092498779297,
      "activations/layer21_attention_weight_max": 26.767606735229492,
      "activations/layer21_attention_weight_min": -23.69895362854004,
      "activations/layer22_attention_weight_max": 38.07658386230469,
      "activations/layer22_attention_weight_min": -31.529979705810547,
      "activations/layer23_attention_weight_max": 31.731542587280273,
      "activations/layer23_attention_weight_min": -25.096885681152344,
      "activations/layer2_attention_weight_max": 31.3201961517334,
      "activations/layer2_attention_weight_min": -34.2088737487793,
      "activations/layer3_attention_weight_max": 91.96234893798828,
      "activations/layer3_attention_weight_min": -97.9003677368164,
      "activations/layer4_attention_weight_max": 103.8661117553711,
      "activations/layer4_attention_weight_min": -107.55989837646484,
      "activations/layer5_attention_weight_max": 49.063514709472656,
      "activations/layer5_attention_weight_min": -60.417694091796875,
      "activations/layer6_attention_weight_max": 40.4652099609375,
      "activations/layer6_attention_weight_min": -46.03068542480469,
      "activations/layer7_attention_weight_max": 83.30591583251953,
      "activations/layer7_attention_weight_min": -86.33882141113281,
      "activations/layer8_attention_weight_max": 37.92662811279297,
      "activations/layer8_attention_weight_min": -39.99637985229492,
      "activations/layer9_attention_weight_max": 31.87662124633789,
      "activations/layer9_attention_weight_min": -31.639860153198242,
      "epoch": 14.36,
      "learning_rate": 5.799318181818182e-05,
      "loss": 2.7489,
      "step": 247100
    },
    {
      "activations/layer0_attention_weight_max": 15.388914108276367,
      "activations/layer0_attention_weight_min": -12.404339790344238,
      "activations/layer10_attention_weight_max": 30.68838882446289,
      "activations/layer10_attention_weight_min": -32.57147216796875,
      "activations/layer11_attention_weight_max": 30.813018798828125,
      "activations/layer11_attention_weight_min": -31.679988861083984,
      "activations/layer12_attention_weight_max": 24.038480758666992,
      "activations/layer12_attention_weight_min": -25.67772102355957,
      "activations/layer13_attention_weight_max": 35.89767837524414,
      "activations/layer13_attention_weight_min": -33.426727294921875,
      "activations/layer14_attention_weight_max": 38.71559143066406,
      "activations/layer14_attention_weight_min": -36.27852249145508,
      "activations/layer15_attention_weight_max": 34.37725067138672,
      "activations/layer15_attention_weight_min": -34.6397819519043,
      "activations/layer16_attention_weight_max": 27.46164321899414,
      "activations/layer16_attention_weight_min": -28.28830337524414,
      "activations/layer17_attention_weight_max": 30.024126052856445,
      "activations/layer17_attention_weight_min": -28.105613708496094,
      "activations/layer18_attention_weight_max": 33.364925384521484,
      "activations/layer18_attention_weight_min": -27.2718505859375,
      "activations/layer19_attention_weight_max": 30.780563354492188,
      "activations/layer19_attention_weight_min": -32.72785568237305,
      "activations/layer1_attention_weight_max": 15.807992935180664,
      "activations/layer1_attention_weight_min": -16.170684814453125,
      "activations/layer20_attention_weight_max": 28.7186336517334,
      "activations/layer20_attention_weight_min": -27.333709716796875,
      "activations/layer21_attention_weight_max": 28.03743553161621,
      "activations/layer21_attention_weight_min": -26.279983520507812,
      "activations/layer22_attention_weight_max": 42.945953369140625,
      "activations/layer22_attention_weight_min": -31.636777877807617,
      "activations/layer23_attention_weight_max": 32.83848571777344,
      "activations/layer23_attention_weight_min": -27.64563751220703,
      "activations/layer2_attention_weight_max": 32.31975555419922,
      "activations/layer2_attention_weight_min": -30.619747161865234,
      "activations/layer3_attention_weight_max": 92.85612487792969,
      "activations/layer3_attention_weight_min": -96.39148712158203,
      "activations/layer4_attention_weight_max": 107.23387908935547,
      "activations/layer4_attention_weight_min": -114.9570541381836,
      "activations/layer5_attention_weight_max": 53.39852523803711,
      "activations/layer5_attention_weight_min": -63.07249069213867,
      "activations/layer6_attention_weight_max": 43.64707565307617,
      "activations/layer6_attention_weight_min": -47.145015716552734,
      "activations/layer7_attention_weight_max": 90.1080322265625,
      "activations/layer7_attention_weight_min": -92.43150329589844,
      "activations/layer8_attention_weight_max": 39.5347785949707,
      "activations/layer8_attention_weight_min": -41.66971206665039,
      "activations/layer9_attention_weight_max": 31.886167526245117,
      "activations/layer9_attention_weight_min": -32.77219772338867,
      "epoch": 14.36,
      "learning_rate": 5.797462121212121e-05,
      "loss": 2.768,
      "step": 247150
    },
    {
      "activations/layer0_attention_weight_max": 15.76276969909668,
      "activations/layer0_attention_weight_min": -13.441128730773926,
      "activations/layer10_attention_weight_max": 31.37225341796875,
      "activations/layer10_attention_weight_min": -33.13520431518555,
      "activations/layer11_attention_weight_max": 32.96091842651367,
      "activations/layer11_attention_weight_min": -31.9058837890625,
      "activations/layer12_attention_weight_max": 27.127805709838867,
      "activations/layer12_attention_weight_min": -26.326215744018555,
      "activations/layer13_attention_weight_max": 42.663368225097656,
      "activations/layer13_attention_weight_min": -35.73379898071289,
      "activations/layer14_attention_weight_max": 47.23075866699219,
      "activations/layer14_attention_weight_min": -37.97585678100586,
      "activations/layer15_attention_weight_max": 40.17395782470703,
      "activations/layer15_attention_weight_min": -34.35691833496094,
      "activations/layer16_attention_weight_max": 32.901634216308594,
      "activations/layer16_attention_weight_min": -29.297868728637695,
      "activations/layer17_attention_weight_max": 32.6711311340332,
      "activations/layer17_attention_weight_min": -25.73720359802246,
      "activations/layer18_attention_weight_max": 33.4986572265625,
      "activations/layer18_attention_weight_min": -24.627885818481445,
      "activations/layer19_attention_weight_max": 39.66151428222656,
      "activations/layer19_attention_weight_min": -31.34229850769043,
      "activations/layer1_attention_weight_max": 16.79384422302246,
      "activations/layer1_attention_weight_min": -15.35696029663086,
      "activations/layer20_attention_weight_max": 32.737186431884766,
      "activations/layer20_attention_weight_min": -25.761137008666992,
      "activations/layer21_attention_weight_max": 31.028518676757812,
      "activations/layer21_attention_weight_min": -24.951292037963867,
      "activations/layer22_attention_weight_max": 42.105777740478516,
      "activations/layer22_attention_weight_min": -32.85056686401367,
      "activations/layer23_attention_weight_max": 35.68091583251953,
      "activations/layer23_attention_weight_min": -26.73343276977539,
      "activations/layer2_attention_weight_max": 29.907669067382812,
      "activations/layer2_attention_weight_min": -30.345741271972656,
      "activations/layer3_attention_weight_max": 90.20882415771484,
      "activations/layer3_attention_weight_min": -95.04256439208984,
      "activations/layer4_attention_weight_max": 109.9098129272461,
      "activations/layer4_attention_weight_min": -114.05667877197266,
      "activations/layer5_attention_weight_max": 50.2066650390625,
      "activations/layer5_attention_weight_min": -63.808815002441406,
      "activations/layer6_attention_weight_max": 43.50636291503906,
      "activations/layer6_attention_weight_min": -48.417606353759766,
      "activations/layer7_attention_weight_max": 89.30061340332031,
      "activations/layer7_attention_weight_min": -94.29463195800781,
      "activations/layer8_attention_weight_max": 37.6459846496582,
      "activations/layer8_attention_weight_min": -42.22212219238281,
      "activations/layer9_attention_weight_max": 31.492874145507812,
      "activations/layer9_attention_weight_min": -33.22553253173828,
      "epoch": 14.36,
      "learning_rate": 5.795568181818181e-05,
      "loss": 2.7614,
      "step": 247200
    },
    {
      "activations/layer0_attention_weight_max": 15.658891677856445,
      "activations/layer0_attention_weight_min": -12.741252899169922,
      "activations/layer10_attention_weight_max": 30.62401580810547,
      "activations/layer10_attention_weight_min": -31.679061889648438,
      "activations/layer11_attention_weight_max": 31.572975158691406,
      "activations/layer11_attention_weight_min": -32.63558578491211,
      "activations/layer12_attention_weight_max": 24.470314025878906,
      "activations/layer12_attention_weight_min": -25.73130989074707,
      "activations/layer13_attention_weight_max": 37.506099700927734,
      "activations/layer13_attention_weight_min": -34.678001403808594,
      "activations/layer14_attention_weight_max": 40.87193298339844,
      "activations/layer14_attention_weight_min": -37.21299362182617,
      "activations/layer15_attention_weight_max": 37.466209411621094,
      "activations/layer15_attention_weight_min": -34.467098236083984,
      "activations/layer16_attention_weight_max": 31.129865646362305,
      "activations/layer16_attention_weight_min": -29.126401901245117,
      "activations/layer17_attention_weight_max": 32.407203674316406,
      "activations/layer17_attention_weight_min": -26.15163230895996,
      "activations/layer18_attention_weight_max": 33.31757354736328,
      "activations/layer18_attention_weight_min": -25.45111656188965,
      "activations/layer19_attention_weight_max": 34.88674545288086,
      "activations/layer19_attention_weight_min": -32.14797592163086,
      "activations/layer1_attention_weight_max": 17.3005313873291,
      "activations/layer1_attention_weight_min": -16.24200439453125,
      "activations/layer20_attention_weight_max": 30.554283142089844,
      "activations/layer20_attention_weight_min": -26.025516510009766,
      "activations/layer21_attention_weight_max": 34.59873962402344,
      "activations/layer21_attention_weight_min": -25.511472702026367,
      "activations/layer22_attention_weight_max": 41.803672790527344,
      "activations/layer22_attention_weight_min": -31.028915405273438,
      "activations/layer23_attention_weight_max": 35.04387664794922,
      "activations/layer23_attention_weight_min": -28.577306747436523,
      "activations/layer2_attention_weight_max": 28.999740600585938,
      "activations/layer2_attention_weight_min": -28.32001304626465,
      "activations/layer3_attention_weight_max": 93.91405487060547,
      "activations/layer3_attention_weight_min": -87.85762786865234,
      "activations/layer4_attention_weight_max": 109.55533599853516,
      "activations/layer4_attention_weight_min": -107.6634521484375,
      "activations/layer5_attention_weight_max": 51.725303649902344,
      "activations/layer5_attention_weight_min": -59.6064453125,
      "activations/layer6_attention_weight_max": 45.20901870727539,
      "activations/layer6_attention_weight_min": -49.30811309814453,
      "activations/layer7_attention_weight_max": 87.32510375976562,
      "activations/layer7_attention_weight_min": -91.15341186523438,
      "activations/layer8_attention_weight_max": 37.24702835083008,
      "activations/layer8_attention_weight_min": -43.32084655761719,
      "activations/layer9_attention_weight_max": 31.02619743347168,
      "activations/layer9_attention_weight_min": -32.70295715332031,
      "epoch": 14.37,
      "learning_rate": 5.7936742424242413e-05,
      "loss": 2.7716,
      "step": 247250
    },
    {
      "activations/layer0_attention_weight_max": 15.877445220947266,
      "activations/layer0_attention_weight_min": -12.749812126159668,
      "activations/layer10_attention_weight_max": 31.665678024291992,
      "activations/layer10_attention_weight_min": -33.89894485473633,
      "activations/layer11_attention_weight_max": 32.154014587402344,
      "activations/layer11_attention_weight_min": -32.26152420043945,
      "activations/layer12_attention_weight_max": 26.88041114807129,
      "activations/layer12_attention_weight_min": -26.724985122680664,
      "activations/layer13_attention_weight_max": 41.46501922607422,
      "activations/layer13_attention_weight_min": -35.19728088378906,
      "activations/layer14_attention_weight_max": 42.854496002197266,
      "activations/layer14_attention_weight_min": -36.66841506958008,
      "activations/layer15_attention_weight_max": 38.07279586791992,
      "activations/layer15_attention_weight_min": -33.46637725830078,
      "activations/layer16_attention_weight_max": 30.669294357299805,
      "activations/layer16_attention_weight_min": -26.75432014465332,
      "activations/layer17_attention_weight_max": 31.72262191772461,
      "activations/layer17_attention_weight_min": -25.106861114501953,
      "activations/layer18_attention_weight_max": 35.99008560180664,
      "activations/layer18_attention_weight_min": -24.978900909423828,
      "activations/layer19_attention_weight_max": 35.172019958496094,
      "activations/layer19_attention_weight_min": -28.552871704101562,
      "activations/layer1_attention_weight_max": 17.683273315429688,
      "activations/layer1_attention_weight_min": -15.440226554870605,
      "activations/layer20_attention_weight_max": 32.14148712158203,
      "activations/layer20_attention_weight_min": -25.020788192749023,
      "activations/layer21_attention_weight_max": 32.06496047973633,
      "activations/layer21_attention_weight_min": -23.99734878540039,
      "activations/layer22_attention_weight_max": 45.35808181762695,
      "activations/layer22_attention_weight_min": -31.056304931640625,
      "activations/layer23_attention_weight_max": 33.48591613769531,
      "activations/layer23_attention_weight_min": -25.881385803222656,
      "activations/layer2_attention_weight_max": 31.223751068115234,
      "activations/layer2_attention_weight_min": -30.681072235107422,
      "activations/layer3_attention_weight_max": 93.3072280883789,
      "activations/layer3_attention_weight_min": -96.61471557617188,
      "activations/layer4_attention_weight_max": 112.2197265625,
      "activations/layer4_attention_weight_min": -111.70660400390625,
      "activations/layer5_attention_weight_max": 52.91819381713867,
      "activations/layer5_attention_weight_min": -66.20606994628906,
      "activations/layer6_attention_weight_max": 45.49003219604492,
      "activations/layer6_attention_weight_min": -46.77638244628906,
      "activations/layer7_attention_weight_max": 90.90879821777344,
      "activations/layer7_attention_weight_min": -90.68538665771484,
      "activations/layer8_attention_weight_max": 40.36830520629883,
      "activations/layer8_attention_weight_min": -42.030025482177734,
      "activations/layer9_attention_weight_max": 30.164676666259766,
      "activations/layer9_attention_weight_min": -33.45411682128906,
      "epoch": 14.37,
      "learning_rate": 5.791780303030303e-05,
      "loss": 2.7631,
      "step": 247300
    },
    {
      "activations/layer0_attention_weight_max": 15.086516380310059,
      "activations/layer0_attention_weight_min": -12.748997688293457,
      "activations/layer10_attention_weight_max": 29.7655086517334,
      "activations/layer10_attention_weight_min": -31.361745834350586,
      "activations/layer11_attention_weight_max": 31.82845687866211,
      "activations/layer11_attention_weight_min": -33.06865692138672,
      "activations/layer12_attention_weight_max": 25.133092880249023,
      "activations/layer12_attention_weight_min": -25.280109405517578,
      "activations/layer13_attention_weight_max": 37.2917594909668,
      "activations/layer13_attention_weight_min": -33.16304016113281,
      "activations/layer14_attention_weight_max": 38.87635803222656,
      "activations/layer14_attention_weight_min": -34.46409606933594,
      "activations/layer15_attention_weight_max": 36.294532775878906,
      "activations/layer15_attention_weight_min": -34.0571174621582,
      "activations/layer16_attention_weight_max": 32.03175735473633,
      "activations/layer16_attention_weight_min": -28.178119659423828,
      "activations/layer17_attention_weight_max": 28.56899642944336,
      "activations/layer17_attention_weight_min": -25.762676239013672,
      "activations/layer18_attention_weight_max": 29.993553161621094,
      "activations/layer18_attention_weight_min": -26.19058609008789,
      "activations/layer19_attention_weight_max": 34.640377044677734,
      "activations/layer19_attention_weight_min": -32.661827087402344,
      "activations/layer1_attention_weight_max": 15.475400924682617,
      "activations/layer1_attention_weight_min": -13.723340034484863,
      "activations/layer20_attention_weight_max": 29.163063049316406,
      "activations/layer20_attention_weight_min": -25.5764102935791,
      "activations/layer21_attention_weight_max": 27.451622009277344,
      "activations/layer21_attention_weight_min": -24.507274627685547,
      "activations/layer22_attention_weight_max": 42.703147888183594,
      "activations/layer22_attention_weight_min": -29.61237335205078,
      "activations/layer23_attention_weight_max": 34.15382385253906,
      "activations/layer23_attention_weight_min": -24.75662612915039,
      "activations/layer2_attention_weight_max": 31.126323699951172,
      "activations/layer2_attention_weight_min": -30.48878288269043,
      "activations/layer3_attention_weight_max": 91.7607192993164,
      "activations/layer3_attention_weight_min": -93.25420379638672,
      "activations/layer4_attention_weight_max": 109.56902313232422,
      "activations/layer4_attention_weight_min": -109.72286224365234,
      "activations/layer5_attention_weight_max": 53.97358322143555,
      "activations/layer5_attention_weight_min": -61.36471176147461,
      "activations/layer6_attention_weight_max": 43.52549362182617,
      "activations/layer6_attention_weight_min": -45.262718200683594,
      "activations/layer7_attention_weight_max": 86.87042999267578,
      "activations/layer7_attention_weight_min": -90.55156707763672,
      "activations/layer8_attention_weight_max": 39.12792205810547,
      "activations/layer8_attention_weight_min": -41.61331558227539,
      "activations/layer9_attention_weight_max": 29.82744789123535,
      "activations/layer9_attention_weight_min": -31.238515853881836,
      "epoch": 14.37,
      "learning_rate": 5.789886363636363e-05,
      "loss": 2.7616,
      "step": 247350
    },
    {
      "activations/layer0_attention_weight_max": 16.39402961730957,
      "activations/layer0_attention_weight_min": -12.882404327392578,
      "activations/layer10_attention_weight_max": 30.68192481994629,
      "activations/layer10_attention_weight_min": -31.5520076751709,
      "activations/layer11_attention_weight_max": 31.303693771362305,
      "activations/layer11_attention_weight_min": -31.863340377807617,
      "activations/layer12_attention_weight_max": 24.75812530517578,
      "activations/layer12_attention_weight_min": -24.350736618041992,
      "activations/layer13_attention_weight_max": 37.646949768066406,
      "activations/layer13_attention_weight_min": -33.633155822753906,
      "activations/layer14_attention_weight_max": 44.80387496948242,
      "activations/layer14_attention_weight_min": -37.07487869262695,
      "activations/layer15_attention_weight_max": 34.42963409423828,
      "activations/layer15_attention_weight_min": -33.186363220214844,
      "activations/layer16_attention_weight_max": 28.222959518432617,
      "activations/layer16_attention_weight_min": -26.847389221191406,
      "activations/layer17_attention_weight_max": 29.95506477355957,
      "activations/layer17_attention_weight_min": -25.708707809448242,
      "activations/layer18_attention_weight_max": 31.53271484375,
      "activations/layer18_attention_weight_min": -25.371706008911133,
      "activations/layer19_attention_weight_max": 35.468658447265625,
      "activations/layer19_attention_weight_min": -31.03889274597168,
      "activations/layer1_attention_weight_max": 16.30060386657715,
      "activations/layer1_attention_weight_min": -13.664326667785645,
      "activations/layer20_attention_weight_max": 31.290447235107422,
      "activations/layer20_attention_weight_min": -24.01820182800293,
      "activations/layer21_attention_weight_max": 30.70265769958496,
      "activations/layer21_attention_weight_min": -23.714324951171875,
      "activations/layer22_attention_weight_max": 48.07773971557617,
      "activations/layer22_attention_weight_min": -30.300601959228516,
      "activations/layer23_attention_weight_max": 34.447059631347656,
      "activations/layer23_attention_weight_min": -26.250165939331055,
      "activations/layer2_attention_weight_max": 30.900625228881836,
      "activations/layer2_attention_weight_min": -30.743925094604492,
      "activations/layer3_attention_weight_max": 90.39598083496094,
      "activations/layer3_attention_weight_min": -89.66851043701172,
      "activations/layer4_attention_weight_max": 104.16035461425781,
      "activations/layer4_attention_weight_min": -106.54845428466797,
      "activations/layer5_attention_weight_max": 48.325828552246094,
      "activations/layer5_attention_weight_min": -60.43272018432617,
      "activations/layer6_attention_weight_max": 42.36737823486328,
      "activations/layer6_attention_weight_min": -47.43946075439453,
      "activations/layer7_attention_weight_max": 86.23628997802734,
      "activations/layer7_attention_weight_min": -91.31981658935547,
      "activations/layer8_attention_weight_max": 39.48060607910156,
      "activations/layer8_attention_weight_min": -41.372276306152344,
      "activations/layer9_attention_weight_max": 32.223567962646484,
      "activations/layer9_attention_weight_min": -31.801599502563477,
      "epoch": 14.38,
      "learning_rate": 5.787992424242424e-05,
      "loss": 2.771,
      "step": 247400
    },
    {
      "activations/layer0_attention_weight_max": 14.962709426879883,
      "activations/layer0_attention_weight_min": -12.857444763183594,
      "activations/layer10_attention_weight_max": 32.40590286254883,
      "activations/layer10_attention_weight_min": -32.520999908447266,
      "activations/layer11_attention_weight_max": 35.4188346862793,
      "activations/layer11_attention_weight_min": -34.22410202026367,
      "activations/layer12_attention_weight_max": 26.737079620361328,
      "activations/layer12_attention_weight_min": -24.86295509338379,
      "activations/layer13_attention_weight_max": 38.861602783203125,
      "activations/layer13_attention_weight_min": -34.34086608886719,
      "activations/layer14_attention_weight_max": 40.170528411865234,
      "activations/layer14_attention_weight_min": -36.22962188720703,
      "activations/layer15_attention_weight_max": 36.359928131103516,
      "activations/layer15_attention_weight_min": -32.286903381347656,
      "activations/layer16_attention_weight_max": 28.599994659423828,
      "activations/layer16_attention_weight_min": -26.81401252746582,
      "activations/layer17_attention_weight_max": 29.4704532623291,
      "activations/layer17_attention_weight_min": -26.381982803344727,
      "activations/layer18_attention_weight_max": 29.86395835876465,
      "activations/layer18_attention_weight_min": -24.066503524780273,
      "activations/layer19_attention_weight_max": 31.746051788330078,
      "activations/layer19_attention_weight_min": -30.203601837158203,
      "activations/layer1_attention_weight_max": 16.442136764526367,
      "activations/layer1_attention_weight_min": -15.748379707336426,
      "activations/layer20_attention_weight_max": 28.35970687866211,
      "activations/layer20_attention_weight_min": -25.901018142700195,
      "activations/layer21_attention_weight_max": 27.098827362060547,
      "activations/layer21_attention_weight_min": -25.601604461669922,
      "activations/layer22_attention_weight_max": 38.6009407043457,
      "activations/layer22_attention_weight_min": -30.769155502319336,
      "activations/layer23_attention_weight_max": 32.886131286621094,
      "activations/layer23_attention_weight_min": -29.63544464111328,
      "activations/layer2_attention_weight_max": 31.477880477905273,
      "activations/layer2_attention_weight_min": -31.649993896484375,
      "activations/layer3_attention_weight_max": 93.28395080566406,
      "activations/layer3_attention_weight_min": -91.49679565429688,
      "activations/layer4_attention_weight_max": 105.1225357055664,
      "activations/layer4_attention_weight_min": -107.4166488647461,
      "activations/layer5_attention_weight_max": 52.6521110534668,
      "activations/layer5_attention_weight_min": -64.03499603271484,
      "activations/layer6_attention_weight_max": 41.73320770263672,
      "activations/layer6_attention_weight_min": -44.9411735534668,
      "activations/layer7_attention_weight_max": 90.25773620605469,
      "activations/layer7_attention_weight_min": -86.61226654052734,
      "activations/layer8_attention_weight_max": 37.97359848022461,
      "activations/layer8_attention_weight_min": -40.071197509765625,
      "activations/layer9_attention_weight_max": 31.162643432617188,
      "activations/layer9_attention_weight_min": -32.229408264160156,
      "epoch": 14.38,
      "learning_rate": 5.786098484848484e-05,
      "loss": 2.7594,
      "step": 247450
    },
    {
      "activations/layer0_attention_weight_max": 15.502737998962402,
      "activations/layer0_attention_weight_min": -13.496905326843262,
      "activations/layer10_attention_weight_max": 34.89542007446289,
      "activations/layer10_attention_weight_min": -33.28950119018555,
      "activations/layer11_attention_weight_max": 35.87788009643555,
      "activations/layer11_attention_weight_min": -32.31341552734375,
      "activations/layer12_attention_weight_max": 28.4016056060791,
      "activations/layer12_attention_weight_min": -24.687360763549805,
      "activations/layer13_attention_weight_max": 39.27782440185547,
      "activations/layer13_attention_weight_min": -34.97050094604492,
      "activations/layer14_attention_weight_max": 49.27821731567383,
      "activations/layer14_attention_weight_min": -35.95968246459961,
      "activations/layer15_attention_weight_max": 40.10074234008789,
      "activations/layer15_attention_weight_min": -32.21613693237305,
      "activations/layer16_attention_weight_max": 36.57935333251953,
      "activations/layer16_attention_weight_min": -30.544795989990234,
      "activations/layer17_attention_weight_max": 32.27924728393555,
      "activations/layer17_attention_weight_min": -27.202157974243164,
      "activations/layer18_attention_weight_max": 37.15647888183594,
      "activations/layer18_attention_weight_min": -24.89667510986328,
      "activations/layer19_attention_weight_max": 38.37089538574219,
      "activations/layer19_attention_weight_min": -31.03108024597168,
      "activations/layer1_attention_weight_max": 17.6746768951416,
      "activations/layer1_attention_weight_min": -16.702171325683594,
      "activations/layer20_attention_weight_max": 35.14406204223633,
      "activations/layer20_attention_weight_min": -24.00199317932129,
      "activations/layer21_attention_weight_max": 33.931617736816406,
      "activations/layer21_attention_weight_min": -22.9334659576416,
      "activations/layer22_attention_weight_max": 54.55002212524414,
      "activations/layer22_attention_weight_min": -33.529109954833984,
      "activations/layer23_attention_weight_max": 38.24462127685547,
      "activations/layer23_attention_weight_min": -25.201635360717773,
      "activations/layer2_attention_weight_max": 33.504215240478516,
      "activations/layer2_attention_weight_min": -32.844093322753906,
      "activations/layer3_attention_weight_max": 94.67926025390625,
      "activations/layer3_attention_weight_min": -98.27576446533203,
      "activations/layer4_attention_weight_max": 107.42726135253906,
      "activations/layer4_attention_weight_min": -108.89411926269531,
      "activations/layer5_attention_weight_max": 51.3525390625,
      "activations/layer5_attention_weight_min": -64.65469360351562,
      "activations/layer6_attention_weight_max": 44.463783264160156,
      "activations/layer6_attention_weight_min": -46.86601638793945,
      "activations/layer7_attention_weight_max": 100.16156768798828,
      "activations/layer7_attention_weight_min": -96.38494110107422,
      "activations/layer8_attention_weight_max": 45.287193298339844,
      "activations/layer8_attention_weight_min": -43.75089645385742,
      "activations/layer9_attention_weight_max": 33.336647033691406,
      "activations/layer9_attention_weight_min": -33.69338607788086,
      "epoch": 14.38,
      "learning_rate": 5.7842045454545455e-05,
      "loss": 2.7565,
      "step": 247500
    },
    {
      "activations/layer0_attention_weight_max": 15.224942207336426,
      "activations/layer0_attention_weight_min": -13.484333038330078,
      "activations/layer10_attention_weight_max": 34.5978889465332,
      "activations/layer10_attention_weight_min": -33.28123474121094,
      "activations/layer11_attention_weight_max": 32.997676849365234,
      "activations/layer11_attention_weight_min": -34.633174896240234,
      "activations/layer12_attention_weight_max": 36.09373092651367,
      "activations/layer12_attention_weight_min": -25.559669494628906,
      "activations/layer13_attention_weight_max": 48.87473678588867,
      "activations/layer13_attention_weight_min": -35.932525634765625,
      "activations/layer14_attention_weight_max": 42.821983337402344,
      "activations/layer14_attention_weight_min": -35.63566970825195,
      "activations/layer15_attention_weight_max": 40.24674606323242,
      "activations/layer15_attention_weight_min": -32.66034698486328,
      "activations/layer16_attention_weight_max": 31.036115646362305,
      "activations/layer16_attention_weight_min": -26.53396987915039,
      "activations/layer17_attention_weight_max": 33.26785659790039,
      "activations/layer17_attention_weight_min": -26.193408966064453,
      "activations/layer18_attention_weight_max": 31.316625595092773,
      "activations/layer18_attention_weight_min": -22.531404495239258,
      "activations/layer19_attention_weight_max": 32.19887924194336,
      "activations/layer19_attention_weight_min": -32.420494079589844,
      "activations/layer1_attention_weight_max": 16.166414260864258,
      "activations/layer1_attention_weight_min": -13.68531608581543,
      "activations/layer20_attention_weight_max": 30.941503524780273,
      "activations/layer20_attention_weight_min": -26.335264205932617,
      "activations/layer21_attention_weight_max": 29.848745346069336,
      "activations/layer21_attention_weight_min": -26.097997665405273,
      "activations/layer22_attention_weight_max": 47.632408142089844,
      "activations/layer22_attention_weight_min": -30.64339828491211,
      "activations/layer23_attention_weight_max": 32.00688552856445,
      "activations/layer23_attention_weight_min": -23.829734802246094,
      "activations/layer2_attention_weight_max": 31.173648834228516,
      "activations/layer2_attention_weight_min": -31.465587615966797,
      "activations/layer3_attention_weight_max": 97.03022766113281,
      "activations/layer3_attention_weight_min": -95.6211929321289,
      "activations/layer4_attention_weight_max": 111.51979064941406,
      "activations/layer4_attention_weight_min": -109.39873504638672,
      "activations/layer5_attention_weight_max": 51.698970794677734,
      "activations/layer5_attention_weight_min": -62.14956283569336,
      "activations/layer6_attention_weight_max": 46.60404968261719,
      "activations/layer6_attention_weight_min": -48.59560012817383,
      "activations/layer7_attention_weight_max": 104.25094604492188,
      "activations/layer7_attention_weight_min": -98.1966552734375,
      "activations/layer8_attention_weight_max": 43.14287567138672,
      "activations/layer8_attention_weight_min": -42.76815414428711,
      "activations/layer9_attention_weight_max": 33.394222259521484,
      "activations/layer9_attention_weight_min": -33.096004486083984,
      "epoch": 14.38,
      "learning_rate": 5.7823106060606056e-05,
      "loss": 2.7614,
      "step": 247550
    },
    {
      "activations/layer0_attention_weight_max": 15.263516426086426,
      "activations/layer0_attention_weight_min": -11.873698234558105,
      "activations/layer10_attention_weight_max": 37.04486083984375,
      "activations/layer10_attention_weight_min": -36.39213943481445,
      "activations/layer11_attention_weight_max": 38.79200744628906,
      "activations/layer11_attention_weight_min": -38.33637237548828,
      "activations/layer12_attention_weight_max": 29.007076263427734,
      "activations/layer12_attention_weight_min": -26.929182052612305,
      "activations/layer13_attention_weight_max": 42.64985275268555,
      "activations/layer13_attention_weight_min": -35.58659744262695,
      "activations/layer14_attention_weight_max": 47.42013931274414,
      "activations/layer14_attention_weight_min": -41.086490631103516,
      "activations/layer15_attention_weight_max": 44.4253044128418,
      "activations/layer15_attention_weight_min": -36.950050354003906,
      "activations/layer16_attention_weight_max": 33.45110321044922,
      "activations/layer16_attention_weight_min": -26.739612579345703,
      "activations/layer17_attention_weight_max": 34.29591369628906,
      "activations/layer17_attention_weight_min": -28.964508056640625,
      "activations/layer18_attention_weight_max": 28.991975784301758,
      "activations/layer18_attention_weight_min": -28.34441566467285,
      "activations/layer19_attention_weight_max": 37.124908447265625,
      "activations/layer19_attention_weight_min": -31.151952743530273,
      "activations/layer1_attention_weight_max": 18.229066848754883,
      "activations/layer1_attention_weight_min": -17.774606704711914,
      "activations/layer20_attention_weight_max": 30.819286346435547,
      "activations/layer20_attention_weight_min": -23.79308319091797,
      "activations/layer21_attention_weight_max": 30.615142822265625,
      "activations/layer21_attention_weight_min": -24.49376678466797,
      "activations/layer22_attention_weight_max": 46.672279357910156,
      "activations/layer22_attention_weight_min": -32.787837982177734,
      "activations/layer23_attention_weight_max": 33.00656509399414,
      "activations/layer23_attention_weight_min": -25.14288902282715,
      "activations/layer2_attention_weight_max": 32.84083938598633,
      "activations/layer2_attention_weight_min": -37.126182556152344,
      "activations/layer3_attention_weight_max": 104.47669219970703,
      "activations/layer3_attention_weight_min": -106.27278137207031,
      "activations/layer4_attention_weight_max": 113.07689666748047,
      "activations/layer4_attention_weight_min": -118.29510498046875,
      "activations/layer5_attention_weight_max": 53.79090118408203,
      "activations/layer5_attention_weight_min": -64.10337829589844,
      "activations/layer6_attention_weight_max": 46.32182693481445,
      "activations/layer6_attention_weight_min": -47.643375396728516,
      "activations/layer7_attention_weight_max": 96.7627182006836,
      "activations/layer7_attention_weight_min": -101.16535186767578,
      "activations/layer8_attention_weight_max": 46.809974670410156,
      "activations/layer8_attention_weight_min": -44.403038024902344,
      "activations/layer9_attention_weight_max": 36.20680236816406,
      "activations/layer9_attention_weight_min": -36.14614486694336,
      "epoch": 14.39,
      "learning_rate": 5.780416666666666e-05,
      "loss": 2.7607,
      "step": 247600
    },
    {
      "activations/layer0_attention_weight_max": 14.836193084716797,
      "activations/layer0_attention_weight_min": -12.473309516906738,
      "activations/layer10_attention_weight_max": 34.06600570678711,
      "activations/layer10_attention_weight_min": -32.46250915527344,
      "activations/layer11_attention_weight_max": 31.377553939819336,
      "activations/layer11_attention_weight_min": -32.771705627441406,
      "activations/layer12_attention_weight_max": 24.500282287597656,
      "activations/layer12_attention_weight_min": -26.170045852661133,
      "activations/layer13_attention_weight_max": 38.186031341552734,
      "activations/layer13_attention_weight_min": -35.038387298583984,
      "activations/layer14_attention_weight_max": 42.72856140136719,
      "activations/layer14_attention_weight_min": -37.70748519897461,
      "activations/layer15_attention_weight_max": 45.241031646728516,
      "activations/layer15_attention_weight_min": -39.70656967163086,
      "activations/layer16_attention_weight_max": 31.308609008789062,
      "activations/layer16_attention_weight_min": -27.161882400512695,
      "activations/layer17_attention_weight_max": 31.08455467224121,
      "activations/layer17_attention_weight_min": -26.00612449645996,
      "activations/layer18_attention_weight_max": 31.884334564208984,
      "activations/layer18_attention_weight_min": -25.274593353271484,
      "activations/layer19_attention_weight_max": 36.67442321777344,
      "activations/layer19_attention_weight_min": -31.306564331054688,
      "activations/layer1_attention_weight_max": 16.219772338867188,
      "activations/layer1_attention_weight_min": -16.149171829223633,
      "activations/layer20_attention_weight_max": 32.229854583740234,
      "activations/layer20_attention_weight_min": -26.1738338470459,
      "activations/layer21_attention_weight_max": 31.549150466918945,
      "activations/layer21_attention_weight_min": -24.657791137695312,
      "activations/layer22_attention_weight_max": 42.11332702636719,
      "activations/layer22_attention_weight_min": -29.14276695251465,
      "activations/layer23_attention_weight_max": 31.264652252197266,
      "activations/layer23_attention_weight_min": -25.010372161865234,
      "activations/layer2_attention_weight_max": 32.66331481933594,
      "activations/layer2_attention_weight_min": -31.313987731933594,
      "activations/layer3_attention_weight_max": 95.05941772460938,
      "activations/layer3_attention_weight_min": -92.5377197265625,
      "activations/layer4_attention_weight_max": 111.533935546875,
      "activations/layer4_attention_weight_min": -106.56053924560547,
      "activations/layer5_attention_weight_max": 53.05683898925781,
      "activations/layer5_attention_weight_min": -62.80664825439453,
      "activations/layer6_attention_weight_max": 43.22549819946289,
      "activations/layer6_attention_weight_min": -46.43101119995117,
      "activations/layer7_attention_weight_max": 89.54358673095703,
      "activations/layer7_attention_weight_min": -90.71501922607422,
      "activations/layer8_attention_weight_max": 38.878639221191406,
      "activations/layer8_attention_weight_min": -42.59148406982422,
      "activations/layer9_attention_weight_max": 32.05488204956055,
      "activations/layer9_attention_weight_min": -33.36721420288086,
      "epoch": 14.39,
      "learning_rate": 5.7785227272727266e-05,
      "loss": 2.7396,
      "step": 247650
    },
    {
      "activations/layer0_attention_weight_max": 15.02686882019043,
      "activations/layer0_attention_weight_min": -12.62014102935791,
      "activations/layer10_attention_weight_max": 37.55697250366211,
      "activations/layer10_attention_weight_min": -34.85015106201172,
      "activations/layer11_attention_weight_max": 36.17906188964844,
      "activations/layer11_attention_weight_min": -36.72190856933594,
      "activations/layer12_attention_weight_max": 32.18856430053711,
      "activations/layer12_attention_weight_min": -30.435253143310547,
      "activations/layer13_attention_weight_max": 49.91999435424805,
      "activations/layer13_attention_weight_min": -34.22441101074219,
      "activations/layer14_attention_weight_max": 49.272830963134766,
      "activations/layer14_attention_weight_min": -36.80894470214844,
      "activations/layer15_attention_weight_max": 52.67371368408203,
      "activations/layer15_attention_weight_min": -39.69902038574219,
      "activations/layer16_attention_weight_max": 31.846723556518555,
      "activations/layer16_attention_weight_min": -26.12700080871582,
      "activations/layer17_attention_weight_max": 27.64598846435547,
      "activations/layer17_attention_weight_min": -28.72727394104004,
      "activations/layer18_attention_weight_max": 28.543119430541992,
      "activations/layer18_attention_weight_min": -25.366262435913086,
      "activations/layer19_attention_weight_max": 34.338096618652344,
      "activations/layer19_attention_weight_min": -29.428695678710938,
      "activations/layer1_attention_weight_max": 17.651968002319336,
      "activations/layer1_attention_weight_min": -15.629782676696777,
      "activations/layer20_attention_weight_max": 25.745588302612305,
      "activations/layer20_attention_weight_min": -24.29894256591797,
      "activations/layer21_attention_weight_max": 25.79673957824707,
      "activations/layer21_attention_weight_min": -25.205474853515625,
      "activations/layer22_attention_weight_max": 40.65974426269531,
      "activations/layer22_attention_weight_min": -32.14436340332031,
      "activations/layer23_attention_weight_max": 29.856754302978516,
      "activations/layer23_attention_weight_min": -24.463932037353516,
      "activations/layer2_attention_weight_max": 32.90913009643555,
      "activations/layer2_attention_weight_min": -32.62143325805664,
      "activations/layer3_attention_weight_max": 96.51414489746094,
      "activations/layer3_attention_weight_min": -96.63153076171875,
      "activations/layer4_attention_weight_max": 109.4601058959961,
      "activations/layer4_attention_weight_min": -107.75616455078125,
      "activations/layer5_attention_weight_max": 55.3907470703125,
      "activations/layer5_attention_weight_min": -60.66743087768555,
      "activations/layer6_attention_weight_max": 46.696434020996094,
      "activations/layer6_attention_weight_min": -47.43631362915039,
      "activations/layer7_attention_weight_max": 102.8467025756836,
      "activations/layer7_attention_weight_min": -92.13468170166016,
      "activations/layer8_attention_weight_max": 43.73101806640625,
      "activations/layer8_attention_weight_min": -43.43496322631836,
      "activations/layer9_attention_weight_max": 33.61838912963867,
      "activations/layer9_attention_weight_min": -33.8368034362793,
      "epoch": 14.39,
      "learning_rate": 5.7766287878787874e-05,
      "loss": 2.7613,
      "step": 247700
    },
    {
      "activations/layer0_attention_weight_max": 17.156753540039062,
      "activations/layer0_attention_weight_min": -12.360118865966797,
      "activations/layer10_attention_weight_max": 35.01711654663086,
      "activations/layer10_attention_weight_min": -34.245155334472656,
      "activations/layer11_attention_weight_max": 36.72862243652344,
      "activations/layer11_attention_weight_min": -36.81927490234375,
      "activations/layer12_attention_weight_max": 25.60314178466797,
      "activations/layer12_attention_weight_min": -28.52117347717285,
      "activations/layer13_attention_weight_max": 37.952388763427734,
      "activations/layer13_attention_weight_min": -37.263668060302734,
      "activations/layer14_attention_weight_max": 43.11957550048828,
      "activations/layer14_attention_weight_min": -41.30350112915039,
      "activations/layer15_attention_weight_max": 34.03901672363281,
      "activations/layer15_attention_weight_min": -34.911224365234375,
      "activations/layer16_attention_weight_max": 28.419004440307617,
      "activations/layer16_attention_weight_min": -29.39217758178711,
      "activations/layer17_attention_weight_max": 27.377758026123047,
      "activations/layer17_attention_weight_min": -28.28373146057129,
      "activations/layer18_attention_weight_max": 31.634904861450195,
      "activations/layer18_attention_weight_min": -25.437095642089844,
      "activations/layer19_attention_weight_max": 34.889156341552734,
      "activations/layer19_attention_weight_min": -33.47698974609375,
      "activations/layer1_attention_weight_max": 15.968341827392578,
      "activations/layer1_attention_weight_min": -13.614936828613281,
      "activations/layer20_attention_weight_max": 29.688194274902344,
      "activations/layer20_attention_weight_min": -25.58418846130371,
      "activations/layer21_attention_weight_max": 31.0860595703125,
      "activations/layer21_attention_weight_min": -25.492013931274414,
      "activations/layer22_attention_weight_max": 51.623775482177734,
      "activations/layer22_attention_weight_min": -32.697933197021484,
      "activations/layer23_attention_weight_max": 33.73667907714844,
      "activations/layer23_attention_weight_min": -25.72481346130371,
      "activations/layer2_attention_weight_max": 32.41521453857422,
      "activations/layer2_attention_weight_min": -30.989395141601562,
      "activations/layer3_attention_weight_max": 96.76981353759766,
      "activations/layer3_attention_weight_min": -98.42707061767578,
      "activations/layer4_attention_weight_max": 111.63802337646484,
      "activations/layer4_attention_weight_min": -106.8822021484375,
      "activations/layer5_attention_weight_max": 50.59667205810547,
      "activations/layer5_attention_weight_min": -58.65754318237305,
      "activations/layer6_attention_weight_max": 44.123924255371094,
      "activations/layer6_attention_weight_min": -46.88703536987305,
      "activations/layer7_attention_weight_max": 91.10393524169922,
      "activations/layer7_attention_weight_min": -93.53678894042969,
      "activations/layer8_attention_weight_max": 40.6655158996582,
      "activations/layer8_attention_weight_min": -42.705474853515625,
      "activations/layer9_attention_weight_max": 31.87919044494629,
      "activations/layer9_attention_weight_min": -33.33246994018555,
      "epoch": 14.4,
      "learning_rate": 5.774734848484848e-05,
      "loss": 2.7719,
      "step": 247750
    },
    {
      "activations/layer0_attention_weight_max": 15.246315956115723,
      "activations/layer0_attention_weight_min": -11.793075561523438,
      "activations/layer10_attention_weight_max": 45.54895782470703,
      "activations/layer10_attention_weight_min": -39.67523956298828,
      "activations/layer11_attention_weight_max": 44.76878356933594,
      "activations/layer11_attention_weight_min": -41.92283630371094,
      "activations/layer12_attention_weight_max": 31.92664337158203,
      "activations/layer12_attention_weight_min": -28.50397300720215,
      "activations/layer13_attention_weight_max": 53.81624221801758,
      "activations/layer13_attention_weight_min": -38.62685012817383,
      "activations/layer14_attention_weight_max": 48.905548095703125,
      "activations/layer14_attention_weight_min": -40.115020751953125,
      "activations/layer15_attention_weight_max": 53.46870040893555,
      "activations/layer15_attention_weight_min": -40.39960861206055,
      "activations/layer16_attention_weight_max": 34.66825866699219,
      "activations/layer16_attention_weight_min": -31.056432723999023,
      "activations/layer17_attention_weight_max": 30.519269943237305,
      "activations/layer17_attention_weight_min": -28.64643096923828,
      "activations/layer18_attention_weight_max": 27.941545486450195,
      "activations/layer18_attention_weight_min": -25.01726531982422,
      "activations/layer19_attention_weight_max": 33.73702621459961,
      "activations/layer19_attention_weight_min": -31.4149112701416,
      "activations/layer1_attention_weight_max": 16.82195281982422,
      "activations/layer1_attention_weight_min": -15.56188678741455,
      "activations/layer20_attention_weight_max": 29.053987503051758,
      "activations/layer20_attention_weight_min": -26.716663360595703,
      "activations/layer21_attention_weight_max": 28.51633071899414,
      "activations/layer21_attention_weight_min": -25.586227416992188,
      "activations/layer22_attention_weight_max": 41.58256149291992,
      "activations/layer22_attention_weight_min": -33.434051513671875,
      "activations/layer23_attention_weight_max": 31.894657135009766,
      "activations/layer23_attention_weight_min": -26.988126754760742,
      "activations/layer2_attention_weight_max": 33.55980682373047,
      "activations/layer2_attention_weight_min": -32.309364318847656,
      "activations/layer3_attention_weight_max": 97.4300537109375,
      "activations/layer3_attention_weight_min": -96.52115631103516,
      "activations/layer4_attention_weight_max": 107.9154281616211,
      "activations/layer4_attention_weight_min": -108.54424285888672,
      "activations/layer5_attention_weight_max": 53.572750091552734,
      "activations/layer5_attention_weight_min": -59.58061981201172,
      "activations/layer6_attention_weight_max": 44.49766159057617,
      "activations/layer6_attention_weight_min": -44.86592102050781,
      "activations/layer7_attention_weight_max": 101.80048370361328,
      "activations/layer7_attention_weight_min": -97.286376953125,
      "activations/layer8_attention_weight_max": 46.190948486328125,
      "activations/layer8_attention_weight_min": -48.009239196777344,
      "activations/layer9_attention_weight_max": 37.07902908325195,
      "activations/layer9_attention_weight_min": -36.38703536987305,
      "epoch": 14.4,
      "learning_rate": 5.7728409090909084e-05,
      "loss": 2.7626,
      "step": 247800
    },
    {
      "activations/layer0_attention_weight_max": 15.348642349243164,
      "activations/layer0_attention_weight_min": -12.136992454528809,
      "activations/layer10_attention_weight_max": 34.00300598144531,
      "activations/layer10_attention_weight_min": -34.101322174072266,
      "activations/layer11_attention_weight_max": 34.56401062011719,
      "activations/layer11_attention_weight_min": -33.736534118652344,
      "activations/layer12_attention_weight_max": 27.690343856811523,
      "activations/layer12_attention_weight_min": -26.491653442382812,
      "activations/layer13_attention_weight_max": 39.857696533203125,
      "activations/layer13_attention_weight_min": -34.177547454833984,
      "activations/layer14_attention_weight_max": 44.142906188964844,
      "activations/layer14_attention_weight_min": -38.86247634887695,
      "activations/layer15_attention_weight_max": 37.39461898803711,
      "activations/layer15_attention_weight_min": -35.41228485107422,
      "activations/layer16_attention_weight_max": 30.130647659301758,
      "activations/layer16_attention_weight_min": -28.80487823486328,
      "activations/layer17_attention_weight_max": 30.41619873046875,
      "activations/layer17_attention_weight_min": -27.619718551635742,
      "activations/layer18_attention_weight_max": 31.71087646484375,
      "activations/layer18_attention_weight_min": -24.33187484741211,
      "activations/layer19_attention_weight_max": 39.610965728759766,
      "activations/layer19_attention_weight_min": -34.074703216552734,
      "activations/layer1_attention_weight_max": 15.467970848083496,
      "activations/layer1_attention_weight_min": -13.948022842407227,
      "activations/layer20_attention_weight_max": 28.90306282043457,
      "activations/layer20_attention_weight_min": -26.1433162689209,
      "activations/layer21_attention_weight_max": 30.177213668823242,
      "activations/layer21_attention_weight_min": -25.358489990234375,
      "activations/layer22_attention_weight_max": 50.1615104675293,
      "activations/layer22_attention_weight_min": -34.16090393066406,
      "activations/layer23_attention_weight_max": 37.75995635986328,
      "activations/layer23_attention_weight_min": -27.57129669189453,
      "activations/layer2_attention_weight_max": 32.66326904296875,
      "activations/layer2_attention_weight_min": -32.018028259277344,
      "activations/layer3_attention_weight_max": 95.0945053100586,
      "activations/layer3_attention_weight_min": -99.44082641601562,
      "activations/layer4_attention_weight_max": 113.55018615722656,
      "activations/layer4_attention_weight_min": -110.28582000732422,
      "activations/layer5_attention_weight_max": 56.13329315185547,
      "activations/layer5_attention_weight_min": -60.582191467285156,
      "activations/layer6_attention_weight_max": 44.43159484863281,
      "activations/layer6_attention_weight_min": -45.08195877075195,
      "activations/layer7_attention_weight_max": 92.64187622070312,
      "activations/layer7_attention_weight_min": -88.99906921386719,
      "activations/layer8_attention_weight_max": 42.32307052612305,
      "activations/layer8_attention_weight_min": -40.16048049926758,
      "activations/layer9_attention_weight_max": 33.99222183227539,
      "activations/layer9_attention_weight_min": -33.67176055908203,
      "epoch": 14.4,
      "learning_rate": 5.77094696969697e-05,
      "loss": 2.7591,
      "step": 247850
    },
    {
      "activations/layer0_attention_weight_max": 15.056500434875488,
      "activations/layer0_attention_weight_min": -12.887685775756836,
      "activations/layer10_attention_weight_max": 35.88494110107422,
      "activations/layer10_attention_weight_min": -35.05182647705078,
      "activations/layer11_attention_weight_max": 35.922142028808594,
      "activations/layer11_attention_weight_min": -35.0275993347168,
      "activations/layer12_attention_weight_max": 28.00078773498535,
      "activations/layer12_attention_weight_min": -26.946632385253906,
      "activations/layer13_attention_weight_max": 41.92571258544922,
      "activations/layer13_attention_weight_min": -34.49652099609375,
      "activations/layer14_attention_weight_max": 48.51618194580078,
      "activations/layer14_attention_weight_min": -38.86506652832031,
      "activations/layer15_attention_weight_max": 42.00532150268555,
      "activations/layer15_attention_weight_min": -33.29892349243164,
      "activations/layer16_attention_weight_max": 34.6115608215332,
      "activations/layer16_attention_weight_min": -29.55473518371582,
      "activations/layer17_attention_weight_max": 33.15000534057617,
      "activations/layer17_attention_weight_min": -28.562007904052734,
      "activations/layer18_attention_weight_max": 35.46639633178711,
      "activations/layer18_attention_weight_min": -26.108558654785156,
      "activations/layer19_attention_weight_max": 39.0419921875,
      "activations/layer19_attention_weight_min": -32.438873291015625,
      "activations/layer1_attention_weight_max": 15.675705909729004,
      "activations/layer1_attention_weight_min": -14.537921905517578,
      "activations/layer20_attention_weight_max": 35.9195556640625,
      "activations/layer20_attention_weight_min": -26.05133056640625,
      "activations/layer21_attention_weight_max": 32.443851470947266,
      "activations/layer21_attention_weight_min": -25.879716873168945,
      "activations/layer22_attention_weight_max": 47.7594108581543,
      "activations/layer22_attention_weight_min": -31.616928100585938,
      "activations/layer23_attention_weight_max": 36.20429992675781,
      "activations/layer23_attention_weight_min": -27.339107513427734,
      "activations/layer2_attention_weight_max": 33.285980224609375,
      "activations/layer2_attention_weight_min": -31.575056076049805,
      "activations/layer3_attention_weight_max": 99.51510620117188,
      "activations/layer3_attention_weight_min": -97.4183349609375,
      "activations/layer4_attention_weight_max": 113.577880859375,
      "activations/layer4_attention_weight_min": -109.5871810913086,
      "activations/layer5_attention_weight_max": 51.727394104003906,
      "activations/layer5_attention_weight_min": -62.464698791503906,
      "activations/layer6_attention_weight_max": 47.130409240722656,
      "activations/layer6_attention_weight_min": -48.72282409667969,
      "activations/layer7_attention_weight_max": 95.2183609008789,
      "activations/layer7_attention_weight_min": -89.67829132080078,
      "activations/layer8_attention_weight_max": 42.8548583984375,
      "activations/layer8_attention_weight_min": -46.30036544799805,
      "activations/layer9_attention_weight_max": 34.47581100463867,
      "activations/layer9_attention_weight_min": -37.93167495727539,
      "epoch": 14.4,
      "learning_rate": 5.76905303030303e-05,
      "loss": 2.7569,
      "step": 247900
    },
    {
      "activations/layer0_attention_weight_max": 16.18284797668457,
      "activations/layer0_attention_weight_min": -12.158472061157227,
      "activations/layer10_attention_weight_max": 30.257633209228516,
      "activations/layer10_attention_weight_min": -32.52808380126953,
      "activations/layer11_attention_weight_max": 31.12379264831543,
      "activations/layer11_attention_weight_min": -32.24908447265625,
      "activations/layer12_attention_weight_max": 25.1159610748291,
      "activations/layer12_attention_weight_min": -25.241342544555664,
      "activations/layer13_attention_weight_max": 39.74678421020508,
      "activations/layer13_attention_weight_min": -34.597877502441406,
      "activations/layer14_attention_weight_max": 40.37799835205078,
      "activations/layer14_attention_weight_min": -41.6265754699707,
      "activations/layer15_attention_weight_max": 37.12363052368164,
      "activations/layer15_attention_weight_min": -32.66781997680664,
      "activations/layer16_attention_weight_max": 29.397493362426758,
      "activations/layer16_attention_weight_min": -28.0696964263916,
      "activations/layer17_attention_weight_max": 34.49611282348633,
      "activations/layer17_attention_weight_min": -26.14759063720703,
      "activations/layer18_attention_weight_max": 32.54195022583008,
      "activations/layer18_attention_weight_min": -22.849437713623047,
      "activations/layer19_attention_weight_max": 35.3737678527832,
      "activations/layer19_attention_weight_min": -30.744491577148438,
      "activations/layer1_attention_weight_max": 16.76064682006836,
      "activations/layer1_attention_weight_min": -14.300307273864746,
      "activations/layer20_attention_weight_max": 29.376968383789062,
      "activations/layer20_attention_weight_min": -23.918603897094727,
      "activations/layer21_attention_weight_max": 28.3353271484375,
      "activations/layer21_attention_weight_min": -24.664226531982422,
      "activations/layer22_attention_weight_max": 43.52878189086914,
      "activations/layer22_attention_weight_min": -30.87131118774414,
      "activations/layer23_attention_weight_max": 34.820648193359375,
      "activations/layer23_attention_weight_min": -27.003156661987305,
      "activations/layer2_attention_weight_max": 33.890541076660156,
      "activations/layer2_attention_weight_min": -33.11288833618164,
      "activations/layer3_attention_weight_max": 95.70918273925781,
      "activations/layer3_attention_weight_min": -94.7975845336914,
      "activations/layer4_attention_weight_max": 107.85892486572266,
      "activations/layer4_attention_weight_min": -110.92037200927734,
      "activations/layer5_attention_weight_max": 52.84403610229492,
      "activations/layer5_attention_weight_min": -62.65534591674805,
      "activations/layer6_attention_weight_max": 42.52318572998047,
      "activations/layer6_attention_weight_min": -45.79401397705078,
      "activations/layer7_attention_weight_max": 87.4778060913086,
      "activations/layer7_attention_weight_min": -92.65987396240234,
      "activations/layer8_attention_weight_max": 39.48855209350586,
      "activations/layer8_attention_weight_min": -41.94910430908203,
      "activations/layer9_attention_weight_max": 32.27415084838867,
      "activations/layer9_attention_weight_min": -32.83426284790039,
      "epoch": 14.41,
      "learning_rate": 5.76715909090909e-05,
      "loss": 2.7503,
      "step": 247950
    },
    {
      "activations/layer0_attention_weight_max": 15.340909957885742,
      "activations/layer0_attention_weight_min": -13.561150550842285,
      "activations/layer10_attention_weight_max": 31.136491775512695,
      "activations/layer10_attention_weight_min": -32.93848419189453,
      "activations/layer11_attention_weight_max": 31.003379821777344,
      "activations/layer11_attention_weight_min": -31.910810470581055,
      "activations/layer12_attention_weight_max": 24.369056701660156,
      "activations/layer12_attention_weight_min": -24.293827056884766,
      "activations/layer13_attention_weight_max": 38.297157287597656,
      "activations/layer13_attention_weight_min": -35.0372428894043,
      "activations/layer14_attention_weight_max": 39.69746017456055,
      "activations/layer14_attention_weight_min": -35.98904800415039,
      "activations/layer15_attention_weight_max": 35.84453201293945,
      "activations/layer15_attention_weight_min": -31.72064781188965,
      "activations/layer16_attention_weight_max": 30.145174026489258,
      "activations/layer16_attention_weight_min": -26.612028121948242,
      "activations/layer17_attention_weight_max": 30.516151428222656,
      "activations/layer17_attention_weight_min": -24.984962463378906,
      "activations/layer18_attention_weight_max": 29.918479919433594,
      "activations/layer18_attention_weight_min": -23.45132064819336,
      "activations/layer19_attention_weight_max": 33.3857536315918,
      "activations/layer19_attention_weight_min": -30.886194229125977,
      "activations/layer1_attention_weight_max": 15.735228538513184,
      "activations/layer1_attention_weight_min": -13.824711799621582,
      "activations/layer20_attention_weight_max": 27.834949493408203,
      "activations/layer20_attention_weight_min": -24.859722137451172,
      "activations/layer21_attention_weight_max": 28.730588912963867,
      "activations/layer21_attention_weight_min": -23.796159744262695,
      "activations/layer22_attention_weight_max": 40.25061798095703,
      "activations/layer22_attention_weight_min": -31.57787322998047,
      "activations/layer23_attention_weight_max": 33.98085021972656,
      "activations/layer23_attention_weight_min": -26.336559295654297,
      "activations/layer2_attention_weight_max": 30.698627471923828,
      "activations/layer2_attention_weight_min": -32.05909729003906,
      "activations/layer3_attention_weight_max": 92.5772705078125,
      "activations/layer3_attention_weight_min": -93.90543365478516,
      "activations/layer4_attention_weight_max": 105.9203109741211,
      "activations/layer4_attention_weight_min": -112.06343078613281,
      "activations/layer5_attention_weight_max": 50.8505859375,
      "activations/layer5_attention_weight_min": -60.692405700683594,
      "activations/layer6_attention_weight_max": 42.70296859741211,
      "activations/layer6_attention_weight_min": -46.64021682739258,
      "activations/layer7_attention_weight_max": 87.58213806152344,
      "activations/layer7_attention_weight_min": -93.93687438964844,
      "activations/layer8_attention_weight_max": 40.270877838134766,
      "activations/layer8_attention_weight_min": -43.81024169921875,
      "activations/layer9_attention_weight_max": 31.50579833984375,
      "activations/layer9_attention_weight_min": -33.3769645690918,
      "epoch": 14.41,
      "learning_rate": 5.765265151515151e-05,
      "loss": 2.7545,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5083,
      "eval_samples_per_second": 504.683,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5083,
      "eval_openwebtext_samples_per_second": 504.683,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 2.0256,
      "eval_wikitext_samples_per_second": 225.119,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_lambada_loss": 2.4609375,
      "eval_lambada_ppl": 11.715789946123383,
      "eval_lambada_runtime": 9.5995,
      "eval_lambada_samples_per_second": 507.216,
      "step": 248000
    },
    {
      "activations/layer0_attention_weight_max": 16.332319259643555,
      "activations/layer0_attention_weight_min": -11.538308143615723,
      "activations/layer10_attention_weight_max": 31.772354125976562,
      "activations/layer10_attention_weight_min": -33.81865692138672,
      "activations/layer11_attention_weight_max": 31.616172790527344,
      "activations/layer11_attention_weight_min": -34.221641540527344,
      "activations/layer12_attention_weight_max": 31.71770477294922,
      "activations/layer12_attention_weight_min": -28.78786849975586,
      "activations/layer13_attention_weight_max": 41.794464111328125,
      "activations/layer13_attention_weight_min": -34.617164611816406,
      "activations/layer14_attention_weight_max": 40.51586151123047,
      "activations/layer14_attention_weight_min": -37.54518508911133,
      "activations/layer15_attention_weight_max": 35.70184326171875,
      "activations/layer15_attention_weight_min": -32.74456787109375,
      "activations/layer16_attention_weight_max": 30.721818923950195,
      "activations/layer16_attention_weight_min": -27.34174346923828,
      "activations/layer17_attention_weight_max": 28.232563018798828,
      "activations/layer17_attention_weight_min": -23.8995304107666,
      "activations/layer18_attention_weight_max": 31.34063148498535,
      "activations/layer18_attention_weight_min": -23.79246711730957,
      "activations/layer19_attention_weight_max": 37.35221862792969,
      "activations/layer19_attention_weight_min": -30.653623580932617,
      "activations/layer1_attention_weight_max": 16.231901168823242,
      "activations/layer1_attention_weight_min": -13.188905715942383,
      "activations/layer20_attention_weight_max": 32.6882438659668,
      "activations/layer20_attention_weight_min": -25.752225875854492,
      "activations/layer21_attention_weight_max": 30.841161727905273,
      "activations/layer21_attention_weight_min": -24.90341567993164,
      "activations/layer22_attention_weight_max": 41.60089111328125,
      "activations/layer22_attention_weight_min": -30.001190185546875,
      "activations/layer23_attention_weight_max": 34.628501892089844,
      "activations/layer23_attention_weight_min": -25.107519149780273,
      "activations/layer2_attention_weight_max": 34.600135803222656,
      "activations/layer2_attention_weight_min": -33.371944427490234,
      "activations/layer3_attention_weight_max": 96.35321044921875,
      "activations/layer3_attention_weight_min": -97.50205993652344,
      "activations/layer4_attention_weight_max": 106.35717010498047,
      "activations/layer4_attention_weight_min": -107.5777587890625,
      "activations/layer5_attention_weight_max": 50.25182342529297,
      "activations/layer5_attention_weight_min": -62.20046615600586,
      "activations/layer6_attention_weight_max": 44.36064147949219,
      "activations/layer6_attention_weight_min": -45.19791030883789,
      "activations/layer7_attention_weight_max": 90.99578857421875,
      "activations/layer7_attention_weight_min": -100.68698120117188,
      "activations/layer8_attention_weight_max": 39.743038177490234,
      "activations/layer8_attention_weight_min": -41.89649963378906,
      "activations/layer9_attention_weight_max": 30.166227340698242,
      "activations/layer9_attention_weight_min": -31.840139389038086,
      "epoch": 14.41,
      "learning_rate": 5.763371212121212e-05,
      "loss": 2.737,
      "step": 248050
    },
    {
      "activations/layer0_attention_weight_max": 14.987722396850586,
      "activations/layer0_attention_weight_min": -12.40289306640625,
      "activations/layer10_attention_weight_max": 32.8330192565918,
      "activations/layer10_attention_weight_min": -33.616146087646484,
      "activations/layer11_attention_weight_max": 33.485267639160156,
      "activations/layer11_attention_weight_min": -31.326995849609375,
      "activations/layer12_attention_weight_max": 26.78516960144043,
      "activations/layer12_attention_weight_min": -25.05624008178711,
      "activations/layer13_attention_weight_max": 41.292022705078125,
      "activations/layer13_attention_weight_min": -33.79738998413086,
      "activations/layer14_attention_weight_max": 41.28251266479492,
      "activations/layer14_attention_weight_min": -37.61777877807617,
      "activations/layer15_attention_weight_max": 37.19184494018555,
      "activations/layer15_attention_weight_min": -32.19744873046875,
      "activations/layer16_attention_weight_max": 28.5133056640625,
      "activations/layer16_attention_weight_min": -25.362783432006836,
      "activations/layer17_attention_weight_max": 34.21209716796875,
      "activations/layer17_attention_weight_min": -24.96796989440918,
      "activations/layer18_attention_weight_max": 31.429180145263672,
      "activations/layer18_attention_weight_min": -23.335960388183594,
      "activations/layer19_attention_weight_max": 32.858272552490234,
      "activations/layer19_attention_weight_min": -29.46142578125,
      "activations/layer1_attention_weight_max": 15.850632667541504,
      "activations/layer1_attention_weight_min": -14.310571670532227,
      "activations/layer20_attention_weight_max": 29.420642852783203,
      "activations/layer20_attention_weight_min": -24.244548797607422,
      "activations/layer21_attention_weight_max": 27.237411499023438,
      "activations/layer21_attention_weight_min": -22.243459701538086,
      "activations/layer22_attention_weight_max": 44.671348571777344,
      "activations/layer22_attention_weight_min": -28.748586654663086,
      "activations/layer23_attention_weight_max": 32.169307708740234,
      "activations/layer23_attention_weight_min": -23.23940086364746,
      "activations/layer2_attention_weight_max": 30.641876220703125,
      "activations/layer2_attention_weight_min": -32.3798828125,
      "activations/layer3_attention_weight_max": 97.36126708984375,
      "activations/layer3_attention_weight_min": -98.71145629882812,
      "activations/layer4_attention_weight_max": 109.34769439697266,
      "activations/layer4_attention_weight_min": -112.25825500488281,
      "activations/layer5_attention_weight_max": 55.44108581542969,
      "activations/layer5_attention_weight_min": -65.6778564453125,
      "activations/layer6_attention_weight_max": 43.87792205810547,
      "activations/layer6_attention_weight_min": -49.28200912475586,
      "activations/layer7_attention_weight_max": 86.63569641113281,
      "activations/layer7_attention_weight_min": -97.8166732788086,
      "activations/layer8_attention_weight_max": 39.63359451293945,
      "activations/layer8_attention_weight_min": -41.58058547973633,
      "activations/layer9_attention_weight_max": 33.2172966003418,
      "activations/layer9_attention_weight_min": -33.93077850341797,
      "epoch": 14.42,
      "learning_rate": 5.761477272727272e-05,
      "loss": 2.7728,
      "step": 248100
    },
    {
      "activations/layer0_attention_weight_max": 15.252134323120117,
      "activations/layer0_attention_weight_min": -12.956459045410156,
      "activations/layer10_attention_weight_max": 32.84061813354492,
      "activations/layer10_attention_weight_min": -34.69855880737305,
      "activations/layer11_attention_weight_max": 33.57502746582031,
      "activations/layer11_attention_weight_min": -33.53438949584961,
      "activations/layer12_attention_weight_max": 26.692880630493164,
      "activations/layer12_attention_weight_min": -26.49942970275879,
      "activations/layer13_attention_weight_max": 39.655426025390625,
      "activations/layer13_attention_weight_min": -34.784637451171875,
      "activations/layer14_attention_weight_max": 38.96183395385742,
      "activations/layer14_attention_weight_min": -38.62068557739258,
      "activations/layer15_attention_weight_max": 37.15467071533203,
      "activations/layer15_attention_weight_min": -33.59396743774414,
      "activations/layer16_attention_weight_max": 32.26911926269531,
      "activations/layer16_attention_weight_min": -29.98847198486328,
      "activations/layer17_attention_weight_max": 30.177608489990234,
      "activations/layer17_attention_weight_min": -27.18922233581543,
      "activations/layer18_attention_weight_max": 30.50751304626465,
      "activations/layer18_attention_weight_min": -24.65599250793457,
      "activations/layer19_attention_weight_max": 32.82114791870117,
      "activations/layer19_attention_weight_min": -32.1325798034668,
      "activations/layer1_attention_weight_max": 16.545597076416016,
      "activations/layer1_attention_weight_min": -14.80443286895752,
      "activations/layer20_attention_weight_max": 29.495616912841797,
      "activations/layer20_attention_weight_min": -25.73576545715332,
      "activations/layer21_attention_weight_max": 29.289306640625,
      "activations/layer21_attention_weight_min": -24.892303466796875,
      "activations/layer22_attention_weight_max": 38.18099594116211,
      "activations/layer22_attention_weight_min": -31.485506057739258,
      "activations/layer23_attention_weight_max": 29.383275985717773,
      "activations/layer23_attention_weight_min": -25.86269187927246,
      "activations/layer2_attention_weight_max": 33.41024398803711,
      "activations/layer2_attention_weight_min": -32.169464111328125,
      "activations/layer3_attention_weight_max": 94.6344223022461,
      "activations/layer3_attention_weight_min": -93.25302124023438,
      "activations/layer4_attention_weight_max": 110.6917495727539,
      "activations/layer4_attention_weight_min": -114.27230072021484,
      "activations/layer5_attention_weight_max": 52.1462516784668,
      "activations/layer5_attention_weight_min": -68.37094116210938,
      "activations/layer6_attention_weight_max": 43.41643142700195,
      "activations/layer6_attention_weight_min": -49.47862243652344,
      "activations/layer7_attention_weight_max": 90.0047836303711,
      "activations/layer7_attention_weight_min": -93.8131332397461,
      "activations/layer8_attention_weight_max": 40.24306106567383,
      "activations/layer8_attention_weight_min": -42.74464797973633,
      "activations/layer9_attention_weight_max": 31.828275680541992,
      "activations/layer9_attention_weight_min": -33.80764389038086,
      "epoch": 14.42,
      "learning_rate": 5.759583333333333e-05,
      "loss": 2.7637,
      "step": 248150
    },
    {
      "activations/layer0_attention_weight_max": 15.343486785888672,
      "activations/layer0_attention_weight_min": -12.004450798034668,
      "activations/layer10_attention_weight_max": 32.74677276611328,
      "activations/layer10_attention_weight_min": -32.724693298339844,
      "activations/layer11_attention_weight_max": 32.36722946166992,
      "activations/layer11_attention_weight_min": -32.6300048828125,
      "activations/layer12_attention_weight_max": 26.319520950317383,
      "activations/layer12_attention_weight_min": -26.242341995239258,
      "activations/layer13_attention_weight_max": 39.46826171875,
      "activations/layer13_attention_weight_min": -34.49213409423828,
      "activations/layer14_attention_weight_max": 43.14433288574219,
      "activations/layer14_attention_weight_min": -39.16960144042969,
      "activations/layer15_attention_weight_max": 38.8480224609375,
      "activations/layer15_attention_weight_min": -33.991127014160156,
      "activations/layer16_attention_weight_max": 30.269702911376953,
      "activations/layer16_attention_weight_min": -27.83705711364746,
      "activations/layer17_attention_weight_max": 32.00189208984375,
      "activations/layer17_attention_weight_min": -26.10601234436035,
      "activations/layer18_attention_weight_max": 33.8754768371582,
      "activations/layer18_attention_weight_min": -22.82048225402832,
      "activations/layer19_attention_weight_max": 37.534873962402344,
      "activations/layer19_attention_weight_min": -29.113412857055664,
      "activations/layer1_attention_weight_max": 16.472103118896484,
      "activations/layer1_attention_weight_min": -15.61659049987793,
      "activations/layer20_attention_weight_max": 31.540353775024414,
      "activations/layer20_attention_weight_min": -23.74103546142578,
      "activations/layer21_attention_weight_max": 29.769975662231445,
      "activations/layer21_attention_weight_min": -22.841176986694336,
      "activations/layer22_attention_weight_max": 45.235809326171875,
      "activations/layer22_attention_weight_min": -30.05685043334961,
      "activations/layer23_attention_weight_max": 35.61661911010742,
      "activations/layer23_attention_weight_min": -23.973066329956055,
      "activations/layer2_attention_weight_max": 31.093692779541016,
      "activations/layer2_attention_weight_min": -31.27076530456543,
      "activations/layer3_attention_weight_max": 93.83001708984375,
      "activations/layer3_attention_weight_min": -97.70081329345703,
      "activations/layer4_attention_weight_max": 110.6446304321289,
      "activations/layer4_attention_weight_min": -111.7137222290039,
      "activations/layer5_attention_weight_max": 51.9553337097168,
      "activations/layer5_attention_weight_min": -61.38517379760742,
      "activations/layer6_attention_weight_max": 44.4704475402832,
      "activations/layer6_attention_weight_min": -47.85070037841797,
      "activations/layer7_attention_weight_max": 90.96538543701172,
      "activations/layer7_attention_weight_min": -97.1836929321289,
      "activations/layer8_attention_weight_max": 41.52451705932617,
      "activations/layer8_attention_weight_min": -42.5043830871582,
      "activations/layer9_attention_weight_max": 32.299556732177734,
      "activations/layer9_attention_weight_min": -33.27535629272461,
      "epoch": 14.42,
      "learning_rate": 5.757689393939393e-05,
      "loss": 2.7474,
      "step": 248200
    },
    {
      "activations/layer0_attention_weight_max": 15.160510063171387,
      "activations/layer0_attention_weight_min": -12.81985855102539,
      "activations/layer10_attention_weight_max": 33.12422561645508,
      "activations/layer10_attention_weight_min": -33.0230598449707,
      "activations/layer11_attention_weight_max": 34.201873779296875,
      "activations/layer11_attention_weight_min": -33.229766845703125,
      "activations/layer12_attention_weight_max": 26.729515075683594,
      "activations/layer12_attention_weight_min": -28.301244735717773,
      "activations/layer13_attention_weight_max": 39.07684326171875,
      "activations/layer13_attention_weight_min": -37.09902572631836,
      "activations/layer14_attention_weight_max": 41.90536880493164,
      "activations/layer14_attention_weight_min": -37.47233200073242,
      "activations/layer15_attention_weight_max": 37.95002365112305,
      "activations/layer15_attention_weight_min": -32.76783752441406,
      "activations/layer16_attention_weight_max": 28.103288650512695,
      "activations/layer16_attention_weight_min": -26.714380264282227,
      "activations/layer17_attention_weight_max": 28.660409927368164,
      "activations/layer17_attention_weight_min": -25.226774215698242,
      "activations/layer18_attention_weight_max": 29.012245178222656,
      "activations/layer18_attention_weight_min": -23.785375595092773,
      "activations/layer19_attention_weight_max": 33.65532302856445,
      "activations/layer19_attention_weight_min": -32.04011154174805,
      "activations/layer1_attention_weight_max": 16.184274673461914,
      "activations/layer1_attention_weight_min": -14.240447044372559,
      "activations/layer20_attention_weight_max": 29.781503677368164,
      "activations/layer20_attention_weight_min": -24.321975708007812,
      "activations/layer21_attention_weight_max": 29.362030029296875,
      "activations/layer21_attention_weight_min": -24.536388397216797,
      "activations/layer22_attention_weight_max": 40.25232696533203,
      "activations/layer22_attention_weight_min": -29.57109832763672,
      "activations/layer23_attention_weight_max": 32.5679817199707,
      "activations/layer23_attention_weight_min": -26.60903549194336,
      "activations/layer2_attention_weight_max": 34.26734161376953,
      "activations/layer2_attention_weight_min": -32.88509750366211,
      "activations/layer3_attention_weight_max": 100.26404571533203,
      "activations/layer3_attention_weight_min": -99.55980682373047,
      "activations/layer4_attention_weight_max": 111.4449234008789,
      "activations/layer4_attention_weight_min": -107.48885345458984,
      "activations/layer5_attention_weight_max": 51.04322814941406,
      "activations/layer5_attention_weight_min": -59.79663848876953,
      "activations/layer6_attention_weight_max": 42.78554916381836,
      "activations/layer6_attention_weight_min": -46.955867767333984,
      "activations/layer7_attention_weight_max": 89.33650207519531,
      "activations/layer7_attention_weight_min": -91.64722442626953,
      "activations/layer8_attention_weight_max": 41.49747085571289,
      "activations/layer8_attention_weight_min": -41.771148681640625,
      "activations/layer9_attention_weight_max": 32.621883392333984,
      "activations/layer9_attention_weight_min": -34.67019271850586,
      "epoch": 14.42,
      "learning_rate": 5.7557954545454545e-05,
      "loss": 2.7531,
      "step": 248250
    },
    {
      "activations/layer0_attention_weight_max": 14.920695304870605,
      "activations/layer0_attention_weight_min": -11.906716346740723,
      "activations/layer10_attention_weight_max": 32.9034423828125,
      "activations/layer10_attention_weight_min": -31.775632858276367,
      "activations/layer11_attention_weight_max": 32.29949951171875,
      "activations/layer11_attention_weight_min": -31.96485137939453,
      "activations/layer12_attention_weight_max": 25.112014770507812,
      "activations/layer12_attention_weight_min": -31.04005241394043,
      "activations/layer13_attention_weight_max": 40.17476272583008,
      "activations/layer13_attention_weight_min": -36.922889709472656,
      "activations/layer14_attention_weight_max": 41.929588317871094,
      "activations/layer14_attention_weight_min": -37.115474700927734,
      "activations/layer15_attention_weight_max": 36.54944610595703,
      "activations/layer15_attention_weight_min": -32.694766998291016,
      "activations/layer16_attention_weight_max": 31.82828712463379,
      "activations/layer16_attention_weight_min": -26.322256088256836,
      "activations/layer17_attention_weight_max": 29.60685920715332,
      "activations/layer17_attention_weight_min": -25.6629581451416,
      "activations/layer18_attention_weight_max": 30.458566665649414,
      "activations/layer18_attention_weight_min": -23.891117095947266,
      "activations/layer19_attention_weight_max": 32.898799896240234,
      "activations/layer19_attention_weight_min": -29.827219009399414,
      "activations/layer1_attention_weight_max": 15.71160888671875,
      "activations/layer1_attention_weight_min": -14.056143760681152,
      "activations/layer20_attention_weight_max": 28.013566970825195,
      "activations/layer20_attention_weight_min": -24.92000389099121,
      "activations/layer21_attention_weight_max": 29.19784927368164,
      "activations/layer21_attention_weight_min": -24.785364151000977,
      "activations/layer22_attention_weight_max": 45.4865837097168,
      "activations/layer22_attention_weight_min": -30.43119239807129,
      "activations/layer23_attention_weight_max": 31.09015464782715,
      "activations/layer23_attention_weight_min": -25.17743492126465,
      "activations/layer2_attention_weight_max": 34.00238800048828,
      "activations/layer2_attention_weight_min": -33.59062957763672,
      "activations/layer3_attention_weight_max": 95.3006362915039,
      "activations/layer3_attention_weight_min": -94.50930786132812,
      "activations/layer4_attention_weight_max": 106.79540252685547,
      "activations/layer4_attention_weight_min": -109.61376190185547,
      "activations/layer5_attention_weight_max": 50.96308898925781,
      "activations/layer5_attention_weight_min": -60.79566955566406,
      "activations/layer6_attention_weight_max": 43.343936920166016,
      "activations/layer6_attention_weight_min": -45.706138610839844,
      "activations/layer7_attention_weight_max": 87.65571594238281,
      "activations/layer7_attention_weight_min": -91.2278060913086,
      "activations/layer8_attention_weight_max": 38.83668899536133,
      "activations/layer8_attention_weight_min": -43.23212432861328,
      "activations/layer9_attention_weight_max": 30.967472076416016,
      "activations/layer9_attention_weight_min": -33.841575622558594,
      "epoch": 14.43,
      "learning_rate": 5.7539015151515146e-05,
      "loss": 2.7679,
      "step": 248300
    },
    {
      "activations/layer0_attention_weight_max": 15.918977737426758,
      "activations/layer0_attention_weight_min": -13.456867218017578,
      "activations/layer10_attention_weight_max": 32.887813568115234,
      "activations/layer10_attention_weight_min": -31.817136764526367,
      "activations/layer11_attention_weight_max": 33.70951843261719,
      "activations/layer11_attention_weight_min": -32.359832763671875,
      "activations/layer12_attention_weight_max": 25.568994522094727,
      "activations/layer12_attention_weight_min": -26.145240783691406,
      "activations/layer13_attention_weight_max": 38.4481086730957,
      "activations/layer13_attention_weight_min": -34.42466735839844,
      "activations/layer14_attention_weight_max": 41.11666488647461,
      "activations/layer14_attention_weight_min": -35.966514587402344,
      "activations/layer15_attention_weight_max": 36.530582427978516,
      "activations/layer15_attention_weight_min": -33.81487274169922,
      "activations/layer16_attention_weight_max": 30.920909881591797,
      "activations/layer16_attention_weight_min": -27.306716918945312,
      "activations/layer17_attention_weight_max": 29.755550384521484,
      "activations/layer17_attention_weight_min": -25.32505226135254,
      "activations/layer18_attention_weight_max": 32.744441986083984,
      "activations/layer18_attention_weight_min": -25.247371673583984,
      "activations/layer19_attention_weight_max": 38.147857666015625,
      "activations/layer19_attention_weight_min": -33.034141540527344,
      "activations/layer1_attention_weight_max": 17.085189819335938,
      "activations/layer1_attention_weight_min": -14.545433044433594,
      "activations/layer20_attention_weight_max": 31.513137817382812,
      "activations/layer20_attention_weight_min": -27.052146911621094,
      "activations/layer21_attention_weight_max": 32.03812789916992,
      "activations/layer21_attention_weight_min": -26.243160247802734,
      "activations/layer22_attention_weight_max": 40.812355041503906,
      "activations/layer22_attention_weight_min": -31.792491912841797,
      "activations/layer23_attention_weight_max": 39.52336502075195,
      "activations/layer23_attention_weight_min": -25.30428123474121,
      "activations/layer2_attention_weight_max": 32.063228607177734,
      "activations/layer2_attention_weight_min": -32.62550735473633,
      "activations/layer3_attention_weight_max": 97.3419418334961,
      "activations/layer3_attention_weight_min": -98.56610870361328,
      "activations/layer4_attention_weight_max": 109.43206787109375,
      "activations/layer4_attention_weight_min": -111.47652435302734,
      "activations/layer5_attention_weight_max": 54.902809143066406,
      "activations/layer5_attention_weight_min": -62.914390563964844,
      "activations/layer6_attention_weight_max": 43.96907043457031,
      "activations/layer6_attention_weight_min": -46.95677185058594,
      "activations/layer7_attention_weight_max": 93.20321655273438,
      "activations/layer7_attention_weight_min": -92.16480255126953,
      "activations/layer8_attention_weight_max": 39.64030838012695,
      "activations/layer8_attention_weight_min": -41.25789260864258,
      "activations/layer9_attention_weight_max": 32.16200637817383,
      "activations/layer9_attention_weight_min": -32.15937042236328,
      "epoch": 14.43,
      "learning_rate": 5.752007575757575e-05,
      "loss": 2.7556,
      "step": 248350
    },
    {
      "activations/layer0_attention_weight_max": 15.982793807983398,
      "activations/layer0_attention_weight_min": -12.020525932312012,
      "activations/layer10_attention_weight_max": 41.71260452270508,
      "activations/layer10_attention_weight_min": -39.12486267089844,
      "activations/layer11_attention_weight_max": 42.154747009277344,
      "activations/layer11_attention_weight_min": -41.110145568847656,
      "activations/layer12_attention_weight_max": 29.064376831054688,
      "activations/layer12_attention_weight_min": -28.444034576416016,
      "activations/layer13_attention_weight_max": 43.17460632324219,
      "activations/layer13_attention_weight_min": -36.14704132080078,
      "activations/layer14_attention_weight_max": 51.602996826171875,
      "activations/layer14_attention_weight_min": -46.75290298461914,
      "activations/layer15_attention_weight_max": 42.31752395629883,
      "activations/layer15_attention_weight_min": -40.06441116333008,
      "activations/layer16_attention_weight_max": 37.26456832885742,
      "activations/layer16_attention_weight_min": -28.557971954345703,
      "activations/layer17_attention_weight_max": 30.556299209594727,
      "activations/layer17_attention_weight_min": -26.669198989868164,
      "activations/layer18_attention_weight_max": 32.04876708984375,
      "activations/layer18_attention_weight_min": -25.64497947692871,
      "activations/layer19_attention_weight_max": 33.300228118896484,
      "activations/layer19_attention_weight_min": -33.56479263305664,
      "activations/layer1_attention_weight_max": 17.4006290435791,
      "activations/layer1_attention_weight_min": -15.681984901428223,
      "activations/layer20_attention_weight_max": 29.069862365722656,
      "activations/layer20_attention_weight_min": -26.670692443847656,
      "activations/layer21_attention_weight_max": 28.60790252685547,
      "activations/layer21_attention_weight_min": -25.35919189453125,
      "activations/layer22_attention_weight_max": 40.88859558105469,
      "activations/layer22_attention_weight_min": -30.06454849243164,
      "activations/layer23_attention_weight_max": 30.942180633544922,
      "activations/layer23_attention_weight_min": -26.856786727905273,
      "activations/layer2_attention_weight_max": 36.59592056274414,
      "activations/layer2_attention_weight_min": -35.26959991455078,
      "activations/layer3_attention_weight_max": 112.73928833007812,
      "activations/layer3_attention_weight_min": -107.35498046875,
      "activations/layer4_attention_weight_max": 115.8819808959961,
      "activations/layer4_attention_weight_min": -113.90980529785156,
      "activations/layer5_attention_weight_max": 55.16752624511719,
      "activations/layer5_attention_weight_min": -65.18708801269531,
      "activations/layer6_attention_weight_max": 49.190284729003906,
      "activations/layer6_attention_weight_min": -48.28158950805664,
      "activations/layer7_attention_weight_max": 113.35099029541016,
      "activations/layer7_attention_weight_min": -98.58644104003906,
      "activations/layer8_attention_weight_max": 49.45619583129883,
      "activations/layer8_attention_weight_min": -46.8810920715332,
      "activations/layer9_attention_weight_max": 41.01247787475586,
      "activations/layer9_attention_weight_min": -39.2310905456543,
      "epoch": 14.43,
      "learning_rate": 5.7501136363636356e-05,
      "loss": 2.7387,
      "step": 248400
    },
    {
      "activations/layer0_attention_weight_max": 15.404960632324219,
      "activations/layer0_attention_weight_min": -12.071191787719727,
      "activations/layer10_attention_weight_max": 37.00654983520508,
      "activations/layer10_attention_weight_min": -36.15427017211914,
      "activations/layer11_attention_weight_max": 35.18482971191406,
      "activations/layer11_attention_weight_min": -35.94829559326172,
      "activations/layer12_attention_weight_max": 28.524168014526367,
      "activations/layer12_attention_weight_min": -28.446979522705078,
      "activations/layer13_attention_weight_max": 43.87588119506836,
      "activations/layer13_attention_weight_min": -36.22880554199219,
      "activations/layer14_attention_weight_max": 48.087955474853516,
      "activations/layer14_attention_weight_min": -38.504051208496094,
      "activations/layer15_attention_weight_max": 45.12492752075195,
      "activations/layer15_attention_weight_min": -35.346839904785156,
      "activations/layer16_attention_weight_max": 32.38711166381836,
      "activations/layer16_attention_weight_min": -29.67317008972168,
      "activations/layer17_attention_weight_max": 31.75895881652832,
      "activations/layer17_attention_weight_min": -25.30972671508789,
      "activations/layer18_attention_weight_max": 33.82454299926758,
      "activations/layer18_attention_weight_min": -25.673599243164062,
      "activations/layer19_attention_weight_max": 40.059486389160156,
      "activations/layer19_attention_weight_min": -33.6734733581543,
      "activations/layer1_attention_weight_max": 16.43383026123047,
      "activations/layer1_attention_weight_min": -15.97205924987793,
      "activations/layer20_attention_weight_max": 33.4273796081543,
      "activations/layer20_attention_weight_min": -23.060049057006836,
      "activations/layer21_attention_weight_max": 31.99132537841797,
      "activations/layer21_attention_weight_min": -22.681888580322266,
      "activations/layer22_attention_weight_max": 49.72880935668945,
      "activations/layer22_attention_weight_min": -28.07417106628418,
      "activations/layer23_attention_weight_max": 35.81077575683594,
      "activations/layer23_attention_weight_min": -23.68715476989746,
      "activations/layer2_attention_weight_max": 33.60590362548828,
      "activations/layer2_attention_weight_min": -32.81914520263672,
      "activations/layer3_attention_weight_max": 99.21253967285156,
      "activations/layer3_attention_weight_min": -100.78541564941406,
      "activations/layer4_attention_weight_max": 113.5750961303711,
      "activations/layer4_attention_weight_min": -110.6994857788086,
      "activations/layer5_attention_weight_max": 51.92446517944336,
      "activations/layer5_attention_weight_min": -59.46977996826172,
      "activations/layer6_attention_weight_max": 44.06998825073242,
      "activations/layer6_attention_weight_min": -47.11821365356445,
      "activations/layer7_attention_weight_max": 93.80805969238281,
      "activations/layer7_attention_weight_min": -92.13397979736328,
      "activations/layer8_attention_weight_max": 42.94906234741211,
      "activations/layer8_attention_weight_min": -42.73311233520508,
      "activations/layer9_attention_weight_max": 32.0367546081543,
      "activations/layer9_attention_weight_min": -34.3799934387207,
      "epoch": 14.44,
      "learning_rate": 5.7482196969696964e-05,
      "loss": 2.7451,
      "step": 248450
    },
    {
      "activations/layer0_attention_weight_max": 16.620595932006836,
      "activations/layer0_attention_weight_min": -13.318629264831543,
      "activations/layer10_attention_weight_max": 34.81643295288086,
      "activations/layer10_attention_weight_min": -32.93778610229492,
      "activations/layer11_attention_weight_max": 34.39047622680664,
      "activations/layer11_attention_weight_min": -33.296241760253906,
      "activations/layer12_attention_weight_max": 27.50931739807129,
      "activations/layer12_attention_weight_min": -24.5975399017334,
      "activations/layer13_attention_weight_max": 42.15647506713867,
      "activations/layer13_attention_weight_min": -35.72243881225586,
      "activations/layer14_attention_weight_max": 45.23908615112305,
      "activations/layer14_attention_weight_min": -36.029754638671875,
      "activations/layer15_attention_weight_max": 41.407493591308594,
      "activations/layer15_attention_weight_min": -31.71474838256836,
      "activations/layer16_attention_weight_max": 29.384445190429688,
      "activations/layer16_attention_weight_min": -26.206357955932617,
      "activations/layer17_attention_weight_max": 31.82637596130371,
      "activations/layer17_attention_weight_min": -24.094011306762695,
      "activations/layer18_attention_weight_max": 32.014949798583984,
      "activations/layer18_attention_weight_min": -23.66681480407715,
      "activations/layer19_attention_weight_max": 33.351806640625,
      "activations/layer19_attention_weight_min": -30.199716567993164,
      "activations/layer1_attention_weight_max": 16.769224166870117,
      "activations/layer1_attention_weight_min": -15.313050270080566,
      "activations/layer20_attention_weight_max": 28.29083251953125,
      "activations/layer20_attention_weight_min": -23.655607223510742,
      "activations/layer21_attention_weight_max": 31.591848373413086,
      "activations/layer21_attention_weight_min": -25.324838638305664,
      "activations/layer22_attention_weight_max": 44.41244888305664,
      "activations/layer22_attention_weight_min": -29.56101417541504,
      "activations/layer23_attention_weight_max": 34.43387985229492,
      "activations/layer23_attention_weight_min": -23.689708709716797,
      "activations/layer2_attention_weight_max": 31.71599578857422,
      "activations/layer2_attention_weight_min": -32.616912841796875,
      "activations/layer3_attention_weight_max": 96.94937133789062,
      "activations/layer3_attention_weight_min": -98.02776336669922,
      "activations/layer4_attention_weight_max": 112.6146469116211,
      "activations/layer4_attention_weight_min": -110.85045623779297,
      "activations/layer5_attention_weight_max": 52.1806755065918,
      "activations/layer5_attention_weight_min": -59.63985824584961,
      "activations/layer6_attention_weight_max": 48.002723693847656,
      "activations/layer6_attention_weight_min": -54.23207473754883,
      "activations/layer7_attention_weight_max": 91.4190673828125,
      "activations/layer7_attention_weight_min": -98.36626434326172,
      "activations/layer8_attention_weight_max": 41.58165740966797,
      "activations/layer8_attention_weight_min": -46.73858642578125,
      "activations/layer9_attention_weight_max": 31.505279541015625,
      "activations/layer9_attention_weight_min": -33.384864807128906,
      "epoch": 14.44,
      "learning_rate": 5.746325757575757e-05,
      "loss": 2.7398,
      "step": 248500
    },
    {
      "activations/layer0_attention_weight_max": 15.357389450073242,
      "activations/layer0_attention_weight_min": -12.38095760345459,
      "activations/layer10_attention_weight_max": 45.84730911254883,
      "activations/layer10_attention_weight_min": -42.71925354003906,
      "activations/layer11_attention_weight_max": 47.05461883544922,
      "activations/layer11_attention_weight_min": -45.109657287597656,
      "activations/layer12_attention_weight_max": 35.07551574707031,
      "activations/layer12_attention_weight_min": -31.6455020904541,
      "activations/layer13_attention_weight_max": 56.64410400390625,
      "activations/layer13_attention_weight_min": -42.918636322021484,
      "activations/layer14_attention_weight_max": 62.487274169921875,
      "activations/layer14_attention_weight_min": -46.34005355834961,
      "activations/layer15_attention_weight_max": 65.20287322998047,
      "activations/layer15_attention_weight_min": -43.668941497802734,
      "activations/layer16_attention_weight_max": 40.20475769042969,
      "activations/layer16_attention_weight_min": -29.047548294067383,
      "activations/layer17_attention_weight_max": 33.513187408447266,
      "activations/layer17_attention_weight_min": -30.02621078491211,
      "activations/layer18_attention_weight_max": 30.000694274902344,
      "activations/layer18_attention_weight_min": -27.731969833374023,
      "activations/layer19_attention_weight_max": 36.85023498535156,
      "activations/layer19_attention_weight_min": -30.632537841796875,
      "activations/layer1_attention_weight_max": 16.902681350708008,
      "activations/layer1_attention_weight_min": -15.608634948730469,
      "activations/layer20_attention_weight_max": 30.470102310180664,
      "activations/layer20_attention_weight_min": -26.552946090698242,
      "activations/layer21_attention_weight_max": 27.912357330322266,
      "activations/layer21_attention_weight_min": -26.886760711669922,
      "activations/layer22_attention_weight_max": 47.99583435058594,
      "activations/layer22_attention_weight_min": -35.90354919433594,
      "activations/layer23_attention_weight_max": 33.075469970703125,
      "activations/layer23_attention_weight_min": -24.02191734313965,
      "activations/layer2_attention_weight_max": 34.73131561279297,
      "activations/layer2_attention_weight_min": -32.728328704833984,
      "activations/layer3_attention_weight_max": 98.32803344726562,
      "activations/layer3_attention_weight_min": -98.90227508544922,
      "activations/layer4_attention_weight_max": 115.92198944091797,
      "activations/layer4_attention_weight_min": -109.93775939941406,
      "activations/layer5_attention_weight_max": 57.23152160644531,
      "activations/layer5_attention_weight_min": -62.067291259765625,
      "activations/layer6_attention_weight_max": 46.06330108642578,
      "activations/layer6_attention_weight_min": -47.3647346496582,
      "activations/layer7_attention_weight_max": 105.1759262084961,
      "activations/layer7_attention_weight_min": -103.16694641113281,
      "activations/layer8_attention_weight_max": 50.47465896606445,
      "activations/layer8_attention_weight_min": -49.2353401184082,
      "activations/layer9_attention_weight_max": 41.880489349365234,
      "activations/layer9_attention_weight_min": -41.18483352661133,
      "epoch": 14.44,
      "learning_rate": 5.7444318181818174e-05,
      "loss": 2.7579,
      "step": 248550
    },
    {
      "activations/layer0_attention_weight_max": 15.329955101013184,
      "activations/layer0_attention_weight_min": -11.751728057861328,
      "activations/layer10_attention_weight_max": 33.522666931152344,
      "activations/layer10_attention_weight_min": -32.056148529052734,
      "activations/layer11_attention_weight_max": 32.84125900268555,
      "activations/layer11_attention_weight_min": -31.96468162536621,
      "activations/layer12_attention_weight_max": 26.648889541625977,
      "activations/layer12_attention_weight_min": -27.96209144592285,
      "activations/layer13_attention_weight_max": 39.35697555541992,
      "activations/layer13_attention_weight_min": -36.54635238647461,
      "activations/layer14_attention_weight_max": 48.210933685302734,
      "activations/layer14_attention_weight_min": -38.846229553222656,
      "activations/layer15_attention_weight_max": 35.952491760253906,
      "activations/layer15_attention_weight_min": -33.685325622558594,
      "activations/layer16_attention_weight_max": 29.4267635345459,
      "activations/layer16_attention_weight_min": -26.855783462524414,
      "activations/layer17_attention_weight_max": 31.723764419555664,
      "activations/layer17_attention_weight_min": -25.481481552124023,
      "activations/layer18_attention_weight_max": 32.85106658935547,
      "activations/layer18_attention_weight_min": -26.208431243896484,
      "activations/layer19_attention_weight_max": 36.820552825927734,
      "activations/layer19_attention_weight_min": -31.961837768554688,
      "activations/layer1_attention_weight_max": 16.321083068847656,
      "activations/layer1_attention_weight_min": -14.242682456970215,
      "activations/layer20_attention_weight_max": 28.65093994140625,
      "activations/layer20_attention_weight_min": -25.599557876586914,
      "activations/layer21_attention_weight_max": 28.0823974609375,
      "activations/layer21_attention_weight_min": -24.1450252532959,
      "activations/layer22_attention_weight_max": 41.53200912475586,
      "activations/layer22_attention_weight_min": -32.21558380126953,
      "activations/layer23_attention_weight_max": 33.54175567626953,
      "activations/layer23_attention_weight_min": -27.429737091064453,
      "activations/layer2_attention_weight_max": 33.926971435546875,
      "activations/layer2_attention_weight_min": -33.07447814941406,
      "activations/layer3_attention_weight_max": 97.61823272705078,
      "activations/layer3_attention_weight_min": -98.5407485961914,
      "activations/layer4_attention_weight_max": 109.41507720947266,
      "activations/layer4_attention_weight_min": -110.40389251708984,
      "activations/layer5_attention_weight_max": 53.21372604370117,
      "activations/layer5_attention_weight_min": -59.67372131347656,
      "activations/layer6_attention_weight_max": 44.28443908691406,
      "activations/layer6_attention_weight_min": -46.27651596069336,
      "activations/layer7_attention_weight_max": 96.45726013183594,
      "activations/layer7_attention_weight_min": -92.18766021728516,
      "activations/layer8_attention_weight_max": 41.64803695678711,
      "activations/layer8_attention_weight_min": -41.57558059692383,
      "activations/layer9_attention_weight_max": 31.455862045288086,
      "activations/layer9_attention_weight_min": -34.13340377807617,
      "epoch": 14.45,
      "learning_rate": 5.742537878787879e-05,
      "loss": 2.7545,
      "step": 248600
    },
    {
      "activations/layer0_attention_weight_max": 15.798598289489746,
      "activations/layer0_attention_weight_min": -12.386531829833984,
      "activations/layer10_attention_weight_max": 36.39033126831055,
      "activations/layer10_attention_weight_min": -34.542415618896484,
      "activations/layer11_attention_weight_max": 36.226463317871094,
      "activations/layer11_attention_weight_min": -33.94769287109375,
      "activations/layer12_attention_weight_max": 27.807580947875977,
      "activations/layer12_attention_weight_min": -26.915315628051758,
      "activations/layer13_attention_weight_max": 44.932804107666016,
      "activations/layer13_attention_weight_min": -36.62097930908203,
      "activations/layer14_attention_weight_max": 43.21437454223633,
      "activations/layer14_attention_weight_min": -37.08369445800781,
      "activations/layer15_attention_weight_max": 38.951637268066406,
      "activations/layer15_attention_weight_min": -32.625545501708984,
      "activations/layer16_attention_weight_max": 29.49755859375,
      "activations/layer16_attention_weight_min": -28.70522117614746,
      "activations/layer17_attention_weight_max": 31.042741775512695,
      "activations/layer17_attention_weight_min": -25.544954299926758,
      "activations/layer18_attention_weight_max": 29.419260025024414,
      "activations/layer18_attention_weight_min": -24.41026496887207,
      "activations/layer19_attention_weight_max": 30.786285400390625,
      "activations/layer19_attention_weight_min": -32.3330192565918,
      "activations/layer1_attention_weight_max": 16.525236129760742,
      "activations/layer1_attention_weight_min": -14.068673133850098,
      "activations/layer20_attention_weight_max": 28.34454917907715,
      "activations/layer20_attention_weight_min": -26.17552947998047,
      "activations/layer21_attention_weight_max": 28.24579429626465,
      "activations/layer21_attention_weight_min": -25.384641647338867,
      "activations/layer22_attention_weight_max": 40.71802520751953,
      "activations/layer22_attention_weight_min": -31.47564697265625,
      "activations/layer23_attention_weight_max": 30.941875457763672,
      "activations/layer23_attention_weight_min": -26.63838768005371,
      "activations/layer2_attention_weight_max": 32.482051849365234,
      "activations/layer2_attention_weight_min": -32.69339370727539,
      "activations/layer3_attention_weight_max": 96.8252182006836,
      "activations/layer3_attention_weight_min": -97.284423828125,
      "activations/layer4_attention_weight_max": 111.9059829711914,
      "activations/layer4_attention_weight_min": -114.34886932373047,
      "activations/layer5_attention_weight_max": 52.07805633544922,
      "activations/layer5_attention_weight_min": -64.47987365722656,
      "activations/layer6_attention_weight_max": 45.61872482299805,
      "activations/layer6_attention_weight_min": -48.45878219604492,
      "activations/layer7_attention_weight_max": 94.34429168701172,
      "activations/layer7_attention_weight_min": -93.54749298095703,
      "activations/layer8_attention_weight_max": 41.5518684387207,
      "activations/layer8_attention_weight_min": -42.172332763671875,
      "activations/layer9_attention_weight_max": 34.703582763671875,
      "activations/layer9_attention_weight_min": -36.40386199951172,
      "epoch": 14.45,
      "learning_rate": 5.740643939393939e-05,
      "loss": 2.7527,
      "step": 248650
    },
    {
      "activations/layer0_attention_weight_max": 16.09320640563965,
      "activations/layer0_attention_weight_min": -12.384064674377441,
      "activations/layer10_attention_weight_max": 47.19276809692383,
      "activations/layer10_attention_weight_min": -46.10047912597656,
      "activations/layer11_attention_weight_max": 49.36140060424805,
      "activations/layer11_attention_weight_min": -47.45194625854492,
      "activations/layer12_attention_weight_max": 39.7386589050293,
      "activations/layer12_attention_weight_min": -34.21830749511719,
      "activations/layer13_attention_weight_max": 51.847896575927734,
      "activations/layer13_attention_weight_min": -46.71294403076172,
      "activations/layer14_attention_weight_max": 68.25965118408203,
      "activations/layer14_attention_weight_min": -55.40009689331055,
      "activations/layer15_attention_weight_max": 57.73067092895508,
      "activations/layer15_attention_weight_min": -51.75811767578125,
      "activations/layer16_attention_weight_max": 45.6988639831543,
      "activations/layer16_attention_weight_min": -38.98320770263672,
      "activations/layer17_attention_weight_max": 54.9582405090332,
      "activations/layer17_attention_weight_min": -41.68739318847656,
      "activations/layer18_attention_weight_max": 44.79534149169922,
      "activations/layer18_attention_weight_min": -39.132137298583984,
      "activations/layer19_attention_weight_max": 44.754268646240234,
      "activations/layer19_attention_weight_min": -37.854583740234375,
      "activations/layer1_attention_weight_max": 16.829240798950195,
      "activations/layer1_attention_weight_min": -15.165173530578613,
      "activations/layer20_attention_weight_max": 36.36579132080078,
      "activations/layer20_attention_weight_min": -28.695873260498047,
      "activations/layer21_attention_weight_max": 34.93885803222656,
      "activations/layer21_attention_weight_min": -28.87375831604004,
      "activations/layer22_attention_weight_max": 59.321136474609375,
      "activations/layer22_attention_weight_min": -40.43063735961914,
      "activations/layer23_attention_weight_max": 41.54446029663086,
      "activations/layer23_attention_weight_min": -32.643089294433594,
      "activations/layer2_attention_weight_max": 35.006195068359375,
      "activations/layer2_attention_weight_min": -34.85920333862305,
      "activations/layer3_attention_weight_max": 101.26912689208984,
      "activations/layer3_attention_weight_min": -99.51565551757812,
      "activations/layer4_attention_weight_max": 116.1902084350586,
      "activations/layer4_attention_weight_min": -107.55550384521484,
      "activations/layer5_attention_weight_max": 52.75480270385742,
      "activations/layer5_attention_weight_min": -62.442726135253906,
      "activations/layer6_attention_weight_max": 48.579490661621094,
      "activations/layer6_attention_weight_min": -47.6324577331543,
      "activations/layer7_attention_weight_max": 111.10388946533203,
      "activations/layer7_attention_weight_min": -101.60823822021484,
      "activations/layer8_attention_weight_max": 51.03548049926758,
      "activations/layer8_attention_weight_min": -51.485008239746094,
      "activations/layer9_attention_weight_max": 43.35995864868164,
      "activations/layer9_attention_weight_min": -42.79256057739258,
      "epoch": 14.45,
      "learning_rate": 5.738749999999999e-05,
      "loss": 2.7437,
      "step": 248700
    },
    {
      "activations/layer0_attention_weight_max": 15.27030086517334,
      "activations/layer0_attention_weight_min": -12.34853458404541,
      "activations/layer10_attention_weight_max": 33.27613830566406,
      "activations/layer10_attention_weight_min": -34.5646858215332,
      "activations/layer11_attention_weight_max": 34.38753128051758,
      "activations/layer11_attention_weight_min": -33.272159576416016,
      "activations/layer12_attention_weight_max": 25.862829208374023,
      "activations/layer12_attention_weight_min": -25.519474029541016,
      "activations/layer13_attention_weight_max": 38.91164779663086,
      "activations/layer13_attention_weight_min": -33.96870803833008,
      "activations/layer14_attention_weight_max": 44.47722625732422,
      "activations/layer14_attention_weight_min": -37.28322982788086,
      "activations/layer15_attention_weight_max": 38.412166595458984,
      "activations/layer15_attention_weight_min": -35.56916427612305,
      "activations/layer16_attention_weight_max": 30.74139404296875,
      "activations/layer16_attention_weight_min": -28.358610153198242,
      "activations/layer17_attention_weight_max": 34.252681732177734,
      "activations/layer17_attention_weight_min": -26.17555046081543,
      "activations/layer18_attention_weight_max": 32.67672348022461,
      "activations/layer18_attention_weight_min": -24.5921573638916,
      "activations/layer19_attention_weight_max": 34.081729888916016,
      "activations/layer19_attention_weight_min": -34.98281478881836,
      "activations/layer1_attention_weight_max": 16.096649169921875,
      "activations/layer1_attention_weight_min": -14.051594734191895,
      "activations/layer20_attention_weight_max": 30.117063522338867,
      "activations/layer20_attention_weight_min": -28.385894775390625,
      "activations/layer21_attention_weight_max": 28.15968894958496,
      "activations/layer21_attention_weight_min": -26.36073875427246,
      "activations/layer22_attention_weight_max": 41.714881896972656,
      "activations/layer22_attention_weight_min": -31.20802879333496,
      "activations/layer23_attention_weight_max": 36.89820861816406,
      "activations/layer23_attention_weight_min": -25.343366622924805,
      "activations/layer2_attention_weight_max": 31.1536865234375,
      "activations/layer2_attention_weight_min": -29.596595764160156,
      "activations/layer3_attention_weight_max": 86.15950775146484,
      "activations/layer3_attention_weight_min": -87.38728332519531,
      "activations/layer4_attention_weight_max": 101.7736587524414,
      "activations/layer4_attention_weight_min": -105.36893463134766,
      "activations/layer5_attention_weight_max": 49.89404296875,
      "activations/layer5_attention_weight_min": -65.16510009765625,
      "activations/layer6_attention_weight_max": 44.07722854614258,
      "activations/layer6_attention_weight_min": -47.28207015991211,
      "activations/layer7_attention_weight_max": 90.6738510131836,
      "activations/layer7_attention_weight_min": -94.04438018798828,
      "activations/layer8_attention_weight_max": 42.147300720214844,
      "activations/layer8_attention_weight_min": -42.186988830566406,
      "activations/layer9_attention_weight_max": 32.2109260559082,
      "activations/layer9_attention_weight_min": -34.02931213378906,
      "epoch": 14.45,
      "learning_rate": 5.73685606060606e-05,
      "loss": 2.7488,
      "step": 248750
    },
    {
      "activations/layer0_attention_weight_max": 17.015525817871094,
      "activations/layer0_attention_weight_min": -11.898442268371582,
      "activations/layer10_attention_weight_max": 34.876197814941406,
      "activations/layer10_attention_weight_min": -32.95724105834961,
      "activations/layer11_attention_weight_max": 34.02183151245117,
      "activations/layer11_attention_weight_min": -31.474205017089844,
      "activations/layer12_attention_weight_max": 25.6339168548584,
      "activations/layer12_attention_weight_min": -27.85948371887207,
      "activations/layer13_attention_weight_max": 39.76155471801758,
      "activations/layer13_attention_weight_min": -36.755523681640625,
      "activations/layer14_attention_weight_max": 42.77294158935547,
      "activations/layer14_attention_weight_min": -37.73976516723633,
      "activations/layer15_attention_weight_max": 40.560890197753906,
      "activations/layer15_attention_weight_min": -34.11833953857422,
      "activations/layer16_attention_weight_max": 35.52983856201172,
      "activations/layer16_attention_weight_min": -29.167804718017578,
      "activations/layer17_attention_weight_max": 32.2404670715332,
      "activations/layer17_attention_weight_min": -26.11855697631836,
      "activations/layer18_attention_weight_max": 29.369150161743164,
      "activations/layer18_attention_weight_min": -24.160999298095703,
      "activations/layer19_attention_weight_max": 35.03399658203125,
      "activations/layer19_attention_weight_min": -34.11186981201172,
      "activations/layer1_attention_weight_max": 17.699384689331055,
      "activations/layer1_attention_weight_min": -16.284992218017578,
      "activations/layer20_attention_weight_max": 32.48365020751953,
      "activations/layer20_attention_weight_min": -26.80877685546875,
      "activations/layer21_attention_weight_max": 32.652923583984375,
      "activations/layer21_attention_weight_min": -23.56412124633789,
      "activations/layer22_attention_weight_max": 44.65696334838867,
      "activations/layer22_attention_weight_min": -31.822799682617188,
      "activations/layer23_attention_weight_max": 33.03436279296875,
      "activations/layer23_attention_weight_min": -26.57057762145996,
      "activations/layer2_attention_weight_max": 33.25727081298828,
      "activations/layer2_attention_weight_min": -32.51225280761719,
      "activations/layer3_attention_weight_max": 95.95648956298828,
      "activations/layer3_attention_weight_min": -95.00123596191406,
      "activations/layer4_attention_weight_max": 109.13130187988281,
      "activations/layer4_attention_weight_min": -107.03032684326172,
      "activations/layer5_attention_weight_max": 52.86102294921875,
      "activations/layer5_attention_weight_min": -65.61985778808594,
      "activations/layer6_attention_weight_max": 44.9734992980957,
      "activations/layer6_attention_weight_min": -46.42559051513672,
      "activations/layer7_attention_weight_max": 88.88410186767578,
      "activations/layer7_attention_weight_min": -92.02446746826172,
      "activations/layer8_attention_weight_max": 39.15424728393555,
      "activations/layer8_attention_weight_min": -39.46138381958008,
      "activations/layer9_attention_weight_max": 33.90354537963867,
      "activations/layer9_attention_weight_min": -33.730552673339844,
      "epoch": 14.46,
      "learning_rate": 5.734962121212121e-05,
      "loss": 2.771,
      "step": 248800
    },
    {
      "activations/layer0_attention_weight_max": 16.05817413330078,
      "activations/layer0_attention_weight_min": -12.38035774230957,
      "activations/layer10_attention_weight_max": 32.725311279296875,
      "activations/layer10_attention_weight_min": -33.8084716796875,
      "activations/layer11_attention_weight_max": 32.87455749511719,
      "activations/layer11_attention_weight_min": -33.82090759277344,
      "activations/layer12_attention_weight_max": 27.87615203857422,
      "activations/layer12_attention_weight_min": -27.67221450805664,
      "activations/layer13_attention_weight_max": 39.720611572265625,
      "activations/layer13_attention_weight_min": -36.387420654296875,
      "activations/layer14_attention_weight_max": 45.599853515625,
      "activations/layer14_attention_weight_min": -37.39269256591797,
      "activations/layer15_attention_weight_max": 37.20494842529297,
      "activations/layer15_attention_weight_min": -37.21410369873047,
      "activations/layer16_attention_weight_max": 29.010515213012695,
      "activations/layer16_attention_weight_min": -29.97852897644043,
      "activations/layer17_attention_weight_max": 30.587087631225586,
      "activations/layer17_attention_weight_min": -26.62421989440918,
      "activations/layer18_attention_weight_max": 31.247018814086914,
      "activations/layer18_attention_weight_min": -24.005489349365234,
      "activations/layer19_attention_weight_max": 32.797794342041016,
      "activations/layer19_attention_weight_min": -30.955856323242188,
      "activations/layer1_attention_weight_max": 15.993110656738281,
      "activations/layer1_attention_weight_min": -14.443449020385742,
      "activations/layer20_attention_weight_max": 28.6650390625,
      "activations/layer20_attention_weight_min": -26.729957580566406,
      "activations/layer21_attention_weight_max": 28.420677185058594,
      "activations/layer21_attention_weight_min": -25.867795944213867,
      "activations/layer22_attention_weight_max": 43.76059341430664,
      "activations/layer22_attention_weight_min": -32.37218475341797,
      "activations/layer23_attention_weight_max": 31.7895565032959,
      "activations/layer23_attention_weight_min": -24.699745178222656,
      "activations/layer2_attention_weight_max": 32.237857818603516,
      "activations/layer2_attention_weight_min": -32.94202423095703,
      "activations/layer3_attention_weight_max": 92.96675109863281,
      "activations/layer3_attention_weight_min": -95.29571533203125,
      "activations/layer4_attention_weight_max": 106.66023254394531,
      "activations/layer4_attention_weight_min": -111.19825744628906,
      "activations/layer5_attention_weight_max": 50.47679901123047,
      "activations/layer5_attention_weight_min": -63.538841247558594,
      "activations/layer6_attention_weight_max": 45.08919143676758,
      "activations/layer6_attention_weight_min": -49.50905990600586,
      "activations/layer7_attention_weight_max": 90.53739166259766,
      "activations/layer7_attention_weight_min": -98.99665832519531,
      "activations/layer8_attention_weight_max": 39.26552963256836,
      "activations/layer8_attention_weight_min": -44.25162887573242,
      "activations/layer9_attention_weight_max": 31.700210571289062,
      "activations/layer9_attention_weight_min": -34.285926818847656,
      "epoch": 14.46,
      "learning_rate": 5.733068181818182e-05,
      "loss": 2.7677,
      "step": 248850
    },
    {
      "activations/layer0_attention_weight_max": 16.5690860748291,
      "activations/layer0_attention_weight_min": -13.174345970153809,
      "activations/layer10_attention_weight_max": 34.12952423095703,
      "activations/layer10_attention_weight_min": -33.49954605102539,
      "activations/layer11_attention_weight_max": 34.96326446533203,
      "activations/layer11_attention_weight_min": -33.86425018310547,
      "activations/layer12_attention_weight_max": 23.951202392578125,
      "activations/layer12_attention_weight_min": -26.042415618896484,
      "activations/layer13_attention_weight_max": 33.206756591796875,
      "activations/layer13_attention_weight_min": -34.5511589050293,
      "activations/layer14_attention_weight_max": 39.241336822509766,
      "activations/layer14_attention_weight_min": -34.674407958984375,
      "activations/layer15_attention_weight_max": 34.194091796875,
      "activations/layer15_attention_weight_min": -35.4409065246582,
      "activations/layer16_attention_weight_max": 28.12394905090332,
      "activations/layer16_attention_weight_min": -26.949352264404297,
      "activations/layer17_attention_weight_max": 27.589319229125977,
      "activations/layer17_attention_weight_min": -25.51133155822754,
      "activations/layer18_attention_weight_max": 30.526765823364258,
      "activations/layer18_attention_weight_min": -24.88395881652832,
      "activations/layer19_attention_weight_max": 32.63618850708008,
      "activations/layer19_attention_weight_min": -30.304553985595703,
      "activations/layer1_attention_weight_max": 16.45026206970215,
      "activations/layer1_attention_weight_min": -13.93640422821045,
      "activations/layer20_attention_weight_max": 26.701372146606445,
      "activations/layer20_attention_weight_min": -24.301666259765625,
      "activations/layer21_attention_weight_max": 27.472509384155273,
      "activations/layer21_attention_weight_min": -24.56041717529297,
      "activations/layer22_attention_weight_max": 42.469966888427734,
      "activations/layer22_attention_weight_min": -32.70123291015625,
      "activations/layer23_attention_weight_max": 35.420860290527344,
      "activations/layer23_attention_weight_min": -26.580825805664062,
      "activations/layer2_attention_weight_max": 32.77249526977539,
      "activations/layer2_attention_weight_min": -33.349647521972656,
      "activations/layer3_attention_weight_max": 96.16971588134766,
      "activations/layer3_attention_weight_min": -98.7303237915039,
      "activations/layer4_attention_weight_max": 110.34112548828125,
      "activations/layer4_attention_weight_min": -109.3843002319336,
      "activations/layer5_attention_weight_max": 51.23014831542969,
      "activations/layer5_attention_weight_min": -59.531890869140625,
      "activations/layer6_attention_weight_max": 42.04859161376953,
      "activations/layer6_attention_weight_min": -47.81391906738281,
      "activations/layer7_attention_weight_max": 84.23926544189453,
      "activations/layer7_attention_weight_min": -89.30620574951172,
      "activations/layer8_attention_weight_max": 40.50749206542969,
      "activations/layer8_attention_weight_min": -41.859375,
      "activations/layer9_attention_weight_max": 32.9427604675293,
      "activations/layer9_attention_weight_min": -35.373687744140625,
      "epoch": 14.46,
      "learning_rate": 5.731174242424242e-05,
      "loss": 2.7564,
      "step": 248900
    },
    {
      "activations/layer0_attention_weight_max": 15.538553237915039,
      "activations/layer0_attention_weight_min": -12.404162406921387,
      "activations/layer10_attention_weight_max": 32.50531768798828,
      "activations/layer10_attention_weight_min": -32.21720886230469,
      "activations/layer11_attention_weight_max": 33.170753479003906,
      "activations/layer11_attention_weight_min": -32.524417877197266,
      "activations/layer12_attention_weight_max": 25.606172561645508,
      "activations/layer12_attention_weight_min": -25.41826820373535,
      "activations/layer13_attention_weight_max": 42.15311813354492,
      "activations/layer13_attention_weight_min": -37.05278778076172,
      "activations/layer14_attention_weight_max": 48.37420654296875,
      "activations/layer14_attention_weight_min": -38.645408630371094,
      "activations/layer15_attention_weight_max": 39.52906799316406,
      "activations/layer15_attention_weight_min": -36.7202033996582,
      "activations/layer16_attention_weight_max": 32.98257064819336,
      "activations/layer16_attention_weight_min": -28.713685989379883,
      "activations/layer17_attention_weight_max": 32.896827697753906,
      "activations/layer17_attention_weight_min": -27.66395378112793,
      "activations/layer18_attention_weight_max": 37.347347259521484,
      "activations/layer18_attention_weight_min": -26.7632999420166,
      "activations/layer19_attention_weight_max": 39.96421813964844,
      "activations/layer19_attention_weight_min": -32.3383674621582,
      "activations/layer1_attention_weight_max": 16.120452880859375,
      "activations/layer1_attention_weight_min": -14.262250900268555,
      "activations/layer20_attention_weight_max": 34.64297866821289,
      "activations/layer20_attention_weight_min": -26.257780075073242,
      "activations/layer21_attention_weight_max": 36.37297821044922,
      "activations/layer21_attention_weight_min": -25.537572860717773,
      "activations/layer22_attention_weight_max": 49.4791374206543,
      "activations/layer22_attention_weight_min": -33.500816345214844,
      "activations/layer23_attention_weight_max": 38.67994689941406,
      "activations/layer23_attention_weight_min": -29.256193161010742,
      "activations/layer2_attention_weight_max": 32.97886276245117,
      "activations/layer2_attention_weight_min": -31.314321517944336,
      "activations/layer3_attention_weight_max": 96.38656616210938,
      "activations/layer3_attention_weight_min": -98.73744201660156,
      "activations/layer4_attention_weight_max": 112.12175750732422,
      "activations/layer4_attention_weight_min": -112.40447235107422,
      "activations/layer5_attention_weight_max": 53.46322250366211,
      "activations/layer5_attention_weight_min": -62.81203842163086,
      "activations/layer6_attention_weight_max": 45.13698959350586,
      "activations/layer6_attention_weight_min": -49.30577850341797,
      "activations/layer7_attention_weight_max": 96.1784896850586,
      "activations/layer7_attention_weight_min": -96.65907287597656,
      "activations/layer8_attention_weight_max": 39.68111038208008,
      "activations/layer8_attention_weight_min": -41.239501953125,
      "activations/layer9_attention_weight_max": 31.13931655883789,
      "activations/layer9_attention_weight_min": -32.682899475097656,
      "epoch": 14.47,
      "learning_rate": 5.729280303030302e-05,
      "loss": 2.7524,
      "step": 248950
    },
    {
      "activations/layer0_attention_weight_max": 14.889238357543945,
      "activations/layer0_attention_weight_min": -11.758718490600586,
      "activations/layer10_attention_weight_max": 31.35638427734375,
      "activations/layer10_attention_weight_min": -33.775291442871094,
      "activations/layer11_attention_weight_max": 34.315162658691406,
      "activations/layer11_attention_weight_min": -34.77418518066406,
      "activations/layer12_attention_weight_max": 42.460384368896484,
      "activations/layer12_attention_weight_min": -26.22447967529297,
      "activations/layer13_attention_weight_max": 49.76264572143555,
      "activations/layer13_attention_weight_min": -36.33795166015625,
      "activations/layer14_attention_weight_max": 47.058895111083984,
      "activations/layer14_attention_weight_min": -36.705997467041016,
      "activations/layer15_attention_weight_max": 42.91517639160156,
      "activations/layer15_attention_weight_min": -35.02202606201172,
      "activations/layer16_attention_weight_max": 37.543426513671875,
      "activations/layer16_attention_weight_min": -28.481950759887695,
      "activations/layer17_attention_weight_max": 33.03946304321289,
      "activations/layer17_attention_weight_min": -25.514469146728516,
      "activations/layer18_attention_weight_max": 32.306461334228516,
      "activations/layer18_attention_weight_min": -24.19603157043457,
      "activations/layer19_attention_weight_max": 40.656368255615234,
      "activations/layer19_attention_weight_min": -30.2056884765625,
      "activations/layer1_attention_weight_max": 16.3821964263916,
      "activations/layer1_attention_weight_min": -14.290412902832031,
      "activations/layer20_attention_weight_max": 36.952964782714844,
      "activations/layer20_attention_weight_min": -23.096145629882812,
      "activations/layer21_attention_weight_max": 35.816131591796875,
      "activations/layer21_attention_weight_min": -24.2586669921875,
      "activations/layer22_attention_weight_max": 44.23460006713867,
      "activations/layer22_attention_weight_min": -29.7595272064209,
      "activations/layer23_attention_weight_max": 36.712196350097656,
      "activations/layer23_attention_weight_min": -26.8000545501709,
      "activations/layer2_attention_weight_max": 34.95903396606445,
      "activations/layer2_attention_weight_min": -33.42710494995117,
      "activations/layer3_attention_weight_max": 101.74816131591797,
      "activations/layer3_attention_weight_min": -97.0418701171875,
      "activations/layer4_attention_weight_max": 110.73291015625,
      "activations/layer4_attention_weight_min": -107.95477294921875,
      "activations/layer5_attention_weight_max": 50.21711349487305,
      "activations/layer5_attention_weight_min": -62.78076171875,
      "activations/layer6_attention_weight_max": 48.780513763427734,
      "activations/layer6_attention_weight_min": -45.27218246459961,
      "activations/layer7_attention_weight_max": 94.9367904663086,
      "activations/layer7_attention_weight_min": -89.97118377685547,
      "activations/layer8_attention_weight_max": 45.30592346191406,
      "activations/layer8_attention_weight_min": -42.28358840942383,
      "activations/layer9_attention_weight_max": 31.96213150024414,
      "activations/layer9_attention_weight_min": -33.935977935791016,
      "epoch": 14.47,
      "learning_rate": 5.7273863636363635e-05,
      "loss": 2.7549,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5147,
      "eval_samples_per_second": 504.305,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5147,
      "eval_openwebtext_samples_per_second": 504.305,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 2.0307,
      "eval_wikitext_samples_per_second": 224.549,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_lambada_loss": 2.435546875,
      "eval_lambada_ppl": 11.422063446614425,
      "eval_lambada_runtime": 9.6554,
      "eval_lambada_samples_per_second": 504.276,
      "step": 249000
    },
    {
      "activations/layer0_attention_weight_max": 15.27924633026123,
      "activations/layer0_attention_weight_min": -12.37620735168457,
      "activations/layer10_attention_weight_max": 37.496925354003906,
      "activations/layer10_attention_weight_min": -35.3200798034668,
      "activations/layer11_attention_weight_max": 35.486854553222656,
      "activations/layer11_attention_weight_min": -35.93423080444336,
      "activations/layer12_attention_weight_max": 26.220773696899414,
      "activations/layer12_attention_weight_min": -28.396299362182617,
      "activations/layer13_attention_weight_max": 41.73255157470703,
      "activations/layer13_attention_weight_min": -41.46376419067383,
      "activations/layer14_attention_weight_max": 45.52096939086914,
      "activations/layer14_attention_weight_min": -41.95903778076172,
      "activations/layer15_attention_weight_max": 37.2110710144043,
      "activations/layer15_attention_weight_min": -37.667423248291016,
      "activations/layer16_attention_weight_max": 28.77923583984375,
      "activations/layer16_attention_weight_min": -27.785175323486328,
      "activations/layer17_attention_weight_max": 30.925092697143555,
      "activations/layer17_attention_weight_min": -25.416107177734375,
      "activations/layer18_attention_weight_max": 31.856204986572266,
      "activations/layer18_attention_weight_min": -26.714155197143555,
      "activations/layer19_attention_weight_max": 33.50583267211914,
      "activations/layer19_attention_weight_min": -30.79878044128418,
      "activations/layer1_attention_weight_max": 16.219514846801758,
      "activations/layer1_attention_weight_min": -15.449002265930176,
      "activations/layer20_attention_weight_max": 28.556034088134766,
      "activations/layer20_attention_weight_min": -24.981481552124023,
      "activations/layer21_attention_weight_max": 30.284624099731445,
      "activations/layer21_attention_weight_min": -24.38487434387207,
      "activations/layer22_attention_weight_max": 40.20425796508789,
      "activations/layer22_attention_weight_min": -31.757274627685547,
      "activations/layer23_attention_weight_max": 31.028539657592773,
      "activations/layer23_attention_weight_min": -26.324918746948242,
      "activations/layer2_attention_weight_max": 32.71407699584961,
      "activations/layer2_attention_weight_min": -33.54228973388672,
      "activations/layer3_attention_weight_max": 98.28129577636719,
      "activations/layer3_attention_weight_min": -102.56366729736328,
      "activations/layer4_attention_weight_max": 109.27558898925781,
      "activations/layer4_attention_weight_min": -112.877685546875,
      "activations/layer5_attention_weight_max": 51.83590316772461,
      "activations/layer5_attention_weight_min": -63.02117156982422,
      "activations/layer6_attention_weight_max": 43.6274528503418,
      "activations/layer6_attention_weight_min": -48.97561264038086,
      "activations/layer7_attention_weight_max": 91.69355010986328,
      "activations/layer7_attention_weight_min": -92.24420166015625,
      "activations/layer8_attention_weight_max": 43.15251541137695,
      "activations/layer8_attention_weight_min": -42.951324462890625,
      "activations/layer9_attention_weight_max": 35.54690170288086,
      "activations/layer9_attention_weight_min": -36.551368713378906,
      "epoch": 14.47,
      "learning_rate": 5.7254924242424237e-05,
      "loss": 2.7742,
      "step": 249050
    },
    {
      "activations/layer0_attention_weight_max": 15.938681602478027,
      "activations/layer0_attention_weight_min": -12.133905410766602,
      "activations/layer10_attention_weight_max": 35.194374084472656,
      "activations/layer10_attention_weight_min": -32.9737548828125,
      "activations/layer11_attention_weight_max": 32.63707733154297,
      "activations/layer11_attention_weight_min": -33.81861877441406,
      "activations/layer12_attention_weight_max": 24.55228614807129,
      "activations/layer12_attention_weight_min": -27.106698989868164,
      "activations/layer13_attention_weight_max": 37.6214714050293,
      "activations/layer13_attention_weight_min": -34.6463623046875,
      "activations/layer14_attention_weight_max": 39.139583587646484,
      "activations/layer14_attention_weight_min": -36.07966613769531,
      "activations/layer15_attention_weight_max": 35.97761154174805,
      "activations/layer15_attention_weight_min": -33.26458740234375,
      "activations/layer16_attention_weight_max": 28.704328536987305,
      "activations/layer16_attention_weight_min": -27.122507095336914,
      "activations/layer17_attention_weight_max": 29.753482818603516,
      "activations/layer17_attention_weight_min": -26.738309860229492,
      "activations/layer18_attention_weight_max": 30.130525588989258,
      "activations/layer18_attention_weight_min": -26.029951095581055,
      "activations/layer19_attention_weight_max": 31.521244049072266,
      "activations/layer19_attention_weight_min": -31.740415573120117,
      "activations/layer1_attention_weight_max": 15.88172435760498,
      "activations/layer1_attention_weight_min": -15.651809692382812,
      "activations/layer20_attention_weight_max": 27.620059967041016,
      "activations/layer20_attention_weight_min": -26.68480682373047,
      "activations/layer21_attention_weight_max": 29.55499267578125,
      "activations/layer21_attention_weight_min": -25.601848602294922,
      "activations/layer22_attention_weight_max": 41.02515411376953,
      "activations/layer22_attention_weight_min": -33.28193283081055,
      "activations/layer23_attention_weight_max": 31.15229606628418,
      "activations/layer23_attention_weight_min": -26.528194427490234,
      "activations/layer2_attention_weight_max": 31.595640182495117,
      "activations/layer2_attention_weight_min": -34.19281005859375,
      "activations/layer3_attention_weight_max": 94.8346176147461,
      "activations/layer3_attention_weight_min": -101.6818618774414,
      "activations/layer4_attention_weight_max": 109.40132904052734,
      "activations/layer4_attention_weight_min": -109.99320220947266,
      "activations/layer5_attention_weight_max": 53.02983856201172,
      "activations/layer5_attention_weight_min": -63.437007904052734,
      "activations/layer6_attention_weight_max": 44.49290084838867,
      "activations/layer6_attention_weight_min": -45.98643112182617,
      "activations/layer7_attention_weight_max": 93.39994049072266,
      "activations/layer7_attention_weight_min": -88.99119567871094,
      "activations/layer8_attention_weight_max": 39.09569549560547,
      "activations/layer8_attention_weight_min": -40.87065505981445,
      "activations/layer9_attention_weight_max": 32.0233268737793,
      "activations/layer9_attention_weight_min": -33.303436279296875,
      "epoch": 14.47,
      "learning_rate": 5.7235984848484845e-05,
      "loss": 2.7567,
      "step": 249100
    },
    {
      "activations/layer0_attention_weight_max": 16.707834243774414,
      "activations/layer0_attention_weight_min": -12.988245010375977,
      "activations/layer10_attention_weight_max": 32.268043518066406,
      "activations/layer10_attention_weight_min": -33.15964126586914,
      "activations/layer11_attention_weight_max": 30.180164337158203,
      "activations/layer11_attention_weight_min": -31.858930587768555,
      "activations/layer12_attention_weight_max": 25.023963928222656,
      "activations/layer12_attention_weight_min": -29.228336334228516,
      "activations/layer13_attention_weight_max": 41.02424621582031,
      "activations/layer13_attention_weight_min": -33.594112396240234,
      "activations/layer14_attention_weight_max": 42.598594665527344,
      "activations/layer14_attention_weight_min": -36.2049560546875,
      "activations/layer15_attention_weight_max": 38.55459213256836,
      "activations/layer15_attention_weight_min": -31.7691650390625,
      "activations/layer16_attention_weight_max": 28.315074920654297,
      "activations/layer16_attention_weight_min": -27.645442962646484,
      "activations/layer17_attention_weight_max": 29.168405532836914,
      "activations/layer17_attention_weight_min": -24.83203125,
      "activations/layer18_attention_weight_max": 30.637985229492188,
      "activations/layer18_attention_weight_min": -25.22206687927246,
      "activations/layer19_attention_weight_max": 33.598602294921875,
      "activations/layer19_attention_weight_min": -31.151647567749023,
      "activations/layer1_attention_weight_max": 16.117082595825195,
      "activations/layer1_attention_weight_min": -14.52617359161377,
      "activations/layer20_attention_weight_max": 29.44074058532715,
      "activations/layer20_attention_weight_min": -25.55742645263672,
      "activations/layer21_attention_weight_max": 27.864458084106445,
      "activations/layer21_attention_weight_min": -24.315135955810547,
      "activations/layer22_attention_weight_max": 42.31781768798828,
      "activations/layer22_attention_weight_min": -33.0507926940918,
      "activations/layer23_attention_weight_max": 31.604482650756836,
      "activations/layer23_attention_weight_min": -25.390005111694336,
      "activations/layer2_attention_weight_max": 33.247615814208984,
      "activations/layer2_attention_weight_min": -33.67631912231445,
      "activations/layer3_attention_weight_max": 99.62108612060547,
      "activations/layer3_attention_weight_min": -104.93195343017578,
      "activations/layer4_attention_weight_max": 109.04668426513672,
      "activations/layer4_attention_weight_min": -106.48381805419922,
      "activations/layer5_attention_weight_max": 50.30414581298828,
      "activations/layer5_attention_weight_min": -60.245765686035156,
      "activations/layer6_attention_weight_max": 42.25103759765625,
      "activations/layer6_attention_weight_min": -44.74236297607422,
      "activations/layer7_attention_weight_max": 89.31272888183594,
      "activations/layer7_attention_weight_min": -91.43019104003906,
      "activations/layer8_attention_weight_max": 39.60634231567383,
      "activations/layer8_attention_weight_min": -41.06026077270508,
      "activations/layer9_attention_weight_max": 30.835744857788086,
      "activations/layer9_attention_weight_min": -33.34848403930664,
      "epoch": 14.48,
      "learning_rate": 5.7217045454545446e-05,
      "loss": 2.7527,
      "step": 249150
    },
    {
      "activations/layer0_attention_weight_max": 15.824934005737305,
      "activations/layer0_attention_weight_min": -12.873639106750488,
      "activations/layer10_attention_weight_max": 34.39111328125,
      "activations/layer10_attention_weight_min": -33.5869255065918,
      "activations/layer11_attention_weight_max": 33.015235900878906,
      "activations/layer11_attention_weight_min": -33.984649658203125,
      "activations/layer12_attention_weight_max": 26.75200080871582,
      "activations/layer12_attention_weight_min": -25.0364933013916,
      "activations/layer13_attention_weight_max": 38.62044906616211,
      "activations/layer13_attention_weight_min": -33.74629211425781,
      "activations/layer14_attention_weight_max": 44.318084716796875,
      "activations/layer14_attention_weight_min": -36.487945556640625,
      "activations/layer15_attention_weight_max": 36.213218688964844,
      "activations/layer15_attention_weight_min": -34.4731559753418,
      "activations/layer16_attention_weight_max": 32.07914352416992,
      "activations/layer16_attention_weight_min": -27.410818099975586,
      "activations/layer17_attention_weight_max": 32.07107925415039,
      "activations/layer17_attention_weight_min": -26.135953903198242,
      "activations/layer18_attention_weight_max": 30.23499870300293,
      "activations/layer18_attention_weight_min": -24.83171844482422,
      "activations/layer19_attention_weight_max": 34.073341369628906,
      "activations/layer19_attention_weight_min": -33.29863357543945,
      "activations/layer1_attention_weight_max": 16.42685317993164,
      "activations/layer1_attention_weight_min": -15.957444190979004,
      "activations/layer20_attention_weight_max": 31.428321838378906,
      "activations/layer20_attention_weight_min": -26.594825744628906,
      "activations/layer21_attention_weight_max": 29.1829833984375,
      "activations/layer21_attention_weight_min": -25.36319923400879,
      "activations/layer22_attention_weight_max": 44.12992477416992,
      "activations/layer22_attention_weight_min": -32.14279556274414,
      "activations/layer23_attention_weight_max": 31.96278953552246,
      "activations/layer23_attention_weight_min": -25.894046783447266,
      "activations/layer2_attention_weight_max": 32.259952545166016,
      "activations/layer2_attention_weight_min": -32.442230224609375,
      "activations/layer3_attention_weight_max": 97.89599609375,
      "activations/layer3_attention_weight_min": -101.04339599609375,
      "activations/layer4_attention_weight_max": 110.2080307006836,
      "activations/layer4_attention_weight_min": -110.96544647216797,
      "activations/layer5_attention_weight_max": 50.92878723144531,
      "activations/layer5_attention_weight_min": -61.59496307373047,
      "activations/layer6_attention_weight_max": 42.821311950683594,
      "activations/layer6_attention_weight_min": -46.44715881347656,
      "activations/layer7_attention_weight_max": 88.29302978515625,
      "activations/layer7_attention_weight_min": -89.49793243408203,
      "activations/layer8_attention_weight_max": 40.68800354003906,
      "activations/layer8_attention_weight_min": -41.6139030456543,
      "activations/layer9_attention_weight_max": 33.233253479003906,
      "activations/layer9_attention_weight_min": -34.02085876464844,
      "epoch": 14.48,
      "learning_rate": 5.719810606060606e-05,
      "loss": 2.7641,
      "step": 249200
    },
    {
      "activations/layer0_attention_weight_max": 15.984064102172852,
      "activations/layer0_attention_weight_min": -12.300745964050293,
      "activations/layer10_attention_weight_max": 33.16298294067383,
      "activations/layer10_attention_weight_min": -32.3275146484375,
      "activations/layer11_attention_weight_max": 35.71240234375,
      "activations/layer11_attention_weight_min": -33.05961608886719,
      "activations/layer12_attention_weight_max": 29.051645278930664,
      "activations/layer12_attention_weight_min": -26.29644203186035,
      "activations/layer13_attention_weight_max": 36.258140563964844,
      "activations/layer13_attention_weight_min": -35.50912094116211,
      "activations/layer14_attention_weight_max": 44.06199264526367,
      "activations/layer14_attention_weight_min": -39.56671905517578,
      "activations/layer15_attention_weight_max": 36.9974250793457,
      "activations/layer15_attention_weight_min": -34.352073669433594,
      "activations/layer16_attention_weight_max": 29.144983291625977,
      "activations/layer16_attention_weight_min": -27.744680404663086,
      "activations/layer17_attention_weight_max": 30.056243896484375,
      "activations/layer17_attention_weight_min": -25.084796905517578,
      "activations/layer18_attention_weight_max": 30.385700225830078,
      "activations/layer18_attention_weight_min": -28.265531539916992,
      "activations/layer19_attention_weight_max": 35.650978088378906,
      "activations/layer19_attention_weight_min": -29.898496627807617,
      "activations/layer1_attention_weight_max": 16.255064010620117,
      "activations/layer1_attention_weight_min": -14.281164169311523,
      "activations/layer20_attention_weight_max": 28.327117919921875,
      "activations/layer20_attention_weight_min": -24.760330200195312,
      "activations/layer21_attention_weight_max": 27.83981704711914,
      "activations/layer21_attention_weight_min": -22.938730239868164,
      "activations/layer22_attention_weight_max": 43.82401657104492,
      "activations/layer22_attention_weight_min": -29.655473709106445,
      "activations/layer23_attention_weight_max": 34.5396728515625,
      "activations/layer23_attention_weight_min": -24.718610763549805,
      "activations/layer2_attention_weight_max": 32.749271392822266,
      "activations/layer2_attention_weight_min": -32.085243225097656,
      "activations/layer3_attention_weight_max": 94.7524642944336,
      "activations/layer3_attention_weight_min": -96.02861785888672,
      "activations/layer4_attention_weight_max": 109.68153381347656,
      "activations/layer4_attention_weight_min": -109.24983978271484,
      "activations/layer5_attention_weight_max": 50.57904052734375,
      "activations/layer5_attention_weight_min": -63.88306427001953,
      "activations/layer6_attention_weight_max": 42.45646667480469,
      "activations/layer6_attention_weight_min": -47.58550262451172,
      "activations/layer7_attention_weight_max": 90.65313720703125,
      "activations/layer7_attention_weight_min": -88.76483917236328,
      "activations/layer8_attention_weight_max": 41.47031021118164,
      "activations/layer8_attention_weight_min": -41.84071731567383,
      "activations/layer9_attention_weight_max": 32.42353057861328,
      "activations/layer9_attention_weight_min": -32.99946212768555,
      "epoch": 14.48,
      "learning_rate": 5.717916666666666e-05,
      "loss": 2.7497,
      "step": 249250
    },
    {
      "activations/layer0_attention_weight_max": 16.058195114135742,
      "activations/layer0_attention_weight_min": -12.985074996948242,
      "activations/layer10_attention_weight_max": 32.346160888671875,
      "activations/layer10_attention_weight_min": -32.21480178833008,
      "activations/layer11_attention_weight_max": 31.072765350341797,
      "activations/layer11_attention_weight_min": -32.03899383544922,
      "activations/layer12_attention_weight_max": 24.398130416870117,
      "activations/layer12_attention_weight_min": -25.29652214050293,
      "activations/layer13_attention_weight_max": 39.32110595703125,
      "activations/layer13_attention_weight_min": -35.92542266845703,
      "activations/layer14_attention_weight_max": 45.130435943603516,
      "activations/layer14_attention_weight_min": -39.451499938964844,
      "activations/layer15_attention_weight_max": 39.11613845825195,
      "activations/layer15_attention_weight_min": -37.52593994140625,
      "activations/layer16_attention_weight_max": 30.87712860107422,
      "activations/layer16_attention_weight_min": -29.87881088256836,
      "activations/layer17_attention_weight_max": 30.807668685913086,
      "activations/layer17_attention_weight_min": -26.100101470947266,
      "activations/layer18_attention_weight_max": 31.2570858001709,
      "activations/layer18_attention_weight_min": -25.380760192871094,
      "activations/layer19_attention_weight_max": 33.72599792480469,
      "activations/layer19_attention_weight_min": -30.570606231689453,
      "activations/layer1_attention_weight_max": 15.6343994140625,
      "activations/layer1_attention_weight_min": -14.047444343566895,
      "activations/layer20_attention_weight_max": 27.485933303833008,
      "activations/layer20_attention_weight_min": -25.258615493774414,
      "activations/layer21_attention_weight_max": 29.782567977905273,
      "activations/layer21_attention_weight_min": -24.6082763671875,
      "activations/layer22_attention_weight_max": 43.329708099365234,
      "activations/layer22_attention_weight_min": -31.37562370300293,
      "activations/layer23_attention_weight_max": 35.16212463378906,
      "activations/layer23_attention_weight_min": -26.014944076538086,
      "activations/layer2_attention_weight_max": 34.19881057739258,
      "activations/layer2_attention_weight_min": -31.0676326751709,
      "activations/layer3_attention_weight_max": 99.21288299560547,
      "activations/layer3_attention_weight_min": -95.00570678710938,
      "activations/layer4_attention_weight_max": 112.56713104248047,
      "activations/layer4_attention_weight_min": -110.76274871826172,
      "activations/layer5_attention_weight_max": 52.10334014892578,
      "activations/layer5_attention_weight_min": -64.91824340820312,
      "activations/layer6_attention_weight_max": 42.964683532714844,
      "activations/layer6_attention_weight_min": -47.84663391113281,
      "activations/layer7_attention_weight_max": 86.89956665039062,
      "activations/layer7_attention_weight_min": -91.58478546142578,
      "activations/layer8_attention_weight_max": 39.38356018066406,
      "activations/layer8_attention_weight_min": -40.503875732421875,
      "activations/layer9_attention_weight_max": 30.387182235717773,
      "activations/layer9_attention_weight_min": -32.6411247253418,
      "epoch": 14.49,
      "learning_rate": 5.7160227272727264e-05,
      "loss": 2.7535,
      "step": 249300
    },
    {
      "activations/layer0_attention_weight_max": 15.444884300231934,
      "activations/layer0_attention_weight_min": -12.26048469543457,
      "activations/layer10_attention_weight_max": 32.79216384887695,
      "activations/layer10_attention_weight_min": -32.80481719970703,
      "activations/layer11_attention_weight_max": 31.48688316345215,
      "activations/layer11_attention_weight_min": -32.33320617675781,
      "activations/layer12_attention_weight_max": 26.90037727355957,
      "activations/layer12_attention_weight_min": -26.89838218688965,
      "activations/layer13_attention_weight_max": 39.2778205871582,
      "activations/layer13_attention_weight_min": -34.94999694824219,
      "activations/layer14_attention_weight_max": 46.79557800292969,
      "activations/layer14_attention_weight_min": -38.44663619995117,
      "activations/layer15_attention_weight_max": 40.50105285644531,
      "activations/layer15_attention_weight_min": -34.5708122253418,
      "activations/layer16_attention_weight_max": 38.67045211791992,
      "activations/layer16_attention_weight_min": -27.827741622924805,
      "activations/layer17_attention_weight_max": 36.13563919067383,
      "activations/layer17_attention_weight_min": -26.38865852355957,
      "activations/layer18_attention_weight_max": 33.97822952270508,
      "activations/layer18_attention_weight_min": -24.858760833740234,
      "activations/layer19_attention_weight_max": 41.30583572387695,
      "activations/layer19_attention_weight_min": -31.461078643798828,
      "activations/layer1_attention_weight_max": 16.684715270996094,
      "activations/layer1_attention_weight_min": -14.365422248840332,
      "activations/layer20_attention_weight_max": 32.60422897338867,
      "activations/layer20_attention_weight_min": -24.10906410217285,
      "activations/layer21_attention_weight_max": 32.238868713378906,
      "activations/layer21_attention_weight_min": -24.47641944885254,
      "activations/layer22_attention_weight_max": 48.24531936645508,
      "activations/layer22_attention_weight_min": -30.843624114990234,
      "activations/layer23_attention_weight_max": 40.87245178222656,
      "activations/layer23_attention_weight_min": -27.04306411743164,
      "activations/layer2_attention_weight_max": 32.8486213684082,
      "activations/layer2_attention_weight_min": -32.56260299682617,
      "activations/layer3_attention_weight_max": 94.8688735961914,
      "activations/layer3_attention_weight_min": -95.339599609375,
      "activations/layer4_attention_weight_max": 109.5721435546875,
      "activations/layer4_attention_weight_min": -114.67817687988281,
      "activations/layer5_attention_weight_max": 50.856788635253906,
      "activations/layer5_attention_weight_min": -61.14414978027344,
      "activations/layer6_attention_weight_max": 43.26579284667969,
      "activations/layer6_attention_weight_min": -46.82382583618164,
      "activations/layer7_attention_weight_max": 94.38378143310547,
      "activations/layer7_attention_weight_min": -92.30096435546875,
      "activations/layer8_attention_weight_max": 41.44997024536133,
      "activations/layer8_attention_weight_min": -42.697025299072266,
      "activations/layer9_attention_weight_max": 31.319076538085938,
      "activations/layer9_attention_weight_min": -32.73908615112305,
      "epoch": 14.49,
      "learning_rate": 5.7141666666666655e-05,
      "loss": 2.7748,
      "step": 249350
    },
    {
      "activations/layer0_attention_weight_max": 15.434959411621094,
      "activations/layer0_attention_weight_min": -12.501198768615723,
      "activations/layer10_attention_weight_max": 38.58665084838867,
      "activations/layer10_attention_weight_min": -33.67148971557617,
      "activations/layer11_attention_weight_max": 41.6484375,
      "activations/layer11_attention_weight_min": -36.17138671875,
      "activations/layer12_attention_weight_max": 25.178312301635742,
      "activations/layer12_attention_weight_min": -26.005552291870117,
      "activations/layer13_attention_weight_max": 36.94276428222656,
      "activations/layer13_attention_weight_min": -34.07136154174805,
      "activations/layer14_attention_weight_max": 44.22079849243164,
      "activations/layer14_attention_weight_min": -38.81929016113281,
      "activations/layer15_attention_weight_max": 36.507598876953125,
      "activations/layer15_attention_weight_min": -32.73308181762695,
      "activations/layer16_attention_weight_max": 31.335628509521484,
      "activations/layer16_attention_weight_min": -27.42149543762207,
      "activations/layer17_attention_weight_max": 32.84696960449219,
      "activations/layer17_attention_weight_min": -28.69892120361328,
      "activations/layer18_attention_weight_max": 30.8365535736084,
      "activations/layer18_attention_weight_min": -25.226070404052734,
      "activations/layer19_attention_weight_max": 34.07990646362305,
      "activations/layer19_attention_weight_min": -33.380672454833984,
      "activations/layer1_attention_weight_max": 16.40167236328125,
      "activations/layer1_attention_weight_min": -14.555747985839844,
      "activations/layer20_attention_weight_max": 32.33556365966797,
      "activations/layer20_attention_weight_min": -25.972627639770508,
      "activations/layer21_attention_weight_max": 32.94287109375,
      "activations/layer21_attention_weight_min": -25.032434463500977,
      "activations/layer22_attention_weight_max": 44.339576721191406,
      "activations/layer22_attention_weight_min": -31.640380859375,
      "activations/layer23_attention_weight_max": 36.36805725097656,
      "activations/layer23_attention_weight_min": -26.351009368896484,
      "activations/layer2_attention_weight_max": 32.864280700683594,
      "activations/layer2_attention_weight_min": -31.182477951049805,
      "activations/layer3_attention_weight_max": 94.39688110351562,
      "activations/layer3_attention_weight_min": -96.13069152832031,
      "activations/layer4_attention_weight_max": 109.22647857666016,
      "activations/layer4_attention_weight_min": -112.03543853759766,
      "activations/layer5_attention_weight_max": 51.11980438232422,
      "activations/layer5_attention_weight_min": -62.15354919433594,
      "activations/layer6_attention_weight_max": 48.66481018066406,
      "activations/layer6_attention_weight_min": -48.18605041503906,
      "activations/layer7_attention_weight_max": 98.11068725585938,
      "activations/layer7_attention_weight_min": -94.09705352783203,
      "activations/layer8_attention_weight_max": 45.64030456542969,
      "activations/layer8_attention_weight_min": -42.39168167114258,
      "activations/layer9_attention_weight_max": 35.85060119628906,
      "activations/layer9_attention_weight_min": -32.354225158691406,
      "epoch": 14.49,
      "learning_rate": 5.712272727272727e-05,
      "loss": 2.7556,
      "step": 249400
    },
    {
      "activations/layer0_attention_weight_max": 16.43848419189453,
      "activations/layer0_attention_weight_min": -12.449917793273926,
      "activations/layer10_attention_weight_max": 35.84938430786133,
      "activations/layer10_attention_weight_min": -33.83234786987305,
      "activations/layer11_attention_weight_max": 37.23403549194336,
      "activations/layer11_attention_weight_min": -37.788814544677734,
      "activations/layer12_attention_weight_max": 30.030677795410156,
      "activations/layer12_attention_weight_min": -25.931859970092773,
      "activations/layer13_attention_weight_max": 45.52268600463867,
      "activations/layer13_attention_weight_min": -35.14142990112305,
      "activations/layer14_attention_weight_max": 49.55303955078125,
      "activations/layer14_attention_weight_min": -39.5666618347168,
      "activations/layer15_attention_weight_max": 50.49701690673828,
      "activations/layer15_attention_weight_min": -37.19895553588867,
      "activations/layer16_attention_weight_max": 40.19416427612305,
      "activations/layer16_attention_weight_min": -27.464618682861328,
      "activations/layer17_attention_weight_max": 34.649879455566406,
      "activations/layer17_attention_weight_min": -28.728715896606445,
      "activations/layer18_attention_weight_max": 31.14752197265625,
      "activations/layer18_attention_weight_min": -25.43400764465332,
      "activations/layer19_attention_weight_max": 39.18160629272461,
      "activations/layer19_attention_weight_min": -30.115121841430664,
      "activations/layer1_attention_weight_max": 16.277673721313477,
      "activations/layer1_attention_weight_min": -14.593513488769531,
      "activations/layer20_attention_weight_max": 33.17145538330078,
      "activations/layer20_attention_weight_min": -25.652387619018555,
      "activations/layer21_attention_weight_max": 31.248472213745117,
      "activations/layer21_attention_weight_min": -25.496103286743164,
      "activations/layer22_attention_weight_max": 50.09341812133789,
      "activations/layer22_attention_weight_min": -32.07910919189453,
      "activations/layer23_attention_weight_max": 37.327186584472656,
      "activations/layer23_attention_weight_min": -26.561765670776367,
      "activations/layer2_attention_weight_max": 35.900794982910156,
      "activations/layer2_attention_weight_min": -32.02651596069336,
      "activations/layer3_attention_weight_max": 97.02555847167969,
      "activations/layer3_attention_weight_min": -102.3325424194336,
      "activations/layer4_attention_weight_max": 111.84020233154297,
      "activations/layer4_attention_weight_min": -108.00862121582031,
      "activations/layer5_attention_weight_max": 56.32682800292969,
      "activations/layer5_attention_weight_min": -62.79935836791992,
      "activations/layer6_attention_weight_max": 44.44044876098633,
      "activations/layer6_attention_weight_min": -44.80842590332031,
      "activations/layer7_attention_weight_max": 97.86534118652344,
      "activations/layer7_attention_weight_min": -91.35038757324219,
      "activations/layer8_attention_weight_max": 41.76886749267578,
      "activations/layer8_attention_weight_min": -43.45204162597656,
      "activations/layer9_attention_weight_max": 34.8112678527832,
      "activations/layer9_attention_weight_min": -33.75697708129883,
      "epoch": 14.49,
      "learning_rate": 5.710378787878787e-05,
      "loss": 2.7726,
      "step": 249450
    },
    {
      "activations/layer0_attention_weight_max": 15.429466247558594,
      "activations/layer0_attention_weight_min": -11.919591903686523,
      "activations/layer10_attention_weight_max": 34.030738830566406,
      "activations/layer10_attention_weight_min": -32.41931915283203,
      "activations/layer11_attention_weight_max": 36.28322219848633,
      "activations/layer11_attention_weight_min": -32.643463134765625,
      "activations/layer12_attention_weight_max": 24.89234161376953,
      "activations/layer12_attention_weight_min": -25.34984016418457,
      "activations/layer13_attention_weight_max": 35.877716064453125,
      "activations/layer13_attention_weight_min": -34.09790802001953,
      "activations/layer14_attention_weight_max": 43.42070770263672,
      "activations/layer14_attention_weight_min": -36.241249084472656,
      "activations/layer15_attention_weight_max": 36.70663833618164,
      "activations/layer15_attention_weight_min": -35.06979751586914,
      "activations/layer16_attention_weight_max": 29.637887954711914,
      "activations/layer16_attention_weight_min": -26.965614318847656,
      "activations/layer17_attention_weight_max": 33.36908721923828,
      "activations/layer17_attention_weight_min": -26.364328384399414,
      "activations/layer18_attention_weight_max": 30.16176986694336,
      "activations/layer18_attention_weight_min": -23.744155883789062,
      "activations/layer19_attention_weight_max": 36.4646110534668,
      "activations/layer19_attention_weight_min": -31.457250595092773,
      "activations/layer1_attention_weight_max": 17.267024993896484,
      "activations/layer1_attention_weight_min": -15.191495895385742,
      "activations/layer20_attention_weight_max": 29.378013610839844,
      "activations/layer20_attention_weight_min": -27.357492446899414,
      "activations/layer21_attention_weight_max": 28.642087936401367,
      "activations/layer21_attention_weight_min": -25.697221755981445,
      "activations/layer22_attention_weight_max": 39.69647216796875,
      "activations/layer22_attention_weight_min": -32.194210052490234,
      "activations/layer23_attention_weight_max": 32.830448150634766,
      "activations/layer23_attention_weight_min": -25.536794662475586,
      "activations/layer2_attention_weight_max": 33.06242370605469,
      "activations/layer2_attention_weight_min": -32.77876281738281,
      "activations/layer3_attention_weight_max": 96.06778717041016,
      "activations/layer3_attention_weight_min": -99.87898254394531,
      "activations/layer4_attention_weight_max": 109.48152160644531,
      "activations/layer4_attention_weight_min": -115.11446380615234,
      "activations/layer5_attention_weight_max": 50.840110778808594,
      "activations/layer5_attention_weight_min": -62.073768615722656,
      "activations/layer6_attention_weight_max": 44.54899597167969,
      "activations/layer6_attention_weight_min": -47.65180206298828,
      "activations/layer7_attention_weight_max": 90.28520202636719,
      "activations/layer7_attention_weight_min": -97.94793701171875,
      "activations/layer8_attention_weight_max": 38.984596252441406,
      "activations/layer8_attention_weight_min": -43.28310012817383,
      "activations/layer9_attention_weight_max": 32.60158920288086,
      "activations/layer9_attention_weight_min": -34.612369537353516,
      "epoch": 14.5,
      "learning_rate": 5.708522727272726e-05,
      "loss": 2.7543,
      "step": 249500
    },
    {
      "activations/layer0_attention_weight_max": 15.800558090209961,
      "activations/layer0_attention_weight_min": -12.567834854125977,
      "activations/layer10_attention_weight_max": 31.622648239135742,
      "activations/layer10_attention_weight_min": -32.276241302490234,
      "activations/layer11_attention_weight_max": 33.56208419799805,
      "activations/layer11_attention_weight_min": -33.370704650878906,
      "activations/layer12_attention_weight_max": 26.303861618041992,
      "activations/layer12_attention_weight_min": -26.370071411132812,
      "activations/layer13_attention_weight_max": 39.35115051269531,
      "activations/layer13_attention_weight_min": -34.968589782714844,
      "activations/layer14_attention_weight_max": 44.61820983886719,
      "activations/layer14_attention_weight_min": -39.75541687011719,
      "activations/layer15_attention_weight_max": 39.09162521362305,
      "activations/layer15_attention_weight_min": -35.98809051513672,
      "activations/layer16_attention_weight_max": 31.955785751342773,
      "activations/layer16_attention_weight_min": -27.869152069091797,
      "activations/layer17_attention_weight_max": 31.82636833190918,
      "activations/layer17_attention_weight_min": -24.920917510986328,
      "activations/layer18_attention_weight_max": 31.28413200378418,
      "activations/layer18_attention_weight_min": -24.869319915771484,
      "activations/layer19_attention_weight_max": 35.98848342895508,
      "activations/layer19_attention_weight_min": -30.713468551635742,
      "activations/layer1_attention_weight_max": 16.33417320251465,
      "activations/layer1_attention_weight_min": -14.826010704040527,
      "activations/layer20_attention_weight_max": 28.77389144897461,
      "activations/layer20_attention_weight_min": -23.938692092895508,
      "activations/layer21_attention_weight_max": 31.926403045654297,
      "activations/layer21_attention_weight_min": -23.0970401763916,
      "activations/layer22_attention_weight_max": 41.852928161621094,
      "activations/layer22_attention_weight_min": -29.749223709106445,
      "activations/layer23_attention_weight_max": 32.61034393310547,
      "activations/layer23_attention_weight_min": -26.514774322509766,
      "activations/layer2_attention_weight_max": 32.109825134277344,
      "activations/layer2_attention_weight_min": -32.088645935058594,
      "activations/layer3_attention_weight_max": 95.74868774414062,
      "activations/layer3_attention_weight_min": -96.11422729492188,
      "activations/layer4_attention_weight_max": 112.63323974609375,
      "activations/layer4_attention_weight_min": -112.62577056884766,
      "activations/layer5_attention_weight_max": 56.186920166015625,
      "activations/layer5_attention_weight_min": -61.861602783203125,
      "activations/layer6_attention_weight_max": 40.771785736083984,
      "activations/layer6_attention_weight_min": -46.08304977416992,
      "activations/layer7_attention_weight_max": 89.43246459960938,
      "activations/layer7_attention_weight_min": -88.00442504882812,
      "activations/layer8_attention_weight_max": 39.74668884277344,
      "activations/layer8_attention_weight_min": -41.76525115966797,
      "activations/layer9_attention_weight_max": 31.05862808227539,
      "activations/layer9_attention_weight_min": -32.709041595458984,
      "epoch": 14.5,
      "learning_rate": 5.706628787878787e-05,
      "loss": 2.7425,
      "step": 249550
    },
    {
      "activations/layer0_attention_weight_max": 15.45952320098877,
      "activations/layer0_attention_weight_min": -12.729348182678223,
      "activations/layer10_attention_weight_max": 33.90290069580078,
      "activations/layer10_attention_weight_min": -35.142616271972656,
      "activations/layer11_attention_weight_max": 34.627586364746094,
      "activations/layer11_attention_weight_min": -33.095680236816406,
      "activations/layer12_attention_weight_max": 39.49049758911133,
      "activations/layer12_attention_weight_min": -27.343605041503906,
      "activations/layer13_attention_weight_max": 47.217220306396484,
      "activations/layer13_attention_weight_min": -36.6804313659668,
      "activations/layer14_attention_weight_max": 54.708984375,
      "activations/layer14_attention_weight_min": -39.87604904174805,
      "activations/layer15_attention_weight_max": 44.44258117675781,
      "activations/layer15_attention_weight_min": -36.004146575927734,
      "activations/layer16_attention_weight_max": 34.37651062011719,
      "activations/layer16_attention_weight_min": -28.306550979614258,
      "activations/layer17_attention_weight_max": 36.95868682861328,
      "activations/layer17_attention_weight_min": -27.119096755981445,
      "activations/layer18_attention_weight_max": 33.99974822998047,
      "activations/layer18_attention_weight_min": -25.34736442565918,
      "activations/layer19_attention_weight_max": 38.507843017578125,
      "activations/layer19_attention_weight_min": -32.44891357421875,
      "activations/layer1_attention_weight_max": 16.990610122680664,
      "activations/layer1_attention_weight_min": -14.345890045166016,
      "activations/layer20_attention_weight_max": 32.56150817871094,
      "activations/layer20_attention_weight_min": -25.51384925842285,
      "activations/layer21_attention_weight_max": 30.621337890625,
      "activations/layer21_attention_weight_min": -25.18684959411621,
      "activations/layer22_attention_weight_max": 50.324241638183594,
      "activations/layer22_attention_weight_min": -31.310773849487305,
      "activations/layer23_attention_weight_max": 36.220298767089844,
      "activations/layer23_attention_weight_min": -25.76717185974121,
      "activations/layer2_attention_weight_max": 32.254486083984375,
      "activations/layer2_attention_weight_min": -33.0556526184082,
      "activations/layer3_attention_weight_max": 98.72810363769531,
      "activations/layer3_attention_weight_min": -101.0541763305664,
      "activations/layer4_attention_weight_max": 118.55644989013672,
      "activations/layer4_attention_weight_min": -115.358642578125,
      "activations/layer5_attention_weight_max": 50.270301818847656,
      "activations/layer5_attention_weight_min": -63.41709518432617,
      "activations/layer6_attention_weight_max": 44.01459503173828,
      "activations/layer6_attention_weight_min": -48.3961296081543,
      "activations/layer7_attention_weight_max": 93.59915161132812,
      "activations/layer7_attention_weight_min": -97.61141967773438,
      "activations/layer8_attention_weight_max": 42.990509033203125,
      "activations/layer8_attention_weight_min": -43.810909271240234,
      "activations/layer9_attention_weight_max": 32.84807205200195,
      "activations/layer9_attention_weight_min": -34.96330642700195,
      "epoch": 14.5,
      "learning_rate": 5.704734848484848e-05,
      "loss": 2.7577,
      "step": 249600
    },
    {
      "activations/layer0_attention_weight_max": 15.706841468811035,
      "activations/layer0_attention_weight_min": -13.056628227233887,
      "activations/layer10_attention_weight_max": 33.5572395324707,
      "activations/layer10_attention_weight_min": -35.66743850708008,
      "activations/layer11_attention_weight_max": 32.46680450439453,
      "activations/layer11_attention_weight_min": -34.68244171142578,
      "activations/layer12_attention_weight_max": 24.806747436523438,
      "activations/layer12_attention_weight_min": -25.061771392822266,
      "activations/layer13_attention_weight_max": 38.00887680053711,
      "activations/layer13_attention_weight_min": -36.85882568359375,
      "activations/layer14_attention_weight_max": 42.64126205444336,
      "activations/layer14_attention_weight_min": -38.22804641723633,
      "activations/layer15_attention_weight_max": 36.259735107421875,
      "activations/layer15_attention_weight_min": -34.68724822998047,
      "activations/layer16_attention_weight_max": 29.358266830444336,
      "activations/layer16_attention_weight_min": -28.154312133789062,
      "activations/layer17_attention_weight_max": 30.109697341918945,
      "activations/layer17_attention_weight_min": -25.708946228027344,
      "activations/layer18_attention_weight_max": 32.2666130065918,
      "activations/layer18_attention_weight_min": -22.963090896606445,
      "activations/layer19_attention_weight_max": 32.29336929321289,
      "activations/layer19_attention_weight_min": -32.60308074951172,
      "activations/layer1_attention_weight_max": 16.085412979125977,
      "activations/layer1_attention_weight_min": -14.664053916931152,
      "activations/layer20_attention_weight_max": 27.409332275390625,
      "activations/layer20_attention_weight_min": -25.424137115478516,
      "activations/layer21_attention_weight_max": 26.539451599121094,
      "activations/layer21_attention_weight_min": -24.963340759277344,
      "activations/layer22_attention_weight_max": 41.47551345825195,
      "activations/layer22_attention_weight_min": -28.96483039855957,
      "activations/layer23_attention_weight_max": 29.627464294433594,
      "activations/layer23_attention_weight_min": -25.89944076538086,
      "activations/layer2_attention_weight_max": 34.37241744995117,
      "activations/layer2_attention_weight_min": -36.643314361572266,
      "activations/layer3_attention_weight_max": 106.02027893066406,
      "activations/layer3_attention_weight_min": -107.34439086914062,
      "activations/layer4_attention_weight_max": 121.272216796875,
      "activations/layer4_attention_weight_min": -120.45861053466797,
      "activations/layer5_attention_weight_max": 52.40727996826172,
      "activations/layer5_attention_weight_min": -65.8074722290039,
      "activations/layer6_attention_weight_max": 46.02140426635742,
      "activations/layer6_attention_weight_min": -48.58755111694336,
      "activations/layer7_attention_weight_max": 89.05345916748047,
      "activations/layer7_attention_weight_min": -97.97098541259766,
      "activations/layer8_attention_weight_max": 41.58898162841797,
      "activations/layer8_attention_weight_min": -45.790504455566406,
      "activations/layer9_attention_weight_max": 34.97566223144531,
      "activations/layer9_attention_weight_min": -35.4109001159668,
      "epoch": 14.51,
      "learning_rate": 5.702840909090909e-05,
      "loss": 2.7812,
      "step": 249650
    },
    {
      "activations/layer0_attention_weight_max": 15.430513381958008,
      "activations/layer0_attention_weight_min": -12.263891220092773,
      "activations/layer10_attention_weight_max": 31.61592674255371,
      "activations/layer10_attention_weight_min": -32.344932556152344,
      "activations/layer11_attention_weight_max": 31.366239547729492,
      "activations/layer11_attention_weight_min": -32.42939376831055,
      "activations/layer12_attention_weight_max": 26.747697830200195,
      "activations/layer12_attention_weight_min": -26.431867599487305,
      "activations/layer13_attention_weight_max": 38.4423942565918,
      "activations/layer13_attention_weight_min": -36.79168701171875,
      "activations/layer14_attention_weight_max": 41.02534866333008,
      "activations/layer14_attention_weight_min": -40.98458480834961,
      "activations/layer15_attention_weight_max": 36.48424530029297,
      "activations/layer15_attention_weight_min": -33.380496978759766,
      "activations/layer16_attention_weight_max": 31.310827255249023,
      "activations/layer16_attention_weight_min": -27.47862434387207,
      "activations/layer17_attention_weight_max": 30.3692569732666,
      "activations/layer17_attention_weight_min": -26.32819175720215,
      "activations/layer18_attention_weight_max": 28.2369384765625,
      "activations/layer18_attention_weight_min": -23.762968063354492,
      "activations/layer19_attention_weight_max": 34.922237396240234,
      "activations/layer19_attention_weight_min": -30.74920654296875,
      "activations/layer1_attention_weight_max": 16.55068016052246,
      "activations/layer1_attention_weight_min": -13.675509452819824,
      "activations/layer20_attention_weight_max": 28.75209617614746,
      "activations/layer20_attention_weight_min": -24.771196365356445,
      "activations/layer21_attention_weight_max": 27.11791229248047,
      "activations/layer21_attention_weight_min": -24.092981338500977,
      "activations/layer22_attention_weight_max": 40.31016159057617,
      "activations/layer22_attention_weight_min": -29.416175842285156,
      "activations/layer23_attention_weight_max": 38.634193420410156,
      "activations/layer23_attention_weight_min": -23.017467498779297,
      "activations/layer2_attention_weight_max": 31.787025451660156,
      "activations/layer2_attention_weight_min": -31.85007095336914,
      "activations/layer3_attention_weight_max": 99.45538330078125,
      "activations/layer3_attention_weight_min": -101.79228210449219,
      "activations/layer4_attention_weight_max": 110.57122039794922,
      "activations/layer4_attention_weight_min": -110.40196990966797,
      "activations/layer5_attention_weight_max": 54.485435485839844,
      "activations/layer5_attention_weight_min": -58.847084045410156,
      "activations/layer6_attention_weight_max": 43.281612396240234,
      "activations/layer6_attention_weight_min": -45.53474044799805,
      "activations/layer7_attention_weight_max": 87.2831802368164,
      "activations/layer7_attention_weight_min": -89.37554931640625,
      "activations/layer8_attention_weight_max": 38.42121887207031,
      "activations/layer8_attention_weight_min": -41.26002883911133,
      "activations/layer9_attention_weight_max": 30.522329330444336,
      "activations/layer9_attention_weight_min": -32.47225570678711,
      "epoch": 14.51,
      "learning_rate": 5.700946969696969e-05,
      "loss": 2.7485,
      "step": 249700
    },
    {
      "activations/layer0_attention_weight_max": 15.329390525817871,
      "activations/layer0_attention_weight_min": -12.768363952636719,
      "activations/layer10_attention_weight_max": 36.278526306152344,
      "activations/layer10_attention_weight_min": -34.322998046875,
      "activations/layer11_attention_weight_max": 36.37137222290039,
      "activations/layer11_attention_weight_min": -32.92357635498047,
      "activations/layer12_attention_weight_max": 25.802335739135742,
      "activations/layer12_attention_weight_min": -25.560993194580078,
      "activations/layer13_attention_weight_max": 37.65974044799805,
      "activations/layer13_attention_weight_min": -34.37027359008789,
      "activations/layer14_attention_weight_max": 44.32313919067383,
      "activations/layer14_attention_weight_min": -36.47858428955078,
      "activations/layer15_attention_weight_max": 37.60310745239258,
      "activations/layer15_attention_weight_min": -33.60835647583008,
      "activations/layer16_attention_weight_max": 31.039993286132812,
      "activations/layer16_attention_weight_min": -27.09017562866211,
      "activations/layer17_attention_weight_max": 29.80278205871582,
      "activations/layer17_attention_weight_min": -25.167123794555664,
      "activations/layer18_attention_weight_max": 31.65118408203125,
      "activations/layer18_attention_weight_min": -24.320812225341797,
      "activations/layer19_attention_weight_max": 33.127567291259766,
      "activations/layer19_attention_weight_min": -31.6534366607666,
      "activations/layer1_attention_weight_max": 16.404857635498047,
      "activations/layer1_attention_weight_min": -14.556774139404297,
      "activations/layer20_attention_weight_max": 29.07050132751465,
      "activations/layer20_attention_weight_min": -24.357702255249023,
      "activations/layer21_attention_weight_max": 29.887619018554688,
      "activations/layer21_attention_weight_min": -23.656394958496094,
      "activations/layer22_attention_weight_max": 41.4470100402832,
      "activations/layer22_attention_weight_min": -30.666109085083008,
      "activations/layer23_attention_weight_max": 32.37614440917969,
      "activations/layer23_attention_weight_min": -25.236881256103516,
      "activations/layer2_attention_weight_max": 32.5999641418457,
      "activations/layer2_attention_weight_min": -33.569549560546875,
      "activations/layer3_attention_weight_max": 97.7959976196289,
      "activations/layer3_attention_weight_min": -99.30239868164062,
      "activations/layer4_attention_weight_max": 114.12938690185547,
      "activations/layer4_attention_weight_min": -114.2384262084961,
      "activations/layer5_attention_weight_max": 51.201377868652344,
      "activations/layer5_attention_weight_min": -62.08073425292969,
      "activations/layer6_attention_weight_max": 48.276912689208984,
      "activations/layer6_attention_weight_min": -49.48001480102539,
      "activations/layer7_attention_weight_max": 93.8035888671875,
      "activations/layer7_attention_weight_min": -99.4580307006836,
      "activations/layer8_attention_weight_max": 44.23575973510742,
      "activations/layer8_attention_weight_min": -45.162227630615234,
      "activations/layer9_attention_weight_max": 34.78334426879883,
      "activations/layer9_attention_weight_min": -36.69483947753906,
      "epoch": 14.51,
      "learning_rate": 5.6990530303030304e-05,
      "loss": 2.7568,
      "step": 249750
    },
    {
      "activations/layer0_attention_weight_max": 15.622222900390625,
      "activations/layer0_attention_weight_min": -12.813019752502441,
      "activations/layer10_attention_weight_max": 35.56391143798828,
      "activations/layer10_attention_weight_min": -35.7411994934082,
      "activations/layer11_attention_weight_max": 35.29664993286133,
      "activations/layer11_attention_weight_min": -34.9456901550293,
      "activations/layer12_attention_weight_max": 27.027236938476562,
      "activations/layer12_attention_weight_min": -26.188703536987305,
      "activations/layer13_attention_weight_max": 40.64510726928711,
      "activations/layer13_attention_weight_min": -37.09999465942383,
      "activations/layer14_attention_weight_max": 43.77721405029297,
      "activations/layer14_attention_weight_min": -38.990264892578125,
      "activations/layer15_attention_weight_max": 46.51840591430664,
      "activations/layer15_attention_weight_min": -34.25679016113281,
      "activations/layer16_attention_weight_max": 32.06304168701172,
      "activations/layer16_attention_weight_min": -27.548973083496094,
      "activations/layer17_attention_weight_max": 29.709909439086914,
      "activations/layer17_attention_weight_min": -25.236862182617188,
      "activations/layer18_attention_weight_max": 31.283117294311523,
      "activations/layer18_attention_weight_min": -23.736461639404297,
      "activations/layer19_attention_weight_max": 39.767459869384766,
      "activations/layer19_attention_weight_min": -32.35664749145508,
      "activations/layer1_attention_weight_max": 16.05374526977539,
      "activations/layer1_attention_weight_min": -15.13646411895752,
      "activations/layer20_attention_weight_max": 33.022403717041016,
      "activations/layer20_attention_weight_min": -24.046310424804688,
      "activations/layer21_attention_weight_max": 31.576101303100586,
      "activations/layer21_attention_weight_min": -22.895021438598633,
      "activations/layer22_attention_weight_max": 50.208438873291016,
      "activations/layer22_attention_weight_min": -28.71883773803711,
      "activations/layer23_attention_weight_max": 38.03443908691406,
      "activations/layer23_attention_weight_min": -24.246841430664062,
      "activations/layer2_attention_weight_max": 34.19496536254883,
      "activations/layer2_attention_weight_min": -32.579036712646484,
      "activations/layer3_attention_weight_max": 99.59341430664062,
      "activations/layer3_attention_weight_min": -101.31773376464844,
      "activations/layer4_attention_weight_max": 114.52144622802734,
      "activations/layer4_attention_weight_min": -116.66310119628906,
      "activations/layer5_attention_weight_max": 53.725074768066406,
      "activations/layer5_attention_weight_min": -64.50445556640625,
      "activations/layer6_attention_weight_max": 46.12245559692383,
      "activations/layer6_attention_weight_min": -49.295372009277344,
      "activations/layer7_attention_weight_max": 94.30058288574219,
      "activations/layer7_attention_weight_min": -97.43253326416016,
      "activations/layer8_attention_weight_max": 45.04109573364258,
      "activations/layer8_attention_weight_min": -42.910972595214844,
      "activations/layer9_attention_weight_max": 33.80210494995117,
      "activations/layer9_attention_weight_min": -33.95320129394531,
      "epoch": 14.51,
      "learning_rate": 5.6971590909090905e-05,
      "loss": 2.7621,
      "step": 249800
    },
    {
      "activations/layer0_attention_weight_max": 15.73607349395752,
      "activations/layer0_attention_weight_min": -12.037585258483887,
      "activations/layer10_attention_weight_max": 34.5267448425293,
      "activations/layer10_attention_weight_min": -34.13251495361328,
      "activations/layer11_attention_weight_max": 33.08308410644531,
      "activations/layer11_attention_weight_min": -33.56255340576172,
      "activations/layer12_attention_weight_max": 27.930265426635742,
      "activations/layer12_attention_weight_min": -26.338516235351562,
      "activations/layer13_attention_weight_max": 43.441837310791016,
      "activations/layer13_attention_weight_min": -35.60921859741211,
      "activations/layer14_attention_weight_max": 45.537994384765625,
      "activations/layer14_attention_weight_min": -39.966129302978516,
      "activations/layer15_attention_weight_max": 40.34731674194336,
      "activations/layer15_attention_weight_min": -33.70689392089844,
      "activations/layer16_attention_weight_max": 30.820430755615234,
      "activations/layer16_attention_weight_min": -27.589797973632812,
      "activations/layer17_attention_weight_max": 33.048545837402344,
      "activations/layer17_attention_weight_min": -25.727724075317383,
      "activations/layer18_attention_weight_max": 33.40992736816406,
      "activations/layer18_attention_weight_min": -24.72559928894043,
      "activations/layer19_attention_weight_max": 36.35517501831055,
      "activations/layer19_attention_weight_min": -30.400510787963867,
      "activations/layer1_attention_weight_max": 17.056915283203125,
      "activations/layer1_attention_weight_min": -15.407044410705566,
      "activations/layer20_attention_weight_max": 31.21208381652832,
      "activations/layer20_attention_weight_min": -25.426475524902344,
      "activations/layer21_attention_weight_max": 29.627025604248047,
      "activations/layer21_attention_weight_min": -24.80900001525879,
      "activations/layer22_attention_weight_max": 43.370670318603516,
      "activations/layer22_attention_weight_min": -30.92531967163086,
      "activations/layer23_attention_weight_max": 33.67366409301758,
      "activations/layer23_attention_weight_min": -26.33381462097168,
      "activations/layer2_attention_weight_max": 35.86384963989258,
      "activations/layer2_attention_weight_min": -34.55860137939453,
      "activations/layer3_attention_weight_max": 99.09967041015625,
      "activations/layer3_attention_weight_min": -100.62835693359375,
      "activations/layer4_attention_weight_max": 112.53917694091797,
      "activations/layer4_attention_weight_min": -111.47834777832031,
      "activations/layer5_attention_weight_max": 55.20245361328125,
      "activations/layer5_attention_weight_min": -64.67626953125,
      "activations/layer6_attention_weight_max": 45.188133239746094,
      "activations/layer6_attention_weight_min": -45.401573181152344,
      "activations/layer7_attention_weight_max": 94.44406127929688,
      "activations/layer7_attention_weight_min": -91.31410217285156,
      "activations/layer8_attention_weight_max": 41.093624114990234,
      "activations/layer8_attention_weight_min": -43.8311653137207,
      "activations/layer9_attention_weight_max": 32.94759750366211,
      "activations/layer9_attention_weight_min": -33.73994064331055,
      "epoch": 14.52,
      "learning_rate": 5.695265151515151e-05,
      "loss": 2.7637,
      "step": 249850
    },
    {
      "activations/layer0_attention_weight_max": 16.290422439575195,
      "activations/layer0_attention_weight_min": -12.63140869140625,
      "activations/layer10_attention_weight_max": 36.1605339050293,
      "activations/layer10_attention_weight_min": -32.07273483276367,
      "activations/layer11_attention_weight_max": 35.94561767578125,
      "activations/layer11_attention_weight_min": -33.124717712402344,
      "activations/layer12_attention_weight_max": 27.067384719848633,
      "activations/layer12_attention_weight_min": -25.09112548828125,
      "activations/layer13_attention_weight_max": 39.05377197265625,
      "activations/layer13_attention_weight_min": -34.055362701416016,
      "activations/layer14_attention_weight_max": 41.875762939453125,
      "activations/layer14_attention_weight_min": -38.2733268737793,
      "activations/layer15_attention_weight_max": 39.469322204589844,
      "activations/layer15_attention_weight_min": -35.88018798828125,
      "activations/layer16_attention_weight_max": 29.87584686279297,
      "activations/layer16_attention_weight_min": -27.208877563476562,
      "activations/layer17_attention_weight_max": 29.749170303344727,
      "activations/layer17_attention_weight_min": -26.854705810546875,
      "activations/layer18_attention_weight_max": 31.747623443603516,
      "activations/layer18_attention_weight_min": -25.117733001708984,
      "activations/layer19_attention_weight_max": 31.63557243347168,
      "activations/layer19_attention_weight_min": -30.56308364868164,
      "activations/layer1_attention_weight_max": 18.99371337890625,
      "activations/layer1_attention_weight_min": -16.601909637451172,
      "activations/layer20_attention_weight_max": 26.38475799560547,
      "activations/layer20_attention_weight_min": -26.484519958496094,
      "activations/layer21_attention_weight_max": 26.159658432006836,
      "activations/layer21_attention_weight_min": -24.23705291748047,
      "activations/layer22_attention_weight_max": 37.87299346923828,
      "activations/layer22_attention_weight_min": -30.7967586517334,
      "activations/layer23_attention_weight_max": 32.71480178833008,
      "activations/layer23_attention_weight_min": -23.83910369873047,
      "activations/layer2_attention_weight_max": 32.75676345825195,
      "activations/layer2_attention_weight_min": -33.08287811279297,
      "activations/layer3_attention_weight_max": 94.87081909179688,
      "activations/layer3_attention_weight_min": -94.68717956542969,
      "activations/layer4_attention_weight_max": 107.16957092285156,
      "activations/layer4_attention_weight_min": -109.06807708740234,
      "activations/layer5_attention_weight_max": 53.983821868896484,
      "activations/layer5_attention_weight_min": -59.122169494628906,
      "activations/layer6_attention_weight_max": 42.97426986694336,
      "activations/layer6_attention_weight_min": -45.64673614501953,
      "activations/layer7_attention_weight_max": 86.021240234375,
      "activations/layer7_attention_weight_min": -88.9234390258789,
      "activations/layer8_attention_weight_max": 41.162322998046875,
      "activations/layer8_attention_weight_min": -40.87083053588867,
      "activations/layer9_attention_weight_max": 33.5988655090332,
      "activations/layer9_attention_weight_min": -33.5789909362793,
      "epoch": 14.52,
      "learning_rate": 5.6933712121212115e-05,
      "loss": 2.7413,
      "step": 249900
    },
    {
      "activations/layer0_attention_weight_max": 15.257596969604492,
      "activations/layer0_attention_weight_min": -13.047078132629395,
      "activations/layer10_attention_weight_max": 33.642311096191406,
      "activations/layer10_attention_weight_min": -34.468597412109375,
      "activations/layer11_attention_weight_max": 32.211639404296875,
      "activations/layer11_attention_weight_min": -34.367530822753906,
      "activations/layer12_attention_weight_max": 28.37073516845703,
      "activations/layer12_attention_weight_min": -28.026002883911133,
      "activations/layer13_attention_weight_max": 42.67372512817383,
      "activations/layer13_attention_weight_min": -37.037445068359375,
      "activations/layer14_attention_weight_max": 43.65396499633789,
      "activations/layer14_attention_weight_min": -40.532630920410156,
      "activations/layer15_attention_weight_max": 38.114383697509766,
      "activations/layer15_attention_weight_min": -34.003082275390625,
      "activations/layer16_attention_weight_max": 31.292699813842773,
      "activations/layer16_attention_weight_min": -28.65237045288086,
      "activations/layer17_attention_weight_max": 31.419641494750977,
      "activations/layer17_attention_weight_min": -26.695749282836914,
      "activations/layer18_attention_weight_max": 32.2559928894043,
      "activations/layer18_attention_weight_min": -25.32550621032715,
      "activations/layer19_attention_weight_max": 34.39840316772461,
      "activations/layer19_attention_weight_min": -31.423654556274414,
      "activations/layer1_attention_weight_max": 15.966157913208008,
      "activations/layer1_attention_weight_min": -14.429498672485352,
      "activations/layer20_attention_weight_max": 29.124343872070312,
      "activations/layer20_attention_weight_min": -28.025041580200195,
      "activations/layer21_attention_weight_max": 28.337982177734375,
      "activations/layer21_attention_weight_min": -25.0306396484375,
      "activations/layer22_attention_weight_max": 44.188926696777344,
      "activations/layer22_attention_weight_min": -31.540369033813477,
      "activations/layer23_attention_weight_max": 34.044189453125,
      "activations/layer23_attention_weight_min": -25.877418518066406,
      "activations/layer2_attention_weight_max": 32.29792785644531,
      "activations/layer2_attention_weight_min": -32.670555114746094,
      "activations/layer3_attention_weight_max": 98.73932647705078,
      "activations/layer3_attention_weight_min": -102.87506866455078,
      "activations/layer4_attention_weight_max": 115.92906951904297,
      "activations/layer4_attention_weight_min": -116.38878631591797,
      "activations/layer5_attention_weight_max": 52.070518493652344,
      "activations/layer5_attention_weight_min": -65.93050384521484,
      "activations/layer6_attention_weight_max": 47.33686828613281,
      "activations/layer6_attention_weight_min": -50.47712707519531,
      "activations/layer7_attention_weight_max": 96.7765884399414,
      "activations/layer7_attention_weight_min": -97.71741485595703,
      "activations/layer8_attention_weight_max": 43.28404235839844,
      "activations/layer8_attention_weight_min": -43.02092742919922,
      "activations/layer9_attention_weight_max": 33.66749572753906,
      "activations/layer9_attention_weight_min": -34.74370574951172,
      "epoch": 14.52,
      "learning_rate": 5.6914772727272723e-05,
      "loss": 2.7492,
      "step": 249950
    },
    {
      "activations/layer0_attention_weight_max": 15.645976066589355,
      "activations/layer0_attention_weight_min": -12.046028137207031,
      "activations/layer10_attention_weight_max": 31.74527931213379,
      "activations/layer10_attention_weight_min": -32.410037994384766,
      "activations/layer11_attention_weight_max": 33.327640533447266,
      "activations/layer11_attention_weight_min": -32.759864807128906,
      "activations/layer12_attention_weight_max": 25.049161911010742,
      "activations/layer12_attention_weight_min": -24.293371200561523,
      "activations/layer13_attention_weight_max": 38.7376594543457,
      "activations/layer13_attention_weight_min": -33.824275970458984,
      "activations/layer14_attention_weight_max": 46.11085891723633,
      "activations/layer14_attention_weight_min": -39.32284164428711,
      "activations/layer15_attention_weight_max": 41.31187438964844,
      "activations/layer15_attention_weight_min": -34.50657272338867,
      "activations/layer16_attention_weight_max": 31.854597091674805,
      "activations/layer16_attention_weight_min": -27.34534454345703,
      "activations/layer17_attention_weight_max": 31.706438064575195,
      "activations/layer17_attention_weight_min": -27.135080337524414,
      "activations/layer18_attention_weight_max": 33.3215217590332,
      "activations/layer18_attention_weight_min": -25.335147857666016,
      "activations/layer19_attention_weight_max": 36.07562255859375,
      "activations/layer19_attention_weight_min": -31.729236602783203,
      "activations/layer1_attention_weight_max": 15.60473918914795,
      "activations/layer1_attention_weight_min": -14.446831703186035,
      "activations/layer20_attention_weight_max": 30.46047592163086,
      "activations/layer20_attention_weight_min": -25.46025276184082,
      "activations/layer21_attention_weight_max": 31.49590301513672,
      "activations/layer21_attention_weight_min": -25.958316802978516,
      "activations/layer22_attention_weight_max": 46.44236373901367,
      "activations/layer22_attention_weight_min": -31.197595596313477,
      "activations/layer23_attention_weight_max": 33.102108001708984,
      "activations/layer23_attention_weight_min": -25.95332145690918,
      "activations/layer2_attention_weight_max": 30.764163970947266,
      "activations/layer2_attention_weight_min": -32.1442985534668,
      "activations/layer3_attention_weight_max": 96.03141021728516,
      "activations/layer3_attention_weight_min": -99.96719360351562,
      "activations/layer4_attention_weight_max": 110.48392486572266,
      "activations/layer4_attention_weight_min": -110.87740325927734,
      "activations/layer5_attention_weight_max": 49.23607635498047,
      "activations/layer5_attention_weight_min": -64.55192565917969,
      "activations/layer6_attention_weight_max": 43.1877555847168,
      "activations/layer6_attention_weight_min": -45.316627502441406,
      "activations/layer7_attention_weight_max": 89.7291488647461,
      "activations/layer7_attention_weight_min": -90.9948501586914,
      "activations/layer8_attention_weight_max": 39.866355895996094,
      "activations/layer8_attention_weight_min": -39.74443817138672,
      "activations/layer9_attention_weight_max": 30.6673583984375,
      "activations/layer9_attention_weight_min": -32.50516128540039,
      "epoch": 14.53,
      "learning_rate": 5.689583333333333e-05,
      "loss": 2.7589,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5059,
      "eval_samples_per_second": 504.827,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5059,
      "eval_openwebtext_samples_per_second": 504.827,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 2.054,
      "eval_wikitext_samples_per_second": 222.007,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_lambada_loss": 2.46875,
      "eval_lambada_ppl": 11.807678025521156,
      "eval_lambada_runtime": 9.6047,
      "eval_lambada_samples_per_second": 506.94,
      "step": 250000
    },
    {
      "activations/layer0_attention_weight_max": 15.515214920043945,
      "activations/layer0_attention_weight_min": -12.766947746276855,
      "activations/layer10_attention_weight_max": 33.4146614074707,
      "activations/layer10_attention_weight_min": -33.243587493896484,
      "activations/layer11_attention_weight_max": 34.79704284667969,
      "activations/layer11_attention_weight_min": -33.60834503173828,
      "activations/layer12_attention_weight_max": 25.282358169555664,
      "activations/layer12_attention_weight_min": -24.229764938354492,
      "activations/layer13_attention_weight_max": 37.26991653442383,
      "activations/layer13_attention_weight_min": -34.93656921386719,
      "activations/layer14_attention_weight_max": 40.682552337646484,
      "activations/layer14_attention_weight_min": -37.66444396972656,
      "activations/layer15_attention_weight_max": 36.51077651977539,
      "activations/layer15_attention_weight_min": -32.82643127441406,
      "activations/layer16_attention_weight_max": 29.399187088012695,
      "activations/layer16_attention_weight_min": -27.53062629699707,
      "activations/layer17_attention_weight_max": 30.5830020904541,
      "activations/layer17_attention_weight_min": -25.251928329467773,
      "activations/layer18_attention_weight_max": 29.209753036499023,
      "activations/layer18_attention_weight_min": -23.07526969909668,
      "activations/layer19_attention_weight_max": 30.952985763549805,
      "activations/layer19_attention_weight_min": -30.312280654907227,
      "activations/layer1_attention_weight_max": 16.413856506347656,
      "activations/layer1_attention_weight_min": -14.960665702819824,
      "activations/layer20_attention_weight_max": 27.497190475463867,
      "activations/layer20_attention_weight_min": -23.323057174682617,
      "activations/layer21_attention_weight_max": 25.8983097076416,
      "activations/layer21_attention_weight_min": -24.388566970825195,
      "activations/layer22_attention_weight_max": 38.435054779052734,
      "activations/layer22_attention_weight_min": -27.52526092529297,
      "activations/layer23_attention_weight_max": 30.679771423339844,
      "activations/layer23_attention_weight_min": -24.27815055847168,
      "activations/layer2_attention_weight_max": 34.78896713256836,
      "activations/layer2_attention_weight_min": -34.11882781982422,
      "activations/layer3_attention_weight_max": 99.68603515625,
      "activations/layer3_attention_weight_min": -102.13290405273438,
      "activations/layer4_attention_weight_max": 107.44271087646484,
      "activations/layer4_attention_weight_min": -111.98005676269531,
      "activations/layer5_attention_weight_max": 49.212554931640625,
      "activations/layer5_attention_weight_min": -62.475990295410156,
      "activations/layer6_attention_weight_max": 45.633907318115234,
      "activations/layer6_attention_weight_min": -46.46892166137695,
      "activations/layer7_attention_weight_max": 90.1974105834961,
      "activations/layer7_attention_weight_min": -95.14942169189453,
      "activations/layer8_attention_weight_max": 41.16143035888672,
      "activations/layer8_attention_weight_min": -43.63735580444336,
      "activations/layer9_attention_weight_max": 33.04172897338867,
      "activations/layer9_attention_weight_min": -34.13802719116211,
      "epoch": 14.53,
      "learning_rate": 5.687689393939393e-05,
      "loss": 2.7652,
      "step": 250050
    },
    {
      "activations/layer0_attention_weight_max": 15.585766792297363,
      "activations/layer0_attention_weight_min": -12.392271041870117,
      "activations/layer10_attention_weight_max": 32.94865417480469,
      "activations/layer10_attention_weight_min": -34.75724792480469,
      "activations/layer11_attention_weight_max": 30.900291442871094,
      "activations/layer11_attention_weight_min": -33.133880615234375,
      "activations/layer12_attention_weight_max": 31.164752960205078,
      "activations/layer12_attention_weight_min": -25.950645446777344,
      "activations/layer13_attention_weight_max": 41.7603645324707,
      "activations/layer13_attention_weight_min": -35.02505111694336,
      "activations/layer14_attention_weight_max": 42.98843002319336,
      "activations/layer14_attention_weight_min": -37.216915130615234,
      "activations/layer15_attention_weight_max": 40.46324157714844,
      "activations/layer15_attention_weight_min": -35.66310501098633,
      "activations/layer16_attention_weight_max": 30.983596801757812,
      "activations/layer16_attention_weight_min": -28.03765296936035,
      "activations/layer17_attention_weight_max": 30.07581901550293,
      "activations/layer17_attention_weight_min": -25.299205780029297,
      "activations/layer18_attention_weight_max": 30.20811653137207,
      "activations/layer18_attention_weight_min": -26.96734619140625,
      "activations/layer19_attention_weight_max": 36.55732727050781,
      "activations/layer19_attention_weight_min": -32.5424919128418,
      "activations/layer1_attention_weight_max": 16.35528564453125,
      "activations/layer1_attention_weight_min": -13.944493293762207,
      "activations/layer20_attention_weight_max": 31.769712448120117,
      "activations/layer20_attention_weight_min": -26.179370880126953,
      "activations/layer21_attention_weight_max": 26.85051155090332,
      "activations/layer21_attention_weight_min": -25.098989486694336,
      "activations/layer22_attention_weight_max": 40.55784225463867,
      "activations/layer22_attention_weight_min": -31.485309600830078,
      "activations/layer23_attention_weight_max": 36.305686950683594,
      "activations/layer23_attention_weight_min": -26.033370971679688,
      "activations/layer2_attention_weight_max": 31.66131591796875,
      "activations/layer2_attention_weight_min": -31.95034408569336,
      "activations/layer3_attention_weight_max": 99.87649536132812,
      "activations/layer3_attention_weight_min": -98.00861358642578,
      "activations/layer4_attention_weight_max": 111.53389739990234,
      "activations/layer4_attention_weight_min": -113.9831771850586,
      "activations/layer5_attention_weight_max": 53.79682922363281,
      "activations/layer5_attention_weight_min": -69.68049621582031,
      "activations/layer6_attention_weight_max": 43.99601745605469,
      "activations/layer6_attention_weight_min": -46.90660095214844,
      "activations/layer7_attention_weight_max": 92.03764343261719,
      "activations/layer7_attention_weight_min": -95.13711547851562,
      "activations/layer8_attention_weight_max": 40.66532897949219,
      "activations/layer8_attention_weight_min": -41.9890022277832,
      "activations/layer9_attention_weight_max": 30.17915153503418,
      "activations/layer9_attention_weight_min": -32.59583282470703,
      "epoch": 14.53,
      "learning_rate": 5.6857954545454535e-05,
      "loss": 2.7578,
      "step": 250100
    },
    {
      "activations/layer0_attention_weight_max": 15.683980941772461,
      "activations/layer0_attention_weight_min": -12.714428901672363,
      "activations/layer10_attention_weight_max": 34.90150833129883,
      "activations/layer10_attention_weight_min": -34.726646423339844,
      "activations/layer11_attention_weight_max": 35.74402618408203,
      "activations/layer11_attention_weight_min": -33.79450988769531,
      "activations/layer12_attention_weight_max": 27.55748176574707,
      "activations/layer12_attention_weight_min": -26.97955894470215,
      "activations/layer13_attention_weight_max": 43.6016960144043,
      "activations/layer13_attention_weight_min": -37.38042449951172,
      "activations/layer14_attention_weight_max": 44.96983337402344,
      "activations/layer14_attention_weight_min": -38.3287467956543,
      "activations/layer15_attention_weight_max": 38.79275894165039,
      "activations/layer15_attention_weight_min": -34.61621856689453,
      "activations/layer16_attention_weight_max": 30.818262100219727,
      "activations/layer16_attention_weight_min": -27.51992416381836,
      "activations/layer17_attention_weight_max": 29.2392520904541,
      "activations/layer17_attention_weight_min": -25.614561080932617,
      "activations/layer18_attention_weight_max": 30.57080078125,
      "activations/layer18_attention_weight_min": -22.377540588378906,
      "activations/layer19_attention_weight_max": 35.944129943847656,
      "activations/layer19_attention_weight_min": -31.48069190979004,
      "activations/layer1_attention_weight_max": 16.141176223754883,
      "activations/layer1_attention_weight_min": -14.308107376098633,
      "activations/layer20_attention_weight_max": 31.95378875732422,
      "activations/layer20_attention_weight_min": -26.300857543945312,
      "activations/layer21_attention_weight_max": 29.501060485839844,
      "activations/layer21_attention_weight_min": -24.623899459838867,
      "activations/layer22_attention_weight_max": 41.294029235839844,
      "activations/layer22_attention_weight_min": -30.66474723815918,
      "activations/layer23_attention_weight_max": 35.198482513427734,
      "activations/layer23_attention_weight_min": -24.987150192260742,
      "activations/layer2_attention_weight_max": 33.74229431152344,
      "activations/layer2_attention_weight_min": -33.435977935791016,
      "activations/layer3_attention_weight_max": 99.99221801757812,
      "activations/layer3_attention_weight_min": -101.2578125,
      "activations/layer4_attention_weight_max": 115.30875396728516,
      "activations/layer4_attention_weight_min": -115.4891128540039,
      "activations/layer5_attention_weight_max": 55.74437713623047,
      "activations/layer5_attention_weight_min": -63.23967742919922,
      "activations/layer6_attention_weight_max": 47.07160949707031,
      "activations/layer6_attention_weight_min": -49.551780700683594,
      "activations/layer7_attention_weight_max": 98.24334716796875,
      "activations/layer7_attention_weight_min": -97.79187774658203,
      "activations/layer8_attention_weight_max": 43.885772705078125,
      "activations/layer8_attention_weight_min": -43.507389068603516,
      "activations/layer9_attention_weight_max": 35.572166442871094,
      "activations/layer9_attention_weight_min": -34.42497634887695,
      "epoch": 14.54,
      "learning_rate": 5.683901515151515e-05,
      "loss": 2.7456,
      "step": 250150
    },
    {
      "activations/layer0_attention_weight_max": 15.920194625854492,
      "activations/layer0_attention_weight_min": -12.240394592285156,
      "activations/layer10_attention_weight_max": 35.71881103515625,
      "activations/layer10_attention_weight_min": -35.502620697021484,
      "activations/layer11_attention_weight_max": 35.552406311035156,
      "activations/layer11_attention_weight_min": -35.1156005859375,
      "activations/layer12_attention_weight_max": 28.30701446533203,
      "activations/layer12_attention_weight_min": -26.94439125061035,
      "activations/layer13_attention_weight_max": 43.05699920654297,
      "activations/layer13_attention_weight_min": -35.41355514526367,
      "activations/layer14_attention_weight_max": 47.8968505859375,
      "activations/layer14_attention_weight_min": -41.37627029418945,
      "activations/layer15_attention_weight_max": 44.02920150756836,
      "activations/layer15_attention_weight_min": -34.8494873046875,
      "activations/layer16_attention_weight_max": 33.94892120361328,
      "activations/layer16_attention_weight_min": -28.30303955078125,
      "activations/layer17_attention_weight_max": 32.24222946166992,
      "activations/layer17_attention_weight_min": -27.73143196105957,
      "activations/layer18_attention_weight_max": 33.42394256591797,
      "activations/layer18_attention_weight_min": -26.098888397216797,
      "activations/layer19_attention_weight_max": 35.98436737060547,
      "activations/layer19_attention_weight_min": -32.45090866088867,
      "activations/layer1_attention_weight_max": 16.282039642333984,
      "activations/layer1_attention_weight_min": -14.82567024230957,
      "activations/layer20_attention_weight_max": 30.898330688476562,
      "activations/layer20_attention_weight_min": -27.578372955322266,
      "activations/layer21_attention_weight_max": 29.778587341308594,
      "activations/layer21_attention_weight_min": -27.317880630493164,
      "activations/layer22_attention_weight_max": 47.122772216796875,
      "activations/layer22_attention_weight_min": -32.9243049621582,
      "activations/layer23_attention_weight_max": 36.49638366699219,
      "activations/layer23_attention_weight_min": -26.604949951171875,
      "activations/layer2_attention_weight_max": 34.496212005615234,
      "activations/layer2_attention_weight_min": -33.97294235229492,
      "activations/layer3_attention_weight_max": 101.53836822509766,
      "activations/layer3_attention_weight_min": -103.24262237548828,
      "activations/layer4_attention_weight_max": 115.84065246582031,
      "activations/layer4_attention_weight_min": -112.6946029663086,
      "activations/layer5_attention_weight_max": 56.073028564453125,
      "activations/layer5_attention_weight_min": -63.12801742553711,
      "activations/layer6_attention_weight_max": 46.3487663269043,
      "activations/layer6_attention_weight_min": -49.156429290771484,
      "activations/layer7_attention_weight_max": 94.58941650390625,
      "activations/layer7_attention_weight_min": -94.05690002441406,
      "activations/layer8_attention_weight_max": 47.485286712646484,
      "activations/layer8_attention_weight_min": -46.748924255371094,
      "activations/layer9_attention_weight_max": 36.11970901489258,
      "activations/layer9_attention_weight_min": -35.38862228393555,
      "epoch": 14.54,
      "learning_rate": 5.682007575757575e-05,
      "loss": 2.7573,
      "step": 250200
    },
    {
      "activations/layer0_attention_weight_max": 15.366937637329102,
      "activations/layer0_attention_weight_min": -12.375347137451172,
      "activations/layer10_attention_weight_max": 34.690452575683594,
      "activations/layer10_attention_weight_min": -34.49471664428711,
      "activations/layer11_attention_weight_max": 35.95567321777344,
      "activations/layer11_attention_weight_min": -34.577545166015625,
      "activations/layer12_attention_weight_max": 27.72641372680664,
      "activations/layer12_attention_weight_min": -27.9866886138916,
      "activations/layer13_attention_weight_max": 41.10064697265625,
      "activations/layer13_attention_weight_min": -36.50689697265625,
      "activations/layer14_attention_weight_max": 43.77236557006836,
      "activations/layer14_attention_weight_min": -41.41670608520508,
      "activations/layer15_attention_weight_max": 42.98638153076172,
      "activations/layer15_attention_weight_min": -36.24338912963867,
      "activations/layer16_attention_weight_max": 30.836816787719727,
      "activations/layer16_attention_weight_min": -28.02025032043457,
      "activations/layer17_attention_weight_max": 32.91390609741211,
      "activations/layer17_attention_weight_min": -26.530935287475586,
      "activations/layer18_attention_weight_max": 34.314971923828125,
      "activations/layer18_attention_weight_min": -24.339805603027344,
      "activations/layer19_attention_weight_max": 39.46917724609375,
      "activations/layer19_attention_weight_min": -33.26325988769531,
      "activations/layer1_attention_weight_max": 16.516300201416016,
      "activations/layer1_attention_weight_min": -15.455967903137207,
      "activations/layer20_attention_weight_max": 34.0827522277832,
      "activations/layer20_attention_weight_min": -24.184568405151367,
      "activations/layer21_attention_weight_max": 33.56350326538086,
      "activations/layer21_attention_weight_min": -25.174991607666016,
      "activations/layer22_attention_weight_max": 48.98837661743164,
      "activations/layer22_attention_weight_min": -29.9720516204834,
      "activations/layer23_attention_weight_max": 38.32197570800781,
      "activations/layer23_attention_weight_min": -25.19582176208496,
      "activations/layer2_attention_weight_max": 34.13623046875,
      "activations/layer2_attention_weight_min": -33.55309295654297,
      "activations/layer3_attention_weight_max": 100.04984283447266,
      "activations/layer3_attention_weight_min": -100.42866516113281,
      "activations/layer4_attention_weight_max": 114.26128387451172,
      "activations/layer4_attention_weight_min": -120.07360076904297,
      "activations/layer5_attention_weight_max": 56.43292999267578,
      "activations/layer5_attention_weight_min": -63.355377197265625,
      "activations/layer6_attention_weight_max": 44.85337448120117,
      "activations/layer6_attention_weight_min": -50.981971740722656,
      "activations/layer7_attention_weight_max": 91.75787353515625,
      "activations/layer7_attention_weight_min": -99.9127426147461,
      "activations/layer8_attention_weight_max": 42.77702713012695,
      "activations/layer8_attention_weight_min": -42.17206573486328,
      "activations/layer9_attention_weight_max": 32.785762786865234,
      "activations/layer9_attention_weight_min": -35.48596954345703,
      "epoch": 14.54,
      "learning_rate": 5.680113636363636e-05,
      "loss": 2.7468,
      "step": 250250
    },
    {
      "activations/layer0_attention_weight_max": 15.69049072265625,
      "activations/layer0_attention_weight_min": -12.838802337646484,
      "activations/layer10_attention_weight_max": 31.715917587280273,
      "activations/layer10_attention_weight_min": -33.103633880615234,
      "activations/layer11_attention_weight_max": 32.1345100402832,
      "activations/layer11_attention_weight_min": -34.00892639160156,
      "activations/layer12_attention_weight_max": 25.259183883666992,
      "activations/layer12_attention_weight_min": -26.042621612548828,
      "activations/layer13_attention_weight_max": 37.42707061767578,
      "activations/layer13_attention_weight_min": -34.49158477783203,
      "activations/layer14_attention_weight_max": 40.0009651184082,
      "activations/layer14_attention_weight_min": -38.21341323852539,
      "activations/layer15_attention_weight_max": 34.56035614013672,
      "activations/layer15_attention_weight_min": -33.347042083740234,
      "activations/layer16_attention_weight_max": 29.446733474731445,
      "activations/layer16_attention_weight_min": -27.488750457763672,
      "activations/layer17_attention_weight_max": 28.841691970825195,
      "activations/layer17_attention_weight_min": -26.624330520629883,
      "activations/layer18_attention_weight_max": 29.82801628112793,
      "activations/layer18_attention_weight_min": -22.484378814697266,
      "activations/layer19_attention_weight_max": 32.65702819824219,
      "activations/layer19_attention_weight_min": -31.225446701049805,
      "activations/layer1_attention_weight_max": 16.539527893066406,
      "activations/layer1_attention_weight_min": -13.983499526977539,
      "activations/layer20_attention_weight_max": 28.34237289428711,
      "activations/layer20_attention_weight_min": -26.015666961669922,
      "activations/layer21_attention_weight_max": 26.63060760498047,
      "activations/layer21_attention_weight_min": -24.092350006103516,
      "activations/layer22_attention_weight_max": 39.43275451660156,
      "activations/layer22_attention_weight_min": -29.626922607421875,
      "activations/layer23_attention_weight_max": 32.49668884277344,
      "activations/layer23_attention_weight_min": -23.705524444580078,
      "activations/layer2_attention_weight_max": 32.94767379760742,
      "activations/layer2_attention_weight_min": -33.0311164855957,
      "activations/layer3_attention_weight_max": 96.00444793701172,
      "activations/layer3_attention_weight_min": -96.81674194335938,
      "activations/layer4_attention_weight_max": 109.99635314941406,
      "activations/layer4_attention_weight_min": -115.85603332519531,
      "activations/layer5_attention_weight_max": 51.128662109375,
      "activations/layer5_attention_weight_min": -65.52637481689453,
      "activations/layer6_attention_weight_max": 45.05508804321289,
      "activations/layer6_attention_weight_min": -48.066490173339844,
      "activations/layer7_attention_weight_max": 92.3954086303711,
      "activations/layer7_attention_weight_min": -91.31520080566406,
      "activations/layer8_attention_weight_max": 38.88037109375,
      "activations/layer8_attention_weight_min": -41.99637222290039,
      "activations/layer9_attention_weight_max": 31.388324737548828,
      "activations/layer9_attention_weight_min": -33.201263427734375,
      "epoch": 14.54,
      "learning_rate": 5.678219696969696e-05,
      "loss": 2.7721,
      "step": 250300
    },
    {
      "activations/layer0_attention_weight_max": 15.42451286315918,
      "activations/layer0_attention_weight_min": -12.701412200927734,
      "activations/layer10_attention_weight_max": 32.91791534423828,
      "activations/layer10_attention_weight_min": -33.220035552978516,
      "activations/layer11_attention_weight_max": 34.05274200439453,
      "activations/layer11_attention_weight_min": -32.423492431640625,
      "activations/layer12_attention_weight_max": 24.3449764251709,
      "activations/layer12_attention_weight_min": -30.63392448425293,
      "activations/layer13_attention_weight_max": 38.38795852661133,
      "activations/layer13_attention_weight_min": -34.07240676879883,
      "activations/layer14_attention_weight_max": 40.956024169921875,
      "activations/layer14_attention_weight_min": -36.34724807739258,
      "activations/layer15_attention_weight_max": 34.29512405395508,
      "activations/layer15_attention_weight_min": -31.829511642456055,
      "activations/layer16_attention_weight_max": 32.123443603515625,
      "activations/layer16_attention_weight_min": -27.41278076171875,
      "activations/layer17_attention_weight_max": 30.171661376953125,
      "activations/layer17_attention_weight_min": -25.607236862182617,
      "activations/layer18_attention_weight_max": 29.478515625,
      "activations/layer18_attention_weight_min": -24.707120895385742,
      "activations/layer19_attention_weight_max": 34.061275482177734,
      "activations/layer19_attention_weight_min": -32.14929962158203,
      "activations/layer1_attention_weight_max": 16.516895294189453,
      "activations/layer1_attention_weight_min": -15.792647361755371,
      "activations/layer20_attention_weight_max": 31.654882431030273,
      "activations/layer20_attention_weight_min": -25.017192840576172,
      "activations/layer21_attention_weight_max": 30.110225677490234,
      "activations/layer21_attention_weight_min": -23.683225631713867,
      "activations/layer22_attention_weight_max": 40.592220306396484,
      "activations/layer22_attention_weight_min": -28.95768928527832,
      "activations/layer23_attention_weight_max": 32.92002868652344,
      "activations/layer23_attention_weight_min": -23.581153869628906,
      "activations/layer2_attention_weight_max": 34.87510681152344,
      "activations/layer2_attention_weight_min": -34.046653747558594,
      "activations/layer3_attention_weight_max": 102.10567474365234,
      "activations/layer3_attention_weight_min": -105.37598419189453,
      "activations/layer4_attention_weight_max": 112.11055755615234,
      "activations/layer4_attention_weight_min": -116.125244140625,
      "activations/layer5_attention_weight_max": 52.24900436401367,
      "activations/layer5_attention_weight_min": -62.588436126708984,
      "activations/layer6_attention_weight_max": 43.96907424926758,
      "activations/layer6_attention_weight_min": -46.161373138427734,
      "activations/layer7_attention_weight_max": 88.78882598876953,
      "activations/layer7_attention_weight_min": -86.11961364746094,
      "activations/layer8_attention_weight_max": 38.19664001464844,
      "activations/layer8_attention_weight_min": -39.14580154418945,
      "activations/layer9_attention_weight_max": 32.042510986328125,
      "activations/layer9_attention_weight_min": -32.4166374206543,
      "epoch": 14.55,
      "learning_rate": 5.6763257575757576e-05,
      "loss": 2.7548,
      "step": 250350
    },
    {
      "activations/layer0_attention_weight_max": 15.807883262634277,
      "activations/layer0_attention_weight_min": -12.173133850097656,
      "activations/layer10_attention_weight_max": 33.93819808959961,
      "activations/layer10_attention_weight_min": -36.31922149658203,
      "activations/layer11_attention_weight_max": 33.6280517578125,
      "activations/layer11_attention_weight_min": -36.76466751098633,
      "activations/layer12_attention_weight_max": 29.839580535888672,
      "activations/layer12_attention_weight_min": -27.578086853027344,
      "activations/layer13_attention_weight_max": 38.72184371948242,
      "activations/layer13_attention_weight_min": -37.22550582885742,
      "activations/layer14_attention_weight_max": 44.053504943847656,
      "activations/layer14_attention_weight_min": -40.406341552734375,
      "activations/layer15_attention_weight_max": 37.1680793762207,
      "activations/layer15_attention_weight_min": -34.36450958251953,
      "activations/layer16_attention_weight_max": 31.443574905395508,
      "activations/layer16_attention_weight_min": -29.590673446655273,
      "activations/layer17_attention_weight_max": 29.768463134765625,
      "activations/layer17_attention_weight_min": -26.57731056213379,
      "activations/layer18_attention_weight_max": 31.340139389038086,
      "activations/layer18_attention_weight_min": -24.94444465637207,
      "activations/layer19_attention_weight_max": 35.28262710571289,
      "activations/layer19_attention_weight_min": -33.807857513427734,
      "activations/layer1_attention_weight_max": 17.43906593322754,
      "activations/layer1_attention_weight_min": -16.33214569091797,
      "activations/layer20_attention_weight_max": 28.324617385864258,
      "activations/layer20_attention_weight_min": -25.009923934936523,
      "activations/layer21_attention_weight_max": 29.480117797851562,
      "activations/layer21_attention_weight_min": -25.507816314697266,
      "activations/layer22_attention_weight_max": 51.41242980957031,
      "activations/layer22_attention_weight_min": -31.11954689025879,
      "activations/layer23_attention_weight_max": 35.21693420410156,
      "activations/layer23_attention_weight_min": -25.072856903076172,
      "activations/layer2_attention_weight_max": 32.43010711669922,
      "activations/layer2_attention_weight_min": -31.114944458007812,
      "activations/layer3_attention_weight_max": 96.55030059814453,
      "activations/layer3_attention_weight_min": -97.35383605957031,
      "activations/layer4_attention_weight_max": 106.37386322021484,
      "activations/layer4_attention_weight_min": -115.8563232421875,
      "activations/layer5_attention_weight_max": 53.773006439208984,
      "activations/layer5_attention_weight_min": -64.17810821533203,
      "activations/layer6_attention_weight_max": 44.826576232910156,
      "activations/layer6_attention_weight_min": -48.444114685058594,
      "activations/layer7_attention_weight_max": 90.74119567871094,
      "activations/layer7_attention_weight_min": -101.58635711669922,
      "activations/layer8_attention_weight_max": 40.561798095703125,
      "activations/layer8_attention_weight_min": -44.6370964050293,
      "activations/layer9_attention_weight_max": 34.35653305053711,
      "activations/layer9_attention_weight_min": -35.41717529296875,
      "epoch": 14.55,
      "learning_rate": 5.674431818181818e-05,
      "loss": 2.7711,
      "step": 250400
    },
    {
      "activations/layer0_attention_weight_max": 16.12492561340332,
      "activations/layer0_attention_weight_min": -12.783876419067383,
      "activations/layer10_attention_weight_max": 33.31390380859375,
      "activations/layer10_attention_weight_min": -33.81852722167969,
      "activations/layer11_attention_weight_max": 34.3304557800293,
      "activations/layer11_attention_weight_min": -33.56779479980469,
      "activations/layer12_attention_weight_max": 28.19361114501953,
      "activations/layer12_attention_weight_min": -25.61079216003418,
      "activations/layer13_attention_weight_max": 43.20659255981445,
      "activations/layer13_attention_weight_min": -33.61993408203125,
      "activations/layer14_attention_weight_max": 47.59916687011719,
      "activations/layer14_attention_weight_min": -36.53742218017578,
      "activations/layer15_attention_weight_max": 44.635223388671875,
      "activations/layer15_attention_weight_min": -35.914794921875,
      "activations/layer16_attention_weight_max": 35.36880874633789,
      "activations/layer16_attention_weight_min": -29.34981918334961,
      "activations/layer17_attention_weight_max": 32.97815704345703,
      "activations/layer17_attention_weight_min": -24.920310974121094,
      "activations/layer18_attention_weight_max": 34.969417572021484,
      "activations/layer18_attention_weight_min": -23.898393630981445,
      "activations/layer19_attention_weight_max": 38.827327728271484,
      "activations/layer19_attention_weight_min": -31.103118896484375,
      "activations/layer1_attention_weight_max": 15.53972053527832,
      "activations/layer1_attention_weight_min": -14.994022369384766,
      "activations/layer20_attention_weight_max": 34.987327575683594,
      "activations/layer20_attention_weight_min": -27.444541931152344,
      "activations/layer21_attention_weight_max": 34.59138870239258,
      "activations/layer21_attention_weight_min": -25.886465072631836,
      "activations/layer22_attention_weight_max": 45.183467864990234,
      "activations/layer22_attention_weight_min": -30.06928062438965,
      "activations/layer23_attention_weight_max": 36.34054183959961,
      "activations/layer23_attention_weight_min": -23.966514587402344,
      "activations/layer2_attention_weight_max": 31.639873504638672,
      "activations/layer2_attention_weight_min": -30.54743194580078,
      "activations/layer3_attention_weight_max": 96.40949249267578,
      "activations/layer3_attention_weight_min": -95.19092559814453,
      "activations/layer4_attention_weight_max": 110.01102447509766,
      "activations/layer4_attention_weight_min": -112.05995178222656,
      "activations/layer5_attention_weight_max": 52.09247589111328,
      "activations/layer5_attention_weight_min": -60.020530700683594,
      "activations/layer6_attention_weight_max": 43.64900588989258,
      "activations/layer6_attention_weight_min": -45.09865951538086,
      "activations/layer7_attention_weight_max": 92.55753326416016,
      "activations/layer7_attention_weight_min": -88.76567840576172,
      "activations/layer8_attention_weight_max": 39.69598388671875,
      "activations/layer8_attention_weight_min": -40.97038650512695,
      "activations/layer9_attention_weight_max": 32.672088623046875,
      "activations/layer9_attention_weight_min": -32.53870391845703,
      "epoch": 14.55,
      "learning_rate": 5.672537878787878e-05,
      "loss": 2.7457,
      "step": 250450
    },
    {
      "activations/layer0_attention_weight_max": 15.879765510559082,
      "activations/layer0_attention_weight_min": -12.827692985534668,
      "activations/layer10_attention_weight_max": 35.873268127441406,
      "activations/layer10_attention_weight_min": -32.92147445678711,
      "activations/layer11_attention_weight_max": 33.6185302734375,
      "activations/layer11_attention_weight_min": -34.1292724609375,
      "activations/layer12_attention_weight_max": 24.252761840820312,
      "activations/layer12_attention_weight_min": -25.882347106933594,
      "activations/layer13_attention_weight_max": 38.74534225463867,
      "activations/layer13_attention_weight_min": -33.21674346923828,
      "activations/layer14_attention_weight_max": 43.38312530517578,
      "activations/layer14_attention_weight_min": -35.91156005859375,
      "activations/layer15_attention_weight_max": 41.737876892089844,
      "activations/layer15_attention_weight_min": -33.285823822021484,
      "activations/layer16_attention_weight_max": 29.181194305419922,
      "activations/layer16_attention_weight_min": -27.839336395263672,
      "activations/layer17_attention_weight_max": 31.80852699279785,
      "activations/layer17_attention_weight_min": -25.29489517211914,
      "activations/layer18_attention_weight_max": 31.312936782836914,
      "activations/layer18_attention_weight_min": -25.233827590942383,
      "activations/layer19_attention_weight_max": 32.499813079833984,
      "activations/layer19_attention_weight_min": -29.8319034576416,
      "activations/layer1_attention_weight_max": 16.882352828979492,
      "activations/layer1_attention_weight_min": -14.308537483215332,
      "activations/layer20_attention_weight_max": 31.8607234954834,
      "activations/layer20_attention_weight_min": -24.96782112121582,
      "activations/layer21_attention_weight_max": 27.617734909057617,
      "activations/layer21_attention_weight_min": -24.841598510742188,
      "activations/layer22_attention_weight_max": 42.179386138916016,
      "activations/layer22_attention_weight_min": -30.509119033813477,
      "activations/layer23_attention_weight_max": 31.910579681396484,
      "activations/layer23_attention_weight_min": -26.781003952026367,
      "activations/layer2_attention_weight_max": 33.548187255859375,
      "activations/layer2_attention_weight_min": -33.5715446472168,
      "activations/layer3_attention_weight_max": 94.59019470214844,
      "activations/layer3_attention_weight_min": -97.52481842041016,
      "activations/layer4_attention_weight_max": 112.09526824951172,
      "activations/layer4_attention_weight_min": -114.89833068847656,
      "activations/layer5_attention_weight_max": 52.91019821166992,
      "activations/layer5_attention_weight_min": -62.73986053466797,
      "activations/layer6_attention_weight_max": 43.68175506591797,
      "activations/layer6_attention_weight_min": -50.17463684082031,
      "activations/layer7_attention_weight_max": 88.53018951416016,
      "activations/layer7_attention_weight_min": -90.38920593261719,
      "activations/layer8_attention_weight_max": 42.783565521240234,
      "activations/layer8_attention_weight_min": -42.84941482543945,
      "activations/layer9_attention_weight_max": 34.35203170776367,
      "activations/layer9_attention_weight_min": -33.36457443237305,
      "epoch": 14.56,
      "learning_rate": 5.6706439393939394e-05,
      "loss": 2.764,
      "step": 250500
    },
    {
      "activations/layer0_attention_weight_max": 16.09017562866211,
      "activations/layer0_attention_weight_min": -12.781142234802246,
      "activations/layer10_attention_weight_max": 34.20374298095703,
      "activations/layer10_attention_weight_min": -34.69388961791992,
      "activations/layer11_attention_weight_max": 33.0942268371582,
      "activations/layer11_attention_weight_min": -33.14502716064453,
      "activations/layer12_attention_weight_max": 27.551313400268555,
      "activations/layer12_attention_weight_min": -26.969133377075195,
      "activations/layer13_attention_weight_max": 40.64784240722656,
      "activations/layer13_attention_weight_min": -34.27301788330078,
      "activations/layer14_attention_weight_max": 44.2911491394043,
      "activations/layer14_attention_weight_min": -36.66914367675781,
      "activations/layer15_attention_weight_max": 37.09690856933594,
      "activations/layer15_attention_weight_min": -32.626522064208984,
      "activations/layer16_attention_weight_max": 29.38592529296875,
      "activations/layer16_attention_weight_min": -28.4902400970459,
      "activations/layer17_attention_weight_max": 28.27846336364746,
      "activations/layer17_attention_weight_min": -26.849960327148438,
      "activations/layer18_attention_weight_max": 27.556392669677734,
      "activations/layer18_attention_weight_min": -23.423391342163086,
      "activations/layer19_attention_weight_max": 34.02705764770508,
      "activations/layer19_attention_weight_min": -32.241886138916016,
      "activations/layer1_attention_weight_max": 19.04319190979004,
      "activations/layer1_attention_weight_min": -19.274446487426758,
      "activations/layer20_attention_weight_max": 27.731842041015625,
      "activations/layer20_attention_weight_min": -26.417024612426758,
      "activations/layer21_attention_weight_max": 27.721004486083984,
      "activations/layer21_attention_weight_min": -23.812206268310547,
      "activations/layer22_attention_weight_max": 45.63137435913086,
      "activations/layer22_attention_weight_min": -29.421051025390625,
      "activations/layer23_attention_weight_max": 31.976032257080078,
      "activations/layer23_attention_weight_min": -25.354171752929688,
      "activations/layer2_attention_weight_max": 32.35447692871094,
      "activations/layer2_attention_weight_min": -31.82583999633789,
      "activations/layer3_attention_weight_max": 101.76273345947266,
      "activations/layer3_attention_weight_min": -100.49606323242188,
      "activations/layer4_attention_weight_max": 117.7010498046875,
      "activations/layer4_attention_weight_min": -115.06011962890625,
      "activations/layer5_attention_weight_max": 52.05918884277344,
      "activations/layer5_attention_weight_min": -62.692893981933594,
      "activations/layer6_attention_weight_max": 46.80362319946289,
      "activations/layer6_attention_weight_min": -49.10261917114258,
      "activations/layer7_attention_weight_max": 97.14564514160156,
      "activations/layer7_attention_weight_min": -94.28028869628906,
      "activations/layer8_attention_weight_max": 42.82624435424805,
      "activations/layer8_attention_weight_min": -43.79182815551758,
      "activations/layer9_attention_weight_max": 33.07791519165039,
      "activations/layer9_attention_weight_min": -34.31454086303711,
      "epoch": 14.56,
      "learning_rate": 5.6687878787878785e-05,
      "loss": 2.746,
      "step": 250550
    },
    {
      "activations/layer0_attention_weight_max": 15.9498872756958,
      "activations/layer0_attention_weight_min": -13.19947624206543,
      "activations/layer10_attention_weight_max": 36.01179885864258,
      "activations/layer10_attention_weight_min": -34.43605422973633,
      "activations/layer11_attention_weight_max": 35.75128173828125,
      "activations/layer11_attention_weight_min": -34.663848876953125,
      "activations/layer12_attention_weight_max": 26.67879867553711,
      "activations/layer12_attention_weight_min": -25.77802276611328,
      "activations/layer13_attention_weight_max": 42.85987091064453,
      "activations/layer13_attention_weight_min": -35.849159240722656,
      "activations/layer14_attention_weight_max": 44.069644927978516,
      "activations/layer14_attention_weight_min": -38.72980499267578,
      "activations/layer15_attention_weight_max": 38.73073959350586,
      "activations/layer15_attention_weight_min": -32.908363342285156,
      "activations/layer16_attention_weight_max": 28.866758346557617,
      "activations/layer16_attention_weight_min": -28.90275764465332,
      "activations/layer17_attention_weight_max": 30.95162582397461,
      "activations/layer17_attention_weight_min": -26.549421310424805,
      "activations/layer18_attention_weight_max": 32.616241455078125,
      "activations/layer18_attention_weight_min": -24.436677932739258,
      "activations/layer19_attention_weight_max": 34.7819938659668,
      "activations/layer19_attention_weight_min": -29.89584732055664,
      "activations/layer1_attention_weight_max": 16.526521682739258,
      "activations/layer1_attention_weight_min": -15.165170669555664,
      "activations/layer20_attention_weight_max": 26.683908462524414,
      "activations/layer20_attention_weight_min": -24.56490135192871,
      "activations/layer21_attention_weight_max": 28.450040817260742,
      "activations/layer21_attention_weight_min": -23.38924217224121,
      "activations/layer22_attention_weight_max": 42.96917724609375,
      "activations/layer22_attention_weight_min": -28.77252769470215,
      "activations/layer23_attention_weight_max": 33.89884567260742,
      "activations/layer23_attention_weight_min": -25.953718185424805,
      "activations/layer2_attention_weight_max": 31.91445541381836,
      "activations/layer2_attention_weight_min": -32.419654846191406,
      "activations/layer3_attention_weight_max": 99.26264953613281,
      "activations/layer3_attention_weight_min": -101.85578918457031,
      "activations/layer4_attention_weight_max": 113.89033508300781,
      "activations/layer4_attention_weight_min": -114.32179260253906,
      "activations/layer5_attention_weight_max": 53.43442916870117,
      "activations/layer5_attention_weight_min": -64.95915222167969,
      "activations/layer6_attention_weight_max": 47.712223052978516,
      "activations/layer6_attention_weight_min": -47.961158752441406,
      "activations/layer7_attention_weight_max": 92.80248260498047,
      "activations/layer7_attention_weight_min": -93.99248504638672,
      "activations/layer8_attention_weight_max": 45.805240631103516,
      "activations/layer8_attention_weight_min": -44.31325149536133,
      "activations/layer9_attention_weight_max": 34.089046478271484,
      "activations/layer9_attention_weight_min": -35.07595443725586,
      "epoch": 14.56,
      "learning_rate": 5.6668939393939386e-05,
      "loss": 2.7655,
      "step": 250600
    },
    {
      "activations/layer0_attention_weight_max": 16.832319259643555,
      "activations/layer0_attention_weight_min": -12.022462844848633,
      "activations/layer10_attention_weight_max": 34.711036682128906,
      "activations/layer10_attention_weight_min": -34.087013244628906,
      "activations/layer11_attention_weight_max": 32.38351821899414,
      "activations/layer11_attention_weight_min": -35.89232635498047,
      "activations/layer12_attention_weight_max": 24.86484146118164,
      "activations/layer12_attention_weight_min": -26.29277229309082,
      "activations/layer13_attention_weight_max": 36.287166595458984,
      "activations/layer13_attention_weight_min": -38.03575897216797,
      "activations/layer14_attention_weight_max": 46.21942901611328,
      "activations/layer14_attention_weight_min": -39.89674758911133,
      "activations/layer15_attention_weight_max": 35.87360382080078,
      "activations/layer15_attention_weight_min": -37.78617858886719,
      "activations/layer16_attention_weight_max": 28.751602172851562,
      "activations/layer16_attention_weight_min": -28.203271865844727,
      "activations/layer17_attention_weight_max": 29.28207778930664,
      "activations/layer17_attention_weight_min": -28.033729553222656,
      "activations/layer18_attention_weight_max": 33.90309524536133,
      "activations/layer18_attention_weight_min": -27.73957633972168,
      "activations/layer19_attention_weight_max": 34.16010284423828,
      "activations/layer19_attention_weight_min": -32.93413543701172,
      "activations/layer1_attention_weight_max": 16.68499755859375,
      "activations/layer1_attention_weight_min": -15.96308708190918,
      "activations/layer20_attention_weight_max": 29.995716094970703,
      "activations/layer20_attention_weight_min": -28.40896224975586,
      "activations/layer21_attention_weight_max": 30.389135360717773,
      "activations/layer21_attention_weight_min": -26.59030532836914,
      "activations/layer22_attention_weight_max": 44.79095458984375,
      "activations/layer22_attention_weight_min": -33.20326232910156,
      "activations/layer23_attention_weight_max": 35.41943359375,
      "activations/layer23_attention_weight_min": -28.883960723876953,
      "activations/layer2_attention_weight_max": 31.980810165405273,
      "activations/layer2_attention_weight_min": -31.54477310180664,
      "activations/layer3_attention_weight_max": 100.169677734375,
      "activations/layer3_attention_weight_min": -102.018798828125,
      "activations/layer4_attention_weight_max": 116.01824951171875,
      "activations/layer4_attention_weight_min": -118.29243469238281,
      "activations/layer5_attention_weight_max": 57.32916259765625,
      "activations/layer5_attention_weight_min": -66.28962707519531,
      "activations/layer6_attention_weight_max": 47.70392990112305,
      "activations/layer6_attention_weight_min": -50.05799865722656,
      "activations/layer7_attention_weight_max": 97.74884796142578,
      "activations/layer7_attention_weight_min": -93.25878143310547,
      "activations/layer8_attention_weight_max": 40.96938705444336,
      "activations/layer8_attention_weight_min": -45.93548583984375,
      "activations/layer9_attention_weight_max": 32.55874252319336,
      "activations/layer9_attention_weight_min": -36.201786041259766,
      "epoch": 14.56,
      "learning_rate": 5.6649999999999995e-05,
      "loss": 2.7688,
      "step": 250650
    },
    {
      "activations/layer0_attention_weight_max": 16.07328987121582,
      "activations/layer0_attention_weight_min": -12.590953826904297,
      "activations/layer10_attention_weight_max": 33.04252624511719,
      "activations/layer10_attention_weight_min": -33.992408752441406,
      "activations/layer11_attention_weight_max": 31.341459274291992,
      "activations/layer11_attention_weight_min": -32.30012512207031,
      "activations/layer12_attention_weight_max": 24.308080673217773,
      "activations/layer12_attention_weight_min": -26.049636840820312,
      "activations/layer13_attention_weight_max": 36.36688232421875,
      "activations/layer13_attention_weight_min": -36.33320236206055,
      "activations/layer14_attention_weight_max": 38.16055679321289,
      "activations/layer14_attention_weight_min": -36.03701400756836,
      "activations/layer15_attention_weight_max": 38.7051887512207,
      "activations/layer15_attention_weight_min": -31.806673049926758,
      "activations/layer16_attention_weight_max": 31.7591552734375,
      "activations/layer16_attention_weight_min": -27.49241828918457,
      "activations/layer17_attention_weight_max": 31.238037109375,
      "activations/layer17_attention_weight_min": -24.07493019104004,
      "activations/layer18_attention_weight_max": 32.4961051940918,
      "activations/layer18_attention_weight_min": -23.63275909423828,
      "activations/layer19_attention_weight_max": 41.85309982299805,
      "activations/layer19_attention_weight_min": -30.488418579101562,
      "activations/layer1_attention_weight_max": 15.940566062927246,
      "activations/layer1_attention_weight_min": -14.342385292053223,
      "activations/layer20_attention_weight_max": 33.270118713378906,
      "activations/layer20_attention_weight_min": -24.001216888427734,
      "activations/layer21_attention_weight_max": 30.942617416381836,
      "activations/layer21_attention_weight_min": -24.004343032836914,
      "activations/layer22_attention_weight_max": 50.80447769165039,
      "activations/layer22_attention_weight_min": -33.125701904296875,
      "activations/layer23_attention_weight_max": 37.89231491088867,
      "activations/layer23_attention_weight_min": -26.22389030456543,
      "activations/layer2_attention_weight_max": 31.086944580078125,
      "activations/layer2_attention_weight_min": -32.389549255371094,
      "activations/layer3_attention_weight_max": 91.18596649169922,
      "activations/layer3_attention_weight_min": -99.21931457519531,
      "activations/layer4_attention_weight_max": 106.24293518066406,
      "activations/layer4_attention_weight_min": -110.68110656738281,
      "activations/layer5_attention_weight_max": 49.89463424682617,
      "activations/layer5_attention_weight_min": -60.20318603515625,
      "activations/layer6_attention_weight_max": 41.25397872924805,
      "activations/layer6_attention_weight_min": -46.20659637451172,
      "activations/layer7_attention_weight_max": 89.59361267089844,
      "activations/layer7_attention_weight_min": -89.05118560791016,
      "activations/layer8_attention_weight_max": 37.50764083862305,
      "activations/layer8_attention_weight_min": -41.59271240234375,
      "activations/layer9_attention_weight_max": 31.66629981994629,
      "activations/layer9_attention_weight_min": -33.28720474243164,
      "epoch": 14.57,
      "learning_rate": 5.66310606060606e-05,
      "loss": 2.7523,
      "step": 250700
    },
    {
      "activations/layer0_attention_weight_max": 15.92542839050293,
      "activations/layer0_attention_weight_min": -12.449691772460938,
      "activations/layer10_attention_weight_max": 32.88677215576172,
      "activations/layer10_attention_weight_min": -33.79804229736328,
      "activations/layer11_attention_weight_max": 31.346311569213867,
      "activations/layer11_attention_weight_min": -33.60808181762695,
      "activations/layer12_attention_weight_max": 25.242856979370117,
      "activations/layer12_attention_weight_min": -25.56512451171875,
      "activations/layer13_attention_weight_max": 38.5572509765625,
      "activations/layer13_attention_weight_min": -36.36735534667969,
      "activations/layer14_attention_weight_max": 42.86194610595703,
      "activations/layer14_attention_weight_min": -40.703514099121094,
      "activations/layer15_attention_weight_max": 34.99054718017578,
      "activations/layer15_attention_weight_min": -33.656856536865234,
      "activations/layer16_attention_weight_max": 29.70292854309082,
      "activations/layer16_attention_weight_min": -27.831768035888672,
      "activations/layer17_attention_weight_max": 30.417903900146484,
      "activations/layer17_attention_weight_min": -26.160858154296875,
      "activations/layer18_attention_weight_max": 32.408447265625,
      "activations/layer18_attention_weight_min": -26.335058212280273,
      "activations/layer19_attention_weight_max": 35.29851531982422,
      "activations/layer19_attention_weight_min": -34.83134078979492,
      "activations/layer1_attention_weight_max": 16.587718963623047,
      "activations/layer1_attention_weight_min": -15.341135025024414,
      "activations/layer20_attention_weight_max": 30.651138305664062,
      "activations/layer20_attention_weight_min": -26.221717834472656,
      "activations/layer21_attention_weight_max": 31.973220825195312,
      "activations/layer21_attention_weight_min": -25.486989974975586,
      "activations/layer22_attention_weight_max": 44.86262893676758,
      "activations/layer22_attention_weight_min": -33.177547454833984,
      "activations/layer23_attention_weight_max": 33.43671798706055,
      "activations/layer23_attention_weight_min": -25.93582534790039,
      "activations/layer2_attention_weight_max": 33.895042419433594,
      "activations/layer2_attention_weight_min": -31.71088981628418,
      "activations/layer3_attention_weight_max": 95.09832763671875,
      "activations/layer3_attention_weight_min": -94.16661071777344,
      "activations/layer4_attention_weight_max": 113.33553314208984,
      "activations/layer4_attention_weight_min": -108.45226287841797,
      "activations/layer5_attention_weight_max": 51.85633850097656,
      "activations/layer5_attention_weight_min": -61.029632568359375,
      "activations/layer6_attention_weight_max": 46.76897430419922,
      "activations/layer6_attention_weight_min": -48.3282356262207,
      "activations/layer7_attention_weight_max": 92.9562759399414,
      "activations/layer7_attention_weight_min": -90.70750427246094,
      "activations/layer8_attention_weight_max": 38.311561584472656,
      "activations/layer8_attention_weight_min": -41.22799301147461,
      "activations/layer9_attention_weight_max": 32.009342193603516,
      "activations/layer9_attention_weight_min": -32.65850067138672,
      "epoch": 14.57,
      "learning_rate": 5.661212121212121e-05,
      "loss": 2.7511,
      "step": 250750
    },
    {
      "activations/layer0_attention_weight_max": 15.74301528930664,
      "activations/layer0_attention_weight_min": -13.186832427978516,
      "activations/layer10_attention_weight_max": 35.14038848876953,
      "activations/layer10_attention_weight_min": -33.730003356933594,
      "activations/layer11_attention_weight_max": 37.37200927734375,
      "activations/layer11_attention_weight_min": -34.24473571777344,
      "activations/layer12_attention_weight_max": 26.538976669311523,
      "activations/layer12_attention_weight_min": -25.452713012695312,
      "activations/layer13_attention_weight_max": 38.92893981933594,
      "activations/layer13_attention_weight_min": -33.42626190185547,
      "activations/layer14_attention_weight_max": 42.38868713378906,
      "activations/layer14_attention_weight_min": -36.71760940551758,
      "activations/layer15_attention_weight_max": 36.34345626831055,
      "activations/layer15_attention_weight_min": -34.97249221801758,
      "activations/layer16_attention_weight_max": 32.376094818115234,
      "activations/layer16_attention_weight_min": -29.609220504760742,
      "activations/layer17_attention_weight_max": 32.581382751464844,
      "activations/layer17_attention_weight_min": -26.036209106445312,
      "activations/layer18_attention_weight_max": 32.37119674682617,
      "activations/layer18_attention_weight_min": -25.538867950439453,
      "activations/layer19_attention_weight_max": 38.86931610107422,
      "activations/layer19_attention_weight_min": -30.73223304748535,
      "activations/layer1_attention_weight_max": 17.62255859375,
      "activations/layer1_attention_weight_min": -16.433820724487305,
      "activations/layer20_attention_weight_max": 31.86998176574707,
      "activations/layer20_attention_weight_min": -26.73753547668457,
      "activations/layer21_attention_weight_max": 31.193206787109375,
      "activations/layer21_attention_weight_min": -27.687673568725586,
      "activations/layer22_attention_weight_max": 46.11077117919922,
      "activations/layer22_attention_weight_min": -32.126548767089844,
      "activations/layer23_attention_weight_max": 31.404088973999023,
      "activations/layer23_attention_weight_min": -26.00545310974121,
      "activations/layer2_attention_weight_max": 34.966217041015625,
      "activations/layer2_attention_weight_min": -33.54485321044922,
      "activations/layer3_attention_weight_max": 105.69310760498047,
      "activations/layer3_attention_weight_min": -105.10908508300781,
      "activations/layer4_attention_weight_max": 120.62102508544922,
      "activations/layer4_attention_weight_min": -111.57588195800781,
      "activations/layer5_attention_weight_max": 52.587013244628906,
      "activations/layer5_attention_weight_min": -62.146209716796875,
      "activations/layer6_attention_weight_max": 43.57625961303711,
      "activations/layer6_attention_weight_min": -46.88322067260742,
      "activations/layer7_attention_weight_max": 93.74231719970703,
      "activations/layer7_attention_weight_min": -88.44136047363281,
      "activations/layer8_attention_weight_max": 42.56953811645508,
      "activations/layer8_attention_weight_min": -43.07125473022461,
      "activations/layer9_attention_weight_max": 35.06023025512695,
      "activations/layer9_attention_weight_min": -34.07291793823242,
      "epoch": 14.57,
      "learning_rate": 5.659318181818181e-05,
      "loss": 2.7543,
      "step": 250800
    },
    {
      "activations/layer0_attention_weight_max": 16.357030868530273,
      "activations/layer0_attention_weight_min": -12.584141731262207,
      "activations/layer10_attention_weight_max": 30.16447639465332,
      "activations/layer10_attention_weight_min": -32.81292724609375,
      "activations/layer11_attention_weight_max": 30.57735824584961,
      "activations/layer11_attention_weight_min": -31.81783103942871,
      "activations/layer12_attention_weight_max": 24.775962829589844,
      "activations/layer12_attention_weight_min": -27.033151626586914,
      "activations/layer13_attention_weight_max": 38.42368698120117,
      "activations/layer13_attention_weight_min": -33.655540466308594,
      "activations/layer14_attention_weight_max": 45.49674987792969,
      "activations/layer14_attention_weight_min": -36.99034118652344,
      "activations/layer15_attention_weight_max": 37.34504699707031,
      "activations/layer15_attention_weight_min": -32.55218505859375,
      "activations/layer16_attention_weight_max": 30.736202239990234,
      "activations/layer16_attention_weight_min": -27.68732452392578,
      "activations/layer17_attention_weight_max": 31.42111587524414,
      "activations/layer17_attention_weight_min": -27.41071891784668,
      "activations/layer18_attention_weight_max": 31.536386489868164,
      "activations/layer18_attention_weight_min": -25.080650329589844,
      "activations/layer19_attention_weight_max": 32.42547607421875,
      "activations/layer19_attention_weight_min": -32.43476486206055,
      "activations/layer1_attention_weight_max": 15.899811744689941,
      "activations/layer1_attention_weight_min": -15.161992073059082,
      "activations/layer20_attention_weight_max": 28.412981033325195,
      "activations/layer20_attention_weight_min": -25.95983123779297,
      "activations/layer21_attention_weight_max": 26.73370933532715,
      "activations/layer21_attention_weight_min": -24.676029205322266,
      "activations/layer22_attention_weight_max": 37.232872009277344,
      "activations/layer22_attention_weight_min": -31.722272872924805,
      "activations/layer23_attention_weight_max": 30.214588165283203,
      "activations/layer23_attention_weight_min": -26.733661651611328,
      "activations/layer2_attention_weight_max": 31.276992797851562,
      "activations/layer2_attention_weight_min": -31.970867156982422,
      "activations/layer3_attention_weight_max": 92.12959289550781,
      "activations/layer3_attention_weight_min": -93.68310546875,
      "activations/layer4_attention_weight_max": 106.1046371459961,
      "activations/layer4_attention_weight_min": -112.4999008178711,
      "activations/layer5_attention_weight_max": 49.25181579589844,
      "activations/layer5_attention_weight_min": -60.97288513183594,
      "activations/layer6_attention_weight_max": 41.57086944580078,
      "activations/layer6_attention_weight_min": -46.69126892089844,
      "activations/layer7_attention_weight_max": 89.40055847167969,
      "activations/layer7_attention_weight_min": -86.4105453491211,
      "activations/layer8_attention_weight_max": 38.19319534301758,
      "activations/layer8_attention_weight_min": -39.304832458496094,
      "activations/layer9_attention_weight_max": 30.307870864868164,
      "activations/layer9_attention_weight_min": -32.17543411254883,
      "epoch": 14.58,
      "learning_rate": 5.6574242424242414e-05,
      "loss": 2.7569,
      "step": 250850
    },
    {
      "activations/layer0_attention_weight_max": 15.782400131225586,
      "activations/layer0_attention_weight_min": -12.12718677520752,
      "activations/layer10_attention_weight_max": 35.03485107421875,
      "activations/layer10_attention_weight_min": -32.76298522949219,
      "activations/layer11_attention_weight_max": 34.078392028808594,
      "activations/layer11_attention_weight_min": -34.614723205566406,
      "activations/layer12_attention_weight_max": 28.063758850097656,
      "activations/layer12_attention_weight_min": -24.44144630432129,
      "activations/layer13_attention_weight_max": 39.61830139160156,
      "activations/layer13_attention_weight_min": -33.470428466796875,
      "activations/layer14_attention_weight_max": 42.11810302734375,
      "activations/layer14_attention_weight_min": -36.30535125732422,
      "activations/layer15_attention_weight_max": 39.346675872802734,
      "activations/layer15_attention_weight_min": -35.66952133178711,
      "activations/layer16_attention_weight_max": 30.810876846313477,
      "activations/layer16_attention_weight_min": -28.657808303833008,
      "activations/layer17_attention_weight_max": 31.88710594177246,
      "activations/layer17_attention_weight_min": -26.94594383239746,
      "activations/layer18_attention_weight_max": 34.5478630065918,
      "activations/layer18_attention_weight_min": -24.89642333984375,
      "activations/layer19_attention_weight_max": 33.58466720581055,
      "activations/layer19_attention_weight_min": -31.573165893554688,
      "activations/layer1_attention_weight_max": 16.41596031188965,
      "activations/layer1_attention_weight_min": -14.869165420532227,
      "activations/layer20_attention_weight_max": 29.075448989868164,
      "activations/layer20_attention_weight_min": -29.2791805267334,
      "activations/layer21_attention_weight_max": 31.72150230407715,
      "activations/layer21_attention_weight_min": -26.72258186340332,
      "activations/layer22_attention_weight_max": 44.452880859375,
      "activations/layer22_attention_weight_min": -33.37036895751953,
      "activations/layer23_attention_weight_max": 34.88205337524414,
      "activations/layer23_attention_weight_min": -26.781837463378906,
      "activations/layer2_attention_weight_max": 31.908035278320312,
      "activations/layer2_attention_weight_min": -31.673810958862305,
      "activations/layer3_attention_weight_max": 97.43632507324219,
      "activations/layer3_attention_weight_min": -95.4889144897461,
      "activations/layer4_attention_weight_max": 111.03531646728516,
      "activations/layer4_attention_weight_min": -111.21053314208984,
      "activations/layer5_attention_weight_max": 52.845394134521484,
      "activations/layer5_attention_weight_min": -63.51690673828125,
      "activations/layer6_attention_weight_max": 45.0058479309082,
      "activations/layer6_attention_weight_min": -46.611576080322266,
      "activations/layer7_attention_weight_max": 92.51404571533203,
      "activations/layer7_attention_weight_min": -90.65300750732422,
      "activations/layer8_attention_weight_max": 40.86292266845703,
      "activations/layer8_attention_weight_min": -41.77287673950195,
      "activations/layer9_attention_weight_max": 33.864131927490234,
      "activations/layer9_attention_weight_min": -34.61676025390625,
      "epoch": 14.58,
      "learning_rate": 5.655530303030303e-05,
      "loss": 2.7713,
      "step": 250900
    },
    {
      "activations/layer0_attention_weight_max": 15.103805541992188,
      "activations/layer0_attention_weight_min": -12.129478454589844,
      "activations/layer10_attention_weight_max": 31.48114013671875,
      "activations/layer10_attention_weight_min": -31.658329010009766,
      "activations/layer11_attention_weight_max": 33.410972595214844,
      "activations/layer11_attention_weight_min": -31.95307159423828,
      "activations/layer12_attention_weight_max": 25.817798614501953,
      "activations/layer12_attention_weight_min": -25.12298583984375,
      "activations/layer13_attention_weight_max": 38.622650146484375,
      "activations/layer13_attention_weight_min": -35.29814910888672,
      "activations/layer14_attention_weight_max": 40.607078552246094,
      "activations/layer14_attention_weight_min": -37.678524017333984,
      "activations/layer15_attention_weight_max": 37.60227584838867,
      "activations/layer15_attention_weight_min": -34.186378479003906,
      "activations/layer16_attention_weight_max": 31.440332412719727,
      "activations/layer16_attention_weight_min": -27.69727897644043,
      "activations/layer17_attention_weight_max": 29.941261291503906,
      "activations/layer17_attention_weight_min": -26.438785552978516,
      "activations/layer18_attention_weight_max": 30.977758407592773,
      "activations/layer18_attention_weight_min": -25.751619338989258,
      "activations/layer19_attention_weight_max": 31.84700584411621,
      "activations/layer19_attention_weight_min": -33.162105560302734,
      "activations/layer1_attention_weight_max": 16.96504020690918,
      "activations/layer1_attention_weight_min": -13.98320198059082,
      "activations/layer20_attention_weight_max": 27.730117797851562,
      "activations/layer20_attention_weight_min": -26.031564712524414,
      "activations/layer21_attention_weight_max": 25.992446899414062,
      "activations/layer21_attention_weight_min": -24.34635353088379,
      "activations/layer22_attention_weight_max": 38.16926193237305,
      "activations/layer22_attention_weight_min": -30.94048309326172,
      "activations/layer23_attention_weight_max": 30.42812728881836,
      "activations/layer23_attention_weight_min": -24.960264205932617,
      "activations/layer2_attention_weight_max": 33.35676193237305,
      "activations/layer2_attention_weight_min": -31.340267181396484,
      "activations/layer3_attention_weight_max": 95.97662353515625,
      "activations/layer3_attention_weight_min": -96.35615539550781,
      "activations/layer4_attention_weight_max": 108.70076751708984,
      "activations/layer4_attention_weight_min": -108.69563293457031,
      "activations/layer5_attention_weight_max": 51.11332321166992,
      "activations/layer5_attention_weight_min": -58.574520111083984,
      "activations/layer6_attention_weight_max": 44.958457946777344,
      "activations/layer6_attention_weight_min": -45.798362731933594,
      "activations/layer7_attention_weight_max": 88.09197998046875,
      "activations/layer7_attention_weight_min": -94.55116271972656,
      "activations/layer8_attention_weight_max": 38.40685272216797,
      "activations/layer8_attention_weight_min": -39.728240966796875,
      "activations/layer9_attention_weight_max": 30.74818992614746,
      "activations/layer9_attention_weight_min": -32.35988235473633,
      "epoch": 14.58,
      "learning_rate": 5.653636363636363e-05,
      "loss": 2.7504,
      "step": 250950
    },
    {
      "activations/layer0_attention_weight_max": 16.0421199798584,
      "activations/layer0_attention_weight_min": -12.26229476928711,
      "activations/layer10_attention_weight_max": 33.78334045410156,
      "activations/layer10_attention_weight_min": -34.462589263916016,
      "activations/layer11_attention_weight_max": 34.86125564575195,
      "activations/layer11_attention_weight_min": -35.99200439453125,
      "activations/layer12_attention_weight_max": 29.675899505615234,
      "activations/layer12_attention_weight_min": -25.300926208496094,
      "activations/layer13_attention_weight_max": 40.69667053222656,
      "activations/layer13_attention_weight_min": -34.2729606628418,
      "activations/layer14_attention_weight_max": 45.645172119140625,
      "activations/layer14_attention_weight_min": -38.4382209777832,
      "activations/layer15_attention_weight_max": 40.732398986816406,
      "activations/layer15_attention_weight_min": -32.896297454833984,
      "activations/layer16_attention_weight_max": 31.799453735351562,
      "activations/layer16_attention_weight_min": -27.52153778076172,
      "activations/layer17_attention_weight_max": 30.702880859375,
      "activations/layer17_attention_weight_min": -24.21893310546875,
      "activations/layer18_attention_weight_max": 30.674640655517578,
      "activations/layer18_attention_weight_min": -24.55251693725586,
      "activations/layer19_attention_weight_max": 33.85171127319336,
      "activations/layer19_attention_weight_min": -29.284738540649414,
      "activations/layer1_attention_weight_max": 16.341651916503906,
      "activations/layer1_attention_weight_min": -15.439116477966309,
      "activations/layer20_attention_weight_max": 30.292661666870117,
      "activations/layer20_attention_weight_min": -25.462045669555664,
      "activations/layer21_attention_weight_max": 33.0880012512207,
      "activations/layer21_attention_weight_min": -25.512510299682617,
      "activations/layer22_attention_weight_max": 44.241798400878906,
      "activations/layer22_attention_weight_min": -29.93347930908203,
      "activations/layer23_attention_weight_max": 34.46791458129883,
      "activations/layer23_attention_weight_min": -25.090740203857422,
      "activations/layer2_attention_weight_max": 32.094139099121094,
      "activations/layer2_attention_weight_min": -31.670970916748047,
      "activations/layer3_attention_weight_max": 94.74907684326172,
      "activations/layer3_attention_weight_min": -99.538818359375,
      "activations/layer4_attention_weight_max": 111.42158508300781,
      "activations/layer4_attention_weight_min": -113.15056610107422,
      "activations/layer5_attention_weight_max": 51.364280700683594,
      "activations/layer5_attention_weight_min": -63.9556884765625,
      "activations/layer6_attention_weight_max": 45.46857452392578,
      "activations/layer6_attention_weight_min": -48.164791107177734,
      "activations/layer7_attention_weight_max": 99.34614562988281,
      "activations/layer7_attention_weight_min": -92.55709075927734,
      "activations/layer8_attention_weight_max": 44.14198684692383,
      "activations/layer8_attention_weight_min": -46.249305725097656,
      "activations/layer9_attention_weight_max": 33.62412643432617,
      "activations/layer9_attention_weight_min": -34.14009475708008,
      "epoch": 14.58,
      "learning_rate": 5.651742424242424e-05,
      "loss": 2.7561,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.5025,
      "eval_samples_per_second": 505.028,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.5025,
      "eval_openwebtext_samples_per_second": 505.028,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 2.0193,
      "eval_wikitext_samples_per_second": 225.823,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_lambada_loss": 2.447265625,
      "eval_lambada_ppl": 11.556703116117722,
      "eval_lambada_runtime": 9.6192,
      "eval_lambada_samples_per_second": 506.173,
      "step": 251000
    },
    {
      "activations/layer0_attention_weight_max": 15.561759948730469,
      "activations/layer0_attention_weight_min": -13.960490226745605,
      "activations/layer10_attention_weight_max": 41.91962814331055,
      "activations/layer10_attention_weight_min": -40.214088439941406,
      "activations/layer11_attention_weight_max": 41.18804931640625,
      "activations/layer11_attention_weight_min": -42.347660064697266,
      "activations/layer12_attention_weight_max": 30.994876861572266,
      "activations/layer12_attention_weight_min": -27.67087173461914,
      "activations/layer13_attention_weight_max": 44.987342834472656,
      "activations/layer13_attention_weight_min": -39.101253509521484,
      "activations/layer14_attention_weight_max": 50.55935287475586,
      "activations/layer14_attention_weight_min": -40.80602264404297,
      "activations/layer15_attention_weight_max": 40.97645950317383,
      "activations/layer15_attention_weight_min": -34.228267669677734,
      "activations/layer16_attention_weight_max": 33.886295318603516,
      "activations/layer16_attention_weight_min": -27.963542938232422,
      "activations/layer17_attention_weight_max": 32.51094436645508,
      "activations/layer17_attention_weight_min": -26.665185928344727,
      "activations/layer18_attention_weight_max": 32.33955001831055,
      "activations/layer18_attention_weight_min": -25.479463577270508,
      "activations/layer19_attention_weight_max": 39.067569732666016,
      "activations/layer19_attention_weight_min": -30.485395431518555,
      "activations/layer1_attention_weight_max": 16.303865432739258,
      "activations/layer1_attention_weight_min": -15.181182861328125,
      "activations/layer20_attention_weight_max": 37.277748107910156,
      "activations/layer20_attention_weight_min": -23.722442626953125,
      "activations/layer21_attention_weight_max": 31.761699676513672,
      "activations/layer21_attention_weight_min": -22.8837833404541,
      "activations/layer22_attention_weight_max": 45.78947067260742,
      "activations/layer22_attention_weight_min": -28.07540512084961,
      "activations/layer23_attention_weight_max": 36.316688537597656,
      "activations/layer23_attention_weight_min": -25.36217498779297,
      "activations/layer2_attention_weight_max": 33.402400970458984,
      "activations/layer2_attention_weight_min": -33.91163635253906,
      "activations/layer3_attention_weight_max": 100.0378646850586,
      "activations/layer3_attention_weight_min": -100.77941131591797,
      "activations/layer4_attention_weight_max": 114.95635986328125,
      "activations/layer4_attention_weight_min": -117.4788818359375,
      "activations/layer5_attention_weight_max": 53.002925872802734,
      "activations/layer5_attention_weight_min": -63.640830993652344,
      "activations/layer6_attention_weight_max": 50.41175842285156,
      "activations/layer6_attention_weight_min": -50.6353874206543,
      "activations/layer7_attention_weight_max": 107.4431381225586,
      "activations/layer7_attention_weight_min": -107.27880096435547,
      "activations/layer8_attention_weight_max": 49.40353012084961,
      "activations/layer8_attention_weight_min": -51.10930252075195,
      "activations/layer9_attention_weight_max": 38.97859191894531,
      "activations/layer9_attention_weight_min": -39.79787826538086,
      "epoch": 14.59,
      "learning_rate": 5.649848484848484e-05,
      "loss": 2.7583,
      "step": 251050
    },
    {
      "activations/layer0_attention_weight_max": 15.685478210449219,
      "activations/layer0_attention_weight_min": -12.616609573364258,
      "activations/layer10_attention_weight_max": 34.900638580322266,
      "activations/layer10_attention_weight_min": -36.7100944519043,
      "activations/layer11_attention_weight_max": 33.410457611083984,
      "activations/layer11_attention_weight_min": -33.70179748535156,
      "activations/layer12_attention_weight_max": 25.992448806762695,
      "activations/layer12_attention_weight_min": -25.784364700317383,
      "activations/layer13_attention_weight_max": 37.59339141845703,
      "activations/layer13_attention_weight_min": -35.726444244384766,
      "activations/layer14_attention_weight_max": 41.02893829345703,
      "activations/layer14_attention_weight_min": -38.854393005371094,
      "activations/layer15_attention_weight_max": 35.02477264404297,
      "activations/layer15_attention_weight_min": -33.04740905761719,
      "activations/layer16_attention_weight_max": 30.996944427490234,
      "activations/layer16_attention_weight_min": -27.94053840637207,
      "activations/layer17_attention_weight_max": 27.809919357299805,
      "activations/layer17_attention_weight_min": -26.051740646362305,
      "activations/layer18_attention_weight_max": 28.953157424926758,
      "activations/layer18_attention_weight_min": -25.58626937866211,
      "activations/layer19_attention_weight_max": 32.69807815551758,
      "activations/layer19_attention_weight_min": -31.53362464904785,
      "activations/layer1_attention_weight_max": 15.873122215270996,
      "activations/layer1_attention_weight_min": -14.761425018310547,
      "activations/layer20_attention_weight_max": 26.60856819152832,
      "activations/layer20_attention_weight_min": -26.6982364654541,
      "activations/layer21_attention_weight_max": 28.073177337646484,
      "activations/layer21_attention_weight_min": -27.367618560791016,
      "activations/layer22_attention_weight_max": 40.88019561767578,
      "activations/layer22_attention_weight_min": -31.17868995666504,
      "activations/layer23_attention_weight_max": 31.605558395385742,
      "activations/layer23_attention_weight_min": -25.695974349975586,
      "activations/layer2_attention_weight_max": 34.263023376464844,
      "activations/layer2_attention_weight_min": -32.37537384033203,
      "activations/layer3_attention_weight_max": 95.17047119140625,
      "activations/layer3_attention_weight_min": -98.73001098632812,
      "activations/layer4_attention_weight_max": 107.28719329833984,
      "activations/layer4_attention_weight_min": -116.54882049560547,
      "activations/layer5_attention_weight_max": 51.23261260986328,
      "activations/layer5_attention_weight_min": -61.451927185058594,
      "activations/layer6_attention_weight_max": 43.536705017089844,
      "activations/layer6_attention_weight_min": -47.94198226928711,
      "activations/layer7_attention_weight_max": 89.44107818603516,
      "activations/layer7_attention_weight_min": -95.98855590820312,
      "activations/layer8_attention_weight_max": 42.29096984863281,
      "activations/layer8_attention_weight_min": -45.47464370727539,
      "activations/layer9_attention_weight_max": 33.454280853271484,
      "activations/layer9_attention_weight_min": -36.474552154541016,
      "epoch": 14.59,
      "learning_rate": 5.6479545454545456e-05,
      "loss": 2.7494,
      "step": 251100
    },
    {
      "activations/layer0_attention_weight_max": 15.149419784545898,
      "activations/layer0_attention_weight_min": -12.283353805541992,
      "activations/layer10_attention_weight_max": 33.463497161865234,
      "activations/layer10_attention_weight_min": -32.806644439697266,
      "activations/layer11_attention_weight_max": 35.14207077026367,
      "activations/layer11_attention_weight_min": -32.947174072265625,
      "activations/layer12_attention_weight_max": 28.532320022583008,
      "activations/layer12_attention_weight_min": -27.412324905395508,
      "activations/layer13_attention_weight_max": 43.94200134277344,
      "activations/layer13_attention_weight_min": -36.93192672729492,
      "activations/layer14_attention_weight_max": 44.12754440307617,
      "activations/layer14_attention_weight_min": -37.79187774658203,
      "activations/layer15_attention_weight_max": 37.4095458984375,
      "activations/layer15_attention_weight_min": -34.7937126159668,
      "activations/layer16_attention_weight_max": 30.55375099182129,
      "activations/layer16_attention_weight_min": -26.956926345825195,
      "activations/layer17_attention_weight_max": 33.638877868652344,
      "activations/layer17_attention_weight_min": -25.89399528503418,
      "activations/layer18_attention_weight_max": 32.71457290649414,
      "activations/layer18_attention_weight_min": -24.343017578125,
      "activations/layer19_attention_weight_max": 34.35746383666992,
      "activations/layer19_attention_weight_min": -30.954425811767578,
      "activations/layer1_attention_weight_max": 16.399124145507812,
      "activations/layer1_attention_weight_min": -15.639310836791992,
      "activations/layer20_attention_weight_max": 29.603574752807617,
      "activations/layer20_attention_weight_min": -26.076169967651367,
      "activations/layer21_attention_weight_max": 27.878677368164062,
      "activations/layer21_attention_weight_min": -25.365196228027344,
      "activations/layer22_attention_weight_max": 42.16670227050781,
      "activations/layer22_attention_weight_min": -32.13380813598633,
      "activations/layer23_attention_weight_max": 32.35603332519531,
      "activations/layer23_attention_weight_min": -25.46605682373047,
      "activations/layer2_attention_weight_max": 33.01564407348633,
      "activations/layer2_attention_weight_min": -31.718769073486328,
      "activations/layer3_attention_weight_max": 96.501220703125,
      "activations/layer3_attention_weight_min": -97.5623550415039,
      "activations/layer4_attention_weight_max": 112.658203125,
      "activations/layer4_attention_weight_min": -113.86949920654297,
      "activations/layer5_attention_weight_max": 52.85470199584961,
      "activations/layer5_attention_weight_min": -60.91986846923828,
      "activations/layer6_attention_weight_max": 45.27625274658203,
      "activations/layer6_attention_weight_min": -47.53379821777344,
      "activations/layer7_attention_weight_max": 99.22962188720703,
      "activations/layer7_attention_weight_min": -94.43891906738281,
      "activations/layer8_attention_weight_max": 43.78895950317383,
      "activations/layer8_attention_weight_min": -45.003910064697266,
      "activations/layer9_attention_weight_max": 34.21780014038086,
      "activations/layer9_attention_weight_min": -33.5434684753418,
      "epoch": 14.59,
      "learning_rate": 5.646060606060606e-05,
      "loss": 2.7617,
      "step": 251150
    },
    {
      "activations/layer0_attention_weight_max": 16.192184448242188,
      "activations/layer0_attention_weight_min": -12.591577529907227,
      "activations/layer10_attention_weight_max": 33.25284194946289,
      "activations/layer10_attention_weight_min": -32.48286819458008,
      "activations/layer11_attention_weight_max": 37.02490234375,
      "activations/layer11_attention_weight_min": -34.46179962158203,
      "activations/layer12_attention_weight_max": 25.367603302001953,
      "activations/layer12_attention_weight_min": -24.25096893310547,
      "activations/layer13_attention_weight_max": 39.54780960083008,
      "activations/layer13_attention_weight_min": -35.45309066772461,
      "activations/layer14_attention_weight_max": 42.33537673950195,
      "activations/layer14_attention_weight_min": -36.81985855102539,
      "activations/layer15_attention_weight_max": 37.85415267944336,
      "activations/layer15_attention_weight_min": -35.69386291503906,
      "activations/layer16_attention_weight_max": 29.43376350402832,
      "activations/layer16_attention_weight_min": -27.355594635009766,
      "activations/layer17_attention_weight_max": 30.22987937927246,
      "activations/layer17_attention_weight_min": -26.00565528869629,
      "activations/layer18_attention_weight_max": 33.03657150268555,
      "activations/layer18_attention_weight_min": -24.623891830444336,
      "activations/layer19_attention_weight_max": 37.57590866088867,
      "activations/layer19_attention_weight_min": -30.636411666870117,
      "activations/layer1_attention_weight_max": 16.42348861694336,
      "activations/layer1_attention_weight_min": -14.832983016967773,
      "activations/layer20_attention_weight_max": 27.952924728393555,
      "activations/layer20_attention_weight_min": -24.70159149169922,
      "activations/layer21_attention_weight_max": 27.274343490600586,
      "activations/layer21_attention_weight_min": -25.27953338623047,
      "activations/layer22_attention_weight_max": 43.0242919921875,
      "activations/layer22_attention_weight_min": -33.98072052001953,
      "activations/layer23_attention_weight_max": 35.322845458984375,
      "activations/layer23_attention_weight_min": -27.0227108001709,
      "activations/layer2_attention_weight_max": 33.56087875366211,
      "activations/layer2_attention_weight_min": -34.26580047607422,
      "activations/layer3_attention_weight_max": 95.21688842773438,
      "activations/layer3_attention_weight_min": -96.7030258178711,
      "activations/layer4_attention_weight_max": 103.32984924316406,
      "activations/layer4_attention_weight_min": -108.5593032836914,
      "activations/layer5_attention_weight_max": 50.05057907104492,
      "activations/layer5_attention_weight_min": -59.899261474609375,
      "activations/layer6_attention_weight_max": 43.14735412597656,
      "activations/layer6_attention_weight_min": -45.41545486450195,
      "activations/layer7_attention_weight_max": 88.5656509399414,
      "activations/layer7_attention_weight_min": -85.63203430175781,
      "activations/layer8_attention_weight_max": 39.79480743408203,
      "activations/layer8_attention_weight_min": -40.318626403808594,
      "activations/layer9_attention_weight_max": 33.46872329711914,
      "activations/layer9_attention_weight_min": -33.0101432800293,
      "epoch": 14.6,
      "learning_rate": 5.644166666666666e-05,
      "loss": 2.766,
      "step": 251200
    },
    {
      "activations/layer0_attention_weight_max": 15.771858215332031,
      "activations/layer0_attention_weight_min": -12.447534561157227,
      "activations/layer10_attention_weight_max": 34.4981803894043,
      "activations/layer10_attention_weight_min": -32.207942962646484,
      "activations/layer11_attention_weight_max": 37.600341796875,
      "activations/layer11_attention_weight_min": -34.76152038574219,
      "activations/layer12_attention_weight_max": 29.82992172241211,
      "activations/layer12_attention_weight_min": -26.58363151550293,
      "activations/layer13_attention_weight_max": 43.117271423339844,
      "activations/layer13_attention_weight_min": -36.18305587768555,
      "activations/layer14_attention_weight_max": 45.23765563964844,
      "activations/layer14_attention_weight_min": -38.234989166259766,
      "activations/layer15_attention_weight_max": 41.146907806396484,
      "activations/layer15_attention_weight_min": -33.13481521606445,
      "activations/layer16_attention_weight_max": 33.8822135925293,
      "activations/layer16_attention_weight_min": -28.68742561340332,
      "activations/layer17_attention_weight_max": 31.04962921142578,
      "activations/layer17_attention_weight_min": -26.810510635375977,
      "activations/layer18_attention_weight_max": 32.96249771118164,
      "activations/layer18_attention_weight_min": -24.498552322387695,
      "activations/layer19_attention_weight_max": 35.27498245239258,
      "activations/layer19_attention_weight_min": -31.111494064331055,
      "activations/layer1_attention_weight_max": 16.6280460357666,
      "activations/layer1_attention_weight_min": -14.302146911621094,
      "activations/layer20_attention_weight_max": 29.9876651763916,
      "activations/layer20_attention_weight_min": -25.447269439697266,
      "activations/layer21_attention_weight_max": 28.920042037963867,
      "activations/layer21_attention_weight_min": -25.39530372619629,
      "activations/layer22_attention_weight_max": 48.120452880859375,
      "activations/layer22_attention_weight_min": -30.255126953125,
      "activations/layer23_attention_weight_max": 34.141807556152344,
      "activations/layer23_attention_weight_min": -26.260404586791992,
      "activations/layer2_attention_weight_max": 32.758094787597656,
      "activations/layer2_attention_weight_min": -30.53609275817871,
      "activations/layer3_attention_weight_max": 96.63304901123047,
      "activations/layer3_attention_weight_min": -96.72078704833984,
      "activations/layer4_attention_weight_max": 110.04084777832031,
      "activations/layer4_attention_weight_min": -110.56624603271484,
      "activations/layer5_attention_weight_max": 52.8432731628418,
      "activations/layer5_attention_weight_min": -61.45676803588867,
      "activations/layer6_attention_weight_max": 45.08384323120117,
      "activations/layer6_attention_weight_min": -50.40267562866211,
      "activations/layer7_attention_weight_max": 91.06793975830078,
      "activations/layer7_attention_weight_min": -100.81637573242188,
      "activations/layer8_attention_weight_max": 38.790714263916016,
      "activations/layer8_attention_weight_min": -41.716514587402344,
      "activations/layer9_attention_weight_max": 32.300228118896484,
      "activations/layer9_attention_weight_min": -33.565181732177734,
      "epoch": 14.6,
      "learning_rate": 5.642272727272727e-05,
      "loss": 2.7533,
      "step": 251250
    },
    {
      "activations/layer0_attention_weight_max": 16.301115036010742,
      "activations/layer0_attention_weight_min": -12.297526359558105,
      "activations/layer10_attention_weight_max": 32.1555061340332,
      "activations/layer10_attention_weight_min": -30.689523696899414,
      "activations/layer11_attention_weight_max": 31.95673179626465,
      "activations/layer11_attention_weight_min": -30.980472564697266,
      "activations/layer12_attention_weight_max": 25.621763229370117,
      "activations/layer12_attention_weight_min": -27.034568786621094,
      "activations/layer13_attention_weight_max": 39.24749755859375,
      "activations/layer13_attention_weight_min": -36.5405387878418,
      "activations/layer14_attention_weight_max": 37.682838439941406,
      "activations/layer14_attention_weight_min": -38.52864456176758,
      "activations/layer15_attention_weight_max": 34.58017349243164,
      "activations/layer15_attention_weight_min": -33.93430709838867,
      "activations/layer16_attention_weight_max": 28.694538116455078,
      "activations/layer16_attention_weight_min": -28.191669464111328,
      "activations/layer17_attention_weight_max": 30.00634765625,
      "activations/layer17_attention_weight_min": -26.58485221862793,
      "activations/layer18_attention_weight_max": 34.22690963745117,
      "activations/layer18_attention_weight_min": -27.621061325073242,
      "activations/layer19_attention_weight_max": 33.213836669921875,
      "activations/layer19_attention_weight_min": -31.243505477905273,
      "activations/layer1_attention_weight_max": 16.756465911865234,
      "activations/layer1_attention_weight_min": -14.70383358001709,
      "activations/layer20_attention_weight_max": 29.899272918701172,
      "activations/layer20_attention_weight_min": -25.766294479370117,
      "activations/layer21_attention_weight_max": 28.94817352294922,
      "activations/layer21_attention_weight_min": -24.55170249938965,
      "activations/layer22_attention_weight_max": 45.570430755615234,
      "activations/layer22_attention_weight_min": -30.48874282836914,
      "activations/layer23_attention_weight_max": 31.075544357299805,
      "activations/layer23_attention_weight_min": -27.65496063232422,
      "activations/layer2_attention_weight_max": 32.71411895751953,
      "activations/layer2_attention_weight_min": -30.731494903564453,
      "activations/layer3_attention_weight_max": 93.83837890625,
      "activations/layer3_attention_weight_min": -92.62651062011719,
      "activations/layer4_attention_weight_max": 105.57637786865234,
      "activations/layer4_attention_weight_min": -111.35558319091797,
      "activations/layer5_attention_weight_max": 52.63134002685547,
      "activations/layer5_attention_weight_min": -61.168121337890625,
      "activations/layer6_attention_weight_max": 41.76011657714844,
      "activations/layer6_attention_weight_min": -43.46272659301758,
      "activations/layer7_attention_weight_max": 91.90422058105469,
      "activations/layer7_attention_weight_min": -86.95323181152344,
      "activations/layer8_attention_weight_max": 39.360355377197266,
      "activations/layer8_attention_weight_min": -40.4168586730957,
      "activations/layer9_attention_weight_max": 31.057706832885742,
      "activations/layer9_attention_weight_min": -31.036727905273438,
      "epoch": 14.6,
      "learning_rate": 5.6403787878787875e-05,
      "loss": 2.7496,
      "step": 251300
    },
    {
      "activations/layer0_attention_weight_max": 15.979679107666016,
      "activations/layer0_attention_weight_min": -12.880788803100586,
      "activations/layer10_attention_weight_max": 35.36473846435547,
      "activations/layer10_attention_weight_min": -31.57879638671875,
      "activations/layer11_attention_weight_max": 33.55954360961914,
      "activations/layer11_attention_weight_min": -32.937522888183594,
      "activations/layer12_attention_weight_max": 25.3172607421875,
      "activations/layer12_attention_weight_min": -26.927658081054688,
      "activations/layer13_attention_weight_max": 37.111610412597656,
      "activations/layer13_attention_weight_min": -36.44210433959961,
      "activations/layer14_attention_weight_max": 41.72305679321289,
      "activations/layer14_attention_weight_min": -39.18433380126953,
      "activations/layer15_attention_weight_max": 34.483543395996094,
      "activations/layer15_attention_weight_min": -33.84355163574219,
      "activations/layer16_attention_weight_max": 30.618005752563477,
      "activations/layer16_attention_weight_min": -29.05501365661621,
      "activations/layer17_attention_weight_max": 29.49036979675293,
      "activations/layer17_attention_weight_min": -26.792980194091797,
      "activations/layer18_attention_weight_max": 31.920936584472656,
      "activations/layer18_attention_weight_min": -24.012022018432617,
      "activations/layer19_attention_weight_max": 34.61094284057617,
      "activations/layer19_attention_weight_min": -31.966970443725586,
      "activations/layer1_attention_weight_max": 17.970714569091797,
      "activations/layer1_attention_weight_min": -15.71032428741455,
      "activations/layer20_attention_weight_max": 27.850452423095703,
      "activations/layer20_attention_weight_min": -25.876148223876953,
      "activations/layer21_attention_weight_max": 27.63176727294922,
      "activations/layer21_attention_weight_min": -25.419145584106445,
      "activations/layer22_attention_weight_max": 38.00947952270508,
      "activations/layer22_attention_weight_min": -31.646821975708008,
      "activations/layer23_attention_weight_max": 32.929901123046875,
      "activations/layer23_attention_weight_min": -25.35614585876465,
      "activations/layer2_attention_weight_max": 33.10619354248047,
      "activations/layer2_attention_weight_min": -32.22562026977539,
      "activations/layer3_attention_weight_max": 95.90045928955078,
      "activations/layer3_attention_weight_min": -96.63008117675781,
      "activations/layer4_attention_weight_max": 110.0745620727539,
      "activations/layer4_attention_weight_min": -112.17689514160156,
      "activations/layer5_attention_weight_max": 54.18685531616211,
      "activations/layer5_attention_weight_min": -65.01979064941406,
      "activations/layer6_attention_weight_max": 43.259281158447266,
      "activations/layer6_attention_weight_min": -48.92955017089844,
      "activations/layer7_attention_weight_max": 88.4587631225586,
      "activations/layer7_attention_weight_min": -93.38809967041016,
      "activations/layer8_attention_weight_max": 38.35182571411133,
      "activations/layer8_attention_weight_min": -40.29512405395508,
      "activations/layer9_attention_weight_max": 33.285396575927734,
      "activations/layer9_attention_weight_min": -33.49190902709961,
      "epoch": 14.6,
      "learning_rate": 5.638484848484848e-05,
      "loss": 2.7561,
      "step": 251350
    },
    {
      "activations/layer0_attention_weight_max": 16.54316520690918,
      "activations/layer0_attention_weight_min": -12.993964195251465,
      "activations/layer10_attention_weight_max": 36.13063049316406,
      "activations/layer10_attention_weight_min": -31.992361068725586,
      "activations/layer11_attention_weight_max": 34.773624420166016,
      "activations/layer11_attention_weight_min": -33.43411636352539,
      "activations/layer12_attention_weight_max": 29.690580368041992,
      "activations/layer12_attention_weight_min": -26.669137954711914,
      "activations/layer13_attention_weight_max": 47.139549255371094,
      "activations/layer13_attention_weight_min": -35.282649993896484,
      "activations/layer14_attention_weight_max": 49.76608657836914,
      "activations/layer14_attention_weight_min": -38.313751220703125,
      "activations/layer15_attention_weight_max": 44.88201141357422,
      "activations/layer15_attention_weight_min": -33.896270751953125,
      "activations/layer16_attention_weight_max": 33.169715881347656,
      "activations/layer16_attention_weight_min": -28.079246520996094,
      "activations/layer17_attention_weight_max": 33.48424530029297,
      "activations/layer17_attention_weight_min": -27.59477424621582,
      "activations/layer18_attention_weight_max": 33.14653778076172,
      "activations/layer18_attention_weight_min": -25.22587776184082,
      "activations/layer19_attention_weight_max": 32.925846099853516,
      "activations/layer19_attention_weight_min": -30.24098777770996,
      "activations/layer1_attention_weight_max": 16.55158042907715,
      "activations/layer1_attention_weight_min": -14.216970443725586,
      "activations/layer20_attention_weight_max": 28.82552146911621,
      "activations/layer20_attention_weight_min": -25.562589645385742,
      "activations/layer21_attention_weight_max": 30.83910369873047,
      "activations/layer21_attention_weight_min": -25.882165908813477,
      "activations/layer22_attention_weight_max": 39.22254180908203,
      "activations/layer22_attention_weight_min": -30.608184814453125,
      "activations/layer23_attention_weight_max": 32.67826461791992,
      "activations/layer23_attention_weight_min": -25.023414611816406,
      "activations/layer2_attention_weight_max": 34.43595886230469,
      "activations/layer2_attention_weight_min": -32.66142272949219,
      "activations/layer3_attention_weight_max": 98.80331420898438,
      "activations/layer3_attention_weight_min": -97.81452941894531,
      "activations/layer4_attention_weight_max": 111.150146484375,
      "activations/layer4_attention_weight_min": -109.6297378540039,
      "activations/layer5_attention_weight_max": 52.515865325927734,
      "activations/layer5_attention_weight_min": -61.33285140991211,
      "activations/layer6_attention_weight_max": 44.99253463745117,
      "activations/layer6_attention_weight_min": -45.064395904541016,
      "activations/layer7_attention_weight_max": 89.10012817382812,
      "activations/layer7_attention_weight_min": -86.71489715576172,
      "activations/layer8_attention_weight_max": 41.02589416503906,
      "activations/layer8_attention_weight_min": -41.9772834777832,
      "activations/layer9_attention_weight_max": 34.382083892822266,
      "activations/layer9_attention_weight_min": -32.58779525756836,
      "epoch": 14.61,
      "learning_rate": 5.6365909090909085e-05,
      "loss": 2.7483,
      "step": 251400
    },
    {
      "activations/layer0_attention_weight_max": 15.731095314025879,
      "activations/layer0_attention_weight_min": -12.589366912841797,
      "activations/layer10_attention_weight_max": 33.76124954223633,
      "activations/layer10_attention_weight_min": -33.61128616333008,
      "activations/layer11_attention_weight_max": 34.503074645996094,
      "activations/layer11_attention_weight_min": -34.48078918457031,
      "activations/layer12_attention_weight_max": 26.19493865966797,
      "activations/layer12_attention_weight_min": -27.093082427978516,
      "activations/layer13_attention_weight_max": 38.68593215942383,
      "activations/layer13_attention_weight_min": -38.84222412109375,
      "activations/layer14_attention_weight_max": 40.59748458862305,
      "activations/layer14_attention_weight_min": -38.43840408325195,
      "activations/layer15_attention_weight_max": 35.54086685180664,
      "activations/layer15_attention_weight_min": -34.5142822265625,
      "activations/layer16_attention_weight_max": 27.965578079223633,
      "activations/layer16_attention_weight_min": -27.70683479309082,
      "activations/layer17_attention_weight_max": 27.964427947998047,
      "activations/layer17_attention_weight_min": -25.83464241027832,
      "activations/layer18_attention_weight_max": 31.615144729614258,
      "activations/layer18_attention_weight_min": -23.110193252563477,
      "activations/layer19_attention_weight_max": 32.32469177246094,
      "activations/layer19_attention_weight_min": -30.837574005126953,
      "activations/layer1_attention_weight_max": 16.00991439819336,
      "activations/layer1_attention_weight_min": -14.67102336883545,
      "activations/layer20_attention_weight_max": 27.050098419189453,
      "activations/layer20_attention_weight_min": -25.349164962768555,
      "activations/layer21_attention_weight_max": 26.176755905151367,
      "activations/layer21_attention_weight_min": -25.997854232788086,
      "activations/layer22_attention_weight_max": 42.04069900512695,
      "activations/layer22_attention_weight_min": -32.43510818481445,
      "activations/layer23_attention_weight_max": 30.11848258972168,
      "activations/layer23_attention_weight_min": -25.234899520874023,
      "activations/layer2_attention_weight_max": 33.2708740234375,
      "activations/layer2_attention_weight_min": -33.118316650390625,
      "activations/layer3_attention_weight_max": 95.66352081298828,
      "activations/layer3_attention_weight_min": -99.84874725341797,
      "activations/layer4_attention_weight_max": 114.4166259765625,
      "activations/layer4_attention_weight_min": -110.28973388671875,
      "activations/layer5_attention_weight_max": 53.494503021240234,
      "activations/layer5_attention_weight_min": -63.49445343017578,
      "activations/layer6_attention_weight_max": 43.08867263793945,
      "activations/layer6_attention_weight_min": -45.94736099243164,
      "activations/layer7_attention_weight_max": 87.52118682861328,
      "activations/layer7_attention_weight_min": -94.37836456298828,
      "activations/layer8_attention_weight_max": 40.540550231933594,
      "activations/layer8_attention_weight_min": -44.43130874633789,
      "activations/layer9_attention_weight_max": 33.45548629760742,
      "activations/layer9_attention_weight_min": -33.59236526489258,
      "epoch": 14.61,
      "learning_rate": 5.634696969696969e-05,
      "loss": 2.7402,
      "step": 251450
    },
    {
      "activations/layer0_attention_weight_max": 15.218378067016602,
      "activations/layer0_attention_weight_min": -12.611830711364746,
      "activations/layer10_attention_weight_max": 31.52632713317871,
      "activations/layer10_attention_weight_min": -32.03459930419922,
      "activations/layer11_attention_weight_max": 32.349037170410156,
      "activations/layer11_attention_weight_min": -31.16274070739746,
      "activations/layer12_attention_weight_max": 24.917587280273438,
      "activations/layer12_attention_weight_min": -25.218788146972656,
      "activations/layer13_attention_weight_max": 37.505733489990234,
      "activations/layer13_attention_weight_min": -34.7308235168457,
      "activations/layer14_attention_weight_max": 50.58601760864258,
      "activations/layer14_attention_weight_min": -37.61091995239258,
      "activations/layer15_attention_weight_max": 37.94120788574219,
      "activations/layer15_attention_weight_min": -35.12092971801758,
      "activations/layer16_attention_weight_max": 33.569419860839844,
      "activations/layer16_attention_weight_min": -28.720930099487305,
      "activations/layer17_attention_weight_max": 29.71539306640625,
      "activations/layer17_attention_weight_min": -25.71599769592285,
      "activations/layer18_attention_weight_max": 30.429019927978516,
      "activations/layer18_attention_weight_min": -26.984420776367188,
      "activations/layer19_attention_weight_max": 35.09765625,
      "activations/layer19_attention_weight_min": -32.48380661010742,
      "activations/layer1_attention_weight_max": 16.276134490966797,
      "activations/layer1_attention_weight_min": -15.501226425170898,
      "activations/layer20_attention_weight_max": 29.465063095092773,
      "activations/layer20_attention_weight_min": -25.138816833496094,
      "activations/layer21_attention_weight_max": 30.59589195251465,
      "activations/layer21_attention_weight_min": -26.076696395874023,
      "activations/layer22_attention_weight_max": 49.284488677978516,
      "activations/layer22_attention_weight_min": -31.736162185668945,
      "activations/layer23_attention_weight_max": 36.383453369140625,
      "activations/layer23_attention_weight_min": -26.871885299682617,
      "activations/layer2_attention_weight_max": 32.58515930175781,
      "activations/layer2_attention_weight_min": -32.2987060546875,
      "activations/layer3_attention_weight_max": 95.6823959350586,
      "activations/layer3_attention_weight_min": -93.44880676269531,
      "activations/layer4_attention_weight_max": 105.4876708984375,
      "activations/layer4_attention_weight_min": -106.59942626953125,
      "activations/layer5_attention_weight_max": 51.60533142089844,
      "activations/layer5_attention_weight_min": -59.00313949584961,
      "activations/layer6_attention_weight_max": 41.803489685058594,
      "activations/layer6_attention_weight_min": -44.071773529052734,
      "activations/layer7_attention_weight_max": 85.44962310791016,
      "activations/layer7_attention_weight_min": -85.17573547363281,
      "activations/layer8_attention_weight_max": 38.21755599975586,
      "activations/layer8_attention_weight_min": -39.22478485107422,
      "activations/layer9_attention_weight_max": 30.674823760986328,
      "activations/layer9_attention_weight_min": -31.824243545532227,
      "epoch": 14.61,
      "learning_rate": 5.63280303030303e-05,
      "loss": 2.759,
      "step": 251500
    },
    {
      "activations/layer0_attention_weight_max": 16.33056640625,
      "activations/layer0_attention_weight_min": -12.907594680786133,
      "activations/layer10_attention_weight_max": 31.28597640991211,
      "activations/layer10_attention_weight_min": -31.639657974243164,
      "activations/layer11_attention_weight_max": 31.084304809570312,
      "activations/layer11_attention_weight_min": -31.99689292907715,
      "activations/layer12_attention_weight_max": 23.308244705200195,
      "activations/layer12_attention_weight_min": -25.362014770507812,
      "activations/layer13_attention_weight_max": 36.32760238647461,
      "activations/layer13_attention_weight_min": -35.07798385620117,
      "activations/layer14_attention_weight_max": 41.54535675048828,
      "activations/layer14_attention_weight_min": -37.50469207763672,
      "activations/layer15_attention_weight_max": 34.460872650146484,
      "activations/layer15_attention_weight_min": -32.830352783203125,
      "activations/layer16_attention_weight_max": 33.54680252075195,
      "activations/layer16_attention_weight_min": -28.04532241821289,
      "activations/layer17_attention_weight_max": 27.924150466918945,
      "activations/layer17_attention_weight_min": -25.23757553100586,
      "activations/layer18_attention_weight_max": 30.764938354492188,
      "activations/layer18_attention_weight_min": -26.61469841003418,
      "activations/layer19_attention_weight_max": 31.010305404663086,
      "activations/layer19_attention_weight_min": -31.898679733276367,
      "activations/layer1_attention_weight_max": 16.222213745117188,
      "activations/layer1_attention_weight_min": -15.263886451721191,
      "activations/layer20_attention_weight_max": 29.00067710876465,
      "activations/layer20_attention_weight_min": -25.567537307739258,
      "activations/layer21_attention_weight_max": 26.15643310546875,
      "activations/layer21_attention_weight_min": -25.320507049560547,
      "activations/layer22_attention_weight_max": 43.056793212890625,
      "activations/layer22_attention_weight_min": -32.100059509277344,
      "activations/layer23_attention_weight_max": 35.916255950927734,
      "activations/layer23_attention_weight_min": -25.202756881713867,
      "activations/layer2_attention_weight_max": 34.99274444580078,
      "activations/layer2_attention_weight_min": -32.214393615722656,
      "activations/layer3_attention_weight_max": 95.3673095703125,
      "activations/layer3_attention_weight_min": -93.23572540283203,
      "activations/layer4_attention_weight_max": 108.92166900634766,
      "activations/layer4_attention_weight_min": -107.07769775390625,
      "activations/layer5_attention_weight_max": 53.4156494140625,
      "activations/layer5_attention_weight_min": -61.25794219970703,
      "activations/layer6_attention_weight_max": 43.62953186035156,
      "activations/layer6_attention_weight_min": -46.59026336669922,
      "activations/layer7_attention_weight_max": 86.41510772705078,
      "activations/layer7_attention_weight_min": -92.76318359375,
      "activations/layer8_attention_weight_max": 38.874000549316406,
      "activations/layer8_attention_weight_min": -39.095149993896484,
      "activations/layer9_attention_weight_max": 29.593917846679688,
      "activations/layer9_attention_weight_min": -31.527097702026367,
      "epoch": 14.62,
      "learning_rate": 5.63090909090909e-05,
      "loss": 2.7562,
      "step": 251550
    },
    {
      "activations/layer0_attention_weight_max": 16.101116180419922,
      "activations/layer0_attention_weight_min": -12.679015159606934,
      "activations/layer10_attention_weight_max": 34.5728645324707,
      "activations/layer10_attention_weight_min": -34.21584701538086,
      "activations/layer11_attention_weight_max": 33.68431091308594,
      "activations/layer11_attention_weight_min": -34.29792404174805,
      "activations/layer12_attention_weight_max": 25.279699325561523,
      "activations/layer12_attention_weight_min": -24.955734252929688,
      "activations/layer13_attention_weight_max": 37.86127853393555,
      "activations/layer13_attention_weight_min": -35.93628692626953,
      "activations/layer14_attention_weight_max": 40.528751373291016,
      "activations/layer14_attention_weight_min": -36.171234130859375,
      "activations/layer15_attention_weight_max": 34.11149215698242,
      "activations/layer15_attention_weight_min": -32.794464111328125,
      "activations/layer16_attention_weight_max": 29.88855743408203,
      "activations/layer16_attention_weight_min": -28.636913299560547,
      "activations/layer17_attention_weight_max": 31.109418869018555,
      "activations/layer17_attention_weight_min": -25.857629776000977,
      "activations/layer18_attention_weight_max": 35.286094665527344,
      "activations/layer18_attention_weight_min": -24.849227905273438,
      "activations/layer19_attention_weight_max": 39.21572494506836,
      "activations/layer19_attention_weight_min": -31.655359268188477,
      "activations/layer1_attention_weight_max": 17.0324649810791,
      "activations/layer1_attention_weight_min": -15.751157760620117,
      "activations/layer20_attention_weight_max": 28.998451232910156,
      "activations/layer20_attention_weight_min": -26.622785568237305,
      "activations/layer21_attention_weight_max": 28.835613250732422,
      "activations/layer21_attention_weight_min": -25.549423217773438,
      "activations/layer22_attention_weight_max": 46.5960693359375,
      "activations/layer22_attention_weight_min": -29.65638542175293,
      "activations/layer23_attention_weight_max": 34.43578338623047,
      "activations/layer23_attention_weight_min": -24.42776870727539,
      "activations/layer2_attention_weight_max": 34.820030212402344,
      "activations/layer2_attention_weight_min": -30.926427841186523,
      "activations/layer3_attention_weight_max": 92.51142120361328,
      "activations/layer3_attention_weight_min": -92.0229721069336,
      "activations/layer4_attention_weight_max": 112.12884521484375,
      "activations/layer4_attention_weight_min": -112.76216125488281,
      "activations/layer5_attention_weight_max": 50.41190719604492,
      "activations/layer5_attention_weight_min": -64.32953643798828,
      "activations/layer6_attention_weight_max": 43.98396682739258,
      "activations/layer6_attention_weight_min": -46.336238861083984,
      "activations/layer7_attention_weight_max": 91.90229034423828,
      "activations/layer7_attention_weight_min": -93.21845245361328,
      "activations/layer8_attention_weight_max": 41.58348846435547,
      "activations/layer8_attention_weight_min": -43.34524154663086,
      "activations/layer9_attention_weight_max": 33.37035369873047,
      "activations/layer9_attention_weight_min": -34.56159591674805,
      "epoch": 14.62,
      "learning_rate": 5.6290530303030294e-05,
      "loss": 2.7578,
      "step": 251600
    },
    {
      "activations/layer0_attention_weight_max": 16.731399536132812,
      "activations/layer0_attention_weight_min": -12.380603790283203,
      "activations/layer10_attention_weight_max": 33.92625045776367,
      "activations/layer10_attention_weight_min": -31.878921508789062,
      "activations/layer11_attention_weight_max": 32.949195861816406,
      "activations/layer11_attention_weight_min": -34.47516632080078,
      "activations/layer12_attention_weight_max": 25.593358993530273,
      "activations/layer12_attention_weight_min": -25.669544219970703,
      "activations/layer13_attention_weight_max": 39.90808868408203,
      "activations/layer13_attention_weight_min": -35.95204162597656,
      "activations/layer14_attention_weight_max": 44.3156852722168,
      "activations/layer14_attention_weight_min": -38.7525749206543,
      "activations/layer15_attention_weight_max": 42.1503791809082,
      "activations/layer15_attention_weight_min": -34.85841369628906,
      "activations/layer16_attention_weight_max": 34.42819595336914,
      "activations/layer16_attention_weight_min": -28.18022918701172,
      "activations/layer17_attention_weight_max": 32.157554626464844,
      "activations/layer17_attention_weight_min": -26.759401321411133,
      "activations/layer18_attention_weight_max": 38.02860641479492,
      "activations/layer18_attention_weight_min": -24.937076568603516,
      "activations/layer19_attention_weight_max": 42.84395980834961,
      "activations/layer19_attention_weight_min": -31.165679931640625,
      "activations/layer1_attention_weight_max": 16.312034606933594,
      "activations/layer1_attention_weight_min": -13.92324447631836,
      "activations/layer20_attention_weight_max": 30.383304595947266,
      "activations/layer20_attention_weight_min": -26.92236328125,
      "activations/layer21_attention_weight_max": 30.312923431396484,
      "activations/layer21_attention_weight_min": -26.16908836364746,
      "activations/layer22_attention_weight_max": 43.352081298828125,
      "activations/layer22_attention_weight_min": -32.684669494628906,
      "activations/layer23_attention_weight_max": 43.85089111328125,
      "activations/layer23_attention_weight_min": -26.489744186401367,
      "activations/layer2_attention_weight_max": 32.64361572265625,
      "activations/layer2_attention_weight_min": -30.484806060791016,
      "activations/layer3_attention_weight_max": 95.92601013183594,
      "activations/layer3_attention_weight_min": -92.86839294433594,
      "activations/layer4_attention_weight_max": 111.24922180175781,
      "activations/layer4_attention_weight_min": -111.10099029541016,
      "activations/layer5_attention_weight_max": 52.700103759765625,
      "activations/layer5_attention_weight_min": -63.337947845458984,
      "activations/layer6_attention_weight_max": 43.01642990112305,
      "activations/layer6_attention_weight_min": -47.228355407714844,
      "activations/layer7_attention_weight_max": 93.31085205078125,
      "activations/layer7_attention_weight_min": -95.15177917480469,
      "activations/layer8_attention_weight_max": 39.912845611572266,
      "activations/layer8_attention_weight_min": -40.67476272583008,
      "activations/layer9_attention_weight_max": 33.58106994628906,
      "activations/layer9_attention_weight_min": -31.894189834594727,
      "epoch": 14.62,
      "learning_rate": 5.627159090909091e-05,
      "loss": 2.7422,
      "step": 251650
    },
    {
      "activations/layer0_attention_weight_max": 15.736247062683105,
      "activations/layer0_attention_weight_min": -12.47394847869873,
      "activations/layer10_attention_weight_max": 31.319957733154297,
      "activations/layer10_attention_weight_min": -31.92340087890625,
      "activations/layer11_attention_weight_max": 31.113645553588867,
      "activations/layer11_attention_weight_min": -33.500370025634766,
      "activations/layer12_attention_weight_max": 24.033483505249023,
      "activations/layer12_attention_weight_min": -26.498592376708984,
      "activations/layer13_attention_weight_max": 36.78014373779297,
      "activations/layer13_attention_weight_min": -35.51203536987305,
      "activations/layer14_attention_weight_max": 41.35240173339844,
      "activations/layer14_attention_weight_min": -37.400474548339844,
      "activations/layer15_attention_weight_max": 37.775630950927734,
      "activations/layer15_attention_weight_min": -35.44228744506836,
      "activations/layer16_attention_weight_max": 30.87567138671875,
      "activations/layer16_attention_weight_min": -29.924325942993164,
      "activations/layer17_attention_weight_max": 30.547719955444336,
      "activations/layer17_attention_weight_min": -25.353166580200195,
      "activations/layer18_attention_weight_max": 33.94968795776367,
      "activations/layer18_attention_weight_min": -23.25014877319336,
      "activations/layer19_attention_weight_max": 35.87061309814453,
      "activations/layer19_attention_weight_min": -31.020227432250977,
      "activations/layer1_attention_weight_max": 17.17144203186035,
      "activations/layer1_attention_weight_min": -14.779143333435059,
      "activations/layer20_attention_weight_max": 33.51173782348633,
      "activations/layer20_attention_weight_min": -25.86410140991211,
      "activations/layer21_attention_weight_max": 31.49784278869629,
      "activations/layer21_attention_weight_min": -25.129972457885742,
      "activations/layer22_attention_weight_max": 47.80699157714844,
      "activations/layer22_attention_weight_min": -29.906091690063477,
      "activations/layer23_attention_weight_max": 38.155941009521484,
      "activations/layer23_attention_weight_min": -25.968677520751953,
      "activations/layer2_attention_weight_max": 32.268226623535156,
      "activations/layer2_attention_weight_min": -31.406652450561523,
      "activations/layer3_attention_weight_max": 94.9901123046875,
      "activations/layer3_attention_weight_min": -95.96772766113281,
      "activations/layer4_attention_weight_max": 110.1358413696289,
      "activations/layer4_attention_weight_min": -107.39339447021484,
      "activations/layer5_attention_weight_max": 52.048301696777344,
      "activations/layer5_attention_weight_min": -63.6727294921875,
      "activations/layer6_attention_weight_max": 44.22478103637695,
      "activations/layer6_attention_weight_min": -46.30847930908203,
      "activations/layer7_attention_weight_max": 91.18303680419922,
      "activations/layer7_attention_weight_min": -88.00122833251953,
      "activations/layer8_attention_weight_max": 39.7241096496582,
      "activations/layer8_attention_weight_min": -41.90819549560547,
      "activations/layer9_attention_weight_max": 30.68336296081543,
      "activations/layer9_attention_weight_min": -32.813602447509766,
      "epoch": 14.63,
      "learning_rate": 5.625265151515151e-05,
      "loss": 2.7588,
      "step": 251700
    },
    {
      "activations/layer0_attention_weight_max": 16.071285247802734,
      "activations/layer0_attention_weight_min": -12.69488525390625,
      "activations/layer10_attention_weight_max": 33.32268524169922,
      "activations/layer10_attention_weight_min": -33.069602966308594,
      "activations/layer11_attention_weight_max": 32.91362380981445,
      "activations/layer11_attention_weight_min": -32.446861267089844,
      "activations/layer12_attention_weight_max": 25.71013069152832,
      "activations/layer12_attention_weight_min": -25.60911750793457,
      "activations/layer13_attention_weight_max": 37.84950637817383,
      "activations/layer13_attention_weight_min": -34.90581130981445,
      "activations/layer14_attention_weight_max": 40.278316497802734,
      "activations/layer14_attention_weight_min": -37.315555572509766,
      "activations/layer15_attention_weight_max": 38.30099868774414,
      "activations/layer15_attention_weight_min": -33.31934356689453,
      "activations/layer16_attention_weight_max": 32.33095169067383,
      "activations/layer16_attention_weight_min": -30.263940811157227,
      "activations/layer17_attention_weight_max": 30.3474063873291,
      "activations/layer17_attention_weight_min": -25.169706344604492,
      "activations/layer18_attention_weight_max": 33.92770004272461,
      "activations/layer18_attention_weight_min": -23.951204299926758,
      "activations/layer19_attention_weight_max": 39.27546310424805,
      "activations/layer19_attention_weight_min": -31.244047164916992,
      "activations/layer1_attention_weight_max": 16.483028411865234,
      "activations/layer1_attention_weight_min": -14.702742576599121,
      "activations/layer20_attention_weight_max": 34.10451889038086,
      "activations/layer20_attention_weight_min": -25.23131561279297,
      "activations/layer21_attention_weight_max": 34.188453674316406,
      "activations/layer21_attention_weight_min": -24.93517303466797,
      "activations/layer22_attention_weight_max": 54.11149215698242,
      "activations/layer22_attention_weight_min": -30.69956398010254,
      "activations/layer23_attention_weight_max": 37.12024688720703,
      "activations/layer23_attention_weight_min": -26.29932403564453,
      "activations/layer2_attention_weight_max": 32.5455322265625,
      "activations/layer2_attention_weight_min": -32.16489791870117,
      "activations/layer3_attention_weight_max": 95.73779296875,
      "activations/layer3_attention_weight_min": -97.39598846435547,
      "activations/layer4_attention_weight_max": 113.25753021240234,
      "activations/layer4_attention_weight_min": -108.80228424072266,
      "activations/layer5_attention_weight_max": 51.650142669677734,
      "activations/layer5_attention_weight_min": -61.43790817260742,
      "activations/layer6_attention_weight_max": 43.02193069458008,
      "activations/layer6_attention_weight_min": -45.20619201660156,
      "activations/layer7_attention_weight_max": 88.80604553222656,
      "activations/layer7_attention_weight_min": -92.6776123046875,
      "activations/layer8_attention_weight_max": 40.13887405395508,
      "activations/layer8_attention_weight_min": -42.27462387084961,
      "activations/layer9_attention_weight_max": 30.98703384399414,
      "activations/layer9_attention_weight_min": -32.72902297973633,
      "epoch": 14.63,
      "learning_rate": 5.623371212121212e-05,
      "loss": 2.7676,
      "step": 251750
    },
    {
      "activations/layer0_attention_weight_max": 15.713415145874023,
      "activations/layer0_attention_weight_min": -13.32976245880127,
      "activations/layer10_attention_weight_max": 32.10490798950195,
      "activations/layer10_attention_weight_min": -34.0084114074707,
      "activations/layer11_attention_weight_max": 30.25570297241211,
      "activations/layer11_attention_weight_min": -32.200252532958984,
      "activations/layer12_attention_weight_max": 24.201574325561523,
      "activations/layer12_attention_weight_min": -24.824827194213867,
      "activations/layer13_attention_weight_max": 36.17314529418945,
      "activations/layer13_attention_weight_min": -33.00218963623047,
      "activations/layer14_attention_weight_max": 37.47209930419922,
      "activations/layer14_attention_weight_min": -35.91080093383789,
      "activations/layer15_attention_weight_max": 34.2172737121582,
      "activations/layer15_attention_weight_min": -34.37728500366211,
      "activations/layer16_attention_weight_max": 29.348148345947266,
      "activations/layer16_attention_weight_min": -29.01103401184082,
      "activations/layer17_attention_weight_max": 28.83169937133789,
      "activations/layer17_attention_weight_min": -25.841472625732422,
      "activations/layer18_attention_weight_max": 29.918624877929688,
      "activations/layer18_attention_weight_min": -23.1962833404541,
      "activations/layer19_attention_weight_max": 33.43927001953125,
      "activations/layer19_attention_weight_min": -32.3199577331543,
      "activations/layer1_attention_weight_max": 16.58663558959961,
      "activations/layer1_attention_weight_min": -13.924564361572266,
      "activations/layer20_attention_weight_max": 25.23906898498535,
      "activations/layer20_attention_weight_min": -25.44774055480957,
      "activations/layer21_attention_weight_max": 26.541179656982422,
      "activations/layer21_attention_weight_min": -24.343400955200195,
      "activations/layer22_attention_weight_max": 38.22928237915039,
      "activations/layer22_attention_weight_min": -29.174497604370117,
      "activations/layer23_attention_weight_max": 27.862483978271484,
      "activations/layer23_attention_weight_min": -25.623098373413086,
      "activations/layer2_attention_weight_max": 33.367820739746094,
      "activations/layer2_attention_weight_min": -31.340999603271484,
      "activations/layer3_attention_weight_max": 94.6751708984375,
      "activations/layer3_attention_weight_min": -96.54602813720703,
      "activations/layer4_attention_weight_max": 110.0188980102539,
      "activations/layer4_attention_weight_min": -113.62945556640625,
      "activations/layer5_attention_weight_max": 51.2200927734375,
      "activations/layer5_attention_weight_min": -63.180908203125,
      "activations/layer6_attention_weight_max": 42.983055114746094,
      "activations/layer6_attention_weight_min": -46.7288932800293,
      "activations/layer7_attention_weight_max": 88.69218444824219,
      "activations/layer7_attention_weight_min": -88.30154418945312,
      "activations/layer8_attention_weight_max": 37.852294921875,
      "activations/layer8_attention_weight_min": -41.88960647583008,
      "activations/layer9_attention_weight_max": 31.646488189697266,
      "activations/layer9_attention_weight_min": -33.4650764465332,
      "epoch": 14.63,
      "learning_rate": 5.621477272727272e-05,
      "loss": 2.758,
      "step": 251800
    },
    {
      "activations/layer0_attention_weight_max": 15.62677001953125,
      "activations/layer0_attention_weight_min": -12.276932716369629,
      "activations/layer10_attention_weight_max": 33.75274658203125,
      "activations/layer10_attention_weight_min": -34.96027374267578,
      "activations/layer11_attention_weight_max": 36.9626350402832,
      "activations/layer11_attention_weight_min": -34.929351806640625,
      "activations/layer12_attention_weight_max": 28.32878303527832,
      "activations/layer12_attention_weight_min": -28.50446891784668,
      "activations/layer13_attention_weight_max": 40.98183822631836,
      "activations/layer13_attention_weight_min": -39.804840087890625,
      "activations/layer14_attention_weight_max": 49.63325119018555,
      "activations/layer14_attention_weight_min": -45.04338455200195,
      "activations/layer15_attention_weight_max": 39.985958099365234,
      "activations/layer15_attention_weight_min": -35.404022216796875,
      "activations/layer16_attention_weight_max": 33.930259704589844,
      "activations/layer16_attention_weight_min": -27.871627807617188,
      "activations/layer17_attention_weight_max": 36.31938934326172,
      "activations/layer17_attention_weight_min": -26.694679260253906,
      "activations/layer18_attention_weight_max": 36.50551986694336,
      "activations/layer18_attention_weight_min": -24.549827575683594,
      "activations/layer19_attention_weight_max": 44.99002456665039,
      "activations/layer19_attention_weight_min": -32.77537155151367,
      "activations/layer1_attention_weight_max": 16.316810607910156,
      "activations/layer1_attention_weight_min": -13.708785057067871,
      "activations/layer20_attention_weight_max": 36.87235641479492,
      "activations/layer20_attention_weight_min": -25.921480178833008,
      "activations/layer21_attention_weight_max": 34.793434143066406,
      "activations/layer21_attention_weight_min": -24.04839324951172,
      "activations/layer22_attention_weight_max": 55.24591064453125,
      "activations/layer22_attention_weight_min": -32.89168930053711,
      "activations/layer23_attention_weight_max": 39.582393646240234,
      "activations/layer23_attention_weight_min": -26.22408103942871,
      "activations/layer2_attention_weight_max": 34.66722106933594,
      "activations/layer2_attention_weight_min": -32.436920166015625,
      "activations/layer3_attention_weight_max": 99.14196014404297,
      "activations/layer3_attention_weight_min": -101.91788482666016,
      "activations/layer4_attention_weight_max": 112.653564453125,
      "activations/layer4_attention_weight_min": -113.5746078491211,
      "activations/layer5_attention_weight_max": 56.33924102783203,
      "activations/layer5_attention_weight_min": -65.66555786132812,
      "activations/layer6_attention_weight_max": 46.06886291503906,
      "activations/layer6_attention_weight_min": -52.266883850097656,
      "activations/layer7_attention_weight_max": 94.23683166503906,
      "activations/layer7_attention_weight_min": -96.20343017578125,
      "activations/layer8_attention_weight_max": 42.075660705566406,
      "activations/layer8_attention_weight_min": -45.541080474853516,
      "activations/layer9_attention_weight_max": 33.66474151611328,
      "activations/layer9_attention_weight_min": -36.24114227294922,
      "epoch": 14.63,
      "learning_rate": 5.619583333333333e-05,
      "loss": 2.7551,
      "step": 251850
    },
    {
      "activations/layer0_attention_weight_max": 15.543380737304688,
      "activations/layer0_attention_weight_min": -12.838449478149414,
      "activations/layer10_attention_weight_max": 31.32522201538086,
      "activations/layer10_attention_weight_min": -31.87527847290039,
      "activations/layer11_attention_weight_max": 32.872779846191406,
      "activations/layer11_attention_weight_min": -33.400943756103516,
      "activations/layer12_attention_weight_max": 28.161535263061523,
      "activations/layer12_attention_weight_min": -25.45295524597168,
      "activations/layer13_attention_weight_max": 41.902244567871094,
      "activations/layer13_attention_weight_min": -36.018306732177734,
      "activations/layer14_attention_weight_max": 49.14191818237305,
      "activations/layer14_attention_weight_min": -41.24837875366211,
      "activations/layer15_attention_weight_max": 42.901702880859375,
      "activations/layer15_attention_weight_min": -38.06416320800781,
      "activations/layer16_attention_weight_max": 35.198001861572266,
      "activations/layer16_attention_weight_min": -30.535188674926758,
      "activations/layer17_attention_weight_max": 35.560813903808594,
      "activations/layer17_attention_weight_min": -29.88336181640625,
      "activations/layer18_attention_weight_max": 38.281105041503906,
      "activations/layer18_attention_weight_min": -27.448434829711914,
      "activations/layer19_attention_weight_max": 38.07664489746094,
      "activations/layer19_attention_weight_min": -33.261417388916016,
      "activations/layer1_attention_weight_max": 17.203266143798828,
      "activations/layer1_attention_weight_min": -15.979753494262695,
      "activations/layer20_attention_weight_max": 34.76992416381836,
      "activations/layer20_attention_weight_min": -26.305150985717773,
      "activations/layer21_attention_weight_max": 33.11227035522461,
      "activations/layer21_attention_weight_min": -25.395421981811523,
      "activations/layer22_attention_weight_max": 44.798404693603516,
      "activations/layer22_attention_weight_min": -32.58322525024414,
      "activations/layer23_attention_weight_max": 34.76205062866211,
      "activations/layer23_attention_weight_min": -30.130836486816406,
      "activations/layer2_attention_weight_max": 33.424278259277344,
      "activations/layer2_attention_weight_min": -32.505279541015625,
      "activations/layer3_attention_weight_max": 97.92730712890625,
      "activations/layer3_attention_weight_min": -97.8277816772461,
      "activations/layer4_attention_weight_max": 114.80382537841797,
      "activations/layer4_attention_weight_min": -110.8141098022461,
      "activations/layer5_attention_weight_max": 50.997859954833984,
      "activations/layer5_attention_weight_min": -66.05118560791016,
      "activations/layer6_attention_weight_max": 44.34153747558594,
      "activations/layer6_attention_weight_min": -46.78450012207031,
      "activations/layer7_attention_weight_max": 91.41755676269531,
      "activations/layer7_attention_weight_min": -93.64073944091797,
      "activations/layer8_attention_weight_max": 40.99269485473633,
      "activations/layer8_attention_weight_min": -43.76353454589844,
      "activations/layer9_attention_weight_max": 31.325363159179688,
      "activations/layer9_attention_weight_min": -32.43094253540039,
      "epoch": 14.64,
      "learning_rate": 5.6176893939393936e-05,
      "loss": 2.753,
      "step": 251900
    },
    {
      "activations/layer0_attention_weight_max": 16.051931381225586,
      "activations/layer0_attention_weight_min": -12.84580135345459,
      "activations/layer10_attention_weight_max": 37.68662643432617,
      "activations/layer10_attention_weight_min": -37.64912796020508,
      "activations/layer11_attention_weight_max": 36.63734436035156,
      "activations/layer11_attention_weight_min": -36.81221389770508,
      "activations/layer12_attention_weight_max": 29.85447883605957,
      "activations/layer12_attention_weight_min": -30.66530990600586,
      "activations/layer13_attention_weight_max": 42.13092041015625,
      "activations/layer13_attention_weight_min": -38.85637283325195,
      "activations/layer14_attention_weight_max": 49.983543395996094,
      "activations/layer14_attention_weight_min": -43.349029541015625,
      "activations/layer15_attention_weight_max": 42.30859375,
      "activations/layer15_attention_weight_min": -37.139652252197266,
      "activations/layer16_attention_weight_max": 31.207530975341797,
      "activations/layer16_attention_weight_min": -28.758419036865234,
      "activations/layer17_attention_weight_max": 31.948015213012695,
      "activations/layer17_attention_weight_min": -26.97624969482422,
      "activations/layer18_attention_weight_max": 30.407756805419922,
      "activations/layer18_attention_weight_min": -25.528247833251953,
      "activations/layer19_attention_weight_max": 36.41061019897461,
      "activations/layer19_attention_weight_min": -31.570022583007812,
      "activations/layer1_attention_weight_max": 15.96374225616455,
      "activations/layer1_attention_weight_min": -15.325054168701172,
      "activations/layer20_attention_weight_max": 31.677221298217773,
      "activations/layer20_attention_weight_min": -25.21654510498047,
      "activations/layer21_attention_weight_max": 35.06212615966797,
      "activations/layer21_attention_weight_min": -26.065744400024414,
      "activations/layer22_attention_weight_max": 46.301265716552734,
      "activations/layer22_attention_weight_min": -31.189517974853516,
      "activations/layer23_attention_weight_max": 33.39109802246094,
      "activations/layer23_attention_weight_min": -24.49787712097168,
      "activations/layer2_attention_weight_max": 33.65756607055664,
      "activations/layer2_attention_weight_min": -34.13621520996094,
      "activations/layer3_attention_weight_max": 97.35447692871094,
      "activations/layer3_attention_weight_min": -99.2638931274414,
      "activations/layer4_attention_weight_max": 114.6116714477539,
      "activations/layer4_attention_weight_min": -114.27178192138672,
      "activations/layer5_attention_weight_max": 54.349266052246094,
      "activations/layer5_attention_weight_min": -68.94389343261719,
      "activations/layer6_attention_weight_max": 45.31763458251953,
      "activations/layer6_attention_weight_min": -46.902732849121094,
      "activations/layer7_attention_weight_max": 96.80748748779297,
      "activations/layer7_attention_weight_min": -97.35767364501953,
      "activations/layer8_attention_weight_max": 44.65557861328125,
      "activations/layer8_attention_weight_min": -44.32309341430664,
      "activations/layer9_attention_weight_max": 35.38378143310547,
      "activations/layer9_attention_weight_min": -37.724910736083984,
      "epoch": 14.64,
      "learning_rate": 5.615795454545454e-05,
      "loss": 2.7588,
      "step": 251950
    },
    {
      "activations/layer0_attention_weight_max": 15.876607894897461,
      "activations/layer0_attention_weight_min": -12.12392520904541,
      "activations/layer10_attention_weight_max": 33.167781829833984,
      "activations/layer10_attention_weight_min": -30.4019775390625,
      "activations/layer11_attention_weight_max": 30.515384674072266,
      "activations/layer11_attention_weight_min": -29.73300552368164,
      "activations/layer12_attention_weight_max": 30.14699935913086,
      "activations/layer12_attention_weight_min": -24.821868896484375,
      "activations/layer13_attention_weight_max": 41.074275970458984,
      "activations/layer13_attention_weight_min": -33.75558090209961,
      "activations/layer14_attention_weight_max": 43.645973205566406,
      "activations/layer14_attention_weight_min": -35.89019775390625,
      "activations/layer15_attention_weight_max": 39.97071075439453,
      "activations/layer15_attention_weight_min": -33.312721252441406,
      "activations/layer16_attention_weight_max": 32.31938171386719,
      "activations/layer16_attention_weight_min": -29.61922264099121,
      "activations/layer17_attention_weight_max": 33.068172454833984,
      "activations/layer17_attention_weight_min": -24.68940544128418,
      "activations/layer18_attention_weight_max": 34.59050750732422,
      "activations/layer18_attention_weight_min": -25.30885124206543,
      "activations/layer19_attention_weight_max": 34.40882110595703,
      "activations/layer19_attention_weight_min": -28.734567642211914,
      "activations/layer1_attention_weight_max": 15.891016006469727,
      "activations/layer1_attention_weight_min": -13.91619873046875,
      "activations/layer20_attention_weight_max": 30.835493087768555,
      "activations/layer20_attention_weight_min": -25.153886795043945,
      "activations/layer21_attention_weight_max": 32.7392578125,
      "activations/layer21_attention_weight_min": -24.22636604309082,
      "activations/layer22_attention_weight_max": 44.82263946533203,
      "activations/layer22_attention_weight_min": -29.238523483276367,
      "activations/layer23_attention_weight_max": 32.59011459350586,
      "activations/layer23_attention_weight_min": -24.312597274780273,
      "activations/layer2_attention_weight_max": 33.97081756591797,
      "activations/layer2_attention_weight_min": -32.90433883666992,
      "activations/layer3_attention_weight_max": 93.22612762451172,
      "activations/layer3_attention_weight_min": -93.36963653564453,
      "activations/layer4_attention_weight_max": 108.69563293457031,
      "activations/layer4_attention_weight_min": -108.04351043701172,
      "activations/layer5_attention_weight_max": 51.32105255126953,
      "activations/layer5_attention_weight_min": -60.45415496826172,
      "activations/layer6_attention_weight_max": 43.6643180847168,
      "activations/layer6_attention_weight_min": -45.916175842285156,
      "activations/layer7_attention_weight_max": 88.15394592285156,
      "activations/layer7_attention_weight_min": -91.92974090576172,
      "activations/layer8_attention_weight_max": 36.4991569519043,
      "activations/layer8_attention_weight_min": -40.68979263305664,
      "activations/layer9_attention_weight_max": 30.295263290405273,
      "activations/layer9_attention_weight_min": -30.95989990234375,
      "epoch": 14.64,
      "learning_rate": 5.6139015151515146e-05,
      "loss": 2.7655,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4818,
      "eval_samples_per_second": 506.262,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4818,
      "eval_openwebtext_samples_per_second": 506.262,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 2.0005,
      "eval_wikitext_samples_per_second": 227.948,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.6232,
      "eval_lambada_samples_per_second": 505.964,
      "step": 252000
    },
    {
      "activations/layer0_attention_weight_max": 15.991451263427734,
      "activations/layer0_attention_weight_min": -12.386589050292969,
      "activations/layer10_attention_weight_max": 32.674720764160156,
      "activations/layer10_attention_weight_min": -31.115854263305664,
      "activations/layer11_attention_weight_max": 31.237918853759766,
      "activations/layer11_attention_weight_min": -30.841678619384766,
      "activations/layer12_attention_weight_max": 26.330781936645508,
      "activations/layer12_attention_weight_min": -25.094438552856445,
      "activations/layer13_attention_weight_max": 38.900917053222656,
      "activations/layer13_attention_weight_min": -36.22352600097656,
      "activations/layer14_attention_weight_max": 41.770545959472656,
      "activations/layer14_attention_weight_min": -36.841609954833984,
      "activations/layer15_attention_weight_max": 38.0196647644043,
      "activations/layer15_attention_weight_min": -31.52396583557129,
      "activations/layer16_attention_weight_max": 29.270998001098633,
      "activations/layer16_attention_weight_min": -26.783666610717773,
      "activations/layer17_attention_weight_max": 32.318233489990234,
      "activations/layer17_attention_weight_min": -24.38968276977539,
      "activations/layer18_attention_weight_max": 32.80836486816406,
      "activations/layer18_attention_weight_min": -23.248939514160156,
      "activations/layer19_attention_weight_max": 32.522727966308594,
      "activations/layer19_attention_weight_min": -30.146957397460938,
      "activations/layer1_attention_weight_max": 15.698075294494629,
      "activations/layer1_attention_weight_min": -13.331381797790527,
      "activations/layer20_attention_weight_max": 29.016368865966797,
      "activations/layer20_attention_weight_min": -24.3512020111084,
      "activations/layer21_attention_weight_max": 30.799896240234375,
      "activations/layer21_attention_weight_min": -24.156291961669922,
      "activations/layer22_attention_weight_max": 42.293785095214844,
      "activations/layer22_attention_weight_min": -28.91018295288086,
      "activations/layer23_attention_weight_max": 33.49895095825195,
      "activations/layer23_attention_weight_min": -24.157638549804688,
      "activations/layer2_attention_weight_max": 32.35637664794922,
      "activations/layer2_attention_weight_min": -31.825435638427734,
      "activations/layer3_attention_weight_max": 92.45693969726562,
      "activations/layer3_attention_weight_min": -95.46419525146484,
      "activations/layer4_attention_weight_max": 105.85963439941406,
      "activations/layer4_attention_weight_min": -107.39106750488281,
      "activations/layer5_attention_weight_max": 50.29541015625,
      "activations/layer5_attention_weight_min": -66.48121643066406,
      "activations/layer6_attention_weight_max": 42.53896713256836,
      "activations/layer6_attention_weight_min": -48.2735481262207,
      "activations/layer7_attention_weight_max": 84.5210952758789,
      "activations/layer7_attention_weight_min": -90.78912353515625,
      "activations/layer8_attention_weight_max": 39.59422302246094,
      "activations/layer8_attention_weight_min": -41.36058807373047,
      "activations/layer9_attention_weight_max": 31.2731990814209,
      "activations/layer9_attention_weight_min": -32.22900390625,
      "epoch": 14.65,
      "learning_rate": 5.6120075757575755e-05,
      "loss": 2.7613,
      "step": 252050
    },
    {
      "activations/layer0_attention_weight_max": 15.959166526794434,
      "activations/layer0_attention_weight_min": -12.434914588928223,
      "activations/layer10_attention_weight_max": 35.0199089050293,
      "activations/layer10_attention_weight_min": -33.61958312988281,
      "activations/layer11_attention_weight_max": 34.67683792114258,
      "activations/layer11_attention_weight_min": -33.31571578979492,
      "activations/layer12_attention_weight_max": 25.15223503112793,
      "activations/layer12_attention_weight_min": -26.41912269592285,
      "activations/layer13_attention_weight_max": 37.890846252441406,
      "activations/layer13_attention_weight_min": -34.76493453979492,
      "activations/layer14_attention_weight_max": 44.830055236816406,
      "activations/layer14_attention_weight_min": -38.396419525146484,
      "activations/layer15_attention_weight_max": 36.96063232421875,
      "activations/layer15_attention_weight_min": -33.50506591796875,
      "activations/layer16_attention_weight_max": 29.8763427734375,
      "activations/layer16_attention_weight_min": -30.42409324645996,
      "activations/layer17_attention_weight_max": 31.21314811706543,
      "activations/layer17_attention_weight_min": -26.61458969116211,
      "activations/layer18_attention_weight_max": 32.06065368652344,
      "activations/layer18_attention_weight_min": -25.57887840270996,
      "activations/layer19_attention_weight_max": 36.82375717163086,
      "activations/layer19_attention_weight_min": -31.990976333618164,
      "activations/layer1_attention_weight_max": 16.13068199157715,
      "activations/layer1_attention_weight_min": -14.968202590942383,
      "activations/layer20_attention_weight_max": 32.124332427978516,
      "activations/layer20_attention_weight_min": -25.585887908935547,
      "activations/layer21_attention_weight_max": 28.89047622680664,
      "activations/layer21_attention_weight_min": -25.616331100463867,
      "activations/layer22_attention_weight_max": 44.72665786743164,
      "activations/layer22_attention_weight_min": -33.82139587402344,
      "activations/layer23_attention_weight_max": 32.59239959716797,
      "activations/layer23_attention_weight_min": -25.539325714111328,
      "activations/layer2_attention_weight_max": 32.936885833740234,
      "activations/layer2_attention_weight_min": -32.26966857910156,
      "activations/layer3_attention_weight_max": 96.8537368774414,
      "activations/layer3_attention_weight_min": -93.7729721069336,
      "activations/layer4_attention_weight_max": 110.42520904541016,
      "activations/layer4_attention_weight_min": -107.94770812988281,
      "activations/layer5_attention_weight_max": 50.09894943237305,
      "activations/layer5_attention_weight_min": -60.017250061035156,
      "activations/layer6_attention_weight_max": 43.9828987121582,
      "activations/layer6_attention_weight_min": -46.05191421508789,
      "activations/layer7_attention_weight_max": 85.72108459472656,
      "activations/layer7_attention_weight_min": -87.9424057006836,
      "activations/layer8_attention_weight_max": 40.20087814331055,
      "activations/layer8_attention_weight_min": -41.68456268310547,
      "activations/layer9_attention_weight_max": 33.43809127807617,
      "activations/layer9_attention_weight_min": -32.84617233276367,
      "epoch": 14.65,
      "learning_rate": 5.6101136363636356e-05,
      "loss": 2.76,
      "step": 252100
    },
    {
      "activations/layer0_attention_weight_max": 16.08524513244629,
      "activations/layer0_attention_weight_min": -11.795411109924316,
      "activations/layer10_attention_weight_max": 33.3776741027832,
      "activations/layer10_attention_weight_min": -33.26563262939453,
      "activations/layer11_attention_weight_max": 33.029197692871094,
      "activations/layer11_attention_weight_min": -31.734291076660156,
      "activations/layer12_attention_weight_max": 25.45843505859375,
      "activations/layer12_attention_weight_min": -25.271116256713867,
      "activations/layer13_attention_weight_max": 38.36992645263672,
      "activations/layer13_attention_weight_min": -32.8377685546875,
      "activations/layer14_attention_weight_max": 41.92702865600586,
      "activations/layer14_attention_weight_min": -35.76837921142578,
      "activations/layer15_attention_weight_max": 37.411842346191406,
      "activations/layer15_attention_weight_min": -34.69877243041992,
      "activations/layer16_attention_weight_max": 30.863859176635742,
      "activations/layer16_attention_weight_min": -27.502872467041016,
      "activations/layer17_attention_weight_max": 34.3122444152832,
      "activations/layer17_attention_weight_min": -24.66006851196289,
      "activations/layer18_attention_weight_max": 32.93247985839844,
      "activations/layer18_attention_weight_min": -24.13559341430664,
      "activations/layer19_attention_weight_max": 36.82597351074219,
      "activations/layer19_attention_weight_min": -29.99184226989746,
      "activations/layer1_attention_weight_max": 15.980386734008789,
      "activations/layer1_attention_weight_min": -14.99103832244873,
      "activations/layer20_attention_weight_max": 33.353572845458984,
      "activations/layer20_attention_weight_min": -23.74724006652832,
      "activations/layer21_attention_weight_max": 34.59981918334961,
      "activations/layer21_attention_weight_min": -24.871152877807617,
      "activations/layer22_attention_weight_max": 42.3221435546875,
      "activations/layer22_attention_weight_min": -29.008291244506836,
      "activations/layer23_attention_weight_max": 35.246124267578125,
      "activations/layer23_attention_weight_min": -25.807119369506836,
      "activations/layer2_attention_weight_max": 32.1593017578125,
      "activations/layer2_attention_weight_min": -33.389678955078125,
      "activations/layer3_attention_weight_max": 90.63548278808594,
      "activations/layer3_attention_weight_min": -95.51699829101562,
      "activations/layer4_attention_weight_max": 104.14188385009766,
      "activations/layer4_attention_weight_min": -106.02052307128906,
      "activations/layer5_attention_weight_max": 49.28502655029297,
      "activations/layer5_attention_weight_min": -60.61167526245117,
      "activations/layer6_attention_weight_max": 42.244773864746094,
      "activations/layer6_attention_weight_min": -46.187381744384766,
      "activations/layer7_attention_weight_max": 91.27648162841797,
      "activations/layer7_attention_weight_min": -95.89429473876953,
      "activations/layer8_attention_weight_max": 39.0765495300293,
      "activations/layer8_attention_weight_min": -43.50495910644531,
      "activations/layer9_attention_weight_max": 31.25441551208496,
      "activations/layer9_attention_weight_min": -34.410072326660156,
      "epoch": 14.65,
      "learning_rate": 5.6082196969696964e-05,
      "loss": 2.7487,
      "step": 252150
    },
    {
      "activations/layer0_attention_weight_max": 15.903865814208984,
      "activations/layer0_attention_weight_min": -12.610404014587402,
      "activations/layer10_attention_weight_max": 32.71846389770508,
      "activations/layer10_attention_weight_min": -34.542179107666016,
      "activations/layer11_attention_weight_max": 32.618255615234375,
      "activations/layer11_attention_weight_min": -32.203704833984375,
      "activations/layer12_attention_weight_max": 29.316850662231445,
      "activations/layer12_attention_weight_min": -25.50053596496582,
      "activations/layer13_attention_weight_max": 41.10374450683594,
      "activations/layer13_attention_weight_min": -35.12208938598633,
      "activations/layer14_attention_weight_max": 47.14154815673828,
      "activations/layer14_attention_weight_min": -36.33802032470703,
      "activations/layer15_attention_weight_max": 40.81502914428711,
      "activations/layer15_attention_weight_min": -33.68064880371094,
      "activations/layer16_attention_weight_max": 32.020999908447266,
      "activations/layer16_attention_weight_min": -28.788047790527344,
      "activations/layer17_attention_weight_max": 31.95465850830078,
      "activations/layer17_attention_weight_min": -25.294010162353516,
      "activations/layer18_attention_weight_max": 33.36322784423828,
      "activations/layer18_attention_weight_min": -24.042598724365234,
      "activations/layer19_attention_weight_max": 35.437923431396484,
      "activations/layer19_attention_weight_min": -32.38649368286133,
      "activations/layer1_attention_weight_max": 15.923746109008789,
      "activations/layer1_attention_weight_min": -14.443889617919922,
      "activations/layer20_attention_weight_max": 32.08624267578125,
      "activations/layer20_attention_weight_min": -25.123149871826172,
      "activations/layer21_attention_weight_max": 30.896194458007812,
      "activations/layer21_attention_weight_min": -25.199905395507812,
      "activations/layer22_attention_weight_max": 54.922996520996094,
      "activations/layer22_attention_weight_min": -30.544843673706055,
      "activations/layer23_attention_weight_max": 34.62963104248047,
      "activations/layer23_attention_weight_min": -24.849029541015625,
      "activations/layer2_attention_weight_max": 34.508697509765625,
      "activations/layer2_attention_weight_min": -31.227209091186523,
      "activations/layer3_attention_weight_max": 95.88802337646484,
      "activations/layer3_attention_weight_min": -93.52749633789062,
      "activations/layer4_attention_weight_max": 109.42433166503906,
      "activations/layer4_attention_weight_min": -106.47804260253906,
      "activations/layer5_attention_weight_max": 54.48822021484375,
      "activations/layer5_attention_weight_min": -61.07363510131836,
      "activations/layer6_attention_weight_max": 44.00205612182617,
      "activations/layer6_attention_weight_min": -47.11833953857422,
      "activations/layer7_attention_weight_max": 96.59532165527344,
      "activations/layer7_attention_weight_min": -87.90442657470703,
      "activations/layer8_attention_weight_max": 41.09933853149414,
      "activations/layer8_attention_weight_min": -41.07911682128906,
      "activations/layer9_attention_weight_max": 31.004995346069336,
      "activations/layer9_attention_weight_min": -33.218719482421875,
      "epoch": 14.65,
      "learning_rate": 5.6063257575757566e-05,
      "loss": 2.7689,
      "step": 252200
    },
    {
      "activations/layer0_attention_weight_max": 16.027816772460938,
      "activations/layer0_attention_weight_min": -12.698481559753418,
      "activations/layer10_attention_weight_max": 33.28822708129883,
      "activations/layer10_attention_weight_min": -33.39402770996094,
      "activations/layer11_attention_weight_max": 33.43010711669922,
      "activations/layer11_attention_weight_min": -34.483726501464844,
      "activations/layer12_attention_weight_max": 27.136354446411133,
      "activations/layer12_attention_weight_min": -25.66742515563965,
      "activations/layer13_attention_weight_max": 37.9274787902832,
      "activations/layer13_attention_weight_min": -33.95817565917969,
      "activations/layer14_attention_weight_max": 41.99538040161133,
      "activations/layer14_attention_weight_min": -35.92622756958008,
      "activations/layer15_attention_weight_max": 38.14531707763672,
      "activations/layer15_attention_weight_min": -34.600643157958984,
      "activations/layer16_attention_weight_max": 31.954389572143555,
      "activations/layer16_attention_weight_min": -28.59478187561035,
      "activations/layer17_attention_weight_max": 31.837467193603516,
      "activations/layer17_attention_weight_min": -25.70096778869629,
      "activations/layer18_attention_weight_max": 29.37285041809082,
      "activations/layer18_attention_weight_min": -23.949026107788086,
      "activations/layer19_attention_weight_max": 33.36851119995117,
      "activations/layer19_attention_weight_min": -28.380125045776367,
      "activations/layer1_attention_weight_max": 16.347763061523438,
      "activations/layer1_attention_weight_min": -18.164203643798828,
      "activations/layer20_attention_weight_max": 29.780344009399414,
      "activations/layer20_attention_weight_min": -24.833316802978516,
      "activations/layer21_attention_weight_max": 29.152328491210938,
      "activations/layer21_attention_weight_min": -24.10227394104004,
      "activations/layer22_attention_weight_max": 46.17735290527344,
      "activations/layer22_attention_weight_min": -30.973163604736328,
      "activations/layer23_attention_weight_max": 35.038818359375,
      "activations/layer23_attention_weight_min": -25.40613555908203,
      "activations/layer2_attention_weight_max": 32.51820755004883,
      "activations/layer2_attention_weight_min": -33.30442428588867,
      "activations/layer3_attention_weight_max": 96.78751373291016,
      "activations/layer3_attention_weight_min": -101.15324401855469,
      "activations/layer4_attention_weight_max": 104.58199310302734,
      "activations/layer4_attention_weight_min": -105.4022445678711,
      "activations/layer5_attention_weight_max": 50.96818161010742,
      "activations/layer5_attention_weight_min": -61.23464584350586,
      "activations/layer6_attention_weight_max": 42.362667083740234,
      "activations/layer6_attention_weight_min": -44.471519470214844,
      "activations/layer7_attention_weight_max": 90.56505584716797,
      "activations/layer7_attention_weight_min": -86.83466339111328,
      "activations/layer8_attention_weight_max": 38.88727951049805,
      "activations/layer8_attention_weight_min": -40.194664001464844,
      "activations/layer9_attention_weight_max": 31.863412857055664,
      "activations/layer9_attention_weight_min": -31.189359664916992,
      "epoch": 14.66,
      "learning_rate": 5.604431818181818e-05,
      "loss": 2.7619,
      "step": 252250
    },
    {
      "activations/layer0_attention_weight_max": 15.875164985656738,
      "activations/layer0_attention_weight_min": -12.367270469665527,
      "activations/layer10_attention_weight_max": 32.25366973876953,
      "activations/layer10_attention_weight_min": -32.820831298828125,
      "activations/layer11_attention_weight_max": 30.943683624267578,
      "activations/layer11_attention_weight_min": -31.717512130737305,
      "activations/layer12_attention_weight_max": 32.66703796386719,
      "activations/layer12_attention_weight_min": -25.82005500793457,
      "activations/layer13_attention_weight_max": 39.61210250854492,
      "activations/layer13_attention_weight_min": -34.24353790283203,
      "activations/layer14_attention_weight_max": 43.5990104675293,
      "activations/layer14_attention_weight_min": -36.053401947021484,
      "activations/layer15_attention_weight_max": 39.23138427734375,
      "activations/layer15_attention_weight_min": -34.95949935913086,
      "activations/layer16_attention_weight_max": 33.282562255859375,
      "activations/layer16_attention_weight_min": -27.898866653442383,
      "activations/layer17_attention_weight_max": 30.80074119567871,
      "activations/layer17_attention_weight_min": -26.256071090698242,
      "activations/layer18_attention_weight_max": 31.041414260864258,
      "activations/layer18_attention_weight_min": -26.499181747436523,
      "activations/layer19_attention_weight_max": 37.83585739135742,
      "activations/layer19_attention_weight_min": -32.10765075683594,
      "activations/layer1_attention_weight_max": 17.03431510925293,
      "activations/layer1_attention_weight_min": -14.327637672424316,
      "activations/layer20_attention_weight_max": 30.324466705322266,
      "activations/layer20_attention_weight_min": -25.86724853515625,
      "activations/layer21_attention_weight_max": 30.40523338317871,
      "activations/layer21_attention_weight_min": -25.865774154663086,
      "activations/layer22_attention_weight_max": 45.29057693481445,
      "activations/layer22_attention_weight_min": -32.63216018676758,
      "activations/layer23_attention_weight_max": 35.80967330932617,
      "activations/layer23_attention_weight_min": -25.502777099609375,
      "activations/layer2_attention_weight_max": 34.28932571411133,
      "activations/layer2_attention_weight_min": -32.857421875,
      "activations/layer3_attention_weight_max": 94.29004669189453,
      "activations/layer3_attention_weight_min": -97.97884368896484,
      "activations/layer4_attention_weight_max": 106.85411834716797,
      "activations/layer4_attention_weight_min": -110.65638732910156,
      "activations/layer5_attention_weight_max": 51.725563049316406,
      "activations/layer5_attention_weight_min": -61.510406494140625,
      "activations/layer6_attention_weight_max": 42.245521545410156,
      "activations/layer6_attention_weight_min": -43.82184982299805,
      "activations/layer7_attention_weight_max": 83.65928649902344,
      "activations/layer7_attention_weight_min": -84.75633239746094,
      "activations/layer8_attention_weight_max": 37.37132263183594,
      "activations/layer8_attention_weight_min": -39.92394256591797,
      "activations/layer9_attention_weight_max": 29.853864669799805,
      "activations/layer9_attention_weight_min": -32.04880142211914,
      "epoch": 14.66,
      "learning_rate": 5.602537878787878e-05,
      "loss": 2.7632,
      "step": 252300
    },
    {
      "activations/layer0_attention_weight_max": 16.857994079589844,
      "activations/layer0_attention_weight_min": -11.768568992614746,
      "activations/layer10_attention_weight_max": 38.04999542236328,
      "activations/layer10_attention_weight_min": -34.23957824707031,
      "activations/layer11_attention_weight_max": 38.67491912841797,
      "activations/layer11_attention_weight_min": -34.53630447387695,
      "activations/layer12_attention_weight_max": 28.99764633178711,
      "activations/layer12_attention_weight_min": -26.78256607055664,
      "activations/layer13_attention_weight_max": 42.39952850341797,
      "activations/layer13_attention_weight_min": -33.27360153198242,
      "activations/layer14_attention_weight_max": 45.20946502685547,
      "activations/layer14_attention_weight_min": -36.22998046875,
      "activations/layer15_attention_weight_max": 40.7562370300293,
      "activations/layer15_attention_weight_min": -33.17277908325195,
      "activations/layer16_attention_weight_max": 31.445499420166016,
      "activations/layer16_attention_weight_min": -27.565017700195312,
      "activations/layer17_attention_weight_max": 32.9498405456543,
      "activations/layer17_attention_weight_min": -26.541488647460938,
      "activations/layer18_attention_weight_max": 31.737096786499023,
      "activations/layer18_attention_weight_min": -24.01774787902832,
      "activations/layer19_attention_weight_max": 37.47477722167969,
      "activations/layer19_attention_weight_min": -29.102365493774414,
      "activations/layer1_attention_weight_max": 17.650604248046875,
      "activations/layer1_attention_weight_min": -14.372920989990234,
      "activations/layer20_attention_weight_max": 32.58176040649414,
      "activations/layer20_attention_weight_min": -24.431011199951172,
      "activations/layer21_attention_weight_max": 31.730966567993164,
      "activations/layer21_attention_weight_min": -24.4425048828125,
      "activations/layer22_attention_weight_max": 45.28342819213867,
      "activations/layer22_attention_weight_min": -32.1183967590332,
      "activations/layer23_attention_weight_max": 35.85353088378906,
      "activations/layer23_attention_weight_min": -26.193286895751953,
      "activations/layer2_attention_weight_max": 34.853485107421875,
      "activations/layer2_attention_weight_min": -32.76279830932617,
      "activations/layer3_attention_weight_max": 96.53741455078125,
      "activations/layer3_attention_weight_min": -96.10973358154297,
      "activations/layer4_attention_weight_max": 109.23526763916016,
      "activations/layer4_attention_weight_min": -106.2208480834961,
      "activations/layer5_attention_weight_max": 50.43626403808594,
      "activations/layer5_attention_weight_min": -60.0419921875,
      "activations/layer6_attention_weight_max": 44.279945373535156,
      "activations/layer6_attention_weight_min": -46.64289474487305,
      "activations/layer7_attention_weight_max": 91.57305145263672,
      "activations/layer7_attention_weight_min": -88.95680236816406,
      "activations/layer8_attention_weight_max": 40.30584716796875,
      "activations/layer8_attention_weight_min": -41.687889099121094,
      "activations/layer9_attention_weight_max": 35.19349670410156,
      "activations/layer9_attention_weight_min": -34.729984283447266,
      "epoch": 14.66,
      "learning_rate": 5.6006439393939384e-05,
      "loss": 2.7402,
      "step": 252350
    },
    {
      "activations/layer0_attention_weight_max": 14.955009460449219,
      "activations/layer0_attention_weight_min": -12.041038513183594,
      "activations/layer10_attention_weight_max": 34.64925003051758,
      "activations/layer10_attention_weight_min": -33.8261833190918,
      "activations/layer11_attention_weight_max": 34.91302490234375,
      "activations/layer11_attention_weight_min": -32.1606559753418,
      "activations/layer12_attention_weight_max": 26.44306182861328,
      "activations/layer12_attention_weight_min": -25.9965877532959,
      "activations/layer13_attention_weight_max": 42.13157272338867,
      "activations/layer13_attention_weight_min": -35.918060302734375,
      "activations/layer14_attention_weight_max": 41.084388732910156,
      "activations/layer14_attention_weight_min": -38.49453353881836,
      "activations/layer15_attention_weight_max": 38.58769607543945,
      "activations/layer15_attention_weight_min": -33.95423889160156,
      "activations/layer16_attention_weight_max": 29.050655364990234,
      "activations/layer16_attention_weight_min": -26.88848304748535,
      "activations/layer17_attention_weight_max": 31.798824310302734,
      "activations/layer17_attention_weight_min": -27.37221336364746,
      "activations/layer18_attention_weight_max": 32.60025405883789,
      "activations/layer18_attention_weight_min": -26.070266723632812,
      "activations/layer19_attention_weight_max": 31.920576095581055,
      "activations/layer19_attention_weight_min": -32.38413619995117,
      "activations/layer1_attention_weight_max": 15.88638973236084,
      "activations/layer1_attention_weight_min": -14.292179107666016,
      "activations/layer20_attention_weight_max": 29.694244384765625,
      "activations/layer20_attention_weight_min": -26.063791275024414,
      "activations/layer21_attention_weight_max": 28.89036750793457,
      "activations/layer21_attention_weight_min": -25.640518188476562,
      "activations/layer22_attention_weight_max": 43.27226638793945,
      "activations/layer22_attention_weight_min": -31.930280685424805,
      "activations/layer23_attention_weight_max": 34.97193145751953,
      "activations/layer23_attention_weight_min": -25.547834396362305,
      "activations/layer2_attention_weight_max": 33.70842361450195,
      "activations/layer2_attention_weight_min": -32.684410095214844,
      "activations/layer3_attention_weight_max": 96.32526397705078,
      "activations/layer3_attention_weight_min": -97.80443572998047,
      "activations/layer4_attention_weight_max": 110.9424819946289,
      "activations/layer4_attention_weight_min": -115.2827377319336,
      "activations/layer5_attention_weight_max": 52.946044921875,
      "activations/layer5_attention_weight_min": -61.073211669921875,
      "activations/layer6_attention_weight_max": 43.82148361206055,
      "activations/layer6_attention_weight_min": -46.36948776245117,
      "activations/layer7_attention_weight_max": 89.83406829833984,
      "activations/layer7_attention_weight_min": -90.50189208984375,
      "activations/layer8_attention_weight_max": 43.84243392944336,
      "activations/layer8_attention_weight_min": -42.64765930175781,
      "activations/layer9_attention_weight_max": 34.41047286987305,
      "activations/layer9_attention_weight_min": -33.96797180175781,
      "epoch": 14.67,
      "learning_rate": 5.59875e-05,
      "loss": 2.7787,
      "step": 252400
    },
    {
      "activations/layer0_attention_weight_max": 15.187275886535645,
      "activations/layer0_attention_weight_min": -12.646860122680664,
      "activations/layer10_attention_weight_max": 43.99162292480469,
      "activations/layer10_attention_weight_min": -42.076480865478516,
      "activations/layer11_attention_weight_max": 45.76739501953125,
      "activations/layer11_attention_weight_min": -42.93077087402344,
      "activations/layer12_attention_weight_max": 33.408447265625,
      "activations/layer12_attention_weight_min": -29.390745162963867,
      "activations/layer13_attention_weight_max": 48.8364372253418,
      "activations/layer13_attention_weight_min": -38.69142150878906,
      "activations/layer14_attention_weight_max": 57.903228759765625,
      "activations/layer14_attention_weight_min": -43.742523193359375,
      "activations/layer15_attention_weight_max": 47.69403839111328,
      "activations/layer15_attention_weight_min": -38.94351577758789,
      "activations/layer16_attention_weight_max": 39.41301345825195,
      "activations/layer16_attention_weight_min": -28.5899658203125,
      "activations/layer17_attention_weight_max": 35.03178787231445,
      "activations/layer17_attention_weight_min": -26.219572067260742,
      "activations/layer18_attention_weight_max": 36.28871536254883,
      "activations/layer18_attention_weight_min": -25.19987678527832,
      "activations/layer19_attention_weight_max": 43.107486724853516,
      "activations/layer19_attention_weight_min": -32.720794677734375,
      "activations/layer1_attention_weight_max": 16.253450393676758,
      "activations/layer1_attention_weight_min": -14.09528636932373,
      "activations/layer20_attention_weight_max": 32.794429779052734,
      "activations/layer20_attention_weight_min": -25.163616180419922,
      "activations/layer21_attention_weight_max": 34.767364501953125,
      "activations/layer21_attention_weight_min": -24.968698501586914,
      "activations/layer22_attention_weight_max": 48.56926727294922,
      "activations/layer22_attention_weight_min": -31.652851104736328,
      "activations/layer23_attention_weight_max": 38.61775588989258,
      "activations/layer23_attention_weight_min": -25.516345977783203,
      "activations/layer2_attention_weight_max": 37.107460021972656,
      "activations/layer2_attention_weight_min": -33.68291473388672,
      "activations/layer3_attention_weight_max": 104.2380142211914,
      "activations/layer3_attention_weight_min": -100.6529312133789,
      "activations/layer4_attention_weight_max": 117.34056854248047,
      "activations/layer4_attention_weight_min": -115.36299133300781,
      "activations/layer5_attention_weight_max": 54.29950714111328,
      "activations/layer5_attention_weight_min": -64.3793716430664,
      "activations/layer6_attention_weight_max": 49.33319854736328,
      "activations/layer6_attention_weight_min": -48.598533630371094,
      "activations/layer7_attention_weight_max": 111.32986450195312,
      "activations/layer7_attention_weight_min": -100.41889190673828,
      "activations/layer8_attention_weight_max": 47.846588134765625,
      "activations/layer8_attention_weight_min": -49.25539016723633,
      "activations/layer9_attention_weight_max": 41.89878463745117,
      "activations/layer9_attention_weight_min": -39.31856918334961,
      "epoch": 14.67,
      "learning_rate": 5.59685606060606e-05,
      "loss": 2.766,
      "step": 252450
    },
    {
      "activations/layer0_attention_weight_max": 16.18297576904297,
      "activations/layer0_attention_weight_min": -12.113581657409668,
      "activations/layer10_attention_weight_max": 30.130428314208984,
      "activations/layer10_attention_weight_min": -32.369197845458984,
      "activations/layer11_attention_weight_max": 31.389808654785156,
      "activations/layer11_attention_weight_min": -31.702184677124023,
      "activations/layer12_attention_weight_max": 23.864166259765625,
      "activations/layer12_attention_weight_min": -24.53816032409668,
      "activations/layer13_attention_weight_max": 37.59288024902344,
      "activations/layer13_attention_weight_min": -35.23300552368164,
      "activations/layer14_attention_weight_max": 40.923770904541016,
      "activations/layer14_attention_weight_min": -35.565467834472656,
      "activations/layer15_attention_weight_max": 35.54972839355469,
      "activations/layer15_attention_weight_min": -35.37495040893555,
      "activations/layer16_attention_weight_max": 31.06744384765625,
      "activations/layer16_attention_weight_min": -28.026121139526367,
      "activations/layer17_attention_weight_max": 29.88006019592285,
      "activations/layer17_attention_weight_min": -26.166528701782227,
      "activations/layer18_attention_weight_max": 34.20964050292969,
      "activations/layer18_attention_weight_min": -23.659175872802734,
      "activations/layer19_attention_weight_max": 34.36311721801758,
      "activations/layer19_attention_weight_min": -31.84102439880371,
      "activations/layer1_attention_weight_max": 16.131057739257812,
      "activations/layer1_attention_weight_min": -13.989999771118164,
      "activations/layer20_attention_weight_max": 30.674705505371094,
      "activations/layer20_attention_weight_min": -23.830751419067383,
      "activations/layer21_attention_weight_max": 29.22124481201172,
      "activations/layer21_attention_weight_min": -24.4150447845459,
      "activations/layer22_attention_weight_max": 41.42765426635742,
      "activations/layer22_attention_weight_min": -30.477069854736328,
      "activations/layer23_attention_weight_max": 35.71025466918945,
      "activations/layer23_attention_weight_min": -27.094833374023438,
      "activations/layer2_attention_weight_max": 30.5826416015625,
      "activations/layer2_attention_weight_min": -32.78190994262695,
      "activations/layer3_attention_weight_max": 90.8313217163086,
      "activations/layer3_attention_weight_min": -91.267333984375,
      "activations/layer4_attention_weight_max": 107.56754302978516,
      "activations/layer4_attention_weight_min": -109.84004211425781,
      "activations/layer5_attention_weight_max": 54.85184860229492,
      "activations/layer5_attention_weight_min": -69.96760559082031,
      "activations/layer6_attention_weight_max": 42.056060791015625,
      "activations/layer6_attention_weight_min": -45.79679489135742,
      "activations/layer7_attention_weight_max": 85.41448211669922,
      "activations/layer7_attention_weight_min": -87.21232604980469,
      "activations/layer8_attention_weight_max": 39.93951416015625,
      "activations/layer8_attention_weight_min": -38.945823669433594,
      "activations/layer9_attention_weight_max": 30.71146011352539,
      "activations/layer9_attention_weight_min": -31.041135787963867,
      "epoch": 14.67,
      "learning_rate": 5.594962121212121e-05,
      "loss": 2.758,
      "step": 252500
    },
    {
      "activations/layer0_attention_weight_max": 16.065000534057617,
      "activations/layer0_attention_weight_min": -12.411311149597168,
      "activations/layer10_attention_weight_max": 37.40862274169922,
      "activations/layer10_attention_weight_min": -38.954254150390625,
      "activations/layer11_attention_weight_max": 33.96137237548828,
      "activations/layer11_attention_weight_min": -36.3446159362793,
      "activations/layer12_attention_weight_max": 27.28523063659668,
      "activations/layer12_attention_weight_min": -25.297359466552734,
      "activations/layer13_attention_weight_max": 37.634822845458984,
      "activations/layer13_attention_weight_min": -34.59689712524414,
      "activations/layer14_attention_weight_max": 42.91090393066406,
      "activations/layer14_attention_weight_min": -36.97841262817383,
      "activations/layer15_attention_weight_max": 43.071842193603516,
      "activations/layer15_attention_weight_min": -33.87582015991211,
      "activations/layer16_attention_weight_max": 30.58537483215332,
      "activations/layer16_attention_weight_min": -26.499052047729492,
      "activations/layer17_attention_weight_max": 29.857051849365234,
      "activations/layer17_attention_weight_min": -25.616683959960938,
      "activations/layer18_attention_weight_max": 30.225553512573242,
      "activations/layer18_attention_weight_min": -25.880115509033203,
      "activations/layer19_attention_weight_max": 31.223093032836914,
      "activations/layer19_attention_weight_min": -30.593225479125977,
      "activations/layer1_attention_weight_max": 16.75663948059082,
      "activations/layer1_attention_weight_min": -16.001686096191406,
      "activations/layer20_attention_weight_max": 26.616315841674805,
      "activations/layer20_attention_weight_min": -26.000654220581055,
      "activations/layer21_attention_weight_max": 29.292877197265625,
      "activations/layer21_attention_weight_min": -25.39651870727539,
      "activations/layer22_attention_weight_max": 39.585384368896484,
      "activations/layer22_attention_weight_min": -30.281858444213867,
      "activations/layer23_attention_weight_max": 30.658836364746094,
      "activations/layer23_attention_weight_min": -25.77685546875,
      "activations/layer2_attention_weight_max": 33.796958923339844,
      "activations/layer2_attention_weight_min": -36.47978210449219,
      "activations/layer3_attention_weight_max": 100.17830657958984,
      "activations/layer3_attention_weight_min": -105.51957702636719,
      "activations/layer4_attention_weight_max": 117.1919174194336,
      "activations/layer4_attention_weight_min": -126.55252838134766,
      "activations/layer5_attention_weight_max": 55.6823844909668,
      "activations/layer5_attention_weight_min": -67.14481353759766,
      "activations/layer6_attention_weight_max": 53.90297317504883,
      "activations/layer6_attention_weight_min": -56.04145431518555,
      "activations/layer7_attention_weight_max": 96.10577392578125,
      "activations/layer7_attention_weight_min": -104.82234191894531,
      "activations/layer8_attention_weight_max": 47.18938064575195,
      "activations/layer8_attention_weight_min": -49.65422439575195,
      "activations/layer9_attention_weight_max": 35.86453628540039,
      "activations/layer9_attention_weight_min": -38.68782043457031,
      "epoch": 14.67,
      "learning_rate": 5.593068181818181e-05,
      "loss": 2.7554,
      "step": 252550
    },
    {
      "activations/layer0_attention_weight_max": 16.28738784790039,
      "activations/layer0_attention_weight_min": -12.110912322998047,
      "activations/layer10_attention_weight_max": 33.92239761352539,
      "activations/layer10_attention_weight_min": -36.12189483642578,
      "activations/layer11_attention_weight_max": 36.048980712890625,
      "activations/layer11_attention_weight_min": -37.84234619140625,
      "activations/layer12_attention_weight_max": 25.210527420043945,
      "activations/layer12_attention_weight_min": -25.130409240722656,
      "activations/layer13_attention_weight_max": 35.872249603271484,
      "activations/layer13_attention_weight_min": -34.47450637817383,
      "activations/layer14_attention_weight_max": 41.324256896972656,
      "activations/layer14_attention_weight_min": -37.2899055480957,
      "activations/layer15_attention_weight_max": 36.198143005371094,
      "activations/layer15_attention_weight_min": -33.32060241699219,
      "activations/layer16_attention_weight_max": 31.24695587158203,
      "activations/layer16_attention_weight_min": -28.397228240966797,
      "activations/layer17_attention_weight_max": 30.025922775268555,
      "activations/layer17_attention_weight_min": -25.167285919189453,
      "activations/layer18_attention_weight_max": 31.9787654876709,
      "activations/layer18_attention_weight_min": -24.89918327331543,
      "activations/layer19_attention_weight_max": 35.554359436035156,
      "activations/layer19_attention_weight_min": -32.6978874206543,
      "activations/layer1_attention_weight_max": 16.48211669921875,
      "activations/layer1_attention_weight_min": -15.561338424682617,
      "activations/layer20_attention_weight_max": 29.320945739746094,
      "activations/layer20_attention_weight_min": -25.955888748168945,
      "activations/layer21_attention_weight_max": 31.643142700195312,
      "activations/layer21_attention_weight_min": -23.563644409179688,
      "activations/layer22_attention_weight_max": 44.862735748291016,
      "activations/layer22_attention_weight_min": -29.50124740600586,
      "activations/layer23_attention_weight_max": 33.90559005737305,
      "activations/layer23_attention_weight_min": -27.446683883666992,
      "activations/layer2_attention_weight_max": 34.070255279541016,
      "activations/layer2_attention_weight_min": -33.406494140625,
      "activations/layer3_attention_weight_max": 98.74822998046875,
      "activations/layer3_attention_weight_min": -102.00740051269531,
      "activations/layer4_attention_weight_max": 111.60448455810547,
      "activations/layer4_attention_weight_min": -107.97830963134766,
      "activations/layer5_attention_weight_max": 51.968605041503906,
      "activations/layer5_attention_weight_min": -62.14769744873047,
      "activations/layer6_attention_weight_max": 42.60160827636719,
      "activations/layer6_attention_weight_min": -44.71290969848633,
      "activations/layer7_attention_weight_max": 89.7976303100586,
      "activations/layer7_attention_weight_min": -90.98487091064453,
      "activations/layer8_attention_weight_max": 41.71113204956055,
      "activations/layer8_attention_weight_min": -43.76533126831055,
      "activations/layer9_attention_weight_max": 34.77083206176758,
      "activations/layer9_attention_weight_min": -34.68464279174805,
      "epoch": 14.68,
      "learning_rate": 5.5911742424242425e-05,
      "loss": 2.7412,
      "step": 252600
    },
    {
      "activations/layer0_attention_weight_max": 15.579024314880371,
      "activations/layer0_attention_weight_min": -13.701878547668457,
      "activations/layer10_attention_weight_max": 30.73101043701172,
      "activations/layer10_attention_weight_min": -34.27016067504883,
      "activations/layer11_attention_weight_max": 31.683277130126953,
      "activations/layer11_attention_weight_min": -34.178375244140625,
      "activations/layer12_attention_weight_max": 27.54378890991211,
      "activations/layer12_attention_weight_min": -25.904178619384766,
      "activations/layer13_attention_weight_max": 39.800621032714844,
      "activations/layer13_attention_weight_min": -34.545570373535156,
      "activations/layer14_attention_weight_max": 43.83165740966797,
      "activations/layer14_attention_weight_min": -38.222206115722656,
      "activations/layer15_attention_weight_max": 37.854713439941406,
      "activations/layer15_attention_weight_min": -35.800472259521484,
      "activations/layer16_attention_weight_max": 29.355243682861328,
      "activations/layer16_attention_weight_min": -28.40186309814453,
      "activations/layer17_attention_weight_max": 30.09617805480957,
      "activations/layer17_attention_weight_min": -27.7613525390625,
      "activations/layer18_attention_weight_max": 31.060890197753906,
      "activations/layer18_attention_weight_min": -26.148731231689453,
      "activations/layer19_attention_weight_max": 32.63474655151367,
      "activations/layer19_attention_weight_min": -32.51713180541992,
      "activations/layer1_attention_weight_max": 16.334367752075195,
      "activations/layer1_attention_weight_min": -14.45370864868164,
      "activations/layer20_attention_weight_max": 28.8964786529541,
      "activations/layer20_attention_weight_min": -26.012527465820312,
      "activations/layer21_attention_weight_max": 27.900304794311523,
      "activations/layer21_attention_weight_min": -24.75722312927246,
      "activations/layer22_attention_weight_max": 45.00076675415039,
      "activations/layer22_attention_weight_min": -33.85944747924805,
      "activations/layer23_attention_weight_max": 32.2057991027832,
      "activations/layer23_attention_weight_min": -29.06351089477539,
      "activations/layer2_attention_weight_max": 31.18024253845215,
      "activations/layer2_attention_weight_min": -29.955564498901367,
      "activations/layer3_attention_weight_max": 95.7564468383789,
      "activations/layer3_attention_weight_min": -92.02436828613281,
      "activations/layer4_attention_weight_max": 101.97591400146484,
      "activations/layer4_attention_weight_min": -107.79644012451172,
      "activations/layer5_attention_weight_max": 48.90802764892578,
      "activations/layer5_attention_weight_min": -60.52093505859375,
      "activations/layer6_attention_weight_max": 42.63935470581055,
      "activations/layer6_attention_weight_min": -44.507225036621094,
      "activations/layer7_attention_weight_max": 92.6208267211914,
      "activations/layer7_attention_weight_min": -93.05753326416016,
      "activations/layer8_attention_weight_max": 37.11270523071289,
      "activations/layer8_attention_weight_min": -39.39723587036133,
      "activations/layer9_attention_weight_max": 30.2646484375,
      "activations/layer9_attention_weight_min": -32.75261688232422,
      "epoch": 14.68,
      "learning_rate": 5.589280303030303e-05,
      "loss": 2.743,
      "step": 252650
    },
    {
      "activations/layer0_attention_weight_max": 16.027996063232422,
      "activations/layer0_attention_weight_min": -12.32215690612793,
      "activations/layer10_attention_weight_max": 35.92232131958008,
      "activations/layer10_attention_weight_min": -38.49071502685547,
      "activations/layer11_attention_weight_max": 35.407752990722656,
      "activations/layer11_attention_weight_min": -37.632667541503906,
      "activations/layer12_attention_weight_max": 28.61754608154297,
      "activations/layer12_attention_weight_min": -29.595645904541016,
      "activations/layer13_attention_weight_max": 43.28525161743164,
      "activations/layer13_attention_weight_min": -40.32867431640625,
      "activations/layer14_attention_weight_max": 45.91450500488281,
      "activations/layer14_attention_weight_min": -45.06909942626953,
      "activations/layer15_attention_weight_max": 41.838008880615234,
      "activations/layer15_attention_weight_min": -41.14112091064453,
      "activations/layer16_attention_weight_max": 30.640682220458984,
      "activations/layer16_attention_weight_min": -32.53335952758789,
      "activations/layer17_attention_weight_max": 33.06245422363281,
      "activations/layer17_attention_weight_min": -28.468441009521484,
      "activations/layer18_attention_weight_max": 35.07809829711914,
      "activations/layer18_attention_weight_min": -27.400554656982422,
      "activations/layer19_attention_weight_max": 37.47107696533203,
      "activations/layer19_attention_weight_min": -37.048919677734375,
      "activations/layer1_attention_weight_max": 17.669281005859375,
      "activations/layer1_attention_weight_min": -14.842818260192871,
      "activations/layer20_attention_weight_max": 32.93148422241211,
      "activations/layer20_attention_weight_min": -27.327741622924805,
      "activations/layer21_attention_weight_max": 30.80303955078125,
      "activations/layer21_attention_weight_min": -26.692026138305664,
      "activations/layer22_attention_weight_max": 43.08574295043945,
      "activations/layer22_attention_weight_min": -33.04410934448242,
      "activations/layer23_attention_weight_max": 32.04951858520508,
      "activations/layer23_attention_weight_min": -24.706523895263672,
      "activations/layer2_attention_weight_max": 33.7459716796875,
      "activations/layer2_attention_weight_min": -32.83687210083008,
      "activations/layer3_attention_weight_max": 98.31407165527344,
      "activations/layer3_attention_weight_min": -99.51960754394531,
      "activations/layer4_attention_weight_max": 121.71581268310547,
      "activations/layer4_attention_weight_min": -118.2774887084961,
      "activations/layer5_attention_weight_max": 55.11698913574219,
      "activations/layer5_attention_weight_min": -64.03544616699219,
      "activations/layer6_attention_weight_max": 48.58621597290039,
      "activations/layer6_attention_weight_min": -48.60773849487305,
      "activations/layer7_attention_weight_max": 94.41340637207031,
      "activations/layer7_attention_weight_min": -97.54267883300781,
      "activations/layer8_attention_weight_max": 45.765602111816406,
      "activations/layer8_attention_weight_min": -49.152503967285156,
      "activations/layer9_attention_weight_max": 36.79378890991211,
      "activations/layer9_attention_weight_min": -40.61703872680664,
      "epoch": 14.68,
      "learning_rate": 5.587386363636363e-05,
      "loss": 2.7426,
      "step": 252700
    },
    {
      "activations/layer0_attention_weight_max": 15.855267524719238,
      "activations/layer0_attention_weight_min": -13.199254989624023,
      "activations/layer10_attention_weight_max": 29.612890243530273,
      "activations/layer10_attention_weight_min": -32.73024368286133,
      "activations/layer11_attention_weight_max": 32.72393798828125,
      "activations/layer11_attention_weight_min": -31.725582122802734,
      "activations/layer12_attention_weight_max": 24.01229476928711,
      "activations/layer12_attention_weight_min": -25.18034553527832,
      "activations/layer13_attention_weight_max": 35.381595611572266,
      "activations/layer13_attention_weight_min": -32.852012634277344,
      "activations/layer14_attention_weight_max": 38.11069107055664,
      "activations/layer14_attention_weight_min": -34.728084564208984,
      "activations/layer15_attention_weight_max": 33.9848518371582,
      "activations/layer15_attention_weight_min": -32.39338302612305,
      "activations/layer16_attention_weight_max": 27.615209579467773,
      "activations/layer16_attention_weight_min": -26.876983642578125,
      "activations/layer17_attention_weight_max": 31.94260025024414,
      "activations/layer17_attention_weight_min": -25.97426986694336,
      "activations/layer18_attention_weight_max": 27.93669319152832,
      "activations/layer18_attention_weight_min": -24.820846557617188,
      "activations/layer19_attention_weight_max": 31.135465621948242,
      "activations/layer19_attention_weight_min": -29.748382568359375,
      "activations/layer1_attention_weight_max": 16.287771224975586,
      "activations/layer1_attention_weight_min": -14.450218200683594,
      "activations/layer20_attention_weight_max": 29.781963348388672,
      "activations/layer20_attention_weight_min": -23.16875457763672,
      "activations/layer21_attention_weight_max": 27.5394344329834,
      "activations/layer21_attention_weight_min": -24.801525115966797,
      "activations/layer22_attention_weight_max": 42.264793395996094,
      "activations/layer22_attention_weight_min": -30.0872859954834,
      "activations/layer23_attention_weight_max": 29.397262573242188,
      "activations/layer23_attention_weight_min": -23.57436180114746,
      "activations/layer2_attention_weight_max": 32.313758850097656,
      "activations/layer2_attention_weight_min": -30.762100219726562,
      "activations/layer3_attention_weight_max": 92.89574432373047,
      "activations/layer3_attention_weight_min": -91.19456481933594,
      "activations/layer4_attention_weight_max": 105.04084777832031,
      "activations/layer4_attention_weight_min": -110.47798156738281,
      "activations/layer5_attention_weight_max": 55.863460540771484,
      "activations/layer5_attention_weight_min": -62.65381622314453,
      "activations/layer6_attention_weight_max": 42.70111846923828,
      "activations/layer6_attention_weight_min": -46.80274200439453,
      "activations/layer7_attention_weight_max": 93.37870788574219,
      "activations/layer7_attention_weight_min": -87.82423400878906,
      "activations/layer8_attention_weight_max": 38.340824127197266,
      "activations/layer8_attention_weight_min": -39.48215866088867,
      "activations/layer9_attention_weight_max": 30.071897506713867,
      "activations/layer9_attention_weight_min": -31.386966705322266,
      "epoch": 14.69,
      "learning_rate": 5.5854924242424236e-05,
      "loss": 2.7632,
      "step": 252750
    },
    {
      "activations/layer0_attention_weight_max": 16.15850257873535,
      "activations/layer0_attention_weight_min": -12.527853012084961,
      "activations/layer10_attention_weight_max": 38.20494842529297,
      "activations/layer10_attention_weight_min": -36.96025466918945,
      "activations/layer11_attention_weight_max": 33.981300354003906,
      "activations/layer11_attention_weight_min": -35.55922317504883,
      "activations/layer12_attention_weight_max": 27.407777786254883,
      "activations/layer12_attention_weight_min": -27.496164321899414,
      "activations/layer13_attention_weight_max": 42.04730987548828,
      "activations/layer13_attention_weight_min": -37.37152099609375,
      "activations/layer14_attention_weight_max": 47.81718444824219,
      "activations/layer14_attention_weight_min": -39.78779983520508,
      "activations/layer15_attention_weight_max": 41.67974090576172,
      "activations/layer15_attention_weight_min": -36.899864196777344,
      "activations/layer16_attention_weight_max": 36.83242416381836,
      "activations/layer16_attention_weight_min": -29.672100067138672,
      "activations/layer17_attention_weight_max": 34.29249954223633,
      "activations/layer17_attention_weight_min": -29.38387107849121,
      "activations/layer18_attention_weight_max": 33.76560592651367,
      "activations/layer18_attention_weight_min": -28.03310775756836,
      "activations/layer19_attention_weight_max": 36.95295333862305,
      "activations/layer19_attention_weight_min": -32.892356872558594,
      "activations/layer1_attention_weight_max": 15.99063491821289,
      "activations/layer1_attention_weight_min": -14.015326499938965,
      "activations/layer20_attention_weight_max": 31.756052017211914,
      "activations/layer20_attention_weight_min": -25.823562622070312,
      "activations/layer21_attention_weight_max": 32.612640380859375,
      "activations/layer21_attention_weight_min": -24.74003791809082,
      "activations/layer22_attention_weight_max": 48.009586334228516,
      "activations/layer22_attention_weight_min": -32.1669807434082,
      "activations/layer23_attention_weight_max": 35.340415954589844,
      "activations/layer23_attention_weight_min": -25.558067321777344,
      "activations/layer2_attention_weight_max": 35.53311538696289,
      "activations/layer2_attention_weight_min": -33.478759765625,
      "activations/layer3_attention_weight_max": 103.3652572631836,
      "activations/layer3_attention_weight_min": -102.32646942138672,
      "activations/layer4_attention_weight_max": 111.43727111816406,
      "activations/layer4_attention_weight_min": -113.9483871459961,
      "activations/layer5_attention_weight_max": 52.475563049316406,
      "activations/layer5_attention_weight_min": -66.8133544921875,
      "activations/layer6_attention_weight_max": 43.875911712646484,
      "activations/layer6_attention_weight_min": -47.56277847290039,
      "activations/layer7_attention_weight_max": 95.44951629638672,
      "activations/layer7_attention_weight_min": -99.50507354736328,
      "activations/layer8_attention_weight_max": 42.900001525878906,
      "activations/layer8_attention_weight_min": -46.223976135253906,
      "activations/layer9_attention_weight_max": 32.32601547241211,
      "activations/layer9_attention_weight_min": -35.884517669677734,
      "epoch": 14.69,
      "learning_rate": 5.5835984848484845e-05,
      "loss": 2.7468,
      "step": 252800
    },
    {
      "activations/layer0_attention_weight_max": 15.80444049835205,
      "activations/layer0_attention_weight_min": -13.14316463470459,
      "activations/layer10_attention_weight_max": 39.055946350097656,
      "activations/layer10_attention_weight_min": -37.81279754638672,
      "activations/layer11_attention_weight_max": 40.30634307861328,
      "activations/layer11_attention_weight_min": -39.579036712646484,
      "activations/layer12_attention_weight_max": 30.109439849853516,
      "activations/layer12_attention_weight_min": -28.727182388305664,
      "activations/layer13_attention_weight_max": 45.04696273803711,
      "activations/layer13_attention_weight_min": -37.69437789916992,
      "activations/layer14_attention_weight_max": 48.762638092041016,
      "activations/layer14_attention_weight_min": -41.76515579223633,
      "activations/layer15_attention_weight_max": 41.92938232421875,
      "activations/layer15_attention_weight_min": -38.943485260009766,
      "activations/layer16_attention_weight_max": 34.574771881103516,
      "activations/layer16_attention_weight_min": -30.174108505249023,
      "activations/layer17_attention_weight_max": 34.053375244140625,
      "activations/layer17_attention_weight_min": -27.59493064880371,
      "activations/layer18_attention_weight_max": 33.43260192871094,
      "activations/layer18_attention_weight_min": -25.6917781829834,
      "activations/layer19_attention_weight_max": 40.45841598510742,
      "activations/layer19_attention_weight_min": -33.151546478271484,
      "activations/layer1_attention_weight_max": 15.838187217712402,
      "activations/layer1_attention_weight_min": -15.036032676696777,
      "activations/layer20_attention_weight_max": 36.13119888305664,
      "activations/layer20_attention_weight_min": -25.761699676513672,
      "activations/layer21_attention_weight_max": 33.173126220703125,
      "activations/layer21_attention_weight_min": -25.60980224609375,
      "activations/layer22_attention_weight_max": 55.52708435058594,
      "activations/layer22_attention_weight_min": -33.554649353027344,
      "activations/layer23_attention_weight_max": 36.95610046386719,
      "activations/layer23_attention_weight_min": -25.63768768310547,
      "activations/layer2_attention_weight_max": 33.34751510620117,
      "activations/layer2_attention_weight_min": -32.191795349121094,
      "activations/layer3_attention_weight_max": 101.03644561767578,
      "activations/layer3_attention_weight_min": -97.48663330078125,
      "activations/layer4_attention_weight_max": 114.37322998046875,
      "activations/layer4_attention_weight_min": -110.90193939208984,
      "activations/layer5_attention_weight_max": 52.87480926513672,
      "activations/layer5_attention_weight_min": -65.66268920898438,
      "activations/layer6_attention_weight_max": 45.00822830200195,
      "activations/layer6_attention_weight_min": -47.109458923339844,
      "activations/layer7_attention_weight_max": 97.64166259765625,
      "activations/layer7_attention_weight_min": -94.39602661132812,
      "activations/layer8_attention_weight_max": 42.57072830200195,
      "activations/layer8_attention_weight_min": -43.64140319824219,
      "activations/layer9_attention_weight_max": 36.29734802246094,
      "activations/layer9_attention_weight_min": -36.72208786010742,
      "epoch": 14.69,
      "learning_rate": 5.581704545454545e-05,
      "loss": 2.7428,
      "step": 252850
    },
    {
      "activations/layer0_attention_weight_max": 15.673833847045898,
      "activations/layer0_attention_weight_min": -13.363523483276367,
      "activations/layer10_attention_weight_max": 37.274139404296875,
      "activations/layer10_attention_weight_min": -36.17348861694336,
      "activations/layer11_attention_weight_max": 35.26693344116211,
      "activations/layer11_attention_weight_min": -34.31084442138672,
      "activations/layer12_attention_weight_max": 27.24212074279785,
      "activations/layer12_attention_weight_min": -25.945091247558594,
      "activations/layer13_attention_weight_max": 41.58802795410156,
      "activations/layer13_attention_weight_min": -34.83041000366211,
      "activations/layer14_attention_weight_max": 49.103240966796875,
      "activations/layer14_attention_weight_min": -41.866416931152344,
      "activations/layer15_attention_weight_max": 41.2060661315918,
      "activations/layer15_attention_weight_min": -33.8744010925293,
      "activations/layer16_attention_weight_max": 31.84947395324707,
      "activations/layer16_attention_weight_min": -28.114978790283203,
      "activations/layer17_attention_weight_max": 31.71626853942871,
      "activations/layer17_attention_weight_min": -25.663475036621094,
      "activations/layer18_attention_weight_max": 34.547115325927734,
      "activations/layer18_attention_weight_min": -25.144073486328125,
      "activations/layer19_attention_weight_max": 36.71639633178711,
      "activations/layer19_attention_weight_min": -29.904653549194336,
      "activations/layer1_attention_weight_max": 20.663089752197266,
      "activations/layer1_attention_weight_min": -17.95839500427246,
      "activations/layer20_attention_weight_max": 33.42967224121094,
      "activations/layer20_attention_weight_min": -24.685884475708008,
      "activations/layer21_attention_weight_max": 34.45532989501953,
      "activations/layer21_attention_weight_min": -24.408550262451172,
      "activations/layer22_attention_weight_max": 46.89585494995117,
      "activations/layer22_attention_weight_min": -28.488691329956055,
      "activations/layer23_attention_weight_max": 33.97886657714844,
      "activations/layer23_attention_weight_min": -24.125045776367188,
      "activations/layer2_attention_weight_max": 34.463645935058594,
      "activations/layer2_attention_weight_min": -33.06584930419922,
      "activations/layer3_attention_weight_max": 96.44139099121094,
      "activations/layer3_attention_weight_min": -97.35131072998047,
      "activations/layer4_attention_weight_max": 107.81178283691406,
      "activations/layer4_attention_weight_min": -110.85442352294922,
      "activations/layer5_attention_weight_max": 51.57152557373047,
      "activations/layer5_attention_weight_min": -62.33144760131836,
      "activations/layer6_attention_weight_max": 45.471073150634766,
      "activations/layer6_attention_weight_min": -47.666526794433594,
      "activations/layer7_attention_weight_max": 92.3865737915039,
      "activations/layer7_attention_weight_min": -95.28536987304688,
      "activations/layer8_attention_weight_max": 43.935245513916016,
      "activations/layer8_attention_weight_min": -42.941078186035156,
      "activations/layer9_attention_weight_max": 33.713748931884766,
      "activations/layer9_attention_weight_min": -35.58253860473633,
      "epoch": 14.69,
      "learning_rate": 5.5798106060606054e-05,
      "loss": 2.7629,
      "step": 252900
    },
    {
      "activations/layer0_attention_weight_max": 16.136510848999023,
      "activations/layer0_attention_weight_min": -12.142962455749512,
      "activations/layer10_attention_weight_max": 33.35333251953125,
      "activations/layer10_attention_weight_min": -32.20396423339844,
      "activations/layer11_attention_weight_max": 33.164710998535156,
      "activations/layer11_attention_weight_min": -35.01084899902344,
      "activations/layer12_attention_weight_max": 26.003250122070312,
      "activations/layer12_attention_weight_min": -26.05710220336914,
      "activations/layer13_attention_weight_max": 41.171653747558594,
      "activations/layer13_attention_weight_min": -36.57539367675781,
      "activations/layer14_attention_weight_max": 44.77275848388672,
      "activations/layer14_attention_weight_min": -40.448890686035156,
      "activations/layer15_attention_weight_max": 38.867767333984375,
      "activations/layer15_attention_weight_min": -34.994693756103516,
      "activations/layer16_attention_weight_max": 32.46366882324219,
      "activations/layer16_attention_weight_min": -28.729278564453125,
      "activations/layer17_attention_weight_max": 31.82634925842285,
      "activations/layer17_attention_weight_min": -26.544885635375977,
      "activations/layer18_attention_weight_max": 35.28219985961914,
      "activations/layer18_attention_weight_min": -25.462162017822266,
      "activations/layer19_attention_weight_max": 34.436195373535156,
      "activations/layer19_attention_weight_min": -31.398710250854492,
      "activations/layer1_attention_weight_max": 16.058889389038086,
      "activations/layer1_attention_weight_min": -13.384034156799316,
      "activations/layer20_attention_weight_max": 31.1020565032959,
      "activations/layer20_attention_weight_min": -27.479108810424805,
      "activations/layer21_attention_weight_max": 28.22287368774414,
      "activations/layer21_attention_weight_min": -24.931713104248047,
      "activations/layer22_attention_weight_max": 46.249752044677734,
      "activations/layer22_attention_weight_min": -33.5953254699707,
      "activations/layer23_attention_weight_max": 36.160606384277344,
      "activations/layer23_attention_weight_min": -26.483577728271484,
      "activations/layer2_attention_weight_max": 30.463756561279297,
      "activations/layer2_attention_weight_min": -30.964452743530273,
      "activations/layer3_attention_weight_max": 90.2931137084961,
      "activations/layer3_attention_weight_min": -94.43444061279297,
      "activations/layer4_attention_weight_max": 110.6794662475586,
      "activations/layer4_attention_weight_min": -109.577880859375,
      "activations/layer5_attention_weight_max": 50.82940673828125,
      "activations/layer5_attention_weight_min": -60.77592086791992,
      "activations/layer6_attention_weight_max": 43.69456481933594,
      "activations/layer6_attention_weight_min": -45.17655563354492,
      "activations/layer7_attention_weight_max": 87.98981475830078,
      "activations/layer7_attention_weight_min": -88.42843627929688,
      "activations/layer8_attention_weight_max": 38.8995246887207,
      "activations/layer8_attention_weight_min": -41.892669677734375,
      "activations/layer9_attention_weight_max": 32.01510238647461,
      "activations/layer9_attention_weight_min": -32.43241500854492,
      "epoch": 14.7,
      "learning_rate": 5.5779166666666656e-05,
      "loss": 2.7559,
      "step": 252950
    },
    {
      "activations/layer0_attention_weight_max": 16.34902000427246,
      "activations/layer0_attention_weight_min": -12.619571685791016,
      "activations/layer10_attention_weight_max": 29.6986141204834,
      "activations/layer10_attention_weight_min": -30.82986831665039,
      "activations/layer11_attention_weight_max": 31.295515060424805,
      "activations/layer11_attention_weight_min": -30.400785446166992,
      "activations/layer12_attention_weight_max": 24.54465103149414,
      "activations/layer12_attention_weight_min": -25.689878463745117,
      "activations/layer13_attention_weight_max": 36.781158447265625,
      "activations/layer13_attention_weight_min": -33.0341911315918,
      "activations/layer14_attention_weight_max": 39.982601165771484,
      "activations/layer14_attention_weight_min": -35.29335403442383,
      "activations/layer15_attention_weight_max": 36.56413269042969,
      "activations/layer15_attention_weight_min": -32.640316009521484,
      "activations/layer16_attention_weight_max": 28.577505111694336,
      "activations/layer16_attention_weight_min": -30.606775283813477,
      "activations/layer17_attention_weight_max": 29.717899322509766,
      "activations/layer17_attention_weight_min": -27.512939453125,
      "activations/layer18_attention_weight_max": 31.739839553833008,
      "activations/layer18_attention_weight_min": -23.951019287109375,
      "activations/layer19_attention_weight_max": 32.16677474975586,
      "activations/layer19_attention_weight_min": -33.29823684692383,
      "activations/layer1_attention_weight_max": 16.432693481445312,
      "activations/layer1_attention_weight_min": -14.581762313842773,
      "activations/layer20_attention_weight_max": 30.006092071533203,
      "activations/layer20_attention_weight_min": -25.217296600341797,
      "activations/layer21_attention_weight_max": 28.32168960571289,
      "activations/layer21_attention_weight_min": -23.752464294433594,
      "activations/layer22_attention_weight_max": 37.45075988769531,
      "activations/layer22_attention_weight_min": -29.353879928588867,
      "activations/layer23_attention_weight_max": 33.176597595214844,
      "activations/layer23_attention_weight_min": -24.182907104492188,
      "activations/layer2_attention_weight_max": 32.73090362548828,
      "activations/layer2_attention_weight_min": -31.930580139160156,
      "activations/layer3_attention_weight_max": 98.95976257324219,
      "activations/layer3_attention_weight_min": -98.10028076171875,
      "activations/layer4_attention_weight_max": 109.9756088256836,
      "activations/layer4_attention_weight_min": -112.77495574951172,
      "activations/layer5_attention_weight_max": 53.14179611206055,
      "activations/layer5_attention_weight_min": -63.45360565185547,
      "activations/layer6_attention_weight_max": 43.387550354003906,
      "activations/layer6_attention_weight_min": -46.58536911010742,
      "activations/layer7_attention_weight_max": 91.95976257324219,
      "activations/layer7_attention_weight_min": -97.01031494140625,
      "activations/layer8_attention_weight_max": 38.10318374633789,
      "activations/layer8_attention_weight_min": -42.260101318359375,
      "activations/layer9_attention_weight_max": 30.752033233642578,
      "activations/layer9_attention_weight_min": -31.90846824645996,
      "epoch": 14.7,
      "learning_rate": 5.576022727272727e-05,
      "loss": 2.7481,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.5244,
      "eval_samples_per_second": 503.728,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.5244,
      "eval_openwebtext_samples_per_second": 503.728,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9928,
      "eval_wikitext_samples_per_second": 228.818,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_lambada_loss": 2.439453125,
      "eval_lambada_ppl": 11.46676813896006,
      "eval_lambada_runtime": 9.5982,
      "eval_lambada_samples_per_second": 507.282,
      "step": 253000
    },
    {
      "activations/layer0_attention_weight_max": 16.50399398803711,
      "activations/layer0_attention_weight_min": -12.837183952331543,
      "activations/layer10_attention_weight_max": 34.85161590576172,
      "activations/layer10_attention_weight_min": -36.86237716674805,
      "activations/layer11_attention_weight_max": 34.80019760131836,
      "activations/layer11_attention_weight_min": -35.39722442626953,
      "activations/layer12_attention_weight_max": 26.295669555664062,
      "activations/layer12_attention_weight_min": -25.82709503173828,
      "activations/layer13_attention_weight_max": 38.23452377319336,
      "activations/layer13_attention_weight_min": -33.916465759277344,
      "activations/layer14_attention_weight_max": 41.9277229309082,
      "activations/layer14_attention_weight_min": -35.928314208984375,
      "activations/layer15_attention_weight_max": 37.62249755859375,
      "activations/layer15_attention_weight_min": -33.9261589050293,
      "activations/layer16_attention_weight_max": 29.45622444152832,
      "activations/layer16_attention_weight_min": -28.793378829956055,
      "activations/layer17_attention_weight_max": 33.0660285949707,
      "activations/layer17_attention_weight_min": -26.136699676513672,
      "activations/layer18_attention_weight_max": 31.818946838378906,
      "activations/layer18_attention_weight_min": -26.829875946044922,
      "activations/layer19_attention_weight_max": 39.478450775146484,
      "activations/layer19_attention_weight_min": -31.043447494506836,
      "activations/layer1_attention_weight_max": 16.15889549255371,
      "activations/layer1_attention_weight_min": -13.416942596435547,
      "activations/layer20_attention_weight_max": 35.260765075683594,
      "activations/layer20_attention_weight_min": -26.21245765686035,
      "activations/layer21_attention_weight_max": 28.929967880249023,
      "activations/layer21_attention_weight_min": -24.0528564453125,
      "activations/layer22_attention_weight_max": 44.43400192260742,
      "activations/layer22_attention_weight_min": -32.78605270385742,
      "activations/layer23_attention_weight_max": 33.89167785644531,
      "activations/layer23_attention_weight_min": -27.552478790283203,
      "activations/layer2_attention_weight_max": 32.95362091064453,
      "activations/layer2_attention_weight_min": -32.09876251220703,
      "activations/layer3_attention_weight_max": 95.9755630493164,
      "activations/layer3_attention_weight_min": -95.51973724365234,
      "activations/layer4_attention_weight_max": 110.99657440185547,
      "activations/layer4_attention_weight_min": -107.60725402832031,
      "activations/layer5_attention_weight_max": 50.46809005737305,
      "activations/layer5_attention_weight_min": -63.734649658203125,
      "activations/layer6_attention_weight_max": 44.982608795166016,
      "activations/layer6_attention_weight_min": -48.891014099121094,
      "activations/layer7_attention_weight_max": 93.92139434814453,
      "activations/layer7_attention_weight_min": -96.34214782714844,
      "activations/layer8_attention_weight_max": 41.92765426635742,
      "activations/layer8_attention_weight_min": -48.947235107421875,
      "activations/layer9_attention_weight_max": 34.09060287475586,
      "activations/layer9_attention_weight_min": -35.26526641845703,
      "epoch": 14.7,
      "learning_rate": 5.574128787878787e-05,
      "loss": 2.7614,
      "step": 253050
    },
    {
      "activations/layer0_attention_weight_max": 15.592209815979004,
      "activations/layer0_attention_weight_min": -12.396472930908203,
      "activations/layer10_attention_weight_max": 32.46484375,
      "activations/layer10_attention_weight_min": -32.65003204345703,
      "activations/layer11_attention_weight_max": 33.357444763183594,
      "activations/layer11_attention_weight_min": -33.268672943115234,
      "activations/layer12_attention_weight_max": 25.328779220581055,
      "activations/layer12_attention_weight_min": -26.38705062866211,
      "activations/layer13_attention_weight_max": 37.88932800292969,
      "activations/layer13_attention_weight_min": -34.687095642089844,
      "activations/layer14_attention_weight_max": 41.477386474609375,
      "activations/layer14_attention_weight_min": -39.117733001708984,
      "activations/layer15_attention_weight_max": 35.291168212890625,
      "activations/layer15_attention_weight_min": -32.627891540527344,
      "activations/layer16_attention_weight_max": 28.659481048583984,
      "activations/layer16_attention_weight_min": -27.431577682495117,
      "activations/layer17_attention_weight_max": 33.347259521484375,
      "activations/layer17_attention_weight_min": -26.232925415039062,
      "activations/layer18_attention_weight_max": 29.995145797729492,
      "activations/layer18_attention_weight_min": -25.268550872802734,
      "activations/layer19_attention_weight_max": 34.27678680419922,
      "activations/layer19_attention_weight_min": -32.87238693237305,
      "activations/layer1_attention_weight_max": 15.3608980178833,
      "activations/layer1_attention_weight_min": -14.164950370788574,
      "activations/layer20_attention_weight_max": 34.678531646728516,
      "activations/layer20_attention_weight_min": -25.15411376953125,
      "activations/layer21_attention_weight_max": 30.57308578491211,
      "activations/layer21_attention_weight_min": -24.70253562927246,
      "activations/layer22_attention_weight_max": 42.24356460571289,
      "activations/layer22_attention_weight_min": -30.19086265563965,
      "activations/layer23_attention_weight_max": 35.31103515625,
      "activations/layer23_attention_weight_min": -27.082958221435547,
      "activations/layer2_attention_weight_max": 31.773178100585938,
      "activations/layer2_attention_weight_min": -32.15137481689453,
      "activations/layer3_attention_weight_max": 94.10125732421875,
      "activations/layer3_attention_weight_min": -97.09547424316406,
      "activations/layer4_attention_weight_max": 104.875244140625,
      "activations/layer4_attention_weight_min": -108.58074188232422,
      "activations/layer5_attention_weight_max": 50.580928802490234,
      "activations/layer5_attention_weight_min": -62.28310775756836,
      "activations/layer6_attention_weight_max": 41.05453109741211,
      "activations/layer6_attention_weight_min": -44.10415267944336,
      "activations/layer7_attention_weight_max": 86.8519058227539,
      "activations/layer7_attention_weight_min": -89.40294647216797,
      "activations/layer8_attention_weight_max": 41.148380279541016,
      "activations/layer8_attention_weight_min": -41.78247833251953,
      "activations/layer9_attention_weight_max": 32.95561218261719,
      "activations/layer9_attention_weight_min": -34.097381591796875,
      "epoch": 14.71,
      "learning_rate": 5.572234848484848e-05,
      "loss": 2.7716,
      "step": 253100
    },
    {
      "activations/layer0_attention_weight_max": 16.724693298339844,
      "activations/layer0_attention_weight_min": -13.153470039367676,
      "activations/layer10_attention_weight_max": 43.0789794921875,
      "activations/layer10_attention_weight_min": -38.136863708496094,
      "activations/layer11_attention_weight_max": 40.1231575012207,
      "activations/layer11_attention_weight_min": -37.96820831298828,
      "activations/layer12_attention_weight_max": 33.4320068359375,
      "activations/layer12_attention_weight_min": -26.521120071411133,
      "activations/layer13_attention_weight_max": 49.98160171508789,
      "activations/layer13_attention_weight_min": -37.398895263671875,
      "activations/layer14_attention_weight_max": 45.848785400390625,
      "activations/layer14_attention_weight_min": -41.039955139160156,
      "activations/layer15_attention_weight_max": 45.46245574951172,
      "activations/layer15_attention_weight_min": -39.71218490600586,
      "activations/layer16_attention_weight_max": 39.349937438964844,
      "activations/layer16_attention_weight_min": -30.51738166809082,
      "activations/layer17_attention_weight_max": 29.24846076965332,
      "activations/layer17_attention_weight_min": -27.123470306396484,
      "activations/layer18_attention_weight_max": 34.06890106201172,
      "activations/layer18_attention_weight_min": -27.34316635131836,
      "activations/layer19_attention_weight_max": 36.57843017578125,
      "activations/layer19_attention_weight_min": -35.065670013427734,
      "activations/layer1_attention_weight_max": 15.959101676940918,
      "activations/layer1_attention_weight_min": -14.194361686706543,
      "activations/layer20_attention_weight_max": 29.530088424682617,
      "activations/layer20_attention_weight_min": -25.430028915405273,
      "activations/layer21_attention_weight_max": 28.24694061279297,
      "activations/layer21_attention_weight_min": -24.03566551208496,
      "activations/layer22_attention_weight_max": 50.28892517089844,
      "activations/layer22_attention_weight_min": -31.2919921875,
      "activations/layer23_attention_weight_max": 37.79859161376953,
      "activations/layer23_attention_weight_min": -22.902629852294922,
      "activations/layer2_attention_weight_max": 33.77693176269531,
      "activations/layer2_attention_weight_min": -33.204200744628906,
      "activations/layer3_attention_weight_max": 98.56966400146484,
      "activations/layer3_attention_weight_min": -103.4423599243164,
      "activations/layer4_attention_weight_max": 110.38793182373047,
      "activations/layer4_attention_weight_min": -112.0029296875,
      "activations/layer5_attention_weight_max": 51.432003021240234,
      "activations/layer5_attention_weight_min": -62.69550704956055,
      "activations/layer6_attention_weight_max": 47.60246276855469,
      "activations/layer6_attention_weight_min": -46.635677337646484,
      "activations/layer7_attention_weight_max": 103.97679138183594,
      "activations/layer7_attention_weight_min": -101.34358215332031,
      "activations/layer8_attention_weight_max": 48.72068786621094,
      "activations/layer8_attention_weight_min": -47.85341262817383,
      "activations/layer9_attention_weight_max": 37.35661315917969,
      "activations/layer9_attention_weight_min": -37.931640625,
      "epoch": 14.71,
      "learning_rate": 5.570340909090908e-05,
      "loss": 2.7485,
      "step": 253150
    },
    {
      "activations/layer0_attention_weight_max": 15.870271682739258,
      "activations/layer0_attention_weight_min": -13.088994026184082,
      "activations/layer10_attention_weight_max": 34.37232208251953,
      "activations/layer10_attention_weight_min": -33.511863708496094,
      "activations/layer11_attention_weight_max": 31.999380111694336,
      "activations/layer11_attention_weight_min": -33.251888275146484,
      "activations/layer12_attention_weight_max": 24.97722816467285,
      "activations/layer12_attention_weight_min": -25.790189743041992,
      "activations/layer13_attention_weight_max": 37.26183319091797,
      "activations/layer13_attention_weight_min": -35.86275863647461,
      "activations/layer14_attention_weight_max": 41.6539421081543,
      "activations/layer14_attention_weight_min": -38.286346435546875,
      "activations/layer15_attention_weight_max": 37.29469299316406,
      "activations/layer15_attention_weight_min": -35.894935607910156,
      "activations/layer16_attention_weight_max": 27.713960647583008,
      "activations/layer16_attention_weight_min": -28.315176010131836,
      "activations/layer17_attention_weight_max": 30.157461166381836,
      "activations/layer17_attention_weight_min": -24.98545265197754,
      "activations/layer18_attention_weight_max": 32.8101692199707,
      "activations/layer18_attention_weight_min": -27.698659896850586,
      "activations/layer19_attention_weight_max": 34.82637405395508,
      "activations/layer19_attention_weight_min": -32.51603698730469,
      "activations/layer1_attention_weight_max": 15.846973419189453,
      "activations/layer1_attention_weight_min": -13.324406623840332,
      "activations/layer20_attention_weight_max": 28.3082332611084,
      "activations/layer20_attention_weight_min": -25.191001892089844,
      "activations/layer21_attention_weight_max": 29.63629150390625,
      "activations/layer21_attention_weight_min": -24.802934646606445,
      "activations/layer22_attention_weight_max": 40.97509765625,
      "activations/layer22_attention_weight_min": -31.393796920776367,
      "activations/layer23_attention_weight_max": 32.26788330078125,
      "activations/layer23_attention_weight_min": -27.920059204101562,
      "activations/layer2_attention_weight_max": 31.462181091308594,
      "activations/layer2_attention_weight_min": -30.67903709411621,
      "activations/layer3_attention_weight_max": 94.59265899658203,
      "activations/layer3_attention_weight_min": -95.86494445800781,
      "activations/layer4_attention_weight_max": 107.18562316894531,
      "activations/layer4_attention_weight_min": -115.24462127685547,
      "activations/layer5_attention_weight_max": 51.182884216308594,
      "activations/layer5_attention_weight_min": -65.52873229980469,
      "activations/layer6_attention_weight_max": 43.78957748413086,
      "activations/layer6_attention_weight_min": -47.84093475341797,
      "activations/layer7_attention_weight_max": 91.44711303710938,
      "activations/layer7_attention_weight_min": -95.01893615722656,
      "activations/layer8_attention_weight_max": 41.569217681884766,
      "activations/layer8_attention_weight_min": -44.427330017089844,
      "activations/layer9_attention_weight_max": 33.24803924560547,
      "activations/layer9_attention_weight_min": -33.4039306640625,
      "epoch": 14.71,
      "learning_rate": 5.56844696969697e-05,
      "loss": 2.7811,
      "step": 253200
    },
    {
      "activations/layer0_attention_weight_max": 16.427448272705078,
      "activations/layer0_attention_weight_min": -12.28194808959961,
      "activations/layer10_attention_weight_max": 33.90037536621094,
      "activations/layer10_attention_weight_min": -33.18092727661133,
      "activations/layer11_attention_weight_max": 31.480224609375,
      "activations/layer11_attention_weight_min": -33.50863265991211,
      "activations/layer12_attention_weight_max": 26.97031593322754,
      "activations/layer12_attention_weight_min": -28.990079879760742,
      "activations/layer13_attention_weight_max": 38.668060302734375,
      "activations/layer13_attention_weight_min": -37.05678176879883,
      "activations/layer14_attention_weight_max": 45.64560317993164,
      "activations/layer14_attention_weight_min": -39.77672576904297,
      "activations/layer15_attention_weight_max": 38.18857192993164,
      "activations/layer15_attention_weight_min": -36.31549072265625,
      "activations/layer16_attention_weight_max": 32.90818405151367,
      "activations/layer16_attention_weight_min": -27.51433563232422,
      "activations/layer17_attention_weight_max": 29.593881607055664,
      "activations/layer17_attention_weight_min": -25.194713592529297,
      "activations/layer18_attention_weight_max": 31.5747013092041,
      "activations/layer18_attention_weight_min": -23.856582641601562,
      "activations/layer19_attention_weight_max": 32.60825729370117,
      "activations/layer19_attention_weight_min": -30.86690330505371,
      "activations/layer1_attention_weight_max": 16.508848190307617,
      "activations/layer1_attention_weight_min": -13.692338943481445,
      "activations/layer20_attention_weight_max": 29.036785125732422,
      "activations/layer20_attention_weight_min": -25.089731216430664,
      "activations/layer21_attention_weight_max": 28.32451057434082,
      "activations/layer21_attention_weight_min": -24.750516891479492,
      "activations/layer22_attention_weight_max": 42.80034637451172,
      "activations/layer22_attention_weight_min": -31.46903419494629,
      "activations/layer23_attention_weight_max": 35.67972946166992,
      "activations/layer23_attention_weight_min": -25.429845809936523,
      "activations/layer2_attention_weight_max": 35.213199615478516,
      "activations/layer2_attention_weight_min": -33.104042053222656,
      "activations/layer3_attention_weight_max": 96.35518646240234,
      "activations/layer3_attention_weight_min": -97.60629272460938,
      "activations/layer4_attention_weight_max": 106.62898254394531,
      "activations/layer4_attention_weight_min": -107.73634338378906,
      "activations/layer5_attention_weight_max": 53.59275817871094,
      "activations/layer5_attention_weight_min": -62.38331604003906,
      "activations/layer6_attention_weight_max": 41.89238739013672,
      "activations/layer6_attention_weight_min": -46.27802658081055,
      "activations/layer7_attention_weight_max": 90.52192687988281,
      "activations/layer7_attention_weight_min": -88.05176544189453,
      "activations/layer8_attention_weight_max": 40.84638595581055,
      "activations/layer8_attention_weight_min": -41.51780700683594,
      "activations/layer9_attention_weight_max": 31.320356369018555,
      "activations/layer9_attention_weight_min": -32.84293746948242,
      "epoch": 14.72,
      "learning_rate": 5.56655303030303e-05,
      "loss": 2.7533,
      "step": 253250
    },
    {
      "activations/layer0_attention_weight_max": 16.233642578125,
      "activations/layer0_attention_weight_min": -12.701172828674316,
      "activations/layer10_attention_weight_max": 33.25526809692383,
      "activations/layer10_attention_weight_min": -31.901580810546875,
      "activations/layer11_attention_weight_max": 31.501094818115234,
      "activations/layer11_attention_weight_min": -32.204017639160156,
      "activations/layer12_attention_weight_max": 24.74606704711914,
      "activations/layer12_attention_weight_min": -24.884037017822266,
      "activations/layer13_attention_weight_max": 37.512786865234375,
      "activations/layer13_attention_weight_min": -32.68238067626953,
      "activations/layer14_attention_weight_max": 43.940521240234375,
      "activations/layer14_attention_weight_min": -35.64439010620117,
      "activations/layer15_attention_weight_max": 36.24421310424805,
      "activations/layer15_attention_weight_min": -32.48941421508789,
      "activations/layer16_attention_weight_max": 30.06627082824707,
      "activations/layer16_attention_weight_min": -27.327533721923828,
      "activations/layer17_attention_weight_max": 31.58572769165039,
      "activations/layer17_attention_weight_min": -25.65281105041504,
      "activations/layer18_attention_weight_max": 31.45197868347168,
      "activations/layer18_attention_weight_min": -22.872150421142578,
      "activations/layer19_attention_weight_max": 32.02632141113281,
      "activations/layer19_attention_weight_min": -30.833301544189453,
      "activations/layer1_attention_weight_max": 16.064990997314453,
      "activations/layer1_attention_weight_min": -14.525723457336426,
      "activations/layer20_attention_weight_max": 28.199905395507812,
      "activations/layer20_attention_weight_min": -24.160247802734375,
      "activations/layer21_attention_weight_max": 27.34554672241211,
      "activations/layer21_attention_weight_min": -23.263063430786133,
      "activations/layer22_attention_weight_max": 43.853111267089844,
      "activations/layer22_attention_weight_min": -29.497833251953125,
      "activations/layer23_attention_weight_max": 31.94738006591797,
      "activations/layer23_attention_weight_min": -24.93419647216797,
      "activations/layer2_attention_weight_max": 30.681396484375,
      "activations/layer2_attention_weight_min": -30.387012481689453,
      "activations/layer3_attention_weight_max": 91.36900329589844,
      "activations/layer3_attention_weight_min": -96.54415130615234,
      "activations/layer4_attention_weight_max": 107.27544403076172,
      "activations/layer4_attention_weight_min": -108.04475402832031,
      "activations/layer5_attention_weight_max": 53.36178207397461,
      "activations/layer5_attention_weight_min": -61.426025390625,
      "activations/layer6_attention_weight_max": 44.3990592956543,
      "activations/layer6_attention_weight_min": -45.82735824584961,
      "activations/layer7_attention_weight_max": 92.06757354736328,
      "activations/layer7_attention_weight_min": -91.2425308227539,
      "activations/layer8_attention_weight_max": 38.95625686645508,
      "activations/layer8_attention_weight_min": -42.59518051147461,
      "activations/layer9_attention_weight_max": 31.389474868774414,
      "activations/layer9_attention_weight_min": -32.7317008972168,
      "epoch": 14.72,
      "learning_rate": 5.56465909090909e-05,
      "loss": 2.7559,
      "step": 253300
    },
    {
      "activations/layer0_attention_weight_max": 16.432897567749023,
      "activations/layer0_attention_weight_min": -13.1405029296875,
      "activations/layer10_attention_weight_max": 31.99765968322754,
      "activations/layer10_attention_weight_min": -31.533653259277344,
      "activations/layer11_attention_weight_max": 31.54947280883789,
      "activations/layer11_attention_weight_min": -32.10423278808594,
      "activations/layer12_attention_weight_max": 25.990676879882812,
      "activations/layer12_attention_weight_min": -26.701889038085938,
      "activations/layer13_attention_weight_max": 41.71366500854492,
      "activations/layer13_attention_weight_min": -35.69947052001953,
      "activations/layer14_attention_weight_max": 46.00116729736328,
      "activations/layer14_attention_weight_min": -39.78055953979492,
      "activations/layer15_attention_weight_max": 41.62173080444336,
      "activations/layer15_attention_weight_min": -36.34312438964844,
      "activations/layer16_attention_weight_max": 30.279680252075195,
      "activations/layer16_attention_weight_min": -28.81770896911621,
      "activations/layer17_attention_weight_max": 30.16388702392578,
      "activations/layer17_attention_weight_min": -26.58614730834961,
      "activations/layer18_attention_weight_max": 33.034908294677734,
      "activations/layer18_attention_weight_min": -25.87466049194336,
      "activations/layer19_attention_weight_max": 35.382259368896484,
      "activations/layer19_attention_weight_min": -30.826322555541992,
      "activations/layer1_attention_weight_max": 16.685806274414062,
      "activations/layer1_attention_weight_min": -14.478654861450195,
      "activations/layer20_attention_weight_max": 32.29865264892578,
      "activations/layer20_attention_weight_min": -26.33958625793457,
      "activations/layer21_attention_weight_max": 28.996074676513672,
      "activations/layer21_attention_weight_min": -24.60487937927246,
      "activations/layer22_attention_weight_max": 47.44202423095703,
      "activations/layer22_attention_weight_min": -30.27410888671875,
      "activations/layer23_attention_weight_max": 35.8209228515625,
      "activations/layer23_attention_weight_min": -25.951183319091797,
      "activations/layer2_attention_weight_max": 32.658660888671875,
      "activations/layer2_attention_weight_min": -32.40317153930664,
      "activations/layer3_attention_weight_max": 96.45673370361328,
      "activations/layer3_attention_weight_min": -101.7535629272461,
      "activations/layer4_attention_weight_max": 109.73162078857422,
      "activations/layer4_attention_weight_min": -109.58668518066406,
      "activations/layer5_attention_weight_max": 51.02904510498047,
      "activations/layer5_attention_weight_min": -61.7242431640625,
      "activations/layer6_attention_weight_max": 43.84823989868164,
      "activations/layer6_attention_weight_min": -45.846981048583984,
      "activations/layer7_attention_weight_max": 91.8899154663086,
      "activations/layer7_attention_weight_min": -92.28106689453125,
      "activations/layer8_attention_weight_max": 38.590721130371094,
      "activations/layer8_attention_weight_min": -43.40579605102539,
      "activations/layer9_attention_weight_max": 31.30315589904785,
      "activations/layer9_attention_weight_min": -32.86600875854492,
      "epoch": 14.72,
      "learning_rate": 5.5628030303030305e-05,
      "loss": 2.7461,
      "step": 253350
    },
    {
      "activations/layer0_attention_weight_max": 16.724925994873047,
      "activations/layer0_attention_weight_min": -12.587674140930176,
      "activations/layer10_attention_weight_max": 35.67239761352539,
      "activations/layer10_attention_weight_min": -36.39063262939453,
      "activations/layer11_attention_weight_max": 35.51750946044922,
      "activations/layer11_attention_weight_min": -38.2398681640625,
      "activations/layer12_attention_weight_max": 28.11992645263672,
      "activations/layer12_attention_weight_min": -27.54806900024414,
      "activations/layer13_attention_weight_max": 41.15947341918945,
      "activations/layer13_attention_weight_min": -36.63662338256836,
      "activations/layer14_attention_weight_max": 47.67512512207031,
      "activations/layer14_attention_weight_min": -40.98667526245117,
      "activations/layer15_attention_weight_max": 41.28305435180664,
      "activations/layer15_attention_weight_min": -39.18646240234375,
      "activations/layer16_attention_weight_max": 33.270877838134766,
      "activations/layer16_attention_weight_min": -29.51776885986328,
      "activations/layer17_attention_weight_max": 31.027048110961914,
      "activations/layer17_attention_weight_min": -27.90268898010254,
      "activations/layer18_attention_weight_max": 33.26826858520508,
      "activations/layer18_attention_weight_min": -25.553468704223633,
      "activations/layer19_attention_weight_max": 36.91672134399414,
      "activations/layer19_attention_weight_min": -31.893110275268555,
      "activations/layer1_attention_weight_max": 15.70969009399414,
      "activations/layer1_attention_weight_min": -16.942066192626953,
      "activations/layer20_attention_weight_max": 32.29884719848633,
      "activations/layer20_attention_weight_min": -26.336814880371094,
      "activations/layer21_attention_weight_max": 30.26176643371582,
      "activations/layer21_attention_weight_min": -24.614177703857422,
      "activations/layer22_attention_weight_max": 51.45284652709961,
      "activations/layer22_attention_weight_min": -34.82231140136719,
      "activations/layer23_attention_weight_max": 35.09626007080078,
      "activations/layer23_attention_weight_min": -28.237051010131836,
      "activations/layer2_attention_weight_max": 34.201698303222656,
      "activations/layer2_attention_weight_min": -31.76589012145996,
      "activations/layer3_attention_weight_max": 97.38748931884766,
      "activations/layer3_attention_weight_min": -98.36835479736328,
      "activations/layer4_attention_weight_max": 112.3921127319336,
      "activations/layer4_attention_weight_min": -111.56536102294922,
      "activations/layer5_attention_weight_max": 53.71099853515625,
      "activations/layer5_attention_weight_min": -63.4675407409668,
      "activations/layer6_attention_weight_max": 44.83595275878906,
      "activations/layer6_attention_weight_min": -47.792930603027344,
      "activations/layer7_attention_weight_max": 92.60306549072266,
      "activations/layer7_attention_weight_min": -92.1205825805664,
      "activations/layer8_attention_weight_max": 43.33351135253906,
      "activations/layer8_attention_weight_min": -44.72608184814453,
      "activations/layer9_attention_weight_max": 34.28044128417969,
      "activations/layer9_attention_weight_min": -34.76497268676758,
      "epoch": 14.72,
      "learning_rate": 5.5609090909090906e-05,
      "loss": 2.7715,
      "step": 253400
    },
    {
      "activations/layer0_attention_weight_max": 16.31960105895996,
      "activations/layer0_attention_weight_min": -12.796463966369629,
      "activations/layer10_attention_weight_max": 33.383689880371094,
      "activations/layer10_attention_weight_min": -34.356571197509766,
      "activations/layer11_attention_weight_max": 33.2426643371582,
      "activations/layer11_attention_weight_min": -32.762413024902344,
      "activations/layer12_attention_weight_max": 26.991901397705078,
      "activations/layer12_attention_weight_min": -25.57720184326172,
      "activations/layer13_attention_weight_max": 41.02177047729492,
      "activations/layer13_attention_weight_min": -35.45762252807617,
      "activations/layer14_attention_weight_max": 43.93598937988281,
      "activations/layer14_attention_weight_min": -37.509063720703125,
      "activations/layer15_attention_weight_max": 38.00627899169922,
      "activations/layer15_attention_weight_min": -33.38629913330078,
      "activations/layer16_attention_weight_max": 30.468517303466797,
      "activations/layer16_attention_weight_min": -27.619216918945312,
      "activations/layer17_attention_weight_max": 31.42225456237793,
      "activations/layer17_attention_weight_min": -25.78641700744629,
      "activations/layer18_attention_weight_max": 33.80707550048828,
      "activations/layer18_attention_weight_min": -25.186994552612305,
      "activations/layer19_attention_weight_max": 34.50897979736328,
      "activations/layer19_attention_weight_min": -31.654157638549805,
      "activations/layer1_attention_weight_max": 16.306480407714844,
      "activations/layer1_attention_weight_min": -15.706631660461426,
      "activations/layer20_attention_weight_max": 30.70733642578125,
      "activations/layer20_attention_weight_min": -25.908790588378906,
      "activations/layer21_attention_weight_max": 29.53337860107422,
      "activations/layer21_attention_weight_min": -24.316091537475586,
      "activations/layer22_attention_weight_max": 44.53089141845703,
      "activations/layer22_attention_weight_min": -30.825435638427734,
      "activations/layer23_attention_weight_max": 36.325260162353516,
      "activations/layer23_attention_weight_min": -28.09438705444336,
      "activations/layer2_attention_weight_max": 34.602638244628906,
      "activations/layer2_attention_weight_min": -32.574859619140625,
      "activations/layer3_attention_weight_max": 99.63338470458984,
      "activations/layer3_attention_weight_min": -98.87456512451172,
      "activations/layer4_attention_weight_max": 114.5059585571289,
      "activations/layer4_attention_weight_min": -109.0182113647461,
      "activations/layer5_attention_weight_max": 52.186302185058594,
      "activations/layer5_attention_weight_min": -62.42521667480469,
      "activations/layer6_attention_weight_max": 45.330692291259766,
      "activations/layer6_attention_weight_min": -46.093746185302734,
      "activations/layer7_attention_weight_max": 90.9009780883789,
      "activations/layer7_attention_weight_min": -92.14594268798828,
      "activations/layer8_attention_weight_max": 41.75093460083008,
      "activations/layer8_attention_weight_min": -41.51356887817383,
      "activations/layer9_attention_weight_max": 33.305824279785156,
      "activations/layer9_attention_weight_min": -33.77754592895508,
      "epoch": 14.73,
      "learning_rate": 5.559015151515151e-05,
      "loss": 2.7343,
      "step": 253450
    },
    {
      "activations/layer0_attention_weight_max": 16.58879852294922,
      "activations/layer0_attention_weight_min": -14.178898811340332,
      "activations/layer10_attention_weight_max": 34.93479537963867,
      "activations/layer10_attention_weight_min": -35.70991516113281,
      "activations/layer11_attention_weight_max": 35.705047607421875,
      "activations/layer11_attention_weight_min": -35.274497985839844,
      "activations/layer12_attention_weight_max": 28.44049835205078,
      "activations/layer12_attention_weight_min": -27.185958862304688,
      "activations/layer13_attention_weight_max": 41.83805847167969,
      "activations/layer13_attention_weight_min": -36.510986328125,
      "activations/layer14_attention_weight_max": 48.251434326171875,
      "activations/layer14_attention_weight_min": -38.00014877319336,
      "activations/layer15_attention_weight_max": 44.021522521972656,
      "activations/layer15_attention_weight_min": -34.92962646484375,
      "activations/layer16_attention_weight_max": 34.4476432800293,
      "activations/layer16_attention_weight_min": -28.837339401245117,
      "activations/layer17_attention_weight_max": 33.81727981567383,
      "activations/layer17_attention_weight_min": -26.68321990966797,
      "activations/layer18_attention_weight_max": 32.856163024902344,
      "activations/layer18_attention_weight_min": -25.917322158813477,
      "activations/layer19_attention_weight_max": 36.58620071411133,
      "activations/layer19_attention_weight_min": -32.85438919067383,
      "activations/layer1_attention_weight_max": 16.984939575195312,
      "activations/layer1_attention_weight_min": -15.059065818786621,
      "activations/layer20_attention_weight_max": 32.635433197021484,
      "activations/layer20_attention_weight_min": -27.363975524902344,
      "activations/layer21_attention_weight_max": 31.690166473388672,
      "activations/layer21_attention_weight_min": -25.826810836791992,
      "activations/layer22_attention_weight_max": 55.94473648071289,
      "activations/layer22_attention_weight_min": -35.084659576416016,
      "activations/layer23_attention_weight_max": 38.6348876953125,
      "activations/layer23_attention_weight_min": -27.48431396484375,
      "activations/layer2_attention_weight_max": 31.708885192871094,
      "activations/layer2_attention_weight_min": -30.694721221923828,
      "activations/layer3_attention_weight_max": 95.68828582763672,
      "activations/layer3_attention_weight_min": -95.79815673828125,
      "activations/layer4_attention_weight_max": 109.93437957763672,
      "activations/layer4_attention_weight_min": -113.87200164794922,
      "activations/layer5_attention_weight_max": 51.84833526611328,
      "activations/layer5_attention_weight_min": -62.516841888427734,
      "activations/layer6_attention_weight_max": 45.54967498779297,
      "activations/layer6_attention_weight_min": -46.85605239868164,
      "activations/layer7_attention_weight_max": 93.87911987304688,
      "activations/layer7_attention_weight_min": -90.18929290771484,
      "activations/layer8_attention_weight_max": 42.8428840637207,
      "activations/layer8_attention_weight_min": -42.7269287109375,
      "activations/layer9_attention_weight_max": 33.23987579345703,
      "activations/layer9_attention_weight_min": -33.09489440917969,
      "epoch": 14.73,
      "learning_rate": 5.5571212121212116e-05,
      "loss": 2.7469,
      "step": 253500
    },
    {
      "activations/layer0_attention_weight_max": 15.963830947875977,
      "activations/layer0_attention_weight_min": -12.912590026855469,
      "activations/layer10_attention_weight_max": 33.45109939575195,
      "activations/layer10_attention_weight_min": -32.70454788208008,
      "activations/layer11_attention_weight_max": 34.61756896972656,
      "activations/layer11_attention_weight_min": -32.07808303833008,
      "activations/layer12_attention_weight_max": 24.616819381713867,
      "activations/layer12_attention_weight_min": -25.427213668823242,
      "activations/layer13_attention_weight_max": 37.617000579833984,
      "activations/layer13_attention_weight_min": -34.17789840698242,
      "activations/layer14_attention_weight_max": 41.941375732421875,
      "activations/layer14_attention_weight_min": -36.79987335205078,
      "activations/layer15_attention_weight_max": 36.81480026245117,
      "activations/layer15_attention_weight_min": -32.05561828613281,
      "activations/layer16_attention_weight_max": 29.57731056213379,
      "activations/layer16_attention_weight_min": -27.702733993530273,
      "activations/layer17_attention_weight_max": 27.157712936401367,
      "activations/layer17_attention_weight_min": -23.22103500366211,
      "activations/layer18_attention_weight_max": 29.428865432739258,
      "activations/layer18_attention_weight_min": -24.32878303527832,
      "activations/layer19_attention_weight_max": 29.837223052978516,
      "activations/layer19_attention_weight_min": -31.475072860717773,
      "activations/layer1_attention_weight_max": 18.297821044921875,
      "activations/layer1_attention_weight_min": -13.99236011505127,
      "activations/layer20_attention_weight_max": 26.822763442993164,
      "activations/layer20_attention_weight_min": -24.458263397216797,
      "activations/layer21_attention_weight_max": 24.4277286529541,
      "activations/layer21_attention_weight_min": -22.12706756591797,
      "activations/layer22_attention_weight_max": 37.84265899658203,
      "activations/layer22_attention_weight_min": -27.563720703125,
      "activations/layer23_attention_weight_max": 29.578115463256836,
      "activations/layer23_attention_weight_min": -24.226362228393555,
      "activations/layer2_attention_weight_max": 34.72246170043945,
      "activations/layer2_attention_weight_min": -30.627696990966797,
      "activations/layer3_attention_weight_max": 94.45881652832031,
      "activations/layer3_attention_weight_min": -93.33373260498047,
      "activations/layer4_attention_weight_max": 113.2067642211914,
      "activations/layer4_attention_weight_min": -104.93456268310547,
      "activations/layer5_attention_weight_max": 54.241451263427734,
      "activations/layer5_attention_weight_min": -62.69403076171875,
      "activations/layer6_attention_weight_max": 44.15990447998047,
      "activations/layer6_attention_weight_min": -44.83803176879883,
      "activations/layer7_attention_weight_max": 94.42510223388672,
      "activations/layer7_attention_weight_min": -88.53231811523438,
      "activations/layer8_attention_weight_max": 40.02678680419922,
      "activations/layer8_attention_weight_min": -40.33506393432617,
      "activations/layer9_attention_weight_max": 32.526519775390625,
      "activations/layer9_attention_weight_min": -34.049076080322266,
      "epoch": 14.73,
      "learning_rate": 5.5552272727272724e-05,
      "loss": 2.7561,
      "step": 253550
    },
    {
      "activations/layer0_attention_weight_max": 16.01999855041504,
      "activations/layer0_attention_weight_min": -12.299074172973633,
      "activations/layer10_attention_weight_max": 30.159440994262695,
      "activations/layer10_attention_weight_min": -32.2071418762207,
      "activations/layer11_attention_weight_max": 31.218669891357422,
      "activations/layer11_attention_weight_min": -31.070133209228516,
      "activations/layer12_attention_weight_max": 26.772396087646484,
      "activations/layer12_attention_weight_min": -25.57048988342285,
      "activations/layer13_attention_weight_max": 41.34300231933594,
      "activations/layer13_attention_weight_min": -33.791465759277344,
      "activations/layer14_attention_weight_max": 41.31395721435547,
      "activations/layer14_attention_weight_min": -36.49012756347656,
      "activations/layer15_attention_weight_max": 40.399559020996094,
      "activations/layer15_attention_weight_min": -34.3095588684082,
      "activations/layer16_attention_weight_max": 27.76934242248535,
      "activations/layer16_attention_weight_min": -27.41767692565918,
      "activations/layer17_attention_weight_max": 28.150068283081055,
      "activations/layer17_attention_weight_min": -24.344799041748047,
      "activations/layer18_attention_weight_max": 26.984615325927734,
      "activations/layer18_attention_weight_min": -24.035924911499023,
      "activations/layer19_attention_weight_max": 32.70644760131836,
      "activations/layer19_attention_weight_min": -28.75174903869629,
      "activations/layer1_attention_weight_max": 15.752004623413086,
      "activations/layer1_attention_weight_min": -14.106648445129395,
      "activations/layer20_attention_weight_max": 26.748023986816406,
      "activations/layer20_attention_weight_min": -25.054492950439453,
      "activations/layer21_attention_weight_max": 27.230863571166992,
      "activations/layer21_attention_weight_min": -24.8255615234375,
      "activations/layer22_attention_weight_max": 40.8674430847168,
      "activations/layer22_attention_weight_min": -29.241092681884766,
      "activations/layer23_attention_weight_max": 36.6513671875,
      "activations/layer23_attention_weight_min": -23.589170455932617,
      "activations/layer2_attention_weight_max": 30.15894317626953,
      "activations/layer2_attention_weight_min": -30.51544761657715,
      "activations/layer3_attention_weight_max": 91.61758422851562,
      "activations/layer3_attention_weight_min": -93.85615539550781,
      "activations/layer4_attention_weight_max": 109.26496887207031,
      "activations/layer4_attention_weight_min": -114.33341217041016,
      "activations/layer5_attention_weight_max": 49.5460205078125,
      "activations/layer5_attention_weight_min": -65.34326171875,
      "activations/layer6_attention_weight_max": 44.301597595214844,
      "activations/layer6_attention_weight_min": -44.58036422729492,
      "activations/layer7_attention_weight_max": 88.05404663085938,
      "activations/layer7_attention_weight_min": -86.49028015136719,
      "activations/layer8_attention_weight_max": 39.370296478271484,
      "activations/layer8_attention_weight_min": -41.687843322753906,
      "activations/layer9_attention_weight_max": 29.720321655273438,
      "activations/layer9_attention_weight_min": -32.2658805847168,
      "epoch": 14.74,
      "learning_rate": 5.553333333333333e-05,
      "loss": 2.7695,
      "step": 253600
    },
    {
      "activations/layer0_attention_weight_max": 17.497140884399414,
      "activations/layer0_attention_weight_min": -13.19091796875,
      "activations/layer10_attention_weight_max": 30.694578170776367,
      "activations/layer10_attention_weight_min": -33.044410705566406,
      "activations/layer11_attention_weight_max": 30.593364715576172,
      "activations/layer11_attention_weight_min": -31.902023315429688,
      "activations/layer12_attention_weight_max": 25.773944854736328,
      "activations/layer12_attention_weight_min": -27.0769100189209,
      "activations/layer13_attention_weight_max": 38.96442413330078,
      "activations/layer13_attention_weight_min": -37.28081512451172,
      "activations/layer14_attention_weight_max": 43.1993522644043,
      "activations/layer14_attention_weight_min": -39.70265579223633,
      "activations/layer15_attention_weight_max": 37.34231185913086,
      "activations/layer15_attention_weight_min": -34.80018615722656,
      "activations/layer16_attention_weight_max": 32.49037170410156,
      "activations/layer16_attention_weight_min": -27.494625091552734,
      "activations/layer17_attention_weight_max": 32.26537322998047,
      "activations/layer17_attention_weight_min": -25.05731773376465,
      "activations/layer18_attention_weight_max": 32.68566131591797,
      "activations/layer18_attention_weight_min": -24.701404571533203,
      "activations/layer19_attention_weight_max": 34.310523986816406,
      "activations/layer19_attention_weight_min": -32.43870544433594,
      "activations/layer1_attention_weight_max": 16.26468276977539,
      "activations/layer1_attention_weight_min": -14.06981372833252,
      "activations/layer20_attention_weight_max": 30.941972732543945,
      "activations/layer20_attention_weight_min": -25.681344985961914,
      "activations/layer21_attention_weight_max": 30.330913543701172,
      "activations/layer21_attention_weight_min": -25.29751968383789,
      "activations/layer22_attention_weight_max": 43.23616409301758,
      "activations/layer22_attention_weight_min": -32.556636810302734,
      "activations/layer23_attention_weight_max": 35.275062561035156,
      "activations/layer23_attention_weight_min": -25.565673828125,
      "activations/layer2_attention_weight_max": 31.821523666381836,
      "activations/layer2_attention_weight_min": -30.086292266845703,
      "activations/layer3_attention_weight_max": 96.71620178222656,
      "activations/layer3_attention_weight_min": -93.12197875976562,
      "activations/layer4_attention_weight_max": 112.25545501708984,
      "activations/layer4_attention_weight_min": -109.57476806640625,
      "activations/layer5_attention_weight_max": 50.449859619140625,
      "activations/layer5_attention_weight_min": -62.28959274291992,
      "activations/layer6_attention_weight_max": 42.76258850097656,
      "activations/layer6_attention_weight_min": -46.1545295715332,
      "activations/layer7_attention_weight_max": 88.43865203857422,
      "activations/layer7_attention_weight_min": -92.17716979980469,
      "activations/layer8_attention_weight_max": 39.22827911376953,
      "activations/layer8_attention_weight_min": -41.769065856933594,
      "activations/layer9_attention_weight_max": 31.81515884399414,
      "activations/layer9_attention_weight_min": -33.71885299682617,
      "epoch": 14.74,
      "learning_rate": 5.5514393939393934e-05,
      "loss": 2.7613,
      "step": 253650
    },
    {
      "activations/layer0_attention_weight_max": 16.52387046813965,
      "activations/layer0_attention_weight_min": -12.170954704284668,
      "activations/layer10_attention_weight_max": 33.2093620300293,
      "activations/layer10_attention_weight_min": -34.98978042602539,
      "activations/layer11_attention_weight_max": 32.25628662109375,
      "activations/layer11_attention_weight_min": -34.961647033691406,
      "activations/layer12_attention_weight_max": 26.87075424194336,
      "activations/layer12_attention_weight_min": -26.26373863220215,
      "activations/layer13_attention_weight_max": 39.5113410949707,
      "activations/layer13_attention_weight_min": -34.74559783935547,
      "activations/layer14_attention_weight_max": 42.935943603515625,
      "activations/layer14_attention_weight_min": -37.06570816040039,
      "activations/layer15_attention_weight_max": 37.78622817993164,
      "activations/layer15_attention_weight_min": -33.74406051635742,
      "activations/layer16_attention_weight_max": 28.876266479492188,
      "activations/layer16_attention_weight_min": -26.658496856689453,
      "activations/layer17_attention_weight_max": 30.069101333618164,
      "activations/layer17_attention_weight_min": -25.237144470214844,
      "activations/layer18_attention_weight_max": 29.513708114624023,
      "activations/layer18_attention_weight_min": -24.07249641418457,
      "activations/layer19_attention_weight_max": 31.652313232421875,
      "activations/layer19_attention_weight_min": -30.292509078979492,
      "activations/layer1_attention_weight_max": 15.972817420959473,
      "activations/layer1_attention_weight_min": -14.815457344055176,
      "activations/layer20_attention_weight_max": 27.079261779785156,
      "activations/layer20_attention_weight_min": -25.26226806640625,
      "activations/layer21_attention_weight_max": 27.29571533203125,
      "activations/layer21_attention_weight_min": -25.152713775634766,
      "activations/layer22_attention_weight_max": 44.34231948852539,
      "activations/layer22_attention_weight_min": -30.294723510742188,
      "activations/layer23_attention_weight_max": 32.810035705566406,
      "activations/layer23_attention_weight_min": -28.916900634765625,
      "activations/layer2_attention_weight_max": 30.62346649169922,
      "activations/layer2_attention_weight_min": -30.36683464050293,
      "activations/layer3_attention_weight_max": 95.8680191040039,
      "activations/layer3_attention_weight_min": -91.89271545410156,
      "activations/layer4_attention_weight_max": 109.24787902832031,
      "activations/layer4_attention_weight_min": -107.30059814453125,
      "activations/layer5_attention_weight_max": 50.48278045654297,
      "activations/layer5_attention_weight_min": -60.22502899169922,
      "activations/layer6_attention_weight_max": 43.177757263183594,
      "activations/layer6_attention_weight_min": -45.67130661010742,
      "activations/layer7_attention_weight_max": 90.23616027832031,
      "activations/layer7_attention_weight_min": -90.65946197509766,
      "activations/layer8_attention_weight_max": 44.68996047973633,
      "activations/layer8_attention_weight_min": -44.5014762878418,
      "activations/layer9_attention_weight_max": 33.47866439819336,
      "activations/layer9_attention_weight_min": -34.1488037109375,
      "epoch": 14.74,
      "learning_rate": 5.5495454545454535e-05,
      "loss": 2.7628,
      "step": 253700
    },
    {
      "activations/layer0_attention_weight_max": 16.040359497070312,
      "activations/layer0_attention_weight_min": -12.926767349243164,
      "activations/layer10_attention_weight_max": 31.997787475585938,
      "activations/layer10_attention_weight_min": -32.66661071777344,
      "activations/layer11_attention_weight_max": 32.28403854370117,
      "activations/layer11_attention_weight_min": -32.15876388549805,
      "activations/layer12_attention_weight_max": 24.767932891845703,
      "activations/layer12_attention_weight_min": -24.03380012512207,
      "activations/layer13_attention_weight_max": 38.822654724121094,
      "activations/layer13_attention_weight_min": -33.123634338378906,
      "activations/layer14_attention_weight_max": 43.98186111450195,
      "activations/layer14_attention_weight_min": -35.14922332763672,
      "activations/layer15_attention_weight_max": 40.9893798828125,
      "activations/layer15_attention_weight_min": -32.894691467285156,
      "activations/layer16_attention_weight_max": 32.06993103027344,
      "activations/layer16_attention_weight_min": -27.17441749572754,
      "activations/layer17_attention_weight_max": 30.28203010559082,
      "activations/layer17_attention_weight_min": -28.52768898010254,
      "activations/layer18_attention_weight_max": 32.99614334106445,
      "activations/layer18_attention_weight_min": -23.72638511657715,
      "activations/layer19_attention_weight_max": 34.46363067626953,
      "activations/layer19_attention_weight_min": -33.3696174621582,
      "activations/layer1_attention_weight_max": 15.976879119873047,
      "activations/layer1_attention_weight_min": -15.805583000183105,
      "activations/layer20_attention_weight_max": 29.868040084838867,
      "activations/layer20_attention_weight_min": -26.998882293701172,
      "activations/layer21_attention_weight_max": 34.61101150512695,
      "activations/layer21_attention_weight_min": -25.82891082763672,
      "activations/layer22_attention_weight_max": 41.145263671875,
      "activations/layer22_attention_weight_min": -31.42230987548828,
      "activations/layer23_attention_weight_max": 34.615760803222656,
      "activations/layer23_attention_weight_min": -27.115219116210938,
      "activations/layer2_attention_weight_max": 33.234031677246094,
      "activations/layer2_attention_weight_min": -30.187530517578125,
      "activations/layer3_attention_weight_max": 94.17776489257812,
      "activations/layer3_attention_weight_min": -93.22181701660156,
      "activations/layer4_attention_weight_max": 109.93624114990234,
      "activations/layer4_attention_weight_min": -111.47542572021484,
      "activations/layer5_attention_weight_max": 53.491920471191406,
      "activations/layer5_attention_weight_min": -64.38204956054688,
      "activations/layer6_attention_weight_max": 43.37141418457031,
      "activations/layer6_attention_weight_min": -45.29190444946289,
      "activations/layer7_attention_weight_max": 92.4618911743164,
      "activations/layer7_attention_weight_min": -88.64947509765625,
      "activations/layer8_attention_weight_max": 40.5201301574707,
      "activations/layer8_attention_weight_min": -40.98390197753906,
      "activations/layer9_attention_weight_max": 33.245826721191406,
      "activations/layer9_attention_weight_min": -31.8870906829834,
      "epoch": 14.74,
      "learning_rate": 5.547651515151515e-05,
      "loss": 2.763,
      "step": 253750
    },
    {
      "activations/layer0_attention_weight_max": 16.412033081054688,
      "activations/layer0_attention_weight_min": -12.748414039611816,
      "activations/layer10_attention_weight_max": 31.56804084777832,
      "activations/layer10_attention_weight_min": -31.29274559020996,
      "activations/layer11_attention_weight_max": 32.53641128540039,
      "activations/layer11_attention_weight_min": -33.33744812011719,
      "activations/layer12_attention_weight_max": 28.022592544555664,
      "activations/layer12_attention_weight_min": -26.066904067993164,
      "activations/layer13_attention_weight_max": 41.505584716796875,
      "activations/layer13_attention_weight_min": -33.902503967285156,
      "activations/layer14_attention_weight_max": 45.021671295166016,
      "activations/layer14_attention_weight_min": -36.340084075927734,
      "activations/layer15_attention_weight_max": 42.707733154296875,
      "activations/layer15_attention_weight_min": -33.603145599365234,
      "activations/layer16_attention_weight_max": 31.556921005249023,
      "activations/layer16_attention_weight_min": -27.139686584472656,
      "activations/layer17_attention_weight_max": 31.366641998291016,
      "activations/layer17_attention_weight_min": -24.91682243347168,
      "activations/layer18_attention_weight_max": 34.77613830566406,
      "activations/layer18_attention_weight_min": -24.232749938964844,
      "activations/layer19_attention_weight_max": 37.515289306640625,
      "activations/layer19_attention_weight_min": -29.820083618164062,
      "activations/layer1_attention_weight_max": 16.043052673339844,
      "activations/layer1_attention_weight_min": -14.305006980895996,
      "activations/layer20_attention_weight_max": 32.22649002075195,
      "activations/layer20_attention_weight_min": -25.705175399780273,
      "activations/layer21_attention_weight_max": 30.605712890625,
      "activations/layer21_attention_weight_min": -24.377111434936523,
      "activations/layer22_attention_weight_max": 47.72438049316406,
      "activations/layer22_attention_weight_min": -29.565387725830078,
      "activations/layer23_attention_weight_max": 34.56687927246094,
      "activations/layer23_attention_weight_min": -26.038898468017578,
      "activations/layer2_attention_weight_max": 31.554380416870117,
      "activations/layer2_attention_weight_min": -31.450759887695312,
      "activations/layer3_attention_weight_max": 96.751220703125,
      "activations/layer3_attention_weight_min": -97.03336334228516,
      "activations/layer4_attention_weight_max": 107.66808319091797,
      "activations/layer4_attention_weight_min": -109.61476135253906,
      "activations/layer5_attention_weight_max": 50.49737548828125,
      "activations/layer5_attention_weight_min": -60.44327926635742,
      "activations/layer6_attention_weight_max": 42.158077239990234,
      "activations/layer6_attention_weight_min": -45.11067199707031,
      "activations/layer7_attention_weight_max": 87.8990478515625,
      "activations/layer7_attention_weight_min": -88.38684844970703,
      "activations/layer8_attention_weight_max": 39.98402786254883,
      "activations/layer8_attention_weight_min": -40.16383361816406,
      "activations/layer9_attention_weight_max": 32.08620834350586,
      "activations/layer9_attention_weight_min": -32.08707046508789,
      "epoch": 14.75,
      "learning_rate": 5.545757575757575e-05,
      "loss": 2.7597,
      "step": 253800
    },
    {
      "activations/layer0_attention_weight_max": 15.517090797424316,
      "activations/layer0_attention_weight_min": -11.881546974182129,
      "activations/layer10_attention_weight_max": 32.54241943359375,
      "activations/layer10_attention_weight_min": -33.951194763183594,
      "activations/layer11_attention_weight_max": 31.985565185546875,
      "activations/layer11_attention_weight_min": -34.13928985595703,
      "activations/layer12_attention_weight_max": 25.263261795043945,
      "activations/layer12_attention_weight_min": -26.360868453979492,
      "activations/layer13_attention_weight_max": 39.13622283935547,
      "activations/layer13_attention_weight_min": -34.652061462402344,
      "activations/layer14_attention_weight_max": 40.073856353759766,
      "activations/layer14_attention_weight_min": -37.06913757324219,
      "activations/layer15_attention_weight_max": 36.071598052978516,
      "activations/layer15_attention_weight_min": -33.303199768066406,
      "activations/layer16_attention_weight_max": 32.34164047241211,
      "activations/layer16_attention_weight_min": -28.10852813720703,
      "activations/layer17_attention_weight_max": 32.301578521728516,
      "activations/layer17_attention_weight_min": -24.961557388305664,
      "activations/layer18_attention_weight_max": 32.18156051635742,
      "activations/layer18_attention_weight_min": -23.803668975830078,
      "activations/layer19_attention_weight_max": 33.27320861816406,
      "activations/layer19_attention_weight_min": -29.980146408081055,
      "activations/layer1_attention_weight_max": 16.643287658691406,
      "activations/layer1_attention_weight_min": -14.743365287780762,
      "activations/layer20_attention_weight_max": 26.927366256713867,
      "activations/layer20_attention_weight_min": -25.633838653564453,
      "activations/layer21_attention_weight_max": 28.620555877685547,
      "activations/layer21_attention_weight_min": -23.92272186279297,
      "activations/layer22_attention_weight_max": 40.07692337036133,
      "activations/layer22_attention_weight_min": -31.57368278503418,
      "activations/layer23_attention_weight_max": 31.11212921142578,
      "activations/layer23_attention_weight_min": -27.753055572509766,
      "activations/layer2_attention_weight_max": 32.0653076171875,
      "activations/layer2_attention_weight_min": -31.636371612548828,
      "activations/layer3_attention_weight_max": 97.42286682128906,
      "activations/layer3_attention_weight_min": -98.39643096923828,
      "activations/layer4_attention_weight_max": 111.7677993774414,
      "activations/layer4_attention_weight_min": -118.32559967041016,
      "activations/layer5_attention_weight_max": 55.197479248046875,
      "activations/layer5_attention_weight_min": -65.26298522949219,
      "activations/layer6_attention_weight_max": 44.29195785522461,
      "activations/layer6_attention_weight_min": -46.494468688964844,
      "activations/layer7_attention_weight_max": 93.7115478515625,
      "activations/layer7_attention_weight_min": -93.47152709960938,
      "activations/layer8_attention_weight_max": 41.26144027709961,
      "activations/layer8_attention_weight_min": -43.98957443237305,
      "activations/layer9_attention_weight_max": 33.70981979370117,
      "activations/layer9_attention_weight_min": -32.69904708862305,
      "epoch": 14.75,
      "learning_rate": 5.543863636363636e-05,
      "loss": 2.7521,
      "step": 253850
    },
    {
      "activations/layer0_attention_weight_max": 16.461122512817383,
      "activations/layer0_attention_weight_min": -13.045841217041016,
      "activations/layer10_attention_weight_max": 32.985355377197266,
      "activations/layer10_attention_weight_min": -32.69687271118164,
      "activations/layer11_attention_weight_max": 32.17565155029297,
      "activations/layer11_attention_weight_min": -33.440330505371094,
      "activations/layer12_attention_weight_max": 25.898033142089844,
      "activations/layer12_attention_weight_min": -25.31926918029785,
      "activations/layer13_attention_weight_max": 40.729515075683594,
      "activations/layer13_attention_weight_min": -36.022544860839844,
      "activations/layer14_attention_weight_max": 43.19694900512695,
      "activations/layer14_attention_weight_min": -39.312496185302734,
      "activations/layer15_attention_weight_max": 40.153324127197266,
      "activations/layer15_attention_weight_min": -35.715576171875,
      "activations/layer16_attention_weight_max": 32.83363342285156,
      "activations/layer16_attention_weight_min": -27.477800369262695,
      "activations/layer17_attention_weight_max": 34.111454010009766,
      "activations/layer17_attention_weight_min": -24.928749084472656,
      "activations/layer18_attention_weight_max": 35.52061080932617,
      "activations/layer18_attention_weight_min": -25.622398376464844,
      "activations/layer19_attention_weight_max": 39.23802947998047,
      "activations/layer19_attention_weight_min": -32.62165832519531,
      "activations/layer1_attention_weight_max": 16.030746459960938,
      "activations/layer1_attention_weight_min": -14.77446460723877,
      "activations/layer20_attention_weight_max": 33.786006927490234,
      "activations/layer20_attention_weight_min": -27.11003875732422,
      "activations/layer21_attention_weight_max": 34.22148513793945,
      "activations/layer21_attention_weight_min": -26.323917388916016,
      "activations/layer22_attention_weight_max": 48.41822052001953,
      "activations/layer22_attention_weight_min": -31.186256408691406,
      "activations/layer23_attention_weight_max": 43.574737548828125,
      "activations/layer23_attention_weight_min": -29.8248348236084,
      "activations/layer2_attention_weight_max": 33.602142333984375,
      "activations/layer2_attention_weight_min": -33.16611099243164,
      "activations/layer3_attention_weight_max": 100.83683013916016,
      "activations/layer3_attention_weight_min": -101.42964172363281,
      "activations/layer4_attention_weight_max": 110.47708892822266,
      "activations/layer4_attention_weight_min": -111.39009094238281,
      "activations/layer5_attention_weight_max": 52.26548767089844,
      "activations/layer5_attention_weight_min": -61.44576644897461,
      "activations/layer6_attention_weight_max": 45.51102066040039,
      "activations/layer6_attention_weight_min": -45.40748596191406,
      "activations/layer7_attention_weight_max": 89.72107696533203,
      "activations/layer7_attention_weight_min": -90.32267761230469,
      "activations/layer8_attention_weight_max": 39.372528076171875,
      "activations/layer8_attention_weight_min": -40.139095306396484,
      "activations/layer9_attention_weight_max": 30.574377059936523,
      "activations/layer9_attention_weight_min": -32.58788299560547,
      "epoch": 14.75,
      "learning_rate": 5.541969696969696e-05,
      "loss": 2.7519,
      "step": 253900
    },
    {
      "activations/layer0_attention_weight_max": 16.689123153686523,
      "activations/layer0_attention_weight_min": -12.857691764831543,
      "activations/layer10_attention_weight_max": 33.819034576416016,
      "activations/layer10_attention_weight_min": -31.54574966430664,
      "activations/layer11_attention_weight_max": 31.771224975585938,
      "activations/layer11_attention_weight_min": -31.933269500732422,
      "activations/layer12_attention_weight_max": 25.413223266601562,
      "activations/layer12_attention_weight_min": -25.322296142578125,
      "activations/layer13_attention_weight_max": 36.459529876708984,
      "activations/layer13_attention_weight_min": -34.828651428222656,
      "activations/layer14_attention_weight_max": 42.3911247253418,
      "activations/layer14_attention_weight_min": -36.11619186401367,
      "activations/layer15_attention_weight_max": 35.956642150878906,
      "activations/layer15_attention_weight_min": -33.74394989013672,
      "activations/layer16_attention_weight_max": 28.9365291595459,
      "activations/layer16_attention_weight_min": -27.803512573242188,
      "activations/layer17_attention_weight_max": 31.41130256652832,
      "activations/layer17_attention_weight_min": -26.840757369995117,
      "activations/layer18_attention_weight_max": 31.503074645996094,
      "activations/layer18_attention_weight_min": -24.616304397583008,
      "activations/layer19_attention_weight_max": 33.28290557861328,
      "activations/layer19_attention_weight_min": -32.3240852355957,
      "activations/layer1_attention_weight_max": 15.824005126953125,
      "activations/layer1_attention_weight_min": -13.671002388000488,
      "activations/layer20_attention_weight_max": 26.831880569458008,
      "activations/layer20_attention_weight_min": -25.562009811401367,
      "activations/layer21_attention_weight_max": 26.98372459411621,
      "activations/layer21_attention_weight_min": -24.80293846130371,
      "activations/layer22_attention_weight_max": 39.37834548950195,
      "activations/layer22_attention_weight_min": -31.543855667114258,
      "activations/layer23_attention_weight_max": 30.063085556030273,
      "activations/layer23_attention_weight_min": -26.968639373779297,
      "activations/layer2_attention_weight_max": 30.50514793395996,
      "activations/layer2_attention_weight_min": -30.09067153930664,
      "activations/layer3_attention_weight_max": 94.55634307861328,
      "activations/layer3_attention_weight_min": -95.7528305053711,
      "activations/layer4_attention_weight_max": 106.08477020263672,
      "activations/layer4_attention_weight_min": -109.415771484375,
      "activations/layer5_attention_weight_max": 51.111907958984375,
      "activations/layer5_attention_weight_min": -62.36394500732422,
      "activations/layer6_attention_weight_max": 43.82353210449219,
      "activations/layer6_attention_weight_min": -47.404117584228516,
      "activations/layer7_attention_weight_max": 88.20967102050781,
      "activations/layer7_attention_weight_min": -87.30792236328125,
      "activations/layer8_attention_weight_max": 41.5997428894043,
      "activations/layer8_attention_weight_min": -43.978084564208984,
      "activations/layer9_attention_weight_max": 31.913333892822266,
      "activations/layer9_attention_weight_min": -32.98976135253906,
      "epoch": 14.76,
      "learning_rate": 5.540075757575757e-05,
      "loss": 2.756,
      "step": 253950
    },
    {
      "activations/layer0_attention_weight_max": 16.691770553588867,
      "activations/layer0_attention_weight_min": -12.653067588806152,
      "activations/layer10_attention_weight_max": 31.062936782836914,
      "activations/layer10_attention_weight_min": -33.42586898803711,
      "activations/layer11_attention_weight_max": 32.52732849121094,
      "activations/layer11_attention_weight_min": -33.356746673583984,
      "activations/layer12_attention_weight_max": 26.755491256713867,
      "activations/layer12_attention_weight_min": -26.577051162719727,
      "activations/layer13_attention_weight_max": 41.22030258178711,
      "activations/layer13_attention_weight_min": -35.39461898803711,
      "activations/layer14_attention_weight_max": 46.72654724121094,
      "activations/layer14_attention_weight_min": -39.64531326293945,
      "activations/layer15_attention_weight_max": 39.28787612915039,
      "activations/layer15_attention_weight_min": -35.50334167480469,
      "activations/layer16_attention_weight_max": 31.63501739501953,
      "activations/layer16_attention_weight_min": -27.56303596496582,
      "activations/layer17_attention_weight_max": 30.38585090637207,
      "activations/layer17_attention_weight_min": -25.609689712524414,
      "activations/layer18_attention_weight_max": 30.544301986694336,
      "activations/layer18_attention_weight_min": -25.06492042541504,
      "activations/layer19_attention_weight_max": 34.69704055786133,
      "activations/layer19_attention_weight_min": -30.75215721130371,
      "activations/layer1_attention_weight_max": 15.533031463623047,
      "activations/layer1_attention_weight_min": -13.684380531311035,
      "activations/layer20_attention_weight_max": 29.437602996826172,
      "activations/layer20_attention_weight_min": -24.078563690185547,
      "activations/layer21_attention_weight_max": 33.56782913208008,
      "activations/layer21_attention_weight_min": -23.7368106842041,
      "activations/layer22_attention_weight_max": 41.44601821899414,
      "activations/layer22_attention_weight_min": -30.03181266784668,
      "activations/layer23_attention_weight_max": 33.13081359863281,
      "activations/layer23_attention_weight_min": -25.236595153808594,
      "activations/layer2_attention_weight_max": 33.68742370605469,
      "activations/layer2_attention_weight_min": -30.98628807067871,
      "activations/layer3_attention_weight_max": 95.39047241210938,
      "activations/layer3_attention_weight_min": -95.0895004272461,
      "activations/layer4_attention_weight_max": 107.37801361083984,
      "activations/layer4_attention_weight_min": -109.17845916748047,
      "activations/layer5_attention_weight_max": 50.907554626464844,
      "activations/layer5_attention_weight_min": -64.8374252319336,
      "activations/layer6_attention_weight_max": 45.10921859741211,
      "activations/layer6_attention_weight_min": -48.730567932128906,
      "activations/layer7_attention_weight_max": 94.42414093017578,
      "activations/layer7_attention_weight_min": -91.05435180664062,
      "activations/layer8_attention_weight_max": 39.999237060546875,
      "activations/layer8_attention_weight_min": -41.91120529174805,
      "activations/layer9_attention_weight_max": 31.054061889648438,
      "activations/layer9_attention_weight_min": -34.336143493652344,
      "epoch": 14.76,
      "learning_rate": 5.538181818181818e-05,
      "loss": 2.7583,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.5149,
      "eval_samples_per_second": 504.295,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.5149,
      "eval_openwebtext_samples_per_second": 504.295,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.986,
      "eval_wikitext_samples_per_second": 229.604,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_lambada_loss": 2.49609375,
      "eval_lambada_ppl": 12.134998917818166,
      "eval_lambada_runtime": 9.5795,
      "eval_lambada_samples_per_second": 508.272,
      "step": 254000
    },
    {
      "activations/layer0_attention_weight_max": 15.109143257141113,
      "activations/layer0_attention_weight_min": -12.797673225402832,
      "activations/layer10_attention_weight_max": 32.7202033996582,
      "activations/layer10_attention_weight_min": -33.88287353515625,
      "activations/layer11_attention_weight_max": 34.22991943359375,
      "activations/layer11_attention_weight_min": -33.688209533691406,
      "activations/layer12_attention_weight_max": 26.807003021240234,
      "activations/layer12_attention_weight_min": -25.132421493530273,
      "activations/layer13_attention_weight_max": 41.98259735107422,
      "activations/layer13_attention_weight_min": -34.166934967041016,
      "activations/layer14_attention_weight_max": 44.10970687866211,
      "activations/layer14_attention_weight_min": -37.2226448059082,
      "activations/layer15_attention_weight_max": 37.91459655761719,
      "activations/layer15_attention_weight_min": -32.938472747802734,
      "activations/layer16_attention_weight_max": 32.334983825683594,
      "activations/layer16_attention_weight_min": -28.506671905517578,
      "activations/layer17_attention_weight_max": 28.333274841308594,
      "activations/layer17_attention_weight_min": -26.335145950317383,
      "activations/layer18_attention_weight_max": 29.32647132873535,
      "activations/layer18_attention_weight_min": -26.69928550720215,
      "activations/layer19_attention_weight_max": 35.6331901550293,
      "activations/layer19_attention_weight_min": -31.30646324157715,
      "activations/layer1_attention_weight_max": 17.526016235351562,
      "activations/layer1_attention_weight_min": -16.892715454101562,
      "activations/layer20_attention_weight_max": 30.80098533630371,
      "activations/layer20_attention_weight_min": -27.47148895263672,
      "activations/layer21_attention_weight_max": 30.62628746032715,
      "activations/layer21_attention_weight_min": -26.321622848510742,
      "activations/layer22_attention_weight_max": 46.210975646972656,
      "activations/layer22_attention_weight_min": -32.0320930480957,
      "activations/layer23_attention_weight_max": 33.22465896606445,
      "activations/layer23_attention_weight_min": -26.47601890563965,
      "activations/layer2_attention_weight_max": 33.12450408935547,
      "activations/layer2_attention_weight_min": -31.285175323486328,
      "activations/layer3_attention_weight_max": 99.96508026123047,
      "activations/layer3_attention_weight_min": -96.06517791748047,
      "activations/layer4_attention_weight_max": 112.94121551513672,
      "activations/layer4_attention_weight_min": -111.37135314941406,
      "activations/layer5_attention_weight_max": 53.630165100097656,
      "activations/layer5_attention_weight_min": -65.75263977050781,
      "activations/layer6_attention_weight_max": 45.667381286621094,
      "activations/layer6_attention_weight_min": -46.72822570800781,
      "activations/layer7_attention_weight_max": 94.19607543945312,
      "activations/layer7_attention_weight_min": -94.02194213867188,
      "activations/layer8_attention_weight_max": 42.45105743408203,
      "activations/layer8_attention_weight_min": -41.37959671020508,
      "activations/layer9_attention_weight_max": 33.97633743286133,
      "activations/layer9_attention_weight_min": -34.19236373901367,
      "epoch": 14.76,
      "learning_rate": 5.536287878787878e-05,
      "loss": 2.7573,
      "step": 254050
    },
    {
      "activations/layer0_attention_weight_max": 16.318817138671875,
      "activations/layer0_attention_weight_min": -11.400002479553223,
      "activations/layer10_attention_weight_max": 29.8028621673584,
      "activations/layer10_attention_weight_min": -32.0404052734375,
      "activations/layer11_attention_weight_max": 33.15618133544922,
      "activations/layer11_attention_weight_min": -30.66584014892578,
      "activations/layer12_attention_weight_max": 22.695880889892578,
      "activations/layer12_attention_weight_min": -26.17173194885254,
      "activations/layer13_attention_weight_max": 34.25059509277344,
      "activations/layer13_attention_weight_min": -34.61823272705078,
      "activations/layer14_attention_weight_max": 38.43980026245117,
      "activations/layer14_attention_weight_min": -38.21818161010742,
      "activations/layer15_attention_weight_max": 33.32351303100586,
      "activations/layer15_attention_weight_min": -35.703792572021484,
      "activations/layer16_attention_weight_max": 27.809062957763672,
      "activations/layer16_attention_weight_min": -28.12499237060547,
      "activations/layer17_attention_weight_max": 26.121376037597656,
      "activations/layer17_attention_weight_min": -26.937313079833984,
      "activations/layer18_attention_weight_max": 28.152055740356445,
      "activations/layer18_attention_weight_min": -24.264225006103516,
      "activations/layer19_attention_weight_max": 33.84268569946289,
      "activations/layer19_attention_weight_min": -32.60072326660156,
      "activations/layer1_attention_weight_max": 16.09737777709961,
      "activations/layer1_attention_weight_min": -14.465481758117676,
      "activations/layer20_attention_weight_max": 29.89324951171875,
      "activations/layer20_attention_weight_min": -26.200050354003906,
      "activations/layer21_attention_weight_max": 27.036649703979492,
      "activations/layer21_attention_weight_min": -25.066808700561523,
      "activations/layer22_attention_weight_max": 37.97317886352539,
      "activations/layer22_attention_weight_min": -31.20522689819336,
      "activations/layer23_attention_weight_max": 32.30259704589844,
      "activations/layer23_attention_weight_min": -25.59906578063965,
      "activations/layer2_attention_weight_max": 32.52227020263672,
      "activations/layer2_attention_weight_min": -31.3211669921875,
      "activations/layer3_attention_weight_max": 93.09294128417969,
      "activations/layer3_attention_weight_min": -97.59291076660156,
      "activations/layer4_attention_weight_max": 107.00819396972656,
      "activations/layer4_attention_weight_min": -111.25389862060547,
      "activations/layer5_attention_weight_max": 50.84429931640625,
      "activations/layer5_attention_weight_min": -62.25109100341797,
      "activations/layer6_attention_weight_max": 43.160579681396484,
      "activations/layer6_attention_weight_min": -46.0475959777832,
      "activations/layer7_attention_weight_max": 89.06876373291016,
      "activations/layer7_attention_weight_min": -86.80195617675781,
      "activations/layer8_attention_weight_max": 38.54903030395508,
      "activations/layer8_attention_weight_min": -39.61880111694336,
      "activations/layer9_attention_weight_max": 28.91875648498535,
      "activations/layer9_attention_weight_min": -31.584060668945312,
      "epoch": 14.76,
      "learning_rate": 5.5343939393939395e-05,
      "loss": 2.75,
      "step": 254100
    },
    {
      "activations/layer0_attention_weight_max": 16.501340866088867,
      "activations/layer0_attention_weight_min": -12.877276420593262,
      "activations/layer10_attention_weight_max": 33.28983688354492,
      "activations/layer10_attention_weight_min": -34.744327545166016,
      "activations/layer11_attention_weight_max": 31.993694305419922,
      "activations/layer11_attention_weight_min": -33.25428771972656,
      "activations/layer12_attention_weight_max": 27.720834732055664,
      "activations/layer12_attention_weight_min": -26.817981719970703,
      "activations/layer13_attention_weight_max": 41.32202911376953,
      "activations/layer13_attention_weight_min": -36.01205062866211,
      "activations/layer14_attention_weight_max": 53.519622802734375,
      "activations/layer14_attention_weight_min": -40.24894714355469,
      "activations/layer15_attention_weight_max": 45.23794174194336,
      "activations/layer15_attention_weight_min": -36.41099166870117,
      "activations/layer16_attention_weight_max": 36.212406158447266,
      "activations/layer16_attention_weight_min": -30.401811599731445,
      "activations/layer17_attention_weight_max": 35.1340217590332,
      "activations/layer17_attention_weight_min": -27.786352157592773,
      "activations/layer18_attention_weight_max": 35.54228973388672,
      "activations/layer18_attention_weight_min": -27.124088287353516,
      "activations/layer19_attention_weight_max": 39.83951187133789,
      "activations/layer19_attention_weight_min": -34.457881927490234,
      "activations/layer1_attention_weight_max": 16.305068969726562,
      "activations/layer1_attention_weight_min": -15.495162010192871,
      "activations/layer20_attention_weight_max": 32.91706085205078,
      "activations/layer20_attention_weight_min": -25.802261352539062,
      "activations/layer21_attention_weight_max": 30.549102783203125,
      "activations/layer21_attention_weight_min": -25.22386932373047,
      "activations/layer22_attention_weight_max": 48.648353576660156,
      "activations/layer22_attention_weight_min": -34.593589782714844,
      "activations/layer23_attention_weight_max": 36.14733123779297,
      "activations/layer23_attention_weight_min": -27.671123504638672,
      "activations/layer2_attention_weight_max": 31.950529098510742,
      "activations/layer2_attention_weight_min": -30.07076644897461,
      "activations/layer3_attention_weight_max": 94.86299133300781,
      "activations/layer3_attention_weight_min": -91.70431518554688,
      "activations/layer4_attention_weight_max": 107.57735443115234,
      "activations/layer4_attention_weight_min": -108.81646728515625,
      "activations/layer5_attention_weight_max": 51.56819152832031,
      "activations/layer5_attention_weight_min": -62.70365524291992,
      "activations/layer6_attention_weight_max": 43.694828033447266,
      "activations/layer6_attention_weight_min": -45.751407623291016,
      "activations/layer7_attention_weight_max": 93.78002166748047,
      "activations/layer7_attention_weight_min": -94.88827514648438,
      "activations/layer8_attention_weight_max": 40.64443588256836,
      "activations/layer8_attention_weight_min": -44.31364059448242,
      "activations/layer9_attention_weight_max": 31.850849151611328,
      "activations/layer9_attention_weight_min": -34.89950180053711,
      "epoch": 14.77,
      "learning_rate": 5.5324999999999996e-05,
      "loss": 2.7445,
      "step": 254150
    },
    {
      "activations/layer0_attention_weight_max": 16.43429946899414,
      "activations/layer0_attention_weight_min": -12.514644622802734,
      "activations/layer10_attention_weight_max": 31.680156707763672,
      "activations/layer10_attention_weight_min": -31.27280044555664,
      "activations/layer11_attention_weight_max": 31.16698455810547,
      "activations/layer11_attention_weight_min": -33.068199157714844,
      "activations/layer12_attention_weight_max": 26.195402145385742,
      "activations/layer12_attention_weight_min": -26.125293731689453,
      "activations/layer13_attention_weight_max": 38.64237594604492,
      "activations/layer13_attention_weight_min": -34.93894958496094,
      "activations/layer14_attention_weight_max": 45.48512649536133,
      "activations/layer14_attention_weight_min": -38.57328414916992,
      "activations/layer15_attention_weight_max": 37.6894645690918,
      "activations/layer15_attention_weight_min": -36.88652801513672,
      "activations/layer16_attention_weight_max": 32.957176208496094,
      "activations/layer16_attention_weight_min": -29.089481353759766,
      "activations/layer17_attention_weight_max": 34.2201042175293,
      "activations/layer17_attention_weight_min": -27.947481155395508,
      "activations/layer18_attention_weight_max": 36.590301513671875,
      "activations/layer18_attention_weight_min": -26.609098434448242,
      "activations/layer19_attention_weight_max": 38.247066497802734,
      "activations/layer19_attention_weight_min": -32.561767578125,
      "activations/layer1_attention_weight_max": 16.756467819213867,
      "activations/layer1_attention_weight_min": -14.804656028747559,
      "activations/layer20_attention_weight_max": 31.12376594543457,
      "activations/layer20_attention_weight_min": -26.720081329345703,
      "activations/layer21_attention_weight_max": 31.944730758666992,
      "activations/layer21_attention_weight_min": -23.931793212890625,
      "activations/layer22_attention_weight_max": 46.844547271728516,
      "activations/layer22_attention_weight_min": -30.79537010192871,
      "activations/layer23_attention_weight_max": 38.151973724365234,
      "activations/layer23_attention_weight_min": -26.05367660522461,
      "activations/layer2_attention_weight_max": 33.0472412109375,
      "activations/layer2_attention_weight_min": -32.53789520263672,
      "activations/layer3_attention_weight_max": 92.09049987792969,
      "activations/layer3_attention_weight_min": -97.2815933227539,
      "activations/layer4_attention_weight_max": 103.83036041259766,
      "activations/layer4_attention_weight_min": -106.65802001953125,
      "activations/layer5_attention_weight_max": 48.3828239440918,
      "activations/layer5_attention_weight_min": -62.560325622558594,
      "activations/layer6_attention_weight_max": 42.41957473754883,
      "activations/layer6_attention_weight_min": -43.99021530151367,
      "activations/layer7_attention_weight_max": 88.67642211914062,
      "activations/layer7_attention_weight_min": -86.77786254882812,
      "activations/layer8_attention_weight_max": 37.61235809326172,
      "activations/layer8_attention_weight_min": -39.75613021850586,
      "activations/layer9_attention_weight_max": 30.05972671508789,
      "activations/layer9_attention_weight_min": -31.848987579345703,
      "epoch": 14.77,
      "learning_rate": 5.53060606060606e-05,
      "loss": 2.7527,
      "step": 254200
    },
    {
      "activations/layer0_attention_weight_max": 16.144392013549805,
      "activations/layer0_attention_weight_min": -12.326598167419434,
      "activations/layer10_attention_weight_max": 35.540443420410156,
      "activations/layer10_attention_weight_min": -33.58726501464844,
      "activations/layer11_attention_weight_max": 34.1546630859375,
      "activations/layer11_attention_weight_min": -33.85960006713867,
      "activations/layer12_attention_weight_max": 25.71678924560547,
      "activations/layer12_attention_weight_min": -27.023578643798828,
      "activations/layer13_attention_weight_max": 39.00849914550781,
      "activations/layer13_attention_weight_min": -36.55030822753906,
      "activations/layer14_attention_weight_max": 43.75067901611328,
      "activations/layer14_attention_weight_min": -38.5531120300293,
      "activations/layer15_attention_weight_max": 39.31974792480469,
      "activations/layer15_attention_weight_min": -35.96989822387695,
      "activations/layer16_attention_weight_max": 31.832719802856445,
      "activations/layer16_attention_weight_min": -28.206552505493164,
      "activations/layer17_attention_weight_max": 30.408681869506836,
      "activations/layer17_attention_weight_min": -26.7628173828125,
      "activations/layer18_attention_weight_max": 32.64960861206055,
      "activations/layer18_attention_weight_min": -24.63725471496582,
      "activations/layer19_attention_weight_max": 36.0199089050293,
      "activations/layer19_attention_weight_min": -30.06536865234375,
      "activations/layer1_attention_weight_max": 16.782474517822266,
      "activations/layer1_attention_weight_min": -14.799277305603027,
      "activations/layer20_attention_weight_max": 29.151193618774414,
      "activations/layer20_attention_weight_min": -25.182510375976562,
      "activations/layer21_attention_weight_max": 26.920732498168945,
      "activations/layer21_attention_weight_min": -24.192285537719727,
      "activations/layer22_attention_weight_max": 42.43225860595703,
      "activations/layer22_attention_weight_min": -30.863521575927734,
      "activations/layer23_attention_weight_max": 33.925743103027344,
      "activations/layer23_attention_weight_min": -26.64199447631836,
      "activations/layer2_attention_weight_max": 33.51892852783203,
      "activations/layer2_attention_weight_min": -33.557132720947266,
      "activations/layer3_attention_weight_max": 95.26033782958984,
      "activations/layer3_attention_weight_min": -96.4457015991211,
      "activations/layer4_attention_weight_max": 111.11505126953125,
      "activations/layer4_attention_weight_min": -109.73487854003906,
      "activations/layer5_attention_weight_max": 48.90773010253906,
      "activations/layer5_attention_weight_min": -61.99339294433594,
      "activations/layer6_attention_weight_max": 45.74943923950195,
      "activations/layer6_attention_weight_min": -47.19823455810547,
      "activations/layer7_attention_weight_max": 93.44316864013672,
      "activations/layer7_attention_weight_min": -99.51887512207031,
      "activations/layer8_attention_weight_max": 41.7996940612793,
      "activations/layer8_attention_weight_min": -43.307804107666016,
      "activations/layer9_attention_weight_max": 33.10611343383789,
      "activations/layer9_attention_weight_min": -35.17409896850586,
      "epoch": 14.77,
      "learning_rate": 5.5287121212121206e-05,
      "loss": 2.7611,
      "step": 254250
    },
    {
      "activations/layer0_attention_weight_max": 16.10821533203125,
      "activations/layer0_attention_weight_min": -12.798826217651367,
      "activations/layer10_attention_weight_max": 32.537986755371094,
      "activations/layer10_attention_weight_min": -32.74920654296875,
      "activations/layer11_attention_weight_max": 32.89823532104492,
      "activations/layer11_attention_weight_min": -32.377140045166016,
      "activations/layer12_attention_weight_max": 25.012739181518555,
      "activations/layer12_attention_weight_min": -24.710567474365234,
      "activations/layer13_attention_weight_max": 39.35771560668945,
      "activations/layer13_attention_weight_min": -33.75921630859375,
      "activations/layer14_attention_weight_max": 41.211265563964844,
      "activations/layer14_attention_weight_min": -38.080833435058594,
      "activations/layer15_attention_weight_max": 36.574581146240234,
      "activations/layer15_attention_weight_min": -32.45704650878906,
      "activations/layer16_attention_weight_max": 29.22504997253418,
      "activations/layer16_attention_weight_min": -27.61675453186035,
      "activations/layer17_attention_weight_max": 30.78011703491211,
      "activations/layer17_attention_weight_min": -24.178730010986328,
      "activations/layer18_attention_weight_max": 32.472225189208984,
      "activations/layer18_attention_weight_min": -26.62856101989746,
      "activations/layer19_attention_weight_max": 32.86115264892578,
      "activations/layer19_attention_weight_min": -33.79475402832031,
      "activations/layer1_attention_weight_max": 16.02013397216797,
      "activations/layer1_attention_weight_min": -14.22382926940918,
      "activations/layer20_attention_weight_max": 27.620216369628906,
      "activations/layer20_attention_weight_min": -25.954345703125,
      "activations/layer21_attention_weight_max": 28.015695571899414,
      "activations/layer21_attention_weight_min": -26.271987915039062,
      "activations/layer22_attention_weight_max": 43.464107513427734,
      "activations/layer22_attention_weight_min": -32.38663101196289,
      "activations/layer23_attention_weight_max": 35.11785125732422,
      "activations/layer23_attention_weight_min": -25.972394943237305,
      "activations/layer2_attention_weight_max": 33.055198669433594,
      "activations/layer2_attention_weight_min": -33.509098052978516,
      "activations/layer3_attention_weight_max": 100.9327621459961,
      "activations/layer3_attention_weight_min": -100.18698120117188,
      "activations/layer4_attention_weight_max": 115.82610321044922,
      "activations/layer4_attention_weight_min": -113.36861419677734,
      "activations/layer5_attention_weight_max": 55.55134201049805,
      "activations/layer5_attention_weight_min": -62.240718841552734,
      "activations/layer6_attention_weight_max": 47.100345611572266,
      "activations/layer6_attention_weight_min": -48.2220573425293,
      "activations/layer7_attention_weight_max": 90.56928253173828,
      "activations/layer7_attention_weight_min": -95.68531799316406,
      "activations/layer8_attention_weight_max": 40.051109313964844,
      "activations/layer8_attention_weight_min": -42.43284225463867,
      "activations/layer9_attention_weight_max": 33.258262634277344,
      "activations/layer9_attention_weight_min": -34.52614212036133,
      "epoch": 14.78,
      "learning_rate": 5.5268181818181814e-05,
      "loss": 2.756,
      "step": 254300
    },
    {
      "activations/layer0_attention_weight_max": 17.274005889892578,
      "activations/layer0_attention_weight_min": -12.691442489624023,
      "activations/layer10_attention_weight_max": 33.91254806518555,
      "activations/layer10_attention_weight_min": -36.20827102661133,
      "activations/layer11_attention_weight_max": 34.04372787475586,
      "activations/layer11_attention_weight_min": -35.64027404785156,
      "activations/layer12_attention_weight_max": 27.895381927490234,
      "activations/layer12_attention_weight_min": -27.544591903686523,
      "activations/layer13_attention_weight_max": 41.1653938293457,
      "activations/layer13_attention_weight_min": -36.45518493652344,
      "activations/layer14_attention_weight_max": 45.225196838378906,
      "activations/layer14_attention_weight_min": -39.08031463623047,
      "activations/layer15_attention_weight_max": 41.932769775390625,
      "activations/layer15_attention_weight_min": -34.750816345214844,
      "activations/layer16_attention_weight_max": 31.713109970092773,
      "activations/layer16_attention_weight_min": -27.634721755981445,
      "activations/layer17_attention_weight_max": 32.70589065551758,
      "activations/layer17_attention_weight_min": -28.19153594970703,
      "activations/layer18_attention_weight_max": 34.16018295288086,
      "activations/layer18_attention_weight_min": -25.995874404907227,
      "activations/layer19_attention_weight_max": 36.894466400146484,
      "activations/layer19_attention_weight_min": -32.062313079833984,
      "activations/layer1_attention_weight_max": 16.632118225097656,
      "activations/layer1_attention_weight_min": -16.165618896484375,
      "activations/layer20_attention_weight_max": 29.925249099731445,
      "activations/layer20_attention_weight_min": -25.2206974029541,
      "activations/layer21_attention_weight_max": 30.193321228027344,
      "activations/layer21_attention_weight_min": -24.348026275634766,
      "activations/layer22_attention_weight_max": 44.94965362548828,
      "activations/layer22_attention_weight_min": -29.72869110107422,
      "activations/layer23_attention_weight_max": 33.41535949707031,
      "activations/layer23_attention_weight_min": -27.297178268432617,
      "activations/layer2_attention_weight_max": 35.24223327636719,
      "activations/layer2_attention_weight_min": -31.943008422851562,
      "activations/layer3_attention_weight_max": 100.84445190429688,
      "activations/layer3_attention_weight_min": -98.98050689697266,
      "activations/layer4_attention_weight_max": 114.0496826171875,
      "activations/layer4_attention_weight_min": -112.21192932128906,
      "activations/layer5_attention_weight_max": 51.418941497802734,
      "activations/layer5_attention_weight_min": -60.37754440307617,
      "activations/layer6_attention_weight_max": 46.199424743652344,
      "activations/layer6_attention_weight_min": -47.646995544433594,
      "activations/layer7_attention_weight_max": 91.61146545410156,
      "activations/layer7_attention_weight_min": -94.8588638305664,
      "activations/layer8_attention_weight_max": 41.053550720214844,
      "activations/layer8_attention_weight_min": -42.69163131713867,
      "activations/layer9_attention_weight_max": 34.88268280029297,
      "activations/layer9_attention_weight_min": -34.273704528808594,
      "epoch": 14.78,
      "learning_rate": 5.524924242424242e-05,
      "loss": 2.7649,
      "step": 254350
    },
    {
      "activations/layer0_attention_weight_max": 16.613645553588867,
      "activations/layer0_attention_weight_min": -12.577449798583984,
      "activations/layer10_attention_weight_max": 34.56830596923828,
      "activations/layer10_attention_weight_min": -36.91887283325195,
      "activations/layer11_attention_weight_max": 33.1878776550293,
      "activations/layer11_attention_weight_min": -34.530948638916016,
      "activations/layer12_attention_weight_max": 26.706703186035156,
      "activations/layer12_attention_weight_min": -26.233253479003906,
      "activations/layer13_attention_weight_max": 40.86556625366211,
      "activations/layer13_attention_weight_min": -33.9254264831543,
      "activations/layer14_attention_weight_max": 41.746891021728516,
      "activations/layer14_attention_weight_min": -36.87599563598633,
      "activations/layer15_attention_weight_max": 39.12487030029297,
      "activations/layer15_attention_weight_min": -33.78355026245117,
      "activations/layer16_attention_weight_max": 28.352930068969727,
      "activations/layer16_attention_weight_min": -30.01847267150879,
      "activations/layer17_attention_weight_max": 31.39750099182129,
      "activations/layer17_attention_weight_min": -25.625215530395508,
      "activations/layer18_attention_weight_max": 30.667268753051758,
      "activations/layer18_attention_weight_min": -23.02288055419922,
      "activations/layer19_attention_weight_max": 31.652240753173828,
      "activations/layer19_attention_weight_min": -29.6077880859375,
      "activations/layer1_attention_weight_max": 16.609352111816406,
      "activations/layer1_attention_weight_min": -14.624458312988281,
      "activations/layer20_attention_weight_max": 27.711957931518555,
      "activations/layer20_attention_weight_min": -24.614017486572266,
      "activations/layer21_attention_weight_max": 28.415939331054688,
      "activations/layer21_attention_weight_min": -24.268798828125,
      "activations/layer22_attention_weight_max": 40.29265213012695,
      "activations/layer22_attention_weight_min": -30.981403350830078,
      "activations/layer23_attention_weight_max": 31.221229553222656,
      "activations/layer23_attention_weight_min": -25.20973014831543,
      "activations/layer2_attention_weight_max": 32.192264556884766,
      "activations/layer2_attention_weight_min": -31.640731811523438,
      "activations/layer3_attention_weight_max": 94.4181137084961,
      "activations/layer3_attention_weight_min": -94.96368408203125,
      "activations/layer4_attention_weight_max": 106.76136779785156,
      "activations/layer4_attention_weight_min": -108.6041259765625,
      "activations/layer5_attention_weight_max": 49.231773376464844,
      "activations/layer5_attention_weight_min": -61.251869201660156,
      "activations/layer6_attention_weight_max": 44.43312072753906,
      "activations/layer6_attention_weight_min": -45.648597717285156,
      "activations/layer7_attention_weight_max": 97.23710632324219,
      "activations/layer7_attention_weight_min": -90.63399505615234,
      "activations/layer8_attention_weight_max": 43.82164764404297,
      "activations/layer8_attention_weight_min": -44.41639709472656,
      "activations/layer9_attention_weight_max": 35.516422271728516,
      "activations/layer9_attention_weight_min": -36.2642707824707,
      "epoch": 14.78,
      "learning_rate": 5.5230303030303024e-05,
      "loss": 2.766,
      "step": 254400
    },
    {
      "activations/layer0_attention_weight_max": 15.974414825439453,
      "activations/layer0_attention_weight_min": -13.212788581848145,
      "activations/layer10_attention_weight_max": 34.563899993896484,
      "activations/layer10_attention_weight_min": -34.110145568847656,
      "activations/layer11_attention_weight_max": 34.3502197265625,
      "activations/layer11_attention_weight_min": -34.30021667480469,
      "activations/layer12_attention_weight_max": 26.21958351135254,
      "activations/layer12_attention_weight_min": -26.63730812072754,
      "activations/layer13_attention_weight_max": 37.3067741394043,
      "activations/layer13_attention_weight_min": -35.35363006591797,
      "activations/layer14_attention_weight_max": 41.38078689575195,
      "activations/layer14_attention_weight_min": -36.32835388183594,
      "activations/layer15_attention_weight_max": 36.46879196166992,
      "activations/layer15_attention_weight_min": -33.23619079589844,
      "activations/layer16_attention_weight_max": 30.34874153137207,
      "activations/layer16_attention_weight_min": -28.015579223632812,
      "activations/layer17_attention_weight_max": 33.49203872680664,
      "activations/layer17_attention_weight_min": -25.87981605529785,
      "activations/layer18_attention_weight_max": 31.017539978027344,
      "activations/layer18_attention_weight_min": -25.77501106262207,
      "activations/layer19_attention_weight_max": 40.959747314453125,
      "activations/layer19_attention_weight_min": -31.962636947631836,
      "activations/layer1_attention_weight_max": 16.164409637451172,
      "activations/layer1_attention_weight_min": -14.070869445800781,
      "activations/layer20_attention_weight_max": 32.88391876220703,
      "activations/layer20_attention_weight_min": -25.052772521972656,
      "activations/layer21_attention_weight_max": 30.50726318359375,
      "activations/layer21_attention_weight_min": -24.660173416137695,
      "activations/layer22_attention_weight_max": 40.18839645385742,
      "activations/layer22_attention_weight_min": -30.820615768432617,
      "activations/layer23_attention_weight_max": 36.02088165283203,
      "activations/layer23_attention_weight_min": -25.47279930114746,
      "activations/layer2_attention_weight_max": 35.56582260131836,
      "activations/layer2_attention_weight_min": -32.74444580078125,
      "activations/layer3_attention_weight_max": 97.67617797851562,
      "activations/layer3_attention_weight_min": -106.0650405883789,
      "activations/layer4_attention_weight_max": 112.4616470336914,
      "activations/layer4_attention_weight_min": -115.8610610961914,
      "activations/layer5_attention_weight_max": 54.79191589355469,
      "activations/layer5_attention_weight_min": -64.86630249023438,
      "activations/layer6_attention_weight_max": 45.04316329956055,
      "activations/layer6_attention_weight_min": -47.04482650756836,
      "activations/layer7_attention_weight_max": 95.80826568603516,
      "activations/layer7_attention_weight_min": -94.71143341064453,
      "activations/layer8_attention_weight_max": 41.79202651977539,
      "activations/layer8_attention_weight_min": -42.56328201293945,
      "activations/layer9_attention_weight_max": 32.94786071777344,
      "activations/layer9_attention_weight_min": -34.84157180786133,
      "epoch": 14.78,
      "learning_rate": 5.5211363636363626e-05,
      "loss": 2.7505,
      "step": 254450
    },
    {
      "activations/layer0_attention_weight_max": 16.387922286987305,
      "activations/layer0_attention_weight_min": -12.6928071975708,
      "activations/layer10_attention_weight_max": 33.828792572021484,
      "activations/layer10_attention_weight_min": -34.055416107177734,
      "activations/layer11_attention_weight_max": 32.87963104248047,
      "activations/layer11_attention_weight_min": -34.425682067871094,
      "activations/layer12_attention_weight_max": 27.482271194458008,
      "activations/layer12_attention_weight_min": -26.618701934814453,
      "activations/layer13_attention_weight_max": 40.7691764831543,
      "activations/layer13_attention_weight_min": -36.33980178833008,
      "activations/layer14_attention_weight_max": 46.31327438354492,
      "activations/layer14_attention_weight_min": -39.42655563354492,
      "activations/layer15_attention_weight_max": 38.507057189941406,
      "activations/layer15_attention_weight_min": -34.058204650878906,
      "activations/layer16_attention_weight_max": 31.89099884033203,
      "activations/layer16_attention_weight_min": -28.501131057739258,
      "activations/layer17_attention_weight_max": 35.250362396240234,
      "activations/layer17_attention_weight_min": -26.81049156188965,
      "activations/layer18_attention_weight_max": 31.72935676574707,
      "activations/layer18_attention_weight_min": -24.22458839416504,
      "activations/layer19_attention_weight_max": 37.74241256713867,
      "activations/layer19_attention_weight_min": -31.282072067260742,
      "activations/layer1_attention_weight_max": 17.379650115966797,
      "activations/layer1_attention_weight_min": -14.174842834472656,
      "activations/layer20_attention_weight_max": 31.54694175720215,
      "activations/layer20_attention_weight_min": -26.07867431640625,
      "activations/layer21_attention_weight_max": 32.465301513671875,
      "activations/layer21_attention_weight_min": -25.187969207763672,
      "activations/layer22_attention_weight_max": 43.695823669433594,
      "activations/layer22_attention_weight_min": -29.965166091918945,
      "activations/layer23_attention_weight_max": 36.1599235534668,
      "activations/layer23_attention_weight_min": -24.913162231445312,
      "activations/layer2_attention_weight_max": 33.42638397216797,
      "activations/layer2_attention_weight_min": -32.370582580566406,
      "activations/layer3_attention_weight_max": 101.6707992553711,
      "activations/layer3_attention_weight_min": -100.02213287353516,
      "activations/layer4_attention_weight_max": 118.09746551513672,
      "activations/layer4_attention_weight_min": -117.7677230834961,
      "activations/layer5_attention_weight_max": 50.940406799316406,
      "activations/layer5_attention_weight_min": -64.59844970703125,
      "activations/layer6_attention_weight_max": 43.45063781738281,
      "activations/layer6_attention_weight_min": -48.48688888549805,
      "activations/layer7_attention_weight_max": 93.79532623291016,
      "activations/layer7_attention_weight_min": -94.119384765625,
      "activations/layer8_attention_weight_max": 42.070430755615234,
      "activations/layer8_attention_weight_min": -41.07687759399414,
      "activations/layer9_attention_weight_max": 33.32648849487305,
      "activations/layer9_attention_weight_min": -33.920101165771484,
      "epoch": 14.79,
      "learning_rate": 5.519242424242424e-05,
      "loss": 2.7558,
      "step": 254500
    },
    {
      "activations/layer0_attention_weight_max": 15.872262001037598,
      "activations/layer0_attention_weight_min": -12.473089218139648,
      "activations/layer10_attention_weight_max": 32.9540901184082,
      "activations/layer10_attention_weight_min": -32.49359130859375,
      "activations/layer11_attention_weight_max": 32.73695755004883,
      "activations/layer11_attention_weight_min": -31.51272201538086,
      "activations/layer12_attention_weight_max": 26.400278091430664,
      "activations/layer12_attention_weight_min": -26.89207649230957,
      "activations/layer13_attention_weight_max": 40.32598876953125,
      "activations/layer13_attention_weight_min": -35.77030563354492,
      "activations/layer14_attention_weight_max": 42.78047180175781,
      "activations/layer14_attention_weight_min": -43.01653289794922,
      "activations/layer15_attention_weight_max": 35.802703857421875,
      "activations/layer15_attention_weight_min": -34.35615921020508,
      "activations/layer16_attention_weight_max": 28.70680809020996,
      "activations/layer16_attention_weight_min": -28.40897560119629,
      "activations/layer17_attention_weight_max": 30.517438888549805,
      "activations/layer17_attention_weight_min": -25.937856674194336,
      "activations/layer18_attention_weight_max": 31.24128532409668,
      "activations/layer18_attention_weight_min": -26.256567001342773,
      "activations/layer19_attention_weight_max": 32.07318115234375,
      "activations/layer19_attention_weight_min": -32.63511657714844,
      "activations/layer1_attention_weight_max": 16.062036514282227,
      "activations/layer1_attention_weight_min": -14.215876579284668,
      "activations/layer20_attention_weight_max": 28.453279495239258,
      "activations/layer20_attention_weight_min": -24.833024978637695,
      "activations/layer21_attention_weight_max": 27.228084564208984,
      "activations/layer21_attention_weight_min": -24.92799949645996,
      "activations/layer22_attention_weight_max": 42.877960205078125,
      "activations/layer22_attention_weight_min": -30.28238296508789,
      "activations/layer23_attention_weight_max": 32.736576080322266,
      "activations/layer23_attention_weight_min": -24.91950798034668,
      "activations/layer2_attention_weight_max": 32.85997772216797,
      "activations/layer2_attention_weight_min": -32.55011749267578,
      "activations/layer3_attention_weight_max": 97.35548400878906,
      "activations/layer3_attention_weight_min": -99.87760925292969,
      "activations/layer4_attention_weight_max": 110.42315673828125,
      "activations/layer4_attention_weight_min": -117.2345199584961,
      "activations/layer5_attention_weight_max": 51.422645568847656,
      "activations/layer5_attention_weight_min": -63.15692138671875,
      "activations/layer6_attention_weight_max": 45.72514343261719,
      "activations/layer6_attention_weight_min": -48.421775817871094,
      "activations/layer7_attention_weight_max": 90.97984313964844,
      "activations/layer7_attention_weight_min": -105.06216430664062,
      "activations/layer8_attention_weight_max": 41.54945755004883,
      "activations/layer8_attention_weight_min": -42.37135696411133,
      "activations/layer9_attention_weight_max": 35.06388473510742,
      "activations/layer9_attention_weight_min": -33.8001708984375,
      "epoch": 14.79,
      "learning_rate": 5.517348484848484e-05,
      "loss": 2.7598,
      "step": 254550
    },
    {
      "activations/layer0_attention_weight_max": 15.97991943359375,
      "activations/layer0_attention_weight_min": -12.72584056854248,
      "activations/layer10_attention_weight_max": 35.6207275390625,
      "activations/layer10_attention_weight_min": -32.497520446777344,
      "activations/layer11_attention_weight_max": 33.74018478393555,
      "activations/layer11_attention_weight_min": -33.16946029663086,
      "activations/layer12_attention_weight_max": 27.091615676879883,
      "activations/layer12_attention_weight_min": -26.22060775756836,
      "activations/layer13_attention_weight_max": 39.019569396972656,
      "activations/layer13_attention_weight_min": -34.036678314208984,
      "activations/layer14_attention_weight_max": 41.13814163208008,
      "activations/layer14_attention_weight_min": -36.546016693115234,
      "activations/layer15_attention_weight_max": 37.45097732543945,
      "activations/layer15_attention_weight_min": -32.976871490478516,
      "activations/layer16_attention_weight_max": 32.02973175048828,
      "activations/layer16_attention_weight_min": -28.826244354248047,
      "activations/layer17_attention_weight_max": 32.46183776855469,
      "activations/layer17_attention_weight_min": -26.5169677734375,
      "activations/layer18_attention_weight_max": 33.26816940307617,
      "activations/layer18_attention_weight_min": -23.08405303955078,
      "activations/layer19_attention_weight_max": 36.193233489990234,
      "activations/layer19_attention_weight_min": -30.22757339477539,
      "activations/layer1_attention_weight_max": 15.853144645690918,
      "activations/layer1_attention_weight_min": -13.676281929016113,
      "activations/layer20_attention_weight_max": 30.796648025512695,
      "activations/layer20_attention_weight_min": -23.264324188232422,
      "activations/layer21_attention_weight_max": 29.86079978942871,
      "activations/layer21_attention_weight_min": -22.382972717285156,
      "activations/layer22_attention_weight_max": 42.10841369628906,
      "activations/layer22_attention_weight_min": -28.49513816833496,
      "activations/layer23_attention_weight_max": 32.96406555175781,
      "activations/layer23_attention_weight_min": -23.57004165649414,
      "activations/layer2_attention_weight_max": 32.68741989135742,
      "activations/layer2_attention_weight_min": -31.222593307495117,
      "activations/layer3_attention_weight_max": 98.04375457763672,
      "activations/layer3_attention_weight_min": -95.98005676269531,
      "activations/layer4_attention_weight_max": 111.59618377685547,
      "activations/layer4_attention_weight_min": -112.96761322021484,
      "activations/layer5_attention_weight_max": 54.55494689941406,
      "activations/layer5_attention_weight_min": -63.59184265136719,
      "activations/layer6_attention_weight_max": 41.967498779296875,
      "activations/layer6_attention_weight_min": -45.650875091552734,
      "activations/layer7_attention_weight_max": 90.53092193603516,
      "activations/layer7_attention_weight_min": -92.1702651977539,
      "activations/layer8_attention_weight_max": 39.853275299072266,
      "activations/layer8_attention_weight_min": -41.62587356567383,
      "activations/layer9_attention_weight_max": 32.94369888305664,
      "activations/layer9_attention_weight_min": -33.85649108886719,
      "epoch": 14.79,
      "learning_rate": 5.515454545454545e-05,
      "loss": 2.7436,
      "step": 254600
    },
    {
      "activations/layer0_attention_weight_max": 15.083160400390625,
      "activations/layer0_attention_weight_min": -11.956660270690918,
      "activations/layer10_attention_weight_max": 32.45683288574219,
      "activations/layer10_attention_weight_min": -33.53611755371094,
      "activations/layer11_attention_weight_max": 33.5092887878418,
      "activations/layer11_attention_weight_min": -33.437068939208984,
      "activations/layer12_attention_weight_max": 27.87455177307129,
      "activations/layer12_attention_weight_min": -25.997148513793945,
      "activations/layer13_attention_weight_max": 40.9353141784668,
      "activations/layer13_attention_weight_min": -37.54505920410156,
      "activations/layer14_attention_weight_max": 46.36920928955078,
      "activations/layer14_attention_weight_min": -39.535491943359375,
      "activations/layer15_attention_weight_max": 40.663368225097656,
      "activations/layer15_attention_weight_min": -34.39179229736328,
      "activations/layer16_attention_weight_max": 33.02689743041992,
      "activations/layer16_attention_weight_min": -27.439172744750977,
      "activations/layer17_attention_weight_max": 31.535701751708984,
      "activations/layer17_attention_weight_min": -25.997941970825195,
      "activations/layer18_attention_weight_max": 33.348873138427734,
      "activations/layer18_attention_weight_min": -25.99565315246582,
      "activations/layer19_attention_weight_max": 36.69662094116211,
      "activations/layer19_attention_weight_min": -30.659305572509766,
      "activations/layer1_attention_weight_max": 16.69081687927246,
      "activations/layer1_attention_weight_min": -13.640398025512695,
      "activations/layer20_attention_weight_max": 31.254892349243164,
      "activations/layer20_attention_weight_min": -24.693157196044922,
      "activations/layer21_attention_weight_max": 32.27134704589844,
      "activations/layer21_attention_weight_min": -24.360321044921875,
      "activations/layer22_attention_weight_max": 43.649810791015625,
      "activations/layer22_attention_weight_min": -30.98614501953125,
      "activations/layer23_attention_weight_max": 37.64368438720703,
      "activations/layer23_attention_weight_min": -24.56740379333496,
      "activations/layer2_attention_weight_max": 31.88278579711914,
      "activations/layer2_attention_weight_min": -31.108793258666992,
      "activations/layer3_attention_weight_max": 93.48957824707031,
      "activations/layer3_attention_weight_min": -95.01771545410156,
      "activations/layer4_attention_weight_max": 110.3297348022461,
      "activations/layer4_attention_weight_min": -121.79115295410156,
      "activations/layer5_attention_weight_max": 49.99566650390625,
      "activations/layer5_attention_weight_min": -63.42600631713867,
      "activations/layer6_attention_weight_max": 44.534950256347656,
      "activations/layer6_attention_weight_min": -48.4937858581543,
      "activations/layer7_attention_weight_max": 90.6021499633789,
      "activations/layer7_attention_weight_min": -94.7922134399414,
      "activations/layer8_attention_weight_max": 40.98855972290039,
      "activations/layer8_attention_weight_min": -43.231693267822266,
      "activations/layer9_attention_weight_max": 32.680397033691406,
      "activations/layer9_attention_weight_min": -34.642662048339844,
      "epoch": 14.8,
      "learning_rate": 5.513560606060605e-05,
      "loss": 2.7506,
      "step": 254650
    },
    {
      "activations/layer0_attention_weight_max": 15.686845779418945,
      "activations/layer0_attention_weight_min": -12.059403419494629,
      "activations/layer10_attention_weight_max": 32.35237121582031,
      "activations/layer10_attention_weight_min": -36.37739562988281,
      "activations/layer11_attention_weight_max": 33.705204010009766,
      "activations/layer11_attention_weight_min": -32.45231628417969,
      "activations/layer12_attention_weight_max": 25.903018951416016,
      "activations/layer12_attention_weight_min": -27.033781051635742,
      "activations/layer13_attention_weight_max": 38.85139465332031,
      "activations/layer13_attention_weight_min": -37.14219284057617,
      "activations/layer14_attention_weight_max": 43.63914108276367,
      "activations/layer14_attention_weight_min": -38.90414047241211,
      "activations/layer15_attention_weight_max": 37.29846954345703,
      "activations/layer15_attention_weight_min": -34.239131927490234,
      "activations/layer16_attention_weight_max": 30.878190994262695,
      "activations/layer16_attention_weight_min": -27.688413619995117,
      "activations/layer17_attention_weight_max": 30.450233459472656,
      "activations/layer17_attention_weight_min": -27.889066696166992,
      "activations/layer18_attention_weight_max": 32.999717712402344,
      "activations/layer18_attention_weight_min": -24.572851181030273,
      "activations/layer19_attention_weight_max": 33.59833526611328,
      "activations/layer19_attention_weight_min": -31.515640258789062,
      "activations/layer1_attention_weight_max": 18.042387008666992,
      "activations/layer1_attention_weight_min": -17.258081436157227,
      "activations/layer20_attention_weight_max": 30.665170669555664,
      "activations/layer20_attention_weight_min": -25.613784790039062,
      "activations/layer21_attention_weight_max": 32.276145935058594,
      "activations/layer21_attention_weight_min": -23.61083984375,
      "activations/layer22_attention_weight_max": 46.961082458496094,
      "activations/layer22_attention_weight_min": -30.428117752075195,
      "activations/layer23_attention_weight_max": 34.46470260620117,
      "activations/layer23_attention_weight_min": -26.242841720581055,
      "activations/layer2_attention_weight_max": 35.7415771484375,
      "activations/layer2_attention_weight_min": -32.72183609008789,
      "activations/layer3_attention_weight_max": 97.47108459472656,
      "activations/layer3_attention_weight_min": -99.0992202758789,
      "activations/layer4_attention_weight_max": 111.21437072753906,
      "activations/layer4_attention_weight_min": -109.25493621826172,
      "activations/layer5_attention_weight_max": 51.50926208496094,
      "activations/layer5_attention_weight_min": -61.748809814453125,
      "activations/layer6_attention_weight_max": 48.09965896606445,
      "activations/layer6_attention_weight_min": -48.77507781982422,
      "activations/layer7_attention_weight_max": 93.06195068359375,
      "activations/layer7_attention_weight_min": -91.9515151977539,
      "activations/layer8_attention_weight_max": 38.561363220214844,
      "activations/layer8_attention_weight_min": -43.48278045654297,
      "activations/layer9_attention_weight_max": 30.893156051635742,
      "activations/layer9_attention_weight_min": -36.73845672607422,
      "epoch": 14.8,
      "learning_rate": 5.511666666666667e-05,
      "loss": 2.7371,
      "step": 254700
    },
    {
      "activations/layer0_attention_weight_max": 15.310225486755371,
      "activations/layer0_attention_weight_min": -12.382685661315918,
      "activations/layer10_attention_weight_max": 34.595638275146484,
      "activations/layer10_attention_weight_min": -33.836002349853516,
      "activations/layer11_attention_weight_max": 37.18272399902344,
      "activations/layer11_attention_weight_min": -34.41392135620117,
      "activations/layer12_attention_weight_max": 25.95273208618164,
      "activations/layer12_attention_weight_min": -27.34912872314453,
      "activations/layer13_attention_weight_max": 39.85124206542969,
      "activations/layer13_attention_weight_min": -35.99314880371094,
      "activations/layer14_attention_weight_max": 42.59748458862305,
      "activations/layer14_attention_weight_min": -38.92490768432617,
      "activations/layer15_attention_weight_max": 37.02627182006836,
      "activations/layer15_attention_weight_min": -36.190025329589844,
      "activations/layer16_attention_weight_max": 28.149599075317383,
      "activations/layer16_attention_weight_min": -29.014867782592773,
      "activations/layer17_attention_weight_max": 34.32918930053711,
      "activations/layer17_attention_weight_min": -27.520551681518555,
      "activations/layer18_attention_weight_max": 36.80927658081055,
      "activations/layer18_attention_weight_min": -25.711273193359375,
      "activations/layer19_attention_weight_max": 37.31423568725586,
      "activations/layer19_attention_weight_min": -33.5400390625,
      "activations/layer1_attention_weight_max": 16.802885055541992,
      "activations/layer1_attention_weight_min": -14.072273254394531,
      "activations/layer20_attention_weight_max": 32.82395553588867,
      "activations/layer20_attention_weight_min": -26.53790283203125,
      "activations/layer21_attention_weight_max": 31.87114715576172,
      "activations/layer21_attention_weight_min": -24.858423233032227,
      "activations/layer22_attention_weight_max": 54.815372467041016,
      "activations/layer22_attention_weight_min": -31.081588745117188,
      "activations/layer23_attention_weight_max": 37.179100036621094,
      "activations/layer23_attention_weight_min": -24.46319580078125,
      "activations/layer2_attention_weight_max": 32.965736389160156,
      "activations/layer2_attention_weight_min": -32.53609085083008,
      "activations/layer3_attention_weight_max": 98.99520111083984,
      "activations/layer3_attention_weight_min": -98.28678894042969,
      "activations/layer4_attention_weight_max": 116.68453216552734,
      "activations/layer4_attention_weight_min": -106.80146789550781,
      "activations/layer5_attention_weight_max": 56.35658264160156,
      "activations/layer5_attention_weight_min": -58.85811996459961,
      "activations/layer6_attention_weight_max": 45.39796447753906,
      "activations/layer6_attention_weight_min": -47.40745162963867,
      "activations/layer7_attention_weight_max": 95.2902603149414,
      "activations/layer7_attention_weight_min": -95.78832244873047,
      "activations/layer8_attention_weight_max": 40.365455627441406,
      "activations/layer8_attention_weight_min": -42.84547424316406,
      "activations/layer9_attention_weight_max": 34.11570358276367,
      "activations/layer9_attention_weight_min": -33.87844467163086,
      "epoch": 14.8,
      "learning_rate": 5.509772727272727e-05,
      "loss": 2.7625,
      "step": 254750
    },
    {
      "activations/layer0_attention_weight_max": 15.867969512939453,
      "activations/layer0_attention_weight_min": -13.080036163330078,
      "activations/layer10_attention_weight_max": 33.79463577270508,
      "activations/layer10_attention_weight_min": -32.2288932800293,
      "activations/layer11_attention_weight_max": 32.883697509765625,
      "activations/layer11_attention_weight_min": -31.784481048583984,
      "activations/layer12_attention_weight_max": 25.500640869140625,
      "activations/layer12_attention_weight_min": -23.998136520385742,
      "activations/layer13_attention_weight_max": 39.86370086669922,
      "activations/layer13_attention_weight_min": -32.381690979003906,
      "activations/layer14_attention_weight_max": 42.28481674194336,
      "activations/layer14_attention_weight_min": -35.10773468017578,
      "activations/layer15_attention_weight_max": 38.490997314453125,
      "activations/layer15_attention_weight_min": -32.68282699584961,
      "activations/layer16_attention_weight_max": 28.226545333862305,
      "activations/layer16_attention_weight_min": -27.349123001098633,
      "activations/layer17_attention_weight_max": 30.24823760986328,
      "activations/layer17_attention_weight_min": -25.25199317932129,
      "activations/layer18_attention_weight_max": 30.975744247436523,
      "activations/layer18_attention_weight_min": -25.498502731323242,
      "activations/layer19_attention_weight_max": 39.49793243408203,
      "activations/layer19_attention_weight_min": -32.14586639404297,
      "activations/layer1_attention_weight_max": 15.744519233703613,
      "activations/layer1_attention_weight_min": -14.553481101989746,
      "activations/layer20_attention_weight_max": 28.54303550720215,
      "activations/layer20_attention_weight_min": -24.41158676147461,
      "activations/layer21_attention_weight_max": 28.135549545288086,
      "activations/layer21_attention_weight_min": -24.878890991210938,
      "activations/layer22_attention_weight_max": 41.8879280090332,
      "activations/layer22_attention_weight_min": -31.888160705566406,
      "activations/layer23_attention_weight_max": 35.1201171875,
      "activations/layer23_attention_weight_min": -24.71442222595215,
      "activations/layer2_attention_weight_max": 32.39714050292969,
      "activations/layer2_attention_weight_min": -31.558319091796875,
      "activations/layer3_attention_weight_max": 96.20201110839844,
      "activations/layer3_attention_weight_min": -97.29156494140625,
      "activations/layer4_attention_weight_max": 108.76634979248047,
      "activations/layer4_attention_weight_min": -111.51671600341797,
      "activations/layer5_attention_weight_max": 51.357383728027344,
      "activations/layer5_attention_weight_min": -61.0548210144043,
      "activations/layer6_attention_weight_max": 41.818145751953125,
      "activations/layer6_attention_weight_min": -46.31057357788086,
      "activations/layer7_attention_weight_max": 87.1295394897461,
      "activations/layer7_attention_weight_min": -90.2281494140625,
      "activations/layer8_attention_weight_max": 40.3673210144043,
      "activations/layer8_attention_weight_min": -41.79690933227539,
      "activations/layer9_attention_weight_max": 32.576141357421875,
      "activations/layer9_attention_weight_min": -33.27122116088867,
      "epoch": 14.81,
      "learning_rate": 5.507878787878787e-05,
      "loss": 2.7475,
      "step": 254800
    },
    {
      "activations/layer0_attention_weight_max": 16.226604461669922,
      "activations/layer0_attention_weight_min": -12.0845365524292,
      "activations/layer10_attention_weight_max": 30.836441040039062,
      "activations/layer10_attention_weight_min": -32.887451171875,
      "activations/layer11_attention_weight_max": 33.78728103637695,
      "activations/layer11_attention_weight_min": -31.906448364257812,
      "activations/layer12_attention_weight_max": 25.631498336791992,
      "activations/layer12_attention_weight_min": -25.09827995300293,
      "activations/layer13_attention_weight_max": 37.87138748168945,
      "activations/layer13_attention_weight_min": -35.32856750488281,
      "activations/layer14_attention_weight_max": 45.49834060668945,
      "activations/layer14_attention_weight_min": -39.53867721557617,
      "activations/layer15_attention_weight_max": 40.03645324707031,
      "activations/layer15_attention_weight_min": -32.97980499267578,
      "activations/layer16_attention_weight_max": 29.767833709716797,
      "activations/layer16_attention_weight_min": -27.675371170043945,
      "activations/layer17_attention_weight_max": 30.733415603637695,
      "activations/layer17_attention_weight_min": -24.188997268676758,
      "activations/layer18_attention_weight_max": 31.03740119934082,
      "activations/layer18_attention_weight_min": -23.520992279052734,
      "activations/layer19_attention_weight_max": 32.616817474365234,
      "activations/layer19_attention_weight_min": -30.219003677368164,
      "activations/layer1_attention_weight_max": 16.65827751159668,
      "activations/layer1_attention_weight_min": -14.875886917114258,
      "activations/layer20_attention_weight_max": 28.485633850097656,
      "activations/layer20_attention_weight_min": -25.65734100341797,
      "activations/layer21_attention_weight_max": 28.920730590820312,
      "activations/layer21_attention_weight_min": -24.214040756225586,
      "activations/layer22_attention_weight_max": 42.27548599243164,
      "activations/layer22_attention_weight_min": -28.2749080657959,
      "activations/layer23_attention_weight_max": 33.433956146240234,
      "activations/layer23_attention_weight_min": -24.75222396850586,
      "activations/layer2_attention_weight_max": 33.09039306640625,
      "activations/layer2_attention_weight_min": -33.23153305053711,
      "activations/layer3_attention_weight_max": 95.33680725097656,
      "activations/layer3_attention_weight_min": -95.66600036621094,
      "activations/layer4_attention_weight_max": 113.874755859375,
      "activations/layer4_attention_weight_min": -113.41422271728516,
      "activations/layer5_attention_weight_max": 57.93909454345703,
      "activations/layer5_attention_weight_min": -63.28327178955078,
      "activations/layer6_attention_weight_max": 43.17129135131836,
      "activations/layer6_attention_weight_min": -47.31607437133789,
      "activations/layer7_attention_weight_max": 92.1442642211914,
      "activations/layer7_attention_weight_min": -92.14838409423828,
      "activations/layer8_attention_weight_max": 41.42387771606445,
      "activations/layer8_attention_weight_min": -40.64271545410156,
      "activations/layer9_attention_weight_max": 31.705610275268555,
      "activations/layer9_attention_weight_min": -33.55072784423828,
      "epoch": 14.81,
      "learning_rate": 5.505984848484848e-05,
      "loss": 2.7682,
      "step": 254850
    },
    {
      "activations/layer0_attention_weight_max": 16.214683532714844,
      "activations/layer0_attention_weight_min": -12.283238410949707,
      "activations/layer10_attention_weight_max": 38.521121978759766,
      "activations/layer10_attention_weight_min": -34.927398681640625,
      "activations/layer11_attention_weight_max": 38.10272979736328,
      "activations/layer11_attention_weight_min": -34.41271209716797,
      "activations/layer12_attention_weight_max": 29.331317901611328,
      "activations/layer12_attention_weight_min": -27.63509750366211,
      "activations/layer13_attention_weight_max": 43.10997009277344,
      "activations/layer13_attention_weight_min": -39.12107849121094,
      "activations/layer14_attention_weight_max": 45.49131393432617,
      "activations/layer14_attention_weight_min": -42.0093879699707,
      "activations/layer15_attention_weight_max": 48.65172576904297,
      "activations/layer15_attention_weight_min": -37.712364196777344,
      "activations/layer16_attention_weight_max": 31.85363006591797,
      "activations/layer16_attention_weight_min": -27.91775894165039,
      "activations/layer17_attention_weight_max": 35.40425491333008,
      "activations/layer17_attention_weight_min": -29.39674186706543,
      "activations/layer18_attention_weight_max": 34.10563659667969,
      "activations/layer18_attention_weight_min": -25.043968200683594,
      "activations/layer19_attention_weight_max": 38.73744201660156,
      "activations/layer19_attention_weight_min": -32.93599319458008,
      "activations/layer1_attention_weight_max": 16.527786254882812,
      "activations/layer1_attention_weight_min": -14.63579273223877,
      "activations/layer20_attention_weight_max": 37.33494186401367,
      "activations/layer20_attention_weight_min": -26.583694458007812,
      "activations/layer21_attention_weight_max": 35.2401008605957,
      "activations/layer21_attention_weight_min": -27.00461196899414,
      "activations/layer22_attention_weight_max": 46.98634338378906,
      "activations/layer22_attention_weight_min": -32.80253219604492,
      "activations/layer23_attention_weight_max": 38.96092987060547,
      "activations/layer23_attention_weight_min": -26.417320251464844,
      "activations/layer2_attention_weight_max": 33.15277099609375,
      "activations/layer2_attention_weight_min": -33.398345947265625,
      "activations/layer3_attention_weight_max": 98.83330535888672,
      "activations/layer3_attention_weight_min": -102.61649322509766,
      "activations/layer4_attention_weight_max": 120.14171600341797,
      "activations/layer4_attention_weight_min": -117.16063690185547,
      "activations/layer5_attention_weight_max": 55.18735885620117,
      "activations/layer5_attention_weight_min": -64.6732406616211,
      "activations/layer6_attention_weight_max": 46.901527404785156,
      "activations/layer6_attention_weight_min": -48.64079284667969,
      "activations/layer7_attention_weight_max": 99.66694641113281,
      "activations/layer7_attention_weight_min": -100.80389404296875,
      "activations/layer8_attention_weight_max": 44.26376724243164,
      "activations/layer8_attention_weight_min": -43.182437896728516,
      "activations/layer9_attention_weight_max": 35.36224365234375,
      "activations/layer9_attention_weight_min": -34.409976959228516,
      "epoch": 14.81,
      "learning_rate": 5.5040909090909086e-05,
      "loss": 2.7453,
      "step": 254900
    },
    {
      "activations/layer0_attention_weight_max": 16.002084732055664,
      "activations/layer0_attention_weight_min": -12.461790084838867,
      "activations/layer10_attention_weight_max": 32.00592803955078,
      "activations/layer10_attention_weight_min": -34.11079788208008,
      "activations/layer11_attention_weight_max": 33.00090789794922,
      "activations/layer11_attention_weight_min": -32.1375732421875,
      "activations/layer12_attention_weight_max": 28.56663703918457,
      "activations/layer12_attention_weight_min": -24.87640953063965,
      "activations/layer13_attention_weight_max": 36.26082992553711,
      "activations/layer13_attention_weight_min": -36.64400100708008,
      "activations/layer14_attention_weight_max": 42.50364303588867,
      "activations/layer14_attention_weight_min": -39.30996322631836,
      "activations/layer15_attention_weight_max": 35.72758483886719,
      "activations/layer15_attention_weight_min": -34.26667404174805,
      "activations/layer16_attention_weight_max": 28.39141273498535,
      "activations/layer16_attention_weight_min": -28.125463485717773,
      "activations/layer17_attention_weight_max": 30.255910873413086,
      "activations/layer17_attention_weight_min": -24.962858200073242,
      "activations/layer18_attention_weight_max": 29.91912269592285,
      "activations/layer18_attention_weight_min": -24.39571189880371,
      "activations/layer19_attention_weight_max": 33.52110290527344,
      "activations/layer19_attention_weight_min": -32.51679992675781,
      "activations/layer1_attention_weight_max": 15.96540355682373,
      "activations/layer1_attention_weight_min": -13.270700454711914,
      "activations/layer20_attention_weight_max": 30.806493759155273,
      "activations/layer20_attention_weight_min": -24.911176681518555,
      "activations/layer21_attention_weight_max": 29.590652465820312,
      "activations/layer21_attention_weight_min": -25.00737762451172,
      "activations/layer22_attention_weight_max": 37.6202278137207,
      "activations/layer22_attention_weight_min": -31.95884132385254,
      "activations/layer23_attention_weight_max": 29.509174346923828,
      "activations/layer23_attention_weight_min": -25.174047470092773,
      "activations/layer2_attention_weight_max": 30.576820373535156,
      "activations/layer2_attention_weight_min": -29.876628875732422,
      "activations/layer3_attention_weight_max": 92.05663299560547,
      "activations/layer3_attention_weight_min": -87.67782592773438,
      "activations/layer4_attention_weight_max": 107.0633773803711,
      "activations/layer4_attention_weight_min": -106.65889739990234,
      "activations/layer5_attention_weight_max": 49.56398010253906,
      "activations/layer5_attention_weight_min": -61.973541259765625,
      "activations/layer6_attention_weight_max": 41.024383544921875,
      "activations/layer6_attention_weight_min": -44.83754348754883,
      "activations/layer7_attention_weight_max": 88.05416107177734,
      "activations/layer7_attention_weight_min": -88.16415405273438,
      "activations/layer8_attention_weight_max": 36.91084671020508,
      "activations/layer8_attention_weight_min": -41.77354431152344,
      "activations/layer9_attention_weight_max": 29.922683715820312,
      "activations/layer9_attention_weight_min": -33.19681930541992,
      "epoch": 14.81,
      "learning_rate": 5.502234848484848e-05,
      "loss": 2.7682,
      "step": 254950
    },
    {
      "activations/layer0_attention_weight_max": 15.272368431091309,
      "activations/layer0_attention_weight_min": -11.93277645111084,
      "activations/layer10_attention_weight_max": 33.05461502075195,
      "activations/layer10_attention_weight_min": -31.8250789642334,
      "activations/layer11_attention_weight_max": 31.887609481811523,
      "activations/layer11_attention_weight_min": -32.085121154785156,
      "activations/layer12_attention_weight_max": 25.47275161743164,
      "activations/layer12_attention_weight_min": -25.548051834106445,
      "activations/layer13_attention_weight_max": 39.0203742980957,
      "activations/layer13_attention_weight_min": -33.67536163330078,
      "activations/layer14_attention_weight_max": 40.15972137451172,
      "activations/layer14_attention_weight_min": -35.98017883300781,
      "activations/layer15_attention_weight_max": 38.13811111450195,
      "activations/layer15_attention_weight_min": -33.344764709472656,
      "activations/layer16_attention_weight_max": 31.37480354309082,
      "activations/layer16_attention_weight_min": -27.709243774414062,
      "activations/layer17_attention_weight_max": 33.98833465576172,
      "activations/layer17_attention_weight_min": -25.25247573852539,
      "activations/layer18_attention_weight_max": 32.045406341552734,
      "activations/layer18_attention_weight_min": -24.175662994384766,
      "activations/layer19_attention_weight_max": 36.968994140625,
      "activations/layer19_attention_weight_min": -28.134851455688477,
      "activations/layer1_attention_weight_max": 17.037687301635742,
      "activations/layer1_attention_weight_min": -14.528726577758789,
      "activations/layer20_attention_weight_max": 29.160888671875,
      "activations/layer20_attention_weight_min": -24.526926040649414,
      "activations/layer21_attention_weight_max": 33.012969970703125,
      "activations/layer21_attention_weight_min": -23.012014389038086,
      "activations/layer22_attention_weight_max": 41.05323028564453,
      "activations/layer22_attention_weight_min": -28.30055809020996,
      "activations/layer23_attention_weight_max": 33.316734313964844,
      "activations/layer23_attention_weight_min": -25.2967586517334,
      "activations/layer2_attention_weight_max": 31.302833557128906,
      "activations/layer2_attention_weight_min": -30.40253448486328,
      "activations/layer3_attention_weight_max": 87.719482421875,
      "activations/layer3_attention_weight_min": -87.08616638183594,
      "activations/layer4_attention_weight_max": 105.05462646484375,
      "activations/layer4_attention_weight_min": -107.90657043457031,
      "activations/layer5_attention_weight_max": 50.75423812866211,
      "activations/layer5_attention_weight_min": -59.5857048034668,
      "activations/layer6_attention_weight_max": 40.947601318359375,
      "activations/layer6_attention_weight_min": -44.54170227050781,
      "activations/layer7_attention_weight_max": 89.89810943603516,
      "activations/layer7_attention_weight_min": -89.87218475341797,
      "activations/layer8_attention_weight_max": 38.536834716796875,
      "activations/layer8_attention_weight_min": -42.19553756713867,
      "activations/layer9_attention_weight_max": 31.87786865234375,
      "activations/layer9_attention_weight_min": -32.67763900756836,
      "epoch": 14.82,
      "learning_rate": 5.5003409090909086e-05,
      "loss": 2.7555,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5152,
      "eval_samples_per_second": 504.277,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5152,
      "eval_openwebtext_samples_per_second": 504.277,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 2.0257,
      "eval_wikitext_samples_per_second": 225.107,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_lambada_loss": 2.57421875,
      "eval_lambada_ppl": 13.121062323276538,
      "eval_lambada_runtime": 9.621,
      "eval_lambada_samples_per_second": 506.08,
      "step": 255000
    },
    {
      "activations/layer0_attention_weight_max": 16.43096160888672,
      "activations/layer0_attention_weight_min": -13.088171005249023,
      "activations/layer10_attention_weight_max": 31.4951229095459,
      "activations/layer10_attention_weight_min": -32.370704650878906,
      "activations/layer11_attention_weight_max": 33.43552780151367,
      "activations/layer11_attention_weight_min": -34.00107192993164,
      "activations/layer12_attention_weight_max": 24.785261154174805,
      "activations/layer12_attention_weight_min": -25.993534088134766,
      "activations/layer13_attention_weight_max": 36.423728942871094,
      "activations/layer13_attention_weight_min": -33.519439697265625,
      "activations/layer14_attention_weight_max": 42.896629333496094,
      "activations/layer14_attention_weight_min": -35.664154052734375,
      "activations/layer15_attention_weight_max": 37.52547836303711,
      "activations/layer15_attention_weight_min": -34.60349655151367,
      "activations/layer16_attention_weight_max": 34.2208137512207,
      "activations/layer16_attention_weight_min": -27.749223709106445,
      "activations/layer17_attention_weight_max": 30.130615234375,
      "activations/layer17_attention_weight_min": -26.84288215637207,
      "activations/layer18_attention_weight_max": 32.1375846862793,
      "activations/layer18_attention_weight_min": -26.232900619506836,
      "activations/layer19_attention_weight_max": 34.87767028808594,
      "activations/layer19_attention_weight_min": -30.40138816833496,
      "activations/layer1_attention_weight_max": 16.47342300415039,
      "activations/layer1_attention_weight_min": -15.87930965423584,
      "activations/layer20_attention_weight_max": 29.18191146850586,
      "activations/layer20_attention_weight_min": -24.084915161132812,
      "activations/layer21_attention_weight_max": 28.4392147064209,
      "activations/layer21_attention_weight_min": -22.971385955810547,
      "activations/layer22_attention_weight_max": 41.29963684082031,
      "activations/layer22_attention_weight_min": -30.768192291259766,
      "activations/layer23_attention_weight_max": 35.81109619140625,
      "activations/layer23_attention_weight_min": -25.170608520507812,
      "activations/layer2_attention_weight_max": 33.8044319152832,
      "activations/layer2_attention_weight_min": -31.68773651123047,
      "activations/layer3_attention_weight_max": 100.14391326904297,
      "activations/layer3_attention_weight_min": -96.62496185302734,
      "activations/layer4_attention_weight_max": 113.60448455810547,
      "activations/layer4_attention_weight_min": -113.5800552368164,
      "activations/layer5_attention_weight_max": 52.74676513671875,
      "activations/layer5_attention_weight_min": -63.749542236328125,
      "activations/layer6_attention_weight_max": 43.661746978759766,
      "activations/layer6_attention_weight_min": -45.85066604614258,
      "activations/layer7_attention_weight_max": 93.46283721923828,
      "activations/layer7_attention_weight_min": -91.0260009765625,
      "activations/layer8_attention_weight_max": 40.163352966308594,
      "activations/layer8_attention_weight_min": -41.67646026611328,
      "activations/layer9_attention_weight_max": 31.802642822265625,
      "activations/layer9_attention_weight_min": -32.744693756103516,
      "epoch": 14.82,
      "learning_rate": 5.4984469696969694e-05,
      "loss": 2.7551,
      "step": 255050
    },
    {
      "activations/layer0_attention_weight_max": 16.259008407592773,
      "activations/layer0_attention_weight_min": -13.656875610351562,
      "activations/layer10_attention_weight_max": 33.09788131713867,
      "activations/layer10_attention_weight_min": -34.42922592163086,
      "activations/layer11_attention_weight_max": 33.0899658203125,
      "activations/layer11_attention_weight_min": -32.34132385253906,
      "activations/layer12_attention_weight_max": 26.056507110595703,
      "activations/layer12_attention_weight_min": -28.483739852905273,
      "activations/layer13_attention_weight_max": 37.60052490234375,
      "activations/layer13_attention_weight_min": -35.915435791015625,
      "activations/layer14_attention_weight_max": 44.34773254394531,
      "activations/layer14_attention_weight_min": -38.36554718017578,
      "activations/layer15_attention_weight_max": 37.70743942260742,
      "activations/layer15_attention_weight_min": -36.73485565185547,
      "activations/layer16_attention_weight_max": 30.660497665405273,
      "activations/layer16_attention_weight_min": -27.394235610961914,
      "activations/layer17_attention_weight_max": 29.7532958984375,
      "activations/layer17_attention_weight_min": -25.84495735168457,
      "activations/layer18_attention_weight_max": 31.7291259765625,
      "activations/layer18_attention_weight_min": -25.662939071655273,
      "activations/layer19_attention_weight_max": 33.08824157714844,
      "activations/layer19_attention_weight_min": -33.04426574707031,
      "activations/layer1_attention_weight_max": 16.31919288635254,
      "activations/layer1_attention_weight_min": -15.48940372467041,
      "activations/layer20_attention_weight_max": 28.45125961303711,
      "activations/layer20_attention_weight_min": -28.291154861450195,
      "activations/layer21_attention_weight_max": 31.09972381591797,
      "activations/layer21_attention_weight_min": -28.043004989624023,
      "activations/layer22_attention_weight_max": 42.11250305175781,
      "activations/layer22_attention_weight_min": -34.44520568847656,
      "activations/layer23_attention_weight_max": 35.099334716796875,
      "activations/layer23_attention_weight_min": -26.584178924560547,
      "activations/layer2_attention_weight_max": 31.14063835144043,
      "activations/layer2_attention_weight_min": -34.000667572021484,
      "activations/layer3_attention_weight_max": 92.1356430053711,
      "activations/layer3_attention_weight_min": -103.3187026977539,
      "activations/layer4_attention_weight_max": 107.54878997802734,
      "activations/layer4_attention_weight_min": -112.3700942993164,
      "activations/layer5_attention_weight_max": 49.43561935424805,
      "activations/layer5_attention_weight_min": -63.16654968261719,
      "activations/layer6_attention_weight_max": 44.11259078979492,
      "activations/layer6_attention_weight_min": -48.532711029052734,
      "activations/layer7_attention_weight_max": 90.11398315429688,
      "activations/layer7_attention_weight_min": -101.2203140258789,
      "activations/layer8_attention_weight_max": 39.42164993286133,
      "activations/layer8_attention_weight_min": -45.71364212036133,
      "activations/layer9_attention_weight_max": 30.372045516967773,
      "activations/layer9_attention_weight_min": -34.650726318359375,
      "epoch": 14.82,
      "learning_rate": 5.49655303030303e-05,
      "loss": 2.754,
      "step": 255100
    },
    {
      "activations/layer0_attention_weight_max": 15.469939231872559,
      "activations/layer0_attention_weight_min": -12.939268112182617,
      "activations/layer10_attention_weight_max": 33.89170455932617,
      "activations/layer10_attention_weight_min": -34.91502380371094,
      "activations/layer11_attention_weight_max": 33.947967529296875,
      "activations/layer11_attention_weight_min": -34.53563690185547,
      "activations/layer12_attention_weight_max": 27.63580894470215,
      "activations/layer12_attention_weight_min": -26.642208099365234,
      "activations/layer13_attention_weight_max": 41.819480895996094,
      "activations/layer13_attention_weight_min": -38.22319030761719,
      "activations/layer14_attention_weight_max": 47.30073165893555,
      "activations/layer14_attention_weight_min": -39.142967224121094,
      "activations/layer15_attention_weight_max": 41.48207092285156,
      "activations/layer15_attention_weight_min": -36.4930419921875,
      "activations/layer16_attention_weight_max": 35.70268630981445,
      "activations/layer16_attention_weight_min": -30.040645599365234,
      "activations/layer17_attention_weight_max": 38.09248733520508,
      "activations/layer17_attention_weight_min": -26.96622657775879,
      "activations/layer18_attention_weight_max": 33.099578857421875,
      "activations/layer18_attention_weight_min": -26.985876083374023,
      "activations/layer19_attention_weight_max": 35.221439361572266,
      "activations/layer19_attention_weight_min": -32.4812126159668,
      "activations/layer1_attention_weight_max": 16.096681594848633,
      "activations/layer1_attention_weight_min": -14.929664611816406,
      "activations/layer20_attention_weight_max": 32.09928894042969,
      "activations/layer20_attention_weight_min": -27.129587173461914,
      "activations/layer21_attention_weight_max": 32.31795120239258,
      "activations/layer21_attention_weight_min": -25.24833106994629,
      "activations/layer22_attention_weight_max": 44.98147964477539,
      "activations/layer22_attention_weight_min": -32.642120361328125,
      "activations/layer23_attention_weight_max": 37.29722595214844,
      "activations/layer23_attention_weight_min": -26.197853088378906,
      "activations/layer2_attention_weight_max": 33.214820861816406,
      "activations/layer2_attention_weight_min": -33.24110412597656,
      "activations/layer3_attention_weight_max": 99.85174560546875,
      "activations/layer3_attention_weight_min": -98.75508117675781,
      "activations/layer4_attention_weight_max": 113.74775695800781,
      "activations/layer4_attention_weight_min": -111.40217590332031,
      "activations/layer5_attention_weight_max": 52.781639099121094,
      "activations/layer5_attention_weight_min": -61.14512634277344,
      "activations/layer6_attention_weight_max": 46.29597854614258,
      "activations/layer6_attention_weight_min": -47.16038131713867,
      "activations/layer7_attention_weight_max": 91.58419036865234,
      "activations/layer7_attention_weight_min": -97.04869842529297,
      "activations/layer8_attention_weight_max": 42.18277359008789,
      "activations/layer8_attention_weight_min": -44.585105895996094,
      "activations/layer9_attention_weight_max": 37.03340530395508,
      "activations/layer9_attention_weight_min": -35.48099899291992,
      "epoch": 14.83,
      "learning_rate": 5.4946590909090904e-05,
      "loss": 2.7692,
      "step": 255150
    },
    {
      "activations/layer0_attention_weight_max": 15.215119361877441,
      "activations/layer0_attention_weight_min": -12.936114311218262,
      "activations/layer10_attention_weight_max": 36.79544448852539,
      "activations/layer10_attention_weight_min": -34.48546600341797,
      "activations/layer11_attention_weight_max": 36.743507385253906,
      "activations/layer11_attention_weight_min": -35.214515686035156,
      "activations/layer12_attention_weight_max": 25.919443130493164,
      "activations/layer12_attention_weight_min": -26.601158142089844,
      "activations/layer13_attention_weight_max": 38.54366683959961,
      "activations/layer13_attention_weight_min": -39.54802703857422,
      "activations/layer14_attention_weight_max": 39.64937973022461,
      "activations/layer14_attention_weight_min": -39.387760162353516,
      "activations/layer15_attention_weight_max": 35.166297912597656,
      "activations/layer15_attention_weight_min": -33.73066711425781,
      "activations/layer16_attention_weight_max": 27.63133430480957,
      "activations/layer16_attention_weight_min": -27.2330265045166,
      "activations/layer17_attention_weight_max": 29.672441482543945,
      "activations/layer17_attention_weight_min": -26.271757125854492,
      "activations/layer18_attention_weight_max": 30.656112670898438,
      "activations/layer18_attention_weight_min": -23.986324310302734,
      "activations/layer19_attention_weight_max": 31.225000381469727,
      "activations/layer19_attention_weight_min": -30.01287841796875,
      "activations/layer1_attention_weight_max": 16.987279891967773,
      "activations/layer1_attention_weight_min": -16.667137145996094,
      "activations/layer20_attention_weight_max": 28.284029006958008,
      "activations/layer20_attention_weight_min": -25.20770835876465,
      "activations/layer21_attention_weight_max": 31.091899871826172,
      "activations/layer21_attention_weight_min": -25.458660125732422,
      "activations/layer22_attention_weight_max": 43.1964111328125,
      "activations/layer22_attention_weight_min": -30.53103256225586,
      "activations/layer23_attention_weight_max": 34.814788818359375,
      "activations/layer23_attention_weight_min": -25.316974639892578,
      "activations/layer2_attention_weight_max": 32.22952651977539,
      "activations/layer2_attention_weight_min": -30.361135482788086,
      "activations/layer3_attention_weight_max": 102.45606231689453,
      "activations/layer3_attention_weight_min": -98.04230499267578,
      "activations/layer4_attention_weight_max": 113.22200775146484,
      "activations/layer4_attention_weight_min": -115.09063720703125,
      "activations/layer5_attention_weight_max": 53.10356140136719,
      "activations/layer5_attention_weight_min": -63.12982177734375,
      "activations/layer6_attention_weight_max": 48.749820709228516,
      "activations/layer6_attention_weight_min": -50.04867935180664,
      "activations/layer7_attention_weight_max": 101.2689437866211,
      "activations/layer7_attention_weight_min": -105.09269714355469,
      "activations/layer8_attention_weight_max": 45.00419616699219,
      "activations/layer8_attention_weight_min": -45.8365592956543,
      "activations/layer9_attention_weight_max": 36.55638122558594,
      "activations/layer9_attention_weight_min": -36.08244705200195,
      "epoch": 14.83,
      "learning_rate": 5.4927651515151505e-05,
      "loss": 2.7538,
      "step": 255200
    },
    {
      "activations/layer0_attention_weight_max": 17.03432273864746,
      "activations/layer0_attention_weight_min": -12.664887428283691,
      "activations/layer10_attention_weight_max": 41.93125534057617,
      "activations/layer10_attention_weight_min": -40.297584533691406,
      "activations/layer11_attention_weight_max": 45.95624542236328,
      "activations/layer11_attention_weight_min": -40.69514465332031,
      "activations/layer12_attention_weight_max": 29.30417823791504,
      "activations/layer12_attention_weight_min": -31.34685707092285,
      "activations/layer13_attention_weight_max": 45.43025588989258,
      "activations/layer13_attention_weight_min": -39.0857048034668,
      "activations/layer14_attention_weight_max": 49.030235290527344,
      "activations/layer14_attention_weight_min": -43.01777267456055,
      "activations/layer15_attention_weight_max": 41.20247268676758,
      "activations/layer15_attention_weight_min": -37.506622314453125,
      "activations/layer16_attention_weight_max": 37.165714263916016,
      "activations/layer16_attention_weight_min": -28.936613082885742,
      "activations/layer17_attention_weight_max": 32.830345153808594,
      "activations/layer17_attention_weight_min": -27.331825256347656,
      "activations/layer18_attention_weight_max": 34.594093322753906,
      "activations/layer18_attention_weight_min": -23.344900131225586,
      "activations/layer19_attention_weight_max": 39.67341232299805,
      "activations/layer19_attention_weight_min": -32.9334716796875,
      "activations/layer1_attention_weight_max": 19.113821029663086,
      "activations/layer1_attention_weight_min": -18.8338565826416,
      "activations/layer20_attention_weight_max": 35.18887710571289,
      "activations/layer20_attention_weight_min": -25.58101463317871,
      "activations/layer21_attention_weight_max": 35.230682373046875,
      "activations/layer21_attention_weight_min": -24.938804626464844,
      "activations/layer22_attention_weight_max": 54.400508880615234,
      "activations/layer22_attention_weight_min": -31.83522605895996,
      "activations/layer23_attention_weight_max": 37.44589614868164,
      "activations/layer23_attention_weight_min": -24.992664337158203,
      "activations/layer2_attention_weight_max": 33.677799224853516,
      "activations/layer2_attention_weight_min": -35.373722076416016,
      "activations/layer3_attention_weight_max": 98.8314208984375,
      "activations/layer3_attention_weight_min": -100.89457702636719,
      "activations/layer4_attention_weight_max": 114.1137466430664,
      "activations/layer4_attention_weight_min": -117.16910552978516,
      "activations/layer5_attention_weight_max": 54.83592224121094,
      "activations/layer5_attention_weight_min": -64.50204467773438,
      "activations/layer6_attention_weight_max": 47.86022186279297,
      "activations/layer6_attention_weight_min": -50.476985931396484,
      "activations/layer7_attention_weight_max": 103.07878112792969,
      "activations/layer7_attention_weight_min": -102.63822937011719,
      "activations/layer8_attention_weight_max": 49.288150787353516,
      "activations/layer8_attention_weight_min": -48.68914794921875,
      "activations/layer9_attention_weight_max": 36.767391204833984,
      "activations/layer9_attention_weight_min": -36.933467864990234,
      "epoch": 14.83,
      "learning_rate": 5.490871212121212e-05,
      "loss": 2.7537,
      "step": 255250
    },
    {
      "activations/layer0_attention_weight_max": 16.170381546020508,
      "activations/layer0_attention_weight_min": -11.912199974060059,
      "activations/layer10_attention_weight_max": 37.6641731262207,
      "activations/layer10_attention_weight_min": -37.44585418701172,
      "activations/layer11_attention_weight_max": 37.81620407104492,
      "activations/layer11_attention_weight_min": -34.6313362121582,
      "activations/layer12_attention_weight_max": 30.133594512939453,
      "activations/layer12_attention_weight_min": -27.932580947875977,
      "activations/layer13_attention_weight_max": 48.61035919189453,
      "activations/layer13_attention_weight_min": -37.7775764465332,
      "activations/layer14_attention_weight_max": 53.570899963378906,
      "activations/layer14_attention_weight_min": -41.29892349243164,
      "activations/layer15_attention_weight_max": 45.320045471191406,
      "activations/layer15_attention_weight_min": -36.736106872558594,
      "activations/layer16_attention_weight_max": 34.71379089355469,
      "activations/layer16_attention_weight_min": -30.157941818237305,
      "activations/layer17_attention_weight_max": 33.37271499633789,
      "activations/layer17_attention_weight_min": -26.766145706176758,
      "activations/layer18_attention_weight_max": 35.14291000366211,
      "activations/layer18_attention_weight_min": -24.51080894470215,
      "activations/layer19_attention_weight_max": 40.957191467285156,
      "activations/layer19_attention_weight_min": -31.5053768157959,
      "activations/layer1_attention_weight_max": 17.073707580566406,
      "activations/layer1_attention_weight_min": -17.124059677124023,
      "activations/layer20_attention_weight_max": 33.39276885986328,
      "activations/layer20_attention_weight_min": -25.693248748779297,
      "activations/layer21_attention_weight_max": 34.1206169128418,
      "activations/layer21_attention_weight_min": -26.44722557067871,
      "activations/layer22_attention_weight_max": 49.352455139160156,
      "activations/layer22_attention_weight_min": -30.78042984008789,
      "activations/layer23_attention_weight_max": 39.88718032836914,
      "activations/layer23_attention_weight_min": -26.268400192260742,
      "activations/layer2_attention_weight_max": 32.93373107910156,
      "activations/layer2_attention_weight_min": -32.78082275390625,
      "activations/layer3_attention_weight_max": 94.41826629638672,
      "activations/layer3_attention_weight_min": -95.5125961303711,
      "activations/layer4_attention_weight_max": 112.20538330078125,
      "activations/layer4_attention_weight_min": -114.4316635131836,
      "activations/layer5_attention_weight_max": 55.90775680541992,
      "activations/layer5_attention_weight_min": -61.9525260925293,
      "activations/layer6_attention_weight_max": 47.498268127441406,
      "activations/layer6_attention_weight_min": -48.17903518676758,
      "activations/layer7_attention_weight_max": 94.59721374511719,
      "activations/layer7_attention_weight_min": -93.6488037109375,
      "activations/layer8_attention_weight_max": 42.996036529541016,
      "activations/layer8_attention_weight_min": -50.69633865356445,
      "activations/layer9_attention_weight_max": 33.989784240722656,
      "activations/layer9_attention_weight_min": -36.80241394042969,
      "epoch": 14.83,
      "learning_rate": 5.488977272727272e-05,
      "loss": 2.7459,
      "step": 255300
    },
    {
      "activations/layer0_attention_weight_max": 16.383705139160156,
      "activations/layer0_attention_weight_min": -12.26006031036377,
      "activations/layer10_attention_weight_max": 35.55133056640625,
      "activations/layer10_attention_weight_min": -33.386146545410156,
      "activations/layer11_attention_weight_max": 34.25691604614258,
      "activations/layer11_attention_weight_min": -33.229644775390625,
      "activations/layer12_attention_weight_max": 26.859453201293945,
      "activations/layer12_attention_weight_min": -27.860322952270508,
      "activations/layer13_attention_weight_max": 42.34960174560547,
      "activations/layer13_attention_weight_min": -34.37422180175781,
      "activations/layer14_attention_weight_max": 39.820526123046875,
      "activations/layer14_attention_weight_min": -36.273826599121094,
      "activations/layer15_attention_weight_max": 36.750457763671875,
      "activations/layer15_attention_weight_min": -33.598365783691406,
      "activations/layer16_attention_weight_max": 31.89716339111328,
      "activations/layer16_attention_weight_min": -27.931886672973633,
      "activations/layer17_attention_weight_max": 31.407819747924805,
      "activations/layer17_attention_weight_min": -26.42216682434082,
      "activations/layer18_attention_weight_max": 31.634300231933594,
      "activations/layer18_attention_weight_min": -24.541017532348633,
      "activations/layer19_attention_weight_max": 33.797420501708984,
      "activations/layer19_attention_weight_min": -32.33638381958008,
      "activations/layer1_attention_weight_max": 15.94617748260498,
      "activations/layer1_attention_weight_min": -14.80640697479248,
      "activations/layer20_attention_weight_max": 28.9781551361084,
      "activations/layer20_attention_weight_min": -25.541015625,
      "activations/layer21_attention_weight_max": 28.357555389404297,
      "activations/layer21_attention_weight_min": -24.830204010009766,
      "activations/layer22_attention_weight_max": 47.15105438232422,
      "activations/layer22_attention_weight_min": -29.923458099365234,
      "activations/layer23_attention_weight_max": 31.500408172607422,
      "activations/layer23_attention_weight_min": -26.13308334350586,
      "activations/layer2_attention_weight_max": 32.64685821533203,
      "activations/layer2_attention_weight_min": -31.222675323486328,
      "activations/layer3_attention_weight_max": 90.25789642333984,
      "activations/layer3_attention_weight_min": -94.20512390136719,
      "activations/layer4_attention_weight_max": 109.38134765625,
      "activations/layer4_attention_weight_min": -112.74739837646484,
      "activations/layer5_attention_weight_max": 52.14510726928711,
      "activations/layer5_attention_weight_min": -60.7852897644043,
      "activations/layer6_attention_weight_max": 44.99794006347656,
      "activations/layer6_attention_weight_min": -49.93217468261719,
      "activations/layer7_attention_weight_max": 105.75776672363281,
      "activations/layer7_attention_weight_min": -94.90614318847656,
      "activations/layer8_attention_weight_max": 40.71676254272461,
      "activations/layer8_attention_weight_min": -42.75242233276367,
      "activations/layer9_attention_weight_max": 33.05509948730469,
      "activations/layer9_attention_weight_min": -33.65104293823242,
      "epoch": 14.84,
      "learning_rate": 5.487083333333333e-05,
      "loss": 2.7635,
      "step": 255350
    },
    {
      "activations/layer0_attention_weight_max": 15.972434043884277,
      "activations/layer0_attention_weight_min": -12.223969459533691,
      "activations/layer10_attention_weight_max": 38.018531799316406,
      "activations/layer10_attention_weight_min": -37.315460205078125,
      "activations/layer11_attention_weight_max": 36.429481506347656,
      "activations/layer11_attention_weight_min": -36.5367431640625,
      "activations/layer12_attention_weight_max": 27.526830673217773,
      "activations/layer12_attention_weight_min": -26.650312423706055,
      "activations/layer13_attention_weight_max": 41.37126541137695,
      "activations/layer13_attention_weight_min": -37.75502395629883,
      "activations/layer14_attention_weight_max": 42.04341506958008,
      "activations/layer14_attention_weight_min": -37.055137634277344,
      "activations/layer15_attention_weight_max": 37.29146194458008,
      "activations/layer15_attention_weight_min": -33.10206604003906,
      "activations/layer16_attention_weight_max": 32.163352966308594,
      "activations/layer16_attention_weight_min": -27.38648796081543,
      "activations/layer17_attention_weight_max": 29.137460708618164,
      "activations/layer17_attention_weight_min": -25.797990798950195,
      "activations/layer18_attention_weight_max": 33.141326904296875,
      "activations/layer18_attention_weight_min": -24.718114852905273,
      "activations/layer19_attention_weight_max": 37.12592315673828,
      "activations/layer19_attention_weight_min": -31.966711044311523,
      "activations/layer1_attention_weight_max": 16.011192321777344,
      "activations/layer1_attention_weight_min": -15.916194915771484,
      "activations/layer20_attention_weight_max": 32.46368408203125,
      "activations/layer20_attention_weight_min": -25.317380905151367,
      "activations/layer21_attention_weight_max": 35.072391510009766,
      "activations/layer21_attention_weight_min": -24.611572265625,
      "activations/layer22_attention_weight_max": 46.18570327758789,
      "activations/layer22_attention_weight_min": -32.77595520019531,
      "activations/layer23_attention_weight_max": 35.694393157958984,
      "activations/layer23_attention_weight_min": -24.613447189331055,
      "activations/layer2_attention_weight_max": 30.70163345336914,
      "activations/layer2_attention_weight_min": -31.301002502441406,
      "activations/layer3_attention_weight_max": 94.46075439453125,
      "activations/layer3_attention_weight_min": -96.75819396972656,
      "activations/layer4_attention_weight_max": 114.79985809326172,
      "activations/layer4_attention_weight_min": -117.30452728271484,
      "activations/layer5_attention_weight_max": 54.27588653564453,
      "activations/layer5_attention_weight_min": -63.11909484863281,
      "activations/layer6_attention_weight_max": 49.617774963378906,
      "activations/layer6_attention_weight_min": -51.06935119628906,
      "activations/layer7_attention_weight_max": 97.04207611083984,
      "activations/layer7_attention_weight_min": -104.83610534667969,
      "activations/layer8_attention_weight_max": 44.59724807739258,
      "activations/layer8_attention_weight_min": -44.20649719238281,
      "activations/layer9_attention_weight_max": 35.972618103027344,
      "activations/layer9_attention_weight_min": -37.1130485534668,
      "epoch": 14.84,
      "learning_rate": 5.485189393939393e-05,
      "loss": 2.7546,
      "step": 255400
    },
    {
      "activations/layer0_attention_weight_max": 16.193424224853516,
      "activations/layer0_attention_weight_min": -12.193094253540039,
      "activations/layer10_attention_weight_max": 32.86863327026367,
      "activations/layer10_attention_weight_min": -33.84078598022461,
      "activations/layer11_attention_weight_max": 31.786460876464844,
      "activations/layer11_attention_weight_min": -33.576446533203125,
      "activations/layer12_attention_weight_max": 26.178808212280273,
      "activations/layer12_attention_weight_min": -25.80252456665039,
      "activations/layer13_attention_weight_max": 40.13007736206055,
      "activations/layer13_attention_weight_min": -34.45821762084961,
      "activations/layer14_attention_weight_max": 42.84212112426758,
      "activations/layer14_attention_weight_min": -37.73487091064453,
      "activations/layer15_attention_weight_max": 39.3850212097168,
      "activations/layer15_attention_weight_min": -32.77334213256836,
      "activations/layer16_attention_weight_max": 30.784534454345703,
      "activations/layer16_attention_weight_min": -27.59690284729004,
      "activations/layer17_attention_weight_max": 32.52415466308594,
      "activations/layer17_attention_weight_min": -26.924314498901367,
      "activations/layer18_attention_weight_max": 32.607696533203125,
      "activations/layer18_attention_weight_min": -26.52516746520996,
      "activations/layer19_attention_weight_max": 34.375396728515625,
      "activations/layer19_attention_weight_min": -32.06586837768555,
      "activations/layer1_attention_weight_max": 17.540077209472656,
      "activations/layer1_attention_weight_min": -15.934745788574219,
      "activations/layer20_attention_weight_max": 31.449859619140625,
      "activations/layer20_attention_weight_min": -27.375194549560547,
      "activations/layer21_attention_weight_max": 30.10118865966797,
      "activations/layer21_attention_weight_min": -27.39297103881836,
      "activations/layer22_attention_weight_max": 44.198158264160156,
      "activations/layer22_attention_weight_min": -34.191688537597656,
      "activations/layer23_attention_weight_max": 37.20179748535156,
      "activations/layer23_attention_weight_min": -29.05480194091797,
      "activations/layer2_attention_weight_max": 32.56398010253906,
      "activations/layer2_attention_weight_min": -32.31267547607422,
      "activations/layer3_attention_weight_max": 98.19405364990234,
      "activations/layer3_attention_weight_min": -94.99454498291016,
      "activations/layer4_attention_weight_max": 107.74100494384766,
      "activations/layer4_attention_weight_min": -109.74415588378906,
      "activations/layer5_attention_weight_max": 51.88433074951172,
      "activations/layer5_attention_weight_min": -61.05810546875,
      "activations/layer6_attention_weight_max": 44.443233489990234,
      "activations/layer6_attention_weight_min": -46.85371398925781,
      "activations/layer7_attention_weight_max": 96.49834442138672,
      "activations/layer7_attention_weight_min": -93.70670318603516,
      "activations/layer8_attention_weight_max": 38.7081298828125,
      "activations/layer8_attention_weight_min": -40.973289489746094,
      "activations/layer9_attention_weight_max": 31.11813735961914,
      "activations/layer9_attention_weight_min": -34.28828811645508,
      "epoch": 14.84,
      "learning_rate": 5.4832954545454546e-05,
      "loss": 2.7614,
      "step": 255450
    },
    {
      "activations/layer0_attention_weight_max": 14.882018089294434,
      "activations/layer0_attention_weight_min": -12.219132423400879,
      "activations/layer10_attention_weight_max": 34.175018310546875,
      "activations/layer10_attention_weight_min": -34.48418426513672,
      "activations/layer11_attention_weight_max": 32.17253875732422,
      "activations/layer11_attention_weight_min": -34.3554801940918,
      "activations/layer12_attention_weight_max": 24.006607055664062,
      "activations/layer12_attention_weight_min": -28.282957077026367,
      "activations/layer13_attention_weight_max": 38.193824768066406,
      "activations/layer13_attention_weight_min": -37.452354431152344,
      "activations/layer14_attention_weight_max": 39.94911575317383,
      "activations/layer14_attention_weight_min": -38.596126556396484,
      "activations/layer15_attention_weight_max": 33.55792999267578,
      "activations/layer15_attention_weight_min": -33.77376937866211,
      "activations/layer16_attention_weight_max": 29.723222732543945,
      "activations/layer16_attention_weight_min": -30.25132942199707,
      "activations/layer17_attention_weight_max": 29.42354393005371,
      "activations/layer17_attention_weight_min": -28.155601501464844,
      "activations/layer18_attention_weight_max": 28.84296226501465,
      "activations/layer18_attention_weight_min": -27.91073226928711,
      "activations/layer19_attention_weight_max": 32.59049987792969,
      "activations/layer19_attention_weight_min": -33.0825309753418,
      "activations/layer1_attention_weight_max": 15.906618118286133,
      "activations/layer1_attention_weight_min": -13.93747615814209,
      "activations/layer20_attention_weight_max": 28.666847229003906,
      "activations/layer20_attention_weight_min": -27.214982986450195,
      "activations/layer21_attention_weight_max": 28.404216766357422,
      "activations/layer21_attention_weight_min": -27.35909652709961,
      "activations/layer22_attention_weight_max": 38.761878967285156,
      "activations/layer22_attention_weight_min": -34.53654861450195,
      "activations/layer23_attention_weight_max": 32.20478820800781,
      "activations/layer23_attention_weight_min": -29.408287048339844,
      "activations/layer2_attention_weight_max": 29.79623794555664,
      "activations/layer2_attention_weight_min": -30.276823043823242,
      "activations/layer3_attention_weight_max": 88.77049255371094,
      "activations/layer3_attention_weight_min": -94.13685607910156,
      "activations/layer4_attention_weight_max": 115.31377410888672,
      "activations/layer4_attention_weight_min": -109.0467529296875,
      "activations/layer5_attention_weight_max": 53.501731872558594,
      "activations/layer5_attention_weight_min": -61.15946578979492,
      "activations/layer6_attention_weight_max": 44.20149612426758,
      "activations/layer6_attention_weight_min": -47.066444396972656,
      "activations/layer7_attention_weight_max": 89.47804260253906,
      "activations/layer7_attention_weight_min": -90.9983139038086,
      "activations/layer8_attention_weight_max": 39.473697662353516,
      "activations/layer8_attention_weight_min": -43.95463562011719,
      "activations/layer9_attention_weight_max": 32.44096374511719,
      "activations/layer9_attention_weight_min": -35.761962890625,
      "epoch": 14.85,
      "learning_rate": 5.481401515151515e-05,
      "loss": 2.7695,
      "step": 255500
    },
    {
      "activations/layer0_attention_weight_max": 15.679698944091797,
      "activations/layer0_attention_weight_min": -12.15225601196289,
      "activations/layer10_attention_weight_max": 30.741546630859375,
      "activations/layer10_attention_weight_min": -32.119972229003906,
      "activations/layer11_attention_weight_max": 29.764537811279297,
      "activations/layer11_attention_weight_min": -32.68059539794922,
      "activations/layer12_attention_weight_max": 25.193275451660156,
      "activations/layer12_attention_weight_min": -26.167926788330078,
      "activations/layer13_attention_weight_max": 38.400856018066406,
      "activations/layer13_attention_weight_min": -34.63044357299805,
      "activations/layer14_attention_weight_max": 37.84923553466797,
      "activations/layer14_attention_weight_min": -36.402854919433594,
      "activations/layer15_attention_weight_max": 34.04082107543945,
      "activations/layer15_attention_weight_min": -34.799468994140625,
      "activations/layer16_attention_weight_max": 28.16240882873535,
      "activations/layer16_attention_weight_min": -28.528913497924805,
      "activations/layer17_attention_weight_max": 32.94388198852539,
      "activations/layer17_attention_weight_min": -27.007932662963867,
      "activations/layer18_attention_weight_max": 31.156930923461914,
      "activations/layer18_attention_weight_min": -25.617040634155273,
      "activations/layer19_attention_weight_max": 34.36343002319336,
      "activations/layer19_attention_weight_min": -30.667343139648438,
      "activations/layer1_attention_weight_max": 16.88250160217285,
      "activations/layer1_attention_weight_min": -14.717525482177734,
      "activations/layer20_attention_weight_max": 28.479236602783203,
      "activations/layer20_attention_weight_min": -26.778867721557617,
      "activations/layer21_attention_weight_max": 29.040231704711914,
      "activations/layer21_attention_weight_min": -25.526317596435547,
      "activations/layer22_attention_weight_max": 43.06105041503906,
      "activations/layer22_attention_weight_min": -32.13467788696289,
      "activations/layer23_attention_weight_max": 34.98505783081055,
      "activations/layer23_attention_weight_min": -26.811630249023438,
      "activations/layer2_attention_weight_max": 31.06019401550293,
      "activations/layer2_attention_weight_min": -31.60540008544922,
      "activations/layer3_attention_weight_max": 92.34829711914062,
      "activations/layer3_attention_weight_min": -90.17813110351562,
      "activations/layer4_attention_weight_max": 109.22772216796875,
      "activations/layer4_attention_weight_min": -110.3438491821289,
      "activations/layer5_attention_weight_max": 53.00376510620117,
      "activations/layer5_attention_weight_min": -67.97015380859375,
      "activations/layer6_attention_weight_max": 43.10110092163086,
      "activations/layer6_attention_weight_min": -45.85340881347656,
      "activations/layer7_attention_weight_max": 89.45894622802734,
      "activations/layer7_attention_weight_min": -91.30873107910156,
      "activations/layer8_attention_weight_max": 37.68027114868164,
      "activations/layer8_attention_weight_min": -40.09803771972656,
      "activations/layer9_attention_weight_max": 28.984100341796875,
      "activations/layer9_attention_weight_min": -31.48848533630371,
      "epoch": 14.85,
      "learning_rate": 5.479507575757575e-05,
      "loss": 2.7624,
      "step": 255550
    },
    {
      "activations/layer0_attention_weight_max": 16.560800552368164,
      "activations/layer0_attention_weight_min": -12.246883392333984,
      "activations/layer10_attention_weight_max": 29.83909034729004,
      "activations/layer10_attention_weight_min": -30.54378318786621,
      "activations/layer11_attention_weight_max": 30.079395294189453,
      "activations/layer11_attention_weight_min": -30.306875228881836,
      "activations/layer12_attention_weight_max": 24.69280242919922,
      "activations/layer12_attention_weight_min": -25.27140235900879,
      "activations/layer13_attention_weight_max": 38.01313781738281,
      "activations/layer13_attention_weight_min": -35.839805603027344,
      "activations/layer14_attention_weight_max": 38.86326217651367,
      "activations/layer14_attention_weight_min": -37.231143951416016,
      "activations/layer15_attention_weight_max": 34.5938720703125,
      "activations/layer15_attention_weight_min": -36.33817672729492,
      "activations/layer16_attention_weight_max": 27.92262840270996,
      "activations/layer16_attention_weight_min": -28.143585205078125,
      "activations/layer17_attention_weight_max": 27.5120906829834,
      "activations/layer17_attention_weight_min": -24.48700714111328,
      "activations/layer18_attention_weight_max": 29.424272537231445,
      "activations/layer18_attention_weight_min": -24.18844223022461,
      "activations/layer19_attention_weight_max": 31.730884552001953,
      "activations/layer19_attention_weight_min": -30.296539306640625,
      "activations/layer1_attention_weight_max": 15.656045913696289,
      "activations/layer1_attention_weight_min": -13.671669960021973,
      "activations/layer20_attention_weight_max": 28.373512268066406,
      "activations/layer20_attention_weight_min": -25.92593002319336,
      "activations/layer21_attention_weight_max": 28.30927085876465,
      "activations/layer21_attention_weight_min": -25.318479537963867,
      "activations/layer22_attention_weight_max": 40.089866638183594,
      "activations/layer22_attention_weight_min": -32.53020477294922,
      "activations/layer23_attention_weight_max": 35.086036682128906,
      "activations/layer23_attention_weight_min": -25.873382568359375,
      "activations/layer2_attention_weight_max": 28.43551254272461,
      "activations/layer2_attention_weight_min": -29.770084381103516,
      "activations/layer3_attention_weight_max": 88.03144073486328,
      "activations/layer3_attention_weight_min": -88.0099868774414,
      "activations/layer4_attention_weight_max": 100.62910461425781,
      "activations/layer4_attention_weight_min": -106.46456146240234,
      "activations/layer5_attention_weight_max": 46.89289474487305,
      "activations/layer5_attention_weight_min": -62.071468353271484,
      "activations/layer6_attention_weight_max": 40.73412322998047,
      "activations/layer6_attention_weight_min": -46.155765533447266,
      "activations/layer7_attention_weight_max": 83.2900161743164,
      "activations/layer7_attention_weight_min": -92.70796966552734,
      "activations/layer8_attention_weight_max": 39.0900764465332,
      "activations/layer8_attention_weight_min": -40.30491256713867,
      "activations/layer9_attention_weight_max": 30.1727294921875,
      "activations/layer9_attention_weight_min": -30.689193725585938,
      "epoch": 14.85,
      "learning_rate": 5.477613636363636e-05,
      "loss": 2.7608,
      "step": 255600
    },
    {
      "activations/layer0_attention_weight_max": 15.444188117980957,
      "activations/layer0_attention_weight_min": -12.419633865356445,
      "activations/layer10_attention_weight_max": 34.89630126953125,
      "activations/layer10_attention_weight_min": -35.14151382446289,
      "activations/layer11_attention_weight_max": 35.94666290283203,
      "activations/layer11_attention_weight_min": -35.52461242675781,
      "activations/layer12_attention_weight_max": 29.36611557006836,
      "activations/layer12_attention_weight_min": -27.426471710205078,
      "activations/layer13_attention_weight_max": 42.40547561645508,
      "activations/layer13_attention_weight_min": -38.47002029418945,
      "activations/layer14_attention_weight_max": 50.22541046142578,
      "activations/layer14_attention_weight_min": -43.75492477416992,
      "activations/layer15_attention_weight_max": 44.014461517333984,
      "activations/layer15_attention_weight_min": -36.86787033081055,
      "activations/layer16_attention_weight_max": 32.97551345825195,
      "activations/layer16_attention_weight_min": -29.106605529785156,
      "activations/layer17_attention_weight_max": 35.51304626464844,
      "activations/layer17_attention_weight_min": -27.62092399597168,
      "activations/layer18_attention_weight_max": 34.368927001953125,
      "activations/layer18_attention_weight_min": -25.655635833740234,
      "activations/layer19_attention_weight_max": 40.53282928466797,
      "activations/layer19_attention_weight_min": -33.76183319091797,
      "activations/layer1_attention_weight_max": 16.432254791259766,
      "activations/layer1_attention_weight_min": -15.134669303894043,
      "activations/layer20_attention_weight_max": 31.59837532043457,
      "activations/layer20_attention_weight_min": -29.20437240600586,
      "activations/layer21_attention_weight_max": 30.797386169433594,
      "activations/layer21_attention_weight_min": -26.5258846282959,
      "activations/layer22_attention_weight_max": 49.92326736450195,
      "activations/layer22_attention_weight_min": -31.150510787963867,
      "activations/layer23_attention_weight_max": 38.27337646484375,
      "activations/layer23_attention_weight_min": -26.01150894165039,
      "activations/layer2_attention_weight_max": 30.126361846923828,
      "activations/layer2_attention_weight_min": -31.081897735595703,
      "activations/layer3_attention_weight_max": 91.52350616455078,
      "activations/layer3_attention_weight_min": -94.04954528808594,
      "activations/layer4_attention_weight_max": 108.5274887084961,
      "activations/layer4_attention_weight_min": -112.27800750732422,
      "activations/layer5_attention_weight_max": 51.657100677490234,
      "activations/layer5_attention_weight_min": -68.46501159667969,
      "activations/layer6_attention_weight_max": 43.54450607299805,
      "activations/layer6_attention_weight_min": -47.32217788696289,
      "activations/layer7_attention_weight_max": 92.89908599853516,
      "activations/layer7_attention_weight_min": -94.09696197509766,
      "activations/layer8_attention_weight_max": 40.179725646972656,
      "activations/layer8_attention_weight_min": -46.058868408203125,
      "activations/layer9_attention_weight_max": 32.446807861328125,
      "activations/layer9_attention_weight_min": -34.94346618652344,
      "epoch": 14.85,
      "learning_rate": 5.4757196969696966e-05,
      "loss": 2.7553,
      "step": 255650
    },
    {
      "activations/layer0_attention_weight_max": 15.266857147216797,
      "activations/layer0_attention_weight_min": -12.986838340759277,
      "activations/layer10_attention_weight_max": 33.3084716796875,
      "activations/layer10_attention_weight_min": -32.458065032958984,
      "activations/layer11_attention_weight_max": 31.24910545349121,
      "activations/layer11_attention_weight_min": -33.694786071777344,
      "activations/layer12_attention_weight_max": 25.530689239501953,
      "activations/layer12_attention_weight_min": -25.938230514526367,
      "activations/layer13_attention_weight_max": 39.942138671875,
      "activations/layer13_attention_weight_min": -37.2125129699707,
      "activations/layer14_attention_weight_max": 40.64427185058594,
      "activations/layer14_attention_weight_min": -40.507423400878906,
      "activations/layer15_attention_weight_max": 35.78073501586914,
      "activations/layer15_attention_weight_min": -34.50870132446289,
      "activations/layer16_attention_weight_max": 28.91295623779297,
      "activations/layer16_attention_weight_min": -31.85064697265625,
      "activations/layer17_attention_weight_max": 31.28224754333496,
      "activations/layer17_attention_weight_min": -28.61952018737793,
      "activations/layer18_attention_weight_max": 31.681598663330078,
      "activations/layer18_attention_weight_min": -26.776771545410156,
      "activations/layer19_attention_weight_max": 34.307247161865234,
      "activations/layer19_attention_weight_min": -34.22164535522461,
      "activations/layer1_attention_weight_max": 16.54426383972168,
      "activations/layer1_attention_weight_min": -13.547835350036621,
      "activations/layer20_attention_weight_max": 29.770580291748047,
      "activations/layer20_attention_weight_min": -26.401920318603516,
      "activations/layer21_attention_weight_max": 31.104063034057617,
      "activations/layer21_attention_weight_min": -25.790306091308594,
      "activations/layer22_attention_weight_max": 43.790252685546875,
      "activations/layer22_attention_weight_min": -32.16310501098633,
      "activations/layer23_attention_weight_max": 34.32060241699219,
      "activations/layer23_attention_weight_min": -27.16823959350586,
      "activations/layer2_attention_weight_max": 30.72393035888672,
      "activations/layer2_attention_weight_min": -30.601097106933594,
      "activations/layer3_attention_weight_max": 86.48332214355469,
      "activations/layer3_attention_weight_min": -85.11771392822266,
      "activations/layer4_attention_weight_max": 99.47745513916016,
      "activations/layer4_attention_weight_min": -104.4171371459961,
      "activations/layer5_attention_weight_max": 53.67827606201172,
      "activations/layer5_attention_weight_min": -58.8587646484375,
      "activations/layer6_attention_weight_max": 40.08867645263672,
      "activations/layer6_attention_weight_min": -43.79323196411133,
      "activations/layer7_attention_weight_max": 86.34123229980469,
      "activations/layer7_attention_weight_min": -89.47269439697266,
      "activations/layer8_attention_weight_max": 39.28032302856445,
      "activations/layer8_attention_weight_min": -41.55815124511719,
      "activations/layer9_attention_weight_max": 30.556913375854492,
      "activations/layer9_attention_weight_min": -32.772369384765625,
      "epoch": 14.86,
      "learning_rate": 5.4738257575757574e-05,
      "loss": 2.761,
      "step": 255700
    },
    {
      "activations/layer0_attention_weight_max": 15.873926162719727,
      "activations/layer0_attention_weight_min": -12.901704788208008,
      "activations/layer10_attention_weight_max": 37.9544563293457,
      "activations/layer10_attention_weight_min": -33.73744201660156,
      "activations/layer11_attention_weight_max": 37.52320098876953,
      "activations/layer11_attention_weight_min": -34.46844482421875,
      "activations/layer12_attention_weight_max": 33.89327621459961,
      "activations/layer12_attention_weight_min": -26.19790267944336,
      "activations/layer13_attention_weight_max": 48.64433288574219,
      "activations/layer13_attention_weight_min": -32.76891326904297,
      "activations/layer14_attention_weight_max": 43.205162048339844,
      "activations/layer14_attention_weight_min": -37.29866409301758,
      "activations/layer15_attention_weight_max": 45.49373245239258,
      "activations/layer15_attention_weight_min": -36.18011474609375,
      "activations/layer16_attention_weight_max": 28.21383285522461,
      "activations/layer16_attention_weight_min": -27.53290367126465,
      "activations/layer17_attention_weight_max": 28.22172737121582,
      "activations/layer17_attention_weight_min": -25.536178588867188,
      "activations/layer18_attention_weight_max": 29.038339614868164,
      "activations/layer18_attention_weight_min": -23.07033920288086,
      "activations/layer19_attention_weight_max": 31.120227813720703,
      "activations/layer19_attention_weight_min": -31.886865615844727,
      "activations/layer1_attention_weight_max": 17.976438522338867,
      "activations/layer1_attention_weight_min": -15.493119239807129,
      "activations/layer20_attention_weight_max": 25.808513641357422,
      "activations/layer20_attention_weight_min": -26.05050277709961,
      "activations/layer21_attention_weight_max": 26.09250831604004,
      "activations/layer21_attention_weight_min": -24.713281631469727,
      "activations/layer22_attention_weight_max": 41.50645065307617,
      "activations/layer22_attention_weight_min": -31.335601806640625,
      "activations/layer23_attention_weight_max": 29.754201889038086,
      "activations/layer23_attention_weight_min": -24.196645736694336,
      "activations/layer2_attention_weight_max": 31.487197875976562,
      "activations/layer2_attention_weight_min": -32.42256164550781,
      "activations/layer3_attention_weight_max": 99.91107940673828,
      "activations/layer3_attention_weight_min": -97.99535369873047,
      "activations/layer4_attention_weight_max": 110.77874755859375,
      "activations/layer4_attention_weight_min": -111.27076721191406,
      "activations/layer5_attention_weight_max": 53.3746452331543,
      "activations/layer5_attention_weight_min": -62.04014205932617,
      "activations/layer6_attention_weight_max": 45.74684143066406,
      "activations/layer6_attention_weight_min": -47.10956954956055,
      "activations/layer7_attention_weight_max": 103.45777893066406,
      "activations/layer7_attention_weight_min": -98.67109680175781,
      "activations/layer8_attention_weight_max": 44.70960235595703,
      "activations/layer8_attention_weight_min": -44.29218292236328,
      "activations/layer9_attention_weight_max": 32.695247650146484,
      "activations/layer9_attention_weight_min": -34.514366149902344,
      "epoch": 14.86,
      "learning_rate": 5.4719318181818176e-05,
      "loss": 2.7456,
      "step": 255750
    },
    {
      "activations/layer0_attention_weight_max": 16.2453556060791,
      "activations/layer0_attention_weight_min": -11.986905097961426,
      "activations/layer10_attention_weight_max": 29.628177642822266,
      "activations/layer10_attention_weight_min": -33.63823699951172,
      "activations/layer11_attention_weight_max": 31.410673141479492,
      "activations/layer11_attention_weight_min": -34.57112121582031,
      "activations/layer12_attention_weight_max": 24.01850128173828,
      "activations/layer12_attention_weight_min": -23.67888069152832,
      "activations/layer13_attention_weight_max": 36.34117889404297,
      "activations/layer13_attention_weight_min": -34.3913688659668,
      "activations/layer14_attention_weight_max": 40.164634704589844,
      "activations/layer14_attention_weight_min": -37.864173889160156,
      "activations/layer15_attention_weight_max": 34.667762756347656,
      "activations/layer15_attention_weight_min": -32.10346984863281,
      "activations/layer16_attention_weight_max": 30.643421173095703,
      "activations/layer16_attention_weight_min": -27.023361206054688,
      "activations/layer17_attention_weight_max": 29.227136611938477,
      "activations/layer17_attention_weight_min": -24.648935317993164,
      "activations/layer18_attention_weight_max": 31.408531188964844,
      "activations/layer18_attention_weight_min": -26.249656677246094,
      "activations/layer19_attention_weight_max": 32.49174118041992,
      "activations/layer19_attention_weight_min": -34.74192810058594,
      "activations/layer1_attention_weight_max": 17.561403274536133,
      "activations/layer1_attention_weight_min": -14.825275421142578,
      "activations/layer20_attention_weight_max": 29.021995544433594,
      "activations/layer20_attention_weight_min": -26.117734909057617,
      "activations/layer21_attention_weight_max": 31.817941665649414,
      "activations/layer21_attention_weight_min": -26.04872703552246,
      "activations/layer22_attention_weight_max": 41.34559631347656,
      "activations/layer22_attention_weight_min": -31.46644401550293,
      "activations/layer23_attention_weight_max": 29.696826934814453,
      "activations/layer23_attention_weight_min": -27.77572250366211,
      "activations/layer2_attention_weight_max": 30.817962646484375,
      "activations/layer2_attention_weight_min": -30.957611083984375,
      "activations/layer3_attention_weight_max": 88.6484375,
      "activations/layer3_attention_weight_min": -92.39671325683594,
      "activations/layer4_attention_weight_max": 105.38265228271484,
      "activations/layer4_attention_weight_min": -108.20143127441406,
      "activations/layer5_attention_weight_max": 48.57182312011719,
      "activations/layer5_attention_weight_min": -61.02259063720703,
      "activations/layer6_attention_weight_max": 41.444068908691406,
      "activations/layer6_attention_weight_min": -45.58824920654297,
      "activations/layer7_attention_weight_max": 89.6902084350586,
      "activations/layer7_attention_weight_min": -92.96683502197266,
      "activations/layer8_attention_weight_max": 38.41606140136719,
      "activations/layer8_attention_weight_min": -39.92274856567383,
      "activations/layer9_attention_weight_max": 29.695852279663086,
      "activations/layer9_attention_weight_min": -32.22304916381836,
      "epoch": 14.86,
      "learning_rate": 5.470037878787878e-05,
      "loss": 2.758,
      "step": 255800
    },
    {
      "activations/layer0_attention_weight_max": 16.18346405029297,
      "activations/layer0_attention_weight_min": -12.266949653625488,
      "activations/layer10_attention_weight_max": 36.87718963623047,
      "activations/layer10_attention_weight_min": -36.72730255126953,
      "activations/layer11_attention_weight_max": 37.6262092590332,
      "activations/layer11_attention_weight_min": -38.616119384765625,
      "activations/layer12_attention_weight_max": 26.493066787719727,
      "activations/layer12_attention_weight_min": -25.418521881103516,
      "activations/layer13_attention_weight_max": 40.53517532348633,
      "activations/layer13_attention_weight_min": -34.72265625,
      "activations/layer14_attention_weight_max": 46.77213668823242,
      "activations/layer14_attention_weight_min": -37.42913055419922,
      "activations/layer15_attention_weight_max": 37.47034454345703,
      "activations/layer15_attention_weight_min": -32.96327590942383,
      "activations/layer16_attention_weight_max": 30.345491409301758,
      "activations/layer16_attention_weight_min": -27.97711753845215,
      "activations/layer17_attention_weight_max": 30.876543045043945,
      "activations/layer17_attention_weight_min": -26.29949188232422,
      "activations/layer18_attention_weight_max": 30.463665008544922,
      "activations/layer18_attention_weight_min": -23.47328758239746,
      "activations/layer19_attention_weight_max": 34.57144546508789,
      "activations/layer19_attention_weight_min": -30.764877319335938,
      "activations/layer1_attention_weight_max": 16.192977905273438,
      "activations/layer1_attention_weight_min": -14.681817054748535,
      "activations/layer20_attention_weight_max": 28.45760154724121,
      "activations/layer20_attention_weight_min": -24.13404083251953,
      "activations/layer21_attention_weight_max": 27.919750213623047,
      "activations/layer21_attention_weight_min": -24.07994270324707,
      "activations/layer22_attention_weight_max": 44.6053466796875,
      "activations/layer22_attention_weight_min": -29.569089889526367,
      "activations/layer23_attention_weight_max": 32.00689697265625,
      "activations/layer23_attention_weight_min": -23.352615356445312,
      "activations/layer2_attention_weight_max": 31.405065536499023,
      "activations/layer2_attention_weight_min": -32.216796875,
      "activations/layer3_attention_weight_max": 93.6662368774414,
      "activations/layer3_attention_weight_min": -98.3397216796875,
      "activations/layer4_attention_weight_max": 107.45003509521484,
      "activations/layer4_attention_weight_min": -113.2016372680664,
      "activations/layer5_attention_weight_max": 50.54517364501953,
      "activations/layer5_attention_weight_min": -61.83695983886719,
      "activations/layer6_attention_weight_max": 43.76545715332031,
      "activations/layer6_attention_weight_min": -48.93366622924805,
      "activations/layer7_attention_weight_max": 91.52753448486328,
      "activations/layer7_attention_weight_min": -98.6113052368164,
      "activations/layer8_attention_weight_max": 41.83879470825195,
      "activations/layer8_attention_weight_min": -44.2949104309082,
      "activations/layer9_attention_weight_max": 35.310359954833984,
      "activations/layer9_attention_weight_min": -36.480567932128906,
      "epoch": 14.87,
      "learning_rate": 5.468143939393939e-05,
      "loss": 2.7601,
      "step": 255850
    },
    {
      "activations/layer0_attention_weight_max": 15.812712669372559,
      "activations/layer0_attention_weight_min": -11.892847061157227,
      "activations/layer10_attention_weight_max": 32.70768356323242,
      "activations/layer10_attention_weight_min": -33.51127624511719,
      "activations/layer11_attention_weight_max": 32.71049499511719,
      "activations/layer11_attention_weight_min": -33.72584915161133,
      "activations/layer12_attention_weight_max": 25.80280113220215,
      "activations/layer12_attention_weight_min": -27.57013702392578,
      "activations/layer13_attention_weight_max": 40.06600570678711,
      "activations/layer13_attention_weight_min": -39.84968566894531,
      "activations/layer14_attention_weight_max": 42.63237762451172,
      "activations/layer14_attention_weight_min": -39.71674346923828,
      "activations/layer15_attention_weight_max": 40.65232849121094,
      "activations/layer15_attention_weight_min": -35.71866989135742,
      "activations/layer16_attention_weight_max": 29.272804260253906,
      "activations/layer16_attention_weight_min": -26.79736328125,
      "activations/layer17_attention_weight_max": 29.58721923828125,
      "activations/layer17_attention_weight_min": -25.721628189086914,
      "activations/layer18_attention_weight_max": 28.99183464050293,
      "activations/layer18_attention_weight_min": -24.719797134399414,
      "activations/layer19_attention_weight_max": 32.751953125,
      "activations/layer19_attention_weight_min": -30.481748580932617,
      "activations/layer1_attention_weight_max": 16.150238037109375,
      "activations/layer1_attention_weight_min": -12.998044967651367,
      "activations/layer20_attention_weight_max": 27.076637268066406,
      "activations/layer20_attention_weight_min": -23.963529586791992,
      "activations/layer21_attention_weight_max": 29.35122299194336,
      "activations/layer21_attention_weight_min": -23.21144676208496,
      "activations/layer22_attention_weight_max": 40.54648208618164,
      "activations/layer22_attention_weight_min": -28.8983154296875,
      "activations/layer23_attention_weight_max": 33.082923889160156,
      "activations/layer23_attention_weight_min": -25.134944915771484,
      "activations/layer2_attention_weight_max": 31.47132110595703,
      "activations/layer2_attention_weight_min": -32.09517288208008,
      "activations/layer3_attention_weight_max": 95.29769897460938,
      "activations/layer3_attention_weight_min": -91.7506103515625,
      "activations/layer4_attention_weight_max": 110.47469329833984,
      "activations/layer4_attention_weight_min": -114.06971740722656,
      "activations/layer5_attention_weight_max": 53.820030212402344,
      "activations/layer5_attention_weight_min": -63.579673767089844,
      "activations/layer6_attention_weight_max": 47.22992706298828,
      "activations/layer6_attention_weight_min": -51.15086364746094,
      "activations/layer7_attention_weight_max": 97.66687774658203,
      "activations/layer7_attention_weight_min": -99.01966857910156,
      "activations/layer8_attention_weight_max": 41.51286315917969,
      "activations/layer8_attention_weight_min": -44.46728515625,
      "activations/layer9_attention_weight_max": 32.57546615600586,
      "activations/layer9_attention_weight_min": -33.8619384765625,
      "epoch": 14.87,
      "learning_rate": 5.4662499999999994e-05,
      "loss": 2.7525,
      "step": 255900
    },
    {
      "activations/layer0_attention_weight_max": 15.67430305480957,
      "activations/layer0_attention_weight_min": -11.81016731262207,
      "activations/layer10_attention_weight_max": 36.32039260864258,
      "activations/layer10_attention_weight_min": -35.6561393737793,
      "activations/layer11_attention_weight_max": 35.386417388916016,
      "activations/layer11_attention_weight_min": -35.306854248046875,
      "activations/layer12_attention_weight_max": 25.498252868652344,
      "activations/layer12_attention_weight_min": -24.790613174438477,
      "activations/layer13_attention_weight_max": 39.02006149291992,
      "activations/layer13_attention_weight_min": -34.788612365722656,
      "activations/layer14_attention_weight_max": 41.31049346923828,
      "activations/layer14_attention_weight_min": -35.79127502441406,
      "activations/layer15_attention_weight_max": 36.81159591674805,
      "activations/layer15_attention_weight_min": -32.30458450317383,
      "activations/layer16_attention_weight_max": 29.671180725097656,
      "activations/layer16_attention_weight_min": -25.756853103637695,
      "activations/layer17_attention_weight_max": 30.6456241607666,
      "activations/layer17_attention_weight_min": -24.21668815612793,
      "activations/layer18_attention_weight_max": 28.61680793762207,
      "activations/layer18_attention_weight_min": -24.695877075195312,
      "activations/layer19_attention_weight_max": 31.823965072631836,
      "activations/layer19_attention_weight_min": -28.999073028564453,
      "activations/layer1_attention_weight_max": 17.308536529541016,
      "activations/layer1_attention_weight_min": -14.136702537536621,
      "activations/layer20_attention_weight_max": 31.323884963989258,
      "activations/layer20_attention_weight_min": -24.129013061523438,
      "activations/layer21_attention_weight_max": 30.18614387512207,
      "activations/layer21_attention_weight_min": -22.705530166625977,
      "activations/layer22_attention_weight_max": 38.58932876586914,
      "activations/layer22_attention_weight_min": -28.21041488647461,
      "activations/layer23_attention_weight_max": 29.701263427734375,
      "activations/layer23_attention_weight_min": -23.385272979736328,
      "activations/layer2_attention_weight_max": 31.80642318725586,
      "activations/layer2_attention_weight_min": -31.183988571166992,
      "activations/layer3_attention_weight_max": 98.3644790649414,
      "activations/layer3_attention_weight_min": -96.85101318359375,
      "activations/layer4_attention_weight_max": 116.86610412597656,
      "activations/layer4_attention_weight_min": -118.10186767578125,
      "activations/layer5_attention_weight_max": 54.34541320800781,
      "activations/layer5_attention_weight_min": -66.64769744873047,
      "activations/layer6_attention_weight_max": 48.08555221557617,
      "activations/layer6_attention_weight_min": -49.63869094848633,
      "activations/layer7_attention_weight_max": 96.49111938476562,
      "activations/layer7_attention_weight_min": -99.67304229736328,
      "activations/layer8_attention_weight_max": 44.184452056884766,
      "activations/layer8_attention_weight_min": -47.17101287841797,
      "activations/layer9_attention_weight_max": 35.56588363647461,
      "activations/layer9_attention_weight_min": -37.77473068237305,
      "epoch": 14.87,
      "learning_rate": 5.46435606060606e-05,
      "loss": 2.7585,
      "step": 255950
    },
    {
      "activations/layer0_attention_weight_max": 15.807014465332031,
      "activations/layer0_attention_weight_min": -12.032349586486816,
      "activations/layer10_attention_weight_max": 33.386741638183594,
      "activations/layer10_attention_weight_min": -34.84654998779297,
      "activations/layer11_attention_weight_max": 31.87392807006836,
      "activations/layer11_attention_weight_min": -33.64790344238281,
      "activations/layer12_attention_weight_max": 27.892337799072266,
      "activations/layer12_attention_weight_min": -26.576005935668945,
      "activations/layer13_attention_weight_max": 43.949058532714844,
      "activations/layer13_attention_weight_min": -36.165687561035156,
      "activations/layer14_attention_weight_max": 47.98344802856445,
      "activations/layer14_attention_weight_min": -38.971553802490234,
      "activations/layer15_attention_weight_max": 40.96171951293945,
      "activations/layer15_attention_weight_min": -37.7437629699707,
      "activations/layer16_attention_weight_max": 32.71635818481445,
      "activations/layer16_attention_weight_min": -28.417726516723633,
      "activations/layer17_attention_weight_max": 32.771671295166016,
      "activations/layer17_attention_weight_min": -26.897024154663086,
      "activations/layer18_attention_weight_max": 33.76323318481445,
      "activations/layer18_attention_weight_min": -26.69469451904297,
      "activations/layer19_attention_weight_max": 34.56438064575195,
      "activations/layer19_attention_weight_min": -31.903528213500977,
      "activations/layer1_attention_weight_max": 16.88622283935547,
      "activations/layer1_attention_weight_min": -13.925721168518066,
      "activations/layer20_attention_weight_max": 31.764575958251953,
      "activations/layer20_attention_weight_min": -25.64281463623047,
      "activations/layer21_attention_weight_max": 29.533935546875,
      "activations/layer21_attention_weight_min": -26.925533294677734,
      "activations/layer22_attention_weight_max": 42.99968338012695,
      "activations/layer22_attention_weight_min": -34.54398727416992,
      "activations/layer23_attention_weight_max": 36.48075485229492,
      "activations/layer23_attention_weight_min": -25.31654930114746,
      "activations/layer2_attention_weight_max": 30.983699798583984,
      "activations/layer2_attention_weight_min": -32.753273010253906,
      "activations/layer3_attention_weight_max": 91.9702377319336,
      "activations/layer3_attention_weight_min": -97.53266906738281,
      "activations/layer4_attention_weight_max": 104.07975006103516,
      "activations/layer4_attention_weight_min": -111.6332015991211,
      "activations/layer5_attention_weight_max": 51.35251998901367,
      "activations/layer5_attention_weight_min": -63.62650680541992,
      "activations/layer6_attention_weight_max": 43.08638381958008,
      "activations/layer6_attention_weight_min": -45.526973724365234,
      "activations/layer7_attention_weight_max": 93.56169891357422,
      "activations/layer7_attention_weight_min": -89.7144775390625,
      "activations/layer8_attention_weight_max": 39.24330520629883,
      "activations/layer8_attention_weight_min": -44.46389389038086,
      "activations/layer9_attention_weight_max": 32.37305450439453,
      "activations/layer9_attention_weight_min": -34.71023178100586,
      "epoch": 14.88,
      "learning_rate": 5.462462121212121e-05,
      "loss": 2.7635,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.5517,
      "eval_samples_per_second": 502.12,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.5517,
      "eval_openwebtext_samples_per_second": 502.12,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9884,
      "eval_wikitext_samples_per_second": 229.33,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_lambada_loss": 2.384765625,
      "eval_lambada_ppl": 10.85651787033819,
      "eval_lambada_runtime": 9.6141,
      "eval_lambada_samples_per_second": 506.445,
      "step": 256000
    },
    {
      "activations/layer0_attention_weight_max": 15.830243110656738,
      "activations/layer0_attention_weight_min": -11.458638191223145,
      "activations/layer10_attention_weight_max": 31.46178436279297,
      "activations/layer10_attention_weight_min": -32.62962341308594,
      "activations/layer11_attention_weight_max": 30.725780487060547,
      "activations/layer11_attention_weight_min": -32.75128173828125,
      "activations/layer12_attention_weight_max": 26.171232223510742,
      "activations/layer12_attention_weight_min": -25.971656799316406,
      "activations/layer13_attention_weight_max": 41.09291076660156,
      "activations/layer13_attention_weight_min": -35.170997619628906,
      "activations/layer14_attention_weight_max": 41.23457336425781,
      "activations/layer14_attention_weight_min": -37.83266830444336,
      "activations/layer15_attention_weight_max": 38.41456985473633,
      "activations/layer15_attention_weight_min": -35.79201889038086,
      "activations/layer16_attention_weight_max": 28.73198699951172,
      "activations/layer16_attention_weight_min": -28.956411361694336,
      "activations/layer17_attention_weight_max": 28.95940589904785,
      "activations/layer17_attention_weight_min": -27.598146438598633,
      "activations/layer18_attention_weight_max": 30.066865921020508,
      "activations/layer18_attention_weight_min": -25.360130310058594,
      "activations/layer19_attention_weight_max": 33.66763687133789,
      "activations/layer19_attention_weight_min": -33.00864791870117,
      "activations/layer1_attention_weight_max": 16.560596466064453,
      "activations/layer1_attention_weight_min": -13.404133796691895,
      "activations/layer20_attention_weight_max": 29.39309310913086,
      "activations/layer20_attention_weight_min": -26.788297653198242,
      "activations/layer21_attention_weight_max": 29.95406723022461,
      "activations/layer21_attention_weight_min": -26.872390747070312,
      "activations/layer22_attention_weight_max": 45.7072868347168,
      "activations/layer22_attention_weight_min": -34.29539489746094,
      "activations/layer23_attention_weight_max": 31.467235565185547,
      "activations/layer23_attention_weight_min": -25.154829025268555,
      "activations/layer2_attention_weight_max": 36.14385986328125,
      "activations/layer2_attention_weight_min": -31.199026107788086,
      "activations/layer3_attention_weight_max": 97.3175048828125,
      "activations/layer3_attention_weight_min": -92.50721740722656,
      "activations/layer4_attention_weight_max": 113.39311981201172,
      "activations/layer4_attention_weight_min": -110.78777313232422,
      "activations/layer5_attention_weight_max": 51.16874313354492,
      "activations/layer5_attention_weight_min": -62.3643798828125,
      "activations/layer6_attention_weight_max": 44.588706970214844,
      "activations/layer6_attention_weight_min": -45.38664627075195,
      "activations/layer7_attention_weight_max": 89.95352935791016,
      "activations/layer7_attention_weight_min": -89.44203186035156,
      "activations/layer8_attention_weight_max": 39.554046630859375,
      "activations/layer8_attention_weight_min": -42.15082550048828,
      "activations/layer9_attention_weight_max": 33.0765380859375,
      "activations/layer9_attention_weight_min": -33.20735168457031,
      "epoch": 14.88,
      "learning_rate": 5.460568181818181e-05,
      "loss": 2.7598,
      "step": 256050
    },
    {
      "activations/layer0_attention_weight_max": 15.871124267578125,
      "activations/layer0_attention_weight_min": -12.532913208007812,
      "activations/layer10_attention_weight_max": 34.191688537597656,
      "activations/layer10_attention_weight_min": -32.73292922973633,
      "activations/layer11_attention_weight_max": 33.48532485961914,
      "activations/layer11_attention_weight_min": -32.81538391113281,
      "activations/layer12_attention_weight_max": 26.074832916259766,
      "activations/layer12_attention_weight_min": -24.934518814086914,
      "activations/layer13_attention_weight_max": 39.75396728515625,
      "activations/layer13_attention_weight_min": -33.769649505615234,
      "activations/layer14_attention_weight_max": 42.39670944213867,
      "activations/layer14_attention_weight_min": -36.46145248413086,
      "activations/layer15_attention_weight_max": 37.23979187011719,
      "activations/layer15_attention_weight_min": -31.884666442871094,
      "activations/layer16_attention_weight_max": 31.123779296875,
      "activations/layer16_attention_weight_min": -27.726242065429688,
      "activations/layer17_attention_weight_max": 30.149465560913086,
      "activations/layer17_attention_weight_min": -25.51287269592285,
      "activations/layer18_attention_weight_max": 33.312255859375,
      "activations/layer18_attention_weight_min": -25.040788650512695,
      "activations/layer19_attention_weight_max": 34.44744873046875,
      "activations/layer19_attention_weight_min": -32.88301086425781,
      "activations/layer1_attention_weight_max": 16.76971435546875,
      "activations/layer1_attention_weight_min": -13.647594451904297,
      "activations/layer20_attention_weight_max": 29.857370376586914,
      "activations/layer20_attention_weight_min": -25.6118106842041,
      "activations/layer21_attention_weight_max": 28.168296813964844,
      "activations/layer21_attention_weight_min": -26.267980575561523,
      "activations/layer22_attention_weight_max": 40.42310333251953,
      "activations/layer22_attention_weight_min": -30.617952346801758,
      "activations/layer23_attention_weight_max": 33.348548889160156,
      "activations/layer23_attention_weight_min": -25.874916076660156,
      "activations/layer2_attention_weight_max": 32.49524688720703,
      "activations/layer2_attention_weight_min": -31.84978485107422,
      "activations/layer3_attention_weight_max": 94.66668701171875,
      "activations/layer3_attention_weight_min": -94.52379608154297,
      "activations/layer4_attention_weight_max": 108.25821685791016,
      "activations/layer4_attention_weight_min": -114.52198791503906,
      "activations/layer5_attention_weight_max": 51.13143539428711,
      "activations/layer5_attention_weight_min": -64.46226501464844,
      "activations/layer6_attention_weight_max": 43.34141540527344,
      "activations/layer6_attention_weight_min": -45.55072784423828,
      "activations/layer7_attention_weight_max": 88.5952377319336,
      "activations/layer7_attention_weight_min": -95.2346420288086,
      "activations/layer8_attention_weight_max": 39.686767578125,
      "activations/layer8_attention_weight_min": -42.042694091796875,
      "activations/layer9_attention_weight_max": 31.94541358947754,
      "activations/layer9_attention_weight_min": -32.309288024902344,
      "epoch": 14.88,
      "learning_rate": 5.458674242424242e-05,
      "loss": 2.7506,
      "step": 256100
    },
    {
      "activations/layer0_attention_weight_max": 15.699934005737305,
      "activations/layer0_attention_weight_min": -11.928566932678223,
      "activations/layer10_attention_weight_max": 34.78411865234375,
      "activations/layer10_attention_weight_min": -34.0258674621582,
      "activations/layer11_attention_weight_max": 32.88233184814453,
      "activations/layer11_attention_weight_min": -35.08839416503906,
      "activations/layer12_attention_weight_max": 27.18537139892578,
      "activations/layer12_attention_weight_min": -25.769601821899414,
      "activations/layer13_attention_weight_max": 38.934783935546875,
      "activations/layer13_attention_weight_min": -34.962615966796875,
      "activations/layer14_attention_weight_max": 42.2923583984375,
      "activations/layer14_attention_weight_min": -37.785728454589844,
      "activations/layer15_attention_weight_max": 38.40528106689453,
      "activations/layer15_attention_weight_min": -32.5811767578125,
      "activations/layer16_attention_weight_max": 28.93646240234375,
      "activations/layer16_attention_weight_min": -28.384309768676758,
      "activations/layer17_attention_weight_max": 29.64691925048828,
      "activations/layer17_attention_weight_min": -25.669841766357422,
      "activations/layer18_attention_weight_max": 31.965618133544922,
      "activations/layer18_attention_weight_min": -25.153703689575195,
      "activations/layer19_attention_weight_max": 36.1657600402832,
      "activations/layer19_attention_weight_min": -31.474166870117188,
      "activations/layer1_attention_weight_max": 16.08784294128418,
      "activations/layer1_attention_weight_min": -15.071815490722656,
      "activations/layer20_attention_weight_max": 29.346588134765625,
      "activations/layer20_attention_weight_min": -25.867891311645508,
      "activations/layer21_attention_weight_max": 27.47066879272461,
      "activations/layer21_attention_weight_min": -25.1704044342041,
      "activations/layer22_attention_weight_max": 40.218902587890625,
      "activations/layer22_attention_weight_min": -31.263851165771484,
      "activations/layer23_attention_weight_max": 35.250213623046875,
      "activations/layer23_attention_weight_min": -24.018497467041016,
      "activations/layer2_attention_weight_max": 31.674583435058594,
      "activations/layer2_attention_weight_min": -31.512691497802734,
      "activations/layer3_attention_weight_max": 96.24504089355469,
      "activations/layer3_attention_weight_min": -98.41839599609375,
      "activations/layer4_attention_weight_max": 105.68585205078125,
      "activations/layer4_attention_weight_min": -110.09077453613281,
      "activations/layer5_attention_weight_max": 50.156005859375,
      "activations/layer5_attention_weight_min": -63.23741149902344,
      "activations/layer6_attention_weight_max": 43.15861511230469,
      "activations/layer6_attention_weight_min": -45.22373580932617,
      "activations/layer7_attention_weight_max": 92.13247680664062,
      "activations/layer7_attention_weight_min": -91.53321075439453,
      "activations/layer8_attention_weight_max": 40.30076599121094,
      "activations/layer8_attention_weight_min": -41.67345428466797,
      "activations/layer9_attention_weight_max": 32.36660385131836,
      "activations/layer9_attention_weight_min": -33.94828414916992,
      "epoch": 14.88,
      "learning_rate": 5.456780303030302e-05,
      "loss": 2.7727,
      "step": 256150
    },
    {
      "activations/layer0_attention_weight_max": 17.042619705200195,
      "activations/layer0_attention_weight_min": -12.369462013244629,
      "activations/layer10_attention_weight_max": 38.79829406738281,
      "activations/layer10_attention_weight_min": -35.12112045288086,
      "activations/layer11_attention_weight_max": 44.68885803222656,
      "activations/layer11_attention_weight_min": -40.14042282104492,
      "activations/layer12_attention_weight_max": 31.730064392089844,
      "activations/layer12_attention_weight_min": -28.608890533447266,
      "activations/layer13_attention_weight_max": 47.39389419555664,
      "activations/layer13_attention_weight_min": -40.98784255981445,
      "activations/layer14_attention_weight_max": 54.599246978759766,
      "activations/layer14_attention_weight_min": -44.911529541015625,
      "activations/layer15_attention_weight_max": 49.1612663269043,
      "activations/layer15_attention_weight_min": -42.13123321533203,
      "activations/layer16_attention_weight_max": 38.865516662597656,
      "activations/layer16_attention_weight_min": -30.372554779052734,
      "activations/layer17_attention_weight_max": 34.89482116699219,
      "activations/layer17_attention_weight_min": -26.657331466674805,
      "activations/layer18_attention_weight_max": 33.41099166870117,
      "activations/layer18_attention_weight_min": -25.403844833374023,
      "activations/layer19_attention_weight_max": 36.08852767944336,
      "activations/layer19_attention_weight_min": -31.319091796875,
      "activations/layer1_attention_weight_max": 15.79671573638916,
      "activations/layer1_attention_weight_min": -13.314657211303711,
      "activations/layer20_attention_weight_max": 31.912878036499023,
      "activations/layer20_attention_weight_min": -26.05434799194336,
      "activations/layer21_attention_weight_max": 31.062984466552734,
      "activations/layer21_attention_weight_min": -23.946409225463867,
      "activations/layer22_attention_weight_max": 46.09695816040039,
      "activations/layer22_attention_weight_min": -31.28362274169922,
      "activations/layer23_attention_weight_max": 34.116371154785156,
      "activations/layer23_attention_weight_min": -24.41543197631836,
      "activations/layer2_attention_weight_max": 31.114072799682617,
      "activations/layer2_attention_weight_min": -30.905515670776367,
      "activations/layer3_attention_weight_max": 97.93370819091797,
      "activations/layer3_attention_weight_min": -93.90271759033203,
      "activations/layer4_attention_weight_max": 111.8357162475586,
      "activations/layer4_attention_weight_min": -107.57563781738281,
      "activations/layer5_attention_weight_max": 54.279090881347656,
      "activations/layer5_attention_weight_min": -62.03927230834961,
      "activations/layer6_attention_weight_max": 46.50304412841797,
      "activations/layer6_attention_weight_min": -46.20613098144531,
      "activations/layer7_attention_weight_max": 94.9178466796875,
      "activations/layer7_attention_weight_min": -95.60774993896484,
      "activations/layer8_attention_weight_max": 43.01087188720703,
      "activations/layer8_attention_weight_min": -44.350914001464844,
      "activations/layer9_attention_weight_max": 36.824642181396484,
      "activations/layer9_attention_weight_min": -34.073543548583984,
      "epoch": 14.89,
      "learning_rate": 5.4548863636363637e-05,
      "loss": 2.7564,
      "step": 256200
    },
    {
      "activations/layer0_attention_weight_max": 16.104759216308594,
      "activations/layer0_attention_weight_min": -12.002801895141602,
      "activations/layer10_attention_weight_max": 31.89848518371582,
      "activations/layer10_attention_weight_min": -34.479896545410156,
      "activations/layer11_attention_weight_max": 34.37017822265625,
      "activations/layer11_attention_weight_min": -32.91676712036133,
      "activations/layer12_attention_weight_max": 27.2296085357666,
      "activations/layer12_attention_weight_min": -26.897541046142578,
      "activations/layer13_attention_weight_max": 39.2680549621582,
      "activations/layer13_attention_weight_min": -34.506500244140625,
      "activations/layer14_attention_weight_max": 44.743412017822266,
      "activations/layer14_attention_weight_min": -36.52702331542969,
      "activations/layer15_attention_weight_max": 40.250205993652344,
      "activations/layer15_attention_weight_min": -35.42063522338867,
      "activations/layer16_attention_weight_max": 32.79085159301758,
      "activations/layer16_attention_weight_min": -29.753284454345703,
      "activations/layer17_attention_weight_max": 33.36396408081055,
      "activations/layer17_attention_weight_min": -27.323680877685547,
      "activations/layer18_attention_weight_max": 30.699331283569336,
      "activations/layer18_attention_weight_min": -25.068418502807617,
      "activations/layer19_attention_weight_max": 32.36809158325195,
      "activations/layer19_attention_weight_min": -32.50392532348633,
      "activations/layer1_attention_weight_max": 16.178983688354492,
      "activations/layer1_attention_weight_min": -13.697244644165039,
      "activations/layer20_attention_weight_max": 28.194229125976562,
      "activations/layer20_attention_weight_min": -27.96602439880371,
      "activations/layer21_attention_weight_max": 27.540584564208984,
      "activations/layer21_attention_weight_min": -25.489225387573242,
      "activations/layer22_attention_weight_max": 42.77525329589844,
      "activations/layer22_attention_weight_min": -29.454431533813477,
      "activations/layer23_attention_weight_max": 33.67508316040039,
      "activations/layer23_attention_weight_min": -25.38292694091797,
      "activations/layer2_attention_weight_max": 29.476444244384766,
      "activations/layer2_attention_weight_min": -29.84902000427246,
      "activations/layer3_attention_weight_max": 87.86946105957031,
      "activations/layer3_attention_weight_min": -93.1831283569336,
      "activations/layer4_attention_weight_max": 101.56099700927734,
      "activations/layer4_attention_weight_min": -110.1556396484375,
      "activations/layer5_attention_weight_max": 49.31759262084961,
      "activations/layer5_attention_weight_min": -64.73489379882812,
      "activations/layer6_attention_weight_max": 43.02018356323242,
      "activations/layer6_attention_weight_min": -46.84623336791992,
      "activations/layer7_attention_weight_max": 85.78289794921875,
      "activations/layer7_attention_weight_min": -95.36846160888672,
      "activations/layer8_attention_weight_max": 40.26695251464844,
      "activations/layer8_attention_weight_min": -43.590763092041016,
      "activations/layer9_attention_weight_max": 30.43721580505371,
      "activations/layer9_attention_weight_min": -34.02260971069336,
      "epoch": 14.89,
      "learning_rate": 5.452992424242424e-05,
      "loss": 2.7516,
      "step": 256250
    },
    {
      "activations/layer0_attention_weight_max": 15.96420669555664,
      "activations/layer0_attention_weight_min": -12.207149505615234,
      "activations/layer10_attention_weight_max": 36.655372619628906,
      "activations/layer10_attention_weight_min": -34.26966094970703,
      "activations/layer11_attention_weight_max": 39.72822189331055,
      "activations/layer11_attention_weight_min": -33.436317443847656,
      "activations/layer12_attention_weight_max": 26.03173065185547,
      "activations/layer12_attention_weight_min": -26.866722106933594,
      "activations/layer13_attention_weight_max": 39.8966064453125,
      "activations/layer13_attention_weight_min": -35.188720703125,
      "activations/layer14_attention_weight_max": 43.32029724121094,
      "activations/layer14_attention_weight_min": -39.59872055053711,
      "activations/layer15_attention_weight_max": 36.22750473022461,
      "activations/layer15_attention_weight_min": -34.61105728149414,
      "activations/layer16_attention_weight_max": 30.661352157592773,
      "activations/layer16_attention_weight_min": -28.19057273864746,
      "activations/layer17_attention_weight_max": 29.566179275512695,
      "activations/layer17_attention_weight_min": -25.487346649169922,
      "activations/layer18_attention_weight_max": 29.99730110168457,
      "activations/layer18_attention_weight_min": -24.68029022216797,
      "activations/layer19_attention_weight_max": 33.0786247253418,
      "activations/layer19_attention_weight_min": -31.985000610351562,
      "activations/layer1_attention_weight_max": 16.454421997070312,
      "activations/layer1_attention_weight_min": -14.816290855407715,
      "activations/layer20_attention_weight_max": 27.777862548828125,
      "activations/layer20_attention_weight_min": -24.22711753845215,
      "activations/layer21_attention_weight_max": 27.519121170043945,
      "activations/layer21_attention_weight_min": -24.431503295898438,
      "activations/layer22_attention_weight_max": 42.65402603149414,
      "activations/layer22_attention_weight_min": -30.06895637512207,
      "activations/layer23_attention_weight_max": 33.17407989501953,
      "activations/layer23_attention_weight_min": -25.53970718383789,
      "activations/layer2_attention_weight_max": 34.044193267822266,
      "activations/layer2_attention_weight_min": -32.70149230957031,
      "activations/layer3_attention_weight_max": 97.3623046875,
      "activations/layer3_attention_weight_min": -99.00899505615234,
      "activations/layer4_attention_weight_max": 109.5255355834961,
      "activations/layer4_attention_weight_min": -110.7304458618164,
      "activations/layer5_attention_weight_max": 52.63832092285156,
      "activations/layer5_attention_weight_min": -59.835994720458984,
      "activations/layer6_attention_weight_max": 46.115116119384766,
      "activations/layer6_attention_weight_min": -48.36616897583008,
      "activations/layer7_attention_weight_max": 91.46748352050781,
      "activations/layer7_attention_weight_min": -95.7033462524414,
      "activations/layer8_attention_weight_max": 42.93559646606445,
      "activations/layer8_attention_weight_min": -42.8575553894043,
      "activations/layer9_attention_weight_max": 33.6728630065918,
      "activations/layer9_attention_weight_min": -35.2418327331543,
      "epoch": 14.89,
      "learning_rate": 5.451098484848484e-05,
      "loss": 2.7503,
      "step": 256300
    },
    {
      "activations/layer0_attention_weight_max": 16.41421127319336,
      "activations/layer0_attention_weight_min": -12.432271003723145,
      "activations/layer10_attention_weight_max": 31.742765426635742,
      "activations/layer10_attention_weight_min": -34.426109313964844,
      "activations/layer11_attention_weight_max": 31.619590759277344,
      "activations/layer11_attention_weight_min": -33.441551208496094,
      "activations/layer12_attention_weight_max": 24.662565231323242,
      "activations/layer12_attention_weight_min": -26.014602661132812,
      "activations/layer13_attention_weight_max": 36.13754653930664,
      "activations/layer13_attention_weight_min": -35.780723571777344,
      "activations/layer14_attention_weight_max": 37.93723678588867,
      "activations/layer14_attention_weight_min": -38.043235778808594,
      "activations/layer15_attention_weight_max": 33.626834869384766,
      "activations/layer15_attention_weight_min": -34.7022705078125,
      "activations/layer16_attention_weight_max": 30.145633697509766,
      "activations/layer16_attention_weight_min": -30.320735931396484,
      "activations/layer17_attention_weight_max": 27.953205108642578,
      "activations/layer17_attention_weight_min": -25.631460189819336,
      "activations/layer18_attention_weight_max": 29.26402473449707,
      "activations/layer18_attention_weight_min": -25.590795516967773,
      "activations/layer19_attention_weight_max": 33.90946578979492,
      "activations/layer19_attention_weight_min": -35.78788375854492,
      "activations/layer1_attention_weight_max": 16.29757308959961,
      "activations/layer1_attention_weight_min": -14.437575340270996,
      "activations/layer20_attention_weight_max": 29.886960983276367,
      "activations/layer20_attention_weight_min": -27.19942283630371,
      "activations/layer21_attention_weight_max": 28.192880630493164,
      "activations/layer21_attention_weight_min": -25.017807006835938,
      "activations/layer22_attention_weight_max": 44.110809326171875,
      "activations/layer22_attention_weight_min": -33.67428970336914,
      "activations/layer23_attention_weight_max": 32.23881530761719,
      "activations/layer23_attention_weight_min": -28.660633087158203,
      "activations/layer2_attention_weight_max": 32.125465393066406,
      "activations/layer2_attention_weight_min": -31.908079147338867,
      "activations/layer3_attention_weight_max": 93.87577819824219,
      "activations/layer3_attention_weight_min": -94.3539047241211,
      "activations/layer4_attention_weight_max": 105.37767791748047,
      "activations/layer4_attention_weight_min": -108.42640686035156,
      "activations/layer5_attention_weight_max": 52.48923873901367,
      "activations/layer5_attention_weight_min": -63.86179733276367,
      "activations/layer6_attention_weight_max": 46.71471405029297,
      "activations/layer6_attention_weight_min": -47.39755630493164,
      "activations/layer7_attention_weight_max": 95.81442260742188,
      "activations/layer7_attention_weight_min": -94.1343002319336,
      "activations/layer8_attention_weight_max": 38.3001823425293,
      "activations/layer8_attention_weight_min": -44.43217086791992,
      "activations/layer9_attention_weight_max": 30.758413314819336,
      "activations/layer9_attention_weight_min": -34.138145446777344,
      "epoch": 14.9,
      "learning_rate": 5.449204545454545e-05,
      "loss": 2.7661,
      "step": 256350
    },
    {
      "activations/layer0_attention_weight_max": 15.847972869873047,
      "activations/layer0_attention_weight_min": -12.746170043945312,
      "activations/layer10_attention_weight_max": 34.77910614013672,
      "activations/layer10_attention_weight_min": -34.03071212768555,
      "activations/layer11_attention_weight_max": 32.729705810546875,
      "activations/layer11_attention_weight_min": -33.098236083984375,
      "activations/layer12_attention_weight_max": 26.41743278503418,
      "activations/layer12_attention_weight_min": -27.395124435424805,
      "activations/layer13_attention_weight_max": 39.66872024536133,
      "activations/layer13_attention_weight_min": -38.143463134765625,
      "activations/layer14_attention_weight_max": 41.22059631347656,
      "activations/layer14_attention_weight_min": -37.78972244262695,
      "activations/layer15_attention_weight_max": 34.973487854003906,
      "activations/layer15_attention_weight_min": -33.79536437988281,
      "activations/layer16_attention_weight_max": 26.971864700317383,
      "activations/layer16_attention_weight_min": -28.304931640625,
      "activations/layer17_attention_weight_max": 29.910797119140625,
      "activations/layer17_attention_weight_min": -25.975189208984375,
      "activations/layer18_attention_weight_max": 31.13441276550293,
      "activations/layer18_attention_weight_min": -25.834640502929688,
      "activations/layer19_attention_weight_max": 31.900876998901367,
      "activations/layer19_attention_weight_min": -32.16520690917969,
      "activations/layer1_attention_weight_max": 17.19989013671875,
      "activations/layer1_attention_weight_min": -15.418205261230469,
      "activations/layer20_attention_weight_max": 27.8480224609375,
      "activations/layer20_attention_weight_min": -25.705631256103516,
      "activations/layer21_attention_weight_max": 28.999547958374023,
      "activations/layer21_attention_weight_min": -25.21162986755371,
      "activations/layer22_attention_weight_max": 38.03825759887695,
      "activations/layer22_attention_weight_min": -31.359004974365234,
      "activations/layer23_attention_weight_max": 30.210872650146484,
      "activations/layer23_attention_weight_min": -25.376800537109375,
      "activations/layer2_attention_weight_max": 30.160865783691406,
      "activations/layer2_attention_weight_min": -30.785465240478516,
      "activations/layer3_attention_weight_max": 90.38819885253906,
      "activations/layer3_attention_weight_min": -94.40301513671875,
      "activations/layer4_attention_weight_max": 108.53704833984375,
      "activations/layer4_attention_weight_min": -109.31497955322266,
      "activations/layer5_attention_weight_max": 48.740814208984375,
      "activations/layer5_attention_weight_min": -65.81063842773438,
      "activations/layer6_attention_weight_max": 42.52534484863281,
      "activations/layer6_attention_weight_min": -47.38805389404297,
      "activations/layer7_attention_weight_max": 87.73747253417969,
      "activations/layer7_attention_weight_min": -89.92717742919922,
      "activations/layer8_attention_weight_max": 39.5291862487793,
      "activations/layer8_attention_weight_min": -41.20576858520508,
      "activations/layer9_attention_weight_max": 34.35390853881836,
      "activations/layer9_attention_weight_min": -34.5522575378418,
      "epoch": 14.9,
      "learning_rate": 5.4473106060606056e-05,
      "loss": 2.7472,
      "step": 256400
    },
    {
      "activations/layer0_attention_weight_max": 16.3642635345459,
      "activations/layer0_attention_weight_min": -11.369600296020508,
      "activations/layer10_attention_weight_max": 34.285064697265625,
      "activations/layer10_attention_weight_min": -32.67422866821289,
      "activations/layer11_attention_weight_max": 32.651519775390625,
      "activations/layer11_attention_weight_min": -33.58653259277344,
      "activations/layer12_attention_weight_max": 26.132661819458008,
      "activations/layer12_attention_weight_min": -26.20501136779785,
      "activations/layer13_attention_weight_max": 40.1248664855957,
      "activations/layer13_attention_weight_min": -33.830196380615234,
      "activations/layer14_attention_weight_max": 42.45941925048828,
      "activations/layer14_attention_weight_min": -38.64069366455078,
      "activations/layer15_attention_weight_max": 36.72006607055664,
      "activations/layer15_attention_weight_min": -33.71090316772461,
      "activations/layer16_attention_weight_max": 31.035364151000977,
      "activations/layer16_attention_weight_min": -28.525775909423828,
      "activations/layer17_attention_weight_max": 32.110595703125,
      "activations/layer17_attention_weight_min": -25.767684936523438,
      "activations/layer18_attention_weight_max": 35.377384185791016,
      "activations/layer18_attention_weight_min": -25.670488357543945,
      "activations/layer19_attention_weight_max": 36.14967727661133,
      "activations/layer19_attention_weight_min": -36.235694885253906,
      "activations/layer1_attention_weight_max": 16.359600067138672,
      "activations/layer1_attention_weight_min": -13.90803050994873,
      "activations/layer20_attention_weight_max": 32.78913879394531,
      "activations/layer20_attention_weight_min": -26.603836059570312,
      "activations/layer21_attention_weight_max": 30.28717613220215,
      "activations/layer21_attention_weight_min": -25.69588279724121,
      "activations/layer22_attention_weight_max": 48.97517395019531,
      "activations/layer22_attention_weight_min": -34.1647834777832,
      "activations/layer23_attention_weight_max": 36.27233123779297,
      "activations/layer23_attention_weight_min": -26.9840145111084,
      "activations/layer2_attention_weight_max": 31.5408992767334,
      "activations/layer2_attention_weight_min": -31.29640769958496,
      "activations/layer3_attention_weight_max": 96.61089324951172,
      "activations/layer3_attention_weight_min": -95.38993072509766,
      "activations/layer4_attention_weight_max": 110.27178955078125,
      "activations/layer4_attention_weight_min": -114.32899475097656,
      "activations/layer5_attention_weight_max": 51.891014099121094,
      "activations/layer5_attention_weight_min": -64.26081848144531,
      "activations/layer6_attention_weight_max": 45.549320220947266,
      "activations/layer6_attention_weight_min": -47.50647735595703,
      "activations/layer7_attention_weight_max": 91.3280258178711,
      "activations/layer7_attention_weight_min": -91.92274475097656,
      "activations/layer8_attention_weight_max": 41.82698059082031,
      "activations/layer8_attention_weight_min": -43.83314895629883,
      "activations/layer9_attention_weight_max": 32.731040954589844,
      "activations/layer9_attention_weight_min": -33.600502014160156,
      "epoch": 14.9,
      "learning_rate": 5.4454166666666664e-05,
      "loss": 2.7567,
      "step": 256450
    },
    {
      "activations/layer0_attention_weight_max": 15.592302322387695,
      "activations/layer0_attention_weight_min": -12.044249534606934,
      "activations/layer10_attention_weight_max": 40.29927062988281,
      "activations/layer10_attention_weight_min": -38.070045471191406,
      "activations/layer11_attention_weight_max": 38.852420806884766,
      "activations/layer11_attention_weight_min": -35.94499206542969,
      "activations/layer12_attention_weight_max": 27.199190139770508,
      "activations/layer12_attention_weight_min": -27.658109664916992,
      "activations/layer13_attention_weight_max": 41.759971618652344,
      "activations/layer13_attention_weight_min": -36.465179443359375,
      "activations/layer14_attention_weight_max": 43.642181396484375,
      "activations/layer14_attention_weight_min": -40.203495025634766,
      "activations/layer15_attention_weight_max": 39.52019119262695,
      "activations/layer15_attention_weight_min": -36.86703872680664,
      "activations/layer16_attention_weight_max": 31.13732147216797,
      "activations/layer16_attention_weight_min": -29.025697708129883,
      "activations/layer17_attention_weight_max": 31.086774826049805,
      "activations/layer17_attention_weight_min": -26.656415939331055,
      "activations/layer18_attention_weight_max": 32.435638427734375,
      "activations/layer18_attention_weight_min": -25.720491409301758,
      "activations/layer19_attention_weight_max": 36.75030517578125,
      "activations/layer19_attention_weight_min": -33.350624084472656,
      "activations/layer1_attention_weight_max": 15.598318099975586,
      "activations/layer1_attention_weight_min": -14.54964828491211,
      "activations/layer20_attention_weight_max": 29.39156723022461,
      "activations/layer20_attention_weight_min": -25.418916702270508,
      "activations/layer21_attention_weight_max": 29.89789390563965,
      "activations/layer21_attention_weight_min": -24.988317489624023,
      "activations/layer22_attention_weight_max": 46.46145248413086,
      "activations/layer22_attention_weight_min": -34.472110748291016,
      "activations/layer23_attention_weight_max": 35.41009521484375,
      "activations/layer23_attention_weight_min": -26.758787155151367,
      "activations/layer2_attention_weight_max": 31.07571029663086,
      "activations/layer2_attention_weight_min": -31.058265686035156,
      "activations/layer3_attention_weight_max": 94.28759765625,
      "activations/layer3_attention_weight_min": -95.46251678466797,
      "activations/layer4_attention_weight_max": 106.76273345947266,
      "activations/layer4_attention_weight_min": -111.6252670288086,
      "activations/layer5_attention_weight_max": 51.74864196777344,
      "activations/layer5_attention_weight_min": -61.28194808959961,
      "activations/layer6_attention_weight_max": 45.9001579284668,
      "activations/layer6_attention_weight_min": -47.93220901489258,
      "activations/layer7_attention_weight_max": 98.43024444580078,
      "activations/layer7_attention_weight_min": -97.71097564697266,
      "activations/layer8_attention_weight_max": 45.47661209106445,
      "activations/layer8_attention_weight_min": -44.532920837402344,
      "activations/layer9_attention_weight_max": 36.832481384277344,
      "activations/layer9_attention_weight_min": -35.87852096557617,
      "epoch": 14.9,
      "learning_rate": 5.4435227272727266e-05,
      "loss": 2.7659,
      "step": 256500
    },
    {
      "activations/layer0_attention_weight_max": 16.835752487182617,
      "activations/layer0_attention_weight_min": -11.375326156616211,
      "activations/layer10_attention_weight_max": 34.6081657409668,
      "activations/layer10_attention_weight_min": -33.502235412597656,
      "activations/layer11_attention_weight_max": 34.90019989013672,
      "activations/layer11_attention_weight_min": -34.012393951416016,
      "activations/layer12_attention_weight_max": 29.942554473876953,
      "activations/layer12_attention_weight_min": -25.59632682800293,
      "activations/layer13_attention_weight_max": 46.1844367980957,
      "activations/layer13_attention_weight_min": -35.0899658203125,
      "activations/layer14_attention_weight_max": 51.615116119384766,
      "activations/layer14_attention_weight_min": -37.70668411254883,
      "activations/layer15_attention_weight_max": 46.01572036743164,
      "activations/layer15_attention_weight_min": -35.83760452270508,
      "activations/layer16_attention_weight_max": 37.11866760253906,
      "activations/layer16_attention_weight_min": -29.37015151977539,
      "activations/layer17_attention_weight_max": 35.3432731628418,
      "activations/layer17_attention_weight_min": -26.523324966430664,
      "activations/layer18_attention_weight_max": 37.288692474365234,
      "activations/layer18_attention_weight_min": -25.87706756591797,
      "activations/layer19_attention_weight_max": 39.21365737915039,
      "activations/layer19_attention_weight_min": -30.910673141479492,
      "activations/layer1_attention_weight_max": 15.814094543457031,
      "activations/layer1_attention_weight_min": -14.574517250061035,
      "activations/layer20_attention_weight_max": 34.85346221923828,
      "activations/layer20_attention_weight_min": -25.267637252807617,
      "activations/layer21_attention_weight_max": 35.866302490234375,
      "activations/layer21_attention_weight_min": -25.57305335998535,
      "activations/layer22_attention_weight_max": 50.948917388916016,
      "activations/layer22_attention_weight_min": -31.077207565307617,
      "activations/layer23_attention_weight_max": 37.96105194091797,
      "activations/layer23_attention_weight_min": -26.519840240478516,
      "activations/layer2_attention_weight_max": 33.18006896972656,
      "activations/layer2_attention_weight_min": -32.08176040649414,
      "activations/layer3_attention_weight_max": 96.86438751220703,
      "activations/layer3_attention_weight_min": -97.78849029541016,
      "activations/layer4_attention_weight_max": 107.7889633178711,
      "activations/layer4_attention_weight_min": -110.47804260253906,
      "activations/layer5_attention_weight_max": 51.99404525756836,
      "activations/layer5_attention_weight_min": -62.27842712402344,
      "activations/layer6_attention_weight_max": 43.25167465209961,
      "activations/layer6_attention_weight_min": -46.4677848815918,
      "activations/layer7_attention_weight_max": 89.57291412353516,
      "activations/layer7_attention_weight_min": -88.78968811035156,
      "activations/layer8_attention_weight_max": 42.47761154174805,
      "activations/layer8_attention_weight_min": -45.91743469238281,
      "activations/layer9_attention_weight_max": 32.18770980834961,
      "activations/layer9_attention_weight_min": -33.99874496459961,
      "epoch": 14.91,
      "learning_rate": 5.441628787878787e-05,
      "loss": 2.7394,
      "step": 256550
    },
    {
      "activations/layer0_attention_weight_max": 16.314964294433594,
      "activations/layer0_attention_weight_min": -11.84807300567627,
      "activations/layer10_attention_weight_max": 35.190155029296875,
      "activations/layer10_attention_weight_min": -34.80290603637695,
      "activations/layer11_attention_weight_max": 34.484004974365234,
      "activations/layer11_attention_weight_min": -37.80615234375,
      "activations/layer12_attention_weight_max": 26.93462371826172,
      "activations/layer12_attention_weight_min": -27.577041625976562,
      "activations/layer13_attention_weight_max": 39.29635238647461,
      "activations/layer13_attention_weight_min": -36.49665832519531,
      "activations/layer14_attention_weight_max": 45.421966552734375,
      "activations/layer14_attention_weight_min": -40.101016998291016,
      "activations/layer15_attention_weight_max": 37.730323791503906,
      "activations/layer15_attention_weight_min": -34.85821533203125,
      "activations/layer16_attention_weight_max": 29.544395446777344,
      "activations/layer16_attention_weight_min": -26.980228424072266,
      "activations/layer17_attention_weight_max": 29.28995704650879,
      "activations/layer17_attention_weight_min": -27.014728546142578,
      "activations/layer18_attention_weight_max": 33.19292068481445,
      "activations/layer18_attention_weight_min": -25.654272079467773,
      "activations/layer19_attention_weight_max": 34.25450134277344,
      "activations/layer19_attention_weight_min": -30.67533302307129,
      "activations/layer1_attention_weight_max": 16.07017707824707,
      "activations/layer1_attention_weight_min": -14.099029541015625,
      "activations/layer20_attention_weight_max": 30.924348831176758,
      "activations/layer20_attention_weight_min": -24.633182525634766,
      "activations/layer21_attention_weight_max": 32.00386047363281,
      "activations/layer21_attention_weight_min": -23.624650955200195,
      "activations/layer22_attention_weight_max": 46.21773147583008,
      "activations/layer22_attention_weight_min": -32.4374885559082,
      "activations/layer23_attention_weight_max": 34.17498016357422,
      "activations/layer23_attention_weight_min": -27.654132843017578,
      "activations/layer2_attention_weight_max": 31.384220123291016,
      "activations/layer2_attention_weight_min": -31.765182495117188,
      "activations/layer3_attention_weight_max": 92.31197357177734,
      "activations/layer3_attention_weight_min": -96.45204162597656,
      "activations/layer4_attention_weight_max": 110.99623107910156,
      "activations/layer4_attention_weight_min": -113.3287353515625,
      "activations/layer5_attention_weight_max": 56.548126220703125,
      "activations/layer5_attention_weight_min": -65.17539978027344,
      "activations/layer6_attention_weight_max": 44.454341888427734,
      "activations/layer6_attention_weight_min": -50.421329498291016,
      "activations/layer7_attention_weight_max": 98.13321685791016,
      "activations/layer7_attention_weight_min": -101.8443832397461,
      "activations/layer8_attention_weight_max": 44.33220672607422,
      "activations/layer8_attention_weight_min": -43.95674133300781,
      "activations/layer9_attention_weight_max": 32.343257904052734,
      "activations/layer9_attention_weight_min": -33.70185470581055,
      "epoch": 14.91,
      "learning_rate": 5.439734848484848e-05,
      "loss": 2.7589,
      "step": 256600
    },
    {
      "activations/layer0_attention_weight_max": 16.188796997070312,
      "activations/layer0_attention_weight_min": -12.203166961669922,
      "activations/layer10_attention_weight_max": 31.75210952758789,
      "activations/layer10_attention_weight_min": -32.98295593261719,
      "activations/layer11_attention_weight_max": 30.733600616455078,
      "activations/layer11_attention_weight_min": -32.13806915283203,
      "activations/layer12_attention_weight_max": 24.974868774414062,
      "activations/layer12_attention_weight_min": -24.646602630615234,
      "activations/layer13_attention_weight_max": 37.40930938720703,
      "activations/layer13_attention_weight_min": -33.27953338623047,
      "activations/layer14_attention_weight_max": 41.212364196777344,
      "activations/layer14_attention_weight_min": -35.42473220825195,
      "activations/layer15_attention_weight_max": 37.800899505615234,
      "activations/layer15_attention_weight_min": -34.24452209472656,
      "activations/layer16_attention_weight_max": 28.549055099487305,
      "activations/layer16_attention_weight_min": -26.94146728515625,
      "activations/layer17_attention_weight_max": 32.51414108276367,
      "activations/layer17_attention_weight_min": -26.10911750793457,
      "activations/layer18_attention_weight_max": 33.30501937866211,
      "activations/layer18_attention_weight_min": -25.166296005249023,
      "activations/layer19_attention_weight_max": 32.07664108276367,
      "activations/layer19_attention_weight_min": -31.3682918548584,
      "activations/layer1_attention_weight_max": 16.422260284423828,
      "activations/layer1_attention_weight_min": -14.237762451171875,
      "activations/layer20_attention_weight_max": 30.19261360168457,
      "activations/layer20_attention_weight_min": -25.09868812561035,
      "activations/layer21_attention_weight_max": 28.863739013671875,
      "activations/layer21_attention_weight_min": -25.49869728088379,
      "activations/layer22_attention_weight_max": 42.26565933227539,
      "activations/layer22_attention_weight_min": -32.35591506958008,
      "activations/layer23_attention_weight_max": 33.43076705932617,
      "activations/layer23_attention_weight_min": -26.65201187133789,
      "activations/layer2_attention_weight_max": 30.35739517211914,
      "activations/layer2_attention_weight_min": -31.086971282958984,
      "activations/layer3_attention_weight_max": 90.071044921875,
      "activations/layer3_attention_weight_min": -92.40377044677734,
      "activations/layer4_attention_weight_max": 105.4488296508789,
      "activations/layer4_attention_weight_min": -109.84979248046875,
      "activations/layer5_attention_weight_max": 53.21891784667969,
      "activations/layer5_attention_weight_min": -61.73080062866211,
      "activations/layer6_attention_weight_max": 42.66243362426758,
      "activations/layer6_attention_weight_min": -45.51674270629883,
      "activations/layer7_attention_weight_max": 89.70234680175781,
      "activations/layer7_attention_weight_min": -88.13900756835938,
      "activations/layer8_attention_weight_max": 40.33892059326172,
      "activations/layer8_attention_weight_min": -40.6683235168457,
      "activations/layer9_attention_weight_max": 30.82999038696289,
      "activations/layer9_attention_weight_min": -32.985939025878906,
      "epoch": 14.91,
      "learning_rate": 5.4378409090909084e-05,
      "loss": 2.7678,
      "step": 256650
    },
    {
      "activations/layer0_attention_weight_max": 15.949320793151855,
      "activations/layer0_attention_weight_min": -12.063933372497559,
      "activations/layer10_attention_weight_max": 38.70527267456055,
      "activations/layer10_attention_weight_min": -34.19276809692383,
      "activations/layer11_attention_weight_max": 40.06298828125,
      "activations/layer11_attention_weight_min": -36.81206512451172,
      "activations/layer12_attention_weight_max": 25.570261001586914,
      "activations/layer12_attention_weight_min": -26.69402503967285,
      "activations/layer13_attention_weight_max": 36.41172790527344,
      "activations/layer13_attention_weight_min": -36.57977294921875,
      "activations/layer14_attention_weight_max": 40.57823181152344,
      "activations/layer14_attention_weight_min": -38.7314338684082,
      "activations/layer15_attention_weight_max": 34.51750183105469,
      "activations/layer15_attention_weight_min": -34.7894287109375,
      "activations/layer16_attention_weight_max": 30.29581642150879,
      "activations/layer16_attention_weight_min": -27.48564910888672,
      "activations/layer17_attention_weight_max": 29.175777435302734,
      "activations/layer17_attention_weight_min": -25.635644912719727,
      "activations/layer18_attention_weight_max": 32.61817169189453,
      "activations/layer18_attention_weight_min": -27.360673904418945,
      "activations/layer19_attention_weight_max": 37.24897003173828,
      "activations/layer19_attention_weight_min": -31.240015029907227,
      "activations/layer1_attention_weight_max": 16.207368850708008,
      "activations/layer1_attention_weight_min": -13.719993591308594,
      "activations/layer20_attention_weight_max": 30.788122177124023,
      "activations/layer20_attention_weight_min": -23.333656311035156,
      "activations/layer21_attention_weight_max": 28.643606185913086,
      "activations/layer21_attention_weight_min": -26.79705047607422,
      "activations/layer22_attention_weight_max": 42.842430114746094,
      "activations/layer22_attention_weight_min": -32.622474670410156,
      "activations/layer23_attention_weight_max": 29.223506927490234,
      "activations/layer23_attention_weight_min": -26.741268157958984,
      "activations/layer2_attention_weight_max": 30.724933624267578,
      "activations/layer2_attention_weight_min": -29.552536010742188,
      "activations/layer3_attention_weight_max": 84.36627197265625,
      "activations/layer3_attention_weight_min": -94.2475814819336,
      "activations/layer4_attention_weight_max": 102.7177963256836,
      "activations/layer4_attention_weight_min": -102.5509033203125,
      "activations/layer5_attention_weight_max": 49.89150619506836,
      "activations/layer5_attention_weight_min": -58.3640251159668,
      "activations/layer6_attention_weight_max": 39.84613037109375,
      "activations/layer6_attention_weight_min": -44.018062591552734,
      "activations/layer7_attention_weight_max": 88.43099975585938,
      "activations/layer7_attention_weight_min": -87.72423553466797,
      "activations/layer8_attention_weight_max": 41.024436950683594,
      "activations/layer8_attention_weight_min": -38.34324645996094,
      "activations/layer9_attention_weight_max": 32.99354553222656,
      "activations/layer9_attention_weight_min": -33.934261322021484,
      "epoch": 14.92,
      "learning_rate": 5.435946969696969e-05,
      "loss": 2.7438,
      "step": 256700
    },
    {
      "activations/layer0_attention_weight_max": 14.978679656982422,
      "activations/layer0_attention_weight_min": -12.077180862426758,
      "activations/layer10_attention_weight_max": 32.83318328857422,
      "activations/layer10_attention_weight_min": -34.54426574707031,
      "activations/layer11_attention_weight_max": 33.72116470336914,
      "activations/layer11_attention_weight_min": -33.056602478027344,
      "activations/layer12_attention_weight_max": 27.670198440551758,
      "activations/layer12_attention_weight_min": -29.618135452270508,
      "activations/layer13_attention_weight_max": 45.51268768310547,
      "activations/layer13_attention_weight_min": -36.01725769042969,
      "activations/layer14_attention_weight_max": 46.254981994628906,
      "activations/layer14_attention_weight_min": -38.96830368041992,
      "activations/layer15_attention_weight_max": 43.260780334472656,
      "activations/layer15_attention_weight_min": -36.32574462890625,
      "activations/layer16_attention_weight_max": 29.317094802856445,
      "activations/layer16_attention_weight_min": -27.57351303100586,
      "activations/layer17_attention_weight_max": 32.33474349975586,
      "activations/layer17_attention_weight_min": -24.79491424560547,
      "activations/layer18_attention_weight_max": 31.04578971862793,
      "activations/layer18_attention_weight_min": -24.328121185302734,
      "activations/layer19_attention_weight_max": 31.056772232055664,
      "activations/layer19_attention_weight_min": -30.866174697875977,
      "activations/layer1_attention_weight_max": 16.109331130981445,
      "activations/layer1_attention_weight_min": -14.741512298583984,
      "activations/layer20_attention_weight_max": 29.203561782836914,
      "activations/layer20_attention_weight_min": -24.412281036376953,
      "activations/layer21_attention_weight_max": 30.025405883789062,
      "activations/layer21_attention_weight_min": -24.283817291259766,
      "activations/layer22_attention_weight_max": 42.56792068481445,
      "activations/layer22_attention_weight_min": -30.67148780822754,
      "activations/layer23_attention_weight_max": 32.67329025268555,
      "activations/layer23_attention_weight_min": -26.84392547607422,
      "activations/layer2_attention_weight_max": 30.696903228759766,
      "activations/layer2_attention_weight_min": -30.316818237304688,
      "activations/layer3_attention_weight_max": 85.73082733154297,
      "activations/layer3_attention_weight_min": -90.01990509033203,
      "activations/layer4_attention_weight_max": 102.81185150146484,
      "activations/layer4_attention_weight_min": -103.4963150024414,
      "activations/layer5_attention_weight_max": 50.37655258178711,
      "activations/layer5_attention_weight_min": -58.93578338623047,
      "activations/layer6_attention_weight_max": 41.424232482910156,
      "activations/layer6_attention_weight_min": -44.50984191894531,
      "activations/layer7_attention_weight_max": 88.84540557861328,
      "activations/layer7_attention_weight_min": -91.94464874267578,
      "activations/layer8_attention_weight_max": 40.51375961303711,
      "activations/layer8_attention_weight_min": -41.95215606689453,
      "activations/layer9_attention_weight_max": 32.2800178527832,
      "activations/layer9_attention_weight_min": -33.0645866394043,
      "epoch": 14.92,
      "learning_rate": 5.43405303030303e-05,
      "loss": 2.7577,
      "step": 256750
    },
    {
      "activations/layer0_attention_weight_max": 15.859289169311523,
      "activations/layer0_attention_weight_min": -12.518875122070312,
      "activations/layer10_attention_weight_max": 35.848628997802734,
      "activations/layer10_attention_weight_min": -34.4302978515625,
      "activations/layer11_attention_weight_max": 34.730499267578125,
      "activations/layer11_attention_weight_min": -35.99409484863281,
      "activations/layer12_attention_weight_max": 28.11504364013672,
      "activations/layer12_attention_weight_min": -28.640810012817383,
      "activations/layer13_attention_weight_max": 44.7564582824707,
      "activations/layer13_attention_weight_min": -39.404090881347656,
      "activations/layer14_attention_weight_max": 46.4304313659668,
      "activations/layer14_attention_weight_min": -42.28225326538086,
      "activations/layer15_attention_weight_max": 42.918540954589844,
      "activations/layer15_attention_weight_min": -37.6528205871582,
      "activations/layer16_attention_weight_max": 32.755775451660156,
      "activations/layer16_attention_weight_min": -29.49861717224121,
      "activations/layer17_attention_weight_max": 32.3582763671875,
      "activations/layer17_attention_weight_min": -27.186410903930664,
      "activations/layer18_attention_weight_max": 31.854122161865234,
      "activations/layer18_attention_weight_min": -26.139659881591797,
      "activations/layer19_attention_weight_max": 37.17264938354492,
      "activations/layer19_attention_weight_min": -33.47434616088867,
      "activations/layer1_attention_weight_max": 15.828936576843262,
      "activations/layer1_attention_weight_min": -14.168622016906738,
      "activations/layer20_attention_weight_max": 30.765729904174805,
      "activations/layer20_attention_weight_min": -26.318418502807617,
      "activations/layer21_attention_weight_max": 29.577348709106445,
      "activations/layer21_attention_weight_min": -24.780710220336914,
      "activations/layer22_attention_weight_max": 47.33313751220703,
      "activations/layer22_attention_weight_min": -31.332624435424805,
      "activations/layer23_attention_weight_max": 34.56774139404297,
      "activations/layer23_attention_weight_min": -26.730756759643555,
      "activations/layer2_attention_weight_max": 31.52196502685547,
      "activations/layer2_attention_weight_min": -30.955276489257812,
      "activations/layer3_attention_weight_max": 87.28665161132812,
      "activations/layer3_attention_weight_min": -88.52639770507812,
      "activations/layer4_attention_weight_max": 100.95781707763672,
      "activations/layer4_attention_weight_min": -108.51409912109375,
      "activations/layer5_attention_weight_max": 50.152931213378906,
      "activations/layer5_attention_weight_min": -59.387325286865234,
      "activations/layer6_attention_weight_max": 44.56062698364258,
      "activations/layer6_attention_weight_min": -45.75229263305664,
      "activations/layer7_attention_weight_max": 98.56926727294922,
      "activations/layer7_attention_weight_min": -95.91197204589844,
      "activations/layer8_attention_weight_max": 44.107765197753906,
      "activations/layer8_attention_weight_min": -45.19633865356445,
      "activations/layer9_attention_weight_max": 34.53340530395508,
      "activations/layer9_attention_weight_min": -34.20918655395508,
      "epoch": 14.92,
      "learning_rate": 5.432159090909091e-05,
      "loss": 2.7548,
      "step": 256800
    },
    {
      "activations/layer0_attention_weight_max": 16.160232543945312,
      "activations/layer0_attention_weight_min": -12.206412315368652,
      "activations/layer10_attention_weight_max": 34.05170440673828,
      "activations/layer10_attention_weight_min": -33.608829498291016,
      "activations/layer11_attention_weight_max": 36.36527633666992,
      "activations/layer11_attention_weight_min": -33.610477447509766,
      "activations/layer12_attention_weight_max": 24.84815216064453,
      "activations/layer12_attention_weight_min": -27.648698806762695,
      "activations/layer13_attention_weight_max": 38.09585189819336,
      "activations/layer13_attention_weight_min": -35.540897369384766,
      "activations/layer14_attention_weight_max": 40.18515396118164,
      "activations/layer14_attention_weight_min": -38.1209602355957,
      "activations/layer15_attention_weight_max": 38.643924713134766,
      "activations/layer15_attention_weight_min": -36.31203842163086,
      "activations/layer16_attention_weight_max": 31.123140335083008,
      "activations/layer16_attention_weight_min": -29.87948989868164,
      "activations/layer17_attention_weight_max": 31.43072509765625,
      "activations/layer17_attention_weight_min": -25.619333267211914,
      "activations/layer18_attention_weight_max": 33.47712707519531,
      "activations/layer18_attention_weight_min": -25.087772369384766,
      "activations/layer19_attention_weight_max": 37.819583892822266,
      "activations/layer19_attention_weight_min": -32.15725326538086,
      "activations/layer1_attention_weight_max": 16.34368133544922,
      "activations/layer1_attention_weight_min": -14.172102928161621,
      "activations/layer20_attention_weight_max": 31.038101196289062,
      "activations/layer20_attention_weight_min": -25.264781951904297,
      "activations/layer21_attention_weight_max": 30.42230796813965,
      "activations/layer21_attention_weight_min": -25.729143142700195,
      "activations/layer22_attention_weight_max": 44.92731475830078,
      "activations/layer22_attention_weight_min": -30.86395835876465,
      "activations/layer23_attention_weight_max": 34.89289855957031,
      "activations/layer23_attention_weight_min": -27.619060516357422,
      "activations/layer2_attention_weight_max": 33.942848205566406,
      "activations/layer2_attention_weight_min": -32.43372344970703,
      "activations/layer3_attention_weight_max": 95.46308135986328,
      "activations/layer3_attention_weight_min": -95.96051025390625,
      "activations/layer4_attention_weight_max": 106.1429672241211,
      "activations/layer4_attention_weight_min": -112.2900161743164,
      "activations/layer5_attention_weight_max": 54.632503509521484,
      "activations/layer5_attention_weight_min": -64.56820678710938,
      "activations/layer6_attention_weight_max": 42.85435485839844,
      "activations/layer6_attention_weight_min": -45.33097839355469,
      "activations/layer7_attention_weight_max": 92.57875061035156,
      "activations/layer7_attention_weight_min": -89.42161560058594,
      "activations/layer8_attention_weight_max": 42.39150619506836,
      "activations/layer8_attention_weight_min": -43.2287483215332,
      "activations/layer9_attention_weight_max": 31.432355880737305,
      "activations/layer9_attention_weight_min": -34.14217758178711,
      "epoch": 14.92,
      "learning_rate": 5.430265151515151e-05,
      "loss": 2.7534,
      "step": 256850
    },
    {
      "activations/layer0_attention_weight_max": 17.148731231689453,
      "activations/layer0_attention_weight_min": -12.784027099609375,
      "activations/layer10_attention_weight_max": 33.15220642089844,
      "activations/layer10_attention_weight_min": -32.254520416259766,
      "activations/layer11_attention_weight_max": 36.00704574584961,
      "activations/layer11_attention_weight_min": -33.81022262573242,
      "activations/layer12_attention_weight_max": 26.226957321166992,
      "activations/layer12_attention_weight_min": -25.131587982177734,
      "activations/layer13_attention_weight_max": 39.07029342651367,
      "activations/layer13_attention_weight_min": -32.706356048583984,
      "activations/layer14_attention_weight_max": 47.218143463134766,
      "activations/layer14_attention_weight_min": -37.03080749511719,
      "activations/layer15_attention_weight_max": 38.22697448730469,
      "activations/layer15_attention_weight_min": -34.86964797973633,
      "activations/layer16_attention_weight_max": 30.281078338623047,
      "activations/layer16_attention_weight_min": -28.02232551574707,
      "activations/layer17_attention_weight_max": 33.9128303527832,
      "activations/layer17_attention_weight_min": -25.629505157470703,
      "activations/layer18_attention_weight_max": 33.33332061767578,
      "activations/layer18_attention_weight_min": -24.556629180908203,
      "activations/layer19_attention_weight_max": 34.485103607177734,
      "activations/layer19_attention_weight_min": -29.533706665039062,
      "activations/layer1_attention_weight_max": 16.891023635864258,
      "activations/layer1_attention_weight_min": -14.120186805725098,
      "activations/layer20_attention_weight_max": 31.608596801757812,
      "activations/layer20_attention_weight_min": -24.231233596801758,
      "activations/layer21_attention_weight_max": 31.21700096130371,
      "activations/layer21_attention_weight_min": -23.950897216796875,
      "activations/layer22_attention_weight_max": 45.48147964477539,
      "activations/layer22_attention_weight_min": -31.12296485900879,
      "activations/layer23_attention_weight_max": 33.590362548828125,
      "activations/layer23_attention_weight_min": -25.8236083984375,
      "activations/layer2_attention_weight_max": 33.011817932128906,
      "activations/layer2_attention_weight_min": -31.91041374206543,
      "activations/layer3_attention_weight_max": 93.60762023925781,
      "activations/layer3_attention_weight_min": -94.30953216552734,
      "activations/layer4_attention_weight_max": 105.42408752441406,
      "activations/layer4_attention_weight_min": -108.5352783203125,
      "activations/layer5_attention_weight_max": 52.21662902832031,
      "activations/layer5_attention_weight_min": -63.25055694580078,
      "activations/layer6_attention_weight_max": 42.3383903503418,
      "activations/layer6_attention_weight_min": -46.71866226196289,
      "activations/layer7_attention_weight_max": 85.5019302368164,
      "activations/layer7_attention_weight_min": -90.22634887695312,
      "activations/layer8_attention_weight_max": 38.7768440246582,
      "activations/layer8_attention_weight_min": -41.068626403808594,
      "activations/layer9_attention_weight_max": 31.568029403686523,
      "activations/layer9_attention_weight_min": -33.17991256713867,
      "epoch": 14.93,
      "learning_rate": 5.428371212121211e-05,
      "loss": 2.7595,
      "step": 256900
    },
    {
      "activations/layer0_attention_weight_max": 15.71279239654541,
      "activations/layer0_attention_weight_min": -11.219504356384277,
      "activations/layer10_attention_weight_max": 35.83169937133789,
      "activations/layer10_attention_weight_min": -33.056461334228516,
      "activations/layer11_attention_weight_max": 35.22186279296875,
      "activations/layer11_attention_weight_min": -34.59820556640625,
      "activations/layer12_attention_weight_max": 29.670377731323242,
      "activations/layer12_attention_weight_min": -27.275527954101562,
      "activations/layer13_attention_weight_max": 45.2745246887207,
      "activations/layer13_attention_weight_min": -35.18479919433594,
      "activations/layer14_attention_weight_max": 47.95438003540039,
      "activations/layer14_attention_weight_min": -39.030303955078125,
      "activations/layer15_attention_weight_max": 43.33218765258789,
      "activations/layer15_attention_weight_min": -37.847110748291016,
      "activations/layer16_attention_weight_max": 33.530357360839844,
      "activations/layer16_attention_weight_min": -28.36652946472168,
      "activations/layer17_attention_weight_max": 33.09144973754883,
      "activations/layer17_attention_weight_min": -26.87790870666504,
      "activations/layer18_attention_weight_max": 29.383712768554688,
      "activations/layer18_attention_weight_min": -25.736671447753906,
      "activations/layer19_attention_weight_max": 40.04798889160156,
      "activations/layer19_attention_weight_min": -35.72574996948242,
      "activations/layer1_attention_weight_max": 16.61347198486328,
      "activations/layer1_attention_weight_min": -15.814071655273438,
      "activations/layer20_attention_weight_max": 29.874515533447266,
      "activations/layer20_attention_weight_min": -24.655811309814453,
      "activations/layer21_attention_weight_max": 27.1959285736084,
      "activations/layer21_attention_weight_min": -24.097301483154297,
      "activations/layer22_attention_weight_max": 40.80867004394531,
      "activations/layer22_attention_weight_min": -32.75126266479492,
      "activations/layer23_attention_weight_max": 35.41223907470703,
      "activations/layer23_attention_weight_min": -27.182395935058594,
      "activations/layer2_attention_weight_max": 32.89964294433594,
      "activations/layer2_attention_weight_min": -32.27855682373047,
      "activations/layer3_attention_weight_max": 100.43710327148438,
      "activations/layer3_attention_weight_min": -99.23506164550781,
      "activations/layer4_attention_weight_max": 115.03041076660156,
      "activations/layer4_attention_weight_min": -114.48186492919922,
      "activations/layer5_attention_weight_max": 55.81590270996094,
      "activations/layer5_attention_weight_min": -64.22789001464844,
      "activations/layer6_attention_weight_max": 46.802162170410156,
      "activations/layer6_attention_weight_min": -50.33287811279297,
      "activations/layer7_attention_weight_max": 95.08953857421875,
      "activations/layer7_attention_weight_min": -104.63628387451172,
      "activations/layer8_attention_weight_max": 43.75057601928711,
      "activations/layer8_attention_weight_min": -42.81570816040039,
      "activations/layer9_attention_weight_max": 34.34451675415039,
      "activations/layer9_attention_weight_min": -34.41362380981445,
      "epoch": 14.93,
      "learning_rate": 5.426477272727273e-05,
      "loss": 2.7511,
      "step": 256950
    },
    {
      "activations/layer0_attention_weight_max": 14.523303985595703,
      "activations/layer0_attention_weight_min": -11.697537422180176,
      "activations/layer10_attention_weight_max": 32.55467987060547,
      "activations/layer10_attention_weight_min": -33.09531784057617,
      "activations/layer11_attention_weight_max": 32.393428802490234,
      "activations/layer11_attention_weight_min": -33.7957763671875,
      "activations/layer12_attention_weight_max": 28.07624053955078,
      "activations/layer12_attention_weight_min": -27.792375564575195,
      "activations/layer13_attention_weight_max": 43.22178268432617,
      "activations/layer13_attention_weight_min": -38.419490814208984,
      "activations/layer14_attention_weight_max": 49.05067443847656,
      "activations/layer14_attention_weight_min": -41.52474594116211,
      "activations/layer15_attention_weight_max": 42.285858154296875,
      "activations/layer15_attention_weight_min": -38.14638137817383,
      "activations/layer16_attention_weight_max": 33.48286437988281,
      "activations/layer16_attention_weight_min": -29.50944709777832,
      "activations/layer17_attention_weight_max": 33.11928176879883,
      "activations/layer17_attention_weight_min": -27.257959365844727,
      "activations/layer18_attention_weight_max": 38.53618240356445,
      "activations/layer18_attention_weight_min": -27.620615005493164,
      "activations/layer19_attention_weight_max": 41.437068939208984,
      "activations/layer19_attention_weight_min": -32.37038803100586,
      "activations/layer1_attention_weight_max": 15.703781127929688,
      "activations/layer1_attention_weight_min": -15.669259071350098,
      "activations/layer20_attention_weight_max": 32.39641571044922,
      "activations/layer20_attention_weight_min": -26.030183792114258,
      "activations/layer21_attention_weight_max": 33.924293518066406,
      "activations/layer21_attention_weight_min": -25.375255584716797,
      "activations/layer22_attention_weight_max": 50.7563362121582,
      "activations/layer22_attention_weight_min": -33.367286682128906,
      "activations/layer23_attention_weight_max": 35.96297836303711,
      "activations/layer23_attention_weight_min": -28.216482162475586,
      "activations/layer2_attention_weight_max": 32.93353271484375,
      "activations/layer2_attention_weight_min": -32.78113555908203,
      "activations/layer3_attention_weight_max": 99.61341094970703,
      "activations/layer3_attention_weight_min": -95.79073333740234,
      "activations/layer4_attention_weight_max": 110.83928680419922,
      "activations/layer4_attention_weight_min": -112.99376678466797,
      "activations/layer5_attention_weight_max": 54.86253356933594,
      "activations/layer5_attention_weight_min": -63.5608024597168,
      "activations/layer6_attention_weight_max": 44.15421676635742,
      "activations/layer6_attention_weight_min": -47.055294036865234,
      "activations/layer7_attention_weight_max": 90.77686309814453,
      "activations/layer7_attention_weight_min": -90.8651351928711,
      "activations/layer8_attention_weight_max": 39.50230026245117,
      "activations/layer8_attention_weight_min": -41.358768463134766,
      "activations/layer9_attention_weight_max": 31.540014266967773,
      "activations/layer9_attention_weight_min": -35.073463439941406,
      "epoch": 14.93,
      "learning_rate": 5.424583333333333e-05,
      "loss": 2.7594,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.5097,
      "eval_samples_per_second": 504.601,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.5097,
      "eval_openwebtext_samples_per_second": 504.601,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 2.0157,
      "eval_wikitext_samples_per_second": 226.222,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_lambada_loss": 2.369140625,
      "eval_lambada_ppl": 10.688203161633492,
      "eval_lambada_runtime": 9.6062,
      "eval_lambada_samples_per_second": 506.86,
      "step": 257000
    },
    {
      "activations/layer0_attention_weight_max": 15.133365631103516,
      "activations/layer0_attention_weight_min": -12.481541633605957,
      "activations/layer10_attention_weight_max": 32.72431564331055,
      "activations/layer10_attention_weight_min": -33.26726150512695,
      "activations/layer11_attention_weight_max": 31.193397521972656,
      "activations/layer11_attention_weight_min": -31.589107513427734,
      "activations/layer12_attention_weight_max": 26.978900909423828,
      "activations/layer12_attention_weight_min": -25.277984619140625,
      "activations/layer13_attention_weight_max": 37.09257888793945,
      "activations/layer13_attention_weight_min": -35.101287841796875,
      "activations/layer14_attention_weight_max": 41.26514434814453,
      "activations/layer14_attention_weight_min": -37.804805755615234,
      "activations/layer15_attention_weight_max": 37.603004455566406,
      "activations/layer15_attention_weight_min": -37.55146789550781,
      "activations/layer16_attention_weight_max": 28.172847747802734,
      "activations/layer16_attention_weight_min": -28.292709350585938,
      "activations/layer17_attention_weight_max": 30.624475479125977,
      "activations/layer17_attention_weight_min": -24.95493507385254,
      "activations/layer18_attention_weight_max": 31.675514221191406,
      "activations/layer18_attention_weight_min": -25.524364471435547,
      "activations/layer19_attention_weight_max": 31.713241577148438,
      "activations/layer19_attention_weight_min": -35.82263946533203,
      "activations/layer1_attention_weight_max": 17.251644134521484,
      "activations/layer1_attention_weight_min": -14.705853462219238,
      "activations/layer20_attention_weight_max": 26.570144653320312,
      "activations/layer20_attention_weight_min": -26.2158203125,
      "activations/layer21_attention_weight_max": 26.50944709777832,
      "activations/layer21_attention_weight_min": -25.73933982849121,
      "activations/layer22_attention_weight_max": 39.53670883178711,
      "activations/layer22_attention_weight_min": -32.911293029785156,
      "activations/layer23_attention_weight_max": 32.90200424194336,
      "activations/layer23_attention_weight_min": -26.002180099487305,
      "activations/layer2_attention_weight_max": 31.99979019165039,
      "activations/layer2_attention_weight_min": -31.253768920898438,
      "activations/layer3_attention_weight_max": 94.75163269042969,
      "activations/layer3_attention_weight_min": -93.574462890625,
      "activations/layer4_attention_weight_max": 110.78932189941406,
      "activations/layer4_attention_weight_min": -112.35729217529297,
      "activations/layer5_attention_weight_max": 54.17876434326172,
      "activations/layer5_attention_weight_min": -60.63924789428711,
      "activations/layer6_attention_weight_max": 44.26368713378906,
      "activations/layer6_attention_weight_min": -47.48165512084961,
      "activations/layer7_attention_weight_max": 92.65381622314453,
      "activations/layer7_attention_weight_min": -90.19697570800781,
      "activations/layer8_attention_weight_max": 40.38880157470703,
      "activations/layer8_attention_weight_min": -40.47258377075195,
      "activations/layer9_attention_weight_max": 32.0319938659668,
      "activations/layer9_attention_weight_min": -32.41316223144531,
      "epoch": 14.94,
      "learning_rate": 5.4226893939393937e-05,
      "loss": 2.7666,
      "step": 257050
    },
    {
      "activations/layer0_attention_weight_max": 16.058855056762695,
      "activations/layer0_attention_weight_min": -11.74105453491211,
      "activations/layer10_attention_weight_max": 36.7781982421875,
      "activations/layer10_attention_weight_min": -34.17527389526367,
      "activations/layer11_attention_weight_max": 38.3780517578125,
      "activations/layer11_attention_weight_min": -34.3607177734375,
      "activations/layer12_attention_weight_max": 27.184249877929688,
      "activations/layer12_attention_weight_min": -25.279132843017578,
      "activations/layer13_attention_weight_max": 44.07791519165039,
      "activations/layer13_attention_weight_min": -33.461936950683594,
      "activations/layer14_attention_weight_max": 43.05268096923828,
      "activations/layer14_attention_weight_min": -36.69310760498047,
      "activations/layer15_attention_weight_max": 41.446815490722656,
      "activations/layer15_attention_weight_min": -34.09333038330078,
      "activations/layer16_attention_weight_max": 34.5348014831543,
      "activations/layer16_attention_weight_min": -27.67593765258789,
      "activations/layer17_attention_weight_max": 33.03070068359375,
      "activations/layer17_attention_weight_min": -26.865375518798828,
      "activations/layer18_attention_weight_max": 37.78351593017578,
      "activations/layer18_attention_weight_min": -26.2139949798584,
      "activations/layer19_attention_weight_max": 36.87417984008789,
      "activations/layer19_attention_weight_min": -33.71100997924805,
      "activations/layer1_attention_weight_max": 17.1285400390625,
      "activations/layer1_attention_weight_min": -14.935227394104004,
      "activations/layer20_attention_weight_max": 34.49676513671875,
      "activations/layer20_attention_weight_min": -29.72994041442871,
      "activations/layer21_attention_weight_max": 32.85559844970703,
      "activations/layer21_attention_weight_min": -27.41786766052246,
      "activations/layer22_attention_weight_max": 46.68440246582031,
      "activations/layer22_attention_weight_min": -34.42933654785156,
      "activations/layer23_attention_weight_max": 36.23015213012695,
      "activations/layer23_attention_weight_min": -26.782257080078125,
      "activations/layer2_attention_weight_max": 31.311473846435547,
      "activations/layer2_attention_weight_min": -30.60858154296875,
      "activations/layer3_attention_weight_max": 91.58624267578125,
      "activations/layer3_attention_weight_min": -91.614990234375,
      "activations/layer4_attention_weight_max": 107.8028564453125,
      "activations/layer4_attention_weight_min": -109.33946990966797,
      "activations/layer5_attention_weight_max": 51.454322814941406,
      "activations/layer5_attention_weight_min": -64.23934936523438,
      "activations/layer6_attention_weight_max": 42.24703598022461,
      "activations/layer6_attention_weight_min": -45.875816345214844,
      "activations/layer7_attention_weight_max": 102.12666320800781,
      "activations/layer7_attention_weight_min": -97.942626953125,
      "activations/layer8_attention_weight_max": 42.14824676513672,
      "activations/layer8_attention_weight_min": -41.04514694213867,
      "activations/layer9_attention_weight_max": 34.84479904174805,
      "activations/layer9_attention_weight_min": -33.16047668457031,
      "epoch": 14.94,
      "learning_rate": 5.420795454545454e-05,
      "loss": 2.7412,
      "step": 257100
    },
    {
      "activations/layer0_attention_weight_max": 16.698434829711914,
      "activations/layer0_attention_weight_min": -12.698193550109863,
      "activations/layer10_attention_weight_max": 31.668540954589844,
      "activations/layer10_attention_weight_min": -32.651424407958984,
      "activations/layer11_attention_weight_max": 31.97663116455078,
      "activations/layer11_attention_weight_min": -32.75002670288086,
      "activations/layer12_attention_weight_max": 26.318927764892578,
      "activations/layer12_attention_weight_min": -25.04790687561035,
      "activations/layer13_attention_weight_max": 37.87840270996094,
      "activations/layer13_attention_weight_min": -35.24110412597656,
      "activations/layer14_attention_weight_max": 44.093936920166016,
      "activations/layer14_attention_weight_min": -36.57850646972656,
      "activations/layer15_attention_weight_max": 38.82804870605469,
      "activations/layer15_attention_weight_min": -35.409400939941406,
      "activations/layer16_attention_weight_max": 29.515485763549805,
      "activations/layer16_attention_weight_min": -29.488971710205078,
      "activations/layer17_attention_weight_max": 30.963003158569336,
      "activations/layer17_attention_weight_min": -26.13388442993164,
      "activations/layer18_attention_weight_max": 31.519445419311523,
      "activations/layer18_attention_weight_min": -25.35124969482422,
      "activations/layer19_attention_weight_max": 32.48265838623047,
      "activations/layer19_attention_weight_min": -30.23163414001465,
      "activations/layer1_attention_weight_max": 15.894296646118164,
      "activations/layer1_attention_weight_min": -14.665102005004883,
      "activations/layer20_attention_weight_max": 31.60036277770996,
      "activations/layer20_attention_weight_min": -24.293256759643555,
      "activations/layer21_attention_weight_max": 29.62318992614746,
      "activations/layer21_attention_weight_min": -24.507280349731445,
      "activations/layer22_attention_weight_max": 38.43611526489258,
      "activations/layer22_attention_weight_min": -29.18756103515625,
      "activations/layer23_attention_weight_max": 34.04269790649414,
      "activations/layer23_attention_weight_min": -27.15560531616211,
      "activations/layer2_attention_weight_max": 32.995323181152344,
      "activations/layer2_attention_weight_min": -31.399168014526367,
      "activations/layer3_attention_weight_max": 97.78024291992188,
      "activations/layer3_attention_weight_min": -96.03069305419922,
      "activations/layer4_attention_weight_max": 115.16507720947266,
      "activations/layer4_attention_weight_min": -111.71036529541016,
      "activations/layer5_attention_weight_max": 52.54349899291992,
      "activations/layer5_attention_weight_min": -60.37348175048828,
      "activations/layer6_attention_weight_max": 43.88872528076172,
      "activations/layer6_attention_weight_min": -46.07064437866211,
      "activations/layer7_attention_weight_max": 93.66187286376953,
      "activations/layer7_attention_weight_min": -98.1263427734375,
      "activations/layer8_attention_weight_max": 40.19594192504883,
      "activations/layer8_attention_weight_min": -42.07762908935547,
      "activations/layer9_attention_weight_max": 32.12733840942383,
      "activations/layer9_attention_weight_min": -33.47093200683594,
      "epoch": 14.94,
      "learning_rate": 5.418901515151515e-05,
      "loss": 2.7596,
      "step": 257150
    },
    {
      "activations/layer0_attention_weight_max": 16.517955780029297,
      "activations/layer0_attention_weight_min": -11.684407234191895,
      "activations/layer10_attention_weight_max": 32.324214935302734,
      "activations/layer10_attention_weight_min": -32.50876998901367,
      "activations/layer11_attention_weight_max": 32.174407958984375,
      "activations/layer11_attention_weight_min": -32.248416900634766,
      "activations/layer12_attention_weight_max": 23.752439498901367,
      "activations/layer12_attention_weight_min": -27.07962417602539,
      "activations/layer13_attention_weight_max": 35.03968048095703,
      "activations/layer13_attention_weight_min": -35.681495666503906,
      "activations/layer14_attention_weight_max": 37.90924835205078,
      "activations/layer14_attention_weight_min": -38.15816116333008,
      "activations/layer15_attention_weight_max": 34.05449295043945,
      "activations/layer15_attention_weight_min": -34.452392578125,
      "activations/layer16_attention_weight_max": 27.51541519165039,
      "activations/layer16_attention_weight_min": -27.867326736450195,
      "activations/layer17_attention_weight_max": 29.857234954833984,
      "activations/layer17_attention_weight_min": -25.03452491760254,
      "activations/layer18_attention_weight_max": 29.97610092163086,
      "activations/layer18_attention_weight_min": -23.91177749633789,
      "activations/layer19_attention_weight_max": 34.55961990356445,
      "activations/layer19_attention_weight_min": -32.180267333984375,
      "activations/layer1_attention_weight_max": 15.886025428771973,
      "activations/layer1_attention_weight_min": -15.425498008728027,
      "activations/layer20_attention_weight_max": 27.188066482543945,
      "activations/layer20_attention_weight_min": -25.683198928833008,
      "activations/layer21_attention_weight_max": 27.848739624023438,
      "activations/layer21_attention_weight_min": -24.365995407104492,
      "activations/layer22_attention_weight_max": 39.842403411865234,
      "activations/layer22_attention_weight_min": -29.84310531616211,
      "activations/layer23_attention_weight_max": 35.76856994628906,
      "activations/layer23_attention_weight_min": -26.894962310791016,
      "activations/layer2_attention_weight_max": 33.226348876953125,
      "activations/layer2_attention_weight_min": -33.291717529296875,
      "activations/layer3_attention_weight_max": 92.60421752929688,
      "activations/layer3_attention_weight_min": -95.33181762695312,
      "activations/layer4_attention_weight_max": 106.239013671875,
      "activations/layer4_attention_weight_min": -108.6117935180664,
      "activations/layer5_attention_weight_max": 52.99454879760742,
      "activations/layer5_attention_weight_min": -58.566627502441406,
      "activations/layer6_attention_weight_max": 44.06868362426758,
      "activations/layer6_attention_weight_min": -44.93070983886719,
      "activations/layer7_attention_weight_max": 90.90619659423828,
      "activations/layer7_attention_weight_min": -92.00469970703125,
      "activations/layer8_attention_weight_max": 41.27292251586914,
      "activations/layer8_attention_weight_min": -42.06190872192383,
      "activations/layer9_attention_weight_max": 31.651151657104492,
      "activations/layer9_attention_weight_min": -33.64878845214844,
      "epoch": 14.94,
      "learning_rate": 5.4170075757575755e-05,
      "loss": 2.7569,
      "step": 257200
    },
    {
      "activations/layer0_attention_weight_max": 15.979007720947266,
      "activations/layer0_attention_weight_min": -12.061609268188477,
      "activations/layer10_attention_weight_max": 33.63473892211914,
      "activations/layer10_attention_weight_min": -32.34500503540039,
      "activations/layer11_attention_weight_max": 33.59422302246094,
      "activations/layer11_attention_weight_min": -32.7596549987793,
      "activations/layer12_attention_weight_max": 25.215656280517578,
      "activations/layer12_attention_weight_min": -26.671165466308594,
      "activations/layer13_attention_weight_max": 37.79484939575195,
      "activations/layer13_attention_weight_min": -36.24627685546875,
      "activations/layer14_attention_weight_max": 42.8906364440918,
      "activations/layer14_attention_weight_min": -36.24232864379883,
      "activations/layer15_attention_weight_max": 36.142974853515625,
      "activations/layer15_attention_weight_min": -34.838356018066406,
      "activations/layer16_attention_weight_max": 31.59461212158203,
      "activations/layer16_attention_weight_min": -26.60694122314453,
      "activations/layer17_attention_weight_max": 30.320112228393555,
      "activations/layer17_attention_weight_min": -24.680240631103516,
      "activations/layer18_attention_weight_max": 29.68633460998535,
      "activations/layer18_attention_weight_min": -22.311800003051758,
      "activations/layer19_attention_weight_max": 31.44904899597168,
      "activations/layer19_attention_weight_min": -29.77741813659668,
      "activations/layer1_attention_weight_max": 16.34232521057129,
      "activations/layer1_attention_weight_min": -15.511208534240723,
      "activations/layer20_attention_weight_max": 29.2137393951416,
      "activations/layer20_attention_weight_min": -23.708091735839844,
      "activations/layer21_attention_weight_max": 29.744609832763672,
      "activations/layer21_attention_weight_min": -22.2265567779541,
      "activations/layer22_attention_weight_max": 39.71650695800781,
      "activations/layer22_attention_weight_min": -29.61329460144043,
      "activations/layer23_attention_weight_max": 30.649246215820312,
      "activations/layer23_attention_weight_min": -25.45370864868164,
      "activations/layer2_attention_weight_max": 30.613906860351562,
      "activations/layer2_attention_weight_min": -29.620121002197266,
      "activations/layer3_attention_weight_max": 89.97786712646484,
      "activations/layer3_attention_weight_min": -92.80462646484375,
      "activations/layer4_attention_weight_max": 107.6329116821289,
      "activations/layer4_attention_weight_min": -110.57435607910156,
      "activations/layer5_attention_weight_max": 54.31714630126953,
      "activations/layer5_attention_weight_min": -59.54902648925781,
      "activations/layer6_attention_weight_max": 42.9321174621582,
      "activations/layer6_attention_weight_min": -46.49448776245117,
      "activations/layer7_attention_weight_max": 90.25406646728516,
      "activations/layer7_attention_weight_min": -93.03199005126953,
      "activations/layer8_attention_weight_max": 41.5610466003418,
      "activations/layer8_attention_weight_min": -42.180274963378906,
      "activations/layer9_attention_weight_max": 32.973751068115234,
      "activations/layer9_attention_weight_min": -34.57681655883789,
      "epoch": 14.95,
      "learning_rate": 5.4151515151515145e-05,
      "loss": 2.7503,
      "step": 257250
    },
    {
      "activations/layer0_attention_weight_max": 16.689823150634766,
      "activations/layer0_attention_weight_min": -11.449934959411621,
      "activations/layer10_attention_weight_max": 31.15890884399414,
      "activations/layer10_attention_weight_min": -32.796714782714844,
      "activations/layer11_attention_weight_max": 32.333709716796875,
      "activations/layer11_attention_weight_min": -31.885221481323242,
      "activations/layer12_attention_weight_max": 26.412599563598633,
      "activations/layer12_attention_weight_min": -27.7165584564209,
      "activations/layer13_attention_weight_max": 41.16724395751953,
      "activations/layer13_attention_weight_min": -35.34086608886719,
      "activations/layer14_attention_weight_max": 44.60810089111328,
      "activations/layer14_attention_weight_min": -41.647926330566406,
      "activations/layer15_attention_weight_max": 40.56569290161133,
      "activations/layer15_attention_weight_min": -37.7371826171875,
      "activations/layer16_attention_weight_max": 31.956144332885742,
      "activations/layer16_attention_weight_min": -30.925045013427734,
      "activations/layer17_attention_weight_max": 30.985328674316406,
      "activations/layer17_attention_weight_min": -27.377405166625977,
      "activations/layer18_attention_weight_max": 31.589008331298828,
      "activations/layer18_attention_weight_min": -25.933467864990234,
      "activations/layer19_attention_weight_max": 38.11597442626953,
      "activations/layer19_attention_weight_min": -29.805343627929688,
      "activations/layer1_attention_weight_max": 15.8667631149292,
      "activations/layer1_attention_weight_min": -13.79909610748291,
      "activations/layer20_attention_weight_max": 31.283796310424805,
      "activations/layer20_attention_weight_min": -24.239490509033203,
      "activations/layer21_attention_weight_max": 29.177616119384766,
      "activations/layer21_attention_weight_min": -24.310842514038086,
      "activations/layer22_attention_weight_max": 46.466453552246094,
      "activations/layer22_attention_weight_min": -35.197349548339844,
      "activations/layer23_attention_weight_max": 34.784114837646484,
      "activations/layer23_attention_weight_min": -27.159805297851562,
      "activations/layer2_attention_weight_max": 32.42268371582031,
      "activations/layer2_attention_weight_min": -32.10584259033203,
      "activations/layer3_attention_weight_max": 92.70781707763672,
      "activations/layer3_attention_weight_min": -98.06490325927734,
      "activations/layer4_attention_weight_max": 108.73065185546875,
      "activations/layer4_attention_weight_min": -112.30632781982422,
      "activations/layer5_attention_weight_max": 50.9110107421875,
      "activations/layer5_attention_weight_min": -60.176971435546875,
      "activations/layer6_attention_weight_max": 44.109275817871094,
      "activations/layer6_attention_weight_min": -44.48072814941406,
      "activations/layer7_attention_weight_max": 91.09766387939453,
      "activations/layer7_attention_weight_min": -89.0877914428711,
      "activations/layer8_attention_weight_max": 38.6255989074707,
      "activations/layer8_attention_weight_min": -40.794368743896484,
      "activations/layer9_attention_weight_max": 32.13032150268555,
      "activations/layer9_attention_weight_min": -31.995290756225586,
      "epoch": 14.95,
      "learning_rate": 5.413257575757575e-05,
      "loss": 2.758,
      "step": 257300
    },
    {
      "activations/layer0_attention_weight_max": 16.258031845092773,
      "activations/layer0_attention_weight_min": -12.330338478088379,
      "activations/layer10_attention_weight_max": 45.11887741088867,
      "activations/layer10_attention_weight_min": -41.64398193359375,
      "activations/layer11_attention_weight_max": 46.670955657958984,
      "activations/layer11_attention_weight_min": -44.390663146972656,
      "activations/layer12_attention_weight_max": 44.80342483520508,
      "activations/layer12_attention_weight_min": -32.733734130859375,
      "activations/layer13_attention_weight_max": 59.915435791015625,
      "activations/layer13_attention_weight_min": -37.61497497558594,
      "activations/layer14_attention_weight_max": 54.95637893676758,
      "activations/layer14_attention_weight_min": -43.252906799316406,
      "activations/layer15_attention_weight_max": 57.969749450683594,
      "activations/layer15_attention_weight_min": -41.421634674072266,
      "activations/layer16_attention_weight_max": 33.64884567260742,
      "activations/layer16_attention_weight_min": -27.76871109008789,
      "activations/layer17_attention_weight_max": 31.82015609741211,
      "activations/layer17_attention_weight_min": -28.788076400756836,
      "activations/layer18_attention_weight_max": 29.636451721191406,
      "activations/layer18_attention_weight_min": -24.615114212036133,
      "activations/layer19_attention_weight_max": 32.711700439453125,
      "activations/layer19_attention_weight_min": -30.972991943359375,
      "activations/layer1_attention_weight_max": 16.22191047668457,
      "activations/layer1_attention_weight_min": -15.452608108520508,
      "activations/layer20_attention_weight_max": 26.1899356842041,
      "activations/layer20_attention_weight_min": -23.25375747680664,
      "activations/layer21_attention_weight_max": 26.366943359375,
      "activations/layer21_attention_weight_min": -26.522014617919922,
      "activations/layer22_attention_weight_max": 39.4616813659668,
      "activations/layer22_attention_weight_min": -30.01942253112793,
      "activations/layer23_attention_weight_max": 32.469451904296875,
      "activations/layer23_attention_weight_min": -27.57015609741211,
      "activations/layer2_attention_weight_max": 36.5730094909668,
      "activations/layer2_attention_weight_min": -37.89470672607422,
      "activations/layer3_attention_weight_max": 101.20832061767578,
      "activations/layer3_attention_weight_min": -106.38796997070312,
      "activations/layer4_attention_weight_max": 115.5357437133789,
      "activations/layer4_attention_weight_min": -111.8024673461914,
      "activations/layer5_attention_weight_max": 58.05426788330078,
      "activations/layer5_attention_weight_min": -62.51299285888672,
      "activations/layer6_attention_weight_max": 44.95490646362305,
      "activations/layer6_attention_weight_min": -47.147464752197266,
      "activations/layer7_attention_weight_max": 113.47216033935547,
      "activations/layer7_attention_weight_min": -95.39155578613281,
      "activations/layer8_attention_weight_max": 51.02745056152344,
      "activations/layer8_attention_weight_min": -47.46609878540039,
      "activations/layer9_attention_weight_max": 40.85737991333008,
      "activations/layer9_attention_weight_min": -40.0238037109375,
      "epoch": 14.95,
      "learning_rate": 5.411363636363636e-05,
      "loss": 2.7426,
      "step": 257350
    },
    {
      "activations/layer0_attention_weight_max": 15.98605728149414,
      "activations/layer0_attention_weight_min": -11.895899772644043,
      "activations/layer10_attention_weight_max": 31.664396286010742,
      "activations/layer10_attention_weight_min": -31.382463455200195,
      "activations/layer11_attention_weight_max": 30.88484001159668,
      "activations/layer11_attention_weight_min": -31.210277557373047,
      "activations/layer12_attention_weight_max": 24.42871856689453,
      "activations/layer12_attention_weight_min": -24.754785537719727,
      "activations/layer13_attention_weight_max": 36.47825241088867,
      "activations/layer13_attention_weight_min": -33.594478607177734,
      "activations/layer14_attention_weight_max": 37.48396301269531,
      "activations/layer14_attention_weight_min": -35.66153335571289,
      "activations/layer15_attention_weight_max": 34.81658935546875,
      "activations/layer15_attention_weight_min": -34.461769104003906,
      "activations/layer16_attention_weight_max": 28.419893264770508,
      "activations/layer16_attention_weight_min": -28.412616729736328,
      "activations/layer17_attention_weight_max": 31.43916893005371,
      "activations/layer17_attention_weight_min": -25.01647186279297,
      "activations/layer18_attention_weight_max": 31.495601654052734,
      "activations/layer18_attention_weight_min": -26.551673889160156,
      "activations/layer19_attention_weight_max": 33.99513626098633,
      "activations/layer19_attention_weight_min": -29.51698875427246,
      "activations/layer1_attention_weight_max": 16.964645385742188,
      "activations/layer1_attention_weight_min": -13.300263404846191,
      "activations/layer20_attention_weight_max": 28.950315475463867,
      "activations/layer20_attention_weight_min": -27.551639556884766,
      "activations/layer21_attention_weight_max": 29.14727783203125,
      "activations/layer21_attention_weight_min": -25.811582565307617,
      "activations/layer22_attention_weight_max": 42.89531707763672,
      "activations/layer22_attention_weight_min": -30.27839469909668,
      "activations/layer23_attention_weight_max": 33.31583786010742,
      "activations/layer23_attention_weight_min": -25.92992401123047,
      "activations/layer2_attention_weight_max": 30.524253845214844,
      "activations/layer2_attention_weight_min": -29.89665985107422,
      "activations/layer3_attention_weight_max": 90.28325653076172,
      "activations/layer3_attention_weight_min": -92.05009460449219,
      "activations/layer4_attention_weight_max": 103.59356689453125,
      "activations/layer4_attention_weight_min": -105.10453033447266,
      "activations/layer5_attention_weight_max": 49.30579376220703,
      "activations/layer5_attention_weight_min": -60.2799072265625,
      "activations/layer6_attention_weight_max": 41.608245849609375,
      "activations/layer6_attention_weight_min": -44.96623611450195,
      "activations/layer7_attention_weight_max": 86.36355590820312,
      "activations/layer7_attention_weight_min": -89.57095336914062,
      "activations/layer8_attention_weight_max": 37.64746856689453,
      "activations/layer8_attention_weight_min": -40.26726150512695,
      "activations/layer9_attention_weight_max": 31.150705337524414,
      "activations/layer9_attention_weight_min": -33.04045486450195,
      "epoch": 14.96,
      "learning_rate": 5.4094696969696963e-05,
      "loss": 2.753,
      "step": 257400
    },
    {
      "activations/layer0_attention_weight_max": 16.16895294189453,
      "activations/layer0_attention_weight_min": -11.74303913116455,
      "activations/layer10_attention_weight_max": 43.026519775390625,
      "activations/layer10_attention_weight_min": -40.246402740478516,
      "activations/layer11_attention_weight_max": 45.13312530517578,
      "activations/layer11_attention_weight_min": -41.46791458129883,
      "activations/layer12_attention_weight_max": 27.487382888793945,
      "activations/layer12_attention_weight_min": -26.755207061767578,
      "activations/layer13_attention_weight_max": 39.308746337890625,
      "activations/layer13_attention_weight_min": -34.418006896972656,
      "activations/layer14_attention_weight_max": 41.08269500732422,
      "activations/layer14_attention_weight_min": -37.25053405761719,
      "activations/layer15_attention_weight_max": 37.0069580078125,
      "activations/layer15_attention_weight_min": -36.10860824584961,
      "activations/layer16_attention_weight_max": 29.63250732421875,
      "activations/layer16_attention_weight_min": -27.607358932495117,
      "activations/layer17_attention_weight_max": 29.505258560180664,
      "activations/layer17_attention_weight_min": -25.66968536376953,
      "activations/layer18_attention_weight_max": 33.03239440917969,
      "activations/layer18_attention_weight_min": -25.175710678100586,
      "activations/layer19_attention_weight_max": 34.610084533691406,
      "activations/layer19_attention_weight_min": -31.617109298706055,
      "activations/layer1_attention_weight_max": 16.596275329589844,
      "activations/layer1_attention_weight_min": -14.799773216247559,
      "activations/layer20_attention_weight_max": 29.581096649169922,
      "activations/layer20_attention_weight_min": -24.862882614135742,
      "activations/layer21_attention_weight_max": 29.747154235839844,
      "activations/layer21_attention_weight_min": -26.60478973388672,
      "activations/layer22_attention_weight_max": 48.31074905395508,
      "activations/layer22_attention_weight_min": -31.139585494995117,
      "activations/layer23_attention_weight_max": 31.559703826904297,
      "activations/layer23_attention_weight_min": -25.671009063720703,
      "activations/layer2_attention_weight_max": 33.04710388183594,
      "activations/layer2_attention_weight_min": -31.856685638427734,
      "activations/layer3_attention_weight_max": 89.26981353759766,
      "activations/layer3_attention_weight_min": -92.87266540527344,
      "activations/layer4_attention_weight_max": 99.73954010009766,
      "activations/layer4_attention_weight_min": -105.2931900024414,
      "activations/layer5_attention_weight_max": 50.469085693359375,
      "activations/layer5_attention_weight_min": -64.31600952148438,
      "activations/layer6_attention_weight_max": 41.28034973144531,
      "activations/layer6_attention_weight_min": -43.30928039550781,
      "activations/layer7_attention_weight_max": 97.77401733398438,
      "activations/layer7_attention_weight_min": -94.14630126953125,
      "activations/layer8_attention_weight_max": 46.36492919921875,
      "activations/layer8_attention_weight_min": -44.470401763916016,
      "activations/layer9_attention_weight_max": 37.673248291015625,
      "activations/layer9_attention_weight_min": -37.394596099853516,
      "epoch": 14.96,
      "learning_rate": 5.4076136363636354e-05,
      "loss": 2.749,
      "step": 257450
    },
    {
      "activations/layer0_attention_weight_max": 16.31616973876953,
      "activations/layer0_attention_weight_min": -11.663871765136719,
      "activations/layer10_attention_weight_max": 31.738676071166992,
      "activations/layer10_attention_weight_min": -33.0745849609375,
      "activations/layer11_attention_weight_max": 32.1488151550293,
      "activations/layer11_attention_weight_min": -33.102752685546875,
      "activations/layer12_attention_weight_max": 26.355070114135742,
      "activations/layer12_attention_weight_min": -26.554367065429688,
      "activations/layer13_attention_weight_max": 40.26293182373047,
      "activations/layer13_attention_weight_min": -34.41114044189453,
      "activations/layer14_attention_weight_max": 41.635658264160156,
      "activations/layer14_attention_weight_min": -38.081295013427734,
      "activations/layer15_attention_weight_max": 37.277645111083984,
      "activations/layer15_attention_weight_min": -33.599937438964844,
      "activations/layer16_attention_weight_max": 30.887292861938477,
      "activations/layer16_attention_weight_min": -27.828752517700195,
      "activations/layer17_attention_weight_max": 31.42685317993164,
      "activations/layer17_attention_weight_min": -25.131446838378906,
      "activations/layer18_attention_weight_max": 32.81421661376953,
      "activations/layer18_attention_weight_min": -27.035799026489258,
      "activations/layer19_attention_weight_max": 34.758216857910156,
      "activations/layer19_attention_weight_min": -31.169921875,
      "activations/layer1_attention_weight_max": 16.666444778442383,
      "activations/layer1_attention_weight_min": -14.299793243408203,
      "activations/layer20_attention_weight_max": 29.153064727783203,
      "activations/layer20_attention_weight_min": -24.886388778686523,
      "activations/layer21_attention_weight_max": 29.634328842163086,
      "activations/layer21_attention_weight_min": -24.6368465423584,
      "activations/layer22_attention_weight_max": 47.84621810913086,
      "activations/layer22_attention_weight_min": -29.97690200805664,
      "activations/layer23_attention_weight_max": 33.84332275390625,
      "activations/layer23_attention_weight_min": -25.29916000366211,
      "activations/layer2_attention_weight_max": 33.31951141357422,
      "activations/layer2_attention_weight_min": -31.925336837768555,
      "activations/layer3_attention_weight_max": 93.50112915039062,
      "activations/layer3_attention_weight_min": -93.18342590332031,
      "activations/layer4_attention_weight_max": 108.1650161743164,
      "activations/layer4_attention_weight_min": -113.8624267578125,
      "activations/layer5_attention_weight_max": 50.42875289916992,
      "activations/layer5_attention_weight_min": -66.42259979248047,
      "activations/layer6_attention_weight_max": 43.7534294128418,
      "activations/layer6_attention_weight_min": -47.65338897705078,
      "activations/layer7_attention_weight_max": 94.96814727783203,
      "activations/layer7_attention_weight_min": -94.56492614746094,
      "activations/layer8_attention_weight_max": 40.963653564453125,
      "activations/layer8_attention_weight_min": -42.98453903198242,
      "activations/layer9_attention_weight_max": 32.11598205566406,
      "activations/layer9_attention_weight_min": -33.476966857910156,
      "epoch": 14.96,
      "learning_rate": 5.405719696969696e-05,
      "loss": 2.7484,
      "step": 257500
    },
    {
      "activations/layer0_attention_weight_max": 15.974529266357422,
      "activations/layer0_attention_weight_min": -12.199712753295898,
      "activations/layer10_attention_weight_max": 33.542884826660156,
      "activations/layer10_attention_weight_min": -35.282814025878906,
      "activations/layer11_attention_weight_max": 36.61238098144531,
      "activations/layer11_attention_weight_min": -34.2342529296875,
      "activations/layer12_attention_weight_max": 26.503971099853516,
      "activations/layer12_attention_weight_min": -25.943449020385742,
      "activations/layer13_attention_weight_max": 37.87257385253906,
      "activations/layer13_attention_weight_min": -34.68301010131836,
      "activations/layer14_attention_weight_max": 39.90682601928711,
      "activations/layer14_attention_weight_min": -36.44976806640625,
      "activations/layer15_attention_weight_max": 33.833770751953125,
      "activations/layer15_attention_weight_min": -34.3095703125,
      "activations/layer16_attention_weight_max": 30.024402618408203,
      "activations/layer16_attention_weight_min": -29.960899353027344,
      "activations/layer17_attention_weight_max": 28.544755935668945,
      "activations/layer17_attention_weight_min": -28.828149795532227,
      "activations/layer18_attention_weight_max": 30.651865005493164,
      "activations/layer18_attention_weight_min": -25.781230926513672,
      "activations/layer19_attention_weight_max": 31.699691772460938,
      "activations/layer19_attention_weight_min": -30.154647827148438,
      "activations/layer1_attention_weight_max": 16.079219818115234,
      "activations/layer1_attention_weight_min": -14.720047950744629,
      "activations/layer20_attention_weight_max": 28.093040466308594,
      "activations/layer20_attention_weight_min": -24.683658599853516,
      "activations/layer21_attention_weight_max": 27.953378677368164,
      "activations/layer21_attention_weight_min": -25.2012882232666,
      "activations/layer22_attention_weight_max": 40.074974060058594,
      "activations/layer22_attention_weight_min": -29.992963790893555,
      "activations/layer23_attention_weight_max": 32.75629425048828,
      "activations/layer23_attention_weight_min": -27.204391479492188,
      "activations/layer2_attention_weight_max": 35.1250114440918,
      "activations/layer2_attention_weight_min": -33.48469924926758,
      "activations/layer3_attention_weight_max": 96.778076171875,
      "activations/layer3_attention_weight_min": -94.72903442382812,
      "activations/layer4_attention_weight_max": 108.8161392211914,
      "activations/layer4_attention_weight_min": -107.99271392822266,
      "activations/layer5_attention_weight_max": 53.83782196044922,
      "activations/layer5_attention_weight_min": -59.64417266845703,
      "activations/layer6_attention_weight_max": 45.54118347167969,
      "activations/layer6_attention_weight_min": -47.300724029541016,
      "activations/layer7_attention_weight_max": 95.98957061767578,
      "activations/layer7_attention_weight_min": -95.34588623046875,
      "activations/layer8_attention_weight_max": 43.13447570800781,
      "activations/layer8_attention_weight_min": -42.66337966918945,
      "activations/layer9_attention_weight_max": 33.58540344238281,
      "activations/layer9_attention_weight_min": -33.687259674072266,
      "epoch": 14.97,
      "learning_rate": 5.403825757575757e-05,
      "loss": 2.7663,
      "step": 257550
    },
    {
      "activations/layer0_attention_weight_max": 15.861677169799805,
      "activations/layer0_attention_weight_min": -12.060924530029297,
      "activations/layer10_attention_weight_max": 33.887969970703125,
      "activations/layer10_attention_weight_min": -34.27334976196289,
      "activations/layer11_attention_weight_max": 33.72314453125,
      "activations/layer11_attention_weight_min": -34.52227020263672,
      "activations/layer12_attention_weight_max": 27.76417350769043,
      "activations/layer12_attention_weight_min": -26.406633377075195,
      "activations/layer13_attention_weight_max": 42.57005310058594,
      "activations/layer13_attention_weight_min": -36.297279357910156,
      "activations/layer14_attention_weight_max": 47.67228698730469,
      "activations/layer14_attention_weight_min": -40.409332275390625,
      "activations/layer15_attention_weight_max": 38.20933532714844,
      "activations/layer15_attention_weight_min": -36.981590270996094,
      "activations/layer16_attention_weight_max": 30.699430465698242,
      "activations/layer16_attention_weight_min": -27.942930221557617,
      "activations/layer17_attention_weight_max": 31.76381492614746,
      "activations/layer17_attention_weight_min": -24.996854782104492,
      "activations/layer18_attention_weight_max": 32.17264938354492,
      "activations/layer18_attention_weight_min": -23.934139251708984,
      "activations/layer19_attention_weight_max": 36.576148986816406,
      "activations/layer19_attention_weight_min": -32.62136459350586,
      "activations/layer1_attention_weight_max": 17.551753997802734,
      "activations/layer1_attention_weight_min": -14.803234100341797,
      "activations/layer20_attention_weight_max": 32.01272964477539,
      "activations/layer20_attention_weight_min": -24.66006088256836,
      "activations/layer21_attention_weight_max": 30.292795181274414,
      "activations/layer21_attention_weight_min": -24.361928939819336,
      "activations/layer22_attention_weight_max": 47.64444351196289,
      "activations/layer22_attention_weight_min": -31.880226135253906,
      "activations/layer23_attention_weight_max": 35.902130126953125,
      "activations/layer23_attention_weight_min": -25.369037628173828,
      "activations/layer2_attention_weight_max": 34.19525909423828,
      "activations/layer2_attention_weight_min": -33.82647705078125,
      "activations/layer3_attention_weight_max": 96.27812194824219,
      "activations/layer3_attention_weight_min": -100.125,
      "activations/layer4_attention_weight_max": 111.33673095703125,
      "activations/layer4_attention_weight_min": -110.97596740722656,
      "activations/layer5_attention_weight_max": 53.74846649169922,
      "activations/layer5_attention_weight_min": -60.5509033203125,
      "activations/layer6_attention_weight_max": 43.70138931274414,
      "activations/layer6_attention_weight_min": -47.88256072998047,
      "activations/layer7_attention_weight_max": 93.19481658935547,
      "activations/layer7_attention_weight_min": -95.57770538330078,
      "activations/layer8_attention_weight_max": 42.272064208984375,
      "activations/layer8_attention_weight_min": -44.107215881347656,
      "activations/layer9_attention_weight_max": 34.30141830444336,
      "activations/layer9_attention_weight_min": -35.39040756225586,
      "epoch": 14.97,
      "learning_rate": 5.401931818181818e-05,
      "loss": 2.7454,
      "step": 257600
    },
    {
      "activations/layer0_attention_weight_max": 15.784409523010254,
      "activations/layer0_attention_weight_min": -11.999469757080078,
      "activations/layer10_attention_weight_max": 31.6204891204834,
      "activations/layer10_attention_weight_min": -31.21138572692871,
      "activations/layer11_attention_weight_max": 32.30552673339844,
      "activations/layer11_attention_weight_min": -32.46202850341797,
      "activations/layer12_attention_weight_max": 27.496843338012695,
      "activations/layer12_attention_weight_min": -26.885025024414062,
      "activations/layer13_attention_weight_max": 40.080535888671875,
      "activations/layer13_attention_weight_min": -35.13520812988281,
      "activations/layer14_attention_weight_max": 46.730995178222656,
      "activations/layer14_attention_weight_min": -37.162193298339844,
      "activations/layer15_attention_weight_max": 39.76549530029297,
      "activations/layer15_attention_weight_min": -33.350189208984375,
      "activations/layer16_attention_weight_max": 29.310117721557617,
      "activations/layer16_attention_weight_min": -27.54960060119629,
      "activations/layer17_attention_weight_max": 29.58180809020996,
      "activations/layer17_attention_weight_min": -26.070327758789062,
      "activations/layer18_attention_weight_max": 30.769865036010742,
      "activations/layer18_attention_weight_min": -26.96721076965332,
      "activations/layer19_attention_weight_max": 30.577301025390625,
      "activations/layer19_attention_weight_min": -31.05572509765625,
      "activations/layer1_attention_weight_max": 16.021114349365234,
      "activations/layer1_attention_weight_min": -15.240410804748535,
      "activations/layer20_attention_weight_max": 26.369028091430664,
      "activations/layer20_attention_weight_min": -25.936922073364258,
      "activations/layer21_attention_weight_max": 29.872520446777344,
      "activations/layer21_attention_weight_min": -26.12087059020996,
      "activations/layer22_attention_weight_max": 41.20264434814453,
      "activations/layer22_attention_weight_min": -33.27130889892578,
      "activations/layer23_attention_weight_max": 31.52727508544922,
      "activations/layer23_attention_weight_min": -26.57826805114746,
      "activations/layer2_attention_weight_max": 32.180450439453125,
      "activations/layer2_attention_weight_min": -29.800016403198242,
      "activations/layer3_attention_weight_max": 92.37928771972656,
      "activations/layer3_attention_weight_min": -90.29890441894531,
      "activations/layer4_attention_weight_max": 106.9403076171875,
      "activations/layer4_attention_weight_min": -107.85137939453125,
      "activations/layer5_attention_weight_max": 54.44025802612305,
      "activations/layer5_attention_weight_min": -61.881988525390625,
      "activations/layer6_attention_weight_max": 42.42279052734375,
      "activations/layer6_attention_weight_min": -45.232887268066406,
      "activations/layer7_attention_weight_max": 94.02715301513672,
      "activations/layer7_attention_weight_min": -86.16336059570312,
      "activations/layer8_attention_weight_max": 39.38969802856445,
      "activations/layer8_attention_weight_min": -42.42906951904297,
      "activations/layer9_attention_weight_max": 30.654953002929688,
      "activations/layer9_attention_weight_min": -30.83222770690918,
      "epoch": 14.97,
      "learning_rate": 5.400037878787878e-05,
      "loss": 2.7683,
      "step": 257650
    },
    {
      "activations/layer0_attention_weight_max": 16.019527435302734,
      "activations/layer0_attention_weight_min": -11.080222129821777,
      "activations/layer10_attention_weight_max": 46.6992301940918,
      "activations/layer10_attention_weight_min": -44.818485260009766,
      "activations/layer11_attention_weight_max": 46.188568115234375,
      "activations/layer11_attention_weight_min": -47.99986267089844,
      "activations/layer12_attention_weight_max": 30.328182220458984,
      "activations/layer12_attention_weight_min": -27.874671936035156,
      "activations/layer13_attention_weight_max": 52.020118713378906,
      "activations/layer13_attention_weight_min": -37.35485076904297,
      "activations/layer14_attention_weight_max": 56.59486770629883,
      "activations/layer14_attention_weight_min": -40.69699478149414,
      "activations/layer15_attention_weight_max": 55.604393005371094,
      "activations/layer15_attention_weight_min": -40.419952392578125,
      "activations/layer16_attention_weight_max": 36.24989318847656,
      "activations/layer16_attention_weight_min": -27.245656967163086,
      "activations/layer17_attention_weight_max": 31.642656326293945,
      "activations/layer17_attention_weight_min": -30.10539436340332,
      "activations/layer18_attention_weight_max": 33.411529541015625,
      "activations/layer18_attention_weight_min": -27.576772689819336,
      "activations/layer19_attention_weight_max": 44.39755630493164,
      "activations/layer19_attention_weight_min": -32.39509201049805,
      "activations/layer1_attention_weight_max": 16.295040130615234,
      "activations/layer1_attention_weight_min": -15.809261322021484,
      "activations/layer20_attention_weight_max": 33.178741455078125,
      "activations/layer20_attention_weight_min": -24.809724807739258,
      "activations/layer21_attention_weight_max": 30.46830177307129,
      "activations/layer21_attention_weight_min": -23.80185890197754,
      "activations/layer22_attention_weight_max": 53.916717529296875,
      "activations/layer22_attention_weight_min": -32.372657775878906,
      "activations/layer23_attention_weight_max": 34.64653015136719,
      "activations/layer23_attention_weight_min": -24.918407440185547,
      "activations/layer2_attention_weight_max": 34.32395553588867,
      "activations/layer2_attention_weight_min": -35.615638732910156,
      "activations/layer3_attention_weight_max": 99.98706817626953,
      "activations/layer3_attention_weight_min": -103.85513305664062,
      "activations/layer4_attention_weight_max": 120.2844009399414,
      "activations/layer4_attention_weight_min": -116.01517486572266,
      "activations/layer5_attention_weight_max": 60.73253631591797,
      "activations/layer5_attention_weight_min": -59.405494689941406,
      "activations/layer6_attention_weight_max": 50.69037628173828,
      "activations/layer6_attention_weight_min": -52.67152786254883,
      "activations/layer7_attention_weight_max": 111.13665008544922,
      "activations/layer7_attention_weight_min": -103.01342010498047,
      "activations/layer8_attention_weight_max": 50.85892868041992,
      "activations/layer8_attention_weight_min": -52.68495178222656,
      "activations/layer9_attention_weight_max": 40.56534194946289,
      "activations/layer9_attention_weight_min": -41.63869857788086,
      "epoch": 14.97,
      "learning_rate": 5.398143939393938e-05,
      "loss": 2.7413,
      "step": 257700
    },
    {
      "activations/layer0_attention_weight_max": 15.636899948120117,
      "activations/layer0_attention_weight_min": -11.97144889831543,
      "activations/layer10_attention_weight_max": 30.84428596496582,
      "activations/layer10_attention_weight_min": -31.45072364807129,
      "activations/layer11_attention_weight_max": 32.339210510253906,
      "activations/layer11_attention_weight_min": -35.18053436279297,
      "activations/layer12_attention_weight_max": 26.173133850097656,
      "activations/layer12_attention_weight_min": -25.21185874938965,
      "activations/layer13_attention_weight_max": 39.01991271972656,
      "activations/layer13_attention_weight_min": -34.579383850097656,
      "activations/layer14_attention_weight_max": 46.041751861572266,
      "activations/layer14_attention_weight_min": -38.41720962524414,
      "activations/layer15_attention_weight_max": 39.71837615966797,
      "activations/layer15_attention_weight_min": -35.960205078125,
      "activations/layer16_attention_weight_max": 30.81722068786621,
      "activations/layer16_attention_weight_min": -28.1611270904541,
      "activations/layer17_attention_weight_max": 29.91006088256836,
      "activations/layer17_attention_weight_min": -27.396535873413086,
      "activations/layer18_attention_weight_max": 31.462675094604492,
      "activations/layer18_attention_weight_min": -26.6021728515625,
      "activations/layer19_attention_weight_max": 37.25593566894531,
      "activations/layer19_attention_weight_min": -31.08512306213379,
      "activations/layer1_attention_weight_max": 15.985857963562012,
      "activations/layer1_attention_weight_min": -14.867082595825195,
      "activations/layer20_attention_weight_max": 29.5202693939209,
      "activations/layer20_attention_weight_min": -24.610393524169922,
      "activations/layer21_attention_weight_max": 28.664915084838867,
      "activations/layer21_attention_weight_min": -24.54231834411621,
      "activations/layer22_attention_weight_max": 45.54124069213867,
      "activations/layer22_attention_weight_min": -32.391502380371094,
      "activations/layer23_attention_weight_max": 36.606597900390625,
      "activations/layer23_attention_weight_min": -26.37912368774414,
      "activations/layer2_attention_weight_max": 32.78355026245117,
      "activations/layer2_attention_weight_min": -32.135128021240234,
      "activations/layer3_attention_weight_max": 88.1950912475586,
      "activations/layer3_attention_weight_min": -92.35468292236328,
      "activations/layer4_attention_weight_max": 103.5356674194336,
      "activations/layer4_attention_weight_min": -105.81139373779297,
      "activations/layer5_attention_weight_max": 50.43523406982422,
      "activations/layer5_attention_weight_min": -62.55634307861328,
      "activations/layer6_attention_weight_max": 40.57006072998047,
      "activations/layer6_attention_weight_min": -45.30820083618164,
      "activations/layer7_attention_weight_max": 91.12706756591797,
      "activations/layer7_attention_weight_min": -86.87574768066406,
      "activations/layer8_attention_weight_max": 37.75251388549805,
      "activations/layer8_attention_weight_min": -40.10301208496094,
      "activations/layer9_attention_weight_max": 29.368818283081055,
      "activations/layer9_attention_weight_min": -30.58847999572754,
      "epoch": 14.98,
      "learning_rate": 5.39625e-05,
      "loss": 2.7451,
      "step": 257750
    },
    {
      "activations/layer0_attention_weight_max": 14.81954574584961,
      "activations/layer0_attention_weight_min": -11.67771053314209,
      "activations/layer10_attention_weight_max": 39.86003494262695,
      "activations/layer10_attention_weight_min": -38.17103958129883,
      "activations/layer11_attention_weight_max": 35.667640686035156,
      "activations/layer11_attention_weight_min": -37.63081359863281,
      "activations/layer12_attention_weight_max": 27.743558883666992,
      "activations/layer12_attention_weight_min": -29.454755783081055,
      "activations/layer13_attention_weight_max": 43.833396911621094,
      "activations/layer13_attention_weight_min": -39.47563934326172,
      "activations/layer14_attention_weight_max": 47.806671142578125,
      "activations/layer14_attention_weight_min": -42.123451232910156,
      "activations/layer15_attention_weight_max": 42.79375457763672,
      "activations/layer15_attention_weight_min": -37.35374450683594,
      "activations/layer16_attention_weight_max": 32.003787994384766,
      "activations/layer16_attention_weight_min": -30.0655460357666,
      "activations/layer17_attention_weight_max": 32.97308349609375,
      "activations/layer17_attention_weight_min": -26.787532806396484,
      "activations/layer18_attention_weight_max": 31.21820068359375,
      "activations/layer18_attention_weight_min": -25.331302642822266,
      "activations/layer19_attention_weight_max": 33.45892333984375,
      "activations/layer19_attention_weight_min": -33.21071243286133,
      "activations/layer1_attention_weight_max": 16.25840187072754,
      "activations/layer1_attention_weight_min": -14.482876777648926,
      "activations/layer20_attention_weight_max": 28.356609344482422,
      "activations/layer20_attention_weight_min": -25.29173469543457,
      "activations/layer21_attention_weight_max": 28.930944442749023,
      "activations/layer21_attention_weight_min": -24.312362670898438,
      "activations/layer22_attention_weight_max": 41.603729248046875,
      "activations/layer22_attention_weight_min": -30.020893096923828,
      "activations/layer23_attention_weight_max": 34.85140609741211,
      "activations/layer23_attention_weight_min": -24.472503662109375,
      "activations/layer2_attention_weight_max": 32.67879104614258,
      "activations/layer2_attention_weight_min": -31.338764190673828,
      "activations/layer3_attention_weight_max": 96.57826232910156,
      "activations/layer3_attention_weight_min": -96.2408676147461,
      "activations/layer4_attention_weight_max": 111.03618621826172,
      "activations/layer4_attention_weight_min": -110.8407211303711,
      "activations/layer5_attention_weight_max": 52.50075912475586,
      "activations/layer5_attention_weight_min": -62.14304733276367,
      "activations/layer6_attention_weight_max": 47.057376861572266,
      "activations/layer6_attention_weight_min": -47.90912628173828,
      "activations/layer7_attention_weight_max": 101.85932922363281,
      "activations/layer7_attention_weight_min": -100.56385040283203,
      "activations/layer8_attention_weight_max": 44.91836929321289,
      "activations/layer8_attention_weight_min": -46.23403549194336,
      "activations/layer9_attention_weight_max": 39.72520446777344,
      "activations/layer9_attention_weight_min": -40.14522933959961,
      "epoch": 14.98,
      "learning_rate": 5.39435606060606e-05,
      "loss": 2.753,
      "step": 257800
    },
    {
      "activations/layer0_attention_weight_max": 14.73270320892334,
      "activations/layer0_attention_weight_min": -11.384862899780273,
      "activations/layer10_attention_weight_max": 32.94496536254883,
      "activations/layer10_attention_weight_min": -34.25460433959961,
      "activations/layer11_attention_weight_max": 32.779930114746094,
      "activations/layer11_attention_weight_min": -33.062477111816406,
      "activations/layer12_attention_weight_max": 26.557703018188477,
      "activations/layer12_attention_weight_min": -26.39994239807129,
      "activations/layer13_attention_weight_max": 39.69436264038086,
      "activations/layer13_attention_weight_min": -35.043087005615234,
      "activations/layer14_attention_weight_max": 40.977867126464844,
      "activations/layer14_attention_weight_min": -37.70044708251953,
      "activations/layer15_attention_weight_max": 38.0178337097168,
      "activations/layer15_attention_weight_min": -34.28360366821289,
      "activations/layer16_attention_weight_max": 31.913074493408203,
      "activations/layer16_attention_weight_min": -26.871475219726562,
      "activations/layer17_attention_weight_max": 30.78181266784668,
      "activations/layer17_attention_weight_min": -24.006668090820312,
      "activations/layer18_attention_weight_max": 32.65690994262695,
      "activations/layer18_attention_weight_min": -24.847246170043945,
      "activations/layer19_attention_weight_max": 31.911991119384766,
      "activations/layer19_attention_weight_min": -32.789581298828125,
      "activations/layer1_attention_weight_max": 16.469743728637695,
      "activations/layer1_attention_weight_min": -14.41640853881836,
      "activations/layer20_attention_weight_max": 28.612953186035156,
      "activations/layer20_attention_weight_min": -25.96380615234375,
      "activations/layer21_attention_weight_max": 28.27358627319336,
      "activations/layer21_attention_weight_min": -26.142534255981445,
      "activations/layer22_attention_weight_max": 44.11540222167969,
      "activations/layer22_attention_weight_min": -32.021236419677734,
      "activations/layer23_attention_weight_max": 30.20654296875,
      "activations/layer23_attention_weight_min": -26.327857971191406,
      "activations/layer2_attention_weight_max": 31.94258689880371,
      "activations/layer2_attention_weight_min": -32.25897979736328,
      "activations/layer3_attention_weight_max": 93.91986846923828,
      "activations/layer3_attention_weight_min": -93.95706939697266,
      "activations/layer4_attention_weight_max": 108.59100341796875,
      "activations/layer4_attention_weight_min": -105.45378875732422,
      "activations/layer5_attention_weight_max": 54.364723205566406,
      "activations/layer5_attention_weight_min": -62.64248275756836,
      "activations/layer6_attention_weight_max": 43.965354919433594,
      "activations/layer6_attention_weight_min": -44.89762878417969,
      "activations/layer7_attention_weight_max": 92.60086822509766,
      "activations/layer7_attention_weight_min": -90.34774017333984,
      "activations/layer8_attention_weight_max": 39.74858474731445,
      "activations/layer8_attention_weight_min": -41.329708099365234,
      "activations/layer9_attention_weight_max": 31.5993595123291,
      "activations/layer9_attention_weight_min": -34.55366516113281,
      "epoch": 14.98,
      "learning_rate": 5.392462121212121e-05,
      "loss": 2.7455,
      "step": 257850
    },
    {
      "activations/layer0_attention_weight_max": 15.598302841186523,
      "activations/layer0_attention_weight_min": -12.017792701721191,
      "activations/layer10_attention_weight_max": 37.19102096557617,
      "activations/layer10_attention_weight_min": -36.477752685546875,
      "activations/layer11_attention_weight_max": 38.89854431152344,
      "activations/layer11_attention_weight_min": -36.183555603027344,
      "activations/layer12_attention_weight_max": 30.115726470947266,
      "activations/layer12_attention_weight_min": -26.359216690063477,
      "activations/layer13_attention_weight_max": 40.04176712036133,
      "activations/layer13_attention_weight_min": -35.00259017944336,
      "activations/layer14_attention_weight_max": 46.705387115478516,
      "activations/layer14_attention_weight_min": -37.69468307495117,
      "activations/layer15_attention_weight_max": 38.200660705566406,
      "activations/layer15_attention_weight_min": -36.055904388427734,
      "activations/layer16_attention_weight_max": 31.091150283813477,
      "activations/layer16_attention_weight_min": -30.349437713623047,
      "activations/layer17_attention_weight_max": 33.52479934692383,
      "activations/layer17_attention_weight_min": -27.308713912963867,
      "activations/layer18_attention_weight_max": 32.660308837890625,
      "activations/layer18_attention_weight_min": -24.90121078491211,
      "activations/layer19_attention_weight_max": 35.60957336425781,
      "activations/layer19_attention_weight_min": -34.13282012939453,
      "activations/layer1_attention_weight_max": 16.493520736694336,
      "activations/layer1_attention_weight_min": -15.416516304016113,
      "activations/layer20_attention_weight_max": 31.853071212768555,
      "activations/layer20_attention_weight_min": -27.977266311645508,
      "activations/layer21_attention_weight_max": 29.403789520263672,
      "activations/layer21_attention_weight_min": -27.27751922607422,
      "activations/layer22_attention_weight_max": 48.15654373168945,
      "activations/layer22_attention_weight_min": -33.06532287597656,
      "activations/layer23_attention_weight_max": 36.785682678222656,
      "activations/layer23_attention_weight_min": -26.687543869018555,
      "activations/layer2_attention_weight_max": 37.31999206542969,
      "activations/layer2_attention_weight_min": -34.950584411621094,
      "activations/layer3_attention_weight_max": 98.9062271118164,
      "activations/layer3_attention_weight_min": -99.60673522949219,
      "activations/layer4_attention_weight_max": 109.78572845458984,
      "activations/layer4_attention_weight_min": -108.666748046875,
      "activations/layer5_attention_weight_max": 52.04922103881836,
      "activations/layer5_attention_weight_min": -63.114845275878906,
      "activations/layer6_attention_weight_max": 42.74769592285156,
      "activations/layer6_attention_weight_min": -45.6131706237793,
      "activations/layer7_attention_weight_max": 105.9798812866211,
      "activations/layer7_attention_weight_min": -94.97857666015625,
      "activations/layer8_attention_weight_max": 44.785789489746094,
      "activations/layer8_attention_weight_min": -44.21488952636719,
      "activations/layer9_attention_weight_max": 34.1277961730957,
      "activations/layer9_attention_weight_min": -35.160152435302734,
      "epoch": 14.99,
      "learning_rate": 5.3905681818181815e-05,
      "loss": 2.7416,
      "step": 257900
    },
    {
      "activations/layer0_attention_weight_max": 15.437238693237305,
      "activations/layer0_attention_weight_min": -12.167562484741211,
      "activations/layer10_attention_weight_max": 32.48897171020508,
      "activations/layer10_attention_weight_min": -33.31480026245117,
      "activations/layer11_attention_weight_max": 32.254730224609375,
      "activations/layer11_attention_weight_min": -35.301361083984375,
      "activations/layer12_attention_weight_max": 36.554805755615234,
      "activations/layer12_attention_weight_min": -26.508543014526367,
      "activations/layer13_attention_weight_max": 44.42377471923828,
      "activations/layer13_attention_weight_min": -35.878135681152344,
      "activations/layer14_attention_weight_max": 43.61455154418945,
      "activations/layer14_attention_weight_min": -40.86174774169922,
      "activations/layer15_attention_weight_max": 36.74551010131836,
      "activations/layer15_attention_weight_min": -35.17189025878906,
      "activations/layer16_attention_weight_max": 28.02117156982422,
      "activations/layer16_attention_weight_min": -28.796142578125,
      "activations/layer17_attention_weight_max": 27.027685165405273,
      "activations/layer17_attention_weight_min": -24.731576919555664,
      "activations/layer18_attention_weight_max": 28.051237106323242,
      "activations/layer18_attention_weight_min": -25.265216827392578,
      "activations/layer19_attention_weight_max": 29.97660255432129,
      "activations/layer19_attention_weight_min": -31.90602684020996,
      "activations/layer1_attention_weight_max": 16.813976287841797,
      "activations/layer1_attention_weight_min": -15.384010314941406,
      "activations/layer20_attention_weight_max": 27.867290496826172,
      "activations/layer20_attention_weight_min": -23.56332778930664,
      "activations/layer21_attention_weight_max": 28.268539428710938,
      "activations/layer21_attention_weight_min": -24.266918182373047,
      "activations/layer22_attention_weight_max": 42.26930236816406,
      "activations/layer22_attention_weight_min": -29.467451095581055,
      "activations/layer23_attention_weight_max": 29.469451904296875,
      "activations/layer23_attention_weight_min": -24.7574462890625,
      "activations/layer2_attention_weight_max": 31.651470184326172,
      "activations/layer2_attention_weight_min": -31.70319366455078,
      "activations/layer3_attention_weight_max": 100.40040588378906,
      "activations/layer3_attention_weight_min": -99.17232513427734,
      "activations/layer4_attention_weight_max": 109.86885833740234,
      "activations/layer4_attention_weight_min": -109.00359344482422,
      "activations/layer5_attention_weight_max": 53.69133758544922,
      "activations/layer5_attention_weight_min": -61.208770751953125,
      "activations/layer6_attention_weight_max": 47.6175422668457,
      "activations/layer6_attention_weight_min": -48.90021896362305,
      "activations/layer7_attention_weight_max": 94.2646255493164,
      "activations/layer7_attention_weight_min": -98.71370697021484,
      "activations/layer8_attention_weight_max": 43.179832458496094,
      "activations/layer8_attention_weight_min": -42.450687408447266,
      "activations/layer9_attention_weight_max": 33.18519973754883,
      "activations/layer9_attention_weight_min": -34.49135971069336,
      "epoch": 14.99,
      "learning_rate": 5.388674242424242e-05,
      "loss": 2.7459,
      "step": 257950
    },
    {
      "activations/layer0_attention_weight_max": 15.35496711730957,
      "activations/layer0_attention_weight_min": -12.481430053710938,
      "activations/layer10_attention_weight_max": 34.1306037902832,
      "activations/layer10_attention_weight_min": -33.19673156738281,
      "activations/layer11_attention_weight_max": 33.303558349609375,
      "activations/layer11_attention_weight_min": -31.257232666015625,
      "activations/layer12_attention_weight_max": 24.93712043762207,
      "activations/layer12_attention_weight_min": -28.267684936523438,
      "activations/layer13_attention_weight_max": 36.62664794921875,
      "activations/layer13_attention_weight_min": -33.814308166503906,
      "activations/layer14_attention_weight_max": 41.98782730102539,
      "activations/layer14_attention_weight_min": -35.5210075378418,
      "activations/layer15_attention_weight_max": 36.524559020996094,
      "activations/layer15_attention_weight_min": -34.832672119140625,
      "activations/layer16_attention_weight_max": 29.148107528686523,
      "activations/layer16_attention_weight_min": -30.45286750793457,
      "activations/layer17_attention_weight_max": 33.080108642578125,
      "activations/layer17_attention_weight_min": -28.204051971435547,
      "activations/layer18_attention_weight_max": 30.60822105407715,
      "activations/layer18_attention_weight_min": -24.454587936401367,
      "activations/layer19_attention_weight_max": 31.295242309570312,
      "activations/layer19_attention_weight_min": -32.63516616821289,
      "activations/layer1_attention_weight_max": 16.47553825378418,
      "activations/layer1_attention_weight_min": -14.26666259765625,
      "activations/layer20_attention_weight_max": 30.513469696044922,
      "activations/layer20_attention_weight_min": -25.44799041748047,
      "activations/layer21_attention_weight_max": 30.096487045288086,
      "activations/layer21_attention_weight_min": -27.45846939086914,
      "activations/layer22_attention_weight_max": 40.27819061279297,
      "activations/layer22_attention_weight_min": -31.045026779174805,
      "activations/layer23_attention_weight_max": 34.65998077392578,
      "activations/layer23_attention_weight_min": -24.59252166748047,
      "activations/layer2_attention_weight_max": 34.414817810058594,
      "activations/layer2_attention_weight_min": -32.04978561401367,
      "activations/layer3_attention_weight_max": 94.6114501953125,
      "activations/layer3_attention_weight_min": -94.76510620117188,
      "activations/layer4_attention_weight_max": 105.76000213623047,
      "activations/layer4_attention_weight_min": -104.1257553100586,
      "activations/layer5_attention_weight_max": 50.2376823425293,
      "activations/layer5_attention_weight_min": -61.206844329833984,
      "activations/layer6_attention_weight_max": 43.03548049926758,
      "activations/layer6_attention_weight_min": -46.75461959838867,
      "activations/layer7_attention_weight_max": 88.41466522216797,
      "activations/layer7_attention_weight_min": -90.11941528320312,
      "activations/layer8_attention_weight_max": 38.4611930847168,
      "activations/layer8_attention_weight_min": -41.04765701293945,
      "activations/layer9_attention_weight_max": 31.212629318237305,
      "activations/layer9_attention_weight_min": -33.01813507080078,
      "epoch": 14.99,
      "learning_rate": 5.3867803030303025e-05,
      "loss": 2.7646,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.5243,
      "eval_samples_per_second": 503.736,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.5243,
      "eval_openwebtext_samples_per_second": 503.736,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9867,
      "eval_wikitext_samples_per_second": 229.528,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_lambada_loss": 2.41015625,
      "eval_lambada_ppl": 11.135700962413912,
      "eval_lambada_runtime": 9.6215,
      "eval_lambada_samples_per_second": 506.055,
      "step": 258000
    },
    {
      "activations/layer0_attention_weight_max": 15.71811580657959,
      "activations/layer0_attention_weight_min": -11.67190933227539,
      "activations/layer10_attention_weight_max": 36.6668815612793,
      "activations/layer10_attention_weight_min": -36.843990325927734,
      "activations/layer11_attention_weight_max": 34.503173828125,
      "activations/layer11_attention_weight_min": -35.41082000732422,
      "activations/layer12_attention_weight_max": 26.473087310791016,
      "activations/layer12_attention_weight_min": -26.130395889282227,
      "activations/layer13_attention_weight_max": 40.483489990234375,
      "activations/layer13_attention_weight_min": -35.26319122314453,
      "activations/layer14_attention_weight_max": 43.3233528137207,
      "activations/layer14_attention_weight_min": -38.76992416381836,
      "activations/layer15_attention_weight_max": 40.53474807739258,
      "activations/layer15_attention_weight_min": -36.9001579284668,
      "activations/layer16_attention_weight_max": 31.008283615112305,
      "activations/layer16_attention_weight_min": -29.84352684020996,
      "activations/layer17_attention_weight_max": 32.79487228393555,
      "activations/layer17_attention_weight_min": -28.635196685791016,
      "activations/layer18_attention_weight_max": 32.0329475402832,
      "activations/layer18_attention_weight_min": -27.031667709350586,
      "activations/layer19_attention_weight_max": 35.60558319091797,
      "activations/layer19_attention_weight_min": -31.405654907226562,
      "activations/layer1_attention_weight_max": 17.8848876953125,
      "activations/layer1_attention_weight_min": -15.450129508972168,
      "activations/layer20_attention_weight_max": 32.44581985473633,
      "activations/layer20_attention_weight_min": -28.611988067626953,
      "activations/layer21_attention_weight_max": 32.62272262573242,
      "activations/layer21_attention_weight_min": -26.736919403076172,
      "activations/layer22_attention_weight_max": 46.60794448852539,
      "activations/layer22_attention_weight_min": -33.27263641357422,
      "activations/layer23_attention_weight_max": 34.66098403930664,
      "activations/layer23_attention_weight_min": -27.15872573852539,
      "activations/layer2_attention_weight_max": 34.21440505981445,
      "activations/layer2_attention_weight_min": -35.16890335083008,
      "activations/layer3_attention_weight_max": 99.75152587890625,
      "activations/layer3_attention_weight_min": -103.14615631103516,
      "activations/layer4_attention_weight_max": 110.8712158203125,
      "activations/layer4_attention_weight_min": -110.072265625,
      "activations/layer5_attention_weight_max": 55.14463806152344,
      "activations/layer5_attention_weight_min": -64.91737365722656,
      "activations/layer6_attention_weight_max": 47.07809829711914,
      "activations/layer6_attention_weight_min": -48.03907012939453,
      "activations/layer7_attention_weight_max": 96.05728912353516,
      "activations/layer7_attention_weight_min": -96.68215942382812,
      "activations/layer8_attention_weight_max": 45.12301254272461,
      "activations/layer8_attention_weight_min": -44.9711799621582,
      "activations/layer9_attention_weight_max": 34.528499603271484,
      "activations/layer9_attention_weight_min": -36.1522216796875,
      "epoch": 14.99,
      "learning_rate": 5.3848863636363626e-05,
      "loss": 2.7426,
      "step": 258050
    },
    {
      "activations/layer0_attention_weight_max": 15.823585510253906,
      "activations/layer0_attention_weight_min": -11.855716705322266,
      "activations/layer10_attention_weight_max": 36.04096221923828,
      "activations/layer10_attention_weight_min": -32.20853805541992,
      "activations/layer11_attention_weight_max": 33.62135314941406,
      "activations/layer11_attention_weight_min": -32.0517578125,
      "activations/layer12_attention_weight_max": 27.335126876831055,
      "activations/layer12_attention_weight_min": -27.385404586791992,
      "activations/layer13_attention_weight_max": 40.806549072265625,
      "activations/layer13_attention_weight_min": -34.78017044067383,
      "activations/layer14_attention_weight_max": 47.14162063598633,
      "activations/layer14_attention_weight_min": -38.48103332519531,
      "activations/layer15_attention_weight_max": 41.14588928222656,
      "activations/layer15_attention_weight_min": -34.90958786010742,
      "activations/layer16_attention_weight_max": 35.12599182128906,
      "activations/layer16_attention_weight_min": -29.242111206054688,
      "activations/layer17_attention_weight_max": 32.576820373535156,
      "activations/layer17_attention_weight_min": -27.547704696655273,
      "activations/layer18_attention_weight_max": 32.47713088989258,
      "activations/layer18_attention_weight_min": -26.392248153686523,
      "activations/layer19_attention_weight_max": 38.64266586303711,
      "activations/layer19_attention_weight_min": -32.445884704589844,
      "activations/layer1_attention_weight_max": 16.308212280273438,
      "activations/layer1_attention_weight_min": -14.82119369506836,
      "activations/layer20_attention_weight_max": 32.60641098022461,
      "activations/layer20_attention_weight_min": -25.717506408691406,
      "activations/layer21_attention_weight_max": 32.952972412109375,
      "activations/layer21_attention_weight_min": -25.81058692932129,
      "activations/layer22_attention_weight_max": 52.87054443359375,
      "activations/layer22_attention_weight_min": -33.316551208496094,
      "activations/layer23_attention_weight_max": 36.19667053222656,
      "activations/layer23_attention_weight_min": -28.13788604736328,
      "activations/layer2_attention_weight_max": 32.48211669921875,
      "activations/layer2_attention_weight_min": -30.24356460571289,
      "activations/layer3_attention_weight_max": 98.09770965576172,
      "activations/layer3_attention_weight_min": -92.94721984863281,
      "activations/layer4_attention_weight_max": 109.4404067993164,
      "activations/layer4_attention_weight_min": -111.96810150146484,
      "activations/layer5_attention_weight_max": 54.033409118652344,
      "activations/layer5_attention_weight_min": -62.28220748901367,
      "activations/layer6_attention_weight_max": 44.19780731201172,
      "activations/layer6_attention_weight_min": -45.338844299316406,
      "activations/layer7_attention_weight_max": 95.597900390625,
      "activations/layer7_attention_weight_min": -91.01077270507812,
      "activations/layer8_attention_weight_max": 42.74182891845703,
      "activations/layer8_attention_weight_min": -40.11526107788086,
      "activations/layer9_attention_weight_max": 33.62070846557617,
      "activations/layer9_attention_weight_min": -32.337425231933594,
      "epoch": 15.0,
      "learning_rate": 5.382992424242424e-05,
      "loss": 2.7568,
      "step": 258100
    },
    {
      "activations/layer0_attention_weight_max": 15.36392879486084,
      "activations/layer0_attention_weight_min": -11.13410472869873,
      "activations/layer10_attention_weight_max": 31.865535736083984,
      "activations/layer10_attention_weight_min": -32.6871337890625,
      "activations/layer11_attention_weight_max": 31.49825668334961,
      "activations/layer11_attention_weight_min": -33.93741989135742,
      "activations/layer12_attention_weight_max": 25.94721221923828,
      "activations/layer12_attention_weight_min": -25.923669815063477,
      "activations/layer13_attention_weight_max": 37.32088851928711,
      "activations/layer13_attention_weight_min": -35.81173324584961,
      "activations/layer14_attention_weight_max": 42.55141830444336,
      "activations/layer14_attention_weight_min": -37.005802154541016,
      "activations/layer15_attention_weight_max": 36.57628631591797,
      "activations/layer15_attention_weight_min": -35.51187515258789,
      "activations/layer16_attention_weight_max": 27.574203491210938,
      "activations/layer16_attention_weight_min": -27.79094123840332,
      "activations/layer17_attention_weight_max": 28.833600997924805,
      "activations/layer17_attention_weight_min": -25.8801326751709,
      "activations/layer18_attention_weight_max": 32.792320251464844,
      "activations/layer18_attention_weight_min": -24.65359115600586,
      "activations/layer19_attention_weight_max": 34.041114807128906,
      "activations/layer19_attention_weight_min": -31.412281036376953,
      "activations/layer1_attention_weight_max": 16.837650299072266,
      "activations/layer1_attention_weight_min": -15.543309211730957,
      "activations/layer20_attention_weight_max": 28.491336822509766,
      "activations/layer20_attention_weight_min": -27.269567489624023,
      "activations/layer21_attention_weight_max": 29.72268295288086,
      "activations/layer21_attention_weight_min": -26.783292770385742,
      "activations/layer22_attention_weight_max": 43.71092224121094,
      "activations/layer22_attention_weight_min": -32.673362731933594,
      "activations/layer23_attention_weight_max": 34.59253692626953,
      "activations/layer23_attention_weight_min": -26.500913619995117,
      "activations/layer2_attention_weight_max": 33.07215118408203,
      "activations/layer2_attention_weight_min": -32.21775817871094,
      "activations/layer3_attention_weight_max": 94.3744888305664,
      "activations/layer3_attention_weight_min": -94.81925964355469,
      "activations/layer4_attention_weight_max": 107.1412582397461,
      "activations/layer4_attention_weight_min": -106.2747802734375,
      "activations/layer5_attention_weight_max": 51.680259704589844,
      "activations/layer5_attention_weight_min": -62.480712890625,
      "activations/layer6_attention_weight_max": 42.309791564941406,
      "activations/layer6_attention_weight_min": -46.252723693847656,
      "activations/layer7_attention_weight_max": 88.13471984863281,
      "activations/layer7_attention_weight_min": -93.33037567138672,
      "activations/layer8_attention_weight_max": 40.517127990722656,
      "activations/layer8_attention_weight_min": -42.938819885253906,
      "activations/layer9_attention_weight_max": 30.585744857788086,
      "activations/layer9_attention_weight_min": -32.50325393676758,
      "epoch": 15.0,
      "learning_rate": 5.381098484848484e-05,
      "loss": 2.7601,
      "step": 258150
    },
    {
      "activations/layer0_attention_weight_max": 14.964820861816406,
      "activations/layer0_attention_weight_min": -11.819602966308594,
      "activations/layer10_attention_weight_max": 32.794342041015625,
      "activations/layer10_attention_weight_min": -33.33681106567383,
      "activations/layer11_attention_weight_max": 32.47222137451172,
      "activations/layer11_attention_weight_min": -33.24716567993164,
      "activations/layer12_attention_weight_max": 31.768537521362305,
      "activations/layer12_attention_weight_min": -27.011959075927734,
      "activations/layer13_attention_weight_max": 42.289947509765625,
      "activations/layer13_attention_weight_min": -35.661705017089844,
      "activations/layer14_attention_weight_max": 46.172977447509766,
      "activations/layer14_attention_weight_min": -38.046024322509766,
      "activations/layer15_attention_weight_max": 41.36649703979492,
      "activations/layer15_attention_weight_min": -34.574302673339844,
      "activations/layer16_attention_weight_max": 33.3156852722168,
      "activations/layer16_attention_weight_min": -29.62640953063965,
      "activations/layer17_attention_weight_max": 30.753313064575195,
      "activations/layer17_attention_weight_min": -27.960159301757812,
      "activations/layer18_attention_weight_max": 34.42193603515625,
      "activations/layer18_attention_weight_min": -25.481760025024414,
      "activations/layer19_attention_weight_max": 39.188995361328125,
      "activations/layer19_attention_weight_min": -32.07624816894531,
      "activations/layer1_attention_weight_max": 15.859236717224121,
      "activations/layer1_attention_weight_min": -14.187031745910645,
      "activations/layer20_attention_weight_max": 33.798526763916016,
      "activations/layer20_attention_weight_min": -28.110050201416016,
      "activations/layer21_attention_weight_max": 30.93251609802246,
      "activations/layer21_attention_weight_min": -25.67399024963379,
      "activations/layer22_attention_weight_max": 48.68030548095703,
      "activations/layer22_attention_weight_min": -31.714244842529297,
      "activations/layer23_attention_weight_max": 36.04941177368164,
      "activations/layer23_attention_weight_min": -24.970731735229492,
      "activations/layer2_attention_weight_max": 32.43464660644531,
      "activations/layer2_attention_weight_min": -31.968975067138672,
      "activations/layer3_attention_weight_max": 95.52350616455078,
      "activations/layer3_attention_weight_min": -93.00631713867188,
      "activations/layer4_attention_weight_max": 109.94062805175781,
      "activations/layer4_attention_weight_min": -110.90084075927734,
      "activations/layer5_attention_weight_max": 51.29954528808594,
      "activations/layer5_attention_weight_min": -65.3428955078125,
      "activations/layer6_attention_weight_max": 44.14452362060547,
      "activations/layer6_attention_weight_min": -48.49818420410156,
      "activations/layer7_attention_weight_max": 92.06249237060547,
      "activations/layer7_attention_weight_min": -97.52088928222656,
      "activations/layer8_attention_weight_max": 39.93803787231445,
      "activations/layer8_attention_weight_min": -42.6436882019043,
      "activations/layer9_attention_weight_max": 32.02806091308594,
      "activations/layer9_attention_weight_min": -33.298973083496094,
      "epoch": 15.0,
      "learning_rate": 5.379166666666666e-05,
      "loss": 2.7991,
      "step": 258200
    },
    {
      "activations/layer0_attention_weight_max": 15.622700691223145,
      "activations/layer0_attention_weight_min": -12.440695762634277,
      "activations/layer10_attention_weight_max": 32.54536056518555,
      "activations/layer10_attention_weight_min": -32.855953216552734,
      "activations/layer11_attention_weight_max": 31.68350601196289,
      "activations/layer11_attention_weight_min": -32.80268859863281,
      "activations/layer12_attention_weight_max": 24.801420211791992,
      "activations/layer12_attention_weight_min": -25.505720138549805,
      "activations/layer13_attention_weight_max": 38.29389572143555,
      "activations/layer13_attention_weight_min": -35.13369369506836,
      "activations/layer14_attention_weight_max": 42.949241638183594,
      "activations/layer14_attention_weight_min": -38.89527130126953,
      "activations/layer15_attention_weight_max": 34.13515090942383,
      "activations/layer15_attention_weight_min": -34.60490798950195,
      "activations/layer16_attention_weight_max": 28.95136833190918,
      "activations/layer16_attention_weight_min": -28.184267044067383,
      "activations/layer17_attention_weight_max": 28.98696517944336,
      "activations/layer17_attention_weight_min": -26.700057983398438,
      "activations/layer18_attention_weight_max": 29.9346866607666,
      "activations/layer18_attention_weight_min": -24.286901473999023,
      "activations/layer19_attention_weight_max": 32.93522644042969,
      "activations/layer19_attention_weight_min": -31.870386123657227,
      "activations/layer1_attention_weight_max": 17.030855178833008,
      "activations/layer1_attention_weight_min": -13.423571586608887,
      "activations/layer20_attention_weight_max": 28.27355194091797,
      "activations/layer20_attention_weight_min": -24.78173065185547,
      "activations/layer21_attention_weight_max": 28.415924072265625,
      "activations/layer21_attention_weight_min": -26.046646118164062,
      "activations/layer22_attention_weight_max": 40.40961456298828,
      "activations/layer22_attention_weight_min": -31.009288787841797,
      "activations/layer23_attention_weight_max": 35.71664810180664,
      "activations/layer23_attention_weight_min": -26.667469024658203,
      "activations/layer2_attention_weight_max": 33.651519775390625,
      "activations/layer2_attention_weight_min": -32.23423767089844,
      "activations/layer3_attention_weight_max": 91.17611694335938,
      "activations/layer3_attention_weight_min": -91.26545715332031,
      "activations/layer4_attention_weight_max": 106.13028717041016,
      "activations/layer4_attention_weight_min": -107.47020721435547,
      "activations/layer5_attention_weight_max": 49.46822738647461,
      "activations/layer5_attention_weight_min": -64.33616638183594,
      "activations/layer6_attention_weight_max": 43.507423400878906,
      "activations/layer6_attention_weight_min": -45.91191482543945,
      "activations/layer7_attention_weight_max": 86.86145782470703,
      "activations/layer7_attention_weight_min": -90.1009750366211,
      "activations/layer8_attention_weight_max": 41.82194137573242,
      "activations/layer8_attention_weight_min": -40.30803298950195,
      "activations/layer9_attention_weight_max": 30.62026023864746,
      "activations/layer9_attention_weight_min": -32.93602752685547,
      "epoch": 15.01,
      "learning_rate": 5.377272727272726e-05,
      "loss": 2.747,
      "step": 258250
    },
    {
      "activations/layer0_attention_weight_max": 16.13343048095703,
      "activations/layer0_attention_weight_min": -12.297805786132812,
      "activations/layer10_attention_weight_max": 32.05116653442383,
      "activations/layer10_attention_weight_min": -30.90157127380371,
      "activations/layer11_attention_weight_max": 32.60814666748047,
      "activations/layer11_attention_weight_min": -31.21587371826172,
      "activations/layer12_attention_weight_max": 23.407800674438477,
      "activations/layer12_attention_weight_min": -24.697343826293945,
      "activations/layer13_attention_weight_max": 38.424072265625,
      "activations/layer13_attention_weight_min": -32.82679748535156,
      "activations/layer14_attention_weight_max": 40.39759063720703,
      "activations/layer14_attention_weight_min": -35.745601654052734,
      "activations/layer15_attention_weight_max": 33.949363708496094,
      "activations/layer15_attention_weight_min": -31.702844619750977,
      "activations/layer16_attention_weight_max": 27.44297218322754,
      "activations/layer16_attention_weight_min": -27.431612014770508,
      "activations/layer17_attention_weight_max": 28.15157699584961,
      "activations/layer17_attention_weight_min": -25.156587600708008,
      "activations/layer18_attention_weight_max": 30.59373664855957,
      "activations/layer18_attention_weight_min": -24.231525421142578,
      "activations/layer19_attention_weight_max": 32.315757751464844,
      "activations/layer19_attention_weight_min": -33.47224044799805,
      "activations/layer1_attention_weight_max": 15.83586597442627,
      "activations/layer1_attention_weight_min": -13.323014259338379,
      "activations/layer20_attention_weight_max": 29.49937629699707,
      "activations/layer20_attention_weight_min": -27.122817993164062,
      "activations/layer21_attention_weight_max": 27.30257797241211,
      "activations/layer21_attention_weight_min": -28.015287399291992,
      "activations/layer22_attention_weight_max": 37.99217224121094,
      "activations/layer22_attention_weight_min": -30.990234375,
      "activations/layer23_attention_weight_max": 33.292232513427734,
      "activations/layer23_attention_weight_min": -28.35466766357422,
      "activations/layer2_attention_weight_max": 32.33268356323242,
      "activations/layer2_attention_weight_min": -31.718223571777344,
      "activations/layer3_attention_weight_max": 92.9561538696289,
      "activations/layer3_attention_weight_min": -93.97534942626953,
      "activations/layer4_attention_weight_max": 107.442138671875,
      "activations/layer4_attention_weight_min": -102.20594024658203,
      "activations/layer5_attention_weight_max": 51.986083984375,
      "activations/layer5_attention_weight_min": -58.59804916381836,
      "activations/layer6_attention_weight_max": 41.68259811401367,
      "activations/layer6_attention_weight_min": -42.44792938232422,
      "activations/layer7_attention_weight_max": 88.45242309570312,
      "activations/layer7_attention_weight_min": -84.5145034790039,
      "activations/layer8_attention_weight_max": 38.29872512817383,
      "activations/layer8_attention_weight_min": -38.23435974121094,
      "activations/layer9_attention_weight_max": 31.87030601501465,
      "activations/layer9_attention_weight_min": -30.365110397338867,
      "epoch": 15.01,
      "learning_rate": 5.375378787878788e-05,
      "loss": 2.7348,
      "step": 258300
    },
    {
      "activations/layer0_attention_weight_max": 15.897514343261719,
      "activations/layer0_attention_weight_min": -11.590181350708008,
      "activations/layer10_attention_weight_max": 32.514286041259766,
      "activations/layer10_attention_weight_min": -31.34817123413086,
      "activations/layer11_attention_weight_max": 34.32013702392578,
      "activations/layer11_attention_weight_min": -32.64820861816406,
      "activations/layer12_attention_weight_max": 25.98859977722168,
      "activations/layer12_attention_weight_min": -25.43183135986328,
      "activations/layer13_attention_weight_max": 36.35345458984375,
      "activations/layer13_attention_weight_min": -33.476497650146484,
      "activations/layer14_attention_weight_max": 39.86977767944336,
      "activations/layer14_attention_weight_min": -36.47910690307617,
      "activations/layer15_attention_weight_max": 36.600852966308594,
      "activations/layer15_attention_weight_min": -31.434402465820312,
      "activations/layer16_attention_weight_max": 29.852012634277344,
      "activations/layer16_attention_weight_min": -25.760568618774414,
      "activations/layer17_attention_weight_max": 32.1860237121582,
      "activations/layer17_attention_weight_min": -26.787385940551758,
      "activations/layer18_attention_weight_max": 31.983016967773438,
      "activations/layer18_attention_weight_min": -26.139034271240234,
      "activations/layer19_attention_weight_max": 32.9849967956543,
      "activations/layer19_attention_weight_min": -32.90174865722656,
      "activations/layer1_attention_weight_max": 16.21556282043457,
      "activations/layer1_attention_weight_min": -13.934151649475098,
      "activations/layer20_attention_weight_max": 29.34663963317871,
      "activations/layer20_attention_weight_min": -25.798931121826172,
      "activations/layer21_attention_weight_max": 30.589956283569336,
      "activations/layer21_attention_weight_min": -28.243045806884766,
      "activations/layer22_attention_weight_max": 39.94607925415039,
      "activations/layer22_attention_weight_min": -34.902530670166016,
      "activations/layer23_attention_weight_max": 34.03213882446289,
      "activations/layer23_attention_weight_min": -29.654266357421875,
      "activations/layer2_attention_weight_max": 31.50737762451172,
      "activations/layer2_attention_weight_min": -31.248199462890625,
      "activations/layer3_attention_weight_max": 89.77652740478516,
      "activations/layer3_attention_weight_min": -92.28060150146484,
      "activations/layer4_attention_weight_max": 104.2676010131836,
      "activations/layer4_attention_weight_min": -102.02621459960938,
      "activations/layer5_attention_weight_max": 53.51858139038086,
      "activations/layer5_attention_weight_min": -57.52428436279297,
      "activations/layer6_attention_weight_max": 42.4924430847168,
      "activations/layer6_attention_weight_min": -43.6856575012207,
      "activations/layer7_attention_weight_max": 88.1761703491211,
      "activations/layer7_attention_weight_min": -82.09610748291016,
      "activations/layer8_attention_weight_max": 40.4022331237793,
      "activations/layer8_attention_weight_min": -38.30855178833008,
      "activations/layer9_attention_weight_max": 32.458072662353516,
      "activations/layer9_attention_weight_min": -32.30402755737305,
      "epoch": 15.01,
      "learning_rate": 5.373484848484848e-05,
      "loss": 2.7517,
      "step": 258350
    },
    {
      "activations/layer0_attention_weight_max": 15.428812980651855,
      "activations/layer0_attention_weight_min": -11.824295043945312,
      "activations/layer10_attention_weight_max": 32.971954345703125,
      "activations/layer10_attention_weight_min": -32.448421478271484,
      "activations/layer11_attention_weight_max": 32.21431350708008,
      "activations/layer11_attention_weight_min": -31.16104507446289,
      "activations/layer12_attention_weight_max": 26.76906967163086,
      "activations/layer12_attention_weight_min": -26.71988296508789,
      "activations/layer13_attention_weight_max": 41.75773239135742,
      "activations/layer13_attention_weight_min": -35.06562423706055,
      "activations/layer14_attention_weight_max": 39.45319747924805,
      "activations/layer14_attention_weight_min": -38.56266403198242,
      "activations/layer15_attention_weight_max": 36.49826431274414,
      "activations/layer15_attention_weight_min": -32.640316009521484,
      "activations/layer16_attention_weight_max": 29.34636116027832,
      "activations/layer16_attention_weight_min": -29.126708984375,
      "activations/layer17_attention_weight_max": 31.110342025756836,
      "activations/layer17_attention_weight_min": -25.050302505493164,
      "activations/layer18_attention_weight_max": 31.585716247558594,
      "activations/layer18_attention_weight_min": -25.124237060546875,
      "activations/layer19_attention_weight_max": 35.38574981689453,
      "activations/layer19_attention_weight_min": -30.90567970275879,
      "activations/layer1_attention_weight_max": 16.647066116333008,
      "activations/layer1_attention_weight_min": -14.505982398986816,
      "activations/layer20_attention_weight_max": 31.017311096191406,
      "activations/layer20_attention_weight_min": -25.375886917114258,
      "activations/layer21_attention_weight_max": 29.966739654541016,
      "activations/layer21_attention_weight_min": -24.305870056152344,
      "activations/layer22_attention_weight_max": 40.36686706542969,
      "activations/layer22_attention_weight_min": -30.45895767211914,
      "activations/layer23_attention_weight_max": 33.40760040283203,
      "activations/layer23_attention_weight_min": -28.36214256286621,
      "activations/layer2_attention_weight_max": 30.959522247314453,
      "activations/layer2_attention_weight_min": -29.77328872680664,
      "activations/layer3_attention_weight_max": 91.07466125488281,
      "activations/layer3_attention_weight_min": -94.88804626464844,
      "activations/layer4_attention_weight_max": 106.65718841552734,
      "activations/layer4_attention_weight_min": -108.31634521484375,
      "activations/layer5_attention_weight_max": 50.18016815185547,
      "activations/layer5_attention_weight_min": -60.19352722167969,
      "activations/layer6_attention_weight_max": 42.650299072265625,
      "activations/layer6_attention_weight_min": -46.36838150024414,
      "activations/layer7_attention_weight_max": 92.72844696044922,
      "activations/layer7_attention_weight_min": -96.38713836669922,
      "activations/layer8_attention_weight_max": 38.879905700683594,
      "activations/layer8_attention_weight_min": -41.28268814086914,
      "activations/layer9_attention_weight_max": 32.18330764770508,
      "activations/layer9_attention_weight_min": -33.127777099609375,
      "epoch": 15.01,
      "learning_rate": 5.371590909090908e-05,
      "loss": 2.7275,
      "step": 258400
    },
    {
      "activations/layer0_attention_weight_max": 16.080598831176758,
      "activations/layer0_attention_weight_min": -11.703620910644531,
      "activations/layer10_attention_weight_max": 31.832717895507812,
      "activations/layer10_attention_weight_min": -32.4559211730957,
      "activations/layer11_attention_weight_max": 32.8172607421875,
      "activations/layer11_attention_weight_min": -32.75403594970703,
      "activations/layer12_attention_weight_max": 24.512842178344727,
      "activations/layer12_attention_weight_min": -25.277788162231445,
      "activations/layer13_attention_weight_max": 36.57951736450195,
      "activations/layer13_attention_weight_min": -35.91035842895508,
      "activations/layer14_attention_weight_max": 40.65079879760742,
      "activations/layer14_attention_weight_min": -38.90580749511719,
      "activations/layer15_attention_weight_max": 36.1558952331543,
      "activations/layer15_attention_weight_min": -35.7980842590332,
      "activations/layer16_attention_weight_max": 28.17384910583496,
      "activations/layer16_attention_weight_min": -27.964466094970703,
      "activations/layer17_attention_weight_max": 31.070098876953125,
      "activations/layer17_attention_weight_min": -26.63484764099121,
      "activations/layer18_attention_weight_max": 30.221256256103516,
      "activations/layer18_attention_weight_min": -24.98056983947754,
      "activations/layer19_attention_weight_max": 33.82993698120117,
      "activations/layer19_attention_weight_min": -30.68636131286621,
      "activations/layer1_attention_weight_max": 16.1888484954834,
      "activations/layer1_attention_weight_min": -14.507357597351074,
      "activations/layer20_attention_weight_max": 28.19230842590332,
      "activations/layer20_attention_weight_min": -25.970317840576172,
      "activations/layer21_attention_weight_max": 29.016759872436523,
      "activations/layer21_attention_weight_min": -25.35515785217285,
      "activations/layer22_attention_weight_max": 43.00785446166992,
      "activations/layer22_attention_weight_min": -30.97953987121582,
      "activations/layer23_attention_weight_max": 35.57707977294922,
      "activations/layer23_attention_weight_min": -24.879196166992188,
      "activations/layer2_attention_weight_max": 31.946666717529297,
      "activations/layer2_attention_weight_min": -30.95690155029297,
      "activations/layer3_attention_weight_max": 95.374755859375,
      "activations/layer3_attention_weight_min": -94.70915985107422,
      "activations/layer4_attention_weight_max": 109.7834243774414,
      "activations/layer4_attention_weight_min": -109.92028045654297,
      "activations/layer5_attention_weight_max": 51.0120849609375,
      "activations/layer5_attention_weight_min": -65.421630859375,
      "activations/layer6_attention_weight_max": 44.114200592041016,
      "activations/layer6_attention_weight_min": -47.979732513427734,
      "activations/layer7_attention_weight_max": 89.03522491455078,
      "activations/layer7_attention_weight_min": -94.33963775634766,
      "activations/layer8_attention_weight_max": 39.321006774902344,
      "activations/layer8_attention_weight_min": -41.585472106933594,
      "activations/layer9_attention_weight_max": 31.251556396484375,
      "activations/layer9_attention_weight_min": -33.11872100830078,
      "epoch": 15.02,
      "learning_rate": 5.369696969696969e-05,
      "loss": 2.7468,
      "step": 258450
    },
    {
      "activations/layer0_attention_weight_max": 15.93187141418457,
      "activations/layer0_attention_weight_min": -11.68154525756836,
      "activations/layer10_attention_weight_max": 31.979381561279297,
      "activations/layer10_attention_weight_min": -31.905929565429688,
      "activations/layer11_attention_weight_max": 34.62523651123047,
      "activations/layer11_attention_weight_min": -31.60175895690918,
      "activations/layer12_attention_weight_max": 26.47986602783203,
      "activations/layer12_attention_weight_min": -28.270084381103516,
      "activations/layer13_attention_weight_max": 39.47411346435547,
      "activations/layer13_attention_weight_min": -37.75752639770508,
      "activations/layer14_attention_weight_max": 41.966556549072266,
      "activations/layer14_attention_weight_min": -39.20006561279297,
      "activations/layer15_attention_weight_max": 37.305423736572266,
      "activations/layer15_attention_weight_min": -34.9546012878418,
      "activations/layer16_attention_weight_max": 29.512802124023438,
      "activations/layer16_attention_weight_min": -26.815757751464844,
      "activations/layer17_attention_weight_max": 33.4715690612793,
      "activations/layer17_attention_weight_min": -25.950469970703125,
      "activations/layer18_attention_weight_max": 33.787269592285156,
      "activations/layer18_attention_weight_min": -25.073265075683594,
      "activations/layer19_attention_weight_max": 32.365116119384766,
      "activations/layer19_attention_weight_min": -31.940200805664062,
      "activations/layer1_attention_weight_max": 16.02640151977539,
      "activations/layer1_attention_weight_min": -16.7528133392334,
      "activations/layer20_attention_weight_max": 29.038352966308594,
      "activations/layer20_attention_weight_min": -24.71266746520996,
      "activations/layer21_attention_weight_max": 30.445953369140625,
      "activations/layer21_attention_weight_min": -24.24460792541504,
      "activations/layer22_attention_weight_max": 41.06330871582031,
      "activations/layer22_attention_weight_min": -30.839502334594727,
      "activations/layer23_attention_weight_max": 34.10313034057617,
      "activations/layer23_attention_weight_min": -25.01681900024414,
      "activations/layer2_attention_weight_max": 29.537837982177734,
      "activations/layer2_attention_weight_min": -30.495861053466797,
      "activations/layer3_attention_weight_max": 91.1589126586914,
      "activations/layer3_attention_weight_min": -95.220947265625,
      "activations/layer4_attention_weight_max": 104.73699188232422,
      "activations/layer4_attention_weight_min": -114.88298797607422,
      "activations/layer5_attention_weight_max": 51.118019104003906,
      "activations/layer5_attention_weight_min": -63.413856506347656,
      "activations/layer6_attention_weight_max": 42.833492279052734,
      "activations/layer6_attention_weight_min": -46.28689956665039,
      "activations/layer7_attention_weight_max": 89.44828033447266,
      "activations/layer7_attention_weight_min": -91.93881225585938,
      "activations/layer8_attention_weight_max": 39.57093811035156,
      "activations/layer8_attention_weight_min": -41.41741943359375,
      "activations/layer9_attention_weight_max": 32.28791809082031,
      "activations/layer9_attention_weight_min": -31.53814697265625,
      "epoch": 15.02,
      "learning_rate": 5.36780303030303e-05,
      "loss": 2.7454,
      "step": 258500
    },
    {
      "activations/layer0_attention_weight_max": 14.866545677185059,
      "activations/layer0_attention_weight_min": -11.070175170898438,
      "activations/layer10_attention_weight_max": 34.20429611206055,
      "activations/layer10_attention_weight_min": -34.628662109375,
      "activations/layer11_attention_weight_max": 32.78216552734375,
      "activations/layer11_attention_weight_min": -33.62889862060547,
      "activations/layer12_attention_weight_max": 26.805322647094727,
      "activations/layer12_attention_weight_min": -26.815353393554688,
      "activations/layer13_attention_weight_max": 40.93418502807617,
      "activations/layer13_attention_weight_min": -38.412925720214844,
      "activations/layer14_attention_weight_max": 44.784732818603516,
      "activations/layer14_attention_weight_min": -39.799991607666016,
      "activations/layer15_attention_weight_max": 39.133583068847656,
      "activations/layer15_attention_weight_min": -36.15439987182617,
      "activations/layer16_attention_weight_max": 33.14959716796875,
      "activations/layer16_attention_weight_min": -29.329668045043945,
      "activations/layer17_attention_weight_max": 30.8784122467041,
      "activations/layer17_attention_weight_min": -26.280839920043945,
      "activations/layer18_attention_weight_max": 34.63665008544922,
      "activations/layer18_attention_weight_min": -25.392593383789062,
      "activations/layer19_attention_weight_max": 40.804927825927734,
      "activations/layer19_attention_weight_min": -33.89830780029297,
      "activations/layer1_attention_weight_max": 16.294130325317383,
      "activations/layer1_attention_weight_min": -14.560118675231934,
      "activations/layer20_attention_weight_max": 33.543399810791016,
      "activations/layer20_attention_weight_min": -27.110754013061523,
      "activations/layer21_attention_weight_max": 31.79521942138672,
      "activations/layer21_attention_weight_min": -25.452451705932617,
      "activations/layer22_attention_weight_max": 47.830848693847656,
      "activations/layer22_attention_weight_min": -31.964698791503906,
      "activations/layer23_attention_weight_max": 35.58372497558594,
      "activations/layer23_attention_weight_min": -27.268024444580078,
      "activations/layer2_attention_weight_max": 32.438838958740234,
      "activations/layer2_attention_weight_min": -30.844696044921875,
      "activations/layer3_attention_weight_max": 95.21955108642578,
      "activations/layer3_attention_weight_min": -94.41928100585938,
      "activations/layer4_attention_weight_max": 110.033935546875,
      "activations/layer4_attention_weight_min": -110.70997619628906,
      "activations/layer5_attention_weight_max": 52.31061553955078,
      "activations/layer5_attention_weight_min": -62.4433708190918,
      "activations/layer6_attention_weight_max": 45.38172149658203,
      "activations/layer6_attention_weight_min": -47.81425476074219,
      "activations/layer7_attention_weight_max": 92.31352996826172,
      "activations/layer7_attention_weight_min": -95.53116607666016,
      "activations/layer8_attention_weight_max": 40.64703369140625,
      "activations/layer8_attention_weight_min": -43.807254791259766,
      "activations/layer9_attention_weight_max": 32.59541702270508,
      "activations/layer9_attention_weight_min": -34.23158645629883,
      "epoch": 15.02,
      "learning_rate": 5.3659090909090906e-05,
      "loss": 2.7445,
      "step": 258550
    },
    {
      "activations/layer0_attention_weight_max": 15.04520320892334,
      "activations/layer0_attention_weight_min": -11.87509536743164,
      "activations/layer10_attention_weight_max": 36.524444580078125,
      "activations/layer10_attention_weight_min": -36.455963134765625,
      "activations/layer11_attention_weight_max": 37.522125244140625,
      "activations/layer11_attention_weight_min": -36.107398986816406,
      "activations/layer12_attention_weight_max": 30.08572769165039,
      "activations/layer12_attention_weight_min": -27.048904418945312,
      "activations/layer13_attention_weight_max": 47.72782897949219,
      "activations/layer13_attention_weight_min": -36.19453430175781,
      "activations/layer14_attention_weight_max": 47.88595199584961,
      "activations/layer14_attention_weight_min": -37.56736755371094,
      "activations/layer15_attention_weight_max": 46.23689651489258,
      "activations/layer15_attention_weight_min": -35.2985954284668,
      "activations/layer16_attention_weight_max": 32.47138977050781,
      "activations/layer16_attention_weight_min": -28.645984649658203,
      "activations/layer17_attention_weight_max": 33.3830451965332,
      "activations/layer17_attention_weight_min": -25.431482315063477,
      "activations/layer18_attention_weight_max": 32.78255081176758,
      "activations/layer18_attention_weight_min": -25.062734603881836,
      "activations/layer19_attention_weight_max": 34.45505905151367,
      "activations/layer19_attention_weight_min": -31.459142684936523,
      "activations/layer1_attention_weight_max": 15.86476993560791,
      "activations/layer1_attention_weight_min": -15.078564643859863,
      "activations/layer20_attention_weight_max": 32.40662384033203,
      "activations/layer20_attention_weight_min": -24.56472396850586,
      "activations/layer21_attention_weight_max": 31.62436866760254,
      "activations/layer21_attention_weight_min": -25.030467987060547,
      "activations/layer22_attention_weight_max": 43.17384719848633,
      "activations/layer22_attention_weight_min": -30.15802001953125,
      "activations/layer23_attention_weight_max": 34.83208084106445,
      "activations/layer23_attention_weight_min": -27.38631820678711,
      "activations/layer2_attention_weight_max": 32.588199615478516,
      "activations/layer2_attention_weight_min": -32.24091720581055,
      "activations/layer3_attention_weight_max": 95.73673248291016,
      "activations/layer3_attention_weight_min": -95.8565902709961,
      "activations/layer4_attention_weight_max": 110.61894989013672,
      "activations/layer4_attention_weight_min": -110.76085662841797,
      "activations/layer5_attention_weight_max": 52.35708236694336,
      "activations/layer5_attention_weight_min": -64.82061767578125,
      "activations/layer6_attention_weight_max": 45.388954162597656,
      "activations/layer6_attention_weight_min": -47.55441665649414,
      "activations/layer7_attention_weight_max": 92.68479919433594,
      "activations/layer7_attention_weight_min": -94.74573516845703,
      "activations/layer8_attention_weight_max": 42.6849250793457,
      "activations/layer8_attention_weight_min": -41.510467529296875,
      "activations/layer9_attention_weight_max": 36.2755241394043,
      "activations/layer9_attention_weight_min": -34.43355941772461,
      "epoch": 15.03,
      "learning_rate": 5.364015151515151e-05,
      "loss": 2.7309,
      "step": 258600
    },
    {
      "activations/layer0_attention_weight_max": 15.51091480255127,
      "activations/layer0_attention_weight_min": -11.904073715209961,
      "activations/layer10_attention_weight_max": 32.15955352783203,
      "activations/layer10_attention_weight_min": -33.29367446899414,
      "activations/layer11_attention_weight_max": 34.3678092956543,
      "activations/layer11_attention_weight_min": -34.68227767944336,
      "activations/layer12_attention_weight_max": 30.160419464111328,
      "activations/layer12_attention_weight_min": -28.47515296936035,
      "activations/layer13_attention_weight_max": 43.60879898071289,
      "activations/layer13_attention_weight_min": -37.48749923706055,
      "activations/layer14_attention_weight_max": 51.87797546386719,
      "activations/layer14_attention_weight_min": -42.94881820678711,
      "activations/layer15_attention_weight_max": 42.90076446533203,
      "activations/layer15_attention_weight_min": -39.9407844543457,
      "activations/layer16_attention_weight_max": 34.05506896972656,
      "activations/layer16_attention_weight_min": -30.559167861938477,
      "activations/layer17_attention_weight_max": 32.53335189819336,
      "activations/layer17_attention_weight_min": -28.46108055114746,
      "activations/layer18_attention_weight_max": 32.58134078979492,
      "activations/layer18_attention_weight_min": -25.4261417388916,
      "activations/layer19_attention_weight_max": 39.525245666503906,
      "activations/layer19_attention_weight_min": -33.77997970581055,
      "activations/layer1_attention_weight_max": 16.65423583984375,
      "activations/layer1_attention_weight_min": -14.591769218444824,
      "activations/layer20_attention_weight_max": 31.346921920776367,
      "activations/layer20_attention_weight_min": -25.358097076416016,
      "activations/layer21_attention_weight_max": 30.56991195678711,
      "activations/layer21_attention_weight_min": -23.82341766357422,
      "activations/layer22_attention_weight_max": 51.80027389526367,
      "activations/layer22_attention_weight_min": -30.563629150390625,
      "activations/layer23_attention_weight_max": 35.82844543457031,
      "activations/layer23_attention_weight_min": -26.129562377929688,
      "activations/layer2_attention_weight_max": 31.251262664794922,
      "activations/layer2_attention_weight_min": -31.371131896972656,
      "activations/layer3_attention_weight_max": 94.38082122802734,
      "activations/layer3_attention_weight_min": -97.69342041015625,
      "activations/layer4_attention_weight_max": 106.29779052734375,
      "activations/layer4_attention_weight_min": -116.47545623779297,
      "activations/layer5_attention_weight_max": 51.972557067871094,
      "activations/layer5_attention_weight_min": -69.39759063720703,
      "activations/layer6_attention_weight_max": 42.7572135925293,
      "activations/layer6_attention_weight_min": -50.191287994384766,
      "activations/layer7_attention_weight_max": 91.01097106933594,
      "activations/layer7_attention_weight_min": -95.24856567382812,
      "activations/layer8_attention_weight_max": 39.33389663696289,
      "activations/layer8_attention_weight_min": -43.8626708984375,
      "activations/layer9_attention_weight_max": 32.850521087646484,
      "activations/layer9_attention_weight_min": -33.76374053955078,
      "epoch": 15.03,
      "learning_rate": 5.362121212121212e-05,
      "loss": 2.7539,
      "step": 258650
    },
    {
      "activations/layer0_attention_weight_max": 15.376291275024414,
      "activations/layer0_attention_weight_min": -12.36305046081543,
      "activations/layer10_attention_weight_max": 51.32871627807617,
      "activations/layer10_attention_weight_min": -48.556182861328125,
      "activations/layer11_attention_weight_max": 53.30500030517578,
      "activations/layer11_attention_weight_min": -50.71681213378906,
      "activations/layer12_attention_weight_max": 39.74026107788086,
      "activations/layer12_attention_weight_min": -32.631370544433594,
      "activations/layer13_attention_weight_max": 63.73683166503906,
      "activations/layer13_attention_weight_min": -43.38558578491211,
      "activations/layer14_attention_weight_max": 61.773033142089844,
      "activations/layer14_attention_weight_min": -50.45184326171875,
      "activations/layer15_attention_weight_max": 60.63200759887695,
      "activations/layer15_attention_weight_min": -49.469459533691406,
      "activations/layer16_attention_weight_max": 40.00727081298828,
      "activations/layer16_attention_weight_min": -30.49370002746582,
      "activations/layer17_attention_weight_max": 37.83695983886719,
      "activations/layer17_attention_weight_min": -33.595123291015625,
      "activations/layer18_attention_weight_max": 33.4425048828125,
      "activations/layer18_attention_weight_min": -29.324508666992188,
      "activations/layer19_attention_weight_max": 39.57009506225586,
      "activations/layer19_attention_weight_min": -33.57575607299805,
      "activations/layer1_attention_weight_max": 16.518898010253906,
      "activations/layer1_attention_weight_min": -15.990509986877441,
      "activations/layer20_attention_weight_max": 32.693946838378906,
      "activations/layer20_attention_weight_min": -27.394573211669922,
      "activations/layer21_attention_weight_max": 32.773101806640625,
      "activations/layer21_attention_weight_min": -26.85837745666504,
      "activations/layer22_attention_weight_max": 56.95015335083008,
      "activations/layer22_attention_weight_min": -37.11991500854492,
      "activations/layer23_attention_weight_max": 36.86737060546875,
      "activations/layer23_attention_weight_min": -26.205503463745117,
      "activations/layer2_attention_weight_max": 34.489139556884766,
      "activations/layer2_attention_weight_min": -33.08107376098633,
      "activations/layer3_attention_weight_max": 103.6933364868164,
      "activations/layer3_attention_weight_min": -107.09078216552734,
      "activations/layer4_attention_weight_max": 121.05901336669922,
      "activations/layer4_attention_weight_min": -115.47773742675781,
      "activations/layer5_attention_weight_max": 57.57999801635742,
      "activations/layer5_attention_weight_min": -62.08045959472656,
      "activations/layer6_attention_weight_max": 52.38730239868164,
      "activations/layer6_attention_weight_min": -50.94081497192383,
      "activations/layer7_attention_weight_max": 120.43419647216797,
      "activations/layer7_attention_weight_min": -109.43021392822266,
      "activations/layer8_attention_weight_max": 55.559288024902344,
      "activations/layer8_attention_weight_min": -53.43603515625,
      "activations/layer9_attention_weight_max": 47.45954895019531,
      "activations/layer9_attention_weight_min": -45.652835845947266,
      "epoch": 15.03,
      "learning_rate": 5.3602272727272724e-05,
      "loss": 2.7656,
      "step": 258700
    },
    {
      "activations/layer0_attention_weight_max": 15.424298286437988,
      "activations/layer0_attention_weight_min": -12.431558609008789,
      "activations/layer10_attention_weight_max": 33.408485412597656,
      "activations/layer10_attention_weight_min": -33.83839797973633,
      "activations/layer11_attention_weight_max": 33.19188690185547,
      "activations/layer11_attention_weight_min": -34.34880065917969,
      "activations/layer12_attention_weight_max": 26.14908790588379,
      "activations/layer12_attention_weight_min": -26.351367950439453,
      "activations/layer13_attention_weight_max": 39.46975326538086,
      "activations/layer13_attention_weight_min": -36.18516159057617,
      "activations/layer14_attention_weight_max": 41.35805130004883,
      "activations/layer14_attention_weight_min": -36.92201232910156,
      "activations/layer15_attention_weight_max": 38.38884735107422,
      "activations/layer15_attention_weight_min": -34.971336364746094,
      "activations/layer16_attention_weight_max": 29.077129364013672,
      "activations/layer16_attention_weight_min": -27.413230895996094,
      "activations/layer17_attention_weight_max": 32.541011810302734,
      "activations/layer17_attention_weight_min": -26.383262634277344,
      "activations/layer18_attention_weight_max": 32.66758346557617,
      "activations/layer18_attention_weight_min": -26.442508697509766,
      "activations/layer19_attention_weight_max": 37.94913864135742,
      "activations/layer19_attention_weight_min": -34.079036712646484,
      "activations/layer1_attention_weight_max": 16.489091873168945,
      "activations/layer1_attention_weight_min": -15.455164909362793,
      "activations/layer20_attention_weight_max": 33.46878433227539,
      "activations/layer20_attention_weight_min": -26.726490020751953,
      "activations/layer21_attention_weight_max": 34.23267364501953,
      "activations/layer21_attention_weight_min": -26.464689254760742,
      "activations/layer22_attention_weight_max": 51.1156005859375,
      "activations/layer22_attention_weight_min": -33.262062072753906,
      "activations/layer23_attention_weight_max": 39.49348831176758,
      "activations/layer23_attention_weight_min": -26.431882858276367,
      "activations/layer2_attention_weight_max": 32.653770446777344,
      "activations/layer2_attention_weight_min": -31.62746238708496,
      "activations/layer3_attention_weight_max": 95.71720123291016,
      "activations/layer3_attention_weight_min": -97.32059478759766,
      "activations/layer4_attention_weight_max": 111.980712890625,
      "activations/layer4_attention_weight_min": -111.3305892944336,
      "activations/layer5_attention_weight_max": 52.92328643798828,
      "activations/layer5_attention_weight_min": -67.10249328613281,
      "activations/layer6_attention_weight_max": 42.88584899902344,
      "activations/layer6_attention_weight_min": -44.42857360839844,
      "activations/layer7_attention_weight_max": 92.1294937133789,
      "activations/layer7_attention_weight_min": -91.32937622070312,
      "activations/layer8_attention_weight_max": 41.10609817504883,
      "activations/layer8_attention_weight_min": -40.37356185913086,
      "activations/layer9_attention_weight_max": 31.554533004760742,
      "activations/layer9_attention_weight_min": -33.26131820678711,
      "epoch": 15.03,
      "learning_rate": 5.3583333333333326e-05,
      "loss": 2.7597,
      "step": 258750
    },
    {
      "activations/layer0_attention_weight_max": 15.064017295837402,
      "activations/layer0_attention_weight_min": -11.245307922363281,
      "activations/layer10_attention_weight_max": 31.46830940246582,
      "activations/layer10_attention_weight_min": -33.419002532958984,
      "activations/layer11_attention_weight_max": 32.974754333496094,
      "activations/layer11_attention_weight_min": -34.230072021484375,
      "activations/layer12_attention_weight_max": 28.60272789001465,
      "activations/layer12_attention_weight_min": -26.48194694519043,
      "activations/layer13_attention_weight_max": 40.672786712646484,
      "activations/layer13_attention_weight_min": -38.19170379638672,
      "activations/layer14_attention_weight_max": 45.6849250793457,
      "activations/layer14_attention_weight_min": -43.5509147644043,
      "activations/layer15_attention_weight_max": 39.868003845214844,
      "activations/layer15_attention_weight_min": -37.23606491088867,
      "activations/layer16_attention_weight_max": 34.38569641113281,
      "activations/layer16_attention_weight_min": -30.743940353393555,
      "activations/layer17_attention_weight_max": 31.72670555114746,
      "activations/layer17_attention_weight_min": -27.599668502807617,
      "activations/layer18_attention_weight_max": 32.082584381103516,
      "activations/layer18_attention_weight_min": -25.4431095123291,
      "activations/layer19_attention_weight_max": 40.33546447753906,
      "activations/layer19_attention_weight_min": -32.28591537475586,
      "activations/layer1_attention_weight_max": 16.996902465820312,
      "activations/layer1_attention_weight_min": -14.356915473937988,
      "activations/layer20_attention_weight_max": 31.07958221435547,
      "activations/layer20_attention_weight_min": -25.110185623168945,
      "activations/layer21_attention_weight_max": 31.555709838867188,
      "activations/layer21_attention_weight_min": -24.494327545166016,
      "activations/layer22_attention_weight_max": 46.21076965332031,
      "activations/layer22_attention_weight_min": -30.49571418762207,
      "activations/layer23_attention_weight_max": 35.06932830810547,
      "activations/layer23_attention_weight_min": -24.700475692749023,
      "activations/layer2_attention_weight_max": 31.40725326538086,
      "activations/layer2_attention_weight_min": -31.262222290039062,
      "activations/layer3_attention_weight_max": 95.75579833984375,
      "activations/layer3_attention_weight_min": -95.69932556152344,
      "activations/layer4_attention_weight_max": 107.6646957397461,
      "activations/layer4_attention_weight_min": -112.74507141113281,
      "activations/layer5_attention_weight_max": 51.15572738647461,
      "activations/layer5_attention_weight_min": -63.330352783203125,
      "activations/layer6_attention_weight_max": 45.61081314086914,
      "activations/layer6_attention_weight_min": -47.631874084472656,
      "activations/layer7_attention_weight_max": 93.84013366699219,
      "activations/layer7_attention_weight_min": -99.63582611083984,
      "activations/layer8_attention_weight_max": 39.79796600341797,
      "activations/layer8_attention_weight_min": -43.3707389831543,
      "activations/layer9_attention_weight_max": 31.322473526000977,
      "activations/layer9_attention_weight_min": -33.811729431152344,
      "epoch": 15.04,
      "learning_rate": 5.3564393939393934e-05,
      "loss": 2.7475,
      "step": 258800
    },
    {
      "activations/layer0_attention_weight_max": 15.262613296508789,
      "activations/layer0_attention_weight_min": -11.899918556213379,
      "activations/layer10_attention_weight_max": 37.093074798583984,
      "activations/layer10_attention_weight_min": -38.78969192504883,
      "activations/layer11_attention_weight_max": 38.11212158203125,
      "activations/layer11_attention_weight_min": -38.62712860107422,
      "activations/layer12_attention_weight_max": 29.772687911987305,
      "activations/layer12_attention_weight_min": -27.64263153076172,
      "activations/layer13_attention_weight_max": 47.252689361572266,
      "activations/layer13_attention_weight_min": -37.37309646606445,
      "activations/layer14_attention_weight_max": 47.65536880493164,
      "activations/layer14_attention_weight_min": -41.18422317504883,
      "activations/layer15_attention_weight_max": 45.03741455078125,
      "activations/layer15_attention_weight_min": -39.33927917480469,
      "activations/layer16_attention_weight_max": 33.23347473144531,
      "activations/layer16_attention_weight_min": -27.874279022216797,
      "activations/layer17_attention_weight_max": 32.53184509277344,
      "activations/layer17_attention_weight_min": -28.48142433166504,
      "activations/layer18_attention_weight_max": 32.53984069824219,
      "activations/layer18_attention_weight_min": -28.895606994628906,
      "activations/layer19_attention_weight_max": 36.66188430786133,
      "activations/layer19_attention_weight_min": -34.56188201904297,
      "activations/layer1_attention_weight_max": 16.301433563232422,
      "activations/layer1_attention_weight_min": -14.776378631591797,
      "activations/layer20_attention_weight_max": 29.479204177856445,
      "activations/layer20_attention_weight_min": -26.137250900268555,
      "activations/layer21_attention_weight_max": 27.0325927734375,
      "activations/layer21_attention_weight_min": -23.212860107421875,
      "activations/layer22_attention_weight_max": 40.841644287109375,
      "activations/layer22_attention_weight_min": -31.55628776550293,
      "activations/layer23_attention_weight_max": 31.825803756713867,
      "activations/layer23_attention_weight_min": -23.473968505859375,
      "activations/layer2_attention_weight_max": 30.78076171875,
      "activations/layer2_attention_weight_min": -30.50666618347168,
      "activations/layer3_attention_weight_max": 92.30004119873047,
      "activations/layer3_attention_weight_min": -93.51309967041016,
      "activations/layer4_attention_weight_max": 103.40593719482422,
      "activations/layer4_attention_weight_min": -108.5311279296875,
      "activations/layer5_attention_weight_max": 52.071533203125,
      "activations/layer5_attention_weight_min": -62.2761344909668,
      "activations/layer6_attention_weight_max": 44.07992935180664,
      "activations/layer6_attention_weight_min": -46.750911712646484,
      "activations/layer7_attention_weight_max": 91.7625961303711,
      "activations/layer7_attention_weight_min": -99.95441436767578,
      "activations/layer8_attention_weight_max": 42.330223083496094,
      "activations/layer8_attention_weight_min": -44.84870529174805,
      "activations/layer9_attention_weight_max": 34.63821029663086,
      "activations/layer9_attention_weight_min": -38.76768112182617,
      "epoch": 15.04,
      "learning_rate": 5.354545454545454e-05,
      "loss": 2.741,
      "step": 258850
    },
    {
      "activations/layer0_attention_weight_max": 16.45594596862793,
      "activations/layer0_attention_weight_min": -12.158401489257812,
      "activations/layer10_attention_weight_max": 32.493614196777344,
      "activations/layer10_attention_weight_min": -34.077484130859375,
      "activations/layer11_attention_weight_max": 32.86096954345703,
      "activations/layer11_attention_weight_min": -34.006385803222656,
      "activations/layer12_attention_weight_max": 25.25853157043457,
      "activations/layer12_attention_weight_min": -27.284584045410156,
      "activations/layer13_attention_weight_max": 38.28977584838867,
      "activations/layer13_attention_weight_min": -36.88923645019531,
      "activations/layer14_attention_weight_max": 44.252811431884766,
      "activations/layer14_attention_weight_min": -40.451133728027344,
      "activations/layer15_attention_weight_max": 35.18598556518555,
      "activations/layer15_attention_weight_min": -35.7924919128418,
      "activations/layer16_attention_weight_max": 26.76386070251465,
      "activations/layer16_attention_weight_min": -28.0560359954834,
      "activations/layer17_attention_weight_max": 29.99481201171875,
      "activations/layer17_attention_weight_min": -26.790815353393555,
      "activations/layer18_attention_weight_max": 30.456701278686523,
      "activations/layer18_attention_weight_min": -27.294832229614258,
      "activations/layer19_attention_weight_max": 33.07664489746094,
      "activations/layer19_attention_weight_min": -33.29594802856445,
      "activations/layer1_attention_weight_max": 16.0841064453125,
      "activations/layer1_attention_weight_min": -14.175790786743164,
      "activations/layer20_attention_weight_max": 26.619861602783203,
      "activations/layer20_attention_weight_min": -25.598186492919922,
      "activations/layer21_attention_weight_max": 29.5606689453125,
      "activations/layer21_attention_weight_min": -25.655424118041992,
      "activations/layer22_attention_weight_max": 42.84590148925781,
      "activations/layer22_attention_weight_min": -33.29548263549805,
      "activations/layer23_attention_weight_max": 32.15843963623047,
      "activations/layer23_attention_weight_min": -27.412038803100586,
      "activations/layer2_attention_weight_max": 32.60340118408203,
      "activations/layer2_attention_weight_min": -31.845693588256836,
      "activations/layer3_attention_weight_max": 95.88262939453125,
      "activations/layer3_attention_weight_min": -97.2376480102539,
      "activations/layer4_attention_weight_max": 108.88805389404297,
      "activations/layer4_attention_weight_min": -110.41959381103516,
      "activations/layer5_attention_weight_max": 48.623802185058594,
      "activations/layer5_attention_weight_min": -62.50688934326172,
      "activations/layer6_attention_weight_max": 41.09938049316406,
      "activations/layer6_attention_weight_min": -44.81239700317383,
      "activations/layer7_attention_weight_max": 85.52338409423828,
      "activations/layer7_attention_weight_min": -88.42158508300781,
      "activations/layer8_attention_weight_max": 38.60597610473633,
      "activations/layer8_attention_weight_min": -40.09915542602539,
      "activations/layer9_attention_weight_max": 30.489477157592773,
      "activations/layer9_attention_weight_min": -32.034576416015625,
      "epoch": 15.04,
      "learning_rate": 5.352689393939393e-05,
      "loss": 2.718,
      "step": 258900
    },
    {
      "activations/layer0_attention_weight_max": 16.307363510131836,
      "activations/layer0_attention_weight_min": -13.579692840576172,
      "activations/layer10_attention_weight_max": 38.257423400878906,
      "activations/layer10_attention_weight_min": -35.73735427856445,
      "activations/layer11_attention_weight_max": 38.569732666015625,
      "activations/layer11_attention_weight_min": -35.95113754272461,
      "activations/layer12_attention_weight_max": 26.34770965576172,
      "activations/layer12_attention_weight_min": -26.534006118774414,
      "activations/layer13_attention_weight_max": 37.120323181152344,
      "activations/layer13_attention_weight_min": -33.38988494873047,
      "activations/layer14_attention_weight_max": 41.59354782104492,
      "activations/layer14_attention_weight_min": -37.11074447631836,
      "activations/layer15_attention_weight_max": 40.834171295166016,
      "activations/layer15_attention_weight_min": -33.25782012939453,
      "activations/layer16_attention_weight_max": 29.5817928314209,
      "activations/layer16_attention_weight_min": -28.351259231567383,
      "activations/layer17_attention_weight_max": 31.85771369934082,
      "activations/layer17_attention_weight_min": -26.028892517089844,
      "activations/layer18_attention_weight_max": 30.65015983581543,
      "activations/layer18_attention_weight_min": -24.82677459716797,
      "activations/layer19_attention_weight_max": 33.91096496582031,
      "activations/layer19_attention_weight_min": -30.167388916015625,
      "activations/layer1_attention_weight_max": 18.2479305267334,
      "activations/layer1_attention_weight_min": -14.01462173461914,
      "activations/layer20_attention_weight_max": 27.84025764465332,
      "activations/layer20_attention_weight_min": -25.698802947998047,
      "activations/layer21_attention_weight_max": 29.99594497680664,
      "activations/layer21_attention_weight_min": -24.527189254760742,
      "activations/layer22_attention_weight_max": 44.03702926635742,
      "activations/layer22_attention_weight_min": -31.136043548583984,
      "activations/layer23_attention_weight_max": 32.71247863769531,
      "activations/layer23_attention_weight_min": -24.941253662109375,
      "activations/layer2_attention_weight_max": 32.22344207763672,
      "activations/layer2_attention_weight_min": -31.643234252929688,
      "activations/layer3_attention_weight_max": 95.90355682373047,
      "activations/layer3_attention_weight_min": -98.02574157714844,
      "activations/layer4_attention_weight_max": 110.34464263916016,
      "activations/layer4_attention_weight_min": -112.12981414794922,
      "activations/layer5_attention_weight_max": 52.272064208984375,
      "activations/layer5_attention_weight_min": -64.24349975585938,
      "activations/layer6_attention_weight_max": 43.92441940307617,
      "activations/layer6_attention_weight_min": -46.46162033081055,
      "activations/layer7_attention_weight_max": 92.72761535644531,
      "activations/layer7_attention_weight_min": -96.89961242675781,
      "activations/layer8_attention_weight_max": 44.26076889038086,
      "activations/layer8_attention_weight_min": -43.5456428527832,
      "activations/layer9_attention_weight_max": 32.957427978515625,
      "activations/layer9_attention_weight_min": -35.21090316772461,
      "epoch": 15.05,
      "learning_rate": 5.350795454545454e-05,
      "loss": 2.748,
      "step": 258950
    },
    {
      "activations/layer0_attention_weight_max": 15.16915225982666,
      "activations/layer0_attention_weight_min": -12.26751708984375,
      "activations/layer10_attention_weight_max": 35.64630126953125,
      "activations/layer10_attention_weight_min": -33.779151916503906,
      "activations/layer11_attention_weight_max": 38.20303726196289,
      "activations/layer11_attention_weight_min": -35.91876983642578,
      "activations/layer12_attention_weight_max": 26.154098510742188,
      "activations/layer12_attention_weight_min": -27.578428268432617,
      "activations/layer13_attention_weight_max": 41.459564208984375,
      "activations/layer13_attention_weight_min": -37.812923431396484,
      "activations/layer14_attention_weight_max": 49.357337951660156,
      "activations/layer14_attention_weight_min": -39.88627243041992,
      "activations/layer15_attention_weight_max": 38.833709716796875,
      "activations/layer15_attention_weight_min": -36.22932052612305,
      "activations/layer16_attention_weight_max": 33.7946891784668,
      "activations/layer16_attention_weight_min": -29.355552673339844,
      "activations/layer17_attention_weight_max": 32.168861389160156,
      "activations/layer17_attention_weight_min": -26.58539390563965,
      "activations/layer18_attention_weight_max": 33.06499481201172,
      "activations/layer18_attention_weight_min": -24.23104476928711,
      "activations/layer19_attention_weight_max": 38.210670471191406,
      "activations/layer19_attention_weight_min": -32.58814239501953,
      "activations/layer1_attention_weight_max": 16.75724983215332,
      "activations/layer1_attention_weight_min": -14.718473434448242,
      "activations/layer20_attention_weight_max": 33.21469497680664,
      "activations/layer20_attention_weight_min": -23.92879295349121,
      "activations/layer21_attention_weight_max": 29.71986961364746,
      "activations/layer21_attention_weight_min": -23.773113250732422,
      "activations/layer22_attention_weight_max": 45.310340881347656,
      "activations/layer22_attention_weight_min": -31.41081428527832,
      "activations/layer23_attention_weight_max": 40.26995086669922,
      "activations/layer23_attention_weight_min": -26.64751434326172,
      "activations/layer2_attention_weight_max": 32.343711853027344,
      "activations/layer2_attention_weight_min": -31.54867935180664,
      "activations/layer3_attention_weight_max": 93.6568603515625,
      "activations/layer3_attention_weight_min": -96.71805572509766,
      "activations/layer4_attention_weight_max": 109.3315200805664,
      "activations/layer4_attention_weight_min": -109.34146881103516,
      "activations/layer5_attention_weight_max": 51.2035026550293,
      "activations/layer5_attention_weight_min": -62.92230987548828,
      "activations/layer6_attention_weight_max": 44.950340270996094,
      "activations/layer6_attention_weight_min": -47.187355041503906,
      "activations/layer7_attention_weight_max": 92.35054779052734,
      "activations/layer7_attention_weight_min": -96.85502624511719,
      "activations/layer8_attention_weight_max": 43.58208084106445,
      "activations/layer8_attention_weight_min": -46.88447189331055,
      "activations/layer9_attention_weight_max": 33.99673080444336,
      "activations/layer9_attention_weight_min": -35.17275619506836,
      "epoch": 15.05,
      "learning_rate": 5.348901515151514e-05,
      "loss": 2.7406,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.5946,
      "eval_samples_per_second": 499.616,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.5946,
      "eval_openwebtext_samples_per_second": 499.616,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 2.0333,
      "eval_wikitext_samples_per_second": 224.271,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_lambada_loss": 2.431640625,
      "eval_lambada_ppl": 11.3775330413472,
      "eval_lambada_runtime": 9.6075,
      "eval_lambada_samples_per_second": 506.794,
      "step": 259000
    },
    {
      "activations/layer0_attention_weight_max": 15.069406509399414,
      "activations/layer0_attention_weight_min": -12.067173957824707,
      "activations/layer10_attention_weight_max": 32.1722297668457,
      "activations/layer10_attention_weight_min": -33.83161163330078,
      "activations/layer11_attention_weight_max": 31.652427673339844,
      "activations/layer11_attention_weight_min": -33.298797607421875,
      "activations/layer12_attention_weight_max": 25.4044189453125,
      "activations/layer12_attention_weight_min": -27.12429428100586,
      "activations/layer13_attention_weight_max": 39.59420394897461,
      "activations/layer13_attention_weight_min": -32.37948989868164,
      "activations/layer14_attention_weight_max": 44.08784484863281,
      "activations/layer14_attention_weight_min": -38.834896087646484,
      "activations/layer15_attention_weight_max": 36.414676666259766,
      "activations/layer15_attention_weight_min": -35.746646881103516,
      "activations/layer16_attention_weight_max": 29.811492919921875,
      "activations/layer16_attention_weight_min": -30.975797653198242,
      "activations/layer17_attention_weight_max": 30.61004066467285,
      "activations/layer17_attention_weight_min": -28.354398727416992,
      "activations/layer18_attention_weight_max": 32.7165641784668,
      "activations/layer18_attention_weight_min": -28.353261947631836,
      "activations/layer19_attention_weight_max": 32.77933883666992,
      "activations/layer19_attention_weight_min": -32.444175720214844,
      "activations/layer1_attention_weight_max": 16.928844451904297,
      "activations/layer1_attention_weight_min": -15.244210243225098,
      "activations/layer20_attention_weight_max": 27.633771896362305,
      "activations/layer20_attention_weight_min": -26.363069534301758,
      "activations/layer21_attention_weight_max": 29.20640754699707,
      "activations/layer21_attention_weight_min": -25.61767578125,
      "activations/layer22_attention_weight_max": 44.41090774536133,
      "activations/layer22_attention_weight_min": -32.1546516418457,
      "activations/layer23_attention_weight_max": 38.33970642089844,
      "activations/layer23_attention_weight_min": -26.49483299255371,
      "activations/layer2_attention_weight_max": 34.470726013183594,
      "activations/layer2_attention_weight_min": -32.579376220703125,
      "activations/layer3_attention_weight_max": 102.80455017089844,
      "activations/layer3_attention_weight_min": -104.51471710205078,
      "activations/layer4_attention_weight_max": 114.8292465209961,
      "activations/layer4_attention_weight_min": -118.23128509521484,
      "activations/layer5_attention_weight_max": 53.04000473022461,
      "activations/layer5_attention_weight_min": -64.44743347167969,
      "activations/layer6_attention_weight_max": 45.65177536010742,
      "activations/layer6_attention_weight_min": -48.33090591430664,
      "activations/layer7_attention_weight_max": 89.92739868164062,
      "activations/layer7_attention_weight_min": -92.48454284667969,
      "activations/layer8_attention_weight_max": 39.63504409790039,
      "activations/layer8_attention_weight_min": -44.02342987060547,
      "activations/layer9_attention_weight_max": 31.627744674682617,
      "activations/layer9_attention_weight_min": -32.569549560546875,
      "epoch": 15.05,
      "learning_rate": 5.347007575757576e-05,
      "loss": 2.7447,
      "step": 259050
    },
    {
      "activations/layer0_attention_weight_max": 15.460532188415527,
      "activations/layer0_attention_weight_min": -12.113484382629395,
      "activations/layer10_attention_weight_max": 33.20539474487305,
      "activations/layer10_attention_weight_min": -32.99319076538086,
      "activations/layer11_attention_weight_max": 33.26116943359375,
      "activations/layer11_attention_weight_min": -34.7585334777832,
      "activations/layer12_attention_weight_max": 26.37653923034668,
      "activations/layer12_attention_weight_min": -27.90354347229004,
      "activations/layer13_attention_weight_max": 39.81861877441406,
      "activations/layer13_attention_weight_min": -39.395469665527344,
      "activations/layer14_attention_weight_max": 42.406463623046875,
      "activations/layer14_attention_weight_min": -42.82762145996094,
      "activations/layer15_attention_weight_max": 38.00608825683594,
      "activations/layer15_attention_weight_min": -34.38703536987305,
      "activations/layer16_attention_weight_max": 29.115942001342773,
      "activations/layer16_attention_weight_min": -28.177627563476562,
      "activations/layer17_attention_weight_max": 30.32856559753418,
      "activations/layer17_attention_weight_min": -25.24299430847168,
      "activations/layer18_attention_weight_max": 35.34307098388672,
      "activations/layer18_attention_weight_min": -24.53746795654297,
      "activations/layer19_attention_weight_max": 37.00289535522461,
      "activations/layer19_attention_weight_min": -31.19429588317871,
      "activations/layer1_attention_weight_max": 16.320802688598633,
      "activations/layer1_attention_weight_min": -15.237709045410156,
      "activations/layer20_attention_weight_max": 28.18602752685547,
      "activations/layer20_attention_weight_min": -23.748428344726562,
      "activations/layer21_attention_weight_max": 27.745750427246094,
      "activations/layer21_attention_weight_min": -23.64157485961914,
      "activations/layer22_attention_weight_max": 41.02734375,
      "activations/layer22_attention_weight_min": -29.700580596923828,
      "activations/layer23_attention_weight_max": 30.935760498046875,
      "activations/layer23_attention_weight_min": -25.27850341796875,
      "activations/layer2_attention_weight_max": 32.080570220947266,
      "activations/layer2_attention_weight_min": -31.3282527923584,
      "activations/layer3_attention_weight_max": 93.07366943359375,
      "activations/layer3_attention_weight_min": -97.57219696044922,
      "activations/layer4_attention_weight_max": 104.76625061035156,
      "activations/layer4_attention_weight_min": -112.7022476196289,
      "activations/layer5_attention_weight_max": 49.34251403808594,
      "activations/layer5_attention_weight_min": -61.35899353027344,
      "activations/layer6_attention_weight_max": 41.05632400512695,
      "activations/layer6_attention_weight_min": -43.79902267456055,
      "activations/layer7_attention_weight_max": 85.50255584716797,
      "activations/layer7_attention_weight_min": -88.6303939819336,
      "activations/layer8_attention_weight_max": 38.131370544433594,
      "activations/layer8_attention_weight_min": -40.27845001220703,
      "activations/layer9_attention_weight_max": 31.81182861328125,
      "activations/layer9_attention_weight_min": -32.72981262207031,
      "epoch": 15.06,
      "learning_rate": 5.345113636363636e-05,
      "loss": 2.7302,
      "step": 259100
    },
    {
      "activations/layer0_attention_weight_max": 16.941986083984375,
      "activations/layer0_attention_weight_min": -12.199767112731934,
      "activations/layer10_attention_weight_max": 30.519145965576172,
      "activations/layer10_attention_weight_min": -33.070045471191406,
      "activations/layer11_attention_weight_max": 31.922252655029297,
      "activations/layer11_attention_weight_min": -31.904592514038086,
      "activations/layer12_attention_weight_max": 25.656030654907227,
      "activations/layer12_attention_weight_min": -25.61859893798828,
      "activations/layer13_attention_weight_max": 37.24488067626953,
      "activations/layer13_attention_weight_min": -31.971240997314453,
      "activations/layer14_attention_weight_max": 40.034278869628906,
      "activations/layer14_attention_weight_min": -36.04850769042969,
      "activations/layer15_attention_weight_max": 40.0203971862793,
      "activations/layer15_attention_weight_min": -33.02165222167969,
      "activations/layer16_attention_weight_max": 29.26992416381836,
      "activations/layer16_attention_weight_min": -26.629375457763672,
      "activations/layer17_attention_weight_max": 33.75848388671875,
      "activations/layer17_attention_weight_min": -24.83327865600586,
      "activations/layer18_attention_weight_max": 29.558090209960938,
      "activations/layer18_attention_weight_min": -25.096975326538086,
      "activations/layer19_attention_weight_max": 33.80335998535156,
      "activations/layer19_attention_weight_min": -31.250659942626953,
      "activations/layer1_attention_weight_max": 16.172006607055664,
      "activations/layer1_attention_weight_min": -15.45779037475586,
      "activations/layer20_attention_weight_max": 32.62076950073242,
      "activations/layer20_attention_weight_min": -24.85323715209961,
      "activations/layer21_attention_weight_max": 29.85856819152832,
      "activations/layer21_attention_weight_min": -23.944480895996094,
      "activations/layer22_attention_weight_max": 41.22089385986328,
      "activations/layer22_attention_weight_min": -32.005516052246094,
      "activations/layer23_attention_weight_max": 33.541259765625,
      "activations/layer23_attention_weight_min": -25.780424118041992,
      "activations/layer2_attention_weight_max": 32.23942565917969,
      "activations/layer2_attention_weight_min": -31.88854217529297,
      "activations/layer3_attention_weight_max": 93.774658203125,
      "activations/layer3_attention_weight_min": -97.83231353759766,
      "activations/layer4_attention_weight_max": 105.27931213378906,
      "activations/layer4_attention_weight_min": -106.12953186035156,
      "activations/layer5_attention_weight_max": 50.982295989990234,
      "activations/layer5_attention_weight_min": -59.6373405456543,
      "activations/layer6_attention_weight_max": 42.03327941894531,
      "activations/layer6_attention_weight_min": -44.55983352661133,
      "activations/layer7_attention_weight_max": 84.7003402709961,
      "activations/layer7_attention_weight_min": -91.37590789794922,
      "activations/layer8_attention_weight_max": 39.979530334472656,
      "activations/layer8_attention_weight_min": -40.851524353027344,
      "activations/layer9_attention_weight_max": 32.903568267822266,
      "activations/layer9_attention_weight_min": -32.460113525390625,
      "epoch": 15.06,
      "learning_rate": 5.343219696969696e-05,
      "loss": 2.7565,
      "step": 259150
    },
    {
      "activations/layer0_attention_weight_max": 15.354446411132812,
      "activations/layer0_attention_weight_min": -12.204451560974121,
      "activations/layer10_attention_weight_max": 32.34956741333008,
      "activations/layer10_attention_weight_min": -33.567413330078125,
      "activations/layer11_attention_weight_max": 31.534976959228516,
      "activations/layer11_attention_weight_min": -32.22957229614258,
      "activations/layer12_attention_weight_max": 25.348264694213867,
      "activations/layer12_attention_weight_min": -24.683963775634766,
      "activations/layer13_attention_weight_max": 39.5047492980957,
      "activations/layer13_attention_weight_min": -34.61273193359375,
      "activations/layer14_attention_weight_max": 42.28968811035156,
      "activations/layer14_attention_weight_min": -36.64017868041992,
      "activations/layer15_attention_weight_max": 37.96514129638672,
      "activations/layer15_attention_weight_min": -37.249935150146484,
      "activations/layer16_attention_weight_max": 30.641979217529297,
      "activations/layer16_attention_weight_min": -30.503726959228516,
      "activations/layer17_attention_weight_max": 29.716339111328125,
      "activations/layer17_attention_weight_min": -26.4736270904541,
      "activations/layer18_attention_weight_max": 32.072784423828125,
      "activations/layer18_attention_weight_min": -24.760046005249023,
      "activations/layer19_attention_weight_max": 33.54498291015625,
      "activations/layer19_attention_weight_min": -31.722518920898438,
      "activations/layer1_attention_weight_max": 16.210189819335938,
      "activations/layer1_attention_weight_min": -17.17116355895996,
      "activations/layer20_attention_weight_max": 30.517162322998047,
      "activations/layer20_attention_weight_min": -23.726137161254883,
      "activations/layer21_attention_weight_max": 28.5968074798584,
      "activations/layer21_attention_weight_min": -22.807092666625977,
      "activations/layer22_attention_weight_max": 40.24783706665039,
      "activations/layer22_attention_weight_min": -27.990985870361328,
      "activations/layer23_attention_weight_max": 30.047155380249023,
      "activations/layer23_attention_weight_min": -23.19964599609375,
      "activations/layer2_attention_weight_max": 32.51509475708008,
      "activations/layer2_attention_weight_min": -32.21302032470703,
      "activations/layer3_attention_weight_max": 97.76219177246094,
      "activations/layer3_attention_weight_min": -100.58544158935547,
      "activations/layer4_attention_weight_max": 108.7064208984375,
      "activations/layer4_attention_weight_min": -115.04792785644531,
      "activations/layer5_attention_weight_max": 50.430503845214844,
      "activations/layer5_attention_weight_min": -65.11161041259766,
      "activations/layer6_attention_weight_max": 43.811336517333984,
      "activations/layer6_attention_weight_min": -49.3709602355957,
      "activations/layer7_attention_weight_max": 94.63838958740234,
      "activations/layer7_attention_weight_min": -96.83776092529297,
      "activations/layer8_attention_weight_max": 40.07347869873047,
      "activations/layer8_attention_weight_min": -42.4738655090332,
      "activations/layer9_attention_weight_max": 31.498044967651367,
      "activations/layer9_attention_weight_min": -32.98051452636719,
      "epoch": 15.06,
      "learning_rate": 5.341325757575757e-05,
      "loss": 2.7536,
      "step": 259200
    },
    {
      "activations/layer0_attention_weight_max": 15.19713306427002,
      "activations/layer0_attention_weight_min": -11.615126609802246,
      "activations/layer10_attention_weight_max": 36.16274642944336,
      "activations/layer10_attention_weight_min": -32.48750686645508,
      "activations/layer11_attention_weight_max": 36.34749221801758,
      "activations/layer11_attention_weight_min": -33.36671447753906,
      "activations/layer12_attention_weight_max": 27.881258010864258,
      "activations/layer12_attention_weight_min": -26.7466983795166,
      "activations/layer13_attention_weight_max": 39.89952850341797,
      "activations/layer13_attention_weight_min": -36.899505615234375,
      "activations/layer14_attention_weight_max": 47.87461471557617,
      "activations/layer14_attention_weight_min": -38.04727554321289,
      "activations/layer15_attention_weight_max": 46.481719970703125,
      "activations/layer15_attention_weight_min": -36.503658294677734,
      "activations/layer16_attention_weight_max": 37.69639205932617,
      "activations/layer16_attention_weight_min": -28.151344299316406,
      "activations/layer17_attention_weight_max": 33.33548355102539,
      "activations/layer17_attention_weight_min": -26.6856689453125,
      "activations/layer18_attention_weight_max": 33.28147888183594,
      "activations/layer18_attention_weight_min": -27.736600875854492,
      "activations/layer19_attention_weight_max": 40.05980682373047,
      "activations/layer19_attention_weight_min": -32.75217819213867,
      "activations/layer1_attention_weight_max": 16.418853759765625,
      "activations/layer1_attention_weight_min": -15.314897537231445,
      "activations/layer20_attention_weight_max": 33.0838508605957,
      "activations/layer20_attention_weight_min": -25.935834884643555,
      "activations/layer21_attention_weight_max": 33.35145950317383,
      "activations/layer21_attention_weight_min": -24.422080993652344,
      "activations/layer22_attention_weight_max": 43.857574462890625,
      "activations/layer22_attention_weight_min": -31.33867073059082,
      "activations/layer23_attention_weight_max": 37.127227783203125,
      "activations/layer23_attention_weight_min": -25.04970359802246,
      "activations/layer2_attention_weight_max": 32.24777603149414,
      "activations/layer2_attention_weight_min": -31.921478271484375,
      "activations/layer3_attention_weight_max": 98.29096221923828,
      "activations/layer3_attention_weight_min": -97.43252563476562,
      "activations/layer4_attention_weight_max": 109.36296844482422,
      "activations/layer4_attention_weight_min": -111.12800598144531,
      "activations/layer5_attention_weight_max": 53.861412048339844,
      "activations/layer5_attention_weight_min": -60.94010925292969,
      "activations/layer6_attention_weight_max": 44.37881088256836,
      "activations/layer6_attention_weight_min": -44.11064529418945,
      "activations/layer7_attention_weight_max": 96.2989730834961,
      "activations/layer7_attention_weight_min": -92.2852554321289,
      "activations/layer8_attention_weight_max": 44.25373458862305,
      "activations/layer8_attention_weight_min": -42.89429473876953,
      "activations/layer9_attention_weight_max": 31.692310333251953,
      "activations/layer9_attention_weight_min": -34.36180877685547,
      "epoch": 15.06,
      "learning_rate": 5.339431818181818e-05,
      "loss": 2.7501,
      "step": 259250
    },
    {
      "activations/layer0_attention_weight_max": 15.555109977722168,
      "activations/layer0_attention_weight_min": -12.320321083068848,
      "activations/layer10_attention_weight_max": 34.254032135009766,
      "activations/layer10_attention_weight_min": -34.31348419189453,
      "activations/layer11_attention_weight_max": 33.32451248168945,
      "activations/layer11_attention_weight_min": -34.271507263183594,
      "activations/layer12_attention_weight_max": 27.3377628326416,
      "activations/layer12_attention_weight_min": -27.72524642944336,
      "activations/layer13_attention_weight_max": 42.28162384033203,
      "activations/layer13_attention_weight_min": -36.20461654663086,
      "activations/layer14_attention_weight_max": 45.73344802856445,
      "activations/layer14_attention_weight_min": -41.32365036010742,
      "activations/layer15_attention_weight_max": 41.557037353515625,
      "activations/layer15_attention_weight_min": -38.056976318359375,
      "activations/layer16_attention_weight_max": 30.368364334106445,
      "activations/layer16_attention_weight_min": -28.137351989746094,
      "activations/layer17_attention_weight_max": 29.72139549255371,
      "activations/layer17_attention_weight_min": -25.797433853149414,
      "activations/layer18_attention_weight_max": 33.11201477050781,
      "activations/layer18_attention_weight_min": -26.581377029418945,
      "activations/layer19_attention_weight_max": 36.408409118652344,
      "activations/layer19_attention_weight_min": -31.784955978393555,
      "activations/layer1_attention_weight_max": 15.606249809265137,
      "activations/layer1_attention_weight_min": -16.281963348388672,
      "activations/layer20_attention_weight_max": 32.87798309326172,
      "activations/layer20_attention_weight_min": -24.687759399414062,
      "activations/layer21_attention_weight_max": 31.166845321655273,
      "activations/layer21_attention_weight_min": -24.433908462524414,
      "activations/layer22_attention_weight_max": 46.7083740234375,
      "activations/layer22_attention_weight_min": -31.89591407775879,
      "activations/layer23_attention_weight_max": 36.941162109375,
      "activations/layer23_attention_weight_min": -27.31146812438965,
      "activations/layer2_attention_weight_max": 31.770118713378906,
      "activations/layer2_attention_weight_min": -32.75496292114258,
      "activations/layer3_attention_weight_max": 94.88529205322266,
      "activations/layer3_attention_weight_min": -98.63755798339844,
      "activations/layer4_attention_weight_max": 108.46666717529297,
      "activations/layer4_attention_weight_min": -113.26869201660156,
      "activations/layer5_attention_weight_max": 50.969017028808594,
      "activations/layer5_attention_weight_min": -60.968990325927734,
      "activations/layer6_attention_weight_max": 42.64018630981445,
      "activations/layer6_attention_weight_min": -46.466617584228516,
      "activations/layer7_attention_weight_max": 89.63456726074219,
      "activations/layer7_attention_weight_min": -92.05001068115234,
      "activations/layer8_attention_weight_max": 39.65401077270508,
      "activations/layer8_attention_weight_min": -42.58745574951172,
      "activations/layer9_attention_weight_max": 33.41293716430664,
      "activations/layer9_attention_weight_min": -34.578369140625,
      "epoch": 15.07,
      "learning_rate": 5.3375378787878786e-05,
      "loss": 2.7343,
      "step": 259300
    },
    {
      "activations/layer0_attention_weight_max": 15.917789459228516,
      "activations/layer0_attention_weight_min": -12.166474342346191,
      "activations/layer10_attention_weight_max": 31.46282196044922,
      "activations/layer10_attention_weight_min": -32.80107116699219,
      "activations/layer11_attention_weight_max": 31.09320640563965,
      "activations/layer11_attention_weight_min": -35.33036804199219,
      "activations/layer12_attention_weight_max": 27.14391326904297,
      "activations/layer12_attention_weight_min": -25.437028884887695,
      "activations/layer13_attention_weight_max": 38.35380554199219,
      "activations/layer13_attention_weight_min": -34.029930114746094,
      "activations/layer14_attention_weight_max": 42.76150131225586,
      "activations/layer14_attention_weight_min": -37.1364860534668,
      "activations/layer15_attention_weight_max": 35.2884521484375,
      "activations/layer15_attention_weight_min": -36.530826568603516,
      "activations/layer16_attention_weight_max": 30.28424835205078,
      "activations/layer16_attention_weight_min": -26.72879981994629,
      "activations/layer17_attention_weight_max": 27.814332962036133,
      "activations/layer17_attention_weight_min": -25.90789222717285,
      "activations/layer18_attention_weight_max": 29.040681838989258,
      "activations/layer18_attention_weight_min": -24.554080963134766,
      "activations/layer19_attention_weight_max": 31.57021141052246,
      "activations/layer19_attention_weight_min": -29.95515251159668,
      "activations/layer1_attention_weight_max": 16.370193481445312,
      "activations/layer1_attention_weight_min": -13.58225154876709,
      "activations/layer20_attention_weight_max": 27.153059005737305,
      "activations/layer20_attention_weight_min": -24.45873260498047,
      "activations/layer21_attention_weight_max": 29.03278160095215,
      "activations/layer21_attention_weight_min": -22.212472915649414,
      "activations/layer22_attention_weight_max": 36.4893684387207,
      "activations/layer22_attention_weight_min": -30.261871337890625,
      "activations/layer23_attention_weight_max": 31.19080352783203,
      "activations/layer23_attention_weight_min": -26.91033172607422,
      "activations/layer2_attention_weight_max": 31.44366455078125,
      "activations/layer2_attention_weight_min": -31.723209381103516,
      "activations/layer3_attention_weight_max": 90.78194427490234,
      "activations/layer3_attention_weight_min": -93.92342376708984,
      "activations/layer4_attention_weight_max": 105.68611907958984,
      "activations/layer4_attention_weight_min": -110.6402359008789,
      "activations/layer5_attention_weight_max": 51.682891845703125,
      "activations/layer5_attention_weight_min": -60.28973388671875,
      "activations/layer6_attention_weight_max": 42.19881057739258,
      "activations/layer6_attention_weight_min": -45.143611907958984,
      "activations/layer7_attention_weight_max": 89.9407730102539,
      "activations/layer7_attention_weight_min": -90.24787902832031,
      "activations/layer8_attention_weight_max": 38.969478607177734,
      "activations/layer8_attention_weight_min": -41.15132522583008,
      "activations/layer9_attention_weight_max": 31.872833251953125,
      "activations/layer9_attention_weight_min": -32.19038009643555,
      "epoch": 15.07,
      "learning_rate": 5.335643939393939e-05,
      "loss": 2.7482,
      "step": 259350
    },
    {
      "activations/layer0_attention_weight_max": 15.398079872131348,
      "activations/layer0_attention_weight_min": -11.57334041595459,
      "activations/layer10_attention_weight_max": 30.77524185180664,
      "activations/layer10_attention_weight_min": -30.86116600036621,
      "activations/layer11_attention_weight_max": 31.274137496948242,
      "activations/layer11_attention_weight_min": -32.103816986083984,
      "activations/layer12_attention_weight_max": 24.628387451171875,
      "activations/layer12_attention_weight_min": -26.155338287353516,
      "activations/layer13_attention_weight_max": 35.05670928955078,
      "activations/layer13_attention_weight_min": -35.064510345458984,
      "activations/layer14_attention_weight_max": 40.603485107421875,
      "activations/layer14_attention_weight_min": -37.26763153076172,
      "activations/layer15_attention_weight_max": 34.82444763183594,
      "activations/layer15_attention_weight_min": -32.741207122802734,
      "activations/layer16_attention_weight_max": 27.03591537475586,
      "activations/layer16_attention_weight_min": -27.30571937561035,
      "activations/layer17_attention_weight_max": 27.148033142089844,
      "activations/layer17_attention_weight_min": -23.945920944213867,
      "activations/layer18_attention_weight_max": 34.5455322265625,
      "activations/layer18_attention_weight_min": -22.712568283081055,
      "activations/layer19_attention_weight_max": 35.98859786987305,
      "activations/layer19_attention_weight_min": -30.936620712280273,
      "activations/layer1_attention_weight_max": 15.436923027038574,
      "activations/layer1_attention_weight_min": -14.910015106201172,
      "activations/layer20_attention_weight_max": 29.239688873291016,
      "activations/layer20_attention_weight_min": -24.206438064575195,
      "activations/layer21_attention_weight_max": 31.533096313476562,
      "activations/layer21_attention_weight_min": -23.307510375976562,
      "activations/layer22_attention_weight_max": 40.49802780151367,
      "activations/layer22_attention_weight_min": -30.341829299926758,
      "activations/layer23_attention_weight_max": 33.80059051513672,
      "activations/layer23_attention_weight_min": -24.90536117553711,
      "activations/layer2_attention_weight_max": 34.09248352050781,
      "activations/layer2_attention_weight_min": -32.465965270996094,
      "activations/layer3_attention_weight_max": 98.38621520996094,
      "activations/layer3_attention_weight_min": -99.04914093017578,
      "activations/layer4_attention_weight_max": 114.5648422241211,
      "activations/layer4_attention_weight_min": -112.49736785888672,
      "activations/layer5_attention_weight_max": 52.795196533203125,
      "activations/layer5_attention_weight_min": -63.72394561767578,
      "activations/layer6_attention_weight_max": 43.128604888916016,
      "activations/layer6_attention_weight_min": -45.839439392089844,
      "activations/layer7_attention_weight_max": 92.430419921875,
      "activations/layer7_attention_weight_min": -91.04530334472656,
      "activations/layer8_attention_weight_max": 41.29880142211914,
      "activations/layer8_attention_weight_min": -42.72221755981445,
      "activations/layer9_attention_weight_max": 32.94704818725586,
      "activations/layer9_attention_weight_min": -32.950347900390625,
      "epoch": 15.07,
      "learning_rate": 5.333749999999999e-05,
      "loss": 2.7332,
      "step": 259400
    },
    {
      "activations/layer0_attention_weight_max": 15.392820358276367,
      "activations/layer0_attention_weight_min": -12.216455459594727,
      "activations/layer10_attention_weight_max": 31.50688934326172,
      "activations/layer10_attention_weight_min": -36.711063385009766,
      "activations/layer11_attention_weight_max": 32.4287109375,
      "activations/layer11_attention_weight_min": -34.43906784057617,
      "activations/layer12_attention_weight_max": 27.977527618408203,
      "activations/layer12_attention_weight_min": -26.049945831298828,
      "activations/layer13_attention_weight_max": 39.76533508300781,
      "activations/layer13_attention_weight_min": -35.598419189453125,
      "activations/layer14_attention_weight_max": 43.97035598754883,
      "activations/layer14_attention_weight_min": -38.135719299316406,
      "activations/layer15_attention_weight_max": 38.879268646240234,
      "activations/layer15_attention_weight_min": -36.005271911621094,
      "activations/layer16_attention_weight_max": 29.663469314575195,
      "activations/layer16_attention_weight_min": -28.744678497314453,
      "activations/layer17_attention_weight_max": 30.20339012145996,
      "activations/layer17_attention_weight_min": -24.84031105041504,
      "activations/layer18_attention_weight_max": 31.82267951965332,
      "activations/layer18_attention_weight_min": -22.797977447509766,
      "activations/layer19_attention_weight_max": 32.676856994628906,
      "activations/layer19_attention_weight_min": -30.06099510192871,
      "activations/layer1_attention_weight_max": 16.218549728393555,
      "activations/layer1_attention_weight_min": -14.331804275512695,
      "activations/layer20_attention_weight_max": 29.005226135253906,
      "activations/layer20_attention_weight_min": -25.335525512695312,
      "activations/layer21_attention_weight_max": 28.237407684326172,
      "activations/layer21_attention_weight_min": -24.011993408203125,
      "activations/layer22_attention_weight_max": 40.259647369384766,
      "activations/layer22_attention_weight_min": -32.21774673461914,
      "activations/layer23_attention_weight_max": 33.50345993041992,
      "activations/layer23_attention_weight_min": -25.14252281188965,
      "activations/layer2_attention_weight_max": 32.72351837158203,
      "activations/layer2_attention_weight_min": -32.43389892578125,
      "activations/layer3_attention_weight_max": 98.01774597167969,
      "activations/layer3_attention_weight_min": -98.0874252319336,
      "activations/layer4_attention_weight_max": 114.4383773803711,
      "activations/layer4_attention_weight_min": -112.79835510253906,
      "activations/layer5_attention_weight_max": 50.41930389404297,
      "activations/layer5_attention_weight_min": -63.844417572021484,
      "activations/layer6_attention_weight_max": 43.7922477722168,
      "activations/layer6_attention_weight_min": -46.486412048339844,
      "activations/layer7_attention_weight_max": 93.32064819335938,
      "activations/layer7_attention_weight_min": -96.80486297607422,
      "activations/layer8_attention_weight_max": 39.364959716796875,
      "activations/layer8_attention_weight_min": -42.010040283203125,
      "activations/layer9_attention_weight_max": 31.175765991210938,
      "activations/layer9_attention_weight_min": -33.961265563964844,
      "epoch": 15.08,
      "learning_rate": 5.3318560606060604e-05,
      "loss": 2.7458,
      "step": 259450
    },
    {
      "activations/layer0_attention_weight_max": 15.4530611038208,
      "activations/layer0_attention_weight_min": -11.952266693115234,
      "activations/layer10_attention_weight_max": 36.92378234863281,
      "activations/layer10_attention_weight_min": -36.8607063293457,
      "activations/layer11_attention_weight_max": 36.27223205566406,
      "activations/layer11_attention_weight_min": -36.934696197509766,
      "activations/layer12_attention_weight_max": 28.177635192871094,
      "activations/layer12_attention_weight_min": -27.374530792236328,
      "activations/layer13_attention_weight_max": 37.85734939575195,
      "activations/layer13_attention_weight_min": -36.09783935546875,
      "activations/layer14_attention_weight_max": 41.379154205322266,
      "activations/layer14_attention_weight_min": -33.97015380859375,
      "activations/layer15_attention_weight_max": 36.95151138305664,
      "activations/layer15_attention_weight_min": -31.61688232421875,
      "activations/layer16_attention_weight_max": 27.83376121520996,
      "activations/layer16_attention_weight_min": -26.616167068481445,
      "activations/layer17_attention_weight_max": 31.012271881103516,
      "activations/layer17_attention_weight_min": -24.353853225708008,
      "activations/layer18_attention_weight_max": 29.151630401611328,
      "activations/layer18_attention_weight_min": -23.66484260559082,
      "activations/layer19_attention_weight_max": 33.05521774291992,
      "activations/layer19_attention_weight_min": -29.910276412963867,
      "activations/layer1_attention_weight_max": 16.660234451293945,
      "activations/layer1_attention_weight_min": -15.440990447998047,
      "activations/layer20_attention_weight_max": 27.784313201904297,
      "activations/layer20_attention_weight_min": -26.5455265045166,
      "activations/layer21_attention_weight_max": 27.893674850463867,
      "activations/layer21_attention_weight_min": -24.331737518310547,
      "activations/layer22_attention_weight_max": 38.32217788696289,
      "activations/layer22_attention_weight_min": -31.087787628173828,
      "activations/layer23_attention_weight_max": 31.539268493652344,
      "activations/layer23_attention_weight_min": -26.4287109375,
      "activations/layer2_attention_weight_max": 34.9144172668457,
      "activations/layer2_attention_weight_min": -32.27680206298828,
      "activations/layer3_attention_weight_max": 95.04264831542969,
      "activations/layer3_attention_weight_min": -97.30162811279297,
      "activations/layer4_attention_weight_max": 111.90602111816406,
      "activations/layer4_attention_weight_min": -110.55378723144531,
      "activations/layer5_attention_weight_max": 56.76833724975586,
      "activations/layer5_attention_weight_min": -61.50599670410156,
      "activations/layer6_attention_weight_max": 43.28175354003906,
      "activations/layer6_attention_weight_min": -44.76533889770508,
      "activations/layer7_attention_weight_max": 101.06571197509766,
      "activations/layer7_attention_weight_min": -96.88274383544922,
      "activations/layer8_attention_weight_max": 46.68981170654297,
      "activations/layer8_attention_weight_min": -41.471492767333984,
      "activations/layer9_attention_weight_max": 32.08462142944336,
      "activations/layer9_attention_weight_min": -34.900569915771484,
      "epoch": 15.08,
      "learning_rate": 5.3299621212121205e-05,
      "loss": 2.7463,
      "step": 259500
    },
    {
      "activations/layer0_attention_weight_max": 15.096707344055176,
      "activations/layer0_attention_weight_min": -12.12768840789795,
      "activations/layer10_attention_weight_max": 31.838090896606445,
      "activations/layer10_attention_weight_min": -33.285797119140625,
      "activations/layer11_attention_weight_max": 31.832130432128906,
      "activations/layer11_attention_weight_min": -33.60103988647461,
      "activations/layer12_attention_weight_max": 25.860319137573242,
      "activations/layer12_attention_weight_min": -26.80339813232422,
      "activations/layer13_attention_weight_max": 39.76481628417969,
      "activations/layer13_attention_weight_min": -35.924560546875,
      "activations/layer14_attention_weight_max": 45.60857391357422,
      "activations/layer14_attention_weight_min": -37.90001678466797,
      "activations/layer15_attention_weight_max": 41.23105239868164,
      "activations/layer15_attention_weight_min": -36.759063720703125,
      "activations/layer16_attention_weight_max": 31.514392852783203,
      "activations/layer16_attention_weight_min": -28.336381912231445,
      "activations/layer17_attention_weight_max": 38.04024124145508,
      "activations/layer17_attention_weight_min": -27.475627899169922,
      "activations/layer18_attention_weight_max": 31.508886337280273,
      "activations/layer18_attention_weight_min": -26.154844284057617,
      "activations/layer19_attention_weight_max": 35.82108688354492,
      "activations/layer19_attention_weight_min": -33.95414733886719,
      "activations/layer1_attention_weight_max": 15.605682373046875,
      "activations/layer1_attention_weight_min": -14.627816200256348,
      "activations/layer20_attention_weight_max": 30.264482498168945,
      "activations/layer20_attention_weight_min": -27.459014892578125,
      "activations/layer21_attention_weight_max": 30.5683650970459,
      "activations/layer21_attention_weight_min": -26.934221267700195,
      "activations/layer22_attention_weight_max": 52.23202896118164,
      "activations/layer22_attention_weight_min": -31.988645553588867,
      "activations/layer23_attention_weight_max": 35.359893798828125,
      "activations/layer23_attention_weight_min": -24.87080192565918,
      "activations/layer2_attention_weight_max": 32.95035934448242,
      "activations/layer2_attention_weight_min": -33.604740142822266,
      "activations/layer3_attention_weight_max": 98.82508850097656,
      "activations/layer3_attention_weight_min": -99.3923568725586,
      "activations/layer4_attention_weight_max": 114.440185546875,
      "activations/layer4_attention_weight_min": -112.88941955566406,
      "activations/layer5_attention_weight_max": 56.518104553222656,
      "activations/layer5_attention_weight_min": -62.61019515991211,
      "activations/layer6_attention_weight_max": 43.269161224365234,
      "activations/layer6_attention_weight_min": -47.58244705200195,
      "activations/layer7_attention_weight_max": 87.70087432861328,
      "activations/layer7_attention_weight_min": -92.60481262207031,
      "activations/layer8_attention_weight_max": 39.07778549194336,
      "activations/layer8_attention_weight_min": -41.86332702636719,
      "activations/layer9_attention_weight_max": 30.3941650390625,
      "activations/layer9_attention_weight_min": -32.81290054321289,
      "epoch": 15.08,
      "learning_rate": 5.3280681818181813e-05,
      "loss": 2.7589,
      "step": 259550
    },
    {
      "activations/layer0_attention_weight_max": 14.980144500732422,
      "activations/layer0_attention_weight_min": -11.868902206420898,
      "activations/layer10_attention_weight_max": 33.84215545654297,
      "activations/layer10_attention_weight_min": -32.99532699584961,
      "activations/layer11_attention_weight_max": 35.77886962890625,
      "activations/layer11_attention_weight_min": -32.30579376220703,
      "activations/layer12_attention_weight_max": 28.48331069946289,
      "activations/layer12_attention_weight_min": -26.357547760009766,
      "activations/layer13_attention_weight_max": 40.45787048339844,
      "activations/layer13_attention_weight_min": -35.15298843383789,
      "activations/layer14_attention_weight_max": 42.596927642822266,
      "activations/layer14_attention_weight_min": -36.30698776245117,
      "activations/layer15_attention_weight_max": 40.541446685791016,
      "activations/layer15_attention_weight_min": -33.461570739746094,
      "activations/layer16_attention_weight_max": 30.395471572875977,
      "activations/layer16_attention_weight_min": -30.42235565185547,
      "activations/layer17_attention_weight_max": 32.57987594604492,
      "activations/layer17_attention_weight_min": -26.764968872070312,
      "activations/layer18_attention_weight_max": 35.661930084228516,
      "activations/layer18_attention_weight_min": -26.465667724609375,
      "activations/layer19_attention_weight_max": 34.54581832885742,
      "activations/layer19_attention_weight_min": -32.05195236206055,
      "activations/layer1_attention_weight_max": 16.142925262451172,
      "activations/layer1_attention_weight_min": -14.511567115783691,
      "activations/layer20_attention_weight_max": 33.16404724121094,
      "activations/layer20_attention_weight_min": -27.051002502441406,
      "activations/layer21_attention_weight_max": 30.57439422607422,
      "activations/layer21_attention_weight_min": -26.95744514465332,
      "activations/layer22_attention_weight_max": 45.00055694580078,
      "activations/layer22_attention_weight_min": -31.785457611083984,
      "activations/layer23_attention_weight_max": 37.25891876220703,
      "activations/layer23_attention_weight_min": -28.50507926940918,
      "activations/layer2_attention_weight_max": 32.57362365722656,
      "activations/layer2_attention_weight_min": -32.39216995239258,
      "activations/layer3_attention_weight_max": 96.28375244140625,
      "activations/layer3_attention_weight_min": -98.09356689453125,
      "activations/layer4_attention_weight_max": 108.26615142822266,
      "activations/layer4_attention_weight_min": -112.29228973388672,
      "activations/layer5_attention_weight_max": 49.940731048583984,
      "activations/layer5_attention_weight_min": -61.38964080810547,
      "activations/layer6_attention_weight_max": 43.21910095214844,
      "activations/layer6_attention_weight_min": -44.80620193481445,
      "activations/layer7_attention_weight_max": 89.03263854980469,
      "activations/layer7_attention_weight_min": -90.23968505859375,
      "activations/layer8_attention_weight_max": 39.53864288330078,
      "activations/layer8_attention_weight_min": -40.79423141479492,
      "activations/layer9_attention_weight_max": 31.000959396362305,
      "activations/layer9_attention_weight_min": -32.36825180053711,
      "epoch": 15.08,
      "learning_rate": 5.326174242424242e-05,
      "loss": 2.7577,
      "step": 259600
    },
    {
      "activations/layer0_attention_weight_max": 15.78663444519043,
      "activations/layer0_attention_weight_min": -11.824728965759277,
      "activations/layer10_attention_weight_max": 38.67692565917969,
      "activations/layer10_attention_weight_min": -37.729583740234375,
      "activations/layer11_attention_weight_max": 34.49317169189453,
      "activations/layer11_attention_weight_min": -36.47371292114258,
      "activations/layer12_attention_weight_max": 27.651859283447266,
      "activations/layer12_attention_weight_min": -26.7033634185791,
      "activations/layer13_attention_weight_max": 41.76667022705078,
      "activations/layer13_attention_weight_min": -36.66596603393555,
      "activations/layer14_attention_weight_max": 47.301605224609375,
      "activations/layer14_attention_weight_min": -38.916893005371094,
      "activations/layer15_attention_weight_max": 38.77421951293945,
      "activations/layer15_attention_weight_min": -34.70175552368164,
      "activations/layer16_attention_weight_max": 30.516191482543945,
      "activations/layer16_attention_weight_min": -28.828020095825195,
      "activations/layer17_attention_weight_max": 31.7731876373291,
      "activations/layer17_attention_weight_min": -25.247859954833984,
      "activations/layer18_attention_weight_max": 33.930931091308594,
      "activations/layer18_attention_weight_min": -26.470930099487305,
      "activations/layer19_attention_weight_max": 36.043392181396484,
      "activations/layer19_attention_weight_min": -35.225189208984375,
      "activations/layer1_attention_weight_max": 15.741621971130371,
      "activations/layer1_attention_weight_min": -15.055938720703125,
      "activations/layer20_attention_weight_max": 30.677967071533203,
      "activations/layer20_attention_weight_min": -27.10988426208496,
      "activations/layer21_attention_weight_max": 29.839889526367188,
      "activations/layer21_attention_weight_min": -26.30306625366211,
      "activations/layer22_attention_weight_max": 49.34189224243164,
      "activations/layer22_attention_weight_min": -33.656734466552734,
      "activations/layer23_attention_weight_max": 35.56313705444336,
      "activations/layer23_attention_weight_min": -24.40216827392578,
      "activations/layer2_attention_weight_max": 33.32683181762695,
      "activations/layer2_attention_weight_min": -33.23956298828125,
      "activations/layer3_attention_weight_max": 100.7994155883789,
      "activations/layer3_attention_weight_min": -99.47205352783203,
      "activations/layer4_attention_weight_max": 109.83238983154297,
      "activations/layer4_attention_weight_min": -111.78560638427734,
      "activations/layer5_attention_weight_max": 51.6217155456543,
      "activations/layer5_attention_weight_min": -63.07497787475586,
      "activations/layer6_attention_weight_max": 46.26923751831055,
      "activations/layer6_attention_weight_min": -47.4861946105957,
      "activations/layer7_attention_weight_max": 96.74640655517578,
      "activations/layer7_attention_weight_min": -98.04393005371094,
      "activations/layer8_attention_weight_max": 43.6915168762207,
      "activations/layer8_attention_weight_min": -48.5445442199707,
      "activations/layer9_attention_weight_max": 37.056884765625,
      "activations/layer9_attention_weight_min": -35.71352767944336,
      "epoch": 15.09,
      "learning_rate": 5.324280303030303e-05,
      "loss": 2.7436,
      "step": 259650
    },
    {
      "activations/layer0_attention_weight_max": 15.611384391784668,
      "activations/layer0_attention_weight_min": -11.874951362609863,
      "activations/layer10_attention_weight_max": 36.097904205322266,
      "activations/layer10_attention_weight_min": -33.78153610229492,
      "activations/layer11_attention_weight_max": 35.601749420166016,
      "activations/layer11_attention_weight_min": -34.87495422363281,
      "activations/layer12_attention_weight_max": 28.198328018188477,
      "activations/layer12_attention_weight_min": -25.46977996826172,
      "activations/layer13_attention_weight_max": 37.75664520263672,
      "activations/layer13_attention_weight_min": -34.43356704711914,
      "activations/layer14_attention_weight_max": 42.857810974121094,
      "activations/layer14_attention_weight_min": -36.17949676513672,
      "activations/layer15_attention_weight_max": 37.1618766784668,
      "activations/layer15_attention_weight_min": -33.386260986328125,
      "activations/layer16_attention_weight_max": 28.19964599609375,
      "activations/layer16_attention_weight_min": -27.88860321044922,
      "activations/layer17_attention_weight_max": 30.560089111328125,
      "activations/layer17_attention_weight_min": -27.547536849975586,
      "activations/layer18_attention_weight_max": 34.88705062866211,
      "activations/layer18_attention_weight_min": -26.034948348999023,
      "activations/layer19_attention_weight_max": 34.00461959838867,
      "activations/layer19_attention_weight_min": -33.09474563598633,
      "activations/layer1_attention_weight_max": 16.403549194335938,
      "activations/layer1_attention_weight_min": -15.702136993408203,
      "activations/layer20_attention_weight_max": 28.8355712890625,
      "activations/layer20_attention_weight_min": -26.97454071044922,
      "activations/layer21_attention_weight_max": 33.520442962646484,
      "activations/layer21_attention_weight_min": -27.120006561279297,
      "activations/layer22_attention_weight_max": 45.36555480957031,
      "activations/layer22_attention_weight_min": -33.93193435668945,
      "activations/layer23_attention_weight_max": 34.164772033691406,
      "activations/layer23_attention_weight_min": -27.576568603515625,
      "activations/layer2_attention_weight_max": 35.33998107910156,
      "activations/layer2_attention_weight_min": -32.303680419921875,
      "activations/layer3_attention_weight_max": 99.1695556640625,
      "activations/layer3_attention_weight_min": -97.47300720214844,
      "activations/layer4_attention_weight_max": 111.52165985107422,
      "activations/layer4_attention_weight_min": -111.24940490722656,
      "activations/layer5_attention_weight_max": 52.943702697753906,
      "activations/layer5_attention_weight_min": -64.12295532226562,
      "activations/layer6_attention_weight_max": 41.8980712890625,
      "activations/layer6_attention_weight_min": -45.23082733154297,
      "activations/layer7_attention_weight_max": 101.1053237915039,
      "activations/layer7_attention_weight_min": -92.41188049316406,
      "activations/layer8_attention_weight_max": 43.222408294677734,
      "activations/layer8_attention_weight_min": -43.11094284057617,
      "activations/layer9_attention_weight_max": 32.407073974609375,
      "activations/layer9_attention_weight_min": -35.45210647583008,
      "epoch": 15.09,
      "learning_rate": 5.322386363636363e-05,
      "loss": 2.7564,
      "step": 259700
    },
    {
      "activations/layer0_attention_weight_max": 14.841714859008789,
      "activations/layer0_attention_weight_min": -12.151988983154297,
      "activations/layer10_attention_weight_max": 31.513273239135742,
      "activations/layer10_attention_weight_min": -35.504676818847656,
      "activations/layer11_attention_weight_max": 32.446495056152344,
      "activations/layer11_attention_weight_min": -33.05106735229492,
      "activations/layer12_attention_weight_max": 27.586769104003906,
      "activations/layer12_attention_weight_min": -26.183124542236328,
      "activations/layer13_attention_weight_max": 41.97698211669922,
      "activations/layer13_attention_weight_min": -34.57477569580078,
      "activations/layer14_attention_weight_max": 41.765159606933594,
      "activations/layer14_attention_weight_min": -37.70024108886719,
      "activations/layer15_attention_weight_max": 37.13596725463867,
      "activations/layer15_attention_weight_min": -36.444156646728516,
      "activations/layer16_attention_weight_max": 31.50695037841797,
      "activations/layer16_attention_weight_min": -28.59100914001465,
      "activations/layer17_attention_weight_max": 32.08866500854492,
      "activations/layer17_attention_weight_min": -25.920358657836914,
      "activations/layer18_attention_weight_max": 29.698434829711914,
      "activations/layer18_attention_weight_min": -26.39637565612793,
      "activations/layer19_attention_weight_max": 35.94149398803711,
      "activations/layer19_attention_weight_min": -30.56012535095215,
      "activations/layer1_attention_weight_max": 16.020565032958984,
      "activations/layer1_attention_weight_min": -14.199450492858887,
      "activations/layer20_attention_weight_max": 31.318464279174805,
      "activations/layer20_attention_weight_min": -24.70707893371582,
      "activations/layer21_attention_weight_max": 28.80352783203125,
      "activations/layer21_attention_weight_min": -25.235782623291016,
      "activations/layer22_attention_weight_max": 42.603599548339844,
      "activations/layer22_attention_weight_min": -33.05148696899414,
      "activations/layer23_attention_weight_max": 32.22277069091797,
      "activations/layer23_attention_weight_min": -24.759944915771484,
      "activations/layer2_attention_weight_max": 32.068302154541016,
      "activations/layer2_attention_weight_min": -31.327220916748047,
      "activations/layer3_attention_weight_max": 92.10755920410156,
      "activations/layer3_attention_weight_min": -96.81929016113281,
      "activations/layer4_attention_weight_max": 104.0286865234375,
      "activations/layer4_attention_weight_min": -108.16104888916016,
      "activations/layer5_attention_weight_max": 50.12936019897461,
      "activations/layer5_attention_weight_min": -60.00739669799805,
      "activations/layer6_attention_weight_max": 42.5130615234375,
      "activations/layer6_attention_weight_min": -45.545616149902344,
      "activations/layer7_attention_weight_max": 86.77960205078125,
      "activations/layer7_attention_weight_min": -92.65200805664062,
      "activations/layer8_attention_weight_max": 40.82273483276367,
      "activations/layer8_attention_weight_min": -41.1470947265625,
      "activations/layer9_attention_weight_max": 31.32038688659668,
      "activations/layer9_attention_weight_min": -34.69308853149414,
      "epoch": 15.09,
      "learning_rate": 5.320492424242423e-05,
      "loss": 2.7567,
      "step": 259750
    },
    {
      "activations/layer0_attention_weight_max": 15.57388687133789,
      "activations/layer0_attention_weight_min": -13.24063777923584,
      "activations/layer10_attention_weight_max": 33.02106857299805,
      "activations/layer10_attention_weight_min": -34.284000396728516,
      "activations/layer11_attention_weight_max": 32.56694793701172,
      "activations/layer11_attention_weight_min": -33.49470138549805,
      "activations/layer12_attention_weight_max": 26.362350463867188,
      "activations/layer12_attention_weight_min": -27.86383628845215,
      "activations/layer13_attention_weight_max": 38.27094650268555,
      "activations/layer13_attention_weight_min": -40.61029052734375,
      "activations/layer14_attention_weight_max": 43.63956832885742,
      "activations/layer14_attention_weight_min": -39.89122009277344,
      "activations/layer15_attention_weight_max": 38.65022277832031,
      "activations/layer15_attention_weight_min": -36.771240234375,
      "activations/layer16_attention_weight_max": 29.409568786621094,
      "activations/layer16_attention_weight_min": -28.99835968017578,
      "activations/layer17_attention_weight_max": 29.684574127197266,
      "activations/layer17_attention_weight_min": -27.450483322143555,
      "activations/layer18_attention_weight_max": 30.59811019897461,
      "activations/layer18_attention_weight_min": -24.14227867126465,
      "activations/layer19_attention_weight_max": 33.278907775878906,
      "activations/layer19_attention_weight_min": -33.03499221801758,
      "activations/layer1_attention_weight_max": 17.32227325439453,
      "activations/layer1_attention_weight_min": -15.709497451782227,
      "activations/layer20_attention_weight_max": 26.020889282226562,
      "activations/layer20_attention_weight_min": -25.464426040649414,
      "activations/layer21_attention_weight_max": 26.760568618774414,
      "activations/layer21_attention_weight_min": -24.12444305419922,
      "activations/layer22_attention_weight_max": 40.73223876953125,
      "activations/layer22_attention_weight_min": -31.40884017944336,
      "activations/layer23_attention_weight_max": 29.95663833618164,
      "activations/layer23_attention_weight_min": -25.946365356445312,
      "activations/layer2_attention_weight_max": 32.486297607421875,
      "activations/layer2_attention_weight_min": -31.30254364013672,
      "activations/layer3_attention_weight_max": 94.8552017211914,
      "activations/layer3_attention_weight_min": -94.5630874633789,
      "activations/layer4_attention_weight_max": 109.03338623046875,
      "activations/layer4_attention_weight_min": -111.75361633300781,
      "activations/layer5_attention_weight_max": 51.2131462097168,
      "activations/layer5_attention_weight_min": -64.22647857666016,
      "activations/layer6_attention_weight_max": 42.740047454833984,
      "activations/layer6_attention_weight_min": -45.082027435302734,
      "activations/layer7_attention_weight_max": 91.93869018554688,
      "activations/layer7_attention_weight_min": -98.6155776977539,
      "activations/layer8_attention_weight_max": 39.50498962402344,
      "activations/layer8_attention_weight_min": -42.71940231323242,
      "activations/layer9_attention_weight_max": 31.005823135375977,
      "activations/layer9_attention_weight_min": -32.673221588134766,
      "epoch": 15.1,
      "learning_rate": 5.318598484848485e-05,
      "loss": 2.7372,
      "step": 259800
    },
    {
      "activations/layer0_attention_weight_max": 15.952953338623047,
      "activations/layer0_attention_weight_min": -12.565276145935059,
      "activations/layer10_attention_weight_max": 36.46238708496094,
      "activations/layer10_attention_weight_min": -36.732547760009766,
      "activations/layer11_attention_weight_max": 34.45372772216797,
      "activations/layer11_attention_weight_min": -35.37032699584961,
      "activations/layer12_attention_weight_max": 28.35049057006836,
      "activations/layer12_attention_weight_min": -26.317996978759766,
      "activations/layer13_attention_weight_max": 43.40354919433594,
      "activations/layer13_attention_weight_min": -36.58646774291992,
      "activations/layer14_attention_weight_max": 46.10658264160156,
      "activations/layer14_attention_weight_min": -38.6346321105957,
      "activations/layer15_attention_weight_max": 43.07578659057617,
      "activations/layer15_attention_weight_min": -34.814517974853516,
      "activations/layer16_attention_weight_max": 30.72368621826172,
      "activations/layer16_attention_weight_min": -29.686494827270508,
      "activations/layer17_attention_weight_max": 31.931190490722656,
      "activations/layer17_attention_weight_min": -27.1717472076416,
      "activations/layer18_attention_weight_max": 33.51848220825195,
      "activations/layer18_attention_weight_min": -25.707996368408203,
      "activations/layer19_attention_weight_max": 35.24135208129883,
      "activations/layer19_attention_weight_min": -30.519620895385742,
      "activations/layer1_attention_weight_max": 16.861892700195312,
      "activations/layer1_attention_weight_min": -14.838956832885742,
      "activations/layer20_attention_weight_max": 30.67229461669922,
      "activations/layer20_attention_weight_min": -25.22026824951172,
      "activations/layer21_attention_weight_max": 30.053569793701172,
      "activations/layer21_attention_weight_min": -23.673614501953125,
      "activations/layer22_attention_weight_max": 46.77946090698242,
      "activations/layer22_attention_weight_min": -30.86761474609375,
      "activations/layer23_attention_weight_max": 34.10139465332031,
      "activations/layer23_attention_weight_min": -25.036046981811523,
      "activations/layer2_attention_weight_max": 31.871417999267578,
      "activations/layer2_attention_weight_min": -31.531057357788086,
      "activations/layer3_attention_weight_max": 95.94680786132812,
      "activations/layer3_attention_weight_min": -99.05064392089844,
      "activations/layer4_attention_weight_max": 109.19139862060547,
      "activations/layer4_attention_weight_min": -109.98970031738281,
      "activations/layer5_attention_weight_max": 51.64973449707031,
      "activations/layer5_attention_weight_min": -60.34702682495117,
      "activations/layer6_attention_weight_max": 45.389930725097656,
      "activations/layer6_attention_weight_min": -46.77085494995117,
      "activations/layer7_attention_weight_max": 92.65514373779297,
      "activations/layer7_attention_weight_min": -92.92333221435547,
      "activations/layer8_attention_weight_max": 42.05937957763672,
      "activations/layer8_attention_weight_min": -42.7157096862793,
      "activations/layer9_attention_weight_max": 33.418453216552734,
      "activations/layer9_attention_weight_min": -36.27324676513672,
      "epoch": 15.1,
      "learning_rate": 5.316704545454545e-05,
      "loss": 2.7559,
      "step": 259850
    },
    {
      "activations/layer0_attention_weight_max": 15.955245018005371,
      "activations/layer0_attention_weight_min": -13.881112098693848,
      "activations/layer10_attention_weight_max": 31.702489852905273,
      "activations/layer10_attention_weight_min": -31.36840057373047,
      "activations/layer11_attention_weight_max": 33.423851013183594,
      "activations/layer11_attention_weight_min": -32.60441589355469,
      "activations/layer12_attention_weight_max": 23.402265548706055,
      "activations/layer12_attention_weight_min": -26.626604080200195,
      "activations/layer13_attention_weight_max": 36.527645111083984,
      "activations/layer13_attention_weight_min": -34.61729431152344,
      "activations/layer14_attention_weight_max": 41.98463821411133,
      "activations/layer14_attention_weight_min": -37.71778869628906,
      "activations/layer15_attention_weight_max": 35.81094741821289,
      "activations/layer15_attention_weight_min": -32.19749069213867,
      "activations/layer16_attention_weight_max": 26.29375648498535,
      "activations/layer16_attention_weight_min": -26.752769470214844,
      "activations/layer17_attention_weight_max": 28.045330047607422,
      "activations/layer17_attention_weight_min": -26.09208106994629,
      "activations/layer18_attention_weight_max": 30.081289291381836,
      "activations/layer18_attention_weight_min": -24.140764236450195,
      "activations/layer19_attention_weight_max": 30.721532821655273,
      "activations/layer19_attention_weight_min": -30.2395076751709,
      "activations/layer1_attention_weight_max": 21.770858764648438,
      "activations/layer1_attention_weight_min": -17.419506072998047,
      "activations/layer20_attention_weight_max": 27.696800231933594,
      "activations/layer20_attention_weight_min": -24.31879234313965,
      "activations/layer21_attention_weight_max": 26.035991668701172,
      "activations/layer21_attention_weight_min": -24.76923179626465,
      "activations/layer22_attention_weight_max": 37.39845657348633,
      "activations/layer22_attention_weight_min": -30.240598678588867,
      "activations/layer23_attention_weight_max": 30.61897850036621,
      "activations/layer23_attention_weight_min": -24.76511001586914,
      "activations/layer2_attention_weight_max": 35.664520263671875,
      "activations/layer2_attention_weight_min": -33.56691360473633,
      "activations/layer3_attention_weight_max": 101.03014373779297,
      "activations/layer3_attention_weight_min": -100.45323181152344,
      "activations/layer4_attention_weight_max": 111.8875503540039,
      "activations/layer4_attention_weight_min": -113.7363052368164,
      "activations/layer5_attention_weight_max": 51.92011260986328,
      "activations/layer5_attention_weight_min": -60.975616455078125,
      "activations/layer6_attention_weight_max": 44.87671661376953,
      "activations/layer6_attention_weight_min": -47.641441345214844,
      "activations/layer7_attention_weight_max": 103.5096435546875,
      "activations/layer7_attention_weight_min": -97.49198913574219,
      "activations/layer8_attention_weight_max": 41.186622619628906,
      "activations/layer8_attention_weight_min": -41.23212814331055,
      "activations/layer9_attention_weight_max": 32.87820053100586,
      "activations/layer9_attention_weight_min": -34.383399963378906,
      "epoch": 15.1,
      "learning_rate": 5.314810606060606e-05,
      "loss": 2.7431,
      "step": 259900
    },
    {
      "activations/layer0_attention_weight_max": 16.056760787963867,
      "activations/layer0_attention_weight_min": -12.313960075378418,
      "activations/layer10_attention_weight_max": 36.61640167236328,
      "activations/layer10_attention_weight_min": -38.13189697265625,
      "activations/layer11_attention_weight_max": 36.15139389038086,
      "activations/layer11_attention_weight_min": -39.281192779541016,
      "activations/layer12_attention_weight_max": 30.75649642944336,
      "activations/layer12_attention_weight_min": -26.55337142944336,
      "activations/layer13_attention_weight_max": 44.74103546142578,
      "activations/layer13_attention_weight_min": -36.27499771118164,
      "activations/layer14_attention_weight_max": 48.35374450683594,
      "activations/layer14_attention_weight_min": -38.07862854003906,
      "activations/layer15_attention_weight_max": 41.057220458984375,
      "activations/layer15_attention_weight_min": -34.795860290527344,
      "activations/layer16_attention_weight_max": 31.16260528564453,
      "activations/layer16_attention_weight_min": -26.92582130432129,
      "activations/layer17_attention_weight_max": 34.255306243896484,
      "activations/layer17_attention_weight_min": -25.795013427734375,
      "activations/layer18_attention_weight_max": 35.585323333740234,
      "activations/layer18_attention_weight_min": -24.623075485229492,
      "activations/layer19_attention_weight_max": 37.502838134765625,
      "activations/layer19_attention_weight_min": -32.278297424316406,
      "activations/layer1_attention_weight_max": 16.051944732666016,
      "activations/layer1_attention_weight_min": -14.539898872375488,
      "activations/layer20_attention_weight_max": 35.32479476928711,
      "activations/layer20_attention_weight_min": -26.071094512939453,
      "activations/layer21_attention_weight_max": 32.405113220214844,
      "activations/layer21_attention_weight_min": -24.153188705444336,
      "activations/layer22_attention_weight_max": 51.32796096801758,
      "activations/layer22_attention_weight_min": -32.83435821533203,
      "activations/layer23_attention_weight_max": 37.436431884765625,
      "activations/layer23_attention_weight_min": -24.620718002319336,
      "activations/layer2_attention_weight_max": 31.552465438842773,
      "activations/layer2_attention_weight_min": -31.009870529174805,
      "activations/layer3_attention_weight_max": 95.83653259277344,
      "activations/layer3_attention_weight_min": -98.25184631347656,
      "activations/layer4_attention_weight_max": 109.23844146728516,
      "activations/layer4_attention_weight_min": -112.37725830078125,
      "activations/layer5_attention_weight_max": 50.98036193847656,
      "activations/layer5_attention_weight_min": -61.18737030029297,
      "activations/layer6_attention_weight_max": 44.72041320800781,
      "activations/layer6_attention_weight_min": -47.69834518432617,
      "activations/layer7_attention_weight_max": 96.73419952392578,
      "activations/layer7_attention_weight_min": -96.86630249023438,
      "activations/layer8_attention_weight_max": 44.00897979736328,
      "activations/layer8_attention_weight_min": -45.238033294677734,
      "activations/layer9_attention_weight_max": 37.83010482788086,
      "activations/layer9_attention_weight_min": -37.01445007324219,
      "epoch": 15.1,
      "learning_rate": 5.312954545454545e-05,
      "loss": 2.7546,
      "step": 259950
    },
    {
      "activations/layer0_attention_weight_max": 15.581552505493164,
      "activations/layer0_attention_weight_min": -13.112385749816895,
      "activations/layer10_attention_weight_max": 33.891639709472656,
      "activations/layer10_attention_weight_min": -34.085506439208984,
      "activations/layer11_attention_weight_max": 34.4766845703125,
      "activations/layer11_attention_weight_min": -34.23463821411133,
      "activations/layer12_attention_weight_max": 24.84482765197754,
      "activations/layer12_attention_weight_min": -25.18324089050293,
      "activations/layer13_attention_weight_max": 39.547874450683594,
      "activations/layer13_attention_weight_min": -36.49144744873047,
      "activations/layer14_attention_weight_max": 40.238033294677734,
      "activations/layer14_attention_weight_min": -37.40274429321289,
      "activations/layer15_attention_weight_max": 34.970489501953125,
      "activations/layer15_attention_weight_min": -34.0372200012207,
      "activations/layer16_attention_weight_max": 29.046022415161133,
      "activations/layer16_attention_weight_min": -29.962472915649414,
      "activations/layer17_attention_weight_max": 32.27473449707031,
      "activations/layer17_attention_weight_min": -27.147926330566406,
      "activations/layer18_attention_weight_max": 31.245723724365234,
      "activations/layer18_attention_weight_min": -24.37922477722168,
      "activations/layer19_attention_weight_max": 33.57321548461914,
      "activations/layer19_attention_weight_min": -33.422733306884766,
      "activations/layer1_attention_weight_max": 20.028722763061523,
      "activations/layer1_attention_weight_min": -15.590816497802734,
      "activations/layer20_attention_weight_max": 28.49965476989746,
      "activations/layer20_attention_weight_min": -27.107749938964844,
      "activations/layer21_attention_weight_max": 29.425432205200195,
      "activations/layer21_attention_weight_min": -24.29802894592285,
      "activations/layer22_attention_weight_max": 44.41012191772461,
      "activations/layer22_attention_weight_min": -31.145986557006836,
      "activations/layer23_attention_weight_max": 34.720245361328125,
      "activations/layer23_attention_weight_min": -27.010038375854492,
      "activations/layer2_attention_weight_max": 30.063901901245117,
      "activations/layer2_attention_weight_min": -30.924766540527344,
      "activations/layer3_attention_weight_max": 96.75069427490234,
      "activations/layer3_attention_weight_min": -100.9103775024414,
      "activations/layer4_attention_weight_max": 110.96624755859375,
      "activations/layer4_attention_weight_min": -109.75044250488281,
      "activations/layer5_attention_weight_max": 55.635963439941406,
      "activations/layer5_attention_weight_min": -64.75958251953125,
      "activations/layer6_attention_weight_max": 49.28728103637695,
      "activations/layer6_attention_weight_min": -49.203426361083984,
      "activations/layer7_attention_weight_max": 93.86275482177734,
      "activations/layer7_attention_weight_min": -98.55082702636719,
      "activations/layer8_attention_weight_max": 43.2620735168457,
      "activations/layer8_attention_weight_min": -45.1204719543457,
      "activations/layer9_attention_weight_max": 33.68923568725586,
      "activations/layer9_attention_weight_min": -35.4183464050293,
      "epoch": 15.11,
      "learning_rate": 5.311060606060606e-05,
      "loss": 2.7532,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.5612,
      "eval_samples_per_second": 501.564,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.5612,
      "eval_openwebtext_samples_per_second": 501.564,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 2.0218,
      "eval_wikitext_samples_per_second": 225.539,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_lambada_loss": 2.435546875,
      "eval_lambada_ppl": 11.422063446614425,
      "eval_lambada_runtime": 9.6193,
      "eval_lambada_samples_per_second": 506.169,
      "step": 260000
    },
    {
      "activations/layer0_attention_weight_max": 15.896284103393555,
      "activations/layer0_attention_weight_min": -12.811054229736328,
      "activations/layer10_attention_weight_max": 32.583839416503906,
      "activations/layer10_attention_weight_min": -34.29168701171875,
      "activations/layer11_attention_weight_max": 33.29472351074219,
      "activations/layer11_attention_weight_min": -33.553863525390625,
      "activations/layer12_attention_weight_max": 26.621644973754883,
      "activations/layer12_attention_weight_min": -25.66454315185547,
      "activations/layer13_attention_weight_max": 39.190608978271484,
      "activations/layer13_attention_weight_min": -34.6231803894043,
      "activations/layer14_attention_weight_max": 46.551212310791016,
      "activations/layer14_attention_weight_min": -39.72919464111328,
      "activations/layer15_attention_weight_max": 39.31864547729492,
      "activations/layer15_attention_weight_min": -35.19443130493164,
      "activations/layer16_attention_weight_max": 30.800634384155273,
      "activations/layer16_attention_weight_min": -32.66039276123047,
      "activations/layer17_attention_weight_max": 30.43001937866211,
      "activations/layer17_attention_weight_min": -25.780481338500977,
      "activations/layer18_attention_weight_max": 31.849468231201172,
      "activations/layer18_attention_weight_min": -24.889057159423828,
      "activations/layer19_attention_weight_max": 39.14207458496094,
      "activations/layer19_attention_weight_min": -30.86884880065918,
      "activations/layer1_attention_weight_max": 16.27840232849121,
      "activations/layer1_attention_weight_min": -14.5482816696167,
      "activations/layer20_attention_weight_max": 31.906349182128906,
      "activations/layer20_attention_weight_min": -26.28219223022461,
      "activations/layer21_attention_weight_max": 32.15111541748047,
      "activations/layer21_attention_weight_min": -23.341623306274414,
      "activations/layer22_attention_weight_max": 40.94593048095703,
      "activations/layer22_attention_weight_min": -30.77521514892578,
      "activations/layer23_attention_weight_max": 35.05085372924805,
      "activations/layer23_attention_weight_min": -25.005672454833984,
      "activations/layer2_attention_weight_max": 33.38420867919922,
      "activations/layer2_attention_weight_min": -32.447608947753906,
      "activations/layer3_attention_weight_max": 96.60211944580078,
      "activations/layer3_attention_weight_min": -92.96430969238281,
      "activations/layer4_attention_weight_max": 109.89116668701172,
      "activations/layer4_attention_weight_min": -109.46356201171875,
      "activations/layer5_attention_weight_max": 52.606746673583984,
      "activations/layer5_attention_weight_min": -63.40959930419922,
      "activations/layer6_attention_weight_max": 43.87077713012695,
      "activations/layer6_attention_weight_min": -46.1605339050293,
      "activations/layer7_attention_weight_max": 90.0179443359375,
      "activations/layer7_attention_weight_min": -93.02542877197266,
      "activations/layer8_attention_weight_max": 41.13157272338867,
      "activations/layer8_attention_weight_min": -44.24150848388672,
      "activations/layer9_attention_weight_max": 32.610389709472656,
      "activations/layer9_attention_weight_min": -34.52859115600586,
      "epoch": 15.11,
      "learning_rate": 5.3091666666666665e-05,
      "loss": 2.7457,
      "step": 260050
    },
    {
      "activations/layer0_attention_weight_max": 17.429941177368164,
      "activations/layer0_attention_weight_min": -12.14488697052002,
      "activations/layer10_attention_weight_max": 31.599777221679688,
      "activations/layer10_attention_weight_min": -34.46597671508789,
      "activations/layer11_attention_weight_max": 33.324928283691406,
      "activations/layer11_attention_weight_min": -36.9722900390625,
      "activations/layer12_attention_weight_max": 24.601648330688477,
      "activations/layer12_attention_weight_min": -25.8652400970459,
      "activations/layer13_attention_weight_max": 40.97463607788086,
      "activations/layer13_attention_weight_min": -34.09782028198242,
      "activations/layer14_attention_weight_max": 40.23616027832031,
      "activations/layer14_attention_weight_min": -37.533599853515625,
      "activations/layer15_attention_weight_max": 36.94526290893555,
      "activations/layer15_attention_weight_min": -32.779056549072266,
      "activations/layer16_attention_weight_max": 29.34628677368164,
      "activations/layer16_attention_weight_min": -27.718950271606445,
      "activations/layer17_attention_weight_max": 32.9271125793457,
      "activations/layer17_attention_weight_min": -26.437999725341797,
      "activations/layer18_attention_weight_max": 31.805465698242188,
      "activations/layer18_attention_weight_min": -25.495023727416992,
      "activations/layer19_attention_weight_max": 33.31155014038086,
      "activations/layer19_attention_weight_min": -32.62068557739258,
      "activations/layer1_attention_weight_max": 16.879629135131836,
      "activations/layer1_attention_weight_min": -14.283133506774902,
      "activations/layer20_attention_weight_max": 30.53053855895996,
      "activations/layer20_attention_weight_min": -24.552526473999023,
      "activations/layer21_attention_weight_max": 30.213714599609375,
      "activations/layer21_attention_weight_min": -23.92902183532715,
      "activations/layer22_attention_weight_max": 44.13147735595703,
      "activations/layer22_attention_weight_min": -29.668901443481445,
      "activations/layer23_attention_weight_max": 34.44580078125,
      "activations/layer23_attention_weight_min": -26.248340606689453,
      "activations/layer2_attention_weight_max": 32.000247955322266,
      "activations/layer2_attention_weight_min": -31.942556381225586,
      "activations/layer3_attention_weight_max": 98.27841186523438,
      "activations/layer3_attention_weight_min": -97.51753234863281,
      "activations/layer4_attention_weight_max": 108.27477264404297,
      "activations/layer4_attention_weight_min": -111.5663070678711,
      "activations/layer5_attention_weight_max": 51.18360137939453,
      "activations/layer5_attention_weight_min": -64.34420013427734,
      "activations/layer6_attention_weight_max": 44.24745559692383,
      "activations/layer6_attention_weight_min": -47.31535339355469,
      "activations/layer7_attention_weight_max": 92.69190979003906,
      "activations/layer7_attention_weight_min": -97.42706298828125,
      "activations/layer8_attention_weight_max": 41.98225784301758,
      "activations/layer8_attention_weight_min": -45.28922653198242,
      "activations/layer9_attention_weight_max": 31.26348876953125,
      "activations/layer9_attention_weight_min": -34.85124206542969,
      "epoch": 15.11,
      "learning_rate": 5.307272727272727e-05,
      "loss": 2.7589,
      "step": 260100
    },
    {
      "activations/layer0_attention_weight_max": 16.788938522338867,
      "activations/layer0_attention_weight_min": -11.386094093322754,
      "activations/layer10_attention_weight_max": 32.81843948364258,
      "activations/layer10_attention_weight_min": -33.84160232543945,
      "activations/layer11_attention_weight_max": 31.558109283447266,
      "activations/layer11_attention_weight_min": -33.74502182006836,
      "activations/layer12_attention_weight_max": 27.64413833618164,
      "activations/layer12_attention_weight_min": -26.016462326049805,
      "activations/layer13_attention_weight_max": 41.030757904052734,
      "activations/layer13_attention_weight_min": -35.098262786865234,
      "activations/layer14_attention_weight_max": 45.42876052856445,
      "activations/layer14_attention_weight_min": -39.826412200927734,
      "activations/layer15_attention_weight_max": 39.75437545776367,
      "activations/layer15_attention_weight_min": -35.571346282958984,
      "activations/layer16_attention_weight_max": 31.951433181762695,
      "activations/layer16_attention_weight_min": -28.855268478393555,
      "activations/layer17_attention_weight_max": 32.178924560546875,
      "activations/layer17_attention_weight_min": -26.447589874267578,
      "activations/layer18_attention_weight_max": 33.742095947265625,
      "activations/layer18_attention_weight_min": -25.145112991333008,
      "activations/layer19_attention_weight_max": 36.407676696777344,
      "activations/layer19_attention_weight_min": -29.960651397705078,
      "activations/layer1_attention_weight_max": 15.65076732635498,
      "activations/layer1_attention_weight_min": -14.540128707885742,
      "activations/layer20_attention_weight_max": 32.03057861328125,
      "activations/layer20_attention_weight_min": -25.364856719970703,
      "activations/layer21_attention_weight_max": 32.170475006103516,
      "activations/layer21_attention_weight_min": -25.357938766479492,
      "activations/layer22_attention_weight_max": 48.19654846191406,
      "activations/layer22_attention_weight_min": -31.892847061157227,
      "activations/layer23_attention_weight_max": 34.995689392089844,
      "activations/layer23_attention_weight_min": -25.336509704589844,
      "activations/layer2_attention_weight_max": 32.24933624267578,
      "activations/layer2_attention_weight_min": -31.726768493652344,
      "activations/layer3_attention_weight_max": 94.50599670410156,
      "activations/layer3_attention_weight_min": -96.01878356933594,
      "activations/layer4_attention_weight_max": 107.02750396728516,
      "activations/layer4_attention_weight_min": -107.30696868896484,
      "activations/layer5_attention_weight_max": 53.316650390625,
      "activations/layer5_attention_weight_min": -61.356788635253906,
      "activations/layer6_attention_weight_max": 44.49317932128906,
      "activations/layer6_attention_weight_min": -46.624202728271484,
      "activations/layer7_attention_weight_max": 90.42322540283203,
      "activations/layer7_attention_weight_min": -98.6724853515625,
      "activations/layer8_attention_weight_max": 44.13847351074219,
      "activations/layer8_attention_weight_min": -45.916873931884766,
      "activations/layer9_attention_weight_max": 32.8824348449707,
      "activations/layer9_attention_weight_min": -35.30896759033203,
      "epoch": 15.12,
      "learning_rate": 5.305378787878787e-05,
      "loss": 2.7537,
      "step": 260150
    },
    {
      "activations/layer0_attention_weight_max": 16.161361694335938,
      "activations/layer0_attention_weight_min": -12.33072566986084,
      "activations/layer10_attention_weight_max": 32.89170455932617,
      "activations/layer10_attention_weight_min": -32.08729934692383,
      "activations/layer11_attention_weight_max": 33.72327423095703,
      "activations/layer11_attention_weight_min": -33.82183074951172,
      "activations/layer12_attention_weight_max": 24.33173179626465,
      "activations/layer12_attention_weight_min": -25.697275161743164,
      "activations/layer13_attention_weight_max": 37.0337028503418,
      "activations/layer13_attention_weight_min": -34.49009323120117,
      "activations/layer14_attention_weight_max": 40.152957916259766,
      "activations/layer14_attention_weight_min": -38.43094253540039,
      "activations/layer15_attention_weight_max": 34.30524826049805,
      "activations/layer15_attention_weight_min": -34.27717590332031,
      "activations/layer16_attention_weight_max": 31.36433982849121,
      "activations/layer16_attention_weight_min": -27.206117630004883,
      "activations/layer17_attention_weight_max": 27.970273971557617,
      "activations/layer17_attention_weight_min": -27.46097755432129,
      "activations/layer18_attention_weight_max": 28.681922912597656,
      "activations/layer18_attention_weight_min": -24.411603927612305,
      "activations/layer19_attention_weight_max": 30.878156661987305,
      "activations/layer19_attention_weight_min": -33.51546096801758,
      "activations/layer1_attention_weight_max": 16.768386840820312,
      "activations/layer1_attention_weight_min": -14.767979621887207,
      "activations/layer20_attention_weight_max": 25.790176391601562,
      "activations/layer20_attention_weight_min": -26.5076847076416,
      "activations/layer21_attention_weight_max": 26.96037483215332,
      "activations/layer21_attention_weight_min": -25.876323699951172,
      "activations/layer22_attention_weight_max": 40.55600357055664,
      "activations/layer22_attention_weight_min": -30.281158447265625,
      "activations/layer23_attention_weight_max": 29.861713409423828,
      "activations/layer23_attention_weight_min": -25.810911178588867,
      "activations/layer2_attention_weight_max": 33.01296615600586,
      "activations/layer2_attention_weight_min": -34.35655212402344,
      "activations/layer3_attention_weight_max": 95.88603210449219,
      "activations/layer3_attention_weight_min": -99.49716186523438,
      "activations/layer4_attention_weight_max": 107.28258514404297,
      "activations/layer4_attention_weight_min": -107.833740234375,
      "activations/layer5_attention_weight_max": 49.949398040771484,
      "activations/layer5_attention_weight_min": -61.72373962402344,
      "activations/layer6_attention_weight_max": 43.31000518798828,
      "activations/layer6_attention_weight_min": -46.2474250793457,
      "activations/layer7_attention_weight_max": 90.47216033935547,
      "activations/layer7_attention_weight_min": -92.62477111816406,
      "activations/layer8_attention_weight_max": 40.72261047363281,
      "activations/layer8_attention_weight_min": -45.10595703125,
      "activations/layer9_attention_weight_max": 31.90106773376465,
      "activations/layer9_attention_weight_min": -33.50873947143555,
      "epoch": 15.12,
      "learning_rate": 5.303484848484848e-05,
      "loss": 2.7681,
      "step": 260200
    },
    {
      "activations/layer0_attention_weight_max": 15.657240867614746,
      "activations/layer0_attention_weight_min": -12.789565086364746,
      "activations/layer10_attention_weight_max": 33.02123260498047,
      "activations/layer10_attention_weight_min": -34.07647705078125,
      "activations/layer11_attention_weight_max": 32.70930099487305,
      "activations/layer11_attention_weight_min": -32.18804168701172,
      "activations/layer12_attention_weight_max": 26.57718849182129,
      "activations/layer12_attention_weight_min": -27.363096237182617,
      "activations/layer13_attention_weight_max": 36.90156173706055,
      "activations/layer13_attention_weight_min": -34.3254280090332,
      "activations/layer14_attention_weight_max": 41.51993179321289,
      "activations/layer14_attention_weight_min": -37.50815200805664,
      "activations/layer15_attention_weight_max": 38.588584899902344,
      "activations/layer15_attention_weight_min": -33.63687515258789,
      "activations/layer16_attention_weight_max": 29.620317459106445,
      "activations/layer16_attention_weight_min": -28.73362922668457,
      "activations/layer17_attention_weight_max": 28.734418869018555,
      "activations/layer17_attention_weight_min": -24.828210830688477,
      "activations/layer18_attention_weight_max": 32.51567077636719,
      "activations/layer18_attention_weight_min": -23.80582618713379,
      "activations/layer19_attention_weight_max": 33.12386703491211,
      "activations/layer19_attention_weight_min": -30.39871597290039,
      "activations/layer1_attention_weight_max": 16.728506088256836,
      "activations/layer1_attention_weight_min": -14.541292190551758,
      "activations/layer20_attention_weight_max": 28.4014835357666,
      "activations/layer20_attention_weight_min": -25.623151779174805,
      "activations/layer21_attention_weight_max": 29.451438903808594,
      "activations/layer21_attention_weight_min": -24.097959518432617,
      "activations/layer22_attention_weight_max": 39.741153717041016,
      "activations/layer22_attention_weight_min": -29.67599868774414,
      "activations/layer23_attention_weight_max": 34.61617660522461,
      "activations/layer23_attention_weight_min": -23.48719596862793,
      "activations/layer2_attention_weight_max": 32.83015060424805,
      "activations/layer2_attention_weight_min": -31.825469970703125,
      "activations/layer3_attention_weight_max": 94.06185913085938,
      "activations/layer3_attention_weight_min": -93.46076202392578,
      "activations/layer4_attention_weight_max": 108.45317840576172,
      "activations/layer4_attention_weight_min": -109.29729461669922,
      "activations/layer5_attention_weight_max": 53.930816650390625,
      "activations/layer5_attention_weight_min": -60.80253601074219,
      "activations/layer6_attention_weight_max": 44.07795333862305,
      "activations/layer6_attention_weight_min": -46.958919525146484,
      "activations/layer7_attention_weight_max": 94.76786041259766,
      "activations/layer7_attention_weight_min": -88.17750549316406,
      "activations/layer8_attention_weight_max": 39.52334213256836,
      "activations/layer8_attention_weight_min": -41.941375732421875,
      "activations/layer9_attention_weight_max": 31.65283203125,
      "activations/layer9_attention_weight_min": -33.90652084350586,
      "epoch": 15.12,
      "learning_rate": 5.3015909090909085e-05,
      "loss": 2.7601,
      "step": 260250
    },
    {
      "activations/layer0_attention_weight_max": 16.15268898010254,
      "activations/layer0_attention_weight_min": -12.46875286102295,
      "activations/layer10_attention_weight_max": 38.230777740478516,
      "activations/layer10_attention_weight_min": -36.61383056640625,
      "activations/layer11_attention_weight_max": 37.426544189453125,
      "activations/layer11_attention_weight_min": -36.01164245605469,
      "activations/layer12_attention_weight_max": 27.756872177124023,
      "activations/layer12_attention_weight_min": -29.158185958862305,
      "activations/layer13_attention_weight_max": 40.097591400146484,
      "activations/layer13_attention_weight_min": -37.84440231323242,
      "activations/layer14_attention_weight_max": 44.95275115966797,
      "activations/layer14_attention_weight_min": -38.477413177490234,
      "activations/layer15_attention_weight_max": 41.92063522338867,
      "activations/layer15_attention_weight_min": -34.2679443359375,
      "activations/layer16_attention_weight_max": 29.385154724121094,
      "activations/layer16_attention_weight_min": -28.007328033447266,
      "activations/layer17_attention_weight_max": 33.2159423828125,
      "activations/layer17_attention_weight_min": -26.144685745239258,
      "activations/layer18_attention_weight_max": 31.874303817749023,
      "activations/layer18_attention_weight_min": -24.17239761352539,
      "activations/layer19_attention_weight_max": 35.38240051269531,
      "activations/layer19_attention_weight_min": -30.49994468688965,
      "activations/layer1_attention_weight_max": 15.93559455871582,
      "activations/layer1_attention_weight_min": -14.754744529724121,
      "activations/layer20_attention_weight_max": 30.813756942749023,
      "activations/layer20_attention_weight_min": -24.14192771911621,
      "activations/layer21_attention_weight_max": 32.019805908203125,
      "activations/layer21_attention_weight_min": -23.585948944091797,
      "activations/layer22_attention_weight_max": 46.85506057739258,
      "activations/layer22_attention_weight_min": -30.220552444458008,
      "activations/layer23_attention_weight_max": 34.0161247253418,
      "activations/layer23_attention_weight_min": -23.600379943847656,
      "activations/layer2_attention_weight_max": 33.5857048034668,
      "activations/layer2_attention_weight_min": -32.252281188964844,
      "activations/layer3_attention_weight_max": 98.11174011230469,
      "activations/layer3_attention_weight_min": -96.24319458007812,
      "activations/layer4_attention_weight_max": 110.1175765991211,
      "activations/layer4_attention_weight_min": -111.158447265625,
      "activations/layer5_attention_weight_max": 51.115867614746094,
      "activations/layer5_attention_weight_min": -62.78666305541992,
      "activations/layer6_attention_weight_max": 48.14989471435547,
      "activations/layer6_attention_weight_min": -51.491214752197266,
      "activations/layer7_attention_weight_max": 102.37423706054688,
      "activations/layer7_attention_weight_min": -108.13094329833984,
      "activations/layer8_attention_weight_max": 51.50880813598633,
      "activations/layer8_attention_weight_min": -49.59296798706055,
      "activations/layer9_attention_weight_max": 39.274635314941406,
      "activations/layer9_attention_weight_min": -38.45625305175781,
      "epoch": 15.12,
      "learning_rate": 5.299696969696969e-05,
      "loss": 2.7662,
      "step": 260300
    },
    {
      "activations/layer0_attention_weight_max": 16.27837371826172,
      "activations/layer0_attention_weight_min": -12.297149658203125,
      "activations/layer10_attention_weight_max": 34.37421417236328,
      "activations/layer10_attention_weight_min": -36.33039093017578,
      "activations/layer11_attention_weight_max": 34.47454071044922,
      "activations/layer11_attention_weight_min": -34.99955368041992,
      "activations/layer12_attention_weight_max": 28.11182403564453,
      "activations/layer12_attention_weight_min": -28.421899795532227,
      "activations/layer13_attention_weight_max": 45.41774368286133,
      "activations/layer13_attention_weight_min": -38.62334060668945,
      "activations/layer14_attention_weight_max": 51.699337005615234,
      "activations/layer14_attention_weight_min": -40.91787338256836,
      "activations/layer15_attention_weight_max": 42.72177505493164,
      "activations/layer15_attention_weight_min": -37.53829574584961,
      "activations/layer16_attention_weight_max": 31.686901092529297,
      "activations/layer16_attention_weight_min": -29.123369216918945,
      "activations/layer17_attention_weight_max": 33.60990905761719,
      "activations/layer17_attention_weight_min": -29.63175392150879,
      "activations/layer18_attention_weight_max": 34.60433578491211,
      "activations/layer18_attention_weight_min": -26.957965850830078,
      "activations/layer19_attention_weight_max": 33.0093879699707,
      "activations/layer19_attention_weight_min": -31.778757095336914,
      "activations/layer1_attention_weight_max": 15.721887588500977,
      "activations/layer1_attention_weight_min": -15.019713401794434,
      "activations/layer20_attention_weight_max": 29.572792053222656,
      "activations/layer20_attention_weight_min": -26.020469665527344,
      "activations/layer21_attention_weight_max": 28.772850036621094,
      "activations/layer21_attention_weight_min": -23.5859317779541,
      "activations/layer22_attention_weight_max": 46.22391891479492,
      "activations/layer22_attention_weight_min": -30.86421775817871,
      "activations/layer23_attention_weight_max": 36.55697250366211,
      "activations/layer23_attention_weight_min": -25.845497131347656,
      "activations/layer2_attention_weight_max": 34.626136779785156,
      "activations/layer2_attention_weight_min": -33.876548767089844,
      "activations/layer3_attention_weight_max": 102.0561752319336,
      "activations/layer3_attention_weight_min": -105.14153289794922,
      "activations/layer4_attention_weight_max": 112.80083465576172,
      "activations/layer4_attention_weight_min": -113.70829010009766,
      "activations/layer5_attention_weight_max": 52.37059020996094,
      "activations/layer5_attention_weight_min": -65.99562072753906,
      "activations/layer6_attention_weight_max": 45.92023468017578,
      "activations/layer6_attention_weight_min": -49.61152267456055,
      "activations/layer7_attention_weight_max": 101.20162963867188,
      "activations/layer7_attention_weight_min": -102.13098907470703,
      "activations/layer8_attention_weight_max": 42.7453727722168,
      "activations/layer8_attention_weight_min": -44.01566696166992,
      "activations/layer9_attention_weight_max": 33.95148849487305,
      "activations/layer9_attention_weight_min": -36.900474548339844,
      "epoch": 15.13,
      "learning_rate": 5.2978030303030294e-05,
      "loss": 2.7473,
      "step": 260350
    },
    {
      "activations/layer0_attention_weight_max": 16.346023559570312,
      "activations/layer0_attention_weight_min": -11.588592529296875,
      "activations/layer10_attention_weight_max": 32.9649772644043,
      "activations/layer10_attention_weight_min": -34.32019805908203,
      "activations/layer11_attention_weight_max": 32.056251525878906,
      "activations/layer11_attention_weight_min": -32.336124420166016,
      "activations/layer12_attention_weight_max": 26.13316535949707,
      "activations/layer12_attention_weight_min": -27.043615341186523,
      "activations/layer13_attention_weight_max": 39.12177276611328,
      "activations/layer13_attention_weight_min": -37.0091667175293,
      "activations/layer14_attention_weight_max": 42.27755355834961,
      "activations/layer14_attention_weight_min": -39.66304397583008,
      "activations/layer15_attention_weight_max": 35.86104202270508,
      "activations/layer15_attention_weight_min": -33.80329895019531,
      "activations/layer16_attention_weight_max": 28.68166732788086,
      "activations/layer16_attention_weight_min": -29.171428680419922,
      "activations/layer17_attention_weight_max": 31.052541732788086,
      "activations/layer17_attention_weight_min": -26.401897430419922,
      "activations/layer18_attention_weight_max": 33.49260711669922,
      "activations/layer18_attention_weight_min": -27.203285217285156,
      "activations/layer19_attention_weight_max": 31.163654327392578,
      "activations/layer19_attention_weight_min": -31.721405029296875,
      "activations/layer1_attention_weight_max": 15.64249324798584,
      "activations/layer1_attention_weight_min": -15.3549222946167,
      "activations/layer20_attention_weight_max": 27.778648376464844,
      "activations/layer20_attention_weight_min": -26.49053955078125,
      "activations/layer21_attention_weight_max": 30.294492721557617,
      "activations/layer21_attention_weight_min": -25.82871437072754,
      "activations/layer22_attention_weight_max": 38.07005310058594,
      "activations/layer22_attention_weight_min": -30.691741943359375,
      "activations/layer23_attention_weight_max": 30.26423454284668,
      "activations/layer23_attention_weight_min": -28.362651824951172,
      "activations/layer2_attention_weight_max": 30.973037719726562,
      "activations/layer2_attention_weight_min": -32.02668380737305,
      "activations/layer3_attention_weight_max": 97.10364532470703,
      "activations/layer3_attention_weight_min": -96.33052062988281,
      "activations/layer4_attention_weight_max": 108.33930969238281,
      "activations/layer4_attention_weight_min": -111.02129364013672,
      "activations/layer5_attention_weight_max": 53.94923400878906,
      "activations/layer5_attention_weight_min": -61.59056091308594,
      "activations/layer6_attention_weight_max": 44.60301971435547,
      "activations/layer6_attention_weight_min": -46.822933197021484,
      "activations/layer7_attention_weight_max": 91.00773620605469,
      "activations/layer7_attention_weight_min": -91.27989196777344,
      "activations/layer8_attention_weight_max": 39.30885696411133,
      "activations/layer8_attention_weight_min": -41.66087341308594,
      "activations/layer9_attention_weight_max": 31.815107345581055,
      "activations/layer9_attention_weight_min": -33.82273483276367,
      "epoch": 15.13,
      "learning_rate": 5.295909090909091e-05,
      "loss": 2.7457,
      "step": 260400
    },
    {
      "activations/layer0_attention_weight_max": 15.840601921081543,
      "activations/layer0_attention_weight_min": -11.768568992614746,
      "activations/layer10_attention_weight_max": 46.23182678222656,
      "activations/layer10_attention_weight_min": -43.34749984741211,
      "activations/layer11_attention_weight_max": 47.162147521972656,
      "activations/layer11_attention_weight_min": -47.46149444580078,
      "activations/layer12_attention_weight_max": 39.368927001953125,
      "activations/layer12_attention_weight_min": -30.33546257019043,
      "activations/layer13_attention_weight_max": 54.9869270324707,
      "activations/layer13_attention_weight_min": -44.38150405883789,
      "activations/layer14_attention_weight_max": 57.731407165527344,
      "activations/layer14_attention_weight_min": -50.09837341308594,
      "activations/layer15_attention_weight_max": 50.53683090209961,
      "activations/layer15_attention_weight_min": -47.449832916259766,
      "activations/layer16_attention_weight_max": 36.591548919677734,
      "activations/layer16_attention_weight_min": -28.371063232421875,
      "activations/layer17_attention_weight_max": 31.254329681396484,
      "activations/layer17_attention_weight_min": -29.46279525756836,
      "activations/layer18_attention_weight_max": 30.763408660888672,
      "activations/layer18_attention_weight_min": -28.45217514038086,
      "activations/layer19_attention_weight_max": 44.38528823852539,
      "activations/layer19_attention_weight_min": -31.18002700805664,
      "activations/layer1_attention_weight_max": 17.520793914794922,
      "activations/layer1_attention_weight_min": -14.853163719177246,
      "activations/layer20_attention_weight_max": 37.168922424316406,
      "activations/layer20_attention_weight_min": -25.19274139404297,
      "activations/layer21_attention_weight_max": 33.74169921875,
      "activations/layer21_attention_weight_min": -24.736175537109375,
      "activations/layer22_attention_weight_max": 51.43650817871094,
      "activations/layer22_attention_weight_min": -32.97484588623047,
      "activations/layer23_attention_weight_max": 34.57551574707031,
      "activations/layer23_attention_weight_min": -24.077207565307617,
      "activations/layer2_attention_weight_max": 34.07341766357422,
      "activations/layer2_attention_weight_min": -32.91431427001953,
      "activations/layer3_attention_weight_max": 101.74317932128906,
      "activations/layer3_attention_weight_min": -97.8360824584961,
      "activations/layer4_attention_weight_max": 114.25148010253906,
      "activations/layer4_attention_weight_min": -112.12491607666016,
      "activations/layer5_attention_weight_max": 55.7525634765625,
      "activations/layer5_attention_weight_min": -62.38735580444336,
      "activations/layer6_attention_weight_max": 47.61647415161133,
      "activations/layer6_attention_weight_min": -46.40208053588867,
      "activations/layer7_attention_weight_max": 110.40265655517578,
      "activations/layer7_attention_weight_min": -110.919677734375,
      "activations/layer8_attention_weight_max": 52.07038116455078,
      "activations/layer8_attention_weight_min": -50.573970794677734,
      "activations/layer9_attention_weight_max": 42.793792724609375,
      "activations/layer9_attention_weight_min": -41.620155334472656,
      "epoch": 15.13,
      "learning_rate": 5.294015151515151e-05,
      "loss": 2.7318,
      "step": 260450
    },
    {
      "activations/layer0_attention_weight_max": 16.24308204650879,
      "activations/layer0_attention_weight_min": -12.566347122192383,
      "activations/layer10_attention_weight_max": 31.917301177978516,
      "activations/layer10_attention_weight_min": -32.54464340209961,
      "activations/layer11_attention_weight_max": 33.50935363769531,
      "activations/layer11_attention_weight_min": -31.59779167175293,
      "activations/layer12_attention_weight_max": 26.3300724029541,
      "activations/layer12_attention_weight_min": -26.406105041503906,
      "activations/layer13_attention_weight_max": 40.2933464050293,
      "activations/layer13_attention_weight_min": -36.2626953125,
      "activations/layer14_attention_weight_max": 43.24380874633789,
      "activations/layer14_attention_weight_min": -37.78200149536133,
      "activations/layer15_attention_weight_max": 38.18046569824219,
      "activations/layer15_attention_weight_min": -35.797401428222656,
      "activations/layer16_attention_weight_max": 32.432064056396484,
      "activations/layer16_attention_weight_min": -27.70960235595703,
      "activations/layer17_attention_weight_max": 30.824886322021484,
      "activations/layer17_attention_weight_min": -26.674514770507812,
      "activations/layer18_attention_weight_max": 33.30613708496094,
      "activations/layer18_attention_weight_min": -23.99885368347168,
      "activations/layer19_attention_weight_max": 39.30050277709961,
      "activations/layer19_attention_weight_min": -31.50341796875,
      "activations/layer1_attention_weight_max": 15.670088768005371,
      "activations/layer1_attention_weight_min": -14.03159236907959,
      "activations/layer20_attention_weight_max": 31.167985916137695,
      "activations/layer20_attention_weight_min": -26.109020233154297,
      "activations/layer21_attention_weight_max": 33.26987075805664,
      "activations/layer21_attention_weight_min": -24.257041931152344,
      "activations/layer22_attention_weight_max": 44.09149932861328,
      "activations/layer22_attention_weight_min": -30.177534103393555,
      "activations/layer23_attention_weight_max": 38.20183181762695,
      "activations/layer23_attention_weight_min": -24.899028778076172,
      "activations/layer2_attention_weight_max": 30.45461082458496,
      "activations/layer2_attention_weight_min": -31.030868530273438,
      "activations/layer3_attention_weight_max": 94.51484680175781,
      "activations/layer3_attention_weight_min": -97.46797180175781,
      "activations/layer4_attention_weight_max": 105.5423812866211,
      "activations/layer4_attention_weight_min": -107.0997543334961,
      "activations/layer5_attention_weight_max": 51.675811767578125,
      "activations/layer5_attention_weight_min": -62.46040344238281,
      "activations/layer6_attention_weight_max": 42.350303649902344,
      "activations/layer6_attention_weight_min": -45.81645965576172,
      "activations/layer7_attention_weight_max": 92.36392211914062,
      "activations/layer7_attention_weight_min": -85.40276336669922,
      "activations/layer8_attention_weight_max": 37.687225341796875,
      "activations/layer8_attention_weight_min": -41.24977493286133,
      "activations/layer9_attention_weight_max": 32.60320281982422,
      "activations/layer9_attention_weight_min": -31.904346466064453,
      "epoch": 15.14,
      "learning_rate": 5.292121212121211e-05,
      "loss": 2.7693,
      "step": 260500
    },
    {
      "activations/layer0_attention_weight_max": 16.44502830505371,
      "activations/layer0_attention_weight_min": -12.251928329467773,
      "activations/layer10_attention_weight_max": 35.69700622558594,
      "activations/layer10_attention_weight_min": -33.879150390625,
      "activations/layer11_attention_weight_max": 34.448448181152344,
      "activations/layer11_attention_weight_min": -34.23342514038086,
      "activations/layer12_attention_weight_max": 29.24901008605957,
      "activations/layer12_attention_weight_min": -26.84588050842285,
      "activations/layer13_attention_weight_max": 42.70886993408203,
      "activations/layer13_attention_weight_min": -35.26340866088867,
      "activations/layer14_attention_weight_max": 47.775901794433594,
      "activations/layer14_attention_weight_min": -38.71330261230469,
      "activations/layer15_attention_weight_max": 39.033626556396484,
      "activations/layer15_attention_weight_min": -34.79859161376953,
      "activations/layer16_attention_weight_max": 30.36973762512207,
      "activations/layer16_attention_weight_min": -28.546499252319336,
      "activations/layer17_attention_weight_max": 29.14009666442871,
      "activations/layer17_attention_weight_min": -25.32879066467285,
      "activations/layer18_attention_weight_max": 30.155254364013672,
      "activations/layer18_attention_weight_min": -25.0042781829834,
      "activations/layer19_attention_weight_max": 35.16548538208008,
      "activations/layer19_attention_weight_min": -32.09806823730469,
      "activations/layer1_attention_weight_max": 15.835911750793457,
      "activations/layer1_attention_weight_min": -15.385608673095703,
      "activations/layer20_attention_weight_max": 28.999040603637695,
      "activations/layer20_attention_weight_min": -27.169300079345703,
      "activations/layer21_attention_weight_max": 27.62803840637207,
      "activations/layer21_attention_weight_min": -25.59677505493164,
      "activations/layer22_attention_weight_max": 40.985923767089844,
      "activations/layer22_attention_weight_min": -33.614524841308594,
      "activations/layer23_attention_weight_max": 30.96975326538086,
      "activations/layer23_attention_weight_min": -24.957937240600586,
      "activations/layer2_attention_weight_max": 31.335371017456055,
      "activations/layer2_attention_weight_min": -32.18151092529297,
      "activations/layer3_attention_weight_max": 95.47811889648438,
      "activations/layer3_attention_weight_min": -96.121337890625,
      "activations/layer4_attention_weight_max": 108.40239715576172,
      "activations/layer4_attention_weight_min": -113.83441162109375,
      "activations/layer5_attention_weight_max": 49.981285095214844,
      "activations/layer5_attention_weight_min": -57.823760986328125,
      "activations/layer6_attention_weight_max": 43.569522857666016,
      "activations/layer6_attention_weight_min": -45.1977653503418,
      "activations/layer7_attention_weight_max": 90.61067199707031,
      "activations/layer7_attention_weight_min": -97.41384887695312,
      "activations/layer8_attention_weight_max": 42.67304992675781,
      "activations/layer8_attention_weight_min": -46.78749465942383,
      "activations/layer9_attention_weight_max": 35.512882232666016,
      "activations/layer9_attention_weight_min": -36.39943313598633,
      "epoch": 15.14,
      "learning_rate": 5.290227272727273e-05,
      "loss": 2.7546,
      "step": 260550
    },
    {
      "activations/layer0_attention_weight_max": 16.600074768066406,
      "activations/layer0_attention_weight_min": -12.63232135772705,
      "activations/layer10_attention_weight_max": 36.9380989074707,
      "activations/layer10_attention_weight_min": -34.82417297363281,
      "activations/layer11_attention_weight_max": 38.13029479980469,
      "activations/layer11_attention_weight_min": -36.47154998779297,
      "activations/layer12_attention_weight_max": 34.56682586669922,
      "activations/layer12_attention_weight_min": -34.598167419433594,
      "activations/layer13_attention_weight_max": 48.17560577392578,
      "activations/layer13_attention_weight_min": -34.45591735839844,
      "activations/layer14_attention_weight_max": 47.053279876708984,
      "activations/layer14_attention_weight_min": -38.895809173583984,
      "activations/layer15_attention_weight_max": 40.78427505493164,
      "activations/layer15_attention_weight_min": -33.99840545654297,
      "activations/layer16_attention_weight_max": 32.6131591796875,
      "activations/layer16_attention_weight_min": -29.219982147216797,
      "activations/layer17_attention_weight_max": 32.615047454833984,
      "activations/layer17_attention_weight_min": -26.366628646850586,
      "activations/layer18_attention_weight_max": 31.034170150756836,
      "activations/layer18_attention_weight_min": -24.038822174072266,
      "activations/layer19_attention_weight_max": 35.27445602416992,
      "activations/layer19_attention_weight_min": -30.372516632080078,
      "activations/layer1_attention_weight_max": 16.852367401123047,
      "activations/layer1_attention_weight_min": -15.747611999511719,
      "activations/layer20_attention_weight_max": 26.16024398803711,
      "activations/layer20_attention_weight_min": -24.298145294189453,
      "activations/layer21_attention_weight_max": 27.791349411010742,
      "activations/layer21_attention_weight_min": -22.840187072753906,
      "activations/layer22_attention_weight_max": 40.41668701171875,
      "activations/layer22_attention_weight_min": -28.200092315673828,
      "activations/layer23_attention_weight_max": 27.784305572509766,
      "activations/layer23_attention_weight_min": -25.27446746826172,
      "activations/layer2_attention_weight_max": 31.434057235717773,
      "activations/layer2_attention_weight_min": -29.797718048095703,
      "activations/layer3_attention_weight_max": 94.79754638671875,
      "activations/layer3_attention_weight_min": -98.18034362792969,
      "activations/layer4_attention_weight_max": 103.15083312988281,
      "activations/layer4_attention_weight_min": -109.17833709716797,
      "activations/layer5_attention_weight_max": 53.32231140136719,
      "activations/layer5_attention_weight_min": -60.809993743896484,
      "activations/layer6_attention_weight_max": 44.00095748901367,
      "activations/layer6_attention_weight_min": -46.296531677246094,
      "activations/layer7_attention_weight_max": 93.62432861328125,
      "activations/layer7_attention_weight_min": -92.83621978759766,
      "activations/layer8_attention_weight_max": 40.842987060546875,
      "activations/layer8_attention_weight_min": -46.41252136230469,
      "activations/layer9_attention_weight_max": 35.1875114440918,
      "activations/layer9_attention_weight_min": -33.885162353515625,
      "epoch": 15.14,
      "learning_rate": 5.288333333333333e-05,
      "loss": 2.7441,
      "step": 260600
    },
    {
      "activations/layer0_attention_weight_max": 16.53537940979004,
      "activations/layer0_attention_weight_min": -13.849135398864746,
      "activations/layer10_attention_weight_max": 32.680816650390625,
      "activations/layer10_attention_weight_min": -34.59935760498047,
      "activations/layer11_attention_weight_max": 33.69303894042969,
      "activations/layer11_attention_weight_min": -33.7665901184082,
      "activations/layer12_attention_weight_max": 26.253145217895508,
      "activations/layer12_attention_weight_min": -26.787473678588867,
      "activations/layer13_attention_weight_max": 47.59235382080078,
      "activations/layer13_attention_weight_min": -37.55613708496094,
      "activations/layer14_attention_weight_max": 46.64194107055664,
      "activations/layer14_attention_weight_min": -40.579166412353516,
      "activations/layer15_attention_weight_max": 41.09980392456055,
      "activations/layer15_attention_weight_min": -35.24644088745117,
      "activations/layer16_attention_weight_max": 31.586170196533203,
      "activations/layer16_attention_weight_min": -28.195871353149414,
      "activations/layer17_attention_weight_max": 34.12153244018555,
      "activations/layer17_attention_weight_min": -25.597021102905273,
      "activations/layer18_attention_weight_max": 34.86904525756836,
      "activations/layer18_attention_weight_min": -25.681230545043945,
      "activations/layer19_attention_weight_max": 35.67441940307617,
      "activations/layer19_attention_weight_min": -31.58441162109375,
      "activations/layer1_attention_weight_max": 16.302106857299805,
      "activations/layer1_attention_weight_min": -15.280885696411133,
      "activations/layer20_attention_weight_max": 31.733240127563477,
      "activations/layer20_attention_weight_min": -25.230070114135742,
      "activations/layer21_attention_weight_max": 29.962907791137695,
      "activations/layer21_attention_weight_min": -25.294260025024414,
      "activations/layer22_attention_weight_max": 41.779727935791016,
      "activations/layer22_attention_weight_min": -32.5466194152832,
      "activations/layer23_attention_weight_max": 32.659812927246094,
      "activations/layer23_attention_weight_min": -24.425949096679688,
      "activations/layer2_attention_weight_max": 32.466060638427734,
      "activations/layer2_attention_weight_min": -32.699951171875,
      "activations/layer3_attention_weight_max": 95.39563751220703,
      "activations/layer3_attention_weight_min": -94.27228546142578,
      "activations/layer4_attention_weight_max": 105.86913299560547,
      "activations/layer4_attention_weight_min": -109.55040740966797,
      "activations/layer5_attention_weight_max": 49.672264099121094,
      "activations/layer5_attention_weight_min": -61.09971618652344,
      "activations/layer6_attention_weight_max": 42.95237731933594,
      "activations/layer6_attention_weight_min": -47.71521759033203,
      "activations/layer7_attention_weight_max": 90.86647033691406,
      "activations/layer7_attention_weight_min": -93.6670913696289,
      "activations/layer8_attention_weight_max": 39.62763977050781,
      "activations/layer8_attention_weight_min": -46.49783706665039,
      "activations/layer9_attention_weight_max": 30.890207290649414,
      "activations/layer9_attention_weight_min": -36.9745979309082,
      "epoch": 15.15,
      "learning_rate": 5.286439393939394e-05,
      "loss": 2.7539,
      "step": 260650
    },
    {
      "activations/layer0_attention_weight_max": 16.336502075195312,
      "activations/layer0_attention_weight_min": -12.124648094177246,
      "activations/layer10_attention_weight_max": 32.90559005737305,
      "activations/layer10_attention_weight_min": -34.94606018066406,
      "activations/layer11_attention_weight_max": 32.37495422363281,
      "activations/layer11_attention_weight_min": -32.96274185180664,
      "activations/layer12_attention_weight_max": 27.15366554260254,
      "activations/layer12_attention_weight_min": -25.812131881713867,
      "activations/layer13_attention_weight_max": 40.760459899902344,
      "activations/layer13_attention_weight_min": -35.424232482910156,
      "activations/layer14_attention_weight_max": 41.72933578491211,
      "activations/layer14_attention_weight_min": -38.29579544067383,
      "activations/layer15_attention_weight_max": 37.867027282714844,
      "activations/layer15_attention_weight_min": -33.595359802246094,
      "activations/layer16_attention_weight_max": 28.534204483032227,
      "activations/layer16_attention_weight_min": -27.757890701293945,
      "activations/layer17_attention_weight_max": 30.748756408691406,
      "activations/layer17_attention_weight_min": -25.62434196472168,
      "activations/layer18_attention_weight_max": 32.83482360839844,
      "activations/layer18_attention_weight_min": -25.73076057434082,
      "activations/layer19_attention_weight_max": 35.84830093383789,
      "activations/layer19_attention_weight_min": -32.20002365112305,
      "activations/layer1_attention_weight_max": 16.637609481811523,
      "activations/layer1_attention_weight_min": -13.82058048248291,
      "activations/layer20_attention_weight_max": 31.494245529174805,
      "activations/layer20_attention_weight_min": -26.03054428100586,
      "activations/layer21_attention_weight_max": 31.278202056884766,
      "activations/layer21_attention_weight_min": -25.232852935791016,
      "activations/layer22_attention_weight_max": 43.18623352050781,
      "activations/layer22_attention_weight_min": -30.128498077392578,
      "activations/layer23_attention_weight_max": 33.94657897949219,
      "activations/layer23_attention_weight_min": -26.237653732299805,
      "activations/layer2_attention_weight_max": 34.038028717041016,
      "activations/layer2_attention_weight_min": -32.31431579589844,
      "activations/layer3_attention_weight_max": 95.17147827148438,
      "activations/layer3_attention_weight_min": -98.67501068115234,
      "activations/layer4_attention_weight_max": 106.97967529296875,
      "activations/layer4_attention_weight_min": -106.54985809326172,
      "activations/layer5_attention_weight_max": 50.345149993896484,
      "activations/layer5_attention_weight_min": -60.51517868041992,
      "activations/layer6_attention_weight_max": 43.68705749511719,
      "activations/layer6_attention_weight_min": -46.085670471191406,
      "activations/layer7_attention_weight_max": 93.89363098144531,
      "activations/layer7_attention_weight_min": -94.7319107055664,
      "activations/layer8_attention_weight_max": 42.90461349487305,
      "activations/layer8_attention_weight_min": -43.82910919189453,
      "activations/layer9_attention_weight_max": 31.49976921081543,
      "activations/layer9_attention_weight_min": -35.05953598022461,
      "epoch": 15.15,
      "learning_rate": 5.284545454545454e-05,
      "loss": 2.7512,
      "step": 260700
    },
    {
      "activations/layer0_attention_weight_max": 16.07680320739746,
      "activations/layer0_attention_weight_min": -11.839874267578125,
      "activations/layer10_attention_weight_max": 33.96491241455078,
      "activations/layer10_attention_weight_min": -33.902740478515625,
      "activations/layer11_attention_weight_max": 33.225399017333984,
      "activations/layer11_attention_weight_min": -35.3177490234375,
      "activations/layer12_attention_weight_max": 24.93398666381836,
      "activations/layer12_attention_weight_min": -27.019025802612305,
      "activations/layer13_attention_weight_max": 36.98917770385742,
      "activations/layer13_attention_weight_min": -36.221885681152344,
      "activations/layer14_attention_weight_max": 39.16183090209961,
      "activations/layer14_attention_weight_min": -38.59160614013672,
      "activations/layer15_attention_weight_max": 34.9659538269043,
      "activations/layer15_attention_weight_min": -33.532405853271484,
      "activations/layer16_attention_weight_max": 29.988256454467773,
      "activations/layer16_attention_weight_min": -29.25688362121582,
      "activations/layer17_attention_weight_max": 32.61614990234375,
      "activations/layer17_attention_weight_min": -28.98334503173828,
      "activations/layer18_attention_weight_max": 34.82720184326172,
      "activations/layer18_attention_weight_min": -25.155765533447266,
      "activations/layer19_attention_weight_max": 35.173709869384766,
      "activations/layer19_attention_weight_min": -33.38465881347656,
      "activations/layer1_attention_weight_max": 15.981826782226562,
      "activations/layer1_attention_weight_min": -14.721213340759277,
      "activations/layer20_attention_weight_max": 29.810686111450195,
      "activations/layer20_attention_weight_min": -28.789960861206055,
      "activations/layer21_attention_weight_max": 31.217824935913086,
      "activations/layer21_attention_weight_min": -26.35711669921875,
      "activations/layer22_attention_weight_max": 47.29425811767578,
      "activations/layer22_attention_weight_min": -34.54062271118164,
      "activations/layer23_attention_weight_max": 35.838924407958984,
      "activations/layer23_attention_weight_min": -26.19253921508789,
      "activations/layer2_attention_weight_max": 31.707233428955078,
      "activations/layer2_attention_weight_min": -32.51069641113281,
      "activations/layer3_attention_weight_max": 96.40308380126953,
      "activations/layer3_attention_weight_min": -92.91329956054688,
      "activations/layer4_attention_weight_max": 106.02363586425781,
      "activations/layer4_attention_weight_min": -105.28814697265625,
      "activations/layer5_attention_weight_max": 48.101593017578125,
      "activations/layer5_attention_weight_min": -64.401611328125,
      "activations/layer6_attention_weight_max": 42.5825309753418,
      "activations/layer6_attention_weight_min": -48.24382781982422,
      "activations/layer7_attention_weight_max": 91.57704162597656,
      "activations/layer7_attention_weight_min": -90.12027740478516,
      "activations/layer8_attention_weight_max": 38.184452056884766,
      "activations/layer8_attention_weight_min": -40.807674407958984,
      "activations/layer9_attention_weight_max": 30.533994674682617,
      "activations/layer9_attention_weight_min": -32.509883880615234,
      "epoch": 15.15,
      "learning_rate": 5.282651515151515e-05,
      "loss": 2.7404,
      "step": 260750
    },
    {
      "activations/layer0_attention_weight_max": 15.509486198425293,
      "activations/layer0_attention_weight_min": -13.09543228149414,
      "activations/layer10_attention_weight_max": 32.247840881347656,
      "activations/layer10_attention_weight_min": -34.7276611328125,
      "activations/layer11_attention_weight_max": 33.17462158203125,
      "activations/layer11_attention_weight_min": -33.170555114746094,
      "activations/layer12_attention_weight_max": 33.22427749633789,
      "activations/layer12_attention_weight_min": -27.635812759399414,
      "activations/layer13_attention_weight_max": 41.078460693359375,
      "activations/layer13_attention_weight_min": -38.32038879394531,
      "activations/layer14_attention_weight_max": 48.27791213989258,
      "activations/layer14_attention_weight_min": -41.3997688293457,
      "activations/layer15_attention_weight_max": 41.84442901611328,
      "activations/layer15_attention_weight_min": -36.660518646240234,
      "activations/layer16_attention_weight_max": 31.676025390625,
      "activations/layer16_attention_weight_min": -29.545612335205078,
      "activations/layer17_attention_weight_max": 29.775217056274414,
      "activations/layer17_attention_weight_min": -26.572526931762695,
      "activations/layer18_attention_weight_max": 29.17547607421875,
      "activations/layer18_attention_weight_min": -24.2376766204834,
      "activations/layer19_attention_weight_max": 37.006263732910156,
      "activations/layer19_attention_weight_min": -31.211660385131836,
      "activations/layer1_attention_weight_max": 16.87899398803711,
      "activations/layer1_attention_weight_min": -14.366888046264648,
      "activations/layer20_attention_weight_max": 29.290342330932617,
      "activations/layer20_attention_weight_min": -25.58698081970215,
      "activations/layer21_attention_weight_max": 28.88646125793457,
      "activations/layer21_attention_weight_min": -24.773027420043945,
      "activations/layer22_attention_weight_max": 47.9929084777832,
      "activations/layer22_attention_weight_min": -30.34185028076172,
      "activations/layer23_attention_weight_max": 34.51323699951172,
      "activations/layer23_attention_weight_min": -26.123579025268555,
      "activations/layer2_attention_weight_max": 31.627920150756836,
      "activations/layer2_attention_weight_min": -32.86583709716797,
      "activations/layer3_attention_weight_max": 95.35419464111328,
      "activations/layer3_attention_weight_min": -97.71733093261719,
      "activations/layer4_attention_weight_max": 108.27310943603516,
      "activations/layer4_attention_weight_min": -107.77677154541016,
      "activations/layer5_attention_weight_max": 57.05270004272461,
      "activations/layer5_attention_weight_min": -61.68120574951172,
      "activations/layer6_attention_weight_max": 43.40740203857422,
      "activations/layer6_attention_weight_min": -46.42392349243164,
      "activations/layer7_attention_weight_max": 92.33610534667969,
      "activations/layer7_attention_weight_min": -90.77576446533203,
      "activations/layer8_attention_weight_max": 42.58696365356445,
      "activations/layer8_attention_weight_min": -41.4492073059082,
      "activations/layer9_attention_weight_max": 30.946500778198242,
      "activations/layer9_attention_weight_min": -34.637943267822266,
      "epoch": 15.15,
      "learning_rate": 5.2807575757575755e-05,
      "loss": 2.7556,
      "step": 260800
    },
    {
      "activations/layer0_attention_weight_max": 15.211043357849121,
      "activations/layer0_attention_weight_min": -11.825702667236328,
      "activations/layer10_attention_weight_max": 34.422607421875,
      "activations/layer10_attention_weight_min": -34.443214416503906,
      "activations/layer11_attention_weight_max": 31.178373336791992,
      "activations/layer11_attention_weight_min": -32.07148742675781,
      "activations/layer12_attention_weight_max": 26.333480834960938,
      "activations/layer12_attention_weight_min": -26.395334243774414,
      "activations/layer13_attention_weight_max": 37.15475082397461,
      "activations/layer13_attention_weight_min": -35.908573150634766,
      "activations/layer14_attention_weight_max": 41.69684600830078,
      "activations/layer14_attention_weight_min": -37.15555191040039,
      "activations/layer15_attention_weight_max": 36.535316467285156,
      "activations/layer15_attention_weight_min": -36.59046936035156,
      "activations/layer16_attention_weight_max": 30.070451736450195,
      "activations/layer16_attention_weight_min": -27.20926856994629,
      "activations/layer17_attention_weight_max": 31.701799392700195,
      "activations/layer17_attention_weight_min": -25.876413345336914,
      "activations/layer18_attention_weight_max": 32.284385681152344,
      "activations/layer18_attention_weight_min": -24.74355697631836,
      "activations/layer19_attention_weight_max": 34.14033508300781,
      "activations/layer19_attention_weight_min": -33.900638580322266,
      "activations/layer1_attention_weight_max": 16.376670837402344,
      "activations/layer1_attention_weight_min": -14.76333999633789,
      "activations/layer20_attention_weight_max": 27.508026123046875,
      "activations/layer20_attention_weight_min": -27.5570068359375,
      "activations/layer21_attention_weight_max": 28.145681381225586,
      "activations/layer21_attention_weight_min": -26.20854949951172,
      "activations/layer22_attention_weight_max": 38.81834030151367,
      "activations/layer22_attention_weight_min": -32.42605972290039,
      "activations/layer23_attention_weight_max": 29.716075897216797,
      "activations/layer23_attention_weight_min": -25.61939239501953,
      "activations/layer2_attention_weight_max": 33.643341064453125,
      "activations/layer2_attention_weight_min": -32.85004806518555,
      "activations/layer3_attention_weight_max": 96.38584899902344,
      "activations/layer3_attention_weight_min": -96.8340835571289,
      "activations/layer4_attention_weight_max": 106.33206939697266,
      "activations/layer4_attention_weight_min": -106.29652404785156,
      "activations/layer5_attention_weight_max": 48.7408561706543,
      "activations/layer5_attention_weight_min": -59.14403533935547,
      "activations/layer6_attention_weight_max": 43.97939682006836,
      "activations/layer6_attention_weight_min": -46.24331283569336,
      "activations/layer7_attention_weight_max": 87.22610473632812,
      "activations/layer7_attention_weight_min": -98.20946502685547,
      "activations/layer8_attention_weight_max": 42.21348190307617,
      "activations/layer8_attention_weight_min": -43.13689041137695,
      "activations/layer9_attention_weight_max": 34.25891876220703,
      "activations/layer9_attention_weight_min": -34.84093475341797,
      "epoch": 15.16,
      "learning_rate": 5.278863636363636e-05,
      "loss": 2.7402,
      "step": 260850
    },
    {
      "activations/layer0_attention_weight_max": 16.365150451660156,
      "activations/layer0_attention_weight_min": -12.173165321350098,
      "activations/layer10_attention_weight_max": 31.5319881439209,
      "activations/layer10_attention_weight_min": -32.74379348754883,
      "activations/layer11_attention_weight_max": 32.80880355834961,
      "activations/layer11_attention_weight_min": -32.568782806396484,
      "activations/layer12_attention_weight_max": 28.848051071166992,
      "activations/layer12_attention_weight_min": -27.84242057800293,
      "activations/layer13_attention_weight_max": 44.45722579956055,
      "activations/layer13_attention_weight_min": -37.42378616333008,
      "activations/layer14_attention_weight_max": 45.07177734375,
      "activations/layer14_attention_weight_min": -39.76034164428711,
      "activations/layer15_attention_weight_max": 42.16400909423828,
      "activations/layer15_attention_weight_min": -34.525978088378906,
      "activations/layer16_attention_weight_max": 32.351566314697266,
      "activations/layer16_attention_weight_min": -27.326196670532227,
      "activations/layer17_attention_weight_max": 31.941139221191406,
      "activations/layer17_attention_weight_min": -26.83269691467285,
      "activations/layer18_attention_weight_max": 33.77306365966797,
      "activations/layer18_attention_weight_min": -23.305471420288086,
      "activations/layer19_attention_weight_max": 41.23324203491211,
      "activations/layer19_attention_weight_min": -30.730356216430664,
      "activations/layer1_attention_weight_max": 16.648244857788086,
      "activations/layer1_attention_weight_min": -14.948034286499023,
      "activations/layer20_attention_weight_max": 33.44835662841797,
      "activations/layer20_attention_weight_min": -23.504430770874023,
      "activations/layer21_attention_weight_max": 35.00225067138672,
      "activations/layer21_attention_weight_min": -23.96097183227539,
      "activations/layer22_attention_weight_max": 44.63529586791992,
      "activations/layer22_attention_weight_min": -29.258543014526367,
      "activations/layer23_attention_weight_max": 41.49614715576172,
      "activations/layer23_attention_weight_min": -26.424396514892578,
      "activations/layer2_attention_weight_max": 30.46367645263672,
      "activations/layer2_attention_weight_min": -31.230871200561523,
      "activations/layer3_attention_weight_max": 93.4682388305664,
      "activations/layer3_attention_weight_min": -99.22392272949219,
      "activations/layer4_attention_weight_max": 102.57630920410156,
      "activations/layer4_attention_weight_min": -111.4133529663086,
      "activations/layer5_attention_weight_max": 48.370849609375,
      "activations/layer5_attention_weight_min": -63.003990173339844,
      "activations/layer6_attention_weight_max": 42.34913635253906,
      "activations/layer6_attention_weight_min": -46.77849197387695,
      "activations/layer7_attention_weight_max": 91.58612060546875,
      "activations/layer7_attention_weight_min": -96.27032470703125,
      "activations/layer8_attention_weight_max": 39.52262496948242,
      "activations/layer8_attention_weight_min": -42.96549606323242,
      "activations/layer9_attention_weight_max": 31.570871353149414,
      "activations/layer9_attention_weight_min": -34.31940460205078,
      "epoch": 15.16,
      "learning_rate": 5.277007575757575e-05,
      "loss": 2.7328,
      "step": 260900
    },
    {
      "activations/layer0_attention_weight_max": 16.322078704833984,
      "activations/layer0_attention_weight_min": -12.515802383422852,
      "activations/layer10_attention_weight_max": 31.41880989074707,
      "activations/layer10_attention_weight_min": -32.32011795043945,
      "activations/layer11_attention_weight_max": 32.7130241394043,
      "activations/layer11_attention_weight_min": -32.628753662109375,
      "activations/layer12_attention_weight_max": 26.530942916870117,
      "activations/layer12_attention_weight_min": -25.966428756713867,
      "activations/layer13_attention_weight_max": 40.86737060546875,
      "activations/layer13_attention_weight_min": -34.92012405395508,
      "activations/layer14_attention_weight_max": 43.95182418823242,
      "activations/layer14_attention_weight_min": -38.903934478759766,
      "activations/layer15_attention_weight_max": 37.42848587036133,
      "activations/layer15_attention_weight_min": -33.47019577026367,
      "activations/layer16_attention_weight_max": 31.76341438293457,
      "activations/layer16_attention_weight_min": -28.83350944519043,
      "activations/layer17_attention_weight_max": 29.4160099029541,
      "activations/layer17_attention_weight_min": -25.101139068603516,
      "activations/layer18_attention_weight_max": 31.129859924316406,
      "activations/layer18_attention_weight_min": -24.431303024291992,
      "activations/layer19_attention_weight_max": 38.875247955322266,
      "activations/layer19_attention_weight_min": -33.912353515625,
      "activations/layer1_attention_weight_max": 16.33527946472168,
      "activations/layer1_attention_weight_min": -14.67850112915039,
      "activations/layer20_attention_weight_max": 30.061344146728516,
      "activations/layer20_attention_weight_min": -26.755931854248047,
      "activations/layer21_attention_weight_max": 30.646642684936523,
      "activations/layer21_attention_weight_min": -25.8044376373291,
      "activations/layer22_attention_weight_max": 41.4642219543457,
      "activations/layer22_attention_weight_min": -32.19051742553711,
      "activations/layer23_attention_weight_max": 34.701332092285156,
      "activations/layer23_attention_weight_min": -24.961830139160156,
      "activations/layer2_attention_weight_max": 32.509029388427734,
      "activations/layer2_attention_weight_min": -33.209102630615234,
      "activations/layer3_attention_weight_max": 100.69261169433594,
      "activations/layer3_attention_weight_min": -98.48210906982422,
      "activations/layer4_attention_weight_max": 113.98355865478516,
      "activations/layer4_attention_weight_min": -108.1893310546875,
      "activations/layer5_attention_weight_max": 53.29014205932617,
      "activations/layer5_attention_weight_min": -60.441864013671875,
      "activations/layer6_attention_weight_max": 44.30360412597656,
      "activations/layer6_attention_weight_min": -46.0065803527832,
      "activations/layer7_attention_weight_max": 93.32582092285156,
      "activations/layer7_attention_weight_min": -89.40630340576172,
      "activations/layer8_attention_weight_max": 39.067840576171875,
      "activations/layer8_attention_weight_min": -43.68791580200195,
      "activations/layer9_attention_weight_max": 31.722614288330078,
      "activations/layer9_attention_weight_min": -32.33389663696289,
      "epoch": 15.16,
      "learning_rate": 5.275113636363636e-05,
      "loss": 2.7375,
      "step": 260950
    },
    {
      "activations/layer0_attention_weight_max": 16.441099166870117,
      "activations/layer0_attention_weight_min": -12.401660919189453,
      "activations/layer10_attention_weight_max": 30.117143630981445,
      "activations/layer10_attention_weight_min": -31.260587692260742,
      "activations/layer11_attention_weight_max": 30.525192260742188,
      "activations/layer11_attention_weight_min": -30.31794548034668,
      "activations/layer12_attention_weight_max": 24.26722526550293,
      "activations/layer12_attention_weight_min": -27.202537536621094,
      "activations/layer13_attention_weight_max": 37.564979553222656,
      "activations/layer13_attention_weight_min": -33.793426513671875,
      "activations/layer14_attention_weight_max": 39.935604095458984,
      "activations/layer14_attention_weight_min": -34.87788772583008,
      "activations/layer15_attention_weight_max": 34.16567611694336,
      "activations/layer15_attention_weight_min": -31.945615768432617,
      "activations/layer16_attention_weight_max": 31.473234176635742,
      "activations/layer16_attention_weight_min": -26.574806213378906,
      "activations/layer17_attention_weight_max": 28.98182487487793,
      "activations/layer17_attention_weight_min": -24.704851150512695,
      "activations/layer18_attention_weight_max": 32.4023551940918,
      "activations/layer18_attention_weight_min": -25.419532775878906,
      "activations/layer19_attention_weight_max": 32.56927490234375,
      "activations/layer19_attention_weight_min": -30.87200355529785,
      "activations/layer1_attention_weight_max": 16.444320678710938,
      "activations/layer1_attention_weight_min": -14.049422264099121,
      "activations/layer20_attention_weight_max": 27.166364669799805,
      "activations/layer20_attention_weight_min": -25.425418853759766,
      "activations/layer21_attention_weight_max": 28.85537338256836,
      "activations/layer21_attention_weight_min": -23.973413467407227,
      "activations/layer22_attention_weight_max": 40.24492263793945,
      "activations/layer22_attention_weight_min": -30.045692443847656,
      "activations/layer23_attention_weight_max": 32.255401611328125,
      "activations/layer23_attention_weight_min": -25.442312240600586,
      "activations/layer2_attention_weight_max": 32.9106559753418,
      "activations/layer2_attention_weight_min": -33.51993942260742,
      "activations/layer3_attention_weight_max": 97.05612182617188,
      "activations/layer3_attention_weight_min": -101.54287719726562,
      "activations/layer4_attention_weight_max": 110.8600082397461,
      "activations/layer4_attention_weight_min": -111.10914611816406,
      "activations/layer5_attention_weight_max": 49.1041259765625,
      "activations/layer5_attention_weight_min": -62.24144744873047,
      "activations/layer6_attention_weight_max": 42.17582702636719,
      "activations/layer6_attention_weight_min": -43.88345718383789,
      "activations/layer7_attention_weight_max": 87.06563568115234,
      "activations/layer7_attention_weight_min": -86.39539337158203,
      "activations/layer8_attention_weight_max": 38.62004470825195,
      "activations/layer8_attention_weight_min": -40.235382080078125,
      "activations/layer9_attention_weight_max": 30.986684799194336,
      "activations/layer9_attention_weight_min": -32.49695587158203,
      "epoch": 15.17,
      "learning_rate": 5.2732196969696964e-05,
      "loss": 2.7444,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.5252,
      "eval_samples_per_second": 503.681,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.5252,
      "eval_openwebtext_samples_per_second": 503.681,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9886,
      "eval_wikitext_samples_per_second": 229.309,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_lambada_loss": 2.3671875,
      "eval_lambada_ppl": 10.667348137697372,
      "eval_lambada_runtime": 9.592,
      "eval_lambada_samples_per_second": 507.613,
      "step": 261000
    },
    {
      "activations/layer0_attention_weight_max": 15.524824142456055,
      "activations/layer0_attention_weight_min": -12.012882232666016,
      "activations/layer10_attention_weight_max": 33.05353546142578,
      "activations/layer10_attention_weight_min": -37.299476623535156,
      "activations/layer11_attention_weight_max": 34.44685363769531,
      "activations/layer11_attention_weight_min": -38.86817169189453,
      "activations/layer12_attention_weight_max": 26.636844635009766,
      "activations/layer12_attention_weight_min": -28.198530197143555,
      "activations/layer13_attention_weight_max": 38.269622802734375,
      "activations/layer13_attention_weight_min": -36.66435623168945,
      "activations/layer14_attention_weight_max": 41.88118362426758,
      "activations/layer14_attention_weight_min": -43.18804931640625,
      "activations/layer15_attention_weight_max": 37.91561508178711,
      "activations/layer15_attention_weight_min": -38.25528335571289,
      "activations/layer16_attention_weight_max": 30.134790420532227,
      "activations/layer16_attention_weight_min": -28.422208786010742,
      "activations/layer17_attention_weight_max": 34.00617980957031,
      "activations/layer17_attention_weight_min": -26.2916202545166,
      "activations/layer18_attention_weight_max": 33.99559783935547,
      "activations/layer18_attention_weight_min": -25.13097381591797,
      "activations/layer19_attention_weight_max": 36.64182662963867,
      "activations/layer19_attention_weight_min": -31.11783790588379,
      "activations/layer1_attention_weight_max": 16.010770797729492,
      "activations/layer1_attention_weight_min": -15.138242721557617,
      "activations/layer20_attention_weight_max": 28.751230239868164,
      "activations/layer20_attention_weight_min": -25.688587188720703,
      "activations/layer21_attention_weight_max": 28.656972885131836,
      "activations/layer21_attention_weight_min": -25.217208862304688,
      "activations/layer22_attention_weight_max": 44.45409393310547,
      "activations/layer22_attention_weight_min": -30.93082618713379,
      "activations/layer23_attention_weight_max": 36.13675308227539,
      "activations/layer23_attention_weight_min": -25.32756805419922,
      "activations/layer2_attention_weight_max": 35.713897705078125,
      "activations/layer2_attention_weight_min": -33.82917022705078,
      "activations/layer3_attention_weight_max": 102.89441680908203,
      "activations/layer3_attention_weight_min": -102.53041076660156,
      "activations/layer4_attention_weight_max": 115.19866180419922,
      "activations/layer4_attention_weight_min": -117.997314453125,
      "activations/layer5_attention_weight_max": 54.67915344238281,
      "activations/layer5_attention_weight_min": -64.23103332519531,
      "activations/layer6_attention_weight_max": 45.84653854370117,
      "activations/layer6_attention_weight_min": -46.97752380371094,
      "activations/layer7_attention_weight_max": 94.06350708007812,
      "activations/layer7_attention_weight_min": -96.39190673828125,
      "activations/layer8_attention_weight_max": 42.69915771484375,
      "activations/layer8_attention_weight_min": -43.11832046508789,
      "activations/layer9_attention_weight_max": 32.39970016479492,
      "activations/layer9_attention_weight_min": -34.73734664916992,
      "epoch": 15.17,
      "learning_rate": 5.271325757575757e-05,
      "loss": 2.7688,
      "step": 261050
    },
    {
      "activations/layer0_attention_weight_max": 16.295055389404297,
      "activations/layer0_attention_weight_min": -12.595420837402344,
      "activations/layer10_attention_weight_max": 37.0784797668457,
      "activations/layer10_attention_weight_min": -33.821258544921875,
      "activations/layer11_attention_weight_max": 37.641639709472656,
      "activations/layer11_attention_weight_min": -33.807228088378906,
      "activations/layer12_attention_weight_max": 29.497913360595703,
      "activations/layer12_attention_weight_min": -24.323163986206055,
      "activations/layer13_attention_weight_max": 42.62180709838867,
      "activations/layer13_attention_weight_min": -33.03485870361328,
      "activations/layer14_attention_weight_max": 46.804908752441406,
      "activations/layer14_attention_weight_min": -37.2621955871582,
      "activations/layer15_attention_weight_max": 44.87256622314453,
      "activations/layer15_attention_weight_min": -36.3752326965332,
      "activations/layer16_attention_weight_max": 35.45755386352539,
      "activations/layer16_attention_weight_min": -28.94346046447754,
      "activations/layer17_attention_weight_max": 30.973798751831055,
      "activations/layer17_attention_weight_min": -25.51662826538086,
      "activations/layer18_attention_weight_max": 33.91349792480469,
      "activations/layer18_attention_weight_min": -24.29709243774414,
      "activations/layer19_attention_weight_max": 37.48461151123047,
      "activations/layer19_attention_weight_min": -33.64265441894531,
      "activations/layer1_attention_weight_max": 18.4647216796875,
      "activations/layer1_attention_weight_min": -15.912174224853516,
      "activations/layer20_attention_weight_max": 33.750545501708984,
      "activations/layer20_attention_weight_min": -24.172874450683594,
      "activations/layer21_attention_weight_max": 33.21088790893555,
      "activations/layer21_attention_weight_min": -25.526031494140625,
      "activations/layer22_attention_weight_max": 50.8267936706543,
      "activations/layer22_attention_weight_min": -31.35523223876953,
      "activations/layer23_attention_weight_max": 36.74707794189453,
      "activations/layer23_attention_weight_min": -26.283954620361328,
      "activations/layer2_attention_weight_max": 31.852558135986328,
      "activations/layer2_attention_weight_min": -33.66217803955078,
      "activations/layer3_attention_weight_max": 95.73993682861328,
      "activations/layer3_attention_weight_min": -97.23197937011719,
      "activations/layer4_attention_weight_max": 107.38433837890625,
      "activations/layer4_attention_weight_min": -111.2060546875,
      "activations/layer5_attention_weight_max": 53.51231002807617,
      "activations/layer5_attention_weight_min": -61.62513732910156,
      "activations/layer6_attention_weight_max": 44.04871368408203,
      "activations/layer6_attention_weight_min": -47.99717330932617,
      "activations/layer7_attention_weight_max": 100.51664733886719,
      "activations/layer7_attention_weight_min": -95.31474304199219,
      "activations/layer8_attention_weight_max": 43.08974838256836,
      "activations/layer8_attention_weight_min": -45.2579231262207,
      "activations/layer9_attention_weight_max": 33.41721725463867,
      "activations/layer9_attention_weight_min": -33.79499435424805,
      "epoch": 15.17,
      "learning_rate": 5.2694318181818174e-05,
      "loss": 2.7561,
      "step": 261100
    },
    {
      "activations/layer0_attention_weight_max": 15.273134231567383,
      "activations/layer0_attention_weight_min": -13.087478637695312,
      "activations/layer10_attention_weight_max": 34.57669448852539,
      "activations/layer10_attention_weight_min": -33.96826171875,
      "activations/layer11_attention_weight_max": 33.32084655761719,
      "activations/layer11_attention_weight_min": -32.41317367553711,
      "activations/layer12_attention_weight_max": 31.28530502319336,
      "activations/layer12_attention_weight_min": -26.246492385864258,
      "activations/layer13_attention_weight_max": 38.26615524291992,
      "activations/layer13_attention_weight_min": -35.540767669677734,
      "activations/layer14_attention_weight_max": 40.04307174682617,
      "activations/layer14_attention_weight_min": -37.821617126464844,
      "activations/layer15_attention_weight_max": 37.0962028503418,
      "activations/layer15_attention_weight_min": -37.23776626586914,
      "activations/layer16_attention_weight_max": 29.202627182006836,
      "activations/layer16_attention_weight_min": -29.649124145507812,
      "activations/layer17_attention_weight_max": 30.325307846069336,
      "activations/layer17_attention_weight_min": -27.590917587280273,
      "activations/layer18_attention_weight_max": 32.33830261230469,
      "activations/layer18_attention_weight_min": -25.077041625976562,
      "activations/layer19_attention_weight_max": 32.60677719116211,
      "activations/layer19_attention_weight_min": -31.39796257019043,
      "activations/layer1_attention_weight_max": 16.43975067138672,
      "activations/layer1_attention_weight_min": -15.09895133972168,
      "activations/layer20_attention_weight_max": 27.190706253051758,
      "activations/layer20_attention_weight_min": -25.61113929748535,
      "activations/layer21_attention_weight_max": 27.266843795776367,
      "activations/layer21_attention_weight_min": -24.23676109313965,
      "activations/layer22_attention_weight_max": 39.10722351074219,
      "activations/layer22_attention_weight_min": -31.815441131591797,
      "activations/layer23_attention_weight_max": 32.13776779174805,
      "activations/layer23_attention_weight_min": -28.007970809936523,
      "activations/layer2_attention_weight_max": 33.92230224609375,
      "activations/layer2_attention_weight_min": -33.149940490722656,
      "activations/layer3_attention_weight_max": 99.45096588134766,
      "activations/layer3_attention_weight_min": -103.52526092529297,
      "activations/layer4_attention_weight_max": 105.15313720703125,
      "activations/layer4_attention_weight_min": -107.57933807373047,
      "activations/layer5_attention_weight_max": 51.55768966674805,
      "activations/layer5_attention_weight_min": -64.21863555908203,
      "activations/layer6_attention_weight_max": 43.175045013427734,
      "activations/layer6_attention_weight_min": -45.60612487792969,
      "activations/layer7_attention_weight_max": 87.43434143066406,
      "activations/layer7_attention_weight_min": -88.60588073730469,
      "activations/layer8_attention_weight_max": 40.542537689208984,
      "activations/layer8_attention_weight_min": -40.9074592590332,
      "activations/layer9_attention_weight_max": 31.097707748413086,
      "activations/layer9_attention_weight_min": -31.571115493774414,
      "epoch": 15.17,
      "learning_rate": 5.267537878787879e-05,
      "loss": 2.7417,
      "step": 261150
    },
    {
      "activations/layer0_attention_weight_max": 15.755840301513672,
      "activations/layer0_attention_weight_min": -12.387444496154785,
      "activations/layer10_attention_weight_max": 28.731388092041016,
      "activations/layer10_attention_weight_min": -32.13457107543945,
      "activations/layer11_attention_weight_max": 29.697675704956055,
      "activations/layer11_attention_weight_min": -32.678749084472656,
      "activations/layer12_attention_weight_max": 27.632661819458008,
      "activations/layer12_attention_weight_min": -24.972858428955078,
      "activations/layer13_attention_weight_max": 39.953575134277344,
      "activations/layer13_attention_weight_min": -34.95255661010742,
      "activations/layer14_attention_weight_max": 46.4949836730957,
      "activations/layer14_attention_weight_min": -36.88459014892578,
      "activations/layer15_attention_weight_max": 40.07954788208008,
      "activations/layer15_attention_weight_min": -33.74889373779297,
      "activations/layer16_attention_weight_max": 32.83148956298828,
      "activations/layer16_attention_weight_min": -27.155559539794922,
      "activations/layer17_attention_weight_max": 35.761444091796875,
      "activations/layer17_attention_weight_min": -25.753450393676758,
      "activations/layer18_attention_weight_max": 31.637197494506836,
      "activations/layer18_attention_weight_min": -24.7663631439209,
      "activations/layer19_attention_weight_max": 39.48733901977539,
      "activations/layer19_attention_weight_min": -30.579252243041992,
      "activations/layer1_attention_weight_max": 17.473100662231445,
      "activations/layer1_attention_weight_min": -14.351524353027344,
      "activations/layer20_attention_weight_max": 34.194210052490234,
      "activations/layer20_attention_weight_min": -25.174697875976562,
      "activations/layer21_attention_weight_max": 30.80867576599121,
      "activations/layer21_attention_weight_min": -23.716062545776367,
      "activations/layer22_attention_weight_max": 42.361534118652344,
      "activations/layer22_attention_weight_min": -30.857568740844727,
      "activations/layer23_attention_weight_max": 36.96929931640625,
      "activations/layer23_attention_weight_min": -25.964824676513672,
      "activations/layer2_attention_weight_max": 32.38569641113281,
      "activations/layer2_attention_weight_min": -32.198760986328125,
      "activations/layer3_attention_weight_max": 95.88977813720703,
      "activations/layer3_attention_weight_min": -95.32041931152344,
      "activations/layer4_attention_weight_max": 104.36797332763672,
      "activations/layer4_attention_weight_min": -107.99413299560547,
      "activations/layer5_attention_weight_max": 50.44416427612305,
      "activations/layer5_attention_weight_min": -59.755157470703125,
      "activations/layer6_attention_weight_max": 41.396705627441406,
      "activations/layer6_attention_weight_min": -43.6274528503418,
      "activations/layer7_attention_weight_max": 86.15997314453125,
      "activations/layer7_attention_weight_min": -89.35936737060547,
      "activations/layer8_attention_weight_max": 37.4009895324707,
      "activations/layer8_attention_weight_min": -41.148681640625,
      "activations/layer9_attention_weight_max": 28.824155807495117,
      "activations/layer9_attention_weight_min": -33.38774108886719,
      "epoch": 15.18,
      "learning_rate": 5.265643939393939e-05,
      "loss": 2.7506,
      "step": 261200
    },
    {
      "activations/layer0_attention_weight_max": 16.209800720214844,
      "activations/layer0_attention_weight_min": -12.723628044128418,
      "activations/layer10_attention_weight_max": 34.03999710083008,
      "activations/layer10_attention_weight_min": -33.5933837890625,
      "activations/layer11_attention_weight_max": 34.595802307128906,
      "activations/layer11_attention_weight_min": -32.49551010131836,
      "activations/layer12_attention_weight_max": 27.782838821411133,
      "activations/layer12_attention_weight_min": -25.934289932250977,
      "activations/layer13_attention_weight_max": 40.683387756347656,
      "activations/layer13_attention_weight_min": -32.86277389526367,
      "activations/layer14_attention_weight_max": 44.265872955322266,
      "activations/layer14_attention_weight_min": -35.92420959472656,
      "activations/layer15_attention_weight_max": 40.67282485961914,
      "activations/layer15_attention_weight_min": -33.48292922973633,
      "activations/layer16_attention_weight_max": 33.65180969238281,
      "activations/layer16_attention_weight_min": -28.82181739807129,
      "activations/layer17_attention_weight_max": 31.883487701416016,
      "activations/layer17_attention_weight_min": -26.54212760925293,
      "activations/layer18_attention_weight_max": 33.97718811035156,
      "activations/layer18_attention_weight_min": -26.281558990478516,
      "activations/layer19_attention_weight_max": 37.536800384521484,
      "activations/layer19_attention_weight_min": -32.461265563964844,
      "activations/layer1_attention_weight_max": 16.261735916137695,
      "activations/layer1_attention_weight_min": -15.138033866882324,
      "activations/layer20_attention_weight_max": 30.79281234741211,
      "activations/layer20_attention_weight_min": -26.6431884765625,
      "activations/layer21_attention_weight_max": 32.663848876953125,
      "activations/layer21_attention_weight_min": -26.60533332824707,
      "activations/layer22_attention_weight_max": 52.4830207824707,
      "activations/layer22_attention_weight_min": -35.0338249206543,
      "activations/layer23_attention_weight_max": 37.45014190673828,
      "activations/layer23_attention_weight_min": -27.578170776367188,
      "activations/layer2_attention_weight_max": 32.946624755859375,
      "activations/layer2_attention_weight_min": -32.1883659362793,
      "activations/layer3_attention_weight_max": 96.28890228271484,
      "activations/layer3_attention_weight_min": -96.81526947021484,
      "activations/layer4_attention_weight_max": 104.5759048461914,
      "activations/layer4_attention_weight_min": -108.7793197631836,
      "activations/layer5_attention_weight_max": 52.825828552246094,
      "activations/layer5_attention_weight_min": -60.863494873046875,
      "activations/layer6_attention_weight_max": 43.1166877746582,
      "activations/layer6_attention_weight_min": -43.40471267700195,
      "activations/layer7_attention_weight_max": 90.95160675048828,
      "activations/layer7_attention_weight_min": -87.40582275390625,
      "activations/layer8_attention_weight_max": 39.16769790649414,
      "activations/layer8_attention_weight_min": -42.9886589050293,
      "activations/layer9_attention_weight_max": 33.11882400512695,
      "activations/layer9_attention_weight_min": -34.535926818847656,
      "epoch": 15.18,
      "learning_rate": 5.263749999999999e-05,
      "loss": 2.7378,
      "step": 261250
    },
    {
      "activations/layer0_attention_weight_max": 15.190828323364258,
      "activations/layer0_attention_weight_min": -12.348730087280273,
      "activations/layer10_attention_weight_max": 32.51827621459961,
      "activations/layer10_attention_weight_min": -32.621376037597656,
      "activations/layer11_attention_weight_max": 32.01384353637695,
      "activations/layer11_attention_weight_min": -30.327789306640625,
      "activations/layer12_attention_weight_max": 26.839704513549805,
      "activations/layer12_attention_weight_min": -26.03250503540039,
      "activations/layer13_attention_weight_max": 41.19114685058594,
      "activations/layer13_attention_weight_min": -37.46964645385742,
      "activations/layer14_attention_weight_max": 47.699684143066406,
      "activations/layer14_attention_weight_min": -39.31085968017578,
      "activations/layer15_attention_weight_max": 41.218292236328125,
      "activations/layer15_attention_weight_min": -34.79519271850586,
      "activations/layer16_attention_weight_max": 30.833768844604492,
      "activations/layer16_attention_weight_min": -30.334367752075195,
      "activations/layer17_attention_weight_max": 33.25592041015625,
      "activations/layer17_attention_weight_min": -25.400928497314453,
      "activations/layer18_attention_weight_max": 32.533180236816406,
      "activations/layer18_attention_weight_min": -24.617921829223633,
      "activations/layer19_attention_weight_max": 35.82262420654297,
      "activations/layer19_attention_weight_min": -30.547866821289062,
      "activations/layer1_attention_weight_max": 15.865447044372559,
      "activations/layer1_attention_weight_min": -14.446829795837402,
      "activations/layer20_attention_weight_max": 30.382871627807617,
      "activations/layer20_attention_weight_min": -25.362043380737305,
      "activations/layer21_attention_weight_max": 30.206979751586914,
      "activations/layer21_attention_weight_min": -24.65648651123047,
      "activations/layer22_attention_weight_max": 45.59226989746094,
      "activations/layer22_attention_weight_min": -32.37026596069336,
      "activations/layer23_attention_weight_max": 36.3246955871582,
      "activations/layer23_attention_weight_min": -26.10232925415039,
      "activations/layer2_attention_weight_max": 32.141780853271484,
      "activations/layer2_attention_weight_min": -34.7656364440918,
      "activations/layer3_attention_weight_max": 98.4870834350586,
      "activations/layer3_attention_weight_min": -100.35398864746094,
      "activations/layer4_attention_weight_max": 105.9864501953125,
      "activations/layer4_attention_weight_min": -111.59063720703125,
      "activations/layer5_attention_weight_max": 51.812191009521484,
      "activations/layer5_attention_weight_min": -63.104820251464844,
      "activations/layer6_attention_weight_max": 43.64077377319336,
      "activations/layer6_attention_weight_min": -45.9644889831543,
      "activations/layer7_attention_weight_max": 93.1575698852539,
      "activations/layer7_attention_weight_min": -89.79350280761719,
      "activations/layer8_attention_weight_max": 38.72517776489258,
      "activations/layer8_attention_weight_min": -39.985782623291016,
      "activations/layer9_attention_weight_max": 31.365497589111328,
      "activations/layer9_attention_weight_min": -32.18940353393555,
      "epoch": 15.18,
      "learning_rate": 5.26185606060606e-05,
      "loss": 2.7442,
      "step": 261300
    },
    {
      "activations/layer0_attention_weight_max": 16.10011863708496,
      "activations/layer0_attention_weight_min": -12.820718765258789,
      "activations/layer10_attention_weight_max": 33.7440071105957,
      "activations/layer10_attention_weight_min": -35.4518928527832,
      "activations/layer11_attention_weight_max": 34.917991638183594,
      "activations/layer11_attention_weight_min": -35.71305465698242,
      "activations/layer12_attention_weight_max": 27.948444366455078,
      "activations/layer12_attention_weight_min": -25.843481063842773,
      "activations/layer13_attention_weight_max": 39.76677703857422,
      "activations/layer13_attention_weight_min": -34.84381866455078,
      "activations/layer14_attention_weight_max": 43.536338806152344,
      "activations/layer14_attention_weight_min": -40.44505310058594,
      "activations/layer15_attention_weight_max": 40.658599853515625,
      "activations/layer15_attention_weight_min": -35.46453857421875,
      "activations/layer16_attention_weight_max": 34.258445739746094,
      "activations/layer16_attention_weight_min": -29.674833297729492,
      "activations/layer17_attention_weight_max": 31.43625259399414,
      "activations/layer17_attention_weight_min": -27.622676849365234,
      "activations/layer18_attention_weight_max": 31.90252113342285,
      "activations/layer18_attention_weight_min": -26.297266006469727,
      "activations/layer19_attention_weight_max": 35.169673919677734,
      "activations/layer19_attention_weight_min": -32.93809127807617,
      "activations/layer1_attention_weight_max": 15.803009033203125,
      "activations/layer1_attention_weight_min": -16.323001861572266,
      "activations/layer20_attention_weight_max": 30.558265686035156,
      "activations/layer20_attention_weight_min": -24.5502872467041,
      "activations/layer21_attention_weight_max": 31.81618309020996,
      "activations/layer21_attention_weight_min": -24.130704879760742,
      "activations/layer22_attention_weight_max": 42.77808380126953,
      "activations/layer22_attention_weight_min": -30.273733139038086,
      "activations/layer23_attention_weight_max": 36.80828094482422,
      "activations/layer23_attention_weight_min": -24.779056549072266,
      "activations/layer2_attention_weight_max": 32.915924072265625,
      "activations/layer2_attention_weight_min": -32.555030822753906,
      "activations/layer3_attention_weight_max": 100.8695068359375,
      "activations/layer3_attention_weight_min": -95.93708038330078,
      "activations/layer4_attention_weight_max": 109.7278060913086,
      "activations/layer4_attention_weight_min": -104.23756408691406,
      "activations/layer5_attention_weight_max": 53.11524963378906,
      "activations/layer5_attention_weight_min": -61.48595428466797,
      "activations/layer6_attention_weight_max": 43.49315643310547,
      "activations/layer6_attention_weight_min": -46.15290069580078,
      "activations/layer7_attention_weight_max": 93.91537475585938,
      "activations/layer7_attention_weight_min": -93.10893249511719,
      "activations/layer8_attention_weight_max": 42.781898498535156,
      "activations/layer8_attention_weight_min": -42.38160705566406,
      "activations/layer9_attention_weight_max": 33.18663024902344,
      "activations/layer9_attention_weight_min": -34.26203536987305,
      "epoch": 15.19,
      "learning_rate": 5.259962121212121e-05,
      "loss": 2.7466,
      "step": 261350
    },
    {
      "activations/layer0_attention_weight_max": 15.22973918914795,
      "activations/layer0_attention_weight_min": -11.895007133483887,
      "activations/layer10_attention_weight_max": 32.790523529052734,
      "activations/layer10_attention_weight_min": -33.519805908203125,
      "activations/layer11_attention_weight_max": 34.352447509765625,
      "activations/layer11_attention_weight_min": -35.67611312866211,
      "activations/layer12_attention_weight_max": 25.43730926513672,
      "activations/layer12_attention_weight_min": -26.296533584594727,
      "activations/layer13_attention_weight_max": 38.97259521484375,
      "activations/layer13_attention_weight_min": -35.68406295776367,
      "activations/layer14_attention_weight_max": 41.66482925415039,
      "activations/layer14_attention_weight_min": -38.191078186035156,
      "activations/layer15_attention_weight_max": 37.04353713989258,
      "activations/layer15_attention_weight_min": -33.96018981933594,
      "activations/layer16_attention_weight_max": 31.800281524658203,
      "activations/layer16_attention_weight_min": -26.686277389526367,
      "activations/layer17_attention_weight_max": 29.12790298461914,
      "activations/layer17_attention_weight_min": -24.24504852294922,
      "activations/layer18_attention_weight_max": 31.55278205871582,
      "activations/layer18_attention_weight_min": -23.841175079345703,
      "activations/layer19_attention_weight_max": 31.11527442932129,
      "activations/layer19_attention_weight_min": -28.386564254760742,
      "activations/layer1_attention_weight_max": 17.32597541809082,
      "activations/layer1_attention_weight_min": -15.572054862976074,
      "activations/layer20_attention_weight_max": 27.475250244140625,
      "activations/layer20_attention_weight_min": -25.133193969726562,
      "activations/layer21_attention_weight_max": 28.955551147460938,
      "activations/layer21_attention_weight_min": -23.370317459106445,
      "activations/layer22_attention_weight_max": 39.606502532958984,
      "activations/layer22_attention_weight_min": -28.448816299438477,
      "activations/layer23_attention_weight_max": 32.874916076660156,
      "activations/layer23_attention_weight_min": -23.303932189941406,
      "activations/layer2_attention_weight_max": 33.88312530517578,
      "activations/layer2_attention_weight_min": -33.55733871459961,
      "activations/layer3_attention_weight_max": 95.16241455078125,
      "activations/layer3_attention_weight_min": -96.19052124023438,
      "activations/layer4_attention_weight_max": 108.1462631225586,
      "activations/layer4_attention_weight_min": -108.46341705322266,
      "activations/layer5_attention_weight_max": 49.72434616088867,
      "activations/layer5_attention_weight_min": -59.98382568359375,
      "activations/layer6_attention_weight_max": 44.34085464477539,
      "activations/layer6_attention_weight_min": -44.45553970336914,
      "activations/layer7_attention_weight_max": 87.16281127929688,
      "activations/layer7_attention_weight_min": -92.68360900878906,
      "activations/layer8_attention_weight_max": 42.56947326660156,
      "activations/layer8_attention_weight_min": -41.896270751953125,
      "activations/layer9_attention_weight_max": 33.330902099609375,
      "activations/layer9_attention_weight_min": -34.83623504638672,
      "epoch": 15.19,
      "learning_rate": 5.258068181818182e-05,
      "loss": 2.7516,
      "step": 261400
    },
    {
      "activations/layer0_attention_weight_max": 16.460346221923828,
      "activations/layer0_attention_weight_min": -12.154707908630371,
      "activations/layer10_attention_weight_max": 39.1224250793457,
      "activations/layer10_attention_weight_min": -38.52659225463867,
      "activations/layer11_attention_weight_max": 43.151756286621094,
      "activations/layer11_attention_weight_min": -40.26472091674805,
      "activations/layer12_attention_weight_max": 27.293115615844727,
      "activations/layer12_attention_weight_min": -27.1953182220459,
      "activations/layer13_attention_weight_max": 42.41901779174805,
      "activations/layer13_attention_weight_min": -39.557674407958984,
      "activations/layer14_attention_weight_max": 46.75904846191406,
      "activations/layer14_attention_weight_min": -46.19657897949219,
      "activations/layer15_attention_weight_max": 44.4273796081543,
      "activations/layer15_attention_weight_min": -38.97277069091797,
      "activations/layer16_attention_weight_max": 36.94321060180664,
      "activations/layer16_attention_weight_min": -29.221723556518555,
      "activations/layer17_attention_weight_max": 34.339927673339844,
      "activations/layer17_attention_weight_min": -27.12099838256836,
      "activations/layer18_attention_weight_max": 32.78327560424805,
      "activations/layer18_attention_weight_min": -25.66132164001465,
      "activations/layer19_attention_weight_max": 36.759769439697266,
      "activations/layer19_attention_weight_min": -30.393461227416992,
      "activations/layer1_attention_weight_max": 17.022584915161133,
      "activations/layer1_attention_weight_min": -15.518753051757812,
      "activations/layer20_attention_weight_max": 31.51382064819336,
      "activations/layer20_attention_weight_min": -25.90570640563965,
      "activations/layer21_attention_weight_max": 28.98223114013672,
      "activations/layer21_attention_weight_min": -26.067752838134766,
      "activations/layer22_attention_weight_max": 42.116172790527344,
      "activations/layer22_attention_weight_min": -32.20341873168945,
      "activations/layer23_attention_weight_max": 32.425750732421875,
      "activations/layer23_attention_weight_min": -25.473798751831055,
      "activations/layer2_attention_weight_max": 33.827293395996094,
      "activations/layer2_attention_weight_min": -33.91704559326172,
      "activations/layer3_attention_weight_max": 95.6454086303711,
      "activations/layer3_attention_weight_min": -96.8310546875,
      "activations/layer4_attention_weight_max": 107.19730377197266,
      "activations/layer4_attention_weight_min": -109.43029022216797,
      "activations/layer5_attention_weight_max": 48.26922607421875,
      "activations/layer5_attention_weight_min": -64.01549530029297,
      "activations/layer6_attention_weight_max": 43.66029357910156,
      "activations/layer6_attention_weight_min": -46.050193786621094,
      "activations/layer7_attention_weight_max": 100.7341079711914,
      "activations/layer7_attention_weight_min": -90.17284393310547,
      "activations/layer8_attention_weight_max": 46.046268463134766,
      "activations/layer8_attention_weight_min": -45.257347106933594,
      "activations/layer9_attention_weight_max": 36.76578903198242,
      "activations/layer9_attention_weight_min": -37.40885543823242,
      "epoch": 15.19,
      "learning_rate": 5.256174242424242e-05,
      "loss": 2.7561,
      "step": 261450
    },
    {
      "activations/layer0_attention_weight_max": 17.113571166992188,
      "activations/layer0_attention_weight_min": -12.826404571533203,
      "activations/layer10_attention_weight_max": 33.42949676513672,
      "activations/layer10_attention_weight_min": -34.756874084472656,
      "activations/layer11_attention_weight_max": 33.09606170654297,
      "activations/layer11_attention_weight_min": -34.53666687011719,
      "activations/layer12_attention_weight_max": 25.046186447143555,
      "activations/layer12_attention_weight_min": -26.20704460144043,
      "activations/layer13_attention_weight_max": 37.07508850097656,
      "activations/layer13_attention_weight_min": -35.96738815307617,
      "activations/layer14_attention_weight_max": 40.67951202392578,
      "activations/layer14_attention_weight_min": -36.724063873291016,
      "activations/layer15_attention_weight_max": 34.48503112792969,
      "activations/layer15_attention_weight_min": -34.306434631347656,
      "activations/layer16_attention_weight_max": 28.22916030883789,
      "activations/layer16_attention_weight_min": -28.17308235168457,
      "activations/layer17_attention_weight_max": 29.579219818115234,
      "activations/layer17_attention_weight_min": -26.548751831054688,
      "activations/layer18_attention_weight_max": 32.4454460144043,
      "activations/layer18_attention_weight_min": -25.250295639038086,
      "activations/layer19_attention_weight_max": 32.966941833496094,
      "activations/layer19_attention_weight_min": -32.47504806518555,
      "activations/layer1_attention_weight_max": 16.458181381225586,
      "activations/layer1_attention_weight_min": -15.723939895629883,
      "activations/layer20_attention_weight_max": 28.971607208251953,
      "activations/layer20_attention_weight_min": -26.328283309936523,
      "activations/layer21_attention_weight_max": 28.185392379760742,
      "activations/layer21_attention_weight_min": -27.733993530273438,
      "activations/layer22_attention_weight_max": 42.91206359863281,
      "activations/layer22_attention_weight_min": -32.54151916503906,
      "activations/layer23_attention_weight_max": 31.10279083251953,
      "activations/layer23_attention_weight_min": -27.34026336669922,
      "activations/layer2_attention_weight_max": 31.788917541503906,
      "activations/layer2_attention_weight_min": -31.814231872558594,
      "activations/layer3_attention_weight_max": 93.53633117675781,
      "activations/layer3_attention_weight_min": -90.98863983154297,
      "activations/layer4_attention_weight_max": 104.19343566894531,
      "activations/layer4_attention_weight_min": -104.65775299072266,
      "activations/layer5_attention_weight_max": 50.960845947265625,
      "activations/layer5_attention_weight_min": -59.07762145996094,
      "activations/layer6_attention_weight_max": 41.743316650390625,
      "activations/layer6_attention_weight_min": -46.68459701538086,
      "activations/layer7_attention_weight_max": 86.38236999511719,
      "activations/layer7_attention_weight_min": -94.05311584472656,
      "activations/layer8_attention_weight_max": 40.223358154296875,
      "activations/layer8_attention_weight_min": -41.71466064453125,
      "activations/layer9_attention_weight_max": 31.19076919555664,
      "activations/layer9_attention_weight_min": -33.13447189331055,
      "epoch": 15.19,
      "learning_rate": 5.2542803030303027e-05,
      "loss": 2.7544,
      "step": 261500
    },
    {
      "activations/layer0_attention_weight_max": 16.377681732177734,
      "activations/layer0_attention_weight_min": -12.842941284179688,
      "activations/layer10_attention_weight_max": 36.78419876098633,
      "activations/layer10_attention_weight_min": -35.18274688720703,
      "activations/layer11_attention_weight_max": 35.76081848144531,
      "activations/layer11_attention_weight_min": -37.26740264892578,
      "activations/layer12_attention_weight_max": 26.087934494018555,
      "activations/layer12_attention_weight_min": -24.53253173828125,
      "activations/layer13_attention_weight_max": 36.12015914916992,
      "activations/layer13_attention_weight_min": -32.41680908203125,
      "activations/layer14_attention_weight_max": 42.75709533691406,
      "activations/layer14_attention_weight_min": -35.52607727050781,
      "activations/layer15_attention_weight_max": 36.56996536254883,
      "activations/layer15_attention_weight_min": -32.31672668457031,
      "activations/layer16_attention_weight_max": 29.057783126831055,
      "activations/layer16_attention_weight_min": -26.94609260559082,
      "activations/layer17_attention_weight_max": 29.33167266845703,
      "activations/layer17_attention_weight_min": -25.45342445373535,
      "activations/layer18_attention_weight_max": 34.49936294555664,
      "activations/layer18_attention_weight_min": -24.938676834106445,
      "activations/layer19_attention_weight_max": 34.929500579833984,
      "activations/layer19_attention_weight_min": -30.919239044189453,
      "activations/layer1_attention_weight_max": 17.206588745117188,
      "activations/layer1_attention_weight_min": -13.656445503234863,
      "activations/layer20_attention_weight_max": 30.925724029541016,
      "activations/layer20_attention_weight_min": -24.773845672607422,
      "activations/layer21_attention_weight_max": 31.155410766601562,
      "activations/layer21_attention_weight_min": -24.5723934173584,
      "activations/layer22_attention_weight_max": 42.58169937133789,
      "activations/layer22_attention_weight_min": -30.262104034423828,
      "activations/layer23_attention_weight_max": 34.109466552734375,
      "activations/layer23_attention_weight_min": -24.968250274658203,
      "activations/layer2_attention_weight_max": 31.784610748291016,
      "activations/layer2_attention_weight_min": -31.84804916381836,
      "activations/layer3_attention_weight_max": 92.24349212646484,
      "activations/layer3_attention_weight_min": -96.18026733398438,
      "activations/layer4_attention_weight_max": 112.34089660644531,
      "activations/layer4_attention_weight_min": -116.3924789428711,
      "activations/layer5_attention_weight_max": 51.592735290527344,
      "activations/layer5_attention_weight_min": -65.55337524414062,
      "activations/layer6_attention_weight_max": 43.82225799560547,
      "activations/layer6_attention_weight_min": -45.80791473388672,
      "activations/layer7_attention_weight_max": 97.02841186523438,
      "activations/layer7_attention_weight_min": -99.29082489013672,
      "activations/layer8_attention_weight_max": 43.35615921020508,
      "activations/layer8_attention_weight_min": -44.80952453613281,
      "activations/layer9_attention_weight_max": 34.223148345947266,
      "activations/layer9_attention_weight_min": -34.84304428100586,
      "epoch": 15.2,
      "learning_rate": 5.2523863636363635e-05,
      "loss": 2.7534,
      "step": 261550
    },
    {
      "activations/layer0_attention_weight_max": 15.416687965393066,
      "activations/layer0_attention_weight_min": -12.328184127807617,
      "activations/layer10_attention_weight_max": 33.00249099731445,
      "activations/layer10_attention_weight_min": -33.68100357055664,
      "activations/layer11_attention_weight_max": 33.10610580444336,
      "activations/layer11_attention_weight_min": -34.551292419433594,
      "activations/layer12_attention_weight_max": 26.10064125061035,
      "activations/layer12_attention_weight_min": -25.45984649658203,
      "activations/layer13_attention_weight_max": 38.839717864990234,
      "activations/layer13_attention_weight_min": -37.14882278442383,
      "activations/layer14_attention_weight_max": 43.419891357421875,
      "activations/layer14_attention_weight_min": -40.87610626220703,
      "activations/layer15_attention_weight_max": 38.93789291381836,
      "activations/layer15_attention_weight_min": -36.447017669677734,
      "activations/layer16_attention_weight_max": 31.570520401000977,
      "activations/layer16_attention_weight_min": -29.42329216003418,
      "activations/layer17_attention_weight_max": 32.35111618041992,
      "activations/layer17_attention_weight_min": -27.66419792175293,
      "activations/layer18_attention_weight_max": 32.512046813964844,
      "activations/layer18_attention_weight_min": -26.24005699157715,
      "activations/layer19_attention_weight_max": 38.95095443725586,
      "activations/layer19_attention_weight_min": -33.35343933105469,
      "activations/layer1_attention_weight_max": 16.028884887695312,
      "activations/layer1_attention_weight_min": -14.399500846862793,
      "activations/layer20_attention_weight_max": 34.64229202270508,
      "activations/layer20_attention_weight_min": -26.77785873413086,
      "activations/layer21_attention_weight_max": 32.91136169433594,
      "activations/layer21_attention_weight_min": -25.201583862304688,
      "activations/layer22_attention_weight_max": 51.64736557006836,
      "activations/layer22_attention_weight_min": -30.916488647460938,
      "activations/layer23_attention_weight_max": 42.58608627319336,
      "activations/layer23_attention_weight_min": -24.932340621948242,
      "activations/layer2_attention_weight_max": 31.57289695739746,
      "activations/layer2_attention_weight_min": -32.46028137207031,
      "activations/layer3_attention_weight_max": 96.12489318847656,
      "activations/layer3_attention_weight_min": -95.38613891601562,
      "activations/layer4_attention_weight_max": 110.0110855102539,
      "activations/layer4_attention_weight_min": -112.70259094238281,
      "activations/layer5_attention_weight_max": 50.56954574584961,
      "activations/layer5_attention_weight_min": -61.9426383972168,
      "activations/layer6_attention_weight_max": 44.57359313964844,
      "activations/layer6_attention_weight_min": -47.614933013916016,
      "activations/layer7_attention_weight_max": 93.16912841796875,
      "activations/layer7_attention_weight_min": -94.46912384033203,
      "activations/layer8_attention_weight_max": 41.58919906616211,
      "activations/layer8_attention_weight_min": -45.17839813232422,
      "activations/layer9_attention_weight_max": 32.58577346801758,
      "activations/layer9_attention_weight_min": -34.248130798339844,
      "epoch": 15.2,
      "learning_rate": 5.2504924242424236e-05,
      "loss": 2.7554,
      "step": 261600
    },
    {
      "activations/layer0_attention_weight_max": 15.536911010742188,
      "activations/layer0_attention_weight_min": -12.363390922546387,
      "activations/layer10_attention_weight_max": 35.55801773071289,
      "activations/layer10_attention_weight_min": -36.03114700317383,
      "activations/layer11_attention_weight_max": 36.35964584350586,
      "activations/layer11_attention_weight_min": -34.873329162597656,
      "activations/layer12_attention_weight_max": 31.144487380981445,
      "activations/layer12_attention_weight_min": -27.74321174621582,
      "activations/layer13_attention_weight_max": 40.84709548950195,
      "activations/layer13_attention_weight_min": -38.542781829833984,
      "activations/layer14_attention_weight_max": 50.08476257324219,
      "activations/layer14_attention_weight_min": -44.89302062988281,
      "activations/layer15_attention_weight_max": 41.062461853027344,
      "activations/layer15_attention_weight_min": -39.9319953918457,
      "activations/layer16_attention_weight_max": 33.21165084838867,
      "activations/layer16_attention_weight_min": -30.44335174560547,
      "activations/layer17_attention_weight_max": 30.961524963378906,
      "activations/layer17_attention_weight_min": -29.705501556396484,
      "activations/layer18_attention_weight_max": 30.560287475585938,
      "activations/layer18_attention_weight_min": -25.664182662963867,
      "activations/layer19_attention_weight_max": 36.15677261352539,
      "activations/layer19_attention_weight_min": -31.448394775390625,
      "activations/layer1_attention_weight_max": 16.11298942565918,
      "activations/layer1_attention_weight_min": -15.197122573852539,
      "activations/layer20_attention_weight_max": 32.47676467895508,
      "activations/layer20_attention_weight_min": -24.79059410095215,
      "activations/layer21_attention_weight_max": 30.35708999633789,
      "activations/layer21_attention_weight_min": -23.90481948852539,
      "activations/layer22_attention_weight_max": 44.44243621826172,
      "activations/layer22_attention_weight_min": -31.170166015625,
      "activations/layer23_attention_weight_max": 36.443058013916016,
      "activations/layer23_attention_weight_min": -25.02426528930664,
      "activations/layer2_attention_weight_max": 34.49462890625,
      "activations/layer2_attention_weight_min": -34.04429244995117,
      "activations/layer3_attention_weight_max": 99.1808853149414,
      "activations/layer3_attention_weight_min": -99.00424194335938,
      "activations/layer4_attention_weight_max": 107.18316650390625,
      "activations/layer4_attention_weight_min": -110.86090087890625,
      "activations/layer5_attention_weight_max": 50.09989929199219,
      "activations/layer5_attention_weight_min": -61.71140670776367,
      "activations/layer6_attention_weight_max": 42.88704299926758,
      "activations/layer6_attention_weight_min": -46.14329528808594,
      "activations/layer7_attention_weight_max": 100.6131820678711,
      "activations/layer7_attention_weight_min": -95.94380187988281,
      "activations/layer8_attention_weight_max": 45.36885070800781,
      "activations/layer8_attention_weight_min": -43.69942092895508,
      "activations/layer9_attention_weight_max": 34.00351333618164,
      "activations/layer9_attention_weight_min": -34.27450180053711,
      "epoch": 15.2,
      "learning_rate": 5.248598484848484e-05,
      "loss": 2.7406,
      "step": 261650
    },
    {
      "activations/layer0_attention_weight_max": 15.307923316955566,
      "activations/layer0_attention_weight_min": -12.519893646240234,
      "activations/layer10_attention_weight_max": 35.108497619628906,
      "activations/layer10_attention_weight_min": -34.918880462646484,
      "activations/layer11_attention_weight_max": 34.248748779296875,
      "activations/layer11_attention_weight_min": -35.195472717285156,
      "activations/layer12_attention_weight_max": 27.70603370666504,
      "activations/layer12_attention_weight_min": -27.89087677001953,
      "activations/layer13_attention_weight_max": 43.7346076965332,
      "activations/layer13_attention_weight_min": -36.086395263671875,
      "activations/layer14_attention_weight_max": 45.59553909301758,
      "activations/layer14_attention_weight_min": -39.517845153808594,
      "activations/layer15_attention_weight_max": 39.90839385986328,
      "activations/layer15_attention_weight_min": -35.502708435058594,
      "activations/layer16_attention_weight_max": 32.02516555786133,
      "activations/layer16_attention_weight_min": -29.296831130981445,
      "activations/layer17_attention_weight_max": 29.743051528930664,
      "activations/layer17_attention_weight_min": -28.566532135009766,
      "activations/layer18_attention_weight_max": 31.901775360107422,
      "activations/layer18_attention_weight_min": -28.540616989135742,
      "activations/layer19_attention_weight_max": 34.93839645385742,
      "activations/layer19_attention_weight_min": -34.21529769897461,
      "activations/layer1_attention_weight_max": 15.531974792480469,
      "activations/layer1_attention_weight_min": -14.367379188537598,
      "activations/layer20_attention_weight_max": 29.234291076660156,
      "activations/layer20_attention_weight_min": -27.185405731201172,
      "activations/layer21_attention_weight_max": 28.987773895263672,
      "activations/layer21_attention_weight_min": -25.340042114257812,
      "activations/layer22_attention_weight_max": 45.996158599853516,
      "activations/layer22_attention_weight_min": -32.12449645996094,
      "activations/layer23_attention_weight_max": 36.08961868286133,
      "activations/layer23_attention_weight_min": -27.873090744018555,
      "activations/layer2_attention_weight_max": 31.56121063232422,
      "activations/layer2_attention_weight_min": -33.215476989746094,
      "activations/layer3_attention_weight_max": 94.81758117675781,
      "activations/layer3_attention_weight_min": -94.42667388916016,
      "activations/layer4_attention_weight_max": 106.3766098022461,
      "activations/layer4_attention_weight_min": -106.2879867553711,
      "activations/layer5_attention_weight_max": 52.01105499267578,
      "activations/layer5_attention_weight_min": -59.01921081542969,
      "activations/layer6_attention_weight_max": 44.82501983642578,
      "activations/layer6_attention_weight_min": -45.1981086730957,
      "activations/layer7_attention_weight_max": 95.23353576660156,
      "activations/layer7_attention_weight_min": -93.05313873291016,
      "activations/layer8_attention_weight_max": 43.33716583251953,
      "activations/layer8_attention_weight_min": -43.968807220458984,
      "activations/layer9_attention_weight_max": 35.45716857910156,
      "activations/layer9_attention_weight_min": -35.708553314208984,
      "epoch": 15.21,
      "learning_rate": 5.246704545454545e-05,
      "loss": 2.746,
      "step": 261700
    },
    {
      "activations/layer0_attention_weight_max": 16.57844352722168,
      "activations/layer0_attention_weight_min": -11.870950698852539,
      "activations/layer10_attention_weight_max": 37.831336975097656,
      "activations/layer10_attention_weight_min": -37.59116744995117,
      "activations/layer11_attention_weight_max": 34.84854507446289,
      "activations/layer11_attention_weight_min": -36.257713317871094,
      "activations/layer12_attention_weight_max": 29.917387008666992,
      "activations/layer12_attention_weight_min": -27.63349151611328,
      "activations/layer13_attention_weight_max": 42.424739837646484,
      "activations/layer13_attention_weight_min": -35.93425750732422,
      "activations/layer14_attention_weight_max": 44.34613037109375,
      "activations/layer14_attention_weight_min": -38.582149505615234,
      "activations/layer15_attention_weight_max": 40.834678649902344,
      "activations/layer15_attention_weight_min": -36.231937408447266,
      "activations/layer16_attention_weight_max": 31.14982795715332,
      "activations/layer16_attention_weight_min": -25.965497970581055,
      "activations/layer17_attention_weight_max": 31.31414222717285,
      "activations/layer17_attention_weight_min": -25.23455238342285,
      "activations/layer18_attention_weight_max": 33.90156936645508,
      "activations/layer18_attention_weight_min": -24.885713577270508,
      "activations/layer19_attention_weight_max": 35.1210823059082,
      "activations/layer19_attention_weight_min": -30.89618492126465,
      "activations/layer1_attention_weight_max": 15.431589126586914,
      "activations/layer1_attention_weight_min": -14.999284744262695,
      "activations/layer20_attention_weight_max": 29.757394790649414,
      "activations/layer20_attention_weight_min": -24.483898162841797,
      "activations/layer21_attention_weight_max": 29.111007690429688,
      "activations/layer21_attention_weight_min": -23.935190200805664,
      "activations/layer22_attention_weight_max": 46.09568786621094,
      "activations/layer22_attention_weight_min": -30.90571403503418,
      "activations/layer23_attention_weight_max": 35.78881072998047,
      "activations/layer23_attention_weight_min": -24.636089324951172,
      "activations/layer2_attention_weight_max": 33.315887451171875,
      "activations/layer2_attention_weight_min": -34.44247817993164,
      "activations/layer3_attention_weight_max": 100.17866516113281,
      "activations/layer3_attention_weight_min": -99.65301513671875,
      "activations/layer4_attention_weight_max": 113.3830795288086,
      "activations/layer4_attention_weight_min": -112.01273345947266,
      "activations/layer5_attention_weight_max": 51.2878532409668,
      "activations/layer5_attention_weight_min": -58.92151641845703,
      "activations/layer6_attention_weight_max": 45.19083023071289,
      "activations/layer6_attention_weight_min": -45.62925338745117,
      "activations/layer7_attention_weight_max": 98.92789459228516,
      "activations/layer7_attention_weight_min": -92.66181182861328,
      "activations/layer8_attention_weight_max": 44.47458267211914,
      "activations/layer8_attention_weight_min": -44.87136459350586,
      "activations/layer9_attention_weight_max": 35.80479049682617,
      "activations/layer9_attention_weight_min": -36.6362190246582,
      "epoch": 15.21,
      "learning_rate": 5.2448106060606054e-05,
      "loss": 2.7556,
      "step": 261750
    },
    {
      "activations/layer0_attention_weight_max": 16.0939998626709,
      "activations/layer0_attention_weight_min": -12.433035850524902,
      "activations/layer10_attention_weight_max": 32.17897415161133,
      "activations/layer10_attention_weight_min": -31.625656127929688,
      "activations/layer11_attention_weight_max": 33.2352294921875,
      "activations/layer11_attention_weight_min": -32.700435638427734,
      "activations/layer12_attention_weight_max": 26.213401794433594,
      "activations/layer12_attention_weight_min": -26.2464542388916,
      "activations/layer13_attention_weight_max": 38.5706901550293,
      "activations/layer13_attention_weight_min": -35.11350631713867,
      "activations/layer14_attention_weight_max": 42.3902473449707,
      "activations/layer14_attention_weight_min": -36.27861022949219,
      "activations/layer15_attention_weight_max": 36.57917404174805,
      "activations/layer15_attention_weight_min": -34.27476501464844,
      "activations/layer16_attention_weight_max": 29.683212280273438,
      "activations/layer16_attention_weight_min": -29.99884605407715,
      "activations/layer17_attention_weight_max": 30.51468276977539,
      "activations/layer17_attention_weight_min": -26.824840545654297,
      "activations/layer18_attention_weight_max": 32.040000915527344,
      "activations/layer18_attention_weight_min": -27.9010066986084,
      "activations/layer19_attention_weight_max": 32.60618209838867,
      "activations/layer19_attention_weight_min": -32.033260345458984,
      "activations/layer1_attention_weight_max": 15.797988891601562,
      "activations/layer1_attention_weight_min": -14.617679595947266,
      "activations/layer20_attention_weight_max": 27.539794921875,
      "activations/layer20_attention_weight_min": -27.759075164794922,
      "activations/layer21_attention_weight_max": 28.197616577148438,
      "activations/layer21_attention_weight_min": -24.605613708496094,
      "activations/layer22_attention_weight_max": 43.395206451416016,
      "activations/layer22_attention_weight_min": -32.01356506347656,
      "activations/layer23_attention_weight_max": 31.73672103881836,
      "activations/layer23_attention_weight_min": -26.392807006835938,
      "activations/layer2_attention_weight_max": 31.316001892089844,
      "activations/layer2_attention_weight_min": -31.911663055419922,
      "activations/layer3_attention_weight_max": 93.85466003417969,
      "activations/layer3_attention_weight_min": -94.07391357421875,
      "activations/layer4_attention_weight_max": 110.91950988769531,
      "activations/layer4_attention_weight_min": -109.4122543334961,
      "activations/layer5_attention_weight_max": 50.50257873535156,
      "activations/layer5_attention_weight_min": -61.63397216796875,
      "activations/layer6_attention_weight_max": 43.17738342285156,
      "activations/layer6_attention_weight_min": -45.151309967041016,
      "activations/layer7_attention_weight_max": 90.15335845947266,
      "activations/layer7_attention_weight_min": -95.14746856689453,
      "activations/layer8_attention_weight_max": 38.13982009887695,
      "activations/layer8_attention_weight_min": -41.181121826171875,
      "activations/layer9_attention_weight_max": 32.20024108886719,
      "activations/layer9_attention_weight_min": -32.533626556396484,
      "epoch": 15.21,
      "learning_rate": 5.242916666666666e-05,
      "loss": 2.7599,
      "step": 261800
    },
    {
      "activations/layer0_attention_weight_max": 15.550285339355469,
      "activations/layer0_attention_weight_min": -12.6348237991333,
      "activations/layer10_attention_weight_max": 38.97380065917969,
      "activations/layer10_attention_weight_min": -36.215911865234375,
      "activations/layer11_attention_weight_max": 37.30271911621094,
      "activations/layer11_attention_weight_min": -36.27008056640625,
      "activations/layer12_attention_weight_max": 25.185869216918945,
      "activations/layer12_attention_weight_min": -30.07019805908203,
      "activations/layer13_attention_weight_max": 39.17540740966797,
      "activations/layer13_attention_weight_min": -35.262672424316406,
      "activations/layer14_attention_weight_max": 44.447696685791016,
      "activations/layer14_attention_weight_min": -38.406742095947266,
      "activations/layer15_attention_weight_max": 36.22095489501953,
      "activations/layer15_attention_weight_min": -34.77232360839844,
      "activations/layer16_attention_weight_max": 28.900836944580078,
      "activations/layer16_attention_weight_min": -27.106746673583984,
      "activations/layer17_attention_weight_max": 29.76898193359375,
      "activations/layer17_attention_weight_min": -25.706201553344727,
      "activations/layer18_attention_weight_max": 29.73292350769043,
      "activations/layer18_attention_weight_min": -24.725425720214844,
      "activations/layer19_attention_weight_max": 30.536264419555664,
      "activations/layer19_attention_weight_min": -31.504735946655273,
      "activations/layer1_attention_weight_max": 16.178430557250977,
      "activations/layer1_attention_weight_min": -15.446852684020996,
      "activations/layer20_attention_weight_max": 26.84271812438965,
      "activations/layer20_attention_weight_min": -26.63652992248535,
      "activations/layer21_attention_weight_max": 28.588457107543945,
      "activations/layer21_attention_weight_min": -27.257841110229492,
      "activations/layer22_attention_weight_max": 42.668212890625,
      "activations/layer22_attention_weight_min": -31.816980361938477,
      "activations/layer23_attention_weight_max": 29.567930221557617,
      "activations/layer23_attention_weight_min": -25.482498168945312,
      "activations/layer2_attention_weight_max": 34.605621337890625,
      "activations/layer2_attention_weight_min": -35.826663970947266,
      "activations/layer3_attention_weight_max": 99.08365631103516,
      "activations/layer3_attention_weight_min": -103.69266510009766,
      "activations/layer4_attention_weight_max": 111.2109603881836,
      "activations/layer4_attention_weight_min": -108.4666976928711,
      "activations/layer5_attention_weight_max": 55.11140441894531,
      "activations/layer5_attention_weight_min": -63.2020263671875,
      "activations/layer6_attention_weight_max": 42.976585388183594,
      "activations/layer6_attention_weight_min": -47.58369445800781,
      "activations/layer7_attention_weight_max": 104.82125854492188,
      "activations/layer7_attention_weight_min": -96.65487670898438,
      "activations/layer8_attention_weight_max": 47.63113021850586,
      "activations/layer8_attention_weight_min": -45.99274826049805,
      "activations/layer9_attention_weight_max": 35.28042221069336,
      "activations/layer9_attention_weight_min": -36.18562698364258,
      "epoch": 15.21,
      "learning_rate": 5.2410227272727264e-05,
      "loss": 2.7403,
      "step": 261850
    },
    {
      "activations/layer0_attention_weight_max": 16.330320358276367,
      "activations/layer0_attention_weight_min": -12.35898494720459,
      "activations/layer10_attention_weight_max": 33.461570739746094,
      "activations/layer10_attention_weight_min": -33.651954650878906,
      "activations/layer11_attention_weight_max": 35.25897216796875,
      "activations/layer11_attention_weight_min": -33.64653778076172,
      "activations/layer12_attention_weight_max": 27.735862731933594,
      "activations/layer12_attention_weight_min": -26.81456184387207,
      "activations/layer13_attention_weight_max": 41.51626205444336,
      "activations/layer13_attention_weight_min": -33.775978088378906,
      "activations/layer14_attention_weight_max": 47.56050109863281,
      "activations/layer14_attention_weight_min": -37.58235168457031,
      "activations/layer15_attention_weight_max": 39.77605056762695,
      "activations/layer15_attention_weight_min": -33.97319030761719,
      "activations/layer16_attention_weight_max": 33.128170013427734,
      "activations/layer16_attention_weight_min": -29.18939208984375,
      "activations/layer17_attention_weight_max": 35.09871292114258,
      "activations/layer17_attention_weight_min": -27.087602615356445,
      "activations/layer18_attention_weight_max": 31.491262435913086,
      "activations/layer18_attention_weight_min": -26.036409378051758,
      "activations/layer19_attention_weight_max": 34.39665222167969,
      "activations/layer19_attention_weight_min": -30.014419555664062,
      "activations/layer1_attention_weight_max": 18.086891174316406,
      "activations/layer1_attention_weight_min": -17.981216430664062,
      "activations/layer20_attention_weight_max": 30.620710372924805,
      "activations/layer20_attention_weight_min": -24.626720428466797,
      "activations/layer21_attention_weight_max": 29.160442352294922,
      "activations/layer21_attention_weight_min": -23.772178649902344,
      "activations/layer22_attention_weight_max": 47.28098678588867,
      "activations/layer22_attention_weight_min": -33.88117218017578,
      "activations/layer23_attention_weight_max": 36.34916305541992,
      "activations/layer23_attention_weight_min": -23.716346740722656,
      "activations/layer2_attention_weight_max": 32.794673919677734,
      "activations/layer2_attention_weight_min": -34.37605285644531,
      "activations/layer3_attention_weight_max": 97.73577117919922,
      "activations/layer3_attention_weight_min": -103.1749496459961,
      "activations/layer4_attention_weight_max": 109.6983642578125,
      "activations/layer4_attention_weight_min": -110.26484680175781,
      "activations/layer5_attention_weight_max": 52.790863037109375,
      "activations/layer5_attention_weight_min": -62.46527099609375,
      "activations/layer6_attention_weight_max": 44.23289489746094,
      "activations/layer6_attention_weight_min": -46.65983581542969,
      "activations/layer7_attention_weight_max": 91.4041748046875,
      "activations/layer7_attention_weight_min": -94.06793212890625,
      "activations/layer8_attention_weight_max": 41.458553314208984,
      "activations/layer8_attention_weight_min": -42.86034393310547,
      "activations/layer9_attention_weight_max": 34.82478713989258,
      "activations/layer9_attention_weight_min": -34.77372360229492,
      "epoch": 15.22,
      "learning_rate": 5.239128787878788e-05,
      "loss": 2.7476,
      "step": 261900
    },
    {
      "activations/layer0_attention_weight_max": 15.990851402282715,
      "activations/layer0_attention_weight_min": -12.393997192382812,
      "activations/layer10_attention_weight_max": 32.09256362915039,
      "activations/layer10_attention_weight_min": -33.558937072753906,
      "activations/layer11_attention_weight_max": 33.990013122558594,
      "activations/layer11_attention_weight_min": -31.451345443725586,
      "activations/layer12_attention_weight_max": 26.946449279785156,
      "activations/layer12_attention_weight_min": -27.464313507080078,
      "activations/layer13_attention_weight_max": 41.95347213745117,
      "activations/layer13_attention_weight_min": -36.57870101928711,
      "activations/layer14_attention_weight_max": 40.88454818725586,
      "activations/layer14_attention_weight_min": -38.96757125854492,
      "activations/layer15_attention_weight_max": 37.25310134887695,
      "activations/layer15_attention_weight_min": -33.00308609008789,
      "activations/layer16_attention_weight_max": 26.83661460876465,
      "activations/layer16_attention_weight_min": -28.330814361572266,
      "activations/layer17_attention_weight_max": 28.45500373840332,
      "activations/layer17_attention_weight_min": -26.593687057495117,
      "activations/layer18_attention_weight_max": 30.02117347717285,
      "activations/layer18_attention_weight_min": -24.934131622314453,
      "activations/layer19_attention_weight_max": 31.98907470703125,
      "activations/layer19_attention_weight_min": -31.272947311401367,
      "activations/layer1_attention_weight_max": 15.92873764038086,
      "activations/layer1_attention_weight_min": -14.378340721130371,
      "activations/layer20_attention_weight_max": 27.77480125427246,
      "activations/layer20_attention_weight_min": -27.818180084228516,
      "activations/layer21_attention_weight_max": 27.79459571838379,
      "activations/layer21_attention_weight_min": -24.263200759887695,
      "activations/layer22_attention_weight_max": 38.32444381713867,
      "activations/layer22_attention_weight_min": -30.024385452270508,
      "activations/layer23_attention_weight_max": 30.664289474487305,
      "activations/layer23_attention_weight_min": -28.187301635742188,
      "activations/layer2_attention_weight_max": 31.260391235351562,
      "activations/layer2_attention_weight_min": -34.12519836425781,
      "activations/layer3_attention_weight_max": 95.07872772216797,
      "activations/layer3_attention_weight_min": -97.74163818359375,
      "activations/layer4_attention_weight_max": 106.9212875366211,
      "activations/layer4_attention_weight_min": -109.84297943115234,
      "activations/layer5_attention_weight_max": 51.04650115966797,
      "activations/layer5_attention_weight_min": -62.34178161621094,
      "activations/layer6_attention_weight_max": 42.85765838623047,
      "activations/layer6_attention_weight_min": -45.347999572753906,
      "activations/layer7_attention_weight_max": 90.74148559570312,
      "activations/layer7_attention_weight_min": -94.4375991821289,
      "activations/layer8_attention_weight_max": 40.20938491821289,
      "activations/layer8_attention_weight_min": -44.12841033935547,
      "activations/layer9_attention_weight_max": 31.552539825439453,
      "activations/layer9_attention_weight_min": -32.14048385620117,
      "epoch": 15.22,
      "learning_rate": 5.237234848484848e-05,
      "loss": 2.7526,
      "step": 261950
    },
    {
      "activations/layer0_attention_weight_max": 15.410425186157227,
      "activations/layer0_attention_weight_min": -11.762940406799316,
      "activations/layer10_attention_weight_max": 36.51197814941406,
      "activations/layer10_attention_weight_min": -34.48430252075195,
      "activations/layer11_attention_weight_max": 37.380836486816406,
      "activations/layer11_attention_weight_min": -36.76933288574219,
      "activations/layer12_attention_weight_max": 26.650175094604492,
      "activations/layer12_attention_weight_min": -27.177507400512695,
      "activations/layer13_attention_weight_max": 39.907371520996094,
      "activations/layer13_attention_weight_min": -34.83616256713867,
      "activations/layer14_attention_weight_max": 40.40633773803711,
      "activations/layer14_attention_weight_min": -37.183223724365234,
      "activations/layer15_attention_weight_max": 41.69375991821289,
      "activations/layer15_attention_weight_min": -34.5438232421875,
      "activations/layer16_attention_weight_max": 29.916475296020508,
      "activations/layer16_attention_weight_min": -28.826162338256836,
      "activations/layer17_attention_weight_max": 33.04270935058594,
      "activations/layer17_attention_weight_min": -25.627235412597656,
      "activations/layer18_attention_weight_max": 30.777889251708984,
      "activations/layer18_attention_weight_min": -23.613170623779297,
      "activations/layer19_attention_weight_max": 34.13703536987305,
      "activations/layer19_attention_weight_min": -31.79575538635254,
      "activations/layer1_attention_weight_max": 16.7499942779541,
      "activations/layer1_attention_weight_min": -16.013418197631836,
      "activations/layer20_attention_weight_max": 30.45421028137207,
      "activations/layer20_attention_weight_min": -24.180583953857422,
      "activations/layer21_attention_weight_max": 29.722402572631836,
      "activations/layer21_attention_weight_min": -23.484996795654297,
      "activations/layer22_attention_weight_max": 41.321006774902344,
      "activations/layer22_attention_weight_min": -30.27192497253418,
      "activations/layer23_attention_weight_max": 33.08952331542969,
      "activations/layer23_attention_weight_min": -23.963443756103516,
      "activations/layer2_attention_weight_max": 35.678077697753906,
      "activations/layer2_attention_weight_min": -34.19596862792969,
      "activations/layer3_attention_weight_max": 98.72576904296875,
      "activations/layer3_attention_weight_min": -102.3566665649414,
      "activations/layer4_attention_weight_max": 105.90803527832031,
      "activations/layer4_attention_weight_min": -113.98097229003906,
      "activations/layer5_attention_weight_max": 53.238555908203125,
      "activations/layer5_attention_weight_min": -63.51948547363281,
      "activations/layer6_attention_weight_max": 42.81163024902344,
      "activations/layer6_attention_weight_min": -45.663028717041016,
      "activations/layer7_attention_weight_max": 96.9036636352539,
      "activations/layer7_attention_weight_min": -94.53634643554688,
      "activations/layer8_attention_weight_max": 42.37861633300781,
      "activations/layer8_attention_weight_min": -42.38077163696289,
      "activations/layer9_attention_weight_max": 32.717288970947266,
      "activations/layer9_attention_weight_min": -33.887481689453125,
      "epoch": 15.22,
      "learning_rate": 5.235340909090908e-05,
      "loss": 2.7492,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.5549,
      "eval_samples_per_second": 501.937,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.5549,
      "eval_openwebtext_samples_per_second": 501.937,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 2.0202,
      "eval_wikitext_samples_per_second": 225.722,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_lambada_loss": 2.44921875,
      "eval_lambada_ppl": 11.57929685891098,
      "eval_lambada_runtime": 9.5722,
      "eval_lambada_samples_per_second": 508.658,
      "step": 262000
    },
    {
      "activations/layer0_attention_weight_max": 16.744552612304688,
      "activations/layer0_attention_weight_min": -11.538093566894531,
      "activations/layer10_attention_weight_max": 31.83598518371582,
      "activations/layer10_attention_weight_min": -33.40613555908203,
      "activations/layer11_attention_weight_max": 32.29582214355469,
      "activations/layer11_attention_weight_min": -32.65406799316406,
      "activations/layer12_attention_weight_max": 25.858861923217773,
      "activations/layer12_attention_weight_min": -26.19309425354004,
      "activations/layer13_attention_weight_max": 39.65907669067383,
      "activations/layer13_attention_weight_min": -34.96302032470703,
      "activations/layer14_attention_weight_max": 44.63711929321289,
      "activations/layer14_attention_weight_min": -36.14191818237305,
      "activations/layer15_attention_weight_max": 40.72383499145508,
      "activations/layer15_attention_weight_min": -33.77526092529297,
      "activations/layer16_attention_weight_max": 29.88986587524414,
      "activations/layer16_attention_weight_min": -27.40727424621582,
      "activations/layer17_attention_weight_max": 29.75507926940918,
      "activations/layer17_attention_weight_min": -25.56194496154785,
      "activations/layer18_attention_weight_max": 30.138620376586914,
      "activations/layer18_attention_weight_min": -23.75405502319336,
      "activations/layer19_attention_weight_max": 31.027847290039062,
      "activations/layer19_attention_weight_min": -30.617094039916992,
      "activations/layer1_attention_weight_max": 17.173471450805664,
      "activations/layer1_attention_weight_min": -14.338891983032227,
      "activations/layer20_attention_weight_max": 29.520008087158203,
      "activations/layer20_attention_weight_min": -25.73720359802246,
      "activations/layer21_attention_weight_max": 29.471500396728516,
      "activations/layer21_attention_weight_min": -24.110071182250977,
      "activations/layer22_attention_weight_max": 40.69304275512695,
      "activations/layer22_attention_weight_min": -28.99884796142578,
      "activations/layer23_attention_weight_max": 38.06304168701172,
      "activations/layer23_attention_weight_min": -25.72859764099121,
      "activations/layer2_attention_weight_max": 32.59367370605469,
      "activations/layer2_attention_weight_min": -33.28601837158203,
      "activations/layer3_attention_weight_max": 95.83975982666016,
      "activations/layer3_attention_weight_min": -97.9039077758789,
      "activations/layer4_attention_weight_max": 108.37694549560547,
      "activations/layer4_attention_weight_min": -110.0103988647461,
      "activations/layer5_attention_weight_max": 52.016258239746094,
      "activations/layer5_attention_weight_min": -60.53448486328125,
      "activations/layer6_attention_weight_max": 41.600914001464844,
      "activations/layer6_attention_weight_min": -44.53821563720703,
      "activations/layer7_attention_weight_max": 87.68927764892578,
      "activations/layer7_attention_weight_min": -90.78172302246094,
      "activations/layer8_attention_weight_max": 40.90107345581055,
      "activations/layer8_attention_weight_min": -42.27700424194336,
      "activations/layer9_attention_weight_max": 32.515254974365234,
      "activations/layer9_attention_weight_min": -33.138423919677734,
      "epoch": 15.23,
      "learning_rate": 5.233446969696969e-05,
      "loss": 2.7372,
      "step": 262050
    },
    {
      "activations/layer0_attention_weight_max": 15.432967185974121,
      "activations/layer0_attention_weight_min": -12.577168464660645,
      "activations/layer10_attention_weight_max": 36.971900939941406,
      "activations/layer10_attention_weight_min": -35.72916793823242,
      "activations/layer11_attention_weight_max": 37.652679443359375,
      "activations/layer11_attention_weight_min": -36.255863189697266,
      "activations/layer12_attention_weight_max": 28.380571365356445,
      "activations/layer12_attention_weight_min": -25.4760799407959,
      "activations/layer13_attention_weight_max": 38.2738151550293,
      "activations/layer13_attention_weight_min": -33.056644439697266,
      "activations/layer14_attention_weight_max": 39.285221099853516,
      "activations/layer14_attention_weight_min": -35.23074722290039,
      "activations/layer15_attention_weight_max": 40.03652572631836,
      "activations/layer15_attention_weight_min": -33.23998260498047,
      "activations/layer16_attention_weight_max": 32.77874755859375,
      "activations/layer16_attention_weight_min": -26.894119262695312,
      "activations/layer17_attention_weight_max": 30.822004318237305,
      "activations/layer17_attention_weight_min": -24.647964477539062,
      "activations/layer18_attention_weight_max": 31.14468002319336,
      "activations/layer18_attention_weight_min": -25.1063289642334,
      "activations/layer19_attention_weight_max": 36.181148529052734,
      "activations/layer19_attention_weight_min": -29.90570640563965,
      "activations/layer1_attention_weight_max": 16.22854232788086,
      "activations/layer1_attention_weight_min": -13.674909591674805,
      "activations/layer20_attention_weight_max": 32.80877685546875,
      "activations/layer20_attention_weight_min": -25.693077087402344,
      "activations/layer21_attention_weight_max": 32.209049224853516,
      "activations/layer21_attention_weight_min": -24.03624725341797,
      "activations/layer22_attention_weight_max": 46.89043045043945,
      "activations/layer22_attention_weight_min": -30.73169708251953,
      "activations/layer23_attention_weight_max": 35.66631317138672,
      "activations/layer23_attention_weight_min": -23.756093978881836,
      "activations/layer2_attention_weight_max": 33.59370422363281,
      "activations/layer2_attention_weight_min": -31.982467651367188,
      "activations/layer3_attention_weight_max": 94.27371215820312,
      "activations/layer3_attention_weight_min": -92.59590148925781,
      "activations/layer4_attention_weight_max": 105.66558837890625,
      "activations/layer4_attention_weight_min": -101.4607925415039,
      "activations/layer5_attention_weight_max": 49.878997802734375,
      "activations/layer5_attention_weight_min": -60.236595153808594,
      "activations/layer6_attention_weight_max": 42.394927978515625,
      "activations/layer6_attention_weight_min": -45.44629669189453,
      "activations/layer7_attention_weight_max": 92.92499542236328,
      "activations/layer7_attention_weight_min": -85.8880844116211,
      "activations/layer8_attention_weight_max": 41.24214172363281,
      "activations/layer8_attention_weight_min": -40.086891174316406,
      "activations/layer9_attention_weight_max": 34.82508087158203,
      "activations/layer9_attention_weight_min": -36.20659637451172,
      "epoch": 15.23,
      "learning_rate": 5.23155303030303e-05,
      "loss": 2.7514,
      "step": 262100
    },
    {
      "activations/layer0_attention_weight_max": 16.692190170288086,
      "activations/layer0_attention_weight_min": -12.181879043579102,
      "activations/layer10_attention_weight_max": 32.60995864868164,
      "activations/layer10_attention_weight_min": -34.80812454223633,
      "activations/layer11_attention_weight_max": 33.13337707519531,
      "activations/layer11_attention_weight_min": -32.292083740234375,
      "activations/layer12_attention_weight_max": 27.94977378845215,
      "activations/layer12_attention_weight_min": -28.301761627197266,
      "activations/layer13_attention_weight_max": 43.258358001708984,
      "activations/layer13_attention_weight_min": -38.64818572998047,
      "activations/layer14_attention_weight_max": 46.55536651611328,
      "activations/layer14_attention_weight_min": -37.62833786010742,
      "activations/layer15_attention_weight_max": 42.378753662109375,
      "activations/layer15_attention_weight_min": -34.4654541015625,
      "activations/layer16_attention_weight_max": 32.70798110961914,
      "activations/layer16_attention_weight_min": -29.449853897094727,
      "activations/layer17_attention_weight_max": 33.897029876708984,
      "activations/layer17_attention_weight_min": -26.441837310791016,
      "activations/layer18_attention_weight_max": 35.72342300415039,
      "activations/layer18_attention_weight_min": -26.206457138061523,
      "activations/layer19_attention_weight_max": 39.45231246948242,
      "activations/layer19_attention_weight_min": -30.318695068359375,
      "activations/layer1_attention_weight_max": 16.584735870361328,
      "activations/layer1_attention_weight_min": -15.7559175491333,
      "activations/layer20_attention_weight_max": 32.76761245727539,
      "activations/layer20_attention_weight_min": -25.504545211791992,
      "activations/layer21_attention_weight_max": 32.974098205566406,
      "activations/layer21_attention_weight_min": -24.416900634765625,
      "activations/layer22_attention_weight_max": 52.54201889038086,
      "activations/layer22_attention_weight_min": -30.353031158447266,
      "activations/layer23_attention_weight_max": 34.621700286865234,
      "activations/layer23_attention_weight_min": -27.084049224853516,
      "activations/layer2_attention_weight_max": 33.71348571777344,
      "activations/layer2_attention_weight_min": -33.007904052734375,
      "activations/layer3_attention_weight_max": 96.4208755493164,
      "activations/layer3_attention_weight_min": -96.59362030029297,
      "activations/layer4_attention_weight_max": 107.67005920410156,
      "activations/layer4_attention_weight_min": -109.3814468383789,
      "activations/layer5_attention_weight_max": 51.56426239013672,
      "activations/layer5_attention_weight_min": -63.64838790893555,
      "activations/layer6_attention_weight_max": 43.88837432861328,
      "activations/layer6_attention_weight_min": -47.26001739501953,
      "activations/layer7_attention_weight_max": 89.39757537841797,
      "activations/layer7_attention_weight_min": -96.71475982666016,
      "activations/layer8_attention_weight_max": 41.37635040283203,
      "activations/layer8_attention_weight_min": -43.94404983520508,
      "activations/layer9_attention_weight_max": 32.81047821044922,
      "activations/layer9_attention_weight_min": -36.37411117553711,
      "epoch": 15.23,
      "learning_rate": 5.229659090909091e-05,
      "loss": 2.7507,
      "step": 262150
    },
    {
      "activations/layer0_attention_weight_max": 16.928356170654297,
      "activations/layer0_attention_weight_min": -13.146405220031738,
      "activations/layer10_attention_weight_max": 32.29409408569336,
      "activations/layer10_attention_weight_min": -33.293548583984375,
      "activations/layer11_attention_weight_max": 33.59703826904297,
      "activations/layer11_attention_weight_min": -32.84025192260742,
      "activations/layer12_attention_weight_max": 26.81597328186035,
      "activations/layer12_attention_weight_min": -27.63455581665039,
      "activations/layer13_attention_weight_max": 38.4555549621582,
      "activations/layer13_attention_weight_min": -35.634239196777344,
      "activations/layer14_attention_weight_max": 45.3093147277832,
      "activations/layer14_attention_weight_min": -37.924949645996094,
      "activations/layer15_attention_weight_max": 36.7745475769043,
      "activations/layer15_attention_weight_min": -36.19578170776367,
      "activations/layer16_attention_weight_max": 33.50274658203125,
      "activations/layer16_attention_weight_min": -28.113216400146484,
      "activations/layer17_attention_weight_max": 30.584253311157227,
      "activations/layer17_attention_weight_min": -25.122533798217773,
      "activations/layer18_attention_weight_max": 28.721603393554688,
      "activations/layer18_attention_weight_min": -23.967689514160156,
      "activations/layer19_attention_weight_max": 36.27652359008789,
      "activations/layer19_attention_weight_min": -31.8015193939209,
      "activations/layer1_attention_weight_max": 16.32551383972168,
      "activations/layer1_attention_weight_min": -15.383557319641113,
      "activations/layer20_attention_weight_max": 28.088205337524414,
      "activations/layer20_attention_weight_min": -26.25047492980957,
      "activations/layer21_attention_weight_max": 27.132543563842773,
      "activations/layer21_attention_weight_min": -25.09703254699707,
      "activations/layer22_attention_weight_max": 39.310794830322266,
      "activations/layer22_attention_weight_min": -31.91536521911621,
      "activations/layer23_attention_weight_max": 30.294471740722656,
      "activations/layer23_attention_weight_min": -26.265342712402344,
      "activations/layer2_attention_weight_max": 32.93259811401367,
      "activations/layer2_attention_weight_min": -32.751522064208984,
      "activations/layer3_attention_weight_max": 96.09352111816406,
      "activations/layer3_attention_weight_min": -97.66400909423828,
      "activations/layer4_attention_weight_max": 107.62459564208984,
      "activations/layer4_attention_weight_min": -110.59852600097656,
      "activations/layer5_attention_weight_max": 50.44775390625,
      "activations/layer5_attention_weight_min": -63.70616149902344,
      "activations/layer6_attention_weight_max": 44.59486770629883,
      "activations/layer6_attention_weight_min": -46.555355072021484,
      "activations/layer7_attention_weight_max": 90.31903839111328,
      "activations/layer7_attention_weight_min": -95.8214340209961,
      "activations/layer8_attention_weight_max": 39.39751052856445,
      "activations/layer8_attention_weight_min": -42.93202590942383,
      "activations/layer9_attention_weight_max": 30.641143798828125,
      "activations/layer9_attention_weight_min": -35.16757583618164,
      "epoch": 15.24,
      "learning_rate": 5.227765151515151e-05,
      "loss": 2.7602,
      "step": 262200
    },
    {
      "activations/layer0_attention_weight_max": 15.002568244934082,
      "activations/layer0_attention_weight_min": -13.140875816345215,
      "activations/layer10_attention_weight_max": 35.10107421875,
      "activations/layer10_attention_weight_min": -35.337158203125,
      "activations/layer11_attention_weight_max": 33.409488677978516,
      "activations/layer11_attention_weight_min": -33.82920455932617,
      "activations/layer12_attention_weight_max": 32.938045501708984,
      "activations/layer12_attention_weight_min": -26.48814582824707,
      "activations/layer13_attention_weight_max": 37.70469665527344,
      "activations/layer13_attention_weight_min": -35.555213928222656,
      "activations/layer14_attention_weight_max": 44.2570915222168,
      "activations/layer14_attention_weight_min": -36.919368743896484,
      "activations/layer15_attention_weight_max": 35.938743591308594,
      "activations/layer15_attention_weight_min": -34.40718460083008,
      "activations/layer16_attention_weight_max": 30.655288696289062,
      "activations/layer16_attention_weight_min": -27.744647979736328,
      "activations/layer17_attention_weight_max": 32.130001068115234,
      "activations/layer17_attention_weight_min": -24.920564651489258,
      "activations/layer18_attention_weight_max": 30.610065460205078,
      "activations/layer18_attention_weight_min": -22.770870208740234,
      "activations/layer19_attention_weight_max": 37.69313049316406,
      "activations/layer19_attention_weight_min": -32.77073669433594,
      "activations/layer1_attention_weight_max": 16.926130294799805,
      "activations/layer1_attention_weight_min": -16.972000122070312,
      "activations/layer20_attention_weight_max": 33.72934341430664,
      "activations/layer20_attention_weight_min": -26.97026824951172,
      "activations/layer21_attention_weight_max": 34.139686584472656,
      "activations/layer21_attention_weight_min": -26.510635375976562,
      "activations/layer22_attention_weight_max": 44.90749740600586,
      "activations/layer22_attention_weight_min": -30.898313522338867,
      "activations/layer23_attention_weight_max": 37.006439208984375,
      "activations/layer23_attention_weight_min": -24.19393539428711,
      "activations/layer2_attention_weight_max": 32.4732780456543,
      "activations/layer2_attention_weight_min": -31.941211700439453,
      "activations/layer3_attention_weight_max": 97.85343933105469,
      "activations/layer3_attention_weight_min": -96.885009765625,
      "activations/layer4_attention_weight_max": 109.18926239013672,
      "activations/layer4_attention_weight_min": -107.90016174316406,
      "activations/layer5_attention_weight_max": 50.480140686035156,
      "activations/layer5_attention_weight_min": -60.05967712402344,
      "activations/layer6_attention_weight_max": 44.48024368286133,
      "activations/layer6_attention_weight_min": -49.18547058105469,
      "activations/layer7_attention_weight_max": 92.22113037109375,
      "activations/layer7_attention_weight_min": -96.3869857788086,
      "activations/layer8_attention_weight_max": 41.61128616333008,
      "activations/layer8_attention_weight_min": -41.92587661743164,
      "activations/layer9_attention_weight_max": 32.169029235839844,
      "activations/layer9_attention_weight_min": -32.855228424072266,
      "epoch": 15.24,
      "learning_rate": 5.2258712121212123e-05,
      "loss": 2.7502,
      "step": 262250
    },
    {
      "activations/layer0_attention_weight_max": 16.785114288330078,
      "activations/layer0_attention_weight_min": -12.0136137008667,
      "activations/layer10_attention_weight_max": 32.101341247558594,
      "activations/layer10_attention_weight_min": -31.628751754760742,
      "activations/layer11_attention_weight_max": 30.556264877319336,
      "activations/layer11_attention_weight_min": -31.979389190673828,
      "activations/layer12_attention_weight_max": 24.45413589477539,
      "activations/layer12_attention_weight_min": -25.85671615600586,
      "activations/layer13_attention_weight_max": 36.77093505859375,
      "activations/layer13_attention_weight_min": -33.82764434814453,
      "activations/layer14_attention_weight_max": 41.28927993774414,
      "activations/layer14_attention_weight_min": -38.53530502319336,
      "activations/layer15_attention_weight_max": 37.61955642700195,
      "activations/layer15_attention_weight_min": -34.22670364379883,
      "activations/layer16_attention_weight_max": 31.896438598632812,
      "activations/layer16_attention_weight_min": -28.38291358947754,
      "activations/layer17_attention_weight_max": 31.23879051208496,
      "activations/layer17_attention_weight_min": -29.44989776611328,
      "activations/layer18_attention_weight_max": 36.220726013183594,
      "activations/layer18_attention_weight_min": -27.556461334228516,
      "activations/layer19_attention_weight_max": 39.48357009887695,
      "activations/layer19_attention_weight_min": -32.635765075683594,
      "activations/layer1_attention_weight_max": 15.775609016418457,
      "activations/layer1_attention_weight_min": -12.675088882446289,
      "activations/layer20_attention_weight_max": 33.32651901245117,
      "activations/layer20_attention_weight_min": -27.460132598876953,
      "activations/layer21_attention_weight_max": 30.799644470214844,
      "activations/layer21_attention_weight_min": -25.488759994506836,
      "activations/layer22_attention_weight_max": 46.19500732421875,
      "activations/layer22_attention_weight_min": -34.53364181518555,
      "activations/layer23_attention_weight_max": 32.66407775878906,
      "activations/layer23_attention_weight_min": -26.917104721069336,
      "activations/layer2_attention_weight_max": 33.36469650268555,
      "activations/layer2_attention_weight_min": -33.88219451904297,
      "activations/layer3_attention_weight_max": 96.39006042480469,
      "activations/layer3_attention_weight_min": -99.18260192871094,
      "activations/layer4_attention_weight_max": 109.3897705078125,
      "activations/layer4_attention_weight_min": -111.30123138427734,
      "activations/layer5_attention_weight_max": 53.58648681640625,
      "activations/layer5_attention_weight_min": -66.1912612915039,
      "activations/layer6_attention_weight_max": 42.02783203125,
      "activations/layer6_attention_weight_min": -46.55119705200195,
      "activations/layer7_attention_weight_max": 88.53023529052734,
      "activations/layer7_attention_weight_min": -90.86375427246094,
      "activations/layer8_attention_weight_max": 41.09719467163086,
      "activations/layer8_attention_weight_min": -42.12941360473633,
      "activations/layer9_attention_weight_max": 30.496368408203125,
      "activations/layer9_attention_weight_min": -34.08906173706055,
      "epoch": 15.24,
      "learning_rate": 5.2239772727272725e-05,
      "loss": 2.7438,
      "step": 262300
    },
    {
      "activations/layer0_attention_weight_max": 16.37615394592285,
      "activations/layer0_attention_weight_min": -13.258207321166992,
      "activations/layer10_attention_weight_max": 36.646629333496094,
      "activations/layer10_attention_weight_min": -35.724342346191406,
      "activations/layer11_attention_weight_max": 34.99270248413086,
      "activations/layer11_attention_weight_min": -34.828956604003906,
      "activations/layer12_attention_weight_max": 28.743986129760742,
      "activations/layer12_attention_weight_min": -28.222745895385742,
      "activations/layer13_attention_weight_max": 41.2495231628418,
      "activations/layer13_attention_weight_min": -37.48210906982422,
      "activations/layer14_attention_weight_max": 44.43132019042969,
      "activations/layer14_attention_weight_min": -38.85698699951172,
      "activations/layer15_attention_weight_max": 37.811397552490234,
      "activations/layer15_attention_weight_min": -34.35245132446289,
      "activations/layer16_attention_weight_max": 30.680923461914062,
      "activations/layer16_attention_weight_min": -28.201309204101562,
      "activations/layer17_attention_weight_max": 29.74884033203125,
      "activations/layer17_attention_weight_min": -25.251989364624023,
      "activations/layer18_attention_weight_max": 38.28075408935547,
      "activations/layer18_attention_weight_min": -25.699974060058594,
      "activations/layer19_attention_weight_max": 33.06891632080078,
      "activations/layer19_attention_weight_min": -30.28879165649414,
      "activations/layer1_attention_weight_max": 16.028593063354492,
      "activations/layer1_attention_weight_min": -14.647651672363281,
      "activations/layer20_attention_weight_max": 29.736328125,
      "activations/layer20_attention_weight_min": -25.859622955322266,
      "activations/layer21_attention_weight_max": 28.70620346069336,
      "activations/layer21_attention_weight_min": -26.34872817993164,
      "activations/layer22_attention_weight_max": 45.9205322265625,
      "activations/layer22_attention_weight_min": -31.014387130737305,
      "activations/layer23_attention_weight_max": 31.983585357666016,
      "activations/layer23_attention_weight_min": -24.92812156677246,
      "activations/layer2_attention_weight_max": 33.02254867553711,
      "activations/layer2_attention_weight_min": -34.92837142944336,
      "activations/layer3_attention_weight_max": 100.7045669555664,
      "activations/layer3_attention_weight_min": -104.8503646850586,
      "activations/layer4_attention_weight_max": 114.3672103881836,
      "activations/layer4_attention_weight_min": -117.23503875732422,
      "activations/layer5_attention_weight_max": 53.302894592285156,
      "activations/layer5_attention_weight_min": -60.22779846191406,
      "activations/layer6_attention_weight_max": 46.46908187866211,
      "activations/layer6_attention_weight_min": -48.849510192871094,
      "activations/layer7_attention_weight_max": 104.72579193115234,
      "activations/layer7_attention_weight_min": -101.98439025878906,
      "activations/layer8_attention_weight_max": 45.275936126708984,
      "activations/layer8_attention_weight_min": -45.61724090576172,
      "activations/layer9_attention_weight_max": 35.76708984375,
      "activations/layer9_attention_weight_min": -37.64505386352539,
      "epoch": 15.24,
      "learning_rate": 5.2220833333333326e-05,
      "loss": 2.7427,
      "step": 262350
    },
    {
      "activations/layer0_attention_weight_max": 15.512555122375488,
      "activations/layer0_attention_weight_min": -11.985884666442871,
      "activations/layer10_attention_weight_max": 33.392181396484375,
      "activations/layer10_attention_weight_min": -32.80994415283203,
      "activations/layer11_attention_weight_max": 32.937652587890625,
      "activations/layer11_attention_weight_min": -32.20731735229492,
      "activations/layer12_attention_weight_max": 26.359634399414062,
      "activations/layer12_attention_weight_min": -26.303556442260742,
      "activations/layer13_attention_weight_max": 37.74359893798828,
      "activations/layer13_attention_weight_min": -35.56956481933594,
      "activations/layer14_attention_weight_max": 43.9642333984375,
      "activations/layer14_attention_weight_min": -38.729366302490234,
      "activations/layer15_attention_weight_max": 37.87247085571289,
      "activations/layer15_attention_weight_min": -34.86579895019531,
      "activations/layer16_attention_weight_max": 30.98511505126953,
      "activations/layer16_attention_weight_min": -33.56891632080078,
      "activations/layer17_attention_weight_max": 32.006046295166016,
      "activations/layer17_attention_weight_min": -26.885881423950195,
      "activations/layer18_attention_weight_max": 34.1702995300293,
      "activations/layer18_attention_weight_min": -26.575607299804688,
      "activations/layer19_attention_weight_max": 34.30826950073242,
      "activations/layer19_attention_weight_min": -33.58290481567383,
      "activations/layer1_attention_weight_max": 16.536392211914062,
      "activations/layer1_attention_weight_min": -15.73084545135498,
      "activations/layer20_attention_weight_max": 31.871047973632812,
      "activations/layer20_attention_weight_min": -27.296783447265625,
      "activations/layer21_attention_weight_max": 29.998979568481445,
      "activations/layer21_attention_weight_min": -25.75762939453125,
      "activations/layer22_attention_weight_max": 46.008460998535156,
      "activations/layer22_attention_weight_min": -31.15650177001953,
      "activations/layer23_attention_weight_max": 32.7480583190918,
      "activations/layer23_attention_weight_min": -26.168371200561523,
      "activations/layer2_attention_weight_max": 33.47393798828125,
      "activations/layer2_attention_weight_min": -33.261505126953125,
      "activations/layer3_attention_weight_max": 98.22628784179688,
      "activations/layer3_attention_weight_min": -98.0156478881836,
      "activations/layer4_attention_weight_max": 111.27168273925781,
      "activations/layer4_attention_weight_min": -111.89884185791016,
      "activations/layer5_attention_weight_max": 52.11933517456055,
      "activations/layer5_attention_weight_min": -65.4695816040039,
      "activations/layer6_attention_weight_max": 45.426639556884766,
      "activations/layer6_attention_weight_min": -48.17311477661133,
      "activations/layer7_attention_weight_max": 94.14669799804688,
      "activations/layer7_attention_weight_min": -91.69981384277344,
      "activations/layer8_attention_weight_max": 45.061710357666016,
      "activations/layer8_attention_weight_min": -47.98891067504883,
      "activations/layer9_attention_weight_max": 35.774662017822266,
      "activations/layer9_attention_weight_min": -37.72014617919922,
      "epoch": 15.25,
      "learning_rate": 5.2201893939393935e-05,
      "loss": 2.7462,
      "step": 262400
    },
    {
      "activations/layer0_attention_weight_max": 15.5022611618042,
      "activations/layer0_attention_weight_min": -12.584671020507812,
      "activations/layer10_attention_weight_max": 32.53455352783203,
      "activations/layer10_attention_weight_min": -33.92644119262695,
      "activations/layer11_attention_weight_max": 34.498233795166016,
      "activations/layer11_attention_weight_min": -33.68168258666992,
      "activations/layer12_attention_weight_max": 29.730697631835938,
      "activations/layer12_attention_weight_min": -27.253395080566406,
      "activations/layer13_attention_weight_max": 41.34056091308594,
      "activations/layer13_attention_weight_min": -37.585693359375,
      "activations/layer14_attention_weight_max": 43.75152587890625,
      "activations/layer14_attention_weight_min": -39.75285339355469,
      "activations/layer15_attention_weight_max": 40.27147674560547,
      "activations/layer15_attention_weight_min": -37.308406829833984,
      "activations/layer16_attention_weight_max": 31.68931007385254,
      "activations/layer16_attention_weight_min": -31.275508880615234,
      "activations/layer17_attention_weight_max": 32.77083969116211,
      "activations/layer17_attention_weight_min": -27.449459075927734,
      "activations/layer18_attention_weight_max": 30.981491088867188,
      "activations/layer18_attention_weight_min": -25.582311630249023,
      "activations/layer19_attention_weight_max": 37.138427734375,
      "activations/layer19_attention_weight_min": -32.60676956176758,
      "activations/layer1_attention_weight_max": 19.351360321044922,
      "activations/layer1_attention_weight_min": -15.022777557373047,
      "activations/layer20_attention_weight_max": 32.445308685302734,
      "activations/layer20_attention_weight_min": -25.662525177001953,
      "activations/layer21_attention_weight_max": 31.676206588745117,
      "activations/layer21_attention_weight_min": -24.062393188476562,
      "activations/layer22_attention_weight_max": 47.86149978637695,
      "activations/layer22_attention_weight_min": -32.45059585571289,
      "activations/layer23_attention_weight_max": 38.52345657348633,
      "activations/layer23_attention_weight_min": -26.71881866455078,
      "activations/layer2_attention_weight_max": 33.836822509765625,
      "activations/layer2_attention_weight_min": -31.196853637695312,
      "activations/layer3_attention_weight_max": 93.24954986572266,
      "activations/layer3_attention_weight_min": -96.12783813476562,
      "activations/layer4_attention_weight_max": 107.05081939697266,
      "activations/layer4_attention_weight_min": -114.55073547363281,
      "activations/layer5_attention_weight_max": 54.13423156738281,
      "activations/layer5_attention_weight_min": -65.02015686035156,
      "activations/layer6_attention_weight_max": 44.02123260498047,
      "activations/layer6_attention_weight_min": -44.26395797729492,
      "activations/layer7_attention_weight_max": 99.37918853759766,
      "activations/layer7_attention_weight_min": -96.56574249267578,
      "activations/layer8_attention_weight_max": 41.588783264160156,
      "activations/layer8_attention_weight_min": -41.679222106933594,
      "activations/layer9_attention_weight_max": 31.618392944335938,
      "activations/layer9_attention_weight_min": -34.08024215698242,
      "epoch": 15.25,
      "learning_rate": 5.218295454545454e-05,
      "loss": 2.7514,
      "step": 262450
    },
    {
      "activations/layer0_attention_weight_max": 16.16313934326172,
      "activations/layer0_attention_weight_min": -11.86713981628418,
      "activations/layer10_attention_weight_max": 34.83424758911133,
      "activations/layer10_attention_weight_min": -35.63518524169922,
      "activations/layer11_attention_weight_max": 34.41924285888672,
      "activations/layer11_attention_weight_min": -35.95905303955078,
      "activations/layer12_attention_weight_max": 26.531787872314453,
      "activations/layer12_attention_weight_min": -25.472864151000977,
      "activations/layer13_attention_weight_max": 38.01403045654297,
      "activations/layer13_attention_weight_min": -34.317543029785156,
      "activations/layer14_attention_weight_max": 44.37028884887695,
      "activations/layer14_attention_weight_min": -38.08620834350586,
      "activations/layer15_attention_weight_max": 40.395484924316406,
      "activations/layer15_attention_weight_min": -34.40081787109375,
      "activations/layer16_attention_weight_max": 28.467998504638672,
      "activations/layer16_attention_weight_min": -28.251644134521484,
      "activations/layer17_attention_weight_max": 30.05509376525879,
      "activations/layer17_attention_weight_min": -25.616741180419922,
      "activations/layer18_attention_weight_max": 29.829593658447266,
      "activations/layer18_attention_weight_min": -23.555822372436523,
      "activations/layer19_attention_weight_max": 33.541534423828125,
      "activations/layer19_attention_weight_min": -30.274585723876953,
      "activations/layer1_attention_weight_max": 16.592069625854492,
      "activations/layer1_attention_weight_min": -16.34037208557129,
      "activations/layer20_attention_weight_max": 29.64360237121582,
      "activations/layer20_attention_weight_min": -25.75362205505371,
      "activations/layer21_attention_weight_max": 29.5157470703125,
      "activations/layer21_attention_weight_min": -24.9202938079834,
      "activations/layer22_attention_weight_max": 42.352745056152344,
      "activations/layer22_attention_weight_min": -31.327573776245117,
      "activations/layer23_attention_weight_max": 36.096923828125,
      "activations/layer23_attention_weight_min": -25.33655548095703,
      "activations/layer2_attention_weight_max": 32.84161376953125,
      "activations/layer2_attention_weight_min": -33.13881301879883,
      "activations/layer3_attention_weight_max": 97.84915161132812,
      "activations/layer3_attention_weight_min": -98.90130615234375,
      "activations/layer4_attention_weight_max": 109.2281494140625,
      "activations/layer4_attention_weight_min": -111.00817108154297,
      "activations/layer5_attention_weight_max": 50.12009811401367,
      "activations/layer5_attention_weight_min": -60.10593795776367,
      "activations/layer6_attention_weight_max": 45.043148040771484,
      "activations/layer6_attention_weight_min": -45.98558044433594,
      "activations/layer7_attention_weight_max": 91.06574249267578,
      "activations/layer7_attention_weight_min": -97.19691467285156,
      "activations/layer8_attention_weight_max": 41.15340805053711,
      "activations/layer8_attention_weight_min": -43.46195983886719,
      "activations/layer9_attention_weight_max": 34.60930252075195,
      "activations/layer9_attention_weight_min": -35.624629974365234,
      "epoch": 15.25,
      "learning_rate": 5.216401515151515e-05,
      "loss": 2.742,
      "step": 262500
    },
    {
      "activations/layer0_attention_weight_max": 15.63523006439209,
      "activations/layer0_attention_weight_min": -12.4070405960083,
      "activations/layer10_attention_weight_max": 33.493934631347656,
      "activations/layer10_attention_weight_min": -32.812538146972656,
      "activations/layer11_attention_weight_max": 33.390804290771484,
      "activations/layer11_attention_weight_min": -34.609806060791016,
      "activations/layer12_attention_weight_max": 27.04673194885254,
      "activations/layer12_attention_weight_min": -26.54926872253418,
      "activations/layer13_attention_weight_max": 40.82722473144531,
      "activations/layer13_attention_weight_min": -35.244171142578125,
      "activations/layer14_attention_weight_max": 45.71014404296875,
      "activations/layer14_attention_weight_min": -36.84431457519531,
      "activations/layer15_attention_weight_max": 38.61090087890625,
      "activations/layer15_attention_weight_min": -34.24774169921875,
      "activations/layer16_attention_weight_max": 31.938220977783203,
      "activations/layer16_attention_weight_min": -26.817358016967773,
      "activations/layer17_attention_weight_max": 32.49638748168945,
      "activations/layer17_attention_weight_min": -26.623422622680664,
      "activations/layer18_attention_weight_max": 37.88410186767578,
      "activations/layer18_attention_weight_min": -25.73305320739746,
      "activations/layer19_attention_weight_max": 39.85054397583008,
      "activations/layer19_attention_weight_min": -31.010496139526367,
      "activations/layer1_attention_weight_max": 15.64863395690918,
      "activations/layer1_attention_weight_min": -15.24994945526123,
      "activations/layer20_attention_weight_max": 35.17641830444336,
      "activations/layer20_attention_weight_min": -25.333595275878906,
      "activations/layer21_attention_weight_max": 35.58704376220703,
      "activations/layer21_attention_weight_min": -24.446367263793945,
      "activations/layer22_attention_weight_max": 57.427364349365234,
      "activations/layer22_attention_weight_min": -30.19191551208496,
      "activations/layer23_attention_weight_max": 40.621742248535156,
      "activations/layer23_attention_weight_min": -25.35489273071289,
      "activations/layer2_attention_weight_max": 32.93699264526367,
      "activations/layer2_attention_weight_min": -32.40755081176758,
      "activations/layer3_attention_weight_max": 95.91194915771484,
      "activations/layer3_attention_weight_min": -100.51624298095703,
      "activations/layer4_attention_weight_max": 106.86663055419922,
      "activations/layer4_attention_weight_min": -110.61756896972656,
      "activations/layer5_attention_weight_max": 50.45698165893555,
      "activations/layer5_attention_weight_min": -63.92344665527344,
      "activations/layer6_attention_weight_max": 42.15438461303711,
      "activations/layer6_attention_weight_min": -44.04798126220703,
      "activations/layer7_attention_weight_max": 87.49512481689453,
      "activations/layer7_attention_weight_min": -90.29498291015625,
      "activations/layer8_attention_weight_max": 41.2950325012207,
      "activations/layer8_attention_weight_min": -42.1758918762207,
      "activations/layer9_attention_weight_max": 33.17844009399414,
      "activations/layer9_attention_weight_min": -33.8758659362793,
      "epoch": 15.26,
      "learning_rate": 5.214507575757575e-05,
      "loss": 2.7508,
      "step": 262550
    },
    {
      "activations/layer0_attention_weight_max": 16.266111373901367,
      "activations/layer0_attention_weight_min": -11.788588523864746,
      "activations/layer10_attention_weight_max": 29.87447738647461,
      "activations/layer10_attention_weight_min": -33.0655517578125,
      "activations/layer11_attention_weight_max": 32.14380645751953,
      "activations/layer11_attention_weight_min": -31.769306182861328,
      "activations/layer12_attention_weight_max": 26.73961639404297,
      "activations/layer12_attention_weight_min": -25.23571014404297,
      "activations/layer13_attention_weight_max": 38.055381774902344,
      "activations/layer13_attention_weight_min": -34.26824188232422,
      "activations/layer14_attention_weight_max": 43.37461853027344,
      "activations/layer14_attention_weight_min": -38.20091247558594,
      "activations/layer15_attention_weight_max": 37.88652801513672,
      "activations/layer15_attention_weight_min": -34.63967514038086,
      "activations/layer16_attention_weight_max": 33.20386505126953,
      "activations/layer16_attention_weight_min": -27.43901252746582,
      "activations/layer17_attention_weight_max": 32.59505081176758,
      "activations/layer17_attention_weight_min": -26.99515724182129,
      "activations/layer18_attention_weight_max": 31.64655113220215,
      "activations/layer18_attention_weight_min": -25.82615089416504,
      "activations/layer19_attention_weight_max": 30.216718673706055,
      "activations/layer19_attention_weight_min": -30.695707321166992,
      "activations/layer1_attention_weight_max": 16.637662887573242,
      "activations/layer1_attention_weight_min": -15.077856063842773,
      "activations/layer20_attention_weight_max": 30.0218448638916,
      "activations/layer20_attention_weight_min": -26.881685256958008,
      "activations/layer21_attention_weight_max": 28.29576301574707,
      "activations/layer21_attention_weight_min": -27.154094696044922,
      "activations/layer22_attention_weight_max": 44.33884048461914,
      "activations/layer22_attention_weight_min": -31.090906143188477,
      "activations/layer23_attention_weight_max": 32.30152893066406,
      "activations/layer23_attention_weight_min": -25.80927276611328,
      "activations/layer2_attention_weight_max": 33.110557556152344,
      "activations/layer2_attention_weight_min": -30.999923706054688,
      "activations/layer3_attention_weight_max": 93.66806030273438,
      "activations/layer3_attention_weight_min": -93.52752685546875,
      "activations/layer4_attention_weight_max": 105.81156158447266,
      "activations/layer4_attention_weight_min": -108.3787612915039,
      "activations/layer5_attention_weight_max": 50.253257751464844,
      "activations/layer5_attention_weight_min": -66.97308349609375,
      "activations/layer6_attention_weight_max": 41.23234939575195,
      "activations/layer6_attention_weight_min": -45.456058502197266,
      "activations/layer7_attention_weight_max": 86.26168823242188,
      "activations/layer7_attention_weight_min": -90.14838409423828,
      "activations/layer8_attention_weight_max": 37.982940673828125,
      "activations/layer8_attention_weight_min": -40.03952407836914,
      "activations/layer9_attention_weight_max": 31.786117553710938,
      "activations/layer9_attention_weight_min": -32.097869873046875,
      "epoch": 15.26,
      "learning_rate": 5.2126136363636354e-05,
      "loss": 2.7544,
      "step": 262600
    },
    {
      "activations/layer0_attention_weight_max": 17.81188201904297,
      "activations/layer0_attention_weight_min": -12.422026634216309,
      "activations/layer10_attention_weight_max": 32.71402359008789,
      "activations/layer10_attention_weight_min": -33.37080764770508,
      "activations/layer11_attention_weight_max": 33.613929748535156,
      "activations/layer11_attention_weight_min": -34.15617370605469,
      "activations/layer12_attention_weight_max": 26.727991104125977,
      "activations/layer12_attention_weight_min": -25.464401245117188,
      "activations/layer13_attention_weight_max": 40.41809844970703,
      "activations/layer13_attention_weight_min": -34.24385452270508,
      "activations/layer14_attention_weight_max": 41.797874450683594,
      "activations/layer14_attention_weight_min": -37.23122787475586,
      "activations/layer15_attention_weight_max": 38.67185592651367,
      "activations/layer15_attention_weight_min": -34.4571533203125,
      "activations/layer16_attention_weight_max": 29.902973175048828,
      "activations/layer16_attention_weight_min": -27.13456153869629,
      "activations/layer17_attention_weight_max": 31.013160705566406,
      "activations/layer17_attention_weight_min": -26.161827087402344,
      "activations/layer18_attention_weight_max": 32.9131965637207,
      "activations/layer18_attention_weight_min": -23.48406410217285,
      "activations/layer19_attention_weight_max": 35.78960037231445,
      "activations/layer19_attention_weight_min": -30.453149795532227,
      "activations/layer1_attention_weight_max": 16.325435638427734,
      "activations/layer1_attention_weight_min": -13.929679870605469,
      "activations/layer20_attention_weight_max": 30.991470336914062,
      "activations/layer20_attention_weight_min": -25.277603149414062,
      "activations/layer21_attention_weight_max": 29.61867332458496,
      "activations/layer21_attention_weight_min": -25.581541061401367,
      "activations/layer22_attention_weight_max": 41.64065170288086,
      "activations/layer22_attention_weight_min": -30.89899253845215,
      "activations/layer23_attention_weight_max": 33.076759338378906,
      "activations/layer23_attention_weight_min": -25.353042602539062,
      "activations/layer2_attention_weight_max": 32.10624313354492,
      "activations/layer2_attention_weight_min": -31.470443725585938,
      "activations/layer3_attention_weight_max": 92.93385314941406,
      "activations/layer3_attention_weight_min": -97.50540924072266,
      "activations/layer4_attention_weight_max": 106.60560607910156,
      "activations/layer4_attention_weight_min": -109.79850006103516,
      "activations/layer5_attention_weight_max": 53.055931091308594,
      "activations/layer5_attention_weight_min": -59.88481903076172,
      "activations/layer6_attention_weight_max": 44.02375030517578,
      "activations/layer6_attention_weight_min": -45.444156646728516,
      "activations/layer7_attention_weight_max": 97.87026977539062,
      "activations/layer7_attention_weight_min": -91.56837463378906,
      "activations/layer8_attention_weight_max": 42.75176239013672,
      "activations/layer8_attention_weight_min": -42.48954391479492,
      "activations/layer9_attention_weight_max": 32.4664421081543,
      "activations/layer9_attention_weight_min": -34.082763671875,
      "epoch": 15.26,
      "learning_rate": 5.210719696969697e-05,
      "loss": 2.7464,
      "step": 262650
    },
    {
      "activations/layer0_attention_weight_max": 16.23306655883789,
      "activations/layer0_attention_weight_min": -13.358946800231934,
      "activations/layer10_attention_weight_max": 31.85323715209961,
      "activations/layer10_attention_weight_min": -32.684505462646484,
      "activations/layer11_attention_weight_max": 33.186004638671875,
      "activations/layer11_attention_weight_min": -33.66671371459961,
      "activations/layer12_attention_weight_max": 25.926939010620117,
      "activations/layer12_attention_weight_min": -26.585046768188477,
      "activations/layer13_attention_weight_max": 40.69397735595703,
      "activations/layer13_attention_weight_min": -35.09526062011719,
      "activations/layer14_attention_weight_max": 43.41529083251953,
      "activations/layer14_attention_weight_min": -37.701202392578125,
      "activations/layer15_attention_weight_max": 37.84840393066406,
      "activations/layer15_attention_weight_min": -33.86680221557617,
      "activations/layer16_attention_weight_max": 27.43587303161621,
      "activations/layer16_attention_weight_min": -27.082719802856445,
      "activations/layer17_attention_weight_max": 29.68515396118164,
      "activations/layer17_attention_weight_min": -25.05890655517578,
      "activations/layer18_attention_weight_max": 31.693151473999023,
      "activations/layer18_attention_weight_min": -24.632503509521484,
      "activations/layer19_attention_weight_max": 35.1715202331543,
      "activations/layer19_attention_weight_min": -33.25811004638672,
      "activations/layer1_attention_weight_max": 16.524377822875977,
      "activations/layer1_attention_weight_min": -17.333396911621094,
      "activations/layer20_attention_weight_max": 29.379844665527344,
      "activations/layer20_attention_weight_min": -26.49876594543457,
      "activations/layer21_attention_weight_max": 29.568405151367188,
      "activations/layer21_attention_weight_min": -26.2122859954834,
      "activations/layer22_attention_weight_max": 48.5280876159668,
      "activations/layer22_attention_weight_min": -33.750511169433594,
      "activations/layer23_attention_weight_max": 36.694801330566406,
      "activations/layer23_attention_weight_min": -25.16023063659668,
      "activations/layer2_attention_weight_max": 32.44297790527344,
      "activations/layer2_attention_weight_min": -33.1472053527832,
      "activations/layer3_attention_weight_max": 97.63894653320312,
      "activations/layer3_attention_weight_min": -99.54057312011719,
      "activations/layer4_attention_weight_max": 109.4771499633789,
      "activations/layer4_attention_weight_min": -109.42607879638672,
      "activations/layer5_attention_weight_max": 52.74491500854492,
      "activations/layer5_attention_weight_min": -61.82129669189453,
      "activations/layer6_attention_weight_max": 43.10404586791992,
      "activations/layer6_attention_weight_min": -48.26192855834961,
      "activations/layer7_attention_weight_max": 96.67909240722656,
      "activations/layer7_attention_weight_min": -94.02131652832031,
      "activations/layer8_attention_weight_max": 43.322723388671875,
      "activations/layer8_attention_weight_min": -44.09701919555664,
      "activations/layer9_attention_weight_max": 31.756704330444336,
      "activations/layer9_attention_weight_min": -33.06525802612305,
      "epoch": 15.26,
      "learning_rate": 5.208825757575757e-05,
      "loss": 2.7664,
      "step": 262700
    },
    {
      "activations/layer0_attention_weight_max": 16.73932647705078,
      "activations/layer0_attention_weight_min": -11.974547386169434,
      "activations/layer10_attention_weight_max": 31.99270248413086,
      "activations/layer10_attention_weight_min": -31.763383865356445,
      "activations/layer11_attention_weight_max": 33.053955078125,
      "activations/layer11_attention_weight_min": -34.09575653076172,
      "activations/layer12_attention_weight_max": 25.27949333190918,
      "activations/layer12_attention_weight_min": -25.355466842651367,
      "activations/layer13_attention_weight_max": 37.56167221069336,
      "activations/layer13_attention_weight_min": -33.838653564453125,
      "activations/layer14_attention_weight_max": 38.240970611572266,
      "activations/layer14_attention_weight_min": -36.35491180419922,
      "activations/layer15_attention_weight_max": 34.60136032104492,
      "activations/layer15_attention_weight_min": -33.49892044067383,
      "activations/layer16_attention_weight_max": 29.49443244934082,
      "activations/layer16_attention_weight_min": -26.824913024902344,
      "activations/layer17_attention_weight_max": 28.56002426147461,
      "activations/layer17_attention_weight_min": -26.005661010742188,
      "activations/layer18_attention_weight_max": 30.802310943603516,
      "activations/layer18_attention_weight_min": -24.31325340270996,
      "activations/layer19_attention_weight_max": 34.89207077026367,
      "activations/layer19_attention_weight_min": -30.6456241607666,
      "activations/layer1_attention_weight_max": 15.900800704956055,
      "activations/layer1_attention_weight_min": -15.251778602600098,
      "activations/layer20_attention_weight_max": 26.83156394958496,
      "activations/layer20_attention_weight_min": -24.67137336730957,
      "activations/layer21_attention_weight_max": 26.572006225585938,
      "activations/layer21_attention_weight_min": -25.662765502929688,
      "activations/layer22_attention_weight_max": 38.57328414916992,
      "activations/layer22_attention_weight_min": -31.52832794189453,
      "activations/layer23_attention_weight_max": 34.06095504760742,
      "activations/layer23_attention_weight_min": -26.243452072143555,
      "activations/layer2_attention_weight_max": 32.6534309387207,
      "activations/layer2_attention_weight_min": -31.94365692138672,
      "activations/layer3_attention_weight_max": 99.3697738647461,
      "activations/layer3_attention_weight_min": -96.52493286132812,
      "activations/layer4_attention_weight_max": 106.27900695800781,
      "activations/layer4_attention_weight_min": -111.81798553466797,
      "activations/layer5_attention_weight_max": 50.10047149658203,
      "activations/layer5_attention_weight_min": -60.641014099121094,
      "activations/layer6_attention_weight_max": 41.82584762573242,
      "activations/layer6_attention_weight_min": -47.293338775634766,
      "activations/layer7_attention_weight_max": 87.69682312011719,
      "activations/layer7_attention_weight_min": -88.85297393798828,
      "activations/layer8_attention_weight_max": 38.992252349853516,
      "activations/layer8_attention_weight_min": -40.1064338684082,
      "activations/layer9_attention_weight_max": 31.23456382751465,
      "activations/layer9_attention_weight_min": -32.6676139831543,
      "epoch": 15.27,
      "learning_rate": 5.206931818181818e-05,
      "loss": 2.7487,
      "step": 262750
    },
    {
      "activations/layer0_attention_weight_max": 16.257097244262695,
      "activations/layer0_attention_weight_min": -11.909546852111816,
      "activations/layer10_attention_weight_max": 33.28164291381836,
      "activations/layer10_attention_weight_min": -32.543174743652344,
      "activations/layer11_attention_weight_max": 34.38517761230469,
      "activations/layer11_attention_weight_min": -33.82060241699219,
      "activations/layer12_attention_weight_max": 26.943870544433594,
      "activations/layer12_attention_weight_min": -27.38102149963379,
      "activations/layer13_attention_weight_max": 39.92448806762695,
      "activations/layer13_attention_weight_min": -36.20360565185547,
      "activations/layer14_attention_weight_max": 44.514488220214844,
      "activations/layer14_attention_weight_min": -40.58256149291992,
      "activations/layer15_attention_weight_max": 40.94805908203125,
      "activations/layer15_attention_weight_min": -35.65489959716797,
      "activations/layer16_attention_weight_max": 31.435550689697266,
      "activations/layer16_attention_weight_min": -30.155628204345703,
      "activations/layer17_attention_weight_max": 32.2183723449707,
      "activations/layer17_attention_weight_min": -26.49085235595703,
      "activations/layer18_attention_weight_max": 33.994895935058594,
      "activations/layer18_attention_weight_min": -24.60871124267578,
      "activations/layer19_attention_weight_max": 34.211795806884766,
      "activations/layer19_attention_weight_min": -31.86638832092285,
      "activations/layer1_attention_weight_max": 16.84772491455078,
      "activations/layer1_attention_weight_min": -13.491565704345703,
      "activations/layer20_attention_weight_max": 30.730884552001953,
      "activations/layer20_attention_weight_min": -24.788061141967773,
      "activations/layer21_attention_weight_max": 27.68973731994629,
      "activations/layer21_attention_weight_min": -24.402822494506836,
      "activations/layer22_attention_weight_max": 43.44255447387695,
      "activations/layer22_attention_weight_min": -31.53083610534668,
      "activations/layer23_attention_weight_max": 34.77708435058594,
      "activations/layer23_attention_weight_min": -27.283416748046875,
      "activations/layer2_attention_weight_max": 33.16531753540039,
      "activations/layer2_attention_weight_min": -33.91790771484375,
      "activations/layer3_attention_weight_max": 99.19170379638672,
      "activations/layer3_attention_weight_min": -99.82086944580078,
      "activations/layer4_attention_weight_max": 110.73186492919922,
      "activations/layer4_attention_weight_min": -110.1643295288086,
      "activations/layer5_attention_weight_max": 52.687252044677734,
      "activations/layer5_attention_weight_min": -66.18746948242188,
      "activations/layer6_attention_weight_max": 45.418758392333984,
      "activations/layer6_attention_weight_min": -45.56037521362305,
      "activations/layer7_attention_weight_max": 91.63966369628906,
      "activations/layer7_attention_weight_min": -94.66474914550781,
      "activations/layer8_attention_weight_max": 41.85566329956055,
      "activations/layer8_attention_weight_min": -42.13854217529297,
      "activations/layer9_attention_weight_max": 33.099544525146484,
      "activations/layer9_attention_weight_min": -34.39698028564453,
      "epoch": 15.27,
      "learning_rate": 5.205037878787878e-05,
      "loss": 2.7491,
      "step": 262800
    },
    {
      "activations/layer0_attention_weight_max": 15.453692436218262,
      "activations/layer0_attention_weight_min": -12.155598640441895,
      "activations/layer10_attention_weight_max": 32.91678237915039,
      "activations/layer10_attention_weight_min": -31.901432037353516,
      "activations/layer11_attention_weight_max": 34.45269012451172,
      "activations/layer11_attention_weight_min": -33.251304626464844,
      "activations/layer12_attention_weight_max": 24.768146514892578,
      "activations/layer12_attention_weight_min": -28.303861618041992,
      "activations/layer13_attention_weight_max": 35.39576721191406,
      "activations/layer13_attention_weight_min": -36.81535720825195,
      "activations/layer14_attention_weight_max": 40.42118453979492,
      "activations/layer14_attention_weight_min": -37.42668533325195,
      "activations/layer15_attention_weight_max": 36.49717330932617,
      "activations/layer15_attention_weight_min": -35.436283111572266,
      "activations/layer16_attention_weight_max": 31.62999153137207,
      "activations/layer16_attention_weight_min": -30.41599464416504,
      "activations/layer17_attention_weight_max": 30.71767234802246,
      "activations/layer17_attention_weight_min": -26.728588104248047,
      "activations/layer18_attention_weight_max": 30.05778694152832,
      "activations/layer18_attention_weight_min": -24.837215423583984,
      "activations/layer19_attention_weight_max": 32.83407211303711,
      "activations/layer19_attention_weight_min": -32.71153259277344,
      "activations/layer1_attention_weight_max": 16.179832458496094,
      "activations/layer1_attention_weight_min": -14.840653419494629,
      "activations/layer20_attention_weight_max": 26.705293655395508,
      "activations/layer20_attention_weight_min": -25.45077133178711,
      "activations/layer21_attention_weight_max": 28.483810424804688,
      "activations/layer21_attention_weight_min": -24.669322967529297,
      "activations/layer22_attention_weight_max": 37.62417221069336,
      "activations/layer22_attention_weight_min": -30.881948471069336,
      "activations/layer23_attention_weight_max": 30.253673553466797,
      "activations/layer23_attention_weight_min": -27.03358268737793,
      "activations/layer2_attention_weight_max": 37.54690933227539,
      "activations/layer2_attention_weight_min": -34.16741943359375,
      "activations/layer3_attention_weight_max": 96.25397491455078,
      "activations/layer3_attention_weight_min": -97.67778015136719,
      "activations/layer4_attention_weight_max": 107.0088119506836,
      "activations/layer4_attention_weight_min": -111.4384994506836,
      "activations/layer5_attention_weight_max": 48.598873138427734,
      "activations/layer5_attention_weight_min": -62.93531036376953,
      "activations/layer6_attention_weight_max": 41.519073486328125,
      "activations/layer6_attention_weight_min": -44.76844787597656,
      "activations/layer7_attention_weight_max": 91.53649139404297,
      "activations/layer7_attention_weight_min": -92.30036926269531,
      "activations/layer8_attention_weight_max": 41.21829605102539,
      "activations/layer8_attention_weight_min": -40.517372131347656,
      "activations/layer9_attention_weight_max": 30.044174194335938,
      "activations/layer9_attention_weight_min": -32.315040588378906,
      "epoch": 15.27,
      "learning_rate": 5.203143939393939e-05,
      "loss": 2.7425,
      "step": 262850
    },
    {
      "activations/layer0_attention_weight_max": 16.699462890625,
      "activations/layer0_attention_weight_min": -11.894524574279785,
      "activations/layer10_attention_weight_max": 38.52089309692383,
      "activations/layer10_attention_weight_min": -32.48988723754883,
      "activations/layer11_attention_weight_max": 34.60886001586914,
      "activations/layer11_attention_weight_min": -33.59315490722656,
      "activations/layer12_attention_weight_max": 22.514732360839844,
      "activations/layer12_attention_weight_min": -26.187448501586914,
      "activations/layer13_attention_weight_max": 32.56439971923828,
      "activations/layer13_attention_weight_min": -34.92784118652344,
      "activations/layer14_attention_weight_max": 37.908416748046875,
      "activations/layer14_attention_weight_min": -37.97788619995117,
      "activations/layer15_attention_weight_max": 32.59361267089844,
      "activations/layer15_attention_weight_min": -34.72348403930664,
      "activations/layer16_attention_weight_max": 27.71502685546875,
      "activations/layer16_attention_weight_min": -27.62824249267578,
      "activations/layer17_attention_weight_max": 26.881532669067383,
      "activations/layer17_attention_weight_min": -26.119524002075195,
      "activations/layer18_attention_weight_max": 28.015592575073242,
      "activations/layer18_attention_weight_min": -24.59064292907715,
      "activations/layer19_attention_weight_max": 31.607803344726562,
      "activations/layer19_attention_weight_min": -28.851806640625,
      "activations/layer1_attention_weight_max": 15.358898162841797,
      "activations/layer1_attention_weight_min": -14.186095237731934,
      "activations/layer20_attention_weight_max": 29.77946662902832,
      "activations/layer20_attention_weight_min": -25.276988983154297,
      "activations/layer21_attention_weight_max": 28.077177047729492,
      "activations/layer21_attention_weight_min": -24.61900520324707,
      "activations/layer22_attention_weight_max": 40.74927520751953,
      "activations/layer22_attention_weight_min": -30.82398223876953,
      "activations/layer23_attention_weight_max": 32.74759292602539,
      "activations/layer23_attention_weight_min": -24.01894187927246,
      "activations/layer2_attention_weight_max": 32.16436767578125,
      "activations/layer2_attention_weight_min": -34.782005310058594,
      "activations/layer3_attention_weight_max": 95.43464660644531,
      "activations/layer3_attention_weight_min": -98.15782928466797,
      "activations/layer4_attention_weight_max": 104.75078582763672,
      "activations/layer4_attention_weight_min": -109.81471252441406,
      "activations/layer5_attention_weight_max": 50.04530334472656,
      "activations/layer5_attention_weight_min": -63.57555389404297,
      "activations/layer6_attention_weight_max": 40.10504913330078,
      "activations/layer6_attention_weight_min": -43.72590637207031,
      "activations/layer7_attention_weight_max": 95.12446594238281,
      "activations/layer7_attention_weight_min": -94.57019805908203,
      "activations/layer8_attention_weight_max": 40.90906524658203,
      "activations/layer8_attention_weight_min": -42.297203063964844,
      "activations/layer9_attention_weight_max": 32.04929733276367,
      "activations/layer9_attention_weight_min": -32.63020706176758,
      "epoch": 15.28,
      "learning_rate": 5.20125e-05,
      "loss": 2.7429,
      "step": 262900
    },
    {
      "activations/layer0_attention_weight_max": 16.588438034057617,
      "activations/layer0_attention_weight_min": -11.662111282348633,
      "activations/layer10_attention_weight_max": 41.48223876953125,
      "activations/layer10_attention_weight_min": -38.027076721191406,
      "activations/layer11_attention_weight_max": 40.78158187866211,
      "activations/layer11_attention_weight_min": -41.23780059814453,
      "activations/layer12_attention_weight_max": 28.903444290161133,
      "activations/layer12_attention_weight_min": -25.37183952331543,
      "activations/layer13_attention_weight_max": 42.434356689453125,
      "activations/layer13_attention_weight_min": -34.92753219604492,
      "activations/layer14_attention_weight_max": 47.411800384521484,
      "activations/layer14_attention_weight_min": -38.056026458740234,
      "activations/layer15_attention_weight_max": 43.70026397705078,
      "activations/layer15_attention_weight_min": -34.964744567871094,
      "activations/layer16_attention_weight_max": 33.31775665283203,
      "activations/layer16_attention_weight_min": -27.19077491760254,
      "activations/layer17_attention_weight_max": 31.533342361450195,
      "activations/layer17_attention_weight_min": -24.88096809387207,
      "activations/layer18_attention_weight_max": 34.19306182861328,
      "activations/layer18_attention_weight_min": -23.71283721923828,
      "activations/layer19_attention_weight_max": 32.335758209228516,
      "activations/layer19_attention_weight_min": -31.618789672851562,
      "activations/layer1_attention_weight_max": 15.332542419433594,
      "activations/layer1_attention_weight_min": -14.453141212463379,
      "activations/layer20_attention_weight_max": 26.475069046020508,
      "activations/layer20_attention_weight_min": -25.43245506286621,
      "activations/layer21_attention_weight_max": 27.96048355102539,
      "activations/layer21_attention_weight_min": -24.360124588012695,
      "activations/layer22_attention_weight_max": 47.86691665649414,
      "activations/layer22_attention_weight_min": -30.58176040649414,
      "activations/layer23_attention_weight_max": 30.458189010620117,
      "activations/layer23_attention_weight_min": -26.727825164794922,
      "activations/layer2_attention_weight_max": 36.691951751708984,
      "activations/layer2_attention_weight_min": -32.93670654296875,
      "activations/layer3_attention_weight_max": 105.25463104248047,
      "activations/layer3_attention_weight_min": -107.36695098876953,
      "activations/layer4_attention_weight_max": 113.50960540771484,
      "activations/layer4_attention_weight_min": -116.9200439453125,
      "activations/layer5_attention_weight_max": 52.564735412597656,
      "activations/layer5_attention_weight_min": -63.6358642578125,
      "activations/layer6_attention_weight_max": 45.08141326904297,
      "activations/layer6_attention_weight_min": -49.81980895996094,
      "activations/layer7_attention_weight_max": 100.48014068603516,
      "activations/layer7_attention_weight_min": -97.74699401855469,
      "activations/layer8_attention_weight_max": 48.0833625793457,
      "activations/layer8_attention_weight_min": -46.46873092651367,
      "activations/layer9_attention_weight_max": 38.66678237915039,
      "activations/layer9_attention_weight_min": -36.84640121459961,
      "epoch": 15.28,
      "learning_rate": 5.199393939393939e-05,
      "loss": 2.758,
      "step": 262950
    },
    {
      "activations/layer0_attention_weight_max": 15.728727340698242,
      "activations/layer0_attention_weight_min": -12.104001998901367,
      "activations/layer10_attention_weight_max": 32.357757568359375,
      "activations/layer10_attention_weight_min": -33.239234924316406,
      "activations/layer11_attention_weight_max": 33.45185089111328,
      "activations/layer11_attention_weight_min": -33.15213394165039,
      "activations/layer12_attention_weight_max": 26.322200775146484,
      "activations/layer12_attention_weight_min": -25.841279983520508,
      "activations/layer13_attention_weight_max": 41.17665481567383,
      "activations/layer13_attention_weight_min": -35.8443603515625,
      "activations/layer14_attention_weight_max": 42.136566162109375,
      "activations/layer14_attention_weight_min": -35.94773483276367,
      "activations/layer15_attention_weight_max": 41.2098503112793,
      "activations/layer15_attention_weight_min": -32.95394515991211,
      "activations/layer16_attention_weight_max": 30.10442352294922,
      "activations/layer16_attention_weight_min": -27.699665069580078,
      "activations/layer17_attention_weight_max": 33.98525619506836,
      "activations/layer17_attention_weight_min": -26.71916389465332,
      "activations/layer18_attention_weight_max": 32.19872283935547,
      "activations/layer18_attention_weight_min": -24.46432113647461,
      "activations/layer19_attention_weight_max": 34.4867057800293,
      "activations/layer19_attention_weight_min": -31.010639190673828,
      "activations/layer1_attention_weight_max": 16.74249267578125,
      "activations/layer1_attention_weight_min": -13.795978546142578,
      "activations/layer20_attention_weight_max": 27.846588134765625,
      "activations/layer20_attention_weight_min": -25.273527145385742,
      "activations/layer21_attention_weight_max": 29.483598709106445,
      "activations/layer21_attention_weight_min": -24.130294799804688,
      "activations/layer22_attention_weight_max": 42.73440933227539,
      "activations/layer22_attention_weight_min": -29.91312026977539,
      "activations/layer23_attention_weight_max": 33.89345169067383,
      "activations/layer23_attention_weight_min": -23.72188377380371,
      "activations/layer2_attention_weight_max": 32.08542251586914,
      "activations/layer2_attention_weight_min": -32.333351135253906,
      "activations/layer3_attention_weight_max": 95.4882583618164,
      "activations/layer3_attention_weight_min": -104.03547668457031,
      "activations/layer4_attention_weight_max": 106.69831848144531,
      "activations/layer4_attention_weight_min": -113.94432830810547,
      "activations/layer5_attention_weight_max": 53.86643981933594,
      "activations/layer5_attention_weight_min": -64.188720703125,
      "activations/layer6_attention_weight_max": 42.813472747802734,
      "activations/layer6_attention_weight_min": -45.87805938720703,
      "activations/layer7_attention_weight_max": 89.94036865234375,
      "activations/layer7_attention_weight_min": -91.89427947998047,
      "activations/layer8_attention_weight_max": 38.855796813964844,
      "activations/layer8_attention_weight_min": -44.57173156738281,
      "activations/layer9_attention_weight_max": 32.96392059326172,
      "activations/layer9_attention_weight_min": -33.40869140625,
      "epoch": 15.28,
      "learning_rate": 5.197499999999999e-05,
      "loss": 2.7645,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.5494,
      "eval_samples_per_second": 502.256,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.5494,
      "eval_openwebtext_samples_per_second": 502.256,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9961,
      "eval_wikitext_samples_per_second": 228.451,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_lambada_loss": 2.51171875,
      "eval_lambada_ppl": 12.326097344475713,
      "eval_lambada_runtime": 9.5723,
      "eval_lambada_samples_per_second": 508.653,
      "step": 263000
    },
    {
      "activations/layer0_attention_weight_max": 15.091179847717285,
      "activations/layer0_attention_weight_min": -13.123825073242188,
      "activations/layer10_attention_weight_max": 31.9255428314209,
      "activations/layer10_attention_weight_min": -32.18405532836914,
      "activations/layer11_attention_weight_max": 33.17143249511719,
      "activations/layer11_attention_weight_min": -33.280662536621094,
      "activations/layer12_attention_weight_max": 26.62809181213379,
      "activations/layer12_attention_weight_min": -27.173383712768555,
      "activations/layer13_attention_weight_max": 39.96455764770508,
      "activations/layer13_attention_weight_min": -33.74516296386719,
      "activations/layer14_attention_weight_max": 42.86628341674805,
      "activations/layer14_attention_weight_min": -36.00631332397461,
      "activations/layer15_attention_weight_max": 38.14476013183594,
      "activations/layer15_attention_weight_min": -32.58782958984375,
      "activations/layer16_attention_weight_max": 28.934833526611328,
      "activations/layer16_attention_weight_min": -29.131633758544922,
      "activations/layer17_attention_weight_max": 29.41800308227539,
      "activations/layer17_attention_weight_min": -23.81741714477539,
      "activations/layer18_attention_weight_max": 30.196359634399414,
      "activations/layer18_attention_weight_min": -24.694759368896484,
      "activations/layer19_attention_weight_max": 33.52589797973633,
      "activations/layer19_attention_weight_min": -30.731739044189453,
      "activations/layer1_attention_weight_max": 16.717308044433594,
      "activations/layer1_attention_weight_min": -13.482714653015137,
      "activations/layer20_attention_weight_max": 31.076852798461914,
      "activations/layer20_attention_weight_min": -24.575815200805664,
      "activations/layer21_attention_weight_max": 29.856990814208984,
      "activations/layer21_attention_weight_min": -23.566125869750977,
      "activations/layer22_attention_weight_max": 45.79883575439453,
      "activations/layer22_attention_weight_min": -30.61703872680664,
      "activations/layer23_attention_weight_max": 32.84020233154297,
      "activations/layer23_attention_weight_min": -24.82012176513672,
      "activations/layer2_attention_weight_max": 32.75395965576172,
      "activations/layer2_attention_weight_min": -31.655498504638672,
      "activations/layer3_attention_weight_max": 97.56552124023438,
      "activations/layer3_attention_weight_min": -96.05461120605469,
      "activations/layer4_attention_weight_max": 107.70027160644531,
      "activations/layer4_attention_weight_min": -105.68448638916016,
      "activations/layer5_attention_weight_max": 52.763267517089844,
      "activations/layer5_attention_weight_min": -59.2012939453125,
      "activations/layer6_attention_weight_max": 43.26736068725586,
      "activations/layer6_attention_weight_min": -45.27067565917969,
      "activations/layer7_attention_weight_max": 90.99829864501953,
      "activations/layer7_attention_weight_min": -85.39115142822266,
      "activations/layer8_attention_weight_max": 38.27964782714844,
      "activations/layer8_attention_weight_min": -41.6155891418457,
      "activations/layer9_attention_weight_max": 32.169700622558594,
      "activations/layer9_attention_weight_min": -33.18784713745117,
      "epoch": 15.28,
      "learning_rate": 5.1956060606060604e-05,
      "loss": 2.7755,
      "step": 263050
    },
    {
      "activations/layer0_attention_weight_max": 15.828195571899414,
      "activations/layer0_attention_weight_min": -11.970878601074219,
      "activations/layer10_attention_weight_max": 32.77846908569336,
      "activations/layer10_attention_weight_min": -33.679840087890625,
      "activations/layer11_attention_weight_max": 32.23012161254883,
      "activations/layer11_attention_weight_min": -32.33744430541992,
      "activations/layer12_attention_weight_max": 25.201751708984375,
      "activations/layer12_attention_weight_min": -26.731037139892578,
      "activations/layer13_attention_weight_max": 35.12117004394531,
      "activations/layer13_attention_weight_min": -33.299495697021484,
      "activations/layer14_attention_weight_max": 39.58695983886719,
      "activations/layer14_attention_weight_min": -35.00917434692383,
      "activations/layer15_attention_weight_max": 35.012847900390625,
      "activations/layer15_attention_weight_min": -32.833683013916016,
      "activations/layer16_attention_weight_max": 28.438398361206055,
      "activations/layer16_attention_weight_min": -27.69143295288086,
      "activations/layer17_attention_weight_max": 30.217201232910156,
      "activations/layer17_attention_weight_min": -24.743450164794922,
      "activations/layer18_attention_weight_max": 35.78107833862305,
      "activations/layer18_attention_weight_min": -24.4523868560791,
      "activations/layer19_attention_weight_max": 34.19527816772461,
      "activations/layer19_attention_weight_min": -30.207242965698242,
      "activations/layer1_attention_weight_max": 16.471635818481445,
      "activations/layer1_attention_weight_min": -15.16851806640625,
      "activations/layer20_attention_weight_max": 29.689184188842773,
      "activations/layer20_attention_weight_min": -25.864234924316406,
      "activations/layer21_attention_weight_max": 29.0312557220459,
      "activations/layer21_attention_weight_min": -25.335357666015625,
      "activations/layer22_attention_weight_max": 44.01056671142578,
      "activations/layer22_attention_weight_min": -30.973773956298828,
      "activations/layer23_attention_weight_max": 32.735904693603516,
      "activations/layer23_attention_weight_min": -25.740867614746094,
      "activations/layer2_attention_weight_max": 32.74653625488281,
      "activations/layer2_attention_weight_min": -32.28832244873047,
      "activations/layer3_attention_weight_max": 104.19666290283203,
      "activations/layer3_attention_weight_min": -102.43821716308594,
      "activations/layer4_attention_weight_max": 113.3768081665039,
      "activations/layer4_attention_weight_min": -111.5968017578125,
      "activations/layer5_attention_weight_max": 58.072227478027344,
      "activations/layer5_attention_weight_min": -61.33954620361328,
      "activations/layer6_attention_weight_max": 44.46234893798828,
      "activations/layer6_attention_weight_min": -46.496360778808594,
      "activations/layer7_attention_weight_max": 89.36885070800781,
      "activations/layer7_attention_weight_min": -99.21344757080078,
      "activations/layer8_attention_weight_max": 39.182273864746094,
      "activations/layer8_attention_weight_min": -42.694725036621094,
      "activations/layer9_attention_weight_max": 31.626707077026367,
      "activations/layer9_attention_weight_min": -33.980350494384766,
      "epoch": 15.29,
      "learning_rate": 5.1937121212121206e-05,
      "loss": 2.7474,
      "step": 263100
    },
    {
      "activations/layer0_attention_weight_max": 16.599384307861328,
      "activations/layer0_attention_weight_min": -12.736157417297363,
      "activations/layer10_attention_weight_max": 35.35030746459961,
      "activations/layer10_attention_weight_min": -36.76530838012695,
      "activations/layer11_attention_weight_max": 34.337669372558594,
      "activations/layer11_attention_weight_min": -35.9838981628418,
      "activations/layer12_attention_weight_max": 26.41106414794922,
      "activations/layer12_attention_weight_min": -28.89226531982422,
      "activations/layer13_attention_weight_max": 38.37484359741211,
      "activations/layer13_attention_weight_min": -35.26547622680664,
      "activations/layer14_attention_weight_max": 44.43440628051758,
      "activations/layer14_attention_weight_min": -38.149261474609375,
      "activations/layer15_attention_weight_max": 38.178627014160156,
      "activations/layer15_attention_weight_min": -34.64701461791992,
      "activations/layer16_attention_weight_max": 27.940752029418945,
      "activations/layer16_attention_weight_min": -30.64314842224121,
      "activations/layer17_attention_weight_max": 30.058908462524414,
      "activations/layer17_attention_weight_min": -27.61393165588379,
      "activations/layer18_attention_weight_max": 28.940603256225586,
      "activations/layer18_attention_weight_min": -24.102237701416016,
      "activations/layer19_attention_weight_max": 32.3784294128418,
      "activations/layer19_attention_weight_min": -30.7675724029541,
      "activations/layer1_attention_weight_max": 16.258344650268555,
      "activations/layer1_attention_weight_min": -14.779664039611816,
      "activations/layer20_attention_weight_max": 29.37567710876465,
      "activations/layer20_attention_weight_min": -24.226016998291016,
      "activations/layer21_attention_weight_max": 27.741636276245117,
      "activations/layer21_attention_weight_min": -23.672021865844727,
      "activations/layer22_attention_weight_max": 38.4694938659668,
      "activations/layer22_attention_weight_min": -29.702119827270508,
      "activations/layer23_attention_weight_max": 31.483190536499023,
      "activations/layer23_attention_weight_min": -23.524099349975586,
      "activations/layer2_attention_weight_max": 32.498558044433594,
      "activations/layer2_attention_weight_min": -31.504308700561523,
      "activations/layer3_attention_weight_max": 95.2219009399414,
      "activations/layer3_attention_weight_min": -94.88929748535156,
      "activations/layer4_attention_weight_max": 107.18524169921875,
      "activations/layer4_attention_weight_min": -109.38075256347656,
      "activations/layer5_attention_weight_max": 55.61644744873047,
      "activations/layer5_attention_weight_min": -63.34785079956055,
      "activations/layer6_attention_weight_max": 44.6008186340332,
      "activations/layer6_attention_weight_min": -46.23720932006836,
      "activations/layer7_attention_weight_max": 96.82611083984375,
      "activations/layer7_attention_weight_min": -98.2669906616211,
      "activations/layer8_attention_weight_max": 41.70115661621094,
      "activations/layer8_attention_weight_min": -43.51327896118164,
      "activations/layer9_attention_weight_max": 33.91854476928711,
      "activations/layer9_attention_weight_min": -35.37753677368164,
      "epoch": 15.29,
      "learning_rate": 5.1918181818181814e-05,
      "loss": 2.7558,
      "step": 263150
    },
    {
      "activations/layer0_attention_weight_max": 16.465564727783203,
      "activations/layer0_attention_weight_min": -12.529439926147461,
      "activations/layer10_attention_weight_max": 36.92546463012695,
      "activations/layer10_attention_weight_min": -32.717350006103516,
      "activations/layer11_attention_weight_max": 39.96980285644531,
      "activations/layer11_attention_weight_min": -32.85850524902344,
      "activations/layer12_attention_weight_max": 27.59408950805664,
      "activations/layer12_attention_weight_min": -26.10833168029785,
      "activations/layer13_attention_weight_max": 38.57310485839844,
      "activations/layer13_attention_weight_min": -35.68566131591797,
      "activations/layer14_attention_weight_max": 43.01213836669922,
      "activations/layer14_attention_weight_min": -38.871402740478516,
      "activations/layer15_attention_weight_max": 36.856040954589844,
      "activations/layer15_attention_weight_min": -35.24000549316406,
      "activations/layer16_attention_weight_max": 27.37864875793457,
      "activations/layer16_attention_weight_min": -28.103803634643555,
      "activations/layer17_attention_weight_max": 28.563053131103516,
      "activations/layer17_attention_weight_min": -25.149106979370117,
      "activations/layer18_attention_weight_max": 30.014989852905273,
      "activations/layer18_attention_weight_min": -26.04256820678711,
      "activations/layer19_attention_weight_max": 32.30418395996094,
      "activations/layer19_attention_weight_min": -31.67484474182129,
      "activations/layer1_attention_weight_max": 16.4283504486084,
      "activations/layer1_attention_weight_min": -14.08043384552002,
      "activations/layer20_attention_weight_max": 29.19769859313965,
      "activations/layer20_attention_weight_min": -24.428142547607422,
      "activations/layer21_attention_weight_max": 28.36117935180664,
      "activations/layer21_attention_weight_min": -25.570852279663086,
      "activations/layer22_attention_weight_max": 46.59795379638672,
      "activations/layer22_attention_weight_min": -33.17475128173828,
      "activations/layer23_attention_weight_max": 33.49249267578125,
      "activations/layer23_attention_weight_min": -26.01596450805664,
      "activations/layer2_attention_weight_max": 33.42490768432617,
      "activations/layer2_attention_weight_min": -34.478389739990234,
      "activations/layer3_attention_weight_max": 96.75379180908203,
      "activations/layer3_attention_weight_min": -106.72571563720703,
      "activations/layer4_attention_weight_max": 107.39845275878906,
      "activations/layer4_attention_weight_min": -115.88005828857422,
      "activations/layer5_attention_weight_max": 51.026390075683594,
      "activations/layer5_attention_weight_min": -64.76985168457031,
      "activations/layer6_attention_weight_max": 46.25702667236328,
      "activations/layer6_attention_weight_min": -46.2789192199707,
      "activations/layer7_attention_weight_max": 88.17317199707031,
      "activations/layer7_attention_weight_min": -97.57398223876953,
      "activations/layer8_attention_weight_max": 43.09816360473633,
      "activations/layer8_attention_weight_min": -42.58284378051758,
      "activations/layer9_attention_weight_max": 33.45539855957031,
      "activations/layer9_attention_weight_min": -35.257991790771484,
      "epoch": 15.29,
      "learning_rate": 5.189924242424242e-05,
      "loss": 2.7543,
      "step": 263200
    },
    {
      "activations/layer0_attention_weight_max": 16.517078399658203,
      "activations/layer0_attention_weight_min": -11.85700511932373,
      "activations/layer10_attention_weight_max": 37.87972640991211,
      "activations/layer10_attention_weight_min": -35.755611419677734,
      "activations/layer11_attention_weight_max": 37.67357635498047,
      "activations/layer11_attention_weight_min": -35.27632141113281,
      "activations/layer12_attention_weight_max": 30.18934440612793,
      "activations/layer12_attention_weight_min": -28.784847259521484,
      "activations/layer13_attention_weight_max": 42.13534164428711,
      "activations/layer13_attention_weight_min": -38.85847091674805,
      "activations/layer14_attention_weight_max": 45.7783317565918,
      "activations/layer14_attention_weight_min": -39.82847213745117,
      "activations/layer15_attention_weight_max": 44.94647216796875,
      "activations/layer15_attention_weight_min": -36.13230514526367,
      "activations/layer16_attention_weight_max": 32.438045501708984,
      "activations/layer16_attention_weight_min": -27.261682510375977,
      "activations/layer17_attention_weight_max": 33.20758056640625,
      "activations/layer17_attention_weight_min": -25.159698486328125,
      "activations/layer18_attention_weight_max": 34.5595588684082,
      "activations/layer18_attention_weight_min": -23.890920639038086,
      "activations/layer19_attention_weight_max": 35.780662536621094,
      "activations/layer19_attention_weight_min": -29.96651268005371,
      "activations/layer1_attention_weight_max": 16.458242416381836,
      "activations/layer1_attention_weight_min": -15.186668395996094,
      "activations/layer20_attention_weight_max": 31.398107528686523,
      "activations/layer20_attention_weight_min": -24.60491371154785,
      "activations/layer21_attention_weight_max": 32.43722915649414,
      "activations/layer21_attention_weight_min": -25.662384033203125,
      "activations/layer22_attention_weight_max": 42.53044509887695,
      "activations/layer22_attention_weight_min": -30.771562576293945,
      "activations/layer23_attention_weight_max": 38.319122314453125,
      "activations/layer23_attention_weight_min": -25.85666275024414,
      "activations/layer2_attention_weight_max": 33.54722595214844,
      "activations/layer2_attention_weight_min": -33.912227630615234,
      "activations/layer3_attention_weight_max": 102.8920669555664,
      "activations/layer3_attention_weight_min": -104.96653747558594,
      "activations/layer4_attention_weight_max": 113.15763092041016,
      "activations/layer4_attention_weight_min": -112.64595031738281,
      "activations/layer5_attention_weight_max": 53.26682662963867,
      "activations/layer5_attention_weight_min": -64.25213623046875,
      "activations/layer6_attention_weight_max": 45.18238067626953,
      "activations/layer6_attention_weight_min": -48.488834381103516,
      "activations/layer7_attention_weight_max": 95.45199584960938,
      "activations/layer7_attention_weight_min": -97.70670318603516,
      "activations/layer8_attention_weight_max": 44.635398864746094,
      "activations/layer8_attention_weight_min": -43.942195892333984,
      "activations/layer9_attention_weight_max": 37.247406005859375,
      "activations/layer9_attention_weight_min": -36.022640228271484,
      "epoch": 15.3,
      "learning_rate": 5.188030303030303e-05,
      "loss": 2.7618,
      "step": 263250
    },
    {
      "activations/layer0_attention_weight_max": 16.80029296875,
      "activations/layer0_attention_weight_min": -11.566437721252441,
      "activations/layer10_attention_weight_max": 33.49038314819336,
      "activations/layer10_attention_weight_min": -35.70771408081055,
      "activations/layer11_attention_weight_max": 34.9378547668457,
      "activations/layer11_attention_weight_min": -35.83964538574219,
      "activations/layer12_attention_weight_max": 25.635534286499023,
      "activations/layer12_attention_weight_min": -27.147253036499023,
      "activations/layer13_attention_weight_max": 41.05999755859375,
      "activations/layer13_attention_weight_min": -37.777069091796875,
      "activations/layer14_attention_weight_max": 45.121864318847656,
      "activations/layer14_attention_weight_min": -42.45754623413086,
      "activations/layer15_attention_weight_max": 39.101158142089844,
      "activations/layer15_attention_weight_min": -38.99563217163086,
      "activations/layer16_attention_weight_max": 33.303733825683594,
      "activations/layer16_attention_weight_min": -29.52949333190918,
      "activations/layer17_attention_weight_max": 29.840967178344727,
      "activations/layer17_attention_weight_min": -27.849035263061523,
      "activations/layer18_attention_weight_max": 31.324935913085938,
      "activations/layer18_attention_weight_min": -26.211135864257812,
      "activations/layer19_attention_weight_max": 34.550357818603516,
      "activations/layer19_attention_weight_min": -32.217830657958984,
      "activations/layer1_attention_weight_max": 15.91545581817627,
      "activations/layer1_attention_weight_min": -14.765058517456055,
      "activations/layer20_attention_weight_max": 28.2923526763916,
      "activations/layer20_attention_weight_min": -24.758222579956055,
      "activations/layer21_attention_weight_max": 27.268041610717773,
      "activations/layer21_attention_weight_min": -24.056344985961914,
      "activations/layer22_attention_weight_max": 45.3712158203125,
      "activations/layer22_attention_weight_min": -31.586830139160156,
      "activations/layer23_attention_weight_max": 33.85270690917969,
      "activations/layer23_attention_weight_min": -25.84407615661621,
      "activations/layer2_attention_weight_max": 32.37853240966797,
      "activations/layer2_attention_weight_min": -32.051300048828125,
      "activations/layer3_attention_weight_max": 96.27469635009766,
      "activations/layer3_attention_weight_min": -98.75765228271484,
      "activations/layer4_attention_weight_max": 108.76116943359375,
      "activations/layer4_attention_weight_min": -109.39522552490234,
      "activations/layer5_attention_weight_max": 50.74828338623047,
      "activations/layer5_attention_weight_min": -62.14124298095703,
      "activations/layer6_attention_weight_max": 43.5975456237793,
      "activations/layer6_attention_weight_min": -46.28086853027344,
      "activations/layer7_attention_weight_max": 88.48036193847656,
      "activations/layer7_attention_weight_min": -93.87831115722656,
      "activations/layer8_attention_weight_max": 41.544090270996094,
      "activations/layer8_attention_weight_min": -43.89097213745117,
      "activations/layer9_attention_weight_max": 33.44430160522461,
      "activations/layer9_attention_weight_min": -38.58829879760742,
      "epoch": 15.3,
      "learning_rate": 5.186136363636363e-05,
      "loss": 2.7553,
      "step": 263300
    },
    {
      "activations/layer0_attention_weight_max": 15.80860424041748,
      "activations/layer0_attention_weight_min": -11.928664207458496,
      "activations/layer10_attention_weight_max": 30.578506469726562,
      "activations/layer10_attention_weight_min": -33.57807922363281,
      "activations/layer11_attention_weight_max": 31.498165130615234,
      "activations/layer11_attention_weight_min": -33.44316482543945,
      "activations/layer12_attention_weight_max": 26.835878372192383,
      "activations/layer12_attention_weight_min": -26.241195678710938,
      "activations/layer13_attention_weight_max": 38.1950798034668,
      "activations/layer13_attention_weight_min": -33.39517593383789,
      "activations/layer14_attention_weight_max": 40.26630401611328,
      "activations/layer14_attention_weight_min": -36.03841018676758,
      "activations/layer15_attention_weight_max": 35.40480041503906,
      "activations/layer15_attention_weight_min": -34.43741226196289,
      "activations/layer16_attention_weight_max": 30.670482635498047,
      "activations/layer16_attention_weight_min": -27.46935272216797,
      "activations/layer17_attention_weight_max": 31.008081436157227,
      "activations/layer17_attention_weight_min": -25.796630859375,
      "activations/layer18_attention_weight_max": 32.77641296386719,
      "activations/layer18_attention_weight_min": -26.218347549438477,
      "activations/layer19_attention_weight_max": 36.301700592041016,
      "activations/layer19_attention_weight_min": -30.943296432495117,
      "activations/layer1_attention_weight_max": 16.341184616088867,
      "activations/layer1_attention_weight_min": -15.718246459960938,
      "activations/layer20_attention_weight_max": 32.40221405029297,
      "activations/layer20_attention_weight_min": -26.5539608001709,
      "activations/layer21_attention_weight_max": 30.251699447631836,
      "activations/layer21_attention_weight_min": -25.64946174621582,
      "activations/layer22_attention_weight_max": 44.745235443115234,
      "activations/layer22_attention_weight_min": -31.8592586517334,
      "activations/layer23_attention_weight_max": 37.44245910644531,
      "activations/layer23_attention_weight_min": -26.612340927124023,
      "activations/layer2_attention_weight_max": 33.26321029663086,
      "activations/layer2_attention_weight_min": -34.34334945678711,
      "activations/layer3_attention_weight_max": 100.29991912841797,
      "activations/layer3_attention_weight_min": -101.9120864868164,
      "activations/layer4_attention_weight_max": 110.62245178222656,
      "activations/layer4_attention_weight_min": -112.28124237060547,
      "activations/layer5_attention_weight_max": 51.529449462890625,
      "activations/layer5_attention_weight_min": -64.26631164550781,
      "activations/layer6_attention_weight_max": 44.528934478759766,
      "activations/layer6_attention_weight_min": -49.29366683959961,
      "activations/layer7_attention_weight_max": 95.91114807128906,
      "activations/layer7_attention_weight_min": -95.4017333984375,
      "activations/layer8_attention_weight_max": 40.5240364074707,
      "activations/layer8_attention_weight_min": -46.4691276550293,
      "activations/layer9_attention_weight_max": 31.514307022094727,
      "activations/layer9_attention_weight_min": -33.21343994140625,
      "epoch": 15.3,
      "learning_rate": 5.1842424242424234e-05,
      "loss": 2.7597,
      "step": 263350
    },
    {
      "activations/layer0_attention_weight_max": 16.448585510253906,
      "activations/layer0_attention_weight_min": -11.987493515014648,
      "activations/layer10_attention_weight_max": 30.67569351196289,
      "activations/layer10_attention_weight_min": -31.789770126342773,
      "activations/layer11_attention_weight_max": 30.879064559936523,
      "activations/layer11_attention_weight_min": -33.43505096435547,
      "activations/layer12_attention_weight_max": 25.553613662719727,
      "activations/layer12_attention_weight_min": -25.849796295166016,
      "activations/layer13_attention_weight_max": 35.70595932006836,
      "activations/layer13_attention_weight_min": -33.47614669799805,
      "activations/layer14_attention_weight_max": 37.43611526489258,
      "activations/layer14_attention_weight_min": -36.66107940673828,
      "activations/layer15_attention_weight_max": 34.769264221191406,
      "activations/layer15_attention_weight_min": -33.73760223388672,
      "activations/layer16_attention_weight_max": 27.550525665283203,
      "activations/layer16_attention_weight_min": -27.702104568481445,
      "activations/layer17_attention_weight_max": 27.327009201049805,
      "activations/layer17_attention_weight_min": -24.183664321899414,
      "activations/layer18_attention_weight_max": 33.03826141357422,
      "activations/layer18_attention_weight_min": -23.65857696533203,
      "activations/layer19_attention_weight_max": 29.198816299438477,
      "activations/layer19_attention_weight_min": -29.241785049438477,
      "activations/layer1_attention_weight_max": 15.833770751953125,
      "activations/layer1_attention_weight_min": -14.472138404846191,
      "activations/layer20_attention_weight_max": 27.323869705200195,
      "activations/layer20_attention_weight_min": -24.99770164489746,
      "activations/layer21_attention_weight_max": 29.2698974609375,
      "activations/layer21_attention_weight_min": -23.088502883911133,
      "activations/layer22_attention_weight_max": 39.58353805541992,
      "activations/layer22_attention_weight_min": -30.397302627563477,
      "activations/layer23_attention_weight_max": 30.75206756591797,
      "activations/layer23_attention_weight_min": -24.22045135498047,
      "activations/layer2_attention_weight_max": 32.20311737060547,
      "activations/layer2_attention_weight_min": -33.42052459716797,
      "activations/layer3_attention_weight_max": 98.79959869384766,
      "activations/layer3_attention_weight_min": -98.62848663330078,
      "activations/layer4_attention_weight_max": 110.12042236328125,
      "activations/layer4_attention_weight_min": -109.24828338623047,
      "activations/layer5_attention_weight_max": 49.473838806152344,
      "activations/layer5_attention_weight_min": -68.1526107788086,
      "activations/layer6_attention_weight_max": 43.834781646728516,
      "activations/layer6_attention_weight_min": -46.48368835449219,
      "activations/layer7_attention_weight_max": 87.15828704833984,
      "activations/layer7_attention_weight_min": -93.33499145507812,
      "activations/layer8_attention_weight_max": 39.08820343017578,
      "activations/layer8_attention_weight_min": -41.71956253051758,
      "activations/layer9_attention_weight_max": 30.48432731628418,
      "activations/layer9_attention_weight_min": -32.394405364990234,
      "epoch": 15.31,
      "learning_rate": 5.182348484848485e-05,
      "loss": 2.7565,
      "step": 263400
    },
    {
      "activations/layer0_attention_weight_max": 16.598726272583008,
      "activations/layer0_attention_weight_min": -12.51800537109375,
      "activations/layer10_attention_weight_max": 32.46564483642578,
      "activations/layer10_attention_weight_min": -34.342689514160156,
      "activations/layer11_attention_weight_max": 32.21436309814453,
      "activations/layer11_attention_weight_min": -33.25135803222656,
      "activations/layer12_attention_weight_max": 33.21189880371094,
      "activations/layer12_attention_weight_min": -26.40793800354004,
      "activations/layer13_attention_weight_max": 38.68412780761719,
      "activations/layer13_attention_weight_min": -35.554100036621094,
      "activations/layer14_attention_weight_max": 42.03243637084961,
      "activations/layer14_attention_weight_min": -38.745018005371094,
      "activations/layer15_attention_weight_max": 36.76376724243164,
      "activations/layer15_attention_weight_min": -36.61741256713867,
      "activations/layer16_attention_weight_max": 31.92976951599121,
      "activations/layer16_attention_weight_min": -28.569854736328125,
      "activations/layer17_attention_weight_max": 31.34716796875,
      "activations/layer17_attention_weight_min": -25.50919532775879,
      "activations/layer18_attention_weight_max": 34.04133224487305,
      "activations/layer18_attention_weight_min": -24.95557403564453,
      "activations/layer19_attention_weight_max": 34.63984680175781,
      "activations/layer19_attention_weight_min": -30.45545768737793,
      "activations/layer1_attention_weight_max": 15.671504020690918,
      "activations/layer1_attention_weight_min": -15.76716136932373,
      "activations/layer20_attention_weight_max": 29.068429946899414,
      "activations/layer20_attention_weight_min": -25.18016815185547,
      "activations/layer21_attention_weight_max": 29.775144577026367,
      "activations/layer21_attention_weight_min": -26.26421356201172,
      "activations/layer22_attention_weight_max": 49.59869384765625,
      "activations/layer22_attention_weight_min": -30.16604232788086,
      "activations/layer23_attention_weight_max": 34.41107177734375,
      "activations/layer23_attention_weight_min": -28.502933502197266,
      "activations/layer2_attention_weight_max": 34.111793518066406,
      "activations/layer2_attention_weight_min": -34.67011642456055,
      "activations/layer3_attention_weight_max": 98.34427642822266,
      "activations/layer3_attention_weight_min": -99.68692779541016,
      "activations/layer4_attention_weight_max": 110.10648345947266,
      "activations/layer4_attention_weight_min": -108.41328430175781,
      "activations/layer5_attention_weight_max": 49.48824691772461,
      "activations/layer5_attention_weight_min": -60.06547164916992,
      "activations/layer6_attention_weight_max": 43.84410858154297,
      "activations/layer6_attention_weight_min": -48.433570861816406,
      "activations/layer7_attention_weight_max": 90.6310806274414,
      "activations/layer7_attention_weight_min": -99.58284759521484,
      "activations/layer8_attention_weight_max": 39.647796630859375,
      "activations/layer8_attention_weight_min": -47.54314041137695,
      "activations/layer9_attention_weight_max": 30.222152709960938,
      "activations/layer9_attention_weight_min": -34.06875228881836,
      "epoch": 15.31,
      "learning_rate": 5.180454545454545e-05,
      "loss": 2.7214,
      "step": 263450
    },
    {
      "activations/layer0_attention_weight_max": 16.122817993164062,
      "activations/layer0_attention_weight_min": -13.83308219909668,
      "activations/layer10_attention_weight_max": 33.78509521484375,
      "activations/layer10_attention_weight_min": -34.710113525390625,
      "activations/layer11_attention_weight_max": 34.933692932128906,
      "activations/layer11_attention_weight_min": -33.57365417480469,
      "activations/layer12_attention_weight_max": 26.86652374267578,
      "activations/layer12_attention_weight_min": -27.4329776763916,
      "activations/layer13_attention_weight_max": 40.6247673034668,
      "activations/layer13_attention_weight_min": -37.66889572143555,
      "activations/layer14_attention_weight_max": 46.041751861572266,
      "activations/layer14_attention_weight_min": -40.1045036315918,
      "activations/layer15_attention_weight_max": 40.71533203125,
      "activations/layer15_attention_weight_min": -35.56423568725586,
      "activations/layer16_attention_weight_max": 30.79095458984375,
      "activations/layer16_attention_weight_min": -27.303852081298828,
      "activations/layer17_attention_weight_max": 31.46537208557129,
      "activations/layer17_attention_weight_min": -25.29313850402832,
      "activations/layer18_attention_weight_max": 32.29989242553711,
      "activations/layer18_attention_weight_min": -23.641353607177734,
      "activations/layer19_attention_weight_max": 34.56250762939453,
      "activations/layer19_attention_weight_min": -32.033382415771484,
      "activations/layer1_attention_weight_max": 15.994959831237793,
      "activations/layer1_attention_weight_min": -14.765158653259277,
      "activations/layer20_attention_weight_max": 32.042938232421875,
      "activations/layer20_attention_weight_min": -26.203880310058594,
      "activations/layer21_attention_weight_max": 29.55502700805664,
      "activations/layer21_attention_weight_min": -25.535232543945312,
      "activations/layer22_attention_weight_max": 42.46046447753906,
      "activations/layer22_attention_weight_min": -31.270511627197266,
      "activations/layer23_attention_weight_max": 33.4921875,
      "activations/layer23_attention_weight_min": -26.809783935546875,
      "activations/layer2_attention_weight_max": 32.48210906982422,
      "activations/layer2_attention_weight_min": -32.05369567871094,
      "activations/layer3_attention_weight_max": 94.90625,
      "activations/layer3_attention_weight_min": -97.2331314086914,
      "activations/layer4_attention_weight_max": 105.39173889160156,
      "activations/layer4_attention_weight_min": -112.88164520263672,
      "activations/layer5_attention_weight_max": 49.872520446777344,
      "activations/layer5_attention_weight_min": -62.65425109863281,
      "activations/layer6_attention_weight_max": 43.31505584716797,
      "activations/layer6_attention_weight_min": -44.974308013916016,
      "activations/layer7_attention_weight_max": 92.79313659667969,
      "activations/layer7_attention_weight_min": -88.45133209228516,
      "activations/layer8_attention_weight_max": 41.6395263671875,
      "activations/layer8_attention_weight_min": -41.88396453857422,
      "activations/layer9_attention_weight_max": 33.10525131225586,
      "activations/layer9_attention_weight_min": -34.189632415771484,
      "epoch": 15.31,
      "learning_rate": 5.178560606060606e-05,
      "loss": 2.7577,
      "step": 263500
    },
    {
      "activations/layer0_attention_weight_max": 16.870473861694336,
      "activations/layer0_attention_weight_min": -12.737288475036621,
      "activations/layer10_attention_weight_max": 33.766883850097656,
      "activations/layer10_attention_weight_min": -33.29622268676758,
      "activations/layer11_attention_weight_max": 32.87303161621094,
      "activations/layer11_attention_weight_min": -33.5811767578125,
      "activations/layer12_attention_weight_max": 25.089515686035156,
      "activations/layer12_attention_weight_min": -26.953073501586914,
      "activations/layer13_attention_weight_max": 38.7213134765625,
      "activations/layer13_attention_weight_min": -35.77236557006836,
      "activations/layer14_attention_weight_max": 38.25996398925781,
      "activations/layer14_attention_weight_min": -39.292694091796875,
      "activations/layer15_attention_weight_max": 35.74720764160156,
      "activations/layer15_attention_weight_min": -35.908329010009766,
      "activations/layer16_attention_weight_max": 30.193920135498047,
      "activations/layer16_attention_weight_min": -28.2879638671875,
      "activations/layer17_attention_weight_max": 29.463775634765625,
      "activations/layer17_attention_weight_min": -27.635360717773438,
      "activations/layer18_attention_weight_max": 29.546329498291016,
      "activations/layer18_attention_weight_min": -27.337154388427734,
      "activations/layer19_attention_weight_max": 33.20673751831055,
      "activations/layer19_attention_weight_min": -31.37369728088379,
      "activations/layer1_attention_weight_max": 16.229217529296875,
      "activations/layer1_attention_weight_min": -14.105196952819824,
      "activations/layer20_attention_weight_max": 28.356937408447266,
      "activations/layer20_attention_weight_min": -26.024362564086914,
      "activations/layer21_attention_weight_max": 26.835891723632812,
      "activations/layer21_attention_weight_min": -25.129526138305664,
      "activations/layer22_attention_weight_max": 41.15614318847656,
      "activations/layer22_attention_weight_min": -32.42750930786133,
      "activations/layer23_attention_weight_max": 32.16640853881836,
      "activations/layer23_attention_weight_min": -27.84160614013672,
      "activations/layer2_attention_weight_max": 31.69569206237793,
      "activations/layer2_attention_weight_min": -32.877994537353516,
      "activations/layer3_attention_weight_max": 94.22832489013672,
      "activations/layer3_attention_weight_min": -94.8901138305664,
      "activations/layer4_attention_weight_max": 109.06070709228516,
      "activations/layer4_attention_weight_min": -109.3315658569336,
      "activations/layer5_attention_weight_max": 50.0224723815918,
      "activations/layer5_attention_weight_min": -60.045440673828125,
      "activations/layer6_attention_weight_max": 44.55073165893555,
      "activations/layer6_attention_weight_min": -45.199241638183594,
      "activations/layer7_attention_weight_max": 83.38761138916016,
      "activations/layer7_attention_weight_min": -90.42408752441406,
      "activations/layer8_attention_weight_max": 40.53541564941406,
      "activations/layer8_attention_weight_min": -44.96327590942383,
      "activations/layer9_attention_weight_max": 30.45538902282715,
      "activations/layer9_attention_weight_min": -33.1023063659668,
      "epoch": 15.31,
      "learning_rate": 5.176666666666666e-05,
      "loss": 2.758,
      "step": 263550
    },
    {
      "activations/layer0_attention_weight_max": 16.340978622436523,
      "activations/layer0_attention_weight_min": -12.039837837219238,
      "activations/layer10_attention_weight_max": 31.037405014038086,
      "activations/layer10_attention_weight_min": -31.234439849853516,
      "activations/layer11_attention_weight_max": 32.1255989074707,
      "activations/layer11_attention_weight_min": -33.25390625,
      "activations/layer12_attention_weight_max": 24.779733657836914,
      "activations/layer12_attention_weight_min": -24.359405517578125,
      "activations/layer13_attention_weight_max": 34.514442443847656,
      "activations/layer13_attention_weight_min": -33.73966979980469,
      "activations/layer14_attention_weight_max": 37.45087814331055,
      "activations/layer14_attention_weight_min": -36.47210693359375,
      "activations/layer15_attention_weight_max": 36.01472854614258,
      "activations/layer15_attention_weight_min": -36.45680236816406,
      "activations/layer16_attention_weight_max": 27.91477394104004,
      "activations/layer16_attention_weight_min": -27.619413375854492,
      "activations/layer17_attention_weight_max": 29.82807159423828,
      "activations/layer17_attention_weight_min": -28.28592872619629,
      "activations/layer18_attention_weight_max": 31.237628936767578,
      "activations/layer18_attention_weight_min": -26.099822998046875,
      "activations/layer19_attention_weight_max": 31.80109977722168,
      "activations/layer19_attention_weight_min": -30.917272567749023,
      "activations/layer1_attention_weight_max": 18.237194061279297,
      "activations/layer1_attention_weight_min": -16.346454620361328,
      "activations/layer20_attention_weight_max": 26.017005920410156,
      "activations/layer20_attention_weight_min": -24.540969848632812,
      "activations/layer21_attention_weight_max": 26.166433334350586,
      "activations/layer21_attention_weight_min": -25.001510620117188,
      "activations/layer22_attention_weight_max": 44.331787109375,
      "activations/layer22_attention_weight_min": -33.30354690551758,
      "activations/layer23_attention_weight_max": 34.25953674316406,
      "activations/layer23_attention_weight_min": -26.432632446289062,
      "activations/layer2_attention_weight_max": 34.807037353515625,
      "activations/layer2_attention_weight_min": -33.078208923339844,
      "activations/layer3_attention_weight_max": 98.1897964477539,
      "activations/layer3_attention_weight_min": -93.63002014160156,
      "activations/layer4_attention_weight_max": 111.0743637084961,
      "activations/layer4_attention_weight_min": -105.50836181640625,
      "activations/layer5_attention_weight_max": 49.09368133544922,
      "activations/layer5_attention_weight_min": -62.87171936035156,
      "activations/layer6_attention_weight_max": 44.89210510253906,
      "activations/layer6_attention_weight_min": -43.773231506347656,
      "activations/layer7_attention_weight_max": 88.67666625976562,
      "activations/layer7_attention_weight_min": -86.7055435180664,
      "activations/layer8_attention_weight_max": 39.63777160644531,
      "activations/layer8_attention_weight_min": -41.37411880493164,
      "activations/layer9_attention_weight_max": 31.80353355407715,
      "activations/layer9_attention_weight_min": -31.890390396118164,
      "epoch": 15.32,
      "learning_rate": 5.174772727272727e-05,
      "loss": 2.7517,
      "step": 263600
    },
    {
      "activations/layer0_attention_weight_max": 15.68369197845459,
      "activations/layer0_attention_weight_min": -11.953474044799805,
      "activations/layer10_attention_weight_max": 32.42201614379883,
      "activations/layer10_attention_weight_min": -34.408843994140625,
      "activations/layer11_attention_weight_max": 31.501781463623047,
      "activations/layer11_attention_weight_min": -35.98312759399414,
      "activations/layer12_attention_weight_max": 26.73552894592285,
      "activations/layer12_attention_weight_min": -26.424955368041992,
      "activations/layer13_attention_weight_max": 40.120941162109375,
      "activations/layer13_attention_weight_min": -36.348114013671875,
      "activations/layer14_attention_weight_max": 48.649452209472656,
      "activations/layer14_attention_weight_min": -42.87302780151367,
      "activations/layer15_attention_weight_max": 43.29558563232422,
      "activations/layer15_attention_weight_min": -38.25041961669922,
      "activations/layer16_attention_weight_max": 33.08225631713867,
      "activations/layer16_attention_weight_min": -28.81245231628418,
      "activations/layer17_attention_weight_max": 34.73004150390625,
      "activations/layer17_attention_weight_min": -29.372528076171875,
      "activations/layer18_attention_weight_max": 34.00907516479492,
      "activations/layer18_attention_weight_min": -26.610633850097656,
      "activations/layer19_attention_weight_max": 37.792396545410156,
      "activations/layer19_attention_weight_min": -31.766223907470703,
      "activations/layer1_attention_weight_max": 16.132190704345703,
      "activations/layer1_attention_weight_min": -14.387438774108887,
      "activations/layer20_attention_weight_max": 31.24888801574707,
      "activations/layer20_attention_weight_min": -26.74721908569336,
      "activations/layer21_attention_weight_max": 32.395668029785156,
      "activations/layer21_attention_weight_min": -25.2814884185791,
      "activations/layer22_attention_weight_max": 41.62709426879883,
      "activations/layer22_attention_weight_min": -32.14481735229492,
      "activations/layer23_attention_weight_max": 32.82844543457031,
      "activations/layer23_attention_weight_min": -26.58258819580078,
      "activations/layer2_attention_weight_max": 32.444217681884766,
      "activations/layer2_attention_weight_min": -32.061851501464844,
      "activations/layer3_attention_weight_max": 95.3531265258789,
      "activations/layer3_attention_weight_min": -94.72962951660156,
      "activations/layer4_attention_weight_max": 105.93157958984375,
      "activations/layer4_attention_weight_min": -105.8672866821289,
      "activations/layer5_attention_weight_max": 50.9306755065918,
      "activations/layer5_attention_weight_min": -63.63802719116211,
      "activations/layer6_attention_weight_max": 41.48896026611328,
      "activations/layer6_attention_weight_min": -45.72651672363281,
      "activations/layer7_attention_weight_max": 90.10848999023438,
      "activations/layer7_attention_weight_min": -89.96434783935547,
      "activations/layer8_attention_weight_max": 40.46877670288086,
      "activations/layer8_attention_weight_min": -40.25189208984375,
      "activations/layer9_attention_weight_max": 30.847074508666992,
      "activations/layer9_attention_weight_min": -33.488948822021484,
      "epoch": 15.32,
      "learning_rate": 5.1728787878787877e-05,
      "loss": 2.7555,
      "step": 263650
    },
    {
      "activations/layer0_attention_weight_max": 16.197359085083008,
      "activations/layer0_attention_weight_min": -11.782713890075684,
      "activations/layer10_attention_weight_max": 32.817569732666016,
      "activations/layer10_attention_weight_min": -32.06528854370117,
      "activations/layer11_attention_weight_max": 32.513465881347656,
      "activations/layer11_attention_weight_min": -32.27558135986328,
      "activations/layer12_attention_weight_max": 24.57388687133789,
      "activations/layer12_attention_weight_min": -27.12244987487793,
      "activations/layer13_attention_weight_max": 36.94423294067383,
      "activations/layer13_attention_weight_min": -35.1403923034668,
      "activations/layer14_attention_weight_max": 40.785518646240234,
      "activations/layer14_attention_weight_min": -38.18062210083008,
      "activations/layer15_attention_weight_max": 37.303462982177734,
      "activations/layer15_attention_weight_min": -35.794979095458984,
      "activations/layer16_attention_weight_max": 29.248918533325195,
      "activations/layer16_attention_weight_min": -29.06103515625,
      "activations/layer17_attention_weight_max": 29.9154052734375,
      "activations/layer17_attention_weight_min": -27.98463249206543,
      "activations/layer18_attention_weight_max": 31.281835556030273,
      "activations/layer18_attention_weight_min": -26.166332244873047,
      "activations/layer19_attention_weight_max": 34.75968551635742,
      "activations/layer19_attention_weight_min": -31.762149810791016,
      "activations/layer1_attention_weight_max": 16.055498123168945,
      "activations/layer1_attention_weight_min": -15.013668060302734,
      "activations/layer20_attention_weight_max": 28.44402313232422,
      "activations/layer20_attention_weight_min": -25.089637756347656,
      "activations/layer21_attention_weight_max": 28.13936424255371,
      "activations/layer21_attention_weight_min": -24.161834716796875,
      "activations/layer22_attention_weight_max": 48.29056167602539,
      "activations/layer22_attention_weight_min": -30.293277740478516,
      "activations/layer23_attention_weight_max": 34.93580627441406,
      "activations/layer23_attention_weight_min": -23.958194732666016,
      "activations/layer2_attention_weight_max": 33.17133712768555,
      "activations/layer2_attention_weight_min": -30.72560691833496,
      "activations/layer3_attention_weight_max": 94.18763732910156,
      "activations/layer3_attention_weight_min": -93.94314575195312,
      "activations/layer4_attention_weight_max": 105.15138244628906,
      "activations/layer4_attention_weight_min": -102.66851043701172,
      "activations/layer5_attention_weight_max": 49.306190490722656,
      "activations/layer5_attention_weight_min": -59.725730895996094,
      "activations/layer6_attention_weight_max": 42.349727630615234,
      "activations/layer6_attention_weight_min": -42.183753967285156,
      "activations/layer7_attention_weight_max": 88.89474487304688,
      "activations/layer7_attention_weight_min": -86.91080474853516,
      "activations/layer8_attention_weight_max": 40.93887710571289,
      "activations/layer8_attention_weight_min": -42.39020919799805,
      "activations/layer9_attention_weight_max": 33.44021224975586,
      "activations/layer9_attention_weight_min": -32.81292724609375,
      "epoch": 15.32,
      "learning_rate": 5.170984848484848e-05,
      "loss": 2.7268,
      "step": 263700
    },
    {
      "activations/layer0_attention_weight_max": 15.18472957611084,
      "activations/layer0_attention_weight_min": -12.120722770690918,
      "activations/layer10_attention_weight_max": 38.28840255737305,
      "activations/layer10_attention_weight_min": -36.557823181152344,
      "activations/layer11_attention_weight_max": 36.377891540527344,
      "activations/layer11_attention_weight_min": -36.22724151611328,
      "activations/layer12_attention_weight_max": 26.819665908813477,
      "activations/layer12_attention_weight_min": -26.04595375061035,
      "activations/layer13_attention_weight_max": 37.80613708496094,
      "activations/layer13_attention_weight_min": -36.83827590942383,
      "activations/layer14_attention_weight_max": 42.45305252075195,
      "activations/layer14_attention_weight_min": -41.205291748046875,
      "activations/layer15_attention_weight_max": 37.3145751953125,
      "activations/layer15_attention_weight_min": -35.733619689941406,
      "activations/layer16_attention_weight_max": 28.22063446044922,
      "activations/layer16_attention_weight_min": -29.93327522277832,
      "activations/layer17_attention_weight_max": 29.19974708557129,
      "activations/layer17_attention_weight_min": -27.010717391967773,
      "activations/layer18_attention_weight_max": 29.997947692871094,
      "activations/layer18_attention_weight_min": -24.649097442626953,
      "activations/layer19_attention_weight_max": 33.46421813964844,
      "activations/layer19_attention_weight_min": -34.72056198120117,
      "activations/layer1_attention_weight_max": 16.543434143066406,
      "activations/layer1_attention_weight_min": -15.378534317016602,
      "activations/layer20_attention_weight_max": 27.922922134399414,
      "activations/layer20_attention_weight_min": -25.283140182495117,
      "activations/layer21_attention_weight_max": 30.487436294555664,
      "activations/layer21_attention_weight_min": -24.47581672668457,
      "activations/layer22_attention_weight_max": 47.04360580444336,
      "activations/layer22_attention_weight_min": -32.253055572509766,
      "activations/layer23_attention_weight_max": 35.67580795288086,
      "activations/layer23_attention_weight_min": -26.29159164428711,
      "activations/layer2_attention_weight_max": 33.30154800415039,
      "activations/layer2_attention_weight_min": -32.957908630371094,
      "activations/layer3_attention_weight_max": 102.07665252685547,
      "activations/layer3_attention_weight_min": -101.85789489746094,
      "activations/layer4_attention_weight_max": 115.89134979248047,
      "activations/layer4_attention_weight_min": -113.6893539428711,
      "activations/layer5_attention_weight_max": 53.97626876831055,
      "activations/layer5_attention_weight_min": -62.32280731201172,
      "activations/layer6_attention_weight_max": 45.349403381347656,
      "activations/layer6_attention_weight_min": -49.00046157836914,
      "activations/layer7_attention_weight_max": 93.9007339477539,
      "activations/layer7_attention_weight_min": -104.07610321044922,
      "activations/layer8_attention_weight_max": 45.041595458984375,
      "activations/layer8_attention_weight_min": -46.04195022583008,
      "activations/layer9_attention_weight_max": 35.58539962768555,
      "activations/layer9_attention_weight_min": -36.2708854675293,
      "epoch": 15.33,
      "learning_rate": 5.169090909090908e-05,
      "loss": 2.7558,
      "step": 263750
    },
    {
      "activations/layer0_attention_weight_max": 16.89335823059082,
      "activations/layer0_attention_weight_min": -11.83389949798584,
      "activations/layer10_attention_weight_max": 34.97922897338867,
      "activations/layer10_attention_weight_min": -34.17774200439453,
      "activations/layer11_attention_weight_max": 34.99940490722656,
      "activations/layer11_attention_weight_min": -34.344364166259766,
      "activations/layer12_attention_weight_max": 26.929340362548828,
      "activations/layer12_attention_weight_min": -25.05343246459961,
      "activations/layer13_attention_weight_max": 39.14268493652344,
      "activations/layer13_attention_weight_min": -34.2507438659668,
      "activations/layer14_attention_weight_max": 41.29640197753906,
      "activations/layer14_attention_weight_min": -37.39665985107422,
      "activations/layer15_attention_weight_max": 38.782005310058594,
      "activations/layer15_attention_weight_min": -33.331417083740234,
      "activations/layer16_attention_weight_max": 32.31223678588867,
      "activations/layer16_attention_weight_min": -26.86570930480957,
      "activations/layer17_attention_weight_max": 32.56208801269531,
      "activations/layer17_attention_weight_min": -24.603206634521484,
      "activations/layer18_attention_weight_max": 33.75387954711914,
      "activations/layer18_attention_weight_min": -25.315811157226562,
      "activations/layer19_attention_weight_max": 31.7003173828125,
      "activations/layer19_attention_weight_min": -32.13570785522461,
      "activations/layer1_attention_weight_max": 15.816826820373535,
      "activations/layer1_attention_weight_min": -14.863189697265625,
      "activations/layer20_attention_weight_max": 30.385900497436523,
      "activations/layer20_attention_weight_min": -24.818490982055664,
      "activations/layer21_attention_weight_max": 29.66069221496582,
      "activations/layer21_attention_weight_min": -24.810033798217773,
      "activations/layer22_attention_weight_max": 40.767967224121094,
      "activations/layer22_attention_weight_min": -31.52367401123047,
      "activations/layer23_attention_weight_max": 36.216983795166016,
      "activations/layer23_attention_weight_min": -26.038549423217773,
      "activations/layer2_attention_weight_max": 32.21729278564453,
      "activations/layer2_attention_weight_min": -32.15021896362305,
      "activations/layer3_attention_weight_max": 95.30935668945312,
      "activations/layer3_attention_weight_min": -96.85708618164062,
      "activations/layer4_attention_weight_max": 109.86217498779297,
      "activations/layer4_attention_weight_min": -113.16400146484375,
      "activations/layer5_attention_weight_max": 50.4884033203125,
      "activations/layer5_attention_weight_min": -64.42039489746094,
      "activations/layer6_attention_weight_max": 43.78593063354492,
      "activations/layer6_attention_weight_min": -45.81118392944336,
      "activations/layer7_attention_weight_max": 88.02708435058594,
      "activations/layer7_attention_weight_min": -92.87136840820312,
      "activations/layer8_attention_weight_max": 40.90418243408203,
      "activations/layer8_attention_weight_min": -41.1579475402832,
      "activations/layer9_attention_weight_max": 32.176456451416016,
      "activations/layer9_attention_weight_min": -33.10940170288086,
      "epoch": 15.33,
      "learning_rate": 5.1671969696969695e-05,
      "loss": 2.7511,
      "step": 263800
    },
    {
      "activations/layer0_attention_weight_max": 15.442078590393066,
      "activations/layer0_attention_weight_min": -12.046736717224121,
      "activations/layer10_attention_weight_max": 34.92403030395508,
      "activations/layer10_attention_weight_min": -35.55073928833008,
      "activations/layer11_attention_weight_max": 33.00911331176758,
      "activations/layer11_attention_weight_min": -34.02854919433594,
      "activations/layer12_attention_weight_max": 27.813358306884766,
      "activations/layer12_attention_weight_min": -26.1572208404541,
      "activations/layer13_attention_weight_max": 42.18890380859375,
      "activations/layer13_attention_weight_min": -36.42760467529297,
      "activations/layer14_attention_weight_max": 43.67807388305664,
      "activations/layer14_attention_weight_min": -38.15163040161133,
      "activations/layer15_attention_weight_max": 40.40068817138672,
      "activations/layer15_attention_weight_min": -34.90019989013672,
      "activations/layer16_attention_weight_max": 32.44258117675781,
      "activations/layer16_attention_weight_min": -27.31100082397461,
      "activations/layer17_attention_weight_max": 29.203426361083984,
      "activations/layer17_attention_weight_min": -26.73414421081543,
      "activations/layer18_attention_weight_max": 31.6900577545166,
      "activations/layer18_attention_weight_min": -24.934844970703125,
      "activations/layer19_attention_weight_max": 34.837093353271484,
      "activations/layer19_attention_weight_min": -30.84162712097168,
      "activations/layer1_attention_weight_max": 17.72177505493164,
      "activations/layer1_attention_weight_min": -14.975852012634277,
      "activations/layer20_attention_weight_max": 30.1099910736084,
      "activations/layer20_attention_weight_min": -24.519886016845703,
      "activations/layer21_attention_weight_max": 26.760513305664062,
      "activations/layer21_attention_weight_min": -25.234384536743164,
      "activations/layer22_attention_weight_max": 46.01605224609375,
      "activations/layer22_attention_weight_min": -32.67348098754883,
      "activations/layer23_attention_weight_max": 35.020389556884766,
      "activations/layer23_attention_weight_min": -27.14931297302246,
      "activations/layer2_attention_weight_max": 34.40257263183594,
      "activations/layer2_attention_weight_min": -31.86357879638672,
      "activations/layer3_attention_weight_max": 100.40938568115234,
      "activations/layer3_attention_weight_min": -96.15921783447266,
      "activations/layer4_attention_weight_max": 106.9267807006836,
      "activations/layer4_attention_weight_min": -110.40262603759766,
      "activations/layer5_attention_weight_max": 51.96699523925781,
      "activations/layer5_attention_weight_min": -63.27980041503906,
      "activations/layer6_attention_weight_max": 43.051353454589844,
      "activations/layer6_attention_weight_min": -47.37738800048828,
      "activations/layer7_attention_weight_max": 87.92100524902344,
      "activations/layer7_attention_weight_min": -94.51005554199219,
      "activations/layer8_attention_weight_max": 39.96004867553711,
      "activations/layer8_attention_weight_min": -45.42243194580078,
      "activations/layer9_attention_weight_max": 31.3365478515625,
      "activations/layer9_attention_weight_min": -34.97127151489258,
      "epoch": 15.33,
      "learning_rate": 5.1653030303030296e-05,
      "loss": 2.7392,
      "step": 263850
    },
    {
      "activations/layer0_attention_weight_max": 15.76400375366211,
      "activations/layer0_attention_weight_min": -12.304719924926758,
      "activations/layer10_attention_weight_max": 34.90494918823242,
      "activations/layer10_attention_weight_min": -35.572025299072266,
      "activations/layer11_attention_weight_max": 34.7172966003418,
      "activations/layer11_attention_weight_min": -33.28578567504883,
      "activations/layer12_attention_weight_max": 30.713062286376953,
      "activations/layer12_attention_weight_min": -26.1734676361084,
      "activations/layer13_attention_weight_max": 41.97126388549805,
      "activations/layer13_attention_weight_min": -35.26429748535156,
      "activations/layer14_attention_weight_max": 45.9957389831543,
      "activations/layer14_attention_weight_min": -38.164207458496094,
      "activations/layer15_attention_weight_max": 41.56436538696289,
      "activations/layer15_attention_weight_min": -37.01654815673828,
      "activations/layer16_attention_weight_max": 30.736629486083984,
      "activations/layer16_attention_weight_min": -27.978412628173828,
      "activations/layer17_attention_weight_max": 31.80572509765625,
      "activations/layer17_attention_weight_min": -27.437469482421875,
      "activations/layer18_attention_weight_max": 37.37416458129883,
      "activations/layer18_attention_weight_min": -26.278167724609375,
      "activations/layer19_attention_weight_max": 35.349334716796875,
      "activations/layer19_attention_weight_min": -32.286956787109375,
      "activations/layer1_attention_weight_max": 17.828275680541992,
      "activations/layer1_attention_weight_min": -17.32830238342285,
      "activations/layer20_attention_weight_max": 30.57711410522461,
      "activations/layer20_attention_weight_min": -26.443042755126953,
      "activations/layer21_attention_weight_max": 29.879125595092773,
      "activations/layer21_attention_weight_min": -25.269195556640625,
      "activations/layer22_attention_weight_max": 49.70185089111328,
      "activations/layer22_attention_weight_min": -34.51530838012695,
      "activations/layer23_attention_weight_max": 35.177738189697266,
      "activations/layer23_attention_weight_min": -29.662656784057617,
      "activations/layer2_attention_weight_max": 33.67529296875,
      "activations/layer2_attention_weight_min": -32.43944549560547,
      "activations/layer3_attention_weight_max": 98.78712463378906,
      "activations/layer3_attention_weight_min": -99.43622589111328,
      "activations/layer4_attention_weight_max": 111.714111328125,
      "activations/layer4_attention_weight_min": -111.985595703125,
      "activations/layer5_attention_weight_max": 51.15247344970703,
      "activations/layer5_attention_weight_min": -65.19771575927734,
      "activations/layer6_attention_weight_max": 46.19536209106445,
      "activations/layer6_attention_weight_min": -46.666690826416016,
      "activations/layer7_attention_weight_max": 91.92521667480469,
      "activations/layer7_attention_weight_min": -93.06893920898438,
      "activations/layer8_attention_weight_max": 45.54087829589844,
      "activations/layer8_attention_weight_min": -45.7038688659668,
      "activations/layer9_attention_weight_max": 33.32870864868164,
      "activations/layer9_attention_weight_min": -36.60601806640625,
      "epoch": 15.33,
      "learning_rate": 5.1634090909090904e-05,
      "loss": 2.7458,
      "step": 263900
    },
    {
      "activations/layer0_attention_weight_max": 16.174949645996094,
      "activations/layer0_attention_weight_min": -12.359771728515625,
      "activations/layer10_attention_weight_max": 31.767845153808594,
      "activations/layer10_attention_weight_min": -31.916540145874023,
      "activations/layer11_attention_weight_max": 32.257606506347656,
      "activations/layer11_attention_weight_min": -33.77513885498047,
      "activations/layer12_attention_weight_max": 28.111129760742188,
      "activations/layer12_attention_weight_min": -25.891193389892578,
      "activations/layer13_attention_weight_max": 41.16331100463867,
      "activations/layer13_attention_weight_min": -35.2243766784668,
      "activations/layer14_attention_weight_max": 42.60639953613281,
      "activations/layer14_attention_weight_min": -38.21201705932617,
      "activations/layer15_attention_weight_max": 38.78512191772461,
      "activations/layer15_attention_weight_min": -36.78852081298828,
      "activations/layer16_attention_weight_max": 32.36509323120117,
      "activations/layer16_attention_weight_min": -27.40007972717285,
      "activations/layer17_attention_weight_max": 32.15227127075195,
      "activations/layer17_attention_weight_min": -25.86363983154297,
      "activations/layer18_attention_weight_max": 35.308570861816406,
      "activations/layer18_attention_weight_min": -26.70796012878418,
      "activations/layer19_attention_weight_max": 37.83154296875,
      "activations/layer19_attention_weight_min": -30.9232177734375,
      "activations/layer1_attention_weight_max": 16.534400939941406,
      "activations/layer1_attention_weight_min": -15.565817832946777,
      "activations/layer20_attention_weight_max": 30.954151153564453,
      "activations/layer20_attention_weight_min": -25.651329040527344,
      "activations/layer21_attention_weight_max": 28.57796287536621,
      "activations/layer21_attention_weight_min": -25.695987701416016,
      "activations/layer22_attention_weight_max": 46.81831741333008,
      "activations/layer22_attention_weight_min": -34.193363189697266,
      "activations/layer23_attention_weight_max": 37.41592025756836,
      "activations/layer23_attention_weight_min": -28.24334716796875,
      "activations/layer2_attention_weight_max": 33.125770568847656,
      "activations/layer2_attention_weight_min": -33.902984619140625,
      "activations/layer3_attention_weight_max": 98.51219940185547,
      "activations/layer3_attention_weight_min": -100.56375122070312,
      "activations/layer4_attention_weight_max": 109.07026672363281,
      "activations/layer4_attention_weight_min": -109.41017150878906,
      "activations/layer5_attention_weight_max": 50.39959716796875,
      "activations/layer5_attention_weight_min": -60.69193649291992,
      "activations/layer6_attention_weight_max": 42.97978973388672,
      "activations/layer6_attention_weight_min": -45.37495422363281,
      "activations/layer7_attention_weight_max": 87.5702133178711,
      "activations/layer7_attention_weight_min": -88.22058868408203,
      "activations/layer8_attention_weight_max": 37.931331634521484,
      "activations/layer8_attention_weight_min": -39.54557418823242,
      "activations/layer9_attention_weight_max": 30.627878189086914,
      "activations/layer9_attention_weight_min": -32.09869384765625,
      "epoch": 15.34,
      "learning_rate": 5.1615151515151506e-05,
      "loss": 2.7307,
      "step": 263950
    },
    {
      "activations/layer0_attention_weight_max": 16.11833381652832,
      "activations/layer0_attention_weight_min": -11.98279857635498,
      "activations/layer10_attention_weight_max": 33.91621780395508,
      "activations/layer10_attention_weight_min": -32.18905258178711,
      "activations/layer11_attention_weight_max": 33.3231315612793,
      "activations/layer11_attention_weight_min": -31.375770568847656,
      "activations/layer12_attention_weight_max": 23.880319595336914,
      "activations/layer12_attention_weight_min": -25.988117218017578,
      "activations/layer13_attention_weight_max": 35.6905517578125,
      "activations/layer13_attention_weight_min": -34.25960922241211,
      "activations/layer14_attention_weight_max": 35.9013557434082,
      "activations/layer14_attention_weight_min": -35.77140808105469,
      "activations/layer15_attention_weight_max": 31.818866729736328,
      "activations/layer15_attention_weight_min": -33.532772064208984,
      "activations/layer16_attention_weight_max": 26.678255081176758,
      "activations/layer16_attention_weight_min": -26.86263084411621,
      "activations/layer17_attention_weight_max": 32.153907775878906,
      "activations/layer17_attention_weight_min": -24.53728675842285,
      "activations/layer18_attention_weight_max": 30.01902961730957,
      "activations/layer18_attention_weight_min": -25.041757583618164,
      "activations/layer19_attention_weight_max": 30.469694137573242,
      "activations/layer19_attention_weight_min": -31.688526153564453,
      "activations/layer1_attention_weight_max": 16.09701919555664,
      "activations/layer1_attention_weight_min": -15.650816917419434,
      "activations/layer20_attention_weight_max": 29.805580139160156,
      "activations/layer20_attention_weight_min": -26.880754470825195,
      "activations/layer21_attention_weight_max": 27.204496383666992,
      "activations/layer21_attention_weight_min": -25.072662353515625,
      "activations/layer22_attention_weight_max": 42.5291633605957,
      "activations/layer22_attention_weight_min": -30.52967071533203,
      "activations/layer23_attention_weight_max": 34.595706939697266,
      "activations/layer23_attention_weight_min": -24.945478439331055,
      "activations/layer2_attention_weight_max": 33.29405212402344,
      "activations/layer2_attention_weight_min": -32.84990692138672,
      "activations/layer3_attention_weight_max": 96.4098892211914,
      "activations/layer3_attention_weight_min": -99.83270263671875,
      "activations/layer4_attention_weight_max": 109.25569152832031,
      "activations/layer4_attention_weight_min": -111.6401138305664,
      "activations/layer5_attention_weight_max": 49.728878021240234,
      "activations/layer5_attention_weight_min": -60.68633270263672,
      "activations/layer6_attention_weight_max": 43.49009704589844,
      "activations/layer6_attention_weight_min": -47.67570114135742,
      "activations/layer7_attention_weight_max": 86.32781219482422,
      "activations/layer7_attention_weight_min": -85.80731201171875,
      "activations/layer8_attention_weight_max": 38.6820068359375,
      "activations/layer8_attention_weight_min": -41.5246696472168,
      "activations/layer9_attention_weight_max": 29.337568283081055,
      "activations/layer9_attention_weight_min": -32.67390060424805,
      "epoch": 15.34,
      "learning_rate": 5.159621212121212e-05,
      "loss": 2.7484,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.5387,
      "eval_samples_per_second": 502.89,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.5387,
      "eval_openwebtext_samples_per_second": 502.89,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 2.003,
      "eval_wikitext_samples_per_second": 227.655,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_lambada_loss": 2.392578125,
      "eval_lambada_ppl": 10.941666595318791,
      "eval_lambada_runtime": 9.5871,
      "eval_lambada_samples_per_second": 507.87,
      "step": 264000
    },
    {
      "activations/layer0_attention_weight_max": 16.46401596069336,
      "activations/layer0_attention_weight_min": -12.648090362548828,
      "activations/layer10_attention_weight_max": 33.33610916137695,
      "activations/layer10_attention_weight_min": -34.1268196105957,
      "activations/layer11_attention_weight_max": 33.2081413269043,
      "activations/layer11_attention_weight_min": -35.3630256652832,
      "activations/layer12_attention_weight_max": 26.987205505371094,
      "activations/layer12_attention_weight_min": -25.395099639892578,
      "activations/layer13_attention_weight_max": 39.15165710449219,
      "activations/layer13_attention_weight_min": -34.689605712890625,
      "activations/layer14_attention_weight_max": 42.92750549316406,
      "activations/layer14_attention_weight_min": -38.887176513671875,
      "activations/layer15_attention_weight_max": 38.597084045410156,
      "activations/layer15_attention_weight_min": -36.07313537597656,
      "activations/layer16_attention_weight_max": 30.241580963134766,
      "activations/layer16_attention_weight_min": -28.77922248840332,
      "activations/layer17_attention_weight_max": 37.3249397277832,
      "activations/layer17_attention_weight_min": -27.532140731811523,
      "activations/layer18_attention_weight_max": 33.317626953125,
      "activations/layer18_attention_weight_min": -24.86241912841797,
      "activations/layer19_attention_weight_max": 35.998756408691406,
      "activations/layer19_attention_weight_min": -32.56063461303711,
      "activations/layer1_attention_weight_max": 17.343576431274414,
      "activations/layer1_attention_weight_min": -16.40797996520996,
      "activations/layer20_attention_weight_max": 31.69530487060547,
      "activations/layer20_attention_weight_min": -26.5727596282959,
      "activations/layer21_attention_weight_max": 31.618528366088867,
      "activations/layer21_attention_weight_min": -25.658140182495117,
      "activations/layer22_attention_weight_max": 45.738224029541016,
      "activations/layer22_attention_weight_min": -35.29469680786133,
      "activations/layer23_attention_weight_max": 36.4855842590332,
      "activations/layer23_attention_weight_min": -26.568647384643555,
      "activations/layer2_attention_weight_max": 33.7651481628418,
      "activations/layer2_attention_weight_min": -33.015811920166016,
      "activations/layer3_attention_weight_max": 97.89532470703125,
      "activations/layer3_attention_weight_min": -99.42465209960938,
      "activations/layer4_attention_weight_max": 113.12999725341797,
      "activations/layer4_attention_weight_min": -111.76018524169922,
      "activations/layer5_attention_weight_max": 51.808067321777344,
      "activations/layer5_attention_weight_min": -62.12928009033203,
      "activations/layer6_attention_weight_max": 42.574607849121094,
      "activations/layer6_attention_weight_min": -46.644710540771484,
      "activations/layer7_attention_weight_max": 92.63754272460938,
      "activations/layer7_attention_weight_min": -90.92436981201172,
      "activations/layer8_attention_weight_max": 44.042877197265625,
      "activations/layer8_attention_weight_min": -43.136470794677734,
      "activations/layer9_attention_weight_max": 30.79953956604004,
      "activations/layer9_attention_weight_min": -33.64937973022461,
      "epoch": 15.34,
      "learning_rate": 5.157727272727272e-05,
      "loss": 2.7491,
      "step": 264050
    },
    {
      "activations/layer0_attention_weight_max": 16.541139602661133,
      "activations/layer0_attention_weight_min": -13.170398712158203,
      "activations/layer10_attention_weight_max": 39.68852615356445,
      "activations/layer10_attention_weight_min": -37.613487243652344,
      "activations/layer11_attention_weight_max": 40.613922119140625,
      "activations/layer11_attention_weight_min": -39.56507873535156,
      "activations/layer12_attention_weight_max": 27.000507354736328,
      "activations/layer12_attention_weight_min": -25.38260269165039,
      "activations/layer13_attention_weight_max": 39.1787223815918,
      "activations/layer13_attention_weight_min": -33.37959671020508,
      "activations/layer14_attention_weight_max": 40.82479476928711,
      "activations/layer14_attention_weight_min": -35.745941162109375,
      "activations/layer15_attention_weight_max": 38.15817642211914,
      "activations/layer15_attention_weight_min": -34.74648666381836,
      "activations/layer16_attention_weight_max": 29.99323081970215,
      "activations/layer16_attention_weight_min": -27.25261116027832,
      "activations/layer17_attention_weight_max": 27.650129318237305,
      "activations/layer17_attention_weight_min": -25.84402084350586,
      "activations/layer18_attention_weight_max": 30.528663635253906,
      "activations/layer18_attention_weight_min": -24.973302841186523,
      "activations/layer19_attention_weight_max": 35.2310791015625,
      "activations/layer19_attention_weight_min": -31.589557647705078,
      "activations/layer1_attention_weight_max": 17.30605697631836,
      "activations/layer1_attention_weight_min": -14.401093482971191,
      "activations/layer20_attention_weight_max": 30.579133987426758,
      "activations/layer20_attention_weight_min": -25.161418914794922,
      "activations/layer21_attention_weight_max": 27.058067321777344,
      "activations/layer21_attention_weight_min": -25.035226821899414,
      "activations/layer22_attention_weight_max": 41.720542907714844,
      "activations/layer22_attention_weight_min": -30.741682052612305,
      "activations/layer23_attention_weight_max": 30.07233428955078,
      "activations/layer23_attention_weight_min": -23.78363037109375,
      "activations/layer2_attention_weight_max": 33.15754699707031,
      "activations/layer2_attention_weight_min": -31.599075317382812,
      "activations/layer3_attention_weight_max": 101.06428527832031,
      "activations/layer3_attention_weight_min": -96.61888885498047,
      "activations/layer4_attention_weight_max": 114.6898422241211,
      "activations/layer4_attention_weight_min": -110.76749420166016,
      "activations/layer5_attention_weight_max": 48.731361389160156,
      "activations/layer5_attention_weight_min": -62.311309814453125,
      "activations/layer6_attention_weight_max": 42.48063278198242,
      "activations/layer6_attention_weight_min": -44.974491119384766,
      "activations/layer7_attention_weight_max": 94.13307189941406,
      "activations/layer7_attention_weight_min": -91.77848815917969,
      "activations/layer8_attention_weight_max": 44.0886116027832,
      "activations/layer8_attention_weight_min": -42.282005310058594,
      "activations/layer9_attention_weight_max": 34.9714241027832,
      "activations/layer9_attention_weight_min": -37.28371047973633,
      "epoch": 15.35,
      "learning_rate": 5.1558333333333324e-05,
      "loss": 2.7376,
      "step": 264100
    },
    {
      "activations/layer0_attention_weight_max": 16.347394943237305,
      "activations/layer0_attention_weight_min": -12.282197952270508,
      "activations/layer10_attention_weight_max": 35.0229377746582,
      "activations/layer10_attention_weight_min": -36.341182708740234,
      "activations/layer11_attention_weight_max": 35.95268249511719,
      "activations/layer11_attention_weight_min": -37.08250427246094,
      "activations/layer12_attention_weight_max": 28.53750991821289,
      "activations/layer12_attention_weight_min": -26.915876388549805,
      "activations/layer13_attention_weight_max": 40.474056243896484,
      "activations/layer13_attention_weight_min": -37.75190353393555,
      "activations/layer14_attention_weight_max": 46.21849060058594,
      "activations/layer14_attention_weight_min": -42.447731018066406,
      "activations/layer15_attention_weight_max": 38.21381759643555,
      "activations/layer15_attention_weight_min": -36.16942596435547,
      "activations/layer16_attention_weight_max": 29.759397506713867,
      "activations/layer16_attention_weight_min": -28.034181594848633,
      "activations/layer17_attention_weight_max": 30.64089584350586,
      "activations/layer17_attention_weight_min": -26.78078842163086,
      "activations/layer18_attention_weight_max": 31.982839584350586,
      "activations/layer18_attention_weight_min": -26.249406814575195,
      "activations/layer19_attention_weight_max": 34.98940658569336,
      "activations/layer19_attention_weight_min": -31.377857208251953,
      "activations/layer1_attention_weight_max": 15.950519561767578,
      "activations/layer1_attention_weight_min": -16.168119430541992,
      "activations/layer20_attention_weight_max": 32.495906829833984,
      "activations/layer20_attention_weight_min": -25.067747116088867,
      "activations/layer21_attention_weight_max": 30.14866828918457,
      "activations/layer21_attention_weight_min": -24.026113510131836,
      "activations/layer22_attention_weight_max": 43.724552154541016,
      "activations/layer22_attention_weight_min": -30.619903564453125,
      "activations/layer23_attention_weight_max": 31.925182342529297,
      "activations/layer23_attention_weight_min": -27.03095245361328,
      "activations/layer2_attention_weight_max": 34.40875244140625,
      "activations/layer2_attention_weight_min": -34.39311218261719,
      "activations/layer3_attention_weight_max": 101.27474975585938,
      "activations/layer3_attention_weight_min": -100.08944702148438,
      "activations/layer4_attention_weight_max": 113.63317108154297,
      "activations/layer4_attention_weight_min": -110.66058349609375,
      "activations/layer5_attention_weight_max": 51.431243896484375,
      "activations/layer5_attention_weight_min": -65.78396606445312,
      "activations/layer6_attention_weight_max": 47.09098434448242,
      "activations/layer6_attention_weight_min": -47.52030944824219,
      "activations/layer7_attention_weight_max": 95.53765106201172,
      "activations/layer7_attention_weight_min": -98.7692642211914,
      "activations/layer8_attention_weight_max": 43.1450080871582,
      "activations/layer8_attention_weight_min": -44.82931137084961,
      "activations/layer9_attention_weight_max": 34.72341537475586,
      "activations/layer9_attention_weight_min": -35.30607604980469,
      "epoch": 15.35,
      "learning_rate": 5.153939393939394e-05,
      "loss": 2.7434,
      "step": 264150
    },
    {
      "activations/layer0_attention_weight_max": 16.4233341217041,
      "activations/layer0_attention_weight_min": -11.720425605773926,
      "activations/layer10_attention_weight_max": 38.74148178100586,
      "activations/layer10_attention_weight_min": -38.67920684814453,
      "activations/layer11_attention_weight_max": 38.770599365234375,
      "activations/layer11_attention_weight_min": -36.667911529541016,
      "activations/layer12_attention_weight_max": 28.672021865844727,
      "activations/layer12_attention_weight_min": -28.384567260742188,
      "activations/layer13_attention_weight_max": 44.19716262817383,
      "activations/layer13_attention_weight_min": -37.32173538208008,
      "activations/layer14_attention_weight_max": 48.605010986328125,
      "activations/layer14_attention_weight_min": -39.6412239074707,
      "activations/layer15_attention_weight_max": 42.31235122680664,
      "activations/layer15_attention_weight_min": -35.232261657714844,
      "activations/layer16_attention_weight_max": 30.026750564575195,
      "activations/layer16_attention_weight_min": -28.632543563842773,
      "activations/layer17_attention_weight_max": 31.430505752563477,
      "activations/layer17_attention_weight_min": -25.813146591186523,
      "activations/layer18_attention_weight_max": 33.45820236206055,
      "activations/layer18_attention_weight_min": -24.1377010345459,
      "activations/layer19_attention_weight_max": 33.572265625,
      "activations/layer19_attention_weight_min": -33.228145599365234,
      "activations/layer1_attention_weight_max": 15.99534797668457,
      "activations/layer1_attention_weight_min": -14.836665153503418,
      "activations/layer20_attention_weight_max": 31.27556037902832,
      "activations/layer20_attention_weight_min": -26.45970344543457,
      "activations/layer21_attention_weight_max": 29.950132369995117,
      "activations/layer21_attention_weight_min": -25.432476043701172,
      "activations/layer22_attention_weight_max": 42.18260192871094,
      "activations/layer22_attention_weight_min": -32.41907501220703,
      "activations/layer23_attention_weight_max": 36.106544494628906,
      "activations/layer23_attention_weight_min": -27.17917251586914,
      "activations/layer2_attention_weight_max": 32.79804229736328,
      "activations/layer2_attention_weight_min": -32.69879913330078,
      "activations/layer3_attention_weight_max": 97.60137176513672,
      "activations/layer3_attention_weight_min": -98.17252349853516,
      "activations/layer4_attention_weight_max": 109.91756439208984,
      "activations/layer4_attention_weight_min": -112.12474822998047,
      "activations/layer5_attention_weight_max": 52.71668243408203,
      "activations/layer5_attention_weight_min": -61.38158416748047,
      "activations/layer6_attention_weight_max": 45.22630310058594,
      "activations/layer6_attention_weight_min": -45.391944885253906,
      "activations/layer7_attention_weight_max": 96.09458923339844,
      "activations/layer7_attention_weight_min": -96.73711395263672,
      "activations/layer8_attention_weight_max": 45.96787643432617,
      "activations/layer8_attention_weight_min": -44.21112823486328,
      "activations/layer9_attention_weight_max": 37.081600189208984,
      "activations/layer9_attention_weight_min": -37.24103927612305,
      "epoch": 15.35,
      "learning_rate": 5.152045454545454e-05,
      "loss": 2.7587,
      "step": 264200
    },
    {
      "activations/layer0_attention_weight_max": 15.85229778289795,
      "activations/layer0_attention_weight_min": -12.226134300231934,
      "activations/layer10_attention_weight_max": 34.1876106262207,
      "activations/layer10_attention_weight_min": -35.236175537109375,
      "activations/layer11_attention_weight_max": 34.22682189941406,
      "activations/layer11_attention_weight_min": -33.747406005859375,
      "activations/layer12_attention_weight_max": 26.475788116455078,
      "activations/layer12_attention_weight_min": -27.406984329223633,
      "activations/layer13_attention_weight_max": 39.41522979736328,
      "activations/layer13_attention_weight_min": -36.18239974975586,
      "activations/layer14_attention_weight_max": 41.502384185791016,
      "activations/layer14_attention_weight_min": -37.722755432128906,
      "activations/layer15_attention_weight_max": 39.18585968017578,
      "activations/layer15_attention_weight_min": -32.73176193237305,
      "activations/layer16_attention_weight_max": 29.9263973236084,
      "activations/layer16_attention_weight_min": -29.943456649780273,
      "activations/layer17_attention_weight_max": 29.887264251708984,
      "activations/layer17_attention_weight_min": -25.374128341674805,
      "activations/layer18_attention_weight_max": 29.561174392700195,
      "activations/layer18_attention_weight_min": -24.189489364624023,
      "activations/layer19_attention_weight_max": 32.65061950683594,
      "activations/layer19_attention_weight_min": -30.925312042236328,
      "activations/layer1_attention_weight_max": 16.840137481689453,
      "activations/layer1_attention_weight_min": -15.403634071350098,
      "activations/layer20_attention_weight_max": 27.900859832763672,
      "activations/layer20_attention_weight_min": -26.544897079467773,
      "activations/layer21_attention_weight_max": 29.660263061523438,
      "activations/layer21_attention_weight_min": -24.74151611328125,
      "activations/layer22_attention_weight_max": 40.31458282470703,
      "activations/layer22_attention_weight_min": -31.70076560974121,
      "activations/layer23_attention_weight_max": 30.801013946533203,
      "activations/layer23_attention_weight_min": -26.463764190673828,
      "activations/layer2_attention_weight_max": 32.6490478515625,
      "activations/layer2_attention_weight_min": -34.09958267211914,
      "activations/layer3_attention_weight_max": 97.3157730102539,
      "activations/layer3_attention_weight_min": -99.76825714111328,
      "activations/layer4_attention_weight_max": 110.8271255493164,
      "activations/layer4_attention_weight_min": -110.04255676269531,
      "activations/layer5_attention_weight_max": 52.7370719909668,
      "activations/layer5_attention_weight_min": -62.945945739746094,
      "activations/layer6_attention_weight_max": 45.028282165527344,
      "activations/layer6_attention_weight_min": -49.44475555419922,
      "activations/layer7_attention_weight_max": 92.38108825683594,
      "activations/layer7_attention_weight_min": -98.21266174316406,
      "activations/layer8_attention_weight_max": 43.321197509765625,
      "activations/layer8_attention_weight_min": -45.39101028442383,
      "activations/layer9_attention_weight_max": 36.47663116455078,
      "activations/layer9_attention_weight_min": -35.73784637451172,
      "epoch": 15.35,
      "learning_rate": 5.150151515151515e-05,
      "loss": 2.7384,
      "step": 264250
    },
    {
      "activations/layer0_attention_weight_max": 16.421239852905273,
      "activations/layer0_attention_weight_min": -12.007469177246094,
      "activations/layer10_attention_weight_max": 38.77980422973633,
      "activations/layer10_attention_weight_min": -39.89999008178711,
      "activations/layer11_attention_weight_max": 39.19581985473633,
      "activations/layer11_attention_weight_min": -41.257198333740234,
      "activations/layer12_attention_weight_max": 28.8774356842041,
      "activations/layer12_attention_weight_min": -26.99814796447754,
      "activations/layer13_attention_weight_max": 42.6732292175293,
      "activations/layer13_attention_weight_min": -35.8441047668457,
      "activations/layer14_attention_weight_max": 44.79442596435547,
      "activations/layer14_attention_weight_min": -40.85801315307617,
      "activations/layer15_attention_weight_max": 40.65647888183594,
      "activations/layer15_attention_weight_min": -38.50580596923828,
      "activations/layer16_attention_weight_max": 29.272714614868164,
      "activations/layer16_attention_weight_min": -27.543779373168945,
      "activations/layer17_attention_weight_max": 29.451656341552734,
      "activations/layer17_attention_weight_min": -26.637798309326172,
      "activations/layer18_attention_weight_max": 31.3530330657959,
      "activations/layer18_attention_weight_min": -26.217683792114258,
      "activations/layer19_attention_weight_max": 31.954267501831055,
      "activations/layer19_attention_weight_min": -30.41837501525879,
      "activations/layer1_attention_weight_max": 15.954875946044922,
      "activations/layer1_attention_weight_min": -15.056086540222168,
      "activations/layer20_attention_weight_max": 28.540149688720703,
      "activations/layer20_attention_weight_min": -25.089082717895508,
      "activations/layer21_attention_weight_max": 27.70401382446289,
      "activations/layer21_attention_weight_min": -24.869964599609375,
      "activations/layer22_attention_weight_max": 39.58193588256836,
      "activations/layer22_attention_weight_min": -34.69980239868164,
      "activations/layer23_attention_weight_max": 32.3712043762207,
      "activations/layer23_attention_weight_min": -26.127521514892578,
      "activations/layer2_attention_weight_max": 32.6331787109375,
      "activations/layer2_attention_weight_min": -34.8812255859375,
      "activations/layer3_attention_weight_max": 99.03601837158203,
      "activations/layer3_attention_weight_min": -101.2995834350586,
      "activations/layer4_attention_weight_max": 112.56534576416016,
      "activations/layer4_attention_weight_min": -114.97753143310547,
      "activations/layer5_attention_weight_max": 51.59710693359375,
      "activations/layer5_attention_weight_min": -60.92220687866211,
      "activations/layer6_attention_weight_max": 45.016605377197266,
      "activations/layer6_attention_weight_min": -45.172611236572266,
      "activations/layer7_attention_weight_max": 92.33781433105469,
      "activations/layer7_attention_weight_min": -94.28886413574219,
      "activations/layer8_attention_weight_max": 43.02082824707031,
      "activations/layer8_attention_weight_min": -46.126678466796875,
      "activations/layer9_attention_weight_max": 36.446964263916016,
      "activations/layer9_attention_weight_min": -38.72377395629883,
      "epoch": 15.36,
      "learning_rate": 5.148257575757575e-05,
      "loss": 2.7496,
      "step": 264300
    },
    {
      "activations/layer0_attention_weight_max": 16.092571258544922,
      "activations/layer0_attention_weight_min": -11.895631790161133,
      "activations/layer10_attention_weight_max": 33.21554946899414,
      "activations/layer10_attention_weight_min": -34.0694580078125,
      "activations/layer11_attention_weight_max": 32.81293487548828,
      "activations/layer11_attention_weight_min": -33.689788818359375,
      "activations/layer12_attention_weight_max": 25.897748947143555,
      "activations/layer12_attention_weight_min": -26.49382781982422,
      "activations/layer13_attention_weight_max": 42.83223342895508,
      "activations/layer13_attention_weight_min": -36.68440628051758,
      "activations/layer14_attention_weight_max": 41.087120056152344,
      "activations/layer14_attention_weight_min": -39.664573669433594,
      "activations/layer15_attention_weight_max": 37.947505950927734,
      "activations/layer15_attention_weight_min": -35.02375030517578,
      "activations/layer16_attention_weight_max": 26.50766372680664,
      "activations/layer16_attention_weight_min": -27.612693786621094,
      "activations/layer17_attention_weight_max": 28.609418869018555,
      "activations/layer17_attention_weight_min": -25.321252822875977,
      "activations/layer18_attention_weight_max": 26.363725662231445,
      "activations/layer18_attention_weight_min": -24.244831085205078,
      "activations/layer19_attention_weight_max": 28.055158615112305,
      "activations/layer19_attention_weight_min": -29.931467056274414,
      "activations/layer1_attention_weight_max": 16.048076629638672,
      "activations/layer1_attention_weight_min": -14.54438304901123,
      "activations/layer20_attention_weight_max": 24.30729866027832,
      "activations/layer20_attention_weight_min": -24.256546020507812,
      "activations/layer21_attention_weight_max": 24.222148895263672,
      "activations/layer21_attention_weight_min": -24.036272048950195,
      "activations/layer22_attention_weight_max": 36.59490203857422,
      "activations/layer22_attention_weight_min": -28.18010711669922,
      "activations/layer23_attention_weight_max": 28.122970581054688,
      "activations/layer23_attention_weight_min": -23.78946304321289,
      "activations/layer2_attention_weight_max": 33.14087677001953,
      "activations/layer2_attention_weight_min": -33.21036148071289,
      "activations/layer3_attention_weight_max": 97.4736328125,
      "activations/layer3_attention_weight_min": -96.2939453125,
      "activations/layer4_attention_weight_max": 108.41979217529297,
      "activations/layer4_attention_weight_min": -109.07667541503906,
      "activations/layer5_attention_weight_max": 51.80884552001953,
      "activations/layer5_attention_weight_min": -66.70728302001953,
      "activations/layer6_attention_weight_max": 43.91126251220703,
      "activations/layer6_attention_weight_min": -46.66282653808594,
      "activations/layer7_attention_weight_max": 90.89048767089844,
      "activations/layer7_attention_weight_min": -88.11042785644531,
      "activations/layer8_attention_weight_max": 40.5162239074707,
      "activations/layer8_attention_weight_min": -41.97014617919922,
      "activations/layer9_attention_weight_max": 32.6831169128418,
      "activations/layer9_attention_weight_min": -32.35282516479492,
      "epoch": 15.36,
      "learning_rate": 5.1463636363636365e-05,
      "loss": 2.758,
      "step": 264350
    },
    {
      "activations/layer0_attention_weight_max": 16.23105239868164,
      "activations/layer0_attention_weight_min": -12.762081146240234,
      "activations/layer10_attention_weight_max": 33.08528137207031,
      "activations/layer10_attention_weight_min": -36.16319274902344,
      "activations/layer11_attention_weight_max": 32.359779357910156,
      "activations/layer11_attention_weight_min": -34.24628829956055,
      "activations/layer12_attention_weight_max": 25.725181579589844,
      "activations/layer12_attention_weight_min": -25.75037384033203,
      "activations/layer13_attention_weight_max": 38.370235443115234,
      "activations/layer13_attention_weight_min": -37.00865173339844,
      "activations/layer14_attention_weight_max": 44.71524429321289,
      "activations/layer14_attention_weight_min": -38.12926483154297,
      "activations/layer15_attention_weight_max": 39.57124328613281,
      "activations/layer15_attention_weight_min": -33.8266487121582,
      "activations/layer16_attention_weight_max": 30.556604385375977,
      "activations/layer16_attention_weight_min": -28.293779373168945,
      "activations/layer17_attention_weight_max": 30.16689682006836,
      "activations/layer17_attention_weight_min": -25.109643936157227,
      "activations/layer18_attention_weight_max": 31.091796875,
      "activations/layer18_attention_weight_min": -24.171768188476562,
      "activations/layer19_attention_weight_max": 35.167293548583984,
      "activations/layer19_attention_weight_min": -32.365779876708984,
      "activations/layer1_attention_weight_max": 16.803871154785156,
      "activations/layer1_attention_weight_min": -15.15534782409668,
      "activations/layer20_attention_weight_max": 31.750627517700195,
      "activations/layer20_attention_weight_min": -25.4177188873291,
      "activations/layer21_attention_weight_max": 33.2444953918457,
      "activations/layer21_attention_weight_min": -25.082347869873047,
      "activations/layer22_attention_weight_max": 47.05274963378906,
      "activations/layer22_attention_weight_min": -33.606998443603516,
      "activations/layer23_attention_weight_max": 35.431243896484375,
      "activations/layer23_attention_weight_min": -27.781375885009766,
      "activations/layer2_attention_weight_max": 32.44457244873047,
      "activations/layer2_attention_weight_min": -34.17974090576172,
      "activations/layer3_attention_weight_max": 100.19818115234375,
      "activations/layer3_attention_weight_min": -102.9690170288086,
      "activations/layer4_attention_weight_max": 109.93114471435547,
      "activations/layer4_attention_weight_min": -111.98136138916016,
      "activations/layer5_attention_weight_max": 52.78118896484375,
      "activations/layer5_attention_weight_min": -64.99129486083984,
      "activations/layer6_attention_weight_max": 45.25261306762695,
      "activations/layer6_attention_weight_min": -48.43462371826172,
      "activations/layer7_attention_weight_max": 88.7011489868164,
      "activations/layer7_attention_weight_min": -96.66989135742188,
      "activations/layer8_attention_weight_max": 40.49908447265625,
      "activations/layer8_attention_weight_min": -45.304500579833984,
      "activations/layer9_attention_weight_max": 32.27040481567383,
      "activations/layer9_attention_weight_min": -36.874141693115234,
      "epoch": 15.36,
      "learning_rate": 5.144469696969697e-05,
      "loss": 2.7589,
      "step": 264400
    },
    {
      "activations/layer0_attention_weight_max": 16.015893936157227,
      "activations/layer0_attention_weight_min": -12.118687629699707,
      "activations/layer10_attention_weight_max": 32.28258514404297,
      "activations/layer10_attention_weight_min": -33.31620788574219,
      "activations/layer11_attention_weight_max": 33.13453674316406,
      "activations/layer11_attention_weight_min": -34.52804946899414,
      "activations/layer12_attention_weight_max": 24.677743911743164,
      "activations/layer12_attention_weight_min": -25.236793518066406,
      "activations/layer13_attention_weight_max": 37.00862121582031,
      "activations/layer13_attention_weight_min": -34.01800537109375,
      "activations/layer14_attention_weight_max": 38.487796783447266,
      "activations/layer14_attention_weight_min": -36.36530303955078,
      "activations/layer15_attention_weight_max": 34.74946212768555,
      "activations/layer15_attention_weight_min": -35.869781494140625,
      "activations/layer16_attention_weight_max": 27.712675094604492,
      "activations/layer16_attention_weight_min": -28.370960235595703,
      "activations/layer17_attention_weight_max": 28.870704650878906,
      "activations/layer17_attention_weight_min": -26.739206314086914,
      "activations/layer18_attention_weight_max": 28.222993850708008,
      "activations/layer18_attention_weight_min": -23.19291877746582,
      "activations/layer19_attention_weight_max": 27.80657958984375,
      "activations/layer19_attention_weight_min": -29.200876235961914,
      "activations/layer1_attention_weight_max": 18.019241333007812,
      "activations/layer1_attention_weight_min": -16.40654182434082,
      "activations/layer20_attention_weight_max": 24.805557250976562,
      "activations/layer20_attention_weight_min": -25.20525360107422,
      "activations/layer21_attention_weight_max": 26.542278289794922,
      "activations/layer21_attention_weight_min": -23.62225341796875,
      "activations/layer22_attention_weight_max": 38.115177154541016,
      "activations/layer22_attention_weight_min": -30.919668197631836,
      "activations/layer23_attention_weight_max": 28.479248046875,
      "activations/layer23_attention_weight_min": -24.79909896850586,
      "activations/layer2_attention_weight_max": 32.412879943847656,
      "activations/layer2_attention_weight_min": -31.56537437438965,
      "activations/layer3_attention_weight_max": 95.10572052001953,
      "activations/layer3_attention_weight_min": -95.85655212402344,
      "activations/layer4_attention_weight_max": 107.5595474243164,
      "activations/layer4_attention_weight_min": -108.09619140625,
      "activations/layer5_attention_weight_max": 49.432273864746094,
      "activations/layer5_attention_weight_min": -61.64893341064453,
      "activations/layer6_attention_weight_max": 41.632904052734375,
      "activations/layer6_attention_weight_min": -45.42673110961914,
      "activations/layer7_attention_weight_max": 92.85333251953125,
      "activations/layer7_attention_weight_min": -88.6104736328125,
      "activations/layer8_attention_weight_max": 39.99407958984375,
      "activations/layer8_attention_weight_min": -43.38841247558594,
      "activations/layer9_attention_weight_max": 31.330148696899414,
      "activations/layer9_attention_weight_min": -35.0899543762207,
      "epoch": 15.37,
      "learning_rate": 5.142575757575757e-05,
      "loss": 2.7506,
      "step": 264450
    },
    {
      "activations/layer0_attention_weight_max": 15.48167610168457,
      "activations/layer0_attention_weight_min": -11.786383628845215,
      "activations/layer10_attention_weight_max": 36.368896484375,
      "activations/layer10_attention_weight_min": -34.14973831176758,
      "activations/layer11_attention_weight_max": 37.95939636230469,
      "activations/layer11_attention_weight_min": -37.51432418823242,
      "activations/layer12_attention_weight_max": 30.318431854248047,
      "activations/layer12_attention_weight_min": -28.22655487060547,
      "activations/layer13_attention_weight_max": 47.07302474975586,
      "activations/layer13_attention_weight_min": -39.78850173950195,
      "activations/layer14_attention_weight_max": 50.177921295166016,
      "activations/layer14_attention_weight_min": -43.113616943359375,
      "activations/layer15_attention_weight_max": 48.11931610107422,
      "activations/layer15_attention_weight_min": -38.53197479248047,
      "activations/layer16_attention_weight_max": 34.93611145019531,
      "activations/layer16_attention_weight_min": -29.090951919555664,
      "activations/layer17_attention_weight_max": 34.06099319458008,
      "activations/layer17_attention_weight_min": -26.47995948791504,
      "activations/layer18_attention_weight_max": 38.913902282714844,
      "activations/layer18_attention_weight_min": -24.665014266967773,
      "activations/layer19_attention_weight_max": 39.836910247802734,
      "activations/layer19_attention_weight_min": -31.028594970703125,
      "activations/layer1_attention_weight_max": 17.753902435302734,
      "activations/layer1_attention_weight_min": -16.36451530456543,
      "activations/layer20_attention_weight_max": 33.44119644165039,
      "activations/layer20_attention_weight_min": -24.413894653320312,
      "activations/layer21_attention_weight_max": 31.191787719726562,
      "activations/layer21_attention_weight_min": -22.991294860839844,
      "activations/layer22_attention_weight_max": 51.161415100097656,
      "activations/layer22_attention_weight_min": -31.01192283630371,
      "activations/layer23_attention_weight_max": 35.96727752685547,
      "activations/layer23_attention_weight_min": -24.438629150390625,
      "activations/layer2_attention_weight_max": 32.82762145996094,
      "activations/layer2_attention_weight_min": -32.00916290283203,
      "activations/layer3_attention_weight_max": 98.01720428466797,
      "activations/layer3_attention_weight_min": -98.29605102539062,
      "activations/layer4_attention_weight_max": 111.08784484863281,
      "activations/layer4_attention_weight_min": -111.60860443115234,
      "activations/layer5_attention_weight_max": 55.86711883544922,
      "activations/layer5_attention_weight_min": -63.66863250732422,
      "activations/layer6_attention_weight_max": 43.23633575439453,
      "activations/layer6_attention_weight_min": -45.623653411865234,
      "activations/layer7_attention_weight_max": 93.8314437866211,
      "activations/layer7_attention_weight_min": -91.3378677368164,
      "activations/layer8_attention_weight_max": 41.38255310058594,
      "activations/layer8_attention_weight_min": -45.281715393066406,
      "activations/layer9_attention_weight_max": 34.4254150390625,
      "activations/layer9_attention_weight_min": -33.78059005737305,
      "epoch": 15.37,
      "learning_rate": 5.1406818181818177e-05,
      "loss": 2.7349,
      "step": 264500
    },
    {
      "activations/layer0_attention_weight_max": 16.58597755432129,
      "activations/layer0_attention_weight_min": -12.210864067077637,
      "activations/layer10_attention_weight_max": 36.097110748291016,
      "activations/layer10_attention_weight_min": -34.58708953857422,
      "activations/layer11_attention_weight_max": 35.227333068847656,
      "activations/layer11_attention_weight_min": -34.2926025390625,
      "activations/layer12_attention_weight_max": 26.801725387573242,
      "activations/layer12_attention_weight_min": -26.77521514892578,
      "activations/layer13_attention_weight_max": 39.57841873168945,
      "activations/layer13_attention_weight_min": -37.269493103027344,
      "activations/layer14_attention_weight_max": 43.35749053955078,
      "activations/layer14_attention_weight_min": -38.28461837768555,
      "activations/layer15_attention_weight_max": 39.070556640625,
      "activations/layer15_attention_weight_min": -35.14940643310547,
      "activations/layer16_attention_weight_max": 36.394752502441406,
      "activations/layer16_attention_weight_min": -31.275405883789062,
      "activations/layer17_attention_weight_max": 36.09849166870117,
      "activations/layer17_attention_weight_min": -27.95041847229004,
      "activations/layer18_attention_weight_max": 33.08549118041992,
      "activations/layer18_attention_weight_min": -25.958715438842773,
      "activations/layer19_attention_weight_max": 37.5281867980957,
      "activations/layer19_attention_weight_min": -33.191585540771484,
      "activations/layer1_attention_weight_max": 16.815776824951172,
      "activations/layer1_attention_weight_min": -14.724020004272461,
      "activations/layer20_attention_weight_max": 32.966060638427734,
      "activations/layer20_attention_weight_min": -28.17833137512207,
      "activations/layer21_attention_weight_max": 31.389389038085938,
      "activations/layer21_attention_weight_min": -26.975204467773438,
      "activations/layer22_attention_weight_max": 46.147117614746094,
      "activations/layer22_attention_weight_min": -34.433223724365234,
      "activations/layer23_attention_weight_max": 31.939617156982422,
      "activations/layer23_attention_weight_min": -28.49365234375,
      "activations/layer2_attention_weight_max": 35.148590087890625,
      "activations/layer2_attention_weight_min": -33.682674407958984,
      "activations/layer3_attention_weight_max": 102.18724060058594,
      "activations/layer3_attention_weight_min": -103.32659149169922,
      "activations/layer4_attention_weight_max": 112.3735122680664,
      "activations/layer4_attention_weight_min": -121.21358489990234,
      "activations/layer5_attention_weight_max": 52.739990234375,
      "activations/layer5_attention_weight_min": -64.58148193359375,
      "activations/layer6_attention_weight_max": 45.94425964355469,
      "activations/layer6_attention_weight_min": -48.26332092285156,
      "activations/layer7_attention_weight_max": 94.70927429199219,
      "activations/layer7_attention_weight_min": -104.67845916748047,
      "activations/layer8_attention_weight_max": 44.50092697143555,
      "activations/layer8_attention_weight_min": -46.36000442504883,
      "activations/layer9_attention_weight_max": 33.676265716552734,
      "activations/layer9_attention_weight_min": -35.128990173339844,
      "epoch": 15.37,
      "learning_rate": 5.1388257575757574e-05,
      "loss": 2.7414,
      "step": 264550
    },
    {
      "activations/layer0_attention_weight_max": 15.811614036560059,
      "activations/layer0_attention_weight_min": -13.655853271484375,
      "activations/layer10_attention_weight_max": 34.50992965698242,
      "activations/layer10_attention_weight_min": -35.977378845214844,
      "activations/layer11_attention_weight_max": 34.52595520019531,
      "activations/layer11_attention_weight_min": -34.29715347290039,
      "activations/layer12_attention_weight_max": 26.987489700317383,
      "activations/layer12_attention_weight_min": -29.67671012878418,
      "activations/layer13_attention_weight_max": 39.53839111328125,
      "activations/layer13_attention_weight_min": -37.06110763549805,
      "activations/layer14_attention_weight_max": 40.3331184387207,
      "activations/layer14_attention_weight_min": -39.447505950927734,
      "activations/layer15_attention_weight_max": 35.558738708496094,
      "activations/layer15_attention_weight_min": -36.056671142578125,
      "activations/layer16_attention_weight_max": 30.154861450195312,
      "activations/layer16_attention_weight_min": -29.856088638305664,
      "activations/layer17_attention_weight_max": 31.93543815612793,
      "activations/layer17_attention_weight_min": -27.932523727416992,
      "activations/layer18_attention_weight_max": 31.22088050842285,
      "activations/layer18_attention_weight_min": -26.016258239746094,
      "activations/layer19_attention_weight_max": 31.779088973999023,
      "activations/layer19_attention_weight_min": -35.87978744506836,
      "activations/layer1_attention_weight_max": 17.123655319213867,
      "activations/layer1_attention_weight_min": -15.35342025756836,
      "activations/layer20_attention_weight_max": 28.307537078857422,
      "activations/layer20_attention_weight_min": -29.370960235595703,
      "activations/layer21_attention_weight_max": 27.792129516601562,
      "activations/layer21_attention_weight_min": -27.830631256103516,
      "activations/layer22_attention_weight_max": 41.4193229675293,
      "activations/layer22_attention_weight_min": -36.12277603149414,
      "activations/layer23_attention_weight_max": 32.82109451293945,
      "activations/layer23_attention_weight_min": -26.58749008178711,
      "activations/layer2_attention_weight_max": 34.51726150512695,
      "activations/layer2_attention_weight_min": -32.77260971069336,
      "activations/layer3_attention_weight_max": 97.55282592773438,
      "activations/layer3_attention_weight_min": -97.11697387695312,
      "activations/layer4_attention_weight_max": 114.7748794555664,
      "activations/layer4_attention_weight_min": -112.20257568359375,
      "activations/layer5_attention_weight_max": 54.10950469970703,
      "activations/layer5_attention_weight_min": -62.58103561401367,
      "activations/layer6_attention_weight_max": 48.08708953857422,
      "activations/layer6_attention_weight_min": -50.11626434326172,
      "activations/layer7_attention_weight_max": 97.4586410522461,
      "activations/layer7_attention_weight_min": -98.9645004272461,
      "activations/layer8_attention_weight_max": 43.42628860473633,
      "activations/layer8_attention_weight_min": -46.13115692138672,
      "activations/layer9_attention_weight_max": 33.75308609008789,
      "activations/layer9_attention_weight_min": -37.613948822021484,
      "epoch": 15.37,
      "learning_rate": 5.1369318181818176e-05,
      "loss": 2.7482,
      "step": 264600
    },
    {
      "activations/layer0_attention_weight_max": 17.38704490661621,
      "activations/layer0_attention_weight_min": -12.75245475769043,
      "activations/layer10_attention_weight_max": 43.55900192260742,
      "activations/layer10_attention_weight_min": -38.0806999206543,
      "activations/layer11_attention_weight_max": 45.118247985839844,
      "activations/layer11_attention_weight_min": -41.1763916015625,
      "activations/layer12_attention_weight_max": 30.49325180053711,
      "activations/layer12_attention_weight_min": -30.34933090209961,
      "activations/layer13_attention_weight_max": 43.05574417114258,
      "activations/layer13_attention_weight_min": -34.46755599975586,
      "activations/layer14_attention_weight_max": 48.0140495300293,
      "activations/layer14_attention_weight_min": -37.49622344970703,
      "activations/layer15_attention_weight_max": 43.16530990600586,
      "activations/layer15_attention_weight_min": -35.37498092651367,
      "activations/layer16_attention_weight_max": 34.091304779052734,
      "activations/layer16_attention_weight_min": -28.715970993041992,
      "activations/layer17_attention_weight_max": 34.421077728271484,
      "activations/layer17_attention_weight_min": -27.6339168548584,
      "activations/layer18_attention_weight_max": 30.50824546813965,
      "activations/layer18_attention_weight_min": -25.134008407592773,
      "activations/layer19_attention_weight_max": 39.20815658569336,
      "activations/layer19_attention_weight_min": -31.500049591064453,
      "activations/layer1_attention_weight_max": 18.14323616027832,
      "activations/layer1_attention_weight_min": -16.204137802124023,
      "activations/layer20_attention_weight_max": 33.128910064697266,
      "activations/layer20_attention_weight_min": -26.004613876342773,
      "activations/layer21_attention_weight_max": 29.39071273803711,
      "activations/layer21_attention_weight_min": -24.39450454711914,
      "activations/layer22_attention_weight_max": 42.38926696777344,
      "activations/layer22_attention_weight_min": -30.65964126586914,
      "activations/layer23_attention_weight_max": 30.301109313964844,
      "activations/layer23_attention_weight_min": -26.51558494567871,
      "activations/layer2_attention_weight_max": 35.685123443603516,
      "activations/layer2_attention_weight_min": -32.22759246826172,
      "activations/layer3_attention_weight_max": 100.75753021240234,
      "activations/layer3_attention_weight_min": -96.44181060791016,
      "activations/layer4_attention_weight_max": 115.85807037353516,
      "activations/layer4_attention_weight_min": -111.32006072998047,
      "activations/layer5_attention_weight_max": 55.7105827331543,
      "activations/layer5_attention_weight_min": -60.88792037963867,
      "activations/layer6_attention_weight_max": 47.20536804199219,
      "activations/layer6_attention_weight_min": -45.05842208862305,
      "activations/layer7_attention_weight_max": 103.493408203125,
      "activations/layer7_attention_weight_min": -94.19607543945312,
      "activations/layer8_attention_weight_max": 50.126426696777344,
      "activations/layer8_attention_weight_min": -45.06760025024414,
      "activations/layer9_attention_weight_max": 38.84248733520508,
      "activations/layer9_attention_weight_min": -37.170616149902344,
      "epoch": 15.38,
      "learning_rate": 5.1350378787878784e-05,
      "loss": 2.7417,
      "step": 264650
    },
    {
      "activations/layer0_attention_weight_max": 15.732419967651367,
      "activations/layer0_attention_weight_min": -12.149808883666992,
      "activations/layer10_attention_weight_max": 33.06890106201172,
      "activations/layer10_attention_weight_min": -33.1422004699707,
      "activations/layer11_attention_weight_max": 33.88030242919922,
      "activations/layer11_attention_weight_min": -33.56334686279297,
      "activations/layer12_attention_weight_max": 32.10701370239258,
      "activations/layer12_attention_weight_min": -29.16534996032715,
      "activations/layer13_attention_weight_max": 48.02082061767578,
      "activations/layer13_attention_weight_min": -38.45078659057617,
      "activations/layer14_attention_weight_max": 53.173946380615234,
      "activations/layer14_attention_weight_min": -40.586421966552734,
      "activations/layer15_attention_weight_max": 47.895751953125,
      "activations/layer15_attention_weight_min": -37.37575149536133,
      "activations/layer16_attention_weight_max": 37.796051025390625,
      "activations/layer16_attention_weight_min": -29.56110382080078,
      "activations/layer17_attention_weight_max": 36.10605239868164,
      "activations/layer17_attention_weight_min": -29.491628646850586,
      "activations/layer18_attention_weight_max": 39.475982666015625,
      "activations/layer18_attention_weight_min": -27.088064193725586,
      "activations/layer19_attention_weight_max": 48.03044509887695,
      "activations/layer19_attention_weight_min": -36.59449768066406,
      "activations/layer1_attention_weight_max": 18.3205623626709,
      "activations/layer1_attention_weight_min": -14.919962882995605,
      "activations/layer20_attention_weight_max": 38.976375579833984,
      "activations/layer20_attention_weight_min": -26.93083381652832,
      "activations/layer21_attention_weight_max": 37.98491668701172,
      "activations/layer21_attention_weight_min": -26.24072265625,
      "activations/layer22_attention_weight_max": 67.7153091430664,
      "activations/layer22_attention_weight_min": -35.53049087524414,
      "activations/layer23_attention_weight_max": 46.91512680053711,
      "activations/layer23_attention_weight_min": -26.7940731048584,
      "activations/layer2_attention_weight_max": 30.95874786376953,
      "activations/layer2_attention_weight_min": -29.74999237060547,
      "activations/layer3_attention_weight_max": 81.8813705444336,
      "activations/layer3_attention_weight_min": -80.43607330322266,
      "activations/layer4_attention_weight_max": 95.66788482666016,
      "activations/layer4_attention_weight_min": -90.50261688232422,
      "activations/layer5_attention_weight_max": 46.96240997314453,
      "activations/layer5_attention_weight_min": -56.60886764526367,
      "activations/layer6_attention_weight_max": 38.58058166503906,
      "activations/layer6_attention_weight_min": -43.17945098876953,
      "activations/layer7_attention_weight_max": 87.48966979980469,
      "activations/layer7_attention_weight_min": -91.35319519042969,
      "activations/layer8_attention_weight_max": 38.77424621582031,
      "activations/layer8_attention_weight_min": -40.773616790771484,
      "activations/layer9_attention_weight_max": 30.701391220092773,
      "activations/layer9_attention_weight_min": -31.638025283813477,
      "epoch": 15.38,
      "learning_rate": 5.1331439393939385e-05,
      "loss": 2.7427,
      "step": 264700
    },
    {
      "activations/layer0_attention_weight_max": 16.725852966308594,
      "activations/layer0_attention_weight_min": -11.807657241821289,
      "activations/layer10_attention_weight_max": 31.81487274169922,
      "activations/layer10_attention_weight_min": -33.8686637878418,
      "activations/layer11_attention_weight_max": 32.341651916503906,
      "activations/layer11_attention_weight_min": -32.91051483154297,
      "activations/layer12_attention_weight_max": 26.60328483581543,
      "activations/layer12_attention_weight_min": -26.72018814086914,
      "activations/layer13_attention_weight_max": 42.86888885498047,
      "activations/layer13_attention_weight_min": -35.85646438598633,
      "activations/layer14_attention_weight_max": 45.26230239868164,
      "activations/layer14_attention_weight_min": -38.185516357421875,
      "activations/layer15_attention_weight_max": 40.475399017333984,
      "activations/layer15_attention_weight_min": -35.82379913330078,
      "activations/layer16_attention_weight_max": 31.614749908447266,
      "activations/layer16_attention_weight_min": -27.41860008239746,
      "activations/layer17_attention_weight_max": 36.70353317260742,
      "activations/layer17_attention_weight_min": -26.50008201599121,
      "activations/layer18_attention_weight_max": 34.274173736572266,
      "activations/layer18_attention_weight_min": -25.734512329101562,
      "activations/layer19_attention_weight_max": 35.174964904785156,
      "activations/layer19_attention_weight_min": -30.330495834350586,
      "activations/layer1_attention_weight_max": 15.642497062683105,
      "activations/layer1_attention_weight_min": -14.5782470703125,
      "activations/layer20_attention_weight_max": 33.861263275146484,
      "activations/layer20_attention_weight_min": -25.1283016204834,
      "activations/layer21_attention_weight_max": 29.845584869384766,
      "activations/layer21_attention_weight_min": -24.05047035217285,
      "activations/layer22_attention_weight_max": 46.17511749267578,
      "activations/layer22_attention_weight_min": -30.47734260559082,
      "activations/layer23_attention_weight_max": 37.13343048095703,
      "activations/layer23_attention_weight_min": -24.969717025756836,
      "activations/layer2_attention_weight_max": 31.388160705566406,
      "activations/layer2_attention_weight_min": -31.117475509643555,
      "activations/layer3_attention_weight_max": 92.74246978759766,
      "activations/layer3_attention_weight_min": -87.31008911132812,
      "activations/layer4_attention_weight_max": 102.8603515625,
      "activations/layer4_attention_weight_min": -102.07587432861328,
      "activations/layer5_attention_weight_max": 49.78837585449219,
      "activations/layer5_attention_weight_min": -61.68998718261719,
      "activations/layer6_attention_weight_max": 41.7861328125,
      "activations/layer6_attention_weight_min": -44.650203704833984,
      "activations/layer7_attention_weight_max": 89.412841796875,
      "activations/layer7_attention_weight_min": -93.42465209960938,
      "activations/layer8_attention_weight_max": 40.787601470947266,
      "activations/layer8_attention_weight_min": -42.48210525512695,
      "activations/layer9_attention_weight_max": 31.855005264282227,
      "activations/layer9_attention_weight_min": -34.4265022277832,
      "epoch": 15.38,
      "learning_rate": 5.13125e-05,
      "loss": 2.7528,
      "step": 264750
    },
    {
      "activations/layer0_attention_weight_max": 16.207420349121094,
      "activations/layer0_attention_weight_min": -13.683967590332031,
      "activations/layer10_attention_weight_max": 35.12153625488281,
      "activations/layer10_attention_weight_min": -33.483856201171875,
      "activations/layer11_attention_weight_max": 33.00727844238281,
      "activations/layer11_attention_weight_min": -31.412586212158203,
      "activations/layer12_attention_weight_max": 26.138517379760742,
      "activations/layer12_attention_weight_min": -28.4508113861084,
      "activations/layer13_attention_weight_max": 38.58274459838867,
      "activations/layer13_attention_weight_min": -37.90664291381836,
      "activations/layer14_attention_weight_max": 45.30455780029297,
      "activations/layer14_attention_weight_min": -38.88957595825195,
      "activations/layer15_attention_weight_max": 35.893672943115234,
      "activations/layer15_attention_weight_min": -35.847816467285156,
      "activations/layer16_attention_weight_max": 30.216154098510742,
      "activations/layer16_attention_weight_min": -28.290302276611328,
      "activations/layer17_attention_weight_max": 31.24024772644043,
      "activations/layer17_attention_weight_min": -27.663846969604492,
      "activations/layer18_attention_weight_max": 32.7307014465332,
      "activations/layer18_attention_weight_min": -26.49005699157715,
      "activations/layer19_attention_weight_max": 37.55643844604492,
      "activations/layer19_attention_weight_min": -34.04642868041992,
      "activations/layer1_attention_weight_max": 16.69953727722168,
      "activations/layer1_attention_weight_min": -14.932267189025879,
      "activations/layer20_attention_weight_max": 31.595458984375,
      "activations/layer20_attention_weight_min": -27.015064239501953,
      "activations/layer21_attention_weight_max": 31.876609802246094,
      "activations/layer21_attention_weight_min": -28.33135223388672,
      "activations/layer22_attention_weight_max": 49.07843017578125,
      "activations/layer22_attention_weight_min": -36.89933776855469,
      "activations/layer23_attention_weight_max": 36.92789077758789,
      "activations/layer23_attention_weight_min": -26.12513542175293,
      "activations/layer2_attention_weight_max": 32.73719787597656,
      "activations/layer2_attention_weight_min": -31.49703598022461,
      "activations/layer3_attention_weight_max": 89.66439056396484,
      "activations/layer3_attention_weight_min": -91.1568603515625,
      "activations/layer4_attention_weight_max": 98.33147430419922,
      "activations/layer4_attention_weight_min": -98.36860656738281,
      "activations/layer5_attention_weight_max": 49.24863815307617,
      "activations/layer5_attention_weight_min": -58.9917106628418,
      "activations/layer6_attention_weight_max": 42.305091857910156,
      "activations/layer6_attention_weight_min": -45.012516021728516,
      "activations/layer7_attention_weight_max": 92.8641128540039,
      "activations/layer7_attention_weight_min": -93.25316619873047,
      "activations/layer8_attention_weight_max": 40.182579040527344,
      "activations/layer8_attention_weight_min": -40.7448844909668,
      "activations/layer9_attention_weight_max": 34.3301887512207,
      "activations/layer9_attention_weight_min": -33.03569793701172,
      "epoch": 15.39,
      "learning_rate": 5.12935606060606e-05,
      "loss": 2.7555,
      "step": 264800
    },
    {
      "activations/layer0_attention_weight_max": 15.492552757263184,
      "activations/layer0_attention_weight_min": -12.43476390838623,
      "activations/layer10_attention_weight_max": 38.965667724609375,
      "activations/layer10_attention_weight_min": -35.99122619628906,
      "activations/layer11_attention_weight_max": 38.67310333251953,
      "activations/layer11_attention_weight_min": -36.375240325927734,
      "activations/layer12_attention_weight_max": 26.76272201538086,
      "activations/layer12_attention_weight_min": -26.44489288330078,
      "activations/layer13_attention_weight_max": 39.42970657348633,
      "activations/layer13_attention_weight_min": -35.21297073364258,
      "activations/layer14_attention_weight_max": 43.01838302612305,
      "activations/layer14_attention_weight_min": -39.52163314819336,
      "activations/layer15_attention_weight_max": 39.151519775390625,
      "activations/layer15_attention_weight_min": -36.1395378112793,
      "activations/layer16_attention_weight_max": 32.88930130004883,
      "activations/layer16_attention_weight_min": -28.625778198242188,
      "activations/layer17_attention_weight_max": 29.330448150634766,
      "activations/layer17_attention_weight_min": -27.245901107788086,
      "activations/layer18_attention_weight_max": 30.979408264160156,
      "activations/layer18_attention_weight_min": -24.610673904418945,
      "activations/layer19_attention_weight_max": 33.29203414916992,
      "activations/layer19_attention_weight_min": -30.630517959594727,
      "activations/layer1_attention_weight_max": 15.734577178955078,
      "activations/layer1_attention_weight_min": -14.25049877166748,
      "activations/layer20_attention_weight_max": 29.678686141967773,
      "activations/layer20_attention_weight_min": -25.310644149780273,
      "activations/layer21_attention_weight_max": 30.677927017211914,
      "activations/layer21_attention_weight_min": -25.194625854492188,
      "activations/layer22_attention_weight_max": 40.67321014404297,
      "activations/layer22_attention_weight_min": -30.645771026611328,
      "activations/layer23_attention_weight_max": 33.34729766845703,
      "activations/layer23_attention_weight_min": -27.305450439453125,
      "activations/layer2_attention_weight_max": 33.52700424194336,
      "activations/layer2_attention_weight_min": -33.40452575683594,
      "activations/layer3_attention_weight_max": 96.95230865478516,
      "activations/layer3_attention_weight_min": -94.96945190429688,
      "activations/layer4_attention_weight_max": 107.80416107177734,
      "activations/layer4_attention_weight_min": -104.4111557006836,
      "activations/layer5_attention_weight_max": 51.239295959472656,
      "activations/layer5_attention_weight_min": -59.41044616699219,
      "activations/layer6_attention_weight_max": 44.98241424560547,
      "activations/layer6_attention_weight_min": -45.59103012084961,
      "activations/layer7_attention_weight_max": 94.85839080810547,
      "activations/layer7_attention_weight_min": -97.53538513183594,
      "activations/layer8_attention_weight_max": 44.38435745239258,
      "activations/layer8_attention_weight_min": -45.69198226928711,
      "activations/layer9_attention_weight_max": 35.65620040893555,
      "activations/layer9_attention_weight_min": -38.39925765991211,
      "epoch": 15.39,
      "learning_rate": 5.1274621212121203e-05,
      "loss": 2.7585,
      "step": 264850
    },
    {
      "activations/layer0_attention_weight_max": 15.539435386657715,
      "activations/layer0_attention_weight_min": -13.198831558227539,
      "activations/layer10_attention_weight_max": 33.70654296875,
      "activations/layer10_attention_weight_min": -32.6397590637207,
      "activations/layer11_attention_weight_max": 34.85602951049805,
      "activations/layer11_attention_weight_min": -33.12296676635742,
      "activations/layer12_attention_weight_max": 27.222742080688477,
      "activations/layer12_attention_weight_min": -25.909381866455078,
      "activations/layer13_attention_weight_max": 40.12152862548828,
      "activations/layer13_attention_weight_min": -35.390323638916016,
      "activations/layer14_attention_weight_max": 41.768558502197266,
      "activations/layer14_attention_weight_min": -38.482086181640625,
      "activations/layer15_attention_weight_max": 36.20266342163086,
      "activations/layer15_attention_weight_min": -34.83313751220703,
      "activations/layer16_attention_weight_max": 29.229448318481445,
      "activations/layer16_attention_weight_min": -26.724695205688477,
      "activations/layer17_attention_weight_max": 30.89999771118164,
      "activations/layer17_attention_weight_min": -26.92058753967285,
      "activations/layer18_attention_weight_max": 31.509971618652344,
      "activations/layer18_attention_weight_min": -26.27568817138672,
      "activations/layer19_attention_weight_max": 32.28153991699219,
      "activations/layer19_attention_weight_min": -31.354246139526367,
      "activations/layer1_attention_weight_max": 16.228116989135742,
      "activations/layer1_attention_weight_min": -16.678327560424805,
      "activations/layer20_attention_weight_max": 29.037363052368164,
      "activations/layer20_attention_weight_min": -27.076786041259766,
      "activations/layer21_attention_weight_max": 28.15593147277832,
      "activations/layer21_attention_weight_min": -25.14794158935547,
      "activations/layer22_attention_weight_max": 43.912418365478516,
      "activations/layer22_attention_weight_min": -30.351648330688477,
      "activations/layer23_attention_weight_max": 30.557926177978516,
      "activations/layer23_attention_weight_min": -23.709508895874023,
      "activations/layer2_attention_weight_max": 32.856773376464844,
      "activations/layer2_attention_weight_min": -33.938777923583984,
      "activations/layer3_attention_weight_max": 92.78547668457031,
      "activations/layer3_attention_weight_min": -91.68273162841797,
      "activations/layer4_attention_weight_max": 106.97810363769531,
      "activations/layer4_attention_weight_min": -104.36405181884766,
      "activations/layer5_attention_weight_max": 49.133846282958984,
      "activations/layer5_attention_weight_min": -60.10337448120117,
      "activations/layer6_attention_weight_max": 44.775596618652344,
      "activations/layer6_attention_weight_min": -48.05488204956055,
      "activations/layer7_attention_weight_max": 91.89337158203125,
      "activations/layer7_attention_weight_min": -97.5263671875,
      "activations/layer8_attention_weight_max": 41.29964828491211,
      "activations/layer8_attention_weight_min": -41.98360061645508,
      "activations/layer9_attention_weight_max": 32.90289306640625,
      "activations/layer9_attention_weight_min": -34.09475326538086,
      "epoch": 15.39,
      "learning_rate": 5.125568181818181e-05,
      "loss": 2.7467,
      "step": 264900
    },
    {
      "activations/layer0_attention_weight_max": 16.122053146362305,
      "activations/layer0_attention_weight_min": -12.638667106628418,
      "activations/layer10_attention_weight_max": 31.282033920288086,
      "activations/layer10_attention_weight_min": -30.726499557495117,
      "activations/layer11_attention_weight_max": 33.603271484375,
      "activations/layer11_attention_weight_min": -33.4669189453125,
      "activations/layer12_attention_weight_max": 27.709875106811523,
      "activations/layer12_attention_weight_min": -26.520912170410156,
      "activations/layer13_attention_weight_max": 44.42720031738281,
      "activations/layer13_attention_weight_min": -37.637176513671875,
      "activations/layer14_attention_weight_max": 48.743873596191406,
      "activations/layer14_attention_weight_min": -39.69321060180664,
      "activations/layer15_attention_weight_max": 43.64686584472656,
      "activations/layer15_attention_weight_min": -36.313873291015625,
      "activations/layer16_attention_weight_max": 34.14242172241211,
      "activations/layer16_attention_weight_min": -29.612985610961914,
      "activations/layer17_attention_weight_max": 31.01589012145996,
      "activations/layer17_attention_weight_min": -27.55043601989746,
      "activations/layer18_attention_weight_max": 34.382137298583984,
      "activations/layer18_attention_weight_min": -26.506263732910156,
      "activations/layer19_attention_weight_max": 36.1262092590332,
      "activations/layer19_attention_weight_min": -31.976694107055664,
      "activations/layer1_attention_weight_max": 16.810916900634766,
      "activations/layer1_attention_weight_min": -15.086827278137207,
      "activations/layer20_attention_weight_max": 32.61528015136719,
      "activations/layer20_attention_weight_min": -26.17049217224121,
      "activations/layer21_attention_weight_max": 30.24117660522461,
      "activations/layer21_attention_weight_min": -25.740758895874023,
      "activations/layer22_attention_weight_max": 50.977142333984375,
      "activations/layer22_attention_weight_min": -31.628416061401367,
      "activations/layer23_attention_weight_max": 34.163673400878906,
      "activations/layer23_attention_weight_min": -26.987144470214844,
      "activations/layer2_attention_weight_max": 28.765819549560547,
      "activations/layer2_attention_weight_min": -30.104454040527344,
      "activations/layer3_attention_weight_max": 91.52718353271484,
      "activations/layer3_attention_weight_min": -91.62582397460938,
      "activations/layer4_attention_weight_max": 103.54476165771484,
      "activations/layer4_attention_weight_min": -99.55217742919922,
      "activations/layer5_attention_weight_max": 51.66440963745117,
      "activations/layer5_attention_weight_min": -58.85651779174805,
      "activations/layer6_attention_weight_max": 42.03456497192383,
      "activations/layer6_attention_weight_min": -46.04426574707031,
      "activations/layer7_attention_weight_max": 84.80538940429688,
      "activations/layer7_attention_weight_min": -88.66641235351562,
      "activations/layer8_attention_weight_max": 38.2149543762207,
      "activations/layer8_attention_weight_min": -41.0118293762207,
      "activations/layer9_attention_weight_max": 31.10040283203125,
      "activations/layer9_attention_weight_min": -31.62862205505371,
      "epoch": 15.4,
      "learning_rate": 5.123674242424242e-05,
      "loss": 2.7564,
      "step": 264950
    },
    {
      "activations/layer0_attention_weight_max": 15.4705171585083,
      "activations/layer0_attention_weight_min": -12.30881404876709,
      "activations/layer10_attention_weight_max": 42.60200881958008,
      "activations/layer10_attention_weight_min": -42.18735885620117,
      "activations/layer11_attention_weight_max": 41.50193405151367,
      "activations/layer11_attention_weight_min": -40.70264434814453,
      "activations/layer12_attention_weight_max": 33.49204635620117,
      "activations/layer12_attention_weight_min": -30.277311325073242,
      "activations/layer13_attention_weight_max": 48.69802474975586,
      "activations/layer13_attention_weight_min": -41.49119186401367,
      "activations/layer14_attention_weight_max": 51.831756591796875,
      "activations/layer14_attention_weight_min": -43.13650131225586,
      "activations/layer15_attention_weight_max": 53.814361572265625,
      "activations/layer15_attention_weight_min": -39.5547981262207,
      "activations/layer16_attention_weight_max": 32.00126266479492,
      "activations/layer16_attention_weight_min": -27.926958084106445,
      "activations/layer17_attention_weight_max": 33.93588638305664,
      "activations/layer17_attention_weight_min": -27.552352905273438,
      "activations/layer18_attention_weight_max": 32.74091720581055,
      "activations/layer18_attention_weight_min": -26.406108856201172,
      "activations/layer19_attention_weight_max": 33.78569412231445,
      "activations/layer19_attention_weight_min": -34.08277130126953,
      "activations/layer1_attention_weight_max": 16.98249626159668,
      "activations/layer1_attention_weight_min": -16.534698486328125,
      "activations/layer20_attention_weight_max": 29.11882781982422,
      "activations/layer20_attention_weight_min": -27.65554428100586,
      "activations/layer21_attention_weight_max": 29.732254028320312,
      "activations/layer21_attention_weight_min": -25.717668533325195,
      "activations/layer22_attention_weight_max": 49.65421676635742,
      "activations/layer22_attention_weight_min": -31.645063400268555,
      "activations/layer23_attention_weight_max": 36.044593811035156,
      "activations/layer23_attention_weight_min": -26.519710540771484,
      "activations/layer2_attention_weight_max": 31.701671600341797,
      "activations/layer2_attention_weight_min": -31.652790069580078,
      "activations/layer3_attention_weight_max": 97.14320373535156,
      "activations/layer3_attention_weight_min": -100.54837799072266,
      "activations/layer4_attention_weight_max": 114.87638092041016,
      "activations/layer4_attention_weight_min": -115.30281066894531,
      "activations/layer5_attention_weight_max": 53.394203186035156,
      "activations/layer5_attention_weight_min": -60.310874938964844,
      "activations/layer6_attention_weight_max": 49.448673248291016,
      "activations/layer6_attention_weight_min": -49.93819046020508,
      "activations/layer7_attention_weight_max": 102.6585693359375,
      "activations/layer7_attention_weight_min": -104.78086853027344,
      "activations/layer8_attention_weight_max": 50.12224197387695,
      "activations/layer8_attention_weight_min": -50.952213287353516,
      "activations/layer9_attention_weight_max": 40.26917266845703,
      "activations/layer9_attention_weight_min": -40.25468063354492,
      "epoch": 15.4,
      "learning_rate": 5.121780303030303e-05,
      "loss": 2.7472,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_loss": 2.703125,
      "eval_runtime": 8.5352,
      "eval_samples_per_second": 503.096,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.5352,
      "eval_openwebtext_samples_per_second": 503.096,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_wikitext_loss": 2.92578125,
      "eval_wikitext_ppl": 18.648789727196185,
      "eval_wikitext_runtime": 2.0005,
      "eval_wikitext_samples_per_second": 227.944,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.56,
      "eval_lambada_samples_per_second": 509.308,
      "step": 265000
    },
    {
      "activations/layer0_attention_weight_max": 16.86354637145996,
      "activations/layer0_attention_weight_min": -12.344103813171387,
      "activations/layer10_attention_weight_max": 33.77759552001953,
      "activations/layer10_attention_weight_min": -33.58283615112305,
      "activations/layer11_attention_weight_max": 34.097816467285156,
      "activations/layer11_attention_weight_min": -36.56691360473633,
      "activations/layer12_attention_weight_max": 29.621000289916992,
      "activations/layer12_attention_weight_min": -28.92647933959961,
      "activations/layer13_attention_weight_max": 47.60594177246094,
      "activations/layer13_attention_weight_min": -42.301910400390625,
      "activations/layer14_attention_weight_max": 49.5854606628418,
      "activations/layer14_attention_weight_min": -41.01681137084961,
      "activations/layer15_attention_weight_max": 45.11653137207031,
      "activations/layer15_attention_weight_min": -37.635658264160156,
      "activations/layer16_attention_weight_max": 32.14501190185547,
      "activations/layer16_attention_weight_min": -29.66777229309082,
      "activations/layer17_attention_weight_max": 33.232421875,
      "activations/layer17_attention_weight_min": -26.181011199951172,
      "activations/layer18_attention_weight_max": 33.285640716552734,
      "activations/layer18_attention_weight_min": -24.88859748840332,
      "activations/layer19_attention_weight_max": 39.29182052612305,
      "activations/layer19_attention_weight_min": -33.00157165527344,
      "activations/layer1_attention_weight_max": 16.609161376953125,
      "activations/layer1_attention_weight_min": -15.525668144226074,
      "activations/layer20_attention_weight_max": 35.846580505371094,
      "activations/layer20_attention_weight_min": -27.541839599609375,
      "activations/layer21_attention_weight_max": 35.02531051635742,
      "activations/layer21_attention_weight_min": -29.19725227355957,
      "activations/layer22_attention_weight_max": 47.035396575927734,
      "activations/layer22_attention_weight_min": -34.515987396240234,
      "activations/layer23_attention_weight_max": 38.985252380371094,
      "activations/layer23_attention_weight_min": -25.383146286010742,
      "activations/layer2_attention_weight_max": 32.112342834472656,
      "activations/layer2_attention_weight_min": -33.7989501953125,
      "activations/layer3_attention_weight_max": 94.6963882446289,
      "activations/layer3_attention_weight_min": -99.24131774902344,
      "activations/layer4_attention_weight_max": 106.17162322998047,
      "activations/layer4_attention_weight_min": -111.2695541381836,
      "activations/layer5_attention_weight_max": 52.96047592163086,
      "activations/layer5_attention_weight_min": -60.78644943237305,
      "activations/layer6_attention_weight_max": 46.33445739746094,
      "activations/layer6_attention_weight_min": -49.55487823486328,
      "activations/layer7_attention_weight_max": 92.03302764892578,
      "activations/layer7_attention_weight_min": -98.38764190673828,
      "activations/layer8_attention_weight_max": 42.90414810180664,
      "activations/layer8_attention_weight_min": -44.24660110473633,
      "activations/layer9_attention_weight_max": 33.27957534790039,
      "activations/layer9_attention_weight_min": -33.50165557861328,
      "epoch": 15.4,
      "learning_rate": 5.119886363636363e-05,
      "loss": 2.7425,
      "step": 265050
    },
    {
      "activations/layer0_attention_weight_max": 15.711267471313477,
      "activations/layer0_attention_weight_min": -12.33743667602539,
      "activations/layer10_attention_weight_max": 33.841522216796875,
      "activations/layer10_attention_weight_min": -33.89583206176758,
      "activations/layer11_attention_weight_max": 33.88728713989258,
      "activations/layer11_attention_weight_min": -33.838706970214844,
      "activations/layer12_attention_weight_max": 28.551410675048828,
      "activations/layer12_attention_weight_min": -25.687253952026367,
      "activations/layer13_attention_weight_max": 41.972259521484375,
      "activations/layer13_attention_weight_min": -34.17750549316406,
      "activations/layer14_attention_weight_max": 41.67551040649414,
      "activations/layer14_attention_weight_min": -36.42672348022461,
      "activations/layer15_attention_weight_max": 40.1713752746582,
      "activations/layer15_attention_weight_min": -34.54657745361328,
      "activations/layer16_attention_weight_max": 30.367170333862305,
      "activations/layer16_attention_weight_min": -28.145431518554688,
      "activations/layer17_attention_weight_max": 28.45228385925293,
      "activations/layer17_attention_weight_min": -26.575693130493164,
      "activations/layer18_attention_weight_max": 31.42451286315918,
      "activations/layer18_attention_weight_min": -24.6976375579834,
      "activations/layer19_attention_weight_max": 31.11979103088379,
      "activations/layer19_attention_weight_min": -32.729862213134766,
      "activations/layer1_attention_weight_max": 16.364521026611328,
      "activations/layer1_attention_weight_min": -15.10888957977295,
      "activations/layer20_attention_weight_max": 29.6492977142334,
      "activations/layer20_attention_weight_min": -26.836109161376953,
      "activations/layer21_attention_weight_max": 27.63187026977539,
      "activations/layer21_attention_weight_min": -24.847654342651367,
      "activations/layer22_attention_weight_max": 41.29246139526367,
      "activations/layer22_attention_weight_min": -30.93581199645996,
      "activations/layer23_attention_weight_max": 31.68178367614746,
      "activations/layer23_attention_weight_min": -28.032455444335938,
      "activations/layer2_attention_weight_max": 30.444995880126953,
      "activations/layer2_attention_weight_min": -30.768293380737305,
      "activations/layer3_attention_weight_max": 93.91365814208984,
      "activations/layer3_attention_weight_min": -91.14166259765625,
      "activations/layer4_attention_weight_max": 104.7860107421875,
      "activations/layer4_attention_weight_min": -107.81591033935547,
      "activations/layer5_attention_weight_max": 49.82122039794922,
      "activations/layer5_attention_weight_min": -63.45995330810547,
      "activations/layer6_attention_weight_max": 43.44485092163086,
      "activations/layer6_attention_weight_min": -45.54438781738281,
      "activations/layer7_attention_weight_max": 88.69478607177734,
      "activations/layer7_attention_weight_min": -91.70127868652344,
      "activations/layer8_attention_weight_max": 39.707149505615234,
      "activations/layer8_attention_weight_min": -44.28376770019531,
      "activations/layer9_attention_weight_max": 32.034637451171875,
      "activations/layer9_attention_weight_min": -32.80242156982422,
      "epoch": 15.4,
      "learning_rate": 5.1179924242424245e-05,
      "loss": 2.7278,
      "step": 265100
    },
    {
      "activations/layer0_attention_weight_max": 15.777649879455566,
      "activations/layer0_attention_weight_min": -12.799764633178711,
      "activations/layer10_attention_weight_max": 31.86472511291504,
      "activations/layer10_attention_weight_min": -31.99678611755371,
      "activations/layer11_attention_weight_max": 31.824848175048828,
      "activations/layer11_attention_weight_min": -32.93384552001953,
      "activations/layer12_attention_weight_max": 25.86690902709961,
      "activations/layer12_attention_weight_min": -25.80170440673828,
      "activations/layer13_attention_weight_max": 36.897804260253906,
      "activations/layer13_attention_weight_min": -35.147274017333984,
      "activations/layer14_attention_weight_max": 41.61263656616211,
      "activations/layer14_attention_weight_min": -36.14435577392578,
      "activations/layer15_attention_weight_max": 36.208641052246094,
      "activations/layer15_attention_weight_min": -35.5915641784668,
      "activations/layer16_attention_weight_max": 30.809398651123047,
      "activations/layer16_attention_weight_min": -28.816162109375,
      "activations/layer17_attention_weight_max": 27.842172622680664,
      "activations/layer17_attention_weight_min": -28.242431640625,
      "activations/layer18_attention_weight_max": 29.792821884155273,
      "activations/layer18_attention_weight_min": -27.985294342041016,
      "activations/layer19_attention_weight_max": 31.657455444335938,
      "activations/layer19_attention_weight_min": -31.01017189025879,
      "activations/layer1_attention_weight_max": 15.710542678833008,
      "activations/layer1_attention_weight_min": -15.305764198303223,
      "activations/layer20_attention_weight_max": 29.095874786376953,
      "activations/layer20_attention_weight_min": -24.515296936035156,
      "activations/layer21_attention_weight_max": 28.066864013671875,
      "activations/layer21_attention_weight_min": -25.3825626373291,
      "activations/layer22_attention_weight_max": 39.79154968261719,
      "activations/layer22_attention_weight_min": -35.21845245361328,
      "activations/layer23_attention_weight_max": 29.471790313720703,
      "activations/layer23_attention_weight_min": -25.206037521362305,
      "activations/layer2_attention_weight_max": 31.156103134155273,
      "activations/layer2_attention_weight_min": -32.28413772583008,
      "activations/layer3_attention_weight_max": 92.58971405029297,
      "activations/layer3_attention_weight_min": -92.76319122314453,
      "activations/layer4_attention_weight_max": 106.33565521240234,
      "activations/layer4_attention_weight_min": -105.6649398803711,
      "activations/layer5_attention_weight_max": 48.658775329589844,
      "activations/layer5_attention_weight_min": -59.57545471191406,
      "activations/layer6_attention_weight_max": 44.455787658691406,
      "activations/layer6_attention_weight_min": -44.4542121887207,
      "activations/layer7_attention_weight_max": 88.80152893066406,
      "activations/layer7_attention_weight_min": -92.79234313964844,
      "activations/layer8_attention_weight_max": 39.70275115966797,
      "activations/layer8_attention_weight_min": -39.92445373535156,
      "activations/layer9_attention_weight_max": 30.9427547454834,
      "activations/layer9_attention_weight_min": -32.13164520263672,
      "epoch": 15.41,
      "learning_rate": 5.1160984848484846e-05,
      "loss": 2.7362,
      "step": 265150
    },
    {
      "activations/layer0_attention_weight_max": 15.576311111450195,
      "activations/layer0_attention_weight_min": -12.64409065246582,
      "activations/layer10_attention_weight_max": 35.71328353881836,
      "activations/layer10_attention_weight_min": -34.79024124145508,
      "activations/layer11_attention_weight_max": 36.532318115234375,
      "activations/layer11_attention_weight_min": -34.29920959472656,
      "activations/layer12_attention_weight_max": 26.609432220458984,
      "activations/layer12_attention_weight_min": -27.1109561920166,
      "activations/layer13_attention_weight_max": 39.86328887939453,
      "activations/layer13_attention_weight_min": -35.28660202026367,
      "activations/layer14_attention_weight_max": 41.892398834228516,
      "activations/layer14_attention_weight_min": -37.881221771240234,
      "activations/layer15_attention_weight_max": 36.2713623046875,
      "activations/layer15_attention_weight_min": -34.911861419677734,
      "activations/layer16_attention_weight_max": 27.664737701416016,
      "activations/layer16_attention_weight_min": -29.582813262939453,
      "activations/layer17_attention_weight_max": 29.188074111938477,
      "activations/layer17_attention_weight_min": -27.10735511779785,
      "activations/layer18_attention_weight_max": 32.65746307373047,
      "activations/layer18_attention_weight_min": -25.98839569091797,
      "activations/layer19_attention_weight_max": 31.230615615844727,
      "activations/layer19_attention_weight_min": -30.5305118560791,
      "activations/layer1_attention_weight_max": 15.94842529296875,
      "activations/layer1_attention_weight_min": -14.970368385314941,
      "activations/layer20_attention_weight_max": 26.322704315185547,
      "activations/layer20_attention_weight_min": -28.86212158203125,
      "activations/layer21_attention_weight_max": 26.765483856201172,
      "activations/layer21_attention_weight_min": -26.020694732666016,
      "activations/layer22_attention_weight_max": 40.964683532714844,
      "activations/layer22_attention_weight_min": -32.09474563598633,
      "activations/layer23_attention_weight_max": 30.15138053894043,
      "activations/layer23_attention_weight_min": -26.474803924560547,
      "activations/layer2_attention_weight_max": 32.59711456298828,
      "activations/layer2_attention_weight_min": -33.41852569580078,
      "activations/layer3_attention_weight_max": 96.31636047363281,
      "activations/layer3_attention_weight_min": -100.01036071777344,
      "activations/layer4_attention_weight_max": 106.38372802734375,
      "activations/layer4_attention_weight_min": -110.70674133300781,
      "activations/layer5_attention_weight_max": 52.81793975830078,
      "activations/layer5_attention_weight_min": -61.9993896484375,
      "activations/layer6_attention_weight_max": 44.50688171386719,
      "activations/layer6_attention_weight_min": -47.1898307800293,
      "activations/layer7_attention_weight_max": 92.07300567626953,
      "activations/layer7_attention_weight_min": -98.54450225830078,
      "activations/layer8_attention_weight_max": 44.84857940673828,
      "activations/layer8_attention_weight_min": -45.238704681396484,
      "activations/layer9_attention_weight_max": 34.60963439941406,
      "activations/layer9_attention_weight_min": -34.95485305786133,
      "epoch": 15.41,
      "learning_rate": 5.114204545454545e-05,
      "loss": 2.7293,
      "step": 265200
    },
    {
      "activations/layer0_attention_weight_max": 15.393342018127441,
      "activations/layer0_attention_weight_min": -12.305534362792969,
      "activations/layer10_attention_weight_max": 31.297649383544922,
      "activations/layer10_attention_weight_min": -31.830739974975586,
      "activations/layer11_attention_weight_max": 31.358978271484375,
      "activations/layer11_attention_weight_min": -32.774845123291016,
      "activations/layer12_attention_weight_max": 27.380701065063477,
      "activations/layer12_attention_weight_min": -25.48922348022461,
      "activations/layer13_attention_weight_max": 38.82289123535156,
      "activations/layer13_attention_weight_min": -33.776676177978516,
      "activations/layer14_attention_weight_max": 40.276939392089844,
      "activations/layer14_attention_weight_min": -38.89500427246094,
      "activations/layer15_attention_weight_max": 37.20552444458008,
      "activations/layer15_attention_weight_min": -35.53350830078125,
      "activations/layer16_attention_weight_max": 29.527910232543945,
      "activations/layer16_attention_weight_min": -28.812021255493164,
      "activations/layer17_attention_weight_max": 28.163022994995117,
      "activations/layer17_attention_weight_min": -26.997989654541016,
      "activations/layer18_attention_weight_max": 29.623571395874023,
      "activations/layer18_attention_weight_min": -25.988998413085938,
      "activations/layer19_attention_weight_max": 34.375057220458984,
      "activations/layer19_attention_weight_min": -30.993656158447266,
      "activations/layer1_attention_weight_max": 16.08934211730957,
      "activations/layer1_attention_weight_min": -15.367460250854492,
      "activations/layer20_attention_weight_max": 32.14743423461914,
      "activations/layer20_attention_weight_min": -25.9272403717041,
      "activations/layer21_attention_weight_max": 33.959869384765625,
      "activations/layer21_attention_weight_min": -26.33340835571289,
      "activations/layer22_attention_weight_max": 42.228328704833984,
      "activations/layer22_attention_weight_min": -32.01181411743164,
      "activations/layer23_attention_weight_max": 33.54194641113281,
      "activations/layer23_attention_weight_min": -25.44171714782715,
      "activations/layer2_attention_weight_max": 31.766111373901367,
      "activations/layer2_attention_weight_min": -32.28102493286133,
      "activations/layer3_attention_weight_max": 94.3089370727539,
      "activations/layer3_attention_weight_min": -97.8580093383789,
      "activations/layer4_attention_weight_max": 102.5648193359375,
      "activations/layer4_attention_weight_min": -105.59905242919922,
      "activations/layer5_attention_weight_max": 47.30216598510742,
      "activations/layer5_attention_weight_min": -63.934349060058594,
      "activations/layer6_attention_weight_max": 43.7771110534668,
      "activations/layer6_attention_weight_min": -45.08909606933594,
      "activations/layer7_attention_weight_max": 92.21509552001953,
      "activations/layer7_attention_weight_min": -88.46078491210938,
      "activations/layer8_attention_weight_max": 39.8315544128418,
      "activations/layer8_attention_weight_min": -40.583457946777344,
      "activations/layer9_attention_weight_max": 30.60455322265625,
      "activations/layer9_attention_weight_min": -33.38534164428711,
      "epoch": 15.41,
      "learning_rate": 5.1123106060606056e-05,
      "loss": 2.7448,
      "step": 265250
    },
    {
      "activations/layer0_attention_weight_max": 16.01255226135254,
      "activations/layer0_attention_weight_min": -12.591766357421875,
      "activations/layer10_attention_weight_max": 34.40578079223633,
      "activations/layer10_attention_weight_min": -31.809293746948242,
      "activations/layer11_attention_weight_max": 34.95098876953125,
      "activations/layer11_attention_weight_min": -32.73799514770508,
      "activations/layer12_attention_weight_max": 27.34790802001953,
      "activations/layer12_attention_weight_min": -26.196027755737305,
      "activations/layer13_attention_weight_max": 40.23667526245117,
      "activations/layer13_attention_weight_min": -35.12570571899414,
      "activations/layer14_attention_weight_max": 42.64600372314453,
      "activations/layer14_attention_weight_min": -38.061702728271484,
      "activations/layer15_attention_weight_max": 38.70232009887695,
      "activations/layer15_attention_weight_min": -33.78274154663086,
      "activations/layer16_attention_weight_max": 30.85698699951172,
      "activations/layer16_attention_weight_min": -27.05181121826172,
      "activations/layer17_attention_weight_max": 31.58677101135254,
      "activations/layer17_attention_weight_min": -26.680339813232422,
      "activations/layer18_attention_weight_max": 31.073284149169922,
      "activations/layer18_attention_weight_min": -25.909719467163086,
      "activations/layer19_attention_weight_max": 31.305530548095703,
      "activations/layer19_attention_weight_min": -32.223388671875,
      "activations/layer1_attention_weight_max": 17.22532081604004,
      "activations/layer1_attention_weight_min": -14.75864315032959,
      "activations/layer20_attention_weight_max": 26.5380916595459,
      "activations/layer20_attention_weight_min": -25.4277286529541,
      "activations/layer21_attention_weight_max": 25.92229652404785,
      "activations/layer21_attention_weight_min": -24.293973922729492,
      "activations/layer22_attention_weight_max": 37.14837646484375,
      "activations/layer22_attention_weight_min": -29.650447845458984,
      "activations/layer23_attention_weight_max": 30.011695861816406,
      "activations/layer23_attention_weight_min": -24.309326171875,
      "activations/layer2_attention_weight_max": 32.13240432739258,
      "activations/layer2_attention_weight_min": -31.645572662353516,
      "activations/layer3_attention_weight_max": 92.35801696777344,
      "activations/layer3_attention_weight_min": -93.34725952148438,
      "activations/layer4_attention_weight_max": 103.31793212890625,
      "activations/layer4_attention_weight_min": -109.49395751953125,
      "activations/layer5_attention_weight_max": 51.71104431152344,
      "activations/layer5_attention_weight_min": -62.17486572265625,
      "activations/layer6_attention_weight_max": 42.6694221496582,
      "activations/layer6_attention_weight_min": -43.97072982788086,
      "activations/layer7_attention_weight_max": 89.676025390625,
      "activations/layer7_attention_weight_min": -91.46231842041016,
      "activations/layer8_attention_weight_max": 41.88691329956055,
      "activations/layer8_attention_weight_min": -40.40999221801758,
      "activations/layer9_attention_weight_max": 33.153324127197266,
      "activations/layer9_attention_weight_min": -33.107452392578125,
      "epoch": 15.42,
      "learning_rate": 5.1104166666666664e-05,
      "loss": 2.7528,
      "step": 265300
    },
    {
      "activations/layer0_attention_weight_max": 15.218110084533691,
      "activations/layer0_attention_weight_min": -11.679877281188965,
      "activations/layer10_attention_weight_max": 37.47407150268555,
      "activations/layer10_attention_weight_min": -36.07587432861328,
      "activations/layer11_attention_weight_max": 38.169776916503906,
      "activations/layer11_attention_weight_min": -37.01588439941406,
      "activations/layer12_attention_weight_max": 27.33310890197754,
      "activations/layer12_attention_weight_min": -26.61737632751465,
      "activations/layer13_attention_weight_max": 39.885894775390625,
      "activations/layer13_attention_weight_min": -35.10219192504883,
      "activations/layer14_attention_weight_max": 41.98567199707031,
      "activations/layer14_attention_weight_min": -37.52124786376953,
      "activations/layer15_attention_weight_max": 39.2387580871582,
      "activations/layer15_attention_weight_min": -34.78947448730469,
      "activations/layer16_attention_weight_max": 30.622997283935547,
      "activations/layer16_attention_weight_min": -27.732744216918945,
      "activations/layer17_attention_weight_max": 31.217411041259766,
      "activations/layer17_attention_weight_min": -26.214845657348633,
      "activations/layer18_attention_weight_max": 31.631441116333008,
      "activations/layer18_attention_weight_min": -24.177547454833984,
      "activations/layer19_attention_weight_max": 35.1759033203125,
      "activations/layer19_attention_weight_min": -30.183551788330078,
      "activations/layer1_attention_weight_max": 16.423419952392578,
      "activations/layer1_attention_weight_min": -14.727519989013672,
      "activations/layer20_attention_weight_max": 29.912235260009766,
      "activations/layer20_attention_weight_min": -24.519254684448242,
      "activations/layer21_attention_weight_max": 28.375822067260742,
      "activations/layer21_attention_weight_min": -23.4581298828125,
      "activations/layer22_attention_weight_max": 42.38344955444336,
      "activations/layer22_attention_weight_min": -33.50020980834961,
      "activations/layer23_attention_weight_max": 31.756162643432617,
      "activations/layer23_attention_weight_min": -26.49025535583496,
      "activations/layer2_attention_weight_max": 35.218544006347656,
      "activations/layer2_attention_weight_min": -33.32048034667969,
      "activations/layer3_attention_weight_max": 99.5030517578125,
      "activations/layer3_attention_weight_min": -99.76959228515625,
      "activations/layer4_attention_weight_max": 114.6526107788086,
      "activations/layer4_attention_weight_min": -116.70145416259766,
      "activations/layer5_attention_weight_max": 53.313690185546875,
      "activations/layer5_attention_weight_min": -62.2223014831543,
      "activations/layer6_attention_weight_max": 47.830223083496094,
      "activations/layer6_attention_weight_min": -47.437232971191406,
      "activations/layer7_attention_weight_max": 101.68480682373047,
      "activations/layer7_attention_weight_min": -101.21963500976562,
      "activations/layer8_attention_weight_max": 45.19715881347656,
      "activations/layer8_attention_weight_min": -46.71873474121094,
      "activations/layer9_attention_weight_max": 34.93400192260742,
      "activations/layer9_attention_weight_min": -36.00552749633789,
      "epoch": 15.42,
      "learning_rate": 5.108522727272727e-05,
      "loss": 2.7279,
      "step": 265350
    },
    {
      "activations/layer0_attention_weight_max": 16.325164794921875,
      "activations/layer0_attention_weight_min": -12.784391403198242,
      "activations/layer10_attention_weight_max": 33.88996505737305,
      "activations/layer10_attention_weight_min": -33.175506591796875,
      "activations/layer11_attention_weight_max": 34.558834075927734,
      "activations/layer11_attention_weight_min": -35.40596008300781,
      "activations/layer12_attention_weight_max": 26.985000610351562,
      "activations/layer12_attention_weight_min": -26.282922744750977,
      "activations/layer13_attention_weight_max": 40.37799835205078,
      "activations/layer13_attention_weight_min": -35.88480758666992,
      "activations/layer14_attention_weight_max": 48.06296157836914,
      "activations/layer14_attention_weight_min": -38.77867126464844,
      "activations/layer15_attention_weight_max": 39.748573303222656,
      "activations/layer15_attention_weight_min": -35.035953521728516,
      "activations/layer16_attention_weight_max": 29.936235427856445,
      "activations/layer16_attention_weight_min": -32.746986389160156,
      "activations/layer17_attention_weight_max": 29.575708389282227,
      "activations/layer17_attention_weight_min": -27.057018280029297,
      "activations/layer18_attention_weight_max": 31.182334899902344,
      "activations/layer18_attention_weight_min": -25.864782333374023,
      "activations/layer19_attention_weight_max": 31.82146644592285,
      "activations/layer19_attention_weight_min": -32.6796875,
      "activations/layer1_attention_weight_max": 16.444150924682617,
      "activations/layer1_attention_weight_min": -15.483671188354492,
      "activations/layer20_attention_weight_max": 28.164716720581055,
      "activations/layer20_attention_weight_min": -24.816755294799805,
      "activations/layer21_attention_weight_max": 27.358585357666016,
      "activations/layer21_attention_weight_min": -23.879064559936523,
      "activations/layer22_attention_weight_max": 42.16661834716797,
      "activations/layer22_attention_weight_min": -29.742889404296875,
      "activations/layer23_attention_weight_max": 31.232257843017578,
      "activations/layer23_attention_weight_min": -25.374767303466797,
      "activations/layer2_attention_weight_max": 31.60724449157715,
      "activations/layer2_attention_weight_min": -32.13633346557617,
      "activations/layer3_attention_weight_max": 91.28253173828125,
      "activations/layer3_attention_weight_min": -96.15082550048828,
      "activations/layer4_attention_weight_max": 104.2500991821289,
      "activations/layer4_attention_weight_min": -105.46748352050781,
      "activations/layer5_attention_weight_max": 50.12510681152344,
      "activations/layer5_attention_weight_min": -59.39081573486328,
      "activations/layer6_attention_weight_max": 43.36692810058594,
      "activations/layer6_attention_weight_min": -45.99848937988281,
      "activations/layer7_attention_weight_max": 91.74810791015625,
      "activations/layer7_attention_weight_min": -92.46924591064453,
      "activations/layer8_attention_weight_max": 41.66838455200195,
      "activations/layer8_attention_weight_min": -42.26145553588867,
      "activations/layer9_attention_weight_max": 34.4796257019043,
      "activations/layer9_attention_weight_min": -33.3697624206543,
      "epoch": 15.42,
      "learning_rate": 5.1066287878787874e-05,
      "loss": 2.7306,
      "step": 265400
    },
    {
      "activations/layer0_attention_weight_max": 15.933672904968262,
      "activations/layer0_attention_weight_min": -12.005982398986816,
      "activations/layer10_attention_weight_max": 32.692787170410156,
      "activations/layer10_attention_weight_min": -32.18156814575195,
      "activations/layer11_attention_weight_max": 32.57878494262695,
      "activations/layer11_attention_weight_min": -31.291534423828125,
      "activations/layer12_attention_weight_max": 26.175556182861328,
      "activations/layer12_attention_weight_min": -25.934589385986328,
      "activations/layer13_attention_weight_max": 40.94551086425781,
      "activations/layer13_attention_weight_min": -36.74518966674805,
      "activations/layer14_attention_weight_max": 42.92207717895508,
      "activations/layer14_attention_weight_min": -38.03083038330078,
      "activations/layer15_attention_weight_max": 38.51433181762695,
      "activations/layer15_attention_weight_min": -35.028236389160156,
      "activations/layer16_attention_weight_max": 29.858877182006836,
      "activations/layer16_attention_weight_min": -27.778770446777344,
      "activations/layer17_attention_weight_max": 29.51956558227539,
      "activations/layer17_attention_weight_min": -26.147981643676758,
      "activations/layer18_attention_weight_max": 30.23518180847168,
      "activations/layer18_attention_weight_min": -23.852964401245117,
      "activations/layer19_attention_weight_max": 34.598228454589844,
      "activations/layer19_attention_weight_min": -30.03952980041504,
      "activations/layer1_attention_weight_max": 15.644739151000977,
      "activations/layer1_attention_weight_min": -13.571757316589355,
      "activations/layer20_attention_weight_max": 29.9507999420166,
      "activations/layer20_attention_weight_min": -24.50982666015625,
      "activations/layer21_attention_weight_max": 29.437734603881836,
      "activations/layer21_attention_weight_min": -25.118026733398438,
      "activations/layer22_attention_weight_max": 41.32406997680664,
      "activations/layer22_attention_weight_min": -29.307506561279297,
      "activations/layer23_attention_weight_max": 36.157630920410156,
      "activations/layer23_attention_weight_min": -24.208845138549805,
      "activations/layer2_attention_weight_max": 32.47822570800781,
      "activations/layer2_attention_weight_min": -32.2742919921875,
      "activations/layer3_attention_weight_max": 94.23102569580078,
      "activations/layer3_attention_weight_min": -92.86907958984375,
      "activations/layer4_attention_weight_max": 108.869140625,
      "activations/layer4_attention_weight_min": -106.12171936035156,
      "activations/layer5_attention_weight_max": 50.59208679199219,
      "activations/layer5_attention_weight_min": -60.894775390625,
      "activations/layer6_attention_weight_max": 43.83714294433594,
      "activations/layer6_attention_weight_min": -44.433128356933594,
      "activations/layer7_attention_weight_max": 97.62683868408203,
      "activations/layer7_attention_weight_min": -97.31672668457031,
      "activations/layer8_attention_weight_max": 44.550743103027344,
      "activations/layer8_attention_weight_min": -44.22189712524414,
      "activations/layer9_attention_weight_max": 32.63964080810547,
      "activations/layer9_attention_weight_min": -33.484683990478516,
      "epoch": 15.42,
      "learning_rate": 5.1047348484848476e-05,
      "loss": 2.7381,
      "step": 265450
    },
    {
      "activations/layer0_attention_weight_max": 15.180733680725098,
      "activations/layer0_attention_weight_min": -12.916175842285156,
      "activations/layer10_attention_weight_max": 31.949546813964844,
      "activations/layer10_attention_weight_min": -33.06233596801758,
      "activations/layer11_attention_weight_max": 33.226585388183594,
      "activations/layer11_attention_weight_min": -32.90944290161133,
      "activations/layer12_attention_weight_max": 26.130821228027344,
      "activations/layer12_attention_weight_min": -27.179256439208984,
      "activations/layer13_attention_weight_max": 39.13875961303711,
      "activations/layer13_attention_weight_min": -35.27375030517578,
      "activations/layer14_attention_weight_max": 41.024845123291016,
      "activations/layer14_attention_weight_min": -37.51557922363281,
      "activations/layer15_attention_weight_max": 36.47585678100586,
      "activations/layer15_attention_weight_min": -35.1905403137207,
      "activations/layer16_attention_weight_max": 29.21327781677246,
      "activations/layer16_attention_weight_min": -27.492359161376953,
      "activations/layer17_attention_weight_max": 29.54776382446289,
      "activations/layer17_attention_weight_min": -27.143152236938477,
      "activations/layer18_attention_weight_max": 32.01095199584961,
      "activations/layer18_attention_weight_min": -26.47281837463379,
      "activations/layer19_attention_weight_max": 34.43027114868164,
      "activations/layer19_attention_weight_min": -34.97443771362305,
      "activations/layer1_attention_weight_max": 16.730051040649414,
      "activations/layer1_attention_weight_min": -17.189510345458984,
      "activations/layer20_attention_weight_max": 28.238231658935547,
      "activations/layer20_attention_weight_min": -27.29022789001465,
      "activations/layer21_attention_weight_max": 28.36993980407715,
      "activations/layer21_attention_weight_min": -25.98550796508789,
      "activations/layer22_attention_weight_max": 49.101016998291016,
      "activations/layer22_attention_weight_min": -32.372982025146484,
      "activations/layer23_attention_weight_max": 32.58260726928711,
      "activations/layer23_attention_weight_min": -25.478694915771484,
      "activations/layer2_attention_weight_max": 32.52789306640625,
      "activations/layer2_attention_weight_min": -33.032875061035156,
      "activations/layer3_attention_weight_max": 94.3461685180664,
      "activations/layer3_attention_weight_min": -96.5960464477539,
      "activations/layer4_attention_weight_max": 105.03386688232422,
      "activations/layer4_attention_weight_min": -107.59614562988281,
      "activations/layer5_attention_weight_max": 51.32206344604492,
      "activations/layer5_attention_weight_min": -60.731605529785156,
      "activations/layer6_attention_weight_max": 41.84501266479492,
      "activations/layer6_attention_weight_min": -44.56871032714844,
      "activations/layer7_attention_weight_max": 90.18991088867188,
      "activations/layer7_attention_weight_min": -93.35700225830078,
      "activations/layer8_attention_weight_max": 40.94950485229492,
      "activations/layer8_attention_weight_min": -42.88674545288086,
      "activations/layer9_attention_weight_max": 31.105976104736328,
      "activations/layer9_attention_weight_min": -34.56144714355469,
      "epoch": 15.43,
      "learning_rate": 5.102840909090909e-05,
      "loss": 2.742,
      "step": 265500
    },
    {
      "activations/layer0_attention_weight_max": 15.948270797729492,
      "activations/layer0_attention_weight_min": -12.733031272888184,
      "activations/layer10_attention_weight_max": 33.32851791381836,
      "activations/layer10_attention_weight_min": -33.13420104980469,
      "activations/layer11_attention_weight_max": 33.48114013671875,
      "activations/layer11_attention_weight_min": -32.9075927734375,
      "activations/layer12_attention_weight_max": 29.488754272460938,
      "activations/layer12_attention_weight_min": -25.895849227905273,
      "activations/layer13_attention_weight_max": 37.40142059326172,
      "activations/layer13_attention_weight_min": -33.774383544921875,
      "activations/layer14_attention_weight_max": 41.21152114868164,
      "activations/layer14_attention_weight_min": -37.9343147277832,
      "activations/layer15_attention_weight_max": 38.72914123535156,
      "activations/layer15_attention_weight_min": -33.745567321777344,
      "activations/layer16_attention_weight_max": 28.259723663330078,
      "activations/layer16_attention_weight_min": -30.910066604614258,
      "activations/layer17_attention_weight_max": 29.23499298095703,
      "activations/layer17_attention_weight_min": -25.343971252441406,
      "activations/layer18_attention_weight_max": 29.93609619140625,
      "activations/layer18_attention_weight_min": -25.660005569458008,
      "activations/layer19_attention_weight_max": 32.498050689697266,
      "activations/layer19_attention_weight_min": -31.05728530883789,
      "activations/layer1_attention_weight_max": 16.65699577331543,
      "activations/layer1_attention_weight_min": -14.163912773132324,
      "activations/layer20_attention_weight_max": 26.518774032592773,
      "activations/layer20_attention_weight_min": -27.128814697265625,
      "activations/layer21_attention_weight_max": 27.115203857421875,
      "activations/layer21_attention_weight_min": -26.25360679626465,
      "activations/layer22_attention_weight_max": 43.53703689575195,
      "activations/layer22_attention_weight_min": -33.588951110839844,
      "activations/layer23_attention_weight_max": 32.34271240234375,
      "activations/layer23_attention_weight_min": -27.149608612060547,
      "activations/layer2_attention_weight_max": 32.587493896484375,
      "activations/layer2_attention_weight_min": -31.81551742553711,
      "activations/layer3_attention_weight_max": 94.18696594238281,
      "activations/layer3_attention_weight_min": -96.32154846191406,
      "activations/layer4_attention_weight_max": 102.78325653076172,
      "activations/layer4_attention_weight_min": -106.1130142211914,
      "activations/layer5_attention_weight_max": 53.39603805541992,
      "activations/layer5_attention_weight_min": -65.42845153808594,
      "activations/layer6_attention_weight_max": 43.64168167114258,
      "activations/layer6_attention_weight_min": -46.54450225830078,
      "activations/layer7_attention_weight_max": 91.54624938964844,
      "activations/layer7_attention_weight_min": -90.15264129638672,
      "activations/layer8_attention_weight_max": 40.26301193237305,
      "activations/layer8_attention_weight_min": -42.32087707519531,
      "activations/layer9_attention_weight_max": 32.78271484375,
      "activations/layer9_attention_weight_min": -33.723812103271484,
      "epoch": 15.43,
      "learning_rate": 5.100946969696969e-05,
      "loss": 2.7575,
      "step": 265550
    },
    {
      "activations/layer0_attention_weight_max": 16.474315643310547,
      "activations/layer0_attention_weight_min": -13.498848915100098,
      "activations/layer10_attention_weight_max": 38.24407958984375,
      "activations/layer10_attention_weight_min": -34.98445510864258,
      "activations/layer11_attention_weight_max": 36.75436782836914,
      "activations/layer11_attention_weight_min": -36.02836608886719,
      "activations/layer12_attention_weight_max": 25.513330459594727,
      "activations/layer12_attention_weight_min": -29.42234992980957,
      "activations/layer13_attention_weight_max": 39.4371223449707,
      "activations/layer13_attention_weight_min": -34.59022521972656,
      "activations/layer14_attention_weight_max": 43.74993896484375,
      "activations/layer14_attention_weight_min": -37.8160285949707,
      "activations/layer15_attention_weight_max": 36.122127532958984,
      "activations/layer15_attention_weight_min": -34.540313720703125,
      "activations/layer16_attention_weight_max": 28.061559677124023,
      "activations/layer16_attention_weight_min": -26.938232421875,
      "activations/layer17_attention_weight_max": 28.75052833557129,
      "activations/layer17_attention_weight_min": -25.158105850219727,
      "activations/layer18_attention_weight_max": 29.48747444152832,
      "activations/layer18_attention_weight_min": -23.841272354125977,
      "activations/layer19_attention_weight_max": 33.901123046875,
      "activations/layer19_attention_weight_min": -30.339040756225586,
      "activations/layer1_attention_weight_max": 16.929462432861328,
      "activations/layer1_attention_weight_min": -14.381268501281738,
      "activations/layer20_attention_weight_max": 29.558870315551758,
      "activations/layer20_attention_weight_min": -25.197952270507812,
      "activations/layer21_attention_weight_max": 27.560867309570312,
      "activations/layer21_attention_weight_min": -25.976959228515625,
      "activations/layer22_attention_weight_max": 38.34965133666992,
      "activations/layer22_attention_weight_min": -28.644317626953125,
      "activations/layer23_attention_weight_max": 33.696414947509766,
      "activations/layer23_attention_weight_min": -25.076725006103516,
      "activations/layer2_attention_weight_max": 31.19955062866211,
      "activations/layer2_attention_weight_min": -32.026493072509766,
      "activations/layer3_attention_weight_max": 91.05320739746094,
      "activations/layer3_attention_weight_min": -99.69535827636719,
      "activations/layer4_attention_weight_max": 104.27644348144531,
      "activations/layer4_attention_weight_min": -108.5207748413086,
      "activations/layer5_attention_weight_max": 55.73208236694336,
      "activations/layer5_attention_weight_min": -57.67213439941406,
      "activations/layer6_attention_weight_max": 42.8443489074707,
      "activations/layer6_attention_weight_min": -43.0409049987793,
      "activations/layer7_attention_weight_max": 104.0804443359375,
      "activations/layer7_attention_weight_min": -96.7423324584961,
      "activations/layer8_attention_weight_max": 44.6570930480957,
      "activations/layer8_attention_weight_min": -46.89339065551758,
      "activations/layer9_attention_weight_max": 33.10702133178711,
      "activations/layer9_attention_weight_min": -35.82167434692383,
      "epoch": 15.43,
      "learning_rate": 5.099090909090908e-05,
      "loss": 2.7275,
      "step": 265600
    },
    {
      "activations/layer0_attention_weight_max": 15.351754188537598,
      "activations/layer0_attention_weight_min": -13.034210205078125,
      "activations/layer10_attention_weight_max": 33.055747985839844,
      "activations/layer10_attention_weight_min": -33.047847747802734,
      "activations/layer11_attention_weight_max": 34.03501510620117,
      "activations/layer11_attention_weight_min": -34.74481201171875,
      "activations/layer12_attention_weight_max": 27.344440460205078,
      "activations/layer12_attention_weight_min": -27.599271774291992,
      "activations/layer13_attention_weight_max": 41.084228515625,
      "activations/layer13_attention_weight_min": -38.08123016357422,
      "activations/layer14_attention_weight_max": 43.02896499633789,
      "activations/layer14_attention_weight_min": -39.581546783447266,
      "activations/layer15_attention_weight_max": 39.30475997924805,
      "activations/layer15_attention_weight_min": -36.109683990478516,
      "activations/layer16_attention_weight_max": 30.986051559448242,
      "activations/layer16_attention_weight_min": -29.438533782958984,
      "activations/layer17_attention_weight_max": 32.194068908691406,
      "activations/layer17_attention_weight_min": -28.166635513305664,
      "activations/layer18_attention_weight_max": 30.465486526489258,
      "activations/layer18_attention_weight_min": -27.44731330871582,
      "activations/layer19_attention_weight_max": 33.903175354003906,
      "activations/layer19_attention_weight_min": -32.73347473144531,
      "activations/layer1_attention_weight_max": 16.444149017333984,
      "activations/layer1_attention_weight_min": -15.096233367919922,
      "activations/layer20_attention_weight_max": 31.356542587280273,
      "activations/layer20_attention_weight_min": -26.18089485168457,
      "activations/layer21_attention_weight_max": 30.455522537231445,
      "activations/layer21_attention_weight_min": -25.56854248046875,
      "activations/layer22_attention_weight_max": 44.85115051269531,
      "activations/layer22_attention_weight_min": -32.97220993041992,
      "activations/layer23_attention_weight_max": 33.222442626953125,
      "activations/layer23_attention_weight_min": -25.868934631347656,
      "activations/layer2_attention_weight_max": 30.144275665283203,
      "activations/layer2_attention_weight_min": -31.096296310424805,
      "activations/layer3_attention_weight_max": 90.17615509033203,
      "activations/layer3_attention_weight_min": -91.18875122070312,
      "activations/layer4_attention_weight_max": 103.93048095703125,
      "activations/layer4_attention_weight_min": -108.02362060546875,
      "activations/layer5_attention_weight_max": 52.5594482421875,
      "activations/layer5_attention_weight_min": -63.00164794921875,
      "activations/layer6_attention_weight_max": 42.619102478027344,
      "activations/layer6_attention_weight_min": -45.649497985839844,
      "activations/layer7_attention_weight_max": 97.74613952636719,
      "activations/layer7_attention_weight_min": -91.08828735351562,
      "activations/layer8_attention_weight_max": 42.96859359741211,
      "activations/layer8_attention_weight_min": -43.194095611572266,
      "activations/layer9_attention_weight_max": 31.870023727416992,
      "activations/layer9_attention_weight_min": -33.113704681396484,
      "epoch": 15.44,
      "learning_rate": 5.097196969696969e-05,
      "loss": 2.7568,
      "step": 265650
    },
    {
      "activations/layer0_attention_weight_max": 16.57377815246582,
      "activations/layer0_attention_weight_min": -12.503170013427734,
      "activations/layer10_attention_weight_max": 33.55942916870117,
      "activations/layer10_attention_weight_min": -34.257179260253906,
      "activations/layer11_attention_weight_max": 33.63120651245117,
      "activations/layer11_attention_weight_min": -35.59067153930664,
      "activations/layer12_attention_weight_max": 27.043394088745117,
      "activations/layer12_attention_weight_min": -25.808387756347656,
      "activations/layer13_attention_weight_max": 38.046653747558594,
      "activations/layer13_attention_weight_min": -34.92324447631836,
      "activations/layer14_attention_weight_max": 40.826290130615234,
      "activations/layer14_attention_weight_min": -36.35129165649414,
      "activations/layer15_attention_weight_max": 35.88661193847656,
      "activations/layer15_attention_weight_min": -34.34529495239258,
      "activations/layer16_attention_weight_max": 29.165437698364258,
      "activations/layer16_attention_weight_min": -28.055673599243164,
      "activations/layer17_attention_weight_max": 28.722755432128906,
      "activations/layer17_attention_weight_min": -26.27802848815918,
      "activations/layer18_attention_weight_max": 31.8662166595459,
      "activations/layer18_attention_weight_min": -23.6749210357666,
      "activations/layer19_attention_weight_max": 34.53868103027344,
      "activations/layer19_attention_weight_min": -30.06684684753418,
      "activations/layer1_attention_weight_max": 15.890689849853516,
      "activations/layer1_attention_weight_min": -14.43181324005127,
      "activations/layer20_attention_weight_max": 30.522680282592773,
      "activations/layer20_attention_weight_min": -23.86542320251465,
      "activations/layer21_attention_weight_max": 28.961406707763672,
      "activations/layer21_attention_weight_min": -24.1019287109375,
      "activations/layer22_attention_weight_max": 39.67943572998047,
      "activations/layer22_attention_weight_min": -28.51626205444336,
      "activations/layer23_attention_weight_max": 29.912097930908203,
      "activations/layer23_attention_weight_min": -24.00996971130371,
      "activations/layer2_attention_weight_max": 31.20046043395996,
      "activations/layer2_attention_weight_min": -32.23908996582031,
      "activations/layer3_attention_weight_max": 96.47573852539062,
      "activations/layer3_attention_weight_min": -95.85675048828125,
      "activations/layer4_attention_weight_max": 110.5063705444336,
      "activations/layer4_attention_weight_min": -111.07098388671875,
      "activations/layer5_attention_weight_max": 52.215980529785156,
      "activations/layer5_attention_weight_min": -63.18856430053711,
      "activations/layer6_attention_weight_max": 47.31739807128906,
      "activations/layer6_attention_weight_min": -49.36448287963867,
      "activations/layer7_attention_weight_max": 91.27554321289062,
      "activations/layer7_attention_weight_min": -100.04254913330078,
      "activations/layer8_attention_weight_max": 41.23079299926758,
      "activations/layer8_attention_weight_min": -44.05002212524414,
      "activations/layer9_attention_weight_max": 32.44234848022461,
      "activations/layer9_attention_weight_min": -34.0650634765625,
      "epoch": 15.44,
      "learning_rate": 5.09530303030303e-05,
      "loss": 2.7527,
      "step": 265700
    },
    {
      "activations/layer0_attention_weight_max": 15.87834358215332,
      "activations/layer0_attention_weight_min": -12.411455154418945,
      "activations/layer10_attention_weight_max": 34.794212341308594,
      "activations/layer10_attention_weight_min": -34.45664978027344,
      "activations/layer11_attention_weight_max": 33.31992721557617,
      "activations/layer11_attention_weight_min": -34.95954132080078,
      "activations/layer12_attention_weight_max": 28.00278663635254,
      "activations/layer12_attention_weight_min": -26.88665199279785,
      "activations/layer13_attention_weight_max": 37.90943908691406,
      "activations/layer13_attention_weight_min": -33.87715148925781,
      "activations/layer14_attention_weight_max": 45.371238708496094,
      "activations/layer14_attention_weight_min": -36.61980438232422,
      "activations/layer15_attention_weight_max": 37.089115142822266,
      "activations/layer15_attention_weight_min": -31.89664077758789,
      "activations/layer16_attention_weight_max": 30.725053787231445,
      "activations/layer16_attention_weight_min": -28.215726852416992,
      "activations/layer17_attention_weight_max": 30.873056411743164,
      "activations/layer17_attention_weight_min": -26.312328338623047,
      "activations/layer18_attention_weight_max": 32.47615432739258,
      "activations/layer18_attention_weight_min": -25.448532104492188,
      "activations/layer19_attention_weight_max": 32.57231521606445,
      "activations/layer19_attention_weight_min": -30.85970115661621,
      "activations/layer1_attention_weight_max": 16.11911964416504,
      "activations/layer1_attention_weight_min": -14.797480583190918,
      "activations/layer20_attention_weight_max": 29.97176170349121,
      "activations/layer20_attention_weight_min": -24.189878463745117,
      "activations/layer21_attention_weight_max": 30.883718490600586,
      "activations/layer21_attention_weight_min": -24.826963424682617,
      "activations/layer22_attention_weight_max": 43.390342712402344,
      "activations/layer22_attention_weight_min": -30.933523178100586,
      "activations/layer23_attention_weight_max": 34.062171936035156,
      "activations/layer23_attention_weight_min": -25.0295467376709,
      "activations/layer2_attention_weight_max": 33.22802734375,
      "activations/layer2_attention_weight_min": -33.81161117553711,
      "activations/layer3_attention_weight_max": 95.66281127929688,
      "activations/layer3_attention_weight_min": -101.13512420654297,
      "activations/layer4_attention_weight_max": 104.6263427734375,
      "activations/layer4_attention_weight_min": -109.8376693725586,
      "activations/layer5_attention_weight_max": 50.75617599487305,
      "activations/layer5_attention_weight_min": -60.04170227050781,
      "activations/layer6_attention_weight_max": 45.8588981628418,
      "activations/layer6_attention_weight_min": -49.23479080200195,
      "activations/layer7_attention_weight_max": 94.65592956542969,
      "activations/layer7_attention_weight_min": -98.61494445800781,
      "activations/layer8_attention_weight_max": 43.182777404785156,
      "activations/layer8_attention_weight_min": -43.9947395324707,
      "activations/layer9_attention_weight_max": 33.24378967285156,
      "activations/layer9_attention_weight_min": -35.61899185180664,
      "epoch": 15.44,
      "learning_rate": 5.093409090909091e-05,
      "loss": 2.7414,
      "step": 265750
    },
    {
      "activations/layer0_attention_weight_max": 16.239986419677734,
      "activations/layer0_attention_weight_min": -11.588606834411621,
      "activations/layer10_attention_weight_max": 32.253753662109375,
      "activations/layer10_attention_weight_min": -33.76814651489258,
      "activations/layer11_attention_weight_max": 33.32900619506836,
      "activations/layer11_attention_weight_min": -33.44981384277344,
      "activations/layer12_attention_weight_max": 27.72745704650879,
      "activations/layer12_attention_weight_min": -26.06647300720215,
      "activations/layer13_attention_weight_max": 39.617366790771484,
      "activations/layer13_attention_weight_min": -35.82106018066406,
      "activations/layer14_attention_weight_max": 41.914337158203125,
      "activations/layer14_attention_weight_min": -38.739097595214844,
      "activations/layer15_attention_weight_max": 37.018463134765625,
      "activations/layer15_attention_weight_min": -35.24580383300781,
      "activations/layer16_attention_weight_max": 29.966022491455078,
      "activations/layer16_attention_weight_min": -28.799718856811523,
      "activations/layer17_attention_weight_max": 31.976444244384766,
      "activations/layer17_attention_weight_min": -25.983482360839844,
      "activations/layer18_attention_weight_max": 30.47768783569336,
      "activations/layer18_attention_weight_min": -25.770278930664062,
      "activations/layer19_attention_weight_max": 32.066043853759766,
      "activations/layer19_attention_weight_min": -32.19268798828125,
      "activations/layer1_attention_weight_max": 17.375965118408203,
      "activations/layer1_attention_weight_min": -15.341073036193848,
      "activations/layer20_attention_weight_max": 29.469947814941406,
      "activations/layer20_attention_weight_min": -26.487369537353516,
      "activations/layer21_attention_weight_max": 30.070329666137695,
      "activations/layer21_attention_weight_min": -26.304733276367188,
      "activations/layer22_attention_weight_max": 42.80826187133789,
      "activations/layer22_attention_weight_min": -32.3828010559082,
      "activations/layer23_attention_weight_max": 32.52528381347656,
      "activations/layer23_attention_weight_min": -26.489004135131836,
      "activations/layer2_attention_weight_max": 33.372314453125,
      "activations/layer2_attention_weight_min": -32.566734313964844,
      "activations/layer3_attention_weight_max": 96.81224822998047,
      "activations/layer3_attention_weight_min": -94.9101333618164,
      "activations/layer4_attention_weight_max": 106.33443450927734,
      "activations/layer4_attention_weight_min": -111.5142822265625,
      "activations/layer5_attention_weight_max": 50.999603271484375,
      "activations/layer5_attention_weight_min": -62.23084259033203,
      "activations/layer6_attention_weight_max": 42.80030059814453,
      "activations/layer6_attention_weight_min": -45.74105453491211,
      "activations/layer7_attention_weight_max": 92.42546081542969,
      "activations/layer7_attention_weight_min": -91.76873779296875,
      "activations/layer8_attention_weight_max": 40.44286346435547,
      "activations/layer8_attention_weight_min": -44.25876998901367,
      "activations/layer9_attention_weight_max": 32.314720153808594,
      "activations/layer9_attention_weight_min": -34.02775192260742,
      "epoch": 15.44,
      "learning_rate": 5.091515151515151e-05,
      "loss": 2.7363,
      "step": 265800
    },
    {
      "activations/layer0_attention_weight_max": 15.884706497192383,
      "activations/layer0_attention_weight_min": -13.073124885559082,
      "activations/layer10_attention_weight_max": 36.55405807495117,
      "activations/layer10_attention_weight_min": -36.67792892456055,
      "activations/layer11_attention_weight_max": 37.12318801879883,
      "activations/layer11_attention_weight_min": -34.691688537597656,
      "activations/layer12_attention_weight_max": 25.11228370666504,
      "activations/layer12_attention_weight_min": -26.844356536865234,
      "activations/layer13_attention_weight_max": 37.48903274536133,
      "activations/layer13_attention_weight_min": -36.13728713989258,
      "activations/layer14_attention_weight_max": 39.78093338012695,
      "activations/layer14_attention_weight_min": -38.44182205200195,
      "activations/layer15_attention_weight_max": 35.82477569580078,
      "activations/layer15_attention_weight_min": -34.07791519165039,
      "activations/layer16_attention_weight_max": 30.642183303833008,
      "activations/layer16_attention_weight_min": -29.206769943237305,
      "activations/layer17_attention_weight_max": 29.97269058227539,
      "activations/layer17_attention_weight_min": -27.85732650756836,
      "activations/layer18_attention_weight_max": 30.3288631439209,
      "activations/layer18_attention_weight_min": -25.70356559753418,
      "activations/layer19_attention_weight_max": 34.54134750366211,
      "activations/layer19_attention_weight_min": -32.60056686401367,
      "activations/layer1_attention_weight_max": 16.628686904907227,
      "activations/layer1_attention_weight_min": -14.674174308776855,
      "activations/layer20_attention_weight_max": 26.90350341796875,
      "activations/layer20_attention_weight_min": -24.45741844177246,
      "activations/layer21_attention_weight_max": 30.4152889251709,
      "activations/layer21_attention_weight_min": -24.062028884887695,
      "activations/layer22_attention_weight_max": 39.91050720214844,
      "activations/layer22_attention_weight_min": -30.370590209960938,
      "activations/layer23_attention_weight_max": 29.983741760253906,
      "activations/layer23_attention_weight_min": -25.65857696533203,
      "activations/layer2_attention_weight_max": 32.08130645751953,
      "activations/layer2_attention_weight_min": -32.004066467285156,
      "activations/layer3_attention_weight_max": 94.12834930419922,
      "activations/layer3_attention_weight_min": -98.12548828125,
      "activations/layer4_attention_weight_max": 111.35396575927734,
      "activations/layer4_attention_weight_min": -109.1117935180664,
      "activations/layer5_attention_weight_max": 51.482444763183594,
      "activations/layer5_attention_weight_min": -71.84523010253906,
      "activations/layer6_attention_weight_max": 44.71598434448242,
      "activations/layer6_attention_weight_min": -49.16551971435547,
      "activations/layer7_attention_weight_max": 96.61434936523438,
      "activations/layer7_attention_weight_min": -99.96377563476562,
      "activations/layer8_attention_weight_max": 44.1645393371582,
      "activations/layer8_attention_weight_min": -45.75478744506836,
      "activations/layer9_attention_weight_max": 34.07025909423828,
      "activations/layer9_attention_weight_min": -36.20967483520508,
      "epoch": 15.45,
      "learning_rate": 5.089621212121211e-05,
      "loss": 2.7478,
      "step": 265850
    },
    {
      "activations/layer0_attention_weight_max": 16.085542678833008,
      "activations/layer0_attention_weight_min": -12.00023365020752,
      "activations/layer10_attention_weight_max": 37.331119537353516,
      "activations/layer10_attention_weight_min": -35.33686828613281,
      "activations/layer11_attention_weight_max": 37.947059631347656,
      "activations/layer11_attention_weight_min": -35.072669982910156,
      "activations/layer12_attention_weight_max": 26.840604782104492,
      "activations/layer12_attention_weight_min": -26.84026336669922,
      "activations/layer13_attention_weight_max": 41.50455856323242,
      "activations/layer13_attention_weight_min": -35.93696975708008,
      "activations/layer14_attention_weight_max": 44.157901763916016,
      "activations/layer14_attention_weight_min": -38.3015022277832,
      "activations/layer15_attention_weight_max": 40.73454666137695,
      "activations/layer15_attention_weight_min": -33.93157958984375,
      "activations/layer16_attention_weight_max": 32.218955993652344,
      "activations/layer16_attention_weight_min": -31.914180755615234,
      "activations/layer17_attention_weight_max": 31.45110511779785,
      "activations/layer17_attention_weight_min": -27.627756118774414,
      "activations/layer18_attention_weight_max": 29.979440689086914,
      "activations/layer18_attention_weight_min": -25.85171127319336,
      "activations/layer19_attention_weight_max": 33.00007629394531,
      "activations/layer19_attention_weight_min": -34.492027282714844,
      "activations/layer1_attention_weight_max": 16.264371871948242,
      "activations/layer1_attention_weight_min": -15.297233581542969,
      "activations/layer20_attention_weight_max": 29.358835220336914,
      "activations/layer20_attention_weight_min": -26.42467498779297,
      "activations/layer21_attention_weight_max": 29.430936813354492,
      "activations/layer21_attention_weight_min": -25.23870086669922,
      "activations/layer22_attention_weight_max": 45.7958869934082,
      "activations/layer22_attention_weight_min": -31.022613525390625,
      "activations/layer23_attention_weight_max": 35.030921936035156,
      "activations/layer23_attention_weight_min": -25.753692626953125,
      "activations/layer2_attention_weight_max": 33.30926513671875,
      "activations/layer2_attention_weight_min": -32.491031646728516,
      "activations/layer3_attention_weight_max": 100.70877838134766,
      "activations/layer3_attention_weight_min": -96.82552337646484,
      "activations/layer4_attention_weight_max": 112.27696990966797,
      "activations/layer4_attention_weight_min": -110.46351623535156,
      "activations/layer5_attention_weight_max": 51.40968322753906,
      "activations/layer5_attention_weight_min": -62.396636962890625,
      "activations/layer6_attention_weight_max": 47.447940826416016,
      "activations/layer6_attention_weight_min": -47.451194763183594,
      "activations/layer7_attention_weight_max": 97.70916748046875,
      "activations/layer7_attention_weight_min": -98.3525161743164,
      "activations/layer8_attention_weight_max": 45.792274475097656,
      "activations/layer8_attention_weight_min": -44.109779357910156,
      "activations/layer9_attention_weight_max": 34.50434494018555,
      "activations/layer9_attention_weight_min": -35.46303176879883,
      "epoch": 15.45,
      "learning_rate": 5.0877272727272726e-05,
      "loss": 2.7427,
      "step": 265900
    },
    {
      "activations/layer0_attention_weight_max": 15.711981773376465,
      "activations/layer0_attention_weight_min": -11.982508659362793,
      "activations/layer10_attention_weight_max": 32.608402252197266,
      "activations/layer10_attention_weight_min": -32.67217254638672,
      "activations/layer11_attention_weight_max": 32.62718200683594,
      "activations/layer11_attention_weight_min": -32.523014068603516,
      "activations/layer12_attention_weight_max": 26.910022735595703,
      "activations/layer12_attention_weight_min": -26.011438369750977,
      "activations/layer13_attention_weight_max": 37.69511795043945,
      "activations/layer13_attention_weight_min": -33.60053634643555,
      "activations/layer14_attention_weight_max": 40.02904510498047,
      "activations/layer14_attention_weight_min": -38.92396545410156,
      "activations/layer15_attention_weight_max": 36.116058349609375,
      "activations/layer15_attention_weight_min": -34.32146072387695,
      "activations/layer16_attention_weight_max": 31.89823341369629,
      "activations/layer16_attention_weight_min": -28.867021560668945,
      "activations/layer17_attention_weight_max": 30.43804359436035,
      "activations/layer17_attention_weight_min": -24.9415225982666,
      "activations/layer18_attention_weight_max": 30.626155853271484,
      "activations/layer18_attention_weight_min": -24.52065658569336,
      "activations/layer19_attention_weight_max": 35.00214767456055,
      "activations/layer19_attention_weight_min": -31.506717681884766,
      "activations/layer1_attention_weight_max": 16.781078338623047,
      "activations/layer1_attention_weight_min": -16.675647735595703,
      "activations/layer20_attention_weight_max": 31.031070709228516,
      "activations/layer20_attention_weight_min": -25.11345672607422,
      "activations/layer21_attention_weight_max": 31.372032165527344,
      "activations/layer21_attention_weight_min": -24.514028549194336,
      "activations/layer22_attention_weight_max": 41.962425231933594,
      "activations/layer22_attention_weight_min": -31.119930267333984,
      "activations/layer23_attention_weight_max": 31.40411376953125,
      "activations/layer23_attention_weight_min": -27.309062957763672,
      "activations/layer2_attention_weight_max": 36.307456970214844,
      "activations/layer2_attention_weight_min": -33.2230224609375,
      "activations/layer3_attention_weight_max": 100.05697631835938,
      "activations/layer3_attention_weight_min": -97.9605484008789,
      "activations/layer4_attention_weight_max": 104.00064849853516,
      "activations/layer4_attention_weight_min": -105.77727508544922,
      "activations/layer5_attention_weight_max": 51.75368881225586,
      "activations/layer5_attention_weight_min": -59.163448333740234,
      "activations/layer6_attention_weight_max": 42.0485725402832,
      "activations/layer6_attention_weight_min": -45.14504623413086,
      "activations/layer7_attention_weight_max": 89.14375305175781,
      "activations/layer7_attention_weight_min": -88.94851684570312,
      "activations/layer8_attention_weight_max": 40.46900177001953,
      "activations/layer8_attention_weight_min": -41.090789794921875,
      "activations/layer9_attention_weight_max": 31.332624435424805,
      "activations/layer9_attention_weight_min": -33.24239730834961,
      "epoch": 15.45,
      "learning_rate": 5.085833333333333e-05,
      "loss": 2.7514,
      "step": 265950
    },
    {
      "activations/layer0_attention_weight_max": 16.15123176574707,
      "activations/layer0_attention_weight_min": -13.136163711547852,
      "activations/layer10_attention_weight_max": 33.2303466796875,
      "activations/layer10_attention_weight_min": -35.07157516479492,
      "activations/layer11_attention_weight_max": 33.494178771972656,
      "activations/layer11_attention_weight_min": -34.06240463256836,
      "activations/layer12_attention_weight_max": 26.44829750061035,
      "activations/layer12_attention_weight_min": -26.786951065063477,
      "activations/layer13_attention_weight_max": 41.68028259277344,
      "activations/layer13_attention_weight_min": -36.5747184753418,
      "activations/layer14_attention_weight_max": 44.41444778442383,
      "activations/layer14_attention_weight_min": -38.260562896728516,
      "activations/layer15_attention_weight_max": 39.29860305786133,
      "activations/layer15_attention_weight_min": -34.56615447998047,
      "activations/layer16_attention_weight_max": 30.522472381591797,
      "activations/layer16_attention_weight_min": -26.97894859313965,
      "activations/layer17_attention_weight_max": 30.903284072875977,
      "activations/layer17_attention_weight_min": -26.201440811157227,
      "activations/layer18_attention_weight_max": 36.647640228271484,
      "activations/layer18_attention_weight_min": -25.03142738342285,
      "activations/layer19_attention_weight_max": 38.0186882019043,
      "activations/layer19_attention_weight_min": -33.01533508300781,
      "activations/layer1_attention_weight_max": 16.471744537353516,
      "activations/layer1_attention_weight_min": -15.43904972076416,
      "activations/layer20_attention_weight_max": 30.633806228637695,
      "activations/layer20_attention_weight_min": -26.66842269897461,
      "activations/layer21_attention_weight_max": 29.651363372802734,
      "activations/layer21_attention_weight_min": -25.240636825561523,
      "activations/layer22_attention_weight_max": 44.66278839111328,
      "activations/layer22_attention_weight_min": -33.1485481262207,
      "activations/layer23_attention_weight_max": 34.605308532714844,
      "activations/layer23_attention_weight_min": -24.916351318359375,
      "activations/layer2_attention_weight_max": 33.431358337402344,
      "activations/layer2_attention_weight_min": -35.073524475097656,
      "activations/layer3_attention_weight_max": 98.96321868896484,
      "activations/layer3_attention_weight_min": -98.41719818115234,
      "activations/layer4_attention_weight_max": 108.91255187988281,
      "activations/layer4_attention_weight_min": -110.3242416381836,
      "activations/layer5_attention_weight_max": 51.125667572021484,
      "activations/layer5_attention_weight_min": -65.57149505615234,
      "activations/layer6_attention_weight_max": 43.99772262573242,
      "activations/layer6_attention_weight_min": -48.218814849853516,
      "activations/layer7_attention_weight_max": 94.66573333740234,
      "activations/layer7_attention_weight_min": -96.46519470214844,
      "activations/layer8_attention_weight_max": 42.99946212768555,
      "activations/layer8_attention_weight_min": -43.8421516418457,
      "activations/layer9_attention_weight_max": 32.000797271728516,
      "activations/layer9_attention_weight_min": -36.70966720581055,
      "epoch": 15.46,
      "learning_rate": 5.0839393939393935e-05,
      "loss": 2.7199,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.7729,
      "eval_samples_per_second": 489.459,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.7729,
      "eval_openwebtext_samples_per_second": 489.459,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9617,
      "eval_wikitext_samples_per_second": 232.446,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_lambada_loss": 2.423828125,
      "eval_lambada_ppl": 11.288992376773315,
      "eval_lambada_runtime": 9.5423,
      "eval_lambada_samples_per_second": 510.255,
      "step": 266000
    },
    {
      "activations/layer0_attention_weight_max": 15.131929397583008,
      "activations/layer0_attention_weight_min": -12.138029098510742,
      "activations/layer10_attention_weight_max": 29.839109420776367,
      "activations/layer10_attention_weight_min": -32.78533172607422,
      "activations/layer11_attention_weight_max": 29.946203231811523,
      "activations/layer11_attention_weight_min": -30.58066177368164,
      "activations/layer12_attention_weight_max": 25.43236541748047,
      "activations/layer12_attention_weight_min": -25.53172492980957,
      "activations/layer13_attention_weight_max": 38.64830780029297,
      "activations/layer13_attention_weight_min": -35.791404724121094,
      "activations/layer14_attention_weight_max": 42.00862121582031,
      "activations/layer14_attention_weight_min": -37.972137451171875,
      "activations/layer15_attention_weight_max": 37.850860595703125,
      "activations/layer15_attention_weight_min": -34.59318161010742,
      "activations/layer16_attention_weight_max": 31.71562385559082,
      "activations/layer16_attention_weight_min": -26.566781997680664,
      "activations/layer17_attention_weight_max": 33.06132888793945,
      "activations/layer17_attention_weight_min": -25.25017738342285,
      "activations/layer18_attention_weight_max": 31.084434509277344,
      "activations/layer18_attention_weight_min": -23.92850685119629,
      "activations/layer19_attention_weight_max": 39.65523147583008,
      "activations/layer19_attention_weight_min": -31.5947322845459,
      "activations/layer1_attention_weight_max": 16.008024215698242,
      "activations/layer1_attention_weight_min": -15.317117691040039,
      "activations/layer20_attention_weight_max": 34.201622009277344,
      "activations/layer20_attention_weight_min": -24.140966415405273,
      "activations/layer21_attention_weight_max": 34.74957275390625,
      "activations/layer21_attention_weight_min": -24.206369400024414,
      "activations/layer22_attention_weight_max": 44.0883674621582,
      "activations/layer22_attention_weight_min": -29.650089263916016,
      "activations/layer23_attention_weight_max": 32.82624816894531,
      "activations/layer23_attention_weight_min": -24.06716537475586,
      "activations/layer2_attention_weight_max": 31.06588363647461,
      "activations/layer2_attention_weight_min": -32.218326568603516,
      "activations/layer3_attention_weight_max": 90.8254623413086,
      "activations/layer3_attention_weight_min": -93.25568389892578,
      "activations/layer4_attention_weight_max": 101.97795104980469,
      "activations/layer4_attention_weight_min": -102.6120376586914,
      "activations/layer5_attention_weight_max": 49.38962936401367,
      "activations/layer5_attention_weight_min": -57.57725524902344,
      "activations/layer6_attention_weight_max": 40.52309036254883,
      "activations/layer6_attention_weight_min": -42.54261016845703,
      "activations/layer7_attention_weight_max": 86.06206512451172,
      "activations/layer7_attention_weight_min": -88.29853057861328,
      "activations/layer8_attention_weight_max": 38.832176208496094,
      "activations/layer8_attention_weight_min": -41.07319641113281,
      "activations/layer9_attention_weight_max": 28.665607452392578,
      "activations/layer9_attention_weight_min": -31.28006362915039,
      "epoch": 15.46,
      "learning_rate": 5.0820454545454544e-05,
      "loss": 2.7465,
      "step": 266050
    },
    {
      "activations/layer0_attention_weight_max": 15.819939613342285,
      "activations/layer0_attention_weight_min": -11.792476654052734,
      "activations/layer10_attention_weight_max": 31.288358688354492,
      "activations/layer10_attention_weight_min": -32.98606491088867,
      "activations/layer11_attention_weight_max": 32.505584716796875,
      "activations/layer11_attention_weight_min": -32.654083251953125,
      "activations/layer12_attention_weight_max": 25.527099609375,
      "activations/layer12_attention_weight_min": -26.576107025146484,
      "activations/layer13_attention_weight_max": 38.14352035522461,
      "activations/layer13_attention_weight_min": -34.63861846923828,
      "activations/layer14_attention_weight_max": 42.03042984008789,
      "activations/layer14_attention_weight_min": -35.626644134521484,
      "activations/layer15_attention_weight_max": 37.52936553955078,
      "activations/layer15_attention_weight_min": -34.12935256958008,
      "activations/layer16_attention_weight_max": 28.40595054626465,
      "activations/layer16_attention_weight_min": -28.968576431274414,
      "activations/layer17_attention_weight_max": 28.802907943725586,
      "activations/layer17_attention_weight_min": -25.10724449157715,
      "activations/layer18_attention_weight_max": 30.675973892211914,
      "activations/layer18_attention_weight_min": -23.230960845947266,
      "activations/layer19_attention_weight_max": 35.23807907104492,
      "activations/layer19_attention_weight_min": -30.148487091064453,
      "activations/layer1_attention_weight_max": 16.47184181213379,
      "activations/layer1_attention_weight_min": -15.544305801391602,
      "activations/layer20_attention_weight_max": 28.258859634399414,
      "activations/layer20_attention_weight_min": -25.2880859375,
      "activations/layer21_attention_weight_max": 28.612152099609375,
      "activations/layer21_attention_weight_min": -23.300457000732422,
      "activations/layer22_attention_weight_max": 45.08689498901367,
      "activations/layer22_attention_weight_min": -29.39533233642578,
      "activations/layer23_attention_weight_max": 32.831504821777344,
      "activations/layer23_attention_weight_min": -22.809093475341797,
      "activations/layer2_attention_weight_max": 32.2701416015625,
      "activations/layer2_attention_weight_min": -30.22665786743164,
      "activations/layer3_attention_weight_max": 94.15318298339844,
      "activations/layer3_attention_weight_min": -93.84137725830078,
      "activations/layer4_attention_weight_max": 105.3205795288086,
      "activations/layer4_attention_weight_min": -105.65815734863281,
      "activations/layer5_attention_weight_max": 50.66764831542969,
      "activations/layer5_attention_weight_min": -63.51064682006836,
      "activations/layer6_attention_weight_max": 44.87101364135742,
      "activations/layer6_attention_weight_min": -43.31752014160156,
      "activations/layer7_attention_weight_max": 102.47064971923828,
      "activations/layer7_attention_weight_min": -87.56840515136719,
      "activations/layer8_attention_weight_max": 40.54777908325195,
      "activations/layer8_attention_weight_min": -41.21122360229492,
      "activations/layer9_attention_weight_max": 31.94266700744629,
      "activations/layer9_attention_weight_min": -33.59466552734375,
      "epoch": 15.46,
      "learning_rate": 5.0801515151515145e-05,
      "loss": 2.7416,
      "step": 266100
    },
    {
      "activations/layer0_attention_weight_max": 16.293968200683594,
      "activations/layer0_attention_weight_min": -13.98907470703125,
      "activations/layer10_attention_weight_max": 32.17005157470703,
      "activations/layer10_attention_weight_min": -34.574501037597656,
      "activations/layer11_attention_weight_max": 34.933956146240234,
      "activations/layer11_attention_weight_min": -32.333778381347656,
      "activations/layer12_attention_weight_max": 26.71967124938965,
      "activations/layer12_attention_weight_min": -26.087074279785156,
      "activations/layer13_attention_weight_max": 39.810089111328125,
      "activations/layer13_attention_weight_min": -36.7192268371582,
      "activations/layer14_attention_weight_max": 44.805877685546875,
      "activations/layer14_attention_weight_min": -41.57949447631836,
      "activations/layer15_attention_weight_max": 38.04417037963867,
      "activations/layer15_attention_weight_min": -36.50251770019531,
      "activations/layer16_attention_weight_max": 32.251285552978516,
      "activations/layer16_attention_weight_min": -27.93913459777832,
      "activations/layer17_attention_weight_max": 30.175920486450195,
      "activations/layer17_attention_weight_min": -26.52324867248535,
      "activations/layer18_attention_weight_max": 34.12852096557617,
      "activations/layer18_attention_weight_min": -26.194992065429688,
      "activations/layer19_attention_weight_max": 32.834049224853516,
      "activations/layer19_attention_weight_min": -31.51325035095215,
      "activations/layer1_attention_weight_max": 17.37372398376465,
      "activations/layer1_attention_weight_min": -18.75556755065918,
      "activations/layer20_attention_weight_max": 27.656509399414062,
      "activations/layer20_attention_weight_min": -25.664583206176758,
      "activations/layer21_attention_weight_max": 28.374778747558594,
      "activations/layer21_attention_weight_min": -24.831167221069336,
      "activations/layer22_attention_weight_max": 44.42646789550781,
      "activations/layer22_attention_weight_min": -30.415531158447266,
      "activations/layer23_attention_weight_max": 31.05337142944336,
      "activations/layer23_attention_weight_min": -24.60413360595703,
      "activations/layer2_attention_weight_max": 32.1707649230957,
      "activations/layer2_attention_weight_min": -32.365936279296875,
      "activations/layer3_attention_weight_max": 91.22701263427734,
      "activations/layer3_attention_weight_min": -92.80796813964844,
      "activations/layer4_attention_weight_max": 103.33385467529297,
      "activations/layer4_attention_weight_min": -104.28700256347656,
      "activations/layer5_attention_weight_max": 53.925933837890625,
      "activations/layer5_attention_weight_min": -60.0791130065918,
      "activations/layer6_attention_weight_max": 43.57546615600586,
      "activations/layer6_attention_weight_min": -45.28970718383789,
      "activations/layer7_attention_weight_max": 92.84465789794922,
      "activations/layer7_attention_weight_min": -91.74769592285156,
      "activations/layer8_attention_weight_max": 41.77583694458008,
      "activations/layer8_attention_weight_min": -42.2785530090332,
      "activations/layer9_attention_weight_max": 31.544870376586914,
      "activations/layer9_attention_weight_min": -32.9337272644043,
      "epoch": 15.46,
      "learning_rate": 5.0782575757575754e-05,
      "loss": 2.7434,
      "step": 266150
    },
    {
      "activations/layer0_attention_weight_max": 15.53079605102539,
      "activations/layer0_attention_weight_min": -12.15533447265625,
      "activations/layer10_attention_weight_max": 33.315608978271484,
      "activations/layer10_attention_weight_min": -33.81379699707031,
      "activations/layer11_attention_weight_max": 33.64033889770508,
      "activations/layer11_attention_weight_min": -34.6069450378418,
      "activations/layer12_attention_weight_max": 26.393741607666016,
      "activations/layer12_attention_weight_min": -29.292219161987305,
      "activations/layer13_attention_weight_max": 39.738433837890625,
      "activations/layer13_attention_weight_min": -39.61516571044922,
      "activations/layer14_attention_weight_max": 45.140750885009766,
      "activations/layer14_attention_weight_min": -40.597286224365234,
      "activations/layer15_attention_weight_max": 37.855567932128906,
      "activations/layer15_attention_weight_min": -35.05760955810547,
      "activations/layer16_attention_weight_max": 29.4005069732666,
      "activations/layer16_attention_weight_min": -28.125471115112305,
      "activations/layer17_attention_weight_max": 29.105709075927734,
      "activations/layer17_attention_weight_min": -25.6929931640625,
      "activations/layer18_attention_weight_max": 30.3774356842041,
      "activations/layer18_attention_weight_min": -24.19573974609375,
      "activations/layer19_attention_weight_max": 34.41001510620117,
      "activations/layer19_attention_weight_min": -31.695302963256836,
      "activations/layer1_attention_weight_max": 15.919893264770508,
      "activations/layer1_attention_weight_min": -13.701957702636719,
      "activations/layer20_attention_weight_max": 31.46251106262207,
      "activations/layer20_attention_weight_min": -25.872922897338867,
      "activations/layer21_attention_weight_max": 29.962425231933594,
      "activations/layer21_attention_weight_min": -24.381755828857422,
      "activations/layer22_attention_weight_max": 40.27070617675781,
      "activations/layer22_attention_weight_min": -30.237878799438477,
      "activations/layer23_attention_weight_max": 36.344600677490234,
      "activations/layer23_attention_weight_min": -24.540128707885742,
      "activations/layer2_attention_weight_max": 31.151752471923828,
      "activations/layer2_attention_weight_min": -32.24966812133789,
      "activations/layer3_attention_weight_max": 94.07610321044922,
      "activations/layer3_attention_weight_min": -95.15975952148438,
      "activations/layer4_attention_weight_max": 103.46453857421875,
      "activations/layer4_attention_weight_min": -106.86083221435547,
      "activations/layer5_attention_weight_max": 51.46571350097656,
      "activations/layer5_attention_weight_min": -64.28181457519531,
      "activations/layer6_attention_weight_max": 43.01962661743164,
      "activations/layer6_attention_weight_min": -45.53446960449219,
      "activations/layer7_attention_weight_max": 93.12606048583984,
      "activations/layer7_attention_weight_min": -90.73222351074219,
      "activations/layer8_attention_weight_max": 40.31231689453125,
      "activations/layer8_attention_weight_min": -43.74873733520508,
      "activations/layer9_attention_weight_max": 31.610807418823242,
      "activations/layer9_attention_weight_min": -34.451541900634766,
      "epoch": 15.47,
      "learning_rate": 5.0763636363636355e-05,
      "loss": 2.7326,
      "step": 266200
    },
    {
      "activations/layer0_attention_weight_max": 15.279518127441406,
      "activations/layer0_attention_weight_min": -12.50430965423584,
      "activations/layer10_attention_weight_max": 31.90023422241211,
      "activations/layer10_attention_weight_min": -30.619863510131836,
      "activations/layer11_attention_weight_max": 32.7678337097168,
      "activations/layer11_attention_weight_min": -30.058135986328125,
      "activations/layer12_attention_weight_max": 26.11620330810547,
      "activations/layer12_attention_weight_min": -26.3343563079834,
      "activations/layer13_attention_weight_max": 38.207237243652344,
      "activations/layer13_attention_weight_min": -34.39628601074219,
      "activations/layer14_attention_weight_max": 40.82021713256836,
      "activations/layer14_attention_weight_min": -38.77177429199219,
      "activations/layer15_attention_weight_max": 38.310184478759766,
      "activations/layer15_attention_weight_min": -35.371742248535156,
      "activations/layer16_attention_weight_max": 33.508750915527344,
      "activations/layer16_attention_weight_min": -28.2277774810791,
      "activations/layer17_attention_weight_max": 30.980634689331055,
      "activations/layer17_attention_weight_min": -25.355514526367188,
      "activations/layer18_attention_weight_max": 34.448883056640625,
      "activations/layer18_attention_weight_min": -24.383630752563477,
      "activations/layer19_attention_weight_max": 34.82009506225586,
      "activations/layer19_attention_weight_min": -31.112945556640625,
      "activations/layer1_attention_weight_max": 16.09088134765625,
      "activations/layer1_attention_weight_min": -14.116345405578613,
      "activations/layer20_attention_weight_max": 30.154638290405273,
      "activations/layer20_attention_weight_min": -24.586973190307617,
      "activations/layer21_attention_weight_max": 29.278545379638672,
      "activations/layer21_attention_weight_min": -25.54717445373535,
      "activations/layer22_attention_weight_max": 42.58210754394531,
      "activations/layer22_attention_weight_min": -30.52968406677246,
      "activations/layer23_attention_weight_max": 34.234230041503906,
      "activations/layer23_attention_weight_min": -27.965728759765625,
      "activations/layer2_attention_weight_max": 31.8869686126709,
      "activations/layer2_attention_weight_min": -31.614673614501953,
      "activations/layer3_attention_weight_max": 92.1723861694336,
      "activations/layer3_attention_weight_min": -93.7697982788086,
      "activations/layer4_attention_weight_max": 105.44197845458984,
      "activations/layer4_attention_weight_min": -104.67989349365234,
      "activations/layer5_attention_weight_max": 51.69240951538086,
      "activations/layer5_attention_weight_min": -61.724300384521484,
      "activations/layer6_attention_weight_max": 43.23996353149414,
      "activations/layer6_attention_weight_min": -46.00619125366211,
      "activations/layer7_attention_weight_max": 92.64611053466797,
      "activations/layer7_attention_weight_min": -92.15010833740234,
      "activations/layer8_attention_weight_max": 40.16360855102539,
      "activations/layer8_attention_weight_min": -42.5250129699707,
      "activations/layer9_attention_weight_max": 31.964553833007812,
      "activations/layer9_attention_weight_min": -32.62531661987305,
      "epoch": 15.47,
      "learning_rate": 5.074469696969697e-05,
      "loss": 2.7389,
      "step": 266250
    },
    {
      "activations/layer0_attention_weight_max": 16.368257522583008,
      "activations/layer0_attention_weight_min": -12.557509422302246,
      "activations/layer10_attention_weight_max": 30.690494537353516,
      "activations/layer10_attention_weight_min": -32.246742248535156,
      "activations/layer11_attention_weight_max": 32.04785919189453,
      "activations/layer11_attention_weight_min": -32.32564926147461,
      "activations/layer12_attention_weight_max": 23.411603927612305,
      "activations/layer12_attention_weight_min": -29.330793380737305,
      "activations/layer13_attention_weight_max": 35.2120475769043,
      "activations/layer13_attention_weight_min": -35.06929016113281,
      "activations/layer14_attention_weight_max": 39.929649353027344,
      "activations/layer14_attention_weight_min": -39.027896881103516,
      "activations/layer15_attention_weight_max": 33.5545768737793,
      "activations/layer15_attention_weight_min": -34.17372131347656,
      "activations/layer16_attention_weight_max": 30.152997970581055,
      "activations/layer16_attention_weight_min": -26.992536544799805,
      "activations/layer17_attention_weight_max": 28.17928123474121,
      "activations/layer17_attention_weight_min": -25.082178115844727,
      "activations/layer18_attention_weight_max": 29.12356185913086,
      "activations/layer18_attention_weight_min": -25.278356552124023,
      "activations/layer19_attention_weight_max": 32.38077926635742,
      "activations/layer19_attention_weight_min": -32.33652877807617,
      "activations/layer1_attention_weight_max": 16.735322952270508,
      "activations/layer1_attention_weight_min": -15.437799453735352,
      "activations/layer20_attention_weight_max": 28.19061279296875,
      "activations/layer20_attention_weight_min": -26.529422760009766,
      "activations/layer21_attention_weight_max": 27.206863403320312,
      "activations/layer21_attention_weight_min": -27.416954040527344,
      "activations/layer22_attention_weight_max": 37.42689514160156,
      "activations/layer22_attention_weight_min": -31.78803825378418,
      "activations/layer23_attention_weight_max": 29.998807907104492,
      "activations/layer23_attention_weight_min": -24.261503219604492,
      "activations/layer2_attention_weight_max": 35.758602142333984,
      "activations/layer2_attention_weight_min": -32.98115158081055,
      "activations/layer3_attention_weight_max": 93.53545379638672,
      "activations/layer3_attention_weight_min": -96.50601196289062,
      "activations/layer4_attention_weight_max": 101.63338470458984,
      "activations/layer4_attention_weight_min": -108.31070709228516,
      "activations/layer5_attention_weight_max": 48.98548889160156,
      "activations/layer5_attention_weight_min": -62.982330322265625,
      "activations/layer6_attention_weight_max": 41.25874328613281,
      "activations/layer6_attention_weight_min": -45.163028717041016,
      "activations/layer7_attention_weight_max": 91.50494384765625,
      "activations/layer7_attention_weight_min": -91.30818176269531,
      "activations/layer8_attention_weight_max": 38.38960266113281,
      "activations/layer8_attention_weight_min": -41.85068130493164,
      "activations/layer9_attention_weight_max": 30.531326293945312,
      "activations/layer9_attention_weight_min": -31.510160446166992,
      "epoch": 15.47,
      "learning_rate": 5.072575757575757e-05,
      "loss": 2.7441,
      "step": 266300
    },
    {
      "activations/layer0_attention_weight_max": 16.21050453186035,
      "activations/layer0_attention_weight_min": -12.736303329467773,
      "activations/layer10_attention_weight_max": 34.539581298828125,
      "activations/layer10_attention_weight_min": -33.758934020996094,
      "activations/layer11_attention_weight_max": 33.41621017456055,
      "activations/layer11_attention_weight_min": -32.41573715209961,
      "activations/layer12_attention_weight_max": 26.067543029785156,
      "activations/layer12_attention_weight_min": -24.848405838012695,
      "activations/layer13_attention_weight_max": 39.33525085449219,
      "activations/layer13_attention_weight_min": -34.721614837646484,
      "activations/layer14_attention_weight_max": 45.84437942504883,
      "activations/layer14_attention_weight_min": -35.7082633972168,
      "activations/layer15_attention_weight_max": 41.410400390625,
      "activations/layer15_attention_weight_min": -33.08210754394531,
      "activations/layer16_attention_weight_max": 29.45652961730957,
      "activations/layer16_attention_weight_min": -27.398029327392578,
      "activations/layer17_attention_weight_max": 33.550514221191406,
      "activations/layer17_attention_weight_min": -25.030799865722656,
      "activations/layer18_attention_weight_max": 31.084007263183594,
      "activations/layer18_attention_weight_min": -25.309247970581055,
      "activations/layer19_attention_weight_max": 33.115882873535156,
      "activations/layer19_attention_weight_min": -30.123245239257812,
      "activations/layer1_attention_weight_max": 16.432659149169922,
      "activations/layer1_attention_weight_min": -14.950030326843262,
      "activations/layer20_attention_weight_max": 28.311992645263672,
      "activations/layer20_attention_weight_min": -25.747962951660156,
      "activations/layer21_attention_weight_max": 30.073373794555664,
      "activations/layer21_attention_weight_min": -25.257505416870117,
      "activations/layer22_attention_weight_max": 43.43970489501953,
      "activations/layer22_attention_weight_min": -33.041996002197266,
      "activations/layer23_attention_weight_max": 35.77653884887695,
      "activations/layer23_attention_weight_min": -25.667654037475586,
      "activations/layer2_attention_weight_max": 31.490455627441406,
      "activations/layer2_attention_weight_min": -33.00651931762695,
      "activations/layer3_attention_weight_max": 96.05632781982422,
      "activations/layer3_attention_weight_min": -95.42049407958984,
      "activations/layer4_attention_weight_max": 107.19878387451172,
      "activations/layer4_attention_weight_min": -104.47258758544922,
      "activations/layer5_attention_weight_max": 50.612693786621094,
      "activations/layer5_attention_weight_min": -60.1346435546875,
      "activations/layer6_attention_weight_max": 43.7995719909668,
      "activations/layer6_attention_weight_min": -45.817665100097656,
      "activations/layer7_attention_weight_max": 91.36360168457031,
      "activations/layer7_attention_weight_min": -88.51730346679688,
      "activations/layer8_attention_weight_max": 40.30975341796875,
      "activations/layer8_attention_weight_min": -40.88591384887695,
      "activations/layer9_attention_weight_max": 31.279556274414062,
      "activations/layer9_attention_weight_min": -33.69038009643555,
      "epoch": 15.48,
      "learning_rate": 5.070681818181817e-05,
      "loss": 2.7583,
      "step": 266350
    },
    {
      "activations/layer0_attention_weight_max": 16.561140060424805,
      "activations/layer0_attention_weight_min": -12.484034538269043,
      "activations/layer10_attention_weight_max": 30.644866943359375,
      "activations/layer10_attention_weight_min": -31.581375122070312,
      "activations/layer11_attention_weight_max": 32.161746978759766,
      "activations/layer11_attention_weight_min": -32.09396743774414,
      "activations/layer12_attention_weight_max": 24.57623863220215,
      "activations/layer12_attention_weight_min": -26.320348739624023,
      "activations/layer13_attention_weight_max": 36.697265625,
      "activations/layer13_attention_weight_min": -35.236663818359375,
      "activations/layer14_attention_weight_max": 41.808868408203125,
      "activations/layer14_attention_weight_min": -37.134952545166016,
      "activations/layer15_attention_weight_max": 37.17937469482422,
      "activations/layer15_attention_weight_min": -34.488197326660156,
      "activations/layer16_attention_weight_max": 27.971477508544922,
      "activations/layer16_attention_weight_min": -26.563879013061523,
      "activations/layer17_attention_weight_max": 28.393230438232422,
      "activations/layer17_attention_weight_min": -26.329679489135742,
      "activations/layer18_attention_weight_max": 33.6694221496582,
      "activations/layer18_attention_weight_min": -24.75241470336914,
      "activations/layer19_attention_weight_max": 31.316877365112305,
      "activations/layer19_attention_weight_min": -31.113157272338867,
      "activations/layer1_attention_weight_max": 15.859047889709473,
      "activations/layer1_attention_weight_min": -15.20574951171875,
      "activations/layer20_attention_weight_max": 25.40733528137207,
      "activations/layer20_attention_weight_min": -24.104753494262695,
      "activations/layer21_attention_weight_max": 26.331188201904297,
      "activations/layer21_attention_weight_min": -23.461345672607422,
      "activations/layer22_attention_weight_max": 39.84628677368164,
      "activations/layer22_attention_weight_min": -29.98554039001465,
      "activations/layer23_attention_weight_max": 31.447433471679688,
      "activations/layer23_attention_weight_min": -25.01727294921875,
      "activations/layer2_attention_weight_max": 33.64215087890625,
      "activations/layer2_attention_weight_min": -34.069454193115234,
      "activations/layer3_attention_weight_max": 93.36097717285156,
      "activations/layer3_attention_weight_min": -97.30418395996094,
      "activations/layer4_attention_weight_max": 102.89595794677734,
      "activations/layer4_attention_weight_min": -103.99344635009766,
      "activations/layer5_attention_weight_max": 50.84937286376953,
      "activations/layer5_attention_weight_min": -61.086952209472656,
      "activations/layer6_attention_weight_max": 43.83460235595703,
      "activations/layer6_attention_weight_min": -45.51729202270508,
      "activations/layer7_attention_weight_max": 91.55686950683594,
      "activations/layer7_attention_weight_min": -89.69197082519531,
      "activations/layer8_attention_weight_max": 39.17851638793945,
      "activations/layer8_attention_weight_min": -42.01362228393555,
      "activations/layer9_attention_weight_max": 30.031269073486328,
      "activations/layer9_attention_weight_min": -32.76791000366211,
      "epoch": 15.48,
      "learning_rate": 5.068787878787878e-05,
      "loss": 2.7525,
      "step": 266400
    },
    {
      "activations/layer0_attention_weight_max": 15.35405158996582,
      "activations/layer0_attention_weight_min": -12.350836753845215,
      "activations/layer10_attention_weight_max": 37.398494720458984,
      "activations/layer10_attention_weight_min": -38.07625198364258,
      "activations/layer11_attention_weight_max": 36.71533203125,
      "activations/layer11_attention_weight_min": -35.785919189453125,
      "activations/layer12_attention_weight_max": 28.029232025146484,
      "activations/layer12_attention_weight_min": -27.347639083862305,
      "activations/layer13_attention_weight_max": 42.84989929199219,
      "activations/layer13_attention_weight_min": -37.85335159301758,
      "activations/layer14_attention_weight_max": 50.99839782714844,
      "activations/layer14_attention_weight_min": -41.17441940307617,
      "activations/layer15_attention_weight_max": 48.20634460449219,
      "activations/layer15_attention_weight_min": -37.05351257324219,
      "activations/layer16_attention_weight_max": 32.73602294921875,
      "activations/layer16_attention_weight_min": -28.090837478637695,
      "activations/layer17_attention_weight_max": 31.443246841430664,
      "activations/layer17_attention_weight_min": -28.317350387573242,
      "activations/layer18_attention_weight_max": 32.79973602294922,
      "activations/layer18_attention_weight_min": -27.262907028198242,
      "activations/layer19_attention_weight_max": 39.8533821105957,
      "activations/layer19_attention_weight_min": -31.28692054748535,
      "activations/layer1_attention_weight_max": 18.506589889526367,
      "activations/layer1_attention_weight_min": -17.557029724121094,
      "activations/layer20_attention_weight_max": 33.88644790649414,
      "activations/layer20_attention_weight_min": -24.882169723510742,
      "activations/layer21_attention_weight_max": 31.042226791381836,
      "activations/layer21_attention_weight_min": -26.5897274017334,
      "activations/layer22_attention_weight_max": 59.328556060791016,
      "activations/layer22_attention_weight_min": -36.89716339111328,
      "activations/layer23_attention_weight_max": 39.62828826904297,
      "activations/layer23_attention_weight_min": -25.20904541015625,
      "activations/layer2_attention_weight_max": 33.085723876953125,
      "activations/layer2_attention_weight_min": -35.656578063964844,
      "activations/layer3_attention_weight_max": 97.32659912109375,
      "activations/layer3_attention_weight_min": -105.12379455566406,
      "activations/layer4_attention_weight_max": 108.1114730834961,
      "activations/layer4_attention_weight_min": -109.40169525146484,
      "activations/layer5_attention_weight_max": 52.29393005371094,
      "activations/layer5_attention_weight_min": -60.474021911621094,
      "activations/layer6_attention_weight_max": 44.831363677978516,
      "activations/layer6_attention_weight_min": -46.70530700683594,
      "activations/layer7_attention_weight_max": 91.66273498535156,
      "activations/layer7_attention_weight_min": -107.38749694824219,
      "activations/layer8_attention_weight_max": 47.0660514831543,
      "activations/layer8_attention_weight_min": -50.17639923095703,
      "activations/layer9_attention_weight_max": 34.6856575012207,
      "activations/layer9_attention_weight_min": -38.1435661315918,
      "epoch": 15.48,
      "learning_rate": 5.066893939393939e-05,
      "loss": 2.7403,
      "step": 266450
    },
    {
      "activations/layer0_attention_weight_max": 15.44253158569336,
      "activations/layer0_attention_weight_min": -12.070436477661133,
      "activations/layer10_attention_weight_max": 35.67256546020508,
      "activations/layer10_attention_weight_min": -35.99102020263672,
      "activations/layer11_attention_weight_max": 34.101959228515625,
      "activations/layer11_attention_weight_min": -36.34585189819336,
      "activations/layer12_attention_weight_max": 26.40557289123535,
      "activations/layer12_attention_weight_min": -26.284975051879883,
      "activations/layer13_attention_weight_max": 39.242279052734375,
      "activations/layer13_attention_weight_min": -35.09546661376953,
      "activations/layer14_attention_weight_max": 43.31413269042969,
      "activations/layer14_attention_weight_min": -37.58127975463867,
      "activations/layer15_attention_weight_max": 36.803504943847656,
      "activations/layer15_attention_weight_min": -35.75284194946289,
      "activations/layer16_attention_weight_max": 30.157398223876953,
      "activations/layer16_attention_weight_min": -26.810407638549805,
      "activations/layer17_attention_weight_max": 29.28230094909668,
      "activations/layer17_attention_weight_min": -26.504770278930664,
      "activations/layer18_attention_weight_max": 29.583101272583008,
      "activations/layer18_attention_weight_min": -26.756383895874023,
      "activations/layer19_attention_weight_max": 38.9309196472168,
      "activations/layer19_attention_weight_min": -32.02020263671875,
      "activations/layer1_attention_weight_max": 17.33810043334961,
      "activations/layer1_attention_weight_min": -15.2111177444458,
      "activations/layer20_attention_weight_max": 28.611011505126953,
      "activations/layer20_attention_weight_min": -25.898359298706055,
      "activations/layer21_attention_weight_max": 26.801284790039062,
      "activations/layer21_attention_weight_min": -24.7130069732666,
      "activations/layer22_attention_weight_max": 44.171783447265625,
      "activations/layer22_attention_weight_min": -30.7047119140625,
      "activations/layer23_attention_weight_max": 37.321834564208984,
      "activations/layer23_attention_weight_min": -24.27011489868164,
      "activations/layer2_attention_weight_max": 34.35089111328125,
      "activations/layer2_attention_weight_min": -34.037147521972656,
      "activations/layer3_attention_weight_max": 98.96209716796875,
      "activations/layer3_attention_weight_min": -97.43660736083984,
      "activations/layer4_attention_weight_max": 107.81990051269531,
      "activations/layer4_attention_weight_min": -109.4771499633789,
      "activations/layer5_attention_weight_max": 52.811744689941406,
      "activations/layer5_attention_weight_min": -62.394989013671875,
      "activations/layer6_attention_weight_max": 44.17933654785156,
      "activations/layer6_attention_weight_min": -48.17744064331055,
      "activations/layer7_attention_weight_max": 90.56523132324219,
      "activations/layer7_attention_weight_min": -90.030029296875,
      "activations/layer8_attention_weight_max": 41.973663330078125,
      "activations/layer8_attention_weight_min": -45.89864730834961,
      "activations/layer9_attention_weight_max": 34.39274215698242,
      "activations/layer9_attention_weight_min": -35.078609466552734,
      "epoch": 15.49,
      "learning_rate": 5.065e-05,
      "loss": 2.754,
      "step": 266500
    },
    {
      "activations/layer0_attention_weight_max": 15.881641387939453,
      "activations/layer0_attention_weight_min": -12.373958587646484,
      "activations/layer10_attention_weight_max": 32.30238723754883,
      "activations/layer10_attention_weight_min": -34.112545013427734,
      "activations/layer11_attention_weight_max": 31.82079315185547,
      "activations/layer11_attention_weight_min": -33.436607360839844,
      "activations/layer12_attention_weight_max": 26.59872817993164,
      "activations/layer12_attention_weight_min": -26.01856231689453,
      "activations/layer13_attention_weight_max": 39.39556121826172,
      "activations/layer13_attention_weight_min": -35.16762924194336,
      "activations/layer14_attention_weight_max": 44.916725158691406,
      "activations/layer14_attention_weight_min": -37.177032470703125,
      "activations/layer15_attention_weight_max": 41.166595458984375,
      "activations/layer15_attention_weight_min": -35.155487060546875,
      "activations/layer16_attention_weight_max": 32.12914276123047,
      "activations/layer16_attention_weight_min": -27.140544891357422,
      "activations/layer17_attention_weight_max": 30.90797233581543,
      "activations/layer17_attention_weight_min": -25.632654190063477,
      "activations/layer18_attention_weight_max": 34.001853942871094,
      "activations/layer18_attention_weight_min": -23.915651321411133,
      "activations/layer19_attention_weight_max": 35.137184143066406,
      "activations/layer19_attention_weight_min": -29.86814308166504,
      "activations/layer1_attention_weight_max": 17.07466697692871,
      "activations/layer1_attention_weight_min": -14.786565780639648,
      "activations/layer20_attention_weight_max": 30.346416473388672,
      "activations/layer20_attention_weight_min": -24.34988784790039,
      "activations/layer21_attention_weight_max": 30.308042526245117,
      "activations/layer21_attention_weight_min": -24.27043342590332,
      "activations/layer22_attention_weight_max": 45.44334411621094,
      "activations/layer22_attention_weight_min": -29.156049728393555,
      "activations/layer23_attention_weight_max": 35.118377685546875,
      "activations/layer23_attention_weight_min": -24.49981689453125,
      "activations/layer2_attention_weight_max": 32.404788970947266,
      "activations/layer2_attention_weight_min": -31.88190269470215,
      "activations/layer3_attention_weight_max": 94.24276733398438,
      "activations/layer3_attention_weight_min": -94.81636810302734,
      "activations/layer4_attention_weight_max": 105.54259490966797,
      "activations/layer4_attention_weight_min": -107.05696868896484,
      "activations/layer5_attention_weight_max": 50.44210433959961,
      "activations/layer5_attention_weight_min": -62.86848449707031,
      "activations/layer6_attention_weight_max": 41.106788635253906,
      "activations/layer6_attention_weight_min": -45.548301696777344,
      "activations/layer7_attention_weight_max": 90.22936248779297,
      "activations/layer7_attention_weight_min": -91.4448013305664,
      "activations/layer8_attention_weight_max": 38.48447799682617,
      "activations/layer8_attention_weight_min": -41.77161407470703,
      "activations/layer9_attention_weight_max": 30.733810424804688,
      "activations/layer9_attention_weight_min": -33.23866271972656,
      "epoch": 15.49,
      "learning_rate": 5.06310606060606e-05,
      "loss": 2.7451,
      "step": 266550
    },
    {
      "activations/layer0_attention_weight_max": 15.512649536132812,
      "activations/layer0_attention_weight_min": -11.255355834960938,
      "activations/layer10_attention_weight_max": 31.906211853027344,
      "activations/layer10_attention_weight_min": -31.409893035888672,
      "activations/layer11_attention_weight_max": 33.05010986328125,
      "activations/layer11_attention_weight_min": -32.158058166503906,
      "activations/layer12_attention_weight_max": 24.03064727783203,
      "activations/layer12_attention_weight_min": -25.30539321899414,
      "activations/layer13_attention_weight_max": 37.04045486450195,
      "activations/layer13_attention_weight_min": -34.461185455322266,
      "activations/layer14_attention_weight_max": 41.31789016723633,
      "activations/layer14_attention_weight_min": -36.943634033203125,
      "activations/layer15_attention_weight_max": 36.184146881103516,
      "activations/layer15_attention_weight_min": -33.656639099121094,
      "activations/layer16_attention_weight_max": 28.465909957885742,
      "activations/layer16_attention_weight_min": -27.618946075439453,
      "activations/layer17_attention_weight_max": 29.16349983215332,
      "activations/layer17_attention_weight_min": -27.39307403564453,
      "activations/layer18_attention_weight_max": 31.1837100982666,
      "activations/layer18_attention_weight_min": -24.582807540893555,
      "activations/layer19_attention_weight_max": 30.97787857055664,
      "activations/layer19_attention_weight_min": -34.90708541870117,
      "activations/layer1_attention_weight_max": 15.524840354919434,
      "activations/layer1_attention_weight_min": -14.7809419631958,
      "activations/layer20_attention_weight_max": 25.962650299072266,
      "activations/layer20_attention_weight_min": -26.39308738708496,
      "activations/layer21_attention_weight_max": 25.885400772094727,
      "activations/layer21_attention_weight_min": -27.70619773864746,
      "activations/layer22_attention_weight_max": 36.426734924316406,
      "activations/layer22_attention_weight_min": -30.738794326782227,
      "activations/layer23_attention_weight_max": 31.41059684753418,
      "activations/layer23_attention_weight_min": -26.779003143310547,
      "activations/layer2_attention_weight_max": 30.50502586364746,
      "activations/layer2_attention_weight_min": -31.608707427978516,
      "activations/layer3_attention_weight_max": 93.34031677246094,
      "activations/layer3_attention_weight_min": -95.892822265625,
      "activations/layer4_attention_weight_max": 106.55902099609375,
      "activations/layer4_attention_weight_min": -108.21025848388672,
      "activations/layer5_attention_weight_max": 51.86711120605469,
      "activations/layer5_attention_weight_min": -62.968536376953125,
      "activations/layer6_attention_weight_max": 41.252685546875,
      "activations/layer6_attention_weight_min": -44.511234283447266,
      "activations/layer7_attention_weight_max": 88.35453796386719,
      "activations/layer7_attention_weight_min": -92.25138092041016,
      "activations/layer8_attention_weight_max": 38.45820999145508,
      "activations/layer8_attention_weight_min": -40.34922409057617,
      "activations/layer9_attention_weight_max": 31.005069732666016,
      "activations/layer9_attention_weight_min": -30.901519775390625,
      "epoch": 15.49,
      "learning_rate": 5.06121212121212e-05,
      "loss": 2.7489,
      "step": 266600
    },
    {
      "activations/layer0_attention_weight_max": 15.599781036376953,
      "activations/layer0_attention_weight_min": -11.915279388427734,
      "activations/layer10_attention_weight_max": 33.8898811340332,
      "activations/layer10_attention_weight_min": -32.1453742980957,
      "activations/layer11_attention_weight_max": 32.79216003417969,
      "activations/layer11_attention_weight_min": -32.19513702392578,
      "activations/layer12_attention_weight_max": 25.651365280151367,
      "activations/layer12_attention_weight_min": -25.062280654907227,
      "activations/layer13_attention_weight_max": 39.35918045043945,
      "activations/layer13_attention_weight_min": -35.058860778808594,
      "activations/layer14_attention_weight_max": 42.07999801635742,
      "activations/layer14_attention_weight_min": -35.38748550415039,
      "activations/layer15_attention_weight_max": 38.279884338378906,
      "activations/layer15_attention_weight_min": -31.065418243408203,
      "activations/layer16_attention_weight_max": 28.14037322998047,
      "activations/layer16_attention_weight_min": -28.715246200561523,
      "activations/layer17_attention_weight_max": 31.93168067932129,
      "activations/layer17_attention_weight_min": -27.844253540039062,
      "activations/layer18_attention_weight_max": 32.25410842895508,
      "activations/layer18_attention_weight_min": -25.142532348632812,
      "activations/layer19_attention_weight_max": 32.49506378173828,
      "activations/layer19_attention_weight_min": -30.155515670776367,
      "activations/layer1_attention_weight_max": 16.78516960144043,
      "activations/layer1_attention_weight_min": -14.545239448547363,
      "activations/layer20_attention_weight_max": 30.548370361328125,
      "activations/layer20_attention_weight_min": -25.38793182373047,
      "activations/layer21_attention_weight_max": 30.59467124938965,
      "activations/layer21_attention_weight_min": -23.920324325561523,
      "activations/layer22_attention_weight_max": 44.0593147277832,
      "activations/layer22_attention_weight_min": -30.79391860961914,
      "activations/layer23_attention_weight_max": 32.66420364379883,
      "activations/layer23_attention_weight_min": -27.08182144165039,
      "activations/layer2_attention_weight_max": 32.53571319580078,
      "activations/layer2_attention_weight_min": -32.29759216308594,
      "activations/layer3_attention_weight_max": 97.49656677246094,
      "activations/layer3_attention_weight_min": -100.51436614990234,
      "activations/layer4_attention_weight_max": 111.09972381591797,
      "activations/layer4_attention_weight_min": -112.43514251708984,
      "activations/layer5_attention_weight_max": 54.973602294921875,
      "activations/layer5_attention_weight_min": -63.603633880615234,
      "activations/layer6_attention_weight_max": 43.86049270629883,
      "activations/layer6_attention_weight_min": -46.163150787353516,
      "activations/layer7_attention_weight_max": 88.24443817138672,
      "activations/layer7_attention_weight_min": -94.19216918945312,
      "activations/layer8_attention_weight_max": 41.05009078979492,
      "activations/layer8_attention_weight_min": -40.66119384765625,
      "activations/layer9_attention_weight_max": 31.90158462524414,
      "activations/layer9_attention_weight_min": -33.28459930419922,
      "epoch": 15.49,
      "learning_rate": 5.0593181818181816e-05,
      "loss": 2.731,
      "step": 266650
    },
    {
      "activations/layer0_attention_weight_max": 16.583786010742188,
      "activations/layer0_attention_weight_min": -12.479543685913086,
      "activations/layer10_attention_weight_max": 38.825469970703125,
      "activations/layer10_attention_weight_min": -34.36198043823242,
      "activations/layer11_attention_weight_max": 37.698036193847656,
      "activations/layer11_attention_weight_min": -34.93785095214844,
      "activations/layer12_attention_weight_max": 27.681659698486328,
      "activations/layer12_attention_weight_min": -26.889684677124023,
      "activations/layer13_attention_weight_max": 42.66876983642578,
      "activations/layer13_attention_weight_min": -36.83240509033203,
      "activations/layer14_attention_weight_max": 44.6827278137207,
      "activations/layer14_attention_weight_min": -38.44626235961914,
      "activations/layer15_attention_weight_max": 41.64974594116211,
      "activations/layer15_attention_weight_min": -34.839717864990234,
      "activations/layer16_attention_weight_max": 32.79008865356445,
      "activations/layer16_attention_weight_min": -29.3840389251709,
      "activations/layer17_attention_weight_max": 29.324642181396484,
      "activations/layer17_attention_weight_min": -26.854419708251953,
      "activations/layer18_attention_weight_max": 29.444072723388672,
      "activations/layer18_attention_weight_min": -25.0310001373291,
      "activations/layer19_attention_weight_max": 34.276283264160156,
      "activations/layer19_attention_weight_min": -32.01261520385742,
      "activations/layer1_attention_weight_max": 16.082008361816406,
      "activations/layer1_attention_weight_min": -15.47419261932373,
      "activations/layer20_attention_weight_max": 31.10735321044922,
      "activations/layer20_attention_weight_min": -26.84579086303711,
      "activations/layer21_attention_weight_max": 30.501279830932617,
      "activations/layer21_attention_weight_min": -26.91375732421875,
      "activations/layer22_attention_weight_max": 43.181556701660156,
      "activations/layer22_attention_weight_min": -31.2363338470459,
      "activations/layer23_attention_weight_max": 31.050857543945312,
      "activations/layer23_attention_weight_min": -26.172344207763672,
      "activations/layer2_attention_weight_max": 32.65242004394531,
      "activations/layer2_attention_weight_min": -32.88233184814453,
      "activations/layer3_attention_weight_max": 95.14904022216797,
      "activations/layer3_attention_weight_min": -96.98155212402344,
      "activations/layer4_attention_weight_max": 105.58601379394531,
      "activations/layer4_attention_weight_min": -107.5263900756836,
      "activations/layer5_attention_weight_max": 52.383846282958984,
      "activations/layer5_attention_weight_min": -67.0778579711914,
      "activations/layer6_attention_weight_max": 44.6486701965332,
      "activations/layer6_attention_weight_min": -46.58988571166992,
      "activations/layer7_attention_weight_max": 99.192138671875,
      "activations/layer7_attention_weight_min": -95.97048950195312,
      "activations/layer8_attention_weight_max": 44.85405349731445,
      "activations/layer8_attention_weight_min": -44.06947708129883,
      "activations/layer9_attention_weight_max": 37.13555145263672,
      "activations/layer9_attention_weight_min": -35.783973693847656,
      "epoch": 15.5,
      "learning_rate": 5.057424242424242e-05,
      "loss": 2.7507,
      "step": 266700
    },
    {
      "activations/layer0_attention_weight_max": 16.05384063720703,
      "activations/layer0_attention_weight_min": -12.199980735778809,
      "activations/layer10_attention_weight_max": 33.196327209472656,
      "activations/layer10_attention_weight_min": -33.265106201171875,
      "activations/layer11_attention_weight_max": 32.475521087646484,
      "activations/layer11_attention_weight_min": -33.23279571533203,
      "activations/layer12_attention_weight_max": 27.71990203857422,
      "activations/layer12_attention_weight_min": -25.222217559814453,
      "activations/layer13_attention_weight_max": 36.68381881713867,
      "activations/layer13_attention_weight_min": -34.21206283569336,
      "activations/layer14_attention_weight_max": 38.88650894165039,
      "activations/layer14_attention_weight_min": -34.44780349731445,
      "activations/layer15_attention_weight_max": 37.052337646484375,
      "activations/layer15_attention_weight_min": -33.85642623901367,
      "activations/layer16_attention_weight_max": 29.3164119720459,
      "activations/layer16_attention_weight_min": -28.919841766357422,
      "activations/layer17_attention_weight_max": 30.692378997802734,
      "activations/layer17_attention_weight_min": -26.131587982177734,
      "activations/layer18_attention_weight_max": 29.6083984375,
      "activations/layer18_attention_weight_min": -23.74105453491211,
      "activations/layer19_attention_weight_max": 33.84382247924805,
      "activations/layer19_attention_weight_min": -29.158212661743164,
      "activations/layer1_attention_weight_max": 15.78380298614502,
      "activations/layer1_attention_weight_min": -15.829602241516113,
      "activations/layer20_attention_weight_max": 29.60905647277832,
      "activations/layer20_attention_weight_min": -25.477420806884766,
      "activations/layer21_attention_weight_max": 26.75419807434082,
      "activations/layer21_attention_weight_min": -25.27422332763672,
      "activations/layer22_attention_weight_max": 39.16114044189453,
      "activations/layer22_attention_weight_min": -31.36798095703125,
      "activations/layer23_attention_weight_max": 34.39307403564453,
      "activations/layer23_attention_weight_min": -24.794578552246094,
      "activations/layer2_attention_weight_max": 31.20954704284668,
      "activations/layer2_attention_weight_min": -31.98263168334961,
      "activations/layer3_attention_weight_max": 90.3435287475586,
      "activations/layer3_attention_weight_min": -89.50226593017578,
      "activations/layer4_attention_weight_max": 102.57244873046875,
      "activations/layer4_attention_weight_min": -104.605224609375,
      "activations/layer5_attention_weight_max": 50.335655212402344,
      "activations/layer5_attention_weight_min": -59.18273162841797,
      "activations/layer6_attention_weight_max": 43.51972198486328,
      "activations/layer6_attention_weight_min": -44.81967544555664,
      "activations/layer7_attention_weight_max": 88.82658386230469,
      "activations/layer7_attention_weight_min": -88.00491333007812,
      "activations/layer8_attention_weight_max": 42.682838439941406,
      "activations/layer8_attention_weight_min": -42.91522979736328,
      "activations/layer9_attention_weight_max": 31.797882080078125,
      "activations/layer9_attention_weight_min": -34.86003112792969,
      "epoch": 15.5,
      "learning_rate": 5.0555303030303026e-05,
      "loss": 2.7315,
      "step": 266750
    },
    {
      "activations/layer0_attention_weight_max": 14.845361709594727,
      "activations/layer0_attention_weight_min": -12.680072784423828,
      "activations/layer10_attention_weight_max": 32.70362091064453,
      "activations/layer10_attention_weight_min": -33.104461669921875,
      "activations/layer11_attention_weight_max": 32.10380172729492,
      "activations/layer11_attention_weight_min": -32.92216491699219,
      "activations/layer12_attention_weight_max": 24.924177169799805,
      "activations/layer12_attention_weight_min": -25.196319580078125,
      "activations/layer13_attention_weight_max": 39.16266632080078,
      "activations/layer13_attention_weight_min": -34.3865966796875,
      "activations/layer14_attention_weight_max": 42.51890563964844,
      "activations/layer14_attention_weight_min": -37.34890365600586,
      "activations/layer15_attention_weight_max": 36.691009521484375,
      "activations/layer15_attention_weight_min": -34.83045959472656,
      "activations/layer16_attention_weight_max": 30.746919631958008,
      "activations/layer16_attention_weight_min": -29.830615997314453,
      "activations/layer17_attention_weight_max": 32.7374153137207,
      "activations/layer17_attention_weight_min": -26.47930335998535,
      "activations/layer18_attention_weight_max": 34.65596008300781,
      "activations/layer18_attention_weight_min": -25.618383407592773,
      "activations/layer19_attention_weight_max": 33.43612289428711,
      "activations/layer19_attention_weight_min": -34.705352783203125,
      "activations/layer1_attention_weight_max": 15.013498306274414,
      "activations/layer1_attention_weight_min": -15.095686912536621,
      "activations/layer20_attention_weight_max": 28.09373664855957,
      "activations/layer20_attention_weight_min": -27.066112518310547,
      "activations/layer21_attention_weight_max": 28.536991119384766,
      "activations/layer21_attention_weight_min": -25.509963989257812,
      "activations/layer22_attention_weight_max": 40.83988952636719,
      "activations/layer22_attention_weight_min": -34.31462478637695,
      "activations/layer23_attention_weight_max": 35.32090759277344,
      "activations/layer23_attention_weight_min": -28.750097274780273,
      "activations/layer2_attention_weight_max": 32.62590789794922,
      "activations/layer2_attention_weight_min": -31.74749755859375,
      "activations/layer3_attention_weight_max": 95.6273193359375,
      "activations/layer3_attention_weight_min": -94.75254821777344,
      "activations/layer4_attention_weight_max": 111.82862854003906,
      "activations/layer4_attention_weight_min": -106.81117248535156,
      "activations/layer5_attention_weight_max": 51.52313232421875,
      "activations/layer5_attention_weight_min": -65.11979675292969,
      "activations/layer6_attention_weight_max": 42.29982376098633,
      "activations/layer6_attention_weight_min": -47.15209197998047,
      "activations/layer7_attention_weight_max": 99.8300552368164,
      "activations/layer7_attention_weight_min": -90.39219665527344,
      "activations/layer8_attention_weight_max": 41.791107177734375,
      "activations/layer8_attention_weight_min": -45.002037048339844,
      "activations/layer9_attention_weight_max": 32.92885971069336,
      "activations/layer9_attention_weight_min": -33.500003814697266,
      "epoch": 15.5,
      "learning_rate": 5.0536363636363634e-05,
      "loss": 2.7519,
      "step": 266800
    },
    {
      "activations/layer0_attention_weight_max": 15.693892478942871,
      "activations/layer0_attention_weight_min": -13.118683815002441,
      "activations/layer10_attention_weight_max": 31.99449348449707,
      "activations/layer10_attention_weight_min": -33.42385482788086,
      "activations/layer11_attention_weight_max": 32.162681579589844,
      "activations/layer11_attention_weight_min": -32.83515930175781,
      "activations/layer12_attention_weight_max": 28.239301681518555,
      "activations/layer12_attention_weight_min": -27.413728713989258,
      "activations/layer13_attention_weight_max": 39.89891052246094,
      "activations/layer13_attention_weight_min": -38.07548522949219,
      "activations/layer14_attention_weight_max": 47.76004409790039,
      "activations/layer14_attention_weight_min": -38.06100845336914,
      "activations/layer15_attention_weight_max": 38.64188003540039,
      "activations/layer15_attention_weight_min": -36.55975341796875,
      "activations/layer16_attention_weight_max": 30.35362434387207,
      "activations/layer16_attention_weight_min": -26.707489013671875,
      "activations/layer17_attention_weight_max": 31.078989028930664,
      "activations/layer17_attention_weight_min": -24.646732330322266,
      "activations/layer18_attention_weight_max": 30.71859359741211,
      "activations/layer18_attention_weight_min": -24.439128875732422,
      "activations/layer19_attention_weight_max": 33.6233024597168,
      "activations/layer19_attention_weight_min": -30.59168815612793,
      "activations/layer1_attention_weight_max": 16.73031234741211,
      "activations/layer1_attention_weight_min": -16.62815284729004,
      "activations/layer20_attention_weight_max": 28.069068908691406,
      "activations/layer20_attention_weight_min": -25.037023544311523,
      "activations/layer21_attention_weight_max": 28.15179443359375,
      "activations/layer21_attention_weight_min": -25.638883590698242,
      "activations/layer22_attention_weight_max": 44.08675765991211,
      "activations/layer22_attention_weight_min": -29.80709457397461,
      "activations/layer23_attention_weight_max": 31.711971282958984,
      "activations/layer23_attention_weight_min": -25.234548568725586,
      "activations/layer2_attention_weight_max": 33.529747009277344,
      "activations/layer2_attention_weight_min": -32.72273254394531,
      "activations/layer3_attention_weight_max": 94.45367431640625,
      "activations/layer3_attention_weight_min": -98.63992309570312,
      "activations/layer4_attention_weight_max": 104.89147186279297,
      "activations/layer4_attention_weight_min": -106.45162200927734,
      "activations/layer5_attention_weight_max": 53.33678436279297,
      "activations/layer5_attention_weight_min": -62.43954086303711,
      "activations/layer6_attention_weight_max": 44.02244567871094,
      "activations/layer6_attention_weight_min": -45.502864837646484,
      "activations/layer7_attention_weight_max": 94.02556610107422,
      "activations/layer7_attention_weight_min": -94.29658508300781,
      "activations/layer8_attention_weight_max": 41.83417892456055,
      "activations/layer8_attention_weight_min": -42.371482849121094,
      "activations/layer9_attention_weight_max": 31.204206466674805,
      "activations/layer9_attention_weight_min": -33.584590911865234,
      "epoch": 15.51,
      "learning_rate": 5.051742424242424e-05,
      "loss": 2.7497,
      "step": 266850
    },
    {
      "activations/layer0_attention_weight_max": 15.625927925109863,
      "activations/layer0_attention_weight_min": -13.13007926940918,
      "activations/layer10_attention_weight_max": 33.0546875,
      "activations/layer10_attention_weight_min": -33.977752685546875,
      "activations/layer11_attention_weight_max": 36.425193786621094,
      "activations/layer11_attention_weight_min": -33.2498893737793,
      "activations/layer12_attention_weight_max": 25.416959762573242,
      "activations/layer12_attention_weight_min": -24.30544662475586,
      "activations/layer13_attention_weight_max": 38.5212516784668,
      "activations/layer13_attention_weight_min": -32.33120346069336,
      "activations/layer14_attention_weight_max": 40.31820297241211,
      "activations/layer14_attention_weight_min": -35.727935791015625,
      "activations/layer15_attention_weight_max": 37.30613327026367,
      "activations/layer15_attention_weight_min": -33.317161560058594,
      "activations/layer16_attention_weight_max": 29.37742042541504,
      "activations/layer16_attention_weight_min": -27.837705612182617,
      "activations/layer17_attention_weight_max": 28.237939834594727,
      "activations/layer17_attention_weight_min": -25.102067947387695,
      "activations/layer18_attention_weight_max": 30.297771453857422,
      "activations/layer18_attention_weight_min": -25.85836410522461,
      "activations/layer19_attention_weight_max": 30.4257869720459,
      "activations/layer19_attention_weight_min": -31.92803955078125,
      "activations/layer1_attention_weight_max": 17.223228454589844,
      "activations/layer1_attention_weight_min": -16.911474227905273,
      "activations/layer20_attention_weight_max": 27.06999397277832,
      "activations/layer20_attention_weight_min": -24.136363983154297,
      "activations/layer21_attention_weight_max": 25.422775268554688,
      "activations/layer21_attention_weight_min": -23.713346481323242,
      "activations/layer22_attention_weight_max": 38.631675720214844,
      "activations/layer22_attention_weight_min": -30.35318374633789,
      "activations/layer23_attention_weight_max": 31.075817108154297,
      "activations/layer23_attention_weight_min": -23.689878463745117,
      "activations/layer2_attention_weight_max": 33.93988037109375,
      "activations/layer2_attention_weight_min": -34.744075775146484,
      "activations/layer3_attention_weight_max": 97.50057983398438,
      "activations/layer3_attention_weight_min": -102.28791046142578,
      "activations/layer4_attention_weight_max": 106.76488494873047,
      "activations/layer4_attention_weight_min": -108.701904296875,
      "activations/layer5_attention_weight_max": 49.40581512451172,
      "activations/layer5_attention_weight_min": -59.54078674316406,
      "activations/layer6_attention_weight_max": 42.724205017089844,
      "activations/layer6_attention_weight_min": -45.96760177612305,
      "activations/layer7_attention_weight_max": 86.4323501586914,
      "activations/layer7_attention_weight_min": -91.16180419921875,
      "activations/layer8_attention_weight_max": 42.101646423339844,
      "activations/layer8_attention_weight_min": -41.16862869262695,
      "activations/layer9_attention_weight_max": 32.99558639526367,
      "activations/layer9_attention_weight_min": -34.49161911010742,
      "epoch": 15.51,
      "learning_rate": 5.049886363636363e-05,
      "loss": 2.7567,
      "step": 266900
    },
    {
      "activations/layer0_attention_weight_max": 16.406801223754883,
      "activations/layer0_attention_weight_min": -11.814201354980469,
      "activations/layer10_attention_weight_max": 37.281272888183594,
      "activations/layer10_attention_weight_min": -35.14509963989258,
      "activations/layer11_attention_weight_max": 38.00002670288086,
      "activations/layer11_attention_weight_min": -36.23762893676758,
      "activations/layer12_attention_weight_max": 25.350370407104492,
      "activations/layer12_attention_weight_min": -26.302810668945312,
      "activations/layer13_attention_weight_max": 36.98929977416992,
      "activations/layer13_attention_weight_min": -35.56278991699219,
      "activations/layer14_attention_weight_max": 44.2900276184082,
      "activations/layer14_attention_weight_min": -37.35708236694336,
      "activations/layer15_attention_weight_max": 40.04693603515625,
      "activations/layer15_attention_weight_min": -34.59916687011719,
      "activations/layer16_attention_weight_max": 35.14822006225586,
      "activations/layer16_attention_weight_min": -30.053348541259766,
      "activations/layer17_attention_weight_max": 31.58587074279785,
      "activations/layer17_attention_weight_min": -25.156435012817383,
      "activations/layer18_attention_weight_max": 32.081241607666016,
      "activations/layer18_attention_weight_min": -24.91915512084961,
      "activations/layer19_attention_weight_max": 40.0387077331543,
      "activations/layer19_attention_weight_min": -34.76866149902344,
      "activations/layer1_attention_weight_max": 17.460458755493164,
      "activations/layer1_attention_weight_min": -14.88990306854248,
      "activations/layer20_attention_weight_max": 35.152652740478516,
      "activations/layer20_attention_weight_min": -25.569551467895508,
      "activations/layer21_attention_weight_max": 32.475460052490234,
      "activations/layer21_attention_weight_min": -24.33279800415039,
      "activations/layer22_attention_weight_max": 43.181236267089844,
      "activations/layer22_attention_weight_min": -33.35740280151367,
      "activations/layer23_attention_weight_max": 38.20807647705078,
      "activations/layer23_attention_weight_min": -25.979976654052734,
      "activations/layer2_attention_weight_max": 32.29155731201172,
      "activations/layer2_attention_weight_min": -33.64529800415039,
      "activations/layer3_attention_weight_max": 94.04924774169922,
      "activations/layer3_attention_weight_min": -96.26790618896484,
      "activations/layer4_attention_weight_max": 106.2145004272461,
      "activations/layer4_attention_weight_min": -105.74187469482422,
      "activations/layer5_attention_weight_max": 51.067466735839844,
      "activations/layer5_attention_weight_min": -63.771484375,
      "activations/layer6_attention_weight_max": 44.55155563354492,
      "activations/layer6_attention_weight_min": -45.57426452636719,
      "activations/layer7_attention_weight_max": 101.5990982055664,
      "activations/layer7_attention_weight_min": -97.56333923339844,
      "activations/layer8_attention_weight_max": 44.95442199707031,
      "activations/layer8_attention_weight_min": -45.81476593017578,
      "activations/layer9_attention_weight_max": 33.073448181152344,
      "activations/layer9_attention_weight_min": -36.87446212768555,
      "epoch": 15.51,
      "learning_rate": 5.0479924242424235e-05,
      "loss": 2.7316,
      "step": 266950
    },
    {
      "activations/layer0_attention_weight_max": 16.09607696533203,
      "activations/layer0_attention_weight_min": -12.599641799926758,
      "activations/layer10_attention_weight_max": 34.111846923828125,
      "activations/layer10_attention_weight_min": -33.933837890625,
      "activations/layer11_attention_weight_max": 32.924652099609375,
      "activations/layer11_attention_weight_min": -34.292236328125,
      "activations/layer12_attention_weight_max": 24.81151008605957,
      "activations/layer12_attention_weight_min": -27.34760856628418,
      "activations/layer13_attention_weight_max": 40.89930725097656,
      "activations/layer13_attention_weight_min": -36.299766540527344,
      "activations/layer14_attention_weight_max": 44.01157760620117,
      "activations/layer14_attention_weight_min": -42.729618072509766,
      "activations/layer15_attention_weight_max": 41.162498474121094,
      "activations/layer15_attention_weight_min": -39.39894104003906,
      "activations/layer16_attention_weight_max": 33.41852951049805,
      "activations/layer16_attention_weight_min": -30.856000900268555,
      "activations/layer17_attention_weight_max": 32.13430404663086,
      "activations/layer17_attention_weight_min": -26.16802978515625,
      "activations/layer18_attention_weight_max": 31.40662956237793,
      "activations/layer18_attention_weight_min": -26.33477020263672,
      "activations/layer19_attention_weight_max": 36.875850677490234,
      "activations/layer19_attention_weight_min": -31.63399314880371,
      "activations/layer1_attention_weight_max": 18.212820053100586,
      "activations/layer1_attention_weight_min": -16.637365341186523,
      "activations/layer20_attention_weight_max": 31.207889556884766,
      "activations/layer20_attention_weight_min": -24.958213806152344,
      "activations/layer21_attention_weight_max": 30.360862731933594,
      "activations/layer21_attention_weight_min": -24.332143783569336,
      "activations/layer22_attention_weight_max": 41.75688934326172,
      "activations/layer22_attention_weight_min": -30.771162033081055,
      "activations/layer23_attention_weight_max": 35.459190368652344,
      "activations/layer23_attention_weight_min": -26.10430145263672,
      "activations/layer2_attention_weight_max": 34.2273063659668,
      "activations/layer2_attention_weight_min": -32.481658935546875,
      "activations/layer3_attention_weight_max": 96.07669067382812,
      "activations/layer3_attention_weight_min": -96.25633239746094,
      "activations/layer4_attention_weight_max": 105.50335693359375,
      "activations/layer4_attention_weight_min": -109.43157958984375,
      "activations/layer5_attention_weight_max": 50.08513641357422,
      "activations/layer5_attention_weight_min": -60.55994415283203,
      "activations/layer6_attention_weight_max": 45.39809036254883,
      "activations/layer6_attention_weight_min": -48.65394973754883,
      "activations/layer7_attention_weight_max": 94.03617095947266,
      "activations/layer7_attention_weight_min": -93.89274597167969,
      "activations/layer8_attention_weight_max": 41.68769073486328,
      "activations/layer8_attention_weight_min": -45.244659423828125,
      "activations/layer9_attention_weight_max": 32.889984130859375,
      "activations/layer9_attention_weight_min": -34.64627456665039,
      "epoch": 15.51,
      "learning_rate": 5.046098484848485e-05,
      "loss": 2.7472,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4565,
      "eval_samples_per_second": 507.774,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4565,
      "eval_openwebtext_samples_per_second": 507.774,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9483,
      "eval_wikitext_samples_per_second": 234.046,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_lambada_loss": 2.42578125,
      "eval_lambada_ppl": 11.311062736078481,
      "eval_lambada_runtime": 9.5144,
      "eval_lambada_samples_per_second": 511.753,
      "step": 267000
    },
    {
      "activations/layer0_attention_weight_max": 16.123340606689453,
      "activations/layer0_attention_weight_min": -12.648755073547363,
      "activations/layer10_attention_weight_max": 32.102943420410156,
      "activations/layer10_attention_weight_min": -33.97459030151367,
      "activations/layer11_attention_weight_max": 30.338451385498047,
      "activations/layer11_attention_weight_min": -33.13764190673828,
      "activations/layer12_attention_weight_max": 25.717863082885742,
      "activations/layer12_attention_weight_min": -25.998960494995117,
      "activations/layer13_attention_weight_max": 37.29595947265625,
      "activations/layer13_attention_weight_min": -35.615997314453125,
      "activations/layer14_attention_weight_max": 39.64523696899414,
      "activations/layer14_attention_weight_min": -36.51728057861328,
      "activations/layer15_attention_weight_max": 36.84127426147461,
      "activations/layer15_attention_weight_min": -32.81070327758789,
      "activations/layer16_attention_weight_max": 30.162813186645508,
      "activations/layer16_attention_weight_min": -27.58883285522461,
      "activations/layer17_attention_weight_max": 30.554927825927734,
      "activations/layer17_attention_weight_min": -24.02745819091797,
      "activations/layer18_attention_weight_max": 28.643901824951172,
      "activations/layer18_attention_weight_min": -26.58572006225586,
      "activations/layer19_attention_weight_max": 35.445735931396484,
      "activations/layer19_attention_weight_min": -33.928810119628906,
      "activations/layer1_attention_weight_max": 15.472705841064453,
      "activations/layer1_attention_weight_min": -14.368714332580566,
      "activations/layer20_attention_weight_max": 28.008892059326172,
      "activations/layer20_attention_weight_min": -26.3535213470459,
      "activations/layer21_attention_weight_max": 29.4278564453125,
      "activations/layer21_attention_weight_min": -26.617557525634766,
      "activations/layer22_attention_weight_max": 39.969451904296875,
      "activations/layer22_attention_weight_min": -33.48801040649414,
      "activations/layer23_attention_weight_max": 33.42524337768555,
      "activations/layer23_attention_weight_min": -26.037883758544922,
      "activations/layer2_attention_weight_max": 32.951202392578125,
      "activations/layer2_attention_weight_min": -32.78376007080078,
      "activations/layer3_attention_weight_max": 99.80365753173828,
      "activations/layer3_attention_weight_min": -99.81878662109375,
      "activations/layer4_attention_weight_max": 109.41975402832031,
      "activations/layer4_attention_weight_min": -112.54437255859375,
      "activations/layer5_attention_weight_max": 55.179176330566406,
      "activations/layer5_attention_weight_min": -65.19196319580078,
      "activations/layer6_attention_weight_max": 44.93824005126953,
      "activations/layer6_attention_weight_min": -46.78768539428711,
      "activations/layer7_attention_weight_max": 93.30672454833984,
      "activations/layer7_attention_weight_min": -93.81761169433594,
      "activations/layer8_attention_weight_max": 41.088871002197266,
      "activations/layer8_attention_weight_min": -44.38229751586914,
      "activations/layer9_attention_weight_max": 31.602359771728516,
      "activations/layer9_attention_weight_min": -33.18462371826172,
      "epoch": 15.52,
      "learning_rate": 5.044204545454545e-05,
      "loss": 2.7688,
      "step": 267050
    },
    {
      "activations/layer0_attention_weight_max": 16.933555603027344,
      "activations/layer0_attention_weight_min": -12.276710510253906,
      "activations/layer10_attention_weight_max": 34.29541015625,
      "activations/layer10_attention_weight_min": -35.207366943359375,
      "activations/layer11_attention_weight_max": 34.5681266784668,
      "activations/layer11_attention_weight_min": -34.41209030151367,
      "activations/layer12_attention_weight_max": 26.83125114440918,
      "activations/layer12_attention_weight_min": -26.967931747436523,
      "activations/layer13_attention_weight_max": 38.67421340942383,
      "activations/layer13_attention_weight_min": -37.102210998535156,
      "activations/layer14_attention_weight_max": 40.76570510864258,
      "activations/layer14_attention_weight_min": -38.16868591308594,
      "activations/layer15_attention_weight_max": 39.864593505859375,
      "activations/layer15_attention_weight_min": -35.5775146484375,
      "activations/layer16_attention_weight_max": 32.21125411987305,
      "activations/layer16_attention_weight_min": -28.113433837890625,
      "activations/layer17_attention_weight_max": 32.921329498291016,
      "activations/layer17_attention_weight_min": -26.426002502441406,
      "activations/layer18_attention_weight_max": 34.63654327392578,
      "activations/layer18_attention_weight_min": -25.601783752441406,
      "activations/layer19_attention_weight_max": 39.49950408935547,
      "activations/layer19_attention_weight_min": -32.507659912109375,
      "activations/layer1_attention_weight_max": 16.24724578857422,
      "activations/layer1_attention_weight_min": -15.124601364135742,
      "activations/layer20_attention_weight_max": 34.2656135559082,
      "activations/layer20_attention_weight_min": -26.1385555267334,
      "activations/layer21_attention_weight_max": 34.02582550048828,
      "activations/layer21_attention_weight_min": -24.576162338256836,
      "activations/layer22_attention_weight_max": 52.20355224609375,
      "activations/layer22_attention_weight_min": -32.127132415771484,
      "activations/layer23_attention_weight_max": 34.79636001586914,
      "activations/layer23_attention_weight_min": -27.918210983276367,
      "activations/layer2_attention_weight_max": 33.676422119140625,
      "activations/layer2_attention_weight_min": -33.708717346191406,
      "activations/layer3_attention_weight_max": 96.32654571533203,
      "activations/layer3_attention_weight_min": -97.37239837646484,
      "activations/layer4_attention_weight_max": 110.28011322021484,
      "activations/layer4_attention_weight_min": -108.62300872802734,
      "activations/layer5_attention_weight_max": 50.16555404663086,
      "activations/layer5_attention_weight_min": -61.62873840332031,
      "activations/layer6_attention_weight_max": 44.4088249206543,
      "activations/layer6_attention_weight_min": -47.03849411010742,
      "activations/layer7_attention_weight_max": 93.70063018798828,
      "activations/layer7_attention_weight_min": -93.8650131225586,
      "activations/layer8_attention_weight_max": 41.75501251220703,
      "activations/layer8_attention_weight_min": -42.82515335083008,
      "activations/layer9_attention_weight_max": 33.41435623168945,
      "activations/layer9_attention_weight_min": -35.622352600097656,
      "epoch": 15.52,
      "learning_rate": 5.042310606060605e-05,
      "loss": 2.7533,
      "step": 267100
    },
    {
      "activations/layer0_attention_weight_max": 15.144640922546387,
      "activations/layer0_attention_weight_min": -12.102730751037598,
      "activations/layer10_attention_weight_max": 31.679594039916992,
      "activations/layer10_attention_weight_min": -31.971187591552734,
      "activations/layer11_attention_weight_max": 31.988876342773438,
      "activations/layer11_attention_weight_min": -32.12302780151367,
      "activations/layer12_attention_weight_max": 24.0235652923584,
      "activations/layer12_attention_weight_min": -26.88398551940918,
      "activations/layer13_attention_weight_max": 36.644649505615234,
      "activations/layer13_attention_weight_min": -37.09538650512695,
      "activations/layer14_attention_weight_max": 39.51832962036133,
      "activations/layer14_attention_weight_min": -37.485328674316406,
      "activations/layer15_attention_weight_max": 37.8424072265625,
      "activations/layer15_attention_weight_min": -33.661312103271484,
      "activations/layer16_attention_weight_max": 26.979448318481445,
      "activations/layer16_attention_weight_min": -30.838199615478516,
      "activations/layer17_attention_weight_max": 29.98785400390625,
      "activations/layer17_attention_weight_min": -27.502344131469727,
      "activations/layer18_attention_weight_max": 35.444210052490234,
      "activations/layer18_attention_weight_min": -26.29530906677246,
      "activations/layer19_attention_weight_max": 35.98902130126953,
      "activations/layer19_attention_weight_min": -32.51124954223633,
      "activations/layer1_attention_weight_max": 16.501800537109375,
      "activations/layer1_attention_weight_min": -14.07353401184082,
      "activations/layer20_attention_weight_max": 31.11631965637207,
      "activations/layer20_attention_weight_min": -25.490575790405273,
      "activations/layer21_attention_weight_max": 30.333885192871094,
      "activations/layer21_attention_weight_min": -26.14611053466797,
      "activations/layer22_attention_weight_max": 40.29889678955078,
      "activations/layer22_attention_weight_min": -32.83313751220703,
      "activations/layer23_attention_weight_max": 35.00666046142578,
      "activations/layer23_attention_weight_min": -24.64950180053711,
      "activations/layer2_attention_weight_max": 32.35076141357422,
      "activations/layer2_attention_weight_min": -32.97938919067383,
      "activations/layer3_attention_weight_max": 94.16571807861328,
      "activations/layer3_attention_weight_min": -97.98174285888672,
      "activations/layer4_attention_weight_max": 106.02876281738281,
      "activations/layer4_attention_weight_min": -110.3490982055664,
      "activations/layer5_attention_weight_max": 51.361812591552734,
      "activations/layer5_attention_weight_min": -65.15901184082031,
      "activations/layer6_attention_weight_max": 42.67040252685547,
      "activations/layer6_attention_weight_min": -48.162593841552734,
      "activations/layer7_attention_weight_max": 87.98750305175781,
      "activations/layer7_attention_weight_min": -87.29653930664062,
      "activations/layer8_attention_weight_max": 39.22123336791992,
      "activations/layer8_attention_weight_min": -40.54994583129883,
      "activations/layer9_attention_weight_max": 30.9379825592041,
      "activations/layer9_attention_weight_min": -32.18246841430664,
      "epoch": 15.52,
      "learning_rate": 5.040416666666666e-05,
      "loss": 2.7452,
      "step": 267150
    },
    {
      "activations/layer0_attention_weight_max": 16.279470443725586,
      "activations/layer0_attention_weight_min": -11.964910507202148,
      "activations/layer10_attention_weight_max": 31.970170974731445,
      "activations/layer10_attention_weight_min": -31.509994506835938,
      "activations/layer11_attention_weight_max": 33.90129470825195,
      "activations/layer11_attention_weight_min": -32.08578872680664,
      "activations/layer12_attention_weight_max": 24.517547607421875,
      "activations/layer12_attention_weight_min": -25.63778305053711,
      "activations/layer13_attention_weight_max": 37.94212341308594,
      "activations/layer13_attention_weight_min": -35.66572952270508,
      "activations/layer14_attention_weight_max": 41.98760223388672,
      "activations/layer14_attention_weight_min": -36.36560821533203,
      "activations/layer15_attention_weight_max": 37.512451171875,
      "activations/layer15_attention_weight_min": -34.64878463745117,
      "activations/layer16_attention_weight_max": 31.357027053833008,
      "activations/layer16_attention_weight_min": -28.12646484375,
      "activations/layer17_attention_weight_max": 31.592529296875,
      "activations/layer17_attention_weight_min": -28.239587783813477,
      "activations/layer18_attention_weight_max": 30.257797241210938,
      "activations/layer18_attention_weight_min": -24.702632904052734,
      "activations/layer19_attention_weight_max": 38.8253173828125,
      "activations/layer19_attention_weight_min": -31.39739990234375,
      "activations/layer1_attention_weight_max": 15.92464828491211,
      "activations/layer1_attention_weight_min": -14.404118537902832,
      "activations/layer20_attention_weight_max": 28.948257446289062,
      "activations/layer20_attention_weight_min": -27.337100982666016,
      "activations/layer21_attention_weight_max": 28.081390380859375,
      "activations/layer21_attention_weight_min": -25.674245834350586,
      "activations/layer22_attention_weight_max": 45.02347946166992,
      "activations/layer22_attention_weight_min": -30.78409194946289,
      "activations/layer23_attention_weight_max": 33.875282287597656,
      "activations/layer23_attention_weight_min": -25.371421813964844,
      "activations/layer2_attention_weight_max": 32.82146453857422,
      "activations/layer2_attention_weight_min": -31.775571823120117,
      "activations/layer3_attention_weight_max": 94.44966125488281,
      "activations/layer3_attention_weight_min": -95.55125427246094,
      "activations/layer4_attention_weight_max": 106.2056655883789,
      "activations/layer4_attention_weight_min": -108.63783264160156,
      "activations/layer5_attention_weight_max": 49.56950378417969,
      "activations/layer5_attention_weight_min": -62.6961784362793,
      "activations/layer6_attention_weight_max": 41.78315734863281,
      "activations/layer6_attention_weight_min": -44.71983337402344,
      "activations/layer7_attention_weight_max": 91.05003356933594,
      "activations/layer7_attention_weight_min": -87.34672546386719,
      "activations/layer8_attention_weight_max": 39.5881233215332,
      "activations/layer8_attention_weight_min": -40.75566482543945,
      "activations/layer9_attention_weight_max": 31.887069702148438,
      "activations/layer9_attention_weight_min": -32.381771087646484,
      "epoch": 15.53,
      "learning_rate": 5.038522727272727e-05,
      "loss": 2.7551,
      "step": 267200
    },
    {
      "activations/layer0_attention_weight_max": 15.927695274353027,
      "activations/layer0_attention_weight_min": -12.767098426818848,
      "activations/layer10_attention_weight_max": 33.24569320678711,
      "activations/layer10_attention_weight_min": -33.73230743408203,
      "activations/layer11_attention_weight_max": 35.187660217285156,
      "activations/layer11_attention_weight_min": -33.70044708251953,
      "activations/layer12_attention_weight_max": 26.340978622436523,
      "activations/layer12_attention_weight_min": -25.73830795288086,
      "activations/layer13_attention_weight_max": 39.810874938964844,
      "activations/layer13_attention_weight_min": -35.492530822753906,
      "activations/layer14_attention_weight_max": 42.33033752441406,
      "activations/layer14_attention_weight_min": -38.11051559448242,
      "activations/layer15_attention_weight_max": 38.7191047668457,
      "activations/layer15_attention_weight_min": -35.411842346191406,
      "activations/layer16_attention_weight_max": 29.7933406829834,
      "activations/layer16_attention_weight_min": -28.73443603515625,
      "activations/layer17_attention_weight_max": 29.447290420532227,
      "activations/layer17_attention_weight_min": -26.490428924560547,
      "activations/layer18_attention_weight_max": 31.882253646850586,
      "activations/layer18_attention_weight_min": -26.01956558227539,
      "activations/layer19_attention_weight_max": 33.76871871948242,
      "activations/layer19_attention_weight_min": -32.227516174316406,
      "activations/layer1_attention_weight_max": 16.686635971069336,
      "activations/layer1_attention_weight_min": -16.29572296142578,
      "activations/layer20_attention_weight_max": 27.38106346130371,
      "activations/layer20_attention_weight_min": -25.37175178527832,
      "activations/layer21_attention_weight_max": 28.049943923950195,
      "activations/layer21_attention_weight_min": -25.904531478881836,
      "activations/layer22_attention_weight_max": 39.259178161621094,
      "activations/layer22_attention_weight_min": -31.577253341674805,
      "activations/layer23_attention_weight_max": 31.538650512695312,
      "activations/layer23_attention_weight_min": -26.982776641845703,
      "activations/layer2_attention_weight_max": 33.25669479370117,
      "activations/layer2_attention_weight_min": -33.59834671020508,
      "activations/layer3_attention_weight_max": 95.2488021850586,
      "activations/layer3_attention_weight_min": -94.09507751464844,
      "activations/layer4_attention_weight_max": 105.4516830444336,
      "activations/layer4_attention_weight_min": -108.1988754272461,
      "activations/layer5_attention_weight_max": 51.29643249511719,
      "activations/layer5_attention_weight_min": -61.921165466308594,
      "activations/layer6_attention_weight_max": 45.406673431396484,
      "activations/layer6_attention_weight_min": -49.08481216430664,
      "activations/layer7_attention_weight_max": 92.19459533691406,
      "activations/layer7_attention_weight_min": -93.86678314208984,
      "activations/layer8_attention_weight_max": 40.66117477416992,
      "activations/layer8_attention_weight_min": -46.49764633178711,
      "activations/layer9_attention_weight_max": 35.32187271118164,
      "activations/layer9_attention_weight_min": -36.64838790893555,
      "epoch": 15.53,
      "learning_rate": 5.036628787878788e-05,
      "loss": 2.7399,
      "step": 267250
    },
    {
      "activations/layer0_attention_weight_max": 17.011825561523438,
      "activations/layer0_attention_weight_min": -13.273177146911621,
      "activations/layer10_attention_weight_max": 31.753488540649414,
      "activations/layer10_attention_weight_min": -32.93011474609375,
      "activations/layer11_attention_weight_max": 31.800270080566406,
      "activations/layer11_attention_weight_min": -32.76606369018555,
      "activations/layer12_attention_weight_max": 25.542407989501953,
      "activations/layer12_attention_weight_min": -26.339954376220703,
      "activations/layer13_attention_weight_max": 38.741390228271484,
      "activations/layer13_attention_weight_min": -35.950496673583984,
      "activations/layer14_attention_weight_max": 42.475929260253906,
      "activations/layer14_attention_weight_min": -38.03147888183594,
      "activations/layer15_attention_weight_max": 39.1416130065918,
      "activations/layer15_attention_weight_min": -35.777400970458984,
      "activations/layer16_attention_weight_max": 30.68793487548828,
      "activations/layer16_attention_weight_min": -27.7838191986084,
      "activations/layer17_attention_weight_max": 29.19976234436035,
      "activations/layer17_attention_weight_min": -26.428564071655273,
      "activations/layer18_attention_weight_max": 29.750896453857422,
      "activations/layer18_attention_weight_min": -28.72490119934082,
      "activations/layer19_attention_weight_max": 34.830257415771484,
      "activations/layer19_attention_weight_min": -33.638587951660156,
      "activations/layer1_attention_weight_max": 16.51909637451172,
      "activations/layer1_attention_weight_min": -15.628827095031738,
      "activations/layer20_attention_weight_max": 30.68963623046875,
      "activations/layer20_attention_weight_min": -25.270591735839844,
      "activations/layer21_attention_weight_max": 27.944372177124023,
      "activations/layer21_attention_weight_min": -24.5314884185791,
      "activations/layer22_attention_weight_max": 41.28010940551758,
      "activations/layer22_attention_weight_min": -30.857955932617188,
      "activations/layer23_attention_weight_max": 35.02586364746094,
      "activations/layer23_attention_weight_min": -25.30525016784668,
      "activations/layer2_attention_weight_max": 31.88966178894043,
      "activations/layer2_attention_weight_min": -33.164451599121094,
      "activations/layer3_attention_weight_max": 94.04300689697266,
      "activations/layer3_attention_weight_min": -94.10172271728516,
      "activations/layer4_attention_weight_max": 103.4273910522461,
      "activations/layer4_attention_weight_min": -107.53401947021484,
      "activations/layer5_attention_weight_max": 50.02317810058594,
      "activations/layer5_attention_weight_min": -61.223289489746094,
      "activations/layer6_attention_weight_max": 43.77684783935547,
      "activations/layer6_attention_weight_min": -46.48798751831055,
      "activations/layer7_attention_weight_max": 96.24486541748047,
      "activations/layer7_attention_weight_min": -96.11815643310547,
      "activations/layer8_attention_weight_max": 41.16648483276367,
      "activations/layer8_attention_weight_min": -43.36054992675781,
      "activations/layer9_attention_weight_max": 31.294387817382812,
      "activations/layer9_attention_weight_min": -35.90003204345703,
      "epoch": 15.53,
      "learning_rate": 5.034734848484848e-05,
      "loss": 2.7388,
      "step": 267300
    },
    {
      "activations/layer0_attention_weight_max": 15.306640625,
      "activations/layer0_attention_weight_min": -12.296759605407715,
      "activations/layer10_attention_weight_max": 34.155914306640625,
      "activations/layer10_attention_weight_min": -35.204750061035156,
      "activations/layer11_attention_weight_max": 34.957916259765625,
      "activations/layer11_attention_weight_min": -33.146827697753906,
      "activations/layer12_attention_weight_max": 26.131650924682617,
      "activations/layer12_attention_weight_min": -26.822351455688477,
      "activations/layer13_attention_weight_max": 36.90057373046875,
      "activations/layer13_attention_weight_min": -34.93228530883789,
      "activations/layer14_attention_weight_max": 43.399314880371094,
      "activations/layer14_attention_weight_min": -37.91398620605469,
      "activations/layer15_attention_weight_max": 38.27494430541992,
      "activations/layer15_attention_weight_min": -34.93291473388672,
      "activations/layer16_attention_weight_max": 32.625946044921875,
      "activations/layer16_attention_weight_min": -28.645893096923828,
      "activations/layer17_attention_weight_max": 35.168365478515625,
      "activations/layer17_attention_weight_min": -27.835588455200195,
      "activations/layer18_attention_weight_max": 35.61109161376953,
      "activations/layer18_attention_weight_min": -27.7353515625,
      "activations/layer19_attention_weight_max": 35.821006774902344,
      "activations/layer19_attention_weight_min": -32.2678337097168,
      "activations/layer1_attention_weight_max": 18.32170295715332,
      "activations/layer1_attention_weight_min": -16.62473487854004,
      "activations/layer20_attention_weight_max": 30.35396385192871,
      "activations/layer20_attention_weight_min": -25.723608016967773,
      "activations/layer21_attention_weight_max": 29.017986297607422,
      "activations/layer21_attention_weight_min": -25.115060806274414,
      "activations/layer22_attention_weight_max": 47.02399444580078,
      "activations/layer22_attention_weight_min": -31.51405143737793,
      "activations/layer23_attention_weight_max": 37.79246139526367,
      "activations/layer23_attention_weight_min": -27.577495574951172,
      "activations/layer2_attention_weight_max": 33.94419860839844,
      "activations/layer2_attention_weight_min": -33.483699798583984,
      "activations/layer3_attention_weight_max": 99.52503204345703,
      "activations/layer3_attention_weight_min": -95.42530059814453,
      "activations/layer4_attention_weight_max": 111.8746109008789,
      "activations/layer4_attention_weight_min": -107.46773529052734,
      "activations/layer5_attention_weight_max": 51.872291564941406,
      "activations/layer5_attention_weight_min": -64.00273895263672,
      "activations/layer6_attention_weight_max": 45.917747497558594,
      "activations/layer6_attention_weight_min": -47.859886169433594,
      "activations/layer7_attention_weight_max": 92.48003387451172,
      "activations/layer7_attention_weight_min": -95.93445587158203,
      "activations/layer8_attention_weight_max": 42.755245208740234,
      "activations/layer8_attention_weight_min": -43.77136993408203,
      "activations/layer9_attention_weight_max": 32.20652389526367,
      "activations/layer9_attention_weight_min": -34.39020919799805,
      "epoch": 15.53,
      "learning_rate": 5.032840909090908e-05,
      "loss": 2.7448,
      "step": 267350
    },
    {
      "activations/layer0_attention_weight_max": 16.285057067871094,
      "activations/layer0_attention_weight_min": -12.599143981933594,
      "activations/layer10_attention_weight_max": 34.452659606933594,
      "activations/layer10_attention_weight_min": -34.555580139160156,
      "activations/layer11_attention_weight_max": 35.15639114379883,
      "activations/layer11_attention_weight_min": -33.00178146362305,
      "activations/layer12_attention_weight_max": 32.62990951538086,
      "activations/layer12_attention_weight_min": -25.844192504882812,
      "activations/layer13_attention_weight_max": 41.61112594604492,
      "activations/layer13_attention_weight_min": -34.732059478759766,
      "activations/layer14_attention_weight_max": 42.91033935546875,
      "activations/layer14_attention_weight_min": -38.16069030761719,
      "activations/layer15_attention_weight_max": 37.6203498840332,
      "activations/layer15_attention_weight_min": -34.98270034790039,
      "activations/layer16_attention_weight_max": 30.397123336791992,
      "activations/layer16_attention_weight_min": -26.754281997680664,
      "activations/layer17_attention_weight_max": 30.47513771057129,
      "activations/layer17_attention_weight_min": -24.448410034179688,
      "activations/layer18_attention_weight_max": 31.075870513916016,
      "activations/layer18_attention_weight_min": -24.713863372802734,
      "activations/layer19_attention_weight_max": 36.548980712890625,
      "activations/layer19_attention_weight_min": -31.395750045776367,
      "activations/layer1_attention_weight_max": 16.449569702148438,
      "activations/layer1_attention_weight_min": -14.64921760559082,
      "activations/layer20_attention_weight_max": 29.482463836669922,
      "activations/layer20_attention_weight_min": -25.446670532226562,
      "activations/layer21_attention_weight_max": 30.861589431762695,
      "activations/layer21_attention_weight_min": -26.42194175720215,
      "activations/layer22_attention_weight_max": 43.23029327392578,
      "activations/layer22_attention_weight_min": -30.66886329650879,
      "activations/layer23_attention_weight_max": 32.38364028930664,
      "activations/layer23_attention_weight_min": -25.536426544189453,
      "activations/layer2_attention_weight_max": 32.800682067871094,
      "activations/layer2_attention_weight_min": -32.90121078491211,
      "activations/layer3_attention_weight_max": 99.9316635131836,
      "activations/layer3_attention_weight_min": -100.35943603515625,
      "activations/layer4_attention_weight_max": 113.2414779663086,
      "activations/layer4_attention_weight_min": -115.48187255859375,
      "activations/layer5_attention_weight_max": 54.772422790527344,
      "activations/layer5_attention_weight_min": -69.02383422851562,
      "activations/layer6_attention_weight_max": 45.08873748779297,
      "activations/layer6_attention_weight_min": -46.39112854003906,
      "activations/layer7_attention_weight_max": 95.6231689453125,
      "activations/layer7_attention_weight_min": -98.27473449707031,
      "activations/layer8_attention_weight_max": 42.093692779541016,
      "activations/layer8_attention_weight_min": -45.43324661254883,
      "activations/layer9_attention_weight_max": 33.81574630737305,
      "activations/layer9_attention_weight_min": -35.241146087646484,
      "epoch": 15.54,
      "learning_rate": 5.0309469696969695e-05,
      "loss": 2.7397,
      "step": 267400
    },
    {
      "activations/layer0_attention_weight_max": 15.87721061706543,
      "activations/layer0_attention_weight_min": -12.522363662719727,
      "activations/layer10_attention_weight_max": 35.393592834472656,
      "activations/layer10_attention_weight_min": -35.416481018066406,
      "activations/layer11_attention_weight_max": 38.29290771484375,
      "activations/layer11_attention_weight_min": -36.42097473144531,
      "activations/layer12_attention_weight_max": 27.502212524414062,
      "activations/layer12_attention_weight_min": -27.79590606689453,
      "activations/layer13_attention_weight_max": 41.3239860534668,
      "activations/layer13_attention_weight_min": -39.84395980834961,
      "activations/layer14_attention_weight_max": 45.89569091796875,
      "activations/layer14_attention_weight_min": -42.49552917480469,
      "activations/layer15_attention_weight_max": 40.556358337402344,
      "activations/layer15_attention_weight_min": -35.81713104248047,
      "activations/layer16_attention_weight_max": 33.13614273071289,
      "activations/layer16_attention_weight_min": -28.779741287231445,
      "activations/layer17_attention_weight_max": 32.80811309814453,
      "activations/layer17_attention_weight_min": -27.22924041748047,
      "activations/layer18_attention_weight_max": 31.234275817871094,
      "activations/layer18_attention_weight_min": -24.51083755493164,
      "activations/layer19_attention_weight_max": 33.15121841430664,
      "activations/layer19_attention_weight_min": -32.962181091308594,
      "activations/layer1_attention_weight_max": 17.227619171142578,
      "activations/layer1_attention_weight_min": -15.98217487335205,
      "activations/layer20_attention_weight_max": 30.084423065185547,
      "activations/layer20_attention_weight_min": -25.166297912597656,
      "activations/layer21_attention_weight_max": 31.46335220336914,
      "activations/layer21_attention_weight_min": -27.24432373046875,
      "activations/layer22_attention_weight_max": 43.661102294921875,
      "activations/layer22_attention_weight_min": -30.987960815429688,
      "activations/layer23_attention_weight_max": 34.426998138427734,
      "activations/layer23_attention_weight_min": -27.206249237060547,
      "activations/layer2_attention_weight_max": 30.168973922729492,
      "activations/layer2_attention_weight_min": -31.92184829711914,
      "activations/layer3_attention_weight_max": 95.51942443847656,
      "activations/layer3_attention_weight_min": -101.69600677490234,
      "activations/layer4_attention_weight_max": 109.63592529296875,
      "activations/layer4_attention_weight_min": -117.88136291503906,
      "activations/layer5_attention_weight_max": 52.915000915527344,
      "activations/layer5_attention_weight_min": -62.99578857421875,
      "activations/layer6_attention_weight_max": 43.545982360839844,
      "activations/layer6_attention_weight_min": -47.39584732055664,
      "activations/layer7_attention_weight_max": 88.95152282714844,
      "activations/layer7_attention_weight_min": -93.12815856933594,
      "activations/layer8_attention_weight_max": 41.44826126098633,
      "activations/layer8_attention_weight_min": -45.76995849609375,
      "activations/layer9_attention_weight_max": 33.2431640625,
      "activations/layer9_attention_weight_min": -35.234554290771484,
      "epoch": 15.54,
      "learning_rate": 5.02905303030303e-05,
      "loss": 2.7448,
      "step": 267450
    },
    {
      "activations/layer0_attention_weight_max": 16.078319549560547,
      "activations/layer0_attention_weight_min": -12.291891098022461,
      "activations/layer10_attention_weight_max": 32.94554138183594,
      "activations/layer10_attention_weight_min": -33.016387939453125,
      "activations/layer11_attention_weight_max": 32.142784118652344,
      "activations/layer11_attention_weight_min": -33.193817138671875,
      "activations/layer12_attention_weight_max": 25.972328186035156,
      "activations/layer12_attention_weight_min": -25.822904586791992,
      "activations/layer13_attention_weight_max": 39.52046585083008,
      "activations/layer13_attention_weight_min": -37.05455780029297,
      "activations/layer14_attention_weight_max": 46.512939453125,
      "activations/layer14_attention_weight_min": -37.3623161315918,
      "activations/layer15_attention_weight_max": 42.80401611328125,
      "activations/layer15_attention_weight_min": -34.855289459228516,
      "activations/layer16_attention_weight_max": 33.92697525024414,
      "activations/layer16_attention_weight_min": -27.461933135986328,
      "activations/layer17_attention_weight_max": 33.51872634887695,
      "activations/layer17_attention_weight_min": -25.631553649902344,
      "activations/layer18_attention_weight_max": 33.91645812988281,
      "activations/layer18_attention_weight_min": -24.464452743530273,
      "activations/layer19_attention_weight_max": 35.486976623535156,
      "activations/layer19_attention_weight_min": -31.018951416015625,
      "activations/layer1_attention_weight_max": 15.741406440734863,
      "activations/layer1_attention_weight_min": -14.375588417053223,
      "activations/layer20_attention_weight_max": 30.381961822509766,
      "activations/layer20_attention_weight_min": -24.709138870239258,
      "activations/layer21_attention_weight_max": 28.92142677307129,
      "activations/layer21_attention_weight_min": -24.406539916992188,
      "activations/layer22_attention_weight_max": 43.48416519165039,
      "activations/layer22_attention_weight_min": -31.639097213745117,
      "activations/layer23_attention_weight_max": 33.397335052490234,
      "activations/layer23_attention_weight_min": -27.161865234375,
      "activations/layer2_attention_weight_max": 31.8150634765625,
      "activations/layer2_attention_weight_min": -31.666547775268555,
      "activations/layer3_attention_weight_max": 91.27243041992188,
      "activations/layer3_attention_weight_min": -94.57477569580078,
      "activations/layer4_attention_weight_max": 101.4811782836914,
      "activations/layer4_attention_weight_min": -107.4834976196289,
      "activations/layer5_attention_weight_max": 52.533668518066406,
      "activations/layer5_attention_weight_min": -64.01171875,
      "activations/layer6_attention_weight_max": 42.78426742553711,
      "activations/layer6_attention_weight_min": -45.450809478759766,
      "activations/layer7_attention_weight_max": 91.86888885498047,
      "activations/layer7_attention_weight_min": -91.4868392944336,
      "activations/layer8_attention_weight_max": 41.770172119140625,
      "activations/layer8_attention_weight_min": -45.412994384765625,
      "activations/layer9_attention_weight_max": 32.263397216796875,
      "activations/layer9_attention_weight_min": -33.190513610839844,
      "epoch": 15.54,
      "learning_rate": 5.0271590909090905e-05,
      "loss": 2.7223,
      "step": 267500
    },
    {
      "activations/layer0_attention_weight_max": 15.374520301818848,
      "activations/layer0_attention_weight_min": -12.007920265197754,
      "activations/layer10_attention_weight_max": 30.54509162902832,
      "activations/layer10_attention_weight_min": -31.869047164916992,
      "activations/layer11_attention_weight_max": 32.19176483154297,
      "activations/layer11_attention_weight_min": -31.54071617126465,
      "activations/layer12_attention_weight_max": 25.738483428955078,
      "activations/layer12_attention_weight_min": -25.796253204345703,
      "activations/layer13_attention_weight_max": 35.72010040283203,
      "activations/layer13_attention_weight_min": -35.15332794189453,
      "activations/layer14_attention_weight_max": 37.48964309692383,
      "activations/layer14_attention_weight_min": -36.493370056152344,
      "activations/layer15_attention_weight_max": 36.10390090942383,
      "activations/layer15_attention_weight_min": -33.948089599609375,
      "activations/layer16_attention_weight_max": 28.470056533813477,
      "activations/layer16_attention_weight_min": -27.63795280456543,
      "activations/layer17_attention_weight_max": 26.982479095458984,
      "activations/layer17_attention_weight_min": -26.775100708007812,
      "activations/layer18_attention_weight_max": 29.64337158203125,
      "activations/layer18_attention_weight_min": -25.50283432006836,
      "activations/layer19_attention_weight_max": 36.01627731323242,
      "activations/layer19_attention_weight_min": -33.46844482421875,
      "activations/layer1_attention_weight_max": 16.559640884399414,
      "activations/layer1_attention_weight_min": -15.577845573425293,
      "activations/layer20_attention_weight_max": 32.31232452392578,
      "activations/layer20_attention_weight_min": -25.9864559173584,
      "activations/layer21_attention_weight_max": 27.999279022216797,
      "activations/layer21_attention_weight_min": -27.532073974609375,
      "activations/layer22_attention_weight_max": 37.61209487915039,
      "activations/layer22_attention_weight_min": -31.309356689453125,
      "activations/layer23_attention_weight_max": 30.022748947143555,
      "activations/layer23_attention_weight_min": -26.709821701049805,
      "activations/layer2_attention_weight_max": 33.18954086303711,
      "activations/layer2_attention_weight_min": -33.4556999206543,
      "activations/layer3_attention_weight_max": 97.67063903808594,
      "activations/layer3_attention_weight_min": -98.64036560058594,
      "activations/layer4_attention_weight_max": 104.32879638671875,
      "activations/layer4_attention_weight_min": -107.40789794921875,
      "activations/layer5_attention_weight_max": 52.97332000732422,
      "activations/layer5_attention_weight_min": -63.79943084716797,
      "activations/layer6_attention_weight_max": 42.2260627746582,
      "activations/layer6_attention_weight_min": -46.28522491455078,
      "activations/layer7_attention_weight_max": 87.14627838134766,
      "activations/layer7_attention_weight_min": -88.46702575683594,
      "activations/layer8_attention_weight_max": 39.43351364135742,
      "activations/layer8_attention_weight_min": -41.353214263916016,
      "activations/layer9_attention_weight_max": 29.163150787353516,
      "activations/layer9_attention_weight_min": -30.800506591796875,
      "epoch": 15.55,
      "learning_rate": 5.025265151515151e-05,
      "loss": 2.7456,
      "step": 267550
    },
    {
      "activations/layer0_attention_weight_max": 16.166160583496094,
      "activations/layer0_attention_weight_min": -11.945490837097168,
      "activations/layer10_attention_weight_max": 34.5411376953125,
      "activations/layer10_attention_weight_min": -34.113311767578125,
      "activations/layer11_attention_weight_max": 34.28277587890625,
      "activations/layer11_attention_weight_min": -34.04698181152344,
      "activations/layer12_attention_weight_max": 29.80187225341797,
      "activations/layer12_attention_weight_min": -28.2110538482666,
      "activations/layer13_attention_weight_max": 41.11948776245117,
      "activations/layer13_attention_weight_min": -36.62843322753906,
      "activations/layer14_attention_weight_max": 41.91556930541992,
      "activations/layer14_attention_weight_min": -38.15465545654297,
      "activations/layer15_attention_weight_max": 41.141334533691406,
      "activations/layer15_attention_weight_min": -34.61638641357422,
      "activations/layer16_attention_weight_max": 29.185501098632812,
      "activations/layer16_attention_weight_min": -27.969831466674805,
      "activations/layer17_attention_weight_max": 31.878597259521484,
      "activations/layer17_attention_weight_min": -26.2507266998291,
      "activations/layer18_attention_weight_max": 30.544260025024414,
      "activations/layer18_attention_weight_min": -24.96303939819336,
      "activations/layer19_attention_weight_max": 31.799579620361328,
      "activations/layer19_attention_weight_min": -31.243698120117188,
      "activations/layer1_attention_weight_max": 17.30023956298828,
      "activations/layer1_attention_weight_min": -17.540271759033203,
      "activations/layer20_attention_weight_max": 26.73400115966797,
      "activations/layer20_attention_weight_min": -24.21625328063965,
      "activations/layer21_attention_weight_max": 25.73622703552246,
      "activations/layer21_attention_weight_min": -24.0322208404541,
      "activations/layer22_attention_weight_max": 39.503196716308594,
      "activations/layer22_attention_weight_min": -32.89424514770508,
      "activations/layer23_attention_weight_max": 31.582632064819336,
      "activations/layer23_attention_weight_min": -27.42233657836914,
      "activations/layer2_attention_weight_max": 33.19032287597656,
      "activations/layer2_attention_weight_min": -33.12165069580078,
      "activations/layer3_attention_weight_max": 93.5728530883789,
      "activations/layer3_attention_weight_min": -96.31208801269531,
      "activations/layer4_attention_weight_max": 101.24392700195312,
      "activations/layer4_attention_weight_min": -105.339111328125,
      "activations/layer5_attention_weight_max": 50.12357711791992,
      "activations/layer5_attention_weight_min": -57.704833984375,
      "activations/layer6_attention_weight_max": 42.734615325927734,
      "activations/layer6_attention_weight_min": -45.78811264038086,
      "activations/layer7_attention_weight_max": 94.93002319335938,
      "activations/layer7_attention_weight_min": -95.36956787109375,
      "activations/layer8_attention_weight_max": 40.33007049560547,
      "activations/layer8_attention_weight_min": -42.1700439453125,
      "activations/layer9_attention_weight_max": 31.323606491088867,
      "activations/layer9_attention_weight_min": -35.61225509643555,
      "epoch": 15.55,
      "learning_rate": 5.023371212121212e-05,
      "loss": 2.7528,
      "step": 267600
    },
    {
      "activations/layer0_attention_weight_max": 15.98882007598877,
      "activations/layer0_attention_weight_min": -12.44900131225586,
      "activations/layer10_attention_weight_max": 35.56084442138672,
      "activations/layer10_attention_weight_min": -34.762611389160156,
      "activations/layer11_attention_weight_max": 35.72336196899414,
      "activations/layer11_attention_weight_min": -36.88114547729492,
      "activations/layer12_attention_weight_max": 26.32942008972168,
      "activations/layer12_attention_weight_min": -26.312477111816406,
      "activations/layer13_attention_weight_max": 39.08715057373047,
      "activations/layer13_attention_weight_min": -35.31160354614258,
      "activations/layer14_attention_weight_max": 43.57835388183594,
      "activations/layer14_attention_weight_min": -39.60460662841797,
      "activations/layer15_attention_weight_max": 39.861610412597656,
      "activations/layer15_attention_weight_min": -35.44691467285156,
      "activations/layer16_attention_weight_max": 31.587984085083008,
      "activations/layer16_attention_weight_min": -29.16581153869629,
      "activations/layer17_attention_weight_max": 32.44071578979492,
      "activations/layer17_attention_weight_min": -25.996572494506836,
      "activations/layer18_attention_weight_max": 34.529388427734375,
      "activations/layer18_attention_weight_min": -25.564422607421875,
      "activations/layer19_attention_weight_max": 41.32766342163086,
      "activations/layer19_attention_weight_min": -32.12702560424805,
      "activations/layer1_attention_weight_max": 16.845579147338867,
      "activations/layer1_attention_weight_min": -16.045372009277344,
      "activations/layer20_attention_weight_max": 35.63188552856445,
      "activations/layer20_attention_weight_min": -25.551755905151367,
      "activations/layer21_attention_weight_max": 31.88597297668457,
      "activations/layer21_attention_weight_min": -24.88371467590332,
      "activations/layer22_attention_weight_max": 46.209808349609375,
      "activations/layer22_attention_weight_min": -32.176761627197266,
      "activations/layer23_attention_weight_max": 35.85529708862305,
      "activations/layer23_attention_weight_min": -27.988811492919922,
      "activations/layer2_attention_weight_max": 32.528656005859375,
      "activations/layer2_attention_weight_min": -33.90056610107422,
      "activations/layer3_attention_weight_max": 93.8307876586914,
      "activations/layer3_attention_weight_min": -96.60108947753906,
      "activations/layer4_attention_weight_max": 104.8626708984375,
      "activations/layer4_attention_weight_min": -106.392333984375,
      "activations/layer5_attention_weight_max": 54.513694763183594,
      "activations/layer5_attention_weight_min": -59.54779052734375,
      "activations/layer6_attention_weight_max": 44.97456741333008,
      "activations/layer6_attention_weight_min": -45.05277633666992,
      "activations/layer7_attention_weight_max": 91.30416870117188,
      "activations/layer7_attention_weight_min": -89.51817321777344,
      "activations/layer8_attention_weight_max": 40.03300857543945,
      "activations/layer8_attention_weight_min": -41.699317932128906,
      "activations/layer9_attention_weight_max": 32.3077392578125,
      "activations/layer9_attention_weight_min": -35.12363815307617,
      "epoch": 15.55,
      "learning_rate": 5.021477272727272e-05,
      "loss": 2.751,
      "step": 267650
    },
    {
      "activations/layer0_attention_weight_max": 16.15492820739746,
      "activations/layer0_attention_weight_min": -12.30582046508789,
      "activations/layer10_attention_weight_max": 34.7095947265625,
      "activations/layer10_attention_weight_min": -37.97967529296875,
      "activations/layer11_attention_weight_max": 36.421852111816406,
      "activations/layer11_attention_weight_min": -38.18585205078125,
      "activations/layer12_attention_weight_max": 32.6158332824707,
      "activations/layer12_attention_weight_min": -27.349496841430664,
      "activations/layer13_attention_weight_max": 42.582759857177734,
      "activations/layer13_attention_weight_min": -35.677608489990234,
      "activations/layer14_attention_weight_max": 43.34372329711914,
      "activations/layer14_attention_weight_min": -39.84779357910156,
      "activations/layer15_attention_weight_max": 38.935333251953125,
      "activations/layer15_attention_weight_min": -36.75811004638672,
      "activations/layer16_attention_weight_max": 32.41344451904297,
      "activations/layer16_attention_weight_min": -28.589073181152344,
      "activations/layer17_attention_weight_max": 32.79938507080078,
      "activations/layer17_attention_weight_min": -27.4963321685791,
      "activations/layer18_attention_weight_max": 31.631486892700195,
      "activations/layer18_attention_weight_min": -27.493703842163086,
      "activations/layer19_attention_weight_max": 35.50484085083008,
      "activations/layer19_attention_weight_min": -33.547794342041016,
      "activations/layer1_attention_weight_max": 17.473384857177734,
      "activations/layer1_attention_weight_min": -14.39271068572998,
      "activations/layer20_attention_weight_max": 32.48454666137695,
      "activations/layer20_attention_weight_min": -27.518369674682617,
      "activations/layer21_attention_weight_max": 32.15727233886719,
      "activations/layer21_attention_weight_min": -27.1909236907959,
      "activations/layer22_attention_weight_max": 41.516178131103516,
      "activations/layer22_attention_weight_min": -32.14342498779297,
      "activations/layer23_attention_weight_max": 35.077293395996094,
      "activations/layer23_attention_weight_min": -25.677963256835938,
      "activations/layer2_attention_weight_max": 33.515716552734375,
      "activations/layer2_attention_weight_min": -33.241695404052734,
      "activations/layer3_attention_weight_max": 96.30216979980469,
      "activations/layer3_attention_weight_min": -98.12718200683594,
      "activations/layer4_attention_weight_max": 106.39763641357422,
      "activations/layer4_attention_weight_min": -109.65811920166016,
      "activations/layer5_attention_weight_max": 52.40562438964844,
      "activations/layer5_attention_weight_min": -61.527442932128906,
      "activations/layer6_attention_weight_max": 45.650291442871094,
      "activations/layer6_attention_weight_min": -45.48033905029297,
      "activations/layer7_attention_weight_max": 92.99761199951172,
      "activations/layer7_attention_weight_min": -96.12346649169922,
      "activations/layer8_attention_weight_max": 41.50260543823242,
      "activations/layer8_attention_weight_min": -45.7485237121582,
      "activations/layer9_attention_weight_max": 33.75707244873047,
      "activations/layer9_attention_weight_min": -36.85331344604492,
      "epoch": 15.55,
      "learning_rate": 5.0195833333333325e-05,
      "loss": 2.7577,
      "step": 267700
    },
    {
      "activations/layer0_attention_weight_max": 16.156389236450195,
      "activations/layer0_attention_weight_min": -12.99704647064209,
      "activations/layer10_attention_weight_max": 36.2698860168457,
      "activations/layer10_attention_weight_min": -35.85034942626953,
      "activations/layer11_attention_weight_max": 36.540225982666016,
      "activations/layer11_attention_weight_min": -37.0074462890625,
      "activations/layer12_attention_weight_max": 27.90658950805664,
      "activations/layer12_attention_weight_min": -27.553903579711914,
      "activations/layer13_attention_weight_max": 42.738304138183594,
      "activations/layer13_attention_weight_min": -35.62446594238281,
      "activations/layer14_attention_weight_max": 42.39556121826172,
      "activations/layer14_attention_weight_min": -38.67918395996094,
      "activations/layer15_attention_weight_max": 37.42226028442383,
      "activations/layer15_attention_weight_min": -35.28060531616211,
      "activations/layer16_attention_weight_max": 29.612903594970703,
      "activations/layer16_attention_weight_min": -27.257965087890625,
      "activations/layer17_attention_weight_max": 31.218069076538086,
      "activations/layer17_attention_weight_min": -26.090755462646484,
      "activations/layer18_attention_weight_max": 29.21525764465332,
      "activations/layer18_attention_weight_min": -25.314109802246094,
      "activations/layer19_attention_weight_max": 32.59250259399414,
      "activations/layer19_attention_weight_min": -30.357934951782227,
      "activations/layer1_attention_weight_max": 17.87575340270996,
      "activations/layer1_attention_weight_min": -16.67827606201172,
      "activations/layer20_attention_weight_max": 28.43413734436035,
      "activations/layer20_attention_weight_min": -23.519140243530273,
      "activations/layer21_attention_weight_max": 28.119186401367188,
      "activations/layer21_attention_weight_min": -24.490921020507812,
      "activations/layer22_attention_weight_max": 39.49924850463867,
      "activations/layer22_attention_weight_min": -30.18239402770996,
      "activations/layer23_attention_weight_max": 32.17361831665039,
      "activations/layer23_attention_weight_min": -24.60426902770996,
      "activations/layer2_attention_weight_max": 37.148075103759766,
      "activations/layer2_attention_weight_min": -35.457359313964844,
      "activations/layer3_attention_weight_max": 102.82421112060547,
      "activations/layer3_attention_weight_min": -100.17316436767578,
      "activations/layer4_attention_weight_max": 113.657958984375,
      "activations/layer4_attention_weight_min": -110.63319396972656,
      "activations/layer5_attention_weight_max": 54.121971130371094,
      "activations/layer5_attention_weight_min": -61.634761810302734,
      "activations/layer6_attention_weight_max": 47.5704460144043,
      "activations/layer6_attention_weight_min": -46.87828826904297,
      "activations/layer7_attention_weight_max": 96.73309326171875,
      "activations/layer7_attention_weight_min": -97.32595825195312,
      "activations/layer8_attention_weight_max": 44.658042907714844,
      "activations/layer8_attention_weight_min": -45.682193756103516,
      "activations/layer9_attention_weight_max": 35.93760299682617,
      "activations/layer9_attention_weight_min": -36.45487976074219,
      "epoch": 15.56,
      "learning_rate": 5.017689393939394e-05,
      "loss": 2.7521,
      "step": 267750
    },
    {
      "activations/layer0_attention_weight_max": 15.658225059509277,
      "activations/layer0_attention_weight_min": -13.234894752502441,
      "activations/layer10_attention_weight_max": 33.119285583496094,
      "activations/layer10_attention_weight_min": -33.17100524902344,
      "activations/layer11_attention_weight_max": 36.524200439453125,
      "activations/layer11_attention_weight_min": -34.22254180908203,
      "activations/layer12_attention_weight_max": 26.51133918762207,
      "activations/layer12_attention_weight_min": -24.9658203125,
      "activations/layer13_attention_weight_max": 37.43185043334961,
      "activations/layer13_attention_weight_min": -34.353355407714844,
      "activations/layer14_attention_weight_max": 41.430503845214844,
      "activations/layer14_attention_weight_min": -37.91624450683594,
      "activations/layer15_attention_weight_max": 38.507110595703125,
      "activations/layer15_attention_weight_min": -36.26587677001953,
      "activations/layer16_attention_weight_max": 32.134429931640625,
      "activations/layer16_attention_weight_min": -28.740556716918945,
      "activations/layer17_attention_weight_max": 32.43780517578125,
      "activations/layer17_attention_weight_min": -27.472684860229492,
      "activations/layer18_attention_weight_max": 35.23236083984375,
      "activations/layer18_attention_weight_min": -25.390539169311523,
      "activations/layer19_attention_weight_max": 37.6173210144043,
      "activations/layer19_attention_weight_min": -33.37948226928711,
      "activations/layer1_attention_weight_max": 16.119909286499023,
      "activations/layer1_attention_weight_min": -14.559381484985352,
      "activations/layer20_attention_weight_max": 31.831340789794922,
      "activations/layer20_attention_weight_min": -26.71635627746582,
      "activations/layer21_attention_weight_max": 31.046171188354492,
      "activations/layer21_attention_weight_min": -26.090139389038086,
      "activations/layer22_attention_weight_max": 46.0744743347168,
      "activations/layer22_attention_weight_min": -32.50444030761719,
      "activations/layer23_attention_weight_max": 35.553260803222656,
      "activations/layer23_attention_weight_min": -28.804561614990234,
      "activations/layer2_attention_weight_max": 32.517059326171875,
      "activations/layer2_attention_weight_min": -32.963584899902344,
      "activations/layer3_attention_weight_max": 97.79317474365234,
      "activations/layer3_attention_weight_min": -99.82383728027344,
      "activations/layer4_attention_weight_max": 105.2205581665039,
      "activations/layer4_attention_weight_min": -106.4026107788086,
      "activations/layer5_attention_weight_max": 51.863121032714844,
      "activations/layer5_attention_weight_min": -63.7054557800293,
      "activations/layer6_attention_weight_max": 44.78690719604492,
      "activations/layer6_attention_weight_min": -44.73527526855469,
      "activations/layer7_attention_weight_max": 95.79656982421875,
      "activations/layer7_attention_weight_min": -90.29942321777344,
      "activations/layer8_attention_weight_max": 42.605831146240234,
      "activations/layer8_attention_weight_min": -42.44062423706055,
      "activations/layer9_attention_weight_max": 32.70027542114258,
      "activations/layer9_attention_weight_min": -35.425506591796875,
      "epoch": 15.56,
      "learning_rate": 5.015795454545454e-05,
      "loss": 2.7574,
      "step": 267800
    },
    {
      "activations/layer0_attention_weight_max": 15.681046485900879,
      "activations/layer0_attention_weight_min": -13.05871295928955,
      "activations/layer10_attention_weight_max": 32.022483825683594,
      "activations/layer10_attention_weight_min": -34.844505310058594,
      "activations/layer11_attention_weight_max": 31.06719207763672,
      "activations/layer11_attention_weight_min": -31.427595138549805,
      "activations/layer12_attention_weight_max": 24.64714241027832,
      "activations/layer12_attention_weight_min": -25.040546417236328,
      "activations/layer13_attention_weight_max": 37.69533157348633,
      "activations/layer13_attention_weight_min": -34.87162399291992,
      "activations/layer14_attention_weight_max": 42.533023834228516,
      "activations/layer14_attention_weight_min": -38.789554595947266,
      "activations/layer15_attention_weight_max": 36.37995529174805,
      "activations/layer15_attention_weight_min": -33.73200607299805,
      "activations/layer16_attention_weight_max": 28.85453987121582,
      "activations/layer16_attention_weight_min": -27.240951538085938,
      "activations/layer17_attention_weight_max": 27.319561004638672,
      "activations/layer17_attention_weight_min": -27.51134490966797,
      "activations/layer18_attention_weight_max": 28.632259368896484,
      "activations/layer18_attention_weight_min": -26.819950103759766,
      "activations/layer19_attention_weight_max": 33.87766647338867,
      "activations/layer19_attention_weight_min": -29.57730484008789,
      "activations/layer1_attention_weight_max": 15.740087509155273,
      "activations/layer1_attention_weight_min": -14.963056564331055,
      "activations/layer20_attention_weight_max": 28.703140258789062,
      "activations/layer20_attention_weight_min": -24.30099105834961,
      "activations/layer21_attention_weight_max": 27.736186981201172,
      "activations/layer21_attention_weight_min": -23.621671676635742,
      "activations/layer22_attention_weight_max": 39.456626892089844,
      "activations/layer22_attention_weight_min": -28.834409713745117,
      "activations/layer23_attention_weight_max": 32.10123062133789,
      "activations/layer23_attention_weight_min": -24.638776779174805,
      "activations/layer2_attention_weight_max": 32.233726501464844,
      "activations/layer2_attention_weight_min": -32.8253173828125,
      "activations/layer3_attention_weight_max": 94.23149108886719,
      "activations/layer3_attention_weight_min": -95.73033905029297,
      "activations/layer4_attention_weight_max": 101.97073364257812,
      "activations/layer4_attention_weight_min": -107.10103607177734,
      "activations/layer5_attention_weight_max": 52.90629577636719,
      "activations/layer5_attention_weight_min": -60.40544891357422,
      "activations/layer6_attention_weight_max": 41.627410888671875,
      "activations/layer6_attention_weight_min": -46.037086486816406,
      "activations/layer7_attention_weight_max": 85.94332122802734,
      "activations/layer7_attention_weight_min": -97.09244537353516,
      "activations/layer8_attention_weight_max": 38.85260009765625,
      "activations/layer8_attention_weight_min": -42.34109878540039,
      "activations/layer9_attention_weight_max": 31.781707763671875,
      "activations/layer9_attention_weight_min": -34.185333251953125,
      "epoch": 15.56,
      "learning_rate": 5.013939393939393e-05,
      "loss": 2.7492,
      "step": 267850
    },
    {
      "activations/layer0_attention_weight_max": 15.907173156738281,
      "activations/layer0_attention_weight_min": -12.625953674316406,
      "activations/layer10_attention_weight_max": 31.420955657958984,
      "activations/layer10_attention_weight_min": -33.23652648925781,
      "activations/layer11_attention_weight_max": 33.72026062011719,
      "activations/layer11_attention_weight_min": -33.36897277832031,
      "activations/layer12_attention_weight_max": 25.282480239868164,
      "activations/layer12_attention_weight_min": -25.515531539916992,
      "activations/layer13_attention_weight_max": 39.33730697631836,
      "activations/layer13_attention_weight_min": -34.33967208862305,
      "activations/layer14_attention_weight_max": 44.547813415527344,
      "activations/layer14_attention_weight_min": -40.21344757080078,
      "activations/layer15_attention_weight_max": 39.186466217041016,
      "activations/layer15_attention_weight_min": -33.10972595214844,
      "activations/layer16_attention_weight_max": 32.88765335083008,
      "activations/layer16_attention_weight_min": -26.544166564941406,
      "activations/layer17_attention_weight_max": 33.66122817993164,
      "activations/layer17_attention_weight_min": -24.71756362915039,
      "activations/layer18_attention_weight_max": 35.46383285522461,
      "activations/layer18_attention_weight_min": -24.796239852905273,
      "activations/layer19_attention_weight_max": 36.56275177001953,
      "activations/layer19_attention_weight_min": -29.0456485748291,
      "activations/layer1_attention_weight_max": 16.482980728149414,
      "activations/layer1_attention_weight_min": -14.59045124053955,
      "activations/layer20_attention_weight_max": 29.788516998291016,
      "activations/layer20_attention_weight_min": -23.542129516601562,
      "activations/layer21_attention_weight_max": 30.451274871826172,
      "activations/layer21_attention_weight_min": -24.128591537475586,
      "activations/layer22_attention_weight_max": 40.038326263427734,
      "activations/layer22_attention_weight_min": -29.250003814697266,
      "activations/layer23_attention_weight_max": 32.614967346191406,
      "activations/layer23_attention_weight_min": -25.43701934814453,
      "activations/layer2_attention_weight_max": 33.64073944091797,
      "activations/layer2_attention_weight_min": -32.64270782470703,
      "activations/layer3_attention_weight_max": 97.75946807861328,
      "activations/layer3_attention_weight_min": -97.89464569091797,
      "activations/layer4_attention_weight_max": 105.4432601928711,
      "activations/layer4_attention_weight_min": -106.6292953491211,
      "activations/layer5_attention_weight_max": 52.851375579833984,
      "activations/layer5_attention_weight_min": -60.512367248535156,
      "activations/layer6_attention_weight_max": 42.60584259033203,
      "activations/layer6_attention_weight_min": -42.56834411621094,
      "activations/layer7_attention_weight_max": 83.75405883789062,
      "activations/layer7_attention_weight_min": -86.99419403076172,
      "activations/layer8_attention_weight_max": 39.991729736328125,
      "activations/layer8_attention_weight_min": -40.89554214477539,
      "activations/layer9_attention_weight_max": 30.91006851196289,
      "activations/layer9_attention_weight_min": -33.319766998291016,
      "epoch": 15.57,
      "learning_rate": 5.012045454545454e-05,
      "loss": 2.7504,
      "step": 267900
    },
    {
      "activations/layer0_attention_weight_max": 15.84341049194336,
      "activations/layer0_attention_weight_min": -13.766104698181152,
      "activations/layer10_attention_weight_max": 38.62751388549805,
      "activations/layer10_attention_weight_min": -34.185760498046875,
      "activations/layer11_attention_weight_max": 36.440887451171875,
      "activations/layer11_attention_weight_min": -35.92029571533203,
      "activations/layer12_attention_weight_max": 28.241580963134766,
      "activations/layer12_attention_weight_min": -26.696062088012695,
      "activations/layer13_attention_weight_max": 40.25275802612305,
      "activations/layer13_attention_weight_min": -34.604736328125,
      "activations/layer14_attention_weight_max": 44.3028450012207,
      "activations/layer14_attention_weight_min": -37.973960876464844,
      "activations/layer15_attention_weight_max": 42.64591979980469,
      "activations/layer15_attention_weight_min": -34.98625183105469,
      "activations/layer16_attention_weight_max": 32.09046936035156,
      "activations/layer16_attention_weight_min": -27.181007385253906,
      "activations/layer17_attention_weight_max": 33.91178512573242,
      "activations/layer17_attention_weight_min": -26.4478759765625,
      "activations/layer18_attention_weight_max": 33.648658752441406,
      "activations/layer18_attention_weight_min": -27.2214412689209,
      "activations/layer19_attention_weight_max": 35.424007415771484,
      "activations/layer19_attention_weight_min": -30.48563575744629,
      "activations/layer1_attention_weight_max": 17.251323699951172,
      "activations/layer1_attention_weight_min": -15.217735290527344,
      "activations/layer20_attention_weight_max": 31.606477737426758,
      "activations/layer20_attention_weight_min": -24.460376739501953,
      "activations/layer21_attention_weight_max": 31.16075325012207,
      "activations/layer21_attention_weight_min": -23.93100929260254,
      "activations/layer22_attention_weight_max": 49.442543029785156,
      "activations/layer22_attention_weight_min": -29.993255615234375,
      "activations/layer23_attention_weight_max": 38.22407531738281,
      "activations/layer23_attention_weight_min": -26.947948455810547,
      "activations/layer2_attention_weight_max": 32.9860954284668,
      "activations/layer2_attention_weight_min": -34.53776931762695,
      "activations/layer3_attention_weight_max": 93.48734283447266,
      "activations/layer3_attention_weight_min": -93.38892364501953,
      "activations/layer4_attention_weight_max": 104.53865814208984,
      "activations/layer4_attention_weight_min": -102.69400787353516,
      "activations/layer5_attention_weight_max": 52.02355194091797,
      "activations/layer5_attention_weight_min": -62.90029525756836,
      "activations/layer6_attention_weight_max": 41.11662673950195,
      "activations/layer6_attention_weight_min": -41.67311477661133,
      "activations/layer7_attention_weight_max": 96.83521270751953,
      "activations/layer7_attention_weight_min": -91.1214370727539,
      "activations/layer8_attention_weight_max": 43.98830032348633,
      "activations/layer8_attention_weight_min": -43.12171936035156,
      "activations/layer9_attention_weight_max": 34.04240036010742,
      "activations/layer9_attention_weight_min": -34.28345489501953,
      "epoch": 15.57,
      "learning_rate": 5.010151515151515e-05,
      "loss": 2.7384,
      "step": 267950
    },
    {
      "activations/layer0_attention_weight_max": 15.682611465454102,
      "activations/layer0_attention_weight_min": -12.468605995178223,
      "activations/layer10_attention_weight_max": 38.40774154663086,
      "activations/layer10_attention_weight_min": -39.852176666259766,
      "activations/layer11_attention_weight_max": 39.35340118408203,
      "activations/layer11_attention_weight_min": -38.12327575683594,
      "activations/layer12_attention_weight_max": 25.572402954101562,
      "activations/layer12_attention_weight_min": -29.77576446533203,
      "activations/layer13_attention_weight_max": 37.03306579589844,
      "activations/layer13_attention_weight_min": -38.87419509887695,
      "activations/layer14_attention_weight_max": 42.49516296386719,
      "activations/layer14_attention_weight_min": -44.49225616455078,
      "activations/layer15_attention_weight_max": 38.026023864746094,
      "activations/layer15_attention_weight_min": -40.07941818237305,
      "activations/layer16_attention_weight_max": 29.169723510742188,
      "activations/layer16_attention_weight_min": -28.13828468322754,
      "activations/layer17_attention_weight_max": 27.78763198852539,
      "activations/layer17_attention_weight_min": -27.790002822875977,
      "activations/layer18_attention_weight_max": 30.638334274291992,
      "activations/layer18_attention_weight_min": -28.389705657958984,
      "activations/layer19_attention_weight_max": 33.26069259643555,
      "activations/layer19_attention_weight_min": -30.598587036132812,
      "activations/layer1_attention_weight_max": 17.13702964782715,
      "activations/layer1_attention_weight_min": -16.727725982666016,
      "activations/layer20_attention_weight_max": 28.112075805664062,
      "activations/layer20_attention_weight_min": -25.106266021728516,
      "activations/layer21_attention_weight_max": 27.57560920715332,
      "activations/layer21_attention_weight_min": -24.264822006225586,
      "activations/layer22_attention_weight_max": 41.529937744140625,
      "activations/layer22_attention_weight_min": -32.37749099731445,
      "activations/layer23_attention_weight_max": 33.521209716796875,
      "activations/layer23_attention_weight_min": -26.190750122070312,
      "activations/layer2_attention_weight_max": 32.07524490356445,
      "activations/layer2_attention_weight_min": -33.76978302001953,
      "activations/layer3_attention_weight_max": 94.3721923828125,
      "activations/layer3_attention_weight_min": -99.26112365722656,
      "activations/layer4_attention_weight_max": 102.42317199707031,
      "activations/layer4_attention_weight_min": -109.08821868896484,
      "activations/layer5_attention_weight_max": 52.37264633178711,
      "activations/layer5_attention_weight_min": -56.7336540222168,
      "activations/layer6_attention_weight_max": 45.61681365966797,
      "activations/layer6_attention_weight_min": -46.37905502319336,
      "activations/layer7_attention_weight_max": 94.05150604248047,
      "activations/layer7_attention_weight_min": -94.46652221679688,
      "activations/layer8_attention_weight_max": 44.2601203918457,
      "activations/layer8_attention_weight_min": -46.1837043762207,
      "activations/layer9_attention_weight_max": 36.82481384277344,
      "activations/layer9_attention_weight_min": -39.675994873046875,
      "epoch": 15.57,
      "learning_rate": 5.008257575757576e-05,
      "loss": 2.7525,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4706,
      "eval_samples_per_second": 506.932,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4706,
      "eval_openwebtext_samples_per_second": 506.932,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9336,
      "eval_wikitext_samples_per_second": 235.83,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_lambada_loss": 2.42578125,
      "eval_lambada_ppl": 11.311062736078481,
      "eval_lambada_runtime": 9.5563,
      "eval_lambada_samples_per_second": 509.506,
      "step": 268000
    },
    {
      "activations/layer0_attention_weight_max": 16.471134185791016,
      "activations/layer0_attention_weight_min": -13.832954406738281,
      "activations/layer10_attention_weight_max": 33.27836990356445,
      "activations/layer10_attention_weight_min": -35.45387649536133,
      "activations/layer11_attention_weight_max": 33.644309997558594,
      "activations/layer11_attention_weight_min": -34.75129699707031,
      "activations/layer12_attention_weight_max": 26.529685974121094,
      "activations/layer12_attention_weight_min": -25.67210578918457,
      "activations/layer13_attention_weight_max": 41.12307357788086,
      "activations/layer13_attention_weight_min": -36.178035736083984,
      "activations/layer14_attention_weight_max": 43.19590759277344,
      "activations/layer14_attention_weight_min": -38.31792449951172,
      "activations/layer15_attention_weight_max": 39.874454498291016,
      "activations/layer15_attention_weight_min": -34.7594108581543,
      "activations/layer16_attention_weight_max": 29.34803581237793,
      "activations/layer16_attention_weight_min": -26.705060958862305,
      "activations/layer17_attention_weight_max": 30.238128662109375,
      "activations/layer17_attention_weight_min": -26.017459869384766,
      "activations/layer18_attention_weight_max": 30.793851852416992,
      "activations/layer18_attention_weight_min": -24.557973861694336,
      "activations/layer19_attention_weight_max": 33.92774200439453,
      "activations/layer19_attention_weight_min": -30.928564071655273,
      "activations/layer1_attention_weight_max": 16.811847686767578,
      "activations/layer1_attention_weight_min": -17.241220474243164,
      "activations/layer20_attention_weight_max": 30.515300750732422,
      "activations/layer20_attention_weight_min": -26.180564880371094,
      "activations/layer21_attention_weight_max": 28.67619514465332,
      "activations/layer21_attention_weight_min": -24.574951171875,
      "activations/layer22_attention_weight_max": 42.49064254760742,
      "activations/layer22_attention_weight_min": -32.27670669555664,
      "activations/layer23_attention_weight_max": 32.28102111816406,
      "activations/layer23_attention_weight_min": -28.0509090423584,
      "activations/layer2_attention_weight_max": 33.60397720336914,
      "activations/layer2_attention_weight_min": -35.19635009765625,
      "activations/layer3_attention_weight_max": 97.84388732910156,
      "activations/layer3_attention_weight_min": -102.23892211914062,
      "activations/layer4_attention_weight_max": 108.82164764404297,
      "activations/layer4_attention_weight_min": -113.74751281738281,
      "activations/layer5_attention_weight_max": 50.996437072753906,
      "activations/layer5_attention_weight_min": -65.6434097290039,
      "activations/layer6_attention_weight_max": 44.41102600097656,
      "activations/layer6_attention_weight_min": -44.665550231933594,
      "activations/layer7_attention_weight_max": 89.8303451538086,
      "activations/layer7_attention_weight_min": -94.7740707397461,
      "activations/layer8_attention_weight_max": 40.92764663696289,
      "activations/layer8_attention_weight_min": -43.19390106201172,
      "activations/layer9_attention_weight_max": 33.85403060913086,
      "activations/layer9_attention_weight_min": -34.4069709777832,
      "epoch": 15.58,
      "learning_rate": 5.006363636363636e-05,
      "loss": 2.7431,
      "step": 268050
    },
    {
      "activations/layer0_attention_weight_max": 16.154415130615234,
      "activations/layer0_attention_weight_min": -12.23628044128418,
      "activations/layer10_attention_weight_max": 34.030967712402344,
      "activations/layer10_attention_weight_min": -32.87998962402344,
      "activations/layer11_attention_weight_max": 33.48160171508789,
      "activations/layer11_attention_weight_min": -34.25419616699219,
      "activations/layer12_attention_weight_max": 28.26246452331543,
      "activations/layer12_attention_weight_min": -27.89596939086914,
      "activations/layer13_attention_weight_max": 42.80950164794922,
      "activations/layer13_attention_weight_min": -37.70150375366211,
      "activations/layer14_attention_weight_max": 44.71197509765625,
      "activations/layer14_attention_weight_min": -41.85696029663086,
      "activations/layer15_attention_weight_max": 38.44565963745117,
      "activations/layer15_attention_weight_min": -35.619197845458984,
      "activations/layer16_attention_weight_max": 31.49302864074707,
      "activations/layer16_attention_weight_min": -30.528844833374023,
      "activations/layer17_attention_weight_max": 30.97066307067871,
      "activations/layer17_attention_weight_min": -26.32276153564453,
      "activations/layer18_attention_weight_max": 32.617862701416016,
      "activations/layer18_attention_weight_min": -25.378515243530273,
      "activations/layer19_attention_weight_max": 42.56733322143555,
      "activations/layer19_attention_weight_min": -31.980905532836914,
      "activations/layer1_attention_weight_max": 16.245811462402344,
      "activations/layer1_attention_weight_min": -15.36879825592041,
      "activations/layer20_attention_weight_max": 33.24740982055664,
      "activations/layer20_attention_weight_min": -25.866676330566406,
      "activations/layer21_attention_weight_max": 31.568822860717773,
      "activations/layer21_attention_weight_min": -25.157073974609375,
      "activations/layer22_attention_weight_max": 47.5328254699707,
      "activations/layer22_attention_weight_min": -32.520713806152344,
      "activations/layer23_attention_weight_max": 37.34192657470703,
      "activations/layer23_attention_weight_min": -27.112773895263672,
      "activations/layer2_attention_weight_max": 34.10131072998047,
      "activations/layer2_attention_weight_min": -34.10929870605469,
      "activations/layer3_attention_weight_max": 95.76232147216797,
      "activations/layer3_attention_weight_min": -98.86213684082031,
      "activations/layer4_attention_weight_max": 107.35161590576172,
      "activations/layer4_attention_weight_min": -105.9133071899414,
      "activations/layer5_attention_weight_max": 51.808509826660156,
      "activations/layer5_attention_weight_min": -60.07353210449219,
      "activations/layer6_attention_weight_max": 45.146873474121094,
      "activations/layer6_attention_weight_min": -46.06053161621094,
      "activations/layer7_attention_weight_max": 94.42183685302734,
      "activations/layer7_attention_weight_min": -90.32403564453125,
      "activations/layer8_attention_weight_max": 42.37904357910156,
      "activations/layer8_attention_weight_min": -43.9005241394043,
      "activations/layer9_attention_weight_max": 33.8010368347168,
      "activations/layer9_attention_weight_min": -34.93197250366211,
      "epoch": 15.58,
      "learning_rate": 5.004469696969696e-05,
      "loss": 2.7574,
      "step": 268100
    },
    {
      "activations/layer0_attention_weight_max": 15.969581604003906,
      "activations/layer0_attention_weight_min": -11.575013160705566,
      "activations/layer10_attention_weight_max": 46.03675079345703,
      "activations/layer10_attention_weight_min": -40.08967208862305,
      "activations/layer11_attention_weight_max": 47.9801025390625,
      "activations/layer11_attention_weight_min": -41.59943771362305,
      "activations/layer12_attention_weight_max": 30.658550262451172,
      "activations/layer12_attention_weight_min": -29.212060928344727,
      "activations/layer13_attention_weight_max": 46.33737564086914,
      "activations/layer13_attention_weight_min": -40.47312927246094,
      "activations/layer14_attention_weight_max": 52.49033737182617,
      "activations/layer14_attention_weight_min": -45.84245300292969,
      "activations/layer15_attention_weight_max": 46.074546813964844,
      "activations/layer15_attention_weight_min": -39.00058364868164,
      "activations/layer16_attention_weight_max": 35.71659851074219,
      "activations/layer16_attention_weight_min": -30.06422996520996,
      "activations/layer17_attention_weight_max": 35.62672424316406,
      "activations/layer17_attention_weight_min": -28.016040802001953,
      "activations/layer18_attention_weight_max": 36.74445343017578,
      "activations/layer18_attention_weight_min": -28.235740661621094,
      "activations/layer19_attention_weight_max": 47.622520446777344,
      "activations/layer19_attention_weight_min": -33.471832275390625,
      "activations/layer1_attention_weight_max": 16.045795440673828,
      "activations/layer1_attention_weight_min": -15.452337265014648,
      "activations/layer20_attention_weight_max": 37.47514343261719,
      "activations/layer20_attention_weight_min": -25.398454666137695,
      "activations/layer21_attention_weight_max": 36.56856918334961,
      "activations/layer21_attention_weight_min": -24.613750457763672,
      "activations/layer22_attention_weight_max": 48.209617614746094,
      "activations/layer22_attention_weight_min": -33.52437973022461,
      "activations/layer23_attention_weight_max": 37.449676513671875,
      "activations/layer23_attention_weight_min": -26.028583526611328,
      "activations/layer2_attention_weight_max": 33.997955322265625,
      "activations/layer2_attention_weight_min": -34.39442443847656,
      "activations/layer3_attention_weight_max": 100.21977996826172,
      "activations/layer3_attention_weight_min": -98.71379852294922,
      "activations/layer4_attention_weight_max": 109.78026580810547,
      "activations/layer4_attention_weight_min": -104.76283264160156,
      "activations/layer5_attention_weight_max": 52.84657287597656,
      "activations/layer5_attention_weight_min": -62.582115173339844,
      "activations/layer6_attention_weight_max": 46.70264434814453,
      "activations/layer6_attention_weight_min": -46.74751663208008,
      "activations/layer7_attention_weight_max": 102.74820709228516,
      "activations/layer7_attention_weight_min": -93.38282012939453,
      "activations/layer8_attention_weight_max": 50.31386184692383,
      "activations/layer8_attention_weight_min": -46.32881546020508,
      "activations/layer9_attention_weight_max": 42.35908889770508,
      "activations/layer9_attention_weight_min": -39.18842697143555,
      "epoch": 15.58,
      "learning_rate": 5.0025757575757575e-05,
      "loss": 2.7502,
      "step": 268150
    },
    {
      "activations/layer0_attention_weight_max": 16.13764762878418,
      "activations/layer0_attention_weight_min": -13.09674072265625,
      "activations/layer10_attention_weight_max": 34.09696578979492,
      "activations/layer10_attention_weight_min": -35.040348052978516,
      "activations/layer11_attention_weight_max": 33.141357421875,
      "activations/layer11_attention_weight_min": -34.03802490234375,
      "activations/layer12_attention_weight_max": 26.431055068969727,
      "activations/layer12_attention_weight_min": -27.172958374023438,
      "activations/layer13_attention_weight_max": 39.741943359375,
      "activations/layer13_attention_weight_min": -37.02796173095703,
      "activations/layer14_attention_weight_max": 40.322410583496094,
      "activations/layer14_attention_weight_min": -38.72272872924805,
      "activations/layer15_attention_weight_max": 36.2469596862793,
      "activations/layer15_attention_weight_min": -36.04618453979492,
      "activations/layer16_attention_weight_max": 29.530921936035156,
      "activations/layer16_attention_weight_min": -27.984905242919922,
      "activations/layer17_attention_weight_max": 31.74372673034668,
      "activations/layer17_attention_weight_min": -27.19620704650879,
      "activations/layer18_attention_weight_max": 32.5274543762207,
      "activations/layer18_attention_weight_min": -24.997222900390625,
      "activations/layer19_attention_weight_max": 34.76416015625,
      "activations/layer19_attention_weight_min": -32.04213333129883,
      "activations/layer1_attention_weight_max": 16.12674903869629,
      "activations/layer1_attention_weight_min": -15.672628402709961,
      "activations/layer20_attention_weight_max": 31.9567813873291,
      "activations/layer20_attention_weight_min": -24.877605438232422,
      "activations/layer21_attention_weight_max": 32.62036895751953,
      "activations/layer21_attention_weight_min": -24.885881423950195,
      "activations/layer22_attention_weight_max": 42.40503692626953,
      "activations/layer22_attention_weight_min": -30.243318557739258,
      "activations/layer23_attention_weight_max": 32.38261032104492,
      "activations/layer23_attention_weight_min": -24.839576721191406,
      "activations/layer2_attention_weight_max": 34.931243896484375,
      "activations/layer2_attention_weight_min": -35.214115142822266,
      "activations/layer3_attention_weight_max": 97.85942077636719,
      "activations/layer3_attention_weight_min": -101.13811492919922,
      "activations/layer4_attention_weight_max": 111.03267669677734,
      "activations/layer4_attention_weight_min": -116.072265625,
      "activations/layer5_attention_weight_max": 55.11212921142578,
      "activations/layer5_attention_weight_min": -65.02539825439453,
      "activations/layer6_attention_weight_max": 48.03395462036133,
      "activations/layer6_attention_weight_min": -52.05718231201172,
      "activations/layer7_attention_weight_max": 95.67140197753906,
      "activations/layer7_attention_weight_min": -103.78887176513672,
      "activations/layer8_attention_weight_max": 44.672119140625,
      "activations/layer8_attention_weight_min": -48.16522216796875,
      "activations/layer9_attention_weight_max": 32.242122650146484,
      "activations/layer9_attention_weight_min": -35.907039642333984,
      "epoch": 15.58,
      "learning_rate": 5.0007196969696966e-05,
      "loss": 2.7397,
      "step": 268200
    },
    {
      "activations/layer0_attention_weight_max": 16.40113067626953,
      "activations/layer0_attention_weight_min": -12.3194580078125,
      "activations/layer10_attention_weight_max": 32.28956604003906,
      "activations/layer10_attention_weight_min": -33.893707275390625,
      "activations/layer11_attention_weight_max": 32.46236038208008,
      "activations/layer11_attention_weight_min": -34.733482360839844,
      "activations/layer12_attention_weight_max": 25.435461044311523,
      "activations/layer12_attention_weight_min": -26.790695190429688,
      "activations/layer13_attention_weight_max": 36.76213073730469,
      "activations/layer13_attention_weight_min": -34.48127365112305,
      "activations/layer14_attention_weight_max": 40.825889587402344,
      "activations/layer14_attention_weight_min": -37.767478942871094,
      "activations/layer15_attention_weight_max": 36.583099365234375,
      "activations/layer15_attention_weight_min": -35.72149658203125,
      "activations/layer16_attention_weight_max": 31.897045135498047,
      "activations/layer16_attention_weight_min": -28.70701789855957,
      "activations/layer17_attention_weight_max": 28.60515022277832,
      "activations/layer17_attention_weight_min": -27.78980255126953,
      "activations/layer18_attention_weight_max": 30.18255043029785,
      "activations/layer18_attention_weight_min": -25.330411911010742,
      "activations/layer19_attention_weight_max": 32.812068939208984,
      "activations/layer19_attention_weight_min": -30.635787963867188,
      "activations/layer1_attention_weight_max": 17.738300323486328,
      "activations/layer1_attention_weight_min": -16.79743766784668,
      "activations/layer20_attention_weight_max": 26.711475372314453,
      "activations/layer20_attention_weight_min": -27.47686767578125,
      "activations/layer21_attention_weight_max": 27.02573585510254,
      "activations/layer21_attention_weight_min": -24.83481788635254,
      "activations/layer22_attention_weight_max": 41.31614685058594,
      "activations/layer22_attention_weight_min": -30.81436538696289,
      "activations/layer23_attention_weight_max": 30.20001792907715,
      "activations/layer23_attention_weight_min": -25.232872009277344,
      "activations/layer2_attention_weight_max": 34.18039321899414,
      "activations/layer2_attention_weight_min": -34.01891326904297,
      "activations/layer3_attention_weight_max": 95.74109649658203,
      "activations/layer3_attention_weight_min": -100.77517700195312,
      "activations/layer4_attention_weight_max": 104.79756164550781,
      "activations/layer4_attention_weight_min": -108.96959686279297,
      "activations/layer5_attention_weight_max": 55.18394470214844,
      "activations/layer5_attention_weight_min": -61.31071090698242,
      "activations/layer6_attention_weight_max": 46.70789337158203,
      "activations/layer6_attention_weight_min": -49.99824142456055,
      "activations/layer7_attention_weight_max": 92.05949401855469,
      "activations/layer7_attention_weight_min": -100.01148223876953,
      "activations/layer8_attention_weight_max": 43.58794021606445,
      "activations/layer8_attention_weight_min": -46.09705352783203,
      "activations/layer9_attention_weight_max": 32.17729568481445,
      "activations/layer9_attention_weight_min": -33.6383171081543,
      "epoch": 15.59,
      "learning_rate": 4.998825757575757e-05,
      "loss": 2.7451,
      "step": 268250
    },
    {
      "activations/layer0_attention_weight_max": 16.010175704956055,
      "activations/layer0_attention_weight_min": -12.396608352661133,
      "activations/layer10_attention_weight_max": 33.11626052856445,
      "activations/layer10_attention_weight_min": -33.235740661621094,
      "activations/layer11_attention_weight_max": 34.169761657714844,
      "activations/layer11_attention_weight_min": -33.18268585205078,
      "activations/layer12_attention_weight_max": 25.472742080688477,
      "activations/layer12_attention_weight_min": -26.975419998168945,
      "activations/layer13_attention_weight_max": 43.71864318847656,
      "activations/layer13_attention_weight_min": -35.78910446166992,
      "activations/layer14_attention_weight_max": 45.54690933227539,
      "activations/layer14_attention_weight_min": -39.61109924316406,
      "activations/layer15_attention_weight_max": 39.06565475463867,
      "activations/layer15_attention_weight_min": -34.80308151245117,
      "activations/layer16_attention_weight_max": 31.327680587768555,
      "activations/layer16_attention_weight_min": -27.462860107421875,
      "activations/layer17_attention_weight_max": 32.62824630737305,
      "activations/layer17_attention_weight_min": -26.446414947509766,
      "activations/layer18_attention_weight_max": 33.02257537841797,
      "activations/layer18_attention_weight_min": -24.925386428833008,
      "activations/layer19_attention_weight_max": 37.193809509277344,
      "activations/layer19_attention_weight_min": -31.974685668945312,
      "activations/layer1_attention_weight_max": 17.14613914489746,
      "activations/layer1_attention_weight_min": -16.44995880126953,
      "activations/layer20_attention_weight_max": 29.39518928527832,
      "activations/layer20_attention_weight_min": -25.00497055053711,
      "activations/layer21_attention_weight_max": 30.731632232666016,
      "activations/layer21_attention_weight_min": -24.51080894470215,
      "activations/layer22_attention_weight_max": 42.11906814575195,
      "activations/layer22_attention_weight_min": -30.40247344970703,
      "activations/layer23_attention_weight_max": 35.62171173095703,
      "activations/layer23_attention_weight_min": -26.031158447265625,
      "activations/layer2_attention_weight_max": 32.96227264404297,
      "activations/layer2_attention_weight_min": -32.49040222167969,
      "activations/layer3_attention_weight_max": 94.53292083740234,
      "activations/layer3_attention_weight_min": -96.34107971191406,
      "activations/layer4_attention_weight_max": 105.7474594116211,
      "activations/layer4_attention_weight_min": -109.36055755615234,
      "activations/layer5_attention_weight_max": 51.2490234375,
      "activations/layer5_attention_weight_min": -61.86820983886719,
      "activations/layer6_attention_weight_max": 44.143184661865234,
      "activations/layer6_attention_weight_min": -46.13762664794922,
      "activations/layer7_attention_weight_max": 92.16249084472656,
      "activations/layer7_attention_weight_min": -95.59440612792969,
      "activations/layer8_attention_weight_max": 41.604515075683594,
      "activations/layer8_attention_weight_min": -47.83845138549805,
      "activations/layer9_attention_weight_max": 31.498563766479492,
      "activations/layer9_attention_weight_min": -32.80978775024414,
      "epoch": 15.59,
      "learning_rate": 4.9969318181818175e-05,
      "loss": 2.7589,
      "step": 268300
    },
    {
      "activations/layer0_attention_weight_max": 15.775177001953125,
      "activations/layer0_attention_weight_min": -13.744296073913574,
      "activations/layer10_attention_weight_max": 36.21385192871094,
      "activations/layer10_attention_weight_min": -35.15104293823242,
      "activations/layer11_attention_weight_max": 35.984554290771484,
      "activations/layer11_attention_weight_min": -34.49188232421875,
      "activations/layer12_attention_weight_max": 25.709720611572266,
      "activations/layer12_attention_weight_min": -25.60056495666504,
      "activations/layer13_attention_weight_max": 37.319496154785156,
      "activations/layer13_attention_weight_min": -36.368865966796875,
      "activations/layer14_attention_weight_max": 40.54595184326172,
      "activations/layer14_attention_weight_min": -36.54049301147461,
      "activations/layer15_attention_weight_max": 35.95609664916992,
      "activations/layer15_attention_weight_min": -35.892478942871094,
      "activations/layer16_attention_weight_max": 27.964778900146484,
      "activations/layer16_attention_weight_min": -28.1532039642334,
      "activations/layer17_attention_weight_max": 27.598783493041992,
      "activations/layer17_attention_weight_min": -26.76972007751465,
      "activations/layer18_attention_weight_max": 29.335969924926758,
      "activations/layer18_attention_weight_min": -26.78365135192871,
      "activations/layer19_attention_weight_max": 32.41452407836914,
      "activations/layer19_attention_weight_min": -30.69718360900879,
      "activations/layer1_attention_weight_max": 17.999813079833984,
      "activations/layer1_attention_weight_min": -16.22824478149414,
      "activations/layer20_attention_weight_max": 28.397403717041016,
      "activations/layer20_attention_weight_min": -24.790122985839844,
      "activations/layer21_attention_weight_max": 26.655471801757812,
      "activations/layer21_attention_weight_min": -24.764982223510742,
      "activations/layer22_attention_weight_max": 40.93852996826172,
      "activations/layer22_attention_weight_min": -32.62761688232422,
      "activations/layer23_attention_weight_max": 31.410770416259766,
      "activations/layer23_attention_weight_min": -25.046546936035156,
      "activations/layer2_attention_weight_max": 33.42422866821289,
      "activations/layer2_attention_weight_min": -32.81917190551758,
      "activations/layer3_attention_weight_max": 93.39599609375,
      "activations/layer3_attention_weight_min": -98.01069641113281,
      "activations/layer4_attention_weight_max": 106.30549621582031,
      "activations/layer4_attention_weight_min": -109.6045150756836,
      "activations/layer5_attention_weight_max": 49.037841796875,
      "activations/layer5_attention_weight_min": -63.89057540893555,
      "activations/layer6_attention_weight_max": 46.340797424316406,
      "activations/layer6_attention_weight_min": -52.29317855834961,
      "activations/layer7_attention_weight_max": 90.57261657714844,
      "activations/layer7_attention_weight_min": -106.6040267944336,
      "activations/layer8_attention_weight_max": 43.09925842285156,
      "activations/layer8_attention_weight_min": -45.6555061340332,
      "activations/layer9_attention_weight_max": 34.75411605834961,
      "activations/layer9_attention_weight_min": -35.200496673583984,
      "epoch": 15.59,
      "learning_rate": 4.9950378787878784e-05,
      "loss": 2.7437,
      "step": 268350
    },
    {
      "activations/layer0_attention_weight_max": 15.605073928833008,
      "activations/layer0_attention_weight_min": -12.363167762756348,
      "activations/layer10_attention_weight_max": 31.511821746826172,
      "activations/layer10_attention_weight_min": -33.004638671875,
      "activations/layer11_attention_weight_max": 31.07916831970215,
      "activations/layer11_attention_weight_min": -32.407161712646484,
      "activations/layer12_attention_weight_max": 23.97869300842285,
      "activations/layer12_attention_weight_min": -25.6687068939209,
      "activations/layer13_attention_weight_max": 34.55682373046875,
      "activations/layer13_attention_weight_min": -33.56383514404297,
      "activations/layer14_attention_weight_max": 42.06767654418945,
      "activations/layer14_attention_weight_min": -37.4097785949707,
      "activations/layer15_attention_weight_max": 34.23456954956055,
      "activations/layer15_attention_weight_min": -32.360233306884766,
      "activations/layer16_attention_weight_max": 29.151336669921875,
      "activations/layer16_attention_weight_min": -27.268892288208008,
      "activations/layer17_attention_weight_max": 29.351823806762695,
      "activations/layer17_attention_weight_min": -26.55130958557129,
      "activations/layer18_attention_weight_max": 32.88114929199219,
      "activations/layer18_attention_weight_min": -25.820960998535156,
      "activations/layer19_attention_weight_max": 33.05634307861328,
      "activations/layer19_attention_weight_min": -33.11141586303711,
      "activations/layer1_attention_weight_max": 16.691030502319336,
      "activations/layer1_attention_weight_min": -17.990877151489258,
      "activations/layer20_attention_weight_max": 29.64336395263672,
      "activations/layer20_attention_weight_min": -25.14971160888672,
      "activations/layer21_attention_weight_max": 27.16278648376465,
      "activations/layer21_attention_weight_min": -24.149259567260742,
      "activations/layer22_attention_weight_max": 39.65852737426758,
      "activations/layer22_attention_weight_min": -31.518016815185547,
      "activations/layer23_attention_weight_max": 32.736045837402344,
      "activations/layer23_attention_weight_min": -29.31866455078125,
      "activations/layer2_attention_weight_max": 32.8614501953125,
      "activations/layer2_attention_weight_min": -32.516475677490234,
      "activations/layer3_attention_weight_max": 93.512939453125,
      "activations/layer3_attention_weight_min": -92.8393783569336,
      "activations/layer4_attention_weight_max": 103.50190734863281,
      "activations/layer4_attention_weight_min": -105.7169418334961,
      "activations/layer5_attention_weight_max": 50.19499206542969,
      "activations/layer5_attention_weight_min": -61.539283752441406,
      "activations/layer6_attention_weight_max": 43.503902435302734,
      "activations/layer6_attention_weight_min": -42.789268493652344,
      "activations/layer7_attention_weight_max": 87.92263793945312,
      "activations/layer7_attention_weight_min": -89.81299591064453,
      "activations/layer8_attention_weight_max": 39.142459869384766,
      "activations/layer8_attention_weight_min": -39.95414733886719,
      "activations/layer9_attention_weight_max": 31.85957908630371,
      "activations/layer9_attention_weight_min": -32.526878356933594,
      "epoch": 15.6,
      "learning_rate": 4.993143939393939e-05,
      "loss": 2.7575,
      "step": 268400
    },
    {
      "activations/layer0_attention_weight_max": 16.322166442871094,
      "activations/layer0_attention_weight_min": -12.554685592651367,
      "activations/layer10_attention_weight_max": 32.25163650512695,
      "activations/layer10_attention_weight_min": -34.8420295715332,
      "activations/layer11_attention_weight_max": 31.814420700073242,
      "activations/layer11_attention_weight_min": -32.21637725830078,
      "activations/layer12_attention_weight_max": 26.18358039855957,
      "activations/layer12_attention_weight_min": -26.098752975463867,
      "activations/layer13_attention_weight_max": 39.867652893066406,
      "activations/layer13_attention_weight_min": -37.029541015625,
      "activations/layer14_attention_weight_max": 41.63701629638672,
      "activations/layer14_attention_weight_min": -37.63494110107422,
      "activations/layer15_attention_weight_max": 38.866302490234375,
      "activations/layer15_attention_weight_min": -34.300498962402344,
      "activations/layer16_attention_weight_max": 30.28264617919922,
      "activations/layer16_attention_weight_min": -26.679162979125977,
      "activations/layer17_attention_weight_max": 31.413888931274414,
      "activations/layer17_attention_weight_min": -24.688983917236328,
      "activations/layer18_attention_weight_max": 32.76898956298828,
      "activations/layer18_attention_weight_min": -24.774553298950195,
      "activations/layer19_attention_weight_max": 34.584625244140625,
      "activations/layer19_attention_weight_min": -32.26472473144531,
      "activations/layer1_attention_weight_max": 16.60699462890625,
      "activations/layer1_attention_weight_min": -16.453468322753906,
      "activations/layer20_attention_weight_max": 28.91826629638672,
      "activations/layer20_attention_weight_min": -26.36397361755371,
      "activations/layer21_attention_weight_max": 27.396434783935547,
      "activations/layer21_attention_weight_min": -25.17338752746582,
      "activations/layer22_attention_weight_max": 41.50335693359375,
      "activations/layer22_attention_weight_min": -30.12006187438965,
      "activations/layer23_attention_weight_max": 33.13446044921875,
      "activations/layer23_attention_weight_min": -26.080821990966797,
      "activations/layer2_attention_weight_max": 32.74296188354492,
      "activations/layer2_attention_weight_min": -33.913963317871094,
      "activations/layer3_attention_weight_max": 93.8659439086914,
      "activations/layer3_attention_weight_min": -96.46336364746094,
      "activations/layer4_attention_weight_max": 106.34161376953125,
      "activations/layer4_attention_weight_min": -103.75190734863281,
      "activations/layer5_attention_weight_max": 50.263912200927734,
      "activations/layer5_attention_weight_min": -61.97262191772461,
      "activations/layer6_attention_weight_max": 41.91065979003906,
      "activations/layer6_attention_weight_min": -46.281314849853516,
      "activations/layer7_attention_weight_max": 91.3034439086914,
      "activations/layer7_attention_weight_min": -90.13145446777344,
      "activations/layer8_attention_weight_max": 38.652896881103516,
      "activations/layer8_attention_weight_min": -43.494144439697266,
      "activations/layer9_attention_weight_max": 30.405317306518555,
      "activations/layer9_attention_weight_min": -34.52037048339844,
      "epoch": 15.6,
      "learning_rate": 4.9912499999999993e-05,
      "loss": 2.7411,
      "step": 268450
    },
    {
      "activations/layer0_attention_weight_max": 15.80123233795166,
      "activations/layer0_attention_weight_min": -12.581304550170898,
      "activations/layer10_attention_weight_max": 32.30305480957031,
      "activations/layer10_attention_weight_min": -33.02653884887695,
      "activations/layer11_attention_weight_max": 33.83937454223633,
      "activations/layer11_attention_weight_min": -32.58827590942383,
      "activations/layer12_attention_weight_max": 27.047616958618164,
      "activations/layer12_attention_weight_min": -25.7011661529541,
      "activations/layer13_attention_weight_max": 38.14350509643555,
      "activations/layer13_attention_weight_min": -37.880672454833984,
      "activations/layer14_attention_weight_max": 45.12196731567383,
      "activations/layer14_attention_weight_min": -42.967132568359375,
      "activations/layer15_attention_weight_max": 39.55501937866211,
      "activations/layer15_attention_weight_min": -36.062721252441406,
      "activations/layer16_attention_weight_max": 29.71463966369629,
      "activations/layer16_attention_weight_min": -26.8468074798584,
      "activations/layer17_attention_weight_max": 29.951847076416016,
      "activations/layer17_attention_weight_min": -25.662254333496094,
      "activations/layer18_attention_weight_max": 30.636268615722656,
      "activations/layer18_attention_weight_min": -27.712739944458008,
      "activations/layer19_attention_weight_max": 33.520294189453125,
      "activations/layer19_attention_weight_min": -35.58881759643555,
      "activations/layer1_attention_weight_max": 16.319355010986328,
      "activations/layer1_attention_weight_min": -15.506532669067383,
      "activations/layer20_attention_weight_max": 31.256135940551758,
      "activations/layer20_attention_weight_min": -28.247329711914062,
      "activations/layer21_attention_weight_max": 30.37735939025879,
      "activations/layer21_attention_weight_min": -25.214431762695312,
      "activations/layer22_attention_weight_max": 40.83476638793945,
      "activations/layer22_attention_weight_min": -30.25864028930664,
      "activations/layer23_attention_weight_max": 33.895790100097656,
      "activations/layer23_attention_weight_min": -26.095182418823242,
      "activations/layer2_attention_weight_max": 32.64495849609375,
      "activations/layer2_attention_weight_min": -33.59566116333008,
      "activations/layer3_attention_weight_max": 94.76495361328125,
      "activations/layer3_attention_weight_min": -97.63929748535156,
      "activations/layer4_attention_weight_max": 106.79139709472656,
      "activations/layer4_attention_weight_min": -108.81925201416016,
      "activations/layer5_attention_weight_max": 51.311458587646484,
      "activations/layer5_attention_weight_min": -62.398189544677734,
      "activations/layer6_attention_weight_max": 45.1761360168457,
      "activations/layer6_attention_weight_min": -44.41370391845703,
      "activations/layer7_attention_weight_max": 93.85922241210938,
      "activations/layer7_attention_weight_min": -91.82295989990234,
      "activations/layer8_attention_weight_max": 38.38475036621094,
      "activations/layer8_attention_weight_min": -41.001976013183594,
      "activations/layer9_attention_weight_max": 29.82381248474121,
      "activations/layer9_attention_weight_min": -32.586509704589844,
      "epoch": 15.6,
      "learning_rate": 4.9893560606060595e-05,
      "loss": 2.7332,
      "step": 268500
    },
    {
      "activations/layer0_attention_weight_max": 15.635087013244629,
      "activations/layer0_attention_weight_min": -13.299983024597168,
      "activations/layer10_attention_weight_max": 31.14423179626465,
      "activations/layer10_attention_weight_min": -31.671016693115234,
      "activations/layer11_attention_weight_max": 31.831153869628906,
      "activations/layer11_attention_weight_min": -32.40909194946289,
      "activations/layer12_attention_weight_max": 25.55471420288086,
      "activations/layer12_attention_weight_min": -25.914793014526367,
      "activations/layer13_attention_weight_max": 37.28126525878906,
      "activations/layer13_attention_weight_min": -34.96331024169922,
      "activations/layer14_attention_weight_max": 41.23986053466797,
      "activations/layer14_attention_weight_min": -37.7200813293457,
      "activations/layer15_attention_weight_max": 35.745365142822266,
      "activations/layer15_attention_weight_min": -33.59577941894531,
      "activations/layer16_attention_weight_max": 27.926488876342773,
      "activations/layer16_attention_weight_min": -26.91485023498535,
      "activations/layer17_attention_weight_max": 29.46725845336914,
      "activations/layer17_attention_weight_min": -25.88417625427246,
      "activations/layer18_attention_weight_max": 28.953359603881836,
      "activations/layer18_attention_weight_min": -24.161046981811523,
      "activations/layer19_attention_weight_max": 34.36367416381836,
      "activations/layer19_attention_weight_min": -29.99588966369629,
      "activations/layer1_attention_weight_max": 17.472007751464844,
      "activations/layer1_attention_weight_min": -16.836095809936523,
      "activations/layer20_attention_weight_max": 28.971445083618164,
      "activations/layer20_attention_weight_min": -24.445322036743164,
      "activations/layer21_attention_weight_max": 30.80757713317871,
      "activations/layer21_attention_weight_min": -25.099376678466797,
      "activations/layer22_attention_weight_max": 38.94575881958008,
      "activations/layer22_attention_weight_min": -33.30187225341797,
      "activations/layer23_attention_weight_max": 34.63768005371094,
      "activations/layer23_attention_weight_min": -25.539695739746094,
      "activations/layer2_attention_weight_max": 34.389892578125,
      "activations/layer2_attention_weight_min": -32.82295227050781,
      "activations/layer3_attention_weight_max": 95.70410919189453,
      "activations/layer3_attention_weight_min": -93.59591674804688,
      "activations/layer4_attention_weight_max": 104.51317596435547,
      "activations/layer4_attention_weight_min": -102.86524963378906,
      "activations/layer5_attention_weight_max": 52.2716064453125,
      "activations/layer5_attention_weight_min": -62.73834228515625,
      "activations/layer6_attention_weight_max": 43.074092864990234,
      "activations/layer6_attention_weight_min": -44.662010192871094,
      "activations/layer7_attention_weight_max": 85.87693786621094,
      "activations/layer7_attention_weight_min": -89.37810516357422,
      "activations/layer8_attention_weight_max": 40.793426513671875,
      "activations/layer8_attention_weight_min": -42.40090560913086,
      "activations/layer9_attention_weight_max": 31.860748291015625,
      "activations/layer9_attention_weight_min": -33.76035690307617,
      "epoch": 15.6,
      "learning_rate": 4.987462121212121e-05,
      "loss": 2.7372,
      "step": 268550
    },
    {
      "activations/layer0_attention_weight_max": 16.086515426635742,
      "activations/layer0_attention_weight_min": -12.502093315124512,
      "activations/layer10_attention_weight_max": 35.89170837402344,
      "activations/layer10_attention_weight_min": -35.76756286621094,
      "activations/layer11_attention_weight_max": 37.641021728515625,
      "activations/layer11_attention_weight_min": -34.5744743347168,
      "activations/layer12_attention_weight_max": 29.138689041137695,
      "activations/layer12_attention_weight_min": -30.148548126220703,
      "activations/layer13_attention_weight_max": 44.42839431762695,
      "activations/layer13_attention_weight_min": -39.64586639404297,
      "activations/layer14_attention_weight_max": 45.3571891784668,
      "activations/layer14_attention_weight_min": -38.887413024902344,
      "activations/layer15_attention_weight_max": 41.009361267089844,
      "activations/layer15_attention_weight_min": -34.2425422668457,
      "activations/layer16_attention_weight_max": 28.895740509033203,
      "activations/layer16_attention_weight_min": -29.645639419555664,
      "activations/layer17_attention_weight_max": 31.51664161682129,
      "activations/layer17_attention_weight_min": -25.701534271240234,
      "activations/layer18_attention_weight_max": 33.33797073364258,
      "activations/layer18_attention_weight_min": -24.22801399230957,
      "activations/layer19_attention_weight_max": 31.67537498474121,
      "activations/layer19_attention_weight_min": -30.015974044799805,
      "activations/layer1_attention_weight_max": 17.188947677612305,
      "activations/layer1_attention_weight_min": -14.803084373474121,
      "activations/layer20_attention_weight_max": 30.668405532836914,
      "activations/layer20_attention_weight_min": -24.375001907348633,
      "activations/layer21_attention_weight_max": 27.569908142089844,
      "activations/layer21_attention_weight_min": -24.8406982421875,
      "activations/layer22_attention_weight_max": 43.609615325927734,
      "activations/layer22_attention_weight_min": -30.015365600585938,
      "activations/layer23_attention_weight_max": 32.297119140625,
      "activations/layer23_attention_weight_min": -23.350276947021484,
      "activations/layer2_attention_weight_max": 35.23262405395508,
      "activations/layer2_attention_weight_min": -36.27004623413086,
      "activations/layer3_attention_weight_max": 100.40790557861328,
      "activations/layer3_attention_weight_min": -106.14684295654297,
      "activations/layer4_attention_weight_max": 109.26338195800781,
      "activations/layer4_attention_weight_min": -111.35981750488281,
      "activations/layer5_attention_weight_max": 54.23712921142578,
      "activations/layer5_attention_weight_min": -64.90348815917969,
      "activations/layer6_attention_weight_max": 45.97239303588867,
      "activations/layer6_attention_weight_min": -47.12788391113281,
      "activations/layer7_attention_weight_max": 96.88987731933594,
      "activations/layer7_attention_weight_min": -95.21395111083984,
      "activations/layer8_attention_weight_max": 46.312721252441406,
      "activations/layer8_attention_weight_min": -44.10196304321289,
      "activations/layer9_attention_weight_max": 35.03233337402344,
      "activations/layer9_attention_weight_min": -34.93064498901367,
      "epoch": 15.61,
      "learning_rate": 4.985568181818181e-05,
      "loss": 2.7284,
      "step": 268600
    },
    {
      "activations/layer0_attention_weight_max": 16.096920013427734,
      "activations/layer0_attention_weight_min": -13.102204322814941,
      "activations/layer10_attention_weight_max": 31.199188232421875,
      "activations/layer10_attention_weight_min": -33.24699783325195,
      "activations/layer11_attention_weight_max": 33.13449478149414,
      "activations/layer11_attention_weight_min": -33.07060623168945,
      "activations/layer12_attention_weight_max": 25.65458869934082,
      "activations/layer12_attention_weight_min": -27.053403854370117,
      "activations/layer13_attention_weight_max": 37.383575439453125,
      "activations/layer13_attention_weight_min": -36.78618240356445,
      "activations/layer14_attention_weight_max": 42.56040954589844,
      "activations/layer14_attention_weight_min": -38.09959411621094,
      "activations/layer15_attention_weight_max": 39.5567626953125,
      "activations/layer15_attention_weight_min": -33.531517028808594,
      "activations/layer16_attention_weight_max": 29.03910255432129,
      "activations/layer16_attention_weight_min": -26.78120994567871,
      "activations/layer17_attention_weight_max": 33.75845718383789,
      "activations/layer17_attention_weight_min": -24.07230567932129,
      "activations/layer18_attention_weight_max": 35.94161605834961,
      "activations/layer18_attention_weight_min": -24.543825149536133,
      "activations/layer19_attention_weight_max": 37.49029541015625,
      "activations/layer19_attention_weight_min": -30.830961227416992,
      "activations/layer1_attention_weight_max": 15.219001770019531,
      "activations/layer1_attention_weight_min": -15.236414909362793,
      "activations/layer20_attention_weight_max": 29.455575942993164,
      "activations/layer20_attention_weight_min": -22.65764808654785,
      "activations/layer21_attention_weight_max": 31.77855110168457,
      "activations/layer21_attention_weight_min": -23.455841064453125,
      "activations/layer22_attention_weight_max": 41.47761535644531,
      "activations/layer22_attention_weight_min": -31.217811584472656,
      "activations/layer23_attention_weight_max": 34.119224548339844,
      "activations/layer23_attention_weight_min": -26.317846298217773,
      "activations/layer2_attention_weight_max": 33.472267150878906,
      "activations/layer2_attention_weight_min": -32.52888107299805,
      "activations/layer3_attention_weight_max": 98.20943450927734,
      "activations/layer3_attention_weight_min": -95.3362808227539,
      "activations/layer4_attention_weight_max": 102.2881088256836,
      "activations/layer4_attention_weight_min": -103.98878479003906,
      "activations/layer5_attention_weight_max": 51.60372543334961,
      "activations/layer5_attention_weight_min": -61.91570281982422,
      "activations/layer6_attention_weight_max": 43.26714324951172,
      "activations/layer6_attention_weight_min": -44.36101150512695,
      "activations/layer7_attention_weight_max": 91.17568969726562,
      "activations/layer7_attention_weight_min": -89.59674072265625,
      "activations/layer8_attention_weight_max": 40.96549987792969,
      "activations/layer8_attention_weight_min": -41.62007141113281,
      "activations/layer9_attention_weight_max": 31.21280288696289,
      "activations/layer9_attention_weight_min": -33.25961685180664,
      "epoch": 15.61,
      "learning_rate": 4.983674242424242e-05,
      "loss": 2.7599,
      "step": 268650
    },
    {
      "activations/layer0_attention_weight_max": 16.352436065673828,
      "activations/layer0_attention_weight_min": -12.545734405517578,
      "activations/layer10_attention_weight_max": 33.19683074951172,
      "activations/layer10_attention_weight_min": -33.30697250366211,
      "activations/layer11_attention_weight_max": 32.563232421875,
      "activations/layer11_attention_weight_min": -33.47564697265625,
      "activations/layer12_attention_weight_max": 25.225101470947266,
      "activations/layer12_attention_weight_min": -26.163087844848633,
      "activations/layer13_attention_weight_max": 38.21247482299805,
      "activations/layer13_attention_weight_min": -34.02831268310547,
      "activations/layer14_attention_weight_max": 42.286903381347656,
      "activations/layer14_attention_weight_min": -36.92625045776367,
      "activations/layer15_attention_weight_max": 38.33716583251953,
      "activations/layer15_attention_weight_min": -34.413330078125,
      "activations/layer16_attention_weight_max": 29.51877212524414,
      "activations/layer16_attention_weight_min": -26.23289680480957,
      "activations/layer17_attention_weight_max": 32.26692581176758,
      "activations/layer17_attention_weight_min": -25.168622970581055,
      "activations/layer18_attention_weight_max": 31.84649658203125,
      "activations/layer18_attention_weight_min": -26.85797882080078,
      "activations/layer19_attention_weight_max": 31.75411033630371,
      "activations/layer19_attention_weight_min": -33.63273239135742,
      "activations/layer1_attention_weight_max": 17.19038963317871,
      "activations/layer1_attention_weight_min": -16.576993942260742,
      "activations/layer20_attention_weight_max": 27.774789810180664,
      "activations/layer20_attention_weight_min": -25.084354400634766,
      "activations/layer21_attention_weight_max": 30.070720672607422,
      "activations/layer21_attention_weight_min": -24.63286018371582,
      "activations/layer22_attention_weight_max": 41.294193267822266,
      "activations/layer22_attention_weight_min": -31.653514862060547,
      "activations/layer23_attention_weight_max": 32.53076934814453,
      "activations/layer23_attention_weight_min": -26.503154754638672,
      "activations/layer2_attention_weight_max": 32.27214050292969,
      "activations/layer2_attention_weight_min": -31.578014373779297,
      "activations/layer3_attention_weight_max": 97.17742156982422,
      "activations/layer3_attention_weight_min": -95.1814956665039,
      "activations/layer4_attention_weight_max": 104.02081298828125,
      "activations/layer4_attention_weight_min": -106.0025634765625,
      "activations/layer5_attention_weight_max": 54.2955436706543,
      "activations/layer5_attention_weight_min": -64.73013305664062,
      "activations/layer6_attention_weight_max": 42.448448181152344,
      "activations/layer6_attention_weight_min": -45.870548248291016,
      "activations/layer7_attention_weight_max": 95.68534088134766,
      "activations/layer7_attention_weight_min": -93.908935546875,
      "activations/layer8_attention_weight_max": 42.58163070678711,
      "activations/layer8_attention_weight_min": -44.43576431274414,
      "activations/layer9_attention_weight_max": 32.06150817871094,
      "activations/layer9_attention_weight_min": -34.65288162231445,
      "epoch": 15.61,
      "learning_rate": 4.981780303030302e-05,
      "loss": 2.7428,
      "step": 268700
    },
    {
      "activations/layer0_attention_weight_max": 16.08550262451172,
      "activations/layer0_attention_weight_min": -13.407771110534668,
      "activations/layer10_attention_weight_max": 33.303070068359375,
      "activations/layer10_attention_weight_min": -33.0057258605957,
      "activations/layer11_attention_weight_max": 33.385162353515625,
      "activations/layer11_attention_weight_min": -32.40079879760742,
      "activations/layer12_attention_weight_max": 25.774574279785156,
      "activations/layer12_attention_weight_min": -25.89678382873535,
      "activations/layer13_attention_weight_max": 39.732948303222656,
      "activations/layer13_attention_weight_min": -35.03879928588867,
      "activations/layer14_attention_weight_max": 42.75535583496094,
      "activations/layer14_attention_weight_min": -36.79446029663086,
      "activations/layer15_attention_weight_max": 39.225589752197266,
      "activations/layer15_attention_weight_min": -32.73063659667969,
      "activations/layer16_attention_weight_max": 33.68959426879883,
      "activations/layer16_attention_weight_min": -27.95271110534668,
      "activations/layer17_attention_weight_max": 33.75831985473633,
      "activations/layer17_attention_weight_min": -26.325414657592773,
      "activations/layer18_attention_weight_max": 35.21906661987305,
      "activations/layer18_attention_weight_min": -25.33103370666504,
      "activations/layer19_attention_weight_max": 32.7874641418457,
      "activations/layer19_attention_weight_min": -32.012298583984375,
      "activations/layer1_attention_weight_max": 16.29192352294922,
      "activations/layer1_attention_weight_min": -15.379533767700195,
      "activations/layer20_attention_weight_max": 31.155338287353516,
      "activations/layer20_attention_weight_min": -25.65553855895996,
      "activations/layer21_attention_weight_max": 27.87718963623047,
      "activations/layer21_attention_weight_min": -24.75053596496582,
      "activations/layer22_attention_weight_max": 45.262611389160156,
      "activations/layer22_attention_weight_min": -29.270492553710938,
      "activations/layer23_attention_weight_max": 34.65187072753906,
      "activations/layer23_attention_weight_min": -25.85414695739746,
      "activations/layer2_attention_weight_max": 34.732269287109375,
      "activations/layer2_attention_weight_min": -32.6681022644043,
      "activations/layer3_attention_weight_max": 98.41699981689453,
      "activations/layer3_attention_weight_min": -96.24732208251953,
      "activations/layer4_attention_weight_max": 106.4230728149414,
      "activations/layer4_attention_weight_min": -108.39019012451172,
      "activations/layer5_attention_weight_max": 51.41843795776367,
      "activations/layer5_attention_weight_min": -64.81459045410156,
      "activations/layer6_attention_weight_max": 41.789363861083984,
      "activations/layer6_attention_weight_min": -43.61001205444336,
      "activations/layer7_attention_weight_max": 89.4887466430664,
      "activations/layer7_attention_weight_min": -93.56770324707031,
      "activations/layer8_attention_weight_max": 39.16843032836914,
      "activations/layer8_attention_weight_min": -42.053070068359375,
      "activations/layer9_attention_weight_max": 33.6647834777832,
      "activations/layer9_attention_weight_min": -32.878700256347656,
      "epoch": 15.62,
      "learning_rate": 4.9798863636363636e-05,
      "loss": 2.7601,
      "step": 268750
    },
    {
      "activations/layer0_attention_weight_max": 16.11433219909668,
      "activations/layer0_attention_weight_min": -14.24536418914795,
      "activations/layer10_attention_weight_max": 36.50135040283203,
      "activations/layer10_attention_weight_min": -34.42938232421875,
      "activations/layer11_attention_weight_max": 36.46592712402344,
      "activations/layer11_attention_weight_min": -35.9375,
      "activations/layer12_attention_weight_max": 28.947763442993164,
      "activations/layer12_attention_weight_min": -30.228778839111328,
      "activations/layer13_attention_weight_max": 45.48860549926758,
      "activations/layer13_attention_weight_min": -39.91865539550781,
      "activations/layer14_attention_weight_max": 51.914920806884766,
      "activations/layer14_attention_weight_min": -42.62599182128906,
      "activations/layer15_attention_weight_max": 45.30902099609375,
      "activations/layer15_attention_weight_min": -38.784263610839844,
      "activations/layer16_attention_weight_max": 33.17684555053711,
      "activations/layer16_attention_weight_min": -31.06553840637207,
      "activations/layer17_attention_weight_max": 32.799930572509766,
      "activations/layer17_attention_weight_min": -26.727365493774414,
      "activations/layer18_attention_weight_max": 34.102352142333984,
      "activations/layer18_attention_weight_min": -27.56989097595215,
      "activations/layer19_attention_weight_max": 40.65473175048828,
      "activations/layer19_attention_weight_min": -33.19930648803711,
      "activations/layer1_attention_weight_max": 16.441072463989258,
      "activations/layer1_attention_weight_min": -14.672998428344727,
      "activations/layer20_attention_weight_max": 37.10200500488281,
      "activations/layer20_attention_weight_min": -27.873090744018555,
      "activations/layer21_attention_weight_max": 32.910579681396484,
      "activations/layer21_attention_weight_min": -25.56967544555664,
      "activations/layer22_attention_weight_max": 56.42868423461914,
      "activations/layer22_attention_weight_min": -34.403953552246094,
      "activations/layer23_attention_weight_max": 38.39680480957031,
      "activations/layer23_attention_weight_min": -27.614837646484375,
      "activations/layer2_attention_weight_max": 33.548187255859375,
      "activations/layer2_attention_weight_min": -33.337459564208984,
      "activations/layer3_attention_weight_max": 97.07093811035156,
      "activations/layer3_attention_weight_min": -99.16557312011719,
      "activations/layer4_attention_weight_max": 109.68810272216797,
      "activations/layer4_attention_weight_min": -111.91377258300781,
      "activations/layer5_attention_weight_max": 52.949851989746094,
      "activations/layer5_attention_weight_min": -62.67613220214844,
      "activations/layer6_attention_weight_max": 46.84297180175781,
      "activations/layer6_attention_weight_min": -49.2985954284668,
      "activations/layer7_attention_weight_max": 97.84341430664062,
      "activations/layer7_attention_weight_min": -96.6552963256836,
      "activations/layer8_attention_weight_max": 43.50120544433594,
      "activations/layer8_attention_weight_min": -46.258270263671875,
      "activations/layer9_attention_weight_max": 33.39884567260742,
      "activations/layer9_attention_weight_min": -35.71639633178711,
      "epoch": 15.62,
      "learning_rate": 4.977992424242424e-05,
      "loss": 2.7471,
      "step": 268800
    },
    {
      "activations/layer0_attention_weight_max": 15.937432289123535,
      "activations/layer0_attention_weight_min": -13.579005241394043,
      "activations/layer10_attention_weight_max": 31.75423240661621,
      "activations/layer10_attention_weight_min": -32.179542541503906,
      "activations/layer11_attention_weight_max": 31.280101776123047,
      "activations/layer11_attention_weight_min": -32.756439208984375,
      "activations/layer12_attention_weight_max": 24.74908447265625,
      "activations/layer12_attention_weight_min": -25.237741470336914,
      "activations/layer13_attention_weight_max": 37.11549758911133,
      "activations/layer13_attention_weight_min": -34.709835052490234,
      "activations/layer14_attention_weight_max": 38.918304443359375,
      "activations/layer14_attention_weight_min": -38.60148239135742,
      "activations/layer15_attention_weight_max": 36.64750289916992,
      "activations/layer15_attention_weight_min": -35.02838134765625,
      "activations/layer16_attention_weight_max": 28.482349395751953,
      "activations/layer16_attention_weight_min": -27.14006233215332,
      "activations/layer17_attention_weight_max": 27.594783782958984,
      "activations/layer17_attention_weight_min": -24.789642333984375,
      "activations/layer18_attention_weight_max": 30.316598892211914,
      "activations/layer18_attention_weight_min": -24.733442306518555,
      "activations/layer19_attention_weight_max": 33.32351303100586,
      "activations/layer19_attention_weight_min": -32.87025451660156,
      "activations/layer1_attention_weight_max": 16.72083854675293,
      "activations/layer1_attention_weight_min": -14.93901252746582,
      "activations/layer20_attention_weight_max": 26.726482391357422,
      "activations/layer20_attention_weight_min": -24.975868225097656,
      "activations/layer21_attention_weight_max": 26.144176483154297,
      "activations/layer21_attention_weight_min": -25.46059226989746,
      "activations/layer22_attention_weight_max": 43.03232955932617,
      "activations/layer22_attention_weight_min": -32.790985107421875,
      "activations/layer23_attention_weight_max": 33.257293701171875,
      "activations/layer23_attention_weight_min": -25.556814193725586,
      "activations/layer2_attention_weight_max": 33.324920654296875,
      "activations/layer2_attention_weight_min": -33.49260711669922,
      "activations/layer3_attention_weight_max": 97.89378356933594,
      "activations/layer3_attention_weight_min": -97.50603485107422,
      "activations/layer4_attention_weight_max": 106.47377014160156,
      "activations/layer4_attention_weight_min": -110.44068908691406,
      "activations/layer5_attention_weight_max": 51.45333480834961,
      "activations/layer5_attention_weight_min": -59.90350341796875,
      "activations/layer6_attention_weight_max": 41.913604736328125,
      "activations/layer6_attention_weight_min": -42.798179626464844,
      "activations/layer7_attention_weight_max": 86.20511627197266,
      "activations/layer7_attention_weight_min": -88.47116088867188,
      "activations/layer8_attention_weight_max": 38.51518630981445,
      "activations/layer8_attention_weight_min": -41.149139404296875,
      "activations/layer9_attention_weight_max": 31.78615379333496,
      "activations/layer9_attention_weight_min": -31.683929443359375,
      "epoch": 15.62,
      "learning_rate": 4.976098484848484e-05,
      "loss": 2.7487,
      "step": 268850
    },
    {
      "activations/layer0_attention_weight_max": 16.27484893798828,
      "activations/layer0_attention_weight_min": -12.488390922546387,
      "activations/layer10_attention_weight_max": 32.341278076171875,
      "activations/layer10_attention_weight_min": -31.217636108398438,
      "activations/layer11_attention_weight_max": 33.044342041015625,
      "activations/layer11_attention_weight_min": -31.466848373413086,
      "activations/layer12_attention_weight_max": 24.50958251953125,
      "activations/layer12_attention_weight_min": -25.219562530517578,
      "activations/layer13_attention_weight_max": 37.84245681762695,
      "activations/layer13_attention_weight_min": -34.79205322265625,
      "activations/layer14_attention_weight_max": 40.24858856201172,
      "activations/layer14_attention_weight_min": -35.886680603027344,
      "activations/layer15_attention_weight_max": 35.853885650634766,
      "activations/layer15_attention_weight_min": -34.421363830566406,
      "activations/layer16_attention_weight_max": 29.85932731628418,
      "activations/layer16_attention_weight_min": -26.407445907592773,
      "activations/layer17_attention_weight_max": 31.253374099731445,
      "activations/layer17_attention_weight_min": -24.486818313598633,
      "activations/layer18_attention_weight_max": 31.997413635253906,
      "activations/layer18_attention_weight_min": -25.62382698059082,
      "activations/layer19_attention_weight_max": 32.08470916748047,
      "activations/layer19_attention_weight_min": -29.658212661743164,
      "activations/layer1_attention_weight_max": 15.566580772399902,
      "activations/layer1_attention_weight_min": -14.996232986450195,
      "activations/layer20_attention_weight_max": 27.36836814880371,
      "activations/layer20_attention_weight_min": -23.785560607910156,
      "activations/layer21_attention_weight_max": 27.774065017700195,
      "activations/layer21_attention_weight_min": -24.35382652282715,
      "activations/layer22_attention_weight_max": 44.79033660888672,
      "activations/layer22_attention_weight_min": -31.10647201538086,
      "activations/layer23_attention_weight_max": 34.329341888427734,
      "activations/layer23_attention_weight_min": -26.49268913269043,
      "activations/layer2_attention_weight_max": 34.710243225097656,
      "activations/layer2_attention_weight_min": -33.749969482421875,
      "activations/layer3_attention_weight_max": 97.69995880126953,
      "activations/layer3_attention_weight_min": -99.86563873291016,
      "activations/layer4_attention_weight_max": 109.1600341796875,
      "activations/layer4_attention_weight_min": -108.85957336425781,
      "activations/layer5_attention_weight_max": 48.92424774169922,
      "activations/layer5_attention_weight_min": -60.36463928222656,
      "activations/layer6_attention_weight_max": 42.08587646484375,
      "activations/layer6_attention_weight_min": -44.703895568847656,
      "activations/layer7_attention_weight_max": 88.29920959472656,
      "activations/layer7_attention_weight_min": -86.56394958496094,
      "activations/layer8_attention_weight_max": 38.99293899536133,
      "activations/layer8_attention_weight_min": -39.331939697265625,
      "activations/layer9_attention_weight_max": 32.239070892333984,
      "activations/layer9_attention_weight_min": -31.84088706970215,
      "epoch": 15.62,
      "learning_rate": 4.9742045454545454e-05,
      "loss": 2.751,
      "step": 268900
    },
    {
      "activations/layer0_attention_weight_max": 16.345905303955078,
      "activations/layer0_attention_weight_min": -12.891353607177734,
      "activations/layer10_attention_weight_max": 32.41661834716797,
      "activations/layer10_attention_weight_min": -32.03943634033203,
      "activations/layer11_attention_weight_max": 32.52621841430664,
      "activations/layer11_attention_weight_min": -32.81507873535156,
      "activations/layer12_attention_weight_max": 25.709489822387695,
      "activations/layer12_attention_weight_min": -25.61681365966797,
      "activations/layer13_attention_weight_max": 38.27622604370117,
      "activations/layer13_attention_weight_min": -34.38102722167969,
      "activations/layer14_attention_weight_max": 42.04427719116211,
      "activations/layer14_attention_weight_min": -36.52263641357422,
      "activations/layer15_attention_weight_max": 38.263309478759766,
      "activations/layer15_attention_weight_min": -34.43349075317383,
      "activations/layer16_attention_weight_max": 31.266918182373047,
      "activations/layer16_attention_weight_min": -27.778221130371094,
      "activations/layer17_attention_weight_max": 31.341325759887695,
      "activations/layer17_attention_weight_min": -27.517656326293945,
      "activations/layer18_attention_weight_max": 33.719642639160156,
      "activations/layer18_attention_weight_min": -25.77719497680664,
      "activations/layer19_attention_weight_max": 33.69773483276367,
      "activations/layer19_attention_weight_min": -30.20148277282715,
      "activations/layer1_attention_weight_max": 15.889404296875,
      "activations/layer1_attention_weight_min": -16.272865295410156,
      "activations/layer20_attention_weight_max": 30.955310821533203,
      "activations/layer20_attention_weight_min": -23.6934814453125,
      "activations/layer21_attention_weight_max": 29.07651138305664,
      "activations/layer21_attention_weight_min": -23.897672653198242,
      "activations/layer22_attention_weight_max": 44.06256103515625,
      "activations/layer22_attention_weight_min": -31.477352142333984,
      "activations/layer23_attention_weight_max": 34.630958557128906,
      "activations/layer23_attention_weight_min": -25.521690368652344,
      "activations/layer2_attention_weight_max": 33.1417350769043,
      "activations/layer2_attention_weight_min": -33.700469970703125,
      "activations/layer3_attention_weight_max": 94.27649688720703,
      "activations/layer3_attention_weight_min": -99.4629135131836,
      "activations/layer4_attention_weight_max": 104.02422332763672,
      "activations/layer4_attention_weight_min": -106.70287322998047,
      "activations/layer5_attention_weight_max": 50.352996826171875,
      "activations/layer5_attention_weight_min": -59.93126678466797,
      "activations/layer6_attention_weight_max": 43.50029754638672,
      "activations/layer6_attention_weight_min": -49.52321243286133,
      "activations/layer7_attention_weight_max": 87.32991790771484,
      "activations/layer7_attention_weight_min": -98.46456909179688,
      "activations/layer8_attention_weight_max": 40.56098937988281,
      "activations/layer8_attention_weight_min": -41.00242233276367,
      "activations/layer9_attention_weight_max": 31.5119571685791,
      "activations/layer9_attention_weight_min": -32.77207565307617,
      "epoch": 15.63,
      "learning_rate": 4.9723106060606056e-05,
      "loss": 2.7562,
      "step": 268950
    },
    {
      "activations/layer0_attention_weight_max": 16.506269454956055,
      "activations/layer0_attention_weight_min": -12.826057434082031,
      "activations/layer10_attention_weight_max": 37.226318359375,
      "activations/layer10_attention_weight_min": -35.5577278137207,
      "activations/layer11_attention_weight_max": 38.774559020996094,
      "activations/layer11_attention_weight_min": -35.360286712646484,
      "activations/layer12_attention_weight_max": 29.65320587158203,
      "activations/layer12_attention_weight_min": -27.38239860534668,
      "activations/layer13_attention_weight_max": 41.787750244140625,
      "activations/layer13_attention_weight_min": -38.74972152709961,
      "activations/layer14_attention_weight_max": 46.69232177734375,
      "activations/layer14_attention_weight_min": -43.93859100341797,
      "activations/layer15_attention_weight_max": 41.68458557128906,
      "activations/layer15_attention_weight_min": -38.13227844238281,
      "activations/layer16_attention_weight_max": 31.25850486755371,
      "activations/layer16_attention_weight_min": -29.46664810180664,
      "activations/layer17_attention_weight_max": 31.817190170288086,
      "activations/layer17_attention_weight_min": -29.818769454956055,
      "activations/layer18_attention_weight_max": 30.469438552856445,
      "activations/layer18_attention_weight_min": -26.326364517211914,
      "activations/layer19_attention_weight_max": 37.48192596435547,
      "activations/layer19_attention_weight_min": -32.45169448852539,
      "activations/layer1_attention_weight_max": 16.079418182373047,
      "activations/layer1_attention_weight_min": -15.118879318237305,
      "activations/layer20_attention_weight_max": 31.9902286529541,
      "activations/layer20_attention_weight_min": -24.989933013916016,
      "activations/layer21_attention_weight_max": 30.796457290649414,
      "activations/layer21_attention_weight_min": -24.161245346069336,
      "activations/layer22_attention_weight_max": 42.4785270690918,
      "activations/layer22_attention_weight_min": -31.61220359802246,
      "activations/layer23_attention_weight_max": 35.13513946533203,
      "activations/layer23_attention_weight_min": -23.044395446777344,
      "activations/layer2_attention_weight_max": 32.71788787841797,
      "activations/layer2_attention_weight_min": -32.69462966918945,
      "activations/layer3_attention_weight_max": 99.65552520751953,
      "activations/layer3_attention_weight_min": -98.6181640625,
      "activations/layer4_attention_weight_max": 110.1120834350586,
      "activations/layer4_attention_weight_min": -107.9217300415039,
      "activations/layer5_attention_weight_max": 53.06428527832031,
      "activations/layer5_attention_weight_min": -62.22494125366211,
      "activations/layer6_attention_weight_max": 44.42003631591797,
      "activations/layer6_attention_weight_min": -47.140995025634766,
      "activations/layer7_attention_weight_max": 94.20352935791016,
      "activations/layer7_attention_weight_min": -93.01625061035156,
      "activations/layer8_attention_weight_max": 44.33744430541992,
      "activations/layer8_attention_weight_min": -44.104637145996094,
      "activations/layer9_attention_weight_max": 35.20331954956055,
      "activations/layer9_attention_weight_min": -33.81836700439453,
      "epoch": 15.63,
      "learning_rate": 4.9704166666666664e-05,
      "loss": 2.7423,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4411,
      "eval_samples_per_second": 508.704,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4411,
      "eval_openwebtext_samples_per_second": 508.704,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9551,
      "eval_wikitext_samples_per_second": 233.238,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_lambada_loss": 2.431640625,
      "eval_lambada_ppl": 11.3775330413472,
      "eval_lambada_runtime": 9.5651,
      "eval_lambada_samples_per_second": 509.037,
      "step": 269000
    },
    {
      "activations/layer0_attention_weight_max": 15.679908752441406,
      "activations/layer0_attention_weight_min": -12.517048835754395,
      "activations/layer10_attention_weight_max": 31.69399642944336,
      "activations/layer10_attention_weight_min": -33.406734466552734,
      "activations/layer11_attention_weight_max": 33.65061950683594,
      "activations/layer11_attention_weight_min": -32.984458923339844,
      "activations/layer12_attention_weight_max": 27.492860794067383,
      "activations/layer12_attention_weight_min": -25.939125061035156,
      "activations/layer13_attention_weight_max": 41.686397552490234,
      "activations/layer13_attention_weight_min": -37.15370559692383,
      "activations/layer14_attention_weight_max": 44.16482162475586,
      "activations/layer14_attention_weight_min": -41.2650260925293,
      "activations/layer15_attention_weight_max": 40.872737884521484,
      "activations/layer15_attention_weight_min": -36.128631591796875,
      "activations/layer16_attention_weight_max": 30.843854904174805,
      "activations/layer16_attention_weight_min": -30.01940155029297,
      "activations/layer17_attention_weight_max": 32.00399398803711,
      "activations/layer17_attention_weight_min": -26.63644790649414,
      "activations/layer18_attention_weight_max": 34.27944564819336,
      "activations/layer18_attention_weight_min": -25.40495491027832,
      "activations/layer19_attention_weight_max": 37.067867279052734,
      "activations/layer19_attention_weight_min": -33.033382415771484,
      "activations/layer1_attention_weight_max": 17.914165496826172,
      "activations/layer1_attention_weight_min": -15.786602020263672,
      "activations/layer20_attention_weight_max": 33.11890411376953,
      "activations/layer20_attention_weight_min": -26.630952835083008,
      "activations/layer21_attention_weight_max": 33.0151481628418,
      "activations/layer21_attention_weight_min": -26.879594802856445,
      "activations/layer22_attention_weight_max": 47.5242805480957,
      "activations/layer22_attention_weight_min": -34.012428283691406,
      "activations/layer23_attention_weight_max": 36.4505615234375,
      "activations/layer23_attention_weight_min": -27.025524139404297,
      "activations/layer2_attention_weight_max": 33.03276062011719,
      "activations/layer2_attention_weight_min": -34.63440704345703,
      "activations/layer3_attention_weight_max": 94.34442138671875,
      "activations/layer3_attention_weight_min": -104.23955535888672,
      "activations/layer4_attention_weight_max": 105.19451141357422,
      "activations/layer4_attention_weight_min": -114.69756317138672,
      "activations/layer5_attention_weight_max": 55.59650802612305,
      "activations/layer5_attention_weight_min": -65.78974914550781,
      "activations/layer6_attention_weight_max": 44.06667709350586,
      "activations/layer6_attention_weight_min": -46.25959396362305,
      "activations/layer7_attention_weight_max": 92.02175903320312,
      "activations/layer7_attention_weight_min": -92.59099578857422,
      "activations/layer8_attention_weight_max": 39.4613151550293,
      "activations/layer8_attention_weight_min": -41.015464782714844,
      "activations/layer9_attention_weight_max": 31.737436294555664,
      "activations/layer9_attention_weight_min": -34.12199020385742,
      "epoch": 15.63,
      "learning_rate": 4.9685227272727266e-05,
      "loss": 2.7533,
      "step": 269050
    },
    {
      "activations/layer0_attention_weight_max": 15.6019926071167,
      "activations/layer0_attention_weight_min": -12.522587776184082,
      "activations/layer10_attention_weight_max": 34.88916778564453,
      "activations/layer10_attention_weight_min": -33.65589141845703,
      "activations/layer11_attention_weight_max": 36.637672424316406,
      "activations/layer11_attention_weight_min": -33.43955612182617,
      "activations/layer12_attention_weight_max": 24.703880310058594,
      "activations/layer12_attention_weight_min": -25.70261573791504,
      "activations/layer13_attention_weight_max": 37.83380889892578,
      "activations/layer13_attention_weight_min": -34.94209289550781,
      "activations/layer14_attention_weight_max": 41.74230194091797,
      "activations/layer14_attention_weight_min": -36.81205749511719,
      "activations/layer15_attention_weight_max": 35.687442779541016,
      "activations/layer15_attention_weight_min": -32.760101318359375,
      "activations/layer16_attention_weight_max": 29.300798416137695,
      "activations/layer16_attention_weight_min": -27.34760093688965,
      "activations/layer17_attention_weight_max": 29.37688636779785,
      "activations/layer17_attention_weight_min": -25.29234504699707,
      "activations/layer18_attention_weight_max": 29.73480796813965,
      "activations/layer18_attention_weight_min": -24.9161376953125,
      "activations/layer19_attention_weight_max": 32.98712158203125,
      "activations/layer19_attention_weight_min": -30.491743087768555,
      "activations/layer1_attention_weight_max": 18.2847843170166,
      "activations/layer1_attention_weight_min": -18.545448303222656,
      "activations/layer20_attention_weight_max": 28.45812225341797,
      "activations/layer20_attention_weight_min": -25.082307815551758,
      "activations/layer21_attention_weight_max": 30.076269149780273,
      "activations/layer21_attention_weight_min": -23.63470458984375,
      "activations/layer22_attention_weight_max": 38.241390228271484,
      "activations/layer22_attention_weight_min": -30.099239349365234,
      "activations/layer23_attention_weight_max": 30.43155288696289,
      "activations/layer23_attention_weight_min": -28.08557891845703,
      "activations/layer2_attention_weight_max": 33.80232238769531,
      "activations/layer2_attention_weight_min": -34.04499053955078,
      "activations/layer3_attention_weight_max": 98.27346801757812,
      "activations/layer3_attention_weight_min": -101.1761245727539,
      "activations/layer4_attention_weight_max": 108.46544647216797,
      "activations/layer4_attention_weight_min": -110.8627700805664,
      "activations/layer5_attention_weight_max": 52.7042236328125,
      "activations/layer5_attention_weight_min": -59.26802062988281,
      "activations/layer6_attention_weight_max": 44.64286804199219,
      "activations/layer6_attention_weight_min": -47.567684173583984,
      "activations/layer7_attention_weight_max": 95.04478454589844,
      "activations/layer7_attention_weight_min": -91.06292724609375,
      "activations/layer8_attention_weight_max": 42.55302810668945,
      "activations/layer8_attention_weight_min": -43.41033935546875,
      "activations/layer9_attention_weight_max": 32.19307327270508,
      "activations/layer9_attention_weight_min": -34.233821868896484,
      "epoch": 15.64,
      "learning_rate": 4.966628787878788e-05,
      "loss": 2.761,
      "step": 269100
    },
    {
      "activations/layer0_attention_weight_max": 15.948368072509766,
      "activations/layer0_attention_weight_min": -11.87691879272461,
      "activations/layer10_attention_weight_max": 30.005489349365234,
      "activations/layer10_attention_weight_min": -30.442087173461914,
      "activations/layer11_attention_weight_max": 32.58230972290039,
      "activations/layer11_attention_weight_min": -31.440845489501953,
      "activations/layer12_attention_weight_max": 29.620365142822266,
      "activations/layer12_attention_weight_min": -26.007795333862305,
      "activations/layer13_attention_weight_max": 39.626609802246094,
      "activations/layer13_attention_weight_min": -35.85395812988281,
      "activations/layer14_attention_weight_max": 42.792564392089844,
      "activations/layer14_attention_weight_min": -39.9440803527832,
      "activations/layer15_attention_weight_max": 39.886436462402344,
      "activations/layer15_attention_weight_min": -37.0582160949707,
      "activations/layer16_attention_weight_max": 29.966951370239258,
      "activations/layer16_attention_weight_min": -29.452863693237305,
      "activations/layer17_attention_weight_max": 32.868316650390625,
      "activations/layer17_attention_weight_min": -26.85795783996582,
      "activations/layer18_attention_weight_max": 33.987548828125,
      "activations/layer18_attention_weight_min": -27.235177993774414,
      "activations/layer19_attention_weight_max": 36.89781188964844,
      "activations/layer19_attention_weight_min": -31.9550724029541,
      "activations/layer1_attention_weight_max": 16.35502052307129,
      "activations/layer1_attention_weight_min": -14.113486289978027,
      "activations/layer20_attention_weight_max": 32.767520904541016,
      "activations/layer20_attention_weight_min": -26.91876220703125,
      "activations/layer21_attention_weight_max": 31.310068130493164,
      "activations/layer21_attention_weight_min": -26.52448081970215,
      "activations/layer22_attention_weight_max": 45.718231201171875,
      "activations/layer22_attention_weight_min": -32.74827575683594,
      "activations/layer23_attention_weight_max": 35.38249206542969,
      "activations/layer23_attention_weight_min": -26.696470260620117,
      "activations/layer2_attention_weight_max": 31.392070770263672,
      "activations/layer2_attention_weight_min": -31.69331169128418,
      "activations/layer3_attention_weight_max": 92.97786712646484,
      "activations/layer3_attention_weight_min": -95.81124114990234,
      "activations/layer4_attention_weight_max": 100.66896057128906,
      "activations/layer4_attention_weight_min": -103.70916748046875,
      "activations/layer5_attention_weight_max": 46.84190368652344,
      "activations/layer5_attention_weight_min": -61.80331039428711,
      "activations/layer6_attention_weight_max": 41.33275604248047,
      "activations/layer6_attention_weight_min": -45.21312713623047,
      "activations/layer7_attention_weight_max": 86.09487915039062,
      "activations/layer7_attention_weight_min": -88.23226165771484,
      "activations/layer8_attention_weight_max": 39.84736633300781,
      "activations/layer8_attention_weight_min": -41.7225456237793,
      "activations/layer9_attention_weight_max": 30.371448516845703,
      "activations/layer9_attention_weight_min": -31.176986694335938,
      "epoch": 15.64,
      "learning_rate": 4.964734848484848e-05,
      "loss": 2.7627,
      "step": 269150
    },
    {
      "activations/layer0_attention_weight_max": 15.37787914276123,
      "activations/layer0_attention_weight_min": -13.206489562988281,
      "activations/layer10_attention_weight_max": 34.71010208129883,
      "activations/layer10_attention_weight_min": -35.45325469970703,
      "activations/layer11_attention_weight_max": 34.83964920043945,
      "activations/layer11_attention_weight_min": -33.99761199951172,
      "activations/layer12_attention_weight_max": 27.962604522705078,
      "activations/layer12_attention_weight_min": -27.07389259338379,
      "activations/layer13_attention_weight_max": 44.54706573486328,
      "activations/layer13_attention_weight_min": -35.77930450439453,
      "activations/layer14_attention_weight_max": 44.55833053588867,
      "activations/layer14_attention_weight_min": -37.13313293457031,
      "activations/layer15_attention_weight_max": 44.08906936645508,
      "activations/layer15_attention_weight_min": -35.59962844848633,
      "activations/layer16_attention_weight_max": 34.02042770385742,
      "activations/layer16_attention_weight_min": -29.197662353515625,
      "activations/layer17_attention_weight_max": 32.187286376953125,
      "activations/layer17_attention_weight_min": -26.93239402770996,
      "activations/layer18_attention_weight_max": 33.446468353271484,
      "activations/layer18_attention_weight_min": -25.2358455657959,
      "activations/layer19_attention_weight_max": 36.604244232177734,
      "activations/layer19_attention_weight_min": -31.288101196289062,
      "activations/layer1_attention_weight_max": 16.173255920410156,
      "activations/layer1_attention_weight_min": -16.195100784301758,
      "activations/layer20_attention_weight_max": 31.477312088012695,
      "activations/layer20_attention_weight_min": -25.309003829956055,
      "activations/layer21_attention_weight_max": 31.087108612060547,
      "activations/layer21_attention_weight_min": -25.199296951293945,
      "activations/layer22_attention_weight_max": 43.44438934326172,
      "activations/layer22_attention_weight_min": -32.471073150634766,
      "activations/layer23_attention_weight_max": 32.76338577270508,
      "activations/layer23_attention_weight_min": -26.237369537353516,
      "activations/layer2_attention_weight_max": 33.2745361328125,
      "activations/layer2_attention_weight_min": -33.644168853759766,
      "activations/layer3_attention_weight_max": 96.43415069580078,
      "activations/layer3_attention_weight_min": -98.4017105102539,
      "activations/layer4_attention_weight_max": 106.9110107421875,
      "activations/layer4_attention_weight_min": -107.1371841430664,
      "activations/layer5_attention_weight_max": 53.4316520690918,
      "activations/layer5_attention_weight_min": -62.75817108154297,
      "activations/layer6_attention_weight_max": 44.12187194824219,
      "activations/layer6_attention_weight_min": -46.00967788696289,
      "activations/layer7_attention_weight_max": 91.39342498779297,
      "activations/layer7_attention_weight_min": -100.7978515625,
      "activations/layer8_attention_weight_max": 42.032623291015625,
      "activations/layer8_attention_weight_min": -43.52564239501953,
      "activations/layer9_attention_weight_max": 35.041751861572266,
      "activations/layer9_attention_weight_min": -34.76216506958008,
      "epoch": 15.64,
      "learning_rate": 4.9628409090909084e-05,
      "loss": 2.7559,
      "step": 269200
    },
    {
      "activations/layer0_attention_weight_max": 16.208372116088867,
      "activations/layer0_attention_weight_min": -12.660872459411621,
      "activations/layer10_attention_weight_max": 32.74279022216797,
      "activations/layer10_attention_weight_min": -36.39296340942383,
      "activations/layer11_attention_weight_max": 33.333656311035156,
      "activations/layer11_attention_weight_min": -32.587860107421875,
      "activations/layer12_attention_weight_max": 25.19881820678711,
      "activations/layer12_attention_weight_min": -27.898725509643555,
      "activations/layer13_attention_weight_max": 38.89037322998047,
      "activations/layer13_attention_weight_min": -34.47576141357422,
      "activations/layer14_attention_weight_max": 44.54146957397461,
      "activations/layer14_attention_weight_min": -38.05449676513672,
      "activations/layer15_attention_weight_max": 38.3010368347168,
      "activations/layer15_attention_weight_min": -35.142723083496094,
      "activations/layer16_attention_weight_max": 32.310237884521484,
      "activations/layer16_attention_weight_min": -27.670141220092773,
      "activations/layer17_attention_weight_max": 30.63101577758789,
      "activations/layer17_attention_weight_min": -26.689804077148438,
      "activations/layer18_attention_weight_max": 34.934566497802734,
      "activations/layer18_attention_weight_min": -25.94980239868164,
      "activations/layer19_attention_weight_max": 34.74949645996094,
      "activations/layer19_attention_weight_min": -30.64981460571289,
      "activations/layer1_attention_weight_max": 15.952335357666016,
      "activations/layer1_attention_weight_min": -15.216912269592285,
      "activations/layer20_attention_weight_max": 30.211681365966797,
      "activations/layer20_attention_weight_min": -26.259471893310547,
      "activations/layer21_attention_weight_max": 29.68617820739746,
      "activations/layer21_attention_weight_min": -26.984548568725586,
      "activations/layer22_attention_weight_max": 47.94516372680664,
      "activations/layer22_attention_weight_min": -32.040164947509766,
      "activations/layer23_attention_weight_max": 32.81594467163086,
      "activations/layer23_attention_weight_min": -25.913793563842773,
      "activations/layer2_attention_weight_max": 33.71906280517578,
      "activations/layer2_attention_weight_min": -36.044342041015625,
      "activations/layer3_attention_weight_max": 99.21953582763672,
      "activations/layer3_attention_weight_min": -106.12249755859375,
      "activations/layer4_attention_weight_max": 107.0567626953125,
      "activations/layer4_attention_weight_min": -104.22737121582031,
      "activations/layer5_attention_weight_max": 50.87845993041992,
      "activations/layer5_attention_weight_min": -61.94350814819336,
      "activations/layer6_attention_weight_max": 44.19553756713867,
      "activations/layer6_attention_weight_min": -46.149166107177734,
      "activations/layer7_attention_weight_max": 88.44075775146484,
      "activations/layer7_attention_weight_min": -94.88516235351562,
      "activations/layer8_attention_weight_max": 41.01808166503906,
      "activations/layer8_attention_weight_min": -43.119300842285156,
      "activations/layer9_attention_weight_max": 30.17929458618164,
      "activations/layer9_attention_weight_min": -33.60693359375,
      "epoch": 15.64,
      "learning_rate": 4.960946969696969e-05,
      "loss": 2.7452,
      "step": 269250
    },
    {
      "activations/layer0_attention_weight_max": 16.18665313720703,
      "activations/layer0_attention_weight_min": -13.188021659851074,
      "activations/layer10_attention_weight_max": 34.54829025268555,
      "activations/layer10_attention_weight_min": -32.22236251831055,
      "activations/layer11_attention_weight_max": 33.40961456298828,
      "activations/layer11_attention_weight_min": -34.70060729980469,
      "activations/layer12_attention_weight_max": 30.540328979492188,
      "activations/layer12_attention_weight_min": -28.567359924316406,
      "activations/layer13_attention_weight_max": 41.46086502075195,
      "activations/layer13_attention_weight_min": -35.930908203125,
      "activations/layer14_attention_weight_max": 44.88728713989258,
      "activations/layer14_attention_weight_min": -38.31306838989258,
      "activations/layer15_attention_weight_max": 39.55868148803711,
      "activations/layer15_attention_weight_min": -34.15152359008789,
      "activations/layer16_attention_weight_max": 29.538570404052734,
      "activations/layer16_attention_weight_min": -29.327272415161133,
      "activations/layer17_attention_weight_max": 29.759498596191406,
      "activations/layer17_attention_weight_min": -25.787263870239258,
      "activations/layer18_attention_weight_max": 33.632022857666016,
      "activations/layer18_attention_weight_min": -25.9464054107666,
      "activations/layer19_attention_weight_max": 33.38200759887695,
      "activations/layer19_attention_weight_min": -31.032461166381836,
      "activations/layer1_attention_weight_max": 16.41740608215332,
      "activations/layer1_attention_weight_min": -15.217517852783203,
      "activations/layer20_attention_weight_max": 27.645235061645508,
      "activations/layer20_attention_weight_min": -26.01939582824707,
      "activations/layer21_attention_weight_max": 29.099124908447266,
      "activations/layer21_attention_weight_min": -26.64175033569336,
      "activations/layer22_attention_weight_max": 41.3660774230957,
      "activations/layer22_attention_weight_min": -31.57895278930664,
      "activations/layer23_attention_weight_max": 31.39655303955078,
      "activations/layer23_attention_weight_min": -26.229801177978516,
      "activations/layer2_attention_weight_max": 33.01881408691406,
      "activations/layer2_attention_weight_min": -33.426517486572266,
      "activations/layer3_attention_weight_max": 96.68151092529297,
      "activations/layer3_attention_weight_min": -100.98542022705078,
      "activations/layer4_attention_weight_max": 111.62422943115234,
      "activations/layer4_attention_weight_min": -109.26289367675781,
      "activations/layer5_attention_weight_max": 50.67524337768555,
      "activations/layer5_attention_weight_min": -61.71891403198242,
      "activations/layer6_attention_weight_max": 43.845314025878906,
      "activations/layer6_attention_weight_min": -44.995887756347656,
      "activations/layer7_attention_weight_max": 90.35614013671875,
      "activations/layer7_attention_weight_min": -91.21182250976562,
      "activations/layer8_attention_weight_max": 39.2862548828125,
      "activations/layer8_attention_weight_min": -41.45740509033203,
      "activations/layer9_attention_weight_max": 32.64775466918945,
      "activations/layer9_attention_weight_min": -33.399505615234375,
      "epoch": 15.65,
      "learning_rate": 4.95905303030303e-05,
      "loss": 2.741,
      "step": 269300
    },
    {
      "activations/layer0_attention_weight_max": 16.296682357788086,
      "activations/layer0_attention_weight_min": -14.117891311645508,
      "activations/layer10_attention_weight_max": 29.57611083984375,
      "activations/layer10_attention_weight_min": -31.782588958740234,
      "activations/layer11_attention_weight_max": 29.872365951538086,
      "activations/layer11_attention_weight_min": -30.592899322509766,
      "activations/layer12_attention_weight_max": 23.99802017211914,
      "activations/layer12_attention_weight_min": -26.717018127441406,
      "activations/layer13_attention_weight_max": 35.40278244018555,
      "activations/layer13_attention_weight_min": -35.38896942138672,
      "activations/layer14_attention_weight_max": 37.52048110961914,
      "activations/layer14_attention_weight_min": -36.618751525878906,
      "activations/layer15_attention_weight_max": 33.73812484741211,
      "activations/layer15_attention_weight_min": -34.11237335205078,
      "activations/layer16_attention_weight_max": 26.588254928588867,
      "activations/layer16_attention_weight_min": -29.337533950805664,
      "activations/layer17_attention_weight_max": 27.581546783447266,
      "activations/layer17_attention_weight_min": -24.931106567382812,
      "activations/layer18_attention_weight_max": 31.21006965637207,
      "activations/layer18_attention_weight_min": -25.340232849121094,
      "activations/layer19_attention_weight_max": 33.29949951171875,
      "activations/layer19_attention_weight_min": -31.550748825073242,
      "activations/layer1_attention_weight_max": 19.75916862487793,
      "activations/layer1_attention_weight_min": -18.148332595825195,
      "activations/layer20_attention_weight_max": 29.589771270751953,
      "activations/layer20_attention_weight_min": -25.423755645751953,
      "activations/layer21_attention_weight_max": 28.261215209960938,
      "activations/layer21_attention_weight_min": -24.19131851196289,
      "activations/layer22_attention_weight_max": 39.6388053894043,
      "activations/layer22_attention_weight_min": -29.24339485168457,
      "activations/layer23_attention_weight_max": 29.494810104370117,
      "activations/layer23_attention_weight_min": -26.23257827758789,
      "activations/layer2_attention_weight_max": 33.09181213378906,
      "activations/layer2_attention_weight_min": -33.39182662963867,
      "activations/layer3_attention_weight_max": 97.35440826416016,
      "activations/layer3_attention_weight_min": -105.54373168945312,
      "activations/layer4_attention_weight_max": 109.10888671875,
      "activations/layer4_attention_weight_min": -109.80538177490234,
      "activations/layer5_attention_weight_max": 51.67797088623047,
      "activations/layer5_attention_weight_min": -62.10602951049805,
      "activations/layer6_attention_weight_max": 40.981353759765625,
      "activations/layer6_attention_weight_min": -43.559574127197266,
      "activations/layer7_attention_weight_max": 85.01362609863281,
      "activations/layer7_attention_weight_min": -90.30419921875,
      "activations/layer8_attention_weight_max": 37.36038589477539,
      "activations/layer8_attention_weight_min": -40.25777053833008,
      "activations/layer9_attention_weight_max": 29.82343864440918,
      "activations/layer9_attention_weight_min": -31.218069076538086,
      "epoch": 15.65,
      "learning_rate": 4.957159090909091e-05,
      "loss": 2.751,
      "step": 269350
    },
    {
      "activations/layer0_attention_weight_max": 15.562834739685059,
      "activations/layer0_attention_weight_min": -11.973550796508789,
      "activations/layer10_attention_weight_max": 33.38554000854492,
      "activations/layer10_attention_weight_min": -35.74188995361328,
      "activations/layer11_attention_weight_max": 33.498558044433594,
      "activations/layer11_attention_weight_min": -36.46360778808594,
      "activations/layer12_attention_weight_max": 27.41383171081543,
      "activations/layer12_attention_weight_min": -28.522193908691406,
      "activations/layer13_attention_weight_max": 39.148433685302734,
      "activations/layer13_attention_weight_min": -35.3131103515625,
      "activations/layer14_attention_weight_max": 44.05906295776367,
      "activations/layer14_attention_weight_min": -38.11506652832031,
      "activations/layer15_attention_weight_max": 38.37690734863281,
      "activations/layer15_attention_weight_min": -36.792842864990234,
      "activations/layer16_attention_weight_max": 32.15827941894531,
      "activations/layer16_attention_weight_min": -29.04071807861328,
      "activations/layer17_attention_weight_max": 29.01420783996582,
      "activations/layer17_attention_weight_min": -25.65235137939453,
      "activations/layer18_attention_weight_max": 30.31396484375,
      "activations/layer18_attention_weight_min": -24.241884231567383,
      "activations/layer19_attention_weight_max": 35.15517044067383,
      "activations/layer19_attention_weight_min": -30.3084716796875,
      "activations/layer1_attention_weight_max": 15.778661727905273,
      "activations/layer1_attention_weight_min": -13.295458793640137,
      "activations/layer20_attention_weight_max": 31.231000900268555,
      "activations/layer20_attention_weight_min": -25.548070907592773,
      "activations/layer21_attention_weight_max": 27.86507797241211,
      "activations/layer21_attention_weight_min": -25.21811294555664,
      "activations/layer22_attention_weight_max": 46.83674621582031,
      "activations/layer22_attention_weight_min": -30.58547592163086,
      "activations/layer23_attention_weight_max": 37.752220153808594,
      "activations/layer23_attention_weight_min": -26.715564727783203,
      "activations/layer2_attention_weight_max": 31.411029815673828,
      "activations/layer2_attention_weight_min": -30.757320404052734,
      "activations/layer3_attention_weight_max": 98.05027770996094,
      "activations/layer3_attention_weight_min": -97.67737579345703,
      "activations/layer4_attention_weight_max": 103.28450012207031,
      "activations/layer4_attention_weight_min": -106.4010238647461,
      "activations/layer5_attention_weight_max": 49.936336517333984,
      "activations/layer5_attention_weight_min": -62.81254577636719,
      "activations/layer6_attention_weight_max": 43.33791732788086,
      "activations/layer6_attention_weight_min": -47.45345687866211,
      "activations/layer7_attention_weight_max": 93.6408920288086,
      "activations/layer7_attention_weight_min": -93.7353286743164,
      "activations/layer8_attention_weight_max": 41.33746337890625,
      "activations/layer8_attention_weight_min": -45.1774787902832,
      "activations/layer9_attention_weight_max": 31.563024520874023,
      "activations/layer9_attention_weight_min": -33.77621078491211,
      "epoch": 15.65,
      "learning_rate": 4.955265151515151e-05,
      "loss": 2.7473,
      "step": 269400
    },
    {
      "activations/layer0_attention_weight_max": 15.848719596862793,
      "activations/layer0_attention_weight_min": -12.387174606323242,
      "activations/layer10_attention_weight_max": 37.07912826538086,
      "activations/layer10_attention_weight_min": -36.465065002441406,
      "activations/layer11_attention_weight_max": 36.993019104003906,
      "activations/layer11_attention_weight_min": -37.85514831542969,
      "activations/layer12_attention_weight_max": 28.785598754882812,
      "activations/layer12_attention_weight_min": -28.22319793701172,
      "activations/layer13_attention_weight_max": 44.56283187866211,
      "activations/layer13_attention_weight_min": -37.77491760253906,
      "activations/layer14_attention_weight_max": 49.70917892456055,
      "activations/layer14_attention_weight_min": -42.98720169067383,
      "activations/layer15_attention_weight_max": 46.575904846191406,
      "activations/layer15_attention_weight_min": -38.35502624511719,
      "activations/layer16_attention_weight_max": 33.78772735595703,
      "activations/layer16_attention_weight_min": -29.869792938232422,
      "activations/layer17_attention_weight_max": 34.14691925048828,
      "activations/layer17_attention_weight_min": -28.47060203552246,
      "activations/layer18_attention_weight_max": 31.300880432128906,
      "activations/layer18_attention_weight_min": -26.611385345458984,
      "activations/layer19_attention_weight_max": 37.974979400634766,
      "activations/layer19_attention_weight_min": -31.725967407226562,
      "activations/layer1_attention_weight_max": 16.188140869140625,
      "activations/layer1_attention_weight_min": -15.168644905090332,
      "activations/layer20_attention_weight_max": 33.083187103271484,
      "activations/layer20_attention_weight_min": -25.523452758789062,
      "activations/layer21_attention_weight_max": 33.08360290527344,
      "activations/layer21_attention_weight_min": -27.628219604492188,
      "activations/layer22_attention_weight_max": 44.013240814208984,
      "activations/layer22_attention_weight_min": -32.04883575439453,
      "activations/layer23_attention_weight_max": 35.157325744628906,
      "activations/layer23_attention_weight_min": -27.412059783935547,
      "activations/layer2_attention_weight_max": 33.06904220581055,
      "activations/layer2_attention_weight_min": -33.561126708984375,
      "activations/layer3_attention_weight_max": 98.05825805664062,
      "activations/layer3_attention_weight_min": -101.84077453613281,
      "activations/layer4_attention_weight_max": 108.08953857421875,
      "activations/layer4_attention_weight_min": -112.1100082397461,
      "activations/layer5_attention_weight_max": 54.396080017089844,
      "activations/layer5_attention_weight_min": -61.858375549316406,
      "activations/layer6_attention_weight_max": 48.10105895996094,
      "activations/layer6_attention_weight_min": -45.934547424316406,
      "activations/layer7_attention_weight_max": 90.10590362548828,
      "activations/layer7_attention_weight_min": -92.07632446289062,
      "activations/layer8_attention_weight_max": 42.40346145629883,
      "activations/layer8_attention_weight_min": -43.15687942504883,
      "activations/layer9_attention_weight_max": 35.09352111816406,
      "activations/layer9_attention_weight_min": -36.631107330322266,
      "epoch": 15.66,
      "learning_rate": 4.953371212121211e-05,
      "loss": 2.748,
      "step": 269450
    },
    {
      "activations/layer0_attention_weight_max": 15.492959022521973,
      "activations/layer0_attention_weight_min": -12.6485013961792,
      "activations/layer10_attention_weight_max": 33.65892791748047,
      "activations/layer10_attention_weight_min": -35.99727249145508,
      "activations/layer11_attention_weight_max": 34.24507141113281,
      "activations/layer11_attention_weight_min": -32.823978424072266,
      "activations/layer12_attention_weight_max": 27.19799041748047,
      "activations/layer12_attention_weight_min": -25.133968353271484,
      "activations/layer13_attention_weight_max": 39.118865966796875,
      "activations/layer13_attention_weight_min": -34.49701690673828,
      "activations/layer14_attention_weight_max": 40.003807067871094,
      "activations/layer14_attention_weight_min": -36.03927993774414,
      "activations/layer15_attention_weight_max": 37.2302131652832,
      "activations/layer15_attention_weight_min": -33.663063049316406,
      "activations/layer16_attention_weight_max": 30.888378143310547,
      "activations/layer16_attention_weight_min": -26.46782875061035,
      "activations/layer17_attention_weight_max": 29.6806583404541,
      "activations/layer17_attention_weight_min": -27.53134536743164,
      "activations/layer18_attention_weight_max": 29.405046463012695,
      "activations/layer18_attention_weight_min": -25.329504013061523,
      "activations/layer19_attention_weight_max": 32.48551559448242,
      "activations/layer19_attention_weight_min": -32.333003997802734,
      "activations/layer1_attention_weight_max": 16.588716506958008,
      "activations/layer1_attention_weight_min": -14.187155723571777,
      "activations/layer20_attention_weight_max": 29.474258422851562,
      "activations/layer20_attention_weight_min": -25.07491111755371,
      "activations/layer21_attention_weight_max": 29.31197738647461,
      "activations/layer21_attention_weight_min": -25.05937957763672,
      "activations/layer22_attention_weight_max": 40.37876510620117,
      "activations/layer22_attention_weight_min": -31.028522491455078,
      "activations/layer23_attention_weight_max": 30.61610984802246,
      "activations/layer23_attention_weight_min": -25.932945251464844,
      "activations/layer2_attention_weight_max": 34.4854850769043,
      "activations/layer2_attention_weight_min": -33.6761474609375,
      "activations/layer3_attention_weight_max": 101.67469787597656,
      "activations/layer3_attention_weight_min": -99.50125122070312,
      "activations/layer4_attention_weight_max": 106.04229736328125,
      "activations/layer4_attention_weight_min": -108.29011535644531,
      "activations/layer5_attention_weight_max": 52.54460906982422,
      "activations/layer5_attention_weight_min": -61.160194396972656,
      "activations/layer6_attention_weight_max": 42.441383361816406,
      "activations/layer6_attention_weight_min": -44.84743118286133,
      "activations/layer7_attention_weight_max": 89.27812194824219,
      "activations/layer7_attention_weight_min": -88.87413787841797,
      "activations/layer8_attention_weight_max": 42.11075973510742,
      "activations/layer8_attention_weight_min": -42.038780212402344,
      "activations/layer9_attention_weight_max": 33.425621032714844,
      "activations/layer9_attention_weight_min": -34.279850006103516,
      "epoch": 15.66,
      "learning_rate": 4.9514772727272726e-05,
      "loss": 2.7419,
      "step": 269500
    },
    {
      "activations/layer0_attention_weight_max": 16.11932945251465,
      "activations/layer0_attention_weight_min": -12.325494766235352,
      "activations/layer10_attention_weight_max": 33.94260787963867,
      "activations/layer10_attention_weight_min": -34.54074478149414,
      "activations/layer11_attention_weight_max": 33.997711181640625,
      "activations/layer11_attention_weight_min": -35.88877487182617,
      "activations/layer12_attention_weight_max": 26.51215362548828,
      "activations/layer12_attention_weight_min": -26.11753273010254,
      "activations/layer13_attention_weight_max": 37.88982391357422,
      "activations/layer13_attention_weight_min": -35.408687591552734,
      "activations/layer14_attention_weight_max": 40.40459442138672,
      "activations/layer14_attention_weight_min": -38.465152740478516,
      "activations/layer15_attention_weight_max": 35.72813034057617,
      "activations/layer15_attention_weight_min": -35.78768539428711,
      "activations/layer16_attention_weight_max": 29.73443603515625,
      "activations/layer16_attention_weight_min": -26.695281982421875,
      "activations/layer17_attention_weight_max": 31.95463752746582,
      "activations/layer17_attention_weight_min": -27.322336196899414,
      "activations/layer18_attention_weight_max": 32.13787078857422,
      "activations/layer18_attention_weight_min": -25.09193229675293,
      "activations/layer19_attention_weight_max": 34.07332229614258,
      "activations/layer19_attention_weight_min": -30.641605377197266,
      "activations/layer1_attention_weight_max": 16.71303939819336,
      "activations/layer1_attention_weight_min": -14.359543800354004,
      "activations/layer20_attention_weight_max": 30.247474670410156,
      "activations/layer20_attention_weight_min": -25.414798736572266,
      "activations/layer21_attention_weight_max": 29.47446632385254,
      "activations/layer21_attention_weight_min": -25.565519332885742,
      "activations/layer22_attention_weight_max": 45.12770080566406,
      "activations/layer22_attention_weight_min": -31.44166374206543,
      "activations/layer23_attention_weight_max": 31.682302474975586,
      "activations/layer23_attention_weight_min": -25.26601219177246,
      "activations/layer2_attention_weight_max": 32.72243881225586,
      "activations/layer2_attention_weight_min": -33.1652717590332,
      "activations/layer3_attention_weight_max": 94.38103485107422,
      "activations/layer3_attention_weight_min": -96.7008056640625,
      "activations/layer4_attention_weight_max": 103.71009826660156,
      "activations/layer4_attention_weight_min": -102.7040786743164,
      "activations/layer5_attention_weight_max": 49.99066925048828,
      "activations/layer5_attention_weight_min": -63.624176025390625,
      "activations/layer6_attention_weight_max": 43.274864196777344,
      "activations/layer6_attention_weight_min": -45.22473907470703,
      "activations/layer7_attention_weight_max": 87.58474731445312,
      "activations/layer7_attention_weight_min": -90.79605102539062,
      "activations/layer8_attention_weight_max": 38.51530075073242,
      "activations/layer8_attention_weight_min": -42.68546676635742,
      "activations/layer9_attention_weight_max": 31.871784210205078,
      "activations/layer9_attention_weight_min": -34.42350387573242,
      "epoch": 15.66,
      "learning_rate": 4.949583333333333e-05,
      "loss": 2.7496,
      "step": 269550
    },
    {
      "activations/layer0_attention_weight_max": 16.159229278564453,
      "activations/layer0_attention_weight_min": -12.320077896118164,
      "activations/layer10_attention_weight_max": 31.55963706970215,
      "activations/layer10_attention_weight_min": -33.05270004272461,
      "activations/layer11_attention_weight_max": 32.75175476074219,
      "activations/layer11_attention_weight_min": -33.38406753540039,
      "activations/layer12_attention_weight_max": 28.179750442504883,
      "activations/layer12_attention_weight_min": -26.809274673461914,
      "activations/layer13_attention_weight_max": 41.49679183959961,
      "activations/layer13_attention_weight_min": -36.1594352722168,
      "activations/layer14_attention_weight_max": 46.70044708251953,
      "activations/layer14_attention_weight_min": -37.53240203857422,
      "activations/layer15_attention_weight_max": 40.77903747558594,
      "activations/layer15_attention_weight_min": -33.161956787109375,
      "activations/layer16_attention_weight_max": 30.90868377685547,
      "activations/layer16_attention_weight_min": -27.985464096069336,
      "activations/layer17_attention_weight_max": 29.6385440826416,
      "activations/layer17_attention_weight_min": -24.578596115112305,
      "activations/layer18_attention_weight_max": 35.935752868652344,
      "activations/layer18_attention_weight_min": -25.84561538696289,
      "activations/layer19_attention_weight_max": 35.38167190551758,
      "activations/layer19_attention_weight_min": -30.05990982055664,
      "activations/layer1_attention_weight_max": 15.987548828125,
      "activations/layer1_attention_weight_min": -14.211735725402832,
      "activations/layer20_attention_weight_max": 31.003860473632812,
      "activations/layer20_attention_weight_min": -24.22774887084961,
      "activations/layer21_attention_weight_max": 30.524028778076172,
      "activations/layer21_attention_weight_min": -23.758703231811523,
      "activations/layer22_attention_weight_max": 44.76405334472656,
      "activations/layer22_attention_weight_min": -29.652572631835938,
      "activations/layer23_attention_weight_max": 32.79266357421875,
      "activations/layer23_attention_weight_min": -25.05304718017578,
      "activations/layer2_attention_weight_max": 32.925514221191406,
      "activations/layer2_attention_weight_min": -31.963050842285156,
      "activations/layer3_attention_weight_max": 93.46312713623047,
      "activations/layer3_attention_weight_min": -94.50841522216797,
      "activations/layer4_attention_weight_max": 102.14669799804688,
      "activations/layer4_attention_weight_min": -105.44029998779297,
      "activations/layer5_attention_weight_max": 51.165794372558594,
      "activations/layer5_attention_weight_min": -60.364356994628906,
      "activations/layer6_attention_weight_max": 42.710567474365234,
      "activations/layer6_attention_weight_min": -45.4096794128418,
      "activations/layer7_attention_weight_max": 88.67318725585938,
      "activations/layer7_attention_weight_min": -88.97664642333984,
      "activations/layer8_attention_weight_max": 38.740169525146484,
      "activations/layer8_attention_weight_min": -41.676109313964844,
      "activations/layer9_attention_weight_max": 30.331207275390625,
      "activations/layer9_attention_weight_min": -33.614990234375,
      "epoch": 15.67,
      "learning_rate": 4.947689393939393e-05,
      "loss": 2.7642,
      "step": 269600
    },
    {
      "activations/layer0_attention_weight_max": 16.123886108398438,
      "activations/layer0_attention_weight_min": -12.66711139678955,
      "activations/layer10_attention_weight_max": 37.78920364379883,
      "activations/layer10_attention_weight_min": -33.356597900390625,
      "activations/layer11_attention_weight_max": 35.87699890136719,
      "activations/layer11_attention_weight_min": -33.33750915527344,
      "activations/layer12_attention_weight_max": 24.408584594726562,
      "activations/layer12_attention_weight_min": -30.343442916870117,
      "activations/layer13_attention_weight_max": 36.38129425048828,
      "activations/layer13_attention_weight_min": -33.52472686767578,
      "activations/layer14_attention_weight_max": 41.300941467285156,
      "activations/layer14_attention_weight_min": -38.2213249206543,
      "activations/layer15_attention_weight_max": 38.50189208984375,
      "activations/layer15_attention_weight_min": -33.35222625732422,
      "activations/layer16_attention_weight_max": 33.64198684692383,
      "activations/layer16_attention_weight_min": -28.675052642822266,
      "activations/layer17_attention_weight_max": 29.94900131225586,
      "activations/layer17_attention_weight_min": -25.17687225341797,
      "activations/layer18_attention_weight_max": 34.02416229248047,
      "activations/layer18_attention_weight_min": -25.139995574951172,
      "activations/layer19_attention_weight_max": 39.8032341003418,
      "activations/layer19_attention_weight_min": -33.07954025268555,
      "activations/layer1_attention_weight_max": 16.56315040588379,
      "activations/layer1_attention_weight_min": -14.328304290771484,
      "activations/layer20_attention_weight_max": 35.15617370605469,
      "activations/layer20_attention_weight_min": -25.748451232910156,
      "activations/layer21_attention_weight_max": 34.59918212890625,
      "activations/layer21_attention_weight_min": -24.017837524414062,
      "activations/layer22_attention_weight_max": 42.50593566894531,
      "activations/layer22_attention_weight_min": -30.841522216796875,
      "activations/layer23_attention_weight_max": 34.744346618652344,
      "activations/layer23_attention_weight_min": -25.505386352539062,
      "activations/layer2_attention_weight_max": 33.647098541259766,
      "activations/layer2_attention_weight_min": -32.075111389160156,
      "activations/layer3_attention_weight_max": 92.47821807861328,
      "activations/layer3_attention_weight_min": -91.91873931884766,
      "activations/layer4_attention_weight_max": 103.218994140625,
      "activations/layer4_attention_weight_min": -97.44290924072266,
      "activations/layer5_attention_weight_max": 52.30174255371094,
      "activations/layer5_attention_weight_min": -61.469139099121094,
      "activations/layer6_attention_weight_max": 43.055702209472656,
      "activations/layer6_attention_weight_min": -42.44626998901367,
      "activations/layer7_attention_weight_max": 99.71420288085938,
      "activations/layer7_attention_weight_min": -92.9817123413086,
      "activations/layer8_attention_weight_max": 44.94252395629883,
      "activations/layer8_attention_weight_min": -42.21062088012695,
      "activations/layer9_attention_weight_max": 33.14838409423828,
      "activations/layer9_attention_weight_min": -34.912227630615234,
      "epoch": 15.67,
      "learning_rate": 4.9457954545454545e-05,
      "loss": 2.7497,
      "step": 269650
    },
    {
      "activations/layer0_attention_weight_max": 16.272865295410156,
      "activations/layer0_attention_weight_min": -15.229106903076172,
      "activations/layer10_attention_weight_max": 36.39042663574219,
      "activations/layer10_attention_weight_min": -35.143741607666016,
      "activations/layer11_attention_weight_max": 34.95130157470703,
      "activations/layer11_attention_weight_min": -34.06522750854492,
      "activations/layer12_attention_weight_max": 27.881559371948242,
      "activations/layer12_attention_weight_min": -26.602876663208008,
      "activations/layer13_attention_weight_max": 41.34820556640625,
      "activations/layer13_attention_weight_min": -36.302486419677734,
      "activations/layer14_attention_weight_max": 45.07861328125,
      "activations/layer14_attention_weight_min": -38.42353820800781,
      "activations/layer15_attention_weight_max": 38.66812515258789,
      "activations/layer15_attention_weight_min": -34.60859680175781,
      "activations/layer16_attention_weight_max": 29.708837509155273,
      "activations/layer16_attention_weight_min": -27.74791145324707,
      "activations/layer17_attention_weight_max": 30.685861587524414,
      "activations/layer17_attention_weight_min": -28.212228775024414,
      "activations/layer18_attention_weight_max": 36.23954391479492,
      "activations/layer18_attention_weight_min": -25.577489852905273,
      "activations/layer19_attention_weight_max": 36.88218688964844,
      "activations/layer19_attention_weight_min": -31.17490005493164,
      "activations/layer1_attention_weight_max": 16.27648162841797,
      "activations/layer1_attention_weight_min": -15.820143699645996,
      "activations/layer20_attention_weight_max": 29.647375106811523,
      "activations/layer20_attention_weight_min": -24.606237411499023,
      "activations/layer21_attention_weight_max": 26.78118896484375,
      "activations/layer21_attention_weight_min": -25.00787925720215,
      "activations/layer22_attention_weight_max": 43.37421417236328,
      "activations/layer22_attention_weight_min": -30.345537185668945,
      "activations/layer23_attention_weight_max": 33.27630615234375,
      "activations/layer23_attention_weight_min": -26.16280174255371,
      "activations/layer2_attention_weight_max": 33.988922119140625,
      "activations/layer2_attention_weight_min": -33.63124465942383,
      "activations/layer3_attention_weight_max": 95.50534057617188,
      "activations/layer3_attention_weight_min": -96.07523345947266,
      "activations/layer4_attention_weight_max": 107.28913879394531,
      "activations/layer4_attention_weight_min": -108.9231185913086,
      "activations/layer5_attention_weight_max": 50.57823944091797,
      "activations/layer5_attention_weight_min": -62.669002532958984,
      "activations/layer6_attention_weight_max": 46.19821548461914,
      "activations/layer6_attention_weight_min": -49.60208511352539,
      "activations/layer7_attention_weight_max": 93.77698516845703,
      "activations/layer7_attention_weight_min": -102.00933837890625,
      "activations/layer8_attention_weight_max": 42.601280212402344,
      "activations/layer8_attention_weight_min": -45.968727111816406,
      "activations/layer9_attention_weight_max": 34.50162124633789,
      "activations/layer9_attention_weight_min": -37.346134185791016,
      "epoch": 15.67,
      "learning_rate": 4.9439015151515146e-05,
      "loss": 2.7457,
      "step": 269700
    },
    {
      "activations/layer0_attention_weight_max": 15.61003303527832,
      "activations/layer0_attention_weight_min": -13.329626083374023,
      "activations/layer10_attention_weight_max": 34.931461334228516,
      "activations/layer10_attention_weight_min": -34.178184509277344,
      "activations/layer11_attention_weight_max": 34.07838821411133,
      "activations/layer11_attention_weight_min": -35.02643585205078,
      "activations/layer12_attention_weight_max": 28.104995727539062,
      "activations/layer12_attention_weight_min": -27.612079620361328,
      "activations/layer13_attention_weight_max": 41.16654586791992,
      "activations/layer13_attention_weight_min": -34.075897216796875,
      "activations/layer14_attention_weight_max": 43.52424240112305,
      "activations/layer14_attention_weight_min": -36.9105110168457,
      "activations/layer15_attention_weight_max": 43.16331100463867,
      "activations/layer15_attention_weight_min": -34.40230178833008,
      "activations/layer16_attention_weight_max": 29.490291595458984,
      "activations/layer16_attention_weight_min": -29.093778610229492,
      "activations/layer17_attention_weight_max": 30.303829193115234,
      "activations/layer17_attention_weight_min": -25.471769332885742,
      "activations/layer18_attention_weight_max": 33.830787658691406,
      "activations/layer18_attention_weight_min": -25.507909774780273,
      "activations/layer19_attention_weight_max": 37.699623107910156,
      "activations/layer19_attention_weight_min": -30.776931762695312,
      "activations/layer1_attention_weight_max": 16.248432159423828,
      "activations/layer1_attention_weight_min": -16.159320831298828,
      "activations/layer20_attention_weight_max": 29.582412719726562,
      "activations/layer20_attention_weight_min": -26.05944061279297,
      "activations/layer21_attention_weight_max": 27.42677879333496,
      "activations/layer21_attention_weight_min": -24.228111267089844,
      "activations/layer22_attention_weight_max": 45.68153381347656,
      "activations/layer22_attention_weight_min": -30.138948440551758,
      "activations/layer23_attention_weight_max": 34.431766510009766,
      "activations/layer23_attention_weight_min": -25.586002349853516,
      "activations/layer2_attention_weight_max": 33.479331970214844,
      "activations/layer2_attention_weight_min": -34.233612060546875,
      "activations/layer3_attention_weight_max": 98.7027816772461,
      "activations/layer3_attention_weight_min": -99.46228790283203,
      "activations/layer4_attention_weight_max": 109.53710174560547,
      "activations/layer4_attention_weight_min": -110.90547180175781,
      "activations/layer5_attention_weight_max": 51.02555847167969,
      "activations/layer5_attention_weight_min": -61.74153137207031,
      "activations/layer6_attention_weight_max": 44.60015106201172,
      "activations/layer6_attention_weight_min": -46.61302947998047,
      "activations/layer7_attention_weight_max": 87.22135925292969,
      "activations/layer7_attention_weight_min": -92.23749542236328,
      "activations/layer8_attention_weight_max": 40.85092544555664,
      "activations/layer8_attention_weight_min": -45.43994140625,
      "activations/layer9_attention_weight_max": 32.54891586303711,
      "activations/layer9_attention_weight_min": -34.65406036376953,
      "epoch": 15.67,
      "learning_rate": 4.9420075757575754e-05,
      "loss": 2.7496,
      "step": 269750
    },
    {
      "activations/layer0_attention_weight_max": 16.019569396972656,
      "activations/layer0_attention_weight_min": -12.328450202941895,
      "activations/layer10_attention_weight_max": 33.813575744628906,
      "activations/layer10_attention_weight_min": -35.7379035949707,
      "activations/layer11_attention_weight_max": 35.034942626953125,
      "activations/layer11_attention_weight_min": -37.068359375,
      "activations/layer12_attention_weight_max": 25.996784210205078,
      "activations/layer12_attention_weight_min": -26.52930450439453,
      "activations/layer13_attention_weight_max": 37.85102462768555,
      "activations/layer13_attention_weight_min": -34.48125457763672,
      "activations/layer14_attention_weight_max": 38.79985809326172,
      "activations/layer14_attention_weight_min": -35.72848129272461,
      "activations/layer15_attention_weight_max": 34.86014938354492,
      "activations/layer15_attention_weight_min": -33.21766662597656,
      "activations/layer16_attention_weight_max": 28.364845275878906,
      "activations/layer16_attention_weight_min": -28.3449649810791,
      "activations/layer17_attention_weight_max": 29.167322158813477,
      "activations/layer17_attention_weight_min": -25.744407653808594,
      "activations/layer18_attention_weight_max": 30.089012145996094,
      "activations/layer18_attention_weight_min": -25.51045799255371,
      "activations/layer19_attention_weight_max": 31.140363693237305,
      "activations/layer19_attention_weight_min": -36.135616302490234,
      "activations/layer1_attention_weight_max": 16.233938217163086,
      "activations/layer1_attention_weight_min": -14.085204124450684,
      "activations/layer20_attention_weight_max": 28.43904685974121,
      "activations/layer20_attention_weight_min": -26.260379791259766,
      "activations/layer21_attention_weight_max": 28.58104133605957,
      "activations/layer21_attention_weight_min": -25.653676986694336,
      "activations/layer22_attention_weight_max": 40.78348159790039,
      "activations/layer22_attention_weight_min": -32.862361907958984,
      "activations/layer23_attention_weight_max": 31.933311462402344,
      "activations/layer23_attention_weight_min": -26.157615661621094,
      "activations/layer2_attention_weight_max": 31.871458053588867,
      "activations/layer2_attention_weight_min": -33.31501007080078,
      "activations/layer3_attention_weight_max": 95.69513702392578,
      "activations/layer3_attention_weight_min": -94.83971405029297,
      "activations/layer4_attention_weight_max": 106.51165008544922,
      "activations/layer4_attention_weight_min": -110.1404800415039,
      "activations/layer5_attention_weight_max": 54.58308410644531,
      "activations/layer5_attention_weight_min": -64.34811401367188,
      "activations/layer6_attention_weight_max": 44.881649017333984,
      "activations/layer6_attention_weight_min": -47.441715240478516,
      "activations/layer7_attention_weight_max": 95.43923950195312,
      "activations/layer7_attention_weight_min": -96.27429962158203,
      "activations/layer8_attention_weight_max": 41.023109436035156,
      "activations/layer8_attention_weight_min": -41.80253982543945,
      "activations/layer9_attention_weight_max": 32.09716033935547,
      "activations/layer9_attention_weight_min": -33.05924987792969,
      "epoch": 15.68,
      "learning_rate": 4.9401136363636356e-05,
      "loss": 2.7393,
      "step": 269800
    },
    {
      "activations/layer0_attention_weight_max": 17.486085891723633,
      "activations/layer0_attention_weight_min": -12.817386627197266,
      "activations/layer10_attention_weight_max": 34.62881088256836,
      "activations/layer10_attention_weight_min": -33.2955436706543,
      "activations/layer11_attention_weight_max": 34.79056930541992,
      "activations/layer11_attention_weight_min": -34.560272216796875,
      "activations/layer12_attention_weight_max": 27.816829681396484,
      "activations/layer12_attention_weight_min": -27.128028869628906,
      "activations/layer13_attention_weight_max": 41.869232177734375,
      "activations/layer13_attention_weight_min": -35.415191650390625,
      "activations/layer14_attention_weight_max": 45.509674072265625,
      "activations/layer14_attention_weight_min": -39.64638900756836,
      "activations/layer15_attention_weight_max": 40.27330780029297,
      "activations/layer15_attention_weight_min": -38.015071868896484,
      "activations/layer16_attention_weight_max": 29.511890411376953,
      "activations/layer16_attention_weight_min": -28.823261260986328,
      "activations/layer17_attention_weight_max": 32.41241455078125,
      "activations/layer17_attention_weight_min": -27.47824478149414,
      "activations/layer18_attention_weight_max": 33.25439453125,
      "activations/layer18_attention_weight_min": -25.9706974029541,
      "activations/layer19_attention_weight_max": 35.87651062011719,
      "activations/layer19_attention_weight_min": -35.44871139526367,
      "activations/layer1_attention_weight_max": 16.56540870666504,
      "activations/layer1_attention_weight_min": -14.89825439453125,
      "activations/layer20_attention_weight_max": 32.08351135253906,
      "activations/layer20_attention_weight_min": -27.56682586669922,
      "activations/layer21_attention_weight_max": 30.10700035095215,
      "activations/layer21_attention_weight_min": -27.544370651245117,
      "activations/layer22_attention_weight_max": 44.41862106323242,
      "activations/layer22_attention_weight_min": -33.94369888305664,
      "activations/layer23_attention_weight_max": 33.998634338378906,
      "activations/layer23_attention_weight_min": -27.244232177734375,
      "activations/layer2_attention_weight_max": 33.639732360839844,
      "activations/layer2_attention_weight_min": -32.819358825683594,
      "activations/layer3_attention_weight_max": 98.18096160888672,
      "activations/layer3_attention_weight_min": -97.04871368408203,
      "activations/layer4_attention_weight_max": 110.3463363647461,
      "activations/layer4_attention_weight_min": -108.02192687988281,
      "activations/layer5_attention_weight_max": 51.854591369628906,
      "activations/layer5_attention_weight_min": -66.67891693115234,
      "activations/layer6_attention_weight_max": 42.79014205932617,
      "activations/layer6_attention_weight_min": -44.41693878173828,
      "activations/layer7_attention_weight_max": 91.93634033203125,
      "activations/layer7_attention_weight_min": -93.22019958496094,
      "activations/layer8_attention_weight_max": 41.57826614379883,
      "activations/layer8_attention_weight_min": -43.132205963134766,
      "activations/layer9_attention_weight_max": 32.97636032104492,
      "activations/layer9_attention_weight_min": -34.85702133178711,
      "epoch": 15.68,
      "learning_rate": 4.938219696969697e-05,
      "loss": 2.7566,
      "step": 269850
    },
    {
      "activations/layer0_attention_weight_max": 15.072587966918945,
      "activations/layer0_attention_weight_min": -12.665336608886719,
      "activations/layer10_attention_weight_max": 34.4821662902832,
      "activations/layer10_attention_weight_min": -37.25712966918945,
      "activations/layer11_attention_weight_max": 35.882659912109375,
      "activations/layer11_attention_weight_min": -39.5843391418457,
      "activations/layer12_attention_weight_max": 27.120410919189453,
      "activations/layer12_attention_weight_min": -26.843263626098633,
      "activations/layer13_attention_weight_max": 41.46487045288086,
      "activations/layer13_attention_weight_min": -36.090972900390625,
      "activations/layer14_attention_weight_max": 44.3819580078125,
      "activations/layer14_attention_weight_min": -38.13880157470703,
      "activations/layer15_attention_weight_max": 41.25843048095703,
      "activations/layer15_attention_weight_min": -35.17388916015625,
      "activations/layer16_attention_weight_max": 33.78630828857422,
      "activations/layer16_attention_weight_min": -29.54094123840332,
      "activations/layer17_attention_weight_max": 37.03996658325195,
      "activations/layer17_attention_weight_min": -26.685626983642578,
      "activations/layer18_attention_weight_max": 34.13518142700195,
      "activations/layer18_attention_weight_min": -25.702072143554688,
      "activations/layer19_attention_weight_max": 37.448402404785156,
      "activations/layer19_attention_weight_min": -32.33193588256836,
      "activations/layer1_attention_weight_max": 16.189939498901367,
      "activations/layer1_attention_weight_min": -15.347970008850098,
      "activations/layer20_attention_weight_max": 33.32203674316406,
      "activations/layer20_attention_weight_min": -27.299633026123047,
      "activations/layer21_attention_weight_max": 31.41292381286621,
      "activations/layer21_attention_weight_min": -25.67862319946289,
      "activations/layer22_attention_weight_max": 42.190940856933594,
      "activations/layer22_attention_weight_min": -32.324424743652344,
      "activations/layer23_attention_weight_max": 34.245933532714844,
      "activations/layer23_attention_weight_min": -28.646657943725586,
      "activations/layer2_attention_weight_max": 36.67911911010742,
      "activations/layer2_attention_weight_min": -32.925437927246094,
      "activations/layer3_attention_weight_max": 102.86956024169922,
      "activations/layer3_attention_weight_min": -100.61225128173828,
      "activations/layer4_attention_weight_max": 111.423583984375,
      "activations/layer4_attention_weight_min": -107.76311492919922,
      "activations/layer5_attention_weight_max": 52.46661376953125,
      "activations/layer5_attention_weight_min": -64.23849487304688,
      "activations/layer6_attention_weight_max": 45.672279357910156,
      "activations/layer6_attention_weight_min": -46.7679557800293,
      "activations/layer7_attention_weight_max": 92.11280822753906,
      "activations/layer7_attention_weight_min": -91.40425109863281,
      "activations/layer8_attention_weight_max": 41.34996795654297,
      "activations/layer8_attention_weight_min": -43.36207962036133,
      "activations/layer9_attention_weight_max": 32.566322326660156,
      "activations/layer9_attention_weight_min": -35.63529968261719,
      "epoch": 15.68,
      "learning_rate": 4.936325757575757e-05,
      "loss": 2.7399,
      "step": 269900
    },
    {
      "activations/layer0_attention_weight_max": 15.957497596740723,
      "activations/layer0_attention_weight_min": -13.85648250579834,
      "activations/layer10_attention_weight_max": 32.981204986572266,
      "activations/layer10_attention_weight_min": -32.63042068481445,
      "activations/layer11_attention_weight_max": 32.18241500854492,
      "activations/layer11_attention_weight_min": -30.908355712890625,
      "activations/layer12_attention_weight_max": 28.815771102905273,
      "activations/layer12_attention_weight_min": -27.316408157348633,
      "activations/layer13_attention_weight_max": 42.30690002441406,
      "activations/layer13_attention_weight_min": -37.89052963256836,
      "activations/layer14_attention_weight_max": 41.755592346191406,
      "activations/layer14_attention_weight_min": -42.672515869140625,
      "activations/layer15_attention_weight_max": 38.56719970703125,
      "activations/layer15_attention_weight_min": -37.980323791503906,
      "activations/layer16_attention_weight_max": 29.368715286254883,
      "activations/layer16_attention_weight_min": -30.38591194152832,
      "activations/layer17_attention_weight_max": 29.851789474487305,
      "activations/layer17_attention_weight_min": -29.04581642150879,
      "activations/layer18_attention_weight_max": 32.30335998535156,
      "activations/layer18_attention_weight_min": -25.40993309020996,
      "activations/layer19_attention_weight_max": 33.673805236816406,
      "activations/layer19_attention_weight_min": -33.11014938354492,
      "activations/layer1_attention_weight_max": 15.552010536193848,
      "activations/layer1_attention_weight_min": -16.2598934173584,
      "activations/layer20_attention_weight_max": 27.33806610107422,
      "activations/layer20_attention_weight_min": -24.71767234802246,
      "activations/layer21_attention_weight_max": 27.072526931762695,
      "activations/layer21_attention_weight_min": -25.54034996032715,
      "activations/layer22_attention_weight_max": 42.06816482543945,
      "activations/layer22_attention_weight_min": -33.59115219116211,
      "activations/layer23_attention_weight_max": 33.45885467529297,
      "activations/layer23_attention_weight_min": -26.45686149597168,
      "activations/layer2_attention_weight_max": 33.999420166015625,
      "activations/layer2_attention_weight_min": -33.502777099609375,
      "activations/layer3_attention_weight_max": 96.29792022705078,
      "activations/layer3_attention_weight_min": -97.38116455078125,
      "activations/layer4_attention_weight_max": 103.30529022216797,
      "activations/layer4_attention_weight_min": -105.6668701171875,
      "activations/layer5_attention_weight_max": 48.29774475097656,
      "activations/layer5_attention_weight_min": -64.00572204589844,
      "activations/layer6_attention_weight_max": 41.117042541503906,
      "activations/layer6_attention_weight_min": -44.91405487060547,
      "activations/layer7_attention_weight_max": 85.3456039428711,
      "activations/layer7_attention_weight_min": -87.88217163085938,
      "activations/layer8_attention_weight_max": 39.26841735839844,
      "activations/layer8_attention_weight_min": -40.33205032348633,
      "activations/layer9_attention_weight_max": 30.905698776245117,
      "activations/layer9_attention_weight_min": -32.976959228515625,
      "epoch": 15.69,
      "learning_rate": 4.9344318181818174e-05,
      "loss": 2.7451,
      "step": 269950
    },
    {
      "activations/layer0_attention_weight_max": 16.248388290405273,
      "activations/layer0_attention_weight_min": -11.950328826904297,
      "activations/layer10_attention_weight_max": 33.96406555175781,
      "activations/layer10_attention_weight_min": -34.4669075012207,
      "activations/layer11_attention_weight_max": 34.13170623779297,
      "activations/layer11_attention_weight_min": -34.967994689941406,
      "activations/layer12_attention_weight_max": 30.40397834777832,
      "activations/layer12_attention_weight_min": -26.731653213500977,
      "activations/layer13_attention_weight_max": 41.23603057861328,
      "activations/layer13_attention_weight_min": -36.64833450317383,
      "activations/layer14_attention_weight_max": 45.4169921875,
      "activations/layer14_attention_weight_min": -37.95473098754883,
      "activations/layer15_attention_weight_max": 41.21409606933594,
      "activations/layer15_attention_weight_min": -34.932762145996094,
      "activations/layer16_attention_weight_max": 32.27191925048828,
      "activations/layer16_attention_weight_min": -27.712526321411133,
      "activations/layer17_attention_weight_max": 33.34709548950195,
      "activations/layer17_attention_weight_min": -26.32810401916504,
      "activations/layer18_attention_weight_max": 33.69874572753906,
      "activations/layer18_attention_weight_min": -25.768693923950195,
      "activations/layer19_attention_weight_max": 37.31999969482422,
      "activations/layer19_attention_weight_min": -30.592126846313477,
      "activations/layer1_attention_weight_max": 15.665210723876953,
      "activations/layer1_attention_weight_min": -14.697561264038086,
      "activations/layer20_attention_weight_max": 32.06487274169922,
      "activations/layer20_attention_weight_min": -24.381145477294922,
      "activations/layer21_attention_weight_max": 33.9134521484375,
      "activations/layer21_attention_weight_min": -26.768918991088867,
      "activations/layer22_attention_weight_max": 45.57328414916992,
      "activations/layer22_attention_weight_min": -34.79258728027344,
      "activations/layer23_attention_weight_max": 38.46046829223633,
      "activations/layer23_attention_weight_min": -28.399002075195312,
      "activations/layer2_attention_weight_max": 36.34662628173828,
      "activations/layer2_attention_weight_min": -32.82337951660156,
      "activations/layer3_attention_weight_max": 97.27357482910156,
      "activations/layer3_attention_weight_min": -93.50146484375,
      "activations/layer4_attention_weight_max": 109.30863189697266,
      "activations/layer4_attention_weight_min": -105.40299224853516,
      "activations/layer5_attention_weight_max": 52.850486755371094,
      "activations/layer5_attention_weight_min": -60.749420166015625,
      "activations/layer6_attention_weight_max": 44.59069061279297,
      "activations/layer6_attention_weight_min": -48.07954788208008,
      "activations/layer7_attention_weight_max": 91.0444564819336,
      "activations/layer7_attention_weight_min": -90.68231964111328,
      "activations/layer8_attention_weight_max": 43.88477325439453,
      "activations/layer8_attention_weight_min": -42.044010162353516,
      "activations/layer9_attention_weight_max": 33.07990264892578,
      "activations/layer9_attention_weight_min": -33.416316986083984,
      "epoch": 15.69,
      "learning_rate": 4.932537878787878e-05,
      "loss": 2.7551,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4475,
      "eval_samples_per_second": 508.314,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4475,
      "eval_openwebtext_samples_per_second": 508.314,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9641,
      "eval_wikitext_samples_per_second": 232.163,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_lambada_loss": 2.447265625,
      "eval_lambada_ppl": 11.556703116117722,
      "eval_lambada_runtime": 9.5454,
      "eval_lambada_samples_per_second": 510.09,
      "step": 270000
    },
    {
      "activations/layer0_attention_weight_max": 16.21944236755371,
      "activations/layer0_attention_weight_min": -12.863991737365723,
      "activations/layer10_attention_weight_max": 35.4772834777832,
      "activations/layer10_attention_weight_min": -33.07503128051758,
      "activations/layer11_attention_weight_max": 36.10066223144531,
      "activations/layer11_attention_weight_min": -33.48877716064453,
      "activations/layer12_attention_weight_max": 28.273273468017578,
      "activations/layer12_attention_weight_min": -25.20743179321289,
      "activations/layer13_attention_weight_max": 39.34495162963867,
      "activations/layer13_attention_weight_min": -34.482574462890625,
      "activations/layer14_attention_weight_max": 43.181800842285156,
      "activations/layer14_attention_weight_min": -37.190757751464844,
      "activations/layer15_attention_weight_max": 39.86957931518555,
      "activations/layer15_attention_weight_min": -33.31277084350586,
      "activations/layer16_attention_weight_max": 30.28122901916504,
      "activations/layer16_attention_weight_min": -28.57250213623047,
      "activations/layer17_attention_weight_max": 33.09351348876953,
      "activations/layer17_attention_weight_min": -28.826786041259766,
      "activations/layer18_attention_weight_max": 31.82049560546875,
      "activations/layer18_attention_weight_min": -26.137720108032227,
      "activations/layer19_attention_weight_max": 36.99724197387695,
      "activations/layer19_attention_weight_min": -35.826900482177734,
      "activations/layer1_attention_weight_max": 15.631133079528809,
      "activations/layer1_attention_weight_min": -14.228659629821777,
      "activations/layer20_attention_weight_max": 33.940711975097656,
      "activations/layer20_attention_weight_min": -28.4263916015625,
      "activations/layer21_attention_weight_max": 33.110591888427734,
      "activations/layer21_attention_weight_min": -26.640398025512695,
      "activations/layer22_attention_weight_max": 48.416748046875,
      "activations/layer22_attention_weight_min": -32.29203414916992,
      "activations/layer23_attention_weight_max": 37.400760650634766,
      "activations/layer23_attention_weight_min": -26.27950668334961,
      "activations/layer2_attention_weight_max": 33.306419372558594,
      "activations/layer2_attention_weight_min": -31.030170440673828,
      "activations/layer3_attention_weight_max": 89.9651107788086,
      "activations/layer3_attention_weight_min": -87.49093627929688,
      "activations/layer4_attention_weight_max": 99.65026092529297,
      "activations/layer4_attention_weight_min": -103.63651275634766,
      "activations/layer5_attention_weight_max": 49.130550384521484,
      "activations/layer5_attention_weight_min": -61.42761993408203,
      "activations/layer6_attention_weight_max": 39.82896423339844,
      "activations/layer6_attention_weight_min": -45.83600616455078,
      "activations/layer7_attention_weight_max": 89.39848327636719,
      "activations/layer7_attention_weight_min": -93.08401489257812,
      "activations/layer8_attention_weight_max": 41.2548713684082,
      "activations/layer8_attention_weight_min": -43.59357452392578,
      "activations/layer9_attention_weight_max": 32.27103042602539,
      "activations/layer9_attention_weight_min": -32.806522369384766,
      "epoch": 15.69,
      "learning_rate": 4.930643939393939e-05,
      "loss": 2.7684,
      "step": 270050
    },
    {
      "activations/layer0_attention_weight_max": 15.942150115966797,
      "activations/layer0_attention_weight_min": -12.808375358581543,
      "activations/layer10_attention_weight_max": 31.357376098632812,
      "activations/layer10_attention_weight_min": -32.71602249145508,
      "activations/layer11_attention_weight_max": 31.810087203979492,
      "activations/layer11_attention_weight_min": -32.105438232421875,
      "activations/layer12_attention_weight_max": 26.73578643798828,
      "activations/layer12_attention_weight_min": -26.36663246154785,
      "activations/layer13_attention_weight_max": 37.22689437866211,
      "activations/layer13_attention_weight_min": -33.4680061340332,
      "activations/layer14_attention_weight_max": 38.84438705444336,
      "activations/layer14_attention_weight_min": -37.648006439208984,
      "activations/layer15_attention_weight_max": 36.700714111328125,
      "activations/layer15_attention_weight_min": -32.633544921875,
      "activations/layer16_attention_weight_max": 28.519752502441406,
      "activations/layer16_attention_weight_min": -28.075319290161133,
      "activations/layer17_attention_weight_max": 30.831329345703125,
      "activations/layer17_attention_weight_min": -25.7995548248291,
      "activations/layer18_attention_weight_max": 32.3105583190918,
      "activations/layer18_attention_weight_min": -25.463794708251953,
      "activations/layer19_attention_weight_max": 31.646011352539062,
      "activations/layer19_attention_weight_min": -30.952716827392578,
      "activations/layer1_attention_weight_max": 15.890558242797852,
      "activations/layer1_attention_weight_min": -14.414678573608398,
      "activations/layer20_attention_weight_max": 27.526716232299805,
      "activations/layer20_attention_weight_min": -25.173233032226562,
      "activations/layer21_attention_weight_max": 27.091264724731445,
      "activations/layer21_attention_weight_min": -24.852895736694336,
      "activations/layer22_attention_weight_max": 39.40534973144531,
      "activations/layer22_attention_weight_min": -28.877065658569336,
      "activations/layer23_attention_weight_max": 33.66658020019531,
      "activations/layer23_attention_weight_min": -25.1513671875,
      "activations/layer2_attention_weight_max": 31.168346405029297,
      "activations/layer2_attention_weight_min": -30.975074768066406,
      "activations/layer3_attention_weight_max": 91.46109008789062,
      "activations/layer3_attention_weight_min": -92.18869018554688,
      "activations/layer4_attention_weight_max": 103.98668670654297,
      "activations/layer4_attention_weight_min": -108.1261978149414,
      "activations/layer5_attention_weight_max": 49.97850799560547,
      "activations/layer5_attention_weight_min": -61.82489013671875,
      "activations/layer6_attention_weight_max": 43.315547943115234,
      "activations/layer6_attention_weight_min": -47.922122955322266,
      "activations/layer7_attention_weight_max": 90.95201873779297,
      "activations/layer7_attention_weight_min": -92.24618530273438,
      "activations/layer8_attention_weight_max": 42.67726516723633,
      "activations/layer8_attention_weight_min": -45.104835510253906,
      "activations/layer9_attention_weight_max": 32.73153305053711,
      "activations/layer9_attention_weight_min": -33.92011260986328,
      "epoch": 15.69,
      "learning_rate": 4.92875e-05,
      "loss": 2.7643,
      "step": 270100
    },
    {
      "activations/layer0_attention_weight_max": 15.56375789642334,
      "activations/layer0_attention_weight_min": -14.7381591796875,
      "activations/layer10_attention_weight_max": 33.328983306884766,
      "activations/layer10_attention_weight_min": -35.68183135986328,
      "activations/layer11_attention_weight_max": 32.39357376098633,
      "activations/layer11_attention_weight_min": -33.06379699707031,
      "activations/layer12_attention_weight_max": 23.377456665039062,
      "activations/layer12_attention_weight_min": -27.464759826660156,
      "activations/layer13_attention_weight_max": 36.35317611694336,
      "activations/layer13_attention_weight_min": -34.48212432861328,
      "activations/layer14_attention_weight_max": 38.92361831665039,
      "activations/layer14_attention_weight_min": -37.13993453979492,
      "activations/layer15_attention_weight_max": 37.23272705078125,
      "activations/layer15_attention_weight_min": -34.51573944091797,
      "activations/layer16_attention_weight_max": 29.956939697265625,
      "activations/layer16_attention_weight_min": -27.679019927978516,
      "activations/layer17_attention_weight_max": 27.857301712036133,
      "activations/layer17_attention_weight_min": -25.28034782409668,
      "activations/layer18_attention_weight_max": 30.919408798217773,
      "activations/layer18_attention_weight_min": -24.2105655670166,
      "activations/layer19_attention_weight_max": 34.15066146850586,
      "activations/layer19_attention_weight_min": -30.589529037475586,
      "activations/layer1_attention_weight_max": 15.32785701751709,
      "activations/layer1_attention_weight_min": -15.9689302444458,
      "activations/layer20_attention_weight_max": 28.770917892456055,
      "activations/layer20_attention_weight_min": -25.16714096069336,
      "activations/layer21_attention_weight_max": 29.622711181640625,
      "activations/layer21_attention_weight_min": -25.376995086669922,
      "activations/layer22_attention_weight_max": 42.35810089111328,
      "activations/layer22_attention_weight_min": -30.489850997924805,
      "activations/layer23_attention_weight_max": 34.94829559326172,
      "activations/layer23_attention_weight_min": -27.23802947998047,
      "activations/layer2_attention_weight_max": 31.740903854370117,
      "activations/layer2_attention_weight_min": -31.651113510131836,
      "activations/layer3_attention_weight_max": 92.70783233642578,
      "activations/layer3_attention_weight_min": -91.62103271484375,
      "activations/layer4_attention_weight_max": 104.05499267578125,
      "activations/layer4_attention_weight_min": -108.04607391357422,
      "activations/layer5_attention_weight_max": 51.68572235107422,
      "activations/layer5_attention_weight_min": -64.4359130859375,
      "activations/layer6_attention_weight_max": 45.295448303222656,
      "activations/layer6_attention_weight_min": -50.477317810058594,
      "activations/layer7_attention_weight_max": 91.04129791259766,
      "activations/layer7_attention_weight_min": -91.43851470947266,
      "activations/layer8_attention_weight_max": 40.865234375,
      "activations/layer8_attention_weight_min": -43.0309944152832,
      "activations/layer9_attention_weight_max": 31.824832916259766,
      "activations/layer9_attention_weight_min": -35.385414123535156,
      "epoch": 15.7,
      "learning_rate": 4.92685606060606e-05,
      "loss": 2.758,
      "step": 270150
    },
    {
      "activations/layer0_attention_weight_max": 16.273645401000977,
      "activations/layer0_attention_weight_min": -12.70711612701416,
      "activations/layer10_attention_weight_max": 30.789810180664062,
      "activations/layer10_attention_weight_min": -31.539318084716797,
      "activations/layer11_attention_weight_max": 34.17951202392578,
      "activations/layer11_attention_weight_min": -32.98991394042969,
      "activations/layer12_attention_weight_max": 26.135889053344727,
      "activations/layer12_attention_weight_min": -28.207712173461914,
      "activations/layer13_attention_weight_max": 38.19794464111328,
      "activations/layer13_attention_weight_min": -34.4208984375,
      "activations/layer14_attention_weight_max": 39.78227615356445,
      "activations/layer14_attention_weight_min": -35.35695266723633,
      "activations/layer15_attention_weight_max": 38.741092681884766,
      "activations/layer15_attention_weight_min": -34.367164611816406,
      "activations/layer16_attention_weight_max": 28.95983123779297,
      "activations/layer16_attention_weight_min": -28.184938430786133,
      "activations/layer17_attention_weight_max": 27.79949951171875,
      "activations/layer17_attention_weight_min": -26.482919692993164,
      "activations/layer18_attention_weight_max": 27.328365325927734,
      "activations/layer18_attention_weight_min": -25.162294387817383,
      "activations/layer19_attention_weight_max": 30.71078872680664,
      "activations/layer19_attention_weight_min": -30.20548439025879,
      "activations/layer1_attention_weight_max": 16.208524703979492,
      "activations/layer1_attention_weight_min": -15.102333068847656,
      "activations/layer20_attention_weight_max": 28.784305572509766,
      "activations/layer20_attention_weight_min": -24.825759887695312,
      "activations/layer21_attention_weight_max": 26.954959869384766,
      "activations/layer21_attention_weight_min": -23.706445693969727,
      "activations/layer22_attention_weight_max": 36.782920837402344,
      "activations/layer22_attention_weight_min": -29.359516143798828,
      "activations/layer23_attention_weight_max": 29.201200485229492,
      "activations/layer23_attention_weight_min": -24.401077270507812,
      "activations/layer2_attention_weight_max": 31.91815185546875,
      "activations/layer2_attention_weight_min": -31.643775939941406,
      "activations/layer3_attention_weight_max": 92.48351287841797,
      "activations/layer3_attention_weight_min": -94.19766235351562,
      "activations/layer4_attention_weight_max": 104.8841323852539,
      "activations/layer4_attention_weight_min": -106.0345458984375,
      "activations/layer5_attention_weight_max": 49.66488265991211,
      "activations/layer5_attention_weight_min": -60.01152038574219,
      "activations/layer6_attention_weight_max": 40.668846130371094,
      "activations/layer6_attention_weight_min": -44.918148040771484,
      "activations/layer7_attention_weight_max": 88.6209945678711,
      "activations/layer7_attention_weight_min": -86.63311004638672,
      "activations/layer8_attention_weight_max": 37.501731872558594,
      "activations/layer8_attention_weight_min": -40.74091339111328,
      "activations/layer9_attention_weight_max": 30.373777389526367,
      "activations/layer9_attention_weight_min": -31.53607749938965,
      "epoch": 15.7,
      "learning_rate": 4.92496212121212e-05,
      "loss": 2.7464,
      "step": 270200
    },
    {
      "activations/layer0_attention_weight_max": 16.126659393310547,
      "activations/layer0_attention_weight_min": -12.643709182739258,
      "activations/layer10_attention_weight_max": 34.63234329223633,
      "activations/layer10_attention_weight_min": -33.7432861328125,
      "activations/layer11_attention_weight_max": 33.0406494140625,
      "activations/layer11_attention_weight_min": -34.342918395996094,
      "activations/layer12_attention_weight_max": 25.396757125854492,
      "activations/layer12_attention_weight_min": -26.24808120727539,
      "activations/layer13_attention_weight_max": 39.92521286010742,
      "activations/layer13_attention_weight_min": -37.49070358276367,
      "activations/layer14_attention_weight_max": 45.67246627807617,
      "activations/layer14_attention_weight_min": -38.8237419128418,
      "activations/layer15_attention_weight_max": 38.34674072265625,
      "activations/layer15_attention_weight_min": -34.84711837768555,
      "activations/layer16_attention_weight_max": 28.276185989379883,
      "activations/layer16_attention_weight_min": -28.161537170410156,
      "activations/layer17_attention_weight_max": 32.0724983215332,
      "activations/layer17_attention_weight_min": -28.836076736450195,
      "activations/layer18_attention_weight_max": 34.06207275390625,
      "activations/layer18_attention_weight_min": -24.451438903808594,
      "activations/layer19_attention_weight_max": 34.80654525756836,
      "activations/layer19_attention_weight_min": -32.15205764770508,
      "activations/layer1_attention_weight_max": 16.180662155151367,
      "activations/layer1_attention_weight_min": -15.381677627563477,
      "activations/layer20_attention_weight_max": 28.48272132873535,
      "activations/layer20_attention_weight_min": -25.660110473632812,
      "activations/layer21_attention_weight_max": 28.59126853942871,
      "activations/layer21_attention_weight_min": -25.41191291809082,
      "activations/layer22_attention_weight_max": 44.739444732666016,
      "activations/layer22_attention_weight_min": -32.03264236450195,
      "activations/layer23_attention_weight_max": 33.59446334838867,
      "activations/layer23_attention_weight_min": -27.560131072998047,
      "activations/layer2_attention_weight_max": 33.33252716064453,
      "activations/layer2_attention_weight_min": -31.645544052124023,
      "activations/layer3_attention_weight_max": 92.84784698486328,
      "activations/layer3_attention_weight_min": -93.64464569091797,
      "activations/layer4_attention_weight_max": 106.41200256347656,
      "activations/layer4_attention_weight_min": -110.78978729248047,
      "activations/layer5_attention_weight_max": 52.29335021972656,
      "activations/layer5_attention_weight_min": -61.091796875,
      "activations/layer6_attention_weight_max": 47.067928314208984,
      "activations/layer6_attention_weight_min": -50.0337028503418,
      "activations/layer7_attention_weight_max": 100.4195327758789,
      "activations/layer7_attention_weight_min": -100.06222534179688,
      "activations/layer8_attention_weight_max": 43.758243560791016,
      "activations/layer8_attention_weight_min": -43.52467346191406,
      "activations/layer9_attention_weight_max": 32.6966667175293,
      "activations/layer9_attention_weight_min": -34.95719528198242,
      "epoch": 15.7,
      "learning_rate": 4.923068181818182e-05,
      "loss": 2.7511,
      "step": 270250
    },
    {
      "activations/layer0_attention_weight_max": 15.673590660095215,
      "activations/layer0_attention_weight_min": -12.93484115600586,
      "activations/layer10_attention_weight_max": 33.52786636352539,
      "activations/layer10_attention_weight_min": -33.75383758544922,
      "activations/layer11_attention_weight_max": 32.36475372314453,
      "activations/layer11_attention_weight_min": -34.00663757324219,
      "activations/layer12_attention_weight_max": 29.27875518798828,
      "activations/layer12_attention_weight_min": -26.913175582885742,
      "activations/layer13_attention_weight_max": 40.41168975830078,
      "activations/layer13_attention_weight_min": -37.07716369628906,
      "activations/layer14_attention_weight_max": 40.96023178100586,
      "activations/layer14_attention_weight_min": -38.06708908081055,
      "activations/layer15_attention_weight_max": 38.01676559448242,
      "activations/layer15_attention_weight_min": -36.86474609375,
      "activations/layer16_attention_weight_max": 32.74169921875,
      "activations/layer16_attention_weight_min": -28.594266891479492,
      "activations/layer17_attention_weight_max": 31.055917739868164,
      "activations/layer17_attention_weight_min": -27.51958656311035,
      "activations/layer18_attention_weight_max": 31.653236389160156,
      "activations/layer18_attention_weight_min": -27.01947593688965,
      "activations/layer19_attention_weight_max": 33.802120208740234,
      "activations/layer19_attention_weight_min": -31.892780303955078,
      "activations/layer1_attention_weight_max": 15.766030311584473,
      "activations/layer1_attention_weight_min": -14.45966911315918,
      "activations/layer20_attention_weight_max": 27.703289031982422,
      "activations/layer20_attention_weight_min": -25.398258209228516,
      "activations/layer21_attention_weight_max": 30.863956451416016,
      "activations/layer21_attention_weight_min": -24.88045310974121,
      "activations/layer22_attention_weight_max": 43.3029670715332,
      "activations/layer22_attention_weight_min": -31.65032196044922,
      "activations/layer23_attention_weight_max": 34.532745361328125,
      "activations/layer23_attention_weight_min": -29.161102294921875,
      "activations/layer2_attention_weight_max": 32.15354919433594,
      "activations/layer2_attention_weight_min": -32.539154052734375,
      "activations/layer3_attention_weight_max": 95.57015228271484,
      "activations/layer3_attention_weight_min": -95.36760711669922,
      "activations/layer4_attention_weight_max": 101.88375091552734,
      "activations/layer4_attention_weight_min": -108.26802825927734,
      "activations/layer5_attention_weight_max": 49.32460021972656,
      "activations/layer5_attention_weight_min": -59.218929290771484,
      "activations/layer6_attention_weight_max": 43.102718353271484,
      "activations/layer6_attention_weight_min": -45.842506408691406,
      "activations/layer7_attention_weight_max": 90.53485870361328,
      "activations/layer7_attention_weight_min": -93.88229370117188,
      "activations/layer8_attention_weight_max": 42.021034240722656,
      "activations/layer8_attention_weight_min": -42.91875457763672,
      "activations/layer9_attention_weight_max": 33.44819259643555,
      "activations/layer9_attention_weight_min": -35.24459457397461,
      "epoch": 15.71,
      "learning_rate": 4.921174242424242e-05,
      "loss": 2.7469,
      "step": 270300
    },
    {
      "activations/layer0_attention_weight_max": 16.41707992553711,
      "activations/layer0_attention_weight_min": -13.12378215789795,
      "activations/layer10_attention_weight_max": 31.078886032104492,
      "activations/layer10_attention_weight_min": -32.55091857910156,
      "activations/layer11_attention_weight_max": 31.145187377929688,
      "activations/layer11_attention_weight_min": -32.713661193847656,
      "activations/layer12_attention_weight_max": 24.141321182250977,
      "activations/layer12_attention_weight_min": -25.71253204345703,
      "activations/layer13_attention_weight_max": 35.706241607666016,
      "activations/layer13_attention_weight_min": -33.73590087890625,
      "activations/layer14_attention_weight_max": 39.89183807373047,
      "activations/layer14_attention_weight_min": -35.2666130065918,
      "activations/layer15_attention_weight_max": 36.30704879760742,
      "activations/layer15_attention_weight_min": -35.62454605102539,
      "activations/layer16_attention_weight_max": 31.510839462280273,
      "activations/layer16_attention_weight_min": -29.59135627746582,
      "activations/layer17_attention_weight_max": 29.672780990600586,
      "activations/layer17_attention_weight_min": -27.016971588134766,
      "activations/layer18_attention_weight_max": 33.92621994018555,
      "activations/layer18_attention_weight_min": -24.79914665222168,
      "activations/layer19_attention_weight_max": 32.47214889526367,
      "activations/layer19_attention_weight_min": -31.774145126342773,
      "activations/layer1_attention_weight_max": 16.2279052734375,
      "activations/layer1_attention_weight_min": -14.013309478759766,
      "activations/layer20_attention_weight_max": 29.616069793701172,
      "activations/layer20_attention_weight_min": -24.782665252685547,
      "activations/layer21_attention_weight_max": 26.508407592773438,
      "activations/layer21_attention_weight_min": -25.20806121826172,
      "activations/layer22_attention_weight_max": 41.415672302246094,
      "activations/layer22_attention_weight_min": -34.173709869384766,
      "activations/layer23_attention_weight_max": 33.642967224121094,
      "activations/layer23_attention_weight_min": -27.610851287841797,
      "activations/layer2_attention_weight_max": 32.38937759399414,
      "activations/layer2_attention_weight_min": -32.22378158569336,
      "activations/layer3_attention_weight_max": 93.01835632324219,
      "activations/layer3_attention_weight_min": -90.65170288085938,
      "activations/layer4_attention_weight_max": 102.3215560913086,
      "activations/layer4_attention_weight_min": -101.14106750488281,
      "activations/layer5_attention_weight_max": 48.18798065185547,
      "activations/layer5_attention_weight_min": -62.386329650878906,
      "activations/layer6_attention_weight_max": 42.77151870727539,
      "activations/layer6_attention_weight_min": -45.22599411010742,
      "activations/layer7_attention_weight_max": 89.79141998291016,
      "activations/layer7_attention_weight_min": -91.63055419921875,
      "activations/layer8_attention_weight_max": 39.76898193359375,
      "activations/layer8_attention_weight_min": -40.62080764770508,
      "activations/layer9_attention_weight_max": 31.8481502532959,
      "activations/layer9_attention_weight_min": -31.204044342041016,
      "epoch": 15.71,
      "learning_rate": 4.9192803030303026e-05,
      "loss": 2.7443,
      "step": 270350
    },
    {
      "activations/layer0_attention_weight_max": 15.899311065673828,
      "activations/layer0_attention_weight_min": -12.356224060058594,
      "activations/layer10_attention_weight_max": 31.012216567993164,
      "activations/layer10_attention_weight_min": -33.25740051269531,
      "activations/layer11_attention_weight_max": 31.23286247253418,
      "activations/layer11_attention_weight_min": -33.20058822631836,
      "activations/layer12_attention_weight_max": 24.93801498413086,
      "activations/layer12_attention_weight_min": -25.1508846282959,
      "activations/layer13_attention_weight_max": 37.04096984863281,
      "activations/layer13_attention_weight_min": -36.35646057128906,
      "activations/layer14_attention_weight_max": 42.78175735473633,
      "activations/layer14_attention_weight_min": -36.38279724121094,
      "activations/layer15_attention_weight_max": 39.95884704589844,
      "activations/layer15_attention_weight_min": -34.81523132324219,
      "activations/layer16_attention_weight_max": 35.87228012084961,
      "activations/layer16_attention_weight_min": -27.380502700805664,
      "activations/layer17_attention_weight_max": 31.771942138671875,
      "activations/layer17_attention_weight_min": -26.205366134643555,
      "activations/layer18_attention_weight_max": 33.112945556640625,
      "activations/layer18_attention_weight_min": -25.978073120117188,
      "activations/layer19_attention_weight_max": 44.08992385864258,
      "activations/layer19_attention_weight_min": -31.37225914001465,
      "activations/layer1_attention_weight_max": 15.516912460327148,
      "activations/layer1_attention_weight_min": -14.741015434265137,
      "activations/layer20_attention_weight_max": 36.51115036010742,
      "activations/layer20_attention_weight_min": -27.90203094482422,
      "activations/layer21_attention_weight_max": 32.2007942199707,
      "activations/layer21_attention_weight_min": -26.43576431274414,
      "activations/layer22_attention_weight_max": 43.99358367919922,
      "activations/layer22_attention_weight_min": -34.132545471191406,
      "activations/layer23_attention_weight_max": 35.182762145996094,
      "activations/layer23_attention_weight_min": -25.588905334472656,
      "activations/layer2_attention_weight_max": 32.35957717895508,
      "activations/layer2_attention_weight_min": -34.78889465332031,
      "activations/layer3_attention_weight_max": 97.8226318359375,
      "activations/layer3_attention_weight_min": -104.90951538085938,
      "activations/layer4_attention_weight_max": 107.38494110107422,
      "activations/layer4_attention_weight_min": -110.73014068603516,
      "activations/layer5_attention_weight_max": 52.144752502441406,
      "activations/layer5_attention_weight_min": -63.46751403808594,
      "activations/layer6_attention_weight_max": 41.85258865356445,
      "activations/layer6_attention_weight_min": -44.3526725769043,
      "activations/layer7_attention_weight_max": 91.77169799804688,
      "activations/layer7_attention_weight_min": -92.34455108642578,
      "activations/layer8_attention_weight_max": 38.561092376708984,
      "activations/layer8_attention_weight_min": -41.95468521118164,
      "activations/layer9_attention_weight_max": 29.6279296875,
      "activations/layer9_attention_weight_min": -32.98125076293945,
      "epoch": 15.71,
      "learning_rate": 4.917386363636363e-05,
      "loss": 2.7541,
      "step": 270400
    },
    {
      "activations/layer0_attention_weight_max": 16.483139038085938,
      "activations/layer0_attention_weight_min": -12.459127426147461,
      "activations/layer10_attention_weight_max": 32.926727294921875,
      "activations/layer10_attention_weight_min": -34.64313507080078,
      "activations/layer11_attention_weight_max": 33.339820861816406,
      "activations/layer11_attention_weight_min": -33.71687316894531,
      "activations/layer12_attention_weight_max": 28.914661407470703,
      "activations/layer12_attention_weight_min": -26.210580825805664,
      "activations/layer13_attention_weight_max": 39.272457122802734,
      "activations/layer13_attention_weight_min": -35.434932708740234,
      "activations/layer14_attention_weight_max": 39.91239547729492,
      "activations/layer14_attention_weight_min": -37.716346740722656,
      "activations/layer15_attention_weight_max": 37.52789306640625,
      "activations/layer15_attention_weight_min": -34.351478576660156,
      "activations/layer16_attention_weight_max": 27.68895149230957,
      "activations/layer16_attention_weight_min": -28.874610900878906,
      "activations/layer17_attention_weight_max": 30.36102294921875,
      "activations/layer17_attention_weight_min": -26.422595977783203,
      "activations/layer18_attention_weight_max": 31.307409286499023,
      "activations/layer18_attention_weight_min": -26.340829849243164,
      "activations/layer19_attention_weight_max": 33.05272674560547,
      "activations/layer19_attention_weight_min": -33.0213508605957,
      "activations/layer1_attention_weight_max": 15.416269302368164,
      "activations/layer1_attention_weight_min": -14.448307991027832,
      "activations/layer20_attention_weight_max": 29.263628005981445,
      "activations/layer20_attention_weight_min": -25.273296356201172,
      "activations/layer21_attention_weight_max": 29.494829177856445,
      "activations/layer21_attention_weight_min": -24.923612594604492,
      "activations/layer22_attention_weight_max": 41.1916618347168,
      "activations/layer22_attention_weight_min": -30.50947380065918,
      "activations/layer23_attention_weight_max": 32.318519592285156,
      "activations/layer23_attention_weight_min": -27.368927001953125,
      "activations/layer2_attention_weight_max": 31.749095916748047,
      "activations/layer2_attention_weight_min": -32.43527603149414,
      "activations/layer3_attention_weight_max": 93.05915069580078,
      "activations/layer3_attention_weight_min": -102.66157531738281,
      "activations/layer4_attention_weight_max": 100.06243133544922,
      "activations/layer4_attention_weight_min": -106.9542007446289,
      "activations/layer5_attention_weight_max": 51.2557373046875,
      "activations/layer5_attention_weight_min": -60.63386535644531,
      "activations/layer6_attention_weight_max": 42.86019515991211,
      "activations/layer6_attention_weight_min": -46.75416564941406,
      "activations/layer7_attention_weight_max": 88.66990661621094,
      "activations/layer7_attention_weight_min": -92.40330505371094,
      "activations/layer8_attention_weight_max": 40.030357360839844,
      "activations/layer8_attention_weight_min": -43.324153900146484,
      "activations/layer9_attention_weight_max": 31.701705932617188,
      "activations/layer9_attention_weight_min": -34.212188720703125,
      "epoch": 15.71,
      "learning_rate": 4.915492424242424e-05,
      "loss": 2.7461,
      "step": 270450
    },
    {
      "activations/layer0_attention_weight_max": 15.94973087310791,
      "activations/layer0_attention_weight_min": -12.7941255569458,
      "activations/layer10_attention_weight_max": 34.44662094116211,
      "activations/layer10_attention_weight_min": -33.11503601074219,
      "activations/layer11_attention_weight_max": 38.66719436645508,
      "activations/layer11_attention_weight_min": -36.57316207885742,
      "activations/layer12_attention_weight_max": 29.380477905273438,
      "activations/layer12_attention_weight_min": -26.71480941772461,
      "activations/layer13_attention_weight_max": 45.27096176147461,
      "activations/layer13_attention_weight_min": -36.52086639404297,
      "activations/layer14_attention_weight_max": 44.10563659667969,
      "activations/layer14_attention_weight_min": -41.3830680847168,
      "activations/layer15_attention_weight_max": 45.016563415527344,
      "activations/layer15_attention_weight_min": -38.90749740600586,
      "activations/layer16_attention_weight_max": 31.352201461791992,
      "activations/layer16_attention_weight_min": -27.564655303955078,
      "activations/layer17_attention_weight_max": 33.540374755859375,
      "activations/layer17_attention_weight_min": -27.263317108154297,
      "activations/layer18_attention_weight_max": 30.814128875732422,
      "activations/layer18_attention_weight_min": -24.81641960144043,
      "activations/layer19_attention_weight_max": 33.2915153503418,
      "activations/layer19_attention_weight_min": -31.949941635131836,
      "activations/layer1_attention_weight_max": 15.813397407531738,
      "activations/layer1_attention_weight_min": -16.024085998535156,
      "activations/layer20_attention_weight_max": 28.220006942749023,
      "activations/layer20_attention_weight_min": -24.545909881591797,
      "activations/layer21_attention_weight_max": 31.11258888244629,
      "activations/layer21_attention_weight_min": -24.961936950683594,
      "activations/layer22_attention_weight_max": 38.55466842651367,
      "activations/layer22_attention_weight_min": -31.672529220581055,
      "activations/layer23_attention_weight_max": 28.058208465576172,
      "activations/layer23_attention_weight_min": -25.77643585205078,
      "activations/layer2_attention_weight_max": 33.006954193115234,
      "activations/layer2_attention_weight_min": -32.53772735595703,
      "activations/layer3_attention_weight_max": 89.83796691894531,
      "activations/layer3_attention_weight_min": -89.9774398803711,
      "activations/layer4_attention_weight_max": 100.63481140136719,
      "activations/layer4_attention_weight_min": -102.83394622802734,
      "activations/layer5_attention_weight_max": 47.535316467285156,
      "activations/layer5_attention_weight_min": -59.08203125,
      "activations/layer6_attention_weight_max": 41.89052200317383,
      "activations/layer6_attention_weight_min": -43.74113845825195,
      "activations/layer7_attention_weight_max": 92.89702606201172,
      "activations/layer7_attention_weight_min": -90.46936798095703,
      "activations/layer8_attention_weight_max": 40.94179916381836,
      "activations/layer8_attention_weight_min": -42.01087951660156,
      "activations/layer9_attention_weight_max": 31.931686401367188,
      "activations/layer9_attention_weight_min": -33.33475875854492,
      "epoch": 15.72,
      "learning_rate": 4.9135984848484844e-05,
      "loss": 2.7694,
      "step": 270500
    },
    {
      "activations/layer0_attention_weight_max": 16.91815948486328,
      "activations/layer0_attention_weight_min": -13.244246482849121,
      "activations/layer10_attention_weight_max": 33.06827926635742,
      "activations/layer10_attention_weight_min": -33.42543411254883,
      "activations/layer11_attention_weight_max": 35.29113006591797,
      "activations/layer11_attention_weight_min": -34.00301742553711,
      "activations/layer12_attention_weight_max": 25.802167892456055,
      "activations/layer12_attention_weight_min": -25.17236328125,
      "activations/layer13_attention_weight_max": 39.71102523803711,
      "activations/layer13_attention_weight_min": -36.10833740234375,
      "activations/layer14_attention_weight_max": 43.09882354736328,
      "activations/layer14_attention_weight_min": -37.5305061340332,
      "activations/layer15_attention_weight_max": 37.96821594238281,
      "activations/layer15_attention_weight_min": -35.465599060058594,
      "activations/layer16_attention_weight_max": 29.76795768737793,
      "activations/layer16_attention_weight_min": -27.285776138305664,
      "activations/layer17_attention_weight_max": 29.91070556640625,
      "activations/layer17_attention_weight_min": -25.764734268188477,
      "activations/layer18_attention_weight_max": 30.2418155670166,
      "activations/layer18_attention_weight_min": -25.336925506591797,
      "activations/layer19_attention_weight_max": 37.81904983520508,
      "activations/layer19_attention_weight_min": -32.53318405151367,
      "activations/layer1_attention_weight_max": 15.54161262512207,
      "activations/layer1_attention_weight_min": -16.376331329345703,
      "activations/layer20_attention_weight_max": 28.24193000793457,
      "activations/layer20_attention_weight_min": -27.312631607055664,
      "activations/layer21_attention_weight_max": 30.39639663696289,
      "activations/layer21_attention_weight_min": -25.178171157836914,
      "activations/layer22_attention_weight_max": 43.193115234375,
      "activations/layer22_attention_weight_min": -31.989702224731445,
      "activations/layer23_attention_weight_max": 31.522415161132812,
      "activations/layer23_attention_weight_min": -24.54340171813965,
      "activations/layer2_attention_weight_max": 31.509021759033203,
      "activations/layer2_attention_weight_min": -32.52522659301758,
      "activations/layer3_attention_weight_max": 92.77220916748047,
      "activations/layer3_attention_weight_min": -93.25633239746094,
      "activations/layer4_attention_weight_max": 106.30916595458984,
      "activations/layer4_attention_weight_min": -106.6453628540039,
      "activations/layer5_attention_weight_max": 51.4659538269043,
      "activations/layer5_attention_weight_min": -58.787593841552734,
      "activations/layer6_attention_weight_max": 43.028316497802734,
      "activations/layer6_attention_weight_min": -45.674747467041016,
      "activations/layer7_attention_weight_max": 90.26536560058594,
      "activations/layer7_attention_weight_min": -89.58247375488281,
      "activations/layer8_attention_weight_max": 38.817203521728516,
      "activations/layer8_attention_weight_min": -44.263492584228516,
      "activations/layer9_attention_weight_max": 31.3526554107666,
      "activations/layer9_attention_weight_min": -36.72703170776367,
      "epoch": 15.72,
      "learning_rate": 4.9117045454545446e-05,
      "loss": 2.7462,
      "step": 270550
    },
    {
      "activations/layer0_attention_weight_max": 18.484474182128906,
      "activations/layer0_attention_weight_min": -12.422517776489258,
      "activations/layer10_attention_weight_max": 32.68407440185547,
      "activations/layer10_attention_weight_min": -35.13715744018555,
      "activations/layer11_attention_weight_max": 32.473533630371094,
      "activations/layer11_attention_weight_min": -32.242008209228516,
      "activations/layer12_attention_weight_max": 25.73356056213379,
      "activations/layer12_attention_weight_min": -26.12493324279785,
      "activations/layer13_attention_weight_max": 36.94844436645508,
      "activations/layer13_attention_weight_min": -34.27534103393555,
      "activations/layer14_attention_weight_max": 41.18587112426758,
      "activations/layer14_attention_weight_min": -37.64230728149414,
      "activations/layer15_attention_weight_max": 37.34966278076172,
      "activations/layer15_attention_weight_min": -34.11569595336914,
      "activations/layer16_attention_weight_max": 29.780155181884766,
      "activations/layer16_attention_weight_min": -27.96629524230957,
      "activations/layer17_attention_weight_max": 31.204248428344727,
      "activations/layer17_attention_weight_min": -25.734622955322266,
      "activations/layer18_attention_weight_max": 31.394752502441406,
      "activations/layer18_attention_weight_min": -26.903926849365234,
      "activations/layer19_attention_weight_max": 36.0739631652832,
      "activations/layer19_attention_weight_min": -30.520675659179688,
      "activations/layer1_attention_weight_max": 16.525035858154297,
      "activations/layer1_attention_weight_min": -14.471733093261719,
      "activations/layer20_attention_weight_max": 32.38405227661133,
      "activations/layer20_attention_weight_min": -24.9818115234375,
      "activations/layer21_attention_weight_max": 33.554298400878906,
      "activations/layer21_attention_weight_min": -25.911392211914062,
      "activations/layer22_attention_weight_max": 44.22098159790039,
      "activations/layer22_attention_weight_min": -33.157936096191406,
      "activations/layer23_attention_weight_max": 34.282691955566406,
      "activations/layer23_attention_weight_min": -27.202665328979492,
      "activations/layer2_attention_weight_max": 34.31222915649414,
      "activations/layer2_attention_weight_min": -30.421051025390625,
      "activations/layer3_attention_weight_max": 91.52177429199219,
      "activations/layer3_attention_weight_min": -92.0928955078125,
      "activations/layer4_attention_weight_max": 100.35932922363281,
      "activations/layer4_attention_weight_min": -103.70108795166016,
      "activations/layer5_attention_weight_max": 50.59031295776367,
      "activations/layer5_attention_weight_min": -59.95161437988281,
      "activations/layer6_attention_weight_max": 43.49380111694336,
      "activations/layer6_attention_weight_min": -43.78461837768555,
      "activations/layer7_attention_weight_max": 84.52606201171875,
      "activations/layer7_attention_weight_min": -86.64985656738281,
      "activations/layer8_attention_weight_max": 37.59511184692383,
      "activations/layer8_attention_weight_min": -40.21190643310547,
      "activations/layer9_attention_weight_max": 31.8979434967041,
      "activations/layer9_attention_weight_min": -32.347755432128906,
      "epoch": 15.72,
      "learning_rate": 4.909810606060606e-05,
      "loss": 2.7558,
      "step": 270600
    },
    {
      "activations/layer0_attention_weight_max": 16.319265365600586,
      "activations/layer0_attention_weight_min": -12.92860221862793,
      "activations/layer10_attention_weight_max": 35.85297393798828,
      "activations/layer10_attention_weight_min": -33.977012634277344,
      "activations/layer11_attention_weight_max": 34.88673400878906,
      "activations/layer11_attention_weight_min": -35.19328308105469,
      "activations/layer12_attention_weight_max": 24.26404571533203,
      "activations/layer12_attention_weight_min": -28.32187843322754,
      "activations/layer13_attention_weight_max": 37.014366149902344,
      "activations/layer13_attention_weight_min": -35.30206298828125,
      "activations/layer14_attention_weight_max": 40.92839431762695,
      "activations/layer14_attention_weight_min": -36.93388366699219,
      "activations/layer15_attention_weight_max": 37.941017150878906,
      "activations/layer15_attention_weight_min": -34.322574615478516,
      "activations/layer16_attention_weight_max": 29.19965362548828,
      "activations/layer16_attention_weight_min": -29.070249557495117,
      "activations/layer17_attention_weight_max": 31.40427589416504,
      "activations/layer17_attention_weight_min": -26.68218421936035,
      "activations/layer18_attention_weight_max": 32.18074035644531,
      "activations/layer18_attention_weight_min": -24.337739944458008,
      "activations/layer19_attention_weight_max": 34.227718353271484,
      "activations/layer19_attention_weight_min": -28.907827377319336,
      "activations/layer1_attention_weight_max": 15.946951866149902,
      "activations/layer1_attention_weight_min": -16.332721710205078,
      "activations/layer20_attention_weight_max": 31.313304901123047,
      "activations/layer20_attention_weight_min": -24.78948402404785,
      "activations/layer21_attention_weight_max": 28.748641967773438,
      "activations/layer21_attention_weight_min": -25.864763259887695,
      "activations/layer22_attention_weight_max": 44.98992156982422,
      "activations/layer22_attention_weight_min": -31.02127456665039,
      "activations/layer23_attention_weight_max": 31.424673080444336,
      "activations/layer23_attention_weight_min": -23.763469696044922,
      "activations/layer2_attention_weight_max": 35.66695022583008,
      "activations/layer2_attention_weight_min": -33.108192443847656,
      "activations/layer3_attention_weight_max": 97.48920440673828,
      "activations/layer3_attention_weight_min": -101.20183563232422,
      "activations/layer4_attention_weight_max": 106.46050262451172,
      "activations/layer4_attention_weight_min": -110.38349914550781,
      "activations/layer5_attention_weight_max": 53.68943405151367,
      "activations/layer5_attention_weight_min": -60.62196350097656,
      "activations/layer6_attention_weight_max": 48.777347564697266,
      "activations/layer6_attention_weight_min": -48.442161560058594,
      "activations/layer7_attention_weight_max": 98.28996276855469,
      "activations/layer7_attention_weight_min": -99.07006072998047,
      "activations/layer8_attention_weight_max": 46.770687103271484,
      "activations/layer8_attention_weight_min": -51.0187873840332,
      "activations/layer9_attention_weight_max": 37.932708740234375,
      "activations/layer9_attention_weight_min": -38.25215530395508,
      "epoch": 15.73,
      "learning_rate": 4.907916666666666e-05,
      "loss": 2.7508,
      "step": 270650
    },
    {
      "activations/layer0_attention_weight_max": 16.285905838012695,
      "activations/layer0_attention_weight_min": -12.964523315429688,
      "activations/layer10_attention_weight_max": 32.24173355102539,
      "activations/layer10_attention_weight_min": -34.29874038696289,
      "activations/layer11_attention_weight_max": 31.815391540527344,
      "activations/layer11_attention_weight_min": -33.933441162109375,
      "activations/layer12_attention_weight_max": 26.977170944213867,
      "activations/layer12_attention_weight_min": -27.431636810302734,
      "activations/layer13_attention_weight_max": 38.57245635986328,
      "activations/layer13_attention_weight_min": -35.74191665649414,
      "activations/layer14_attention_weight_max": 42.63743591308594,
      "activations/layer14_attention_weight_min": -37.15444564819336,
      "activations/layer15_attention_weight_max": 34.066184997558594,
      "activations/layer15_attention_weight_min": -35.559810638427734,
      "activations/layer16_attention_weight_max": 28.458810806274414,
      "activations/layer16_attention_weight_min": -28.757755279541016,
      "activations/layer17_attention_weight_max": 33.937313079833984,
      "activations/layer17_attention_weight_min": -25.306589126586914,
      "activations/layer18_attention_weight_max": 35.56318283081055,
      "activations/layer18_attention_weight_min": -25.939903259277344,
      "activations/layer19_attention_weight_max": 36.2962760925293,
      "activations/layer19_attention_weight_min": -32.97807693481445,
      "activations/layer1_attention_weight_max": 15.829239845275879,
      "activations/layer1_attention_weight_min": -15.732380867004395,
      "activations/layer20_attention_weight_max": 33.482666015625,
      "activations/layer20_attention_weight_min": -27.474163055419922,
      "activations/layer21_attention_weight_max": 36.55634307861328,
      "activations/layer21_attention_weight_min": -27.274717330932617,
      "activations/layer22_attention_weight_max": 49.072845458984375,
      "activations/layer22_attention_weight_min": -34.67385482788086,
      "activations/layer23_attention_weight_max": 39.72679138183594,
      "activations/layer23_attention_weight_min": -27.312314987182617,
      "activations/layer2_attention_weight_max": 31.560752868652344,
      "activations/layer2_attention_weight_min": -31.49913787841797,
      "activations/layer3_attention_weight_max": 90.1369400024414,
      "activations/layer3_attention_weight_min": -91.9839096069336,
      "activations/layer4_attention_weight_max": 99.98319244384766,
      "activations/layer4_attention_weight_min": -99.13887786865234,
      "activations/layer5_attention_weight_max": 48.536842346191406,
      "activations/layer5_attention_weight_min": -59.95277404785156,
      "activations/layer6_attention_weight_max": 41.17158126831055,
      "activations/layer6_attention_weight_min": -42.99058151245117,
      "activations/layer7_attention_weight_max": 89.1875228881836,
      "activations/layer7_attention_weight_min": -89.74870300292969,
      "activations/layer8_attention_weight_max": 39.4674072265625,
      "activations/layer8_attention_weight_min": -41.40573501586914,
      "activations/layer9_attention_weight_max": 31.39218521118164,
      "activations/layer9_attention_weight_min": -33.00143051147461,
      "epoch": 15.73,
      "learning_rate": 4.906022727272727e-05,
      "loss": 2.75,
      "step": 270700
    },
    {
      "activations/layer0_attention_weight_max": 16.33077621459961,
      "activations/layer0_attention_weight_min": -12.478730201721191,
      "activations/layer10_attention_weight_max": 41.22399139404297,
      "activations/layer10_attention_weight_min": -36.586463928222656,
      "activations/layer11_attention_weight_max": 39.48975372314453,
      "activations/layer11_attention_weight_min": -38.27601623535156,
      "activations/layer12_attention_weight_max": 25.70191192626953,
      "activations/layer12_attention_weight_min": -26.899229049682617,
      "activations/layer13_attention_weight_max": 39.014732360839844,
      "activations/layer13_attention_weight_min": -36.6541633605957,
      "activations/layer14_attention_weight_max": 43.50648498535156,
      "activations/layer14_attention_weight_min": -37.37489318847656,
      "activations/layer15_attention_weight_max": 38.3461799621582,
      "activations/layer15_attention_weight_min": -35.843833923339844,
      "activations/layer16_attention_weight_max": 30.048358917236328,
      "activations/layer16_attention_weight_min": -28.124134063720703,
      "activations/layer17_attention_weight_max": 33.38885498046875,
      "activations/layer17_attention_weight_min": -24.96773338317871,
      "activations/layer18_attention_weight_max": 29.559589385986328,
      "activations/layer18_attention_weight_min": -26.2640380859375,
      "activations/layer19_attention_weight_max": 33.69639587402344,
      "activations/layer19_attention_weight_min": -30.167789459228516,
      "activations/layer1_attention_weight_max": 16.108434677124023,
      "activations/layer1_attention_weight_min": -15.237299919128418,
      "activations/layer20_attention_weight_max": 28.375255584716797,
      "activations/layer20_attention_weight_min": -25.6840877532959,
      "activations/layer21_attention_weight_max": 26.17589569091797,
      "activations/layer21_attention_weight_min": -24.499258041381836,
      "activations/layer22_attention_weight_max": 39.95561599731445,
      "activations/layer22_attention_weight_min": -29.87620735168457,
      "activations/layer23_attention_weight_max": 27.341506958007812,
      "activations/layer23_attention_weight_min": -27.69713592529297,
      "activations/layer2_attention_weight_max": 35.18408203125,
      "activations/layer2_attention_weight_min": -35.810428619384766,
      "activations/layer3_attention_weight_max": 95.37812042236328,
      "activations/layer3_attention_weight_min": -97.01478576660156,
      "activations/layer4_attention_weight_max": 106.52284240722656,
      "activations/layer4_attention_weight_min": -106.64896392822266,
      "activations/layer5_attention_weight_max": 53.87185287475586,
      "activations/layer5_attention_weight_min": -61.16642761230469,
      "activations/layer6_attention_weight_max": 43.89043426513672,
      "activations/layer6_attention_weight_min": -49.13540267944336,
      "activations/layer7_attention_weight_max": 101.80352783203125,
      "activations/layer7_attention_weight_min": -92.93164825439453,
      "activations/layer8_attention_weight_max": 50.01570510864258,
      "activations/layer8_attention_weight_min": -44.37647247314453,
      "activations/layer9_attention_weight_max": 35.69758605957031,
      "activations/layer9_attention_weight_min": -36.63005065917969,
      "epoch": 15.73,
      "learning_rate": 4.904128787878787e-05,
      "loss": 2.7435,
      "step": 270750
    },
    {
      "activations/layer0_attention_weight_max": 15.704917907714844,
      "activations/layer0_attention_weight_min": -12.163274765014648,
      "activations/layer10_attention_weight_max": 35.306495666503906,
      "activations/layer10_attention_weight_min": -35.46507263183594,
      "activations/layer11_attention_weight_max": 37.27210998535156,
      "activations/layer11_attention_weight_min": -37.16908645629883,
      "activations/layer12_attention_weight_max": 28.10392951965332,
      "activations/layer12_attention_weight_min": -26.736337661743164,
      "activations/layer13_attention_weight_max": 42.79579544067383,
      "activations/layer13_attention_weight_min": -36.5319938659668,
      "activations/layer14_attention_weight_max": 47.10430908203125,
      "activations/layer14_attention_weight_min": -39.165714263916016,
      "activations/layer15_attention_weight_max": 43.34430694580078,
      "activations/layer15_attention_weight_min": -35.05327224731445,
      "activations/layer16_attention_weight_max": 34.4221076965332,
      "activations/layer16_attention_weight_min": -29.503110885620117,
      "activations/layer17_attention_weight_max": 32.43860626220703,
      "activations/layer17_attention_weight_min": -26.845849990844727,
      "activations/layer18_attention_weight_max": 35.31074905395508,
      "activations/layer18_attention_weight_min": -26.26483154296875,
      "activations/layer19_attention_weight_max": 38.56122589111328,
      "activations/layer19_attention_weight_min": -32.87801742553711,
      "activations/layer1_attention_weight_max": 15.788887977600098,
      "activations/layer1_attention_weight_min": -14.60693359375,
      "activations/layer20_attention_weight_max": 31.933088302612305,
      "activations/layer20_attention_weight_min": -25.696434020996094,
      "activations/layer21_attention_weight_max": 31.6246395111084,
      "activations/layer21_attention_weight_min": -25.169445037841797,
      "activations/layer22_attention_weight_max": 51.14102554321289,
      "activations/layer22_attention_weight_min": -31.592126846313477,
      "activations/layer23_attention_weight_max": 35.83744430541992,
      "activations/layer23_attention_weight_min": -26.494037628173828,
      "activations/layer2_attention_weight_max": 31.45039939880371,
      "activations/layer2_attention_weight_min": -31.753887176513672,
      "activations/layer3_attention_weight_max": 92.904296875,
      "activations/layer3_attention_weight_min": -93.79431915283203,
      "activations/layer4_attention_weight_max": 107.5398941040039,
      "activations/layer4_attention_weight_min": -106.8789291381836,
      "activations/layer5_attention_weight_max": 50.72179412841797,
      "activations/layer5_attention_weight_min": -60.287025451660156,
      "activations/layer6_attention_weight_max": 44.66294479370117,
      "activations/layer6_attention_weight_min": -45.78099060058594,
      "activations/layer7_attention_weight_max": 102.28833770751953,
      "activations/layer7_attention_weight_min": -90.83306884765625,
      "activations/layer8_attention_weight_max": 40.61188507080078,
      "activations/layer8_attention_weight_min": -43.36125946044922,
      "activations/layer9_attention_weight_max": 34.81980514526367,
      "activations/layer9_attention_weight_min": -34.88656997680664,
      "epoch": 15.74,
      "learning_rate": 4.902234848484848e-05,
      "loss": 2.7533,
      "step": 270800
    },
    {
      "activations/layer0_attention_weight_max": 15.885539054870605,
      "activations/layer0_attention_weight_min": -12.038399696350098,
      "activations/layer10_attention_weight_max": 37.98780059814453,
      "activations/layer10_attention_weight_min": -40.73313522338867,
      "activations/layer11_attention_weight_max": 37.417991638183594,
      "activations/layer11_attention_weight_min": -38.42510986328125,
      "activations/layer12_attention_weight_max": 26.590761184692383,
      "activations/layer12_attention_weight_min": -29.03301429748535,
      "activations/layer13_attention_weight_max": 38.31825256347656,
      "activations/layer13_attention_weight_min": -38.669586181640625,
      "activations/layer14_attention_weight_max": 45.89606857299805,
      "activations/layer14_attention_weight_min": -44.48750686645508,
      "activations/layer15_attention_weight_max": 37.511749267578125,
      "activations/layer15_attention_weight_min": -40.152339935302734,
      "activations/layer16_attention_weight_max": 32.473018646240234,
      "activations/layer16_attention_weight_min": -30.82265281677246,
      "activations/layer17_attention_weight_max": 33.2783088684082,
      "activations/layer17_attention_weight_min": -28.65486717224121,
      "activations/layer18_attention_weight_max": 32.572357177734375,
      "activations/layer18_attention_weight_min": -27.708003997802734,
      "activations/layer19_attention_weight_max": 33.059425354003906,
      "activations/layer19_attention_weight_min": -32.60997772216797,
      "activations/layer1_attention_weight_max": 16.8236141204834,
      "activations/layer1_attention_weight_min": -15.492959976196289,
      "activations/layer20_attention_weight_max": 28.119972229003906,
      "activations/layer20_attention_weight_min": -26.708234786987305,
      "activations/layer21_attention_weight_max": 26.505239486694336,
      "activations/layer21_attention_weight_min": -25.538986206054688,
      "activations/layer22_attention_weight_max": 40.8345832824707,
      "activations/layer22_attention_weight_min": -32.868804931640625,
      "activations/layer23_attention_weight_max": 32.38478088378906,
      "activations/layer23_attention_weight_min": -27.491260528564453,
      "activations/layer2_attention_weight_max": 34.35769271850586,
      "activations/layer2_attention_weight_min": -35.50857925415039,
      "activations/layer3_attention_weight_max": 98.49642181396484,
      "activations/layer3_attention_weight_min": -103.61512756347656,
      "activations/layer4_attention_weight_max": 110.40169525146484,
      "activations/layer4_attention_weight_min": -111.10340881347656,
      "activations/layer5_attention_weight_max": 52.26627731323242,
      "activations/layer5_attention_weight_min": -60.753997802734375,
      "activations/layer6_attention_weight_max": 45.04907989501953,
      "activations/layer6_attention_weight_min": -46.29993438720703,
      "activations/layer7_attention_weight_max": 100.68892669677734,
      "activations/layer7_attention_weight_min": -96.87444305419922,
      "activations/layer8_attention_weight_max": 45.1244010925293,
      "activations/layer8_attention_weight_min": -45.7803840637207,
      "activations/layer9_attention_weight_max": 34.923126220703125,
      "activations/layer9_attention_weight_min": -38.60003662109375,
      "epoch": 15.74,
      "learning_rate": 4.900340909090909e-05,
      "loss": 2.7291,
      "step": 270850
    },
    {
      "activations/layer0_attention_weight_max": 16.69032859802246,
      "activations/layer0_attention_weight_min": -13.396708488464355,
      "activations/layer10_attention_weight_max": 36.37842559814453,
      "activations/layer10_attention_weight_min": -33.709110260009766,
      "activations/layer11_attention_weight_max": 38.06980895996094,
      "activations/layer11_attention_weight_min": -33.88376235961914,
      "activations/layer12_attention_weight_max": 24.757553100585938,
      "activations/layer12_attention_weight_min": -25.83234214782715,
      "activations/layer13_attention_weight_max": 37.23973846435547,
      "activations/layer13_attention_weight_min": -35.11613082885742,
      "activations/layer14_attention_weight_max": 39.89311599731445,
      "activations/layer14_attention_weight_min": -34.60432434082031,
      "activations/layer15_attention_weight_max": 35.084938049316406,
      "activations/layer15_attention_weight_min": -34.00869369506836,
      "activations/layer16_attention_weight_max": 29.478771209716797,
      "activations/layer16_attention_weight_min": -28.20416259765625,
      "activations/layer17_attention_weight_max": 27.947219848632812,
      "activations/layer17_attention_weight_min": -24.73250389099121,
      "activations/layer18_attention_weight_max": 30.818401336669922,
      "activations/layer18_attention_weight_min": -25.01262664794922,
      "activations/layer19_attention_weight_max": 31.96360206604004,
      "activations/layer19_attention_weight_min": -29.59588623046875,
      "activations/layer1_attention_weight_max": 16.12447166442871,
      "activations/layer1_attention_weight_min": -14.631606101989746,
      "activations/layer20_attention_weight_max": 26.601062774658203,
      "activations/layer20_attention_weight_min": -24.775304794311523,
      "activations/layer21_attention_weight_max": 27.106168746948242,
      "activations/layer21_attention_weight_min": -23.46501350402832,
      "activations/layer22_attention_weight_max": 41.490318298339844,
      "activations/layer22_attention_weight_min": -31.125165939331055,
      "activations/layer23_attention_weight_max": 31.12964630126953,
      "activations/layer23_attention_weight_min": -24.48877716064453,
      "activations/layer2_attention_weight_max": 33.99677658081055,
      "activations/layer2_attention_weight_min": -35.19343185424805,
      "activations/layer3_attention_weight_max": 93.69062805175781,
      "activations/layer3_attention_weight_min": -95.92717742919922,
      "activations/layer4_attention_weight_max": 101.96985626220703,
      "activations/layer4_attention_weight_min": -101.91410064697266,
      "activations/layer5_attention_weight_max": 48.84083557128906,
      "activations/layer5_attention_weight_min": -59.370582580566406,
      "activations/layer6_attention_weight_max": 40.082664489746094,
      "activations/layer6_attention_weight_min": -43.02302169799805,
      "activations/layer7_attention_weight_max": 94.04562377929688,
      "activations/layer7_attention_weight_min": -89.69082641601562,
      "activations/layer8_attention_weight_max": 41.56083297729492,
      "activations/layer8_attention_weight_min": -41.96781921386719,
      "activations/layer9_attention_weight_max": 30.798843383789062,
      "activations/layer9_attention_weight_min": -32.808311462402344,
      "epoch": 15.74,
      "learning_rate": 4.898446969696969e-05,
      "loss": 2.7367,
      "step": 270900
    },
    {
      "activations/layer0_attention_weight_max": 17.507301330566406,
      "activations/layer0_attention_weight_min": -11.825520515441895,
      "activations/layer10_attention_weight_max": 30.71451759338379,
      "activations/layer10_attention_weight_min": -29.8802490234375,
      "activations/layer11_attention_weight_max": 31.61890411376953,
      "activations/layer11_attention_weight_min": -30.846237182617188,
      "activations/layer12_attention_weight_max": 26.170244216918945,
      "activations/layer12_attention_weight_min": -26.11173439025879,
      "activations/layer13_attention_weight_max": 36.010005950927734,
      "activations/layer13_attention_weight_min": -36.5841064453125,
      "activations/layer14_attention_weight_max": 43.81351089477539,
      "activations/layer14_attention_weight_min": -37.37525177001953,
      "activations/layer15_attention_weight_max": 35.03588104248047,
      "activations/layer15_attention_weight_min": -35.65066146850586,
      "activations/layer16_attention_weight_max": 29.828317642211914,
      "activations/layer16_attention_weight_min": -30.37689208984375,
      "activations/layer17_attention_weight_max": 32.63445281982422,
      "activations/layer17_attention_weight_min": -25.420515060424805,
      "activations/layer18_attention_weight_max": 32.69881057739258,
      "activations/layer18_attention_weight_min": -25.450973510742188,
      "activations/layer19_attention_weight_max": 33.38519287109375,
      "activations/layer19_attention_weight_min": -32.20262908935547,
      "activations/layer1_attention_weight_max": 15.808128356933594,
      "activations/layer1_attention_weight_min": -14.443361282348633,
      "activations/layer20_attention_weight_max": 29.39790916442871,
      "activations/layer20_attention_weight_min": -24.645261764526367,
      "activations/layer21_attention_weight_max": 30.215612411499023,
      "activations/layer21_attention_weight_min": -24.201927185058594,
      "activations/layer22_attention_weight_max": 46.43097686767578,
      "activations/layer22_attention_weight_min": -30.692060470581055,
      "activations/layer23_attention_weight_max": 32.8517951965332,
      "activations/layer23_attention_weight_min": -25.066360473632812,
      "activations/layer2_attention_weight_max": 32.22173309326172,
      "activations/layer2_attention_weight_min": -31.708465576171875,
      "activations/layer3_attention_weight_max": 95.52139282226562,
      "activations/layer3_attention_weight_min": -94.8871078491211,
      "activations/layer4_attention_weight_max": 107.65278625488281,
      "activations/layer4_attention_weight_min": -107.23738861083984,
      "activations/layer5_attention_weight_max": 49.52930450439453,
      "activations/layer5_attention_weight_min": -59.843536376953125,
      "activations/layer6_attention_weight_max": 41.16013717651367,
      "activations/layer6_attention_weight_min": -44.57268142700195,
      "activations/layer7_attention_weight_max": 90.66541290283203,
      "activations/layer7_attention_weight_min": -83.55828094482422,
      "activations/layer8_attention_weight_max": 36.87843704223633,
      "activations/layer8_attention_weight_min": -39.23809051513672,
      "activations/layer9_attention_weight_max": 30.18829917907715,
      "activations/layer9_attention_weight_min": -29.209787368774414,
      "epoch": 15.74,
      "learning_rate": 4.89655303030303e-05,
      "loss": 2.7553,
      "step": 270950
    },
    {
      "activations/layer0_attention_weight_max": 16.31613540649414,
      "activations/layer0_attention_weight_min": -12.356982231140137,
      "activations/layer10_attention_weight_max": 32.1879997253418,
      "activations/layer10_attention_weight_min": -36.19973373413086,
      "activations/layer11_attention_weight_max": 31.761985778808594,
      "activations/layer11_attention_weight_min": -32.06056213378906,
      "activations/layer12_attention_weight_max": 24.920013427734375,
      "activations/layer12_attention_weight_min": -25.511146545410156,
      "activations/layer13_attention_weight_max": 37.379085540771484,
      "activations/layer13_attention_weight_min": -36.839778900146484,
      "activations/layer14_attention_weight_max": 40.21311569213867,
      "activations/layer14_attention_weight_min": -36.88349533081055,
      "activations/layer15_attention_weight_max": 36.440330505371094,
      "activations/layer15_attention_weight_min": -35.021907806396484,
      "activations/layer16_attention_weight_max": 29.532447814941406,
      "activations/layer16_attention_weight_min": -27.663602828979492,
      "activations/layer17_attention_weight_max": 29.172040939331055,
      "activations/layer17_attention_weight_min": -24.566478729248047,
      "activations/layer18_attention_weight_max": 31.73904037475586,
      "activations/layer18_attention_weight_min": -23.117868423461914,
      "activations/layer19_attention_weight_max": 34.73588943481445,
      "activations/layer19_attention_weight_min": -30.000165939331055,
      "activations/layer1_attention_weight_max": 16.008481979370117,
      "activations/layer1_attention_weight_min": -14.356893539428711,
      "activations/layer20_attention_weight_max": 31.139009475708008,
      "activations/layer20_attention_weight_min": -24.920475006103516,
      "activations/layer21_attention_weight_max": 30.47311782836914,
      "activations/layer21_attention_weight_min": -24.440948486328125,
      "activations/layer22_attention_weight_max": 45.50568389892578,
      "activations/layer22_attention_weight_min": -30.73752784729004,
      "activations/layer23_attention_weight_max": 30.014371871948242,
      "activations/layer23_attention_weight_min": -26.195083618164062,
      "activations/layer2_attention_weight_max": 32.694217681884766,
      "activations/layer2_attention_weight_min": -32.17060852050781,
      "activations/layer3_attention_weight_max": 93.38578033447266,
      "activations/layer3_attention_weight_min": -96.77590942382812,
      "activations/layer4_attention_weight_max": 103.45304107666016,
      "activations/layer4_attention_weight_min": -104.7427749633789,
      "activations/layer5_attention_weight_max": 47.65715026855469,
      "activations/layer5_attention_weight_min": -57.742881774902344,
      "activations/layer6_attention_weight_max": 42.41947937011719,
      "activations/layer6_attention_weight_min": -47.45039367675781,
      "activations/layer7_attention_weight_max": 86.8852310180664,
      "activations/layer7_attention_weight_min": -94.47979736328125,
      "activations/layer8_attention_weight_max": 37.64488220214844,
      "activations/layer8_attention_weight_min": -41.06121826171875,
      "activations/layer9_attention_weight_max": 30.51508140563965,
      "activations/layer9_attention_weight_min": -33.04624938964844,
      "epoch": 15.75,
      "learning_rate": 4.894659090909091e-05,
      "loss": 2.7427,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4531,
      "eval_samples_per_second": 507.979,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4531,
      "eval_openwebtext_samples_per_second": 507.979,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_wikitext_loss": 2.92578125,
      "eval_wikitext_ppl": 18.648789727196185,
      "eval_wikitext_runtime": 1.9459,
      "eval_wikitext_samples_per_second": 234.338,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_lambada_loss": 2.4296875,
      "eval_lambada_ppl": 11.3553329839266,
      "eval_lambada_runtime": 9.5349,
      "eval_lambada_samples_per_second": 510.651,
      "step": 271000
    },
    {
      "activations/layer0_attention_weight_max": 16.12416648864746,
      "activations/layer0_attention_weight_min": -12.559157371520996,
      "activations/layer10_attention_weight_max": 33.308868408203125,
      "activations/layer10_attention_weight_min": -34.800994873046875,
      "activations/layer11_attention_weight_max": 32.758094787597656,
      "activations/layer11_attention_weight_min": -35.1064453125,
      "activations/layer12_attention_weight_max": 24.943994522094727,
      "activations/layer12_attention_weight_min": -26.555665969848633,
      "activations/layer13_attention_weight_max": 37.47010803222656,
      "activations/layer13_attention_weight_min": -36.11164093017578,
      "activations/layer14_attention_weight_max": 39.649166107177734,
      "activations/layer14_attention_weight_min": -39.336341857910156,
      "activations/layer15_attention_weight_max": 37.340877532958984,
      "activations/layer15_attention_weight_min": -37.52699279785156,
      "activations/layer16_attention_weight_max": 30.22197151184082,
      "activations/layer16_attention_weight_min": -29.369308471679688,
      "activations/layer17_attention_weight_max": 30.633544921875,
      "activations/layer17_attention_weight_min": -26.87958526611328,
      "activations/layer18_attention_weight_max": 32.831077575683594,
      "activations/layer18_attention_weight_min": -25.031579971313477,
      "activations/layer19_attention_weight_max": 31.033620834350586,
      "activations/layer19_attention_weight_min": -31.63102149963379,
      "activations/layer1_attention_weight_max": 16.30693817138672,
      "activations/layer1_attention_weight_min": -15.037566184997559,
      "activations/layer20_attention_weight_max": 27.900012969970703,
      "activations/layer20_attention_weight_min": -25.31256103515625,
      "activations/layer21_attention_weight_max": 25.95062828063965,
      "activations/layer21_attention_weight_min": -24.882156372070312,
      "activations/layer22_attention_weight_max": 39.298091888427734,
      "activations/layer22_attention_weight_min": -31.3856201171875,
      "activations/layer23_attention_weight_max": 31.602720260620117,
      "activations/layer23_attention_weight_min": -27.92084312438965,
      "activations/layer2_attention_weight_max": 32.41230010986328,
      "activations/layer2_attention_weight_min": -32.898277282714844,
      "activations/layer3_attention_weight_max": 95.10039520263672,
      "activations/layer3_attention_weight_min": -94.6451187133789,
      "activations/layer4_attention_weight_max": 106.48248291015625,
      "activations/layer4_attention_weight_min": -109.1977767944336,
      "activations/layer5_attention_weight_max": 51.02973175048828,
      "activations/layer5_attention_weight_min": -62.253047943115234,
      "activations/layer6_attention_weight_max": 44.21475601196289,
      "activations/layer6_attention_weight_min": -44.79239273071289,
      "activations/layer7_attention_weight_max": 94.37446594238281,
      "activations/layer7_attention_weight_min": -92.94062805175781,
      "activations/layer8_attention_weight_max": 40.9435920715332,
      "activations/layer8_attention_weight_min": -43.945796966552734,
      "activations/layer9_attention_weight_max": 33.56106185913086,
      "activations/layer9_attention_weight_min": -33.75849533081055,
      "epoch": 15.75,
      "learning_rate": 4.892765151515151e-05,
      "loss": 2.7655,
      "step": 271050
    },
    {
      "activations/layer0_attention_weight_max": 15.825557708740234,
      "activations/layer0_attention_weight_min": -12.312832832336426,
      "activations/layer10_attention_weight_max": 32.08314895629883,
      "activations/layer10_attention_weight_min": -33.636070251464844,
      "activations/layer11_attention_weight_max": 31.530967712402344,
      "activations/layer11_attention_weight_min": -33.167236328125,
      "activations/layer12_attention_weight_max": 25.13902473449707,
      "activations/layer12_attention_weight_min": -27.46161460876465,
      "activations/layer13_attention_weight_max": 37.42387008666992,
      "activations/layer13_attention_weight_min": -34.53738021850586,
      "activations/layer14_attention_weight_max": 38.13559341430664,
      "activations/layer14_attention_weight_min": -37.14261245727539,
      "activations/layer15_attention_weight_max": 35.42606735229492,
      "activations/layer15_attention_weight_min": -33.27022933959961,
      "activations/layer16_attention_weight_max": 28.89554214477539,
      "activations/layer16_attention_weight_min": -26.530412673950195,
      "activations/layer17_attention_weight_max": 26.87433433532715,
      "activations/layer17_attention_weight_min": -27.418399810791016,
      "activations/layer18_attention_weight_max": 33.48312759399414,
      "activations/layer18_attention_weight_min": -24.381986618041992,
      "activations/layer19_attention_weight_max": 33.33707809448242,
      "activations/layer19_attention_weight_min": -31.450977325439453,
      "activations/layer1_attention_weight_max": 16.421470642089844,
      "activations/layer1_attention_weight_min": -15.056533813476562,
      "activations/layer20_attention_weight_max": 27.971439361572266,
      "activations/layer20_attention_weight_min": -26.43829345703125,
      "activations/layer21_attention_weight_max": 26.26064682006836,
      "activations/layer21_attention_weight_min": -23.62826919555664,
      "activations/layer22_attention_weight_max": 42.651424407958984,
      "activations/layer22_attention_weight_min": -31.05558204650879,
      "activations/layer23_attention_weight_max": 30.900394439697266,
      "activations/layer23_attention_weight_min": -24.487668991088867,
      "activations/layer2_attention_weight_max": 31.447072982788086,
      "activations/layer2_attention_weight_min": -32.60921096801758,
      "activations/layer3_attention_weight_max": 94.59484100341797,
      "activations/layer3_attention_weight_min": -92.66349792480469,
      "activations/layer4_attention_weight_max": 105.01909637451172,
      "activations/layer4_attention_weight_min": -105.11608123779297,
      "activations/layer5_attention_weight_max": 47.26142120361328,
      "activations/layer5_attention_weight_min": -60.74760055541992,
      "activations/layer6_attention_weight_max": 42.342281341552734,
      "activations/layer6_attention_weight_min": -47.52296829223633,
      "activations/layer7_attention_weight_max": 83.62191009521484,
      "activations/layer7_attention_weight_min": -88.21614074707031,
      "activations/layer8_attention_weight_max": 40.33317947387695,
      "activations/layer8_attention_weight_min": -42.11272048950195,
      "activations/layer9_attention_weight_max": 32.126991271972656,
      "activations/layer9_attention_weight_min": -32.24312210083008,
      "epoch": 15.75,
      "learning_rate": 4.8908712121212117e-05,
      "loss": 2.7644,
      "step": 271100
    },
    {
      "activations/layer0_attention_weight_max": 15.855130195617676,
      "activations/layer0_attention_weight_min": -11.705753326416016,
      "activations/layer10_attention_weight_max": 37.42165756225586,
      "activations/layer10_attention_weight_min": -33.538848876953125,
      "activations/layer11_attention_weight_max": 37.565853118896484,
      "activations/layer11_attention_weight_min": -34.00054931640625,
      "activations/layer12_attention_weight_max": 24.58975601196289,
      "activations/layer12_attention_weight_min": -27.168725967407227,
      "activations/layer13_attention_weight_max": 37.428627014160156,
      "activations/layer13_attention_weight_min": -36.29125213623047,
      "activations/layer14_attention_weight_max": 42.97159194946289,
      "activations/layer14_attention_weight_min": -40.538848876953125,
      "activations/layer15_attention_weight_max": 35.430599212646484,
      "activations/layer15_attention_weight_min": -35.21906661987305,
      "activations/layer16_attention_weight_max": 28.904088973999023,
      "activations/layer16_attention_weight_min": -29.35089683532715,
      "activations/layer17_attention_weight_max": 31.49142837524414,
      "activations/layer17_attention_weight_min": -26.59781837463379,
      "activations/layer18_attention_weight_max": 30.566364288330078,
      "activations/layer18_attention_weight_min": -25.627927780151367,
      "activations/layer19_attention_weight_max": 31.750070571899414,
      "activations/layer19_attention_weight_min": -32.65900802612305,
      "activations/layer1_attention_weight_max": 17.007143020629883,
      "activations/layer1_attention_weight_min": -16.295961380004883,
      "activations/layer20_attention_weight_max": 28.924177169799805,
      "activations/layer20_attention_weight_min": -25.736026763916016,
      "activations/layer21_attention_weight_max": 28.851409912109375,
      "activations/layer21_attention_weight_min": -27.106998443603516,
      "activations/layer22_attention_weight_max": 40.75227737426758,
      "activations/layer22_attention_weight_min": -33.69659423828125,
      "activations/layer23_attention_weight_max": 30.752761840820312,
      "activations/layer23_attention_weight_min": -27.000242233276367,
      "activations/layer2_attention_weight_max": 32.68456268310547,
      "activations/layer2_attention_weight_min": -32.44091033935547,
      "activations/layer3_attention_weight_max": 94.77151489257812,
      "activations/layer3_attention_weight_min": -95.85731506347656,
      "activations/layer4_attention_weight_max": 104.36019134521484,
      "activations/layer4_attention_weight_min": -106.50140380859375,
      "activations/layer5_attention_weight_max": 49.71546936035156,
      "activations/layer5_attention_weight_min": -59.891414642333984,
      "activations/layer6_attention_weight_max": 44.4350471496582,
      "activations/layer6_attention_weight_min": -47.34524154663086,
      "activations/layer7_attention_weight_max": 93.17279815673828,
      "activations/layer7_attention_weight_min": -94.56112670898438,
      "activations/layer8_attention_weight_max": 41.50981521606445,
      "activations/layer8_attention_weight_min": -45.94902801513672,
      "activations/layer9_attention_weight_max": 32.18972396850586,
      "activations/layer9_attention_weight_min": -33.26471710205078,
      "epoch": 15.76,
      "learning_rate": 4.888977272727272e-05,
      "loss": 2.747,
      "step": 271150
    },
    {
      "activations/layer0_attention_weight_max": 15.928752899169922,
      "activations/layer0_attention_weight_min": -13.794445991516113,
      "activations/layer10_attention_weight_max": 36.575523376464844,
      "activations/layer10_attention_weight_min": -35.03498840332031,
      "activations/layer11_attention_weight_max": 36.15909957885742,
      "activations/layer11_attention_weight_min": -34.1090087890625,
      "activations/layer12_attention_weight_max": 25.020954132080078,
      "activations/layer12_attention_weight_min": -25.606178283691406,
      "activations/layer13_attention_weight_max": 37.19891357421875,
      "activations/layer13_attention_weight_min": -33.85367965698242,
      "activations/layer14_attention_weight_max": 40.17121505737305,
      "activations/layer14_attention_weight_min": -36.790340423583984,
      "activations/layer15_attention_weight_max": 35.927555084228516,
      "activations/layer15_attention_weight_min": -33.79235076904297,
      "activations/layer16_attention_weight_max": 28.724475860595703,
      "activations/layer16_attention_weight_min": -28.078588485717773,
      "activations/layer17_attention_weight_max": 30.045984268188477,
      "activations/layer17_attention_weight_min": -26.4169864654541,
      "activations/layer18_attention_weight_max": 30.52781867980957,
      "activations/layer18_attention_weight_min": -23.861825942993164,
      "activations/layer19_attention_weight_max": 33.62920379638672,
      "activations/layer19_attention_weight_min": -31.90339469909668,
      "activations/layer1_attention_weight_max": 18.000627517700195,
      "activations/layer1_attention_weight_min": -15.844971656799316,
      "activations/layer20_attention_weight_max": 27.74944496154785,
      "activations/layer20_attention_weight_min": -27.290138244628906,
      "activations/layer21_attention_weight_max": 27.204418182373047,
      "activations/layer21_attention_weight_min": -25.75290870666504,
      "activations/layer22_attention_weight_max": 38.686466217041016,
      "activations/layer22_attention_weight_min": -32.426239013671875,
      "activations/layer23_attention_weight_max": 31.874013900756836,
      "activations/layer23_attention_weight_min": -26.42219352722168,
      "activations/layer2_attention_weight_max": 31.53077507019043,
      "activations/layer2_attention_weight_min": -32.18034744262695,
      "activations/layer3_attention_weight_max": 94.89425659179688,
      "activations/layer3_attention_weight_min": -93.66352844238281,
      "activations/layer4_attention_weight_max": 106.1312484741211,
      "activations/layer4_attention_weight_min": -107.12623596191406,
      "activations/layer5_attention_weight_max": 53.197689056396484,
      "activations/layer5_attention_weight_min": -63.129150390625,
      "activations/layer6_attention_weight_max": 43.07749938964844,
      "activations/layer6_attention_weight_min": -45.16852951049805,
      "activations/layer7_attention_weight_max": 86.09573364257812,
      "activations/layer7_attention_weight_min": -96.66429138183594,
      "activations/layer8_attention_weight_max": 43.383182525634766,
      "activations/layer8_attention_weight_min": -45.54143142700195,
      "activations/layer9_attention_weight_max": 33.34370422363281,
      "activations/layer9_attention_weight_min": -34.404884338378906,
      "epoch": 15.76,
      "learning_rate": 4.887083333333333e-05,
      "loss": 2.7384,
      "step": 271200
    },
    {
      "activations/layer0_attention_weight_max": 15.716170310974121,
      "activations/layer0_attention_weight_min": -12.600522994995117,
      "activations/layer10_attention_weight_max": 34.3035888671875,
      "activations/layer10_attention_weight_min": -33.37947463989258,
      "activations/layer11_attention_weight_max": 34.62421417236328,
      "activations/layer11_attention_weight_min": -35.66533279418945,
      "activations/layer12_attention_weight_max": 26.34377670288086,
      "activations/layer12_attention_weight_min": -25.273212432861328,
      "activations/layer13_attention_weight_max": 39.61967849731445,
      "activations/layer13_attention_weight_min": -35.23800277709961,
      "activations/layer14_attention_weight_max": 45.994659423828125,
      "activations/layer14_attention_weight_min": -36.49937438964844,
      "activations/layer15_attention_weight_max": 40.40455627441406,
      "activations/layer15_attention_weight_min": -34.97908020019531,
      "activations/layer16_attention_weight_max": 35.83683776855469,
      "activations/layer16_attention_weight_min": -28.15668296813965,
      "activations/layer17_attention_weight_max": 33.47685241699219,
      "activations/layer17_attention_weight_min": -26.47088623046875,
      "activations/layer18_attention_weight_max": 32.90229415893555,
      "activations/layer18_attention_weight_min": -25.663372039794922,
      "activations/layer19_attention_weight_max": 41.75751876831055,
      "activations/layer19_attention_weight_min": -32.878841400146484,
      "activations/layer1_attention_weight_max": 17.933568954467773,
      "activations/layer1_attention_weight_min": -16.61655616760254,
      "activations/layer20_attention_weight_max": 34.54830551147461,
      "activations/layer20_attention_weight_min": -25.829334259033203,
      "activations/layer21_attention_weight_max": 32.093727111816406,
      "activations/layer21_attention_weight_min": -24.9162654876709,
      "activations/layer22_attention_weight_max": 53.0164794921875,
      "activations/layer22_attention_weight_min": -32.34132766723633,
      "activations/layer23_attention_weight_max": 44.35879135131836,
      "activations/layer23_attention_weight_min": -25.75147819519043,
      "activations/layer2_attention_weight_max": 33.729515075683594,
      "activations/layer2_attention_weight_min": -32.496795654296875,
      "activations/layer3_attention_weight_max": 97.44274139404297,
      "activations/layer3_attention_weight_min": -95.25923919677734,
      "activations/layer4_attention_weight_max": 107.58354949951172,
      "activations/layer4_attention_weight_min": -107.33373260498047,
      "activations/layer5_attention_weight_max": 50.58143615722656,
      "activations/layer5_attention_weight_min": -62.26776885986328,
      "activations/layer6_attention_weight_max": 44.69179916381836,
      "activations/layer6_attention_weight_min": -47.266746520996094,
      "activations/layer7_attention_weight_max": 96.74191284179688,
      "activations/layer7_attention_weight_min": -89.52584838867188,
      "activations/layer8_attention_weight_max": 42.055458068847656,
      "activations/layer8_attention_weight_min": -43.25526428222656,
      "activations/layer9_attention_weight_max": 34.57383346557617,
      "activations/layer9_attention_weight_min": -33.676170349121094,
      "epoch": 15.76,
      "learning_rate": 4.8851893939393935e-05,
      "loss": 2.7315,
      "step": 271250
    },
    {
      "activations/layer0_attention_weight_max": 15.84023666381836,
      "activations/layer0_attention_weight_min": -12.939825057983398,
      "activations/layer10_attention_weight_max": 35.10625076293945,
      "activations/layer10_attention_weight_min": -35.2333984375,
      "activations/layer11_attention_weight_max": 35.44734191894531,
      "activations/layer11_attention_weight_min": -34.7603874206543,
      "activations/layer12_attention_weight_max": 24.695663452148438,
      "activations/layer12_attention_weight_min": -25.34166145324707,
      "activations/layer13_attention_weight_max": 36.46974563598633,
      "activations/layer13_attention_weight_min": -34.73439025878906,
      "activations/layer14_attention_weight_max": 40.191749572753906,
      "activations/layer14_attention_weight_min": -35.673683166503906,
      "activations/layer15_attention_weight_max": 40.18863296508789,
      "activations/layer15_attention_weight_min": -33.44819259643555,
      "activations/layer16_attention_weight_max": 32.599117279052734,
      "activations/layer16_attention_weight_min": -27.268321990966797,
      "activations/layer17_attention_weight_max": 31.772377014160156,
      "activations/layer17_attention_weight_min": -25.310340881347656,
      "activations/layer18_attention_weight_max": 34.1356315612793,
      "activations/layer18_attention_weight_min": -25.94008445739746,
      "activations/layer19_attention_weight_max": 36.1788215637207,
      "activations/layer19_attention_weight_min": -33.02497100830078,
      "activations/layer1_attention_weight_max": 16.145862579345703,
      "activations/layer1_attention_weight_min": -14.788969993591309,
      "activations/layer20_attention_weight_max": 31.255096435546875,
      "activations/layer20_attention_weight_min": -25.654067993164062,
      "activations/layer21_attention_weight_max": 35.352027893066406,
      "activations/layer21_attention_weight_min": -27.458261489868164,
      "activations/layer22_attention_weight_max": 48.26035690307617,
      "activations/layer22_attention_weight_min": -34.26970291137695,
      "activations/layer23_attention_weight_max": 36.28604507446289,
      "activations/layer23_attention_weight_min": -27.447864532470703,
      "activations/layer2_attention_weight_max": 31.081594467163086,
      "activations/layer2_attention_weight_min": -32.81575012207031,
      "activations/layer3_attention_weight_max": 98.99285888671875,
      "activations/layer3_attention_weight_min": -94.74726104736328,
      "activations/layer4_attention_weight_max": 107.92378997802734,
      "activations/layer4_attention_weight_min": -104.08975982666016,
      "activations/layer5_attention_weight_max": 48.02682113647461,
      "activations/layer5_attention_weight_min": -60.269752502441406,
      "activations/layer6_attention_weight_max": 44.87356948852539,
      "activations/layer6_attention_weight_min": -44.06509017944336,
      "activations/layer7_attention_weight_max": 91.94778442382812,
      "activations/layer7_attention_weight_min": -89.17184448242188,
      "activations/layer8_attention_weight_max": 40.4422721862793,
      "activations/layer8_attention_weight_min": -40.991058349609375,
      "activations/layer9_attention_weight_max": 31.255355834960938,
      "activations/layer9_attention_weight_min": -34.56980895996094,
      "epoch": 15.76,
      "learning_rate": 4.8832954545454536e-05,
      "loss": 2.7518,
      "step": 271300
    },
    {
      "activations/layer0_attention_weight_max": 15.93162727355957,
      "activations/layer0_attention_weight_min": -12.333540916442871,
      "activations/layer10_attention_weight_max": 32.93602752685547,
      "activations/layer10_attention_weight_min": -33.65222930908203,
      "activations/layer11_attention_weight_max": 34.1220703125,
      "activations/layer11_attention_weight_min": -34.06558609008789,
      "activations/layer12_attention_weight_max": 26.029712677001953,
      "activations/layer12_attention_weight_min": -25.585203170776367,
      "activations/layer13_attention_weight_max": 39.7906379699707,
      "activations/layer13_attention_weight_min": -35.536739349365234,
      "activations/layer14_attention_weight_max": 43.998477935791016,
      "activations/layer14_attention_weight_min": -36.86687088012695,
      "activations/layer15_attention_weight_max": 37.72784423828125,
      "activations/layer15_attention_weight_min": -32.507991790771484,
      "activations/layer16_attention_weight_max": 31.314346313476562,
      "activations/layer16_attention_weight_min": -26.983531951904297,
      "activations/layer17_attention_weight_max": 32.23063278198242,
      "activations/layer17_attention_weight_min": -25.1983585357666,
      "activations/layer18_attention_weight_max": 34.03443908691406,
      "activations/layer18_attention_weight_min": -24.591753005981445,
      "activations/layer19_attention_weight_max": 34.22480392456055,
      "activations/layer19_attention_weight_min": -32.63423156738281,
      "activations/layer1_attention_weight_max": 15.738134384155273,
      "activations/layer1_attention_weight_min": -14.907430648803711,
      "activations/layer20_attention_weight_max": 30.170995712280273,
      "activations/layer20_attention_weight_min": -25.97052574157715,
      "activations/layer21_attention_weight_max": 30.439586639404297,
      "activations/layer21_attention_weight_min": -26.0426082611084,
      "activations/layer22_attention_weight_max": 41.84870529174805,
      "activations/layer22_attention_weight_min": -32.4282112121582,
      "activations/layer23_attention_weight_max": 31.646055221557617,
      "activations/layer23_attention_weight_min": -26.823402404785156,
      "activations/layer2_attention_weight_max": 32.371009826660156,
      "activations/layer2_attention_weight_min": -32.26148223876953,
      "activations/layer3_attention_weight_max": 95.5930404663086,
      "activations/layer3_attention_weight_min": -97.6394271850586,
      "activations/layer4_attention_weight_max": 103.90455627441406,
      "activations/layer4_attention_weight_min": -110.05714416503906,
      "activations/layer5_attention_weight_max": 51.815704345703125,
      "activations/layer5_attention_weight_min": -62.83858871459961,
      "activations/layer6_attention_weight_max": 44.52641296386719,
      "activations/layer6_attention_weight_min": -45.86841583251953,
      "activations/layer7_attention_weight_max": 92.80374908447266,
      "activations/layer7_attention_weight_min": -92.7144775390625,
      "activations/layer8_attention_weight_max": 40.64643096923828,
      "activations/layer8_attention_weight_min": -42.44363784790039,
      "activations/layer9_attention_weight_max": 33.0546989440918,
      "activations/layer9_attention_weight_min": -34.587127685546875,
      "epoch": 15.77,
      "learning_rate": 4.881401515151515e-05,
      "loss": 2.7365,
      "step": 271350
    },
    {
      "activations/layer0_attention_weight_max": 16.246259689331055,
      "activations/layer0_attention_weight_min": -12.29356861114502,
      "activations/layer10_attention_weight_max": 32.73131561279297,
      "activations/layer10_attention_weight_min": -35.17439651489258,
      "activations/layer11_attention_weight_max": 32.918880462646484,
      "activations/layer11_attention_weight_min": -34.7316780090332,
      "activations/layer12_attention_weight_max": 26.657712936401367,
      "activations/layer12_attention_weight_min": -27.126785278320312,
      "activations/layer13_attention_weight_max": 38.670406341552734,
      "activations/layer13_attention_weight_min": -35.87177658081055,
      "activations/layer14_attention_weight_max": 44.24082565307617,
      "activations/layer14_attention_weight_min": -40.06522750854492,
      "activations/layer15_attention_weight_max": 37.06328582763672,
      "activations/layer15_attention_weight_min": -37.37104415893555,
      "activations/layer16_attention_weight_max": 28.611244201660156,
      "activations/layer16_attention_weight_min": -27.840906143188477,
      "activations/layer17_attention_weight_max": 29.900041580200195,
      "activations/layer17_attention_weight_min": -27.222057342529297,
      "activations/layer18_attention_weight_max": 33.2262077331543,
      "activations/layer18_attention_weight_min": -26.58222007751465,
      "activations/layer19_attention_weight_max": 37.03740692138672,
      "activations/layer19_attention_weight_min": -32.874210357666016,
      "activations/layer1_attention_weight_max": 17.21782684326172,
      "activations/layer1_attention_weight_min": -14.578856468200684,
      "activations/layer20_attention_weight_max": 29.55978012084961,
      "activations/layer20_attention_weight_min": -25.0660400390625,
      "activations/layer21_attention_weight_max": 28.520450592041016,
      "activations/layer21_attention_weight_min": -24.45189666748047,
      "activations/layer22_attention_weight_max": 43.764427185058594,
      "activations/layer22_attention_weight_min": -31.534162521362305,
      "activations/layer23_attention_weight_max": 37.7416877746582,
      "activations/layer23_attention_weight_min": -28.275909423828125,
      "activations/layer2_attention_weight_max": 34.27796173095703,
      "activations/layer2_attention_weight_min": -32.01050567626953,
      "activations/layer3_attention_weight_max": 95.98905944824219,
      "activations/layer3_attention_weight_min": -97.63095092773438,
      "activations/layer4_attention_weight_max": 106.90716552734375,
      "activations/layer4_attention_weight_min": -107.24430084228516,
      "activations/layer5_attention_weight_max": 52.91880798339844,
      "activations/layer5_attention_weight_min": -64.37142944335938,
      "activations/layer6_attention_weight_max": 47.22069549560547,
      "activations/layer6_attention_weight_min": -46.1813850402832,
      "activations/layer7_attention_weight_max": 94.07830810546875,
      "activations/layer7_attention_weight_min": -95.8118896484375,
      "activations/layer8_attention_weight_max": 40.53163528442383,
      "activations/layer8_attention_weight_min": -42.191322326660156,
      "activations/layer9_attention_weight_max": 30.171005249023438,
      "activations/layer9_attention_weight_min": -32.47208786010742,
      "epoch": 15.77,
      "learning_rate": 4.879545454545454e-05,
      "loss": 2.7657,
      "step": 271400
    },
    {
      "activations/layer0_attention_weight_max": 15.745863914489746,
      "activations/layer0_attention_weight_min": -12.402010917663574,
      "activations/layer10_attention_weight_max": 32.513702392578125,
      "activations/layer10_attention_weight_min": -34.06897735595703,
      "activations/layer11_attention_weight_max": 32.40726089477539,
      "activations/layer11_attention_weight_min": -32.96184158325195,
      "activations/layer12_attention_weight_max": 25.421016693115234,
      "activations/layer12_attention_weight_min": -25.611276626586914,
      "activations/layer13_attention_weight_max": 37.00910949707031,
      "activations/layer13_attention_weight_min": -35.09988021850586,
      "activations/layer14_attention_weight_max": 42.76259994506836,
      "activations/layer14_attention_weight_min": -37.42898941040039,
      "activations/layer15_attention_weight_max": 36.41475296020508,
      "activations/layer15_attention_weight_min": -32.915592193603516,
      "activations/layer16_attention_weight_max": 28.81156349182129,
      "activations/layer16_attention_weight_min": -25.870176315307617,
      "activations/layer17_attention_weight_max": 27.462987899780273,
      "activations/layer17_attention_weight_min": -24.904708862304688,
      "activations/layer18_attention_weight_max": 30.137630462646484,
      "activations/layer18_attention_weight_min": -25.314653396606445,
      "activations/layer19_attention_weight_max": 33.099464416503906,
      "activations/layer19_attention_weight_min": -30.735021591186523,
      "activations/layer1_attention_weight_max": 16.278358459472656,
      "activations/layer1_attention_weight_min": -15.461862564086914,
      "activations/layer20_attention_weight_max": 28.058382034301758,
      "activations/layer20_attention_weight_min": -26.3498592376709,
      "activations/layer21_attention_weight_max": 28.71800422668457,
      "activations/layer21_attention_weight_min": -25.005046844482422,
      "activations/layer22_attention_weight_max": 38.0787239074707,
      "activations/layer22_attention_weight_min": -31.343761444091797,
      "activations/layer23_attention_weight_max": 30.931262969970703,
      "activations/layer23_attention_weight_min": -27.053543090820312,
      "activations/layer2_attention_weight_max": 33.90187072753906,
      "activations/layer2_attention_weight_min": -33.2479248046875,
      "activations/layer3_attention_weight_max": 95.63034057617188,
      "activations/layer3_attention_weight_min": -101.90557098388672,
      "activations/layer4_attention_weight_max": 108.81269836425781,
      "activations/layer4_attention_weight_min": -109.79723358154297,
      "activations/layer5_attention_weight_max": 54.20664596557617,
      "activations/layer5_attention_weight_min": -61.87449264526367,
      "activations/layer6_attention_weight_max": 44.27211380004883,
      "activations/layer6_attention_weight_min": -45.10991287231445,
      "activations/layer7_attention_weight_max": 93.31555938720703,
      "activations/layer7_attention_weight_min": -90.95244598388672,
      "activations/layer8_attention_weight_max": 38.50456619262695,
      "activations/layer8_attention_weight_min": -40.932456970214844,
      "activations/layer9_attention_weight_max": 30.945165634155273,
      "activations/layer9_attention_weight_min": -33.066768646240234,
      "epoch": 15.77,
      "learning_rate": 4.877651515151515e-05,
      "loss": 2.7552,
      "step": 271450
    },
    {
      "activations/layer0_attention_weight_max": 17.11993408203125,
      "activations/layer0_attention_weight_min": -13.0699462890625,
      "activations/layer10_attention_weight_max": 39.55902862548828,
      "activations/layer10_attention_weight_min": -38.2247314453125,
      "activations/layer11_attention_weight_max": 38.27442932128906,
      "activations/layer11_attention_weight_min": -39.84541320800781,
      "activations/layer12_attention_weight_max": 30.988561630249023,
      "activations/layer12_attention_weight_min": -28.39378547668457,
      "activations/layer13_attention_weight_max": 49.72470474243164,
      "activations/layer13_attention_weight_min": -39.95787811279297,
      "activations/layer14_attention_weight_max": 57.37211227416992,
      "activations/layer14_attention_weight_min": -43.262027740478516,
      "activations/layer15_attention_weight_max": 48.200469970703125,
      "activations/layer15_attention_weight_min": -41.48503875732422,
      "activations/layer16_attention_weight_max": 34.09524154663086,
      "activations/layer16_attention_weight_min": -33.28140640258789,
      "activations/layer17_attention_weight_max": 33.558589935302734,
      "activations/layer17_attention_weight_min": -30.826297760009766,
      "activations/layer18_attention_weight_max": 32.94630813598633,
      "activations/layer18_attention_weight_min": -28.69438362121582,
      "activations/layer19_attention_weight_max": 38.170162200927734,
      "activations/layer19_attention_weight_min": -34.374061584472656,
      "activations/layer1_attention_weight_max": 19.00958824157715,
      "activations/layer1_attention_weight_min": -17.38669776916504,
      "activations/layer20_attention_weight_max": 31.49237632751465,
      "activations/layer20_attention_weight_min": -26.188077926635742,
      "activations/layer21_attention_weight_max": 29.21059226989746,
      "activations/layer21_attention_weight_min": -25.789737701416016,
      "activations/layer22_attention_weight_max": 52.802310943603516,
      "activations/layer22_attention_weight_min": -34.21079635620117,
      "activations/layer23_attention_weight_max": 36.056514739990234,
      "activations/layer23_attention_weight_min": -25.4411678314209,
      "activations/layer2_attention_weight_max": 36.061424255371094,
      "activations/layer2_attention_weight_min": -34.76557922363281,
      "activations/layer3_attention_weight_max": 100.71703338623047,
      "activations/layer3_attention_weight_min": -100.99081420898438,
      "activations/layer4_attention_weight_max": 111.05610656738281,
      "activations/layer4_attention_weight_min": -110.83831787109375,
      "activations/layer5_attention_weight_max": 51.19265365600586,
      "activations/layer5_attention_weight_min": -63.51742172241211,
      "activations/layer6_attention_weight_max": 45.801082611083984,
      "activations/layer6_attention_weight_min": -47.589149475097656,
      "activations/layer7_attention_weight_max": 98.4769287109375,
      "activations/layer7_attention_weight_min": -100.58106994628906,
      "activations/layer8_attention_weight_max": 44.986263275146484,
      "activations/layer8_attention_weight_min": -47.22285842895508,
      "activations/layer9_attention_weight_max": 36.192134857177734,
      "activations/layer9_attention_weight_min": -37.296661376953125,
      "epoch": 15.78,
      "learning_rate": 4.875757575757575e-05,
      "loss": 2.7412,
      "step": 271500
    },
    {
      "activations/layer0_attention_weight_max": 15.84123706817627,
      "activations/layer0_attention_weight_min": -13.695945739746094,
      "activations/layer10_attention_weight_max": 39.27882766723633,
      "activations/layer10_attention_weight_min": -35.20053482055664,
      "activations/layer11_attention_weight_max": 40.335166931152344,
      "activations/layer11_attention_weight_min": -40.82849884033203,
      "activations/layer12_attention_weight_max": 26.07132911682129,
      "activations/layer12_attention_weight_min": -27.197017669677734,
      "activations/layer13_attention_weight_max": 36.76017761230469,
      "activations/layer13_attention_weight_min": -37.0294189453125,
      "activations/layer14_attention_weight_max": 45.19771194458008,
      "activations/layer14_attention_weight_min": -41.91902160644531,
      "activations/layer15_attention_weight_max": 42.94230651855469,
      "activations/layer15_attention_weight_min": -35.798583984375,
      "activations/layer16_attention_weight_max": 35.087249755859375,
      "activations/layer16_attention_weight_min": -27.909595489501953,
      "activations/layer17_attention_weight_max": 32.2121467590332,
      "activations/layer17_attention_weight_min": -29.793020248413086,
      "activations/layer18_attention_weight_max": 30.19655990600586,
      "activations/layer18_attention_weight_min": -26.156661987304688,
      "activations/layer19_attention_weight_max": 35.98427963256836,
      "activations/layer19_attention_weight_min": -31.247573852539062,
      "activations/layer1_attention_weight_max": 16.676963806152344,
      "activations/layer1_attention_weight_min": -15.358270645141602,
      "activations/layer20_attention_weight_max": 29.46236228942871,
      "activations/layer20_attention_weight_min": -24.66075897216797,
      "activations/layer21_attention_weight_max": 28.20749282836914,
      "activations/layer21_attention_weight_min": -25.191787719726562,
      "activations/layer22_attention_weight_max": 41.12946701049805,
      "activations/layer22_attention_weight_min": -31.65329360961914,
      "activations/layer23_attention_weight_max": 32.38566207885742,
      "activations/layer23_attention_weight_min": -23.948848724365234,
      "activations/layer2_attention_weight_max": 34.51636505126953,
      "activations/layer2_attention_weight_min": -34.12657928466797,
      "activations/layer3_attention_weight_max": 98.49921417236328,
      "activations/layer3_attention_weight_min": -97.89683532714844,
      "activations/layer4_attention_weight_max": 107.50239562988281,
      "activations/layer4_attention_weight_min": -110.09429168701172,
      "activations/layer5_attention_weight_max": 49.14896774291992,
      "activations/layer5_attention_weight_min": -61.70184326171875,
      "activations/layer6_attention_weight_max": 43.95488357543945,
      "activations/layer6_attention_weight_min": -48.019798278808594,
      "activations/layer7_attention_weight_max": 102.47035217285156,
      "activations/layer7_attention_weight_min": -95.60000610351562,
      "activations/layer8_attention_weight_max": 43.83657455444336,
      "activations/layer8_attention_weight_min": -45.43764114379883,
      "activations/layer9_attention_weight_max": 35.41857147216797,
      "activations/layer9_attention_weight_min": -35.41087341308594,
      "epoch": 15.78,
      "learning_rate": 4.873863636363636e-05,
      "loss": 2.7571,
      "step": 271550
    },
    {
      "activations/layer0_attention_weight_max": 15.969281196594238,
      "activations/layer0_attention_weight_min": -13.027206420898438,
      "activations/layer10_attention_weight_max": 36.56081008911133,
      "activations/layer10_attention_weight_min": -35.605003356933594,
      "activations/layer11_attention_weight_max": 36.27973937988281,
      "activations/layer11_attention_weight_min": -33.99858093261719,
      "activations/layer12_attention_weight_max": 27.814111709594727,
      "activations/layer12_attention_weight_min": -28.41273307800293,
      "activations/layer13_attention_weight_max": 39.96343231201172,
      "activations/layer13_attention_weight_min": -35.121463775634766,
      "activations/layer14_attention_weight_max": 45.528594970703125,
      "activations/layer14_attention_weight_min": -36.087093353271484,
      "activations/layer15_attention_weight_max": 37.1269416809082,
      "activations/layer15_attention_weight_min": -34.7109489440918,
      "activations/layer16_attention_weight_max": 31.157894134521484,
      "activations/layer16_attention_weight_min": -27.34377670288086,
      "activations/layer17_attention_weight_max": 29.25705909729004,
      "activations/layer17_attention_weight_min": -25.620685577392578,
      "activations/layer18_attention_weight_max": 33.23466491699219,
      "activations/layer18_attention_weight_min": -24.960187911987305,
      "activations/layer19_attention_weight_max": 33.29728698730469,
      "activations/layer19_attention_weight_min": -30.663721084594727,
      "activations/layer1_attention_weight_max": 15.702811241149902,
      "activations/layer1_attention_weight_min": -14.329655647277832,
      "activations/layer20_attention_weight_max": 27.89164924621582,
      "activations/layer20_attention_weight_min": -26.28799057006836,
      "activations/layer21_attention_weight_max": 28.434844970703125,
      "activations/layer21_attention_weight_min": -25.910839080810547,
      "activations/layer22_attention_weight_max": 44.324398040771484,
      "activations/layer22_attention_weight_min": -33.41792297363281,
      "activations/layer23_attention_weight_max": 33.412696838378906,
      "activations/layer23_attention_weight_min": -26.064334869384766,
      "activations/layer2_attention_weight_max": 31.53541374206543,
      "activations/layer2_attention_weight_min": -31.856557846069336,
      "activations/layer3_attention_weight_max": 92.9649429321289,
      "activations/layer3_attention_weight_min": -92.18228912353516,
      "activations/layer4_attention_weight_max": 102.90045166015625,
      "activations/layer4_attention_weight_min": -101.92525482177734,
      "activations/layer5_attention_weight_max": 49.58580017089844,
      "activations/layer5_attention_weight_min": -60.61448669433594,
      "activations/layer6_attention_weight_max": 42.95227813720703,
      "activations/layer6_attention_weight_min": -45.51970672607422,
      "activations/layer7_attention_weight_max": 98.14116668701172,
      "activations/layer7_attention_weight_min": -91.08458709716797,
      "activations/layer8_attention_weight_max": 43.18584442138672,
      "activations/layer8_attention_weight_min": -42.60409927368164,
      "activations/layer9_attention_weight_max": 33.32383346557617,
      "activations/layer9_attention_weight_min": -33.996273040771484,
      "epoch": 15.78,
      "learning_rate": 4.871969696969697e-05,
      "loss": 2.7475,
      "step": 271600
    },
    {
      "activations/layer0_attention_weight_max": 15.849749565124512,
      "activations/layer0_attention_weight_min": -13.37493896484375,
      "activations/layer10_attention_weight_max": 31.899686813354492,
      "activations/layer10_attention_weight_min": -33.09278869628906,
      "activations/layer11_attention_weight_max": 32.439971923828125,
      "activations/layer11_attention_weight_min": -33.2865104675293,
      "activations/layer12_attention_weight_max": 27.594114303588867,
      "activations/layer12_attention_weight_min": -25.314346313476562,
      "activations/layer13_attention_weight_max": 41.91840744018555,
      "activations/layer13_attention_weight_min": -34.47965621948242,
      "activations/layer14_attention_weight_max": 50.224525451660156,
      "activations/layer14_attention_weight_min": -36.2136116027832,
      "activations/layer15_attention_weight_max": 42.78078842163086,
      "activations/layer15_attention_weight_min": -35.71620178222656,
      "activations/layer16_attention_weight_max": 31.480224609375,
      "activations/layer16_attention_weight_min": -28.698444366455078,
      "activations/layer17_attention_weight_max": 28.84026527404785,
      "activations/layer17_attention_weight_min": -25.180234909057617,
      "activations/layer18_attention_weight_max": 33.38901901245117,
      "activations/layer18_attention_weight_min": -27.0976505279541,
      "activations/layer19_attention_weight_max": 32.68940734863281,
      "activations/layer19_attention_weight_min": -30.436660766601562,
      "activations/layer1_attention_weight_max": 16.636497497558594,
      "activations/layer1_attention_weight_min": -15.504389762878418,
      "activations/layer20_attention_weight_max": 27.441612243652344,
      "activations/layer20_attention_weight_min": -27.75743293762207,
      "activations/layer21_attention_weight_max": 28.303049087524414,
      "activations/layer21_attention_weight_min": -24.400066375732422,
      "activations/layer22_attention_weight_max": 44.78793716430664,
      "activations/layer22_attention_weight_min": -29.976438522338867,
      "activations/layer23_attention_weight_max": 31.90383529663086,
      "activations/layer23_attention_weight_min": -24.878433227539062,
      "activations/layer2_attention_weight_max": 33.05937194824219,
      "activations/layer2_attention_weight_min": -32.55878829956055,
      "activations/layer3_attention_weight_max": 94.84441375732422,
      "activations/layer3_attention_weight_min": -98.03771209716797,
      "activations/layer4_attention_weight_max": 103.14287567138672,
      "activations/layer4_attention_weight_min": -104.6595230102539,
      "activations/layer5_attention_weight_max": 48.081207275390625,
      "activations/layer5_attention_weight_min": -61.944313049316406,
      "activations/layer6_attention_weight_max": 42.56508255004883,
      "activations/layer6_attention_weight_min": -45.63267135620117,
      "activations/layer7_attention_weight_max": 91.58173370361328,
      "activations/layer7_attention_weight_min": -95.06906127929688,
      "activations/layer8_attention_weight_max": 39.98392868041992,
      "activations/layer8_attention_weight_min": -42.970123291015625,
      "activations/layer9_attention_weight_max": 32.03572463989258,
      "activations/layer9_attention_weight_min": -34.98077392578125,
      "epoch": 15.78,
      "learning_rate": 4.870075757575757e-05,
      "loss": 2.7412,
      "step": 271650
    },
    {
      "activations/layer0_attention_weight_max": 15.523036003112793,
      "activations/layer0_attention_weight_min": -12.815170288085938,
      "activations/layer10_attention_weight_max": 36.487335205078125,
      "activations/layer10_attention_weight_min": -33.00291061401367,
      "activations/layer11_attention_weight_max": 34.68260192871094,
      "activations/layer11_attention_weight_min": -34.16849899291992,
      "activations/layer12_attention_weight_max": 24.411596298217773,
      "activations/layer12_attention_weight_min": -26.026582717895508,
      "activations/layer13_attention_weight_max": 34.40250778198242,
      "activations/layer13_attention_weight_min": -33.71523666381836,
      "activations/layer14_attention_weight_max": 37.98784255981445,
      "activations/layer14_attention_weight_min": -34.73337173461914,
      "activations/layer15_attention_weight_max": 34.69293975830078,
      "activations/layer15_attention_weight_min": -32.043582916259766,
      "activations/layer16_attention_weight_max": 27.576339721679688,
      "activations/layer16_attention_weight_min": -27.7977237701416,
      "activations/layer17_attention_weight_max": 30.13901710510254,
      "activations/layer17_attention_weight_min": -25.192523956298828,
      "activations/layer18_attention_weight_max": 29.757768630981445,
      "activations/layer18_attention_weight_min": -24.38178062438965,
      "activations/layer19_attention_weight_max": 31.641347885131836,
      "activations/layer19_attention_weight_min": -30.239126205444336,
      "activations/layer1_attention_weight_max": 16.965194702148438,
      "activations/layer1_attention_weight_min": -14.649554252624512,
      "activations/layer20_attention_weight_max": 29.034801483154297,
      "activations/layer20_attention_weight_min": -23.88007164001465,
      "activations/layer21_attention_weight_max": 29.34259033203125,
      "activations/layer21_attention_weight_min": -23.00652503967285,
      "activations/layer22_attention_weight_max": 43.53066635131836,
      "activations/layer22_attention_weight_min": -30.42457389831543,
      "activations/layer23_attention_weight_max": 33.659088134765625,
      "activations/layer23_attention_weight_min": -26.723587036132812,
      "activations/layer2_attention_weight_max": 35.899070739746094,
      "activations/layer2_attention_weight_min": -34.65595245361328,
      "activations/layer3_attention_weight_max": 102.35778045654297,
      "activations/layer3_attention_weight_min": -102.21975708007812,
      "activations/layer4_attention_weight_max": 110.4295883178711,
      "activations/layer4_attention_weight_min": -106.36669158935547,
      "activations/layer5_attention_weight_max": 53.609397888183594,
      "activations/layer5_attention_weight_min": -63.30227279663086,
      "activations/layer6_attention_weight_max": 41.82366943359375,
      "activations/layer6_attention_weight_min": -42.720760345458984,
      "activations/layer7_attention_weight_max": 98.5277099609375,
      "activations/layer7_attention_weight_min": -92.38532257080078,
      "activations/layer8_attention_weight_max": 43.15144348144531,
      "activations/layer8_attention_weight_min": -41.37275695800781,
      "activations/layer9_attention_weight_max": 31.6844425201416,
      "activations/layer9_attention_weight_min": -33.63581466674805,
      "epoch": 15.79,
      "learning_rate": 4.868181818181817e-05,
      "loss": 2.7515,
      "step": 271700
    },
    {
      "activations/layer0_attention_weight_max": 17.010835647583008,
      "activations/layer0_attention_weight_min": -11.881658554077148,
      "activations/layer10_attention_weight_max": 33.90007400512695,
      "activations/layer10_attention_weight_min": -34.506080627441406,
      "activations/layer11_attention_weight_max": 34.966617584228516,
      "activations/layer11_attention_weight_min": -34.73894500732422,
      "activations/layer12_attention_weight_max": 27.08654022216797,
      "activations/layer12_attention_weight_min": -28.199914932250977,
      "activations/layer13_attention_weight_max": 41.54479217529297,
      "activations/layer13_attention_weight_min": -36.1514892578125,
      "activations/layer14_attention_weight_max": 45.421207427978516,
      "activations/layer14_attention_weight_min": -37.726966857910156,
      "activations/layer15_attention_weight_max": 38.52994155883789,
      "activations/layer15_attention_weight_min": -35.30062484741211,
      "activations/layer16_attention_weight_max": 29.167707443237305,
      "activations/layer16_attention_weight_min": -28.148317337036133,
      "activations/layer17_attention_weight_max": 29.91680908203125,
      "activations/layer17_attention_weight_min": -27.413341522216797,
      "activations/layer18_attention_weight_max": 34.91329574584961,
      "activations/layer18_attention_weight_min": -25.294958114624023,
      "activations/layer19_attention_weight_max": 33.905094146728516,
      "activations/layer19_attention_weight_min": -32.323326110839844,
      "activations/layer1_attention_weight_max": 16.767107009887695,
      "activations/layer1_attention_weight_min": -15.50800895690918,
      "activations/layer20_attention_weight_max": 30.1470947265625,
      "activations/layer20_attention_weight_min": -26.786624908447266,
      "activations/layer21_attention_weight_max": 29.013566970825195,
      "activations/layer21_attention_weight_min": -25.49631690979004,
      "activations/layer22_attention_weight_max": 40.11286544799805,
      "activations/layer22_attention_weight_min": -30.89626693725586,
      "activations/layer23_attention_weight_max": 35.86058807373047,
      "activations/layer23_attention_weight_min": -25.514114379882812,
      "activations/layer2_attention_weight_max": 32.179481506347656,
      "activations/layer2_attention_weight_min": -32.24629211425781,
      "activations/layer3_attention_weight_max": 97.0096664428711,
      "activations/layer3_attention_weight_min": -101.2662353515625,
      "activations/layer4_attention_weight_max": 107.1231689453125,
      "activations/layer4_attention_weight_min": -108.5812759399414,
      "activations/layer5_attention_weight_max": 51.272132873535156,
      "activations/layer5_attention_weight_min": -60.44541931152344,
      "activations/layer6_attention_weight_max": 44.64059066772461,
      "activations/layer6_attention_weight_min": -47.88205337524414,
      "activations/layer7_attention_weight_max": 92.62906646728516,
      "activations/layer7_attention_weight_min": -94.85908508300781,
      "activations/layer8_attention_weight_max": 43.08589172363281,
      "activations/layer8_attention_weight_min": -43.00251770019531,
      "activations/layer9_attention_weight_max": 34.02595138549805,
      "activations/layer9_attention_weight_min": -34.195762634277344,
      "epoch": 15.79,
      "learning_rate": 4.8662878787878786e-05,
      "loss": 2.7507,
      "step": 271750
    },
    {
      "activations/layer0_attention_weight_max": 15.30445384979248,
      "activations/layer0_attention_weight_min": -12.784000396728516,
      "activations/layer10_attention_weight_max": 33.98985290527344,
      "activations/layer10_attention_weight_min": -36.00030517578125,
      "activations/layer11_attention_weight_max": 33.52772521972656,
      "activations/layer11_attention_weight_min": -34.33258056640625,
      "activations/layer12_attention_weight_max": 26.497880935668945,
      "activations/layer12_attention_weight_min": -26.976472854614258,
      "activations/layer13_attention_weight_max": 39.32624816894531,
      "activations/layer13_attention_weight_min": -37.30284881591797,
      "activations/layer14_attention_weight_max": 41.11201477050781,
      "activations/layer14_attention_weight_min": -39.06502151489258,
      "activations/layer15_attention_weight_max": 38.01950454711914,
      "activations/layer15_attention_weight_min": -35.12321472167969,
      "activations/layer16_attention_weight_max": 29.07765007019043,
      "activations/layer16_attention_weight_min": -26.77593421936035,
      "activations/layer17_attention_weight_max": 32.45376968383789,
      "activations/layer17_attention_weight_min": -27.602291107177734,
      "activations/layer18_attention_weight_max": 33.05888748168945,
      "activations/layer18_attention_weight_min": -25.52480697631836,
      "activations/layer19_attention_weight_max": 33.001461029052734,
      "activations/layer19_attention_weight_min": -35.403648376464844,
      "activations/layer1_attention_weight_max": 16.553600311279297,
      "activations/layer1_attention_weight_min": -16.64188575744629,
      "activations/layer20_attention_weight_max": 29.700401306152344,
      "activations/layer20_attention_weight_min": -28.274280548095703,
      "activations/layer21_attention_weight_max": 29.871335983276367,
      "activations/layer21_attention_weight_min": -27.549741744995117,
      "activations/layer22_attention_weight_max": 41.8890380859375,
      "activations/layer22_attention_weight_min": -32.206092834472656,
      "activations/layer23_attention_weight_max": 32.520938873291016,
      "activations/layer23_attention_weight_min": -25.652843475341797,
      "activations/layer2_attention_weight_max": 33.923858642578125,
      "activations/layer2_attention_weight_min": -33.12111282348633,
      "activations/layer3_attention_weight_max": 103.22846984863281,
      "activations/layer3_attention_weight_min": -102.840087890625,
      "activations/layer4_attention_weight_max": 110.60314178466797,
      "activations/layer4_attention_weight_min": -114.40856170654297,
      "activations/layer5_attention_weight_max": 53.421199798583984,
      "activations/layer5_attention_weight_min": -62.185211181640625,
      "activations/layer6_attention_weight_max": 46.2127799987793,
      "activations/layer6_attention_weight_min": -49.841407775878906,
      "activations/layer7_attention_weight_max": 95.3792495727539,
      "activations/layer7_attention_weight_min": -98.25849151611328,
      "activations/layer8_attention_weight_max": 42.8192024230957,
      "activations/layer8_attention_weight_min": -43.9853515625,
      "activations/layer9_attention_weight_max": 32.66794967651367,
      "activations/layer9_attention_weight_min": -34.91096115112305,
      "epoch": 15.79,
      "learning_rate": 4.864393939393939e-05,
      "loss": 2.7493,
      "step": 271800
    },
    {
      "activations/layer0_attention_weight_max": 15.88431453704834,
      "activations/layer0_attention_weight_min": -13.329901695251465,
      "activations/layer10_attention_weight_max": 33.450233459472656,
      "activations/layer10_attention_weight_min": -33.656585693359375,
      "activations/layer11_attention_weight_max": 31.076862335205078,
      "activations/layer11_attention_weight_min": -34.2266845703125,
      "activations/layer12_attention_weight_max": 25.6219539642334,
      "activations/layer12_attention_weight_min": -27.3280086517334,
      "activations/layer13_attention_weight_max": 36.37971496582031,
      "activations/layer13_attention_weight_min": -33.399070739746094,
      "activations/layer14_attention_weight_max": 40.66244125366211,
      "activations/layer14_attention_weight_min": -35.653831481933594,
      "activations/layer15_attention_weight_max": 37.11472702026367,
      "activations/layer15_attention_weight_min": -34.0039176940918,
      "activations/layer16_attention_weight_max": 30.854394912719727,
      "activations/layer16_attention_weight_min": -28.06111717224121,
      "activations/layer17_attention_weight_max": 32.0424919128418,
      "activations/layer17_attention_weight_min": -25.728275299072266,
      "activations/layer18_attention_weight_max": 33.745941162109375,
      "activations/layer18_attention_weight_min": -27.231054306030273,
      "activations/layer19_attention_weight_max": 38.710872650146484,
      "activations/layer19_attention_weight_min": -32.9716911315918,
      "activations/layer1_attention_weight_max": 16.412302017211914,
      "activations/layer1_attention_weight_min": -15.866534233093262,
      "activations/layer20_attention_weight_max": 35.943809509277344,
      "activations/layer20_attention_weight_min": -25.656108856201172,
      "activations/layer21_attention_weight_max": 34.67665481567383,
      "activations/layer21_attention_weight_min": -24.727249145507812,
      "activations/layer22_attention_weight_max": 46.41400909423828,
      "activations/layer22_attention_weight_min": -31.24895477294922,
      "activations/layer23_attention_weight_max": 37.73211669921875,
      "activations/layer23_attention_weight_min": -25.9415340423584,
      "activations/layer2_attention_weight_max": 32.576961517333984,
      "activations/layer2_attention_weight_min": -32.5099983215332,
      "activations/layer3_attention_weight_max": 97.5732192993164,
      "activations/layer3_attention_weight_min": -97.87477111816406,
      "activations/layer4_attention_weight_max": 105.72452545166016,
      "activations/layer4_attention_weight_min": -105.53551483154297,
      "activations/layer5_attention_weight_max": 49.8798828125,
      "activations/layer5_attention_weight_min": -62.27965545654297,
      "activations/layer6_attention_weight_max": 44.42624282836914,
      "activations/layer6_attention_weight_min": -46.46809768676758,
      "activations/layer7_attention_weight_max": 90.67774200439453,
      "activations/layer7_attention_weight_min": -94.13497924804688,
      "activations/layer8_attention_weight_max": 40.015830993652344,
      "activations/layer8_attention_weight_min": -43.55718231201172,
      "activations/layer9_attention_weight_max": 32.650367736816406,
      "activations/layer9_attention_weight_min": -33.550193786621094,
      "epoch": 15.8,
      "learning_rate": 4.8624999999999996e-05,
      "loss": 2.7544,
      "step": 271850
    },
    {
      "activations/layer0_attention_weight_max": 15.783592224121094,
      "activations/layer0_attention_weight_min": -11.966358184814453,
      "activations/layer10_attention_weight_max": 33.248329162597656,
      "activations/layer10_attention_weight_min": -33.29206085205078,
      "activations/layer11_attention_weight_max": 33.49946594238281,
      "activations/layer11_attention_weight_min": -32.66065979003906,
      "activations/layer12_attention_weight_max": 25.94253158569336,
      "activations/layer12_attention_weight_min": -27.6097354888916,
      "activations/layer13_attention_weight_max": 38.347503662109375,
      "activations/layer13_attention_weight_min": -37.08793640136719,
      "activations/layer14_attention_weight_max": 39.778900146484375,
      "activations/layer14_attention_weight_min": -39.53458023071289,
      "activations/layer15_attention_weight_max": 36.416011810302734,
      "activations/layer15_attention_weight_min": -39.05195617675781,
      "activations/layer16_attention_weight_max": 31.990467071533203,
      "activations/layer16_attention_weight_min": -28.424314498901367,
      "activations/layer17_attention_weight_max": 34.073760986328125,
      "activations/layer17_attention_weight_min": -27.174718856811523,
      "activations/layer18_attention_weight_max": 34.53956985473633,
      "activations/layer18_attention_weight_min": -27.38874626159668,
      "activations/layer19_attention_weight_max": 38.01435852050781,
      "activations/layer19_attention_weight_min": -33.172977447509766,
      "activations/layer1_attention_weight_max": 16.33607292175293,
      "activations/layer1_attention_weight_min": -15.500789642333984,
      "activations/layer20_attention_weight_max": 31.058792114257812,
      "activations/layer20_attention_weight_min": -26.170137405395508,
      "activations/layer21_attention_weight_max": 31.3590087890625,
      "activations/layer21_attention_weight_min": -27.69331932067871,
      "activations/layer22_attention_weight_max": 42.546409606933594,
      "activations/layer22_attention_weight_min": -31.576433181762695,
      "activations/layer23_attention_weight_max": 34.90693664550781,
      "activations/layer23_attention_weight_min": -27.2163143157959,
      "activations/layer2_attention_weight_max": 33.968116760253906,
      "activations/layer2_attention_weight_min": -33.27073669433594,
      "activations/layer3_attention_weight_max": 96.6048812866211,
      "activations/layer3_attention_weight_min": -100.36985778808594,
      "activations/layer4_attention_weight_max": 101.1922607421875,
      "activations/layer4_attention_weight_min": -107.90665435791016,
      "activations/layer5_attention_weight_max": 49.47908020019531,
      "activations/layer5_attention_weight_min": -59.74890899658203,
      "activations/layer6_attention_weight_max": 44.00968933105469,
      "activations/layer6_attention_weight_min": -44.86300277709961,
      "activations/layer7_attention_weight_max": 87.12450408935547,
      "activations/layer7_attention_weight_min": -90.49962615966797,
      "activations/layer8_attention_weight_max": 39.8570556640625,
      "activations/layer8_attention_weight_min": -41.948974609375,
      "activations/layer9_attention_weight_max": 31.779752731323242,
      "activations/layer9_attention_weight_min": -34.3400764465332,
      "epoch": 15.8,
      "learning_rate": 4.86060606060606e-05,
      "loss": 2.7463,
      "step": 271900
    },
    {
      "activations/layer0_attention_weight_max": 15.573650360107422,
      "activations/layer0_attention_weight_min": -13.26135540008545,
      "activations/layer10_attention_weight_max": 30.4571533203125,
      "activations/layer10_attention_weight_min": -33.79793167114258,
      "activations/layer11_attention_weight_max": 31.207351684570312,
      "activations/layer11_attention_weight_min": -35.37002944946289,
      "activations/layer12_attention_weight_max": 25.284133911132812,
      "activations/layer12_attention_weight_min": -26.257522583007812,
      "activations/layer13_attention_weight_max": 38.63790512084961,
      "activations/layer13_attention_weight_min": -34.40401077270508,
      "activations/layer14_attention_weight_max": 42.13938903808594,
      "activations/layer14_attention_weight_min": -35.514892578125,
      "activations/layer15_attention_weight_max": 36.98752975463867,
      "activations/layer15_attention_weight_min": -33.605201721191406,
      "activations/layer16_attention_weight_max": 30.01112174987793,
      "activations/layer16_attention_weight_min": -30.378320693969727,
      "activations/layer17_attention_weight_max": 29.870866775512695,
      "activations/layer17_attention_weight_min": -26.035377502441406,
      "activations/layer18_attention_weight_max": 30.144756317138672,
      "activations/layer18_attention_weight_min": -23.838157653808594,
      "activations/layer19_attention_weight_max": 33.79922103881836,
      "activations/layer19_attention_weight_min": -31.768217086791992,
      "activations/layer1_attention_weight_max": 16.83926010131836,
      "activations/layer1_attention_weight_min": -13.91749382019043,
      "activations/layer20_attention_weight_max": 32.15855026245117,
      "activations/layer20_attention_weight_min": -29.12078094482422,
      "activations/layer21_attention_weight_max": 34.19016647338867,
      "activations/layer21_attention_weight_min": -28.125524520874023,
      "activations/layer22_attention_weight_max": 40.89134979248047,
      "activations/layer22_attention_weight_min": -32.35982894897461,
      "activations/layer23_attention_weight_max": 34.296329498291016,
      "activations/layer23_attention_weight_min": -26.421207427978516,
      "activations/layer2_attention_weight_max": 32.36574172973633,
      "activations/layer2_attention_weight_min": -30.78449821472168,
      "activations/layer3_attention_weight_max": 90.03001403808594,
      "activations/layer3_attention_weight_min": -90.21379852294922,
      "activations/layer4_attention_weight_max": 97.37987518310547,
      "activations/layer4_attention_weight_min": -99.89010620117188,
      "activations/layer5_attention_weight_max": 48.694061279296875,
      "activations/layer5_attention_weight_min": -58.17585754394531,
      "activations/layer6_attention_weight_max": 39.11913299560547,
      "activations/layer6_attention_weight_min": -42.91440963745117,
      "activations/layer7_attention_weight_max": 84.79273223876953,
      "activations/layer7_attention_weight_min": -91.74642944335938,
      "activations/layer8_attention_weight_max": 36.82750701904297,
      "activations/layer8_attention_weight_min": -40.842472076416016,
      "activations/layer9_attention_weight_max": 30.306076049804688,
      "activations/layer9_attention_weight_min": -30.99916648864746,
      "epoch": 15.8,
      "learning_rate": 4.858712121212121e-05,
      "loss": 2.7411,
      "step": 271950
    },
    {
      "activations/layer0_attention_weight_max": 16.140249252319336,
      "activations/layer0_attention_weight_min": -13.042216300964355,
      "activations/layer10_attention_weight_max": 31.20646858215332,
      "activations/layer10_attention_weight_min": -34.535518646240234,
      "activations/layer11_attention_weight_max": 33.08255386352539,
      "activations/layer11_attention_weight_min": -33.39404296875,
      "activations/layer12_attention_weight_max": 29.6671199798584,
      "activations/layer12_attention_weight_min": -27.146589279174805,
      "activations/layer13_attention_weight_max": 44.00474548339844,
      "activations/layer13_attention_weight_min": -38.46156311035156,
      "activations/layer14_attention_weight_max": 48.12948989868164,
      "activations/layer14_attention_weight_min": -39.83931350708008,
      "activations/layer15_attention_weight_max": 44.03404235839844,
      "activations/layer15_attention_weight_min": -35.9189338684082,
      "activations/layer16_attention_weight_max": 30.75611114501953,
      "activations/layer16_attention_weight_min": -28.375160217285156,
      "activations/layer17_attention_weight_max": 34.21302795410156,
      "activations/layer17_attention_weight_min": -27.312150955200195,
      "activations/layer18_attention_weight_max": 34.82118225097656,
      "activations/layer18_attention_weight_min": -25.109512329101562,
      "activations/layer19_attention_weight_max": 34.87102508544922,
      "activations/layer19_attention_weight_min": -32.63350296020508,
      "activations/layer1_attention_weight_max": 16.429241180419922,
      "activations/layer1_attention_weight_min": -14.899182319641113,
      "activations/layer20_attention_weight_max": 30.06161117553711,
      "activations/layer20_attention_weight_min": -27.32439613342285,
      "activations/layer21_attention_weight_max": 30.531036376953125,
      "activations/layer21_attention_weight_min": -25.52627944946289,
      "activations/layer22_attention_weight_max": 44.24129104614258,
      "activations/layer22_attention_weight_min": -33.24708557128906,
      "activations/layer23_attention_weight_max": 36.76094055175781,
      "activations/layer23_attention_weight_min": -26.347415924072266,
      "activations/layer2_attention_weight_max": 33.767662048339844,
      "activations/layer2_attention_weight_min": -31.009990692138672,
      "activations/layer3_attention_weight_max": 97.8863296508789,
      "activations/layer3_attention_weight_min": -94.30400085449219,
      "activations/layer4_attention_weight_max": 108.73500061035156,
      "activations/layer4_attention_weight_min": -108.64845275878906,
      "activations/layer5_attention_weight_max": 51.798133850097656,
      "activations/layer5_attention_weight_min": -65.38673400878906,
      "activations/layer6_attention_weight_max": 43.958805084228516,
      "activations/layer6_attention_weight_min": -45.783809661865234,
      "activations/layer7_attention_weight_max": 92.54405975341797,
      "activations/layer7_attention_weight_min": -89.83155822753906,
      "activations/layer8_attention_weight_max": 39.77913284301758,
      "activations/layer8_attention_weight_min": -42.12192153930664,
      "activations/layer9_attention_weight_max": 31.25067710876465,
      "activations/layer9_attention_weight_min": -32.547611236572266,
      "epoch": 15.8,
      "learning_rate": 4.8568181818181814e-05,
      "loss": 2.7664,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4871,
      "eval_samples_per_second": 505.943,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4871,
      "eval_openwebtext_samples_per_second": 505.943,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 2.0105,
      "eval_wikitext_samples_per_second": 226.807,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_lambada_loss": 2.384765625,
      "eval_lambada_ppl": 10.85651787033819,
      "eval_lambada_runtime": 9.556,
      "eval_lambada_samples_per_second": 509.521,
      "step": 272000
    },
    {
      "activations/layer0_attention_weight_max": 15.50363540649414,
      "activations/layer0_attention_weight_min": -13.462000846862793,
      "activations/layer10_attention_weight_max": 32.96983337402344,
      "activations/layer10_attention_weight_min": -34.94461441040039,
      "activations/layer11_attention_weight_max": 34.352752685546875,
      "activations/layer11_attention_weight_min": -36.519046783447266,
      "activations/layer12_attention_weight_max": 25.716110229492188,
      "activations/layer12_attention_weight_min": -27.46744728088379,
      "activations/layer13_attention_weight_max": 37.26457977294922,
      "activations/layer13_attention_weight_min": -36.77894592285156,
      "activations/layer14_attention_weight_max": 42.00014114379883,
      "activations/layer14_attention_weight_min": -39.648433685302734,
      "activations/layer15_attention_weight_max": 38.51700210571289,
      "activations/layer15_attention_weight_min": -36.342674255371094,
      "activations/layer16_attention_weight_max": 29.128633499145508,
      "activations/layer16_attention_weight_min": -30.42289161682129,
      "activations/layer17_attention_weight_max": 29.172895431518555,
      "activations/layer17_attention_weight_min": -30.84861946105957,
      "activations/layer18_attention_weight_max": 33.207481384277344,
      "activations/layer18_attention_weight_min": -29.171056747436523,
      "activations/layer19_attention_weight_max": 32.941932678222656,
      "activations/layer19_attention_weight_min": -33.51034927368164,
      "activations/layer1_attention_weight_max": 15.606270790100098,
      "activations/layer1_attention_weight_min": -15.268239974975586,
      "activations/layer20_attention_weight_max": 26.926258087158203,
      "activations/layer20_attention_weight_min": -24.800397872924805,
      "activations/layer21_attention_weight_max": 27.734975814819336,
      "activations/layer21_attention_weight_min": -24.323497772216797,
      "activations/layer22_attention_weight_max": 48.37893295288086,
      "activations/layer22_attention_weight_min": -34.25450897216797,
      "activations/layer23_attention_weight_max": 33.1580696105957,
      "activations/layer23_attention_weight_min": -25.243839263916016,
      "activations/layer2_attention_weight_max": 32.78937530517578,
      "activations/layer2_attention_weight_min": -35.578956604003906,
      "activations/layer3_attention_weight_max": 97.1534194946289,
      "activations/layer3_attention_weight_min": -104.6994400024414,
      "activations/layer4_attention_weight_max": 107.7029037475586,
      "activations/layer4_attention_weight_min": -113.7111587524414,
      "activations/layer5_attention_weight_max": 53.15790557861328,
      "activations/layer5_attention_weight_min": -63.078102111816406,
      "activations/layer6_attention_weight_max": 46.03995132446289,
      "activations/layer6_attention_weight_min": -47.306053161621094,
      "activations/layer7_attention_weight_max": 92.50823211669922,
      "activations/layer7_attention_weight_min": -100.15232849121094,
      "activations/layer8_attention_weight_max": 43.29069519042969,
      "activations/layer8_attention_weight_min": -47.09456253051758,
      "activations/layer9_attention_weight_max": 33.09777069091797,
      "activations/layer9_attention_weight_min": -36.191741943359375,
      "epoch": 15.81,
      "learning_rate": 4.8549242424242416e-05,
      "loss": 2.7569,
      "step": 272050
    },
    {
      "activations/layer0_attention_weight_max": 15.337479591369629,
      "activations/layer0_attention_weight_min": -12.76477336883545,
      "activations/layer10_attention_weight_max": 32.480506896972656,
      "activations/layer10_attention_weight_min": -36.83086013793945,
      "activations/layer11_attention_weight_max": 33.31158447265625,
      "activations/layer11_attention_weight_min": -34.98502731323242,
      "activations/layer12_attention_weight_max": 25.936689376831055,
      "activations/layer12_attention_weight_min": -25.999385833740234,
      "activations/layer13_attention_weight_max": 39.29216766357422,
      "activations/layer13_attention_weight_min": -35.187774658203125,
      "activations/layer14_attention_weight_max": 40.931880950927734,
      "activations/layer14_attention_weight_min": -38.383052825927734,
      "activations/layer15_attention_weight_max": 39.75711441040039,
      "activations/layer15_attention_weight_min": -34.866363525390625,
      "activations/layer16_attention_weight_max": 30.511714935302734,
      "activations/layer16_attention_weight_min": -29.357425689697266,
      "activations/layer17_attention_weight_max": 33.38671112060547,
      "activations/layer17_attention_weight_min": -28.054231643676758,
      "activations/layer18_attention_weight_max": 35.237850189208984,
      "activations/layer18_attention_weight_min": -25.277095794677734,
      "activations/layer19_attention_weight_max": 33.4578742980957,
      "activations/layer19_attention_weight_min": -33.63602828979492,
      "activations/layer1_attention_weight_max": 16.146818161010742,
      "activations/layer1_attention_weight_min": -14.695119857788086,
      "activations/layer20_attention_weight_max": 28.70619773864746,
      "activations/layer20_attention_weight_min": -27.281280517578125,
      "activations/layer21_attention_weight_max": 30.983882904052734,
      "activations/layer21_attention_weight_min": -25.817535400390625,
      "activations/layer22_attention_weight_max": 43.19137191772461,
      "activations/layer22_attention_weight_min": -32.80892562866211,
      "activations/layer23_attention_weight_max": 35.63521194458008,
      "activations/layer23_attention_weight_min": -26.527692794799805,
      "activations/layer2_attention_weight_max": 31.396785736083984,
      "activations/layer2_attention_weight_min": -31.096309661865234,
      "activations/layer3_attention_weight_max": 93.4460220336914,
      "activations/layer3_attention_weight_min": -94.28441619873047,
      "activations/layer4_attention_weight_max": 100.55675506591797,
      "activations/layer4_attention_weight_min": -107.58629608154297,
      "activations/layer5_attention_weight_max": 51.10512924194336,
      "activations/layer5_attention_weight_min": -64.41421508789062,
      "activations/layer6_attention_weight_max": 43.07353210449219,
      "activations/layer6_attention_weight_min": -48.69069290161133,
      "activations/layer7_attention_weight_max": 86.44368743896484,
      "activations/layer7_attention_weight_min": -93.7536392211914,
      "activations/layer8_attention_weight_max": 38.58186721801758,
      "activations/layer8_attention_weight_min": -42.48031997680664,
      "activations/layer9_attention_weight_max": 30.085113525390625,
      "activations/layer9_attention_weight_min": -33.312705993652344,
      "epoch": 15.81,
      "learning_rate": 4.853068181818181e-05,
      "loss": 2.7504,
      "step": 272100
    },
    {
      "activations/layer0_attention_weight_max": 15.439539909362793,
      "activations/layer0_attention_weight_min": -12.471427917480469,
      "activations/layer10_attention_weight_max": 31.072795867919922,
      "activations/layer10_attention_weight_min": -33.4356575012207,
      "activations/layer11_attention_weight_max": 30.88124656677246,
      "activations/layer11_attention_weight_min": -32.50498580932617,
      "activations/layer12_attention_weight_max": 25.45317268371582,
      "activations/layer12_attention_weight_min": -26.908287048339844,
      "activations/layer13_attention_weight_max": 39.32143020629883,
      "activations/layer13_attention_weight_min": -36.64222717285156,
      "activations/layer14_attention_weight_max": 42.42823028564453,
      "activations/layer14_attention_weight_min": -36.473365783691406,
      "activations/layer15_attention_weight_max": 35.73817443847656,
      "activations/layer15_attention_weight_min": -34.57957077026367,
      "activations/layer16_attention_weight_max": 29.919282913208008,
      "activations/layer16_attention_weight_min": -29.484254837036133,
      "activations/layer17_attention_weight_max": 28.47432518005371,
      "activations/layer17_attention_weight_min": -25.7018985748291,
      "activations/layer18_attention_weight_max": 31.037864685058594,
      "activations/layer18_attention_weight_min": -25.075862884521484,
      "activations/layer19_attention_weight_max": 33.439083099365234,
      "activations/layer19_attention_weight_min": -32.37928009033203,
      "activations/layer1_attention_weight_max": 16.262474060058594,
      "activations/layer1_attention_weight_min": -15.548380851745605,
      "activations/layer20_attention_weight_max": 25.762001037597656,
      "activations/layer20_attention_weight_min": -25.809385299682617,
      "activations/layer21_attention_weight_max": 25.71881675720215,
      "activations/layer21_attention_weight_min": -25.266515731811523,
      "activations/layer22_attention_weight_max": 41.521461486816406,
      "activations/layer22_attention_weight_min": -32.296688079833984,
      "activations/layer23_attention_weight_max": 31.164562225341797,
      "activations/layer23_attention_weight_min": -27.918338775634766,
      "activations/layer2_attention_weight_max": 32.60540008544922,
      "activations/layer2_attention_weight_min": -32.2595100402832,
      "activations/layer3_attention_weight_max": 93.83274841308594,
      "activations/layer3_attention_weight_min": -98.14534759521484,
      "activations/layer4_attention_weight_max": 103.23616790771484,
      "activations/layer4_attention_weight_min": -105.81060028076172,
      "activations/layer5_attention_weight_max": 52.71934509277344,
      "activations/layer5_attention_weight_min": -61.63489532470703,
      "activations/layer6_attention_weight_max": 43.560306549072266,
      "activations/layer6_attention_weight_min": -45.050113677978516,
      "activations/layer7_attention_weight_max": 86.98193359375,
      "activations/layer7_attention_weight_min": -91.23434448242188,
      "activations/layer8_attention_weight_max": 38.58430099487305,
      "activations/layer8_attention_weight_min": -41.96915817260742,
      "activations/layer9_attention_weight_max": 28.857816696166992,
      "activations/layer9_attention_weight_min": -32.04850387573242,
      "epoch": 15.81,
      "learning_rate": 4.851174242424242e-05,
      "loss": 2.7492,
      "step": 272150
    },
    {
      "activations/layer0_attention_weight_max": 15.371674537658691,
      "activations/layer0_attention_weight_min": -13.426922798156738,
      "activations/layer10_attention_weight_max": 33.56905746459961,
      "activations/layer10_attention_weight_min": -33.61708068847656,
      "activations/layer11_attention_weight_max": 33.446231842041016,
      "activations/layer11_attention_weight_min": -33.959510803222656,
      "activations/layer12_attention_weight_max": 26.338891983032227,
      "activations/layer12_attention_weight_min": -26.15274429321289,
      "activations/layer13_attention_weight_max": 40.806549072265625,
      "activations/layer13_attention_weight_min": -36.39282989501953,
      "activations/layer14_attention_weight_max": 40.92321014404297,
      "activations/layer14_attention_weight_min": -37.18411636352539,
      "activations/layer15_attention_weight_max": 36.149085998535156,
      "activations/layer15_attention_weight_min": -33.97920608520508,
      "activations/layer16_attention_weight_max": 31.027177810668945,
      "activations/layer16_attention_weight_min": -28.56835174560547,
      "activations/layer17_attention_weight_max": 31.134199142456055,
      "activations/layer17_attention_weight_min": -24.35471534729004,
      "activations/layer18_attention_weight_max": 32.350032806396484,
      "activations/layer18_attention_weight_min": -25.02911949157715,
      "activations/layer19_attention_weight_max": 36.434547424316406,
      "activations/layer19_attention_weight_min": -30.753665924072266,
      "activations/layer1_attention_weight_max": 17.030855178833008,
      "activations/layer1_attention_weight_min": -14.981353759765625,
      "activations/layer20_attention_weight_max": 29.298229217529297,
      "activations/layer20_attention_weight_min": -24.21958351135254,
      "activations/layer21_attention_weight_max": 29.44411277770996,
      "activations/layer21_attention_weight_min": -24.757055282592773,
      "activations/layer22_attention_weight_max": 42.369911193847656,
      "activations/layer22_attention_weight_min": -31.164262771606445,
      "activations/layer23_attention_weight_max": 37.04522705078125,
      "activations/layer23_attention_weight_min": -24.692602157592773,
      "activations/layer2_attention_weight_max": 33.38470458984375,
      "activations/layer2_attention_weight_min": -31.049314498901367,
      "activations/layer3_attention_weight_max": 94.09626007080078,
      "activations/layer3_attention_weight_min": -91.78461456298828,
      "activations/layer4_attention_weight_max": 105.06002044677734,
      "activations/layer4_attention_weight_min": -102.524169921875,
      "activations/layer5_attention_weight_max": 50.29108428955078,
      "activations/layer5_attention_weight_min": -61.05857849121094,
      "activations/layer6_attention_weight_max": 43.734336853027344,
      "activations/layer6_attention_weight_min": -44.900779724121094,
      "activations/layer7_attention_weight_max": 91.70848846435547,
      "activations/layer7_attention_weight_min": -94.4159927368164,
      "activations/layer8_attention_weight_max": 41.377769470214844,
      "activations/layer8_attention_weight_min": -42.59343338012695,
      "activations/layer9_attention_weight_max": 32.412330627441406,
      "activations/layer9_attention_weight_min": -34.24103927612305,
      "epoch": 15.82,
      "learning_rate": 4.849280303030302e-05,
      "loss": 2.7424,
      "step": 272200
    },
    {
      "activations/layer0_attention_weight_max": 16.03925132751465,
      "activations/layer0_attention_weight_min": -12.343416213989258,
      "activations/layer10_attention_weight_max": 34.96457290649414,
      "activations/layer10_attention_weight_min": -34.799198150634766,
      "activations/layer11_attention_weight_max": 36.03895950317383,
      "activations/layer11_attention_weight_min": -36.73408508300781,
      "activations/layer12_attention_weight_max": 26.19615364074707,
      "activations/layer12_attention_weight_min": -25.324628829956055,
      "activations/layer13_attention_weight_max": 41.93678665161133,
      "activations/layer13_attention_weight_min": -35.37270736694336,
      "activations/layer14_attention_weight_max": 41.423954010009766,
      "activations/layer14_attention_weight_min": -36.998653411865234,
      "activations/layer15_attention_weight_max": 37.11390686035156,
      "activations/layer15_attention_weight_min": -34.022056579589844,
      "activations/layer16_attention_weight_max": 28.02600860595703,
      "activations/layer16_attention_weight_min": -28.48457145690918,
      "activations/layer17_attention_weight_max": 31.50874900817871,
      "activations/layer17_attention_weight_min": -28.354644775390625,
      "activations/layer18_attention_weight_max": 29.443735122680664,
      "activations/layer18_attention_weight_min": -24.978130340576172,
      "activations/layer19_attention_weight_max": 34.094913482666016,
      "activations/layer19_attention_weight_min": -33.06643295288086,
      "activations/layer1_attention_weight_max": 16.373136520385742,
      "activations/layer1_attention_weight_min": -15.300148010253906,
      "activations/layer20_attention_weight_max": 28.93408203125,
      "activations/layer20_attention_weight_min": -25.233352661132812,
      "activations/layer21_attention_weight_max": 28.049407958984375,
      "activations/layer21_attention_weight_min": -25.075674057006836,
      "activations/layer22_attention_weight_max": 40.33097839355469,
      "activations/layer22_attention_weight_min": -30.21730613708496,
      "activations/layer23_attention_weight_max": 36.44829559326172,
      "activations/layer23_attention_weight_min": -25.13532257080078,
      "activations/layer2_attention_weight_max": 33.33546829223633,
      "activations/layer2_attention_weight_min": -31.509994506835938,
      "activations/layer3_attention_weight_max": 97.21047973632812,
      "activations/layer3_attention_weight_min": -101.62494659423828,
      "activations/layer4_attention_weight_max": 108.2598648071289,
      "activations/layer4_attention_weight_min": -109.69416809082031,
      "activations/layer5_attention_weight_max": 51.61235809326172,
      "activations/layer5_attention_weight_min": -59.371971130371094,
      "activations/layer6_attention_weight_max": 44.77916717529297,
      "activations/layer6_attention_weight_min": -45.75205612182617,
      "activations/layer7_attention_weight_max": 92.76256561279297,
      "activations/layer7_attention_weight_min": -95.51399230957031,
      "activations/layer8_attention_weight_max": 42.59175109863281,
      "activations/layer8_attention_weight_min": -43.35345458984375,
      "activations/layer9_attention_weight_max": 33.491878509521484,
      "activations/layer9_attention_weight_min": -34.70559310913086,
      "epoch": 15.82,
      "learning_rate": 4.847386363636363e-05,
      "loss": 2.7546,
      "step": 272250
    },
    {
      "activations/layer0_attention_weight_max": 15.901548385620117,
      "activations/layer0_attention_weight_min": -12.240943908691406,
      "activations/layer10_attention_weight_max": 37.34838104248047,
      "activations/layer10_attention_weight_min": -36.252174377441406,
      "activations/layer11_attention_weight_max": 35.65673065185547,
      "activations/layer11_attention_weight_min": -34.41327667236328,
      "activations/layer12_attention_weight_max": 24.045740127563477,
      "activations/layer12_attention_weight_min": -26.656038284301758,
      "activations/layer13_attention_weight_max": 37.30104064941406,
      "activations/layer13_attention_weight_min": -36.64826202392578,
      "activations/layer14_attention_weight_max": 41.68994140625,
      "activations/layer14_attention_weight_min": -37.61774826049805,
      "activations/layer15_attention_weight_max": 37.21381759643555,
      "activations/layer15_attention_weight_min": -35.14912033081055,
      "activations/layer16_attention_weight_max": 26.76083755493164,
      "activations/layer16_attention_weight_min": -26.887889862060547,
      "activations/layer17_attention_weight_max": 32.0365104675293,
      "activations/layer17_attention_weight_min": -26.368234634399414,
      "activations/layer18_attention_weight_max": 31.348222732543945,
      "activations/layer18_attention_weight_min": -24.25973892211914,
      "activations/layer19_attention_weight_max": 34.14440155029297,
      "activations/layer19_attention_weight_min": -30.99300193786621,
      "activations/layer1_attention_weight_max": 16.66223907470703,
      "activations/layer1_attention_weight_min": -15.698118209838867,
      "activations/layer20_attention_weight_max": 28.7767333984375,
      "activations/layer20_attention_weight_min": -25.104345321655273,
      "activations/layer21_attention_weight_max": 26.57112693786621,
      "activations/layer21_attention_weight_min": -23.963647842407227,
      "activations/layer22_attention_weight_max": 41.510536193847656,
      "activations/layer22_attention_weight_min": -30.14347267150879,
      "activations/layer23_attention_weight_max": 31.58477783203125,
      "activations/layer23_attention_weight_min": -26.23395347595215,
      "activations/layer2_attention_weight_max": 31.872093200683594,
      "activations/layer2_attention_weight_min": -32.85970687866211,
      "activations/layer3_attention_weight_max": 93.50985717773438,
      "activations/layer3_attention_weight_min": -98.62776947021484,
      "activations/layer4_attention_weight_max": 103.18053436279297,
      "activations/layer4_attention_weight_min": -110.12129211425781,
      "activations/layer5_attention_weight_max": 50.746063232421875,
      "activations/layer5_attention_weight_min": -63.934688568115234,
      "activations/layer6_attention_weight_max": 43.09354019165039,
      "activations/layer6_attention_weight_min": -47.97818374633789,
      "activations/layer7_attention_weight_max": 90.95584106445312,
      "activations/layer7_attention_weight_min": -94.62712860107422,
      "activations/layer8_attention_weight_max": 42.41943359375,
      "activations/layer8_attention_weight_min": -45.009765625,
      "activations/layer9_attention_weight_max": 33.10380172729492,
      "activations/layer9_attention_weight_min": -37.61918258666992,
      "epoch": 15.82,
      "learning_rate": 4.845492424242424e-05,
      "loss": 2.7696,
      "step": 272300
    },
    {
      "activations/layer0_attention_weight_max": 15.962944984436035,
      "activations/layer0_attention_weight_min": -12.910407066345215,
      "activations/layer10_attention_weight_max": 32.73048782348633,
      "activations/layer10_attention_weight_min": -36.09036636352539,
      "activations/layer11_attention_weight_max": 32.440185546875,
      "activations/layer11_attention_weight_min": -34.153160095214844,
      "activations/layer12_attention_weight_max": 23.67698860168457,
      "activations/layer12_attention_weight_min": -28.46713638305664,
      "activations/layer13_attention_weight_max": 36.033973693847656,
      "activations/layer13_attention_weight_min": -33.1135139465332,
      "activations/layer14_attention_weight_max": 39.24074935913086,
      "activations/layer14_attention_weight_min": -35.88138198852539,
      "activations/layer15_attention_weight_max": 35.58586502075195,
      "activations/layer15_attention_weight_min": -33.260433197021484,
      "activations/layer16_attention_weight_max": 30.44527244567871,
      "activations/layer16_attention_weight_min": -28.88069725036621,
      "activations/layer17_attention_weight_max": 31.581804275512695,
      "activations/layer17_attention_weight_min": -25.701993942260742,
      "activations/layer18_attention_weight_max": 30.275869369506836,
      "activations/layer18_attention_weight_min": -27.04954719543457,
      "activations/layer19_attention_weight_max": 35.66890335083008,
      "activations/layer19_attention_weight_min": -33.20479202270508,
      "activations/layer1_attention_weight_max": 15.993654251098633,
      "activations/layer1_attention_weight_min": -15.572205543518066,
      "activations/layer20_attention_weight_max": 30.739240646362305,
      "activations/layer20_attention_weight_min": -26.946001052856445,
      "activations/layer21_attention_weight_max": 30.0898494720459,
      "activations/layer21_attention_weight_min": -24.835866928100586,
      "activations/layer22_attention_weight_max": 49.22270965576172,
      "activations/layer22_attention_weight_min": -32.866600036621094,
      "activations/layer23_attention_weight_max": 33.0198974609375,
      "activations/layer23_attention_weight_min": -26.718711853027344,
      "activations/layer2_attention_weight_max": 33.27591323852539,
      "activations/layer2_attention_weight_min": -33.700897216796875,
      "activations/layer3_attention_weight_max": 96.99280548095703,
      "activations/layer3_attention_weight_min": -100.97074127197266,
      "activations/layer4_attention_weight_max": 108.7019271850586,
      "activations/layer4_attention_weight_min": -113.95216369628906,
      "activations/layer5_attention_weight_max": 53.25715255737305,
      "activations/layer5_attention_weight_min": -62.62797164916992,
      "activations/layer6_attention_weight_max": 43.97658157348633,
      "activations/layer6_attention_weight_min": -47.2471809387207,
      "activations/layer7_attention_weight_max": 91.2801284790039,
      "activations/layer7_attention_weight_min": -99.40290069580078,
      "activations/layer8_attention_weight_max": 42.468299865722656,
      "activations/layer8_attention_weight_min": -45.09420394897461,
      "activations/layer9_attention_weight_max": 31.343992233276367,
      "activations/layer9_attention_weight_min": -34.28231430053711,
      "epoch": 15.83,
      "learning_rate": 4.843598484848485e-05,
      "loss": 2.7449,
      "step": 272350
    },
    {
      "activations/layer0_attention_weight_max": 15.93557357788086,
      "activations/layer0_attention_weight_min": -12.16645622253418,
      "activations/layer10_attention_weight_max": 31.4787654876709,
      "activations/layer10_attention_weight_min": -34.93858337402344,
      "activations/layer11_attention_weight_max": 33.278892517089844,
      "activations/layer11_attention_weight_min": -34.608890533447266,
      "activations/layer12_attention_weight_max": 28.673099517822266,
      "activations/layer12_attention_weight_min": -26.085832595825195,
      "activations/layer13_attention_weight_max": 40.522911071777344,
      "activations/layer13_attention_weight_min": -37.163597106933594,
      "activations/layer14_attention_weight_max": 44.03001022338867,
      "activations/layer14_attention_weight_min": -38.50692367553711,
      "activations/layer15_attention_weight_max": 40.44875717163086,
      "activations/layer15_attention_weight_min": -34.21996307373047,
      "activations/layer16_attention_weight_max": 31.750459671020508,
      "activations/layer16_attention_weight_min": -29.58598518371582,
      "activations/layer17_attention_weight_max": 32.43011474609375,
      "activations/layer17_attention_weight_min": -25.363855361938477,
      "activations/layer18_attention_weight_max": 32.05550765991211,
      "activations/layer18_attention_weight_min": -27.202245712280273,
      "activations/layer19_attention_weight_max": 33.97383499145508,
      "activations/layer19_attention_weight_min": -31.88886833190918,
      "activations/layer1_attention_weight_max": 16.858705520629883,
      "activations/layer1_attention_weight_min": -16.635488510131836,
      "activations/layer20_attention_weight_max": 29.82933807373047,
      "activations/layer20_attention_weight_min": -27.671348571777344,
      "activations/layer21_attention_weight_max": 29.026626586914062,
      "activations/layer21_attention_weight_min": -25.992624282836914,
      "activations/layer22_attention_weight_max": 40.15471649169922,
      "activations/layer22_attention_weight_min": -30.91083335876465,
      "activations/layer23_attention_weight_max": 34.55351638793945,
      "activations/layer23_attention_weight_min": -28.240720748901367,
      "activations/layer2_attention_weight_max": 33.97245788574219,
      "activations/layer2_attention_weight_min": -35.27336883544922,
      "activations/layer3_attention_weight_max": 101.8405990600586,
      "activations/layer3_attention_weight_min": -104.26966094970703,
      "activations/layer4_attention_weight_max": 110.39583587646484,
      "activations/layer4_attention_weight_min": -115.3278579711914,
      "activations/layer5_attention_weight_max": 52.88597869873047,
      "activations/layer5_attention_weight_min": -67.20783996582031,
      "activations/layer6_attention_weight_max": 43.658851623535156,
      "activations/layer6_attention_weight_min": -47.63217544555664,
      "activations/layer7_attention_weight_max": 90.35932159423828,
      "activations/layer7_attention_weight_min": -97.75800323486328,
      "activations/layer8_attention_weight_max": 41.13369369506836,
      "activations/layer8_attention_weight_min": -44.55504608154297,
      "activations/layer9_attention_weight_max": 32.75251388549805,
      "activations/layer9_attention_weight_min": -35.197566986083984,
      "epoch": 15.83,
      "learning_rate": 4.841704545454545e-05,
      "loss": 2.76,
      "step": 272400
    },
    {
      "activations/layer0_attention_weight_max": 15.403230667114258,
      "activations/layer0_attention_weight_min": -12.755464553833008,
      "activations/layer10_attention_weight_max": 30.975210189819336,
      "activations/layer10_attention_weight_min": -32.59798812866211,
      "activations/layer11_attention_weight_max": 32.02971649169922,
      "activations/layer11_attention_weight_min": -31.81147003173828,
      "activations/layer12_attention_weight_max": 29.15992546081543,
      "activations/layer12_attention_weight_min": -27.767688751220703,
      "activations/layer13_attention_weight_max": 38.69903564453125,
      "activations/layer13_attention_weight_min": -37.22478485107422,
      "activations/layer14_attention_weight_max": 46.73145294189453,
      "activations/layer14_attention_weight_min": -44.72065734863281,
      "activations/layer15_attention_weight_max": 36.91521453857422,
      "activations/layer15_attention_weight_min": -37.61283493041992,
      "activations/layer16_attention_weight_max": 31.801834106445312,
      "activations/layer16_attention_weight_min": -27.635881423950195,
      "activations/layer17_attention_weight_max": 30.20295524597168,
      "activations/layer17_attention_weight_min": -26.280235290527344,
      "activations/layer18_attention_weight_max": 30.554180145263672,
      "activations/layer18_attention_weight_min": -24.587993621826172,
      "activations/layer19_attention_weight_max": 38.18242645263672,
      "activations/layer19_attention_weight_min": -33.80845260620117,
      "activations/layer1_attention_weight_max": 16.834732055664062,
      "activations/layer1_attention_weight_min": -16.50891876220703,
      "activations/layer20_attention_weight_max": 30.839521408081055,
      "activations/layer20_attention_weight_min": -25.614904403686523,
      "activations/layer21_attention_weight_max": 30.881460189819336,
      "activations/layer21_attention_weight_min": -26.112560272216797,
      "activations/layer22_attention_weight_max": 39.870628356933594,
      "activations/layer22_attention_weight_min": -30.24850082397461,
      "activations/layer23_attention_weight_max": 36.91117858886719,
      "activations/layer23_attention_weight_min": -25.33087921142578,
      "activations/layer2_attention_weight_max": 33.61598587036133,
      "activations/layer2_attention_weight_min": -32.448753356933594,
      "activations/layer3_attention_weight_max": 95.24034881591797,
      "activations/layer3_attention_weight_min": -94.64382934570312,
      "activations/layer4_attention_weight_max": 105.19850158691406,
      "activations/layer4_attention_weight_min": -105.28743743896484,
      "activations/layer5_attention_weight_max": 49.649139404296875,
      "activations/layer5_attention_weight_min": -60.516353607177734,
      "activations/layer6_attention_weight_max": 41.68722152709961,
      "activations/layer6_attention_weight_min": -44.431678771972656,
      "activations/layer7_attention_weight_max": 88.34626007080078,
      "activations/layer7_attention_weight_min": -92.01123046875,
      "activations/layer8_attention_weight_max": 37.64266586303711,
      "activations/layer8_attention_weight_min": -40.79853057861328,
      "activations/layer9_attention_weight_max": 29.204425811767578,
      "activations/layer9_attention_weight_min": -33.55239486694336,
      "epoch": 15.83,
      "learning_rate": 4.839810606060605e-05,
      "loss": 2.7461,
      "step": 272450
    },
    {
      "activations/layer0_attention_weight_max": 15.558106422424316,
      "activations/layer0_attention_weight_min": -12.278816223144531,
      "activations/layer10_attention_weight_max": 32.0578498840332,
      "activations/layer10_attention_weight_min": -34.72587203979492,
      "activations/layer11_attention_weight_max": 31.773761749267578,
      "activations/layer11_attention_weight_min": -33.46104431152344,
      "activations/layer12_attention_weight_max": 27.891382217407227,
      "activations/layer12_attention_weight_min": -28.340232849121094,
      "activations/layer13_attention_weight_max": 42.97200012207031,
      "activations/layer13_attention_weight_min": -36.801063537597656,
      "activations/layer14_attention_weight_max": 42.516300201416016,
      "activations/layer14_attention_weight_min": -39.41271209716797,
      "activations/layer15_attention_weight_max": 38.07179260253906,
      "activations/layer15_attention_weight_min": -35.07974624633789,
      "activations/layer16_attention_weight_max": 30.084646224975586,
      "activations/layer16_attention_weight_min": -28.50222396850586,
      "activations/layer17_attention_weight_max": 29.806177139282227,
      "activations/layer17_attention_weight_min": -25.419496536254883,
      "activations/layer18_attention_weight_max": 32.93944549560547,
      "activations/layer18_attention_weight_min": -24.553272247314453,
      "activations/layer19_attention_weight_max": 35.827945709228516,
      "activations/layer19_attention_weight_min": -31.979724884033203,
      "activations/layer1_attention_weight_max": 16.354721069335938,
      "activations/layer1_attention_weight_min": -14.790839195251465,
      "activations/layer20_attention_weight_max": 26.037208557128906,
      "activations/layer20_attention_weight_min": -25.39278221130371,
      "activations/layer21_attention_weight_max": 28.01919937133789,
      "activations/layer21_attention_weight_min": -24.721940994262695,
      "activations/layer22_attention_weight_max": 38.99358367919922,
      "activations/layer22_attention_weight_min": -30.888805389404297,
      "activations/layer23_attention_weight_max": 34.50544357299805,
      "activations/layer23_attention_weight_min": -26.702747344970703,
      "activations/layer2_attention_weight_max": 33.40857696533203,
      "activations/layer2_attention_weight_min": -32.69516372680664,
      "activations/layer3_attention_weight_max": 95.64913177490234,
      "activations/layer3_attention_weight_min": -98.76016235351562,
      "activations/layer4_attention_weight_max": 104.6893081665039,
      "activations/layer4_attention_weight_min": -107.7429428100586,
      "activations/layer5_attention_weight_max": 51.23710632324219,
      "activations/layer5_attention_weight_min": -64.3023452758789,
      "activations/layer6_attention_weight_max": 43.585636138916016,
      "activations/layer6_attention_weight_min": -47.3283805847168,
      "activations/layer7_attention_weight_max": 87.7668685913086,
      "activations/layer7_attention_weight_min": -96.76886749267578,
      "activations/layer8_attention_weight_max": 42.43211364746094,
      "activations/layer8_attention_weight_min": -42.829856872558594,
      "activations/layer9_attention_weight_max": 32.114013671875,
      "activations/layer9_attention_weight_min": -34.61323165893555,
      "epoch": 15.83,
      "learning_rate": 4.8379166666666666e-05,
      "loss": 2.7523,
      "step": 272500
    },
    {
      "activations/layer0_attention_weight_max": 15.845840454101562,
      "activations/layer0_attention_weight_min": -14.02609920501709,
      "activations/layer10_attention_weight_max": 33.081668853759766,
      "activations/layer10_attention_weight_min": -33.712100982666016,
      "activations/layer11_attention_weight_max": 32.79437255859375,
      "activations/layer11_attention_weight_min": -33.76885986328125,
      "activations/layer12_attention_weight_max": 24.426212310791016,
      "activations/layer12_attention_weight_min": -24.839263916015625,
      "activations/layer13_attention_weight_max": 35.47852325439453,
      "activations/layer13_attention_weight_min": -36.440364837646484,
      "activations/layer14_attention_weight_max": 39.703895568847656,
      "activations/layer14_attention_weight_min": -40.01856231689453,
      "activations/layer15_attention_weight_max": 36.523712158203125,
      "activations/layer15_attention_weight_min": -34.44235610961914,
      "activations/layer16_attention_weight_max": 28.820392608642578,
      "activations/layer16_attention_weight_min": -29.53200912475586,
      "activations/layer17_attention_weight_max": 30.8917179107666,
      "activations/layer17_attention_weight_min": -26.71927833557129,
      "activations/layer18_attention_weight_max": 31.516925811767578,
      "activations/layer18_attention_weight_min": -25.20785140991211,
      "activations/layer19_attention_weight_max": 33.21507263183594,
      "activations/layer19_attention_weight_min": -31.93744468688965,
      "activations/layer1_attention_weight_max": 16.590978622436523,
      "activations/layer1_attention_weight_min": -15.059173583984375,
      "activations/layer20_attention_weight_max": 26.897674560546875,
      "activations/layer20_attention_weight_min": -26.727033615112305,
      "activations/layer21_attention_weight_max": 28.157817840576172,
      "activations/layer21_attention_weight_min": -25.36920166015625,
      "activations/layer22_attention_weight_max": 40.271324157714844,
      "activations/layer22_attention_weight_min": -31.884565353393555,
      "activations/layer23_attention_weight_max": 32.10731887817383,
      "activations/layer23_attention_weight_min": -27.549917221069336,
      "activations/layer2_attention_weight_max": 34.855018615722656,
      "activations/layer2_attention_weight_min": -34.688697814941406,
      "activations/layer3_attention_weight_max": 98.02033233642578,
      "activations/layer3_attention_weight_min": -101.1513900756836,
      "activations/layer4_attention_weight_max": 108.3482894897461,
      "activations/layer4_attention_weight_min": -107.0938720703125,
      "activations/layer5_attention_weight_max": 50.467750549316406,
      "activations/layer5_attention_weight_min": -57.64802551269531,
      "activations/layer6_attention_weight_max": 45.24403381347656,
      "activations/layer6_attention_weight_min": -46.24408721923828,
      "activations/layer7_attention_weight_max": 92.82456970214844,
      "activations/layer7_attention_weight_min": -89.95504760742188,
      "activations/layer8_attention_weight_max": 39.86684036254883,
      "activations/layer8_attention_weight_min": -40.95122146606445,
      "activations/layer9_attention_weight_max": 32.38912582397461,
      "activations/layer9_attention_weight_min": -33.187095642089844,
      "epoch": 15.84,
      "learning_rate": 4.836022727272727e-05,
      "loss": 2.7538,
      "step": 272550
    },
    {
      "activations/layer0_attention_weight_max": 15.761130332946777,
      "activations/layer0_attention_weight_min": -12.152033805847168,
      "activations/layer10_attention_weight_max": 32.6309814453125,
      "activations/layer10_attention_weight_min": -34.90130615234375,
      "activations/layer11_attention_weight_max": 32.3263053894043,
      "activations/layer11_attention_weight_min": -33.91928482055664,
      "activations/layer12_attention_weight_max": 26.329669952392578,
      "activations/layer12_attention_weight_min": -25.627948760986328,
      "activations/layer13_attention_weight_max": 38.98444366455078,
      "activations/layer13_attention_weight_min": -34.63641357421875,
      "activations/layer14_attention_weight_max": 41.37333297729492,
      "activations/layer14_attention_weight_min": -36.87208938598633,
      "activations/layer15_attention_weight_max": 37.68915557861328,
      "activations/layer15_attention_weight_min": -32.65964889526367,
      "activations/layer16_attention_weight_max": 29.479515075683594,
      "activations/layer16_attention_weight_min": -28.508073806762695,
      "activations/layer17_attention_weight_max": 29.88970375061035,
      "activations/layer17_attention_weight_min": -26.115495681762695,
      "activations/layer18_attention_weight_max": 30.928447723388672,
      "activations/layer18_attention_weight_min": -24.5634822845459,
      "activations/layer19_attention_weight_max": 37.022396087646484,
      "activations/layer19_attention_weight_min": -32.33028793334961,
      "activations/layer1_attention_weight_max": 18.575510025024414,
      "activations/layer1_attention_weight_min": -15.440082550048828,
      "activations/layer20_attention_weight_max": 31.01051902770996,
      "activations/layer20_attention_weight_min": -26.939302444458008,
      "activations/layer21_attention_weight_max": 28.564769744873047,
      "activations/layer21_attention_weight_min": -26.906600952148438,
      "activations/layer22_attention_weight_max": 45.755123138427734,
      "activations/layer22_attention_weight_min": -30.34012794494629,
      "activations/layer23_attention_weight_max": 37.539955139160156,
      "activations/layer23_attention_weight_min": -23.536880493164062,
      "activations/layer2_attention_weight_max": 33.30327224731445,
      "activations/layer2_attention_weight_min": -33.44541931152344,
      "activations/layer3_attention_weight_max": 97.11080169677734,
      "activations/layer3_attention_weight_min": -100.39366912841797,
      "activations/layer4_attention_weight_max": 112.7537612915039,
      "activations/layer4_attention_weight_min": -114.62938690185547,
      "activations/layer5_attention_weight_max": 52.24109649658203,
      "activations/layer5_attention_weight_min": -62.91726303100586,
      "activations/layer6_attention_weight_max": 44.00295639038086,
      "activations/layer6_attention_weight_min": -48.78826141357422,
      "activations/layer7_attention_weight_max": 89.41109466552734,
      "activations/layer7_attention_weight_min": -101.41364288330078,
      "activations/layer8_attention_weight_max": 42.19687271118164,
      "activations/layer8_attention_weight_min": -43.30881881713867,
      "activations/layer9_attention_weight_max": 31.264392852783203,
      "activations/layer9_attention_weight_min": -34.32963180541992,
      "epoch": 15.84,
      "learning_rate": 4.8341287878787876e-05,
      "loss": 2.7373,
      "step": 272600
    },
    {
      "activations/layer0_attention_weight_max": 15.805737495422363,
      "activations/layer0_attention_weight_min": -12.481460571289062,
      "activations/layer10_attention_weight_max": 46.47304153442383,
      "activations/layer10_attention_weight_min": -42.29827880859375,
      "activations/layer11_attention_weight_max": 44.798500061035156,
      "activations/layer11_attention_weight_min": -43.66947555541992,
      "activations/layer12_attention_weight_max": 47.73891067504883,
      "activations/layer12_attention_weight_min": -31.162137985229492,
      "activations/layer13_attention_weight_max": 61.576847076416016,
      "activations/layer13_attention_weight_min": -43.363948822021484,
      "activations/layer14_attention_weight_max": 59.54628372192383,
      "activations/layer14_attention_weight_min": -48.257164001464844,
      "activations/layer15_attention_weight_max": 61.44182205200195,
      "activations/layer15_attention_weight_min": -49.982669830322266,
      "activations/layer16_attention_weight_max": 37.07373809814453,
      "activations/layer16_attention_weight_min": -30.823013305664062,
      "activations/layer17_attention_weight_max": 32.919010162353516,
      "activations/layer17_attention_weight_min": -33.483699798583984,
      "activations/layer18_attention_weight_max": 30.619060516357422,
      "activations/layer18_attention_weight_min": -30.489681243896484,
      "activations/layer19_attention_weight_max": 33.54595947265625,
      "activations/layer19_attention_weight_min": -32.627662658691406,
      "activations/layer1_attention_weight_max": 17.548664093017578,
      "activations/layer1_attention_weight_min": -15.248189926147461,
      "activations/layer20_attention_weight_max": 29.53264808654785,
      "activations/layer20_attention_weight_min": -23.98408317565918,
      "activations/layer21_attention_weight_max": 28.093685150146484,
      "activations/layer21_attention_weight_min": -25.0518856048584,
      "activations/layer22_attention_weight_max": 46.58208084106445,
      "activations/layer22_attention_weight_min": -35.19097137451172,
      "activations/layer23_attention_weight_max": 35.02350616455078,
      "activations/layer23_attention_weight_min": -26.44331932067871,
      "activations/layer2_attention_weight_max": 32.271095275878906,
      "activations/layer2_attention_weight_min": -32.21327209472656,
      "activations/layer3_attention_weight_max": 95.02666473388672,
      "activations/layer3_attention_weight_min": -95.4059066772461,
      "activations/layer4_attention_weight_max": 108.44170379638672,
      "activations/layer4_attention_weight_min": -106.8203125,
      "activations/layer5_attention_weight_max": 54.835426330566406,
      "activations/layer5_attention_weight_min": -59.077178955078125,
      "activations/layer6_attention_weight_max": 47.077980041503906,
      "activations/layer6_attention_weight_min": -45.17527389526367,
      "activations/layer7_attention_weight_max": 112.3556900024414,
      "activations/layer7_attention_weight_min": -94.50602722167969,
      "activations/layer8_attention_weight_max": 52.35991668701172,
      "activations/layer8_attention_weight_min": -46.99101638793945,
      "activations/layer9_attention_weight_max": 42.250885009765625,
      "activations/layer9_attention_weight_min": -41.05651092529297,
      "epoch": 15.84,
      "learning_rate": 4.832234848484848e-05,
      "loss": 2.7538,
      "step": 272650
    },
    {
      "activations/layer0_attention_weight_max": 15.233031272888184,
      "activations/layer0_attention_weight_min": -13.78893756866455,
      "activations/layer10_attention_weight_max": 32.3757209777832,
      "activations/layer10_attention_weight_min": -32.101654052734375,
      "activations/layer11_attention_weight_max": 33.11506652832031,
      "activations/layer11_attention_weight_min": -33.05693435668945,
      "activations/layer12_attention_weight_max": 26.50960922241211,
      "activations/layer12_attention_weight_min": -26.752199172973633,
      "activations/layer13_attention_weight_max": 38.96929931640625,
      "activations/layer13_attention_weight_min": -34.62666702270508,
      "activations/layer14_attention_weight_max": 41.36536407470703,
      "activations/layer14_attention_weight_min": -37.059593200683594,
      "activations/layer15_attention_weight_max": 37.25160217285156,
      "activations/layer15_attention_weight_min": -34.78771209716797,
      "activations/layer16_attention_weight_max": 28.760385513305664,
      "activations/layer16_attention_weight_min": -29.377315521240234,
      "activations/layer17_attention_weight_max": 30.38168716430664,
      "activations/layer17_attention_weight_min": -25.933658599853516,
      "activations/layer18_attention_weight_max": 30.836715698242188,
      "activations/layer18_attention_weight_min": -24.169618606567383,
      "activations/layer19_attention_weight_max": 35.51518630981445,
      "activations/layer19_attention_weight_min": -33.37495040893555,
      "activations/layer1_attention_weight_max": 17.12201690673828,
      "activations/layer1_attention_weight_min": -15.438507080078125,
      "activations/layer20_attention_weight_max": 29.522132873535156,
      "activations/layer20_attention_weight_min": -24.995994567871094,
      "activations/layer21_attention_weight_max": 29.646114349365234,
      "activations/layer21_attention_weight_min": -24.761259078979492,
      "activations/layer22_attention_weight_max": 41.67657470703125,
      "activations/layer22_attention_weight_min": -32.21367263793945,
      "activations/layer23_attention_weight_max": 32.45952224731445,
      "activations/layer23_attention_weight_min": -24.819961547851562,
      "activations/layer2_attention_weight_max": 31.50244140625,
      "activations/layer2_attention_weight_min": -32.06327438354492,
      "activations/layer3_attention_weight_max": 98.09861755371094,
      "activations/layer3_attention_weight_min": -97.14054870605469,
      "activations/layer4_attention_weight_max": 107.5702133178711,
      "activations/layer4_attention_weight_min": -109.40435791015625,
      "activations/layer5_attention_weight_max": 52.19563293457031,
      "activations/layer5_attention_weight_min": -63.216156005859375,
      "activations/layer6_attention_weight_max": 43.44679260253906,
      "activations/layer6_attention_weight_min": -47.62681198120117,
      "activations/layer7_attention_weight_max": 93.4124526977539,
      "activations/layer7_attention_weight_min": -100.26327514648438,
      "activations/layer8_attention_weight_max": 39.46715545654297,
      "activations/layer8_attention_weight_min": -42.49099349975586,
      "activations/layer9_attention_weight_max": 31.656982421875,
      "activations/layer9_attention_weight_min": -31.5797061920166,
      "epoch": 15.85,
      "learning_rate": 4.830340909090909e-05,
      "loss": 2.748,
      "step": 272700
    },
    {
      "activations/layer0_attention_weight_max": 15.890409469604492,
      "activations/layer0_attention_weight_min": -12.43213176727295,
      "activations/layer10_attention_weight_max": 32.82363510131836,
      "activations/layer10_attention_weight_min": -34.127601623535156,
      "activations/layer11_attention_weight_max": 32.005226135253906,
      "activations/layer11_attention_weight_min": -31.55682373046875,
      "activations/layer12_attention_weight_max": 28.481176376342773,
      "activations/layer12_attention_weight_min": -26.390098571777344,
      "activations/layer13_attention_weight_max": 44.09412384033203,
      "activations/layer13_attention_weight_min": -35.581478118896484,
      "activations/layer14_attention_weight_max": 42.25283432006836,
      "activations/layer14_attention_weight_min": -37.38942337036133,
      "activations/layer15_attention_weight_max": 42.20171356201172,
      "activations/layer15_attention_weight_min": -33.91072463989258,
      "activations/layer16_attention_weight_max": 28.627029418945312,
      "activations/layer16_attention_weight_min": -27.228689193725586,
      "activations/layer17_attention_weight_max": 28.84061622619629,
      "activations/layer17_attention_weight_min": -25.505985260009766,
      "activations/layer18_attention_weight_max": 31.931011199951172,
      "activations/layer18_attention_weight_min": -25.890941619873047,
      "activations/layer19_attention_weight_max": 37.9158821105957,
      "activations/layer19_attention_weight_min": -32.585693359375,
      "activations/layer1_attention_weight_max": 17.23145866394043,
      "activations/layer1_attention_weight_min": -16.527193069458008,
      "activations/layer20_attention_weight_max": 34.58914566040039,
      "activations/layer20_attention_weight_min": -25.39764976501465,
      "activations/layer21_attention_weight_max": 30.67300796508789,
      "activations/layer21_attention_weight_min": -25.612895965576172,
      "activations/layer22_attention_weight_max": 43.4589729309082,
      "activations/layer22_attention_weight_min": -30.73017692565918,
      "activations/layer23_attention_weight_max": 34.05512237548828,
      "activations/layer23_attention_weight_min": -24.989192962646484,
      "activations/layer2_attention_weight_max": 33.22307586669922,
      "activations/layer2_attention_weight_min": -33.48036193847656,
      "activations/layer3_attention_weight_max": 95.40657806396484,
      "activations/layer3_attention_weight_min": -97.58297729492188,
      "activations/layer4_attention_weight_max": 104.9214096069336,
      "activations/layer4_attention_weight_min": -105.08038330078125,
      "activations/layer5_attention_weight_max": 48.847740173339844,
      "activations/layer5_attention_weight_min": -63.88450622558594,
      "activations/layer6_attention_weight_max": 43.15547561645508,
      "activations/layer6_attention_weight_min": -44.4413948059082,
      "activations/layer7_attention_weight_max": 85.29962921142578,
      "activations/layer7_attention_weight_min": -94.09740447998047,
      "activations/layer8_attention_weight_max": 39.71149826049805,
      "activations/layer8_attention_weight_min": -41.591983795166016,
      "activations/layer9_attention_weight_max": 30.343297958374023,
      "activations/layer9_attention_weight_min": -33.85654067993164,
      "epoch": 15.85,
      "learning_rate": 4.8284469696969694e-05,
      "loss": 2.7336,
      "step": 272750
    },
    {
      "activations/layer0_attention_weight_max": 15.586592674255371,
      "activations/layer0_attention_weight_min": -12.331338882446289,
      "activations/layer10_attention_weight_max": 30.91927146911621,
      "activations/layer10_attention_weight_min": -33.793392181396484,
      "activations/layer11_attention_weight_max": 30.303646087646484,
      "activations/layer11_attention_weight_min": -33.80883026123047,
      "activations/layer12_attention_weight_max": 24.20391082763672,
      "activations/layer12_attention_weight_min": -26.264793395996094,
      "activations/layer13_attention_weight_max": 40.249080657958984,
      "activations/layer13_attention_weight_min": -36.53953552246094,
      "activations/layer14_attention_weight_max": 42.3909912109375,
      "activations/layer14_attention_weight_min": -37.3857536315918,
      "activations/layer15_attention_weight_max": 45.49461364746094,
      "activations/layer15_attention_weight_min": -34.71624755859375,
      "activations/layer16_attention_weight_max": 32.3631591796875,
      "activations/layer16_attention_weight_min": -27.00071907043457,
      "activations/layer17_attention_weight_max": 28.5001220703125,
      "activations/layer17_attention_weight_min": -27.657163619995117,
      "activations/layer18_attention_weight_max": 30.9628963470459,
      "activations/layer18_attention_weight_min": -27.101280212402344,
      "activations/layer19_attention_weight_max": 34.1843376159668,
      "activations/layer19_attention_weight_min": -30.553466796875,
      "activations/layer1_attention_weight_max": 18.960100173950195,
      "activations/layer1_attention_weight_min": -18.59360694885254,
      "activations/layer20_attention_weight_max": 29.928001403808594,
      "activations/layer20_attention_weight_min": -26.32265281677246,
      "activations/layer21_attention_weight_max": 28.74502182006836,
      "activations/layer21_attention_weight_min": -24.907365798950195,
      "activations/layer22_attention_weight_max": 40.59238815307617,
      "activations/layer22_attention_weight_min": -31.831825256347656,
      "activations/layer23_attention_weight_max": 30.390544891357422,
      "activations/layer23_attention_weight_min": -26.357894897460938,
      "activations/layer2_attention_weight_max": 33.87785720825195,
      "activations/layer2_attention_weight_min": -33.53510284423828,
      "activations/layer3_attention_weight_max": 96.82463073730469,
      "activations/layer3_attention_weight_min": -96.8145980834961,
      "activations/layer4_attention_weight_max": 102.00840759277344,
      "activations/layer4_attention_weight_min": -104.0171127319336,
      "activations/layer5_attention_weight_max": 50.840389251708984,
      "activations/layer5_attention_weight_min": -60.39237594604492,
      "activations/layer6_attention_weight_max": 41.1188850402832,
      "activations/layer6_attention_weight_min": -45.21638107299805,
      "activations/layer7_attention_weight_max": 84.58846282958984,
      "activations/layer7_attention_weight_min": -90.6894760131836,
      "activations/layer8_attention_weight_max": 38.4658203125,
      "activations/layer8_attention_weight_min": -39.61326217651367,
      "activations/layer9_attention_weight_max": 31.511493682861328,
      "activations/layer9_attention_weight_min": -31.8586483001709,
      "epoch": 15.85,
      "learning_rate": 4.8265530303030295e-05,
      "loss": 2.7489,
      "step": 272800
    },
    {
      "activations/layer0_attention_weight_max": 16.242895126342773,
      "activations/layer0_attention_weight_min": -12.296491622924805,
      "activations/layer10_attention_weight_max": 31.0025691986084,
      "activations/layer10_attention_weight_min": -34.239501953125,
      "activations/layer11_attention_weight_max": 32.65530776977539,
      "activations/layer11_attention_weight_min": -34.76779556274414,
      "activations/layer12_attention_weight_max": 24.672666549682617,
      "activations/layer12_attention_weight_min": -26.09885025024414,
      "activations/layer13_attention_weight_max": 38.70687484741211,
      "activations/layer13_attention_weight_min": -34.44646072387695,
      "activations/layer14_attention_weight_max": 40.323883056640625,
      "activations/layer14_attention_weight_min": -36.34140396118164,
      "activations/layer15_attention_weight_max": 35.859798431396484,
      "activations/layer15_attention_weight_min": -32.447723388671875,
      "activations/layer16_attention_weight_max": 27.346149444580078,
      "activations/layer16_attention_weight_min": -27.13001251220703,
      "activations/layer17_attention_weight_max": 30.35856819152832,
      "activations/layer17_attention_weight_min": -24.77642822265625,
      "activations/layer18_attention_weight_max": 29.46554183959961,
      "activations/layer18_attention_weight_min": -23.483680725097656,
      "activations/layer19_attention_weight_max": 29.120258331298828,
      "activations/layer19_attention_weight_min": -29.27374267578125,
      "activations/layer1_attention_weight_max": 15.439112663269043,
      "activations/layer1_attention_weight_min": -13.918301582336426,
      "activations/layer20_attention_weight_max": 26.888898849487305,
      "activations/layer20_attention_weight_min": -23.36229133605957,
      "activations/layer21_attention_weight_max": 26.141965866088867,
      "activations/layer21_attention_weight_min": -23.375391006469727,
      "activations/layer22_attention_weight_max": 37.88689041137695,
      "activations/layer22_attention_weight_min": -29.939451217651367,
      "activations/layer23_attention_weight_max": 31.846519470214844,
      "activations/layer23_attention_weight_min": -24.01547622680664,
      "activations/layer2_attention_weight_max": 29.506824493408203,
      "activations/layer2_attention_weight_min": -31.03376007080078,
      "activations/layer3_attention_weight_max": 90.35485076904297,
      "activations/layer3_attention_weight_min": -93.47693634033203,
      "activations/layer4_attention_weight_max": 102.89813232421875,
      "activations/layer4_attention_weight_min": -107.77188873291016,
      "activations/layer5_attention_weight_max": 50.1084098815918,
      "activations/layer5_attention_weight_min": -62.991172790527344,
      "activations/layer6_attention_weight_max": 43.66957092285156,
      "activations/layer6_attention_weight_min": -46.46464157104492,
      "activations/layer7_attention_weight_max": 93.34044647216797,
      "activations/layer7_attention_weight_min": -94.66046142578125,
      "activations/layer8_attention_weight_max": 38.23302459716797,
      "activations/layer8_attention_weight_min": -42.244972229003906,
      "activations/layer9_attention_weight_max": 31.369464874267578,
      "activations/layer9_attention_weight_min": -32.99758529663086,
      "epoch": 15.85,
      "learning_rate": 4.82465909090909e-05,
      "loss": 2.7525,
      "step": 272850
    },
    {
      "activations/layer0_attention_weight_max": 15.322786331176758,
      "activations/layer0_attention_weight_min": -12.660240173339844,
      "activations/layer10_attention_weight_max": 32.90004348754883,
      "activations/layer10_attention_weight_min": -33.41156005859375,
      "activations/layer11_attention_weight_max": 34.043601989746094,
      "activations/layer11_attention_weight_min": -34.181907653808594,
      "activations/layer12_attention_weight_max": 26.414823532104492,
      "activations/layer12_attention_weight_min": -26.079517364501953,
      "activations/layer13_attention_weight_max": 39.839881896972656,
      "activations/layer13_attention_weight_min": -33.70368957519531,
      "activations/layer14_attention_weight_max": 41.209537506103516,
      "activations/layer14_attention_weight_min": -37.14965057373047,
      "activations/layer15_attention_weight_max": 36.15839385986328,
      "activations/layer15_attention_weight_min": -36.82707214355469,
      "activations/layer16_attention_weight_max": 29.35858154296875,
      "activations/layer16_attention_weight_min": -29.937538146972656,
      "activations/layer17_attention_weight_max": 31.94757843017578,
      "activations/layer17_attention_weight_min": -26.6658992767334,
      "activations/layer18_attention_weight_max": 33.855682373046875,
      "activations/layer18_attention_weight_min": -27.173969268798828,
      "activations/layer19_attention_weight_max": 33.28754425048828,
      "activations/layer19_attention_weight_min": -30.67856788635254,
      "activations/layer1_attention_weight_max": 15.789809226989746,
      "activations/layer1_attention_weight_min": -15.886114120483398,
      "activations/layer20_attention_weight_max": 28.656158447265625,
      "activations/layer20_attention_weight_min": -24.874460220336914,
      "activations/layer21_attention_weight_max": 30.649593353271484,
      "activations/layer21_attention_weight_min": -23.379539489746094,
      "activations/layer22_attention_weight_max": 44.03644561767578,
      "activations/layer22_attention_weight_min": -31.768856048583984,
      "activations/layer23_attention_weight_max": 37.3846435546875,
      "activations/layer23_attention_weight_min": -26.150686264038086,
      "activations/layer2_attention_weight_max": 32.836822509765625,
      "activations/layer2_attention_weight_min": -31.579551696777344,
      "activations/layer3_attention_weight_max": 97.27731323242188,
      "activations/layer3_attention_weight_min": -99.02906036376953,
      "activations/layer4_attention_weight_max": 104.98046112060547,
      "activations/layer4_attention_weight_min": -106.9803695678711,
      "activations/layer5_attention_weight_max": 49.797969818115234,
      "activations/layer5_attention_weight_min": -62.43642807006836,
      "activations/layer6_attention_weight_max": 45.09735870361328,
      "activations/layer6_attention_weight_min": -47.77415466308594,
      "activations/layer7_attention_weight_max": 88.10432434082031,
      "activations/layer7_attention_weight_min": -96.69125366210938,
      "activations/layer8_attention_weight_max": 39.609519958496094,
      "activations/layer8_attention_weight_min": -42.632354736328125,
      "activations/layer9_attention_weight_max": 31.551496505737305,
      "activations/layer9_attention_weight_min": -33.657470703125,
      "epoch": 15.86,
      "learning_rate": 4.822765151515151e-05,
      "loss": 2.7531,
      "step": 272900
    },
    {
      "activations/layer0_attention_weight_max": 16.77668571472168,
      "activations/layer0_attention_weight_min": -12.502001762390137,
      "activations/layer10_attention_weight_max": 34.59074020385742,
      "activations/layer10_attention_weight_min": -34.84627151489258,
      "activations/layer11_attention_weight_max": 37.28421401977539,
      "activations/layer11_attention_weight_min": -33.00627899169922,
      "activations/layer12_attention_weight_max": 29.034992218017578,
      "activations/layer12_attention_weight_min": -27.004905700683594,
      "activations/layer13_attention_weight_max": 44.06826400756836,
      "activations/layer13_attention_weight_min": -37.994319915771484,
      "activations/layer14_attention_weight_max": 46.477561950683594,
      "activations/layer14_attention_weight_min": -40.78929138183594,
      "activations/layer15_attention_weight_max": 40.850223541259766,
      "activations/layer15_attention_weight_min": -36.31574630737305,
      "activations/layer16_attention_weight_max": 31.530101776123047,
      "activations/layer16_attention_weight_min": -28.909116744995117,
      "activations/layer17_attention_weight_max": 31.64788246154785,
      "activations/layer17_attention_weight_min": -27.8554744720459,
      "activations/layer18_attention_weight_max": 32.21854019165039,
      "activations/layer18_attention_weight_min": -25.91046714782715,
      "activations/layer19_attention_weight_max": 37.18998336791992,
      "activations/layer19_attention_weight_min": -32.298336029052734,
      "activations/layer1_attention_weight_max": 15.712778091430664,
      "activations/layer1_attention_weight_min": -15.696820259094238,
      "activations/layer20_attention_weight_max": 31.121591567993164,
      "activations/layer20_attention_weight_min": -26.244304656982422,
      "activations/layer21_attention_weight_max": 33.10276412963867,
      "activations/layer21_attention_weight_min": -26.070709228515625,
      "activations/layer22_attention_weight_max": 44.71353530883789,
      "activations/layer22_attention_weight_min": -31.266315460205078,
      "activations/layer23_attention_weight_max": 36.85202407836914,
      "activations/layer23_attention_weight_min": -26.501569747924805,
      "activations/layer2_attention_weight_max": 32.86635971069336,
      "activations/layer2_attention_weight_min": -32.872928619384766,
      "activations/layer3_attention_weight_max": 99.25453186035156,
      "activations/layer3_attention_weight_min": -96.06339263916016,
      "activations/layer4_attention_weight_max": 110.2635498046875,
      "activations/layer4_attention_weight_min": -112.58404541015625,
      "activations/layer5_attention_weight_max": 51.17646408081055,
      "activations/layer5_attention_weight_min": -61.60065841674805,
      "activations/layer6_attention_weight_max": 44.485328674316406,
      "activations/layer6_attention_weight_min": -46.73359298706055,
      "activations/layer7_attention_weight_max": 95.289794921875,
      "activations/layer7_attention_weight_min": -100.49762725830078,
      "activations/layer8_attention_weight_max": 42.48952865600586,
      "activations/layer8_attention_weight_min": -44.245025634765625,
      "activations/layer9_attention_weight_max": 36.661537170410156,
      "activations/layer9_attention_weight_min": -34.277740478515625,
      "epoch": 15.86,
      "learning_rate": 4.820871212121212e-05,
      "loss": 2.7424,
      "step": 272950
    },
    {
      "activations/layer0_attention_weight_max": 15.981362342834473,
      "activations/layer0_attention_weight_min": -13.283941268920898,
      "activations/layer10_attention_weight_max": 31.46957778930664,
      "activations/layer10_attention_weight_min": -32.4958381652832,
      "activations/layer11_attention_weight_max": 30.52723503112793,
      "activations/layer11_attention_weight_min": -33.21470642089844,
      "activations/layer12_attention_weight_max": 25.739154815673828,
      "activations/layer12_attention_weight_min": -26.758804321289062,
      "activations/layer13_attention_weight_max": 38.24061965942383,
      "activations/layer13_attention_weight_min": -33.244598388671875,
      "activations/layer14_attention_weight_max": 40.95536804199219,
      "activations/layer14_attention_weight_min": -37.194976806640625,
      "activations/layer15_attention_weight_max": 35.68121337890625,
      "activations/layer15_attention_weight_min": -33.127838134765625,
      "activations/layer16_attention_weight_max": 31.307815551757812,
      "activations/layer16_attention_weight_min": -30.221302032470703,
      "activations/layer17_attention_weight_max": 31.641719818115234,
      "activations/layer17_attention_weight_min": -25.26760482788086,
      "activations/layer18_attention_weight_max": 32.11296844482422,
      "activations/layer18_attention_weight_min": -24.61867904663086,
      "activations/layer19_attention_weight_max": 36.37961959838867,
      "activations/layer19_attention_weight_min": -32.35721969604492,
      "activations/layer1_attention_weight_max": 16.19614028930664,
      "activations/layer1_attention_weight_min": -14.898813247680664,
      "activations/layer20_attention_weight_max": 30.059192657470703,
      "activations/layer20_attention_weight_min": -27.114479064941406,
      "activations/layer21_attention_weight_max": 31.267553329467773,
      "activations/layer21_attention_weight_min": -26.100051879882812,
      "activations/layer22_attention_weight_max": 42.51884460449219,
      "activations/layer22_attention_weight_min": -31.370494842529297,
      "activations/layer23_attention_weight_max": 36.41559982299805,
      "activations/layer23_attention_weight_min": -27.066612243652344,
      "activations/layer2_attention_weight_max": 34.454368591308594,
      "activations/layer2_attention_weight_min": -33.09196853637695,
      "activations/layer3_attention_weight_max": 95.41011047363281,
      "activations/layer3_attention_weight_min": -97.68209075927734,
      "activations/layer4_attention_weight_max": 107.00899505615234,
      "activations/layer4_attention_weight_min": -107.10877990722656,
      "activations/layer5_attention_weight_max": 51.31541442871094,
      "activations/layer5_attention_weight_min": -59.15871810913086,
      "activations/layer6_attention_weight_max": 43.161354064941406,
      "activations/layer6_attention_weight_min": -46.52284622192383,
      "activations/layer7_attention_weight_max": 86.6196060180664,
      "activations/layer7_attention_weight_min": -95.74577331542969,
      "activations/layer8_attention_weight_max": 39.31770706176758,
      "activations/layer8_attention_weight_min": -41.59343338012695,
      "activations/layer9_attention_weight_max": 31.18692398071289,
      "activations/layer9_attention_weight_min": -33.187156677246094,
      "epoch": 15.86,
      "learning_rate": 4.818977272727272e-05,
      "loss": 2.7446,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4515,
      "eval_samples_per_second": 508.073,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4515,
      "eval_openwebtext_samples_per_second": 508.073,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_wikitext_loss": 2.92578125,
      "eval_wikitext_ppl": 18.648789727196185,
      "eval_wikitext_runtime": 1.9614,
      "eval_wikitext_samples_per_second": 232.492,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_lambada_loss": 2.462890625,
      "eval_lambada_ppl": 11.738694709013188,
      "eval_lambada_runtime": 9.5514,
      "eval_lambada_samples_per_second": 509.77,
      "step": 273000
    },
    {
      "activations/layer0_attention_weight_max": 16.513145446777344,
      "activations/layer0_attention_weight_min": -12.551291465759277,
      "activations/layer10_attention_weight_max": 32.648704528808594,
      "activations/layer10_attention_weight_min": -31.67515754699707,
      "activations/layer11_attention_weight_max": 32.69355773925781,
      "activations/layer11_attention_weight_min": -31.53938102722168,
      "activations/layer12_attention_weight_max": 25.40667152404785,
      "activations/layer12_attention_weight_min": -25.10569190979004,
      "activations/layer13_attention_weight_max": 36.98073959350586,
      "activations/layer13_attention_weight_min": -35.3935661315918,
      "activations/layer14_attention_weight_max": 40.64904022216797,
      "activations/layer14_attention_weight_min": -37.22554397583008,
      "activations/layer15_attention_weight_max": 37.470577239990234,
      "activations/layer15_attention_weight_min": -34.025997161865234,
      "activations/layer16_attention_weight_max": 30.191654205322266,
      "activations/layer16_attention_weight_min": -27.897308349609375,
      "activations/layer17_attention_weight_max": 30.85833168029785,
      "activations/layer17_attention_weight_min": -25.270092010498047,
      "activations/layer18_attention_weight_max": 32.99015808105469,
      "activations/layer18_attention_weight_min": -25.722387313842773,
      "activations/layer19_attention_weight_max": 36.77529525756836,
      "activations/layer19_attention_weight_min": -31.139633178710938,
      "activations/layer1_attention_weight_max": 16.369049072265625,
      "activations/layer1_attention_weight_min": -14.804207801818848,
      "activations/layer20_attention_weight_max": 32.554439544677734,
      "activations/layer20_attention_weight_min": -26.263099670410156,
      "activations/layer21_attention_weight_max": 31.72423553466797,
      "activations/layer21_attention_weight_min": -26.0861759185791,
      "activations/layer22_attention_weight_max": 45.997432708740234,
      "activations/layer22_attention_weight_min": -30.903841018676758,
      "activations/layer23_attention_weight_max": 35.52724838256836,
      "activations/layer23_attention_weight_min": -24.802907943725586,
      "activations/layer2_attention_weight_max": 34.3072395324707,
      "activations/layer2_attention_weight_min": -33.02325439453125,
      "activations/layer3_attention_weight_max": 100.03557586669922,
      "activations/layer3_attention_weight_min": -98.07416534423828,
      "activations/layer4_attention_weight_max": 108.21734619140625,
      "activations/layer4_attention_weight_min": -106.78169250488281,
      "activations/layer5_attention_weight_max": 52.86662292480469,
      "activations/layer5_attention_weight_min": -61.76068115234375,
      "activations/layer6_attention_weight_max": 42.88594436645508,
      "activations/layer6_attention_weight_min": -44.4682502746582,
      "activations/layer7_attention_weight_max": 90.27774047851562,
      "activations/layer7_attention_weight_min": -92.2371597290039,
      "activations/layer8_attention_weight_max": 40.845619201660156,
      "activations/layer8_attention_weight_min": -41.17379379272461,
      "activations/layer9_attention_weight_max": 33.533836364746094,
      "activations/layer9_attention_weight_min": -32.817405700683594,
      "epoch": 15.87,
      "learning_rate": 4.817083333333332e-05,
      "loss": 2.7586,
      "step": 273050
    },
    {
      "activations/layer0_attention_weight_max": 15.518403053283691,
      "activations/layer0_attention_weight_min": -12.858772277832031,
      "activations/layer10_attention_weight_max": 34.77348709106445,
      "activations/layer10_attention_weight_min": -34.29967498779297,
      "activations/layer11_attention_weight_max": 33.414146423339844,
      "activations/layer11_attention_weight_min": -32.43608474731445,
      "activations/layer12_attention_weight_max": 28.546777725219727,
      "activations/layer12_attention_weight_min": -25.540855407714844,
      "activations/layer13_attention_weight_max": 44.29644012451172,
      "activations/layer13_attention_weight_min": -35.519081115722656,
      "activations/layer14_attention_weight_max": 45.36565017700195,
      "activations/layer14_attention_weight_min": -38.57823181152344,
      "activations/layer15_attention_weight_max": 42.746700286865234,
      "activations/layer15_attention_weight_min": -35.36320877075195,
      "activations/layer16_attention_weight_max": 33.17694854736328,
      "activations/layer16_attention_weight_min": -29.148208618164062,
      "activations/layer17_attention_weight_max": 33.094974517822266,
      "activations/layer17_attention_weight_min": -28.579511642456055,
      "activations/layer18_attention_weight_max": 36.92490768432617,
      "activations/layer18_attention_weight_min": -27.7257137298584,
      "activations/layer19_attention_weight_max": 38.1427116394043,
      "activations/layer19_attention_weight_min": -35.06522750854492,
      "activations/layer1_attention_weight_max": 16.86605453491211,
      "activations/layer1_attention_weight_min": -17.753185272216797,
      "activations/layer20_attention_weight_max": 33.24820327758789,
      "activations/layer20_attention_weight_min": -28.324995040893555,
      "activations/layer21_attention_weight_max": 33.30498504638672,
      "activations/layer21_attention_weight_min": -27.328977584838867,
      "activations/layer22_attention_weight_max": 46.32353591918945,
      "activations/layer22_attention_weight_min": -37.55061340332031,
      "activations/layer23_attention_weight_max": 36.028785705566406,
      "activations/layer23_attention_weight_min": -27.935565948486328,
      "activations/layer2_attention_weight_max": 33.75336456298828,
      "activations/layer2_attention_weight_min": -32.17755126953125,
      "activations/layer3_attention_weight_max": 101.01020812988281,
      "activations/layer3_attention_weight_min": -96.32460021972656,
      "activations/layer4_attention_weight_max": 110.57649230957031,
      "activations/layer4_attention_weight_min": -109.3744888305664,
      "activations/layer5_attention_weight_max": 50.668975830078125,
      "activations/layer5_attention_weight_min": -62.74605941772461,
      "activations/layer6_attention_weight_max": 44.88943099975586,
      "activations/layer6_attention_weight_min": -48.42363357543945,
      "activations/layer7_attention_weight_max": 95.14057159423828,
      "activations/layer7_attention_weight_min": -92.57501983642578,
      "activations/layer8_attention_weight_max": 40.540531158447266,
      "activations/layer8_attention_weight_min": -41.67332458496094,
      "activations/layer9_attention_weight_max": 32.45320510864258,
      "activations/layer9_attention_weight_min": -33.37704849243164,
      "epoch": 15.87,
      "learning_rate": 4.815189393939394e-05,
      "loss": 2.7567,
      "step": 273100
    },
    {
      "activations/layer0_attention_weight_max": 16.07891082763672,
      "activations/layer0_attention_weight_min": -12.473024368286133,
      "activations/layer10_attention_weight_max": 34.472076416015625,
      "activations/layer10_attention_weight_min": -33.16554641723633,
      "activations/layer11_attention_weight_max": 32.57569885253906,
      "activations/layer11_attention_weight_min": -33.573089599609375,
      "activations/layer12_attention_weight_max": 25.120054244995117,
      "activations/layer12_attention_weight_min": -25.55352210998535,
      "activations/layer13_attention_weight_max": 36.61346435546875,
      "activations/layer13_attention_weight_min": -34.282657623291016,
      "activations/layer14_attention_weight_max": 43.36475372314453,
      "activations/layer14_attention_weight_min": -35.11494827270508,
      "activations/layer15_attention_weight_max": 39.38260269165039,
      "activations/layer15_attention_weight_min": -36.066078186035156,
      "activations/layer16_attention_weight_max": 29.463417053222656,
      "activations/layer16_attention_weight_min": -28.538238525390625,
      "activations/layer17_attention_weight_max": 28.61947250366211,
      "activations/layer17_attention_weight_min": -25.3863468170166,
      "activations/layer18_attention_weight_max": 33.665306091308594,
      "activations/layer18_attention_weight_min": -25.15648651123047,
      "activations/layer19_attention_weight_max": 32.94668197631836,
      "activations/layer19_attention_weight_min": -31.03553581237793,
      "activations/layer1_attention_weight_max": 17.22304344177246,
      "activations/layer1_attention_weight_min": -16.543649673461914,
      "activations/layer20_attention_weight_max": 26.841806411743164,
      "activations/layer20_attention_weight_min": -26.010074615478516,
      "activations/layer21_attention_weight_max": 28.502151489257812,
      "activations/layer21_attention_weight_min": -23.634714126586914,
      "activations/layer22_attention_weight_max": 42.682594299316406,
      "activations/layer22_attention_weight_min": -29.8699951171875,
      "activations/layer23_attention_weight_max": 33.5311279296875,
      "activations/layer23_attention_weight_min": -26.6037654876709,
      "activations/layer2_attention_weight_max": 32.525657653808594,
      "activations/layer2_attention_weight_min": -32.70716857910156,
      "activations/layer3_attention_weight_max": 96.3208236694336,
      "activations/layer3_attention_weight_min": -96.2087173461914,
      "activations/layer4_attention_weight_max": 105.3521728515625,
      "activations/layer4_attention_weight_min": -108.17835998535156,
      "activations/layer5_attention_weight_max": 53.026119232177734,
      "activations/layer5_attention_weight_min": -65.30760955810547,
      "activations/layer6_attention_weight_max": 42.1662712097168,
      "activations/layer6_attention_weight_min": -45.798988342285156,
      "activations/layer7_attention_weight_max": 89.66741943359375,
      "activations/layer7_attention_weight_min": -95.57244110107422,
      "activations/layer8_attention_weight_max": 41.63426208496094,
      "activations/layer8_attention_weight_min": -41.34684371948242,
      "activations/layer9_attention_weight_max": 31.218570709228516,
      "activations/layer9_attention_weight_min": -33.4715576171875,
      "epoch": 15.87,
      "learning_rate": 4.813295454545454e-05,
      "loss": 2.7557,
      "step": 273150
    },
    {
      "activations/layer0_attention_weight_max": 15.272727966308594,
      "activations/layer0_attention_weight_min": -12.648752212524414,
      "activations/layer10_attention_weight_max": 35.24774169921875,
      "activations/layer10_attention_weight_min": -35.354915618896484,
      "activations/layer11_attention_weight_max": 34.71913528442383,
      "activations/layer11_attention_weight_min": -33.73783874511719,
      "activations/layer12_attention_weight_max": 28.876829147338867,
      "activations/layer12_attention_weight_min": -33.260711669921875,
      "activations/layer13_attention_weight_max": 39.9574089050293,
      "activations/layer13_attention_weight_min": -35.367393493652344,
      "activations/layer14_attention_weight_max": 48.49928665161133,
      "activations/layer14_attention_weight_min": -37.86770248413086,
      "activations/layer15_attention_weight_max": 45.47624969482422,
      "activations/layer15_attention_weight_min": -35.84342956542969,
      "activations/layer16_attention_weight_max": 36.99989318847656,
      "activations/layer16_attention_weight_min": -26.564016342163086,
      "activations/layer17_attention_weight_max": 34.260623931884766,
      "activations/layer17_attention_weight_min": -25.574228286743164,
      "activations/layer18_attention_weight_max": 29.611928939819336,
      "activations/layer18_attention_weight_min": -25.127796173095703,
      "activations/layer19_attention_weight_max": 38.281341552734375,
      "activations/layer19_attention_weight_min": -29.657791137695312,
      "activations/layer1_attention_weight_max": 18.22270393371582,
      "activations/layer1_attention_weight_min": -17.23691177368164,
      "activations/layer20_attention_weight_max": 31.28118324279785,
      "activations/layer20_attention_weight_min": -24.803544998168945,
      "activations/layer21_attention_weight_max": 28.882150650024414,
      "activations/layer21_attention_weight_min": -24.88629722595215,
      "activations/layer22_attention_weight_max": 43.98638916015625,
      "activations/layer22_attention_weight_min": -31.12749481201172,
      "activations/layer23_attention_weight_max": 35.263057708740234,
      "activations/layer23_attention_weight_min": -26.339712142944336,
      "activations/layer2_attention_weight_max": 33.278533935546875,
      "activations/layer2_attention_weight_min": -31.857158660888672,
      "activations/layer3_attention_weight_max": 94.17583465576172,
      "activations/layer3_attention_weight_min": -96.32770538330078,
      "activations/layer4_attention_weight_max": 104.392333984375,
      "activations/layer4_attention_weight_min": -106.12054443359375,
      "activations/layer5_attention_weight_max": 48.445369720458984,
      "activations/layer5_attention_weight_min": -60.049705505371094,
      "activations/layer6_attention_weight_max": 43.550392150878906,
      "activations/layer6_attention_weight_min": -44.872779846191406,
      "activations/layer7_attention_weight_max": 92.1146240234375,
      "activations/layer7_attention_weight_min": -90.73300170898438,
      "activations/layer8_attention_weight_max": 40.763763427734375,
      "activations/layer8_attention_weight_min": -42.80646896362305,
      "activations/layer9_attention_weight_max": 32.11483383178711,
      "activations/layer9_attention_weight_min": -35.27219009399414,
      "epoch": 15.87,
      "learning_rate": 4.811401515151515e-05,
      "loss": 2.7432,
      "step": 273200
    },
    {
      "activations/layer0_attention_weight_max": 15.953475952148438,
      "activations/layer0_attention_weight_min": -13.014690399169922,
      "activations/layer10_attention_weight_max": 34.62509536743164,
      "activations/layer10_attention_weight_min": -33.82612991333008,
      "activations/layer11_attention_weight_max": 36.026588439941406,
      "activations/layer11_attention_weight_min": -34.260040283203125,
      "activations/layer12_attention_weight_max": 27.089988708496094,
      "activations/layer12_attention_weight_min": -25.76464080810547,
      "activations/layer13_attention_weight_max": 42.450164794921875,
      "activations/layer13_attention_weight_min": -34.51738357543945,
      "activations/layer14_attention_weight_max": 45.91112518310547,
      "activations/layer14_attention_weight_min": -36.49249267578125,
      "activations/layer15_attention_weight_max": 40.2125129699707,
      "activations/layer15_attention_weight_min": -34.02039337158203,
      "activations/layer16_attention_weight_max": 30.261762619018555,
      "activations/layer16_attention_weight_min": -27.32122802734375,
      "activations/layer17_attention_weight_max": 31.32004737854004,
      "activations/layer17_attention_weight_min": -25.760915756225586,
      "activations/layer18_attention_weight_max": 35.522544860839844,
      "activations/layer18_attention_weight_min": -23.853187561035156,
      "activations/layer19_attention_weight_max": 37.73073196411133,
      "activations/layer19_attention_weight_min": -30.218759536743164,
      "activations/layer1_attention_weight_max": 16.600433349609375,
      "activations/layer1_attention_weight_min": -15.41079044342041,
      "activations/layer20_attention_weight_max": 32.33327102661133,
      "activations/layer20_attention_weight_min": -24.75136375427246,
      "activations/layer21_attention_weight_max": 33.560909271240234,
      "activations/layer21_attention_weight_min": -26.6064510345459,
      "activations/layer22_attention_weight_max": 45.47575759887695,
      "activations/layer22_attention_weight_min": -31.826780319213867,
      "activations/layer23_attention_weight_max": 37.01918029785156,
      "activations/layer23_attention_weight_min": -27.422887802124023,
      "activations/layer2_attention_weight_max": 34.69413757324219,
      "activations/layer2_attention_weight_min": -32.72386169433594,
      "activations/layer3_attention_weight_max": 99.33004760742188,
      "activations/layer3_attention_weight_min": -99.42040252685547,
      "activations/layer4_attention_weight_max": 107.68531799316406,
      "activations/layer4_attention_weight_min": -108.18053436279297,
      "activations/layer5_attention_weight_max": 51.431060791015625,
      "activations/layer5_attention_weight_min": -62.465858459472656,
      "activations/layer6_attention_weight_max": 43.64162063598633,
      "activations/layer6_attention_weight_min": -44.1114616394043,
      "activations/layer7_attention_weight_max": 95.61276245117188,
      "activations/layer7_attention_weight_min": -95.39498138427734,
      "activations/layer8_attention_weight_max": 40.0414924621582,
      "activations/layer8_attention_weight_min": -41.4592399597168,
      "activations/layer9_attention_weight_max": 33.075592041015625,
      "activations/layer9_attention_weight_min": -34.0189094543457,
      "epoch": 15.88,
      "learning_rate": 4.8095075757575756e-05,
      "loss": 2.7417,
      "step": 273250
    },
    {
      "activations/layer0_attention_weight_max": 15.938485145568848,
      "activations/layer0_attention_weight_min": -12.218323707580566,
      "activations/layer10_attention_weight_max": 32.47312927246094,
      "activations/layer10_attention_weight_min": -32.57041549682617,
      "activations/layer11_attention_weight_max": 32.154701232910156,
      "activations/layer11_attention_weight_min": -31.945972442626953,
      "activations/layer12_attention_weight_max": 24.539560317993164,
      "activations/layer12_attention_weight_min": -25.417449951171875,
      "activations/layer13_attention_weight_max": 37.46324920654297,
      "activations/layer13_attention_weight_min": -34.67482376098633,
      "activations/layer14_attention_weight_max": 37.38966751098633,
      "activations/layer14_attention_weight_min": -35.92939376831055,
      "activations/layer15_attention_weight_max": 36.34767532348633,
      "activations/layer15_attention_weight_min": -33.283267974853516,
      "activations/layer16_attention_weight_max": 30.070045471191406,
      "activations/layer16_attention_weight_min": -27.60635757446289,
      "activations/layer17_attention_weight_max": 27.795207977294922,
      "activations/layer17_attention_weight_min": -26.392240524291992,
      "activations/layer18_attention_weight_max": 32.8044319152832,
      "activations/layer18_attention_weight_min": -26.346445083618164,
      "activations/layer19_attention_weight_max": 40.02585983276367,
      "activations/layer19_attention_weight_min": -32.88207244873047,
      "activations/layer1_attention_weight_max": 16.332408905029297,
      "activations/layer1_attention_weight_min": -14.844902038574219,
      "activations/layer20_attention_weight_max": 36.31059265136719,
      "activations/layer20_attention_weight_min": -28.731843948364258,
      "activations/layer21_attention_weight_max": 33.51385498046875,
      "activations/layer21_attention_weight_min": -27.06404685974121,
      "activations/layer22_attention_weight_max": 47.62778091430664,
      "activations/layer22_attention_weight_min": -35.25822067260742,
      "activations/layer23_attention_weight_max": 38.448307037353516,
      "activations/layer23_attention_weight_min": -30.466136932373047,
      "activations/layer2_attention_weight_max": 32.72829818725586,
      "activations/layer2_attention_weight_min": -32.96879577636719,
      "activations/layer3_attention_weight_max": 95.63175964355469,
      "activations/layer3_attention_weight_min": -97.30133056640625,
      "activations/layer4_attention_weight_max": 107.02811431884766,
      "activations/layer4_attention_weight_min": -106.12451171875,
      "activations/layer5_attention_weight_max": 49.71714401245117,
      "activations/layer5_attention_weight_min": -62.971614837646484,
      "activations/layer6_attention_weight_max": 42.83740997314453,
      "activations/layer6_attention_weight_min": -47.98335647583008,
      "activations/layer7_attention_weight_max": 91.3415756225586,
      "activations/layer7_attention_weight_min": -97.77008056640625,
      "activations/layer8_attention_weight_max": 40.81597900390625,
      "activations/layer8_attention_weight_min": -41.11750793457031,
      "activations/layer9_attention_weight_max": 29.909696578979492,
      "activations/layer9_attention_weight_min": -30.830121994018555,
      "epoch": 15.88,
      "learning_rate": 4.8076136363636364e-05,
      "loss": 2.762,
      "step": 273300
    },
    {
      "activations/layer0_attention_weight_max": 16.017351150512695,
      "activations/layer0_attention_weight_min": -12.727086067199707,
      "activations/layer10_attention_weight_max": 33.728004455566406,
      "activations/layer10_attention_weight_min": -32.30204391479492,
      "activations/layer11_attention_weight_max": 31.935590744018555,
      "activations/layer11_attention_weight_min": -33.45407485961914,
      "activations/layer12_attention_weight_max": 24.612567901611328,
      "activations/layer12_attention_weight_min": -26.43659782409668,
      "activations/layer13_attention_weight_max": 36.8984375,
      "activations/layer13_attention_weight_min": -35.80630111694336,
      "activations/layer14_attention_weight_max": 40.253631591796875,
      "activations/layer14_attention_weight_min": -35.76397705078125,
      "activations/layer15_attention_weight_max": 34.46311569213867,
      "activations/layer15_attention_weight_min": -32.51080322265625,
      "activations/layer16_attention_weight_max": 27.580198287963867,
      "activations/layer16_attention_weight_min": -26.640380859375,
      "activations/layer17_attention_weight_max": 28.375070571899414,
      "activations/layer17_attention_weight_min": -24.760751724243164,
      "activations/layer18_attention_weight_max": 29.979251861572266,
      "activations/layer18_attention_weight_min": -25.330429077148438,
      "activations/layer19_attention_weight_max": 34.748043060302734,
      "activations/layer19_attention_weight_min": -31.804452896118164,
      "activations/layer1_attention_weight_max": 17.001741409301758,
      "activations/layer1_attention_weight_min": -14.96460247039795,
      "activations/layer20_attention_weight_max": 28.080265045166016,
      "activations/layer20_attention_weight_min": -25.04747200012207,
      "activations/layer21_attention_weight_max": 26.558523178100586,
      "activations/layer21_attention_weight_min": -24.622539520263672,
      "activations/layer22_attention_weight_max": 41.28997039794922,
      "activations/layer22_attention_weight_min": -31.36463737487793,
      "activations/layer23_attention_weight_max": 28.636489868164062,
      "activations/layer23_attention_weight_min": -24.996633529663086,
      "activations/layer2_attention_weight_max": 33.398460388183594,
      "activations/layer2_attention_weight_min": -33.965370178222656,
      "activations/layer3_attention_weight_max": 97.27299499511719,
      "activations/layer3_attention_weight_min": -95.98393249511719,
      "activations/layer4_attention_weight_max": 104.77519989013672,
      "activations/layer4_attention_weight_min": -105.1050796508789,
      "activations/layer5_attention_weight_max": 47.96875,
      "activations/layer5_attention_weight_min": -61.42531967163086,
      "activations/layer6_attention_weight_max": 44.13059616088867,
      "activations/layer6_attention_weight_min": -46.70344924926758,
      "activations/layer7_attention_weight_max": 92.40437316894531,
      "activations/layer7_attention_weight_min": -91.8470230102539,
      "activations/layer8_attention_weight_max": 40.88082504272461,
      "activations/layer8_attention_weight_min": -44.57182693481445,
      "activations/layer9_attention_weight_max": 33.016231536865234,
      "activations/layer9_attention_weight_min": -34.657562255859375,
      "epoch": 15.88,
      "learning_rate": 4.8057196969696966e-05,
      "loss": 2.7544,
      "step": 273350
    },
    {
      "activations/layer0_attention_weight_max": 16.408206939697266,
      "activations/layer0_attention_weight_min": -12.93724250793457,
      "activations/layer10_attention_weight_max": 31.861974716186523,
      "activations/layer10_attention_weight_min": -35.325164794921875,
      "activations/layer11_attention_weight_max": 31.923274993896484,
      "activations/layer11_attention_weight_min": -36.000030517578125,
      "activations/layer12_attention_weight_max": 25.580142974853516,
      "activations/layer12_attention_weight_min": -27.816062927246094,
      "activations/layer13_attention_weight_max": 41.36634826660156,
      "activations/layer13_attention_weight_min": -37.520172119140625,
      "activations/layer14_attention_weight_max": 40.24583435058594,
      "activations/layer14_attention_weight_min": -38.17399215698242,
      "activations/layer15_attention_weight_max": 37.59635925292969,
      "activations/layer15_attention_weight_min": -36.191585540771484,
      "activations/layer16_attention_weight_max": 33.698246002197266,
      "activations/layer16_attention_weight_min": -29.47435188293457,
      "activations/layer17_attention_weight_max": 31.87435531616211,
      "activations/layer17_attention_weight_min": -28.105026245117188,
      "activations/layer18_attention_weight_max": 33.48435974121094,
      "activations/layer18_attention_weight_min": -26.237316131591797,
      "activations/layer19_attention_weight_max": 35.36022186279297,
      "activations/layer19_attention_weight_min": -34.54098129272461,
      "activations/layer1_attention_weight_max": 15.709789276123047,
      "activations/layer1_attention_weight_min": -14.510486602783203,
      "activations/layer20_attention_weight_max": 29.41231346130371,
      "activations/layer20_attention_weight_min": -26.860809326171875,
      "activations/layer21_attention_weight_max": 31.998807907104492,
      "activations/layer21_attention_weight_min": -27.45241355895996,
      "activations/layer22_attention_weight_max": 45.225852966308594,
      "activations/layer22_attention_weight_min": -33.81254196166992,
      "activations/layer23_attention_weight_max": 37.871376037597656,
      "activations/layer23_attention_weight_min": -28.56298065185547,
      "activations/layer2_attention_weight_max": 32.79651641845703,
      "activations/layer2_attention_weight_min": -31.398595809936523,
      "activations/layer3_attention_weight_max": 92.12297821044922,
      "activations/layer3_attention_weight_min": -92.12977600097656,
      "activations/layer4_attention_weight_max": 106.10487365722656,
      "activations/layer4_attention_weight_min": -107.95564270019531,
      "activations/layer5_attention_weight_max": 49.581024169921875,
      "activations/layer5_attention_weight_min": -62.334556579589844,
      "activations/layer6_attention_weight_max": 42.73728942871094,
      "activations/layer6_attention_weight_min": -44.83671569824219,
      "activations/layer7_attention_weight_max": 89.12605285644531,
      "activations/layer7_attention_weight_min": -88.57520294189453,
      "activations/layer8_attention_weight_max": 41.9908561706543,
      "activations/layer8_attention_weight_min": -44.016178131103516,
      "activations/layer9_attention_weight_max": 30.9970760345459,
      "activations/layer9_attention_weight_min": -34.39509582519531,
      "epoch": 15.89,
      "learning_rate": 4.803825757575757e-05,
      "loss": 2.7443,
      "step": 273400
    },
    {
      "activations/layer0_attention_weight_max": 15.1609468460083,
      "activations/layer0_attention_weight_min": -13.128150939941406,
      "activations/layer10_attention_weight_max": 33.60576248168945,
      "activations/layer10_attention_weight_min": -34.56250762939453,
      "activations/layer11_attention_weight_max": 31.99341583251953,
      "activations/layer11_attention_weight_min": -33.76283264160156,
      "activations/layer12_attention_weight_max": 24.623950958251953,
      "activations/layer12_attention_weight_min": -24.962011337280273,
      "activations/layer13_attention_weight_max": 35.47000503540039,
      "activations/layer13_attention_weight_min": -32.94658660888672,
      "activations/layer14_attention_weight_max": 38.7289924621582,
      "activations/layer14_attention_weight_min": -34.985103607177734,
      "activations/layer15_attention_weight_max": 35.04951858520508,
      "activations/layer15_attention_weight_min": -31.801036834716797,
      "activations/layer16_attention_weight_max": 27.463083267211914,
      "activations/layer16_attention_weight_min": -26.92906951904297,
      "activations/layer17_attention_weight_max": 29.645511627197266,
      "activations/layer17_attention_weight_min": -26.362329483032227,
      "activations/layer18_attention_weight_max": 29.788827896118164,
      "activations/layer18_attention_weight_min": -26.558385848999023,
      "activations/layer19_attention_weight_max": 34.57439041137695,
      "activations/layer19_attention_weight_min": -32.23468780517578,
      "activations/layer1_attention_weight_max": 16.049945831298828,
      "activations/layer1_attention_weight_min": -15.137563705444336,
      "activations/layer20_attention_weight_max": 28.088788986206055,
      "activations/layer20_attention_weight_min": -26.174888610839844,
      "activations/layer21_attention_weight_max": 28.29345703125,
      "activations/layer21_attention_weight_min": -26.625579833984375,
      "activations/layer22_attention_weight_max": 36.61927032470703,
      "activations/layer22_attention_weight_min": -32.1475715637207,
      "activations/layer23_attention_weight_max": 31.328298568725586,
      "activations/layer23_attention_weight_min": -28.61005973815918,
      "activations/layer2_attention_weight_max": 32.63788986206055,
      "activations/layer2_attention_weight_min": -32.1794548034668,
      "activations/layer3_attention_weight_max": 98.32183074951172,
      "activations/layer3_attention_weight_min": -93.9290771484375,
      "activations/layer4_attention_weight_max": 105.06233978271484,
      "activations/layer4_attention_weight_min": -107.974853515625,
      "activations/layer5_attention_weight_max": 51.629573822021484,
      "activations/layer5_attention_weight_min": -62.350257873535156,
      "activations/layer6_attention_weight_max": 43.83970642089844,
      "activations/layer6_attention_weight_min": -46.054080963134766,
      "activations/layer7_attention_weight_max": 94.34134674072266,
      "activations/layer7_attention_weight_min": -95.64253997802734,
      "activations/layer8_attention_weight_max": 43.09334182739258,
      "activations/layer8_attention_weight_min": -42.95813751220703,
      "activations/layer9_attention_weight_max": 31.743600845336914,
      "activations/layer9_attention_weight_min": -33.2403450012207,
      "epoch": 15.89,
      "learning_rate": 4.801931818181818e-05,
      "loss": 2.7672,
      "step": 273450
    },
    {
      "activations/layer0_attention_weight_max": 16.36705207824707,
      "activations/layer0_attention_weight_min": -12.881279945373535,
      "activations/layer10_attention_weight_max": 31.907682418823242,
      "activations/layer10_attention_weight_min": -33.069793701171875,
      "activations/layer11_attention_weight_max": 34.12529754638672,
      "activations/layer11_attention_weight_min": -34.68366622924805,
      "activations/layer12_attention_weight_max": 27.970115661621094,
      "activations/layer12_attention_weight_min": -28.15326690673828,
      "activations/layer13_attention_weight_max": 39.70095443725586,
      "activations/layer13_attention_weight_min": -36.66847229003906,
      "activations/layer14_attention_weight_max": 43.883216857910156,
      "activations/layer14_attention_weight_min": -38.75917053222656,
      "activations/layer15_attention_weight_max": 43.57606506347656,
      "activations/layer15_attention_weight_min": -35.646358489990234,
      "activations/layer16_attention_weight_max": 35.41276550292969,
      "activations/layer16_attention_weight_min": -29.649869918823242,
      "activations/layer17_attention_weight_max": 38.42692184448242,
      "activations/layer17_attention_weight_min": -27.365854263305664,
      "activations/layer18_attention_weight_max": 38.95452117919922,
      "activations/layer18_attention_weight_min": -28.983041763305664,
      "activations/layer19_attention_weight_max": 38.8114013671875,
      "activations/layer19_attention_weight_min": -33.731685638427734,
      "activations/layer1_attention_weight_max": 17.158700942993164,
      "activations/layer1_attention_weight_min": -14.913070678710938,
      "activations/layer20_attention_weight_max": 30.478586196899414,
      "activations/layer20_attention_weight_min": -26.587156295776367,
      "activations/layer21_attention_weight_max": 29.531064987182617,
      "activations/layer21_attention_weight_min": -25.5241641998291,
      "activations/layer22_attention_weight_max": 47.283294677734375,
      "activations/layer22_attention_weight_min": -33.08928298950195,
      "activations/layer23_attention_weight_max": 33.9965934753418,
      "activations/layer23_attention_weight_min": -26.439077377319336,
      "activations/layer2_attention_weight_max": 35.827110290527344,
      "activations/layer2_attention_weight_min": -31.83924102783203,
      "activations/layer3_attention_weight_max": 97.60089874267578,
      "activations/layer3_attention_weight_min": -94.87783813476562,
      "activations/layer4_attention_weight_max": 105.3526611328125,
      "activations/layer4_attention_weight_min": -107.13031005859375,
      "activations/layer5_attention_weight_max": 52.84379577636719,
      "activations/layer5_attention_weight_min": -62.19792938232422,
      "activations/layer6_attention_weight_max": 41.99302673339844,
      "activations/layer6_attention_weight_min": -45.24660110473633,
      "activations/layer7_attention_weight_max": 93.12702178955078,
      "activations/layer7_attention_weight_min": -92.38323974609375,
      "activations/layer8_attention_weight_max": 40.09364318847656,
      "activations/layer8_attention_weight_min": -41.30840301513672,
      "activations/layer9_attention_weight_max": 32.39753341674805,
      "activations/layer9_attention_weight_min": -32.383235931396484,
      "epoch": 15.89,
      "learning_rate": 4.8000378787878784e-05,
      "loss": 2.7541,
      "step": 273500
    },
    {
      "activations/layer0_attention_weight_max": 16.610153198242188,
      "activations/layer0_attention_weight_min": -11.787545204162598,
      "activations/layer10_attention_weight_max": 40.659751892089844,
      "activations/layer10_attention_weight_min": -39.202239990234375,
      "activations/layer11_attention_weight_max": 39.824615478515625,
      "activations/layer11_attention_weight_min": -38.3853645324707,
      "activations/layer12_attention_weight_max": 27.303327560424805,
      "activations/layer12_attention_weight_min": -27.523303985595703,
      "activations/layer13_attention_weight_max": 39.3473014831543,
      "activations/layer13_attention_weight_min": -35.22196578979492,
      "activations/layer14_attention_weight_max": 40.75588607788086,
      "activations/layer14_attention_weight_min": -38.84218978881836,
      "activations/layer15_attention_weight_max": 41.7401237487793,
      "activations/layer15_attention_weight_min": -32.961883544921875,
      "activations/layer16_attention_weight_max": 27.56072235107422,
      "activations/layer16_attention_weight_min": -25.888437271118164,
      "activations/layer17_attention_weight_max": 28.8919677734375,
      "activations/layer17_attention_weight_min": -25.799636840820312,
      "activations/layer18_attention_weight_max": 31.067338943481445,
      "activations/layer18_attention_weight_min": -23.451501846313477,
      "activations/layer19_attention_weight_max": 30.481094360351562,
      "activations/layer19_attention_weight_min": -30.64432716369629,
      "activations/layer1_attention_weight_max": 17.2152099609375,
      "activations/layer1_attention_weight_min": -15.678531646728516,
      "activations/layer20_attention_weight_max": 26.323867797851562,
      "activations/layer20_attention_weight_min": -25.046977996826172,
      "activations/layer21_attention_weight_max": 25.447805404663086,
      "activations/layer21_attention_weight_min": -24.742082595825195,
      "activations/layer22_attention_weight_max": 43.05859375,
      "activations/layer22_attention_weight_min": -29.37611961364746,
      "activations/layer23_attention_weight_max": 31.3330135345459,
      "activations/layer23_attention_weight_min": -24.303451538085938,
      "activations/layer2_attention_weight_max": 33.755958557128906,
      "activations/layer2_attention_weight_min": -35.46636962890625,
      "activations/layer3_attention_weight_max": 103.36393737792969,
      "activations/layer3_attention_weight_min": -104.97693634033203,
      "activations/layer4_attention_weight_max": 114.34600067138672,
      "activations/layer4_attention_weight_min": -109.8564224243164,
      "activations/layer5_attention_weight_max": 57.932777404785156,
      "activations/layer5_attention_weight_min": -62.45600891113281,
      "activations/layer6_attention_weight_max": 43.83711242675781,
      "activations/layer6_attention_weight_min": -46.612953186035156,
      "activations/layer7_attention_weight_max": 104.1766357421875,
      "activations/layer7_attention_weight_min": -98.06537628173828,
      "activations/layer8_attention_weight_max": 47.204078674316406,
      "activations/layer8_attention_weight_min": -47.28178405761719,
      "activations/layer9_attention_weight_max": 36.98296356201172,
      "activations/layer9_attention_weight_min": -37.28701400756836,
      "epoch": 15.89,
      "learning_rate": 4.798143939393939e-05,
      "loss": 2.7493,
      "step": 273550
    },
    {
      "activations/layer0_attention_weight_max": 16.10235023498535,
      "activations/layer0_attention_weight_min": -12.054117202758789,
      "activations/layer10_attention_weight_max": 33.72129440307617,
      "activations/layer10_attention_weight_min": -33.6639289855957,
      "activations/layer11_attention_weight_max": 32.14384460449219,
      "activations/layer11_attention_weight_min": -32.51789855957031,
      "activations/layer12_attention_weight_max": 25.994054794311523,
      "activations/layer12_attention_weight_min": -26.263160705566406,
      "activations/layer13_attention_weight_max": 35.494205474853516,
      "activations/layer13_attention_weight_min": -34.51789093017578,
      "activations/layer14_attention_weight_max": 42.193992614746094,
      "activations/layer14_attention_weight_min": -36.316123962402344,
      "activations/layer15_attention_weight_max": 37.17778396606445,
      "activations/layer15_attention_weight_min": -35.41958236694336,
      "activations/layer16_attention_weight_max": 28.195632934570312,
      "activations/layer16_attention_weight_min": -29.903905868530273,
      "activations/layer17_attention_weight_max": 29.11689567565918,
      "activations/layer17_attention_weight_min": -27.11829376220703,
      "activations/layer18_attention_weight_max": 32.06834030151367,
      "activations/layer18_attention_weight_min": -26.46963882446289,
      "activations/layer19_attention_weight_max": 34.13701248168945,
      "activations/layer19_attention_weight_min": -33.172767639160156,
      "activations/layer1_attention_weight_max": 16.73641014099121,
      "activations/layer1_attention_weight_min": -15.931761741638184,
      "activations/layer20_attention_weight_max": 28.466495513916016,
      "activations/layer20_attention_weight_min": -26.206693649291992,
      "activations/layer21_attention_weight_max": 29.259883880615234,
      "activations/layer21_attention_weight_min": -25.522014617919922,
      "activations/layer22_attention_weight_max": 38.98891830444336,
      "activations/layer22_attention_weight_min": -31.705278396606445,
      "activations/layer23_attention_weight_max": 30.16591453552246,
      "activations/layer23_attention_weight_min": -27.756574630737305,
      "activations/layer2_attention_weight_max": 33.6558723449707,
      "activations/layer2_attention_weight_min": -32.79792785644531,
      "activations/layer3_attention_weight_max": 96.49158477783203,
      "activations/layer3_attention_weight_min": -99.28189086914062,
      "activations/layer4_attention_weight_max": 108.43364715576172,
      "activations/layer4_attention_weight_min": -104.37703704833984,
      "activations/layer5_attention_weight_max": 52.27314376831055,
      "activations/layer5_attention_weight_min": -58.81903839111328,
      "activations/layer6_attention_weight_max": 44.05990219116211,
      "activations/layer6_attention_weight_min": -45.94524002075195,
      "activations/layer7_attention_weight_max": 89.12008666992188,
      "activations/layer7_attention_weight_min": -92.14290618896484,
      "activations/layer8_attention_weight_max": 41.079551696777344,
      "activations/layer8_attention_weight_min": -43.23851013183594,
      "activations/layer9_attention_weight_max": 32.20783615112305,
      "activations/layer9_attention_weight_min": -33.54775619506836,
      "epoch": 15.9,
      "learning_rate": 4.7962499999999994e-05,
      "loss": 2.7509,
      "step": 273600
    },
    {
      "activations/layer0_attention_weight_max": 15.51252555847168,
      "activations/layer0_attention_weight_min": -12.73798942565918,
      "activations/layer10_attention_weight_max": 33.82343673706055,
      "activations/layer10_attention_weight_min": -35.294288635253906,
      "activations/layer11_attention_weight_max": 34.7315673828125,
      "activations/layer11_attention_weight_min": -36.53550338745117,
      "activations/layer12_attention_weight_max": 27.35067367553711,
      "activations/layer12_attention_weight_min": -27.294898986816406,
      "activations/layer13_attention_weight_max": 39.5296516418457,
      "activations/layer13_attention_weight_min": -37.501380920410156,
      "activations/layer14_attention_weight_max": 42.79685592651367,
      "activations/layer14_attention_weight_min": -40.17271423339844,
      "activations/layer15_attention_weight_max": 37.52507400512695,
      "activations/layer15_attention_weight_min": -35.2554817199707,
      "activations/layer16_attention_weight_max": 29.347789764404297,
      "activations/layer16_attention_weight_min": -29.806306838989258,
      "activations/layer17_attention_weight_max": 28.5042724609375,
      "activations/layer17_attention_weight_min": -25.45779037475586,
      "activations/layer18_attention_weight_max": 29.173805236816406,
      "activations/layer18_attention_weight_min": -25.4705753326416,
      "activations/layer19_attention_weight_max": 32.00179672241211,
      "activations/layer19_attention_weight_min": -33.21477508544922,
      "activations/layer1_attention_weight_max": 16.17176628112793,
      "activations/layer1_attention_weight_min": -15.73823070526123,
      "activations/layer20_attention_weight_max": 30.453622817993164,
      "activations/layer20_attention_weight_min": -26.663135528564453,
      "activations/layer21_attention_weight_max": 27.819286346435547,
      "activations/layer21_attention_weight_min": -27.185558319091797,
      "activations/layer22_attention_weight_max": 45.136924743652344,
      "activations/layer22_attention_weight_min": -30.86270523071289,
      "activations/layer23_attention_weight_max": 32.73538589477539,
      "activations/layer23_attention_weight_min": -25.768842697143555,
      "activations/layer2_attention_weight_max": 31.349340438842773,
      "activations/layer2_attention_weight_min": -32.8270263671875,
      "activations/layer3_attention_weight_max": 95.12665557861328,
      "activations/layer3_attention_weight_min": -96.00232696533203,
      "activations/layer4_attention_weight_max": 101.42772674560547,
      "activations/layer4_attention_weight_min": -103.20894622802734,
      "activations/layer5_attention_weight_max": 47.401390075683594,
      "activations/layer5_attention_weight_min": -60.271080017089844,
      "activations/layer6_attention_weight_max": 42.63967514038086,
      "activations/layer6_attention_weight_min": -46.03923797607422,
      "activations/layer7_attention_weight_max": 94.3071517944336,
      "activations/layer7_attention_weight_min": -96.07676696777344,
      "activations/layer8_attention_weight_max": 39.55831527709961,
      "activations/layer8_attention_weight_min": -45.65586471557617,
      "activations/layer9_attention_weight_max": 33.641357421875,
      "activations/layer9_attention_weight_min": -37.39767074584961,
      "epoch": 15.9,
      "learning_rate": 4.79435606060606e-05,
      "loss": 2.7367,
      "step": 273650
    },
    {
      "activations/layer0_attention_weight_max": 15.303207397460938,
      "activations/layer0_attention_weight_min": -13.54240894317627,
      "activations/layer10_attention_weight_max": 33.88853454589844,
      "activations/layer10_attention_weight_min": -34.408973693847656,
      "activations/layer11_attention_weight_max": 32.166927337646484,
      "activations/layer11_attention_weight_min": -34.03032684326172,
      "activations/layer12_attention_weight_max": 25.041101455688477,
      "activations/layer12_attention_weight_min": -27.69249153137207,
      "activations/layer13_attention_weight_max": 39.82754135131836,
      "activations/layer13_attention_weight_min": -35.86976623535156,
      "activations/layer14_attention_weight_max": 40.67744064331055,
      "activations/layer14_attention_weight_min": -40.43173599243164,
      "activations/layer15_attention_weight_max": 35.95193099975586,
      "activations/layer15_attention_weight_min": -36.20627975463867,
      "activations/layer16_attention_weight_max": 28.449092864990234,
      "activations/layer16_attention_weight_min": -28.23713493347168,
      "activations/layer17_attention_weight_max": 30.387128829956055,
      "activations/layer17_attention_weight_min": -27.19644546508789,
      "activations/layer18_attention_weight_max": 32.83859634399414,
      "activations/layer18_attention_weight_min": -26.309072494506836,
      "activations/layer19_attention_weight_max": 34.551387786865234,
      "activations/layer19_attention_weight_min": -31.67873191833496,
      "activations/layer1_attention_weight_max": 16.99250602722168,
      "activations/layer1_attention_weight_min": -17.673891067504883,
      "activations/layer20_attention_weight_max": 30.087339401245117,
      "activations/layer20_attention_weight_min": -27.973175048828125,
      "activations/layer21_attention_weight_max": 29.446712493896484,
      "activations/layer21_attention_weight_min": -27.897977828979492,
      "activations/layer22_attention_weight_max": 38.23984909057617,
      "activations/layer22_attention_weight_min": -34.79170227050781,
      "activations/layer23_attention_weight_max": 31.844223022460938,
      "activations/layer23_attention_weight_min": -29.22177505493164,
      "activations/layer2_attention_weight_max": 32.41791534423828,
      "activations/layer2_attention_weight_min": -32.38642120361328,
      "activations/layer3_attention_weight_max": 92.74422454833984,
      "activations/layer3_attention_weight_min": -94.27376556396484,
      "activations/layer4_attention_weight_max": 101.79017639160156,
      "activations/layer4_attention_weight_min": -105.51497650146484,
      "activations/layer5_attention_weight_max": 50.17814254760742,
      "activations/layer5_attention_weight_min": -62.017005920410156,
      "activations/layer6_attention_weight_max": 43.99678421020508,
      "activations/layer6_attention_weight_min": -45.84947967529297,
      "activations/layer7_attention_weight_max": 93.09616088867188,
      "activations/layer7_attention_weight_min": -93.96600341796875,
      "activations/layer8_attention_weight_max": 40.69136428833008,
      "activations/layer8_attention_weight_min": -44.93960189819336,
      "activations/layer9_attention_weight_max": 32.19599533081055,
      "activations/layer9_attention_weight_min": -32.6964225769043,
      "epoch": 15.9,
      "learning_rate": 4.792462121212121e-05,
      "loss": 2.763,
      "step": 273700
    },
    {
      "activations/layer0_attention_weight_max": 15.895939826965332,
      "activations/layer0_attention_weight_min": -13.769364356994629,
      "activations/layer10_attention_weight_max": 31.490581512451172,
      "activations/layer10_attention_weight_min": -32.72673797607422,
      "activations/layer11_attention_weight_max": 32.041648864746094,
      "activations/layer11_attention_weight_min": -34.13560485839844,
      "activations/layer12_attention_weight_max": 30.97899627685547,
      "activations/layer12_attention_weight_min": -28.083805084228516,
      "activations/layer13_attention_weight_max": 38.829689025878906,
      "activations/layer13_attention_weight_min": -35.8145866394043,
      "activations/layer14_attention_weight_max": 40.71894454956055,
      "activations/layer14_attention_weight_min": -37.043148040771484,
      "activations/layer15_attention_weight_max": 37.90279769897461,
      "activations/layer15_attention_weight_min": -35.599586486816406,
      "activations/layer16_attention_weight_max": 31.163646697998047,
      "activations/layer16_attention_weight_min": -28.577608108520508,
      "activations/layer17_attention_weight_max": 29.42219352722168,
      "activations/layer17_attention_weight_min": -25.85801124572754,
      "activations/layer18_attention_weight_max": 32.57662582397461,
      "activations/layer18_attention_weight_min": -24.410308837890625,
      "activations/layer19_attention_weight_max": 32.93647384643555,
      "activations/layer19_attention_weight_min": -32.093589782714844,
      "activations/layer1_attention_weight_max": 16.532302856445312,
      "activations/layer1_attention_weight_min": -15.489182472229004,
      "activations/layer20_attention_weight_max": 28.842845916748047,
      "activations/layer20_attention_weight_min": -24.010629653930664,
      "activations/layer21_attention_weight_max": 28.830785751342773,
      "activations/layer21_attention_weight_min": -22.759103775024414,
      "activations/layer22_attention_weight_max": 45.70329666137695,
      "activations/layer22_attention_weight_min": -29.98169708251953,
      "activations/layer23_attention_weight_max": 32.24399948120117,
      "activations/layer23_attention_weight_min": -27.006206512451172,
      "activations/layer2_attention_weight_max": 32.31201934814453,
      "activations/layer2_attention_weight_min": -33.34247589111328,
      "activations/layer3_attention_weight_max": 95.39237976074219,
      "activations/layer3_attention_weight_min": -99.11170196533203,
      "activations/layer4_attention_weight_max": 105.64397430419922,
      "activations/layer4_attention_weight_min": -108.92101287841797,
      "activations/layer5_attention_weight_max": 49.199668884277344,
      "activations/layer5_attention_weight_min": -65.86610412597656,
      "activations/layer6_attention_weight_max": 43.48918151855469,
      "activations/layer6_attention_weight_min": -46.2485237121582,
      "activations/layer7_attention_weight_max": 89.2142333984375,
      "activations/layer7_attention_weight_min": -92.87956237792969,
      "activations/layer8_attention_weight_max": 39.24579620361328,
      "activations/layer8_attention_weight_min": -41.701541900634766,
      "activations/layer9_attention_weight_max": 30.95711326599121,
      "activations/layer9_attention_weight_min": -35.65061950683594,
      "epoch": 15.91,
      "learning_rate": 4.790568181818181e-05,
      "loss": 2.7506,
      "step": 273750
    },
    {
      "activations/layer0_attention_weight_max": 16.157106399536133,
      "activations/layer0_attention_weight_min": -11.825067520141602,
      "activations/layer10_attention_weight_max": 42.41028594970703,
      "activations/layer10_attention_weight_min": -37.769630432128906,
      "activations/layer11_attention_weight_max": 40.66716766357422,
      "activations/layer11_attention_weight_min": -37.4158935546875,
      "activations/layer12_attention_weight_max": 28.932456970214844,
      "activations/layer12_attention_weight_min": -26.236249923706055,
      "activations/layer13_attention_weight_max": 38.735958099365234,
      "activations/layer13_attention_weight_min": -33.59767150878906,
      "activations/layer14_attention_weight_max": 40.99634552001953,
      "activations/layer14_attention_weight_min": -36.109771728515625,
      "activations/layer15_attention_weight_max": 39.04658508300781,
      "activations/layer15_attention_weight_min": -36.18741989135742,
      "activations/layer16_attention_weight_max": 31.61351776123047,
      "activations/layer16_attention_weight_min": -27.1539363861084,
      "activations/layer17_attention_weight_max": 27.717039108276367,
      "activations/layer17_attention_weight_min": -25.044645309448242,
      "activations/layer18_attention_weight_max": 28.443449020385742,
      "activations/layer18_attention_weight_min": -25.632396697998047,
      "activations/layer19_attention_weight_max": 33.645729064941406,
      "activations/layer19_attention_weight_min": -31.363819122314453,
      "activations/layer1_attention_weight_max": 16.873291015625,
      "activations/layer1_attention_weight_min": -16.124025344848633,
      "activations/layer20_attention_weight_max": 27.76953125,
      "activations/layer20_attention_weight_min": -26.21349334716797,
      "activations/layer21_attention_weight_max": 26.911907196044922,
      "activations/layer21_attention_weight_min": -25.055450439453125,
      "activations/layer22_attention_weight_max": 39.835853576660156,
      "activations/layer22_attention_weight_min": -31.458621978759766,
      "activations/layer23_attention_weight_max": 32.18057632446289,
      "activations/layer23_attention_weight_min": -26.162654876708984,
      "activations/layer2_attention_weight_max": 35.739662170410156,
      "activations/layer2_attention_weight_min": -33.539581298828125,
      "activations/layer3_attention_weight_max": 102.12799835205078,
      "activations/layer3_attention_weight_min": -100.64868927001953,
      "activations/layer4_attention_weight_max": 114.10597229003906,
      "activations/layer4_attention_weight_min": -106.80224609375,
      "activations/layer5_attention_weight_max": 55.30573654174805,
      "activations/layer5_attention_weight_min": -60.12860107421875,
      "activations/layer6_attention_weight_max": 43.657432556152344,
      "activations/layer6_attention_weight_min": -45.96165466308594,
      "activations/layer7_attention_weight_max": 110.63306427001953,
      "activations/layer7_attention_weight_min": -97.56897735595703,
      "activations/layer8_attention_weight_max": 49.6291618347168,
      "activations/layer8_attention_weight_min": -45.566776275634766,
      "activations/layer9_attention_weight_max": 37.900917053222656,
      "activations/layer9_attention_weight_min": -38.07228088378906,
      "epoch": 15.91,
      "learning_rate": 4.788674242424241e-05,
      "loss": 2.7661,
      "step": 273800
    },
    {
      "activations/layer0_attention_weight_max": 15.50403118133545,
      "activations/layer0_attention_weight_min": -12.371733665466309,
      "activations/layer10_attention_weight_max": 31.8314208984375,
      "activations/layer10_attention_weight_min": -33.965248107910156,
      "activations/layer11_attention_weight_max": 33.66669845581055,
      "activations/layer11_attention_weight_min": -34.62434387207031,
      "activations/layer12_attention_weight_max": 27.395021438598633,
      "activations/layer12_attention_weight_min": -26.75713539123535,
      "activations/layer13_attention_weight_max": 43.522743225097656,
      "activations/layer13_attention_weight_min": -38.05073928833008,
      "activations/layer14_attention_weight_max": 44.836524963378906,
      "activations/layer14_attention_weight_min": -38.75590515136719,
      "activations/layer15_attention_weight_max": 42.33585739135742,
      "activations/layer15_attention_weight_min": -37.0922737121582,
      "activations/layer16_attention_weight_max": 30.504276275634766,
      "activations/layer16_attention_weight_min": -28.873760223388672,
      "activations/layer17_attention_weight_max": 30.20167350769043,
      "activations/layer17_attention_weight_min": -27.233070373535156,
      "activations/layer18_attention_weight_max": 30.641206741333008,
      "activations/layer18_attention_weight_min": -25.87582778930664,
      "activations/layer19_attention_weight_max": 35.22666549682617,
      "activations/layer19_attention_weight_min": -31.117630004882812,
      "activations/layer1_attention_weight_max": 15.352238655090332,
      "activations/layer1_attention_weight_min": -15.743864059448242,
      "activations/layer20_attention_weight_max": 28.84886360168457,
      "activations/layer20_attention_weight_min": -25.173704147338867,
      "activations/layer21_attention_weight_max": 28.298870086669922,
      "activations/layer21_attention_weight_min": -24.76641845703125,
      "activations/layer22_attention_weight_max": 38.12272262573242,
      "activations/layer22_attention_weight_min": -30.805681228637695,
      "activations/layer23_attention_weight_max": 28.50404930114746,
      "activations/layer23_attention_weight_min": -26.685020446777344,
      "activations/layer2_attention_weight_max": 33.39745330810547,
      "activations/layer2_attention_weight_min": -31.86017608642578,
      "activations/layer3_attention_weight_max": 99.75695037841797,
      "activations/layer3_attention_weight_min": -94.8746566772461,
      "activations/layer4_attention_weight_max": 109.10920715332031,
      "activations/layer4_attention_weight_min": -105.51545715332031,
      "activations/layer5_attention_weight_max": 50.30707550048828,
      "activations/layer5_attention_weight_min": -61.13973617553711,
      "activations/layer6_attention_weight_max": 43.695980072021484,
      "activations/layer6_attention_weight_min": -44.27901077270508,
      "activations/layer7_attention_weight_max": 89.40145874023438,
      "activations/layer7_attention_weight_min": -88.78978729248047,
      "activations/layer8_attention_weight_max": 39.97376251220703,
      "activations/layer8_attention_weight_min": -43.763919830322266,
      "activations/layer9_attention_weight_max": 30.190753936767578,
      "activations/layer9_attention_weight_min": -32.88890838623047,
      "epoch": 15.91,
      "learning_rate": 4.786780303030303e-05,
      "loss": 2.7508,
      "step": 273850
    },
    {
      "activations/layer0_attention_weight_max": 15.471039772033691,
      "activations/layer0_attention_weight_min": -12.77299690246582,
      "activations/layer10_attention_weight_max": 38.06145095825195,
      "activations/layer10_attention_weight_min": -35.440975189208984,
      "activations/layer11_attention_weight_max": 36.98754119873047,
      "activations/layer11_attention_weight_min": -36.44468688964844,
      "activations/layer12_attention_weight_max": 28.50853157043457,
      "activations/layer12_attention_weight_min": -26.692684173583984,
      "activations/layer13_attention_weight_max": 42.625099182128906,
      "activations/layer13_attention_weight_min": -35.403907775878906,
      "activations/layer14_attention_weight_max": 45.562110900878906,
      "activations/layer14_attention_weight_min": -38.75174331665039,
      "activations/layer15_attention_weight_max": 40.711158752441406,
      "activations/layer15_attention_weight_min": -35.73728561401367,
      "activations/layer16_attention_weight_max": 33.4908447265625,
      "activations/layer16_attention_weight_min": -28.19925880432129,
      "activations/layer17_attention_weight_max": 33.755210876464844,
      "activations/layer17_attention_weight_min": -25.88912582397461,
      "activations/layer18_attention_weight_max": 30.91303825378418,
      "activations/layer18_attention_weight_min": -26.375831604003906,
      "activations/layer19_attention_weight_max": 32.931182861328125,
      "activations/layer19_attention_weight_min": -34.671932220458984,
      "activations/layer1_attention_weight_max": 16.27265739440918,
      "activations/layer1_attention_weight_min": -15.461533546447754,
      "activations/layer20_attention_weight_max": 31.232336044311523,
      "activations/layer20_attention_weight_min": -27.098281860351562,
      "activations/layer21_attention_weight_max": 29.43174171447754,
      "activations/layer21_attention_weight_min": -27.59874725341797,
      "activations/layer22_attention_weight_max": 45.891571044921875,
      "activations/layer22_attention_weight_min": -34.67278289794922,
      "activations/layer23_attention_weight_max": 33.498897552490234,
      "activations/layer23_attention_weight_min": -27.19351577758789,
      "activations/layer2_attention_weight_max": 34.73064041137695,
      "activations/layer2_attention_weight_min": -35.81072235107422,
      "activations/layer3_attention_weight_max": 101.17202758789062,
      "activations/layer3_attention_weight_min": -102.66848754882812,
      "activations/layer4_attention_weight_max": 109.17451477050781,
      "activations/layer4_attention_weight_min": -109.12383270263672,
      "activations/layer5_attention_weight_max": 50.399810791015625,
      "activations/layer5_attention_weight_min": -60.198646545410156,
      "activations/layer6_attention_weight_max": 42.68693542480469,
      "activations/layer6_attention_weight_min": -44.288108825683594,
      "activations/layer7_attention_weight_max": 103.42032623291016,
      "activations/layer7_attention_weight_min": -91.9273452758789,
      "activations/layer8_attention_weight_max": 47.88994216918945,
      "activations/layer8_attention_weight_min": -41.665985107421875,
      "activations/layer9_attention_weight_max": 34.56411361694336,
      "activations/layer9_attention_weight_min": -35.22241973876953,
      "epoch": 15.92,
      "learning_rate": 4.784886363636363e-05,
      "loss": 2.742,
      "step": 273900
    },
    {
      "activations/layer0_attention_weight_max": 15.923796653747559,
      "activations/layer0_attention_weight_min": -13.597429275512695,
      "activations/layer10_attention_weight_max": 32.5883674621582,
      "activations/layer10_attention_weight_min": -34.078948974609375,
      "activations/layer11_attention_weight_max": 36.258853912353516,
      "activations/layer11_attention_weight_min": -33.98965072631836,
      "activations/layer12_attention_weight_max": 28.63082504272461,
      "activations/layer12_attention_weight_min": -26.971193313598633,
      "activations/layer13_attention_weight_max": 42.91458511352539,
      "activations/layer13_attention_weight_min": -34.6796760559082,
      "activations/layer14_attention_weight_max": 45.20612716674805,
      "activations/layer14_attention_weight_min": -36.35823059082031,
      "activations/layer15_attention_weight_max": 40.12001037597656,
      "activations/layer15_attention_weight_min": -34.495323181152344,
      "activations/layer16_attention_weight_max": 30.421003341674805,
      "activations/layer16_attention_weight_min": -27.822998046875,
      "activations/layer17_attention_weight_max": 32.518577575683594,
      "activations/layer17_attention_weight_min": -24.787111282348633,
      "activations/layer18_attention_weight_max": 35.39657974243164,
      "activations/layer18_attention_weight_min": -25.146095275878906,
      "activations/layer19_attention_weight_max": 36.810516357421875,
      "activations/layer19_attention_weight_min": -29.948795318603516,
      "activations/layer1_attention_weight_max": 16.800186157226562,
      "activations/layer1_attention_weight_min": -15.740638732910156,
      "activations/layer20_attention_weight_max": 31.73647689819336,
      "activations/layer20_attention_weight_min": -24.774229049682617,
      "activations/layer21_attention_weight_max": 28.75542449951172,
      "activations/layer21_attention_weight_min": -24.73814582824707,
      "activations/layer22_attention_weight_max": 49.632568359375,
      "activations/layer22_attention_weight_min": -30.18454360961914,
      "activations/layer23_attention_weight_max": 35.33681869506836,
      "activations/layer23_attention_weight_min": -24.718191146850586,
      "activations/layer2_attention_weight_max": 32.984683990478516,
      "activations/layer2_attention_weight_min": -34.058509826660156,
      "activations/layer3_attention_weight_max": 101.06060028076172,
      "activations/layer3_attention_weight_min": -100.36221313476562,
      "activations/layer4_attention_weight_max": 108.5807876586914,
      "activations/layer4_attention_weight_min": -111.030517578125,
      "activations/layer5_attention_weight_max": 52.414588928222656,
      "activations/layer5_attention_weight_min": -59.24114227294922,
      "activations/layer6_attention_weight_max": 43.546775817871094,
      "activations/layer6_attention_weight_min": -46.51521682739258,
      "activations/layer7_attention_weight_max": 89.08781433105469,
      "activations/layer7_attention_weight_min": -93.76988983154297,
      "activations/layer8_attention_weight_max": 41.67925262451172,
      "activations/layer8_attention_weight_min": -43.715450286865234,
      "activations/layer9_attention_weight_max": 32.465938568115234,
      "activations/layer9_attention_weight_min": -34.41900634765625,
      "epoch": 15.92,
      "learning_rate": 4.782992424242424e-05,
      "loss": 2.7484,
      "step": 273950
    },
    {
      "activations/layer0_attention_weight_max": 16.07464027404785,
      "activations/layer0_attention_weight_min": -13.039446830749512,
      "activations/layer10_attention_weight_max": 32.775474548339844,
      "activations/layer10_attention_weight_min": -34.957489013671875,
      "activations/layer11_attention_weight_max": 32.293174743652344,
      "activations/layer11_attention_weight_min": -33.80610656738281,
      "activations/layer12_attention_weight_max": 29.707141876220703,
      "activations/layer12_attention_weight_min": -25.91486167907715,
      "activations/layer13_attention_weight_max": 39.6624870300293,
      "activations/layer13_attention_weight_min": -35.80928039550781,
      "activations/layer14_attention_weight_max": 43.78213119506836,
      "activations/layer14_attention_weight_min": -38.33898162841797,
      "activations/layer15_attention_weight_max": 37.695255279541016,
      "activations/layer15_attention_weight_min": -33.712013244628906,
      "activations/layer16_attention_weight_max": 32.10852813720703,
      "activations/layer16_attention_weight_min": -28.879230499267578,
      "activations/layer17_attention_weight_max": 30.193742752075195,
      "activations/layer17_attention_weight_min": -27.355220794677734,
      "activations/layer18_attention_weight_max": 33.36420822143555,
      "activations/layer18_attention_weight_min": -27.013286590576172,
      "activations/layer19_attention_weight_max": 35.332366943359375,
      "activations/layer19_attention_weight_min": -33.12444305419922,
      "activations/layer1_attention_weight_max": 16.001123428344727,
      "activations/layer1_attention_weight_min": -15.308457374572754,
      "activations/layer20_attention_weight_max": 30.800853729248047,
      "activations/layer20_attention_weight_min": -26.793800354003906,
      "activations/layer21_attention_weight_max": 29.151926040649414,
      "activations/layer21_attention_weight_min": -24.487096786499023,
      "activations/layer22_attention_weight_max": 40.355979919433594,
      "activations/layer22_attention_weight_min": -29.810314178466797,
      "activations/layer23_attention_weight_max": 32.20607376098633,
      "activations/layer23_attention_weight_min": -28.966129302978516,
      "activations/layer2_attention_weight_max": 32.51129913330078,
      "activations/layer2_attention_weight_min": -32.79106903076172,
      "activations/layer3_attention_weight_max": 98.58524322509766,
      "activations/layer3_attention_weight_min": -100.24007415771484,
      "activations/layer4_attention_weight_max": 107.68896484375,
      "activations/layer4_attention_weight_min": -112.20816802978516,
      "activations/layer5_attention_weight_max": 50.54493713378906,
      "activations/layer5_attention_weight_min": -64.76681518554688,
      "activations/layer6_attention_weight_max": 44.13318634033203,
      "activations/layer6_attention_weight_min": -48.08090591430664,
      "activations/layer7_attention_weight_max": 95.29730224609375,
      "activations/layer7_attention_weight_min": -94.27822875976562,
      "activations/layer8_attention_weight_max": 40.15773391723633,
      "activations/layer8_attention_weight_min": -43.29707336425781,
      "activations/layer9_attention_weight_max": 32.6571159362793,
      "activations/layer9_attention_weight_min": -33.1800651550293,
      "epoch": 15.92,
      "learning_rate": 4.7810984848484846e-05,
      "loss": 2.7467,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4752,
      "eval_samples_per_second": 506.655,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4752,
      "eval_openwebtext_samples_per_second": 506.655,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9549,
      "eval_wikitext_samples_per_second": 233.262,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.5396,
      "eval_lambada_samples_per_second": 510.397,
      "step": 274000
    },
    {
      "activations/layer0_attention_weight_max": 16.09853744506836,
      "activations/layer0_attention_weight_min": -13.482994079589844,
      "activations/layer10_attention_weight_max": 33.1503791809082,
      "activations/layer10_attention_weight_min": -32.769187927246094,
      "activations/layer11_attention_weight_max": 34.158905029296875,
      "activations/layer11_attention_weight_min": -32.908302307128906,
      "activations/layer12_attention_weight_max": 23.87374496459961,
      "activations/layer12_attention_weight_min": -26.745328903198242,
      "activations/layer13_attention_weight_max": 35.45565414428711,
      "activations/layer13_attention_weight_min": -35.58506393432617,
      "activations/layer14_attention_weight_max": 42.050132751464844,
      "activations/layer14_attention_weight_min": -38.84614944458008,
      "activations/layer15_attention_weight_max": 35.41549301147461,
      "activations/layer15_attention_weight_min": -32.571224212646484,
      "activations/layer16_attention_weight_max": 30.844507217407227,
      "activations/layer16_attention_weight_min": -27.990930557250977,
      "activations/layer17_attention_weight_max": 31.577816009521484,
      "activations/layer17_attention_weight_min": -24.47975730895996,
      "activations/layer18_attention_weight_max": 32.27119064331055,
      "activations/layer18_attention_weight_min": -23.785518646240234,
      "activations/layer19_attention_weight_max": 37.60602951049805,
      "activations/layer19_attention_weight_min": -30.424625396728516,
      "activations/layer1_attention_weight_max": 17.100008010864258,
      "activations/layer1_attention_weight_min": -14.383687973022461,
      "activations/layer20_attention_weight_max": 30.971351623535156,
      "activations/layer20_attention_weight_min": -24.29368019104004,
      "activations/layer21_attention_weight_max": 28.76983642578125,
      "activations/layer21_attention_weight_min": -25.273393630981445,
      "activations/layer22_attention_weight_max": 45.57999801635742,
      "activations/layer22_attention_weight_min": -30.597761154174805,
      "activations/layer23_attention_weight_max": 38.23020553588867,
      "activations/layer23_attention_weight_min": -28.171451568603516,
      "activations/layer2_attention_weight_max": 34.199134826660156,
      "activations/layer2_attention_weight_min": -33.872657775878906,
      "activations/layer3_attention_weight_max": 99.09717559814453,
      "activations/layer3_attention_weight_min": -95.74312591552734,
      "activations/layer4_attention_weight_max": 107.29972076416016,
      "activations/layer4_attention_weight_min": -106.82128143310547,
      "activations/layer5_attention_weight_max": 51.37453842163086,
      "activations/layer5_attention_weight_min": -63.04759979248047,
      "activations/layer6_attention_weight_max": 43.17345428466797,
      "activations/layer6_attention_weight_min": -45.55434036254883,
      "activations/layer7_attention_weight_max": 88.32266998291016,
      "activations/layer7_attention_weight_min": -90.61044311523438,
      "activations/layer8_attention_weight_max": 39.72932052612305,
      "activations/layer8_attention_weight_min": -42.073673248291016,
      "activations/layer9_attention_weight_max": 30.147825241088867,
      "activations/layer9_attention_weight_min": -32.56989669799805,
      "epoch": 15.92,
      "learning_rate": 4.7792045454545454e-05,
      "loss": 2.749,
      "step": 274050
    },
    {
      "activations/layer0_attention_weight_max": 15.87936019897461,
      "activations/layer0_attention_weight_min": -12.521459579467773,
      "activations/layer10_attention_weight_max": 32.87268829345703,
      "activations/layer10_attention_weight_min": -32.772281646728516,
      "activations/layer11_attention_weight_max": 33.250572204589844,
      "activations/layer11_attention_weight_min": -33.076107025146484,
      "activations/layer12_attention_weight_max": 24.35833740234375,
      "activations/layer12_attention_weight_min": -27.364917755126953,
      "activations/layer13_attention_weight_max": 36.807701110839844,
      "activations/layer13_attention_weight_min": -39.36534881591797,
      "activations/layer14_attention_weight_max": 42.10737228393555,
      "activations/layer14_attention_weight_min": -39.34748458862305,
      "activations/layer15_attention_weight_max": 36.83787536621094,
      "activations/layer15_attention_weight_min": -36.06953048706055,
      "activations/layer16_attention_weight_max": 30.985794067382812,
      "activations/layer16_attention_weight_min": -34.184730529785156,
      "activations/layer17_attention_weight_max": 29.43699073791504,
      "activations/layer17_attention_weight_min": -27.083538055419922,
      "activations/layer18_attention_weight_max": 35.17396545410156,
      "activations/layer18_attention_weight_min": -27.172977447509766,
      "activations/layer19_attention_weight_max": 39.79055404663086,
      "activations/layer19_attention_weight_min": -33.38951873779297,
      "activations/layer1_attention_weight_max": 16.54899787902832,
      "activations/layer1_attention_weight_min": -14.56213092803955,
      "activations/layer20_attention_weight_max": 37.88233184814453,
      "activations/layer20_attention_weight_min": -26.80489730834961,
      "activations/layer21_attention_weight_max": 32.52679443359375,
      "activations/layer21_attention_weight_min": -24.81148338317871,
      "activations/layer22_attention_weight_max": 44.74285888671875,
      "activations/layer22_attention_weight_min": -35.01433181762695,
      "activations/layer23_attention_weight_max": 34.99814224243164,
      "activations/layer23_attention_weight_min": -26.753324508666992,
      "activations/layer2_attention_weight_max": 32.52006912231445,
      "activations/layer2_attention_weight_min": -33.16007995605469,
      "activations/layer3_attention_weight_max": 94.90731811523438,
      "activations/layer3_attention_weight_min": -95.6123275756836,
      "activations/layer4_attention_weight_max": 105.17791748046875,
      "activations/layer4_attention_weight_min": -107.70206451416016,
      "activations/layer5_attention_weight_max": 51.3548698425293,
      "activations/layer5_attention_weight_min": -61.30232238769531,
      "activations/layer6_attention_weight_max": 43.11074447631836,
      "activations/layer6_attention_weight_min": -45.430301666259766,
      "activations/layer7_attention_weight_max": 99.76658630371094,
      "activations/layer7_attention_weight_min": -92.42013549804688,
      "activations/layer8_attention_weight_max": 40.10068130493164,
      "activations/layer8_attention_weight_min": -44.864418029785156,
      "activations/layer9_attention_weight_max": 30.04638671875,
      "activations/layer9_attention_weight_min": -34.00325393676758,
      "epoch": 15.93,
      "learning_rate": 4.7773106060606056e-05,
      "loss": 2.7373,
      "step": 274100
    },
    {
      "activations/layer0_attention_weight_max": 15.874073028564453,
      "activations/layer0_attention_weight_min": -12.275772094726562,
      "activations/layer10_attention_weight_max": 37.802879333496094,
      "activations/layer10_attention_weight_min": -36.55792999267578,
      "activations/layer11_attention_weight_max": 37.277923583984375,
      "activations/layer11_attention_weight_min": -36.403968811035156,
      "activations/layer12_attention_weight_max": 30.536951065063477,
      "activations/layer12_attention_weight_min": -28.09642791748047,
      "activations/layer13_attention_weight_max": 44.819522857666016,
      "activations/layer13_attention_weight_min": -40.8626823425293,
      "activations/layer14_attention_weight_max": 54.13557815551758,
      "activations/layer14_attention_weight_min": -43.50862503051758,
      "activations/layer15_attention_weight_max": 46.698699951171875,
      "activations/layer15_attention_weight_min": -38.728912353515625,
      "activations/layer16_attention_weight_max": 34.9561767578125,
      "activations/layer16_attention_weight_min": -28.158016204833984,
      "activations/layer17_attention_weight_max": 31.573434829711914,
      "activations/layer17_attention_weight_min": -27.05936622619629,
      "activations/layer18_attention_weight_max": 30.94403076171875,
      "activations/layer18_attention_weight_min": -25.754119873046875,
      "activations/layer19_attention_weight_max": 34.15329360961914,
      "activations/layer19_attention_weight_min": -30.24906349182129,
      "activations/layer1_attention_weight_max": 16.162790298461914,
      "activations/layer1_attention_weight_min": -14.657037734985352,
      "activations/layer20_attention_weight_max": 30.51476287841797,
      "activations/layer20_attention_weight_min": -23.931835174560547,
      "activations/layer21_attention_weight_max": 25.954076766967773,
      "activations/layer21_attention_weight_min": -23.937681198120117,
      "activations/layer22_attention_weight_max": 44.674556732177734,
      "activations/layer22_attention_weight_min": -29.949739456176758,
      "activations/layer23_attention_weight_max": 34.043277740478516,
      "activations/layer23_attention_weight_min": -24.64411163330078,
      "activations/layer2_attention_weight_max": 34.059730529785156,
      "activations/layer2_attention_weight_min": -33.84009552001953,
      "activations/layer3_attention_weight_max": 96.31744384765625,
      "activations/layer3_attention_weight_min": -101.55921173095703,
      "activations/layer4_attention_weight_max": 107.16072082519531,
      "activations/layer4_attention_weight_min": -110.0003662109375,
      "activations/layer5_attention_weight_max": 53.58971405029297,
      "activations/layer5_attention_weight_min": -62.184967041015625,
      "activations/layer6_attention_weight_max": 42.88915252685547,
      "activations/layer6_attention_weight_min": -45.47139358520508,
      "activations/layer7_attention_weight_max": 91.13998413085938,
      "activations/layer7_attention_weight_min": -97.4239730834961,
      "activations/layer8_attention_weight_max": 41.81959915161133,
      "activations/layer8_attention_weight_min": -44.40943145751953,
      "activations/layer9_attention_weight_max": 36.46717071533203,
      "activations/layer9_attention_weight_min": -38.11225128173828,
      "epoch": 15.93,
      "learning_rate": 4.775416666666666e-05,
      "loss": 2.7529,
      "step": 274150
    },
    {
      "activations/layer0_attention_weight_max": 15.597427368164062,
      "activations/layer0_attention_weight_min": -12.765701293945312,
      "activations/layer10_attention_weight_max": 41.05130386352539,
      "activations/layer10_attention_weight_min": -36.08610916137695,
      "activations/layer11_attention_weight_max": 40.80950164794922,
      "activations/layer11_attention_weight_min": -38.385581970214844,
      "activations/layer12_attention_weight_max": 33.545555114746094,
      "activations/layer12_attention_weight_min": -27.288911819458008,
      "activations/layer13_attention_weight_max": 46.49119567871094,
      "activations/layer13_attention_weight_min": -34.872276306152344,
      "activations/layer14_attention_weight_max": 42.46485137939453,
      "activations/layer14_attention_weight_min": -38.42112350463867,
      "activations/layer15_attention_weight_max": 45.1700325012207,
      "activations/layer15_attention_weight_min": -36.637489318847656,
      "activations/layer16_attention_weight_max": 33.01710891723633,
      "activations/layer16_attention_weight_min": -28.667753219604492,
      "activations/layer17_attention_weight_max": 31.444421768188477,
      "activations/layer17_attention_weight_min": -25.76926040649414,
      "activations/layer18_attention_weight_max": 34.224571228027344,
      "activations/layer18_attention_weight_min": -24.568193435668945,
      "activations/layer19_attention_weight_max": 34.66972732543945,
      "activations/layer19_attention_weight_min": -32.982826232910156,
      "activations/layer1_attention_weight_max": 16.350482940673828,
      "activations/layer1_attention_weight_min": -14.863784790039062,
      "activations/layer20_attention_weight_max": 30.506168365478516,
      "activations/layer20_attention_weight_min": -27.173858642578125,
      "activations/layer21_attention_weight_max": 29.912288665771484,
      "activations/layer21_attention_weight_min": -25.316326141357422,
      "activations/layer22_attention_weight_max": 42.197265625,
      "activations/layer22_attention_weight_min": -34.18071365356445,
      "activations/layer23_attention_weight_max": 35.98912811279297,
      "activations/layer23_attention_weight_min": -28.14236831665039,
      "activations/layer2_attention_weight_max": 35.57326889038086,
      "activations/layer2_attention_weight_min": -32.65816879272461,
      "activations/layer3_attention_weight_max": 101.54364776611328,
      "activations/layer3_attention_weight_min": -103.55178833007812,
      "activations/layer4_attention_weight_max": 107.36730194091797,
      "activations/layer4_attention_weight_min": -109.40767669677734,
      "activations/layer5_attention_weight_max": 54.80555725097656,
      "activations/layer5_attention_weight_min": -62.70254135131836,
      "activations/layer6_attention_weight_max": 44.51958084106445,
      "activations/layer6_attention_weight_min": -47.973758697509766,
      "activations/layer7_attention_weight_max": 105.01300811767578,
      "activations/layer7_attention_weight_min": -93.08421325683594,
      "activations/layer8_attention_weight_max": 46.74177932739258,
      "activations/layer8_attention_weight_min": -41.731163024902344,
      "activations/layer9_attention_weight_max": 37.790794372558594,
      "activations/layer9_attention_weight_min": -36.27643966674805,
      "epoch": 15.93,
      "learning_rate": 4.773522727272727e-05,
      "loss": 2.7452,
      "step": 274200
    },
    {
      "activations/layer0_attention_weight_max": 15.39936637878418,
      "activations/layer0_attention_weight_min": -12.74248218536377,
      "activations/layer10_attention_weight_max": 31.92564582824707,
      "activations/layer10_attention_weight_min": -34.082252502441406,
      "activations/layer11_attention_weight_max": 32.91658401489258,
      "activations/layer11_attention_weight_min": -33.72407150268555,
      "activations/layer12_attention_weight_max": 27.089231491088867,
      "activations/layer12_attention_weight_min": -26.105592727661133,
      "activations/layer13_attention_weight_max": 39.347137451171875,
      "activations/layer13_attention_weight_min": -34.62052917480469,
      "activations/layer14_attention_weight_max": 43.0821647644043,
      "activations/layer14_attention_weight_min": -37.77696228027344,
      "activations/layer15_attention_weight_max": 39.82807159423828,
      "activations/layer15_attention_weight_min": -34.43986511230469,
      "activations/layer16_attention_weight_max": 31.330095291137695,
      "activations/layer16_attention_weight_min": -28.19527816772461,
      "activations/layer17_attention_weight_max": 32.45047378540039,
      "activations/layer17_attention_weight_min": -27.034379959106445,
      "activations/layer18_attention_weight_max": 31.327293395996094,
      "activations/layer18_attention_weight_min": -25.242816925048828,
      "activations/layer19_attention_weight_max": 38.04979705810547,
      "activations/layer19_attention_weight_min": -32.28505325317383,
      "activations/layer1_attention_weight_max": 16.54074478149414,
      "activations/layer1_attention_weight_min": -13.3034086227417,
      "activations/layer20_attention_weight_max": 30.299388885498047,
      "activations/layer20_attention_weight_min": -26.630781173706055,
      "activations/layer21_attention_weight_max": 28.67148780822754,
      "activations/layer21_attention_weight_min": -26.17146873474121,
      "activations/layer22_attention_weight_max": 39.68437957763672,
      "activations/layer22_attention_weight_min": -30.95681381225586,
      "activations/layer23_attention_weight_max": 34.52386474609375,
      "activations/layer23_attention_weight_min": -27.1423397064209,
      "activations/layer2_attention_weight_max": 33.35562515258789,
      "activations/layer2_attention_weight_min": -34.471656799316406,
      "activations/layer3_attention_weight_max": 99.56505584716797,
      "activations/layer3_attention_weight_min": -101.77093505859375,
      "activations/layer4_attention_weight_max": 111.8670883178711,
      "activations/layer4_attention_weight_min": -117.63020324707031,
      "activations/layer5_attention_weight_max": 52.78277587890625,
      "activations/layer5_attention_weight_min": -64.63920593261719,
      "activations/layer6_attention_weight_max": 45.85843276977539,
      "activations/layer6_attention_weight_min": -50.3746452331543,
      "activations/layer7_attention_weight_max": 91.33917999267578,
      "activations/layer7_attention_weight_min": -97.46446990966797,
      "activations/layer8_attention_weight_max": 42.46479415893555,
      "activations/layer8_attention_weight_min": -45.55140686035156,
      "activations/layer9_attention_weight_max": 31.54916763305664,
      "activations/layer9_attention_weight_min": -33.437896728515625,
      "epoch": 15.94,
      "learning_rate": 4.7716287878787874e-05,
      "loss": 2.7353,
      "step": 274250
    },
    {
      "activations/layer0_attention_weight_max": 15.938054084777832,
      "activations/layer0_attention_weight_min": -13.938334465026855,
      "activations/layer10_attention_weight_max": 37.22699737548828,
      "activations/layer10_attention_weight_min": -34.283199310302734,
      "activations/layer11_attention_weight_max": 36.79199981689453,
      "activations/layer11_attention_weight_min": -35.452919006347656,
      "activations/layer12_attention_weight_max": 26.9399471282959,
      "activations/layer12_attention_weight_min": -28.640348434448242,
      "activations/layer13_attention_weight_max": 39.90205764770508,
      "activations/layer13_attention_weight_min": -35.555667877197266,
      "activations/layer14_attention_weight_max": 45.85824203491211,
      "activations/layer14_attention_weight_min": -39.436248779296875,
      "activations/layer15_attention_weight_max": 39.22369384765625,
      "activations/layer15_attention_weight_min": -35.561161041259766,
      "activations/layer16_attention_weight_max": 30.09564781188965,
      "activations/layer16_attention_weight_min": -26.884851455688477,
      "activations/layer17_attention_weight_max": 33.46257400512695,
      "activations/layer17_attention_weight_min": -26.696025848388672,
      "activations/layer18_attention_weight_max": 32.982173919677734,
      "activations/layer18_attention_weight_min": -23.904199600219727,
      "activations/layer19_attention_weight_max": 35.85477828979492,
      "activations/layer19_attention_weight_min": -28.729894638061523,
      "activations/layer1_attention_weight_max": 16.405088424682617,
      "activations/layer1_attention_weight_min": -14.169642448425293,
      "activations/layer20_attention_weight_max": 28.572105407714844,
      "activations/layer20_attention_weight_min": -24.087160110473633,
      "activations/layer21_attention_weight_max": 28.14989471435547,
      "activations/layer21_attention_weight_min": -24.26727867126465,
      "activations/layer22_attention_weight_max": 41.62141036987305,
      "activations/layer22_attention_weight_min": -29.51209831237793,
      "activations/layer23_attention_weight_max": 30.951919555664062,
      "activations/layer23_attention_weight_min": -23.456172943115234,
      "activations/layer2_attention_weight_max": 33.533287048339844,
      "activations/layer2_attention_weight_min": -33.49485778808594,
      "activations/layer3_attention_weight_max": 98.6126708984375,
      "activations/layer3_attention_weight_min": -97.65448760986328,
      "activations/layer4_attention_weight_max": 108.4749984741211,
      "activations/layer4_attention_weight_min": -105.665283203125,
      "activations/layer5_attention_weight_max": 51.797760009765625,
      "activations/layer5_attention_weight_min": -60.1352653503418,
      "activations/layer6_attention_weight_max": 43.56632614135742,
      "activations/layer6_attention_weight_min": -44.039981842041016,
      "activations/layer7_attention_weight_max": 96.2353744506836,
      "activations/layer7_attention_weight_min": -95.0896224975586,
      "activations/layer8_attention_weight_max": 43.44392776489258,
      "activations/layer8_attention_weight_min": -43.69978713989258,
      "activations/layer9_attention_weight_max": 35.58091354370117,
      "activations/layer9_attention_weight_min": -34.88863754272461,
      "epoch": 15.94,
      "learning_rate": 4.769734848484848e-05,
      "loss": 2.7548,
      "step": 274300
    },
    {
      "activations/layer0_attention_weight_max": 15.839788436889648,
      "activations/layer0_attention_weight_min": -12.490819931030273,
      "activations/layer10_attention_weight_max": 32.02783203125,
      "activations/layer10_attention_weight_min": -33.3116455078125,
      "activations/layer11_attention_weight_max": 33.697235107421875,
      "activations/layer11_attention_weight_min": -35.155540466308594,
      "activations/layer12_attention_weight_max": 30.61862564086914,
      "activations/layer12_attention_weight_min": -26.256383895874023,
      "activations/layer13_attention_weight_max": 42.84822082519531,
      "activations/layer13_attention_weight_min": -35.872989654541016,
      "activations/layer14_attention_weight_max": 49.37223434448242,
      "activations/layer14_attention_weight_min": -38.35313415527344,
      "activations/layer15_attention_weight_max": 46.34373474121094,
      "activations/layer15_attention_weight_min": -35.625064849853516,
      "activations/layer16_attention_weight_max": 33.53757095336914,
      "activations/layer16_attention_weight_min": -28.11328887939453,
      "activations/layer17_attention_weight_max": 31.115604400634766,
      "activations/layer17_attention_weight_min": -26.683277130126953,
      "activations/layer18_attention_weight_max": 31.28056526184082,
      "activations/layer18_attention_weight_min": -25.6735782623291,
      "activations/layer19_attention_weight_max": 35.3552360534668,
      "activations/layer19_attention_weight_min": -31.720407485961914,
      "activations/layer1_attention_weight_max": 16.902490615844727,
      "activations/layer1_attention_weight_min": -15.155914306640625,
      "activations/layer20_attention_weight_max": 31.18962287902832,
      "activations/layer20_attention_weight_min": -25.55339813232422,
      "activations/layer21_attention_weight_max": 32.59624099731445,
      "activations/layer21_attention_weight_min": -25.549840927124023,
      "activations/layer22_attention_weight_max": 44.76435089111328,
      "activations/layer22_attention_weight_min": -32.30091857910156,
      "activations/layer23_attention_weight_max": 33.36943054199219,
      "activations/layer23_attention_weight_min": -25.639602661132812,
      "activations/layer2_attention_weight_max": 35.47043991088867,
      "activations/layer2_attention_weight_min": -34.00440216064453,
      "activations/layer3_attention_weight_max": 98.51669311523438,
      "activations/layer3_attention_weight_min": -103.31511688232422,
      "activations/layer4_attention_weight_max": 108.6931381225586,
      "activations/layer4_attention_weight_min": -110.11029052734375,
      "activations/layer5_attention_weight_max": 57.79682540893555,
      "activations/layer5_attention_weight_min": -61.921356201171875,
      "activations/layer6_attention_weight_max": 43.22329330444336,
      "activations/layer6_attention_weight_min": -44.63835525512695,
      "activations/layer7_attention_weight_max": 93.36939239501953,
      "activations/layer7_attention_weight_min": -91.62689208984375,
      "activations/layer8_attention_weight_max": 40.31047439575195,
      "activations/layer8_attention_weight_min": -41.795135498046875,
      "activations/layer9_attention_weight_max": 31.944839477539062,
      "activations/layer9_attention_weight_min": -34.66743087768555,
      "epoch": 15.94,
      "learning_rate": 4.7678409090909084e-05,
      "loss": 2.7442,
      "step": 274350
    },
    {
      "activations/layer0_attention_weight_max": 16.263975143432617,
      "activations/layer0_attention_weight_min": -12.519133567810059,
      "activations/layer10_attention_weight_max": 31.947954177856445,
      "activations/layer10_attention_weight_min": -33.29932403564453,
      "activations/layer11_attention_weight_max": 32.80850601196289,
      "activations/layer11_attention_weight_min": -32.86587142944336,
      "activations/layer12_attention_weight_max": 24.949689865112305,
      "activations/layer12_attention_weight_min": -26.169145584106445,
      "activations/layer13_attention_weight_max": 38.90791320800781,
      "activations/layer13_attention_weight_min": -32.65314865112305,
      "activations/layer14_attention_weight_max": 41.99164581298828,
      "activations/layer14_attention_weight_min": -36.16074752807617,
      "activations/layer15_attention_weight_max": 36.76737594604492,
      "activations/layer15_attention_weight_min": -33.80586242675781,
      "activations/layer16_attention_weight_max": 32.67805480957031,
      "activations/layer16_attention_weight_min": -28.056936264038086,
      "activations/layer17_attention_weight_max": 28.842248916625977,
      "activations/layer17_attention_weight_min": -25.708261489868164,
      "activations/layer18_attention_weight_max": 29.950538635253906,
      "activations/layer18_attention_weight_min": -24.656896591186523,
      "activations/layer19_attention_weight_max": 33.80178451538086,
      "activations/layer19_attention_weight_min": -30.87867546081543,
      "activations/layer1_attention_weight_max": 16.785503387451172,
      "activations/layer1_attention_weight_min": -14.714597702026367,
      "activations/layer20_attention_weight_max": 26.989511489868164,
      "activations/layer20_attention_weight_min": -23.966184616088867,
      "activations/layer21_attention_weight_max": 27.00589370727539,
      "activations/layer21_attention_weight_min": -22.874603271484375,
      "activations/layer22_attention_weight_max": 40.56282043457031,
      "activations/layer22_attention_weight_min": -30.181915283203125,
      "activations/layer23_attention_weight_max": 33.23047637939453,
      "activations/layer23_attention_weight_min": -25.356977462768555,
      "activations/layer2_attention_weight_max": 34.71440124511719,
      "activations/layer2_attention_weight_min": -34.24840545654297,
      "activations/layer3_attention_weight_max": 98.49606323242188,
      "activations/layer3_attention_weight_min": -96.51226806640625,
      "activations/layer4_attention_weight_max": 112.57384490966797,
      "activations/layer4_attention_weight_min": -105.48778533935547,
      "activations/layer5_attention_weight_max": 53.3438835144043,
      "activations/layer5_attention_weight_min": -62.560585021972656,
      "activations/layer6_attention_weight_max": 45.29462814331055,
      "activations/layer6_attention_weight_min": -45.24955749511719,
      "activations/layer7_attention_weight_max": 93.72602081298828,
      "activations/layer7_attention_weight_min": -89.45250701904297,
      "activations/layer8_attention_weight_max": 41.18343734741211,
      "activations/layer8_attention_weight_min": -42.1545295715332,
      "activations/layer9_attention_weight_max": 31.455068588256836,
      "activations/layer9_attention_weight_min": -31.485319137573242,
      "epoch": 15.94,
      "learning_rate": 4.76594696969697e-05,
      "loss": 2.7441,
      "step": 274400
    },
    {
      "activations/layer0_attention_weight_max": 16.067829132080078,
      "activations/layer0_attention_weight_min": -13.442648887634277,
      "activations/layer10_attention_weight_max": 32.729862213134766,
      "activations/layer10_attention_weight_min": -32.02383041381836,
      "activations/layer11_attention_weight_max": 32.61835479736328,
      "activations/layer11_attention_weight_min": -34.19359588623047,
      "activations/layer12_attention_weight_max": 28.783496856689453,
      "activations/layer12_attention_weight_min": -26.26837921142578,
      "activations/layer13_attention_weight_max": 43.56916809082031,
      "activations/layer13_attention_weight_min": -35.39374542236328,
      "activations/layer14_attention_weight_max": 49.43417739868164,
      "activations/layer14_attention_weight_min": -39.47877883911133,
      "activations/layer15_attention_weight_max": 39.9052619934082,
      "activations/layer15_attention_weight_min": -34.85177230834961,
      "activations/layer16_attention_weight_max": 33.9322395324707,
      "activations/layer16_attention_weight_min": -28.199987411499023,
      "activations/layer17_attention_weight_max": 32.35158157348633,
      "activations/layer17_attention_weight_min": -28.542009353637695,
      "activations/layer18_attention_weight_max": 34.761722564697266,
      "activations/layer18_attention_weight_min": -25.21883201599121,
      "activations/layer19_attention_weight_max": 36.95729064941406,
      "activations/layer19_attention_weight_min": -32.512603759765625,
      "activations/layer1_attention_weight_max": 15.670881271362305,
      "activations/layer1_attention_weight_min": -15.135172843933105,
      "activations/layer20_attention_weight_max": 33.69453430175781,
      "activations/layer20_attention_weight_min": -25.25487518310547,
      "activations/layer21_attention_weight_max": 31.706655502319336,
      "activations/layer21_attention_weight_min": -26.445804595947266,
      "activations/layer22_attention_weight_max": 44.7423210144043,
      "activations/layer22_attention_weight_min": -29.567623138427734,
      "activations/layer23_attention_weight_max": 36.381229400634766,
      "activations/layer23_attention_weight_min": -25.361446380615234,
      "activations/layer2_attention_weight_max": 35.57464599609375,
      "activations/layer2_attention_weight_min": -33.74794006347656,
      "activations/layer3_attention_weight_max": 100.2699203491211,
      "activations/layer3_attention_weight_min": -99.91524505615234,
      "activations/layer4_attention_weight_max": 110.78192138671875,
      "activations/layer4_attention_weight_min": -113.21065521240234,
      "activations/layer5_attention_weight_max": 51.0108642578125,
      "activations/layer5_attention_weight_min": -63.13847351074219,
      "activations/layer6_attention_weight_max": 44.44755935668945,
      "activations/layer6_attention_weight_min": -49.881675720214844,
      "activations/layer7_attention_weight_max": 94.60505676269531,
      "activations/layer7_attention_weight_min": -97.63121032714844,
      "activations/layer8_attention_weight_max": 41.35399627685547,
      "activations/layer8_attention_weight_min": -42.38956832885742,
      "activations/layer9_attention_weight_max": 32.72301483154297,
      "activations/layer9_attention_weight_min": -32.36098861694336,
      "epoch": 15.95,
      "learning_rate": 4.76405303030303e-05,
      "loss": 2.7431,
      "step": 274450
    },
    {
      "activations/layer0_attention_weight_max": 16.2342472076416,
      "activations/layer0_attention_weight_min": -13.067256927490234,
      "activations/layer10_attention_weight_max": 48.9362907409668,
      "activations/layer10_attention_weight_min": -47.76520919799805,
      "activations/layer11_attention_weight_max": 47.74177551269531,
      "activations/layer11_attention_weight_min": -48.886070251464844,
      "activations/layer12_attention_weight_max": 32.5385627746582,
      "activations/layer12_attention_weight_min": -30.398387908935547,
      "activations/layer13_attention_weight_max": 46.483394622802734,
      "activations/layer13_attention_weight_min": -38.47660827636719,
      "activations/layer14_attention_weight_max": 58.280887603759766,
      "activations/layer14_attention_weight_min": -42.641761779785156,
      "activations/layer15_attention_weight_max": 58.75643539428711,
      "activations/layer15_attention_weight_min": -41.0030403137207,
      "activations/layer16_attention_weight_max": 36.38404083251953,
      "activations/layer16_attention_weight_min": -29.557870864868164,
      "activations/layer17_attention_weight_max": 34.24948501586914,
      "activations/layer17_attention_weight_min": -27.587833404541016,
      "activations/layer18_attention_weight_max": 32.11701583862305,
      "activations/layer18_attention_weight_min": -25.388322830200195,
      "activations/layer19_attention_weight_max": 36.148597717285156,
      "activations/layer19_attention_weight_min": -34.24286651611328,
      "activations/layer1_attention_weight_max": 18.82889747619629,
      "activations/layer1_attention_weight_min": -15.583741188049316,
      "activations/layer20_attention_weight_max": 29.493213653564453,
      "activations/layer20_attention_weight_min": -26.11394691467285,
      "activations/layer21_attention_weight_max": 28.04193115234375,
      "activations/layer21_attention_weight_min": -25.438657760620117,
      "activations/layer22_attention_weight_max": 47.89657974243164,
      "activations/layer22_attention_weight_min": -32.65180969238281,
      "activations/layer23_attention_weight_max": 33.93753433227539,
      "activations/layer23_attention_weight_min": -27.500547409057617,
      "activations/layer2_attention_weight_max": 36.26971435546875,
      "activations/layer2_attention_weight_min": -36.682640075683594,
      "activations/layer3_attention_weight_max": 102.02020263671875,
      "activations/layer3_attention_weight_min": -104.22982025146484,
      "activations/layer4_attention_weight_max": 111.59529876708984,
      "activations/layer4_attention_weight_min": -109.39885711669922,
      "activations/layer5_attention_weight_max": 54.545528411865234,
      "activations/layer5_attention_weight_min": -60.71887969970703,
      "activations/layer6_attention_weight_max": 52.60811233520508,
      "activations/layer6_attention_weight_min": -50.93907165527344,
      "activations/layer7_attention_weight_max": 116.7204360961914,
      "activations/layer7_attention_weight_min": -107.80960083007812,
      "activations/layer8_attention_weight_max": 55.55006790161133,
      "activations/layer8_attention_weight_min": -54.90083312988281,
      "activations/layer9_attention_weight_max": 44.3565788269043,
      "activations/layer9_attention_weight_min": -43.961700439453125,
      "epoch": 15.95,
      "learning_rate": 4.76215909090909e-05,
      "loss": 2.7447,
      "step": 274500
    },
    {
      "activations/layer0_attention_weight_max": 15.656927108764648,
      "activations/layer0_attention_weight_min": -12.356621742248535,
      "activations/layer10_attention_weight_max": 33.13899612426758,
      "activations/layer10_attention_weight_min": -33.645904541015625,
      "activations/layer11_attention_weight_max": 34.3037109375,
      "activations/layer11_attention_weight_min": -32.54951858520508,
      "activations/layer12_attention_weight_max": 26.784822463989258,
      "activations/layer12_attention_weight_min": -27.101144790649414,
      "activations/layer13_attention_weight_max": 41.30354309082031,
      "activations/layer13_attention_weight_min": -34.252166748046875,
      "activations/layer14_attention_weight_max": 44.48310470581055,
      "activations/layer14_attention_weight_min": -37.09532928466797,
      "activations/layer15_attention_weight_max": 40.46937942504883,
      "activations/layer15_attention_weight_min": -33.669551849365234,
      "activations/layer16_attention_weight_max": 31.613712310791016,
      "activations/layer16_attention_weight_min": -27.882871627807617,
      "activations/layer17_attention_weight_max": 31.853750228881836,
      "activations/layer17_attention_weight_min": -25.527420043945312,
      "activations/layer18_attention_weight_max": 32.65189743041992,
      "activations/layer18_attention_weight_min": -25.749996185302734,
      "activations/layer19_attention_weight_max": 37.31819152832031,
      "activations/layer19_attention_weight_min": -32.45137023925781,
      "activations/layer1_attention_weight_max": 16.481460571289062,
      "activations/layer1_attention_weight_min": -14.223564147949219,
      "activations/layer20_attention_weight_max": 32.10390090942383,
      "activations/layer20_attention_weight_min": -27.477537155151367,
      "activations/layer21_attention_weight_max": 30.743999481201172,
      "activations/layer21_attention_weight_min": -27.060338973999023,
      "activations/layer22_attention_weight_max": 47.794010162353516,
      "activations/layer22_attention_weight_min": -33.929683685302734,
      "activations/layer23_attention_weight_max": 35.05574035644531,
      "activations/layer23_attention_weight_min": -28.337509155273438,
      "activations/layer2_attention_weight_max": 32.716949462890625,
      "activations/layer2_attention_weight_min": -34.1728630065918,
      "activations/layer3_attention_weight_max": 94.66602325439453,
      "activations/layer3_attention_weight_min": -97.55911254882812,
      "activations/layer4_attention_weight_max": 101.34886932373047,
      "activations/layer4_attention_weight_min": -106.16497039794922,
      "activations/layer5_attention_weight_max": 47.65278244018555,
      "activations/layer5_attention_weight_min": -59.53178405761719,
      "activations/layer6_attention_weight_max": 42.59583282470703,
      "activations/layer6_attention_weight_min": -45.177608489990234,
      "activations/layer7_attention_weight_max": 91.77490234375,
      "activations/layer7_attention_weight_min": -91.54264831542969,
      "activations/layer8_attention_weight_max": 42.44407272338867,
      "activations/layer8_attention_weight_min": -43.74831008911133,
      "activations/layer9_attention_weight_max": 31.244680404663086,
      "activations/layer9_attention_weight_min": -33.334503173828125,
      "epoch": 15.95,
      "learning_rate": 4.760265151515151e-05,
      "loss": 2.7451,
      "step": 274550
    },
    {
      "activations/layer0_attention_weight_max": 16.35744285583496,
      "activations/layer0_attention_weight_min": -12.438486099243164,
      "activations/layer10_attention_weight_max": 36.93796920776367,
      "activations/layer10_attention_weight_min": -35.11146545410156,
      "activations/layer11_attention_weight_max": 36.32257080078125,
      "activations/layer11_attention_weight_min": -37.07292175292969,
      "activations/layer12_attention_weight_max": 38.91999816894531,
      "activations/layer12_attention_weight_min": -27.228477478027344,
      "activations/layer13_attention_weight_max": 45.27085876464844,
      "activations/layer13_attention_weight_min": -37.36542892456055,
      "activations/layer14_attention_weight_max": 42.47556686401367,
      "activations/layer14_attention_weight_min": -37.76897048950195,
      "activations/layer15_attention_weight_max": 40.0989990234375,
      "activations/layer15_attention_weight_min": -35.4031867980957,
      "activations/layer16_attention_weight_max": 29.964303970336914,
      "activations/layer16_attention_weight_min": -28.705095291137695,
      "activations/layer17_attention_weight_max": 31.84364891052246,
      "activations/layer17_attention_weight_min": -26.555400848388672,
      "activations/layer18_attention_weight_max": 30.106210708618164,
      "activations/layer18_attention_weight_min": -28.866792678833008,
      "activations/layer19_attention_weight_max": 31.83349609375,
      "activations/layer19_attention_weight_min": -32.08625411987305,
      "activations/layer1_attention_weight_max": 17.099483489990234,
      "activations/layer1_attention_weight_min": -14.860647201538086,
      "activations/layer20_attention_weight_max": 28.60834312438965,
      "activations/layer20_attention_weight_min": -28.46100425720215,
      "activations/layer21_attention_weight_max": 28.112810134887695,
      "activations/layer21_attention_weight_min": -26.812358856201172,
      "activations/layer22_attention_weight_max": 46.698307037353516,
      "activations/layer22_attention_weight_min": -34.0229377746582,
      "activations/layer23_attention_weight_max": 33.4959716796875,
      "activations/layer23_attention_weight_min": -26.671871185302734,
      "activations/layer2_attention_weight_max": 34.467002868652344,
      "activations/layer2_attention_weight_min": -32.02339172363281,
      "activations/layer3_attention_weight_max": 102.3365707397461,
      "activations/layer3_attention_weight_min": -100.59761047363281,
      "activations/layer4_attention_weight_max": 107.63399505615234,
      "activations/layer4_attention_weight_min": -107.49446868896484,
      "activations/layer5_attention_weight_max": 52.55515670776367,
      "activations/layer5_attention_weight_min": -61.8383903503418,
      "activations/layer6_attention_weight_max": 46.22461700439453,
      "activations/layer6_attention_weight_min": -46.67277145385742,
      "activations/layer7_attention_weight_max": 98.18714141845703,
      "activations/layer7_attention_weight_min": -96.68016815185547,
      "activations/layer8_attention_weight_max": 42.819541931152344,
      "activations/layer8_attention_weight_min": -46.2009162902832,
      "activations/layer9_attention_weight_max": 35.055362701416016,
      "activations/layer9_attention_weight_min": -36.339744567871094,
      "epoch": 15.96,
      "learning_rate": 4.758371212121212e-05,
      "loss": 2.7586,
      "step": 274600
    },
    {
      "activations/layer0_attention_weight_max": 16.624935150146484,
      "activations/layer0_attention_weight_min": -13.062504768371582,
      "activations/layer10_attention_weight_max": 33.07518005371094,
      "activations/layer10_attention_weight_min": -33.537418365478516,
      "activations/layer11_attention_weight_max": 32.73008728027344,
      "activations/layer11_attention_weight_min": -34.45946502685547,
      "activations/layer12_attention_weight_max": 26.601213455200195,
      "activations/layer12_attention_weight_min": -29.078086853027344,
      "activations/layer13_attention_weight_max": 42.36559295654297,
      "activations/layer13_attention_weight_min": -36.69078826904297,
      "activations/layer14_attention_weight_max": 42.99895095825195,
      "activations/layer14_attention_weight_min": -41.59226989746094,
      "activations/layer15_attention_weight_max": 43.71305847167969,
      "activations/layer15_attention_weight_min": -35.43998718261719,
      "activations/layer16_attention_weight_max": 33.41094970703125,
      "activations/layer16_attention_weight_min": -30.214859008789062,
      "activations/layer17_attention_weight_max": 32.13895797729492,
      "activations/layer17_attention_weight_min": -27.353729248046875,
      "activations/layer18_attention_weight_max": 32.212886810302734,
      "activations/layer18_attention_weight_min": -24.143465042114258,
      "activations/layer19_attention_weight_max": 34.794090270996094,
      "activations/layer19_attention_weight_min": -31.066404342651367,
      "activations/layer1_attention_weight_max": 16.282962799072266,
      "activations/layer1_attention_weight_min": -15.464129447937012,
      "activations/layer20_attention_weight_max": 30.064897537231445,
      "activations/layer20_attention_weight_min": -23.344688415527344,
      "activations/layer21_attention_weight_max": 29.07695198059082,
      "activations/layer21_attention_weight_min": -23.227415084838867,
      "activations/layer22_attention_weight_max": 44.08342361450195,
      "activations/layer22_attention_weight_min": -31.289291381835938,
      "activations/layer23_attention_weight_max": 33.22271728515625,
      "activations/layer23_attention_weight_min": -26.65814971923828,
      "activations/layer2_attention_weight_max": 33.175071716308594,
      "activations/layer2_attention_weight_min": -32.579933166503906,
      "activations/layer3_attention_weight_max": 96.235595703125,
      "activations/layer3_attention_weight_min": -98.30897521972656,
      "activations/layer4_attention_weight_max": 107.1873550415039,
      "activations/layer4_attention_weight_min": -109.43804931640625,
      "activations/layer5_attention_weight_max": 51.40610885620117,
      "activations/layer5_attention_weight_min": -63.781978607177734,
      "activations/layer6_attention_weight_max": 44.47901153564453,
      "activations/layer6_attention_weight_min": -48.1627311706543,
      "activations/layer7_attention_weight_max": 93.9486312866211,
      "activations/layer7_attention_weight_min": -94.99466705322266,
      "activations/layer8_attention_weight_max": 39.90143966674805,
      "activations/layer8_attention_weight_min": -42.901546478271484,
      "activations/layer9_attention_weight_max": 32.82021713256836,
      "activations/layer9_attention_weight_min": -32.493778228759766,
      "epoch": 15.96,
      "learning_rate": 4.7564772727272727e-05,
      "loss": 2.7606,
      "step": 274650
    },
    {
      "activations/layer0_attention_weight_max": 15.165809631347656,
      "activations/layer0_attention_weight_min": -12.581972122192383,
      "activations/layer10_attention_weight_max": 31.16817283630371,
      "activations/layer10_attention_weight_min": -31.895263671875,
      "activations/layer11_attention_weight_max": 32.83439636230469,
      "activations/layer11_attention_weight_min": -30.928966522216797,
      "activations/layer12_attention_weight_max": 25.831575393676758,
      "activations/layer12_attention_weight_min": -25.99584197998047,
      "activations/layer13_attention_weight_max": 38.16544723510742,
      "activations/layer13_attention_weight_min": -37.61421585083008,
      "activations/layer14_attention_weight_max": 42.81128692626953,
      "activations/layer14_attention_weight_min": -39.731781005859375,
      "activations/layer15_attention_weight_max": 36.36652755737305,
      "activations/layer15_attention_weight_min": -34.55622100830078,
      "activations/layer16_attention_weight_max": 26.859987258911133,
      "activations/layer16_attention_weight_min": -27.598424911499023,
      "activations/layer17_attention_weight_max": 31.165283203125,
      "activations/layer17_attention_weight_min": -25.023653030395508,
      "activations/layer18_attention_weight_max": 30.9864444732666,
      "activations/layer18_attention_weight_min": -26.346445083618164,
      "activations/layer19_attention_weight_max": 33.078243255615234,
      "activations/layer19_attention_weight_min": -28.54926872253418,
      "activations/layer1_attention_weight_max": 18.2147159576416,
      "activations/layer1_attention_weight_min": -14.358931541442871,
      "activations/layer20_attention_weight_max": 28.931306838989258,
      "activations/layer20_attention_weight_min": -25.562328338623047,
      "activations/layer21_attention_weight_max": 31.91452980041504,
      "activations/layer21_attention_weight_min": -24.28473472595215,
      "activations/layer22_attention_weight_max": 41.2841911315918,
      "activations/layer22_attention_weight_min": -31.237613677978516,
      "activations/layer23_attention_weight_max": 35.15163040161133,
      "activations/layer23_attention_weight_min": -25.381620407104492,
      "activations/layer2_attention_weight_max": 31.08693504333496,
      "activations/layer2_attention_weight_min": -33.22291946411133,
      "activations/layer3_attention_weight_max": 93.56007385253906,
      "activations/layer3_attention_weight_min": -96.11442565917969,
      "activations/layer4_attention_weight_max": 104.09151458740234,
      "activations/layer4_attention_weight_min": -107.89315032958984,
      "activations/layer5_attention_weight_max": 50.17582321166992,
      "activations/layer5_attention_weight_min": -60.820640563964844,
      "activations/layer6_attention_weight_max": 41.584144592285156,
      "activations/layer6_attention_weight_min": -46.596839904785156,
      "activations/layer7_attention_weight_max": 84.77641296386719,
      "activations/layer7_attention_weight_min": -89.82550811767578,
      "activations/layer8_attention_weight_max": 38.72859191894531,
      "activations/layer8_attention_weight_min": -39.73134231567383,
      "activations/layer9_attention_weight_max": 30.348859786987305,
      "activations/layer9_attention_weight_min": -31.906957626342773,
      "epoch": 15.96,
      "learning_rate": 4.754583333333333e-05,
      "loss": 2.7397,
      "step": 274700
    },
    {
      "activations/layer0_attention_weight_max": 15.746308326721191,
      "activations/layer0_attention_weight_min": -13.598912239074707,
      "activations/layer10_attention_weight_max": 31.790536880493164,
      "activations/layer10_attention_weight_min": -32.14049530029297,
      "activations/layer11_attention_weight_max": 30.953018188476562,
      "activations/layer11_attention_weight_min": -30.73351287841797,
      "activations/layer12_attention_weight_max": 24.455644607543945,
      "activations/layer12_attention_weight_min": -26.384767532348633,
      "activations/layer13_attention_weight_max": 38.42858123779297,
      "activations/layer13_attention_weight_min": -34.18531036376953,
      "activations/layer14_attention_weight_max": 40.06398010253906,
      "activations/layer14_attention_weight_min": -37.6103401184082,
      "activations/layer15_attention_weight_max": 34.58037185668945,
      "activations/layer15_attention_weight_min": -34.78127670288086,
      "activations/layer16_attention_weight_max": 28.746456146240234,
      "activations/layer16_attention_weight_min": -27.97265625,
      "activations/layer17_attention_weight_max": 32.05766296386719,
      "activations/layer17_attention_weight_min": -25.953405380249023,
      "activations/layer18_attention_weight_max": 30.9246768951416,
      "activations/layer18_attention_weight_min": -25.02391242980957,
      "activations/layer19_attention_weight_max": 34.15387725830078,
      "activations/layer19_attention_weight_min": -30.891006469726562,
      "activations/layer1_attention_weight_max": 17.33687973022461,
      "activations/layer1_attention_weight_min": -15.614862442016602,
      "activations/layer20_attention_weight_max": 27.863189697265625,
      "activations/layer20_attention_weight_min": -25.647581100463867,
      "activations/layer21_attention_weight_max": 27.148826599121094,
      "activations/layer21_attention_weight_min": -23.722087860107422,
      "activations/layer22_attention_weight_max": 41.91132736206055,
      "activations/layer22_attention_weight_min": -29.233999252319336,
      "activations/layer23_attention_weight_max": 32.81840896606445,
      "activations/layer23_attention_weight_min": -24.343107223510742,
      "activations/layer2_attention_weight_max": 31.834468841552734,
      "activations/layer2_attention_weight_min": -32.03717803955078,
      "activations/layer3_attention_weight_max": 97.24793243408203,
      "activations/layer3_attention_weight_min": -95.15019226074219,
      "activations/layer4_attention_weight_max": 106.97801971435547,
      "activations/layer4_attention_weight_min": -116.32286834716797,
      "activations/layer5_attention_weight_max": 52.879173278808594,
      "activations/layer5_attention_weight_min": -63.915035247802734,
      "activations/layer6_attention_weight_max": 41.09617233276367,
      "activations/layer6_attention_weight_min": -44.285396575927734,
      "activations/layer7_attention_weight_max": 86.10771942138672,
      "activations/layer7_attention_weight_min": -90.71154022216797,
      "activations/layer8_attention_weight_max": 38.470035552978516,
      "activations/layer8_attention_weight_min": -40.75025939941406,
      "activations/layer9_attention_weight_max": 30.349472045898438,
      "activations/layer9_attention_weight_min": -31.838388442993164,
      "epoch": 15.96,
      "learning_rate": 4.752689393939393e-05,
      "loss": 2.7397,
      "step": 274750
    },
    {
      "activations/layer0_attention_weight_max": 16.49928092956543,
      "activations/layer0_attention_weight_min": -13.224541664123535,
      "activations/layer10_attention_weight_max": 41.939781188964844,
      "activations/layer10_attention_weight_min": -39.60908508300781,
      "activations/layer11_attention_weight_max": 39.88707733154297,
      "activations/layer11_attention_weight_min": -39.71687316894531,
      "activations/layer12_attention_weight_max": 27.311019897460938,
      "activations/layer12_attention_weight_min": -26.124357223510742,
      "activations/layer13_attention_weight_max": 41.84797286987305,
      "activations/layer13_attention_weight_min": -34.80074691772461,
      "activations/layer14_attention_weight_max": 41.35977554321289,
      "activations/layer14_attention_weight_min": -37.07646942138672,
      "activations/layer15_attention_weight_max": 39.53126525878906,
      "activations/layer15_attention_weight_min": -36.54798889160156,
      "activations/layer16_attention_weight_max": 32.74729537963867,
      "activations/layer16_attention_weight_min": -28.28597068786621,
      "activations/layer17_attention_weight_max": 34.12782669067383,
      "activations/layer17_attention_weight_min": -26.848051071166992,
      "activations/layer18_attention_weight_max": 33.52876281738281,
      "activations/layer18_attention_weight_min": -23.752334594726562,
      "activations/layer19_attention_weight_max": 35.19037628173828,
      "activations/layer19_attention_weight_min": -29.94525146484375,
      "activations/layer1_attention_weight_max": 16.583213806152344,
      "activations/layer1_attention_weight_min": -14.430577278137207,
      "activations/layer20_attention_weight_max": 32.75693893432617,
      "activations/layer20_attention_weight_min": -24.11284065246582,
      "activations/layer21_attention_weight_max": 31.51646614074707,
      "activations/layer21_attention_weight_min": -24.257102966308594,
      "activations/layer22_attention_weight_max": 42.14860153198242,
      "activations/layer22_attention_weight_min": -29.960189819335938,
      "activations/layer23_attention_weight_max": 36.00692367553711,
      "activations/layer23_attention_weight_min": -24.251876831054688,
      "activations/layer2_attention_weight_max": 31.806791305541992,
      "activations/layer2_attention_weight_min": -32.39109420776367,
      "activations/layer3_attention_weight_max": 96.9996109008789,
      "activations/layer3_attention_weight_min": -95.82077026367188,
      "activations/layer4_attention_weight_max": 108.26738739013672,
      "activations/layer4_attention_weight_min": -109.62403869628906,
      "activations/layer5_attention_weight_max": 51.155738830566406,
      "activations/layer5_attention_weight_min": -58.77415084838867,
      "activations/layer6_attention_weight_max": 42.8950080871582,
      "activations/layer6_attention_weight_min": -46.17427444458008,
      "activations/layer7_attention_weight_max": 103.81101989746094,
      "activations/layer7_attention_weight_min": -90.3106460571289,
      "activations/layer8_attention_weight_max": 47.768821716308594,
      "activations/layer8_attention_weight_min": -48.48060989379883,
      "activations/layer9_attention_weight_max": 37.34572982788086,
      "activations/layer9_attention_weight_min": -40.94590377807617,
      "epoch": 15.97,
      "learning_rate": 4.7507954545454545e-05,
      "loss": 2.7452,
      "step": 274800
    },
    {
      "activations/layer0_attention_weight_max": 15.850319862365723,
      "activations/layer0_attention_weight_min": -13.497320175170898,
      "activations/layer10_attention_weight_max": 34.83858108520508,
      "activations/layer10_attention_weight_min": -35.90099334716797,
      "activations/layer11_attention_weight_max": 35.4691162109375,
      "activations/layer11_attention_weight_min": -35.016021728515625,
      "activations/layer12_attention_weight_max": 29.926607131958008,
      "activations/layer12_attention_weight_min": -29.119821548461914,
      "activations/layer13_attention_weight_max": 44.79853820800781,
      "activations/layer13_attention_weight_min": -39.92516326904297,
      "activations/layer14_attention_weight_max": 45.58012008666992,
      "activations/layer14_attention_weight_min": -40.336910247802734,
      "activations/layer15_attention_weight_max": 42.860740661621094,
      "activations/layer15_attention_weight_min": -36.1075439453125,
      "activations/layer16_attention_weight_max": 31.568811416625977,
      "activations/layer16_attention_weight_min": -27.349931716918945,
      "activations/layer17_attention_weight_max": 30.66729164123535,
      "activations/layer17_attention_weight_min": -25.9121150970459,
      "activations/layer18_attention_weight_max": 30.795724868774414,
      "activations/layer18_attention_weight_min": -24.85993003845215,
      "activations/layer19_attention_weight_max": 32.96712875366211,
      "activations/layer19_attention_weight_min": -30.453678131103516,
      "activations/layer1_attention_weight_max": 16.21004867553711,
      "activations/layer1_attention_weight_min": -15.250778198242188,
      "activations/layer20_attention_weight_max": 32.04749298095703,
      "activations/layer20_attention_weight_min": -24.185483932495117,
      "activations/layer21_attention_weight_max": 32.26068115234375,
      "activations/layer21_attention_weight_min": -25.046621322631836,
      "activations/layer22_attention_weight_max": 41.9385986328125,
      "activations/layer22_attention_weight_min": -28.28675651550293,
      "activations/layer23_attention_weight_max": 32.490211486816406,
      "activations/layer23_attention_weight_min": -25.71746063232422,
      "activations/layer2_attention_weight_max": 32.005958557128906,
      "activations/layer2_attention_weight_min": -33.13589096069336,
      "activations/layer3_attention_weight_max": 102.07249450683594,
      "activations/layer3_attention_weight_min": -101.11203002929688,
      "activations/layer4_attention_weight_max": 109.82672119140625,
      "activations/layer4_attention_weight_min": -111.80269622802734,
      "activations/layer5_attention_weight_max": 56.287200927734375,
      "activations/layer5_attention_weight_min": -64.65245819091797,
      "activations/layer6_attention_weight_max": 44.69389343261719,
      "activations/layer6_attention_weight_min": -47.73522186279297,
      "activations/layer7_attention_weight_max": 99.91355895996094,
      "activations/layer7_attention_weight_min": -100.71635437011719,
      "activations/layer8_attention_weight_max": 44.127166748046875,
      "activations/layer8_attention_weight_min": -45.43510437011719,
      "activations/layer9_attention_weight_max": 32.88686752319336,
      "activations/layer9_attention_weight_min": -36.1373176574707,
      "epoch": 15.97,
      "learning_rate": 4.7489015151515146e-05,
      "loss": 2.7493,
      "step": 274850
    },
    {
      "activations/layer0_attention_weight_max": 16.168804168701172,
      "activations/layer0_attention_weight_min": -12.882318496704102,
      "activations/layer10_attention_weight_max": 32.533775329589844,
      "activations/layer10_attention_weight_min": -32.288307189941406,
      "activations/layer11_attention_weight_max": 33.51136016845703,
      "activations/layer11_attention_weight_min": -31.567306518554688,
      "activations/layer12_attention_weight_max": 25.80921173095703,
      "activations/layer12_attention_weight_min": -25.548030853271484,
      "activations/layer13_attention_weight_max": 41.01972198486328,
      "activations/layer13_attention_weight_min": -34.80251693725586,
      "activations/layer14_attention_weight_max": 40.527732849121094,
      "activations/layer14_attention_weight_min": -37.919029235839844,
      "activations/layer15_attention_weight_max": 37.75578689575195,
      "activations/layer15_attention_weight_min": -32.51277160644531,
      "activations/layer16_attention_weight_max": 30.244234085083008,
      "activations/layer16_attention_weight_min": -26.612077713012695,
      "activations/layer17_attention_weight_max": 30.57077980041504,
      "activations/layer17_attention_weight_min": -26.188756942749023,
      "activations/layer18_attention_weight_max": 30.75153923034668,
      "activations/layer18_attention_weight_min": -25.12715721130371,
      "activations/layer19_attention_weight_max": 34.064971923828125,
      "activations/layer19_attention_weight_min": -32.790504455566406,
      "activations/layer1_attention_weight_max": 16.691654205322266,
      "activations/layer1_attention_weight_min": -16.365447998046875,
      "activations/layer20_attention_weight_max": 29.483890533447266,
      "activations/layer20_attention_weight_min": -27.035438537597656,
      "activations/layer21_attention_weight_max": 30.88265609741211,
      "activations/layer21_attention_weight_min": -26.87779998779297,
      "activations/layer22_attention_weight_max": 39.975215911865234,
      "activations/layer22_attention_weight_min": -33.06895065307617,
      "activations/layer23_attention_weight_max": 31.06189727783203,
      "activations/layer23_attention_weight_min": -26.68031883239746,
      "activations/layer2_attention_weight_max": 34.212364196777344,
      "activations/layer2_attention_weight_min": -33.540767669677734,
      "activations/layer3_attention_weight_max": 96.89938354492188,
      "activations/layer3_attention_weight_min": -98.42802429199219,
      "activations/layer4_attention_weight_max": 106.30596160888672,
      "activations/layer4_attention_weight_min": -105.24845123291016,
      "activations/layer5_attention_weight_max": 50.821041107177734,
      "activations/layer5_attention_weight_min": -60.36248779296875,
      "activations/layer6_attention_weight_max": 45.05917739868164,
      "activations/layer6_attention_weight_min": -43.390872955322266,
      "activations/layer7_attention_weight_max": 88.38156127929688,
      "activations/layer7_attention_weight_min": -85.11856842041016,
      "activations/layer8_attention_weight_max": 38.82307434082031,
      "activations/layer8_attention_weight_min": -41.04593276977539,
      "activations/layer9_attention_weight_max": 31.621679306030273,
      "activations/layer9_attention_weight_min": -32.16819381713867,
      "epoch": 15.97,
      "learning_rate": 4.7470075757575754e-05,
      "loss": 2.7322,
      "step": 274900
    },
    {
      "activations/layer0_attention_weight_max": 15.786733627319336,
      "activations/layer0_attention_weight_min": -12.715288162231445,
      "activations/layer10_attention_weight_max": 33.51936721801758,
      "activations/layer10_attention_weight_min": -34.178958892822266,
      "activations/layer11_attention_weight_max": 32.37067413330078,
      "activations/layer11_attention_weight_min": -34.0323371887207,
      "activations/layer12_attention_weight_max": 26.45142364501953,
      "activations/layer12_attention_weight_min": -27.185766220092773,
      "activations/layer13_attention_weight_max": 41.06237030029297,
      "activations/layer13_attention_weight_min": -37.479854583740234,
      "activations/layer14_attention_weight_max": 42.78553771972656,
      "activations/layer14_attention_weight_min": -39.63129806518555,
      "activations/layer15_attention_weight_max": 37.00836181640625,
      "activations/layer15_attention_weight_min": -36.098636627197266,
      "activations/layer16_attention_weight_max": 27.52580451965332,
      "activations/layer16_attention_weight_min": -28.981351852416992,
      "activations/layer17_attention_weight_max": 29.900325775146484,
      "activations/layer17_attention_weight_min": -25.878847122192383,
      "activations/layer18_attention_weight_max": 34.19977951049805,
      "activations/layer18_attention_weight_min": -25.91562271118164,
      "activations/layer19_attention_weight_max": 32.25271224975586,
      "activations/layer19_attention_weight_min": -30.59449577331543,
      "activations/layer1_attention_weight_max": 16.382183074951172,
      "activations/layer1_attention_weight_min": -16.112449645996094,
      "activations/layer20_attention_weight_max": 29.247386932373047,
      "activations/layer20_attention_weight_min": -26.710613250732422,
      "activations/layer21_attention_weight_max": 30.043996810913086,
      "activations/layer21_attention_weight_min": -25.830448150634766,
      "activations/layer22_attention_weight_max": 41.939659118652344,
      "activations/layer22_attention_weight_min": -32.69401931762695,
      "activations/layer23_attention_weight_max": 32.62106704711914,
      "activations/layer23_attention_weight_min": -26.856037139892578,
      "activations/layer2_attention_weight_max": 32.65830993652344,
      "activations/layer2_attention_weight_min": -34.12096405029297,
      "activations/layer3_attention_weight_max": 97.63029479980469,
      "activations/layer3_attention_weight_min": -98.18289184570312,
      "activations/layer4_attention_weight_max": 105.94145965576172,
      "activations/layer4_attention_weight_min": -110.4537353515625,
      "activations/layer5_attention_weight_max": 49.39146041870117,
      "activations/layer5_attention_weight_min": -63.643402099609375,
      "activations/layer6_attention_weight_max": 42.0341796875,
      "activations/layer6_attention_weight_min": -49.48594665527344,
      "activations/layer7_attention_weight_max": 88.00827026367188,
      "activations/layer7_attention_weight_min": -90.8336181640625,
      "activations/layer8_attention_weight_max": 39.90480422973633,
      "activations/layer8_attention_weight_min": -42.85986328125,
      "activations/layer9_attention_weight_max": 31.467132568359375,
      "activations/layer9_attention_weight_min": -34.15391159057617,
      "epoch": 15.98,
      "learning_rate": 4.745113636363636e-05,
      "loss": 2.7381,
      "step": 274950
    },
    {
      "activations/layer0_attention_weight_max": 15.788755416870117,
      "activations/layer0_attention_weight_min": -12.966972351074219,
      "activations/layer10_attention_weight_max": 33.289459228515625,
      "activations/layer10_attention_weight_min": -34.88241195678711,
      "activations/layer11_attention_weight_max": 35.440216064453125,
      "activations/layer11_attention_weight_min": -33.97319793701172,
      "activations/layer12_attention_weight_max": 28.37851333618164,
      "activations/layer12_attention_weight_min": -27.89106559753418,
      "activations/layer13_attention_weight_max": 41.179710388183594,
      "activations/layer13_attention_weight_min": -36.7142448425293,
      "activations/layer14_attention_weight_max": 49.527042388916016,
      "activations/layer14_attention_weight_min": -40.519073486328125,
      "activations/layer15_attention_weight_max": 44.55604934692383,
      "activations/layer15_attention_weight_min": -37.131595611572266,
      "activations/layer16_attention_weight_max": 31.913904190063477,
      "activations/layer16_attention_weight_min": -30.489770889282227,
      "activations/layer17_attention_weight_max": 32.961421966552734,
      "activations/layer17_attention_weight_min": -28.291635513305664,
      "activations/layer18_attention_weight_max": 31.078214645385742,
      "activations/layer18_attention_weight_min": -25.787485122680664,
      "activations/layer19_attention_weight_max": 35.60506057739258,
      "activations/layer19_attention_weight_min": -32.63204574584961,
      "activations/layer1_attention_weight_max": 17.079614639282227,
      "activations/layer1_attention_weight_min": -14.682409286499023,
      "activations/layer20_attention_weight_max": 32.06229019165039,
      "activations/layer20_attention_weight_min": -24.867759704589844,
      "activations/layer21_attention_weight_max": 32.641292572021484,
      "activations/layer21_attention_weight_min": -24.798828125,
      "activations/layer22_attention_weight_max": 44.6488037109375,
      "activations/layer22_attention_weight_min": -32.19054412841797,
      "activations/layer23_attention_weight_max": 33.0767822265625,
      "activations/layer23_attention_weight_min": -24.803203582763672,
      "activations/layer2_attention_weight_max": 33.45965576171875,
      "activations/layer2_attention_weight_min": -34.111412048339844,
      "activations/layer3_attention_weight_max": 97.97554016113281,
      "activations/layer3_attention_weight_min": -98.32329559326172,
      "activations/layer4_attention_weight_max": 107.34271240234375,
      "activations/layer4_attention_weight_min": -108.88270568847656,
      "activations/layer5_attention_weight_max": 54.76930236816406,
      "activations/layer5_attention_weight_min": -63.0085563659668,
      "activations/layer6_attention_weight_max": 43.831424713134766,
      "activations/layer6_attention_weight_min": -47.52090072631836,
      "activations/layer7_attention_weight_max": 91.40410614013672,
      "activations/layer7_attention_weight_min": -103.40892028808594,
      "activations/layer8_attention_weight_max": 41.141822814941406,
      "activations/layer8_attention_weight_min": -43.79338455200195,
      "activations/layer9_attention_weight_max": 32.40025329589844,
      "activations/layer9_attention_weight_min": -35.32077407836914,
      "epoch": 15.98,
      "learning_rate": 4.7432196969696964e-05,
      "loss": 2.7474,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.5162,
      "eval_samples_per_second": 504.214,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.5162,
      "eval_openwebtext_samples_per_second": 504.214,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9818,
      "eval_wikitext_samples_per_second": 230.089,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_lambada_loss": 2.39453125,
      "eval_lambada_ppl": 10.963057921054194,
      "eval_lambada_runtime": 9.5326,
      "eval_lambada_samples_per_second": 510.773,
      "step": 275000
    },
    {
      "activations/layer0_attention_weight_max": 16.6041202545166,
      "activations/layer0_attention_weight_min": -13.157827377319336,
      "activations/layer10_attention_weight_max": 31.27683448791504,
      "activations/layer10_attention_weight_min": -31.17166519165039,
      "activations/layer11_attention_weight_max": 30.846046447753906,
      "activations/layer11_attention_weight_min": -31.05356216430664,
      "activations/layer12_attention_weight_max": 27.193111419677734,
      "activations/layer12_attention_weight_min": -26.058393478393555,
      "activations/layer13_attention_weight_max": 40.44590377807617,
      "activations/layer13_attention_weight_min": -36.58130645751953,
      "activations/layer14_attention_weight_max": 41.47205352783203,
      "activations/layer14_attention_weight_min": -38.37269592285156,
      "activations/layer15_attention_weight_max": 37.84193801879883,
      "activations/layer15_attention_weight_min": -32.96357345581055,
      "activations/layer16_attention_weight_max": 29.11293601989746,
      "activations/layer16_attention_weight_min": -26.827444076538086,
      "activations/layer17_attention_weight_max": 31.215240478515625,
      "activations/layer17_attention_weight_min": -24.733747482299805,
      "activations/layer18_attention_weight_max": 30.480079650878906,
      "activations/layer18_attention_weight_min": -26.089420318603516,
      "activations/layer19_attention_weight_max": 34.208778381347656,
      "activations/layer19_attention_weight_min": -29.967655181884766,
      "activations/layer1_attention_weight_max": 15.91115665435791,
      "activations/layer1_attention_weight_min": -13.76787281036377,
      "activations/layer20_attention_weight_max": 28.329221725463867,
      "activations/layer20_attention_weight_min": -24.21074867248535,
      "activations/layer21_attention_weight_max": 28.869773864746094,
      "activations/layer21_attention_weight_min": -25.197229385375977,
      "activations/layer22_attention_weight_max": 42.022216796875,
      "activations/layer22_attention_weight_min": -29.79947853088379,
      "activations/layer23_attention_weight_max": 34.933570861816406,
      "activations/layer23_attention_weight_min": -23.94112205505371,
      "activations/layer2_attention_weight_max": 36.94855499267578,
      "activations/layer2_attention_weight_min": -34.1552734375,
      "activations/layer3_attention_weight_max": 102.4863510131836,
      "activations/layer3_attention_weight_min": -105.97041320800781,
      "activations/layer4_attention_weight_max": 109.46089172363281,
      "activations/layer4_attention_weight_min": -108.05670166015625,
      "activations/layer5_attention_weight_max": 52.018123626708984,
      "activations/layer5_attention_weight_min": -57.46487045288086,
      "activations/layer6_attention_weight_max": 42.15605926513672,
      "activations/layer6_attention_weight_min": -43.82273864746094,
      "activations/layer7_attention_weight_max": 89.04920959472656,
      "activations/layer7_attention_weight_min": -90.15206146240234,
      "activations/layer8_attention_weight_max": 38.3959846496582,
      "activations/layer8_attention_weight_min": -40.39458084106445,
      "activations/layer9_attention_weight_max": 29.790557861328125,
      "activations/layer9_attention_weight_min": -31.323535919189453,
      "epoch": 15.98,
      "learning_rate": 4.741325757575757e-05,
      "loss": 2.7583,
      "step": 275050
    },
    {
      "activations/layer0_attention_weight_max": 16.697702407836914,
      "activations/layer0_attention_weight_min": -12.180716514587402,
      "activations/layer10_attention_weight_max": 31.517303466796875,
      "activations/layer10_attention_weight_min": -32.9349479675293,
      "activations/layer11_attention_weight_max": 31.497770309448242,
      "activations/layer11_attention_weight_min": -33.22282791137695,
      "activations/layer12_attention_weight_max": 26.0476131439209,
      "activations/layer12_attention_weight_min": -29.323286056518555,
      "activations/layer13_attention_weight_max": 41.869808197021484,
      "activations/layer13_attention_weight_min": -34.14509582519531,
      "activations/layer14_attention_weight_max": 42.37556076049805,
      "activations/layer14_attention_weight_min": -38.2436408996582,
      "activations/layer15_attention_weight_max": 38.30507278442383,
      "activations/layer15_attention_weight_min": -33.93125915527344,
      "activations/layer16_attention_weight_max": 30.184158325195312,
      "activations/layer16_attention_weight_min": -28.24015235900879,
      "activations/layer17_attention_weight_max": 29.266183853149414,
      "activations/layer17_attention_weight_min": -25.591144561767578,
      "activations/layer18_attention_weight_max": 31.66437339782715,
      "activations/layer18_attention_weight_min": -24.343706130981445,
      "activations/layer19_attention_weight_max": 34.2894287109375,
      "activations/layer19_attention_weight_min": -31.768651962280273,
      "activations/layer1_attention_weight_max": 16.168136596679688,
      "activations/layer1_attention_weight_min": -14.575549125671387,
      "activations/layer20_attention_weight_max": 28.7417049407959,
      "activations/layer20_attention_weight_min": -24.655860900878906,
      "activations/layer21_attention_weight_max": 29.16376304626465,
      "activations/layer21_attention_weight_min": -21.79949378967285,
      "activations/layer22_attention_weight_max": 46.7393798828125,
      "activations/layer22_attention_weight_min": -30.30215072631836,
      "activations/layer23_attention_weight_max": 33.55846405029297,
      "activations/layer23_attention_weight_min": -23.981002807617188,
      "activations/layer2_attention_weight_max": 32.3871955871582,
      "activations/layer2_attention_weight_min": -31.58466911315918,
      "activations/layer3_attention_weight_max": 93.47354888916016,
      "activations/layer3_attention_weight_min": -94.1050033569336,
      "activations/layer4_attention_weight_max": 101.15983581542969,
      "activations/layer4_attention_weight_min": -103.84798431396484,
      "activations/layer5_attention_weight_max": 47.52404022216797,
      "activations/layer5_attention_weight_min": -61.31463623046875,
      "activations/layer6_attention_weight_max": 41.20335006713867,
      "activations/layer6_attention_weight_min": -45.19508361816406,
      "activations/layer7_attention_weight_max": 87.4975814819336,
      "activations/layer7_attention_weight_min": -92.45516967773438,
      "activations/layer8_attention_weight_max": 41.89862060546875,
      "activations/layer8_attention_weight_min": -44.046485900878906,
      "activations/layer9_attention_weight_max": 31.30466079711914,
      "activations/layer9_attention_weight_min": -33.885799407958984,
      "epoch": 15.98,
      "learning_rate": 4.7394318181818174e-05,
      "loss": 2.7404,
      "step": 275100
    },
    {
      "activations/layer0_attention_weight_max": 17.066102981567383,
      "activations/layer0_attention_weight_min": -12.864301681518555,
      "activations/layer10_attention_weight_max": 32.067138671875,
      "activations/layer10_attention_weight_min": -33.38230895996094,
      "activations/layer11_attention_weight_max": 33.91884231567383,
      "activations/layer11_attention_weight_min": -33.70580291748047,
      "activations/layer12_attention_weight_max": 24.756301879882812,
      "activations/layer12_attention_weight_min": -26.441972732543945,
      "activations/layer13_attention_weight_max": 36.48147201538086,
      "activations/layer13_attention_weight_min": -34.391510009765625,
      "activations/layer14_attention_weight_max": 39.757469177246094,
      "activations/layer14_attention_weight_min": -37.55165100097656,
      "activations/layer15_attention_weight_max": 36.189002990722656,
      "activations/layer15_attention_weight_min": -33.17481994628906,
      "activations/layer16_attention_weight_max": 29.312511444091797,
      "activations/layer16_attention_weight_min": -27.830167770385742,
      "activations/layer17_attention_weight_max": 29.379751205444336,
      "activations/layer17_attention_weight_min": -29.76353645324707,
      "activations/layer18_attention_weight_max": 30.46392822265625,
      "activations/layer18_attention_weight_min": -26.152767181396484,
      "activations/layer19_attention_weight_max": 33.10251235961914,
      "activations/layer19_attention_weight_min": -30.238632202148438,
      "activations/layer1_attention_weight_max": 16.825841903686523,
      "activations/layer1_attention_weight_min": -16.12665557861328,
      "activations/layer20_attention_weight_max": 27.257347106933594,
      "activations/layer20_attention_weight_min": -25.365556716918945,
      "activations/layer21_attention_weight_max": 25.889158248901367,
      "activations/layer21_attention_weight_min": -24.28643035888672,
      "activations/layer22_attention_weight_max": 41.07622146606445,
      "activations/layer22_attention_weight_min": -31.71773910522461,
      "activations/layer23_attention_weight_max": 31.39772605895996,
      "activations/layer23_attention_weight_min": -25.884654998779297,
      "activations/layer2_attention_weight_max": 32.98213577270508,
      "activations/layer2_attention_weight_min": -31.687816619873047,
      "activations/layer3_attention_weight_max": 94.31973266601562,
      "activations/layer3_attention_weight_min": -93.34813690185547,
      "activations/layer4_attention_weight_max": 106.44939422607422,
      "activations/layer4_attention_weight_min": -103.7664566040039,
      "activations/layer5_attention_weight_max": 52.057525634765625,
      "activations/layer5_attention_weight_min": -58.939292907714844,
      "activations/layer6_attention_weight_max": 44.133419036865234,
      "activations/layer6_attention_weight_min": -46.60714340209961,
      "activations/layer7_attention_weight_max": 93.01903533935547,
      "activations/layer7_attention_weight_min": -89.89139556884766,
      "activations/layer8_attention_weight_max": 39.769493103027344,
      "activations/layer8_attention_weight_min": -40.79324722290039,
      "activations/layer9_attention_weight_max": 31.100570678710938,
      "activations/layer9_attention_weight_min": -33.49717330932617,
      "epoch": 15.99,
      "learning_rate": 4.737537878787879e-05,
      "loss": 2.7495,
      "step": 275150
    },
    {
      "activations/layer0_attention_weight_max": 15.689117431640625,
      "activations/layer0_attention_weight_min": -13.022794723510742,
      "activations/layer10_attention_weight_max": 34.50692367553711,
      "activations/layer10_attention_weight_min": -34.59652328491211,
      "activations/layer11_attention_weight_max": 33.93256759643555,
      "activations/layer11_attention_weight_min": -34.290592193603516,
      "activations/layer12_attention_weight_max": 27.88574981689453,
      "activations/layer12_attention_weight_min": -24.584760665893555,
      "activations/layer13_attention_weight_max": 41.26304244995117,
      "activations/layer13_attention_weight_min": -34.68818664550781,
      "activations/layer14_attention_weight_max": 45.90321350097656,
      "activations/layer14_attention_weight_min": -38.32859802246094,
      "activations/layer15_attention_weight_max": 42.29459762573242,
      "activations/layer15_attention_weight_min": -33.630218505859375,
      "activations/layer16_attention_weight_max": 35.28342819213867,
      "activations/layer16_attention_weight_min": -28.13959312438965,
      "activations/layer17_attention_weight_max": 32.433841705322266,
      "activations/layer17_attention_weight_min": -24.258893966674805,
      "activations/layer18_attention_weight_max": 34.1358757019043,
      "activations/layer18_attention_weight_min": -23.864675521850586,
      "activations/layer19_attention_weight_max": 36.513816833496094,
      "activations/layer19_attention_weight_min": -28.629831314086914,
      "activations/layer1_attention_weight_max": 15.782894134521484,
      "activations/layer1_attention_weight_min": -14.984707832336426,
      "activations/layer20_attention_weight_max": 31.209808349609375,
      "activations/layer20_attention_weight_min": -23.6947078704834,
      "activations/layer21_attention_weight_max": 30.77646827697754,
      "activations/layer21_attention_weight_min": -22.314449310302734,
      "activations/layer22_attention_weight_max": 42.18204879760742,
      "activations/layer22_attention_weight_min": -28.141443252563477,
      "activations/layer23_attention_weight_max": 32.616432189941406,
      "activations/layer23_attention_weight_min": -25.685285568237305,
      "activations/layer2_attention_weight_max": 32.69301223754883,
      "activations/layer2_attention_weight_min": -33.49375915527344,
      "activations/layer3_attention_weight_max": 99.04962921142578,
      "activations/layer3_attention_weight_min": -104.55303955078125,
      "activations/layer4_attention_weight_max": 109.8864974975586,
      "activations/layer4_attention_weight_min": -108.44600677490234,
      "activations/layer5_attention_weight_max": 49.43630599975586,
      "activations/layer5_attention_weight_min": -60.20066833496094,
      "activations/layer6_attention_weight_max": 44.230369567871094,
      "activations/layer6_attention_weight_min": -48.74230194091797,
      "activations/layer7_attention_weight_max": 91.71672821044922,
      "activations/layer7_attention_weight_min": -103.93905639648438,
      "activations/layer8_attention_weight_max": 45.0428352355957,
      "activations/layer8_attention_weight_min": -46.812889099121094,
      "activations/layer9_attention_weight_max": 32.88349914550781,
      "activations/layer9_attention_weight_min": -36.25886535644531,
      "epoch": 15.99,
      "learning_rate": 4.735643939393939e-05,
      "loss": 2.7473,
      "step": 275200
    },
    {
      "activations/layer0_attention_weight_max": 15.724198341369629,
      "activations/layer0_attention_weight_min": -12.073906898498535,
      "activations/layer10_attention_weight_max": 34.51461410522461,
      "activations/layer10_attention_weight_min": -34.0130500793457,
      "activations/layer11_attention_weight_max": 34.75080871582031,
      "activations/layer11_attention_weight_min": -34.038448333740234,
      "activations/layer12_attention_weight_max": 27.342044830322266,
      "activations/layer12_attention_weight_min": -26.491453170776367,
      "activations/layer13_attention_weight_max": 42.843360900878906,
      "activations/layer13_attention_weight_min": -38.712154388427734,
      "activations/layer14_attention_weight_max": 49.623313903808594,
      "activations/layer14_attention_weight_min": -41.60352325439453,
      "activations/layer15_attention_weight_max": 48.9217529296875,
      "activations/layer15_attention_weight_min": -43.261253356933594,
      "activations/layer16_attention_weight_max": 34.73273468017578,
      "activations/layer16_attention_weight_min": -28.312070846557617,
      "activations/layer17_attention_weight_max": 34.53937911987305,
      "activations/layer17_attention_weight_min": -26.926427841186523,
      "activations/layer18_attention_weight_max": 35.75669479370117,
      "activations/layer18_attention_weight_min": -26.78434181213379,
      "activations/layer19_attention_weight_max": 39.058597564697266,
      "activations/layer19_attention_weight_min": -34.83280563354492,
      "activations/layer1_attention_weight_max": 15.850330352783203,
      "activations/layer1_attention_weight_min": -15.892699241638184,
      "activations/layer20_attention_weight_max": 31.53362274169922,
      "activations/layer20_attention_weight_min": -23.287216186523438,
      "activations/layer21_attention_weight_max": 30.44443130493164,
      "activations/layer21_attention_weight_min": -24.561920166015625,
      "activations/layer22_attention_weight_max": 46.87839889526367,
      "activations/layer22_attention_weight_min": -34.269737243652344,
      "activations/layer23_attention_weight_max": 40.065696716308594,
      "activations/layer23_attention_weight_min": -25.516925811767578,
      "activations/layer2_attention_weight_max": 35.35456085205078,
      "activations/layer2_attention_weight_min": -34.66993713378906,
      "activations/layer3_attention_weight_max": 98.19412994384766,
      "activations/layer3_attention_weight_min": -99.6049575805664,
      "activations/layer4_attention_weight_max": 110.13541412353516,
      "activations/layer4_attention_weight_min": -108.34188079833984,
      "activations/layer5_attention_weight_max": 52.246212005615234,
      "activations/layer5_attention_weight_min": -60.734413146972656,
      "activations/layer6_attention_weight_max": 45.89985275268555,
      "activations/layer6_attention_weight_min": -46.55132293701172,
      "activations/layer7_attention_weight_max": 99.55342102050781,
      "activations/layer7_attention_weight_min": -104.77978515625,
      "activations/layer8_attention_weight_max": 43.12910079956055,
      "activations/layer8_attention_weight_min": -44.35075759887695,
      "activations/layer9_attention_weight_max": 31.97236442565918,
      "activations/layer9_attention_weight_min": -34.30493927001953,
      "epoch": 15.99,
      "learning_rate": 4.733749999999999e-05,
      "loss": 2.7419,
      "step": 275250
    },
    {
      "activations/layer0_attention_weight_max": 15.666206359863281,
      "activations/layer0_attention_weight_min": -12.979462623596191,
      "activations/layer10_attention_weight_max": 33.88193130493164,
      "activations/layer10_attention_weight_min": -32.900482177734375,
      "activations/layer11_attention_weight_max": 31.784873962402344,
      "activations/layer11_attention_weight_min": -32.063209533691406,
      "activations/layer12_attention_weight_max": 26.188684463500977,
      "activations/layer12_attention_weight_min": -24.82588005065918,
      "activations/layer13_attention_weight_max": 42.32125473022461,
      "activations/layer13_attention_weight_min": -34.66462707519531,
      "activations/layer14_attention_weight_max": 46.18656921386719,
      "activations/layer14_attention_weight_min": -35.942100524902344,
      "activations/layer15_attention_weight_max": 41.38351058959961,
      "activations/layer15_attention_weight_min": -35.63588333129883,
      "activations/layer16_attention_weight_max": 30.83055877685547,
      "activations/layer16_attention_weight_min": -29.209196090698242,
      "activations/layer17_attention_weight_max": 30.097272872924805,
      "activations/layer17_attention_weight_min": -27.319995880126953,
      "activations/layer18_attention_weight_max": 34.56830596923828,
      "activations/layer18_attention_weight_min": -27.335298538208008,
      "activations/layer19_attention_weight_max": 35.697715759277344,
      "activations/layer19_attention_weight_min": -32.5226936340332,
      "activations/layer1_attention_weight_max": 15.790410995483398,
      "activations/layer1_attention_weight_min": -14.83655071258545,
      "activations/layer20_attention_weight_max": 28.595108032226562,
      "activations/layer20_attention_weight_min": -27.8321533203125,
      "activations/layer21_attention_weight_max": 29.574167251586914,
      "activations/layer21_attention_weight_min": -24.260765075683594,
      "activations/layer22_attention_weight_max": 41.618446350097656,
      "activations/layer22_attention_weight_min": -31.025192260742188,
      "activations/layer23_attention_weight_max": 33.654747009277344,
      "activations/layer23_attention_weight_min": -25.448257446289062,
      "activations/layer2_attention_weight_max": 32.40665817260742,
      "activations/layer2_attention_weight_min": -33.40065002441406,
      "activations/layer3_attention_weight_max": 94.76917266845703,
      "activations/layer3_attention_weight_min": -99.20109558105469,
      "activations/layer4_attention_weight_max": 105.76805877685547,
      "activations/layer4_attention_weight_min": -106.04357147216797,
      "activations/layer5_attention_weight_max": 49.83897018432617,
      "activations/layer5_attention_weight_min": -62.832908630371094,
      "activations/layer6_attention_weight_max": 43.504295349121094,
      "activations/layer6_attention_weight_min": -45.22071838378906,
      "activations/layer7_attention_weight_max": 93.13809967041016,
      "activations/layer7_attention_weight_min": -94.19288635253906,
      "activations/layer8_attention_weight_max": 38.886810302734375,
      "activations/layer8_attention_weight_min": -43.028076171875,
      "activations/layer9_attention_weight_max": 33.757137298583984,
      "activations/layer9_attention_weight_min": -33.899131774902344,
      "epoch": 16.0,
      "learning_rate": 4.73185606060606e-05,
      "loss": 2.7434,
      "step": 275300
    },
    {
      "activations/layer0_attention_weight_max": 17.04770851135254,
      "activations/layer0_attention_weight_min": -12.75307559967041,
      "activations/layer10_attention_weight_max": 33.404693603515625,
      "activations/layer10_attention_weight_min": -33.38896179199219,
      "activations/layer11_attention_weight_max": 32.98325729370117,
      "activations/layer11_attention_weight_min": -32.88166046142578,
      "activations/layer12_attention_weight_max": 26.029142379760742,
      "activations/layer12_attention_weight_min": -24.667333602905273,
      "activations/layer13_attention_weight_max": 39.14227294921875,
      "activations/layer13_attention_weight_min": -35.44794845581055,
      "activations/layer14_attention_weight_max": 43.89497756958008,
      "activations/layer14_attention_weight_min": -36.444515228271484,
      "activations/layer15_attention_weight_max": 39.078269958496094,
      "activations/layer15_attention_weight_min": -33.34550476074219,
      "activations/layer16_attention_weight_max": 34.48111343383789,
      "activations/layer16_attention_weight_min": -28.176589965820312,
      "activations/layer17_attention_weight_max": 31.35722541809082,
      "activations/layer17_attention_weight_min": -25.19487190246582,
      "activations/layer18_attention_weight_max": 29.449594497680664,
      "activations/layer18_attention_weight_min": -25.440338134765625,
      "activations/layer19_attention_weight_max": 32.91518020629883,
      "activations/layer19_attention_weight_min": -30.335315704345703,
      "activations/layer1_attention_weight_max": 16.57016372680664,
      "activations/layer1_attention_weight_min": -15.863637924194336,
      "activations/layer20_attention_weight_max": 27.173704147338867,
      "activations/layer20_attention_weight_min": -25.37494468688965,
      "activations/layer21_attention_weight_max": 32.07330322265625,
      "activations/layer21_attention_weight_min": -24.50497817993164,
      "activations/layer22_attention_weight_max": 38.785011291503906,
      "activations/layer22_attention_weight_min": -32.79494094848633,
      "activations/layer23_attention_weight_max": 31.77361297607422,
      "activations/layer23_attention_weight_min": -28.01770782470703,
      "activations/layer2_attention_weight_max": 33.646690368652344,
      "activations/layer2_attention_weight_min": -32.54344177246094,
      "activations/layer3_attention_weight_max": 94.58851623535156,
      "activations/layer3_attention_weight_min": -97.59866333007812,
      "activations/layer4_attention_weight_max": 105.53590393066406,
      "activations/layer4_attention_weight_min": -106.3042221069336,
      "activations/layer5_attention_weight_max": 51.48271179199219,
      "activations/layer5_attention_weight_min": -70.0892333984375,
      "activations/layer6_attention_weight_max": 45.87892150878906,
      "activations/layer6_attention_weight_min": -45.43367004394531,
      "activations/layer7_attention_weight_max": 91.7944564819336,
      "activations/layer7_attention_weight_min": -91.76841735839844,
      "activations/layer8_attention_weight_max": 40.0219612121582,
      "activations/layer8_attention_weight_min": -45.368133544921875,
      "activations/layer9_attention_weight_max": 30.96722412109375,
      "activations/layer9_attention_weight_min": -34.175994873046875,
      "epoch": 16.0,
      "learning_rate": 4.729962121212121e-05,
      "loss": 2.7621,
      "step": 275350
    },
    {
      "activations/layer0_attention_weight_max": 16.328231811523438,
      "activations/layer0_attention_weight_min": -12.88255786895752,
      "activations/layer10_attention_weight_max": 34.45235061645508,
      "activations/layer10_attention_weight_min": -36.70057678222656,
      "activations/layer11_attention_weight_max": 36.845367431640625,
      "activations/layer11_attention_weight_min": -37.146751403808594,
      "activations/layer12_attention_weight_max": 25.78718376159668,
      "activations/layer12_attention_weight_min": -25.92001724243164,
      "activations/layer13_attention_weight_max": 38.00436019897461,
      "activations/layer13_attention_weight_min": -37.59819412231445,
      "activations/layer14_attention_weight_max": 45.10732650756836,
      "activations/layer14_attention_weight_min": -43.783042907714844,
      "activations/layer15_attention_weight_max": 40.23020553588867,
      "activations/layer15_attention_weight_min": -40.16474533081055,
      "activations/layer16_attention_weight_max": 35.4804573059082,
      "activations/layer16_attention_weight_min": -29.794301986694336,
      "activations/layer17_attention_weight_max": 30.76240348815918,
      "activations/layer17_attention_weight_min": -28.68816566467285,
      "activations/layer18_attention_weight_max": 32.622066497802734,
      "activations/layer18_attention_weight_min": -26.48956871032715,
      "activations/layer19_attention_weight_max": 35.42058181762695,
      "activations/layer19_attention_weight_min": -31.62592124938965,
      "activations/layer1_attention_weight_max": 15.932853698730469,
      "activations/layer1_attention_weight_min": -14.438286781311035,
      "activations/layer20_attention_weight_max": 31.932292938232422,
      "activations/layer20_attention_weight_min": -25.0491943359375,
      "activations/layer21_attention_weight_max": 32.114845275878906,
      "activations/layer21_attention_weight_min": -26.284378051757812,
      "activations/layer22_attention_weight_max": 46.19389724731445,
      "activations/layer22_attention_weight_min": -33.84591293334961,
      "activations/layer23_attention_weight_max": 32.48895263671875,
      "activations/layer23_attention_weight_min": -27.406429290771484,
      "activations/layer2_attention_weight_max": 32.60249710083008,
      "activations/layer2_attention_weight_min": -32.75178909301758,
      "activations/layer3_attention_weight_max": 95.63043212890625,
      "activations/layer3_attention_weight_min": -94.92388153076172,
      "activations/layer4_attention_weight_max": 101.85228729248047,
      "activations/layer4_attention_weight_min": -101.65560150146484,
      "activations/layer5_attention_weight_max": 48.541683197021484,
      "activations/layer5_attention_weight_min": -60.09228515625,
      "activations/layer6_attention_weight_max": 43.94779586791992,
      "activations/layer6_attention_weight_min": -45.39289093017578,
      "activations/layer7_attention_weight_max": 88.45449829101562,
      "activations/layer7_attention_weight_min": -91.93675994873047,
      "activations/layer8_attention_weight_max": 39.90528869628906,
      "activations/layer8_attention_weight_min": -42.362125396728516,
      "activations/layer9_attention_weight_max": 35.55210494995117,
      "activations/layer9_attention_weight_min": -36.28596878051758,
      "epoch": 16.0,
      "learning_rate": 4.728068181818182e-05,
      "loss": 2.7853,
      "step": 275400
    },
    {
      "activations/layer0_attention_weight_max": 15.62265396118164,
      "activations/layer0_attention_weight_min": -12.382617950439453,
      "activations/layer10_attention_weight_max": 33.935211181640625,
      "activations/layer10_attention_weight_min": -35.260948181152344,
      "activations/layer11_attention_weight_max": 33.790687561035156,
      "activations/layer11_attention_weight_min": -35.01268005371094,
      "activations/layer12_attention_weight_max": 26.543216705322266,
      "activations/layer12_attention_weight_min": -26.349929809570312,
      "activations/layer13_attention_weight_max": 38.20106506347656,
      "activations/layer13_attention_weight_min": -36.09811782836914,
      "activations/layer14_attention_weight_max": 41.27589416503906,
      "activations/layer14_attention_weight_min": -36.960697174072266,
      "activations/layer15_attention_weight_max": 41.52864074707031,
      "activations/layer15_attention_weight_min": -37.23139953613281,
      "activations/layer16_attention_weight_max": 31.085100173950195,
      "activations/layer16_attention_weight_min": -28.965572357177734,
      "activations/layer17_attention_weight_max": 30.603952407836914,
      "activations/layer17_attention_weight_min": -27.810895919799805,
      "activations/layer18_attention_weight_max": 31.0499210357666,
      "activations/layer18_attention_weight_min": -26.736000061035156,
      "activations/layer19_attention_weight_max": 33.92574691772461,
      "activations/layer19_attention_weight_min": -30.918655395507812,
      "activations/layer1_attention_weight_max": 16.854692459106445,
      "activations/layer1_attention_weight_min": -15.094010353088379,
      "activations/layer20_attention_weight_max": 28.465103149414062,
      "activations/layer20_attention_weight_min": -26.789710998535156,
      "activations/layer21_attention_weight_max": 28.659006118774414,
      "activations/layer21_attention_weight_min": -25.375011444091797,
      "activations/layer22_attention_weight_max": 37.82036209106445,
      "activations/layer22_attention_weight_min": -32.71815490722656,
      "activations/layer23_attention_weight_max": 33.348541259765625,
      "activations/layer23_attention_weight_min": -25.561193466186523,
      "activations/layer2_attention_weight_max": 31.76942253112793,
      "activations/layer2_attention_weight_min": -32.025970458984375,
      "activations/layer3_attention_weight_max": 97.35575866699219,
      "activations/layer3_attention_weight_min": -96.16670989990234,
      "activations/layer4_attention_weight_max": 106.72315979003906,
      "activations/layer4_attention_weight_min": -106.9653091430664,
      "activations/layer5_attention_weight_max": 50.52312469482422,
      "activations/layer5_attention_weight_min": -60.97276306152344,
      "activations/layer6_attention_weight_max": 44.55777359008789,
      "activations/layer6_attention_weight_min": -45.45317459106445,
      "activations/layer7_attention_weight_max": 98.82171630859375,
      "activations/layer7_attention_weight_min": -90.90955352783203,
      "activations/layer8_attention_weight_max": 43.33968734741211,
      "activations/layer8_attention_weight_min": -41.69647216796875,
      "activations/layer9_attention_weight_max": 32.57172775268555,
      "activations/layer9_attention_weight_min": -33.81169128417969,
      "epoch": 16.01,
      "learning_rate": 4.726174242424242e-05,
      "loss": 2.7297,
      "step": 275450
    },
    {
      "activations/layer0_attention_weight_max": 15.774433135986328,
      "activations/layer0_attention_weight_min": -13.569404602050781,
      "activations/layer10_attention_weight_max": 34.580081939697266,
      "activations/layer10_attention_weight_min": -34.266868591308594,
      "activations/layer11_attention_weight_max": 33.05369567871094,
      "activations/layer11_attention_weight_min": -34.34450149536133,
      "activations/layer12_attention_weight_max": 26.225099563598633,
      "activations/layer12_attention_weight_min": -27.01658821105957,
      "activations/layer13_attention_weight_max": 38.70625686645508,
      "activations/layer13_attention_weight_min": -37.643959045410156,
      "activations/layer14_attention_weight_max": 42.22665786743164,
      "activations/layer14_attention_weight_min": -38.18210983276367,
      "activations/layer15_attention_weight_max": 37.82014846801758,
      "activations/layer15_attention_weight_min": -34.723201751708984,
      "activations/layer16_attention_weight_max": 31.828853607177734,
      "activations/layer16_attention_weight_min": -28.256196975708008,
      "activations/layer17_attention_weight_max": 31.14670753479004,
      "activations/layer17_attention_weight_min": -24.623085021972656,
      "activations/layer18_attention_weight_max": 30.940195083618164,
      "activations/layer18_attention_weight_min": -24.250473022460938,
      "activations/layer19_attention_weight_max": 36.26267623901367,
      "activations/layer19_attention_weight_min": -33.37313461303711,
      "activations/layer1_attention_weight_max": 17.08012580871582,
      "activations/layer1_attention_weight_min": -15.675375938415527,
      "activations/layer20_attention_weight_max": 29.741527557373047,
      "activations/layer20_attention_weight_min": -26.917545318603516,
      "activations/layer21_attention_weight_max": 27.740758895874023,
      "activations/layer21_attention_weight_min": -24.920080184936523,
      "activations/layer22_attention_weight_max": 41.25812530517578,
      "activations/layer22_attention_weight_min": -29.775178909301758,
      "activations/layer23_attention_weight_max": 32.28788757324219,
      "activations/layer23_attention_weight_min": -29.54037094116211,
      "activations/layer2_attention_weight_max": 34.02384948730469,
      "activations/layer2_attention_weight_min": -32.126220703125,
      "activations/layer3_attention_weight_max": 100.75869750976562,
      "activations/layer3_attention_weight_min": -97.36070251464844,
      "activations/layer4_attention_weight_max": 109.02037811279297,
      "activations/layer4_attention_weight_min": -106.40386962890625,
      "activations/layer5_attention_weight_max": 48.186195373535156,
      "activations/layer5_attention_weight_min": -62.13425827026367,
      "activations/layer6_attention_weight_max": 43.19172286987305,
      "activations/layer6_attention_weight_min": -47.525882720947266,
      "activations/layer7_attention_weight_max": 86.98109436035156,
      "activations/layer7_attention_weight_min": -92.20181274414062,
      "activations/layer8_attention_weight_max": 40.904109954833984,
      "activations/layer8_attention_weight_min": -42.22592544555664,
      "activations/layer9_attention_weight_max": 33.11735916137695,
      "activations/layer9_attention_weight_min": -34.72917175292969,
      "epoch": 16.01,
      "learning_rate": 4.724280303030302e-05,
      "loss": 2.7422,
      "step": 275500
    },
    {
      "activations/layer0_attention_weight_max": 16.329021453857422,
      "activations/layer0_attention_weight_min": -13.005864143371582,
      "activations/layer10_attention_weight_max": 33.31805419921875,
      "activations/layer10_attention_weight_min": -33.94963455200195,
      "activations/layer11_attention_weight_max": 31.857494354248047,
      "activations/layer11_attention_weight_min": -31.763744354248047,
      "activations/layer12_attention_weight_max": 27.873884201049805,
      "activations/layer12_attention_weight_min": -26.11168670654297,
      "activations/layer13_attention_weight_max": 38.176490783691406,
      "activations/layer13_attention_weight_min": -36.624412536621094,
      "activations/layer14_attention_weight_max": 46.80253601074219,
      "activations/layer14_attention_weight_min": -38.01786422729492,
      "activations/layer15_attention_weight_max": 39.05531692504883,
      "activations/layer15_attention_weight_min": -36.44567108154297,
      "activations/layer16_attention_weight_max": 31.227449417114258,
      "activations/layer16_attention_weight_min": -28.563859939575195,
      "activations/layer17_attention_weight_max": 30.86182403564453,
      "activations/layer17_attention_weight_min": -26.695846557617188,
      "activations/layer18_attention_weight_max": 35.472625732421875,
      "activations/layer18_attention_weight_min": -25.968196868896484,
      "activations/layer19_attention_weight_max": 35.496395111083984,
      "activations/layer19_attention_weight_min": -30.56780433654785,
      "activations/layer1_attention_weight_max": 15.722846984863281,
      "activations/layer1_attention_weight_min": -15.367264747619629,
      "activations/layer20_attention_weight_max": 34.360347747802734,
      "activations/layer20_attention_weight_min": -26.029766082763672,
      "activations/layer21_attention_weight_max": 35.325660705566406,
      "activations/layer21_attention_weight_min": -26.043607711791992,
      "activations/layer22_attention_weight_max": 47.75885772705078,
      "activations/layer22_attention_weight_min": -33.067100524902344,
      "activations/layer23_attention_weight_max": 35.70634460449219,
      "activations/layer23_attention_weight_min": -25.599258422851562,
      "activations/layer2_attention_weight_max": 32.9575309753418,
      "activations/layer2_attention_weight_min": -32.00270462036133,
      "activations/layer3_attention_weight_max": 94.2149658203125,
      "activations/layer3_attention_weight_min": -94.70038604736328,
      "activations/layer4_attention_weight_max": 106.64686584472656,
      "activations/layer4_attention_weight_min": -108.80904388427734,
      "activations/layer5_attention_weight_max": 52.90130615234375,
      "activations/layer5_attention_weight_min": -60.71715545654297,
      "activations/layer6_attention_weight_max": 42.77735137939453,
      "activations/layer6_attention_weight_min": -44.91801834106445,
      "activations/layer7_attention_weight_max": 91.29071044921875,
      "activations/layer7_attention_weight_min": -94.12521362304688,
      "activations/layer8_attention_weight_max": 39.84589385986328,
      "activations/layer8_attention_weight_min": -42.01697540283203,
      "activations/layer9_attention_weight_max": 32.44368362426758,
      "activations/layer9_attention_weight_min": -34.35374069213867,
      "epoch": 16.01,
      "learning_rate": 4.7223863636363635e-05,
      "loss": 2.7337,
      "step": 275550
    },
    {
      "activations/layer0_attention_weight_max": 16.69107437133789,
      "activations/layer0_attention_weight_min": -12.8526611328125,
      "activations/layer10_attention_weight_max": 30.29342269897461,
      "activations/layer10_attention_weight_min": -32.426727294921875,
      "activations/layer11_attention_weight_max": 30.455162048339844,
      "activations/layer11_attention_weight_min": -31.94887351989746,
      "activations/layer12_attention_weight_max": 25.990070343017578,
      "activations/layer12_attention_weight_min": -25.79193687438965,
      "activations/layer13_attention_weight_max": 38.14289093017578,
      "activations/layer13_attention_weight_min": -34.269989013671875,
      "activations/layer14_attention_weight_max": 42.99307632446289,
      "activations/layer14_attention_weight_min": -35.73123550415039,
      "activations/layer15_attention_weight_max": 36.81503677368164,
      "activations/layer15_attention_weight_min": -34.37717056274414,
      "activations/layer16_attention_weight_max": 30.34177589416504,
      "activations/layer16_attention_weight_min": -28.472171783447266,
      "activations/layer17_attention_weight_max": 31.14529800415039,
      "activations/layer17_attention_weight_min": -26.95989227294922,
      "activations/layer18_attention_weight_max": 33.54690170288086,
      "activations/layer18_attention_weight_min": -25.99789810180664,
      "activations/layer19_attention_weight_max": 33.668663024902344,
      "activations/layer19_attention_weight_min": -31.983678817749023,
      "activations/layer1_attention_weight_max": 15.812381744384766,
      "activations/layer1_attention_weight_min": -16.220212936401367,
      "activations/layer20_attention_weight_max": 34.219722747802734,
      "activations/layer20_attention_weight_min": -25.75008201599121,
      "activations/layer21_attention_weight_max": 31.71303367614746,
      "activations/layer21_attention_weight_min": -25.805078506469727,
      "activations/layer22_attention_weight_max": 39.02557373046875,
      "activations/layer22_attention_weight_min": -31.94727897644043,
      "activations/layer23_attention_weight_max": 33.98384475708008,
      "activations/layer23_attention_weight_min": -26.30660057067871,
      "activations/layer2_attention_weight_max": 32.83599853515625,
      "activations/layer2_attention_weight_min": -31.34088897705078,
      "activations/layer3_attention_weight_max": 94.80470275878906,
      "activations/layer3_attention_weight_min": -92.45215606689453,
      "activations/layer4_attention_weight_max": 102.17340087890625,
      "activations/layer4_attention_weight_min": -106.86956787109375,
      "activations/layer5_attention_weight_max": 51.29810333251953,
      "activations/layer5_attention_weight_min": -59.42927551269531,
      "activations/layer6_attention_weight_max": 43.27180862426758,
      "activations/layer6_attention_weight_min": -46.89569854736328,
      "activations/layer7_attention_weight_max": 84.61820220947266,
      "activations/layer7_attention_weight_min": -84.8947982788086,
      "activations/layer8_attention_weight_max": 39.08806228637695,
      "activations/layer8_attention_weight_min": -41.829734802246094,
      "activations/layer9_attention_weight_max": 29.741199493408203,
      "activations/layer9_attention_weight_min": -32.599449157714844,
      "epoch": 16.01,
      "learning_rate": 4.7204924242424236e-05,
      "loss": 2.7437,
      "step": 275600
    },
    {
      "activations/layer0_attention_weight_max": 16.81679344177246,
      "activations/layer0_attention_weight_min": -12.53866195678711,
      "activations/layer10_attention_weight_max": 34.37300109863281,
      "activations/layer10_attention_weight_min": -34.86088180541992,
      "activations/layer11_attention_weight_max": 33.408409118652344,
      "activations/layer11_attention_weight_min": -33.4239616394043,
      "activations/layer12_attention_weight_max": 27.64550018310547,
      "activations/layer12_attention_weight_min": -28.243677139282227,
      "activations/layer13_attention_weight_max": 39.431602478027344,
      "activations/layer13_attention_weight_min": -36.39662551879883,
      "activations/layer14_attention_weight_max": 44.90215301513672,
      "activations/layer14_attention_weight_min": -41.91187286376953,
      "activations/layer15_attention_weight_max": 45.238712310791016,
      "activations/layer15_attention_weight_min": -37.403053283691406,
      "activations/layer16_attention_weight_max": 33.670005798339844,
      "activations/layer16_attention_weight_min": -27.689098358154297,
      "activations/layer17_attention_weight_max": 32.51906204223633,
      "activations/layer17_attention_weight_min": -27.571847915649414,
      "activations/layer18_attention_weight_max": 34.318031311035156,
      "activations/layer18_attention_weight_min": -25.950611114501953,
      "activations/layer19_attention_weight_max": 38.0393180847168,
      "activations/layer19_attention_weight_min": -32.657405853271484,
      "activations/layer1_attention_weight_max": 16.323335647583008,
      "activations/layer1_attention_weight_min": -14.375523567199707,
      "activations/layer20_attention_weight_max": 32.105281829833984,
      "activations/layer20_attention_weight_min": -26.57518196105957,
      "activations/layer21_attention_weight_max": 33.158748626708984,
      "activations/layer21_attention_weight_min": -25.290348052978516,
      "activations/layer22_attention_weight_max": 46.45029830932617,
      "activations/layer22_attention_weight_min": -33.72111892700195,
      "activations/layer23_attention_weight_max": 37.65713119506836,
      "activations/layer23_attention_weight_min": -25.816892623901367,
      "activations/layer2_attention_weight_max": 32.3411865234375,
      "activations/layer2_attention_weight_min": -32.0821418762207,
      "activations/layer3_attention_weight_max": 97.18403625488281,
      "activations/layer3_attention_weight_min": -93.38701629638672,
      "activations/layer4_attention_weight_max": 104.32069396972656,
      "activations/layer4_attention_weight_min": -102.5835952758789,
      "activations/layer5_attention_weight_max": 53.195579528808594,
      "activations/layer5_attention_weight_min": -61.12910461425781,
      "activations/layer6_attention_weight_max": 42.339210510253906,
      "activations/layer6_attention_weight_min": -45.07392120361328,
      "activations/layer7_attention_weight_max": 90.27725982666016,
      "activations/layer7_attention_weight_min": -90.83283996582031,
      "activations/layer8_attention_weight_max": 46.15222930908203,
      "activations/layer8_attention_weight_min": -44.089542388916016,
      "activations/layer9_attention_weight_max": 31.706998825073242,
      "activations/layer9_attention_weight_min": -34.14768600463867,
      "epoch": 16.02,
      "learning_rate": 4.7185984848484844e-05,
      "loss": 2.735,
      "step": 275650
    },
    {
      "activations/layer0_attention_weight_max": 15.912115097045898,
      "activations/layer0_attention_weight_min": -13.429944038391113,
      "activations/layer10_attention_weight_max": 34.52386474609375,
      "activations/layer10_attention_weight_min": -36.13779830932617,
      "activations/layer11_attention_weight_max": 36.23090744018555,
      "activations/layer11_attention_weight_min": -36.865299224853516,
      "activations/layer12_attention_weight_max": 25.466093063354492,
      "activations/layer12_attention_weight_min": -25.82783317565918,
      "activations/layer13_attention_weight_max": 37.05083465576172,
      "activations/layer13_attention_weight_min": -34.17546844482422,
      "activations/layer14_attention_weight_max": 39.09358215332031,
      "activations/layer14_attention_weight_min": -35.44274139404297,
      "activations/layer15_attention_weight_max": 36.3094596862793,
      "activations/layer15_attention_weight_min": -33.736183166503906,
      "activations/layer16_attention_weight_max": 30.00798225402832,
      "activations/layer16_attention_weight_min": -28.172101974487305,
      "activations/layer17_attention_weight_max": 27.56342124938965,
      "activations/layer17_attention_weight_min": -26.137371063232422,
      "activations/layer18_attention_weight_max": 30.125,
      "activations/layer18_attention_weight_min": -26.26706314086914,
      "activations/layer19_attention_weight_max": 32.39930725097656,
      "activations/layer19_attention_weight_min": -29.411666870117188,
      "activations/layer1_attention_weight_max": 16.720823287963867,
      "activations/layer1_attention_weight_min": -15.517744064331055,
      "activations/layer20_attention_weight_max": 28.252056121826172,
      "activations/layer20_attention_weight_min": -24.675386428833008,
      "activations/layer21_attention_weight_max": 30.00356101989746,
      "activations/layer21_attention_weight_min": -23.405424118041992,
      "activations/layer22_attention_weight_max": 38.79008865356445,
      "activations/layer22_attention_weight_min": -28.946205139160156,
      "activations/layer23_attention_weight_max": 29.65631866455078,
      "activations/layer23_attention_weight_min": -25.52899169921875,
      "activations/layer2_attention_weight_max": 32.57929992675781,
      "activations/layer2_attention_weight_min": -33.06093978881836,
      "activations/layer3_attention_weight_max": 102.70369720458984,
      "activations/layer3_attention_weight_min": -100.27091217041016,
      "activations/layer4_attention_weight_max": 109.83821105957031,
      "activations/layer4_attention_weight_min": -111.21849822998047,
      "activations/layer5_attention_weight_max": 53.181488037109375,
      "activations/layer5_attention_weight_min": -60.0066032409668,
      "activations/layer6_attention_weight_max": 45.851890563964844,
      "activations/layer6_attention_weight_min": -47.63692855834961,
      "activations/layer7_attention_weight_max": 94.18470001220703,
      "activations/layer7_attention_weight_min": -104.4176254272461,
      "activations/layer8_attention_weight_max": 41.73994064331055,
      "activations/layer8_attention_weight_min": -45.8378791809082,
      "activations/layer9_attention_weight_max": 33.29165267944336,
      "activations/layer9_attention_weight_min": -36.31033706665039,
      "epoch": 16.02,
      "learning_rate": 4.7167045454545446e-05,
      "loss": 2.7328,
      "step": 275700
    },
    {
      "activations/layer0_attention_weight_max": 16.17568588256836,
      "activations/layer0_attention_weight_min": -13.390105247497559,
      "activations/layer10_attention_weight_max": 33.67218017578125,
      "activations/layer10_attention_weight_min": -32.0593147277832,
      "activations/layer11_attention_weight_max": 33.31987380981445,
      "activations/layer11_attention_weight_min": -31.974929809570312,
      "activations/layer12_attention_weight_max": 25.913448333740234,
      "activations/layer12_attention_weight_min": -24.89847755432129,
      "activations/layer13_attention_weight_max": 36.600074768066406,
      "activations/layer13_attention_weight_min": -33.82575225830078,
      "activations/layer14_attention_weight_max": 44.62744140625,
      "activations/layer14_attention_weight_min": -36.96867370605469,
      "activations/layer15_attention_weight_max": 37.44425964355469,
      "activations/layer15_attention_weight_min": -33.75288772583008,
      "activations/layer16_attention_weight_max": 31.359127044677734,
      "activations/layer16_attention_weight_min": -27.593101501464844,
      "activations/layer17_attention_weight_max": 30.744882583618164,
      "activations/layer17_attention_weight_min": -26.39789390563965,
      "activations/layer18_attention_weight_max": 30.545812606811523,
      "activations/layer18_attention_weight_min": -25.93669319152832,
      "activations/layer19_attention_weight_max": 36.84379196166992,
      "activations/layer19_attention_weight_min": -30.266952514648438,
      "activations/layer1_attention_weight_max": 16.226133346557617,
      "activations/layer1_attention_weight_min": -14.875359535217285,
      "activations/layer20_attention_weight_max": 29.8475399017334,
      "activations/layer20_attention_weight_min": -25.750349044799805,
      "activations/layer21_attention_weight_max": 28.85366439819336,
      "activations/layer21_attention_weight_min": -24.787425994873047,
      "activations/layer22_attention_weight_max": 42.17998504638672,
      "activations/layer22_attention_weight_min": -30.88575553894043,
      "activations/layer23_attention_weight_max": 35.085044860839844,
      "activations/layer23_attention_weight_min": -27.17221450805664,
      "activations/layer2_attention_weight_max": 33.86003494262695,
      "activations/layer2_attention_weight_min": -33.615089416503906,
      "activations/layer3_attention_weight_max": 98.32441711425781,
      "activations/layer3_attention_weight_min": -98.28118896484375,
      "activations/layer4_attention_weight_max": 105.49867248535156,
      "activations/layer4_attention_weight_min": -109.41564178466797,
      "activations/layer5_attention_weight_max": 51.58058166503906,
      "activations/layer5_attention_weight_min": -63.45690155029297,
      "activations/layer6_attention_weight_max": 42.21116256713867,
      "activations/layer6_attention_weight_min": -45.089439392089844,
      "activations/layer7_attention_weight_max": 85.888916015625,
      "activations/layer7_attention_weight_min": -86.8856430053711,
      "activations/layer8_attention_weight_max": 38.41328430175781,
      "activations/layer8_attention_weight_min": -41.73539352416992,
      "activations/layer9_attention_weight_max": 31.17994499206543,
      "activations/layer9_attention_weight_min": -32.711036682128906,
      "epoch": 16.02,
      "learning_rate": 4.714810606060606e-05,
      "loss": 2.7306,
      "step": 275750
    },
    {
      "activations/layer0_attention_weight_max": 16.321195602416992,
      "activations/layer0_attention_weight_min": -12.376769065856934,
      "activations/layer10_attention_weight_max": 49.77543258666992,
      "activations/layer10_attention_weight_min": -47.13837814331055,
      "activations/layer11_attention_weight_max": 53.64183044433594,
      "activations/layer11_attention_weight_min": -52.5538330078125,
      "activations/layer12_attention_weight_max": 31.174503326416016,
      "activations/layer12_attention_weight_min": -29.46475601196289,
      "activations/layer13_attention_weight_max": 45.42487716674805,
      "activations/layer13_attention_weight_min": -39.041160583496094,
      "activations/layer14_attention_weight_max": 50.63490676879883,
      "activations/layer14_attention_weight_min": -42.829132080078125,
      "activations/layer15_attention_weight_max": 44.508548736572266,
      "activations/layer15_attention_weight_min": -39.96941375732422,
      "activations/layer16_attention_weight_max": 34.258094787597656,
      "activations/layer16_attention_weight_min": -28.689313888549805,
      "activations/layer17_attention_weight_max": 29.8261661529541,
      "activations/layer17_attention_weight_min": -25.03972625732422,
      "activations/layer18_attention_weight_max": 31.019906997680664,
      "activations/layer18_attention_weight_min": -24.243560791015625,
      "activations/layer19_attention_weight_max": 35.76984786987305,
      "activations/layer19_attention_weight_min": -30.803394317626953,
      "activations/layer1_attention_weight_max": 15.778786659240723,
      "activations/layer1_attention_weight_min": -14.716883659362793,
      "activations/layer20_attention_weight_max": 28.568700790405273,
      "activations/layer20_attention_weight_min": -25.587251663208008,
      "activations/layer21_attention_weight_max": 26.878698348999023,
      "activations/layer21_attention_weight_min": -23.90470314025879,
      "activations/layer22_attention_weight_max": 42.542213439941406,
      "activations/layer22_attention_weight_min": -30.855451583862305,
      "activations/layer23_attention_weight_max": 30.821910858154297,
      "activations/layer23_attention_weight_min": -24.88924789428711,
      "activations/layer2_attention_weight_max": 38.13797378540039,
      "activations/layer2_attention_weight_min": -33.760616302490234,
      "activations/layer3_attention_weight_max": 103.9267578125,
      "activations/layer3_attention_weight_min": -100.66333770751953,
      "activations/layer4_attention_weight_max": 111.78279876708984,
      "activations/layer4_attention_weight_min": -110.55145263671875,
      "activations/layer5_attention_weight_max": 51.875877380371094,
      "activations/layer5_attention_weight_min": -62.43732833862305,
      "activations/layer6_attention_weight_max": 49.51245880126953,
      "activations/layer6_attention_weight_min": -46.65453338623047,
      "activations/layer7_attention_weight_max": 120.89790344238281,
      "activations/layer7_attention_weight_min": -107.34894561767578,
      "activations/layer8_attention_weight_max": 53.7408332824707,
      "activations/layer8_attention_weight_min": -50.008460998535156,
      "activations/layer9_attention_weight_max": 45.32963562011719,
      "activations/layer9_attention_weight_min": -43.80894088745117,
      "epoch": 16.03,
      "learning_rate": 4.712916666666666e-05,
      "loss": 2.7614,
      "step": 275800
    },
    {
      "activations/layer0_attention_weight_max": 15.855388641357422,
      "activations/layer0_attention_weight_min": -14.044805526733398,
      "activations/layer10_attention_weight_max": 33.41313552856445,
      "activations/layer10_attention_weight_min": -34.54517364501953,
      "activations/layer11_attention_weight_max": 33.82335662841797,
      "activations/layer11_attention_weight_min": -33.285789489746094,
      "activations/layer12_attention_weight_max": 31.61132049560547,
      "activations/layer12_attention_weight_min": -26.41448402404785,
      "activations/layer13_attention_weight_max": 47.39469528198242,
      "activations/layer13_attention_weight_min": -37.78093719482422,
      "activations/layer14_attention_weight_max": 45.74538040161133,
      "activations/layer14_attention_weight_min": -37.30007553100586,
      "activations/layer15_attention_weight_max": 42.42166519165039,
      "activations/layer15_attention_weight_min": -36.184288024902344,
      "activations/layer16_attention_weight_max": 33.71350860595703,
      "activations/layer16_attention_weight_min": -30.077211380004883,
      "activations/layer17_attention_weight_max": 29.994062423706055,
      "activations/layer17_attention_weight_min": -25.593137741088867,
      "activations/layer18_attention_weight_max": 31.742168426513672,
      "activations/layer18_attention_weight_min": -24.43191146850586,
      "activations/layer19_attention_weight_max": 36.09893798828125,
      "activations/layer19_attention_weight_min": -35.16167449951172,
      "activations/layer1_attention_weight_max": 17.45758819580078,
      "activations/layer1_attention_weight_min": -16.588470458984375,
      "activations/layer20_attention_weight_max": 32.04378128051758,
      "activations/layer20_attention_weight_min": -28.1649227142334,
      "activations/layer21_attention_weight_max": 29.326738357543945,
      "activations/layer21_attention_weight_min": -26.176586151123047,
      "activations/layer22_attention_weight_max": 41.37895584106445,
      "activations/layer22_attention_weight_min": -36.25954055786133,
      "activations/layer23_attention_weight_max": 33.47184753417969,
      "activations/layer23_attention_weight_min": -26.97045135498047,
      "activations/layer2_attention_weight_max": 34.267799377441406,
      "activations/layer2_attention_weight_min": -33.41636657714844,
      "activations/layer3_attention_weight_max": 95.08657836914062,
      "activations/layer3_attention_weight_min": -94.91702270507812,
      "activations/layer4_attention_weight_max": 103.29273986816406,
      "activations/layer4_attention_weight_min": -106.28592681884766,
      "activations/layer5_attention_weight_max": 53.03330612182617,
      "activations/layer5_attention_weight_min": -64.80117797851562,
      "activations/layer6_attention_weight_max": 42.47304153442383,
      "activations/layer6_attention_weight_min": -46.39308547973633,
      "activations/layer7_attention_weight_max": 92.27082824707031,
      "activations/layer7_attention_weight_min": -92.24870300292969,
      "activations/layer8_attention_weight_max": 39.96199417114258,
      "activations/layer8_attention_weight_min": -41.9465217590332,
      "activations/layer9_attention_weight_max": 32.782928466796875,
      "activations/layer9_attention_weight_min": -33.08345031738281,
      "epoch": 16.03,
      "learning_rate": 4.7110227272727264e-05,
      "loss": 2.7443,
      "step": 275850
    },
    {
      "activations/layer0_attention_weight_max": 15.662311553955078,
      "activations/layer0_attention_weight_min": -12.224991798400879,
      "activations/layer10_attention_weight_max": 33.74774932861328,
      "activations/layer10_attention_weight_min": -32.34177017211914,
      "activations/layer11_attention_weight_max": 33.59516906738281,
      "activations/layer11_attention_weight_min": -32.40919876098633,
      "activations/layer12_attention_weight_max": 26.30084800720215,
      "activations/layer12_attention_weight_min": -26.12186050415039,
      "activations/layer13_attention_weight_max": 37.718528747558594,
      "activations/layer13_attention_weight_min": -36.200408935546875,
      "activations/layer14_attention_weight_max": 41.15275955200195,
      "activations/layer14_attention_weight_min": -39.41226577758789,
      "activations/layer15_attention_weight_max": 39.84500503540039,
      "activations/layer15_attention_weight_min": -35.542110443115234,
      "activations/layer16_attention_weight_max": 29.601194381713867,
      "activations/layer16_attention_weight_min": -28.19903564453125,
      "activations/layer17_attention_weight_max": 26.393096923828125,
      "activations/layer17_attention_weight_min": -24.869590759277344,
      "activations/layer18_attention_weight_max": 31.072635650634766,
      "activations/layer18_attention_weight_min": -24.22694969177246,
      "activations/layer19_attention_weight_max": 31.95917320251465,
      "activations/layer19_attention_weight_min": -31.503864288330078,
      "activations/layer1_attention_weight_max": 16.368925094604492,
      "activations/layer1_attention_weight_min": -15.087514877319336,
      "activations/layer20_attention_weight_max": 26.319297790527344,
      "activations/layer20_attention_weight_min": -25.174327850341797,
      "activations/layer21_attention_weight_max": 26.748077392578125,
      "activations/layer21_attention_weight_min": -23.70072364807129,
      "activations/layer22_attention_weight_max": 40.72428512573242,
      "activations/layer22_attention_weight_min": -29.821910858154297,
      "activations/layer23_attention_weight_max": 33.452125549316406,
      "activations/layer23_attention_weight_min": -26.868236541748047,
      "activations/layer2_attention_weight_max": 32.672019958496094,
      "activations/layer2_attention_weight_min": -33.09800338745117,
      "activations/layer3_attention_weight_max": 97.36930084228516,
      "activations/layer3_attention_weight_min": -98.62196350097656,
      "activations/layer4_attention_weight_max": 104.65047454833984,
      "activations/layer4_attention_weight_min": -112.65970611572266,
      "activations/layer5_attention_weight_max": 53.537261962890625,
      "activations/layer5_attention_weight_min": -61.49555587768555,
      "activations/layer6_attention_weight_max": 44.336204528808594,
      "activations/layer6_attention_weight_min": -45.55732345581055,
      "activations/layer7_attention_weight_max": 90.536865234375,
      "activations/layer7_attention_weight_min": -92.73160552978516,
      "activations/layer8_attention_weight_max": 41.832035064697266,
      "activations/layer8_attention_weight_min": -41.63544845581055,
      "activations/layer9_attention_weight_max": 31.9854736328125,
      "activations/layer9_attention_weight_min": -34.69211959838867,
      "epoch": 16.03,
      "learning_rate": 4.709128787878788e-05,
      "loss": 2.7508,
      "step": 275900
    },
    {
      "activations/layer0_attention_weight_max": 15.593157768249512,
      "activations/layer0_attention_weight_min": -14.057245254516602,
      "activations/layer10_attention_weight_max": 31.57334327697754,
      "activations/layer10_attention_weight_min": -32.00189971923828,
      "activations/layer11_attention_weight_max": 31.30603790283203,
      "activations/layer11_attention_weight_min": -32.068511962890625,
      "activations/layer12_attention_weight_max": 30.037273406982422,
      "activations/layer12_attention_weight_min": -25.334075927734375,
      "activations/layer13_attention_weight_max": 41.87263870239258,
      "activations/layer13_attention_weight_min": -35.71238708496094,
      "activations/layer14_attention_weight_max": 42.58778762817383,
      "activations/layer14_attention_weight_min": -39.8421516418457,
      "activations/layer15_attention_weight_max": 36.0285530090332,
      "activations/layer15_attention_weight_min": -35.73014450073242,
      "activations/layer16_attention_weight_max": 27.639667510986328,
      "activations/layer16_attention_weight_min": -27.645248413085938,
      "activations/layer17_attention_weight_max": 30.08390235900879,
      "activations/layer17_attention_weight_min": -23.857685089111328,
      "activations/layer18_attention_weight_max": 30.493404388427734,
      "activations/layer18_attention_weight_min": -24.43025779724121,
      "activations/layer19_attention_weight_max": 31.293371200561523,
      "activations/layer19_attention_weight_min": -30.506473541259766,
      "activations/layer1_attention_weight_max": 16.878202438354492,
      "activations/layer1_attention_weight_min": -15.635926246643066,
      "activations/layer20_attention_weight_max": 28.24916648864746,
      "activations/layer20_attention_weight_min": -24.372716903686523,
      "activations/layer21_attention_weight_max": 30.235164642333984,
      "activations/layer21_attention_weight_min": -24.203933715820312,
      "activations/layer22_attention_weight_max": 38.962547302246094,
      "activations/layer22_attention_weight_min": -28.91176414489746,
      "activations/layer23_attention_weight_max": 30.223514556884766,
      "activations/layer23_attention_weight_min": -24.658668518066406,
      "activations/layer2_attention_weight_max": 33.52524948120117,
      "activations/layer2_attention_weight_min": -33.89318084716797,
      "activations/layer3_attention_weight_max": 91.09751892089844,
      "activations/layer3_attention_weight_min": -98.19445037841797,
      "activations/layer4_attention_weight_max": 100.79607391357422,
      "activations/layer4_attention_weight_min": -108.18153381347656,
      "activations/layer5_attention_weight_max": 48.027801513671875,
      "activations/layer5_attention_weight_min": -61.453651428222656,
      "activations/layer6_attention_weight_max": 41.82746887207031,
      "activations/layer6_attention_weight_min": -44.97525405883789,
      "activations/layer7_attention_weight_max": 88.42633819580078,
      "activations/layer7_attention_weight_min": -86.74496459960938,
      "activations/layer8_attention_weight_max": 37.714599609375,
      "activations/layer8_attention_weight_min": -39.76354217529297,
      "activations/layer9_attention_weight_max": 29.125385284423828,
      "activations/layer9_attention_weight_min": -31.980274200439453,
      "epoch": 16.03,
      "learning_rate": 4.707234848484848e-05,
      "loss": 2.7326,
      "step": 275950
    },
    {
      "activations/layer0_attention_weight_max": 16.028261184692383,
      "activations/layer0_attention_weight_min": -12.800460815429688,
      "activations/layer10_attention_weight_max": 36.116737365722656,
      "activations/layer10_attention_weight_min": -35.211669921875,
      "activations/layer11_attention_weight_max": 36.09064483642578,
      "activations/layer11_attention_weight_min": -36.43799591064453,
      "activations/layer12_attention_weight_max": 30.779970169067383,
      "activations/layer12_attention_weight_min": -27.372886657714844,
      "activations/layer13_attention_weight_max": 40.59028625488281,
      "activations/layer13_attention_weight_min": -36.059391021728516,
      "activations/layer14_attention_weight_max": 45.6463623046875,
      "activations/layer14_attention_weight_min": -39.92220687866211,
      "activations/layer15_attention_weight_max": 44.57429504394531,
      "activations/layer15_attention_weight_min": -36.69776153564453,
      "activations/layer16_attention_weight_max": 32.73796844482422,
      "activations/layer16_attention_weight_min": -27.344257354736328,
      "activations/layer17_attention_weight_max": 32.84523010253906,
      "activations/layer17_attention_weight_min": -26.375829696655273,
      "activations/layer18_attention_weight_max": 35.18688201904297,
      "activations/layer18_attention_weight_min": -26.517045974731445,
      "activations/layer19_attention_weight_max": 39.107234954833984,
      "activations/layer19_attention_weight_min": -31.674835205078125,
      "activations/layer1_attention_weight_max": 15.810321807861328,
      "activations/layer1_attention_weight_min": -16.019466400146484,
      "activations/layer20_attention_weight_max": 30.31077766418457,
      "activations/layer20_attention_weight_min": -26.640718460083008,
      "activations/layer21_attention_weight_max": 32.3770866394043,
      "activations/layer21_attention_weight_min": -24.369842529296875,
      "activations/layer22_attention_weight_max": 48.615081787109375,
      "activations/layer22_attention_weight_min": -31.15308380126953,
      "activations/layer23_attention_weight_max": 36.940391540527344,
      "activations/layer23_attention_weight_min": -23.204994201660156,
      "activations/layer2_attention_weight_max": 34.68548583984375,
      "activations/layer2_attention_weight_min": -35.58222198486328,
      "activations/layer3_attention_weight_max": 98.74343872070312,
      "activations/layer3_attention_weight_min": -103.20502471923828,
      "activations/layer4_attention_weight_max": 108.13189697265625,
      "activations/layer4_attention_weight_min": -109.50528717041016,
      "activations/layer5_attention_weight_max": 50.13235092163086,
      "activations/layer5_attention_weight_min": -60.00200653076172,
      "activations/layer6_attention_weight_max": 44.58409881591797,
      "activations/layer6_attention_weight_min": -45.16144943237305,
      "activations/layer7_attention_weight_max": 95.37513732910156,
      "activations/layer7_attention_weight_min": -93.30464935302734,
      "activations/layer8_attention_weight_max": 42.77250671386719,
      "activations/layer8_attention_weight_min": -44.103485107421875,
      "activations/layer9_attention_weight_max": 33.117610931396484,
      "activations/layer9_attention_weight_min": -34.12026596069336,
      "epoch": 16.04,
      "learning_rate": 4.705340909090909e-05,
      "loss": 2.733,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.5508,
      "eval_samples_per_second": 502.173,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.5508,
      "eval_openwebtext_samples_per_second": 502.173,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 2.0101,
      "eval_wikitext_samples_per_second": 226.856,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_lambada_loss": 2.4453125,
      "eval_lambada_ppl": 11.534153458662256,
      "eval_lambada_runtime": 9.6165,
      "eval_lambada_samples_per_second": 506.318,
      "step": 276000
    },
    {
      "activations/layer0_attention_weight_max": 15.634661674499512,
      "activations/layer0_attention_weight_min": -13.681970596313477,
      "activations/layer10_attention_weight_max": 37.853145599365234,
      "activations/layer10_attention_weight_min": -35.739742279052734,
      "activations/layer11_attention_weight_max": 36.118408203125,
      "activations/layer11_attention_weight_min": -35.82973098754883,
      "activations/layer12_attention_weight_max": 27.103776931762695,
      "activations/layer12_attention_weight_min": -25.997255325317383,
      "activations/layer13_attention_weight_max": 41.918853759765625,
      "activations/layer13_attention_weight_min": -35.47230911254883,
      "activations/layer14_attention_weight_max": 43.46974563598633,
      "activations/layer14_attention_weight_min": -38.62087631225586,
      "activations/layer15_attention_weight_max": 45.45105743408203,
      "activations/layer15_attention_weight_min": -36.44987106323242,
      "activations/layer16_attention_weight_max": 30.276782989501953,
      "activations/layer16_attention_weight_min": -30.382959365844727,
      "activations/layer17_attention_weight_max": 31.442806243896484,
      "activations/layer17_attention_weight_min": -26.37936782836914,
      "activations/layer18_attention_weight_max": 29.85211753845215,
      "activations/layer18_attention_weight_min": -25.35144805908203,
      "activations/layer19_attention_weight_max": 33.43760299682617,
      "activations/layer19_attention_weight_min": -30.145456314086914,
      "activations/layer1_attention_weight_max": 15.597817420959473,
      "activations/layer1_attention_weight_min": -16.868053436279297,
      "activations/layer20_attention_weight_max": 29.558568954467773,
      "activations/layer20_attention_weight_min": -25.61965560913086,
      "activations/layer21_attention_weight_max": 28.594554901123047,
      "activations/layer21_attention_weight_min": -23.801549911499023,
      "activations/layer22_attention_weight_max": 43.99591827392578,
      "activations/layer22_attention_weight_min": -31.63344955444336,
      "activations/layer23_attention_weight_max": 31.895023345947266,
      "activations/layer23_attention_weight_min": -25.523895263671875,
      "activations/layer2_attention_weight_max": 32.86688995361328,
      "activations/layer2_attention_weight_min": -33.711368560791016,
      "activations/layer3_attention_weight_max": 98.97987365722656,
      "activations/layer3_attention_weight_min": -98.41939544677734,
      "activations/layer4_attention_weight_max": 110.11921691894531,
      "activations/layer4_attention_weight_min": -107.7577896118164,
      "activations/layer5_attention_weight_max": 54.31555938720703,
      "activations/layer5_attention_weight_min": -60.140010833740234,
      "activations/layer6_attention_weight_max": 43.06694793701172,
      "activations/layer6_attention_weight_min": -47.10207748413086,
      "activations/layer7_attention_weight_max": 94.5050277709961,
      "activations/layer7_attention_weight_min": -97.14347839355469,
      "activations/layer8_attention_weight_max": 42.91468048095703,
      "activations/layer8_attention_weight_min": -43.82242202758789,
      "activations/layer9_attention_weight_max": 35.54750061035156,
      "activations/layer9_attention_weight_min": -33.745513916015625,
      "epoch": 16.04,
      "learning_rate": 4.703446969696969e-05,
      "loss": 2.7554,
      "step": 276050
    },
    {
      "activations/layer0_attention_weight_max": 15.981876373291016,
      "activations/layer0_attention_weight_min": -13.504415512084961,
      "activations/layer10_attention_weight_max": 36.30424880981445,
      "activations/layer10_attention_weight_min": -34.84720230102539,
      "activations/layer11_attention_weight_max": 34.604766845703125,
      "activations/layer11_attention_weight_min": -34.163734436035156,
      "activations/layer12_attention_weight_max": 27.294113159179688,
      "activations/layer12_attention_weight_min": -26.789884567260742,
      "activations/layer13_attention_weight_max": 39.36988830566406,
      "activations/layer13_attention_weight_min": -35.78038787841797,
      "activations/layer14_attention_weight_max": 40.86624526977539,
      "activations/layer14_attention_weight_min": -36.668033599853516,
      "activations/layer15_attention_weight_max": 38.359561920166016,
      "activations/layer15_attention_weight_min": -35.45444869995117,
      "activations/layer16_attention_weight_max": 30.050472259521484,
      "activations/layer16_attention_weight_min": -28.55113983154297,
      "activations/layer17_attention_weight_max": 28.71530532836914,
      "activations/layer17_attention_weight_min": -26.73508071899414,
      "activations/layer18_attention_weight_max": 31.9693546295166,
      "activations/layer18_attention_weight_min": -26.03124237060547,
      "activations/layer19_attention_weight_max": 35.59467315673828,
      "activations/layer19_attention_weight_min": -30.750701904296875,
      "activations/layer1_attention_weight_max": 15.883581161499023,
      "activations/layer1_attention_weight_min": -15.6283597946167,
      "activations/layer20_attention_weight_max": 29.631345748901367,
      "activations/layer20_attention_weight_min": -24.77303695678711,
      "activations/layer21_attention_weight_max": 28.7243709564209,
      "activations/layer21_attention_weight_min": -25.429025650024414,
      "activations/layer22_attention_weight_max": 41.580230712890625,
      "activations/layer22_attention_weight_min": -34.222862243652344,
      "activations/layer23_attention_weight_max": 36.48876190185547,
      "activations/layer23_attention_weight_min": -26.60792350769043,
      "activations/layer2_attention_weight_max": 33.54171371459961,
      "activations/layer2_attention_weight_min": -34.930877685546875,
      "activations/layer3_attention_weight_max": 98.77046203613281,
      "activations/layer3_attention_weight_min": -101.27062225341797,
      "activations/layer4_attention_weight_max": 108.5130386352539,
      "activations/layer4_attention_weight_min": -111.2298812866211,
      "activations/layer5_attention_weight_max": 49.8343620300293,
      "activations/layer5_attention_weight_min": -59.697200775146484,
      "activations/layer6_attention_weight_max": 43.650291442871094,
      "activations/layer6_attention_weight_min": -48.42502212524414,
      "activations/layer7_attention_weight_max": 93.46717071533203,
      "activations/layer7_attention_weight_min": -95.53755187988281,
      "activations/layer8_attention_weight_max": 44.6655158996582,
      "activations/layer8_attention_weight_min": -48.844940185546875,
      "activations/layer9_attention_weight_max": 33.98995590209961,
      "activations/layer9_attention_weight_min": -35.7066650390625,
      "epoch": 16.04,
      "learning_rate": 4.7015530303030305e-05,
      "loss": 2.7671,
      "step": 276100
    },
    {
      "activations/layer0_attention_weight_max": 15.73233699798584,
      "activations/layer0_attention_weight_min": -12.818343162536621,
      "activations/layer10_attention_weight_max": 30.916805267333984,
      "activations/layer10_attention_weight_min": -33.074615478515625,
      "activations/layer11_attention_weight_max": 30.366239547729492,
      "activations/layer11_attention_weight_min": -32.786048889160156,
      "activations/layer12_attention_weight_max": 24.856685638427734,
      "activations/layer12_attention_weight_min": -25.251054763793945,
      "activations/layer13_attention_weight_max": 37.56555938720703,
      "activations/layer13_attention_weight_min": -35.47506332397461,
      "activations/layer14_attention_weight_max": 41.540306091308594,
      "activations/layer14_attention_weight_min": -37.25802993774414,
      "activations/layer15_attention_weight_max": 35.33101272583008,
      "activations/layer15_attention_weight_min": -36.863365173339844,
      "activations/layer16_attention_weight_max": 30.00214195251465,
      "activations/layer16_attention_weight_min": -30.272342681884766,
      "activations/layer17_attention_weight_max": 29.15594482421875,
      "activations/layer17_attention_weight_min": -25.628517150878906,
      "activations/layer18_attention_weight_max": 30.466793060302734,
      "activations/layer18_attention_weight_min": -25.451021194458008,
      "activations/layer19_attention_weight_max": 33.065547943115234,
      "activations/layer19_attention_weight_min": -31.146665573120117,
      "activations/layer1_attention_weight_max": 16.068626403808594,
      "activations/layer1_attention_weight_min": -15.585453987121582,
      "activations/layer20_attention_weight_max": 29.109912872314453,
      "activations/layer20_attention_weight_min": -25.199907302856445,
      "activations/layer21_attention_weight_max": 29.993436813354492,
      "activations/layer21_attention_weight_min": -24.83098793029785,
      "activations/layer22_attention_weight_max": 41.8134765625,
      "activations/layer22_attention_weight_min": -32.06373596191406,
      "activations/layer23_attention_weight_max": 32.98853302001953,
      "activations/layer23_attention_weight_min": -26.595426559448242,
      "activations/layer2_attention_weight_max": 33.639217376708984,
      "activations/layer2_attention_weight_min": -34.42346954345703,
      "activations/layer3_attention_weight_max": 95.71463775634766,
      "activations/layer3_attention_weight_min": -95.36591339111328,
      "activations/layer4_attention_weight_max": 105.86598205566406,
      "activations/layer4_attention_weight_min": -104.42073059082031,
      "activations/layer5_attention_weight_max": 48.690670013427734,
      "activations/layer5_attention_weight_min": -62.67350769042969,
      "activations/layer6_attention_weight_max": 42.133785247802734,
      "activations/layer6_attention_weight_min": -46.071292877197266,
      "activations/layer7_attention_weight_max": 89.17940521240234,
      "activations/layer7_attention_weight_min": -90.66630554199219,
      "activations/layer8_attention_weight_max": 37.26631164550781,
      "activations/layer8_attention_weight_min": -42.655181884765625,
      "activations/layer9_attention_weight_max": 29.63111686706543,
      "activations/layer9_attention_weight_min": -33.87717819213867,
      "epoch": 16.05,
      "learning_rate": 4.699659090909091e-05,
      "loss": 2.7371,
      "step": 276150
    },
    {
      "activations/layer0_attention_weight_max": 15.602216720581055,
      "activations/layer0_attention_weight_min": -12.291988372802734,
      "activations/layer10_attention_weight_max": 38.70278549194336,
      "activations/layer10_attention_weight_min": -35.70164489746094,
      "activations/layer11_attention_weight_max": 37.291927337646484,
      "activations/layer11_attention_weight_min": -34.659034729003906,
      "activations/layer12_attention_weight_max": 35.483070373535156,
      "activations/layer12_attention_weight_min": -28.02829360961914,
      "activations/layer13_attention_weight_max": 57.048709869384766,
      "activations/layer13_attention_weight_min": -36.14083480834961,
      "activations/layer14_attention_weight_max": 53.43951416015625,
      "activations/layer14_attention_weight_min": -40.66657638549805,
      "activations/layer15_attention_weight_max": 57.04383850097656,
      "activations/layer15_attention_weight_min": -40.82392501831055,
      "activations/layer16_attention_weight_max": 32.65666198730469,
      "activations/layer16_attention_weight_min": -28.544939041137695,
      "activations/layer17_attention_weight_max": 32.352745056152344,
      "activations/layer17_attention_weight_min": -28.715702056884766,
      "activations/layer18_attention_weight_max": 36.54582214355469,
      "activations/layer18_attention_weight_min": -29.27972412109375,
      "activations/layer19_attention_weight_max": 40.273704528808594,
      "activations/layer19_attention_weight_min": -34.2463493347168,
      "activations/layer1_attention_weight_max": 15.268314361572266,
      "activations/layer1_attention_weight_min": -14.81869888305664,
      "activations/layer20_attention_weight_max": 33.036048889160156,
      "activations/layer20_attention_weight_min": -27.202863693237305,
      "activations/layer21_attention_weight_max": 33.18907928466797,
      "activations/layer21_attention_weight_min": -26.449430465698242,
      "activations/layer22_attention_weight_max": 48.56928253173828,
      "activations/layer22_attention_weight_min": -35.595611572265625,
      "activations/layer23_attention_weight_max": 38.53544616699219,
      "activations/layer23_attention_weight_min": -27.371694564819336,
      "activations/layer2_attention_weight_max": 34.93291473388672,
      "activations/layer2_attention_weight_min": -32.848690032958984,
      "activations/layer3_attention_weight_max": 102.61128234863281,
      "activations/layer3_attention_weight_min": -98.88493347167969,
      "activations/layer4_attention_weight_max": 104.47545623779297,
      "activations/layer4_attention_weight_min": -104.09777069091797,
      "activations/layer5_attention_weight_max": 50.630332946777344,
      "activations/layer5_attention_weight_min": -61.515804290771484,
      "activations/layer6_attention_weight_max": 42.15633010864258,
      "activations/layer6_attention_weight_min": -46.64195251464844,
      "activations/layer7_attention_weight_max": 92.29195404052734,
      "activations/layer7_attention_weight_min": -93.24385833740234,
      "activations/layer8_attention_weight_max": 40.93860626220703,
      "activations/layer8_attention_weight_min": -43.115962982177734,
      "activations/layer9_attention_weight_max": 33.45185470581055,
      "activations/layer9_attention_weight_min": -34.24959945678711,
      "epoch": 16.05,
      "learning_rate": 4.697765151515151e-05,
      "loss": 2.7483,
      "step": 276200
    },
    {
      "activations/layer0_attention_weight_max": 16.068063735961914,
      "activations/layer0_attention_weight_min": -12.956976890563965,
      "activations/layer10_attention_weight_max": 35.47883224487305,
      "activations/layer10_attention_weight_min": -33.08839797973633,
      "activations/layer11_attention_weight_max": 34.75894546508789,
      "activations/layer11_attention_weight_min": -32.71495819091797,
      "activations/layer12_attention_weight_max": 30.675060272216797,
      "activations/layer12_attention_weight_min": -26.633848190307617,
      "activations/layer13_attention_weight_max": 43.31891632080078,
      "activations/layer13_attention_weight_min": -34.40208053588867,
      "activations/layer14_attention_weight_max": 45.92853927612305,
      "activations/layer14_attention_weight_min": -36.30477523803711,
      "activations/layer15_attention_weight_max": 42.02333450317383,
      "activations/layer15_attention_weight_min": -35.31386947631836,
      "activations/layer16_attention_weight_max": 33.403560638427734,
      "activations/layer16_attention_weight_min": -28.464515686035156,
      "activations/layer17_attention_weight_max": 31.43024444580078,
      "activations/layer17_attention_weight_min": -25.3631534576416,
      "activations/layer18_attention_weight_max": 33.40460968017578,
      "activations/layer18_attention_weight_min": -27.174474716186523,
      "activations/layer19_attention_weight_max": 35.33857345581055,
      "activations/layer19_attention_weight_min": -32.81713104248047,
      "activations/layer1_attention_weight_max": 15.981592178344727,
      "activations/layer1_attention_weight_min": -15.090441703796387,
      "activations/layer20_attention_weight_max": 31.367624282836914,
      "activations/layer20_attention_weight_min": -26.556255340576172,
      "activations/layer21_attention_weight_max": 31.254085540771484,
      "activations/layer21_attention_weight_min": -26.08670425415039,
      "activations/layer22_attention_weight_max": 45.857421875,
      "activations/layer22_attention_weight_min": -32.910240173339844,
      "activations/layer23_attention_weight_max": 39.31568908691406,
      "activations/layer23_attention_weight_min": -26.25356101989746,
      "activations/layer2_attention_weight_max": 33.760528564453125,
      "activations/layer2_attention_weight_min": -32.63192367553711,
      "activations/layer3_attention_weight_max": 95.42042541503906,
      "activations/layer3_attention_weight_min": -92.96356201171875,
      "activations/layer4_attention_weight_max": 107.0765380859375,
      "activations/layer4_attention_weight_min": -109.63903045654297,
      "activations/layer5_attention_weight_max": 49.053009033203125,
      "activations/layer5_attention_weight_min": -61.080604553222656,
      "activations/layer6_attention_weight_max": 44.93690490722656,
      "activations/layer6_attention_weight_min": -45.66478729248047,
      "activations/layer7_attention_weight_max": 93.2131118774414,
      "activations/layer7_attention_weight_min": -91.1824951171875,
      "activations/layer8_attention_weight_max": 44.16481399536133,
      "activations/layer8_attention_weight_min": -46.26283645629883,
      "activations/layer9_attention_weight_max": 32.992427825927734,
      "activations/layer9_attention_weight_min": -36.09299087524414,
      "epoch": 16.05,
      "learning_rate": 4.69590909090909e-05,
      "loss": 2.7226,
      "step": 276250
    },
    {
      "activations/layer0_attention_weight_max": 15.842144012451172,
      "activations/layer0_attention_weight_min": -13.172883987426758,
      "activations/layer10_attention_weight_max": 31.45714569091797,
      "activations/layer10_attention_weight_min": -32.96596908569336,
      "activations/layer11_attention_weight_max": 32.32585525512695,
      "activations/layer11_attention_weight_min": -32.146724700927734,
      "activations/layer12_attention_weight_max": 26.621591567993164,
      "activations/layer12_attention_weight_min": -25.47490119934082,
      "activations/layer13_attention_weight_max": 41.0745735168457,
      "activations/layer13_attention_weight_min": -35.58857727050781,
      "activations/layer14_attention_weight_max": 42.806610107421875,
      "activations/layer14_attention_weight_min": -35.97395706176758,
      "activations/layer15_attention_weight_max": 38.2277946472168,
      "activations/layer15_attention_weight_min": -33.502685546875,
      "activations/layer16_attention_weight_max": 31.333858489990234,
      "activations/layer16_attention_weight_min": -28.906278610229492,
      "activations/layer17_attention_weight_max": 33.39829635620117,
      "activations/layer17_attention_weight_min": -26.012683868408203,
      "activations/layer18_attention_weight_max": 34.17409896850586,
      "activations/layer18_attention_weight_min": -24.111974716186523,
      "activations/layer19_attention_weight_max": 37.95192337036133,
      "activations/layer19_attention_weight_min": -33.884212493896484,
      "activations/layer1_attention_weight_max": 15.697731018066406,
      "activations/layer1_attention_weight_min": -14.977502822875977,
      "activations/layer20_attention_weight_max": 32.90285873413086,
      "activations/layer20_attention_weight_min": -25.68046760559082,
      "activations/layer21_attention_weight_max": 31.8562068939209,
      "activations/layer21_attention_weight_min": -25.99951171875,
      "activations/layer22_attention_weight_max": 42.68461227416992,
      "activations/layer22_attention_weight_min": -32.233699798583984,
      "activations/layer23_attention_weight_max": 36.81536102294922,
      "activations/layer23_attention_weight_min": -26.81900405883789,
      "activations/layer2_attention_weight_max": 32.66557312011719,
      "activations/layer2_attention_weight_min": -33.360321044921875,
      "activations/layer3_attention_weight_max": 99.34268188476562,
      "activations/layer3_attention_weight_min": -102.77587890625,
      "activations/layer4_attention_weight_max": 107.3108901977539,
      "activations/layer4_attention_weight_min": -109.7891616821289,
      "activations/layer5_attention_weight_max": 51.5765380859375,
      "activations/layer5_attention_weight_min": -65.8815689086914,
      "activations/layer6_attention_weight_max": 43.13039779663086,
      "activations/layer6_attention_weight_min": -46.62724685668945,
      "activations/layer7_attention_weight_max": 86.73768615722656,
      "activations/layer7_attention_weight_min": -94.02017974853516,
      "activations/layer8_attention_weight_max": 39.43079376220703,
      "activations/layer8_attention_weight_min": -42.787254333496094,
      "activations/layer9_attention_weight_max": 30.32082176208496,
      "activations/layer9_attention_weight_min": -33.3568229675293,
      "epoch": 16.05,
      "learning_rate": 4.6940151515151514e-05,
      "loss": 2.7611,
      "step": 276300
    },
    {
      "activations/layer0_attention_weight_max": 16.0539493560791,
      "activations/layer0_attention_weight_min": -13.810090065002441,
      "activations/layer10_attention_weight_max": 31.80484962463379,
      "activations/layer10_attention_weight_min": -31.223613739013672,
      "activations/layer11_attention_weight_max": 32.88142776489258,
      "activations/layer11_attention_weight_min": -31.75794219970703,
      "activations/layer12_attention_weight_max": 27.278156280517578,
      "activations/layer12_attention_weight_min": -24.122421264648438,
      "activations/layer13_attention_weight_max": 40.18742370605469,
      "activations/layer13_attention_weight_min": -34.240543365478516,
      "activations/layer14_attention_weight_max": 42.87744903564453,
      "activations/layer14_attention_weight_min": -34.412254333496094,
      "activations/layer15_attention_weight_max": 40.10678482055664,
      "activations/layer15_attention_weight_min": -31.856660842895508,
      "activations/layer16_attention_weight_max": 30.87290382385254,
      "activations/layer16_attention_weight_min": -27.893630981445312,
      "activations/layer17_attention_weight_max": 31.11960220336914,
      "activations/layer17_attention_weight_min": -24.650371551513672,
      "activations/layer18_attention_weight_max": 33.42395782470703,
      "activations/layer18_attention_weight_min": -23.440567016601562,
      "activations/layer19_attention_weight_max": 32.47786331176758,
      "activations/layer19_attention_weight_min": -29.1619815826416,
      "activations/layer1_attention_weight_max": 16.063941955566406,
      "activations/layer1_attention_weight_min": -13.98032283782959,
      "activations/layer20_attention_weight_max": 29.32274627685547,
      "activations/layer20_attention_weight_min": -23.424421310424805,
      "activations/layer21_attention_weight_max": 28.80095100402832,
      "activations/layer21_attention_weight_min": -24.16319465637207,
      "activations/layer22_attention_weight_max": 41.64326858520508,
      "activations/layer22_attention_weight_min": -31.222042083740234,
      "activations/layer23_attention_weight_max": 33.193267822265625,
      "activations/layer23_attention_weight_min": -27.174724578857422,
      "activations/layer2_attention_weight_max": 32.8179931640625,
      "activations/layer2_attention_weight_min": -33.320274353027344,
      "activations/layer3_attention_weight_max": 93.23836517333984,
      "activations/layer3_attention_weight_min": -98.3909912109375,
      "activations/layer4_attention_weight_max": 99.67224884033203,
      "activations/layer4_attention_weight_min": -104.3108139038086,
      "activations/layer5_attention_weight_max": 48.532588958740234,
      "activations/layer5_attention_weight_min": -61.40336608886719,
      "activations/layer6_attention_weight_max": 42.26750946044922,
      "activations/layer6_attention_weight_min": -45.135826110839844,
      "activations/layer7_attention_weight_max": 87.59468078613281,
      "activations/layer7_attention_weight_min": -86.79901123046875,
      "activations/layer8_attention_weight_max": 38.096885681152344,
      "activations/layer8_attention_weight_min": -40.220054626464844,
      "activations/layer9_attention_weight_max": 33.340293884277344,
      "activations/layer9_attention_weight_min": -31.728635787963867,
      "epoch": 16.06,
      "learning_rate": 4.6921212121212116e-05,
      "loss": 2.7464,
      "step": 276350
    },
    {
      "activations/layer0_attention_weight_max": 15.82485580444336,
      "activations/layer0_attention_weight_min": -12.601410865783691,
      "activations/layer10_attention_weight_max": 37.004554748535156,
      "activations/layer10_attention_weight_min": -34.19728469848633,
      "activations/layer11_attention_weight_max": 36.40168380737305,
      "activations/layer11_attention_weight_min": -34.58249282836914,
      "activations/layer12_attention_weight_max": 28.82077980041504,
      "activations/layer12_attention_weight_min": -26.910621643066406,
      "activations/layer13_attention_weight_max": 46.802772521972656,
      "activations/layer13_attention_weight_min": -37.14372253417969,
      "activations/layer14_attention_weight_max": 53.15800094604492,
      "activations/layer14_attention_weight_min": -42.026187896728516,
      "activations/layer15_attention_weight_max": 46.49576187133789,
      "activations/layer15_attention_weight_min": -37.921348571777344,
      "activations/layer16_attention_weight_max": 35.734046936035156,
      "activations/layer16_attention_weight_min": -28.677581787109375,
      "activations/layer17_attention_weight_max": 35.80284118652344,
      "activations/layer17_attention_weight_min": -27.745582580566406,
      "activations/layer18_attention_weight_max": 33.691558837890625,
      "activations/layer18_attention_weight_min": -29.353666305541992,
      "activations/layer19_attention_weight_max": 38.37434005737305,
      "activations/layer19_attention_weight_min": -33.670982360839844,
      "activations/layer1_attention_weight_max": 16.589000701904297,
      "activations/layer1_attention_weight_min": -16.230892181396484,
      "activations/layer20_attention_weight_max": 33.68928909301758,
      "activations/layer20_attention_weight_min": -26.981611251831055,
      "activations/layer21_attention_weight_max": 32.8170166015625,
      "activations/layer21_attention_weight_min": -25.88591957092285,
      "activations/layer22_attention_weight_max": 49.96345520019531,
      "activations/layer22_attention_weight_min": -34.56721496582031,
      "activations/layer23_attention_weight_max": 37.594173431396484,
      "activations/layer23_attention_weight_min": -28.19440460205078,
      "activations/layer2_attention_weight_max": 33.93543243408203,
      "activations/layer2_attention_weight_min": -32.795013427734375,
      "activations/layer3_attention_weight_max": 99.75357818603516,
      "activations/layer3_attention_weight_min": -96.86153411865234,
      "activations/layer4_attention_weight_max": 112.51761627197266,
      "activations/layer4_attention_weight_min": -108.08428192138672,
      "activations/layer5_attention_weight_max": 52.642677307128906,
      "activations/layer5_attention_weight_min": -63.18193054199219,
      "activations/layer6_attention_weight_max": 45.72452163696289,
      "activations/layer6_attention_weight_min": -47.57491683959961,
      "activations/layer7_attention_weight_max": 98.36495971679688,
      "activations/layer7_attention_weight_min": -97.2341079711914,
      "activations/layer8_attention_weight_max": 42.994407653808594,
      "activations/layer8_attention_weight_min": -46.25725173950195,
      "activations/layer9_attention_weight_max": 35.062808990478516,
      "activations/layer9_attention_weight_min": -34.69850158691406,
      "epoch": 16.06,
      "learning_rate": 4.6902272727272724e-05,
      "loss": 2.7436,
      "step": 276400
    },
    {
      "activations/layer0_attention_weight_max": 15.924345970153809,
      "activations/layer0_attention_weight_min": -12.925969123840332,
      "activations/layer10_attention_weight_max": 33.591644287109375,
      "activations/layer10_attention_weight_min": -34.697906494140625,
      "activations/layer11_attention_weight_max": 33.83335876464844,
      "activations/layer11_attention_weight_min": -35.24060821533203,
      "activations/layer12_attention_weight_max": 27.092206954956055,
      "activations/layer12_attention_weight_min": -27.637744903564453,
      "activations/layer13_attention_weight_max": 38.7828254699707,
      "activations/layer13_attention_weight_min": -38.62865447998047,
      "activations/layer14_attention_weight_max": 41.23856735229492,
      "activations/layer14_attention_weight_min": -42.640201568603516,
      "activations/layer15_attention_weight_max": 39.659915924072266,
      "activations/layer15_attention_weight_min": -35.06584548950195,
      "activations/layer16_attention_weight_max": 30.672754287719727,
      "activations/layer16_attention_weight_min": -29.27364158630371,
      "activations/layer17_attention_weight_max": 28.75434684753418,
      "activations/layer17_attention_weight_min": -25.006921768188477,
      "activations/layer18_attention_weight_max": 30.74601936340332,
      "activations/layer18_attention_weight_min": -26.17156982421875,
      "activations/layer19_attention_weight_max": 36.49880599975586,
      "activations/layer19_attention_weight_min": -30.79945182800293,
      "activations/layer1_attention_weight_max": 16.29534912109375,
      "activations/layer1_attention_weight_min": -15.108121871948242,
      "activations/layer20_attention_weight_max": 29.931156158447266,
      "activations/layer20_attention_weight_min": -24.291709899902344,
      "activations/layer21_attention_weight_max": 29.20113182067871,
      "activations/layer21_attention_weight_min": -23.090349197387695,
      "activations/layer22_attention_weight_max": 46.978633880615234,
      "activations/layer22_attention_weight_min": -30.053539276123047,
      "activations/layer23_attention_weight_max": 33.998046875,
      "activations/layer23_attention_weight_min": -25.438947677612305,
      "activations/layer2_attention_weight_max": 33.952144622802734,
      "activations/layer2_attention_weight_min": -32.584007263183594,
      "activations/layer3_attention_weight_max": 98.92792510986328,
      "activations/layer3_attention_weight_min": -97.81415557861328,
      "activations/layer4_attention_weight_max": 107.2115478515625,
      "activations/layer4_attention_weight_min": -111.42665100097656,
      "activations/layer5_attention_weight_max": 50.46482467651367,
      "activations/layer5_attention_weight_min": -61.21327209472656,
      "activations/layer6_attention_weight_max": 46.41999053955078,
      "activations/layer6_attention_weight_min": -46.187374114990234,
      "activations/layer7_attention_weight_max": 91.73600769042969,
      "activations/layer7_attention_weight_min": -99.3168716430664,
      "activations/layer8_attention_weight_max": 40.942665100097656,
      "activations/layer8_attention_weight_min": -44.913204193115234,
      "activations/layer9_attention_weight_max": 31.854476928710938,
      "activations/layer9_attention_weight_min": -35.90339279174805,
      "epoch": 16.06,
      "learning_rate": 4.6883333333333325e-05,
      "loss": 2.761,
      "step": 276450
    },
    {
      "activations/layer0_attention_weight_max": 16.429790496826172,
      "activations/layer0_attention_weight_min": -13.081583023071289,
      "activations/layer10_attention_weight_max": 32.492923736572266,
      "activations/layer10_attention_weight_min": -34.25666046142578,
      "activations/layer11_attention_weight_max": 32.57683563232422,
      "activations/layer11_attention_weight_min": -33.26630401611328,
      "activations/layer12_attention_weight_max": 24.833600997924805,
      "activations/layer12_attention_weight_min": -24.89875030517578,
      "activations/layer13_attention_weight_max": 38.92529296875,
      "activations/layer13_attention_weight_min": -33.277523040771484,
      "activations/layer14_attention_weight_max": 39.734825134277344,
      "activations/layer14_attention_weight_min": -36.169715881347656,
      "activations/layer15_attention_weight_max": 35.5584716796875,
      "activations/layer15_attention_weight_min": -32.9334831237793,
      "activations/layer16_attention_weight_max": 33.922996520996094,
      "activations/layer16_attention_weight_min": -29.66585922241211,
      "activations/layer17_attention_weight_max": 30.53573989868164,
      "activations/layer17_attention_weight_min": -24.825159072875977,
      "activations/layer18_attention_weight_max": 31.805212020874023,
      "activations/layer18_attention_weight_min": -25.716339111328125,
      "activations/layer19_attention_weight_max": 39.20789337158203,
      "activations/layer19_attention_weight_min": -32.45875930786133,
      "activations/layer1_attention_weight_max": 15.806358337402344,
      "activations/layer1_attention_weight_min": -14.611581802368164,
      "activations/layer20_attention_weight_max": 34.61006546020508,
      "activations/layer20_attention_weight_min": -26.3399715423584,
      "activations/layer21_attention_weight_max": 31.297964096069336,
      "activations/layer21_attention_weight_min": -25.065505981445312,
      "activations/layer22_attention_weight_max": 41.741294860839844,
      "activations/layer22_attention_weight_min": -31.83631134033203,
      "activations/layer23_attention_weight_max": 35.01776123046875,
      "activations/layer23_attention_weight_min": -26.588241577148438,
      "activations/layer2_attention_weight_max": 33.34257507324219,
      "activations/layer2_attention_weight_min": -34.16066360473633,
      "activations/layer3_attention_weight_max": 101.3118667602539,
      "activations/layer3_attention_weight_min": -98.71357727050781,
      "activations/layer4_attention_weight_max": 111.85607147216797,
      "activations/layer4_attention_weight_min": -110.9813461303711,
      "activations/layer5_attention_weight_max": 50.498558044433594,
      "activations/layer5_attention_weight_min": -62.14691925048828,
      "activations/layer6_attention_weight_max": 43.33320617675781,
      "activations/layer6_attention_weight_min": -45.3045768737793,
      "activations/layer7_attention_weight_max": 91.14020538330078,
      "activations/layer7_attention_weight_min": -91.26341247558594,
      "activations/layer8_attention_weight_max": 40.093135833740234,
      "activations/layer8_attention_weight_min": -43.84868240356445,
      "activations/layer9_attention_weight_max": 32.44385528564453,
      "activations/layer9_attention_weight_min": -33.681365966796875,
      "epoch": 16.07,
      "learning_rate": 4.686439393939394e-05,
      "loss": 2.7434,
      "step": 276500
    },
    {
      "activations/layer0_attention_weight_max": 16.577880859375,
      "activations/layer0_attention_weight_min": -13.489350318908691,
      "activations/layer10_attention_weight_max": 32.762001037597656,
      "activations/layer10_attention_weight_min": -35.347843170166016,
      "activations/layer11_attention_weight_max": 34.74723815917969,
      "activations/layer11_attention_weight_min": -33.20427703857422,
      "activations/layer12_attention_weight_max": 28.570743560791016,
      "activations/layer12_attention_weight_min": -26.783748626708984,
      "activations/layer13_attention_weight_max": 43.209598541259766,
      "activations/layer13_attention_weight_min": -36.27885437011719,
      "activations/layer14_attention_weight_max": 46.34088134765625,
      "activations/layer14_attention_weight_min": -36.9048957824707,
      "activations/layer15_attention_weight_max": 43.52474594116211,
      "activations/layer15_attention_weight_min": -33.481201171875,
      "activations/layer16_attention_weight_max": 30.83133888244629,
      "activations/layer16_attention_weight_min": -29.5894775390625,
      "activations/layer17_attention_weight_max": 32.84798049926758,
      "activations/layer17_attention_weight_min": -26.32263946533203,
      "activations/layer18_attention_weight_max": 34.95553970336914,
      "activations/layer18_attention_weight_min": -25.430681228637695,
      "activations/layer19_attention_weight_max": 37.265724182128906,
      "activations/layer19_attention_weight_min": -31.42194175720215,
      "activations/layer1_attention_weight_max": 16.683027267456055,
      "activations/layer1_attention_weight_min": -16.235580444335938,
      "activations/layer20_attention_weight_max": 30.426761627197266,
      "activations/layer20_attention_weight_min": -26.087841033935547,
      "activations/layer21_attention_weight_max": 30.121734619140625,
      "activations/layer21_attention_weight_min": -23.891263961791992,
      "activations/layer22_attention_weight_max": 48.26564025878906,
      "activations/layer22_attention_weight_min": -29.48097801208496,
      "activations/layer23_attention_weight_max": 34.174041748046875,
      "activations/layer23_attention_weight_min": -24.537078857421875,
      "activations/layer2_attention_weight_max": 33.91712188720703,
      "activations/layer2_attention_weight_min": -32.68467712402344,
      "activations/layer3_attention_weight_max": 94.49329376220703,
      "activations/layer3_attention_weight_min": -98.57536315917969,
      "activations/layer4_attention_weight_max": 107.52506256103516,
      "activations/layer4_attention_weight_min": -110.29215240478516,
      "activations/layer5_attention_weight_max": 54.03235626220703,
      "activations/layer5_attention_weight_min": -61.09292221069336,
      "activations/layer6_attention_weight_max": 42.37649154663086,
      "activations/layer6_attention_weight_min": -44.6530876159668,
      "activations/layer7_attention_weight_max": 88.4057388305664,
      "activations/layer7_attention_weight_min": -93.30702209472656,
      "activations/layer8_attention_weight_max": 40.01185989379883,
      "activations/layer8_attention_weight_min": -40.616912841796875,
      "activations/layer9_attention_weight_max": 31.639455795288086,
      "activations/layer9_attention_weight_min": -33.34400177001953,
      "epoch": 16.07,
      "learning_rate": 4.684545454545454e-05,
      "loss": 2.7372,
      "step": 276550
    },
    {
      "activations/layer0_attention_weight_max": 16.289119720458984,
      "activations/layer0_attention_weight_min": -13.085144996643066,
      "activations/layer10_attention_weight_max": 35.28616714477539,
      "activations/layer10_attention_weight_min": -35.78959655761719,
      "activations/layer11_attention_weight_max": 34.184574127197266,
      "activations/layer11_attention_weight_min": -35.124691009521484,
      "activations/layer12_attention_weight_max": 25.914281845092773,
      "activations/layer12_attention_weight_min": -26.73534393310547,
      "activations/layer13_attention_weight_max": 38.263763427734375,
      "activations/layer13_attention_weight_min": -36.53593444824219,
      "activations/layer14_attention_weight_max": 41.68038558959961,
      "activations/layer14_attention_weight_min": -35.54401397705078,
      "activations/layer15_attention_weight_max": 39.209842681884766,
      "activations/layer15_attention_weight_min": -35.656982421875,
      "activations/layer16_attention_weight_max": 32.783321380615234,
      "activations/layer16_attention_weight_min": -29.750396728515625,
      "activations/layer17_attention_weight_max": 29.435747146606445,
      "activations/layer17_attention_weight_min": -25.851816177368164,
      "activations/layer18_attention_weight_max": 30.103525161743164,
      "activations/layer18_attention_weight_min": -25.51181983947754,
      "activations/layer19_attention_weight_max": 32.87580871582031,
      "activations/layer19_attention_weight_min": -32.21030807495117,
      "activations/layer1_attention_weight_max": 17.183223724365234,
      "activations/layer1_attention_weight_min": -16.694841384887695,
      "activations/layer20_attention_weight_max": 28.91857147216797,
      "activations/layer20_attention_weight_min": -25.248695373535156,
      "activations/layer21_attention_weight_max": 27.793819427490234,
      "activations/layer21_attention_weight_min": -26.177841186523438,
      "activations/layer22_attention_weight_max": 38.174217224121094,
      "activations/layer22_attention_weight_min": -30.46137809753418,
      "activations/layer23_attention_weight_max": 30.345951080322266,
      "activations/layer23_attention_weight_min": -26.4089412689209,
      "activations/layer2_attention_weight_max": 33.545021057128906,
      "activations/layer2_attention_weight_min": -33.9561767578125,
      "activations/layer3_attention_weight_max": 103.8822021484375,
      "activations/layer3_attention_weight_min": -105.11256408691406,
      "activations/layer4_attention_weight_max": 113.75563049316406,
      "activations/layer4_attention_weight_min": -111.4514389038086,
      "activations/layer5_attention_weight_max": 51.11451721191406,
      "activations/layer5_attention_weight_min": -61.98568344116211,
      "activations/layer6_attention_weight_max": 45.440528869628906,
      "activations/layer6_attention_weight_min": -48.59056854248047,
      "activations/layer7_attention_weight_max": 93.57830810546875,
      "activations/layer7_attention_weight_min": -96.22480773925781,
      "activations/layer8_attention_weight_max": 43.61629104614258,
      "activations/layer8_attention_weight_min": -44.20424270629883,
      "activations/layer9_attention_weight_max": 33.10128402709961,
      "activations/layer9_attention_weight_min": -35.11883544921875,
      "epoch": 16.07,
      "learning_rate": 4.6826515151515144e-05,
      "loss": 2.7494,
      "step": 276600
    },
    {
      "activations/layer0_attention_weight_max": 16.207666397094727,
      "activations/layer0_attention_weight_min": -12.155716896057129,
      "activations/layer10_attention_weight_max": 35.577945709228516,
      "activations/layer10_attention_weight_min": -34.080745697021484,
      "activations/layer11_attention_weight_max": 36.09235382080078,
      "activations/layer11_attention_weight_min": -35.66779708862305,
      "activations/layer12_attention_weight_max": 43.23499298095703,
      "activations/layer12_attention_weight_min": -28.485666275024414,
      "activations/layer13_attention_weight_max": 52.58403396606445,
      "activations/layer13_attention_weight_min": -36.98167419433594,
      "activations/layer14_attention_weight_max": 48.00254440307617,
      "activations/layer14_attention_weight_min": -39.76014709472656,
      "activations/layer15_attention_weight_max": 41.46281433105469,
      "activations/layer15_attention_weight_min": -35.34022903442383,
      "activations/layer16_attention_weight_max": 32.12372970581055,
      "activations/layer16_attention_weight_min": -28.255884170532227,
      "activations/layer17_attention_weight_max": 33.24857711791992,
      "activations/layer17_attention_weight_min": -25.729511260986328,
      "activations/layer18_attention_weight_max": 34.70552062988281,
      "activations/layer18_attention_weight_min": -24.2694149017334,
      "activations/layer19_attention_weight_max": 36.96377182006836,
      "activations/layer19_attention_weight_min": -30.928213119506836,
      "activations/layer1_attention_weight_max": 16.60525894165039,
      "activations/layer1_attention_weight_min": -15.341147422790527,
      "activations/layer20_attention_weight_max": 31.751617431640625,
      "activations/layer20_attention_weight_min": -24.668127059936523,
      "activations/layer21_attention_weight_max": 31.577024459838867,
      "activations/layer21_attention_weight_min": -24.248796463012695,
      "activations/layer22_attention_weight_max": 53.31203842163086,
      "activations/layer22_attention_weight_min": -31.646244049072266,
      "activations/layer23_attention_weight_max": 38.24091720581055,
      "activations/layer23_attention_weight_min": -24.19103240966797,
      "activations/layer2_attention_weight_max": 32.47986602783203,
      "activations/layer2_attention_weight_min": -32.77387237548828,
      "activations/layer3_attention_weight_max": 97.30754089355469,
      "activations/layer3_attention_weight_min": -98.5146713256836,
      "activations/layer4_attention_weight_max": 106.20169830322266,
      "activations/layer4_attention_weight_min": -105.18498992919922,
      "activations/layer5_attention_weight_max": 50.259639739990234,
      "activations/layer5_attention_weight_min": -61.835060119628906,
      "activations/layer6_attention_weight_max": 44.36245346069336,
      "activations/layer6_attention_weight_min": -50.04334259033203,
      "activations/layer7_attention_weight_max": 91.71644592285156,
      "activations/layer7_attention_weight_min": -97.48291778564453,
      "activations/layer8_attention_weight_max": 43.17237854003906,
      "activations/layer8_attention_weight_min": -43.93825149536133,
      "activations/layer9_attention_weight_max": 34.689815521240234,
      "activations/layer9_attention_weight_min": -33.50631332397461,
      "epoch": 16.07,
      "learning_rate": 4.680757575757576e-05,
      "loss": 2.7482,
      "step": 276650
    },
    {
      "activations/layer0_attention_weight_max": 16.424541473388672,
      "activations/layer0_attention_weight_min": -12.995457649230957,
      "activations/layer10_attention_weight_max": 41.93406295776367,
      "activations/layer10_attention_weight_min": -39.167724609375,
      "activations/layer11_attention_weight_max": 43.91065216064453,
      "activations/layer11_attention_weight_min": -39.69560241699219,
      "activations/layer12_attention_weight_max": 26.908626556396484,
      "activations/layer12_attention_weight_min": -28.395259857177734,
      "activations/layer13_attention_weight_max": 38.2999153137207,
      "activations/layer13_attention_weight_min": -33.206180572509766,
      "activations/layer14_attention_weight_max": 42.553348541259766,
      "activations/layer14_attention_weight_min": -38.38589096069336,
      "activations/layer15_attention_weight_max": 40.915802001953125,
      "activations/layer15_attention_weight_min": -36.82024002075195,
      "activations/layer16_attention_weight_max": 30.324430465698242,
      "activations/layer16_attention_weight_min": -28.592309951782227,
      "activations/layer17_attention_weight_max": 27.130674362182617,
      "activations/layer17_attention_weight_min": -26.68355941772461,
      "activations/layer18_attention_weight_max": 27.289840698242188,
      "activations/layer18_attention_weight_min": -25.690147399902344,
      "activations/layer19_attention_weight_max": 32.37575912475586,
      "activations/layer19_attention_weight_min": -32.84376525878906,
      "activations/layer1_attention_weight_max": 16.83316993713379,
      "activations/layer1_attention_weight_min": -15.993571281433105,
      "activations/layer20_attention_weight_max": 27.036924362182617,
      "activations/layer20_attention_weight_min": -26.21910285949707,
      "activations/layer21_attention_weight_max": 26.683544158935547,
      "activations/layer21_attention_weight_min": -27.60431480407715,
      "activations/layer22_attention_weight_max": 38.145660400390625,
      "activations/layer22_attention_weight_min": -32.867408752441406,
      "activations/layer23_attention_weight_max": 29.82843589782715,
      "activations/layer23_attention_weight_min": -23.918548583984375,
      "activations/layer2_attention_weight_max": 35.74908447265625,
      "activations/layer2_attention_weight_min": -35.99175262451172,
      "activations/layer3_attention_weight_max": 101.18125915527344,
      "activations/layer3_attention_weight_min": -104.12344360351562,
      "activations/layer4_attention_weight_max": 110.11077880859375,
      "activations/layer4_attention_weight_min": -112.09098052978516,
      "activations/layer5_attention_weight_max": 50.429649353027344,
      "activations/layer5_attention_weight_min": -58.43696594238281,
      "activations/layer6_attention_weight_max": 44.40464782714844,
      "activations/layer6_attention_weight_min": -46.30046081542969,
      "activations/layer7_attention_weight_max": 99.24000549316406,
      "activations/layer7_attention_weight_min": -93.21662139892578,
      "activations/layer8_attention_weight_max": 46.5724983215332,
      "activations/layer8_attention_weight_min": -44.696285247802734,
      "activations/layer9_attention_weight_max": 38.0853271484375,
      "activations/layer9_attention_weight_min": -38.76443099975586,
      "epoch": 16.08,
      "learning_rate": 4.678863636363636e-05,
      "loss": 2.745,
      "step": 276700
    },
    {
      "activations/layer0_attention_weight_max": 16.000202178955078,
      "activations/layer0_attention_weight_min": -12.515937805175781,
      "activations/layer10_attention_weight_max": 32.32691955566406,
      "activations/layer10_attention_weight_min": -33.35536193847656,
      "activations/layer11_attention_weight_max": 33.06270980834961,
      "activations/layer11_attention_weight_min": -33.08375930786133,
      "activations/layer12_attention_weight_max": 24.223447799682617,
      "activations/layer12_attention_weight_min": -25.354248046875,
      "activations/layer13_attention_weight_max": 36.30561065673828,
      "activations/layer13_attention_weight_min": -32.7542724609375,
      "activations/layer14_attention_weight_max": 40.630306243896484,
      "activations/layer14_attention_weight_min": -36.10918045043945,
      "activations/layer15_attention_weight_max": 35.53834533691406,
      "activations/layer15_attention_weight_min": -33.56696319580078,
      "activations/layer16_attention_weight_max": 29.960649490356445,
      "activations/layer16_attention_weight_min": -28.41034698486328,
      "activations/layer17_attention_weight_max": 30.719573974609375,
      "activations/layer17_attention_weight_min": -25.180347442626953,
      "activations/layer18_attention_weight_max": 30.6355037689209,
      "activations/layer18_attention_weight_min": -23.680124282836914,
      "activations/layer19_attention_weight_max": 32.02477264404297,
      "activations/layer19_attention_weight_min": -29.996891021728516,
      "activations/layer1_attention_weight_max": 16.516326904296875,
      "activations/layer1_attention_weight_min": -15.056379318237305,
      "activations/layer20_attention_weight_max": 27.581565856933594,
      "activations/layer20_attention_weight_min": -24.523277282714844,
      "activations/layer21_attention_weight_max": 27.113231658935547,
      "activations/layer21_attention_weight_min": -24.40485191345215,
      "activations/layer22_attention_weight_max": 40.16551208496094,
      "activations/layer22_attention_weight_min": -30.379838943481445,
      "activations/layer23_attention_weight_max": 31.2315673828125,
      "activations/layer23_attention_weight_min": -26.288074493408203,
      "activations/layer2_attention_weight_max": 32.67604446411133,
      "activations/layer2_attention_weight_min": -33.75432586669922,
      "activations/layer3_attention_weight_max": 97.1044692993164,
      "activations/layer3_attention_weight_min": -98.77682495117188,
      "activations/layer4_attention_weight_max": 107.40682220458984,
      "activations/layer4_attention_weight_min": -107.07405853271484,
      "activations/layer5_attention_weight_max": 48.288055419921875,
      "activations/layer5_attention_weight_min": -61.74617004394531,
      "activations/layer6_attention_weight_max": 41.565364837646484,
      "activations/layer6_attention_weight_min": -44.37905502319336,
      "activations/layer7_attention_weight_max": 89.50413513183594,
      "activations/layer7_attention_weight_min": -91.29019927978516,
      "activations/layer8_attention_weight_max": 38.821197509765625,
      "activations/layer8_attention_weight_min": -43.167537689208984,
      "activations/layer9_attention_weight_max": 32.69599151611328,
      "activations/layer9_attention_weight_min": -35.58710479736328,
      "epoch": 16.08,
      "learning_rate": 4.677007575757575e-05,
      "loss": 2.7341,
      "step": 276750
    },
    {
      "activations/layer0_attention_weight_max": 15.661017417907715,
      "activations/layer0_attention_weight_min": -13.973002433776855,
      "activations/layer10_attention_weight_max": 34.54926681518555,
      "activations/layer10_attention_weight_min": -34.591270446777344,
      "activations/layer11_attention_weight_max": 35.66756057739258,
      "activations/layer11_attention_weight_min": -33.84621047973633,
      "activations/layer12_attention_weight_max": 27.203954696655273,
      "activations/layer12_attention_weight_min": -25.71949577331543,
      "activations/layer13_attention_weight_max": 38.809112548828125,
      "activations/layer13_attention_weight_min": -36.58579635620117,
      "activations/layer14_attention_weight_max": 45.14718246459961,
      "activations/layer14_attention_weight_min": -39.121803283691406,
      "activations/layer15_attention_weight_max": 39.98630905151367,
      "activations/layer15_attention_weight_min": -36.57602310180664,
      "activations/layer16_attention_weight_max": 30.065763473510742,
      "activations/layer16_attention_weight_min": -27.09848403930664,
      "activations/layer17_attention_weight_max": 29.706613540649414,
      "activations/layer17_attention_weight_min": -25.83962631225586,
      "activations/layer18_attention_weight_max": 34.220619201660156,
      "activations/layer18_attention_weight_min": -26.966161727905273,
      "activations/layer19_attention_weight_max": 35.677303314208984,
      "activations/layer19_attention_weight_min": -33.193885803222656,
      "activations/layer1_attention_weight_max": 18.056528091430664,
      "activations/layer1_attention_weight_min": -16.620811462402344,
      "activations/layer20_attention_weight_max": 29.655033111572266,
      "activations/layer20_attention_weight_min": -25.312725067138672,
      "activations/layer21_attention_weight_max": 30.55220603942871,
      "activations/layer21_attention_weight_min": -24.00809669494629,
      "activations/layer22_attention_weight_max": 44.466190338134766,
      "activations/layer22_attention_weight_min": -31.2762451171875,
      "activations/layer23_attention_weight_max": 33.73193359375,
      "activations/layer23_attention_weight_min": -29.987506866455078,
      "activations/layer2_attention_weight_max": 34.6121711730957,
      "activations/layer2_attention_weight_min": -34.436946868896484,
      "activations/layer3_attention_weight_max": 100.70441436767578,
      "activations/layer3_attention_weight_min": -100.9133071899414,
      "activations/layer4_attention_weight_max": 106.89781951904297,
      "activations/layer4_attention_weight_min": -110.19671630859375,
      "activations/layer5_attention_weight_max": 51.22358322143555,
      "activations/layer5_attention_weight_min": -62.88444137573242,
      "activations/layer6_attention_weight_max": 45.083457946777344,
      "activations/layer6_attention_weight_min": -46.61333084106445,
      "activations/layer7_attention_weight_max": 90.69317626953125,
      "activations/layer7_attention_weight_min": -92.7061767578125,
      "activations/layer8_attention_weight_max": 40.149879455566406,
      "activations/layer8_attention_weight_min": -42.48744583129883,
      "activations/layer9_attention_weight_max": 31.915525436401367,
      "activations/layer9_attention_weight_min": -32.211910247802734,
      "epoch": 16.08,
      "learning_rate": 4.675113636363636e-05,
      "loss": 2.7427,
      "step": 276800
    },
    {
      "activations/layer0_attention_weight_max": 16.091524124145508,
      "activations/layer0_attention_weight_min": -12.7237548828125,
      "activations/layer10_attention_weight_max": 33.87914276123047,
      "activations/layer10_attention_weight_min": -34.987060546875,
      "activations/layer11_attention_weight_max": 34.35731506347656,
      "activations/layer11_attention_weight_min": -35.68634033203125,
      "activations/layer12_attention_weight_max": 26.09191131591797,
      "activations/layer12_attention_weight_min": -28.071815490722656,
      "activations/layer13_attention_weight_max": 38.88267135620117,
      "activations/layer13_attention_weight_min": -37.31178283691406,
      "activations/layer14_attention_weight_max": 42.61818313598633,
      "activations/layer14_attention_weight_min": -38.48124694824219,
      "activations/layer15_attention_weight_max": 39.94540786743164,
      "activations/layer15_attention_weight_min": -38.19569396972656,
      "activations/layer16_attention_weight_max": 29.542234420776367,
      "activations/layer16_attention_weight_min": -27.897411346435547,
      "activations/layer17_attention_weight_max": 30.05362319946289,
      "activations/layer17_attention_weight_min": -25.087841033935547,
      "activations/layer18_attention_weight_max": 29.625398635864258,
      "activations/layer18_attention_weight_min": -25.27684783935547,
      "activations/layer19_attention_weight_max": 35.33409881591797,
      "activations/layer19_attention_weight_min": -31.38722801208496,
      "activations/layer1_attention_weight_max": 16.607030868530273,
      "activations/layer1_attention_weight_min": -15.97445011138916,
      "activations/layer20_attention_weight_max": 27.746280670166016,
      "activations/layer20_attention_weight_min": -26.03986167907715,
      "activations/layer21_attention_weight_max": 28.389415740966797,
      "activations/layer21_attention_weight_min": -25.498966217041016,
      "activations/layer22_attention_weight_max": 40.69679260253906,
      "activations/layer22_attention_weight_min": -31.250120162963867,
      "activations/layer23_attention_weight_max": 31.314136505126953,
      "activations/layer23_attention_weight_min": -26.115331649780273,
      "activations/layer2_attention_weight_max": 31.674652099609375,
      "activations/layer2_attention_weight_min": -32.19682312011719,
      "activations/layer3_attention_weight_max": 99.84998321533203,
      "activations/layer3_attention_weight_min": -99.9879150390625,
      "activations/layer4_attention_weight_max": 107.45826721191406,
      "activations/layer4_attention_weight_min": -110.74922943115234,
      "activations/layer5_attention_weight_max": 53.626487731933594,
      "activations/layer5_attention_weight_min": -60.834651947021484,
      "activations/layer6_attention_weight_max": 45.211143493652344,
      "activations/layer6_attention_weight_min": -48.485355377197266,
      "activations/layer7_attention_weight_max": 94.19652557373047,
      "activations/layer7_attention_weight_min": -97.89366149902344,
      "activations/layer8_attention_weight_max": 42.296321868896484,
      "activations/layer8_attention_weight_min": -45.03520965576172,
      "activations/layer9_attention_weight_max": 33.35273361206055,
      "activations/layer9_attention_weight_min": -36.73703384399414,
      "epoch": 16.09,
      "learning_rate": 4.673219696969697e-05,
      "loss": 2.731,
      "step": 276850
    },
    {
      "activations/layer0_attention_weight_max": 15.829900741577148,
      "activations/layer0_attention_weight_min": -13.023608207702637,
      "activations/layer10_attention_weight_max": 33.2065315246582,
      "activations/layer10_attention_weight_min": -32.933441162109375,
      "activations/layer11_attention_weight_max": 34.42530822753906,
      "activations/layer11_attention_weight_min": -32.14373779296875,
      "activations/layer12_attention_weight_max": 25.680219650268555,
      "activations/layer12_attention_weight_min": -26.085107803344727,
      "activations/layer13_attention_weight_max": 39.3160514831543,
      "activations/layer13_attention_weight_min": -34.50720977783203,
      "activations/layer14_attention_weight_max": 42.2989387512207,
      "activations/layer14_attention_weight_min": -38.329219818115234,
      "activations/layer15_attention_weight_max": 38.86538314819336,
      "activations/layer15_attention_weight_min": -34.63416290283203,
      "activations/layer16_attention_weight_max": 33.055633544921875,
      "activations/layer16_attention_weight_min": -27.517269134521484,
      "activations/layer17_attention_weight_max": 35.61882400512695,
      "activations/layer17_attention_weight_min": -24.540342330932617,
      "activations/layer18_attention_weight_max": 35.27790069580078,
      "activations/layer18_attention_weight_min": -25.70269775390625,
      "activations/layer19_attention_weight_max": 39.990177154541016,
      "activations/layer19_attention_weight_min": -29.05568504333496,
      "activations/layer1_attention_weight_max": 17.714275360107422,
      "activations/layer1_attention_weight_min": -16.760372161865234,
      "activations/layer20_attention_weight_max": 33.33741760253906,
      "activations/layer20_attention_weight_min": -23.376293182373047,
      "activations/layer21_attention_weight_max": 31.95452880859375,
      "activations/layer21_attention_weight_min": -25.375513076782227,
      "activations/layer22_attention_weight_max": 46.70149612426758,
      "activations/layer22_attention_weight_min": -29.641902923583984,
      "activations/layer23_attention_weight_max": 35.61029815673828,
      "activations/layer23_attention_weight_min": -25.16290855407715,
      "activations/layer2_attention_weight_max": 32.430213928222656,
      "activations/layer2_attention_weight_min": -29.965057373046875,
      "activations/layer3_attention_weight_max": 95.35664367675781,
      "activations/layer3_attention_weight_min": -94.64664459228516,
      "activations/layer4_attention_weight_max": 107.37525939941406,
      "activations/layer4_attention_weight_min": -108.5684585571289,
      "activations/layer5_attention_weight_max": 53.486175537109375,
      "activations/layer5_attention_weight_min": -64.72315979003906,
      "activations/layer6_attention_weight_max": 45.30404281616211,
      "activations/layer6_attention_weight_min": -46.92108917236328,
      "activations/layer7_attention_weight_max": 91.08394622802734,
      "activations/layer7_attention_weight_min": -93.22196960449219,
      "activations/layer8_attention_weight_max": 40.634273529052734,
      "activations/layer8_attention_weight_min": -42.167991638183594,
      "activations/layer9_attention_weight_max": 32.2920036315918,
      "activations/layer9_attention_weight_min": -33.27857208251953,
      "epoch": 16.09,
      "learning_rate": 4.6713257575757576e-05,
      "loss": 2.7366,
      "step": 276900
    },
    {
      "activations/layer0_attention_weight_max": 15.777692794799805,
      "activations/layer0_attention_weight_min": -12.875469207763672,
      "activations/layer10_attention_weight_max": 32.875572204589844,
      "activations/layer10_attention_weight_min": -35.42207717895508,
      "activations/layer11_attention_weight_max": 32.88103485107422,
      "activations/layer11_attention_weight_min": -33.43517303466797,
      "activations/layer12_attention_weight_max": 24.24498748779297,
      "activations/layer12_attention_weight_min": -25.188379287719727,
      "activations/layer13_attention_weight_max": 36.881256103515625,
      "activations/layer13_attention_weight_min": -34.70518112182617,
      "activations/layer14_attention_weight_max": 39.23247146606445,
      "activations/layer14_attention_weight_min": -36.13243865966797,
      "activations/layer15_attention_weight_max": 34.996337890625,
      "activations/layer15_attention_weight_min": -32.15171432495117,
      "activations/layer16_attention_weight_max": 29.250810623168945,
      "activations/layer16_attention_weight_min": -26.55446434020996,
      "activations/layer17_attention_weight_max": 29.211782455444336,
      "activations/layer17_attention_weight_min": -26.660242080688477,
      "activations/layer18_attention_weight_max": 33.430938720703125,
      "activations/layer18_attention_weight_min": -23.671463012695312,
      "activations/layer19_attention_weight_max": 35.50767135620117,
      "activations/layer19_attention_weight_min": -30.86032485961914,
      "activations/layer1_attention_weight_max": 16.85614585876465,
      "activations/layer1_attention_weight_min": -14.843026161193848,
      "activations/layer20_attention_weight_max": 28.264286041259766,
      "activations/layer20_attention_weight_min": -24.90178108215332,
      "activations/layer21_attention_weight_max": 28.67667579650879,
      "activations/layer21_attention_weight_min": -25.07339859008789,
      "activations/layer22_attention_weight_max": 40.37864303588867,
      "activations/layer22_attention_weight_min": -30.12751579284668,
      "activations/layer23_attention_weight_max": 32.74664306640625,
      "activations/layer23_attention_weight_min": -24.565086364746094,
      "activations/layer2_attention_weight_max": 35.30784606933594,
      "activations/layer2_attention_weight_min": -31.99697494506836,
      "activations/layer3_attention_weight_max": 93.91004943847656,
      "activations/layer3_attention_weight_min": -93.60673522949219,
      "activations/layer4_attention_weight_max": 103.94585418701172,
      "activations/layer4_attention_weight_min": -105.10143280029297,
      "activations/layer5_attention_weight_max": 52.53203582763672,
      "activations/layer5_attention_weight_min": -61.481239318847656,
      "activations/layer6_attention_weight_max": 44.05378723144531,
      "activations/layer6_attention_weight_min": -46.594261169433594,
      "activations/layer7_attention_weight_max": 89.43777465820312,
      "activations/layer7_attention_weight_min": -91.23493194580078,
      "activations/layer8_attention_weight_max": 40.16249465942383,
      "activations/layer8_attention_weight_min": -42.31126022338867,
      "activations/layer9_attention_weight_max": 31.30230712890625,
      "activations/layer9_attention_weight_min": -34.142112731933594,
      "epoch": 16.09,
      "learning_rate": 4.669431818181818e-05,
      "loss": 2.7377,
      "step": 276950
    },
    {
      "activations/layer0_attention_weight_max": 17.12108039855957,
      "activations/layer0_attention_weight_min": -12.563620567321777,
      "activations/layer10_attention_weight_max": 34.42646026611328,
      "activations/layer10_attention_weight_min": -33.37816619873047,
      "activations/layer11_attention_weight_max": 32.85206604003906,
      "activations/layer11_attention_weight_min": -34.670127868652344,
      "activations/layer12_attention_weight_max": 26.265398025512695,
      "activations/layer12_attention_weight_min": -34.587913513183594,
      "activations/layer13_attention_weight_max": 38.943233489990234,
      "activations/layer13_attention_weight_min": -34.03899383544922,
      "activations/layer14_attention_weight_max": 42.52409744262695,
      "activations/layer14_attention_weight_min": -34.57645797729492,
      "activations/layer15_attention_weight_max": 36.02467346191406,
      "activations/layer15_attention_weight_min": -31.812849044799805,
      "activations/layer16_attention_weight_max": 32.40974426269531,
      "activations/layer16_attention_weight_min": -27.870487213134766,
      "activations/layer17_attention_weight_max": 31.378347396850586,
      "activations/layer17_attention_weight_min": -24.965784072875977,
      "activations/layer18_attention_weight_max": 32.1590690612793,
      "activations/layer18_attention_weight_min": -25.45693588256836,
      "activations/layer19_attention_weight_max": 34.758148193359375,
      "activations/layer19_attention_weight_min": -33.266605377197266,
      "activations/layer1_attention_weight_max": 15.828909873962402,
      "activations/layer1_attention_weight_min": -14.452825546264648,
      "activations/layer20_attention_weight_max": 30.077808380126953,
      "activations/layer20_attention_weight_min": -26.796977996826172,
      "activations/layer21_attention_weight_max": 30.718381881713867,
      "activations/layer21_attention_weight_min": -25.449874877929688,
      "activations/layer22_attention_weight_max": 42.5369873046875,
      "activations/layer22_attention_weight_min": -30.396671295166016,
      "activations/layer23_attention_weight_max": 35.96059036254883,
      "activations/layer23_attention_weight_min": -25.253276824951172,
      "activations/layer2_attention_weight_max": 34.42298126220703,
      "activations/layer2_attention_weight_min": -32.646324157714844,
      "activations/layer3_attention_weight_max": 94.9988021850586,
      "activations/layer3_attention_weight_min": -94.32856750488281,
      "activations/layer4_attention_weight_max": 107.57186126708984,
      "activations/layer4_attention_weight_min": -112.27174377441406,
      "activations/layer5_attention_weight_max": 53.21320724487305,
      "activations/layer5_attention_weight_min": -61.433650970458984,
      "activations/layer6_attention_weight_max": 41.56856155395508,
      "activations/layer6_attention_weight_min": -44.17929458618164,
      "activations/layer7_attention_weight_max": 97.29560089111328,
      "activations/layer7_attention_weight_min": -91.33338928222656,
      "activations/layer8_attention_weight_max": 40.850006103515625,
      "activations/layer8_attention_weight_min": -43.06304931640625,
      "activations/layer9_attention_weight_max": 31.410207748413086,
      "activations/layer9_attention_weight_min": -33.97190475463867,
      "epoch": 16.1,
      "learning_rate": 4.667537878787878e-05,
      "loss": 2.7421,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4784,
      "eval_samples_per_second": 506.465,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4784,
      "eval_openwebtext_samples_per_second": 506.465,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_wikitext_loss": 2.92578125,
      "eval_wikitext_ppl": 18.648789727196185,
      "eval_wikitext_runtime": 1.9978,
      "eval_wikitext_samples_per_second": 228.25,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.5916,
      "eval_lambada_samples_per_second": 507.634,
      "step": 277000
    },
    {
      "activations/layer0_attention_weight_max": 16.26914405822754,
      "activations/layer0_attention_weight_min": -12.733738899230957,
      "activations/layer10_attention_weight_max": 35.8421630859375,
      "activations/layer10_attention_weight_min": -33.8756217956543,
      "activations/layer11_attention_weight_max": 33.989540100097656,
      "activations/layer11_attention_weight_min": -33.73860168457031,
      "activations/layer12_attention_weight_max": 29.760522842407227,
      "activations/layer12_attention_weight_min": -26.62043571472168,
      "activations/layer13_attention_weight_max": 43.49972915649414,
      "activations/layer13_attention_weight_min": -35.3842658996582,
      "activations/layer14_attention_weight_max": 49.417720794677734,
      "activations/layer14_attention_weight_min": -38.90848159790039,
      "activations/layer15_attention_weight_max": 39.253517150878906,
      "activations/layer15_attention_weight_min": -34.61117935180664,
      "activations/layer16_attention_weight_max": 30.26153564453125,
      "activations/layer16_attention_weight_min": -27.234760284423828,
      "activations/layer17_attention_weight_max": 28.531824111938477,
      "activations/layer17_attention_weight_min": -25.99212646484375,
      "activations/layer18_attention_weight_max": 33.199920654296875,
      "activations/layer18_attention_weight_min": -24.48553466796875,
      "activations/layer19_attention_weight_max": 34.88242721557617,
      "activations/layer19_attention_weight_min": -29.5391788482666,
      "activations/layer1_attention_weight_max": 16.348600387573242,
      "activations/layer1_attention_weight_min": -14.42566967010498,
      "activations/layer20_attention_weight_max": 28.904190063476562,
      "activations/layer20_attention_weight_min": -25.962263107299805,
      "activations/layer21_attention_weight_max": 27.596059799194336,
      "activations/layer21_attention_weight_min": -25.201236724853516,
      "activations/layer22_attention_weight_max": 39.72791290283203,
      "activations/layer22_attention_weight_min": -30.369417190551758,
      "activations/layer23_attention_weight_max": 31.737775802612305,
      "activations/layer23_attention_weight_min": -26.288814544677734,
      "activations/layer2_attention_weight_max": 33.68691635131836,
      "activations/layer2_attention_weight_min": -32.62055969238281,
      "activations/layer3_attention_weight_max": 98.50665283203125,
      "activations/layer3_attention_weight_min": -98.25682830810547,
      "activations/layer4_attention_weight_max": 112.66082000732422,
      "activations/layer4_attention_weight_min": -110.2762680053711,
      "activations/layer5_attention_weight_max": 51.5362548828125,
      "activations/layer5_attention_weight_min": -62.43496322631836,
      "activations/layer6_attention_weight_max": 45.386348724365234,
      "activations/layer6_attention_weight_min": -47.022193908691406,
      "activations/layer7_attention_weight_max": 97.52203369140625,
      "activations/layer7_attention_weight_min": -98.99019622802734,
      "activations/layer8_attention_weight_max": 43.0350227355957,
      "activations/layer8_attention_weight_min": -44.657230377197266,
      "activations/layer9_attention_weight_max": 35.011470794677734,
      "activations/layer9_attention_weight_min": -34.51521301269531,
      "epoch": 16.1,
      "learning_rate": 4.6656439393939394e-05,
      "loss": 2.7359,
      "step": 277050
    },
    {
      "activations/layer0_attention_weight_max": 17.12430763244629,
      "activations/layer0_attention_weight_min": -14.458680152893066,
      "activations/layer10_attention_weight_max": 32.367164611816406,
      "activations/layer10_attention_weight_min": -35.03615188598633,
      "activations/layer11_attention_weight_max": 35.276641845703125,
      "activations/layer11_attention_weight_min": -32.92060089111328,
      "activations/layer12_attention_weight_max": 25.916959762573242,
      "activations/layer12_attention_weight_min": -26.836715698242188,
      "activations/layer13_attention_weight_max": 40.28430938720703,
      "activations/layer13_attention_weight_min": -36.62528991699219,
      "activations/layer14_attention_weight_max": 44.69819259643555,
      "activations/layer14_attention_weight_min": -39.64081954956055,
      "activations/layer15_attention_weight_max": 42.86975860595703,
      "activations/layer15_attention_weight_min": -34.224666595458984,
      "activations/layer16_attention_weight_max": 29.62306785583496,
      "activations/layer16_attention_weight_min": -27.631227493286133,
      "activations/layer17_attention_weight_max": 30.549686431884766,
      "activations/layer17_attention_weight_min": -25.33753776550293,
      "activations/layer18_attention_weight_max": 33.90143966674805,
      "activations/layer18_attention_weight_min": -25.12213706970215,
      "activations/layer19_attention_weight_max": 33.64861297607422,
      "activations/layer19_attention_weight_min": -32.429779052734375,
      "activations/layer1_attention_weight_max": 16.16272735595703,
      "activations/layer1_attention_weight_min": -14.829767227172852,
      "activations/layer20_attention_weight_max": 28.41670799255371,
      "activations/layer20_attention_weight_min": -24.52606201171875,
      "activations/layer21_attention_weight_max": 26.963823318481445,
      "activations/layer21_attention_weight_min": -24.282556533813477,
      "activations/layer22_attention_weight_max": 43.685237884521484,
      "activations/layer22_attention_weight_min": -31.25417137145996,
      "activations/layer23_attention_weight_max": 32.37425994873047,
      "activations/layer23_attention_weight_min": -25.27315330505371,
      "activations/layer2_attention_weight_max": 33.57415771484375,
      "activations/layer2_attention_weight_min": -32.496829986572266,
      "activations/layer3_attention_weight_max": 95.46318817138672,
      "activations/layer3_attention_weight_min": -93.73150634765625,
      "activations/layer4_attention_weight_max": 107.47515106201172,
      "activations/layer4_attention_weight_min": -101.39395904541016,
      "activations/layer5_attention_weight_max": 51.23358917236328,
      "activations/layer5_attention_weight_min": -58.37495422363281,
      "activations/layer6_attention_weight_max": 43.87648010253906,
      "activations/layer6_attention_weight_min": -44.2072868347168,
      "activations/layer7_attention_weight_max": 97.93438720703125,
      "activations/layer7_attention_weight_min": -93.96559143066406,
      "activations/layer8_attention_weight_max": 40.21236038208008,
      "activations/layer8_attention_weight_min": -41.8508186340332,
      "activations/layer9_attention_weight_max": 32.62167739868164,
      "activations/layer9_attention_weight_min": -33.34092330932617,
      "epoch": 16.1,
      "learning_rate": 4.6637499999999995e-05,
      "loss": 2.729,
      "step": 277100
    },
    {
      "activations/layer0_attention_weight_max": 16.95935821533203,
      "activations/layer0_attention_weight_min": -13.674610137939453,
      "activations/layer10_attention_weight_max": 34.27299499511719,
      "activations/layer10_attention_weight_min": -36.959144592285156,
      "activations/layer11_attention_weight_max": 33.64979553222656,
      "activations/layer11_attention_weight_min": -33.986019134521484,
      "activations/layer12_attention_weight_max": 27.93326759338379,
      "activations/layer12_attention_weight_min": -26.460657119750977,
      "activations/layer13_attention_weight_max": 42.586177825927734,
      "activations/layer13_attention_weight_min": -34.745025634765625,
      "activations/layer14_attention_weight_max": 46.321929931640625,
      "activations/layer14_attention_weight_min": -37.6505012512207,
      "activations/layer15_attention_weight_max": 40.44679260253906,
      "activations/layer15_attention_weight_min": -34.43730163574219,
      "activations/layer16_attention_weight_max": 30.465225219726562,
      "activations/layer16_attention_weight_min": -27.375337600708008,
      "activations/layer17_attention_weight_max": 32.7536506652832,
      "activations/layer17_attention_weight_min": -24.86690330505371,
      "activations/layer18_attention_weight_max": 30.751821517944336,
      "activations/layer18_attention_weight_min": -24.575666427612305,
      "activations/layer19_attention_weight_max": 34.7794303894043,
      "activations/layer19_attention_weight_min": -31.215341567993164,
      "activations/layer1_attention_weight_max": 15.708602905273438,
      "activations/layer1_attention_weight_min": -14.733177185058594,
      "activations/layer20_attention_weight_max": 27.7662353515625,
      "activations/layer20_attention_weight_min": -27.121082305908203,
      "activations/layer21_attention_weight_max": 29.18749237060547,
      "activations/layer21_attention_weight_min": -25.129655838012695,
      "activations/layer22_attention_weight_max": 42.22987747192383,
      "activations/layer22_attention_weight_min": -30.44784164428711,
      "activations/layer23_attention_weight_max": 31.8077392578125,
      "activations/layer23_attention_weight_min": -26.434619903564453,
      "activations/layer2_attention_weight_max": 33.6593132019043,
      "activations/layer2_attention_weight_min": -33.8642578125,
      "activations/layer3_attention_weight_max": 93.18843841552734,
      "activations/layer3_attention_weight_min": -99.16014862060547,
      "activations/layer4_attention_weight_max": 106.47119140625,
      "activations/layer4_attention_weight_min": -106.06082916259766,
      "activations/layer5_attention_weight_max": 54.4593505859375,
      "activations/layer5_attention_weight_min": -59.5418701171875,
      "activations/layer6_attention_weight_max": 43.50066375732422,
      "activations/layer6_attention_weight_min": -44.62373352050781,
      "activations/layer7_attention_weight_max": 91.81938934326172,
      "activations/layer7_attention_weight_min": -93.22692108154297,
      "activations/layer8_attention_weight_max": 42.89302062988281,
      "activations/layer8_attention_weight_min": -46.419620513916016,
      "activations/layer9_attention_weight_max": 33.215084075927734,
      "activations/layer9_attention_weight_min": -36.256290435791016,
      "epoch": 16.1,
      "learning_rate": 4.6618560606060603e-05,
      "loss": 2.7388,
      "step": 277150
    },
    {
      "activations/layer0_attention_weight_max": 16.451805114746094,
      "activations/layer0_attention_weight_min": -13.213652610778809,
      "activations/layer10_attention_weight_max": 36.439720153808594,
      "activations/layer10_attention_weight_min": -37.26117706298828,
      "activations/layer11_attention_weight_max": 38.37432098388672,
      "activations/layer11_attention_weight_min": -38.00769805908203,
      "activations/layer12_attention_weight_max": 27.907121658325195,
      "activations/layer12_attention_weight_min": -26.209823608398438,
      "activations/layer13_attention_weight_max": 39.33941650390625,
      "activations/layer13_attention_weight_min": -34.22863006591797,
      "activations/layer14_attention_weight_max": 39.89609909057617,
      "activations/layer14_attention_weight_min": -35.951595306396484,
      "activations/layer15_attention_weight_max": 36.224388122558594,
      "activations/layer15_attention_weight_min": -32.68544387817383,
      "activations/layer16_attention_weight_max": 27.674638748168945,
      "activations/layer16_attention_weight_min": -26.080482482910156,
      "activations/layer17_attention_weight_max": 30.02546501159668,
      "activations/layer17_attention_weight_min": -24.83734703063965,
      "activations/layer18_attention_weight_max": 29.604787826538086,
      "activations/layer18_attention_weight_min": -24.978160858154297,
      "activations/layer19_attention_weight_max": 31.599720001220703,
      "activations/layer19_attention_weight_min": -31.55853271484375,
      "activations/layer1_attention_weight_max": 15.617082595825195,
      "activations/layer1_attention_weight_min": -14.498062133789062,
      "activations/layer20_attention_weight_max": 27.842845916748047,
      "activations/layer20_attention_weight_min": -25.17753791809082,
      "activations/layer21_attention_weight_max": 28.2845458984375,
      "activations/layer21_attention_weight_min": -24.42990493774414,
      "activations/layer22_attention_weight_max": 43.42979431152344,
      "activations/layer22_attention_weight_min": -30.020599365234375,
      "activations/layer23_attention_weight_max": 32.69959259033203,
      "activations/layer23_attention_weight_min": -25.789932250976562,
      "activations/layer2_attention_weight_max": 33.62807083129883,
      "activations/layer2_attention_weight_min": -32.00018310546875,
      "activations/layer3_attention_weight_max": 95.39578247070312,
      "activations/layer3_attention_weight_min": -96.07073974609375,
      "activations/layer4_attention_weight_max": 104.94072723388672,
      "activations/layer4_attention_weight_min": -107.24065399169922,
      "activations/layer5_attention_weight_max": 50.60784912109375,
      "activations/layer5_attention_weight_min": -63.49932861328125,
      "activations/layer6_attention_weight_max": 44.731361389160156,
      "activations/layer6_attention_weight_min": -46.64055633544922,
      "activations/layer7_attention_weight_max": 88.45154571533203,
      "activations/layer7_attention_weight_min": -98.16297149658203,
      "activations/layer8_attention_weight_max": 41.51633834838867,
      "activations/layer8_attention_weight_min": -43.828041076660156,
      "activations/layer9_attention_weight_max": 32.847251892089844,
      "activations/layer9_attention_weight_min": -36.11028289794922,
      "epoch": 16.11,
      "learning_rate": 4.6599621212121205e-05,
      "loss": 2.7588,
      "step": 277200
    },
    {
      "activations/layer0_attention_weight_max": 16.469175338745117,
      "activations/layer0_attention_weight_min": -13.536178588867188,
      "activations/layer10_attention_weight_max": 33.00116729736328,
      "activations/layer10_attention_weight_min": -33.29534149169922,
      "activations/layer11_attention_weight_max": 32.20891571044922,
      "activations/layer11_attention_weight_min": -32.109619140625,
      "activations/layer12_attention_weight_max": 25.573747634887695,
      "activations/layer12_attention_weight_min": -25.546037673950195,
      "activations/layer13_attention_weight_max": 36.1773681640625,
      "activations/layer13_attention_weight_min": -34.481021881103516,
      "activations/layer14_attention_weight_max": 41.04347610473633,
      "activations/layer14_attention_weight_min": -34.69424057006836,
      "activations/layer15_attention_weight_max": 35.10654067993164,
      "activations/layer15_attention_weight_min": -31.284530639648438,
      "activations/layer16_attention_weight_max": 29.156221389770508,
      "activations/layer16_attention_weight_min": -25.718759536743164,
      "activations/layer17_attention_weight_max": 28.67987060546875,
      "activations/layer17_attention_weight_min": -25.227584838867188,
      "activations/layer18_attention_weight_max": 32.6204948425293,
      "activations/layer18_attention_weight_min": -24.850337982177734,
      "activations/layer19_attention_weight_max": 30.140735626220703,
      "activations/layer19_attention_weight_min": -30.201139450073242,
      "activations/layer1_attention_weight_max": 16.136465072631836,
      "activations/layer1_attention_weight_min": -15.653103828430176,
      "activations/layer20_attention_weight_max": 26.41621971130371,
      "activations/layer20_attention_weight_min": -24.966140747070312,
      "activations/layer21_attention_weight_max": 28.392465591430664,
      "activations/layer21_attention_weight_min": -24.321720123291016,
      "activations/layer22_attention_weight_max": 40.90161895751953,
      "activations/layer22_attention_weight_min": -31.67757797241211,
      "activations/layer23_attention_weight_max": 31.142372131347656,
      "activations/layer23_attention_weight_min": -25.425106048583984,
      "activations/layer2_attention_weight_max": 31.815601348876953,
      "activations/layer2_attention_weight_min": -33.33769989013672,
      "activations/layer3_attention_weight_max": 96.32927703857422,
      "activations/layer3_attention_weight_min": -97.40680694580078,
      "activations/layer4_attention_weight_max": 103.1450424194336,
      "activations/layer4_attention_weight_min": -106.5878677368164,
      "activations/layer5_attention_weight_max": 49.09585189819336,
      "activations/layer5_attention_weight_min": -59.15934371948242,
      "activations/layer6_attention_weight_max": 42.77985763549805,
      "activations/layer6_attention_weight_min": -44.43686294555664,
      "activations/layer7_attention_weight_max": 88.1923828125,
      "activations/layer7_attention_weight_min": -93.44957733154297,
      "activations/layer8_attention_weight_max": 39.59977340698242,
      "activations/layer8_attention_weight_min": -42.51704406738281,
      "activations/layer9_attention_weight_max": 31.031354904174805,
      "activations/layer9_attention_weight_min": -32.87376022338867,
      "epoch": 16.11,
      "learning_rate": 4.65810606060606e-05,
      "loss": 2.7401,
      "step": 277250
    },
    {
      "activations/layer0_attention_weight_max": 15.656015396118164,
      "activations/layer0_attention_weight_min": -13.24354362487793,
      "activations/layer10_attention_weight_max": 32.05958938598633,
      "activations/layer10_attention_weight_min": -34.3922004699707,
      "activations/layer11_attention_weight_max": 33.47552490234375,
      "activations/layer11_attention_weight_min": -35.14897537231445,
      "activations/layer12_attention_weight_max": 25.867023468017578,
      "activations/layer12_attention_weight_min": -26.67107391357422,
      "activations/layer13_attention_weight_max": 37.595130920410156,
      "activations/layer13_attention_weight_min": -35.18020248413086,
      "activations/layer14_attention_weight_max": 40.529483795166016,
      "activations/layer14_attention_weight_min": -38.3938102722168,
      "activations/layer15_attention_weight_max": 37.35670852661133,
      "activations/layer15_attention_weight_min": -35.3197021484375,
      "activations/layer16_attention_weight_max": 31.399744033813477,
      "activations/layer16_attention_weight_min": -28.315086364746094,
      "activations/layer17_attention_weight_max": 29.135047912597656,
      "activations/layer17_attention_weight_min": -25.298992156982422,
      "activations/layer18_attention_weight_max": 28.47040367126465,
      "activations/layer18_attention_weight_min": -22.998001098632812,
      "activations/layer19_attention_weight_max": 35.28797149658203,
      "activations/layer19_attention_weight_min": -31.087656021118164,
      "activations/layer1_attention_weight_max": 15.605659484863281,
      "activations/layer1_attention_weight_min": -15.392033576965332,
      "activations/layer20_attention_weight_max": 28.759098052978516,
      "activations/layer20_attention_weight_min": -25.27099609375,
      "activations/layer21_attention_weight_max": 28.763687133789062,
      "activations/layer21_attention_weight_min": -24.05927085876465,
      "activations/layer22_attention_weight_max": 39.63473892211914,
      "activations/layer22_attention_weight_min": -29.60045623779297,
      "activations/layer23_attention_weight_max": 33.392269134521484,
      "activations/layer23_attention_weight_min": -24.868080139160156,
      "activations/layer2_attention_weight_max": 32.157920837402344,
      "activations/layer2_attention_weight_min": -32.63456344604492,
      "activations/layer3_attention_weight_max": 99.08553314208984,
      "activations/layer3_attention_weight_min": -96.98948669433594,
      "activations/layer4_attention_weight_max": 105.93070220947266,
      "activations/layer4_attention_weight_min": -108.82628631591797,
      "activations/layer5_attention_weight_max": 50.31139373779297,
      "activations/layer5_attention_weight_min": -60.99993133544922,
      "activations/layer6_attention_weight_max": 41.74619674682617,
      "activations/layer6_attention_weight_min": -44.97236633300781,
      "activations/layer7_attention_weight_max": 94.48324584960938,
      "activations/layer7_attention_weight_min": -89.12918090820312,
      "activations/layer8_attention_weight_max": 39.8719367980957,
      "activations/layer8_attention_weight_min": -42.15565872192383,
      "activations/layer9_attention_weight_max": 31.41702651977539,
      "activations/layer9_attention_weight_min": -34.56803894042969,
      "epoch": 16.11,
      "learning_rate": 4.656212121212121e-05,
      "loss": 2.7381,
      "step": 277300
    },
    {
      "activations/layer0_attention_weight_max": 16.48040199279785,
      "activations/layer0_attention_weight_min": -12.889745712280273,
      "activations/layer10_attention_weight_max": 32.824710845947266,
      "activations/layer10_attention_weight_min": -32.89543914794922,
      "activations/layer11_attention_weight_max": 32.74925994873047,
      "activations/layer11_attention_weight_min": -33.930503845214844,
      "activations/layer12_attention_weight_max": 26.38290786743164,
      "activations/layer12_attention_weight_min": -26.633073806762695,
      "activations/layer13_attention_weight_max": 38.600433349609375,
      "activations/layer13_attention_weight_min": -34.9829216003418,
      "activations/layer14_attention_weight_max": 44.47043991088867,
      "activations/layer14_attention_weight_min": -37.27476119995117,
      "activations/layer15_attention_weight_max": 37.420989990234375,
      "activations/layer15_attention_weight_min": -33.81298065185547,
      "activations/layer16_attention_weight_max": 29.67729949951172,
      "activations/layer16_attention_weight_min": -27.635683059692383,
      "activations/layer17_attention_weight_max": 31.301137924194336,
      "activations/layer17_attention_weight_min": -25.994461059570312,
      "activations/layer18_attention_weight_max": 32.16877365112305,
      "activations/layer18_attention_weight_min": -25.678503036499023,
      "activations/layer19_attention_weight_max": 32.99384689331055,
      "activations/layer19_attention_weight_min": -30.02376365661621,
      "activations/layer1_attention_weight_max": 16.256410598754883,
      "activations/layer1_attention_weight_min": -14.5726957321167,
      "activations/layer20_attention_weight_max": 28.094566345214844,
      "activations/layer20_attention_weight_min": -24.261943817138672,
      "activations/layer21_attention_weight_max": 29.46486473083496,
      "activations/layer21_attention_weight_min": -23.865787506103516,
      "activations/layer22_attention_weight_max": 45.27204895019531,
      "activations/layer22_attention_weight_min": -31.242761611938477,
      "activations/layer23_attention_weight_max": 30.314741134643555,
      "activations/layer23_attention_weight_min": -26.540319442749023,
      "activations/layer2_attention_weight_max": 30.653797149658203,
      "activations/layer2_attention_weight_min": -31.071142196655273,
      "activations/layer3_attention_weight_max": 91.72010803222656,
      "activations/layer3_attention_weight_min": -90.78462219238281,
      "activations/layer4_attention_weight_max": 102.6133041381836,
      "activations/layer4_attention_weight_min": -106.41847229003906,
      "activations/layer5_attention_weight_max": 48.493038177490234,
      "activations/layer5_attention_weight_min": -62.023475646972656,
      "activations/layer6_attention_weight_max": 41.504817962646484,
      "activations/layer6_attention_weight_min": -44.291439056396484,
      "activations/layer7_attention_weight_max": 87.81497955322266,
      "activations/layer7_attention_weight_min": -90.86640930175781,
      "activations/layer8_attention_weight_max": 39.14309310913086,
      "activations/layer8_attention_weight_min": -40.35218048095703,
      "activations/layer9_attention_weight_max": 31.87139892578125,
      "activations/layer9_attention_weight_min": -32.16872024536133,
      "epoch": 16.12,
      "learning_rate": 4.654318181818181e-05,
      "loss": 2.7268,
      "step": 277350
    },
    {
      "activations/layer0_attention_weight_max": 16.09531021118164,
      "activations/layer0_attention_weight_min": -12.527029991149902,
      "activations/layer10_attention_weight_max": 34.71852493286133,
      "activations/layer10_attention_weight_min": -35.37617874145508,
      "activations/layer11_attention_weight_max": 35.388179779052734,
      "activations/layer11_attention_weight_min": -35.29708480834961,
      "activations/layer12_attention_weight_max": 25.742563247680664,
      "activations/layer12_attention_weight_min": -26.462251663208008,
      "activations/layer13_attention_weight_max": 39.362586975097656,
      "activations/layer13_attention_weight_min": -34.836814880371094,
      "activations/layer14_attention_weight_max": 43.44505310058594,
      "activations/layer14_attention_weight_min": -38.30797576904297,
      "activations/layer15_attention_weight_max": 37.85297393798828,
      "activations/layer15_attention_weight_min": -33.39309310913086,
      "activations/layer16_attention_weight_max": 31.41497039794922,
      "activations/layer16_attention_weight_min": -28.38088035583496,
      "activations/layer17_attention_weight_max": 33.67914962768555,
      "activations/layer17_attention_weight_min": -25.305198669433594,
      "activations/layer18_attention_weight_max": 33.80537414550781,
      "activations/layer18_attention_weight_min": -24.506254196166992,
      "activations/layer19_attention_weight_max": 35.01891326904297,
      "activations/layer19_attention_weight_min": -32.235313415527344,
      "activations/layer1_attention_weight_max": 17.056968688964844,
      "activations/layer1_attention_weight_min": -16.303144454956055,
      "activations/layer20_attention_weight_max": 29.677322387695312,
      "activations/layer20_attention_weight_min": -29.57257652282715,
      "activations/layer21_attention_weight_max": 28.509904861450195,
      "activations/layer21_attention_weight_min": -25.717187881469727,
      "activations/layer22_attention_weight_max": 45.10429000854492,
      "activations/layer22_attention_weight_min": -33.547813415527344,
      "activations/layer23_attention_weight_max": 33.625736236572266,
      "activations/layer23_attention_weight_min": -26.521968841552734,
      "activations/layer2_attention_weight_max": 32.31718063354492,
      "activations/layer2_attention_weight_min": -33.196529388427734,
      "activations/layer3_attention_weight_max": 97.78936767578125,
      "activations/layer3_attention_weight_min": -98.56558990478516,
      "activations/layer4_attention_weight_max": 108.94339752197266,
      "activations/layer4_attention_weight_min": -113.21734619140625,
      "activations/layer5_attention_weight_max": 53.89553451538086,
      "activations/layer5_attention_weight_min": -65.11328125,
      "activations/layer6_attention_weight_max": 46.06159973144531,
      "activations/layer6_attention_weight_min": -47.45378112792969,
      "activations/layer7_attention_weight_max": 95.10572814941406,
      "activations/layer7_attention_weight_min": -94.49334716796875,
      "activations/layer8_attention_weight_max": 44.96466827392578,
      "activations/layer8_attention_weight_min": -45.950775146484375,
      "activations/layer9_attention_weight_max": 37.01057052612305,
      "activations/layer9_attention_weight_min": -35.012325286865234,
      "epoch": 16.12,
      "learning_rate": 4.6524242424242414e-05,
      "loss": 2.7459,
      "step": 277400
    },
    {
      "activations/layer0_attention_weight_max": 16.682632446289062,
      "activations/layer0_attention_weight_min": -14.686168670654297,
      "activations/layer10_attention_weight_max": 31.219139099121094,
      "activations/layer10_attention_weight_min": -32.48095703125,
      "activations/layer11_attention_weight_max": 31.10887908935547,
      "activations/layer11_attention_weight_min": -31.826702117919922,
      "activations/layer12_attention_weight_max": 26.172178268432617,
      "activations/layer12_attention_weight_min": -25.505294799804688,
      "activations/layer13_attention_weight_max": 40.65317916870117,
      "activations/layer13_attention_weight_min": -34.031070709228516,
      "activations/layer14_attention_weight_max": 40.13195037841797,
      "activations/layer14_attention_weight_min": -37.63945007324219,
      "activations/layer15_attention_weight_max": 40.622337341308594,
      "activations/layer15_attention_weight_min": -35.82600402832031,
      "activations/layer16_attention_weight_max": 26.949216842651367,
      "activations/layer16_attention_weight_min": -29.118167877197266,
      "activations/layer17_attention_weight_max": 29.4906005859375,
      "activations/layer17_attention_weight_min": -24.53615951538086,
      "activations/layer18_attention_weight_max": 30.89384651184082,
      "activations/layer18_attention_weight_min": -25.005525588989258,
      "activations/layer19_attention_weight_max": 29.292800903320312,
      "activations/layer19_attention_weight_min": -32.3015251159668,
      "activations/layer1_attention_weight_max": 16.139596939086914,
      "activations/layer1_attention_weight_min": -16.535930633544922,
      "activations/layer20_attention_weight_max": 26.507543563842773,
      "activations/layer20_attention_weight_min": -23.567663192749023,
      "activations/layer21_attention_weight_max": 25.84904670715332,
      "activations/layer21_attention_weight_min": -22.75052833557129,
      "activations/layer22_attention_weight_max": 39.616024017333984,
      "activations/layer22_attention_weight_min": -28.60269546508789,
      "activations/layer23_attention_weight_max": 31.24941062927246,
      "activations/layer23_attention_weight_min": -23.844823837280273,
      "activations/layer2_attention_weight_max": 33.03002166748047,
      "activations/layer2_attention_weight_min": -34.84808349609375,
      "activations/layer3_attention_weight_max": 99.44923400878906,
      "activations/layer3_attention_weight_min": -94.44388580322266,
      "activations/layer4_attention_weight_max": 108.55806732177734,
      "activations/layer4_attention_weight_min": -106.7248306274414,
      "activations/layer5_attention_weight_max": 47.82160568237305,
      "activations/layer5_attention_weight_min": -64.66094970703125,
      "activations/layer6_attention_weight_max": 43.569786071777344,
      "activations/layer6_attention_weight_min": -48.10313415527344,
      "activations/layer7_attention_weight_max": 88.80310821533203,
      "activations/layer7_attention_weight_min": -92.04376983642578,
      "activations/layer8_attention_weight_max": 40.877593994140625,
      "activations/layer8_attention_weight_min": -41.56307601928711,
      "activations/layer9_attention_weight_max": 30.346582412719727,
      "activations/layer9_attention_weight_min": -32.10447311401367,
      "epoch": 16.12,
      "learning_rate": 4.650530303030303e-05,
      "loss": 2.7516,
      "step": 277450
    },
    {
      "activations/layer0_attention_weight_max": 15.259038925170898,
      "activations/layer0_attention_weight_min": -12.570263862609863,
      "activations/layer10_attention_weight_max": 34.177345275878906,
      "activations/layer10_attention_weight_min": -34.14168930053711,
      "activations/layer11_attention_weight_max": 33.432334899902344,
      "activations/layer11_attention_weight_min": -32.27388381958008,
      "activations/layer12_attention_weight_max": 27.875938415527344,
      "activations/layer12_attention_weight_min": -25.813920974731445,
      "activations/layer13_attention_weight_max": 40.72626495361328,
      "activations/layer13_attention_weight_min": -35.21092987060547,
      "activations/layer14_attention_weight_max": 47.55961227416992,
      "activations/layer14_attention_weight_min": -37.93143844604492,
      "activations/layer15_attention_weight_max": 45.42943572998047,
      "activations/layer15_attention_weight_min": -35.62807846069336,
      "activations/layer16_attention_weight_max": 34.763492584228516,
      "activations/layer16_attention_weight_min": -30.83721160888672,
      "activations/layer17_attention_weight_max": 41.43173599243164,
      "activations/layer17_attention_weight_min": -29.677974700927734,
      "activations/layer18_attention_weight_max": 40.071266174316406,
      "activations/layer18_attention_weight_min": -28.345077514648438,
      "activations/layer19_attention_weight_max": 40.63770294189453,
      "activations/layer19_attention_weight_min": -31.81398582458496,
      "activations/layer1_attention_weight_max": 16.683914184570312,
      "activations/layer1_attention_weight_min": -16.76630973815918,
      "activations/layer20_attention_weight_max": 35.25247573852539,
      "activations/layer20_attention_weight_min": -24.844459533691406,
      "activations/layer21_attention_weight_max": 33.82551574707031,
      "activations/layer21_attention_weight_min": -24.579809188842773,
      "activations/layer22_attention_weight_max": 49.53111267089844,
      "activations/layer22_attention_weight_min": -32.89959716796875,
      "activations/layer23_attention_weight_max": 38.40287399291992,
      "activations/layer23_attention_weight_min": -25.684370040893555,
      "activations/layer2_attention_weight_max": 31.672866821289062,
      "activations/layer2_attention_weight_min": -32.1054573059082,
      "activations/layer3_attention_weight_max": 91.82015228271484,
      "activations/layer3_attention_weight_min": -94.91370391845703,
      "activations/layer4_attention_weight_max": 105.47216796875,
      "activations/layer4_attention_weight_min": -105.63685607910156,
      "activations/layer5_attention_weight_max": 53.183258056640625,
      "activations/layer5_attention_weight_min": -66.12181091308594,
      "activations/layer6_attention_weight_max": 43.12821578979492,
      "activations/layer6_attention_weight_min": -47.0485954284668,
      "activations/layer7_attention_weight_max": 92.0782470703125,
      "activations/layer7_attention_weight_min": -93.42117309570312,
      "activations/layer8_attention_weight_max": 41.147132873535156,
      "activations/layer8_attention_weight_min": -42.20221710205078,
      "activations/layer9_attention_weight_max": 32.46419906616211,
      "activations/layer9_attention_weight_min": -34.46681213378906,
      "epoch": 16.12,
      "learning_rate": 4.648636363636363e-05,
      "loss": 2.7494,
      "step": 277500
    },
    {
      "activations/layer0_attention_weight_max": 15.913606643676758,
      "activations/layer0_attention_weight_min": -12.991451263427734,
      "activations/layer10_attention_weight_max": 35.90303421020508,
      "activations/layer10_attention_weight_min": -34.354793548583984,
      "activations/layer11_attention_weight_max": 37.04282760620117,
      "activations/layer11_attention_weight_min": -34.66516876220703,
      "activations/layer12_attention_weight_max": 33.89458084106445,
      "activations/layer12_attention_weight_min": -26.892168045043945,
      "activations/layer13_attention_weight_max": 42.40813446044922,
      "activations/layer13_attention_weight_min": -35.47408676147461,
      "activations/layer14_attention_weight_max": 42.59467697143555,
      "activations/layer14_attention_weight_min": -38.976558685302734,
      "activations/layer15_attention_weight_max": 38.74856948852539,
      "activations/layer15_attention_weight_min": -38.227542877197266,
      "activations/layer16_attention_weight_max": 36.377197265625,
      "activations/layer16_attention_weight_min": -29.29802894592285,
      "activations/layer17_attention_weight_max": 29.061092376708984,
      "activations/layer17_attention_weight_min": -26.276899337768555,
      "activations/layer18_attention_weight_max": 31.9276180267334,
      "activations/layer18_attention_weight_min": -25.79998779296875,
      "activations/layer19_attention_weight_max": 39.171485900878906,
      "activations/layer19_attention_weight_min": -36.83448791503906,
      "activations/layer1_attention_weight_max": 17.136423110961914,
      "activations/layer1_attention_weight_min": -16.017379760742188,
      "activations/layer20_attention_weight_max": 31.24435043334961,
      "activations/layer20_attention_weight_min": -27.09685707092285,
      "activations/layer21_attention_weight_max": 27.54593849182129,
      "activations/layer21_attention_weight_min": -26.502817153930664,
      "activations/layer22_attention_weight_max": 41.03287887573242,
      "activations/layer22_attention_weight_min": -32.95858383178711,
      "activations/layer23_attention_weight_max": 32.54084777832031,
      "activations/layer23_attention_weight_min": -29.063068389892578,
      "activations/layer2_attention_weight_max": 35.399253845214844,
      "activations/layer2_attention_weight_min": -33.98674011230469,
      "activations/layer3_attention_weight_max": 99.53245544433594,
      "activations/layer3_attention_weight_min": -97.063720703125,
      "activations/layer4_attention_weight_max": 104.99256896972656,
      "activations/layer4_attention_weight_min": -108.4692153930664,
      "activations/layer5_attention_weight_max": 50.704551696777344,
      "activations/layer5_attention_weight_min": -62.3194694519043,
      "activations/layer6_attention_weight_max": 44.53201675415039,
      "activations/layer6_attention_weight_min": -44.96671676635742,
      "activations/layer7_attention_weight_max": 91.18909454345703,
      "activations/layer7_attention_weight_min": -88.42240905761719,
      "activations/layer8_attention_weight_max": 42.48701477050781,
      "activations/layer8_attention_weight_min": -43.630271911621094,
      "activations/layer9_attention_weight_max": 33.61996078491211,
      "activations/layer9_attention_weight_min": -34.395668029785156,
      "epoch": 16.13,
      "learning_rate": 4.646742424242424e-05,
      "loss": 2.7245,
      "step": 277550
    },
    {
      "activations/layer0_attention_weight_max": 15.869599342346191,
      "activations/layer0_attention_weight_min": -12.7648286819458,
      "activations/layer10_attention_weight_max": 31.98855209350586,
      "activations/layer10_attention_weight_min": -31.539060592651367,
      "activations/layer11_attention_weight_max": 32.21531295776367,
      "activations/layer11_attention_weight_min": -31.61420440673828,
      "activations/layer12_attention_weight_max": 26.878583908081055,
      "activations/layer12_attention_weight_min": -25.878257751464844,
      "activations/layer13_attention_weight_max": 39.19279098510742,
      "activations/layer13_attention_weight_min": -37.069183349609375,
      "activations/layer14_attention_weight_max": 46.16844177246094,
      "activations/layer14_attention_weight_min": -40.269004821777344,
      "activations/layer15_attention_weight_max": 36.26307678222656,
      "activations/layer15_attention_weight_min": -34.29189682006836,
      "activations/layer16_attention_weight_max": 31.28862190246582,
      "activations/layer16_attention_weight_min": -26.736352920532227,
      "activations/layer17_attention_weight_max": 27.940555572509766,
      "activations/layer17_attention_weight_min": -25.967039108276367,
      "activations/layer18_attention_weight_max": 29.126792907714844,
      "activations/layer18_attention_weight_min": -25.12872314453125,
      "activations/layer19_attention_weight_max": 34.28975296020508,
      "activations/layer19_attention_weight_min": -33.18268966674805,
      "activations/layer1_attention_weight_max": 16.128326416015625,
      "activations/layer1_attention_weight_min": -15.551261901855469,
      "activations/layer20_attention_weight_max": 28.141387939453125,
      "activations/layer20_attention_weight_min": -25.664287567138672,
      "activations/layer21_attention_weight_max": 29.011714935302734,
      "activations/layer21_attention_weight_min": -24.67964744567871,
      "activations/layer22_attention_weight_max": 40.144126892089844,
      "activations/layer22_attention_weight_min": -32.34014892578125,
      "activations/layer23_attention_weight_max": 30.133413314819336,
      "activations/layer23_attention_weight_min": -26.34027099609375,
      "activations/layer2_attention_weight_max": 32.59832000732422,
      "activations/layer2_attention_weight_min": -33.689918518066406,
      "activations/layer3_attention_weight_max": 96.48135375976562,
      "activations/layer3_attention_weight_min": -97.34833526611328,
      "activations/layer4_attention_weight_max": 109.5975112915039,
      "activations/layer4_attention_weight_min": -108.559326171875,
      "activations/layer5_attention_weight_max": 50.047767639160156,
      "activations/layer5_attention_weight_min": -62.03890609741211,
      "activations/layer6_attention_weight_max": 43.04834747314453,
      "activations/layer6_attention_weight_min": -48.12919616699219,
      "activations/layer7_attention_weight_max": 89.75383758544922,
      "activations/layer7_attention_weight_min": -97.84859466552734,
      "activations/layer8_attention_weight_max": 39.176265716552734,
      "activations/layer8_attention_weight_min": -43.44860076904297,
      "activations/layer9_attention_weight_max": 30.975234985351562,
      "activations/layer9_attention_weight_min": -33.09653854370117,
      "epoch": 16.13,
      "learning_rate": 4.644848484848484e-05,
      "loss": 2.7499,
      "step": 277600
    },
    {
      "activations/layer0_attention_weight_max": 16.42794418334961,
      "activations/layer0_attention_weight_min": -12.627039909362793,
      "activations/layer10_attention_weight_max": 32.17140579223633,
      "activations/layer10_attention_weight_min": -32.00790023803711,
      "activations/layer11_attention_weight_max": 31.879817962646484,
      "activations/layer11_attention_weight_min": -31.756153106689453,
      "activations/layer12_attention_weight_max": 25.28180694580078,
      "activations/layer12_attention_weight_min": -27.493797302246094,
      "activations/layer13_attention_weight_max": 37.27328109741211,
      "activations/layer13_attention_weight_min": -37.573585510253906,
      "activations/layer14_attention_weight_max": 38.77519607543945,
      "activations/layer14_attention_weight_min": -38.36491775512695,
      "activations/layer15_attention_weight_max": 35.86482238769531,
      "activations/layer15_attention_weight_min": -32.75939178466797,
      "activations/layer16_attention_weight_max": 27.014392852783203,
      "activations/layer16_attention_weight_min": -28.713626861572266,
      "activations/layer17_attention_weight_max": 27.259300231933594,
      "activations/layer17_attention_weight_min": -25.542552947998047,
      "activations/layer18_attention_weight_max": 32.60741424560547,
      "activations/layer18_attention_weight_min": -23.976009368896484,
      "activations/layer19_attention_weight_max": 32.06892776489258,
      "activations/layer19_attention_weight_min": -29.9086856842041,
      "activations/layer1_attention_weight_max": 15.969239234924316,
      "activations/layer1_attention_weight_min": -15.46226692199707,
      "activations/layer20_attention_weight_max": 26.374309539794922,
      "activations/layer20_attention_weight_min": -22.948253631591797,
      "activations/layer21_attention_weight_max": 26.37213134765625,
      "activations/layer21_attention_weight_min": -22.66666603088379,
      "activations/layer22_attention_weight_max": 39.9897575378418,
      "activations/layer22_attention_weight_min": -29.981660842895508,
      "activations/layer23_attention_weight_max": 32.57941436767578,
      "activations/layer23_attention_weight_min": -23.905746459960938,
      "activations/layer2_attention_weight_max": 31.981216430664062,
      "activations/layer2_attention_weight_min": -34.91584777832031,
      "activations/layer3_attention_weight_max": 93.79226684570312,
      "activations/layer3_attention_weight_min": -96.94281005859375,
      "activations/layer4_attention_weight_max": 105.96862030029297,
      "activations/layer4_attention_weight_min": -107.76326751708984,
      "activations/layer5_attention_weight_max": 51.914974212646484,
      "activations/layer5_attention_weight_min": -61.58134460449219,
      "activations/layer6_attention_weight_max": 43.55583572387695,
      "activations/layer6_attention_weight_min": -44.56140899658203,
      "activations/layer7_attention_weight_max": 87.13093566894531,
      "activations/layer7_attention_weight_min": -95.0721435546875,
      "activations/layer8_attention_weight_max": 41.48235321044922,
      "activations/layer8_attention_weight_min": -41.43867492675781,
      "activations/layer9_attention_weight_max": 31.81014060974121,
      "activations/layer9_attention_weight_min": -33.166744232177734,
      "epoch": 16.13,
      "learning_rate": 4.6429545454545455e-05,
      "loss": 2.7541,
      "step": 277650
    },
    {
      "activations/layer0_attention_weight_max": 16.04055404663086,
      "activations/layer0_attention_weight_min": -13.034693717956543,
      "activations/layer10_attention_weight_max": 29.835172653198242,
      "activations/layer10_attention_weight_min": -32.54619216918945,
      "activations/layer11_attention_weight_max": 30.510623931884766,
      "activations/layer11_attention_weight_min": -31.356603622436523,
      "activations/layer12_attention_weight_max": 25.59858512878418,
      "activations/layer12_attention_weight_min": -25.73063850402832,
      "activations/layer13_attention_weight_max": 35.57087326049805,
      "activations/layer13_attention_weight_min": -33.6563835144043,
      "activations/layer14_attention_weight_max": 44.61780548095703,
      "activations/layer14_attention_weight_min": -36.44898986816406,
      "activations/layer15_attention_weight_max": 38.294036865234375,
      "activations/layer15_attention_weight_min": -32.71210861206055,
      "activations/layer16_attention_weight_max": 32.53302764892578,
      "activations/layer16_attention_weight_min": -26.612932205200195,
      "activations/layer17_attention_weight_max": 30.3756046295166,
      "activations/layer17_attention_weight_min": -25.29766273498535,
      "activations/layer18_attention_weight_max": 30.469942092895508,
      "activations/layer18_attention_weight_min": -24.720054626464844,
      "activations/layer19_attention_weight_max": 30.933149337768555,
      "activations/layer19_attention_weight_min": -30.145273208618164,
      "activations/layer1_attention_weight_max": 17.055601119995117,
      "activations/layer1_attention_weight_min": -16.471107482910156,
      "activations/layer20_attention_weight_max": 28.515485763549805,
      "activations/layer20_attention_weight_min": -23.987361907958984,
      "activations/layer21_attention_weight_max": 28.939908981323242,
      "activations/layer21_attention_weight_min": -23.002914428710938,
      "activations/layer22_attention_weight_max": 39.44064712524414,
      "activations/layer22_attention_weight_min": -29.348262786865234,
      "activations/layer23_attention_weight_max": 30.947834014892578,
      "activations/layer23_attention_weight_min": -25.644277572631836,
      "activations/layer2_attention_weight_max": 31.737438201904297,
      "activations/layer2_attention_weight_min": -32.770660400390625,
      "activations/layer3_attention_weight_max": 96.2482681274414,
      "activations/layer3_attention_weight_min": -96.15975952148438,
      "activations/layer4_attention_weight_max": 104.95094299316406,
      "activations/layer4_attention_weight_min": -105.48408508300781,
      "activations/layer5_attention_weight_max": 51.409095764160156,
      "activations/layer5_attention_weight_min": -58.46317672729492,
      "activations/layer6_attention_weight_max": 41.24494171142578,
      "activations/layer6_attention_weight_min": -43.43458938598633,
      "activations/layer7_attention_weight_max": 88.57901000976562,
      "activations/layer7_attention_weight_min": -87.00872802734375,
      "activations/layer8_attention_weight_max": 37.51953125,
      "activations/layer8_attention_weight_min": -39.290443420410156,
      "activations/layer9_attention_weight_max": 30.955245971679688,
      "activations/layer9_attention_weight_min": -31.25498390197754,
      "epoch": 16.14,
      "learning_rate": 4.641060606060606e-05,
      "loss": 2.736,
      "step": 277700
    },
    {
      "activations/layer0_attention_weight_max": 15.902836799621582,
      "activations/layer0_attention_weight_min": -11.968733787536621,
      "activations/layer10_attention_weight_max": 31.906755447387695,
      "activations/layer10_attention_weight_min": -31.14980697631836,
      "activations/layer11_attention_weight_max": 31.589176177978516,
      "activations/layer11_attention_weight_min": -30.878103256225586,
      "activations/layer12_attention_weight_max": 26.81871223449707,
      "activations/layer12_attention_weight_min": -25.09437370300293,
      "activations/layer13_attention_weight_max": 40.59364700317383,
      "activations/layer13_attention_weight_min": -32.683021545410156,
      "activations/layer14_attention_weight_max": 42.59999465942383,
      "activations/layer14_attention_weight_min": -37.69281768798828,
      "activations/layer15_attention_weight_max": 35.1264762878418,
      "activations/layer15_attention_weight_min": -32.91096878051758,
      "activations/layer16_attention_weight_max": 29.22007179260254,
      "activations/layer16_attention_weight_min": -27.299182891845703,
      "activations/layer17_attention_weight_max": 29.89688491821289,
      "activations/layer17_attention_weight_min": -25.82822608947754,
      "activations/layer18_attention_weight_max": 31.426668167114258,
      "activations/layer18_attention_weight_min": -25.443052291870117,
      "activations/layer19_attention_weight_max": 31.14529800415039,
      "activations/layer19_attention_weight_min": -29.610273361206055,
      "activations/layer1_attention_weight_max": 16.31161117553711,
      "activations/layer1_attention_weight_min": -14.542078018188477,
      "activations/layer20_attention_weight_max": 26.691816329956055,
      "activations/layer20_attention_weight_min": -24.393674850463867,
      "activations/layer21_attention_weight_max": 27.75596046447754,
      "activations/layer21_attention_weight_min": -23.750539779663086,
      "activations/layer22_attention_weight_max": 42.87507247924805,
      "activations/layer22_attention_weight_min": -28.71023178100586,
      "activations/layer23_attention_weight_max": 33.599754333496094,
      "activations/layer23_attention_weight_min": -25.71092987060547,
      "activations/layer2_attention_weight_max": 33.14183044433594,
      "activations/layer2_attention_weight_min": -33.09278106689453,
      "activations/layer3_attention_weight_max": 95.65496826171875,
      "activations/layer3_attention_weight_min": -97.35974884033203,
      "activations/layer4_attention_weight_max": 103.3928451538086,
      "activations/layer4_attention_weight_min": -104.27941131591797,
      "activations/layer5_attention_weight_max": 49.29108810424805,
      "activations/layer5_attention_weight_min": -60.225093841552734,
      "activations/layer6_attention_weight_max": 40.56959533691406,
      "activations/layer6_attention_weight_min": -42.40822219848633,
      "activations/layer7_attention_weight_max": 87.74345397949219,
      "activations/layer7_attention_weight_min": -86.6619644165039,
      "activations/layer8_attention_weight_max": 37.610843658447266,
      "activations/layer8_attention_weight_min": -39.71623611450195,
      "activations/layer9_attention_weight_max": 29.7231502532959,
      "activations/layer9_attention_weight_min": -31.298999786376953,
      "epoch": 16.14,
      "learning_rate": 4.639166666666666e-05,
      "loss": 2.7543,
      "step": 277750
    },
    {
      "activations/layer0_attention_weight_max": 16.490375518798828,
      "activations/layer0_attention_weight_min": -13.47896671295166,
      "activations/layer10_attention_weight_max": 30.45352554321289,
      "activations/layer10_attention_weight_min": -34.987178802490234,
      "activations/layer11_attention_weight_max": 31.855792999267578,
      "activations/layer11_attention_weight_min": -33.578392028808594,
      "activations/layer12_attention_weight_max": 26.018373489379883,
      "activations/layer12_attention_weight_min": -26.03293228149414,
      "activations/layer13_attention_weight_max": 38.31040954589844,
      "activations/layer13_attention_weight_min": -35.88801574707031,
      "activations/layer14_attention_weight_max": 45.42967987060547,
      "activations/layer14_attention_weight_min": -37.60422897338867,
      "activations/layer15_attention_weight_max": 36.289546966552734,
      "activations/layer15_attention_weight_min": -34.817020416259766,
      "activations/layer16_attention_weight_max": 28.546804428100586,
      "activations/layer16_attention_weight_min": -27.360713958740234,
      "activations/layer17_attention_weight_max": 28.72547721862793,
      "activations/layer17_attention_weight_min": -26.274158477783203,
      "activations/layer18_attention_weight_max": 31.2198543548584,
      "activations/layer18_attention_weight_min": -24.896507263183594,
      "activations/layer19_attention_weight_max": 33.56733322143555,
      "activations/layer19_attention_weight_min": -31.1463623046875,
      "activations/layer1_attention_weight_max": 15.591519355773926,
      "activations/layer1_attention_weight_min": -14.434928894042969,
      "activations/layer20_attention_weight_max": 28.344087600708008,
      "activations/layer20_attention_weight_min": -24.91365623474121,
      "activations/layer21_attention_weight_max": 28.34086799621582,
      "activations/layer21_attention_weight_min": -24.66739845275879,
      "activations/layer22_attention_weight_max": 39.62046813964844,
      "activations/layer22_attention_weight_min": -31.261804580688477,
      "activations/layer23_attention_weight_max": 31.143095016479492,
      "activations/layer23_attention_weight_min": -26.30816650390625,
      "activations/layer2_attention_weight_max": 33.22755813598633,
      "activations/layer2_attention_weight_min": -34.013221740722656,
      "activations/layer3_attention_weight_max": 103.79067993164062,
      "activations/layer3_attention_weight_min": -101.43965148925781,
      "activations/layer4_attention_weight_max": 108.68055725097656,
      "activations/layer4_attention_weight_min": -106.42315673828125,
      "activations/layer5_attention_weight_max": 48.9171257019043,
      "activations/layer5_attention_weight_min": -57.38756561279297,
      "activations/layer6_attention_weight_max": 40.61486053466797,
      "activations/layer6_attention_weight_min": -43.49020004272461,
      "activations/layer7_attention_weight_max": 88.7400131225586,
      "activations/layer7_attention_weight_min": -89.84845733642578,
      "activations/layer8_attention_weight_max": 39.621360778808594,
      "activations/layer8_attention_weight_min": -40.46176528930664,
      "activations/layer9_attention_weight_max": 30.77958106994629,
      "activations/layer9_attention_weight_min": -33.650856018066406,
      "epoch": 16.14,
      "learning_rate": 4.637272727272727e-05,
      "loss": 2.7568,
      "step": 277800
    },
    {
      "activations/layer0_attention_weight_max": 16.150278091430664,
      "activations/layer0_attention_weight_min": -13.109275817871094,
      "activations/layer10_attention_weight_max": 32.29912567138672,
      "activations/layer10_attention_weight_min": -33.70750045776367,
      "activations/layer11_attention_weight_max": 33.618228912353516,
      "activations/layer11_attention_weight_min": -33.907493591308594,
      "activations/layer12_attention_weight_max": 27.97927474975586,
      "activations/layer12_attention_weight_min": -26.856712341308594,
      "activations/layer13_attention_weight_max": 42.74482727050781,
      "activations/layer13_attention_weight_min": -37.212364196777344,
      "activations/layer14_attention_weight_max": 45.52431869506836,
      "activations/layer14_attention_weight_min": -41.07835388183594,
      "activations/layer15_attention_weight_max": 41.26751708984375,
      "activations/layer15_attention_weight_min": -36.598445892333984,
      "activations/layer16_attention_weight_max": 28.708696365356445,
      "activations/layer16_attention_weight_min": -29.671846389770508,
      "activations/layer17_attention_weight_max": 32.676937103271484,
      "activations/layer17_attention_weight_min": -28.15018653869629,
      "activations/layer18_attention_weight_max": 31.158447265625,
      "activations/layer18_attention_weight_min": -26.498056411743164,
      "activations/layer19_attention_weight_max": 34.49427795410156,
      "activations/layer19_attention_weight_min": -33.39875411987305,
      "activations/layer1_attention_weight_max": 18.20255470275879,
      "activations/layer1_attention_weight_min": -16.649980545043945,
      "activations/layer20_attention_weight_max": 30.11577606201172,
      "activations/layer20_attention_weight_min": -25.986101150512695,
      "activations/layer21_attention_weight_max": 31.59112548828125,
      "activations/layer21_attention_weight_min": -25.754228591918945,
      "activations/layer22_attention_weight_max": 41.03504943847656,
      "activations/layer22_attention_weight_min": -33.256507873535156,
      "activations/layer23_attention_weight_max": 30.55514144897461,
      "activations/layer23_attention_weight_min": -25.642742156982422,
      "activations/layer2_attention_weight_max": 32.68687438964844,
      "activations/layer2_attention_weight_min": -32.95979690551758,
      "activations/layer3_attention_weight_max": 91.21964263916016,
      "activations/layer3_attention_weight_min": -93.79105377197266,
      "activations/layer4_attention_weight_max": 101.48822784423828,
      "activations/layer4_attention_weight_min": -106.67115783691406,
      "activations/layer5_attention_weight_max": 49.039119720458984,
      "activations/layer5_attention_weight_min": -64.06083679199219,
      "activations/layer6_attention_weight_max": 44.14821243286133,
      "activations/layer6_attention_weight_min": -50.52686309814453,
      "activations/layer7_attention_weight_max": 93.58866119384766,
      "activations/layer7_attention_weight_min": -101.76720428466797,
      "activations/layer8_attention_weight_max": 40.85330581665039,
      "activations/layer8_attention_weight_min": -46.9627685546875,
      "activations/layer9_attention_weight_max": 32.023536682128906,
      "activations/layer9_attention_weight_min": -34.61863327026367,
      "epoch": 16.14,
      "learning_rate": 4.6353787878787875e-05,
      "loss": 2.7577,
      "step": 277850
    },
    {
      "activations/layer0_attention_weight_max": 16.023910522460938,
      "activations/layer0_attention_weight_min": -12.897881507873535,
      "activations/layer10_attention_weight_max": 34.581329345703125,
      "activations/layer10_attention_weight_min": -35.28715896606445,
      "activations/layer11_attention_weight_max": 36.01911163330078,
      "activations/layer11_attention_weight_min": -35.39421844482422,
      "activations/layer12_attention_weight_max": 28.54987335205078,
      "activations/layer12_attention_weight_min": -28.13666534423828,
      "activations/layer13_attention_weight_max": 41.894287109375,
      "activations/layer13_attention_weight_min": -36.27892303466797,
      "activations/layer14_attention_weight_max": 45.81163024902344,
      "activations/layer14_attention_weight_min": -41.85774230957031,
      "activations/layer15_attention_weight_max": 39.17500305175781,
      "activations/layer15_attention_weight_min": -35.694175720214844,
      "activations/layer16_attention_weight_max": 30.80004119873047,
      "activations/layer16_attention_weight_min": -29.237268447875977,
      "activations/layer17_attention_weight_max": 30.904932022094727,
      "activations/layer17_attention_weight_min": -25.735048294067383,
      "activations/layer18_attention_weight_max": 32.8824462890625,
      "activations/layer18_attention_weight_min": -27.555599212646484,
      "activations/layer19_attention_weight_max": 35.08814239501953,
      "activations/layer19_attention_weight_min": -32.7894287109375,
      "activations/layer1_attention_weight_max": 19.448881149291992,
      "activations/layer1_attention_weight_min": -16.360849380493164,
      "activations/layer20_attention_weight_max": 28.043296813964844,
      "activations/layer20_attention_weight_min": -27.408971786499023,
      "activations/layer21_attention_weight_max": 28.869609832763672,
      "activations/layer21_attention_weight_min": -26.100698471069336,
      "activations/layer22_attention_weight_max": 41.62721633911133,
      "activations/layer22_attention_weight_min": -31.644086837768555,
      "activations/layer23_attention_weight_max": 33.60013961791992,
      "activations/layer23_attention_weight_min": -26.722187042236328,
      "activations/layer2_attention_weight_max": 33.058860778808594,
      "activations/layer2_attention_weight_min": -32.899322509765625,
      "activations/layer3_attention_weight_max": 96.38975524902344,
      "activations/layer3_attention_weight_min": -96.85077667236328,
      "activations/layer4_attention_weight_max": 107.73345184326172,
      "activations/layer4_attention_weight_min": -105.87957763671875,
      "activations/layer5_attention_weight_max": 50.859230041503906,
      "activations/layer5_attention_weight_min": -60.35116195678711,
      "activations/layer6_attention_weight_max": 45.96321487426758,
      "activations/layer6_attention_weight_min": -48.24831771850586,
      "activations/layer7_attention_weight_max": 92.01547241210938,
      "activations/layer7_attention_weight_min": -99.15032196044922,
      "activations/layer8_attention_weight_max": 40.77711868286133,
      "activations/layer8_attention_weight_min": -40.5911979675293,
      "activations/layer9_attention_weight_max": 32.30666732788086,
      "activations/layer9_attention_weight_min": -33.933753967285156,
      "epoch": 16.15,
      "learning_rate": 4.633484848484848e-05,
      "loss": 2.7339,
      "step": 277900
    },
    {
      "activations/layer0_attention_weight_max": 15.726316452026367,
      "activations/layer0_attention_weight_min": -13.532083511352539,
      "activations/layer10_attention_weight_max": 33.6329231262207,
      "activations/layer10_attention_weight_min": -34.15806579589844,
      "activations/layer11_attention_weight_max": 32.89834976196289,
      "activations/layer11_attention_weight_min": -32.2200927734375,
      "activations/layer12_attention_weight_max": 26.60521125793457,
      "activations/layer12_attention_weight_min": -26.774362564086914,
      "activations/layer13_attention_weight_max": 39.13790512084961,
      "activations/layer13_attention_weight_min": -34.55906677246094,
      "activations/layer14_attention_weight_max": 42.75126266479492,
      "activations/layer14_attention_weight_min": -36.21711349487305,
      "activations/layer15_attention_weight_max": 37.25992965698242,
      "activations/layer15_attention_weight_min": -34.34585952758789,
      "activations/layer16_attention_weight_max": 29.78899383544922,
      "activations/layer16_attention_weight_min": -27.21990394592285,
      "activations/layer17_attention_weight_max": 32.75230026245117,
      "activations/layer17_attention_weight_min": -26.455102920532227,
      "activations/layer18_attention_weight_max": 32.51344680786133,
      "activations/layer18_attention_weight_min": -25.069978713989258,
      "activations/layer19_attention_weight_max": 31.63691520690918,
      "activations/layer19_attention_weight_min": -30.074872970581055,
      "activations/layer1_attention_weight_max": 16.621212005615234,
      "activations/layer1_attention_weight_min": -15.347076416015625,
      "activations/layer20_attention_weight_max": 29.180957794189453,
      "activations/layer20_attention_weight_min": -25.3730411529541,
      "activations/layer21_attention_weight_max": 29.33867645263672,
      "activations/layer21_attention_weight_min": -27.76324462890625,
      "activations/layer22_attention_weight_max": 42.566368103027344,
      "activations/layer22_attention_weight_min": -30.780452728271484,
      "activations/layer23_attention_weight_max": 32.31187057495117,
      "activations/layer23_attention_weight_min": -27.09274673461914,
      "activations/layer2_attention_weight_max": 31.70852279663086,
      "activations/layer2_attention_weight_min": -32.56304931640625,
      "activations/layer3_attention_weight_max": 96.24153900146484,
      "activations/layer3_attention_weight_min": -96.50032043457031,
      "activations/layer4_attention_weight_max": 105.84478759765625,
      "activations/layer4_attention_weight_min": -108.22281646728516,
      "activations/layer5_attention_weight_max": 51.57049560546875,
      "activations/layer5_attention_weight_min": -64.57698059082031,
      "activations/layer6_attention_weight_max": 42.9117317199707,
      "activations/layer6_attention_weight_min": -47.19086837768555,
      "activations/layer7_attention_weight_max": 91.48841857910156,
      "activations/layer7_attention_weight_min": -93.0880355834961,
      "activations/layer8_attention_weight_max": 41.694698333740234,
      "activations/layer8_attention_weight_min": -42.336673736572266,
      "activations/layer9_attention_weight_max": 32.86699676513672,
      "activations/layer9_attention_weight_min": -34.47980880737305,
      "epoch": 16.15,
      "learning_rate": 4.6316287878787874e-05,
      "loss": 2.7496,
      "step": 277950
    },
    {
      "activations/layer0_attention_weight_max": 15.657942771911621,
      "activations/layer0_attention_weight_min": -14.005057334899902,
      "activations/layer10_attention_weight_max": 32.00059509277344,
      "activations/layer10_attention_weight_min": -34.003395080566406,
      "activations/layer11_attention_weight_max": 32.397823333740234,
      "activations/layer11_attention_weight_min": -33.225502014160156,
      "activations/layer12_attention_weight_max": 23.862165451049805,
      "activations/layer12_attention_weight_min": -25.007734298706055,
      "activations/layer13_attention_weight_max": 35.34114456176758,
      "activations/layer13_attention_weight_min": -33.62974548339844,
      "activations/layer14_attention_weight_max": 39.65317153930664,
      "activations/layer14_attention_weight_min": -37.21599578857422,
      "activations/layer15_attention_weight_max": 34.9040641784668,
      "activations/layer15_attention_weight_min": -34.193843841552734,
      "activations/layer16_attention_weight_max": 29.037479400634766,
      "activations/layer16_attention_weight_min": -27.352901458740234,
      "activations/layer17_attention_weight_max": 28.2414493560791,
      "activations/layer17_attention_weight_min": -24.400283813476562,
      "activations/layer18_attention_weight_max": 32.16591262817383,
      "activations/layer18_attention_weight_min": -24.292728424072266,
      "activations/layer19_attention_weight_max": 37.64182662963867,
      "activations/layer19_attention_weight_min": -30.590595245361328,
      "activations/layer1_attention_weight_max": 17.695470809936523,
      "activations/layer1_attention_weight_min": -17.019439697265625,
      "activations/layer20_attention_weight_max": 33.18124771118164,
      "activations/layer20_attention_weight_min": -25.48687744140625,
      "activations/layer21_attention_weight_max": 33.49119186401367,
      "activations/layer21_attention_weight_min": -24.641098022460938,
      "activations/layer22_attention_weight_max": 42.87012481689453,
      "activations/layer22_attention_weight_min": -32.156639099121094,
      "activations/layer23_attention_weight_max": 34.937538146972656,
      "activations/layer23_attention_weight_min": -26.663593292236328,
      "activations/layer2_attention_weight_max": 31.898115158081055,
      "activations/layer2_attention_weight_min": -33.95439147949219,
      "activations/layer3_attention_weight_max": 100.5766372680664,
      "activations/layer3_attention_weight_min": -97.27619934082031,
      "activations/layer4_attention_weight_max": 111.10455322265625,
      "activations/layer4_attention_weight_min": -108.92278289794922,
      "activations/layer5_attention_weight_max": 50.439605712890625,
      "activations/layer5_attention_weight_min": -61.93069076538086,
      "activations/layer6_attention_weight_max": 44.896446228027344,
      "activations/layer6_attention_weight_min": -46.98080062866211,
      "activations/layer7_attention_weight_max": 89.08829498291016,
      "activations/layer7_attention_weight_min": -93.47237396240234,
      "activations/layer8_attention_weight_max": 40.61317825317383,
      "activations/layer8_attention_weight_min": -44.30933380126953,
      "activations/layer9_attention_weight_max": 29.942901611328125,
      "activations/layer9_attention_weight_min": -32.20336151123047,
      "epoch": 16.15,
      "learning_rate": 4.629734848484848e-05,
      "loss": 2.738,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.5144,
      "eval_samples_per_second": 504.32,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.5144,
      "eval_openwebtext_samples_per_second": 504.32,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 2.0259,
      "eval_wikitext_samples_per_second": 225.084,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_lambada_loss": 2.376953125,
      "eval_lambada_ppl": 10.77203177799245,
      "eval_lambada_runtime": 9.6127,
      "eval_lambada_samples_per_second": 506.52,
      "step": 278000
    },
    {
      "activations/layer0_attention_weight_max": 17.149600982666016,
      "activations/layer0_attention_weight_min": -12.459291458129883,
      "activations/layer10_attention_weight_max": 36.55006790161133,
      "activations/layer10_attention_weight_min": -38.02464294433594,
      "activations/layer11_attention_weight_max": 35.630897521972656,
      "activations/layer11_attention_weight_min": -36.60982131958008,
      "activations/layer12_attention_weight_max": 30.924327850341797,
      "activations/layer12_attention_weight_min": -28.961326599121094,
      "activations/layer13_attention_weight_max": 45.202945709228516,
      "activations/layer13_attention_weight_min": -39.66610336303711,
      "activations/layer14_attention_weight_max": 45.39203643798828,
      "activations/layer14_attention_weight_min": -40.29785919189453,
      "activations/layer15_attention_weight_max": 44.95079803466797,
      "activations/layer15_attention_weight_min": -37.133277893066406,
      "activations/layer16_attention_weight_max": 34.178550720214844,
      "activations/layer16_attention_weight_min": -28.30562400817871,
      "activations/layer17_attention_weight_max": 34.79707717895508,
      "activations/layer17_attention_weight_min": -27.426597595214844,
      "activations/layer18_attention_weight_max": 33.4472770690918,
      "activations/layer18_attention_weight_min": -26.261892318725586,
      "activations/layer19_attention_weight_max": 35.48722457885742,
      "activations/layer19_attention_weight_min": -32.91769790649414,
      "activations/layer1_attention_weight_max": 15.972546577453613,
      "activations/layer1_attention_weight_min": -13.766386985778809,
      "activations/layer20_attention_weight_max": 31.037620544433594,
      "activations/layer20_attention_weight_min": -27.540943145751953,
      "activations/layer21_attention_weight_max": 29.75248146057129,
      "activations/layer21_attention_weight_min": -24.83213996887207,
      "activations/layer22_attention_weight_max": 42.313720703125,
      "activations/layer22_attention_weight_min": -32.15272521972656,
      "activations/layer23_attention_weight_max": 32.90228271484375,
      "activations/layer23_attention_weight_min": -23.992727279663086,
      "activations/layer2_attention_weight_max": 31.63437271118164,
      "activations/layer2_attention_weight_min": -32.949249267578125,
      "activations/layer3_attention_weight_max": 97.6343002319336,
      "activations/layer3_attention_weight_min": -98.71373748779297,
      "activations/layer4_attention_weight_max": 106.42744445800781,
      "activations/layer4_attention_weight_min": -110.44660949707031,
      "activations/layer5_attention_weight_max": 51.548519134521484,
      "activations/layer5_attention_weight_min": -63.15204620361328,
      "activations/layer6_attention_weight_max": 45.1861457824707,
      "activations/layer6_attention_weight_min": -48.59224319458008,
      "activations/layer7_attention_weight_max": 94.1260757446289,
      "activations/layer7_attention_weight_min": -106.92384338378906,
      "activations/layer8_attention_weight_max": 43.48031997680664,
      "activations/layer8_attention_weight_min": -46.97410583496094,
      "activations/layer9_attention_weight_max": 36.33490753173828,
      "activations/layer9_attention_weight_min": -36.60737991333008,
      "epoch": 16.16,
      "learning_rate": 4.627840909090909e-05,
      "loss": 2.7351,
      "step": 278050
    },
    {
      "activations/layer0_attention_weight_max": 15.757347106933594,
      "activations/layer0_attention_weight_min": -12.88941478729248,
      "activations/layer10_attention_weight_max": 36.02522659301758,
      "activations/layer10_attention_weight_min": -37.60883331298828,
      "activations/layer11_attention_weight_max": 36.99396514892578,
      "activations/layer11_attention_weight_min": -37.808349609375,
      "activations/layer12_attention_weight_max": 28.219867706298828,
      "activations/layer12_attention_weight_min": -28.051836013793945,
      "activations/layer13_attention_weight_max": 42.11189651489258,
      "activations/layer13_attention_weight_min": -37.8470344543457,
      "activations/layer14_attention_weight_max": 47.07089614868164,
      "activations/layer14_attention_weight_min": -38.9013786315918,
      "activations/layer15_attention_weight_max": 40.733375549316406,
      "activations/layer15_attention_weight_min": -35.209625244140625,
      "activations/layer16_attention_weight_max": 30.53277587890625,
      "activations/layer16_attention_weight_min": -29.393543243408203,
      "activations/layer17_attention_weight_max": 33.38322830200195,
      "activations/layer17_attention_weight_min": -28.791603088378906,
      "activations/layer18_attention_weight_max": 29.80360221862793,
      "activations/layer18_attention_weight_min": -24.288869857788086,
      "activations/layer19_attention_weight_max": 35.03754806518555,
      "activations/layer19_attention_weight_min": -30.070898056030273,
      "activations/layer1_attention_weight_max": 16.10550308227539,
      "activations/layer1_attention_weight_min": -16.09102439880371,
      "activations/layer20_attention_weight_max": 31.08358383178711,
      "activations/layer20_attention_weight_min": -25.866376876831055,
      "activations/layer21_attention_weight_max": 29.63591766357422,
      "activations/layer21_attention_weight_min": -23.44750213623047,
      "activations/layer22_attention_weight_max": 41.833980560302734,
      "activations/layer22_attention_weight_min": -33.14793395996094,
      "activations/layer23_attention_weight_max": 36.228965759277344,
      "activations/layer23_attention_weight_min": -25.05555534362793,
      "activations/layer2_attention_weight_max": 32.788082122802734,
      "activations/layer2_attention_weight_min": -31.93617057800293,
      "activations/layer3_attention_weight_max": 99.863525390625,
      "activations/layer3_attention_weight_min": -101.52530670166016,
      "activations/layer4_attention_weight_max": 115.30889892578125,
      "activations/layer4_attention_weight_min": -113.55694580078125,
      "activations/layer5_attention_weight_max": 53.10497283935547,
      "activations/layer5_attention_weight_min": -66.95611572265625,
      "activations/layer6_attention_weight_max": 48.15966033935547,
      "activations/layer6_attention_weight_min": -48.34795379638672,
      "activations/layer7_attention_weight_max": 96.78179168701172,
      "activations/layer7_attention_weight_min": -100.2393569946289,
      "activations/layer8_attention_weight_max": 43.38368225097656,
      "activations/layer8_attention_weight_min": -45.36501693725586,
      "activations/layer9_attention_weight_max": 34.15117645263672,
      "activations/layer9_attention_weight_min": -35.959224700927734,
      "epoch": 16.16,
      "learning_rate": 4.625946969696969e-05,
      "loss": 2.7425,
      "step": 278100
    },
    {
      "activations/layer0_attention_weight_max": 15.960615158081055,
      "activations/layer0_attention_weight_min": -14.161724090576172,
      "activations/layer10_attention_weight_max": 32.327857971191406,
      "activations/layer10_attention_weight_min": -35.72795486450195,
      "activations/layer11_attention_weight_max": 34.4970703125,
      "activations/layer11_attention_weight_min": -34.20597839355469,
      "activations/layer12_attention_weight_max": 24.24073028564453,
      "activations/layer12_attention_weight_min": -27.58824920654297,
      "activations/layer13_attention_weight_max": 35.68146896362305,
      "activations/layer13_attention_weight_min": -34.499900817871094,
      "activations/layer14_attention_weight_max": 40.451499938964844,
      "activations/layer14_attention_weight_min": -37.17876052856445,
      "activations/layer15_attention_weight_max": 37.04460906982422,
      "activations/layer15_attention_weight_min": -34.01766586303711,
      "activations/layer16_attention_weight_max": 29.200361251831055,
      "activations/layer16_attention_weight_min": -28.295812606811523,
      "activations/layer17_attention_weight_max": 28.66519546508789,
      "activations/layer17_attention_weight_min": -27.83076286315918,
      "activations/layer18_attention_weight_max": 32.12009811401367,
      "activations/layer18_attention_weight_min": -24.736209869384766,
      "activations/layer19_attention_weight_max": 35.970699310302734,
      "activations/layer19_attention_weight_min": -31.42669677734375,
      "activations/layer1_attention_weight_max": 16.265920639038086,
      "activations/layer1_attention_weight_min": -14.389678955078125,
      "activations/layer20_attention_weight_max": 28.068450927734375,
      "activations/layer20_attention_weight_min": -25.626493453979492,
      "activations/layer21_attention_weight_max": 26.872817993164062,
      "activations/layer21_attention_weight_min": -23.640207290649414,
      "activations/layer22_attention_weight_max": 45.7772216796875,
      "activations/layer22_attention_weight_min": -30.44749641418457,
      "activations/layer23_attention_weight_max": 33.828792572021484,
      "activations/layer23_attention_weight_min": -26.152095794677734,
      "activations/layer2_attention_weight_max": 31.06210708618164,
      "activations/layer2_attention_weight_min": -31.43439292907715,
      "activations/layer3_attention_weight_max": 91.36754608154297,
      "activations/layer3_attention_weight_min": -95.39219665527344,
      "activations/layer4_attention_weight_max": 104.43355560302734,
      "activations/layer4_attention_weight_min": -109.7783432006836,
      "activations/layer5_attention_weight_max": 50.627479553222656,
      "activations/layer5_attention_weight_min": -64.19679260253906,
      "activations/layer6_attention_weight_max": 43.676883697509766,
      "activations/layer6_attention_weight_min": -48.393245697021484,
      "activations/layer7_attention_weight_max": 89.44017791748047,
      "activations/layer7_attention_weight_min": -97.69707489013672,
      "activations/layer8_attention_weight_max": 41.47072982788086,
      "activations/layer8_attention_weight_min": -45.909507751464844,
      "activations/layer9_attention_weight_max": 33.00517654418945,
      "activations/layer9_attention_weight_min": -35.29384994506836,
      "epoch": 16.16,
      "learning_rate": 4.624053030303029e-05,
      "loss": 2.7552,
      "step": 278150
    },
    {
      "activations/layer0_attention_weight_max": 15.600013732910156,
      "activations/layer0_attention_weight_min": -13.349963188171387,
      "activations/layer10_attention_weight_max": 35.5775032043457,
      "activations/layer10_attention_weight_min": -35.995338439941406,
      "activations/layer11_attention_weight_max": 33.749027252197266,
      "activations/layer11_attention_weight_min": -36.008338928222656,
      "activations/layer12_attention_weight_max": 26.678678512573242,
      "activations/layer12_attention_weight_min": -27.223127365112305,
      "activations/layer13_attention_weight_max": 43.434349060058594,
      "activations/layer13_attention_weight_min": -35.499053955078125,
      "activations/layer14_attention_weight_max": 47.17471694946289,
      "activations/layer14_attention_weight_min": -36.52132034301758,
      "activations/layer15_attention_weight_max": 42.11449432373047,
      "activations/layer15_attention_weight_min": -33.80405044555664,
      "activations/layer16_attention_weight_max": 30.68166160583496,
      "activations/layer16_attention_weight_min": -28.088159561157227,
      "activations/layer17_attention_weight_max": 30.801605224609375,
      "activations/layer17_attention_weight_min": -24.938133239746094,
      "activations/layer18_attention_weight_max": 32.615665435791016,
      "activations/layer18_attention_weight_min": -24.40494728088379,
      "activations/layer19_attention_weight_max": 32.656272888183594,
      "activations/layer19_attention_weight_min": -31.382099151611328,
      "activations/layer1_attention_weight_max": 16.2866153717041,
      "activations/layer1_attention_weight_min": -15.402100563049316,
      "activations/layer20_attention_weight_max": 27.916404724121094,
      "activations/layer20_attention_weight_min": -25.107343673706055,
      "activations/layer21_attention_weight_max": 27.630765914916992,
      "activations/layer21_attention_weight_min": -23.416391372680664,
      "activations/layer22_attention_weight_max": 43.95243453979492,
      "activations/layer22_attention_weight_min": -29.41691017150879,
      "activations/layer23_attention_weight_max": 30.378032684326172,
      "activations/layer23_attention_weight_min": -26.096302032470703,
      "activations/layer2_attention_weight_max": 32.65131378173828,
      "activations/layer2_attention_weight_min": -32.590389251708984,
      "activations/layer3_attention_weight_max": 95.18299102783203,
      "activations/layer3_attention_weight_min": -94.81742095947266,
      "activations/layer4_attention_weight_max": 107.010009765625,
      "activations/layer4_attention_weight_min": -111.18878173828125,
      "activations/layer5_attention_weight_max": 52.27926254272461,
      "activations/layer5_attention_weight_min": -63.28478240966797,
      "activations/layer6_attention_weight_max": 43.70878982543945,
      "activations/layer6_attention_weight_min": -46.07988739013672,
      "activations/layer7_attention_weight_max": 94.18409729003906,
      "activations/layer7_attention_weight_min": -96.79801177978516,
      "activations/layer8_attention_weight_max": 43.48848342895508,
      "activations/layer8_attention_weight_min": -45.670066833496094,
      "activations/layer9_attention_weight_max": 35.1519660949707,
      "activations/layer9_attention_weight_min": -34.48087692260742,
      "epoch": 16.17,
      "learning_rate": 4.622159090909091e-05,
      "loss": 2.7244,
      "step": 278200
    },
    {
      "activations/layer0_attention_weight_max": 15.901957511901855,
      "activations/layer0_attention_weight_min": -12.893477439880371,
      "activations/layer10_attention_weight_max": 32.60247802734375,
      "activations/layer10_attention_weight_min": -32.37324905395508,
      "activations/layer11_attention_weight_max": 33.397987365722656,
      "activations/layer11_attention_weight_min": -34.503326416015625,
      "activations/layer12_attention_weight_max": 29.146305084228516,
      "activations/layer12_attention_weight_min": -33.076969146728516,
      "activations/layer13_attention_weight_max": 42.972434997558594,
      "activations/layer13_attention_weight_min": -35.16169738769531,
      "activations/layer14_attention_weight_max": 42.807037353515625,
      "activations/layer14_attention_weight_min": -38.42576217651367,
      "activations/layer15_attention_weight_max": 39.083641052246094,
      "activations/layer15_attention_weight_min": -35.760467529296875,
      "activations/layer16_attention_weight_max": 28.38666343688965,
      "activations/layer16_attention_weight_min": -28.329505920410156,
      "activations/layer17_attention_weight_max": 31.920061111450195,
      "activations/layer17_attention_weight_min": -25.719816207885742,
      "activations/layer18_attention_weight_max": 31.253437042236328,
      "activations/layer18_attention_weight_min": -25.275236129760742,
      "activations/layer19_attention_weight_max": 33.658451080322266,
      "activations/layer19_attention_weight_min": -30.002187728881836,
      "activations/layer1_attention_weight_max": 15.275833129882812,
      "activations/layer1_attention_weight_min": -14.151800155639648,
      "activations/layer20_attention_weight_max": 29.241981506347656,
      "activations/layer20_attention_weight_min": -25.118024826049805,
      "activations/layer21_attention_weight_max": 26.971561431884766,
      "activations/layer21_attention_weight_min": -25.038724899291992,
      "activations/layer22_attention_weight_max": 42.273136138916016,
      "activations/layer22_attention_weight_min": -31.73798942565918,
      "activations/layer23_attention_weight_max": 34.5251350402832,
      "activations/layer23_attention_weight_min": -25.86586570739746,
      "activations/layer2_attention_weight_max": 31.188884735107422,
      "activations/layer2_attention_weight_min": -31.185224533081055,
      "activations/layer3_attention_weight_max": 88.31362915039062,
      "activations/layer3_attention_weight_min": -92.89079284667969,
      "activations/layer4_attention_weight_max": 101.7100830078125,
      "activations/layer4_attention_weight_min": -107.354736328125,
      "activations/layer5_attention_weight_max": 50.22059631347656,
      "activations/layer5_attention_weight_min": -64.1112060546875,
      "activations/layer6_attention_weight_max": 40.386497497558594,
      "activations/layer6_attention_weight_min": -44.24992370605469,
      "activations/layer7_attention_weight_max": 89.36936950683594,
      "activations/layer7_attention_weight_min": -87.72621154785156,
      "activations/layer8_attention_weight_max": 42.077144622802734,
      "activations/layer8_attention_weight_min": -42.00203323364258,
      "activations/layer9_attention_weight_max": 31.683135986328125,
      "activations/layer9_attention_weight_min": -31.84458351135254,
      "epoch": 16.17,
      "learning_rate": 4.620265151515151e-05,
      "loss": 2.7367,
      "step": 278250
    },
    {
      "activations/layer0_attention_weight_max": 15.738946914672852,
      "activations/layer0_attention_weight_min": -12.86152458190918,
      "activations/layer10_attention_weight_max": 29.781064987182617,
      "activations/layer10_attention_weight_min": -30.197263717651367,
      "activations/layer11_attention_weight_max": 30.448291778564453,
      "activations/layer11_attention_weight_min": -31.399892807006836,
      "activations/layer12_attention_weight_max": 24.08102035522461,
      "activations/layer12_attention_weight_min": -23.894004821777344,
      "activations/layer13_attention_weight_max": 36.50304412841797,
      "activations/layer13_attention_weight_min": -31.92350196838379,
      "activations/layer14_attention_weight_max": 38.67967987060547,
      "activations/layer14_attention_weight_min": -34.142574310302734,
      "activations/layer15_attention_weight_max": 37.48562240600586,
      "activations/layer15_attention_weight_min": -33.73603820800781,
      "activations/layer16_attention_weight_max": 28.836626052856445,
      "activations/layer16_attention_weight_min": -27.21736717224121,
      "activations/layer17_attention_weight_max": 27.918649673461914,
      "activations/layer17_attention_weight_min": -24.847209930419922,
      "activations/layer18_attention_weight_max": 28.733488082885742,
      "activations/layer18_attention_weight_min": -26.26664924621582,
      "activations/layer19_attention_weight_max": 33.855552673339844,
      "activations/layer19_attention_weight_min": -30.627796173095703,
      "activations/layer1_attention_weight_max": 15.7750244140625,
      "activations/layer1_attention_weight_min": -14.871111869812012,
      "activations/layer20_attention_weight_max": 29.341039657592773,
      "activations/layer20_attention_weight_min": -25.19914436340332,
      "activations/layer21_attention_weight_max": 29.240184783935547,
      "activations/layer21_attention_weight_min": -24.01359748840332,
      "activations/layer22_attention_weight_max": 37.618404388427734,
      "activations/layer22_attention_weight_min": -29.82859992980957,
      "activations/layer23_attention_weight_max": 32.376380920410156,
      "activations/layer23_attention_weight_min": -25.135175704956055,
      "activations/layer2_attention_weight_max": 29.869548797607422,
      "activations/layer2_attention_weight_min": -30.95172882080078,
      "activations/layer3_attention_weight_max": 89.6144790649414,
      "activations/layer3_attention_weight_min": -91.9579086303711,
      "activations/layer4_attention_weight_max": 99.66564178466797,
      "activations/layer4_attention_weight_min": -100.7577896118164,
      "activations/layer5_attention_weight_max": 47.988914489746094,
      "activations/layer5_attention_weight_min": -56.714019775390625,
      "activations/layer6_attention_weight_max": 39.66474533081055,
      "activations/layer6_attention_weight_min": -45.32231903076172,
      "activations/layer7_attention_weight_max": 85.87593078613281,
      "activations/layer7_attention_weight_min": -91.14608001708984,
      "activations/layer8_attention_weight_max": 38.94093322753906,
      "activations/layer8_attention_weight_min": -40.632747650146484,
      "activations/layer9_attention_weight_max": 28.403257369995117,
      "activations/layer9_attention_weight_min": -31.138639450073242,
      "epoch": 16.17,
      "learning_rate": 4.618371212121212e-05,
      "loss": 2.7409,
      "step": 278300
    },
    {
      "activations/layer0_attention_weight_max": 15.989433288574219,
      "activations/layer0_attention_weight_min": -12.383347511291504,
      "activations/layer10_attention_weight_max": 35.98054885864258,
      "activations/layer10_attention_weight_min": -37.018333435058594,
      "activations/layer11_attention_weight_max": 35.869388580322266,
      "activations/layer11_attention_weight_min": -38.262001037597656,
      "activations/layer12_attention_weight_max": 26.984973907470703,
      "activations/layer12_attention_weight_min": -26.484575271606445,
      "activations/layer13_attention_weight_max": 39.77788162231445,
      "activations/layer13_attention_weight_min": -36.964073181152344,
      "activations/layer14_attention_weight_max": 48.16688537597656,
      "activations/layer14_attention_weight_min": -40.19750213623047,
      "activations/layer15_attention_weight_max": 39.87151336669922,
      "activations/layer15_attention_weight_min": -35.478309631347656,
      "activations/layer16_attention_weight_max": 32.469573974609375,
      "activations/layer16_attention_weight_min": -29.138145446777344,
      "activations/layer17_attention_weight_max": 30.50278663635254,
      "activations/layer17_attention_weight_min": -28.325456619262695,
      "activations/layer18_attention_weight_max": 32.079952239990234,
      "activations/layer18_attention_weight_min": -27.981178283691406,
      "activations/layer19_attention_weight_max": 38.24758529663086,
      "activations/layer19_attention_weight_min": -36.635169982910156,
      "activations/layer1_attention_weight_max": 16.801481246948242,
      "activations/layer1_attention_weight_min": -17.46259880065918,
      "activations/layer20_attention_weight_max": 33.31578063964844,
      "activations/layer20_attention_weight_min": -29.555479049682617,
      "activations/layer21_attention_weight_max": 31.67686653137207,
      "activations/layer21_attention_weight_min": -28.79235076904297,
      "activations/layer22_attention_weight_max": 49.858314514160156,
      "activations/layer22_attention_weight_min": -34.56174850463867,
      "activations/layer23_attention_weight_max": 34.13510513305664,
      "activations/layer23_attention_weight_min": -29.036718368530273,
      "activations/layer2_attention_weight_max": 33.07779312133789,
      "activations/layer2_attention_weight_min": -31.53175163269043,
      "activations/layer3_attention_weight_max": 98.05003356933594,
      "activations/layer3_attention_weight_min": -96.88662719726562,
      "activations/layer4_attention_weight_max": 104.64873504638672,
      "activations/layer4_attention_weight_min": -108.02254486083984,
      "activations/layer5_attention_weight_max": 49.910823822021484,
      "activations/layer5_attention_weight_min": -62.66309356689453,
      "activations/layer6_attention_weight_max": 44.375221252441406,
      "activations/layer6_attention_weight_min": -47.2462272644043,
      "activations/layer7_attention_weight_max": 93.81551361083984,
      "activations/layer7_attention_weight_min": -103.24532318115234,
      "activations/layer8_attention_weight_max": 43.987220764160156,
      "activations/layer8_attention_weight_min": -46.749656677246094,
      "activations/layer9_attention_weight_max": 30.836780548095703,
      "activations/layer9_attention_weight_min": -34.388763427734375,
      "epoch": 16.17,
      "learning_rate": 4.616477272727272e-05,
      "loss": 2.7415,
      "step": 278350
    },
    {
      "activations/layer0_attention_weight_max": 16.526302337646484,
      "activations/layer0_attention_weight_min": -12.935025215148926,
      "activations/layer10_attention_weight_max": 36.63533401489258,
      "activations/layer10_attention_weight_min": -34.65559005737305,
      "activations/layer11_attention_weight_max": 36.049102783203125,
      "activations/layer11_attention_weight_min": -36.526588439941406,
      "activations/layer12_attention_weight_max": 30.662094116210938,
      "activations/layer12_attention_weight_min": -28.86449432373047,
      "activations/layer13_attention_weight_max": 45.62322235107422,
      "activations/layer13_attention_weight_min": -37.979248046875,
      "activations/layer14_attention_weight_max": 49.373714447021484,
      "activations/layer14_attention_weight_min": -43.85224151611328,
      "activations/layer15_attention_weight_max": 48.45907974243164,
      "activations/layer15_attention_weight_min": -40.989662170410156,
      "activations/layer16_attention_weight_max": 35.37654495239258,
      "activations/layer16_attention_weight_min": -31.162097930908203,
      "activations/layer17_attention_weight_max": 36.44878005981445,
      "activations/layer17_attention_weight_min": -29.8944034576416,
      "activations/layer18_attention_weight_max": 38.43705368041992,
      "activations/layer18_attention_weight_min": -28.179887771606445,
      "activations/layer19_attention_weight_max": 43.237754821777344,
      "activations/layer19_attention_weight_min": -36.23078155517578,
      "activations/layer1_attention_weight_max": 17.358936309814453,
      "activations/layer1_attention_weight_min": -15.50775146484375,
      "activations/layer20_attention_weight_max": 36.035316467285156,
      "activations/layer20_attention_weight_min": -28.688634872436523,
      "activations/layer21_attention_weight_max": 35.655521392822266,
      "activations/layer21_attention_weight_min": -27.892118453979492,
      "activations/layer22_attention_weight_max": 56.17274475097656,
      "activations/layer22_attention_weight_min": -37.70594024658203,
      "activations/layer23_attention_weight_max": 44.0581169128418,
      "activations/layer23_attention_weight_min": -28.83087730407715,
      "activations/layer2_attention_weight_max": 31.056594848632812,
      "activations/layer2_attention_weight_min": -32.631874084472656,
      "activations/layer3_attention_weight_max": 93.5545883178711,
      "activations/layer3_attention_weight_min": -94.26898956298828,
      "activations/layer4_attention_weight_max": 101.19632720947266,
      "activations/layer4_attention_weight_min": -106.75288391113281,
      "activations/layer5_attention_weight_max": 51.02611541748047,
      "activations/layer5_attention_weight_min": -61.12522888183594,
      "activations/layer6_attention_weight_max": 42.57583236694336,
      "activations/layer6_attention_weight_min": -44.68180847167969,
      "activations/layer7_attention_weight_max": 91.09146118164062,
      "activations/layer7_attention_weight_min": -94.11949157714844,
      "activations/layer8_attention_weight_max": 41.18681716918945,
      "activations/layer8_attention_weight_min": -44.898799896240234,
      "activations/layer9_attention_weight_max": 35.15224075317383,
      "activations/layer9_attention_weight_min": -37.71891784667969,
      "epoch": 16.18,
      "learning_rate": 4.6145833333333335e-05,
      "loss": 2.7353,
      "step": 278400
    },
    {
      "activations/layer0_attention_weight_max": 16.501296997070312,
      "activations/layer0_attention_weight_min": -12.502144813537598,
      "activations/layer10_attention_weight_max": 32.11771774291992,
      "activations/layer10_attention_weight_min": -32.605865478515625,
      "activations/layer11_attention_weight_max": 30.655256271362305,
      "activations/layer11_attention_weight_min": -30.791271209716797,
      "activations/layer12_attention_weight_max": 24.34111976623535,
      "activations/layer12_attention_weight_min": -26.139575958251953,
      "activations/layer13_attention_weight_max": 34.412776947021484,
      "activations/layer13_attention_weight_min": -32.26899337768555,
      "activations/layer14_attention_weight_max": 37.75944137573242,
      "activations/layer14_attention_weight_min": -33.13256072998047,
      "activations/layer15_attention_weight_max": 33.4925537109375,
      "activations/layer15_attention_weight_min": -31.984621047973633,
      "activations/layer16_attention_weight_max": 25.779890060424805,
      "activations/layer16_attention_weight_min": -25.43851089477539,
      "activations/layer17_attention_weight_max": 30.03627586364746,
      "activations/layer17_attention_weight_min": -24.812320709228516,
      "activations/layer18_attention_weight_max": 32.09375762939453,
      "activations/layer18_attention_weight_min": -26.978805541992188,
      "activations/layer19_attention_weight_max": 31.73597526550293,
      "activations/layer19_attention_weight_min": -29.73787498474121,
      "activations/layer1_attention_weight_max": 16.498544692993164,
      "activations/layer1_attention_weight_min": -15.234570503234863,
      "activations/layer20_attention_weight_max": 29.08718490600586,
      "activations/layer20_attention_weight_min": -25.513465881347656,
      "activations/layer21_attention_weight_max": 32.715476989746094,
      "activations/layer21_attention_weight_min": -23.75738525390625,
      "activations/layer22_attention_weight_max": 45.690181732177734,
      "activations/layer22_attention_weight_min": -29.34173583984375,
      "activations/layer23_attention_weight_max": 32.87946319580078,
      "activations/layer23_attention_weight_min": -24.72210693359375,
      "activations/layer2_attention_weight_max": 31.43813133239746,
      "activations/layer2_attention_weight_min": -31.492389678955078,
      "activations/layer3_attention_weight_max": 95.2934799194336,
      "activations/layer3_attention_weight_min": -99.2835922241211,
      "activations/layer4_attention_weight_max": 102.21768951416016,
      "activations/layer4_attention_weight_min": -103.75540924072266,
      "activations/layer5_attention_weight_max": 46.906822204589844,
      "activations/layer5_attention_weight_min": -60.20447540283203,
      "activations/layer6_attention_weight_max": 41.1242790222168,
      "activations/layer6_attention_weight_min": -45.03427505493164,
      "activations/layer7_attention_weight_max": 88.2726821899414,
      "activations/layer7_attention_weight_min": -91.5425796508789,
      "activations/layer8_attention_weight_max": 39.40004348754883,
      "activations/layer8_attention_weight_min": -44.743995666503906,
      "activations/layer9_attention_weight_max": 32.598575592041016,
      "activations/layer9_attention_weight_min": -35.54153823852539,
      "epoch": 16.18,
      "learning_rate": 4.6126893939393936e-05,
      "loss": 2.7459,
      "step": 278450
    },
    {
      "activations/layer0_attention_weight_max": 15.683343887329102,
      "activations/layer0_attention_weight_min": -12.303559303283691,
      "activations/layer10_attention_weight_max": 35.60109329223633,
      "activations/layer10_attention_weight_min": -33.641273498535156,
      "activations/layer11_attention_weight_max": 34.296424865722656,
      "activations/layer11_attention_weight_min": -32.992523193359375,
      "activations/layer12_attention_weight_max": 25.861831665039062,
      "activations/layer12_attention_weight_min": -25.8190975189209,
      "activations/layer13_attention_weight_max": 40.702449798583984,
      "activations/layer13_attention_weight_min": -35.867549896240234,
      "activations/layer14_attention_weight_max": 40.2061653137207,
      "activations/layer14_attention_weight_min": -36.05927658081055,
      "activations/layer15_attention_weight_max": 36.04154968261719,
      "activations/layer15_attention_weight_min": -32.70448684692383,
      "activations/layer16_attention_weight_max": 27.50094223022461,
      "activations/layer16_attention_weight_min": -27.269367218017578,
      "activations/layer17_attention_weight_max": 31.151405334472656,
      "activations/layer17_attention_weight_min": -24.57969093322754,
      "activations/layer18_attention_weight_max": 30.808212280273438,
      "activations/layer18_attention_weight_min": -23.81594467163086,
      "activations/layer19_attention_weight_max": 32.52177810668945,
      "activations/layer19_attention_weight_min": -31.872486114501953,
      "activations/layer1_attention_weight_max": 15.707128524780273,
      "activations/layer1_attention_weight_min": -15.9370756149292,
      "activations/layer20_attention_weight_max": 26.977615356445312,
      "activations/layer20_attention_weight_min": -25.597549438476562,
      "activations/layer21_attention_weight_max": 26.323461532592773,
      "activations/layer21_attention_weight_min": -23.484018325805664,
      "activations/layer22_attention_weight_max": 38.64536666870117,
      "activations/layer22_attention_weight_min": -28.38846778869629,
      "activations/layer23_attention_weight_max": 32.684295654296875,
      "activations/layer23_attention_weight_min": -24.781890869140625,
      "activations/layer2_attention_weight_max": 32.12916946411133,
      "activations/layer2_attention_weight_min": -31.921064376831055,
      "activations/layer3_attention_weight_max": 94.25211334228516,
      "activations/layer3_attention_weight_min": -95.88339233398438,
      "activations/layer4_attention_weight_max": 103.46282958984375,
      "activations/layer4_attention_weight_min": -107.1321792602539,
      "activations/layer5_attention_weight_max": 52.21723175048828,
      "activations/layer5_attention_weight_min": -62.26439666748047,
      "activations/layer6_attention_weight_max": 42.97537612915039,
      "activations/layer6_attention_weight_min": -46.63789749145508,
      "activations/layer7_attention_weight_max": 90.47814178466797,
      "activations/layer7_attention_weight_min": -93.79928588867188,
      "activations/layer8_attention_weight_max": 42.800559997558594,
      "activations/layer8_attention_weight_min": -43.62215042114258,
      "activations/layer9_attention_weight_max": 32.88709259033203,
      "activations/layer9_attention_weight_min": -34.47583770751953,
      "epoch": 16.18,
      "learning_rate": 4.610795454545454e-05,
      "loss": 2.7623,
      "step": 278500
    },
    {
      "activations/layer0_attention_weight_max": 15.611489295959473,
      "activations/layer0_attention_weight_min": -12.536055564880371,
      "activations/layer10_attention_weight_max": 34.147682189941406,
      "activations/layer10_attention_weight_min": -35.46745300292969,
      "activations/layer11_attention_weight_max": 33.214324951171875,
      "activations/layer11_attention_weight_min": -33.65629959106445,
      "activations/layer12_attention_weight_max": 25.860187530517578,
      "activations/layer12_attention_weight_min": -25.85987663269043,
      "activations/layer13_attention_weight_max": 39.875030517578125,
      "activations/layer13_attention_weight_min": -35.285858154296875,
      "activations/layer14_attention_weight_max": 42.688507080078125,
      "activations/layer14_attention_weight_min": -37.58755874633789,
      "activations/layer15_attention_weight_max": 39.25969696044922,
      "activations/layer15_attention_weight_min": -34.38111114501953,
      "activations/layer16_attention_weight_max": 30.62255096435547,
      "activations/layer16_attention_weight_min": -26.854143142700195,
      "activations/layer17_attention_weight_max": 31.408185958862305,
      "activations/layer17_attention_weight_min": -26.556169509887695,
      "activations/layer18_attention_weight_max": 33.110450744628906,
      "activations/layer18_attention_weight_min": -25.81753158569336,
      "activations/layer19_attention_weight_max": 37.815826416015625,
      "activations/layer19_attention_weight_min": -30.589155197143555,
      "activations/layer1_attention_weight_max": 16.559202194213867,
      "activations/layer1_attention_weight_min": -15.636213302612305,
      "activations/layer20_attention_weight_max": 33.24560546875,
      "activations/layer20_attention_weight_min": -25.990360260009766,
      "activations/layer21_attention_weight_max": 32.63018798828125,
      "activations/layer21_attention_weight_min": -26.29624366760254,
      "activations/layer22_attention_weight_max": 49.94776916503906,
      "activations/layer22_attention_weight_min": -33.42169952392578,
      "activations/layer23_attention_weight_max": 37.92655944824219,
      "activations/layer23_attention_weight_min": -29.664236068725586,
      "activations/layer2_attention_weight_max": 34.492103576660156,
      "activations/layer2_attention_weight_min": -33.79010772705078,
      "activations/layer3_attention_weight_max": 101.08348846435547,
      "activations/layer3_attention_weight_min": -97.63494873046875,
      "activations/layer4_attention_weight_max": 108.97926330566406,
      "activations/layer4_attention_weight_min": -107.7869644165039,
      "activations/layer5_attention_weight_max": 50.60361862182617,
      "activations/layer5_attention_weight_min": -59.69123840332031,
      "activations/layer6_attention_weight_max": 42.30831527709961,
      "activations/layer6_attention_weight_min": -46.53129196166992,
      "activations/layer7_attention_weight_max": 92.49677276611328,
      "activations/layer7_attention_weight_min": -95.399658203125,
      "activations/layer8_attention_weight_max": 41.79405975341797,
      "activations/layer8_attention_weight_min": -41.733604431152344,
      "activations/layer9_attention_weight_max": 33.12774658203125,
      "activations/layer9_attention_weight_min": -33.59852600097656,
      "epoch": 16.19,
      "learning_rate": 4.6089015151515146e-05,
      "loss": 2.7503,
      "step": 278550
    },
    {
      "activations/layer0_attention_weight_max": 16.27174186706543,
      "activations/layer0_attention_weight_min": -12.817392349243164,
      "activations/layer10_attention_weight_max": 35.05128860473633,
      "activations/layer10_attention_weight_min": -33.798126220703125,
      "activations/layer11_attention_weight_max": 35.28312301635742,
      "activations/layer11_attention_weight_min": -35.75360870361328,
      "activations/layer12_attention_weight_max": 27.777616500854492,
      "activations/layer12_attention_weight_min": -26.824689865112305,
      "activations/layer13_attention_weight_max": 44.65413284301758,
      "activations/layer13_attention_weight_min": -36.94879913330078,
      "activations/layer14_attention_weight_max": 43.08899688720703,
      "activations/layer14_attention_weight_min": -37.532737731933594,
      "activations/layer15_attention_weight_max": 38.696075439453125,
      "activations/layer15_attention_weight_min": -34.308074951171875,
      "activations/layer16_attention_weight_max": 29.660770416259766,
      "activations/layer16_attention_weight_min": -26.097888946533203,
      "activations/layer17_attention_weight_max": 31.136539459228516,
      "activations/layer17_attention_weight_min": -25.961734771728516,
      "activations/layer18_attention_weight_max": 31.7220458984375,
      "activations/layer18_attention_weight_min": -23.384679794311523,
      "activations/layer19_attention_weight_max": 34.34937286376953,
      "activations/layer19_attention_weight_min": -28.942157745361328,
      "activations/layer1_attention_weight_max": 16.594058990478516,
      "activations/layer1_attention_weight_min": -15.207571983337402,
      "activations/layer20_attention_weight_max": 30.047529220581055,
      "activations/layer20_attention_weight_min": -23.862638473510742,
      "activations/layer21_attention_weight_max": 30.9890193939209,
      "activations/layer21_attention_weight_min": -25.050193786621094,
      "activations/layer22_attention_weight_max": 43.07444381713867,
      "activations/layer22_attention_weight_min": -31.252588272094727,
      "activations/layer23_attention_weight_max": 33.22621154785156,
      "activations/layer23_attention_weight_min": -24.873306274414062,
      "activations/layer2_attention_weight_max": 32.64683151245117,
      "activations/layer2_attention_weight_min": -33.23558044433594,
      "activations/layer3_attention_weight_max": 92.44527435302734,
      "activations/layer3_attention_weight_min": -95.79226684570312,
      "activations/layer4_attention_weight_max": 101.93486785888672,
      "activations/layer4_attention_weight_min": -108.4284439086914,
      "activations/layer5_attention_weight_max": 50.845672607421875,
      "activations/layer5_attention_weight_min": -59.39986801147461,
      "activations/layer6_attention_weight_max": 43.75861358642578,
      "activations/layer6_attention_weight_min": -47.64545822143555,
      "activations/layer7_attention_weight_max": 87.91586303710938,
      "activations/layer7_attention_weight_min": -94.87358093261719,
      "activations/layer8_attention_weight_max": 41.79864501953125,
      "activations/layer8_attention_weight_min": -41.492244720458984,
      "activations/layer9_attention_weight_max": 33.724273681640625,
      "activations/layer9_attention_weight_min": -34.18613052368164,
      "epoch": 16.19,
      "learning_rate": 4.6070075757575754e-05,
      "loss": 2.7398,
      "step": 278600
    },
    {
      "activations/layer0_attention_weight_max": 15.918980598449707,
      "activations/layer0_attention_weight_min": -12.21494197845459,
      "activations/layer10_attention_weight_max": 42.41454315185547,
      "activations/layer10_attention_weight_min": -39.40766143798828,
      "activations/layer11_attention_weight_max": 43.698848724365234,
      "activations/layer11_attention_weight_min": -41.857608795166016,
      "activations/layer12_attention_weight_max": 27.12371253967285,
      "activations/layer12_attention_weight_min": -28.805408477783203,
      "activations/layer13_attention_weight_max": 39.42830276489258,
      "activations/layer13_attention_weight_min": -35.77198791503906,
      "activations/layer14_attention_weight_max": 45.145965576171875,
      "activations/layer14_attention_weight_min": -39.63426208496094,
      "activations/layer15_attention_weight_max": 37.538047790527344,
      "activations/layer15_attention_weight_min": -38.324806213378906,
      "activations/layer16_attention_weight_max": 33.22842788696289,
      "activations/layer16_attention_weight_min": -29.57538604736328,
      "activations/layer17_attention_weight_max": 31.852596282958984,
      "activations/layer17_attention_weight_min": -27.449216842651367,
      "activations/layer18_attention_weight_max": 35.710121154785156,
      "activations/layer18_attention_weight_min": -28.171066284179688,
      "activations/layer19_attention_weight_max": 35.19496154785156,
      "activations/layer19_attention_weight_min": -33.702938079833984,
      "activations/layer1_attention_weight_max": 16.005815505981445,
      "activations/layer1_attention_weight_min": -16.19263458251953,
      "activations/layer20_attention_weight_max": 29.995943069458008,
      "activations/layer20_attention_weight_min": -29.533653259277344,
      "activations/layer21_attention_weight_max": 27.775657653808594,
      "activations/layer21_attention_weight_min": -26.644681930541992,
      "activations/layer22_attention_weight_max": 42.812950134277344,
      "activations/layer22_attention_weight_min": -31.833419799804688,
      "activations/layer23_attention_weight_max": 33.8098258972168,
      "activations/layer23_attention_weight_min": -26.729206085205078,
      "activations/layer2_attention_weight_max": 33.976409912109375,
      "activations/layer2_attention_weight_min": -33.807533264160156,
      "activations/layer3_attention_weight_max": 96.31832885742188,
      "activations/layer3_attention_weight_min": -102.5169677734375,
      "activations/layer4_attention_weight_max": 109.86669158935547,
      "activations/layer4_attention_weight_min": -115.11026763916016,
      "activations/layer5_attention_weight_max": 51.00947570800781,
      "activations/layer5_attention_weight_min": -65.4921875,
      "activations/layer6_attention_weight_max": 47.84737777709961,
      "activations/layer6_attention_weight_min": -50.039852142333984,
      "activations/layer7_attention_weight_max": 100.9896240234375,
      "activations/layer7_attention_weight_min": -96.72677612304688,
      "activations/layer8_attention_weight_max": 47.8326416015625,
      "activations/layer8_attention_weight_min": -47.5113410949707,
      "activations/layer9_attention_weight_max": 37.65358352661133,
      "activations/layer9_attention_weight_min": -42.499202728271484,
      "epoch": 16.19,
      "learning_rate": 4.6051515151515145e-05,
      "loss": 2.7249,
      "step": 278650
    },
    {
      "activations/layer0_attention_weight_max": 15.542587280273438,
      "activations/layer0_attention_weight_min": -14.43708610534668,
      "activations/layer10_attention_weight_max": 36.62862777709961,
      "activations/layer10_attention_weight_min": -36.050052642822266,
      "activations/layer11_attention_weight_max": 36.59233856201172,
      "activations/layer11_attention_weight_min": -36.60908508300781,
      "activations/layer12_attention_weight_max": 27.376996994018555,
      "activations/layer12_attention_weight_min": -26.554805755615234,
      "activations/layer13_attention_weight_max": 41.961463928222656,
      "activations/layer13_attention_weight_min": -35.39680480957031,
      "activations/layer14_attention_weight_max": 47.091861724853516,
      "activations/layer14_attention_weight_min": -38.824466705322266,
      "activations/layer15_attention_weight_max": 40.43565368652344,
      "activations/layer15_attention_weight_min": -35.254661560058594,
      "activations/layer16_attention_weight_max": 32.840179443359375,
      "activations/layer16_attention_weight_min": -27.45410919189453,
      "activations/layer17_attention_weight_max": 31.46888542175293,
      "activations/layer17_attention_weight_min": -26.086097717285156,
      "activations/layer18_attention_weight_max": 32.893157958984375,
      "activations/layer18_attention_weight_min": -27.029342651367188,
      "activations/layer19_attention_weight_max": 35.33671188354492,
      "activations/layer19_attention_weight_min": -32.33208084106445,
      "activations/layer1_attention_weight_max": 15.941014289855957,
      "activations/layer1_attention_weight_min": -14.263163566589355,
      "activations/layer20_attention_weight_max": 30.264190673828125,
      "activations/layer20_attention_weight_min": -24.373184204101562,
      "activations/layer21_attention_weight_max": 28.71139907836914,
      "activations/layer21_attention_weight_min": -23.57603645324707,
      "activations/layer22_attention_weight_max": 44.916664123535156,
      "activations/layer22_attention_weight_min": -29.457429885864258,
      "activations/layer23_attention_weight_max": 35.286773681640625,
      "activations/layer23_attention_weight_min": -24.951866149902344,
      "activations/layer2_attention_weight_max": 32.3394775390625,
      "activations/layer2_attention_weight_min": -30.986270904541016,
      "activations/layer3_attention_weight_max": 96.57125854492188,
      "activations/layer3_attention_weight_min": -95.27169799804688,
      "activations/layer4_attention_weight_max": 107.73968505859375,
      "activations/layer4_attention_weight_min": -110.62259674072266,
      "activations/layer5_attention_weight_max": 51.6355094909668,
      "activations/layer5_attention_weight_min": -64.09967803955078,
      "activations/layer6_attention_weight_max": 43.87592315673828,
      "activations/layer6_attention_weight_min": -44.32004165649414,
      "activations/layer7_attention_weight_max": 97.12721252441406,
      "activations/layer7_attention_weight_min": -94.66805267333984,
      "activations/layer8_attention_weight_max": 43.82064437866211,
      "activations/layer8_attention_weight_min": -44.18269729614258,
      "activations/layer9_attention_weight_max": 34.27573776245117,
      "activations/layer9_attention_weight_min": -35.62734603881836,
      "epoch": 16.19,
      "learning_rate": 4.603257575757575e-05,
      "loss": 2.7438,
      "step": 278700
    },
    {
      "activations/layer0_attention_weight_max": 15.840439796447754,
      "activations/layer0_attention_weight_min": -13.208695411682129,
      "activations/layer10_attention_weight_max": 37.89972686767578,
      "activations/layer10_attention_weight_min": -36.92781448364258,
      "activations/layer11_attention_weight_max": 37.87560272216797,
      "activations/layer11_attention_weight_min": -35.54994201660156,
      "activations/layer12_attention_weight_max": 27.453462600708008,
      "activations/layer12_attention_weight_min": -25.913408279418945,
      "activations/layer13_attention_weight_max": 40.83789825439453,
      "activations/layer13_attention_weight_min": -36.912010192871094,
      "activations/layer14_attention_weight_max": 46.26981735229492,
      "activations/layer14_attention_weight_min": -39.12205123901367,
      "activations/layer15_attention_weight_max": 38.737613677978516,
      "activations/layer15_attention_weight_min": -34.49863815307617,
      "activations/layer16_attention_weight_max": 30.565872192382812,
      "activations/layer16_attention_weight_min": -27.472076416015625,
      "activations/layer17_attention_weight_max": 34.527767181396484,
      "activations/layer17_attention_weight_min": -25.408823013305664,
      "activations/layer18_attention_weight_max": 33.88725280761719,
      "activations/layer18_attention_weight_min": -24.65350341796875,
      "activations/layer19_attention_weight_max": 35.28354263305664,
      "activations/layer19_attention_weight_min": -31.54524040222168,
      "activations/layer1_attention_weight_max": 17.510427474975586,
      "activations/layer1_attention_weight_min": -16.71941375732422,
      "activations/layer20_attention_weight_max": 31.438243865966797,
      "activations/layer20_attention_weight_min": -26.331937789916992,
      "activations/layer21_attention_weight_max": 30.831405639648438,
      "activations/layer21_attention_weight_min": -26.414493560791016,
      "activations/layer22_attention_weight_max": 45.36162567138672,
      "activations/layer22_attention_weight_min": -30.00205421447754,
      "activations/layer23_attention_weight_max": 35.72100067138672,
      "activations/layer23_attention_weight_min": -25.54113006591797,
      "activations/layer2_attention_weight_max": 34.89317321777344,
      "activations/layer2_attention_weight_min": -31.78223419189453,
      "activations/layer3_attention_weight_max": 96.07477569580078,
      "activations/layer3_attention_weight_min": -97.5497055053711,
      "activations/layer4_attention_weight_max": 100.05638122558594,
      "activations/layer4_attention_weight_min": -102.37169647216797,
      "activations/layer5_attention_weight_max": 51.1665153503418,
      "activations/layer5_attention_weight_min": -61.03790283203125,
      "activations/layer6_attention_weight_max": 43.0926399230957,
      "activations/layer6_attention_weight_min": -44.42835235595703,
      "activations/layer7_attention_weight_max": 98.8175277709961,
      "activations/layer7_attention_weight_min": -92.2753677368164,
      "activations/layer8_attention_weight_max": 42.49555969238281,
      "activations/layer8_attention_weight_min": -43.83406066894531,
      "activations/layer9_attention_weight_max": 35.2288703918457,
      "activations/layer9_attention_weight_min": -36.331031799316406,
      "epoch": 16.2,
      "learning_rate": 4.601363636363636e-05,
      "loss": 2.7423,
      "step": 278750
    },
    {
      "activations/layer0_attention_weight_max": 15.272635459899902,
      "activations/layer0_attention_weight_min": -12.987095832824707,
      "activations/layer10_attention_weight_max": 35.744598388671875,
      "activations/layer10_attention_weight_min": -33.7869873046875,
      "activations/layer11_attention_weight_max": 36.52393341064453,
      "activations/layer11_attention_weight_min": -35.6584358215332,
      "activations/layer12_attention_weight_max": 26.836095809936523,
      "activations/layer12_attention_weight_min": -25.9405574798584,
      "activations/layer13_attention_weight_max": 43.63431167602539,
      "activations/layer13_attention_weight_min": -35.22025680541992,
      "activations/layer14_attention_weight_max": 43.43852615356445,
      "activations/layer14_attention_weight_min": -35.91709518432617,
      "activations/layer15_attention_weight_max": 42.990875244140625,
      "activations/layer15_attention_weight_min": -34.73181915283203,
      "activations/layer16_attention_weight_max": 31.576631546020508,
      "activations/layer16_attention_weight_min": -30.681623458862305,
      "activations/layer17_attention_weight_max": 34.72038650512695,
      "activations/layer17_attention_weight_min": -25.513946533203125,
      "activations/layer18_attention_weight_max": 34.1854248046875,
      "activations/layer18_attention_weight_min": -26.272140502929688,
      "activations/layer19_attention_weight_max": 35.07676315307617,
      "activations/layer19_attention_weight_min": -32.00020217895508,
      "activations/layer1_attention_weight_max": 16.348777770996094,
      "activations/layer1_attention_weight_min": -13.999086380004883,
      "activations/layer20_attention_weight_max": 31.695398330688477,
      "activations/layer20_attention_weight_min": -26.664011001586914,
      "activations/layer21_attention_weight_max": 34.61997985839844,
      "activations/layer21_attention_weight_min": -25.114065170288086,
      "activations/layer22_attention_weight_max": 41.63604736328125,
      "activations/layer22_attention_weight_min": -30.50982666015625,
      "activations/layer23_attention_weight_max": 34.180274963378906,
      "activations/layer23_attention_weight_min": -24.709897994995117,
      "activations/layer2_attention_weight_max": 34.350555419921875,
      "activations/layer2_attention_weight_min": -34.26095199584961,
      "activations/layer3_attention_weight_max": 98.5628662109375,
      "activations/layer3_attention_weight_min": -98.91621398925781,
      "activations/layer4_attention_weight_max": 110.07015228271484,
      "activations/layer4_attention_weight_min": -106.96315002441406,
      "activations/layer5_attention_weight_max": 50.56386947631836,
      "activations/layer5_attention_weight_min": -62.04905700683594,
      "activations/layer6_attention_weight_max": 45.16680145263672,
      "activations/layer6_attention_weight_min": -49.09638977050781,
      "activations/layer7_attention_weight_max": 97.3816909790039,
      "activations/layer7_attention_weight_min": -100.98644256591797,
      "activations/layer8_attention_weight_max": 43.65375900268555,
      "activations/layer8_attention_weight_min": -47.3467903137207,
      "activations/layer9_attention_weight_max": 33.80238723754883,
      "activations/layer9_attention_weight_min": -37.09184646606445,
      "epoch": 16.2,
      "learning_rate": 4.599469696969697e-05,
      "loss": 2.7564,
      "step": 278800
    },
    {
      "activations/layer0_attention_weight_max": 15.581676483154297,
      "activations/layer0_attention_weight_min": -15.12833309173584,
      "activations/layer10_attention_weight_max": 35.26923370361328,
      "activations/layer10_attention_weight_min": -36.35711669921875,
      "activations/layer11_attention_weight_max": 36.35686492919922,
      "activations/layer11_attention_weight_min": -35.29943084716797,
      "activations/layer12_attention_weight_max": 27.76095962524414,
      "activations/layer12_attention_weight_min": -27.37743377685547,
      "activations/layer13_attention_weight_max": 43.66285705566406,
      "activations/layer13_attention_weight_min": -37.11109161376953,
      "activations/layer14_attention_weight_max": 43.04957580566406,
      "activations/layer14_attention_weight_min": -38.763240814208984,
      "activations/layer15_attention_weight_max": 41.936561584472656,
      "activations/layer15_attention_weight_min": -33.59125900268555,
      "activations/layer16_attention_weight_max": 30.31542205810547,
      "activations/layer16_attention_weight_min": -27.636138916015625,
      "activations/layer17_attention_weight_max": 30.77475929260254,
      "activations/layer17_attention_weight_min": -24.87694549560547,
      "activations/layer18_attention_weight_max": 31.2551212310791,
      "activations/layer18_attention_weight_min": -23.551254272460938,
      "activations/layer19_attention_weight_max": 32.32356643676758,
      "activations/layer19_attention_weight_min": -29.167760848999023,
      "activations/layer1_attention_weight_max": 17.832904815673828,
      "activations/layer1_attention_weight_min": -16.589569091796875,
      "activations/layer20_attention_weight_max": 27.467105865478516,
      "activations/layer20_attention_weight_min": -23.567684173583984,
      "activations/layer21_attention_weight_max": 28.24376106262207,
      "activations/layer21_attention_weight_min": -22.922677993774414,
      "activations/layer22_attention_weight_max": 43.405052185058594,
      "activations/layer22_attention_weight_min": -30.523176193237305,
      "activations/layer23_attention_weight_max": 33.947261810302734,
      "activations/layer23_attention_weight_min": -25.048095703125,
      "activations/layer2_attention_weight_max": 34.848506927490234,
      "activations/layer2_attention_weight_min": -32.502174377441406,
      "activations/layer3_attention_weight_max": 98.04251861572266,
      "activations/layer3_attention_weight_min": -95.78128051757812,
      "activations/layer4_attention_weight_max": 106.53912353515625,
      "activations/layer4_attention_weight_min": -103.03923797607422,
      "activations/layer5_attention_weight_max": 49.806175231933594,
      "activations/layer5_attention_weight_min": -63.60765075683594,
      "activations/layer6_attention_weight_max": 46.66841125488281,
      "activations/layer6_attention_weight_min": -49.34003448486328,
      "activations/layer7_attention_weight_max": 95.97140502929688,
      "activations/layer7_attention_weight_min": -100.8256607055664,
      "activations/layer8_attention_weight_max": 45.08919143676758,
      "activations/layer8_attention_weight_min": -44.43456268310547,
      "activations/layer9_attention_weight_max": 35.1457633972168,
      "activations/layer9_attention_weight_min": -35.137508392333984,
      "epoch": 16.2,
      "learning_rate": 4.597575757575757e-05,
      "loss": 2.7552,
      "step": 278850
    },
    {
      "activations/layer0_attention_weight_max": 15.502958297729492,
      "activations/layer0_attention_weight_min": -12.84386157989502,
      "activations/layer10_attention_weight_max": 33.120426177978516,
      "activations/layer10_attention_weight_min": -34.16498947143555,
      "activations/layer11_attention_weight_max": 34.22962188720703,
      "activations/layer11_attention_weight_min": -32.58367156982422,
      "activations/layer12_attention_weight_max": 28.81684112548828,
      "activations/layer12_attention_weight_min": -25.43124008178711,
      "activations/layer13_attention_weight_max": 40.99276351928711,
      "activations/layer13_attention_weight_min": -35.20331954956055,
      "activations/layer14_attention_weight_max": 44.907203674316406,
      "activations/layer14_attention_weight_min": -38.91962814331055,
      "activations/layer15_attention_weight_max": 40.888370513916016,
      "activations/layer15_attention_weight_min": -34.13237380981445,
      "activations/layer16_attention_weight_max": 29.49962043762207,
      "activations/layer16_attention_weight_min": -31.192895889282227,
      "activations/layer17_attention_weight_max": 32.426185607910156,
      "activations/layer17_attention_weight_min": -29.727386474609375,
      "activations/layer18_attention_weight_max": 32.87242889404297,
      "activations/layer18_attention_weight_min": -25.911130905151367,
      "activations/layer19_attention_weight_max": 37.073543548583984,
      "activations/layer19_attention_weight_min": -30.98095703125,
      "activations/layer1_attention_weight_max": 15.700079917907715,
      "activations/layer1_attention_weight_min": -15.148387908935547,
      "activations/layer20_attention_weight_max": 30.912918090820312,
      "activations/layer20_attention_weight_min": -24.9118595123291,
      "activations/layer21_attention_weight_max": 30.156719207763672,
      "activations/layer21_attention_weight_min": -24.747282028198242,
      "activations/layer22_attention_weight_max": 44.26131057739258,
      "activations/layer22_attention_weight_min": -31.70321273803711,
      "activations/layer23_attention_weight_max": 35.72437286376953,
      "activations/layer23_attention_weight_min": -25.582035064697266,
      "activations/layer2_attention_weight_max": 32.519615173339844,
      "activations/layer2_attention_weight_min": -32.58106994628906,
      "activations/layer3_attention_weight_max": 91.85781860351562,
      "activations/layer3_attention_weight_min": -96.45957946777344,
      "activations/layer4_attention_weight_max": 104.63575744628906,
      "activations/layer4_attention_weight_min": -104.40870666503906,
      "activations/layer5_attention_weight_max": 49.55962371826172,
      "activations/layer5_attention_weight_min": -61.875667572021484,
      "activations/layer6_attention_weight_max": 42.43745803833008,
      "activations/layer6_attention_weight_min": -44.77633285522461,
      "activations/layer7_attention_weight_max": 94.49801635742188,
      "activations/layer7_attention_weight_min": -99.59783172607422,
      "activations/layer8_attention_weight_max": 41.23640823364258,
      "activations/layer8_attention_weight_min": -42.58363723754883,
      "activations/layer9_attention_weight_max": 33.07646942138672,
      "activations/layer9_attention_weight_min": -33.660579681396484,
      "epoch": 16.21,
      "learning_rate": 4.595681818181817e-05,
      "loss": 2.7531,
      "step": 278900
    },
    {
      "activations/layer0_attention_weight_max": 15.56612491607666,
      "activations/layer0_attention_weight_min": -12.816267967224121,
      "activations/layer10_attention_weight_max": 32.70983123779297,
      "activations/layer10_attention_weight_min": -33.15544509887695,
      "activations/layer11_attention_weight_max": 32.357784271240234,
      "activations/layer11_attention_weight_min": -34.331390380859375,
      "activations/layer12_attention_weight_max": 28.302814483642578,
      "activations/layer12_attention_weight_min": -26.907215118408203,
      "activations/layer13_attention_weight_max": 40.0297737121582,
      "activations/layer13_attention_weight_min": -35.66508483886719,
      "activations/layer14_attention_weight_max": 44.13385772705078,
      "activations/layer14_attention_weight_min": -36.2277717590332,
      "activations/layer15_attention_weight_max": 39.20124053955078,
      "activations/layer15_attention_weight_min": -35.486480712890625,
      "activations/layer16_attention_weight_max": 31.004100799560547,
      "activations/layer16_attention_weight_min": -28.104114532470703,
      "activations/layer17_attention_weight_max": 33.21506881713867,
      "activations/layer17_attention_weight_min": -27.84998321533203,
      "activations/layer18_attention_weight_max": 37.10529708862305,
      "activations/layer18_attention_weight_min": -27.202011108398438,
      "activations/layer19_attention_weight_max": 36.32937240600586,
      "activations/layer19_attention_weight_min": -32.57297134399414,
      "activations/layer1_attention_weight_max": 17.37767219543457,
      "activations/layer1_attention_weight_min": -16.403533935546875,
      "activations/layer20_attention_weight_max": 29.935029983520508,
      "activations/layer20_attention_weight_min": -27.828941345214844,
      "activations/layer21_attention_weight_max": 36.20947265625,
      "activations/layer21_attention_weight_min": -26.033201217651367,
      "activations/layer22_attention_weight_max": 44.96485900878906,
      "activations/layer22_attention_weight_min": -33.31271743774414,
      "activations/layer23_attention_weight_max": 35.53272247314453,
      "activations/layer23_attention_weight_min": -26.432798385620117,
      "activations/layer2_attention_weight_max": 33.302276611328125,
      "activations/layer2_attention_weight_min": -31.91236114501953,
      "activations/layer3_attention_weight_max": 93.24288940429688,
      "activations/layer3_attention_weight_min": -95.26663970947266,
      "activations/layer4_attention_weight_max": 105.81888580322266,
      "activations/layer4_attention_weight_min": -102.75958251953125,
      "activations/layer5_attention_weight_max": 53.21541976928711,
      "activations/layer5_attention_weight_min": -60.47509002685547,
      "activations/layer6_attention_weight_max": 42.522911071777344,
      "activations/layer6_attention_weight_min": -46.68223190307617,
      "activations/layer7_attention_weight_max": 93.7285385131836,
      "activations/layer7_attention_weight_min": -97.40767669677734,
      "activations/layer8_attention_weight_max": 40.11050796508789,
      "activations/layer8_attention_weight_min": -41.4682502746582,
      "activations/layer9_attention_weight_max": 31.83271598815918,
      "activations/layer9_attention_weight_min": -34.146018981933594,
      "epoch": 16.21,
      "learning_rate": 4.593787878787879e-05,
      "loss": 2.7376,
      "step": 278950
    },
    {
      "activations/layer0_attention_weight_max": 15.319684982299805,
      "activations/layer0_attention_weight_min": -12.790425300598145,
      "activations/layer10_attention_weight_max": 36.15216827392578,
      "activations/layer10_attention_weight_min": -36.10503005981445,
      "activations/layer11_attention_weight_max": 38.22382354736328,
      "activations/layer11_attention_weight_min": -37.421749114990234,
      "activations/layer12_attention_weight_max": 30.20028305053711,
      "activations/layer12_attention_weight_min": -27.227678298950195,
      "activations/layer13_attention_weight_max": 44.59700393676758,
      "activations/layer13_attention_weight_min": -35.191715240478516,
      "activations/layer14_attention_weight_max": 52.59990692138672,
      "activations/layer14_attention_weight_min": -41.85498809814453,
      "activations/layer15_attention_weight_max": 52.144229888916016,
      "activations/layer15_attention_weight_min": -39.51565170288086,
      "activations/layer16_attention_weight_max": 41.52273941040039,
      "activations/layer16_attention_weight_min": -30.98137092590332,
      "activations/layer17_attention_weight_max": 35.88948440551758,
      "activations/layer17_attention_weight_min": -29.288700103759766,
      "activations/layer18_attention_weight_max": 40.23717498779297,
      "activations/layer18_attention_weight_min": -32.52705764770508,
      "activations/layer19_attention_weight_max": 49.37895965576172,
      "activations/layer19_attention_weight_min": -34.75629806518555,
      "activations/layer1_attention_weight_max": 17.02193260192871,
      "activations/layer1_attention_weight_min": -14.7214994430542,
      "activations/layer20_attention_weight_max": 40.02316665649414,
      "activations/layer20_attention_weight_min": -25.682178497314453,
      "activations/layer21_attention_weight_max": 35.39313888549805,
      "activations/layer21_attention_weight_min": -26.314640045166016,
      "activations/layer22_attention_weight_max": 54.89558410644531,
      "activations/layer22_attention_weight_min": -35.589237213134766,
      "activations/layer23_attention_weight_max": 42.02405548095703,
      "activations/layer23_attention_weight_min": -26.70599365234375,
      "activations/layer2_attention_weight_max": 31.80194091796875,
      "activations/layer2_attention_weight_min": -31.970367431640625,
      "activations/layer3_attention_weight_max": 95.12049102783203,
      "activations/layer3_attention_weight_min": -94.04788970947266,
      "activations/layer4_attention_weight_max": 103.071533203125,
      "activations/layer4_attention_weight_min": -108.71197509765625,
      "activations/layer5_attention_weight_max": 48.33808135986328,
      "activations/layer5_attention_weight_min": -62.897804260253906,
      "activations/layer6_attention_weight_max": 45.80641555786133,
      "activations/layer6_attention_weight_min": -45.88955307006836,
      "activations/layer7_attention_weight_max": 100.1854019165039,
      "activations/layer7_attention_weight_min": -97.46763610839844,
      "activations/layer8_attention_weight_max": 40.74737548828125,
      "activations/layer8_attention_weight_min": -41.61985397338867,
      "activations/layer9_attention_weight_max": 33.096527099609375,
      "activations/layer9_attention_weight_min": -34.97609329223633,
      "epoch": 16.21,
      "learning_rate": 4.591893939393939e-05,
      "loss": 2.7479,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4783,
      "eval_samples_per_second": 506.47,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4783,
      "eval_openwebtext_samples_per_second": 506.47,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9899,
      "eval_wikitext_samples_per_second": 229.157,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_lambada_loss": 2.330078125,
      "eval_lambada_ppl": 10.278744528592322,
      "eval_lambada_runtime": 9.571,
      "eval_lambada_samples_per_second": 508.723,
      "step": 279000
    },
    {
      "activations/layer0_attention_weight_max": 15.857341766357422,
      "activations/layer0_attention_weight_min": -12.75194263458252,
      "activations/layer10_attention_weight_max": 33.61384963989258,
      "activations/layer10_attention_weight_min": -33.70417785644531,
      "activations/layer11_attention_weight_max": 34.76612854003906,
      "activations/layer11_attention_weight_min": -34.111412048339844,
      "activations/layer12_attention_weight_max": 29.689836502075195,
      "activations/layer12_attention_weight_min": -26.48563003540039,
      "activations/layer13_attention_weight_max": 39.431270599365234,
      "activations/layer13_attention_weight_min": -37.422813415527344,
      "activations/layer14_attention_weight_max": 43.58512496948242,
      "activations/layer14_attention_weight_min": -41.8837776184082,
      "activations/layer15_attention_weight_max": 41.93989181518555,
      "activations/layer15_attention_weight_min": -40.22966384887695,
      "activations/layer16_attention_weight_max": 30.900476455688477,
      "activations/layer16_attention_weight_min": -29.810691833496094,
      "activations/layer17_attention_weight_max": 30.580116271972656,
      "activations/layer17_attention_weight_min": -27.969789505004883,
      "activations/layer18_attention_weight_max": 36.4848747253418,
      "activations/layer18_attention_weight_min": -27.584196090698242,
      "activations/layer19_attention_weight_max": 34.64470672607422,
      "activations/layer19_attention_weight_min": -31.55489158630371,
      "activations/layer1_attention_weight_max": 17.6381778717041,
      "activations/layer1_attention_weight_min": -14.58669376373291,
      "activations/layer20_attention_weight_max": 28.252412796020508,
      "activations/layer20_attention_weight_min": -25.41551399230957,
      "activations/layer21_attention_weight_max": 28.064367294311523,
      "activations/layer21_attention_weight_min": -24.811609268188477,
      "activations/layer22_attention_weight_max": 45.594791412353516,
      "activations/layer22_attention_weight_min": -32.16883087158203,
      "activations/layer23_attention_weight_max": 35.11782455444336,
      "activations/layer23_attention_weight_min": -26.762325286865234,
      "activations/layer2_attention_weight_max": 32.21527099609375,
      "activations/layer2_attention_weight_min": -32.98207092285156,
      "activations/layer3_attention_weight_max": 96.16749572753906,
      "activations/layer3_attention_weight_min": -95.98585510253906,
      "activations/layer4_attention_weight_max": 106.17571258544922,
      "activations/layer4_attention_weight_min": -108.06195068359375,
      "activations/layer5_attention_weight_max": 49.56293487548828,
      "activations/layer5_attention_weight_min": -59.033790588378906,
      "activations/layer6_attention_weight_max": 43.98038864135742,
      "activations/layer6_attention_weight_min": -46.956302642822266,
      "activations/layer7_attention_weight_max": 95.37957000732422,
      "activations/layer7_attention_weight_min": -96.28335571289062,
      "activations/layer8_attention_weight_max": 39.940853118896484,
      "activations/layer8_attention_weight_min": -41.967750549316406,
      "activations/layer9_attention_weight_max": 32.032981872558594,
      "activations/layer9_attention_weight_min": -34.416725158691406,
      "epoch": 16.21,
      "learning_rate": 4.59e-05,
      "loss": 2.7519,
      "step": 279050
    },
    {
      "activations/layer0_attention_weight_max": 16.142030715942383,
      "activations/layer0_attention_weight_min": -12.705820083618164,
      "activations/layer10_attention_weight_max": 31.856786727905273,
      "activations/layer10_attention_weight_min": -31.841232299804688,
      "activations/layer11_attention_weight_max": 30.855329513549805,
      "activations/layer11_attention_weight_min": -31.72850799560547,
      "activations/layer12_attention_weight_max": 26.042892456054688,
      "activations/layer12_attention_weight_min": -25.77814292907715,
      "activations/layer13_attention_weight_max": 39.50529861450195,
      "activations/layer13_attention_weight_min": -34.35378646850586,
      "activations/layer14_attention_weight_max": 44.83802795410156,
      "activations/layer14_attention_weight_min": -37.610347747802734,
      "activations/layer15_attention_weight_max": 36.72868347167969,
      "activations/layer15_attention_weight_min": -33.25720977783203,
      "activations/layer16_attention_weight_max": 26.83614730834961,
      "activations/layer16_attention_weight_min": -26.29503631591797,
      "activations/layer17_attention_weight_max": 29.739301681518555,
      "activations/layer17_attention_weight_min": -25.87546730041504,
      "activations/layer18_attention_weight_max": 31.553447723388672,
      "activations/layer18_attention_weight_min": -26.622310638427734,
      "activations/layer19_attention_weight_max": 31.790618896484375,
      "activations/layer19_attention_weight_min": -32.22855758666992,
      "activations/layer1_attention_weight_max": 16.705419540405273,
      "activations/layer1_attention_weight_min": -15.952047348022461,
      "activations/layer20_attention_weight_max": 27.592687606811523,
      "activations/layer20_attention_weight_min": -29.336681365966797,
      "activations/layer21_attention_weight_max": 29.686626434326172,
      "activations/layer21_attention_weight_min": -26.92856788635254,
      "activations/layer22_attention_weight_max": 37.256919860839844,
      "activations/layer22_attention_weight_min": -31.86670684814453,
      "activations/layer23_attention_weight_max": 29.96224021911621,
      "activations/layer23_attention_weight_min": -27.63991355895996,
      "activations/layer2_attention_weight_max": 32.243770599365234,
      "activations/layer2_attention_weight_min": -32.79148864746094,
      "activations/layer3_attention_weight_max": 95.6016845703125,
      "activations/layer3_attention_weight_min": -93.814697265625,
      "activations/layer4_attention_weight_max": 107.3609390258789,
      "activations/layer4_attention_weight_min": -104.47135162353516,
      "activations/layer5_attention_weight_max": 48.94195556640625,
      "activations/layer5_attention_weight_min": -61.32262420654297,
      "activations/layer6_attention_weight_max": 41.41254806518555,
      "activations/layer6_attention_weight_min": -45.02438735961914,
      "activations/layer7_attention_weight_max": 85.18551635742188,
      "activations/layer7_attention_weight_min": -90.38995361328125,
      "activations/layer8_attention_weight_max": 39.52445602416992,
      "activations/layer8_attention_weight_min": -40.97951126098633,
      "activations/layer9_attention_weight_max": 32.8045539855957,
      "activations/layer9_attention_weight_min": -32.48612976074219,
      "epoch": 16.22,
      "learning_rate": 4.58810606060606e-05,
      "loss": 2.7474,
      "step": 279100
    },
    {
      "activations/layer0_attention_weight_max": 16.051877975463867,
      "activations/layer0_attention_weight_min": -12.48725700378418,
      "activations/layer10_attention_weight_max": 34.64231491088867,
      "activations/layer10_attention_weight_min": -35.42664337158203,
      "activations/layer11_attention_weight_max": 36.10174560546875,
      "activations/layer11_attention_weight_min": -33.174686431884766,
      "activations/layer12_attention_weight_max": 25.223325729370117,
      "activations/layer12_attention_weight_min": -25.892858505249023,
      "activations/layer13_attention_weight_max": 37.2685546875,
      "activations/layer13_attention_weight_min": -35.3896369934082,
      "activations/layer14_attention_weight_max": 39.4593391418457,
      "activations/layer14_attention_weight_min": -37.29079055786133,
      "activations/layer15_attention_weight_max": 37.78965377807617,
      "activations/layer15_attention_weight_min": -33.51802444458008,
      "activations/layer16_attention_weight_max": 29.960466384887695,
      "activations/layer16_attention_weight_min": -28.164384841918945,
      "activations/layer17_attention_weight_max": 27.44281005859375,
      "activations/layer17_attention_weight_min": -25.40859031677246,
      "activations/layer18_attention_weight_max": 28.089323043823242,
      "activations/layer18_attention_weight_min": -25.510005950927734,
      "activations/layer19_attention_weight_max": 31.793684005737305,
      "activations/layer19_attention_weight_min": -32.4825439453125,
      "activations/layer1_attention_weight_max": 17.582565307617188,
      "activations/layer1_attention_weight_min": -17.349441528320312,
      "activations/layer20_attention_weight_max": 27.70013999938965,
      "activations/layer20_attention_weight_min": -26.313926696777344,
      "activations/layer21_attention_weight_max": 29.723247528076172,
      "activations/layer21_attention_weight_min": -24.70201873779297,
      "activations/layer22_attention_weight_max": 41.26601028442383,
      "activations/layer22_attention_weight_min": -33.973793029785156,
      "activations/layer23_attention_weight_max": 31.22188949584961,
      "activations/layer23_attention_weight_min": -24.81513023376465,
      "activations/layer2_attention_weight_max": 32.882789611816406,
      "activations/layer2_attention_weight_min": -31.263221740722656,
      "activations/layer3_attention_weight_max": 95.25955963134766,
      "activations/layer3_attention_weight_min": -91.46930694580078,
      "activations/layer4_attention_weight_max": 102.8873062133789,
      "activations/layer4_attention_weight_min": -104.40077209472656,
      "activations/layer5_attention_weight_max": 50.160133361816406,
      "activations/layer5_attention_weight_min": -61.0372428894043,
      "activations/layer6_attention_weight_max": 45.23255920410156,
      "activations/layer6_attention_weight_min": -45.00920486450195,
      "activations/layer7_attention_weight_max": 96.03646850585938,
      "activations/layer7_attention_weight_min": -91.94094848632812,
      "activations/layer8_attention_weight_max": 42.97019958496094,
      "activations/layer8_attention_weight_min": -45.117061614990234,
      "activations/layer9_attention_weight_max": 34.01348876953125,
      "activations/layer9_attention_weight_min": -34.2528190612793,
      "epoch": 16.22,
      "learning_rate": 4.5862121212121214e-05,
      "loss": 2.7522,
      "step": 279150
    },
    {
      "activations/layer0_attention_weight_max": 15.875090599060059,
      "activations/layer0_attention_weight_min": -12.798258781433105,
      "activations/layer10_attention_weight_max": 36.00130844116211,
      "activations/layer10_attention_weight_min": -37.12812805175781,
      "activations/layer11_attention_weight_max": 37.59996032714844,
      "activations/layer11_attention_weight_min": -35.11235427856445,
      "activations/layer12_attention_weight_max": 28.28887367248535,
      "activations/layer12_attention_weight_min": -26.7580509185791,
      "activations/layer13_attention_weight_max": 42.29469680786133,
      "activations/layer13_attention_weight_min": -36.367767333984375,
      "activations/layer14_attention_weight_max": 41.96883010864258,
      "activations/layer14_attention_weight_min": -39.02737808227539,
      "activations/layer15_attention_weight_max": 40.03522872924805,
      "activations/layer15_attention_weight_min": -36.113216400146484,
      "activations/layer16_attention_weight_max": 30.768705368041992,
      "activations/layer16_attention_weight_min": -31.811914443969727,
      "activations/layer17_attention_weight_max": 32.70570755004883,
      "activations/layer17_attention_weight_min": -27.201690673828125,
      "activations/layer18_attention_weight_max": 31.083627700805664,
      "activations/layer18_attention_weight_min": -25.17142105102539,
      "activations/layer19_attention_weight_max": 38.386775970458984,
      "activations/layer19_attention_weight_min": -34.13712692260742,
      "activations/layer1_attention_weight_max": 16.54496955871582,
      "activations/layer1_attention_weight_min": -14.658286094665527,
      "activations/layer20_attention_weight_max": 34.246822357177734,
      "activations/layer20_attention_weight_min": -27.137977600097656,
      "activations/layer21_attention_weight_max": 32.147762298583984,
      "activations/layer21_attention_weight_min": -26.995445251464844,
      "activations/layer22_attention_weight_max": 44.426475524902344,
      "activations/layer22_attention_weight_min": -31.847402572631836,
      "activations/layer23_attention_weight_max": 33.41815185546875,
      "activations/layer23_attention_weight_min": -26.87860870361328,
      "activations/layer2_attention_weight_max": 33.50684356689453,
      "activations/layer2_attention_weight_min": -32.350765228271484,
      "activations/layer3_attention_weight_max": 98.4063949584961,
      "activations/layer3_attention_weight_min": -99.7712631225586,
      "activations/layer4_attention_weight_max": 108.0396728515625,
      "activations/layer4_attention_weight_min": -110.72322082519531,
      "activations/layer5_attention_weight_max": 54.800594329833984,
      "activations/layer5_attention_weight_min": -65.05201721191406,
      "activations/layer6_attention_weight_max": 45.93314743041992,
      "activations/layer6_attention_weight_min": -47.61101150512695,
      "activations/layer7_attention_weight_max": 94.03726196289062,
      "activations/layer7_attention_weight_min": -97.09395599365234,
      "activations/layer8_attention_weight_max": 47.56942367553711,
      "activations/layer8_attention_weight_min": -47.06316375732422,
      "activations/layer9_attention_weight_max": 34.975345611572266,
      "activations/layer9_attention_weight_min": -36.365142822265625,
      "epoch": 16.22,
      "learning_rate": 4.5843181818181816e-05,
      "loss": 2.7511,
      "step": 279200
    },
    {
      "activations/layer0_attention_weight_max": 16.35965919494629,
      "activations/layer0_attention_weight_min": -12.902392387390137,
      "activations/layer10_attention_weight_max": 31.5496826171875,
      "activations/layer10_attention_weight_min": -32.325138092041016,
      "activations/layer11_attention_weight_max": 31.43359375,
      "activations/layer11_attention_weight_min": -31.473979949951172,
      "activations/layer12_attention_weight_max": 25.36789321899414,
      "activations/layer12_attention_weight_min": -25.853063583374023,
      "activations/layer13_attention_weight_max": 35.49856185913086,
      "activations/layer13_attention_weight_min": -35.8900260925293,
      "activations/layer14_attention_weight_max": 38.56328201293945,
      "activations/layer14_attention_weight_min": -37.847923278808594,
      "activations/layer15_attention_weight_max": 33.690555572509766,
      "activations/layer15_attention_weight_min": -34.257240295410156,
      "activations/layer16_attention_weight_max": 27.580278396606445,
      "activations/layer16_attention_weight_min": -28.261018753051758,
      "activations/layer17_attention_weight_max": 29.81505012512207,
      "activations/layer17_attention_weight_min": -25.56056785583496,
      "activations/layer18_attention_weight_max": 29.4758358001709,
      "activations/layer18_attention_weight_min": -26.159521102905273,
      "activations/layer19_attention_weight_max": 32.352630615234375,
      "activations/layer19_attention_weight_min": -32.270347595214844,
      "activations/layer1_attention_weight_max": 17.089460372924805,
      "activations/layer1_attention_weight_min": -13.828298568725586,
      "activations/layer20_attention_weight_max": 26.557109832763672,
      "activations/layer20_attention_weight_min": -25.136594772338867,
      "activations/layer21_attention_weight_max": 27.01507568359375,
      "activations/layer21_attention_weight_min": -27.16606330871582,
      "activations/layer22_attention_weight_max": 40.27162170410156,
      "activations/layer22_attention_weight_min": -31.739362716674805,
      "activations/layer23_attention_weight_max": 33.42267608642578,
      "activations/layer23_attention_weight_min": -27.426258087158203,
      "activations/layer2_attention_weight_max": 30.455108642578125,
      "activations/layer2_attention_weight_min": -32.16005325317383,
      "activations/layer3_attention_weight_max": 88.56922149658203,
      "activations/layer3_attention_weight_min": -90.30819702148438,
      "activations/layer4_attention_weight_max": 98.8623275756836,
      "activations/layer4_attention_weight_min": -102.31919860839844,
      "activations/layer5_attention_weight_max": 48.75136947631836,
      "activations/layer5_attention_weight_min": -62.49543380737305,
      "activations/layer6_attention_weight_max": 39.5637321472168,
      "activations/layer6_attention_weight_min": -43.3997688293457,
      "activations/layer7_attention_weight_max": 92.97506713867188,
      "activations/layer7_attention_weight_min": -85.20657348632812,
      "activations/layer8_attention_weight_max": 39.80952835083008,
      "activations/layer8_attention_weight_min": -40.95660400390625,
      "activations/layer9_attention_weight_max": 30.372297286987305,
      "activations/layer9_attention_weight_min": -32.20980453491211,
      "epoch": 16.23,
      "learning_rate": 4.582424242424242e-05,
      "loss": 2.736,
      "step": 279250
    },
    {
      "activations/layer0_attention_weight_max": 16.995046615600586,
      "activations/layer0_attention_weight_min": -12.684412956237793,
      "activations/layer10_attention_weight_max": 30.036529541015625,
      "activations/layer10_attention_weight_min": -33.78374099731445,
      "activations/layer11_attention_weight_max": 30.176742553710938,
      "activations/layer11_attention_weight_min": -34.0311279296875,
      "activations/layer12_attention_weight_max": 24.759735107421875,
      "activations/layer12_attention_weight_min": -28.11332893371582,
      "activations/layer13_attention_weight_max": 38.980709075927734,
      "activations/layer13_attention_weight_min": -38.07225799560547,
      "activations/layer14_attention_weight_max": 41.98548126220703,
      "activations/layer14_attention_weight_min": -38.485206604003906,
      "activations/layer15_attention_weight_max": 37.15043258666992,
      "activations/layer15_attention_weight_min": -34.83433532714844,
      "activations/layer16_attention_weight_max": 28.52793312072754,
      "activations/layer16_attention_weight_min": -31.378442764282227,
      "activations/layer17_attention_weight_max": 31.14236831665039,
      "activations/layer17_attention_weight_min": -27.321958541870117,
      "activations/layer18_attention_weight_max": 33.06647872924805,
      "activations/layer18_attention_weight_min": -27.43842887878418,
      "activations/layer19_attention_weight_max": 33.53911590576172,
      "activations/layer19_attention_weight_min": -32.265926361083984,
      "activations/layer1_attention_weight_max": 15.523763656616211,
      "activations/layer1_attention_weight_min": -14.780314445495605,
      "activations/layer20_attention_weight_max": 29.911319732666016,
      "activations/layer20_attention_weight_min": -26.855632781982422,
      "activations/layer21_attention_weight_max": 29.80265235900879,
      "activations/layer21_attention_weight_min": -26.564346313476562,
      "activations/layer22_attention_weight_max": 43.50067138671875,
      "activations/layer22_attention_weight_min": -33.2204475402832,
      "activations/layer23_attention_weight_max": 33.54207229614258,
      "activations/layer23_attention_weight_min": -26.32204246520996,
      "activations/layer2_attention_weight_max": 30.887319564819336,
      "activations/layer2_attention_weight_min": -33.732120513916016,
      "activations/layer3_attention_weight_max": 90.1092529296875,
      "activations/layer3_attention_weight_min": -92.9603042602539,
      "activations/layer4_attention_weight_max": 100.36959838867188,
      "activations/layer4_attention_weight_min": -101.77901458740234,
      "activations/layer5_attention_weight_max": 52.576324462890625,
      "activations/layer5_attention_weight_min": -63.36066818237305,
      "activations/layer6_attention_weight_max": 40.95379638671875,
      "activations/layer6_attention_weight_min": -44.43349075317383,
      "activations/layer7_attention_weight_max": 89.03785705566406,
      "activations/layer7_attention_weight_min": -94.83924865722656,
      "activations/layer8_attention_weight_max": 39.784141540527344,
      "activations/layer8_attention_weight_min": -40.21227264404297,
      "activations/layer9_attention_weight_max": 28.675687789916992,
      "activations/layer9_attention_weight_min": -31.46555519104004,
      "epoch": 16.23,
      "learning_rate": 4.5805303030303025e-05,
      "loss": 2.7308,
      "step": 279300
    },
    {
      "activations/layer0_attention_weight_max": 15.802138328552246,
      "activations/layer0_attention_weight_min": -12.55711555480957,
      "activations/layer10_attention_weight_max": 32.89791488647461,
      "activations/layer10_attention_weight_min": -34.216888427734375,
      "activations/layer11_attention_weight_max": 32.570350646972656,
      "activations/layer11_attention_weight_min": -34.29560852050781,
      "activations/layer12_attention_weight_max": 26.55712127685547,
      "activations/layer12_attention_weight_min": -25.78346061706543,
      "activations/layer13_attention_weight_max": 40.490272521972656,
      "activations/layer13_attention_weight_min": -35.99991989135742,
      "activations/layer14_attention_weight_max": 45.03253173828125,
      "activations/layer14_attention_weight_min": -38.78569030761719,
      "activations/layer15_attention_weight_max": 37.25096893310547,
      "activations/layer15_attention_weight_min": -35.297279357910156,
      "activations/layer16_attention_weight_max": 32.35727310180664,
      "activations/layer16_attention_weight_min": -28.724958419799805,
      "activations/layer17_attention_weight_max": 30.79145050048828,
      "activations/layer17_attention_weight_min": -27.92879295349121,
      "activations/layer18_attention_weight_max": 31.475149154663086,
      "activations/layer18_attention_weight_min": -26.681264877319336,
      "activations/layer19_attention_weight_max": 36.1250114440918,
      "activations/layer19_attention_weight_min": -33.04273223876953,
      "activations/layer1_attention_weight_max": 16.034286499023438,
      "activations/layer1_attention_weight_min": -14.483736991882324,
      "activations/layer20_attention_weight_max": 29.626609802246094,
      "activations/layer20_attention_weight_min": -25.806852340698242,
      "activations/layer21_attention_weight_max": 30.005849838256836,
      "activations/layer21_attention_weight_min": -27.68431854248047,
      "activations/layer22_attention_weight_max": 44.56357955932617,
      "activations/layer22_attention_weight_min": -32.5689811706543,
      "activations/layer23_attention_weight_max": 34.14747619628906,
      "activations/layer23_attention_weight_min": -24.930633544921875,
      "activations/layer2_attention_weight_max": 30.212932586669922,
      "activations/layer2_attention_weight_min": -32.2967414855957,
      "activations/layer3_attention_weight_max": 91.48894500732422,
      "activations/layer3_attention_weight_min": -95.04890441894531,
      "activations/layer4_attention_weight_max": 102.52083587646484,
      "activations/layer4_attention_weight_min": -105.9157485961914,
      "activations/layer5_attention_weight_max": 51.26461410522461,
      "activations/layer5_attention_weight_min": -64.98026275634766,
      "activations/layer6_attention_weight_max": 44.62733459472656,
      "activations/layer6_attention_weight_min": -45.817161560058594,
      "activations/layer7_attention_weight_max": 87.74430847167969,
      "activations/layer7_attention_weight_min": -93.28681945800781,
      "activations/layer8_attention_weight_max": 41.8292121887207,
      "activations/layer8_attention_weight_min": -46.00482940673828,
      "activations/layer9_attention_weight_max": 32.325767517089844,
      "activations/layer9_attention_weight_min": -35.59425735473633,
      "epoch": 16.23,
      "learning_rate": 4.5786363636363634e-05,
      "loss": 2.73,
      "step": 279350
    },
    {
      "activations/layer0_attention_weight_max": 15.364470481872559,
      "activations/layer0_attention_weight_min": -12.16678237915039,
      "activations/layer10_attention_weight_max": 33.36758804321289,
      "activations/layer10_attention_weight_min": -32.34633255004883,
      "activations/layer11_attention_weight_max": 32.534278869628906,
      "activations/layer11_attention_weight_min": -31.301237106323242,
      "activations/layer12_attention_weight_max": 25.168848037719727,
      "activations/layer12_attention_weight_min": -26.181434631347656,
      "activations/layer13_attention_weight_max": 37.3127326965332,
      "activations/layer13_attention_weight_min": -36.074119567871094,
      "activations/layer14_attention_weight_max": 39.909217834472656,
      "activations/layer14_attention_weight_min": -40.09690475463867,
      "activations/layer15_attention_weight_max": 38.572654724121094,
      "activations/layer15_attention_weight_min": -37.69976043701172,
      "activations/layer16_attention_weight_max": 30.742280960083008,
      "activations/layer16_attention_weight_min": -28.660987854003906,
      "activations/layer17_attention_weight_max": 30.98135757446289,
      "activations/layer17_attention_weight_min": -31.88009262084961,
      "activations/layer18_attention_weight_max": 31.08783531188965,
      "activations/layer18_attention_weight_min": -29.359703063964844,
      "activations/layer19_attention_weight_max": 30.89581298828125,
      "activations/layer19_attention_weight_min": -32.94664001464844,
      "activations/layer1_attention_weight_max": 16.698732376098633,
      "activations/layer1_attention_weight_min": -14.587390899658203,
      "activations/layer20_attention_weight_max": 27.62297821044922,
      "activations/layer20_attention_weight_min": -26.728321075439453,
      "activations/layer21_attention_weight_max": 29.25150489807129,
      "activations/layer21_attention_weight_min": -27.252670288085938,
      "activations/layer22_attention_weight_max": 41.28934860229492,
      "activations/layer22_attention_weight_min": -38.188941955566406,
      "activations/layer23_attention_weight_max": 30.608572006225586,
      "activations/layer23_attention_weight_min": -27.814617156982422,
      "activations/layer2_attention_weight_max": 29.641849517822266,
      "activations/layer2_attention_weight_min": -30.02214813232422,
      "activations/layer3_attention_weight_max": 88.834228515625,
      "activations/layer3_attention_weight_min": -93.15460968017578,
      "activations/layer4_attention_weight_max": 97.20792388916016,
      "activations/layer4_attention_weight_min": -100.1180648803711,
      "activations/layer5_attention_weight_max": 48.423065185546875,
      "activations/layer5_attention_weight_min": -62.70042037963867,
      "activations/layer6_attention_weight_max": 42.29586410522461,
      "activations/layer6_attention_weight_min": -44.678104400634766,
      "activations/layer7_attention_weight_max": 90.28160858154297,
      "activations/layer7_attention_weight_min": -97.20423889160156,
      "activations/layer8_attention_weight_max": 43.357765197753906,
      "activations/layer8_attention_weight_min": -42.11723327636719,
      "activations/layer9_attention_weight_max": 32.29791259765625,
      "activations/layer9_attention_weight_min": -32.56081771850586,
      "epoch": 16.23,
      "learning_rate": 4.576742424242424e-05,
      "loss": 2.7417,
      "step": 279400
    },
    {
      "activations/layer0_attention_weight_max": 15.64834213256836,
      "activations/layer0_attention_weight_min": -13.910237312316895,
      "activations/layer10_attention_weight_max": 33.072113037109375,
      "activations/layer10_attention_weight_min": -33.73468017578125,
      "activations/layer11_attention_weight_max": 31.237560272216797,
      "activations/layer11_attention_weight_min": -32.80388641357422,
      "activations/layer12_attention_weight_max": 26.601987838745117,
      "activations/layer12_attention_weight_min": -27.245805740356445,
      "activations/layer13_attention_weight_max": 38.87417221069336,
      "activations/layer13_attention_weight_min": -35.58293151855469,
      "activations/layer14_attention_weight_max": 41.846649169921875,
      "activations/layer14_attention_weight_min": -37.843902587890625,
      "activations/layer15_attention_weight_max": 36.48431396484375,
      "activations/layer15_attention_weight_min": -35.195858001708984,
      "activations/layer16_attention_weight_max": 26.82198143005371,
      "activations/layer16_attention_weight_min": -28.459102630615234,
      "activations/layer17_attention_weight_max": 30.842960357666016,
      "activations/layer17_attention_weight_min": -26.69236183166504,
      "activations/layer18_attention_weight_max": 33.416603088378906,
      "activations/layer18_attention_weight_min": -26.77462387084961,
      "activations/layer19_attention_weight_max": 33.694175720214844,
      "activations/layer19_attention_weight_min": -32.504486083984375,
      "activations/layer1_attention_weight_max": 16.355804443359375,
      "activations/layer1_attention_weight_min": -13.915355682373047,
      "activations/layer20_attention_weight_max": 28.784942626953125,
      "activations/layer20_attention_weight_min": -24.962617874145508,
      "activations/layer21_attention_weight_max": 28.67604637145996,
      "activations/layer21_attention_weight_min": -24.246862411499023,
      "activations/layer22_attention_weight_max": 39.07221221923828,
      "activations/layer22_attention_weight_min": -34.00063705444336,
      "activations/layer23_attention_weight_max": 34.98023223876953,
      "activations/layer23_attention_weight_min": -28.025634765625,
      "activations/layer2_attention_weight_max": 29.459110260009766,
      "activations/layer2_attention_weight_min": -31.106016159057617,
      "activations/layer3_attention_weight_max": 92.5352554321289,
      "activations/layer3_attention_weight_min": -94.90684509277344,
      "activations/layer4_attention_weight_max": 100.50555419921875,
      "activations/layer4_attention_weight_min": -103.61241912841797,
      "activations/layer5_attention_weight_max": 52.60972595214844,
      "activations/layer5_attention_weight_min": -60.4163818359375,
      "activations/layer6_attention_weight_max": 42.33777618408203,
      "activations/layer6_attention_weight_min": -49.20998764038086,
      "activations/layer7_attention_weight_max": 93.38835906982422,
      "activations/layer7_attention_weight_min": -94.6390380859375,
      "activations/layer8_attention_weight_max": 40.63103103637695,
      "activations/layer8_attention_weight_min": -43.24665451049805,
      "activations/layer9_attention_weight_max": 30.322446823120117,
      "activations/layer9_attention_weight_min": -33.37763214111328,
      "epoch": 16.24,
      "learning_rate": 4.5748484848484843e-05,
      "loss": 2.743,
      "step": 279450
    },
    {
      "activations/layer0_attention_weight_max": 15.946157455444336,
      "activations/layer0_attention_weight_min": -13.377071380615234,
      "activations/layer10_attention_weight_max": 33.56510543823242,
      "activations/layer10_attention_weight_min": -34.13770294189453,
      "activations/layer11_attention_weight_max": 35.20559310913086,
      "activations/layer11_attention_weight_min": -33.159881591796875,
      "activations/layer12_attention_weight_max": 27.676511764526367,
      "activations/layer12_attention_weight_min": -26.605424880981445,
      "activations/layer13_attention_weight_max": 40.160499572753906,
      "activations/layer13_attention_weight_min": -36.55035400390625,
      "activations/layer14_attention_weight_max": 45.4161376953125,
      "activations/layer14_attention_weight_min": -40.715782165527344,
      "activations/layer15_attention_weight_max": 41.25030517578125,
      "activations/layer15_attention_weight_min": -37.38506317138672,
      "activations/layer16_attention_weight_max": 28.834644317626953,
      "activations/layer16_attention_weight_min": -28.872669219970703,
      "activations/layer17_attention_weight_max": 30.271474838256836,
      "activations/layer17_attention_weight_min": -25.874731063842773,
      "activations/layer18_attention_weight_max": 32.457366943359375,
      "activations/layer18_attention_weight_min": -26.63595199584961,
      "activations/layer19_attention_weight_max": 34.13252639770508,
      "activations/layer19_attention_weight_min": -33.92117691040039,
      "activations/layer1_attention_weight_max": 16.7979793548584,
      "activations/layer1_attention_weight_min": -14.779834747314453,
      "activations/layer20_attention_weight_max": 29.384857177734375,
      "activations/layer20_attention_weight_min": -25.369007110595703,
      "activations/layer21_attention_weight_max": 27.651811599731445,
      "activations/layer21_attention_weight_min": -24.978363037109375,
      "activations/layer22_attention_weight_max": 39.59712600708008,
      "activations/layer22_attention_weight_min": -31.71010398864746,
      "activations/layer23_attention_weight_max": 33.19636535644531,
      "activations/layer23_attention_weight_min": -26.672279357910156,
      "activations/layer2_attention_weight_max": 30.76358413696289,
      "activations/layer2_attention_weight_min": -31.038406372070312,
      "activations/layer3_attention_weight_max": 96.40333557128906,
      "activations/layer3_attention_weight_min": -93.46678161621094,
      "activations/layer4_attention_weight_max": 103.87589263916016,
      "activations/layer4_attention_weight_min": -101.19697570800781,
      "activations/layer5_attention_weight_max": 48.98838806152344,
      "activations/layer5_attention_weight_min": -64.22784423828125,
      "activations/layer6_attention_weight_max": 42.82007598876953,
      "activations/layer6_attention_weight_min": -46.573184967041016,
      "activations/layer7_attention_weight_max": 90.13323974609375,
      "activations/layer7_attention_weight_min": -93.63434600830078,
      "activations/layer8_attention_weight_max": 39.87561798095703,
      "activations/layer8_attention_weight_min": -40.898887634277344,
      "activations/layer9_attention_weight_max": 32.273006439208984,
      "activations/layer9_attention_weight_min": -33.49214172363281,
      "epoch": 16.24,
      "learning_rate": 4.5729545454545445e-05,
      "loss": 2.7356,
      "step": 279500
    },
    {
      "activations/layer0_attention_weight_max": 15.553093910217285,
      "activations/layer0_attention_weight_min": -11.876941680908203,
      "activations/layer10_attention_weight_max": 31.358531951904297,
      "activations/layer10_attention_weight_min": -35.490882873535156,
      "activations/layer11_attention_weight_max": 31.817279815673828,
      "activations/layer11_attention_weight_min": -32.8658561706543,
      "activations/layer12_attention_weight_max": 26.50416374206543,
      "activations/layer12_attention_weight_min": -26.19791030883789,
      "activations/layer13_attention_weight_max": 43.9744987487793,
      "activations/layer13_attention_weight_min": -37.37682342529297,
      "activations/layer14_attention_weight_max": 47.141761779785156,
      "activations/layer14_attention_weight_min": -39.140010833740234,
      "activations/layer15_attention_weight_max": 39.93028259277344,
      "activations/layer15_attention_weight_min": -38.198341369628906,
      "activations/layer16_attention_weight_max": 29.195253372192383,
      "activations/layer16_attention_weight_min": -27.694488525390625,
      "activations/layer17_attention_weight_max": 31.42512321472168,
      "activations/layer17_attention_weight_min": -26.36551284790039,
      "activations/layer18_attention_weight_max": 34.106231689453125,
      "activations/layer18_attention_weight_min": -25.41329574584961,
      "activations/layer19_attention_weight_max": 38.58088302612305,
      "activations/layer19_attention_weight_min": -30.754308700561523,
      "activations/layer1_attention_weight_max": 15.458040237426758,
      "activations/layer1_attention_weight_min": -14.782861709594727,
      "activations/layer20_attention_weight_max": 31.860881805419922,
      "activations/layer20_attention_weight_min": -23.994760513305664,
      "activations/layer21_attention_weight_max": 30.764019012451172,
      "activations/layer21_attention_weight_min": -24.38287925720215,
      "activations/layer22_attention_weight_max": 45.56140899658203,
      "activations/layer22_attention_weight_min": -32.8957633972168,
      "activations/layer23_attention_weight_max": 35.19948196411133,
      "activations/layer23_attention_weight_min": -26.756824493408203,
      "activations/layer2_attention_weight_max": 31.122596740722656,
      "activations/layer2_attention_weight_min": -31.596092224121094,
      "activations/layer3_attention_weight_max": 94.7096176147461,
      "activations/layer3_attention_weight_min": -94.65914154052734,
      "activations/layer4_attention_weight_max": 105.90616607666016,
      "activations/layer4_attention_weight_min": -106.7862777709961,
      "activations/layer5_attention_weight_max": 51.73167419433594,
      "activations/layer5_attention_weight_min": -62.31500244140625,
      "activations/layer6_attention_weight_max": 43.95204162597656,
      "activations/layer6_attention_weight_min": -50.94489669799805,
      "activations/layer7_attention_weight_max": 88.70049285888672,
      "activations/layer7_attention_weight_min": -95.77373504638672,
      "activations/layer8_attention_weight_max": 41.54035568237305,
      "activations/layer8_attention_weight_min": -45.96812057495117,
      "activations/layer9_attention_weight_max": 31.310216903686523,
      "activations/layer9_attention_weight_min": -35.10403060913086,
      "epoch": 16.24,
      "learning_rate": 4.571060606060606e-05,
      "loss": 2.736,
      "step": 279550
    },
    {
      "activations/layer0_attention_weight_max": 17.131441116333008,
      "activations/layer0_attention_weight_min": -12.475799560546875,
      "activations/layer10_attention_weight_max": 34.28829574584961,
      "activations/layer10_attention_weight_min": -35.43743896484375,
      "activations/layer11_attention_weight_max": 37.90863037109375,
      "activations/layer11_attention_weight_min": -36.475196838378906,
      "activations/layer12_attention_weight_max": 29.17519760131836,
      "activations/layer12_attention_weight_min": -28.27625846862793,
      "activations/layer13_attention_weight_max": 43.87363052368164,
      "activations/layer13_attention_weight_min": -37.9320182800293,
      "activations/layer14_attention_weight_max": 48.29696273803711,
      "activations/layer14_attention_weight_min": -42.0108528137207,
      "activations/layer15_attention_weight_max": 50.635562896728516,
      "activations/layer15_attention_weight_min": -37.65988540649414,
      "activations/layer16_attention_weight_max": 34.928741455078125,
      "activations/layer16_attention_weight_min": -30.664066314697266,
      "activations/layer17_attention_weight_max": 32.451454162597656,
      "activations/layer17_attention_weight_min": -27.5659122467041,
      "activations/layer18_attention_weight_max": 33.95368576049805,
      "activations/layer18_attention_weight_min": -25.69505500793457,
      "activations/layer19_attention_weight_max": 38.47700500488281,
      "activations/layer19_attention_weight_min": -33.27757263183594,
      "activations/layer1_attention_weight_max": 16.516450881958008,
      "activations/layer1_attention_weight_min": -14.683053970336914,
      "activations/layer20_attention_weight_max": 31.02397346496582,
      "activations/layer20_attention_weight_min": -25.581880569458008,
      "activations/layer21_attention_weight_max": 28.999576568603516,
      "activations/layer21_attention_weight_min": -26.816341400146484,
      "activations/layer22_attention_weight_max": 49.89019775390625,
      "activations/layer22_attention_weight_min": -31.94215965270996,
      "activations/layer23_attention_weight_max": 33.65406799316406,
      "activations/layer23_attention_weight_min": -25.122726440429688,
      "activations/layer2_attention_weight_max": 33.982017517089844,
      "activations/layer2_attention_weight_min": -32.75973892211914,
      "activations/layer3_attention_weight_max": 95.35128784179688,
      "activations/layer3_attention_weight_min": -98.35918426513672,
      "activations/layer4_attention_weight_max": 103.9476089477539,
      "activations/layer4_attention_weight_min": -102.6565933227539,
      "activations/layer5_attention_weight_max": 49.17076873779297,
      "activations/layer5_attention_weight_min": -63.4181022644043,
      "activations/layer6_attention_weight_max": 44.381717681884766,
      "activations/layer6_attention_weight_min": -45.70439910888672,
      "activations/layer7_attention_weight_max": 91.29046630859375,
      "activations/layer7_attention_weight_min": -91.74629211425781,
      "activations/layer8_attention_weight_max": 41.13712692260742,
      "activations/layer8_attention_weight_min": -41.33515930175781,
      "activations/layer9_attention_weight_max": 35.315338134765625,
      "activations/layer9_attention_weight_min": -35.57745361328125,
      "epoch": 16.25,
      "learning_rate": 4.569166666666666e-05,
      "loss": 2.7356,
      "step": 279600
    },
    {
      "activations/layer0_attention_weight_max": 14.89692497253418,
      "activations/layer0_attention_weight_min": -12.51788330078125,
      "activations/layer10_attention_weight_max": 36.962989807128906,
      "activations/layer10_attention_weight_min": -36.38507843017578,
      "activations/layer11_attention_weight_max": 36.34014129638672,
      "activations/layer11_attention_weight_min": -40.10508346557617,
      "activations/layer12_attention_weight_max": 28.654239654541016,
      "activations/layer12_attention_weight_min": -27.22710418701172,
      "activations/layer13_attention_weight_max": 46.076290130615234,
      "activations/layer13_attention_weight_min": -38.5941162109375,
      "activations/layer14_attention_weight_max": 49.7575798034668,
      "activations/layer14_attention_weight_min": -41.196563720703125,
      "activations/layer15_attention_weight_max": 46.644954681396484,
      "activations/layer15_attention_weight_min": -36.54660415649414,
      "activations/layer16_attention_weight_max": 34.41370391845703,
      "activations/layer16_attention_weight_min": -28.824411392211914,
      "activations/layer17_attention_weight_max": 33.094974517822266,
      "activations/layer17_attention_weight_min": -26.130149841308594,
      "activations/layer18_attention_weight_max": 36.28877639770508,
      "activations/layer18_attention_weight_min": -27.0873966217041,
      "activations/layer19_attention_weight_max": 42.43204879760742,
      "activations/layer19_attention_weight_min": -33.871665954589844,
      "activations/layer1_attention_weight_max": 16.860570907592773,
      "activations/layer1_attention_weight_min": -14.500812530517578,
      "activations/layer20_attention_weight_max": 36.94377136230469,
      "activations/layer20_attention_weight_min": -27.825044631958008,
      "activations/layer21_attention_weight_max": 33.0496711730957,
      "activations/layer21_attention_weight_min": -25.533451080322266,
      "activations/layer22_attention_weight_max": 49.77478790283203,
      "activations/layer22_attention_weight_min": -35.29526901245117,
      "activations/layer23_attention_weight_max": 42.208091735839844,
      "activations/layer23_attention_weight_min": -26.61532974243164,
      "activations/layer2_attention_weight_max": 31.875473022460938,
      "activations/layer2_attention_weight_min": -33.03668975830078,
      "activations/layer3_attention_weight_max": 96.08995819091797,
      "activations/layer3_attention_weight_min": -98.48255920410156,
      "activations/layer4_attention_weight_max": 105.0008773803711,
      "activations/layer4_attention_weight_min": -110.18914794921875,
      "activations/layer5_attention_weight_max": 52.10240173339844,
      "activations/layer5_attention_weight_min": -61.74067687988281,
      "activations/layer6_attention_weight_max": 42.856571197509766,
      "activations/layer6_attention_weight_min": -45.630123138427734,
      "activations/layer7_attention_weight_max": 95.41668701171875,
      "activations/layer7_attention_weight_min": -94.1252212524414,
      "activations/layer8_attention_weight_max": 41.31026077270508,
      "activations/layer8_attention_weight_min": -45.99187469482422,
      "activations/layer9_attention_weight_max": 32.417911529541016,
      "activations/layer9_attention_weight_min": -35.987003326416016,
      "epoch": 16.25,
      "learning_rate": 4.567272727272727e-05,
      "loss": 2.7502,
      "step": 279650
    },
    {
      "activations/layer0_attention_weight_max": 15.97423267364502,
      "activations/layer0_attention_weight_min": -13.250824928283691,
      "activations/layer10_attention_weight_max": 33.57178497314453,
      "activations/layer10_attention_weight_min": -32.737144470214844,
      "activations/layer11_attention_weight_max": 32.323631286621094,
      "activations/layer11_attention_weight_min": -34.2064208984375,
      "activations/layer12_attention_weight_max": 25.683473587036133,
      "activations/layer12_attention_weight_min": -25.73335075378418,
      "activations/layer13_attention_weight_max": 40.370460510253906,
      "activations/layer13_attention_weight_min": -34.66936492919922,
      "activations/layer14_attention_weight_max": 42.36338806152344,
      "activations/layer14_attention_weight_min": -36.69839096069336,
      "activations/layer15_attention_weight_max": 36.719966888427734,
      "activations/layer15_attention_weight_min": -34.17298126220703,
      "activations/layer16_attention_weight_max": 30.807796478271484,
      "activations/layer16_attention_weight_min": -28.73581314086914,
      "activations/layer17_attention_weight_max": 30.778079986572266,
      "activations/layer17_attention_weight_min": -27.0493106842041,
      "activations/layer18_attention_weight_max": 32.26437759399414,
      "activations/layer18_attention_weight_min": -26.602657318115234,
      "activations/layer19_attention_weight_max": 32.20089340209961,
      "activations/layer19_attention_weight_min": -31.786191940307617,
      "activations/layer1_attention_weight_max": 17.303464889526367,
      "activations/layer1_attention_weight_min": -16.360214233398438,
      "activations/layer20_attention_weight_max": 29.836584091186523,
      "activations/layer20_attention_weight_min": -26.839183807373047,
      "activations/layer21_attention_weight_max": 28.63579559326172,
      "activations/layer21_attention_weight_min": -27.341861724853516,
      "activations/layer22_attention_weight_max": 41.175209045410156,
      "activations/layer22_attention_weight_min": -31.274282455444336,
      "activations/layer23_attention_weight_max": 33.60171127319336,
      "activations/layer23_attention_weight_min": -25.96817398071289,
      "activations/layer2_attention_weight_max": 33.49715805053711,
      "activations/layer2_attention_weight_min": -32.683040618896484,
      "activations/layer3_attention_weight_max": 96.5341567993164,
      "activations/layer3_attention_weight_min": -99.8646011352539,
      "activations/layer4_attention_weight_max": 109.73116302490234,
      "activations/layer4_attention_weight_min": -110.64598846435547,
      "activations/layer5_attention_weight_max": 52.461883544921875,
      "activations/layer5_attention_weight_min": -60.402008056640625,
      "activations/layer6_attention_weight_max": 44.50205993652344,
      "activations/layer6_attention_weight_min": -44.818824768066406,
      "activations/layer7_attention_weight_max": 88.75611114501953,
      "activations/layer7_attention_weight_min": -89.90306091308594,
      "activations/layer8_attention_weight_max": 40.088191986083984,
      "activations/layer8_attention_weight_min": -43.28636932373047,
      "activations/layer9_attention_weight_max": 32.00197219848633,
      "activations/layer9_attention_weight_min": -33.381168365478516,
      "epoch": 16.25,
      "learning_rate": 4.565378787878788e-05,
      "loss": 2.7452,
      "step": 279700
    },
    {
      "activations/layer0_attention_weight_max": 15.627514839172363,
      "activations/layer0_attention_weight_min": -13.234016418457031,
      "activations/layer10_attention_weight_max": 33.24550247192383,
      "activations/layer10_attention_weight_min": -32.8265380859375,
      "activations/layer11_attention_weight_max": 33.55502700805664,
      "activations/layer11_attention_weight_min": -33.247314453125,
      "activations/layer12_attention_weight_max": 24.506729125976562,
      "activations/layer12_attention_weight_min": -25.1463680267334,
      "activations/layer13_attention_weight_max": 37.12810516357422,
      "activations/layer13_attention_weight_min": -32.740386962890625,
      "activations/layer14_attention_weight_max": 39.95230484008789,
      "activations/layer14_attention_weight_min": -37.67435073852539,
      "activations/layer15_attention_weight_max": 34.63005065917969,
      "activations/layer15_attention_weight_min": -32.96491622924805,
      "activations/layer16_attention_weight_max": 31.079805374145508,
      "activations/layer16_attention_weight_min": -26.71633529663086,
      "activations/layer17_attention_weight_max": 26.121606826782227,
      "activations/layer17_attention_weight_min": -26.128620147705078,
      "activations/layer18_attention_weight_max": 30.886648178100586,
      "activations/layer18_attention_weight_min": -25.99610710144043,
      "activations/layer19_attention_weight_max": 33.174888610839844,
      "activations/layer19_attention_weight_min": -30.062503814697266,
      "activations/layer1_attention_weight_max": 17.020954132080078,
      "activations/layer1_attention_weight_min": -16.44594955444336,
      "activations/layer20_attention_weight_max": 27.359697341918945,
      "activations/layer20_attention_weight_min": -24.339126586914062,
      "activations/layer21_attention_weight_max": 27.944326400756836,
      "activations/layer21_attention_weight_min": -25.666641235351562,
      "activations/layer22_attention_weight_max": 38.2696418762207,
      "activations/layer22_attention_weight_min": -31.623640060424805,
      "activations/layer23_attention_weight_max": 34.4018440246582,
      "activations/layer23_attention_weight_min": -27.263900756835938,
      "activations/layer2_attention_weight_max": 33.05937957763672,
      "activations/layer2_attention_weight_min": -34.093021392822266,
      "activations/layer3_attention_weight_max": 96.32843780517578,
      "activations/layer3_attention_weight_min": -98.0995864868164,
      "activations/layer4_attention_weight_max": 104.69793701171875,
      "activations/layer4_attention_weight_min": -109.2389144897461,
      "activations/layer5_attention_weight_max": 48.666259765625,
      "activations/layer5_attention_weight_min": -61.53921127319336,
      "activations/layer6_attention_weight_max": 43.18349838256836,
      "activations/layer6_attention_weight_min": -48.27164077758789,
      "activations/layer7_attention_weight_max": 87.97144317626953,
      "activations/layer7_attention_weight_min": -92.78185272216797,
      "activations/layer8_attention_weight_max": 40.32687759399414,
      "activations/layer8_attention_weight_min": -42.14898681640625,
      "activations/layer9_attention_weight_max": 31.272937774658203,
      "activations/layer9_attention_weight_min": -31.983346939086914,
      "epoch": 16.26,
      "learning_rate": 4.563484848484848e-05,
      "loss": 2.7698,
      "step": 279750
    },
    {
      "activations/layer0_attention_weight_max": 15.830280303955078,
      "activations/layer0_attention_weight_min": -12.309674263000488,
      "activations/layer10_attention_weight_max": 40.368743896484375,
      "activations/layer10_attention_weight_min": -36.213253021240234,
      "activations/layer11_attention_weight_max": 41.005165100097656,
      "activations/layer11_attention_weight_min": -37.57305908203125,
      "activations/layer12_attention_weight_max": 30.299068450927734,
      "activations/layer12_attention_weight_min": -26.54351234436035,
      "activations/layer13_attention_weight_max": 43.636287689208984,
      "activations/layer13_attention_weight_min": -34.72964096069336,
      "activations/layer14_attention_weight_max": 56.465938568115234,
      "activations/layer14_attention_weight_min": -38.371829986572266,
      "activations/layer15_attention_weight_max": 50.639442443847656,
      "activations/layer15_attention_weight_min": -39.23530578613281,
      "activations/layer16_attention_weight_max": 40.11619186401367,
      "activations/layer16_attention_weight_min": -26.901243209838867,
      "activations/layer17_attention_weight_max": 48.081214904785156,
      "activations/layer17_attention_weight_min": -30.845001220703125,
      "activations/layer18_attention_weight_max": 39.80192565917969,
      "activations/layer18_attention_weight_min": -27.97457504272461,
      "activations/layer19_attention_weight_max": 40.40190505981445,
      "activations/layer19_attention_weight_min": -31.56220054626465,
      "activations/layer1_attention_weight_max": 16.202438354492188,
      "activations/layer1_attention_weight_min": -15.093188285827637,
      "activations/layer20_attention_weight_max": 32.97752380371094,
      "activations/layer20_attention_weight_min": -26.500314712524414,
      "activations/layer21_attention_weight_max": 34.225257873535156,
      "activations/layer21_attention_weight_min": -25.4144229888916,
      "activations/layer22_attention_weight_max": 50.66714859008789,
      "activations/layer22_attention_weight_min": -34.24778366088867,
      "activations/layer23_attention_weight_max": 38.26882553100586,
      "activations/layer23_attention_weight_min": -27.38010025024414,
      "activations/layer2_attention_weight_max": 33.06132507324219,
      "activations/layer2_attention_weight_min": -33.571922302246094,
      "activations/layer3_attention_weight_max": 96.81416320800781,
      "activations/layer3_attention_weight_min": -96.50798034667969,
      "activations/layer4_attention_weight_max": 108.32860565185547,
      "activations/layer4_attention_weight_min": -111.4352798461914,
      "activations/layer5_attention_weight_max": 52.314186096191406,
      "activations/layer5_attention_weight_min": -63.59027099609375,
      "activations/layer6_attention_weight_max": 45.13066482543945,
      "activations/layer6_attention_weight_min": -48.28251266479492,
      "activations/layer7_attention_weight_max": 104.08753204345703,
      "activations/layer7_attention_weight_min": -99.33642578125,
      "activations/layer8_attention_weight_max": 47.3721809387207,
      "activations/layer8_attention_weight_min": -46.721248626708984,
      "activations/layer9_attention_weight_max": 36.82688522338867,
      "activations/layer9_attention_weight_min": -37.75355529785156,
      "epoch": 16.26,
      "learning_rate": 4.561590909090909e-05,
      "loss": 2.739,
      "step": 279800
    },
    {
      "activations/layer0_attention_weight_max": 15.919615745544434,
      "activations/layer0_attention_weight_min": -12.794894218444824,
      "activations/layer10_attention_weight_max": 36.68201446533203,
      "activations/layer10_attention_weight_min": -36.56396484375,
      "activations/layer11_attention_weight_max": 36.67588806152344,
      "activations/layer11_attention_weight_min": -36.14662170410156,
      "activations/layer12_attention_weight_max": 27.226707458496094,
      "activations/layer12_attention_weight_min": -27.368425369262695,
      "activations/layer13_attention_weight_max": 41.76276779174805,
      "activations/layer13_attention_weight_min": -37.2278938293457,
      "activations/layer14_attention_weight_max": 47.16049575805664,
      "activations/layer14_attention_weight_min": -40.212249755859375,
      "activations/layer15_attention_weight_max": 42.884342193603516,
      "activations/layer15_attention_weight_min": -36.81196594238281,
      "activations/layer16_attention_weight_max": 34.10734939575195,
      "activations/layer16_attention_weight_min": -28.30821990966797,
      "activations/layer17_attention_weight_max": 31.683963775634766,
      "activations/layer17_attention_weight_min": -25.94947624206543,
      "activations/layer18_attention_weight_max": 33.646785736083984,
      "activations/layer18_attention_weight_min": -25.02334213256836,
      "activations/layer19_attention_weight_max": 39.92112350463867,
      "activations/layer19_attention_weight_min": -32.52761459350586,
      "activations/layer1_attention_weight_max": 18.076807022094727,
      "activations/layer1_attention_weight_min": -15.20319938659668,
      "activations/layer20_attention_weight_max": 32.93551254272461,
      "activations/layer20_attention_weight_min": -25.872228622436523,
      "activations/layer21_attention_weight_max": 31.12363624572754,
      "activations/layer21_attention_weight_min": -24.782730102539062,
      "activations/layer22_attention_weight_max": 56.53921890258789,
      "activations/layer22_attention_weight_min": -32.04458999633789,
      "activations/layer23_attention_weight_max": 37.292930603027344,
      "activations/layer23_attention_weight_min": -26.04218101501465,
      "activations/layer2_attention_weight_max": 35.67267990112305,
      "activations/layer2_attention_weight_min": -35.54502487182617,
      "activations/layer3_attention_weight_max": 102.49555206298828,
      "activations/layer3_attention_weight_min": -98.35308074951172,
      "activations/layer4_attention_weight_max": 113.17225646972656,
      "activations/layer4_attention_weight_min": -116.2758560180664,
      "activations/layer5_attention_weight_max": 51.430511474609375,
      "activations/layer5_attention_weight_min": -64.28775024414062,
      "activations/layer6_attention_weight_max": 48.78669357299805,
      "activations/layer6_attention_weight_min": -47.653526306152344,
      "activations/layer7_attention_weight_max": 101.95549011230469,
      "activations/layer7_attention_weight_min": -97.3015365600586,
      "activations/layer8_attention_weight_max": 45.16469192504883,
      "activations/layer8_attention_weight_min": -43.83023452758789,
      "activations/layer9_attention_weight_max": 35.13157272338867,
      "activations/layer9_attention_weight_min": -36.34773254394531,
      "epoch": 16.26,
      "learning_rate": 4.559696969696969e-05,
      "loss": 2.7487,
      "step": 279850
    },
    {
      "activations/layer0_attention_weight_max": 16.517704010009766,
      "activations/layer0_attention_weight_min": -14.179657936096191,
      "activations/layer10_attention_weight_max": 33.053863525390625,
      "activations/layer10_attention_weight_min": -34.78374099731445,
      "activations/layer11_attention_weight_max": 34.607357025146484,
      "activations/layer11_attention_weight_min": -33.32557678222656,
      "activations/layer12_attention_weight_max": 26.378145217895508,
      "activations/layer12_attention_weight_min": -25.856674194335938,
      "activations/layer13_attention_weight_max": 36.84806823730469,
      "activations/layer13_attention_weight_min": -35.87302780151367,
      "activations/layer14_attention_weight_max": 44.59733963012695,
      "activations/layer14_attention_weight_min": -36.21675491333008,
      "activations/layer15_attention_weight_max": 35.84116744995117,
      "activations/layer15_attention_weight_min": -36.50495529174805,
      "activations/layer16_attention_weight_max": 29.43570899963379,
      "activations/layer16_attention_weight_min": -28.733610153198242,
      "activations/layer17_attention_weight_max": 30.56160545349121,
      "activations/layer17_attention_weight_min": -26.44310188293457,
      "activations/layer18_attention_weight_max": 32.0151252746582,
      "activations/layer18_attention_weight_min": -26.587581634521484,
      "activations/layer19_attention_weight_max": 33.300140380859375,
      "activations/layer19_attention_weight_min": -34.18636703491211,
      "activations/layer1_attention_weight_max": 16.064407348632812,
      "activations/layer1_attention_weight_min": -14.631390571594238,
      "activations/layer20_attention_weight_max": 27.64463996887207,
      "activations/layer20_attention_weight_min": -26.367013931274414,
      "activations/layer21_attention_weight_max": 25.891441345214844,
      "activations/layer21_attention_weight_min": -24.54814338684082,
      "activations/layer22_attention_weight_max": 43.6512565612793,
      "activations/layer22_attention_weight_min": -30.857709884643555,
      "activations/layer23_attention_weight_max": 27.624380111694336,
      "activations/layer23_attention_weight_min": -26.12311363220215,
      "activations/layer2_attention_weight_max": 31.13536834716797,
      "activations/layer2_attention_weight_min": -31.21234703063965,
      "activations/layer3_attention_weight_max": 94.69357299804688,
      "activations/layer3_attention_weight_min": -99.57122039794922,
      "activations/layer4_attention_weight_max": 103.62166595458984,
      "activations/layer4_attention_weight_min": -110.6711196899414,
      "activations/layer5_attention_weight_max": 48.90496063232422,
      "activations/layer5_attention_weight_min": -62.89556121826172,
      "activations/layer6_attention_weight_max": 42.014915466308594,
      "activations/layer6_attention_weight_min": -45.378414154052734,
      "activations/layer7_attention_weight_max": 89.7812728881836,
      "activations/layer7_attention_weight_min": -91.1467514038086,
      "activations/layer8_attention_weight_max": 40.23371124267578,
      "activations/layer8_attention_weight_min": -41.73408126831055,
      "activations/layer9_attention_weight_max": 31.661001205444336,
      "activations/layer9_attention_weight_min": -34.82713317871094,
      "epoch": 16.26,
      "learning_rate": 4.5578030303030304e-05,
      "loss": 2.741,
      "step": 279900
    },
    {
      "activations/layer0_attention_weight_max": 16.45182991027832,
      "activations/layer0_attention_weight_min": -12.167780876159668,
      "activations/layer10_attention_weight_max": 41.17722702026367,
      "activations/layer10_attention_weight_min": -34.36673355102539,
      "activations/layer11_attention_weight_max": 41.78301239013672,
      "activations/layer11_attention_weight_min": -36.19926071166992,
      "activations/layer12_attention_weight_max": 26.606409072875977,
      "activations/layer12_attention_weight_min": -26.442651748657227,
      "activations/layer13_attention_weight_max": 35.85521697998047,
      "activations/layer13_attention_weight_min": -36.84345245361328,
      "activations/layer14_attention_weight_max": 48.80934524536133,
      "activations/layer14_attention_weight_min": -39.733890533447266,
      "activations/layer15_attention_weight_max": 38.766937255859375,
      "activations/layer15_attention_weight_min": -38.49371337890625,
      "activations/layer16_attention_weight_max": 28.804874420166016,
      "activations/layer16_attention_weight_min": -29.643360137939453,
      "activations/layer17_attention_weight_max": 27.87322425842285,
      "activations/layer17_attention_weight_min": -25.582368850708008,
      "activations/layer18_attention_weight_max": 31.1082763671875,
      "activations/layer18_attention_weight_min": -25.28997039794922,
      "activations/layer19_attention_weight_max": 37.90960693359375,
      "activations/layer19_attention_weight_min": -32.32466506958008,
      "activations/layer1_attention_weight_max": 16.298460006713867,
      "activations/layer1_attention_weight_min": -13.720442771911621,
      "activations/layer20_attention_weight_max": 26.18843650817871,
      "activations/layer20_attention_weight_min": -26.25945472717285,
      "activations/layer21_attention_weight_max": 27.849637985229492,
      "activations/layer21_attention_weight_min": -25.801340103149414,
      "activations/layer22_attention_weight_max": 42.14103698730469,
      "activations/layer22_attention_weight_min": -33.15491485595703,
      "activations/layer23_attention_weight_max": 33.39470291137695,
      "activations/layer23_attention_weight_min": -26.710851669311523,
      "activations/layer2_attention_weight_max": 31.765439987182617,
      "activations/layer2_attention_weight_min": -32.97297668457031,
      "activations/layer3_attention_weight_max": 96.7234115600586,
      "activations/layer3_attention_weight_min": -97.18914031982422,
      "activations/layer4_attention_weight_max": 108.64083099365234,
      "activations/layer4_attention_weight_min": -105.21468353271484,
      "activations/layer5_attention_weight_max": 52.86488342285156,
      "activations/layer5_attention_weight_min": -61.80818176269531,
      "activations/layer6_attention_weight_max": 44.615333557128906,
      "activations/layer6_attention_weight_min": -45.28915786743164,
      "activations/layer7_attention_weight_max": 102.41866302490234,
      "activations/layer7_attention_weight_min": -91.26521301269531,
      "activations/layer8_attention_weight_max": 48.90190505981445,
      "activations/layer8_attention_weight_min": -41.352760314941406,
      "activations/layer9_attention_weight_max": 34.18648147583008,
      "activations/layer9_attention_weight_min": -34.15270233154297,
      "epoch": 16.27,
      "learning_rate": 4.5559090909090906e-05,
      "loss": 2.7459,
      "step": 279950
    },
    {
      "activations/layer0_attention_weight_max": 16.48609161376953,
      "activations/layer0_attention_weight_min": -14.985791206359863,
      "activations/layer10_attention_weight_max": 33.679229736328125,
      "activations/layer10_attention_weight_min": -34.24505615234375,
      "activations/layer11_attention_weight_max": 33.75572204589844,
      "activations/layer11_attention_weight_min": -33.521385192871094,
      "activations/layer12_attention_weight_max": 27.572628021240234,
      "activations/layer12_attention_weight_min": -27.927915573120117,
      "activations/layer13_attention_weight_max": 40.2621955871582,
      "activations/layer13_attention_weight_min": -37.238136291503906,
      "activations/layer14_attention_weight_max": 41.51185607910156,
      "activations/layer14_attention_weight_min": -38.97770309448242,
      "activations/layer15_attention_weight_max": 38.847190856933594,
      "activations/layer15_attention_weight_min": -35.4559211730957,
      "activations/layer16_attention_weight_max": 29.883996963500977,
      "activations/layer16_attention_weight_min": -26.6680965423584,
      "activations/layer17_attention_weight_max": 28.816118240356445,
      "activations/layer17_attention_weight_min": -26.665529251098633,
      "activations/layer18_attention_weight_max": 33.763484954833984,
      "activations/layer18_attention_weight_min": -24.868837356567383,
      "activations/layer19_attention_weight_max": 34.09000015258789,
      "activations/layer19_attention_weight_min": -32.67717742919922,
      "activations/layer1_attention_weight_max": 16.234899520874023,
      "activations/layer1_attention_weight_min": -15.5824556350708,
      "activations/layer20_attention_weight_max": 29.250341415405273,
      "activations/layer20_attention_weight_min": -25.266149520874023,
      "activations/layer21_attention_weight_max": 29.184255599975586,
      "activations/layer21_attention_weight_min": -24.844091415405273,
      "activations/layer22_attention_weight_max": 45.46306610107422,
      "activations/layer22_attention_weight_min": -30.67548179626465,
      "activations/layer23_attention_weight_max": 33.137229919433594,
      "activations/layer23_attention_weight_min": -25.517759323120117,
      "activations/layer2_attention_weight_max": 30.891132354736328,
      "activations/layer2_attention_weight_min": -31.04871368408203,
      "activations/layer3_attention_weight_max": 95.2468490600586,
      "activations/layer3_attention_weight_min": -95.27898406982422,
      "activations/layer4_attention_weight_max": 106.1716537475586,
      "activations/layer4_attention_weight_min": -109.32398986816406,
      "activations/layer5_attention_weight_max": 53.3353271484375,
      "activations/layer5_attention_weight_min": -63.13432312011719,
      "activations/layer6_attention_weight_max": 44.8758659362793,
      "activations/layer6_attention_weight_min": -45.86998748779297,
      "activations/layer7_attention_weight_max": 94.67326354980469,
      "activations/layer7_attention_weight_min": -87.52153778076172,
      "activations/layer8_attention_weight_max": 41.89447784423828,
      "activations/layer8_attention_weight_min": -44.358612060546875,
      "activations/layer9_attention_weight_max": 32.669734954833984,
      "activations/layer9_attention_weight_min": -35.2033576965332,
      "epoch": 16.27,
      "learning_rate": 4.554015151515151e-05,
      "loss": 2.7335,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4939,
      "eval_samples_per_second": 505.54,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4939,
      "eval_openwebtext_samples_per_second": 505.54,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9815,
      "eval_wikitext_samples_per_second": 230.127,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_lambada_loss": 2.5078125,
      "eval_lambada_ppl": 12.278042445054236,
      "eval_lambada_runtime": 9.5426,
      "eval_lambada_samples_per_second": 510.237,
      "step": 280000
    },
    {
      "activations/layer0_attention_weight_max": 16.796537399291992,
      "activations/layer0_attention_weight_min": -11.220531463623047,
      "activations/layer10_attention_weight_max": 32.21826171875,
      "activations/layer10_attention_weight_min": -32.67139434814453,
      "activations/layer11_attention_weight_max": 32.57970428466797,
      "activations/layer11_attention_weight_min": -31.276443481445312,
      "activations/layer12_attention_weight_max": 27.869083404541016,
      "activations/layer12_attention_weight_min": -30.374897003173828,
      "activations/layer13_attention_weight_max": 40.90974807739258,
      "activations/layer13_attention_weight_min": -34.585304260253906,
      "activations/layer14_attention_weight_max": 43.76014709472656,
      "activations/layer14_attention_weight_min": -37.45429229736328,
      "activations/layer15_attention_weight_max": 40.01963806152344,
      "activations/layer15_attention_weight_min": -38.041236877441406,
      "activations/layer16_attention_weight_max": 29.793352127075195,
      "activations/layer16_attention_weight_min": -25.474624633789062,
      "activations/layer17_attention_weight_max": 30.542043685913086,
      "activations/layer17_attention_weight_min": -26.702960968017578,
      "activations/layer18_attention_weight_max": 28.83015251159668,
      "activations/layer18_attention_weight_min": -24.89136505126953,
      "activations/layer19_attention_weight_max": 34.3953971862793,
      "activations/layer19_attention_weight_min": -31.00044059753418,
      "activations/layer1_attention_weight_max": 16.36959457397461,
      "activations/layer1_attention_weight_min": -13.092497825622559,
      "activations/layer20_attention_weight_max": 27.99334144592285,
      "activations/layer20_attention_weight_min": -24.97393226623535,
      "activations/layer21_attention_weight_max": 26.337100982666016,
      "activations/layer21_attention_weight_min": -25.09317398071289,
      "activations/layer22_attention_weight_max": 40.28092956542969,
      "activations/layer22_attention_weight_min": -33.244293212890625,
      "activations/layer23_attention_weight_max": 31.954757690429688,
      "activations/layer23_attention_weight_min": -26.793315887451172,
      "activations/layer2_attention_weight_max": 34.1578254699707,
      "activations/layer2_attention_weight_min": -33.306861877441406,
      "activations/layer3_attention_weight_max": 100.71366119384766,
      "activations/layer3_attention_weight_min": -106.83173370361328,
      "activations/layer4_attention_weight_max": 109.05216217041016,
      "activations/layer4_attention_weight_min": -109.4483871459961,
      "activations/layer5_attention_weight_max": 50.91602325439453,
      "activations/layer5_attention_weight_min": -58.9277458190918,
      "activations/layer6_attention_weight_max": 46.46135330200195,
      "activations/layer6_attention_weight_min": -45.44422912597656,
      "activations/layer7_attention_weight_max": 93.66336822509766,
      "activations/layer7_attention_weight_min": -91.10195922851562,
      "activations/layer8_attention_weight_max": 39.91297912597656,
      "activations/layer8_attention_weight_min": -41.765071868896484,
      "activations/layer9_attention_weight_max": 31.75876235961914,
      "activations/layer9_attention_weight_min": -34.4166145324707,
      "epoch": 16.27,
      "learning_rate": 4.5521212121212116e-05,
      "loss": 2.7492,
      "step": 280050
    },
    {
      "activations/layer0_attention_weight_max": 15.756414413452148,
      "activations/layer0_attention_weight_min": -13.255620956420898,
      "activations/layer10_attention_weight_max": 32.65590286254883,
      "activations/layer10_attention_weight_min": -32.27912139892578,
      "activations/layer11_attention_weight_max": 33.34128189086914,
      "activations/layer11_attention_weight_min": -32.05964279174805,
      "activations/layer12_attention_weight_max": 26.341358184814453,
      "activations/layer12_attention_weight_min": -26.710655212402344,
      "activations/layer13_attention_weight_max": 40.72738265991211,
      "activations/layer13_attention_weight_min": -34.722511291503906,
      "activations/layer14_attention_weight_max": 44.338539123535156,
      "activations/layer14_attention_weight_min": -34.260162353515625,
      "activations/layer15_attention_weight_max": 38.0858268737793,
      "activations/layer15_attention_weight_min": -33.37601852416992,
      "activations/layer16_attention_weight_max": 28.133808135986328,
      "activations/layer16_attention_weight_min": -26.864269256591797,
      "activations/layer17_attention_weight_max": 29.52358627319336,
      "activations/layer17_attention_weight_min": -23.87320899963379,
      "activations/layer18_attention_weight_max": 30.55662727355957,
      "activations/layer18_attention_weight_min": -26.7840633392334,
      "activations/layer19_attention_weight_max": 32.46157455444336,
      "activations/layer19_attention_weight_min": -29.631229400634766,
      "activations/layer1_attention_weight_max": 16.00206756591797,
      "activations/layer1_attention_weight_min": -15.520895957946777,
      "activations/layer20_attention_weight_max": 26.14482879638672,
      "activations/layer20_attention_weight_min": -23.88850975036621,
      "activations/layer21_attention_weight_max": 26.066808700561523,
      "activations/layer21_attention_weight_min": -24.634897232055664,
      "activations/layer22_attention_weight_max": 43.38066101074219,
      "activations/layer22_attention_weight_min": -29.240434646606445,
      "activations/layer23_attention_weight_max": 32.466461181640625,
      "activations/layer23_attention_weight_min": -24.207969665527344,
      "activations/layer2_attention_weight_max": 34.13307189941406,
      "activations/layer2_attention_weight_min": -32.578773498535156,
      "activations/layer3_attention_weight_max": 94.17700958251953,
      "activations/layer3_attention_weight_min": -99.89390563964844,
      "activations/layer4_attention_weight_max": 105.9409408569336,
      "activations/layer4_attention_weight_min": -103.95299530029297,
      "activations/layer5_attention_weight_max": 49.62025451660156,
      "activations/layer5_attention_weight_min": -63.308349609375,
      "activations/layer6_attention_weight_max": 42.06135940551758,
      "activations/layer6_attention_weight_min": -47.825687408447266,
      "activations/layer7_attention_weight_max": 88.3840103149414,
      "activations/layer7_attention_weight_min": -94.38882446289062,
      "activations/layer8_attention_weight_max": 40.02216339111328,
      "activations/layer8_attention_weight_min": -41.298282623291016,
      "activations/layer9_attention_weight_max": 33.17250061035156,
      "activations/layer9_attention_weight_min": -35.444210052490234,
      "epoch": 16.28,
      "learning_rate": 4.5502272727272724e-05,
      "loss": 2.7365,
      "step": 280100
    },
    {
      "activations/layer0_attention_weight_max": 16.295806884765625,
      "activations/layer0_attention_weight_min": -12.912981033325195,
      "activations/layer10_attention_weight_max": 30.821287155151367,
      "activations/layer10_attention_weight_min": -30.998580932617188,
      "activations/layer11_attention_weight_max": 30.716732025146484,
      "activations/layer11_attention_weight_min": -30.645986557006836,
      "activations/layer12_attention_weight_max": 26.007652282714844,
      "activations/layer12_attention_weight_min": -24.710960388183594,
      "activations/layer13_attention_weight_max": 37.852325439453125,
      "activations/layer13_attention_weight_min": -33.71044921875,
      "activations/layer14_attention_weight_max": 39.813514709472656,
      "activations/layer14_attention_weight_min": -34.86186981201172,
      "activations/layer15_attention_weight_max": 35.914852142333984,
      "activations/layer15_attention_weight_min": -33.37174606323242,
      "activations/layer16_attention_weight_max": 26.549545288085938,
      "activations/layer16_attention_weight_min": -26.4185791015625,
      "activations/layer17_attention_weight_max": 27.90071678161621,
      "activations/layer17_attention_weight_min": -24.244218826293945,
      "activations/layer18_attention_weight_max": 31.198339462280273,
      "activations/layer18_attention_weight_min": -23.976945877075195,
      "activations/layer19_attention_weight_max": 31.741910934448242,
      "activations/layer19_attention_weight_min": -29.25876808166504,
      "activations/layer1_attention_weight_max": 15.513535499572754,
      "activations/layer1_attention_weight_min": -16.2126407623291,
      "activations/layer20_attention_weight_max": 28.269014358520508,
      "activations/layer20_attention_weight_min": -25.485538482666016,
      "activations/layer21_attention_weight_max": 28.030895233154297,
      "activations/layer21_attention_weight_min": -23.468170166015625,
      "activations/layer22_attention_weight_max": 38.185203552246094,
      "activations/layer22_attention_weight_min": -28.94287872314453,
      "activations/layer23_attention_weight_max": 30.546995162963867,
      "activations/layer23_attention_weight_min": -26.561323165893555,
      "activations/layer2_attention_weight_max": 32.722633361816406,
      "activations/layer2_attention_weight_min": -33.2989387512207,
      "activations/layer3_attention_weight_max": 94.80181884765625,
      "activations/layer3_attention_weight_min": -95.86949157714844,
      "activations/layer4_attention_weight_max": 102.1144027709961,
      "activations/layer4_attention_weight_min": -110.67733001708984,
      "activations/layer5_attention_weight_max": 49.789703369140625,
      "activations/layer5_attention_weight_min": -61.39546203613281,
      "activations/layer6_attention_weight_max": 40.9913330078125,
      "activations/layer6_attention_weight_min": -46.943092346191406,
      "activations/layer7_attention_weight_max": 90.63224792480469,
      "activations/layer7_attention_weight_min": -88.22163391113281,
      "activations/layer8_attention_weight_max": 37.8146858215332,
      "activations/layer8_attention_weight_min": -40.548805236816406,
      "activations/layer9_attention_weight_max": 30.08194923400879,
      "activations/layer9_attention_weight_min": -32.78175354003906,
      "epoch": 16.28,
      "learning_rate": 4.548333333333333e-05,
      "loss": 2.7483,
      "step": 280150
    },
    {
      "activations/layer0_attention_weight_max": 16.395614624023438,
      "activations/layer0_attention_weight_min": -12.58281135559082,
      "activations/layer10_attention_weight_max": 32.56897735595703,
      "activations/layer10_attention_weight_min": -34.9680290222168,
      "activations/layer11_attention_weight_max": 32.56889724731445,
      "activations/layer11_attention_weight_min": -33.726776123046875,
      "activations/layer12_attention_weight_max": 24.3992919921875,
      "activations/layer12_attention_weight_min": -25.83427619934082,
      "activations/layer13_attention_weight_max": 37.611572265625,
      "activations/layer13_attention_weight_min": -36.25221633911133,
      "activations/layer14_attention_weight_max": 42.05719757080078,
      "activations/layer14_attention_weight_min": -35.32072830200195,
      "activations/layer15_attention_weight_max": 34.90810775756836,
      "activations/layer15_attention_weight_min": -33.76933288574219,
      "activations/layer16_attention_weight_max": 27.664844512939453,
      "activations/layer16_attention_weight_min": -30.996305465698242,
      "activations/layer17_attention_weight_max": 29.467262268066406,
      "activations/layer17_attention_weight_min": -27.33393096923828,
      "activations/layer18_attention_weight_max": 32.34233093261719,
      "activations/layer18_attention_weight_min": -25.804195404052734,
      "activations/layer19_attention_weight_max": 31.620153427124023,
      "activations/layer19_attention_weight_min": -35.05940246582031,
      "activations/layer1_attention_weight_max": 15.766743659973145,
      "activations/layer1_attention_weight_min": -15.208696365356445,
      "activations/layer20_attention_weight_max": 28.407934188842773,
      "activations/layer20_attention_weight_min": -27.007667541503906,
      "activations/layer21_attention_weight_max": 26.588151931762695,
      "activations/layer21_attention_weight_min": -25.941665649414062,
      "activations/layer22_attention_weight_max": 44.51984786987305,
      "activations/layer22_attention_weight_min": -31.451032638549805,
      "activations/layer23_attention_weight_max": 33.277198791503906,
      "activations/layer23_attention_weight_min": -24.71931266784668,
      "activations/layer2_attention_weight_max": 30.803268432617188,
      "activations/layer2_attention_weight_min": -32.988182067871094,
      "activations/layer3_attention_weight_max": 91.38928985595703,
      "activations/layer3_attention_weight_min": -95.36272430419922,
      "activations/layer4_attention_weight_max": 99.95037078857422,
      "activations/layer4_attention_weight_min": -108.5705795288086,
      "activations/layer5_attention_weight_max": 50.96941375732422,
      "activations/layer5_attention_weight_min": -62.44026184082031,
      "activations/layer6_attention_weight_max": 43.32741165161133,
      "activations/layer6_attention_weight_min": -47.056739807128906,
      "activations/layer7_attention_weight_max": 88.86767578125,
      "activations/layer7_attention_weight_min": -93.69236755371094,
      "activations/layer8_attention_weight_max": 40.5963020324707,
      "activations/layer8_attention_weight_min": -45.94563293457031,
      "activations/layer9_attention_weight_max": 31.029510498046875,
      "activations/layer9_attention_weight_min": -35.839317321777344,
      "epoch": 16.28,
      "learning_rate": 4.5464393939393934e-05,
      "loss": 2.7346,
      "step": 280200
    },
    {
      "activations/layer0_attention_weight_max": 14.990774154663086,
      "activations/layer0_attention_weight_min": -12.740875244140625,
      "activations/layer10_attention_weight_max": 37.98728942871094,
      "activations/layer10_attention_weight_min": -36.245548248291016,
      "activations/layer11_attention_weight_max": 38.17689514160156,
      "activations/layer11_attention_weight_min": -38.77289581298828,
      "activations/layer12_attention_weight_max": 34.01109313964844,
      "activations/layer12_attention_weight_min": -29.859729766845703,
      "activations/layer13_attention_weight_max": 50.03757095336914,
      "activations/layer13_attention_weight_min": -40.62847137451172,
      "activations/layer14_attention_weight_max": 49.33988952636719,
      "activations/layer14_attention_weight_min": -43.685245513916016,
      "activations/layer15_attention_weight_max": 49.428558349609375,
      "activations/layer15_attention_weight_min": -41.76513671875,
      "activations/layer16_attention_weight_max": 35.800472259521484,
      "activations/layer16_attention_weight_min": -32.33208084106445,
      "activations/layer17_attention_weight_max": 35.09105682373047,
      "activations/layer17_attention_weight_min": -31.04781150817871,
      "activations/layer18_attention_weight_max": 35.147735595703125,
      "activations/layer18_attention_weight_min": -31.97791862487793,
      "activations/layer19_attention_weight_max": 43.07090377807617,
      "activations/layer19_attention_weight_min": -35.21464157104492,
      "activations/layer1_attention_weight_max": 16.34400749206543,
      "activations/layer1_attention_weight_min": -15.54033088684082,
      "activations/layer20_attention_weight_max": 35.464664459228516,
      "activations/layer20_attention_weight_min": -28.354097366333008,
      "activations/layer21_attention_weight_max": 34.945106506347656,
      "activations/layer21_attention_weight_min": -28.9372615814209,
      "activations/layer22_attention_weight_max": 55.60208511352539,
      "activations/layer22_attention_weight_min": -39.68398666381836,
      "activations/layer23_attention_weight_max": 37.91814041137695,
      "activations/layer23_attention_weight_min": -29.185970306396484,
      "activations/layer2_attention_weight_max": 32.48046112060547,
      "activations/layer2_attention_weight_min": -32.480613708496094,
      "activations/layer3_attention_weight_max": 100.07291412353516,
      "activations/layer3_attention_weight_min": -97.6790542602539,
      "activations/layer4_attention_weight_max": 107.17921447753906,
      "activations/layer4_attention_weight_min": -105.54741668701172,
      "activations/layer5_attention_weight_max": 50.279815673828125,
      "activations/layer5_attention_weight_min": -59.98223876953125,
      "activations/layer6_attention_weight_max": 44.60945510864258,
      "activations/layer6_attention_weight_min": -46.37285614013672,
      "activations/layer7_attention_weight_max": 97.87577819824219,
      "activations/layer7_attention_weight_min": -94.98336029052734,
      "activations/layer8_attention_weight_max": 46.63906478881836,
      "activations/layer8_attention_weight_min": -43.225215911865234,
      "activations/layer9_attention_weight_max": 34.884525299072266,
      "activations/layer9_attention_weight_min": -35.42606735229492,
      "epoch": 16.28,
      "learning_rate": 4.5445454545454535e-05,
      "loss": 2.7465,
      "step": 280250
    },
    {
      "activations/layer0_attention_weight_max": 16.263790130615234,
      "activations/layer0_attention_weight_min": -12.683494567871094,
      "activations/layer10_attention_weight_max": 35.96382522583008,
      "activations/layer10_attention_weight_min": -36.816566467285156,
      "activations/layer11_attention_weight_max": 33.303802490234375,
      "activations/layer11_attention_weight_min": -34.878570556640625,
      "activations/layer12_attention_weight_max": 28.304737091064453,
      "activations/layer12_attention_weight_min": -26.168729782104492,
      "activations/layer13_attention_weight_max": 42.733177185058594,
      "activations/layer13_attention_weight_min": -38.87403869628906,
      "activations/layer14_attention_weight_max": 48.566246032714844,
      "activations/layer14_attention_weight_min": -40.645198822021484,
      "activations/layer15_attention_weight_max": 42.09037399291992,
      "activations/layer15_attention_weight_min": -36.65178680419922,
      "activations/layer16_attention_weight_max": 33.07239532470703,
      "activations/layer16_attention_weight_min": -29.69432830810547,
      "activations/layer17_attention_weight_max": 32.22517395019531,
      "activations/layer17_attention_weight_min": -27.0367374420166,
      "activations/layer18_attention_weight_max": 33.791168212890625,
      "activations/layer18_attention_weight_min": -26.05992317199707,
      "activations/layer19_attention_weight_max": 40.79176712036133,
      "activations/layer19_attention_weight_min": -31.342844009399414,
      "activations/layer1_attention_weight_max": 15.897272109985352,
      "activations/layer1_attention_weight_min": -14.61844539642334,
      "activations/layer20_attention_weight_max": 34.00467300415039,
      "activations/layer20_attention_weight_min": -26.83711814880371,
      "activations/layer21_attention_weight_max": 32.72233200073242,
      "activations/layer21_attention_weight_min": -24.610679626464844,
      "activations/layer22_attention_weight_max": 57.610496520996094,
      "activations/layer22_attention_weight_min": -30.73999786376953,
      "activations/layer23_attention_weight_max": 38.19845962524414,
      "activations/layer23_attention_weight_min": -29.80207061767578,
      "activations/layer2_attention_weight_max": 31.987228393554688,
      "activations/layer2_attention_weight_min": -32.03502655029297,
      "activations/layer3_attention_weight_max": 93.94441223144531,
      "activations/layer3_attention_weight_min": -95.22512817382812,
      "activations/layer4_attention_weight_max": 103.2263412475586,
      "activations/layer4_attention_weight_min": -112.77900695800781,
      "activations/layer5_attention_weight_max": 49.151580810546875,
      "activations/layer5_attention_weight_min": -60.89358139038086,
      "activations/layer6_attention_weight_max": 42.470638275146484,
      "activations/layer6_attention_weight_min": -46.77632141113281,
      "activations/layer7_attention_weight_max": 91.55866241455078,
      "activations/layer7_attention_weight_min": -95.48983764648438,
      "activations/layer8_attention_weight_max": 39.769264221191406,
      "activations/layer8_attention_weight_min": -45.378292083740234,
      "activations/layer9_attention_weight_max": 33.679874420166016,
      "activations/layer9_attention_weight_min": -34.79259490966797,
      "epoch": 16.29,
      "learning_rate": 4.542651515151515e-05,
      "loss": 2.7344,
      "step": 280300
    },
    {
      "activations/layer0_attention_weight_max": 16.725772857666016,
      "activations/layer0_attention_weight_min": -13.348409652709961,
      "activations/layer10_attention_weight_max": 33.77373504638672,
      "activations/layer10_attention_weight_min": -33.456512451171875,
      "activations/layer11_attention_weight_max": 33.31442642211914,
      "activations/layer11_attention_weight_min": -33.38880157470703,
      "activations/layer12_attention_weight_max": 25.84450912475586,
      "activations/layer12_attention_weight_min": -27.646604537963867,
      "activations/layer13_attention_weight_max": 39.745479583740234,
      "activations/layer13_attention_weight_min": -35.7906608581543,
      "activations/layer14_attention_weight_max": 42.53984069824219,
      "activations/layer14_attention_weight_min": -39.34962844848633,
      "activations/layer15_attention_weight_max": 38.408748626708984,
      "activations/layer15_attention_weight_min": -36.728271484375,
      "activations/layer16_attention_weight_max": 29.580949783325195,
      "activations/layer16_attention_weight_min": -28.623682022094727,
      "activations/layer17_attention_weight_max": 32.10240936279297,
      "activations/layer17_attention_weight_min": -26.0556640625,
      "activations/layer18_attention_weight_max": 34.79430389404297,
      "activations/layer18_attention_weight_min": -24.039823532104492,
      "activations/layer19_attention_weight_max": 35.181636810302734,
      "activations/layer19_attention_weight_min": -31.485305786132812,
      "activations/layer1_attention_weight_max": 16.741378784179688,
      "activations/layer1_attention_weight_min": -15.441933631896973,
      "activations/layer20_attention_weight_max": 29.398420333862305,
      "activations/layer20_attention_weight_min": -24.05646324157715,
      "activations/layer21_attention_weight_max": 29.675254821777344,
      "activations/layer21_attention_weight_min": -23.673858642578125,
      "activations/layer22_attention_weight_max": 43.234169006347656,
      "activations/layer22_attention_weight_min": -29.93723487854004,
      "activations/layer23_attention_weight_max": 32.44715881347656,
      "activations/layer23_attention_weight_min": -25.607656478881836,
      "activations/layer2_attention_weight_max": 32.34572982788086,
      "activations/layer2_attention_weight_min": -31.629552841186523,
      "activations/layer3_attention_weight_max": 96.21849060058594,
      "activations/layer3_attention_weight_min": -93.52761840820312,
      "activations/layer4_attention_weight_max": 102.64546966552734,
      "activations/layer4_attention_weight_min": -108.652587890625,
      "activations/layer5_attention_weight_max": 50.72864532470703,
      "activations/layer5_attention_weight_min": -63.926475524902344,
      "activations/layer6_attention_weight_max": 42.54708480834961,
      "activations/layer6_attention_weight_min": -45.20534133911133,
      "activations/layer7_attention_weight_max": 92.3468246459961,
      "activations/layer7_attention_weight_min": -91.74030303955078,
      "activations/layer8_attention_weight_max": 39.496036529541016,
      "activations/layer8_attention_weight_min": -42.30818176269531,
      "activations/layer9_attention_weight_max": 33.8896484375,
      "activations/layer9_attention_weight_min": -34.19866180419922,
      "epoch": 16.29,
      "learning_rate": 4.540757575757575e-05,
      "loss": 2.7563,
      "step": 280350
    },
    {
      "activations/layer0_attention_weight_max": 15.969719886779785,
      "activations/layer0_attention_weight_min": -13.541573524475098,
      "activations/layer10_attention_weight_max": 34.475982666015625,
      "activations/layer10_attention_weight_min": -35.28911209106445,
      "activations/layer11_attention_weight_max": 34.57290267944336,
      "activations/layer11_attention_weight_min": -34.10112762451172,
      "activations/layer12_attention_weight_max": 25.746355056762695,
      "activations/layer12_attention_weight_min": -29.59537124633789,
      "activations/layer13_attention_weight_max": 36.508140563964844,
      "activations/layer13_attention_weight_min": -35.01043701171875,
      "activations/layer14_attention_weight_max": 42.64094161987305,
      "activations/layer14_attention_weight_min": -37.15654754638672,
      "activations/layer15_attention_weight_max": 36.61604690551758,
      "activations/layer15_attention_weight_min": -34.87837219238281,
      "activations/layer16_attention_weight_max": 26.237077713012695,
      "activations/layer16_attention_weight_min": -28.28734016418457,
      "activations/layer17_attention_weight_max": 32.7657356262207,
      "activations/layer17_attention_weight_min": -26.057933807373047,
      "activations/layer18_attention_weight_max": 28.302846908569336,
      "activations/layer18_attention_weight_min": -25.149202346801758,
      "activations/layer19_attention_weight_max": 29.192615509033203,
      "activations/layer19_attention_weight_min": -31.991994857788086,
      "activations/layer1_attention_weight_max": 16.237478256225586,
      "activations/layer1_attention_weight_min": -16.955026626586914,
      "activations/layer20_attention_weight_max": 25.589387893676758,
      "activations/layer20_attention_weight_min": -24.23979377746582,
      "activations/layer21_attention_weight_max": 24.39198875427246,
      "activations/layer21_attention_weight_min": -24.44723892211914,
      "activations/layer22_attention_weight_max": 35.557491302490234,
      "activations/layer22_attention_weight_min": -29.519441604614258,
      "activations/layer23_attention_weight_max": 29.426910400390625,
      "activations/layer23_attention_weight_min": -25.353103637695312,
      "activations/layer2_attention_weight_max": 34.62236404418945,
      "activations/layer2_attention_weight_min": -33.375885009765625,
      "activations/layer3_attention_weight_max": 100.44670867919922,
      "activations/layer3_attention_weight_min": -98.1087875366211,
      "activations/layer4_attention_weight_max": 106.4226303100586,
      "activations/layer4_attention_weight_min": -108.6466064453125,
      "activations/layer5_attention_weight_max": 51.019248962402344,
      "activations/layer5_attention_weight_min": -63.0883903503418,
      "activations/layer6_attention_weight_max": 44.50771713256836,
      "activations/layer6_attention_weight_min": -46.05099105834961,
      "activations/layer7_attention_weight_max": 95.2282485961914,
      "activations/layer7_attention_weight_min": -91.85636901855469,
      "activations/layer8_attention_weight_max": 43.292274475097656,
      "activations/layer8_attention_weight_min": -42.368682861328125,
      "activations/layer9_attention_weight_max": 34.277183532714844,
      "activations/layer9_attention_weight_min": -35.368385314941406,
      "epoch": 16.29,
      "learning_rate": 4.538863636363636e-05,
      "loss": 2.7461,
      "step": 280400
    },
    {
      "activations/layer0_attention_weight_max": 16.13559913635254,
      "activations/layer0_attention_weight_min": -13.560823440551758,
      "activations/layer10_attention_weight_max": 33.20802688598633,
      "activations/layer10_attention_weight_min": -32.11065673828125,
      "activations/layer11_attention_weight_max": 34.53596115112305,
      "activations/layer11_attention_weight_min": -32.98453903198242,
      "activations/layer12_attention_weight_max": 26.440549850463867,
      "activations/layer12_attention_weight_min": -25.672727584838867,
      "activations/layer13_attention_weight_max": 36.422027587890625,
      "activations/layer13_attention_weight_min": -33.11328887939453,
      "activations/layer14_attention_weight_max": 43.22636413574219,
      "activations/layer14_attention_weight_min": -37.0806999206543,
      "activations/layer15_attention_weight_max": 35.77007293701172,
      "activations/layer15_attention_weight_min": -32.785667419433594,
      "activations/layer16_attention_weight_max": 28.166091918945312,
      "activations/layer16_attention_weight_min": -26.729549407958984,
      "activations/layer17_attention_weight_max": 32.004371643066406,
      "activations/layer17_attention_weight_min": -25.696372985839844,
      "activations/layer18_attention_weight_max": 28.630361557006836,
      "activations/layer18_attention_weight_min": -25.15622329711914,
      "activations/layer19_attention_weight_max": 31.420469284057617,
      "activations/layer19_attention_weight_min": -31.300888061523438,
      "activations/layer1_attention_weight_max": 16.818527221679688,
      "activations/layer1_attention_weight_min": -15.092860221862793,
      "activations/layer20_attention_weight_max": 26.802173614501953,
      "activations/layer20_attention_weight_min": -25.116186141967773,
      "activations/layer21_attention_weight_max": 28.01370620727539,
      "activations/layer21_attention_weight_min": -23.631942749023438,
      "activations/layer22_attention_weight_max": 39.02051544189453,
      "activations/layer22_attention_weight_min": -30.68832015991211,
      "activations/layer23_attention_weight_max": 36.073768615722656,
      "activations/layer23_attention_weight_min": -27.16396713256836,
      "activations/layer2_attention_weight_max": 29.81775665283203,
      "activations/layer2_attention_weight_min": -32.27717971801758,
      "activations/layer3_attention_weight_max": 93.36509704589844,
      "activations/layer3_attention_weight_min": -97.4307632446289,
      "activations/layer4_attention_weight_max": 107.36913299560547,
      "activations/layer4_attention_weight_min": -107.99176025390625,
      "activations/layer5_attention_weight_max": 50.498435974121094,
      "activations/layer5_attention_weight_min": -63.135223388671875,
      "activations/layer6_attention_weight_max": 40.85393142700195,
      "activations/layer6_attention_weight_min": -44.98893356323242,
      "activations/layer7_attention_weight_max": 90.54714965820312,
      "activations/layer7_attention_weight_min": -86.14955139160156,
      "activations/layer8_attention_weight_max": 38.15166091918945,
      "activations/layer8_attention_weight_min": -40.53312301635742,
      "activations/layer9_attention_weight_max": 28.703332901000977,
      "activations/layer9_attention_weight_min": -34.44520568847656,
      "epoch": 16.3,
      "learning_rate": 4.536969696969697e-05,
      "loss": 2.7373,
      "step": 280450
    },
    {
      "activations/layer0_attention_weight_max": 15.251785278320312,
      "activations/layer0_attention_weight_min": -12.017345428466797,
      "activations/layer10_attention_weight_max": 34.2197265625,
      "activations/layer10_attention_weight_min": -35.58050537109375,
      "activations/layer11_attention_weight_max": 35.18306350708008,
      "activations/layer11_attention_weight_min": -35.716590881347656,
      "activations/layer12_attention_weight_max": 25.303287506103516,
      "activations/layer12_attention_weight_min": -26.132287979125977,
      "activations/layer13_attention_weight_max": 41.75084686279297,
      "activations/layer13_attention_weight_min": -33.75066375732422,
      "activations/layer14_attention_weight_max": 40.7891731262207,
      "activations/layer14_attention_weight_min": -36.31023406982422,
      "activations/layer15_attention_weight_max": 38.08472442626953,
      "activations/layer15_attention_weight_min": -36.15053176879883,
      "activations/layer16_attention_weight_max": 28.167102813720703,
      "activations/layer16_attention_weight_min": -27.429174423217773,
      "activations/layer17_attention_weight_max": 30.469146728515625,
      "activations/layer17_attention_weight_min": -26.67354393005371,
      "activations/layer18_attention_weight_max": 31.411781311035156,
      "activations/layer18_attention_weight_min": -26.42885971069336,
      "activations/layer19_attention_weight_max": 32.81022262573242,
      "activations/layer19_attention_weight_min": -31.877273559570312,
      "activations/layer1_attention_weight_max": 15.714616775512695,
      "activations/layer1_attention_weight_min": -14.381872177124023,
      "activations/layer20_attention_weight_max": 29.630590438842773,
      "activations/layer20_attention_weight_min": -28.5601749420166,
      "activations/layer21_attention_weight_max": 28.50783348083496,
      "activations/layer21_attention_weight_min": -25.246353149414062,
      "activations/layer22_attention_weight_max": 39.27869415283203,
      "activations/layer22_attention_weight_min": -31.29960060119629,
      "activations/layer23_attention_weight_max": 30.736318588256836,
      "activations/layer23_attention_weight_min": -27.18140983581543,
      "activations/layer2_attention_weight_max": 32.01129150390625,
      "activations/layer2_attention_weight_min": -32.42466354370117,
      "activations/layer3_attention_weight_max": 92.11331176757812,
      "activations/layer3_attention_weight_min": -91.9469985961914,
      "activations/layer4_attention_weight_max": 102.20909881591797,
      "activations/layer4_attention_weight_min": -104.9430160522461,
      "activations/layer5_attention_weight_max": 48.25796127319336,
      "activations/layer5_attention_weight_min": -61.64768600463867,
      "activations/layer6_attention_weight_max": 41.678768157958984,
      "activations/layer6_attention_weight_min": -47.35704040527344,
      "activations/layer7_attention_weight_max": 93.72356414794922,
      "activations/layer7_attention_weight_min": -92.97273254394531,
      "activations/layer8_attention_weight_max": 43.20402908325195,
      "activations/layer8_attention_weight_min": -41.8563117980957,
      "activations/layer9_attention_weight_max": 32.09957504272461,
      "activations/layer9_attention_weight_min": -36.72443771362305,
      "epoch": 16.3,
      "learning_rate": 4.5350757575757576e-05,
      "loss": 2.7532,
      "step": 280500
    },
    {
      "activations/layer0_attention_weight_max": 15.622884750366211,
      "activations/layer0_attention_weight_min": -12.343124389648438,
      "activations/layer10_attention_weight_max": 34.53765106201172,
      "activations/layer10_attention_weight_min": -35.30667495727539,
      "activations/layer11_attention_weight_max": 34.01904296875,
      "activations/layer11_attention_weight_min": -35.70623779296875,
      "activations/layer12_attention_weight_max": 28.21041488647461,
      "activations/layer12_attention_weight_min": -27.959741592407227,
      "activations/layer13_attention_weight_max": 39.32164001464844,
      "activations/layer13_attention_weight_min": -36.62044143676758,
      "activations/layer14_attention_weight_max": 42.82740020751953,
      "activations/layer14_attention_weight_min": -37.28561019897461,
      "activations/layer15_attention_weight_max": 39.5048828125,
      "activations/layer15_attention_weight_min": -36.28860092163086,
      "activations/layer16_attention_weight_max": 28.016071319580078,
      "activations/layer16_attention_weight_min": -28.689430236816406,
      "activations/layer17_attention_weight_max": 28.36682891845703,
      "activations/layer17_attention_weight_min": -26.770967483520508,
      "activations/layer18_attention_weight_max": 31.625797271728516,
      "activations/layer18_attention_weight_min": -24.19099998474121,
      "activations/layer19_attention_weight_max": 37.80042266845703,
      "activations/layer19_attention_weight_min": -34.39924240112305,
      "activations/layer1_attention_weight_max": 16.415891647338867,
      "activations/layer1_attention_weight_min": -17.28145408630371,
      "activations/layer20_attention_weight_max": 28.335403442382812,
      "activations/layer20_attention_weight_min": -25.886035919189453,
      "activations/layer21_attention_weight_max": 27.0921630859375,
      "activations/layer21_attention_weight_min": -24.90233039855957,
      "activations/layer22_attention_weight_max": 43.20505905151367,
      "activations/layer22_attention_weight_min": -31.769041061401367,
      "activations/layer23_attention_weight_max": 37.581787109375,
      "activations/layer23_attention_weight_min": -26.68649673461914,
      "activations/layer2_attention_weight_max": 30.88962173461914,
      "activations/layer2_attention_weight_min": -30.07575225830078,
      "activations/layer3_attention_weight_max": 91.45549011230469,
      "activations/layer3_attention_weight_min": -89.24563598632812,
      "activations/layer4_attention_weight_max": 101.06584930419922,
      "activations/layer4_attention_weight_min": -105.39399719238281,
      "activations/layer5_attention_weight_max": 47.82463073730469,
      "activations/layer5_attention_weight_min": -60.464752197265625,
      "activations/layer6_attention_weight_max": 42.88611602783203,
      "activations/layer6_attention_weight_min": -44.133792877197266,
      "activations/layer7_attention_weight_max": 92.00851440429688,
      "activations/layer7_attention_weight_min": -92.21405792236328,
      "activations/layer8_attention_weight_max": 41.00713348388672,
      "activations/layer8_attention_weight_min": -43.51631546020508,
      "activations/layer9_attention_weight_max": 34.97610092163086,
      "activations/layer9_attention_weight_min": -33.54876708984375,
      "epoch": 16.3,
      "learning_rate": 4.533181818181818e-05,
      "loss": 2.7464,
      "step": 280550
    },
    {
      "activations/layer0_attention_weight_max": 16.00889778137207,
      "activations/layer0_attention_weight_min": -12.281866073608398,
      "activations/layer10_attention_weight_max": 37.065269470214844,
      "activations/layer10_attention_weight_min": -36.53447341918945,
      "activations/layer11_attention_weight_max": 34.786231994628906,
      "activations/layer11_attention_weight_min": -36.66290283203125,
      "activations/layer12_attention_weight_max": 26.93674659729004,
      "activations/layer12_attention_weight_min": -30.830507278442383,
      "activations/layer13_attention_weight_max": 39.961021423339844,
      "activations/layer13_attention_weight_min": -36.6929817199707,
      "activations/layer14_attention_weight_max": 39.88314437866211,
      "activations/layer14_attention_weight_min": -38.296356201171875,
      "activations/layer15_attention_weight_max": 36.8198127746582,
      "activations/layer15_attention_weight_min": -34.77505874633789,
      "activations/layer16_attention_weight_max": 29.579381942749023,
      "activations/layer16_attention_weight_min": -27.80232048034668,
      "activations/layer17_attention_weight_max": 28.44536781311035,
      "activations/layer17_attention_weight_min": -25.45841407775879,
      "activations/layer18_attention_weight_max": 31.946815490722656,
      "activations/layer18_attention_weight_min": -24.39206886291504,
      "activations/layer19_attention_weight_max": 30.39451026916504,
      "activations/layer19_attention_weight_min": -31.396207809448242,
      "activations/layer1_attention_weight_max": 16.92987060546875,
      "activations/layer1_attention_weight_min": -16.601938247680664,
      "activations/layer20_attention_weight_max": 26.4241886138916,
      "activations/layer20_attention_weight_min": -24.486656188964844,
      "activations/layer21_attention_weight_max": 26.836551666259766,
      "activations/layer21_attention_weight_min": -23.915983200073242,
      "activations/layer22_attention_weight_max": 38.69752502441406,
      "activations/layer22_attention_weight_min": -30.090179443359375,
      "activations/layer23_attention_weight_max": 31.1784725189209,
      "activations/layer23_attention_weight_min": -24.283445358276367,
      "activations/layer2_attention_weight_max": 32.78306198120117,
      "activations/layer2_attention_weight_min": -32.26443862915039,
      "activations/layer3_attention_weight_max": 92.36092376708984,
      "activations/layer3_attention_weight_min": -92.86723327636719,
      "activations/layer4_attention_weight_max": 104.83209228515625,
      "activations/layer4_attention_weight_min": -107.91716766357422,
      "activations/layer5_attention_weight_max": 50.34967803955078,
      "activations/layer5_attention_weight_min": -56.60883712768555,
      "activations/layer6_attention_weight_max": 46.08201217651367,
      "activations/layer6_attention_weight_min": -46.953311920166016,
      "activations/layer7_attention_weight_max": 93.86517333984375,
      "activations/layer7_attention_weight_min": -96.07257843017578,
      "activations/layer8_attention_weight_max": 44.29957962036133,
      "activations/layer8_attention_weight_min": -46.1633186340332,
      "activations/layer9_attention_weight_max": 35.25215148925781,
      "activations/layer9_attention_weight_min": -37.268009185791016,
      "epoch": 16.3,
      "learning_rate": 4.531287878787878e-05,
      "loss": 2.7407,
      "step": 280600
    },
    {
      "activations/layer0_attention_weight_max": 15.86121940612793,
      "activations/layer0_attention_weight_min": -12.054956436157227,
      "activations/layer10_attention_weight_max": 37.376922607421875,
      "activations/layer10_attention_weight_min": -35.10730743408203,
      "activations/layer11_attention_weight_max": 37.89629364013672,
      "activations/layer11_attention_weight_min": -34.70194625854492,
      "activations/layer12_attention_weight_max": 28.54593276977539,
      "activations/layer12_attention_weight_min": -27.28281021118164,
      "activations/layer13_attention_weight_max": 41.035789489746094,
      "activations/layer13_attention_weight_min": -34.957008361816406,
      "activations/layer14_attention_weight_max": 45.19562530517578,
      "activations/layer14_attention_weight_min": -38.778263092041016,
      "activations/layer15_attention_weight_max": 43.574546813964844,
      "activations/layer15_attention_weight_min": -34.82651138305664,
      "activations/layer16_attention_weight_max": 33.59485626220703,
      "activations/layer16_attention_weight_min": -27.537790298461914,
      "activations/layer17_attention_weight_max": 29.479103088378906,
      "activations/layer17_attention_weight_min": -25.631296157836914,
      "activations/layer18_attention_weight_max": 31.306325912475586,
      "activations/layer18_attention_weight_min": -27.261207580566406,
      "activations/layer19_attention_weight_max": 32.433467864990234,
      "activations/layer19_attention_weight_min": -30.51309585571289,
      "activations/layer1_attention_weight_max": 17.282474517822266,
      "activations/layer1_attention_weight_min": -16.36263656616211,
      "activations/layer20_attention_weight_max": 27.935707092285156,
      "activations/layer20_attention_weight_min": -25.051549911499023,
      "activations/layer21_attention_weight_max": 29.242631912231445,
      "activations/layer21_attention_weight_min": -23.894681930541992,
      "activations/layer22_attention_weight_max": 42.72545623779297,
      "activations/layer22_attention_weight_min": -30.340681076049805,
      "activations/layer23_attention_weight_max": 32.39656448364258,
      "activations/layer23_attention_weight_min": -26.456899642944336,
      "activations/layer2_attention_weight_max": 32.770084381103516,
      "activations/layer2_attention_weight_min": -34.595211029052734,
      "activations/layer3_attention_weight_max": 98.33041381835938,
      "activations/layer3_attention_weight_min": -97.82655334472656,
      "activations/layer4_attention_weight_max": 110.58072662353516,
      "activations/layer4_attention_weight_min": -104.4698257446289,
      "activations/layer5_attention_weight_max": 55.68798065185547,
      "activations/layer5_attention_weight_min": -60.38837432861328,
      "activations/layer6_attention_weight_max": 48.01070785522461,
      "activations/layer6_attention_weight_min": -47.094627380371094,
      "activations/layer7_attention_weight_max": 96.02407836914062,
      "activations/layer7_attention_weight_min": -96.1952133178711,
      "activations/layer8_attention_weight_max": 45.29713439941406,
      "activations/layer8_attention_weight_min": -44.01258087158203,
      "activations/layer9_attention_weight_max": 36.0344123840332,
      "activations/layer9_attention_weight_min": -35.77433395385742,
      "epoch": 16.31,
      "learning_rate": 4.5293939393939394e-05,
      "loss": 2.7524,
      "step": 280650
    },
    {
      "activations/layer0_attention_weight_max": 14.774713516235352,
      "activations/layer0_attention_weight_min": -14.027009010314941,
      "activations/layer10_attention_weight_max": 34.0428466796875,
      "activations/layer10_attention_weight_min": -33.709861755371094,
      "activations/layer11_attention_weight_max": 33.25962829589844,
      "activations/layer11_attention_weight_min": -35.029449462890625,
      "activations/layer12_attention_weight_max": 26.387046813964844,
      "activations/layer12_attention_weight_min": -25.69816017150879,
      "activations/layer13_attention_weight_max": 39.664852142333984,
      "activations/layer13_attention_weight_min": -35.6342658996582,
      "activations/layer14_attention_weight_max": 40.99235153198242,
      "activations/layer14_attention_weight_min": -36.35101318359375,
      "activations/layer15_attention_weight_max": 36.08915328979492,
      "activations/layer15_attention_weight_min": -33.09248352050781,
      "activations/layer16_attention_weight_max": 29.21619987487793,
      "activations/layer16_attention_weight_min": -27.00425148010254,
      "activations/layer17_attention_weight_max": 30.598047256469727,
      "activations/layer17_attention_weight_min": -25.05831527709961,
      "activations/layer18_attention_weight_max": 30.03973388671875,
      "activations/layer18_attention_weight_min": -25.272829055786133,
      "activations/layer19_attention_weight_max": 33.93828582763672,
      "activations/layer19_attention_weight_min": -35.69230270385742,
      "activations/layer1_attention_weight_max": 16.16811180114746,
      "activations/layer1_attention_weight_min": -15.483891487121582,
      "activations/layer20_attention_weight_max": 29.6999454498291,
      "activations/layer20_attention_weight_min": -26.83809471130371,
      "activations/layer21_attention_weight_max": 28.721702575683594,
      "activations/layer21_attention_weight_min": -25.814617156982422,
      "activations/layer22_attention_weight_max": 42.66965866088867,
      "activations/layer22_attention_weight_min": -29.751075744628906,
      "activations/layer23_attention_weight_max": 33.062679290771484,
      "activations/layer23_attention_weight_min": -26.49045181274414,
      "activations/layer2_attention_weight_max": 32.52259826660156,
      "activations/layer2_attention_weight_min": -31.3746395111084,
      "activations/layer3_attention_weight_max": 95.04505920410156,
      "activations/layer3_attention_weight_min": -93.6585922241211,
      "activations/layer4_attention_weight_max": 102.3724594116211,
      "activations/layer4_attention_weight_min": -105.53192138671875,
      "activations/layer5_attention_weight_max": 49.6024284362793,
      "activations/layer5_attention_weight_min": -64.92289733886719,
      "activations/layer6_attention_weight_max": 43.49178695678711,
      "activations/layer6_attention_weight_min": -45.911582946777344,
      "activations/layer7_attention_weight_max": 87.89386749267578,
      "activations/layer7_attention_weight_min": -93.90686798095703,
      "activations/layer8_attention_weight_max": 39.71488571166992,
      "activations/layer8_attention_weight_min": -42.848724365234375,
      "activations/layer9_attention_weight_max": 32.056434631347656,
      "activations/layer9_attention_weight_min": -33.636680603027344,
      "epoch": 16.31,
      "learning_rate": 4.5274999999999996e-05,
      "loss": 2.7546,
      "step": 280700
    },
    {
      "activations/layer0_attention_weight_max": 16.141643524169922,
      "activations/layer0_attention_weight_min": -12.816618919372559,
      "activations/layer10_attention_weight_max": 34.158973693847656,
      "activations/layer10_attention_weight_min": -32.79231262207031,
      "activations/layer11_attention_weight_max": 33.34140396118164,
      "activations/layer11_attention_weight_min": -32.094757080078125,
      "activations/layer12_attention_weight_max": 26.453651428222656,
      "activations/layer12_attention_weight_min": -25.568572998046875,
      "activations/layer13_attention_weight_max": 40.49296569824219,
      "activations/layer13_attention_weight_min": -33.34479522705078,
      "activations/layer14_attention_weight_max": 41.24221420288086,
      "activations/layer14_attention_weight_min": -36.27210235595703,
      "activations/layer15_attention_weight_max": 35.48741149902344,
      "activations/layer15_attention_weight_min": -32.51462936401367,
      "activations/layer16_attention_weight_max": 30.14500617980957,
      "activations/layer16_attention_weight_min": -27.937711715698242,
      "activations/layer17_attention_weight_max": 30.42612648010254,
      "activations/layer17_attention_weight_min": -25.217988967895508,
      "activations/layer18_attention_weight_max": 29.739835739135742,
      "activations/layer18_attention_weight_min": -24.666505813598633,
      "activations/layer19_attention_weight_max": 30.317371368408203,
      "activations/layer19_attention_weight_min": -30.64597511291504,
      "activations/layer1_attention_weight_max": 16.913148880004883,
      "activations/layer1_attention_weight_min": -15.220817565917969,
      "activations/layer20_attention_weight_max": 26.469621658325195,
      "activations/layer20_attention_weight_min": -24.903553009033203,
      "activations/layer21_attention_weight_max": 26.047441482543945,
      "activations/layer21_attention_weight_min": -24.869165420532227,
      "activations/layer22_attention_weight_max": 38.45518112182617,
      "activations/layer22_attention_weight_min": -29.515546798706055,
      "activations/layer23_attention_weight_max": 29.90359878540039,
      "activations/layer23_attention_weight_min": -25.536794662475586,
      "activations/layer2_attention_weight_max": 33.7813835144043,
      "activations/layer2_attention_weight_min": -32.48442840576172,
      "activations/layer3_attention_weight_max": 95.72889709472656,
      "activations/layer3_attention_weight_min": -98.50926208496094,
      "activations/layer4_attention_weight_max": 103.4165267944336,
      "activations/layer4_attention_weight_min": -107.78380584716797,
      "activations/layer5_attention_weight_max": 49.14582061767578,
      "activations/layer5_attention_weight_min": -61.06842803955078,
      "activations/layer6_attention_weight_max": 43.100791931152344,
      "activations/layer6_attention_weight_min": -45.73686218261719,
      "activations/layer7_attention_weight_max": 87.7015609741211,
      "activations/layer7_attention_weight_min": -95.16907501220703,
      "activations/layer8_attention_weight_max": 39.257286071777344,
      "activations/layer8_attention_weight_min": -44.428428649902344,
      "activations/layer9_attention_weight_max": 31.436038970947266,
      "activations/layer9_attention_weight_min": -34.02769470214844,
      "epoch": 16.31,
      "learning_rate": 4.5256060606060604e-05,
      "loss": 2.7343,
      "step": 280750
    },
    {
      "activations/layer0_attention_weight_max": 15.4521484375,
      "activations/layer0_attention_weight_min": -12.632296562194824,
      "activations/layer10_attention_weight_max": 34.340457916259766,
      "activations/layer10_attention_weight_min": -34.340354919433594,
      "activations/layer11_attention_weight_max": 35.34308624267578,
      "activations/layer11_attention_weight_min": -35.34438705444336,
      "activations/layer12_attention_weight_max": 27.696062088012695,
      "activations/layer12_attention_weight_min": -27.339515686035156,
      "activations/layer13_attention_weight_max": 41.61670684814453,
      "activations/layer13_attention_weight_min": -35.21681213378906,
      "activations/layer14_attention_weight_max": 42.00623321533203,
      "activations/layer14_attention_weight_min": -39.11024856567383,
      "activations/layer15_attention_weight_max": 38.67902374267578,
      "activations/layer15_attention_weight_min": -34.46451187133789,
      "activations/layer16_attention_weight_max": 28.648765563964844,
      "activations/layer16_attention_weight_min": -28.462190628051758,
      "activations/layer17_attention_weight_max": 31.35466194152832,
      "activations/layer17_attention_weight_min": -27.605722427368164,
      "activations/layer18_attention_weight_max": 31.04803466796875,
      "activations/layer18_attention_weight_min": -26.060388565063477,
      "activations/layer19_attention_weight_max": 36.05570602416992,
      "activations/layer19_attention_weight_min": -32.23441696166992,
      "activations/layer1_attention_weight_max": 16.300464630126953,
      "activations/layer1_attention_weight_min": -14.543289184570312,
      "activations/layer20_attention_weight_max": 28.60187339782715,
      "activations/layer20_attention_weight_min": -26.650951385498047,
      "activations/layer21_attention_weight_max": 27.835153579711914,
      "activations/layer21_attention_weight_min": -27.427291870117188,
      "activations/layer22_attention_weight_max": 46.9540901184082,
      "activations/layer22_attention_weight_min": -35.0790901184082,
      "activations/layer23_attention_weight_max": 34.06707000732422,
      "activations/layer23_attention_weight_min": -27.035625457763672,
      "activations/layer2_attention_weight_max": 33.015281677246094,
      "activations/layer2_attention_weight_min": -33.767066955566406,
      "activations/layer3_attention_weight_max": 96.04620361328125,
      "activations/layer3_attention_weight_min": -99.6994400024414,
      "activations/layer4_attention_weight_max": 106.26153564453125,
      "activations/layer4_attention_weight_min": -118.91768646240234,
      "activations/layer5_attention_weight_max": 53.675472259521484,
      "activations/layer5_attention_weight_min": -64.07899475097656,
      "activations/layer6_attention_weight_max": 45.57099151611328,
      "activations/layer6_attention_weight_min": -45.21272277832031,
      "activations/layer7_attention_weight_max": 95.88155364990234,
      "activations/layer7_attention_weight_min": -94.15912628173828,
      "activations/layer8_attention_weight_max": 43.179264068603516,
      "activations/layer8_attention_weight_min": -42.90729904174805,
      "activations/layer9_attention_weight_max": 33.28554153442383,
      "activations/layer9_attention_weight_min": -34.177154541015625,
      "epoch": 16.32,
      "learning_rate": 4.5237121212121206e-05,
      "loss": 2.745,
      "step": 280800
    },
    {
      "activations/layer0_attention_weight_max": 16.136032104492188,
      "activations/layer0_attention_weight_min": -12.640029907226562,
      "activations/layer10_attention_weight_max": 41.473514556884766,
      "activations/layer10_attention_weight_min": -39.71722412109375,
      "activations/layer11_attention_weight_max": 41.69124984741211,
      "activations/layer11_attention_weight_min": -41.180030822753906,
      "activations/layer12_attention_weight_max": 26.456439971923828,
      "activations/layer12_attention_weight_min": -27.015783309936523,
      "activations/layer13_attention_weight_max": 37.103294372558594,
      "activations/layer13_attention_weight_min": -35.848899841308594,
      "activations/layer14_attention_weight_max": 41.52522659301758,
      "activations/layer14_attention_weight_min": -39.260494232177734,
      "activations/layer15_attention_weight_max": 42.14223098754883,
      "activations/layer15_attention_weight_min": -36.73539352416992,
      "activations/layer16_attention_weight_max": 29.83768653869629,
      "activations/layer16_attention_weight_min": -26.993457794189453,
      "activations/layer17_attention_weight_max": 31.611915588378906,
      "activations/layer17_attention_weight_min": -25.38155746459961,
      "activations/layer18_attention_weight_max": 28.6487979888916,
      "activations/layer18_attention_weight_min": -24.69553565979004,
      "activations/layer19_attention_weight_max": 33.763912200927734,
      "activations/layer19_attention_weight_min": -29.183507919311523,
      "activations/layer1_attention_weight_max": 16.251514434814453,
      "activations/layer1_attention_weight_min": -14.799912452697754,
      "activations/layer20_attention_weight_max": 28.636993408203125,
      "activations/layer20_attention_weight_min": -26.36220932006836,
      "activations/layer21_attention_weight_max": 28.517667770385742,
      "activations/layer21_attention_weight_min": -24.20792007446289,
      "activations/layer22_attention_weight_max": 42.70854568481445,
      "activations/layer22_attention_weight_min": -29.150592803955078,
      "activations/layer23_attention_weight_max": 33.06200408935547,
      "activations/layer23_attention_weight_min": -27.058528900146484,
      "activations/layer2_attention_weight_max": 31.421920776367188,
      "activations/layer2_attention_weight_min": -32.336971282958984,
      "activations/layer3_attention_weight_max": 95.06578826904297,
      "activations/layer3_attention_weight_min": -97.40560913085938,
      "activations/layer4_attention_weight_max": 103.29621887207031,
      "activations/layer4_attention_weight_min": -104.85550689697266,
      "activations/layer5_attention_weight_max": 50.567481994628906,
      "activations/layer5_attention_weight_min": -59.407020568847656,
      "activations/layer6_attention_weight_max": 45.241920471191406,
      "activations/layer6_attention_weight_min": -45.41263961791992,
      "activations/layer7_attention_weight_max": 90.97531127929688,
      "activations/layer7_attention_weight_min": -94.21701049804688,
      "activations/layer8_attention_weight_max": 45.49631118774414,
      "activations/layer8_attention_weight_min": -45.243324279785156,
      "activations/layer9_attention_weight_max": 39.05918502807617,
      "activations/layer9_attention_weight_min": -37.3232421875,
      "epoch": 16.32,
      "learning_rate": 4.5218181818181814e-05,
      "loss": 2.7469,
      "step": 280850
    },
    {
      "activations/layer0_attention_weight_max": 15.606165885925293,
      "activations/layer0_attention_weight_min": -12.081561088562012,
      "activations/layer10_attention_weight_max": 34.09713363647461,
      "activations/layer10_attention_weight_min": -35.110286712646484,
      "activations/layer11_attention_weight_max": 34.29914474487305,
      "activations/layer11_attention_weight_min": -33.91423797607422,
      "activations/layer12_attention_weight_max": 26.603519439697266,
      "activations/layer12_attention_weight_min": -26.840717315673828,
      "activations/layer13_attention_weight_max": 38.31730270385742,
      "activations/layer13_attention_weight_min": -37.62712097167969,
      "activations/layer14_attention_weight_max": 46.93718338012695,
      "activations/layer14_attention_weight_min": -40.4152946472168,
      "activations/layer15_attention_weight_max": 38.86941146850586,
      "activations/layer15_attention_weight_min": -36.347816467285156,
      "activations/layer16_attention_weight_max": 32.316734313964844,
      "activations/layer16_attention_weight_min": -27.714962005615234,
      "activations/layer17_attention_weight_max": 30.50284194946289,
      "activations/layer17_attention_weight_min": -26.029212951660156,
      "activations/layer18_attention_weight_max": 30.046424865722656,
      "activations/layer18_attention_weight_min": -24.863988876342773,
      "activations/layer19_attention_weight_max": 37.21820831298828,
      "activations/layer19_attention_weight_min": -32.609588623046875,
      "activations/layer1_attention_weight_max": 17.27414894104004,
      "activations/layer1_attention_weight_min": -15.493680000305176,
      "activations/layer20_attention_weight_max": 31.148387908935547,
      "activations/layer20_attention_weight_min": -24.962614059448242,
      "activations/layer21_attention_weight_max": 28.308385848999023,
      "activations/layer21_attention_weight_min": -24.003463745117188,
      "activations/layer22_attention_weight_max": 41.96736145019531,
      "activations/layer22_attention_weight_min": -30.67865753173828,
      "activations/layer23_attention_weight_max": 35.91976547241211,
      "activations/layer23_attention_weight_min": -25.344085693359375,
      "activations/layer2_attention_weight_max": 32.75251770019531,
      "activations/layer2_attention_weight_min": -32.36874771118164,
      "activations/layer3_attention_weight_max": 93.49768829345703,
      "activations/layer3_attention_weight_min": -95.44298553466797,
      "activations/layer4_attention_weight_max": 101.9129638671875,
      "activations/layer4_attention_weight_min": -109.47216796875,
      "activations/layer5_attention_weight_max": 50.58821105957031,
      "activations/layer5_attention_weight_min": -60.79400634765625,
      "activations/layer6_attention_weight_max": 44.03668975830078,
      "activations/layer6_attention_weight_min": -47.318355560302734,
      "activations/layer7_attention_weight_max": 90.78815460205078,
      "activations/layer7_attention_weight_min": -96.1440200805664,
      "activations/layer8_attention_weight_max": 42.30381774902344,
      "activations/layer8_attention_weight_min": -44.539371490478516,
      "activations/layer9_attention_weight_max": 34.01285934448242,
      "activations/layer9_attention_weight_min": -33.54657745361328,
      "epoch": 16.32,
      "learning_rate": 4.519924242424242e-05,
      "loss": 2.7461,
      "step": 280900
    },
    {
      "activations/layer0_attention_weight_max": 16.223764419555664,
      "activations/layer0_attention_weight_min": -12.363621711730957,
      "activations/layer10_attention_weight_max": 34.11442184448242,
      "activations/layer10_attention_weight_min": -33.4958381652832,
      "activations/layer11_attention_weight_max": 33.968597412109375,
      "activations/layer11_attention_weight_min": -33.174476623535156,
      "activations/layer12_attention_weight_max": 27.401521682739258,
      "activations/layer12_attention_weight_min": -26.245485305786133,
      "activations/layer13_attention_weight_max": 41.707000732421875,
      "activations/layer13_attention_weight_min": -36.19145965576172,
      "activations/layer14_attention_weight_max": 45.948394775390625,
      "activations/layer14_attention_weight_min": -38.93955612182617,
      "activations/layer15_attention_weight_max": 41.805686950683594,
      "activations/layer15_attention_weight_min": -36.60835266113281,
      "activations/layer16_attention_weight_max": 31.107105255126953,
      "activations/layer16_attention_weight_min": -28.843997955322266,
      "activations/layer17_attention_weight_max": 32.87451934814453,
      "activations/layer17_attention_weight_min": -27.92738151550293,
      "activations/layer18_attention_weight_max": 31.804582595825195,
      "activations/layer18_attention_weight_min": -25.581199645996094,
      "activations/layer19_attention_weight_max": 36.04316329956055,
      "activations/layer19_attention_weight_min": -33.37748718261719,
      "activations/layer1_attention_weight_max": 16.79287338256836,
      "activations/layer1_attention_weight_min": -14.479535102844238,
      "activations/layer20_attention_weight_max": 28.637121200561523,
      "activations/layer20_attention_weight_min": -26.49666976928711,
      "activations/layer21_attention_weight_max": 28.40604591369629,
      "activations/layer21_attention_weight_min": -26.242380142211914,
      "activations/layer22_attention_weight_max": 41.077293395996094,
      "activations/layer22_attention_weight_min": -33.26457595825195,
      "activations/layer23_attention_weight_max": 34.565486907958984,
      "activations/layer23_attention_weight_min": -25.129161834716797,
      "activations/layer2_attention_weight_max": 32.25563049316406,
      "activations/layer2_attention_weight_min": -29.77480697631836,
      "activations/layer3_attention_weight_max": 95.47537231445312,
      "activations/layer3_attention_weight_min": -93.37855529785156,
      "activations/layer4_attention_weight_max": 107.6573486328125,
      "activations/layer4_attention_weight_min": -111.85250091552734,
      "activations/layer5_attention_weight_max": 48.06184005737305,
      "activations/layer5_attention_weight_min": -62.12488555908203,
      "activations/layer6_attention_weight_max": 44.47307205200195,
      "activations/layer6_attention_weight_min": -49.25875473022461,
      "activations/layer7_attention_weight_max": 86.33873748779297,
      "activations/layer7_attention_weight_min": -92.40103912353516,
      "activations/layer8_attention_weight_max": 41.6236457824707,
      "activations/layer8_attention_weight_min": -43.65898132324219,
      "activations/layer9_attention_weight_max": 33.42753219604492,
      "activations/layer9_attention_weight_min": -35.73068618774414,
      "epoch": 16.32,
      "learning_rate": 4.5180303030303024e-05,
      "loss": 2.7402,
      "step": 280950
    },
    {
      "activations/layer0_attention_weight_max": 16.757631301879883,
      "activations/layer0_attention_weight_min": -13.405404090881348,
      "activations/layer10_attention_weight_max": 34.260440826416016,
      "activations/layer10_attention_weight_min": -32.5881462097168,
      "activations/layer11_attention_weight_max": 34.521995544433594,
      "activations/layer11_attention_weight_min": -32.7767448425293,
      "activations/layer12_attention_weight_max": 27.100202560424805,
      "activations/layer12_attention_weight_min": -27.008913040161133,
      "activations/layer13_attention_weight_max": 40.603233337402344,
      "activations/layer13_attention_weight_min": -35.98438262939453,
      "activations/layer14_attention_weight_max": 43.53603744506836,
      "activations/layer14_attention_weight_min": -40.705894470214844,
      "activations/layer15_attention_weight_max": 38.9439582824707,
      "activations/layer15_attention_weight_min": -35.18152618408203,
      "activations/layer16_attention_weight_max": 32.526947021484375,
      "activations/layer16_attention_weight_min": -28.286949157714844,
      "activations/layer17_attention_weight_max": 31.788686752319336,
      "activations/layer17_attention_weight_min": -26.350427627563477,
      "activations/layer18_attention_weight_max": 37.617095947265625,
      "activations/layer18_attention_weight_min": -27.352645874023438,
      "activations/layer19_attention_weight_max": 39.34664535522461,
      "activations/layer19_attention_weight_min": -32.898441314697266,
      "activations/layer1_attention_weight_max": 15.878153800964355,
      "activations/layer1_attention_weight_min": -14.450570106506348,
      "activations/layer20_attention_weight_max": 32.95823669433594,
      "activations/layer20_attention_weight_min": -25.2841796875,
      "activations/layer21_attention_weight_max": 33.42868423461914,
      "activations/layer21_attention_weight_min": -25.062692642211914,
      "activations/layer22_attention_weight_max": 51.13468551635742,
      "activations/layer22_attention_weight_min": -35.01360321044922,
      "activations/layer23_attention_weight_max": 36.45778274536133,
      "activations/layer23_attention_weight_min": -26.625751495361328,
      "activations/layer2_attention_weight_max": 33.37310791015625,
      "activations/layer2_attention_weight_min": -33.82651138305664,
      "activations/layer3_attention_weight_max": 97.09123992919922,
      "activations/layer3_attention_weight_min": -97.60165405273438,
      "activations/layer4_attention_weight_max": 106.57279205322266,
      "activations/layer4_attention_weight_min": -110.9057388305664,
      "activations/layer5_attention_weight_max": 52.34081268310547,
      "activations/layer5_attention_weight_min": -62.53254699707031,
      "activations/layer6_attention_weight_max": 43.711238861083984,
      "activations/layer6_attention_weight_min": -45.60984802246094,
      "activations/layer7_attention_weight_max": 88.51130676269531,
      "activations/layer7_attention_weight_min": -92.34771728515625,
      "activations/layer8_attention_weight_max": 39.67340087890625,
      "activations/layer8_attention_weight_min": -42.06191635131836,
      "activations/layer9_attention_weight_max": 30.454904556274414,
      "activations/layer9_attention_weight_min": -33.8919792175293,
      "epoch": 16.33,
      "learning_rate": 4.516136363636363e-05,
      "loss": 2.7531,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4264,
      "eval_samples_per_second": 509.589,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4264,
      "eval_openwebtext_samples_per_second": 509.589,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9548,
      "eval_wikitext_samples_per_second": 233.27,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_lambada_loss": 2.443359375,
      "eval_lambada_ppl": 11.511647800524438,
      "eval_lambada_runtime": 9.5589,
      "eval_lambada_samples_per_second": 509.369,
      "step": 281000
    },
    {
      "activations/layer0_attention_weight_max": 15.755932807922363,
      "activations/layer0_attention_weight_min": -12.979771614074707,
      "activations/layer10_attention_weight_max": 33.41977310180664,
      "activations/layer10_attention_weight_min": -32.80710220336914,
      "activations/layer11_attention_weight_max": 34.858909606933594,
      "activations/layer11_attention_weight_min": -33.431983947753906,
      "activations/layer12_attention_weight_max": 26.137252807617188,
      "activations/layer12_attention_weight_min": -25.6148624420166,
      "activations/layer13_attention_weight_max": 39.926300048828125,
      "activations/layer13_attention_weight_min": -35.1828498840332,
      "activations/layer14_attention_weight_max": 44.34402084350586,
      "activations/layer14_attention_weight_min": -36.74224090576172,
      "activations/layer15_attention_weight_max": 38.87106704711914,
      "activations/layer15_attention_weight_min": -34.39314651489258,
      "activations/layer16_attention_weight_max": 30.49855613708496,
      "activations/layer16_attention_weight_min": -28.462108612060547,
      "activations/layer17_attention_weight_max": 30.151519775390625,
      "activations/layer17_attention_weight_min": -25.18912696838379,
      "activations/layer18_attention_weight_max": 31.620162963867188,
      "activations/layer18_attention_weight_min": -25.60692024230957,
      "activations/layer19_attention_weight_max": 32.27130126953125,
      "activations/layer19_attention_weight_min": -30.596784591674805,
      "activations/layer1_attention_weight_max": 15.502586364746094,
      "activations/layer1_attention_weight_min": -15.101546287536621,
      "activations/layer20_attention_weight_max": 27.711008071899414,
      "activations/layer20_attention_weight_min": -27.618755340576172,
      "activations/layer21_attention_weight_max": 27.30194091796875,
      "activations/layer21_attention_weight_min": -27.411540985107422,
      "activations/layer22_attention_weight_max": 39.2488899230957,
      "activations/layer22_attention_weight_min": -35.41946792602539,
      "activations/layer23_attention_weight_max": 31.206024169921875,
      "activations/layer23_attention_weight_min": -28.340688705444336,
      "activations/layer2_attention_weight_max": 32.02174758911133,
      "activations/layer2_attention_weight_min": -32.626712799072266,
      "activations/layer3_attention_weight_max": 94.1973648071289,
      "activations/layer3_attention_weight_min": -93.30406951904297,
      "activations/layer4_attention_weight_max": 102.63639068603516,
      "activations/layer4_attention_weight_min": -111.5203857421875,
      "activations/layer5_attention_weight_max": 48.232032775878906,
      "activations/layer5_attention_weight_min": -64.41400909423828,
      "activations/layer6_attention_weight_max": 41.7647705078125,
      "activations/layer6_attention_weight_min": -46.505455017089844,
      "activations/layer7_attention_weight_max": 88.64807891845703,
      "activations/layer7_attention_weight_min": -92.3968276977539,
      "activations/layer8_attention_weight_max": 41.31486511230469,
      "activations/layer8_attention_weight_min": -46.64336013793945,
      "activations/layer9_attention_weight_max": 33.19622039794922,
      "activations/layer9_attention_weight_min": -34.740150451660156,
      "epoch": 16.33,
      "learning_rate": 4.514242424242424e-05,
      "loss": 2.7515,
      "step": 281050
    },
    {
      "activations/layer0_attention_weight_max": 15.593048095703125,
      "activations/layer0_attention_weight_min": -12.744166374206543,
      "activations/layer10_attention_weight_max": 32.69969177246094,
      "activations/layer10_attention_weight_min": -35.18439483642578,
      "activations/layer11_attention_weight_max": 34.27662658691406,
      "activations/layer11_attention_weight_min": -36.92711639404297,
      "activations/layer12_attention_weight_max": 26.85837745666504,
      "activations/layer12_attention_weight_min": -27.18576431274414,
      "activations/layer13_attention_weight_max": 40.29936218261719,
      "activations/layer13_attention_weight_min": -37.20310974121094,
      "activations/layer14_attention_weight_max": 41.54600524902344,
      "activations/layer14_attention_weight_min": -40.444393157958984,
      "activations/layer15_attention_weight_max": 38.351585388183594,
      "activations/layer15_attention_weight_min": -37.939697265625,
      "activations/layer16_attention_weight_max": 30.02362060546875,
      "activations/layer16_attention_weight_min": -28.10419273376465,
      "activations/layer17_attention_weight_max": 28.412307739257812,
      "activations/layer17_attention_weight_min": -26.756006240844727,
      "activations/layer18_attention_weight_max": 31.93768310546875,
      "activations/layer18_attention_weight_min": -26.65268325805664,
      "activations/layer19_attention_weight_max": 30.520404815673828,
      "activations/layer19_attention_weight_min": -31.533573150634766,
      "activations/layer1_attention_weight_max": 16.9824275970459,
      "activations/layer1_attention_weight_min": -15.689360618591309,
      "activations/layer20_attention_weight_max": 27.359169006347656,
      "activations/layer20_attention_weight_min": -25.129222869873047,
      "activations/layer21_attention_weight_max": 26.6591796875,
      "activations/layer21_attention_weight_min": -24.057010650634766,
      "activations/layer22_attention_weight_max": 35.80832290649414,
      "activations/layer22_attention_weight_min": -30.42200469970703,
      "activations/layer23_attention_weight_max": 28.631635665893555,
      "activations/layer23_attention_weight_min": -25.3121395111084,
      "activations/layer2_attention_weight_max": 30.952739715576172,
      "activations/layer2_attention_weight_min": -31.781658172607422,
      "activations/layer3_attention_weight_max": 95.49066162109375,
      "activations/layer3_attention_weight_min": -99.45917510986328,
      "activations/layer4_attention_weight_max": 102.5333251953125,
      "activations/layer4_attention_weight_min": -109.16462707519531,
      "activations/layer5_attention_weight_max": 49.3590087890625,
      "activations/layer5_attention_weight_min": -62.093299865722656,
      "activations/layer6_attention_weight_max": 42.97177505493164,
      "activations/layer6_attention_weight_min": -47.560401916503906,
      "activations/layer7_attention_weight_max": 96.02161407470703,
      "activations/layer7_attention_weight_min": -96.86604309082031,
      "activations/layer8_attention_weight_max": 41.73352813720703,
      "activations/layer8_attention_weight_min": -43.33602523803711,
      "activations/layer9_attention_weight_max": 31.804006576538086,
      "activations/layer9_attention_weight_min": -34.5526237487793,
      "epoch": 16.33,
      "learning_rate": 4.512348484848484e-05,
      "loss": 2.7513,
      "step": 281100
    },
    {
      "activations/layer0_attention_weight_max": 16.24749755859375,
      "activations/layer0_attention_weight_min": -13.20721435546875,
      "activations/layer10_attention_weight_max": 40.36455154418945,
      "activations/layer10_attention_weight_min": -40.681087493896484,
      "activations/layer11_attention_weight_max": 41.827354431152344,
      "activations/layer11_attention_weight_min": -42.97605895996094,
      "activations/layer12_attention_weight_max": 30.575504302978516,
      "activations/layer12_attention_weight_min": -26.705856323242188,
      "activations/layer13_attention_weight_max": 47.59421157836914,
      "activations/layer13_attention_weight_min": -38.394622802734375,
      "activations/layer14_attention_weight_max": 53.37429428100586,
      "activations/layer14_attention_weight_min": -41.193016052246094,
      "activations/layer15_attention_weight_max": 45.974361419677734,
      "activations/layer15_attention_weight_min": -37.73634338378906,
      "activations/layer16_attention_weight_max": 34.61239242553711,
      "activations/layer16_attention_weight_min": -26.353803634643555,
      "activations/layer17_attention_weight_max": 33.243343353271484,
      "activations/layer17_attention_weight_min": -27.270206451416016,
      "activations/layer18_attention_weight_max": 32.114810943603516,
      "activations/layer18_attention_weight_min": -26.484357833862305,
      "activations/layer19_attention_weight_max": 33.793495178222656,
      "activations/layer19_attention_weight_min": -30.369871139526367,
      "activations/layer1_attention_weight_max": 16.94991683959961,
      "activations/layer1_attention_weight_min": -15.738450050354004,
      "activations/layer20_attention_weight_max": 29.025300979614258,
      "activations/layer20_attention_weight_min": -25.054075241088867,
      "activations/layer21_attention_weight_max": 27.574743270874023,
      "activations/layer21_attention_weight_min": -24.428579330444336,
      "activations/layer22_attention_weight_max": 41.950462341308594,
      "activations/layer22_attention_weight_min": -32.91926574707031,
      "activations/layer23_attention_weight_max": 30.009084701538086,
      "activations/layer23_attention_weight_min": -25.747989654541016,
      "activations/layer2_attention_weight_max": 34.444732666015625,
      "activations/layer2_attention_weight_min": -32.21367645263672,
      "activations/layer3_attention_weight_max": 96.47335052490234,
      "activations/layer3_attention_weight_min": -97.18074798583984,
      "activations/layer4_attention_weight_max": 105.00045013427734,
      "activations/layer4_attention_weight_min": -108.1301040649414,
      "activations/layer5_attention_weight_max": 50.289852142333984,
      "activations/layer5_attention_weight_min": -60.142921447753906,
      "activations/layer6_attention_weight_max": 49.937137603759766,
      "activations/layer6_attention_weight_min": -47.11265563964844,
      "activations/layer7_attention_weight_max": 98.72271728515625,
      "activations/layer7_attention_weight_min": -91.87288665771484,
      "activations/layer8_attention_weight_max": 47.67469787597656,
      "activations/layer8_attention_weight_min": -47.230690002441406,
      "activations/layer9_attention_weight_max": 38.878849029541016,
      "activations/layer9_attention_weight_min": -39.136810302734375,
      "epoch": 16.34,
      "learning_rate": 4.510454545454545e-05,
      "loss": 2.736,
      "step": 281150
    },
    {
      "activations/layer0_attention_weight_max": 16.27338409423828,
      "activations/layer0_attention_weight_min": -12.790838241577148,
      "activations/layer10_attention_weight_max": 33.16154098510742,
      "activations/layer10_attention_weight_min": -33.036354064941406,
      "activations/layer11_attention_weight_max": 34.05291748046875,
      "activations/layer11_attention_weight_min": -31.498716354370117,
      "activations/layer12_attention_weight_max": 27.046598434448242,
      "activations/layer12_attention_weight_min": -26.759220123291016,
      "activations/layer13_attention_weight_max": 42.299339294433594,
      "activations/layer13_attention_weight_min": -35.50431823730469,
      "activations/layer14_attention_weight_max": 41.40192413330078,
      "activations/layer14_attention_weight_min": -38.011444091796875,
      "activations/layer15_attention_weight_max": 43.71553421020508,
      "activations/layer15_attention_weight_min": -36.039127349853516,
      "activations/layer16_attention_weight_max": 29.450225830078125,
      "activations/layer16_attention_weight_min": -27.87291145324707,
      "activations/layer17_attention_weight_max": 29.903406143188477,
      "activations/layer17_attention_weight_min": -25.971223831176758,
      "activations/layer18_attention_weight_max": 34.0552864074707,
      "activations/layer18_attention_weight_min": -25.118743896484375,
      "activations/layer19_attention_weight_max": 33.03618621826172,
      "activations/layer19_attention_weight_min": -30.296588897705078,
      "activations/layer1_attention_weight_max": 19.989816665649414,
      "activations/layer1_attention_weight_min": -14.1399507522583,
      "activations/layer20_attention_weight_max": 30.735811233520508,
      "activations/layer20_attention_weight_min": -24.160791397094727,
      "activations/layer21_attention_weight_max": 29.533395767211914,
      "activations/layer21_attention_weight_min": -23.225088119506836,
      "activations/layer22_attention_weight_max": 42.5999755859375,
      "activations/layer22_attention_weight_min": -30.913410186767578,
      "activations/layer23_attention_weight_max": 30.21511459350586,
      "activations/layer23_attention_weight_min": -24.862165451049805,
      "activations/layer2_attention_weight_max": 31.650754928588867,
      "activations/layer2_attention_weight_min": -31.41301727294922,
      "activations/layer3_attention_weight_max": 90.3875961303711,
      "activations/layer3_attention_weight_min": -90.28702545166016,
      "activations/layer4_attention_weight_max": 99.86509704589844,
      "activations/layer4_attention_weight_min": -102.30245971679688,
      "activations/layer5_attention_weight_max": 50.942596435546875,
      "activations/layer5_attention_weight_min": -66.26699829101562,
      "activations/layer6_attention_weight_max": 41.3182373046875,
      "activations/layer6_attention_weight_min": -42.673194885253906,
      "activations/layer7_attention_weight_max": 91.71796417236328,
      "activations/layer7_attention_weight_min": -87.29376983642578,
      "activations/layer8_attention_weight_max": 39.09915542602539,
      "activations/layer8_attention_weight_min": -40.53996276855469,
      "activations/layer9_attention_weight_max": 32.96380615234375,
      "activations/layer9_attention_weight_min": -32.56785202026367,
      "epoch": 16.34,
      "learning_rate": 4.508560606060605e-05,
      "loss": 2.7441,
      "step": 281200
    },
    {
      "activations/layer0_attention_weight_max": 15.217644691467285,
      "activations/layer0_attention_weight_min": -12.632845878601074,
      "activations/layer10_attention_weight_max": 34.974403381347656,
      "activations/layer10_attention_weight_min": -33.922672271728516,
      "activations/layer11_attention_weight_max": 34.650394439697266,
      "activations/layer11_attention_weight_min": -33.87232971191406,
      "activations/layer12_attention_weight_max": 28.085769653320312,
      "activations/layer12_attention_weight_min": -28.088703155517578,
      "activations/layer13_attention_weight_max": 41.157752990722656,
      "activations/layer13_attention_weight_min": -35.84598922729492,
      "activations/layer14_attention_weight_max": 41.555450439453125,
      "activations/layer14_attention_weight_min": -36.864654541015625,
      "activations/layer15_attention_weight_max": 39.70186233520508,
      "activations/layer15_attention_weight_min": -34.97554397583008,
      "activations/layer16_attention_weight_max": 32.108455657958984,
      "activations/layer16_attention_weight_min": -28.956504821777344,
      "activations/layer17_attention_weight_max": 31.484914779663086,
      "activations/layer17_attention_weight_min": -26.923660278320312,
      "activations/layer18_attention_weight_max": 30.230966567993164,
      "activations/layer18_attention_weight_min": -24.709938049316406,
      "activations/layer19_attention_weight_max": 35.07656478881836,
      "activations/layer19_attention_weight_min": -32.24345016479492,
      "activations/layer1_attention_weight_max": 16.16233253479004,
      "activations/layer1_attention_weight_min": -15.636066436767578,
      "activations/layer20_attention_weight_max": 28.425792694091797,
      "activations/layer20_attention_weight_min": -29.366497039794922,
      "activations/layer21_attention_weight_max": 28.37731170654297,
      "activations/layer21_attention_weight_min": -25.84325408935547,
      "activations/layer22_attention_weight_max": 40.22940444946289,
      "activations/layer22_attention_weight_min": -34.310218811035156,
      "activations/layer23_attention_weight_max": 32.6935920715332,
      "activations/layer23_attention_weight_min": -24.698917388916016,
      "activations/layer2_attention_weight_max": 32.97319030761719,
      "activations/layer2_attention_weight_min": -31.486309051513672,
      "activations/layer3_attention_weight_max": 94.7885513305664,
      "activations/layer3_attention_weight_min": -94.4164047241211,
      "activations/layer4_attention_weight_max": 103.48381805419922,
      "activations/layer4_attention_weight_min": -104.180908203125,
      "activations/layer5_attention_weight_max": 52.28843307495117,
      "activations/layer5_attention_weight_min": -59.66576385498047,
      "activations/layer6_attention_weight_max": 45.379173278808594,
      "activations/layer6_attention_weight_min": -48.34724807739258,
      "activations/layer7_attention_weight_max": 88.61349487304688,
      "activations/layer7_attention_weight_min": -96.0463638305664,
      "activations/layer8_attention_weight_max": 42.34401321411133,
      "activations/layer8_attention_weight_min": -44.439361572265625,
      "activations/layer9_attention_weight_max": 34.032413482666016,
      "activations/layer9_attention_weight_min": -34.096656799316406,
      "epoch": 16.34,
      "learning_rate": 4.5066666666666667e-05,
      "loss": 2.7463,
      "step": 281250
    },
    {
      "activations/layer0_attention_weight_max": 16.04042625427246,
      "activations/layer0_attention_weight_min": -13.237748146057129,
      "activations/layer10_attention_weight_max": 40.303436279296875,
      "activations/layer10_attention_weight_min": -38.24968338012695,
      "activations/layer11_attention_weight_max": 41.577789306640625,
      "activations/layer11_attention_weight_min": -40.51658630371094,
      "activations/layer12_attention_weight_max": 30.136934280395508,
      "activations/layer12_attention_weight_min": -29.525297164916992,
      "activations/layer13_attention_weight_max": 44.68339538574219,
      "activations/layer13_attention_weight_min": -43.06087112426758,
      "activations/layer14_attention_weight_max": 56.74020767211914,
      "activations/layer14_attention_weight_min": -51.924049377441406,
      "activations/layer15_attention_weight_max": 48.50440979003906,
      "activations/layer15_attention_weight_min": -45.41303634643555,
      "activations/layer16_attention_weight_max": 35.82563018798828,
      "activations/layer16_attention_weight_min": -27.491788864135742,
      "activations/layer17_attention_weight_max": 29.689334869384766,
      "activations/layer17_attention_weight_min": -28.456003189086914,
      "activations/layer18_attention_weight_max": 32.10361862182617,
      "activations/layer18_attention_weight_min": -28.000465393066406,
      "activations/layer19_attention_weight_max": 41.9549674987793,
      "activations/layer19_attention_weight_min": -31.7453670501709,
      "activations/layer1_attention_weight_max": 15.944671630859375,
      "activations/layer1_attention_weight_min": -14.150314331054688,
      "activations/layer20_attention_weight_max": 31.05242347717285,
      "activations/layer20_attention_weight_min": -27.51591682434082,
      "activations/layer21_attention_weight_max": 28.383358001708984,
      "activations/layer21_attention_weight_min": -26.284982681274414,
      "activations/layer22_attention_weight_max": 46.93312072753906,
      "activations/layer22_attention_weight_min": -37.15301513671875,
      "activations/layer23_attention_weight_max": 33.34666442871094,
      "activations/layer23_attention_weight_min": -27.931236267089844,
      "activations/layer2_attention_weight_max": 32.06243133544922,
      "activations/layer2_attention_weight_min": -31.41623306274414,
      "activations/layer3_attention_weight_max": 92.96126556396484,
      "activations/layer3_attention_weight_min": -91.59925842285156,
      "activations/layer4_attention_weight_max": 104.20356750488281,
      "activations/layer4_attention_weight_min": -104.2864761352539,
      "activations/layer5_attention_weight_max": 51.81562042236328,
      "activations/layer5_attention_weight_min": -62.026649475097656,
      "activations/layer6_attention_weight_max": 44.43061065673828,
      "activations/layer6_attention_weight_min": -45.97946548461914,
      "activations/layer7_attention_weight_max": 101.98882293701172,
      "activations/layer7_attention_weight_min": -96.77021789550781,
      "activations/layer8_attention_weight_max": 46.22490692138672,
      "activations/layer8_attention_weight_min": -45.880409240722656,
      "activations/layer9_attention_weight_max": 35.31390380859375,
      "activations/layer9_attention_weight_min": -38.002315521240234,
      "epoch": 16.35,
      "learning_rate": 4.504772727272727e-05,
      "loss": 2.7379,
      "step": 281300
    },
    {
      "activations/layer0_attention_weight_max": 16.07773208618164,
      "activations/layer0_attention_weight_min": -11.987831115722656,
      "activations/layer10_attention_weight_max": 34.550106048583984,
      "activations/layer10_attention_weight_min": -37.69435501098633,
      "activations/layer11_attention_weight_max": 33.95248031616211,
      "activations/layer11_attention_weight_min": -36.17894744873047,
      "activations/layer12_attention_weight_max": 28.0943546295166,
      "activations/layer12_attention_weight_min": -26.73900604248047,
      "activations/layer13_attention_weight_max": 38.921974182128906,
      "activations/layer13_attention_weight_min": -38.387081146240234,
      "activations/layer14_attention_weight_max": 42.26408767700195,
      "activations/layer14_attention_weight_min": -39.98154067993164,
      "activations/layer15_attention_weight_max": 38.84895706176758,
      "activations/layer15_attention_weight_min": -37.10350799560547,
      "activations/layer16_attention_weight_max": 29.741254806518555,
      "activations/layer16_attention_weight_min": -27.316577911376953,
      "activations/layer17_attention_weight_max": 30.303823471069336,
      "activations/layer17_attention_weight_min": -26.14234161376953,
      "activations/layer18_attention_weight_max": 32.57112503051758,
      "activations/layer18_attention_weight_min": -29.74443244934082,
      "activations/layer19_attention_weight_max": 34.04714584350586,
      "activations/layer19_attention_weight_min": -33.40126419067383,
      "activations/layer1_attention_weight_max": 19.081552505493164,
      "activations/layer1_attention_weight_min": -16.96239471435547,
      "activations/layer20_attention_weight_max": 28.796873092651367,
      "activations/layer20_attention_weight_min": -26.280437469482422,
      "activations/layer21_attention_weight_max": 30.13805389404297,
      "activations/layer21_attention_weight_min": -25.76227378845215,
      "activations/layer22_attention_weight_max": 45.97951889038086,
      "activations/layer22_attention_weight_min": -34.869873046875,
      "activations/layer23_attention_weight_max": 36.427879333496094,
      "activations/layer23_attention_weight_min": -25.607023239135742,
      "activations/layer2_attention_weight_max": 32.738121032714844,
      "activations/layer2_attention_weight_min": -31.430723190307617,
      "activations/layer3_attention_weight_max": 92.34239959716797,
      "activations/layer3_attention_weight_min": -97.58244323730469,
      "activations/layer4_attention_weight_max": 103.6553955078125,
      "activations/layer4_attention_weight_min": -107.24895477294922,
      "activations/layer5_attention_weight_max": 49.72615051269531,
      "activations/layer5_attention_weight_min": -68.22447204589844,
      "activations/layer6_attention_weight_max": 44.817447662353516,
      "activations/layer6_attention_weight_min": -46.12638473510742,
      "activations/layer7_attention_weight_max": 92.24043273925781,
      "activations/layer7_attention_weight_min": -99.41824340820312,
      "activations/layer8_attention_weight_max": 43.81928634643555,
      "activations/layer8_attention_weight_min": -46.55646896362305,
      "activations/layer9_attention_weight_max": 33.95795822143555,
      "activations/layer9_attention_weight_min": -38.29153823852539,
      "epoch": 16.35,
      "learning_rate": 4.502878787878787e-05,
      "loss": 2.7365,
      "step": 281350
    },
    {
      "activations/layer0_attention_weight_max": 15.789604187011719,
      "activations/layer0_attention_weight_min": -12.663468360900879,
      "activations/layer10_attention_weight_max": 33.608360290527344,
      "activations/layer10_attention_weight_min": -34.234535217285156,
      "activations/layer11_attention_weight_max": 31.610685348510742,
      "activations/layer11_attention_weight_min": -34.82338333129883,
      "activations/layer12_attention_weight_max": 24.113914489746094,
      "activations/layer12_attention_weight_min": -25.445035934448242,
      "activations/layer13_attention_weight_max": 36.22659683227539,
      "activations/layer13_attention_weight_min": -35.90751266479492,
      "activations/layer14_attention_weight_max": 41.24259948730469,
      "activations/layer14_attention_weight_min": -39.11399841308594,
      "activations/layer15_attention_weight_max": 36.088897705078125,
      "activations/layer15_attention_weight_min": -35.98455047607422,
      "activations/layer16_attention_weight_max": 30.834732055664062,
      "activations/layer16_attention_weight_min": -30.966445922851562,
      "activations/layer17_attention_weight_max": 30.54537582397461,
      "activations/layer17_attention_weight_min": -26.210599899291992,
      "activations/layer18_attention_weight_max": 31.163488388061523,
      "activations/layer18_attention_weight_min": -25.65880584716797,
      "activations/layer19_attention_weight_max": 33.92826843261719,
      "activations/layer19_attention_weight_min": -34.96073532104492,
      "activations/layer1_attention_weight_max": 17.06332015991211,
      "activations/layer1_attention_weight_min": -16.337066650390625,
      "activations/layer20_attention_weight_max": 28.251108169555664,
      "activations/layer20_attention_weight_min": -29.88924789428711,
      "activations/layer21_attention_weight_max": 28.664480209350586,
      "activations/layer21_attention_weight_min": -27.585113525390625,
      "activations/layer22_attention_weight_max": 45.60454177856445,
      "activations/layer22_attention_weight_min": -34.42965316772461,
      "activations/layer23_attention_weight_max": 33.58837127685547,
      "activations/layer23_attention_weight_min": -28.560834884643555,
      "activations/layer2_attention_weight_max": 32.26786804199219,
      "activations/layer2_attention_weight_min": -31.202556610107422,
      "activations/layer3_attention_weight_max": 94.77855682373047,
      "activations/layer3_attention_weight_min": -95.16283416748047,
      "activations/layer4_attention_weight_max": 107.439208984375,
      "activations/layer4_attention_weight_min": -107.0888671875,
      "activations/layer5_attention_weight_max": 53.0904655456543,
      "activations/layer5_attention_weight_min": -64.7801284790039,
      "activations/layer6_attention_weight_max": 44.37802505493164,
      "activations/layer6_attention_weight_min": -47.594703674316406,
      "activations/layer7_attention_weight_max": 88.96800994873047,
      "activations/layer7_attention_weight_min": -92.03019714355469,
      "activations/layer8_attention_weight_max": 40.55910873413086,
      "activations/layer8_attention_weight_min": -42.63395309448242,
      "activations/layer9_attention_weight_max": 32.70606994628906,
      "activations/layer9_attention_weight_min": -35.075103759765625,
      "epoch": 16.35,
      "learning_rate": 4.5009848484848485e-05,
      "loss": 2.7314,
      "step": 281400
    },
    {
      "activations/layer0_attention_weight_max": 15.141355514526367,
      "activations/layer0_attention_weight_min": -12.78248119354248,
      "activations/layer10_attention_weight_max": 37.2388801574707,
      "activations/layer10_attention_weight_min": -34.9869384765625,
      "activations/layer11_attention_weight_max": 36.43126678466797,
      "activations/layer11_attention_weight_min": -34.68408966064453,
      "activations/layer12_attention_weight_max": 27.732088088989258,
      "activations/layer12_attention_weight_min": -27.528907775878906,
      "activations/layer13_attention_weight_max": 38.974891662597656,
      "activations/layer13_attention_weight_min": -37.114681243896484,
      "activations/layer14_attention_weight_max": 43.38438034057617,
      "activations/layer14_attention_weight_min": -40.27710723876953,
      "activations/layer15_attention_weight_max": 40.5277099609375,
      "activations/layer15_attention_weight_min": -36.608367919921875,
      "activations/layer16_attention_weight_max": 30.024555206298828,
      "activations/layer16_attention_weight_min": -27.35359764099121,
      "activations/layer17_attention_weight_max": 32.45934295654297,
      "activations/layer17_attention_weight_min": -26.696840286254883,
      "activations/layer18_attention_weight_max": 33.488243103027344,
      "activations/layer18_attention_weight_min": -27.39226722717285,
      "activations/layer19_attention_weight_max": 34.748680114746094,
      "activations/layer19_attention_weight_min": -32.25865936279297,
      "activations/layer1_attention_weight_max": 16.56248664855957,
      "activations/layer1_attention_weight_min": -15.452536582946777,
      "activations/layer20_attention_weight_max": 32.26359558105469,
      "activations/layer20_attention_weight_min": -26.10696029663086,
      "activations/layer21_attention_weight_max": 30.770706176757812,
      "activations/layer21_attention_weight_min": -26.042930603027344,
      "activations/layer22_attention_weight_max": 43.41478729248047,
      "activations/layer22_attention_weight_min": -31.97671127319336,
      "activations/layer23_attention_weight_max": 34.33039093017578,
      "activations/layer23_attention_weight_min": -27.776714324951172,
      "activations/layer2_attention_weight_max": 33.22316360473633,
      "activations/layer2_attention_weight_min": -32.39701843261719,
      "activations/layer3_attention_weight_max": 95.76628112792969,
      "activations/layer3_attention_weight_min": -100.10350799560547,
      "activations/layer4_attention_weight_max": 106.234619140625,
      "activations/layer4_attention_weight_min": -108.66065979003906,
      "activations/layer5_attention_weight_max": 48.63427734375,
      "activations/layer5_attention_weight_min": -59.907630920410156,
      "activations/layer6_attention_weight_max": 43.79433059692383,
      "activations/layer6_attention_weight_min": -45.661903381347656,
      "activations/layer7_attention_weight_max": 93.0223159790039,
      "activations/layer7_attention_weight_min": -94.55704498291016,
      "activations/layer8_attention_weight_max": 42.28572463989258,
      "activations/layer8_attention_weight_min": -43.2564697265625,
      "activations/layer9_attention_weight_max": 33.48998260498047,
      "activations/layer9_attention_weight_min": -37.37382125854492,
      "epoch": 16.35,
      "learning_rate": 4.4990909090909086e-05,
      "loss": 2.743,
      "step": 281450
    },
    {
      "activations/layer0_attention_weight_max": 15.458690643310547,
      "activations/layer0_attention_weight_min": -12.393963813781738,
      "activations/layer10_attention_weight_max": 33.77926254272461,
      "activations/layer10_attention_weight_min": -32.766754150390625,
      "activations/layer11_attention_weight_max": 32.41616439819336,
      "activations/layer11_attention_weight_min": -33.74501037597656,
      "activations/layer12_attention_weight_max": 25.87415885925293,
      "activations/layer12_attention_weight_min": -25.580820083618164,
      "activations/layer13_attention_weight_max": 37.0455207824707,
      "activations/layer13_attention_weight_min": -34.1650390625,
      "activations/layer14_attention_weight_max": 43.40494918823242,
      "activations/layer14_attention_weight_min": -35.93204879760742,
      "activations/layer15_attention_weight_max": 35.9283332824707,
      "activations/layer15_attention_weight_min": -32.90656280517578,
      "activations/layer16_attention_weight_max": 27.114564895629883,
      "activations/layer16_attention_weight_min": -30.800479888916016,
      "activations/layer17_attention_weight_max": 30.72224998474121,
      "activations/layer17_attention_weight_min": -24.491138458251953,
      "activations/layer18_attention_weight_max": 29.492422103881836,
      "activations/layer18_attention_weight_min": -25.196924209594727,
      "activations/layer19_attention_weight_max": 31.962915420532227,
      "activations/layer19_attention_weight_min": -32.14350509643555,
      "activations/layer1_attention_weight_max": 17.0302734375,
      "activations/layer1_attention_weight_min": -15.08719253540039,
      "activations/layer20_attention_weight_max": 27.753734588623047,
      "activations/layer20_attention_weight_min": -28.31111717224121,
      "activations/layer21_attention_weight_max": 28.435781478881836,
      "activations/layer21_attention_weight_min": -25.75303077697754,
      "activations/layer22_attention_weight_max": 47.10380172729492,
      "activations/layer22_attention_weight_min": -31.163585662841797,
      "activations/layer23_attention_weight_max": 33.55842590332031,
      "activations/layer23_attention_weight_min": -27.582202911376953,
      "activations/layer2_attention_weight_max": 31.1495361328125,
      "activations/layer2_attention_weight_min": -31.971206665039062,
      "activations/layer3_attention_weight_max": 90.36913299560547,
      "activations/layer3_attention_weight_min": -96.00847625732422,
      "activations/layer4_attention_weight_max": 104.93184661865234,
      "activations/layer4_attention_weight_min": -106.82911682128906,
      "activations/layer5_attention_weight_max": 52.180755615234375,
      "activations/layer5_attention_weight_min": -64.45366668701172,
      "activations/layer6_attention_weight_max": 45.57126235961914,
      "activations/layer6_attention_weight_min": -46.85166549682617,
      "activations/layer7_attention_weight_max": 91.71385955810547,
      "activations/layer7_attention_weight_min": -94.37884521484375,
      "activations/layer8_attention_weight_max": 43.265289306640625,
      "activations/layer8_attention_weight_min": -43.75724792480469,
      "activations/layer9_attention_weight_max": 33.74506378173828,
      "activations/layer9_attention_weight_min": -34.042205810546875,
      "epoch": 16.36,
      "learning_rate": 4.4971969696969694e-05,
      "loss": 2.7479,
      "step": 281500
    },
    {
      "activations/layer0_attention_weight_max": 15.512523651123047,
      "activations/layer0_attention_weight_min": -14.716996192932129,
      "activations/layer10_attention_weight_max": 37.529945373535156,
      "activations/layer10_attention_weight_min": -37.53562545776367,
      "activations/layer11_attention_weight_max": 37.04657745361328,
      "activations/layer11_attention_weight_min": -35.812286376953125,
      "activations/layer12_attention_weight_max": 29.893766403198242,
      "activations/layer12_attention_weight_min": -28.83514976501465,
      "activations/layer13_attention_weight_max": 43.64850997924805,
      "activations/layer13_attention_weight_min": -38.12061309814453,
      "activations/layer14_attention_weight_max": 43.93915557861328,
      "activations/layer14_attention_weight_min": -41.68135070800781,
      "activations/layer15_attention_weight_max": 40.5677604675293,
      "activations/layer15_attention_weight_min": -35.47753143310547,
      "activations/layer16_attention_weight_max": 29.91317367553711,
      "activations/layer16_attention_weight_min": -27.631616592407227,
      "activations/layer17_attention_weight_max": 30.42119026184082,
      "activations/layer17_attention_weight_min": -26.377492904663086,
      "activations/layer18_attention_weight_max": 32.50364685058594,
      "activations/layer18_attention_weight_min": -26.536823272705078,
      "activations/layer19_attention_weight_max": 35.51051330566406,
      "activations/layer19_attention_weight_min": -29.996021270751953,
      "activations/layer1_attention_weight_max": 17.26947593688965,
      "activations/layer1_attention_weight_min": -14.643580436706543,
      "activations/layer20_attention_weight_max": 27.872783660888672,
      "activations/layer20_attention_weight_min": -24.805828094482422,
      "activations/layer21_attention_weight_max": 26.417625427246094,
      "activations/layer21_attention_weight_min": -24.380565643310547,
      "activations/layer22_attention_weight_max": 41.44570541381836,
      "activations/layer22_attention_weight_min": -31.985111236572266,
      "activations/layer23_attention_weight_max": 33.17918395996094,
      "activations/layer23_attention_weight_min": -28.024795532226562,
      "activations/layer2_attention_weight_max": 32.26626968383789,
      "activations/layer2_attention_weight_min": -33.72766876220703,
      "activations/layer3_attention_weight_max": 94.94957733154297,
      "activations/layer3_attention_weight_min": -100.24397277832031,
      "activations/layer4_attention_weight_max": 111.22108459472656,
      "activations/layer4_attention_weight_min": -111.91120910644531,
      "activations/layer5_attention_weight_max": 51.30543518066406,
      "activations/layer5_attention_weight_min": -62.522056579589844,
      "activations/layer6_attention_weight_max": 45.15596389770508,
      "activations/layer6_attention_weight_min": -48.99863052368164,
      "activations/layer7_attention_weight_max": 97.6451644897461,
      "activations/layer7_attention_weight_min": -99.61691284179688,
      "activations/layer8_attention_weight_max": 44.74791717529297,
      "activations/layer8_attention_weight_min": -47.929264068603516,
      "activations/layer9_attention_weight_max": 35.5021858215332,
      "activations/layer9_attention_weight_min": -36.47806167602539,
      "epoch": 16.36,
      "learning_rate": 4.4953030303030296e-05,
      "loss": 2.7496,
      "step": 281550
    },
    {
      "activations/layer0_attention_weight_max": 15.412712097167969,
      "activations/layer0_attention_weight_min": -12.652228355407715,
      "activations/layer10_attention_weight_max": 33.463382720947266,
      "activations/layer10_attention_weight_min": -34.73147964477539,
      "activations/layer11_attention_weight_max": 35.41346740722656,
      "activations/layer11_attention_weight_min": -34.22527313232422,
      "activations/layer12_attention_weight_max": 28.509212493896484,
      "activations/layer12_attention_weight_min": -26.759031295776367,
      "activations/layer13_attention_weight_max": 40.83625793457031,
      "activations/layer13_attention_weight_min": -35.2956428527832,
      "activations/layer14_attention_weight_max": 41.39961242675781,
      "activations/layer14_attention_weight_min": -39.48204803466797,
      "activations/layer15_attention_weight_max": 38.828857421875,
      "activations/layer15_attention_weight_min": -34.89181900024414,
      "activations/layer16_attention_weight_max": 34.08708953857422,
      "activations/layer16_attention_weight_min": -27.715734481811523,
      "activations/layer17_attention_weight_max": 32.32637405395508,
      "activations/layer17_attention_weight_min": -27.073047637939453,
      "activations/layer18_attention_weight_max": 32.636627197265625,
      "activations/layer18_attention_weight_min": -26.55127716064453,
      "activations/layer19_attention_weight_max": 38.54588317871094,
      "activations/layer19_attention_weight_min": -30.958642959594727,
      "activations/layer1_attention_weight_max": 15.737787246704102,
      "activations/layer1_attention_weight_min": -15.344098091125488,
      "activations/layer20_attention_weight_max": 31.93120574951172,
      "activations/layer20_attention_weight_min": -25.61827850341797,
      "activations/layer21_attention_weight_max": 31.46324920654297,
      "activations/layer21_attention_weight_min": -24.270275115966797,
      "activations/layer22_attention_weight_max": 46.6870231628418,
      "activations/layer22_attention_weight_min": -31.12446403503418,
      "activations/layer23_attention_weight_max": 35.19199752807617,
      "activations/layer23_attention_weight_min": -27.61795425415039,
      "activations/layer2_attention_weight_max": 31.94852066040039,
      "activations/layer2_attention_weight_min": -33.07640075683594,
      "activations/layer3_attention_weight_max": 96.36236572265625,
      "activations/layer3_attention_weight_min": -97.92123413085938,
      "activations/layer4_attention_weight_max": 107.02473449707031,
      "activations/layer4_attention_weight_min": -111.09716796875,
      "activations/layer5_attention_weight_max": 53.428035736083984,
      "activations/layer5_attention_weight_min": -63.861812591552734,
      "activations/layer6_attention_weight_max": 45.4103889465332,
      "activations/layer6_attention_weight_min": -47.98735046386719,
      "activations/layer7_attention_weight_max": 90.96739959716797,
      "activations/layer7_attention_weight_min": -100.94629669189453,
      "activations/layer8_attention_weight_max": 41.92026901245117,
      "activations/layer8_attention_weight_min": -44.13492965698242,
      "activations/layer9_attention_weight_max": 34.63014602661133,
      "activations/layer9_attention_weight_min": -33.49959945678711,
      "epoch": 16.36,
      "learning_rate": 4.493409090909091e-05,
      "loss": 2.7253,
      "step": 281600
    },
    {
      "activations/layer0_attention_weight_max": 16.220369338989258,
      "activations/layer0_attention_weight_min": -12.133822441101074,
      "activations/layer10_attention_weight_max": 39.00664520263672,
      "activations/layer10_attention_weight_min": -34.98456573486328,
      "activations/layer11_attention_weight_max": 38.81148910522461,
      "activations/layer11_attention_weight_min": -35.516944885253906,
      "activations/layer12_attention_weight_max": 26.669578552246094,
      "activations/layer12_attention_weight_min": -26.97312355041504,
      "activations/layer13_attention_weight_max": 38.549095153808594,
      "activations/layer13_attention_weight_min": -36.168212890625,
      "activations/layer14_attention_weight_max": 46.50627136230469,
      "activations/layer14_attention_weight_min": -38.7013053894043,
      "activations/layer15_attention_weight_max": 39.45447540283203,
      "activations/layer15_attention_weight_min": -36.146507263183594,
      "activations/layer16_attention_weight_max": 28.312498092651367,
      "activations/layer16_attention_weight_min": -26.147979736328125,
      "activations/layer17_attention_weight_max": 33.17338562011719,
      "activations/layer17_attention_weight_min": -25.527576446533203,
      "activations/layer18_attention_weight_max": 29.62342643737793,
      "activations/layer18_attention_weight_min": -23.555349349975586,
      "activations/layer19_attention_weight_max": 32.8884391784668,
      "activations/layer19_attention_weight_min": -30.367507934570312,
      "activations/layer1_attention_weight_max": 16.212562561035156,
      "activations/layer1_attention_weight_min": -14.16838550567627,
      "activations/layer20_attention_weight_max": 32.67470932006836,
      "activations/layer20_attention_weight_min": -27.110233306884766,
      "activations/layer21_attention_weight_max": 31.54960823059082,
      "activations/layer21_attention_weight_min": -25.388320922851562,
      "activations/layer22_attention_weight_max": 38.413612365722656,
      "activations/layer22_attention_weight_min": -31.89813232421875,
      "activations/layer23_attention_weight_max": 31.855411529541016,
      "activations/layer23_attention_weight_min": -25.85262107849121,
      "activations/layer2_attention_weight_max": 32.85032653808594,
      "activations/layer2_attention_weight_min": -31.659759521484375,
      "activations/layer3_attention_weight_max": 100.31891632080078,
      "activations/layer3_attention_weight_min": -98.23717498779297,
      "activations/layer4_attention_weight_max": 108.42878723144531,
      "activations/layer4_attention_weight_min": -105.6378402709961,
      "activations/layer5_attention_weight_max": 51.944862365722656,
      "activations/layer5_attention_weight_min": -60.13710021972656,
      "activations/layer6_attention_weight_max": 46.726417541503906,
      "activations/layer6_attention_weight_min": -47.900455474853516,
      "activations/layer7_attention_weight_max": 98.49433898925781,
      "activations/layer7_attention_weight_min": -97.07449340820312,
      "activations/layer8_attention_weight_max": 44.72859573364258,
      "activations/layer8_attention_weight_min": -44.29011535644531,
      "activations/layer9_attention_weight_max": 35.18514633178711,
      "activations/layer9_attention_weight_min": -34.44099807739258,
      "epoch": 16.37,
      "learning_rate": 4.49155303030303e-05,
      "loss": 2.7425,
      "step": 281650
    },
    {
      "activations/layer0_attention_weight_max": 16.74384117126465,
      "activations/layer0_attention_weight_min": -12.55846881866455,
      "activations/layer10_attention_weight_max": 34.39895248413086,
      "activations/layer10_attention_weight_min": -33.14735794067383,
      "activations/layer11_attention_weight_max": 32.703460693359375,
      "activations/layer11_attention_weight_min": -34.77527618408203,
      "activations/layer12_attention_weight_max": 27.31112289428711,
      "activations/layer12_attention_weight_min": -25.571680068969727,
      "activations/layer13_attention_weight_max": 41.018306732177734,
      "activations/layer13_attention_weight_min": -36.40908432006836,
      "activations/layer14_attention_weight_max": 42.7272834777832,
      "activations/layer14_attention_weight_min": -37.714378356933594,
      "activations/layer15_attention_weight_max": 41.16851043701172,
      "activations/layer15_attention_weight_min": -34.258853912353516,
      "activations/layer16_attention_weight_max": 34.222747802734375,
      "activations/layer16_attention_weight_min": -28.189443588256836,
      "activations/layer17_attention_weight_max": 35.002357482910156,
      "activations/layer17_attention_weight_min": -26.673616409301758,
      "activations/layer18_attention_weight_max": 31.927005767822266,
      "activations/layer18_attention_weight_min": -24.259872436523438,
      "activations/layer19_attention_weight_max": 35.93768310546875,
      "activations/layer19_attention_weight_min": -34.731510162353516,
      "activations/layer1_attention_weight_max": 16.318405151367188,
      "activations/layer1_attention_weight_min": -15.059830665588379,
      "activations/layer20_attention_weight_max": 32.37105178833008,
      "activations/layer20_attention_weight_min": -27.252458572387695,
      "activations/layer21_attention_weight_max": 30.700254440307617,
      "activations/layer21_attention_weight_min": -25.950382232666016,
      "activations/layer22_attention_weight_max": 40.72600555419922,
      "activations/layer22_attention_weight_min": -31.40901756286621,
      "activations/layer23_attention_weight_max": 34.656150817871094,
      "activations/layer23_attention_weight_min": -24.58419418334961,
      "activations/layer2_attention_weight_max": 31.944923400878906,
      "activations/layer2_attention_weight_min": -33.122920989990234,
      "activations/layer3_attention_weight_max": 93.75860595703125,
      "activations/layer3_attention_weight_min": -95.91564178466797,
      "activations/layer4_attention_weight_max": 102.92278289794922,
      "activations/layer4_attention_weight_min": -108.2768783569336,
      "activations/layer5_attention_weight_max": 51.123878479003906,
      "activations/layer5_attention_weight_min": -62.57994079589844,
      "activations/layer6_attention_weight_max": 42.630733489990234,
      "activations/layer6_attention_weight_min": -47.342933654785156,
      "activations/layer7_attention_weight_max": 90.9399642944336,
      "activations/layer7_attention_weight_min": -95.81237030029297,
      "activations/layer8_attention_weight_max": 43.51142501831055,
      "activations/layer8_attention_weight_min": -44.781314849853516,
      "activations/layer9_attention_weight_max": 33.80727005004883,
      "activations/layer9_attention_weight_min": -33.446041107177734,
      "epoch": 16.37,
      "learning_rate": 4.48965909090909e-05,
      "loss": 2.7371,
      "step": 281700
    },
    {
      "activations/layer0_attention_weight_max": 14.975719451904297,
      "activations/layer0_attention_weight_min": -13.05879020690918,
      "activations/layer10_attention_weight_max": 33.800758361816406,
      "activations/layer10_attention_weight_min": -35.2839241027832,
      "activations/layer11_attention_weight_max": 35.747703552246094,
      "activations/layer11_attention_weight_min": -33.292057037353516,
      "activations/layer12_attention_weight_max": 28.154630661010742,
      "activations/layer12_attention_weight_min": -26.879789352416992,
      "activations/layer13_attention_weight_max": 42.257408142089844,
      "activations/layer13_attention_weight_min": -35.19373321533203,
      "activations/layer14_attention_weight_max": 42.84003448486328,
      "activations/layer14_attention_weight_min": -36.44386672973633,
      "activations/layer15_attention_weight_max": 41.987552642822266,
      "activations/layer15_attention_weight_min": -34.353389739990234,
      "activations/layer16_attention_weight_max": 30.000640869140625,
      "activations/layer16_attention_weight_min": -28.96376609802246,
      "activations/layer17_attention_weight_max": 31.248369216918945,
      "activations/layer17_attention_weight_min": -27.255321502685547,
      "activations/layer18_attention_weight_max": 31.75710678100586,
      "activations/layer18_attention_weight_min": -26.4799747467041,
      "activations/layer19_attention_weight_max": 34.74982833862305,
      "activations/layer19_attention_weight_min": -33.43213653564453,
      "activations/layer1_attention_weight_max": 16.08431625366211,
      "activations/layer1_attention_weight_min": -16.785354614257812,
      "activations/layer20_attention_weight_max": 29.18313217163086,
      "activations/layer20_attention_weight_min": -26.503026962280273,
      "activations/layer21_attention_weight_max": 28.637479782104492,
      "activations/layer21_attention_weight_min": -25.73716926574707,
      "activations/layer22_attention_weight_max": 44.21268844604492,
      "activations/layer22_attention_weight_min": -34.727699279785156,
      "activations/layer23_attention_weight_max": 35.174461364746094,
      "activations/layer23_attention_weight_min": -29.77168846130371,
      "activations/layer2_attention_weight_max": 32.801273345947266,
      "activations/layer2_attention_weight_min": -33.657806396484375,
      "activations/layer3_attention_weight_max": 94.65314483642578,
      "activations/layer3_attention_weight_min": -96.18659210205078,
      "activations/layer4_attention_weight_max": 107.21068572998047,
      "activations/layer4_attention_weight_min": -106.7596206665039,
      "activations/layer5_attention_weight_max": 49.75444793701172,
      "activations/layer5_attention_weight_min": -63.62240219116211,
      "activations/layer6_attention_weight_max": 44.24962615966797,
      "activations/layer6_attention_weight_min": -46.63472366333008,
      "activations/layer7_attention_weight_max": 92.57715606689453,
      "activations/layer7_attention_weight_min": -91.4632568359375,
      "activations/layer8_attention_weight_max": 42.99135208129883,
      "activations/layer8_attention_weight_min": -47.12638473510742,
      "activations/layer9_attention_weight_max": 33.91755294799805,
      "activations/layer9_attention_weight_min": -35.16144943237305,
      "epoch": 16.37,
      "learning_rate": 4.4878030303030294e-05,
      "loss": 2.7603,
      "step": 281750
    },
    {
      "activations/layer0_attention_weight_max": 15.419170379638672,
      "activations/layer0_attention_weight_min": -12.400729179382324,
      "activations/layer10_attention_weight_max": 33.45796203613281,
      "activations/layer10_attention_weight_min": -32.22517776489258,
      "activations/layer11_attention_weight_max": 35.011837005615234,
      "activations/layer11_attention_weight_min": -33.713932037353516,
      "activations/layer12_attention_weight_max": 25.975929260253906,
      "activations/layer12_attention_weight_min": -24.892810821533203,
      "activations/layer13_attention_weight_max": 40.13372802734375,
      "activations/layer13_attention_weight_min": -34.08578872680664,
      "activations/layer14_attention_weight_max": 44.395938873291016,
      "activations/layer14_attention_weight_min": -37.90701675415039,
      "activations/layer15_attention_weight_max": 41.288978576660156,
      "activations/layer15_attention_weight_min": -35.209373474121094,
      "activations/layer16_attention_weight_max": 31.074804306030273,
      "activations/layer16_attention_weight_min": -29.551538467407227,
      "activations/layer17_attention_weight_max": 31.191667556762695,
      "activations/layer17_attention_weight_min": -25.230819702148438,
      "activations/layer18_attention_weight_max": 31.153902053833008,
      "activations/layer18_attention_weight_min": -25.164663314819336,
      "activations/layer19_attention_weight_max": 33.04841232299805,
      "activations/layer19_attention_weight_min": -31.887670516967773,
      "activations/layer1_attention_weight_max": 17.80680274963379,
      "activations/layer1_attention_weight_min": -16.461849212646484,
      "activations/layer20_attention_weight_max": 27.692840576171875,
      "activations/layer20_attention_weight_min": -26.282392501831055,
      "activations/layer21_attention_weight_max": 28.18077278137207,
      "activations/layer21_attention_weight_min": -24.878183364868164,
      "activations/layer22_attention_weight_max": 41.52908706665039,
      "activations/layer22_attention_weight_min": -32.0801887512207,
      "activations/layer23_attention_weight_max": 31.727535247802734,
      "activations/layer23_attention_weight_min": -26.87192153930664,
      "activations/layer2_attention_weight_max": 31.72157096862793,
      "activations/layer2_attention_weight_min": -31.337453842163086,
      "activations/layer3_attention_weight_max": 95.37456512451172,
      "activations/layer3_attention_weight_min": -93.89867401123047,
      "activations/layer4_attention_weight_max": 101.93211364746094,
      "activations/layer4_attention_weight_min": -102.83070373535156,
      "activations/layer5_attention_weight_max": 48.526100158691406,
      "activations/layer5_attention_weight_min": -58.48371887207031,
      "activations/layer6_attention_weight_max": 41.14595031738281,
      "activations/layer6_attention_weight_min": -44.206092834472656,
      "activations/layer7_attention_weight_max": 89.26303100585938,
      "activations/layer7_attention_weight_min": -91.98756408691406,
      "activations/layer8_attention_weight_max": 41.78578186035156,
      "activations/layer8_attention_weight_min": -42.661556243896484,
      "activations/layer9_attention_weight_max": 32.232608795166016,
      "activations/layer9_attention_weight_min": -33.545162200927734,
      "epoch": 16.37,
      "learning_rate": 4.485909090909091e-05,
      "loss": 2.7337,
      "step": 281800
    },
    {
      "activations/layer0_attention_weight_max": 15.952893257141113,
      "activations/layer0_attention_weight_min": -12.268895149230957,
      "activations/layer10_attention_weight_max": 34.89577865600586,
      "activations/layer10_attention_weight_min": -34.69157409667969,
      "activations/layer11_attention_weight_max": 33.39619445800781,
      "activations/layer11_attention_weight_min": -35.39680480957031,
      "activations/layer12_attention_weight_max": 26.18730926513672,
      "activations/layer12_attention_weight_min": -27.693038940429688,
      "activations/layer13_attention_weight_max": 40.09840774536133,
      "activations/layer13_attention_weight_min": -36.58148193359375,
      "activations/layer14_attention_weight_max": 42.66529083251953,
      "activations/layer14_attention_weight_min": -38.41332244873047,
      "activations/layer15_attention_weight_max": 40.81502151489258,
      "activations/layer15_attention_weight_min": -35.911712646484375,
      "activations/layer16_attention_weight_max": 30.509841918945312,
      "activations/layer16_attention_weight_min": -28.358211517333984,
      "activations/layer17_attention_weight_max": 28.963077545166016,
      "activations/layer17_attention_weight_min": -26.830049514770508,
      "activations/layer18_attention_weight_max": 29.53630256652832,
      "activations/layer18_attention_weight_min": -24.099199295043945,
      "activations/layer19_attention_weight_max": 32.947444915771484,
      "activations/layer19_attention_weight_min": -30.923398971557617,
      "activations/layer1_attention_weight_max": 15.913666725158691,
      "activations/layer1_attention_weight_min": -13.973676681518555,
      "activations/layer20_attention_weight_max": 28.404247283935547,
      "activations/layer20_attention_weight_min": -24.060022354125977,
      "activations/layer21_attention_weight_max": 28.929447174072266,
      "activations/layer21_attention_weight_min": -24.315860748291016,
      "activations/layer22_attention_weight_max": 43.83994674682617,
      "activations/layer22_attention_weight_min": -31.772907257080078,
      "activations/layer23_attention_weight_max": 30.583354949951172,
      "activations/layer23_attention_weight_min": -26.95342254638672,
      "activations/layer2_attention_weight_max": 31.795852661132812,
      "activations/layer2_attention_weight_min": -31.519309997558594,
      "activations/layer3_attention_weight_max": 94.26912689208984,
      "activations/layer3_attention_weight_min": -98.80575561523438,
      "activations/layer4_attention_weight_max": 105.8095474243164,
      "activations/layer4_attention_weight_min": -112.29833221435547,
      "activations/layer5_attention_weight_max": 53.77118682861328,
      "activations/layer5_attention_weight_min": -63.3326530456543,
      "activations/layer6_attention_weight_max": 43.58652877807617,
      "activations/layer6_attention_weight_min": -49.5240364074707,
      "activations/layer7_attention_weight_max": 93.32060241699219,
      "activations/layer7_attention_weight_min": -95.8920669555664,
      "activations/layer8_attention_weight_max": 43.81899642944336,
      "activations/layer8_attention_weight_min": -43.07898712158203,
      "activations/layer9_attention_weight_max": 34.74748611450195,
      "activations/layer9_attention_weight_min": -35.470401763916016,
      "epoch": 16.38,
      "learning_rate": 4.484015151515151e-05,
      "loss": 2.7435,
      "step": 281850
    },
    {
      "activations/layer0_attention_weight_max": 15.23405933380127,
      "activations/layer0_attention_weight_min": -12.530771255493164,
      "activations/layer10_attention_weight_max": 33.15748977661133,
      "activations/layer10_attention_weight_min": -34.10646438598633,
      "activations/layer11_attention_weight_max": 33.47881317138672,
      "activations/layer11_attention_weight_min": -33.35340118408203,
      "activations/layer12_attention_weight_max": 30.816734313964844,
      "activations/layer12_attention_weight_min": -25.442304611206055,
      "activations/layer13_attention_weight_max": 39.71821212768555,
      "activations/layer13_attention_weight_min": -34.000946044921875,
      "activations/layer14_attention_weight_max": 47.64131546020508,
      "activations/layer14_attention_weight_min": -37.99000549316406,
      "activations/layer15_attention_weight_max": 39.736812591552734,
      "activations/layer15_attention_weight_min": -36.419891357421875,
      "activations/layer16_attention_weight_max": 33.178592681884766,
      "activations/layer16_attention_weight_min": -26.90538215637207,
      "activations/layer17_attention_weight_max": 33.37300109863281,
      "activations/layer17_attention_weight_min": -25.23211097717285,
      "activations/layer18_attention_weight_max": 30.37331771850586,
      "activations/layer18_attention_weight_min": -24.601842880249023,
      "activations/layer19_attention_weight_max": 33.7809944152832,
      "activations/layer19_attention_weight_min": -30.35792350769043,
      "activations/layer1_attention_weight_max": 16.457212448120117,
      "activations/layer1_attention_weight_min": -14.999814987182617,
      "activations/layer20_attention_weight_max": 27.957529067993164,
      "activations/layer20_attention_weight_min": -23.93671417236328,
      "activations/layer21_attention_weight_max": 28.079641342163086,
      "activations/layer21_attention_weight_min": -24.469430923461914,
      "activations/layer22_attention_weight_max": 46.13577651977539,
      "activations/layer22_attention_weight_min": -31.37737274169922,
      "activations/layer23_attention_weight_max": 34.75856399536133,
      "activations/layer23_attention_weight_min": -27.178937911987305,
      "activations/layer2_attention_weight_max": 31.418170928955078,
      "activations/layer2_attention_weight_min": -32.33441925048828,
      "activations/layer3_attention_weight_max": 94.72632598876953,
      "activations/layer3_attention_weight_min": -97.12591552734375,
      "activations/layer4_attention_weight_max": 106.72792053222656,
      "activations/layer4_attention_weight_min": -112.09614562988281,
      "activations/layer5_attention_weight_max": 48.683372497558594,
      "activations/layer5_attention_weight_min": -61.32878112792969,
      "activations/layer6_attention_weight_max": 42.94144821166992,
      "activations/layer6_attention_weight_min": -46.220619201660156,
      "activations/layer7_attention_weight_max": 93.73535919189453,
      "activations/layer7_attention_weight_min": -93.18814086914062,
      "activations/layer8_attention_weight_max": 41.16756057739258,
      "activations/layer8_attention_weight_min": -43.48902130126953,
      "activations/layer9_attention_weight_max": 32.94224166870117,
      "activations/layer9_attention_weight_min": -35.25602340698242,
      "epoch": 16.38,
      "learning_rate": 4.482121212121212e-05,
      "loss": 2.7469,
      "step": 281900
    },
    {
      "activations/layer0_attention_weight_max": 15.830931663513184,
      "activations/layer0_attention_weight_min": -12.402267456054688,
      "activations/layer10_attention_weight_max": 33.225257873535156,
      "activations/layer10_attention_weight_min": -33.35388946533203,
      "activations/layer11_attention_weight_max": 33.27440643310547,
      "activations/layer11_attention_weight_min": -33.22998809814453,
      "activations/layer12_attention_weight_max": 29.622482299804688,
      "activations/layer12_attention_weight_min": -26.399320602416992,
      "activations/layer13_attention_weight_max": 40.424713134765625,
      "activations/layer13_attention_weight_min": -35.396026611328125,
      "activations/layer14_attention_weight_max": 39.501861572265625,
      "activations/layer14_attention_weight_min": -37.49467468261719,
      "activations/layer15_attention_weight_max": 38.29651641845703,
      "activations/layer15_attention_weight_min": -35.44011306762695,
      "activations/layer16_attention_weight_max": 29.612491607666016,
      "activations/layer16_attention_weight_min": -28.63887596130371,
      "activations/layer17_attention_weight_max": 28.10161590576172,
      "activations/layer17_attention_weight_min": -26.230358123779297,
      "activations/layer18_attention_weight_max": 30.122201919555664,
      "activations/layer18_attention_weight_min": -25.538326263427734,
      "activations/layer19_attention_weight_max": 31.676794052124023,
      "activations/layer19_attention_weight_min": -31.176334381103516,
      "activations/layer1_attention_weight_max": 16.61339569091797,
      "activations/layer1_attention_weight_min": -15.10377025604248,
      "activations/layer20_attention_weight_max": 27.988740921020508,
      "activations/layer20_attention_weight_min": -25.995092391967773,
      "activations/layer21_attention_weight_max": 28.165189743041992,
      "activations/layer21_attention_weight_min": -25.844938278198242,
      "activations/layer22_attention_weight_max": 38.744937896728516,
      "activations/layer22_attention_weight_min": -31.831079483032227,
      "activations/layer23_attention_weight_max": 30.629507064819336,
      "activations/layer23_attention_weight_min": -26.58124351501465,
      "activations/layer2_attention_weight_max": 32.553733825683594,
      "activations/layer2_attention_weight_min": -31.92181968688965,
      "activations/layer3_attention_weight_max": 97.78206634521484,
      "activations/layer3_attention_weight_min": -96.41597747802734,
      "activations/layer4_attention_weight_max": 105.25709533691406,
      "activations/layer4_attention_weight_min": -105.2896728515625,
      "activations/layer5_attention_weight_max": 49.168453216552734,
      "activations/layer5_attention_weight_min": -62.05775451660156,
      "activations/layer6_attention_weight_max": 45.46245574951172,
      "activations/layer6_attention_weight_min": -45.50682830810547,
      "activations/layer7_attention_weight_max": 99.44841003417969,
      "activations/layer7_attention_weight_min": -93.77633666992188,
      "activations/layer8_attention_weight_max": 44.30292892456055,
      "activations/layer8_attention_weight_min": -42.97834014892578,
      "activations/layer9_attention_weight_max": 33.892425537109375,
      "activations/layer9_attention_weight_min": -35.1195068359375,
      "epoch": 16.38,
      "learning_rate": 4.480227272727272e-05,
      "loss": 2.7365,
      "step": 281950
    },
    {
      "activations/layer0_attention_weight_max": 15.785204887390137,
      "activations/layer0_attention_weight_min": -12.304420471191406,
      "activations/layer10_attention_weight_max": 30.828947067260742,
      "activations/layer10_attention_weight_min": -32.40481185913086,
      "activations/layer11_attention_weight_max": 30.759267807006836,
      "activations/layer11_attention_weight_min": -33.11635208129883,
      "activations/layer12_attention_weight_max": 25.46474838256836,
      "activations/layer12_attention_weight_min": -25.67258644104004,
      "activations/layer13_attention_weight_max": 43.84812927246094,
      "activations/layer13_attention_weight_min": -35.136478424072266,
      "activations/layer14_attention_weight_max": 44.47404479980469,
      "activations/layer14_attention_weight_min": -37.148929595947266,
      "activations/layer15_attention_weight_max": 39.01756286621094,
      "activations/layer15_attention_weight_min": -35.37005615234375,
      "activations/layer16_attention_weight_max": 31.615371704101562,
      "activations/layer16_attention_weight_min": -26.394052505493164,
      "activations/layer17_attention_weight_max": 30.911638259887695,
      "activations/layer17_attention_weight_min": -26.677106857299805,
      "activations/layer18_attention_weight_max": 28.69000244140625,
      "activations/layer18_attention_weight_min": -27.025432586669922,
      "activations/layer19_attention_weight_max": 31.063879013061523,
      "activations/layer19_attention_weight_min": -30.148517608642578,
      "activations/layer1_attention_weight_max": 16.362899780273438,
      "activations/layer1_attention_weight_min": -14.988964080810547,
      "activations/layer20_attention_weight_max": 27.36867332458496,
      "activations/layer20_attention_weight_min": -26.185611724853516,
      "activations/layer21_attention_weight_max": 25.525053024291992,
      "activations/layer21_attention_weight_min": -24.117523193359375,
      "activations/layer22_attention_weight_max": 38.4314079284668,
      "activations/layer22_attention_weight_min": -31.00747299194336,
      "activations/layer23_attention_weight_max": 28.517871856689453,
      "activations/layer23_attention_weight_min": -24.169178009033203,
      "activations/layer2_attention_weight_max": 32.309383392333984,
      "activations/layer2_attention_weight_min": -32.56843566894531,
      "activations/layer3_attention_weight_max": 96.09022521972656,
      "activations/layer3_attention_weight_min": -97.17871856689453,
      "activations/layer4_attention_weight_max": 102.99037170410156,
      "activations/layer4_attention_weight_min": -111.6240463256836,
      "activations/layer5_attention_weight_max": 49.94879913330078,
      "activations/layer5_attention_weight_min": -61.03852462768555,
      "activations/layer6_attention_weight_max": 43.24955368041992,
      "activations/layer6_attention_weight_min": -44.42876052856445,
      "activations/layer7_attention_weight_max": 91.51910400390625,
      "activations/layer7_attention_weight_min": -94.10652923583984,
      "activations/layer8_attention_weight_max": 41.499046325683594,
      "activations/layer8_attention_weight_min": -43.166446685791016,
      "activations/layer9_attention_weight_max": 31.39700698852539,
      "activations/layer9_attention_weight_min": -33.6331901550293,
      "epoch": 16.39,
      "learning_rate": 4.4783333333333335e-05,
      "loss": 2.7476,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4325,
      "eval_samples_per_second": 509.218,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4325,
      "eval_openwebtext_samples_per_second": 509.218,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9839,
      "eval_wikitext_samples_per_second": 229.851,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_lambada_loss": 2.43359375,
      "eval_lambada_ppl": 11.399776500625777,
      "eval_lambada_runtime": 9.5451,
      "eval_lambada_samples_per_second": 510.104,
      "step": 282000
    },
    {
      "activations/layer0_attention_weight_max": 16.089658737182617,
      "activations/layer0_attention_weight_min": -12.079927444458008,
      "activations/layer10_attention_weight_max": 35.50133514404297,
      "activations/layer10_attention_weight_min": -34.80165100097656,
      "activations/layer11_attention_weight_max": 33.36026382446289,
      "activations/layer11_attention_weight_min": -34.15247344970703,
      "activations/layer12_attention_weight_max": 27.097631454467773,
      "activations/layer12_attention_weight_min": -26.43253517150879,
      "activations/layer13_attention_weight_max": 41.12989044189453,
      "activations/layer13_attention_weight_min": -36.06749725341797,
      "activations/layer14_attention_weight_max": 42.990333557128906,
      "activations/layer14_attention_weight_min": -37.694374084472656,
      "activations/layer15_attention_weight_max": 38.50727844238281,
      "activations/layer15_attention_weight_min": -35.74640655517578,
      "activations/layer16_attention_weight_max": 29.090442657470703,
      "activations/layer16_attention_weight_min": -27.65547752380371,
      "activations/layer17_attention_weight_max": 29.615699768066406,
      "activations/layer17_attention_weight_min": -25.703330993652344,
      "activations/layer18_attention_weight_max": 30.502958297729492,
      "activations/layer18_attention_weight_min": -27.147615432739258,
      "activations/layer19_attention_weight_max": 31.811471939086914,
      "activations/layer19_attention_weight_min": -33.33723068237305,
      "activations/layer1_attention_weight_max": 16.6668643951416,
      "activations/layer1_attention_weight_min": -15.07690143585205,
      "activations/layer20_attention_weight_max": 27.124195098876953,
      "activations/layer20_attention_weight_min": -26.80767250061035,
      "activations/layer21_attention_weight_max": 25.609628677368164,
      "activations/layer21_attention_weight_min": -26.413774490356445,
      "activations/layer22_attention_weight_max": 37.17110824584961,
      "activations/layer22_attention_weight_min": -32.86592102050781,
      "activations/layer23_attention_weight_max": 30.152997970581055,
      "activations/layer23_attention_weight_min": -26.095443725585938,
      "activations/layer2_attention_weight_max": 32.6673698425293,
      "activations/layer2_attention_weight_min": -32.792083740234375,
      "activations/layer3_attention_weight_max": 91.58785247802734,
      "activations/layer3_attention_weight_min": -93.41143798828125,
      "activations/layer4_attention_weight_max": 104.25849914550781,
      "activations/layer4_attention_weight_min": -105.89403533935547,
      "activations/layer5_attention_weight_max": 49.390289306640625,
      "activations/layer5_attention_weight_min": -59.22512435913086,
      "activations/layer6_attention_weight_max": 42.65611267089844,
      "activations/layer6_attention_weight_min": -45.26270294189453,
      "activations/layer7_attention_weight_max": 92.04676818847656,
      "activations/layer7_attention_weight_min": -96.85801696777344,
      "activations/layer8_attention_weight_max": 43.047508239746094,
      "activations/layer8_attention_weight_min": -45.73994064331055,
      "activations/layer9_attention_weight_max": 33.236507415771484,
      "activations/layer9_attention_weight_min": -34.68170928955078,
      "epoch": 16.39,
      "learning_rate": 4.476439393939394e-05,
      "loss": 2.7371,
      "step": 282050
    },
    {
      "activations/layer0_attention_weight_max": 15.552253723144531,
      "activations/layer0_attention_weight_min": -12.530150413513184,
      "activations/layer10_attention_weight_max": 33.362876892089844,
      "activations/layer10_attention_weight_min": -34.81803512573242,
      "activations/layer11_attention_weight_max": 35.23796081542969,
      "activations/layer11_attention_weight_min": -34.58984375,
      "activations/layer12_attention_weight_max": 27.883968353271484,
      "activations/layer12_attention_weight_min": -26.11905860900879,
      "activations/layer13_attention_weight_max": 42.40028381347656,
      "activations/layer13_attention_weight_min": -34.290618896484375,
      "activations/layer14_attention_weight_max": 42.83881759643555,
      "activations/layer14_attention_weight_min": -36.03398895263672,
      "activations/layer15_attention_weight_max": 39.06003189086914,
      "activations/layer15_attention_weight_min": -33.87434768676758,
      "activations/layer16_attention_weight_max": 29.14582061767578,
      "activations/layer16_attention_weight_min": -28.035655975341797,
      "activations/layer17_attention_weight_max": 29.53526496887207,
      "activations/layer17_attention_weight_min": -26.667877197265625,
      "activations/layer18_attention_weight_max": 34.56848907470703,
      "activations/layer18_attention_weight_min": -27.686012268066406,
      "activations/layer19_attention_weight_max": 33.8124885559082,
      "activations/layer19_attention_weight_min": -31.614343643188477,
      "activations/layer1_attention_weight_max": 16.270795822143555,
      "activations/layer1_attention_weight_min": -14.566396713256836,
      "activations/layer20_attention_weight_max": 30.044145584106445,
      "activations/layer20_attention_weight_min": -23.88139533996582,
      "activations/layer21_attention_weight_max": 27.935203552246094,
      "activations/layer21_attention_weight_min": -25.081140518188477,
      "activations/layer22_attention_weight_max": 43.09190368652344,
      "activations/layer22_attention_weight_min": -31.84896469116211,
      "activations/layer23_attention_weight_max": 33.98291778564453,
      "activations/layer23_attention_weight_min": -24.796375274658203,
      "activations/layer2_attention_weight_max": 31.414844512939453,
      "activations/layer2_attention_weight_min": -30.201309204101562,
      "activations/layer3_attention_weight_max": 95.32219696044922,
      "activations/layer3_attention_weight_min": -93.24597930908203,
      "activations/layer4_attention_weight_max": 110.17400360107422,
      "activations/layer4_attention_weight_min": -104.42659759521484,
      "activations/layer5_attention_weight_max": 53.070411682128906,
      "activations/layer5_attention_weight_min": -61.848106384277344,
      "activations/layer6_attention_weight_max": 44.374778747558594,
      "activations/layer6_attention_weight_min": -45.312889099121094,
      "activations/layer7_attention_weight_max": 92.5338363647461,
      "activations/layer7_attention_weight_min": -96.76311492919922,
      "activations/layer8_attention_weight_max": 42.73187255859375,
      "activations/layer8_attention_weight_min": -42.646873474121094,
      "activations/layer9_attention_weight_max": 33.468177795410156,
      "activations/layer9_attention_weight_min": -33.73795700073242,
      "epoch": 16.39,
      "learning_rate": 4.474545454545454e-05,
      "loss": 2.7498,
      "step": 282100
    },
    {
      "activations/layer0_attention_weight_max": 15.03739070892334,
      "activations/layer0_attention_weight_min": -12.890031814575195,
      "activations/layer10_attention_weight_max": 34.184810638427734,
      "activations/layer10_attention_weight_min": -34.875709533691406,
      "activations/layer11_attention_weight_max": 32.83558654785156,
      "activations/layer11_attention_weight_min": -34.07267761230469,
      "activations/layer12_attention_weight_max": 24.701688766479492,
      "activations/layer12_attention_weight_min": -26.158798217773438,
      "activations/layer13_attention_weight_max": 38.9194450378418,
      "activations/layer13_attention_weight_min": -38.15297317504883,
      "activations/layer14_attention_weight_max": 45.19648361206055,
      "activations/layer14_attention_weight_min": -37.95209503173828,
      "activations/layer15_attention_weight_max": 37.02017593383789,
      "activations/layer15_attention_weight_min": -33.97865676879883,
      "activations/layer16_attention_weight_max": 30.839879989624023,
      "activations/layer16_attention_weight_min": -27.769371032714844,
      "activations/layer17_attention_weight_max": 32.08620071411133,
      "activations/layer17_attention_weight_min": -26.660730361938477,
      "activations/layer18_attention_weight_max": 31.6010799407959,
      "activations/layer18_attention_weight_min": -24.488088607788086,
      "activations/layer19_attention_weight_max": 34.150272369384766,
      "activations/layer19_attention_weight_min": -32.090694427490234,
      "activations/layer1_attention_weight_max": 17.37533187866211,
      "activations/layer1_attention_weight_min": -15.065406799316406,
      "activations/layer20_attention_weight_max": 29.779117584228516,
      "activations/layer20_attention_weight_min": -26.556779861450195,
      "activations/layer21_attention_weight_max": 28.845073699951172,
      "activations/layer21_attention_weight_min": -25.853374481201172,
      "activations/layer22_attention_weight_max": 40.64980697631836,
      "activations/layer22_attention_weight_min": -31.909698486328125,
      "activations/layer23_attention_weight_max": 33.32815933227539,
      "activations/layer23_attention_weight_min": -25.479228973388672,
      "activations/layer2_attention_weight_max": 32.32342529296875,
      "activations/layer2_attention_weight_min": -30.634754180908203,
      "activations/layer3_attention_weight_max": 96.08727264404297,
      "activations/layer3_attention_weight_min": -97.10350036621094,
      "activations/layer4_attention_weight_max": 104.27127838134766,
      "activations/layer4_attention_weight_min": -108.79048919677734,
      "activations/layer5_attention_weight_max": 50.6363639831543,
      "activations/layer5_attention_weight_min": -58.989723205566406,
      "activations/layer6_attention_weight_max": 43.4300651550293,
      "activations/layer6_attention_weight_min": -47.24143600463867,
      "activations/layer7_attention_weight_max": 89.15679168701172,
      "activations/layer7_attention_weight_min": -91.5146713256836,
      "activations/layer8_attention_weight_max": 41.10747528076172,
      "activations/layer8_attention_weight_min": -42.79501724243164,
      "activations/layer9_attention_weight_max": 32.39132308959961,
      "activations/layer9_attention_weight_min": -33.35786437988281,
      "epoch": 16.39,
      "learning_rate": 4.472651515151515e-05,
      "loss": 2.7444,
      "step": 282150
    },
    {
      "activations/layer0_attention_weight_max": 15.921133041381836,
      "activations/layer0_attention_weight_min": -12.864964485168457,
      "activations/layer10_attention_weight_max": 38.337745666503906,
      "activations/layer10_attention_weight_min": -36.47211837768555,
      "activations/layer11_attention_weight_max": 35.00102615356445,
      "activations/layer11_attention_weight_min": -36.15628433227539,
      "activations/layer12_attention_weight_max": 27.436479568481445,
      "activations/layer12_attention_weight_min": -28.998865127563477,
      "activations/layer13_attention_weight_max": 44.707420349121094,
      "activations/layer13_attention_weight_min": -40.22536087036133,
      "activations/layer14_attention_weight_max": 47.157432556152344,
      "activations/layer14_attention_weight_min": -45.57005310058594,
      "activations/layer15_attention_weight_max": 47.127647399902344,
      "activations/layer15_attention_weight_min": -39.2905158996582,
      "activations/layer16_attention_weight_max": 33.33366394042969,
      "activations/layer16_attention_weight_min": -30.909143447875977,
      "activations/layer17_attention_weight_max": 30.2807559967041,
      "activations/layer17_attention_weight_min": -27.7335262298584,
      "activations/layer18_attention_weight_max": 34.0661506652832,
      "activations/layer18_attention_weight_min": -25.170949935913086,
      "activations/layer19_attention_weight_max": 35.53683090209961,
      "activations/layer19_attention_weight_min": -32.33442306518555,
      "activations/layer1_attention_weight_max": 16.384462356567383,
      "activations/layer1_attention_weight_min": -14.767454147338867,
      "activations/layer20_attention_weight_max": 28.813751220703125,
      "activations/layer20_attention_weight_min": -26.888797760009766,
      "activations/layer21_attention_weight_max": 29.304813385009766,
      "activations/layer21_attention_weight_min": -26.146514892578125,
      "activations/layer22_attention_weight_max": 52.2703971862793,
      "activations/layer22_attention_weight_min": -32.8272705078125,
      "activations/layer23_attention_weight_max": 36.46495819091797,
      "activations/layer23_attention_weight_min": -24.92414665222168,
      "activations/layer2_attention_weight_max": 33.124839782714844,
      "activations/layer2_attention_weight_min": -32.895442962646484,
      "activations/layer3_attention_weight_max": 96.55806732177734,
      "activations/layer3_attention_weight_min": -97.95963287353516,
      "activations/layer4_attention_weight_max": 105.2459716796875,
      "activations/layer4_attention_weight_min": -106.25084686279297,
      "activations/layer5_attention_weight_max": 51.54548263549805,
      "activations/layer5_attention_weight_min": -61.67394256591797,
      "activations/layer6_attention_weight_max": 44.56087112426758,
      "activations/layer6_attention_weight_min": -46.848548889160156,
      "activations/layer7_attention_weight_max": 95.88887023925781,
      "activations/layer7_attention_weight_min": -92.62955474853516,
      "activations/layer8_attention_weight_max": 41.76026916503906,
      "activations/layer8_attention_weight_min": -44.16105270385742,
      "activations/layer9_attention_weight_max": 35.38249588012695,
      "activations/layer9_attention_weight_min": -35.54896926879883,
      "epoch": 16.4,
      "learning_rate": 4.4707575757575755e-05,
      "loss": 2.7468,
      "step": 282200
    },
    {
      "activations/layer0_attention_weight_max": 16.146486282348633,
      "activations/layer0_attention_weight_min": -12.35358715057373,
      "activations/layer10_attention_weight_max": 32.894142150878906,
      "activations/layer10_attention_weight_min": -34.712154388427734,
      "activations/layer11_attention_weight_max": 33.369972229003906,
      "activations/layer11_attention_weight_min": -34.264442443847656,
      "activations/layer12_attention_weight_max": 26.843441009521484,
      "activations/layer12_attention_weight_min": -26.937761306762695,
      "activations/layer13_attention_weight_max": 41.17501449584961,
      "activations/layer13_attention_weight_min": -37.97779083251953,
      "activations/layer14_attention_weight_max": 45.2202262878418,
      "activations/layer14_attention_weight_min": -39.18334197998047,
      "activations/layer15_attention_weight_max": 38.54362487792969,
      "activations/layer15_attention_weight_min": -35.426517486572266,
      "activations/layer16_attention_weight_max": 31.20090675354004,
      "activations/layer16_attention_weight_min": -28.369239807128906,
      "activations/layer17_attention_weight_max": 33.72036361694336,
      "activations/layer17_attention_weight_min": -25.396087646484375,
      "activations/layer18_attention_weight_max": 31.192550659179688,
      "activations/layer18_attention_weight_min": -23.957626342773438,
      "activations/layer19_attention_weight_max": 35.15413284301758,
      "activations/layer19_attention_weight_min": -31.84677505493164,
      "activations/layer1_attention_weight_max": 16.202011108398438,
      "activations/layer1_attention_weight_min": -14.05246639251709,
      "activations/layer20_attention_weight_max": 30.548229217529297,
      "activations/layer20_attention_weight_min": -26.533628463745117,
      "activations/layer21_attention_weight_max": 31.04464340209961,
      "activations/layer21_attention_weight_min": -25.562931060791016,
      "activations/layer22_attention_weight_max": 44.8581428527832,
      "activations/layer22_attention_weight_min": -31.80116844177246,
      "activations/layer23_attention_weight_max": 32.050758361816406,
      "activations/layer23_attention_weight_min": -26.633588790893555,
      "activations/layer2_attention_weight_max": 33.39573287963867,
      "activations/layer2_attention_weight_min": -31.54507827758789,
      "activations/layer3_attention_weight_max": 94.96782684326172,
      "activations/layer3_attention_weight_min": -95.62666320800781,
      "activations/layer4_attention_weight_max": 104.20588684082031,
      "activations/layer4_attention_weight_min": -110.49788665771484,
      "activations/layer5_attention_weight_max": 50.68334197998047,
      "activations/layer5_attention_weight_min": -62.96628952026367,
      "activations/layer6_attention_weight_max": 43.38683319091797,
      "activations/layer6_attention_weight_min": -45.13420486450195,
      "activations/layer7_attention_weight_max": 94.47862243652344,
      "activations/layer7_attention_weight_min": -92.95154571533203,
      "activations/layer8_attention_weight_max": 41.00859451293945,
      "activations/layer8_attention_weight_min": -44.267662048339844,
      "activations/layer9_attention_weight_max": 34.81370162963867,
      "activations/layer9_attention_weight_min": -32.89338684082031,
      "epoch": 16.4,
      "learning_rate": 4.4688636363636356e-05,
      "loss": 2.7383,
      "step": 282250
    },
    {
      "activations/layer0_attention_weight_max": 16.088441848754883,
      "activations/layer0_attention_weight_min": -12.477851867675781,
      "activations/layer10_attention_weight_max": 32.23294448852539,
      "activations/layer10_attention_weight_min": -33.958587646484375,
      "activations/layer11_attention_weight_max": 33.45376968383789,
      "activations/layer11_attention_weight_min": -33.973777770996094,
      "activations/layer12_attention_weight_max": 26.603879928588867,
      "activations/layer12_attention_weight_min": -26.58558464050293,
      "activations/layer13_attention_weight_max": 38.16486740112305,
      "activations/layer13_attention_weight_min": -34.81922149658203,
      "activations/layer14_attention_weight_max": 43.076148986816406,
      "activations/layer14_attention_weight_min": -37.32723617553711,
      "activations/layer15_attention_weight_max": 40.6053352355957,
      "activations/layer15_attention_weight_min": -36.080360412597656,
      "activations/layer16_attention_weight_max": 31.12642478942871,
      "activations/layer16_attention_weight_min": -29.4669189453125,
      "activations/layer17_attention_weight_max": 30.22522735595703,
      "activations/layer17_attention_weight_min": -26.619577407836914,
      "activations/layer18_attention_weight_max": 31.9163818359375,
      "activations/layer18_attention_weight_min": -26.78192710876465,
      "activations/layer19_attention_weight_max": 34.919105529785156,
      "activations/layer19_attention_weight_min": -33.158668518066406,
      "activations/layer1_attention_weight_max": 15.674479484558105,
      "activations/layer1_attention_weight_min": -14.577129364013672,
      "activations/layer20_attention_weight_max": 30.68013572692871,
      "activations/layer20_attention_weight_min": -25.486534118652344,
      "activations/layer21_attention_weight_max": 28.294485092163086,
      "activations/layer21_attention_weight_min": -23.986242294311523,
      "activations/layer22_attention_weight_max": 47.7060432434082,
      "activations/layer22_attention_weight_min": -31.51662254333496,
      "activations/layer23_attention_weight_max": 35.77851104736328,
      "activations/layer23_attention_weight_min": -27.689552307128906,
      "activations/layer2_attention_weight_max": 31.811176300048828,
      "activations/layer2_attention_weight_min": -32.17132568359375,
      "activations/layer3_attention_weight_max": 91.32975006103516,
      "activations/layer3_attention_weight_min": -95.71224212646484,
      "activations/layer4_attention_weight_max": 108.26985168457031,
      "activations/layer4_attention_weight_min": -106.0657958984375,
      "activations/layer5_attention_weight_max": 51.934051513671875,
      "activations/layer5_attention_weight_min": -64.23757934570312,
      "activations/layer6_attention_weight_max": 42.653377532958984,
      "activations/layer6_attention_weight_min": -46.08171844482422,
      "activations/layer7_attention_weight_max": 91.04153442382812,
      "activations/layer7_attention_weight_min": -91.75794982910156,
      "activations/layer8_attention_weight_max": 43.009315490722656,
      "activations/layer8_attention_weight_min": -43.5711555480957,
      "activations/layer9_attention_weight_max": 31.918832778930664,
      "activations/layer9_attention_weight_min": -34.93236541748047,
      "epoch": 16.4,
      "learning_rate": 4.4669696969696965e-05,
      "loss": 2.7403,
      "step": 282300
    },
    {
      "activations/layer0_attention_weight_max": 16.42667579650879,
      "activations/layer0_attention_weight_min": -12.8696928024292,
      "activations/layer10_attention_weight_max": 32.17341232299805,
      "activations/layer10_attention_weight_min": -33.28569412231445,
      "activations/layer11_attention_weight_max": 31.115535736083984,
      "activations/layer11_attention_weight_min": -33.35758590698242,
      "activations/layer12_attention_weight_max": 23.94293212890625,
      "activations/layer12_attention_weight_min": -26.923389434814453,
      "activations/layer13_attention_weight_max": 36.562294006347656,
      "activations/layer13_attention_weight_min": -37.51210403442383,
      "activations/layer14_attention_weight_max": 38.888465881347656,
      "activations/layer14_attention_weight_min": -40.04557418823242,
      "activations/layer15_attention_weight_max": 35.80510330200195,
      "activations/layer15_attention_weight_min": -34.39895248413086,
      "activations/layer16_attention_weight_max": 26.941015243530273,
      "activations/layer16_attention_weight_min": -27.399551391601562,
      "activations/layer17_attention_weight_max": 27.46580696105957,
      "activations/layer17_attention_weight_min": -26.07982635498047,
      "activations/layer18_attention_weight_max": 29.60003662109375,
      "activations/layer18_attention_weight_min": -25.38441276550293,
      "activations/layer19_attention_weight_max": 30.336105346679688,
      "activations/layer19_attention_weight_min": -32.59202575683594,
      "activations/layer1_attention_weight_max": 18.719940185546875,
      "activations/layer1_attention_weight_min": -15.20315933227539,
      "activations/layer20_attention_weight_max": 26.2553768157959,
      "activations/layer20_attention_weight_min": -26.22929573059082,
      "activations/layer21_attention_weight_max": 24.514556884765625,
      "activations/layer21_attention_weight_min": -24.973358154296875,
      "activations/layer22_attention_weight_max": 43.83131790161133,
      "activations/layer22_attention_weight_min": -31.239994049072266,
      "activations/layer23_attention_weight_max": 31.04096794128418,
      "activations/layer23_attention_weight_min": -27.089950561523438,
      "activations/layer2_attention_weight_max": 30.668659210205078,
      "activations/layer2_attention_weight_min": -30.166234970092773,
      "activations/layer3_attention_weight_max": 87.05384826660156,
      "activations/layer3_attention_weight_min": -89.0632553100586,
      "activations/layer4_attention_weight_max": 96.9572525024414,
      "activations/layer4_attention_weight_min": -106.52300262451172,
      "activations/layer5_attention_weight_max": 46.868492126464844,
      "activations/layer5_attention_weight_min": -58.249568939208984,
      "activations/layer6_attention_weight_max": 41.291748046875,
      "activations/layer6_attention_weight_min": -44.91652297973633,
      "activations/layer7_attention_weight_max": 88.51492309570312,
      "activations/layer7_attention_weight_min": -91.57962036132812,
      "activations/layer8_attention_weight_max": 38.33944320678711,
      "activations/layer8_attention_weight_min": -42.28319549560547,
      "activations/layer9_attention_weight_max": 29.899860382080078,
      "activations/layer9_attention_weight_min": -32.01445007324219,
      "epoch": 16.41,
      "learning_rate": 4.465075757575757e-05,
      "loss": 2.7613,
      "step": 282350
    },
    {
      "activations/layer0_attention_weight_max": 16.172286987304688,
      "activations/layer0_attention_weight_min": -12.294182777404785,
      "activations/layer10_attention_weight_max": 32.16778564453125,
      "activations/layer10_attention_weight_min": -33.9515380859375,
      "activations/layer11_attention_weight_max": 34.288822174072266,
      "activations/layer11_attention_weight_min": -35.51395034790039,
      "activations/layer12_attention_weight_max": 25.458885192871094,
      "activations/layer12_attention_weight_min": -27.07685089111328,
      "activations/layer13_attention_weight_max": 39.04503631591797,
      "activations/layer13_attention_weight_min": -37.452781677246094,
      "activations/layer14_attention_weight_max": 43.66722106933594,
      "activations/layer14_attention_weight_min": -38.219818115234375,
      "activations/layer15_attention_weight_max": 40.55884552001953,
      "activations/layer15_attention_weight_min": -37.55393981933594,
      "activations/layer16_attention_weight_max": 35.57557678222656,
      "activations/layer16_attention_weight_min": -29.67428970336914,
      "activations/layer17_attention_weight_max": 32.77299880981445,
      "activations/layer17_attention_weight_min": -27.52141571044922,
      "activations/layer18_attention_weight_max": 35.05265808105469,
      "activations/layer18_attention_weight_min": -26.881362915039062,
      "activations/layer19_attention_weight_max": 35.2790641784668,
      "activations/layer19_attention_weight_min": -33.122413635253906,
      "activations/layer1_attention_weight_max": 17.50141143798828,
      "activations/layer1_attention_weight_min": -17.057340621948242,
      "activations/layer20_attention_weight_max": 33.385860443115234,
      "activations/layer20_attention_weight_min": -26.777692794799805,
      "activations/layer21_attention_weight_max": 32.41936111450195,
      "activations/layer21_attention_weight_min": -25.570024490356445,
      "activations/layer22_attention_weight_max": 45.01692581176758,
      "activations/layer22_attention_weight_min": -32.64316177368164,
      "activations/layer23_attention_weight_max": 34.278133392333984,
      "activations/layer23_attention_weight_min": -25.796497344970703,
      "activations/layer2_attention_weight_max": 30.71521759033203,
      "activations/layer2_attention_weight_min": -30.757871627807617,
      "activations/layer3_attention_weight_max": 93.14884948730469,
      "activations/layer3_attention_weight_min": -95.72663879394531,
      "activations/layer4_attention_weight_max": 100.33534240722656,
      "activations/layer4_attention_weight_min": -106.4930191040039,
      "activations/layer5_attention_weight_max": 50.404415130615234,
      "activations/layer5_attention_weight_min": -62.013404846191406,
      "activations/layer6_attention_weight_max": 44.06330490112305,
      "activations/layer6_attention_weight_min": -45.44151306152344,
      "activations/layer7_attention_weight_max": 93.2421875,
      "activations/layer7_attention_weight_min": -95.69074249267578,
      "activations/layer8_attention_weight_max": 41.33496856689453,
      "activations/layer8_attention_weight_min": -44.196189880371094,
      "activations/layer9_attention_weight_max": 32.32404708862305,
      "activations/layer9_attention_weight_min": -34.74120330810547,
      "epoch": 16.41,
      "learning_rate": 4.463181818181818e-05,
      "loss": 2.7376,
      "step": 282400
    },
    {
      "activations/layer0_attention_weight_max": 14.785157203674316,
      "activations/layer0_attention_weight_min": -12.338985443115234,
      "activations/layer10_attention_weight_max": 31.90902328491211,
      "activations/layer10_attention_weight_min": -33.0684814453125,
      "activations/layer11_attention_weight_max": 32.48782730102539,
      "activations/layer11_attention_weight_min": -33.34143829345703,
      "activations/layer12_attention_weight_max": 25.77924156188965,
      "activations/layer12_attention_weight_min": -26.434059143066406,
      "activations/layer13_attention_weight_max": 39.88204574584961,
      "activations/layer13_attention_weight_min": -36.246986389160156,
      "activations/layer14_attention_weight_max": 42.15616989135742,
      "activations/layer14_attention_weight_min": -37.984092712402344,
      "activations/layer15_attention_weight_max": 38.669036865234375,
      "activations/layer15_attention_weight_min": -34.642295837402344,
      "activations/layer16_attention_weight_max": 30.978540420532227,
      "activations/layer16_attention_weight_min": -28.10557746887207,
      "activations/layer17_attention_weight_max": 29.42392349243164,
      "activations/layer17_attention_weight_min": -27.485605239868164,
      "activations/layer18_attention_weight_max": 30.872793197631836,
      "activations/layer18_attention_weight_min": -25.443458557128906,
      "activations/layer19_attention_weight_max": 32.477378845214844,
      "activations/layer19_attention_weight_min": -31.475101470947266,
      "activations/layer1_attention_weight_max": 15.953888893127441,
      "activations/layer1_attention_weight_min": -15.46961784362793,
      "activations/layer20_attention_weight_max": 27.01308250427246,
      "activations/layer20_attention_weight_min": -25.551984786987305,
      "activations/layer21_attention_weight_max": 26.639873504638672,
      "activations/layer21_attention_weight_min": -26.11332130432129,
      "activations/layer22_attention_weight_max": 41.91242218017578,
      "activations/layer22_attention_weight_min": -34.09999084472656,
      "activations/layer23_attention_weight_max": 32.47715759277344,
      "activations/layer23_attention_weight_min": -27.094562530517578,
      "activations/layer2_attention_weight_max": 31.333248138427734,
      "activations/layer2_attention_weight_min": -31.87224578857422,
      "activations/layer3_attention_weight_max": 102.71723175048828,
      "activations/layer3_attention_weight_min": -103.27574157714844,
      "activations/layer4_attention_weight_max": 111.14668273925781,
      "activations/layer4_attention_weight_min": -108.86425018310547,
      "activations/layer5_attention_weight_max": 52.13862609863281,
      "activations/layer5_attention_weight_min": -60.838157653808594,
      "activations/layer6_attention_weight_max": 44.34402847290039,
      "activations/layer6_attention_weight_min": -46.433082580566406,
      "activations/layer7_attention_weight_max": 94.49617767333984,
      "activations/layer7_attention_weight_min": -98.30836486816406,
      "activations/layer8_attention_weight_max": 40.132110595703125,
      "activations/layer8_attention_weight_min": -44.02229309082031,
      "activations/layer9_attention_weight_max": 32.411888122558594,
      "activations/layer9_attention_weight_min": -33.02085876464844,
      "epoch": 16.41,
      "learning_rate": 4.461287878787878e-05,
      "loss": 2.7566,
      "step": 282450
    },
    {
      "activations/layer0_attention_weight_max": 16.39119529724121,
      "activations/layer0_attention_weight_min": -12.14472770690918,
      "activations/layer10_attention_weight_max": 34.582523345947266,
      "activations/layer10_attention_weight_min": -34.320892333984375,
      "activations/layer11_attention_weight_max": 36.240234375,
      "activations/layer11_attention_weight_min": -35.76294708251953,
      "activations/layer12_attention_weight_max": 27.97638702392578,
      "activations/layer12_attention_weight_min": -27.539522171020508,
      "activations/layer13_attention_weight_max": 41.39991760253906,
      "activations/layer13_attention_weight_min": -37.36277389526367,
      "activations/layer14_attention_weight_max": 40.708290100097656,
      "activations/layer14_attention_weight_min": -37.433162689208984,
      "activations/layer15_attention_weight_max": 35.84612274169922,
      "activations/layer15_attention_weight_min": -34.998634338378906,
      "activations/layer16_attention_weight_max": 29.29393768310547,
      "activations/layer16_attention_weight_min": -27.64737319946289,
      "activations/layer17_attention_weight_max": 27.967248916625977,
      "activations/layer17_attention_weight_min": -26.40569305419922,
      "activations/layer18_attention_weight_max": 34.79941940307617,
      "activations/layer18_attention_weight_min": -28.284610748291016,
      "activations/layer19_attention_weight_max": 34.642086029052734,
      "activations/layer19_attention_weight_min": -31.90793800354004,
      "activations/layer1_attention_weight_max": 17.319345474243164,
      "activations/layer1_attention_weight_min": -16.4390869140625,
      "activations/layer20_attention_weight_max": 29.085803985595703,
      "activations/layer20_attention_weight_min": -26.268754959106445,
      "activations/layer21_attention_weight_max": 28.286691665649414,
      "activations/layer21_attention_weight_min": -25.105484008789062,
      "activations/layer22_attention_weight_max": 44.616886138916016,
      "activations/layer22_attention_weight_min": -31.95944595336914,
      "activations/layer23_attention_weight_max": 33.120574951171875,
      "activations/layer23_attention_weight_min": -25.693317413330078,
      "activations/layer2_attention_weight_max": 31.67650604248047,
      "activations/layer2_attention_weight_min": -32.17826461791992,
      "activations/layer3_attention_weight_max": 96.64282989501953,
      "activations/layer3_attention_weight_min": -96.47563934326172,
      "activations/layer4_attention_weight_max": 105.072509765625,
      "activations/layer4_attention_weight_min": -106.45751190185547,
      "activations/layer5_attention_weight_max": 49.348289489746094,
      "activations/layer5_attention_weight_min": -58.393795013427734,
      "activations/layer6_attention_weight_max": 44.537532806396484,
      "activations/layer6_attention_weight_min": -46.63357162475586,
      "activations/layer7_attention_weight_max": 88.39828491210938,
      "activations/layer7_attention_weight_min": -93.7579345703125,
      "activations/layer8_attention_weight_max": 39.24828338623047,
      "activations/layer8_attention_weight_min": -42.9207763671875,
      "activations/layer9_attention_weight_max": 32.5697135925293,
      "activations/layer9_attention_weight_min": -34.74166488647461,
      "epoch": 16.41,
      "learning_rate": 4.4593939393939384e-05,
      "loss": 2.7471,
      "step": 282500
    },
    {
      "activations/layer0_attention_weight_max": 16.026762008666992,
      "activations/layer0_attention_weight_min": -12.460311889648438,
      "activations/layer10_attention_weight_max": 30.054582595825195,
      "activations/layer10_attention_weight_min": -33.60459899902344,
      "activations/layer11_attention_weight_max": 31.2242431640625,
      "activations/layer11_attention_weight_min": -32.22572326660156,
      "activations/layer12_attention_weight_max": 24.41038703918457,
      "activations/layer12_attention_weight_min": -25.994779586791992,
      "activations/layer13_attention_weight_max": 38.50309371948242,
      "activations/layer13_attention_weight_min": -34.36292266845703,
      "activations/layer14_attention_weight_max": 38.99127960205078,
      "activations/layer14_attention_weight_min": -37.35127258300781,
      "activations/layer15_attention_weight_max": 37.34159469604492,
      "activations/layer15_attention_weight_min": -35.19484329223633,
      "activations/layer16_attention_weight_max": 27.469453811645508,
      "activations/layer16_attention_weight_min": -29.32770538330078,
      "activations/layer17_attention_weight_max": 27.110626220703125,
      "activations/layer17_attention_weight_min": -25.4614200592041,
      "activations/layer18_attention_weight_max": 29.776145935058594,
      "activations/layer18_attention_weight_min": -26.775224685668945,
      "activations/layer19_attention_weight_max": 32.5145378112793,
      "activations/layer19_attention_weight_min": -31.380258560180664,
      "activations/layer1_attention_weight_max": 16.100261688232422,
      "activations/layer1_attention_weight_min": -14.775712966918945,
      "activations/layer20_attention_weight_max": 28.3692626953125,
      "activations/layer20_attention_weight_min": -26.328462600708008,
      "activations/layer21_attention_weight_max": 28.139921188354492,
      "activations/layer21_attention_weight_min": -25.338674545288086,
      "activations/layer22_attention_weight_max": 41.3868293762207,
      "activations/layer22_attention_weight_min": -31.769582748413086,
      "activations/layer23_attention_weight_max": 29.943878173828125,
      "activations/layer23_attention_weight_min": -25.32379722595215,
      "activations/layer2_attention_weight_max": 31.08564567565918,
      "activations/layer2_attention_weight_min": -31.84004783630371,
      "activations/layer3_attention_weight_max": 96.04206848144531,
      "activations/layer3_attention_weight_min": -92.84590148925781,
      "activations/layer4_attention_weight_max": 103.77214813232422,
      "activations/layer4_attention_weight_min": -107.23005676269531,
      "activations/layer5_attention_weight_max": 49.08592224121094,
      "activations/layer5_attention_weight_min": -60.99687957763672,
      "activations/layer6_attention_weight_max": 42.156890869140625,
      "activations/layer6_attention_weight_min": -43.555259704589844,
      "activations/layer7_attention_weight_max": 84.71288299560547,
      "activations/layer7_attention_weight_min": -91.07167053222656,
      "activations/layer8_attention_weight_max": 41.6536979675293,
      "activations/layer8_attention_weight_min": -43.04726791381836,
      "activations/layer9_attention_weight_max": 29.615957260131836,
      "activations/layer9_attention_weight_min": -33.36206817626953,
      "epoch": 16.42,
      "learning_rate": 4.4575e-05,
      "loss": 2.743,
      "step": 282550
    },
    {
      "activations/layer0_attention_weight_max": 16.06520652770996,
      "activations/layer0_attention_weight_min": -14.325833320617676,
      "activations/layer10_attention_weight_max": 35.059635162353516,
      "activations/layer10_attention_weight_min": -32.85615921020508,
      "activations/layer11_attention_weight_max": 31.691579818725586,
      "activations/layer11_attention_weight_min": -32.331146240234375,
      "activations/layer12_attention_weight_max": 27.625350952148438,
      "activations/layer12_attention_weight_min": -26.636865615844727,
      "activations/layer13_attention_weight_max": 40.36469268798828,
      "activations/layer13_attention_weight_min": -36.893272399902344,
      "activations/layer14_attention_weight_max": 44.53428268432617,
      "activations/layer14_attention_weight_min": -38.71364212036133,
      "activations/layer15_attention_weight_max": 37.759151458740234,
      "activations/layer15_attention_weight_min": -34.40334701538086,
      "activations/layer16_attention_weight_max": 31.01658821105957,
      "activations/layer16_attention_weight_min": -27.650487899780273,
      "activations/layer17_attention_weight_max": 29.696266174316406,
      "activations/layer17_attention_weight_min": -24.441425323486328,
      "activations/layer18_attention_weight_max": 31.572269439697266,
      "activations/layer18_attention_weight_min": -25.949766159057617,
      "activations/layer19_attention_weight_max": 31.408538818359375,
      "activations/layer19_attention_weight_min": -31.55841064453125,
      "activations/layer1_attention_weight_max": 15.642949104309082,
      "activations/layer1_attention_weight_min": -14.54723834991455,
      "activations/layer20_attention_weight_max": 28.929231643676758,
      "activations/layer20_attention_weight_min": -25.45882797241211,
      "activations/layer21_attention_weight_max": 29.32805633544922,
      "activations/layer21_attention_weight_min": -26.044042587280273,
      "activations/layer22_attention_weight_max": 39.752235412597656,
      "activations/layer22_attention_weight_min": -34.42979049682617,
      "activations/layer23_attention_weight_max": 29.338104248046875,
      "activations/layer23_attention_weight_min": -27.305273056030273,
      "activations/layer2_attention_weight_max": 32.29290008544922,
      "activations/layer2_attention_weight_min": -32.043235778808594,
      "activations/layer3_attention_weight_max": 94.47611236572266,
      "activations/layer3_attention_weight_min": -97.40518951416016,
      "activations/layer4_attention_weight_max": 107.53539276123047,
      "activations/layer4_attention_weight_min": -107.78034973144531,
      "activations/layer5_attention_weight_max": 47.928932189941406,
      "activations/layer5_attention_weight_min": -61.03509521484375,
      "activations/layer6_attention_weight_max": 43.095699310302734,
      "activations/layer6_attention_weight_min": -46.50596618652344,
      "activations/layer7_attention_weight_max": 88.45565795898438,
      "activations/layer7_attention_weight_min": -93.4944076538086,
      "activations/layer8_attention_weight_max": 43.34015655517578,
      "activations/layer8_attention_weight_min": -43.31180191040039,
      "activations/layer9_attention_weight_max": 32.5874137878418,
      "activations/layer9_attention_weight_min": -33.99295425415039,
      "epoch": 16.42,
      "learning_rate": 4.455643939393939e-05,
      "loss": 2.758,
      "step": 282600
    },
    {
      "activations/layer0_attention_weight_max": 16.306970596313477,
      "activations/layer0_attention_weight_min": -12.940275192260742,
      "activations/layer10_attention_weight_max": 30.672903060913086,
      "activations/layer10_attention_weight_min": -33.505584716796875,
      "activations/layer11_attention_weight_max": 31.715808868408203,
      "activations/layer11_attention_weight_min": -33.51622772216797,
      "activations/layer12_attention_weight_max": 25.27511215209961,
      "activations/layer12_attention_weight_min": -26.2868595123291,
      "activations/layer13_attention_weight_max": 38.81081008911133,
      "activations/layer13_attention_weight_min": -35.37400817871094,
      "activations/layer14_attention_weight_max": 40.258399963378906,
      "activations/layer14_attention_weight_min": -36.23830032348633,
      "activations/layer15_attention_weight_max": 37.09555435180664,
      "activations/layer15_attention_weight_min": -33.13591384887695,
      "activations/layer16_attention_weight_max": 29.854900360107422,
      "activations/layer16_attention_weight_min": -29.889141082763672,
      "activations/layer17_attention_weight_max": 31.607421875,
      "activations/layer17_attention_weight_min": -27.56597328186035,
      "activations/layer18_attention_weight_max": 33.87185287475586,
      "activations/layer18_attention_weight_min": -24.542236328125,
      "activations/layer19_attention_weight_max": 35.30733108520508,
      "activations/layer19_attention_weight_min": -31.523862838745117,
      "activations/layer1_attention_weight_max": 17.996566772460938,
      "activations/layer1_attention_weight_min": -16.206941604614258,
      "activations/layer20_attention_weight_max": 28.84450340270996,
      "activations/layer20_attention_weight_min": -25.937084197998047,
      "activations/layer21_attention_weight_max": 28.27617645263672,
      "activations/layer21_attention_weight_min": -26.750469207763672,
      "activations/layer22_attention_weight_max": 41.92605972290039,
      "activations/layer22_attention_weight_min": -33.47411346435547,
      "activations/layer23_attention_weight_max": 32.958412170410156,
      "activations/layer23_attention_weight_min": -29.181011199951172,
      "activations/layer2_attention_weight_max": 30.244834899902344,
      "activations/layer2_attention_weight_min": -30.994428634643555,
      "activations/layer3_attention_weight_max": 94.94951629638672,
      "activations/layer3_attention_weight_min": -96.80181884765625,
      "activations/layer4_attention_weight_max": 106.14239501953125,
      "activations/layer4_attention_weight_min": -110.46854400634766,
      "activations/layer5_attention_weight_max": 50.106536865234375,
      "activations/layer5_attention_weight_min": -64.71894073486328,
      "activations/layer6_attention_weight_max": 41.49231719970703,
      "activations/layer6_attention_weight_min": -46.00300979614258,
      "activations/layer7_attention_weight_max": 87.25870513916016,
      "activations/layer7_attention_weight_min": -88.4425048828125,
      "activations/layer8_attention_weight_max": 39.334651947021484,
      "activations/layer8_attention_weight_min": -41.083248138427734,
      "activations/layer9_attention_weight_max": 30.576887130737305,
      "activations/layer9_attention_weight_min": -31.535104751586914,
      "epoch": 16.42,
      "learning_rate": 4.45375e-05,
      "loss": 2.7448,
      "step": 282650
    },
    {
      "activations/layer0_attention_weight_max": 16.435649871826172,
      "activations/layer0_attention_weight_min": -11.793218612670898,
      "activations/layer10_attention_weight_max": 33.16071319580078,
      "activations/layer10_attention_weight_min": -32.217132568359375,
      "activations/layer11_attention_weight_max": 33.81779479980469,
      "activations/layer11_attention_weight_min": -32.36500549316406,
      "activations/layer12_attention_weight_max": 27.950464248657227,
      "activations/layer12_attention_weight_min": -26.858983993530273,
      "activations/layer13_attention_weight_max": 40.969482421875,
      "activations/layer13_attention_weight_min": -34.84326934814453,
      "activations/layer14_attention_weight_max": 43.36396408081055,
      "activations/layer14_attention_weight_min": -39.719505310058594,
      "activations/layer15_attention_weight_max": 40.10999298095703,
      "activations/layer15_attention_weight_min": -36.42068099975586,
      "activations/layer16_attention_weight_max": 29.562849044799805,
      "activations/layer16_attention_weight_min": -29.287860870361328,
      "activations/layer17_attention_weight_max": 27.848670959472656,
      "activations/layer17_attention_weight_min": -27.56024932861328,
      "activations/layer18_attention_weight_max": 31.715641021728516,
      "activations/layer18_attention_weight_min": -26.33138084411621,
      "activations/layer19_attention_weight_max": 31.700824737548828,
      "activations/layer19_attention_weight_min": -32.57279586791992,
      "activations/layer1_attention_weight_max": 15.945741653442383,
      "activations/layer1_attention_weight_min": -14.995429039001465,
      "activations/layer20_attention_weight_max": 27.360334396362305,
      "activations/layer20_attention_weight_min": -26.017248153686523,
      "activations/layer21_attention_weight_max": 28.81791114807129,
      "activations/layer21_attention_weight_min": -25.30016326904297,
      "activations/layer22_attention_weight_max": 40.75765609741211,
      "activations/layer22_attention_weight_min": -31.64166259765625,
      "activations/layer23_attention_weight_max": 32.47205352783203,
      "activations/layer23_attention_weight_min": -27.261371612548828,
      "activations/layer2_attention_weight_max": 32.4946403503418,
      "activations/layer2_attention_weight_min": -32.87883377075195,
      "activations/layer3_attention_weight_max": 95.66349029541016,
      "activations/layer3_attention_weight_min": -94.37206268310547,
      "activations/layer4_attention_weight_max": 104.2100601196289,
      "activations/layer4_attention_weight_min": -108.335693359375,
      "activations/layer5_attention_weight_max": 50.303768157958984,
      "activations/layer5_attention_weight_min": -60.98033142089844,
      "activations/layer6_attention_weight_max": 43.06315994262695,
      "activations/layer6_attention_weight_min": -48.785221099853516,
      "activations/layer7_attention_weight_max": 86.36296844482422,
      "activations/layer7_attention_weight_min": -92.64449310302734,
      "activations/layer8_attention_weight_max": 42.81252670288086,
      "activations/layer8_attention_weight_min": -42.37721633911133,
      "activations/layer9_attention_weight_max": 33.68307113647461,
      "activations/layer9_attention_weight_min": -32.971923828125,
      "epoch": 16.43,
      "learning_rate": 4.45185606060606e-05,
      "loss": 2.7326,
      "step": 282700
    },
    {
      "activations/layer0_attention_weight_max": 15.924670219421387,
      "activations/layer0_attention_weight_min": -12.344325065612793,
      "activations/layer10_attention_weight_max": 33.1884765625,
      "activations/layer10_attention_weight_min": -31.972307205200195,
      "activations/layer11_attention_weight_max": 34.10719299316406,
      "activations/layer11_attention_weight_min": -34.39104080200195,
      "activations/layer12_attention_weight_max": 24.012020111083984,
      "activations/layer12_attention_weight_min": -24.528564453125,
      "activations/layer13_attention_weight_max": 34.32442855834961,
      "activations/layer13_attention_weight_min": -35.93619918823242,
      "activations/layer14_attention_weight_max": 42.84599685668945,
      "activations/layer14_attention_weight_min": -39.313453674316406,
      "activations/layer15_attention_weight_max": 34.88162612915039,
      "activations/layer15_attention_weight_min": -38.12181091308594,
      "activations/layer16_attention_weight_max": 31.7572021484375,
      "activations/layer16_attention_weight_min": -29.241535186767578,
      "activations/layer17_attention_weight_max": 29.87481689453125,
      "activations/layer17_attention_weight_min": -28.34444236755371,
      "activations/layer18_attention_weight_max": 31.739187240600586,
      "activations/layer18_attention_weight_min": -26.86345100402832,
      "activations/layer19_attention_weight_max": 33.75383377075195,
      "activations/layer19_attention_weight_min": -31.780765533447266,
      "activations/layer1_attention_weight_max": 17.8513126373291,
      "activations/layer1_attention_weight_min": -17.157188415527344,
      "activations/layer20_attention_weight_max": 28.478830337524414,
      "activations/layer20_attention_weight_min": -25.393415451049805,
      "activations/layer21_attention_weight_max": 29.071969985961914,
      "activations/layer21_attention_weight_min": -26.73845100402832,
      "activations/layer22_attention_weight_max": 41.645286560058594,
      "activations/layer22_attention_weight_min": -32.0283203125,
      "activations/layer23_attention_weight_max": 32.071937561035156,
      "activations/layer23_attention_weight_min": -27.187162399291992,
      "activations/layer2_attention_weight_max": 32.4956169128418,
      "activations/layer2_attention_weight_min": -32.090240478515625,
      "activations/layer3_attention_weight_max": 98.72488403320312,
      "activations/layer3_attention_weight_min": -94.03246307373047,
      "activations/layer4_attention_weight_max": 112.1321029663086,
      "activations/layer4_attention_weight_min": -102.27738952636719,
      "activations/layer5_attention_weight_max": 51.81333541870117,
      "activations/layer5_attention_weight_min": -61.16554260253906,
      "activations/layer6_attention_weight_max": 43.46033477783203,
      "activations/layer6_attention_weight_min": -46.639122009277344,
      "activations/layer7_attention_weight_max": 94.55247497558594,
      "activations/layer7_attention_weight_min": -91.45802307128906,
      "activations/layer8_attention_weight_max": 42.1019172668457,
      "activations/layer8_attention_weight_min": -42.72990417480469,
      "activations/layer9_attention_weight_max": 32.01184844970703,
      "activations/layer9_attention_weight_min": -32.70180130004883,
      "epoch": 16.43,
      "learning_rate": 4.449962121212121e-05,
      "loss": 2.7461,
      "step": 282750
    },
    {
      "activations/layer0_attention_weight_max": 15.697860717773438,
      "activations/layer0_attention_weight_min": -12.118127822875977,
      "activations/layer10_attention_weight_max": 35.71688461303711,
      "activations/layer10_attention_weight_min": -34.39533233642578,
      "activations/layer11_attention_weight_max": 37.102378845214844,
      "activations/layer11_attention_weight_min": -36.281280517578125,
      "activations/layer12_attention_weight_max": 29.02409553527832,
      "activations/layer12_attention_weight_min": -26.41454315185547,
      "activations/layer13_attention_weight_max": 39.06243896484375,
      "activations/layer13_attention_weight_min": -34.22400665283203,
      "activations/layer14_attention_weight_max": 44.1711311340332,
      "activations/layer14_attention_weight_min": -38.18754577636719,
      "activations/layer15_attention_weight_max": 45.28632354736328,
      "activations/layer15_attention_weight_min": -38.51298141479492,
      "activations/layer16_attention_weight_max": 36.2991828918457,
      "activations/layer16_attention_weight_min": -27.881391525268555,
      "activations/layer17_attention_weight_max": 33.17630386352539,
      "activations/layer17_attention_weight_min": -28.268789291381836,
      "activations/layer18_attention_weight_max": 34.01259231567383,
      "activations/layer18_attention_weight_min": -28.271141052246094,
      "activations/layer19_attention_weight_max": 42.307823181152344,
      "activations/layer19_attention_weight_min": -34.54875564575195,
      "activations/layer1_attention_weight_max": 16.657161712646484,
      "activations/layer1_attention_weight_min": -15.187865257263184,
      "activations/layer20_attention_weight_max": 36.20561218261719,
      "activations/layer20_attention_weight_min": -25.829084396362305,
      "activations/layer21_attention_weight_max": 33.291717529296875,
      "activations/layer21_attention_weight_min": -26.744503021240234,
      "activations/layer22_attention_weight_max": 52.61622619628906,
      "activations/layer22_attention_weight_min": -36.65840148925781,
      "activations/layer23_attention_weight_max": 40.285980224609375,
      "activations/layer23_attention_weight_min": -26.490596771240234,
      "activations/layer2_attention_weight_max": 30.23671531677246,
      "activations/layer2_attention_weight_min": -30.94481658935547,
      "activations/layer3_attention_weight_max": 94.27588653564453,
      "activations/layer3_attention_weight_min": -93.17822265625,
      "activations/layer4_attention_weight_max": 108.4102783203125,
      "activations/layer4_attention_weight_min": -103.71695709228516,
      "activations/layer5_attention_weight_max": 50.284141540527344,
      "activations/layer5_attention_weight_min": -62.426780700683594,
      "activations/layer6_attention_weight_max": 41.2317008972168,
      "activations/layer6_attention_weight_min": -43.54860305786133,
      "activations/layer7_attention_weight_max": 103.72844696044922,
      "activations/layer7_attention_weight_min": -94.38296508789062,
      "activations/layer8_attention_weight_max": 43.97343826293945,
      "activations/layer8_attention_weight_min": -45.31670379638672,
      "activations/layer9_attention_weight_max": 34.4794921875,
      "activations/layer9_attention_weight_min": -36.00400924682617,
      "epoch": 16.43,
      "learning_rate": 4.4480681818181816e-05,
      "loss": 2.7447,
      "step": 282800
    },
    {
      "activations/layer0_attention_weight_max": 16.537593841552734,
      "activations/layer0_attention_weight_min": -12.46496295928955,
      "activations/layer10_attention_weight_max": 31.40700912475586,
      "activations/layer10_attention_weight_min": -33.14748001098633,
      "activations/layer11_attention_weight_max": 33.80234146118164,
      "activations/layer11_attention_weight_min": -32.96880340576172,
      "activations/layer12_attention_weight_max": 28.139156341552734,
      "activations/layer12_attention_weight_min": -25.74443817138672,
      "activations/layer13_attention_weight_max": 38.963802337646484,
      "activations/layer13_attention_weight_min": -35.50640869140625,
      "activations/layer14_attention_weight_max": 41.86445236206055,
      "activations/layer14_attention_weight_min": -36.0684928894043,
      "activations/layer15_attention_weight_max": 37.699771881103516,
      "activations/layer15_attention_weight_min": -33.09882354736328,
      "activations/layer16_attention_weight_max": 28.64429473876953,
      "activations/layer16_attention_weight_min": -27.032480239868164,
      "activations/layer17_attention_weight_max": 28.80075454711914,
      "activations/layer17_attention_weight_min": -25.253671646118164,
      "activations/layer18_attention_weight_max": 31.263816833496094,
      "activations/layer18_attention_weight_min": -25.09857177734375,
      "activations/layer19_attention_weight_max": 33.94802474975586,
      "activations/layer19_attention_weight_min": -31.125808715820312,
      "activations/layer1_attention_weight_max": 17.870542526245117,
      "activations/layer1_attention_weight_min": -17.06304931640625,
      "activations/layer20_attention_weight_max": 27.78953742980957,
      "activations/layer20_attention_weight_min": -26.56602668762207,
      "activations/layer21_attention_weight_max": 28.303634643554688,
      "activations/layer21_attention_weight_min": -23.84733009338379,
      "activations/layer22_attention_weight_max": 39.01923370361328,
      "activations/layer22_attention_weight_min": -30.77261734008789,
      "activations/layer23_attention_weight_max": 31.17281723022461,
      "activations/layer23_attention_weight_min": -25.236392974853516,
      "activations/layer2_attention_weight_max": 31.962799072265625,
      "activations/layer2_attention_weight_min": -31.2745361328125,
      "activations/layer3_attention_weight_max": 95.09232330322266,
      "activations/layer3_attention_weight_min": -94.14077758789062,
      "activations/layer4_attention_weight_max": 98.58185577392578,
      "activations/layer4_attention_weight_min": -100.10954284667969,
      "activations/layer5_attention_weight_max": 48.97926330566406,
      "activations/layer5_attention_weight_min": -60.893577575683594,
      "activations/layer6_attention_weight_max": 43.96111297607422,
      "activations/layer6_attention_weight_min": -48.14143371582031,
      "activations/layer7_attention_weight_max": 85.7691650390625,
      "activations/layer7_attention_weight_min": -94.15681457519531,
      "activations/layer8_attention_weight_max": 38.889495849609375,
      "activations/layer8_attention_weight_min": -39.85438919067383,
      "activations/layer9_attention_weight_max": 30.9857120513916,
      "activations/layer9_attention_weight_min": -31.86252212524414,
      "epoch": 16.44,
      "learning_rate": 4.446174242424242e-05,
      "loss": 2.7514,
      "step": 282850
    },
    {
      "activations/layer0_attention_weight_max": 15.127883911132812,
      "activations/layer0_attention_weight_min": -12.2306547164917,
      "activations/layer10_attention_weight_max": 31.383209228515625,
      "activations/layer10_attention_weight_min": -32.2384033203125,
      "activations/layer11_attention_weight_max": 32.06647491455078,
      "activations/layer11_attention_weight_min": -32.47431182861328,
      "activations/layer12_attention_weight_max": 25.91443634033203,
      "activations/layer12_attention_weight_min": -26.04168128967285,
      "activations/layer13_attention_weight_max": 40.43095397949219,
      "activations/layer13_attention_weight_min": -34.819332122802734,
      "activations/layer14_attention_weight_max": 43.5982780456543,
      "activations/layer14_attention_weight_min": -36.609092712402344,
      "activations/layer15_attention_weight_max": 41.108585357666016,
      "activations/layer15_attention_weight_min": -34.40740966796875,
      "activations/layer16_attention_weight_max": 31.530614852905273,
      "activations/layer16_attention_weight_min": -29.692338943481445,
      "activations/layer17_attention_weight_max": 31.48598861694336,
      "activations/layer17_attention_weight_min": -26.88104248046875,
      "activations/layer18_attention_weight_max": 33.565982818603516,
      "activations/layer18_attention_weight_min": -27.35063362121582,
      "activations/layer19_attention_weight_max": 34.61783218383789,
      "activations/layer19_attention_weight_min": -34.628597259521484,
      "activations/layer1_attention_weight_max": 17.700674057006836,
      "activations/layer1_attention_weight_min": -15.584574699401855,
      "activations/layer20_attention_weight_max": 31.586334228515625,
      "activations/layer20_attention_weight_min": -27.886451721191406,
      "activations/layer21_attention_weight_max": 30.882152557373047,
      "activations/layer21_attention_weight_min": -26.52017593383789,
      "activations/layer22_attention_weight_max": 51.78488540649414,
      "activations/layer22_attention_weight_min": -31.84869956970215,
      "activations/layer23_attention_weight_max": 35.62268829345703,
      "activations/layer23_attention_weight_min": -26.769912719726562,
      "activations/layer2_attention_weight_max": 31.741628646850586,
      "activations/layer2_attention_weight_min": -31.86431884765625,
      "activations/layer3_attention_weight_max": 91.48956298828125,
      "activations/layer3_attention_weight_min": -93.51441192626953,
      "activations/layer4_attention_weight_max": 98.56317138671875,
      "activations/layer4_attention_weight_min": -103.071533203125,
      "activations/layer5_attention_weight_max": 50.28046798706055,
      "activations/layer5_attention_weight_min": -61.95231246948242,
      "activations/layer6_attention_weight_max": 41.21059036254883,
      "activations/layer6_attention_weight_min": -45.38639831542969,
      "activations/layer7_attention_weight_max": 86.57083892822266,
      "activations/layer7_attention_weight_min": -92.89067077636719,
      "activations/layer8_attention_weight_max": 40.550411224365234,
      "activations/layer8_attention_weight_min": -40.82298278808594,
      "activations/layer9_attention_weight_max": 30.743743896484375,
      "activations/layer9_attention_weight_min": -30.731199264526367,
      "epoch": 16.44,
      "learning_rate": 4.4442803030303026e-05,
      "loss": 2.7606,
      "step": 282900
    },
    {
      "activations/layer0_attention_weight_max": 16.782577514648438,
      "activations/layer0_attention_weight_min": -12.531939506530762,
      "activations/layer10_attention_weight_max": 33.4134635925293,
      "activations/layer10_attention_weight_min": -36.6282844543457,
      "activations/layer11_attention_weight_max": 33.264671325683594,
      "activations/layer11_attention_weight_min": -36.858604431152344,
      "activations/layer12_attention_weight_max": 27.352252960205078,
      "activations/layer12_attention_weight_min": -25.93505096435547,
      "activations/layer13_attention_weight_max": 39.669979095458984,
      "activations/layer13_attention_weight_min": -35.467529296875,
      "activations/layer14_attention_weight_max": 43.30799102783203,
      "activations/layer14_attention_weight_min": -36.48847579956055,
      "activations/layer15_attention_weight_max": 38.47837829589844,
      "activations/layer15_attention_weight_min": -36.25446319580078,
      "activations/layer16_attention_weight_max": 29.635271072387695,
      "activations/layer16_attention_weight_min": -27.070693969726562,
      "activations/layer17_attention_weight_max": 28.75286865234375,
      "activations/layer17_attention_weight_min": -27.050621032714844,
      "activations/layer18_attention_weight_max": 31.917499542236328,
      "activations/layer18_attention_weight_min": -27.230735778808594,
      "activations/layer19_attention_weight_max": 32.33041000366211,
      "activations/layer19_attention_weight_min": -31.064390182495117,
      "activations/layer1_attention_weight_max": 18.877641677856445,
      "activations/layer1_attention_weight_min": -17.49468421936035,
      "activations/layer20_attention_weight_max": 28.494295120239258,
      "activations/layer20_attention_weight_min": -27.138111114501953,
      "activations/layer21_attention_weight_max": 28.52964210510254,
      "activations/layer21_attention_weight_min": -24.61789321899414,
      "activations/layer22_attention_weight_max": 38.94044494628906,
      "activations/layer22_attention_weight_min": -29.470951080322266,
      "activations/layer23_attention_weight_max": 32.449974060058594,
      "activations/layer23_attention_weight_min": -25.093420028686523,
      "activations/layer2_attention_weight_max": 34.47462463378906,
      "activations/layer2_attention_weight_min": -34.21151351928711,
      "activations/layer3_attention_weight_max": 94.3658447265625,
      "activations/layer3_attention_weight_min": -97.95854949951172,
      "activations/layer4_attention_weight_max": 104.74958801269531,
      "activations/layer4_attention_weight_min": -104.36165618896484,
      "activations/layer5_attention_weight_max": 51.70022201538086,
      "activations/layer5_attention_weight_min": -64.29190063476562,
      "activations/layer6_attention_weight_max": 46.41865539550781,
      "activations/layer6_attention_weight_min": -48.96910858154297,
      "activations/layer7_attention_weight_max": 99.0806655883789,
      "activations/layer7_attention_weight_min": -101.29874420166016,
      "activations/layer8_attention_weight_max": 42.743717193603516,
      "activations/layer8_attention_weight_min": -45.57292556762695,
      "activations/layer9_attention_weight_max": 32.03010177612305,
      "activations/layer9_attention_weight_min": -34.79838562011719,
      "epoch": 16.44,
      "learning_rate": 4.4423863636363634e-05,
      "loss": 2.7586,
      "step": 282950
    },
    {
      "activations/layer0_attention_weight_max": 16.31157684326172,
      "activations/layer0_attention_weight_min": -12.739823341369629,
      "activations/layer10_attention_weight_max": 32.45709228515625,
      "activations/layer10_attention_weight_min": -36.7159538269043,
      "activations/layer11_attention_weight_max": 33.776405334472656,
      "activations/layer11_attention_weight_min": -35.88751220703125,
      "activations/layer12_attention_weight_max": 26.468111038208008,
      "activations/layer12_attention_weight_min": -27.8624324798584,
      "activations/layer13_attention_weight_max": 41.52766036987305,
      "activations/layer13_attention_weight_min": -35.98834228515625,
      "activations/layer14_attention_weight_max": 42.127689361572266,
      "activations/layer14_attention_weight_min": -39.03034973144531,
      "activations/layer15_attention_weight_max": 40.44761657714844,
      "activations/layer15_attention_weight_min": -35.54082489013672,
      "activations/layer16_attention_weight_max": 29.118377685546875,
      "activations/layer16_attention_weight_min": -28.378904342651367,
      "activations/layer17_attention_weight_max": 29.967342376708984,
      "activations/layer17_attention_weight_min": -27.484834671020508,
      "activations/layer18_attention_weight_max": 31.49273681640625,
      "activations/layer18_attention_weight_min": -25.90656852722168,
      "activations/layer19_attention_weight_max": 35.78547668457031,
      "activations/layer19_attention_weight_min": -30.980478286743164,
      "activations/layer1_attention_weight_max": 16.297367095947266,
      "activations/layer1_attention_weight_min": -14.689374923706055,
      "activations/layer20_attention_weight_max": 29.188547134399414,
      "activations/layer20_attention_weight_min": -24.142274856567383,
      "activations/layer21_attention_weight_max": 28.550060272216797,
      "activations/layer21_attention_weight_min": -24.316085815429688,
      "activations/layer22_attention_weight_max": 41.75783157348633,
      "activations/layer22_attention_weight_min": -31.10491943359375,
      "activations/layer23_attention_weight_max": 32.68159484863281,
      "activations/layer23_attention_weight_min": -25.73394012451172,
      "activations/layer2_attention_weight_max": 35.224063873291016,
      "activations/layer2_attention_weight_min": -33.7823486328125,
      "activations/layer3_attention_weight_max": 96.59801483154297,
      "activations/layer3_attention_weight_min": -98.29645538330078,
      "activations/layer4_attention_weight_max": 104.23625946044922,
      "activations/layer4_attention_weight_min": -104.3847885131836,
      "activations/layer5_attention_weight_max": 49.45237350463867,
      "activations/layer5_attention_weight_min": -61.30312728881836,
      "activations/layer6_attention_weight_max": 42.57828903198242,
      "activations/layer6_attention_weight_min": -46.731422424316406,
      "activations/layer7_attention_weight_max": 99.42060852050781,
      "activations/layer7_attention_weight_min": -90.09600830078125,
      "activations/layer8_attention_weight_max": 41.0230598449707,
      "activations/layer8_attention_weight_min": -42.894107818603516,
      "activations/layer9_attention_weight_max": 32.554962158203125,
      "activations/layer9_attention_weight_min": -33.58728790283203,
      "epoch": 16.44,
      "learning_rate": 4.4404924242424236e-05,
      "loss": 2.7343,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.5651,
      "eval_samples_per_second": 501.339,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.5651,
      "eval_openwebtext_samples_per_second": 501.339,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.0171,
      "eval_wikitext_samples_per_second": 226.062,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.5639,
      "eval_lambada_samples_per_second": 509.1,
      "step": 283000
    },
    {
      "activations/layer0_attention_weight_max": 16.19576072692871,
      "activations/layer0_attention_weight_min": -12.84858512878418,
      "activations/layer10_attention_weight_max": 33.9776611328125,
      "activations/layer10_attention_weight_min": -32.52838897705078,
      "activations/layer11_attention_weight_max": 35.11933135986328,
      "activations/layer11_attention_weight_min": -33.19312286376953,
      "activations/layer12_attention_weight_max": 24.2503719329834,
      "activations/layer12_attention_weight_min": -25.94446563720703,
      "activations/layer13_attention_weight_max": 35.78118896484375,
      "activations/layer13_attention_weight_min": -34.75914001464844,
      "activations/layer14_attention_weight_max": 43.682151794433594,
      "activations/layer14_attention_weight_min": -38.52247619628906,
      "activations/layer15_attention_weight_max": 36.93505096435547,
      "activations/layer15_attention_weight_min": -33.27626419067383,
      "activations/layer16_attention_weight_max": 31.010770797729492,
      "activations/layer16_attention_weight_min": -27.980039596557617,
      "activations/layer17_attention_weight_max": 28.84652328491211,
      "activations/layer17_attention_weight_min": -25.961214065551758,
      "activations/layer18_attention_weight_max": 29.53156089782715,
      "activations/layer18_attention_weight_min": -24.738523483276367,
      "activations/layer19_attention_weight_max": 33.97727584838867,
      "activations/layer19_attention_weight_min": -32.038455963134766,
      "activations/layer1_attention_weight_max": 16.83588409423828,
      "activations/layer1_attention_weight_min": -15.917548179626465,
      "activations/layer20_attention_weight_max": 27.384632110595703,
      "activations/layer20_attention_weight_min": -25.447477340698242,
      "activations/layer21_attention_weight_max": 26.493053436279297,
      "activations/layer21_attention_weight_min": -23.936723709106445,
      "activations/layer22_attention_weight_max": 43.82449722290039,
      "activations/layer22_attention_weight_min": -30.21213150024414,
      "activations/layer23_attention_weight_max": 31.931747436523438,
      "activations/layer23_attention_weight_min": -24.30791473388672,
      "activations/layer2_attention_weight_max": 33.48768997192383,
      "activations/layer2_attention_weight_min": -30.644132614135742,
      "activations/layer3_attention_weight_max": 93.19361114501953,
      "activations/layer3_attention_weight_min": -95.48652648925781,
      "activations/layer4_attention_weight_max": 104.32340240478516,
      "activations/layer4_attention_weight_min": -102.3868179321289,
      "activations/layer5_attention_weight_max": 55.9154167175293,
      "activations/layer5_attention_weight_min": -62.71722412109375,
      "activations/layer6_attention_weight_max": 41.70050048828125,
      "activations/layer6_attention_weight_min": -44.32175064086914,
      "activations/layer7_attention_weight_max": 88.62297821044922,
      "activations/layer7_attention_weight_min": -89.52650451660156,
      "activations/layer8_attention_weight_max": 39.74855422973633,
      "activations/layer8_attention_weight_min": -42.144996643066406,
      "activations/layer9_attention_weight_max": 32.240074157714844,
      "activations/layer9_attention_weight_min": -32.588314056396484,
      "epoch": 16.45,
      "learning_rate": 4.4385984848484844e-05,
      "loss": 2.7412,
      "step": 283050
    },
    {
      "activations/layer0_attention_weight_max": 16.35117530822754,
      "activations/layer0_attention_weight_min": -12.785889625549316,
      "activations/layer10_attention_weight_max": 35.04143142700195,
      "activations/layer10_attention_weight_min": -33.792640686035156,
      "activations/layer11_attention_weight_max": 34.663204193115234,
      "activations/layer11_attention_weight_min": -32.729061126708984,
      "activations/layer12_attention_weight_max": 27.838777542114258,
      "activations/layer12_attention_weight_min": -25.83945655822754,
      "activations/layer13_attention_weight_max": 42.20018768310547,
      "activations/layer13_attention_weight_min": -35.437889099121094,
      "activations/layer14_attention_weight_max": 43.53496551513672,
      "activations/layer14_attention_weight_min": -37.027610778808594,
      "activations/layer15_attention_weight_max": 44.775508880615234,
      "activations/layer15_attention_weight_min": -34.263362884521484,
      "activations/layer16_attention_weight_max": 31.787275314331055,
      "activations/layer16_attention_weight_min": -27.9152774810791,
      "activations/layer17_attention_weight_max": 33.05061721801758,
      "activations/layer17_attention_weight_min": -26.249345779418945,
      "activations/layer18_attention_weight_max": 30.661277770996094,
      "activations/layer18_attention_weight_min": -27.528459548950195,
      "activations/layer19_attention_weight_max": 33.03465270996094,
      "activations/layer19_attention_weight_min": -30.803714752197266,
      "activations/layer1_attention_weight_max": 17.596084594726562,
      "activations/layer1_attention_weight_min": -14.261801719665527,
      "activations/layer20_attention_weight_max": 32.507137298583984,
      "activations/layer20_attention_weight_min": -25.934459686279297,
      "activations/layer21_attention_weight_max": 29.80912208557129,
      "activations/layer21_attention_weight_min": -24.786046981811523,
      "activations/layer22_attention_weight_max": 41.94831085205078,
      "activations/layer22_attention_weight_min": -31.695987701416016,
      "activations/layer23_attention_weight_max": 32.85870361328125,
      "activations/layer23_attention_weight_min": -25.36962127685547,
      "activations/layer2_attention_weight_max": 31.988157272338867,
      "activations/layer2_attention_weight_min": -31.193843841552734,
      "activations/layer3_attention_weight_max": 94.15690612792969,
      "activations/layer3_attention_weight_min": -95.8363265991211,
      "activations/layer4_attention_weight_max": 100.26195526123047,
      "activations/layer4_attention_weight_min": -105.50091552734375,
      "activations/layer5_attention_weight_max": 51.82952117919922,
      "activations/layer5_attention_weight_min": -60.05504608154297,
      "activations/layer6_attention_weight_max": 43.65266418457031,
      "activations/layer6_attention_weight_min": -45.468894958496094,
      "activations/layer7_attention_weight_max": 89.61304473876953,
      "activations/layer7_attention_weight_min": -97.97245788574219,
      "activations/layer8_attention_weight_max": 41.69292068481445,
      "activations/layer8_attention_weight_min": -41.05842208862305,
      "activations/layer9_attention_weight_max": 32.81843185424805,
      "activations/layer9_attention_weight_min": -31.5416316986084,
      "epoch": 16.45,
      "learning_rate": 4.4367045454545446e-05,
      "loss": 2.7467,
      "step": 283100
    },
    {
      "activations/layer0_attention_weight_max": 16.443389892578125,
      "activations/layer0_attention_weight_min": -12.763725280761719,
      "activations/layer10_attention_weight_max": 34.98033905029297,
      "activations/layer10_attention_weight_min": -34.32701873779297,
      "activations/layer11_attention_weight_max": 33.81570053100586,
      "activations/layer11_attention_weight_min": -34.614356994628906,
      "activations/layer12_attention_weight_max": 26.150362014770508,
      "activations/layer12_attention_weight_min": -25.00701141357422,
      "activations/layer13_attention_weight_max": 38.75770950317383,
      "activations/layer13_attention_weight_min": -33.7273063659668,
      "activations/layer14_attention_weight_max": 41.60173416137695,
      "activations/layer14_attention_weight_min": -36.162567138671875,
      "activations/layer15_attention_weight_max": 37.498416900634766,
      "activations/layer15_attention_weight_min": -33.00772476196289,
      "activations/layer16_attention_weight_max": 30.25380516052246,
      "activations/layer16_attention_weight_min": -29.408287048339844,
      "activations/layer17_attention_weight_max": 30.59427833557129,
      "activations/layer17_attention_weight_min": -25.529083251953125,
      "activations/layer18_attention_weight_max": 35.96230697631836,
      "activations/layer18_attention_weight_min": -24.52962875366211,
      "activations/layer19_attention_weight_max": 35.5681266784668,
      "activations/layer19_attention_weight_min": -33.719764709472656,
      "activations/layer1_attention_weight_max": 16.11338233947754,
      "activations/layer1_attention_weight_min": -14.638225555419922,
      "activations/layer20_attention_weight_max": 29.182266235351562,
      "activations/layer20_attention_weight_min": -26.000011444091797,
      "activations/layer21_attention_weight_max": 31.535202026367188,
      "activations/layer21_attention_weight_min": -26.90365982055664,
      "activations/layer22_attention_weight_max": 42.5617790222168,
      "activations/layer22_attention_weight_min": -32.75390625,
      "activations/layer23_attention_weight_max": 33.30022430419922,
      "activations/layer23_attention_weight_min": -25.792213439941406,
      "activations/layer2_attention_weight_max": 32.544647216796875,
      "activations/layer2_attention_weight_min": -33.0594482421875,
      "activations/layer3_attention_weight_max": 93.60358428955078,
      "activations/layer3_attention_weight_min": -95.46800994873047,
      "activations/layer4_attention_weight_max": 107.2512435913086,
      "activations/layer4_attention_weight_min": -109.40142059326172,
      "activations/layer5_attention_weight_max": 51.03669357299805,
      "activations/layer5_attention_weight_min": -64.55500030517578,
      "activations/layer6_attention_weight_max": 44.71194076538086,
      "activations/layer6_attention_weight_min": -46.82186508178711,
      "activations/layer7_attention_weight_max": 101.20130157470703,
      "activations/layer7_attention_weight_min": -96.99098205566406,
      "activations/layer8_attention_weight_max": 44.47690963745117,
      "activations/layer8_attention_weight_min": -45.556373596191406,
      "activations/layer9_attention_weight_max": 32.66908264160156,
      "activations/layer9_attention_weight_min": -32.489070892333984,
      "epoch": 16.45,
      "learning_rate": 4.434810606060606e-05,
      "loss": 2.7474,
      "step": 283150
    },
    {
      "activations/layer0_attention_weight_max": 15.371223449707031,
      "activations/layer0_attention_weight_min": -12.541342735290527,
      "activations/layer10_attention_weight_max": 36.736454010009766,
      "activations/layer10_attention_weight_min": -35.23371505737305,
      "activations/layer11_attention_weight_max": 35.36797332763672,
      "activations/layer11_attention_weight_min": -34.674461364746094,
      "activations/layer12_attention_weight_max": 31.251371383666992,
      "activations/layer12_attention_weight_min": -26.782175064086914,
      "activations/layer13_attention_weight_max": 45.36194610595703,
      "activations/layer13_attention_weight_min": -36.42416763305664,
      "activations/layer14_attention_weight_max": 47.8035774230957,
      "activations/layer14_attention_weight_min": -38.84751510620117,
      "activations/layer15_attention_weight_max": 41.07500076293945,
      "activations/layer15_attention_weight_min": -35.51829528808594,
      "activations/layer16_attention_weight_max": 31.41521453857422,
      "activations/layer16_attention_weight_min": -28.000913619995117,
      "activations/layer17_attention_weight_max": 32.45719909667969,
      "activations/layer17_attention_weight_min": -25.90131187438965,
      "activations/layer18_attention_weight_max": 35.3823127746582,
      "activations/layer18_attention_weight_min": -25.419403076171875,
      "activations/layer19_attention_weight_max": 41.15287399291992,
      "activations/layer19_attention_weight_min": -32.694034576416016,
      "activations/layer1_attention_weight_max": 16.684425354003906,
      "activations/layer1_attention_weight_min": -15.327285766601562,
      "activations/layer20_attention_weight_max": 35.795711517333984,
      "activations/layer20_attention_weight_min": -26.619882583618164,
      "activations/layer21_attention_weight_max": 38.89267349243164,
      "activations/layer21_attention_weight_min": -25.227148056030273,
      "activations/layer22_attention_weight_max": 51.25888442993164,
      "activations/layer22_attention_weight_min": -34.79976272583008,
      "activations/layer23_attention_weight_max": 39.62236404418945,
      "activations/layer23_attention_weight_min": -25.847379684448242,
      "activations/layer2_attention_weight_max": 31.44051170349121,
      "activations/layer2_attention_weight_min": -31.799968719482422,
      "activations/layer3_attention_weight_max": 93.99127960205078,
      "activations/layer3_attention_weight_min": -96.26518249511719,
      "activations/layer4_attention_weight_max": 104.35302734375,
      "activations/layer4_attention_weight_min": -111.40093231201172,
      "activations/layer5_attention_weight_max": 49.75238037109375,
      "activations/layer5_attention_weight_min": -63.65690612792969,
      "activations/layer6_attention_weight_max": 44.95811080932617,
      "activations/layer6_attention_weight_min": -47.96794891357422,
      "activations/layer7_attention_weight_max": 101.05487060546875,
      "activations/layer7_attention_weight_min": -97.50555419921875,
      "activations/layer8_attention_weight_max": 42.3336067199707,
      "activations/layer8_attention_weight_min": -44.72993087768555,
      "activations/layer9_attention_weight_max": 34.23116683959961,
      "activations/layer9_attention_weight_min": -35.54030990600586,
      "epoch": 16.46,
      "learning_rate": 4.432916666666666e-05,
      "loss": 2.7343,
      "step": 283200
    },
    {
      "activations/layer0_attention_weight_max": 15.906105041503906,
      "activations/layer0_attention_weight_min": -12.360883712768555,
      "activations/layer10_attention_weight_max": 33.15217590332031,
      "activations/layer10_attention_weight_min": -33.727359771728516,
      "activations/layer11_attention_weight_max": 32.67713165283203,
      "activations/layer11_attention_weight_min": -33.86064529418945,
      "activations/layer12_attention_weight_max": 27.17135238647461,
      "activations/layer12_attention_weight_min": -27.04017448425293,
      "activations/layer13_attention_weight_max": 40.71989822387695,
      "activations/layer13_attention_weight_min": -37.0961799621582,
      "activations/layer14_attention_weight_max": 47.601078033447266,
      "activations/layer14_attention_weight_min": -39.7122917175293,
      "activations/layer15_attention_weight_max": 38.98780059814453,
      "activations/layer15_attention_weight_min": -37.27558135986328,
      "activations/layer16_attention_weight_max": 32.009620666503906,
      "activations/layer16_attention_weight_min": -27.470592498779297,
      "activations/layer17_attention_weight_max": 31.573577880859375,
      "activations/layer17_attention_weight_min": -25.827505111694336,
      "activations/layer18_attention_weight_max": 34.2276725769043,
      "activations/layer18_attention_weight_min": -26.396669387817383,
      "activations/layer19_attention_weight_max": 40.63703536987305,
      "activations/layer19_attention_weight_min": -32.44087219238281,
      "activations/layer1_attention_weight_max": 16.80482292175293,
      "activations/layer1_attention_weight_min": -14.89126968383789,
      "activations/layer20_attention_weight_max": 29.28325080871582,
      "activations/layer20_attention_weight_min": -24.692819595336914,
      "activations/layer21_attention_weight_max": 30.937681198120117,
      "activations/layer21_attention_weight_min": -24.8001651763916,
      "activations/layer22_attention_weight_max": 44.34497833251953,
      "activations/layer22_attention_weight_min": -30.791873931884766,
      "activations/layer23_attention_weight_max": 43.134803771972656,
      "activations/layer23_attention_weight_min": -26.783811569213867,
      "activations/layer2_attention_weight_max": 31.791303634643555,
      "activations/layer2_attention_weight_min": -30.67073631286621,
      "activations/layer3_attention_weight_max": 90.19741821289062,
      "activations/layer3_attention_weight_min": -92.77603149414062,
      "activations/layer4_attention_weight_max": 100.49211120605469,
      "activations/layer4_attention_weight_min": -105.86363983154297,
      "activations/layer5_attention_weight_max": 49.37716293334961,
      "activations/layer5_attention_weight_min": -61.05987548828125,
      "activations/layer6_attention_weight_max": 42.32616424560547,
      "activations/layer6_attention_weight_min": -46.198646545410156,
      "activations/layer7_attention_weight_max": 91.40345764160156,
      "activations/layer7_attention_weight_min": -94.57850646972656,
      "activations/layer8_attention_weight_max": 40.96034622192383,
      "activations/layer8_attention_weight_min": -44.487098693847656,
      "activations/layer9_attention_weight_max": 32.019866943359375,
      "activations/layer9_attention_weight_min": -34.909423828125,
      "epoch": 16.46,
      "learning_rate": 4.4310227272727264e-05,
      "loss": 2.7449,
      "step": 283250
    },
    {
      "activations/layer0_attention_weight_max": 17.059730529785156,
      "activations/layer0_attention_weight_min": -13.106620788574219,
      "activations/layer10_attention_weight_max": 34.878822326660156,
      "activations/layer10_attention_weight_min": -37.59839630126953,
      "activations/layer11_attention_weight_max": 33.10586166381836,
      "activations/layer11_attention_weight_min": -36.224063873291016,
      "activations/layer12_attention_weight_max": 29.152772903442383,
      "activations/layer12_attention_weight_min": -33.7075080871582,
      "activations/layer13_attention_weight_max": 42.7061882019043,
      "activations/layer13_attention_weight_min": -35.93089294433594,
      "activations/layer14_attention_weight_max": 46.300235748291016,
      "activations/layer14_attention_weight_min": -37.69607162475586,
      "activations/layer15_attention_weight_max": 41.129356384277344,
      "activations/layer15_attention_weight_min": -39.316036224365234,
      "activations/layer16_attention_weight_max": 33.599464416503906,
      "activations/layer16_attention_weight_min": -28.256694793701172,
      "activations/layer17_attention_weight_max": 33.96918869018555,
      "activations/layer17_attention_weight_min": -27.45298957824707,
      "activations/layer18_attention_weight_max": 37.702186584472656,
      "activations/layer18_attention_weight_min": -26.64036750793457,
      "activations/layer19_attention_weight_max": 43.37568283081055,
      "activations/layer19_attention_weight_min": -33.28014373779297,
      "activations/layer1_attention_weight_max": 15.932032585144043,
      "activations/layer1_attention_weight_min": -15.824164390563965,
      "activations/layer20_attention_weight_max": 34.93851089477539,
      "activations/layer20_attention_weight_min": -26.067790985107422,
      "activations/layer21_attention_weight_max": 33.69509506225586,
      "activations/layer21_attention_weight_min": -26.991641998291016,
      "activations/layer22_attention_weight_max": 49.4254150390625,
      "activations/layer22_attention_weight_min": -37.15387725830078,
      "activations/layer23_attention_weight_max": 41.108856201171875,
      "activations/layer23_attention_weight_min": -26.306060791015625,
      "activations/layer2_attention_weight_max": 31.339292526245117,
      "activations/layer2_attention_weight_min": -32.1645622253418,
      "activations/layer3_attention_weight_max": 91.62770080566406,
      "activations/layer3_attention_weight_min": -92.85906982421875,
      "activations/layer4_attention_weight_max": 103.45589447021484,
      "activations/layer4_attention_weight_min": -104.26876068115234,
      "activations/layer5_attention_weight_max": 48.38429260253906,
      "activations/layer5_attention_weight_min": -65.01316833496094,
      "activations/layer6_attention_weight_max": 42.589107513427734,
      "activations/layer6_attention_weight_min": -44.619388580322266,
      "activations/layer7_attention_weight_max": 92.81961059570312,
      "activations/layer7_attention_weight_min": -92.9359359741211,
      "activations/layer8_attention_weight_max": 39.56047058105469,
      "activations/layer8_attention_weight_min": -43.18179702758789,
      "activations/layer9_attention_weight_max": 32.55018997192383,
      "activations/layer9_attention_weight_min": -36.960227966308594,
      "epoch": 16.46,
      "learning_rate": 4.429128787878788e-05,
      "loss": 2.7318,
      "step": 283300
    },
    {
      "activations/layer0_attention_weight_max": 15.601614952087402,
      "activations/layer0_attention_weight_min": -12.691861152648926,
      "activations/layer10_attention_weight_max": 35.06470489501953,
      "activations/layer10_attention_weight_min": -34.050636291503906,
      "activations/layer11_attention_weight_max": 34.018577575683594,
      "activations/layer11_attention_weight_min": -32.33544921875,
      "activations/layer12_attention_weight_max": 27.363367080688477,
      "activations/layer12_attention_weight_min": -25.687131881713867,
      "activations/layer13_attention_weight_max": 37.54351806640625,
      "activations/layer13_attention_weight_min": -35.04637908935547,
      "activations/layer14_attention_weight_max": 41.161460876464844,
      "activations/layer14_attention_weight_min": -36.97354507446289,
      "activations/layer15_attention_weight_max": 37.346336364746094,
      "activations/layer15_attention_weight_min": -34.429603576660156,
      "activations/layer16_attention_weight_max": 31.867691040039062,
      "activations/layer16_attention_weight_min": -27.510448455810547,
      "activations/layer17_attention_weight_max": 29.14547348022461,
      "activations/layer17_attention_weight_min": -26.266016006469727,
      "activations/layer18_attention_weight_max": 32.110877990722656,
      "activations/layer18_attention_weight_min": -26.59502601623535,
      "activations/layer19_attention_weight_max": 33.86566162109375,
      "activations/layer19_attention_weight_min": -29.231000900268555,
      "activations/layer1_attention_weight_max": 16.039958953857422,
      "activations/layer1_attention_weight_min": -14.61340618133545,
      "activations/layer20_attention_weight_max": 30.054752349853516,
      "activations/layer20_attention_weight_min": -24.030046463012695,
      "activations/layer21_attention_weight_max": 28.43930435180664,
      "activations/layer21_attention_weight_min": -25.085067749023438,
      "activations/layer22_attention_weight_max": 38.869083404541016,
      "activations/layer22_attention_weight_min": -31.938140869140625,
      "activations/layer23_attention_weight_max": 30.486183166503906,
      "activations/layer23_attention_weight_min": -26.053882598876953,
      "activations/layer2_attention_weight_max": 30.32935905456543,
      "activations/layer2_attention_weight_min": -29.766820907592773,
      "activations/layer3_attention_weight_max": 89.21358489990234,
      "activations/layer3_attention_weight_min": -86.72583770751953,
      "activations/layer4_attention_weight_max": 98.33558654785156,
      "activations/layer4_attention_weight_min": -99.6225814819336,
      "activations/layer5_attention_weight_max": 46.51157760620117,
      "activations/layer5_attention_weight_min": -59.411170959472656,
      "activations/layer6_attention_weight_max": 41.202980041503906,
      "activations/layer6_attention_weight_min": -44.24850845336914,
      "activations/layer7_attention_weight_max": 88.43225860595703,
      "activations/layer7_attention_weight_min": -88.54719543457031,
      "activations/layer8_attention_weight_max": 40.151729583740234,
      "activations/layer8_attention_weight_min": -40.527530670166016,
      "activations/layer9_attention_weight_max": 33.79623031616211,
      "activations/layer9_attention_weight_min": -33.605098724365234,
      "epoch": 16.46,
      "learning_rate": 4.427234848484848e-05,
      "loss": 2.7314,
      "step": 283350
    },
    {
      "activations/layer0_attention_weight_max": 15.766478538513184,
      "activations/layer0_attention_weight_min": -13.220633506774902,
      "activations/layer10_attention_weight_max": 33.844154357910156,
      "activations/layer10_attention_weight_min": -32.43669891357422,
      "activations/layer11_attention_weight_max": 35.03409957885742,
      "activations/layer11_attention_weight_min": -31.85953140258789,
      "activations/layer12_attention_weight_max": 26.793460845947266,
      "activations/layer12_attention_weight_min": -25.0570125579834,
      "activations/layer13_attention_weight_max": 40.64714050292969,
      "activations/layer13_attention_weight_min": -34.82072067260742,
      "activations/layer14_attention_weight_max": 41.59916687011719,
      "activations/layer14_attention_weight_min": -37.09015655517578,
      "activations/layer15_attention_weight_max": 38.68230438232422,
      "activations/layer15_attention_weight_min": -34.0664176940918,
      "activations/layer16_attention_weight_max": 29.426544189453125,
      "activations/layer16_attention_weight_min": -27.861669540405273,
      "activations/layer17_attention_weight_max": 30.75617790222168,
      "activations/layer17_attention_weight_min": -25.389007568359375,
      "activations/layer18_attention_weight_max": 31.768821716308594,
      "activations/layer18_attention_weight_min": -24.012001037597656,
      "activations/layer19_attention_weight_max": 34.63931655883789,
      "activations/layer19_attention_weight_min": -30.467899322509766,
      "activations/layer1_attention_weight_max": 16.21331787109375,
      "activations/layer1_attention_weight_min": -15.377065658569336,
      "activations/layer20_attention_weight_max": 30.67852783203125,
      "activations/layer20_attention_weight_min": -25.312894821166992,
      "activations/layer21_attention_weight_max": 29.82125473022461,
      "activations/layer21_attention_weight_min": -24.67508888244629,
      "activations/layer22_attention_weight_max": 42.62192916870117,
      "activations/layer22_attention_weight_min": -31.159393310546875,
      "activations/layer23_attention_weight_max": 34.93064880371094,
      "activations/layer23_attention_weight_min": -26.024837493896484,
      "activations/layer2_attention_weight_max": 32.27186584472656,
      "activations/layer2_attention_weight_min": -31.865758895874023,
      "activations/layer3_attention_weight_max": 96.94633483886719,
      "activations/layer3_attention_weight_min": -97.02381896972656,
      "activations/layer4_attention_weight_max": 103.69554901123047,
      "activations/layer4_attention_weight_min": -108.0956802368164,
      "activations/layer5_attention_weight_max": 51.151275634765625,
      "activations/layer5_attention_weight_min": -61.73974609375,
      "activations/layer6_attention_weight_max": 42.13650131225586,
      "activations/layer6_attention_weight_min": -47.61650848388672,
      "activations/layer7_attention_weight_max": 87.94192504882812,
      "activations/layer7_attention_weight_min": -99.72415161132812,
      "activations/layer8_attention_weight_max": 40.14585494995117,
      "activations/layer8_attention_weight_min": -40.69273376464844,
      "activations/layer9_attention_weight_max": 31.760791778564453,
      "activations/layer9_attention_weight_min": -32.74172592163086,
      "epoch": 16.47,
      "learning_rate": 4.425340909090909e-05,
      "loss": 2.752,
      "step": 283400
    },
    {
      "activations/layer0_attention_weight_max": 16.840436935424805,
      "activations/layer0_attention_weight_min": -12.229146957397461,
      "activations/layer10_attention_weight_max": 43.20907974243164,
      "activations/layer10_attention_weight_min": -39.68901824951172,
      "activations/layer11_attention_weight_max": 39.81475830078125,
      "activations/layer11_attention_weight_min": -39.674774169921875,
      "activations/layer12_attention_weight_max": 31.282485961914062,
      "activations/layer12_attention_weight_min": -30.862533569335938,
      "activations/layer13_attention_weight_max": 45.785030364990234,
      "activations/layer13_attention_weight_min": -38.7347526550293,
      "activations/layer14_attention_weight_max": 50.13279724121094,
      "activations/layer14_attention_weight_min": -43.27621841430664,
      "activations/layer15_attention_weight_max": 45.76197814941406,
      "activations/layer15_attention_weight_min": -41.164093017578125,
      "activations/layer16_attention_weight_max": 35.61846160888672,
      "activations/layer16_attention_weight_min": -30.04583740234375,
      "activations/layer17_attention_weight_max": 35.90226745605469,
      "activations/layer17_attention_weight_min": -27.693483352661133,
      "activations/layer18_attention_weight_max": 35.578739166259766,
      "activations/layer18_attention_weight_min": -27.044776916503906,
      "activations/layer19_attention_weight_max": 41.847496032714844,
      "activations/layer19_attention_weight_min": -34.58936309814453,
      "activations/layer1_attention_weight_max": 15.76816177368164,
      "activations/layer1_attention_weight_min": -14.199231147766113,
      "activations/layer20_attention_weight_max": 36.11945724487305,
      "activations/layer20_attention_weight_min": -26.519248962402344,
      "activations/layer21_attention_weight_max": 32.5765495300293,
      "activations/layer21_attention_weight_min": -25.442350387573242,
      "activations/layer22_attention_weight_max": 53.46118927001953,
      "activations/layer22_attention_weight_min": -34.44827651977539,
      "activations/layer23_attention_weight_max": 39.0743408203125,
      "activations/layer23_attention_weight_min": -25.036672592163086,
      "activations/layer2_attention_weight_max": 35.764434814453125,
      "activations/layer2_attention_weight_min": -32.38771057128906,
      "activations/layer3_attention_weight_max": 97.81957244873047,
      "activations/layer3_attention_weight_min": -98.73226165771484,
      "activations/layer4_attention_weight_max": 108.73931884765625,
      "activations/layer4_attention_weight_min": -110.5361557006836,
      "activations/layer5_attention_weight_max": 55.94463348388672,
      "activations/layer5_attention_weight_min": -62.34577178955078,
      "activations/layer6_attention_weight_max": 45.6032600402832,
      "activations/layer6_attention_weight_min": -46.645103454589844,
      "activations/layer7_attention_weight_max": 107.43082427978516,
      "activations/layer7_attention_weight_min": -104.63664245605469,
      "activations/layer8_attention_weight_max": 47.84272384643555,
      "activations/layer8_attention_weight_min": -50.280799865722656,
      "activations/layer9_attention_weight_max": 36.7413444519043,
      "activations/layer9_attention_weight_min": -39.560970306396484,
      "epoch": 16.47,
      "learning_rate": 4.423446969696969e-05,
      "loss": 2.7503,
      "step": 283450
    },
    {
      "activations/layer0_attention_weight_max": 15.867497444152832,
      "activations/layer0_attention_weight_min": -12.403853416442871,
      "activations/layer10_attention_weight_max": 30.5520076751709,
      "activations/layer10_attention_weight_min": -34.24498748779297,
      "activations/layer11_attention_weight_max": 33.912925720214844,
      "activations/layer11_attention_weight_min": -32.93251037597656,
      "activations/layer12_attention_weight_max": 26.703372955322266,
      "activations/layer12_attention_weight_min": -25.123550415039062,
      "activations/layer13_attention_weight_max": 40.97983932495117,
      "activations/layer13_attention_weight_min": -36.081642150878906,
      "activations/layer14_attention_weight_max": 42.82892990112305,
      "activations/layer14_attention_weight_min": -39.10407257080078,
      "activations/layer15_attention_weight_max": 38.38656997680664,
      "activations/layer15_attention_weight_min": -35.83014678955078,
      "activations/layer16_attention_weight_max": 30.13474464416504,
      "activations/layer16_attention_weight_min": -28.400066375732422,
      "activations/layer17_attention_weight_max": 29.77658462524414,
      "activations/layer17_attention_weight_min": -25.437498092651367,
      "activations/layer18_attention_weight_max": 32.74966812133789,
      "activations/layer18_attention_weight_min": -24.74435043334961,
      "activations/layer19_attention_weight_max": 37.50952911376953,
      "activations/layer19_attention_weight_min": -35.61298751831055,
      "activations/layer1_attention_weight_max": 16.25568389892578,
      "activations/layer1_attention_weight_min": -16.164417266845703,
      "activations/layer20_attention_weight_max": 30.752721786499023,
      "activations/layer20_attention_weight_min": -26.851646423339844,
      "activations/layer21_attention_weight_max": 31.80467414855957,
      "activations/layer21_attention_weight_min": -25.354801177978516,
      "activations/layer22_attention_weight_max": 42.567317962646484,
      "activations/layer22_attention_weight_min": -31.578672409057617,
      "activations/layer23_attention_weight_max": 33.46743392944336,
      "activations/layer23_attention_weight_min": -26.866722106933594,
      "activations/layer2_attention_weight_max": 31.73834800720215,
      "activations/layer2_attention_weight_min": -31.963237762451172,
      "activations/layer3_attention_weight_max": 91.9080810546875,
      "activations/layer3_attention_weight_min": -95.18792724609375,
      "activations/layer4_attention_weight_max": 102.71065521240234,
      "activations/layer4_attention_weight_min": -104.6778335571289,
      "activations/layer5_attention_weight_max": 49.14490509033203,
      "activations/layer5_attention_weight_min": -62.51707458496094,
      "activations/layer6_attention_weight_max": 42.591217041015625,
      "activations/layer6_attention_weight_min": -43.8479118347168,
      "activations/layer7_attention_weight_max": 90.85855865478516,
      "activations/layer7_attention_weight_min": -87.42758178710938,
      "activations/layer8_attention_weight_max": 40.075199127197266,
      "activations/layer8_attention_weight_min": -40.8109245300293,
      "activations/layer9_attention_weight_max": 30.80544090270996,
      "activations/layer9_attention_weight_min": -33.316890716552734,
      "epoch": 16.47,
      "learning_rate": 4.4215530303030305e-05,
      "loss": 2.7461,
      "step": 283500
    },
    {
      "activations/layer0_attention_weight_max": 16.183948516845703,
      "activations/layer0_attention_weight_min": -12.833681106567383,
      "activations/layer10_attention_weight_max": 35.17745590209961,
      "activations/layer10_attention_weight_min": -35.459136962890625,
      "activations/layer11_attention_weight_max": 35.371437072753906,
      "activations/layer11_attention_weight_min": -35.97510528564453,
      "activations/layer12_attention_weight_max": 26.829891204833984,
      "activations/layer12_attention_weight_min": -26.1240291595459,
      "activations/layer13_attention_weight_max": 39.753211975097656,
      "activations/layer13_attention_weight_min": -33.69721603393555,
      "activations/layer14_attention_weight_max": 41.96760559082031,
      "activations/layer14_attention_weight_min": -36.44117736816406,
      "activations/layer15_attention_weight_max": 37.724853515625,
      "activations/layer15_attention_weight_min": -35.01966094970703,
      "activations/layer16_attention_weight_max": 32.06334686279297,
      "activations/layer16_attention_weight_min": -28.179122924804688,
      "activations/layer17_attention_weight_max": 33.07323455810547,
      "activations/layer17_attention_weight_min": -26.3155460357666,
      "activations/layer18_attention_weight_max": 33.481964111328125,
      "activations/layer18_attention_weight_min": -24.668197631835938,
      "activations/layer19_attention_weight_max": 34.32414627075195,
      "activations/layer19_attention_weight_min": -33.229591369628906,
      "activations/layer1_attention_weight_max": 16.242183685302734,
      "activations/layer1_attention_weight_min": -13.013721466064453,
      "activations/layer20_attention_weight_max": 29.156036376953125,
      "activations/layer20_attention_weight_min": -26.66839027404785,
      "activations/layer21_attention_weight_max": 29.912845611572266,
      "activations/layer21_attention_weight_min": -25.94736099243164,
      "activations/layer22_attention_weight_max": 39.473506927490234,
      "activations/layer22_attention_weight_min": -32.40016174316406,
      "activations/layer23_attention_weight_max": 30.69005584716797,
      "activations/layer23_attention_weight_min": -27.943439483642578,
      "activations/layer2_attention_weight_max": 33.77534103393555,
      "activations/layer2_attention_weight_min": -32.00357437133789,
      "activations/layer3_attention_weight_max": 94.82738494873047,
      "activations/layer3_attention_weight_min": -96.30908203125,
      "activations/layer4_attention_weight_max": 105.24242401123047,
      "activations/layer4_attention_weight_min": -106.80877685546875,
      "activations/layer5_attention_weight_max": 59.314369201660156,
      "activations/layer5_attention_weight_min": -62.009307861328125,
      "activations/layer6_attention_weight_max": 45.26857376098633,
      "activations/layer6_attention_weight_min": -45.642539978027344,
      "activations/layer7_attention_weight_max": 101.55836486816406,
      "activations/layer7_attention_weight_min": -92.2417984008789,
      "activations/layer8_attention_weight_max": 43.979373931884766,
      "activations/layer8_attention_weight_min": -43.404659271240234,
      "activations/layer9_attention_weight_max": 33.767208099365234,
      "activations/layer9_attention_weight_min": -35.04544448852539,
      "epoch": 16.48,
      "learning_rate": 4.4196590909090907e-05,
      "loss": 2.7346,
      "step": 283550
    },
    {
      "activations/layer0_attention_weight_max": 15.177456855773926,
      "activations/layer0_attention_weight_min": -12.85763168334961,
      "activations/layer10_attention_weight_max": 36.05345153808594,
      "activations/layer10_attention_weight_min": -36.058815002441406,
      "activations/layer11_attention_weight_max": 35.631866455078125,
      "activations/layer11_attention_weight_min": -35.267578125,
      "activations/layer12_attention_weight_max": 28.333768844604492,
      "activations/layer12_attention_weight_min": -27.76176643371582,
      "activations/layer13_attention_weight_max": 44.85676193237305,
      "activations/layer13_attention_weight_min": -37.900203704833984,
      "activations/layer14_attention_weight_max": 42.61545181274414,
      "activations/layer14_attention_weight_min": -37.76258087158203,
      "activations/layer15_attention_weight_max": 39.41364669799805,
      "activations/layer15_attention_weight_min": -34.14152908325195,
      "activations/layer16_attention_weight_max": 28.748279571533203,
      "activations/layer16_attention_weight_min": -27.090517044067383,
      "activations/layer17_attention_weight_max": 29.748891830444336,
      "activations/layer17_attention_weight_min": -25.58203887939453,
      "activations/layer18_attention_weight_max": 31.214923858642578,
      "activations/layer18_attention_weight_min": -25.457040786743164,
      "activations/layer19_attention_weight_max": 33.588016510009766,
      "activations/layer19_attention_weight_min": -33.06512451171875,
      "activations/layer1_attention_weight_max": 15.606232643127441,
      "activations/layer1_attention_weight_min": -14.314970970153809,
      "activations/layer20_attention_weight_max": 28.674699783325195,
      "activations/layer20_attention_weight_min": -25.464126586914062,
      "activations/layer21_attention_weight_max": 30.550338745117188,
      "activations/layer21_attention_weight_min": -26.484237670898438,
      "activations/layer22_attention_weight_max": 44.01369857788086,
      "activations/layer22_attention_weight_min": -30.834285736083984,
      "activations/layer23_attention_weight_max": 32.84143829345703,
      "activations/layer23_attention_weight_min": -29.515140533447266,
      "activations/layer2_attention_weight_max": 31.925434112548828,
      "activations/layer2_attention_weight_min": -31.64789581298828,
      "activations/layer3_attention_weight_max": 95.5196304321289,
      "activations/layer3_attention_weight_min": -94.71704864501953,
      "activations/layer4_attention_weight_max": 110.05065155029297,
      "activations/layer4_attention_weight_min": -107.19075775146484,
      "activations/layer5_attention_weight_max": 50.42877960205078,
      "activations/layer5_attention_weight_min": -64.2255859375,
      "activations/layer6_attention_weight_max": 46.47355651855469,
      "activations/layer6_attention_weight_min": -46.98343276977539,
      "activations/layer7_attention_weight_max": 92.4856185913086,
      "activations/layer7_attention_weight_min": -96.78739929199219,
      "activations/layer8_attention_weight_max": 45.18211364746094,
      "activations/layer8_attention_weight_min": -46.65877914428711,
      "activations/layer9_attention_weight_max": 35.74216079711914,
      "activations/layer9_attention_weight_min": -35.88972091674805,
      "epoch": 16.48,
      "learning_rate": 4.417765151515151e-05,
      "loss": 2.7334,
      "step": 283600
    },
    {
      "activations/layer0_attention_weight_max": 16.1613826751709,
      "activations/layer0_attention_weight_min": -12.173842430114746,
      "activations/layer10_attention_weight_max": 33.82481384277344,
      "activations/layer10_attention_weight_min": -35.33694076538086,
      "activations/layer11_attention_weight_max": 35.47818374633789,
      "activations/layer11_attention_weight_min": -35.67304992675781,
      "activations/layer12_attention_weight_max": 27.770511627197266,
      "activations/layer12_attention_weight_min": -27.09390640258789,
      "activations/layer13_attention_weight_max": 41.19438552856445,
      "activations/layer13_attention_weight_min": -35.056365966796875,
      "activations/layer14_attention_weight_max": 42.8568115234375,
      "activations/layer14_attention_weight_min": -39.524497985839844,
      "activations/layer15_attention_weight_max": 38.9683952331543,
      "activations/layer15_attention_weight_min": -35.45955276489258,
      "activations/layer16_attention_weight_max": 31.367761611938477,
      "activations/layer16_attention_weight_min": -27.998918533325195,
      "activations/layer17_attention_weight_max": 35.76383590698242,
      "activations/layer17_attention_weight_min": -26.058570861816406,
      "activations/layer18_attention_weight_max": 35.09797286987305,
      "activations/layer18_attention_weight_min": -24.746084213256836,
      "activations/layer19_attention_weight_max": 36.639747619628906,
      "activations/layer19_attention_weight_min": -30.73105239868164,
      "activations/layer1_attention_weight_max": 16.383317947387695,
      "activations/layer1_attention_weight_min": -15.1093168258667,
      "activations/layer20_attention_weight_max": 33.602500915527344,
      "activations/layer20_attention_weight_min": -24.267229080200195,
      "activations/layer21_attention_weight_max": 31.860586166381836,
      "activations/layer21_attention_weight_min": -23.012683868408203,
      "activations/layer22_attention_weight_max": 46.98611068725586,
      "activations/layer22_attention_weight_min": -30.856775283813477,
      "activations/layer23_attention_weight_max": 38.25412368774414,
      "activations/layer23_attention_weight_min": -27.035879135131836,
      "activations/layer2_attention_weight_max": 34.486663818359375,
      "activations/layer2_attention_weight_min": -32.79962158203125,
      "activations/layer3_attention_weight_max": 100.22296905517578,
      "activations/layer3_attention_weight_min": -96.37660217285156,
      "activations/layer4_attention_weight_max": 106.78081512451172,
      "activations/layer4_attention_weight_min": -108.97637176513672,
      "activations/layer5_attention_weight_max": 54.760005950927734,
      "activations/layer5_attention_weight_min": -64.48119354248047,
      "activations/layer6_attention_weight_max": 46.48460006713867,
      "activations/layer6_attention_weight_min": -48.34041976928711,
      "activations/layer7_attention_weight_max": 94.07682800292969,
      "activations/layer7_attention_weight_min": -94.52447509765625,
      "activations/layer8_attention_weight_max": 41.572418212890625,
      "activations/layer8_attention_weight_min": -43.55341720581055,
      "activations/layer9_attention_weight_max": 34.12495040893555,
      "activations/layer9_attention_weight_min": -34.082908630371094,
      "epoch": 16.48,
      "learning_rate": 4.4158712121212116e-05,
      "loss": 2.7379,
      "step": 283650
    },
    {
      "activations/layer0_attention_weight_max": 15.699874877929688,
      "activations/layer0_attention_weight_min": -12.333643913269043,
      "activations/layer10_attention_weight_max": 35.952091217041016,
      "activations/layer10_attention_weight_min": -33.79731369018555,
      "activations/layer11_attention_weight_max": 35.50743103027344,
      "activations/layer11_attention_weight_min": -34.57883834838867,
      "activations/layer12_attention_weight_max": 32.40711212158203,
      "activations/layer12_attention_weight_min": -27.17618179321289,
      "activations/layer13_attention_weight_max": 44.76028823852539,
      "activations/layer13_attention_weight_min": -36.353309631347656,
      "activations/layer14_attention_weight_max": 43.569190979003906,
      "activations/layer14_attention_weight_min": -41.804840087890625,
      "activations/layer15_attention_weight_max": 43.160545349121094,
      "activations/layer15_attention_weight_min": -36.47270584106445,
      "activations/layer16_attention_weight_max": 31.62982177734375,
      "activations/layer16_attention_weight_min": -29.096101760864258,
      "activations/layer17_attention_weight_max": 32.19343948364258,
      "activations/layer17_attention_weight_min": -26.914270401000977,
      "activations/layer18_attention_weight_max": 32.332454681396484,
      "activations/layer18_attention_weight_min": -26.495954513549805,
      "activations/layer19_attention_weight_max": 38.24263381958008,
      "activations/layer19_attention_weight_min": -33.0614013671875,
      "activations/layer1_attention_weight_max": 16.931692123413086,
      "activations/layer1_attention_weight_min": -15.414641380310059,
      "activations/layer20_attention_weight_max": 31.685195922851562,
      "activations/layer20_attention_weight_min": -26.401620864868164,
      "activations/layer21_attention_weight_max": 30.41094207763672,
      "activations/layer21_attention_weight_min": -26.102331161499023,
      "activations/layer22_attention_weight_max": 41.11345672607422,
      "activations/layer22_attention_weight_min": -30.740713119506836,
      "activations/layer23_attention_weight_max": 35.68324661254883,
      "activations/layer23_attention_weight_min": -28.719154357910156,
      "activations/layer2_attention_weight_max": 32.74443054199219,
      "activations/layer2_attention_weight_min": -31.05315399169922,
      "activations/layer3_attention_weight_max": 98.02006530761719,
      "activations/layer3_attention_weight_min": -95.62759399414062,
      "activations/layer4_attention_weight_max": 107.108154296875,
      "activations/layer4_attention_weight_min": -105.3095703125,
      "activations/layer5_attention_weight_max": 55.441261291503906,
      "activations/layer5_attention_weight_min": -64.59534454345703,
      "activations/layer6_attention_weight_max": 46.81925964355469,
      "activations/layer6_attention_weight_min": -45.332645416259766,
      "activations/layer7_attention_weight_max": 100.3294906616211,
      "activations/layer7_attention_weight_min": -93.49391174316406,
      "activations/layer8_attention_weight_max": 41.12409591674805,
      "activations/layer8_attention_weight_min": -41.754539489746094,
      "activations/layer9_attention_weight_max": 32.55912780761719,
      "activations/layer9_attention_weight_min": -34.04033660888672,
      "epoch": 16.48,
      "learning_rate": 4.4139772727272725e-05,
      "loss": 2.7412,
      "step": 283700
    },
    {
      "activations/layer0_attention_weight_max": 15.836034774780273,
      "activations/layer0_attention_weight_min": -12.859829902648926,
      "activations/layer10_attention_weight_max": 39.31163024902344,
      "activations/layer10_attention_weight_min": -36.24945068359375,
      "activations/layer11_attention_weight_max": 37.20244216918945,
      "activations/layer11_attention_weight_min": -36.27381896972656,
      "activations/layer12_attention_weight_max": 28.196653366088867,
      "activations/layer12_attention_weight_min": -25.74631690979004,
      "activations/layer13_attention_weight_max": 41.97404479980469,
      "activations/layer13_attention_weight_min": -36.569732666015625,
      "activations/layer14_attention_weight_max": 47.75941848754883,
      "activations/layer14_attention_weight_min": -38.61119079589844,
      "activations/layer15_attention_weight_max": 41.18460464477539,
      "activations/layer15_attention_weight_min": -33.81816864013672,
      "activations/layer16_attention_weight_max": 31.887832641601562,
      "activations/layer16_attention_weight_min": -32.92429733276367,
      "activations/layer17_attention_weight_max": 30.265703201293945,
      "activations/layer17_attention_weight_min": -27.039365768432617,
      "activations/layer18_attention_weight_max": 33.11650085449219,
      "activations/layer18_attention_weight_min": -26.046323776245117,
      "activations/layer19_attention_weight_max": 33.84506607055664,
      "activations/layer19_attention_weight_min": -33.61466979980469,
      "activations/layer1_attention_weight_max": 16.671056747436523,
      "activations/layer1_attention_weight_min": -15.772218704223633,
      "activations/layer20_attention_weight_max": 28.529512405395508,
      "activations/layer20_attention_weight_min": -27.13753890991211,
      "activations/layer21_attention_weight_max": 29.699119567871094,
      "activations/layer21_attention_weight_min": -24.77977180480957,
      "activations/layer22_attention_weight_max": 40.09126281738281,
      "activations/layer22_attention_weight_min": -31.96463394165039,
      "activations/layer23_attention_weight_max": 30.817474365234375,
      "activations/layer23_attention_weight_min": -28.43074607849121,
      "activations/layer2_attention_weight_max": 34.512638092041016,
      "activations/layer2_attention_weight_min": -33.247764587402344,
      "activations/layer3_attention_weight_max": 101.2694320678711,
      "activations/layer3_attention_weight_min": -100.4161605834961,
      "activations/layer4_attention_weight_max": 111.45610809326172,
      "activations/layer4_attention_weight_min": -108.37438201904297,
      "activations/layer5_attention_weight_max": 52.02125549316406,
      "activations/layer5_attention_weight_min": -64.98603820800781,
      "activations/layer6_attention_weight_max": 46.10153579711914,
      "activations/layer6_attention_weight_min": -45.584434509277344,
      "activations/layer7_attention_weight_max": 95.1267318725586,
      "activations/layer7_attention_weight_min": -94.65335845947266,
      "activations/layer8_attention_weight_max": 45.736080169677734,
      "activations/layer8_attention_weight_min": -45.77309036254883,
      "activations/layer9_attention_weight_max": 34.93142318725586,
      "activations/layer9_attention_weight_min": -34.42127990722656,
      "epoch": 16.49,
      "learning_rate": 4.412083333333333e-05,
      "loss": 2.7406,
      "step": 283750
    },
    {
      "activations/layer0_attention_weight_max": 15.745091438293457,
      "activations/layer0_attention_weight_min": -12.831828117370605,
      "activations/layer10_attention_weight_max": 36.55231475830078,
      "activations/layer10_attention_weight_min": -34.420555114746094,
      "activations/layer11_attention_weight_max": 35.3854866027832,
      "activations/layer11_attention_weight_min": -34.852294921875,
      "activations/layer12_attention_weight_max": 28.671358108520508,
      "activations/layer12_attention_weight_min": -26.401939392089844,
      "activations/layer13_attention_weight_max": 44.83101272583008,
      "activations/layer13_attention_weight_min": -37.69818878173828,
      "activations/layer14_attention_weight_max": 47.412696838378906,
      "activations/layer14_attention_weight_min": -38.908687591552734,
      "activations/layer15_attention_weight_max": 43.64234924316406,
      "activations/layer15_attention_weight_min": -35.762184143066406,
      "activations/layer16_attention_weight_max": 30.15028953552246,
      "activations/layer16_attention_weight_min": -29.466249465942383,
      "activations/layer17_attention_weight_max": 33.501407623291016,
      "activations/layer17_attention_weight_min": -28.182138442993164,
      "activations/layer18_attention_weight_max": 30.570110321044922,
      "activations/layer18_attention_weight_min": -26.201696395874023,
      "activations/layer19_attention_weight_max": 35.45698165893555,
      "activations/layer19_attention_weight_min": -31.82710838317871,
      "activations/layer1_attention_weight_max": 17.13729476928711,
      "activations/layer1_attention_weight_min": -14.505838394165039,
      "activations/layer20_attention_weight_max": 32.66374206542969,
      "activations/layer20_attention_weight_min": -25.763370513916016,
      "activations/layer21_attention_weight_max": 27.814908981323242,
      "activations/layer21_attention_weight_min": -25.50701141357422,
      "activations/layer22_attention_weight_max": 43.01215744018555,
      "activations/layer22_attention_weight_min": -30.533830642700195,
      "activations/layer23_attention_weight_max": 34.012962341308594,
      "activations/layer23_attention_weight_min": -28.880413055419922,
      "activations/layer2_attention_weight_max": 32.33383560180664,
      "activations/layer2_attention_weight_min": -32.148834228515625,
      "activations/layer3_attention_weight_max": 96.3308334350586,
      "activations/layer3_attention_weight_min": -97.46929168701172,
      "activations/layer4_attention_weight_max": 104.90290832519531,
      "activations/layer4_attention_weight_min": -110.88594055175781,
      "activations/layer5_attention_weight_max": 51.73447799682617,
      "activations/layer5_attention_weight_min": -63.16436767578125,
      "activations/layer6_attention_weight_max": 43.60663986206055,
      "activations/layer6_attention_weight_min": -47.05280685424805,
      "activations/layer7_attention_weight_max": 93.68780517578125,
      "activations/layer7_attention_weight_min": -90.87873077392578,
      "activations/layer8_attention_weight_max": 40.916778564453125,
      "activations/layer8_attention_weight_min": -41.368194580078125,
      "activations/layer9_attention_weight_max": 32.66179656982422,
      "activations/layer9_attention_weight_min": -33.20232391357422,
      "epoch": 16.49,
      "learning_rate": 4.4101893939393934e-05,
      "loss": 2.7381,
      "step": 283800
    },
    {
      "activations/layer0_attention_weight_max": 15.604215621948242,
      "activations/layer0_attention_weight_min": -12.318537712097168,
      "activations/layer10_attention_weight_max": 35.21714782714844,
      "activations/layer10_attention_weight_min": -33.96492004394531,
      "activations/layer11_attention_weight_max": 32.31085205078125,
      "activations/layer11_attention_weight_min": -33.030052185058594,
      "activations/layer12_attention_weight_max": 27.30317497253418,
      "activations/layer12_attention_weight_min": -27.307518005371094,
      "activations/layer13_attention_weight_max": 40.49854278564453,
      "activations/layer13_attention_weight_min": -35.675315856933594,
      "activations/layer14_attention_weight_max": 43.78042221069336,
      "activations/layer14_attention_weight_min": -38.28754806518555,
      "activations/layer15_attention_weight_max": 39.79656219482422,
      "activations/layer15_attention_weight_min": -35.16116714477539,
      "activations/layer16_attention_weight_max": 29.09904670715332,
      "activations/layer16_attention_weight_min": -28.323795318603516,
      "activations/layer17_attention_weight_max": 33.42448806762695,
      "activations/layer17_attention_weight_min": -26.394596099853516,
      "activations/layer18_attention_weight_max": 32.751976013183594,
      "activations/layer18_attention_weight_min": -25.54308319091797,
      "activations/layer19_attention_weight_max": 33.05690383911133,
      "activations/layer19_attention_weight_min": -32.97711944580078,
      "activations/layer1_attention_weight_max": 17.995481491088867,
      "activations/layer1_attention_weight_min": -15.141786575317383,
      "activations/layer20_attention_weight_max": 31.521493911743164,
      "activations/layer20_attention_weight_min": -26.68328857421875,
      "activations/layer21_attention_weight_max": 32.38904571533203,
      "activations/layer21_attention_weight_min": -26.751338958740234,
      "activations/layer22_attention_weight_max": 40.860557556152344,
      "activations/layer22_attention_weight_min": -32.571876525878906,
      "activations/layer23_attention_weight_max": 33.1175537109375,
      "activations/layer23_attention_weight_min": -28.1204891204834,
      "activations/layer2_attention_weight_max": 31.941207885742188,
      "activations/layer2_attention_weight_min": -30.97927474975586,
      "activations/layer3_attention_weight_max": 95.16331481933594,
      "activations/layer3_attention_weight_min": -93.51451873779297,
      "activations/layer4_attention_weight_max": 101.86770629882812,
      "activations/layer4_attention_weight_min": -106.19476318359375,
      "activations/layer5_attention_weight_max": 50.01932907104492,
      "activations/layer5_attention_weight_min": -62.00708770751953,
      "activations/layer6_attention_weight_max": 45.59358596801758,
      "activations/layer6_attention_weight_min": -44.317222595214844,
      "activations/layer7_attention_weight_max": 89.8811264038086,
      "activations/layer7_attention_weight_min": -91.5523452758789,
      "activations/layer8_attention_weight_max": 40.84035110473633,
      "activations/layer8_attention_weight_min": -42.78860855102539,
      "activations/layer9_attention_weight_max": 31.504545211791992,
      "activations/layer9_attention_weight_min": -33.6651611328125,
      "epoch": 16.49,
      "learning_rate": 4.4082954545454536e-05,
      "loss": 2.7599,
      "step": 283850
    },
    {
      "activations/layer0_attention_weight_max": 15.990771293640137,
      "activations/layer0_attention_weight_min": -12.458672523498535,
      "activations/layer10_attention_weight_max": 47.2131462097168,
      "activations/layer10_attention_weight_min": -46.637142181396484,
      "activations/layer11_attention_weight_max": 46.33262634277344,
      "activations/layer11_attention_weight_min": -46.96946716308594,
      "activations/layer12_attention_weight_max": 34.633811950683594,
      "activations/layer12_attention_weight_min": -29.996505737304688,
      "activations/layer13_attention_weight_max": 49.32987594604492,
      "activations/layer13_attention_weight_min": -39.804317474365234,
      "activations/layer14_attention_weight_max": 53.69039535522461,
      "activations/layer14_attention_weight_min": -44.72966766357422,
      "activations/layer15_attention_weight_max": 56.32782745361328,
      "activations/layer15_attention_weight_min": -43.33732223510742,
      "activations/layer16_attention_weight_max": 35.33090591430664,
      "activations/layer16_attention_weight_min": -26.77372932434082,
      "activations/layer17_attention_weight_max": 29.93226432800293,
      "activations/layer17_attention_weight_min": -27.040800094604492,
      "activations/layer18_attention_weight_max": 31.466054916381836,
      "activations/layer18_attention_weight_min": -26.719947814941406,
      "activations/layer19_attention_weight_max": 36.82181930541992,
      "activations/layer19_attention_weight_min": -32.13729476928711,
      "activations/layer1_attention_weight_max": 15.522209167480469,
      "activations/layer1_attention_weight_min": -15.685784339904785,
      "activations/layer20_attention_weight_max": 31.78744125366211,
      "activations/layer20_attention_weight_min": -26.643659591674805,
      "activations/layer21_attention_weight_max": 29.214488983154297,
      "activations/layer21_attention_weight_min": -26.381704330444336,
      "activations/layer22_attention_weight_max": 41.76171112060547,
      "activations/layer22_attention_weight_min": -32.487789154052734,
      "activations/layer23_attention_weight_max": 31.906452178955078,
      "activations/layer23_attention_weight_min": -23.97566032409668,
      "activations/layer2_attention_weight_max": 39.916282653808594,
      "activations/layer2_attention_weight_min": -37.27943801879883,
      "activations/layer3_attention_weight_max": 107.41920471191406,
      "activations/layer3_attention_weight_min": -103.89864349365234,
      "activations/layer4_attention_weight_max": 119.28429412841797,
      "activations/layer4_attention_weight_min": -114.5711898803711,
      "activations/layer5_attention_weight_max": 56.1338005065918,
      "activations/layer5_attention_weight_min": -61.09735107421875,
      "activations/layer6_attention_weight_max": 49.47010803222656,
      "activations/layer6_attention_weight_min": -47.88924789428711,
      "activations/layer7_attention_weight_max": 117.71244049072266,
      "activations/layer7_attention_weight_min": -104.08444213867188,
      "activations/layer8_attention_weight_max": 55.033546447753906,
      "activations/layer8_attention_weight_min": -51.90379333496094,
      "activations/layer9_attention_weight_max": 43.085853576660156,
      "activations/layer9_attention_weight_min": -42.24686050415039,
      "epoch": 16.5,
      "learning_rate": 4.406401515151515e-05,
      "loss": 2.7408,
      "step": 283900
    },
    {
      "activations/layer0_attention_weight_max": 15.382420539855957,
      "activations/layer0_attention_weight_min": -12.177424430847168,
      "activations/layer10_attention_weight_max": 32.09508514404297,
      "activations/layer10_attention_weight_min": -34.909828186035156,
      "activations/layer11_attention_weight_max": 31.99616241455078,
      "activations/layer11_attention_weight_min": -34.54756546020508,
      "activations/layer12_attention_weight_max": 27.16216468811035,
      "activations/layer12_attention_weight_min": -26.155845642089844,
      "activations/layer13_attention_weight_max": 40.488033294677734,
      "activations/layer13_attention_weight_min": -36.8460693359375,
      "activations/layer14_attention_weight_max": 43.74005126953125,
      "activations/layer14_attention_weight_min": -41.03425979614258,
      "activations/layer15_attention_weight_max": 38.20609664916992,
      "activations/layer15_attention_weight_min": -37.32426071166992,
      "activations/layer16_attention_weight_max": 28.98760414123535,
      "activations/layer16_attention_weight_min": -29.08621597290039,
      "activations/layer17_attention_weight_max": 28.39469337463379,
      "activations/layer17_attention_weight_min": -27.352127075195312,
      "activations/layer18_attention_weight_max": 29.91035270690918,
      "activations/layer18_attention_weight_min": -25.65873146057129,
      "activations/layer19_attention_weight_max": 32.6015739440918,
      "activations/layer19_attention_weight_min": -35.08247756958008,
      "activations/layer1_attention_weight_max": 15.731534004211426,
      "activations/layer1_attention_weight_min": -14.137200355529785,
      "activations/layer20_attention_weight_max": 27.613475799560547,
      "activations/layer20_attention_weight_min": -27.313913345336914,
      "activations/layer21_attention_weight_max": 27.134855270385742,
      "activations/layer21_attention_weight_min": -26.744503021240234,
      "activations/layer22_attention_weight_max": 43.060951232910156,
      "activations/layer22_attention_weight_min": -33.09382629394531,
      "activations/layer23_attention_weight_max": 32.02676010131836,
      "activations/layer23_attention_weight_min": -28.603588104248047,
      "activations/layer2_attention_weight_max": 30.742958068847656,
      "activations/layer2_attention_weight_min": -30.71276092529297,
      "activations/layer3_attention_weight_max": 86.1583480834961,
      "activations/layer3_attention_weight_min": -93.76344299316406,
      "activations/layer4_attention_weight_max": 98.40768432617188,
      "activations/layer4_attention_weight_min": -103.45423889160156,
      "activations/layer5_attention_weight_max": 48.463600158691406,
      "activations/layer5_attention_weight_min": -58.40996551513672,
      "activations/layer6_attention_weight_max": 41.192710876464844,
      "activations/layer6_attention_weight_min": -45.18016052246094,
      "activations/layer7_attention_weight_max": 88.16244506835938,
      "activations/layer7_attention_weight_min": -92.65221405029297,
      "activations/layer8_attention_weight_max": 39.0213737487793,
      "activations/layer8_attention_weight_min": -43.91267013549805,
      "activations/layer9_attention_weight_max": 30.425535202026367,
      "activations/layer9_attention_weight_min": -35.54219055175781,
      "epoch": 16.5,
      "learning_rate": 4.404507575757575e-05,
      "loss": 2.7433,
      "step": 283950
    },
    {
      "activations/layer0_attention_weight_max": 16.150482177734375,
      "activations/layer0_attention_weight_min": -12.744844436645508,
      "activations/layer10_attention_weight_max": 34.71651840209961,
      "activations/layer10_attention_weight_min": -35.14055252075195,
      "activations/layer11_attention_weight_max": 33.81642150878906,
      "activations/layer11_attention_weight_min": -34.6073112487793,
      "activations/layer12_attention_weight_max": 28.733810424804688,
      "activations/layer12_attention_weight_min": -27.853654861450195,
      "activations/layer13_attention_weight_max": 44.10775375366211,
      "activations/layer13_attention_weight_min": -37.44096374511719,
      "activations/layer14_attention_weight_max": 43.75156784057617,
      "activations/layer14_attention_weight_min": -39.39456558227539,
      "activations/layer15_attention_weight_max": 39.75870895385742,
      "activations/layer15_attention_weight_min": -35.99852752685547,
      "activations/layer16_attention_weight_max": 34.04234313964844,
      "activations/layer16_attention_weight_min": -27.673263549804688,
      "activations/layer17_attention_weight_max": 32.46151351928711,
      "activations/layer17_attention_weight_min": -26.40102195739746,
      "activations/layer18_attention_weight_max": 33.64381408691406,
      "activations/layer18_attention_weight_min": -27.366558074951172,
      "activations/layer19_attention_weight_max": 40.9533576965332,
      "activations/layer19_attention_weight_min": -35.18792724609375,
      "activations/layer1_attention_weight_max": 15.487275123596191,
      "activations/layer1_attention_weight_min": -14.410130500793457,
      "activations/layer20_attention_weight_max": 32.84375,
      "activations/layer20_attention_weight_min": -26.7559757232666,
      "activations/layer21_attention_weight_max": 30.424522399902344,
      "activations/layer21_attention_weight_min": -25.53875732421875,
      "activations/layer22_attention_weight_max": 46.88077926635742,
      "activations/layer22_attention_weight_min": -32.67527389526367,
      "activations/layer23_attention_weight_max": 35.94048309326172,
      "activations/layer23_attention_weight_min": -27.202598571777344,
      "activations/layer2_attention_weight_max": 32.67003631591797,
      "activations/layer2_attention_weight_min": -33.93597412109375,
      "activations/layer3_attention_weight_max": 93.97481536865234,
      "activations/layer3_attention_weight_min": -99.3751220703125,
      "activations/layer4_attention_weight_max": 106.08192443847656,
      "activations/layer4_attention_weight_min": -105.50090789794922,
      "activations/layer5_attention_weight_max": 50.3287467956543,
      "activations/layer5_attention_weight_min": -62.50782012939453,
      "activations/layer6_attention_weight_max": 43.43800735473633,
      "activations/layer6_attention_weight_min": -48.21590042114258,
      "activations/layer7_attention_weight_max": 92.14776611328125,
      "activations/layer7_attention_weight_min": -98.48597717285156,
      "activations/layer8_attention_weight_max": 41.53435134887695,
      "activations/layer8_attention_weight_min": -46.193511962890625,
      "activations/layer9_attention_weight_max": 33.365474700927734,
      "activations/layer9_attention_weight_min": -35.619873046875,
      "epoch": 16.5,
      "learning_rate": 4.402613636363636e-05,
      "loss": 2.7365,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4634,
      "eval_samples_per_second": 507.361,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4634,
      "eval_openwebtext_samples_per_second": 507.361,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9833,
      "eval_wikitext_samples_per_second": 229.918,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_lambada_loss": 2.373046875,
      "eval_lambada_ppl": 10.730035606033903,
      "eval_lambada_runtime": 9.5582,
      "eval_lambada_samples_per_second": 509.407,
      "step": 284000
    },
    {
      "activations/layer0_attention_weight_max": 16.742176055908203,
      "activations/layer0_attention_weight_min": -11.92729377746582,
      "activations/layer10_attention_weight_max": 32.984588623046875,
      "activations/layer10_attention_weight_min": -32.44559860229492,
      "activations/layer11_attention_weight_max": 32.59971618652344,
      "activations/layer11_attention_weight_min": -33.59365463256836,
      "activations/layer12_attention_weight_max": 24.88459587097168,
      "activations/layer12_attention_weight_min": -34.38774490356445,
      "activations/layer13_attention_weight_max": 37.804134368896484,
      "activations/layer13_attention_weight_min": -35.655052185058594,
      "activations/layer14_attention_weight_max": 40.077301025390625,
      "activations/layer14_attention_weight_min": -40.5423698425293,
      "activations/layer15_attention_weight_max": 37.72371292114258,
      "activations/layer15_attention_weight_min": -37.43630599975586,
      "activations/layer16_attention_weight_max": 32.34006118774414,
      "activations/layer16_attention_weight_min": -27.22418785095215,
      "activations/layer17_attention_weight_max": 31.5537052154541,
      "activations/layer17_attention_weight_min": -24.529172897338867,
      "activations/layer18_attention_weight_max": 35.060882568359375,
      "activations/layer18_attention_weight_min": -26.401098251342773,
      "activations/layer19_attention_weight_max": 32.16367721557617,
      "activations/layer19_attention_weight_min": -32.02299499511719,
      "activations/layer1_attention_weight_max": 17.485492706298828,
      "activations/layer1_attention_weight_min": -14.418183326721191,
      "activations/layer20_attention_weight_max": 25.82562828063965,
      "activations/layer20_attention_weight_min": -27.561479568481445,
      "activations/layer21_attention_weight_max": 29.11624526977539,
      "activations/layer21_attention_weight_min": -23.851465225219727,
      "activations/layer22_attention_weight_max": 40.69287109375,
      "activations/layer22_attention_weight_min": -30.2481746673584,
      "activations/layer23_attention_weight_max": 31.662532806396484,
      "activations/layer23_attention_weight_min": -26.831340789794922,
      "activations/layer2_attention_weight_max": 32.09065246582031,
      "activations/layer2_attention_weight_min": -30.478328704833984,
      "activations/layer3_attention_weight_max": 91.98567962646484,
      "activations/layer3_attention_weight_min": -91.3302001953125,
      "activations/layer4_attention_weight_max": 104.61866760253906,
      "activations/layer4_attention_weight_min": -101.62490844726562,
      "activations/layer5_attention_weight_max": 51.29670715332031,
      "activations/layer5_attention_weight_min": -59.758705139160156,
      "activations/layer6_attention_weight_max": 41.43004608154297,
      "activations/layer6_attention_weight_min": -42.40960693359375,
      "activations/layer7_attention_weight_max": 95.12352752685547,
      "activations/layer7_attention_weight_min": -89.5254135131836,
      "activations/layer8_attention_weight_max": 39.45680618286133,
      "activations/layer8_attention_weight_min": -40.433837890625,
      "activations/layer9_attention_weight_max": 29.379535675048828,
      "activations/layer9_attention_weight_min": -33.09920120239258,
      "epoch": 16.5,
      "learning_rate": 4.400719696969696e-05,
      "loss": 2.7335,
      "step": 284050
    },
    {
      "activations/layer0_attention_weight_max": 15.114079475402832,
      "activations/layer0_attention_weight_min": -12.11467170715332,
      "activations/layer10_attention_weight_max": 33.00651168823242,
      "activations/layer10_attention_weight_min": -37.02644729614258,
      "activations/layer11_attention_weight_max": 33.57038116455078,
      "activations/layer11_attention_weight_min": -35.7629280090332,
      "activations/layer12_attention_weight_max": 30.349306106567383,
      "activations/layer12_attention_weight_min": -29.59878158569336,
      "activations/layer13_attention_weight_max": 44.372215270996094,
      "activations/layer13_attention_weight_min": -36.85017395019531,
      "activations/layer14_attention_weight_max": 46.15121078491211,
      "activations/layer14_attention_weight_min": -40.828033447265625,
      "activations/layer15_attention_weight_max": 44.75507736206055,
      "activations/layer15_attention_weight_min": -42.543190002441406,
      "activations/layer16_attention_weight_max": 34.1102294921875,
      "activations/layer16_attention_weight_min": -28.97420883178711,
      "activations/layer17_attention_weight_max": 33.688480377197266,
      "activations/layer17_attention_weight_min": -32.64697265625,
      "activations/layer18_attention_weight_max": 35.23946762084961,
      "activations/layer18_attention_weight_min": -32.6632194519043,
      "activations/layer19_attention_weight_max": 33.683448791503906,
      "activations/layer19_attention_weight_min": -34.09770965576172,
      "activations/layer1_attention_weight_max": 16.29240608215332,
      "activations/layer1_attention_weight_min": -14.764165878295898,
      "activations/layer20_attention_weight_max": 29.14259910583496,
      "activations/layer20_attention_weight_min": -26.241056442260742,
      "activations/layer21_attention_weight_max": 27.868637084960938,
      "activations/layer21_attention_weight_min": -25.904794692993164,
      "activations/layer22_attention_weight_max": 41.63792037963867,
      "activations/layer22_attention_weight_min": -32.89137268066406,
      "activations/layer23_attention_weight_max": 30.205947875976562,
      "activations/layer23_attention_weight_min": -26.702190399169922,
      "activations/layer2_attention_weight_max": 31.200664520263672,
      "activations/layer2_attention_weight_min": -33.22553634643555,
      "activations/layer3_attention_weight_max": 96.95661163330078,
      "activations/layer3_attention_weight_min": -101.18708801269531,
      "activations/layer4_attention_weight_max": 109.77050018310547,
      "activations/layer4_attention_weight_min": -110.01397705078125,
      "activations/layer5_attention_weight_max": 52.38932418823242,
      "activations/layer5_attention_weight_min": -59.383636474609375,
      "activations/layer6_attention_weight_max": 43.99825668334961,
      "activations/layer6_attention_weight_min": -46.70353317260742,
      "activations/layer7_attention_weight_max": 91.32111358642578,
      "activations/layer7_attention_weight_min": -95.26921844482422,
      "activations/layer8_attention_weight_max": 40.665985107421875,
      "activations/layer8_attention_weight_min": -44.54889678955078,
      "activations/layer9_attention_weight_max": 33.47654342651367,
      "activations/layer9_attention_weight_min": -34.7672233581543,
      "epoch": 16.51,
      "learning_rate": 4.398825757575758e-05,
      "loss": 2.7437,
      "step": 284100
    },
    {
      "activations/layer0_attention_weight_max": 16.232357025146484,
      "activations/layer0_attention_weight_min": -12.64620590209961,
      "activations/layer10_attention_weight_max": 33.61761474609375,
      "activations/layer10_attention_weight_min": -33.53142166137695,
      "activations/layer11_attention_weight_max": 35.14890670776367,
      "activations/layer11_attention_weight_min": -33.463645935058594,
      "activations/layer12_attention_weight_max": 28.311227798461914,
      "activations/layer12_attention_weight_min": -26.414627075195312,
      "activations/layer13_attention_weight_max": 39.66184997558594,
      "activations/layer13_attention_weight_min": -33.67369842529297,
      "activations/layer14_attention_weight_max": 44.81205368041992,
      "activations/layer14_attention_weight_min": -36.39951705932617,
      "activations/layer15_attention_weight_max": 41.41095733642578,
      "activations/layer15_attention_weight_min": -34.360530853271484,
      "activations/layer16_attention_weight_max": 30.24547004699707,
      "activations/layer16_attention_weight_min": -26.99017333984375,
      "activations/layer17_attention_weight_max": 31.98277473449707,
      "activations/layer17_attention_weight_min": -26.196001052856445,
      "activations/layer18_attention_weight_max": 31.4081974029541,
      "activations/layer18_attention_weight_min": -27.661954879760742,
      "activations/layer19_attention_weight_max": 34.98422622680664,
      "activations/layer19_attention_weight_min": -30.142642974853516,
      "activations/layer1_attention_weight_max": 17.141067504882812,
      "activations/layer1_attention_weight_min": -14.359513282775879,
      "activations/layer20_attention_weight_max": 32.3244514465332,
      "activations/layer20_attention_weight_min": -24.547447204589844,
      "activations/layer21_attention_weight_max": 32.2567024230957,
      "activations/layer21_attention_weight_min": -24.60021209716797,
      "activations/layer22_attention_weight_max": 44.4864501953125,
      "activations/layer22_attention_weight_min": -31.057899475097656,
      "activations/layer23_attention_weight_max": 34.50788116455078,
      "activations/layer23_attention_weight_min": -26.207427978515625,
      "activations/layer2_attention_weight_max": 32.953338623046875,
      "activations/layer2_attention_weight_min": -33.48151779174805,
      "activations/layer3_attention_weight_max": 95.19798278808594,
      "activations/layer3_attention_weight_min": -99.76156616210938,
      "activations/layer4_attention_weight_max": 103.31819152832031,
      "activations/layer4_attention_weight_min": -106.9049301147461,
      "activations/layer5_attention_weight_max": 50.10354232788086,
      "activations/layer5_attention_weight_min": -59.476558685302734,
      "activations/layer6_attention_weight_max": 43.98954391479492,
      "activations/layer6_attention_weight_min": -46.24729919433594,
      "activations/layer7_attention_weight_max": 92.48192596435547,
      "activations/layer7_attention_weight_min": -94.44806671142578,
      "activations/layer8_attention_weight_max": 42.95126724243164,
      "activations/layer8_attention_weight_min": -41.94973373413086,
      "activations/layer9_attention_weight_max": 32.928070068359375,
      "activations/layer9_attention_weight_min": -34.327083587646484,
      "epoch": 16.51,
      "learning_rate": 4.396931818181818e-05,
      "loss": 2.7366,
      "step": 284150
    },
    {
      "activations/layer0_attention_weight_max": 15.967203140258789,
      "activations/layer0_attention_weight_min": -12.369527816772461,
      "activations/layer10_attention_weight_max": 41.146705627441406,
      "activations/layer10_attention_weight_min": -40.63153076171875,
      "activations/layer11_attention_weight_max": 42.465572357177734,
      "activations/layer11_attention_weight_min": -40.866241455078125,
      "activations/layer12_attention_weight_max": 28.629331588745117,
      "activations/layer12_attention_weight_min": -27.926454544067383,
      "activations/layer13_attention_weight_max": 43.2358512878418,
      "activations/layer13_attention_weight_min": -37.35740280151367,
      "activations/layer14_attention_weight_max": 45.22233581542969,
      "activations/layer14_attention_weight_min": -41.43220901489258,
      "activations/layer15_attention_weight_max": 42.43619155883789,
      "activations/layer15_attention_weight_min": -38.2546272277832,
      "activations/layer16_attention_weight_max": 33.294578552246094,
      "activations/layer16_attention_weight_min": -28.416242599487305,
      "activations/layer17_attention_weight_max": 28.959136962890625,
      "activations/layer17_attention_weight_min": -26.1199893951416,
      "activations/layer18_attention_weight_max": 32.62692642211914,
      "activations/layer18_attention_weight_min": -25.964134216308594,
      "activations/layer19_attention_weight_max": 35.74939727783203,
      "activations/layer19_attention_weight_min": -31.21335220336914,
      "activations/layer1_attention_weight_max": 18.401123046875,
      "activations/layer1_attention_weight_min": -14.380282402038574,
      "activations/layer20_attention_weight_max": 30.411592483520508,
      "activations/layer20_attention_weight_min": -23.632564544677734,
      "activations/layer21_attention_weight_max": 28.40972900390625,
      "activations/layer21_attention_weight_min": -23.422216415405273,
      "activations/layer22_attention_weight_max": 43.5426025390625,
      "activations/layer22_attention_weight_min": -32.23693084716797,
      "activations/layer23_attention_weight_max": 34.60519027709961,
      "activations/layer23_attention_weight_min": -27.86683464050293,
      "activations/layer2_attention_weight_max": 37.40095520019531,
      "activations/layer2_attention_weight_min": -33.89882278442383,
      "activations/layer3_attention_weight_max": 106.26592254638672,
      "activations/layer3_attention_weight_min": -105.6786117553711,
      "activations/layer4_attention_weight_max": 114.0199966430664,
      "activations/layer4_attention_weight_min": -112.27202606201172,
      "activations/layer5_attention_weight_max": 56.66770553588867,
      "activations/layer5_attention_weight_min": -66.82820129394531,
      "activations/layer6_attention_weight_max": 49.439064025878906,
      "activations/layer6_attention_weight_min": -50.66533279418945,
      "activations/layer7_attention_weight_max": 114.2614517211914,
      "activations/layer7_attention_weight_min": -107.70829772949219,
      "activations/layer8_attention_weight_max": 54.1696662902832,
      "activations/layer8_attention_weight_min": -52.886199951171875,
      "activations/layer9_attention_weight_max": 38.46676254272461,
      "activations/layer9_attention_weight_min": -39.698326110839844,
      "epoch": 16.51,
      "learning_rate": 4.395037878787878e-05,
      "loss": 2.7413,
      "step": 284200
    },
    {
      "activations/layer0_attention_weight_max": 15.985183715820312,
      "activations/layer0_attention_weight_min": -11.959193229675293,
      "activations/layer10_attention_weight_max": 34.405982971191406,
      "activations/layer10_attention_weight_min": -32.42619705200195,
      "activations/layer11_attention_weight_max": 34.151851654052734,
      "activations/layer11_attention_weight_min": -32.421348571777344,
      "activations/layer12_attention_weight_max": 26.67605209350586,
      "activations/layer12_attention_weight_min": -26.721519470214844,
      "activations/layer13_attention_weight_max": 39.73164367675781,
      "activations/layer13_attention_weight_min": -35.882545471191406,
      "activations/layer14_attention_weight_max": 47.37533950805664,
      "activations/layer14_attention_weight_min": -43.2396240234375,
      "activations/layer15_attention_weight_max": 37.34333801269531,
      "activations/layer15_attention_weight_min": -36.4151725769043,
      "activations/layer16_attention_weight_max": 28.99920654296875,
      "activations/layer16_attention_weight_min": -29.606740951538086,
      "activations/layer17_attention_weight_max": 30.61992073059082,
      "activations/layer17_attention_weight_min": -25.737899780273438,
      "activations/layer18_attention_weight_max": 31.709020614624023,
      "activations/layer18_attention_weight_min": -25.853626251220703,
      "activations/layer19_attention_weight_max": 32.7033576965332,
      "activations/layer19_attention_weight_min": -31.360403060913086,
      "activations/layer1_attention_weight_max": 16.39668083190918,
      "activations/layer1_attention_weight_min": -14.218435287475586,
      "activations/layer20_attention_weight_max": 28.967988967895508,
      "activations/layer20_attention_weight_min": -25.759315490722656,
      "activations/layer21_attention_weight_max": 29.995718002319336,
      "activations/layer21_attention_weight_min": -25.146177291870117,
      "activations/layer22_attention_weight_max": 38.6224250793457,
      "activations/layer22_attention_weight_min": -31.752058029174805,
      "activations/layer23_attention_weight_max": 30.80013656616211,
      "activations/layer23_attention_weight_min": -26.079193115234375,
      "activations/layer2_attention_weight_max": 34.27811813354492,
      "activations/layer2_attention_weight_min": -32.472808837890625,
      "activations/layer3_attention_weight_max": 92.85411071777344,
      "activations/layer3_attention_weight_min": -93.86793518066406,
      "activations/layer4_attention_weight_max": 101.47807312011719,
      "activations/layer4_attention_weight_min": -104.06488037109375,
      "activations/layer5_attention_weight_max": 53.634544372558594,
      "activations/layer5_attention_weight_min": -62.54066467285156,
      "activations/layer6_attention_weight_max": 42.97327423095703,
      "activations/layer6_attention_weight_min": -46.14928436279297,
      "activations/layer7_attention_weight_max": 88.2333755493164,
      "activations/layer7_attention_weight_min": -93.92671203613281,
      "activations/layer8_attention_weight_max": 38.70267868041992,
      "activations/layer8_attention_weight_min": -40.191837310791016,
      "activations/layer9_attention_weight_max": 31.21053123474121,
      "activations/layer9_attention_weight_min": -32.86355209350586,
      "epoch": 16.52,
      "learning_rate": 4.3931439393939395e-05,
      "loss": 2.7529,
      "step": 284250
    },
    {
      "activations/layer0_attention_weight_max": 16.10359764099121,
      "activations/layer0_attention_weight_min": -13.100780487060547,
      "activations/layer10_attention_weight_max": 30.929744720458984,
      "activations/layer10_attention_weight_min": -31.201519012451172,
      "activations/layer11_attention_weight_max": 32.666107177734375,
      "activations/layer11_attention_weight_min": -30.889434814453125,
      "activations/layer12_attention_weight_max": 25.717411041259766,
      "activations/layer12_attention_weight_min": -24.745136260986328,
      "activations/layer13_attention_weight_max": 37.420379638671875,
      "activations/layer13_attention_weight_min": -34.655296325683594,
      "activations/layer14_attention_weight_max": 42.42158126831055,
      "activations/layer14_attention_weight_min": -38.23170471191406,
      "activations/layer15_attention_weight_max": 36.72819519042969,
      "activations/layer15_attention_weight_min": -34.839561462402344,
      "activations/layer16_attention_weight_max": 31.29184913635254,
      "activations/layer16_attention_weight_min": -28.268178939819336,
      "activations/layer17_attention_weight_max": 32.52263259887695,
      "activations/layer17_attention_weight_min": -26.01560401916504,
      "activations/layer18_attention_weight_max": 35.04313278198242,
      "activations/layer18_attention_weight_min": -25.935827255249023,
      "activations/layer19_attention_weight_max": 37.693748474121094,
      "activations/layer19_attention_weight_min": -32.584842681884766,
      "activations/layer1_attention_weight_max": 15.597464561462402,
      "activations/layer1_attention_weight_min": -14.054834365844727,
      "activations/layer20_attention_weight_max": 29.36928939819336,
      "activations/layer20_attention_weight_min": -25.2427978515625,
      "activations/layer21_attention_weight_max": 28.65859031677246,
      "activations/layer21_attention_weight_min": -25.026073455810547,
      "activations/layer22_attention_weight_max": 47.12747573852539,
      "activations/layer22_attention_weight_min": -32.53521728515625,
      "activations/layer23_attention_weight_max": 33.38669204711914,
      "activations/layer23_attention_weight_min": -28.12430191040039,
      "activations/layer2_attention_weight_max": 30.728225708007812,
      "activations/layer2_attention_weight_min": -33.21110534667969,
      "activations/layer3_attention_weight_max": 92.627197265625,
      "activations/layer3_attention_weight_min": -92.95881652832031,
      "activations/layer4_attention_weight_max": 102.83267974853516,
      "activations/layer4_attention_weight_min": -107.42076873779297,
      "activations/layer5_attention_weight_max": 51.263885498046875,
      "activations/layer5_attention_weight_min": -61.32305908203125,
      "activations/layer6_attention_weight_max": 43.22230529785156,
      "activations/layer6_attention_weight_min": -46.20191955566406,
      "activations/layer7_attention_weight_max": 92.5176010131836,
      "activations/layer7_attention_weight_min": -96.08316802978516,
      "activations/layer8_attention_weight_max": 41.788204193115234,
      "activations/layer8_attention_weight_min": -43.4007453918457,
      "activations/layer9_attention_weight_max": 30.12371826171875,
      "activations/layer9_attention_weight_min": -32.17938995361328,
      "epoch": 16.52,
      "learning_rate": 4.39125e-05,
      "loss": 2.7369,
      "step": 284300
    },
    {
      "activations/layer0_attention_weight_max": 17.45771598815918,
      "activations/layer0_attention_weight_min": -12.99310302734375,
      "activations/layer10_attention_weight_max": 31.187767028808594,
      "activations/layer10_attention_weight_min": -30.96979331970215,
      "activations/layer11_attention_weight_max": 30.410808563232422,
      "activations/layer11_attention_weight_min": -31.427440643310547,
      "activations/layer12_attention_weight_max": 24.634916305541992,
      "activations/layer12_attention_weight_min": -25.525802612304688,
      "activations/layer13_attention_weight_max": 35.71156311035156,
      "activations/layer13_attention_weight_min": -34.01542663574219,
      "activations/layer14_attention_weight_max": 40.30554962158203,
      "activations/layer14_attention_weight_min": -36.26499557495117,
      "activations/layer15_attention_weight_max": 34.110713958740234,
      "activations/layer15_attention_weight_min": -36.2341194152832,
      "activations/layer16_attention_weight_max": 27.89546012878418,
      "activations/layer16_attention_weight_min": -28.42853355407715,
      "activations/layer17_attention_weight_max": 30.019968032836914,
      "activations/layer17_attention_weight_min": -26.44423484802246,
      "activations/layer18_attention_weight_max": 30.152015686035156,
      "activations/layer18_attention_weight_min": -25.873416900634766,
      "activations/layer19_attention_weight_max": 32.038448333740234,
      "activations/layer19_attention_weight_min": -30.53835105895996,
      "activations/layer1_attention_weight_max": 16.10956382751465,
      "activations/layer1_attention_weight_min": -14.504258155822754,
      "activations/layer20_attention_weight_max": 29.111101150512695,
      "activations/layer20_attention_weight_min": -24.269163131713867,
      "activations/layer21_attention_weight_max": 29.574935913085938,
      "activations/layer21_attention_weight_min": -25.349679946899414,
      "activations/layer22_attention_weight_max": 37.463844299316406,
      "activations/layer22_attention_weight_min": -31.263086318969727,
      "activations/layer23_attention_weight_max": 31.37885093688965,
      "activations/layer23_attention_weight_min": -25.740171432495117,
      "activations/layer2_attention_weight_max": 31.575748443603516,
      "activations/layer2_attention_weight_min": -32.24992752075195,
      "activations/layer3_attention_weight_max": 95.2787857055664,
      "activations/layer3_attention_weight_min": -98.17819213867188,
      "activations/layer4_attention_weight_max": 104.3193130493164,
      "activations/layer4_attention_weight_min": -108.5435562133789,
      "activations/layer5_attention_weight_max": 50.94422912597656,
      "activations/layer5_attention_weight_min": -64.86531829833984,
      "activations/layer6_attention_weight_max": 42.89667892456055,
      "activations/layer6_attention_weight_min": -46.21432113647461,
      "activations/layer7_attention_weight_max": 87.9849624633789,
      "activations/layer7_attention_weight_min": -87.58759307861328,
      "activations/layer8_attention_weight_max": 39.93077850341797,
      "activations/layer8_attention_weight_min": -41.928794860839844,
      "activations/layer9_attention_weight_max": 32.444034576416016,
      "activations/layer9_attention_weight_min": -34.233341217041016,
      "epoch": 16.52,
      "learning_rate": 4.38935606060606e-05,
      "loss": 2.7444,
      "step": 284350
    },
    {
      "activations/layer0_attention_weight_max": 16.444992065429688,
      "activations/layer0_attention_weight_min": -12.313047409057617,
      "activations/layer10_attention_weight_max": 31.077781677246094,
      "activations/layer10_attention_weight_min": -33.2234992980957,
      "activations/layer11_attention_weight_max": 31.336599349975586,
      "activations/layer11_attention_weight_min": -34.58087158203125,
      "activations/layer12_attention_weight_max": 25.15384292602539,
      "activations/layer12_attention_weight_min": -25.523345947265625,
      "activations/layer13_attention_weight_max": 38.14320373535156,
      "activations/layer13_attention_weight_min": -33.628753662109375,
      "activations/layer14_attention_weight_max": 39.69047164916992,
      "activations/layer14_attention_weight_min": -37.60038757324219,
      "activations/layer15_attention_weight_max": 41.24346160888672,
      "activations/layer15_attention_weight_min": -35.540382385253906,
      "activations/layer16_attention_weight_max": 29.181333541870117,
      "activations/layer16_attention_weight_min": -27.98130226135254,
      "activations/layer17_attention_weight_max": 32.33784103393555,
      "activations/layer17_attention_weight_min": -25.44725227355957,
      "activations/layer18_attention_weight_max": 31.35896110534668,
      "activations/layer18_attention_weight_min": -24.6680850982666,
      "activations/layer19_attention_weight_max": 36.74594497680664,
      "activations/layer19_attention_weight_min": -30.749906539916992,
      "activations/layer1_attention_weight_max": 15.912571907043457,
      "activations/layer1_attention_weight_min": -15.488893508911133,
      "activations/layer20_attention_weight_max": 30.951818466186523,
      "activations/layer20_attention_weight_min": -25.783828735351562,
      "activations/layer21_attention_weight_max": 31.998062133789062,
      "activations/layer21_attention_weight_min": -25.238534927368164,
      "activations/layer22_attention_weight_max": 43.394771575927734,
      "activations/layer22_attention_weight_min": -31.625444412231445,
      "activations/layer23_attention_weight_max": 33.75184631347656,
      "activations/layer23_attention_weight_min": -26.31743621826172,
      "activations/layer2_attention_weight_max": 31.85591697692871,
      "activations/layer2_attention_weight_min": -32.62501907348633,
      "activations/layer3_attention_weight_max": 94.95970916748047,
      "activations/layer3_attention_weight_min": -94.8792953491211,
      "activations/layer4_attention_weight_max": 106.01091766357422,
      "activations/layer4_attention_weight_min": -108.23088836669922,
      "activations/layer5_attention_weight_max": 51.023250579833984,
      "activations/layer5_attention_weight_min": -65.48562622070312,
      "activations/layer6_attention_weight_max": 42.81733322143555,
      "activations/layer6_attention_weight_min": -45.514156341552734,
      "activations/layer7_attention_weight_max": 94.95465087890625,
      "activations/layer7_attention_weight_min": -95.00359344482422,
      "activations/layer8_attention_weight_max": 39.309425354003906,
      "activations/layer8_attention_weight_min": -40.803367614746094,
      "activations/layer9_attention_weight_max": 29.82021141052246,
      "activations/layer9_attention_weight_min": -33.703914642333984,
      "epoch": 16.53,
      "learning_rate": 4.3874621212121206e-05,
      "loss": 2.7504,
      "step": 284400
    },
    {
      "activations/layer0_attention_weight_max": 16.15522575378418,
      "activations/layer0_attention_weight_min": -13.661596298217773,
      "activations/layer10_attention_weight_max": 34.98542022705078,
      "activations/layer10_attention_weight_min": -36.7991943359375,
      "activations/layer11_attention_weight_max": 35.75554275512695,
      "activations/layer11_attention_weight_min": -35.79008865356445,
      "activations/layer12_attention_weight_max": 29.300722122192383,
      "activations/layer12_attention_weight_min": -26.96666717529297,
      "activations/layer13_attention_weight_max": 45.51164627075195,
      "activations/layer13_attention_weight_min": -37.87473678588867,
      "activations/layer14_attention_weight_max": 52.529151916503906,
      "activations/layer14_attention_weight_min": -38.602149963378906,
      "activations/layer15_attention_weight_max": 43.28506088256836,
      "activations/layer15_attention_weight_min": -36.72567367553711,
      "activations/layer16_attention_weight_max": 31.73777961730957,
      "activations/layer16_attention_weight_min": -30.94084358215332,
      "activations/layer17_attention_weight_max": 34.24556350708008,
      "activations/layer17_attention_weight_min": -26.891511917114258,
      "activations/layer18_attention_weight_max": 35.28254318237305,
      "activations/layer18_attention_weight_min": -25.428293228149414,
      "activations/layer19_attention_weight_max": 42.46088790893555,
      "activations/layer19_attention_weight_min": -30.294504165649414,
      "activations/layer1_attention_weight_max": 16.61138916015625,
      "activations/layer1_attention_weight_min": -14.296921730041504,
      "activations/layer20_attention_weight_max": 35.90542984008789,
      "activations/layer20_attention_weight_min": -24.800086975097656,
      "activations/layer21_attention_weight_max": 35.090179443359375,
      "activations/layer21_attention_weight_min": -23.49445152282715,
      "activations/layer22_attention_weight_max": 47.50883102416992,
      "activations/layer22_attention_weight_min": -33.498836517333984,
      "activations/layer23_attention_weight_max": 38.62788391113281,
      "activations/layer23_attention_weight_min": -25.30129623413086,
      "activations/layer2_attention_weight_max": 32.02708053588867,
      "activations/layer2_attention_weight_min": -32.37046813964844,
      "activations/layer3_attention_weight_max": 93.3083267211914,
      "activations/layer3_attention_weight_min": -97.9402847290039,
      "activations/layer4_attention_weight_max": 105.26444244384766,
      "activations/layer4_attention_weight_min": -109.8670883178711,
      "activations/layer5_attention_weight_max": 52.376312255859375,
      "activations/layer5_attention_weight_min": -60.35136413574219,
      "activations/layer6_attention_weight_max": 44.80493927001953,
      "activations/layer6_attention_weight_min": -48.96582794189453,
      "activations/layer7_attention_weight_max": 100.96418762207031,
      "activations/layer7_attention_weight_min": -99.03810119628906,
      "activations/layer8_attention_weight_max": 43.201778411865234,
      "activations/layer8_attention_weight_min": -43.53773880004883,
      "activations/layer9_attention_weight_max": 33.02360534667969,
      "activations/layer9_attention_weight_min": -34.34324645996094,
      "epoch": 16.53,
      "learning_rate": 4.3855681818181815e-05,
      "loss": 2.7437,
      "step": 284450
    },
    {
      "activations/layer0_attention_weight_max": 15.924604415893555,
      "activations/layer0_attention_weight_min": -12.141485214233398,
      "activations/layer10_attention_weight_max": 31.836729049682617,
      "activations/layer10_attention_weight_min": -32.81121826171875,
      "activations/layer11_attention_weight_max": 32.53236770629883,
      "activations/layer11_attention_weight_min": -34.16183090209961,
      "activations/layer12_attention_weight_max": 23.776172637939453,
      "activations/layer12_attention_weight_min": -25.822837829589844,
      "activations/layer13_attention_weight_max": 36.0535888671875,
      "activations/layer13_attention_weight_min": -34.60206985473633,
      "activations/layer14_attention_weight_max": 37.478477478027344,
      "activations/layer14_attention_weight_min": -36.648582458496094,
      "activations/layer15_attention_weight_max": 35.33863067626953,
      "activations/layer15_attention_weight_min": -34.237525939941406,
      "activations/layer16_attention_weight_max": 28.02287483215332,
      "activations/layer16_attention_weight_min": -26.86113929748535,
      "activations/layer17_attention_weight_max": 28.032032012939453,
      "activations/layer17_attention_weight_min": -25.961223602294922,
      "activations/layer18_attention_weight_max": 27.955177307128906,
      "activations/layer18_attention_weight_min": -26.021520614624023,
      "activations/layer19_attention_weight_max": 34.78864669799805,
      "activations/layer19_attention_weight_min": -33.632320404052734,
      "activations/layer1_attention_weight_max": 16.386577606201172,
      "activations/layer1_attention_weight_min": -14.770451545715332,
      "activations/layer20_attention_weight_max": 31.5101261138916,
      "activations/layer20_attention_weight_min": -24.24542808532715,
      "activations/layer21_attention_weight_max": 31.210031509399414,
      "activations/layer21_attention_weight_min": -23.707101821899414,
      "activations/layer22_attention_weight_max": 40.2547607421875,
      "activations/layer22_attention_weight_min": -32.03270721435547,
      "activations/layer23_attention_weight_max": 33.546546936035156,
      "activations/layer23_attention_weight_min": -26.45590591430664,
      "activations/layer2_attention_weight_max": 34.25384521484375,
      "activations/layer2_attention_weight_min": -34.69673156738281,
      "activations/layer3_attention_weight_max": 97.4786148071289,
      "activations/layer3_attention_weight_min": -100.84904479980469,
      "activations/layer4_attention_weight_max": 110.24903869628906,
      "activations/layer4_attention_weight_min": -110.08357238769531,
      "activations/layer5_attention_weight_max": 49.23451232910156,
      "activations/layer5_attention_weight_min": -60.141563415527344,
      "activations/layer6_attention_weight_max": 46.11427688598633,
      "activations/layer6_attention_weight_min": -48.506656646728516,
      "activations/layer7_attention_weight_max": 92.83106994628906,
      "activations/layer7_attention_weight_min": -96.92488861083984,
      "activations/layer8_attention_weight_max": 42.311279296875,
      "activations/layer8_attention_weight_min": -41.93716049194336,
      "activations/layer9_attention_weight_max": 32.52895736694336,
      "activations/layer9_attention_weight_min": -34.567726135253906,
      "epoch": 16.53,
      "learning_rate": 4.383674242424242e-05,
      "loss": 2.7484,
      "step": 284500
    },
    {
      "activations/layer0_attention_weight_max": 16.211475372314453,
      "activations/layer0_attention_weight_min": -11.860563278198242,
      "activations/layer10_attention_weight_max": 31.376007080078125,
      "activations/layer10_attention_weight_min": -32.986351013183594,
      "activations/layer11_attention_weight_max": 32.50634002685547,
      "activations/layer11_attention_weight_min": -32.935821533203125,
      "activations/layer12_attention_weight_max": 26.20349884033203,
      "activations/layer12_attention_weight_min": -25.736488342285156,
      "activations/layer13_attention_weight_max": 38.57146453857422,
      "activations/layer13_attention_weight_min": -33.25020980834961,
      "activations/layer14_attention_weight_max": 39.900230407714844,
      "activations/layer14_attention_weight_min": -34.8967170715332,
      "activations/layer15_attention_weight_max": 38.638973236083984,
      "activations/layer15_attention_weight_min": -33.29604721069336,
      "activations/layer16_attention_weight_max": 29.509929656982422,
      "activations/layer16_attention_weight_min": -28.838695526123047,
      "activations/layer17_attention_weight_max": 31.843713760375977,
      "activations/layer17_attention_weight_min": -26.32318687438965,
      "activations/layer18_attention_weight_max": 35.257476806640625,
      "activations/layer18_attention_weight_min": -25.075639724731445,
      "activations/layer19_attention_weight_max": 35.0794792175293,
      "activations/layer19_attention_weight_min": -31.173542022705078,
      "activations/layer1_attention_weight_max": 17.279972076416016,
      "activations/layer1_attention_weight_min": -14.983545303344727,
      "activations/layer20_attention_weight_max": 30.720918655395508,
      "activations/layer20_attention_weight_min": -25.2581729888916,
      "activations/layer21_attention_weight_max": 29.662242889404297,
      "activations/layer21_attention_weight_min": -24.98931884765625,
      "activations/layer22_attention_weight_max": 49.459442138671875,
      "activations/layer22_attention_weight_min": -31.67812156677246,
      "activations/layer23_attention_weight_max": 33.02197265625,
      "activations/layer23_attention_weight_min": -24.703344345092773,
      "activations/layer2_attention_weight_max": 32.526344299316406,
      "activations/layer2_attention_weight_min": -31.370708465576172,
      "activations/layer3_attention_weight_max": 93.68405151367188,
      "activations/layer3_attention_weight_min": -95.04015350341797,
      "activations/layer4_attention_weight_max": 106.31550598144531,
      "activations/layer4_attention_weight_min": -108.18839263916016,
      "activations/layer5_attention_weight_max": 50.667694091796875,
      "activations/layer5_attention_weight_min": -61.508567810058594,
      "activations/layer6_attention_weight_max": 42.495391845703125,
      "activations/layer6_attention_weight_min": -47.732383728027344,
      "activations/layer7_attention_weight_max": 90.88439178466797,
      "activations/layer7_attention_weight_min": -95.46672058105469,
      "activations/layer8_attention_weight_max": 40.39052200317383,
      "activations/layer8_attention_weight_min": -43.41346740722656,
      "activations/layer9_attention_weight_max": 32.08039474487305,
      "activations/layer9_attention_weight_min": -34.243350982666016,
      "epoch": 16.53,
      "learning_rate": 4.3817803030303025e-05,
      "loss": 2.7473,
      "step": 284550
    },
    {
      "activations/layer0_attention_weight_max": 15.41264533996582,
      "activations/layer0_attention_weight_min": -12.595839500427246,
      "activations/layer10_attention_weight_max": 29.94944953918457,
      "activations/layer10_attention_weight_min": -31.721921920776367,
      "activations/layer11_attention_weight_max": 30.93147850036621,
      "activations/layer11_attention_weight_min": -32.1746826171875,
      "activations/layer12_attention_weight_max": 26.05141830444336,
      "activations/layer12_attention_weight_min": -32.0147705078125,
      "activations/layer13_attention_weight_max": 36.32696533203125,
      "activations/layer13_attention_weight_min": -33.602989196777344,
      "activations/layer14_attention_weight_max": 39.91276931762695,
      "activations/layer14_attention_weight_min": -36.352535247802734,
      "activations/layer15_attention_weight_max": 34.82772445678711,
      "activations/layer15_attention_weight_min": -31.984275817871094,
      "activations/layer16_attention_weight_max": 28.45588493347168,
      "activations/layer16_attention_weight_min": -26.496501922607422,
      "activations/layer17_attention_weight_max": 30.795244216918945,
      "activations/layer17_attention_weight_min": -24.76457977294922,
      "activations/layer18_attention_weight_max": 32.321353912353516,
      "activations/layer18_attention_weight_min": -24.470844268798828,
      "activations/layer19_attention_weight_max": 29.03065299987793,
      "activations/layer19_attention_weight_min": -31.141687393188477,
      "activations/layer1_attention_weight_max": 16.4075984954834,
      "activations/layer1_attention_weight_min": -13.819510459899902,
      "activations/layer20_attention_weight_max": 25.427188873291016,
      "activations/layer20_attention_weight_min": -24.90147590637207,
      "activations/layer21_attention_weight_max": 24.00518798828125,
      "activations/layer21_attention_weight_min": -23.68387222290039,
      "activations/layer22_attention_weight_max": 36.18043518066406,
      "activations/layer22_attention_weight_min": -29.989025115966797,
      "activations/layer23_attention_weight_max": 28.5932674407959,
      "activations/layer23_attention_weight_min": -25.115020751953125,
      "activations/layer2_attention_weight_max": 33.034427642822266,
      "activations/layer2_attention_weight_min": -30.519729614257812,
      "activations/layer3_attention_weight_max": 88.45502471923828,
      "activations/layer3_attention_weight_min": -92.1872329711914,
      "activations/layer4_attention_weight_max": 96.29374694824219,
      "activations/layer4_attention_weight_min": -105.59874725341797,
      "activations/layer5_attention_weight_max": 47.0272216796875,
      "activations/layer5_attention_weight_min": -58.93017578125,
      "activations/layer6_attention_weight_max": 39.37989044189453,
      "activations/layer6_attention_weight_min": -44.19114303588867,
      "activations/layer7_attention_weight_max": 84.08622741699219,
      "activations/layer7_attention_weight_min": -89.42018127441406,
      "activations/layer8_attention_weight_max": 38.92953872680664,
      "activations/layer8_attention_weight_min": -39.70880889892578,
      "activations/layer9_attention_weight_max": 29.69758415222168,
      "activations/layer9_attention_weight_min": -30.825016021728516,
      "epoch": 16.54,
      "learning_rate": 4.3798863636363626e-05,
      "loss": 2.7398,
      "step": 284600
    },
    {
      "activations/layer0_attention_weight_max": 16.20920181274414,
      "activations/layer0_attention_weight_min": -12.395198822021484,
      "activations/layer10_attention_weight_max": 33.96693801879883,
      "activations/layer10_attention_weight_min": -33.81231689453125,
      "activations/layer11_attention_weight_max": 32.678497314453125,
      "activations/layer11_attention_weight_min": -35.079742431640625,
      "activations/layer12_attention_weight_max": 29.478548049926758,
      "activations/layer12_attention_weight_min": -27.4516544342041,
      "activations/layer13_attention_weight_max": 43.46000289916992,
      "activations/layer13_attention_weight_min": -36.95507049560547,
      "activations/layer14_attention_weight_max": 45.030357360839844,
      "activations/layer14_attention_weight_min": -37.924068450927734,
      "activations/layer15_attention_weight_max": 40.74753952026367,
      "activations/layer15_attention_weight_min": -36.41134262084961,
      "activations/layer16_attention_weight_max": 32.54530715942383,
      "activations/layer16_attention_weight_min": -28.258983612060547,
      "activations/layer17_attention_weight_max": 32.337013244628906,
      "activations/layer17_attention_weight_min": -28.211633682250977,
      "activations/layer18_attention_weight_max": 33.45173263549805,
      "activations/layer18_attention_weight_min": -27.09853744506836,
      "activations/layer19_attention_weight_max": 36.87196350097656,
      "activations/layer19_attention_weight_min": -31.979825973510742,
      "activations/layer1_attention_weight_max": 18.492298126220703,
      "activations/layer1_attention_weight_min": -15.769806861877441,
      "activations/layer20_attention_weight_max": 30.415771484375,
      "activations/layer20_attention_weight_min": -27.018918991088867,
      "activations/layer21_attention_weight_max": 31.41917610168457,
      "activations/layer21_attention_weight_min": -27.42991065979004,
      "activations/layer22_attention_weight_max": 44.78589630126953,
      "activations/layer22_attention_weight_min": -34.92935562133789,
      "activations/layer23_attention_weight_max": 36.17578125,
      "activations/layer23_attention_weight_min": -27.870895385742188,
      "activations/layer2_attention_weight_max": 34.174034118652344,
      "activations/layer2_attention_weight_min": -28.655677795410156,
      "activations/layer3_attention_weight_max": 88.07582092285156,
      "activations/layer3_attention_weight_min": -90.21287536621094,
      "activations/layer4_attention_weight_max": 101.63277435302734,
      "activations/layer4_attention_weight_min": -102.17330169677734,
      "activations/layer5_attention_weight_max": 47.859371185302734,
      "activations/layer5_attention_weight_min": -58.506866455078125,
      "activations/layer6_attention_weight_max": 44.39798355102539,
      "activations/layer6_attention_weight_min": -45.58351135253906,
      "activations/layer7_attention_weight_max": 92.52621459960938,
      "activations/layer7_attention_weight_min": -92.46369934082031,
      "activations/layer8_attention_weight_max": 42.837337493896484,
      "activations/layer8_attention_weight_min": -41.6512336730957,
      "activations/layer9_attention_weight_max": 34.393184661865234,
      "activations/layer9_attention_weight_min": -34.24369430541992,
      "epoch": 16.54,
      "learning_rate": 4.377992424242424e-05,
      "loss": 2.7527,
      "step": 284650
    },
    {
      "activations/layer0_attention_weight_max": 16.653858184814453,
      "activations/layer0_attention_weight_min": -13.026117324829102,
      "activations/layer10_attention_weight_max": 34.810264587402344,
      "activations/layer10_attention_weight_min": -34.18962860107422,
      "activations/layer11_attention_weight_max": 33.474220275878906,
      "activations/layer11_attention_weight_min": -33.130062103271484,
      "activations/layer12_attention_weight_max": 26.08167839050293,
      "activations/layer12_attention_weight_min": -28.289342880249023,
      "activations/layer13_attention_weight_max": 36.26335144042969,
      "activations/layer13_attention_weight_min": -37.83350372314453,
      "activations/layer14_attention_weight_max": 40.82865524291992,
      "activations/layer14_attention_weight_min": -39.700374603271484,
      "activations/layer15_attention_weight_max": 35.4931640625,
      "activations/layer15_attention_weight_min": -35.61016082763672,
      "activations/layer16_attention_weight_max": 28.863609313964844,
      "activations/layer16_attention_weight_min": -26.955698013305664,
      "activations/layer17_attention_weight_max": 31.694204330444336,
      "activations/layer17_attention_weight_min": -26.75558090209961,
      "activations/layer18_attention_weight_max": 27.914268493652344,
      "activations/layer18_attention_weight_min": -24.970901489257812,
      "activations/layer19_attention_weight_max": 30.779251098632812,
      "activations/layer19_attention_weight_min": -29.759140014648438,
      "activations/layer1_attention_weight_max": 16.0512638092041,
      "activations/layer1_attention_weight_min": -14.96883487701416,
      "activations/layer20_attention_weight_max": 26.1646671295166,
      "activations/layer20_attention_weight_min": -24.227325439453125,
      "activations/layer21_attention_weight_max": 26.38549041748047,
      "activations/layer21_attention_weight_min": -26.213579177856445,
      "activations/layer22_attention_weight_max": 38.16712188720703,
      "activations/layer22_attention_weight_min": -31.130706787109375,
      "activations/layer23_attention_weight_max": 29.2900447845459,
      "activations/layer23_attention_weight_min": -26.89411735534668,
      "activations/layer2_attention_weight_max": 32.838958740234375,
      "activations/layer2_attention_weight_min": -31.645870208740234,
      "activations/layer3_attention_weight_max": 95.23621368408203,
      "activations/layer3_attention_weight_min": -97.08383178710938,
      "activations/layer4_attention_weight_max": 103.29054260253906,
      "activations/layer4_attention_weight_min": -106.12611389160156,
      "activations/layer5_attention_weight_max": 50.22162628173828,
      "activations/layer5_attention_weight_min": -61.96788024902344,
      "activations/layer6_attention_weight_max": 42.49331283569336,
      "activations/layer6_attention_weight_min": -45.78290939331055,
      "activations/layer7_attention_weight_max": 97.522216796875,
      "activations/layer7_attention_weight_min": -96.18482971191406,
      "activations/layer8_attention_weight_max": 42.996646881103516,
      "activations/layer8_attention_weight_min": -46.61896514892578,
      "activations/layer9_attention_weight_max": 34.97572708129883,
      "activations/layer9_attention_weight_min": -34.05967330932617,
      "epoch": 16.54,
      "learning_rate": 4.376098484848484e-05,
      "loss": 2.735,
      "step": 284700
    },
    {
      "activations/layer0_attention_weight_max": 15.157357215881348,
      "activations/layer0_attention_weight_min": -12.474875450134277,
      "activations/layer10_attention_weight_max": 34.437435150146484,
      "activations/layer10_attention_weight_min": -33.796051025390625,
      "activations/layer11_attention_weight_max": 37.0966682434082,
      "activations/layer11_attention_weight_min": -34.363800048828125,
      "activations/layer12_attention_weight_max": 28.250038146972656,
      "activations/layer12_attention_weight_min": -26.92625617980957,
      "activations/layer13_attention_weight_max": 38.709571838378906,
      "activations/layer13_attention_weight_min": -35.25294494628906,
      "activations/layer14_attention_weight_max": 41.44533920288086,
      "activations/layer14_attention_weight_min": -38.857627868652344,
      "activations/layer15_attention_weight_max": 36.808387756347656,
      "activations/layer15_attention_weight_min": -35.515987396240234,
      "activations/layer16_attention_weight_max": 28.734235763549805,
      "activations/layer16_attention_weight_min": -32.122886657714844,
      "activations/layer17_attention_weight_max": 28.79421043395996,
      "activations/layer17_attention_weight_min": -25.8128662109375,
      "activations/layer18_attention_weight_max": 31.15214729309082,
      "activations/layer18_attention_weight_min": -24.25424575805664,
      "activations/layer19_attention_weight_max": 31.217905044555664,
      "activations/layer19_attention_weight_min": -31.362577438354492,
      "activations/layer1_attention_weight_max": 16.496694564819336,
      "activations/layer1_attention_weight_min": -13.940645217895508,
      "activations/layer20_attention_weight_max": 27.693065643310547,
      "activations/layer20_attention_weight_min": -26.161596298217773,
      "activations/layer21_attention_weight_max": 27.724233627319336,
      "activations/layer21_attention_weight_min": -26.133148193359375,
      "activations/layer22_attention_weight_max": 45.06168746948242,
      "activations/layer22_attention_weight_min": -31.98859977722168,
      "activations/layer23_attention_weight_max": 30.830358505249023,
      "activations/layer23_attention_weight_min": -25.66653823852539,
      "activations/layer2_attention_weight_max": 30.760406494140625,
      "activations/layer2_attention_weight_min": -31.847980499267578,
      "activations/layer3_attention_weight_max": 94.90018463134766,
      "activations/layer3_attention_weight_min": -97.01930236816406,
      "activations/layer4_attention_weight_max": 106.5733871459961,
      "activations/layer4_attention_weight_min": -106.2670669555664,
      "activations/layer5_attention_weight_max": 56.26905059814453,
      "activations/layer5_attention_weight_min": -65.12157440185547,
      "activations/layer6_attention_weight_max": 43.9962272644043,
      "activations/layer6_attention_weight_min": -48.12607955932617,
      "activations/layer7_attention_weight_max": 91.02750396728516,
      "activations/layer7_attention_weight_min": -97.88238525390625,
      "activations/layer8_attention_weight_max": 39.454345703125,
      "activations/layer8_attention_weight_min": -42.554630279541016,
      "activations/layer9_attention_weight_max": 31.889829635620117,
      "activations/layer9_attention_weight_min": -32.8576774597168,
      "epoch": 16.55,
      "learning_rate": 4.374204545454545e-05,
      "loss": 2.737,
      "step": 284750
    },
    {
      "activations/layer0_attention_weight_max": 14.794069290161133,
      "activations/layer0_attention_weight_min": -14.165664672851562,
      "activations/layer10_attention_weight_max": 32.827816009521484,
      "activations/layer10_attention_weight_min": -31.73732566833496,
      "activations/layer11_attention_weight_max": 32.786651611328125,
      "activations/layer11_attention_weight_min": -32.61204147338867,
      "activations/layer12_attention_weight_max": 26.088773727416992,
      "activations/layer12_attention_weight_min": -26.919479370117188,
      "activations/layer13_attention_weight_max": 38.76340866088867,
      "activations/layer13_attention_weight_min": -36.105430603027344,
      "activations/layer14_attention_weight_max": 40.763458251953125,
      "activations/layer14_attention_weight_min": -37.6082649230957,
      "activations/layer15_attention_weight_max": 37.68827438354492,
      "activations/layer15_attention_weight_min": -35.80906677246094,
      "activations/layer16_attention_weight_max": 29.40493392944336,
      "activations/layer16_attention_weight_min": -25.9818115234375,
      "activations/layer17_attention_weight_max": 27.49558448791504,
      "activations/layer17_attention_weight_min": -24.638137817382812,
      "activations/layer18_attention_weight_max": 30.27785873413086,
      "activations/layer18_attention_weight_min": -25.40816307067871,
      "activations/layer19_attention_weight_max": 33.266292572021484,
      "activations/layer19_attention_weight_min": -29.1715145111084,
      "activations/layer1_attention_weight_max": 16.57830047607422,
      "activations/layer1_attention_weight_min": -15.159531593322754,
      "activations/layer20_attention_weight_max": 27.541004180908203,
      "activations/layer20_attention_weight_min": -26.992931365966797,
      "activations/layer21_attention_weight_max": 26.323110580444336,
      "activations/layer21_attention_weight_min": -23.747512817382812,
      "activations/layer22_attention_weight_max": 37.361392974853516,
      "activations/layer22_attention_weight_min": -28.783708572387695,
      "activations/layer23_attention_weight_max": 29.163330078125,
      "activations/layer23_attention_weight_min": -24.279651641845703,
      "activations/layer2_attention_weight_max": 32.18119812011719,
      "activations/layer2_attention_weight_min": -31.122970581054688,
      "activations/layer3_attention_weight_max": 95.86893463134766,
      "activations/layer3_attention_weight_min": -95.96002960205078,
      "activations/layer4_attention_weight_max": 107.56233215332031,
      "activations/layer4_attention_weight_min": -108.74720764160156,
      "activations/layer5_attention_weight_max": 53.2988395690918,
      "activations/layer5_attention_weight_min": -63.350608825683594,
      "activations/layer6_attention_weight_max": 43.50717544555664,
      "activations/layer6_attention_weight_min": -45.364139556884766,
      "activations/layer7_attention_weight_max": 92.24337768554688,
      "activations/layer7_attention_weight_min": -92.75102233886719,
      "activations/layer8_attention_weight_max": 40.4083137512207,
      "activations/layer8_attention_weight_min": -41.802764892578125,
      "activations/layer9_attention_weight_max": 32.4969596862793,
      "activations/layer9_attention_weight_min": -34.77535629272461,
      "epoch": 16.55,
      "learning_rate": 4.372310606060605e-05,
      "loss": 2.7396,
      "step": 284800
    },
    {
      "activations/layer0_attention_weight_max": 16.369403839111328,
      "activations/layer0_attention_weight_min": -11.690494537353516,
      "activations/layer10_attention_weight_max": 32.24007797241211,
      "activations/layer10_attention_weight_min": -32.399192810058594,
      "activations/layer11_attention_weight_max": 33.399436950683594,
      "activations/layer11_attention_weight_min": -34.727806091308594,
      "activations/layer12_attention_weight_max": 31.29592514038086,
      "activations/layer12_attention_weight_min": -26.180376052856445,
      "activations/layer13_attention_weight_max": 39.416221618652344,
      "activations/layer13_attention_weight_min": -35.776126861572266,
      "activations/layer14_attention_weight_max": 40.46543884277344,
      "activations/layer14_attention_weight_min": -38.661460876464844,
      "activations/layer15_attention_weight_max": 37.36530303955078,
      "activations/layer15_attention_weight_min": -36.480430603027344,
      "activations/layer16_attention_weight_max": 31.707275390625,
      "activations/layer16_attention_weight_min": -28.260143280029297,
      "activations/layer17_attention_weight_max": 30.60317039489746,
      "activations/layer17_attention_weight_min": -28.90825080871582,
      "activations/layer18_attention_weight_max": 31.8566951751709,
      "activations/layer18_attention_weight_min": -26.620235443115234,
      "activations/layer19_attention_weight_max": 33.771812438964844,
      "activations/layer19_attention_weight_min": -32.9978141784668,
      "activations/layer1_attention_weight_max": 16.0697021484375,
      "activations/layer1_attention_weight_min": -14.500041007995605,
      "activations/layer20_attention_weight_max": 27.072044372558594,
      "activations/layer20_attention_weight_min": -26.972383499145508,
      "activations/layer21_attention_weight_max": 28.479793548583984,
      "activations/layer21_attention_weight_min": -27.66234588623047,
      "activations/layer22_attention_weight_max": 41.20256805419922,
      "activations/layer22_attention_weight_min": -33.54643630981445,
      "activations/layer23_attention_weight_max": 32.35520553588867,
      "activations/layer23_attention_weight_min": -27.52396011352539,
      "activations/layer2_attention_weight_max": 33.07720947265625,
      "activations/layer2_attention_weight_min": -31.721927642822266,
      "activations/layer3_attention_weight_max": 96.48910522460938,
      "activations/layer3_attention_weight_min": -96.6685791015625,
      "activations/layer4_attention_weight_max": 105.92388916015625,
      "activations/layer4_attention_weight_min": -106.28414154052734,
      "activations/layer5_attention_weight_max": 52.40024948120117,
      "activations/layer5_attention_weight_min": -65.66683197021484,
      "activations/layer6_attention_weight_max": 42.33970642089844,
      "activations/layer6_attention_weight_min": -46.33448028564453,
      "activations/layer7_attention_weight_max": 95.41706848144531,
      "activations/layer7_attention_weight_min": -95.27989959716797,
      "activations/layer8_attention_weight_max": 40.2803840637207,
      "activations/layer8_attention_weight_min": -43.12083435058594,
      "activations/layer9_attention_weight_max": 31.1451416015625,
      "activations/layer9_attention_weight_min": -32.12248611450195,
      "epoch": 16.55,
      "learning_rate": 4.370416666666667e-05,
      "loss": 2.7395,
      "step": 284850
    },
    {
      "activations/layer0_attention_weight_max": 16.107120513916016,
      "activations/layer0_attention_weight_min": -13.021268844604492,
      "activations/layer10_attention_weight_max": 33.0737190246582,
      "activations/layer10_attention_weight_min": -35.0816650390625,
      "activations/layer11_attention_weight_max": 31.727840423583984,
      "activations/layer11_attention_weight_min": -35.42669677734375,
      "activations/layer12_attention_weight_max": 23.703739166259766,
      "activations/layer12_attention_weight_min": -26.04125213623047,
      "activations/layer13_attention_weight_max": 36.65492248535156,
      "activations/layer13_attention_weight_min": -34.719932556152344,
      "activations/layer14_attention_weight_max": 38.424922943115234,
      "activations/layer14_attention_weight_min": -37.290985107421875,
      "activations/layer15_attention_weight_max": 35.68669891357422,
      "activations/layer15_attention_weight_min": -34.733463287353516,
      "activations/layer16_attention_weight_max": 30.69622039794922,
      "activations/layer16_attention_weight_min": -29.574180603027344,
      "activations/layer17_attention_weight_max": 28.486963272094727,
      "activations/layer17_attention_weight_min": -26.820999145507812,
      "activations/layer18_attention_weight_max": 30.653291702270508,
      "activations/layer18_attention_weight_min": -26.71462059020996,
      "activations/layer19_attention_weight_max": 35.0284423828125,
      "activations/layer19_attention_weight_min": -36.53068161010742,
      "activations/layer1_attention_weight_max": 16.482065200805664,
      "activations/layer1_attention_weight_min": -14.399849891662598,
      "activations/layer20_attention_weight_max": 30.02361488342285,
      "activations/layer20_attention_weight_min": -26.830333709716797,
      "activations/layer21_attention_weight_max": 29.982975006103516,
      "activations/layer21_attention_weight_min": -26.152917861938477,
      "activations/layer22_attention_weight_max": 45.8232536315918,
      "activations/layer22_attention_weight_min": -34.377403259277344,
      "activations/layer23_attention_weight_max": 32.858741760253906,
      "activations/layer23_attention_weight_min": -26.21438217163086,
      "activations/layer2_attention_weight_max": 32.913536071777344,
      "activations/layer2_attention_weight_min": -32.592430114746094,
      "activations/layer3_attention_weight_max": 91.99711608886719,
      "activations/layer3_attention_weight_min": -94.77470397949219,
      "activations/layer4_attention_weight_max": 106.18031311035156,
      "activations/layer4_attention_weight_min": -106.29583740234375,
      "activations/layer5_attention_weight_max": 47.44802474975586,
      "activations/layer5_attention_weight_min": -60.93362045288086,
      "activations/layer6_attention_weight_max": 39.73646926879883,
      "activations/layer6_attention_weight_min": -42.778038024902344,
      "activations/layer7_attention_weight_max": 86.18307495117188,
      "activations/layer7_attention_weight_min": -87.1517333984375,
      "activations/layer8_attention_weight_max": 37.753074645996094,
      "activations/layer8_attention_weight_min": -39.96458053588867,
      "activations/layer9_attention_weight_max": 29.540271759033203,
      "activations/layer9_attention_weight_min": -34.00865173339844,
      "epoch": 16.55,
      "learning_rate": 4.368522727272727e-05,
      "loss": 2.7345,
      "step": 284900
    },
    {
      "activations/layer0_attention_weight_max": 16.209991455078125,
      "activations/layer0_attention_weight_min": -14.269015312194824,
      "activations/layer10_attention_weight_max": 34.24019241333008,
      "activations/layer10_attention_weight_min": -34.569435119628906,
      "activations/layer11_attention_weight_max": 35.38050842285156,
      "activations/layer11_attention_weight_min": -32.0797119140625,
      "activations/layer12_attention_weight_max": 29.650890350341797,
      "activations/layer12_attention_weight_min": -27.555456161499023,
      "activations/layer13_attention_weight_max": 45.74446105957031,
      "activations/layer13_attention_weight_min": -36.12566375732422,
      "activations/layer14_attention_weight_max": 49.325565338134766,
      "activations/layer14_attention_weight_min": -37.71939468383789,
      "activations/layer15_attention_weight_max": 43.09626007080078,
      "activations/layer15_attention_weight_min": -36.0462646484375,
      "activations/layer16_attention_weight_max": 30.675830841064453,
      "activations/layer16_attention_weight_min": -27.419776916503906,
      "activations/layer17_attention_weight_max": 33.65329360961914,
      "activations/layer17_attention_weight_min": -25.527761459350586,
      "activations/layer18_attention_weight_max": 31.59480094909668,
      "activations/layer18_attention_weight_min": -25.415603637695312,
      "activations/layer19_attention_weight_max": 37.58037185668945,
      "activations/layer19_attention_weight_min": -30.445514678955078,
      "activations/layer1_attention_weight_max": 15.490381240844727,
      "activations/layer1_attention_weight_min": -14.370379447937012,
      "activations/layer20_attention_weight_max": 32.11918258666992,
      "activations/layer20_attention_weight_min": -23.703716278076172,
      "activations/layer21_attention_weight_max": 32.6019172668457,
      "activations/layer21_attention_weight_min": -23.81587791442871,
      "activations/layer22_attention_weight_max": 43.39070510864258,
      "activations/layer22_attention_weight_min": -31.374284744262695,
      "activations/layer23_attention_weight_max": 32.45362091064453,
      "activations/layer23_attention_weight_min": -26.346641540527344,
      "activations/layer2_attention_weight_max": 36.063594818115234,
      "activations/layer2_attention_weight_min": -34.616790771484375,
      "activations/layer3_attention_weight_max": 103.29444885253906,
      "activations/layer3_attention_weight_min": -104.29446411132812,
      "activations/layer4_attention_weight_max": 109.88825988769531,
      "activations/layer4_attention_weight_min": -108.81624603271484,
      "activations/layer5_attention_weight_max": 49.15351104736328,
      "activations/layer5_attention_weight_min": -64.09579467773438,
      "activations/layer6_attention_weight_max": 43.4266471862793,
      "activations/layer6_attention_weight_min": -47.025169372558594,
      "activations/layer7_attention_weight_max": 86.88343048095703,
      "activations/layer7_attention_weight_min": -96.01412200927734,
      "activations/layer8_attention_weight_max": 40.38899612426758,
      "activations/layer8_attention_weight_min": -41.35652542114258,
      "activations/layer9_attention_weight_max": 33.30019760131836,
      "activations/layer9_attention_weight_min": -34.50896072387695,
      "epoch": 16.56,
      "learning_rate": 4.366628787878787e-05,
      "loss": 2.7427,
      "step": 284950
    },
    {
      "activations/layer0_attention_weight_max": 16.51580238342285,
      "activations/layer0_attention_weight_min": -12.191648483276367,
      "activations/layer10_attention_weight_max": 34.91156768798828,
      "activations/layer10_attention_weight_min": -34.86450958251953,
      "activations/layer11_attention_weight_max": 34.652652740478516,
      "activations/layer11_attention_weight_min": -36.937686920166016,
      "activations/layer12_attention_weight_max": 28.221988677978516,
      "activations/layer12_attention_weight_min": -27.411340713500977,
      "activations/layer13_attention_weight_max": 43.42817687988281,
      "activations/layer13_attention_weight_min": -37.60865020751953,
      "activations/layer14_attention_weight_max": 43.126739501953125,
      "activations/layer14_attention_weight_min": -40.4080696105957,
      "activations/layer15_attention_weight_max": 40.20531463623047,
      "activations/layer15_attention_weight_min": -36.81772232055664,
      "activations/layer16_attention_weight_max": 30.962491989135742,
      "activations/layer16_attention_weight_min": -29.65553092956543,
      "activations/layer17_attention_weight_max": 32.458984375,
      "activations/layer17_attention_weight_min": -27.9849853515625,
      "activations/layer18_attention_weight_max": 37.03675079345703,
      "activations/layer18_attention_weight_min": -26.98542594909668,
      "activations/layer19_attention_weight_max": 39.6385498046875,
      "activations/layer19_attention_weight_min": -33.65059280395508,
      "activations/layer1_attention_weight_max": 16.600473403930664,
      "activations/layer1_attention_weight_min": -13.67699909210205,
      "activations/layer20_attention_weight_max": 33.60933303833008,
      "activations/layer20_attention_weight_min": -26.571304321289062,
      "activations/layer21_attention_weight_max": 33.38734817504883,
      "activations/layer21_attention_weight_min": -26.607566833496094,
      "activations/layer22_attention_weight_max": 50.54785919189453,
      "activations/layer22_attention_weight_min": -37.51495361328125,
      "activations/layer23_attention_weight_max": 38.057228088378906,
      "activations/layer23_attention_weight_min": -29.985332489013672,
      "activations/layer2_attention_weight_max": 32.09912109375,
      "activations/layer2_attention_weight_min": -32.136940002441406,
      "activations/layer3_attention_weight_max": 95.11670684814453,
      "activations/layer3_attention_weight_min": -95.60847473144531,
      "activations/layer4_attention_weight_max": 104.7302017211914,
      "activations/layer4_attention_weight_min": -108.25882720947266,
      "activations/layer5_attention_weight_max": 50.6083984375,
      "activations/layer5_attention_weight_min": -60.776485443115234,
      "activations/layer6_attention_weight_max": 42.877132415771484,
      "activations/layer6_attention_weight_min": -48.104957580566406,
      "activations/layer7_attention_weight_max": 90.61599731445312,
      "activations/layer7_attention_weight_min": -93.24825286865234,
      "activations/layer8_attention_weight_max": 41.044456481933594,
      "activations/layer8_attention_weight_min": -42.85961151123047,
      "activations/layer9_attention_weight_max": 33.1791877746582,
      "activations/layer9_attention_weight_min": -34.876487731933594,
      "epoch": 16.56,
      "learning_rate": 4.3647348484848485e-05,
      "loss": 2.7385,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4837,
      "eval_samples_per_second": 506.148,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4837,
      "eval_openwebtext_samples_per_second": 506.148,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9748,
      "eval_wikitext_samples_per_second": 230.909,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_lambada_loss": 2.38671875,
      "eval_lambada_ppl": 10.877742727456024,
      "eval_lambada_runtime": 9.5563,
      "eval_lambada_samples_per_second": 509.505,
      "step": 285000
    },
    {
      "activations/layer0_attention_weight_max": 16.058900833129883,
      "activations/layer0_attention_weight_min": -13.530800819396973,
      "activations/layer10_attention_weight_max": 34.449554443359375,
      "activations/layer10_attention_weight_min": -33.881202697753906,
      "activations/layer11_attention_weight_max": 32.76506423950195,
      "activations/layer11_attention_weight_min": -33.170814514160156,
      "activations/layer12_attention_weight_max": 24.68667984008789,
      "activations/layer12_attention_weight_min": -24.8576717376709,
      "activations/layer13_attention_weight_max": 36.45734405517578,
      "activations/layer13_attention_weight_min": -34.335777282714844,
      "activations/layer14_attention_weight_max": 43.185707092285156,
      "activations/layer14_attention_weight_min": -38.51553726196289,
      "activations/layer15_attention_weight_max": 41.127471923828125,
      "activations/layer15_attention_weight_min": -34.849510192871094,
      "activations/layer16_attention_weight_max": 27.200923919677734,
      "activations/layer16_attention_weight_min": -26.511220932006836,
      "activations/layer17_attention_weight_max": 29.512969970703125,
      "activations/layer17_attention_weight_min": -25.447961807250977,
      "activations/layer18_attention_weight_max": 28.6519718170166,
      "activations/layer18_attention_weight_min": -25.693893432617188,
      "activations/layer19_attention_weight_max": 30.009626388549805,
      "activations/layer19_attention_weight_min": -30.88233757019043,
      "activations/layer1_attention_weight_max": 15.736199378967285,
      "activations/layer1_attention_weight_min": -14.397269248962402,
      "activations/layer20_attention_weight_max": 26.492719650268555,
      "activations/layer20_attention_weight_min": -24.4838924407959,
      "activations/layer21_attention_weight_max": 28.384227752685547,
      "activations/layer21_attention_weight_min": -24.703771591186523,
      "activations/layer22_attention_weight_max": 35.87905502319336,
      "activations/layer22_attention_weight_min": -30.68549919128418,
      "activations/layer23_attention_weight_max": 33.539955139160156,
      "activations/layer23_attention_weight_min": -25.283000946044922,
      "activations/layer2_attention_weight_max": 33.29585266113281,
      "activations/layer2_attention_weight_min": -33.93325424194336,
      "activations/layer3_attention_weight_max": 97.51028442382812,
      "activations/layer3_attention_weight_min": -95.79904174804688,
      "activations/layer4_attention_weight_max": 104.44203186035156,
      "activations/layer4_attention_weight_min": -106.34378814697266,
      "activations/layer5_attention_weight_max": 48.733787536621094,
      "activations/layer5_attention_weight_min": -58.33897399902344,
      "activations/layer6_attention_weight_max": 43.16840744018555,
      "activations/layer6_attention_weight_min": -46.84059143066406,
      "activations/layer7_attention_weight_max": 87.95890808105469,
      "activations/layer7_attention_weight_min": -94.65643310546875,
      "activations/layer8_attention_weight_max": 40.535335540771484,
      "activations/layer8_attention_weight_min": -46.42202377319336,
      "activations/layer9_attention_weight_max": 32.19172286987305,
      "activations/layer9_attention_weight_min": -34.37553024291992,
      "epoch": 16.56,
      "learning_rate": 4.362840909090909e-05,
      "loss": 2.7409,
      "step": 285050
    },
    {
      "activations/layer0_attention_weight_max": 17.311290740966797,
      "activations/layer0_attention_weight_min": -12.50687026977539,
      "activations/layer10_attention_weight_max": 33.6905403137207,
      "activations/layer10_attention_weight_min": -34.26759719848633,
      "activations/layer11_attention_weight_max": 34.100502014160156,
      "activations/layer11_attention_weight_min": -35.00830078125,
      "activations/layer12_attention_weight_max": 28.18572425842285,
      "activations/layer12_attention_weight_min": -27.153696060180664,
      "activations/layer13_attention_weight_max": 42.9007682800293,
      "activations/layer13_attention_weight_min": -37.0007209777832,
      "activations/layer14_attention_weight_max": 43.07636260986328,
      "activations/layer14_attention_weight_min": -41.16139602661133,
      "activations/layer15_attention_weight_max": 41.83551025390625,
      "activations/layer15_attention_weight_min": -37.55982208251953,
      "activations/layer16_attention_weight_max": 33.50754928588867,
      "activations/layer16_attention_weight_min": -30.78882598876953,
      "activations/layer17_attention_weight_max": 32.06010818481445,
      "activations/layer17_attention_weight_min": -26.733354568481445,
      "activations/layer18_attention_weight_max": 32.57136535644531,
      "activations/layer18_attention_weight_min": -25.66966438293457,
      "activations/layer19_attention_weight_max": 37.74369812011719,
      "activations/layer19_attention_weight_min": -33.37857437133789,
      "activations/layer1_attention_weight_max": 16.871013641357422,
      "activations/layer1_attention_weight_min": -14.735194206237793,
      "activations/layer20_attention_weight_max": 33.61566925048828,
      "activations/layer20_attention_weight_min": -26.803184509277344,
      "activations/layer21_attention_weight_max": 32.060997009277344,
      "activations/layer21_attention_weight_min": -25.23750877380371,
      "activations/layer22_attention_weight_max": 48.12919235229492,
      "activations/layer22_attention_weight_min": -32.76024627685547,
      "activations/layer23_attention_weight_max": 38.40622329711914,
      "activations/layer23_attention_weight_min": -25.59140396118164,
      "activations/layer2_attention_weight_max": 32.62178039550781,
      "activations/layer2_attention_weight_min": -33.58442687988281,
      "activations/layer3_attention_weight_max": 96.00391387939453,
      "activations/layer3_attention_weight_min": -97.67381286621094,
      "activations/layer4_attention_weight_max": 104.09050750732422,
      "activations/layer4_attention_weight_min": -110.69766998291016,
      "activations/layer5_attention_weight_max": 49.458274841308594,
      "activations/layer5_attention_weight_min": -65.43822479248047,
      "activations/layer6_attention_weight_max": 42.41853332519531,
      "activations/layer6_attention_weight_min": -47.638023376464844,
      "activations/layer7_attention_weight_max": 96.6907958984375,
      "activations/layer7_attention_weight_min": -98.32933807373047,
      "activations/layer8_attention_weight_max": 42.531253814697266,
      "activations/layer8_attention_weight_min": -43.91669845581055,
      "activations/layer9_attention_weight_max": 31.209585189819336,
      "activations/layer9_attention_weight_min": -33.0855712890625,
      "epoch": 16.57,
      "learning_rate": 4.3609469696969695e-05,
      "loss": 2.7456,
      "step": 285100
    },
    {
      "activations/layer0_attention_weight_max": 14.692667007446289,
      "activations/layer0_attention_weight_min": -12.173901557922363,
      "activations/layer10_attention_weight_max": 31.275943756103516,
      "activations/layer10_attention_weight_min": -31.911344528198242,
      "activations/layer11_attention_weight_max": 32.56716537475586,
      "activations/layer11_attention_weight_min": -32.505393981933594,
      "activations/layer12_attention_weight_max": 24.71187973022461,
      "activations/layer12_attention_weight_min": -25.226423263549805,
      "activations/layer13_attention_weight_max": 38.5008659362793,
      "activations/layer13_attention_weight_min": -35.6966438293457,
      "activations/layer14_attention_weight_max": 40.67622375488281,
      "activations/layer14_attention_weight_min": -36.74756622314453,
      "activations/layer15_attention_weight_max": 38.22504425048828,
      "activations/layer15_attention_weight_min": -34.575462341308594,
      "activations/layer16_attention_weight_max": 30.43919563293457,
      "activations/layer16_attention_weight_min": -26.378299713134766,
      "activations/layer17_attention_weight_max": 28.274581909179688,
      "activations/layer17_attention_weight_min": -26.567434310913086,
      "activations/layer18_attention_weight_max": 30.832927703857422,
      "activations/layer18_attention_weight_min": -24.3065185546875,
      "activations/layer19_attention_weight_max": 31.29102897644043,
      "activations/layer19_attention_weight_min": -31.729536056518555,
      "activations/layer1_attention_weight_max": 15.897140502929688,
      "activations/layer1_attention_weight_min": -14.743192672729492,
      "activations/layer20_attention_weight_max": 26.27614974975586,
      "activations/layer20_attention_weight_min": -25.998706817626953,
      "activations/layer21_attention_weight_max": 26.57219123840332,
      "activations/layer21_attention_weight_min": -25.158767700195312,
      "activations/layer22_attention_weight_max": 38.87057876586914,
      "activations/layer22_attention_weight_min": -29.633808135986328,
      "activations/layer23_attention_weight_max": 31.155691146850586,
      "activations/layer23_attention_weight_min": -25.762327194213867,
      "activations/layer2_attention_weight_max": 32.025001525878906,
      "activations/layer2_attention_weight_min": -30.854055404663086,
      "activations/layer3_attention_weight_max": 97.53848266601562,
      "activations/layer3_attention_weight_min": -96.24845886230469,
      "activations/layer4_attention_weight_max": 105.20692443847656,
      "activations/layer4_attention_weight_min": -106.7305679321289,
      "activations/layer5_attention_weight_max": 51.488746643066406,
      "activations/layer5_attention_weight_min": -59.59112548828125,
      "activations/layer6_attention_weight_max": 43.092857360839844,
      "activations/layer6_attention_weight_min": -44.242916107177734,
      "activations/layer7_attention_weight_max": 91.14232635498047,
      "activations/layer7_attention_weight_min": -92.23763275146484,
      "activations/layer8_attention_weight_max": 40.3406867980957,
      "activations/layer8_attention_weight_min": -42.540199279785156,
      "activations/layer9_attention_weight_max": 29.779041290283203,
      "activations/layer9_attention_weight_min": -31.678903579711914,
      "epoch": 16.57,
      "learning_rate": 4.35905303030303e-05,
      "loss": 2.7273,
      "step": 285150
    },
    {
      "activations/layer0_attention_weight_max": 16.002639770507812,
      "activations/layer0_attention_weight_min": -12.643173217773438,
      "activations/layer10_attention_weight_max": 34.840362548828125,
      "activations/layer10_attention_weight_min": -33.84678649902344,
      "activations/layer11_attention_weight_max": 34.76213455200195,
      "activations/layer11_attention_weight_min": -33.85847854614258,
      "activations/layer12_attention_weight_max": 25.984777450561523,
      "activations/layer12_attention_weight_min": -24.988399505615234,
      "activations/layer13_attention_weight_max": 39.620052337646484,
      "activations/layer13_attention_weight_min": -36.01288986206055,
      "activations/layer14_attention_weight_max": 43.54261016845703,
      "activations/layer14_attention_weight_min": -36.11722946166992,
      "activations/layer15_attention_weight_max": 37.776390075683594,
      "activations/layer15_attention_weight_min": -33.639686584472656,
      "activations/layer16_attention_weight_max": 32.167171478271484,
      "activations/layer16_attention_weight_min": -28.069162368774414,
      "activations/layer17_attention_weight_max": 32.51447677612305,
      "activations/layer17_attention_weight_min": -25.967493057250977,
      "activations/layer18_attention_weight_max": 33.02860641479492,
      "activations/layer18_attention_weight_min": -25.711214065551758,
      "activations/layer19_attention_weight_max": 40.64103698730469,
      "activations/layer19_attention_weight_min": -32.904727935791016,
      "activations/layer1_attention_weight_max": 15.584236145019531,
      "activations/layer1_attention_weight_min": -15.84034538269043,
      "activations/layer20_attention_weight_max": 31.666725158691406,
      "activations/layer20_attention_weight_min": -26.954383850097656,
      "activations/layer21_attention_weight_max": 29.369455337524414,
      "activations/layer21_attention_weight_min": -26.74697494506836,
      "activations/layer22_attention_weight_max": 50.02546691894531,
      "activations/layer22_attention_weight_min": -32.9675178527832,
      "activations/layer23_attention_weight_max": 41.070987701416016,
      "activations/layer23_attention_weight_min": -26.737564086914062,
      "activations/layer2_attention_weight_max": 33.1502685546875,
      "activations/layer2_attention_weight_min": -32.26695251464844,
      "activations/layer3_attention_weight_max": 97.03411102294922,
      "activations/layer3_attention_weight_min": -95.83009338378906,
      "activations/layer4_attention_weight_max": 105.0932388305664,
      "activations/layer4_attention_weight_min": -102.94945526123047,
      "activations/layer5_attention_weight_max": 51.8222770690918,
      "activations/layer5_attention_weight_min": -62.99480438232422,
      "activations/layer6_attention_weight_max": 43.361656188964844,
      "activations/layer6_attention_weight_min": -45.07564163208008,
      "activations/layer7_attention_weight_max": 89.44783782958984,
      "activations/layer7_attention_weight_min": -94.02131652832031,
      "activations/layer8_attention_weight_max": 41.897579193115234,
      "activations/layer8_attention_weight_min": -41.70979690551758,
      "activations/layer9_attention_weight_max": 31.99445152282715,
      "activations/layer9_attention_weight_min": -33.50625228881836,
      "epoch": 16.57,
      "learning_rate": 4.357159090909091e-05,
      "loss": 2.7303,
      "step": 285200
    },
    {
      "activations/layer0_attention_weight_max": 16.29653549194336,
      "activations/layer0_attention_weight_min": -13.130817413330078,
      "activations/layer10_attention_weight_max": 37.589271545410156,
      "activations/layer10_attention_weight_min": -37.718719482421875,
      "activations/layer11_attention_weight_max": 36.457603454589844,
      "activations/layer11_attention_weight_min": -35.753421783447266,
      "activations/layer12_attention_weight_max": 26.621976852416992,
      "activations/layer12_attention_weight_min": -26.726680755615234,
      "activations/layer13_attention_weight_max": 39.8294677734375,
      "activations/layer13_attention_weight_min": -39.040802001953125,
      "activations/layer14_attention_weight_max": 43.723209381103516,
      "activations/layer14_attention_weight_min": -38.736324310302734,
      "activations/layer15_attention_weight_max": 39.60986328125,
      "activations/layer15_attention_weight_min": -33.50804901123047,
      "activations/layer16_attention_weight_max": 30.319128036499023,
      "activations/layer16_attention_weight_min": -28.501707077026367,
      "activations/layer17_attention_weight_max": 30.341094970703125,
      "activations/layer17_attention_weight_min": -26.696653366088867,
      "activations/layer18_attention_weight_max": 32.66835021972656,
      "activations/layer18_attention_weight_min": -25.14899253845215,
      "activations/layer19_attention_weight_max": 34.39529037475586,
      "activations/layer19_attention_weight_min": -30.31254005432129,
      "activations/layer1_attention_weight_max": 15.323563575744629,
      "activations/layer1_attention_weight_min": -15.562019348144531,
      "activations/layer20_attention_weight_max": 27.222156524658203,
      "activations/layer20_attention_weight_min": -23.84397315979004,
      "activations/layer21_attention_weight_max": 29.52323341369629,
      "activations/layer21_attention_weight_min": -23.847158432006836,
      "activations/layer22_attention_weight_max": 42.00938034057617,
      "activations/layer22_attention_weight_min": -32.44312286376953,
      "activations/layer23_attention_weight_max": 33.156986236572266,
      "activations/layer23_attention_weight_min": -25.80079460144043,
      "activations/layer2_attention_weight_max": 33.637115478515625,
      "activations/layer2_attention_weight_min": -32.752017974853516,
      "activations/layer3_attention_weight_max": 99.70745086669922,
      "activations/layer3_attention_weight_min": -97.72679138183594,
      "activations/layer4_attention_weight_max": 107.51106262207031,
      "activations/layer4_attention_weight_min": -107.8918228149414,
      "activations/layer5_attention_weight_max": 53.10393524169922,
      "activations/layer5_attention_weight_min": -60.3718376159668,
      "activations/layer6_attention_weight_max": 43.45662307739258,
      "activations/layer6_attention_weight_min": -45.77504348754883,
      "activations/layer7_attention_weight_max": 94.44656372070312,
      "activations/layer7_attention_weight_min": -97.48789978027344,
      "activations/layer8_attention_weight_max": 44.49004364013672,
      "activations/layer8_attention_weight_min": -47.08604431152344,
      "activations/layer9_attention_weight_max": 34.53531265258789,
      "activations/layer9_attention_weight_min": -37.36939239501953,
      "epoch": 16.57,
      "learning_rate": 4.355265151515151e-05,
      "loss": 2.746,
      "step": 285250
    },
    {
      "activations/layer0_attention_weight_max": 16.60757827758789,
      "activations/layer0_attention_weight_min": -12.514067649841309,
      "activations/layer10_attention_weight_max": 41.1899528503418,
      "activations/layer10_attention_weight_min": -39.04069900512695,
      "activations/layer11_attention_weight_max": 39.053436279296875,
      "activations/layer11_attention_weight_min": -38.061031341552734,
      "activations/layer12_attention_weight_max": 32.18310546875,
      "activations/layer12_attention_weight_min": -29.57500648498535,
      "activations/layer13_attention_weight_max": 44.17265701293945,
      "activations/layer13_attention_weight_min": -39.78404235839844,
      "activations/layer14_attention_weight_max": 47.3633918762207,
      "activations/layer14_attention_weight_min": -43.40951919555664,
      "activations/layer15_attention_weight_max": 41.03643035888672,
      "activations/layer15_attention_weight_min": -38.87097930908203,
      "activations/layer16_attention_weight_max": 29.556344985961914,
      "activations/layer16_attention_weight_min": -28.56669807434082,
      "activations/layer17_attention_weight_max": 29.00234031677246,
      "activations/layer17_attention_weight_min": -26.574872970581055,
      "activations/layer18_attention_weight_max": 27.328876495361328,
      "activations/layer18_attention_weight_min": -25.691701889038086,
      "activations/layer19_attention_weight_max": 29.94329261779785,
      "activations/layer19_attention_weight_min": -30.629459381103516,
      "activations/layer1_attention_weight_max": 15.471911430358887,
      "activations/layer1_attention_weight_min": -14.415945053100586,
      "activations/layer20_attention_weight_max": 25.629793167114258,
      "activations/layer20_attention_weight_min": -25.27351951599121,
      "activations/layer21_attention_weight_max": 26.81606674194336,
      "activations/layer21_attention_weight_min": -25.056352615356445,
      "activations/layer22_attention_weight_max": 38.581939697265625,
      "activations/layer22_attention_weight_min": -29.77948760986328,
      "activations/layer23_attention_weight_max": 30.458410263061523,
      "activations/layer23_attention_weight_min": -25.0778865814209,
      "activations/layer2_attention_weight_max": 33.71666717529297,
      "activations/layer2_attention_weight_min": -32.854740142822266,
      "activations/layer3_attention_weight_max": 94.25829315185547,
      "activations/layer3_attention_weight_min": -94.84034729003906,
      "activations/layer4_attention_weight_max": 104.05303955078125,
      "activations/layer4_attention_weight_min": -105.64874267578125,
      "activations/layer5_attention_weight_max": 49.53668212890625,
      "activations/layer5_attention_weight_min": -58.01932907104492,
      "activations/layer6_attention_weight_max": 45.30632019042969,
      "activations/layer6_attention_weight_min": -46.04806900024414,
      "activations/layer7_attention_weight_max": 96.8309326171875,
      "activations/layer7_attention_weight_min": -101.69306945800781,
      "activations/layer8_attention_weight_max": 47.245704650878906,
      "activations/layer8_attention_weight_min": -48.5163688659668,
      "activations/layer9_attention_weight_max": 37.21237564086914,
      "activations/layer9_attention_weight_min": -37.83821105957031,
      "epoch": 16.58,
      "learning_rate": 4.3533712121212115e-05,
      "loss": 2.7512,
      "step": 285300
    },
    {
      "activations/layer0_attention_weight_max": 16.133285522460938,
      "activations/layer0_attention_weight_min": -12.412056922912598,
      "activations/layer10_attention_weight_max": 33.367618560791016,
      "activations/layer10_attention_weight_min": -33.7552490234375,
      "activations/layer11_attention_weight_max": 34.24720764160156,
      "activations/layer11_attention_weight_min": -35.16926574707031,
      "activations/layer12_attention_weight_max": 26.859786987304688,
      "activations/layer12_attention_weight_min": -26.080446243286133,
      "activations/layer13_attention_weight_max": 39.12260055541992,
      "activations/layer13_attention_weight_min": -34.28722381591797,
      "activations/layer14_attention_weight_max": 41.4218864440918,
      "activations/layer14_attention_weight_min": -36.43278884887695,
      "activations/layer15_attention_weight_max": 34.794857025146484,
      "activations/layer15_attention_weight_min": -34.99592971801758,
      "activations/layer16_attention_weight_max": 27.36522102355957,
      "activations/layer16_attention_weight_min": -30.060646057128906,
      "activations/layer17_attention_weight_max": 31.61985969543457,
      "activations/layer17_attention_weight_min": -26.467254638671875,
      "activations/layer18_attention_weight_max": 30.855060577392578,
      "activations/layer18_attention_weight_min": -27.437822341918945,
      "activations/layer19_attention_weight_max": 34.995948791503906,
      "activations/layer19_attention_weight_min": -31.390750885009766,
      "activations/layer1_attention_weight_max": 16.36587905883789,
      "activations/layer1_attention_weight_min": -14.018701553344727,
      "activations/layer20_attention_weight_max": 31.061738967895508,
      "activations/layer20_attention_weight_min": -24.778358459472656,
      "activations/layer21_attention_weight_max": 32.1959228515625,
      "activations/layer21_attention_weight_min": -24.258575439453125,
      "activations/layer22_attention_weight_max": 40.887664794921875,
      "activations/layer22_attention_weight_min": -32.41932678222656,
      "activations/layer23_attention_weight_max": 29.892730712890625,
      "activations/layer23_attention_weight_min": -25.099193572998047,
      "activations/layer2_attention_weight_max": 31.377395629882812,
      "activations/layer2_attention_weight_min": -32.45806884765625,
      "activations/layer3_attention_weight_max": 94.36283874511719,
      "activations/layer3_attention_weight_min": -96.3699722290039,
      "activations/layer4_attention_weight_max": 103.04540252685547,
      "activations/layer4_attention_weight_min": -102.0604019165039,
      "activations/layer5_attention_weight_max": 51.93772888183594,
      "activations/layer5_attention_weight_min": -61.99934387207031,
      "activations/layer6_attention_weight_max": 40.58913803100586,
      "activations/layer6_attention_weight_min": -43.652706146240234,
      "activations/layer7_attention_weight_max": 91.40386962890625,
      "activations/layer7_attention_weight_min": -88.1258544921875,
      "activations/layer8_attention_weight_max": 41.283565521240234,
      "activations/layer8_attention_weight_min": -40.576683044433594,
      "activations/layer9_attention_weight_max": 32.48537063598633,
      "activations/layer9_attention_weight_min": -33.89616394042969,
      "epoch": 16.58,
      "learning_rate": 4.351477272727272e-05,
      "loss": 2.7363,
      "step": 285350
    },
    {
      "activations/layer0_attention_weight_max": 15.953399658203125,
      "activations/layer0_attention_weight_min": -12.618132591247559,
      "activations/layer10_attention_weight_max": 33.316932678222656,
      "activations/layer10_attention_weight_min": -35.8270149230957,
      "activations/layer11_attention_weight_max": 32.49440383911133,
      "activations/layer11_attention_weight_min": -35.98307800292969,
      "activations/layer12_attention_weight_max": 27.443317413330078,
      "activations/layer12_attention_weight_min": -26.0510196685791,
      "activations/layer13_attention_weight_max": 41.05952453613281,
      "activations/layer13_attention_weight_min": -35.85712814331055,
      "activations/layer14_attention_weight_max": 43.718414306640625,
      "activations/layer14_attention_weight_min": -38.827049255371094,
      "activations/layer15_attention_weight_max": 39.36967849731445,
      "activations/layer15_attention_weight_min": -37.61518096923828,
      "activations/layer16_attention_weight_max": 31.25531005859375,
      "activations/layer16_attention_weight_min": -28.52903175354004,
      "activations/layer17_attention_weight_max": 29.875268936157227,
      "activations/layer17_attention_weight_min": -26.59111976623535,
      "activations/layer18_attention_weight_max": 30.730892181396484,
      "activations/layer18_attention_weight_min": -25.05915641784668,
      "activations/layer19_attention_weight_max": 31.14988136291504,
      "activations/layer19_attention_weight_min": -30.26894187927246,
      "activations/layer1_attention_weight_max": 16.90479850769043,
      "activations/layer1_attention_weight_min": -15.045339584350586,
      "activations/layer20_attention_weight_max": 28.010208129882812,
      "activations/layer20_attention_weight_min": -25.322553634643555,
      "activations/layer21_attention_weight_max": 27.1888427734375,
      "activations/layer21_attention_weight_min": -23.548583984375,
      "activations/layer22_attention_weight_max": 39.390228271484375,
      "activations/layer22_attention_weight_min": -30.28122329711914,
      "activations/layer23_attention_weight_max": 33.0273323059082,
      "activations/layer23_attention_weight_min": -25.290830612182617,
      "activations/layer2_attention_weight_max": 32.708675384521484,
      "activations/layer2_attention_weight_min": -33.21275329589844,
      "activations/layer3_attention_weight_max": 97.1117172241211,
      "activations/layer3_attention_weight_min": -96.27333068847656,
      "activations/layer4_attention_weight_max": 107.0167236328125,
      "activations/layer4_attention_weight_min": -107.58931732177734,
      "activations/layer5_attention_weight_max": 50.69977951049805,
      "activations/layer5_attention_weight_min": -61.723236083984375,
      "activations/layer6_attention_weight_max": 43.1365852355957,
      "activations/layer6_attention_weight_min": -48.21340560913086,
      "activations/layer7_attention_weight_max": 88.3231201171875,
      "activations/layer7_attention_weight_min": -97.53926086425781,
      "activations/layer8_attention_weight_max": 40.097801208496094,
      "activations/layer8_attention_weight_min": -43.50519943237305,
      "activations/layer9_attention_weight_max": 30.216861724853516,
      "activations/layer9_attention_weight_min": -34.03419876098633,
      "epoch": 16.58,
      "learning_rate": 4.349583333333333e-05,
      "loss": 2.7462,
      "step": 285400
    },
    {
      "activations/layer0_attention_weight_max": 14.963205337524414,
      "activations/layer0_attention_weight_min": -12.173806190490723,
      "activations/layer10_attention_weight_max": 30.743093490600586,
      "activations/layer10_attention_weight_min": -31.460378646850586,
      "activations/layer11_attention_weight_max": 34.14094543457031,
      "activations/layer11_attention_weight_min": -31.06566619873047,
      "activations/layer12_attention_weight_max": 25.94567108154297,
      "activations/layer12_attention_weight_min": -25.227325439453125,
      "activations/layer13_attention_weight_max": 37.850730895996094,
      "activations/layer13_attention_weight_min": -33.73714828491211,
      "activations/layer14_attention_weight_max": 39.79433822631836,
      "activations/layer14_attention_weight_min": -37.416229248046875,
      "activations/layer15_attention_weight_max": 37.47840881347656,
      "activations/layer15_attention_weight_min": -34.274810791015625,
      "activations/layer16_attention_weight_max": 29.39437484741211,
      "activations/layer16_attention_weight_min": -30.022192001342773,
      "activations/layer17_attention_weight_max": 30.988338470458984,
      "activations/layer17_attention_weight_min": -27.360519409179688,
      "activations/layer18_attention_weight_max": 33.3647575378418,
      "activations/layer18_attention_weight_min": -25.998741149902344,
      "activations/layer19_attention_weight_max": 35.33357238769531,
      "activations/layer19_attention_weight_min": -32.61214065551758,
      "activations/layer1_attention_weight_max": 14.997824668884277,
      "activations/layer1_attention_weight_min": -15.282800674438477,
      "activations/layer20_attention_weight_max": 28.951335906982422,
      "activations/layer20_attention_weight_min": -26.130163192749023,
      "activations/layer21_attention_weight_max": 30.252519607543945,
      "activations/layer21_attention_weight_min": -25.042367935180664,
      "activations/layer22_attention_weight_max": 42.88905334472656,
      "activations/layer22_attention_weight_min": -32.28445053100586,
      "activations/layer23_attention_weight_max": 31.829490661621094,
      "activations/layer23_attention_weight_min": -26.24970245361328,
      "activations/layer2_attention_weight_max": 32.20906066894531,
      "activations/layer2_attention_weight_min": -33.276405334472656,
      "activations/layer3_attention_weight_max": 93.20747375488281,
      "activations/layer3_attention_weight_min": -96.13126373291016,
      "activations/layer4_attention_weight_max": 102.73441314697266,
      "activations/layer4_attention_weight_min": -105.44752502441406,
      "activations/layer5_attention_weight_max": 49.89014434814453,
      "activations/layer5_attention_weight_min": -61.67376708984375,
      "activations/layer6_attention_weight_max": 41.571533203125,
      "activations/layer6_attention_weight_min": -43.96289825439453,
      "activations/layer7_attention_weight_max": 85.89557647705078,
      "activations/layer7_attention_weight_min": -89.13390350341797,
      "activations/layer8_attention_weight_max": 40.8388557434082,
      "activations/layer8_attention_weight_min": -39.680458068847656,
      "activations/layer9_attention_weight_max": 29.680774688720703,
      "activations/layer9_attention_weight_min": -31.64023780822754,
      "epoch": 16.59,
      "learning_rate": 4.347689393939394e-05,
      "loss": 2.7603,
      "step": 285450
    },
    {
      "activations/layer0_attention_weight_max": 16.35325813293457,
      "activations/layer0_attention_weight_min": -12.49106216430664,
      "activations/layer10_attention_weight_max": 36.47282791137695,
      "activations/layer10_attention_weight_min": -36.285301208496094,
      "activations/layer11_attention_weight_max": 36.924591064453125,
      "activations/layer11_attention_weight_min": -38.21092224121094,
      "activations/layer12_attention_weight_max": 26.56959342956543,
      "activations/layer12_attention_weight_min": -25.902326583862305,
      "activations/layer13_attention_weight_max": 39.42394256591797,
      "activations/layer13_attention_weight_min": -35.78491973876953,
      "activations/layer14_attention_weight_max": 43.44321060180664,
      "activations/layer14_attention_weight_min": -36.62166976928711,
      "activations/layer15_attention_weight_max": 39.553497314453125,
      "activations/layer15_attention_weight_min": -33.873695373535156,
      "activations/layer16_attention_weight_max": 30.299055099487305,
      "activations/layer16_attention_weight_min": -28.575510025024414,
      "activations/layer17_attention_weight_max": 31.58112335205078,
      "activations/layer17_attention_weight_min": -25.616374969482422,
      "activations/layer18_attention_weight_max": 30.203678131103516,
      "activations/layer18_attention_weight_min": -24.920169830322266,
      "activations/layer19_attention_weight_max": 31.76405906677246,
      "activations/layer19_attention_weight_min": -30.456396102905273,
      "activations/layer1_attention_weight_max": 16.59156608581543,
      "activations/layer1_attention_weight_min": -14.822591781616211,
      "activations/layer20_attention_weight_max": 29.68877601623535,
      "activations/layer20_attention_weight_min": -27.855504989624023,
      "activations/layer21_attention_weight_max": 29.04880714416504,
      "activations/layer21_attention_weight_min": -25.062541961669922,
      "activations/layer22_attention_weight_max": 40.405067443847656,
      "activations/layer22_attention_weight_min": -31.889877319335938,
      "activations/layer23_attention_weight_max": 33.54439163208008,
      "activations/layer23_attention_weight_min": -25.792829513549805,
      "activations/layer2_attention_weight_max": 32.92422866821289,
      "activations/layer2_attention_weight_min": -33.22069549560547,
      "activations/layer3_attention_weight_max": 97.90753173828125,
      "activations/layer3_attention_weight_min": -97.05380249023438,
      "activations/layer4_attention_weight_max": 110.30885314941406,
      "activations/layer4_attention_weight_min": -109.3004379272461,
      "activations/layer5_attention_weight_max": 50.53173065185547,
      "activations/layer5_attention_weight_min": -60.519203186035156,
      "activations/layer6_attention_weight_max": 42.689247131347656,
      "activations/layer6_attention_weight_min": -44.0517692565918,
      "activations/layer7_attention_weight_max": 97.42044067382812,
      "activations/layer7_attention_weight_min": -94.19942474365234,
      "activations/layer8_attention_weight_max": 46.04220199584961,
      "activations/layer8_attention_weight_min": -43.96590042114258,
      "activations/layer9_attention_weight_max": 32.7056770324707,
      "activations/layer9_attention_weight_min": -35.36177444458008,
      "epoch": 16.59,
      "learning_rate": 4.345795454545454e-05,
      "loss": 2.745,
      "step": 285500
    },
    {
      "activations/layer0_attention_weight_max": 16.027584075927734,
      "activations/layer0_attention_weight_min": -12.460174560546875,
      "activations/layer10_attention_weight_max": 39.820556640625,
      "activations/layer10_attention_weight_min": -40.187255859375,
      "activations/layer11_attention_weight_max": 40.061153411865234,
      "activations/layer11_attention_weight_min": -42.15439224243164,
      "activations/layer12_attention_weight_max": 28.01117706298828,
      "activations/layer12_attention_weight_min": -29.990604400634766,
      "activations/layer13_attention_weight_max": 40.17232131958008,
      "activations/layer13_attention_weight_min": -39.67234420776367,
      "activations/layer14_attention_weight_max": 45.070533752441406,
      "activations/layer14_attention_weight_min": -46.19947052001953,
      "activations/layer15_attention_weight_max": 44.94087600708008,
      "activations/layer15_attention_weight_min": -43.084815979003906,
      "activations/layer16_attention_weight_max": 32.57724380493164,
      "activations/layer16_attention_weight_min": -28.593734741210938,
      "activations/layer17_attention_weight_max": 29.69642448425293,
      "activations/layer17_attention_weight_min": -27.649572372436523,
      "activations/layer18_attention_weight_max": 31.440677642822266,
      "activations/layer18_attention_weight_min": -25.16583251953125,
      "activations/layer19_attention_weight_max": 29.893375396728516,
      "activations/layer19_attention_weight_min": -29.453046798706055,
      "activations/layer1_attention_weight_max": 17.103452682495117,
      "activations/layer1_attention_weight_min": -15.314918518066406,
      "activations/layer20_attention_weight_max": 25.316871643066406,
      "activations/layer20_attention_weight_min": -24.75114631652832,
      "activations/layer21_attention_weight_max": 24.40453338623047,
      "activations/layer21_attention_weight_min": -23.417739868164062,
      "activations/layer22_attention_weight_max": 36.48194885253906,
      "activations/layer22_attention_weight_min": -29.471176147460938,
      "activations/layer23_attention_weight_max": 27.953954696655273,
      "activations/layer23_attention_weight_min": -24.50288200378418,
      "activations/layer2_attention_weight_max": 36.78038024902344,
      "activations/layer2_attention_weight_min": -34.86641311645508,
      "activations/layer3_attention_weight_max": 107.51569366455078,
      "activations/layer3_attention_weight_min": -101.36406707763672,
      "activations/layer4_attention_weight_max": 104.4332504272461,
      "activations/layer4_attention_weight_min": -108.6172103881836,
      "activations/layer5_attention_weight_max": 51.073020935058594,
      "activations/layer5_attention_weight_min": -60.14752960205078,
      "activations/layer6_attention_weight_max": 46.20481491088867,
      "activations/layer6_attention_weight_min": -47.07802200317383,
      "activations/layer7_attention_weight_max": 98.24898529052734,
      "activations/layer7_attention_weight_min": -95.96023559570312,
      "activations/layer8_attention_weight_max": 46.79166030883789,
      "activations/layer8_attention_weight_min": -48.25928497314453,
      "activations/layer9_attention_weight_max": 35.399017333984375,
      "activations/layer9_attention_weight_min": -38.07828903198242,
      "epoch": 16.59,
      "learning_rate": 4.343901515151514e-05,
      "loss": 2.7275,
      "step": 285550
    },
    {
      "activations/layer0_attention_weight_max": 16.016300201416016,
      "activations/layer0_attention_weight_min": -12.53113842010498,
      "activations/layer10_attention_weight_max": 31.437387466430664,
      "activations/layer10_attention_weight_min": -32.89118194580078,
      "activations/layer11_attention_weight_max": 30.388202667236328,
      "activations/layer11_attention_weight_min": -32.41173553466797,
      "activations/layer12_attention_weight_max": 26.752822875976562,
      "activations/layer12_attention_weight_min": -26.929975509643555,
      "activations/layer13_attention_weight_max": 39.692718505859375,
      "activations/layer13_attention_weight_min": -35.11266326904297,
      "activations/layer14_attention_weight_max": 45.84755325317383,
      "activations/layer14_attention_weight_min": -38.63618850708008,
      "activations/layer15_attention_weight_max": 41.266414642333984,
      "activations/layer15_attention_weight_min": -33.70698928833008,
      "activations/layer16_attention_weight_max": 30.79543113708496,
      "activations/layer16_attention_weight_min": -27.381391525268555,
      "activations/layer17_attention_weight_max": 30.534128189086914,
      "activations/layer17_attention_weight_min": -25.172842025756836,
      "activations/layer18_attention_weight_max": 31.032756805419922,
      "activations/layer18_attention_weight_min": -26.737390518188477,
      "activations/layer19_attention_weight_max": 35.08881759643555,
      "activations/layer19_attention_weight_min": -31.031625747680664,
      "activations/layer1_attention_weight_max": 20.32523536682129,
      "activations/layer1_attention_weight_min": -13.819714546203613,
      "activations/layer20_attention_weight_max": 28.843421936035156,
      "activations/layer20_attention_weight_min": -25.600563049316406,
      "activations/layer21_attention_weight_max": 28.29998779296875,
      "activations/layer21_attention_weight_min": -24.2086238861084,
      "activations/layer22_attention_weight_max": 44.90958786010742,
      "activations/layer22_attention_weight_min": -30.8299617767334,
      "activations/layer23_attention_weight_max": 34.74759292602539,
      "activations/layer23_attention_weight_min": -27.78030014038086,
      "activations/layer2_attention_weight_max": 31.82909393310547,
      "activations/layer2_attention_weight_min": -32.41301727294922,
      "activations/layer3_attention_weight_max": 93.93383026123047,
      "activations/layer3_attention_weight_min": -96.54505920410156,
      "activations/layer4_attention_weight_max": 102.11827850341797,
      "activations/layer4_attention_weight_min": -108.50372314453125,
      "activations/layer5_attention_weight_max": 48.808494567871094,
      "activations/layer5_attention_weight_min": -65.53529357910156,
      "activations/layer6_attention_weight_max": 40.89955520629883,
      "activations/layer6_attention_weight_min": -45.72276306152344,
      "activations/layer7_attention_weight_max": 85.67156982421875,
      "activations/layer7_attention_weight_min": -91.1111068725586,
      "activations/layer8_attention_weight_max": 38.75180435180664,
      "activations/layer8_attention_weight_min": -42.22909164428711,
      "activations/layer9_attention_weight_max": 31.598798751831055,
      "activations/layer9_attention_weight_min": -32.91359329223633,
      "epoch": 16.59,
      "learning_rate": 4.342007575757576e-05,
      "loss": 2.7322,
      "step": 285600
    },
    {
      "activations/layer0_attention_weight_max": 16.3865966796875,
      "activations/layer0_attention_weight_min": -12.359344482421875,
      "activations/layer10_attention_weight_max": 35.073299407958984,
      "activations/layer10_attention_weight_min": -32.71983337402344,
      "activations/layer11_attention_weight_max": 32.321754455566406,
      "activations/layer11_attention_weight_min": -33.448753356933594,
      "activations/layer12_attention_weight_max": 26.86038589477539,
      "activations/layer12_attention_weight_min": -26.892005920410156,
      "activations/layer13_attention_weight_max": 37.312713623046875,
      "activations/layer13_attention_weight_min": -36.074974060058594,
      "activations/layer14_attention_weight_max": 40.91749954223633,
      "activations/layer14_attention_weight_min": -40.862186431884766,
      "activations/layer15_attention_weight_max": 33.45899200439453,
      "activations/layer15_attention_weight_min": -37.281551361083984,
      "activations/layer16_attention_weight_max": 28.721818923950195,
      "activations/layer16_attention_weight_min": -26.618589401245117,
      "activations/layer17_attention_weight_max": 27.059926986694336,
      "activations/layer17_attention_weight_min": -26.58237075805664,
      "activations/layer18_attention_weight_max": 28.71226692199707,
      "activations/layer18_attention_weight_min": -26.67116355895996,
      "activations/layer19_attention_weight_max": 33.26356506347656,
      "activations/layer19_attention_weight_min": -32.80716323852539,
      "activations/layer1_attention_weight_max": 16.70549964904785,
      "activations/layer1_attention_weight_min": -15.05006217956543,
      "activations/layer20_attention_weight_max": 26.11370086669922,
      "activations/layer20_attention_weight_min": -25.634658813476562,
      "activations/layer21_attention_weight_max": 26.77018928527832,
      "activations/layer21_attention_weight_min": -24.629074096679688,
      "activations/layer22_attention_weight_max": 41.5552864074707,
      "activations/layer22_attention_weight_min": -30.33587646484375,
      "activations/layer23_attention_weight_max": 30.63421630859375,
      "activations/layer23_attention_weight_min": -24.794103622436523,
      "activations/layer2_attention_weight_max": 33.661338806152344,
      "activations/layer2_attention_weight_min": -31.834823608398438,
      "activations/layer3_attention_weight_max": 100.97460174560547,
      "activations/layer3_attention_weight_min": -96.38626098632812,
      "activations/layer4_attention_weight_max": 105.33270263671875,
      "activations/layer4_attention_weight_min": -106.90654754638672,
      "activations/layer5_attention_weight_max": 49.9205436706543,
      "activations/layer5_attention_weight_min": -61.297882080078125,
      "activations/layer6_attention_weight_max": 42.16875076293945,
      "activations/layer6_attention_weight_min": -44.873905181884766,
      "activations/layer7_attention_weight_max": 89.47589874267578,
      "activations/layer7_attention_weight_min": -93.43736267089844,
      "activations/layer8_attention_weight_max": 41.292213439941406,
      "activations/layer8_attention_weight_min": -41.531089782714844,
      "activations/layer9_attention_weight_max": 31.084171295166016,
      "activations/layer9_attention_weight_min": -33.65852737426758,
      "epoch": 16.6,
      "learning_rate": 4.340113636363636e-05,
      "loss": 2.7609,
      "step": 285650
    },
    {
      "activations/layer0_attention_weight_max": 15.303696632385254,
      "activations/layer0_attention_weight_min": -12.819053649902344,
      "activations/layer10_attention_weight_max": 33.89162826538086,
      "activations/layer10_attention_weight_min": -37.45833206176758,
      "activations/layer11_attention_weight_max": 35.7719612121582,
      "activations/layer11_attention_weight_min": -36.50679016113281,
      "activations/layer12_attention_weight_max": 27.486698150634766,
      "activations/layer12_attention_weight_min": -25.682043075561523,
      "activations/layer13_attention_weight_max": 38.967071533203125,
      "activations/layer13_attention_weight_min": -35.08015060424805,
      "activations/layer14_attention_weight_max": 45.28712844848633,
      "activations/layer14_attention_weight_min": -35.922420501708984,
      "activations/layer15_attention_weight_max": 39.50370788574219,
      "activations/layer15_attention_weight_min": -35.20230484008789,
      "activations/layer16_attention_weight_max": 29.68781852722168,
      "activations/layer16_attention_weight_min": -29.030860900878906,
      "activations/layer17_attention_weight_max": 31.22524070739746,
      "activations/layer17_attention_weight_min": -27.4041690826416,
      "activations/layer18_attention_weight_max": 36.231544494628906,
      "activations/layer18_attention_weight_min": -25.876928329467773,
      "activations/layer19_attention_weight_max": 35.267791748046875,
      "activations/layer19_attention_weight_min": -33.7382698059082,
      "activations/layer1_attention_weight_max": 16.140029907226562,
      "activations/layer1_attention_weight_min": -16.62283706665039,
      "activations/layer20_attention_weight_max": 29.944799423217773,
      "activations/layer20_attention_weight_min": -26.70121955871582,
      "activations/layer21_attention_weight_max": 30.98012924194336,
      "activations/layer21_attention_weight_min": -27.334726333618164,
      "activations/layer22_attention_weight_max": 45.288761138916016,
      "activations/layer22_attention_weight_min": -33.305206298828125,
      "activations/layer23_attention_weight_max": 33.61835861206055,
      "activations/layer23_attention_weight_min": -26.35344696044922,
      "activations/layer2_attention_weight_max": 32.94469451904297,
      "activations/layer2_attention_weight_min": -34.42457962036133,
      "activations/layer3_attention_weight_max": 95.52288818359375,
      "activations/layer3_attention_weight_min": -94.60466003417969,
      "activations/layer4_attention_weight_max": 105.0979995727539,
      "activations/layer4_attention_weight_min": -107.63612365722656,
      "activations/layer5_attention_weight_max": 52.63302230834961,
      "activations/layer5_attention_weight_min": -60.87261199951172,
      "activations/layer6_attention_weight_max": 41.81077194213867,
      "activations/layer6_attention_weight_min": -44.053279876708984,
      "activations/layer7_attention_weight_max": 94.7221908569336,
      "activations/layer7_attention_weight_min": -100.33544158935547,
      "activations/layer8_attention_weight_max": 40.56009292602539,
      "activations/layer8_attention_weight_min": -42.09785079956055,
      "activations/layer9_attention_weight_max": 31.89632225036621,
      "activations/layer9_attention_weight_min": -34.41447448730469,
      "epoch": 16.6,
      "learning_rate": 4.338219696969697e-05,
      "loss": 2.7489,
      "step": 285700
    },
    {
      "activations/layer0_attention_weight_max": 16.02147102355957,
      "activations/layer0_attention_weight_min": -12.596822738647461,
      "activations/layer10_attention_weight_max": 33.22076416015625,
      "activations/layer10_attention_weight_min": -32.73243713378906,
      "activations/layer11_attention_weight_max": 32.731353759765625,
      "activations/layer11_attention_weight_min": -32.930999755859375,
      "activations/layer12_attention_weight_max": 27.458759307861328,
      "activations/layer12_attention_weight_min": -25.822738647460938,
      "activations/layer13_attention_weight_max": 40.560638427734375,
      "activations/layer13_attention_weight_min": -33.972633361816406,
      "activations/layer14_attention_weight_max": 45.16118621826172,
      "activations/layer14_attention_weight_min": -36.84525680541992,
      "activations/layer15_attention_weight_max": 41.43022155761719,
      "activations/layer15_attention_weight_min": -35.36701583862305,
      "activations/layer16_attention_weight_max": 31.069623947143555,
      "activations/layer16_attention_weight_min": -30.422618865966797,
      "activations/layer17_attention_weight_max": 31.340518951416016,
      "activations/layer17_attention_weight_min": -25.868928909301758,
      "activations/layer18_attention_weight_max": 36.24764633178711,
      "activations/layer18_attention_weight_min": -27.565570831298828,
      "activations/layer19_attention_weight_max": 37.25397872924805,
      "activations/layer19_attention_weight_min": -32.76199722290039,
      "activations/layer1_attention_weight_max": 15.71437931060791,
      "activations/layer1_attention_weight_min": -15.185770034790039,
      "activations/layer20_attention_weight_max": 31.095317840576172,
      "activations/layer20_attention_weight_min": -25.187965393066406,
      "activations/layer21_attention_weight_max": 30.96173667907715,
      "activations/layer21_attention_weight_min": -24.333086013793945,
      "activations/layer22_attention_weight_max": 51.610172271728516,
      "activations/layer22_attention_weight_min": -33.47609329223633,
      "activations/layer23_attention_weight_max": 37.0521240234375,
      "activations/layer23_attention_weight_min": -29.576683044433594,
      "activations/layer2_attention_weight_max": 33.35865020751953,
      "activations/layer2_attention_weight_min": -34.5606689453125,
      "activations/layer3_attention_weight_max": 96.25018310546875,
      "activations/layer3_attention_weight_min": -101.76664733886719,
      "activations/layer4_attention_weight_max": 107.2847900390625,
      "activations/layer4_attention_weight_min": -107.13018035888672,
      "activations/layer5_attention_weight_max": 51.551124572753906,
      "activations/layer5_attention_weight_min": -66.10848236083984,
      "activations/layer6_attention_weight_max": 42.93761444091797,
      "activations/layer6_attention_weight_min": -46.52425003051758,
      "activations/layer7_attention_weight_max": 90.45686340332031,
      "activations/layer7_attention_weight_min": -94.78437042236328,
      "activations/layer8_attention_weight_max": 41.20866012573242,
      "activations/layer8_attention_weight_min": -41.1601448059082,
      "activations/layer9_attention_weight_max": 30.935178756713867,
      "activations/layer9_attention_weight_min": -33.67298126220703,
      "epoch": 16.6,
      "learning_rate": 4.336325757575757e-05,
      "loss": 2.7432,
      "step": 285750
    },
    {
      "activations/layer0_attention_weight_max": 15.589442253112793,
      "activations/layer0_attention_weight_min": -12.365194320678711,
      "activations/layer10_attention_weight_max": 32.352115631103516,
      "activations/layer10_attention_weight_min": -33.49055862426758,
      "activations/layer11_attention_weight_max": 33.80241775512695,
      "activations/layer11_attention_weight_min": -35.354339599609375,
      "activations/layer12_attention_weight_max": 27.177656173706055,
      "activations/layer12_attention_weight_min": -26.903120040893555,
      "activations/layer13_attention_weight_max": 41.290855407714844,
      "activations/layer13_attention_weight_min": -35.74748992919922,
      "activations/layer14_attention_weight_max": 43.77958297729492,
      "activations/layer14_attention_weight_min": -39.60795211791992,
      "activations/layer15_attention_weight_max": 38.79343795776367,
      "activations/layer15_attention_weight_min": -36.01446533203125,
      "activations/layer16_attention_weight_max": 31.04041290283203,
      "activations/layer16_attention_weight_min": -31.30230712890625,
      "activations/layer17_attention_weight_max": 30.62228012084961,
      "activations/layer17_attention_weight_min": -26.395706176757812,
      "activations/layer18_attention_weight_max": 31.205293655395508,
      "activations/layer18_attention_weight_min": -26.729000091552734,
      "activations/layer19_attention_weight_max": 33.842159271240234,
      "activations/layer19_attention_weight_min": -31.617307662963867,
      "activations/layer1_attention_weight_max": 16.796123504638672,
      "activations/layer1_attention_weight_min": -14.607645988464355,
      "activations/layer20_attention_weight_max": 29.277915954589844,
      "activations/layer20_attention_weight_min": -25.853330612182617,
      "activations/layer21_attention_weight_max": 29.01742935180664,
      "activations/layer21_attention_weight_min": -25.09917449951172,
      "activations/layer22_attention_weight_max": 43.195987701416016,
      "activations/layer22_attention_weight_min": -31.144365310668945,
      "activations/layer23_attention_weight_max": 32.4034423828125,
      "activations/layer23_attention_weight_min": -27.199499130249023,
      "activations/layer2_attention_weight_max": 33.586875915527344,
      "activations/layer2_attention_weight_min": -32.1154670715332,
      "activations/layer3_attention_weight_max": 101.6910629272461,
      "activations/layer3_attention_weight_min": -99.1859359741211,
      "activations/layer4_attention_weight_max": 111.769775390625,
      "activations/layer4_attention_weight_min": -110.55854797363281,
      "activations/layer5_attention_weight_max": 52.23567199707031,
      "activations/layer5_attention_weight_min": -63.37113571166992,
      "activations/layer6_attention_weight_max": 47.31748580932617,
      "activations/layer6_attention_weight_min": -48.136173248291016,
      "activations/layer7_attention_weight_max": 95.29356384277344,
      "activations/layer7_attention_weight_min": -92.52689361572266,
      "activations/layer8_attention_weight_max": 42.44841384887695,
      "activations/layer8_attention_weight_min": -41.89805603027344,
      "activations/layer9_attention_weight_max": 31.485265731811523,
      "activations/layer9_attention_weight_min": -34.548091888427734,
      "epoch": 16.61,
      "learning_rate": 4.334431818181818e-05,
      "loss": 2.7386,
      "step": 285800
    },
    {
      "activations/layer0_attention_weight_max": 15.839481353759766,
      "activations/layer0_attention_weight_min": -12.52962589263916,
      "activations/layer10_attention_weight_max": 34.932003021240234,
      "activations/layer10_attention_weight_min": -38.20884323120117,
      "activations/layer11_attention_weight_max": 35.7474479675293,
      "activations/layer11_attention_weight_min": -38.066226959228516,
      "activations/layer12_attention_weight_max": 29.8205509185791,
      "activations/layer12_attention_weight_min": -28.829774856567383,
      "activations/layer13_attention_weight_max": 43.65971755981445,
      "activations/layer13_attention_weight_min": -38.632747650146484,
      "activations/layer14_attention_weight_max": 53.285953521728516,
      "activations/layer14_attention_weight_min": -42.81765365600586,
      "activations/layer15_attention_weight_max": 41.808048248291016,
      "activations/layer15_attention_weight_min": -39.143341064453125,
      "activations/layer16_attention_weight_max": 33.803096771240234,
      "activations/layer16_attention_weight_min": -30.25189971923828,
      "activations/layer17_attention_weight_max": 31.602014541625977,
      "activations/layer17_attention_weight_min": -28.911434173583984,
      "activations/layer18_attention_weight_max": 33.03109359741211,
      "activations/layer18_attention_weight_min": -28.143901824951172,
      "activations/layer19_attention_weight_max": 34.079429626464844,
      "activations/layer19_attention_weight_min": -35.3946418762207,
      "activations/layer1_attention_weight_max": 16.705480575561523,
      "activations/layer1_attention_weight_min": -14.814618110656738,
      "activations/layer20_attention_weight_max": 30.157236099243164,
      "activations/layer20_attention_weight_min": -26.76259994506836,
      "activations/layer21_attention_weight_max": 27.52480125427246,
      "activations/layer21_attention_weight_min": -26.30154037475586,
      "activations/layer22_attention_weight_max": 43.28700256347656,
      "activations/layer22_attention_weight_min": -34.459068298339844,
      "activations/layer23_attention_weight_max": 30.04757308959961,
      "activations/layer23_attention_weight_min": -27.283079147338867,
      "activations/layer2_attention_weight_max": 32.153785705566406,
      "activations/layer2_attention_weight_min": -34.28063201904297,
      "activations/layer3_attention_weight_max": 94.8631591796875,
      "activations/layer3_attention_weight_min": -103.84037017822266,
      "activations/layer4_attention_weight_max": 105.63032531738281,
      "activations/layer4_attention_weight_min": -109.02709197998047,
      "activations/layer5_attention_weight_max": 52.95466232299805,
      "activations/layer5_attention_weight_min": -60.00960922241211,
      "activations/layer6_attention_weight_max": 43.90401840209961,
      "activations/layer6_attention_weight_min": -49.444786071777344,
      "activations/layer7_attention_weight_max": 98.48296356201172,
      "activations/layer7_attention_weight_min": -102.95048522949219,
      "activations/layer8_attention_weight_max": 42.25320816040039,
      "activations/layer8_attention_weight_min": -46.22645568847656,
      "activations/layer9_attention_weight_max": 33.75751876831055,
      "activations/layer9_attention_weight_min": -36.87970733642578,
      "epoch": 16.61,
      "learning_rate": 4.3325378787878785e-05,
      "loss": 2.7591,
      "step": 285850
    },
    {
      "activations/layer0_attention_weight_max": 15.990604400634766,
      "activations/layer0_attention_weight_min": -12.620906829833984,
      "activations/layer10_attention_weight_max": 34.970008850097656,
      "activations/layer10_attention_weight_min": -34.908416748046875,
      "activations/layer11_attention_weight_max": 34.687744140625,
      "activations/layer11_attention_weight_min": -35.09187316894531,
      "activations/layer12_attention_weight_max": 26.66492462158203,
      "activations/layer12_attention_weight_min": -28.23456573486328,
      "activations/layer13_attention_weight_max": 37.834407806396484,
      "activations/layer13_attention_weight_min": -36.97176742553711,
      "activations/layer14_attention_weight_max": 40.163917541503906,
      "activations/layer14_attention_weight_min": -37.49838638305664,
      "activations/layer15_attention_weight_max": 39.79110336303711,
      "activations/layer15_attention_weight_min": -33.92212677001953,
      "activations/layer16_attention_weight_max": 29.299272537231445,
      "activations/layer16_attention_weight_min": -27.22075080871582,
      "activations/layer17_attention_weight_max": 29.38057518005371,
      "activations/layer17_attention_weight_min": -25.864707946777344,
      "activations/layer18_attention_weight_max": 32.860626220703125,
      "activations/layer18_attention_weight_min": -25.21158790588379,
      "activations/layer19_attention_weight_max": 36.565372467041016,
      "activations/layer19_attention_weight_min": -29.922006607055664,
      "activations/layer1_attention_weight_max": 17.163326263427734,
      "activations/layer1_attention_weight_min": -15.704429626464844,
      "activations/layer20_attention_weight_max": 31.9439697265625,
      "activations/layer20_attention_weight_min": -25.050914764404297,
      "activations/layer21_attention_weight_max": 31.865690231323242,
      "activations/layer21_attention_weight_min": -25.785978317260742,
      "activations/layer22_attention_weight_max": 40.9239501953125,
      "activations/layer22_attention_weight_min": -31.90019989013672,
      "activations/layer23_attention_weight_max": 30.922977447509766,
      "activations/layer23_attention_weight_min": -25.157211303710938,
      "activations/layer2_attention_weight_max": 34.146759033203125,
      "activations/layer2_attention_weight_min": -34.77751541137695,
      "activations/layer3_attention_weight_max": 97.41925048828125,
      "activations/layer3_attention_weight_min": -101.60143280029297,
      "activations/layer4_attention_weight_max": 110.07466888427734,
      "activations/layer4_attention_weight_min": -110.4461898803711,
      "activations/layer5_attention_weight_max": 53.316062927246094,
      "activations/layer5_attention_weight_min": -59.814998626708984,
      "activations/layer6_attention_weight_max": 44.28927993774414,
      "activations/layer6_attention_weight_min": -46.071842193603516,
      "activations/layer7_attention_weight_max": 96.67711639404297,
      "activations/layer7_attention_weight_min": -98.21419525146484,
      "activations/layer8_attention_weight_max": 41.35356521606445,
      "activations/layer8_attention_weight_min": -43.86296081542969,
      "activations/layer9_attention_weight_max": 33.26815414428711,
      "activations/layer9_attention_weight_min": -35.11738204956055,
      "epoch": 16.61,
      "learning_rate": 4.330643939393939e-05,
      "loss": 2.7513,
      "step": 285900
    },
    {
      "activations/layer0_attention_weight_max": 15.463839530944824,
      "activations/layer0_attention_weight_min": -13.107870101928711,
      "activations/layer10_attention_weight_max": 31.662466049194336,
      "activations/layer10_attention_weight_min": -31.596172332763672,
      "activations/layer11_attention_weight_max": 32.21562576293945,
      "activations/layer11_attention_weight_min": -32.424903869628906,
      "activations/layer12_attention_weight_max": 26.85883903503418,
      "activations/layer12_attention_weight_min": -26.045127868652344,
      "activations/layer13_attention_weight_max": 38.34783935546875,
      "activations/layer13_attention_weight_min": -35.456817626953125,
      "activations/layer14_attention_weight_max": 45.82429885864258,
      "activations/layer14_attention_weight_min": -38.666053771972656,
      "activations/layer15_attention_weight_max": 43.50424575805664,
      "activations/layer15_attention_weight_min": -36.4134407043457,
      "activations/layer16_attention_weight_max": 34.427276611328125,
      "activations/layer16_attention_weight_min": -30.210248947143555,
      "activations/layer17_attention_weight_max": 31.741193771362305,
      "activations/layer17_attention_weight_min": -27.326013565063477,
      "activations/layer18_attention_weight_max": 32.70401382446289,
      "activations/layer18_attention_weight_min": -25.75961685180664,
      "activations/layer19_attention_weight_max": 38.14413070678711,
      "activations/layer19_attention_weight_min": -32.97837829589844,
      "activations/layer1_attention_weight_max": 16.171348571777344,
      "activations/layer1_attention_weight_min": -14.241862297058105,
      "activations/layer20_attention_weight_max": 32.215023040771484,
      "activations/layer20_attention_weight_min": -25.876752853393555,
      "activations/layer21_attention_weight_max": 30.749242782592773,
      "activations/layer21_attention_weight_min": -26.640853881835938,
      "activations/layer22_attention_weight_max": 44.83975601196289,
      "activations/layer22_attention_weight_min": -33.27006530761719,
      "activations/layer23_attention_weight_max": 36.42603302001953,
      "activations/layer23_attention_weight_min": -27.290878295898438,
      "activations/layer2_attention_weight_max": 31.42728042602539,
      "activations/layer2_attention_weight_min": -32.55236053466797,
      "activations/layer3_attention_weight_max": 89.45503997802734,
      "activations/layer3_attention_weight_min": -93.55364990234375,
      "activations/layer4_attention_weight_max": 100.71267700195312,
      "activations/layer4_attention_weight_min": -101.0796127319336,
      "activations/layer5_attention_weight_max": 49.10976791381836,
      "activations/layer5_attention_weight_min": -60.9525146484375,
      "activations/layer6_attention_weight_max": 41.85775375366211,
      "activations/layer6_attention_weight_min": -44.96638488769531,
      "activations/layer7_attention_weight_max": 92.427001953125,
      "activations/layer7_attention_weight_min": -89.14185333251953,
      "activations/layer8_attention_weight_max": 39.31526184082031,
      "activations/layer8_attention_weight_min": -41.13149642944336,
      "activations/layer9_attention_weight_max": 32.16848373413086,
      "activations/layer9_attention_weight_min": -34.11500549316406,
      "epoch": 16.62,
      "learning_rate": 4.32875e-05,
      "loss": 2.7484,
      "step": 285950
    },
    {
      "activations/layer0_attention_weight_max": 17.009767532348633,
      "activations/layer0_attention_weight_min": -12.604544639587402,
      "activations/layer10_attention_weight_max": 37.43053436279297,
      "activations/layer10_attention_weight_min": -37.287681579589844,
      "activations/layer11_attention_weight_max": 38.92189407348633,
      "activations/layer11_attention_weight_min": -36.18183135986328,
      "activations/layer12_attention_weight_max": 27.927303314208984,
      "activations/layer12_attention_weight_min": -26.578426361083984,
      "activations/layer13_attention_weight_max": 41.23972702026367,
      "activations/layer13_attention_weight_min": -38.76066589355469,
      "activations/layer14_attention_weight_max": 47.77967071533203,
      "activations/layer14_attention_weight_min": -44.3792724609375,
      "activations/layer15_attention_weight_max": 54.35262680053711,
      "activations/layer15_attention_weight_min": -43.11149978637695,
      "activations/layer16_attention_weight_max": 35.744815826416016,
      "activations/layer16_attention_weight_min": -31.576221466064453,
      "activations/layer17_attention_weight_max": 33.96007537841797,
      "activations/layer17_attention_weight_min": -28.693628311157227,
      "activations/layer18_attention_weight_max": 34.93400955200195,
      "activations/layer18_attention_weight_min": -29.561418533325195,
      "activations/layer19_attention_weight_max": 43.078636169433594,
      "activations/layer19_attention_weight_min": -35.96147155761719,
      "activations/layer1_attention_weight_max": 16.215435028076172,
      "activations/layer1_attention_weight_min": -14.283797264099121,
      "activations/layer20_attention_weight_max": 33.61521530151367,
      "activations/layer20_attention_weight_min": -27.425188064575195,
      "activations/layer21_attention_weight_max": 29.676815032958984,
      "activations/layer21_attention_weight_min": -25.35293960571289,
      "activations/layer22_attention_weight_max": 49.569236755371094,
      "activations/layer22_attention_weight_min": -34.0916633605957,
      "activations/layer23_attention_weight_max": 33.99951934814453,
      "activations/layer23_attention_weight_min": -24.798330307006836,
      "activations/layer2_attention_weight_max": 35.80896759033203,
      "activations/layer2_attention_weight_min": -33.25496292114258,
      "activations/layer3_attention_weight_max": 97.28569793701172,
      "activations/layer3_attention_weight_min": -96.75868225097656,
      "activations/layer4_attention_weight_max": 109.1265640258789,
      "activations/layer4_attention_weight_min": -107.78678131103516,
      "activations/layer5_attention_weight_max": 52.413055419921875,
      "activations/layer5_attention_weight_min": -60.827938079833984,
      "activations/layer6_attention_weight_max": 40.881011962890625,
      "activations/layer6_attention_weight_min": -44.093807220458984,
      "activations/layer7_attention_weight_max": 101.1208267211914,
      "activations/layer7_attention_weight_min": -92.5286865234375,
      "activations/layer8_attention_weight_max": 45.46906280517578,
      "activations/layer8_attention_weight_min": -43.27811050415039,
      "activations/layer9_attention_weight_max": 32.274505615234375,
      "activations/layer9_attention_weight_min": -37.41374969482422,
      "epoch": 16.62,
      "learning_rate": 4.32685606060606e-05,
      "loss": 2.745,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.549,
      "eval_samples_per_second": 502.28,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.549,
      "eval_openwebtext_samples_per_second": 502.28,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9686,
      "eval_wikitext_samples_per_second": 231.635,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_lambada_loss": 2.478515625,
      "eval_lambada_ppl": 11.923552252478798,
      "eval_lambada_runtime": 9.6026,
      "eval_lambada_samples_per_second": 507.049,
      "step": 286000
    },
    {
      "activations/layer0_attention_weight_max": 14.986205101013184,
      "activations/layer0_attention_weight_min": -12.262744903564453,
      "activations/layer10_attention_weight_max": 32.926231384277344,
      "activations/layer10_attention_weight_min": -34.01283645629883,
      "activations/layer11_attention_weight_max": 32.57402038574219,
      "activations/layer11_attention_weight_min": -34.311614990234375,
      "activations/layer12_attention_weight_max": 26.431415557861328,
      "activations/layer12_attention_weight_min": -28.593341827392578,
      "activations/layer13_attention_weight_max": 38.81056213378906,
      "activations/layer13_attention_weight_min": -39.817176818847656,
      "activations/layer14_attention_weight_max": 41.86027526855469,
      "activations/layer14_attention_weight_min": -41.82396697998047,
      "activations/layer15_attention_weight_max": 36.808837890625,
      "activations/layer15_attention_weight_min": -39.29866409301758,
      "activations/layer16_attention_weight_max": 29.807973861694336,
      "activations/layer16_attention_weight_min": -30.606889724731445,
      "activations/layer17_attention_weight_max": 29.36798095703125,
      "activations/layer17_attention_weight_min": -27.705381393432617,
      "activations/layer18_attention_weight_max": 30.247703552246094,
      "activations/layer18_attention_weight_min": -30.0904541015625,
      "activations/layer19_attention_weight_max": 35.90348434448242,
      "activations/layer19_attention_weight_min": -36.709712982177734,
      "activations/layer1_attention_weight_max": 16.302753448486328,
      "activations/layer1_attention_weight_min": -14.949938774108887,
      "activations/layer20_attention_weight_max": 31.388254165649414,
      "activations/layer20_attention_weight_min": -26.38198471069336,
      "activations/layer21_attention_weight_max": 29.393569946289062,
      "activations/layer21_attention_weight_min": -26.138011932373047,
      "activations/layer22_attention_weight_max": 45.23836135864258,
      "activations/layer22_attention_weight_min": -32.635528564453125,
      "activations/layer23_attention_weight_max": 34.974647521972656,
      "activations/layer23_attention_weight_min": -26.27688980102539,
      "activations/layer2_attention_weight_max": 34.04413604736328,
      "activations/layer2_attention_weight_min": -31.4262752532959,
      "activations/layer3_attention_weight_max": 97.641845703125,
      "activations/layer3_attention_weight_min": -96.69454193115234,
      "activations/layer4_attention_weight_max": 108.74629974365234,
      "activations/layer4_attention_weight_min": -109.3941421508789,
      "activations/layer5_attention_weight_max": 49.396602630615234,
      "activations/layer5_attention_weight_min": -58.957130432128906,
      "activations/layer6_attention_weight_max": 44.274314880371094,
      "activations/layer6_attention_weight_min": -44.104793548583984,
      "activations/layer7_attention_weight_max": 91.43474578857422,
      "activations/layer7_attention_weight_min": -90.76502227783203,
      "activations/layer8_attention_weight_max": 41.94020080566406,
      "activations/layer8_attention_weight_min": -42.00346374511719,
      "activations/layer9_attention_weight_max": 33.23849105834961,
      "activations/layer9_attention_weight_min": -34.988914489746094,
      "epoch": 16.62,
      "learning_rate": 4.3249621212121205e-05,
      "loss": 2.7437,
      "step": 286050
    },
    {
      "activations/layer0_attention_weight_max": 15.89767074584961,
      "activations/layer0_attention_weight_min": -12.61164665222168,
      "activations/layer10_attention_weight_max": 33.90625762939453,
      "activations/layer10_attention_weight_min": -33.11184310913086,
      "activations/layer11_attention_weight_max": 33.61943054199219,
      "activations/layer11_attention_weight_min": -32.81623077392578,
      "activations/layer12_attention_weight_max": 28.597169876098633,
      "activations/layer12_attention_weight_min": -26.425634384155273,
      "activations/layer13_attention_weight_max": 40.78651809692383,
      "activations/layer13_attention_weight_min": -35.44101333618164,
      "activations/layer14_attention_weight_max": 46.093162536621094,
      "activations/layer14_attention_weight_min": -37.68940734863281,
      "activations/layer15_attention_weight_max": 37.93703842163086,
      "activations/layer15_attention_weight_min": -34.08894729614258,
      "activations/layer16_attention_weight_max": 28.10822868347168,
      "activations/layer16_attention_weight_min": -30.174617767333984,
      "activations/layer17_attention_weight_max": 32.212432861328125,
      "activations/layer17_attention_weight_min": -26.21366310119629,
      "activations/layer18_attention_weight_max": 34.89069366455078,
      "activations/layer18_attention_weight_min": -24.813589096069336,
      "activations/layer19_attention_weight_max": 35.13936233520508,
      "activations/layer19_attention_weight_min": -31.975828170776367,
      "activations/layer1_attention_weight_max": 15.73596477508545,
      "activations/layer1_attention_weight_min": -15.109453201293945,
      "activations/layer20_attention_weight_max": 31.14681625366211,
      "activations/layer20_attention_weight_min": -27.457805633544922,
      "activations/layer21_attention_weight_max": 32.06226348876953,
      "activations/layer21_attention_weight_min": -25.826467514038086,
      "activations/layer22_attention_weight_max": 44.97525405883789,
      "activations/layer22_attention_weight_min": -33.60359191894531,
      "activations/layer23_attention_weight_max": 35.06267547607422,
      "activations/layer23_attention_weight_min": -26.182571411132812,
      "activations/layer2_attention_weight_max": 31.75128936767578,
      "activations/layer2_attention_weight_min": -31.98774528503418,
      "activations/layer3_attention_weight_max": 96.30400848388672,
      "activations/layer3_attention_weight_min": -95.50198364257812,
      "activations/layer4_attention_weight_max": 107.72676849365234,
      "activations/layer4_attention_weight_min": -106.54902648925781,
      "activations/layer5_attention_weight_max": 50.46900939941406,
      "activations/layer5_attention_weight_min": -62.16579818725586,
      "activations/layer6_attention_weight_max": 44.74258041381836,
      "activations/layer6_attention_weight_min": -44.92221450805664,
      "activations/layer7_attention_weight_max": 88.7551040649414,
      "activations/layer7_attention_weight_min": -100.20519256591797,
      "activations/layer8_attention_weight_max": 40.13444900512695,
      "activations/layer8_attention_weight_min": -41.70097732543945,
      "activations/layer9_attention_weight_max": 32.663726806640625,
      "activations/layer9_attention_weight_min": -34.74075698852539,
      "epoch": 16.62,
      "learning_rate": 4.323068181818181e-05,
      "loss": 2.7495,
      "step": 286100
    },
    {
      "activations/layer0_attention_weight_max": 16.238258361816406,
      "activations/layer0_attention_weight_min": -13.107794761657715,
      "activations/layer10_attention_weight_max": 33.06634521484375,
      "activations/layer10_attention_weight_min": -35.19646453857422,
      "activations/layer11_attention_weight_max": 34.834869384765625,
      "activations/layer11_attention_weight_min": -35.3155403137207,
      "activations/layer12_attention_weight_max": 25.937576293945312,
      "activations/layer12_attention_weight_min": -26.337976455688477,
      "activations/layer13_attention_weight_max": 37.902671813964844,
      "activations/layer13_attention_weight_min": -34.28243637084961,
      "activations/layer14_attention_weight_max": 40.43757247924805,
      "activations/layer14_attention_weight_min": -35.76755142211914,
      "activations/layer15_attention_weight_max": 40.98855972290039,
      "activations/layer15_attention_weight_min": -34.478267669677734,
      "activations/layer16_attention_weight_max": 28.38239288330078,
      "activations/layer16_attention_weight_min": -27.139020919799805,
      "activations/layer17_attention_weight_max": 29.10349464416504,
      "activations/layer17_attention_weight_min": -26.610647201538086,
      "activations/layer18_attention_weight_max": 31.3298282623291,
      "activations/layer18_attention_weight_min": -27.240983963012695,
      "activations/layer19_attention_weight_max": 31.48760986328125,
      "activations/layer19_attention_weight_min": -32.71644592285156,
      "activations/layer1_attention_weight_max": 16.2180118560791,
      "activations/layer1_attention_weight_min": -14.611167907714844,
      "activations/layer20_attention_weight_max": 27.117801666259766,
      "activations/layer20_attention_weight_min": -25.518810272216797,
      "activations/layer21_attention_weight_max": 28.003108978271484,
      "activations/layer21_attention_weight_min": -25.126720428466797,
      "activations/layer22_attention_weight_max": 43.44633865356445,
      "activations/layer22_attention_weight_min": -31.957252502441406,
      "activations/layer23_attention_weight_max": 31.353851318359375,
      "activations/layer23_attention_weight_min": -25.86663055419922,
      "activations/layer2_attention_weight_max": 32.5726318359375,
      "activations/layer2_attention_weight_min": -32.47654342651367,
      "activations/layer3_attention_weight_max": 97.09329223632812,
      "activations/layer3_attention_weight_min": -97.27709197998047,
      "activations/layer4_attention_weight_max": 105.83216857910156,
      "activations/layer4_attention_weight_min": -110.33281707763672,
      "activations/layer5_attention_weight_max": 50.4401969909668,
      "activations/layer5_attention_weight_min": -65.93363952636719,
      "activations/layer6_attention_weight_max": 41.902252197265625,
      "activations/layer6_attention_weight_min": -46.79671859741211,
      "activations/layer7_attention_weight_max": 88.2664566040039,
      "activations/layer7_attention_weight_min": -92.68640899658203,
      "activations/layer8_attention_weight_max": 42.71759033203125,
      "activations/layer8_attention_weight_min": -45.255271911621094,
      "activations/layer9_attention_weight_max": 31.46710205078125,
      "activations/layer9_attention_weight_min": -35.54008102416992,
      "epoch": 16.63,
      "learning_rate": 4.321174242424242e-05,
      "loss": 2.7433,
      "step": 286150
    },
    {
      "activations/layer0_attention_weight_max": 16.154945373535156,
      "activations/layer0_attention_weight_min": -12.084433555603027,
      "activations/layer10_attention_weight_max": 32.75787353515625,
      "activations/layer10_attention_weight_min": -33.78980255126953,
      "activations/layer11_attention_weight_max": 34.58539581298828,
      "activations/layer11_attention_weight_min": -33.63438415527344,
      "activations/layer12_attention_weight_max": 28.410884857177734,
      "activations/layer12_attention_weight_min": -27.991186141967773,
      "activations/layer13_attention_weight_max": 42.446346282958984,
      "activations/layer13_attention_weight_min": -39.37671661376953,
      "activations/layer14_attention_weight_max": 50.348777770996094,
      "activations/layer14_attention_weight_min": -41.13473129272461,
      "activations/layer15_attention_weight_max": 43.64542007446289,
      "activations/layer15_attention_weight_min": -36.585044860839844,
      "activations/layer16_attention_weight_max": 31.973058700561523,
      "activations/layer16_attention_weight_min": -27.484968185424805,
      "activations/layer17_attention_weight_max": 32.76455307006836,
      "activations/layer17_attention_weight_min": -26.476911544799805,
      "activations/layer18_attention_weight_max": 34.4426155090332,
      "activations/layer18_attention_weight_min": -27.205429077148438,
      "activations/layer19_attention_weight_max": 36.67925262451172,
      "activations/layer19_attention_weight_min": -32.72809600830078,
      "activations/layer1_attention_weight_max": 15.505106925964355,
      "activations/layer1_attention_weight_min": -13.504332542419434,
      "activations/layer20_attention_weight_max": 31.599517822265625,
      "activations/layer20_attention_weight_min": -25.34027099609375,
      "activations/layer21_attention_weight_max": 31.339157104492188,
      "activations/layer21_attention_weight_min": -23.823495864868164,
      "activations/layer22_attention_weight_max": 50.98992919921875,
      "activations/layer22_attention_weight_min": -33.40799331665039,
      "activations/layer23_attention_weight_max": 36.96605682373047,
      "activations/layer23_attention_weight_min": -26.658756256103516,
      "activations/layer2_attention_weight_max": 31.676063537597656,
      "activations/layer2_attention_weight_min": -32.692665100097656,
      "activations/layer3_attention_weight_max": 95.08104705810547,
      "activations/layer3_attention_weight_min": -97.10728454589844,
      "activations/layer4_attention_weight_max": 104.8646240234375,
      "activations/layer4_attention_weight_min": -108.10375213623047,
      "activations/layer5_attention_weight_max": 51.52547836303711,
      "activations/layer5_attention_weight_min": -60.65141296386719,
      "activations/layer6_attention_weight_max": 43.6102409362793,
      "activations/layer6_attention_weight_min": -45.29044723510742,
      "activations/layer7_attention_weight_max": 94.62234497070312,
      "activations/layer7_attention_weight_min": -97.1103515625,
      "activations/layer8_attention_weight_max": 39.424678802490234,
      "activations/layer8_attention_weight_min": -46.25026321411133,
      "activations/layer9_attention_weight_max": 32.61529541015625,
      "activations/layer9_attention_weight_min": -33.618865966796875,
      "epoch": 16.63,
      "learning_rate": 4.319280303030303e-05,
      "loss": 2.7507,
      "step": 286200
    },
    {
      "activations/layer0_attention_weight_max": 16.332351684570312,
      "activations/layer0_attention_weight_min": -12.680371284484863,
      "activations/layer10_attention_weight_max": 36.4069938659668,
      "activations/layer10_attention_weight_min": -35.57957077026367,
      "activations/layer11_attention_weight_max": 36.72600555419922,
      "activations/layer11_attention_weight_min": -35.987586975097656,
      "activations/layer12_attention_weight_max": 29.26693344116211,
      "activations/layer12_attention_weight_min": -29.23686981201172,
      "activations/layer13_attention_weight_max": 45.43364334106445,
      "activations/layer13_attention_weight_min": -41.209373474121094,
      "activations/layer14_attention_weight_max": 50.36545181274414,
      "activations/layer14_attention_weight_min": -43.602699279785156,
      "activations/layer15_attention_weight_max": 45.78911590576172,
      "activations/layer15_attention_weight_min": -39.57573699951172,
      "activations/layer16_attention_weight_max": 32.37748336791992,
      "activations/layer16_attention_weight_min": -28.513132095336914,
      "activations/layer17_attention_weight_max": 32.05070114135742,
      "activations/layer17_attention_weight_min": -26.650821685791016,
      "activations/layer18_attention_weight_max": 29.745174407958984,
      "activations/layer18_attention_weight_min": -26.076601028442383,
      "activations/layer19_attention_weight_max": 41.239044189453125,
      "activations/layer19_attention_weight_min": -33.06995391845703,
      "activations/layer1_attention_weight_max": 17.15815544128418,
      "activations/layer1_attention_weight_min": -15.454171180725098,
      "activations/layer20_attention_weight_max": 30.153213500976562,
      "activations/layer20_attention_weight_min": -25.801862716674805,
      "activations/layer21_attention_weight_max": 28.590646743774414,
      "activations/layer21_attention_weight_min": -24.1370906829834,
      "activations/layer22_attention_weight_max": 40.7878303527832,
      "activations/layer22_attention_weight_min": -31.655704498291016,
      "activations/layer23_attention_weight_max": 34.154388427734375,
      "activations/layer23_attention_weight_min": -28.277252197265625,
      "activations/layer2_attention_weight_max": 33.55610656738281,
      "activations/layer2_attention_weight_min": -32.871726989746094,
      "activations/layer3_attention_weight_max": 101.3434066772461,
      "activations/layer3_attention_weight_min": -100.9797592163086,
      "activations/layer4_attention_weight_max": 107.72621154785156,
      "activations/layer4_attention_weight_min": -105.00904846191406,
      "activations/layer5_attention_weight_max": 51.81220626831055,
      "activations/layer5_attention_weight_min": -58.639732360839844,
      "activations/layer6_attention_weight_max": 44.546546936035156,
      "activations/layer6_attention_weight_min": -46.35358428955078,
      "activations/layer7_attention_weight_max": 96.76109313964844,
      "activations/layer7_attention_weight_min": -99.50045776367188,
      "activations/layer8_attention_weight_max": 43.51932907104492,
      "activations/layer8_attention_weight_min": -44.787445068359375,
      "activations/layer9_attention_weight_max": 33.4565315246582,
      "activations/layer9_attention_weight_min": -35.76581573486328,
      "epoch": 16.63,
      "learning_rate": 4.317386363636363e-05,
      "loss": 2.7514,
      "step": 286250
    },
    {
      "activations/layer0_attention_weight_max": 16.19503402709961,
      "activations/layer0_attention_weight_min": -12.60904598236084,
      "activations/layer10_attention_weight_max": 32.395748138427734,
      "activations/layer10_attention_weight_min": -32.61757278442383,
      "activations/layer11_attention_weight_max": 32.1843147277832,
      "activations/layer11_attention_weight_min": -35.0400505065918,
      "activations/layer12_attention_weight_max": 26.74102020263672,
      "activations/layer12_attention_weight_min": -26.241525650024414,
      "activations/layer13_attention_weight_max": 40.279991149902344,
      "activations/layer13_attention_weight_min": -36.47088623046875,
      "activations/layer14_attention_weight_max": 45.019004821777344,
      "activations/layer14_attention_weight_min": -40.21167755126953,
      "activations/layer15_attention_weight_max": 40.41976547241211,
      "activations/layer15_attention_weight_min": -34.349910736083984,
      "activations/layer16_attention_weight_max": 31.935924530029297,
      "activations/layer16_attention_weight_min": -28.23531723022461,
      "activations/layer17_attention_weight_max": 30.588830947875977,
      "activations/layer17_attention_weight_min": -26.91623306274414,
      "activations/layer18_attention_weight_max": 33.11643981933594,
      "activations/layer18_attention_weight_min": -27.001195907592773,
      "activations/layer19_attention_weight_max": 35.935096740722656,
      "activations/layer19_attention_weight_min": -30.766538619995117,
      "activations/layer1_attention_weight_max": 16.502281188964844,
      "activations/layer1_attention_weight_min": -14.445581436157227,
      "activations/layer20_attention_weight_max": 29.371536254882812,
      "activations/layer20_attention_weight_min": -25.65916633605957,
      "activations/layer21_attention_weight_max": 27.844600677490234,
      "activations/layer21_attention_weight_min": -23.962371826171875,
      "activations/layer22_attention_weight_max": 42.47041320800781,
      "activations/layer22_attention_weight_min": -31.309465408325195,
      "activations/layer23_attention_weight_max": 32.715850830078125,
      "activations/layer23_attention_weight_min": -29.043760299682617,
      "activations/layer2_attention_weight_max": 35.52642059326172,
      "activations/layer2_attention_weight_min": -35.11760330200195,
      "activations/layer3_attention_weight_max": 106.30311584472656,
      "activations/layer3_attention_weight_min": -106.07066345214844,
      "activations/layer4_attention_weight_max": 114.95155334472656,
      "activations/layer4_attention_weight_min": -117.07540130615234,
      "activations/layer5_attention_weight_max": 56.7623291015625,
      "activations/layer5_attention_weight_min": -66.15940856933594,
      "activations/layer6_attention_weight_max": 43.36507034301758,
      "activations/layer6_attention_weight_min": -48.41048049926758,
      "activations/layer7_attention_weight_max": 94.9644775390625,
      "activations/layer7_attention_weight_min": -94.91741180419922,
      "activations/layer8_attention_weight_max": 40.71959686279297,
      "activations/layer8_attention_weight_min": -44.824920654296875,
      "activations/layer9_attention_weight_max": 32.1214714050293,
      "activations/layer9_attention_weight_min": -33.699886322021484,
      "epoch": 16.64,
      "learning_rate": 4.315492424242423e-05,
      "loss": 2.748,
      "step": 286300
    },
    {
      "activations/layer0_attention_weight_max": 16.360748291015625,
      "activations/layer0_attention_weight_min": -12.653861999511719,
      "activations/layer10_attention_weight_max": 32.94930648803711,
      "activations/layer10_attention_weight_min": -35.35739517211914,
      "activations/layer11_attention_weight_max": 34.143592834472656,
      "activations/layer11_attention_weight_min": -35.00933074951172,
      "activations/layer12_attention_weight_max": 26.611000061035156,
      "activations/layer12_attention_weight_min": -27.787630081176758,
      "activations/layer13_attention_weight_max": 43.27877426147461,
      "activations/layer13_attention_weight_min": -36.880638122558594,
      "activations/layer14_attention_weight_max": 41.333736419677734,
      "activations/layer14_attention_weight_min": -42.76351547241211,
      "activations/layer15_attention_weight_max": 38.2385368347168,
      "activations/layer15_attention_weight_min": -39.54439163208008,
      "activations/layer16_attention_weight_max": 29.40195655822754,
      "activations/layer16_attention_weight_min": -30.62938690185547,
      "activations/layer17_attention_weight_max": 31.391496658325195,
      "activations/layer17_attention_weight_min": -28.48938751220703,
      "activations/layer18_attention_weight_max": 32.78763961791992,
      "activations/layer18_attention_weight_min": -27.971323013305664,
      "activations/layer19_attention_weight_max": 34.61247634887695,
      "activations/layer19_attention_weight_min": -32.950523376464844,
      "activations/layer1_attention_weight_max": 17.914640426635742,
      "activations/layer1_attention_weight_min": -17.106224060058594,
      "activations/layer20_attention_weight_max": 31.58486557006836,
      "activations/layer20_attention_weight_min": -27.864471435546875,
      "activations/layer21_attention_weight_max": 31.85460662841797,
      "activations/layer21_attention_weight_min": -28.00713348388672,
      "activations/layer22_attention_weight_max": 47.95468521118164,
      "activations/layer22_attention_weight_min": -34.135643005371094,
      "activations/layer23_attention_weight_max": 31.48792839050293,
      "activations/layer23_attention_weight_min": -28.862091064453125,
      "activations/layer2_attention_weight_max": 32.041473388671875,
      "activations/layer2_attention_weight_min": -32.454925537109375,
      "activations/layer3_attention_weight_max": 97.38774871826172,
      "activations/layer3_attention_weight_min": -101.50914764404297,
      "activations/layer4_attention_weight_max": 105.77156829833984,
      "activations/layer4_attention_weight_min": -105.5052490234375,
      "activations/layer5_attention_weight_max": 52.9596061706543,
      "activations/layer5_attention_weight_min": -63.15650177001953,
      "activations/layer6_attention_weight_max": 42.90666961669922,
      "activations/layer6_attention_weight_min": -46.734779357910156,
      "activations/layer7_attention_weight_max": 88.85042572021484,
      "activations/layer7_attention_weight_min": -94.1612319946289,
      "activations/layer8_attention_weight_max": 42.373043060302734,
      "activations/layer8_attention_weight_min": -42.30845260620117,
      "activations/layer9_attention_weight_max": 33.005523681640625,
      "activations/layer9_attention_weight_min": -33.57266616821289,
      "epoch": 16.64,
      "learning_rate": 4.313636363636364e-05,
      "loss": 2.7525,
      "step": 286350
    },
    {
      "activations/layer0_attention_weight_max": 16.178089141845703,
      "activations/layer0_attention_weight_min": -12.76470947265625,
      "activations/layer10_attention_weight_max": 34.334449768066406,
      "activations/layer10_attention_weight_min": -35.426002502441406,
      "activations/layer11_attention_weight_max": 34.41057205200195,
      "activations/layer11_attention_weight_min": -33.92438507080078,
      "activations/layer12_attention_weight_max": 26.379108428955078,
      "activations/layer12_attention_weight_min": -26.151351928710938,
      "activations/layer13_attention_weight_max": 37.93867492675781,
      "activations/layer13_attention_weight_min": -35.750518798828125,
      "activations/layer14_attention_weight_max": 37.78532791137695,
      "activations/layer14_attention_weight_min": -36.747398376464844,
      "activations/layer15_attention_weight_max": 37.21500778198242,
      "activations/layer15_attention_weight_min": -36.14459991455078,
      "activations/layer16_attention_weight_max": 28.379905700683594,
      "activations/layer16_attention_weight_min": -30.141857147216797,
      "activations/layer17_attention_weight_max": 27.728534698486328,
      "activations/layer17_attention_weight_min": -24.782360076904297,
      "activations/layer18_attention_weight_max": 30.393455505371094,
      "activations/layer18_attention_weight_min": -24.84870147705078,
      "activations/layer19_attention_weight_max": 31.529647827148438,
      "activations/layer19_attention_weight_min": -32.811580657958984,
      "activations/layer1_attention_weight_max": 16.25897979736328,
      "activations/layer1_attention_weight_min": -14.460893630981445,
      "activations/layer20_attention_weight_max": 27.219497680664062,
      "activations/layer20_attention_weight_min": -25.53582191467285,
      "activations/layer21_attention_weight_max": 29.40652084350586,
      "activations/layer21_attention_weight_min": -25.209123611450195,
      "activations/layer22_attention_weight_max": 39.10551834106445,
      "activations/layer22_attention_weight_min": -33.31427764892578,
      "activations/layer23_attention_weight_max": 30.558637619018555,
      "activations/layer23_attention_weight_min": -26.952388763427734,
      "activations/layer2_attention_weight_max": 32.43684387207031,
      "activations/layer2_attention_weight_min": -32.5908203125,
      "activations/layer3_attention_weight_max": 97.20587158203125,
      "activations/layer3_attention_weight_min": -100.7146224975586,
      "activations/layer4_attention_weight_max": 105.53006744384766,
      "activations/layer4_attention_weight_min": -107.037841796875,
      "activations/layer5_attention_weight_max": 49.15612030029297,
      "activations/layer5_attention_weight_min": -63.77477264404297,
      "activations/layer6_attention_weight_max": 43.77537155151367,
      "activations/layer6_attention_weight_min": -48.84549331665039,
      "activations/layer7_attention_weight_max": 96.14188385009766,
      "activations/layer7_attention_weight_min": -102.3793716430664,
      "activations/layer8_attention_weight_max": 44.13809585571289,
      "activations/layer8_attention_weight_min": -46.5814323425293,
      "activations/layer9_attention_weight_max": 35.488338470458984,
      "activations/layer9_attention_weight_min": -35.832786560058594,
      "epoch": 16.64,
      "learning_rate": 4.311742424242424e-05,
      "loss": 2.7469,
      "step": 286400
    },
    {
      "activations/layer0_attention_weight_max": 15.376980781555176,
      "activations/layer0_attention_weight_min": -12.965961456298828,
      "activations/layer10_attention_weight_max": 33.477108001708984,
      "activations/layer10_attention_weight_min": -32.06166458129883,
      "activations/layer11_attention_weight_max": 32.80636978149414,
      "activations/layer11_attention_weight_min": -32.61509323120117,
      "activations/layer12_attention_weight_max": 26.295120239257812,
      "activations/layer12_attention_weight_min": -26.174352645874023,
      "activations/layer13_attention_weight_max": 39.40621566772461,
      "activations/layer13_attention_weight_min": -34.829864501953125,
      "activations/layer14_attention_weight_max": 41.73219680786133,
      "activations/layer14_attention_weight_min": -39.571739196777344,
      "activations/layer15_attention_weight_max": 40.761295318603516,
      "activations/layer15_attention_weight_min": -35.73390579223633,
      "activations/layer16_attention_weight_max": 29.74434471130371,
      "activations/layer16_attention_weight_min": -27.74705696105957,
      "activations/layer17_attention_weight_max": 29.60279655456543,
      "activations/layer17_attention_weight_min": -26.3747501373291,
      "activations/layer18_attention_weight_max": 31.052345275878906,
      "activations/layer18_attention_weight_min": -24.752397537231445,
      "activations/layer19_attention_weight_max": 31.188566207885742,
      "activations/layer19_attention_weight_min": -31.178241729736328,
      "activations/layer1_attention_weight_max": 15.64074993133545,
      "activations/layer1_attention_weight_min": -14.812121391296387,
      "activations/layer20_attention_weight_max": 28.194042205810547,
      "activations/layer20_attention_weight_min": -24.613645553588867,
      "activations/layer21_attention_weight_max": 26.799222946166992,
      "activations/layer21_attention_weight_min": -25.431625366210938,
      "activations/layer22_attention_weight_max": 36.702945709228516,
      "activations/layer22_attention_weight_min": -30.12144660949707,
      "activations/layer23_attention_weight_max": 30.775693893432617,
      "activations/layer23_attention_weight_min": -24.89946746826172,
      "activations/layer2_attention_weight_max": 33.98155975341797,
      "activations/layer2_attention_weight_min": -31.78386688232422,
      "activations/layer3_attention_weight_max": 96.84825897216797,
      "activations/layer3_attention_weight_min": -93.03005981445312,
      "activations/layer4_attention_weight_max": 105.9759750366211,
      "activations/layer4_attention_weight_min": -102.56451416015625,
      "activations/layer5_attention_weight_max": 51.08706283569336,
      "activations/layer5_attention_weight_min": -59.4791374206543,
      "activations/layer6_attention_weight_max": 44.72648620605469,
      "activations/layer6_attention_weight_min": -44.66786193847656,
      "activations/layer7_attention_weight_max": 94.07868957519531,
      "activations/layer7_attention_weight_min": -87.27721405029297,
      "activations/layer8_attention_weight_max": 44.20792007446289,
      "activations/layer8_attention_weight_min": -41.66658020019531,
      "activations/layer9_attention_weight_max": 32.290611267089844,
      "activations/layer9_attention_weight_min": -32.41350555419922,
      "epoch": 16.64,
      "learning_rate": 4.309848484848484e-05,
      "loss": 2.7497,
      "step": 286450
    },
    {
      "activations/layer0_attention_weight_max": 15.90646743774414,
      "activations/layer0_attention_weight_min": -12.566426277160645,
      "activations/layer10_attention_weight_max": 31.637718200683594,
      "activations/layer10_attention_weight_min": -32.425880432128906,
      "activations/layer11_attention_weight_max": 33.59126281738281,
      "activations/layer11_attention_weight_min": -31.669891357421875,
      "activations/layer12_attention_weight_max": 26.09716033935547,
      "activations/layer12_attention_weight_min": -31.345537185668945,
      "activations/layer13_attention_weight_max": 39.14585876464844,
      "activations/layer13_attention_weight_min": -34.097572326660156,
      "activations/layer14_attention_weight_max": 42.65723419189453,
      "activations/layer14_attention_weight_min": -37.470760345458984,
      "activations/layer15_attention_weight_max": 40.08454895019531,
      "activations/layer15_attention_weight_min": -36.83647537231445,
      "activations/layer16_attention_weight_max": 29.80039405822754,
      "activations/layer16_attention_weight_min": -27.70814323425293,
      "activations/layer17_attention_weight_max": 30.92234992980957,
      "activations/layer17_attention_weight_min": -25.308561325073242,
      "activations/layer18_attention_weight_max": 32.402259826660156,
      "activations/layer18_attention_weight_min": -25.57474708557129,
      "activations/layer19_attention_weight_max": 32.69411849975586,
      "activations/layer19_attention_weight_min": -30.634008407592773,
      "activations/layer1_attention_weight_max": 16.699623107910156,
      "activations/layer1_attention_weight_min": -14.199825286865234,
      "activations/layer20_attention_weight_max": 27.932348251342773,
      "activations/layer20_attention_weight_min": -26.014835357666016,
      "activations/layer21_attention_weight_max": 29.008073806762695,
      "activations/layer21_attention_weight_min": -24.313949584960938,
      "activations/layer22_attention_weight_max": 42.794212341308594,
      "activations/layer22_attention_weight_min": -31.57063102722168,
      "activations/layer23_attention_weight_max": 32.20438766479492,
      "activations/layer23_attention_weight_min": -25.867786407470703,
      "activations/layer2_attention_weight_max": 33.525291442871094,
      "activations/layer2_attention_weight_min": -31.99449920654297,
      "activations/layer3_attention_weight_max": 97.01301574707031,
      "activations/layer3_attention_weight_min": -94.92361450195312,
      "activations/layer4_attention_weight_max": 107.27962493896484,
      "activations/layer4_attention_weight_min": -110.64348602294922,
      "activations/layer5_attention_weight_max": 50.300655364990234,
      "activations/layer5_attention_weight_min": -61.3786735534668,
      "activations/layer6_attention_weight_max": 41.68346405029297,
      "activations/layer6_attention_weight_min": -46.797996520996094,
      "activations/layer7_attention_weight_max": 89.38650512695312,
      "activations/layer7_attention_weight_min": -93.44293975830078,
      "activations/layer8_attention_weight_max": 40.78150939941406,
      "activations/layer8_attention_weight_min": -42.833797454833984,
      "activations/layer9_attention_weight_max": 31.842498779296875,
      "activations/layer9_attention_weight_min": -32.24336624145508,
      "epoch": 16.65,
      "learning_rate": 4.307954545454545e-05,
      "loss": 2.7466,
      "step": 286500
    },
    {
      "activations/layer0_attention_weight_max": 16.39813804626465,
      "activations/layer0_attention_weight_min": -12.098567962646484,
      "activations/layer10_attention_weight_max": 33.85084533691406,
      "activations/layer10_attention_weight_min": -36.286460876464844,
      "activations/layer11_attention_weight_max": 32.86970520019531,
      "activations/layer11_attention_weight_min": -35.70674133300781,
      "activations/layer12_attention_weight_max": 31.349990844726562,
      "activations/layer12_attention_weight_min": -28.281314849853516,
      "activations/layer13_attention_weight_max": 46.48450469970703,
      "activations/layer13_attention_weight_min": -37.74888229370117,
      "activations/layer14_attention_weight_max": 52.99066925048828,
      "activations/layer14_attention_weight_min": -41.0281867980957,
      "activations/layer15_attention_weight_max": 47.34843063354492,
      "activations/layer15_attention_weight_min": -39.35527420043945,
      "activations/layer16_attention_weight_max": 36.655921936035156,
      "activations/layer16_attention_weight_min": -29.255733489990234,
      "activations/layer17_attention_weight_max": 35.80588912963867,
      "activations/layer17_attention_weight_min": -30.424564361572266,
      "activations/layer18_attention_weight_max": 33.82609176635742,
      "activations/layer18_attention_weight_min": -27.928987503051758,
      "activations/layer19_attention_weight_max": 38.19382095336914,
      "activations/layer19_attention_weight_min": -32.758914947509766,
      "activations/layer1_attention_weight_max": 15.795639038085938,
      "activations/layer1_attention_weight_min": -13.829840660095215,
      "activations/layer20_attention_weight_max": 31.782690048217773,
      "activations/layer20_attention_weight_min": -26.382129669189453,
      "activations/layer21_attention_weight_max": 32.31423568725586,
      "activations/layer21_attention_weight_min": -27.217905044555664,
      "activations/layer22_attention_weight_max": 52.34640121459961,
      "activations/layer22_attention_weight_min": -37.31283187866211,
      "activations/layer23_attention_weight_max": 35.12974548339844,
      "activations/layer23_attention_weight_min": -27.45515251159668,
      "activations/layer2_attention_weight_max": 32.3867301940918,
      "activations/layer2_attention_weight_min": -31.264480590820312,
      "activations/layer3_attention_weight_max": 93.89849853515625,
      "activations/layer3_attention_weight_min": -94.67889404296875,
      "activations/layer4_attention_weight_max": 101.7944107055664,
      "activations/layer4_attention_weight_min": -102.26961517333984,
      "activations/layer5_attention_weight_max": 51.96039962768555,
      "activations/layer5_attention_weight_min": -61.298614501953125,
      "activations/layer6_attention_weight_max": 43.17722702026367,
      "activations/layer6_attention_weight_min": -45.57918167114258,
      "activations/layer7_attention_weight_max": 93.57376098632812,
      "activations/layer7_attention_weight_min": -99.22126007080078,
      "activations/layer8_attention_weight_max": 39.765785217285156,
      "activations/layer8_attention_weight_min": -41.50029373168945,
      "activations/layer9_attention_weight_max": 30.141904830932617,
      "activations/layer9_attention_weight_min": -33.36660385131836,
      "epoch": 16.65,
      "learning_rate": 4.3060606060606057e-05,
      "loss": 2.757,
      "step": 286550
    },
    {
      "activations/layer0_attention_weight_max": 15.633696556091309,
      "activations/layer0_attention_weight_min": -11.816141128540039,
      "activations/layer10_attention_weight_max": 34.6765022277832,
      "activations/layer10_attention_weight_min": -36.66535186767578,
      "activations/layer11_attention_weight_max": 36.95964813232422,
      "activations/layer11_attention_weight_min": -36.326568603515625,
      "activations/layer12_attention_weight_max": 30.372940063476562,
      "activations/layer12_attention_weight_min": -28.2559757232666,
      "activations/layer13_attention_weight_max": 40.598472595214844,
      "activations/layer13_attention_weight_min": -35.3028564453125,
      "activations/layer14_attention_weight_max": 39.77047348022461,
      "activations/layer14_attention_weight_min": -35.23973083496094,
      "activations/layer15_attention_weight_max": 42.323753356933594,
      "activations/layer15_attention_weight_min": -35.514732360839844,
      "activations/layer16_attention_weight_max": 32.05012512207031,
      "activations/layer16_attention_weight_min": -29.926212310791016,
      "activations/layer17_attention_weight_max": 31.02433967590332,
      "activations/layer17_attention_weight_min": -27.075960159301758,
      "activations/layer18_attention_weight_max": 30.757736206054688,
      "activations/layer18_attention_weight_min": -26.649263381958008,
      "activations/layer19_attention_weight_max": 33.68462371826172,
      "activations/layer19_attention_weight_min": -31.218692779541016,
      "activations/layer1_attention_weight_max": 16.61705780029297,
      "activations/layer1_attention_weight_min": -14.429167747497559,
      "activations/layer20_attention_weight_max": 28.1883544921875,
      "activations/layer20_attention_weight_min": -26.20503807067871,
      "activations/layer21_attention_weight_max": 27.848772048950195,
      "activations/layer21_attention_weight_min": -25.00640106201172,
      "activations/layer22_attention_weight_max": 38.44791793823242,
      "activations/layer22_attention_weight_min": -31.576921463012695,
      "activations/layer23_attention_weight_max": 32.355106353759766,
      "activations/layer23_attention_weight_min": -26.761608123779297,
      "activations/layer2_attention_weight_max": 31.283506393432617,
      "activations/layer2_attention_weight_min": -30.137008666992188,
      "activations/layer3_attention_weight_max": 90.41854095458984,
      "activations/layer3_attention_weight_min": -89.81617736816406,
      "activations/layer4_attention_weight_max": 101.6248779296875,
      "activations/layer4_attention_weight_min": -98.89051055908203,
      "activations/layer5_attention_weight_max": 48.42735290527344,
      "activations/layer5_attention_weight_min": -60.59768295288086,
      "activations/layer6_attention_weight_max": 41.366119384765625,
      "activations/layer6_attention_weight_min": -42.95073318481445,
      "activations/layer7_attention_weight_max": 89.29145050048828,
      "activations/layer7_attention_weight_min": -94.21531677246094,
      "activations/layer8_attention_weight_max": 39.03125,
      "activations/layer8_attention_weight_min": -41.48904800415039,
      "activations/layer9_attention_weight_max": 33.26374435424805,
      "activations/layer9_attention_weight_min": -36.5384407043457,
      "epoch": 16.65,
      "learning_rate": 4.3041666666666665e-05,
      "loss": 2.737,
      "step": 286600
    },
    {
      "activations/layer0_attention_weight_max": 16.3400936126709,
      "activations/layer0_attention_weight_min": -12.540369033813477,
      "activations/layer10_attention_weight_max": 30.306264877319336,
      "activations/layer10_attention_weight_min": -32.8195915222168,
      "activations/layer11_attention_weight_max": 30.26545524597168,
      "activations/layer11_attention_weight_min": -31.37664031982422,
      "activations/layer12_attention_weight_max": 24.943517684936523,
      "activations/layer12_attention_weight_min": -26.544404983520508,
      "activations/layer13_attention_weight_max": 38.3335075378418,
      "activations/layer13_attention_weight_min": -34.11884689331055,
      "activations/layer14_attention_weight_max": 39.31343078613281,
      "activations/layer14_attention_weight_min": -36.97816848754883,
      "activations/layer15_attention_weight_max": 38.33458709716797,
      "activations/layer15_attention_weight_min": -34.982234954833984,
      "activations/layer16_attention_weight_max": 28.451507568359375,
      "activations/layer16_attention_weight_min": -26.756595611572266,
      "activations/layer17_attention_weight_max": 27.67853546142578,
      "activations/layer17_attention_weight_min": -24.648225784301758,
      "activations/layer18_attention_weight_max": 32.219974517822266,
      "activations/layer18_attention_weight_min": -23.663599014282227,
      "activations/layer19_attention_weight_max": 33.968605041503906,
      "activations/layer19_attention_weight_min": -31.037628173828125,
      "activations/layer1_attention_weight_max": 16.778627395629883,
      "activations/layer1_attention_weight_min": -13.552961349487305,
      "activations/layer20_attention_weight_max": 27.82866096496582,
      "activations/layer20_attention_weight_min": -25.585847854614258,
      "activations/layer21_attention_weight_max": 26.488590240478516,
      "activations/layer21_attention_weight_min": -25.133203506469727,
      "activations/layer22_attention_weight_max": 38.50334930419922,
      "activations/layer22_attention_weight_min": -29.624048233032227,
      "activations/layer23_attention_weight_max": 31.015132904052734,
      "activations/layer23_attention_weight_min": -23.039512634277344,
      "activations/layer2_attention_weight_max": 31.458736419677734,
      "activations/layer2_attention_weight_min": -32.787841796875,
      "activations/layer3_attention_weight_max": 93.50627136230469,
      "activations/layer3_attention_weight_min": -95.9561996459961,
      "activations/layer4_attention_weight_max": 101.37466430664062,
      "activations/layer4_attention_weight_min": -105.79204559326172,
      "activations/layer5_attention_weight_max": 48.31902313232422,
      "activations/layer5_attention_weight_min": -61.61161804199219,
      "activations/layer6_attention_weight_max": 42.73680114746094,
      "activations/layer6_attention_weight_min": -44.97748565673828,
      "activations/layer7_attention_weight_max": 91.87835693359375,
      "activations/layer7_attention_weight_min": -99.72564697265625,
      "activations/layer8_attention_weight_max": 41.77674865722656,
      "activations/layer8_attention_weight_min": -42.2757682800293,
      "activations/layer9_attention_weight_max": 29.787403106689453,
      "activations/layer9_attention_weight_min": -32.62920379638672,
      "epoch": 16.66,
      "learning_rate": 4.3022727272727266e-05,
      "loss": 2.7421,
      "step": 286650
    },
    {
      "activations/layer0_attention_weight_max": 15.837432861328125,
      "activations/layer0_attention_weight_min": -12.046208381652832,
      "activations/layer10_attention_weight_max": 31.480865478515625,
      "activations/layer10_attention_weight_min": -32.0601692199707,
      "activations/layer11_attention_weight_max": 32.399818420410156,
      "activations/layer11_attention_weight_min": -33.41482925415039,
      "activations/layer12_attention_weight_max": 26.56406021118164,
      "activations/layer12_attention_weight_min": -25.787185668945312,
      "activations/layer13_attention_weight_max": 38.24298858642578,
      "activations/layer13_attention_weight_min": -34.88541793823242,
      "activations/layer14_attention_weight_max": 41.78612518310547,
      "activations/layer14_attention_weight_min": -36.65266799926758,
      "activations/layer15_attention_weight_max": 38.32703399658203,
      "activations/layer15_attention_weight_min": -34.53190994262695,
      "activations/layer16_attention_weight_max": 32.046390533447266,
      "activations/layer16_attention_weight_min": -27.07109832763672,
      "activations/layer17_attention_weight_max": 33.23726272583008,
      "activations/layer17_attention_weight_min": -26.484010696411133,
      "activations/layer18_attention_weight_max": 36.46989822387695,
      "activations/layer18_attention_weight_min": -25.340885162353516,
      "activations/layer19_attention_weight_max": 37.189300537109375,
      "activations/layer19_attention_weight_min": -32.750396728515625,
      "activations/layer1_attention_weight_max": 15.914670944213867,
      "activations/layer1_attention_weight_min": -14.559500694274902,
      "activations/layer20_attention_weight_max": 34.18932342529297,
      "activations/layer20_attention_weight_min": -26.794898986816406,
      "activations/layer21_attention_weight_max": 29.05998992919922,
      "activations/layer21_attention_weight_min": -26.08732795715332,
      "activations/layer22_attention_weight_max": 44.23472595214844,
      "activations/layer22_attention_weight_min": -31.904144287109375,
      "activations/layer23_attention_weight_max": 35.0336799621582,
      "activations/layer23_attention_weight_min": -25.23444938659668,
      "activations/layer2_attention_weight_max": 31.899555206298828,
      "activations/layer2_attention_weight_min": -31.498458862304688,
      "activations/layer3_attention_weight_max": 98.6177978515625,
      "activations/layer3_attention_weight_min": -100.0956802368164,
      "activations/layer4_attention_weight_max": 109.14735412597656,
      "activations/layer4_attention_weight_min": -109.55487060546875,
      "activations/layer5_attention_weight_max": 50.295326232910156,
      "activations/layer5_attention_weight_min": -66.90852355957031,
      "activations/layer6_attention_weight_max": 43.89566421508789,
      "activations/layer6_attention_weight_min": -46.213829040527344,
      "activations/layer7_attention_weight_max": 88.8628158569336,
      "activations/layer7_attention_weight_min": -95.7327651977539,
      "activations/layer8_attention_weight_max": 40.35338592529297,
      "activations/layer8_attention_weight_min": -41.69660186767578,
      "activations/layer9_attention_weight_max": 30.928510665893555,
      "activations/layer9_attention_weight_min": -33.272037506103516,
      "epoch": 16.66,
      "learning_rate": 4.300378787878787e-05,
      "loss": 2.7534,
      "step": 286700
    },
    {
      "activations/layer0_attention_weight_max": 16.1583194732666,
      "activations/layer0_attention_weight_min": -13.234090805053711,
      "activations/layer10_attention_weight_max": 36.12837219238281,
      "activations/layer10_attention_weight_min": -34.99591827392578,
      "activations/layer11_attention_weight_max": 35.912322998046875,
      "activations/layer11_attention_weight_min": -32.722686767578125,
      "activations/layer12_attention_weight_max": 28.227962493896484,
      "activations/layer12_attention_weight_min": -30.0484676361084,
      "activations/layer13_attention_weight_max": 42.24636459350586,
      "activations/layer13_attention_weight_min": -36.20337677001953,
      "activations/layer14_attention_weight_max": 44.1544303894043,
      "activations/layer14_attention_weight_min": -37.99300765991211,
      "activations/layer15_attention_weight_max": 40.55256652832031,
      "activations/layer15_attention_weight_min": -36.3687629699707,
      "activations/layer16_attention_weight_max": 29.294618606567383,
      "activations/layer16_attention_weight_min": -28.400272369384766,
      "activations/layer17_attention_weight_max": 31.98970603942871,
      "activations/layer17_attention_weight_min": -26.434412002563477,
      "activations/layer18_attention_weight_max": 32.46806716918945,
      "activations/layer18_attention_weight_min": -26.056459426879883,
      "activations/layer19_attention_weight_max": 34.53184127807617,
      "activations/layer19_attention_weight_min": -31.913772583007812,
      "activations/layer1_attention_weight_max": 17.19598388671875,
      "activations/layer1_attention_weight_min": -14.93814754486084,
      "activations/layer20_attention_weight_max": 32.32143020629883,
      "activations/layer20_attention_weight_min": -27.498422622680664,
      "activations/layer21_attention_weight_max": 30.71569061279297,
      "activations/layer21_attention_weight_min": -27.40013885498047,
      "activations/layer22_attention_weight_max": 50.83523178100586,
      "activations/layer22_attention_weight_min": -33.5352668762207,
      "activations/layer23_attention_weight_max": 37.5235595703125,
      "activations/layer23_attention_weight_min": -27.03280258178711,
      "activations/layer2_attention_weight_max": 32.73855972290039,
      "activations/layer2_attention_weight_min": -30.566661834716797,
      "activations/layer3_attention_weight_max": 95.26049041748047,
      "activations/layer3_attention_weight_min": -92.45262145996094,
      "activations/layer4_attention_weight_max": 106.20507049560547,
      "activations/layer4_attention_weight_min": -102.5074462890625,
      "activations/layer5_attention_weight_max": 52.05303192138672,
      "activations/layer5_attention_weight_min": -58.61898422241211,
      "activations/layer6_attention_weight_max": 44.369441986083984,
      "activations/layer6_attention_weight_min": -43.9409294128418,
      "activations/layer7_attention_weight_max": 94.42974853515625,
      "activations/layer7_attention_weight_min": -91.30489349365234,
      "activations/layer8_attention_weight_max": 42.10365676879883,
      "activations/layer8_attention_weight_min": -42.58701705932617,
      "activations/layer9_attention_weight_max": 34.20486068725586,
      "activations/layer9_attention_weight_min": -34.5205078125,
      "epoch": 16.66,
      "learning_rate": 4.298484848484848e-05,
      "loss": 2.7538,
      "step": 286750
    },
    {
      "activations/layer0_attention_weight_max": 15.404248237609863,
      "activations/layer0_attention_weight_min": -12.499112129211426,
      "activations/layer10_attention_weight_max": 32.45192337036133,
      "activations/layer10_attention_weight_min": -34.0709114074707,
      "activations/layer11_attention_weight_max": 32.33346939086914,
      "activations/layer11_attention_weight_min": -32.127838134765625,
      "activations/layer12_attention_weight_max": 25.576255798339844,
      "activations/layer12_attention_weight_min": -25.55051612854004,
      "activations/layer13_attention_weight_max": 36.56749725341797,
      "activations/layer13_attention_weight_min": -34.7165641784668,
      "activations/layer14_attention_weight_max": 41.45455551147461,
      "activations/layer14_attention_weight_min": -37.23467254638672,
      "activations/layer15_attention_weight_max": 35.868167877197266,
      "activations/layer15_attention_weight_min": -32.99673843383789,
      "activations/layer16_attention_weight_max": 29.136737823486328,
      "activations/layer16_attention_weight_min": -30.976490020751953,
      "activations/layer17_attention_weight_max": 29.604631423950195,
      "activations/layer17_attention_weight_min": -26.527854919433594,
      "activations/layer18_attention_weight_max": 32.398014068603516,
      "activations/layer18_attention_weight_min": -25.179744720458984,
      "activations/layer19_attention_weight_max": 34.05329895019531,
      "activations/layer19_attention_weight_min": -31.302831649780273,
      "activations/layer1_attention_weight_max": 15.838298797607422,
      "activations/layer1_attention_weight_min": -13.731795310974121,
      "activations/layer20_attention_weight_max": 28.133644104003906,
      "activations/layer20_attention_weight_min": -26.11847496032715,
      "activations/layer21_attention_weight_max": 26.37457275390625,
      "activations/layer21_attention_weight_min": -26.404312133789062,
      "activations/layer22_attention_weight_max": 45.4036865234375,
      "activations/layer22_attention_weight_min": -31.03573226928711,
      "activations/layer23_attention_weight_max": 29.017288208007812,
      "activations/layer23_attention_weight_min": -26.63091278076172,
      "activations/layer2_attention_weight_max": 33.990291595458984,
      "activations/layer2_attention_weight_min": -31.65420150756836,
      "activations/layer3_attention_weight_max": 101.45697021484375,
      "activations/layer3_attention_weight_min": -102.6919174194336,
      "activations/layer4_attention_weight_max": 106.01434326171875,
      "activations/layer4_attention_weight_min": -112.8853759765625,
      "activations/layer5_attention_weight_max": 51.56341552734375,
      "activations/layer5_attention_weight_min": -63.712013244628906,
      "activations/layer6_attention_weight_max": 44.40737533569336,
      "activations/layer6_attention_weight_min": -47.175289154052734,
      "activations/layer7_attention_weight_max": 92.83753967285156,
      "activations/layer7_attention_weight_min": -96.22924041748047,
      "activations/layer8_attention_weight_max": 43.68357467651367,
      "activations/layer8_attention_weight_min": -47.668113708496094,
      "activations/layer9_attention_weight_max": 31.527942657470703,
      "activations/layer9_attention_weight_min": -35.60517501831055,
      "epoch": 16.66,
      "learning_rate": 4.2965909090909084e-05,
      "loss": 2.7425,
      "step": 286800
    },
    {
      "activations/layer0_attention_weight_max": 15.261335372924805,
      "activations/layer0_attention_weight_min": -12.552414894104004,
      "activations/layer10_attention_weight_max": 35.11757278442383,
      "activations/layer10_attention_weight_min": -34.222450256347656,
      "activations/layer11_attention_weight_max": 37.90024948120117,
      "activations/layer11_attention_weight_min": -39.01185607910156,
      "activations/layer12_attention_weight_max": 30.65339469909668,
      "activations/layer12_attention_weight_min": -26.63473129272461,
      "activations/layer13_attention_weight_max": 47.22964859008789,
      "activations/layer13_attention_weight_min": -36.43735885620117,
      "activations/layer14_attention_weight_max": 47.124813079833984,
      "activations/layer14_attention_weight_min": -36.488704681396484,
      "activations/layer15_attention_weight_max": 40.294857025146484,
      "activations/layer15_attention_weight_min": -35.25310516357422,
      "activations/layer16_attention_weight_max": 32.67906951904297,
      "activations/layer16_attention_weight_min": -28.648921966552734,
      "activations/layer17_attention_weight_max": 32.96501541137695,
      "activations/layer17_attention_weight_min": -25.950267791748047,
      "activations/layer18_attention_weight_max": 33.041908264160156,
      "activations/layer18_attention_weight_min": -26.348369598388672,
      "activations/layer19_attention_weight_max": 34.862632751464844,
      "activations/layer19_attention_weight_min": -30.37118911743164,
      "activations/layer1_attention_weight_max": 18.721994400024414,
      "activations/layer1_attention_weight_min": -17.216705322265625,
      "activations/layer20_attention_weight_max": 31.884075164794922,
      "activations/layer20_attention_weight_min": -26.45904541015625,
      "activations/layer21_attention_weight_max": 33.87531280517578,
      "activations/layer21_attention_weight_min": -26.672832489013672,
      "activations/layer22_attention_weight_max": 45.24392318725586,
      "activations/layer22_attention_weight_min": -34.091796875,
      "activations/layer23_attention_weight_max": 34.00785827636719,
      "activations/layer23_attention_weight_min": -25.82819175720215,
      "activations/layer2_attention_weight_max": 36.39927291870117,
      "activations/layer2_attention_weight_min": -32.72660827636719,
      "activations/layer3_attention_weight_max": 107.02367401123047,
      "activations/layer3_attention_weight_min": -103.28692626953125,
      "activations/layer4_attention_weight_max": 116.38494110107422,
      "activations/layer4_attention_weight_min": -111.92867279052734,
      "activations/layer5_attention_weight_max": 55.23623275756836,
      "activations/layer5_attention_weight_min": -63.39426803588867,
      "activations/layer6_attention_weight_max": 47.066261291503906,
      "activations/layer6_attention_weight_min": -49.772727966308594,
      "activations/layer7_attention_weight_max": 96.18357849121094,
      "activations/layer7_attention_weight_min": -100.0877456665039,
      "activations/layer8_attention_weight_max": 48.77870178222656,
      "activations/layer8_attention_weight_min": -48.593360900878906,
      "activations/layer9_attention_weight_max": 35.4023323059082,
      "activations/layer9_attention_weight_min": -36.39561462402344,
      "epoch": 16.67,
      "learning_rate": 4.294696969696969e-05,
      "loss": 2.7564,
      "step": 286850
    },
    {
      "activations/layer0_attention_weight_max": 15.727179527282715,
      "activations/layer0_attention_weight_min": -12.882379531860352,
      "activations/layer10_attention_weight_max": 39.212554931640625,
      "activations/layer10_attention_weight_min": -36.71849822998047,
      "activations/layer11_attention_weight_max": 42.006595611572266,
      "activations/layer11_attention_weight_min": -36.13496017456055,
      "activations/layer12_attention_weight_max": 27.420564651489258,
      "activations/layer12_attention_weight_min": -26.647233963012695,
      "activations/layer13_attention_weight_max": 41.46595764160156,
      "activations/layer13_attention_weight_min": -35.21030807495117,
      "activations/layer14_attention_weight_max": 43.92007827758789,
      "activations/layer14_attention_weight_min": -36.928001403808594,
      "activations/layer15_attention_weight_max": 40.88009262084961,
      "activations/layer15_attention_weight_min": -33.05427932739258,
      "activations/layer16_attention_weight_max": 30.453182220458984,
      "activations/layer16_attention_weight_min": -25.749494552612305,
      "activations/layer17_attention_weight_max": 30.26762580871582,
      "activations/layer17_attention_weight_min": -25.737529754638672,
      "activations/layer18_attention_weight_max": 29.82020378112793,
      "activations/layer18_attention_weight_min": -23.947826385498047,
      "activations/layer19_attention_weight_max": 31.957931518554688,
      "activations/layer19_attention_weight_min": -29.8748836517334,
      "activations/layer1_attention_weight_max": 17.11705207824707,
      "activations/layer1_attention_weight_min": -15.879039764404297,
      "activations/layer20_attention_weight_max": 26.62911033630371,
      "activations/layer20_attention_weight_min": -24.85295867919922,
      "activations/layer21_attention_weight_max": 26.00688934326172,
      "activations/layer21_attention_weight_min": -24.4786319732666,
      "activations/layer22_attention_weight_max": 38.83000183105469,
      "activations/layer22_attention_weight_min": -30.8586483001709,
      "activations/layer23_attention_weight_max": 33.226409912109375,
      "activations/layer23_attention_weight_min": -24.67591667175293,
      "activations/layer2_attention_weight_max": 32.38608169555664,
      "activations/layer2_attention_weight_min": -32.963539123535156,
      "activations/layer3_attention_weight_max": 97.56854248046875,
      "activations/layer3_attention_weight_min": -104.5010986328125,
      "activations/layer4_attention_weight_max": 110.76210021972656,
      "activations/layer4_attention_weight_min": -118.23664855957031,
      "activations/layer5_attention_weight_max": 51.22113800048828,
      "activations/layer5_attention_weight_min": -61.79857635498047,
      "activations/layer6_attention_weight_max": 44.29418182373047,
      "activations/layer6_attention_weight_min": -45.94778823852539,
      "activations/layer7_attention_weight_max": 97.43125915527344,
      "activations/layer7_attention_weight_min": -96.18273162841797,
      "activations/layer8_attention_weight_max": 45.783226013183594,
      "activations/layer8_attention_weight_min": -46.6036491394043,
      "activations/layer9_attention_weight_max": 35.41714096069336,
      "activations/layer9_attention_weight_min": -34.51670837402344,
      "epoch": 16.67,
      "learning_rate": 4.29280303030303e-05,
      "loss": 2.7408,
      "step": 286900
    },
    {
      "activations/layer0_attention_weight_max": 16.69662857055664,
      "activations/layer0_attention_weight_min": -12.64115047454834,
      "activations/layer10_attention_weight_max": 35.028968811035156,
      "activations/layer10_attention_weight_min": -35.15364456176758,
      "activations/layer11_attention_weight_max": 33.781639099121094,
      "activations/layer11_attention_weight_min": -35.09982681274414,
      "activations/layer12_attention_weight_max": 25.830646514892578,
      "activations/layer12_attention_weight_min": -26.580408096313477,
      "activations/layer13_attention_weight_max": 40.51754379272461,
      "activations/layer13_attention_weight_min": -38.06372833251953,
      "activations/layer14_attention_weight_max": 41.44460678100586,
      "activations/layer14_attention_weight_min": -38.637210845947266,
      "activations/layer15_attention_weight_max": 41.83445358276367,
      "activations/layer15_attention_weight_min": -33.816261291503906,
      "activations/layer16_attention_weight_max": 28.14812660217285,
      "activations/layer16_attention_weight_min": -26.26434326171875,
      "activations/layer17_attention_weight_max": 30.270172119140625,
      "activations/layer17_attention_weight_min": -25.227886199951172,
      "activations/layer18_attention_weight_max": 29.712486267089844,
      "activations/layer18_attention_weight_min": -24.73113250732422,
      "activations/layer19_attention_weight_max": 31.172483444213867,
      "activations/layer19_attention_weight_min": -29.31734275817871,
      "activations/layer1_attention_weight_max": 15.789575576782227,
      "activations/layer1_attention_weight_min": -15.316340446472168,
      "activations/layer20_attention_weight_max": 28.833507537841797,
      "activations/layer20_attention_weight_min": -24.069921493530273,
      "activations/layer21_attention_weight_max": 28.830820083618164,
      "activations/layer21_attention_weight_min": -24.39371681213379,
      "activations/layer22_attention_weight_max": 42.36345672607422,
      "activations/layer22_attention_weight_min": -29.22260093688965,
      "activations/layer23_attention_weight_max": 32.955352783203125,
      "activations/layer23_attention_weight_min": -23.61823844909668,
      "activations/layer2_attention_weight_max": 32.28253173828125,
      "activations/layer2_attention_weight_min": -31.776092529296875,
      "activations/layer3_attention_weight_max": 93.45867156982422,
      "activations/layer3_attention_weight_min": -92.98165130615234,
      "activations/layer4_attention_weight_max": 102.55777740478516,
      "activations/layer4_attention_weight_min": -107.3931655883789,
      "activations/layer5_attention_weight_max": 47.751487731933594,
      "activations/layer5_attention_weight_min": -61.839317321777344,
      "activations/layer6_attention_weight_max": 42.791404724121094,
      "activations/layer6_attention_weight_min": -44.79338073730469,
      "activations/layer7_attention_weight_max": 93.38163757324219,
      "activations/layer7_attention_weight_min": -98.09215545654297,
      "activations/layer8_attention_weight_max": 44.02967071533203,
      "activations/layer8_attention_weight_min": -45.86232376098633,
      "activations/layer9_attention_weight_max": 34.75868606567383,
      "activations/layer9_attention_weight_min": -35.361080169677734,
      "epoch": 16.67,
      "learning_rate": 4.290909090909091e-05,
      "loss": 2.7433,
      "step": 286950
    },
    {
      "activations/layer0_attention_weight_max": 16.391786575317383,
      "activations/layer0_attention_weight_min": -12.175031661987305,
      "activations/layer10_attention_weight_max": 36.15179443359375,
      "activations/layer10_attention_weight_min": -37.16562271118164,
      "activations/layer11_attention_weight_max": 35.627445220947266,
      "activations/layer11_attention_weight_min": -37.02220153808594,
      "activations/layer12_attention_weight_max": 31.163291931152344,
      "activations/layer12_attention_weight_min": -28.10774040222168,
      "activations/layer13_attention_weight_max": 46.79994201660156,
      "activations/layer13_attention_weight_min": -36.37135696411133,
      "activations/layer14_attention_weight_max": 47.700103759765625,
      "activations/layer14_attention_weight_min": -37.95618438720703,
      "activations/layer15_attention_weight_max": 40.46919631958008,
      "activations/layer15_attention_weight_min": -35.892887115478516,
      "activations/layer16_attention_weight_max": 34.57490158081055,
      "activations/layer16_attention_weight_min": -30.06729507446289,
      "activations/layer17_attention_weight_max": 35.08186340332031,
      "activations/layer17_attention_weight_min": -29.296762466430664,
      "activations/layer18_attention_weight_max": 31.6429386138916,
      "activations/layer18_attention_weight_min": -29.392995834350586,
      "activations/layer19_attention_weight_max": 35.35521697998047,
      "activations/layer19_attention_weight_min": -31.929914474487305,
      "activations/layer1_attention_weight_max": 16.793384552001953,
      "activations/layer1_attention_weight_min": -15.189876556396484,
      "activations/layer20_attention_weight_max": 29.229589462280273,
      "activations/layer20_attention_weight_min": -26.29241180419922,
      "activations/layer21_attention_weight_max": 31.143247604370117,
      "activations/layer21_attention_weight_min": -25.92828941345215,
      "activations/layer22_attention_weight_max": 40.862831115722656,
      "activations/layer22_attention_weight_min": -34.429718017578125,
      "activations/layer23_attention_weight_max": 34.55781173706055,
      "activations/layer23_attention_weight_min": -29.02029037475586,
      "activations/layer2_attention_weight_max": 32.94206619262695,
      "activations/layer2_attention_weight_min": -32.552513122558594,
      "activations/layer3_attention_weight_max": 95.25511932373047,
      "activations/layer3_attention_weight_min": -97.37814331054688,
      "activations/layer4_attention_weight_max": 105.07708740234375,
      "activations/layer4_attention_weight_min": -104.29058837890625,
      "activations/layer5_attention_weight_max": 49.20260238647461,
      "activations/layer5_attention_weight_min": -58.51165771484375,
      "activations/layer6_attention_weight_max": 42.9930305480957,
      "activations/layer6_attention_weight_min": -44.46065139770508,
      "activations/layer7_attention_weight_max": 95.40458679199219,
      "activations/layer7_attention_weight_min": -91.7569808959961,
      "activations/layer8_attention_weight_max": 42.755191802978516,
      "activations/layer8_attention_weight_min": -45.90537643432617,
      "activations/layer9_attention_weight_max": 34.260921478271484,
      "activations/layer9_attention_weight_min": -35.15953826904297,
      "epoch": 16.68,
      "learning_rate": 4.289015151515151e-05,
      "loss": 2.7462,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.546,
      "eval_samples_per_second": 502.457,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.546,
      "eval_openwebtext_samples_per_second": 502.457,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9853,
      "eval_wikitext_samples_per_second": 229.692,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_lambada_loss": 2.4140625,
      "eval_lambada_ppl": 11.17928486368573,
      "eval_lambada_runtime": 9.5574,
      "eval_lambada_samples_per_second": 509.446,
      "step": 287000
    },
    {
      "activations/layer0_attention_weight_max": 15.982345581054688,
      "activations/layer0_attention_weight_min": -12.671510696411133,
      "activations/layer10_attention_weight_max": 39.690940856933594,
      "activations/layer10_attention_weight_min": -38.09182357788086,
      "activations/layer11_attention_weight_max": 40.081260681152344,
      "activations/layer11_attention_weight_min": -37.36731719970703,
      "activations/layer12_attention_weight_max": 26.861875534057617,
      "activations/layer12_attention_weight_min": -26.398191452026367,
      "activations/layer13_attention_weight_max": 38.24027633666992,
      "activations/layer13_attention_weight_min": -35.33992004394531,
      "activations/layer14_attention_weight_max": 45.36531448364258,
      "activations/layer14_attention_weight_min": -38.493282318115234,
      "activations/layer15_attention_weight_max": 38.95851135253906,
      "activations/layer15_attention_weight_min": -38.3786506652832,
      "activations/layer16_attention_weight_max": 28.238994598388672,
      "activations/layer16_attention_weight_min": -26.767271041870117,
      "activations/layer17_attention_weight_max": 27.9489688873291,
      "activations/layer17_attention_weight_min": -24.129291534423828,
      "activations/layer18_attention_weight_max": 28.010475158691406,
      "activations/layer18_attention_weight_min": -25.32082748413086,
      "activations/layer19_attention_weight_max": 33.902095794677734,
      "activations/layer19_attention_weight_min": -30.8118953704834,
      "activations/layer1_attention_weight_max": 16.3934326171875,
      "activations/layer1_attention_weight_min": -15.32435131072998,
      "activations/layer20_attention_weight_max": 29.08462142944336,
      "activations/layer20_attention_weight_min": -23.95406150817871,
      "activations/layer21_attention_weight_max": 27.562692642211914,
      "activations/layer21_attention_weight_min": -24.866287231445312,
      "activations/layer22_attention_weight_max": 39.540340423583984,
      "activations/layer22_attention_weight_min": -33.12773132324219,
      "activations/layer23_attention_weight_max": 30.258665084838867,
      "activations/layer23_attention_weight_min": -25.811620712280273,
      "activations/layer2_attention_weight_max": 34.13709259033203,
      "activations/layer2_attention_weight_min": -33.477394104003906,
      "activations/layer3_attention_weight_max": 96.75932312011719,
      "activations/layer3_attention_weight_min": -98.32074737548828,
      "activations/layer4_attention_weight_max": 107.46209716796875,
      "activations/layer4_attention_weight_min": -105.0109634399414,
      "activations/layer5_attention_weight_max": 49.94557189941406,
      "activations/layer5_attention_weight_min": -61.15653991699219,
      "activations/layer6_attention_weight_max": 44.718570709228516,
      "activations/layer6_attention_weight_min": -44.94752883911133,
      "activations/layer7_attention_weight_max": 95.73680877685547,
      "activations/layer7_attention_weight_min": -97.22364807128906,
      "activations/layer8_attention_weight_max": 47.67851257324219,
      "activations/layer8_attention_weight_min": -45.44233322143555,
      "activations/layer9_attention_weight_max": 37.916805267333984,
      "activations/layer9_attention_weight_min": -39.0170783996582,
      "epoch": 16.68,
      "learning_rate": 4.287121212121211e-05,
      "loss": 2.7449,
      "step": 287050
    },
    {
      "activations/layer0_attention_weight_max": 16.52456283569336,
      "activations/layer0_attention_weight_min": -12.033028602600098,
      "activations/layer10_attention_weight_max": 35.37612533569336,
      "activations/layer10_attention_weight_min": -33.18690872192383,
      "activations/layer11_attention_weight_max": 36.75645446777344,
      "activations/layer11_attention_weight_min": -31.834461212158203,
      "activations/layer12_attention_weight_max": 26.59374237060547,
      "activations/layer12_attention_weight_min": -25.259708404541016,
      "activations/layer13_attention_weight_max": 40.53212356567383,
      "activations/layer13_attention_weight_min": -34.89961242675781,
      "activations/layer14_attention_weight_max": 40.580970764160156,
      "activations/layer14_attention_weight_min": -36.29670333862305,
      "activations/layer15_attention_weight_max": 38.37189865112305,
      "activations/layer15_attention_weight_min": -34.86655807495117,
      "activations/layer16_attention_weight_max": 29.443729400634766,
      "activations/layer16_attention_weight_min": -28.11513328552246,
      "activations/layer17_attention_weight_max": 33.67816925048828,
      "activations/layer17_attention_weight_min": -25.706945419311523,
      "activations/layer18_attention_weight_max": 32.60798263549805,
      "activations/layer18_attention_weight_min": -24.249767303466797,
      "activations/layer19_attention_weight_max": 35.70464324951172,
      "activations/layer19_attention_weight_min": -30.09119987487793,
      "activations/layer1_attention_weight_max": 17.404478073120117,
      "activations/layer1_attention_weight_min": -15.085365295410156,
      "activations/layer20_attention_weight_max": 28.956119537353516,
      "activations/layer20_attention_weight_min": -23.736194610595703,
      "activations/layer21_attention_weight_max": 27.600433349609375,
      "activations/layer21_attention_weight_min": -23.677261352539062,
      "activations/layer22_attention_weight_max": 40.42435836791992,
      "activations/layer22_attention_weight_min": -30.754934310913086,
      "activations/layer23_attention_weight_max": 33.61235809326172,
      "activations/layer23_attention_weight_min": -23.468564987182617,
      "activations/layer2_attention_weight_max": 34.797508239746094,
      "activations/layer2_attention_weight_min": -32.91947937011719,
      "activations/layer3_attention_weight_max": 92.85598754882812,
      "activations/layer3_attention_weight_min": -98.97642517089844,
      "activations/layer4_attention_weight_max": 103.57767486572266,
      "activations/layer4_attention_weight_min": -105.1698226928711,
      "activations/layer5_attention_weight_max": 49.34798812866211,
      "activations/layer5_attention_weight_min": -61.0325927734375,
      "activations/layer6_attention_weight_max": 43.31539535522461,
      "activations/layer6_attention_weight_min": -46.5757942199707,
      "activations/layer7_attention_weight_max": 90.99432373046875,
      "activations/layer7_attention_weight_min": -104.59589385986328,
      "activations/layer8_attention_weight_max": 44.2127685546875,
      "activations/layer8_attention_weight_min": -44.408790588378906,
      "activations/layer9_attention_weight_max": 33.64035415649414,
      "activations/layer9_attention_weight_min": -35.47829055786133,
      "epoch": 16.68,
      "learning_rate": 4.285227272727273e-05,
      "loss": 2.7459,
      "step": 287100
    },
    {
      "activations/layer0_attention_weight_max": 15.771897315979004,
      "activations/layer0_attention_weight_min": -12.915154457092285,
      "activations/layer10_attention_weight_max": 34.21058654785156,
      "activations/layer10_attention_weight_min": -36.05949401855469,
      "activations/layer11_attention_weight_max": 37.155982971191406,
      "activations/layer11_attention_weight_min": -36.44426727294922,
      "activations/layer12_attention_weight_max": 25.757747650146484,
      "activations/layer12_attention_weight_min": -27.116979598999023,
      "activations/layer13_attention_weight_max": 37.98708724975586,
      "activations/layer13_attention_weight_min": -35.379302978515625,
      "activations/layer14_attention_weight_max": 43.66460418701172,
      "activations/layer14_attention_weight_min": -38.600921630859375,
      "activations/layer15_attention_weight_max": 40.271114349365234,
      "activations/layer15_attention_weight_min": -36.31707000732422,
      "activations/layer16_attention_weight_max": 30.37023162841797,
      "activations/layer16_attention_weight_min": -28.287065505981445,
      "activations/layer17_attention_weight_max": 29.461849212646484,
      "activations/layer17_attention_weight_min": -25.981557846069336,
      "activations/layer18_attention_weight_max": 31.583322525024414,
      "activations/layer18_attention_weight_min": -26.423391342163086,
      "activations/layer19_attention_weight_max": 37.42227554321289,
      "activations/layer19_attention_weight_min": -32.84379959106445,
      "activations/layer1_attention_weight_max": 15.891542434692383,
      "activations/layer1_attention_weight_min": -16.034076690673828,
      "activations/layer20_attention_weight_max": 28.3503475189209,
      "activations/layer20_attention_weight_min": -26.159500122070312,
      "activations/layer21_attention_weight_max": 27.821903228759766,
      "activations/layer21_attention_weight_min": -25.85809326171875,
      "activations/layer22_attention_weight_max": 38.484378814697266,
      "activations/layer22_attention_weight_min": -33.74761962890625,
      "activations/layer23_attention_weight_max": 31.27028465270996,
      "activations/layer23_attention_weight_min": -27.20216941833496,
      "activations/layer2_attention_weight_max": 34.619049072265625,
      "activations/layer2_attention_weight_min": -31.853675842285156,
      "activations/layer3_attention_weight_max": 97.41522216796875,
      "activations/layer3_attention_weight_min": -95.03553771972656,
      "activations/layer4_attention_weight_max": 111.75882720947266,
      "activations/layer4_attention_weight_min": -103.6983642578125,
      "activations/layer5_attention_weight_max": 52.45986557006836,
      "activations/layer5_attention_weight_min": -59.998321533203125,
      "activations/layer6_attention_weight_max": 43.260520935058594,
      "activations/layer6_attention_weight_min": -45.050899505615234,
      "activations/layer7_attention_weight_max": 90.8189468383789,
      "activations/layer7_attention_weight_min": -100.91462707519531,
      "activations/layer8_attention_weight_max": 41.63453674316406,
      "activations/layer8_attention_weight_min": -42.246280670166016,
      "activations/layer9_attention_weight_max": 31.8277530670166,
      "activations/layer9_attention_weight_min": -34.471641540527344,
      "epoch": 16.69,
      "learning_rate": 4.283333333333333e-05,
      "loss": 2.7342,
      "step": 287150
    },
    {
      "activations/layer0_attention_weight_max": 15.660989761352539,
      "activations/layer0_attention_weight_min": -12.419578552246094,
      "activations/layer10_attention_weight_max": 31.8588924407959,
      "activations/layer10_attention_weight_min": -32.521427154541016,
      "activations/layer11_attention_weight_max": 33.118988037109375,
      "activations/layer11_attention_weight_min": -33.43335723876953,
      "activations/layer12_attention_weight_max": 26.244369506835938,
      "activations/layer12_attention_weight_min": -26.904760360717773,
      "activations/layer13_attention_weight_max": 38.819580078125,
      "activations/layer13_attention_weight_min": -37.37067794799805,
      "activations/layer14_attention_weight_max": 43.417755126953125,
      "activations/layer14_attention_weight_min": -37.34644317626953,
      "activations/layer15_attention_weight_max": 36.77167892456055,
      "activations/layer15_attention_weight_min": -36.65896224975586,
      "activations/layer16_attention_weight_max": 29.732295989990234,
      "activations/layer16_attention_weight_min": -27.62816619873047,
      "activations/layer17_attention_weight_max": 29.329044342041016,
      "activations/layer17_attention_weight_min": -24.607255935668945,
      "activations/layer18_attention_weight_max": 31.71259117126465,
      "activations/layer18_attention_weight_min": -23.761615753173828,
      "activations/layer19_attention_weight_max": 35.61439895629883,
      "activations/layer19_attention_weight_min": -30.503026962280273,
      "activations/layer1_attention_weight_max": 16.22882080078125,
      "activations/layer1_attention_weight_min": -14.063966751098633,
      "activations/layer20_attention_weight_max": 29.854101181030273,
      "activations/layer20_attention_weight_min": -25.98335838317871,
      "activations/layer21_attention_weight_max": 26.758464813232422,
      "activations/layer21_attention_weight_min": -24.05662727355957,
      "activations/layer22_attention_weight_max": 40.40917205810547,
      "activations/layer22_attention_weight_min": -32.009979248046875,
      "activations/layer23_attention_weight_max": 34.8184928894043,
      "activations/layer23_attention_weight_min": -24.108905792236328,
      "activations/layer2_attention_weight_max": 31.555217742919922,
      "activations/layer2_attention_weight_min": -30.91583824157715,
      "activations/layer3_attention_weight_max": 94.12689971923828,
      "activations/layer3_attention_weight_min": -94.13836669921875,
      "activations/layer4_attention_weight_max": 104.03211975097656,
      "activations/layer4_attention_weight_min": -107.45812225341797,
      "activations/layer5_attention_weight_max": 49.87158966064453,
      "activations/layer5_attention_weight_min": -59.63590621948242,
      "activations/layer6_attention_weight_max": 43.56340408325195,
      "activations/layer6_attention_weight_min": -44.66049575805664,
      "activations/layer7_attention_weight_max": 91.32989501953125,
      "activations/layer7_attention_weight_min": -97.83130645751953,
      "activations/layer8_attention_weight_max": 40.63136291503906,
      "activations/layer8_attention_weight_min": -42.837059020996094,
      "activations/layer9_attention_weight_max": 31.4799861907959,
      "activations/layer9_attention_weight_min": -32.567081451416016,
      "epoch": 16.69,
      "learning_rate": 4.281439393939394e-05,
      "loss": 2.7296,
      "step": 287200
    },
    {
      "activations/layer0_attention_weight_max": 15.17554759979248,
      "activations/layer0_attention_weight_min": -13.65422534942627,
      "activations/layer10_attention_weight_max": 36.17511749267578,
      "activations/layer10_attention_weight_min": -36.51487731933594,
      "activations/layer11_attention_weight_max": 36.542503356933594,
      "activations/layer11_attention_weight_min": -37.54233169555664,
      "activations/layer12_attention_weight_max": 29.344898223876953,
      "activations/layer12_attention_weight_min": -28.18744468688965,
      "activations/layer13_attention_weight_max": 41.59248352050781,
      "activations/layer13_attention_weight_min": -36.740806579589844,
      "activations/layer14_attention_weight_max": 45.27682876586914,
      "activations/layer14_attention_weight_min": -39.936134338378906,
      "activations/layer15_attention_weight_max": 40.402549743652344,
      "activations/layer15_attention_weight_min": -39.031089782714844,
      "activations/layer16_attention_weight_max": 31.571210861206055,
      "activations/layer16_attention_weight_min": -32.81440353393555,
      "activations/layer17_attention_weight_max": 33.59682846069336,
      "activations/layer17_attention_weight_min": -31.378705978393555,
      "activations/layer18_attention_weight_max": 30.855134963989258,
      "activations/layer18_attention_weight_min": -27.27482795715332,
      "activations/layer19_attention_weight_max": 36.98502731323242,
      "activations/layer19_attention_weight_min": -30.970565795898438,
      "activations/layer1_attention_weight_max": 15.439034461975098,
      "activations/layer1_attention_weight_min": -13.711216926574707,
      "activations/layer20_attention_weight_max": 30.0113525390625,
      "activations/layer20_attention_weight_min": -25.617830276489258,
      "activations/layer21_attention_weight_max": 28.26567268371582,
      "activations/layer21_attention_weight_min": -25.30702781677246,
      "activations/layer22_attention_weight_max": 42.29102325439453,
      "activations/layer22_attention_weight_min": -34.01469039916992,
      "activations/layer23_attention_weight_max": 30.412128448486328,
      "activations/layer23_attention_weight_min": -26.289886474609375,
      "activations/layer2_attention_weight_max": 32.347633361816406,
      "activations/layer2_attention_weight_min": -31.83614158630371,
      "activations/layer3_attention_weight_max": 96.22176361083984,
      "activations/layer3_attention_weight_min": -98.95994567871094,
      "activations/layer4_attention_weight_max": 108.55255126953125,
      "activations/layer4_attention_weight_min": -106.94856262207031,
      "activations/layer5_attention_weight_max": 53.3650016784668,
      "activations/layer5_attention_weight_min": -62.26430892944336,
      "activations/layer6_attention_weight_max": 45.95223617553711,
      "activations/layer6_attention_weight_min": -47.973941802978516,
      "activations/layer7_attention_weight_max": 97.94779205322266,
      "activations/layer7_attention_weight_min": -97.60820770263672,
      "activations/layer8_attention_weight_max": 45.211177825927734,
      "activations/layer8_attention_weight_min": -46.49726486206055,
      "activations/layer9_attention_weight_max": 36.158905029296875,
      "activations/layer9_attention_weight_min": -35.114681243896484,
      "epoch": 16.69,
      "learning_rate": 4.279545454545454e-05,
      "loss": 2.736,
      "step": 287250
    },
    {
      "activations/layer0_attention_weight_max": 16.26173210144043,
      "activations/layer0_attention_weight_min": -13.225349426269531,
      "activations/layer10_attention_weight_max": 33.9996223449707,
      "activations/layer10_attention_weight_min": -33.86956024169922,
      "activations/layer11_attention_weight_max": 35.06867218017578,
      "activations/layer11_attention_weight_min": -35.34953308105469,
      "activations/layer12_attention_weight_max": 27.249841690063477,
      "activations/layer12_attention_weight_min": -27.564428329467773,
      "activations/layer13_attention_weight_max": 42.12914276123047,
      "activations/layer13_attention_weight_min": -37.67515563964844,
      "activations/layer14_attention_weight_max": 44.54887008666992,
      "activations/layer14_attention_weight_min": -40.36757278442383,
      "activations/layer15_attention_weight_max": 38.84901809692383,
      "activations/layer15_attention_weight_min": -36.67341232299805,
      "activations/layer16_attention_weight_max": 30.837635040283203,
      "activations/layer16_attention_weight_min": -29.12775993347168,
      "activations/layer17_attention_weight_max": 28.641021728515625,
      "activations/layer17_attention_weight_min": -25.972997665405273,
      "activations/layer18_attention_weight_max": 29.946308135986328,
      "activations/layer18_attention_weight_min": -26.86273765563965,
      "activations/layer19_attention_weight_max": 32.49778366088867,
      "activations/layer19_attention_weight_min": -31.5992374420166,
      "activations/layer1_attention_weight_max": 16.66059684753418,
      "activations/layer1_attention_weight_min": -15.414114952087402,
      "activations/layer20_attention_weight_max": 29.12325668334961,
      "activations/layer20_attention_weight_min": -25.09502601623535,
      "activations/layer21_attention_weight_max": 28.776058197021484,
      "activations/layer21_attention_weight_min": -25.153732299804688,
      "activations/layer22_attention_weight_max": 43.7504768371582,
      "activations/layer22_attention_weight_min": -31.923486709594727,
      "activations/layer23_attention_weight_max": 31.095333099365234,
      "activations/layer23_attention_weight_min": -27.450895309448242,
      "activations/layer2_attention_weight_max": 31.59248924255371,
      "activations/layer2_attention_weight_min": -30.91156005859375,
      "activations/layer3_attention_weight_max": 97.57791137695312,
      "activations/layer3_attention_weight_min": -98.34869384765625,
      "activations/layer4_attention_weight_max": 105.23457336425781,
      "activations/layer4_attention_weight_min": -105.1511001586914,
      "activations/layer5_attention_weight_max": 48.736717224121094,
      "activations/layer5_attention_weight_min": -63.75172424316406,
      "activations/layer6_attention_weight_max": 44.18673324584961,
      "activations/layer6_attention_weight_min": -45.1701774597168,
      "activations/layer7_attention_weight_max": 96.57145690917969,
      "activations/layer7_attention_weight_min": -93.27689361572266,
      "activations/layer8_attention_weight_max": 43.227561950683594,
      "activations/layer8_attention_weight_min": -43.895416259765625,
      "activations/layer9_attention_weight_max": 33.72695541381836,
      "activations/layer9_attention_weight_min": -36.07034683227539,
      "epoch": 16.69,
      "learning_rate": 4.2776515151515153e-05,
      "loss": 2.7299,
      "step": 287300
    },
    {
      "activations/layer0_attention_weight_max": 16.92742347717285,
      "activations/layer0_attention_weight_min": -12.7103853225708,
      "activations/layer10_attention_weight_max": 35.5794677734375,
      "activations/layer10_attention_weight_min": -35.86981201171875,
      "activations/layer11_attention_weight_max": 34.86247253417969,
      "activations/layer11_attention_weight_min": -37.04106903076172,
      "activations/layer12_attention_weight_max": 24.647354125976562,
      "activations/layer12_attention_weight_min": -30.723817825317383,
      "activations/layer13_attention_weight_max": 36.44997787475586,
      "activations/layer13_attention_weight_min": -36.59302520751953,
      "activations/layer14_attention_weight_max": 43.554527282714844,
      "activations/layer14_attention_weight_min": -39.18656921386719,
      "activations/layer15_attention_weight_max": 35.91975784301758,
      "activations/layer15_attention_weight_min": -36.82170867919922,
      "activations/layer16_attention_weight_max": 27.689294815063477,
      "activations/layer16_attention_weight_min": -28.372709274291992,
      "activations/layer17_attention_weight_max": 29.266157150268555,
      "activations/layer17_attention_weight_min": -26.55742073059082,
      "activations/layer18_attention_weight_max": 29.132463455200195,
      "activations/layer18_attention_weight_min": -25.582487106323242,
      "activations/layer19_attention_weight_max": 32.23017120361328,
      "activations/layer19_attention_weight_min": -31.06337547302246,
      "activations/layer1_attention_weight_max": 17.38774299621582,
      "activations/layer1_attention_weight_min": -15.525660514831543,
      "activations/layer20_attention_weight_max": 29.297468185424805,
      "activations/layer20_attention_weight_min": -24.96569061279297,
      "activations/layer21_attention_weight_max": 28.469085693359375,
      "activations/layer21_attention_weight_min": -23.683658599853516,
      "activations/layer22_attention_weight_max": 39.2829704284668,
      "activations/layer22_attention_weight_min": -32.760398864746094,
      "activations/layer23_attention_weight_max": 29.856056213378906,
      "activations/layer23_attention_weight_min": -25.438838958740234,
      "activations/layer2_attention_weight_max": 34.64594650268555,
      "activations/layer2_attention_weight_min": -32.68889236450195,
      "activations/layer3_attention_weight_max": 96.22988891601562,
      "activations/layer3_attention_weight_min": -97.83877563476562,
      "activations/layer4_attention_weight_max": 107.74449920654297,
      "activations/layer4_attention_weight_min": -106.32402801513672,
      "activations/layer5_attention_weight_max": 52.195899963378906,
      "activations/layer5_attention_weight_min": -57.79450225830078,
      "activations/layer6_attention_weight_max": 42.59884262084961,
      "activations/layer6_attention_weight_min": -45.53105926513672,
      "activations/layer7_attention_weight_max": 85.68682098388672,
      "activations/layer7_attention_weight_min": -90.54740905761719,
      "activations/layer8_attention_weight_max": 41.15351486206055,
      "activations/layer8_attention_weight_min": -44.66838836669922,
      "activations/layer9_attention_weight_max": 32.95458221435547,
      "activations/layer9_attention_weight_min": -34.50069046020508,
      "epoch": 16.7,
      "learning_rate": 4.2757575757575755e-05,
      "loss": 2.7434,
      "step": 287350
    },
    {
      "activations/layer0_attention_weight_max": 14.908767700195312,
      "activations/layer0_attention_weight_min": -12.828154563903809,
      "activations/layer10_attention_weight_max": 32.8774299621582,
      "activations/layer10_attention_weight_min": -35.98112869262695,
      "activations/layer11_attention_weight_max": 34.12678146362305,
      "activations/layer11_attention_weight_min": -34.91902542114258,
      "activations/layer12_attention_weight_max": 26.78820037841797,
      "activations/layer12_attention_weight_min": -27.405473709106445,
      "activations/layer13_attention_weight_max": 39.82655715942383,
      "activations/layer13_attention_weight_min": -35.15583419799805,
      "activations/layer14_attention_weight_max": 41.306732177734375,
      "activations/layer14_attention_weight_min": -39.80492401123047,
      "activations/layer15_attention_weight_max": 38.938350677490234,
      "activations/layer15_attention_weight_min": -34.39724349975586,
      "activations/layer16_attention_weight_max": 28.784448623657227,
      "activations/layer16_attention_weight_min": -27.86823081970215,
      "activations/layer17_attention_weight_max": 30.12960433959961,
      "activations/layer17_attention_weight_min": -25.18849754333496,
      "activations/layer18_attention_weight_max": 34.51327133178711,
      "activations/layer18_attention_weight_min": -25.041399002075195,
      "activations/layer19_attention_weight_max": 32.30367660522461,
      "activations/layer19_attention_weight_min": -30.91788673400879,
      "activations/layer1_attention_weight_max": 16.20996856689453,
      "activations/layer1_attention_weight_min": -14.949418067932129,
      "activations/layer20_attention_weight_max": 27.19017791748047,
      "activations/layer20_attention_weight_min": -25.40765380859375,
      "activations/layer21_attention_weight_max": 27.453283309936523,
      "activations/layer21_attention_weight_min": -23.954294204711914,
      "activations/layer22_attention_weight_max": 41.616172790527344,
      "activations/layer22_attention_weight_min": -33.47282028198242,
      "activations/layer23_attention_weight_max": 33.00983428955078,
      "activations/layer23_attention_weight_min": -24.554908752441406,
      "activations/layer2_attention_weight_max": 33.62883758544922,
      "activations/layer2_attention_weight_min": -30.918167114257812,
      "activations/layer3_attention_weight_max": 98.20872497558594,
      "activations/layer3_attention_weight_min": -94.5528564453125,
      "activations/layer4_attention_weight_max": 106.6212158203125,
      "activations/layer4_attention_weight_min": -107.66043853759766,
      "activations/layer5_attention_weight_max": 50.1731071472168,
      "activations/layer5_attention_weight_min": -61.22040557861328,
      "activations/layer6_attention_weight_max": 43.1105842590332,
      "activations/layer6_attention_weight_min": -45.047359466552734,
      "activations/layer7_attention_weight_max": 91.00175476074219,
      "activations/layer7_attention_weight_min": -97.08155059814453,
      "activations/layer8_attention_weight_max": 41.25429916381836,
      "activations/layer8_attention_weight_min": -42.85039138793945,
      "activations/layer9_attention_weight_max": 32.3834342956543,
      "activations/layer9_attention_weight_min": -33.14493179321289,
      "epoch": 16.7,
      "learning_rate": 4.2738636363636356e-05,
      "loss": 2.7246,
      "step": 287400
    },
    {
      "activations/layer0_attention_weight_max": 15.513288497924805,
      "activations/layer0_attention_weight_min": -13.413558959960938,
      "activations/layer10_attention_weight_max": 37.36337661743164,
      "activations/layer10_attention_weight_min": -34.75484848022461,
      "activations/layer11_attention_weight_max": 36.366790771484375,
      "activations/layer11_attention_weight_min": -35.59348678588867,
      "activations/layer12_attention_weight_max": 27.300682067871094,
      "activations/layer12_attention_weight_min": -28.03473472595215,
      "activations/layer13_attention_weight_max": 44.499305725097656,
      "activations/layer13_attention_weight_min": -39.51262283325195,
      "activations/layer14_attention_weight_max": 50.890865325927734,
      "activations/layer14_attention_weight_min": -45.21124267578125,
      "activations/layer15_attention_weight_max": 43.73358917236328,
      "activations/layer15_attention_weight_min": -39.739234924316406,
      "activations/layer16_attention_weight_max": 33.79874801635742,
      "activations/layer16_attention_weight_min": -29.260921478271484,
      "activations/layer17_attention_weight_max": 33.899993896484375,
      "activations/layer17_attention_weight_min": -27.858051300048828,
      "activations/layer18_attention_weight_max": 33.461509704589844,
      "activations/layer18_attention_weight_min": -27.46090316772461,
      "activations/layer19_attention_weight_max": 44.41708755493164,
      "activations/layer19_attention_weight_min": -38.14683151245117,
      "activations/layer1_attention_weight_max": 16.67261505126953,
      "activations/layer1_attention_weight_min": -14.505657196044922,
      "activations/layer20_attention_weight_max": 35.075199127197266,
      "activations/layer20_attention_weight_min": -27.178808212280273,
      "activations/layer21_attention_weight_max": 30.351383209228516,
      "activations/layer21_attention_weight_min": -25.06389808654785,
      "activations/layer22_attention_weight_max": 56.0406379699707,
      "activations/layer22_attention_weight_min": -33.62404251098633,
      "activations/layer23_attention_weight_max": 36.45454406738281,
      "activations/layer23_attention_weight_min": -25.53396224975586,
      "activations/layer2_attention_weight_max": 33.851829528808594,
      "activations/layer2_attention_weight_min": -33.104034423828125,
      "activations/layer3_attention_weight_max": 101.74469757080078,
      "activations/layer3_attention_weight_min": -102.67091369628906,
      "activations/layer4_attention_weight_max": 107.5852279663086,
      "activations/layer4_attention_weight_min": -108.15093994140625,
      "activations/layer5_attention_weight_max": 53.278785705566406,
      "activations/layer5_attention_weight_min": -61.185096740722656,
      "activations/layer6_attention_weight_max": 43.30590057373047,
      "activations/layer6_attention_weight_min": -47.5457649230957,
      "activations/layer7_attention_weight_max": 96.3070297241211,
      "activations/layer7_attention_weight_min": -99.23303985595703,
      "activations/layer8_attention_weight_max": 43.32191467285156,
      "activations/layer8_attention_weight_min": -45.60598373413086,
      "activations/layer9_attention_weight_max": 34.84174346923828,
      "activations/layer9_attention_weight_min": -35.282989501953125,
      "epoch": 16.7,
      "learning_rate": 4.2719696969696965e-05,
      "loss": 2.7272,
      "step": 287450
    },
    {
      "activations/layer0_attention_weight_max": 15.139138221740723,
      "activations/layer0_attention_weight_min": -13.396328926086426,
      "activations/layer10_attention_weight_max": 37.50679016113281,
      "activations/layer10_attention_weight_min": -36.7754020690918,
      "activations/layer11_attention_weight_max": 35.28547286987305,
      "activations/layer11_attention_weight_min": -37.461326599121094,
      "activations/layer12_attention_weight_max": 26.75501823425293,
      "activations/layer12_attention_weight_min": -27.297901153564453,
      "activations/layer13_attention_weight_max": 40.07130813598633,
      "activations/layer13_attention_weight_min": -35.769344329833984,
      "activations/layer14_attention_weight_max": 45.966033935546875,
      "activations/layer14_attention_weight_min": -37.64802932739258,
      "activations/layer15_attention_weight_max": 38.922794342041016,
      "activations/layer15_attention_weight_min": -36.94564437866211,
      "activations/layer16_attention_weight_max": 31.026485443115234,
      "activations/layer16_attention_weight_min": -29.063814163208008,
      "activations/layer17_attention_weight_max": 29.251062393188477,
      "activations/layer17_attention_weight_min": -26.921449661254883,
      "activations/layer18_attention_weight_max": 31.49110221862793,
      "activations/layer18_attention_weight_min": -26.117998123168945,
      "activations/layer19_attention_weight_max": 35.53791427612305,
      "activations/layer19_attention_weight_min": -31.825292587280273,
      "activations/layer1_attention_weight_max": 15.277315139770508,
      "activations/layer1_attention_weight_min": -14.89321231842041,
      "activations/layer20_attention_weight_max": 30.004764556884766,
      "activations/layer20_attention_weight_min": -25.2987003326416,
      "activations/layer21_attention_weight_max": 30.072111129760742,
      "activations/layer21_attention_weight_min": -24.062034606933594,
      "activations/layer22_attention_weight_max": 45.58592224121094,
      "activations/layer22_attention_weight_min": -32.70759582519531,
      "activations/layer23_attention_weight_max": 35.56734848022461,
      "activations/layer23_attention_weight_min": -26.107091903686523,
      "activations/layer2_attention_weight_max": 32.21361541748047,
      "activations/layer2_attention_weight_min": -32.372920989990234,
      "activations/layer3_attention_weight_max": 96.23322296142578,
      "activations/layer3_attention_weight_min": -95.40684509277344,
      "activations/layer4_attention_weight_max": 107.484375,
      "activations/layer4_attention_weight_min": -108.0012435913086,
      "activations/layer5_attention_weight_max": 49.17756652832031,
      "activations/layer5_attention_weight_min": -61.00666809082031,
      "activations/layer6_attention_weight_max": 43.68097686767578,
      "activations/layer6_attention_weight_min": -44.56589889526367,
      "activations/layer7_attention_weight_max": 90.88638305664062,
      "activations/layer7_attention_weight_min": -91.25745391845703,
      "activations/layer8_attention_weight_max": 40.537803649902344,
      "activations/layer8_attention_weight_min": -42.232608795166016,
      "activations/layer9_attention_weight_max": 32.29854965209961,
      "activations/layer9_attention_weight_min": -34.64295959472656,
      "epoch": 16.71,
      "learning_rate": 4.270075757575757e-05,
      "loss": 2.7371,
      "step": 287500
    },
    {
      "activations/layer0_attention_weight_max": 15.97744083404541,
      "activations/layer0_attention_weight_min": -12.524115562438965,
      "activations/layer10_attention_weight_max": 32.888336181640625,
      "activations/layer10_attention_weight_min": -35.22737503051758,
      "activations/layer11_attention_weight_max": 31.21595573425293,
      "activations/layer11_attention_weight_min": -33.80919647216797,
      "activations/layer12_attention_weight_max": 24.02158546447754,
      "activations/layer12_attention_weight_min": -26.27041244506836,
      "activations/layer13_attention_weight_max": 36.96928787231445,
      "activations/layer13_attention_weight_min": -34.38072204589844,
      "activations/layer14_attention_weight_max": 41.295719146728516,
      "activations/layer14_attention_weight_min": -36.35002136230469,
      "activations/layer15_attention_weight_max": 34.81333541870117,
      "activations/layer15_attention_weight_min": -33.789390563964844,
      "activations/layer16_attention_weight_max": 27.538249969482422,
      "activations/layer16_attention_weight_min": -31.43564796447754,
      "activations/layer17_attention_weight_max": 27.898283004760742,
      "activations/layer17_attention_weight_min": -27.726171493530273,
      "activations/layer18_attention_weight_max": 32.65827178955078,
      "activations/layer18_attention_weight_min": -26.340042114257812,
      "activations/layer19_attention_weight_max": 30.676843643188477,
      "activations/layer19_attention_weight_min": -30.359317779541016,
      "activations/layer1_attention_weight_max": 15.746929168701172,
      "activations/layer1_attention_weight_min": -14.763439178466797,
      "activations/layer20_attention_weight_max": 26.642194747924805,
      "activations/layer20_attention_weight_min": -25.117042541503906,
      "activations/layer21_attention_weight_max": 27.261777877807617,
      "activations/layer21_attention_weight_min": -24.89898109436035,
      "activations/layer22_attention_weight_max": 39.54865646362305,
      "activations/layer22_attention_weight_min": -30.63669776916504,
      "activations/layer23_attention_weight_max": 30.353851318359375,
      "activations/layer23_attention_weight_min": -26.084936141967773,
      "activations/layer2_attention_weight_max": 30.842750549316406,
      "activations/layer2_attention_weight_min": -31.644695281982422,
      "activations/layer3_attention_weight_max": 92.42432403564453,
      "activations/layer3_attention_weight_min": -95.10038757324219,
      "activations/layer4_attention_weight_max": 104.6030044555664,
      "activations/layer4_attention_weight_min": -106.6661605834961,
      "activations/layer5_attention_weight_max": 47.15666198730469,
      "activations/layer5_attention_weight_min": -60.632503509521484,
      "activations/layer6_attention_weight_max": 41.27178955078125,
      "activations/layer6_attention_weight_min": -45.707679748535156,
      "activations/layer7_attention_weight_max": 91.74864196777344,
      "activations/layer7_attention_weight_min": -94.66918182373047,
      "activations/layer8_attention_weight_max": 39.60211181640625,
      "activations/layer8_attention_weight_min": -43.175262451171875,
      "activations/layer9_attention_weight_max": 30.64505958557129,
      "activations/layer9_attention_weight_min": -34.867984771728516,
      "epoch": 16.71,
      "learning_rate": 4.268181818181818e-05,
      "loss": 2.7392,
      "step": 287550
    },
    {
      "activations/layer0_attention_weight_max": 15.939699172973633,
      "activations/layer0_attention_weight_min": -12.194682121276855,
      "activations/layer10_attention_weight_max": 34.85620880126953,
      "activations/layer10_attention_weight_min": -35.2725830078125,
      "activations/layer11_attention_weight_max": 34.06627655029297,
      "activations/layer11_attention_weight_min": -34.39725875854492,
      "activations/layer12_attention_weight_max": 26.872583389282227,
      "activations/layer12_attention_weight_min": -26.708789825439453,
      "activations/layer13_attention_weight_max": 40.027610778808594,
      "activations/layer13_attention_weight_min": -36.10121154785156,
      "activations/layer14_attention_weight_max": 41.7808952331543,
      "activations/layer14_attention_weight_min": -37.81907653808594,
      "activations/layer15_attention_weight_max": 42.12458038330078,
      "activations/layer15_attention_weight_min": -34.94624710083008,
      "activations/layer16_attention_weight_max": 29.83916664123535,
      "activations/layer16_attention_weight_min": -29.118860244750977,
      "activations/layer17_attention_weight_max": 29.472110748291016,
      "activations/layer17_attention_weight_min": -26.085969924926758,
      "activations/layer18_attention_weight_max": 30.255220413208008,
      "activations/layer18_attention_weight_min": -25.782270431518555,
      "activations/layer19_attention_weight_max": 32.82822799682617,
      "activations/layer19_attention_weight_min": -31.81336784362793,
      "activations/layer1_attention_weight_max": 15.606154441833496,
      "activations/layer1_attention_weight_min": -14.64625072479248,
      "activations/layer20_attention_weight_max": 29.928144454956055,
      "activations/layer20_attention_weight_min": -26.65459442138672,
      "activations/layer21_attention_weight_max": 31.70100975036621,
      "activations/layer21_attention_weight_min": -27.323678970336914,
      "activations/layer22_attention_weight_max": 47.253684997558594,
      "activations/layer22_attention_weight_min": -33.07419967651367,
      "activations/layer23_attention_weight_max": 34.47038650512695,
      "activations/layer23_attention_weight_min": -25.951587677001953,
      "activations/layer2_attention_weight_max": 32.6204948425293,
      "activations/layer2_attention_weight_min": -33.29821014404297,
      "activations/layer3_attention_weight_max": 97.56497192382812,
      "activations/layer3_attention_weight_min": -97.56822204589844,
      "activations/layer4_attention_weight_max": 107.72757720947266,
      "activations/layer4_attention_weight_min": -108.93680572509766,
      "activations/layer5_attention_weight_max": 53.03691864013672,
      "activations/layer5_attention_weight_min": -60.30634689331055,
      "activations/layer6_attention_weight_max": 43.33848571777344,
      "activations/layer6_attention_weight_min": -46.13978958129883,
      "activations/layer7_attention_weight_max": 90.97393035888672,
      "activations/layer7_attention_weight_min": -95.40545654296875,
      "activations/layer8_attention_weight_max": 40.67536163330078,
      "activations/layer8_attention_weight_min": -45.93033981323242,
      "activations/layer9_attention_weight_max": 33.52048873901367,
      "activations/layer9_attention_weight_min": -35.18709182739258,
      "epoch": 16.71,
      "learning_rate": 4.266287878787878e-05,
      "loss": 2.736,
      "step": 287600
    },
    {
      "activations/layer0_attention_weight_max": 15.64298152923584,
      "activations/layer0_attention_weight_min": -12.948800086975098,
      "activations/layer10_attention_weight_max": 32.01490020751953,
      "activations/layer10_attention_weight_min": -32.96260452270508,
      "activations/layer11_attention_weight_max": 31.881866455078125,
      "activations/layer11_attention_weight_min": -34.10834503173828,
      "activations/layer12_attention_weight_max": 25.588214874267578,
      "activations/layer12_attention_weight_min": -26.38223648071289,
      "activations/layer13_attention_weight_max": 36.76862335205078,
      "activations/layer13_attention_weight_min": -36.41135025024414,
      "activations/layer14_attention_weight_max": 42.33768081665039,
      "activations/layer14_attention_weight_min": -38.43515396118164,
      "activations/layer15_attention_weight_max": 38.0594596862793,
      "activations/layer15_attention_weight_min": -36.64509201049805,
      "activations/layer16_attention_weight_max": 28.803951263427734,
      "activations/layer16_attention_weight_min": -29.30972671508789,
      "activations/layer17_attention_weight_max": 30.574440002441406,
      "activations/layer17_attention_weight_min": -28.70069694519043,
      "activations/layer18_attention_weight_max": 31.416765213012695,
      "activations/layer18_attention_weight_min": -30.587800979614258,
      "activations/layer19_attention_weight_max": 34.172325134277344,
      "activations/layer19_attention_weight_min": -33.755218505859375,
      "activations/layer1_attention_weight_max": 16.213350296020508,
      "activations/layer1_attention_weight_min": -16.398441314697266,
      "activations/layer20_attention_weight_max": 30.298870086669922,
      "activations/layer20_attention_weight_min": -28.18358039855957,
      "activations/layer21_attention_weight_max": 30.57888412475586,
      "activations/layer21_attention_weight_min": -28.274288177490234,
      "activations/layer22_attention_weight_max": 41.74225616455078,
      "activations/layer22_attention_weight_min": -35.77790451049805,
      "activations/layer23_attention_weight_max": 32.38275146484375,
      "activations/layer23_attention_weight_min": -29.01483917236328,
      "activations/layer2_attention_weight_max": 31.47976303100586,
      "activations/layer2_attention_weight_min": -30.671857833862305,
      "activations/layer3_attention_weight_max": 91.76758575439453,
      "activations/layer3_attention_weight_min": -93.61217498779297,
      "activations/layer4_attention_weight_max": 101.24602508544922,
      "activations/layer4_attention_weight_min": -102.09347534179688,
      "activations/layer5_attention_weight_max": 51.62602233886719,
      "activations/layer5_attention_weight_min": -59.199615478515625,
      "activations/layer6_attention_weight_max": 41.93357467651367,
      "activations/layer6_attention_weight_min": -44.22669219970703,
      "activations/layer7_attention_weight_max": 90.01914978027344,
      "activations/layer7_attention_weight_min": -92.02677917480469,
      "activations/layer8_attention_weight_max": 38.63276290893555,
      "activations/layer8_attention_weight_min": -43.8765869140625,
      "activations/layer9_attention_weight_max": 31.136667251586914,
      "activations/layer9_attention_weight_min": -34.206825256347656,
      "epoch": 16.71,
      "learning_rate": 4.2644318181818174e-05,
      "loss": 2.7461,
      "step": 287650
    },
    {
      "activations/layer0_attention_weight_max": 17.263996124267578,
      "activations/layer0_attention_weight_min": -13.113574028015137,
      "activations/layer10_attention_weight_max": 35.607887268066406,
      "activations/layer10_attention_weight_min": -34.27534866333008,
      "activations/layer11_attention_weight_max": 34.34431457519531,
      "activations/layer11_attention_weight_min": -33.9019889831543,
      "activations/layer12_attention_weight_max": 28.945175170898438,
      "activations/layer12_attention_weight_min": -27.589820861816406,
      "activations/layer13_attention_weight_max": 42.19462966918945,
      "activations/layer13_attention_weight_min": -36.27580642700195,
      "activations/layer14_attention_weight_max": 44.97480392456055,
      "activations/layer14_attention_weight_min": -39.85453796386719,
      "activations/layer15_attention_weight_max": 41.583168029785156,
      "activations/layer15_attention_weight_min": -37.82530212402344,
      "activations/layer16_attention_weight_max": 30.050230026245117,
      "activations/layer16_attention_weight_min": -27.129655838012695,
      "activations/layer17_attention_weight_max": 31.713056564331055,
      "activations/layer17_attention_weight_min": -25.386789321899414,
      "activations/layer18_attention_weight_max": 32.36884689331055,
      "activations/layer18_attention_weight_min": -24.825618743896484,
      "activations/layer19_attention_weight_max": 35.09163284301758,
      "activations/layer19_attention_weight_min": -29.939102172851562,
      "activations/layer1_attention_weight_max": 16.046899795532227,
      "activations/layer1_attention_weight_min": -13.52151107788086,
      "activations/layer20_attention_weight_max": 31.82765769958496,
      "activations/layer20_attention_weight_min": -25.586729049682617,
      "activations/layer21_attention_weight_max": 33.46937942504883,
      "activations/layer21_attention_weight_min": -25.57671356201172,
      "activations/layer22_attention_weight_max": 41.401123046875,
      "activations/layer22_attention_weight_min": -32.74350357055664,
      "activations/layer23_attention_weight_max": 33.400245666503906,
      "activations/layer23_attention_weight_min": -27.96358299255371,
      "activations/layer2_attention_weight_max": 30.843595504760742,
      "activations/layer2_attention_weight_min": -28.646678924560547,
      "activations/layer3_attention_weight_max": 87.03529357910156,
      "activations/layer3_attention_weight_min": -89.42720794677734,
      "activations/layer4_attention_weight_max": 97.27540588378906,
      "activations/layer4_attention_weight_min": -98.98052215576172,
      "activations/layer5_attention_weight_max": 48.10892868041992,
      "activations/layer5_attention_weight_min": -57.555931091308594,
      "activations/layer6_attention_weight_max": 42.35238265991211,
      "activations/layer6_attention_weight_min": -43.401004791259766,
      "activations/layer7_attention_weight_max": 91.15776062011719,
      "activations/layer7_attention_weight_min": -90.5834732055664,
      "activations/layer8_attention_weight_max": 41.56888961791992,
      "activations/layer8_attention_weight_min": -42.083194732666016,
      "activations/layer9_attention_weight_max": 35.17582321166992,
      "activations/layer9_attention_weight_min": -34.714866638183594,
      "epoch": 16.72,
      "learning_rate": 4.262537878787879e-05,
      "loss": 2.7291,
      "step": 287700
    },
    {
      "activations/layer0_attention_weight_max": 15.565483093261719,
      "activations/layer0_attention_weight_min": -12.57686710357666,
      "activations/layer10_attention_weight_max": 38.121490478515625,
      "activations/layer10_attention_weight_min": -38.21820831298828,
      "activations/layer11_attention_weight_max": 38.30234909057617,
      "activations/layer11_attention_weight_min": -36.10542297363281,
      "activations/layer12_attention_weight_max": 32.39650344848633,
      "activations/layer12_attention_weight_min": -27.007020950317383,
      "activations/layer13_attention_weight_max": 41.161163330078125,
      "activations/layer13_attention_weight_min": -35.34941101074219,
      "activations/layer14_attention_weight_max": 42.48823928833008,
      "activations/layer14_attention_weight_min": -38.12919235229492,
      "activations/layer15_attention_weight_max": 41.504276275634766,
      "activations/layer15_attention_weight_min": -38.009578704833984,
      "activations/layer16_attention_weight_max": 31.042293548583984,
      "activations/layer16_attention_weight_min": -28.50577735900879,
      "activations/layer17_attention_weight_max": 32.70642852783203,
      "activations/layer17_attention_weight_min": -26.903470993041992,
      "activations/layer18_attention_weight_max": 35.19524383544922,
      "activations/layer18_attention_weight_min": -25.99156951904297,
      "activations/layer19_attention_weight_max": 39.18132781982422,
      "activations/layer19_attention_weight_min": -35.7136344909668,
      "activations/layer1_attention_weight_max": 17.210309982299805,
      "activations/layer1_attention_weight_min": -16.47893714904785,
      "activations/layer20_attention_weight_max": 33.63727951049805,
      "activations/layer20_attention_weight_min": -28.756580352783203,
      "activations/layer21_attention_weight_max": 31.43306541442871,
      "activations/layer21_attention_weight_min": -28.860876083374023,
      "activations/layer22_attention_weight_max": 45.12348175048828,
      "activations/layer22_attention_weight_min": -34.96583938598633,
      "activations/layer23_attention_weight_max": 34.735198974609375,
      "activations/layer23_attention_weight_min": -26.036298751831055,
      "activations/layer2_attention_weight_max": 33.44508743286133,
      "activations/layer2_attention_weight_min": -33.149539947509766,
      "activations/layer3_attention_weight_max": 94.37110900878906,
      "activations/layer3_attention_weight_min": -98.71109771728516,
      "activations/layer4_attention_weight_max": 104.14857482910156,
      "activations/layer4_attention_weight_min": -106.8405532836914,
      "activations/layer5_attention_weight_max": 51.06492614746094,
      "activations/layer5_attention_weight_min": -56.81903076171875,
      "activations/layer6_attention_weight_max": 43.87083435058594,
      "activations/layer6_attention_weight_min": -45.043704986572266,
      "activations/layer7_attention_weight_max": 100.8211669921875,
      "activations/layer7_attention_weight_min": -99.22142791748047,
      "activations/layer8_attention_weight_max": 46.80220413208008,
      "activations/layer8_attention_weight_min": -48.028350830078125,
      "activations/layer9_attention_weight_max": 34.67853927612305,
      "activations/layer9_attention_weight_min": -37.684730529785156,
      "epoch": 16.72,
      "learning_rate": 4.260643939393939e-05,
      "loss": 2.7362,
      "step": 287750
    },
    {
      "activations/layer0_attention_weight_max": 16.20775604248047,
      "activations/layer0_attention_weight_min": -11.727431297302246,
      "activations/layer10_attention_weight_max": 31.38875389099121,
      "activations/layer10_attention_weight_min": -33.716880798339844,
      "activations/layer11_attention_weight_max": 32.622161865234375,
      "activations/layer11_attention_weight_min": -32.600502014160156,
      "activations/layer12_attention_weight_max": 28.347610473632812,
      "activations/layer12_attention_weight_min": -25.705217361450195,
      "activations/layer13_attention_weight_max": 42.52277374267578,
      "activations/layer13_attention_weight_min": -36.64617919921875,
      "activations/layer14_attention_weight_max": 47.129520416259766,
      "activations/layer14_attention_weight_min": -36.76121520996094,
      "activations/layer15_attention_weight_max": 41.9937858581543,
      "activations/layer15_attention_weight_min": -34.21631622314453,
      "activations/layer16_attention_weight_max": 33.091800689697266,
      "activations/layer16_attention_weight_min": -26.632991790771484,
      "activations/layer17_attention_weight_max": 30.20420265197754,
      "activations/layer17_attention_weight_min": -24.422494888305664,
      "activations/layer18_attention_weight_max": 30.572481155395508,
      "activations/layer18_attention_weight_min": -24.75043296813965,
      "activations/layer19_attention_weight_max": 34.31924057006836,
      "activations/layer19_attention_weight_min": -30.144418716430664,
      "activations/layer1_attention_weight_max": 17.61224937438965,
      "activations/layer1_attention_weight_min": -15.518364906311035,
      "activations/layer20_attention_weight_max": 29.783689498901367,
      "activations/layer20_attention_weight_min": -23.919536590576172,
      "activations/layer21_attention_weight_max": 29.494237899780273,
      "activations/layer21_attention_weight_min": -22.839412689208984,
      "activations/layer22_attention_weight_max": 42.15925598144531,
      "activations/layer22_attention_weight_min": -28.978046417236328,
      "activations/layer23_attention_weight_max": 33.909751892089844,
      "activations/layer23_attention_weight_min": -24.212196350097656,
      "activations/layer2_attention_weight_max": 32.95073699951172,
      "activations/layer2_attention_weight_min": -32.613372802734375,
      "activations/layer3_attention_weight_max": 95.38934326171875,
      "activations/layer3_attention_weight_min": -97.908203125,
      "activations/layer4_attention_weight_max": 104.78910064697266,
      "activations/layer4_attention_weight_min": -105.99421691894531,
      "activations/layer5_attention_weight_max": 51.85943603515625,
      "activations/layer5_attention_weight_min": -59.94884490966797,
      "activations/layer6_attention_weight_max": 42.071083068847656,
      "activations/layer6_attention_weight_min": -46.07857894897461,
      "activations/layer7_attention_weight_max": 86.80816650390625,
      "activations/layer7_attention_weight_min": -91.30176544189453,
      "activations/layer8_attention_weight_max": 40.81833267211914,
      "activations/layer8_attention_weight_min": -42.27495193481445,
      "activations/layer9_attention_weight_max": 31.086713790893555,
      "activations/layer9_attention_weight_min": -33.80583190917969,
      "epoch": 16.72,
      "learning_rate": 4.258749999999999e-05,
      "loss": 2.7419,
      "step": 287800
    },
    {
      "activations/layer0_attention_weight_max": 17.182849884033203,
      "activations/layer0_attention_weight_min": -13.23753547668457,
      "activations/layer10_attention_weight_max": 41.47064971923828,
      "activations/layer10_attention_weight_min": -37.8867073059082,
      "activations/layer11_attention_weight_max": 38.56106948852539,
      "activations/layer11_attention_weight_min": -37.62980651855469,
      "activations/layer12_attention_weight_max": 30.96534538269043,
      "activations/layer12_attention_weight_min": -29.729209899902344,
      "activations/layer13_attention_weight_max": 48.07801055908203,
      "activations/layer13_attention_weight_min": -39.34770584106445,
      "activations/layer14_attention_weight_max": 58.3072624206543,
      "activations/layer14_attention_weight_min": -44.1185188293457,
      "activations/layer15_attention_weight_max": 53.58732986450195,
      "activations/layer15_attention_weight_min": -41.61087417602539,
      "activations/layer16_attention_weight_max": 37.11330795288086,
      "activations/layer16_attention_weight_min": -28.861909866333008,
      "activations/layer17_attention_weight_max": 36.10188674926758,
      "activations/layer17_attention_weight_min": -29.31612777709961,
      "activations/layer18_attention_weight_max": 38.544219970703125,
      "activations/layer18_attention_weight_min": -30.93157958984375,
      "activations/layer19_attention_weight_max": 49.33693313598633,
      "activations/layer19_attention_weight_min": -38.97756576538086,
      "activations/layer1_attention_weight_max": 16.6126651763916,
      "activations/layer1_attention_weight_min": -15.04163646697998,
      "activations/layer20_attention_weight_max": 37.81114196777344,
      "activations/layer20_attention_weight_min": -31.032819747924805,
      "activations/layer21_attention_weight_max": 38.254302978515625,
      "activations/layer21_attention_weight_min": -30.006729125976562,
      "activations/layer22_attention_weight_max": 60.42215347290039,
      "activations/layer22_attention_weight_min": -40.837398529052734,
      "activations/layer23_attention_weight_max": 43.02497863769531,
      "activations/layer23_attention_weight_min": -31.143543243408203,
      "activations/layer2_attention_weight_max": 39.33116149902344,
      "activations/layer2_attention_weight_min": -33.282344818115234,
      "activations/layer3_attention_weight_max": 102.09391784667969,
      "activations/layer3_attention_weight_min": -101.2806167602539,
      "activations/layer4_attention_weight_max": 106.1167221069336,
      "activations/layer4_attention_weight_min": -112.08000946044922,
      "activations/layer5_attention_weight_max": 51.084556579589844,
      "activations/layer5_attention_weight_min": -66.38563537597656,
      "activations/layer6_attention_weight_max": 44.1765022277832,
      "activations/layer6_attention_weight_min": -50.55470275878906,
      "activations/layer7_attention_weight_max": 100.29576110839844,
      "activations/layer7_attention_weight_min": -102.20020294189453,
      "activations/layer8_attention_weight_max": 45.467403411865234,
      "activations/layer8_attention_weight_min": -49.696834564208984,
      "activations/layer9_attention_weight_max": 36.668949127197266,
      "activations/layer9_attention_weight_min": -36.50217819213867,
      "epoch": 16.73,
      "learning_rate": 4.256856060606061e-05,
      "loss": 2.7421,
      "step": 287850
    },
    {
      "activations/layer0_attention_weight_max": 16.261600494384766,
      "activations/layer0_attention_weight_min": -12.439010620117188,
      "activations/layer10_attention_weight_max": 36.684444427490234,
      "activations/layer10_attention_weight_min": -39.73492431640625,
      "activations/layer11_attention_weight_max": 34.38540267944336,
      "activations/layer11_attention_weight_min": -37.63188171386719,
      "activations/layer12_attention_weight_max": 27.68159294128418,
      "activations/layer12_attention_weight_min": -27.94959831237793,
      "activations/layer13_attention_weight_max": 40.66135787963867,
      "activations/layer13_attention_weight_min": -39.33525085449219,
      "activations/layer14_attention_weight_max": 43.12116622924805,
      "activations/layer14_attention_weight_min": -39.95304870605469,
      "activations/layer15_attention_weight_max": 39.26736068725586,
      "activations/layer15_attention_weight_min": -36.271846771240234,
      "activations/layer16_attention_weight_max": 29.71418571472168,
      "activations/layer16_attention_weight_min": -27.810998916625977,
      "activations/layer17_attention_weight_max": 31.199575424194336,
      "activations/layer17_attention_weight_min": -26.5068416595459,
      "activations/layer18_attention_weight_max": 30.214805603027344,
      "activations/layer18_attention_weight_min": -25.292640686035156,
      "activations/layer19_attention_weight_max": 33.78334045410156,
      "activations/layer19_attention_weight_min": -30.760557174682617,
      "activations/layer1_attention_weight_max": 15.851659774780273,
      "activations/layer1_attention_weight_min": -15.39730167388916,
      "activations/layer20_attention_weight_max": 28.14312744140625,
      "activations/layer20_attention_weight_min": -25.08778953552246,
      "activations/layer21_attention_weight_max": 29.164684295654297,
      "activations/layer21_attention_weight_min": -24.850942611694336,
      "activations/layer22_attention_weight_max": 41.07929229736328,
      "activations/layer22_attention_weight_min": -31.42436981201172,
      "activations/layer23_attention_weight_max": 31.147445678710938,
      "activations/layer23_attention_weight_min": -25.331005096435547,
      "activations/layer2_attention_weight_max": 33.930458068847656,
      "activations/layer2_attention_weight_min": -34.34452819824219,
      "activations/layer3_attention_weight_max": 100.61904907226562,
      "activations/layer3_attention_weight_min": -105.96192932128906,
      "activations/layer4_attention_weight_max": 116.23741912841797,
      "activations/layer4_attention_weight_min": -113.29945373535156,
      "activations/layer5_attention_weight_max": 53.56622314453125,
      "activations/layer5_attention_weight_min": -61.614532470703125,
      "activations/layer6_attention_weight_max": 46.19267654418945,
      "activations/layer6_attention_weight_min": -50.121543884277344,
      "activations/layer7_attention_weight_max": 100.86628723144531,
      "activations/layer7_attention_weight_min": -107.8714370727539,
      "activations/layer8_attention_weight_max": 45.2123908996582,
      "activations/layer8_attention_weight_min": -47.644676208496094,
      "activations/layer9_attention_weight_max": 34.955223083496094,
      "activations/layer9_attention_weight_min": -38.5357666015625,
      "epoch": 16.73,
      "learning_rate": 4.254962121212121e-05,
      "loss": 2.7358,
      "step": 287900
    },
    {
      "activations/layer0_attention_weight_max": 15.822693824768066,
      "activations/layer0_attention_weight_min": -13.109702110290527,
      "activations/layer10_attention_weight_max": 34.72211837768555,
      "activations/layer10_attention_weight_min": -32.770118713378906,
      "activations/layer11_attention_weight_max": 35.082401275634766,
      "activations/layer11_attention_weight_min": -34.389930725097656,
      "activations/layer12_attention_weight_max": 24.154163360595703,
      "activations/layer12_attention_weight_min": -26.06255531311035,
      "activations/layer13_attention_weight_max": 35.74003982543945,
      "activations/layer13_attention_weight_min": -34.75267028808594,
      "activations/layer14_attention_weight_max": 38.54997634887695,
      "activations/layer14_attention_weight_min": -37.343238830566406,
      "activations/layer15_attention_weight_max": 33.555152893066406,
      "activations/layer15_attention_weight_min": -34.819061279296875,
      "activations/layer16_attention_weight_max": 27.949060440063477,
      "activations/layer16_attention_weight_min": -27.212570190429688,
      "activations/layer17_attention_weight_max": 28.54564094543457,
      "activations/layer17_attention_weight_min": -26.286195755004883,
      "activations/layer18_attention_weight_max": 28.20538330078125,
      "activations/layer18_attention_weight_min": -26.31559944152832,
      "activations/layer19_attention_weight_max": 32.325904846191406,
      "activations/layer19_attention_weight_min": -30.34574317932129,
      "activations/layer1_attention_weight_max": 16.4439697265625,
      "activations/layer1_attention_weight_min": -16.30225944519043,
      "activations/layer20_attention_weight_max": 26.078723907470703,
      "activations/layer20_attention_weight_min": -24.433591842651367,
      "activations/layer21_attention_weight_max": 26.897964477539062,
      "activations/layer21_attention_weight_min": -23.91552734375,
      "activations/layer22_attention_weight_max": 37.39993667602539,
      "activations/layer22_attention_weight_min": -31.553279876708984,
      "activations/layer23_attention_weight_max": 31.54646873474121,
      "activations/layer23_attention_weight_min": -25.12722396850586,
      "activations/layer2_attention_weight_max": 32.55354309082031,
      "activations/layer2_attention_weight_min": -32.835731506347656,
      "activations/layer3_attention_weight_max": 97.67498779296875,
      "activations/layer3_attention_weight_min": -98.26181030273438,
      "activations/layer4_attention_weight_max": 110.41300201416016,
      "activations/layer4_attention_weight_min": -107.3087158203125,
      "activations/layer5_attention_weight_max": 49.254913330078125,
      "activations/layer5_attention_weight_min": -65.72181701660156,
      "activations/layer6_attention_weight_max": 43.96520233154297,
      "activations/layer6_attention_weight_min": -45.83977127075195,
      "activations/layer7_attention_weight_max": 91.43201446533203,
      "activations/layer7_attention_weight_min": -92.4258041381836,
      "activations/layer8_attention_weight_max": 41.20325469970703,
      "activations/layer8_attention_weight_min": -41.41314697265625,
      "activations/layer9_attention_weight_max": 32.58356857299805,
      "activations/layer9_attention_weight_min": -32.820743560791016,
      "epoch": 16.73,
      "learning_rate": 4.2530681818181816e-05,
      "loss": 2.7417,
      "step": 287950
    },
    {
      "activations/layer0_attention_weight_max": 15.541447639465332,
      "activations/layer0_attention_weight_min": -11.90048599243164,
      "activations/layer10_attention_weight_max": 33.680274963378906,
      "activations/layer10_attention_weight_min": -33.39460754394531,
      "activations/layer11_attention_weight_max": 32.90835189819336,
      "activations/layer11_attention_weight_min": -35.64132308959961,
      "activations/layer12_attention_weight_max": 25.925813674926758,
      "activations/layer12_attention_weight_min": -25.720651626586914,
      "activations/layer13_attention_weight_max": 41.9044075012207,
      "activations/layer13_attention_weight_min": -33.60376739501953,
      "activations/layer14_attention_weight_max": 41.688133239746094,
      "activations/layer14_attention_weight_min": -36.82169723510742,
      "activations/layer15_attention_weight_max": 39.72487258911133,
      "activations/layer15_attention_weight_min": -33.05288314819336,
      "activations/layer16_attention_weight_max": 28.963483810424805,
      "activations/layer16_attention_weight_min": -27.57090950012207,
      "activations/layer17_attention_weight_max": 30.39324378967285,
      "activations/layer17_attention_weight_min": -25.381107330322266,
      "activations/layer18_attention_weight_max": 32.36194610595703,
      "activations/layer18_attention_weight_min": -25.439666748046875,
      "activations/layer19_attention_weight_max": 31.9755802154541,
      "activations/layer19_attention_weight_min": -30.622314453125,
      "activations/layer1_attention_weight_max": 15.9756441116333,
      "activations/layer1_attention_weight_min": -14.560308456420898,
      "activations/layer20_attention_weight_max": 28.712873458862305,
      "activations/layer20_attention_weight_min": -25.802478790283203,
      "activations/layer21_attention_weight_max": 29.152725219726562,
      "activations/layer21_attention_weight_min": -24.33762550354004,
      "activations/layer22_attention_weight_max": 41.16737365722656,
      "activations/layer22_attention_weight_min": -31.07709312438965,
      "activations/layer23_attention_weight_max": 31.348419189453125,
      "activations/layer23_attention_weight_min": -26.637451171875,
      "activations/layer2_attention_weight_max": 32.51346206665039,
      "activations/layer2_attention_weight_min": -32.34560775756836,
      "activations/layer3_attention_weight_max": 99.9222640991211,
      "activations/layer3_attention_weight_min": -96.48141479492188,
      "activations/layer4_attention_weight_max": 108.7511215209961,
      "activations/layer4_attention_weight_min": -109.51744842529297,
      "activations/layer5_attention_weight_max": 49.89825439453125,
      "activations/layer5_attention_weight_min": -60.876869201660156,
      "activations/layer6_attention_weight_max": 43.48606872558594,
      "activations/layer6_attention_weight_min": -46.488624572753906,
      "activations/layer7_attention_weight_max": 94.24124908447266,
      "activations/layer7_attention_weight_min": -94.04867553710938,
      "activations/layer8_attention_weight_max": 40.91498565673828,
      "activations/layer8_attention_weight_min": -43.3851432800293,
      "activations/layer9_attention_weight_max": 32.74541473388672,
      "activations/layer9_attention_weight_min": -33.49836349487305,
      "epoch": 16.73,
      "learning_rate": 4.251174242424242e-05,
      "loss": 2.7531,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.5326,
      "eval_samples_per_second": 503.246,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.5326,
      "eval_openwebtext_samples_per_second": 503.246,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.953,
      "eval_wikitext_samples_per_second": 233.487,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.5408,
      "eval_lambada_samples_per_second": 510.335,
      "step": 288000
    },
    {
      "activations/layer0_attention_weight_max": 16.174341201782227,
      "activations/layer0_attention_weight_min": -12.537409782409668,
      "activations/layer10_attention_weight_max": 35.96182632446289,
      "activations/layer10_attention_weight_min": -34.246246337890625,
      "activations/layer11_attention_weight_max": 37.549198150634766,
      "activations/layer11_attention_weight_min": -34.0237922668457,
      "activations/layer12_attention_weight_max": 27.568763732910156,
      "activations/layer12_attention_weight_min": -27.46575164794922,
      "activations/layer13_attention_weight_max": 40.10236740112305,
      "activations/layer13_attention_weight_min": -37.14067459106445,
      "activations/layer14_attention_weight_max": 44.18758010864258,
      "activations/layer14_attention_weight_min": -40.373600006103516,
      "activations/layer15_attention_weight_max": 37.62635040283203,
      "activations/layer15_attention_weight_min": -35.305782318115234,
      "activations/layer16_attention_weight_max": 30.184106826782227,
      "activations/layer16_attention_weight_min": -26.8402042388916,
      "activations/layer17_attention_weight_max": 30.985443115234375,
      "activations/layer17_attention_weight_min": -25.995126724243164,
      "activations/layer18_attention_weight_max": 29.657211303710938,
      "activations/layer18_attention_weight_min": -27.620277404785156,
      "activations/layer19_attention_weight_max": 33.66728210449219,
      "activations/layer19_attention_weight_min": -29.822982788085938,
      "activations/layer1_attention_weight_max": 16.614089965820312,
      "activations/layer1_attention_weight_min": -14.639032363891602,
      "activations/layer20_attention_weight_max": 29.8034610748291,
      "activations/layer20_attention_weight_min": -24.449613571166992,
      "activations/layer21_attention_weight_max": 27.308671951293945,
      "activations/layer21_attention_weight_min": -25.53215980529785,
      "activations/layer22_attention_weight_max": 41.80784225463867,
      "activations/layer22_attention_weight_min": -30.532501220703125,
      "activations/layer23_attention_weight_max": 31.893203735351562,
      "activations/layer23_attention_weight_min": -25.4769287109375,
      "activations/layer2_attention_weight_max": 31.11445426940918,
      "activations/layer2_attention_weight_min": -32.5031852722168,
      "activations/layer3_attention_weight_max": 92.65485382080078,
      "activations/layer3_attention_weight_min": -96.19821166992188,
      "activations/layer4_attention_weight_max": 103.82524871826172,
      "activations/layer4_attention_weight_min": -102.4840087890625,
      "activations/layer5_attention_weight_max": 51.07563018798828,
      "activations/layer5_attention_weight_min": -58.403804779052734,
      "activations/layer6_attention_weight_max": 43.70280838012695,
      "activations/layer6_attention_weight_min": -43.66387176513672,
      "activations/layer7_attention_weight_max": 91.52796936035156,
      "activations/layer7_attention_weight_min": -91.03205871582031,
      "activations/layer8_attention_weight_max": 44.760433197021484,
      "activations/layer8_attention_weight_min": -43.19554901123047,
      "activations/layer9_attention_weight_max": 35.24387741088867,
      "activations/layer9_attention_weight_min": -33.827247619628906,
      "epoch": 16.74,
      "learning_rate": 4.249280303030303e-05,
      "loss": 2.7571,
      "step": 288050
    },
    {
      "activations/layer0_attention_weight_max": 17.348546981811523,
      "activations/layer0_attention_weight_min": -14.470767974853516,
      "activations/layer10_attention_weight_max": 32.90607833862305,
      "activations/layer10_attention_weight_min": -32.88584899902344,
      "activations/layer11_attention_weight_max": 34.029762268066406,
      "activations/layer11_attention_weight_min": -33.92134475708008,
      "activations/layer12_attention_weight_max": 26.61988639831543,
      "activations/layer12_attention_weight_min": -27.0792293548584,
      "activations/layer13_attention_weight_max": 41.15779113769531,
      "activations/layer13_attention_weight_min": -35.66286087036133,
      "activations/layer14_attention_weight_max": 39.708614349365234,
      "activations/layer14_attention_weight_min": -37.864723205566406,
      "activations/layer15_attention_weight_max": 38.435020446777344,
      "activations/layer15_attention_weight_min": -36.619590759277344,
      "activations/layer16_attention_weight_max": 28.37394905090332,
      "activations/layer16_attention_weight_min": -29.154386520385742,
      "activations/layer17_attention_weight_max": 30.44134521484375,
      "activations/layer17_attention_weight_min": -25.79302406311035,
      "activations/layer18_attention_weight_max": 31.915817260742188,
      "activations/layer18_attention_weight_min": -24.119998931884766,
      "activations/layer19_attention_weight_max": 34.168617248535156,
      "activations/layer19_attention_weight_min": -30.881589889526367,
      "activations/layer1_attention_weight_max": 17.128231048583984,
      "activations/layer1_attention_weight_min": -15.277520179748535,
      "activations/layer20_attention_weight_max": 32.81376266479492,
      "activations/layer20_attention_weight_min": -24.759891510009766,
      "activations/layer21_attention_weight_max": 29.859375,
      "activations/layer21_attention_weight_min": -23.862957000732422,
      "activations/layer22_attention_weight_max": 38.91559982299805,
      "activations/layer22_attention_weight_min": -32.00214385986328,
      "activations/layer23_attention_weight_max": 32.514007568359375,
      "activations/layer23_attention_weight_min": -26.096206665039062,
      "activations/layer2_attention_weight_max": 31.596431732177734,
      "activations/layer2_attention_weight_min": -32.442779541015625,
      "activations/layer3_attention_weight_max": 94.89195251464844,
      "activations/layer3_attention_weight_min": -98.17696380615234,
      "activations/layer4_attention_weight_max": 108.1375732421875,
      "activations/layer4_attention_weight_min": -111.5130386352539,
      "activations/layer5_attention_weight_max": 53.0587158203125,
      "activations/layer5_attention_weight_min": -61.04772186279297,
      "activations/layer6_attention_weight_max": 43.68126678466797,
      "activations/layer6_attention_weight_min": -46.81403732299805,
      "activations/layer7_attention_weight_max": 91.84766387939453,
      "activations/layer7_attention_weight_min": -95.78104400634766,
      "activations/layer8_attention_weight_max": 39.09645462036133,
      "activations/layer8_attention_weight_min": -43.02766799926758,
      "activations/layer9_attention_weight_max": 32.79099655151367,
      "activations/layer9_attention_weight_min": -33.09553146362305,
      "epoch": 16.74,
      "learning_rate": 4.2473863636363634e-05,
      "loss": 2.7436,
      "step": 288100
    },
    {
      "activations/layer0_attention_weight_max": 15.814568519592285,
      "activations/layer0_attention_weight_min": -12.616729736328125,
      "activations/layer10_attention_weight_max": 32.93887710571289,
      "activations/layer10_attention_weight_min": -35.55438232421875,
      "activations/layer11_attention_weight_max": 33.09043884277344,
      "activations/layer11_attention_weight_min": -33.23152542114258,
      "activations/layer12_attention_weight_max": 27.326595306396484,
      "activations/layer12_attention_weight_min": -26.148704528808594,
      "activations/layer13_attention_weight_max": 41.47553634643555,
      "activations/layer13_attention_weight_min": -35.89446258544922,
      "activations/layer14_attention_weight_max": 43.772544860839844,
      "activations/layer14_attention_weight_min": -40.3588981628418,
      "activations/layer15_attention_weight_max": 41.19214630126953,
      "activations/layer15_attention_weight_min": -34.741580963134766,
      "activations/layer16_attention_weight_max": 31.630752563476562,
      "activations/layer16_attention_weight_min": -28.902158737182617,
      "activations/layer17_attention_weight_max": 32.25629425048828,
      "activations/layer17_attention_weight_min": -26.309946060180664,
      "activations/layer18_attention_weight_max": 32.255401611328125,
      "activations/layer18_attention_weight_min": -25.113142013549805,
      "activations/layer19_attention_weight_max": 34.377933502197266,
      "activations/layer19_attention_weight_min": -30.94618034362793,
      "activations/layer1_attention_weight_max": 16.621517181396484,
      "activations/layer1_attention_weight_min": -16.173931121826172,
      "activations/layer20_attention_weight_max": 29.772010803222656,
      "activations/layer20_attention_weight_min": -24.963411331176758,
      "activations/layer21_attention_weight_max": 28.390329360961914,
      "activations/layer21_attention_weight_min": -24.887523651123047,
      "activations/layer22_attention_weight_max": 42.37620544433594,
      "activations/layer22_attention_weight_min": -30.72412109375,
      "activations/layer23_attention_weight_max": 34.759002685546875,
      "activations/layer23_attention_weight_min": -26.84841537475586,
      "activations/layer2_attention_weight_max": 34.32598114013672,
      "activations/layer2_attention_weight_min": -33.61863327026367,
      "activations/layer3_attention_weight_max": 100.2335205078125,
      "activations/layer3_attention_weight_min": -101.71121215820312,
      "activations/layer4_attention_weight_max": 107.55091857910156,
      "activations/layer4_attention_weight_min": -114.34944915771484,
      "activations/layer5_attention_weight_max": 50.507606506347656,
      "activations/layer5_attention_weight_min": -62.20355224609375,
      "activations/layer6_attention_weight_max": 44.912174224853516,
      "activations/layer6_attention_weight_min": -46.70622253417969,
      "activations/layer7_attention_weight_max": 90.23751831054688,
      "activations/layer7_attention_weight_min": -93.72652435302734,
      "activations/layer8_attention_weight_max": 42.452449798583984,
      "activations/layer8_attention_weight_min": -44.04518127441406,
      "activations/layer9_attention_weight_max": 32.2419548034668,
      "activations/layer9_attention_weight_min": -35.32166290283203,
      "epoch": 16.74,
      "learning_rate": 4.2454924242424236e-05,
      "loss": 2.7257,
      "step": 288150
    },
    {
      "activations/layer0_attention_weight_max": 16.284408569335938,
      "activations/layer0_attention_weight_min": -14.202187538146973,
      "activations/layer10_attention_weight_max": 34.72621154785156,
      "activations/layer10_attention_weight_min": -31.859697341918945,
      "activations/layer11_attention_weight_max": 34.392913818359375,
      "activations/layer11_attention_weight_min": -32.56303024291992,
      "activations/layer12_attention_weight_max": 25.894285202026367,
      "activations/layer12_attention_weight_min": -26.36605453491211,
      "activations/layer13_attention_weight_max": 39.08332443237305,
      "activations/layer13_attention_weight_min": -35.35081100463867,
      "activations/layer14_attention_weight_max": 44.099308013916016,
      "activations/layer14_attention_weight_min": -36.83893966674805,
      "activations/layer15_attention_weight_max": 40.04857635498047,
      "activations/layer15_attention_weight_min": -33.994022369384766,
      "activations/layer16_attention_weight_max": 29.619096755981445,
      "activations/layer16_attention_weight_min": -28.42904281616211,
      "activations/layer17_attention_weight_max": 30.567853927612305,
      "activations/layer17_attention_weight_min": -27.434566497802734,
      "activations/layer18_attention_weight_max": 32.35615158081055,
      "activations/layer18_attention_weight_min": -26.136051177978516,
      "activations/layer19_attention_weight_max": 33.182003021240234,
      "activations/layer19_attention_weight_min": -31.261707305908203,
      "activations/layer1_attention_weight_max": 16.70404624938965,
      "activations/layer1_attention_weight_min": -15.902226448059082,
      "activations/layer20_attention_weight_max": 28.066532135009766,
      "activations/layer20_attention_weight_min": -27.338010787963867,
      "activations/layer21_attention_weight_max": 28.282114028930664,
      "activations/layer21_attention_weight_min": -25.405029296875,
      "activations/layer22_attention_weight_max": 44.243953704833984,
      "activations/layer22_attention_weight_min": -32.7486457824707,
      "activations/layer23_attention_weight_max": 36.98822784423828,
      "activations/layer23_attention_weight_min": -27.179637908935547,
      "activations/layer2_attention_weight_max": 31.923507690429688,
      "activations/layer2_attention_weight_min": -30.436033248901367,
      "activations/layer3_attention_weight_max": 89.6301498413086,
      "activations/layer3_attention_weight_min": -94.03080749511719,
      "activations/layer4_attention_weight_max": 100.32721710205078,
      "activations/layer4_attention_weight_min": -102.82894134521484,
      "activations/layer5_attention_weight_max": 47.96576690673828,
      "activations/layer5_attention_weight_min": -60.341854095458984,
      "activations/layer6_attention_weight_max": 41.81980514526367,
      "activations/layer6_attention_weight_min": -43.79918670654297,
      "activations/layer7_attention_weight_max": 83.8681869506836,
      "activations/layer7_attention_weight_min": -91.54418182373047,
      "activations/layer8_attention_weight_max": 37.81232833862305,
      "activations/layer8_attention_weight_min": -39.41203689575195,
      "activations/layer9_attention_weight_max": 31.097745895385742,
      "activations/layer9_attention_weight_min": -32.232234954833984,
      "epoch": 16.75,
      "learning_rate": 4.2435984848484844e-05,
      "loss": 2.7331,
      "step": 288200
    },
    {
      "activations/layer0_attention_weight_max": 15.216324806213379,
      "activations/layer0_attention_weight_min": -12.963653564453125,
      "activations/layer10_attention_weight_max": 33.03829574584961,
      "activations/layer10_attention_weight_min": -33.162960052490234,
      "activations/layer11_attention_weight_max": 34.80828857421875,
      "activations/layer11_attention_weight_min": -35.157081604003906,
      "activations/layer12_attention_weight_max": 26.09222984313965,
      "activations/layer12_attention_weight_min": -27.59379005432129,
      "activations/layer13_attention_weight_max": 39.9570426940918,
      "activations/layer13_attention_weight_min": -35.98530960083008,
      "activations/layer14_attention_weight_max": 42.18960952758789,
      "activations/layer14_attention_weight_min": -35.94268798828125,
      "activations/layer15_attention_weight_max": 41.38469314575195,
      "activations/layer15_attention_weight_min": -35.288082122802734,
      "activations/layer16_attention_weight_max": 32.354122161865234,
      "activations/layer16_attention_weight_min": -27.668170928955078,
      "activations/layer17_attention_weight_max": 33.613853454589844,
      "activations/layer17_attention_weight_min": -27.344623565673828,
      "activations/layer18_attention_weight_max": 35.547828674316406,
      "activations/layer18_attention_weight_min": -23.788530349731445,
      "activations/layer19_attention_weight_max": 40.72831726074219,
      "activations/layer19_attention_weight_min": -31.337020874023438,
      "activations/layer1_attention_weight_max": 16.229928970336914,
      "activations/layer1_attention_weight_min": -15.418136596679688,
      "activations/layer20_attention_weight_max": 33.66162872314453,
      "activations/layer20_attention_weight_min": -25.16033172607422,
      "activations/layer21_attention_weight_max": 33.46990203857422,
      "activations/layer21_attention_weight_min": -23.773723602294922,
      "activations/layer22_attention_weight_max": 45.01752471923828,
      "activations/layer22_attention_weight_min": -31.662630081176758,
      "activations/layer23_attention_weight_max": 36.25884246826172,
      "activations/layer23_attention_weight_min": -25.58718490600586,
      "activations/layer2_attention_weight_max": 33.32606506347656,
      "activations/layer2_attention_weight_min": -33.54461669921875,
      "activations/layer3_attention_weight_max": 94.8769760131836,
      "activations/layer3_attention_weight_min": -99.44061279296875,
      "activations/layer4_attention_weight_max": 105.63298797607422,
      "activations/layer4_attention_weight_min": -108.96111297607422,
      "activations/layer5_attention_weight_max": 49.47297668457031,
      "activations/layer5_attention_weight_min": -63.47612762451172,
      "activations/layer6_attention_weight_max": 43.88810348510742,
      "activations/layer6_attention_weight_min": -47.70438766479492,
      "activations/layer7_attention_weight_max": 92.66702270507812,
      "activations/layer7_attention_weight_min": -92.08504486083984,
      "activations/layer8_attention_weight_max": 41.465919494628906,
      "activations/layer8_attention_weight_min": -40.34290313720703,
      "activations/layer9_attention_weight_max": 32.45158004760742,
      "activations/layer9_attention_weight_min": -32.63930892944336,
      "epoch": 16.75,
      "learning_rate": 4.241704545454545e-05,
      "loss": 2.7565,
      "step": 288250
    },
    {
      "activations/layer0_attention_weight_max": 16.18899154663086,
      "activations/layer0_attention_weight_min": -13.815861701965332,
      "activations/layer10_attention_weight_max": 33.013275146484375,
      "activations/layer10_attention_weight_min": -32.03019332885742,
      "activations/layer11_attention_weight_max": 33.1605110168457,
      "activations/layer11_attention_weight_min": -33.88481903076172,
      "activations/layer12_attention_weight_max": 24.939895629882812,
      "activations/layer12_attention_weight_min": -28.022993087768555,
      "activations/layer13_attention_weight_max": 39.585811614990234,
      "activations/layer13_attention_weight_min": -34.91829299926758,
      "activations/layer14_attention_weight_max": 41.984642028808594,
      "activations/layer14_attention_weight_min": -36.753211975097656,
      "activations/layer15_attention_weight_max": 36.202232360839844,
      "activations/layer15_attention_weight_min": -33.567562103271484,
      "activations/layer16_attention_weight_max": 30.118871688842773,
      "activations/layer16_attention_weight_min": -28.00855827331543,
      "activations/layer17_attention_weight_max": 27.921918869018555,
      "activations/layer17_attention_weight_min": -26.35015106201172,
      "activations/layer18_attention_weight_max": 30.81736946105957,
      "activations/layer18_attention_weight_min": -25.537492752075195,
      "activations/layer19_attention_weight_max": 33.02190017700195,
      "activations/layer19_attention_weight_min": -30.843475341796875,
      "activations/layer1_attention_weight_max": 16.968008041381836,
      "activations/layer1_attention_weight_min": -14.62539005279541,
      "activations/layer20_attention_weight_max": 28.90281867980957,
      "activations/layer20_attention_weight_min": -25.32257080078125,
      "activations/layer21_attention_weight_max": 29.922515869140625,
      "activations/layer21_attention_weight_min": -24.557554244995117,
      "activations/layer22_attention_weight_max": 39.67547607421875,
      "activations/layer22_attention_weight_min": -30.415172576904297,
      "activations/layer23_attention_weight_max": 30.902408599853516,
      "activations/layer23_attention_weight_min": -25.2489013671875,
      "activations/layer2_attention_weight_max": 35.4474983215332,
      "activations/layer2_attention_weight_min": -34.320159912109375,
      "activations/layer3_attention_weight_max": 101.92143249511719,
      "activations/layer3_attention_weight_min": -101.34981536865234,
      "activations/layer4_attention_weight_max": 107.44830322265625,
      "activations/layer4_attention_weight_min": -108.7242431640625,
      "activations/layer5_attention_weight_max": 51.05110168457031,
      "activations/layer5_attention_weight_min": -59.25818634033203,
      "activations/layer6_attention_weight_max": 44.00403594970703,
      "activations/layer6_attention_weight_min": -45.3046760559082,
      "activations/layer7_attention_weight_max": 95.38217163085938,
      "activations/layer7_attention_weight_min": -89.92039489746094,
      "activations/layer8_attention_weight_max": 42.635169982910156,
      "activations/layer8_attention_weight_min": -41.476932525634766,
      "activations/layer9_attention_weight_max": 31.200143814086914,
      "activations/layer9_attention_weight_min": -34.01992416381836,
      "epoch": 16.75,
      "learning_rate": 4.239810606060606e-05,
      "loss": 2.7326,
      "step": 288300
    },
    {
      "activations/layer0_attention_weight_max": 15.907914161682129,
      "activations/layer0_attention_weight_min": -12.796013832092285,
      "activations/layer10_attention_weight_max": 34.71669006347656,
      "activations/layer10_attention_weight_min": -36.555755615234375,
      "activations/layer11_attention_weight_max": 33.192108154296875,
      "activations/layer11_attention_weight_min": -34.143760681152344,
      "activations/layer12_attention_weight_max": 26.436975479125977,
      "activations/layer12_attention_weight_min": -26.841388702392578,
      "activations/layer13_attention_weight_max": 39.19999313354492,
      "activations/layer13_attention_weight_min": -35.25065612792969,
      "activations/layer14_attention_weight_max": 40.52021026611328,
      "activations/layer14_attention_weight_min": -37.54172897338867,
      "activations/layer15_attention_weight_max": 38.65086364746094,
      "activations/layer15_attention_weight_min": -35.7797737121582,
      "activations/layer16_attention_weight_max": 28.693950653076172,
      "activations/layer16_attention_weight_min": -28.040512084960938,
      "activations/layer17_attention_weight_max": 31.06067657470703,
      "activations/layer17_attention_weight_min": -24.477245330810547,
      "activations/layer18_attention_weight_max": 29.84228515625,
      "activations/layer18_attention_weight_min": -25.011505126953125,
      "activations/layer19_attention_weight_max": 32.376583099365234,
      "activations/layer19_attention_weight_min": -31.10555076599121,
      "activations/layer1_attention_weight_max": 16.388870239257812,
      "activations/layer1_attention_weight_min": -14.693526268005371,
      "activations/layer20_attention_weight_max": 28.459077835083008,
      "activations/layer20_attention_weight_min": -26.27862548828125,
      "activations/layer21_attention_weight_max": 33.048500061035156,
      "activations/layer21_attention_weight_min": -26.485368728637695,
      "activations/layer22_attention_weight_max": 40.384727478027344,
      "activations/layer22_attention_weight_min": -30.46206283569336,
      "activations/layer23_attention_weight_max": 34.008331298828125,
      "activations/layer23_attention_weight_min": -26.421995162963867,
      "activations/layer2_attention_weight_max": 32.657989501953125,
      "activations/layer2_attention_weight_min": -31.68740463256836,
      "activations/layer3_attention_weight_max": 94.50884246826172,
      "activations/layer3_attention_weight_min": -97.93987274169922,
      "activations/layer4_attention_weight_max": 107.68568420410156,
      "activations/layer4_attention_weight_min": -106.56361389160156,
      "activations/layer5_attention_weight_max": 51.16120910644531,
      "activations/layer5_attention_weight_min": -62.5291748046875,
      "activations/layer6_attention_weight_max": 44.44386291503906,
      "activations/layer6_attention_weight_min": -46.96723556518555,
      "activations/layer7_attention_weight_max": 92.70044708251953,
      "activations/layer7_attention_weight_min": -97.82762908935547,
      "activations/layer8_attention_weight_max": 42.5265007019043,
      "activations/layer8_attention_weight_min": -45.93961715698242,
      "activations/layer9_attention_weight_max": 31.95063591003418,
      "activations/layer9_attention_weight_min": -36.512786865234375,
      "epoch": 16.75,
      "learning_rate": 4.237916666666666e-05,
      "loss": 2.7476,
      "step": 288350
    },
    {
      "activations/layer0_attention_weight_max": 15.140865325927734,
      "activations/layer0_attention_weight_min": -12.524375915527344,
      "activations/layer10_attention_weight_max": 36.67216491699219,
      "activations/layer10_attention_weight_min": -34.333534240722656,
      "activations/layer11_attention_weight_max": 34.312843322753906,
      "activations/layer11_attention_weight_min": -34.35350036621094,
      "activations/layer12_attention_weight_max": 28.487760543823242,
      "activations/layer12_attention_weight_min": -28.94559097290039,
      "activations/layer13_attention_weight_max": 40.34047317504883,
      "activations/layer13_attention_weight_min": -40.34395217895508,
      "activations/layer14_attention_weight_max": 43.58806610107422,
      "activations/layer14_attention_weight_min": -42.24864959716797,
      "activations/layer15_attention_weight_max": 38.241573333740234,
      "activations/layer15_attention_weight_min": -36.9096565246582,
      "activations/layer16_attention_weight_max": 28.134004592895508,
      "activations/layer16_attention_weight_min": -30.262210845947266,
      "activations/layer17_attention_weight_max": 28.473670959472656,
      "activations/layer17_attention_weight_min": -29.077234268188477,
      "activations/layer18_attention_weight_max": 30.408489227294922,
      "activations/layer18_attention_weight_min": -25.18004035949707,
      "activations/layer19_attention_weight_max": 31.7701473236084,
      "activations/layer19_attention_weight_min": -30.632156372070312,
      "activations/layer1_attention_weight_max": 16.75370216369629,
      "activations/layer1_attention_weight_min": -16.033954620361328,
      "activations/layer20_attention_weight_max": 26.348554611206055,
      "activations/layer20_attention_weight_min": -25.77410316467285,
      "activations/layer21_attention_weight_max": 25.990453720092773,
      "activations/layer21_attention_weight_min": -25.64397430419922,
      "activations/layer22_attention_weight_max": 42.708518981933594,
      "activations/layer22_attention_weight_min": -31.18644142150879,
      "activations/layer23_attention_weight_max": 31.260086059570312,
      "activations/layer23_attention_weight_min": -27.977672576904297,
      "activations/layer2_attention_weight_max": 33.98546600341797,
      "activations/layer2_attention_weight_min": -33.578147888183594,
      "activations/layer3_attention_weight_max": 95.30927276611328,
      "activations/layer3_attention_weight_min": -93.68417358398438,
      "activations/layer4_attention_weight_max": 107.5726547241211,
      "activations/layer4_attention_weight_min": -105.12940979003906,
      "activations/layer5_attention_weight_max": 51.95087432861328,
      "activations/layer5_attention_weight_min": -64.65571594238281,
      "activations/layer6_attention_weight_max": 42.130279541015625,
      "activations/layer6_attention_weight_min": -47.32850646972656,
      "activations/layer7_attention_weight_max": 93.66873168945312,
      "activations/layer7_attention_weight_min": -88.78102111816406,
      "activations/layer8_attention_weight_max": 44.2994499206543,
      "activations/layer8_attention_weight_min": -43.00455856323242,
      "activations/layer9_attention_weight_max": 36.466434478759766,
      "activations/layer9_attention_weight_min": -33.15778732299805,
      "epoch": 16.76,
      "learning_rate": 4.2360227272727264e-05,
      "loss": 2.7505,
      "step": 288400
    },
    {
      "activations/layer0_attention_weight_max": 15.91727066040039,
      "activations/layer0_attention_weight_min": -12.999153137207031,
      "activations/layer10_attention_weight_max": 38.32541275024414,
      "activations/layer10_attention_weight_min": -40.44423294067383,
      "activations/layer11_attention_weight_max": 38.50102996826172,
      "activations/layer11_attention_weight_min": -39.39466094970703,
      "activations/layer12_attention_weight_max": 26.05985450744629,
      "activations/layer12_attention_weight_min": -27.634180068969727,
      "activations/layer13_attention_weight_max": 35.932472229003906,
      "activations/layer13_attention_weight_min": -39.24501419067383,
      "activations/layer14_attention_weight_max": 44.423221588134766,
      "activations/layer14_attention_weight_min": -44.016090393066406,
      "activations/layer15_attention_weight_max": 39.182029724121094,
      "activations/layer15_attention_weight_min": -38.89535903930664,
      "activations/layer16_attention_weight_max": 28.70790672302246,
      "activations/layer16_attention_weight_min": -30.936681747436523,
      "activations/layer17_attention_weight_max": 32.86739730834961,
      "activations/layer17_attention_weight_min": -26.34067153930664,
      "activations/layer18_attention_weight_max": 31.38652229309082,
      "activations/layer18_attention_weight_min": -28.1070499420166,
      "activations/layer19_attention_weight_max": 31.941526412963867,
      "activations/layer19_attention_weight_min": -32.0166015625,
      "activations/layer1_attention_weight_max": 18.218154907226562,
      "activations/layer1_attention_weight_min": -17.519075393676758,
      "activations/layer20_attention_weight_max": 28.07550048828125,
      "activations/layer20_attention_weight_min": -27.588787078857422,
      "activations/layer21_attention_weight_max": 26.64981460571289,
      "activations/layer21_attention_weight_min": -25.61299705505371,
      "activations/layer22_attention_weight_max": 40.56189727783203,
      "activations/layer22_attention_weight_min": -31.930967330932617,
      "activations/layer23_attention_weight_max": 31.13739585876465,
      "activations/layer23_attention_weight_min": -26.13055992126465,
      "activations/layer2_attention_weight_max": 33.109928131103516,
      "activations/layer2_attention_weight_min": -36.777931213378906,
      "activations/layer3_attention_weight_max": 91.50662231445312,
      "activations/layer3_attention_weight_min": -103.01721954345703,
      "activations/layer4_attention_weight_max": 103.85540771484375,
      "activations/layer4_attention_weight_min": -98.95004272460938,
      "activations/layer5_attention_weight_max": 49.498531341552734,
      "activations/layer5_attention_weight_min": -56.98342514038086,
      "activations/layer6_attention_weight_max": 41.768218994140625,
      "activations/layer6_attention_weight_min": -43.91509246826172,
      "activations/layer7_attention_weight_max": 94.1837158203125,
      "activations/layer7_attention_weight_min": -95.357421875,
      "activations/layer8_attention_weight_max": 42.310386657714844,
      "activations/layer8_attention_weight_min": -43.619232177734375,
      "activations/layer9_attention_weight_max": 34.09572219848633,
      "activations/layer9_attention_weight_min": -34.69774627685547,
      "epoch": 16.76,
      "learning_rate": 4.234128787878788e-05,
      "loss": 2.7428,
      "step": 288450
    },
    {
      "activations/layer0_attention_weight_max": 16.033113479614258,
      "activations/layer0_attention_weight_min": -12.564345359802246,
      "activations/layer10_attention_weight_max": 35.3931884765625,
      "activations/layer10_attention_weight_min": -34.965572357177734,
      "activations/layer11_attention_weight_max": 36.079933166503906,
      "activations/layer11_attention_weight_min": -34.419822692871094,
      "activations/layer12_attention_weight_max": 34.915122985839844,
      "activations/layer12_attention_weight_min": -28.022701263427734,
      "activations/layer13_attention_weight_max": 46.24710464477539,
      "activations/layer13_attention_weight_min": -38.2917366027832,
      "activations/layer14_attention_weight_max": 49.91455841064453,
      "activations/layer14_attention_weight_min": -42.17738342285156,
      "activations/layer15_attention_weight_max": 46.624542236328125,
      "activations/layer15_attention_weight_min": -38.45592498779297,
      "activations/layer16_attention_weight_max": 35.835472106933594,
      "activations/layer16_attention_weight_min": -30.290908813476562,
      "activations/layer17_attention_weight_max": 33.82427215576172,
      "activations/layer17_attention_weight_min": -26.46062660217285,
      "activations/layer18_attention_weight_max": 34.404075622558594,
      "activations/layer18_attention_weight_min": -29.08404541015625,
      "activations/layer19_attention_weight_max": 37.351802825927734,
      "activations/layer19_attention_weight_min": -33.2317008972168,
      "activations/layer1_attention_weight_max": 16.15496826171875,
      "activations/layer1_attention_weight_min": -15.552205085754395,
      "activations/layer20_attention_weight_max": 35.41016387939453,
      "activations/layer20_attention_weight_min": -27.014013290405273,
      "activations/layer21_attention_weight_max": 35.741127014160156,
      "activations/layer21_attention_weight_min": -26.561002731323242,
      "activations/layer22_attention_weight_max": 51.14710998535156,
      "activations/layer22_attention_weight_min": -33.808380126953125,
      "activations/layer23_attention_weight_max": 35.8348503112793,
      "activations/layer23_attention_weight_min": -26.69200897216797,
      "activations/layer2_attention_weight_max": 30.980140686035156,
      "activations/layer2_attention_weight_min": -30.527183532714844,
      "activations/layer3_attention_weight_max": 95.24319458007812,
      "activations/layer3_attention_weight_min": -93.69245910644531,
      "activations/layer4_attention_weight_max": 110.8305435180664,
      "activations/layer4_attention_weight_min": -110.9153823852539,
      "activations/layer5_attention_weight_max": 52.1607666015625,
      "activations/layer5_attention_weight_min": -61.298919677734375,
      "activations/layer6_attention_weight_max": 46.08341598510742,
      "activations/layer6_attention_weight_min": -46.886436462402344,
      "activations/layer7_attention_weight_max": 97.90455627441406,
      "activations/layer7_attention_weight_min": -99.05142974853516,
      "activations/layer8_attention_weight_max": 43.212921142578125,
      "activations/layer8_attention_weight_min": -44.575218200683594,
      "activations/layer9_attention_weight_max": 34.50867462158203,
      "activations/layer9_attention_weight_min": -34.61961364746094,
      "epoch": 16.76,
      "learning_rate": 4.232234848484848e-05,
      "loss": 2.7573,
      "step": 288500
    },
    {
      "activations/layer0_attention_weight_max": 15.916358947753906,
      "activations/layer0_attention_weight_min": -13.3823823928833,
      "activations/layer10_attention_weight_max": 34.76213455200195,
      "activations/layer10_attention_weight_min": -31.95070457458496,
      "activations/layer11_attention_weight_max": 33.8155632019043,
      "activations/layer11_attention_weight_min": -31.598867416381836,
      "activations/layer12_attention_weight_max": 28.07094955444336,
      "activations/layer12_attention_weight_min": -25.922346115112305,
      "activations/layer13_attention_weight_max": 43.6565055847168,
      "activations/layer13_attention_weight_min": -33.94456481933594,
      "activations/layer14_attention_weight_max": 39.795135498046875,
      "activations/layer14_attention_weight_min": -37.4817008972168,
      "activations/layer15_attention_weight_max": 36.611061096191406,
      "activations/layer15_attention_weight_min": -33.73104476928711,
      "activations/layer16_attention_weight_max": 27.628311157226562,
      "activations/layer16_attention_weight_min": -28.238109588623047,
      "activations/layer17_attention_weight_max": 29.903011322021484,
      "activations/layer17_attention_weight_min": -24.92557144165039,
      "activations/layer18_attention_weight_max": 35.359832763671875,
      "activations/layer18_attention_weight_min": -24.956375122070312,
      "activations/layer19_attention_weight_max": 31.233413696289062,
      "activations/layer19_attention_weight_min": -32.13468551635742,
      "activations/layer1_attention_weight_max": 16.002038955688477,
      "activations/layer1_attention_weight_min": -15.438023567199707,
      "activations/layer20_attention_weight_max": 27.82594871520996,
      "activations/layer20_attention_weight_min": -24.78066635131836,
      "activations/layer21_attention_weight_max": 27.797319412231445,
      "activations/layer21_attention_weight_min": -24.63836097717285,
      "activations/layer22_attention_weight_max": 43.021728515625,
      "activations/layer22_attention_weight_min": -32.0435676574707,
      "activations/layer23_attention_weight_max": 30.97132110595703,
      "activations/layer23_attention_weight_min": -27.704219818115234,
      "activations/layer2_attention_weight_max": 32.27032470703125,
      "activations/layer2_attention_weight_min": -32.26270294189453,
      "activations/layer3_attention_weight_max": 94.27774810791016,
      "activations/layer3_attention_weight_min": -96.84497833251953,
      "activations/layer4_attention_weight_max": 103.21687316894531,
      "activations/layer4_attention_weight_min": -105.12374114990234,
      "activations/layer5_attention_weight_max": 49.62336730957031,
      "activations/layer5_attention_weight_min": -60.34905242919922,
      "activations/layer6_attention_weight_max": 46.20218276977539,
      "activations/layer6_attention_weight_min": -44.75971984863281,
      "activations/layer7_attention_weight_max": 92.6502456665039,
      "activations/layer7_attention_weight_min": -97.01471710205078,
      "activations/layer8_attention_weight_max": 41.68991470336914,
      "activations/layer8_attention_weight_min": -41.75728225708008,
      "activations/layer9_attention_weight_max": 31.96526527404785,
      "activations/layer9_attention_weight_min": -34.011505126953125,
      "epoch": 16.77,
      "learning_rate": 4.230340909090908e-05,
      "loss": 2.7354,
      "step": 288550
    },
    {
      "activations/layer0_attention_weight_max": 16.14512062072754,
      "activations/layer0_attention_weight_min": -13.130077362060547,
      "activations/layer10_attention_weight_max": 35.095516204833984,
      "activations/layer10_attention_weight_min": -33.54254150390625,
      "activations/layer11_attention_weight_max": 33.84039306640625,
      "activations/layer11_attention_weight_min": -34.4017219543457,
      "activations/layer12_attention_weight_max": 26.496171951293945,
      "activations/layer12_attention_weight_min": -26.552982330322266,
      "activations/layer13_attention_weight_max": 40.513465881347656,
      "activations/layer13_attention_weight_min": -36.25926208496094,
      "activations/layer14_attention_weight_max": 42.07048416137695,
      "activations/layer14_attention_weight_min": -39.20674133300781,
      "activations/layer15_attention_weight_max": 42.109859466552734,
      "activations/layer15_attention_weight_min": -37.52519226074219,
      "activations/layer16_attention_weight_max": 32.25569534301758,
      "activations/layer16_attention_weight_min": -28.734163284301758,
      "activations/layer17_attention_weight_max": 32.609649658203125,
      "activations/layer17_attention_weight_min": -27.457534790039062,
      "activations/layer18_attention_weight_max": 33.95720291137695,
      "activations/layer18_attention_weight_min": -26.39382553100586,
      "activations/layer19_attention_weight_max": 39.891998291015625,
      "activations/layer19_attention_weight_min": -33.950130462646484,
      "activations/layer1_attention_weight_max": 17.434480667114258,
      "activations/layer1_attention_weight_min": -15.068004608154297,
      "activations/layer20_attention_weight_max": 32.26832962036133,
      "activations/layer20_attention_weight_min": -25.931556701660156,
      "activations/layer21_attention_weight_max": 31.795438766479492,
      "activations/layer21_attention_weight_min": -24.643766403198242,
      "activations/layer22_attention_weight_max": 41.349266052246094,
      "activations/layer22_attention_weight_min": -32.03087615966797,
      "activations/layer23_attention_weight_max": 35.172523498535156,
      "activations/layer23_attention_weight_min": -25.0616455078125,
      "activations/layer2_attention_weight_max": 32.36613464355469,
      "activations/layer2_attention_weight_min": -28.614906311035156,
      "activations/layer3_attention_weight_max": 84.8001937866211,
      "activations/layer3_attention_weight_min": -89.0421371459961,
      "activations/layer4_attention_weight_max": 99.59268188476562,
      "activations/layer4_attention_weight_min": -102.552001953125,
      "activations/layer5_attention_weight_max": 47.095664978027344,
      "activations/layer5_attention_weight_min": -58.13655090332031,
      "activations/layer6_attention_weight_max": 42.106178283691406,
      "activations/layer6_attention_weight_min": -45.3360481262207,
      "activations/layer7_attention_weight_max": 90.2287826538086,
      "activations/layer7_attention_weight_min": -96.7498779296875,
      "activations/layer8_attention_weight_max": 42.0435676574707,
      "activations/layer8_attention_weight_min": -45.542503356933594,
      "activations/layer9_attention_weight_max": 33.08186721801758,
      "activations/layer9_attention_weight_min": -35.01069259643555,
      "epoch": 16.77,
      "learning_rate": 4.22844696969697e-05,
      "loss": 2.7484,
      "step": 288600
    },
    {
      "activations/layer0_attention_weight_max": 15.823747634887695,
      "activations/layer0_attention_weight_min": -12.592103004455566,
      "activations/layer10_attention_weight_max": 35.47270584106445,
      "activations/layer10_attention_weight_min": -36.514984130859375,
      "activations/layer11_attention_weight_max": 34.86439895629883,
      "activations/layer11_attention_weight_min": -36.48035430908203,
      "activations/layer12_attention_weight_max": 27.17266273498535,
      "activations/layer12_attention_weight_min": -28.048681259155273,
      "activations/layer13_attention_weight_max": 39.02201843261719,
      "activations/layer13_attention_weight_min": -36.346893310546875,
      "activations/layer14_attention_weight_max": 46.05527114868164,
      "activations/layer14_attention_weight_min": -41.39474868774414,
      "activations/layer15_attention_weight_max": 40.10496520996094,
      "activations/layer15_attention_weight_min": -37.90636444091797,
      "activations/layer16_attention_weight_max": 36.599178314208984,
      "activations/layer16_attention_weight_min": -27.979108810424805,
      "activations/layer17_attention_weight_max": 30.58407974243164,
      "activations/layer17_attention_weight_min": -26.425840377807617,
      "activations/layer18_attention_weight_max": 34.6130485534668,
      "activations/layer18_attention_weight_min": -26.52305793762207,
      "activations/layer19_attention_weight_max": 38.8958854675293,
      "activations/layer19_attention_weight_min": -31.94516372680664,
      "activations/layer1_attention_weight_max": 17.195205688476562,
      "activations/layer1_attention_weight_min": -16.015596389770508,
      "activations/layer20_attention_weight_max": 32.34690856933594,
      "activations/layer20_attention_weight_min": -25.125194549560547,
      "activations/layer21_attention_weight_max": 31.288284301757812,
      "activations/layer21_attention_weight_min": -25.838035583496094,
      "activations/layer22_attention_weight_max": 43.89689254760742,
      "activations/layer22_attention_weight_min": -33.47449493408203,
      "activations/layer23_attention_weight_max": 38.37504577636719,
      "activations/layer23_attention_weight_min": -25.98653221130371,
      "activations/layer2_attention_weight_max": 33.60761642456055,
      "activations/layer2_attention_weight_min": -31.60375213623047,
      "activations/layer3_attention_weight_max": 96.5447769165039,
      "activations/layer3_attention_weight_min": -95.94722747802734,
      "activations/layer4_attention_weight_max": 107.04398345947266,
      "activations/layer4_attention_weight_min": -110.67726135253906,
      "activations/layer5_attention_weight_max": 51.071929931640625,
      "activations/layer5_attention_weight_min": -64.70311737060547,
      "activations/layer6_attention_weight_max": 44.66648864746094,
      "activations/layer6_attention_weight_min": -45.763492584228516,
      "activations/layer7_attention_weight_max": 96.04798126220703,
      "activations/layer7_attention_weight_min": -98.93629455566406,
      "activations/layer8_attention_weight_max": 43.18955993652344,
      "activations/layer8_attention_weight_min": -45.11008071899414,
      "activations/layer9_attention_weight_max": 33.51659393310547,
      "activations/layer9_attention_weight_min": -35.34371566772461,
      "epoch": 16.77,
      "learning_rate": 4.22655303030303e-05,
      "loss": 2.7318,
      "step": 288650
    },
    {
      "activations/layer0_attention_weight_max": 15.573967933654785,
      "activations/layer0_attention_weight_min": -13.15174388885498,
      "activations/layer10_attention_weight_max": 32.40916061401367,
      "activations/layer10_attention_weight_min": -31.8450984954834,
      "activations/layer11_attention_weight_max": 31.569835662841797,
      "activations/layer11_attention_weight_min": -32.44868469238281,
      "activations/layer12_attention_weight_max": 27.583566665649414,
      "activations/layer12_attention_weight_min": -26.353473663330078,
      "activations/layer13_attention_weight_max": 40.725547790527344,
      "activations/layer13_attention_weight_min": -36.397186279296875,
      "activations/layer14_attention_weight_max": 44.68522262573242,
      "activations/layer14_attention_weight_min": -38.02457809448242,
      "activations/layer15_attention_weight_max": 40.234291076660156,
      "activations/layer15_attention_weight_min": -36.492069244384766,
      "activations/layer16_attention_weight_max": 29.675443649291992,
      "activations/layer16_attention_weight_min": -28.057147979736328,
      "activations/layer17_attention_weight_max": 27.362268447875977,
      "activations/layer17_attention_weight_min": -25.466651916503906,
      "activations/layer18_attention_weight_max": 29.868398666381836,
      "activations/layer18_attention_weight_min": -24.115053176879883,
      "activations/layer19_attention_weight_max": 33.466957092285156,
      "activations/layer19_attention_weight_min": -30.98258399963379,
      "activations/layer1_attention_weight_max": 16.963762283325195,
      "activations/layer1_attention_weight_min": -14.164477348327637,
      "activations/layer20_attention_weight_max": 27.785110473632812,
      "activations/layer20_attention_weight_min": -24.81887435913086,
      "activations/layer21_attention_weight_max": 28.21532440185547,
      "activations/layer21_attention_weight_min": -25.576927185058594,
      "activations/layer22_attention_weight_max": 37.9528923034668,
      "activations/layer22_attention_weight_min": -31.82369041442871,
      "activations/layer23_attention_weight_max": 31.923450469970703,
      "activations/layer23_attention_weight_min": -26.785314559936523,
      "activations/layer2_attention_weight_max": 32.671356201171875,
      "activations/layer2_attention_weight_min": -30.754566192626953,
      "activations/layer3_attention_weight_max": 93.1032943725586,
      "activations/layer3_attention_weight_min": -92.43221282958984,
      "activations/layer4_attention_weight_max": 106.03514862060547,
      "activations/layer4_attention_weight_min": -107.6142807006836,
      "activations/layer5_attention_weight_max": 51.92133331298828,
      "activations/layer5_attention_weight_min": -63.66840744018555,
      "activations/layer6_attention_weight_max": 43.550880432128906,
      "activations/layer6_attention_weight_min": -44.64385223388672,
      "activations/layer7_attention_weight_max": 90.32734680175781,
      "activations/layer7_attention_weight_min": -91.70196533203125,
      "activations/layer8_attention_weight_max": 40.23601531982422,
      "activations/layer8_attention_weight_min": -40.94197082519531,
      "activations/layer9_attention_weight_max": 31.169782638549805,
      "activations/layer9_attention_weight_min": -32.31293869018555,
      "epoch": 16.78,
      "learning_rate": 4.2246590909090907e-05,
      "loss": 2.7249,
      "step": 288700
    },
    {
      "activations/layer0_attention_weight_max": 15.928640365600586,
      "activations/layer0_attention_weight_min": -12.207736015319824,
      "activations/layer10_attention_weight_max": 35.895423889160156,
      "activations/layer10_attention_weight_min": -35.70008087158203,
      "activations/layer11_attention_weight_max": 36.59806442260742,
      "activations/layer11_attention_weight_min": -36.197227478027344,
      "activations/layer12_attention_weight_max": 28.20334815979004,
      "activations/layer12_attention_weight_min": -28.753602981567383,
      "activations/layer13_attention_weight_max": 39.78129196166992,
      "activations/layer13_attention_weight_min": -35.645652770996094,
      "activations/layer14_attention_weight_max": 44.14661407470703,
      "activations/layer14_attention_weight_min": -39.89845657348633,
      "activations/layer15_attention_weight_max": 40.919822692871094,
      "activations/layer15_attention_weight_min": -37.74231719970703,
      "activations/layer16_attention_weight_max": 33.40803909301758,
      "activations/layer16_attention_weight_min": -29.062950134277344,
      "activations/layer17_attention_weight_max": 33.01148986816406,
      "activations/layer17_attention_weight_min": -26.339101791381836,
      "activations/layer18_attention_weight_max": 32.01583480834961,
      "activations/layer18_attention_weight_min": -25.639251708984375,
      "activations/layer19_attention_weight_max": 40.346221923828125,
      "activations/layer19_attention_weight_min": -34.34077453613281,
      "activations/layer1_attention_weight_max": 16.584293365478516,
      "activations/layer1_attention_weight_min": -16.9908390045166,
      "activations/layer20_attention_weight_max": 32.100399017333984,
      "activations/layer20_attention_weight_min": -28.376934051513672,
      "activations/layer21_attention_weight_max": 30.652095794677734,
      "activations/layer21_attention_weight_min": -26.061965942382812,
      "activations/layer22_attention_weight_max": 42.89445495605469,
      "activations/layer22_attention_weight_min": -30.80841827392578,
      "activations/layer23_attention_weight_max": 33.083251953125,
      "activations/layer23_attention_weight_min": -25.06375503540039,
      "activations/layer2_attention_weight_max": 32.813899993896484,
      "activations/layer2_attention_weight_min": -32.01074981689453,
      "activations/layer3_attention_weight_max": 98.80105590820312,
      "activations/layer3_attention_weight_min": -96.63662719726562,
      "activations/layer4_attention_weight_max": 111.22560119628906,
      "activations/layer4_attention_weight_min": -106.27471923828125,
      "activations/layer5_attention_weight_max": 52.1282958984375,
      "activations/layer5_attention_weight_min": -68.09224700927734,
      "activations/layer6_attention_weight_max": 46.71510696411133,
      "activations/layer6_attention_weight_min": -47.244789123535156,
      "activations/layer7_attention_weight_max": 93.84321594238281,
      "activations/layer7_attention_weight_min": -95.4386215209961,
      "activations/layer8_attention_weight_max": 41.78096389770508,
      "activations/layer8_attention_weight_min": -43.757843017578125,
      "activations/layer9_attention_weight_max": 31.951391220092773,
      "activations/layer9_attention_weight_min": -35.785037994384766,
      "epoch": 16.78,
      "learning_rate": 4.222765151515151e-05,
      "loss": 2.751,
      "step": 288750
    },
    {
      "activations/layer0_attention_weight_max": 15.965177536010742,
      "activations/layer0_attention_weight_min": -12.358304023742676,
      "activations/layer10_attention_weight_max": 37.65932846069336,
      "activations/layer10_attention_weight_min": -37.44231033325195,
      "activations/layer11_attention_weight_max": 37.73767852783203,
      "activations/layer11_attention_weight_min": -36.877464294433594,
      "activations/layer12_attention_weight_max": 31.62090301513672,
      "activations/layer12_attention_weight_min": -26.337459564208984,
      "activations/layer13_attention_weight_max": 42.91789245605469,
      "activations/layer13_attention_weight_min": -35.74039840698242,
      "activations/layer14_attention_weight_max": 43.8233528137207,
      "activations/layer14_attention_weight_min": -38.51161193847656,
      "activations/layer15_attention_weight_max": 43.999698638916016,
      "activations/layer15_attention_weight_min": -35.97285079956055,
      "activations/layer16_attention_weight_max": 31.72045135498047,
      "activations/layer16_attention_weight_min": -28.65264320373535,
      "activations/layer17_attention_weight_max": 31.13376808166504,
      "activations/layer17_attention_weight_min": -25.580442428588867,
      "activations/layer18_attention_weight_max": 32.50675582885742,
      "activations/layer18_attention_weight_min": -25.005611419677734,
      "activations/layer19_attention_weight_max": 31.28132438659668,
      "activations/layer19_attention_weight_min": -31.36195182800293,
      "activations/layer1_attention_weight_max": 16.492223739624023,
      "activations/layer1_attention_weight_min": -15.389581680297852,
      "activations/layer20_attention_weight_max": 28.735994338989258,
      "activations/layer20_attention_weight_min": -26.3758602142334,
      "activations/layer21_attention_weight_max": 28.073070526123047,
      "activations/layer21_attention_weight_min": -24.368799209594727,
      "activations/layer22_attention_weight_max": 41.07746505737305,
      "activations/layer22_attention_weight_min": -30.63763427734375,
      "activations/layer23_attention_weight_max": 32.650753021240234,
      "activations/layer23_attention_weight_min": -24.29135513305664,
      "activations/layer2_attention_weight_max": 31.498764038085938,
      "activations/layer2_attention_weight_min": -31.849510192871094,
      "activations/layer3_attention_weight_max": 95.30999755859375,
      "activations/layer3_attention_weight_min": -98.23192596435547,
      "activations/layer4_attention_weight_max": 105.50594329833984,
      "activations/layer4_attention_weight_min": -109.5222396850586,
      "activations/layer5_attention_weight_max": 48.29815673828125,
      "activations/layer5_attention_weight_min": -63.45648956298828,
      "activations/layer6_attention_weight_max": 44.90863800048828,
      "activations/layer6_attention_weight_min": -48.7260856628418,
      "activations/layer7_attention_weight_max": 95.92448425292969,
      "activations/layer7_attention_weight_min": -98.08393859863281,
      "activations/layer8_attention_weight_max": 43.73337936401367,
      "activations/layer8_attention_weight_min": -45.668949127197266,
      "activations/layer9_attention_weight_max": 35.02489471435547,
      "activations/layer9_attention_weight_min": -36.415279388427734,
      "epoch": 16.78,
      "learning_rate": 4.220871212121212e-05,
      "loss": 2.7464,
      "step": 288800
    },
    {
      "activations/layer0_attention_weight_max": 15.507746696472168,
      "activations/layer0_attention_weight_min": -12.369161605834961,
      "activations/layer10_attention_weight_max": 33.3272590637207,
      "activations/layer10_attention_weight_min": -33.20499801635742,
      "activations/layer11_attention_weight_max": 31.262847900390625,
      "activations/layer11_attention_weight_min": -32.758323669433594,
      "activations/layer12_attention_weight_max": 28.18303680419922,
      "activations/layer12_attention_weight_min": -25.670442581176758,
      "activations/layer13_attention_weight_max": 39.1302604675293,
      "activations/layer13_attention_weight_min": -33.96951675415039,
      "activations/layer14_attention_weight_max": 41.66717529296875,
      "activations/layer14_attention_weight_min": -37.47276306152344,
      "activations/layer15_attention_weight_max": 35.692935943603516,
      "activations/layer15_attention_weight_min": -36.310142517089844,
      "activations/layer16_attention_weight_max": 30.654939651489258,
      "activations/layer16_attention_weight_min": -28.992395401000977,
      "activations/layer17_attention_weight_max": 29.084346771240234,
      "activations/layer17_attention_weight_min": -25.69350814819336,
      "activations/layer18_attention_weight_max": 34.819366455078125,
      "activations/layer18_attention_weight_min": -26.586870193481445,
      "activations/layer19_attention_weight_max": 32.29811477661133,
      "activations/layer19_attention_weight_min": -30.780324935913086,
      "activations/layer1_attention_weight_max": 17.291807174682617,
      "activations/layer1_attention_weight_min": -17.146957397460938,
      "activations/layer20_attention_weight_max": 28.38082504272461,
      "activations/layer20_attention_weight_min": -26.29704475402832,
      "activations/layer21_attention_weight_max": 29.82145118713379,
      "activations/layer21_attention_weight_min": -25.799057006835938,
      "activations/layer22_attention_weight_max": 40.89950180053711,
      "activations/layer22_attention_weight_min": -32.331993103027344,
      "activations/layer23_attention_weight_max": 32.35758590698242,
      "activations/layer23_attention_weight_min": -27.44461441040039,
      "activations/layer2_attention_weight_max": 33.690460205078125,
      "activations/layer2_attention_weight_min": -33.29604721069336,
      "activations/layer3_attention_weight_max": 99.3612060546875,
      "activations/layer3_attention_weight_min": -95.53556823730469,
      "activations/layer4_attention_weight_max": 106.8438491821289,
      "activations/layer4_attention_weight_min": -103.46870422363281,
      "activations/layer5_attention_weight_max": 51.39056396484375,
      "activations/layer5_attention_weight_min": -61.622100830078125,
      "activations/layer6_attention_weight_max": 42.219337463378906,
      "activations/layer6_attention_weight_min": -45.67966079711914,
      "activations/layer7_attention_weight_max": 89.1214370727539,
      "activations/layer7_attention_weight_min": -99.26508331298828,
      "activations/layer8_attention_weight_max": 39.33232498168945,
      "activations/layer8_attention_weight_min": -45.46977615356445,
      "activations/layer9_attention_weight_max": 29.6335391998291,
      "activations/layer9_attention_weight_min": -33.0547981262207,
      "epoch": 16.78,
      "learning_rate": 4.2189772727272725e-05,
      "loss": 2.7242,
      "step": 288850
    },
    {
      "activations/layer0_attention_weight_max": 15.48377799987793,
      "activations/layer0_attention_weight_min": -12.72651195526123,
      "activations/layer10_attention_weight_max": 32.96763229370117,
      "activations/layer10_attention_weight_min": -36.27039337158203,
      "activations/layer11_attention_weight_max": 34.192604064941406,
      "activations/layer11_attention_weight_min": -35.69106674194336,
      "activations/layer12_attention_weight_max": 25.775545120239258,
      "activations/layer12_attention_weight_min": -26.388690948486328,
      "activations/layer13_attention_weight_max": 36.87785339355469,
      "activations/layer13_attention_weight_min": -35.59883117675781,
      "activations/layer14_attention_weight_max": 42.29419708251953,
      "activations/layer14_attention_weight_min": -36.469444274902344,
      "activations/layer15_attention_weight_max": 35.151424407958984,
      "activations/layer15_attention_weight_min": -35.41094970703125,
      "activations/layer16_attention_weight_max": 29.430280685424805,
      "activations/layer16_attention_weight_min": -27.9744930267334,
      "activations/layer17_attention_weight_max": 29.04351234436035,
      "activations/layer17_attention_weight_min": -27.786056518554688,
      "activations/layer18_attention_weight_max": 29.101755142211914,
      "activations/layer18_attention_weight_min": -27.61224365234375,
      "activations/layer19_attention_weight_max": 35.786067962646484,
      "activations/layer19_attention_weight_min": -31.7133731842041,
      "activations/layer1_attention_weight_max": 16.72921371459961,
      "activations/layer1_attention_weight_min": -14.195818901062012,
      "activations/layer20_attention_weight_max": 28.13173484802246,
      "activations/layer20_attention_weight_min": -26.9216365814209,
      "activations/layer21_attention_weight_max": 30.066373825073242,
      "activations/layer21_attention_weight_min": -25.540225982666016,
      "activations/layer22_attention_weight_max": 39.923851013183594,
      "activations/layer22_attention_weight_min": -31.702993392944336,
      "activations/layer23_attention_weight_max": 35.86083221435547,
      "activations/layer23_attention_weight_min": -28.68537139892578,
      "activations/layer2_attention_weight_max": 32.397621154785156,
      "activations/layer2_attention_weight_min": -30.339595794677734,
      "activations/layer3_attention_weight_max": 92.40164184570312,
      "activations/layer3_attention_weight_min": -92.3417739868164,
      "activations/layer4_attention_weight_max": 102.69313049316406,
      "activations/layer4_attention_weight_min": -103.6812515258789,
      "activations/layer5_attention_weight_max": 50.033485412597656,
      "activations/layer5_attention_weight_min": -60.8186149597168,
      "activations/layer6_attention_weight_max": 41.88319778442383,
      "activations/layer6_attention_weight_min": -44.35401916503906,
      "activations/layer7_attention_weight_max": 91.36542510986328,
      "activations/layer7_attention_weight_min": -90.94424438476562,
      "activations/layer8_attention_weight_max": 40.441627502441406,
      "activations/layer8_attention_weight_min": -41.3197135925293,
      "activations/layer9_attention_weight_max": 31.351409912109375,
      "activations/layer9_attention_weight_min": -34.709808349609375,
      "epoch": 16.79,
      "learning_rate": 4.2170833333333326e-05,
      "loss": 2.7535,
      "step": 288900
    },
    {
      "activations/layer0_attention_weight_max": 15.997983932495117,
      "activations/layer0_attention_weight_min": -13.783936500549316,
      "activations/layer10_attention_weight_max": 35.33740234375,
      "activations/layer10_attention_weight_min": -34.673667907714844,
      "activations/layer11_attention_weight_max": 35.9256591796875,
      "activations/layer11_attention_weight_min": -33.65408706665039,
      "activations/layer12_attention_weight_max": 26.5245304107666,
      "activations/layer12_attention_weight_min": -27.352054595947266,
      "activations/layer13_attention_weight_max": 39.55542755126953,
      "activations/layer13_attention_weight_min": -36.51165008544922,
      "activations/layer14_attention_weight_max": 42.20299530029297,
      "activations/layer14_attention_weight_min": -40.53818130493164,
      "activations/layer15_attention_weight_max": 38.31290817260742,
      "activations/layer15_attention_weight_min": -36.94459915161133,
      "activations/layer16_attention_weight_max": 29.87985610961914,
      "activations/layer16_attention_weight_min": -29.06536865234375,
      "activations/layer17_attention_weight_max": 28.60872459411621,
      "activations/layer17_attention_weight_min": -26.17148780822754,
      "activations/layer18_attention_weight_max": 30.093326568603516,
      "activations/layer18_attention_weight_min": -25.027652740478516,
      "activations/layer19_attention_weight_max": 30.71868896484375,
      "activations/layer19_attention_weight_min": -32.7363166809082,
      "activations/layer1_attention_weight_max": 18.43720817565918,
      "activations/layer1_attention_weight_min": -16.733131408691406,
      "activations/layer20_attention_weight_max": 26.706741333007812,
      "activations/layer20_attention_weight_min": -25.718204498291016,
      "activations/layer21_attention_weight_max": 25.2783145904541,
      "activations/layer21_attention_weight_min": -24.570322036743164,
      "activations/layer22_attention_weight_max": 36.05095672607422,
      "activations/layer22_attention_weight_min": -30.2927303314209,
      "activations/layer23_attention_weight_max": 27.94247817993164,
      "activations/layer23_attention_weight_min": -26.02589225769043,
      "activations/layer2_attention_weight_max": 34.85483169555664,
      "activations/layer2_attention_weight_min": -34.668025970458984,
      "activations/layer3_attention_weight_max": 99.96167755126953,
      "activations/layer3_attention_weight_min": -98.46041107177734,
      "activations/layer4_attention_weight_max": 108.7686767578125,
      "activations/layer4_attention_weight_min": -108.9283676147461,
      "activations/layer5_attention_weight_max": 50.049354553222656,
      "activations/layer5_attention_weight_min": -60.59259033203125,
      "activations/layer6_attention_weight_max": 44.85694122314453,
      "activations/layer6_attention_weight_min": -45.71818923950195,
      "activations/layer7_attention_weight_max": 95.82714080810547,
      "activations/layer7_attention_weight_min": -94.31134033203125,
      "activations/layer8_attention_weight_max": 41.92982864379883,
      "activations/layer8_attention_weight_min": -43.4724006652832,
      "activations/layer9_attention_weight_max": 32.627235412597656,
      "activations/layer9_attention_weight_min": -34.373226165771484,
      "epoch": 16.79,
      "learning_rate": 4.2151893939393934e-05,
      "loss": 2.7421,
      "step": 288950
    },
    {
      "activations/layer0_attention_weight_max": 15.840168952941895,
      "activations/layer0_attention_weight_min": -12.491089820861816,
      "activations/layer10_attention_weight_max": 31.07285499572754,
      "activations/layer10_attention_weight_min": -33.24623489379883,
      "activations/layer11_attention_weight_max": 31.3690242767334,
      "activations/layer11_attention_weight_min": -33.66783905029297,
      "activations/layer12_attention_weight_max": 29.225120544433594,
      "activations/layer12_attention_weight_min": -26.48750114440918,
      "activations/layer13_attention_weight_max": 41.56193542480469,
      "activations/layer13_attention_weight_min": -34.208091735839844,
      "activations/layer14_attention_weight_max": 43.92577362060547,
      "activations/layer14_attention_weight_min": -36.86333465576172,
      "activations/layer15_attention_weight_max": 42.767066955566406,
      "activations/layer15_attention_weight_min": -36.08866500854492,
      "activations/layer16_attention_weight_max": 31.455278396606445,
      "activations/layer16_attention_weight_min": -28.49571418762207,
      "activations/layer17_attention_weight_max": 30.787841796875,
      "activations/layer17_attention_weight_min": -24.381103515625,
      "activations/layer18_attention_weight_max": 32.14056396484375,
      "activations/layer18_attention_weight_min": -26.13900375366211,
      "activations/layer19_attention_weight_max": 33.909236907958984,
      "activations/layer19_attention_weight_min": -31.4903507232666,
      "activations/layer1_attention_weight_max": 17.273014068603516,
      "activations/layer1_attention_weight_min": -14.564356803894043,
      "activations/layer20_attention_weight_max": 29.331968307495117,
      "activations/layer20_attention_weight_min": -24.71746826171875,
      "activations/layer21_attention_weight_max": 28.88675308227539,
      "activations/layer21_attention_weight_min": -25.108427047729492,
      "activations/layer22_attention_weight_max": 41.78230285644531,
      "activations/layer22_attention_weight_min": -31.633440017700195,
      "activations/layer23_attention_weight_max": 35.55953598022461,
      "activations/layer23_attention_weight_min": -27.023822784423828,
      "activations/layer2_attention_weight_max": 32.44200897216797,
      "activations/layer2_attention_weight_min": -30.57996368408203,
      "activations/layer3_attention_weight_max": 92.81275177001953,
      "activations/layer3_attention_weight_min": -94.3178939819336,
      "activations/layer4_attention_weight_max": 101.88582611083984,
      "activations/layer4_attention_weight_min": -104.68267059326172,
      "activations/layer5_attention_weight_max": 49.80431365966797,
      "activations/layer5_attention_weight_min": -57.31617736816406,
      "activations/layer6_attention_weight_max": 43.609066009521484,
      "activations/layer6_attention_weight_min": -46.916236877441406,
      "activations/layer7_attention_weight_max": 85.65087890625,
      "activations/layer7_attention_weight_min": -91.1911849975586,
      "activations/layer8_attention_weight_max": 40.380027770996094,
      "activations/layer8_attention_weight_min": -42.386016845703125,
      "activations/layer9_attention_weight_max": 30.80109977722168,
      "activations/layer9_attention_weight_min": -33.39553451538086,
      "epoch": 16.79,
      "learning_rate": 4.213295454545454e-05,
      "loss": 2.7449,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4436,
      "eval_samples_per_second": 508.549,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4436,
      "eval_openwebtext_samples_per_second": 508.549,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 2.0076,
      "eval_wikitext_samples_per_second": 227.132,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_lambada_loss": 2.341796875,
      "eval_lambada_ppl": 10.399907116482579,
      "eval_lambada_runtime": 9.6048,
      "eval_lambada_samples_per_second": 506.932,
      "step": 289000
    },
    {
      "activations/layer0_attention_weight_max": 15.833639144897461,
      "activations/layer0_attention_weight_min": -12.484556198120117,
      "activations/layer10_attention_weight_max": 35.35982131958008,
      "activations/layer10_attention_weight_min": -35.11479187011719,
      "activations/layer11_attention_weight_max": 34.58709716796875,
      "activations/layer11_attention_weight_min": -34.065269470214844,
      "activations/layer12_attention_weight_max": 26.191499710083008,
      "activations/layer12_attention_weight_min": -26.88831901550293,
      "activations/layer13_attention_weight_max": 39.740028381347656,
      "activations/layer13_attention_weight_min": -38.73305892944336,
      "activations/layer14_attention_weight_max": 42.06755828857422,
      "activations/layer14_attention_weight_min": -38.93697738647461,
      "activations/layer15_attention_weight_max": 42.522422790527344,
      "activations/layer15_attention_weight_min": -35.39712905883789,
      "activations/layer16_attention_weight_max": 31.027664184570312,
      "activations/layer16_attention_weight_min": -26.555599212646484,
      "activations/layer17_attention_weight_max": 29.86084747314453,
      "activations/layer17_attention_weight_min": -25.210494995117188,
      "activations/layer18_attention_weight_max": 33.758514404296875,
      "activations/layer18_attention_weight_min": -24.521547317504883,
      "activations/layer19_attention_weight_max": 36.62351608276367,
      "activations/layer19_attention_weight_min": -30.828128814697266,
      "activations/layer1_attention_weight_max": 15.661046028137207,
      "activations/layer1_attention_weight_min": -14.476605415344238,
      "activations/layer20_attention_weight_max": 31.435413360595703,
      "activations/layer20_attention_weight_min": -24.716169357299805,
      "activations/layer21_attention_weight_max": 31.99799156188965,
      "activations/layer21_attention_weight_min": -24.18120574951172,
      "activations/layer22_attention_weight_max": 46.0704460144043,
      "activations/layer22_attention_weight_min": -31.501216888427734,
      "activations/layer23_attention_weight_max": 34.28520202636719,
      "activations/layer23_attention_weight_min": -27.5977840423584,
      "activations/layer2_attention_weight_max": 32.78835678100586,
      "activations/layer2_attention_weight_min": -32.95863342285156,
      "activations/layer3_attention_weight_max": 97.75021362304688,
      "activations/layer3_attention_weight_min": -100.59123229980469,
      "activations/layer4_attention_weight_max": 108.96681213378906,
      "activations/layer4_attention_weight_min": -114.1932144165039,
      "activations/layer5_attention_weight_max": 49.85345458984375,
      "activations/layer5_attention_weight_min": -62.410316467285156,
      "activations/layer6_attention_weight_max": 44.585365295410156,
      "activations/layer6_attention_weight_min": -47.5840950012207,
      "activations/layer7_attention_weight_max": 87.38338470458984,
      "activations/layer7_attention_weight_min": -98.45996856689453,
      "activations/layer8_attention_weight_max": 42.768917083740234,
      "activations/layer8_attention_weight_min": -44.81375503540039,
      "activations/layer9_attention_weight_max": 32.11201858520508,
      "activations/layer9_attention_weight_min": -33.788330078125,
      "epoch": 16.8,
      "learning_rate": 4.2114393939393933e-05,
      "loss": 2.7343,
      "step": 289050
    },
    {
      "activations/layer0_attention_weight_max": 15.374975204467773,
      "activations/layer0_attention_weight_min": -12.625396728515625,
      "activations/layer10_attention_weight_max": 36.995216369628906,
      "activations/layer10_attention_weight_min": -35.60868453979492,
      "activations/layer11_attention_weight_max": 35.828163146972656,
      "activations/layer11_attention_weight_min": -35.03900909423828,
      "activations/layer12_attention_weight_max": 30.55573272705078,
      "activations/layer12_attention_weight_min": -26.954729080200195,
      "activations/layer13_attention_weight_max": 43.35498809814453,
      "activations/layer13_attention_weight_min": -35.05587387084961,
      "activations/layer14_attention_weight_max": 47.02261734008789,
      "activations/layer14_attention_weight_min": -38.66657257080078,
      "activations/layer15_attention_weight_max": 43.13587188720703,
      "activations/layer15_attention_weight_min": -37.392024993896484,
      "activations/layer16_attention_weight_max": 34.12007141113281,
      "activations/layer16_attention_weight_min": -28.42510223388672,
      "activations/layer17_attention_weight_max": 33.461326599121094,
      "activations/layer17_attention_weight_min": -26.845060348510742,
      "activations/layer18_attention_weight_max": 34.480712890625,
      "activations/layer18_attention_weight_min": -28.965341567993164,
      "activations/layer19_attention_weight_max": 38.29807662963867,
      "activations/layer19_attention_weight_min": -33.51203536987305,
      "activations/layer1_attention_weight_max": 16.69523811340332,
      "activations/layer1_attention_weight_min": -14.741127967834473,
      "activations/layer20_attention_weight_max": 32.66542434692383,
      "activations/layer20_attention_weight_min": -26.06743049621582,
      "activations/layer21_attention_weight_max": 32.7196044921875,
      "activations/layer21_attention_weight_min": -25.643661499023438,
      "activations/layer22_attention_weight_max": 47.605655670166016,
      "activations/layer22_attention_weight_min": -34.06447982788086,
      "activations/layer23_attention_weight_max": 38.75904846191406,
      "activations/layer23_attention_weight_min": -25.827444076538086,
      "activations/layer2_attention_weight_max": 32.075443267822266,
      "activations/layer2_attention_weight_min": -32.991668701171875,
      "activations/layer3_attention_weight_max": 98.15960693359375,
      "activations/layer3_attention_weight_min": -100.58612060546875,
      "activations/layer4_attention_weight_max": 109.09637451171875,
      "activations/layer4_attention_weight_min": -112.03813934326172,
      "activations/layer5_attention_weight_max": 55.71214294433594,
      "activations/layer5_attention_weight_min": -63.1505241394043,
      "activations/layer6_attention_weight_max": 46.22660827636719,
      "activations/layer6_attention_weight_min": -49.40220260620117,
      "activations/layer7_attention_weight_max": 99.80159759521484,
      "activations/layer7_attention_weight_min": -102.6888198852539,
      "activations/layer8_attention_weight_max": 43.72486877441406,
      "activations/layer8_attention_weight_min": -47.046688079833984,
      "activations/layer9_attention_weight_max": 34.052207946777344,
      "activations/layer9_attention_weight_min": -35.887054443359375,
      "epoch": 16.8,
      "learning_rate": 4.209545454545454e-05,
      "loss": 2.7466,
      "step": 289100
    },
    {
      "activations/layer0_attention_weight_max": 15.972615242004395,
      "activations/layer0_attention_weight_min": -12.965046882629395,
      "activations/layer10_attention_weight_max": 34.12178421020508,
      "activations/layer10_attention_weight_min": -33.41394805908203,
      "activations/layer11_attention_weight_max": 34.656551361083984,
      "activations/layer11_attention_weight_min": -34.23331832885742,
      "activations/layer12_attention_weight_max": 25.714658737182617,
      "activations/layer12_attention_weight_min": -25.956132888793945,
      "activations/layer13_attention_weight_max": 36.67216873168945,
      "activations/layer13_attention_weight_min": -35.625144958496094,
      "activations/layer14_attention_weight_max": 42.802345275878906,
      "activations/layer14_attention_weight_min": -36.72951126098633,
      "activations/layer15_attention_weight_max": 36.624263763427734,
      "activations/layer15_attention_weight_min": -34.1341438293457,
      "activations/layer16_attention_weight_max": 28.933874130249023,
      "activations/layer16_attention_weight_min": -27.561155319213867,
      "activations/layer17_attention_weight_max": 29.324737548828125,
      "activations/layer17_attention_weight_min": -25.225921630859375,
      "activations/layer18_attention_weight_max": 30.87494468688965,
      "activations/layer18_attention_weight_min": -25.325586318969727,
      "activations/layer19_attention_weight_max": 35.259681701660156,
      "activations/layer19_attention_weight_min": -31.261960983276367,
      "activations/layer1_attention_weight_max": 15.786526679992676,
      "activations/layer1_attention_weight_min": -15.22808837890625,
      "activations/layer20_attention_weight_max": 29.514081954956055,
      "activations/layer20_attention_weight_min": -24.596023559570312,
      "activations/layer21_attention_weight_max": 30.335586547851562,
      "activations/layer21_attention_weight_min": -26.218704223632812,
      "activations/layer22_attention_weight_max": 39.63423538208008,
      "activations/layer22_attention_weight_min": -33.16108322143555,
      "activations/layer23_attention_weight_max": 32.285301208496094,
      "activations/layer23_attention_weight_min": -26.124256134033203,
      "activations/layer2_attention_weight_max": 31.470434188842773,
      "activations/layer2_attention_weight_min": -32.852542877197266,
      "activations/layer3_attention_weight_max": 93.68366241455078,
      "activations/layer3_attention_weight_min": -100.39617156982422,
      "activations/layer4_attention_weight_max": 104.74776458740234,
      "activations/layer4_attention_weight_min": -108.32543182373047,
      "activations/layer5_attention_weight_max": 49.97184753417969,
      "activations/layer5_attention_weight_min": -60.183250427246094,
      "activations/layer6_attention_weight_max": 43.1075325012207,
      "activations/layer6_attention_weight_min": -47.1798210144043,
      "activations/layer7_attention_weight_max": 92.82560729980469,
      "activations/layer7_attention_weight_min": -94.02272033691406,
      "activations/layer8_attention_weight_max": 40.71519088745117,
      "activations/layer8_attention_weight_min": -41.8574104309082,
      "activations/layer9_attention_weight_max": 31.77052879333496,
      "activations/layer9_attention_weight_min": -33.52714920043945,
      "epoch": 16.8,
      "learning_rate": 4.207651515151514e-05,
      "loss": 2.7345,
      "step": 289150
    },
    {
      "activations/layer0_attention_weight_max": 15.69346809387207,
      "activations/layer0_attention_weight_min": -12.925679206848145,
      "activations/layer10_attention_weight_max": 33.7932014465332,
      "activations/layer10_attention_weight_min": -37.9613151550293,
      "activations/layer11_attention_weight_max": 33.444252014160156,
      "activations/layer11_attention_weight_min": -37.5810661315918,
      "activations/layer12_attention_weight_max": 27.31201934814453,
      "activations/layer12_attention_weight_min": -27.090795516967773,
      "activations/layer13_attention_weight_max": 38.49638366699219,
      "activations/layer13_attention_weight_min": -37.843788146972656,
      "activations/layer14_attention_weight_max": 41.902069091796875,
      "activations/layer14_attention_weight_min": -40.23419952392578,
      "activations/layer15_attention_weight_max": 39.940147399902344,
      "activations/layer15_attention_weight_min": -35.236351013183594,
      "activations/layer16_attention_weight_max": 31.26866912841797,
      "activations/layer16_attention_weight_min": -27.9017391204834,
      "activations/layer17_attention_weight_max": 29.143869400024414,
      "activations/layer17_attention_weight_min": -26.247053146362305,
      "activations/layer18_attention_weight_max": 32.41462326049805,
      "activations/layer18_attention_weight_min": -25.246822357177734,
      "activations/layer19_attention_weight_max": 31.298765182495117,
      "activations/layer19_attention_weight_min": -31.337818145751953,
      "activations/layer1_attention_weight_max": 15.644145011901855,
      "activations/layer1_attention_weight_min": -14.358059883117676,
      "activations/layer20_attention_weight_max": 28.07213592529297,
      "activations/layer20_attention_weight_min": -25.426525115966797,
      "activations/layer21_attention_weight_max": 27.78128433227539,
      "activations/layer21_attention_weight_min": -26.0223388671875,
      "activations/layer22_attention_weight_max": 44.141624450683594,
      "activations/layer22_attention_weight_min": -31.258399963378906,
      "activations/layer23_attention_weight_max": 31.990375518798828,
      "activations/layer23_attention_weight_min": -25.643186569213867,
      "activations/layer2_attention_weight_max": 32.073760986328125,
      "activations/layer2_attention_weight_min": -32.718143463134766,
      "activations/layer3_attention_weight_max": 97.78934478759766,
      "activations/layer3_attention_weight_min": -97.28035736083984,
      "activations/layer4_attention_weight_max": 107.00874328613281,
      "activations/layer4_attention_weight_min": -111.69986724853516,
      "activations/layer5_attention_weight_max": 49.742034912109375,
      "activations/layer5_attention_weight_min": -61.068946838378906,
      "activations/layer6_attention_weight_max": 43.641597747802734,
      "activations/layer6_attention_weight_min": -46.23289489746094,
      "activations/layer7_attention_weight_max": 93.26496124267578,
      "activations/layer7_attention_weight_min": -95.56842803955078,
      "activations/layer8_attention_weight_max": 42.233642578125,
      "activations/layer8_attention_weight_min": -43.960060119628906,
      "activations/layer9_attention_weight_max": 32.68966293334961,
      "activations/layer9_attention_weight_min": -37.01105880737305,
      "epoch": 16.8,
      "learning_rate": 4.205757575757576e-05,
      "loss": 2.7465,
      "step": 289200
    },
    {
      "activations/layer0_attention_weight_max": 16.053274154663086,
      "activations/layer0_attention_weight_min": -13.399572372436523,
      "activations/layer10_attention_weight_max": 32.15471267700195,
      "activations/layer10_attention_weight_min": -33.654197692871094,
      "activations/layer11_attention_weight_max": 31.03290557861328,
      "activations/layer11_attention_weight_min": -31.99139404296875,
      "activations/layer12_attention_weight_max": 27.0726261138916,
      "activations/layer12_attention_weight_min": -26.343408584594727,
      "activations/layer13_attention_weight_max": 39.190792083740234,
      "activations/layer13_attention_weight_min": -36.380157470703125,
      "activations/layer14_attention_weight_max": 42.2308235168457,
      "activations/layer14_attention_weight_min": -37.345062255859375,
      "activations/layer15_attention_weight_max": 36.11768341064453,
      "activations/layer15_attention_weight_min": -34.137577056884766,
      "activations/layer16_attention_weight_max": 26.05719566345215,
      "activations/layer16_attention_weight_min": -29.786907196044922,
      "activations/layer17_attention_weight_max": 29.20347785949707,
      "activations/layer17_attention_weight_min": -28.102231979370117,
      "activations/layer18_attention_weight_max": 32.46125411987305,
      "activations/layer18_attention_weight_min": -26.151338577270508,
      "activations/layer19_attention_weight_max": 32.20710754394531,
      "activations/layer19_attention_weight_min": -32.66215133666992,
      "activations/layer1_attention_weight_max": 16.688945770263672,
      "activations/layer1_attention_weight_min": -16.30769920349121,
      "activations/layer20_attention_weight_max": 28.773412704467773,
      "activations/layer20_attention_weight_min": -28.91399574279785,
      "activations/layer21_attention_weight_max": 29.743444442749023,
      "activations/layer21_attention_weight_min": -26.409318923950195,
      "activations/layer22_attention_weight_max": 44.37569808959961,
      "activations/layer22_attention_weight_min": -31.885786056518555,
      "activations/layer23_attention_weight_max": 30.172195434570312,
      "activations/layer23_attention_weight_min": -26.307357788085938,
      "activations/layer2_attention_weight_max": 31.9295711517334,
      "activations/layer2_attention_weight_min": -32.47523498535156,
      "activations/layer3_attention_weight_max": 96.48495483398438,
      "activations/layer3_attention_weight_min": -96.53450775146484,
      "activations/layer4_attention_weight_max": 109.0000228881836,
      "activations/layer4_attention_weight_min": -109.7015380859375,
      "activations/layer5_attention_weight_max": 48.19218444824219,
      "activations/layer5_attention_weight_min": -62.14377975463867,
      "activations/layer6_attention_weight_max": 43.41086959838867,
      "activations/layer6_attention_weight_min": -45.22239303588867,
      "activations/layer7_attention_weight_max": 86.55714416503906,
      "activations/layer7_attention_weight_min": -93.188232421875,
      "activations/layer8_attention_weight_max": 38.72761154174805,
      "activations/layer8_attention_weight_min": -41.973026275634766,
      "activations/layer9_attention_weight_max": 31.09256935119629,
      "activations/layer9_attention_weight_min": -34.02290344238281,
      "epoch": 16.81,
      "learning_rate": 4.203863636363636e-05,
      "loss": 2.7584,
      "step": 289250
    },
    {
      "activations/layer0_attention_weight_max": 16.135860443115234,
      "activations/layer0_attention_weight_min": -12.78814697265625,
      "activations/layer10_attention_weight_max": 35.17131042480469,
      "activations/layer10_attention_weight_min": -34.817440032958984,
      "activations/layer11_attention_weight_max": 34.072021484375,
      "activations/layer11_attention_weight_min": -33.93309020996094,
      "activations/layer12_attention_weight_max": 26.979108810424805,
      "activations/layer12_attention_weight_min": -27.187667846679688,
      "activations/layer13_attention_weight_max": 41.27164840698242,
      "activations/layer13_attention_weight_min": -40.49456024169922,
      "activations/layer14_attention_weight_max": 44.50886535644531,
      "activations/layer14_attention_weight_min": -42.89022445678711,
      "activations/layer15_attention_weight_max": 42.087608337402344,
      "activations/layer15_attention_weight_min": -38.908729553222656,
      "activations/layer16_attention_weight_max": 32.48017883300781,
      "activations/layer16_attention_weight_min": -28.87682342529297,
      "activations/layer17_attention_weight_max": 33.9879035949707,
      "activations/layer17_attention_weight_min": -29.18149757385254,
      "activations/layer18_attention_weight_max": 34.44615936279297,
      "activations/layer18_attention_weight_min": -28.51748275756836,
      "activations/layer19_attention_weight_max": 34.85268783569336,
      "activations/layer19_attention_weight_min": -33.8841667175293,
      "activations/layer1_attention_weight_max": 15.670827865600586,
      "activations/layer1_attention_weight_min": -15.02405071258545,
      "activations/layer20_attention_weight_max": 31.53944969177246,
      "activations/layer20_attention_weight_min": -28.045299530029297,
      "activations/layer21_attention_weight_max": 29.365909576416016,
      "activations/layer21_attention_weight_min": -25.741674423217773,
      "activations/layer22_attention_weight_max": 42.35547637939453,
      "activations/layer22_attention_weight_min": -35.769493103027344,
      "activations/layer23_attention_weight_max": 34.72267150878906,
      "activations/layer23_attention_weight_min": -29.599475860595703,
      "activations/layer2_attention_weight_max": 32.299373626708984,
      "activations/layer2_attention_weight_min": -33.2689208984375,
      "activations/layer3_attention_weight_max": 98.7325439453125,
      "activations/layer3_attention_weight_min": -101.42724609375,
      "activations/layer4_attention_weight_max": 111.56599426269531,
      "activations/layer4_attention_weight_min": -114.08438873291016,
      "activations/layer5_attention_weight_max": 53.17106628417969,
      "activations/layer5_attention_weight_min": -64.40425109863281,
      "activations/layer6_attention_weight_max": 45.01918029785156,
      "activations/layer6_attention_weight_min": -46.37931823730469,
      "activations/layer7_attention_weight_max": 92.08361053466797,
      "activations/layer7_attention_weight_min": -99.52140808105469,
      "activations/layer8_attention_weight_max": 42.91421890258789,
      "activations/layer8_attention_weight_min": -44.26716613769531,
      "activations/layer9_attention_weight_max": 35.09211349487305,
      "activations/layer9_attention_weight_min": -34.64138412475586,
      "epoch": 16.81,
      "learning_rate": 4.201969696969696e-05,
      "loss": 2.7454,
      "step": 289300
    },
    {
      "activations/layer0_attention_weight_max": 16.650598526000977,
      "activations/layer0_attention_weight_min": -12.949466705322266,
      "activations/layer10_attention_weight_max": 32.79010772705078,
      "activations/layer10_attention_weight_min": -35.15837478637695,
      "activations/layer11_attention_weight_max": 33.64949035644531,
      "activations/layer11_attention_weight_min": -34.23185729980469,
      "activations/layer12_attention_weight_max": 25.288286209106445,
      "activations/layer12_attention_weight_min": -25.75711441040039,
      "activations/layer13_attention_weight_max": 37.63459014892578,
      "activations/layer13_attention_weight_min": -34.95701599121094,
      "activations/layer14_attention_weight_max": 39.962703704833984,
      "activations/layer14_attention_weight_min": -40.924198150634766,
      "activations/layer15_attention_weight_max": 35.496986389160156,
      "activations/layer15_attention_weight_min": -33.45819091796875,
      "activations/layer16_attention_weight_max": 30.265291213989258,
      "activations/layer16_attention_weight_min": -27.814720153808594,
      "activations/layer17_attention_weight_max": 29.49761962890625,
      "activations/layer17_attention_weight_min": -25.950349807739258,
      "activations/layer18_attention_weight_max": 35.1103515625,
      "activations/layer18_attention_weight_min": -26.895305633544922,
      "activations/layer19_attention_weight_max": 34.186946868896484,
      "activations/layer19_attention_weight_min": -30.283483505249023,
      "activations/layer1_attention_weight_max": 16.653846740722656,
      "activations/layer1_attention_weight_min": -16.069583892822266,
      "activations/layer20_attention_weight_max": 28.97015953063965,
      "activations/layer20_attention_weight_min": -24.739116668701172,
      "activations/layer21_attention_weight_max": 27.975351333618164,
      "activations/layer21_attention_weight_min": -24.685422897338867,
      "activations/layer22_attention_weight_max": 42.72908020019531,
      "activations/layer22_attention_weight_min": -29.42221450805664,
      "activations/layer23_attention_weight_max": 32.84703826904297,
      "activations/layer23_attention_weight_min": -24.380535125732422,
      "activations/layer2_attention_weight_max": 32.84867477416992,
      "activations/layer2_attention_weight_min": -33.2033576965332,
      "activations/layer3_attention_weight_max": 96.30084228515625,
      "activations/layer3_attention_weight_min": -97.90676879882812,
      "activations/layer4_attention_weight_max": 108.7685546875,
      "activations/layer4_attention_weight_min": -106.44719696044922,
      "activations/layer5_attention_weight_max": 52.15203094482422,
      "activations/layer5_attention_weight_min": -60.19152069091797,
      "activations/layer6_attention_weight_max": 42.07375717163086,
      "activations/layer6_attention_weight_min": -46.579166412353516,
      "activations/layer7_attention_weight_max": 91.69225311279297,
      "activations/layer7_attention_weight_min": -93.34856414794922,
      "activations/layer8_attention_weight_max": 39.977142333984375,
      "activations/layer8_attention_weight_min": -42.50231170654297,
      "activations/layer9_attention_weight_max": 30.886249542236328,
      "activations/layer9_attention_weight_min": -34.923240661621094,
      "epoch": 16.81,
      "learning_rate": 4.200075757575757e-05,
      "loss": 2.7455,
      "step": 289350
    },
    {
      "activations/layer0_attention_weight_max": 17.082408905029297,
      "activations/layer0_attention_weight_min": -14.620104789733887,
      "activations/layer10_attention_weight_max": 32.83137130737305,
      "activations/layer10_attention_weight_min": -31.0460205078125,
      "activations/layer11_attention_weight_max": 34.312889099121094,
      "activations/layer11_attention_weight_min": -31.994176864624023,
      "activations/layer12_attention_weight_max": 28.876554489135742,
      "activations/layer12_attention_weight_min": -25.120956420898438,
      "activations/layer13_attention_weight_max": 38.848670959472656,
      "activations/layer13_attention_weight_min": -33.8865966796875,
      "activations/layer14_attention_weight_max": 42.950340270996094,
      "activations/layer14_attention_weight_min": -35.90702438354492,
      "activations/layer15_attention_weight_max": 37.928401947021484,
      "activations/layer15_attention_weight_min": -36.90245056152344,
      "activations/layer16_attention_weight_max": 36.311744689941406,
      "activations/layer16_attention_weight_min": -26.995223999023438,
      "activations/layer17_attention_weight_max": 29.005205154418945,
      "activations/layer17_attention_weight_min": -25.253154754638672,
      "activations/layer18_attention_weight_max": 30.928508758544922,
      "activations/layer18_attention_weight_min": -25.993927001953125,
      "activations/layer19_attention_weight_max": 33.348697662353516,
      "activations/layer19_attention_weight_min": -30.749481201171875,
      "activations/layer1_attention_weight_max": 16.509496688842773,
      "activations/layer1_attention_weight_min": -14.633735656738281,
      "activations/layer20_attention_weight_max": 28.73284912109375,
      "activations/layer20_attention_weight_min": -25.163406372070312,
      "activations/layer21_attention_weight_max": 29.086923599243164,
      "activations/layer21_attention_weight_min": -23.665023803710938,
      "activations/layer22_attention_weight_max": 43.15487289428711,
      "activations/layer22_attention_weight_min": -33.69470977783203,
      "activations/layer23_attention_weight_max": 37.12293243408203,
      "activations/layer23_attention_weight_min": -26.53999900817871,
      "activations/layer2_attention_weight_max": 33.72178649902344,
      "activations/layer2_attention_weight_min": -32.72981262207031,
      "activations/layer3_attention_weight_max": 92.04987335205078,
      "activations/layer3_attention_weight_min": -98.65567016601562,
      "activations/layer4_attention_weight_max": 102.3800277709961,
      "activations/layer4_attention_weight_min": -107.27398681640625,
      "activations/layer5_attention_weight_max": 51.93798828125,
      "activations/layer5_attention_weight_min": -59.25004577636719,
      "activations/layer6_attention_weight_max": 42.55023193359375,
      "activations/layer6_attention_weight_min": -44.85654067993164,
      "activations/layer7_attention_weight_max": 88.87605285644531,
      "activations/layer7_attention_weight_min": -88.58134460449219,
      "activations/layer8_attention_weight_max": 40.22469711303711,
      "activations/layer8_attention_weight_min": -41.04283905029297,
      "activations/layer9_attention_weight_max": 30.131488800048828,
      "activations/layer9_attention_weight_min": -31.587604522705078,
      "epoch": 16.82,
      "learning_rate": 4.198181818181818e-05,
      "loss": 2.74,
      "step": 289400
    },
    {
      "activations/layer0_attention_weight_max": 14.909613609313965,
      "activations/layer0_attention_weight_min": -12.018553733825684,
      "activations/layer10_attention_weight_max": 33.82876205444336,
      "activations/layer10_attention_weight_min": -33.575111389160156,
      "activations/layer11_attention_weight_max": 32.894187927246094,
      "activations/layer11_attention_weight_min": -32.75190353393555,
      "activations/layer12_attention_weight_max": 26.164289474487305,
      "activations/layer12_attention_weight_min": -26.283727645874023,
      "activations/layer13_attention_weight_max": 37.73700714111328,
      "activations/layer13_attention_weight_min": -35.0380973815918,
      "activations/layer14_attention_weight_max": 44.28452682495117,
      "activations/layer14_attention_weight_min": -39.768890380859375,
      "activations/layer15_attention_weight_max": 37.708011627197266,
      "activations/layer15_attention_weight_min": -35.20154571533203,
      "activations/layer16_attention_weight_max": 29.16105842590332,
      "activations/layer16_attention_weight_min": -29.60047149658203,
      "activations/layer17_attention_weight_max": 31.31917953491211,
      "activations/layer17_attention_weight_min": -25.915058135986328,
      "activations/layer18_attention_weight_max": 30.738862991333008,
      "activations/layer18_attention_weight_min": -25.726720809936523,
      "activations/layer19_attention_weight_max": 33.14152145385742,
      "activations/layer19_attention_weight_min": -32.35981369018555,
      "activations/layer1_attention_weight_max": 15.75320053100586,
      "activations/layer1_attention_weight_min": -13.946575164794922,
      "activations/layer20_attention_weight_max": 27.474681854248047,
      "activations/layer20_attention_weight_min": -26.434507369995117,
      "activations/layer21_attention_weight_max": 30.433727264404297,
      "activations/layer21_attention_weight_min": -25.774456024169922,
      "activations/layer22_attention_weight_max": 37.99216842651367,
      "activations/layer22_attention_weight_min": -31.79798698425293,
      "activations/layer23_attention_weight_max": 30.800533294677734,
      "activations/layer23_attention_weight_min": -25.293867111206055,
      "activations/layer2_attention_weight_max": 33.3265266418457,
      "activations/layer2_attention_weight_min": -32.358402252197266,
      "activations/layer3_attention_weight_max": 96.47071838378906,
      "activations/layer3_attention_weight_min": -96.15884399414062,
      "activations/layer4_attention_weight_max": 108.79280853271484,
      "activations/layer4_attention_weight_min": -114.17366027832031,
      "activations/layer5_attention_weight_max": 50.472984313964844,
      "activations/layer5_attention_weight_min": -62.11467742919922,
      "activations/layer6_attention_weight_max": 43.98978805541992,
      "activations/layer6_attention_weight_min": -45.27438735961914,
      "activations/layer7_attention_weight_max": 91.38623809814453,
      "activations/layer7_attention_weight_min": -89.40225982666016,
      "activations/layer8_attention_weight_max": 44.60346984863281,
      "activations/layer8_attention_weight_min": -43.94316482543945,
      "activations/layer9_attention_weight_max": 33.08846664428711,
      "activations/layer9_attention_weight_min": -33.677825927734375,
      "epoch": 16.82,
      "learning_rate": 4.1962878787878786e-05,
      "loss": 2.7144,
      "step": 289450
    },
    {
      "activations/layer0_attention_weight_max": 15.676506996154785,
      "activations/layer0_attention_weight_min": -12.874771118164062,
      "activations/layer10_attention_weight_max": 36.91383743286133,
      "activations/layer10_attention_weight_min": -33.915870666503906,
      "activations/layer11_attention_weight_max": 38.465087890625,
      "activations/layer11_attention_weight_min": -33.34016418457031,
      "activations/layer12_attention_weight_max": 25.839475631713867,
      "activations/layer12_attention_weight_min": -26.20907211303711,
      "activations/layer13_attention_weight_max": 38.00620651245117,
      "activations/layer13_attention_weight_min": -33.925270080566406,
      "activations/layer14_attention_weight_max": 43.38981628417969,
      "activations/layer14_attention_weight_min": -38.084171295166016,
      "activations/layer15_attention_weight_max": 38.85016632080078,
      "activations/layer15_attention_weight_min": -37.766475677490234,
      "activations/layer16_attention_weight_max": 30.072166442871094,
      "activations/layer16_attention_weight_min": -29.04843521118164,
      "activations/layer17_attention_weight_max": 29.326698303222656,
      "activations/layer17_attention_weight_min": -26.06858253479004,
      "activations/layer18_attention_weight_max": 30.30322265625,
      "activations/layer18_attention_weight_min": -26.900630950927734,
      "activations/layer19_attention_weight_max": 35.1878776550293,
      "activations/layer19_attention_weight_min": -30.129690170288086,
      "activations/layer1_attention_weight_max": 18.102197647094727,
      "activations/layer1_attention_weight_min": -14.535305976867676,
      "activations/layer20_attention_weight_max": 30.377391815185547,
      "activations/layer20_attention_weight_min": -25.7078857421875,
      "activations/layer21_attention_weight_max": 29.003890991210938,
      "activations/layer21_attention_weight_min": -22.825138092041016,
      "activations/layer22_attention_weight_max": 39.40812301635742,
      "activations/layer22_attention_weight_min": -29.99463653564453,
      "activations/layer23_attention_weight_max": 34.93193817138672,
      "activations/layer23_attention_weight_min": -25.138996124267578,
      "activations/layer2_attention_weight_max": 33.43079376220703,
      "activations/layer2_attention_weight_min": -30.22407341003418,
      "activations/layer3_attention_weight_max": 100.37580108642578,
      "activations/layer3_attention_weight_min": -94.1362075805664,
      "activations/layer4_attention_weight_max": 103.05213928222656,
      "activations/layer4_attention_weight_min": -104.02210998535156,
      "activations/layer5_attention_weight_max": 50.36231994628906,
      "activations/layer5_attention_weight_min": -61.60760498046875,
      "activations/layer6_attention_weight_max": 42.5763053894043,
      "activations/layer6_attention_weight_min": -42.07326889038086,
      "activations/layer7_attention_weight_max": 96.03097534179688,
      "activations/layer7_attention_weight_min": -98.00779724121094,
      "activations/layer8_attention_weight_max": 44.48526382446289,
      "activations/layer8_attention_weight_min": -42.01507568359375,
      "activations/layer9_attention_weight_max": 34.97764587402344,
      "activations/layer9_attention_weight_min": -33.958221435546875,
      "epoch": 16.82,
      "learning_rate": 4.194393939393939e-05,
      "loss": 2.7394,
      "step": 289500
    },
    {
      "activations/layer0_attention_weight_max": 15.142399787902832,
      "activations/layer0_attention_weight_min": -12.423394203186035,
      "activations/layer10_attention_weight_max": 36.23090362548828,
      "activations/layer10_attention_weight_min": -33.73707962036133,
      "activations/layer11_attention_weight_max": 34.52062225341797,
      "activations/layer11_attention_weight_min": -36.13028335571289,
      "activations/layer12_attention_weight_max": 26.9064884185791,
      "activations/layer12_attention_weight_min": -26.22646713256836,
      "activations/layer13_attention_weight_max": 37.925567626953125,
      "activations/layer13_attention_weight_min": -36.04550552368164,
      "activations/layer14_attention_weight_max": 40.97084045410156,
      "activations/layer14_attention_weight_min": -35.79251480102539,
      "activations/layer15_attention_weight_max": 38.780635833740234,
      "activations/layer15_attention_weight_min": -34.395198822021484,
      "activations/layer16_attention_weight_max": 30.19801902770996,
      "activations/layer16_attention_weight_min": -26.50545310974121,
      "activations/layer17_attention_weight_max": 34.25426483154297,
      "activations/layer17_attention_weight_min": -27.524124145507812,
      "activations/layer18_attention_weight_max": 33.134620666503906,
      "activations/layer18_attention_weight_min": -26.814739227294922,
      "activations/layer19_attention_weight_max": 36.7743034362793,
      "activations/layer19_attention_weight_min": -32.35979080200195,
      "activations/layer1_attention_weight_max": 16.165950775146484,
      "activations/layer1_attention_weight_min": -15.00631332397461,
      "activations/layer20_attention_weight_max": 32.093204498291016,
      "activations/layer20_attention_weight_min": -26.614246368408203,
      "activations/layer21_attention_weight_max": 29.742721557617188,
      "activations/layer21_attention_weight_min": -24.84781837463379,
      "activations/layer22_attention_weight_max": 39.89020538330078,
      "activations/layer22_attention_weight_min": -31.910058975219727,
      "activations/layer23_attention_weight_max": 33.35211944580078,
      "activations/layer23_attention_weight_min": -26.487667083740234,
      "activations/layer2_attention_weight_max": 34.486778259277344,
      "activations/layer2_attention_weight_min": -32.859230041503906,
      "activations/layer3_attention_weight_max": 97.74813079833984,
      "activations/layer3_attention_weight_min": -98.36434936523438,
      "activations/layer4_attention_weight_max": 106.61324310302734,
      "activations/layer4_attention_weight_min": -110.26241302490234,
      "activations/layer5_attention_weight_max": 50.24119186401367,
      "activations/layer5_attention_weight_min": -61.89363098144531,
      "activations/layer6_attention_weight_max": 43.96908950805664,
      "activations/layer6_attention_weight_min": -47.98332977294922,
      "activations/layer7_attention_weight_max": 93.80023193359375,
      "activations/layer7_attention_weight_min": -100.4584732055664,
      "activations/layer8_attention_weight_max": 43.36359405517578,
      "activations/layer8_attention_weight_min": -44.70009994506836,
      "activations/layer9_attention_weight_max": 32.44875717163086,
      "activations/layer9_attention_weight_min": -34.433502197265625,
      "epoch": 16.82,
      "learning_rate": 4.1925e-05,
      "loss": 2.7476,
      "step": 289550
    },
    {
      "activations/layer0_attention_weight_max": 15.35749626159668,
      "activations/layer0_attention_weight_min": -13.107979774475098,
      "activations/layer10_attention_weight_max": 34.460853576660156,
      "activations/layer10_attention_weight_min": -35.347412109375,
      "activations/layer11_attention_weight_max": 33.989234924316406,
      "activations/layer11_attention_weight_min": -35.9561767578125,
      "activations/layer12_attention_weight_max": 26.833473205566406,
      "activations/layer12_attention_weight_min": -26.696077346801758,
      "activations/layer13_attention_weight_max": 39.33082962036133,
      "activations/layer13_attention_weight_min": -34.92021942138672,
      "activations/layer14_attention_weight_max": 43.48709487915039,
      "activations/layer14_attention_weight_min": -38.39328384399414,
      "activations/layer15_attention_weight_max": 37.674652099609375,
      "activations/layer15_attention_weight_min": -35.70836639404297,
      "activations/layer16_attention_weight_max": 29.869340896606445,
      "activations/layer16_attention_weight_min": -27.37119483947754,
      "activations/layer17_attention_weight_max": 30.43631362915039,
      "activations/layer17_attention_weight_min": -27.768583297729492,
      "activations/layer18_attention_weight_max": 30.961942672729492,
      "activations/layer18_attention_weight_min": -27.247482299804688,
      "activations/layer19_attention_weight_max": 32.66880416870117,
      "activations/layer19_attention_weight_min": -30.668012619018555,
      "activations/layer1_attention_weight_max": 16.39780044555664,
      "activations/layer1_attention_weight_min": -14.99002456665039,
      "activations/layer20_attention_weight_max": 29.65724754333496,
      "activations/layer20_attention_weight_min": -25.711353302001953,
      "activations/layer21_attention_weight_max": 29.72412109375,
      "activations/layer21_attention_weight_min": -23.98284912109375,
      "activations/layer22_attention_weight_max": 41.99848556518555,
      "activations/layer22_attention_weight_min": -32.05104064941406,
      "activations/layer23_attention_weight_max": 32.2495002746582,
      "activations/layer23_attention_weight_min": -25.894874572753906,
      "activations/layer2_attention_weight_max": 32.731544494628906,
      "activations/layer2_attention_weight_min": -34.0345458984375,
      "activations/layer3_attention_weight_max": 99.87309265136719,
      "activations/layer3_attention_weight_min": -102.27812194824219,
      "activations/layer4_attention_weight_max": 108.94781494140625,
      "activations/layer4_attention_weight_min": -110.91051483154297,
      "activations/layer5_attention_weight_max": 49.902008056640625,
      "activations/layer5_attention_weight_min": -61.394954681396484,
      "activations/layer6_attention_weight_max": 44.713314056396484,
      "activations/layer6_attention_weight_min": -46.653865814208984,
      "activations/layer7_attention_weight_max": 92.0497055053711,
      "activations/layer7_attention_weight_min": -99.80445098876953,
      "activations/layer8_attention_weight_max": 41.65617752075195,
      "activations/layer8_attention_weight_min": -43.68696975708008,
      "activations/layer9_attention_weight_max": 33.13777542114258,
      "activations/layer9_attention_weight_min": -35.26539611816406,
      "epoch": 16.83,
      "learning_rate": 4.1906060606060604e-05,
      "loss": 2.7618,
      "step": 289600
    },
    {
      "activations/layer0_attention_weight_max": 16.279558181762695,
      "activations/layer0_attention_weight_min": -12.090105056762695,
      "activations/layer10_attention_weight_max": 36.5057373046875,
      "activations/layer10_attention_weight_min": -34.87467575073242,
      "activations/layer11_attention_weight_max": 34.77318572998047,
      "activations/layer11_attention_weight_min": -35.729461669921875,
      "activations/layer12_attention_weight_max": 29.49372673034668,
      "activations/layer12_attention_weight_min": -27.406299591064453,
      "activations/layer13_attention_weight_max": 44.64125442504883,
      "activations/layer13_attention_weight_min": -36.55455017089844,
      "activations/layer14_attention_weight_max": 46.560523986816406,
      "activations/layer14_attention_weight_min": -38.9329948425293,
      "activations/layer15_attention_weight_max": 44.95499038696289,
      "activations/layer15_attention_weight_min": -36.05968475341797,
      "activations/layer16_attention_weight_max": 31.453588485717773,
      "activations/layer16_attention_weight_min": -27.49629020690918,
      "activations/layer17_attention_weight_max": 35.754234313964844,
      "activations/layer17_attention_weight_min": -25.530176162719727,
      "activations/layer18_attention_weight_max": 36.675838470458984,
      "activations/layer18_attention_weight_min": -25.714529037475586,
      "activations/layer19_attention_weight_max": 36.935516357421875,
      "activations/layer19_attention_weight_min": -29.912534713745117,
      "activations/layer1_attention_weight_max": 16.312559127807617,
      "activations/layer1_attention_weight_min": -14.971895217895508,
      "activations/layer20_attention_weight_max": 34.9239501953125,
      "activations/layer20_attention_weight_min": -26.041160583496094,
      "activations/layer21_attention_weight_max": 34.402198791503906,
      "activations/layer21_attention_weight_min": -23.742507934570312,
      "activations/layer22_attention_weight_max": 51.80868911743164,
      "activations/layer22_attention_weight_min": -32.4625129699707,
      "activations/layer23_attention_weight_max": 37.59610366821289,
      "activations/layer23_attention_weight_min": -25.19964599609375,
      "activations/layer2_attention_weight_max": 32.686866760253906,
      "activations/layer2_attention_weight_min": -31.13695526123047,
      "activations/layer3_attention_weight_max": 91.96833801269531,
      "activations/layer3_attention_weight_min": -96.32451629638672,
      "activations/layer4_attention_weight_max": 104.6521987915039,
      "activations/layer4_attention_weight_min": -109.18609619140625,
      "activations/layer5_attention_weight_max": 50.344207763671875,
      "activations/layer5_attention_weight_min": -59.230220794677734,
      "activations/layer6_attention_weight_max": 40.826812744140625,
      "activations/layer6_attention_weight_min": -44.65843963623047,
      "activations/layer7_attention_weight_max": 95.41645050048828,
      "activations/layer7_attention_weight_min": -90.43463134765625,
      "activations/layer8_attention_weight_max": 42.10932922363281,
      "activations/layer8_attention_weight_min": -42.52443313598633,
      "activations/layer9_attention_weight_max": 34.357234954833984,
      "activations/layer9_attention_weight_min": -34.00310134887695,
      "epoch": 16.83,
      "learning_rate": 4.1887121212121206e-05,
      "loss": 2.7466,
      "step": 289650
    },
    {
      "activations/layer0_attention_weight_max": 15.838266372680664,
      "activations/layer0_attention_weight_min": -13.632037162780762,
      "activations/layer10_attention_weight_max": 35.05946731567383,
      "activations/layer10_attention_weight_min": -36.10470199584961,
      "activations/layer11_attention_weight_max": 34.37675476074219,
      "activations/layer11_attention_weight_min": -34.63543701171875,
      "activations/layer12_attention_weight_max": 27.218585968017578,
      "activations/layer12_attention_weight_min": -27.115461349487305,
      "activations/layer13_attention_weight_max": 41.952938079833984,
      "activations/layer13_attention_weight_min": -36.22492218017578,
      "activations/layer14_attention_weight_max": 43.050506591796875,
      "activations/layer14_attention_weight_min": -38.224422454833984,
      "activations/layer15_attention_weight_max": 41.88862609863281,
      "activations/layer15_attention_weight_min": -35.10563659667969,
      "activations/layer16_attention_weight_max": 30.147424697875977,
      "activations/layer16_attention_weight_min": -28.007774353027344,
      "activations/layer17_attention_weight_max": 33.36500930786133,
      "activations/layer17_attention_weight_min": -25.57289695739746,
      "activations/layer18_attention_weight_max": 32.63660430908203,
      "activations/layer18_attention_weight_min": -23.218666076660156,
      "activations/layer19_attention_weight_max": 33.992374420166016,
      "activations/layer19_attention_weight_min": -31.087432861328125,
      "activations/layer1_attention_weight_max": 16.09131622314453,
      "activations/layer1_attention_weight_min": -13.573258399963379,
      "activations/layer20_attention_weight_max": 29.631492614746094,
      "activations/layer20_attention_weight_min": -24.55864143371582,
      "activations/layer21_attention_weight_max": 27.359270095825195,
      "activations/layer21_attention_weight_min": -22.50893211364746,
      "activations/layer22_attention_weight_max": 38.837284088134766,
      "activations/layer22_attention_weight_min": -29.263776779174805,
      "activations/layer23_attention_weight_max": 30.71173095703125,
      "activations/layer23_attention_weight_min": -24.524269104003906,
      "activations/layer2_attention_weight_max": 30.985538482666016,
      "activations/layer2_attention_weight_min": -30.858203887939453,
      "activations/layer3_attention_weight_max": 95.8775405883789,
      "activations/layer3_attention_weight_min": -96.89347076416016,
      "activations/layer4_attention_weight_max": 109.06831359863281,
      "activations/layer4_attention_weight_min": -107.40755462646484,
      "activations/layer5_attention_weight_max": 50.55819320678711,
      "activations/layer5_attention_weight_min": -61.3587646484375,
      "activations/layer6_attention_weight_max": 45.069496154785156,
      "activations/layer6_attention_weight_min": -46.50639724731445,
      "activations/layer7_attention_weight_max": 92.04255676269531,
      "activations/layer7_attention_weight_min": -98.59111785888672,
      "activations/layer8_attention_weight_max": 43.00267028808594,
      "activations/layer8_attention_weight_min": -43.90303421020508,
      "activations/layer9_attention_weight_max": 34.16558837890625,
      "activations/layer9_attention_weight_min": -36.234981536865234,
      "epoch": 16.83,
      "learning_rate": 4.1868181818181814e-05,
      "loss": 2.7499,
      "step": 289700
    },
    {
      "activations/layer0_attention_weight_max": 15.74709701538086,
      "activations/layer0_attention_weight_min": -12.257976531982422,
      "activations/layer10_attention_weight_max": 34.62502670288086,
      "activations/layer10_attention_weight_min": -36.03781509399414,
      "activations/layer11_attention_weight_max": 33.45048904418945,
      "activations/layer11_attention_weight_min": -33.88392639160156,
      "activations/layer12_attention_weight_max": 30.834009170532227,
      "activations/layer12_attention_weight_min": -26.51816177368164,
      "activations/layer13_attention_weight_max": 39.10682678222656,
      "activations/layer13_attention_weight_min": -36.519535064697266,
      "activations/layer14_attention_weight_max": 46.299232482910156,
      "activations/layer14_attention_weight_min": -35.684295654296875,
      "activations/layer15_attention_weight_max": 38.380126953125,
      "activations/layer15_attention_weight_min": -34.21930694580078,
      "activations/layer16_attention_weight_max": 31.21248435974121,
      "activations/layer16_attention_weight_min": -27.68022346496582,
      "activations/layer17_attention_weight_max": 28.734079360961914,
      "activations/layer17_attention_weight_min": -25.044519424438477,
      "activations/layer18_attention_weight_max": 33.08989715576172,
      "activations/layer18_attention_weight_min": -23.82930564880371,
      "activations/layer19_attention_weight_max": 35.69590759277344,
      "activations/layer19_attention_weight_min": -30.599592208862305,
      "activations/layer1_attention_weight_max": 16.091419219970703,
      "activations/layer1_attention_weight_min": -14.473394393920898,
      "activations/layer20_attention_weight_max": 29.867679595947266,
      "activations/layer20_attention_weight_min": -24.98293113708496,
      "activations/layer21_attention_weight_max": 28.670621871948242,
      "activations/layer21_attention_weight_min": -24.622282028198242,
      "activations/layer22_attention_weight_max": 45.614768981933594,
      "activations/layer22_attention_weight_min": -32.59757995605469,
      "activations/layer23_attention_weight_max": 32.75861358642578,
      "activations/layer23_attention_weight_min": -28.56013298034668,
      "activations/layer2_attention_weight_max": 32.997886657714844,
      "activations/layer2_attention_weight_min": -32.5527229309082,
      "activations/layer3_attention_weight_max": 101.62276458740234,
      "activations/layer3_attention_weight_min": -98.79235076904297,
      "activations/layer4_attention_weight_max": 109.93668365478516,
      "activations/layer4_attention_weight_min": -108.9242935180664,
      "activations/layer5_attention_weight_max": 52.58294677734375,
      "activations/layer5_attention_weight_min": -59.38132858276367,
      "activations/layer6_attention_weight_max": 44.56343078613281,
      "activations/layer6_attention_weight_min": -47.18977355957031,
      "activations/layer7_attention_weight_max": 92.70806884765625,
      "activations/layer7_attention_weight_min": -93.89443969726562,
      "activations/layer8_attention_weight_max": 41.53425216674805,
      "activations/layer8_attention_weight_min": -45.18182373046875,
      "activations/layer9_attention_weight_max": 32.95408630371094,
      "activations/layer9_attention_weight_min": -34.50130081176758,
      "epoch": 16.84,
      "learning_rate": 4.184924242424242e-05,
      "loss": 2.7538,
      "step": 289750
    },
    {
      "activations/layer0_attention_weight_max": 16.451618194580078,
      "activations/layer0_attention_weight_min": -13.314827919006348,
      "activations/layer10_attention_weight_max": 36.97917556762695,
      "activations/layer10_attention_weight_min": -35.934974670410156,
      "activations/layer11_attention_weight_max": 39.439361572265625,
      "activations/layer11_attention_weight_min": -37.57829666137695,
      "activations/layer12_attention_weight_max": 27.761686325073242,
      "activations/layer12_attention_weight_min": -28.41493034362793,
      "activations/layer13_attention_weight_max": 44.124671936035156,
      "activations/layer13_attention_weight_min": -38.2428092956543,
      "activations/layer14_attention_weight_max": 47.22383117675781,
      "activations/layer14_attention_weight_min": -38.98415756225586,
      "activations/layer15_attention_weight_max": 43.92310333251953,
      "activations/layer15_attention_weight_min": -35.968318939208984,
      "activations/layer16_attention_weight_max": 29.181135177612305,
      "activations/layer16_attention_weight_min": -30.082305908203125,
      "activations/layer17_attention_weight_max": 31.61977767944336,
      "activations/layer17_attention_weight_min": -26.109689712524414,
      "activations/layer18_attention_weight_max": 30.069364547729492,
      "activations/layer18_attention_weight_min": -24.52230453491211,
      "activations/layer19_attention_weight_max": 32.17317581176758,
      "activations/layer19_attention_weight_min": -29.761388778686523,
      "activations/layer1_attention_weight_max": 17.093473434448242,
      "activations/layer1_attention_weight_min": -15.630903244018555,
      "activations/layer20_attention_weight_max": 26.89620590209961,
      "activations/layer20_attention_weight_min": -23.81455421447754,
      "activations/layer21_attention_weight_max": 26.49106788635254,
      "activations/layer21_attention_weight_min": -23.374704360961914,
      "activations/layer22_attention_weight_max": 41.43165588378906,
      "activations/layer22_attention_weight_min": -29.766719818115234,
      "activations/layer23_attention_weight_max": 30.163616180419922,
      "activations/layer23_attention_weight_min": -23.78879165649414,
      "activations/layer2_attention_weight_max": 34.58176040649414,
      "activations/layer2_attention_weight_min": -32.813026428222656,
      "activations/layer3_attention_weight_max": 98.91207885742188,
      "activations/layer3_attention_weight_min": -97.71981048583984,
      "activations/layer4_attention_weight_max": 110.24676513671875,
      "activations/layer4_attention_weight_min": -111.3594970703125,
      "activations/layer5_attention_weight_max": 50.21453857421875,
      "activations/layer5_attention_weight_min": -63.879844665527344,
      "activations/layer6_attention_weight_max": 45.21925354003906,
      "activations/layer6_attention_weight_min": -46.645423889160156,
      "activations/layer7_attention_weight_max": 96.65857696533203,
      "activations/layer7_attention_weight_min": -94.34156036376953,
      "activations/layer8_attention_weight_max": 45.46424102783203,
      "activations/layer8_attention_weight_min": -45.06837844848633,
      "activations/layer9_attention_weight_max": 35.90669250488281,
      "activations/layer9_attention_weight_min": -38.02267837524414,
      "epoch": 16.84,
      "learning_rate": 4.183030303030303e-05,
      "loss": 2.7336,
      "step": 289800
    },
    {
      "activations/layer0_attention_weight_max": 15.691995620727539,
      "activations/layer0_attention_weight_min": -12.066205024719238,
      "activations/layer10_attention_weight_max": 35.33189392089844,
      "activations/layer10_attention_weight_min": -33.44154739379883,
      "activations/layer11_attention_weight_max": 34.3396110534668,
      "activations/layer11_attention_weight_min": -33.716758728027344,
      "activations/layer12_attention_weight_max": 26.609962463378906,
      "activations/layer12_attention_weight_min": -31.675601959228516,
      "activations/layer13_attention_weight_max": 35.893402099609375,
      "activations/layer13_attention_weight_min": -36.19840621948242,
      "activations/layer14_attention_weight_max": 40.162357330322266,
      "activations/layer14_attention_weight_min": -38.412872314453125,
      "activations/layer15_attention_weight_max": 36.165287017822266,
      "activations/layer15_attention_weight_min": -35.36114501953125,
      "activations/layer16_attention_weight_max": 28.593822479248047,
      "activations/layer16_attention_weight_min": -26.935686111450195,
      "activations/layer17_attention_weight_max": 27.728107452392578,
      "activations/layer17_attention_weight_min": -25.797334671020508,
      "activations/layer18_attention_weight_max": 29.39434242248535,
      "activations/layer18_attention_weight_min": -24.051496505737305,
      "activations/layer19_attention_weight_max": 32.58012390136719,
      "activations/layer19_attention_weight_min": -28.223276138305664,
      "activations/layer1_attention_weight_max": 16.345626831054688,
      "activations/layer1_attention_weight_min": -15.742271423339844,
      "activations/layer20_attention_weight_max": 25.706497192382812,
      "activations/layer20_attention_weight_min": -23.723974227905273,
      "activations/layer21_attention_weight_max": 25.83617401123047,
      "activations/layer21_attention_weight_min": -23.25686264038086,
      "activations/layer22_attention_weight_max": 40.11920166015625,
      "activations/layer22_attention_weight_min": -28.88370132446289,
      "activations/layer23_attention_weight_max": 31.502418518066406,
      "activations/layer23_attention_weight_min": -24.887115478515625,
      "activations/layer2_attention_weight_max": 32.81848907470703,
      "activations/layer2_attention_weight_min": -33.79539108276367,
      "activations/layer3_attention_weight_max": 95.4864730834961,
      "activations/layer3_attention_weight_min": -97.3963851928711,
      "activations/layer4_attention_weight_max": 106.50270080566406,
      "activations/layer4_attention_weight_min": -106.6483383178711,
      "activations/layer5_attention_weight_max": 51.86363220214844,
      "activations/layer5_attention_weight_min": -61.810585021972656,
      "activations/layer6_attention_weight_max": 41.14617156982422,
      "activations/layer6_attention_weight_min": -43.845985412597656,
      "activations/layer7_attention_weight_max": 94.9516830444336,
      "activations/layer7_attention_weight_min": -92.98567199707031,
      "activations/layer8_attention_weight_max": 41.36775207519531,
      "activations/layer8_attention_weight_min": -45.73666763305664,
      "activations/layer9_attention_weight_max": 30.759912490844727,
      "activations/layer9_attention_weight_min": -33.76348876953125,
      "epoch": 16.84,
      "learning_rate": 4.181136363636363e-05,
      "loss": 2.7573,
      "step": 289850
    },
    {
      "activations/layer0_attention_weight_max": 15.0119047164917,
      "activations/layer0_attention_weight_min": -12.824503898620605,
      "activations/layer10_attention_weight_max": 37.10966873168945,
      "activations/layer10_attention_weight_min": -33.50570297241211,
      "activations/layer11_attention_weight_max": 39.229087829589844,
      "activations/layer11_attention_weight_min": -36.496498107910156,
      "activations/layer12_attention_weight_max": 23.91377067565918,
      "activations/layer12_attention_weight_min": -25.40986442565918,
      "activations/layer13_attention_weight_max": 34.83022689819336,
      "activations/layer13_attention_weight_min": -36.12539291381836,
      "activations/layer14_attention_weight_max": 38.03946304321289,
      "activations/layer14_attention_weight_min": -36.70026397705078,
      "activations/layer15_attention_weight_max": 35.800315856933594,
      "activations/layer15_attention_weight_min": -32.967811584472656,
      "activations/layer16_attention_weight_max": 29.746078491210938,
      "activations/layer16_attention_weight_min": -26.420461654663086,
      "activations/layer17_attention_weight_max": 27.110034942626953,
      "activations/layer17_attention_weight_min": -23.792652130126953,
      "activations/layer18_attention_weight_max": 32.09089279174805,
      "activations/layer18_attention_weight_min": -24.604345321655273,
      "activations/layer19_attention_weight_max": 33.622032165527344,
      "activations/layer19_attention_weight_min": -29.956058502197266,
      "activations/layer1_attention_weight_max": 15.747149467468262,
      "activations/layer1_attention_weight_min": -15.110897064208984,
      "activations/layer20_attention_weight_max": 27.737648010253906,
      "activations/layer20_attention_weight_min": -24.90533447265625,
      "activations/layer21_attention_weight_max": 27.02524757385254,
      "activations/layer21_attention_weight_min": -24.31892967224121,
      "activations/layer22_attention_weight_max": 36.812198638916016,
      "activations/layer22_attention_weight_min": -29.635272979736328,
      "activations/layer23_attention_weight_max": 32.38262939453125,
      "activations/layer23_attention_weight_min": -27.433568954467773,
      "activations/layer2_attention_weight_max": 33.226131439208984,
      "activations/layer2_attention_weight_min": -33.480751037597656,
      "activations/layer3_attention_weight_max": 100.31887817382812,
      "activations/layer3_attention_weight_min": -99.67607879638672,
      "activations/layer4_attention_weight_max": 112.51859283447266,
      "activations/layer4_attention_weight_min": -108.14720916748047,
      "activations/layer5_attention_weight_max": 54.55344772338867,
      "activations/layer5_attention_weight_min": -60.256317138671875,
      "activations/layer6_attention_weight_max": 43.67040252685547,
      "activations/layer6_attention_weight_min": -46.147335052490234,
      "activations/layer7_attention_weight_max": 98.25757598876953,
      "activations/layer7_attention_weight_min": -93.60314178466797,
      "activations/layer8_attention_weight_max": 46.28000259399414,
      "activations/layer8_attention_weight_min": -41.23845291137695,
      "activations/layer9_attention_weight_max": 31.987003326416016,
      "activations/layer9_attention_weight_min": -33.98014831542969,
      "epoch": 16.84,
      "learning_rate": 4.1792424242424233e-05,
      "loss": 2.7533,
      "step": 289900
    },
    {
      "activations/layer0_attention_weight_max": 16.282251358032227,
      "activations/layer0_attention_weight_min": -12.327512741088867,
      "activations/layer10_attention_weight_max": 32.340206146240234,
      "activations/layer10_attention_weight_min": -34.6830940246582,
      "activations/layer11_attention_weight_max": 31.645797729492188,
      "activations/layer11_attention_weight_min": -34.729862213134766,
      "activations/layer12_attention_weight_max": 26.48436737060547,
      "activations/layer12_attention_weight_min": -27.704565048217773,
      "activations/layer13_attention_weight_max": 40.31426239013672,
      "activations/layer13_attention_weight_min": -35.990814208984375,
      "activations/layer14_attention_weight_max": 44.25556945800781,
      "activations/layer14_attention_weight_min": -39.85623550415039,
      "activations/layer15_attention_weight_max": 37.89751434326172,
      "activations/layer15_attention_weight_min": -37.536231994628906,
      "activations/layer16_attention_weight_max": 33.78667449951172,
      "activations/layer16_attention_weight_min": -27.718536376953125,
      "activations/layer17_attention_weight_max": 33.730289459228516,
      "activations/layer17_attention_weight_min": -27.347610473632812,
      "activations/layer18_attention_weight_max": 34.16240692138672,
      "activations/layer18_attention_weight_min": -26.222314834594727,
      "activations/layer19_attention_weight_max": 36.906856536865234,
      "activations/layer19_attention_weight_min": -32.55143737792969,
      "activations/layer1_attention_weight_max": 16.33011245727539,
      "activations/layer1_attention_weight_min": -15.815852165222168,
      "activations/layer20_attention_weight_max": 32.59667205810547,
      "activations/layer20_attention_weight_min": -24.956613540649414,
      "activations/layer21_attention_weight_max": 29.158702850341797,
      "activations/layer21_attention_weight_min": -24.376562118530273,
      "activations/layer22_attention_weight_max": 48.286739349365234,
      "activations/layer22_attention_weight_min": -32.56755447387695,
      "activations/layer23_attention_weight_max": 31.792560577392578,
      "activations/layer23_attention_weight_min": -27.501209259033203,
      "activations/layer2_attention_weight_max": 32.03783416748047,
      "activations/layer2_attention_weight_min": -31.87347412109375,
      "activations/layer3_attention_weight_max": 93.63668060302734,
      "activations/layer3_attention_weight_min": -99.14891052246094,
      "activations/layer4_attention_weight_max": 106.064697265625,
      "activations/layer4_attention_weight_min": -105.806640625,
      "activations/layer5_attention_weight_max": 50.98704147338867,
      "activations/layer5_attention_weight_min": -62.10010528564453,
      "activations/layer6_attention_weight_max": 43.889163970947266,
      "activations/layer6_attention_weight_min": -45.4274787902832,
      "activations/layer7_attention_weight_max": 90.47718811035156,
      "activations/layer7_attention_weight_min": -91.22063446044922,
      "activations/layer8_attention_weight_max": 40.456748962402344,
      "activations/layer8_attention_weight_min": -41.66057205200195,
      "activations/layer9_attention_weight_max": 32.79518127441406,
      "activations/layer9_attention_weight_min": -33.634666442871094,
      "epoch": 16.85,
      "learning_rate": 4.177348484848485e-05,
      "loss": 2.7223,
      "step": 289950
    },
    {
      "activations/layer0_attention_weight_max": 16.86227035522461,
      "activations/layer0_attention_weight_min": -13.182307243347168,
      "activations/layer10_attention_weight_max": 31.385223388671875,
      "activations/layer10_attention_weight_min": -33.92266082763672,
      "activations/layer11_attention_weight_max": 31.21100616455078,
      "activations/layer11_attention_weight_min": -34.86839294433594,
      "activations/layer12_attention_weight_max": 24.82195472717285,
      "activations/layer12_attention_weight_min": -26.217931747436523,
      "activations/layer13_attention_weight_max": 36.274147033691406,
      "activations/layer13_attention_weight_min": -34.66013717651367,
      "activations/layer14_attention_weight_max": 38.88276672363281,
      "activations/layer14_attention_weight_min": -37.450557708740234,
      "activations/layer15_attention_weight_max": 36.30220413208008,
      "activations/layer15_attention_weight_min": -34.89363479614258,
      "activations/layer16_attention_weight_max": 31.043163299560547,
      "activations/layer16_attention_weight_min": -28.705604553222656,
      "activations/layer17_attention_weight_max": 28.265045166015625,
      "activations/layer17_attention_weight_min": -25.99788475036621,
      "activations/layer18_attention_weight_max": 30.0989933013916,
      "activations/layer18_attention_weight_min": -25.45799446105957,
      "activations/layer19_attention_weight_max": 32.44460678100586,
      "activations/layer19_attention_weight_min": -30.70208740234375,
      "activations/layer1_attention_weight_max": 16.6093692779541,
      "activations/layer1_attention_weight_min": -14.850259780883789,
      "activations/layer20_attention_weight_max": 26.909832000732422,
      "activations/layer20_attention_weight_min": -25.474102020263672,
      "activations/layer21_attention_weight_max": 26.484092712402344,
      "activations/layer21_attention_weight_min": -24.66648292541504,
      "activations/layer22_attention_weight_max": 39.26242446899414,
      "activations/layer22_attention_weight_min": -32.71055221557617,
      "activations/layer23_attention_weight_max": 32.16128158569336,
      "activations/layer23_attention_weight_min": -27.281728744506836,
      "activations/layer2_attention_weight_max": 31.5221004486084,
      "activations/layer2_attention_weight_min": -32.8216667175293,
      "activations/layer3_attention_weight_max": 94.7085189819336,
      "activations/layer3_attention_weight_min": -99.2913818359375,
      "activations/layer4_attention_weight_max": 105.4599838256836,
      "activations/layer4_attention_weight_min": -106.84300994873047,
      "activations/layer5_attention_weight_max": 50.27474594116211,
      "activations/layer5_attention_weight_min": -65.2000961303711,
      "activations/layer6_attention_weight_max": 41.87035369873047,
      "activations/layer6_attention_weight_min": -44.92231369018555,
      "activations/layer7_attention_weight_max": 89.41233825683594,
      "activations/layer7_attention_weight_min": -86.77146911621094,
      "activations/layer8_attention_weight_max": 38.32016372680664,
      "activations/layer8_attention_weight_min": -41.63154220581055,
      "activations/layer9_attention_weight_max": 30.829410552978516,
      "activations/layer9_attention_weight_min": -32.835182189941406,
      "epoch": 16.85,
      "learning_rate": 4.175454545454545e-05,
      "loss": 2.7493,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.533,
      "eval_samples_per_second": 503.222,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.533,
      "eval_openwebtext_samples_per_second": 503.222,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9583,
      "eval_wikitext_samples_per_second": 232.855,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_lambada_loss": 2.4140625,
      "eval_lambada_ppl": 11.17928486368573,
      "eval_lambada_runtime": 9.5884,
      "eval_lambada_samples_per_second": 507.799,
      "step": 290000
    },
    {
      "activations/layer0_attention_weight_max": 16.26618003845215,
      "activations/layer0_attention_weight_min": -13.34384822845459,
      "activations/layer10_attention_weight_max": 36.912994384765625,
      "activations/layer10_attention_weight_min": -35.349578857421875,
      "activations/layer11_attention_weight_max": 35.71997833251953,
      "activations/layer11_attention_weight_min": -35.01276779174805,
      "activations/layer12_attention_weight_max": 25.88262367248535,
      "activations/layer12_attention_weight_min": -26.89030647277832,
      "activations/layer13_attention_weight_max": 38.256858825683594,
      "activations/layer13_attention_weight_min": -36.192020416259766,
      "activations/layer14_attention_weight_max": 41.04829025268555,
      "activations/layer14_attention_weight_min": -39.34535217285156,
      "activations/layer15_attention_weight_max": 38.55660629272461,
      "activations/layer15_attention_weight_min": -35.24637222290039,
      "activations/layer16_attention_weight_max": 28.743844985961914,
      "activations/layer16_attention_weight_min": -29.661346435546875,
      "activations/layer17_attention_weight_max": 30.03940773010254,
      "activations/layer17_attention_weight_min": -27.984275817871094,
      "activations/layer18_attention_weight_max": 32.78978729248047,
      "activations/layer18_attention_weight_min": -27.785072326660156,
      "activations/layer19_attention_weight_max": 33.38734436035156,
      "activations/layer19_attention_weight_min": -34.25320053100586,
      "activations/layer1_attention_weight_max": 15.690885543823242,
      "activations/layer1_attention_weight_min": -15.497575759887695,
      "activations/layer20_attention_weight_max": 26.945659637451172,
      "activations/layer20_attention_weight_min": -28.161537170410156,
      "activations/layer21_attention_weight_max": 26.071928024291992,
      "activations/layer21_attention_weight_min": -27.737749099731445,
      "activations/layer22_attention_weight_max": 42.63272476196289,
      "activations/layer22_attention_weight_min": -34.807308197021484,
      "activations/layer23_attention_weight_max": 33.5035285949707,
      "activations/layer23_attention_weight_min": -30.26253890991211,
      "activations/layer2_attention_weight_max": 32.359622955322266,
      "activations/layer2_attention_weight_min": -32.80247497558594,
      "activations/layer3_attention_weight_max": 98.03863525390625,
      "activations/layer3_attention_weight_min": -100.62772369384766,
      "activations/layer4_attention_weight_max": 108.48628234863281,
      "activations/layer4_attention_weight_min": -111.10157775878906,
      "activations/layer5_attention_weight_max": 49.252777099609375,
      "activations/layer5_attention_weight_min": -60.90098571777344,
      "activations/layer6_attention_weight_max": 45.8975944519043,
      "activations/layer6_attention_weight_min": -46.440975189208984,
      "activations/layer7_attention_weight_max": 96.15107727050781,
      "activations/layer7_attention_weight_min": -95.61500549316406,
      "activations/layer8_attention_weight_max": 44.85475540161133,
      "activations/layer8_attention_weight_min": -46.754188537597656,
      "activations/layer9_attention_weight_max": 35.565574645996094,
      "activations/layer9_attention_weight_min": -34.75178527832031,
      "epoch": 16.85,
      "learning_rate": 4.173560606060606e-05,
      "loss": 2.7445,
      "step": 290050
    },
    {
      "activations/layer0_attention_weight_max": 16.84848976135254,
      "activations/layer0_attention_weight_min": -13.902628898620605,
      "activations/layer10_attention_weight_max": 41.15940475463867,
      "activations/layer10_attention_weight_min": -37.36150360107422,
      "activations/layer11_attention_weight_max": 37.87079620361328,
      "activations/layer11_attention_weight_min": -38.20069885253906,
      "activations/layer12_attention_weight_max": 30.80870246887207,
      "activations/layer12_attention_weight_min": -26.430030822753906,
      "activations/layer13_attention_weight_max": 39.16604232788086,
      "activations/layer13_attention_weight_min": -35.571258544921875,
      "activations/layer14_attention_weight_max": 44.91994094848633,
      "activations/layer14_attention_weight_min": -37.723052978515625,
      "activations/layer15_attention_weight_max": 40.3138542175293,
      "activations/layer15_attention_weight_min": -37.68572235107422,
      "activations/layer16_attention_weight_max": 31.957950592041016,
      "activations/layer16_attention_weight_min": -28.376161575317383,
      "activations/layer17_attention_weight_max": 29.44088363647461,
      "activations/layer17_attention_weight_min": -26.264171600341797,
      "activations/layer18_attention_weight_max": 30.5595760345459,
      "activations/layer18_attention_weight_min": -25.572921752929688,
      "activations/layer19_attention_weight_max": 35.291866302490234,
      "activations/layer19_attention_weight_min": -32.417457580566406,
      "activations/layer1_attention_weight_max": 16.23274040222168,
      "activations/layer1_attention_weight_min": -16.496063232421875,
      "activations/layer20_attention_weight_max": 31.001182556152344,
      "activations/layer20_attention_weight_min": -25.8417911529541,
      "activations/layer21_attention_weight_max": 32.13648223876953,
      "activations/layer21_attention_weight_min": -26.187503814697266,
      "activations/layer22_attention_weight_max": 41.280208587646484,
      "activations/layer22_attention_weight_min": -32.43018341064453,
      "activations/layer23_attention_weight_max": 32.28849792480469,
      "activations/layer23_attention_weight_min": -27.73668670654297,
      "activations/layer2_attention_weight_max": 33.47705841064453,
      "activations/layer2_attention_weight_min": -32.9748649597168,
      "activations/layer3_attention_weight_max": 97.98249053955078,
      "activations/layer3_attention_weight_min": -100.63377380371094,
      "activations/layer4_attention_weight_max": 108.17529296875,
      "activations/layer4_attention_weight_min": -107.64229583740234,
      "activations/layer5_attention_weight_max": 51.615692138671875,
      "activations/layer5_attention_weight_min": -62.28544616699219,
      "activations/layer6_attention_weight_max": 41.87499237060547,
      "activations/layer6_attention_weight_min": -44.306705474853516,
      "activations/layer7_attention_weight_max": 107.3381118774414,
      "activations/layer7_attention_weight_min": -92.91316223144531,
      "activations/layer8_attention_weight_max": 45.71084976196289,
      "activations/layer8_attention_weight_min": -43.84397506713867,
      "activations/layer9_attention_weight_max": 34.493045806884766,
      "activations/layer9_attention_weight_min": -38.376190185546875,
      "epoch": 16.86,
      "learning_rate": 4.171666666666666e-05,
      "loss": 2.7391,
      "step": 290100
    },
    {
      "activations/layer0_attention_weight_max": 15.448685646057129,
      "activations/layer0_attention_weight_min": -13.014840126037598,
      "activations/layer10_attention_weight_max": 31.914810180664062,
      "activations/layer10_attention_weight_min": -32.506324768066406,
      "activations/layer11_attention_weight_max": 32.39923095703125,
      "activations/layer11_attention_weight_min": -32.52454376220703,
      "activations/layer12_attention_weight_max": 27.674617767333984,
      "activations/layer12_attention_weight_min": -27.23050308227539,
      "activations/layer13_attention_weight_max": 38.79384231567383,
      "activations/layer13_attention_weight_min": -34.984710693359375,
      "activations/layer14_attention_weight_max": 44.31388473510742,
      "activations/layer14_attention_weight_min": -37.76583480834961,
      "activations/layer15_attention_weight_max": 38.53764343261719,
      "activations/layer15_attention_weight_min": -34.17940139770508,
      "activations/layer16_attention_weight_max": 29.8970947265625,
      "activations/layer16_attention_weight_min": -28.728492736816406,
      "activations/layer17_attention_weight_max": 31.35679817199707,
      "activations/layer17_attention_weight_min": -25.858722686767578,
      "activations/layer18_attention_weight_max": 38.476104736328125,
      "activations/layer18_attention_weight_min": -26.079227447509766,
      "activations/layer19_attention_weight_max": 35.65715408325195,
      "activations/layer19_attention_weight_min": -34.718753814697266,
      "activations/layer1_attention_weight_max": 15.788541793823242,
      "activations/layer1_attention_weight_min": -14.178276062011719,
      "activations/layer20_attention_weight_max": 34.245235443115234,
      "activations/layer20_attention_weight_min": -26.078794479370117,
      "activations/layer21_attention_weight_max": 33.22252655029297,
      "activations/layer21_attention_weight_min": -25.884801864624023,
      "activations/layer22_attention_weight_max": 45.895751953125,
      "activations/layer22_attention_weight_min": -33.71961975097656,
      "activations/layer23_attention_weight_max": 37.170230865478516,
      "activations/layer23_attention_weight_min": -25.907230377197266,
      "activations/layer2_attention_weight_max": 31.482603073120117,
      "activations/layer2_attention_weight_min": -31.43020248413086,
      "activations/layer3_attention_weight_max": 93.16975402832031,
      "activations/layer3_attention_weight_min": -92.63632202148438,
      "activations/layer4_attention_weight_max": 103.25249481201172,
      "activations/layer4_attention_weight_min": -106.4833984375,
      "activations/layer5_attention_weight_max": 49.03471374511719,
      "activations/layer5_attention_weight_min": -62.20612716674805,
      "activations/layer6_attention_weight_max": 41.27949142456055,
      "activations/layer6_attention_weight_min": -44.446842193603516,
      "activations/layer7_attention_weight_max": 86.68779754638672,
      "activations/layer7_attention_weight_min": -90.43418884277344,
      "activations/layer8_attention_weight_max": 38.6731071472168,
      "activations/layer8_attention_weight_min": -40.095054626464844,
      "activations/layer9_attention_weight_max": 30.880407333374023,
      "activations/layer9_attention_weight_min": -32.53941345214844,
      "epoch": 16.86,
      "learning_rate": 4.1697727272727275e-05,
      "loss": 2.7473,
      "step": 290150
    },
    {
      "activations/layer0_attention_weight_max": 15.812437057495117,
      "activations/layer0_attention_weight_min": -12.103256225585938,
      "activations/layer10_attention_weight_max": 32.12358474731445,
      "activations/layer10_attention_weight_min": -33.68635177612305,
      "activations/layer11_attention_weight_max": 32.79389572143555,
      "activations/layer11_attention_weight_min": -33.87279510498047,
      "activations/layer12_attention_weight_max": 26.642305374145508,
      "activations/layer12_attention_weight_min": -27.204105377197266,
      "activations/layer13_attention_weight_max": 38.70759201049805,
      "activations/layer13_attention_weight_min": -34.73450469970703,
      "activations/layer14_attention_weight_max": 41.23579788208008,
      "activations/layer14_attention_weight_min": -44.152183532714844,
      "activations/layer15_attention_weight_max": 36.750850677490234,
      "activations/layer15_attention_weight_min": -37.11057662963867,
      "activations/layer16_attention_weight_max": 30.83793830871582,
      "activations/layer16_attention_weight_min": -29.000411987304688,
      "activations/layer17_attention_weight_max": 31.18800163269043,
      "activations/layer17_attention_weight_min": -28.09912109375,
      "activations/layer18_attention_weight_max": 31.467641830444336,
      "activations/layer18_attention_weight_min": -26.108579635620117,
      "activations/layer19_attention_weight_max": 32.2244987487793,
      "activations/layer19_attention_weight_min": -32.4165153503418,
      "activations/layer1_attention_weight_max": 15.749201774597168,
      "activations/layer1_attention_weight_min": -14.962151527404785,
      "activations/layer20_attention_weight_max": 26.739355087280273,
      "activations/layer20_attention_weight_min": -27.151384353637695,
      "activations/layer21_attention_weight_max": 25.620290756225586,
      "activations/layer21_attention_weight_min": -24.80986213684082,
      "activations/layer22_attention_weight_max": 44.035274505615234,
      "activations/layer22_attention_weight_min": -32.26218795776367,
      "activations/layer23_attention_weight_max": 30.373321533203125,
      "activations/layer23_attention_weight_min": -26.11727523803711,
      "activations/layer2_attention_weight_max": 32.562870025634766,
      "activations/layer2_attention_weight_min": -31.373899459838867,
      "activations/layer3_attention_weight_max": 93.42440032958984,
      "activations/layer3_attention_weight_min": -99.32510375976562,
      "activations/layer4_attention_weight_max": 102.3402328491211,
      "activations/layer4_attention_weight_min": -107.50907135009766,
      "activations/layer5_attention_weight_max": 51.469627380371094,
      "activations/layer5_attention_weight_min": -60.184181213378906,
      "activations/layer6_attention_weight_max": 40.79231262207031,
      "activations/layer6_attention_weight_min": -43.88457489013672,
      "activations/layer7_attention_weight_max": 86.99929809570312,
      "activations/layer7_attention_weight_min": -86.30049896240234,
      "activations/layer8_attention_weight_max": 38.070491790771484,
      "activations/layer8_attention_weight_min": -39.82665252685547,
      "activations/layer9_attention_weight_max": 31.099811553955078,
      "activations/layer9_attention_weight_min": -35.50136947631836,
      "epoch": 16.86,
      "learning_rate": 4.1678787878787876e-05,
      "loss": 2.7358,
      "step": 290200
    },
    {
      "activations/layer0_attention_weight_max": 15.723742485046387,
      "activations/layer0_attention_weight_min": -12.45973014831543,
      "activations/layer10_attention_weight_max": 33.2134895324707,
      "activations/layer10_attention_weight_min": -34.02351760864258,
      "activations/layer11_attention_weight_max": 33.64179229736328,
      "activations/layer11_attention_weight_min": -33.70185470581055,
      "activations/layer12_attention_weight_max": 26.64339828491211,
      "activations/layer12_attention_weight_min": -26.810428619384766,
      "activations/layer13_attention_weight_max": 41.7006950378418,
      "activations/layer13_attention_weight_min": -34.8402214050293,
      "activations/layer14_attention_weight_max": 43.45349884033203,
      "activations/layer14_attention_weight_min": -38.45642852783203,
      "activations/layer15_attention_weight_max": 41.4388313293457,
      "activations/layer15_attention_weight_min": -35.483978271484375,
      "activations/layer16_attention_weight_max": 32.16423797607422,
      "activations/layer16_attention_weight_min": -28.17183494567871,
      "activations/layer17_attention_weight_max": 31.70535659790039,
      "activations/layer17_attention_weight_min": -25.93379020690918,
      "activations/layer18_attention_weight_max": 32.39570617675781,
      "activations/layer18_attention_weight_min": -25.127153396606445,
      "activations/layer19_attention_weight_max": 36.04685592651367,
      "activations/layer19_attention_weight_min": -33.47948455810547,
      "activations/layer1_attention_weight_max": 16.079561233520508,
      "activations/layer1_attention_weight_min": -14.691679954528809,
      "activations/layer20_attention_weight_max": 34.27927017211914,
      "activations/layer20_attention_weight_min": -26.27587127685547,
      "activations/layer21_attention_weight_max": 32.166160583496094,
      "activations/layer21_attention_weight_min": -24.56022071838379,
      "activations/layer22_attention_weight_max": 43.615806579589844,
      "activations/layer22_attention_weight_min": -29.986671447753906,
      "activations/layer23_attention_weight_max": 33.78397750854492,
      "activations/layer23_attention_weight_min": -25.825664520263672,
      "activations/layer2_attention_weight_max": 31.88481903076172,
      "activations/layer2_attention_weight_min": -32.734275817871094,
      "activations/layer3_attention_weight_max": 93.42662048339844,
      "activations/layer3_attention_weight_min": -97.47825622558594,
      "activations/layer4_attention_weight_max": 103.04998779296875,
      "activations/layer4_attention_weight_min": -104.75179290771484,
      "activations/layer5_attention_weight_max": 49.037139892578125,
      "activations/layer5_attention_weight_min": -57.8708610534668,
      "activations/layer6_attention_weight_max": 42.12239074707031,
      "activations/layer6_attention_weight_min": -46.24812316894531,
      "activations/layer7_attention_weight_max": 88.38538360595703,
      "activations/layer7_attention_weight_min": -97.32967376708984,
      "activations/layer8_attention_weight_max": 40.30257797241211,
      "activations/layer8_attention_weight_min": -43.32561492919922,
      "activations/layer9_attention_weight_max": 32.815887451171875,
      "activations/layer9_attention_weight_min": -34.1252555847168,
      "epoch": 16.87,
      "learning_rate": 4.165984848484848e-05,
      "loss": 2.7316,
      "step": 290250
    },
    {
      "activations/layer0_attention_weight_max": 15.591484069824219,
      "activations/layer0_attention_weight_min": -12.109575271606445,
      "activations/layer10_attention_weight_max": 33.54246520996094,
      "activations/layer10_attention_weight_min": -32.737972259521484,
      "activations/layer11_attention_weight_max": 34.821937561035156,
      "activations/layer11_attention_weight_min": -33.59770202636719,
      "activations/layer12_attention_weight_max": 28.168195724487305,
      "activations/layer12_attention_weight_min": -26.43809700012207,
      "activations/layer13_attention_weight_max": 44.29463577270508,
      "activations/layer13_attention_weight_min": -37.0254020690918,
      "activations/layer14_attention_weight_max": 45.17070388793945,
      "activations/layer14_attention_weight_min": -38.26653289794922,
      "activations/layer15_attention_weight_max": 44.47410583496094,
      "activations/layer15_attention_weight_min": -37.480045318603516,
      "activations/layer16_attention_weight_max": 33.34061813354492,
      "activations/layer16_attention_weight_min": -30.24765968322754,
      "activations/layer17_attention_weight_max": 32.46004867553711,
      "activations/layer17_attention_weight_min": -26.638870239257812,
      "activations/layer18_attention_weight_max": 37.23249435424805,
      "activations/layer18_attention_weight_min": -25.800798416137695,
      "activations/layer19_attention_weight_max": 38.80790328979492,
      "activations/layer19_attention_weight_min": -33.835628509521484,
      "activations/layer1_attention_weight_max": 16.813522338867188,
      "activations/layer1_attention_weight_min": -14.77680492401123,
      "activations/layer20_attention_weight_max": 33.22251892089844,
      "activations/layer20_attention_weight_min": -26.36591148376465,
      "activations/layer21_attention_weight_max": 32.172264099121094,
      "activations/layer21_attention_weight_min": -25.46126365661621,
      "activations/layer22_attention_weight_max": 49.66482925415039,
      "activations/layer22_attention_weight_min": -32.670433044433594,
      "activations/layer23_attention_weight_max": 37.415809631347656,
      "activations/layer23_attention_weight_min": -28.0074520111084,
      "activations/layer2_attention_weight_max": 32.36558532714844,
      "activations/layer2_attention_weight_min": -31.410078048706055,
      "activations/layer3_attention_weight_max": 92.30479431152344,
      "activations/layer3_attention_weight_min": -94.41226959228516,
      "activations/layer4_attention_weight_max": 101.17668914794922,
      "activations/layer4_attention_weight_min": -105.39790344238281,
      "activations/layer5_attention_weight_max": 49.53681945800781,
      "activations/layer5_attention_weight_min": -58.31742477416992,
      "activations/layer6_attention_weight_max": 42.39551544189453,
      "activations/layer6_attention_weight_min": -45.5554084777832,
      "activations/layer7_attention_weight_max": 88.0011215209961,
      "activations/layer7_attention_weight_min": -90.36995697021484,
      "activations/layer8_attention_weight_max": 40.20740509033203,
      "activations/layer8_attention_weight_min": -42.48952865600586,
      "activations/layer9_attention_weight_max": 31.109481811523438,
      "activations/layer9_attention_weight_min": -32.60877227783203,
      "epoch": 16.87,
      "learning_rate": 4.1640909090909086e-05,
      "loss": 2.7551,
      "step": 290300
    },
    {
      "activations/layer0_attention_weight_max": 16.209003448486328,
      "activations/layer0_attention_weight_min": -13.113858222961426,
      "activations/layer10_attention_weight_max": 34.52096176147461,
      "activations/layer10_attention_weight_min": -35.753623962402344,
      "activations/layer11_attention_weight_max": 34.126380920410156,
      "activations/layer11_attention_weight_min": -32.98527145385742,
      "activations/layer12_attention_weight_max": 29.28470802307129,
      "activations/layer12_attention_weight_min": -28.375293731689453,
      "activations/layer13_attention_weight_max": 41.482818603515625,
      "activations/layer13_attention_weight_min": -37.20789337158203,
      "activations/layer14_attention_weight_max": 48.60086441040039,
      "activations/layer14_attention_weight_min": -39.723419189453125,
      "activations/layer15_attention_weight_max": 42.003910064697266,
      "activations/layer15_attention_weight_min": -36.166202545166016,
      "activations/layer16_attention_weight_max": 31.62010383605957,
      "activations/layer16_attention_weight_min": -29.11760139465332,
      "activations/layer17_attention_weight_max": 34.846229553222656,
      "activations/layer17_attention_weight_min": -30.863306045532227,
      "activations/layer18_attention_weight_max": 38.3853874206543,
      "activations/layer18_attention_weight_min": -27.9171085357666,
      "activations/layer19_attention_weight_max": 42.43679428100586,
      "activations/layer19_attention_weight_min": -34.01454162597656,
      "activations/layer1_attention_weight_max": 17.155101776123047,
      "activations/layer1_attention_weight_min": -15.04819393157959,
      "activations/layer20_attention_weight_max": 36.81602478027344,
      "activations/layer20_attention_weight_min": -25.720928192138672,
      "activations/layer21_attention_weight_max": 35.81805419921875,
      "activations/layer21_attention_weight_min": -25.90982437133789,
      "activations/layer22_attention_weight_max": 56.637760162353516,
      "activations/layer22_attention_weight_min": -35.96601867675781,
      "activations/layer23_attention_weight_max": 41.9501953125,
      "activations/layer23_attention_weight_min": -28.22547721862793,
      "activations/layer2_attention_weight_max": 32.72062683105469,
      "activations/layer2_attention_weight_min": -33.65349578857422,
      "activations/layer3_attention_weight_max": 97.22701263427734,
      "activations/layer3_attention_weight_min": -99.04261016845703,
      "activations/layer4_attention_weight_max": 104.1141586303711,
      "activations/layer4_attention_weight_min": -104.3560562133789,
      "activations/layer5_attention_weight_max": 48.24671936035156,
      "activations/layer5_attention_weight_min": -63.319557189941406,
      "activations/layer6_attention_weight_max": 42.80374526977539,
      "activations/layer6_attention_weight_min": -46.460060119628906,
      "activations/layer7_attention_weight_max": 92.56975555419922,
      "activations/layer7_attention_weight_min": -96.42381286621094,
      "activations/layer8_attention_weight_max": 41.27688217163086,
      "activations/layer8_attention_weight_min": -45.35871505737305,
      "activations/layer9_attention_weight_max": 32.23653030395508,
      "activations/layer9_attention_weight_min": -35.194828033447266,
      "epoch": 16.87,
      "learning_rate": 4.1621969696969694e-05,
      "loss": 2.7392,
      "step": 290350
    },
    {
      "activations/layer0_attention_weight_max": 15.720223426818848,
      "activations/layer0_attention_weight_min": -13.104969024658203,
      "activations/layer10_attention_weight_max": 33.294921875,
      "activations/layer10_attention_weight_min": -33.56553649902344,
      "activations/layer11_attention_weight_max": 32.261444091796875,
      "activations/layer11_attention_weight_min": -33.26248550415039,
      "activations/layer12_attention_weight_max": 25.037500381469727,
      "activations/layer12_attention_weight_min": -25.955368041992188,
      "activations/layer13_attention_weight_max": 36.984371185302734,
      "activations/layer13_attention_weight_min": -35.48579406738281,
      "activations/layer14_attention_weight_max": 39.12678909301758,
      "activations/layer14_attention_weight_min": -36.53960037231445,
      "activations/layer15_attention_weight_max": 37.912776947021484,
      "activations/layer15_attention_weight_min": -35.797908782958984,
      "activations/layer16_attention_weight_max": 31.282596588134766,
      "activations/layer16_attention_weight_min": -28.912139892578125,
      "activations/layer17_attention_weight_max": 27.927213668823242,
      "activations/layer17_attention_weight_min": -26.076086044311523,
      "activations/layer18_attention_weight_max": 32.99311828613281,
      "activations/layer18_attention_weight_min": -27.00810432434082,
      "activations/layer19_attention_weight_max": 30.706457138061523,
      "activations/layer19_attention_weight_min": -32.51048278808594,
      "activations/layer1_attention_weight_max": 16.323585510253906,
      "activations/layer1_attention_weight_min": -15.47487735748291,
      "activations/layer20_attention_weight_max": 25.772897720336914,
      "activations/layer20_attention_weight_min": -25.79140853881836,
      "activations/layer21_attention_weight_max": 27.309616088867188,
      "activations/layer21_attention_weight_min": -24.984569549560547,
      "activations/layer22_attention_weight_max": 38.969478607177734,
      "activations/layer22_attention_weight_min": -31.59014892578125,
      "activations/layer23_attention_weight_max": 29.061246871948242,
      "activations/layer23_attention_weight_min": -26.814857482910156,
      "activations/layer2_attention_weight_max": 34.87165832519531,
      "activations/layer2_attention_weight_min": -32.7781982421875,
      "activations/layer3_attention_weight_max": 100.0904541015625,
      "activations/layer3_attention_weight_min": -99.61365509033203,
      "activations/layer4_attention_weight_max": 103.43505096435547,
      "activations/layer4_attention_weight_min": -104.85900115966797,
      "activations/layer5_attention_weight_max": 51.260963439941406,
      "activations/layer5_attention_weight_min": -60.778900146484375,
      "activations/layer6_attention_weight_max": 45.92875289916992,
      "activations/layer6_attention_weight_min": -45.71699142456055,
      "activations/layer7_attention_weight_max": 92.18675994873047,
      "activations/layer7_attention_weight_min": -94.65431213378906,
      "activations/layer8_attention_weight_max": 41.41572952270508,
      "activations/layer8_attention_weight_min": -44.838497161865234,
      "activations/layer9_attention_weight_max": 30.83249855041504,
      "activations/layer9_attention_weight_min": -34.31587600708008,
      "epoch": 16.87,
      "learning_rate": 4.1603030303030296e-05,
      "loss": 2.7399,
      "step": 290400
    },
    {
      "activations/layer0_attention_weight_max": 16.313495635986328,
      "activations/layer0_attention_weight_min": -11.973995208740234,
      "activations/layer10_attention_weight_max": 32.547210693359375,
      "activations/layer10_attention_weight_min": -32.778533935546875,
      "activations/layer11_attention_weight_max": 33.73915100097656,
      "activations/layer11_attention_weight_min": -32.48719024658203,
      "activations/layer12_attention_weight_max": 26.83639144897461,
      "activations/layer12_attention_weight_min": -25.89248275756836,
      "activations/layer13_attention_weight_max": 39.066612243652344,
      "activations/layer13_attention_weight_min": -35.41300582885742,
      "activations/layer14_attention_weight_max": 43.3460693359375,
      "activations/layer14_attention_weight_min": -40.82255935668945,
      "activations/layer15_attention_weight_max": 38.169437408447266,
      "activations/layer15_attention_weight_min": -35.761810302734375,
      "activations/layer16_attention_weight_max": 27.134767532348633,
      "activations/layer16_attention_weight_min": -28.90972137451172,
      "activations/layer17_attention_weight_max": 28.14927101135254,
      "activations/layer17_attention_weight_min": -25.6196231842041,
      "activations/layer18_attention_weight_max": 29.77562141418457,
      "activations/layer18_attention_weight_min": -24.11857795715332,
      "activations/layer19_attention_weight_max": 30.478992462158203,
      "activations/layer19_attention_weight_min": -31.00006103515625,
      "activations/layer1_attention_weight_max": 16.828449249267578,
      "activations/layer1_attention_weight_min": -15.087916374206543,
      "activations/layer20_attention_weight_max": 28.24819564819336,
      "activations/layer20_attention_weight_min": -25.0644588470459,
      "activations/layer21_attention_weight_max": 27.938663482666016,
      "activations/layer21_attention_weight_min": -24.942703247070312,
      "activations/layer22_attention_weight_max": 42.46848678588867,
      "activations/layer22_attention_weight_min": -31.57883644104004,
      "activations/layer23_attention_weight_max": 33.37731170654297,
      "activations/layer23_attention_weight_min": -28.037532806396484,
      "activations/layer2_attention_weight_max": 31.100894927978516,
      "activations/layer2_attention_weight_min": -31.43042755126953,
      "activations/layer3_attention_weight_max": 94.48184204101562,
      "activations/layer3_attention_weight_min": -96.26274108886719,
      "activations/layer4_attention_weight_max": 109.1684799194336,
      "activations/layer4_attention_weight_min": -105.75831604003906,
      "activations/layer5_attention_weight_max": 53.65453338623047,
      "activations/layer5_attention_weight_min": -61.28037643432617,
      "activations/layer6_attention_weight_max": 45.62459945678711,
      "activations/layer6_attention_weight_min": -46.81050109863281,
      "activations/layer7_attention_weight_max": 89.2588119506836,
      "activations/layer7_attention_weight_min": -93.0811538696289,
      "activations/layer8_attention_weight_max": 41.14735794067383,
      "activations/layer8_attention_weight_min": -42.927310943603516,
      "activations/layer9_attention_weight_max": 30.390562057495117,
      "activations/layer9_attention_weight_min": -34.22028732299805,
      "epoch": 16.88,
      "learning_rate": 4.1584090909090904e-05,
      "loss": 2.7265,
      "step": 290450
    },
    {
      "activations/layer0_attention_weight_max": 16.087156295776367,
      "activations/layer0_attention_weight_min": -13.17403793334961,
      "activations/layer10_attention_weight_max": 32.914039611816406,
      "activations/layer10_attention_weight_min": -35.6281623840332,
      "activations/layer11_attention_weight_max": 31.135066986083984,
      "activations/layer11_attention_weight_min": -32.935787200927734,
      "activations/layer12_attention_weight_max": 25.79508399963379,
      "activations/layer12_attention_weight_min": -25.996274948120117,
      "activations/layer13_attention_weight_max": 38.64278030395508,
      "activations/layer13_attention_weight_min": -34.096107482910156,
      "activations/layer14_attention_weight_max": 42.6676139831543,
      "activations/layer14_attention_weight_min": -36.9969596862793,
      "activations/layer15_attention_weight_max": 37.83065414428711,
      "activations/layer15_attention_weight_min": -34.395225524902344,
      "activations/layer16_attention_weight_max": 28.62164878845215,
      "activations/layer16_attention_weight_min": -27.866498947143555,
      "activations/layer17_attention_weight_max": 29.569108963012695,
      "activations/layer17_attention_weight_min": -25.632863998413086,
      "activations/layer18_attention_weight_max": 31.9827880859375,
      "activations/layer18_attention_weight_min": -25.475339889526367,
      "activations/layer19_attention_weight_max": 32.72944259643555,
      "activations/layer19_attention_weight_min": -30.275541305541992,
      "activations/layer1_attention_weight_max": 15.988017082214355,
      "activations/layer1_attention_weight_min": -14.708968162536621,
      "activations/layer20_attention_weight_max": 30.948368072509766,
      "activations/layer20_attention_weight_min": -25.096891403198242,
      "activations/layer21_attention_weight_max": 30.244016647338867,
      "activations/layer21_attention_weight_min": -24.546764373779297,
      "activations/layer22_attention_weight_max": 46.29848861694336,
      "activations/layer22_attention_weight_min": -31.176654815673828,
      "activations/layer23_attention_weight_max": 32.383846282958984,
      "activations/layer23_attention_weight_min": -26.649572372436523,
      "activations/layer2_attention_weight_max": 33.13317108154297,
      "activations/layer2_attention_weight_min": -33.78192901611328,
      "activations/layer3_attention_weight_max": 96.64202880859375,
      "activations/layer3_attention_weight_min": -96.2552261352539,
      "activations/layer4_attention_weight_max": 103.26521301269531,
      "activations/layer4_attention_weight_min": -110.5804443359375,
      "activations/layer5_attention_weight_max": 52.527610778808594,
      "activations/layer5_attention_weight_min": -60.14215087890625,
      "activations/layer6_attention_weight_max": 42.192832946777344,
      "activations/layer6_attention_weight_min": -44.97895050048828,
      "activations/layer7_attention_weight_max": 91.9410629272461,
      "activations/layer7_attention_weight_min": -93.11559295654297,
      "activations/layer8_attention_weight_max": 41.71902847290039,
      "activations/layer8_attention_weight_min": -41.966983795166016,
      "activations/layer9_attention_weight_max": 31.74129295349121,
      "activations/layer9_attention_weight_min": -34.24929428100586,
      "epoch": 16.88,
      "learning_rate": 4.156515151515151e-05,
      "loss": 2.7276,
      "step": 290500
    },
    {
      "activations/layer0_attention_weight_max": 15.905378341674805,
      "activations/layer0_attention_weight_min": -12.158818244934082,
      "activations/layer10_attention_weight_max": 38.256099700927734,
      "activations/layer10_attention_weight_min": -34.868408203125,
      "activations/layer11_attention_weight_max": 39.70591735839844,
      "activations/layer11_attention_weight_min": -36.02235794067383,
      "activations/layer12_attention_weight_max": 25.944108963012695,
      "activations/layer12_attention_weight_min": -26.381370544433594,
      "activations/layer13_attention_weight_max": 38.22400665283203,
      "activations/layer13_attention_weight_min": -35.886226654052734,
      "activations/layer14_attention_weight_max": 44.97794723510742,
      "activations/layer14_attention_weight_min": -36.867637634277344,
      "activations/layer15_attention_weight_max": 40.453636169433594,
      "activations/layer15_attention_weight_min": -35.590396881103516,
      "activations/layer16_attention_weight_max": 31.223772048950195,
      "activations/layer16_attention_weight_min": -29.959096908569336,
      "activations/layer17_attention_weight_max": 35.822505950927734,
      "activations/layer17_attention_weight_min": -26.813766479492188,
      "activations/layer18_attention_weight_max": 34.559322357177734,
      "activations/layer18_attention_weight_min": -26.82052230834961,
      "activations/layer19_attention_weight_max": 33.23484420776367,
      "activations/layer19_attention_weight_min": -32.66728210449219,
      "activations/layer1_attention_weight_max": 16.213802337646484,
      "activations/layer1_attention_weight_min": -15.826025009155273,
      "activations/layer20_attention_weight_max": 31.065366744995117,
      "activations/layer20_attention_weight_min": -26.291175842285156,
      "activations/layer21_attention_weight_max": 30.433317184448242,
      "activations/layer21_attention_weight_min": -26.38018035888672,
      "activations/layer22_attention_weight_max": 41.929161071777344,
      "activations/layer22_attention_weight_min": -30.2818660736084,
      "activations/layer23_attention_weight_max": 33.95183563232422,
      "activations/layer23_attention_weight_min": -25.18783187866211,
      "activations/layer2_attention_weight_max": 34.94680404663086,
      "activations/layer2_attention_weight_min": -33.08376693725586,
      "activations/layer3_attention_weight_max": 98.60735321044922,
      "activations/layer3_attention_weight_min": -96.79682159423828,
      "activations/layer4_attention_weight_max": 108.3647232055664,
      "activations/layer4_attention_weight_min": -109.67391204833984,
      "activations/layer5_attention_weight_max": 51.58431625366211,
      "activations/layer5_attention_weight_min": -61.44306564331055,
      "activations/layer6_attention_weight_max": 45.23332977294922,
      "activations/layer6_attention_weight_min": -46.858375549316406,
      "activations/layer7_attention_weight_max": 98.64192962646484,
      "activations/layer7_attention_weight_min": -95.5129623413086,
      "activations/layer8_attention_weight_max": 44.745601654052734,
      "activations/layer8_attention_weight_min": -43.5892219543457,
      "activations/layer9_attention_weight_max": 34.44065475463867,
      "activations/layer9_attention_weight_min": -35.00370407104492,
      "epoch": 16.88,
      "learning_rate": 4.154621212121212e-05,
      "loss": 2.7373,
      "step": 290550
    },
    {
      "activations/layer0_attention_weight_max": 16.440828323364258,
      "activations/layer0_attention_weight_min": -12.547956466674805,
      "activations/layer10_attention_weight_max": 33.115962982177734,
      "activations/layer10_attention_weight_min": -34.70012283325195,
      "activations/layer11_attention_weight_max": 34.460609436035156,
      "activations/layer11_attention_weight_min": -35.07139587402344,
      "activations/layer12_attention_weight_max": 27.620742797851562,
      "activations/layer12_attention_weight_min": -27.468149185180664,
      "activations/layer13_attention_weight_max": 40.51567840576172,
      "activations/layer13_attention_weight_min": -36.544586181640625,
      "activations/layer14_attention_weight_max": 39.310447692871094,
      "activations/layer14_attention_weight_min": -39.700042724609375,
      "activations/layer15_attention_weight_max": 39.82334899902344,
      "activations/layer15_attention_weight_min": -35.12819290161133,
      "activations/layer16_attention_weight_max": 30.158781051635742,
      "activations/layer16_attention_weight_min": -27.706584930419922,
      "activations/layer17_attention_weight_max": 29.55640411376953,
      "activations/layer17_attention_weight_min": -26.05088233947754,
      "activations/layer18_attention_weight_max": 32.10209655761719,
      "activations/layer18_attention_weight_min": -25.085420608520508,
      "activations/layer19_attention_weight_max": 31.554885864257812,
      "activations/layer19_attention_weight_min": -30.56276512145996,
      "activations/layer1_attention_weight_max": 15.81635856628418,
      "activations/layer1_attention_weight_min": -16.491470336914062,
      "activations/layer20_attention_weight_max": 28.510051727294922,
      "activations/layer20_attention_weight_min": -25.603376388549805,
      "activations/layer21_attention_weight_max": 28.138212203979492,
      "activations/layer21_attention_weight_min": -26.75670623779297,
      "activations/layer22_attention_weight_max": 42.62266159057617,
      "activations/layer22_attention_weight_min": -31.176660537719727,
      "activations/layer23_attention_weight_max": 32.53080749511719,
      "activations/layer23_attention_weight_min": -24.524211883544922,
      "activations/layer2_attention_weight_max": 32.491859436035156,
      "activations/layer2_attention_weight_min": -33.211265563964844,
      "activations/layer3_attention_weight_max": 96.47954559326172,
      "activations/layer3_attention_weight_min": -100.72049713134766,
      "activations/layer4_attention_weight_max": 107.33184051513672,
      "activations/layer4_attention_weight_min": -106.5096664428711,
      "activations/layer5_attention_weight_max": 52.96208953857422,
      "activations/layer5_attention_weight_min": -64.9151840209961,
      "activations/layer6_attention_weight_max": 43.45258712768555,
      "activations/layer6_attention_weight_min": -47.173458099365234,
      "activations/layer7_attention_weight_max": 91.14520263671875,
      "activations/layer7_attention_weight_min": -100.07179260253906,
      "activations/layer8_attention_weight_max": 41.7056770324707,
      "activations/layer8_attention_weight_min": -47.12254333496094,
      "activations/layer9_attention_weight_max": 33.324161529541016,
      "activations/layer9_attention_weight_min": -36.298336029052734,
      "epoch": 16.89,
      "learning_rate": 4.152727272727272e-05,
      "loss": 2.7492,
      "step": 290600
    },
    {
      "activations/layer0_attention_weight_max": 16.5275936126709,
      "activations/layer0_attention_weight_min": -13.556377410888672,
      "activations/layer10_attention_weight_max": 32.25042724609375,
      "activations/layer10_attention_weight_min": -36.68238067626953,
      "activations/layer11_attention_weight_max": 32.97398376464844,
      "activations/layer11_attention_weight_min": -34.31046676635742,
      "activations/layer12_attention_weight_max": 26.804920196533203,
      "activations/layer12_attention_weight_min": -27.072006225585938,
      "activations/layer13_attention_weight_max": 41.16593551635742,
      "activations/layer13_attention_weight_min": -35.68677520751953,
      "activations/layer14_attention_weight_max": 48.35874938964844,
      "activations/layer14_attention_weight_min": -40.789241790771484,
      "activations/layer15_attention_weight_max": 39.21017074584961,
      "activations/layer15_attention_weight_min": -37.03825378417969,
      "activations/layer16_attention_weight_max": 29.87010955810547,
      "activations/layer16_attention_weight_min": -27.428712844848633,
      "activations/layer17_attention_weight_max": 31.09764289855957,
      "activations/layer17_attention_weight_min": -24.94285774230957,
      "activations/layer18_attention_weight_max": 32.45772171020508,
      "activations/layer18_attention_weight_min": -24.84648895263672,
      "activations/layer19_attention_weight_max": 38.50338363647461,
      "activations/layer19_attention_weight_min": -30.81565284729004,
      "activations/layer1_attention_weight_max": 16.017810821533203,
      "activations/layer1_attention_weight_min": -14.27515697479248,
      "activations/layer20_attention_weight_max": 32.363590240478516,
      "activations/layer20_attention_weight_min": -25.411144256591797,
      "activations/layer21_attention_weight_max": 32.60376739501953,
      "activations/layer21_attention_weight_min": -26.008634567260742,
      "activations/layer22_attention_weight_max": 45.386192321777344,
      "activations/layer22_attention_weight_min": -34.04259490966797,
      "activations/layer23_attention_weight_max": 34.809085845947266,
      "activations/layer23_attention_weight_min": -28.09601593017578,
      "activations/layer2_attention_weight_max": 32.92176055908203,
      "activations/layer2_attention_weight_min": -31.827444076538086,
      "activations/layer3_attention_weight_max": 99.44512939453125,
      "activations/layer3_attention_weight_min": -98.99617004394531,
      "activations/layer4_attention_weight_max": 104.36876678466797,
      "activations/layer4_attention_weight_min": -107.44579315185547,
      "activations/layer5_attention_weight_max": 47.888790130615234,
      "activations/layer5_attention_weight_min": -62.82326889038086,
      "activations/layer6_attention_weight_max": 45.154476165771484,
      "activations/layer6_attention_weight_min": -47.203575134277344,
      "activations/layer7_attention_weight_max": 92.29974365234375,
      "activations/layer7_attention_weight_min": -92.16827392578125,
      "activations/layer8_attention_weight_max": 40.40240478515625,
      "activations/layer8_attention_weight_min": -41.185550689697266,
      "activations/layer9_attention_weight_max": 30.619903564453125,
      "activations/layer9_attention_weight_min": -35.046566009521484,
      "epoch": 16.89,
      "learning_rate": 4.1508333333333324e-05,
      "loss": 2.74,
      "step": 290650
    },
    {
      "activations/layer0_attention_weight_max": 16.04296112060547,
      "activations/layer0_attention_weight_min": -12.33596134185791,
      "activations/layer10_attention_weight_max": 32.13343811035156,
      "activations/layer10_attention_weight_min": -32.53541946411133,
      "activations/layer11_attention_weight_max": 31.264684677124023,
      "activations/layer11_attention_weight_min": -31.694683074951172,
      "activations/layer12_attention_weight_max": 24.53753089904785,
      "activations/layer12_attention_weight_min": -25.98808479309082,
      "activations/layer13_attention_weight_max": 40.94863510131836,
      "activations/layer13_attention_weight_min": -38.66306686401367,
      "activations/layer14_attention_weight_max": 42.36227035522461,
      "activations/layer14_attention_weight_min": -36.63334655761719,
      "activations/layer15_attention_weight_max": 40.30028533935547,
      "activations/layer15_attention_weight_min": -34.09971237182617,
      "activations/layer16_attention_weight_max": 28.365970611572266,
      "activations/layer16_attention_weight_min": -28.321779251098633,
      "activations/layer17_attention_weight_max": 31.56684684753418,
      "activations/layer17_attention_weight_min": -25.923118591308594,
      "activations/layer18_attention_weight_max": 31.970487594604492,
      "activations/layer18_attention_weight_min": -24.204607009887695,
      "activations/layer19_attention_weight_max": 33.38703536987305,
      "activations/layer19_attention_weight_min": -30.087369918823242,
      "activations/layer1_attention_weight_max": 16.226041793823242,
      "activations/layer1_attention_weight_min": -14.455948829650879,
      "activations/layer20_attention_weight_max": 28.301313400268555,
      "activations/layer20_attention_weight_min": -23.757856369018555,
      "activations/layer21_attention_weight_max": 27.812644958496094,
      "activations/layer21_attention_weight_min": -23.58115005493164,
      "activations/layer22_attention_weight_max": 40.262481689453125,
      "activations/layer22_attention_weight_min": -29.169410705566406,
      "activations/layer23_attention_weight_max": 31.757598876953125,
      "activations/layer23_attention_weight_min": -25.46705436706543,
      "activations/layer2_attention_weight_max": 32.93401336669922,
      "activations/layer2_attention_weight_min": -32.35690689086914,
      "activations/layer3_attention_weight_max": 96.09581756591797,
      "activations/layer3_attention_weight_min": -98.17542266845703,
      "activations/layer4_attention_weight_max": 105.11002349853516,
      "activations/layer4_attention_weight_min": -105.9302978515625,
      "activations/layer5_attention_weight_max": 48.241451263427734,
      "activations/layer5_attention_weight_min": -60.709808349609375,
      "activations/layer6_attention_weight_max": 42.316856384277344,
      "activations/layer6_attention_weight_min": -47.85834503173828,
      "activations/layer7_attention_weight_max": 86.4320068359375,
      "activations/layer7_attention_weight_min": -90.87276458740234,
      "activations/layer8_attention_weight_max": 37.15687942504883,
      "activations/layer8_attention_weight_min": -42.151123046875,
      "activations/layer9_attention_weight_max": 31.309301376342773,
      "activations/layer9_attention_weight_min": -31.849714279174805,
      "epoch": 16.89,
      "learning_rate": 4.148939393939394e-05,
      "loss": 2.7436,
      "step": 290700
    },
    {
      "activations/layer0_attention_weight_max": 16.22848129272461,
      "activations/layer0_attention_weight_min": -12.787013053894043,
      "activations/layer10_attention_weight_max": 31.781230926513672,
      "activations/layer10_attention_weight_min": -32.12483215332031,
      "activations/layer11_attention_weight_max": 33.7150993347168,
      "activations/layer11_attention_weight_min": -31.95439910888672,
      "activations/layer12_attention_weight_max": 26.74295425415039,
      "activations/layer12_attention_weight_min": -25.744647979736328,
      "activations/layer13_attention_weight_max": 39.349205017089844,
      "activations/layer13_attention_weight_min": -35.1646842956543,
      "activations/layer14_attention_weight_max": 41.43473434448242,
      "activations/layer14_attention_weight_min": -37.69674301147461,
      "activations/layer15_attention_weight_max": 38.36305236816406,
      "activations/layer15_attention_weight_min": -35.3237190246582,
      "activations/layer16_attention_weight_max": 28.97035026550293,
      "activations/layer16_attention_weight_min": -28.691734313964844,
      "activations/layer17_attention_weight_max": 32.90406036376953,
      "activations/layer17_attention_weight_min": -26.258068084716797,
      "activations/layer18_attention_weight_max": 38.77655029296875,
      "activations/layer18_attention_weight_min": -26.627986907958984,
      "activations/layer19_attention_weight_max": 36.34721374511719,
      "activations/layer19_attention_weight_min": -32.23575973510742,
      "activations/layer1_attention_weight_max": 16.292325973510742,
      "activations/layer1_attention_weight_min": -14.979362487792969,
      "activations/layer20_attention_weight_max": 30.826671600341797,
      "activations/layer20_attention_weight_min": -24.492347717285156,
      "activations/layer21_attention_weight_max": 33.97616195678711,
      "activations/layer21_attention_weight_min": -25.44364356994629,
      "activations/layer22_attention_weight_max": 44.6420783996582,
      "activations/layer22_attention_weight_min": -31.623966217041016,
      "activations/layer23_attention_weight_max": 35.697242736816406,
      "activations/layer23_attention_weight_min": -29.2646541595459,
      "activations/layer2_attention_weight_max": 34.38404083251953,
      "activations/layer2_attention_weight_min": -32.991371154785156,
      "activations/layer3_attention_weight_max": 97.6647720336914,
      "activations/layer3_attention_weight_min": -99.95665740966797,
      "activations/layer4_attention_weight_max": 106.69439697265625,
      "activations/layer4_attention_weight_min": -109.06904602050781,
      "activations/layer5_attention_weight_max": 47.4544677734375,
      "activations/layer5_attention_weight_min": -61.282100677490234,
      "activations/layer6_attention_weight_max": 42.33439636230469,
      "activations/layer6_attention_weight_min": -45.165138244628906,
      "activations/layer7_attention_weight_max": 91.48027038574219,
      "activations/layer7_attention_weight_min": -96.0477066040039,
      "activations/layer8_attention_weight_max": 38.79213333129883,
      "activations/layer8_attention_weight_min": -41.65238952636719,
      "activations/layer9_attention_weight_max": 32.56325912475586,
      "activations/layer9_attention_weight_min": -32.820125579833984,
      "epoch": 16.89,
      "learning_rate": 4.147045454545454e-05,
      "loss": 2.7313,
      "step": 290750
    },
    {
      "activations/layer0_attention_weight_max": 16.705650329589844,
      "activations/layer0_attention_weight_min": -13.127368927001953,
      "activations/layer10_attention_weight_max": 36.85612869262695,
      "activations/layer10_attention_weight_min": -35.43997573852539,
      "activations/layer11_attention_weight_max": 36.69071578979492,
      "activations/layer11_attention_weight_min": -34.21784591674805,
      "activations/layer12_attention_weight_max": 25.05048370361328,
      "activations/layer12_attention_weight_min": -24.74842643737793,
      "activations/layer13_attention_weight_max": 36.86619567871094,
      "activations/layer13_attention_weight_min": -33.66847229003906,
      "activations/layer14_attention_weight_max": 40.11207962036133,
      "activations/layer14_attention_weight_min": -35.72328186035156,
      "activations/layer15_attention_weight_max": 36.298004150390625,
      "activations/layer15_attention_weight_min": -36.82609939575195,
      "activations/layer16_attention_weight_max": 27.113311767578125,
      "activations/layer16_attention_weight_min": -28.593900680541992,
      "activations/layer17_attention_weight_max": 30.682111740112305,
      "activations/layer17_attention_weight_min": -25.806608200073242,
      "activations/layer18_attention_weight_max": 31.9776611328125,
      "activations/layer18_attention_weight_min": -27.97549057006836,
      "activations/layer19_attention_weight_max": 30.396102905273438,
      "activations/layer19_attention_weight_min": -30.757108688354492,
      "activations/layer1_attention_weight_max": 17.253173828125,
      "activations/layer1_attention_weight_min": -14.239827156066895,
      "activations/layer20_attention_weight_max": 26.57732582092285,
      "activations/layer20_attention_weight_min": -25.495267868041992,
      "activations/layer21_attention_weight_max": 27.811019897460938,
      "activations/layer21_attention_weight_min": -24.574949264526367,
      "activations/layer22_attention_weight_max": 40.01806640625,
      "activations/layer22_attention_weight_min": -30.48470687866211,
      "activations/layer23_attention_weight_max": 32.6615104675293,
      "activations/layer23_attention_weight_min": -26.31080436706543,
      "activations/layer2_attention_weight_max": 32.186126708984375,
      "activations/layer2_attention_weight_min": -33.83596420288086,
      "activations/layer3_attention_weight_max": 95.66967010498047,
      "activations/layer3_attention_weight_min": -100.63838195800781,
      "activations/layer4_attention_weight_max": 102.3984603881836,
      "activations/layer4_attention_weight_min": -107.55680084228516,
      "activations/layer5_attention_weight_max": 50.980194091796875,
      "activations/layer5_attention_weight_min": -61.834022521972656,
      "activations/layer6_attention_weight_max": 41.953834533691406,
      "activations/layer6_attention_weight_min": -44.61116027832031,
      "activations/layer7_attention_weight_max": 97.90129089355469,
      "activations/layer7_attention_weight_min": -89.07109832763672,
      "activations/layer8_attention_weight_max": 41.532012939453125,
      "activations/layer8_attention_weight_min": -43.2139778137207,
      "activations/layer9_attention_weight_max": 32.382259368896484,
      "activations/layer9_attention_weight_min": -32.78655242919922,
      "epoch": 16.9,
      "learning_rate": 4.145151515151515e-05,
      "loss": 2.7496,
      "step": 290800
    },
    {
      "activations/layer0_attention_weight_max": 15.200784683227539,
      "activations/layer0_attention_weight_min": -12.62929916381836,
      "activations/layer10_attention_weight_max": 41.074005126953125,
      "activations/layer10_attention_weight_min": -37.567169189453125,
      "activations/layer11_attention_weight_max": 39.99956512451172,
      "activations/layer11_attention_weight_min": -39.44571304321289,
      "activations/layer12_attention_weight_max": 26.696529388427734,
      "activations/layer12_attention_weight_min": -27.56443977355957,
      "activations/layer13_attention_weight_max": 41.461997985839844,
      "activations/layer13_attention_weight_min": -37.20504379272461,
      "activations/layer14_attention_weight_max": 44.994728088378906,
      "activations/layer14_attention_weight_min": -40.00072479248047,
      "activations/layer15_attention_weight_max": 40.83869552612305,
      "activations/layer15_attention_weight_min": -39.335792541503906,
      "activations/layer16_attention_weight_max": 32.5008544921875,
      "activations/layer16_attention_weight_min": -27.49293327331543,
      "activations/layer17_attention_weight_max": 30.850589752197266,
      "activations/layer17_attention_weight_min": -26.08808135986328,
      "activations/layer18_attention_weight_max": 31.608016967773438,
      "activations/layer18_attention_weight_min": -25.704809188842773,
      "activations/layer19_attention_weight_max": 35.79753494262695,
      "activations/layer19_attention_weight_min": -31.322385787963867,
      "activations/layer1_attention_weight_max": 16.44500732421875,
      "activations/layer1_attention_weight_min": -14.979249000549316,
      "activations/layer20_attention_weight_max": 30.446943283081055,
      "activations/layer20_attention_weight_min": -24.991308212280273,
      "activations/layer21_attention_weight_max": 28.84935188293457,
      "activations/layer21_attention_weight_min": -23.901500701904297,
      "activations/layer22_attention_weight_max": 42.453941345214844,
      "activations/layer22_attention_weight_min": -33.675575256347656,
      "activations/layer23_attention_weight_max": 36.59648132324219,
      "activations/layer23_attention_weight_min": -26.458908081054688,
      "activations/layer2_attention_weight_max": 36.03417205810547,
      "activations/layer2_attention_weight_min": -34.432682037353516,
      "activations/layer3_attention_weight_max": 102.56720733642578,
      "activations/layer3_attention_weight_min": -104.47740173339844,
      "activations/layer4_attention_weight_max": 110.50164794921875,
      "activations/layer4_attention_weight_min": -106.90325164794922,
      "activations/layer5_attention_weight_max": 54.48933410644531,
      "activations/layer5_attention_weight_min": -61.57331848144531,
      "activations/layer6_attention_weight_max": 43.32109451293945,
      "activations/layer6_attention_weight_min": -44.512245178222656,
      "activations/layer7_attention_weight_max": 102.1158676147461,
      "activations/layer7_attention_weight_min": -93.55037689208984,
      "activations/layer8_attention_weight_max": 44.721073150634766,
      "activations/layer8_attention_weight_min": -44.34946060180664,
      "activations/layer9_attention_weight_max": 36.321617126464844,
      "activations/layer9_attention_weight_min": -37.14181137084961,
      "epoch": 16.9,
      "learning_rate": 4.143257575757575e-05,
      "loss": 2.7431,
      "step": 290850
    },
    {
      "activations/layer0_attention_weight_max": 14.828350067138672,
      "activations/layer0_attention_weight_min": -12.407654762268066,
      "activations/layer10_attention_weight_max": 35.610286712646484,
      "activations/layer10_attention_weight_min": -34.08671951293945,
      "activations/layer11_attention_weight_max": 34.88086700439453,
      "activations/layer11_attention_weight_min": -36.1309928894043,
      "activations/layer12_attention_weight_max": 26.80072593688965,
      "activations/layer12_attention_weight_min": -24.88547706604004,
      "activations/layer13_attention_weight_max": 39.78420639038086,
      "activations/layer13_attention_weight_min": -34.47774887084961,
      "activations/layer14_attention_weight_max": 41.457157135009766,
      "activations/layer14_attention_weight_min": -36.85578536987305,
      "activations/layer15_attention_weight_max": 37.53195571899414,
      "activations/layer15_attention_weight_min": -33.83831024169922,
      "activations/layer16_attention_weight_max": 28.62485122680664,
      "activations/layer16_attention_weight_min": -28.1071720123291,
      "activations/layer17_attention_weight_max": 28.954797744750977,
      "activations/layer17_attention_weight_min": -26.162094116210938,
      "activations/layer18_attention_weight_max": 29.6153621673584,
      "activations/layer18_attention_weight_min": -24.736268997192383,
      "activations/layer19_attention_weight_max": 30.640729904174805,
      "activations/layer19_attention_weight_min": -30.328535079956055,
      "activations/layer1_attention_weight_max": 16.477415084838867,
      "activations/layer1_attention_weight_min": -15.265353202819824,
      "activations/layer20_attention_weight_max": 27.688114166259766,
      "activations/layer20_attention_weight_min": -26.65486717224121,
      "activations/layer21_attention_weight_max": 27.340843200683594,
      "activations/layer21_attention_weight_min": -24.86972427368164,
      "activations/layer22_attention_weight_max": 38.78972244262695,
      "activations/layer22_attention_weight_min": -31.473630905151367,
      "activations/layer23_attention_weight_max": 32.3803596496582,
      "activations/layer23_attention_weight_min": -26.684072494506836,
      "activations/layer2_attention_weight_max": 33.0514030456543,
      "activations/layer2_attention_weight_min": -32.5031623840332,
      "activations/layer3_attention_weight_max": 96.57843017578125,
      "activations/layer3_attention_weight_min": -101.85574340820312,
      "activations/layer4_attention_weight_max": 107.27050018310547,
      "activations/layer4_attention_weight_min": -103.83917236328125,
      "activations/layer5_attention_weight_max": 49.04517364501953,
      "activations/layer5_attention_weight_min": -62.4764518737793,
      "activations/layer6_attention_weight_max": 42.47737503051758,
      "activations/layer6_attention_weight_min": -44.22807312011719,
      "activations/layer7_attention_weight_max": 88.43685913085938,
      "activations/layer7_attention_weight_min": -92.4262466430664,
      "activations/layer8_attention_weight_max": 43.278316497802734,
      "activations/layer8_attention_weight_min": -43.4581184387207,
      "activations/layer9_attention_weight_max": 34.606990814208984,
      "activations/layer9_attention_weight_min": -34.353816986083984,
      "epoch": 16.9,
      "learning_rate": 4.1413636363636365e-05,
      "loss": 2.7385,
      "step": 290900
    },
    {
      "activations/layer0_attention_weight_max": 15.783499717712402,
      "activations/layer0_attention_weight_min": -12.652750015258789,
      "activations/layer10_attention_weight_max": 34.321533203125,
      "activations/layer10_attention_weight_min": -33.829124450683594,
      "activations/layer11_attention_weight_max": 32.840476989746094,
      "activations/layer11_attention_weight_min": -33.63759994506836,
      "activations/layer12_attention_weight_max": 27.120712280273438,
      "activations/layer12_attention_weight_min": -25.999170303344727,
      "activations/layer13_attention_weight_max": 42.25905227661133,
      "activations/layer13_attention_weight_min": -34.21451187133789,
      "activations/layer14_attention_weight_max": 44.13627624511719,
      "activations/layer14_attention_weight_min": -36.05607986450195,
      "activations/layer15_attention_weight_max": 40.628822326660156,
      "activations/layer15_attention_weight_min": -33.45158386230469,
      "activations/layer16_attention_weight_max": 28.352529525756836,
      "activations/layer16_attention_weight_min": -27.14535140991211,
      "activations/layer17_attention_weight_max": 30.901025772094727,
      "activations/layer17_attention_weight_min": -24.85605812072754,
      "activations/layer18_attention_weight_max": 29.69572639465332,
      "activations/layer18_attention_weight_min": -23.12722396850586,
      "activations/layer19_attention_weight_max": 31.371784210205078,
      "activations/layer19_attention_weight_min": -29.91037368774414,
      "activations/layer1_attention_weight_max": 15.081624984741211,
      "activations/layer1_attention_weight_min": -14.415212631225586,
      "activations/layer20_attention_weight_max": 26.69200897216797,
      "activations/layer20_attention_weight_min": -25.288373947143555,
      "activations/layer21_attention_weight_max": 27.674663543701172,
      "activations/layer21_attention_weight_min": -24.992151260375977,
      "activations/layer22_attention_weight_max": 39.66033935546875,
      "activations/layer22_attention_weight_min": -30.345348358154297,
      "activations/layer23_attention_weight_max": 32.251869201660156,
      "activations/layer23_attention_weight_min": -25.96933364868164,
      "activations/layer2_attention_weight_max": 32.2599983215332,
      "activations/layer2_attention_weight_min": -33.389617919921875,
      "activations/layer3_attention_weight_max": 94.87921905517578,
      "activations/layer3_attention_weight_min": -97.41231536865234,
      "activations/layer4_attention_weight_max": 103.5657958984375,
      "activations/layer4_attention_weight_min": -101.93113708496094,
      "activations/layer5_attention_weight_max": 48.11851501464844,
      "activations/layer5_attention_weight_min": -58.250850677490234,
      "activations/layer6_attention_weight_max": 43.293514251708984,
      "activations/layer6_attention_weight_min": -43.55625915527344,
      "activations/layer7_attention_weight_max": 91.03093719482422,
      "activations/layer7_attention_weight_min": -90.10332489013672,
      "activations/layer8_attention_weight_max": 42.1830940246582,
      "activations/layer8_attention_weight_min": -41.321319580078125,
      "activations/layer9_attention_weight_max": 32.274620056152344,
      "activations/layer9_attention_weight_min": -33.690853118896484,
      "epoch": 16.91,
      "learning_rate": 4.1394696969696966e-05,
      "loss": 2.7548,
      "step": 290950
    },
    {
      "activations/layer0_attention_weight_max": 16.205965042114258,
      "activations/layer0_attention_weight_min": -13.116636276245117,
      "activations/layer10_attention_weight_max": 31.981523513793945,
      "activations/layer10_attention_weight_min": -33.910484313964844,
      "activations/layer11_attention_weight_max": 33.620155334472656,
      "activations/layer11_attention_weight_min": -34.555938720703125,
      "activations/layer12_attention_weight_max": 24.986957550048828,
      "activations/layer12_attention_weight_min": -29.32851791381836,
      "activations/layer13_attention_weight_max": 35.367244720458984,
      "activations/layer13_attention_weight_min": -37.48432540893555,
      "activations/layer14_attention_weight_max": 37.368412017822266,
      "activations/layer14_attention_weight_min": -39.65876770019531,
      "activations/layer15_attention_weight_max": 38.3598518371582,
      "activations/layer15_attention_weight_min": -38.131832122802734,
      "activations/layer16_attention_weight_max": 29.904830932617188,
      "activations/layer16_attention_weight_min": -32.90878677368164,
      "activations/layer17_attention_weight_max": 30.138200759887695,
      "activations/layer17_attention_weight_min": -28.89971351623535,
      "activations/layer18_attention_weight_max": 29.803415298461914,
      "activations/layer18_attention_weight_min": -25.88100814819336,
      "activations/layer19_attention_weight_max": 33.461666107177734,
      "activations/layer19_attention_weight_min": -30.133676528930664,
      "activations/layer1_attention_weight_max": 16.4323673248291,
      "activations/layer1_attention_weight_min": -14.791193008422852,
      "activations/layer20_attention_weight_max": 28.424882888793945,
      "activations/layer20_attention_weight_min": -26.294466018676758,
      "activations/layer21_attention_weight_max": 27.06369400024414,
      "activations/layer21_attention_weight_min": -24.507341384887695,
      "activations/layer22_attention_weight_max": 38.65122985839844,
      "activations/layer22_attention_weight_min": -30.767499923706055,
      "activations/layer23_attention_weight_max": 32.44794464111328,
      "activations/layer23_attention_weight_min": -25.29798126220703,
      "activations/layer2_attention_weight_max": 33.013938903808594,
      "activations/layer2_attention_weight_min": -32.85077667236328,
      "activations/layer3_attention_weight_max": 93.73310089111328,
      "activations/layer3_attention_weight_min": -94.73246002197266,
      "activations/layer4_attention_weight_max": 106.3633804321289,
      "activations/layer4_attention_weight_min": -101.9557876586914,
      "activations/layer5_attention_weight_max": 49.79973602294922,
      "activations/layer5_attention_weight_min": -57.845237731933594,
      "activations/layer6_attention_weight_max": 41.66501998901367,
      "activations/layer6_attention_weight_min": -42.07383728027344,
      "activations/layer7_attention_weight_max": 93.15259552001953,
      "activations/layer7_attention_weight_min": -89.75007629394531,
      "activations/layer8_attention_weight_max": 41.32866668701172,
      "activations/layer8_attention_weight_min": -39.8617057800293,
      "activations/layer9_attention_weight_max": 32.062232971191406,
      "activations/layer9_attention_weight_min": -33.21515655517578,
      "epoch": 16.91,
      "learning_rate": 4.137575757575757e-05,
      "loss": 2.7503,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4911,
      "eval_samples_per_second": 505.704,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4911,
      "eval_openwebtext_samples_per_second": 505.704,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9879,
      "eval_wikitext_samples_per_second": 229.383,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5325,
      "eval_lambada_samples_per_second": 510.777,
      "step": 291000
    },
    {
      "activations/layer0_attention_weight_max": 16.478553771972656,
      "activations/layer0_attention_weight_min": -13.854103088378906,
      "activations/layer10_attention_weight_max": 32.90767288208008,
      "activations/layer10_attention_weight_min": -33.335357666015625,
      "activations/layer11_attention_weight_max": 33.01685333251953,
      "activations/layer11_attention_weight_min": -33.99824523925781,
      "activations/layer12_attention_weight_max": 26.757658004760742,
      "activations/layer12_attention_weight_min": -27.101600646972656,
      "activations/layer13_attention_weight_max": 40.08195114135742,
      "activations/layer13_attention_weight_min": -37.807491302490234,
      "activations/layer14_attention_weight_max": 47.52851486206055,
      "activations/layer14_attention_weight_min": -40.92313003540039,
      "activations/layer15_attention_weight_max": 41.354942321777344,
      "activations/layer15_attention_weight_min": -36.10614013671875,
      "activations/layer16_attention_weight_max": 30.55348777770996,
      "activations/layer16_attention_weight_min": -29.662824630737305,
      "activations/layer17_attention_weight_max": 31.100948333740234,
      "activations/layer17_attention_weight_min": -27.778705596923828,
      "activations/layer18_attention_weight_max": 33.278411865234375,
      "activations/layer18_attention_weight_min": -26.23185920715332,
      "activations/layer19_attention_weight_max": 34.87217330932617,
      "activations/layer19_attention_weight_min": -33.517906188964844,
      "activations/layer1_attention_weight_max": 16.75810432434082,
      "activations/layer1_attention_weight_min": -14.906389236450195,
      "activations/layer20_attention_weight_max": 33.623355865478516,
      "activations/layer20_attention_weight_min": -27.690349578857422,
      "activations/layer21_attention_weight_max": 32.46144485473633,
      "activations/layer21_attention_weight_min": -27.73433494567871,
      "activations/layer22_attention_weight_max": 46.034324645996094,
      "activations/layer22_attention_weight_min": -33.051536560058594,
      "activations/layer23_attention_weight_max": 36.18699645996094,
      "activations/layer23_attention_weight_min": -26.97952651977539,
      "activations/layer2_attention_weight_max": 35.14259719848633,
      "activations/layer2_attention_weight_min": -34.516910552978516,
      "activations/layer3_attention_weight_max": 98.2960433959961,
      "activations/layer3_attention_weight_min": -98.99830627441406,
      "activations/layer4_attention_weight_max": 106.55644989013672,
      "activations/layer4_attention_weight_min": -105.5467529296875,
      "activations/layer5_attention_weight_max": 54.455135345458984,
      "activations/layer5_attention_weight_min": -64.88548278808594,
      "activations/layer6_attention_weight_max": 45.67851257324219,
      "activations/layer6_attention_weight_min": -46.1940803527832,
      "activations/layer7_attention_weight_max": 93.6240463256836,
      "activations/layer7_attention_weight_min": -96.29744720458984,
      "activations/layer8_attention_weight_max": 41.899044036865234,
      "activations/layer8_attention_weight_min": -46.80227279663086,
      "activations/layer9_attention_weight_max": 31.852277755737305,
      "activations/layer9_attention_weight_min": -34.47565841674805,
      "epoch": 16.91,
      "learning_rate": 4.1356818181818176e-05,
      "loss": 2.7363,
      "step": 291050
    },
    {
      "activations/layer0_attention_weight_max": 15.541696548461914,
      "activations/layer0_attention_weight_min": -13.222457885742188,
      "activations/layer10_attention_weight_max": 33.43451690673828,
      "activations/layer10_attention_weight_min": -34.52771759033203,
      "activations/layer11_attention_weight_max": 34.021785736083984,
      "activations/layer11_attention_weight_min": -33.54275131225586,
      "activations/layer12_attention_weight_max": 26.730440139770508,
      "activations/layer12_attention_weight_min": -25.857059478759766,
      "activations/layer13_attention_weight_max": 39.13383102416992,
      "activations/layer13_attention_weight_min": -36.39446258544922,
      "activations/layer14_attention_weight_max": 43.85099411010742,
      "activations/layer14_attention_weight_min": -39.326351165771484,
      "activations/layer15_attention_weight_max": 36.85190963745117,
      "activations/layer15_attention_weight_min": -37.36524200439453,
      "activations/layer16_attention_weight_max": 28.158245086669922,
      "activations/layer16_attention_weight_min": -27.705791473388672,
      "activations/layer17_attention_weight_max": 33.3823356628418,
      "activations/layer17_attention_weight_min": -26.678356170654297,
      "activations/layer18_attention_weight_max": 36.57283020019531,
      "activations/layer18_attention_weight_min": -26.220516204833984,
      "activations/layer19_attention_weight_max": 35.23272705078125,
      "activations/layer19_attention_weight_min": -32.85515594482422,
      "activations/layer1_attention_weight_max": 16.476680755615234,
      "activations/layer1_attention_weight_min": -16.337583541870117,
      "activations/layer20_attention_weight_max": 30.966278076171875,
      "activations/layer20_attention_weight_min": -25.442913055419922,
      "activations/layer21_attention_weight_max": 33.591102600097656,
      "activations/layer21_attention_weight_min": -26.91621971130371,
      "activations/layer22_attention_weight_max": 41.97339630126953,
      "activations/layer22_attention_weight_min": -31.684459686279297,
      "activations/layer23_attention_weight_max": 32.977970123291016,
      "activations/layer23_attention_weight_min": -26.305095672607422,
      "activations/layer2_attention_weight_max": 33.1900749206543,
      "activations/layer2_attention_weight_min": -33.431907653808594,
      "activations/layer3_attention_weight_max": 96.03591918945312,
      "activations/layer3_attention_weight_min": -98.19161987304688,
      "activations/layer4_attention_weight_max": 107.3278579711914,
      "activations/layer4_attention_weight_min": -110.95018768310547,
      "activations/layer5_attention_weight_max": 52.24736785888672,
      "activations/layer5_attention_weight_min": -61.770263671875,
      "activations/layer6_attention_weight_max": 43.31416320800781,
      "activations/layer6_attention_weight_min": -47.418861389160156,
      "activations/layer7_attention_weight_max": 91.20491790771484,
      "activations/layer7_attention_weight_min": -98.18941497802734,
      "activations/layer8_attention_weight_max": 42.24727249145508,
      "activations/layer8_attention_weight_min": -42.94277572631836,
      "activations/layer9_attention_weight_max": 32.42820358276367,
      "activations/layer9_attention_weight_min": -33.89658737182617,
      "epoch": 16.91,
      "learning_rate": 4.1337878787878784e-05,
      "loss": 2.7404,
      "step": 291100
    },
    {
      "activations/layer0_attention_weight_max": 15.847769737243652,
      "activations/layer0_attention_weight_min": -12.239924430847168,
      "activations/layer10_attention_weight_max": 32.625648498535156,
      "activations/layer10_attention_weight_min": -34.458900451660156,
      "activations/layer11_attention_weight_max": 35.81352996826172,
      "activations/layer11_attention_weight_min": -36.9248046875,
      "activations/layer12_attention_weight_max": 26.76279067993164,
      "activations/layer12_attention_weight_min": -27.34569549560547,
      "activations/layer13_attention_weight_max": 39.16613006591797,
      "activations/layer13_attention_weight_min": -34.536529541015625,
      "activations/layer14_attention_weight_max": 44.48750305175781,
      "activations/layer14_attention_weight_min": -37.79227828979492,
      "activations/layer15_attention_weight_max": 39.03702926635742,
      "activations/layer15_attention_weight_min": -35.65290832519531,
      "activations/layer16_attention_weight_max": 31.075679779052734,
      "activations/layer16_attention_weight_min": -29.20748519897461,
      "activations/layer17_attention_weight_max": 30.50408935546875,
      "activations/layer17_attention_weight_min": -26.22261619567871,
      "activations/layer18_attention_weight_max": 31.632539749145508,
      "activations/layer18_attention_weight_min": -26.438631057739258,
      "activations/layer19_attention_weight_max": 35.79500961303711,
      "activations/layer19_attention_weight_min": -30.692358016967773,
      "activations/layer1_attention_weight_max": 16.590911865234375,
      "activations/layer1_attention_weight_min": -14.896580696105957,
      "activations/layer20_attention_weight_max": 28.58384895324707,
      "activations/layer20_attention_weight_min": -25.349580764770508,
      "activations/layer21_attention_weight_max": 28.68571662902832,
      "activations/layer21_attention_weight_min": -25.697038650512695,
      "activations/layer22_attention_weight_max": 43.06495666503906,
      "activations/layer22_attention_weight_min": -32.42069625854492,
      "activations/layer23_attention_weight_max": 32.05451202392578,
      "activations/layer23_attention_weight_min": -26.003650665283203,
      "activations/layer2_attention_weight_max": 34.470245361328125,
      "activations/layer2_attention_weight_min": -32.21424102783203,
      "activations/layer3_attention_weight_max": 96.32329559326172,
      "activations/layer3_attention_weight_min": -96.08773803710938,
      "activations/layer4_attention_weight_max": 104.62188720703125,
      "activations/layer4_attention_weight_min": -106.0748062133789,
      "activations/layer5_attention_weight_max": 52.94166564941406,
      "activations/layer5_attention_weight_min": -59.337066650390625,
      "activations/layer6_attention_weight_max": 41.97441482543945,
      "activations/layer6_attention_weight_min": -44.61178970336914,
      "activations/layer7_attention_weight_max": 92.9417724609375,
      "activations/layer7_attention_weight_min": -90.76377868652344,
      "activations/layer8_attention_weight_max": 39.993072509765625,
      "activations/layer8_attention_weight_min": -41.56312561035156,
      "activations/layer9_attention_weight_max": 31.0861873626709,
      "activations/layer9_attention_weight_min": -33.75590515136719,
      "epoch": 16.92,
      "learning_rate": 4.1319318181818175e-05,
      "loss": 2.7419,
      "step": 291150
    },
    {
      "activations/layer0_attention_weight_max": 16.241703033447266,
      "activations/layer0_attention_weight_min": -14.087382316589355,
      "activations/layer10_attention_weight_max": 38.46344757080078,
      "activations/layer10_attention_weight_min": -38.00834274291992,
      "activations/layer11_attention_weight_max": 37.368812561035156,
      "activations/layer11_attention_weight_min": -36.61505126953125,
      "activations/layer12_attention_weight_max": 30.48678970336914,
      "activations/layer12_attention_weight_min": -27.949893951416016,
      "activations/layer13_attention_weight_max": 44.76518249511719,
      "activations/layer13_attention_weight_min": -39.19844436645508,
      "activations/layer14_attention_weight_max": 46.04957580566406,
      "activations/layer14_attention_weight_min": -40.598304748535156,
      "activations/layer15_attention_weight_max": 43.687591552734375,
      "activations/layer15_attention_weight_min": -38.13414001464844,
      "activations/layer16_attention_weight_max": 31.80188751220703,
      "activations/layer16_attention_weight_min": -31.13147735595703,
      "activations/layer17_attention_weight_max": 31.99169921875,
      "activations/layer17_attention_weight_min": -29.27579116821289,
      "activations/layer18_attention_weight_max": 34.69355773925781,
      "activations/layer18_attention_weight_min": -26.818010330200195,
      "activations/layer19_attention_weight_max": 35.59258270263672,
      "activations/layer19_attention_weight_min": -35.478271484375,
      "activations/layer1_attention_weight_max": 15.497611045837402,
      "activations/layer1_attention_weight_min": -14.545737266540527,
      "activations/layer20_attention_weight_max": 29.134265899658203,
      "activations/layer20_attention_weight_min": -27.028396606445312,
      "activations/layer21_attention_weight_max": 30.75967788696289,
      "activations/layer21_attention_weight_min": -26.443248748779297,
      "activations/layer22_attention_weight_max": 43.235069274902344,
      "activations/layer22_attention_weight_min": -32.481910705566406,
      "activations/layer23_attention_weight_max": 33.64002990722656,
      "activations/layer23_attention_weight_min": -25.61766815185547,
      "activations/layer2_attention_weight_max": 33.02225875854492,
      "activations/layer2_attention_weight_min": -34.00824737548828,
      "activations/layer3_attention_weight_max": 98.5192642211914,
      "activations/layer3_attention_weight_min": -97.54464721679688,
      "activations/layer4_attention_weight_max": 109.3469467163086,
      "activations/layer4_attention_weight_min": -116.5667724609375,
      "activations/layer5_attention_weight_max": 53.7539176940918,
      "activations/layer5_attention_weight_min": -68.98458099365234,
      "activations/layer6_attention_weight_max": 47.19966125488281,
      "activations/layer6_attention_weight_min": -50.22168731689453,
      "activations/layer7_attention_weight_max": 95.54904174804688,
      "activations/layer7_attention_weight_min": -102.588623046875,
      "activations/layer8_attention_weight_max": 45.955657958984375,
      "activations/layer8_attention_weight_min": -48.25088119506836,
      "activations/layer9_attention_weight_max": 35.50728988647461,
      "activations/layer9_attention_weight_min": -36.09284591674805,
      "epoch": 16.92,
      "learning_rate": 4.1300378787878784e-05,
      "loss": 2.7596,
      "step": 291200
    },
    {
      "activations/layer0_attention_weight_max": 15.497833251953125,
      "activations/layer0_attention_weight_min": -13.402252197265625,
      "activations/layer10_attention_weight_max": 33.13483428955078,
      "activations/layer10_attention_weight_min": -33.92665481567383,
      "activations/layer11_attention_weight_max": 33.67408752441406,
      "activations/layer11_attention_weight_min": -33.18930435180664,
      "activations/layer12_attention_weight_max": 26.083946228027344,
      "activations/layer12_attention_weight_min": -27.771081924438477,
      "activations/layer13_attention_weight_max": 37.943904876708984,
      "activations/layer13_attention_weight_min": -35.626617431640625,
      "activations/layer14_attention_weight_max": 41.64572525024414,
      "activations/layer14_attention_weight_min": -37.064788818359375,
      "activations/layer15_attention_weight_max": 35.80006408691406,
      "activations/layer15_attention_weight_min": -34.2730712890625,
      "activations/layer16_attention_weight_max": 28.131567001342773,
      "activations/layer16_attention_weight_min": -27.695213317871094,
      "activations/layer17_attention_weight_max": 28.245922088623047,
      "activations/layer17_attention_weight_min": -26.00961685180664,
      "activations/layer18_attention_weight_max": 32.73591232299805,
      "activations/layer18_attention_weight_min": -25.039234161376953,
      "activations/layer19_attention_weight_max": 34.02098083496094,
      "activations/layer19_attention_weight_min": -31.296207427978516,
      "activations/layer1_attention_weight_max": 17.07326889038086,
      "activations/layer1_attention_weight_min": -14.149720191955566,
      "activations/layer20_attention_weight_max": 31.707378387451172,
      "activations/layer20_attention_weight_min": -26.585054397583008,
      "activations/layer21_attention_weight_max": 33.1672248840332,
      "activations/layer21_attention_weight_min": -25.530344009399414,
      "activations/layer22_attention_weight_max": 45.21995544433594,
      "activations/layer22_attention_weight_min": -31.775476455688477,
      "activations/layer23_attention_weight_max": 33.28292465209961,
      "activations/layer23_attention_weight_min": -25.19886589050293,
      "activations/layer2_attention_weight_max": 32.668914794921875,
      "activations/layer2_attention_weight_min": -32.539939880371094,
      "activations/layer3_attention_weight_max": 101.15013122558594,
      "activations/layer3_attention_weight_min": -96.92090606689453,
      "activations/layer4_attention_weight_max": 105.8930435180664,
      "activations/layer4_attention_weight_min": -104.62409973144531,
      "activations/layer5_attention_weight_max": 50.384300231933594,
      "activations/layer5_attention_weight_min": -60.55906295776367,
      "activations/layer6_attention_weight_max": 44.02638626098633,
      "activations/layer6_attention_weight_min": -46.03670120239258,
      "activations/layer7_attention_weight_max": 94.38960266113281,
      "activations/layer7_attention_weight_min": -97.07084655761719,
      "activations/layer8_attention_weight_max": 42.554012298583984,
      "activations/layer8_attention_weight_min": -45.01408386230469,
      "activations/layer9_attention_weight_max": 32.379215240478516,
      "activations/layer9_attention_weight_min": -34.806419372558594,
      "epoch": 16.92,
      "learning_rate": 4.128143939393939e-05,
      "loss": 2.7328,
      "step": 291250
    },
    {
      "activations/layer0_attention_weight_max": 15.89493179321289,
      "activations/layer0_attention_weight_min": -12.707547187805176,
      "activations/layer10_attention_weight_max": 35.46186065673828,
      "activations/layer10_attention_weight_min": -34.79317855834961,
      "activations/layer11_attention_weight_max": 33.675113677978516,
      "activations/layer11_attention_weight_min": -35.64246368408203,
      "activations/layer12_attention_weight_max": 27.22049903869629,
      "activations/layer12_attention_weight_min": -29.04789161682129,
      "activations/layer13_attention_weight_max": 39.96444320678711,
      "activations/layer13_attention_weight_min": -36.263031005859375,
      "activations/layer14_attention_weight_max": 43.634037017822266,
      "activations/layer14_attention_weight_min": -38.70173645019531,
      "activations/layer15_attention_weight_max": 41.60590362548828,
      "activations/layer15_attention_weight_min": -35.705970764160156,
      "activations/layer16_attention_weight_max": 29.514970779418945,
      "activations/layer16_attention_weight_min": -28.37333869934082,
      "activations/layer17_attention_weight_max": 30.42608642578125,
      "activations/layer17_attention_weight_min": -25.033212661743164,
      "activations/layer18_attention_weight_max": 34.60649871826172,
      "activations/layer18_attention_weight_min": -24.878747940063477,
      "activations/layer19_attention_weight_max": 35.620906829833984,
      "activations/layer19_attention_weight_min": -32.08262252807617,
      "activations/layer1_attention_weight_max": 17.013586044311523,
      "activations/layer1_attention_weight_min": -15.610923767089844,
      "activations/layer20_attention_weight_max": 32.33058547973633,
      "activations/layer20_attention_weight_min": -24.57329750061035,
      "activations/layer21_attention_weight_max": 31.457656860351562,
      "activations/layer21_attention_weight_min": -25.053421020507812,
      "activations/layer22_attention_weight_max": 41.9679069519043,
      "activations/layer22_attention_weight_min": -30.18992042541504,
      "activations/layer23_attention_weight_max": 33.480499267578125,
      "activations/layer23_attention_weight_min": -25.223495483398438,
      "activations/layer2_attention_weight_max": 34.287353515625,
      "activations/layer2_attention_weight_min": -32.98443603515625,
      "activations/layer3_attention_weight_max": 97.77908325195312,
      "activations/layer3_attention_weight_min": -99.6133041381836,
      "activations/layer4_attention_weight_max": 106.25231170654297,
      "activations/layer4_attention_weight_min": -110.7440414428711,
      "activations/layer5_attention_weight_max": 49.95890808105469,
      "activations/layer5_attention_weight_min": -63.80556869506836,
      "activations/layer6_attention_weight_max": 45.88618087768555,
      "activations/layer6_attention_weight_min": -47.02573013305664,
      "activations/layer7_attention_weight_max": 96.01737213134766,
      "activations/layer7_attention_weight_min": -98.42598724365234,
      "activations/layer8_attention_weight_max": 44.98073196411133,
      "activations/layer8_attention_weight_min": -46.63447570800781,
      "activations/layer9_attention_weight_max": 34.979454040527344,
      "activations/layer9_attention_weight_min": -36.69801712036133,
      "epoch": 16.93,
      "learning_rate": 4.126287878787879e-05,
      "loss": 2.7513,
      "step": 291300
    },
    {
      "activations/layer0_attention_weight_max": 15.648362159729004,
      "activations/layer0_attention_weight_min": -12.702996253967285,
      "activations/layer10_attention_weight_max": 35.21895217895508,
      "activations/layer10_attention_weight_min": -33.99787902832031,
      "activations/layer11_attention_weight_max": 37.748687744140625,
      "activations/layer11_attention_weight_min": -37.0384521484375,
      "activations/layer12_attention_weight_max": 29.49388885498047,
      "activations/layer12_attention_weight_min": -27.476619720458984,
      "activations/layer13_attention_weight_max": 45.91161346435547,
      "activations/layer13_attention_weight_min": -38.593505859375,
      "activations/layer14_attention_weight_max": 49.393253326416016,
      "activations/layer14_attention_weight_min": -41.69862747192383,
      "activations/layer15_attention_weight_max": 45.95004653930664,
      "activations/layer15_attention_weight_min": -39.212860107421875,
      "activations/layer16_attention_weight_max": 36.08526611328125,
      "activations/layer16_attention_weight_min": -28.752498626708984,
      "activations/layer17_attention_weight_max": 33.32057571411133,
      "activations/layer17_attention_weight_min": -27.514665603637695,
      "activations/layer18_attention_weight_max": 34.30756759643555,
      "activations/layer18_attention_weight_min": -25.535694122314453,
      "activations/layer19_attention_weight_max": 38.333431243896484,
      "activations/layer19_attention_weight_min": -32.43925476074219,
      "activations/layer1_attention_weight_max": 16.806400299072266,
      "activations/layer1_attention_weight_min": -14.653299331665039,
      "activations/layer20_attention_weight_max": 32.87318801879883,
      "activations/layer20_attention_weight_min": -25.932016372680664,
      "activations/layer21_attention_weight_max": 29.371658325195312,
      "activations/layer21_attention_weight_min": -26.66366958618164,
      "activations/layer22_attention_weight_max": 38.299137115478516,
      "activations/layer22_attention_weight_min": -33.02103042602539,
      "activations/layer23_attention_weight_max": 32.61586380004883,
      "activations/layer23_attention_weight_min": -26.488752365112305,
      "activations/layer2_attention_weight_max": 32.94648742675781,
      "activations/layer2_attention_weight_min": -32.274864196777344,
      "activations/layer3_attention_weight_max": 98.07421875,
      "activations/layer3_attention_weight_min": -97.92589569091797,
      "activations/layer4_attention_weight_max": 112.2636489868164,
      "activations/layer4_attention_weight_min": -109.63912200927734,
      "activations/layer5_attention_weight_max": 52.63134002685547,
      "activations/layer5_attention_weight_min": -61.2052001953125,
      "activations/layer6_attention_weight_max": 45.74458312988281,
      "activations/layer6_attention_weight_min": -45.40919494628906,
      "activations/layer7_attention_weight_max": 100.465576171875,
      "activations/layer7_attention_weight_min": -94.95938873291016,
      "activations/layer8_attention_weight_max": 43.21311950683594,
      "activations/layer8_attention_weight_min": -44.527530670166016,
      "activations/layer9_attention_weight_max": 33.9078254699707,
      "activations/layer9_attention_weight_min": -34.62194061279297,
      "epoch": 16.93,
      "learning_rate": 4.124393939393939e-05,
      "loss": 2.7447,
      "step": 291350
    },
    {
      "activations/layer0_attention_weight_max": 15.986459732055664,
      "activations/layer0_attention_weight_min": -13.30489730834961,
      "activations/layer10_attention_weight_max": 34.084598541259766,
      "activations/layer10_attention_weight_min": -34.456947326660156,
      "activations/layer11_attention_weight_max": 36.41217803955078,
      "activations/layer11_attention_weight_min": -33.90488052368164,
      "activations/layer12_attention_weight_max": 30.494726181030273,
      "activations/layer12_attention_weight_min": -27.59026336669922,
      "activations/layer13_attention_weight_max": 46.28757095336914,
      "activations/layer13_attention_weight_min": -36.49204635620117,
      "activations/layer14_attention_weight_max": 56.69746017456055,
      "activations/layer14_attention_weight_min": -39.90462875366211,
      "activations/layer15_attention_weight_max": 48.107383728027344,
      "activations/layer15_attention_weight_min": -38.60454177856445,
      "activations/layer16_attention_weight_max": 40.81607437133789,
      "activations/layer16_attention_weight_min": -31.617292404174805,
      "activations/layer17_attention_weight_max": 38.094970703125,
      "activations/layer17_attention_weight_min": -27.280141830444336,
      "activations/layer18_attention_weight_max": 35.309974670410156,
      "activations/layer18_attention_weight_min": -25.668331146240234,
      "activations/layer19_attention_weight_max": 42.45841979980469,
      "activations/layer19_attention_weight_min": -32.21087646484375,
      "activations/layer1_attention_weight_max": 16.09722137451172,
      "activations/layer1_attention_weight_min": -15.850790977478027,
      "activations/layer20_attention_weight_max": 35.61653137207031,
      "activations/layer20_attention_weight_min": -24.794872283935547,
      "activations/layer21_attention_weight_max": 32.528358459472656,
      "activations/layer21_attention_weight_min": -23.49108123779297,
      "activations/layer22_attention_weight_max": 50.42535400390625,
      "activations/layer22_attention_weight_min": -31.080265045166016,
      "activations/layer23_attention_weight_max": 38.43379211425781,
      "activations/layer23_attention_weight_min": -25.240615844726562,
      "activations/layer2_attention_weight_max": 33.369529724121094,
      "activations/layer2_attention_weight_min": -32.92831802368164,
      "activations/layer3_attention_weight_max": 97.00627899169922,
      "activations/layer3_attention_weight_min": -96.49983215332031,
      "activations/layer4_attention_weight_max": 103.02083587646484,
      "activations/layer4_attention_weight_min": -102.99224090576172,
      "activations/layer5_attention_weight_max": 50.60755920410156,
      "activations/layer5_attention_weight_min": -58.21751403808594,
      "activations/layer6_attention_weight_max": 44.08811950683594,
      "activations/layer6_attention_weight_min": -43.8040657043457,
      "activations/layer7_attention_weight_max": 94.7464370727539,
      "activations/layer7_attention_weight_min": -90.11357116699219,
      "activations/layer8_attention_weight_max": 42.88003158569336,
      "activations/layer8_attention_weight_min": -42.298011779785156,
      "activations/layer9_attention_weight_max": 36.40430450439453,
      "activations/layer9_attention_weight_min": -35.6268424987793,
      "epoch": 16.93,
      "learning_rate": 4.122499999999999e-05,
      "loss": 2.7368,
      "step": 291400
    },
    {
      "activations/layer0_attention_weight_max": 15.857545852661133,
      "activations/layer0_attention_weight_min": -13.607526779174805,
      "activations/layer10_attention_weight_max": 34.139225006103516,
      "activations/layer10_attention_weight_min": -34.169368743896484,
      "activations/layer11_attention_weight_max": 35.87022399902344,
      "activations/layer11_attention_weight_min": -35.647056579589844,
      "activations/layer12_attention_weight_max": 29.222211837768555,
      "activations/layer12_attention_weight_min": -27.369394302368164,
      "activations/layer13_attention_weight_max": 45.260276794433594,
      "activations/layer13_attention_weight_min": -40.51136779785156,
      "activations/layer14_attention_weight_max": 48.70695495605469,
      "activations/layer14_attention_weight_min": -39.327659606933594,
      "activations/layer15_attention_weight_max": 47.97953796386719,
      "activations/layer15_attention_weight_min": -39.54292297363281,
      "activations/layer16_attention_weight_max": 31.27701759338379,
      "activations/layer16_attention_weight_min": -26.98011589050293,
      "activations/layer17_attention_weight_max": 28.75074577331543,
      "activations/layer17_attention_weight_min": -25.436431884765625,
      "activations/layer18_attention_weight_max": 32.71434020996094,
      "activations/layer18_attention_weight_min": -26.26412582397461,
      "activations/layer19_attention_weight_max": 36.834049224853516,
      "activations/layer19_attention_weight_min": -34.11220932006836,
      "activations/layer1_attention_weight_max": 16.465641021728516,
      "activations/layer1_attention_weight_min": -15.010592460632324,
      "activations/layer20_attention_weight_max": 30.84310531616211,
      "activations/layer20_attention_weight_min": -26.92704963684082,
      "activations/layer21_attention_weight_max": 29.8577938079834,
      "activations/layer21_attention_weight_min": -26.905529022216797,
      "activations/layer22_attention_weight_max": 47.185325622558594,
      "activations/layer22_attention_weight_min": -32.064476013183594,
      "activations/layer23_attention_weight_max": 34.40736389160156,
      "activations/layer23_attention_weight_min": -25.77107810974121,
      "activations/layer2_attention_weight_max": 32.399742126464844,
      "activations/layer2_attention_weight_min": -33.249427795410156,
      "activations/layer3_attention_weight_max": 96.6470718383789,
      "activations/layer3_attention_weight_min": -94.38739013671875,
      "activations/layer4_attention_weight_max": 104.71998596191406,
      "activations/layer4_attention_weight_min": -108.75060272216797,
      "activations/layer5_attention_weight_max": 50.76014709472656,
      "activations/layer5_attention_weight_min": -60.07943344116211,
      "activations/layer6_attention_weight_max": 43.26669692993164,
      "activations/layer6_attention_weight_min": -45.21460723876953,
      "activations/layer7_attention_weight_max": 92.29728698730469,
      "activations/layer7_attention_weight_min": -95.6869125366211,
      "activations/layer8_attention_weight_max": 42.70740509033203,
      "activations/layer8_attention_weight_min": -44.4791259765625,
      "activations/layer9_attention_weight_max": 34.247100830078125,
      "activations/layer9_attention_weight_min": -34.17373275756836,
      "epoch": 16.93,
      "learning_rate": 4.120606060606061e-05,
      "loss": 2.7378,
      "step": 291450
    },
    {
      "activations/layer0_attention_weight_max": 16.43761444091797,
      "activations/layer0_attention_weight_min": -13.411242485046387,
      "activations/layer10_attention_weight_max": 33.62196350097656,
      "activations/layer10_attention_weight_min": -34.85417938232422,
      "activations/layer11_attention_weight_max": 33.72214126586914,
      "activations/layer11_attention_weight_min": -33.96003723144531,
      "activations/layer12_attention_weight_max": 27.770902633666992,
      "activations/layer12_attention_weight_min": -29.57280158996582,
      "activations/layer13_attention_weight_max": 43.69362258911133,
      "activations/layer13_attention_weight_min": -39.369056701660156,
      "activations/layer14_attention_weight_max": 41.88990783691406,
      "activations/layer14_attention_weight_min": -38.858646392822266,
      "activations/layer15_attention_weight_max": 40.701026916503906,
      "activations/layer15_attention_weight_min": -36.569969177246094,
      "activations/layer16_attention_weight_max": 30.64115333557129,
      "activations/layer16_attention_weight_min": -27.62502098083496,
      "activations/layer17_attention_weight_max": 30.88157081604004,
      "activations/layer17_attention_weight_min": -28.07791519165039,
      "activations/layer18_attention_weight_max": 33.71114730834961,
      "activations/layer18_attention_weight_min": -26.18492317199707,
      "activations/layer19_attention_weight_max": 32.409889221191406,
      "activations/layer19_attention_weight_min": -32.251895904541016,
      "activations/layer1_attention_weight_max": 15.810685157775879,
      "activations/layer1_attention_weight_min": -14.60834789276123,
      "activations/layer20_attention_weight_max": 29.169628143310547,
      "activations/layer20_attention_weight_min": -25.743528366088867,
      "activations/layer21_attention_weight_max": 29.154760360717773,
      "activations/layer21_attention_weight_min": -23.5595760345459,
      "activations/layer22_attention_weight_max": 44.52075958251953,
      "activations/layer22_attention_weight_min": -32.31324005126953,
      "activations/layer23_attention_weight_max": 32.36143493652344,
      "activations/layer23_attention_weight_min": -24.042938232421875,
      "activations/layer2_attention_weight_max": 33.03276824951172,
      "activations/layer2_attention_weight_min": -32.432064056396484,
      "activations/layer3_attention_weight_max": 95.92481994628906,
      "activations/layer3_attention_weight_min": -97.66154479980469,
      "activations/layer4_attention_weight_max": 103.3692855834961,
      "activations/layer4_attention_weight_min": -109.0516357421875,
      "activations/layer5_attention_weight_max": 49.8076171875,
      "activations/layer5_attention_weight_min": -61.72892761230469,
      "activations/layer6_attention_weight_max": 42.72270965576172,
      "activations/layer6_attention_weight_min": -45.34465026855469,
      "activations/layer7_attention_weight_max": 92.07928466796875,
      "activations/layer7_attention_weight_min": -101.51447296142578,
      "activations/layer8_attention_weight_max": 42.17445755004883,
      "activations/layer8_attention_weight_min": -44.03807830810547,
      "activations/layer9_attention_weight_max": 31.964630126953125,
      "activations/layer9_attention_weight_min": -34.46889877319336,
      "epoch": 16.94,
      "learning_rate": 4.118712121212121e-05,
      "loss": 2.7356,
      "step": 291500
    },
    {
      "activations/layer0_attention_weight_max": 16.268314361572266,
      "activations/layer0_attention_weight_min": -12.872945785522461,
      "activations/layer10_attention_weight_max": 30.832088470458984,
      "activations/layer10_attention_weight_min": -31.671321868896484,
      "activations/layer11_attention_weight_max": 32.06338882446289,
      "activations/layer11_attention_weight_min": -33.102989196777344,
      "activations/layer12_attention_weight_max": 44.002655029296875,
      "activations/layer12_attention_weight_min": -27.9978084564209,
      "activations/layer13_attention_weight_max": 55.46369552612305,
      "activations/layer13_attention_weight_min": -37.253475189208984,
      "activations/layer14_attention_weight_max": 52.76328659057617,
      "activations/layer14_attention_weight_min": -39.311092376708984,
      "activations/layer15_attention_weight_max": 46.45513916015625,
      "activations/layer15_attention_weight_min": -37.124351501464844,
      "activations/layer16_attention_weight_max": 33.1647834777832,
      "activations/layer16_attention_weight_min": -28.395450592041016,
      "activations/layer17_attention_weight_max": 31.49198341369629,
      "activations/layer17_attention_weight_min": -26.487382888793945,
      "activations/layer18_attention_weight_max": 33.48120880126953,
      "activations/layer18_attention_weight_min": -25.637943267822266,
      "activations/layer19_attention_weight_max": 39.09811019897461,
      "activations/layer19_attention_weight_min": -33.5954475402832,
      "activations/layer1_attention_weight_max": 15.990873336791992,
      "activations/layer1_attention_weight_min": -15.061777114868164,
      "activations/layer20_attention_weight_max": 36.62845230102539,
      "activations/layer20_attention_weight_min": -26.7707576751709,
      "activations/layer21_attention_weight_max": 34.67919921875,
      "activations/layer21_attention_weight_min": -27.03301239013672,
      "activations/layer22_attention_weight_max": 48.04120635986328,
      "activations/layer22_attention_weight_min": -36.254486083984375,
      "activations/layer23_attention_weight_max": 35.233802795410156,
      "activations/layer23_attention_weight_min": -26.82892417907715,
      "activations/layer2_attention_weight_max": 29.3399715423584,
      "activations/layer2_attention_weight_min": -30.82630157470703,
      "activations/layer3_attention_weight_max": 87.1919937133789,
      "activations/layer3_attention_weight_min": -90.44135284423828,
      "activations/layer4_attention_weight_max": 95.65266418457031,
      "activations/layer4_attention_weight_min": -99.15670776367188,
      "activations/layer5_attention_weight_max": 48.88337707519531,
      "activations/layer5_attention_weight_min": -55.858394622802734,
      "activations/layer6_attention_weight_max": 39.772186279296875,
      "activations/layer6_attention_weight_min": -43.23421096801758,
      "activations/layer7_attention_weight_max": 82.27628326416016,
      "activations/layer7_attention_weight_min": -88.73472595214844,
      "activations/layer8_attention_weight_max": 37.391841888427734,
      "activations/layer8_attention_weight_min": -40.221927642822266,
      "activations/layer9_attention_weight_max": 29.847387313842773,
      "activations/layer9_attention_weight_min": -31.0712833404541,
      "epoch": 16.94,
      "learning_rate": 4.116818181818182e-05,
      "loss": 2.7439,
      "step": 291550
    },
    {
      "activations/layer0_attention_weight_max": 15.44663143157959,
      "activations/layer0_attention_weight_min": -13.457390785217285,
      "activations/layer10_attention_weight_max": 36.81578826904297,
      "activations/layer10_attention_weight_min": -37.46741485595703,
      "activations/layer11_attention_weight_max": 34.642066955566406,
      "activations/layer11_attention_weight_min": -34.852386474609375,
      "activations/layer12_attention_weight_max": 27.20562171936035,
      "activations/layer12_attention_weight_min": -28.47530746459961,
      "activations/layer13_attention_weight_max": 39.80315017700195,
      "activations/layer13_attention_weight_min": -37.44727325439453,
      "activations/layer14_attention_weight_max": 43.733680725097656,
      "activations/layer14_attention_weight_min": -38.82502365112305,
      "activations/layer15_attention_weight_max": 41.83516311645508,
      "activations/layer15_attention_weight_min": -36.885643005371094,
      "activations/layer16_attention_weight_max": 29.353431701660156,
      "activations/layer16_attention_weight_min": -28.384986877441406,
      "activations/layer17_attention_weight_max": 32.1551399230957,
      "activations/layer17_attention_weight_min": -25.915796279907227,
      "activations/layer18_attention_weight_max": 32.44041061401367,
      "activations/layer18_attention_weight_min": -24.48710823059082,
      "activations/layer19_attention_weight_max": 36.459007263183594,
      "activations/layer19_attention_weight_min": -31.06879997253418,
      "activations/layer1_attention_weight_max": 16.315696716308594,
      "activations/layer1_attention_weight_min": -15.392878532409668,
      "activations/layer20_attention_weight_max": 30.145456314086914,
      "activations/layer20_attention_weight_min": -25.58294677734375,
      "activations/layer21_attention_weight_max": 29.52102279663086,
      "activations/layer21_attention_weight_min": -24.31451416015625,
      "activations/layer22_attention_weight_max": 44.15188980102539,
      "activations/layer22_attention_weight_min": -32.580440521240234,
      "activations/layer23_attention_weight_max": 34.38648223876953,
      "activations/layer23_attention_weight_min": -26.89636993408203,
      "activations/layer2_attention_weight_max": 34.58927536010742,
      "activations/layer2_attention_weight_min": -32.219482421875,
      "activations/layer3_attention_weight_max": 97.12777709960938,
      "activations/layer3_attention_weight_min": -97.45977783203125,
      "activations/layer4_attention_weight_max": 108.94514465332031,
      "activations/layer4_attention_weight_min": -108.68836975097656,
      "activations/layer5_attention_weight_max": 51.39431381225586,
      "activations/layer5_attention_weight_min": -60.24125671386719,
      "activations/layer6_attention_weight_max": 44.532962799072266,
      "activations/layer6_attention_weight_min": -47.99663162231445,
      "activations/layer7_attention_weight_max": 97.9998550415039,
      "activations/layer7_attention_weight_min": -97.25115966796875,
      "activations/layer8_attention_weight_max": 42.063812255859375,
      "activations/layer8_attention_weight_min": -45.11595153808594,
      "activations/layer9_attention_weight_max": 34.60771560668945,
      "activations/layer9_attention_weight_min": -35.464515686035156,
      "epoch": 16.94,
      "learning_rate": 4.114924242424242e-05,
      "loss": 2.7464,
      "step": 291600
    },
    {
      "activations/layer0_attention_weight_max": 15.195481300354004,
      "activations/layer0_attention_weight_min": -13.195646286010742,
      "activations/layer10_attention_weight_max": 46.99692153930664,
      "activations/layer10_attention_weight_min": -44.0675163269043,
      "activations/layer11_attention_weight_max": 42.51618957519531,
      "activations/layer11_attention_weight_min": -43.74652099609375,
      "activations/layer12_attention_weight_max": 37.04043197631836,
      "activations/layer12_attention_weight_min": -32.82377243041992,
      "activations/layer13_attention_weight_max": 58.64833068847656,
      "activations/layer13_attention_weight_min": -46.83744430541992,
      "activations/layer14_attention_weight_max": 64.93069458007812,
      "activations/layer14_attention_weight_min": -52.245723724365234,
      "activations/layer15_attention_weight_max": 59.46847915649414,
      "activations/layer15_attention_weight_min": -52.123443603515625,
      "activations/layer16_attention_weight_max": 42.887962341308594,
      "activations/layer16_attention_weight_min": -30.30674171447754,
      "activations/layer17_attention_weight_max": 37.05015182495117,
      "activations/layer17_attention_weight_min": -30.817012786865234,
      "activations/layer18_attention_weight_max": 35.64958572387695,
      "activations/layer18_attention_weight_min": -29.683879852294922,
      "activations/layer19_attention_weight_max": 46.76240158081055,
      "activations/layer19_attention_weight_min": -34.722259521484375,
      "activations/layer1_attention_weight_max": 15.929022789001465,
      "activations/layer1_attention_weight_min": -14.87356185913086,
      "activations/layer20_attention_weight_max": 36.514060974121094,
      "activations/layer20_attention_weight_min": -25.751115798950195,
      "activations/layer21_attention_weight_max": 34.27822494506836,
      "activations/layer21_attention_weight_min": -26.525558471679688,
      "activations/layer22_attention_weight_max": 58.11355972290039,
      "activations/layer22_attention_weight_min": -36.20011520385742,
      "activations/layer23_attention_weight_max": 36.28789520263672,
      "activations/layer23_attention_weight_min": -23.8892822265625,
      "activations/layer2_attention_weight_max": 36.02552795410156,
      "activations/layer2_attention_weight_min": -35.00082778930664,
      "activations/layer3_attention_weight_max": 106.32218170166016,
      "activations/layer3_attention_weight_min": -106.68250274658203,
      "activations/layer4_attention_weight_max": 113.53424072265625,
      "activations/layer4_attention_weight_min": -113.31829071044922,
      "activations/layer5_attention_weight_max": 52.90149688720703,
      "activations/layer5_attention_weight_min": -62.87846374511719,
      "activations/layer6_attention_weight_max": 48.340110778808594,
      "activations/layer6_attention_weight_min": -48.00773620605469,
      "activations/layer7_attention_weight_max": 104.73621368408203,
      "activations/layer7_attention_weight_min": -100.06719207763672,
      "activations/layer8_attention_weight_max": 54.1882209777832,
      "activations/layer8_attention_weight_min": -50.816017150878906,
      "activations/layer9_attention_weight_max": 40.56002426147461,
      "activations/layer9_attention_weight_min": -43.74232864379883,
      "epoch": 16.95,
      "learning_rate": 4.113030303030303e-05,
      "loss": 2.7423,
      "step": 291650
    },
    {
      "activations/layer0_attention_weight_max": 16.221187591552734,
      "activations/layer0_attention_weight_min": -12.258984565734863,
      "activations/layer10_attention_weight_max": 33.51632308959961,
      "activations/layer10_attention_weight_min": -35.613800048828125,
      "activations/layer11_attention_weight_max": 34.103271484375,
      "activations/layer11_attention_weight_min": -34.84648132324219,
      "activations/layer12_attention_weight_max": 27.10696029663086,
      "activations/layer12_attention_weight_min": -27.866926193237305,
      "activations/layer13_attention_weight_max": 38.943763732910156,
      "activations/layer13_attention_weight_min": -37.66771697998047,
      "activations/layer14_attention_weight_max": 41.77983093261719,
      "activations/layer14_attention_weight_min": -41.97557067871094,
      "activations/layer15_attention_weight_max": 41.134525299072266,
      "activations/layer15_attention_weight_min": -38.208030700683594,
      "activations/layer16_attention_weight_max": 30.461687088012695,
      "activations/layer16_attention_weight_min": -30.214553833007812,
      "activations/layer17_attention_weight_max": 30.760860443115234,
      "activations/layer17_attention_weight_min": -27.28383445739746,
      "activations/layer18_attention_weight_max": 31.949071884155273,
      "activations/layer18_attention_weight_min": -26.146875381469727,
      "activations/layer19_attention_weight_max": 35.8760986328125,
      "activations/layer19_attention_weight_min": -36.814666748046875,
      "activations/layer1_attention_weight_max": 16.60552406311035,
      "activations/layer1_attention_weight_min": -15.311378479003906,
      "activations/layer20_attention_weight_max": 30.070100784301758,
      "activations/layer20_attention_weight_min": -25.421951293945312,
      "activations/layer21_attention_weight_max": 28.544921875,
      "activations/layer21_attention_weight_min": -26.16427993774414,
      "activations/layer22_attention_weight_max": 39.518680572509766,
      "activations/layer22_attention_weight_min": -35.15460205078125,
      "activations/layer23_attention_weight_max": 31.862377166748047,
      "activations/layer23_attention_weight_min": -26.37051010131836,
      "activations/layer2_attention_weight_max": 33.195899963378906,
      "activations/layer2_attention_weight_min": -32.254371643066406,
      "activations/layer3_attention_weight_max": 97.76457977294922,
      "activations/layer3_attention_weight_min": -98.24784851074219,
      "activations/layer4_attention_weight_max": 109.46868133544922,
      "activations/layer4_attention_weight_min": -111.43779754638672,
      "activations/layer5_attention_weight_max": 51.93037033081055,
      "activations/layer5_attention_weight_min": -60.393760681152344,
      "activations/layer6_attention_weight_max": 46.534610748291016,
      "activations/layer6_attention_weight_min": -49.11075973510742,
      "activations/layer7_attention_weight_max": 94.72087097167969,
      "activations/layer7_attention_weight_min": -100.01817321777344,
      "activations/layer8_attention_weight_max": 42.829071044921875,
      "activations/layer8_attention_weight_min": -45.72890090942383,
      "activations/layer9_attention_weight_max": 34.44166946411133,
      "activations/layer9_attention_weight_min": -36.57912063598633,
      "epoch": 16.95,
      "learning_rate": 4.1111363636363635e-05,
      "loss": 2.7405,
      "step": 291700
    },
    {
      "activations/layer0_attention_weight_max": 15.52689266204834,
      "activations/layer0_attention_weight_min": -13.187657356262207,
      "activations/layer10_attention_weight_max": 32.65769958496094,
      "activations/layer10_attention_weight_min": -35.63570022583008,
      "activations/layer11_attention_weight_max": 34.0576286315918,
      "activations/layer11_attention_weight_min": -35.07044982910156,
      "activations/layer12_attention_weight_max": 27.581850051879883,
      "activations/layer12_attention_weight_min": -27.602998733520508,
      "activations/layer13_attention_weight_max": 40.96306610107422,
      "activations/layer13_attention_weight_min": -35.7135009765625,
      "activations/layer14_attention_weight_max": 43.23332595825195,
      "activations/layer14_attention_weight_min": -38.248451232910156,
      "activations/layer15_attention_weight_max": 40.04207992553711,
      "activations/layer15_attention_weight_min": -35.280609130859375,
      "activations/layer16_attention_weight_max": 30.392732620239258,
      "activations/layer16_attention_weight_min": -28.429609298706055,
      "activations/layer17_attention_weight_max": 34.63397216796875,
      "activations/layer17_attention_weight_min": -27.81402015686035,
      "activations/layer18_attention_weight_max": 33.565765380859375,
      "activations/layer18_attention_weight_min": -27.6872615814209,
      "activations/layer19_attention_weight_max": 36.43258285522461,
      "activations/layer19_attention_weight_min": -32.96653747558594,
      "activations/layer1_attention_weight_max": 15.4324951171875,
      "activations/layer1_attention_weight_min": -14.687410354614258,
      "activations/layer20_attention_weight_max": 33.219390869140625,
      "activations/layer20_attention_weight_min": -26.867525100708008,
      "activations/layer21_attention_weight_max": 31.789756774902344,
      "activations/layer21_attention_weight_min": -28.34221076965332,
      "activations/layer22_attention_weight_max": 54.03744888305664,
      "activations/layer22_attention_weight_min": -34.74699783325195,
      "activations/layer23_attention_weight_max": 35.671024322509766,
      "activations/layer23_attention_weight_min": -30.39949607849121,
      "activations/layer2_attention_weight_max": 33.206825256347656,
      "activations/layer2_attention_weight_min": -34.832489013671875,
      "activations/layer3_attention_weight_max": 96.5497817993164,
      "activations/layer3_attention_weight_min": -97.4188232421875,
      "activations/layer4_attention_weight_max": 105.68251037597656,
      "activations/layer4_attention_weight_min": -108.88887786865234,
      "activations/layer5_attention_weight_max": 49.106014251708984,
      "activations/layer5_attention_weight_min": -60.941062927246094,
      "activations/layer6_attention_weight_max": 44.86260223388672,
      "activations/layer6_attention_weight_min": -49.51976013183594,
      "activations/layer7_attention_weight_max": 95.18115997314453,
      "activations/layer7_attention_weight_min": -96.69996643066406,
      "activations/layer8_attention_weight_max": 42.99002456665039,
      "activations/layer8_attention_weight_min": -46.242069244384766,
      "activations/layer9_attention_weight_max": 33.30839920043945,
      "activations/layer9_attention_weight_min": -34.76273727416992,
      "epoch": 16.95,
      "learning_rate": 4.109242424242424e-05,
      "loss": 2.7458,
      "step": 291750
    },
    {
      "activations/layer0_attention_weight_max": 16.45347785949707,
      "activations/layer0_attention_weight_min": -13.584193229675293,
      "activations/layer10_attention_weight_max": 32.073665618896484,
      "activations/layer10_attention_weight_min": -32.17758560180664,
      "activations/layer11_attention_weight_max": 31.6988582611084,
      "activations/layer11_attention_weight_min": -32.97224426269531,
      "activations/layer12_attention_weight_max": 25.562965393066406,
      "activations/layer12_attention_weight_min": -24.425491333007812,
      "activations/layer13_attention_weight_max": 37.92856979370117,
      "activations/layer13_attention_weight_min": -33.04106903076172,
      "activations/layer14_attention_weight_max": 42.352516174316406,
      "activations/layer14_attention_weight_min": -36.15324401855469,
      "activations/layer15_attention_weight_max": 37.31767654418945,
      "activations/layer15_attention_weight_min": -33.09212112426758,
      "activations/layer16_attention_weight_max": 30.72399139404297,
      "activations/layer16_attention_weight_min": -26.419776916503906,
      "activations/layer17_attention_weight_max": 28.73512077331543,
      "activations/layer17_attention_weight_min": -25.794790267944336,
      "activations/layer18_attention_weight_max": 32.79420852661133,
      "activations/layer18_attention_weight_min": -24.722166061401367,
      "activations/layer19_attention_weight_max": 34.88555908203125,
      "activations/layer19_attention_weight_min": -36.190425872802734,
      "activations/layer1_attention_weight_max": 16.495506286621094,
      "activations/layer1_attention_weight_min": -14.897717475891113,
      "activations/layer20_attention_weight_max": 30.023387908935547,
      "activations/layer20_attention_weight_min": -26.882164001464844,
      "activations/layer21_attention_weight_max": 29.71426773071289,
      "activations/layer21_attention_weight_min": -25.020963668823242,
      "activations/layer22_attention_weight_max": 43.23992156982422,
      "activations/layer22_attention_weight_min": -33.3729133605957,
      "activations/layer23_attention_weight_max": 33.97007369995117,
      "activations/layer23_attention_weight_min": -27.101659774780273,
      "activations/layer2_attention_weight_max": 34.18553161621094,
      "activations/layer2_attention_weight_min": -33.900428771972656,
      "activations/layer3_attention_weight_max": 97.8904800415039,
      "activations/layer3_attention_weight_min": -98.5333251953125,
      "activations/layer4_attention_weight_max": 108.47417449951172,
      "activations/layer4_attention_weight_min": -112.3315658569336,
      "activations/layer5_attention_weight_max": 50.11207580566406,
      "activations/layer5_attention_weight_min": -62.607173919677734,
      "activations/layer6_attention_weight_max": 43.96524429321289,
      "activations/layer6_attention_weight_min": -45.71517562866211,
      "activations/layer7_attention_weight_max": 91.46297454833984,
      "activations/layer7_attention_weight_min": -91.7413101196289,
      "activations/layer8_attention_weight_max": 41.37959671020508,
      "activations/layer8_attention_weight_min": -43.1338005065918,
      "activations/layer9_attention_weight_max": 33.72970962524414,
      "activations/layer9_attention_weight_min": -32.76528549194336,
      "epoch": 16.96,
      "learning_rate": 4.1073484848484845e-05,
      "loss": 2.752,
      "step": 291800
    },
    {
      "activations/layer0_attention_weight_max": 16.239578247070312,
      "activations/layer0_attention_weight_min": -12.946736335754395,
      "activations/layer10_attention_weight_max": 34.757747650146484,
      "activations/layer10_attention_weight_min": -36.18294143676758,
      "activations/layer11_attention_weight_max": 35.84441375732422,
      "activations/layer11_attention_weight_min": -36.719478607177734,
      "activations/layer12_attention_weight_max": 28.269710540771484,
      "activations/layer12_attention_weight_min": -27.724042892456055,
      "activations/layer13_attention_weight_max": 42.66078567504883,
      "activations/layer13_attention_weight_min": -37.584877014160156,
      "activations/layer14_attention_weight_max": 47.07086181640625,
      "activations/layer14_attention_weight_min": -39.39125442504883,
      "activations/layer15_attention_weight_max": 40.74676513671875,
      "activations/layer15_attention_weight_min": -36.37498092651367,
      "activations/layer16_attention_weight_max": 32.78555679321289,
      "activations/layer16_attention_weight_min": -28.412202835083008,
      "activations/layer17_attention_weight_max": 33.15144729614258,
      "activations/layer17_attention_weight_min": -26.040857315063477,
      "activations/layer18_attention_weight_max": 34.16494369506836,
      "activations/layer18_attention_weight_min": -24.61626434326172,
      "activations/layer19_attention_weight_max": 36.4581184387207,
      "activations/layer19_attention_weight_min": -31.293670654296875,
      "activations/layer1_attention_weight_max": 15.891789436340332,
      "activations/layer1_attention_weight_min": -14.530540466308594,
      "activations/layer20_attention_weight_max": 31.46700668334961,
      "activations/layer20_attention_weight_min": -23.409269332885742,
      "activations/layer21_attention_weight_max": 29.968162536621094,
      "activations/layer21_attention_weight_min": -23.507932662963867,
      "activations/layer22_attention_weight_max": 44.38893127441406,
      "activations/layer22_attention_weight_min": -33.0860595703125,
      "activations/layer23_attention_weight_max": 32.60377502441406,
      "activations/layer23_attention_weight_min": -24.652761459350586,
      "activations/layer2_attention_weight_max": 33.078704833984375,
      "activations/layer2_attention_weight_min": -33.655601501464844,
      "activations/layer3_attention_weight_max": 103.37323760986328,
      "activations/layer3_attention_weight_min": -103.5068588256836,
      "activations/layer4_attention_weight_max": 111.6625747680664,
      "activations/layer4_attention_weight_min": -109.45694732666016,
      "activations/layer5_attention_weight_max": 51.44103240966797,
      "activations/layer5_attention_weight_min": -60.031578063964844,
      "activations/layer6_attention_weight_max": 46.941505432128906,
      "activations/layer6_attention_weight_min": -49.89826202392578,
      "activations/layer7_attention_weight_max": 93.9334487915039,
      "activations/layer7_attention_weight_min": -100.68711853027344,
      "activations/layer8_attention_weight_max": 43.753597259521484,
      "activations/layer8_attention_weight_min": -44.0493049621582,
      "activations/layer9_attention_weight_max": 32.76226806640625,
      "activations/layer9_attention_weight_min": -35.76029586791992,
      "epoch": 16.96,
      "learning_rate": 4.105454545454545e-05,
      "loss": 2.7442,
      "step": 291850
    },
    {
      "activations/layer0_attention_weight_max": 16.206287384033203,
      "activations/layer0_attention_weight_min": -13.641145706176758,
      "activations/layer10_attention_weight_max": 33.6768913269043,
      "activations/layer10_attention_weight_min": -34.20772171020508,
      "activations/layer11_attention_weight_max": 33.850372314453125,
      "activations/layer11_attention_weight_min": -34.022945404052734,
      "activations/layer12_attention_weight_max": 26.44827651977539,
      "activations/layer12_attention_weight_min": -25.926563262939453,
      "activations/layer13_attention_weight_max": 41.09770965576172,
      "activations/layer13_attention_weight_min": -37.1702880859375,
      "activations/layer14_attention_weight_max": 43.97706985473633,
      "activations/layer14_attention_weight_min": -39.090850830078125,
      "activations/layer15_attention_weight_max": 39.198795318603516,
      "activations/layer15_attention_weight_min": -36.18224334716797,
      "activations/layer16_attention_weight_max": 30.97564697265625,
      "activations/layer16_attention_weight_min": -28.448381423950195,
      "activations/layer17_attention_weight_max": 30.720415115356445,
      "activations/layer17_attention_weight_min": -26.09950065612793,
      "activations/layer18_attention_weight_max": 36.0632438659668,
      "activations/layer18_attention_weight_min": -26.536590576171875,
      "activations/layer19_attention_weight_max": 34.90971374511719,
      "activations/layer19_attention_weight_min": -34.397884368896484,
      "activations/layer1_attention_weight_max": 15.975446701049805,
      "activations/layer1_attention_weight_min": -16.777179718017578,
      "activations/layer20_attention_weight_max": 28.6143856048584,
      "activations/layer20_attention_weight_min": -25.11900520324707,
      "activations/layer21_attention_weight_max": 28.733428955078125,
      "activations/layer21_attention_weight_min": -25.724449157714844,
      "activations/layer22_attention_weight_max": 48.40620422363281,
      "activations/layer22_attention_weight_min": -34.124210357666016,
      "activations/layer23_attention_weight_max": 33.016441345214844,
      "activations/layer23_attention_weight_min": -28.858028411865234,
      "activations/layer2_attention_weight_max": 32.063560485839844,
      "activations/layer2_attention_weight_min": -33.078067779541016,
      "activations/layer3_attention_weight_max": 98.21475982666016,
      "activations/layer3_attention_weight_min": -98.0627670288086,
      "activations/layer4_attention_weight_max": 109.40123748779297,
      "activations/layer4_attention_weight_min": -107.0732421875,
      "activations/layer5_attention_weight_max": 51.500343322753906,
      "activations/layer5_attention_weight_min": -64.33899688720703,
      "activations/layer6_attention_weight_max": 43.19832992553711,
      "activations/layer6_attention_weight_min": -47.176231384277344,
      "activations/layer7_attention_weight_max": 95.7227554321289,
      "activations/layer7_attention_weight_min": -96.30284881591797,
      "activations/layer8_attention_weight_max": 41.123714447021484,
      "activations/layer8_attention_weight_min": -45.13158416748047,
      "activations/layer9_attention_weight_max": 32.099945068359375,
      "activations/layer9_attention_weight_min": -33.900089263916016,
      "epoch": 16.96,
      "learning_rate": 4.1035606060606055e-05,
      "loss": 2.7231,
      "step": 291900
    },
    {
      "activations/layer0_attention_weight_max": 16.195999145507812,
      "activations/layer0_attention_weight_min": -13.635766983032227,
      "activations/layer10_attention_weight_max": 34.33109664916992,
      "activations/layer10_attention_weight_min": -34.02002716064453,
      "activations/layer11_attention_weight_max": 36.07029724121094,
      "activations/layer11_attention_weight_min": -34.62507629394531,
      "activations/layer12_attention_weight_max": 26.376867294311523,
      "activations/layer12_attention_weight_min": -28.85252571105957,
      "activations/layer13_attention_weight_max": 38.7974853515625,
      "activations/layer13_attention_weight_min": -36.62614059448242,
      "activations/layer14_attention_weight_max": 43.32903289794922,
      "activations/layer14_attention_weight_min": -39.76818084716797,
      "activations/layer15_attention_weight_max": 40.03323745727539,
      "activations/layer15_attention_weight_min": -37.89248275756836,
      "activations/layer16_attention_weight_max": 29.14780616760254,
      "activations/layer16_attention_weight_min": -28.626102447509766,
      "activations/layer17_attention_weight_max": 30.372982025146484,
      "activations/layer17_attention_weight_min": -26.08355140686035,
      "activations/layer18_attention_weight_max": 34.027137756347656,
      "activations/layer18_attention_weight_min": -26.78099250793457,
      "activations/layer19_attention_weight_max": 35.68327331542969,
      "activations/layer19_attention_weight_min": -31.65555763244629,
      "activations/layer1_attention_weight_max": 15.549581527709961,
      "activations/layer1_attention_weight_min": -15.144092559814453,
      "activations/layer20_attention_weight_max": 29.254016876220703,
      "activations/layer20_attention_weight_min": -25.984561920166016,
      "activations/layer21_attention_weight_max": 27.856069564819336,
      "activations/layer21_attention_weight_min": -24.918970108032227,
      "activations/layer22_attention_weight_max": 38.234107971191406,
      "activations/layer22_attention_weight_min": -31.745264053344727,
      "activations/layer23_attention_weight_max": 31.72218132019043,
      "activations/layer23_attention_weight_min": -27.3466796875,
      "activations/layer2_attention_weight_max": 32.801570892333984,
      "activations/layer2_attention_weight_min": -32.341064453125,
      "activations/layer3_attention_weight_max": 95.68110656738281,
      "activations/layer3_attention_weight_min": -98.81591033935547,
      "activations/layer4_attention_weight_max": 107.32606506347656,
      "activations/layer4_attention_weight_min": -109.39139556884766,
      "activations/layer5_attention_weight_max": 51.474273681640625,
      "activations/layer5_attention_weight_min": -58.3345947265625,
      "activations/layer6_attention_weight_max": 45.53168487548828,
      "activations/layer6_attention_weight_min": -44.883724212646484,
      "activations/layer7_attention_weight_max": 91.97505950927734,
      "activations/layer7_attention_weight_min": -91.47457885742188,
      "activations/layer8_attention_weight_max": 39.67172622680664,
      "activations/layer8_attention_weight_min": -44.22357940673828,
      "activations/layer9_attention_weight_max": 32.376548767089844,
      "activations/layer9_attention_weight_min": -33.2296257019043,
      "epoch": 16.96,
      "learning_rate": 4.101666666666666e-05,
      "loss": 2.7413,
      "step": 291950
    },
    {
      "activations/layer0_attention_weight_max": 16.10900115966797,
      "activations/layer0_attention_weight_min": -12.385689735412598,
      "activations/layer10_attention_weight_max": 33.03632354736328,
      "activations/layer10_attention_weight_min": -32.84962463378906,
      "activations/layer11_attention_weight_max": 33.75669860839844,
      "activations/layer11_attention_weight_min": -33.047183990478516,
      "activations/layer12_attention_weight_max": 25.620691299438477,
      "activations/layer12_attention_weight_min": -24.327529907226562,
      "activations/layer13_attention_weight_max": 39.00463104248047,
      "activations/layer13_attention_weight_min": -33.63042068481445,
      "activations/layer14_attention_weight_max": 41.987186431884766,
      "activations/layer14_attention_weight_min": -36.52949523925781,
      "activations/layer15_attention_weight_max": 38.840091705322266,
      "activations/layer15_attention_weight_min": -33.504581451416016,
      "activations/layer16_attention_weight_max": 31.505903244018555,
      "activations/layer16_attention_weight_min": -29.451370239257812,
      "activations/layer17_attention_weight_max": 28.92520523071289,
      "activations/layer17_attention_weight_min": -25.339323043823242,
      "activations/layer18_attention_weight_max": 34.33407211303711,
      "activations/layer18_attention_weight_min": -26.5017147064209,
      "activations/layer19_attention_weight_max": 34.13020706176758,
      "activations/layer19_attention_weight_min": -32.58307647705078,
      "activations/layer1_attention_weight_max": 15.757070541381836,
      "activations/layer1_attention_weight_min": -14.29766845703125,
      "activations/layer20_attention_weight_max": 30.2117862701416,
      "activations/layer20_attention_weight_min": -25.469758987426758,
      "activations/layer21_attention_weight_max": 29.62148094177246,
      "activations/layer21_attention_weight_min": -26.478647232055664,
      "activations/layer22_attention_weight_max": 42.872493743896484,
      "activations/layer22_attention_weight_min": -35.861717224121094,
      "activations/layer23_attention_weight_max": 34.67454528808594,
      "activations/layer23_attention_weight_min": -26.351131439208984,
      "activations/layer2_attention_weight_max": 32.018516540527344,
      "activations/layer2_attention_weight_min": -32.94355773925781,
      "activations/layer3_attention_weight_max": 97.85565185546875,
      "activations/layer3_attention_weight_min": -98.13031005859375,
      "activations/layer4_attention_weight_max": 105.79500579833984,
      "activations/layer4_attention_weight_min": -106.30419921875,
      "activations/layer5_attention_weight_max": 49.68101119995117,
      "activations/layer5_attention_weight_min": -60.844444274902344,
      "activations/layer6_attention_weight_max": 42.27290344238281,
      "activations/layer6_attention_weight_min": -43.900794982910156,
      "activations/layer7_attention_weight_max": 87.60022735595703,
      "activations/layer7_attention_weight_min": -89.32134246826172,
      "activations/layer8_attention_weight_max": 40.585472106933594,
      "activations/layer8_attention_weight_min": -41.56171798706055,
      "activations/layer9_attention_weight_max": 31.200210571289062,
      "activations/layer9_attention_weight_min": -33.899417877197266,
      "epoch": 16.97,
      "learning_rate": 4.0997727272727265e-05,
      "loss": 2.7346,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.5102,
      "eval_samples_per_second": 504.572,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.5102,
      "eval_openwebtext_samples_per_second": 504.572,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9744,
      "eval_wikitext_samples_per_second": 230.953,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.6173,
      "eval_lambada_samples_per_second": 506.277,
      "step": 292000
    },
    {
      "activations/layer0_attention_weight_max": 17.661026000976562,
      "activations/layer0_attention_weight_min": -12.809290885925293,
      "activations/layer10_attention_weight_max": 44.44472122192383,
      "activations/layer10_attention_weight_min": -43.50004959106445,
      "activations/layer11_attention_weight_max": 43.21528244018555,
      "activations/layer11_attention_weight_min": -44.8653564453125,
      "activations/layer12_attention_weight_max": 28.49551773071289,
      "activations/layer12_attention_weight_min": -28.353805541992188,
      "activations/layer13_attention_weight_max": 38.583396911621094,
      "activations/layer13_attention_weight_min": -39.25621032714844,
      "activations/layer14_attention_weight_max": 46.20582962036133,
      "activations/layer14_attention_weight_min": -44.46209716796875,
      "activations/layer15_attention_weight_max": 41.463775634765625,
      "activations/layer15_attention_weight_min": -45.620521545410156,
      "activations/layer16_attention_weight_max": 30.221263885498047,
      "activations/layer16_attention_weight_min": -29.957611083984375,
      "activations/layer17_attention_weight_max": 29.940996170043945,
      "activations/layer17_attention_weight_min": -27.81248664855957,
      "activations/layer18_attention_weight_max": 35.75667953491211,
      "activations/layer18_attention_weight_min": -27.316097259521484,
      "activations/layer19_attention_weight_max": 32.070098876953125,
      "activations/layer19_attention_weight_min": -32.25254440307617,
      "activations/layer1_attention_weight_max": 16.573564529418945,
      "activations/layer1_attention_weight_min": -14.742679595947266,
      "activations/layer20_attention_weight_max": 28.43123435974121,
      "activations/layer20_attention_weight_min": -27.3332576751709,
      "activations/layer21_attention_weight_max": 28.61172866821289,
      "activations/layer21_attention_weight_min": -26.772552490234375,
      "activations/layer22_attention_weight_max": 39.54487228393555,
      "activations/layer22_attention_weight_min": -34.21310806274414,
      "activations/layer23_attention_weight_max": 30.820293426513672,
      "activations/layer23_attention_weight_min": -28.478696823120117,
      "activations/layer2_attention_weight_max": 35.600486755371094,
      "activations/layer2_attention_weight_min": -33.35173797607422,
      "activations/layer3_attention_weight_max": 101.77632904052734,
      "activations/layer3_attention_weight_min": -102.01026153564453,
      "activations/layer4_attention_weight_max": 108.68670654296875,
      "activations/layer4_attention_weight_min": -108.58302307128906,
      "activations/layer5_attention_weight_max": 45.73419952392578,
      "activations/layer5_attention_weight_min": -59.1930046081543,
      "activations/layer6_attention_weight_max": 41.760520935058594,
      "activations/layer6_attention_weight_min": -43.84889221191406,
      "activations/layer7_attention_weight_max": 95.7467041015625,
      "activations/layer7_attention_weight_min": -92.1653060913086,
      "activations/layer8_attention_weight_max": 44.96623229980469,
      "activations/layer8_attention_weight_min": -48.07195281982422,
      "activations/layer9_attention_weight_max": 39.474876403808594,
      "activations/layer9_attention_weight_min": -40.779319763183594,
      "epoch": 16.97,
      "learning_rate": 4.097878787878788e-05,
      "loss": 2.7462,
      "step": 292050
    },
    {
      "activations/layer0_attention_weight_max": 16.013612747192383,
      "activations/layer0_attention_weight_min": -13.966148376464844,
      "activations/layer10_attention_weight_max": 32.498287200927734,
      "activations/layer10_attention_weight_min": -33.67464828491211,
      "activations/layer11_attention_weight_max": 32.68008804321289,
      "activations/layer11_attention_weight_min": -32.39908981323242,
      "activations/layer12_attention_weight_max": 28.18803596496582,
      "activations/layer12_attention_weight_min": -27.147541046142578,
      "activations/layer13_attention_weight_max": 38.870155334472656,
      "activations/layer13_attention_weight_min": -37.66571044921875,
      "activations/layer14_attention_weight_max": 45.81050491333008,
      "activations/layer14_attention_weight_min": -38.675689697265625,
      "activations/layer15_attention_weight_max": 38.25615310668945,
      "activations/layer15_attention_weight_min": -35.297176361083984,
      "activations/layer16_attention_weight_max": 30.667016983032227,
      "activations/layer16_attention_weight_min": -27.455615997314453,
      "activations/layer17_attention_weight_max": 30.289377212524414,
      "activations/layer17_attention_weight_min": -26.814308166503906,
      "activations/layer18_attention_weight_max": 31.859882354736328,
      "activations/layer18_attention_weight_min": -25.574867248535156,
      "activations/layer19_attention_weight_max": 30.288639068603516,
      "activations/layer19_attention_weight_min": -30.559751510620117,
      "activations/layer1_attention_weight_max": 15.82262134552002,
      "activations/layer1_attention_weight_min": -16.427410125732422,
      "activations/layer20_attention_weight_max": 27.348787307739258,
      "activations/layer20_attention_weight_min": -27.078041076660156,
      "activations/layer21_attention_weight_max": 27.157438278198242,
      "activations/layer21_attention_weight_min": -24.804922103881836,
      "activations/layer22_attention_weight_max": 41.06317138671875,
      "activations/layer22_attention_weight_min": -31.74908447265625,
      "activations/layer23_attention_weight_max": 30.898365020751953,
      "activations/layer23_attention_weight_min": -27.313278198242188,
      "activations/layer2_attention_weight_max": 32.788639068603516,
      "activations/layer2_attention_weight_min": -33.50462341308594,
      "activations/layer3_attention_weight_max": 97.35529327392578,
      "activations/layer3_attention_weight_min": -99.58987426757812,
      "activations/layer4_attention_weight_max": 104.19136810302734,
      "activations/layer4_attention_weight_min": -107.67513275146484,
      "activations/layer5_attention_weight_max": 54.178550720214844,
      "activations/layer5_attention_weight_min": -61.21063995361328,
      "activations/layer6_attention_weight_max": 42.87404251098633,
      "activations/layer6_attention_weight_min": -44.71077346801758,
      "activations/layer7_attention_weight_max": 89.54782104492188,
      "activations/layer7_attention_weight_min": -90.40043640136719,
      "activations/layer8_attention_weight_max": 40.558223724365234,
      "activations/layer8_attention_weight_min": -42.213199615478516,
      "activations/layer9_attention_weight_max": 31.07832908630371,
      "activations/layer9_attention_weight_min": -33.925289154052734,
      "epoch": 16.97,
      "learning_rate": 4.095984848484848e-05,
      "loss": 2.745,
      "step": 292100
    },
    {
      "activations/layer0_attention_weight_max": 15.389168739318848,
      "activations/layer0_attention_weight_min": -12.872257232666016,
      "activations/layer10_attention_weight_max": 32.59379196166992,
      "activations/layer10_attention_weight_min": -34.75897216796875,
      "activations/layer11_attention_weight_max": 31.23444938659668,
      "activations/layer11_attention_weight_min": -32.121456146240234,
      "activations/layer12_attention_weight_max": 24.954607009887695,
      "activations/layer12_attention_weight_min": -24.95095443725586,
      "activations/layer13_attention_weight_max": 37.363258361816406,
      "activations/layer13_attention_weight_min": -35.178401947021484,
      "activations/layer14_attention_weight_max": 39.336151123046875,
      "activations/layer14_attention_weight_min": -35.59883117675781,
      "activations/layer15_attention_weight_max": 35.31358337402344,
      "activations/layer15_attention_weight_min": -36.63092041015625,
      "activations/layer16_attention_weight_max": 27.30495262145996,
      "activations/layer16_attention_weight_min": -28.073776245117188,
      "activations/layer17_attention_weight_max": 29.890811920166016,
      "activations/layer17_attention_weight_min": -26.11935043334961,
      "activations/layer18_attention_weight_max": 30.706430435180664,
      "activations/layer18_attention_weight_min": -25.66876983642578,
      "activations/layer19_attention_weight_max": 33.07099914550781,
      "activations/layer19_attention_weight_min": -30.750219345092773,
      "activations/layer1_attention_weight_max": 16.028846740722656,
      "activations/layer1_attention_weight_min": -14.695908546447754,
      "activations/layer20_attention_weight_max": 30.17804527282715,
      "activations/layer20_attention_weight_min": -24.719383239746094,
      "activations/layer21_attention_weight_max": 31.45676040649414,
      "activations/layer21_attention_weight_min": -24.120346069335938,
      "activations/layer22_attention_weight_max": 45.24676513671875,
      "activations/layer22_attention_weight_min": -30.58061981201172,
      "activations/layer23_attention_weight_max": 30.510093688964844,
      "activations/layer23_attention_weight_min": -27.120807647705078,
      "activations/layer2_attention_weight_max": 33.116641998291016,
      "activations/layer2_attention_weight_min": -33.13429641723633,
      "activations/layer3_attention_weight_max": 97.28810119628906,
      "activations/layer3_attention_weight_min": -98.62141418457031,
      "activations/layer4_attention_weight_max": 107.6735610961914,
      "activations/layer4_attention_weight_min": -109.20436096191406,
      "activations/layer5_attention_weight_max": 53.88967514038086,
      "activations/layer5_attention_weight_min": -62.8841667175293,
      "activations/layer6_attention_weight_max": 42.43299865722656,
      "activations/layer6_attention_weight_min": -45.646156311035156,
      "activations/layer7_attention_weight_max": 88.30686950683594,
      "activations/layer7_attention_weight_min": -96.42794036865234,
      "activations/layer8_attention_weight_max": 41.90257263183594,
      "activations/layer8_attention_weight_min": -42.53627014160156,
      "activations/layer9_attention_weight_max": 31.293943405151367,
      "activations/layer9_attention_weight_min": -32.96418762207031,
      "epoch": 16.98,
      "learning_rate": 4.094090909090908e-05,
      "loss": 2.7566,
      "step": 292150
    },
    {
      "activations/layer0_attention_weight_max": 16.176132202148438,
      "activations/layer0_attention_weight_min": -13.31859016418457,
      "activations/layer10_attention_weight_max": 33.141963958740234,
      "activations/layer10_attention_weight_min": -34.87153244018555,
      "activations/layer11_attention_weight_max": 32.96746063232422,
      "activations/layer11_attention_weight_min": -34.99085235595703,
      "activations/layer12_attention_weight_max": 24.69126319885254,
      "activations/layer12_attention_weight_min": -27.273670196533203,
      "activations/layer13_attention_weight_max": 37.1082649230957,
      "activations/layer13_attention_weight_min": -35.817325592041016,
      "activations/layer14_attention_weight_max": 43.881561279296875,
      "activations/layer14_attention_weight_min": -44.68798828125,
      "activations/layer15_attention_weight_max": 35.434906005859375,
      "activations/layer15_attention_weight_min": -38.544898986816406,
      "activations/layer16_attention_weight_max": 28.281190872192383,
      "activations/layer16_attention_weight_min": -29.524431228637695,
      "activations/layer17_attention_weight_max": 28.50632095336914,
      "activations/layer17_attention_weight_min": -29.623422622680664,
      "activations/layer18_attention_weight_max": 28.77460289001465,
      "activations/layer18_attention_weight_min": -27.276100158691406,
      "activations/layer19_attention_weight_max": 30.652524948120117,
      "activations/layer19_attention_weight_min": -31.312955856323242,
      "activations/layer1_attention_weight_max": 16.14836311340332,
      "activations/layer1_attention_weight_min": -13.968052864074707,
      "activations/layer20_attention_weight_max": 29.169570922851562,
      "activations/layer20_attention_weight_min": -25.661359786987305,
      "activations/layer21_attention_weight_max": 27.525901794433594,
      "activations/layer21_attention_weight_min": -25.532255172729492,
      "activations/layer22_attention_weight_max": 40.57367706298828,
      "activations/layer22_attention_weight_min": -31.260740280151367,
      "activations/layer23_attention_weight_max": 29.687292098999023,
      "activations/layer23_attention_weight_min": -27.0474910736084,
      "activations/layer2_attention_weight_max": 32.56058120727539,
      "activations/layer2_attention_weight_min": -33.503238677978516,
      "activations/layer3_attention_weight_max": 96.33966064453125,
      "activations/layer3_attention_weight_min": -99.43291473388672,
      "activations/layer4_attention_weight_max": 111.89920806884766,
      "activations/layer4_attention_weight_min": -107.25890350341797,
      "activations/layer5_attention_weight_max": 49.495819091796875,
      "activations/layer5_attention_weight_min": -66.63427734375,
      "activations/layer6_attention_weight_max": 44.77083969116211,
      "activations/layer6_attention_weight_min": -45.36918258666992,
      "activations/layer7_attention_weight_max": 96.05060577392578,
      "activations/layer7_attention_weight_min": -93.14338684082031,
      "activations/layer8_attention_weight_max": 39.72929763793945,
      "activations/layer8_attention_weight_min": -43.17916488647461,
      "activations/layer9_attention_weight_max": 31.778390884399414,
      "activations/layer9_attention_weight_min": -34.052574157714844,
      "epoch": 16.98,
      "learning_rate": 4.09219696969697e-05,
      "loss": 2.7391,
      "step": 292200
    },
    {
      "activations/layer0_attention_weight_max": 16.11404037475586,
      "activations/layer0_attention_weight_min": -13.171730995178223,
      "activations/layer10_attention_weight_max": 30.594552993774414,
      "activations/layer10_attention_weight_min": -31.209957122802734,
      "activations/layer11_attention_weight_max": 31.397573471069336,
      "activations/layer11_attention_weight_min": -31.602384567260742,
      "activations/layer12_attention_weight_max": 24.08111000061035,
      "activations/layer12_attention_weight_min": -24.803510665893555,
      "activations/layer13_attention_weight_max": 38.080196380615234,
      "activations/layer13_attention_weight_min": -35.37120056152344,
      "activations/layer14_attention_weight_max": 41.22129821777344,
      "activations/layer14_attention_weight_min": -34.97394561767578,
      "activations/layer15_attention_weight_max": 37.92255401611328,
      "activations/layer15_attention_weight_min": -34.97945022583008,
      "activations/layer16_attention_weight_max": 29.690797805786133,
      "activations/layer16_attention_weight_min": -26.783702850341797,
      "activations/layer17_attention_weight_max": 29.18720817565918,
      "activations/layer17_attention_weight_min": -27.690343856811523,
      "activations/layer18_attention_weight_max": 29.632402420043945,
      "activations/layer18_attention_weight_min": -24.537128448486328,
      "activations/layer19_attention_weight_max": 30.881025314331055,
      "activations/layer19_attention_weight_min": -30.321691513061523,
      "activations/layer1_attention_weight_max": 16.790565490722656,
      "activations/layer1_attention_weight_min": -14.704060554504395,
      "activations/layer20_attention_weight_max": 25.886611938476562,
      "activations/layer20_attention_weight_min": -25.310710906982422,
      "activations/layer21_attention_weight_max": 27.978538513183594,
      "activations/layer21_attention_weight_min": -24.712766647338867,
      "activations/layer22_attention_weight_max": 40.45996856689453,
      "activations/layer22_attention_weight_min": -30.015316009521484,
      "activations/layer23_attention_weight_max": 33.26869583129883,
      "activations/layer23_attention_weight_min": -23.929279327392578,
      "activations/layer2_attention_weight_max": 33.47854995727539,
      "activations/layer2_attention_weight_min": -34.76774215698242,
      "activations/layer3_attention_weight_max": 97.9687271118164,
      "activations/layer3_attention_weight_min": -96.83056640625,
      "activations/layer4_attention_weight_max": 105.80860900878906,
      "activations/layer4_attention_weight_min": -107.4751205444336,
      "activations/layer5_attention_weight_max": 50.04595947265625,
      "activations/layer5_attention_weight_min": -66.56008911132812,
      "activations/layer6_attention_weight_max": 42.41203689575195,
      "activations/layer6_attention_weight_min": -45.97599411010742,
      "activations/layer7_attention_weight_max": 87.05317687988281,
      "activations/layer7_attention_weight_min": -89.68123626708984,
      "activations/layer8_attention_weight_max": 38.0665283203125,
      "activations/layer8_attention_weight_min": -41.40572738647461,
      "activations/layer9_attention_weight_max": 30.78864860534668,
      "activations/layer9_attention_weight_min": -32.17271041870117,
      "epoch": 16.98,
      "learning_rate": 4.09030303030303e-05,
      "loss": 2.7313,
      "step": 292250
    },
    {
      "activations/layer0_attention_weight_max": 15.870162010192871,
      "activations/layer0_attention_weight_min": -12.475517272949219,
      "activations/layer10_attention_weight_max": 32.45703125,
      "activations/layer10_attention_weight_min": -31.592464447021484,
      "activations/layer11_attention_weight_max": 31.605396270751953,
      "activations/layer11_attention_weight_min": -31.05678939819336,
      "activations/layer12_attention_weight_max": 23.249725341796875,
      "activations/layer12_attention_weight_min": -24.868772506713867,
      "activations/layer13_attention_weight_max": 34.173824310302734,
      "activations/layer13_attention_weight_min": -36.42251205444336,
      "activations/layer14_attention_weight_max": 38.38984298706055,
      "activations/layer14_attention_weight_min": -38.35793685913086,
      "activations/layer15_attention_weight_max": 34.86621856689453,
      "activations/layer15_attention_weight_min": -37.41101837158203,
      "activations/layer16_attention_weight_max": 28.783594131469727,
      "activations/layer16_attention_weight_min": -28.650550842285156,
      "activations/layer17_attention_weight_max": 28.342771530151367,
      "activations/layer17_attention_weight_min": -25.43152618408203,
      "activations/layer18_attention_weight_max": 30.012760162353516,
      "activations/layer18_attention_weight_min": -26.671276092529297,
      "activations/layer19_attention_weight_max": 31.393085479736328,
      "activations/layer19_attention_weight_min": -35.933956146240234,
      "activations/layer1_attention_weight_max": 16.219356536865234,
      "activations/layer1_attention_weight_min": -17.78367042541504,
      "activations/layer20_attention_weight_max": 26.581876754760742,
      "activations/layer20_attention_weight_min": -27.184120178222656,
      "activations/layer21_attention_weight_max": 24.883054733276367,
      "activations/layer21_attention_weight_min": -26.291545867919922,
      "activations/layer22_attention_weight_max": 35.90872573852539,
      "activations/layer22_attention_weight_min": -33.39265441894531,
      "activations/layer23_attention_weight_max": 30.54111099243164,
      "activations/layer23_attention_weight_min": -27.63812255859375,
      "activations/layer2_attention_weight_max": 34.164344787597656,
      "activations/layer2_attention_weight_min": -34.81406784057617,
      "activations/layer3_attention_weight_max": 100.2352523803711,
      "activations/layer3_attention_weight_min": -104.85598754882812,
      "activations/layer4_attention_weight_max": 110.09871673583984,
      "activations/layer4_attention_weight_min": -120.1771240234375,
      "activations/layer5_attention_weight_max": 49.80794906616211,
      "activations/layer5_attention_weight_min": -62.868560791015625,
      "activations/layer6_attention_weight_max": 44.20582962036133,
      "activations/layer6_attention_weight_min": -49.53346633911133,
      "activations/layer7_attention_weight_max": 90.65525817871094,
      "activations/layer7_attention_weight_min": -95.12466430664062,
      "activations/layer8_attention_weight_max": 37.85459899902344,
      "activations/layer8_attention_weight_min": -43.34670639038086,
      "activations/layer9_attention_weight_max": 31.056299209594727,
      "activations/layer9_attention_weight_min": -32.20746994018555,
      "epoch": 16.98,
      "learning_rate": 4.088409090909091e-05,
      "loss": 2.7421,
      "step": 292300
    },
    {
      "activations/layer0_attention_weight_max": 14.85300064086914,
      "activations/layer0_attention_weight_min": -12.45148754119873,
      "activations/layer10_attention_weight_max": 33.94776916503906,
      "activations/layer10_attention_weight_min": -33.32313537597656,
      "activations/layer11_attention_weight_max": 34.716651916503906,
      "activations/layer11_attention_weight_min": -36.81671905517578,
      "activations/layer12_attention_weight_max": 26.971208572387695,
      "activations/layer12_attention_weight_min": -27.639589309692383,
      "activations/layer13_attention_weight_max": 40.47279357910156,
      "activations/layer13_attention_weight_min": -38.001792907714844,
      "activations/layer14_attention_weight_max": 41.8759651184082,
      "activations/layer14_attention_weight_min": -40.28550338745117,
      "activations/layer15_attention_weight_max": 43.335357666015625,
      "activations/layer15_attention_weight_min": -39.600791931152344,
      "activations/layer16_attention_weight_max": 32.00445556640625,
      "activations/layer16_attention_weight_min": -31.142480850219727,
      "activations/layer17_attention_weight_max": 29.67810821533203,
      "activations/layer17_attention_weight_min": -27.839487075805664,
      "activations/layer18_attention_weight_max": 32.532814025878906,
      "activations/layer18_attention_weight_min": -27.429988861083984,
      "activations/layer19_attention_weight_max": 39.08670425415039,
      "activations/layer19_attention_weight_min": -34.796871185302734,
      "activations/layer1_attention_weight_max": 15.853286743164062,
      "activations/layer1_attention_weight_min": -15.773526191711426,
      "activations/layer20_attention_weight_max": 31.647897720336914,
      "activations/layer20_attention_weight_min": -26.429080963134766,
      "activations/layer21_attention_weight_max": 29.848541259765625,
      "activations/layer21_attention_weight_min": -25.276769638061523,
      "activations/layer22_attention_weight_max": 42.786075592041016,
      "activations/layer22_attention_weight_min": -35.11398696899414,
      "activations/layer23_attention_weight_max": 31.20800018310547,
      "activations/layer23_attention_weight_min": -27.07221031188965,
      "activations/layer2_attention_weight_max": 34.31569290161133,
      "activations/layer2_attention_weight_min": -34.551612854003906,
      "activations/layer3_attention_weight_max": 102.6593017578125,
      "activations/layer3_attention_weight_min": -103.24124145507812,
      "activations/layer4_attention_weight_max": 111.9639892578125,
      "activations/layer4_attention_weight_min": -111.06951904296875,
      "activations/layer5_attention_weight_max": 52.423095703125,
      "activations/layer5_attention_weight_min": -63.77088928222656,
      "activations/layer6_attention_weight_max": 45.538429260253906,
      "activations/layer6_attention_weight_min": -47.384029388427734,
      "activations/layer7_attention_weight_max": 96.0146255493164,
      "activations/layer7_attention_weight_min": -97.94286346435547,
      "activations/layer8_attention_weight_max": 41.777435302734375,
      "activations/layer8_attention_weight_min": -43.75332260131836,
      "activations/layer9_attention_weight_max": 34.04585647583008,
      "activations/layer9_attention_weight_min": -33.1973991394043,
      "epoch": 16.99,
      "learning_rate": 4.086515151515151e-05,
      "loss": 2.7435,
      "step": 292350
    },
    {
      "activations/layer0_attention_weight_max": 15.649930000305176,
      "activations/layer0_attention_weight_min": -13.328880310058594,
      "activations/layer10_attention_weight_max": 42.62662887573242,
      "activations/layer10_attention_weight_min": -38.92721176147461,
      "activations/layer11_attention_weight_max": 40.020782470703125,
      "activations/layer11_attention_weight_min": -40.825401306152344,
      "activations/layer12_attention_weight_max": 28.686735153198242,
      "activations/layer12_attention_weight_min": -28.364072799682617,
      "activations/layer13_attention_weight_max": 42.82921600341797,
      "activations/layer13_attention_weight_min": -38.08524703979492,
      "activations/layer14_attention_weight_max": 46.674049377441406,
      "activations/layer14_attention_weight_min": -42.45882034301758,
      "activations/layer15_attention_weight_max": 45.452754974365234,
      "activations/layer15_attention_weight_min": -37.814056396484375,
      "activations/layer16_attention_weight_max": 34.59622573852539,
      "activations/layer16_attention_weight_min": -29.113697052001953,
      "activations/layer17_attention_weight_max": 30.525320053100586,
      "activations/layer17_attention_weight_min": -27.66706085205078,
      "activations/layer18_attention_weight_max": 33.28956985473633,
      "activations/layer18_attention_weight_min": -25.22011947631836,
      "activations/layer19_attention_weight_max": 37.41874313354492,
      "activations/layer19_attention_weight_min": -33.079811096191406,
      "activations/layer1_attention_weight_max": 16.5372257232666,
      "activations/layer1_attention_weight_min": -14.971671104431152,
      "activations/layer20_attention_weight_max": 29.84663963317871,
      "activations/layer20_attention_weight_min": -28.366714477539062,
      "activations/layer21_attention_weight_max": 33.9783935546875,
      "activations/layer21_attention_weight_min": -26.33614158630371,
      "activations/layer22_attention_weight_max": 44.88498306274414,
      "activations/layer22_attention_weight_min": -32.07113265991211,
      "activations/layer23_attention_weight_max": 34.576988220214844,
      "activations/layer23_attention_weight_min": -25.971988677978516,
      "activations/layer2_attention_weight_max": 34.76356887817383,
      "activations/layer2_attention_weight_min": -33.309932708740234,
      "activations/layer3_attention_weight_max": 100.02263641357422,
      "activations/layer3_attention_weight_min": -101.81043243408203,
      "activations/layer4_attention_weight_max": 109.49991607666016,
      "activations/layer4_attention_weight_min": -108.87689208984375,
      "activations/layer5_attention_weight_max": 54.931846618652344,
      "activations/layer5_attention_weight_min": -61.141578674316406,
      "activations/layer6_attention_weight_max": 46.04814147949219,
      "activations/layer6_attention_weight_min": -46.18170928955078,
      "activations/layer7_attention_weight_max": 102.88314056396484,
      "activations/layer7_attention_weight_min": -93.52579498291016,
      "activations/layer8_attention_weight_max": 46.90507888793945,
      "activations/layer8_attention_weight_min": -45.08363342285156,
      "activations/layer9_attention_weight_max": 36.57679748535156,
      "activations/layer9_attention_weight_min": -37.81901168823242,
      "epoch": 16.99,
      "learning_rate": 4.0846212121212124e-05,
      "loss": 2.7303,
      "step": 292400
    },
    {
      "activations/layer0_attention_weight_max": 15.623759269714355,
      "activations/layer0_attention_weight_min": -12.35896110534668,
      "activations/layer10_attention_weight_max": 35.03596878051758,
      "activations/layer10_attention_weight_min": -39.886566162109375,
      "activations/layer11_attention_weight_max": 34.02907180786133,
      "activations/layer11_attention_weight_min": -37.11448287963867,
      "activations/layer12_attention_weight_max": 26.491043090820312,
      "activations/layer12_attention_weight_min": -25.89877700805664,
      "activations/layer13_attention_weight_max": 38.806724548339844,
      "activations/layer13_attention_weight_min": -38.6854133605957,
      "activations/layer14_attention_weight_max": 40.828834533691406,
      "activations/layer14_attention_weight_min": -37.6790885925293,
      "activations/layer15_attention_weight_max": 39.494171142578125,
      "activations/layer15_attention_weight_min": -33.95111846923828,
      "activations/layer16_attention_weight_max": 28.995086669921875,
      "activations/layer16_attention_weight_min": -28.26323890686035,
      "activations/layer17_attention_weight_max": 27.67891502380371,
      "activations/layer17_attention_weight_min": -25.629432678222656,
      "activations/layer18_attention_weight_max": 29.522377014160156,
      "activations/layer18_attention_weight_min": -24.639951705932617,
      "activations/layer19_attention_weight_max": 33.80122375488281,
      "activations/layer19_attention_weight_min": -32.133277893066406,
      "activations/layer1_attention_weight_max": 15.811107635498047,
      "activations/layer1_attention_weight_min": -15.004813194274902,
      "activations/layer20_attention_weight_max": 31.97032356262207,
      "activations/layer20_attention_weight_min": -25.554027557373047,
      "activations/layer21_attention_weight_max": 30.292699813842773,
      "activations/layer21_attention_weight_min": -25.843875885009766,
      "activations/layer22_attention_weight_max": 39.93977355957031,
      "activations/layer22_attention_weight_min": -31.52915382385254,
      "activations/layer23_attention_weight_max": 31.48715591430664,
      "activations/layer23_attention_weight_min": -24.933170318603516,
      "activations/layer2_attention_weight_max": 32.76731872558594,
      "activations/layer2_attention_weight_min": -33.460723876953125,
      "activations/layer3_attention_weight_max": 99.9837417602539,
      "activations/layer3_attention_weight_min": -100.15767669677734,
      "activations/layer4_attention_weight_max": 109.00186920166016,
      "activations/layer4_attention_weight_min": -106.35560607910156,
      "activations/layer5_attention_weight_max": 52.756221771240234,
      "activations/layer5_attention_weight_min": -63.65480041503906,
      "activations/layer6_attention_weight_max": 46.32606506347656,
      "activations/layer6_attention_weight_min": -50.13505172729492,
      "activations/layer7_attention_weight_max": 95.82687377929688,
      "activations/layer7_attention_weight_min": -103.32979583740234,
      "activations/layer8_attention_weight_max": 43.80247116088867,
      "activations/layer8_attention_weight_min": -50.29963302612305,
      "activations/layer9_attention_weight_max": 32.93081283569336,
      "activations/layer9_attention_weight_min": -39.29963302612305,
      "epoch": 16.99,
      "learning_rate": 4.0827272727272725e-05,
      "loss": 2.7494,
      "step": 292450
    },
    {
      "activations/layer0_attention_weight_max": 16.230575561523438,
      "activations/layer0_attention_weight_min": -12.851456642150879,
      "activations/layer10_attention_weight_max": 32.55426788330078,
      "activations/layer10_attention_weight_min": -33.57183837890625,
      "activations/layer11_attention_weight_max": 32.673377990722656,
      "activations/layer11_attention_weight_min": -33.86548614501953,
      "activations/layer12_attention_weight_max": 24.1444034576416,
      "activations/layer12_attention_weight_min": -25.3880672454834,
      "activations/layer13_attention_weight_max": 37.32455062866211,
      "activations/layer13_attention_weight_min": -34.518882751464844,
      "activations/layer14_attention_weight_max": 38.110103607177734,
      "activations/layer14_attention_weight_min": -36.74795913696289,
      "activations/layer15_attention_weight_max": 35.703514099121094,
      "activations/layer15_attention_weight_min": -33.43341064453125,
      "activations/layer16_attention_weight_max": 28.048477172851562,
      "activations/layer16_attention_weight_min": -26.486431121826172,
      "activations/layer17_attention_weight_max": 27.000551223754883,
      "activations/layer17_attention_weight_min": -25.363887786865234,
      "activations/layer18_attention_weight_max": 30.821123123168945,
      "activations/layer18_attention_weight_min": -28.145099639892578,
      "activations/layer19_attention_weight_max": 33.14336013793945,
      "activations/layer19_attention_weight_min": -32.5876579284668,
      "activations/layer1_attention_weight_max": 16.28191566467285,
      "activations/layer1_attention_weight_min": -15.126677513122559,
      "activations/layer20_attention_weight_max": 29.348119735717773,
      "activations/layer20_attention_weight_min": -28.108806610107422,
      "activations/layer21_attention_weight_max": 28.434429168701172,
      "activations/layer21_attention_weight_min": -25.627405166625977,
      "activations/layer22_attention_weight_max": 38.58602523803711,
      "activations/layer22_attention_weight_min": -33.083377838134766,
      "activations/layer23_attention_weight_max": 30.884679794311523,
      "activations/layer23_attention_weight_min": -26.404403686523438,
      "activations/layer2_attention_weight_max": 33.127845764160156,
      "activations/layer2_attention_weight_min": -32.53036880493164,
      "activations/layer3_attention_weight_max": 93.28583526611328,
      "activations/layer3_attention_weight_min": -95.64221954345703,
      "activations/layer4_attention_weight_max": 105.50847625732422,
      "activations/layer4_attention_weight_min": -105.6152572631836,
      "activations/layer5_attention_weight_max": 48.91947937011719,
      "activations/layer5_attention_weight_min": -61.146263122558594,
      "activations/layer6_attention_weight_max": 41.213661193847656,
      "activations/layer6_attention_weight_min": -45.64847183227539,
      "activations/layer7_attention_weight_max": 87.15815734863281,
      "activations/layer7_attention_weight_min": -95.50590515136719,
      "activations/layer8_attention_weight_max": 40.768798828125,
      "activations/layer8_attention_weight_min": -41.0311164855957,
      "activations/layer9_attention_weight_max": 30.917499542236328,
      "activations/layer9_attention_weight_min": -34.71257400512695,
      "epoch": 17.0,
      "learning_rate": 4.080833333333333e-05,
      "loss": 2.7391,
      "step": 292500
    },
    {
      "activations/layer0_attention_weight_max": 16.19615364074707,
      "activations/layer0_attention_weight_min": -13.839715957641602,
      "activations/layer10_attention_weight_max": 33.81450653076172,
      "activations/layer10_attention_weight_min": -32.9110107421875,
      "activations/layer11_attention_weight_max": 33.11703109741211,
      "activations/layer11_attention_weight_min": -34.633819580078125,
      "activations/layer12_attention_weight_max": 25.47315216064453,
      "activations/layer12_attention_weight_min": -25.012866973876953,
      "activations/layer13_attention_weight_max": 37.78539276123047,
      "activations/layer13_attention_weight_min": -33.215152740478516,
      "activations/layer14_attention_weight_max": 40.645877838134766,
      "activations/layer14_attention_weight_min": -35.24154281616211,
      "activations/layer15_attention_weight_max": 35.97743225097656,
      "activations/layer15_attention_weight_min": -34.01499938964844,
      "activations/layer16_attention_weight_max": 30.710176467895508,
      "activations/layer16_attention_weight_min": -28.65477180480957,
      "activations/layer17_attention_weight_max": 31.032915115356445,
      "activations/layer17_attention_weight_min": -26.014007568359375,
      "activations/layer18_attention_weight_max": 30.310251235961914,
      "activations/layer18_attention_weight_min": -25.690778732299805,
      "activations/layer19_attention_weight_max": 31.604568481445312,
      "activations/layer19_attention_weight_min": -31.77534294128418,
      "activations/layer1_attention_weight_max": 16.45867156982422,
      "activations/layer1_attention_weight_min": -15.282551765441895,
      "activations/layer20_attention_weight_max": 27.19347381591797,
      "activations/layer20_attention_weight_min": -24.824068069458008,
      "activations/layer21_attention_weight_max": 27.061277389526367,
      "activations/layer21_attention_weight_min": -25.478599548339844,
      "activations/layer22_attention_weight_max": 40.82892608642578,
      "activations/layer22_attention_weight_min": -30.81369400024414,
      "activations/layer23_attention_weight_max": 31.511493682861328,
      "activations/layer23_attention_weight_min": -26.494489669799805,
      "activations/layer2_attention_weight_max": 37.24688720703125,
      "activations/layer2_attention_weight_min": -35.483734130859375,
      "activations/layer3_attention_weight_max": 101.97987365722656,
      "activations/layer3_attention_weight_min": -100.21903228759766,
      "activations/layer4_attention_weight_max": 107.36418914794922,
      "activations/layer4_attention_weight_min": -114.20006561279297,
      "activations/layer5_attention_weight_max": 52.78459930419922,
      "activations/layer5_attention_weight_min": -62.68658447265625,
      "activations/layer6_attention_weight_max": 43.55122756958008,
      "activations/layer6_attention_weight_min": -45.45736312866211,
      "activations/layer7_attention_weight_max": 91.09573364257812,
      "activations/layer7_attention_weight_min": -97.78280639648438,
      "activations/layer8_attention_weight_max": 42.885257720947266,
      "activations/layer8_attention_weight_min": -44.35221862792969,
      "activations/layer9_attention_weight_max": 30.204730987548828,
      "activations/layer9_attention_weight_min": -34.279380798339844,
      "epoch": 17.0,
      "learning_rate": 4.0789393939393935e-05,
      "loss": 2.7388,
      "step": 292550
    },
    {
      "activations/layer0_attention_weight_max": 16.040523529052734,
      "activations/layer0_attention_weight_min": -14.76910400390625,
      "activations/layer10_attention_weight_max": 35.49268341064453,
      "activations/layer10_attention_weight_min": -33.240806579589844,
      "activations/layer11_attention_weight_max": 33.25390625,
      "activations/layer11_attention_weight_min": -33.57390213012695,
      "activations/layer12_attention_weight_max": 28.079219818115234,
      "activations/layer12_attention_weight_min": -27.943464279174805,
      "activations/layer13_attention_weight_max": 43.753623962402344,
      "activations/layer13_attention_weight_min": -37.9640998840332,
      "activations/layer14_attention_weight_max": 44.086856842041016,
      "activations/layer14_attention_weight_min": -38.85895538330078,
      "activations/layer15_attention_weight_max": 43.863887786865234,
      "activations/layer15_attention_weight_min": -36.69023513793945,
      "activations/layer16_attention_weight_max": 30.874988555908203,
      "activations/layer16_attention_weight_min": -28.860464096069336,
      "activations/layer17_attention_weight_max": 30.86737823486328,
      "activations/layer17_attention_weight_min": -25.49911117553711,
      "activations/layer18_attention_weight_max": 34.5933723449707,
      "activations/layer18_attention_weight_min": -25.909303665161133,
      "activations/layer19_attention_weight_max": 37.084434509277344,
      "activations/layer19_attention_weight_min": -30.033674240112305,
      "activations/layer1_attention_weight_max": 17.654619216918945,
      "activations/layer1_attention_weight_min": -14.88940143585205,
      "activations/layer20_attention_weight_max": 32.62440872192383,
      "activations/layer20_attention_weight_min": -25.01498031616211,
      "activations/layer21_attention_weight_max": 34.36960220336914,
      "activations/layer21_attention_weight_min": -25.957286834716797,
      "activations/layer22_attention_weight_max": 44.54053497314453,
      "activations/layer22_attention_weight_min": -30.92110824584961,
      "activations/layer23_attention_weight_max": 33.32107925415039,
      "activations/layer23_attention_weight_min": -24.17146873474121,
      "activations/layer2_attention_weight_max": 33.54721450805664,
      "activations/layer2_attention_weight_min": -34.079715728759766,
      "activations/layer3_attention_weight_max": 90.87450408935547,
      "activations/layer3_attention_weight_min": -96.98098754882812,
      "activations/layer4_attention_weight_max": 101.85738372802734,
      "activations/layer4_attention_weight_min": -106.35533905029297,
      "activations/layer5_attention_weight_max": 50.28170394897461,
      "activations/layer5_attention_weight_min": -65.23995208740234,
      "activations/layer6_attention_weight_max": 44.05038070678711,
      "activations/layer6_attention_weight_min": -45.825984954833984,
      "activations/layer7_attention_weight_max": 88.95106506347656,
      "activations/layer7_attention_weight_min": -92.90129089355469,
      "activations/layer8_attention_weight_max": 41.794090270996094,
      "activations/layer8_attention_weight_min": -41.504188537597656,
      "activations/layer9_attention_weight_max": 33.11444091796875,
      "activations/layer9_attention_weight_min": -33.1670036315918,
      "epoch": 17.0,
      "learning_rate": 4.0770075757575754e-05,
      "loss": 2.7856,
      "step": 292600
    },
    {
      "activations/layer0_attention_weight_max": 16.02269744873047,
      "activations/layer0_attention_weight_min": -13.09703540802002,
      "activations/layer10_attention_weight_max": 34.877593994140625,
      "activations/layer10_attention_weight_min": -34.0426139831543,
      "activations/layer11_attention_weight_max": 32.960479736328125,
      "activations/layer11_attention_weight_min": -32.8534049987793,
      "activations/layer12_attention_weight_max": 27.771493911743164,
      "activations/layer12_attention_weight_min": -27.948043823242188,
      "activations/layer13_attention_weight_max": 40.2381591796875,
      "activations/layer13_attention_weight_min": -36.45000457763672,
      "activations/layer14_attention_weight_max": 43.01528549194336,
      "activations/layer14_attention_weight_min": -40.20688247680664,
      "activations/layer15_attention_weight_max": 37.593807220458984,
      "activations/layer15_attention_weight_min": -37.225215911865234,
      "activations/layer16_attention_weight_max": 27.65127182006836,
      "activations/layer16_attention_weight_min": -28.91750717163086,
      "activations/layer17_attention_weight_max": 27.760417938232422,
      "activations/layer17_attention_weight_min": -27.922510147094727,
      "activations/layer18_attention_weight_max": 31.776079177856445,
      "activations/layer18_attention_weight_min": -26.87687873840332,
      "activations/layer19_attention_weight_max": 33.99644470214844,
      "activations/layer19_attention_weight_min": -35.39553451538086,
      "activations/layer1_attention_weight_max": 15.741739273071289,
      "activations/layer1_attention_weight_min": -14.973199844360352,
      "activations/layer20_attention_weight_max": 29.769683837890625,
      "activations/layer20_attention_weight_min": -26.406078338623047,
      "activations/layer21_attention_weight_max": 32.54695510864258,
      "activations/layer21_attention_weight_min": -25.532609939575195,
      "activations/layer22_attention_weight_max": 46.05119705200195,
      "activations/layer22_attention_weight_min": -35.03614807128906,
      "activations/layer23_attention_weight_max": 31.252309799194336,
      "activations/layer23_attention_weight_min": -29.82073974609375,
      "activations/layer2_attention_weight_max": 33.96027755737305,
      "activations/layer2_attention_weight_min": -33.7758674621582,
      "activations/layer3_attention_weight_max": 97.92054748535156,
      "activations/layer3_attention_weight_min": -102.60184478759766,
      "activations/layer4_attention_weight_max": 105.86014556884766,
      "activations/layer4_attention_weight_min": -113.91719818115234,
      "activations/layer5_attention_weight_max": 49.288726806640625,
      "activations/layer5_attention_weight_min": -60.456642150878906,
      "activations/layer6_attention_weight_max": 44.30830383300781,
      "activations/layer6_attention_weight_min": -45.72980499267578,
      "activations/layer7_attention_weight_max": 91.9260025024414,
      "activations/layer7_attention_weight_min": -96.94347381591797,
      "activations/layer8_attention_weight_max": 40.20860290527344,
      "activations/layer8_attention_weight_min": -43.07522964477539,
      "activations/layer9_attention_weight_max": 33.113590240478516,
      "activations/layer9_attention_weight_min": -34.77775573730469,
      "epoch": 17.0,
      "learning_rate": 4.075113636363636e-05,
      "loss": 2.7237,
      "step": 292650
    },
    {
      "activations/layer0_attention_weight_max": 16.13237190246582,
      "activations/layer0_attention_weight_min": -14.155341148376465,
      "activations/layer10_attention_weight_max": 32.53596878051758,
      "activations/layer10_attention_weight_min": -33.31904602050781,
      "activations/layer11_attention_weight_max": 35.73573303222656,
      "activations/layer11_attention_weight_min": -34.21782684326172,
      "activations/layer12_attention_weight_max": 25.74816131591797,
      "activations/layer12_attention_weight_min": -26.46306037902832,
      "activations/layer13_attention_weight_max": 37.4400634765625,
      "activations/layer13_attention_weight_min": -35.83929443359375,
      "activations/layer14_attention_weight_max": 42.75922775268555,
      "activations/layer14_attention_weight_min": -37.88614273071289,
      "activations/layer15_attention_weight_max": 39.0336799621582,
      "activations/layer15_attention_weight_min": -34.263980865478516,
      "activations/layer16_attention_weight_max": 30.008535385131836,
      "activations/layer16_attention_weight_min": -27.68484115600586,
      "activations/layer17_attention_weight_max": 29.31425666809082,
      "activations/layer17_attention_weight_min": -25.9351749420166,
      "activations/layer18_attention_weight_max": 32.69817352294922,
      "activations/layer18_attention_weight_min": -24.69377326965332,
      "activations/layer19_attention_weight_max": 33.41510009765625,
      "activations/layer19_attention_weight_min": -30.427011489868164,
      "activations/layer1_attention_weight_max": 15.881775856018066,
      "activations/layer1_attention_weight_min": -15.092103004455566,
      "activations/layer20_attention_weight_max": 30.29616928100586,
      "activations/layer20_attention_weight_min": -26.921892166137695,
      "activations/layer21_attention_weight_max": 29.574691772460938,
      "activations/layer21_attention_weight_min": -27.084056854248047,
      "activations/layer22_attention_weight_max": 41.00978088378906,
      "activations/layer22_attention_weight_min": -33.27756881713867,
      "activations/layer23_attention_weight_max": 34.55201721191406,
      "activations/layer23_attention_weight_min": -26.351219177246094,
      "activations/layer2_attention_weight_max": 33.57316589355469,
      "activations/layer2_attention_weight_min": -32.54613494873047,
      "activations/layer3_attention_weight_max": 94.97171020507812,
      "activations/layer3_attention_weight_min": -92.38520050048828,
      "activations/layer4_attention_weight_max": 104.36956024169922,
      "activations/layer4_attention_weight_min": -106.40870666503906,
      "activations/layer5_attention_weight_max": 48.48075485229492,
      "activations/layer5_attention_weight_min": -57.9677734375,
      "activations/layer6_attention_weight_max": 44.417823791503906,
      "activations/layer6_attention_weight_min": -45.84990310668945,
      "activations/layer7_attention_weight_max": 93.31912231445312,
      "activations/layer7_attention_weight_min": -90.05144500732422,
      "activations/layer8_attention_weight_max": 40.112064361572266,
      "activations/layer8_attention_weight_min": -42.93604278564453,
      "activations/layer9_attention_weight_max": 32.22426986694336,
      "activations/layer9_attention_weight_min": -33.11778259277344,
      "epoch": 17.01,
      "learning_rate": 4.073257575757575e-05,
      "loss": 2.7476,
      "step": 292700
    },
    {
      "activations/layer0_attention_weight_max": 16.28142738342285,
      "activations/layer0_attention_weight_min": -13.483192443847656,
      "activations/layer10_attention_weight_max": 33.53189468383789,
      "activations/layer10_attention_weight_min": -34.34381103515625,
      "activations/layer11_attention_weight_max": 32.76595687866211,
      "activations/layer11_attention_weight_min": -32.08876037597656,
      "activations/layer12_attention_weight_max": 27.418079376220703,
      "activations/layer12_attention_weight_min": -25.947593688964844,
      "activations/layer13_attention_weight_max": 39.62832260131836,
      "activations/layer13_attention_weight_min": -36.77354049682617,
      "activations/layer14_attention_weight_max": 42.844383239746094,
      "activations/layer14_attention_weight_min": -36.98881149291992,
      "activations/layer15_attention_weight_max": 38.34724044799805,
      "activations/layer15_attention_weight_min": -35.58409118652344,
      "activations/layer16_attention_weight_max": 30.00237464904785,
      "activations/layer16_attention_weight_min": -26.482723236083984,
      "activations/layer17_attention_weight_max": 29.75355339050293,
      "activations/layer17_attention_weight_min": -24.797086715698242,
      "activations/layer18_attention_weight_max": 29.775352478027344,
      "activations/layer18_attention_weight_min": -24.932159423828125,
      "activations/layer19_attention_weight_max": 30.032835006713867,
      "activations/layer19_attention_weight_min": -28.940359115600586,
      "activations/layer1_attention_weight_max": 16.175874710083008,
      "activations/layer1_attention_weight_min": -15.844255447387695,
      "activations/layer20_attention_weight_max": 26.010021209716797,
      "activations/layer20_attention_weight_min": -25.385295867919922,
      "activations/layer21_attention_weight_max": 26.1276798248291,
      "activations/layer21_attention_weight_min": -23.93932342529297,
      "activations/layer22_attention_weight_max": 39.828861236572266,
      "activations/layer22_attention_weight_min": -32.55257797241211,
      "activations/layer23_attention_weight_max": 29.234416961669922,
      "activations/layer23_attention_weight_min": -26.337940216064453,
      "activations/layer2_attention_weight_max": 33.217384338378906,
      "activations/layer2_attention_weight_min": -31.94265365600586,
      "activations/layer3_attention_weight_max": 97.65912628173828,
      "activations/layer3_attention_weight_min": -93.93601989746094,
      "activations/layer4_attention_weight_max": 107.79869842529297,
      "activations/layer4_attention_weight_min": -107.58394622802734,
      "activations/layer5_attention_weight_max": 51.081199645996094,
      "activations/layer5_attention_weight_min": -63.94688034057617,
      "activations/layer6_attention_weight_max": 43.99677276611328,
      "activations/layer6_attention_weight_min": -46.05418014526367,
      "activations/layer7_attention_weight_max": 96.04813385009766,
      "activations/layer7_attention_weight_min": -92.83948516845703,
      "activations/layer8_attention_weight_max": 43.20444107055664,
      "activations/layer8_attention_weight_min": -45.36404800415039,
      "activations/layer9_attention_weight_max": 32.8635368347168,
      "activations/layer9_attention_weight_min": -35.25701904296875,
      "epoch": 17.01,
      "learning_rate": 4.0713636363636355e-05,
      "loss": 2.736,
      "step": 292750
    },
    {
      "activations/layer0_attention_weight_max": 16.516498565673828,
      "activations/layer0_attention_weight_min": -13.690834999084473,
      "activations/layer10_attention_weight_max": 32.928951263427734,
      "activations/layer10_attention_weight_min": -32.48999786376953,
      "activations/layer11_attention_weight_max": 33.82562255859375,
      "activations/layer11_attention_weight_min": -32.107059478759766,
      "activations/layer12_attention_weight_max": 26.916370391845703,
      "activations/layer12_attention_weight_min": -26.189882278442383,
      "activations/layer13_attention_weight_max": 40.415931701660156,
      "activations/layer13_attention_weight_min": -35.65584945678711,
      "activations/layer14_attention_weight_max": 44.03730392456055,
      "activations/layer14_attention_weight_min": -39.909271240234375,
      "activations/layer15_attention_weight_max": 40.469627380371094,
      "activations/layer15_attention_weight_min": -35.62077713012695,
      "activations/layer16_attention_weight_max": 31.65431785583496,
      "activations/layer16_attention_weight_min": -28.080827713012695,
      "activations/layer17_attention_weight_max": 32.05374526977539,
      "activations/layer17_attention_weight_min": -26.696395874023438,
      "activations/layer18_attention_weight_max": 33.6716423034668,
      "activations/layer18_attention_weight_min": -24.381505966186523,
      "activations/layer19_attention_weight_max": 34.760658264160156,
      "activations/layer19_attention_weight_min": -31.033023834228516,
      "activations/layer1_attention_weight_max": 16.596527099609375,
      "activations/layer1_attention_weight_min": -14.631609916687012,
      "activations/layer20_attention_weight_max": 31.255048751831055,
      "activations/layer20_attention_weight_min": -24.810617446899414,
      "activations/layer21_attention_weight_max": 28.660837173461914,
      "activations/layer21_attention_weight_min": -24.003177642822266,
      "activations/layer22_attention_weight_max": 40.836673736572266,
      "activations/layer22_attention_weight_min": -31.344684600830078,
      "activations/layer23_attention_weight_max": 34.578895568847656,
      "activations/layer23_attention_weight_min": -25.421857833862305,
      "activations/layer2_attention_weight_max": 33.840572357177734,
      "activations/layer2_attention_weight_min": -32.38178253173828,
      "activations/layer3_attention_weight_max": 96.9906997680664,
      "activations/layer3_attention_weight_min": -94.54081726074219,
      "activations/layer4_attention_weight_max": 104.04254150390625,
      "activations/layer4_attention_weight_min": -105.72916412353516,
      "activations/layer5_attention_weight_max": 51.54521179199219,
      "activations/layer5_attention_weight_min": -61.09453582763672,
      "activations/layer6_attention_weight_max": 46.174503326416016,
      "activations/layer6_attention_weight_min": -46.25135040283203,
      "activations/layer7_attention_weight_max": 93.95639038085938,
      "activations/layer7_attention_weight_min": -94.92696380615234,
      "activations/layer8_attention_weight_max": 41.389808654785156,
      "activations/layer8_attention_weight_min": -47.62581253051758,
      "activations/layer9_attention_weight_max": 30.756174087524414,
      "activations/layer9_attention_weight_min": -34.86031723022461,
      "epoch": 17.01,
      "learning_rate": 4.069469696969697e-05,
      "loss": 2.7288,
      "step": 292800
    },
    {
      "activations/layer0_attention_weight_max": 15.654814720153809,
      "activations/layer0_attention_weight_min": -12.255494117736816,
      "activations/layer10_attention_weight_max": 35.656436920166016,
      "activations/layer10_attention_weight_min": -32.958919525146484,
      "activations/layer11_attention_weight_max": 34.05850601196289,
      "activations/layer11_attention_weight_min": -33.645729064941406,
      "activations/layer12_attention_weight_max": 25.449560165405273,
      "activations/layer12_attention_weight_min": -25.764209747314453,
      "activations/layer13_attention_weight_max": 38.94878005981445,
      "activations/layer13_attention_weight_min": -33.94835662841797,
      "activations/layer14_attention_weight_max": 43.182945251464844,
      "activations/layer14_attention_weight_min": -36.108673095703125,
      "activations/layer15_attention_weight_max": 44.48569869995117,
      "activations/layer15_attention_weight_min": -33.15403747558594,
      "activations/layer16_attention_weight_max": 31.056758880615234,
      "activations/layer16_attention_weight_min": -27.77022933959961,
      "activations/layer17_attention_weight_max": 29.50936508178711,
      "activations/layer17_attention_weight_min": -26.31851577758789,
      "activations/layer18_attention_weight_max": 31.413944244384766,
      "activations/layer18_attention_weight_min": -24.232816696166992,
      "activations/layer19_attention_weight_max": 35.8277702331543,
      "activations/layer19_attention_weight_min": -33.10836410522461,
      "activations/layer1_attention_weight_max": 17.14137077331543,
      "activations/layer1_attention_weight_min": -15.953103065490723,
      "activations/layer20_attention_weight_max": 30.049930572509766,
      "activations/layer20_attention_weight_min": -25.137609481811523,
      "activations/layer21_attention_weight_max": 28.896310806274414,
      "activations/layer21_attention_weight_min": -24.879165649414062,
      "activations/layer22_attention_weight_max": 40.861873626708984,
      "activations/layer22_attention_weight_min": -31.259185791015625,
      "activations/layer23_attention_weight_max": 34.47509002685547,
      "activations/layer23_attention_weight_min": -25.62495994567871,
      "activations/layer2_attention_weight_max": 32.896942138671875,
      "activations/layer2_attention_weight_min": -33.2421760559082,
      "activations/layer3_attention_weight_max": 99.00312042236328,
      "activations/layer3_attention_weight_min": -97.63117218017578,
      "activations/layer4_attention_weight_max": 108.02274322509766,
      "activations/layer4_attention_weight_min": -106.81041717529297,
      "activations/layer5_attention_weight_max": 49.31804656982422,
      "activations/layer5_attention_weight_min": -60.836055755615234,
      "activations/layer6_attention_weight_max": 44.39509201049805,
      "activations/layer6_attention_weight_min": -46.47077178955078,
      "activations/layer7_attention_weight_max": 93.9203872680664,
      "activations/layer7_attention_weight_min": -102.44110870361328,
      "activations/layer8_attention_weight_max": 44.10466384887695,
      "activations/layer8_attention_weight_min": -49.3828010559082,
      "activations/layer9_attention_weight_max": 35.11124038696289,
      "activations/layer9_attention_weight_min": -33.97697067260742,
      "epoch": 17.02,
      "learning_rate": 4.067575757575757e-05,
      "loss": 2.7171,
      "step": 292850
    },
    {
      "activations/layer0_attention_weight_max": 16.14552116394043,
      "activations/layer0_attention_weight_min": -13.261248588562012,
      "activations/layer10_attention_weight_max": 40.45783233642578,
      "activations/layer10_attention_weight_min": -38.49812698364258,
      "activations/layer11_attention_weight_max": 39.862274169921875,
      "activations/layer11_attention_weight_min": -40.881744384765625,
      "activations/layer12_attention_weight_max": 24.74042320251465,
      "activations/layer12_attention_weight_min": -27.51707649230957,
      "activations/layer13_attention_weight_max": 38.10657501220703,
      "activations/layer13_attention_weight_min": -36.18976974487305,
      "activations/layer14_attention_weight_max": 40.57304000854492,
      "activations/layer14_attention_weight_min": -38.95380783081055,
      "activations/layer15_attention_weight_max": 39.632869720458984,
      "activations/layer15_attention_weight_min": -35.50813293457031,
      "activations/layer16_attention_weight_max": 32.52960968017578,
      "activations/layer16_attention_weight_min": -27.709503173828125,
      "activations/layer17_attention_weight_max": 29.432861328125,
      "activations/layer17_attention_weight_min": -26.701404571533203,
      "activations/layer18_attention_weight_max": 30.021751403808594,
      "activations/layer18_attention_weight_min": -26.540817260742188,
      "activations/layer19_attention_weight_max": 36.50849151611328,
      "activations/layer19_attention_weight_min": -31.121295928955078,
      "activations/layer1_attention_weight_max": 16.948509216308594,
      "activations/layer1_attention_weight_min": -15.321895599365234,
      "activations/layer20_attention_weight_max": 29.97918128967285,
      "activations/layer20_attention_weight_min": -23.743717193603516,
      "activations/layer21_attention_weight_max": 29.70641326904297,
      "activations/layer21_attention_weight_min": -23.82353973388672,
      "activations/layer22_attention_weight_max": 46.42544174194336,
      "activations/layer22_attention_weight_min": -33.78889083862305,
      "activations/layer23_attention_weight_max": 36.038368225097656,
      "activations/layer23_attention_weight_min": -26.487258911132812,
      "activations/layer2_attention_weight_max": 34.578269958496094,
      "activations/layer2_attention_weight_min": -36.208770751953125,
      "activations/layer3_attention_weight_max": 97.75973510742188,
      "activations/layer3_attention_weight_min": -105.62960052490234,
      "activations/layer4_attention_weight_max": 108.71920013427734,
      "activations/layer4_attention_weight_min": -107.4747543334961,
      "activations/layer5_attention_weight_max": 51.43967819213867,
      "activations/layer5_attention_weight_min": -60.8341064453125,
      "activations/layer6_attention_weight_max": 41.891387939453125,
      "activations/layer6_attention_weight_min": -43.191646575927734,
      "activations/layer7_attention_weight_max": 104.03690338134766,
      "activations/layer7_attention_weight_min": -100.09923553466797,
      "activations/layer8_attention_weight_max": 45.27613830566406,
      "activations/layer8_attention_weight_min": -48.4135627746582,
      "activations/layer9_attention_weight_max": 35.353050231933594,
      "activations/layer9_attention_weight_min": -38.677181243896484,
      "epoch": 17.02,
      "learning_rate": 4.065681818181818e-05,
      "loss": 2.734,
      "step": 292900
    },
    {
      "activations/layer0_attention_weight_max": 15.706239700317383,
      "activations/layer0_attention_weight_min": -13.403128623962402,
      "activations/layer10_attention_weight_max": 32.686988830566406,
      "activations/layer10_attention_weight_min": -33.94912338256836,
      "activations/layer11_attention_weight_max": 32.427852630615234,
      "activations/layer11_attention_weight_min": -34.134185791015625,
      "activations/layer12_attention_weight_max": 26.372140884399414,
      "activations/layer12_attention_weight_min": -26.82711410522461,
      "activations/layer13_attention_weight_max": 39.37158966064453,
      "activations/layer13_attention_weight_min": -37.15192794799805,
      "activations/layer14_attention_weight_max": 41.31840515136719,
      "activations/layer14_attention_weight_min": -42.75941467285156,
      "activations/layer15_attention_weight_max": 38.28654098510742,
      "activations/layer15_attention_weight_min": -39.111934661865234,
      "activations/layer16_attention_weight_max": 30.954057693481445,
      "activations/layer16_attention_weight_min": -26.908784866333008,
      "activations/layer17_attention_weight_max": 29.68706703186035,
      "activations/layer17_attention_weight_min": -25.86602210998535,
      "activations/layer18_attention_weight_max": 30.53367805480957,
      "activations/layer18_attention_weight_min": -25.251752853393555,
      "activations/layer19_attention_weight_max": 35.01375198364258,
      "activations/layer19_attention_weight_min": -32.70510482788086,
      "activations/layer1_attention_weight_max": 16.00884437561035,
      "activations/layer1_attention_weight_min": -16.200212478637695,
      "activations/layer20_attention_weight_max": 29.149126052856445,
      "activations/layer20_attention_weight_min": -26.400798797607422,
      "activations/layer21_attention_weight_max": 29.92808723449707,
      "activations/layer21_attention_weight_min": -25.765981674194336,
      "activations/layer22_attention_weight_max": 44.28015899658203,
      "activations/layer22_attention_weight_min": -33.1860466003418,
      "activations/layer23_attention_weight_max": 32.186561584472656,
      "activations/layer23_attention_weight_min": -29.2294921875,
      "activations/layer2_attention_weight_max": 33.0131950378418,
      "activations/layer2_attention_weight_min": -33.323448181152344,
      "activations/layer3_attention_weight_max": 98.64324188232422,
      "activations/layer3_attention_weight_min": -100.99626159667969,
      "activations/layer4_attention_weight_max": 109.12688446044922,
      "activations/layer4_attention_weight_min": -108.71752166748047,
      "activations/layer5_attention_weight_max": 51.281063079833984,
      "activations/layer5_attention_weight_min": -62.306671142578125,
      "activations/layer6_attention_weight_max": 41.590782165527344,
      "activations/layer6_attention_weight_min": -44.157127380371094,
      "activations/layer7_attention_weight_max": 94.89280700683594,
      "activations/layer7_attention_weight_min": -89.8011474609375,
      "activations/layer8_attention_weight_max": 39.3281135559082,
      "activations/layer8_attention_weight_min": -42.78182601928711,
      "activations/layer9_attention_weight_max": 30.494901657104492,
      "activations/layer9_attention_weight_min": -32.831146240234375,
      "epoch": 17.02,
      "learning_rate": 4.063787878787878e-05,
      "loss": 2.7342,
      "step": 292950
    },
    {
      "activations/layer0_attention_weight_max": 16.594703674316406,
      "activations/layer0_attention_weight_min": -12.20220947265625,
      "activations/layer10_attention_weight_max": 35.5511360168457,
      "activations/layer10_attention_weight_min": -37.280860900878906,
      "activations/layer11_attention_weight_max": 37.34054946899414,
      "activations/layer11_attention_weight_min": -35.589324951171875,
      "activations/layer12_attention_weight_max": 29.712308883666992,
      "activations/layer12_attention_weight_min": -29.72154426574707,
      "activations/layer13_attention_weight_max": 45.62308120727539,
      "activations/layer13_attention_weight_min": -36.816246032714844,
      "activations/layer14_attention_weight_max": 45.65502166748047,
      "activations/layer14_attention_weight_min": -37.97385787963867,
      "activations/layer15_attention_weight_max": 43.538875579833984,
      "activations/layer15_attention_weight_min": -36.86006546020508,
      "activations/layer16_attention_weight_max": 32.28327178955078,
      "activations/layer16_attention_weight_min": -28.097816467285156,
      "activations/layer17_attention_weight_max": 31.31477165222168,
      "activations/layer17_attention_weight_min": -25.541330337524414,
      "activations/layer18_attention_weight_max": 33.60251998901367,
      "activations/layer18_attention_weight_min": -27.03802490234375,
      "activations/layer19_attention_weight_max": 36.78334426879883,
      "activations/layer19_attention_weight_min": -32.42583465576172,
      "activations/layer1_attention_weight_max": 16.118024826049805,
      "activations/layer1_attention_weight_min": -15.273340225219727,
      "activations/layer20_attention_weight_max": 31.1534366607666,
      "activations/layer20_attention_weight_min": -25.405105590820312,
      "activations/layer21_attention_weight_max": 28.385826110839844,
      "activations/layer21_attention_weight_min": -25.623594284057617,
      "activations/layer22_attention_weight_max": 43.34848403930664,
      "activations/layer22_attention_weight_min": -32.19656753540039,
      "activations/layer23_attention_weight_max": 33.189430236816406,
      "activations/layer23_attention_weight_min": -28.163846969604492,
      "activations/layer2_attention_weight_max": 33.96282958984375,
      "activations/layer2_attention_weight_min": -34.02638244628906,
      "activations/layer3_attention_weight_max": 98.78690338134766,
      "activations/layer3_attention_weight_min": -100.5135726928711,
      "activations/layer4_attention_weight_max": 109.21675872802734,
      "activations/layer4_attention_weight_min": -118.33619689941406,
      "activations/layer5_attention_weight_max": 51.100372314453125,
      "activations/layer5_attention_weight_min": -65.94032287597656,
      "activations/layer6_attention_weight_max": 46.28669357299805,
      "activations/layer6_attention_weight_min": -47.44890594482422,
      "activations/layer7_attention_weight_max": 95.31709289550781,
      "activations/layer7_attention_weight_min": -99.18023681640625,
      "activations/layer8_attention_weight_max": 41.64802169799805,
      "activations/layer8_attention_weight_min": -45.7507209777832,
      "activations/layer9_attention_weight_max": 33.9077262878418,
      "activations/layer9_attention_weight_min": -37.59433364868164,
      "epoch": 17.02,
      "learning_rate": 4.061893939393939e-05,
      "loss": 2.7465,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.5382,
      "eval_samples_per_second": 502.916,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.5382,
      "eval_openwebtext_samples_per_second": 502.916,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0701,
      "eval_wikitext_samples_per_second": 220.28,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_lambada_loss": 2.40234375,
      "eval_lambada_ppl": 11.049042250008942,
      "eval_lambada_runtime": 9.5564,
      "eval_lambada_samples_per_second": 509.501,
      "step": 293000
    },
    {
      "activations/layer0_attention_weight_max": 15.305242538452148,
      "activations/layer0_attention_weight_min": -12.402998924255371,
      "activations/layer10_attention_weight_max": 34.35762405395508,
      "activations/layer10_attention_weight_min": -35.10075759887695,
      "activations/layer11_attention_weight_max": 32.73936080932617,
      "activations/layer11_attention_weight_min": -34.271202087402344,
      "activations/layer12_attention_weight_max": 27.295536041259766,
      "activations/layer12_attention_weight_min": -28.155176162719727,
      "activations/layer13_attention_weight_max": 39.62425994873047,
      "activations/layer13_attention_weight_min": -38.58742904663086,
      "activations/layer14_attention_weight_max": 41.750343322753906,
      "activations/layer14_attention_weight_min": -39.71086883544922,
      "activations/layer15_attention_weight_max": 37.881690979003906,
      "activations/layer15_attention_weight_min": -38.06825637817383,
      "activations/layer16_attention_weight_max": 31.46173858642578,
      "activations/layer16_attention_weight_min": -28.987945556640625,
      "activations/layer17_attention_weight_max": 29.900136947631836,
      "activations/layer17_attention_weight_min": -27.545352935791016,
      "activations/layer18_attention_weight_max": 30.6901798248291,
      "activations/layer18_attention_weight_min": -26.899105072021484,
      "activations/layer19_attention_weight_max": 35.41372299194336,
      "activations/layer19_attention_weight_min": -34.30416488647461,
      "activations/layer1_attention_weight_max": 16.94810676574707,
      "activations/layer1_attention_weight_min": -14.677502632141113,
      "activations/layer20_attention_weight_max": 31.782901763916016,
      "activations/layer20_attention_weight_min": -27.08946418762207,
      "activations/layer21_attention_weight_max": 28.83250617980957,
      "activations/layer21_attention_weight_min": -26.476665496826172,
      "activations/layer22_attention_weight_max": 43.6741943359375,
      "activations/layer22_attention_weight_min": -33.59968185424805,
      "activations/layer23_attention_weight_max": 32.75429153442383,
      "activations/layer23_attention_weight_min": -27.861225128173828,
      "activations/layer2_attention_weight_max": 34.14499282836914,
      "activations/layer2_attention_weight_min": -33.938941955566406,
      "activations/layer3_attention_weight_max": 98.68075561523438,
      "activations/layer3_attention_weight_min": -99.45736694335938,
      "activations/layer4_attention_weight_max": 108.03189849853516,
      "activations/layer4_attention_weight_min": -107.89647674560547,
      "activations/layer5_attention_weight_max": 51.271636962890625,
      "activations/layer5_attention_weight_min": -60.939971923828125,
      "activations/layer6_attention_weight_max": 44.41212844848633,
      "activations/layer6_attention_weight_min": -45.3359489440918,
      "activations/layer7_attention_weight_max": 96.56332397460938,
      "activations/layer7_attention_weight_min": -94.94020080566406,
      "activations/layer8_attention_weight_max": 43.580257415771484,
      "activations/layer8_attention_weight_min": -43.96352005004883,
      "activations/layer9_attention_weight_max": 33.909610748291016,
      "activations/layer9_attention_weight_min": -35.08362579345703,
      "epoch": 17.03,
      "learning_rate": 4.06e-05,
      "loss": 2.7257,
      "step": 293050
    },
    {
      "activations/layer0_attention_weight_max": 16.167057037353516,
      "activations/layer0_attention_weight_min": -12.163128852844238,
      "activations/layer10_attention_weight_max": 33.86341857910156,
      "activations/layer10_attention_weight_min": -35.7032470703125,
      "activations/layer11_attention_weight_max": 33.64070129394531,
      "activations/layer11_attention_weight_min": -35.48207092285156,
      "activations/layer12_attention_weight_max": 26.696430206298828,
      "activations/layer12_attention_weight_min": -25.437498092651367,
      "activations/layer13_attention_weight_max": 39.234954833984375,
      "activations/layer13_attention_weight_min": -34.522029876708984,
      "activations/layer14_attention_weight_max": 45.09928512573242,
      "activations/layer14_attention_weight_min": -36.536155700683594,
      "activations/layer15_attention_weight_max": 37.883331298828125,
      "activations/layer15_attention_weight_min": -34.3631706237793,
      "activations/layer16_attention_weight_max": 31.074724197387695,
      "activations/layer16_attention_weight_min": -29.06688117980957,
      "activations/layer17_attention_weight_max": 32.078121185302734,
      "activations/layer17_attention_weight_min": -26.47490882873535,
      "activations/layer18_attention_weight_max": 31.444557189941406,
      "activations/layer18_attention_weight_min": -26.689517974853516,
      "activations/layer19_attention_weight_max": 35.0693473815918,
      "activations/layer19_attention_weight_min": -30.978200912475586,
      "activations/layer1_attention_weight_max": 16.935718536376953,
      "activations/layer1_attention_weight_min": -14.742012977600098,
      "activations/layer20_attention_weight_max": 30.202064514160156,
      "activations/layer20_attention_weight_min": -23.795055389404297,
      "activations/layer21_attention_weight_max": 30.655332565307617,
      "activations/layer21_attention_weight_min": -24.214889526367188,
      "activations/layer22_attention_weight_max": 40.42051315307617,
      "activations/layer22_attention_weight_min": -32.379966735839844,
      "activations/layer23_attention_weight_max": 32.03589630126953,
      "activations/layer23_attention_weight_min": -24.644269943237305,
      "activations/layer2_attention_weight_max": 34.46131896972656,
      "activations/layer2_attention_weight_min": -36.6990852355957,
      "activations/layer3_attention_weight_max": 99.06470489501953,
      "activations/layer3_attention_weight_min": -104.16983795166016,
      "activations/layer4_attention_weight_max": 113.40693664550781,
      "activations/layer4_attention_weight_min": -110.33839416503906,
      "activations/layer5_attention_weight_max": 51.924129486083984,
      "activations/layer5_attention_weight_min": -64.58380126953125,
      "activations/layer6_attention_weight_max": 45.800621032714844,
      "activations/layer6_attention_weight_min": -48.881752014160156,
      "activations/layer7_attention_weight_max": 91.10844421386719,
      "activations/layer7_attention_weight_min": -96.76908111572266,
      "activations/layer8_attention_weight_max": 40.8316535949707,
      "activations/layer8_attention_weight_min": -44.46550750732422,
      "activations/layer9_attention_weight_max": 32.431636810302734,
      "activations/layer9_attention_weight_min": -36.319801330566406,
      "epoch": 17.03,
      "learning_rate": 4.05810606060606e-05,
      "loss": 2.738,
      "step": 293100
    },
    {
      "activations/layer0_attention_weight_max": 16.7681827545166,
      "activations/layer0_attention_weight_min": -13.780815124511719,
      "activations/layer10_attention_weight_max": 31.053298950195312,
      "activations/layer10_attention_weight_min": -35.780372619628906,
      "activations/layer11_attention_weight_max": 32.45140838623047,
      "activations/layer11_attention_weight_min": -36.90190887451172,
      "activations/layer12_attention_weight_max": 26.913644790649414,
      "activations/layer12_attention_weight_min": -27.3158016204834,
      "activations/layer13_attention_weight_max": 38.485015869140625,
      "activations/layer13_attention_weight_min": -36.22975158691406,
      "activations/layer14_attention_weight_max": 40.0060920715332,
      "activations/layer14_attention_weight_min": -37.28804397583008,
      "activations/layer15_attention_weight_max": 36.737884521484375,
      "activations/layer15_attention_weight_min": -34.893062591552734,
      "activations/layer16_attention_weight_max": 29.536197662353516,
      "activations/layer16_attention_weight_min": -26.847429275512695,
      "activations/layer17_attention_weight_max": 29.824344635009766,
      "activations/layer17_attention_weight_min": -26.298830032348633,
      "activations/layer18_attention_weight_max": 32.8038444519043,
      "activations/layer18_attention_weight_min": -24.948537826538086,
      "activations/layer19_attention_weight_max": 32.32772445678711,
      "activations/layer19_attention_weight_min": -31.20344352722168,
      "activations/layer1_attention_weight_max": 16.859445571899414,
      "activations/layer1_attention_weight_min": -15.650493621826172,
      "activations/layer20_attention_weight_max": 27.029205322265625,
      "activations/layer20_attention_weight_min": -24.63903045654297,
      "activations/layer21_attention_weight_max": 26.651382446289062,
      "activations/layer21_attention_weight_min": -24.03544044494629,
      "activations/layer22_attention_weight_max": 39.227054595947266,
      "activations/layer22_attention_weight_min": -31.58245849609375,
      "activations/layer23_attention_weight_max": 32.99004364013672,
      "activations/layer23_attention_weight_min": -28.048391342163086,
      "activations/layer2_attention_weight_max": 33.85758590698242,
      "activations/layer2_attention_weight_min": -35.11391067504883,
      "activations/layer3_attention_weight_max": 95.50331115722656,
      "activations/layer3_attention_weight_min": -97.8499755859375,
      "activations/layer4_attention_weight_max": 106.00325012207031,
      "activations/layer4_attention_weight_min": -111.83585357666016,
      "activations/layer5_attention_weight_max": 50.184425354003906,
      "activations/layer5_attention_weight_min": -65.60444641113281,
      "activations/layer6_attention_weight_max": 46.307437896728516,
      "activations/layer6_attention_weight_min": -48.80179214477539,
      "activations/layer7_attention_weight_max": 92.7688217163086,
      "activations/layer7_attention_weight_min": -97.25196838378906,
      "activations/layer8_attention_weight_max": 40.798912048339844,
      "activations/layer8_attention_weight_min": -44.44324493408203,
      "activations/layer9_attention_weight_max": 30.831689834594727,
      "activations/layer9_attention_weight_min": -37.37950134277344,
      "epoch": 17.03,
      "learning_rate": 4.0562121212121214e-05,
      "loss": 2.733,
      "step": 293150
    },
    {
      "activations/layer0_attention_weight_max": 16.105409622192383,
      "activations/layer0_attention_weight_min": -12.42844295501709,
      "activations/layer10_attention_weight_max": 30.291030883789062,
      "activations/layer10_attention_weight_min": -32.598388671875,
      "activations/layer11_attention_weight_max": 31.788524627685547,
      "activations/layer11_attention_weight_min": -31.95343780517578,
      "activations/layer12_attention_weight_max": 27.88675308227539,
      "activations/layer12_attention_weight_min": -26.595041275024414,
      "activations/layer13_attention_weight_max": 38.90678024291992,
      "activations/layer13_attention_weight_min": -36.17924880981445,
      "activations/layer14_attention_weight_max": 44.93727493286133,
      "activations/layer14_attention_weight_min": -38.74392318725586,
      "activations/layer15_attention_weight_max": 40.533382415771484,
      "activations/layer15_attention_weight_min": -34.406005859375,
      "activations/layer16_attention_weight_max": 29.582134246826172,
      "activations/layer16_attention_weight_min": -26.16251564025879,
      "activations/layer17_attention_weight_max": 29.20842933654785,
      "activations/layer17_attention_weight_min": -24.612329483032227,
      "activations/layer18_attention_weight_max": 30.68109130859375,
      "activations/layer18_attention_weight_min": -24.454303741455078,
      "activations/layer19_attention_weight_max": 34.0389404296875,
      "activations/layer19_attention_weight_min": -29.78931999206543,
      "activations/layer1_attention_weight_max": 16.48764419555664,
      "activations/layer1_attention_weight_min": -15.018674850463867,
      "activations/layer20_attention_weight_max": 29.956087112426758,
      "activations/layer20_attention_weight_min": -24.919116973876953,
      "activations/layer21_attention_weight_max": 29.561946868896484,
      "activations/layer21_attention_weight_min": -23.722564697265625,
      "activations/layer22_attention_weight_max": 42.38227462768555,
      "activations/layer22_attention_weight_min": -31.720714569091797,
      "activations/layer23_attention_weight_max": 35.08326721191406,
      "activations/layer23_attention_weight_min": -24.617483139038086,
      "activations/layer2_attention_weight_max": 35.335296630859375,
      "activations/layer2_attention_weight_min": -32.443199157714844,
      "activations/layer3_attention_weight_max": 94.8265380859375,
      "activations/layer3_attention_weight_min": -96.40460205078125,
      "activations/layer4_attention_weight_max": 101.02172088623047,
      "activations/layer4_attention_weight_min": -106.1378402709961,
      "activations/layer5_attention_weight_max": 48.90077209472656,
      "activations/layer5_attention_weight_min": -60.16815948486328,
      "activations/layer6_attention_weight_max": 42.767906188964844,
      "activations/layer6_attention_weight_min": -47.52470397949219,
      "activations/layer7_attention_weight_max": 88.70836639404297,
      "activations/layer7_attention_weight_min": -101.64451599121094,
      "activations/layer8_attention_weight_max": 39.78437042236328,
      "activations/layer8_attention_weight_min": -41.65973663330078,
      "activations/layer9_attention_weight_max": 30.77461051940918,
      "activations/layer9_attention_weight_min": -31.771530151367188,
      "epoch": 17.04,
      "learning_rate": 4.0543181818181816e-05,
      "loss": 2.7249,
      "step": 293200
    },
    {
      "activations/layer0_attention_weight_max": 16.53373146057129,
      "activations/layer0_attention_weight_min": -12.853315353393555,
      "activations/layer10_attention_weight_max": 33.36338424682617,
      "activations/layer10_attention_weight_min": -35.008888244628906,
      "activations/layer11_attention_weight_max": 32.415260314941406,
      "activations/layer11_attention_weight_min": -35.12327194213867,
      "activations/layer12_attention_weight_max": 24.266334533691406,
      "activations/layer12_attention_weight_min": -27.291959762573242,
      "activations/layer13_attention_weight_max": 37.26116943359375,
      "activations/layer13_attention_weight_min": -36.115638732910156,
      "activations/layer14_attention_weight_max": 38.77417755126953,
      "activations/layer14_attention_weight_min": -36.473297119140625,
      "activations/layer15_attention_weight_max": 37.02383041381836,
      "activations/layer15_attention_weight_min": -33.57585144042969,
      "activations/layer16_attention_weight_max": 28.64388656616211,
      "activations/layer16_attention_weight_min": -28.593769073486328,
      "activations/layer17_attention_weight_max": 30.04402732849121,
      "activations/layer17_attention_weight_min": -27.368249893188477,
      "activations/layer18_attention_weight_max": 31.002864837646484,
      "activations/layer18_attention_weight_min": -25.598493576049805,
      "activations/layer19_attention_weight_max": 30.61404800415039,
      "activations/layer19_attention_weight_min": -33.35987091064453,
      "activations/layer1_attention_weight_max": 16.35738754272461,
      "activations/layer1_attention_weight_min": -15.471247673034668,
      "activations/layer20_attention_weight_max": 26.520374298095703,
      "activations/layer20_attention_weight_min": -26.550559997558594,
      "activations/layer21_attention_weight_max": 27.509267807006836,
      "activations/layer21_attention_weight_min": -25.159618377685547,
      "activations/layer22_attention_weight_max": 39.2429084777832,
      "activations/layer22_attention_weight_min": -31.568023681640625,
      "activations/layer23_attention_weight_max": 31.838476181030273,
      "activations/layer23_attention_weight_min": -26.960227966308594,
      "activations/layer2_attention_weight_max": 31.81609344482422,
      "activations/layer2_attention_weight_min": -32.46961975097656,
      "activations/layer3_attention_weight_max": 98.58720397949219,
      "activations/layer3_attention_weight_min": -97.82001495361328,
      "activations/layer4_attention_weight_max": 105.55449676513672,
      "activations/layer4_attention_weight_min": -105.73492431640625,
      "activations/layer5_attention_weight_max": 49.14315414428711,
      "activations/layer5_attention_weight_min": -60.32000732421875,
      "activations/layer6_attention_weight_max": 43.82175064086914,
      "activations/layer6_attention_weight_min": -46.73764419555664,
      "activations/layer7_attention_weight_max": 91.3342056274414,
      "activations/layer7_attention_weight_min": -95.77413940429688,
      "activations/layer8_attention_weight_max": 39.68547821044922,
      "activations/layer8_attention_weight_min": -43.8797492980957,
      "activations/layer9_attention_weight_max": 30.42508888244629,
      "activations/layer9_attention_weight_min": -35.49118423461914,
      "epoch": 17.04,
      "learning_rate": 4.052424242424242e-05,
      "loss": 2.722,
      "step": 293250
    },
    {
      "activations/layer0_attention_weight_max": 15.765288352966309,
      "activations/layer0_attention_weight_min": -13.237210273742676,
      "activations/layer10_attention_weight_max": 33.09290313720703,
      "activations/layer10_attention_weight_min": -33.56214141845703,
      "activations/layer11_attention_weight_max": 33.08160400390625,
      "activations/layer11_attention_weight_min": -33.57551574707031,
      "activations/layer12_attention_weight_max": 28.34986686706543,
      "activations/layer12_attention_weight_min": -27.175113677978516,
      "activations/layer13_attention_weight_max": 42.31822204589844,
      "activations/layer13_attention_weight_min": -36.45024871826172,
      "activations/layer14_attention_weight_max": 50.1040153503418,
      "activations/layer14_attention_weight_min": -41.228050231933594,
      "activations/layer15_attention_weight_max": 41.25680923461914,
      "activations/layer15_attention_weight_min": -36.021610260009766,
      "activations/layer16_attention_weight_max": 30.517412185668945,
      "activations/layer16_attention_weight_min": -31.096240997314453,
      "activations/layer17_attention_weight_max": 30.699562072753906,
      "activations/layer17_attention_weight_min": -27.829025268554688,
      "activations/layer18_attention_weight_max": 33.846073150634766,
      "activations/layer18_attention_weight_min": -26.1203556060791,
      "activations/layer19_attention_weight_max": 35.28280258178711,
      "activations/layer19_attention_weight_min": -30.723356246948242,
      "activations/layer1_attention_weight_max": 16.153461456298828,
      "activations/layer1_attention_weight_min": -15.648149490356445,
      "activations/layer20_attention_weight_max": 30.264564514160156,
      "activations/layer20_attention_weight_min": -24.7482967376709,
      "activations/layer21_attention_weight_max": 31.48331642150879,
      "activations/layer21_attention_weight_min": -26.326534271240234,
      "activations/layer22_attention_weight_max": 43.33711242675781,
      "activations/layer22_attention_weight_min": -31.66351318359375,
      "activations/layer23_attention_weight_max": 35.145389556884766,
      "activations/layer23_attention_weight_min": -24.865917205810547,
      "activations/layer2_attention_weight_max": 33.422489166259766,
      "activations/layer2_attention_weight_min": -35.65901184082031,
      "activations/layer3_attention_weight_max": 101.18814086914062,
      "activations/layer3_attention_weight_min": -99.74542999267578,
      "activations/layer4_attention_weight_max": 105.47342681884766,
      "activations/layer4_attention_weight_min": -108.78816223144531,
      "activations/layer5_attention_weight_max": 53.295562744140625,
      "activations/layer5_attention_weight_min": -61.354820251464844,
      "activations/layer6_attention_weight_max": 44.40251541137695,
      "activations/layer6_attention_weight_min": -45.8172492980957,
      "activations/layer7_attention_weight_max": 94.4265365600586,
      "activations/layer7_attention_weight_min": -97.75537109375,
      "activations/layer8_attention_weight_max": 43.029537200927734,
      "activations/layer8_attention_weight_min": -43.04188919067383,
      "activations/layer9_attention_weight_max": 32.62499237060547,
      "activations/layer9_attention_weight_min": -33.83498001098633,
      "epoch": 17.04,
      "learning_rate": 4.0505303030303025e-05,
      "loss": 2.7265,
      "step": 293300
    },
    {
      "activations/layer0_attention_weight_max": 16.31196403503418,
      "activations/layer0_attention_weight_min": -12.303335189819336,
      "activations/layer10_attention_weight_max": 41.477317810058594,
      "activations/layer10_attention_weight_min": -38.188270568847656,
      "activations/layer11_attention_weight_max": 44.12556076049805,
      "activations/layer11_attention_weight_min": -40.62071990966797,
      "activations/layer12_attention_weight_max": 29.723892211914062,
      "activations/layer12_attention_weight_min": -28.818851470947266,
      "activations/layer13_attention_weight_max": 40.045501708984375,
      "activations/layer13_attention_weight_min": -37.29118347167969,
      "activations/layer14_attention_weight_max": 42.71839141845703,
      "activations/layer14_attention_weight_min": -42.26686477661133,
      "activations/layer15_attention_weight_max": 43.753143310546875,
      "activations/layer15_attention_weight_min": -36.92094039916992,
      "activations/layer16_attention_weight_max": 31.97687530517578,
      "activations/layer16_attention_weight_min": -28.18334197998047,
      "activations/layer17_attention_weight_max": 31.41155433654785,
      "activations/layer17_attention_weight_min": -28.37343406677246,
      "activations/layer18_attention_weight_max": 29.19642448425293,
      "activations/layer18_attention_weight_min": -26.823047637939453,
      "activations/layer19_attention_weight_max": 36.07477951049805,
      "activations/layer19_attention_weight_min": -32.2716178894043,
      "activations/layer1_attention_weight_max": 16.42787742614746,
      "activations/layer1_attention_weight_min": -15.299253463745117,
      "activations/layer20_attention_weight_max": 29.63323211669922,
      "activations/layer20_attention_weight_min": -24.273418426513672,
      "activations/layer21_attention_weight_max": 30.064041137695312,
      "activations/layer21_attention_weight_min": -25.226835250854492,
      "activations/layer22_attention_weight_max": 39.51145935058594,
      "activations/layer22_attention_weight_min": -33.87391662597656,
      "activations/layer23_attention_weight_max": 34.661949157714844,
      "activations/layer23_attention_weight_min": -26.301008224487305,
      "activations/layer2_attention_weight_max": 35.556793212890625,
      "activations/layer2_attention_weight_min": -34.39729309082031,
      "activations/layer3_attention_weight_max": 98.46762084960938,
      "activations/layer3_attention_weight_min": -100.67162322998047,
      "activations/layer4_attention_weight_max": 106.31221008300781,
      "activations/layer4_attention_weight_min": -110.8780517578125,
      "activations/layer5_attention_weight_max": 50.511817932128906,
      "activations/layer5_attention_weight_min": -60.33312225341797,
      "activations/layer6_attention_weight_max": 43.88686752319336,
      "activations/layer6_attention_weight_min": -44.83491897583008,
      "activations/layer7_attention_weight_max": 96.0428695678711,
      "activations/layer7_attention_weight_min": -96.23905181884766,
      "activations/layer8_attention_weight_max": 46.91561508178711,
      "activations/layer8_attention_weight_min": -43.8363151550293,
      "activations/layer9_attention_weight_max": 36.825050354003906,
      "activations/layer9_attention_weight_min": -36.57725143432617,
      "epoch": 17.05,
      "learning_rate": 4.0486363636363634e-05,
      "loss": 2.7229,
      "step": 293350
    },
    {
      "activations/layer0_attention_weight_max": 16.402952194213867,
      "activations/layer0_attention_weight_min": -13.895942687988281,
      "activations/layer10_attention_weight_max": 31.72865867614746,
      "activations/layer10_attention_weight_min": -33.05944061279297,
      "activations/layer11_attention_weight_max": 32.19560241699219,
      "activations/layer11_attention_weight_min": -31.46633529663086,
      "activations/layer12_attention_weight_max": 24.445499420166016,
      "activations/layer12_attention_weight_min": -26.432104110717773,
      "activations/layer13_attention_weight_max": 34.79923629760742,
      "activations/layer13_attention_weight_min": -34.182701110839844,
      "activations/layer14_attention_weight_max": 38.23587417602539,
      "activations/layer14_attention_weight_min": -38.1627311706543,
      "activations/layer15_attention_weight_max": 35.05624008178711,
      "activations/layer15_attention_weight_min": -37.28456115722656,
      "activations/layer16_attention_weight_max": 26.600542068481445,
      "activations/layer16_attention_weight_min": -28.758808135986328,
      "activations/layer17_attention_weight_max": 29.54666519165039,
      "activations/layer17_attention_weight_min": -26.01418685913086,
      "activations/layer18_attention_weight_max": 31.826889038085938,
      "activations/layer18_attention_weight_min": -27.011756896972656,
      "activations/layer19_attention_weight_max": 33.773929595947266,
      "activations/layer19_attention_weight_min": -33.46458435058594,
      "activations/layer1_attention_weight_max": 16.875089645385742,
      "activations/layer1_attention_weight_min": -15.628266334533691,
      "activations/layer20_attention_weight_max": 27.311887741088867,
      "activations/layer20_attention_weight_min": -28.7006893157959,
      "activations/layer21_attention_weight_max": 27.771255493164062,
      "activations/layer21_attention_weight_min": -26.197729110717773,
      "activations/layer22_attention_weight_max": 41.17234420776367,
      "activations/layer22_attention_weight_min": -32.51091384887695,
      "activations/layer23_attention_weight_max": 34.54350280761719,
      "activations/layer23_attention_weight_min": -26.900352478027344,
      "activations/layer2_attention_weight_max": 33.95024108886719,
      "activations/layer2_attention_weight_min": -34.35053253173828,
      "activations/layer3_attention_weight_max": 96.09698486328125,
      "activations/layer3_attention_weight_min": -99.711181640625,
      "activations/layer4_attention_weight_max": 105.0333251953125,
      "activations/layer4_attention_weight_min": -108.48919677734375,
      "activations/layer5_attention_weight_max": 50.653926849365234,
      "activations/layer5_attention_weight_min": -61.283443450927734,
      "activations/layer6_attention_weight_max": 44.39680862426758,
      "activations/layer6_attention_weight_min": -47.07634353637695,
      "activations/layer7_attention_weight_max": 91.76021575927734,
      "activations/layer7_attention_weight_min": -88.42669677734375,
      "activations/layer8_attention_weight_max": 39.47970962524414,
      "activations/layer8_attention_weight_min": -41.57341384887695,
      "activations/layer9_attention_weight_max": 31.766040802001953,
      "activations/layer9_attention_weight_min": -32.536373138427734,
      "epoch": 17.05,
      "learning_rate": 4.046742424242424e-05,
      "loss": 2.7342,
      "step": 293400
    },
    {
      "activations/layer0_attention_weight_max": 15.933669090270996,
      "activations/layer0_attention_weight_min": -13.557010650634766,
      "activations/layer10_attention_weight_max": 32.242469787597656,
      "activations/layer10_attention_weight_min": -34.57111740112305,
      "activations/layer11_attention_weight_max": 33.4896354675293,
      "activations/layer11_attention_weight_min": -33.645782470703125,
      "activations/layer12_attention_weight_max": 26.487897872924805,
      "activations/layer12_attention_weight_min": -27.4683895111084,
      "activations/layer13_attention_weight_max": 38.61445236206055,
      "activations/layer13_attention_weight_min": -36.489681243896484,
      "activations/layer14_attention_weight_max": 41.76681137084961,
      "activations/layer14_attention_weight_min": -39.95287322998047,
      "activations/layer15_attention_weight_max": 39.781246185302734,
      "activations/layer15_attention_weight_min": -36.49356460571289,
      "activations/layer16_attention_weight_max": 29.062320709228516,
      "activations/layer16_attention_weight_min": -28.19315528869629,
      "activations/layer17_attention_weight_max": 30.94285011291504,
      "activations/layer17_attention_weight_min": -27.393653869628906,
      "activations/layer18_attention_weight_max": 33.310821533203125,
      "activations/layer18_attention_weight_min": -26.59722900390625,
      "activations/layer19_attention_weight_max": 36.556060791015625,
      "activations/layer19_attention_weight_min": -32.9818115234375,
      "activations/layer1_attention_weight_max": 15.9791259765625,
      "activations/layer1_attention_weight_min": -15.14728832244873,
      "activations/layer20_attention_weight_max": 29.904876708984375,
      "activations/layer20_attention_weight_min": -26.820850372314453,
      "activations/layer21_attention_weight_max": 31.195751190185547,
      "activations/layer21_attention_weight_min": -26.161569595336914,
      "activations/layer22_attention_weight_max": 43.873268127441406,
      "activations/layer22_attention_weight_min": -34.65275192260742,
      "activations/layer23_attention_weight_max": 33.14094543457031,
      "activations/layer23_attention_weight_min": -27.978313446044922,
      "activations/layer2_attention_weight_max": 33.974693298339844,
      "activations/layer2_attention_weight_min": -35.095272064208984,
      "activations/layer3_attention_weight_max": 102.39124298095703,
      "activations/layer3_attention_weight_min": -102.70128631591797,
      "activations/layer4_attention_weight_max": 110.7904281616211,
      "activations/layer4_attention_weight_min": -111.52386474609375,
      "activations/layer5_attention_weight_max": 51.70740509033203,
      "activations/layer5_attention_weight_min": -65.93588256835938,
      "activations/layer6_attention_weight_max": 43.97368621826172,
      "activations/layer6_attention_weight_min": -46.93813705444336,
      "activations/layer7_attention_weight_max": 90.7494888305664,
      "activations/layer7_attention_weight_min": -98.8790283203125,
      "activations/layer8_attention_weight_max": 40.22770690917969,
      "activations/layer8_attention_weight_min": -42.630523681640625,
      "activations/layer9_attention_weight_max": 31.276905059814453,
      "activations/layer9_attention_weight_min": -34.544246673583984,
      "epoch": 17.05,
      "learning_rate": 4.044848484848484e-05,
      "loss": 2.7454,
      "step": 293450
    },
    {
      "activations/layer0_attention_weight_max": 16.674524307250977,
      "activations/layer0_attention_weight_min": -13.107206344604492,
      "activations/layer10_attention_weight_max": 35.19612121582031,
      "activations/layer10_attention_weight_min": -34.410125732421875,
      "activations/layer11_attention_weight_max": 37.602325439453125,
      "activations/layer11_attention_weight_min": -37.16349792480469,
      "activations/layer12_attention_weight_max": 28.791086196899414,
      "activations/layer12_attention_weight_min": -25.328311920166016,
      "activations/layer13_attention_weight_max": 41.66755676269531,
      "activations/layer13_attention_weight_min": -35.55416488647461,
      "activations/layer14_attention_weight_max": 43.11673355102539,
      "activations/layer14_attention_weight_min": -37.505592346191406,
      "activations/layer15_attention_weight_max": 42.822166442871094,
      "activations/layer15_attention_weight_min": -34.90943145751953,
      "activations/layer16_attention_weight_max": 30.062185287475586,
      "activations/layer16_attention_weight_min": -29.220922470092773,
      "activations/layer17_attention_weight_max": 32.8494758605957,
      "activations/layer17_attention_weight_min": -28.161714553833008,
      "activations/layer18_attention_weight_max": 31.580425262451172,
      "activations/layer18_attention_weight_min": -25.219459533691406,
      "activations/layer19_attention_weight_max": 35.83088684082031,
      "activations/layer19_attention_weight_min": -30.841552734375,
      "activations/layer1_attention_weight_max": 16.002696990966797,
      "activations/layer1_attention_weight_min": -16.00267219543457,
      "activations/layer20_attention_weight_max": 32.293941497802734,
      "activations/layer20_attention_weight_min": -26.427518844604492,
      "activations/layer21_attention_weight_max": 32.1075325012207,
      "activations/layer21_attention_weight_min": -26.724180221557617,
      "activations/layer22_attention_weight_max": 42.222023010253906,
      "activations/layer22_attention_weight_min": -31.560123443603516,
      "activations/layer23_attention_weight_max": 37.016239166259766,
      "activations/layer23_attention_weight_min": -26.24184799194336,
      "activations/layer2_attention_weight_max": 34.97875213623047,
      "activations/layer2_attention_weight_min": -35.401405334472656,
      "activations/layer3_attention_weight_max": 105.23030090332031,
      "activations/layer3_attention_weight_min": -102.76951599121094,
      "activations/layer4_attention_weight_max": 104.5793228149414,
      "activations/layer4_attention_weight_min": -108.55413055419922,
      "activations/layer5_attention_weight_max": 51.59673309326172,
      "activations/layer5_attention_weight_min": -60.165611267089844,
      "activations/layer6_attention_weight_max": 46.68773651123047,
      "activations/layer6_attention_weight_min": -47.70466995239258,
      "activations/layer7_attention_weight_max": 91.93921661376953,
      "activations/layer7_attention_weight_min": -91.63154602050781,
      "activations/layer8_attention_weight_max": 43.72747039794922,
      "activations/layer8_attention_weight_min": -44.59981155395508,
      "activations/layer9_attention_weight_max": 34.77006149291992,
      "activations/layer9_attention_weight_min": -35.14115524291992,
      "epoch": 17.05,
      "learning_rate": 4.0429545454545445e-05,
      "loss": 2.7399,
      "step": 293500
    },
    {
      "activations/layer0_attention_weight_max": 16.618972778320312,
      "activations/layer0_attention_weight_min": -13.452516555786133,
      "activations/layer10_attention_weight_max": 36.41593551635742,
      "activations/layer10_attention_weight_min": -35.509742736816406,
      "activations/layer11_attention_weight_max": 36.205326080322266,
      "activations/layer11_attention_weight_min": -33.26193618774414,
      "activations/layer12_attention_weight_max": 25.088151931762695,
      "activations/layer12_attention_weight_min": -33.19863510131836,
      "activations/layer13_attention_weight_max": 38.01444625854492,
      "activations/layer13_attention_weight_min": -35.27273178100586,
      "activations/layer14_attention_weight_max": 41.41316223144531,
      "activations/layer14_attention_weight_min": -37.830238342285156,
      "activations/layer15_attention_weight_max": 38.0940055847168,
      "activations/layer15_attention_weight_min": -34.24971008300781,
      "activations/layer16_attention_weight_max": 28.566884994506836,
      "activations/layer16_attention_weight_min": -28.010149002075195,
      "activations/layer17_attention_weight_max": 30.88557243347168,
      "activations/layer17_attention_weight_min": -25.38560676574707,
      "activations/layer18_attention_weight_max": 30.924495697021484,
      "activations/layer18_attention_weight_min": -24.39478874206543,
      "activations/layer19_attention_weight_max": 32.324058532714844,
      "activations/layer19_attention_weight_min": -29.96795082092285,
      "activations/layer1_attention_weight_max": 16.36612892150879,
      "activations/layer1_attention_weight_min": -14.55640983581543,
      "activations/layer20_attention_weight_max": 27.095312118530273,
      "activations/layer20_attention_weight_min": -23.948535919189453,
      "activations/layer21_attention_weight_max": 29.101736068725586,
      "activations/layer21_attention_weight_min": -24.8109073638916,
      "activations/layer22_attention_weight_max": 36.59819412231445,
      "activations/layer22_attention_weight_min": -29.087238311767578,
      "activations/layer23_attention_weight_max": 31.806255340576172,
      "activations/layer23_attention_weight_min": -24.511899948120117,
      "activations/layer2_attention_weight_max": 33.548370361328125,
      "activations/layer2_attention_weight_min": -34.79833984375,
      "activations/layer3_attention_weight_max": 101.548583984375,
      "activations/layer3_attention_weight_min": -101.26659393310547,
      "activations/layer4_attention_weight_max": 120.72601318359375,
      "activations/layer4_attention_weight_min": -116.86067962646484,
      "activations/layer5_attention_weight_max": 50.136962890625,
      "activations/layer5_attention_weight_min": -64.16644287109375,
      "activations/layer6_attention_weight_max": 43.10071563720703,
      "activations/layer6_attention_weight_min": -45.62699508666992,
      "activations/layer7_attention_weight_max": 96.26211547851562,
      "activations/layer7_attention_weight_min": -95.22791290283203,
      "activations/layer8_attention_weight_max": 40.31516647338867,
      "activations/layer8_attention_weight_min": -43.07497024536133,
      "activations/layer9_attention_weight_max": 32.60599136352539,
      "activations/layer9_attention_weight_min": -34.404396057128906,
      "epoch": 17.06,
      "learning_rate": 4.041060606060606e-05,
      "loss": 2.756,
      "step": 293550
    },
    {
      "activations/layer0_attention_weight_max": 16.261085510253906,
      "activations/layer0_attention_weight_min": -13.575448989868164,
      "activations/layer10_attention_weight_max": 45.10752487182617,
      "activations/layer10_attention_weight_min": -45.604454040527344,
      "activations/layer11_attention_weight_max": 45.25177001953125,
      "activations/layer11_attention_weight_min": -46.205013275146484,
      "activations/layer12_attention_weight_max": 34.5604248046875,
      "activations/layer12_attention_weight_min": -31.269041061401367,
      "activations/layer13_attention_weight_max": 51.35794448852539,
      "activations/layer13_attention_weight_min": -46.73501968383789,
      "activations/layer14_attention_weight_max": 59.9876823425293,
      "activations/layer14_attention_weight_min": -52.4340934753418,
      "activations/layer15_attention_weight_max": 57.266441345214844,
      "activations/layer15_attention_weight_min": -53.586875915527344,
      "activations/layer16_attention_weight_max": 39.41484832763672,
      "activations/layer16_attention_weight_min": -29.509490966796875,
      "activations/layer17_attention_weight_max": 31.27698516845703,
      "activations/layer17_attention_weight_min": -28.110952377319336,
      "activations/layer18_attention_weight_max": 30.027536392211914,
      "activations/layer18_attention_weight_min": -28.396268844604492,
      "activations/layer19_attention_weight_max": 34.91026306152344,
      "activations/layer19_attention_weight_min": -31.9950008392334,
      "activations/layer1_attention_weight_max": 17.762954711914062,
      "activations/layer1_attention_weight_min": -16.434715270996094,
      "activations/layer20_attention_weight_max": 29.41461181640625,
      "activations/layer20_attention_weight_min": -25.771278381347656,
      "activations/layer21_attention_weight_max": 28.8720760345459,
      "activations/layer21_attention_weight_min": -25.057653427124023,
      "activations/layer22_attention_weight_max": 41.103450775146484,
      "activations/layer22_attention_weight_min": -34.343467712402344,
      "activations/layer23_attention_weight_max": 30.628957748413086,
      "activations/layer23_attention_weight_min": -25.361438751220703,
      "activations/layer2_attention_weight_max": 36.47927474975586,
      "activations/layer2_attention_weight_min": -37.10302734375,
      "activations/layer3_attention_weight_max": 101.81356811523438,
      "activations/layer3_attention_weight_min": -108.99137878417969,
      "activations/layer4_attention_weight_max": 106.8707046508789,
      "activations/layer4_attention_weight_min": -106.11132049560547,
      "activations/layer5_attention_weight_max": 53.69297409057617,
      "activations/layer5_attention_weight_min": -61.03047180175781,
      "activations/layer6_attention_weight_max": 47.732906341552734,
      "activations/layer6_attention_weight_min": -47.57343673706055,
      "activations/layer7_attention_weight_max": 112.14643859863281,
      "activations/layer7_attention_weight_min": -105.60124969482422,
      "activations/layer8_attention_weight_max": 50.35869216918945,
      "activations/layer8_attention_weight_min": -51.7221565246582,
      "activations/layer9_attention_weight_max": 39.70652389526367,
      "activations/layer9_attention_weight_min": -42.601558685302734,
      "epoch": 17.06,
      "learning_rate": 4.039166666666666e-05,
      "loss": 2.7351,
      "step": 293600
    },
    {
      "activations/layer0_attention_weight_max": 15.980436325073242,
      "activations/layer0_attention_weight_min": -12.279460906982422,
      "activations/layer10_attention_weight_max": 35.56504821777344,
      "activations/layer10_attention_weight_min": -34.10573959350586,
      "activations/layer11_attention_weight_max": 36.13335418701172,
      "activations/layer11_attention_weight_min": -34.41743469238281,
      "activations/layer12_attention_weight_max": 28.833044052124023,
      "activations/layer12_attention_weight_min": -27.051912307739258,
      "activations/layer13_attention_weight_max": 42.79961013793945,
      "activations/layer13_attention_weight_min": -38.142208099365234,
      "activations/layer14_attention_weight_max": 43.81117630004883,
      "activations/layer14_attention_weight_min": -39.72782897949219,
      "activations/layer15_attention_weight_max": 40.84722137451172,
      "activations/layer15_attention_weight_min": -36.85765075683594,
      "activations/layer16_attention_weight_max": 29.72027587890625,
      "activations/layer16_attention_weight_min": -29.430465698242188,
      "activations/layer17_attention_weight_max": 30.143585205078125,
      "activations/layer17_attention_weight_min": -26.58261489868164,
      "activations/layer18_attention_weight_max": 32.81404495239258,
      "activations/layer18_attention_weight_min": -26.857242584228516,
      "activations/layer19_attention_weight_max": 33.899471282958984,
      "activations/layer19_attention_weight_min": -32.04964065551758,
      "activations/layer1_attention_weight_max": 16.06547737121582,
      "activations/layer1_attention_weight_min": -14.785911560058594,
      "activations/layer20_attention_weight_max": 28.801616668701172,
      "activations/layer20_attention_weight_min": -26.48150634765625,
      "activations/layer21_attention_weight_max": 30.759029388427734,
      "activations/layer21_attention_weight_min": -26.441837310791016,
      "activations/layer22_attention_weight_max": 42.1889533996582,
      "activations/layer22_attention_weight_min": -33.052730560302734,
      "activations/layer23_attention_weight_max": 31.978618621826172,
      "activations/layer23_attention_weight_min": -28.751379013061523,
      "activations/layer2_attention_weight_max": 33.524993896484375,
      "activations/layer2_attention_weight_min": -33.21303176879883,
      "activations/layer3_attention_weight_max": 98.91059875488281,
      "activations/layer3_attention_weight_min": -98.64974975585938,
      "activations/layer4_attention_weight_max": 110.76419830322266,
      "activations/layer4_attention_weight_min": -108.61998748779297,
      "activations/layer5_attention_weight_max": 51.814613342285156,
      "activations/layer5_attention_weight_min": -62.830650329589844,
      "activations/layer6_attention_weight_max": 45.69902801513672,
      "activations/layer6_attention_weight_min": -48.965599060058594,
      "activations/layer7_attention_weight_max": 93.26555633544922,
      "activations/layer7_attention_weight_min": -97.19429779052734,
      "activations/layer8_attention_weight_max": 43.000553131103516,
      "activations/layer8_attention_weight_min": -44.2205810546875,
      "activations/layer9_attention_weight_max": 33.59494400024414,
      "activations/layer9_attention_weight_min": -35.171409606933594,
      "epoch": 17.06,
      "learning_rate": 4.037272727272727e-05,
      "loss": 2.7355,
      "step": 293650
    },
    {
      "activations/layer0_attention_weight_max": 15.32095718383789,
      "activations/layer0_attention_weight_min": -13.630271911621094,
      "activations/layer10_attention_weight_max": 31.38140869140625,
      "activations/layer10_attention_weight_min": -32.86891174316406,
      "activations/layer11_attention_weight_max": 32.07732009887695,
      "activations/layer11_attention_weight_min": -33.431907653808594,
      "activations/layer12_attention_weight_max": 28.69251251220703,
      "activations/layer12_attention_weight_min": -25.49217987060547,
      "activations/layer13_attention_weight_max": 40.60194396972656,
      "activations/layer13_attention_weight_min": -33.60175323486328,
      "activations/layer14_attention_weight_max": 42.54422378540039,
      "activations/layer14_attention_weight_min": -35.55906295776367,
      "activations/layer15_attention_weight_max": 40.46052169799805,
      "activations/layer15_attention_weight_min": -33.42628479003906,
      "activations/layer16_attention_weight_max": 33.9892578125,
      "activations/layer16_attention_weight_min": -29.06780242919922,
      "activations/layer17_attention_weight_max": 36.083404541015625,
      "activations/layer17_attention_weight_min": -26.547992706298828,
      "activations/layer18_attention_weight_max": 32.285728454589844,
      "activations/layer18_attention_weight_min": -23.83708953857422,
      "activations/layer19_attention_weight_max": 37.88167953491211,
      "activations/layer19_attention_weight_min": -31.162931442260742,
      "activations/layer1_attention_weight_max": 16.433879852294922,
      "activations/layer1_attention_weight_min": -15.627522468566895,
      "activations/layer20_attention_weight_max": 29.950910568237305,
      "activations/layer20_attention_weight_min": -24.43511390686035,
      "activations/layer21_attention_weight_max": 32.712608337402344,
      "activations/layer21_attention_weight_min": -25.26902198791504,
      "activations/layer22_attention_weight_max": 43.09746551513672,
      "activations/layer22_attention_weight_min": -30.44619369506836,
      "activations/layer23_attention_weight_max": 36.78167724609375,
      "activations/layer23_attention_weight_min": -26.002037048339844,
      "activations/layer2_attention_weight_max": 33.05378341674805,
      "activations/layer2_attention_weight_min": -33.86369323730469,
      "activations/layer3_attention_weight_max": 95.75757598876953,
      "activations/layer3_attention_weight_min": -94.67024993896484,
      "activations/layer4_attention_weight_max": 105.02252197265625,
      "activations/layer4_attention_weight_min": -105.9608383178711,
      "activations/layer5_attention_weight_max": 49.308284759521484,
      "activations/layer5_attention_weight_min": -59.48358917236328,
      "activations/layer6_attention_weight_max": 42.5987548828125,
      "activations/layer6_attention_weight_min": -46.022804260253906,
      "activations/layer7_attention_weight_max": 90.33124542236328,
      "activations/layer7_attention_weight_min": -90.50214385986328,
      "activations/layer8_attention_weight_max": 40.32187271118164,
      "activations/layer8_attention_weight_min": -43.66616439819336,
      "activations/layer9_attention_weight_max": 30.47259521484375,
      "activations/layer9_attention_weight_min": -33.99558639526367,
      "epoch": 17.07,
      "learning_rate": 4.035378787878787e-05,
      "loss": 2.7307,
      "step": 293700
    },
    {
      "activations/layer0_attention_weight_max": 16.351791381835938,
      "activations/layer0_attention_weight_min": -13.169055938720703,
      "activations/layer10_attention_weight_max": 32.337093353271484,
      "activations/layer10_attention_weight_min": -34.41387176513672,
      "activations/layer11_attention_weight_max": 32.21031188964844,
      "activations/layer11_attention_weight_min": -33.32780075073242,
      "activations/layer12_attention_weight_max": 28.230445861816406,
      "activations/layer12_attention_weight_min": -26.273820877075195,
      "activations/layer13_attention_weight_max": 39.78626251220703,
      "activations/layer13_attention_weight_min": -35.97178268432617,
      "activations/layer14_attention_weight_max": 45.2059211730957,
      "activations/layer14_attention_weight_min": -38.446617126464844,
      "activations/layer15_attention_weight_max": 39.649417877197266,
      "activations/layer15_attention_weight_min": -34.39147186279297,
      "activations/layer16_attention_weight_max": 29.37131690979004,
      "activations/layer16_attention_weight_min": -27.308666229248047,
      "activations/layer17_attention_weight_max": 30.465389251708984,
      "activations/layer17_attention_weight_min": -27.393003463745117,
      "activations/layer18_attention_weight_max": 32.822349548339844,
      "activations/layer18_attention_weight_min": -28.326913833618164,
      "activations/layer19_attention_weight_max": 32.662654876708984,
      "activations/layer19_attention_weight_min": -31.610382080078125,
      "activations/layer1_attention_weight_max": 16.511821746826172,
      "activations/layer1_attention_weight_min": -14.161429405212402,
      "activations/layer20_attention_weight_max": 30.0701904296875,
      "activations/layer20_attention_weight_min": -25.578439712524414,
      "activations/layer21_attention_weight_max": 31.16615867614746,
      "activations/layer21_attention_weight_min": -24.584951400756836,
      "activations/layer22_attention_weight_max": 40.84814453125,
      "activations/layer22_attention_weight_min": -31.23129653930664,
      "activations/layer23_attention_weight_max": 33.59995651245117,
      "activations/layer23_attention_weight_min": -28.514514923095703,
      "activations/layer2_attention_weight_max": 32.53547668457031,
      "activations/layer2_attention_weight_min": -33.48776626586914,
      "activations/layer3_attention_weight_max": 97.38367462158203,
      "activations/layer3_attention_weight_min": -98.73973846435547,
      "activations/layer4_attention_weight_max": 105.19551086425781,
      "activations/layer4_attention_weight_min": -108.6344223022461,
      "activations/layer5_attention_weight_max": 50.57986068725586,
      "activations/layer5_attention_weight_min": -62.32033920288086,
      "activations/layer6_attention_weight_max": 44.642250061035156,
      "activations/layer6_attention_weight_min": -49.494468688964844,
      "activations/layer7_attention_weight_max": 92.15361022949219,
      "activations/layer7_attention_weight_min": -98.47650146484375,
      "activations/layer8_attention_weight_max": 40.97859191894531,
      "activations/layer8_attention_weight_min": -46.03737258911133,
      "activations/layer9_attention_weight_max": 30.997617721557617,
      "activations/layer9_attention_weight_min": -36.08693313598633,
      "epoch": 17.07,
      "learning_rate": 4.0334848484848486e-05,
      "loss": 2.7202,
      "step": 293750
    },
    {
      "activations/layer0_attention_weight_max": 16.16928482055664,
      "activations/layer0_attention_weight_min": -13.278278350830078,
      "activations/layer10_attention_weight_max": 31.71445083618164,
      "activations/layer10_attention_weight_min": -32.25639343261719,
      "activations/layer11_attention_weight_max": 32.9187126159668,
      "activations/layer11_attention_weight_min": -32.06346130371094,
      "activations/layer12_attention_weight_max": 24.112592697143555,
      "activations/layer12_attention_weight_min": -26.174509048461914,
      "activations/layer13_attention_weight_max": 35.471622467041016,
      "activations/layer13_attention_weight_min": -34.81992721557617,
      "activations/layer14_attention_weight_max": 37.950748443603516,
      "activations/layer14_attention_weight_min": -36.17424011230469,
      "activations/layer15_attention_weight_max": 35.88089370727539,
      "activations/layer15_attention_weight_min": -35.49315643310547,
      "activations/layer16_attention_weight_max": 28.338558197021484,
      "activations/layer16_attention_weight_min": -29.580263137817383,
      "activations/layer17_attention_weight_max": 28.649675369262695,
      "activations/layer17_attention_weight_min": -25.48570442199707,
      "activations/layer18_attention_weight_max": 31.717269897460938,
      "activations/layer18_attention_weight_min": -27.6366024017334,
      "activations/layer19_attention_weight_max": 34.096839904785156,
      "activations/layer19_attention_weight_min": -34.27497482299805,
      "activations/layer1_attention_weight_max": 16.24700355529785,
      "activations/layer1_attention_weight_min": -14.917163848876953,
      "activations/layer20_attention_weight_max": 31.17751121520996,
      "activations/layer20_attention_weight_min": -28.73931312561035,
      "activations/layer21_attention_weight_max": 29.20754623413086,
      "activations/layer21_attention_weight_min": -26.01426887512207,
      "activations/layer22_attention_weight_max": 38.88252639770508,
      "activations/layer22_attention_weight_min": -30.211118698120117,
      "activations/layer23_attention_weight_max": 34.0706672668457,
      "activations/layer23_attention_weight_min": -26.66351890563965,
      "activations/layer2_attention_weight_max": 34.43071746826172,
      "activations/layer2_attention_weight_min": -33.76998519897461,
      "activations/layer3_attention_weight_max": 99.54193115234375,
      "activations/layer3_attention_weight_min": -99.80314636230469,
      "activations/layer4_attention_weight_max": 107.877197265625,
      "activations/layer4_attention_weight_min": -106.70702362060547,
      "activations/layer5_attention_weight_max": 49.90779113769531,
      "activations/layer5_attention_weight_min": -63.90797424316406,
      "activations/layer6_attention_weight_max": 42.425079345703125,
      "activations/layer6_attention_weight_min": -43.95560836791992,
      "activations/layer7_attention_weight_max": 91.13880157470703,
      "activations/layer7_attention_weight_min": -94.5579605102539,
      "activations/layer8_attention_weight_max": 38.295467376708984,
      "activations/layer8_attention_weight_min": -42.99436569213867,
      "activations/layer9_attention_weight_max": 30.386091232299805,
      "activations/layer9_attention_weight_min": -32.49144744873047,
      "epoch": 17.07,
      "learning_rate": 4.031590909090909e-05,
      "loss": 2.7512,
      "step": 293800
    },
    {
      "activations/layer0_attention_weight_max": 17.819974899291992,
      "activations/layer0_attention_weight_min": -12.478450775146484,
      "activations/layer10_attention_weight_max": 37.714664459228516,
      "activations/layer10_attention_weight_min": -36.90447998046875,
      "activations/layer11_attention_weight_max": 36.51874923706055,
      "activations/layer11_attention_weight_min": -37.67585754394531,
      "activations/layer12_attention_weight_max": 28.718734741210938,
      "activations/layer12_attention_weight_min": -30.19104766845703,
      "activations/layer13_attention_weight_max": 41.86045837402344,
      "activations/layer13_attention_weight_min": -40.95709228515625,
      "activations/layer14_attention_weight_max": 48.23362731933594,
      "activations/layer14_attention_weight_min": -41.2429084777832,
      "activations/layer15_attention_weight_max": 43.398868560791016,
      "activations/layer15_attention_weight_min": -38.32474136352539,
      "activations/layer16_attention_weight_max": 31.159223556518555,
      "activations/layer16_attention_weight_min": -29.290685653686523,
      "activations/layer17_attention_weight_max": 31.747615814208984,
      "activations/layer17_attention_weight_min": -28.228015899658203,
      "activations/layer18_attention_weight_max": 32.98517990112305,
      "activations/layer18_attention_weight_min": -27.890905380249023,
      "activations/layer19_attention_weight_max": 35.5657958984375,
      "activations/layer19_attention_weight_min": -31.764575958251953,
      "activations/layer1_attention_weight_max": 16.355728149414062,
      "activations/layer1_attention_weight_min": -14.550679206848145,
      "activations/layer20_attention_weight_max": 32.18852233886719,
      "activations/layer20_attention_weight_min": -27.435888290405273,
      "activations/layer21_attention_weight_max": 30.37558937072754,
      "activations/layer21_attention_weight_min": -27.11979866027832,
      "activations/layer22_attention_weight_max": 49.42766189575195,
      "activations/layer22_attention_weight_min": -31.487350463867188,
      "activations/layer23_attention_weight_max": 35.65127182006836,
      "activations/layer23_attention_weight_min": -25.790889739990234,
      "activations/layer2_attention_weight_max": 33.648101806640625,
      "activations/layer2_attention_weight_min": -33.96589660644531,
      "activations/layer3_attention_weight_max": 99.94055938720703,
      "activations/layer3_attention_weight_min": -97.54363250732422,
      "activations/layer4_attention_weight_max": 108.86779022216797,
      "activations/layer4_attention_weight_min": -108.91902160644531,
      "activations/layer5_attention_weight_max": 53.98876190185547,
      "activations/layer5_attention_weight_min": -63.04649353027344,
      "activations/layer6_attention_weight_max": 47.63264465332031,
      "activations/layer6_attention_weight_min": -48.92204284667969,
      "activations/layer7_attention_weight_max": 100.00586700439453,
      "activations/layer7_attention_weight_min": -100.39061737060547,
      "activations/layer8_attention_weight_max": 48.774959564208984,
      "activations/layer8_attention_weight_min": -52.08383560180664,
      "activations/layer9_attention_weight_max": 37.01374435424805,
      "activations/layer9_attention_weight_min": -39.622982025146484,
      "epoch": 17.07,
      "learning_rate": 4.029734848484848e-05,
      "loss": 2.7408,
      "step": 293850
    },
    {
      "activations/layer0_attention_weight_max": 16.443073272705078,
      "activations/layer0_attention_weight_min": -13.18055534362793,
      "activations/layer10_attention_weight_max": 36.95815658569336,
      "activations/layer10_attention_weight_min": -36.68872833251953,
      "activations/layer11_attention_weight_max": 35.37068557739258,
      "activations/layer11_attention_weight_min": -35.487953186035156,
      "activations/layer12_attention_weight_max": 29.631261825561523,
      "activations/layer12_attention_weight_min": -25.463668823242188,
      "activations/layer13_attention_weight_max": 46.87843704223633,
      "activations/layer13_attention_weight_min": -35.44597625732422,
      "activations/layer14_attention_weight_max": 51.78644561767578,
      "activations/layer14_attention_weight_min": -39.85745620727539,
      "activations/layer15_attention_weight_max": 44.57872772216797,
      "activations/layer15_attention_weight_min": -34.73585891723633,
      "activations/layer16_attention_weight_max": 35.531715393066406,
      "activations/layer16_attention_weight_min": -28.69297981262207,
      "activations/layer17_attention_weight_max": 34.77958679199219,
      "activations/layer17_attention_weight_min": -26.414854049682617,
      "activations/layer18_attention_weight_max": 30.539165496826172,
      "activations/layer18_attention_weight_min": -24.916873931884766,
      "activations/layer19_attention_weight_max": 35.41265869140625,
      "activations/layer19_attention_weight_min": -33.42933654785156,
      "activations/layer1_attention_weight_max": 16.332181930541992,
      "activations/layer1_attention_weight_min": -15.529106140136719,
      "activations/layer20_attention_weight_max": 30.045507431030273,
      "activations/layer20_attention_weight_min": -25.744138717651367,
      "activations/layer21_attention_weight_max": 29.79683494567871,
      "activations/layer21_attention_weight_min": -25.09914207458496,
      "activations/layer22_attention_weight_max": 50.54377746582031,
      "activations/layer22_attention_weight_min": -33.82498550415039,
      "activations/layer23_attention_weight_max": 34.791259765625,
      "activations/layer23_attention_weight_min": -26.443357467651367,
      "activations/layer2_attention_weight_max": 32.095848083496094,
      "activations/layer2_attention_weight_min": -33.461673736572266,
      "activations/layer3_attention_weight_max": 90.99153900146484,
      "activations/layer3_attention_weight_min": -96.7672348022461,
      "activations/layer4_attention_weight_max": 104.07964324951172,
      "activations/layer4_attention_weight_min": -105.74828338623047,
      "activations/layer5_attention_weight_max": 51.25991439819336,
      "activations/layer5_attention_weight_min": -64.74140930175781,
      "activations/layer6_attention_weight_max": 45.048072814941406,
      "activations/layer6_attention_weight_min": -47.6214714050293,
      "activations/layer7_attention_weight_max": 89.70498657226562,
      "activations/layer7_attention_weight_min": -95.51390838623047,
      "activations/layer8_attention_weight_max": 40.890140533447266,
      "activations/layer8_attention_weight_min": -42.4557991027832,
      "activations/layer9_attention_weight_max": 35.16056823730469,
      "activations/layer9_attention_weight_min": -36.232200622558594,
      "epoch": 17.08,
      "learning_rate": 4.027840909090909e-05,
      "loss": 2.735,
      "step": 293900
    },
    {
      "activations/layer0_attention_weight_max": 16.398799896240234,
      "activations/layer0_attention_weight_min": -12.131957054138184,
      "activations/layer10_attention_weight_max": 32.83020782470703,
      "activations/layer10_attention_weight_min": -32.702239990234375,
      "activations/layer11_attention_weight_max": 33.81334686279297,
      "activations/layer11_attention_weight_min": -34.509910583496094,
      "activations/layer12_attention_weight_max": 24.9951171875,
      "activations/layer12_attention_weight_min": -24.88495445251465,
      "activations/layer13_attention_weight_max": 37.92066192626953,
      "activations/layer13_attention_weight_min": -33.59791946411133,
      "activations/layer14_attention_weight_max": 40.686439514160156,
      "activations/layer14_attention_weight_min": -35.272010803222656,
      "activations/layer15_attention_weight_max": 36.29973220825195,
      "activations/layer15_attention_weight_min": -33.84880828857422,
      "activations/layer16_attention_weight_max": 31.37380027770996,
      "activations/layer16_attention_weight_min": -29.090728759765625,
      "activations/layer17_attention_weight_max": 32.764705657958984,
      "activations/layer17_attention_weight_min": -25.16143798828125,
      "activations/layer18_attention_weight_max": 31.202226638793945,
      "activations/layer18_attention_weight_min": -25.361743927001953,
      "activations/layer19_attention_weight_max": 32.04889678955078,
      "activations/layer19_attention_weight_min": -31.11353874206543,
      "activations/layer1_attention_weight_max": 16.56711196899414,
      "activations/layer1_attention_weight_min": -15.070966720581055,
      "activations/layer20_attention_weight_max": 28.027841567993164,
      "activations/layer20_attention_weight_min": -24.656009674072266,
      "activations/layer21_attention_weight_max": 29.766002655029297,
      "activations/layer21_attention_weight_min": -24.75567626953125,
      "activations/layer22_attention_weight_max": 38.89248275756836,
      "activations/layer22_attention_weight_min": -31.069360733032227,
      "activations/layer23_attention_weight_max": 34.616119384765625,
      "activations/layer23_attention_weight_min": -28.04842758178711,
      "activations/layer2_attention_weight_max": 33.76984405517578,
      "activations/layer2_attention_weight_min": -32.450111389160156,
      "activations/layer3_attention_weight_max": 97.03477478027344,
      "activations/layer3_attention_weight_min": -94.44122314453125,
      "activations/layer4_attention_weight_max": 102.77532196044922,
      "activations/layer4_attention_weight_min": -101.78936767578125,
      "activations/layer5_attention_weight_max": 50.56472396850586,
      "activations/layer5_attention_weight_min": -58.6501579284668,
      "activations/layer6_attention_weight_max": 43.89950180053711,
      "activations/layer6_attention_weight_min": -44.451805114746094,
      "activations/layer7_attention_weight_max": 88.89401245117188,
      "activations/layer7_attention_weight_min": -95.03594207763672,
      "activations/layer8_attention_weight_max": 40.09160614013672,
      "activations/layer8_attention_weight_min": -41.84237289428711,
      "activations/layer9_attention_weight_max": 29.646554946899414,
      "activations/layer9_attention_weight_min": -32.382476806640625,
      "epoch": 17.08,
      "learning_rate": 4.0259469696969695e-05,
      "loss": 2.7377,
      "step": 293950
    },
    {
      "activations/layer0_attention_weight_max": 16.572900772094727,
      "activations/layer0_attention_weight_min": -12.91055965423584,
      "activations/layer10_attention_weight_max": 29.712799072265625,
      "activations/layer10_attention_weight_min": -31.4398193359375,
      "activations/layer11_attention_weight_max": 31.428449630737305,
      "activations/layer11_attention_weight_min": -32.09844970703125,
      "activations/layer12_attention_weight_max": 26.544944763183594,
      "activations/layer12_attention_weight_min": -25.90104866027832,
      "activations/layer13_attention_weight_max": 37.58962631225586,
      "activations/layer13_attention_weight_min": -35.38215637207031,
      "activations/layer14_attention_weight_max": 40.81450653076172,
      "activations/layer14_attention_weight_min": -38.75468063354492,
      "activations/layer15_attention_weight_max": 38.559539794921875,
      "activations/layer15_attention_weight_min": -35.16989517211914,
      "activations/layer16_attention_weight_max": 28.09235382080078,
      "activations/layer16_attention_weight_min": -27.62178611755371,
      "activations/layer17_attention_weight_max": 29.071510314941406,
      "activations/layer17_attention_weight_min": -26.127662658691406,
      "activations/layer18_attention_weight_max": 31.858314514160156,
      "activations/layer18_attention_weight_min": -25.106691360473633,
      "activations/layer19_attention_weight_max": 38.236839294433594,
      "activations/layer19_attention_weight_min": -30.412811279296875,
      "activations/layer1_attention_weight_max": 15.420482635498047,
      "activations/layer1_attention_weight_min": -14.866251945495605,
      "activations/layer20_attention_weight_max": 30.754066467285156,
      "activations/layer20_attention_weight_min": -24.966411590576172,
      "activations/layer21_attention_weight_max": 29.445737838745117,
      "activations/layer21_attention_weight_min": -24.852006912231445,
      "activations/layer22_attention_weight_max": 44.504180908203125,
      "activations/layer22_attention_weight_min": -32.279685974121094,
      "activations/layer23_attention_weight_max": 33.820411682128906,
      "activations/layer23_attention_weight_min": -25.579021453857422,
      "activations/layer2_attention_weight_max": 31.00909996032715,
      "activations/layer2_attention_weight_min": -32.802276611328125,
      "activations/layer3_attention_weight_max": 90.53441619873047,
      "activations/layer3_attention_weight_min": -98.04499053955078,
      "activations/layer4_attention_weight_max": 101.66624450683594,
      "activations/layer4_attention_weight_min": -100.16694641113281,
      "activations/layer5_attention_weight_max": 49.5675163269043,
      "activations/layer5_attention_weight_min": -61.41100311279297,
      "activations/layer6_attention_weight_max": 42.69404220581055,
      "activations/layer6_attention_weight_min": -45.18279266357422,
      "activations/layer7_attention_weight_max": 87.00458526611328,
      "activations/layer7_attention_weight_min": -91.29998016357422,
      "activations/layer8_attention_weight_max": 36.49629211425781,
      "activations/layer8_attention_weight_min": -39.57974624633789,
      "activations/layer9_attention_weight_max": 29.79561424255371,
      "activations/layer9_attention_weight_min": -30.678730010986328,
      "epoch": 17.08,
      "learning_rate": 4.0240530303030297e-05,
      "loss": 2.7303,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_loss": 2.697265625,
      "eval_runtime": 9.8677,
      "eval_samples_per_second": 435.155,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 9.8677,
      "eval_openwebtext_samples_per_second": 435.155,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9689,
      "eval_wikitext_samples_per_second": 231.606,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_lambada_loss": 2.388671875,
      "eval_lambada_ppl": 10.899009079882486,
      "eval_lambada_runtime": 9.5687,
      "eval_lambada_samples_per_second": 508.846,
      "step": 294000
    },
    {
      "activations/layer0_attention_weight_max": 16.411420822143555,
      "activations/layer0_attention_weight_min": -13.453655242919922,
      "activations/layer10_attention_weight_max": 32.83821105957031,
      "activations/layer10_attention_weight_min": -32.95220184326172,
      "activations/layer11_attention_weight_max": 32.653141021728516,
      "activations/layer11_attention_weight_min": -33.89926528930664,
      "activations/layer12_attention_weight_max": 25.693477630615234,
      "activations/layer12_attention_weight_min": -25.235666275024414,
      "activations/layer13_attention_weight_max": 40.225257873535156,
      "activations/layer13_attention_weight_min": -37.627071380615234,
      "activations/layer14_attention_weight_max": 40.61674880981445,
      "activations/layer14_attention_weight_min": -37.26567840576172,
      "activations/layer15_attention_weight_max": 35.88694381713867,
      "activations/layer15_attention_weight_min": -33.80350112915039,
      "activations/layer16_attention_weight_max": 27.733909606933594,
      "activations/layer16_attention_weight_min": -27.68991470336914,
      "activations/layer17_attention_weight_max": 29.899391174316406,
      "activations/layer17_attention_weight_min": -25.166240692138672,
      "activations/layer18_attention_weight_max": 30.342670440673828,
      "activations/layer18_attention_weight_min": -24.798091888427734,
      "activations/layer19_attention_weight_max": 32.872806549072266,
      "activations/layer19_attention_weight_min": -29.926706314086914,
      "activations/layer1_attention_weight_max": 17.070039749145508,
      "activations/layer1_attention_weight_min": -15.521039009094238,
      "activations/layer20_attention_weight_max": 28.3929500579834,
      "activations/layer20_attention_weight_min": -25.074230194091797,
      "activations/layer21_attention_weight_max": 28.752206802368164,
      "activations/layer21_attention_weight_min": -24.226232528686523,
      "activations/layer22_attention_weight_max": 37.05875015258789,
      "activations/layer22_attention_weight_min": -29.666677474975586,
      "activations/layer23_attention_weight_max": 34.025970458984375,
      "activations/layer23_attention_weight_min": -25.590362548828125,
      "activations/layer2_attention_weight_max": 29.927335739135742,
      "activations/layer2_attention_weight_min": -31.05891990661621,
      "activations/layer3_attention_weight_max": 93.61315155029297,
      "activations/layer3_attention_weight_min": -95.6697998046875,
      "activations/layer4_attention_weight_max": 102.2651596069336,
      "activations/layer4_attention_weight_min": -103.68548583984375,
      "activations/layer5_attention_weight_max": 49.331233978271484,
      "activations/layer5_attention_weight_min": -59.692962646484375,
      "activations/layer6_attention_weight_max": 42.359832763671875,
      "activations/layer6_attention_weight_min": -46.57723617553711,
      "activations/layer7_attention_weight_max": 92.45818328857422,
      "activations/layer7_attention_weight_min": -91.1093978881836,
      "activations/layer8_attention_weight_max": 39.726505279541016,
      "activations/layer8_attention_weight_min": -41.69126510620117,
      "activations/layer9_attention_weight_max": 31.90065574645996,
      "activations/layer9_attention_weight_min": -33.37234115600586,
      "epoch": 17.09,
      "learning_rate": 4.0221590909090905e-05,
      "loss": 2.7325,
      "step": 294050
    },
    {
      "activations/layer0_attention_weight_max": 16.947059631347656,
      "activations/layer0_attention_weight_min": -13.010262489318848,
      "activations/layer10_attention_weight_max": 33.17621612548828,
      "activations/layer10_attention_weight_min": -33.33453369140625,
      "activations/layer11_attention_weight_max": 33.172515869140625,
      "activations/layer11_attention_weight_min": -34.626670837402344,
      "activations/layer12_attention_weight_max": 27.639734268188477,
      "activations/layer12_attention_weight_min": -27.700342178344727,
      "activations/layer13_attention_weight_max": 37.924278259277344,
      "activations/layer13_attention_weight_min": -35.49668502807617,
      "activations/layer14_attention_weight_max": 40.184635162353516,
      "activations/layer14_attention_weight_min": -38.385128021240234,
      "activations/layer15_attention_weight_max": 36.49378967285156,
      "activations/layer15_attention_weight_min": -33.70063018798828,
      "activations/layer16_attention_weight_max": 27.944639205932617,
      "activations/layer16_attention_weight_min": -28.437273025512695,
      "activations/layer17_attention_weight_max": 32.53611755371094,
      "activations/layer17_attention_weight_min": -26.16815185546875,
      "activations/layer18_attention_weight_max": 32.347843170166016,
      "activations/layer18_attention_weight_min": -28.3167667388916,
      "activations/layer19_attention_weight_max": 31.679523468017578,
      "activations/layer19_attention_weight_min": -31.193979263305664,
      "activations/layer1_attention_weight_max": 16.580228805541992,
      "activations/layer1_attention_weight_min": -15.467954635620117,
      "activations/layer20_attention_weight_max": 26.65483283996582,
      "activations/layer20_attention_weight_min": -25.873023986816406,
      "activations/layer21_attention_weight_max": 27.965118408203125,
      "activations/layer21_attention_weight_min": -24.948232650756836,
      "activations/layer22_attention_weight_max": 40.29717254638672,
      "activations/layer22_attention_weight_min": -31.15871810913086,
      "activations/layer23_attention_weight_max": 30.671117782592773,
      "activations/layer23_attention_weight_min": -26.505088806152344,
      "activations/layer2_attention_weight_max": 31.99069595336914,
      "activations/layer2_attention_weight_min": -33.177330017089844,
      "activations/layer3_attention_weight_max": 97.00984191894531,
      "activations/layer3_attention_weight_min": -96.84493255615234,
      "activations/layer4_attention_weight_max": 104.16088104248047,
      "activations/layer4_attention_weight_min": -103.85099029541016,
      "activations/layer5_attention_weight_max": 49.9058952331543,
      "activations/layer5_attention_weight_min": -58.323020935058594,
      "activations/layer6_attention_weight_max": 44.624061584472656,
      "activations/layer6_attention_weight_min": -46.10338592529297,
      "activations/layer7_attention_weight_max": 89.18303680419922,
      "activations/layer7_attention_weight_min": -95.16649627685547,
      "activations/layer8_attention_weight_max": 40.49885559082031,
      "activations/layer8_attention_weight_min": -44.470802307128906,
      "activations/layer9_attention_weight_max": 32.142791748046875,
      "activations/layer9_attention_weight_min": -34.08161926269531,
      "epoch": 17.09,
      "learning_rate": 4.020265151515151e-05,
      "loss": 2.7274,
      "step": 294100
    },
    {
      "activations/layer0_attention_weight_max": 16.18936538696289,
      "activations/layer0_attention_weight_min": -12.724185943603516,
      "activations/layer10_attention_weight_max": 32.60493469238281,
      "activations/layer10_attention_weight_min": -33.476383209228516,
      "activations/layer11_attention_weight_max": 33.659584045410156,
      "activations/layer11_attention_weight_min": -33.465789794921875,
      "activations/layer12_attention_weight_max": 27.438722610473633,
      "activations/layer12_attention_weight_min": -28.02350425720215,
      "activations/layer13_attention_weight_max": 39.967193603515625,
      "activations/layer13_attention_weight_min": -37.67483139038086,
      "activations/layer14_attention_weight_max": 42.90194320678711,
      "activations/layer14_attention_weight_min": -42.02033615112305,
      "activations/layer15_attention_weight_max": 38.77116394042969,
      "activations/layer15_attention_weight_min": -38.5677490234375,
      "activations/layer16_attention_weight_max": 29.28960609436035,
      "activations/layer16_attention_weight_min": -30.765090942382812,
      "activations/layer17_attention_weight_max": 30.425920486450195,
      "activations/layer17_attention_weight_min": -31.213985443115234,
      "activations/layer18_attention_weight_max": 30.266605377197266,
      "activations/layer18_attention_weight_min": -28.05811882019043,
      "activations/layer19_attention_weight_max": 35.303707122802734,
      "activations/layer19_attention_weight_min": -33.86428451538086,
      "activations/layer1_attention_weight_max": 16.418773651123047,
      "activations/layer1_attention_weight_min": -16.407976150512695,
      "activations/layer20_attention_weight_max": 29.964082717895508,
      "activations/layer20_attention_weight_min": -26.234041213989258,
      "activations/layer21_attention_weight_max": 28.246978759765625,
      "activations/layer21_attention_weight_min": -27.26735496520996,
      "activations/layer22_attention_weight_max": 40.14876937866211,
      "activations/layer22_attention_weight_min": -35.34439468383789,
      "activations/layer23_attention_weight_max": 34.3961181640625,
      "activations/layer23_attention_weight_min": -29.040708541870117,
      "activations/layer2_attention_weight_max": 33.214599609375,
      "activations/layer2_attention_weight_min": -34.03134536743164,
      "activations/layer3_attention_weight_max": 94.24076080322266,
      "activations/layer3_attention_weight_min": -102.18501281738281,
      "activations/layer4_attention_weight_max": 105.48701477050781,
      "activations/layer4_attention_weight_min": -107.95784759521484,
      "activations/layer5_attention_weight_max": 50.85346221923828,
      "activations/layer5_attention_weight_min": -69.18584442138672,
      "activations/layer6_attention_weight_max": 45.56742477416992,
      "activations/layer6_attention_weight_min": -49.17323684692383,
      "activations/layer7_attention_weight_max": 89.60128021240234,
      "activations/layer7_attention_weight_min": -98.71956634521484,
      "activations/layer8_attention_weight_max": 40.14234161376953,
      "activations/layer8_attention_weight_min": -43.06267166137695,
      "activations/layer9_attention_weight_max": 31.450143814086914,
      "activations/layer9_attention_weight_min": -33.860042572021484,
      "epoch": 17.09,
      "learning_rate": 4.018371212121212e-05,
      "loss": 2.7315,
      "step": 294150
    },
    {
      "activations/layer0_attention_weight_max": 16.0955753326416,
      "activations/layer0_attention_weight_min": -12.669790267944336,
      "activations/layer10_attention_weight_max": 36.063812255859375,
      "activations/layer10_attention_weight_min": -34.79063034057617,
      "activations/layer11_attention_weight_max": 34.5026741027832,
      "activations/layer11_attention_weight_min": -34.20349884033203,
      "activations/layer12_attention_weight_max": 26.87517738342285,
      "activations/layer12_attention_weight_min": -25.89040756225586,
      "activations/layer13_attention_weight_max": 39.958839416503906,
      "activations/layer13_attention_weight_min": -36.12235641479492,
      "activations/layer14_attention_weight_max": 43.70997619628906,
      "activations/layer14_attention_weight_min": -37.18523025512695,
      "activations/layer15_attention_weight_max": 40.92601013183594,
      "activations/layer15_attention_weight_min": -38.11148452758789,
      "activations/layer16_attention_weight_max": 30.309810638427734,
      "activations/layer16_attention_weight_min": -29.512842178344727,
      "activations/layer17_attention_weight_max": 30.337100982666016,
      "activations/layer17_attention_weight_min": -27.804954528808594,
      "activations/layer18_attention_weight_max": 31.82282829284668,
      "activations/layer18_attention_weight_min": -25.24667739868164,
      "activations/layer19_attention_weight_max": 33.2811164855957,
      "activations/layer19_attention_weight_min": -32.39934158325195,
      "activations/layer1_attention_weight_max": 16.433107376098633,
      "activations/layer1_attention_weight_min": -14.440354347229004,
      "activations/layer20_attention_weight_max": 27.86922836303711,
      "activations/layer20_attention_weight_min": -24.92193031311035,
      "activations/layer21_attention_weight_max": 28.413082122802734,
      "activations/layer21_attention_weight_min": -25.132497787475586,
      "activations/layer22_attention_weight_max": 41.97883224487305,
      "activations/layer22_attention_weight_min": -30.7540340423584,
      "activations/layer23_attention_weight_max": 30.630027770996094,
      "activations/layer23_attention_weight_min": -26.26852798461914,
      "activations/layer2_attention_weight_max": 32.91940689086914,
      "activations/layer2_attention_weight_min": -33.45619201660156,
      "activations/layer3_attention_weight_max": 93.80377197265625,
      "activations/layer3_attention_weight_min": -95.04011535644531,
      "activations/layer4_attention_weight_max": 105.4080810546875,
      "activations/layer4_attention_weight_min": -109.61616516113281,
      "activations/layer5_attention_weight_max": 53.18647003173828,
      "activations/layer5_attention_weight_min": -66.15740966796875,
      "activations/layer6_attention_weight_max": 43.953712463378906,
      "activations/layer6_attention_weight_min": -45.68291473388672,
      "activations/layer7_attention_weight_max": 92.46868133544922,
      "activations/layer7_attention_weight_min": -96.3578109741211,
      "activations/layer8_attention_weight_max": 43.56700134277344,
      "activations/layer8_attention_weight_min": -43.557064056396484,
      "activations/layer9_attention_weight_max": 33.62884521484375,
      "activations/layer9_attention_weight_min": -35.46137619018555,
      "epoch": 17.09,
      "learning_rate": 4.016477272727272e-05,
      "loss": 2.7399,
      "step": 294200
    },
    {
      "activations/layer0_attention_weight_max": 16.2907657623291,
      "activations/layer0_attention_weight_min": -12.951138496398926,
      "activations/layer10_attention_weight_max": 32.452552795410156,
      "activations/layer10_attention_weight_min": -33.195777893066406,
      "activations/layer11_attention_weight_max": 34.43063735961914,
      "activations/layer11_attention_weight_min": -33.99114990234375,
      "activations/layer12_attention_weight_max": 26.98432731628418,
      "activations/layer12_attention_weight_min": -27.452959060668945,
      "activations/layer13_attention_weight_max": 40.20461654663086,
      "activations/layer13_attention_weight_min": -36.74690246582031,
      "activations/layer14_attention_weight_max": 44.712318420410156,
      "activations/layer14_attention_weight_min": -39.79792404174805,
      "activations/layer15_attention_weight_max": 39.26808547973633,
      "activations/layer15_attention_weight_min": -36.490692138671875,
      "activations/layer16_attention_weight_max": 31.082990646362305,
      "activations/layer16_attention_weight_min": -29.56181526184082,
      "activations/layer17_attention_weight_max": 30.921125411987305,
      "activations/layer17_attention_weight_min": -27.979944229125977,
      "activations/layer18_attention_weight_max": 36.335479736328125,
      "activations/layer18_attention_weight_min": -26.343881607055664,
      "activations/layer19_attention_weight_max": 37.009151458740234,
      "activations/layer19_attention_weight_min": -32.42643356323242,
      "activations/layer1_attention_weight_max": 16.46903419494629,
      "activations/layer1_attention_weight_min": -15.311967849731445,
      "activations/layer20_attention_weight_max": 35.437950134277344,
      "activations/layer20_attention_weight_min": -26.70802879333496,
      "activations/layer21_attention_weight_max": 35.53193283081055,
      "activations/layer21_attention_weight_min": -29.02037239074707,
      "activations/layer22_attention_weight_max": 47.180416107177734,
      "activations/layer22_attention_weight_min": -33.90168762207031,
      "activations/layer23_attention_weight_max": 36.20465087890625,
      "activations/layer23_attention_weight_min": -26.383987426757812,
      "activations/layer2_attention_weight_max": 31.719282150268555,
      "activations/layer2_attention_weight_min": -32.43519592285156,
      "activations/layer3_attention_weight_max": 94.34178924560547,
      "activations/layer3_attention_weight_min": -94.93489837646484,
      "activations/layer4_attention_weight_max": 102.59513092041016,
      "activations/layer4_attention_weight_min": -108.91732025146484,
      "activations/layer5_attention_weight_max": 48.198482513427734,
      "activations/layer5_attention_weight_min": -62.02525329589844,
      "activations/layer6_attention_weight_max": 44.06328582763672,
      "activations/layer6_attention_weight_min": -47.30380630493164,
      "activations/layer7_attention_weight_max": 90.7764663696289,
      "activations/layer7_attention_weight_min": -103.57797241210938,
      "activations/layer8_attention_weight_max": 40.385250091552734,
      "activations/layer8_attention_weight_min": -44.2009391784668,
      "activations/layer9_attention_weight_max": 32.87950134277344,
      "activations/layer9_attention_weight_min": -35.02555847167969,
      "epoch": 17.1,
      "learning_rate": 4.0145833333333324e-05,
      "loss": 2.7104,
      "step": 294250
    },
    {
      "activations/layer0_attention_weight_max": 15.371258735656738,
      "activations/layer0_attention_weight_min": -12.902120590209961,
      "activations/layer10_attention_weight_max": 38.961334228515625,
      "activations/layer10_attention_weight_min": -37.244293212890625,
      "activations/layer11_attention_weight_max": 40.32806396484375,
      "activations/layer11_attention_weight_min": -37.728702545166016,
      "activations/layer12_attention_weight_max": 30.405967712402344,
      "activations/layer12_attention_weight_min": -25.005538940429688,
      "activations/layer13_attention_weight_max": 43.35282897949219,
      "activations/layer13_attention_weight_min": -35.653724670410156,
      "activations/layer14_attention_weight_max": 55.03357696533203,
      "activations/layer14_attention_weight_min": -39.10464096069336,
      "activations/layer15_attention_weight_max": 44.85742950439453,
      "activations/layer15_attention_weight_min": -38.62306213378906,
      "activations/layer16_attention_weight_max": 34.009117126464844,
      "activations/layer16_attention_weight_min": -26.57918357849121,
      "activations/layer17_attention_weight_max": 31.18250274658203,
      "activations/layer17_attention_weight_min": -26.86033821105957,
      "activations/layer18_attention_weight_max": 31.634456634521484,
      "activations/layer18_attention_weight_min": -25.02437973022461,
      "activations/layer19_attention_weight_max": 38.9244384765625,
      "activations/layer19_attention_weight_min": -31.411890029907227,
      "activations/layer1_attention_weight_max": 16.629852294921875,
      "activations/layer1_attention_weight_min": -15.887524604797363,
      "activations/layer20_attention_weight_max": 32.243343353271484,
      "activations/layer20_attention_weight_min": -27.209428787231445,
      "activations/layer21_attention_weight_max": 29.751089096069336,
      "activations/layer21_attention_weight_min": -25.750308990478516,
      "activations/layer22_attention_weight_max": 49.118473052978516,
      "activations/layer22_attention_weight_min": -33.53009796142578,
      "activations/layer23_attention_weight_max": 31.489120483398438,
      "activations/layer23_attention_weight_min": -26.584516525268555,
      "activations/layer2_attention_weight_max": 37.01225280761719,
      "activations/layer2_attention_weight_min": -34.111480712890625,
      "activations/layer3_attention_weight_max": 100.48754119873047,
      "activations/layer3_attention_weight_min": -104.94166564941406,
      "activations/layer4_attention_weight_max": 106.51583099365234,
      "activations/layer4_attention_weight_min": -106.58756256103516,
      "activations/layer5_attention_weight_max": 51.863399505615234,
      "activations/layer5_attention_weight_min": -60.456886291503906,
      "activations/layer6_attention_weight_max": 45.134742736816406,
      "activations/layer6_attention_weight_min": -45.97335433959961,
      "activations/layer7_attention_weight_max": 103.33085632324219,
      "activations/layer7_attention_weight_min": -102.30719757080078,
      "activations/layer8_attention_weight_max": 48.18505096435547,
      "activations/layer8_attention_weight_min": -48.577274322509766,
      "activations/layer9_attention_weight_max": 33.5609245300293,
      "activations/layer9_attention_weight_min": -36.26371765136719,
      "epoch": 17.1,
      "learning_rate": 4.012689393939394e-05,
      "loss": 2.7217,
      "step": 294300
    },
    {
      "activations/layer0_attention_weight_max": 16.382997512817383,
      "activations/layer0_attention_weight_min": -14.13540267944336,
      "activations/layer10_attention_weight_max": 33.85495376586914,
      "activations/layer10_attention_weight_min": -35.15351104736328,
      "activations/layer11_attention_weight_max": 33.3308219909668,
      "activations/layer11_attention_weight_min": -34.266815185546875,
      "activations/layer12_attention_weight_max": 28.211694717407227,
      "activations/layer12_attention_weight_min": -28.810819625854492,
      "activations/layer13_attention_weight_max": 41.23383712768555,
      "activations/layer13_attention_weight_min": -37.43547821044922,
      "activations/layer14_attention_weight_max": 41.00270080566406,
      "activations/layer14_attention_weight_min": -39.74568176269531,
      "activations/layer15_attention_weight_max": 38.83513641357422,
      "activations/layer15_attention_weight_min": -34.35038757324219,
      "activations/layer16_attention_weight_max": 32.10527420043945,
      "activations/layer16_attention_weight_min": -28.02311897277832,
      "activations/layer17_attention_weight_max": 31.153154373168945,
      "activations/layer17_attention_weight_min": -26.758865356445312,
      "activations/layer18_attention_weight_max": 33.08253479003906,
      "activations/layer18_attention_weight_min": -28.235563278198242,
      "activations/layer19_attention_weight_max": 34.53840255737305,
      "activations/layer19_attention_weight_min": -30.86685562133789,
      "activations/layer1_attention_weight_max": 16.846328735351562,
      "activations/layer1_attention_weight_min": -15.294818878173828,
      "activations/layer20_attention_weight_max": 28.61693000793457,
      "activations/layer20_attention_weight_min": -26.339635848999023,
      "activations/layer21_attention_weight_max": 27.03586769104004,
      "activations/layer21_attention_weight_min": -25.4128360748291,
      "activations/layer22_attention_weight_max": 41.15549087524414,
      "activations/layer22_attention_weight_min": -30.354402542114258,
      "activations/layer23_attention_weight_max": 34.20203399658203,
      "activations/layer23_attention_weight_min": -25.38957977294922,
      "activations/layer2_attention_weight_max": 31.511199951171875,
      "activations/layer2_attention_weight_min": -33.340328216552734,
      "activations/layer3_attention_weight_max": 91.54202270507812,
      "activations/layer3_attention_weight_min": -96.1536865234375,
      "activations/layer4_attention_weight_max": 100.94197082519531,
      "activations/layer4_attention_weight_min": -104.79994201660156,
      "activations/layer5_attention_weight_max": 49.92327880859375,
      "activations/layer5_attention_weight_min": -61.371299743652344,
      "activations/layer6_attention_weight_max": 43.50526809692383,
      "activations/layer6_attention_weight_min": -48.866981506347656,
      "activations/layer7_attention_weight_max": 90.78446960449219,
      "activations/layer7_attention_weight_min": -94.27418518066406,
      "activations/layer8_attention_weight_max": 40.651145935058594,
      "activations/layer8_attention_weight_min": -44.468284606933594,
      "activations/layer9_attention_weight_max": 33.464534759521484,
      "activations/layer9_attention_weight_min": -33.588314056396484,
      "epoch": 17.1,
      "learning_rate": 4.010795454545454e-05,
      "loss": 2.7263,
      "step": 294350
    },
    {
      "activations/layer0_attention_weight_max": 16.226659774780273,
      "activations/layer0_attention_weight_min": -12.323972702026367,
      "activations/layer10_attention_weight_max": 34.566795349121094,
      "activations/layer10_attention_weight_min": -34.92252731323242,
      "activations/layer11_attention_weight_max": 34.87406921386719,
      "activations/layer11_attention_weight_min": -38.36646270751953,
      "activations/layer12_attention_weight_max": 27.37761688232422,
      "activations/layer12_attention_weight_min": -26.444608688354492,
      "activations/layer13_attention_weight_max": 40.33332824707031,
      "activations/layer13_attention_weight_min": -35.54712677001953,
      "activations/layer14_attention_weight_max": 46.986297607421875,
      "activations/layer14_attention_weight_min": -38.69651412963867,
      "activations/layer15_attention_weight_max": 42.349979400634766,
      "activations/layer15_attention_weight_min": -37.76993942260742,
      "activations/layer16_attention_weight_max": 34.32853698730469,
      "activations/layer16_attention_weight_min": -28.461294174194336,
      "activations/layer17_attention_weight_max": 33.13984680175781,
      "activations/layer17_attention_weight_min": -27.862686157226562,
      "activations/layer18_attention_weight_max": 30.858182907104492,
      "activations/layer18_attention_weight_min": -28.925636291503906,
      "activations/layer19_attention_weight_max": 35.6922607421875,
      "activations/layer19_attention_weight_min": -33.5127067565918,
      "activations/layer1_attention_weight_max": 17.686294555664062,
      "activations/layer1_attention_weight_min": -15.860089302062988,
      "activations/layer20_attention_weight_max": 30.09389877319336,
      "activations/layer20_attention_weight_min": -24.243818283081055,
      "activations/layer21_attention_weight_max": 26.51432228088379,
      "activations/layer21_attention_weight_min": -25.102468490600586,
      "activations/layer22_attention_weight_max": 41.663490295410156,
      "activations/layer22_attention_weight_min": -32.77179718017578,
      "activations/layer23_attention_weight_max": 33.81080627441406,
      "activations/layer23_attention_weight_min": -28.163671493530273,
      "activations/layer2_attention_weight_max": 34.05925750732422,
      "activations/layer2_attention_weight_min": -33.653656005859375,
      "activations/layer3_attention_weight_max": 99.64411926269531,
      "activations/layer3_attention_weight_min": -102.148193359375,
      "activations/layer4_attention_weight_max": 106.77485656738281,
      "activations/layer4_attention_weight_min": -105.18181610107422,
      "activations/layer5_attention_weight_max": 48.62073516845703,
      "activations/layer5_attention_weight_min": -58.354286193847656,
      "activations/layer6_attention_weight_max": 45.19068908691406,
      "activations/layer6_attention_weight_min": -47.6328010559082,
      "activations/layer7_attention_weight_max": 94.95182800292969,
      "activations/layer7_attention_weight_min": -101.69478607177734,
      "activations/layer8_attention_weight_max": 41.36655044555664,
      "activations/layer8_attention_weight_min": -46.57309341430664,
      "activations/layer9_attention_weight_max": 31.481794357299805,
      "activations/layer9_attention_weight_min": -32.91205978393555,
      "epoch": 17.11,
      "learning_rate": 4.008901515151515e-05,
      "loss": 2.7353,
      "step": 294400
    },
    {
      "activations/layer0_attention_weight_max": 16.572349548339844,
      "activations/layer0_attention_weight_min": -12.32135009765625,
      "activations/layer10_attention_weight_max": 35.60853958129883,
      "activations/layer10_attention_weight_min": -35.4349365234375,
      "activations/layer11_attention_weight_max": 35.88999557495117,
      "activations/layer11_attention_weight_min": -35.55229568481445,
      "activations/layer12_attention_weight_max": 27.931842803955078,
      "activations/layer12_attention_weight_min": -27.14676856994629,
      "activations/layer13_attention_weight_max": 39.625179290771484,
      "activations/layer13_attention_weight_min": -37.164974212646484,
      "activations/layer14_attention_weight_max": 43.092185974121094,
      "activations/layer14_attention_weight_min": -37.444618225097656,
      "activations/layer15_attention_weight_max": 38.627403259277344,
      "activations/layer15_attention_weight_min": -33.71817398071289,
      "activations/layer16_attention_weight_max": 29.121732711791992,
      "activations/layer16_attention_weight_min": -27.587343215942383,
      "activations/layer17_attention_weight_max": 30.40523910522461,
      "activations/layer17_attention_weight_min": -27.397132873535156,
      "activations/layer18_attention_weight_max": 31.558889389038086,
      "activations/layer18_attention_weight_min": -27.14213752746582,
      "activations/layer19_attention_weight_max": 33.95032501220703,
      "activations/layer19_attention_weight_min": -33.959842681884766,
      "activations/layer1_attention_weight_max": 16.310352325439453,
      "activations/layer1_attention_weight_min": -15.44624137878418,
      "activations/layer20_attention_weight_max": 28.54391098022461,
      "activations/layer20_attention_weight_min": -24.914487838745117,
      "activations/layer21_attention_weight_max": 29.29443359375,
      "activations/layer21_attention_weight_min": -24.141386032104492,
      "activations/layer22_attention_weight_max": 43.552764892578125,
      "activations/layer22_attention_weight_min": -31.319454193115234,
      "activations/layer23_attention_weight_max": 31.9377498626709,
      "activations/layer23_attention_weight_min": -26.366134643554688,
      "activations/layer2_attention_weight_max": 33.37046432495117,
      "activations/layer2_attention_weight_min": -32.929115295410156,
      "activations/layer3_attention_weight_max": 96.71479034423828,
      "activations/layer3_attention_weight_min": -100.04129791259766,
      "activations/layer4_attention_weight_max": 106.93035888671875,
      "activations/layer4_attention_weight_min": -113.71331787109375,
      "activations/layer5_attention_weight_max": 51.3848991394043,
      "activations/layer5_attention_weight_min": -62.820533752441406,
      "activations/layer6_attention_weight_max": 44.60905075073242,
      "activations/layer6_attention_weight_min": -46.458683013916016,
      "activations/layer7_attention_weight_max": 92.6805419921875,
      "activations/layer7_attention_weight_min": -93.66856384277344,
      "activations/layer8_attention_weight_max": 41.2411994934082,
      "activations/layer8_attention_weight_min": -43.31416320800781,
      "activations/layer9_attention_weight_max": 34.83462142944336,
      "activations/layer9_attention_weight_min": -34.76276779174805,
      "epoch": 17.11,
      "learning_rate": 4.007007575757575e-05,
      "loss": 2.7282,
      "step": 294450
    },
    {
      "activations/layer0_attention_weight_max": 16.05999755859375,
      "activations/layer0_attention_weight_min": -12.044268608093262,
      "activations/layer10_attention_weight_max": 36.70180130004883,
      "activations/layer10_attention_weight_min": -35.22978591918945,
      "activations/layer11_attention_weight_max": 33.8038330078125,
      "activations/layer11_attention_weight_min": -34.34446716308594,
      "activations/layer12_attention_weight_max": 28.014991760253906,
      "activations/layer12_attention_weight_min": -26.56713104248047,
      "activations/layer13_attention_weight_max": 40.28629684448242,
      "activations/layer13_attention_weight_min": -37.24152755737305,
      "activations/layer14_attention_weight_max": 44.76418685913086,
      "activations/layer14_attention_weight_min": -40.4023551940918,
      "activations/layer15_attention_weight_max": 42.460086822509766,
      "activations/layer15_attention_weight_min": -37.36455535888672,
      "activations/layer16_attention_weight_max": 32.27049255371094,
      "activations/layer16_attention_weight_min": -29.431657791137695,
      "activations/layer17_attention_weight_max": 34.386287689208984,
      "activations/layer17_attention_weight_min": -29.970882415771484,
      "activations/layer18_attention_weight_max": 35.79991912841797,
      "activations/layer18_attention_weight_min": -28.795978546142578,
      "activations/layer19_attention_weight_max": 35.7157096862793,
      "activations/layer19_attention_weight_min": -30.905981063842773,
      "activations/layer1_attention_weight_max": 16.801191329956055,
      "activations/layer1_attention_weight_min": -15.321582794189453,
      "activations/layer20_attention_weight_max": 30.325843811035156,
      "activations/layer20_attention_weight_min": -24.601787567138672,
      "activations/layer21_attention_weight_max": 30.199705123901367,
      "activations/layer21_attention_weight_min": -25.811307907104492,
      "activations/layer22_attention_weight_max": 44.538246154785156,
      "activations/layer22_attention_weight_min": -31.72309112548828,
      "activations/layer23_attention_weight_max": 32.525733947753906,
      "activations/layer23_attention_weight_min": -25.768753051757812,
      "activations/layer2_attention_weight_max": 33.34269714355469,
      "activations/layer2_attention_weight_min": -34.96868133544922,
      "activations/layer3_attention_weight_max": 97.87800598144531,
      "activations/layer3_attention_weight_min": -102.52059936523438,
      "activations/layer4_attention_weight_max": 106.39653778076172,
      "activations/layer4_attention_weight_min": -108.18123626708984,
      "activations/layer5_attention_weight_max": 50.15154266357422,
      "activations/layer5_attention_weight_min": -63.21568298339844,
      "activations/layer6_attention_weight_max": 44.49063491821289,
      "activations/layer6_attention_weight_min": -47.59225845336914,
      "activations/layer7_attention_weight_max": 96.34906005859375,
      "activations/layer7_attention_weight_min": -95.22367858886719,
      "activations/layer8_attention_weight_max": 43.95973205566406,
      "activations/layer8_attention_weight_min": -42.913490295410156,
      "activations/layer9_attention_weight_max": 33.785430908203125,
      "activations/layer9_attention_weight_min": -36.0103874206543,
      "epoch": 17.11,
      "learning_rate": 4.0051136363636366e-05,
      "loss": 2.7185,
      "step": 294500
    },
    {
      "activations/layer0_attention_weight_max": 16.674951553344727,
      "activations/layer0_attention_weight_min": -12.37702465057373,
      "activations/layer10_attention_weight_max": 35.2969856262207,
      "activations/layer10_attention_weight_min": -32.92291259765625,
      "activations/layer11_attention_weight_max": 35.34730911254883,
      "activations/layer11_attention_weight_min": -33.59699630737305,
      "activations/layer12_attention_weight_max": 28.889060974121094,
      "activations/layer12_attention_weight_min": -26.677698135375977,
      "activations/layer13_attention_weight_max": 42.33454895019531,
      "activations/layer13_attention_weight_min": -39.54930114746094,
      "activations/layer14_attention_weight_max": 48.40655517578125,
      "activations/layer14_attention_weight_min": -41.24989700317383,
      "activations/layer15_attention_weight_max": 42.170475006103516,
      "activations/layer15_attention_weight_min": -35.16267395019531,
      "activations/layer16_attention_weight_max": 30.99833869934082,
      "activations/layer16_attention_weight_min": -27.131317138671875,
      "activations/layer17_attention_weight_max": 30.278738021850586,
      "activations/layer17_attention_weight_min": -24.828216552734375,
      "activations/layer18_attention_weight_max": 32.51856231689453,
      "activations/layer18_attention_weight_min": -24.916563034057617,
      "activations/layer19_attention_weight_max": 37.385833740234375,
      "activations/layer19_attention_weight_min": -30.140090942382812,
      "activations/layer1_attention_weight_max": 15.890630722045898,
      "activations/layer1_attention_weight_min": -14.176718711853027,
      "activations/layer20_attention_weight_max": 30.36354637145996,
      "activations/layer20_attention_weight_min": -26.259469985961914,
      "activations/layer21_attention_weight_max": 31.952667236328125,
      "activations/layer21_attention_weight_min": -27.417509078979492,
      "activations/layer22_attention_weight_max": 40.051551818847656,
      "activations/layer22_attention_weight_min": -31.53037452697754,
      "activations/layer23_attention_weight_max": 33.011592864990234,
      "activations/layer23_attention_weight_min": -24.59198760986328,
      "activations/layer2_attention_weight_max": 33.46052551269531,
      "activations/layer2_attention_weight_min": -34.54615783691406,
      "activations/layer3_attention_weight_max": 99.25619506835938,
      "activations/layer3_attention_weight_min": -98.34585571289062,
      "activations/layer4_attention_weight_max": 107.3310546875,
      "activations/layer4_attention_weight_min": -107.58349609375,
      "activations/layer5_attention_weight_max": 52.98899841308594,
      "activations/layer5_attention_weight_min": -61.0528678894043,
      "activations/layer6_attention_weight_max": 46.44194793701172,
      "activations/layer6_attention_weight_min": -47.42661666870117,
      "activations/layer7_attention_weight_max": 92.78730773925781,
      "activations/layer7_attention_weight_min": -98.28811645507812,
      "activations/layer8_attention_weight_max": 43.944095611572266,
      "activations/layer8_attention_weight_min": -43.66473388671875,
      "activations/layer9_attention_weight_max": 33.148189544677734,
      "activations/layer9_attention_weight_min": -33.56535339355469,
      "epoch": 17.12,
      "learning_rate": 4.003219696969697e-05,
      "loss": 2.7247,
      "step": 294550
    },
    {
      "activations/layer0_attention_weight_max": 15.510591506958008,
      "activations/layer0_attention_weight_min": -13.43967342376709,
      "activations/layer10_attention_weight_max": 33.271671295166016,
      "activations/layer10_attention_weight_min": -32.73478698730469,
      "activations/layer11_attention_weight_max": 32.619781494140625,
      "activations/layer11_attention_weight_min": -34.71756362915039,
      "activations/layer12_attention_weight_max": 25.095977783203125,
      "activations/layer12_attention_weight_min": -36.189857482910156,
      "activations/layer13_attention_weight_max": 36.01510238647461,
      "activations/layer13_attention_weight_min": -33.10396957397461,
      "activations/layer14_attention_weight_max": 38.404815673828125,
      "activations/layer14_attention_weight_min": -34.899696350097656,
      "activations/layer15_attention_weight_max": 35.096656799316406,
      "activations/layer15_attention_weight_min": -32.5313606262207,
      "activations/layer16_attention_weight_max": 27.17078971862793,
      "activations/layer16_attention_weight_min": -27.157669067382812,
      "activations/layer17_attention_weight_max": 28.366769790649414,
      "activations/layer17_attention_weight_min": -27.271860122680664,
      "activations/layer18_attention_weight_max": 30.48528480529785,
      "activations/layer18_attention_weight_min": -25.50251007080078,
      "activations/layer19_attention_weight_max": 35.22872543334961,
      "activations/layer19_attention_weight_min": -30.725677490234375,
      "activations/layer1_attention_weight_max": 16.891115188598633,
      "activations/layer1_attention_weight_min": -15.274660110473633,
      "activations/layer20_attention_weight_max": 27.0147762298584,
      "activations/layer20_attention_weight_min": -24.532541275024414,
      "activations/layer21_attention_weight_max": 27.293142318725586,
      "activations/layer21_attention_weight_min": -24.803991317749023,
      "activations/layer22_attention_weight_max": 38.975921630859375,
      "activations/layer22_attention_weight_min": -32.24034881591797,
      "activations/layer23_attention_weight_max": 31.95000457763672,
      "activations/layer23_attention_weight_min": -27.7581787109375,
      "activations/layer2_attention_weight_max": 32.56348419189453,
      "activations/layer2_attention_weight_min": -32.79056167602539,
      "activations/layer3_attention_weight_max": 100.42259979248047,
      "activations/layer3_attention_weight_min": -96.94731140136719,
      "activations/layer4_attention_weight_max": 105.64350891113281,
      "activations/layer4_attention_weight_min": -103.27205657958984,
      "activations/layer5_attention_weight_max": 51.629371643066406,
      "activations/layer5_attention_weight_min": -63.005393981933594,
      "activations/layer6_attention_weight_max": 43.769107818603516,
      "activations/layer6_attention_weight_min": -43.89516067504883,
      "activations/layer7_attention_weight_max": 96.15947723388672,
      "activations/layer7_attention_weight_min": -95.50833129882812,
      "activations/layer8_attention_weight_max": 39.954891204833984,
      "activations/layer8_attention_weight_min": -41.29417037963867,
      "activations/layer9_attention_weight_max": 30.192859649658203,
      "activations/layer9_attention_weight_min": -32.891971588134766,
      "epoch": 17.12,
      "learning_rate": 4.001325757575757e-05,
      "loss": 2.7203,
      "step": 294600
    },
    {
      "activations/layer0_attention_weight_max": 16.308929443359375,
      "activations/layer0_attention_weight_min": -11.848721504211426,
      "activations/layer10_attention_weight_max": 37.44636917114258,
      "activations/layer10_attention_weight_min": -35.00961685180664,
      "activations/layer11_attention_weight_max": 38.18765640258789,
      "activations/layer11_attention_weight_min": -35.47089385986328,
      "activations/layer12_attention_weight_max": 28.752403259277344,
      "activations/layer12_attention_weight_min": -26.746339797973633,
      "activations/layer13_attention_weight_max": 41.6383056640625,
      "activations/layer13_attention_weight_min": -36.74647903442383,
      "activations/layer14_attention_weight_max": 44.079105377197266,
      "activations/layer14_attention_weight_min": -39.46208572387695,
      "activations/layer15_attention_weight_max": 41.18861770629883,
      "activations/layer15_attention_weight_min": -35.475196838378906,
      "activations/layer16_attention_weight_max": 30.544355392456055,
      "activations/layer16_attention_weight_min": -28.276384353637695,
      "activations/layer17_attention_weight_max": 32.872135162353516,
      "activations/layer17_attention_weight_min": -25.900772094726562,
      "activations/layer18_attention_weight_max": 33.33133316040039,
      "activations/layer18_attention_weight_min": -26.593215942382812,
      "activations/layer19_attention_weight_max": 38.61045455932617,
      "activations/layer19_attention_weight_min": -32.7381706237793,
      "activations/layer1_attention_weight_max": 16.259572982788086,
      "activations/layer1_attention_weight_min": -14.437609672546387,
      "activations/layer20_attention_weight_max": 36.773765563964844,
      "activations/layer20_attention_weight_min": -26.76753807067871,
      "activations/layer21_attention_weight_max": 36.55937957763672,
      "activations/layer21_attention_weight_min": -25.578100204467773,
      "activations/layer22_attention_weight_max": 47.88767623901367,
      "activations/layer22_attention_weight_min": -31.842161178588867,
      "activations/layer23_attention_weight_max": 37.734230041503906,
      "activations/layer23_attention_weight_min": -27.123382568359375,
      "activations/layer2_attention_weight_max": 33.848388671875,
      "activations/layer2_attention_weight_min": -33.22032165527344,
      "activations/layer3_attention_weight_max": 94.8880386352539,
      "activations/layer3_attention_weight_min": -97.2681655883789,
      "activations/layer4_attention_weight_max": 100.5186996459961,
      "activations/layer4_attention_weight_min": -105.50308990478516,
      "activations/layer5_attention_weight_max": 48.55647277832031,
      "activations/layer5_attention_weight_min": -62.606483459472656,
      "activations/layer6_attention_weight_max": 44.79838562011719,
      "activations/layer6_attention_weight_min": -45.6716423034668,
      "activations/layer7_attention_weight_max": 94.77501678466797,
      "activations/layer7_attention_weight_min": -99.85262298583984,
      "activations/layer8_attention_weight_max": 45.85670852661133,
      "activations/layer8_attention_weight_min": -46.49457931518555,
      "activations/layer9_attention_weight_max": 34.76768112182617,
      "activations/layer9_attention_weight_min": -35.658870697021484,
      "epoch": 17.12,
      "learning_rate": 3.999431818181818e-05,
      "loss": 2.7418,
      "step": 294650
    },
    {
      "activations/layer0_attention_weight_max": 16.84218978881836,
      "activations/layer0_attention_weight_min": -12.828788757324219,
      "activations/layer10_attention_weight_max": 36.50428771972656,
      "activations/layer10_attention_weight_min": -36.43280792236328,
      "activations/layer11_attention_weight_max": 37.24458312988281,
      "activations/layer11_attention_weight_min": -37.15354919433594,
      "activations/layer12_attention_weight_max": 29.41956901550293,
      "activations/layer12_attention_weight_min": -26.78986358642578,
      "activations/layer13_attention_weight_max": 39.818031311035156,
      "activations/layer13_attention_weight_min": -38.058067321777344,
      "activations/layer14_attention_weight_max": 44.118797302246094,
      "activations/layer14_attention_weight_min": -38.39548873901367,
      "activations/layer15_attention_weight_max": 38.081939697265625,
      "activations/layer15_attention_weight_min": -36.5332145690918,
      "activations/layer16_attention_weight_max": 28.009111404418945,
      "activations/layer16_attention_weight_min": -29.40155601501465,
      "activations/layer17_attention_weight_max": 28.29856300354004,
      "activations/layer17_attention_weight_min": -26.242963790893555,
      "activations/layer18_attention_weight_max": 33.432289123535156,
      "activations/layer18_attention_weight_min": -25.482666015625,
      "activations/layer19_attention_weight_max": 34.49453353881836,
      "activations/layer19_attention_weight_min": -30.98489761352539,
      "activations/layer1_attention_weight_max": 16.323007583618164,
      "activations/layer1_attention_weight_min": -14.603959083557129,
      "activations/layer20_attention_weight_max": 29.074472427368164,
      "activations/layer20_attention_weight_min": -24.90884017944336,
      "activations/layer21_attention_weight_max": 29.701730728149414,
      "activations/layer21_attention_weight_min": -24.4105167388916,
      "activations/layer22_attention_weight_max": 46.04233932495117,
      "activations/layer22_attention_weight_min": -34.11830139160156,
      "activations/layer23_attention_weight_max": 33.73576736450195,
      "activations/layer23_attention_weight_min": -24.982677459716797,
      "activations/layer2_attention_weight_max": 32.497528076171875,
      "activations/layer2_attention_weight_min": -33.8187255859375,
      "activations/layer3_attention_weight_max": 96.79528045654297,
      "activations/layer3_attention_weight_min": -98.46497344970703,
      "activations/layer4_attention_weight_max": 106.63966369628906,
      "activations/layer4_attention_weight_min": -107.87762451171875,
      "activations/layer5_attention_weight_max": 53.678199768066406,
      "activations/layer5_attention_weight_min": -62.75869369506836,
      "activations/layer6_attention_weight_max": 45.33015060424805,
      "activations/layer6_attention_weight_min": -47.14048385620117,
      "activations/layer7_attention_weight_max": 102.2687759399414,
      "activations/layer7_attention_weight_min": -103.48907470703125,
      "activations/layer8_attention_weight_max": 42.857269287109375,
      "activations/layer8_attention_weight_min": -47.55470657348633,
      "activations/layer9_attention_weight_max": 33.95109939575195,
      "activations/layer9_attention_weight_min": -35.77178192138672,
      "epoch": 17.12,
      "learning_rate": 3.9975378787878785e-05,
      "loss": 2.7237,
      "step": 294700
    },
    {
      "activations/layer0_attention_weight_max": 16.35816192626953,
      "activations/layer0_attention_weight_min": -13.735527038574219,
      "activations/layer10_attention_weight_max": 35.65330123901367,
      "activations/layer10_attention_weight_min": -35.60582733154297,
      "activations/layer11_attention_weight_max": 34.99364471435547,
      "activations/layer11_attention_weight_min": -36.617759704589844,
      "activations/layer12_attention_weight_max": 29.899755477905273,
      "activations/layer12_attention_weight_min": -28.687807083129883,
      "activations/layer13_attention_weight_max": 42.029014587402344,
      "activations/layer13_attention_weight_min": -36.961524963378906,
      "activations/layer14_attention_weight_max": 47.40195846557617,
      "activations/layer14_attention_weight_min": -39.97342300415039,
      "activations/layer15_attention_weight_max": 45.14164352416992,
      "activations/layer15_attention_weight_min": -43.40740966796875,
      "activations/layer16_attention_weight_max": 35.44647979736328,
      "activations/layer16_attention_weight_min": -28.56515884399414,
      "activations/layer17_attention_weight_max": 34.12605285644531,
      "activations/layer17_attention_weight_min": -28.27815055847168,
      "activations/layer18_attention_weight_max": 33.9963264465332,
      "activations/layer18_attention_weight_min": -27.4180965423584,
      "activations/layer19_attention_weight_max": 36.2277946472168,
      "activations/layer19_attention_weight_min": -32.124202728271484,
      "activations/layer1_attention_weight_max": 15.925113677978516,
      "activations/layer1_attention_weight_min": -15.098217964172363,
      "activations/layer20_attention_weight_max": 29.779003143310547,
      "activations/layer20_attention_weight_min": -26.877962112426758,
      "activations/layer21_attention_weight_max": 29.049631118774414,
      "activations/layer21_attention_weight_min": -24.243114471435547,
      "activations/layer22_attention_weight_max": 42.96918869018555,
      "activations/layer22_attention_weight_min": -33.49280548095703,
      "activations/layer23_attention_weight_max": 32.51342010498047,
      "activations/layer23_attention_weight_min": -24.480907440185547,
      "activations/layer2_attention_weight_max": 31.470678329467773,
      "activations/layer2_attention_weight_min": -31.852161407470703,
      "activations/layer3_attention_weight_max": 95.01695251464844,
      "activations/layer3_attention_weight_min": -96.30555725097656,
      "activations/layer4_attention_weight_max": 100.48433685302734,
      "activations/layer4_attention_weight_min": -106.2215347290039,
      "activations/layer5_attention_weight_max": 48.476375579833984,
      "activations/layer5_attention_weight_min": -61.7802734375,
      "activations/layer6_attention_weight_max": 43.77760696411133,
      "activations/layer6_attention_weight_min": -44.6695442199707,
      "activations/layer7_attention_weight_max": 88.88187408447266,
      "activations/layer7_attention_weight_min": -92.9535140991211,
      "activations/layer8_attention_weight_max": 45.078697204589844,
      "activations/layer8_attention_weight_min": -44.44597625732422,
      "activations/layer9_attention_weight_max": 33.49565887451172,
      "activations/layer9_attention_weight_min": -35.2053108215332,
      "epoch": 17.13,
      "learning_rate": 3.9956439393939393e-05,
      "loss": 2.7298,
      "step": 294750
    },
    {
      "activations/layer0_attention_weight_max": 16.34943199157715,
      "activations/layer0_attention_weight_min": -13.36961841583252,
      "activations/layer10_attention_weight_max": 31.849679946899414,
      "activations/layer10_attention_weight_min": -32.61091995239258,
      "activations/layer11_attention_weight_max": 31.397565841674805,
      "activations/layer11_attention_weight_min": -31.281646728515625,
      "activations/layer12_attention_weight_max": 24.7487735748291,
      "activations/layer12_attention_weight_min": -26.310518264770508,
      "activations/layer13_attention_weight_max": 37.946189880371094,
      "activations/layer13_attention_weight_min": -32.730220794677734,
      "activations/layer14_attention_weight_max": 39.58951950073242,
      "activations/layer14_attention_weight_min": -35.15898895263672,
      "activations/layer15_attention_weight_max": 34.90745544433594,
      "activations/layer15_attention_weight_min": -33.79912185668945,
      "activations/layer16_attention_weight_max": 29.858539581298828,
      "activations/layer16_attention_weight_min": -28.945999145507812,
      "activations/layer17_attention_weight_max": 29.417871475219727,
      "activations/layer17_attention_weight_min": -25.900556564331055,
      "activations/layer18_attention_weight_max": 29.91486167907715,
      "activations/layer18_attention_weight_min": -25.758745193481445,
      "activations/layer19_attention_weight_max": 33.09136199951172,
      "activations/layer19_attention_weight_min": -31.094324111938477,
      "activations/layer1_attention_weight_max": 16.153837203979492,
      "activations/layer1_attention_weight_min": -14.03892707824707,
      "activations/layer20_attention_weight_max": 27.571943283081055,
      "activations/layer20_attention_weight_min": -24.871990203857422,
      "activations/layer21_attention_weight_max": 26.657108306884766,
      "activations/layer21_attention_weight_min": -24.290910720825195,
      "activations/layer22_attention_weight_max": 36.43867111206055,
      "activations/layer22_attention_weight_min": -30.716447830200195,
      "activations/layer23_attention_weight_max": 31.345800399780273,
      "activations/layer23_attention_weight_min": -27.026777267456055,
      "activations/layer2_attention_weight_max": 32.44529724121094,
      "activations/layer2_attention_weight_min": -35.00980758666992,
      "activations/layer3_attention_weight_max": 100.07090759277344,
      "activations/layer3_attention_weight_min": -104.08241271972656,
      "activations/layer4_attention_weight_max": 105.52108001708984,
      "activations/layer4_attention_weight_min": -107.23529052734375,
      "activations/layer5_attention_weight_max": 48.57769775390625,
      "activations/layer5_attention_weight_min": -57.55570983886719,
      "activations/layer6_attention_weight_max": 41.48727035522461,
      "activations/layer6_attention_weight_min": -43.58931350708008,
      "activations/layer7_attention_weight_max": 92.23808288574219,
      "activations/layer7_attention_weight_min": -90.79313659667969,
      "activations/layer8_attention_weight_max": 41.13490676879883,
      "activations/layer8_attention_weight_min": -42.222347259521484,
      "activations/layer9_attention_weight_max": 32.53212356567383,
      "activations/layer9_attention_weight_min": -32.75234603881836,
      "epoch": 17.13,
      "learning_rate": 3.9937499999999995e-05,
      "loss": 2.74,
      "step": 294800
    },
    {
      "activations/layer0_attention_weight_max": 15.965046882629395,
      "activations/layer0_attention_weight_min": -12.066055297851562,
      "activations/layer10_attention_weight_max": 34.396217346191406,
      "activations/layer10_attention_weight_min": -35.15653991699219,
      "activations/layer11_attention_weight_max": 34.89046096801758,
      "activations/layer11_attention_weight_min": -34.3996696472168,
      "activations/layer12_attention_weight_max": 29.386125564575195,
      "activations/layer12_attention_weight_min": -26.864885330200195,
      "activations/layer13_attention_weight_max": 42.36275100708008,
      "activations/layer13_attention_weight_min": -35.2194938659668,
      "activations/layer14_attention_weight_max": 49.635581970214844,
      "activations/layer14_attention_weight_min": -37.0389289855957,
      "activations/layer15_attention_weight_max": 40.529808044433594,
      "activations/layer15_attention_weight_min": -35.94517135620117,
      "activations/layer16_attention_weight_max": 35.411746978759766,
      "activations/layer16_attention_weight_min": -28.694828033447266,
      "activations/layer17_attention_weight_max": 31.25065803527832,
      "activations/layer17_attention_weight_min": -27.730045318603516,
      "activations/layer18_attention_weight_max": 33.32768249511719,
      "activations/layer18_attention_weight_min": -25.608253479003906,
      "activations/layer19_attention_weight_max": 38.98542785644531,
      "activations/layer19_attention_weight_min": -32.8006706237793,
      "activations/layer1_attention_weight_max": 15.639871597290039,
      "activations/layer1_attention_weight_min": -15.120330810546875,
      "activations/layer20_attention_weight_max": 34.10490036010742,
      "activations/layer20_attention_weight_min": -26.389848709106445,
      "activations/layer21_attention_weight_max": 34.51753616333008,
      "activations/layer21_attention_weight_min": -25.883026123046875,
      "activations/layer22_attention_weight_max": 55.861785888671875,
      "activations/layer22_attention_weight_min": -32.08304977416992,
      "activations/layer23_attention_weight_max": 38.000701904296875,
      "activations/layer23_attention_weight_min": -26.352134704589844,
      "activations/layer2_attention_weight_max": 34.73219299316406,
      "activations/layer2_attention_weight_min": -33.77172088623047,
      "activations/layer3_attention_weight_max": 102.46321868896484,
      "activations/layer3_attention_weight_min": -103.5792007446289,
      "activations/layer4_attention_weight_max": 108.38138580322266,
      "activations/layer4_attention_weight_min": -107.59626770019531,
      "activations/layer5_attention_weight_max": 52.46574401855469,
      "activations/layer5_attention_weight_min": -61.186012268066406,
      "activations/layer6_attention_weight_max": 45.14452362060547,
      "activations/layer6_attention_weight_min": -47.05516815185547,
      "activations/layer7_attention_weight_max": 96.8455810546875,
      "activations/layer7_attention_weight_min": -98.46589660644531,
      "activations/layer8_attention_weight_max": 42.1685676574707,
      "activations/layer8_attention_weight_min": -43.9694709777832,
      "activations/layer9_attention_weight_max": 33.6606559753418,
      "activations/layer9_attention_weight_min": -33.36743927001953,
      "epoch": 17.13,
      "learning_rate": 3.991856060606061e-05,
      "loss": 2.7255,
      "step": 294850
    },
    {
      "activations/layer0_attention_weight_max": 16.265541076660156,
      "activations/layer0_attention_weight_min": -11.752670288085938,
      "activations/layer10_attention_weight_max": 35.19782638549805,
      "activations/layer10_attention_weight_min": -34.93759536743164,
      "activations/layer11_attention_weight_max": 35.210086822509766,
      "activations/layer11_attention_weight_min": -35.247379302978516,
      "activations/layer12_attention_weight_max": 28.23124122619629,
      "activations/layer12_attention_weight_min": -25.738969802856445,
      "activations/layer13_attention_weight_max": 44.474491119384766,
      "activations/layer13_attention_weight_min": -35.76906967163086,
      "activations/layer14_attention_weight_max": 46.94599151611328,
      "activations/layer14_attention_weight_min": -39.65404510498047,
      "activations/layer15_attention_weight_max": 43.11936950683594,
      "activations/layer15_attention_weight_min": -36.56945037841797,
      "activations/layer16_attention_weight_max": 30.609851837158203,
      "activations/layer16_attention_weight_min": -29.419429779052734,
      "activations/layer17_attention_weight_max": 30.18626594543457,
      "activations/layer17_attention_weight_min": -27.69635581970215,
      "activations/layer18_attention_weight_max": 32.53158950805664,
      "activations/layer18_attention_weight_min": -30.64618492126465,
      "activations/layer19_attention_weight_max": 34.20832824707031,
      "activations/layer19_attention_weight_min": -35.63226318359375,
      "activations/layer1_attention_weight_max": 16.273517608642578,
      "activations/layer1_attention_weight_min": -16.36288070678711,
      "activations/layer20_attention_weight_max": 28.195329666137695,
      "activations/layer20_attention_weight_min": -27.745248794555664,
      "activations/layer21_attention_weight_max": 26.755107879638672,
      "activations/layer21_attention_weight_min": -26.358983993530273,
      "activations/layer22_attention_weight_max": 43.121028900146484,
      "activations/layer22_attention_weight_min": -34.02410125732422,
      "activations/layer23_attention_weight_max": 30.431509017944336,
      "activations/layer23_attention_weight_min": -27.361984252929688,
      "activations/layer2_attention_weight_max": 36.302223205566406,
      "activations/layer2_attention_weight_min": -32.98267364501953,
      "activations/layer3_attention_weight_max": 103.28728485107422,
      "activations/layer3_attention_weight_min": -105.4865493774414,
      "activations/layer4_attention_weight_max": 112.15040588378906,
      "activations/layer4_attention_weight_min": -108.06391906738281,
      "activations/layer5_attention_weight_max": 50.30134201049805,
      "activations/layer5_attention_weight_min": -60.31013488769531,
      "activations/layer6_attention_weight_max": 45.88385772705078,
      "activations/layer6_attention_weight_min": -46.334407806396484,
      "activations/layer7_attention_weight_max": 94.2022705078125,
      "activations/layer7_attention_weight_min": -95.70507049560547,
      "activations/layer8_attention_weight_max": 42.106204986572266,
      "activations/layer8_attention_weight_min": -43.55836486816406,
      "activations/layer9_attention_weight_max": 33.93040466308594,
      "activations/layer9_attention_weight_min": -33.684322357177734,
      "epoch": 17.14,
      "learning_rate": 3.989962121212121e-05,
      "loss": 2.7477,
      "step": 294900
    },
    {
      "activations/layer0_attention_weight_max": 16.314292907714844,
      "activations/layer0_attention_weight_min": -14.43883228302002,
      "activations/layer10_attention_weight_max": 34.51637268066406,
      "activations/layer10_attention_weight_min": -34.94995880126953,
      "activations/layer11_attention_weight_max": 33.666778564453125,
      "activations/layer11_attention_weight_min": -38.98350524902344,
      "activations/layer12_attention_weight_max": 24.837732315063477,
      "activations/layer12_attention_weight_min": -26.186952590942383,
      "activations/layer13_attention_weight_max": 37.68657302856445,
      "activations/layer13_attention_weight_min": -35.43317413330078,
      "activations/layer14_attention_weight_max": 39.101768493652344,
      "activations/layer14_attention_weight_min": -36.26079177856445,
      "activations/layer15_attention_weight_max": 37.33953094482422,
      "activations/layer15_attention_weight_min": -34.320404052734375,
      "activations/layer16_attention_weight_max": 26.14353370666504,
      "activations/layer16_attention_weight_min": -27.8778018951416,
      "activations/layer17_attention_weight_max": 27.856698989868164,
      "activations/layer17_attention_weight_min": -25.226165771484375,
      "activations/layer18_attention_weight_max": 31.25579261779785,
      "activations/layer18_attention_weight_min": -26.049224853515625,
      "activations/layer19_attention_weight_max": 36.53847885131836,
      "activations/layer19_attention_weight_min": -30.115238189697266,
      "activations/layer1_attention_weight_max": 16.938573837280273,
      "activations/layer1_attention_weight_min": -14.625494956970215,
      "activations/layer20_attention_weight_max": 27.771045684814453,
      "activations/layer20_attention_weight_min": -23.485591888427734,
      "activations/layer21_attention_weight_max": 26.805553436279297,
      "activations/layer21_attention_weight_min": -25.384050369262695,
      "activations/layer22_attention_weight_max": 38.081398010253906,
      "activations/layer22_attention_weight_min": -28.42323875427246,
      "activations/layer23_attention_weight_max": 37.31053161621094,
      "activations/layer23_attention_weight_min": -23.61273765563965,
      "activations/layer2_attention_weight_max": 34.161285400390625,
      "activations/layer2_attention_weight_min": -35.92023468017578,
      "activations/layer3_attention_weight_max": 97.23139190673828,
      "activations/layer3_attention_weight_min": -101.1717529296875,
      "activations/layer4_attention_weight_max": 102.52165985107422,
      "activations/layer4_attention_weight_min": -102.00847625732422,
      "activations/layer5_attention_weight_max": 49.4652099609375,
      "activations/layer5_attention_weight_min": -59.386451721191406,
      "activations/layer6_attention_weight_max": 42.750587463378906,
      "activations/layer6_attention_weight_min": -47.17253875732422,
      "activations/layer7_attention_weight_max": 87.03189086914062,
      "activations/layer7_attention_weight_min": -90.08663177490234,
      "activations/layer8_attention_weight_max": 41.26979446411133,
      "activations/layer8_attention_weight_min": -45.34661865234375,
      "activations/layer9_attention_weight_max": 31.698184967041016,
      "activations/layer9_attention_weight_min": -35.667354583740234,
      "epoch": 17.14,
      "learning_rate": 3.988068181818181e-05,
      "loss": 2.7274,
      "step": 294950
    },
    {
      "activations/layer0_attention_weight_max": 16.438112258911133,
      "activations/layer0_attention_weight_min": -12.85696792602539,
      "activations/layer10_attention_weight_max": 38.64039993286133,
      "activations/layer10_attention_weight_min": -38.381378173828125,
      "activations/layer11_attention_weight_max": 42.60402297973633,
      "activations/layer11_attention_weight_min": -39.55470275878906,
      "activations/layer12_attention_weight_max": 29.100557327270508,
      "activations/layer12_attention_weight_min": -27.513294219970703,
      "activations/layer13_attention_weight_max": 41.08233642578125,
      "activations/layer13_attention_weight_min": -36.90681076049805,
      "activations/layer14_attention_weight_max": 48.516998291015625,
      "activations/layer14_attention_weight_min": -39.54252624511719,
      "activations/layer15_attention_weight_max": 46.5748291015625,
      "activations/layer15_attention_weight_min": -39.5587158203125,
      "activations/layer16_attention_weight_max": 34.697269439697266,
      "activations/layer16_attention_weight_min": -28.171588897705078,
      "activations/layer17_attention_weight_max": 33.0910758972168,
      "activations/layer17_attention_weight_min": -27.415483474731445,
      "activations/layer18_attention_weight_max": 30.20807456970215,
      "activations/layer18_attention_weight_min": -25.647319793701172,
      "activations/layer19_attention_weight_max": 38.17247009277344,
      "activations/layer19_attention_weight_min": -34.2291145324707,
      "activations/layer1_attention_weight_max": 16.789920806884766,
      "activations/layer1_attention_weight_min": -15.554754257202148,
      "activations/layer20_attention_weight_max": 34.13170623779297,
      "activations/layer20_attention_weight_min": -25.414836883544922,
      "activations/layer21_attention_weight_max": 33.2602424621582,
      "activations/layer21_attention_weight_min": -23.641277313232422,
      "activations/layer22_attention_weight_max": 46.743568420410156,
      "activations/layer22_attention_weight_min": -30.855207443237305,
      "activations/layer23_attention_weight_max": 36.8779296875,
      "activations/layer23_attention_weight_min": -24.371196746826172,
      "activations/layer2_attention_weight_max": 36.99623107910156,
      "activations/layer2_attention_weight_min": -35.50822830200195,
      "activations/layer3_attention_weight_max": 102.5906982421875,
      "activations/layer3_attention_weight_min": -104.35477447509766,
      "activations/layer4_attention_weight_max": 108.4543228149414,
      "activations/layer4_attention_weight_min": -107.12259674072266,
      "activations/layer5_attention_weight_max": 50.06553649902344,
      "activations/layer5_attention_weight_min": -60.09132385253906,
      "activations/layer6_attention_weight_max": 45.38438034057617,
      "activations/layer6_attention_weight_min": -47.23811340332031,
      "activations/layer7_attention_weight_max": 101.79013061523438,
      "activations/layer7_attention_weight_min": -100.8387451171875,
      "activations/layer8_attention_weight_max": 49.28572463989258,
      "activations/layer8_attention_weight_min": -49.43008804321289,
      "activations/layer9_attention_weight_max": 36.17839431762695,
      "activations/layer9_attention_weight_min": -37.889495849609375,
      "epoch": 17.14,
      "learning_rate": 3.986174242424242e-05,
      "loss": 2.726,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4637,
      "eval_samples_per_second": 507.345,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4637,
      "eval_openwebtext_samples_per_second": 507.345,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9521,
      "eval_wikitext_samples_per_second": 233.59,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_lambada_loss": 2.375,
      "eval_lambada_ppl": 10.751013186076355,
      "eval_lambada_runtime": 9.5451,
      "eval_lambada_samples_per_second": 510.106,
      "step": 295000
    },
    {
      "activations/layer0_attention_weight_max": 16.148099899291992,
      "activations/layer0_attention_weight_min": -12.369800567626953,
      "activations/layer10_attention_weight_max": 36.13427734375,
      "activations/layer10_attention_weight_min": -35.5362434387207,
      "activations/layer11_attention_weight_max": 34.81977081298828,
      "activations/layer11_attention_weight_min": -36.07845687866211,
      "activations/layer12_attention_weight_max": 28.366334915161133,
      "activations/layer12_attention_weight_min": -29.4715518951416,
      "activations/layer13_attention_weight_max": 42.87979507446289,
      "activations/layer13_attention_weight_min": -41.3436393737793,
      "activations/layer14_attention_weight_max": 46.12913513183594,
      "activations/layer14_attention_weight_min": -44.14290237426758,
      "activations/layer15_attention_weight_max": 42.880653381347656,
      "activations/layer15_attention_weight_min": -39.01689147949219,
      "activations/layer16_attention_weight_max": 29.47261619567871,
      "activations/layer16_attention_weight_min": -27.19170570373535,
      "activations/layer17_attention_weight_max": 33.41037368774414,
      "activations/layer17_attention_weight_min": -25.514860153198242,
      "activations/layer18_attention_weight_max": 32.55779266357422,
      "activations/layer18_attention_weight_min": -27.100971221923828,
      "activations/layer19_attention_weight_max": 33.70359802246094,
      "activations/layer19_attention_weight_min": -30.56155776977539,
      "activations/layer1_attention_weight_max": 16.500652313232422,
      "activations/layer1_attention_weight_min": -14.235851287841797,
      "activations/layer20_attention_weight_max": 31.165632247924805,
      "activations/layer20_attention_weight_min": -24.089862823486328,
      "activations/layer21_attention_weight_max": 30.898038864135742,
      "activations/layer21_attention_weight_min": -25.07036590576172,
      "activations/layer22_attention_weight_max": 40.121707916259766,
      "activations/layer22_attention_weight_min": -30.15390396118164,
      "activations/layer23_attention_weight_max": 31.346914291381836,
      "activations/layer23_attention_weight_min": -25.52467918395996,
      "activations/layer2_attention_weight_max": 33.44013977050781,
      "activations/layer2_attention_weight_min": -33.4281120300293,
      "activations/layer3_attention_weight_max": 100.28534698486328,
      "activations/layer3_attention_weight_min": -100.38475799560547,
      "activations/layer4_attention_weight_max": 106.58843994140625,
      "activations/layer4_attention_weight_min": -109.83428955078125,
      "activations/layer5_attention_weight_max": 54.083587646484375,
      "activations/layer5_attention_weight_min": -60.960784912109375,
      "activations/layer6_attention_weight_max": 43.65097427368164,
      "activations/layer6_attention_weight_min": -47.83047103881836,
      "activations/layer7_attention_weight_max": 97.08543395996094,
      "activations/layer7_attention_weight_min": -106.36375427246094,
      "activations/layer8_attention_weight_max": 41.8614501953125,
      "activations/layer8_attention_weight_min": -45.93216323852539,
      "activations/layer9_attention_weight_max": 34.110164642333984,
      "activations/layer9_attention_weight_min": -35.23003005981445,
      "epoch": 17.14,
      "learning_rate": 3.984280303030303e-05,
      "loss": 2.7275,
      "step": 295050
    },
    {
      "activations/layer0_attention_weight_max": 16.207590103149414,
      "activations/layer0_attention_weight_min": -14.633665084838867,
      "activations/layer10_attention_weight_max": 38.02897262573242,
      "activations/layer10_attention_weight_min": -35.76984786987305,
      "activations/layer11_attention_weight_max": 36.42033767700195,
      "activations/layer11_attention_weight_min": -34.73472213745117,
      "activations/layer12_attention_weight_max": 27.515169143676758,
      "activations/layer12_attention_weight_min": -27.531896591186523,
      "activations/layer13_attention_weight_max": 42.75011444091797,
      "activations/layer13_attention_weight_min": -35.12401580810547,
      "activations/layer14_attention_weight_max": 46.813011169433594,
      "activations/layer14_attention_weight_min": -38.49354553222656,
      "activations/layer15_attention_weight_max": 40.0274658203125,
      "activations/layer15_attention_weight_min": -34.733219146728516,
      "activations/layer16_attention_weight_max": 32.613670349121094,
      "activations/layer16_attention_weight_min": -28.968637466430664,
      "activations/layer17_attention_weight_max": 30.027286529541016,
      "activations/layer17_attention_weight_min": -27.185510635375977,
      "activations/layer18_attention_weight_max": 32.84626007080078,
      "activations/layer18_attention_weight_min": -26.7489070892334,
      "activations/layer19_attention_weight_max": 36.80746078491211,
      "activations/layer19_attention_weight_min": -30.353260040283203,
      "activations/layer1_attention_weight_max": 16.32890510559082,
      "activations/layer1_attention_weight_min": -15.27908706665039,
      "activations/layer20_attention_weight_max": 30.468799591064453,
      "activations/layer20_attention_weight_min": -26.66402244567871,
      "activations/layer21_attention_weight_max": 33.86023712158203,
      "activations/layer21_attention_weight_min": -25.750577926635742,
      "activations/layer22_attention_weight_max": 47.01377868652344,
      "activations/layer22_attention_weight_min": -31.49835205078125,
      "activations/layer23_attention_weight_max": 33.44228744506836,
      "activations/layer23_attention_weight_min": -26.03975486755371,
      "activations/layer2_attention_weight_max": 33.36914825439453,
      "activations/layer2_attention_weight_min": -33.63705062866211,
      "activations/layer3_attention_weight_max": 103.46216583251953,
      "activations/layer3_attention_weight_min": -99.43822479248047,
      "activations/layer4_attention_weight_max": 108.78252410888672,
      "activations/layer4_attention_weight_min": -110.46473693847656,
      "activations/layer5_attention_weight_max": 53.447959899902344,
      "activations/layer5_attention_weight_min": -61.51186752319336,
      "activations/layer6_attention_weight_max": 46.45030212402344,
      "activations/layer6_attention_weight_min": -46.817630767822266,
      "activations/layer7_attention_weight_max": 101.49717712402344,
      "activations/layer7_attention_weight_min": -97.10810852050781,
      "activations/layer8_attention_weight_max": 45.01543426513672,
      "activations/layer8_attention_weight_min": -44.321861267089844,
      "activations/layer9_attention_weight_max": 34.714229583740234,
      "activations/layer9_attention_weight_min": -35.338340759277344,
      "epoch": 17.15,
      "learning_rate": 3.982386363636363e-05,
      "loss": 2.7408,
      "step": 295100
    },
    {
      "activations/layer0_attention_weight_max": 16.264570236206055,
      "activations/layer0_attention_weight_min": -12.097005844116211,
      "activations/layer10_attention_weight_max": 31.276988983154297,
      "activations/layer10_attention_weight_min": -32.81597137451172,
      "activations/layer11_attention_weight_max": 31.542797088623047,
      "activations/layer11_attention_weight_min": -32.30495834350586,
      "activations/layer12_attention_weight_max": 28.16337776184082,
      "activations/layer12_attention_weight_min": -27.026212692260742,
      "activations/layer13_attention_weight_max": 40.02783966064453,
      "activations/layer13_attention_weight_min": -37.2673454284668,
      "activations/layer14_attention_weight_max": 42.308570861816406,
      "activations/layer14_attention_weight_min": -38.52848434448242,
      "activations/layer15_attention_weight_max": 39.057289123535156,
      "activations/layer15_attention_weight_min": -34.86521911621094,
      "activations/layer16_attention_weight_max": 29.480619430541992,
      "activations/layer16_attention_weight_min": -27.06546974182129,
      "activations/layer17_attention_weight_max": 29.119049072265625,
      "activations/layer17_attention_weight_min": -26.013193130493164,
      "activations/layer18_attention_weight_max": 31.09419059753418,
      "activations/layer18_attention_weight_min": -25.159093856811523,
      "activations/layer19_attention_weight_max": 32.19063949584961,
      "activations/layer19_attention_weight_min": -31.647802352905273,
      "activations/layer1_attention_weight_max": 15.67984676361084,
      "activations/layer1_attention_weight_min": -14.402280807495117,
      "activations/layer20_attention_weight_max": 28.141212463378906,
      "activations/layer20_attention_weight_min": -24.251487731933594,
      "activations/layer21_attention_weight_max": 27.03527069091797,
      "activations/layer21_attention_weight_min": -23.572845458984375,
      "activations/layer22_attention_weight_max": 38.44729232788086,
      "activations/layer22_attention_weight_min": -29.468326568603516,
      "activations/layer23_attention_weight_max": 32.12898254394531,
      "activations/layer23_attention_weight_min": -24.732608795166016,
      "activations/layer2_attention_weight_max": 32.58609390258789,
      "activations/layer2_attention_weight_min": -33.18315124511719,
      "activations/layer3_attention_weight_max": 97.9106674194336,
      "activations/layer3_attention_weight_min": -96.8033676147461,
      "activations/layer4_attention_weight_max": 106.4761962890625,
      "activations/layer4_attention_weight_min": -106.47911834716797,
      "activations/layer5_attention_weight_max": 54.15276336669922,
      "activations/layer5_attention_weight_min": -61.789337158203125,
      "activations/layer6_attention_weight_max": 43.0120964050293,
      "activations/layer6_attention_weight_min": -46.71235656738281,
      "activations/layer7_attention_weight_max": 92.64633178710938,
      "activations/layer7_attention_weight_min": -91.94503784179688,
      "activations/layer8_attention_weight_max": 39.11253356933594,
      "activations/layer8_attention_weight_min": -42.30926513671875,
      "activations/layer9_attention_weight_max": 30.14409828186035,
      "activations/layer9_attention_weight_min": -33.427120208740234,
      "epoch": 17.15,
      "learning_rate": 3.980492424242424e-05,
      "loss": 2.7332,
      "step": 295150
    },
    {
      "activations/layer0_attention_weight_max": 16.161474227905273,
      "activations/layer0_attention_weight_min": -13.770779609680176,
      "activations/layer10_attention_weight_max": 34.21516799926758,
      "activations/layer10_attention_weight_min": -34.08134841918945,
      "activations/layer11_attention_weight_max": 33.101844787597656,
      "activations/layer11_attention_weight_min": -32.954017639160156,
      "activations/layer12_attention_weight_max": 25.929012298583984,
      "activations/layer12_attention_weight_min": -27.097867965698242,
      "activations/layer13_attention_weight_max": 39.3786735534668,
      "activations/layer13_attention_weight_min": -35.42295455932617,
      "activations/layer14_attention_weight_max": 39.81574630737305,
      "activations/layer14_attention_weight_min": -36.92533493041992,
      "activations/layer15_attention_weight_max": 37.48214340209961,
      "activations/layer15_attention_weight_min": -33.38100051879883,
      "activations/layer16_attention_weight_max": 27.744808197021484,
      "activations/layer16_attention_weight_min": -26.69771385192871,
      "activations/layer17_attention_weight_max": 30.422027587890625,
      "activations/layer17_attention_weight_min": -25.3448486328125,
      "activations/layer18_attention_weight_max": 34.5368537902832,
      "activations/layer18_attention_weight_min": -24.70893096923828,
      "activations/layer19_attention_weight_max": 32.813636779785156,
      "activations/layer19_attention_weight_min": -31.050125122070312,
      "activations/layer1_attention_weight_max": 16.000242233276367,
      "activations/layer1_attention_weight_min": -14.489994049072266,
      "activations/layer20_attention_weight_max": 28.8486270904541,
      "activations/layer20_attention_weight_min": -25.171308517456055,
      "activations/layer21_attention_weight_max": 28.272207260131836,
      "activations/layer21_attention_weight_min": -23.753721237182617,
      "activations/layer22_attention_weight_max": 42.39918899536133,
      "activations/layer22_attention_weight_min": -30.30408477783203,
      "activations/layer23_attention_weight_max": 33.02296447753906,
      "activations/layer23_attention_weight_min": -24.231388092041016,
      "activations/layer2_attention_weight_max": 32.70357894897461,
      "activations/layer2_attention_weight_min": -32.98250961303711,
      "activations/layer3_attention_weight_max": 100.24178314208984,
      "activations/layer3_attention_weight_min": -97.50736236572266,
      "activations/layer4_attention_weight_max": 105.6312026977539,
      "activations/layer4_attention_weight_min": -107.66246795654297,
      "activations/layer5_attention_weight_max": 49.19608688354492,
      "activations/layer5_attention_weight_min": -61.971405029296875,
      "activations/layer6_attention_weight_max": 44.01914978027344,
      "activations/layer6_attention_weight_min": -47.41069793701172,
      "activations/layer7_attention_weight_max": 92.16557312011719,
      "activations/layer7_attention_weight_min": -94.60302734375,
      "activations/layer8_attention_weight_max": 42.87528991699219,
      "activations/layer8_attention_weight_min": -43.33953857421875,
      "activations/layer9_attention_weight_max": 33.081146240234375,
      "activations/layer9_attention_weight_min": -34.8775634765625,
      "epoch": 17.15,
      "learning_rate": 3.978598484848484e-05,
      "loss": 2.7368,
      "step": 295200
    },
    {
      "activations/layer0_attention_weight_max": 15.872427940368652,
      "activations/layer0_attention_weight_min": -12.739750862121582,
      "activations/layer10_attention_weight_max": 32.304569244384766,
      "activations/layer10_attention_weight_min": -34.82405471801758,
      "activations/layer11_attention_weight_max": 32.165802001953125,
      "activations/layer11_attention_weight_min": -35.69258117675781,
      "activations/layer12_attention_weight_max": 26.158145904541016,
      "activations/layer12_attention_weight_min": -25.344221115112305,
      "activations/layer13_attention_weight_max": 35.56599807739258,
      "activations/layer13_attention_weight_min": -35.222259521484375,
      "activations/layer14_attention_weight_max": 40.97245788574219,
      "activations/layer14_attention_weight_min": -36.17082595825195,
      "activations/layer15_attention_weight_max": 37.914024353027344,
      "activations/layer15_attention_weight_min": -35.22820281982422,
      "activations/layer16_attention_weight_max": 29.118331909179688,
      "activations/layer16_attention_weight_min": -28.575069427490234,
      "activations/layer17_attention_weight_max": 32.9678840637207,
      "activations/layer17_attention_weight_min": -28.02703857421875,
      "activations/layer18_attention_weight_max": 32.90839767456055,
      "activations/layer18_attention_weight_min": -27.43107795715332,
      "activations/layer19_attention_weight_max": 32.06359100341797,
      "activations/layer19_attention_weight_min": -31.378372192382812,
      "activations/layer1_attention_weight_max": 16.19032859802246,
      "activations/layer1_attention_weight_min": -14.377233505249023,
      "activations/layer20_attention_weight_max": 27.356599807739258,
      "activations/layer20_attention_weight_min": -27.980146408081055,
      "activations/layer21_attention_weight_max": 26.993358612060547,
      "activations/layer21_attention_weight_min": -25.38743019104004,
      "activations/layer22_attention_weight_max": 40.37348937988281,
      "activations/layer22_attention_weight_min": -33.02836227416992,
      "activations/layer23_attention_weight_max": 32.42455291748047,
      "activations/layer23_attention_weight_min": -28.16883087158203,
      "activations/layer2_attention_weight_max": 33.36448669433594,
      "activations/layer2_attention_weight_min": -33.4455451965332,
      "activations/layer3_attention_weight_max": 100.7779769897461,
      "activations/layer3_attention_weight_min": -101.1629867553711,
      "activations/layer4_attention_weight_max": 105.71046447753906,
      "activations/layer4_attention_weight_min": -107.46309661865234,
      "activations/layer5_attention_weight_max": 53.17755126953125,
      "activations/layer5_attention_weight_min": -60.23350524902344,
      "activations/layer6_attention_weight_max": 44.259315490722656,
      "activations/layer6_attention_weight_min": -45.923343658447266,
      "activations/layer7_attention_weight_max": 89.09452819824219,
      "activations/layer7_attention_weight_min": -98.38129425048828,
      "activations/layer8_attention_weight_max": 41.85892868041992,
      "activations/layer8_attention_weight_min": -45.85429000854492,
      "activations/layer9_attention_weight_max": 30.602405548095703,
      "activations/layer9_attention_weight_min": -34.51049041748047,
      "epoch": 17.16,
      "learning_rate": 3.9767045454545456e-05,
      "loss": 2.7275,
      "step": 295250
    },
    {
      "activations/layer0_attention_weight_max": 15.87309741973877,
      "activations/layer0_attention_weight_min": -14.138059616088867,
      "activations/layer10_attention_weight_max": 34.76254653930664,
      "activations/layer10_attention_weight_min": -33.49757766723633,
      "activations/layer11_attention_weight_max": 35.47773742675781,
      "activations/layer11_attention_weight_min": -35.644168853759766,
      "activations/layer12_attention_weight_max": 29.264936447143555,
      "activations/layer12_attention_weight_min": -27.36644172668457,
      "activations/layer13_attention_weight_max": 40.386695861816406,
      "activations/layer13_attention_weight_min": -36.41468048095703,
      "activations/layer14_attention_weight_max": 44.01152038574219,
      "activations/layer14_attention_weight_min": -39.819271087646484,
      "activations/layer15_attention_weight_max": 39.49384689331055,
      "activations/layer15_attention_weight_min": -34.760929107666016,
      "activations/layer16_attention_weight_max": 30.066116333007812,
      "activations/layer16_attention_weight_min": -27.4852237701416,
      "activations/layer17_attention_weight_max": 31.420698165893555,
      "activations/layer17_attention_weight_min": -27.57415199279785,
      "activations/layer18_attention_weight_max": 33.32786178588867,
      "activations/layer18_attention_weight_min": -26.838546752929688,
      "activations/layer19_attention_weight_max": 35.66530990600586,
      "activations/layer19_attention_weight_min": -32.15785217285156,
      "activations/layer1_attention_weight_max": 16.463775634765625,
      "activations/layer1_attention_weight_min": -14.8831205368042,
      "activations/layer20_attention_weight_max": 29.469066619873047,
      "activations/layer20_attention_weight_min": -26.729393005371094,
      "activations/layer21_attention_weight_max": 31.08278465270996,
      "activations/layer21_attention_weight_min": -24.858789443969727,
      "activations/layer22_attention_weight_max": 42.93284225463867,
      "activations/layer22_attention_weight_min": -32.56172180175781,
      "activations/layer23_attention_weight_max": 34.62059783935547,
      "activations/layer23_attention_weight_min": -26.154508590698242,
      "activations/layer2_attention_weight_max": 34.52817153930664,
      "activations/layer2_attention_weight_min": -34.09326934814453,
      "activations/layer3_attention_weight_max": 101.87960815429688,
      "activations/layer3_attention_weight_min": -101.5940170288086,
      "activations/layer4_attention_weight_max": 111.18538665771484,
      "activations/layer4_attention_weight_min": -104.227294921875,
      "activations/layer5_attention_weight_max": 50.93915557861328,
      "activations/layer5_attention_weight_min": -60.94492721557617,
      "activations/layer6_attention_weight_max": 45.93318557739258,
      "activations/layer6_attention_weight_min": -45.707191467285156,
      "activations/layer7_attention_weight_max": 92.21131134033203,
      "activations/layer7_attention_weight_min": -93.56024932861328,
      "activations/layer8_attention_weight_max": 43.32946014404297,
      "activations/layer8_attention_weight_min": -44.30921173095703,
      "activations/layer9_attention_weight_max": 32.72834777832031,
      "activations/layer9_attention_weight_min": -34.013370513916016,
      "epoch": 17.16,
      "learning_rate": 3.974848484848485e-05,
      "loss": 2.7246,
      "step": 295300
    },
    {
      "activations/layer0_attention_weight_max": 16.739221572875977,
      "activations/layer0_attention_weight_min": -12.930133819580078,
      "activations/layer10_attention_weight_max": 38.12533950805664,
      "activations/layer10_attention_weight_min": -35.81645202636719,
      "activations/layer11_attention_weight_max": 36.95527648925781,
      "activations/layer11_attention_weight_min": -36.332584381103516,
      "activations/layer12_attention_weight_max": 26.501283645629883,
      "activations/layer12_attention_weight_min": -25.550506591796875,
      "activations/layer13_attention_weight_max": 39.311805725097656,
      "activations/layer13_attention_weight_min": -34.29084777832031,
      "activations/layer14_attention_weight_max": 42.68633270263672,
      "activations/layer14_attention_weight_min": -37.09604263305664,
      "activations/layer15_attention_weight_max": 39.068721771240234,
      "activations/layer15_attention_weight_min": -35.30242919921875,
      "activations/layer16_attention_weight_max": 30.17487335205078,
      "activations/layer16_attention_weight_min": -28.149425506591797,
      "activations/layer17_attention_weight_max": 30.949426651000977,
      "activations/layer17_attention_weight_min": -26.082855224609375,
      "activations/layer18_attention_weight_max": 32.2899284362793,
      "activations/layer18_attention_weight_min": -26.314455032348633,
      "activations/layer19_attention_weight_max": 37.8385124206543,
      "activations/layer19_attention_weight_min": -31.31159019470215,
      "activations/layer1_attention_weight_max": 15.824263572692871,
      "activations/layer1_attention_weight_min": -15.579896926879883,
      "activations/layer20_attention_weight_max": 30.687652587890625,
      "activations/layer20_attention_weight_min": -25.813322067260742,
      "activations/layer21_attention_weight_max": 32.72667694091797,
      "activations/layer21_attention_weight_min": -25.745370864868164,
      "activations/layer22_attention_weight_max": 43.97139358520508,
      "activations/layer22_attention_weight_min": -34.68708419799805,
      "activations/layer23_attention_weight_max": 35.252410888671875,
      "activations/layer23_attention_weight_min": -26.7443904876709,
      "activations/layer2_attention_weight_max": 32.57566833496094,
      "activations/layer2_attention_weight_min": -33.758365631103516,
      "activations/layer3_attention_weight_max": 103.22735595703125,
      "activations/layer3_attention_weight_min": -99.32846069335938,
      "activations/layer4_attention_weight_max": 115.50923156738281,
      "activations/layer4_attention_weight_min": -112.96002197265625,
      "activations/layer5_attention_weight_max": 51.18551254272461,
      "activations/layer5_attention_weight_min": -61.867149353027344,
      "activations/layer6_attention_weight_max": 46.6695671081543,
      "activations/layer6_attention_weight_min": -48.65049743652344,
      "activations/layer7_attention_weight_max": 92.27064514160156,
      "activations/layer7_attention_weight_min": -96.58650970458984,
      "activations/layer8_attention_weight_max": 42.66456985473633,
      "activations/layer8_attention_weight_min": -48.7135124206543,
      "activations/layer9_attention_weight_max": 35.13273239135742,
      "activations/layer9_attention_weight_min": -35.289058685302734,
      "epoch": 17.16,
      "learning_rate": 3.972954545454545e-05,
      "loss": 2.7249,
      "step": 295350
    },
    {
      "activations/layer0_attention_weight_max": 15.89168643951416,
      "activations/layer0_attention_weight_min": -11.89580249786377,
      "activations/layer10_attention_weight_max": 34.648406982421875,
      "activations/layer10_attention_weight_min": -33.95036315917969,
      "activations/layer11_attention_weight_max": 34.488548278808594,
      "activations/layer11_attention_weight_min": -33.845340728759766,
      "activations/layer12_attention_weight_max": 27.711280822753906,
      "activations/layer12_attention_weight_min": -26.25699806213379,
      "activations/layer13_attention_weight_max": 41.81660842895508,
      "activations/layer13_attention_weight_min": -35.83563232421875,
      "activations/layer14_attention_weight_max": 43.46042251586914,
      "activations/layer14_attention_weight_min": -36.94023513793945,
      "activations/layer15_attention_weight_max": 40.4820671081543,
      "activations/layer15_attention_weight_min": -35.10614013671875,
      "activations/layer16_attention_weight_max": 29.747766494750977,
      "activations/layer16_attention_weight_min": -27.951265335083008,
      "activations/layer17_attention_weight_max": 31.01152229309082,
      "activations/layer17_attention_weight_min": -27.97040557861328,
      "activations/layer18_attention_weight_max": 30.025503158569336,
      "activations/layer18_attention_weight_min": -27.237510681152344,
      "activations/layer19_attention_weight_max": 35.47586441040039,
      "activations/layer19_attention_weight_min": -32.64242172241211,
      "activations/layer1_attention_weight_max": 16.22486686706543,
      "activations/layer1_attention_weight_min": -14.493906021118164,
      "activations/layer20_attention_weight_max": 28.831541061401367,
      "activations/layer20_attention_weight_min": -27.68572998046875,
      "activations/layer21_attention_weight_max": 28.9599666595459,
      "activations/layer21_attention_weight_min": -26.662662506103516,
      "activations/layer22_attention_weight_max": 39.52994155883789,
      "activations/layer22_attention_weight_min": -31.449825286865234,
      "activations/layer23_attention_weight_max": 32.459510803222656,
      "activations/layer23_attention_weight_min": -26.425052642822266,
      "activations/layer2_attention_weight_max": 33.83638000488281,
      "activations/layer2_attention_weight_min": -33.842864990234375,
      "activations/layer3_attention_weight_max": 101.75524139404297,
      "activations/layer3_attention_weight_min": -97.09111022949219,
      "activations/layer4_attention_weight_max": 109.9345474243164,
      "activations/layer4_attention_weight_min": -108.14857482910156,
      "activations/layer5_attention_weight_max": 53.86968231201172,
      "activations/layer5_attention_weight_min": -63.82392883300781,
      "activations/layer6_attention_weight_max": 42.43355178833008,
      "activations/layer6_attention_weight_min": -45.139217376708984,
      "activations/layer7_attention_weight_max": 95.10238647460938,
      "activations/layer7_attention_weight_min": -101.37373352050781,
      "activations/layer8_attention_weight_max": 43.97333526611328,
      "activations/layer8_attention_weight_min": -43.408416748046875,
      "activations/layer9_attention_weight_max": 33.18851852416992,
      "activations/layer9_attention_weight_min": -34.370399475097656,
      "epoch": 17.16,
      "learning_rate": 3.9710606060606056e-05,
      "loss": 2.7189,
      "step": 295400
    },
    {
      "activations/layer0_attention_weight_max": 16.72506332397461,
      "activations/layer0_attention_weight_min": -12.453152656555176,
      "activations/layer10_attention_weight_max": 33.12215042114258,
      "activations/layer10_attention_weight_min": -32.29172134399414,
      "activations/layer11_attention_weight_max": 33.27870559692383,
      "activations/layer11_attention_weight_min": -32.79324722290039,
      "activations/layer12_attention_weight_max": 25.616458892822266,
      "activations/layer12_attention_weight_min": -26.496173858642578,
      "activations/layer13_attention_weight_max": 39.155296325683594,
      "activations/layer13_attention_weight_min": -34.965126037597656,
      "activations/layer14_attention_weight_max": 43.56997299194336,
      "activations/layer14_attention_weight_min": -37.61849594116211,
      "activations/layer15_attention_weight_max": 39.65301513671875,
      "activations/layer15_attention_weight_min": -35.23942565917969,
      "activations/layer16_attention_weight_max": 32.541683197021484,
      "activations/layer16_attention_weight_min": -26.583518981933594,
      "activations/layer17_attention_weight_max": 31.65802001953125,
      "activations/layer17_attention_weight_min": -25.51188850402832,
      "activations/layer18_attention_weight_max": 35.101402282714844,
      "activations/layer18_attention_weight_min": -27.08148765563965,
      "activations/layer19_attention_weight_max": 34.5241813659668,
      "activations/layer19_attention_weight_min": -32.387638092041016,
      "activations/layer1_attention_weight_max": 16.249176025390625,
      "activations/layer1_attention_weight_min": -15.86750316619873,
      "activations/layer20_attention_weight_max": 29.317310333251953,
      "activations/layer20_attention_weight_min": -25.899396896362305,
      "activations/layer21_attention_weight_max": 28.950057983398438,
      "activations/layer21_attention_weight_min": -25.755666732788086,
      "activations/layer22_attention_weight_max": 45.67308044433594,
      "activations/layer22_attention_weight_min": -33.35197830200195,
      "activations/layer23_attention_weight_max": 34.72986602783203,
      "activations/layer23_attention_weight_min": -26.659828186035156,
      "activations/layer2_attention_weight_max": 33.60722732543945,
      "activations/layer2_attention_weight_min": -34.281761169433594,
      "activations/layer3_attention_weight_max": 97.94267272949219,
      "activations/layer3_attention_weight_min": -100.26317596435547,
      "activations/layer4_attention_weight_max": 102.36522674560547,
      "activations/layer4_attention_weight_min": -106.61540985107422,
      "activations/layer5_attention_weight_max": 52.16314697265625,
      "activations/layer5_attention_weight_min": -62.820838928222656,
      "activations/layer6_attention_weight_max": 43.00294494628906,
      "activations/layer6_attention_weight_min": -46.094486236572266,
      "activations/layer7_attention_weight_max": 95.12855529785156,
      "activations/layer7_attention_weight_min": -92.29803466796875,
      "activations/layer8_attention_weight_max": 39.846927642822266,
      "activations/layer8_attention_weight_min": -42.53166580200195,
      "activations/layer9_attention_weight_max": 31.427209854125977,
      "activations/layer9_attention_weight_min": -33.312808990478516,
      "epoch": 17.17,
      "learning_rate": 3.9691666666666665e-05,
      "loss": 2.7386,
      "step": 295450
    },
    {
      "activations/layer0_attention_weight_max": 15.82837200164795,
      "activations/layer0_attention_weight_min": -12.530244827270508,
      "activations/layer10_attention_weight_max": 36.219154357910156,
      "activations/layer10_attention_weight_min": -34.152992248535156,
      "activations/layer11_attention_weight_max": 33.617427825927734,
      "activations/layer11_attention_weight_min": -34.3865966796875,
      "activations/layer12_attention_weight_max": 27.146394729614258,
      "activations/layer12_attention_weight_min": -26.899473190307617,
      "activations/layer13_attention_weight_max": 42.1125373840332,
      "activations/layer13_attention_weight_min": -38.76632308959961,
      "activations/layer14_attention_weight_max": 44.8422737121582,
      "activations/layer14_attention_weight_min": -40.00168991088867,
      "activations/layer15_attention_weight_max": 38.38996124267578,
      "activations/layer15_attention_weight_min": -36.46015930175781,
      "activations/layer16_attention_weight_max": 32.18168640136719,
      "activations/layer16_attention_weight_min": -28.032215118408203,
      "activations/layer17_attention_weight_max": 30.548561096191406,
      "activations/layer17_attention_weight_min": -26.045372009277344,
      "activations/layer18_attention_weight_max": 30.768709182739258,
      "activations/layer18_attention_weight_min": -27.383512496948242,
      "activations/layer19_attention_weight_max": 34.41661071777344,
      "activations/layer19_attention_weight_min": -31.018741607666016,
      "activations/layer1_attention_weight_max": 16.01399803161621,
      "activations/layer1_attention_weight_min": -14.776094436645508,
      "activations/layer20_attention_weight_max": 29.600717544555664,
      "activations/layer20_attention_weight_min": -27.12620735168457,
      "activations/layer21_attention_weight_max": 29.54677963256836,
      "activations/layer21_attention_weight_min": -25.977956771850586,
      "activations/layer22_attention_weight_max": 46.37321853637695,
      "activations/layer22_attention_weight_min": -34.55213165283203,
      "activations/layer23_attention_weight_max": 34.071998596191406,
      "activations/layer23_attention_weight_min": -27.415599822998047,
      "activations/layer2_attention_weight_max": 33.655792236328125,
      "activations/layer2_attention_weight_min": -32.93871307373047,
      "activations/layer3_attention_weight_max": 99.95153045654297,
      "activations/layer3_attention_weight_min": -98.53694915771484,
      "activations/layer4_attention_weight_max": 107.06066131591797,
      "activations/layer4_attention_weight_min": -108.568115234375,
      "activations/layer5_attention_weight_max": 53.80802536010742,
      "activations/layer5_attention_weight_min": -62.481849670410156,
      "activations/layer6_attention_weight_max": 46.13795471191406,
      "activations/layer6_attention_weight_min": -47.83639907836914,
      "activations/layer7_attention_weight_max": 96.34017944335938,
      "activations/layer7_attention_weight_min": -102.10301208496094,
      "activations/layer8_attention_weight_max": 43.554718017578125,
      "activations/layer8_attention_weight_min": -45.02457809448242,
      "activations/layer9_attention_weight_max": 34.46720504760742,
      "activations/layer9_attention_weight_min": -34.636661529541016,
      "epoch": 17.17,
      "learning_rate": 3.967272727272727e-05,
      "loss": 2.7443,
      "step": 295500
    },
    {
      "activations/layer0_attention_weight_max": 16.354520797729492,
      "activations/layer0_attention_weight_min": -14.35981559753418,
      "activations/layer10_attention_weight_max": 36.09973907470703,
      "activations/layer10_attention_weight_min": -33.6103515625,
      "activations/layer11_attention_weight_max": 38.589691162109375,
      "activations/layer11_attention_weight_min": -35.561588287353516,
      "activations/layer12_attention_weight_max": 28.689210891723633,
      "activations/layer12_attention_weight_min": -27.527637481689453,
      "activations/layer13_attention_weight_max": 42.469566345214844,
      "activations/layer13_attention_weight_min": -37.62299728393555,
      "activations/layer14_attention_weight_max": 47.12239074707031,
      "activations/layer14_attention_weight_min": -38.93681716918945,
      "activations/layer15_attention_weight_max": 43.205841064453125,
      "activations/layer15_attention_weight_min": -38.96409225463867,
      "activations/layer16_attention_weight_max": 32.54153823852539,
      "activations/layer16_attention_weight_min": -28.78948974609375,
      "activations/layer17_attention_weight_max": 32.80155563354492,
      "activations/layer17_attention_weight_min": -27.07265281677246,
      "activations/layer18_attention_weight_max": 31.632919311523438,
      "activations/layer18_attention_weight_min": -28.64885139465332,
      "activations/layer19_attention_weight_max": 34.89596176147461,
      "activations/layer19_attention_weight_min": -32.84526824951172,
      "activations/layer1_attention_weight_max": 16.014251708984375,
      "activations/layer1_attention_weight_min": -14.622321128845215,
      "activations/layer20_attention_weight_max": 30.126996994018555,
      "activations/layer20_attention_weight_min": -28.853713989257812,
      "activations/layer21_attention_weight_max": 28.591922760009766,
      "activations/layer21_attention_weight_min": -27.27322006225586,
      "activations/layer22_attention_weight_max": 39.896263122558594,
      "activations/layer22_attention_weight_min": -34.362274169921875,
      "activations/layer23_attention_weight_max": 29.731061935424805,
      "activations/layer23_attention_weight_min": -27.491323471069336,
      "activations/layer2_attention_weight_max": 31.900732040405273,
      "activations/layer2_attention_weight_min": -32.998409271240234,
      "activations/layer3_attention_weight_max": 97.36841583251953,
      "activations/layer3_attention_weight_min": -95.84364318847656,
      "activations/layer4_attention_weight_max": 102.82464599609375,
      "activations/layer4_attention_weight_min": -107.3714828491211,
      "activations/layer5_attention_weight_max": 48.377655029296875,
      "activations/layer5_attention_weight_min": -62.35274887084961,
      "activations/layer6_attention_weight_max": 43.924198150634766,
      "activations/layer6_attention_weight_min": -47.31272506713867,
      "activations/layer7_attention_weight_max": 93.94055938720703,
      "activations/layer7_attention_weight_min": -92.4563980102539,
      "activations/layer8_attention_weight_max": 43.245357513427734,
      "activations/layer8_attention_weight_min": -43.61210250854492,
      "activations/layer9_attention_weight_max": 32.896358489990234,
      "activations/layer9_attention_weight_min": -34.8726692199707,
      "epoch": 17.17,
      "learning_rate": 3.9653787878787874e-05,
      "loss": 2.7253,
      "step": 295550
    },
    {
      "activations/layer0_attention_weight_max": 16.543594360351562,
      "activations/layer0_attention_weight_min": -13.009917259216309,
      "activations/layer10_attention_weight_max": 34.92289733886719,
      "activations/layer10_attention_weight_min": -34.2369270324707,
      "activations/layer11_attention_weight_max": 36.54756546020508,
      "activations/layer11_attention_weight_min": -33.72853469848633,
      "activations/layer12_attention_weight_max": 29.048437118530273,
      "activations/layer12_attention_weight_min": -27.137191772460938,
      "activations/layer13_attention_weight_max": 41.80464172363281,
      "activations/layer13_attention_weight_min": -36.65156936645508,
      "activations/layer14_attention_weight_max": 43.7770881652832,
      "activations/layer14_attention_weight_min": -37.9351921081543,
      "activations/layer15_attention_weight_max": 41.221435546875,
      "activations/layer15_attention_weight_min": -35.95455551147461,
      "activations/layer16_attention_weight_max": 29.813907623291016,
      "activations/layer16_attention_weight_min": -28.4497013092041,
      "activations/layer17_attention_weight_max": 27.96375846862793,
      "activations/layer17_attention_weight_min": -25.883285522460938,
      "activations/layer18_attention_weight_max": 29.28040885925293,
      "activations/layer18_attention_weight_min": -24.20218276977539,
      "activations/layer19_attention_weight_max": 34.82469940185547,
      "activations/layer19_attention_weight_min": -30.403928756713867,
      "activations/layer1_attention_weight_max": 16.682422637939453,
      "activations/layer1_attention_weight_min": -15.794957160949707,
      "activations/layer20_attention_weight_max": 28.68568992614746,
      "activations/layer20_attention_weight_min": -25.330175399780273,
      "activations/layer21_attention_weight_max": 27.836101531982422,
      "activations/layer21_attention_weight_min": -24.96734046936035,
      "activations/layer22_attention_weight_max": 41.56470489501953,
      "activations/layer22_attention_weight_min": -32.6768684387207,
      "activations/layer23_attention_weight_max": 34.20928955078125,
      "activations/layer23_attention_weight_min": -25.285114288330078,
      "activations/layer2_attention_weight_max": 34.32219696044922,
      "activations/layer2_attention_weight_min": -33.307403564453125,
      "activations/layer3_attention_weight_max": 99.64454650878906,
      "activations/layer3_attention_weight_min": -99.10237121582031,
      "activations/layer4_attention_weight_max": 106.593505859375,
      "activations/layer4_attention_weight_min": -105.19544982910156,
      "activations/layer5_attention_weight_max": 48.49700927734375,
      "activations/layer5_attention_weight_min": -59.494537353515625,
      "activations/layer6_attention_weight_max": 42.88191223144531,
      "activations/layer6_attention_weight_min": -44.18832015991211,
      "activations/layer7_attention_weight_max": 91.13703918457031,
      "activations/layer7_attention_weight_min": -91.81560516357422,
      "activations/layer8_attention_weight_max": 43.476383209228516,
      "activations/layer8_attention_weight_min": -43.27259826660156,
      "activations/layer9_attention_weight_max": 32.96688461303711,
      "activations/layer9_attention_weight_min": -32.63022994995117,
      "epoch": 17.18,
      "learning_rate": 3.9634848484848476e-05,
      "loss": 2.7289,
      "step": 295600
    },
    {
      "activations/layer0_attention_weight_max": 16.766143798828125,
      "activations/layer0_attention_weight_min": -13.309765815734863,
      "activations/layer10_attention_weight_max": 32.20824432373047,
      "activations/layer10_attention_weight_min": -34.19350814819336,
      "activations/layer11_attention_weight_max": 33.38976287841797,
      "activations/layer11_attention_weight_min": -33.184471130371094,
      "activations/layer12_attention_weight_max": 23.68979835510254,
      "activations/layer12_attention_weight_min": -25.594390869140625,
      "activations/layer13_attention_weight_max": 35.997779846191406,
      "activations/layer13_attention_weight_min": -35.094688415527344,
      "activations/layer14_attention_weight_max": 40.15727996826172,
      "activations/layer14_attention_weight_min": -39.463687896728516,
      "activations/layer15_attention_weight_max": 37.247188568115234,
      "activations/layer15_attention_weight_min": -34.23866271972656,
      "activations/layer16_attention_weight_max": 27.544921875,
      "activations/layer16_attention_weight_min": -27.553279876708984,
      "activations/layer17_attention_weight_max": 29.571496963500977,
      "activations/layer17_attention_weight_min": -25.522260665893555,
      "activations/layer18_attention_weight_max": 31.360755920410156,
      "activations/layer18_attention_weight_min": -25.006633758544922,
      "activations/layer19_attention_weight_max": 30.552637100219727,
      "activations/layer19_attention_weight_min": -30.9644718170166,
      "activations/layer1_attention_weight_max": 16.243154525756836,
      "activations/layer1_attention_weight_min": -15.410425186157227,
      "activations/layer20_attention_weight_max": 27.417478561401367,
      "activations/layer20_attention_weight_min": -24.417041778564453,
      "activations/layer21_attention_weight_max": 27.842113494873047,
      "activations/layer21_attention_weight_min": -23.88997459411621,
      "activations/layer22_attention_weight_max": 43.20597457885742,
      "activations/layer22_attention_weight_min": -30.913267135620117,
      "activations/layer23_attention_weight_max": 33.44818878173828,
      "activations/layer23_attention_weight_min": -25.36901092529297,
      "activations/layer2_attention_weight_max": 32.19064712524414,
      "activations/layer2_attention_weight_min": -32.64038848876953,
      "activations/layer3_attention_weight_max": 92.75206756591797,
      "activations/layer3_attention_weight_min": -94.53995513916016,
      "activations/layer4_attention_weight_max": 105.3551254272461,
      "activations/layer4_attention_weight_min": -106.6329345703125,
      "activations/layer5_attention_weight_max": 48.625885009765625,
      "activations/layer5_attention_weight_min": -60.711517333984375,
      "activations/layer6_attention_weight_max": 42.621307373046875,
      "activations/layer6_attention_weight_min": -45.79513168334961,
      "activations/layer7_attention_weight_max": 91.28919219970703,
      "activations/layer7_attention_weight_min": -95.46749114990234,
      "activations/layer8_attention_weight_max": 41.45830535888672,
      "activations/layer8_attention_weight_min": -43.45281219482422,
      "activations/layer9_attention_weight_max": 30.955846786499023,
      "activations/layer9_attention_weight_min": -35.29243850708008,
      "epoch": 17.18,
      "learning_rate": 3.961590909090909e-05,
      "loss": 2.7227,
      "step": 295650
    },
    {
      "activations/layer0_attention_weight_max": 17.590391159057617,
      "activations/layer0_attention_weight_min": -14.334426879882812,
      "activations/layer10_attention_weight_max": 31.750316619873047,
      "activations/layer10_attention_weight_min": -32.27802658081055,
      "activations/layer11_attention_weight_max": 31.55171012878418,
      "activations/layer11_attention_weight_min": -31.199317932128906,
      "activations/layer12_attention_weight_max": 25.865188598632812,
      "activations/layer12_attention_weight_min": -25.293527603149414,
      "activations/layer13_attention_weight_max": 39.265968322753906,
      "activations/layer13_attention_weight_min": -34.510719299316406,
      "activations/layer14_attention_weight_max": 43.56949996948242,
      "activations/layer14_attention_weight_min": -35.96137619018555,
      "activations/layer15_attention_weight_max": 37.07021713256836,
      "activations/layer15_attention_weight_min": -36.403175354003906,
      "activations/layer16_attention_weight_max": 27.930540084838867,
      "activations/layer16_attention_weight_min": -26.896751403808594,
      "activations/layer17_attention_weight_max": 30.482568740844727,
      "activations/layer17_attention_weight_min": -26.338918685913086,
      "activations/layer18_attention_weight_max": 31.925390243530273,
      "activations/layer18_attention_weight_min": -24.53743553161621,
      "activations/layer19_attention_weight_max": 31.92714500427246,
      "activations/layer19_attention_weight_min": -32.42922592163086,
      "activations/layer1_attention_weight_max": 16.73919677734375,
      "activations/layer1_attention_weight_min": -14.847055435180664,
      "activations/layer20_attention_weight_max": 29.782970428466797,
      "activations/layer20_attention_weight_min": -25.3915958404541,
      "activations/layer21_attention_weight_max": 29.04317283630371,
      "activations/layer21_attention_weight_min": -27.268253326416016,
      "activations/layer22_attention_weight_max": 42.8835563659668,
      "activations/layer22_attention_weight_min": -30.250812530517578,
      "activations/layer23_attention_weight_max": 30.69500732421875,
      "activations/layer23_attention_weight_min": -24.935766220092773,
      "activations/layer2_attention_weight_max": 33.435245513916016,
      "activations/layer2_attention_weight_min": -32.89763641357422,
      "activations/layer3_attention_weight_max": 96.98287963867188,
      "activations/layer3_attention_weight_min": -96.00545501708984,
      "activations/layer4_attention_weight_max": 108.04534149169922,
      "activations/layer4_attention_weight_min": -109.29625701904297,
      "activations/layer5_attention_weight_max": 50.62214660644531,
      "activations/layer5_attention_weight_min": -64.0185317993164,
      "activations/layer6_attention_weight_max": 43.658626556396484,
      "activations/layer6_attention_weight_min": -47.69588851928711,
      "activations/layer7_attention_weight_max": 91.98299407958984,
      "activations/layer7_attention_weight_min": -95.3836898803711,
      "activations/layer8_attention_weight_max": 38.835819244384766,
      "activations/layer8_attention_weight_min": -41.85258483886719,
      "activations/layer9_attention_weight_max": 28.87803077697754,
      "activations/layer9_attention_weight_min": -32.48306655883789,
      "epoch": 17.18,
      "learning_rate": 3.959696969696969e-05,
      "loss": 2.732,
      "step": 295700
    },
    {
      "activations/layer0_attention_weight_max": 17.25266456604004,
      "activations/layer0_attention_weight_min": -14.331655502319336,
      "activations/layer10_attention_weight_max": 34.474056243896484,
      "activations/layer10_attention_weight_min": -34.25487518310547,
      "activations/layer11_attention_weight_max": 34.38390350341797,
      "activations/layer11_attention_weight_min": -35.03587341308594,
      "activations/layer12_attention_weight_max": 27.289140701293945,
      "activations/layer12_attention_weight_min": -31.133350372314453,
      "activations/layer13_attention_weight_max": 39.94450378417969,
      "activations/layer13_attention_weight_min": -34.99443054199219,
      "activations/layer14_attention_weight_max": 49.42666244506836,
      "activations/layer14_attention_weight_min": -37.92599868774414,
      "activations/layer15_attention_weight_max": 41.38370895385742,
      "activations/layer15_attention_weight_min": -34.558494567871094,
      "activations/layer16_attention_weight_max": 31.3453369140625,
      "activations/layer16_attention_weight_min": -27.86154556274414,
      "activations/layer17_attention_weight_max": 29.734521865844727,
      "activations/layer17_attention_weight_min": -26.075841903686523,
      "activations/layer18_attention_weight_max": 31.236026763916016,
      "activations/layer18_attention_weight_min": -25.177059173583984,
      "activations/layer19_attention_weight_max": 34.40647506713867,
      "activations/layer19_attention_weight_min": -33.56010818481445,
      "activations/layer1_attention_weight_max": 16.794994354248047,
      "activations/layer1_attention_weight_min": -15.37545108795166,
      "activations/layer20_attention_weight_max": 29.25299072265625,
      "activations/layer20_attention_weight_min": -26.320720672607422,
      "activations/layer21_attention_weight_max": 29.545408248901367,
      "activations/layer21_attention_weight_min": -26.025964736938477,
      "activations/layer22_attention_weight_max": 41.57984924316406,
      "activations/layer22_attention_weight_min": -35.9093017578125,
      "activations/layer23_attention_weight_max": 35.64317321777344,
      "activations/layer23_attention_weight_min": -25.62643814086914,
      "activations/layer2_attention_weight_max": 34.83026123046875,
      "activations/layer2_attention_weight_min": -35.052452087402344,
      "activations/layer3_attention_weight_max": 97.56279754638672,
      "activations/layer3_attention_weight_min": -99.29489135742188,
      "activations/layer4_attention_weight_max": 105.66088104248047,
      "activations/layer4_attention_weight_min": -109.74360656738281,
      "activations/layer5_attention_weight_max": 53.52955627441406,
      "activations/layer5_attention_weight_min": -60.70661926269531,
      "activations/layer6_attention_weight_max": 44.95037078857422,
      "activations/layer6_attention_weight_min": -47.756385803222656,
      "activations/layer7_attention_weight_max": 94.06917572021484,
      "activations/layer7_attention_weight_min": -102.07855224609375,
      "activations/layer8_attention_weight_max": 41.67908477783203,
      "activations/layer8_attention_weight_min": -46.47441864013672,
      "activations/layer9_attention_weight_max": 33.024627685546875,
      "activations/layer9_attention_weight_min": -33.978092193603516,
      "epoch": 17.18,
      "learning_rate": 3.95780303030303e-05,
      "loss": 2.7317,
      "step": 295750
    },
    {
      "activations/layer0_attention_weight_max": 17.482358932495117,
      "activations/layer0_attention_weight_min": -13.13758373260498,
      "activations/layer10_attention_weight_max": 33.93238067626953,
      "activations/layer10_attention_weight_min": -34.02920913696289,
      "activations/layer11_attention_weight_max": 36.70256805419922,
      "activations/layer11_attention_weight_min": -34.12530517578125,
      "activations/layer12_attention_weight_max": 26.270540237426758,
      "activations/layer12_attention_weight_min": -29.698997497558594,
      "activations/layer13_attention_weight_max": 39.03774642944336,
      "activations/layer13_attention_weight_min": -34.349510192871094,
      "activations/layer14_attention_weight_max": 41.49650955200195,
      "activations/layer14_attention_weight_min": -36.728416442871094,
      "activations/layer15_attention_weight_max": 41.046180725097656,
      "activations/layer15_attention_weight_min": -33.95835876464844,
      "activations/layer16_attention_weight_max": 31.09291648864746,
      "activations/layer16_attention_weight_min": -29.672998428344727,
      "activations/layer17_attention_weight_max": 37.227760314941406,
      "activations/layer17_attention_weight_min": -30.97271728515625,
      "activations/layer18_attention_weight_max": 39.32432556152344,
      "activations/layer18_attention_weight_min": -28.7894344329834,
      "activations/layer19_attention_weight_max": 43.04203796386719,
      "activations/layer19_attention_weight_min": -32.136207580566406,
      "activations/layer1_attention_weight_max": 16.374324798583984,
      "activations/layer1_attention_weight_min": -15.309788703918457,
      "activations/layer20_attention_weight_max": 34.5610466003418,
      "activations/layer20_attention_weight_min": -25.55007553100586,
      "activations/layer21_attention_weight_max": 34.916439056396484,
      "activations/layer21_attention_weight_min": -24.396820068359375,
      "activations/layer22_attention_weight_max": 52.92166519165039,
      "activations/layer22_attention_weight_min": -32.61724090576172,
      "activations/layer23_attention_weight_max": 42.26394271850586,
      "activations/layer23_attention_weight_min": -27.226985931396484,
      "activations/layer2_attention_weight_max": 32.73665237426758,
      "activations/layer2_attention_weight_min": -34.46125793457031,
      "activations/layer3_attention_weight_max": 100.37549591064453,
      "activations/layer3_attention_weight_min": -98.49089050292969,
      "activations/layer4_attention_weight_max": 109.0591812133789,
      "activations/layer4_attention_weight_min": -105.0374526977539,
      "activations/layer5_attention_weight_max": 49.63589096069336,
      "activations/layer5_attention_weight_min": -62.87831115722656,
      "activations/layer6_attention_weight_max": 48.28331756591797,
      "activations/layer6_attention_weight_min": -48.927066802978516,
      "activations/layer7_attention_weight_max": 95.41433715820312,
      "activations/layer7_attention_weight_min": -100.93536376953125,
      "activations/layer8_attention_weight_max": 40.70838928222656,
      "activations/layer8_attention_weight_min": -44.77729415893555,
      "activations/layer9_attention_weight_max": 32.494598388671875,
      "activations/layer9_attention_weight_min": -32.438255310058594,
      "epoch": 17.19,
      "learning_rate": 3.955909090909091e-05,
      "loss": 2.7289,
      "step": 295800
    },
    {
      "activations/layer0_attention_weight_max": 17.014324188232422,
      "activations/layer0_attention_weight_min": -13.232199668884277,
      "activations/layer10_attention_weight_max": 34.57037353515625,
      "activations/layer10_attention_weight_min": -31.86481285095215,
      "activations/layer11_attention_weight_max": 34.831138610839844,
      "activations/layer11_attention_weight_min": -33.048194885253906,
      "activations/layer12_attention_weight_max": 26.056196212768555,
      "activations/layer12_attention_weight_min": -26.927160263061523,
      "activations/layer13_attention_weight_max": 39.01374816894531,
      "activations/layer13_attention_weight_min": -33.707759857177734,
      "activations/layer14_attention_weight_max": 43.523563385009766,
      "activations/layer14_attention_weight_min": -41.38945007324219,
      "activations/layer15_attention_weight_max": 39.01508712768555,
      "activations/layer15_attention_weight_min": -39.538692474365234,
      "activations/layer16_attention_weight_max": 28.763566970825195,
      "activations/layer16_attention_weight_min": -30.04490852355957,
      "activations/layer17_attention_weight_max": 27.028701782226562,
      "activations/layer17_attention_weight_min": -30.414270401000977,
      "activations/layer18_attention_weight_max": 32.46257781982422,
      "activations/layer18_attention_weight_min": -28.086772918701172,
      "activations/layer19_attention_weight_max": 30.666257858276367,
      "activations/layer19_attention_weight_min": -32.0604133605957,
      "activations/layer1_attention_weight_max": 16.877222061157227,
      "activations/layer1_attention_weight_min": -14.789031028747559,
      "activations/layer20_attention_weight_max": 25.63311195373535,
      "activations/layer20_attention_weight_min": -25.330942153930664,
      "activations/layer21_attention_weight_max": 25.284488677978516,
      "activations/layer21_attention_weight_min": -25.004600524902344,
      "activations/layer22_attention_weight_max": 42.35505676269531,
      "activations/layer22_attention_weight_min": -35.62730026245117,
      "activations/layer23_attention_weight_max": 28.645675659179688,
      "activations/layer23_attention_weight_min": -26.507978439331055,
      "activations/layer2_attention_weight_max": 33.29771041870117,
      "activations/layer2_attention_weight_min": -31.843402862548828,
      "activations/layer3_attention_weight_max": 101.2631607055664,
      "activations/layer3_attention_weight_min": -98.09477996826172,
      "activations/layer4_attention_weight_max": 109.9664306640625,
      "activations/layer4_attention_weight_min": -105.0476303100586,
      "activations/layer5_attention_weight_max": 49.615211486816406,
      "activations/layer5_attention_weight_min": -61.261810302734375,
      "activations/layer6_attention_weight_max": 43.48967742919922,
      "activations/layer6_attention_weight_min": -46.408668518066406,
      "activations/layer7_attention_weight_max": 89.13314056396484,
      "activations/layer7_attention_weight_min": -91.54595184326172,
      "activations/layer8_attention_weight_max": 42.56599426269531,
      "activations/layer8_attention_weight_min": -42.966773986816406,
      "activations/layer9_attention_weight_max": 31.89114761352539,
      "activations/layer9_attention_weight_min": -32.01278305053711,
      "epoch": 17.19,
      "learning_rate": 3.954015151515151e-05,
      "loss": 2.7461,
      "step": 295850
    },
    {
      "activations/layer0_attention_weight_max": 17.760242462158203,
      "activations/layer0_attention_weight_min": -13.038050651550293,
      "activations/layer10_attention_weight_max": 43.59666061401367,
      "activations/layer10_attention_weight_min": -39.765174865722656,
      "activations/layer11_attention_weight_max": 41.84252166748047,
      "activations/layer11_attention_weight_min": -39.70870590209961,
      "activations/layer12_attention_weight_max": 31.847492218017578,
      "activations/layer12_attention_weight_min": -28.838829040527344,
      "activations/layer13_attention_weight_max": 49.56587219238281,
      "activations/layer13_attention_weight_min": -39.264381408691406,
      "activations/layer14_attention_weight_max": 49.827823638916016,
      "activations/layer14_attention_weight_min": -46.988948822021484,
      "activations/layer15_attention_weight_max": 55.88858413696289,
      "activations/layer15_attention_weight_min": -44.14491653442383,
      "activations/layer16_attention_weight_max": 35.37900161743164,
      "activations/layer16_attention_weight_min": -27.57774543762207,
      "activations/layer17_attention_weight_max": 31.73736572265625,
      "activations/layer17_attention_weight_min": -26.988662719726562,
      "activations/layer18_attention_weight_max": 32.41264724731445,
      "activations/layer18_attention_weight_min": -24.491674423217773,
      "activations/layer19_attention_weight_max": 37.79326629638672,
      "activations/layer19_attention_weight_min": -31.626483917236328,
      "activations/layer1_attention_weight_max": 15.950868606567383,
      "activations/layer1_attention_weight_min": -15.097454071044922,
      "activations/layer20_attention_weight_max": 31.79705810546875,
      "activations/layer20_attention_weight_min": -24.461454391479492,
      "activations/layer21_attention_weight_max": 31.94192886352539,
      "activations/layer21_attention_weight_min": -25.31577491760254,
      "activations/layer22_attention_weight_max": 46.9493522644043,
      "activations/layer22_attention_weight_min": -31.033893585205078,
      "activations/layer23_attention_weight_max": 34.33394241333008,
      "activations/layer23_attention_weight_min": -26.767942428588867,
      "activations/layer2_attention_weight_max": 32.849647521972656,
      "activations/layer2_attention_weight_min": -35.256614685058594,
      "activations/layer3_attention_weight_max": 98.08533477783203,
      "activations/layer3_attention_weight_min": -98.71105194091797,
      "activations/layer4_attention_weight_max": 104.29471588134766,
      "activations/layer4_attention_weight_min": -107.02498626708984,
      "activations/layer5_attention_weight_max": 52.550621032714844,
      "activations/layer5_attention_weight_min": -60.0954704284668,
      "activations/layer6_attention_weight_max": 46.58340072631836,
      "activations/layer6_attention_weight_min": -48.65446853637695,
      "activations/layer7_attention_weight_max": 97.58804321289062,
      "activations/layer7_attention_weight_min": -103.41585540771484,
      "activations/layer8_attention_weight_max": 46.915531158447266,
      "activations/layer8_attention_weight_min": -46.62713623046875,
      "activations/layer9_attention_weight_max": 37.460533142089844,
      "activations/layer9_attention_weight_min": -37.958675384521484,
      "epoch": 17.19,
      "learning_rate": 3.952121212121212e-05,
      "loss": 2.7396,
      "step": 295900
    },
    {
      "activations/layer0_attention_weight_max": 16.972925186157227,
      "activations/layer0_attention_weight_min": -12.915704727172852,
      "activations/layer10_attention_weight_max": 34.438968658447266,
      "activations/layer10_attention_weight_min": -34.93183135986328,
      "activations/layer11_attention_weight_max": 33.695770263671875,
      "activations/layer11_attention_weight_min": -36.6804084777832,
      "activations/layer12_attention_weight_max": 24.052879333496094,
      "activations/layer12_attention_weight_min": -27.597414016723633,
      "activations/layer13_attention_weight_max": 37.281700134277344,
      "activations/layer13_attention_weight_min": -36.43781280517578,
      "activations/layer14_attention_weight_max": 40.117130279541016,
      "activations/layer14_attention_weight_min": -38.0499382019043,
      "activations/layer15_attention_weight_max": 33.744754791259766,
      "activations/layer15_attention_weight_min": -34.50975036621094,
      "activations/layer16_attention_weight_max": 28.63794708251953,
      "activations/layer16_attention_weight_min": -27.998470306396484,
      "activations/layer17_attention_weight_max": 27.090404510498047,
      "activations/layer17_attention_weight_min": -25.23963737487793,
      "activations/layer18_attention_weight_max": 28.687467575073242,
      "activations/layer18_attention_weight_min": -26.950016021728516,
      "activations/layer19_attention_weight_max": 30.9617919921875,
      "activations/layer19_attention_weight_min": -30.173696517944336,
      "activations/layer1_attention_weight_max": 16.047470092773438,
      "activations/layer1_attention_weight_min": -14.791698455810547,
      "activations/layer20_attention_weight_max": 26.98241424560547,
      "activations/layer20_attention_weight_min": -25.287490844726562,
      "activations/layer21_attention_weight_max": 27.698497772216797,
      "activations/layer21_attention_weight_min": -24.548038482666016,
      "activations/layer22_attention_weight_max": 42.44029998779297,
      "activations/layer22_attention_weight_min": -30.804304122924805,
      "activations/layer23_attention_weight_max": 30.750059127807617,
      "activations/layer23_attention_weight_min": -26.082164764404297,
      "activations/layer2_attention_weight_max": 32.77225875854492,
      "activations/layer2_attention_weight_min": -33.30840301513672,
      "activations/layer3_attention_weight_max": 95.107421875,
      "activations/layer3_attention_weight_min": -98.47937774658203,
      "activations/layer4_attention_weight_max": 106.4776611328125,
      "activations/layer4_attention_weight_min": -104.80895233154297,
      "activations/layer5_attention_weight_max": 50.755882263183594,
      "activations/layer5_attention_weight_min": -58.579524993896484,
      "activations/layer6_attention_weight_max": 46.08317947387695,
      "activations/layer6_attention_weight_min": -46.31421661376953,
      "activations/layer7_attention_weight_max": 102.9792709350586,
      "activations/layer7_attention_weight_min": -97.86505889892578,
      "activations/layer8_attention_weight_max": 42.603858947753906,
      "activations/layer8_attention_weight_min": -42.848846435546875,
      "activations/layer9_attention_weight_max": 32.47018814086914,
      "activations/layer9_attention_weight_min": -35.143157958984375,
      "epoch": 17.2,
      "learning_rate": 3.950227272727272e-05,
      "loss": 2.7301,
      "step": 295950
    },
    {
      "activations/layer0_attention_weight_max": 16.85951042175293,
      "activations/layer0_attention_weight_min": -14.451037406921387,
      "activations/layer10_attention_weight_max": 34.197547912597656,
      "activations/layer10_attention_weight_min": -35.97944259643555,
      "activations/layer11_attention_weight_max": 34.992218017578125,
      "activations/layer11_attention_weight_min": -33.725379943847656,
      "activations/layer12_attention_weight_max": 26.21982192993164,
      "activations/layer12_attention_weight_min": -26.108360290527344,
      "activations/layer13_attention_weight_max": 39.844173431396484,
      "activations/layer13_attention_weight_min": -36.153804779052734,
      "activations/layer14_attention_weight_max": 44.5717658996582,
      "activations/layer14_attention_weight_min": -37.81550598144531,
      "activations/layer15_attention_weight_max": 39.377593994140625,
      "activations/layer15_attention_weight_min": -36.640071868896484,
      "activations/layer16_attention_weight_max": 31.730525970458984,
      "activations/layer16_attention_weight_min": -30.112958908081055,
      "activations/layer17_attention_weight_max": 29.744232177734375,
      "activations/layer17_attention_weight_min": -27.253154754638672,
      "activations/layer18_attention_weight_max": 31.548507690429688,
      "activations/layer18_attention_weight_min": -26.494386672973633,
      "activations/layer19_attention_weight_max": 31.439910888671875,
      "activations/layer19_attention_weight_min": -31.941694259643555,
      "activations/layer1_attention_weight_max": 16.510896682739258,
      "activations/layer1_attention_weight_min": -13.686226844787598,
      "activations/layer20_attention_weight_max": 28.964277267456055,
      "activations/layer20_attention_weight_min": -26.795093536376953,
      "activations/layer21_attention_weight_max": 29.62731170654297,
      "activations/layer21_attention_weight_min": -25.475778579711914,
      "activations/layer22_attention_weight_max": 51.0340461730957,
      "activations/layer22_attention_weight_min": -31.759389877319336,
      "activations/layer23_attention_weight_max": 34.324302673339844,
      "activations/layer23_attention_weight_min": -25.390043258666992,
      "activations/layer2_attention_weight_max": 32.320552825927734,
      "activations/layer2_attention_weight_min": -32.612449645996094,
      "activations/layer3_attention_weight_max": 93.70703887939453,
      "activations/layer3_attention_weight_min": -96.03150939941406,
      "activations/layer4_attention_weight_max": 100.61190795898438,
      "activations/layer4_attention_weight_min": -103.12690734863281,
      "activations/layer5_attention_weight_max": 47.875450134277344,
      "activations/layer5_attention_weight_min": -60.679908752441406,
      "activations/layer6_attention_weight_max": 42.06077194213867,
      "activations/layer6_attention_weight_min": -44.97822952270508,
      "activations/layer7_attention_weight_max": 87.00965881347656,
      "activations/layer7_attention_weight_min": -93.08184814453125,
      "activations/layer8_attention_weight_max": 41.01103591918945,
      "activations/layer8_attention_weight_min": -42.9315299987793,
      "activations/layer9_attention_weight_max": 33.162391662597656,
      "activations/layer9_attention_weight_min": -32.96186828613281,
      "epoch": 17.2,
      "learning_rate": 3.9483333333333335e-05,
      "loss": 2.7291,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4572,
      "eval_samples_per_second": 507.735,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4572,
      "eval_openwebtext_samples_per_second": 507.735,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9547,
      "eval_wikitext_samples_per_second": 233.285,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.5299,
      "eval_lambada_samples_per_second": 510.92,
      "step": 296000
    },
    {
      "activations/layer0_attention_weight_max": 16.321138381958008,
      "activations/layer0_attention_weight_min": -13.443406105041504,
      "activations/layer10_attention_weight_max": 38.78847885131836,
      "activations/layer10_attention_weight_min": -35.604286193847656,
      "activations/layer11_attention_weight_max": 37.88274002075195,
      "activations/layer11_attention_weight_min": -38.126625061035156,
      "activations/layer12_attention_weight_max": 28.21988868713379,
      "activations/layer12_attention_weight_min": -26.83399772644043,
      "activations/layer13_attention_weight_max": 43.66224670410156,
      "activations/layer13_attention_weight_min": -37.73039627075195,
      "activations/layer14_attention_weight_max": 47.447940826416016,
      "activations/layer14_attention_weight_min": -42.20040512084961,
      "activations/layer15_attention_weight_max": 46.08843994140625,
      "activations/layer15_attention_weight_min": -38.00373458862305,
      "activations/layer16_attention_weight_max": 34.186153411865234,
      "activations/layer16_attention_weight_min": -28.730619430541992,
      "activations/layer17_attention_weight_max": 32.11250305175781,
      "activations/layer17_attention_weight_min": -27.27937126159668,
      "activations/layer18_attention_weight_max": 35.0709342956543,
      "activations/layer18_attention_weight_min": -27.52604866027832,
      "activations/layer19_attention_weight_max": 35.8322868347168,
      "activations/layer19_attention_weight_min": -33.752899169921875,
      "activations/layer1_attention_weight_max": 16.361555099487305,
      "activations/layer1_attention_weight_min": -14.812228202819824,
      "activations/layer20_attention_weight_max": 32.2215576171875,
      "activations/layer20_attention_weight_min": -29.01695442199707,
      "activations/layer21_attention_weight_max": 31.941844940185547,
      "activations/layer21_attention_weight_min": -27.153484344482422,
      "activations/layer22_attention_weight_max": 46.616451263427734,
      "activations/layer22_attention_weight_min": -34.150840759277344,
      "activations/layer23_attention_weight_max": 35.77986526489258,
      "activations/layer23_attention_weight_min": -25.748001098632812,
      "activations/layer2_attention_weight_max": 34.28567123413086,
      "activations/layer2_attention_weight_min": -32.76487731933594,
      "activations/layer3_attention_weight_max": 100.9588394165039,
      "activations/layer3_attention_weight_min": -99.87416076660156,
      "activations/layer4_attention_weight_max": 107.8135757446289,
      "activations/layer4_attention_weight_min": -108.53372955322266,
      "activations/layer5_attention_weight_max": 54.80017852783203,
      "activations/layer5_attention_weight_min": -61.14870834350586,
      "activations/layer6_attention_weight_max": 45.43429183959961,
      "activations/layer6_attention_weight_min": -46.41188430786133,
      "activations/layer7_attention_weight_max": 96.2225570678711,
      "activations/layer7_attention_weight_min": -99.39759826660156,
      "activations/layer8_attention_weight_max": 44.74116134643555,
      "activations/layer8_attention_weight_min": -44.537410736083984,
      "activations/layer9_attention_weight_max": 37.3472785949707,
      "activations/layer9_attention_weight_min": -37.15250015258789,
      "epoch": 17.2,
      "learning_rate": 3.946439393939394e-05,
      "loss": 2.7322,
      "step": 296050
    },
    {
      "activations/layer0_attention_weight_max": 16.874624252319336,
      "activations/layer0_attention_weight_min": -14.45871353149414,
      "activations/layer10_attention_weight_max": 30.45357322692871,
      "activations/layer10_attention_weight_min": -31.346172332763672,
      "activations/layer11_attention_weight_max": 30.86553192138672,
      "activations/layer11_attention_weight_min": -30.939754486083984,
      "activations/layer12_attention_weight_max": 24.690183639526367,
      "activations/layer12_attention_weight_min": -24.69558334350586,
      "activations/layer13_attention_weight_max": 36.21186447143555,
      "activations/layer13_attention_weight_min": -33.976078033447266,
      "activations/layer14_attention_weight_max": 43.5281982421875,
      "activations/layer14_attention_weight_min": -40.03212356567383,
      "activations/layer15_attention_weight_max": 36.18419647216797,
      "activations/layer15_attention_weight_min": -34.85869598388672,
      "activations/layer16_attention_weight_max": 30.203813552856445,
      "activations/layer16_attention_weight_min": -28.70266342163086,
      "activations/layer17_attention_weight_max": 27.831344604492188,
      "activations/layer17_attention_weight_min": -26.10333251953125,
      "activations/layer18_attention_weight_max": 30.852130889892578,
      "activations/layer18_attention_weight_min": -26.526920318603516,
      "activations/layer19_attention_weight_max": 33.560726165771484,
      "activations/layer19_attention_weight_min": -32.179168701171875,
      "activations/layer1_attention_weight_max": 18.077383041381836,
      "activations/layer1_attention_weight_min": -15.871541023254395,
      "activations/layer20_attention_weight_max": 28.809797286987305,
      "activations/layer20_attention_weight_min": -27.017791748046875,
      "activations/layer21_attention_weight_max": 28.355880737304688,
      "activations/layer21_attention_weight_min": -26.319137573242188,
      "activations/layer22_attention_weight_max": 44.14889907836914,
      "activations/layer22_attention_weight_min": -34.189422607421875,
      "activations/layer23_attention_weight_max": 31.752981185913086,
      "activations/layer23_attention_weight_min": -26.402080535888672,
      "activations/layer2_attention_weight_max": 34.82379913330078,
      "activations/layer2_attention_weight_min": -33.14854431152344,
      "activations/layer3_attention_weight_max": 98.67870330810547,
      "activations/layer3_attention_weight_min": -99.92386627197266,
      "activations/layer4_attention_weight_max": 103.48259735107422,
      "activations/layer4_attention_weight_min": -102.42516326904297,
      "activations/layer5_attention_weight_max": 48.45519256591797,
      "activations/layer5_attention_weight_min": -61.01472473144531,
      "activations/layer6_attention_weight_max": 42.7856330871582,
      "activations/layer6_attention_weight_min": -43.72905731201172,
      "activations/layer7_attention_weight_max": 84.11482238769531,
      "activations/layer7_attention_weight_min": -86.5081558227539,
      "activations/layer8_attention_weight_max": 37.44620132446289,
      "activations/layer8_attention_weight_min": -40.10530471801758,
      "activations/layer9_attention_weight_max": 29.622095108032227,
      "activations/layer9_attention_weight_min": -31.302249908447266,
      "epoch": 17.21,
      "learning_rate": 3.944583333333333e-05,
      "loss": 2.7417,
      "step": 296100
    },
    {
      "activations/layer0_attention_weight_max": 16.183753967285156,
      "activations/layer0_attention_weight_min": -12.249951362609863,
      "activations/layer10_attention_weight_max": 35.60487747192383,
      "activations/layer10_attention_weight_min": -35.1412467956543,
      "activations/layer11_attention_weight_max": 32.60990905761719,
      "activations/layer11_attention_weight_min": -33.645973205566406,
      "activations/layer12_attention_weight_max": 25.99799346923828,
      "activations/layer12_attention_weight_min": -26.359657287597656,
      "activations/layer13_attention_weight_max": 37.8204460144043,
      "activations/layer13_attention_weight_min": -33.340728759765625,
      "activations/layer14_attention_weight_max": 44.57978057861328,
      "activations/layer14_attention_weight_min": -39.374446868896484,
      "activations/layer15_attention_weight_max": 43.50236129760742,
      "activations/layer15_attention_weight_min": -37.64542007446289,
      "activations/layer16_attention_weight_max": 32.80385971069336,
      "activations/layer16_attention_weight_min": -27.796133041381836,
      "activations/layer17_attention_weight_max": 29.90833282470703,
      "activations/layer17_attention_weight_min": -25.176599502563477,
      "activations/layer18_attention_weight_max": 28.960861206054688,
      "activations/layer18_attention_weight_min": -24.55389976501465,
      "activations/layer19_attention_weight_max": 31.703466415405273,
      "activations/layer19_attention_weight_min": -31.360858917236328,
      "activations/layer1_attention_weight_max": 15.64929485321045,
      "activations/layer1_attention_weight_min": -15.089682579040527,
      "activations/layer20_attention_weight_max": 26.68166160583496,
      "activations/layer20_attention_weight_min": -26.14470863342285,
      "activations/layer21_attention_weight_max": 28.17184066772461,
      "activations/layer21_attention_weight_min": -24.861968994140625,
      "activations/layer22_attention_weight_max": 44.636810302734375,
      "activations/layer22_attention_weight_min": -33.31268310546875,
      "activations/layer23_attention_weight_max": 31.52023696899414,
      "activations/layer23_attention_weight_min": -26.730878829956055,
      "activations/layer2_attention_weight_max": 33.423362731933594,
      "activations/layer2_attention_weight_min": -32.73365783691406,
      "activations/layer3_attention_weight_max": 97.35669708251953,
      "activations/layer3_attention_weight_min": -98.355224609375,
      "activations/layer4_attention_weight_max": 105.78958892822266,
      "activations/layer4_attention_weight_min": -111.4324951171875,
      "activations/layer5_attention_weight_max": 51.26089096069336,
      "activations/layer5_attention_weight_min": -62.38484191894531,
      "activations/layer6_attention_weight_max": 43.3439826965332,
      "activations/layer6_attention_weight_min": -48.55023956298828,
      "activations/layer7_attention_weight_max": 95.58698272705078,
      "activations/layer7_attention_weight_min": -99.14212036132812,
      "activations/layer8_attention_weight_max": 40.1723518371582,
      "activations/layer8_attention_weight_min": -41.70431900024414,
      "activations/layer9_attention_weight_max": 31.425878524780273,
      "activations/layer9_attention_weight_min": -33.70878219604492,
      "epoch": 17.21,
      "learning_rate": 3.9426893939393936e-05,
      "loss": 2.7112,
      "step": 296150
    },
    {
      "activations/layer0_attention_weight_max": 16.501291275024414,
      "activations/layer0_attention_weight_min": -14.151484489440918,
      "activations/layer10_attention_weight_max": 33.77824020385742,
      "activations/layer10_attention_weight_min": -35.83848571777344,
      "activations/layer11_attention_weight_max": 33.57593536376953,
      "activations/layer11_attention_weight_min": -35.17671203613281,
      "activations/layer12_attention_weight_max": 26.295801162719727,
      "activations/layer12_attention_weight_min": -28.39287567138672,
      "activations/layer13_attention_weight_max": 37.59664535522461,
      "activations/layer13_attention_weight_min": -36.40947723388672,
      "activations/layer14_attention_weight_max": 42.16878128051758,
      "activations/layer14_attention_weight_min": -36.13063430786133,
      "activations/layer15_attention_weight_max": 39.31914138793945,
      "activations/layer15_attention_weight_min": -34.67488098144531,
      "activations/layer16_attention_weight_max": 28.33439064025879,
      "activations/layer16_attention_weight_min": -28.244901657104492,
      "activations/layer17_attention_weight_max": 28.961795806884766,
      "activations/layer17_attention_weight_min": -25.38759994506836,
      "activations/layer18_attention_weight_max": 33.99733352661133,
      "activations/layer18_attention_weight_min": -24.688817977905273,
      "activations/layer19_attention_weight_max": 39.33393859863281,
      "activations/layer19_attention_weight_min": -30.69252586364746,
      "activations/layer1_attention_weight_max": 17.35379409790039,
      "activations/layer1_attention_weight_min": -15.067465782165527,
      "activations/layer20_attention_weight_max": 32.764732360839844,
      "activations/layer20_attention_weight_min": -24.617673873901367,
      "activations/layer21_attention_weight_max": 31.891765594482422,
      "activations/layer21_attention_weight_min": -24.268394470214844,
      "activations/layer22_attention_weight_max": 43.97841262817383,
      "activations/layer22_attention_weight_min": -30.28692626953125,
      "activations/layer23_attention_weight_max": 33.77016830444336,
      "activations/layer23_attention_weight_min": -24.845579147338867,
      "activations/layer2_attention_weight_max": 33.457801818847656,
      "activations/layer2_attention_weight_min": -33.392906188964844,
      "activations/layer3_attention_weight_max": 101.77969360351562,
      "activations/layer3_attention_weight_min": -104.43241119384766,
      "activations/layer4_attention_weight_max": 108.46659088134766,
      "activations/layer4_attention_weight_min": -114.68685150146484,
      "activations/layer5_attention_weight_max": 56.247802734375,
      "activations/layer5_attention_weight_min": -63.33418273925781,
      "activations/layer6_attention_weight_max": 46.80876922607422,
      "activations/layer6_attention_weight_min": -48.96288299560547,
      "activations/layer7_attention_weight_max": 94.61308288574219,
      "activations/layer7_attention_weight_min": -97.17784118652344,
      "activations/layer8_attention_weight_max": 42.145957946777344,
      "activations/layer8_attention_weight_min": -45.62709426879883,
      "activations/layer9_attention_weight_max": 34.405155181884766,
      "activations/layer9_attention_weight_min": -34.89899826049805,
      "epoch": 17.21,
      "learning_rate": 3.9407954545454544e-05,
      "loss": 2.7292,
      "step": 296200
    },
    {
      "activations/layer0_attention_weight_max": 17.390295028686523,
      "activations/layer0_attention_weight_min": -13.674795150756836,
      "activations/layer10_attention_weight_max": 30.928728103637695,
      "activations/layer10_attention_weight_min": -32.013282775878906,
      "activations/layer11_attention_weight_max": 32.33208465576172,
      "activations/layer11_attention_weight_min": -33.68954086303711,
      "activations/layer12_attention_weight_max": 26.945369720458984,
      "activations/layer12_attention_weight_min": -26.113258361816406,
      "activations/layer13_attention_weight_max": 40.57761764526367,
      "activations/layer13_attention_weight_min": -36.33917236328125,
      "activations/layer14_attention_weight_max": 41.056915283203125,
      "activations/layer14_attention_weight_min": -37.83717346191406,
      "activations/layer15_attention_weight_max": 38.368255615234375,
      "activations/layer15_attention_weight_min": -35.61174392700195,
      "activations/layer16_attention_weight_max": 31.37639045715332,
      "activations/layer16_attention_weight_min": -29.01821517944336,
      "activations/layer17_attention_weight_max": 30.48387336730957,
      "activations/layer17_attention_weight_min": -26.306299209594727,
      "activations/layer18_attention_weight_max": 32.611637115478516,
      "activations/layer18_attention_weight_min": -26.594812393188477,
      "activations/layer19_attention_weight_max": 35.740352630615234,
      "activations/layer19_attention_weight_min": -35.82902145385742,
      "activations/layer1_attention_weight_max": 17.255619049072266,
      "activations/layer1_attention_weight_min": -16.58563995361328,
      "activations/layer20_attention_weight_max": 30.15530014038086,
      "activations/layer20_attention_weight_min": -27.561180114746094,
      "activations/layer21_attention_weight_max": 30.155059814453125,
      "activations/layer21_attention_weight_min": -25.97783851623535,
      "activations/layer22_attention_weight_max": 45.7717399597168,
      "activations/layer22_attention_weight_min": -35.92009735107422,
      "activations/layer23_attention_weight_max": 33.23530960083008,
      "activations/layer23_attention_weight_min": -29.227651596069336,
      "activations/layer2_attention_weight_max": 31.679588317871094,
      "activations/layer2_attention_weight_min": -32.25786590576172,
      "activations/layer3_attention_weight_max": 92.71055603027344,
      "activations/layer3_attention_weight_min": -96.35987854003906,
      "activations/layer4_attention_weight_max": 103.44464111328125,
      "activations/layer4_attention_weight_min": -103.50244140625,
      "activations/layer5_attention_weight_max": 53.565975189208984,
      "activations/layer5_attention_weight_min": -56.40653610229492,
      "activations/layer6_attention_weight_max": 41.73728561401367,
      "activations/layer6_attention_weight_min": -45.00996780395508,
      "activations/layer7_attention_weight_max": 87.43656921386719,
      "activations/layer7_attention_weight_min": -89.44789123535156,
      "activations/layer8_attention_weight_max": 38.865455627441406,
      "activations/layer8_attention_weight_min": -40.979366302490234,
      "activations/layer9_attention_weight_max": 29.921567916870117,
      "activations/layer9_attention_weight_min": -32.18507385253906,
      "epoch": 17.21,
      "learning_rate": 3.938901515151515e-05,
      "loss": 2.7206,
      "step": 296250
    },
    {
      "activations/layer0_attention_weight_max": 16.32624626159668,
      "activations/layer0_attention_weight_min": -12.336112022399902,
      "activations/layer10_attention_weight_max": 35.609039306640625,
      "activations/layer10_attention_weight_min": -35.22098159790039,
      "activations/layer11_attention_weight_max": 33.13011932373047,
      "activations/layer11_attention_weight_min": -35.11930847167969,
      "activations/layer12_attention_weight_max": 25.55605697631836,
      "activations/layer12_attention_weight_min": -26.18065643310547,
      "activations/layer13_attention_weight_max": 37.17793273925781,
      "activations/layer13_attention_weight_min": -35.44331359863281,
      "activations/layer14_attention_weight_max": 39.57870864868164,
      "activations/layer14_attention_weight_min": -36.29036331176758,
      "activations/layer15_attention_weight_max": 34.54342269897461,
      "activations/layer15_attention_weight_min": -33.59772872924805,
      "activations/layer16_attention_weight_max": 27.511056900024414,
      "activations/layer16_attention_weight_min": -27.18508529663086,
      "activations/layer17_attention_weight_max": 28.49842071533203,
      "activations/layer17_attention_weight_min": -24.490835189819336,
      "activations/layer18_attention_weight_max": 32.75477981567383,
      "activations/layer18_attention_weight_min": -25.227462768554688,
      "activations/layer19_attention_weight_max": 31.923704147338867,
      "activations/layer19_attention_weight_min": -31.007587432861328,
      "activations/layer1_attention_weight_max": 16.015666961669922,
      "activations/layer1_attention_weight_min": -17.320520401000977,
      "activations/layer20_attention_weight_max": 29.423789978027344,
      "activations/layer20_attention_weight_min": -26.111488342285156,
      "activations/layer21_attention_weight_max": 30.22162437438965,
      "activations/layer21_attention_weight_min": -24.93596649169922,
      "activations/layer22_attention_weight_max": 36.68052291870117,
      "activations/layer22_attention_weight_min": -30.3446102142334,
      "activations/layer23_attention_weight_max": 30.58034324645996,
      "activations/layer23_attention_weight_min": -24.789016723632812,
      "activations/layer2_attention_weight_max": 33.68640899658203,
      "activations/layer2_attention_weight_min": -36.52278518676758,
      "activations/layer3_attention_weight_max": 98.7296142578125,
      "activations/layer3_attention_weight_min": -102.21051788330078,
      "activations/layer4_attention_weight_max": 107.7445297241211,
      "activations/layer4_attention_weight_min": -114.89739990234375,
      "activations/layer5_attention_weight_max": 50.75240707397461,
      "activations/layer5_attention_weight_min": -67.67725372314453,
      "activations/layer6_attention_weight_max": 47.2385139465332,
      "activations/layer6_attention_weight_min": -49.928775787353516,
      "activations/layer7_attention_weight_max": 91.03633117675781,
      "activations/layer7_attention_weight_min": -100.35893249511719,
      "activations/layer8_attention_weight_max": 43.28955078125,
      "activations/layer8_attention_weight_min": -47.05336380004883,
      "activations/layer9_attention_weight_max": 33.294525146484375,
      "activations/layer9_attention_weight_min": -35.509456634521484,
      "epoch": 17.22,
      "learning_rate": 3.9370075757575754e-05,
      "loss": 2.7293,
      "step": 296300
    },
    {
      "activations/layer0_attention_weight_max": 16.242509841918945,
      "activations/layer0_attention_weight_min": -13.10551929473877,
      "activations/layer10_attention_weight_max": 34.25749206542969,
      "activations/layer10_attention_weight_min": -34.6751708984375,
      "activations/layer11_attention_weight_max": 35.23170852661133,
      "activations/layer11_attention_weight_min": -35.41603469848633,
      "activations/layer12_attention_weight_max": 28.870153427124023,
      "activations/layer12_attention_weight_min": -26.377050399780273,
      "activations/layer13_attention_weight_max": 41.28980255126953,
      "activations/layer13_attention_weight_min": -36.22639083862305,
      "activations/layer14_attention_weight_max": 45.54528045654297,
      "activations/layer14_attention_weight_min": -37.517887115478516,
      "activations/layer15_attention_weight_max": 39.041011810302734,
      "activations/layer15_attention_weight_min": -35.3012580871582,
      "activations/layer16_attention_weight_max": 31.753442764282227,
      "activations/layer16_attention_weight_min": -29.118330001831055,
      "activations/layer17_attention_weight_max": 32.11397171020508,
      "activations/layer17_attention_weight_min": -26.874160766601562,
      "activations/layer18_attention_weight_max": 34.75456237792969,
      "activations/layer18_attention_weight_min": -25.928564071655273,
      "activations/layer19_attention_weight_max": 37.47819900512695,
      "activations/layer19_attention_weight_min": -32.149986267089844,
      "activations/layer1_attention_weight_max": 17.534955978393555,
      "activations/layer1_attention_weight_min": -15.239764213562012,
      "activations/layer20_attention_weight_max": 33.88869094848633,
      "activations/layer20_attention_weight_min": -26.138200759887695,
      "activations/layer21_attention_weight_max": 35.97590255737305,
      "activations/layer21_attention_weight_min": -25.90135383605957,
      "activations/layer22_attention_weight_max": 47.5640754699707,
      "activations/layer22_attention_weight_min": -33.16049575805664,
      "activations/layer23_attention_weight_max": 37.07088851928711,
      "activations/layer23_attention_weight_min": -27.33613395690918,
      "activations/layer2_attention_weight_max": 32.48227310180664,
      "activations/layer2_attention_weight_min": -32.43836975097656,
      "activations/layer3_attention_weight_max": 95.06660461425781,
      "activations/layer3_attention_weight_min": -95.11121368408203,
      "activations/layer4_attention_weight_max": 103.1481704711914,
      "activations/layer4_attention_weight_min": -106.12711334228516,
      "activations/layer5_attention_weight_max": 48.04465866088867,
      "activations/layer5_attention_weight_min": -58.9886474609375,
      "activations/layer6_attention_weight_max": 43.9173698425293,
      "activations/layer6_attention_weight_min": -44.79480743408203,
      "activations/layer7_attention_weight_max": 88.70874786376953,
      "activations/layer7_attention_weight_min": -91.863037109375,
      "activations/layer8_attention_weight_max": 40.97322463989258,
      "activations/layer8_attention_weight_min": -43.83784866333008,
      "activations/layer9_attention_weight_max": 31.904306411743164,
      "activations/layer9_attention_weight_min": -34.588863372802734,
      "epoch": 17.22,
      "learning_rate": 3.9351136363636355e-05,
      "loss": 2.7208,
      "step": 296350
    },
    {
      "activations/layer0_attention_weight_max": 16.369903564453125,
      "activations/layer0_attention_weight_min": -15.060495376586914,
      "activations/layer10_attention_weight_max": 36.557640075683594,
      "activations/layer10_attention_weight_min": -35.07419204711914,
      "activations/layer11_attention_weight_max": 37.526344299316406,
      "activations/layer11_attention_weight_min": -36.51322937011719,
      "activations/layer12_attention_weight_max": 26.941560745239258,
      "activations/layer12_attention_weight_min": -27.337865829467773,
      "activations/layer13_attention_weight_max": 39.77519226074219,
      "activations/layer13_attention_weight_min": -36.3679084777832,
      "activations/layer14_attention_weight_max": 46.13951110839844,
      "activations/layer14_attention_weight_min": -43.43912124633789,
      "activations/layer15_attention_weight_max": 40.29545974731445,
      "activations/layer15_attention_weight_min": -37.18223571777344,
      "activations/layer16_attention_weight_max": 32.55320358276367,
      "activations/layer16_attention_weight_min": -30.036645889282227,
      "activations/layer17_attention_weight_max": 31.70974349975586,
      "activations/layer17_attention_weight_min": -28.029014587402344,
      "activations/layer18_attention_weight_max": 30.1809139251709,
      "activations/layer18_attention_weight_min": -26.593664169311523,
      "activations/layer19_attention_weight_max": 37.614906311035156,
      "activations/layer19_attention_weight_min": -30.141544342041016,
      "activations/layer1_attention_weight_max": 15.375039100646973,
      "activations/layer1_attention_weight_min": -14.036579132080078,
      "activations/layer20_attention_weight_max": 30.168561935424805,
      "activations/layer20_attention_weight_min": -24.84228515625,
      "activations/layer21_attention_weight_max": 28.266414642333984,
      "activations/layer21_attention_weight_min": -23.706933975219727,
      "activations/layer22_attention_weight_max": 42.2442626953125,
      "activations/layer22_attention_weight_min": -32.126644134521484,
      "activations/layer23_attention_weight_max": 32.888851165771484,
      "activations/layer23_attention_weight_min": -25.914127349853516,
      "activations/layer2_attention_weight_max": 31.397289276123047,
      "activations/layer2_attention_weight_min": -30.92738914489746,
      "activations/layer3_attention_weight_max": 95.17992401123047,
      "activations/layer3_attention_weight_min": -94.11133575439453,
      "activations/layer4_attention_weight_max": 108.5265121459961,
      "activations/layer4_attention_weight_min": -109.64070892333984,
      "activations/layer5_attention_weight_max": 52.3173828125,
      "activations/layer5_attention_weight_min": -61.28154754638672,
      "activations/layer6_attention_weight_max": 46.81985855102539,
      "activations/layer6_attention_weight_min": -47.25651931762695,
      "activations/layer7_attention_weight_max": 100.46713256835938,
      "activations/layer7_attention_weight_min": -97.04523468017578,
      "activations/layer8_attention_weight_max": 45.136783599853516,
      "activations/layer8_attention_weight_min": -44.51725387573242,
      "activations/layer9_attention_weight_max": 33.98743438720703,
      "activations/layer9_attention_weight_min": -34.06302261352539,
      "epoch": 17.22,
      "learning_rate": 3.933219696969697e-05,
      "loss": 2.7419,
      "step": 296400
    },
    {
      "activations/layer0_attention_weight_max": 17.132230758666992,
      "activations/layer0_attention_weight_min": -13.665648460388184,
      "activations/layer10_attention_weight_max": 34.099700927734375,
      "activations/layer10_attention_weight_min": -35.65057373046875,
      "activations/layer11_attention_weight_max": 34.867408752441406,
      "activations/layer11_attention_weight_min": -34.09391784667969,
      "activations/layer12_attention_weight_max": 26.740158081054688,
      "activations/layer12_attention_weight_min": -26.629634857177734,
      "activations/layer13_attention_weight_max": 38.540462493896484,
      "activations/layer13_attention_weight_min": -34.83100891113281,
      "activations/layer14_attention_weight_max": 38.98856735229492,
      "activations/layer14_attention_weight_min": -38.89039611816406,
      "activations/layer15_attention_weight_max": 34.937442779541016,
      "activations/layer15_attention_weight_min": -35.529605865478516,
      "activations/layer16_attention_weight_max": 27.54340934753418,
      "activations/layer16_attention_weight_min": -27.655410766601562,
      "activations/layer17_attention_weight_max": 30.58611488342285,
      "activations/layer17_attention_weight_min": -29.31275177001953,
      "activations/layer18_attention_weight_max": 32.98735046386719,
      "activations/layer18_attention_weight_min": -27.139057159423828,
      "activations/layer19_attention_weight_max": 33.439910888671875,
      "activations/layer19_attention_weight_min": -31.961002349853516,
      "activations/layer1_attention_weight_max": 15.789989471435547,
      "activations/layer1_attention_weight_min": -15.848529815673828,
      "activations/layer20_attention_weight_max": 30.237930297851562,
      "activations/layer20_attention_weight_min": -26.712980270385742,
      "activations/layer21_attention_weight_max": 28.44171714782715,
      "activations/layer21_attention_weight_min": -25.66180419921875,
      "activations/layer22_attention_weight_max": 42.40216827392578,
      "activations/layer22_attention_weight_min": -34.47013473510742,
      "activations/layer23_attention_weight_max": 30.494956970214844,
      "activations/layer23_attention_weight_min": -28.389341354370117,
      "activations/layer2_attention_weight_max": 32.759132385253906,
      "activations/layer2_attention_weight_min": -32.90592956542969,
      "activations/layer3_attention_weight_max": 99.05699920654297,
      "activations/layer3_attention_weight_min": -102.14239501953125,
      "activations/layer4_attention_weight_max": 107.77406311035156,
      "activations/layer4_attention_weight_min": -110.69795227050781,
      "activations/layer5_attention_weight_max": 50.856300354003906,
      "activations/layer5_attention_weight_min": -65.7990951538086,
      "activations/layer6_attention_weight_max": 44.28437805175781,
      "activations/layer6_attention_weight_min": -48.65739822387695,
      "activations/layer7_attention_weight_max": 92.54660034179688,
      "activations/layer7_attention_weight_min": -96.56204223632812,
      "activations/layer8_attention_weight_max": 43.8849983215332,
      "activations/layer8_attention_weight_min": -44.77849197387695,
      "activations/layer9_attention_weight_max": 33.91477584838867,
      "activations/layer9_attention_weight_min": -34.82815170288086,
      "epoch": 17.23,
      "learning_rate": 3.931325757575757e-05,
      "loss": 2.7366,
      "step": 296450
    },
    {
      "activations/layer0_attention_weight_max": 16.98661994934082,
      "activations/layer0_attention_weight_min": -13.130807876586914,
      "activations/layer10_attention_weight_max": 34.55493927001953,
      "activations/layer10_attention_weight_min": -33.778221130371094,
      "activations/layer11_attention_weight_max": 34.247764587402344,
      "activations/layer11_attention_weight_min": -35.085227966308594,
      "activations/layer12_attention_weight_max": 26.562637329101562,
      "activations/layer12_attention_weight_min": -26.288022994995117,
      "activations/layer13_attention_weight_max": 38.40525817871094,
      "activations/layer13_attention_weight_min": -35.119285583496094,
      "activations/layer14_attention_weight_max": 43.067752838134766,
      "activations/layer14_attention_weight_min": -38.316375732421875,
      "activations/layer15_attention_weight_max": 37.800804138183594,
      "activations/layer15_attention_weight_min": -35.687862396240234,
      "activations/layer16_attention_weight_max": 29.020790100097656,
      "activations/layer16_attention_weight_min": -29.715991973876953,
      "activations/layer17_attention_weight_max": 29.03743553161621,
      "activations/layer17_attention_weight_min": -25.31620216369629,
      "activations/layer18_attention_weight_max": 31.350345611572266,
      "activations/layer18_attention_weight_min": -25.958065032958984,
      "activations/layer19_attention_weight_max": 31.711181640625,
      "activations/layer19_attention_weight_min": -31.313980102539062,
      "activations/layer1_attention_weight_max": 16.929346084594727,
      "activations/layer1_attention_weight_min": -15.636990547180176,
      "activations/layer20_attention_weight_max": 28.326011657714844,
      "activations/layer20_attention_weight_min": -23.90114974975586,
      "activations/layer21_attention_weight_max": 27.59299659729004,
      "activations/layer21_attention_weight_min": -24.386980056762695,
      "activations/layer22_attention_weight_max": 39.38437271118164,
      "activations/layer22_attention_weight_min": -30.74940299987793,
      "activations/layer23_attention_weight_max": 32.31587219238281,
      "activations/layer23_attention_weight_min": -25.459075927734375,
      "activations/layer2_attention_weight_max": 33.53248977661133,
      "activations/layer2_attention_weight_min": -34.06986999511719,
      "activations/layer3_attention_weight_max": 98.72566986083984,
      "activations/layer3_attention_weight_min": -98.91970825195312,
      "activations/layer4_attention_weight_max": 104.0151596069336,
      "activations/layer4_attention_weight_min": -109.532470703125,
      "activations/layer5_attention_weight_max": 50.355506896972656,
      "activations/layer5_attention_weight_min": -62.517860412597656,
      "activations/layer6_attention_weight_max": 44.047794342041016,
      "activations/layer6_attention_weight_min": -46.28730392456055,
      "activations/layer7_attention_weight_max": 91.08655548095703,
      "activations/layer7_attention_weight_min": -93.76651000976562,
      "activations/layer8_attention_weight_max": 41.42485046386719,
      "activations/layer8_attention_weight_min": -43.7027702331543,
      "activations/layer9_attention_weight_max": 33.918357849121094,
      "activations/layer9_attention_weight_min": -33.74934005737305,
      "epoch": 17.23,
      "learning_rate": 3.9294318181818173e-05,
      "loss": 2.7498,
      "step": 296500
    },
    {
      "activations/layer0_attention_weight_max": 16.02682113647461,
      "activations/layer0_attention_weight_min": -12.728869438171387,
      "activations/layer10_attention_weight_max": 31.663522720336914,
      "activations/layer10_attention_weight_min": -33.05877685546875,
      "activations/layer11_attention_weight_max": 33.45365905761719,
      "activations/layer11_attention_weight_min": -33.54412078857422,
      "activations/layer12_attention_weight_max": 24.462711334228516,
      "activations/layer12_attention_weight_min": -25.59196662902832,
      "activations/layer13_attention_weight_max": 37.023921966552734,
      "activations/layer13_attention_weight_min": -36.46409225463867,
      "activations/layer14_attention_weight_max": 39.388423919677734,
      "activations/layer14_attention_weight_min": -36.918338775634766,
      "activations/layer15_attention_weight_max": 40.62195587158203,
      "activations/layer15_attention_weight_min": -34.00206756591797,
      "activations/layer16_attention_weight_max": 33.05550765991211,
      "activations/layer16_attention_weight_min": -27.639057159423828,
      "activations/layer17_attention_weight_max": 27.731285095214844,
      "activations/layer17_attention_weight_min": -25.423372268676758,
      "activations/layer18_attention_weight_max": 30.436973571777344,
      "activations/layer18_attention_weight_min": -25.012012481689453,
      "activations/layer19_attention_weight_max": 32.9302864074707,
      "activations/layer19_attention_weight_min": -32.26874542236328,
      "activations/layer1_attention_weight_max": 16.104475021362305,
      "activations/layer1_attention_weight_min": -15.020416259765625,
      "activations/layer20_attention_weight_max": 29.640880584716797,
      "activations/layer20_attention_weight_min": -24.67507553100586,
      "activations/layer21_attention_weight_max": 28.02764892578125,
      "activations/layer21_attention_weight_min": -23.509057998657227,
      "activations/layer22_attention_weight_max": 39.73996353149414,
      "activations/layer22_attention_weight_min": -29.95246696472168,
      "activations/layer23_attention_weight_max": 32.8075065612793,
      "activations/layer23_attention_weight_min": -27.003944396972656,
      "activations/layer2_attention_weight_max": 35.889686584472656,
      "activations/layer2_attention_weight_min": -33.30820846557617,
      "activations/layer3_attention_weight_max": 98.36449432373047,
      "activations/layer3_attention_weight_min": -100.88069152832031,
      "activations/layer4_attention_weight_max": 105.15406799316406,
      "activations/layer4_attention_weight_min": -105.98982238769531,
      "activations/layer5_attention_weight_max": 50.552459716796875,
      "activations/layer5_attention_weight_min": -59.515220642089844,
      "activations/layer6_attention_weight_max": 43.33418273925781,
      "activations/layer6_attention_weight_min": -46.73743438720703,
      "activations/layer7_attention_weight_max": 88.89787292480469,
      "activations/layer7_attention_weight_min": -89.07134246826172,
      "activations/layer8_attention_weight_max": 42.291664123535156,
      "activations/layer8_attention_weight_min": -39.88225555419922,
      "activations/layer9_attention_weight_max": 31.246200561523438,
      "activations/layer9_attention_weight_min": -32.076168060302734,
      "epoch": 17.23,
      "learning_rate": 3.927537878787878e-05,
      "loss": 2.7394,
      "step": 296550
    },
    {
      "activations/layer0_attention_weight_max": 16.338533401489258,
      "activations/layer0_attention_weight_min": -12.723102569580078,
      "activations/layer10_attention_weight_max": 32.5527458190918,
      "activations/layer10_attention_weight_min": -33.22444152832031,
      "activations/layer11_attention_weight_max": 32.77855682373047,
      "activations/layer11_attention_weight_min": -32.82767105102539,
      "activations/layer12_attention_weight_max": 26.488523483276367,
      "activations/layer12_attention_weight_min": -26.465425491333008,
      "activations/layer13_attention_weight_max": 37.77738571166992,
      "activations/layer13_attention_weight_min": -36.535945892333984,
      "activations/layer14_attention_weight_max": 41.43390655517578,
      "activations/layer14_attention_weight_min": -40.50011444091797,
      "activations/layer15_attention_weight_max": 39.934051513671875,
      "activations/layer15_attention_weight_min": -40.018089294433594,
      "activations/layer16_attention_weight_max": 31.533491134643555,
      "activations/layer16_attention_weight_min": -30.832653045654297,
      "activations/layer17_attention_weight_max": 30.19918441772461,
      "activations/layer17_attention_weight_min": -28.225269317626953,
      "activations/layer18_attention_weight_max": 29.988948822021484,
      "activations/layer18_attention_weight_min": -25.455196380615234,
      "activations/layer19_attention_weight_max": 32.88821029663086,
      "activations/layer19_attention_weight_min": -32.37893295288086,
      "activations/layer1_attention_weight_max": 16.325361251831055,
      "activations/layer1_attention_weight_min": -15.372518539428711,
      "activations/layer20_attention_weight_max": 30.328136444091797,
      "activations/layer20_attention_weight_min": -27.791791915893555,
      "activations/layer21_attention_weight_max": 30.728759765625,
      "activations/layer21_attention_weight_min": -24.907861709594727,
      "activations/layer22_attention_weight_max": 43.899227142333984,
      "activations/layer22_attention_weight_min": -32.68917465209961,
      "activations/layer23_attention_weight_max": 34.066261291503906,
      "activations/layer23_attention_weight_min": -26.815303802490234,
      "activations/layer2_attention_weight_max": 34.09590530395508,
      "activations/layer2_attention_weight_min": -33.173828125,
      "activations/layer3_attention_weight_max": 99.39971160888672,
      "activations/layer3_attention_weight_min": -103.37699127197266,
      "activations/layer4_attention_weight_max": 108.55816650390625,
      "activations/layer4_attention_weight_min": -109.87261962890625,
      "activations/layer5_attention_weight_max": 51.17377471923828,
      "activations/layer5_attention_weight_min": -63.37830352783203,
      "activations/layer6_attention_weight_max": 43.768882751464844,
      "activations/layer6_attention_weight_min": -46.18260192871094,
      "activations/layer7_attention_weight_max": 93.93621063232422,
      "activations/layer7_attention_weight_min": -95.30817413330078,
      "activations/layer8_attention_weight_max": 39.889076232910156,
      "activations/layer8_attention_weight_min": -41.355445861816406,
      "activations/layer9_attention_weight_max": 31.05451011657715,
      "activations/layer9_attention_weight_min": -33.83934783935547,
      "epoch": 17.23,
      "learning_rate": 3.925643939393939e-05,
      "loss": 2.7252,
      "step": 296600
    },
    {
      "activations/layer0_attention_weight_max": 15.340620994567871,
      "activations/layer0_attention_weight_min": -13.535253524780273,
      "activations/layer10_attention_weight_max": 35.482757568359375,
      "activations/layer10_attention_weight_min": -35.7434196472168,
      "activations/layer11_attention_weight_max": 34.379150390625,
      "activations/layer11_attention_weight_min": -35.125770568847656,
      "activations/layer12_attention_weight_max": 26.747472763061523,
      "activations/layer12_attention_weight_min": -27.353113174438477,
      "activations/layer13_attention_weight_max": 41.46802520751953,
      "activations/layer13_attention_weight_min": -37.818809509277344,
      "activations/layer14_attention_weight_max": 44.43984603881836,
      "activations/layer14_attention_weight_min": -39.74723815917969,
      "activations/layer15_attention_weight_max": 39.348541259765625,
      "activations/layer15_attention_weight_min": -38.31589889526367,
      "activations/layer16_attention_weight_max": 32.315467834472656,
      "activations/layer16_attention_weight_min": -27.11562156677246,
      "activations/layer17_attention_weight_max": 29.693410873413086,
      "activations/layer17_attention_weight_min": -26.220441818237305,
      "activations/layer18_attention_weight_max": 29.58968734741211,
      "activations/layer18_attention_weight_min": -27.065053939819336,
      "activations/layer19_attention_weight_max": 34.51863479614258,
      "activations/layer19_attention_weight_min": -31.91925621032715,
      "activations/layer1_attention_weight_max": 16.437335968017578,
      "activations/layer1_attention_weight_min": -15.792550086975098,
      "activations/layer20_attention_weight_max": 26.8404541015625,
      "activations/layer20_attention_weight_min": -25.36397933959961,
      "activations/layer21_attention_weight_max": 28.00054168701172,
      "activations/layer21_attention_weight_min": -24.900266647338867,
      "activations/layer22_attention_weight_max": 37.18376541137695,
      "activations/layer22_attention_weight_min": -31.859909057617188,
      "activations/layer23_attention_weight_max": 31.665611267089844,
      "activations/layer23_attention_weight_min": -27.501638412475586,
      "activations/layer2_attention_weight_max": 33.64970397949219,
      "activations/layer2_attention_weight_min": -33.32358169555664,
      "activations/layer3_attention_weight_max": 98.47338104248047,
      "activations/layer3_attention_weight_min": -96.68474578857422,
      "activations/layer4_attention_weight_max": 106.86223602294922,
      "activations/layer4_attention_weight_min": -107.34349822998047,
      "activations/layer5_attention_weight_max": 48.84515380859375,
      "activations/layer5_attention_weight_min": -62.097442626953125,
      "activations/layer6_attention_weight_max": 43.59547805786133,
      "activations/layer6_attention_weight_min": -47.95242691040039,
      "activations/layer7_attention_weight_max": 93.43646240234375,
      "activations/layer7_attention_weight_min": -96.91954040527344,
      "activations/layer8_attention_weight_max": 43.32862854003906,
      "activations/layer8_attention_weight_min": -44.628623962402344,
      "activations/layer9_attention_weight_max": 32.681602478027344,
      "activations/layer9_attention_weight_min": -36.41102981567383,
      "epoch": 17.24,
      "learning_rate": 3.92375e-05,
      "loss": 2.7285,
      "step": 296650
    },
    {
      "activations/layer0_attention_weight_max": 16.04538345336914,
      "activations/layer0_attention_weight_min": -12.409562110900879,
      "activations/layer10_attention_weight_max": 35.07464599609375,
      "activations/layer10_attention_weight_min": -35.019657135009766,
      "activations/layer11_attention_weight_max": 34.63706588745117,
      "activations/layer11_attention_weight_min": -34.93408966064453,
      "activations/layer12_attention_weight_max": 28.29881477355957,
      "activations/layer12_attention_weight_min": -27.0731143951416,
      "activations/layer13_attention_weight_max": 43.41975784301758,
      "activations/layer13_attention_weight_min": -37.12413787841797,
      "activations/layer14_attention_weight_max": 44.856571197509766,
      "activations/layer14_attention_weight_min": -39.706993103027344,
      "activations/layer15_attention_weight_max": 42.37565612792969,
      "activations/layer15_attention_weight_min": -37.360740661621094,
      "activations/layer16_attention_weight_max": 31.690282821655273,
      "activations/layer16_attention_weight_min": -30.301998138427734,
      "activations/layer17_attention_weight_max": 28.90013313293457,
      "activations/layer17_attention_weight_min": -29.892953872680664,
      "activations/layer18_attention_weight_max": 29.46371078491211,
      "activations/layer18_attention_weight_min": -26.77851676940918,
      "activations/layer19_attention_weight_max": 30.480871200561523,
      "activations/layer19_attention_weight_min": -32.01321792602539,
      "activations/layer1_attention_weight_max": 15.477400779724121,
      "activations/layer1_attention_weight_min": -15.196188926696777,
      "activations/layer20_attention_weight_max": 26.82842445373535,
      "activations/layer20_attention_weight_min": -25.420774459838867,
      "activations/layer21_attention_weight_max": 28.229923248291016,
      "activations/layer21_attention_weight_min": -25.64689064025879,
      "activations/layer22_attention_weight_max": 41.79414367675781,
      "activations/layer22_attention_weight_min": -30.74958038330078,
      "activations/layer23_attention_weight_max": 31.472326278686523,
      "activations/layer23_attention_weight_min": -26.445594787597656,
      "activations/layer2_attention_weight_max": 33.24659729003906,
      "activations/layer2_attention_weight_min": -33.50032043457031,
      "activations/layer3_attention_weight_max": 98.57977294921875,
      "activations/layer3_attention_weight_min": -101.03325653076172,
      "activations/layer4_attention_weight_max": 109.2673110961914,
      "activations/layer4_attention_weight_min": -107.87923431396484,
      "activations/layer5_attention_weight_max": 49.845863342285156,
      "activations/layer5_attention_weight_min": -61.57537078857422,
      "activations/layer6_attention_weight_max": 44.03473663330078,
      "activations/layer6_attention_weight_min": -45.778316497802734,
      "activations/layer7_attention_weight_max": 92.07049560546875,
      "activations/layer7_attention_weight_min": -97.75542449951172,
      "activations/layer8_attention_weight_max": 41.64844512939453,
      "activations/layer8_attention_weight_min": -46.467124938964844,
      "activations/layer9_attention_weight_max": 33.63227081298828,
      "activations/layer9_attention_weight_min": -34.1885871887207,
      "epoch": 17.24,
      "learning_rate": 3.92185606060606e-05,
      "loss": 2.7432,
      "step": 296700
    },
    {
      "activations/layer0_attention_weight_max": 16.621139526367188,
      "activations/layer0_attention_weight_min": -12.863536834716797,
      "activations/layer10_attention_weight_max": 33.46113586425781,
      "activations/layer10_attention_weight_min": -32.143959045410156,
      "activations/layer11_attention_weight_max": 33.14218521118164,
      "activations/layer11_attention_weight_min": -34.98821258544922,
      "activations/layer12_attention_weight_max": 27.796812057495117,
      "activations/layer12_attention_weight_min": -25.93165397644043,
      "activations/layer13_attention_weight_max": 42.23948287963867,
      "activations/layer13_attention_weight_min": -35.59876251220703,
      "activations/layer14_attention_weight_max": 43.308021545410156,
      "activations/layer14_attention_weight_min": -38.38629913330078,
      "activations/layer15_attention_weight_max": 41.94598388671875,
      "activations/layer15_attention_weight_min": -35.14177703857422,
      "activations/layer16_attention_weight_max": 28.989656448364258,
      "activations/layer16_attention_weight_min": -26.796613693237305,
      "activations/layer17_attention_weight_max": 29.97304344177246,
      "activations/layer17_attention_weight_min": -24.341768264770508,
      "activations/layer18_attention_weight_max": 31.283302307128906,
      "activations/layer18_attention_weight_min": -26.457456588745117,
      "activations/layer19_attention_weight_max": 33.0004768371582,
      "activations/layer19_attention_weight_min": -30.69692039489746,
      "activations/layer1_attention_weight_max": 16.32481575012207,
      "activations/layer1_attention_weight_min": -14.2567777633667,
      "activations/layer20_attention_weight_max": 29.501829147338867,
      "activations/layer20_attention_weight_min": -24.844526290893555,
      "activations/layer21_attention_weight_max": 30.357702255249023,
      "activations/layer21_attention_weight_min": -24.40610122680664,
      "activations/layer22_attention_weight_max": 47.14621353149414,
      "activations/layer22_attention_weight_min": -30.72133445739746,
      "activations/layer23_attention_weight_max": 32.14286422729492,
      "activations/layer23_attention_weight_min": -27.89118766784668,
      "activations/layer2_attention_weight_max": 32.82607650756836,
      "activations/layer2_attention_weight_min": -31.24481201171875,
      "activations/layer3_attention_weight_max": 97.19339752197266,
      "activations/layer3_attention_weight_min": -94.04585266113281,
      "activations/layer4_attention_weight_max": 102.23727416992188,
      "activations/layer4_attention_weight_min": -105.23912811279297,
      "activations/layer5_attention_weight_max": 47.40922546386719,
      "activations/layer5_attention_weight_min": -59.35102462768555,
      "activations/layer6_attention_weight_max": 43.56355285644531,
      "activations/layer6_attention_weight_min": -45.2612190246582,
      "activations/layer7_attention_weight_max": 92.67757415771484,
      "activations/layer7_attention_weight_min": -92.55438995361328,
      "activations/layer8_attention_weight_max": 40.47050476074219,
      "activations/layer8_attention_weight_min": -40.88722229003906,
      "activations/layer9_attention_weight_max": 31.563556671142578,
      "activations/layer9_attention_weight_min": -31.833694458007812,
      "epoch": 17.24,
      "learning_rate": 3.9199621212121215e-05,
      "loss": 2.744,
      "step": 296750
    },
    {
      "activations/layer0_attention_weight_max": 16.01433753967285,
      "activations/layer0_attention_weight_min": -13.13201904296875,
      "activations/layer10_attention_weight_max": 35.28528594970703,
      "activations/layer10_attention_weight_min": -33.10779571533203,
      "activations/layer11_attention_weight_max": 35.410335540771484,
      "activations/layer11_attention_weight_min": -33.27750778198242,
      "activations/layer12_attention_weight_max": 26.00173568725586,
      "activations/layer12_attention_weight_min": -28.061466217041016,
      "activations/layer13_attention_weight_max": 36.819366455078125,
      "activations/layer13_attention_weight_min": -38.437782287597656,
      "activations/layer14_attention_weight_max": 46.92865753173828,
      "activations/layer14_attention_weight_min": -40.978553771972656,
      "activations/layer15_attention_weight_max": 36.899330139160156,
      "activations/layer15_attention_weight_min": -37.45157241821289,
      "activations/layer16_attention_weight_max": 31.290422439575195,
      "activations/layer16_attention_weight_min": -27.6246280670166,
      "activations/layer17_attention_weight_max": 30.0871524810791,
      "activations/layer17_attention_weight_min": -28.96705436706543,
      "activations/layer18_attention_weight_max": 32.579837799072266,
      "activations/layer18_attention_weight_min": -30.39940071105957,
      "activations/layer19_attention_weight_max": 35.67654037475586,
      "activations/layer19_attention_weight_min": -33.44792175292969,
      "activations/layer1_attention_weight_max": 16.5816650390625,
      "activations/layer1_attention_weight_min": -15.672089576721191,
      "activations/layer20_attention_weight_max": 31.167308807373047,
      "activations/layer20_attention_weight_min": -26.338956832885742,
      "activations/layer21_attention_weight_max": 31.088029861450195,
      "activations/layer21_attention_weight_min": -25.729598999023438,
      "activations/layer22_attention_weight_max": 46.77076721191406,
      "activations/layer22_attention_weight_min": -33.31392288208008,
      "activations/layer23_attention_weight_max": 37.53116226196289,
      "activations/layer23_attention_weight_min": -26.168140411376953,
      "activations/layer2_attention_weight_max": 33.40190887451172,
      "activations/layer2_attention_weight_min": -32.61354064941406,
      "activations/layer3_attention_weight_max": 97.0797348022461,
      "activations/layer3_attention_weight_min": -97.0044174194336,
      "activations/layer4_attention_weight_max": 107.1490707397461,
      "activations/layer4_attention_weight_min": -108.21195220947266,
      "activations/layer5_attention_weight_max": 49.74626922607422,
      "activations/layer5_attention_weight_min": -64.10032653808594,
      "activations/layer6_attention_weight_max": 44.63438034057617,
      "activations/layer6_attention_weight_min": -48.79722213745117,
      "activations/layer7_attention_weight_max": 90.28076171875,
      "activations/layer7_attention_weight_min": -93.58560943603516,
      "activations/layer8_attention_weight_max": 41.648651123046875,
      "activations/layer8_attention_weight_min": -41.33464813232422,
      "activations/layer9_attention_weight_max": 31.770694732666016,
      "activations/layer9_attention_weight_min": -33.10175704956055,
      "epoch": 17.25,
      "learning_rate": 3.9180681818181816e-05,
      "loss": 2.7406,
      "step": 296800
    },
    {
      "activations/layer0_attention_weight_max": 15.9911470413208,
      "activations/layer0_attention_weight_min": -12.531935691833496,
      "activations/layer10_attention_weight_max": 33.93537139892578,
      "activations/layer10_attention_weight_min": -31.959671020507812,
      "activations/layer11_attention_weight_max": 35.4483642578125,
      "activations/layer11_attention_weight_min": -33.465179443359375,
      "activations/layer12_attention_weight_max": 27.479902267456055,
      "activations/layer12_attention_weight_min": -29.075037002563477,
      "activations/layer13_attention_weight_max": 43.76826858520508,
      "activations/layer13_attention_weight_min": -30.920217514038086,
      "activations/layer14_attention_weight_max": 38.85871124267578,
      "activations/layer14_attention_weight_min": -34.2011833190918,
      "activations/layer15_attention_weight_max": 38.747703552246094,
      "activations/layer15_attention_weight_min": -35.69086456298828,
      "activations/layer16_attention_weight_max": 30.168359756469727,
      "activations/layer16_attention_weight_min": -26.320846557617188,
      "activations/layer17_attention_weight_max": 27.418725967407227,
      "activations/layer17_attention_weight_min": -25.46068572998047,
      "activations/layer18_attention_weight_max": 25.933408737182617,
      "activations/layer18_attention_weight_min": -26.713184356689453,
      "activations/layer19_attention_weight_max": 31.68216323852539,
      "activations/layer19_attention_weight_min": -29.696063995361328,
      "activations/layer1_attention_weight_max": 17.400190353393555,
      "activations/layer1_attention_weight_min": -14.971126556396484,
      "activations/layer20_attention_weight_max": 27.396583557128906,
      "activations/layer20_attention_weight_min": -23.232213973999023,
      "activations/layer21_attention_weight_max": 25.49130630493164,
      "activations/layer21_attention_weight_min": -23.36513328552246,
      "activations/layer22_attention_weight_max": 39.88449478149414,
      "activations/layer22_attention_weight_min": -34.100406646728516,
      "activations/layer23_attention_weight_max": 29.55067253112793,
      "activations/layer23_attention_weight_min": -25.062467575073242,
      "activations/layer2_attention_weight_max": 38.14511489868164,
      "activations/layer2_attention_weight_min": -37.66545867919922,
      "activations/layer3_attention_weight_max": 108.70492553710938,
      "activations/layer3_attention_weight_min": -104.07742309570312,
      "activations/layer4_attention_weight_max": 112.17658996582031,
      "activations/layer4_attention_weight_min": -108.1969223022461,
      "activations/layer5_attention_weight_max": 52.615760803222656,
      "activations/layer5_attention_weight_min": -63.60451889038086,
      "activations/layer6_attention_weight_max": 42.269691467285156,
      "activations/layer6_attention_weight_min": -43.96332931518555,
      "activations/layer7_attention_weight_max": 97.62561798095703,
      "activations/layer7_attention_weight_min": -93.64583587646484,
      "activations/layer8_attention_weight_max": 42.81205749511719,
      "activations/layer8_attention_weight_min": -41.97052001953125,
      "activations/layer9_attention_weight_max": 29.191129684448242,
      "activations/layer9_attention_weight_min": -32.76398849487305,
      "epoch": 17.25,
      "learning_rate": 3.916174242424242e-05,
      "loss": 2.726,
      "step": 296850
    },
    {
      "activations/layer0_attention_weight_max": 16.5950927734375,
      "activations/layer0_attention_weight_min": -13.276453971862793,
      "activations/layer10_attention_weight_max": 35.78295135498047,
      "activations/layer10_attention_weight_min": -37.2018928527832,
      "activations/layer11_attention_weight_max": 34.667762756347656,
      "activations/layer11_attention_weight_min": -35.689456939697266,
      "activations/layer12_attention_weight_max": 27.13440704345703,
      "activations/layer12_attention_weight_min": -25.501691818237305,
      "activations/layer13_attention_weight_max": 40.196258544921875,
      "activations/layer13_attention_weight_min": -35.67100143432617,
      "activations/layer14_attention_weight_max": 41.174400329589844,
      "activations/layer14_attention_weight_min": -37.78108215332031,
      "activations/layer15_attention_weight_max": 39.99800491333008,
      "activations/layer15_attention_weight_min": -34.14432907104492,
      "activations/layer16_attention_weight_max": 31.359647750854492,
      "activations/layer16_attention_weight_min": -27.759693145751953,
      "activations/layer17_attention_weight_max": 28.975847244262695,
      "activations/layer17_attention_weight_min": -24.685482025146484,
      "activations/layer18_attention_weight_max": 32.79001998901367,
      "activations/layer18_attention_weight_min": -26.35527801513672,
      "activations/layer19_attention_weight_max": 32.358150482177734,
      "activations/layer19_attention_weight_min": -29.468114852905273,
      "activations/layer1_attention_weight_max": 15.648637771606445,
      "activations/layer1_attention_weight_min": -15.317906379699707,
      "activations/layer20_attention_weight_max": 28.92458152770996,
      "activations/layer20_attention_weight_min": -24.840774536132812,
      "activations/layer21_attention_weight_max": 30.107450485229492,
      "activations/layer21_attention_weight_min": -24.60832977294922,
      "activations/layer22_attention_weight_max": 43.86881637573242,
      "activations/layer22_attention_weight_min": -30.8154354095459,
      "activations/layer23_attention_weight_max": 34.91978073120117,
      "activations/layer23_attention_weight_min": -27.017311096191406,
      "activations/layer2_attention_weight_max": 32.304752349853516,
      "activations/layer2_attention_weight_min": -32.48444366455078,
      "activations/layer3_attention_weight_max": 98.81672668457031,
      "activations/layer3_attention_weight_min": -98.8794174194336,
      "activations/layer4_attention_weight_max": 108.40670013427734,
      "activations/layer4_attention_weight_min": -110.4816665649414,
      "activations/layer5_attention_weight_max": 49.67135238647461,
      "activations/layer5_attention_weight_min": -62.370521545410156,
      "activations/layer6_attention_weight_max": 44.12424850463867,
      "activations/layer6_attention_weight_min": -45.338584899902344,
      "activations/layer7_attention_weight_max": 86.20130157470703,
      "activations/layer7_attention_weight_min": -91.30587768554688,
      "activations/layer8_attention_weight_max": 41.10334777832031,
      "activations/layer8_attention_weight_min": -42.88774490356445,
      "activations/layer9_attention_weight_max": 33.21116256713867,
      "activations/layer9_attention_weight_min": -34.56669235229492,
      "epoch": 17.25,
      "learning_rate": 3.9142803030303026e-05,
      "loss": 2.7492,
      "step": 296900
    },
    {
      "activations/layer0_attention_weight_max": 15.858384132385254,
      "activations/layer0_attention_weight_min": -15.927006721496582,
      "activations/layer10_attention_weight_max": 35.36558151245117,
      "activations/layer10_attention_weight_min": -33.17499542236328,
      "activations/layer11_attention_weight_max": 37.87969970703125,
      "activations/layer11_attention_weight_min": -33.70499038696289,
      "activations/layer12_attention_weight_max": 27.458053588867188,
      "activations/layer12_attention_weight_min": -28.0755672454834,
      "activations/layer13_attention_weight_max": 39.13182067871094,
      "activations/layer13_attention_weight_min": -37.16402816772461,
      "activations/layer14_attention_weight_max": 44.165443420410156,
      "activations/layer14_attention_weight_min": -37.6475944519043,
      "activations/layer15_attention_weight_max": 42.084754943847656,
      "activations/layer15_attention_weight_min": -36.2769889831543,
      "activations/layer16_attention_weight_max": 32.40159225463867,
      "activations/layer16_attention_weight_min": -28.608604431152344,
      "activations/layer17_attention_weight_max": 30.546281814575195,
      "activations/layer17_attention_weight_min": -26.200159072875977,
      "activations/layer18_attention_weight_max": 28.486392974853516,
      "activations/layer18_attention_weight_min": -26.72355842590332,
      "activations/layer19_attention_weight_max": 35.14311981201172,
      "activations/layer19_attention_weight_min": -32.694358825683594,
      "activations/layer1_attention_weight_max": 16.816617965698242,
      "activations/layer1_attention_weight_min": -15.927286148071289,
      "activations/layer20_attention_weight_max": 29.390256881713867,
      "activations/layer20_attention_weight_min": -26.413972854614258,
      "activations/layer21_attention_weight_max": 27.991952896118164,
      "activations/layer21_attention_weight_min": -27.388193130493164,
      "activations/layer22_attention_weight_max": 40.44383239746094,
      "activations/layer22_attention_weight_min": -30.639272689819336,
      "activations/layer23_attention_weight_max": 31.516376495361328,
      "activations/layer23_attention_weight_min": -25.27463150024414,
      "activations/layer2_attention_weight_max": 34.34651565551758,
      "activations/layer2_attention_weight_min": -33.816078186035156,
      "activations/layer3_attention_weight_max": 98.48439025878906,
      "activations/layer3_attention_weight_min": -103.18816375732422,
      "activations/layer4_attention_weight_max": 105.20367431640625,
      "activations/layer4_attention_weight_min": -109.6570816040039,
      "activations/layer5_attention_weight_max": 50.59442138671875,
      "activations/layer5_attention_weight_min": -60.51231384277344,
      "activations/layer6_attention_weight_max": 43.683448791503906,
      "activations/layer6_attention_weight_min": -46.70077896118164,
      "activations/layer7_attention_weight_max": 92.20255279541016,
      "activations/layer7_attention_weight_min": -94.14836883544922,
      "activations/layer8_attention_weight_max": 42.76316452026367,
      "activations/layer8_attention_weight_min": -41.84923553466797,
      "activations/layer9_attention_weight_max": 31.569976806640625,
      "activations/layer9_attention_weight_min": -34.10635757446289,
      "epoch": 17.25,
      "learning_rate": 3.9123863636363634e-05,
      "loss": 2.7205,
      "step": 296950
    },
    {
      "activations/layer0_attention_weight_max": 15.364253997802734,
      "activations/layer0_attention_weight_min": -12.685676574707031,
      "activations/layer10_attention_weight_max": 31.696533203125,
      "activations/layer10_attention_weight_min": -34.1612548828125,
      "activations/layer11_attention_weight_max": 31.311294555664062,
      "activations/layer11_attention_weight_min": -33.16595458984375,
      "activations/layer12_attention_weight_max": 28.4354190826416,
      "activations/layer12_attention_weight_min": -25.521438598632812,
      "activations/layer13_attention_weight_max": 40.513275146484375,
      "activations/layer13_attention_weight_min": -36.409278869628906,
      "activations/layer14_attention_weight_max": 40.86564254760742,
      "activations/layer14_attention_weight_min": -37.392032623291016,
      "activations/layer15_attention_weight_max": 36.24693298339844,
      "activations/layer15_attention_weight_min": -35.40226745605469,
      "activations/layer16_attention_weight_max": 29.879858016967773,
      "activations/layer16_attention_weight_min": -28.10826301574707,
      "activations/layer17_attention_weight_max": 30.120126724243164,
      "activations/layer17_attention_weight_min": -26.865741729736328,
      "activations/layer18_attention_weight_max": 31.483985900878906,
      "activations/layer18_attention_weight_min": -25.80502700805664,
      "activations/layer19_attention_weight_max": 33.323116302490234,
      "activations/layer19_attention_weight_min": -29.758563995361328,
      "activations/layer1_attention_weight_max": 17.372568130493164,
      "activations/layer1_attention_weight_min": -16.110191345214844,
      "activations/layer20_attention_weight_max": 31.049816131591797,
      "activations/layer20_attention_weight_min": -23.64276885986328,
      "activations/layer21_attention_weight_max": 29.403947830200195,
      "activations/layer21_attention_weight_min": -22.626312255859375,
      "activations/layer22_attention_weight_max": 39.41597366333008,
      "activations/layer22_attention_weight_min": -30.144855499267578,
      "activations/layer23_attention_weight_max": 34.15607833862305,
      "activations/layer23_attention_weight_min": -25.383098602294922,
      "activations/layer2_attention_weight_max": 34.95126724243164,
      "activations/layer2_attention_weight_min": -33.588783264160156,
      "activations/layer3_attention_weight_max": 98.10693359375,
      "activations/layer3_attention_weight_min": -98.13754272460938,
      "activations/layer4_attention_weight_max": 105.97796630859375,
      "activations/layer4_attention_weight_min": -109.09674072265625,
      "activations/layer5_attention_weight_max": 51.327510833740234,
      "activations/layer5_attention_weight_min": -64.78504943847656,
      "activations/layer6_attention_weight_max": 43.180747985839844,
      "activations/layer6_attention_weight_min": -44.999759674072266,
      "activations/layer7_attention_weight_max": 94.17603302001953,
      "activations/layer7_attention_weight_min": -93.10261535644531,
      "activations/layer8_attention_weight_max": 40.13145065307617,
      "activations/layer8_attention_weight_min": -43.9205207824707,
      "activations/layer9_attention_weight_max": 30.804086685180664,
      "activations/layer9_attention_weight_min": -34.445980072021484,
      "epoch": 17.26,
      "learning_rate": 3.910492424242424e-05,
      "loss": 2.7245,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4637,
      "eval_samples_per_second": 507.346,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4637,
      "eval_openwebtext_samples_per_second": 507.346,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9563,
      "eval_wikitext_samples_per_second": 233.09,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_lambada_loss": 2.392578125,
      "eval_lambada_ppl": 10.941666595318791,
      "eval_lambada_runtime": 9.5385,
      "eval_lambada_samples_per_second": 510.459,
      "step": 297000
    },
    {
      "activations/layer0_attention_weight_max": 16.26007652282715,
      "activations/layer0_attention_weight_min": -12.837785720825195,
      "activations/layer10_attention_weight_max": 33.87269973754883,
      "activations/layer10_attention_weight_min": -35.942352294921875,
      "activations/layer11_attention_weight_max": 33.66022491455078,
      "activations/layer11_attention_weight_min": -35.140892028808594,
      "activations/layer12_attention_weight_max": 28.224746704101562,
      "activations/layer12_attention_weight_min": -27.031333923339844,
      "activations/layer13_attention_weight_max": 43.018001556396484,
      "activations/layer13_attention_weight_min": -36.77592849731445,
      "activations/layer14_attention_weight_max": 42.29111862182617,
      "activations/layer14_attention_weight_min": -38.16786575317383,
      "activations/layer15_attention_weight_max": 42.226654052734375,
      "activations/layer15_attention_weight_min": -37.70290756225586,
      "activations/layer16_attention_weight_max": 27.883420944213867,
      "activations/layer16_attention_weight_min": -26.95229148864746,
      "activations/layer17_attention_weight_max": 30.130746841430664,
      "activations/layer17_attention_weight_min": -25.39963722229004,
      "activations/layer18_attention_weight_max": 30.003759384155273,
      "activations/layer18_attention_weight_min": -24.47028350830078,
      "activations/layer19_attention_weight_max": 35.329036712646484,
      "activations/layer19_attention_weight_min": -31.595108032226562,
      "activations/layer1_attention_weight_max": 15.303168296813965,
      "activations/layer1_attention_weight_min": -14.824122428894043,
      "activations/layer20_attention_weight_max": 33.54254150390625,
      "activations/layer20_attention_weight_min": -26.09896469116211,
      "activations/layer21_attention_weight_max": 32.5320930480957,
      "activations/layer21_attention_weight_min": -26.556100845336914,
      "activations/layer22_attention_weight_max": 39.99524688720703,
      "activations/layer22_attention_weight_min": -34.02964401245117,
      "activations/layer23_attention_weight_max": 32.20273971557617,
      "activations/layer23_attention_weight_min": -24.63177490234375,
      "activations/layer2_attention_weight_max": 32.9722785949707,
      "activations/layer2_attention_weight_min": -32.28236389160156,
      "activations/layer3_attention_weight_max": 100.27949523925781,
      "activations/layer3_attention_weight_min": -99.36833190917969,
      "activations/layer4_attention_weight_max": 105.04180908203125,
      "activations/layer4_attention_weight_min": -106.72309875488281,
      "activations/layer5_attention_weight_max": 51.53907012939453,
      "activations/layer5_attention_weight_min": -62.26042175292969,
      "activations/layer6_attention_weight_max": 45.68733215332031,
      "activations/layer6_attention_weight_min": -48.82734298706055,
      "activations/layer7_attention_weight_max": 94.89002990722656,
      "activations/layer7_attention_weight_min": -101.5457763671875,
      "activations/layer8_attention_weight_max": 41.793731689453125,
      "activations/layer8_attention_weight_min": -48.452842712402344,
      "activations/layer9_attention_weight_max": 33.334922790527344,
      "activations/layer9_attention_weight_min": -34.821510314941406,
      "epoch": 17.26,
      "learning_rate": 3.9085984848484844e-05,
      "loss": 2.715,
      "step": 297050
    },
    {
      "activations/layer0_attention_weight_max": 16.253559112548828,
      "activations/layer0_attention_weight_min": -13.759431838989258,
      "activations/layer10_attention_weight_max": 39.09830856323242,
      "activations/layer10_attention_weight_min": -36.27591323852539,
      "activations/layer11_attention_weight_max": 38.23869323730469,
      "activations/layer11_attention_weight_min": -35.40186309814453,
      "activations/layer12_attention_weight_max": 29.912555694580078,
      "activations/layer12_attention_weight_min": -27.864599227905273,
      "activations/layer13_attention_weight_max": 43.62074279785156,
      "activations/layer13_attention_weight_min": -37.00093078613281,
      "activations/layer14_attention_weight_max": 49.010902404785156,
      "activations/layer14_attention_weight_min": -38.677371978759766,
      "activations/layer15_attention_weight_max": 40.91499710083008,
      "activations/layer15_attention_weight_min": -34.13437271118164,
      "activations/layer16_attention_weight_max": 30.626909255981445,
      "activations/layer16_attention_weight_min": -27.797626495361328,
      "activations/layer17_attention_weight_max": 32.84437561035156,
      "activations/layer17_attention_weight_min": -24.550817489624023,
      "activations/layer18_attention_weight_max": 32.251522064208984,
      "activations/layer18_attention_weight_min": -25.709936141967773,
      "activations/layer19_attention_weight_max": 37.382232666015625,
      "activations/layer19_attention_weight_min": -33.29581832885742,
      "activations/layer1_attention_weight_max": 15.963029861450195,
      "activations/layer1_attention_weight_min": -14.856550216674805,
      "activations/layer20_attention_weight_max": 30.124521255493164,
      "activations/layer20_attention_weight_min": -26.024375915527344,
      "activations/layer21_attention_weight_max": 31.24530601501465,
      "activations/layer21_attention_weight_min": -25.761465072631836,
      "activations/layer22_attention_weight_max": 46.849266052246094,
      "activations/layer22_attention_weight_min": -35.67686080932617,
      "activations/layer23_attention_weight_max": 38.923988342285156,
      "activations/layer23_attention_weight_min": -24.268051147460938,
      "activations/layer2_attention_weight_max": 33.74616241455078,
      "activations/layer2_attention_weight_min": -33.888938903808594,
      "activations/layer3_attention_weight_max": 101.93453216552734,
      "activations/layer3_attention_weight_min": -99.93873596191406,
      "activations/layer4_attention_weight_max": 109.50040435791016,
      "activations/layer4_attention_weight_min": -104.25121307373047,
      "activations/layer5_attention_weight_max": 48.873687744140625,
      "activations/layer5_attention_weight_min": -63.660701751708984,
      "activations/layer6_attention_weight_max": 45.75992965698242,
      "activations/layer6_attention_weight_min": -45.471866607666016,
      "activations/layer7_attention_weight_max": 93.6597671508789,
      "activations/layer7_attention_weight_min": -98.29232788085938,
      "activations/layer8_attention_weight_max": 43.623252868652344,
      "activations/layer8_attention_weight_min": -45.21256637573242,
      "activations/layer9_attention_weight_max": 36.292720794677734,
      "activations/layer9_attention_weight_min": -36.66720199584961,
      "epoch": 17.26,
      "learning_rate": 3.9067045454545446e-05,
      "loss": 2.7479,
      "step": 297100
    },
    {
      "activations/layer0_attention_weight_max": 17.189565658569336,
      "activations/layer0_attention_weight_min": -12.872739791870117,
      "activations/layer10_attention_weight_max": 39.44334411621094,
      "activations/layer10_attention_weight_min": -38.34945297241211,
      "activations/layer11_attention_weight_max": 39.79405212402344,
      "activations/layer11_attention_weight_min": -38.35064697265625,
      "activations/layer12_attention_weight_max": 30.278963088989258,
      "activations/layer12_attention_weight_min": -29.164987564086914,
      "activations/layer13_attention_weight_max": 45.86989212036133,
      "activations/layer13_attention_weight_min": -40.00994873046875,
      "activations/layer14_attention_weight_max": 52.52602005004883,
      "activations/layer14_attention_weight_min": -43.467628479003906,
      "activations/layer15_attention_weight_max": 46.2598876953125,
      "activations/layer15_attention_weight_min": -41.98869323730469,
      "activations/layer16_attention_weight_max": 36.06583786010742,
      "activations/layer16_attention_weight_min": -29.596763610839844,
      "activations/layer17_attention_weight_max": 35.87297058105469,
      "activations/layer17_attention_weight_min": -28.786964416503906,
      "activations/layer18_attention_weight_max": 36.71870422363281,
      "activations/layer18_attention_weight_min": -27.378507614135742,
      "activations/layer19_attention_weight_max": 40.98537826538086,
      "activations/layer19_attention_weight_min": -35.90278244018555,
      "activations/layer1_attention_weight_max": 16.08518409729004,
      "activations/layer1_attention_weight_min": -14.390239715576172,
      "activations/layer20_attention_weight_max": 34.78632736206055,
      "activations/layer20_attention_weight_min": -27.296968460083008,
      "activations/layer21_attention_weight_max": 33.86884689331055,
      "activations/layer21_attention_weight_min": -25.368825912475586,
      "activations/layer22_attention_weight_max": 49.7195930480957,
      "activations/layer22_attention_weight_min": -35.20236587524414,
      "activations/layer23_attention_weight_max": 42.91184616088867,
      "activations/layer23_attention_weight_min": -29.0091609954834,
      "activations/layer2_attention_weight_max": 33.23865509033203,
      "activations/layer2_attention_weight_min": -33.67093276977539,
      "activations/layer3_attention_weight_max": 99.87694549560547,
      "activations/layer3_attention_weight_min": -103.35066223144531,
      "activations/layer4_attention_weight_max": 106.69905853271484,
      "activations/layer4_attention_weight_min": -106.45708465576172,
      "activations/layer5_attention_weight_max": 50.86033630371094,
      "activations/layer5_attention_weight_min": -60.470619201660156,
      "activations/layer6_attention_weight_max": 43.55337142944336,
      "activations/layer6_attention_weight_min": -44.56992721557617,
      "activations/layer7_attention_weight_max": 93.7584457397461,
      "activations/layer7_attention_weight_min": -94.35872650146484,
      "activations/layer8_attention_weight_max": 42.04658126831055,
      "activations/layer8_attention_weight_min": -43.42879867553711,
      "activations/layer9_attention_weight_max": 35.89493942260742,
      "activations/layer9_attention_weight_min": -37.191768646240234,
      "epoch": 17.27,
      "learning_rate": 3.904810606060606e-05,
      "loss": 2.7427,
      "step": 297150
    },
    {
      "activations/layer0_attention_weight_max": 16.086929321289062,
      "activations/layer0_attention_weight_min": -12.026411056518555,
      "activations/layer10_attention_weight_max": 36.330711364746094,
      "activations/layer10_attention_weight_min": -36.68693161010742,
      "activations/layer11_attention_weight_max": 35.093414306640625,
      "activations/layer11_attention_weight_min": -34.89492416381836,
      "activations/layer12_attention_weight_max": 27.381893157958984,
      "activations/layer12_attention_weight_min": -25.303361892700195,
      "activations/layer13_attention_weight_max": 41.57887268066406,
      "activations/layer13_attention_weight_min": -34.999786376953125,
      "activations/layer14_attention_weight_max": 44.983707427978516,
      "activations/layer14_attention_weight_min": -40.18566131591797,
      "activations/layer15_attention_weight_max": 42.19401168823242,
      "activations/layer15_attention_weight_min": -37.36129379272461,
      "activations/layer16_attention_weight_max": 31.727832794189453,
      "activations/layer16_attention_weight_min": -27.29612159729004,
      "activations/layer17_attention_weight_max": 32.16661071777344,
      "activations/layer17_attention_weight_min": -25.505271911621094,
      "activations/layer18_attention_weight_max": 31.388893127441406,
      "activations/layer18_attention_weight_min": -26.332124710083008,
      "activations/layer19_attention_weight_max": 31.8710994720459,
      "activations/layer19_attention_weight_min": -30.6088924407959,
      "activations/layer1_attention_weight_max": 15.826554298400879,
      "activations/layer1_attention_weight_min": -14.869893074035645,
      "activations/layer20_attention_weight_max": 26.913854598999023,
      "activations/layer20_attention_weight_min": -24.907854080200195,
      "activations/layer21_attention_weight_max": 27.380643844604492,
      "activations/layer21_attention_weight_min": -24.273361206054688,
      "activations/layer22_attention_weight_max": 43.48518371582031,
      "activations/layer22_attention_weight_min": -31.66844367980957,
      "activations/layer23_attention_weight_max": 32.7602424621582,
      "activations/layer23_attention_weight_min": -25.70745086669922,
      "activations/layer2_attention_weight_max": 33.08427429199219,
      "activations/layer2_attention_weight_min": -33.7313346862793,
      "activations/layer3_attention_weight_max": 101.61106872558594,
      "activations/layer3_attention_weight_min": -102.48887634277344,
      "activations/layer4_attention_weight_max": 110.5254135131836,
      "activations/layer4_attention_weight_min": -109.94325256347656,
      "activations/layer5_attention_weight_max": 51.76275634765625,
      "activations/layer5_attention_weight_min": -61.399375915527344,
      "activations/layer6_attention_weight_max": 45.67802047729492,
      "activations/layer6_attention_weight_min": -46.834678649902344,
      "activations/layer7_attention_weight_max": 91.73693084716797,
      "activations/layer7_attention_weight_min": -96.14083862304688,
      "activations/layer8_attention_weight_max": 42.70262145996094,
      "activations/layer8_attention_weight_min": -44.505313873291016,
      "activations/layer9_attention_weight_max": 34.27846908569336,
      "activations/layer9_attention_weight_min": -34.84588623046875,
      "epoch": 17.27,
      "learning_rate": 3.902916666666666e-05,
      "loss": 2.7394,
      "step": 297200
    },
    {
      "activations/layer0_attention_weight_max": 15.72506046295166,
      "activations/layer0_attention_weight_min": -13.301692008972168,
      "activations/layer10_attention_weight_max": 36.83817672729492,
      "activations/layer10_attention_weight_min": -35.66427230834961,
      "activations/layer11_attention_weight_max": 37.73130798339844,
      "activations/layer11_attention_weight_min": -35.46788024902344,
      "activations/layer12_attention_weight_max": 35.00984573364258,
      "activations/layer12_attention_weight_min": -26.820053100585938,
      "activations/layer13_attention_weight_max": 44.44489669799805,
      "activations/layer13_attention_weight_min": -35.97947311401367,
      "activations/layer14_attention_weight_max": 45.80743408203125,
      "activations/layer14_attention_weight_min": -39.89734649658203,
      "activations/layer15_attention_weight_max": 42.71987533569336,
      "activations/layer15_attention_weight_min": -38.36534118652344,
      "activations/layer16_attention_weight_max": 30.877832412719727,
      "activations/layer16_attention_weight_min": -28.70366096496582,
      "activations/layer17_attention_weight_max": 29.809675216674805,
      "activations/layer17_attention_weight_min": -26.00150489807129,
      "activations/layer18_attention_weight_max": 33.01525115966797,
      "activations/layer18_attention_weight_min": -25.474502563476562,
      "activations/layer19_attention_weight_max": 31.331695556640625,
      "activations/layer19_attention_weight_min": -33.771968841552734,
      "activations/layer1_attention_weight_max": 17.166919708251953,
      "activations/layer1_attention_weight_min": -14.833078384399414,
      "activations/layer20_attention_weight_max": 27.962196350097656,
      "activations/layer20_attention_weight_min": -24.972911834716797,
      "activations/layer21_attention_weight_max": 28.549911499023438,
      "activations/layer21_attention_weight_min": -24.82137107849121,
      "activations/layer22_attention_weight_max": 40.909549713134766,
      "activations/layer22_attention_weight_min": -31.75560188293457,
      "activations/layer23_attention_weight_max": 31.972288131713867,
      "activations/layer23_attention_weight_min": -25.89813232421875,
      "activations/layer2_attention_weight_max": 31.999164581298828,
      "activations/layer2_attention_weight_min": -31.620975494384766,
      "activations/layer3_attention_weight_max": 95.31114196777344,
      "activations/layer3_attention_weight_min": -100.1696548461914,
      "activations/layer4_attention_weight_max": 102.8448715209961,
      "activations/layer4_attention_weight_min": -105.23760986328125,
      "activations/layer5_attention_weight_max": 48.676265716552734,
      "activations/layer5_attention_weight_min": -61.58590316772461,
      "activations/layer6_attention_weight_max": 43.75328063964844,
      "activations/layer6_attention_weight_min": -46.55839157104492,
      "activations/layer7_attention_weight_max": 95.24671173095703,
      "activations/layer7_attention_weight_min": -95.2938003540039,
      "activations/layer8_attention_weight_max": 43.94793701171875,
      "activations/layer8_attention_weight_min": -43.653385162353516,
      "activations/layer9_attention_weight_max": 36.830806732177734,
      "activations/layer9_attention_weight_min": -35.10365676879883,
      "epoch": 17.27,
      "learning_rate": 3.901022727272727e-05,
      "loss": 2.7212,
      "step": 297250
    },
    {
      "activations/layer0_attention_weight_max": 16.0388240814209,
      "activations/layer0_attention_weight_min": -13.560935974121094,
      "activations/layer10_attention_weight_max": 31.531295776367188,
      "activations/layer10_attention_weight_min": -33.96961975097656,
      "activations/layer11_attention_weight_max": 33.352516174316406,
      "activations/layer11_attention_weight_min": -34.57616424560547,
      "activations/layer12_attention_weight_max": 25.364221572875977,
      "activations/layer12_attention_weight_min": -28.38092613220215,
      "activations/layer13_attention_weight_max": 39.790870666503906,
      "activations/layer13_attention_weight_min": -36.30830001831055,
      "activations/layer14_attention_weight_max": 44.67763900756836,
      "activations/layer14_attention_weight_min": -39.129024505615234,
      "activations/layer15_attention_weight_max": 37.53028869628906,
      "activations/layer15_attention_weight_min": -36.55113983154297,
      "activations/layer16_attention_weight_max": 33.25559997558594,
      "activations/layer16_attention_weight_min": -30.320999145507812,
      "activations/layer17_attention_weight_max": 34.268104553222656,
      "activations/layer17_attention_weight_min": -28.061708450317383,
      "activations/layer18_attention_weight_max": 35.41438293457031,
      "activations/layer18_attention_weight_min": -27.147010803222656,
      "activations/layer19_attention_weight_max": 38.162227630615234,
      "activations/layer19_attention_weight_min": -32.80720901489258,
      "activations/layer1_attention_weight_max": 17.16004753112793,
      "activations/layer1_attention_weight_min": -16.516401290893555,
      "activations/layer20_attention_weight_max": 33.67314147949219,
      "activations/layer20_attention_weight_min": -25.113603591918945,
      "activations/layer21_attention_weight_max": 29.178800582885742,
      "activations/layer21_attention_weight_min": -25.619558334350586,
      "activations/layer22_attention_weight_max": 42.97430419921875,
      "activations/layer22_attention_weight_min": -32.24853515625,
      "activations/layer23_attention_weight_max": 32.50548553466797,
      "activations/layer23_attention_weight_min": -25.72625732421875,
      "activations/layer2_attention_weight_max": 34.647953033447266,
      "activations/layer2_attention_weight_min": -33.874576568603516,
      "activations/layer3_attention_weight_max": 99.8226318359375,
      "activations/layer3_attention_weight_min": -100.65452575683594,
      "activations/layer4_attention_weight_max": 105.57960510253906,
      "activations/layer4_attention_weight_min": -108.6612777709961,
      "activations/layer5_attention_weight_max": 52.13363265991211,
      "activations/layer5_attention_weight_min": -63.40841293334961,
      "activations/layer6_attention_weight_max": 43.85599136352539,
      "activations/layer6_attention_weight_min": -45.943336486816406,
      "activations/layer7_attention_weight_max": 89.31324005126953,
      "activations/layer7_attention_weight_min": -90.92514038085938,
      "activations/layer8_attention_weight_max": 38.650970458984375,
      "activations/layer8_attention_weight_min": -41.80369186401367,
      "activations/layer9_attention_weight_max": 30.7557430267334,
      "activations/layer9_attention_weight_min": -32.7821159362793,
      "epoch": 17.27,
      "learning_rate": 3.899128787878787e-05,
      "loss": 2.7273,
      "step": 297300
    },
    {
      "activations/layer0_attention_weight_max": 16.60959815979004,
      "activations/layer0_attention_weight_min": -13.17922306060791,
      "activations/layer10_attention_weight_max": 36.69227600097656,
      "activations/layer10_attention_weight_min": -37.517940521240234,
      "activations/layer11_attention_weight_max": 35.82331848144531,
      "activations/layer11_attention_weight_min": -39.127384185791016,
      "activations/layer12_attention_weight_max": 28.053470611572266,
      "activations/layer12_attention_weight_min": -27.41411781311035,
      "activations/layer13_attention_weight_max": 41.57224655151367,
      "activations/layer13_attention_weight_min": -37.24712371826172,
      "activations/layer14_attention_weight_max": 45.33827590942383,
      "activations/layer14_attention_weight_min": -38.77290344238281,
      "activations/layer15_attention_weight_max": 41.767574310302734,
      "activations/layer15_attention_weight_min": -36.39795684814453,
      "activations/layer16_attention_weight_max": 28.263816833496094,
      "activations/layer16_attention_weight_min": -26.97763442993164,
      "activations/layer17_attention_weight_max": 29.60662841796875,
      "activations/layer17_attention_weight_min": -25.186664581298828,
      "activations/layer18_attention_weight_max": 33.46742248535156,
      "activations/layer18_attention_weight_min": -25.089323043823242,
      "activations/layer19_attention_weight_max": 35.01066207885742,
      "activations/layer19_attention_weight_min": -30.557071685791016,
      "activations/layer1_attention_weight_max": 15.826744079589844,
      "activations/layer1_attention_weight_min": -15.558796882629395,
      "activations/layer20_attention_weight_max": 31.409576416015625,
      "activations/layer20_attention_weight_min": -25.543193817138672,
      "activations/layer21_attention_weight_max": 30.23546600341797,
      "activations/layer21_attention_weight_min": -26.278823852539062,
      "activations/layer22_attention_weight_max": 48.273475646972656,
      "activations/layer22_attention_weight_min": -29.60727882385254,
      "activations/layer23_attention_weight_max": 35.348854064941406,
      "activations/layer23_attention_weight_min": -24.591907501220703,
      "activations/layer2_attention_weight_max": 35.7413330078125,
      "activations/layer2_attention_weight_min": -34.813297271728516,
      "activations/layer3_attention_weight_max": 102.83653259277344,
      "activations/layer3_attention_weight_min": -105.54173278808594,
      "activations/layer4_attention_weight_max": 112.57574462890625,
      "activations/layer4_attention_weight_min": -114.77894592285156,
      "activations/layer5_attention_weight_max": 52.047935485839844,
      "activations/layer5_attention_weight_min": -62.636688232421875,
      "activations/layer6_attention_weight_max": 45.20175552368164,
      "activations/layer6_attention_weight_min": -48.3271369934082,
      "activations/layer7_attention_weight_max": 97.14938354492188,
      "activations/layer7_attention_weight_min": -101.88124084472656,
      "activations/layer8_attention_weight_max": 44.419586181640625,
      "activations/layer8_attention_weight_min": -45.20619583129883,
      "activations/layer9_attention_weight_max": 35.18256378173828,
      "activations/layer9_attention_weight_min": -35.982337951660156,
      "epoch": 17.28,
      "learning_rate": 3.897234848484849e-05,
      "loss": 2.7332,
      "step": 297350
    },
    {
      "activations/layer0_attention_weight_max": 16.264177322387695,
      "activations/layer0_attention_weight_min": -13.223373413085938,
      "activations/layer10_attention_weight_max": 33.165626525878906,
      "activations/layer10_attention_weight_min": -36.49074172973633,
      "activations/layer11_attention_weight_max": 34.70554733276367,
      "activations/layer11_attention_weight_min": -36.8135986328125,
      "activations/layer12_attention_weight_max": 28.118783950805664,
      "activations/layer12_attention_weight_min": -26.503572463989258,
      "activations/layer13_attention_weight_max": 39.6356201171875,
      "activations/layer13_attention_weight_min": -37.28070831298828,
      "activations/layer14_attention_weight_max": 41.835655212402344,
      "activations/layer14_attention_weight_min": -40.53163528442383,
      "activations/layer15_attention_weight_max": 37.70187759399414,
      "activations/layer15_attention_weight_min": -37.836158752441406,
      "activations/layer16_attention_weight_max": 27.76290512084961,
      "activations/layer16_attention_weight_min": -27.133363723754883,
      "activations/layer17_attention_weight_max": 30.817779541015625,
      "activations/layer17_attention_weight_min": -24.549701690673828,
      "activations/layer18_attention_weight_max": 30.835355758666992,
      "activations/layer18_attention_weight_min": -25.02583885192871,
      "activations/layer19_attention_weight_max": 31.268417358398438,
      "activations/layer19_attention_weight_min": -29.58099365234375,
      "activations/layer1_attention_weight_max": 16.5230770111084,
      "activations/layer1_attention_weight_min": -15.131699562072754,
      "activations/layer20_attention_weight_max": 27.879472732543945,
      "activations/layer20_attention_weight_min": -23.858564376831055,
      "activations/layer21_attention_weight_max": 26.670541763305664,
      "activations/layer21_attention_weight_min": -23.658437728881836,
      "activations/layer22_attention_weight_max": 39.178001403808594,
      "activations/layer22_attention_weight_min": -30.57921028137207,
      "activations/layer23_attention_weight_max": 29.489322662353516,
      "activations/layer23_attention_weight_min": -24.993614196777344,
      "activations/layer2_attention_weight_max": 33.64512634277344,
      "activations/layer2_attention_weight_min": -33.835655212402344,
      "activations/layer3_attention_weight_max": 104.23817443847656,
      "activations/layer3_attention_weight_min": -103.87548065185547,
      "activations/layer4_attention_weight_max": 111.29345703125,
      "activations/layer4_attention_weight_min": -106.64202117919922,
      "activations/layer5_attention_weight_max": 50.9024658203125,
      "activations/layer5_attention_weight_min": -61.031837463378906,
      "activations/layer6_attention_weight_max": 44.39736557006836,
      "activations/layer6_attention_weight_min": -45.08003616333008,
      "activations/layer7_attention_weight_max": 86.94095611572266,
      "activations/layer7_attention_weight_min": -88.95316314697266,
      "activations/layer8_attention_weight_max": 39.90188980102539,
      "activations/layer8_attention_weight_min": -41.65995788574219,
      "activations/layer9_attention_weight_max": 30.96162223815918,
      "activations/layer9_attention_weight_min": -33.90696716308594,
      "epoch": 17.28,
      "learning_rate": 3.895340909090909e-05,
      "loss": 2.737,
      "step": 297400
    },
    {
      "activations/layer0_attention_weight_max": 16.80516242980957,
      "activations/layer0_attention_weight_min": -12.587093353271484,
      "activations/layer10_attention_weight_max": 32.448822021484375,
      "activations/layer10_attention_weight_min": -32.41335678100586,
      "activations/layer11_attention_weight_max": 33.224266052246094,
      "activations/layer11_attention_weight_min": -33.216495513916016,
      "activations/layer12_attention_weight_max": 23.416364669799805,
      "activations/layer12_attention_weight_min": -25.633663177490234,
      "activations/layer13_attention_weight_max": 34.697505950927734,
      "activations/layer13_attention_weight_min": -33.97795867919922,
      "activations/layer14_attention_weight_max": 38.770137786865234,
      "activations/layer14_attention_weight_min": -36.85917663574219,
      "activations/layer15_attention_weight_max": 32.922306060791016,
      "activations/layer15_attention_weight_min": -33.705482482910156,
      "activations/layer16_attention_weight_max": 28.312034606933594,
      "activations/layer16_attention_weight_min": -29.54746437072754,
      "activations/layer17_attention_weight_max": 27.93155860900879,
      "activations/layer17_attention_weight_min": -26.1184024810791,
      "activations/layer18_attention_weight_max": 28.944082260131836,
      "activations/layer18_attention_weight_min": -26.096046447753906,
      "activations/layer19_attention_weight_max": 30.950729370117188,
      "activations/layer19_attention_weight_min": -29.555219650268555,
      "activations/layer1_attention_weight_max": 15.730782508850098,
      "activations/layer1_attention_weight_min": -14.405508995056152,
      "activations/layer20_attention_weight_max": 25.889225006103516,
      "activations/layer20_attention_weight_min": -24.124265670776367,
      "activations/layer21_attention_weight_max": 26.495697021484375,
      "activations/layer21_attention_weight_min": -24.316858291625977,
      "activations/layer22_attention_weight_max": 35.92837905883789,
      "activations/layer22_attention_weight_min": -30.068086624145508,
      "activations/layer23_attention_weight_max": 27.51262664794922,
      "activations/layer23_attention_weight_min": -24.38155174255371,
      "activations/layer2_attention_weight_max": 32.69715118408203,
      "activations/layer2_attention_weight_min": -33.097774505615234,
      "activations/layer3_attention_weight_max": 95.05667114257812,
      "activations/layer3_attention_weight_min": -98.3953628540039,
      "activations/layer4_attention_weight_max": 103.13002014160156,
      "activations/layer4_attention_weight_min": -108.67414855957031,
      "activations/layer5_attention_weight_max": 48.67112350463867,
      "activations/layer5_attention_weight_min": -61.43075180053711,
      "activations/layer6_attention_weight_max": 42.0938606262207,
      "activations/layer6_attention_weight_min": -44.78123474121094,
      "activations/layer7_attention_weight_max": 89.68565368652344,
      "activations/layer7_attention_weight_min": -93.40906524658203,
      "activations/layer8_attention_weight_max": 42.532474517822266,
      "activations/layer8_attention_weight_min": -44.05270004272461,
      "activations/layer9_attention_weight_max": 31.353900909423828,
      "activations/layer9_attention_weight_min": -32.502559661865234,
      "epoch": 17.28,
      "learning_rate": 3.893446969696969e-05,
      "loss": 2.7179,
      "step": 297450
    },
    {
      "activations/layer0_attention_weight_max": 17.07997703552246,
      "activations/layer0_attention_weight_min": -14.061362266540527,
      "activations/layer10_attention_weight_max": 34.758846282958984,
      "activations/layer10_attention_weight_min": -34.552528381347656,
      "activations/layer11_attention_weight_max": 34.348392486572266,
      "activations/layer11_attention_weight_min": -35.51109313964844,
      "activations/layer12_attention_weight_max": 26.531299591064453,
      "activations/layer12_attention_weight_min": -26.22995948791504,
      "activations/layer13_attention_weight_max": 37.36417770385742,
      "activations/layer13_attention_weight_min": -35.219383239746094,
      "activations/layer14_attention_weight_max": 46.59562301635742,
      "activations/layer14_attention_weight_min": -40.84042739868164,
      "activations/layer15_attention_weight_max": 38.6114616394043,
      "activations/layer15_attention_weight_min": -32.794830322265625,
      "activations/layer16_attention_weight_max": 30.819421768188477,
      "activations/layer16_attention_weight_min": -25.70376968383789,
      "activations/layer17_attention_weight_max": 31.0620059967041,
      "activations/layer17_attention_weight_min": -23.96698570251465,
      "activations/layer18_attention_weight_max": 33.53847122192383,
      "activations/layer18_attention_weight_min": -22.690540313720703,
      "activations/layer19_attention_weight_max": 37.304386138916016,
      "activations/layer19_attention_weight_min": -30.601032257080078,
      "activations/layer1_attention_weight_max": 16.450115203857422,
      "activations/layer1_attention_weight_min": -16.363901138305664,
      "activations/layer20_attention_weight_max": 29.90041732788086,
      "activations/layer20_attention_weight_min": -25.363651275634766,
      "activations/layer21_attention_weight_max": 29.575668334960938,
      "activations/layer21_attention_weight_min": -25.43449592590332,
      "activations/layer22_attention_weight_max": 43.64744186401367,
      "activations/layer22_attention_weight_min": -30.572725296020508,
      "activations/layer23_attention_weight_max": 35.17723846435547,
      "activations/layer23_attention_weight_min": -25.133888244628906,
      "activations/layer2_attention_weight_max": 34.437232971191406,
      "activations/layer2_attention_weight_min": -36.0412712097168,
      "activations/layer3_attention_weight_max": 99.68972778320312,
      "activations/layer3_attention_weight_min": -101.30979919433594,
      "activations/layer4_attention_weight_max": 107.91749572753906,
      "activations/layer4_attention_weight_min": -104.05059814453125,
      "activations/layer5_attention_weight_max": 51.56298828125,
      "activations/layer5_attention_weight_min": -61.31965637207031,
      "activations/layer6_attention_weight_max": 43.82456970214844,
      "activations/layer6_attention_weight_min": -47.09635925292969,
      "activations/layer7_attention_weight_max": 92.94309997558594,
      "activations/layer7_attention_weight_min": -106.46000671386719,
      "activations/layer8_attention_weight_max": 41.66966247558594,
      "activations/layer8_attention_weight_min": -42.539859771728516,
      "activations/layer9_attention_weight_max": 32.5540771484375,
      "activations/layer9_attention_weight_min": -34.517127990722656,
      "epoch": 17.29,
      "learning_rate": 3.89155303030303e-05,
      "loss": 2.7434,
      "step": 297500
    },
    {
      "activations/layer0_attention_weight_max": 16.634347915649414,
      "activations/layer0_attention_weight_min": -13.631587982177734,
      "activations/layer10_attention_weight_max": 43.864891052246094,
      "activations/layer10_attention_weight_min": -37.91944885253906,
      "activations/layer11_attention_weight_max": 42.43329620361328,
      "activations/layer11_attention_weight_min": -39.191993713378906,
      "activations/layer12_attention_weight_max": 34.541725158691406,
      "activations/layer12_attention_weight_min": -29.876874923706055,
      "activations/layer13_attention_weight_max": 51.1006965637207,
      "activations/layer13_attention_weight_min": -38.30530548095703,
      "activations/layer14_attention_weight_max": 53.82114791870117,
      "activations/layer14_attention_weight_min": -42.802764892578125,
      "activations/layer15_attention_weight_max": 49.229923248291016,
      "activations/layer15_attention_weight_min": -39.70713424682617,
      "activations/layer16_attention_weight_max": 33.75104522705078,
      "activations/layer16_attention_weight_min": -27.80156135559082,
      "activations/layer17_attention_weight_max": 34.911827087402344,
      "activations/layer17_attention_weight_min": -28.4224796295166,
      "activations/layer18_attention_weight_max": 36.63762664794922,
      "activations/layer18_attention_weight_min": -26.97642707824707,
      "activations/layer19_attention_weight_max": 38.43069839477539,
      "activations/layer19_attention_weight_min": -34.41524887084961,
      "activations/layer1_attention_weight_max": 15.743460655212402,
      "activations/layer1_attention_weight_min": -14.265158653259277,
      "activations/layer20_attention_weight_max": 35.3101921081543,
      "activations/layer20_attention_weight_min": -26.18029022216797,
      "activations/layer21_attention_weight_max": 35.23487091064453,
      "activations/layer21_attention_weight_min": -26.936691284179688,
      "activations/layer22_attention_weight_max": 56.47273635864258,
      "activations/layer22_attention_weight_min": -37.24958038330078,
      "activations/layer23_attention_weight_max": 42.379127502441406,
      "activations/layer23_attention_weight_min": -26.537025451660156,
      "activations/layer2_attention_weight_max": 32.87760925292969,
      "activations/layer2_attention_weight_min": -33.49718475341797,
      "activations/layer3_attention_weight_max": 101.76331329345703,
      "activations/layer3_attention_weight_min": -102.16584777832031,
      "activations/layer4_attention_weight_max": 106.75531005859375,
      "activations/layer4_attention_weight_min": -109.56341552734375,
      "activations/layer5_attention_weight_max": 53.07234191894531,
      "activations/layer5_attention_weight_min": -60.683494567871094,
      "activations/layer6_attention_weight_max": 45.0154914855957,
      "activations/layer6_attention_weight_min": -46.69890594482422,
      "activations/layer7_attention_weight_max": 101.6664047241211,
      "activations/layer7_attention_weight_min": -105.96800231933594,
      "activations/layer8_attention_weight_max": 46.170013427734375,
      "activations/layer8_attention_weight_min": -45.23731231689453,
      "activations/layer9_attention_weight_max": 38.008697509765625,
      "activations/layer9_attention_weight_min": -36.914772033691406,
      "epoch": 17.29,
      "learning_rate": 3.8896590909090906e-05,
      "loss": 2.7268,
      "step": 297550
    },
    {
      "activations/layer0_attention_weight_max": 16.763166427612305,
      "activations/layer0_attention_weight_min": -12.553256034851074,
      "activations/layer10_attention_weight_max": 36.285560607910156,
      "activations/layer10_attention_weight_min": -35.27959442138672,
      "activations/layer11_attention_weight_max": 38.49549102783203,
      "activations/layer11_attention_weight_min": -35.366451263427734,
      "activations/layer12_attention_weight_max": 25.776765823364258,
      "activations/layer12_attention_weight_min": -28.223154067993164,
      "activations/layer13_attention_weight_max": 37.97612762451172,
      "activations/layer13_attention_weight_min": -37.05945587158203,
      "activations/layer14_attention_weight_max": 42.53318405151367,
      "activations/layer14_attention_weight_min": -41.2072639465332,
      "activations/layer15_attention_weight_max": 42.159645080566406,
      "activations/layer15_attention_weight_min": -42.08547592163086,
      "activations/layer16_attention_weight_max": 28.486879348754883,
      "activations/layer16_attention_weight_min": -28.234573364257812,
      "activations/layer17_attention_weight_max": 29.43204116821289,
      "activations/layer17_attention_weight_min": -24.54145622253418,
      "activations/layer18_attention_weight_max": 29.742042541503906,
      "activations/layer18_attention_weight_min": -24.859569549560547,
      "activations/layer19_attention_weight_max": 35.603878021240234,
      "activations/layer19_attention_weight_min": -31.595672607421875,
      "activations/layer1_attention_weight_max": 16.205106735229492,
      "activations/layer1_attention_weight_min": -15.852219581604004,
      "activations/layer20_attention_weight_max": 29.477745056152344,
      "activations/layer20_attention_weight_min": -23.641098022460938,
      "activations/layer21_attention_weight_max": 28.826940536499023,
      "activations/layer21_attention_weight_min": -26.21714973449707,
      "activations/layer22_attention_weight_max": 44.58766174316406,
      "activations/layer22_attention_weight_min": -29.778671264648438,
      "activations/layer23_attention_weight_max": 30.96308135986328,
      "activations/layer23_attention_weight_min": -25.381561279296875,
      "activations/layer2_attention_weight_max": 34.380104064941406,
      "activations/layer2_attention_weight_min": -34.46476745605469,
      "activations/layer3_attention_weight_max": 100.67327117919922,
      "activations/layer3_attention_weight_min": -104.5103530883789,
      "activations/layer4_attention_weight_max": 108.34149169921875,
      "activations/layer4_attention_weight_min": -103.83995819091797,
      "activations/layer5_attention_weight_max": 49.633583068847656,
      "activations/layer5_attention_weight_min": -57.421043395996094,
      "activations/layer6_attention_weight_max": 45.73579406738281,
      "activations/layer6_attention_weight_min": -45.305912017822266,
      "activations/layer7_attention_weight_max": 96.49039459228516,
      "activations/layer7_attention_weight_min": -92.95293426513672,
      "activations/layer8_attention_weight_max": 45.37063217163086,
      "activations/layer8_attention_weight_min": -45.36772155761719,
      "activations/layer9_attention_weight_max": 34.29006576538086,
      "activations/layer9_attention_weight_min": -35.90795135498047,
      "epoch": 17.29,
      "learning_rate": 3.8877651515151515e-05,
      "loss": 2.7347,
      "step": 297600
    },
    {
      "activations/layer0_attention_weight_max": 15.463499069213867,
      "activations/layer0_attention_weight_min": -13.940688133239746,
      "activations/layer10_attention_weight_max": 34.39664840698242,
      "activations/layer10_attention_weight_min": -35.012813568115234,
      "activations/layer11_attention_weight_max": 35.08267593383789,
      "activations/layer11_attention_weight_min": -33.48214340209961,
      "activations/layer12_attention_weight_max": 27.1306095123291,
      "activations/layer12_attention_weight_min": -28.85148811340332,
      "activations/layer13_attention_weight_max": 40.55226135253906,
      "activations/layer13_attention_weight_min": -36.9566535949707,
      "activations/layer14_attention_weight_max": 44.728641510009766,
      "activations/layer14_attention_weight_min": -38.31913757324219,
      "activations/layer15_attention_weight_max": 40.78485107421875,
      "activations/layer15_attention_weight_min": -36.755611419677734,
      "activations/layer16_attention_weight_max": 30.443994522094727,
      "activations/layer16_attention_weight_min": -27.71337127685547,
      "activations/layer17_attention_weight_max": 34.851356506347656,
      "activations/layer17_attention_weight_min": -27.200016021728516,
      "activations/layer18_attention_weight_max": 35.33719253540039,
      "activations/layer18_attention_weight_min": -27.368261337280273,
      "activations/layer19_attention_weight_max": 41.00767135620117,
      "activations/layer19_attention_weight_min": -34.15760040283203,
      "activations/layer1_attention_weight_max": 16.030242919921875,
      "activations/layer1_attention_weight_min": -16.029300689697266,
      "activations/layer20_attention_weight_max": 34.882450103759766,
      "activations/layer20_attention_weight_min": -28.098398208618164,
      "activations/layer21_attention_weight_max": 32.120662689208984,
      "activations/layer21_attention_weight_min": -28.47372055053711,
      "activations/layer22_attention_weight_max": 45.65557861328125,
      "activations/layer22_attention_weight_min": -34.55082702636719,
      "activations/layer23_attention_weight_max": 37.40410614013672,
      "activations/layer23_attention_weight_min": -27.468799591064453,
      "activations/layer2_attention_weight_max": 32.529998779296875,
      "activations/layer2_attention_weight_min": -33.79010009765625,
      "activations/layer3_attention_weight_max": 102.70545959472656,
      "activations/layer3_attention_weight_min": -102.19908142089844,
      "activations/layer4_attention_weight_max": 109.2859878540039,
      "activations/layer4_attention_weight_min": -107.331298828125,
      "activations/layer5_attention_weight_max": 50.003822326660156,
      "activations/layer5_attention_weight_min": -59.46247482299805,
      "activations/layer6_attention_weight_max": 43.53621292114258,
      "activations/layer6_attention_weight_min": -47.02244186401367,
      "activations/layer7_attention_weight_max": 96.44802856445312,
      "activations/layer7_attention_weight_min": -95.67155456542969,
      "activations/layer8_attention_weight_max": 44.86587905883789,
      "activations/layer8_attention_weight_min": -46.69032287597656,
      "activations/layer9_attention_weight_max": 35.04890441894531,
      "activations/layer9_attention_weight_min": -36.24687957763672,
      "epoch": 17.3,
      "learning_rate": 3.8858712121212116e-05,
      "loss": 2.7568,
      "step": 297650
    },
    {
      "activations/layer0_attention_weight_max": 15.589615821838379,
      "activations/layer0_attention_weight_min": -11.923176765441895,
      "activations/layer10_attention_weight_max": 38.921424865722656,
      "activations/layer10_attention_weight_min": -35.54085159301758,
      "activations/layer11_attention_weight_max": 40.19599151611328,
      "activations/layer11_attention_weight_min": -37.35553741455078,
      "activations/layer12_attention_weight_max": 33.1318244934082,
      "activations/layer12_attention_weight_min": -28.385534286499023,
      "activations/layer13_attention_weight_max": 49.19880294799805,
      "activations/layer13_attention_weight_min": -38.8925895690918,
      "activations/layer14_attention_weight_max": 57.07588577270508,
      "activations/layer14_attention_weight_min": -44.63469696044922,
      "activations/layer15_attention_weight_max": 51.263179779052734,
      "activations/layer15_attention_weight_min": -44.4196891784668,
      "activations/layer16_attention_weight_max": 38.81028747558594,
      "activations/layer16_attention_weight_min": -30.533403396606445,
      "activations/layer17_attention_weight_max": 43.195594787597656,
      "activations/layer17_attention_weight_min": -30.747684478759766,
      "activations/layer18_attention_weight_max": 41.4880485534668,
      "activations/layer18_attention_weight_min": -33.207271575927734,
      "activations/layer19_attention_weight_max": 49.578617095947266,
      "activations/layer19_attention_weight_min": -38.88576126098633,
      "activations/layer1_attention_weight_max": 15.950958251953125,
      "activations/layer1_attention_weight_min": -16.498329162597656,
      "activations/layer20_attention_weight_max": 37.63489532470703,
      "activations/layer20_attention_weight_min": -30.349334716796875,
      "activations/layer21_attention_weight_max": 38.61345291137695,
      "activations/layer21_attention_weight_min": -32.032447814941406,
      "activations/layer22_attention_weight_max": 63.95718002319336,
      "activations/layer22_attention_weight_min": -43.4544563293457,
      "activations/layer23_attention_weight_max": 40.90309524536133,
      "activations/layer23_attention_weight_min": -31.48561668395996,
      "activations/layer2_attention_weight_max": 32.429840087890625,
      "activations/layer2_attention_weight_min": -33.77640914916992,
      "activations/layer3_attention_weight_max": 101.18434143066406,
      "activations/layer3_attention_weight_min": -104.9321517944336,
      "activations/layer4_attention_weight_max": 106.3133544921875,
      "activations/layer4_attention_weight_min": -106.52080535888672,
      "activations/layer5_attention_weight_max": 47.76754379272461,
      "activations/layer5_attention_weight_min": -60.02001953125,
      "activations/layer6_attention_weight_max": 42.35349655151367,
      "activations/layer6_attention_weight_min": -45.3321418762207,
      "activations/layer7_attention_weight_max": 100.73101043701172,
      "activations/layer7_attention_weight_min": -93.80997467041016,
      "activations/layer8_attention_weight_max": 44.863487243652344,
      "activations/layer8_attention_weight_min": -44.10994338989258,
      "activations/layer9_attention_weight_max": 37.366546630859375,
      "activations/layer9_attention_weight_min": -36.165897369384766,
      "epoch": 17.3,
      "learning_rate": 3.8839772727272725e-05,
      "loss": 2.7462,
      "step": 297700
    },
    {
      "activations/layer0_attention_weight_max": 16.15321922302246,
      "activations/layer0_attention_weight_min": -12.486312866210938,
      "activations/layer10_attention_weight_max": 35.62285232543945,
      "activations/layer10_attention_weight_min": -34.18659210205078,
      "activations/layer11_attention_weight_max": 35.07566833496094,
      "activations/layer11_attention_weight_min": -33.78092956542969,
      "activations/layer12_attention_weight_max": 30.77222442626953,
      "activations/layer12_attention_weight_min": -26.0916690826416,
      "activations/layer13_attention_weight_max": 39.956077575683594,
      "activations/layer13_attention_weight_min": -35.20409393310547,
      "activations/layer14_attention_weight_max": 41.5738639831543,
      "activations/layer14_attention_weight_min": -37.38803482055664,
      "activations/layer15_attention_weight_max": 38.43064498901367,
      "activations/layer15_attention_weight_min": -37.089542388916016,
      "activations/layer16_attention_weight_max": 32.20244216918945,
      "activations/layer16_attention_weight_min": -27.64492416381836,
      "activations/layer17_attention_weight_max": 30.09066390991211,
      "activations/layer17_attention_weight_min": -25.493701934814453,
      "activations/layer18_attention_weight_max": 29.77581214904785,
      "activations/layer18_attention_weight_min": -26.759225845336914,
      "activations/layer19_attention_weight_max": 36.2944450378418,
      "activations/layer19_attention_weight_min": -30.079986572265625,
      "activations/layer1_attention_weight_max": 16.108135223388672,
      "activations/layer1_attention_weight_min": -15.192407608032227,
      "activations/layer20_attention_weight_max": 31.573057174682617,
      "activations/layer20_attention_weight_min": -25.404102325439453,
      "activations/layer21_attention_weight_max": 32.61583709716797,
      "activations/layer21_attention_weight_min": -23.33365249633789,
      "activations/layer22_attention_weight_max": 43.36330795288086,
      "activations/layer22_attention_weight_min": -30.081010818481445,
      "activations/layer23_attention_weight_max": 36.51985168457031,
      "activations/layer23_attention_weight_min": -24.82482147216797,
      "activations/layer2_attention_weight_max": 32.03425979614258,
      "activations/layer2_attention_weight_min": -32.94941711425781,
      "activations/layer3_attention_weight_max": 99.78479766845703,
      "activations/layer3_attention_weight_min": -99.21541595458984,
      "activations/layer4_attention_weight_max": 107.38600158691406,
      "activations/layer4_attention_weight_min": -109.81327056884766,
      "activations/layer5_attention_weight_max": 53.28241729736328,
      "activations/layer5_attention_weight_min": -64.7802734375,
      "activations/layer6_attention_weight_max": 43.978031158447266,
      "activations/layer6_attention_weight_min": -47.0710563659668,
      "activations/layer7_attention_weight_max": 94.7131118774414,
      "activations/layer7_attention_weight_min": -93.28253173828125,
      "activations/layer8_attention_weight_max": 41.2446174621582,
      "activations/layer8_attention_weight_min": -42.220001220703125,
      "activations/layer9_attention_weight_max": 33.19332504272461,
      "activations/layer9_attention_weight_min": -33.96799850463867,
      "epoch": 17.3,
      "learning_rate": 3.882083333333333e-05,
      "loss": 2.7372,
      "step": 297750
    },
    {
      "activations/layer0_attention_weight_max": 15.954416275024414,
      "activations/layer0_attention_weight_min": -12.599353790283203,
      "activations/layer10_attention_weight_max": 33.842742919921875,
      "activations/layer10_attention_weight_min": -34.23814392089844,
      "activations/layer11_attention_weight_max": 33.57435989379883,
      "activations/layer11_attention_weight_min": -34.06830978393555,
      "activations/layer12_attention_weight_max": 25.803613662719727,
      "activations/layer12_attention_weight_min": -25.501907348632812,
      "activations/layer13_attention_weight_max": 38.8992805480957,
      "activations/layer13_attention_weight_min": -34.834659576416016,
      "activations/layer14_attention_weight_max": 44.06327819824219,
      "activations/layer14_attention_weight_min": -37.22429275512695,
      "activations/layer15_attention_weight_max": 38.68531799316406,
      "activations/layer15_attention_weight_min": -37.058956146240234,
      "activations/layer16_attention_weight_max": 30.08563804626465,
      "activations/layer16_attention_weight_min": -28.176753997802734,
      "activations/layer17_attention_weight_max": 29.27985382080078,
      "activations/layer17_attention_weight_min": -25.882123947143555,
      "activations/layer18_attention_weight_max": 30.90908432006836,
      "activations/layer18_attention_weight_min": -25.24284553527832,
      "activations/layer19_attention_weight_max": 30.552921295166016,
      "activations/layer19_attention_weight_min": -29.123571395874023,
      "activations/layer1_attention_weight_max": 16.283512115478516,
      "activations/layer1_attention_weight_min": -14.740324020385742,
      "activations/layer20_attention_weight_max": 27.026451110839844,
      "activations/layer20_attention_weight_min": -27.302345275878906,
      "activations/layer21_attention_weight_max": 27.192119598388672,
      "activations/layer21_attention_weight_min": -27.439977645874023,
      "activations/layer22_attention_weight_max": 40.70623779296875,
      "activations/layer22_attention_weight_min": -32.8426628112793,
      "activations/layer23_attention_weight_max": 32.10717010498047,
      "activations/layer23_attention_weight_min": -28.02370262145996,
      "activations/layer2_attention_weight_max": 33.48213195800781,
      "activations/layer2_attention_weight_min": -34.06969451904297,
      "activations/layer3_attention_weight_max": 101.6292495727539,
      "activations/layer3_attention_weight_min": -102.48057556152344,
      "activations/layer4_attention_weight_max": 106.70122528076172,
      "activations/layer4_attention_weight_min": -107.0443115234375,
      "activations/layer5_attention_weight_max": 48.980751037597656,
      "activations/layer5_attention_weight_min": -60.69462585449219,
      "activations/layer6_attention_weight_max": 45.47736740112305,
      "activations/layer6_attention_weight_min": -47.96590042114258,
      "activations/layer7_attention_weight_max": 92.72590637207031,
      "activations/layer7_attention_weight_min": -94.36279296875,
      "activations/layer8_attention_weight_max": 41.39885330200195,
      "activations/layer8_attention_weight_min": -42.49715042114258,
      "activations/layer9_attention_weight_max": 33.680015563964844,
      "activations/layer9_attention_weight_min": -35.19411087036133,
      "epoch": 17.3,
      "learning_rate": 3.8801893939393934e-05,
      "loss": 2.7538,
      "step": 297800
    },
    {
      "activations/layer0_attention_weight_max": 16.773353576660156,
      "activations/layer0_attention_weight_min": -14.89931869506836,
      "activations/layer10_attention_weight_max": 36.45000076293945,
      "activations/layer10_attention_weight_min": -35.20195388793945,
      "activations/layer11_attention_weight_max": 36.34442901611328,
      "activations/layer11_attention_weight_min": -36.12903594970703,
      "activations/layer12_attention_weight_max": 27.736698150634766,
      "activations/layer12_attention_weight_min": -26.64211654663086,
      "activations/layer13_attention_weight_max": 43.832645416259766,
      "activations/layer13_attention_weight_min": -37.34978103637695,
      "activations/layer14_attention_weight_max": 44.173667907714844,
      "activations/layer14_attention_weight_min": -39.13007354736328,
      "activations/layer15_attention_weight_max": 40.47217559814453,
      "activations/layer15_attention_weight_min": -39.57166290283203,
      "activations/layer16_attention_weight_max": 30.070268630981445,
      "activations/layer16_attention_weight_min": -29.159883499145508,
      "activations/layer17_attention_weight_max": 30.558244705200195,
      "activations/layer17_attention_weight_min": -27.304121017456055,
      "activations/layer18_attention_weight_max": 35.62940216064453,
      "activations/layer18_attention_weight_min": -25.628812789916992,
      "activations/layer19_attention_weight_max": 32.16254425048828,
      "activations/layer19_attention_weight_min": -30.9249324798584,
      "activations/layer1_attention_weight_max": 16.03763771057129,
      "activations/layer1_attention_weight_min": -14.931981086730957,
      "activations/layer20_attention_weight_max": 26.83186912536621,
      "activations/layer20_attention_weight_min": -26.05864715576172,
      "activations/layer21_attention_weight_max": 29.167203903198242,
      "activations/layer21_attention_weight_min": -25.36869239807129,
      "activations/layer22_attention_weight_max": 40.55424118041992,
      "activations/layer22_attention_weight_min": -30.28632164001465,
      "activations/layer23_attention_weight_max": 30.8504695892334,
      "activations/layer23_attention_weight_min": -25.47449493408203,
      "activations/layer2_attention_weight_max": 33.33516311645508,
      "activations/layer2_attention_weight_min": -34.217010498046875,
      "activations/layer3_attention_weight_max": 99.88924407958984,
      "activations/layer3_attention_weight_min": -100.11511993408203,
      "activations/layer4_attention_weight_max": 107.22712707519531,
      "activations/layer4_attention_weight_min": -114.41650390625,
      "activations/layer5_attention_weight_max": 49.9961051940918,
      "activations/layer5_attention_weight_min": -63.16918182373047,
      "activations/layer6_attention_weight_max": 42.978084564208984,
      "activations/layer6_attention_weight_min": -43.799930572509766,
      "activations/layer7_attention_weight_max": 91.94815063476562,
      "activations/layer7_attention_weight_min": -91.66643524169922,
      "activations/layer8_attention_weight_max": 40.97800064086914,
      "activations/layer8_attention_weight_min": -45.3785285949707,
      "activations/layer9_attention_weight_max": 34.082889556884766,
      "activations/layer9_attention_weight_min": -34.3500862121582,
      "epoch": 17.31,
      "learning_rate": 3.878295454545454e-05,
      "loss": 2.7374,
      "step": 297850
    },
    {
      "activations/layer0_attention_weight_max": 15.829463958740234,
      "activations/layer0_attention_weight_min": -14.085491180419922,
      "activations/layer10_attention_weight_max": 32.542415618896484,
      "activations/layer10_attention_weight_min": -33.585384368896484,
      "activations/layer11_attention_weight_max": 32.68528747558594,
      "activations/layer11_attention_weight_min": -32.88412857055664,
      "activations/layer12_attention_weight_max": 27.107234954833984,
      "activations/layer12_attention_weight_min": -26.42365074157715,
      "activations/layer13_attention_weight_max": 39.30720138549805,
      "activations/layer13_attention_weight_min": -35.63400650024414,
      "activations/layer14_attention_weight_max": 43.04608917236328,
      "activations/layer14_attention_weight_min": -38.25260543823242,
      "activations/layer15_attention_weight_max": 40.043148040771484,
      "activations/layer15_attention_weight_min": -37.36643981933594,
      "activations/layer16_attention_weight_max": 31.530654907226562,
      "activations/layer16_attention_weight_min": -27.596681594848633,
      "activations/layer17_attention_weight_max": 29.68482208251953,
      "activations/layer17_attention_weight_min": -26.8095645904541,
      "activations/layer18_attention_weight_max": 30.736331939697266,
      "activations/layer18_attention_weight_min": -27.699993133544922,
      "activations/layer19_attention_weight_max": 33.33852005004883,
      "activations/layer19_attention_weight_min": -32.16571044921875,
      "activations/layer1_attention_weight_max": 15.993268013000488,
      "activations/layer1_attention_weight_min": -15.831335067749023,
      "activations/layer20_attention_weight_max": 26.942626953125,
      "activations/layer20_attention_weight_min": -25.845096588134766,
      "activations/layer21_attention_weight_max": 28.871036529541016,
      "activations/layer21_attention_weight_min": -26.36779022216797,
      "activations/layer22_attention_weight_max": 38.00043869018555,
      "activations/layer22_attention_weight_min": -32.9271125793457,
      "activations/layer23_attention_weight_max": 30.91254997253418,
      "activations/layer23_attention_weight_min": -28.0511531829834,
      "activations/layer2_attention_weight_max": 31.506553649902344,
      "activations/layer2_attention_weight_min": -32.109619140625,
      "activations/layer3_attention_weight_max": 96.31074523925781,
      "activations/layer3_attention_weight_min": -95.83512115478516,
      "activations/layer4_attention_weight_max": 103.70677947998047,
      "activations/layer4_attention_weight_min": -102.43688201904297,
      "activations/layer5_attention_weight_max": 48.78404998779297,
      "activations/layer5_attention_weight_min": -59.97816467285156,
      "activations/layer6_attention_weight_max": 42.86498260498047,
      "activations/layer6_attention_weight_min": -43.60462188720703,
      "activations/layer7_attention_weight_max": 84.44588470458984,
      "activations/layer7_attention_weight_min": -89.4148178100586,
      "activations/layer8_attention_weight_max": 41.45232391357422,
      "activations/layer8_attention_weight_min": -41.198673248291016,
      "activations/layer9_attention_weight_max": 33.12506866455078,
      "activations/layer9_attention_weight_min": -32.83833312988281,
      "epoch": 17.31,
      "learning_rate": 3.876401515151515e-05,
      "loss": 2.7417,
      "step": 297900
    },
    {
      "activations/layer0_attention_weight_max": 16.721982955932617,
      "activations/layer0_attention_weight_min": -13.114051818847656,
      "activations/layer10_attention_weight_max": 30.29241943359375,
      "activations/layer10_attention_weight_min": -33.78403091430664,
      "activations/layer11_attention_weight_max": 31.894927978515625,
      "activations/layer11_attention_weight_min": -32.659393310546875,
      "activations/layer12_attention_weight_max": 24.495986938476562,
      "activations/layer12_attention_weight_min": -25.992591857910156,
      "activations/layer13_attention_weight_max": 36.4327507019043,
      "activations/layer13_attention_weight_min": -33.359771728515625,
      "activations/layer14_attention_weight_max": 39.48454666137695,
      "activations/layer14_attention_weight_min": -35.468910217285156,
      "activations/layer15_attention_weight_max": 34.822452545166016,
      "activations/layer15_attention_weight_min": -33.6438102722168,
      "activations/layer16_attention_weight_max": 28.744251251220703,
      "activations/layer16_attention_weight_min": -26.778968811035156,
      "activations/layer17_attention_weight_max": 31.045862197875977,
      "activations/layer17_attention_weight_min": -24.651086807250977,
      "activations/layer18_attention_weight_max": 31.287668228149414,
      "activations/layer18_attention_weight_min": -23.509984970092773,
      "activations/layer19_attention_weight_max": 35.29618835449219,
      "activations/layer19_attention_weight_min": -30.647750854492188,
      "activations/layer1_attention_weight_max": 15.725146293640137,
      "activations/layer1_attention_weight_min": -15.57778263092041,
      "activations/layer20_attention_weight_max": 28.60704803466797,
      "activations/layer20_attention_weight_min": -23.593395233154297,
      "activations/layer21_attention_weight_max": 29.364837646484375,
      "activations/layer21_attention_weight_min": -25.025293350219727,
      "activations/layer22_attention_weight_max": 42.13754653930664,
      "activations/layer22_attention_weight_min": -30.012245178222656,
      "activations/layer23_attention_weight_max": 33.569976806640625,
      "activations/layer23_attention_weight_min": -25.43324089050293,
      "activations/layer2_attention_weight_max": 33.18265151977539,
      "activations/layer2_attention_weight_min": -33.01456069946289,
      "activations/layer3_attention_weight_max": 97.39569091796875,
      "activations/layer3_attention_weight_min": -97.86893463134766,
      "activations/layer4_attention_weight_max": 103.47869873046875,
      "activations/layer4_attention_weight_min": -106.61993408203125,
      "activations/layer5_attention_weight_max": 49.62672424316406,
      "activations/layer5_attention_weight_min": -57.60039520263672,
      "activations/layer6_attention_weight_max": 41.32866668701172,
      "activations/layer6_attention_weight_min": -46.83953094482422,
      "activations/layer7_attention_weight_max": 87.01160430908203,
      "activations/layer7_attention_weight_min": -91.62580871582031,
      "activations/layer8_attention_weight_max": 39.919002532958984,
      "activations/layer8_attention_weight_min": -43.12921905517578,
      "activations/layer9_attention_weight_max": 32.18345260620117,
      "activations/layer9_attention_weight_min": -33.54951858520508,
      "epoch": 17.31,
      "learning_rate": 3.874507575757575e-05,
      "loss": 2.7223,
      "step": 297950
    },
    {
      "activations/layer0_attention_weight_max": 16.35631561279297,
      "activations/layer0_attention_weight_min": -12.969137191772461,
      "activations/layer10_attention_weight_max": 33.4442138671875,
      "activations/layer10_attention_weight_min": -34.35857391357422,
      "activations/layer11_attention_weight_max": 33.96613693237305,
      "activations/layer11_attention_weight_min": -33.889671325683594,
      "activations/layer12_attention_weight_max": 25.89582633972168,
      "activations/layer12_attention_weight_min": -27.672513961791992,
      "activations/layer13_attention_weight_max": 40.050811767578125,
      "activations/layer13_attention_weight_min": -35.44172668457031,
      "activations/layer14_attention_weight_max": 42.71272659301758,
      "activations/layer14_attention_weight_min": -37.39840316772461,
      "activations/layer15_attention_weight_max": 40.0951042175293,
      "activations/layer15_attention_weight_min": -37.70615768432617,
      "activations/layer16_attention_weight_max": 28.811805725097656,
      "activations/layer16_attention_weight_min": -29.342269897460938,
      "activations/layer17_attention_weight_max": 29.910871505737305,
      "activations/layer17_attention_weight_min": -26.613431930541992,
      "activations/layer18_attention_weight_max": 31.992111206054688,
      "activations/layer18_attention_weight_min": -25.826515197753906,
      "activations/layer19_attention_weight_max": 36.35662841796875,
      "activations/layer19_attention_weight_min": -29.58262062072754,
      "activations/layer1_attention_weight_max": 16.18671417236328,
      "activations/layer1_attention_weight_min": -16.4325008392334,
      "activations/layer20_attention_weight_max": 30.52880859375,
      "activations/layer20_attention_weight_min": -23.950408935546875,
      "activations/layer21_attention_weight_max": 30.17211151123047,
      "activations/layer21_attention_weight_min": -23.99639320373535,
      "activations/layer22_attention_weight_max": 40.320491790771484,
      "activations/layer22_attention_weight_min": -29.53750991821289,
      "activations/layer23_attention_weight_max": 32.86540603637695,
      "activations/layer23_attention_weight_min": -23.91779899597168,
      "activations/layer2_attention_weight_max": 33.88405990600586,
      "activations/layer2_attention_weight_min": -33.63424301147461,
      "activations/layer3_attention_weight_max": 99.9064712524414,
      "activations/layer3_attention_weight_min": -102.29950714111328,
      "activations/layer4_attention_weight_max": 107.6079330444336,
      "activations/layer4_attention_weight_min": -109.37676239013672,
      "activations/layer5_attention_weight_max": 50.64451217651367,
      "activations/layer5_attention_weight_min": -62.25462341308594,
      "activations/layer6_attention_weight_max": 44.15486145019531,
      "activations/layer6_attention_weight_min": -46.717891693115234,
      "activations/layer7_attention_weight_max": 94.81732177734375,
      "activations/layer7_attention_weight_min": -99.79034423828125,
      "activations/layer8_attention_weight_max": 40.15984344482422,
      "activations/layer8_attention_weight_min": -44.061092376708984,
      "activations/layer9_attention_weight_max": 31.646621704101562,
      "activations/layer9_attention_weight_min": -32.795475006103516,
      "epoch": 17.32,
      "learning_rate": 3.872613636363636e-05,
      "loss": 2.7325,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4403,
      "eval_samples_per_second": 508.747,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4403,
      "eval_openwebtext_samples_per_second": 508.747,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9832,
      "eval_wikitext_samples_per_second": 229.933,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_lambada_loss": 2.486328125,
      "eval_lambada_ppl": 12.01706983184115,
      "eval_lambada_runtime": 9.5566,
      "eval_lambada_samples_per_second": 509.49,
      "step": 298000
    },
    {
      "activations/layer0_attention_weight_max": 15.947566032409668,
      "activations/layer0_attention_weight_min": -12.582091331481934,
      "activations/layer10_attention_weight_max": 31.22109603881836,
      "activations/layer10_attention_weight_min": -33.578643798828125,
      "activations/layer11_attention_weight_max": 31.847965240478516,
      "activations/layer11_attention_weight_min": -32.49067687988281,
      "activations/layer12_attention_weight_max": 27.4128475189209,
      "activations/layer12_attention_weight_min": -27.375185012817383,
      "activations/layer13_attention_weight_max": 38.57929229736328,
      "activations/layer13_attention_weight_min": -37.61208724975586,
      "activations/layer14_attention_weight_max": 42.6655387878418,
      "activations/layer14_attention_weight_min": -38.07139587402344,
      "activations/layer15_attention_weight_max": 41.18863296508789,
      "activations/layer15_attention_weight_min": -36.6657829284668,
      "activations/layer16_attention_weight_max": 32.27017593383789,
      "activations/layer16_attention_weight_min": -26.78338623046875,
      "activations/layer17_attention_weight_max": 32.128047943115234,
      "activations/layer17_attention_weight_min": -26.32273292541504,
      "activations/layer18_attention_weight_max": 33.36376190185547,
      "activations/layer18_attention_weight_min": -25.31874656677246,
      "activations/layer19_attention_weight_max": 41.04237747192383,
      "activations/layer19_attention_weight_min": -36.03162384033203,
      "activations/layer1_attention_weight_max": 16.48841094970703,
      "activations/layer1_attention_weight_min": -13.973361015319824,
      "activations/layer20_attention_weight_max": 35.80630874633789,
      "activations/layer20_attention_weight_min": -27.439035415649414,
      "activations/layer21_attention_weight_max": 33.677520751953125,
      "activations/layer21_attention_weight_min": -24.38982391357422,
      "activations/layer22_attention_weight_max": 45.97177505493164,
      "activations/layer22_attention_weight_min": -30.984453201293945,
      "activations/layer23_attention_weight_max": 37.589935302734375,
      "activations/layer23_attention_weight_min": -25.802001953125,
      "activations/layer2_attention_weight_max": 32.35016632080078,
      "activations/layer2_attention_weight_min": -34.062255859375,
      "activations/layer3_attention_weight_max": 102.74004364013672,
      "activations/layer3_attention_weight_min": -103.35908508300781,
      "activations/layer4_attention_weight_max": 107.0440444946289,
      "activations/layer4_attention_weight_min": -108.1545181274414,
      "activations/layer5_attention_weight_max": 51.638702392578125,
      "activations/layer5_attention_weight_min": -60.130157470703125,
      "activations/layer6_attention_weight_max": 44.934085845947266,
      "activations/layer6_attention_weight_min": -47.98878479003906,
      "activations/layer7_attention_weight_max": 93.8586196899414,
      "activations/layer7_attention_weight_min": -96.03485107421875,
      "activations/layer8_attention_weight_max": 40.21149826049805,
      "activations/layer8_attention_weight_min": -43.1792106628418,
      "activations/layer9_attention_weight_max": 30.966312408447266,
      "activations/layer9_attention_weight_min": -34.464786529541016,
      "epoch": 17.32,
      "learning_rate": 3.870719696969696e-05,
      "loss": 2.7246,
      "step": 298050
    },
    {
      "activations/layer0_attention_weight_max": 16.15019989013672,
      "activations/layer0_attention_weight_min": -13.319722175598145,
      "activations/layer10_attention_weight_max": 34.06693649291992,
      "activations/layer10_attention_weight_min": -33.751243591308594,
      "activations/layer11_attention_weight_max": 33.90301513671875,
      "activations/layer11_attention_weight_min": -33.42378234863281,
      "activations/layer12_attention_weight_max": 27.92612075805664,
      "activations/layer12_attention_weight_min": -26.162744522094727,
      "activations/layer13_attention_weight_max": 41.74763107299805,
      "activations/layer13_attention_weight_min": -35.10573959350586,
      "activations/layer14_attention_weight_max": 43.26115036010742,
      "activations/layer14_attention_weight_min": -39.949493408203125,
      "activations/layer15_attention_weight_max": 39.89811325073242,
      "activations/layer15_attention_weight_min": -34.37421417236328,
      "activations/layer16_attention_weight_max": 29.878538131713867,
      "activations/layer16_attention_weight_min": -26.911088943481445,
      "activations/layer17_attention_weight_max": 28.894338607788086,
      "activations/layer17_attention_weight_min": -24.340036392211914,
      "activations/layer18_attention_weight_max": 32.275115966796875,
      "activations/layer18_attention_weight_min": -24.168474197387695,
      "activations/layer19_attention_weight_max": 36.986968994140625,
      "activations/layer19_attention_weight_min": -31.616973876953125,
      "activations/layer1_attention_weight_max": 16.175580978393555,
      "activations/layer1_attention_weight_min": -15.199678421020508,
      "activations/layer20_attention_weight_max": 31.389019012451172,
      "activations/layer20_attention_weight_min": -25.92281723022461,
      "activations/layer21_attention_weight_max": 27.80754852294922,
      "activations/layer21_attention_weight_min": -24.339702606201172,
      "activations/layer22_attention_weight_max": 38.449546813964844,
      "activations/layer22_attention_weight_min": -31.077802658081055,
      "activations/layer23_attention_weight_max": 31.784269332885742,
      "activations/layer23_attention_weight_min": -27.246318817138672,
      "activations/layer2_attention_weight_max": 34.48154067993164,
      "activations/layer2_attention_weight_min": -33.405174255371094,
      "activations/layer3_attention_weight_max": 103.07695770263672,
      "activations/layer3_attention_weight_min": -99.87255859375,
      "activations/layer4_attention_weight_max": 104.46366119384766,
      "activations/layer4_attention_weight_min": -106.2444076538086,
      "activations/layer5_attention_weight_max": 51.44108200073242,
      "activations/layer5_attention_weight_min": -61.377037048339844,
      "activations/layer6_attention_weight_max": 43.021202087402344,
      "activations/layer6_attention_weight_min": -46.14707946777344,
      "activations/layer7_attention_weight_max": 92.53500366210938,
      "activations/layer7_attention_weight_min": -96.58879089355469,
      "activations/layer8_attention_weight_max": 40.430442810058594,
      "activations/layer8_attention_weight_min": -42.56099319458008,
      "activations/layer9_attention_weight_max": 31.83003807067871,
      "activations/layer9_attention_weight_min": -33.87430953979492,
      "epoch": 17.32,
      "learning_rate": 3.868825757575758e-05,
      "loss": 2.753,
      "step": 298100
    },
    {
      "activations/layer0_attention_weight_max": 16.218088150024414,
      "activations/layer0_attention_weight_min": -12.476916313171387,
      "activations/layer10_attention_weight_max": 35.670352935791016,
      "activations/layer10_attention_weight_min": -38.45390319824219,
      "activations/layer11_attention_weight_max": 35.594844818115234,
      "activations/layer11_attention_weight_min": -38.02614974975586,
      "activations/layer12_attention_weight_max": 26.248964309692383,
      "activations/layer12_attention_weight_min": -27.86786651611328,
      "activations/layer13_attention_weight_max": 39.05000686645508,
      "activations/layer13_attention_weight_min": -37.815216064453125,
      "activations/layer14_attention_weight_max": 41.20671081542969,
      "activations/layer14_attention_weight_min": -40.63753890991211,
      "activations/layer15_attention_weight_max": 39.581851959228516,
      "activations/layer15_attention_weight_min": -41.38148880004883,
      "activations/layer16_attention_weight_max": 29.794353485107422,
      "activations/layer16_attention_weight_min": -29.486370086669922,
      "activations/layer17_attention_weight_max": 28.53989028930664,
      "activations/layer17_attention_weight_min": -27.02825355529785,
      "activations/layer18_attention_weight_max": 34.26793670654297,
      "activations/layer18_attention_weight_min": -26.39661407470703,
      "activations/layer19_attention_weight_max": 32.65982437133789,
      "activations/layer19_attention_weight_min": -33.571773529052734,
      "activations/layer1_attention_weight_max": 16.346298217773438,
      "activations/layer1_attention_weight_min": -16.539508819580078,
      "activations/layer20_attention_weight_max": 28.1760311126709,
      "activations/layer20_attention_weight_min": -26.059629440307617,
      "activations/layer21_attention_weight_max": 28.28226661682129,
      "activations/layer21_attention_weight_min": -24.422313690185547,
      "activations/layer22_attention_weight_max": 44.948455810546875,
      "activations/layer22_attention_weight_min": -33.39945602416992,
      "activations/layer23_attention_weight_max": 36.030601501464844,
      "activations/layer23_attention_weight_min": -26.89295196533203,
      "activations/layer2_attention_weight_max": 35.044837951660156,
      "activations/layer2_attention_weight_min": -34.82413101196289,
      "activations/layer3_attention_weight_max": 102.80682373046875,
      "activations/layer3_attention_weight_min": -105.37044525146484,
      "activations/layer4_attention_weight_max": 107.92115783691406,
      "activations/layer4_attention_weight_min": -113.048095703125,
      "activations/layer5_attention_weight_max": 52.44352340698242,
      "activations/layer5_attention_weight_min": -64.85588073730469,
      "activations/layer6_attention_weight_max": 47.03725051879883,
      "activations/layer6_attention_weight_min": -47.941287994384766,
      "activations/layer7_attention_weight_max": 94.48182678222656,
      "activations/layer7_attention_weight_min": -92.68003845214844,
      "activations/layer8_attention_weight_max": 42.88359832763672,
      "activations/layer8_attention_weight_min": -46.50655746459961,
      "activations/layer9_attention_weight_max": 33.437049865722656,
      "activations/layer9_attention_weight_min": -38.34885787963867,
      "epoch": 17.32,
      "learning_rate": 3.866931818181818e-05,
      "loss": 2.7302,
      "step": 298150
    },
    {
      "activations/layer0_attention_weight_max": 16.50632095336914,
      "activations/layer0_attention_weight_min": -13.075404167175293,
      "activations/layer10_attention_weight_max": 35.11623764038086,
      "activations/layer10_attention_weight_min": -33.46013259887695,
      "activations/layer11_attention_weight_max": 33.22370529174805,
      "activations/layer11_attention_weight_min": -33.51525115966797,
      "activations/layer12_attention_weight_max": 26.025224685668945,
      "activations/layer12_attention_weight_min": -24.92314338684082,
      "activations/layer13_attention_weight_max": 36.67026901245117,
      "activations/layer13_attention_weight_min": -35.119285583496094,
      "activations/layer14_attention_weight_max": 38.68120574951172,
      "activations/layer14_attention_weight_min": -38.60800552368164,
      "activations/layer15_attention_weight_max": 34.03630828857422,
      "activations/layer15_attention_weight_min": -36.62644577026367,
      "activations/layer16_attention_weight_max": 28.837053298950195,
      "activations/layer16_attention_weight_min": -29.42210578918457,
      "activations/layer17_attention_weight_max": 27.75890350341797,
      "activations/layer17_attention_weight_min": -28.1290340423584,
      "activations/layer18_attention_weight_max": 33.19169235229492,
      "activations/layer18_attention_weight_min": -26.601512908935547,
      "activations/layer19_attention_weight_max": 34.21780776977539,
      "activations/layer19_attention_weight_min": -34.29812240600586,
      "activations/layer1_attention_weight_max": 16.336706161499023,
      "activations/layer1_attention_weight_min": -15.21398639678955,
      "activations/layer20_attention_weight_max": 32.953636169433594,
      "activations/layer20_attention_weight_min": -27.557451248168945,
      "activations/layer21_attention_weight_max": 31.911907196044922,
      "activations/layer21_attention_weight_min": -27.196605682373047,
      "activations/layer22_attention_weight_max": 44.25656509399414,
      "activations/layer22_attention_weight_min": -34.08116149902344,
      "activations/layer23_attention_weight_max": 36.35256576538086,
      "activations/layer23_attention_weight_min": -26.74343490600586,
      "activations/layer2_attention_weight_max": 34.327789306640625,
      "activations/layer2_attention_weight_min": -35.14612579345703,
      "activations/layer3_attention_weight_max": 101.67455291748047,
      "activations/layer3_attention_weight_min": -105.03548431396484,
      "activations/layer4_attention_weight_max": 106.70488739013672,
      "activations/layer4_attention_weight_min": -106.5948257446289,
      "activations/layer5_attention_weight_max": 49.84398651123047,
      "activations/layer5_attention_weight_min": -64.66812133789062,
      "activations/layer6_attention_weight_max": 44.18219757080078,
      "activations/layer6_attention_weight_min": -47.101558685302734,
      "activations/layer7_attention_weight_max": 96.00106048583984,
      "activations/layer7_attention_weight_min": -95.66425323486328,
      "activations/layer8_attention_weight_max": 43.68067169189453,
      "activations/layer8_attention_weight_min": -45.63188552856445,
      "activations/layer9_attention_weight_max": 32.23924255371094,
      "activations/layer9_attention_weight_min": -33.29377746582031,
      "epoch": 17.33,
      "learning_rate": 3.865037878787878e-05,
      "loss": 2.7323,
      "step": 298200
    },
    {
      "activations/layer0_attention_weight_max": 17.267126083374023,
      "activations/layer0_attention_weight_min": -13.144145011901855,
      "activations/layer10_attention_weight_max": 33.25752639770508,
      "activations/layer10_attention_weight_min": -36.765830993652344,
      "activations/layer11_attention_weight_max": 35.5013542175293,
      "activations/layer11_attention_weight_min": -35.71938705444336,
      "activations/layer12_attention_weight_max": 25.796998977661133,
      "activations/layer12_attention_weight_min": -25.61005973815918,
      "activations/layer13_attention_weight_max": 38.36469268798828,
      "activations/layer13_attention_weight_min": -35.39219665527344,
      "activations/layer14_attention_weight_max": 42.585533142089844,
      "activations/layer14_attention_weight_min": -37.34708023071289,
      "activations/layer15_attention_weight_max": 36.52069091796875,
      "activations/layer15_attention_weight_min": -34.99131774902344,
      "activations/layer16_attention_weight_max": 31.43138313293457,
      "activations/layer16_attention_weight_min": -27.141847610473633,
      "activations/layer17_attention_weight_max": 28.21602439880371,
      "activations/layer17_attention_weight_min": -25.28498077392578,
      "activations/layer18_attention_weight_max": 29.96021842956543,
      "activations/layer18_attention_weight_min": -24.882137298583984,
      "activations/layer19_attention_weight_max": 34.871543884277344,
      "activations/layer19_attention_weight_min": -30.0822811126709,
      "activations/layer1_attention_weight_max": 15.536725044250488,
      "activations/layer1_attention_weight_min": -14.497883796691895,
      "activations/layer20_attention_weight_max": 29.045669555664062,
      "activations/layer20_attention_weight_min": -24.6480712890625,
      "activations/layer21_attention_weight_max": 27.746652603149414,
      "activations/layer21_attention_weight_min": -24.70853042602539,
      "activations/layer22_attention_weight_max": 40.407596588134766,
      "activations/layer22_attention_weight_min": -31.68107032775879,
      "activations/layer23_attention_weight_max": 30.85220718383789,
      "activations/layer23_attention_weight_min": -24.44835662841797,
      "activations/layer2_attention_weight_max": 33.388328552246094,
      "activations/layer2_attention_weight_min": -33.22174072265625,
      "activations/layer3_attention_weight_max": 100.37519073486328,
      "activations/layer3_attention_weight_min": -101.2013931274414,
      "activations/layer4_attention_weight_max": 109.20106506347656,
      "activations/layer4_attention_weight_min": -110.64387512207031,
      "activations/layer5_attention_weight_max": 51.629425048828125,
      "activations/layer5_attention_weight_min": -58.99250030517578,
      "activations/layer6_attention_weight_max": 46.77519989013672,
      "activations/layer6_attention_weight_min": -49.046409606933594,
      "activations/layer7_attention_weight_max": 90.8480453491211,
      "activations/layer7_attention_weight_min": -99.29859161376953,
      "activations/layer8_attention_weight_max": 42.59058380126953,
      "activations/layer8_attention_weight_min": -44.3646240234375,
      "activations/layer9_attention_weight_max": 33.55845260620117,
      "activations/layer9_attention_weight_min": -35.64412307739258,
      "epoch": 17.33,
      "learning_rate": 3.863143939393939e-05,
      "loss": 2.7286,
      "step": 298250
    },
    {
      "activations/layer0_attention_weight_max": 15.753090858459473,
      "activations/layer0_attention_weight_min": -12.686988830566406,
      "activations/layer10_attention_weight_max": 31.788602828979492,
      "activations/layer10_attention_weight_min": -32.60551071166992,
      "activations/layer11_attention_weight_max": 32.32862091064453,
      "activations/layer11_attention_weight_min": -32.751243591308594,
      "activations/layer12_attention_weight_max": 25.899930953979492,
      "activations/layer12_attention_weight_min": -26.707786560058594,
      "activations/layer13_attention_weight_max": 39.52866744995117,
      "activations/layer13_attention_weight_min": -35.008384704589844,
      "activations/layer14_attention_weight_max": 42.01206970214844,
      "activations/layer14_attention_weight_min": -38.962772369384766,
      "activations/layer15_attention_weight_max": 39.137046813964844,
      "activations/layer15_attention_weight_min": -37.02312469482422,
      "activations/layer16_attention_weight_max": 29.13070297241211,
      "activations/layer16_attention_weight_min": -26.539724349975586,
      "activations/layer17_attention_weight_max": 28.316123962402344,
      "activations/layer17_attention_weight_min": -25.431053161621094,
      "activations/layer18_attention_weight_max": 29.81252670288086,
      "activations/layer18_attention_weight_min": -27.53817367553711,
      "activations/layer19_attention_weight_max": 32.31415939331055,
      "activations/layer19_attention_weight_min": -30.75929832458496,
      "activations/layer1_attention_weight_max": 15.605155944824219,
      "activations/layer1_attention_weight_min": -14.434803009033203,
      "activations/layer20_attention_weight_max": 29.734996795654297,
      "activations/layer20_attention_weight_min": -24.917003631591797,
      "activations/layer21_attention_weight_max": 30.22734832763672,
      "activations/layer21_attention_weight_min": -24.383636474609375,
      "activations/layer22_attention_weight_max": 42.59636306762695,
      "activations/layer22_attention_weight_min": -30.34580421447754,
      "activations/layer23_attention_weight_max": 29.554851531982422,
      "activations/layer23_attention_weight_min": -24.24930191040039,
      "activations/layer2_attention_weight_max": 34.7983512878418,
      "activations/layer2_attention_weight_min": -33.47652816772461,
      "activations/layer3_attention_weight_max": 102.85140228271484,
      "activations/layer3_attention_weight_min": -101.5229263305664,
      "activations/layer4_attention_weight_max": 107.1190414428711,
      "activations/layer4_attention_weight_min": -111.63653564453125,
      "activations/layer5_attention_weight_max": 48.69453048706055,
      "activations/layer5_attention_weight_min": -62.1883659362793,
      "activations/layer6_attention_weight_max": 42.577667236328125,
      "activations/layer6_attention_weight_min": -45.13304901123047,
      "activations/layer7_attention_weight_max": 90.24726104736328,
      "activations/layer7_attention_weight_min": -93.23229217529297,
      "activations/layer8_attention_weight_max": 41.463462829589844,
      "activations/layer8_attention_weight_min": -42.44619369506836,
      "activations/layer9_attention_weight_max": 30.135089874267578,
      "activations/layer9_attention_weight_min": -35.017818450927734,
      "epoch": 17.33,
      "learning_rate": 3.86125e-05,
      "loss": 2.7445,
      "step": 298300
    },
    {
      "activations/layer0_attention_weight_max": 16.88518524169922,
      "activations/layer0_attention_weight_min": -13.116256713867188,
      "activations/layer10_attention_weight_max": 32.4976806640625,
      "activations/layer10_attention_weight_min": -33.3280029296875,
      "activations/layer11_attention_weight_max": 31.974843978881836,
      "activations/layer11_attention_weight_min": -32.93975067138672,
      "activations/layer12_attention_weight_max": 25.803550720214844,
      "activations/layer12_attention_weight_min": -26.155418395996094,
      "activations/layer13_attention_weight_max": 39.62848663330078,
      "activations/layer13_attention_weight_min": -35.55309295654297,
      "activations/layer14_attention_weight_max": 45.65824890136719,
      "activations/layer14_attention_weight_min": -35.88139724731445,
      "activations/layer15_attention_weight_max": 38.24441909790039,
      "activations/layer15_attention_weight_min": -35.32112503051758,
      "activations/layer16_attention_weight_max": 30.676647186279297,
      "activations/layer16_attention_weight_min": -28.29933738708496,
      "activations/layer17_attention_weight_max": 29.15991973876953,
      "activations/layer17_attention_weight_min": -25.212818145751953,
      "activations/layer18_attention_weight_max": 30.842504501342773,
      "activations/layer18_attention_weight_min": -24.204435348510742,
      "activations/layer19_attention_weight_max": 32.67463684082031,
      "activations/layer19_attention_weight_min": -29.436120986938477,
      "activations/layer1_attention_weight_max": 17.6107177734375,
      "activations/layer1_attention_weight_min": -15.689815521240234,
      "activations/layer20_attention_weight_max": 29.73558807373047,
      "activations/layer20_attention_weight_min": -23.86836814880371,
      "activations/layer21_attention_weight_max": 28.769065856933594,
      "activations/layer21_attention_weight_min": -23.778514862060547,
      "activations/layer22_attention_weight_max": 45.498111724853516,
      "activations/layer22_attention_weight_min": -29.528657913208008,
      "activations/layer23_attention_weight_max": 32.03287887573242,
      "activations/layer23_attention_weight_min": -25.115276336669922,
      "activations/layer2_attention_weight_max": 33.866363525390625,
      "activations/layer2_attention_weight_min": -32.98385238647461,
      "activations/layer3_attention_weight_max": 101.51573944091797,
      "activations/layer3_attention_weight_min": -100.92366790771484,
      "activations/layer4_attention_weight_max": 104.4671859741211,
      "activations/layer4_attention_weight_min": -103.85904693603516,
      "activations/layer5_attention_weight_max": 50.79425811767578,
      "activations/layer5_attention_weight_min": -58.7698974609375,
      "activations/layer6_attention_weight_max": 43.3785285949707,
      "activations/layer6_attention_weight_min": -45.332923889160156,
      "activations/layer7_attention_weight_max": 87.78192138671875,
      "activations/layer7_attention_weight_min": -96.21488189697266,
      "activations/layer8_attention_weight_max": 39.990447998046875,
      "activations/layer8_attention_weight_min": -42.2937126159668,
      "activations/layer9_attention_weight_max": 30.45819091796875,
      "activations/layer9_attention_weight_min": -32.08500289916992,
      "epoch": 17.34,
      "learning_rate": 3.8593560606060605e-05,
      "loss": 2.7255,
      "step": 298350
    },
    {
      "activations/layer0_attention_weight_max": 16.886625289916992,
      "activations/layer0_attention_weight_min": -14.008952140808105,
      "activations/layer10_attention_weight_max": 36.444061279296875,
      "activations/layer10_attention_weight_min": -35.157371520996094,
      "activations/layer11_attention_weight_max": 36.361297607421875,
      "activations/layer11_attention_weight_min": -34.460655212402344,
      "activations/layer12_attention_weight_max": 25.694238662719727,
      "activations/layer12_attention_weight_min": -25.47336196899414,
      "activations/layer13_attention_weight_max": 36.79301834106445,
      "activations/layer13_attention_weight_min": -36.6528434753418,
      "activations/layer14_attention_weight_max": 40.156009674072266,
      "activations/layer14_attention_weight_min": -38.136226654052734,
      "activations/layer15_attention_weight_max": 35.46659469604492,
      "activations/layer15_attention_weight_min": -34.52863693237305,
      "activations/layer16_attention_weight_max": 29.877573013305664,
      "activations/layer16_attention_weight_min": -27.27081871032715,
      "activations/layer17_attention_weight_max": 28.4893741607666,
      "activations/layer17_attention_weight_min": -25.654783248901367,
      "activations/layer18_attention_weight_max": 30.93988037109375,
      "activations/layer18_attention_weight_min": -24.980880737304688,
      "activations/layer19_attention_weight_max": 34.18281173706055,
      "activations/layer19_attention_weight_min": -30.3019962310791,
      "activations/layer1_attention_weight_max": 16.9202880859375,
      "activations/layer1_attention_weight_min": -15.013914108276367,
      "activations/layer20_attention_weight_max": 27.46831703186035,
      "activations/layer20_attention_weight_min": -25.062082290649414,
      "activations/layer21_attention_weight_max": 27.54437255859375,
      "activations/layer21_attention_weight_min": -25.236295700073242,
      "activations/layer22_attention_weight_max": 39.61078643798828,
      "activations/layer22_attention_weight_min": -30.782808303833008,
      "activations/layer23_attention_weight_max": 31.198942184448242,
      "activations/layer23_attention_weight_min": -25.21481704711914,
      "activations/layer2_attention_weight_max": 33.03053283691406,
      "activations/layer2_attention_weight_min": -32.42516326904297,
      "activations/layer3_attention_weight_max": 98.90618133544922,
      "activations/layer3_attention_weight_min": -98.77799224853516,
      "activations/layer4_attention_weight_max": 106.9687728881836,
      "activations/layer4_attention_weight_min": -108.2343978881836,
      "activations/layer5_attention_weight_max": 51.218238830566406,
      "activations/layer5_attention_weight_min": -59.25799560546875,
      "activations/layer6_attention_weight_max": 43.44690704345703,
      "activations/layer6_attention_weight_min": -46.405635833740234,
      "activations/layer7_attention_weight_max": 94.9189682006836,
      "activations/layer7_attention_weight_min": -96.50296020507812,
      "activations/layer8_attention_weight_max": 40.913814544677734,
      "activations/layer8_attention_weight_min": -42.273380279541016,
      "activations/layer9_attention_weight_max": 31.59598731994629,
      "activations/layer9_attention_weight_min": -33.44529342651367,
      "epoch": 17.34,
      "learning_rate": 3.8574621212121206e-05,
      "loss": 2.7444,
      "step": 298400
    },
    {
      "activations/layer0_attention_weight_max": 15.83216381072998,
      "activations/layer0_attention_weight_min": -12.461256980895996,
      "activations/layer10_attention_weight_max": 35.28190231323242,
      "activations/layer10_attention_weight_min": -34.62287139892578,
      "activations/layer11_attention_weight_max": 37.14885330200195,
      "activations/layer11_attention_weight_min": -36.91176223754883,
      "activations/layer12_attention_weight_max": 27.799579620361328,
      "activations/layer12_attention_weight_min": -27.058908462524414,
      "activations/layer13_attention_weight_max": 40.36705780029297,
      "activations/layer13_attention_weight_min": -39.033687591552734,
      "activations/layer14_attention_weight_max": 49.728885650634766,
      "activations/layer14_attention_weight_min": -39.61069869995117,
      "activations/layer15_attention_weight_max": 39.551883697509766,
      "activations/layer15_attention_weight_min": -36.24373245239258,
      "activations/layer16_attention_weight_max": 33.00664520263672,
      "activations/layer16_attention_weight_min": -26.58926773071289,
      "activations/layer17_attention_weight_max": 33.045833587646484,
      "activations/layer17_attention_weight_min": -26.369112014770508,
      "activations/layer18_attention_weight_max": 34.57193374633789,
      "activations/layer18_attention_weight_min": -24.40532112121582,
      "activations/layer19_attention_weight_max": 35.56554412841797,
      "activations/layer19_attention_weight_min": -31.616439819335938,
      "activations/layer1_attention_weight_max": 16.34667205810547,
      "activations/layer1_attention_weight_min": -15.120715141296387,
      "activations/layer20_attention_weight_max": 31.804624557495117,
      "activations/layer20_attention_weight_min": -23.562480926513672,
      "activations/layer21_attention_weight_max": 33.19184494018555,
      "activations/layer21_attention_weight_min": -22.424571990966797,
      "activations/layer22_attention_weight_max": 46.37613296508789,
      "activations/layer22_attention_weight_min": -31.007173538208008,
      "activations/layer23_attention_weight_max": 32.59510803222656,
      "activations/layer23_attention_weight_min": -25.56674575805664,
      "activations/layer2_attention_weight_max": 33.6552734375,
      "activations/layer2_attention_weight_min": -34.277198791503906,
      "activations/layer3_attention_weight_max": 104.06234741210938,
      "activations/layer3_attention_weight_min": -111.40938568115234,
      "activations/layer4_attention_weight_max": 108.75897216796875,
      "activations/layer4_attention_weight_min": -108.2683334350586,
      "activations/layer5_attention_weight_max": 50.29619598388672,
      "activations/layer5_attention_weight_min": -61.10034942626953,
      "activations/layer6_attention_weight_max": 43.76681137084961,
      "activations/layer6_attention_weight_min": -47.143287658691406,
      "activations/layer7_attention_weight_max": 94.52369689941406,
      "activations/layer7_attention_weight_min": -91.93708038330078,
      "activations/layer8_attention_weight_max": 42.22331237792969,
      "activations/layer8_attention_weight_min": -41.939735412597656,
      "activations/layer9_attention_weight_max": 33.33338928222656,
      "activations/layer9_attention_weight_min": -34.19186782836914,
      "epoch": 17.34,
      "learning_rate": 3.855568181818182e-05,
      "loss": 2.7285,
      "step": 298450
    },
    {
      "activations/layer0_attention_weight_max": 16.22216033935547,
      "activations/layer0_attention_weight_min": -12.800870895385742,
      "activations/layer10_attention_weight_max": 33.66632080078125,
      "activations/layer10_attention_weight_min": -32.941078186035156,
      "activations/layer11_attention_weight_max": 32.110076904296875,
      "activations/layer11_attention_weight_min": -33.01666259765625,
      "activations/layer12_attention_weight_max": 24.81866455078125,
      "activations/layer12_attention_weight_min": -26.10437774658203,
      "activations/layer13_attention_weight_max": 35.89581298828125,
      "activations/layer13_attention_weight_min": -33.35707092285156,
      "activations/layer14_attention_weight_max": 39.005027770996094,
      "activations/layer14_attention_weight_min": -37.39058303833008,
      "activations/layer15_attention_weight_max": 36.06529235839844,
      "activations/layer15_attention_weight_min": -32.730003356933594,
      "activations/layer16_attention_weight_max": 28.742055892944336,
      "activations/layer16_attention_weight_min": -27.1021785736084,
      "activations/layer17_attention_weight_max": 28.25295639038086,
      "activations/layer17_attention_weight_min": -25.24650001525879,
      "activations/layer18_attention_weight_max": 32.32661056518555,
      "activations/layer18_attention_weight_min": -24.66434097290039,
      "activations/layer19_attention_weight_max": 34.289791107177734,
      "activations/layer19_attention_weight_min": -33.176937103271484,
      "activations/layer1_attention_weight_max": 16.040653228759766,
      "activations/layer1_attention_weight_min": -15.470035552978516,
      "activations/layer20_attention_weight_max": 28.638832092285156,
      "activations/layer20_attention_weight_min": -28.3378963470459,
      "activations/layer21_attention_weight_max": 26.607620239257812,
      "activations/layer21_attention_weight_min": -27.442663192749023,
      "activations/layer22_attention_weight_max": 43.56488037109375,
      "activations/layer22_attention_weight_min": -31.951202392578125,
      "activations/layer23_attention_weight_max": 30.20136070251465,
      "activations/layer23_attention_weight_min": -25.480533599853516,
      "activations/layer2_attention_weight_max": 32.73228454589844,
      "activations/layer2_attention_weight_min": -34.714290618896484,
      "activations/layer3_attention_weight_max": 97.91796875,
      "activations/layer3_attention_weight_min": -100.61760711669922,
      "activations/layer4_attention_weight_max": 103.1694107055664,
      "activations/layer4_attention_weight_min": -112.50817108154297,
      "activations/layer5_attention_weight_max": 50.153961181640625,
      "activations/layer5_attention_weight_min": -60.67236328125,
      "activations/layer6_attention_weight_max": 44.04331588745117,
      "activations/layer6_attention_weight_min": -46.226829528808594,
      "activations/layer7_attention_weight_max": 90.40553283691406,
      "activations/layer7_attention_weight_min": -95.39939880371094,
      "activations/layer8_attention_weight_max": 43.10026550292969,
      "activations/layer8_attention_weight_min": -42.423927307128906,
      "activations/layer9_attention_weight_max": 32.311302185058594,
      "activations/layer9_attention_weight_min": -33.19167709350586,
      "epoch": 17.34,
      "learning_rate": 3.853674242424242e-05,
      "loss": 2.7235,
      "step": 298500
    },
    {
      "activations/layer0_attention_weight_max": 16.590635299682617,
      "activations/layer0_attention_weight_min": -13.922578811645508,
      "activations/layer10_attention_weight_max": 35.00544357299805,
      "activations/layer10_attention_weight_min": -33.75363540649414,
      "activations/layer11_attention_weight_max": 35.16863250732422,
      "activations/layer11_attention_weight_min": -34.39692306518555,
      "activations/layer12_attention_weight_max": 27.97956657409668,
      "activations/layer12_attention_weight_min": -26.442533493041992,
      "activations/layer13_attention_weight_max": 40.820289611816406,
      "activations/layer13_attention_weight_min": -36.06715393066406,
      "activations/layer14_attention_weight_max": 47.2354736328125,
      "activations/layer14_attention_weight_min": -38.65224075317383,
      "activations/layer15_attention_weight_max": 42.30074691772461,
      "activations/layer15_attention_weight_min": -35.94538497924805,
      "activations/layer16_attention_weight_max": 33.05035400390625,
      "activations/layer16_attention_weight_min": -27.5245304107666,
      "activations/layer17_attention_weight_max": 31.110687255859375,
      "activations/layer17_attention_weight_min": -27.008033752441406,
      "activations/layer18_attention_weight_max": 33.171531677246094,
      "activations/layer18_attention_weight_min": -24.843673706054688,
      "activations/layer19_attention_weight_max": 39.26144027709961,
      "activations/layer19_attention_weight_min": -30.50390625,
      "activations/layer1_attention_weight_max": 16.34246826171875,
      "activations/layer1_attention_weight_min": -15.663904190063477,
      "activations/layer20_attention_weight_max": 35.95219802856445,
      "activations/layer20_attention_weight_min": -25.067209243774414,
      "activations/layer21_attention_weight_max": 33.187828063964844,
      "activations/layer21_attention_weight_min": -24.364065170288086,
      "activations/layer22_attention_weight_max": 46.20405578613281,
      "activations/layer22_attention_weight_min": -31.10790252685547,
      "activations/layer23_attention_weight_max": 39.433597564697266,
      "activations/layer23_attention_weight_min": -27.199115753173828,
      "activations/layer2_attention_weight_max": 33.47521209716797,
      "activations/layer2_attention_weight_min": -34.03474044799805,
      "activations/layer3_attention_weight_max": 104.35531616210938,
      "activations/layer3_attention_weight_min": -100.1416244506836,
      "activations/layer4_attention_weight_max": 109.85442352294922,
      "activations/layer4_attention_weight_min": -107.7162857055664,
      "activations/layer5_attention_weight_max": 52.62261962890625,
      "activations/layer5_attention_weight_min": -64.84270477294922,
      "activations/layer6_attention_weight_max": 43.47285461425781,
      "activations/layer6_attention_weight_min": -47.12568664550781,
      "activations/layer7_attention_weight_max": 92.64839172363281,
      "activations/layer7_attention_weight_min": -97.81853485107422,
      "activations/layer8_attention_weight_max": 42.86549758911133,
      "activations/layer8_attention_weight_min": -43.98378372192383,
      "activations/layer9_attention_weight_max": 33.061641693115234,
      "activations/layer9_attention_weight_min": -35.50183868408203,
      "epoch": 17.35,
      "learning_rate": 3.8517803030303024e-05,
      "loss": 2.7321,
      "step": 298550
    },
    {
      "activations/layer0_attention_weight_max": 17.023210525512695,
      "activations/layer0_attention_weight_min": -12.678022384643555,
      "activations/layer10_attention_weight_max": 31.755355834960938,
      "activations/layer10_attention_weight_min": -33.81528854370117,
      "activations/layer11_attention_weight_max": 32.691612243652344,
      "activations/layer11_attention_weight_min": -34.44713592529297,
      "activations/layer12_attention_weight_max": 25.337228775024414,
      "activations/layer12_attention_weight_min": -24.893966674804688,
      "activations/layer13_attention_weight_max": 36.47346115112305,
      "activations/layer13_attention_weight_min": -35.76275634765625,
      "activations/layer14_attention_weight_max": 43.035160064697266,
      "activations/layer14_attention_weight_min": -37.56633758544922,
      "activations/layer15_attention_weight_max": 34.618526458740234,
      "activations/layer15_attention_weight_min": -35.083621978759766,
      "activations/layer16_attention_weight_max": 28.40970802307129,
      "activations/layer16_attention_weight_min": -28.975399017333984,
      "activations/layer17_attention_weight_max": 30.090702056884766,
      "activations/layer17_attention_weight_min": -26.469823837280273,
      "activations/layer18_attention_weight_max": 31.73951530456543,
      "activations/layer18_attention_weight_min": -26.405656814575195,
      "activations/layer19_attention_weight_max": 31.9315128326416,
      "activations/layer19_attention_weight_min": -33.24934005737305,
      "activations/layer1_attention_weight_max": 16.25032615661621,
      "activations/layer1_attention_weight_min": -15.248198509216309,
      "activations/layer20_attention_weight_max": 27.736093521118164,
      "activations/layer20_attention_weight_min": -25.454395294189453,
      "activations/layer21_attention_weight_max": 29.81553077697754,
      "activations/layer21_attention_weight_min": -25.102685928344727,
      "activations/layer22_attention_weight_max": 42.335025787353516,
      "activations/layer22_attention_weight_min": -31.78754997253418,
      "activations/layer23_attention_weight_max": 32.673126220703125,
      "activations/layer23_attention_weight_min": -25.537473678588867,
      "activations/layer2_attention_weight_max": 33.91261291503906,
      "activations/layer2_attention_weight_min": -33.68087387084961,
      "activations/layer3_attention_weight_max": 99.30131530761719,
      "activations/layer3_attention_weight_min": -99.65966796875,
      "activations/layer4_attention_weight_max": 107.17659759521484,
      "activations/layer4_attention_weight_min": -107.03295135498047,
      "activations/layer5_attention_weight_max": 50.9273681640625,
      "activations/layer5_attention_weight_min": -60.55859375,
      "activations/layer6_attention_weight_max": 43.698116302490234,
      "activations/layer6_attention_weight_min": -45.25789260864258,
      "activations/layer7_attention_weight_max": 90.9777603149414,
      "activations/layer7_attention_weight_min": -94.59559631347656,
      "activations/layer8_attention_weight_max": 39.738346099853516,
      "activations/layer8_attention_weight_min": -41.01601028442383,
      "activations/layer9_attention_weight_max": 30.950607299804688,
      "activations/layer9_attention_weight_min": -33.14280319213867,
      "epoch": 17.35,
      "learning_rate": 3.849886363636363e-05,
      "loss": 2.7292,
      "step": 298600
    },
    {
      "activations/layer0_attention_weight_max": 17.30833625793457,
      "activations/layer0_attention_weight_min": -12.561700820922852,
      "activations/layer10_attention_weight_max": 32.44330596923828,
      "activations/layer10_attention_weight_min": -33.50634765625,
      "activations/layer11_attention_weight_max": 32.033721923828125,
      "activations/layer11_attention_weight_min": -32.444576263427734,
      "activations/layer12_attention_weight_max": 24.048063278198242,
      "activations/layer12_attention_weight_min": -25.46530532836914,
      "activations/layer13_attention_weight_max": 37.85083770751953,
      "activations/layer13_attention_weight_min": -34.197166442871094,
      "activations/layer14_attention_weight_max": 38.36688995361328,
      "activations/layer14_attention_weight_min": -36.47202682495117,
      "activations/layer15_attention_weight_max": 37.03644943237305,
      "activations/layer15_attention_weight_min": -33.432979583740234,
      "activations/layer16_attention_weight_max": 28.657760620117188,
      "activations/layer16_attention_weight_min": -27.723670959472656,
      "activations/layer17_attention_weight_max": 34.32745361328125,
      "activations/layer17_attention_weight_min": -25.544775009155273,
      "activations/layer18_attention_weight_max": 36.11540985107422,
      "activations/layer18_attention_weight_min": -25.06376838684082,
      "activations/layer19_attention_weight_max": 34.93704605102539,
      "activations/layer19_attention_weight_min": -28.948474884033203,
      "activations/layer1_attention_weight_max": 16.436819076538086,
      "activations/layer1_attention_weight_min": -15.395088195800781,
      "activations/layer20_attention_weight_max": 32.09242630004883,
      "activations/layer20_attention_weight_min": -23.573570251464844,
      "activations/layer21_attention_weight_max": 31.81510353088379,
      "activations/layer21_attention_weight_min": -23.901769638061523,
      "activations/layer22_attention_weight_max": 39.87803268432617,
      "activations/layer22_attention_weight_min": -29.17375946044922,
      "activations/layer23_attention_weight_max": 35.58676528930664,
      "activations/layer23_attention_weight_min": -23.653783798217773,
      "activations/layer2_attention_weight_max": 32.86190414428711,
      "activations/layer2_attention_weight_min": -33.3568000793457,
      "activations/layer3_attention_weight_max": 96.9721450805664,
      "activations/layer3_attention_weight_min": -98.80095672607422,
      "activations/layer4_attention_weight_max": 102.22502136230469,
      "activations/layer4_attention_weight_min": -106.9159164428711,
      "activations/layer5_attention_weight_max": 48.33625793457031,
      "activations/layer5_attention_weight_min": -62.74274826049805,
      "activations/layer6_attention_weight_max": 41.47028732299805,
      "activations/layer6_attention_weight_min": -45.73946762084961,
      "activations/layer7_attention_weight_max": 87.54912567138672,
      "activations/layer7_attention_weight_min": -91.16643524169922,
      "activations/layer8_attention_weight_max": 39.157920837402344,
      "activations/layer8_attention_weight_min": -41.96435546875,
      "activations/layer9_attention_weight_max": 30.553430557250977,
      "activations/layer9_attention_weight_min": -33.284053802490234,
      "epoch": 17.35,
      "learning_rate": 3.847992424242424e-05,
      "loss": 2.746,
      "step": 298650
    },
    {
      "activations/layer0_attention_weight_max": 16.379894256591797,
      "activations/layer0_attention_weight_min": -13.108565330505371,
      "activations/layer10_attention_weight_max": 32.71147537231445,
      "activations/layer10_attention_weight_min": -33.51676559448242,
      "activations/layer11_attention_weight_max": 32.41400909423828,
      "activations/layer11_attention_weight_min": -32.16606903076172,
      "activations/layer12_attention_weight_max": 25.710935592651367,
      "activations/layer12_attention_weight_min": -25.86472511291504,
      "activations/layer13_attention_weight_max": 37.39097595214844,
      "activations/layer13_attention_weight_min": -34.15702438354492,
      "activations/layer14_attention_weight_max": 40.586856842041016,
      "activations/layer14_attention_weight_min": -34.93678665161133,
      "activations/layer15_attention_weight_max": 35.90850067138672,
      "activations/layer15_attention_weight_min": -32.92350769042969,
      "activations/layer16_attention_weight_max": 28.36979866027832,
      "activations/layer16_attention_weight_min": -26.53709602355957,
      "activations/layer17_attention_weight_max": 29.109506607055664,
      "activations/layer17_attention_weight_min": -25.49309730529785,
      "activations/layer18_attention_weight_max": 30.359554290771484,
      "activations/layer18_attention_weight_min": -24.24262046813965,
      "activations/layer19_attention_weight_max": 31.304128646850586,
      "activations/layer19_attention_weight_min": -29.920557022094727,
      "activations/layer1_attention_weight_max": 16.182523727416992,
      "activations/layer1_attention_weight_min": -14.306243896484375,
      "activations/layer20_attention_weight_max": 30.114810943603516,
      "activations/layer20_attention_weight_min": -24.105928421020508,
      "activations/layer21_attention_weight_max": 27.884531021118164,
      "activations/layer21_attention_weight_min": -23.76444435119629,
      "activations/layer22_attention_weight_max": 37.6818733215332,
      "activations/layer22_attention_weight_min": -29.622028350830078,
      "activations/layer23_attention_weight_max": 31.63739585876465,
      "activations/layer23_attention_weight_min": -24.588598251342773,
      "activations/layer2_attention_weight_max": 31.37922477722168,
      "activations/layer2_attention_weight_min": -32.090877532958984,
      "activations/layer3_attention_weight_max": 94.70528411865234,
      "activations/layer3_attention_weight_min": -96.63896179199219,
      "activations/layer4_attention_weight_max": 104.86617279052734,
      "activations/layer4_attention_weight_min": -105.03436279296875,
      "activations/layer5_attention_weight_max": 50.781639099121094,
      "activations/layer5_attention_weight_min": -60.08317184448242,
      "activations/layer6_attention_weight_max": 42.13166809082031,
      "activations/layer6_attention_weight_min": -44.19792175292969,
      "activations/layer7_attention_weight_max": 90.86625671386719,
      "activations/layer7_attention_weight_min": -93.74514770507812,
      "activations/layer8_attention_weight_max": 38.91059112548828,
      "activations/layer8_attention_weight_min": -43.968387603759766,
      "activations/layer9_attention_weight_max": 31.30780601501465,
      "activations/layer9_attention_weight_min": -32.949886322021484,
      "epoch": 17.36,
      "learning_rate": 3.846098484848485e-05,
      "loss": 2.736,
      "step": 298700
    },
    {
      "activations/layer0_attention_weight_max": 16.911088943481445,
      "activations/layer0_attention_weight_min": -13.994172096252441,
      "activations/layer10_attention_weight_max": 32.4005241394043,
      "activations/layer10_attention_weight_min": -33.086097717285156,
      "activations/layer11_attention_weight_max": 32.45890808105469,
      "activations/layer11_attention_weight_min": -33.725608825683594,
      "activations/layer12_attention_weight_max": 27.882850646972656,
      "activations/layer12_attention_weight_min": -27.161048889160156,
      "activations/layer13_attention_weight_max": 41.59113311767578,
      "activations/layer13_attention_weight_min": -36.39588165283203,
      "activations/layer14_attention_weight_max": 45.31613540649414,
      "activations/layer14_attention_weight_min": -38.368228912353516,
      "activations/layer15_attention_weight_max": 38.140167236328125,
      "activations/layer15_attention_weight_min": -35.041873931884766,
      "activations/layer16_attention_weight_max": 34.458038330078125,
      "activations/layer16_attention_weight_min": -28.211639404296875,
      "activations/layer17_attention_weight_max": 29.032724380493164,
      "activations/layer17_attention_weight_min": -25.367414474487305,
      "activations/layer18_attention_weight_max": 32.391719818115234,
      "activations/layer18_attention_weight_min": -25.799232482910156,
      "activations/layer19_attention_weight_max": 38.384517669677734,
      "activations/layer19_attention_weight_min": -30.78189468383789,
      "activations/layer1_attention_weight_max": 17.38747215270996,
      "activations/layer1_attention_weight_min": -14.998690605163574,
      "activations/layer20_attention_weight_max": 32.04280471801758,
      "activations/layer20_attention_weight_min": -25.357702255249023,
      "activations/layer21_attention_weight_max": 33.937618255615234,
      "activations/layer21_attention_weight_min": -26.357797622680664,
      "activations/layer22_attention_weight_max": 48.82305908203125,
      "activations/layer22_attention_weight_min": -32.46635437011719,
      "activations/layer23_attention_weight_max": 36.307456970214844,
      "activations/layer23_attention_weight_min": -25.41594696044922,
      "activations/layer2_attention_weight_max": 32.72365951538086,
      "activations/layer2_attention_weight_min": -33.14942169189453,
      "activations/layer3_attention_weight_max": 102.34996795654297,
      "activations/layer3_attention_weight_min": -102.28532409667969,
      "activations/layer4_attention_weight_max": 109.24834442138672,
      "activations/layer4_attention_weight_min": -108.25817108154297,
      "activations/layer5_attention_weight_max": 59.02357482910156,
      "activations/layer5_attention_weight_min": -63.005088806152344,
      "activations/layer6_attention_weight_max": 47.09918975830078,
      "activations/layer6_attention_weight_min": -47.839515686035156,
      "activations/layer7_attention_weight_max": 95.0230941772461,
      "activations/layer7_attention_weight_min": -95.10673522949219,
      "activations/layer8_attention_weight_max": 43.146183013916016,
      "activations/layer8_attention_weight_min": -43.52860641479492,
      "activations/layer9_attention_weight_max": 31.673017501831055,
      "activations/layer9_attention_weight_min": -34.281700134277344,
      "epoch": 17.36,
      "learning_rate": 3.844204545454545e-05,
      "loss": 2.7115,
      "step": 298750
    },
    {
      "activations/layer0_attention_weight_max": 16.25860595703125,
      "activations/layer0_attention_weight_min": -12.732626914978027,
      "activations/layer10_attention_weight_max": 41.30580520629883,
      "activations/layer10_attention_weight_min": -40.00625991821289,
      "activations/layer11_attention_weight_max": 40.67901611328125,
      "activations/layer11_attention_weight_min": -38.26385498046875,
      "activations/layer12_attention_weight_max": 32.71330261230469,
      "activations/layer12_attention_weight_min": -33.43427658081055,
      "activations/layer13_attention_weight_max": 47.096500396728516,
      "activations/layer13_attention_weight_min": -41.32970428466797,
      "activations/layer14_attention_weight_max": 55.761138916015625,
      "activations/layer14_attention_weight_min": -44.21879196166992,
      "activations/layer15_attention_weight_max": 48.662391662597656,
      "activations/layer15_attention_weight_min": -42.72132110595703,
      "activations/layer16_attention_weight_max": 35.95053482055664,
      "activations/layer16_attention_weight_min": -33.593597412109375,
      "activations/layer17_attention_weight_max": 43.11492919921875,
      "activations/layer17_attention_weight_min": -32.14531326293945,
      "activations/layer18_attention_weight_max": 39.49855422973633,
      "activations/layer18_attention_weight_min": -30.09237289428711,
      "activations/layer19_attention_weight_max": 44.074092864990234,
      "activations/layer19_attention_weight_min": -35.54117965698242,
      "activations/layer1_attention_weight_max": 16.45176124572754,
      "activations/layer1_attention_weight_min": -15.134726524353027,
      "activations/layer20_attention_weight_max": 35.24755096435547,
      "activations/layer20_attention_weight_min": -28.506534576416016,
      "activations/layer21_attention_weight_max": 37.89105224609375,
      "activations/layer21_attention_weight_min": -28.060253143310547,
      "activations/layer22_attention_weight_max": 61.125282287597656,
      "activations/layer22_attention_weight_min": -38.487152099609375,
      "activations/layer23_attention_weight_max": 38.766353607177734,
      "activations/layer23_attention_weight_min": -27.345478057861328,
      "activations/layer2_attention_weight_max": 37.26971435546875,
      "activations/layer2_attention_weight_min": -35.16765213012695,
      "activations/layer3_attention_weight_max": 101.499267578125,
      "activations/layer3_attention_weight_min": -103.17013549804688,
      "activations/layer4_attention_weight_max": 103.49637603759766,
      "activations/layer4_attention_weight_min": -111.76802825927734,
      "activations/layer5_attention_weight_max": 49.62276840209961,
      "activations/layer5_attention_weight_min": -63.500301361083984,
      "activations/layer6_attention_weight_max": 44.25874328613281,
      "activations/layer6_attention_weight_min": -47.541202545166016,
      "activations/layer7_attention_weight_max": 94.16044616699219,
      "activations/layer7_attention_weight_min": -102.35234832763672,
      "activations/layer8_attention_weight_max": 45.30205535888672,
      "activations/layer8_attention_weight_min": -46.34075164794922,
      "activations/layer9_attention_weight_max": 36.715370178222656,
      "activations/layer9_attention_weight_min": -37.110286712646484,
      "epoch": 17.36,
      "learning_rate": 3.842310606060605e-05,
      "loss": 2.7444,
      "step": 298800
    },
    {
      "activations/layer0_attention_weight_max": 16.407238006591797,
      "activations/layer0_attention_weight_min": -14.595383644104004,
      "activations/layer10_attention_weight_max": 39.875553131103516,
      "activations/layer10_attention_weight_min": -37.72195053100586,
      "activations/layer11_attention_weight_max": 36.92509841918945,
      "activations/layer11_attention_weight_min": -35.80324172973633,
      "activations/layer12_attention_weight_max": 27.222721099853516,
      "activations/layer12_attention_weight_min": -27.037776947021484,
      "activations/layer13_attention_weight_max": 40.583824157714844,
      "activations/layer13_attention_weight_min": -36.836402893066406,
      "activations/layer14_attention_weight_max": 41.77357864379883,
      "activations/layer14_attention_weight_min": -38.869773864746094,
      "activations/layer15_attention_weight_max": 38.226436614990234,
      "activations/layer15_attention_weight_min": -36.32829666137695,
      "activations/layer16_attention_weight_max": 29.220172882080078,
      "activations/layer16_attention_weight_min": -28.853378295898438,
      "activations/layer17_attention_weight_max": 29.455657958984375,
      "activations/layer17_attention_weight_min": -25.2534236907959,
      "activations/layer18_attention_weight_max": 29.932025909423828,
      "activations/layer18_attention_weight_min": -24.62953758239746,
      "activations/layer19_attention_weight_max": 30.078384399414062,
      "activations/layer19_attention_weight_min": -30.8648681640625,
      "activations/layer1_attention_weight_max": 16.584836959838867,
      "activations/layer1_attention_weight_min": -15.135866165161133,
      "activations/layer20_attention_weight_max": 25.410112380981445,
      "activations/layer20_attention_weight_min": -24.42464256286621,
      "activations/layer21_attention_weight_max": 25.36752700805664,
      "activations/layer21_attention_weight_min": -22.471715927124023,
      "activations/layer22_attention_weight_max": 35.49749755859375,
      "activations/layer22_attention_weight_min": -33.40888214111328,
      "activations/layer23_attention_weight_max": 31.816646575927734,
      "activations/layer23_attention_weight_min": -24.816661834716797,
      "activations/layer2_attention_weight_max": 34.456825256347656,
      "activations/layer2_attention_weight_min": -34.15388870239258,
      "activations/layer3_attention_weight_max": 105.21403503417969,
      "activations/layer3_attention_weight_min": -107.98555755615234,
      "activations/layer4_attention_weight_max": 110.4885482788086,
      "activations/layer4_attention_weight_min": -109.89996337890625,
      "activations/layer5_attention_weight_max": 51.23897933959961,
      "activations/layer5_attention_weight_min": -62.35920333862305,
      "activations/layer6_attention_weight_max": 45.805335998535156,
      "activations/layer6_attention_weight_min": -48.41948699951172,
      "activations/layer7_attention_weight_max": 103.24568939208984,
      "activations/layer7_attention_weight_min": -96.15470123291016,
      "activations/layer8_attention_weight_max": 46.328086853027344,
      "activations/layer8_attention_weight_min": -45.612449645996094,
      "activations/layer9_attention_weight_max": 37.978607177734375,
      "activations/layer9_attention_weight_min": -37.60646057128906,
      "epoch": 17.36,
      "learning_rate": 3.840416666666667e-05,
      "loss": 2.7218,
      "step": 298850
    },
    {
      "activations/layer0_attention_weight_max": 16.397199630737305,
      "activations/layer0_attention_weight_min": -13.537684440612793,
      "activations/layer10_attention_weight_max": 35.329063415527344,
      "activations/layer10_attention_weight_min": -35.58077621459961,
      "activations/layer11_attention_weight_max": 35.89649200439453,
      "activations/layer11_attention_weight_min": -39.06037139892578,
      "activations/layer12_attention_weight_max": 30.653215408325195,
      "activations/layer12_attention_weight_min": -27.12187385559082,
      "activations/layer13_attention_weight_max": 46.87293243408203,
      "activations/layer13_attention_weight_min": -36.15966033935547,
      "activations/layer14_attention_weight_max": 51.60529327392578,
      "activations/layer14_attention_weight_min": -38.042091369628906,
      "activations/layer15_attention_weight_max": 43.75065612792969,
      "activations/layer15_attention_weight_min": -36.609004974365234,
      "activations/layer16_attention_weight_max": 31.929655075073242,
      "activations/layer16_attention_weight_min": -26.346817016601562,
      "activations/layer17_attention_weight_max": 31.22464370727539,
      "activations/layer17_attention_weight_min": -26.449838638305664,
      "activations/layer18_attention_weight_max": 30.44269371032715,
      "activations/layer18_attention_weight_min": -24.11819076538086,
      "activations/layer19_attention_weight_max": 32.461605072021484,
      "activations/layer19_attention_weight_min": -31.622251510620117,
      "activations/layer1_attention_weight_max": 17.925588607788086,
      "activations/layer1_attention_weight_min": -16.706951141357422,
      "activations/layer20_attention_weight_max": 28.874982833862305,
      "activations/layer20_attention_weight_min": -25.309036254882812,
      "activations/layer21_attention_weight_max": 28.01706886291504,
      "activations/layer21_attention_weight_min": -23.05765724182129,
      "activations/layer22_attention_weight_max": 40.16440200805664,
      "activations/layer22_attention_weight_min": -29.46942710876465,
      "activations/layer23_attention_weight_max": 31.342161178588867,
      "activations/layer23_attention_weight_min": -23.423015594482422,
      "activations/layer2_attention_weight_max": 37.5863037109375,
      "activations/layer2_attention_weight_min": -36.21048355102539,
      "activations/layer3_attention_weight_max": 103.25196838378906,
      "activations/layer3_attention_weight_min": -106.89164733886719,
      "activations/layer4_attention_weight_max": 112.07991790771484,
      "activations/layer4_attention_weight_min": -111.7405014038086,
      "activations/layer5_attention_weight_max": 52.74436569213867,
      "activations/layer5_attention_weight_min": -62.07148742675781,
      "activations/layer6_attention_weight_max": 46.594852447509766,
      "activations/layer6_attention_weight_min": -48.448280334472656,
      "activations/layer7_attention_weight_max": 95.47197723388672,
      "activations/layer7_attention_weight_min": -102.25968170166016,
      "activations/layer8_attention_weight_max": 43.44359588623047,
      "activations/layer8_attention_weight_min": -48.423946380615234,
      "activations/layer9_attention_weight_max": 32.47694778442383,
      "activations/layer9_attention_weight_min": -34.92261505126953,
      "epoch": 17.37,
      "learning_rate": 3.838522727272727e-05,
      "loss": 2.7341,
      "step": 298900
    },
    {
      "activations/layer0_attention_weight_max": 16.393775939941406,
      "activations/layer0_attention_weight_min": -13.04129409790039,
      "activations/layer10_attention_weight_max": 31.429119110107422,
      "activations/layer10_attention_weight_min": -32.85022735595703,
      "activations/layer11_attention_weight_max": 33.39201736450195,
      "activations/layer11_attention_weight_min": -33.86579895019531,
      "activations/layer12_attention_weight_max": 25.897966384887695,
      "activations/layer12_attention_weight_min": -26.619890213012695,
      "activations/layer13_attention_weight_max": 36.820674896240234,
      "activations/layer13_attention_weight_min": -33.8176155090332,
      "activations/layer14_attention_weight_max": 41.64686584472656,
      "activations/layer14_attention_weight_min": -36.241424560546875,
      "activations/layer15_attention_weight_max": 35.37489700317383,
      "activations/layer15_attention_weight_min": -33.85589599609375,
      "activations/layer16_attention_weight_max": 26.966262817382812,
      "activations/layer16_attention_weight_min": -28.102842330932617,
      "activations/layer17_attention_weight_max": 28.417577743530273,
      "activations/layer17_attention_weight_min": -25.618295669555664,
      "activations/layer18_attention_weight_max": 32.733001708984375,
      "activations/layer18_attention_weight_min": -24.861291885375977,
      "activations/layer19_attention_weight_max": 30.719406127929688,
      "activations/layer19_attention_weight_min": -31.683866500854492,
      "activations/layer1_attention_weight_max": 17.166034698486328,
      "activations/layer1_attention_weight_min": -16.353031158447266,
      "activations/layer20_attention_weight_max": 27.179182052612305,
      "activations/layer20_attention_weight_min": -24.323810577392578,
      "activations/layer21_attention_weight_max": 27.99489974975586,
      "activations/layer21_attention_weight_min": -24.67645835876465,
      "activations/layer22_attention_weight_max": 40.79745864868164,
      "activations/layer22_attention_weight_min": -29.711809158325195,
      "activations/layer23_attention_weight_max": 30.425045013427734,
      "activations/layer23_attention_weight_min": -27.064388275146484,
      "activations/layer2_attention_weight_max": 36.481910705566406,
      "activations/layer2_attention_weight_min": -33.961585998535156,
      "activations/layer3_attention_weight_max": 101.2650146484375,
      "activations/layer3_attention_weight_min": -98.27577209472656,
      "activations/layer4_attention_weight_max": 103.8931884765625,
      "activations/layer4_attention_weight_min": -104.01033020019531,
      "activations/layer5_attention_weight_max": 48.748268127441406,
      "activations/layer5_attention_weight_min": -63.220664978027344,
      "activations/layer6_attention_weight_max": 43.489803314208984,
      "activations/layer6_attention_weight_min": -47.82728958129883,
      "activations/layer7_attention_weight_max": 88.56185150146484,
      "activations/layer7_attention_weight_min": -90.42213439941406,
      "activations/layer8_attention_weight_max": 39.385719299316406,
      "activations/layer8_attention_weight_min": -40.78732681274414,
      "activations/layer9_attention_weight_max": 31.4064884185791,
      "activations/layer9_attention_weight_min": -32.41634750366211,
      "epoch": 17.37,
      "learning_rate": 3.836628787878788e-05,
      "loss": 2.7297,
      "step": 298950
    },
    {
      "activations/layer0_attention_weight_max": 16.643268585205078,
      "activations/layer0_attention_weight_min": -12.613192558288574,
      "activations/layer10_attention_weight_max": 35.137203216552734,
      "activations/layer10_attention_weight_min": -36.371665954589844,
      "activations/layer11_attention_weight_max": 31.82692527770996,
      "activations/layer11_attention_weight_min": -34.61851119995117,
      "activations/layer12_attention_weight_max": 25.539321899414062,
      "activations/layer12_attention_weight_min": -26.384872436523438,
      "activations/layer13_attention_weight_max": 37.51710891723633,
      "activations/layer13_attention_weight_min": -34.92589569091797,
      "activations/layer14_attention_weight_max": 40.100040435791016,
      "activations/layer14_attention_weight_min": -37.518028259277344,
      "activations/layer15_attention_weight_max": 40.59370040893555,
      "activations/layer15_attention_weight_min": -37.23189163208008,
      "activations/layer16_attention_weight_max": 33.00194549560547,
      "activations/layer16_attention_weight_min": -29.06941032409668,
      "activations/layer17_attention_weight_max": 32.72469711303711,
      "activations/layer17_attention_weight_min": -27.06352996826172,
      "activations/layer18_attention_weight_max": 30.325578689575195,
      "activations/layer18_attention_weight_min": -24.250940322875977,
      "activations/layer19_attention_weight_max": 34.29308319091797,
      "activations/layer19_attention_weight_min": -31.40704345703125,
      "activations/layer1_attention_weight_max": 16.422941207885742,
      "activations/layer1_attention_weight_min": -14.570197105407715,
      "activations/layer20_attention_weight_max": 27.779117584228516,
      "activations/layer20_attention_weight_min": -25.878000259399414,
      "activations/layer21_attention_weight_max": 25.615346908569336,
      "activations/layer21_attention_weight_min": -25.115018844604492,
      "activations/layer22_attention_weight_max": 39.57157516479492,
      "activations/layer22_attention_weight_min": -34.09784698486328,
      "activations/layer23_attention_weight_max": 30.898860931396484,
      "activations/layer23_attention_weight_min": -28.02341079711914,
      "activations/layer2_attention_weight_max": 32.931983947753906,
      "activations/layer2_attention_weight_min": -33.51128387451172,
      "activations/layer3_attention_weight_max": 97.28250122070312,
      "activations/layer3_attention_weight_min": -98.56875610351562,
      "activations/layer4_attention_weight_max": 102.67525482177734,
      "activations/layer4_attention_weight_min": -107.7317123413086,
      "activations/layer5_attention_weight_max": 47.511512756347656,
      "activations/layer5_attention_weight_min": -61.950862884521484,
      "activations/layer6_attention_weight_max": 44.71729278564453,
      "activations/layer6_attention_weight_min": -46.392818450927734,
      "activations/layer7_attention_weight_max": 93.30097198486328,
      "activations/layer7_attention_weight_min": -98.04012298583984,
      "activations/layer8_attention_weight_max": 40.25618362426758,
      "activations/layer8_attention_weight_min": -43.803443908691406,
      "activations/layer9_attention_weight_max": 32.95661163330078,
      "activations/layer9_attention_weight_min": -35.86213302612305,
      "epoch": 17.37,
      "learning_rate": 3.834734848484848e-05,
      "loss": 2.7452,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4457,
      "eval_samples_per_second": 508.424,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4457,
      "eval_openwebtext_samples_per_second": 508.424,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9469,
      "eval_wikitext_samples_per_second": 234.224,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_lambada_loss": 2.421875,
      "eval_lambada_ppl": 11.26696508157019,
      "eval_lambada_runtime": 9.5616,
      "eval_lambada_samples_per_second": 509.224,
      "step": 299000
    },
    {
      "activations/layer0_attention_weight_max": 16.049720764160156,
      "activations/layer0_attention_weight_min": -13.888178825378418,
      "activations/layer10_attention_weight_max": 32.223575592041016,
      "activations/layer10_attention_weight_min": -34.304718017578125,
      "activations/layer11_attention_weight_max": 33.169429779052734,
      "activations/layer11_attention_weight_min": -36.150264739990234,
      "activations/layer12_attention_weight_max": 27.43667984008789,
      "activations/layer12_attention_weight_min": -25.930965423583984,
      "activations/layer13_attention_weight_max": 42.289833068847656,
      "activations/layer13_attention_weight_min": -34.50048065185547,
      "activations/layer14_attention_weight_max": 45.41117858886719,
      "activations/layer14_attention_weight_min": -37.502769470214844,
      "activations/layer15_attention_weight_max": 41.71602249145508,
      "activations/layer15_attention_weight_min": -35.85934066772461,
      "activations/layer16_attention_weight_max": 33.56744384765625,
      "activations/layer16_attention_weight_min": -28.265094757080078,
      "activations/layer17_attention_weight_max": 28.87775421142578,
      "activations/layer17_attention_weight_min": -25.92119789123535,
      "activations/layer18_attention_weight_max": 31.262487411499023,
      "activations/layer18_attention_weight_min": -25.7420597076416,
      "activations/layer19_attention_weight_max": 38.28440475463867,
      "activations/layer19_attention_weight_min": -34.2336540222168,
      "activations/layer1_attention_weight_max": 15.706262588500977,
      "activations/layer1_attention_weight_min": -14.383162498474121,
      "activations/layer20_attention_weight_max": 31.76616096496582,
      "activations/layer20_attention_weight_min": -25.480701446533203,
      "activations/layer21_attention_weight_max": 31.221603393554688,
      "activations/layer21_attention_weight_min": -24.384458541870117,
      "activations/layer22_attention_weight_max": 41.06540298461914,
      "activations/layer22_attention_weight_min": -30.41653060913086,
      "activations/layer23_attention_weight_max": 34.39623260498047,
      "activations/layer23_attention_weight_min": -25.55518341064453,
      "activations/layer2_attention_weight_max": 33.521427154541016,
      "activations/layer2_attention_weight_min": -33.758426666259766,
      "activations/layer3_attention_weight_max": 101.09241485595703,
      "activations/layer3_attention_weight_min": -99.76343536376953,
      "activations/layer4_attention_weight_max": 108.4054946899414,
      "activations/layer4_attention_weight_min": -106.82994842529297,
      "activations/layer5_attention_weight_max": 49.30607604980469,
      "activations/layer5_attention_weight_min": -62.661834716796875,
      "activations/layer6_attention_weight_max": 42.87224578857422,
      "activations/layer6_attention_weight_min": -43.09434127807617,
      "activations/layer7_attention_weight_max": 92.30584716796875,
      "activations/layer7_attention_weight_min": -90.25199890136719,
      "activations/layer8_attention_weight_max": 42.626441955566406,
      "activations/layer8_attention_weight_min": -42.00603103637695,
      "activations/layer9_attention_weight_max": 30.44708824157715,
      "activations/layer9_attention_weight_min": -33.727455139160156,
      "epoch": 17.38,
      "learning_rate": 3.832840909090909e-05,
      "loss": 2.7321,
      "step": 299050
    },
    {
      "activations/layer0_attention_weight_max": 15.422006607055664,
      "activations/layer0_attention_weight_min": -12.633514404296875,
      "activations/layer10_attention_weight_max": 38.33218765258789,
      "activations/layer10_attention_weight_min": -36.368743896484375,
      "activations/layer11_attention_weight_max": 39.393798828125,
      "activations/layer11_attention_weight_min": -37.058868408203125,
      "activations/layer12_attention_weight_max": 29.104660034179688,
      "activations/layer12_attention_weight_min": -28.328290939331055,
      "activations/layer13_attention_weight_max": 45.31709671020508,
      "activations/layer13_attention_weight_min": -40.31173324584961,
      "activations/layer14_attention_weight_max": 45.193603515625,
      "activations/layer14_attention_weight_min": -44.22760009765625,
      "activations/layer15_attention_weight_max": 43.1065673828125,
      "activations/layer15_attention_weight_min": -39.92013931274414,
      "activations/layer16_attention_weight_max": 30.852819442749023,
      "activations/layer16_attention_weight_min": -28.89267349243164,
      "activations/layer17_attention_weight_max": 33.610511779785156,
      "activations/layer17_attention_weight_min": -26.792680740356445,
      "activations/layer18_attention_weight_max": 33.53590393066406,
      "activations/layer18_attention_weight_min": -24.758594512939453,
      "activations/layer19_attention_weight_max": 33.83066940307617,
      "activations/layer19_attention_weight_min": -31.733572006225586,
      "activations/layer1_attention_weight_max": 17.250675201416016,
      "activations/layer1_attention_weight_min": -14.735710144042969,
      "activations/layer20_attention_weight_max": 27.75832748413086,
      "activations/layer20_attention_weight_min": -25.00167465209961,
      "activations/layer21_attention_weight_max": 26.961782455444336,
      "activations/layer21_attention_weight_min": -24.808212280273438,
      "activations/layer22_attention_weight_max": 38.84295654296875,
      "activations/layer22_attention_weight_min": -31.283601760864258,
      "activations/layer23_attention_weight_max": 35.28617477416992,
      "activations/layer23_attention_weight_min": -24.876060485839844,
      "activations/layer2_attention_weight_max": 33.469505310058594,
      "activations/layer2_attention_weight_min": -35.33522415161133,
      "activations/layer3_attention_weight_max": 102.89634704589844,
      "activations/layer3_attention_weight_min": -103.7075424194336,
      "activations/layer4_attention_weight_max": 107.1476058959961,
      "activations/layer4_attention_weight_min": -109.86878967285156,
      "activations/layer5_attention_weight_max": 51.997222900390625,
      "activations/layer5_attention_weight_min": -64.88748168945312,
      "activations/layer6_attention_weight_max": 46.6068229675293,
      "activations/layer6_attention_weight_min": -47.135948181152344,
      "activations/layer7_attention_weight_max": 98.37568664550781,
      "activations/layer7_attention_weight_min": -96.03276824951172,
      "activations/layer8_attention_weight_max": 42.2957763671875,
      "activations/layer8_attention_weight_min": -43.291255950927734,
      "activations/layer9_attention_weight_max": 33.90781021118164,
      "activations/layer9_attention_weight_min": -35.064273834228516,
      "epoch": 17.38,
      "learning_rate": 3.8309469696969695e-05,
      "loss": 2.7236,
      "step": 299100
    },
    {
      "activations/layer0_attention_weight_max": 16.097152709960938,
      "activations/layer0_attention_weight_min": -12.800025939941406,
      "activations/layer10_attention_weight_max": 38.842628479003906,
      "activations/layer10_attention_weight_min": -37.08394241333008,
      "activations/layer11_attention_weight_max": 39.945884704589844,
      "activations/layer11_attention_weight_min": -36.650482177734375,
      "activations/layer12_attention_weight_max": 32.679527282714844,
      "activations/layer12_attention_weight_min": -28.51952362060547,
      "activations/layer13_attention_weight_max": 45.303062438964844,
      "activations/layer13_attention_weight_min": -37.898189544677734,
      "activations/layer14_attention_weight_max": 45.787418365478516,
      "activations/layer14_attention_weight_min": -39.226951599121094,
      "activations/layer15_attention_weight_max": 42.70890808105469,
      "activations/layer15_attention_weight_min": -38.01175308227539,
      "activations/layer16_attention_weight_max": 33.97954177856445,
      "activations/layer16_attention_weight_min": -27.594675064086914,
      "activations/layer17_attention_weight_max": 30.94736099243164,
      "activations/layer17_attention_weight_min": -26.200284957885742,
      "activations/layer18_attention_weight_max": 32.180999755859375,
      "activations/layer18_attention_weight_min": -27.62999153137207,
      "activations/layer19_attention_weight_max": 33.993186950683594,
      "activations/layer19_attention_weight_min": -31.782699584960938,
      "activations/layer1_attention_weight_max": 17.706153869628906,
      "activations/layer1_attention_weight_min": -15.581214904785156,
      "activations/layer20_attention_weight_max": 30.247318267822266,
      "activations/layer20_attention_weight_min": -25.816604614257812,
      "activations/layer21_attention_weight_max": 29.485105514526367,
      "activations/layer21_attention_weight_min": -26.73563575744629,
      "activations/layer22_attention_weight_max": 44.714027404785156,
      "activations/layer22_attention_weight_min": -33.239933013916016,
      "activations/layer23_attention_weight_max": 32.6932373046875,
      "activations/layer23_attention_weight_min": -25.9317684173584,
      "activations/layer2_attention_weight_max": 33.837242126464844,
      "activations/layer2_attention_weight_min": -34.446937561035156,
      "activations/layer3_attention_weight_max": 95.9822998046875,
      "activations/layer3_attention_weight_min": -98.77973937988281,
      "activations/layer4_attention_weight_max": 104.96748352050781,
      "activations/layer4_attention_weight_min": -107.454345703125,
      "activations/layer5_attention_weight_max": 53.1365966796875,
      "activations/layer5_attention_weight_min": -63.03407287597656,
      "activations/layer6_attention_weight_max": 46.20685958862305,
      "activations/layer6_attention_weight_min": -47.974830627441406,
      "activations/layer7_attention_weight_max": 99.83203887939453,
      "activations/layer7_attention_weight_min": -104.01577758789062,
      "activations/layer8_attention_weight_max": 46.10945510864258,
      "activations/layer8_attention_weight_min": -47.04642868041992,
      "activations/layer9_attention_weight_max": 35.5645637512207,
      "activations/layer9_attention_weight_min": -37.794795989990234,
      "epoch": 17.38,
      "learning_rate": 3.8290530303030297e-05,
      "loss": 2.7245,
      "step": 299150
    },
    {
      "activations/layer0_attention_weight_max": 16.60340690612793,
      "activations/layer0_attention_weight_min": -13.929022789001465,
      "activations/layer10_attention_weight_max": 34.3057746887207,
      "activations/layer10_attention_weight_min": -34.00330352783203,
      "activations/layer11_attention_weight_max": 33.958160400390625,
      "activations/layer11_attention_weight_min": -33.17926788330078,
      "activations/layer12_attention_weight_max": 26.93404769897461,
      "activations/layer12_attention_weight_min": -26.924951553344727,
      "activations/layer13_attention_weight_max": 42.17066955566406,
      "activations/layer13_attention_weight_min": -35.436927795410156,
      "activations/layer14_attention_weight_max": 44.32307052612305,
      "activations/layer14_attention_weight_min": -38.1434440612793,
      "activations/layer15_attention_weight_max": 42.84239959716797,
      "activations/layer15_attention_weight_min": -34.75924301147461,
      "activations/layer16_attention_weight_max": 30.238985061645508,
      "activations/layer16_attention_weight_min": -28.893924713134766,
      "activations/layer17_attention_weight_max": 32.061405181884766,
      "activations/layer17_attention_weight_min": -25.6644344329834,
      "activations/layer18_attention_weight_max": 32.131004333496094,
      "activations/layer18_attention_weight_min": -26.428390502929688,
      "activations/layer19_attention_weight_max": 34.985050201416016,
      "activations/layer19_attention_weight_min": -32.17511749267578,
      "activations/layer1_attention_weight_max": 15.650550842285156,
      "activations/layer1_attention_weight_min": -15.892097473144531,
      "activations/layer20_attention_weight_max": 31.645118713378906,
      "activations/layer20_attention_weight_min": -25.7884578704834,
      "activations/layer21_attention_weight_max": 31.775075912475586,
      "activations/layer21_attention_weight_min": -24.422765731811523,
      "activations/layer22_attention_weight_max": 42.62942886352539,
      "activations/layer22_attention_weight_min": -31.24336051940918,
      "activations/layer23_attention_weight_max": 35.180389404296875,
      "activations/layer23_attention_weight_min": -24.764930725097656,
      "activations/layer2_attention_weight_max": 34.217689514160156,
      "activations/layer2_attention_weight_min": -33.55063247680664,
      "activations/layer3_attention_weight_max": 103.67008972167969,
      "activations/layer3_attention_weight_min": -102.85603332519531,
      "activations/layer4_attention_weight_max": 107.45094299316406,
      "activations/layer4_attention_weight_min": -108.57185363769531,
      "activations/layer5_attention_weight_max": 51.577369689941406,
      "activations/layer5_attention_weight_min": -60.770198822021484,
      "activations/layer6_attention_weight_max": 44.658592224121094,
      "activations/layer6_attention_weight_min": -46.4326057434082,
      "activations/layer7_attention_weight_max": 93.0798110961914,
      "activations/layer7_attention_weight_min": -95.06575775146484,
      "activations/layer8_attention_weight_max": 41.786319732666016,
      "activations/layer8_attention_weight_min": -43.22111129760742,
      "activations/layer9_attention_weight_max": 35.31706237792969,
      "activations/layer9_attention_weight_min": -34.58228302001953,
      "epoch": 17.39,
      "learning_rate": 3.827196969696969e-05,
      "loss": 2.7246,
      "step": 299200
    },
    {
      "activations/layer0_attention_weight_max": 16.548324584960938,
      "activations/layer0_attention_weight_min": -12.436162948608398,
      "activations/layer10_attention_weight_max": 41.147003173828125,
      "activations/layer10_attention_weight_min": -35.177738189697266,
      "activations/layer11_attention_weight_max": 39.530067443847656,
      "activations/layer11_attention_weight_min": -35.779720306396484,
      "activations/layer12_attention_weight_max": 27.77834701538086,
      "activations/layer12_attention_weight_min": -26.70343780517578,
      "activations/layer13_attention_weight_max": 41.13393020629883,
      "activations/layer13_attention_weight_min": -37.57653045654297,
      "activations/layer14_attention_weight_max": 46.013694763183594,
      "activations/layer14_attention_weight_min": -41.01014709472656,
      "activations/layer15_attention_weight_max": 43.89427947998047,
      "activations/layer15_attention_weight_min": -38.81851577758789,
      "activations/layer16_attention_weight_max": 31.33878517150879,
      "activations/layer16_attention_weight_min": -29.009700775146484,
      "activations/layer17_attention_weight_max": 30.944814682006836,
      "activations/layer17_attention_weight_min": -27.831056594848633,
      "activations/layer18_attention_weight_max": 32.830074310302734,
      "activations/layer18_attention_weight_min": -26.165620803833008,
      "activations/layer19_attention_weight_max": 37.73191452026367,
      "activations/layer19_attention_weight_min": -31.766462326049805,
      "activations/layer1_attention_weight_max": 15.982416152954102,
      "activations/layer1_attention_weight_min": -14.57944107055664,
      "activations/layer20_attention_weight_max": 29.32888412475586,
      "activations/layer20_attention_weight_min": -24.733165740966797,
      "activations/layer21_attention_weight_max": 28.712921142578125,
      "activations/layer21_attention_weight_min": -25.270977020263672,
      "activations/layer22_attention_weight_max": 42.36900329589844,
      "activations/layer22_attention_weight_min": -32.11393737792969,
      "activations/layer23_attention_weight_max": 34.498374938964844,
      "activations/layer23_attention_weight_min": -26.103988647460938,
      "activations/layer2_attention_weight_max": 32.87687683105469,
      "activations/layer2_attention_weight_min": -33.15940856933594,
      "activations/layer3_attention_weight_max": 99.20340728759766,
      "activations/layer3_attention_weight_min": -101.8537826538086,
      "activations/layer4_attention_weight_max": 106.52850341796875,
      "activations/layer4_attention_weight_min": -111.9939193725586,
      "activations/layer5_attention_weight_max": 51.2818603515625,
      "activations/layer5_attention_weight_min": -65.04454040527344,
      "activations/layer6_attention_weight_max": 44.05626678466797,
      "activations/layer6_attention_weight_min": -48.227787017822266,
      "activations/layer7_attention_weight_max": 91.28170013427734,
      "activations/layer7_attention_weight_min": -95.81044006347656,
      "activations/layer8_attention_weight_max": 43.70214080810547,
      "activations/layer8_attention_weight_min": -44.13001251220703,
      "activations/layer9_attention_weight_max": 36.841087341308594,
      "activations/layer9_attention_weight_min": -34.83257293701172,
      "epoch": 17.39,
      "learning_rate": 3.82530303030303e-05,
      "loss": 2.7494,
      "step": 299250
    },
    {
      "activations/layer0_attention_weight_max": 16.824403762817383,
      "activations/layer0_attention_weight_min": -14.389065742492676,
      "activations/layer10_attention_weight_max": 43.983436584472656,
      "activations/layer10_attention_weight_min": -39.8246955871582,
      "activations/layer11_attention_weight_max": 43.97877502441406,
      "activations/layer11_attention_weight_min": -40.97443389892578,
      "activations/layer12_attention_weight_max": 27.4239444732666,
      "activations/layer12_attention_weight_min": -26.009323120117188,
      "activations/layer13_attention_weight_max": 39.46995162963867,
      "activations/layer13_attention_weight_min": -34.50570297241211,
      "activations/layer14_attention_weight_max": 44.53453826904297,
      "activations/layer14_attention_weight_min": -37.45732498168945,
      "activations/layer15_attention_weight_max": 45.30351257324219,
      "activations/layer15_attention_weight_min": -36.09419631958008,
      "activations/layer16_attention_weight_max": 33.644588470458984,
      "activations/layer16_attention_weight_min": -29.98529815673828,
      "activations/layer17_attention_weight_max": 29.78797721862793,
      "activations/layer17_attention_weight_min": -25.106082916259766,
      "activations/layer18_attention_weight_max": 30.42803382873535,
      "activations/layer18_attention_weight_min": -24.91748809814453,
      "activations/layer19_attention_weight_max": 35.251373291015625,
      "activations/layer19_attention_weight_min": -32.99139404296875,
      "activations/layer1_attention_weight_max": 17.32139778137207,
      "activations/layer1_attention_weight_min": -17.964433670043945,
      "activations/layer20_attention_weight_max": 28.499143600463867,
      "activations/layer20_attention_weight_min": -25.163129806518555,
      "activations/layer21_attention_weight_max": 26.1235294342041,
      "activations/layer21_attention_weight_min": -24.59810447692871,
      "activations/layer22_attention_weight_max": 40.35852813720703,
      "activations/layer22_attention_weight_min": -31.23179054260254,
      "activations/layer23_attention_weight_max": 32.693050384521484,
      "activations/layer23_attention_weight_min": -25.588613510131836,
      "activations/layer2_attention_weight_max": 36.124755859375,
      "activations/layer2_attention_weight_min": -35.15880584716797,
      "activations/layer3_attention_weight_max": 106.25918579101562,
      "activations/layer3_attention_weight_min": -101.64911651611328,
      "activations/layer4_attention_weight_max": 119.09230041503906,
      "activations/layer4_attention_weight_min": -110.60387420654297,
      "activations/layer5_attention_weight_max": 58.2303581237793,
      "activations/layer5_attention_weight_min": -63.22413635253906,
      "activations/layer6_attention_weight_max": 48.951683044433594,
      "activations/layer6_attention_weight_min": -46.19194412231445,
      "activations/layer7_attention_weight_max": 111.00932312011719,
      "activations/layer7_attention_weight_min": -102.5561294555664,
      "activations/layer8_attention_weight_max": 52.48017120361328,
      "activations/layer8_attention_weight_min": -49.84650421142578,
      "activations/layer9_attention_weight_max": 39.09398651123047,
      "activations/layer9_attention_weight_min": -38.19246292114258,
      "epoch": 17.39,
      "learning_rate": 3.8234090909090904e-05,
      "loss": 2.7295,
      "step": 299300
    },
    {
      "activations/layer0_attention_weight_max": 16.17687225341797,
      "activations/layer0_attention_weight_min": -13.260293960571289,
      "activations/layer10_attention_weight_max": 34.23755645751953,
      "activations/layer10_attention_weight_min": -33.764896392822266,
      "activations/layer11_attention_weight_max": 36.420326232910156,
      "activations/layer11_attention_weight_min": -33.951419830322266,
      "activations/layer12_attention_weight_max": 27.35118865966797,
      "activations/layer12_attention_weight_min": -26.99215316772461,
      "activations/layer13_attention_weight_max": 41.55048751831055,
      "activations/layer13_attention_weight_min": -33.82699203491211,
      "activations/layer14_attention_weight_max": 46.72791290283203,
      "activations/layer14_attention_weight_min": -38.14356231689453,
      "activations/layer15_attention_weight_max": 39.18989562988281,
      "activations/layer15_attention_weight_min": -33.49666213989258,
      "activations/layer16_attention_weight_max": 31.871305465698242,
      "activations/layer16_attention_weight_min": -28.430950164794922,
      "activations/layer17_attention_weight_max": 29.285823822021484,
      "activations/layer17_attention_weight_min": -26.280088424682617,
      "activations/layer18_attention_weight_max": 31.32368278503418,
      "activations/layer18_attention_weight_min": -24.591426849365234,
      "activations/layer19_attention_weight_max": 30.373929977416992,
      "activations/layer19_attention_weight_min": -29.936969757080078,
      "activations/layer1_attention_weight_max": 15.742866516113281,
      "activations/layer1_attention_weight_min": -16.19661521911621,
      "activations/layer20_attention_weight_max": 28.920814514160156,
      "activations/layer20_attention_weight_min": -24.08272361755371,
      "activations/layer21_attention_weight_max": 25.093791961669922,
      "activations/layer21_attention_weight_min": -23.440038681030273,
      "activations/layer22_attention_weight_max": 40.57771301269531,
      "activations/layer22_attention_weight_min": -29.004894256591797,
      "activations/layer23_attention_weight_max": 30.4814453125,
      "activations/layer23_attention_weight_min": -24.92171287536621,
      "activations/layer2_attention_weight_max": 33.18426513671875,
      "activations/layer2_attention_weight_min": -32.347434997558594,
      "activations/layer3_attention_weight_max": 98.50811767578125,
      "activations/layer3_attention_weight_min": -97.98470306396484,
      "activations/layer4_attention_weight_max": 104.42958068847656,
      "activations/layer4_attention_weight_min": -107.08251190185547,
      "activations/layer5_attention_weight_max": 48.94648361206055,
      "activations/layer5_attention_weight_min": -62.425960540771484,
      "activations/layer6_attention_weight_max": 42.189247131347656,
      "activations/layer6_attention_weight_min": -44.52629470825195,
      "activations/layer7_attention_weight_max": 92.05335235595703,
      "activations/layer7_attention_weight_min": -95.09709930419922,
      "activations/layer8_attention_weight_max": 40.61610412597656,
      "activations/layer8_attention_weight_min": -41.334693908691406,
      "activations/layer9_attention_weight_max": 31.98219871520996,
      "activations/layer9_attention_weight_min": -32.91653060913086,
      "epoch": 17.39,
      "learning_rate": 3.821515151515151e-05,
      "loss": 2.7383,
      "step": 299350
    },
    {
      "activations/layer0_attention_weight_max": 16.682968139648438,
      "activations/layer0_attention_weight_min": -13.085352897644043,
      "activations/layer10_attention_weight_max": 34.55992126464844,
      "activations/layer10_attention_weight_min": -33.18367385864258,
      "activations/layer11_attention_weight_max": 32.45479202270508,
      "activations/layer11_attention_weight_min": -32.2526741027832,
      "activations/layer12_attention_weight_max": 26.056257247924805,
      "activations/layer12_attention_weight_min": -26.687332153320312,
      "activations/layer13_attention_weight_max": 38.4910888671875,
      "activations/layer13_attention_weight_min": -35.544639587402344,
      "activations/layer14_attention_weight_max": 40.5661506652832,
      "activations/layer14_attention_weight_min": -36.738006591796875,
      "activations/layer15_attention_weight_max": 37.68865203857422,
      "activations/layer15_attention_weight_min": -33.64118194580078,
      "activations/layer16_attention_weight_max": 29.034902572631836,
      "activations/layer16_attention_weight_min": -27.45362091064453,
      "activations/layer17_attention_weight_max": 29.059125900268555,
      "activations/layer17_attention_weight_min": -24.04346466064453,
      "activations/layer18_attention_weight_max": 30.683170318603516,
      "activations/layer18_attention_weight_min": -22.243942260742188,
      "activations/layer19_attention_weight_max": 31.9008731842041,
      "activations/layer19_attention_weight_min": -29.163618087768555,
      "activations/layer1_attention_weight_max": 16.140653610229492,
      "activations/layer1_attention_weight_min": -14.292378425598145,
      "activations/layer20_attention_weight_max": 27.153215408325195,
      "activations/layer20_attention_weight_min": -23.82029914855957,
      "activations/layer21_attention_weight_max": 27.68095588684082,
      "activations/layer21_attention_weight_min": -23.835586547851562,
      "activations/layer22_attention_weight_max": 39.02069854736328,
      "activations/layer22_attention_weight_min": -30.38920021057129,
      "activations/layer23_attention_weight_max": 32.71564865112305,
      "activations/layer23_attention_weight_min": -24.90198516845703,
      "activations/layer2_attention_weight_max": 32.98253631591797,
      "activations/layer2_attention_weight_min": -32.28973388671875,
      "activations/layer3_attention_weight_max": 98.34744262695312,
      "activations/layer3_attention_weight_min": -98.98055267333984,
      "activations/layer4_attention_weight_max": 105.7838363647461,
      "activations/layer4_attention_weight_min": -108.80549621582031,
      "activations/layer5_attention_weight_max": 50.37303161621094,
      "activations/layer5_attention_weight_min": -61.67916488647461,
      "activations/layer6_attention_weight_max": 45.83346939086914,
      "activations/layer6_attention_weight_min": -50.396263122558594,
      "activations/layer7_attention_weight_max": 93.19792938232422,
      "activations/layer7_attention_weight_min": -99.4376220703125,
      "activations/layer8_attention_weight_max": 44.51799392700195,
      "activations/layer8_attention_weight_min": -45.039527893066406,
      "activations/layer9_attention_weight_max": 33.17688751220703,
      "activations/layer9_attention_weight_min": -33.304866790771484,
      "epoch": 17.4,
      "learning_rate": 3.819621212121212e-05,
      "loss": 2.726,
      "step": 299400
    },
    {
      "activations/layer0_attention_weight_max": 17.25503158569336,
      "activations/layer0_attention_weight_min": -12.87501335144043,
      "activations/layer10_attention_weight_max": 29.421356201171875,
      "activations/layer10_attention_weight_min": -31.940086364746094,
      "activations/layer11_attention_weight_max": 29.415042877197266,
      "activations/layer11_attention_weight_min": -32.817649841308594,
      "activations/layer12_attention_weight_max": 23.510190963745117,
      "activations/layer12_attention_weight_min": -25.66421127319336,
      "activations/layer13_attention_weight_max": 38.03545379638672,
      "activations/layer13_attention_weight_min": -33.70846176147461,
      "activations/layer14_attention_weight_max": 37.59689712524414,
      "activations/layer14_attention_weight_min": -35.333065032958984,
      "activations/layer15_attention_weight_max": 36.627105712890625,
      "activations/layer15_attention_weight_min": -31.746274948120117,
      "activations/layer16_attention_weight_max": 26.38109016418457,
      "activations/layer16_attention_weight_min": -27.305583953857422,
      "activations/layer17_attention_weight_max": 27.829660415649414,
      "activations/layer17_attention_weight_min": -26.466197967529297,
      "activations/layer18_attention_weight_max": 30.180461883544922,
      "activations/layer18_attention_weight_min": -24.0333194732666,
      "activations/layer19_attention_weight_max": 29.87042236328125,
      "activations/layer19_attention_weight_min": -30.514596939086914,
      "activations/layer1_attention_weight_max": 16.122360229492188,
      "activations/layer1_attention_weight_min": -14.46640396118164,
      "activations/layer20_attention_weight_max": 26.724836349487305,
      "activations/layer20_attention_weight_min": -24.553770065307617,
      "activations/layer21_attention_weight_max": 26.975051879882812,
      "activations/layer21_attention_weight_min": -24.54817771911621,
      "activations/layer22_attention_weight_max": 39.81023025512695,
      "activations/layer22_attention_weight_min": -30.684906005859375,
      "activations/layer23_attention_weight_max": 29.971569061279297,
      "activations/layer23_attention_weight_min": -27.052709579467773,
      "activations/layer2_attention_weight_max": 32.692420959472656,
      "activations/layer2_attention_weight_min": -32.537506103515625,
      "activations/layer3_attention_weight_max": 91.90070343017578,
      "activations/layer3_attention_weight_min": -96.83757781982422,
      "activations/layer4_attention_weight_max": 96.67882537841797,
      "activations/layer4_attention_weight_min": -100.05711364746094,
      "activations/layer5_attention_weight_max": 48.40486145019531,
      "activations/layer5_attention_weight_min": -61.21971130371094,
      "activations/layer6_attention_weight_max": 38.56550598144531,
      "activations/layer6_attention_weight_min": -41.217010498046875,
      "activations/layer7_attention_weight_max": 79.54818725585938,
      "activations/layer7_attention_weight_min": -89.37846374511719,
      "activations/layer8_attention_weight_max": 35.35627746582031,
      "activations/layer8_attention_weight_min": -38.525733947753906,
      "activations/layer9_attention_weight_max": 27.759183883666992,
      "activations/layer9_attention_weight_min": -30.51338005065918,
      "epoch": 17.4,
      "learning_rate": 3.817727272727273e-05,
      "loss": 2.7246,
      "step": 299450
    },
    {
      "activations/layer0_attention_weight_max": 16.09959602355957,
      "activations/layer0_attention_weight_min": -12.221673965454102,
      "activations/layer10_attention_weight_max": 34.27610778808594,
      "activations/layer10_attention_weight_min": -33.07365798950195,
      "activations/layer11_attention_weight_max": 35.168251037597656,
      "activations/layer11_attention_weight_min": -33.89435958862305,
      "activations/layer12_attention_weight_max": 31.113880157470703,
      "activations/layer12_attention_weight_min": -26.699966430664062,
      "activations/layer13_attention_weight_max": 41.08888244628906,
      "activations/layer13_attention_weight_min": -36.68157958984375,
      "activations/layer14_attention_weight_max": 52.72779083251953,
      "activations/layer14_attention_weight_min": -40.47187805175781,
      "activations/layer15_attention_weight_max": 43.67485046386719,
      "activations/layer15_attention_weight_min": -35.88318634033203,
      "activations/layer16_attention_weight_max": 31.16005516052246,
      "activations/layer16_attention_weight_min": -26.877763748168945,
      "activations/layer17_attention_weight_max": 30.144893646240234,
      "activations/layer17_attention_weight_min": -25.54325294494629,
      "activations/layer18_attention_weight_max": 33.429725646972656,
      "activations/layer18_attention_weight_min": -25.779752731323242,
      "activations/layer19_attention_weight_max": 39.25613021850586,
      "activations/layer19_attention_weight_min": -30.31412124633789,
      "activations/layer1_attention_weight_max": 16.280319213867188,
      "activations/layer1_attention_weight_min": -15.029976844787598,
      "activations/layer20_attention_weight_max": 33.158958435058594,
      "activations/layer20_attention_weight_min": -25.323204040527344,
      "activations/layer21_attention_weight_max": 31.82107925415039,
      "activations/layer21_attention_weight_min": -23.884380340576172,
      "activations/layer22_attention_weight_max": 54.115474700927734,
      "activations/layer22_attention_weight_min": -31.99189567565918,
      "activations/layer23_attention_weight_max": 41.547760009765625,
      "activations/layer23_attention_weight_min": -25.21521759033203,
      "activations/layer2_attention_weight_max": 35.603431701660156,
      "activations/layer2_attention_weight_min": -33.762168884277344,
      "activations/layer3_attention_weight_max": 96.77613830566406,
      "activations/layer3_attention_weight_min": -97.84535217285156,
      "activations/layer4_attention_weight_max": 103.65997314453125,
      "activations/layer4_attention_weight_min": -103.79803466796875,
      "activations/layer5_attention_weight_max": 52.65446090698242,
      "activations/layer5_attention_weight_min": -58.900718688964844,
      "activations/layer6_attention_weight_max": 43.07041549682617,
      "activations/layer6_attention_weight_min": -45.000892639160156,
      "activations/layer7_attention_weight_max": 87.94934844970703,
      "activations/layer7_attention_weight_min": -94.1400146484375,
      "activations/layer8_attention_weight_max": 42.07604217529297,
      "activations/layer8_attention_weight_min": -42.756404876708984,
      "activations/layer9_attention_weight_max": 31.96259117126465,
      "activations/layer9_attention_weight_min": -34.35477066040039,
      "epoch": 17.4,
      "learning_rate": 3.815833333333333e-05,
      "loss": 2.7372,
      "step": 299500
    },
    {
      "activations/layer0_attention_weight_max": 16.11003303527832,
      "activations/layer0_attention_weight_min": -12.83871841430664,
      "activations/layer10_attention_weight_max": 33.774593353271484,
      "activations/layer10_attention_weight_min": -33.72956466674805,
      "activations/layer11_attention_weight_max": 35.94181442260742,
      "activations/layer11_attention_weight_min": -33.493263244628906,
      "activations/layer12_attention_weight_max": 25.192626953125,
      "activations/layer12_attention_weight_min": -25.35586929321289,
      "activations/layer13_attention_weight_max": 37.10760498046875,
      "activations/layer13_attention_weight_min": -35.617401123046875,
      "activations/layer14_attention_weight_max": 39.562068939208984,
      "activations/layer14_attention_weight_min": -37.471473693847656,
      "activations/layer15_attention_weight_max": 37.63982009887695,
      "activations/layer15_attention_weight_min": -37.44563293457031,
      "activations/layer16_attention_weight_max": 29.159873962402344,
      "activations/layer16_attention_weight_min": -27.29129981994629,
      "activations/layer17_attention_weight_max": 33.1890754699707,
      "activations/layer17_attention_weight_min": -25.635387420654297,
      "activations/layer18_attention_weight_max": 31.76300621032715,
      "activations/layer18_attention_weight_min": -25.059860229492188,
      "activations/layer19_attention_weight_max": 31.984542846679688,
      "activations/layer19_attention_weight_min": -30.92429542541504,
      "activations/layer1_attention_weight_max": 15.536154747009277,
      "activations/layer1_attention_weight_min": -15.440587997436523,
      "activations/layer20_attention_weight_max": 28.491863250732422,
      "activations/layer20_attention_weight_min": -25.630069732666016,
      "activations/layer21_attention_weight_max": 30.39337730407715,
      "activations/layer21_attention_weight_min": -24.33407974243164,
      "activations/layer22_attention_weight_max": 40.184791564941406,
      "activations/layer22_attention_weight_min": -33.281944274902344,
      "activations/layer23_attention_weight_max": 31.34512710571289,
      "activations/layer23_attention_weight_min": -25.983694076538086,
      "activations/layer2_attention_weight_max": 32.961856842041016,
      "activations/layer2_attention_weight_min": -30.516523361206055,
      "activations/layer3_attention_weight_max": 98.63092041015625,
      "activations/layer3_attention_weight_min": -98.93958282470703,
      "activations/layer4_attention_weight_max": 107.12629699707031,
      "activations/layer4_attention_weight_min": -105.53128814697266,
      "activations/layer5_attention_weight_max": 51.567649841308594,
      "activations/layer5_attention_weight_min": -59.3111572265625,
      "activations/layer6_attention_weight_max": 46.23972702026367,
      "activations/layer6_attention_weight_min": -46.12342071533203,
      "activations/layer7_attention_weight_max": 92.16714477539062,
      "activations/layer7_attention_weight_min": -89.79924011230469,
      "activations/layer8_attention_weight_max": 40.76437759399414,
      "activations/layer8_attention_weight_min": -41.16389083862305,
      "activations/layer9_attention_weight_max": 30.91086769104004,
      "activations/layer9_attention_weight_min": -32.57524871826172,
      "epoch": 17.41,
      "learning_rate": 3.813939393939393e-05,
      "loss": 2.747,
      "step": 299550
    },
    {
      "activations/layer0_attention_weight_max": 15.878552436828613,
      "activations/layer0_attention_weight_min": -13.545351028442383,
      "activations/layer10_attention_weight_max": 32.65069580078125,
      "activations/layer10_attention_weight_min": -35.32136917114258,
      "activations/layer11_attention_weight_max": 33.78050231933594,
      "activations/layer11_attention_weight_min": -35.58006286621094,
      "activations/layer12_attention_weight_max": 24.566123962402344,
      "activations/layer12_attention_weight_min": -26.845420837402344,
      "activations/layer13_attention_weight_max": 36.960052490234375,
      "activations/layer13_attention_weight_min": -35.07394790649414,
      "activations/layer14_attention_weight_max": 44.56180191040039,
      "activations/layer14_attention_weight_min": -38.358055114746094,
      "activations/layer15_attention_weight_max": 36.754676818847656,
      "activations/layer15_attention_weight_min": -36.18726348876953,
      "activations/layer16_attention_weight_max": 28.587242126464844,
      "activations/layer16_attention_weight_min": -26.907054901123047,
      "activations/layer17_attention_weight_max": 26.778249740600586,
      "activations/layer17_attention_weight_min": -25.38334083557129,
      "activations/layer18_attention_weight_max": 30.187414169311523,
      "activations/layer18_attention_weight_min": -25.163970947265625,
      "activations/layer19_attention_weight_max": 30.88020133972168,
      "activations/layer19_attention_weight_min": -30.85322380065918,
      "activations/layer1_attention_weight_max": 16.787086486816406,
      "activations/layer1_attention_weight_min": -16.108686447143555,
      "activations/layer20_attention_weight_max": 26.914226531982422,
      "activations/layer20_attention_weight_min": -26.647254943847656,
      "activations/layer21_attention_weight_max": 27.754222869873047,
      "activations/layer21_attention_weight_min": -23.901174545288086,
      "activations/layer22_attention_weight_max": 38.77468490600586,
      "activations/layer22_attention_weight_min": -29.596799850463867,
      "activations/layer23_attention_weight_max": 29.961658477783203,
      "activations/layer23_attention_weight_min": -24.626728057861328,
      "activations/layer2_attention_weight_max": 32.609622955322266,
      "activations/layer2_attention_weight_min": -33.641822814941406,
      "activations/layer3_attention_weight_max": 104.95081329345703,
      "activations/layer3_attention_weight_min": -107.71251678466797,
      "activations/layer4_attention_weight_max": 105.1163330078125,
      "activations/layer4_attention_weight_min": -111.22454071044922,
      "activations/layer5_attention_weight_max": 49.09507369995117,
      "activations/layer5_attention_weight_min": -60.826446533203125,
      "activations/layer6_attention_weight_max": 44.56143569946289,
      "activations/layer6_attention_weight_min": -47.34436798095703,
      "activations/layer7_attention_weight_max": 88.61054229736328,
      "activations/layer7_attention_weight_min": -92.5847396850586,
      "activations/layer8_attention_weight_max": 40.20491409301758,
      "activations/layer8_attention_weight_min": -42.480079650878906,
      "activations/layer9_attention_weight_max": 31.77359390258789,
      "activations/layer9_attention_weight_min": -33.72644805908203,
      "epoch": 17.41,
      "learning_rate": 3.812045454545455e-05,
      "loss": 2.7343,
      "step": 299600
    },
    {
      "activations/layer0_attention_weight_max": 16.44671630859375,
      "activations/layer0_attention_weight_min": -13.41120719909668,
      "activations/layer10_attention_weight_max": 33.0328369140625,
      "activations/layer10_attention_weight_min": -34.95097732543945,
      "activations/layer11_attention_weight_max": 32.27802658081055,
      "activations/layer11_attention_weight_min": -34.28370666503906,
      "activations/layer12_attention_weight_max": 25.673439025878906,
      "activations/layer12_attention_weight_min": -28.205018997192383,
      "activations/layer13_attention_weight_max": 39.029293060302734,
      "activations/layer13_attention_weight_min": -41.22849655151367,
      "activations/layer14_attention_weight_max": 42.51576614379883,
      "activations/layer14_attention_weight_min": -41.96345901489258,
      "activations/layer15_attention_weight_max": 37.40003204345703,
      "activations/layer15_attention_weight_min": -36.9306755065918,
      "activations/layer16_attention_weight_max": 27.913352966308594,
      "activations/layer16_attention_weight_min": -27.54466438293457,
      "activations/layer17_attention_weight_max": 27.398014068603516,
      "activations/layer17_attention_weight_min": -26.31438446044922,
      "activations/layer18_attention_weight_max": 29.663490295410156,
      "activations/layer18_attention_weight_min": -23.26312828063965,
      "activations/layer19_attention_weight_max": 39.70097732543945,
      "activations/layer19_attention_weight_min": -30.114511489868164,
      "activations/layer1_attention_weight_max": 16.206384658813477,
      "activations/layer1_attention_weight_min": -15.23941421508789,
      "activations/layer20_attention_weight_max": 31.356809616088867,
      "activations/layer20_attention_weight_min": -23.347932815551758,
      "activations/layer21_attention_weight_max": 27.53607177734375,
      "activations/layer21_attention_weight_min": -23.14227294921875,
      "activations/layer22_attention_weight_max": 39.15727996826172,
      "activations/layer22_attention_weight_min": -29.7169189453125,
      "activations/layer23_attention_weight_max": 30.695249557495117,
      "activations/layer23_attention_weight_min": -25.090909957885742,
      "activations/layer2_attention_weight_max": 33.927734375,
      "activations/layer2_attention_weight_min": -32.69520568847656,
      "activations/layer3_attention_weight_max": 104.60884094238281,
      "activations/layer3_attention_weight_min": -104.73960876464844,
      "activations/layer4_attention_weight_max": 116.78211975097656,
      "activations/layer4_attention_weight_min": -113.13201904296875,
      "activations/layer5_attention_weight_max": 51.53536605834961,
      "activations/layer5_attention_weight_min": -64.447265625,
      "activations/layer6_attention_weight_max": 47.1170539855957,
      "activations/layer6_attention_weight_min": -46.09025192260742,
      "activations/layer7_attention_weight_max": 91.0441665649414,
      "activations/layer7_attention_weight_min": -92.68174743652344,
      "activations/layer8_attention_weight_max": 40.825767517089844,
      "activations/layer8_attention_weight_min": -42.356075286865234,
      "activations/layer9_attention_weight_max": 30.345426559448242,
      "activations/layer9_attention_weight_min": -34.198028564453125,
      "epoch": 17.41,
      "learning_rate": 3.810151515151515e-05,
      "loss": 2.7361,
      "step": 299650
    },
    {
      "activations/layer0_attention_weight_max": 15.358755111694336,
      "activations/layer0_attention_weight_min": -12.766024589538574,
      "activations/layer10_attention_weight_max": 32.1561164855957,
      "activations/layer10_attention_weight_min": -33.65699005126953,
      "activations/layer11_attention_weight_max": 32.257896423339844,
      "activations/layer11_attention_weight_min": -32.866607666015625,
      "activations/layer12_attention_weight_max": 27.115074157714844,
      "activations/layer12_attention_weight_min": -27.88723373413086,
      "activations/layer13_attention_weight_max": 39.87427520751953,
      "activations/layer13_attention_weight_min": -36.461029052734375,
      "activations/layer14_attention_weight_max": 45.41917419433594,
      "activations/layer14_attention_weight_min": -38.109737396240234,
      "activations/layer15_attention_weight_max": 38.10783004760742,
      "activations/layer15_attention_weight_min": -38.9714241027832,
      "activations/layer16_attention_weight_max": 31.07921600341797,
      "activations/layer16_attention_weight_min": -28.5479679107666,
      "activations/layer17_attention_weight_max": 28.54903221130371,
      "activations/layer17_attention_weight_min": -25.948570251464844,
      "activations/layer18_attention_weight_max": 30.032705307006836,
      "activations/layer18_attention_weight_min": -26.623422622680664,
      "activations/layer19_attention_weight_max": 29.12116050720215,
      "activations/layer19_attention_weight_min": -30.8927001953125,
      "activations/layer1_attention_weight_max": 15.65770435333252,
      "activations/layer1_attention_weight_min": -15.448224067687988,
      "activations/layer20_attention_weight_max": 28.800790786743164,
      "activations/layer20_attention_weight_min": -25.32210922241211,
      "activations/layer21_attention_weight_max": 28.470937728881836,
      "activations/layer21_attention_weight_min": -24.48146629333496,
      "activations/layer22_attention_weight_max": 39.19916534423828,
      "activations/layer22_attention_weight_min": -31.653841018676758,
      "activations/layer23_attention_weight_max": 30.615108489990234,
      "activations/layer23_attention_weight_min": -26.671401977539062,
      "activations/layer2_attention_weight_max": 33.167850494384766,
      "activations/layer2_attention_weight_min": -33.6292724609375,
      "activations/layer3_attention_weight_max": 99.64396667480469,
      "activations/layer3_attention_weight_min": -100.75139617919922,
      "activations/layer4_attention_weight_max": 105.80818176269531,
      "activations/layer4_attention_weight_min": -106.18395233154297,
      "activations/layer5_attention_weight_max": 49.95058822631836,
      "activations/layer5_attention_weight_min": -61.10035705566406,
      "activations/layer6_attention_weight_max": 43.848880767822266,
      "activations/layer6_attention_weight_min": -48.322471618652344,
      "activations/layer7_attention_weight_max": 85.82435607910156,
      "activations/layer7_attention_weight_min": -94.70030212402344,
      "activations/layer8_attention_weight_max": 38.800777435302734,
      "activations/layer8_attention_weight_min": -44.251739501953125,
      "activations/layer9_attention_weight_max": 31.590377807617188,
      "activations/layer9_attention_weight_min": -34.328712463378906,
      "epoch": 17.41,
      "learning_rate": 3.8082575757575757e-05,
      "loss": 2.7357,
      "step": 299700
    },
    {
      "activations/layer0_attention_weight_max": 16.575395584106445,
      "activations/layer0_attention_weight_min": -14.019265174865723,
      "activations/layer10_attention_weight_max": 32.44221496582031,
      "activations/layer10_attention_weight_min": -33.64180374145508,
      "activations/layer11_attention_weight_max": 32.428016662597656,
      "activations/layer11_attention_weight_min": -33.323856353759766,
      "activations/layer12_attention_weight_max": 39.258819580078125,
      "activations/layer12_attention_weight_min": -26.395694732666016,
      "activations/layer13_attention_weight_max": 45.39173126220703,
      "activations/layer13_attention_weight_min": -35.286685943603516,
      "activations/layer14_attention_weight_max": 41.176849365234375,
      "activations/layer14_attention_weight_min": -37.98628234863281,
      "activations/layer15_attention_weight_max": 40.695472717285156,
      "activations/layer15_attention_weight_min": -36.13839340209961,
      "activations/layer16_attention_weight_max": 31.81913185119629,
      "activations/layer16_attention_weight_min": -27.636747360229492,
      "activations/layer17_attention_weight_max": 28.913421630859375,
      "activations/layer17_attention_weight_min": -26.274953842163086,
      "activations/layer18_attention_weight_max": 28.26799964904785,
      "activations/layer18_attention_weight_min": -24.9031982421875,
      "activations/layer19_attention_weight_max": 31.77503776550293,
      "activations/layer19_attention_weight_min": -30.245853424072266,
      "activations/layer1_attention_weight_max": 16.392444610595703,
      "activations/layer1_attention_weight_min": -14.557462692260742,
      "activations/layer20_attention_weight_max": 26.798192977905273,
      "activations/layer20_attention_weight_min": -25.289810180664062,
      "activations/layer21_attention_weight_max": 27.28817367553711,
      "activations/layer21_attention_weight_min": -25.223100662231445,
      "activations/layer22_attention_weight_max": 35.08242416381836,
      "activations/layer22_attention_weight_min": -30.531511306762695,
      "activations/layer23_attention_weight_max": 29.255451202392578,
      "activations/layer23_attention_weight_min": -26.863691329956055,
      "activations/layer2_attention_weight_max": 35.29303741455078,
      "activations/layer2_attention_weight_min": -32.870262145996094,
      "activations/layer3_attention_weight_max": 97.61427307128906,
      "activations/layer3_attention_weight_min": -101.79753112792969,
      "activations/layer4_attention_weight_max": 107.0947265625,
      "activations/layer4_attention_weight_min": -112.1123275756836,
      "activations/layer5_attention_weight_max": 50.727943420410156,
      "activations/layer5_attention_weight_min": -65.19043731689453,
      "activations/layer6_attention_weight_max": 41.92842102050781,
      "activations/layer6_attention_weight_min": -45.81173324584961,
      "activations/layer7_attention_weight_max": 86.40998077392578,
      "activations/layer7_attention_weight_min": -93.77164459228516,
      "activations/layer8_attention_weight_max": 37.94671630859375,
      "activations/layer8_attention_weight_min": -42.36083984375,
      "activations/layer9_attention_weight_max": 31.51445960998535,
      "activations/layer9_attention_weight_min": -33.08272933959961,
      "epoch": 17.42,
      "learning_rate": 3.806363636363636e-05,
      "loss": 2.7315,
      "step": 299750
    },
    {
      "activations/layer0_attention_weight_max": 16.158084869384766,
      "activations/layer0_attention_weight_min": -13.961514472961426,
      "activations/layer10_attention_weight_max": 31.500661849975586,
      "activations/layer10_attention_weight_min": -32.15275955200195,
      "activations/layer11_attention_weight_max": 31.864356994628906,
      "activations/layer11_attention_weight_min": -32.60423278808594,
      "activations/layer12_attention_weight_max": 26.781970977783203,
      "activations/layer12_attention_weight_min": -25.263614654541016,
      "activations/layer13_attention_weight_max": 39.30531311035156,
      "activations/layer13_attention_weight_min": -34.91716003417969,
      "activations/layer14_attention_weight_max": 44.40018081665039,
      "activations/layer14_attention_weight_min": -36.69497299194336,
      "activations/layer15_attention_weight_max": 39.84251403808594,
      "activations/layer15_attention_weight_min": -33.39851760864258,
      "activations/layer16_attention_weight_max": 30.474145889282227,
      "activations/layer16_attention_weight_min": -27.445032119750977,
      "activations/layer17_attention_weight_max": 31.245004653930664,
      "activations/layer17_attention_weight_min": -25.21751594543457,
      "activations/layer18_attention_weight_max": 34.78373336791992,
      "activations/layer18_attention_weight_min": -24.55918312072754,
      "activations/layer19_attention_weight_max": 36.832706451416016,
      "activations/layer19_attention_weight_min": -31.441190719604492,
      "activations/layer1_attention_weight_max": 16.66061782836914,
      "activations/layer1_attention_weight_min": -16.20542335510254,
      "activations/layer20_attention_weight_max": 31.868745803833008,
      "activations/layer20_attention_weight_min": -25.483970642089844,
      "activations/layer21_attention_weight_max": 31.135684967041016,
      "activations/layer21_attention_weight_min": -24.967626571655273,
      "activations/layer22_attention_weight_max": 53.56804275512695,
      "activations/layer22_attention_weight_min": -30.90011978149414,
      "activations/layer23_attention_weight_max": 36.937461853027344,
      "activations/layer23_attention_weight_min": -26.560707092285156,
      "activations/layer2_attention_weight_max": 32.620452880859375,
      "activations/layer2_attention_weight_min": -33.13861846923828,
      "activations/layer3_attention_weight_max": 95.36412811279297,
      "activations/layer3_attention_weight_min": -96.0600357055664,
      "activations/layer4_attention_weight_max": 109.464111328125,
      "activations/layer4_attention_weight_min": -109.41996002197266,
      "activations/layer5_attention_weight_max": 50.12458038330078,
      "activations/layer5_attention_weight_min": -59.34166717529297,
      "activations/layer6_attention_weight_max": 42.345760345458984,
      "activations/layer6_attention_weight_min": -45.5918083190918,
      "activations/layer7_attention_weight_max": 92.05404663085938,
      "activations/layer7_attention_weight_min": -91.3913345336914,
      "activations/layer8_attention_weight_max": 40.16184616088867,
      "activations/layer8_attention_weight_min": -41.59829330444336,
      "activations/layer9_attention_weight_max": 31.905790328979492,
      "activations/layer9_attention_weight_min": -32.39579391479492,
      "epoch": 17.42,
      "learning_rate": 3.8044696969696966e-05,
      "loss": 2.7479,
      "step": 299800
    },
    {
      "activations/layer0_attention_weight_max": 15.739843368530273,
      "activations/layer0_attention_weight_min": -13.810051918029785,
      "activations/layer10_attention_weight_max": 35.755001068115234,
      "activations/layer10_attention_weight_min": -33.78535079956055,
      "activations/layer11_attention_weight_max": 33.156978607177734,
      "activations/layer11_attention_weight_min": -34.16839599609375,
      "activations/layer12_attention_weight_max": 25.79767417907715,
      "activations/layer12_attention_weight_min": -26.278873443603516,
      "activations/layer13_attention_weight_max": 38.54194259643555,
      "activations/layer13_attention_weight_min": -35.264217376708984,
      "activations/layer14_attention_weight_max": 46.397926330566406,
      "activations/layer14_attention_weight_min": -38.40736389160156,
      "activations/layer15_attention_weight_max": 42.537147521972656,
      "activations/layer15_attention_weight_min": -35.94406509399414,
      "activations/layer16_attention_weight_max": 30.31144142150879,
      "activations/layer16_attention_weight_min": -27.509918212890625,
      "activations/layer17_attention_weight_max": 30.74802017211914,
      "activations/layer17_attention_weight_min": -25.78565788269043,
      "activations/layer18_attention_weight_max": 30.23200798034668,
      "activations/layer18_attention_weight_min": -26.013166427612305,
      "activations/layer19_attention_weight_max": 34.209102630615234,
      "activations/layer19_attention_weight_min": -35.25893020629883,
      "activations/layer1_attention_weight_max": 17.0926456451416,
      "activations/layer1_attention_weight_min": -16.311317443847656,
      "activations/layer20_attention_weight_max": 31.950410842895508,
      "activations/layer20_attention_weight_min": -27.42668914794922,
      "activations/layer21_attention_weight_max": 27.576406478881836,
      "activations/layer21_attention_weight_min": -26.623716354370117,
      "activations/layer22_attention_weight_max": 43.47689437866211,
      "activations/layer22_attention_weight_min": -33.132015228271484,
      "activations/layer23_attention_weight_max": 34.35661315917969,
      "activations/layer23_attention_weight_min": -25.564834594726562,
      "activations/layer2_attention_weight_max": 33.05849838256836,
      "activations/layer2_attention_weight_min": -33.446533203125,
      "activations/layer3_attention_weight_max": 98.59479522705078,
      "activations/layer3_attention_weight_min": -100.66399383544922,
      "activations/layer4_attention_weight_max": 106.31127166748047,
      "activations/layer4_attention_weight_min": -108.98274230957031,
      "activations/layer5_attention_weight_max": 50.229347229003906,
      "activations/layer5_attention_weight_min": -60.25782775878906,
      "activations/layer6_attention_weight_max": 42.64445877075195,
      "activations/layer6_attention_weight_min": -44.69894790649414,
      "activations/layer7_attention_weight_max": 93.3973617553711,
      "activations/layer7_attention_weight_min": -94.02629089355469,
      "activations/layer8_attention_weight_max": 41.497467041015625,
      "activations/layer8_attention_weight_min": -43.27412796020508,
      "activations/layer9_attention_weight_max": 32.95923614501953,
      "activations/layer9_attention_weight_min": -34.325218200683594,
      "epoch": 17.42,
      "learning_rate": 3.8025757575757575e-05,
      "loss": 2.7151,
      "step": 299850
    },
    {
      "activations/layer0_attention_weight_max": 15.562962532043457,
      "activations/layer0_attention_weight_min": -12.955377578735352,
      "activations/layer10_attention_weight_max": 34.94779968261719,
      "activations/layer10_attention_weight_min": -34.37944030761719,
      "activations/layer11_attention_weight_max": 33.8460578918457,
      "activations/layer11_attention_weight_min": -34.39178466796875,
      "activations/layer12_attention_weight_max": 27.195005416870117,
      "activations/layer12_attention_weight_min": -25.822710037231445,
      "activations/layer13_attention_weight_max": 39.33559799194336,
      "activations/layer13_attention_weight_min": -34.675819396972656,
      "activations/layer14_attention_weight_max": 43.116050720214844,
      "activations/layer14_attention_weight_min": -36.833980560302734,
      "activations/layer15_attention_weight_max": 39.604278564453125,
      "activations/layer15_attention_weight_min": -33.905670166015625,
      "activations/layer16_attention_weight_max": 28.98424530029297,
      "activations/layer16_attention_weight_min": -27.299589157104492,
      "activations/layer17_attention_weight_max": 29.20888900756836,
      "activations/layer17_attention_weight_min": -25.735450744628906,
      "activations/layer18_attention_weight_max": 33.67795181274414,
      "activations/layer18_attention_weight_min": -25.704912185668945,
      "activations/layer19_attention_weight_max": 34.60595703125,
      "activations/layer19_attention_weight_min": -31.47141456604004,
      "activations/layer1_attention_weight_max": 15.757905960083008,
      "activations/layer1_attention_weight_min": -15.53746223449707,
      "activations/layer20_attention_weight_max": 30.307079315185547,
      "activations/layer20_attention_weight_min": -24.9373722076416,
      "activations/layer21_attention_weight_max": 27.690568923950195,
      "activations/layer21_attention_weight_min": -24.81646156311035,
      "activations/layer22_attention_weight_max": 40.69905471801758,
      "activations/layer22_attention_weight_min": -31.468164443969727,
      "activations/layer23_attention_weight_max": 35.10712432861328,
      "activations/layer23_attention_weight_min": -27.497404098510742,
      "activations/layer2_attention_weight_max": 32.810394287109375,
      "activations/layer2_attention_weight_min": -32.580833435058594,
      "activations/layer3_attention_weight_max": 96.31705474853516,
      "activations/layer3_attention_weight_min": -95.51859283447266,
      "activations/layer4_attention_weight_max": 105.06689453125,
      "activations/layer4_attention_weight_min": -108.92093658447266,
      "activations/layer5_attention_weight_max": 47.86334991455078,
      "activations/layer5_attention_weight_min": -59.402889251708984,
      "activations/layer6_attention_weight_max": 43.269222259521484,
      "activations/layer6_attention_weight_min": -46.46439743041992,
      "activations/layer7_attention_weight_max": 88.90817260742188,
      "activations/layer7_attention_weight_min": -92.10123443603516,
      "activations/layer8_attention_weight_max": 41.717735290527344,
      "activations/layer8_attention_weight_min": -43.93275833129883,
      "activations/layer9_attention_weight_max": 31.689117431640625,
      "activations/layer9_attention_weight_min": -34.430110931396484,
      "epoch": 17.43,
      "learning_rate": 3.8006818181818176e-05,
      "loss": 2.7327,
      "step": 299900
    },
    {
      "activations/layer0_attention_weight_max": 15.119329452514648,
      "activations/layer0_attention_weight_min": -12.745183944702148,
      "activations/layer10_attention_weight_max": 32.4472541809082,
      "activations/layer10_attention_weight_min": -32.488304138183594,
      "activations/layer11_attention_weight_max": 32.352806091308594,
      "activations/layer11_attention_weight_min": -32.613037109375,
      "activations/layer12_attention_weight_max": 28.71164894104004,
      "activations/layer12_attention_weight_min": -25.96316909790039,
      "activations/layer13_attention_weight_max": 40.80216598510742,
      "activations/layer13_attention_weight_min": -35.05628204345703,
      "activations/layer14_attention_weight_max": 45.24692916870117,
      "activations/layer14_attention_weight_min": -39.406394958496094,
      "activations/layer15_attention_weight_max": 39.549129486083984,
      "activations/layer15_attention_weight_min": -34.54090881347656,
      "activations/layer16_attention_weight_max": 33.19173049926758,
      "activations/layer16_attention_weight_min": -29.72808837890625,
      "activations/layer17_attention_weight_max": 30.421077728271484,
      "activations/layer17_attention_weight_min": -25.81671142578125,
      "activations/layer18_attention_weight_max": 31.308141708374023,
      "activations/layer18_attention_weight_min": -26.46568489074707,
      "activations/layer19_attention_weight_max": 36.35873031616211,
      "activations/layer19_attention_weight_min": -33.579490661621094,
      "activations/layer1_attention_weight_max": 16.89936637878418,
      "activations/layer1_attention_weight_min": -14.83548641204834,
      "activations/layer20_attention_weight_max": 31.771465301513672,
      "activations/layer20_attention_weight_min": -27.803571701049805,
      "activations/layer21_attention_weight_max": 31.02543067932129,
      "activations/layer21_attention_weight_min": -29.59604263305664,
      "activations/layer22_attention_weight_max": 45.001502990722656,
      "activations/layer22_attention_weight_min": -34.50849533081055,
      "activations/layer23_attention_weight_max": 36.874385833740234,
      "activations/layer23_attention_weight_min": -25.94411849975586,
      "activations/layer2_attention_weight_max": 31.90741539001465,
      "activations/layer2_attention_weight_min": -31.76649284362793,
      "activations/layer3_attention_weight_max": 98.45201110839844,
      "activations/layer3_attention_weight_min": -95.35812377929688,
      "activations/layer4_attention_weight_max": 102.7812728881836,
      "activations/layer4_attention_weight_min": -103.30995178222656,
      "activations/layer5_attention_weight_max": 47.90485763549805,
      "activations/layer5_attention_weight_min": -63.59548568725586,
      "activations/layer6_attention_weight_max": 46.61463165283203,
      "activations/layer6_attention_weight_min": -45.18744659423828,
      "activations/layer7_attention_weight_max": 94.79947662353516,
      "activations/layer7_attention_weight_min": -90.53112030029297,
      "activations/layer8_attention_weight_max": 41.05933380126953,
      "activations/layer8_attention_weight_min": -43.44401931762695,
      "activations/layer9_attention_weight_max": 32.29050064086914,
      "activations/layer9_attention_weight_min": -33.67660140991211,
      "epoch": 17.43,
      "learning_rate": 3.7987878787878784e-05,
      "loss": 2.7274,
      "step": 299950
    },
    {
      "activations/layer0_attention_weight_max": 16.585607528686523,
      "activations/layer0_attention_weight_min": -13.80669116973877,
      "activations/layer10_attention_weight_max": 31.591259002685547,
      "activations/layer10_attention_weight_min": -32.75239181518555,
      "activations/layer11_attention_weight_max": 33.373897552490234,
      "activations/layer11_attention_weight_min": -31.836040496826172,
      "activations/layer12_attention_weight_max": 25.71184730529785,
      "activations/layer12_attention_weight_min": -24.826921463012695,
      "activations/layer13_attention_weight_max": 41.5655403137207,
      "activations/layer13_attention_weight_min": -34.32695388793945,
      "activations/layer14_attention_weight_max": 42.943851470947266,
      "activations/layer14_attention_weight_min": -36.392860412597656,
      "activations/layer15_attention_weight_max": 40.288543701171875,
      "activations/layer15_attention_weight_min": -35.77941131591797,
      "activations/layer16_attention_weight_max": 29.86198616027832,
      "activations/layer16_attention_weight_min": -29.22848129272461,
      "activations/layer17_attention_weight_max": 30.05779457092285,
      "activations/layer17_attention_weight_min": -28.897050857543945,
      "activations/layer18_attention_weight_max": 33.86022186279297,
      "activations/layer18_attention_weight_min": -28.6627140045166,
      "activations/layer19_attention_weight_max": 37.30280685424805,
      "activations/layer19_attention_weight_min": -32.4423942565918,
      "activations/layer1_attention_weight_max": 17.98805809020996,
      "activations/layer1_attention_weight_min": -16.515663146972656,
      "activations/layer20_attention_weight_max": 31.32931137084961,
      "activations/layer20_attention_weight_min": -26.587352752685547,
      "activations/layer21_attention_weight_max": 29.854076385498047,
      "activations/layer21_attention_weight_min": -24.4905948638916,
      "activations/layer22_attention_weight_max": 44.16085433959961,
      "activations/layer22_attention_weight_min": -32.10540008544922,
      "activations/layer23_attention_weight_max": 33.923728942871094,
      "activations/layer23_attention_weight_min": -26.23787498474121,
      "activations/layer2_attention_weight_max": 34.65972900390625,
      "activations/layer2_attention_weight_min": -32.058990478515625,
      "activations/layer3_attention_weight_max": 99.83023834228516,
      "activations/layer3_attention_weight_min": -101.14989471435547,
      "activations/layer4_attention_weight_max": 105.07036590576172,
      "activations/layer4_attention_weight_min": -104.00770568847656,
      "activations/layer5_attention_weight_max": 50.39164733886719,
      "activations/layer5_attention_weight_min": -57.95570373535156,
      "activations/layer6_attention_weight_max": 42.54694747924805,
      "activations/layer6_attention_weight_min": -44.77450942993164,
      "activations/layer7_attention_weight_max": 90.992431640625,
      "activations/layer7_attention_weight_min": -86.41807556152344,
      "activations/layer8_attention_weight_max": 39.50727844238281,
      "activations/layer8_attention_weight_min": -41.28764724731445,
      "activations/layer9_attention_weight_max": 30.523651123046875,
      "activations/layer9_attention_weight_min": -33.400569915771484,
      "epoch": 17.43,
      "learning_rate": 3.796893939393939e-05,
      "loss": 2.7419,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4555,
      "eval_samples_per_second": 507.838,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4555,
      "eval_openwebtext_samples_per_second": 507.838,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9892,
      "eval_wikitext_samples_per_second": 229.234,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.5651,
      "eval_lambada_samples_per_second": 509.036,
      "step": 300000
    },
    {
      "activations/layer0_attention_weight_max": 15.152509689331055,
      "activations/layer0_attention_weight_min": -13.902412414550781,
      "activations/layer10_attention_weight_max": 32.207237243652344,
      "activations/layer10_attention_weight_min": -31.774465560913086,
      "activations/layer11_attention_weight_max": 31.998945236206055,
      "activations/layer11_attention_weight_min": -33.215789794921875,
      "activations/layer12_attention_weight_max": 24.94382095336914,
      "activations/layer12_attention_weight_min": -26.148242950439453,
      "activations/layer13_attention_weight_max": 37.572784423828125,
      "activations/layer13_attention_weight_min": -37.19962692260742,
      "activations/layer14_attention_weight_max": 39.34066390991211,
      "activations/layer14_attention_weight_min": -38.99203109741211,
      "activations/layer15_attention_weight_max": 38.78932189941406,
      "activations/layer15_attention_weight_min": -37.87552261352539,
      "activations/layer16_attention_weight_max": 30.90697479248047,
      "activations/layer16_attention_weight_min": -31.015396118164062,
      "activations/layer17_attention_weight_max": 28.71718978881836,
      "activations/layer17_attention_weight_min": -26.061965942382812,
      "activations/layer18_attention_weight_max": 31.57805061340332,
      "activations/layer18_attention_weight_min": -27.094655990600586,
      "activations/layer19_attention_weight_max": 35.705284118652344,
      "activations/layer19_attention_weight_min": -31.42555046081543,
      "activations/layer1_attention_weight_max": 16.965953826904297,
      "activations/layer1_attention_weight_min": -14.064615249633789,
      "activations/layer20_attention_weight_max": 29.705608367919922,
      "activations/layer20_attention_weight_min": -25.737285614013672,
      "activations/layer21_attention_weight_max": 29.396268844604492,
      "activations/layer21_attention_weight_min": -25.247283935546875,
      "activations/layer22_attention_weight_max": 42.22734451293945,
      "activations/layer22_attention_weight_min": -31.518959045410156,
      "activations/layer23_attention_weight_max": 31.138090133666992,
      "activations/layer23_attention_weight_min": -25.704998016357422,
      "activations/layer2_attention_weight_max": 33.159324645996094,
      "activations/layer2_attention_weight_min": -31.758075714111328,
      "activations/layer3_attention_weight_max": 98.9339370727539,
      "activations/layer3_attention_weight_min": -95.208251953125,
      "activations/layer4_attention_weight_max": 106.72845458984375,
      "activations/layer4_attention_weight_min": -105.89044189453125,
      "activations/layer5_attention_weight_max": 50.3426628112793,
      "activations/layer5_attention_weight_min": -62.121395111083984,
      "activations/layer6_attention_weight_max": 43.74263381958008,
      "activations/layer6_attention_weight_min": -45.498023986816406,
      "activations/layer7_attention_weight_max": 93.17581176757812,
      "activations/layer7_attention_weight_min": -92.66549682617188,
      "activations/layer8_attention_weight_max": 39.679107666015625,
      "activations/layer8_attention_weight_min": -41.96748733520508,
      "activations/layer9_attention_weight_max": 29.642370223999023,
      "activations/layer9_attention_weight_min": -32.095943450927734,
      "epoch": 17.43,
      "learning_rate": 3.7949999999999994e-05,
      "loss": 2.7397,
      "step": 300050
    },
    {
      "activations/layer0_attention_weight_max": 16.67527198791504,
      "activations/layer0_attention_weight_min": -14.81677532196045,
      "activations/layer10_attention_weight_max": 31.32053565979004,
      "activations/layer10_attention_weight_min": -34.2935905456543,
      "activations/layer11_attention_weight_max": 31.833805084228516,
      "activations/layer11_attention_weight_min": -33.69550704956055,
      "activations/layer12_attention_weight_max": 24.65285873413086,
      "activations/layer12_attention_weight_min": -26.60802459716797,
      "activations/layer13_attention_weight_max": 36.01163864135742,
      "activations/layer13_attention_weight_min": -35.99457550048828,
      "activations/layer14_attention_weight_max": 39.91735076904297,
      "activations/layer14_attention_weight_min": -38.264617919921875,
      "activations/layer15_attention_weight_max": 36.59059143066406,
      "activations/layer15_attention_weight_min": -36.27189254760742,
      "activations/layer16_attention_weight_max": 29.03850746154785,
      "activations/layer16_attention_weight_min": -27.454397201538086,
      "activations/layer17_attention_weight_max": 31.835256576538086,
      "activations/layer17_attention_weight_min": -27.47434425354004,
      "activations/layer18_attention_weight_max": 31.24327278137207,
      "activations/layer18_attention_weight_min": -24.75564193725586,
      "activations/layer19_attention_weight_max": 35.28424835205078,
      "activations/layer19_attention_weight_min": -30.089078903198242,
      "activations/layer1_attention_weight_max": 16.967967987060547,
      "activations/layer1_attention_weight_min": -15.046008110046387,
      "activations/layer20_attention_weight_max": 29.524559020996094,
      "activations/layer20_attention_weight_min": -24.708717346191406,
      "activations/layer21_attention_weight_max": 30.90274429321289,
      "activations/layer21_attention_weight_min": -22.9290771484375,
      "activations/layer22_attention_weight_max": 42.24072265625,
      "activations/layer22_attention_weight_min": -29.58156967163086,
      "activations/layer23_attention_weight_max": 31.449337005615234,
      "activations/layer23_attention_weight_min": -24.793880462646484,
      "activations/layer2_attention_weight_max": 32.82695007324219,
      "activations/layer2_attention_weight_min": -32.50384521484375,
      "activations/layer3_attention_weight_max": 97.94009399414062,
      "activations/layer3_attention_weight_min": -99.46449279785156,
      "activations/layer4_attention_weight_max": 103.65853118896484,
      "activations/layer4_attention_weight_min": -104.40010833740234,
      "activations/layer5_attention_weight_max": 50.21979904174805,
      "activations/layer5_attention_weight_min": -61.92701721191406,
      "activations/layer6_attention_weight_max": 42.385231018066406,
      "activations/layer6_attention_weight_min": -45.047115325927734,
      "activations/layer7_attention_weight_max": 92.02545166015625,
      "activations/layer7_attention_weight_min": -91.18497467041016,
      "activations/layer8_attention_weight_max": 37.73082733154297,
      "activations/layer8_attention_weight_min": -43.20542526245117,
      "activations/layer9_attention_weight_max": 29.71095085144043,
      "activations/layer9_attention_weight_min": -31.544666290283203,
      "epoch": 17.44,
      "learning_rate": 3.79310606060606e-05,
      "loss": 2.721,
      "step": 300100
    },
    {
      "activations/layer0_attention_weight_max": 16.796052932739258,
      "activations/layer0_attention_weight_min": -13.908814430236816,
      "activations/layer10_attention_weight_max": 34.712684631347656,
      "activations/layer10_attention_weight_min": -33.3953742980957,
      "activations/layer11_attention_weight_max": 34.77232360839844,
      "activations/layer11_attention_weight_min": -34.28497314453125,
      "activations/layer12_attention_weight_max": 26.67763900756836,
      "activations/layer12_attention_weight_min": -25.60913848876953,
      "activations/layer13_attention_weight_max": 39.902130126953125,
      "activations/layer13_attention_weight_min": -35.497310638427734,
      "activations/layer14_attention_weight_max": 40.58540725708008,
      "activations/layer14_attention_weight_min": -39.01682662963867,
      "activations/layer15_attention_weight_max": 37.69862365722656,
      "activations/layer15_attention_weight_min": -36.549468994140625,
      "activations/layer16_attention_weight_max": 29.419475555419922,
      "activations/layer16_attention_weight_min": -28.417293548583984,
      "activations/layer17_attention_weight_max": 34.69953536987305,
      "activations/layer17_attention_weight_min": -26.893945693969727,
      "activations/layer18_attention_weight_max": 32.4078369140625,
      "activations/layer18_attention_weight_min": -27.679067611694336,
      "activations/layer19_attention_weight_max": 33.351558685302734,
      "activations/layer19_attention_weight_min": -31.769384384155273,
      "activations/layer1_attention_weight_max": 16.362606048583984,
      "activations/layer1_attention_weight_min": -13.902444839477539,
      "activations/layer20_attention_weight_max": 30.5998477935791,
      "activations/layer20_attention_weight_min": -26.847034454345703,
      "activations/layer21_attention_weight_max": 26.928503036499023,
      "activations/layer21_attention_weight_min": -26.22329330444336,
      "activations/layer22_attention_weight_max": 40.66740036010742,
      "activations/layer22_attention_weight_min": -33.56132888793945,
      "activations/layer23_attention_weight_max": 29.7789306640625,
      "activations/layer23_attention_weight_min": -26.744962692260742,
      "activations/layer2_attention_weight_max": 32.995887756347656,
      "activations/layer2_attention_weight_min": -32.37049102783203,
      "activations/layer3_attention_weight_max": 98.82952880859375,
      "activations/layer3_attention_weight_min": -99.31708526611328,
      "activations/layer4_attention_weight_max": 108.1744613647461,
      "activations/layer4_attention_weight_min": -104.63188934326172,
      "activations/layer5_attention_weight_max": 49.1783561706543,
      "activations/layer5_attention_weight_min": -60.081443786621094,
      "activations/layer6_attention_weight_max": 44.14564514160156,
      "activations/layer6_attention_weight_min": -45.684444427490234,
      "activations/layer7_attention_weight_max": 94.0199966430664,
      "activations/layer7_attention_weight_min": -92.18167114257812,
      "activations/layer8_attention_weight_max": 40.61085891723633,
      "activations/layer8_attention_weight_min": -44.08836364746094,
      "activations/layer9_attention_weight_max": 32.50282669067383,
      "activations/layer9_attention_weight_min": -33.98183822631836,
      "epoch": 17.44,
      "learning_rate": 3.7912121212121204e-05,
      "loss": 2.721,
      "step": 300150
    },
    {
      "activations/layer0_attention_weight_max": 15.546686172485352,
      "activations/layer0_attention_weight_min": -12.478409767150879,
      "activations/layer10_attention_weight_max": 38.915191650390625,
      "activations/layer10_attention_weight_min": -38.429752349853516,
      "activations/layer11_attention_weight_max": 37.66251754760742,
      "activations/layer11_attention_weight_min": -38.003597259521484,
      "activations/layer12_attention_weight_max": 27.69318962097168,
      "activations/layer12_attention_weight_min": -29.0490779876709,
      "activations/layer13_attention_weight_max": 40.61794662475586,
      "activations/layer13_attention_weight_min": -39.8659553527832,
      "activations/layer14_attention_weight_max": 46.67519760131836,
      "activations/layer14_attention_weight_min": -43.175567626953125,
      "activations/layer15_attention_weight_max": 42.15898132324219,
      "activations/layer15_attention_weight_min": -40.767757415771484,
      "activations/layer16_attention_weight_max": 30.174654006958008,
      "activations/layer16_attention_weight_min": -28.463136672973633,
      "activations/layer17_attention_weight_max": 29.48699951171875,
      "activations/layer17_attention_weight_min": -28.2626953125,
      "activations/layer18_attention_weight_max": 31.60515785217285,
      "activations/layer18_attention_weight_min": -26.779052734375,
      "activations/layer19_attention_weight_max": 36.15806198120117,
      "activations/layer19_attention_weight_min": -32.167274475097656,
      "activations/layer1_attention_weight_max": 15.704204559326172,
      "activations/layer1_attention_weight_min": -13.968026161193848,
      "activations/layer20_attention_weight_max": 30.25177574157715,
      "activations/layer20_attention_weight_min": -24.802696228027344,
      "activations/layer21_attention_weight_max": 30.39833641052246,
      "activations/layer21_attention_weight_min": -26.499603271484375,
      "activations/layer22_attention_weight_max": 41.81209182739258,
      "activations/layer22_attention_weight_min": -33.38771057128906,
      "activations/layer23_attention_weight_max": 34.503963470458984,
      "activations/layer23_attention_weight_min": -25.96632194519043,
      "activations/layer2_attention_weight_max": 32.96553039550781,
      "activations/layer2_attention_weight_min": -34.58174514770508,
      "activations/layer3_attention_weight_max": 96.92265319824219,
      "activations/layer3_attention_weight_min": -101.80896759033203,
      "activations/layer4_attention_weight_max": 108.1979751586914,
      "activations/layer4_attention_weight_min": -106.7020492553711,
      "activations/layer5_attention_weight_max": 48.005859375,
      "activations/layer5_attention_weight_min": -60.18973922729492,
      "activations/layer6_attention_weight_max": 43.90428924560547,
      "activations/layer6_attention_weight_min": -47.4423942565918,
      "activations/layer7_attention_weight_max": 96.2520980834961,
      "activations/layer7_attention_weight_min": -98.92169952392578,
      "activations/layer8_attention_weight_max": 42.143150329589844,
      "activations/layer8_attention_weight_min": -44.94573211669922,
      "activations/layer9_attention_weight_max": 33.57250213623047,
      "activations/layer9_attention_weight_min": -36.46875762939453,
      "epoch": 17.44,
      "learning_rate": 3.789318181818182e-05,
      "loss": 2.7402,
      "step": 300200
    },
    {
      "activations/layer0_attention_weight_max": 15.461800575256348,
      "activations/layer0_attention_weight_min": -13.055758476257324,
      "activations/layer10_attention_weight_max": 33.37309265136719,
      "activations/layer10_attention_weight_min": -35.398292541503906,
      "activations/layer11_attention_weight_max": 32.979190826416016,
      "activations/layer11_attention_weight_min": -33.04035949707031,
      "activations/layer12_attention_weight_max": 26.70579719543457,
      "activations/layer12_attention_weight_min": -27.724868774414062,
      "activations/layer13_attention_weight_max": 39.0206184387207,
      "activations/layer13_attention_weight_min": -35.688167572021484,
      "activations/layer14_attention_weight_max": 41.809967041015625,
      "activations/layer14_attention_weight_min": -37.411739349365234,
      "activations/layer15_attention_weight_max": 38.926944732666016,
      "activations/layer15_attention_weight_min": -37.53902053833008,
      "activations/layer16_attention_weight_max": 29.88932991027832,
      "activations/layer16_attention_weight_min": -30.4976806640625,
      "activations/layer17_attention_weight_max": 30.05495262145996,
      "activations/layer17_attention_weight_min": -25.56818962097168,
      "activations/layer18_attention_weight_max": 36.917755126953125,
      "activations/layer18_attention_weight_min": -27.33295249938965,
      "activations/layer19_attention_weight_max": 37.19252014160156,
      "activations/layer19_attention_weight_min": -31.09710693359375,
      "activations/layer1_attention_weight_max": 16.4099178314209,
      "activations/layer1_attention_weight_min": -15.392006874084473,
      "activations/layer20_attention_weight_max": 28.73949432373047,
      "activations/layer20_attention_weight_min": -25.02736473083496,
      "activations/layer21_attention_weight_max": 29.026445388793945,
      "activations/layer21_attention_weight_min": -24.299015045166016,
      "activations/layer22_attention_weight_max": 43.80289077758789,
      "activations/layer22_attention_weight_min": -30.6545467376709,
      "activations/layer23_attention_weight_max": 31.73123550415039,
      "activations/layer23_attention_weight_min": -25.441396713256836,
      "activations/layer2_attention_weight_max": 33.173072814941406,
      "activations/layer2_attention_weight_min": -32.52555847167969,
      "activations/layer3_attention_weight_max": 98.93035125732422,
      "activations/layer3_attention_weight_min": -105.37528228759766,
      "activations/layer4_attention_weight_max": 105.13887786865234,
      "activations/layer4_attention_weight_min": -111.58203125,
      "activations/layer5_attention_weight_max": 49.6077995300293,
      "activations/layer5_attention_weight_min": -61.85901641845703,
      "activations/layer6_attention_weight_max": 44.81576919555664,
      "activations/layer6_attention_weight_min": -46.89066696166992,
      "activations/layer7_attention_weight_max": 92.12312316894531,
      "activations/layer7_attention_weight_min": -96.73212432861328,
      "activations/layer8_attention_weight_max": 40.39350891113281,
      "activations/layer8_attention_weight_min": -42.669857025146484,
      "activations/layer9_attention_weight_max": 30.903308868408203,
      "activations/layer9_attention_weight_min": -35.91892623901367,
      "epoch": 17.45,
      "learning_rate": 3.787424242424242e-05,
      "loss": 2.7416,
      "step": 300250
    },
    {
      "activations/layer0_attention_weight_max": 15.746793746948242,
      "activations/layer0_attention_weight_min": -12.353075981140137,
      "activations/layer10_attention_weight_max": 31.584365844726562,
      "activations/layer10_attention_weight_min": -32.562747955322266,
      "activations/layer11_attention_weight_max": 32.14702606201172,
      "activations/layer11_attention_weight_min": -35.84661102294922,
      "activations/layer12_attention_weight_max": 27.468887329101562,
      "activations/layer12_attention_weight_min": -27.1077823638916,
      "activations/layer13_attention_weight_max": 39.66279220581055,
      "activations/layer13_attention_weight_min": -35.44464874267578,
      "activations/layer14_attention_weight_max": 44.30302810668945,
      "activations/layer14_attention_weight_min": -38.92900085449219,
      "activations/layer15_attention_weight_max": 43.66878128051758,
      "activations/layer15_attention_weight_min": -36.46835708618164,
      "activations/layer16_attention_weight_max": 32.306156158447266,
      "activations/layer16_attention_weight_min": -28.420516967773438,
      "activations/layer17_attention_weight_max": 32.96933364868164,
      "activations/layer17_attention_weight_min": -25.694183349609375,
      "activations/layer18_attention_weight_max": 35.0821647644043,
      "activations/layer18_attention_weight_min": -27.102664947509766,
      "activations/layer19_attention_weight_max": 38.224281311035156,
      "activations/layer19_attention_weight_min": -33.79771423339844,
      "activations/layer1_attention_weight_max": 16.48491668701172,
      "activations/layer1_attention_weight_min": -15.630398750305176,
      "activations/layer20_attention_weight_max": 32.732784271240234,
      "activations/layer20_attention_weight_min": -25.875059127807617,
      "activations/layer21_attention_weight_max": 30.752487182617188,
      "activations/layer21_attention_weight_min": -27.273900985717773,
      "activations/layer22_attention_weight_max": 48.33933639526367,
      "activations/layer22_attention_weight_min": -33.733360290527344,
      "activations/layer23_attention_weight_max": 36.53209686279297,
      "activations/layer23_attention_weight_min": -27.33709716796875,
      "activations/layer2_attention_weight_max": 33.49016571044922,
      "activations/layer2_attention_weight_min": -32.36500930786133,
      "activations/layer3_attention_weight_max": 98.26468658447266,
      "activations/layer3_attention_weight_min": -98.54097747802734,
      "activations/layer4_attention_weight_max": 103.271240234375,
      "activations/layer4_attention_weight_min": -108.0787124633789,
      "activations/layer5_attention_weight_max": 49.17140197753906,
      "activations/layer5_attention_weight_min": -62.26331329345703,
      "activations/layer6_attention_weight_max": 45.64183807373047,
      "activations/layer6_attention_weight_min": -49.02269744873047,
      "activations/layer7_attention_weight_max": 90.91722106933594,
      "activations/layer7_attention_weight_min": -100.50151062011719,
      "activations/layer8_attention_weight_max": 40.81880569458008,
      "activations/layer8_attention_weight_min": -43.65526580810547,
      "activations/layer9_attention_weight_max": 30.850534439086914,
      "activations/layer9_attention_weight_min": -32.7481803894043,
      "epoch": 17.45,
      "learning_rate": 3.785530303030302e-05,
      "loss": 2.7217,
      "step": 300300
    },
    {
      "activations/layer0_attention_weight_max": 16.169546127319336,
      "activations/layer0_attention_weight_min": -13.165406227111816,
      "activations/layer10_attention_weight_max": 32.7926025390625,
      "activations/layer10_attention_weight_min": -32.67462921142578,
      "activations/layer11_attention_weight_max": 29.559499740600586,
      "activations/layer11_attention_weight_min": -31.583799362182617,
      "activations/layer12_attention_weight_max": 24.769458770751953,
      "activations/layer12_attention_weight_min": -25.596256256103516,
      "activations/layer13_attention_weight_max": 37.8248291015625,
      "activations/layer13_attention_weight_min": -34.8046989440918,
      "activations/layer14_attention_weight_max": 38.586631774902344,
      "activations/layer14_attention_weight_min": -38.42207336425781,
      "activations/layer15_attention_weight_max": 36.67060089111328,
      "activations/layer15_attention_weight_min": -35.5755500793457,
      "activations/layer16_attention_weight_max": 28.91861915588379,
      "activations/layer16_attention_weight_min": -28.775123596191406,
      "activations/layer17_attention_weight_max": 29.867448806762695,
      "activations/layer17_attention_weight_min": -25.85516357421875,
      "activations/layer18_attention_weight_max": 33.778167724609375,
      "activations/layer18_attention_weight_min": -26.00516128540039,
      "activations/layer19_attention_weight_max": 34.495811462402344,
      "activations/layer19_attention_weight_min": -30.38765525817871,
      "activations/layer1_attention_weight_max": 16.879608154296875,
      "activations/layer1_attention_weight_min": -14.754830360412598,
      "activations/layer20_attention_weight_max": 30.345447540283203,
      "activations/layer20_attention_weight_min": -23.671262741088867,
      "activations/layer21_attention_weight_max": 30.827043533325195,
      "activations/layer21_attention_weight_min": -25.146947860717773,
      "activations/layer22_attention_weight_max": 41.950706481933594,
      "activations/layer22_attention_weight_min": -30.23710823059082,
      "activations/layer23_attention_weight_max": 37.93251419067383,
      "activations/layer23_attention_weight_min": -25.17949676513672,
      "activations/layer2_attention_weight_max": 32.09619903564453,
      "activations/layer2_attention_weight_min": -32.60615539550781,
      "activations/layer3_attention_weight_max": 99.86937713623047,
      "activations/layer3_attention_weight_min": -98.35481262207031,
      "activations/layer4_attention_weight_max": 102.96357727050781,
      "activations/layer4_attention_weight_min": -105.15340423583984,
      "activations/layer5_attention_weight_max": 51.85197067260742,
      "activations/layer5_attention_weight_min": -62.20396041870117,
      "activations/layer6_attention_weight_max": 42.48054885864258,
      "activations/layer6_attention_weight_min": -46.29018020629883,
      "activations/layer7_attention_weight_max": 88.92564392089844,
      "activations/layer7_attention_weight_min": -100.8120346069336,
      "activations/layer8_attention_weight_max": 38.957481384277344,
      "activations/layer8_attention_weight_min": -41.37038803100586,
      "activations/layer9_attention_weight_max": 32.168251037597656,
      "activations/layer9_attention_weight_min": -32.67387771606445,
      "epoch": 17.45,
      "learning_rate": 3.783636363636364e-05,
      "loss": 2.7354,
      "step": 300350
    },
    {
      "activations/layer0_attention_weight_max": 15.609167098999023,
      "activations/layer0_attention_weight_min": -13.961743354797363,
      "activations/layer10_attention_weight_max": 29.165660858154297,
      "activations/layer10_attention_weight_min": -31.62930679321289,
      "activations/layer11_attention_weight_max": 30.390682220458984,
      "activations/layer11_attention_weight_min": -33.325355529785156,
      "activations/layer12_attention_weight_max": 25.971864700317383,
      "activations/layer12_attention_weight_min": -25.613889694213867,
      "activations/layer13_attention_weight_max": 40.315513610839844,
      "activations/layer13_attention_weight_min": -35.81942367553711,
      "activations/layer14_attention_weight_max": 42.55097198486328,
      "activations/layer14_attention_weight_min": -37.024375915527344,
      "activations/layer15_attention_weight_max": 39.79902267456055,
      "activations/layer15_attention_weight_min": -35.42856216430664,
      "activations/layer16_attention_weight_max": 32.439002990722656,
      "activations/layer16_attention_weight_min": -27.77837371826172,
      "activations/layer17_attention_weight_max": 33.22575759887695,
      "activations/layer17_attention_weight_min": -25.54129409790039,
      "activations/layer18_attention_weight_max": 29.9559268951416,
      "activations/layer18_attention_weight_min": -25.8449764251709,
      "activations/layer19_attention_weight_max": 37.81448745727539,
      "activations/layer19_attention_weight_min": -32.843502044677734,
      "activations/layer1_attention_weight_max": 16.27410125732422,
      "activations/layer1_attention_weight_min": -15.225651741027832,
      "activations/layer20_attention_weight_max": 32.58414077758789,
      "activations/layer20_attention_weight_min": -24.66874885559082,
      "activations/layer21_attention_weight_max": 31.036521911621094,
      "activations/layer21_attention_weight_min": -24.541259765625,
      "activations/layer22_attention_weight_max": 44.9012336730957,
      "activations/layer22_attention_weight_min": -30.09938621520996,
      "activations/layer23_attention_weight_max": 35.156829833984375,
      "activations/layer23_attention_weight_min": -27.420866012573242,
      "activations/layer2_attention_weight_max": 35.729373931884766,
      "activations/layer2_attention_weight_min": -32.95098876953125,
      "activations/layer3_attention_weight_max": 102.49034118652344,
      "activations/layer3_attention_weight_min": -98.88095092773438,
      "activations/layer4_attention_weight_max": 105.2835464477539,
      "activations/layer4_attention_weight_min": -108.2166519165039,
      "activations/layer5_attention_weight_max": 53.84613800048828,
      "activations/layer5_attention_weight_min": -62.75509262084961,
      "activations/layer6_attention_weight_max": 43.50664520263672,
      "activations/layer6_attention_weight_min": -44.0111198425293,
      "activations/layer7_attention_weight_max": 90.21916961669922,
      "activations/layer7_attention_weight_min": -90.46442413330078,
      "activations/layer8_attention_weight_max": 39.4532356262207,
      "activations/layer8_attention_weight_min": -39.870765686035156,
      "activations/layer9_attention_weight_max": 29.679306030273438,
      "activations/layer9_attention_weight_min": -30.937021255493164,
      "epoch": 17.45,
      "learning_rate": 3.781742424242424e-05,
      "loss": 2.7382,
      "step": 300400
    },
    {
      "activations/layer0_attention_weight_max": 16.849628448486328,
      "activations/layer0_attention_weight_min": -13.182241439819336,
      "activations/layer10_attention_weight_max": 36.76263427734375,
      "activations/layer10_attention_weight_min": -34.993621826171875,
      "activations/layer11_attention_weight_max": 39.79800033569336,
      "activations/layer11_attention_weight_min": -36.379554748535156,
      "activations/layer12_attention_weight_max": 27.109086990356445,
      "activations/layer12_attention_weight_min": -27.53182601928711,
      "activations/layer13_attention_weight_max": 38.73917770385742,
      "activations/layer13_attention_weight_min": -37.62889099121094,
      "activations/layer14_attention_weight_max": 43.19998550415039,
      "activations/layer14_attention_weight_min": -41.284812927246094,
      "activations/layer15_attention_weight_max": 39.49713134765625,
      "activations/layer15_attention_weight_min": -36.55593490600586,
      "activations/layer16_attention_weight_max": 30.202661514282227,
      "activations/layer16_attention_weight_min": -27.697200775146484,
      "activations/layer17_attention_weight_max": 27.18423843383789,
      "activations/layer17_attention_weight_min": -24.813135147094727,
      "activations/layer18_attention_weight_max": 31.460430145263672,
      "activations/layer18_attention_weight_min": -24.175479888916016,
      "activations/layer19_attention_weight_max": 32.970821380615234,
      "activations/layer19_attention_weight_min": -30.339460372924805,
      "activations/layer1_attention_weight_max": 17.225746154785156,
      "activations/layer1_attention_weight_min": -17.44101905822754,
      "activations/layer20_attention_weight_max": 29.654512405395508,
      "activations/layer20_attention_weight_min": -24.74913787841797,
      "activations/layer21_attention_weight_max": 30.155071258544922,
      "activations/layer21_attention_weight_min": -24.25881004333496,
      "activations/layer22_attention_weight_max": 43.12283706665039,
      "activations/layer22_attention_weight_min": -31.09505844116211,
      "activations/layer23_attention_weight_max": 29.890430450439453,
      "activations/layer23_attention_weight_min": -24.313459396362305,
      "activations/layer2_attention_weight_max": 34.80332946777344,
      "activations/layer2_attention_weight_min": -33.34705352783203,
      "activations/layer3_attention_weight_max": 100.4146957397461,
      "activations/layer3_attention_weight_min": -103.09353637695312,
      "activations/layer4_attention_weight_max": 105.43145751953125,
      "activations/layer4_attention_weight_min": -105.40068817138672,
      "activations/layer5_attention_weight_max": 52.01043701171875,
      "activations/layer5_attention_weight_min": -63.06623840332031,
      "activations/layer6_attention_weight_max": 43.774085998535156,
      "activations/layer6_attention_weight_min": -45.63529586791992,
      "activations/layer7_attention_weight_max": 89.90872955322266,
      "activations/layer7_attention_weight_min": -91.78486633300781,
      "activations/layer8_attention_weight_max": 43.26004409790039,
      "activations/layer8_attention_weight_min": -44.16035842895508,
      "activations/layer9_attention_weight_max": 35.30219268798828,
      "activations/layer9_attention_weight_min": -34.65990447998047,
      "epoch": 17.46,
      "learning_rate": 3.779848484848485e-05,
      "loss": 2.7209,
      "step": 300450
    },
    {
      "activations/layer0_attention_weight_max": 15.973249435424805,
      "activations/layer0_attention_weight_min": -13.004583358764648,
      "activations/layer10_attention_weight_max": 31.425315856933594,
      "activations/layer10_attention_weight_min": -33.11122131347656,
      "activations/layer11_attention_weight_max": 31.572914123535156,
      "activations/layer11_attention_weight_min": -31.9852294921875,
      "activations/layer12_attention_weight_max": 26.067110061645508,
      "activations/layer12_attention_weight_min": -24.7279052734375,
      "activations/layer13_attention_weight_max": 40.14949035644531,
      "activations/layer13_attention_weight_min": -33.697242736816406,
      "activations/layer14_attention_weight_max": 39.952980041503906,
      "activations/layer14_attention_weight_min": -34.178123474121094,
      "activations/layer15_attention_weight_max": 39.658634185791016,
      "activations/layer15_attention_weight_min": -32.26323699951172,
      "activations/layer16_attention_weight_max": 27.61478614807129,
      "activations/layer16_attention_weight_min": -28.168249130249023,
      "activations/layer17_attention_weight_max": 28.427135467529297,
      "activations/layer17_attention_weight_min": -26.89701271057129,
      "activations/layer18_attention_weight_max": 32.76359176635742,
      "activations/layer18_attention_weight_min": -28.404399871826172,
      "activations/layer19_attention_weight_max": 36.830116271972656,
      "activations/layer19_attention_weight_min": -33.31101608276367,
      "activations/layer1_attention_weight_max": 15.952186584472656,
      "activations/layer1_attention_weight_min": -14.97840404510498,
      "activations/layer20_attention_weight_max": 32.959835052490234,
      "activations/layer20_attention_weight_min": -27.463376998901367,
      "activations/layer21_attention_weight_max": 30.381927490234375,
      "activations/layer21_attention_weight_min": -25.194854736328125,
      "activations/layer22_attention_weight_max": 40.59628677368164,
      "activations/layer22_attention_weight_min": -33.8760871887207,
      "activations/layer23_attention_weight_max": 34.04133987426758,
      "activations/layer23_attention_weight_min": -25.786453247070312,
      "activations/layer2_attention_weight_max": 33.15431594848633,
      "activations/layer2_attention_weight_min": -32.96862030029297,
      "activations/layer3_attention_weight_max": 97.1521987915039,
      "activations/layer3_attention_weight_min": -96.10116577148438,
      "activations/layer4_attention_weight_max": 104.1291275024414,
      "activations/layer4_attention_weight_min": -107.0329818725586,
      "activations/layer5_attention_weight_max": 48.770301818847656,
      "activations/layer5_attention_weight_min": -66.09886932373047,
      "activations/layer6_attention_weight_max": 41.036476135253906,
      "activations/layer6_attention_weight_min": -43.204917907714844,
      "activations/layer7_attention_weight_max": 90.04255676269531,
      "activations/layer7_attention_weight_min": -89.94011688232422,
      "activations/layer8_attention_weight_max": 39.358158111572266,
      "activations/layer8_attention_weight_min": -41.8453369140625,
      "activations/layer9_attention_weight_max": 30.265222549438477,
      "activations/layer9_attention_weight_min": -34.684322357177734,
      "epoch": 17.46,
      "learning_rate": 3.777954545454545e-05,
      "loss": 2.7442,
      "step": 300500
    },
    {
      "activations/layer0_attention_weight_max": 16.206375122070312,
      "activations/layer0_attention_weight_min": -13.35018253326416,
      "activations/layer10_attention_weight_max": 30.371273040771484,
      "activations/layer10_attention_weight_min": -32.45941925048828,
      "activations/layer11_attention_weight_max": 31.421905517578125,
      "activations/layer11_attention_weight_min": -32.498695373535156,
      "activations/layer12_attention_weight_max": 26.78794288635254,
      "activations/layer12_attention_weight_min": -25.819272994995117,
      "activations/layer13_attention_weight_max": 41.1141357421875,
      "activations/layer13_attention_weight_min": -34.73051071166992,
      "activations/layer14_attention_weight_max": 41.583805084228516,
      "activations/layer14_attention_weight_min": -37.93672180175781,
      "activations/layer15_attention_weight_max": 38.898597717285156,
      "activations/layer15_attention_weight_min": -34.10568618774414,
      "activations/layer16_attention_weight_max": 31.38869285583496,
      "activations/layer16_attention_weight_min": -27.619112014770508,
      "activations/layer17_attention_weight_max": 28.218921661376953,
      "activations/layer17_attention_weight_min": -26.03083610534668,
      "activations/layer18_attention_weight_max": 30.820791244506836,
      "activations/layer18_attention_weight_min": -25.81559181213379,
      "activations/layer19_attention_weight_max": 31.443374633789062,
      "activations/layer19_attention_weight_min": -29.79144859313965,
      "activations/layer1_attention_weight_max": 15.464592933654785,
      "activations/layer1_attention_weight_min": -14.817320823669434,
      "activations/layer20_attention_weight_max": 30.96352195739746,
      "activations/layer20_attention_weight_min": -26.710966110229492,
      "activations/layer21_attention_weight_max": 31.4890193939209,
      "activations/layer21_attention_weight_min": -24.697044372558594,
      "activations/layer22_attention_weight_max": 42.89612579345703,
      "activations/layer22_attention_weight_min": -31.785724639892578,
      "activations/layer23_attention_weight_max": 29.829666137695312,
      "activations/layer23_attention_weight_min": -27.12373924255371,
      "activations/layer2_attention_weight_max": 32.936431884765625,
      "activations/layer2_attention_weight_min": -32.31302261352539,
      "activations/layer3_attention_weight_max": 96.53839111328125,
      "activations/layer3_attention_weight_min": -99.26976776123047,
      "activations/layer4_attention_weight_max": 107.01122283935547,
      "activations/layer4_attention_weight_min": -106.79176330566406,
      "activations/layer5_attention_weight_max": 50.2523193359375,
      "activations/layer5_attention_weight_min": -60.38003158569336,
      "activations/layer6_attention_weight_max": 42.67639923095703,
      "activations/layer6_attention_weight_min": -47.472835540771484,
      "activations/layer7_attention_weight_max": 91.41960144042969,
      "activations/layer7_attention_weight_min": -93.12691497802734,
      "activations/layer8_attention_weight_max": 39.35966873168945,
      "activations/layer8_attention_weight_min": -43.04512405395508,
      "activations/layer9_attention_weight_max": 30.349180221557617,
      "activations/layer9_attention_weight_min": -35.15142822265625,
      "epoch": 17.46,
      "learning_rate": 3.776060606060606e-05,
      "loss": 2.7375,
      "step": 300550
    },
    {
      "activations/layer0_attention_weight_max": 16.47543716430664,
      "activations/layer0_attention_weight_min": -13.444379806518555,
      "activations/layer10_attention_weight_max": 30.14986228942871,
      "activations/layer10_attention_weight_min": -31.981075286865234,
      "activations/layer11_attention_weight_max": 31.190326690673828,
      "activations/layer11_attention_weight_min": -32.514434814453125,
      "activations/layer12_attention_weight_max": 28.86604118347168,
      "activations/layer12_attention_weight_min": -26.59942054748535,
      "activations/layer13_attention_weight_max": 43.348487854003906,
      "activations/layer13_attention_weight_min": -34.49506759643555,
      "activations/layer14_attention_weight_max": 42.25833511352539,
      "activations/layer14_attention_weight_min": -37.6402473449707,
      "activations/layer15_attention_weight_max": 37.85818099975586,
      "activations/layer15_attention_weight_min": -34.14748001098633,
      "activations/layer16_attention_weight_max": 29.586240768432617,
      "activations/layer16_attention_weight_min": -27.141597747802734,
      "activations/layer17_attention_weight_max": 28.9642276763916,
      "activations/layer17_attention_weight_min": -26.258304595947266,
      "activations/layer18_attention_weight_max": 30.43040657043457,
      "activations/layer18_attention_weight_min": -26.007190704345703,
      "activations/layer19_attention_weight_max": 33.7535400390625,
      "activations/layer19_attention_weight_min": -30.622671127319336,
      "activations/layer1_attention_weight_max": 16.432645797729492,
      "activations/layer1_attention_weight_min": -13.858591079711914,
      "activations/layer20_attention_weight_max": 27.986980438232422,
      "activations/layer20_attention_weight_min": -25.705049514770508,
      "activations/layer21_attention_weight_max": 26.93498992919922,
      "activations/layer21_attention_weight_min": -25.301244735717773,
      "activations/layer22_attention_weight_max": 45.58069610595703,
      "activations/layer22_attention_weight_min": -32.267417907714844,
      "activations/layer23_attention_weight_max": 30.983816146850586,
      "activations/layer23_attention_weight_min": -26.84524154663086,
      "activations/layer2_attention_weight_max": 31.757802963256836,
      "activations/layer2_attention_weight_min": -32.534912109375,
      "activations/layer3_attention_weight_max": 98.11292266845703,
      "activations/layer3_attention_weight_min": -99.85916137695312,
      "activations/layer4_attention_weight_max": 109.9740982055664,
      "activations/layer4_attention_weight_min": -109.6009750366211,
      "activations/layer5_attention_weight_max": 49.011287689208984,
      "activations/layer5_attention_weight_min": -62.25840377807617,
      "activations/layer6_attention_weight_max": 40.41529083251953,
      "activations/layer6_attention_weight_min": -44.50160217285156,
      "activations/layer7_attention_weight_max": 86.80040740966797,
      "activations/layer7_attention_weight_min": -92.28227233886719,
      "activations/layer8_attention_weight_max": 36.977561950683594,
      "activations/layer8_attention_weight_min": -41.681819915771484,
      "activations/layer9_attention_weight_max": 29.77760887145996,
      "activations/layer9_attention_weight_min": -32.884517669677734,
      "epoch": 17.47,
      "learning_rate": 3.7741666666666665e-05,
      "loss": 2.739,
      "step": 300600
    },
    {
      "activations/layer0_attention_weight_max": 16.021142959594727,
      "activations/layer0_attention_weight_min": -12.605363845825195,
      "activations/layer10_attention_weight_max": 38.217464447021484,
      "activations/layer10_attention_weight_min": -38.94902038574219,
      "activations/layer11_attention_weight_max": 40.389678955078125,
      "activations/layer11_attention_weight_min": -40.32292175292969,
      "activations/layer12_attention_weight_max": 29.606494903564453,
      "activations/layer12_attention_weight_min": -27.7130184173584,
      "activations/layer13_attention_weight_max": 42.874027252197266,
      "activations/layer13_attention_weight_min": -42.68254089355469,
      "activations/layer14_attention_weight_max": 51.37868881225586,
      "activations/layer14_attention_weight_min": -45.80426025390625,
      "activations/layer15_attention_weight_max": 42.701534271240234,
      "activations/layer15_attention_weight_min": -41.910709381103516,
      "activations/layer16_attention_weight_max": 30.770275115966797,
      "activations/layer16_attention_weight_min": -29.253822326660156,
      "activations/layer17_attention_weight_max": 27.243921279907227,
      "activations/layer17_attention_weight_min": -26.59221839904785,
      "activations/layer18_attention_weight_max": 29.948226928710938,
      "activations/layer18_attention_weight_min": -25.33694076538086,
      "activations/layer19_attention_weight_max": 34.0452766418457,
      "activations/layer19_attention_weight_min": -34.18305587768555,
      "activations/layer1_attention_weight_max": 16.755958557128906,
      "activations/layer1_attention_weight_min": -17.412992477416992,
      "activations/layer20_attention_weight_max": 28.506990432739258,
      "activations/layer20_attention_weight_min": -26.650793075561523,
      "activations/layer21_attention_weight_max": 27.681997299194336,
      "activations/layer21_attention_weight_min": -25.042198181152344,
      "activations/layer22_attention_weight_max": 41.54929733276367,
      "activations/layer22_attention_weight_min": -31.45592498779297,
      "activations/layer23_attention_weight_max": 31.170265197753906,
      "activations/layer23_attention_weight_min": -25.961172103881836,
      "activations/layer2_attention_weight_max": 37.333858489990234,
      "activations/layer2_attention_weight_min": -36.015193939208984,
      "activations/layer3_attention_weight_max": 104.90264892578125,
      "activations/layer3_attention_weight_min": -100.25349426269531,
      "activations/layer4_attention_weight_max": 113.77337646484375,
      "activations/layer4_attention_weight_min": -109.99433898925781,
      "activations/layer5_attention_weight_max": 50.81291961669922,
      "activations/layer5_attention_weight_min": -59.06871795654297,
      "activations/layer6_attention_weight_max": 46.38507843017578,
      "activations/layer6_attention_weight_min": -45.02891159057617,
      "activations/layer7_attention_weight_max": 98.61997985839844,
      "activations/layer7_attention_weight_min": -93.62359619140625,
      "activations/layer8_attention_weight_max": 44.50477981567383,
      "activations/layer8_attention_weight_min": -46.340614318847656,
      "activations/layer9_attention_weight_max": 34.41294860839844,
      "activations/layer9_attention_weight_min": -37.55143356323242,
      "epoch": 17.47,
      "learning_rate": 3.7723106060606056e-05,
      "loss": 2.7404,
      "step": 300650
    },
    {
      "activations/layer0_attention_weight_max": 16.542308807373047,
      "activations/layer0_attention_weight_min": -13.000273704528809,
      "activations/layer10_attention_weight_max": 41.64365005493164,
      "activations/layer10_attention_weight_min": -39.814979553222656,
      "activations/layer11_attention_weight_max": 42.689422607421875,
      "activations/layer11_attention_weight_min": -40.68336486816406,
      "activations/layer12_attention_weight_max": 28.921161651611328,
      "activations/layer12_attention_weight_min": -27.193758010864258,
      "activations/layer13_attention_weight_max": 40.93574523925781,
      "activations/layer13_attention_weight_min": -37.00228500366211,
      "activations/layer14_attention_weight_max": 45.99964141845703,
      "activations/layer14_attention_weight_min": -42.377567291259766,
      "activations/layer15_attention_weight_max": 43.693138122558594,
      "activations/layer15_attention_weight_min": -39.583805084228516,
      "activations/layer16_attention_weight_max": 32.24169921875,
      "activations/layer16_attention_weight_min": -27.385343551635742,
      "activations/layer17_attention_weight_max": 32.62569046020508,
      "activations/layer17_attention_weight_min": -26.924055099487305,
      "activations/layer18_attention_weight_max": 31.055702209472656,
      "activations/layer18_attention_weight_min": -27.230815887451172,
      "activations/layer19_attention_weight_max": 37.1284065246582,
      "activations/layer19_attention_weight_min": -29.658126831054688,
      "activations/layer1_attention_weight_max": 16.06068229675293,
      "activations/layer1_attention_weight_min": -16.01504898071289,
      "activations/layer20_attention_weight_max": 30.506370544433594,
      "activations/layer20_attention_weight_min": -24.257164001464844,
      "activations/layer21_attention_weight_max": 27.427846908569336,
      "activations/layer21_attention_weight_min": -23.94591522216797,
      "activations/layer22_attention_weight_max": 47.24710464477539,
      "activations/layer22_attention_weight_min": -32.361045837402344,
      "activations/layer23_attention_weight_max": 31.314632415771484,
      "activations/layer23_attention_weight_min": -26.85909652709961,
      "activations/layer2_attention_weight_max": 34.94971466064453,
      "activations/layer2_attention_weight_min": -37.004276275634766,
      "activations/layer3_attention_weight_max": 106.56175231933594,
      "activations/layer3_attention_weight_min": -114.37699127197266,
      "activations/layer4_attention_weight_max": 110.69609832763672,
      "activations/layer4_attention_weight_min": -109.57347869873047,
      "activations/layer5_attention_weight_max": 53.3140869140625,
      "activations/layer5_attention_weight_min": -56.897369384765625,
      "activations/layer6_attention_weight_max": 46.30678176879883,
      "activations/layer6_attention_weight_min": -45.522708892822266,
      "activations/layer7_attention_weight_max": 109.10511779785156,
      "activations/layer7_attention_weight_min": -100.10173034667969,
      "activations/layer8_attention_weight_max": 48.43472671508789,
      "activations/layer8_attention_weight_min": -47.772422790527344,
      "activations/layer9_attention_weight_max": 36.77814865112305,
      "activations/layer9_attention_weight_min": -39.086692810058594,
      "epoch": 17.47,
      "learning_rate": 3.770416666666666e-05,
      "loss": 2.7377,
      "step": 300700
    },
    {
      "activations/layer0_attention_weight_max": 16.068628311157227,
      "activations/layer0_attention_weight_min": -13.076074600219727,
      "activations/layer10_attention_weight_max": 33.891876220703125,
      "activations/layer10_attention_weight_min": -32.03253173828125,
      "activations/layer11_attention_weight_max": 33.7760009765625,
      "activations/layer11_attention_weight_min": -32.525264739990234,
      "activations/layer12_attention_weight_max": 26.88827133178711,
      "activations/layer12_attention_weight_min": -26.4427433013916,
      "activations/layer13_attention_weight_max": 37.592559814453125,
      "activations/layer13_attention_weight_min": -35.70487976074219,
      "activations/layer14_attention_weight_max": 39.072357177734375,
      "activations/layer14_attention_weight_min": -38.3381233215332,
      "activations/layer15_attention_weight_max": 38.51774978637695,
      "activations/layer15_attention_weight_min": -35.835506439208984,
      "activations/layer16_attention_weight_max": 28.601736068725586,
      "activations/layer16_attention_weight_min": -28.171035766601562,
      "activations/layer17_attention_weight_max": 27.816499710083008,
      "activations/layer17_attention_weight_min": -24.1603946685791,
      "activations/layer18_attention_weight_max": 29.613779067993164,
      "activations/layer18_attention_weight_min": -26.304935455322266,
      "activations/layer19_attention_weight_max": 31.882123947143555,
      "activations/layer19_attention_weight_min": -32.68171310424805,
      "activations/layer1_attention_weight_max": 15.954440116882324,
      "activations/layer1_attention_weight_min": -14.65082836151123,
      "activations/layer20_attention_weight_max": 26.862836837768555,
      "activations/layer20_attention_weight_min": -25.341617584228516,
      "activations/layer21_attention_weight_max": 24.552305221557617,
      "activations/layer21_attention_weight_min": -24.363101959228516,
      "activations/layer22_attention_weight_max": 34.296875,
      "activations/layer22_attention_weight_min": -29.32636260986328,
      "activations/layer23_attention_weight_max": 27.489437103271484,
      "activations/layer23_attention_weight_min": -25.487213134765625,
      "activations/layer2_attention_weight_max": 33.860572814941406,
      "activations/layer2_attention_weight_min": -33.21656799316406,
      "activations/layer3_attention_weight_max": 98.48568725585938,
      "activations/layer3_attention_weight_min": -97.45320129394531,
      "activations/layer4_attention_weight_max": 105.64027404785156,
      "activations/layer4_attention_weight_min": -107.20082092285156,
      "activations/layer5_attention_weight_max": 49.01800537109375,
      "activations/layer5_attention_weight_min": -60.70743942260742,
      "activations/layer6_attention_weight_max": 43.84587860107422,
      "activations/layer6_attention_weight_min": -46.7722282409668,
      "activations/layer7_attention_weight_max": 96.68265533447266,
      "activations/layer7_attention_weight_min": -94.77352142333984,
      "activations/layer8_attention_weight_max": 42.73217010498047,
      "activations/layer8_attention_weight_min": -41.7811393737793,
      "activations/layer9_attention_weight_max": 33.6441535949707,
      "activations/layer9_attention_weight_min": -34.221412658691406,
      "epoch": 17.48,
      "learning_rate": 3.768522727272727e-05,
      "loss": 2.7273,
      "step": 300750
    },
    {
      "activations/layer0_attention_weight_max": 16.787982940673828,
      "activations/layer0_attention_weight_min": -12.70486831665039,
      "activations/layer10_attention_weight_max": 36.646610260009766,
      "activations/layer10_attention_weight_min": -35.99650955200195,
      "activations/layer11_attention_weight_max": 34.754150390625,
      "activations/layer11_attention_weight_min": -34.9913215637207,
      "activations/layer12_attention_weight_max": 27.796167373657227,
      "activations/layer12_attention_weight_min": -26.530176162719727,
      "activations/layer13_attention_weight_max": 38.81781005859375,
      "activations/layer13_attention_weight_min": -36.47557067871094,
      "activations/layer14_attention_weight_max": 41.58375930786133,
      "activations/layer14_attention_weight_min": -37.822906494140625,
      "activations/layer15_attention_weight_max": 43.21968078613281,
      "activations/layer15_attention_weight_min": -37.876644134521484,
      "activations/layer16_attention_weight_max": 28.53866195678711,
      "activations/layer16_attention_weight_min": -27.428266525268555,
      "activations/layer17_attention_weight_max": 30.8691349029541,
      "activations/layer17_attention_weight_min": -27.39080810546875,
      "activations/layer18_attention_weight_max": 30.997093200683594,
      "activations/layer18_attention_weight_min": -26.03170394897461,
      "activations/layer19_attention_weight_max": 32.91490173339844,
      "activations/layer19_attention_weight_min": -30.92374038696289,
      "activations/layer1_attention_weight_max": 16.62354278564453,
      "activations/layer1_attention_weight_min": -15.091641426086426,
      "activations/layer20_attention_weight_max": 31.32686996459961,
      "activations/layer20_attention_weight_min": -24.354671478271484,
      "activations/layer21_attention_weight_max": 29.825742721557617,
      "activations/layer21_attention_weight_min": -24.583934783935547,
      "activations/layer22_attention_weight_max": 39.68860626220703,
      "activations/layer22_attention_weight_min": -31.919830322265625,
      "activations/layer23_attention_weight_max": 30.774593353271484,
      "activations/layer23_attention_weight_min": -25.941570281982422,
      "activations/layer2_attention_weight_max": 33.55926513671875,
      "activations/layer2_attention_weight_min": -32.76435852050781,
      "activations/layer3_attention_weight_max": 100.66666412353516,
      "activations/layer3_attention_weight_min": -101.56404876708984,
      "activations/layer4_attention_weight_max": 106.52294921875,
      "activations/layer4_attention_weight_min": -106.3557357788086,
      "activations/layer5_attention_weight_max": 50.026451110839844,
      "activations/layer5_attention_weight_min": -59.74093246459961,
      "activations/layer6_attention_weight_max": 42.71229934692383,
      "activations/layer6_attention_weight_min": -46.23582458496094,
      "activations/layer7_attention_weight_max": 91.48011779785156,
      "activations/layer7_attention_weight_min": -95.7107162475586,
      "activations/layer8_attention_weight_max": 42.740440368652344,
      "activations/layer8_attention_weight_min": -45.19365310668945,
      "activations/layer9_attention_weight_max": 33.54850387573242,
      "activations/layer9_attention_weight_min": -35.698062896728516,
      "epoch": 17.48,
      "learning_rate": 3.7666287878787874e-05,
      "loss": 2.7268,
      "step": 300800
    },
    {
      "activations/layer0_attention_weight_max": 15.398343086242676,
      "activations/layer0_attention_weight_min": -12.393525123596191,
      "activations/layer10_attention_weight_max": 33.84627151489258,
      "activations/layer10_attention_weight_min": -34.81059265136719,
      "activations/layer11_attention_weight_max": 33.88078308105469,
      "activations/layer11_attention_weight_min": -33.40325927734375,
      "activations/layer12_attention_weight_max": 25.47266960144043,
      "activations/layer12_attention_weight_min": -26.497806549072266,
      "activations/layer13_attention_weight_max": 40.76283264160156,
      "activations/layer13_attention_weight_min": -36.45085906982422,
      "activations/layer14_attention_weight_max": 43.31203842163086,
      "activations/layer14_attention_weight_min": -39.41730499267578,
      "activations/layer15_attention_weight_max": 39.715187072753906,
      "activations/layer15_attention_weight_min": -35.799522399902344,
      "activations/layer16_attention_weight_max": 31.148958206176758,
      "activations/layer16_attention_weight_min": -27.89665985107422,
      "activations/layer17_attention_weight_max": 33.48028564453125,
      "activations/layer17_attention_weight_min": -27.426694869995117,
      "activations/layer18_attention_weight_max": 33.798370361328125,
      "activations/layer18_attention_weight_min": -25.956478118896484,
      "activations/layer19_attention_weight_max": 35.70133590698242,
      "activations/layer19_attention_weight_min": -36.230255126953125,
      "activations/layer1_attention_weight_max": 16.054166793823242,
      "activations/layer1_attention_weight_min": -14.29511833190918,
      "activations/layer20_attention_weight_max": 30.462738037109375,
      "activations/layer20_attention_weight_min": -28.9241943359375,
      "activations/layer21_attention_weight_max": 29.758665084838867,
      "activations/layer21_attention_weight_min": -28.496017456054688,
      "activations/layer22_attention_weight_max": 43.495567321777344,
      "activations/layer22_attention_weight_min": -34.120059967041016,
      "activations/layer23_attention_weight_max": 36.11030578613281,
      "activations/layer23_attention_weight_min": -27.254201889038086,
      "activations/layer2_attention_weight_max": 32.59158706665039,
      "activations/layer2_attention_weight_min": -32.476654052734375,
      "activations/layer3_attention_weight_max": 97.698486328125,
      "activations/layer3_attention_weight_min": -98.01439666748047,
      "activations/layer4_attention_weight_max": 103.81981658935547,
      "activations/layer4_attention_weight_min": -105.01932525634766,
      "activations/layer5_attention_weight_max": 47.606014251708984,
      "activations/layer5_attention_weight_min": -58.94889831542969,
      "activations/layer6_attention_weight_max": 43.54572296142578,
      "activations/layer6_attention_weight_min": -47.51460266113281,
      "activations/layer7_attention_weight_max": 93.10633850097656,
      "activations/layer7_attention_weight_min": -96.02523040771484,
      "activations/layer8_attention_weight_max": 41.88351058959961,
      "activations/layer8_attention_weight_min": -42.92544174194336,
      "activations/layer9_attention_weight_max": 32.47380065917969,
      "activations/layer9_attention_weight_min": -32.59813690185547,
      "epoch": 17.48,
      "learning_rate": 3.764734848484848e-05,
      "loss": 2.7611,
      "step": 300850
    },
    {
      "activations/layer0_attention_weight_max": 15.972865104675293,
      "activations/layer0_attention_weight_min": -12.699299812316895,
      "activations/layer10_attention_weight_max": 33.65030288696289,
      "activations/layer10_attention_weight_min": -34.929847717285156,
      "activations/layer11_attention_weight_max": 34.15194320678711,
      "activations/layer11_attention_weight_min": -35.05202102661133,
      "activations/layer12_attention_weight_max": 26.283288955688477,
      "activations/layer12_attention_weight_min": -27.599332809448242,
      "activations/layer13_attention_weight_max": 41.25299072265625,
      "activations/layer13_attention_weight_min": -35.0150146484375,
      "activations/layer14_attention_weight_max": 40.65410232543945,
      "activations/layer14_attention_weight_min": -36.73908996582031,
      "activations/layer15_attention_weight_max": 41.18598175048828,
      "activations/layer15_attention_weight_min": -34.52356719970703,
      "activations/layer16_attention_weight_max": 32.38373947143555,
      "activations/layer16_attention_weight_min": -27.787240982055664,
      "activations/layer17_attention_weight_max": 32.685543060302734,
      "activations/layer17_attention_weight_min": -26.304162979125977,
      "activations/layer18_attention_weight_max": 32.016845703125,
      "activations/layer18_attention_weight_min": -24.96002197265625,
      "activations/layer19_attention_weight_max": 34.75629425048828,
      "activations/layer19_attention_weight_min": -30.117517471313477,
      "activations/layer1_attention_weight_max": 16.521718978881836,
      "activations/layer1_attention_weight_min": -15.271293640136719,
      "activations/layer20_attention_weight_max": 27.780683517456055,
      "activations/layer20_attention_weight_min": -25.507062911987305,
      "activations/layer21_attention_weight_max": 26.935546875,
      "activations/layer21_attention_weight_min": -25.84943199157715,
      "activations/layer22_attention_weight_max": 40.62688064575195,
      "activations/layer22_attention_weight_min": -31.40778160095215,
      "activations/layer23_attention_weight_max": 33.9761962890625,
      "activations/layer23_attention_weight_min": -27.12804412841797,
      "activations/layer2_attention_weight_max": 32.96056365966797,
      "activations/layer2_attention_weight_min": -35.52113342285156,
      "activations/layer3_attention_weight_max": 99.97319030761719,
      "activations/layer3_attention_weight_min": -105.98816680908203,
      "activations/layer4_attention_weight_max": 102.94058990478516,
      "activations/layer4_attention_weight_min": -104.00787353515625,
      "activations/layer5_attention_weight_max": 47.42238998413086,
      "activations/layer5_attention_weight_min": -61.964439392089844,
      "activations/layer6_attention_weight_max": 41.98268127441406,
      "activations/layer6_attention_weight_min": -45.80691909790039,
      "activations/layer7_attention_weight_max": 87.3244400024414,
      "activations/layer7_attention_weight_min": -94.33582305908203,
      "activations/layer8_attention_weight_max": 40.716285705566406,
      "activations/layer8_attention_weight_min": -45.31262969970703,
      "activations/layer9_attention_weight_max": 30.034473419189453,
      "activations/layer9_attention_weight_min": -34.692840576171875,
      "epoch": 17.48,
      "learning_rate": 3.762840909090908e-05,
      "loss": 2.7239,
      "step": 300900
    },
    {
      "activations/layer0_attention_weight_max": 16.362470626831055,
      "activations/layer0_attention_weight_min": -13.009939193725586,
      "activations/layer10_attention_weight_max": 33.300048828125,
      "activations/layer10_attention_weight_min": -34.85097122192383,
      "activations/layer11_attention_weight_max": 33.26714324951172,
      "activations/layer11_attention_weight_min": -34.658790588378906,
      "activations/layer12_attention_weight_max": 26.04955291748047,
      "activations/layer12_attention_weight_min": -26.1088809967041,
      "activations/layer13_attention_weight_max": 39.0513801574707,
      "activations/layer13_attention_weight_min": -36.78242111206055,
      "activations/layer14_attention_weight_max": 41.22237777709961,
      "activations/layer14_attention_weight_min": -39.83268356323242,
      "activations/layer15_attention_weight_max": 37.12631607055664,
      "activations/layer15_attention_weight_min": -36.3885498046875,
      "activations/layer16_attention_weight_max": 28.6106014251709,
      "activations/layer16_attention_weight_min": -31.27179718017578,
      "activations/layer17_attention_weight_max": 29.62394142150879,
      "activations/layer17_attention_weight_min": -29.12351417541504,
      "activations/layer18_attention_weight_max": 33.43280792236328,
      "activations/layer18_attention_weight_min": -27.86823081970215,
      "activations/layer19_attention_weight_max": 33.790557861328125,
      "activations/layer19_attention_weight_min": -31.481521606445312,
      "activations/layer1_attention_weight_max": 16.56105613708496,
      "activations/layer1_attention_weight_min": -15.157727241516113,
      "activations/layer20_attention_weight_max": 29.244871139526367,
      "activations/layer20_attention_weight_min": -25.394546508789062,
      "activations/layer21_attention_weight_max": 28.186729431152344,
      "activations/layer21_attention_weight_min": -25.048734664916992,
      "activations/layer22_attention_weight_max": 45.41878128051758,
      "activations/layer22_attention_weight_min": -30.271087646484375,
      "activations/layer23_attention_weight_max": 34.95185089111328,
      "activations/layer23_attention_weight_min": -25.720558166503906,
      "activations/layer2_attention_weight_max": 34.87480545043945,
      "activations/layer2_attention_weight_min": -34.53453826904297,
      "activations/layer3_attention_weight_max": 101.50581359863281,
      "activations/layer3_attention_weight_min": -104.87842559814453,
      "activations/layer4_attention_weight_max": 108.30074310302734,
      "activations/layer4_attention_weight_min": -116.38056945800781,
      "activations/layer5_attention_weight_max": 51.313690185546875,
      "activations/layer5_attention_weight_min": -61.06852340698242,
      "activations/layer6_attention_weight_max": 45.78499221801758,
      "activations/layer6_attention_weight_min": -53.02565383911133,
      "activations/layer7_attention_weight_max": 94.65310668945312,
      "activations/layer7_attention_weight_min": -106.00289916992188,
      "activations/layer8_attention_weight_max": 42.34439468383789,
      "activations/layer8_attention_weight_min": -46.521759033203125,
      "activations/layer9_attention_weight_max": 32.758026123046875,
      "activations/layer9_attention_weight_min": -36.42734909057617,
      "epoch": 17.49,
      "learning_rate": 3.76094696969697e-05,
      "loss": 2.7361,
      "step": 300950
    },
    {
      "activations/layer0_attention_weight_max": 15.941070556640625,
      "activations/layer0_attention_weight_min": -12.217050552368164,
      "activations/layer10_attention_weight_max": 35.770347595214844,
      "activations/layer10_attention_weight_min": -37.64369583129883,
      "activations/layer11_attention_weight_max": 33.60009002685547,
      "activations/layer11_attention_weight_min": -38.37141418457031,
      "activations/layer12_attention_weight_max": 26.534133911132812,
      "activations/layer12_attention_weight_min": -27.558443069458008,
      "activations/layer13_attention_weight_max": 40.52247619628906,
      "activations/layer13_attention_weight_min": -35.8631706237793,
      "activations/layer14_attention_weight_max": 43.24852752685547,
      "activations/layer14_attention_weight_min": -40.3958854675293,
      "activations/layer15_attention_weight_max": 38.22489929199219,
      "activations/layer15_attention_weight_min": -36.12220764160156,
      "activations/layer16_attention_weight_max": 29.690515518188477,
      "activations/layer16_attention_weight_min": -27.188907623291016,
      "activations/layer17_attention_weight_max": 30.300561904907227,
      "activations/layer17_attention_weight_min": -26.15509796142578,
      "activations/layer18_attention_weight_max": 33.61574172973633,
      "activations/layer18_attention_weight_min": -26.84341049194336,
      "activations/layer19_attention_weight_max": 34.70376968383789,
      "activations/layer19_attention_weight_min": -31.43267822265625,
      "activations/layer1_attention_weight_max": 16.419492721557617,
      "activations/layer1_attention_weight_min": -14.902316093444824,
      "activations/layer20_attention_weight_max": 29.028446197509766,
      "activations/layer20_attention_weight_min": -25.604984283447266,
      "activations/layer21_attention_weight_max": 27.510465621948242,
      "activations/layer21_attention_weight_min": -23.965904235839844,
      "activations/layer22_attention_weight_max": 47.7862548828125,
      "activations/layer22_attention_weight_min": -32.728172302246094,
      "activations/layer23_attention_weight_max": 31.732463836669922,
      "activations/layer23_attention_weight_min": -26.874568939208984,
      "activations/layer2_attention_weight_max": 32.87158203125,
      "activations/layer2_attention_weight_min": -34.271385192871094,
      "activations/layer3_attention_weight_max": 101.59588623046875,
      "activations/layer3_attention_weight_min": -105.51759338378906,
      "activations/layer4_attention_weight_max": 109.6148910522461,
      "activations/layer4_attention_weight_min": -113.5497055053711,
      "activations/layer5_attention_weight_max": 54.69023895263672,
      "activations/layer5_attention_weight_min": -63.24449920654297,
      "activations/layer6_attention_weight_max": 46.33513641357422,
      "activations/layer6_attention_weight_min": -48.82020950317383,
      "activations/layer7_attention_weight_max": 104.59371185302734,
      "activations/layer7_attention_weight_min": -95.76876831054688,
      "activations/layer8_attention_weight_max": 44.60215759277344,
      "activations/layer8_attention_weight_min": -46.92369842529297,
      "activations/layer9_attention_weight_max": 32.1363639831543,
      "activations/layer9_attention_weight_min": -35.58828353881836,
      "epoch": 17.49,
      "learning_rate": 3.75905303030303e-05,
      "loss": 2.7443,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.468,
      "eval_samples_per_second": 507.088,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.468,
      "eval_openwebtext_samples_per_second": 507.088,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_wikitext_loss": 2.92578125,
      "eval_wikitext_ppl": 18.648789727196185,
      "eval_wikitext_runtime": 1.9569,
      "eval_wikitext_samples_per_second": 233.023,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_lambada_loss": 2.419921875,
      "eval_lambada_ppl": 11.244980766441616,
      "eval_lambada_runtime": 9.5229,
      "eval_lambada_samples_per_second": 511.293,
      "step": 301000
    },
    {
      "activations/layer0_attention_weight_max": 16.05903434753418,
      "activations/layer0_attention_weight_min": -13.440963745117188,
      "activations/layer10_attention_weight_max": 32.499305725097656,
      "activations/layer10_attention_weight_min": -32.00160217285156,
      "activations/layer11_attention_weight_max": 32.87662887573242,
      "activations/layer11_attention_weight_min": -31.687450408935547,
      "activations/layer12_attention_weight_max": 28.340003967285156,
      "activations/layer12_attention_weight_min": -26.21978187561035,
      "activations/layer13_attention_weight_max": 40.56188201904297,
      "activations/layer13_attention_weight_min": -35.792545318603516,
      "activations/layer14_attention_weight_max": 43.232486724853516,
      "activations/layer14_attention_weight_min": -36.41475296020508,
      "activations/layer15_attention_weight_max": 37.77885818481445,
      "activations/layer15_attention_weight_min": -35.364994049072266,
      "activations/layer16_attention_weight_max": 28.842260360717773,
      "activations/layer16_attention_weight_min": -28.496685028076172,
      "activations/layer17_attention_weight_max": 29.815555572509766,
      "activations/layer17_attention_weight_min": -26.635080337524414,
      "activations/layer18_attention_weight_max": 35.71617126464844,
      "activations/layer18_attention_weight_min": -27.219572067260742,
      "activations/layer19_attention_weight_max": 31.58282470703125,
      "activations/layer19_attention_weight_min": -35.32114028930664,
      "activations/layer1_attention_weight_max": 15.689285278320312,
      "activations/layer1_attention_weight_min": -14.660958290100098,
      "activations/layer20_attention_weight_max": 26.950233459472656,
      "activations/layer20_attention_weight_min": -25.880083084106445,
      "activations/layer21_attention_weight_max": 26.990779876708984,
      "activations/layer21_attention_weight_min": -24.617380142211914,
      "activations/layer22_attention_weight_max": 45.107810974121094,
      "activations/layer22_attention_weight_min": -31.83570098876953,
      "activations/layer23_attention_weight_max": 29.450870513916016,
      "activations/layer23_attention_weight_min": -28.10402488708496,
      "activations/layer2_attention_weight_max": 34.59519958496094,
      "activations/layer2_attention_weight_min": -32.729248046875,
      "activations/layer3_attention_weight_max": 102.08970642089844,
      "activations/layer3_attention_weight_min": -95.98408508300781,
      "activations/layer4_attention_weight_max": 104.654541015625,
      "activations/layer4_attention_weight_min": -107.87480926513672,
      "activations/layer5_attention_weight_max": 51.65720748901367,
      "activations/layer5_attention_weight_min": -60.43479919433594,
      "activations/layer6_attention_weight_max": 44.29123306274414,
      "activations/layer6_attention_weight_min": -44.95456314086914,
      "activations/layer7_attention_weight_max": 89.51519012451172,
      "activations/layer7_attention_weight_min": -92.45738220214844,
      "activations/layer8_attention_weight_max": 38.938961029052734,
      "activations/layer8_attention_weight_min": -41.2039909362793,
      "activations/layer9_attention_weight_max": 32.44887924194336,
      "activations/layer9_attention_weight_min": -32.2724723815918,
      "epoch": 17.49,
      "learning_rate": 3.75715909090909e-05,
      "loss": 2.7258,
      "step": 301050
    },
    {
      "activations/layer0_attention_weight_max": 16.419719696044922,
      "activations/layer0_attention_weight_min": -13.343391418457031,
      "activations/layer10_attention_weight_max": 35.77305221557617,
      "activations/layer10_attention_weight_min": -37.20612716674805,
      "activations/layer11_attention_weight_max": 37.91394805908203,
      "activations/layer11_attention_weight_min": -36.35261917114258,
      "activations/layer12_attention_weight_max": 29.027423858642578,
      "activations/layer12_attention_weight_min": -27.891284942626953,
      "activations/layer13_attention_weight_max": 45.46521759033203,
      "activations/layer13_attention_weight_min": -40.063323974609375,
      "activations/layer14_attention_weight_max": 51.776851654052734,
      "activations/layer14_attention_weight_min": -44.69233703613281,
      "activations/layer15_attention_weight_max": 43.60057830810547,
      "activations/layer15_attention_weight_min": -39.471675872802734,
      "activations/layer16_attention_weight_max": 36.112613677978516,
      "activations/layer16_attention_weight_min": -30.148710250854492,
      "activations/layer17_attention_weight_max": 34.63212966918945,
      "activations/layer17_attention_weight_min": -28.023147583007812,
      "activations/layer18_attention_weight_max": 33.45465850830078,
      "activations/layer18_attention_weight_min": -27.18760871887207,
      "activations/layer19_attention_weight_max": 40.941650390625,
      "activations/layer19_attention_weight_min": -31.49567985534668,
      "activations/layer1_attention_weight_max": 15.733564376831055,
      "activations/layer1_attention_weight_min": -13.450139999389648,
      "activations/layer20_attention_weight_max": 32.40367889404297,
      "activations/layer20_attention_weight_min": -25.613779067993164,
      "activations/layer21_attention_weight_max": 32.12453842163086,
      "activations/layer21_attention_weight_min": -25.689559936523438,
      "activations/layer22_attention_weight_max": 50.05071258544922,
      "activations/layer22_attention_weight_min": -33.60065841674805,
      "activations/layer23_attention_weight_max": 40.129119873046875,
      "activations/layer23_attention_weight_min": -27.234012603759766,
      "activations/layer2_attention_weight_max": 34.29938888549805,
      "activations/layer2_attention_weight_min": -33.75418472290039,
      "activations/layer3_attention_weight_max": 95.42859649658203,
      "activations/layer3_attention_weight_min": -97.37998962402344,
      "activations/layer4_attention_weight_max": 106.79296112060547,
      "activations/layer4_attention_weight_min": -105.46028900146484,
      "activations/layer5_attention_weight_max": 50.90796661376953,
      "activations/layer5_attention_weight_min": -67.22659301757812,
      "activations/layer6_attention_weight_max": 44.95811462402344,
      "activations/layer6_attention_weight_min": -47.74622344970703,
      "activations/layer7_attention_weight_max": 94.10881042480469,
      "activations/layer7_attention_weight_min": -107.21315002441406,
      "activations/layer8_attention_weight_max": 41.958229064941406,
      "activations/layer8_attention_weight_min": -48.2778434753418,
      "activations/layer9_attention_weight_max": 33.30293655395508,
      "activations/layer9_attention_weight_min": -36.8324089050293,
      "epoch": 17.5,
      "learning_rate": 3.755265151515151e-05,
      "loss": 2.7321,
      "step": 301100
    },
    {
      "activations/layer0_attention_weight_max": 16.022518157958984,
      "activations/layer0_attention_weight_min": -12.615184783935547,
      "activations/layer10_attention_weight_max": 36.24066925048828,
      "activations/layer10_attention_weight_min": -33.97129821777344,
      "activations/layer11_attention_weight_max": 34.71963882446289,
      "activations/layer11_attention_weight_min": -33.81753158569336,
      "activations/layer12_attention_weight_max": 27.22085189819336,
      "activations/layer12_attention_weight_min": -26.556230545043945,
      "activations/layer13_attention_weight_max": 37.94982147216797,
      "activations/layer13_attention_weight_min": -34.44451141357422,
      "activations/layer14_attention_weight_max": 40.188079833984375,
      "activations/layer14_attention_weight_min": -37.139564514160156,
      "activations/layer15_attention_weight_max": 37.05271911621094,
      "activations/layer15_attention_weight_min": -34.123722076416016,
      "activations/layer16_attention_weight_max": 27.162899017333984,
      "activations/layer16_attention_weight_min": -26.552757263183594,
      "activations/layer17_attention_weight_max": 28.6561279296875,
      "activations/layer17_attention_weight_min": -26.767114639282227,
      "activations/layer18_attention_weight_max": 31.832782745361328,
      "activations/layer18_attention_weight_min": -25.473772048950195,
      "activations/layer19_attention_weight_max": 30.592166900634766,
      "activations/layer19_attention_weight_min": -33.293514251708984,
      "activations/layer1_attention_weight_max": 16.747936248779297,
      "activations/layer1_attention_weight_min": -14.845808982849121,
      "activations/layer20_attention_weight_max": 26.32965087890625,
      "activations/layer20_attention_weight_min": -26.763948440551758,
      "activations/layer21_attention_weight_max": 27.858028411865234,
      "activations/layer21_attention_weight_min": -26.86125373840332,
      "activations/layer22_attention_weight_max": 36.565834045410156,
      "activations/layer22_attention_weight_min": -32.88425064086914,
      "activations/layer23_attention_weight_max": 29.772581100463867,
      "activations/layer23_attention_weight_min": -26.811017990112305,
      "activations/layer2_attention_weight_max": 31.091978073120117,
      "activations/layer2_attention_weight_min": -32.36373519897461,
      "activations/layer3_attention_weight_max": 92.17572021484375,
      "activations/layer3_attention_weight_min": -96.96923828125,
      "activations/layer4_attention_weight_max": 99.84253692626953,
      "activations/layer4_attention_weight_min": -102.64000701904297,
      "activations/layer5_attention_weight_max": 52.07328796386719,
      "activations/layer5_attention_weight_min": -55.777034759521484,
      "activations/layer6_attention_weight_max": 40.59815979003906,
      "activations/layer6_attention_weight_min": -44.09962463378906,
      "activations/layer7_attention_weight_max": 91.76740264892578,
      "activations/layer7_attention_weight_min": -88.2005386352539,
      "activations/layer8_attention_weight_max": 39.668975830078125,
      "activations/layer8_attention_weight_min": -43.29267120361328,
      "activations/layer9_attention_weight_max": 33.54311752319336,
      "activations/layer9_attention_weight_min": -37.46161651611328,
      "epoch": 17.5,
      "learning_rate": 3.753371212121212e-05,
      "loss": 2.7415,
      "step": 301150
    },
    {
      "activations/layer0_attention_weight_max": 17.844579696655273,
      "activations/layer0_attention_weight_min": -12.620136260986328,
      "activations/layer10_attention_weight_max": 33.3992919921875,
      "activations/layer10_attention_weight_min": -34.052547454833984,
      "activations/layer11_attention_weight_max": 32.567893981933594,
      "activations/layer11_attention_weight_min": -33.12025451660156,
      "activations/layer12_attention_weight_max": 25.860000610351562,
      "activations/layer12_attention_weight_min": -26.001792907714844,
      "activations/layer13_attention_weight_max": 38.739463806152344,
      "activations/layer13_attention_weight_min": -34.8569221496582,
      "activations/layer14_attention_weight_max": 40.11582946777344,
      "activations/layer14_attention_weight_min": -37.41488265991211,
      "activations/layer15_attention_weight_max": 37.654869079589844,
      "activations/layer15_attention_weight_min": -35.118560791015625,
      "activations/layer16_attention_weight_max": 27.150375366210938,
      "activations/layer16_attention_weight_min": -28.256893157958984,
      "activations/layer17_attention_weight_max": 26.972787857055664,
      "activations/layer17_attention_weight_min": -25.895883560180664,
      "activations/layer18_attention_weight_max": 29.900127410888672,
      "activations/layer18_attention_weight_min": -25.80117416381836,
      "activations/layer19_attention_weight_max": 33.23073196411133,
      "activations/layer19_attention_weight_min": -29.474979400634766,
      "activations/layer1_attention_weight_max": 15.985405921936035,
      "activations/layer1_attention_weight_min": -13.3114013671875,
      "activations/layer20_attention_weight_max": 28.59904670715332,
      "activations/layer20_attention_weight_min": -24.156314849853516,
      "activations/layer21_attention_weight_max": 28.385433197021484,
      "activations/layer21_attention_weight_min": -23.84593963623047,
      "activations/layer22_attention_weight_max": 40.2611083984375,
      "activations/layer22_attention_weight_min": -30.30410385131836,
      "activations/layer23_attention_weight_max": 30.37184715270996,
      "activations/layer23_attention_weight_min": -24.94280242919922,
      "activations/layer2_attention_weight_max": 32.77262496948242,
      "activations/layer2_attention_weight_min": -35.252113342285156,
      "activations/layer3_attention_weight_max": 99.99878692626953,
      "activations/layer3_attention_weight_min": -106.26256561279297,
      "activations/layer4_attention_weight_max": 106.75701141357422,
      "activations/layer4_attention_weight_min": -115.59847259521484,
      "activations/layer5_attention_weight_max": 48.9596061706543,
      "activations/layer5_attention_weight_min": -59.404296875,
      "activations/layer6_attention_weight_max": 44.73466491699219,
      "activations/layer6_attention_weight_min": -46.11698532104492,
      "activations/layer7_attention_weight_max": 90.18010711669922,
      "activations/layer7_attention_weight_min": -98.05841827392578,
      "activations/layer8_attention_weight_max": 40.0493278503418,
      "activations/layer8_attention_weight_min": -46.056297302246094,
      "activations/layer9_attention_weight_max": 31.034818649291992,
      "activations/layer9_attention_weight_min": -33.51322555541992,
      "epoch": 17.5,
      "learning_rate": 3.7514772727272726e-05,
      "loss": 2.737,
      "step": 301200
    },
    {
      "activations/layer0_attention_weight_max": 16.35057258605957,
      "activations/layer0_attention_weight_min": -13.494156837463379,
      "activations/layer10_attention_weight_max": 36.63957214355469,
      "activations/layer10_attention_weight_min": -35.227821350097656,
      "activations/layer11_attention_weight_max": 35.936302185058594,
      "activations/layer11_attention_weight_min": -34.52745819091797,
      "activations/layer12_attention_weight_max": 27.36811637878418,
      "activations/layer12_attention_weight_min": -27.853425979614258,
      "activations/layer13_attention_weight_max": 39.45103454589844,
      "activations/layer13_attention_weight_min": -37.45758056640625,
      "activations/layer14_attention_weight_max": 42.339027404785156,
      "activations/layer14_attention_weight_min": -40.514671325683594,
      "activations/layer15_attention_weight_max": 38.533905029296875,
      "activations/layer15_attention_weight_min": -37.655662536621094,
      "activations/layer16_attention_weight_max": 28.945680618286133,
      "activations/layer16_attention_weight_min": -28.408727645874023,
      "activations/layer17_attention_weight_max": 30.526777267456055,
      "activations/layer17_attention_weight_min": -26.1987361907959,
      "activations/layer18_attention_weight_max": 32.12751388549805,
      "activations/layer18_attention_weight_min": -26.784805297851562,
      "activations/layer19_attention_weight_max": 32.98020553588867,
      "activations/layer19_attention_weight_min": -31.8687686920166,
      "activations/layer1_attention_weight_max": 15.899690628051758,
      "activations/layer1_attention_weight_min": -13.844630241394043,
      "activations/layer20_attention_weight_max": 26.01409149169922,
      "activations/layer20_attention_weight_min": -26.352365493774414,
      "activations/layer21_attention_weight_max": 26.910417556762695,
      "activations/layer21_attention_weight_min": -25.070140838623047,
      "activations/layer22_attention_weight_max": 46.709468841552734,
      "activations/layer22_attention_weight_min": -34.1268196105957,
      "activations/layer23_attention_weight_max": 31.760147094726562,
      "activations/layer23_attention_weight_min": -25.70050811767578,
      "activations/layer2_attention_weight_max": 33.25896453857422,
      "activations/layer2_attention_weight_min": -35.14607238769531,
      "activations/layer3_attention_weight_max": 98.84881591796875,
      "activations/layer3_attention_weight_min": -99.84654235839844,
      "activations/layer4_attention_weight_max": 108.98931121826172,
      "activations/layer4_attention_weight_min": -108.26244354248047,
      "activations/layer5_attention_weight_max": 49.49601364135742,
      "activations/layer5_attention_weight_min": -62.06072998046875,
      "activations/layer6_attention_weight_max": 45.01504135131836,
      "activations/layer6_attention_weight_min": -48.20771026611328,
      "activations/layer7_attention_weight_max": 91.58212280273438,
      "activations/layer7_attention_weight_min": -102.17115783691406,
      "activations/layer8_attention_weight_max": 41.97514724731445,
      "activations/layer8_attention_weight_min": -44.79341506958008,
      "activations/layer9_attention_weight_max": 34.25614929199219,
      "activations/layer9_attention_weight_min": -33.439117431640625,
      "epoch": 17.5,
      "learning_rate": 3.749621212121212e-05,
      "loss": 2.7357,
      "step": 301250
    },
    {
      "activations/layer0_attention_weight_max": 15.855740547180176,
      "activations/layer0_attention_weight_min": -12.230008125305176,
      "activations/layer10_attention_weight_max": 32.29784393310547,
      "activations/layer10_attention_weight_min": -32.64790344238281,
      "activations/layer11_attention_weight_max": 32.075050354003906,
      "activations/layer11_attention_weight_min": -33.66498565673828,
      "activations/layer12_attention_weight_max": 23.021591186523438,
      "activations/layer12_attention_weight_min": -25.39011573791504,
      "activations/layer13_attention_weight_max": 35.434967041015625,
      "activations/layer13_attention_weight_min": -38.46916580200195,
      "activations/layer14_attention_weight_max": 40.820430755615234,
      "activations/layer14_attention_weight_min": -43.72145462036133,
      "activations/layer15_attention_weight_max": 39.56733703613281,
      "activations/layer15_attention_weight_min": -37.57121276855469,
      "activations/layer16_attention_weight_max": 28.294490814208984,
      "activations/layer16_attention_weight_min": -30.096534729003906,
      "activations/layer17_attention_weight_max": 27.622222900390625,
      "activations/layer17_attention_weight_min": -25.090896606445312,
      "activations/layer18_attention_weight_max": 32.12965774536133,
      "activations/layer18_attention_weight_min": -25.609603881835938,
      "activations/layer19_attention_weight_max": 33.0025634765625,
      "activations/layer19_attention_weight_min": -31.240957260131836,
      "activations/layer1_attention_weight_max": 16.241586685180664,
      "activations/layer1_attention_weight_min": -13.731922149658203,
      "activations/layer20_attention_weight_max": 31.452306747436523,
      "activations/layer20_attention_weight_min": -24.759992599487305,
      "activations/layer21_attention_weight_max": 28.30016326904297,
      "activations/layer21_attention_weight_min": -25.537670135498047,
      "activations/layer22_attention_weight_max": 39.56275939941406,
      "activations/layer22_attention_weight_min": -31.671567916870117,
      "activations/layer23_attention_weight_max": 34.0258903503418,
      "activations/layer23_attention_weight_min": -27.54515838623047,
      "activations/layer2_attention_weight_max": 33.25013732910156,
      "activations/layer2_attention_weight_min": -35.843467712402344,
      "activations/layer3_attention_weight_max": 98.7629165649414,
      "activations/layer3_attention_weight_min": -100.896240234375,
      "activations/layer4_attention_weight_max": 104.06996154785156,
      "activations/layer4_attention_weight_min": -104.5042495727539,
      "activations/layer5_attention_weight_max": 46.59125900268555,
      "activations/layer5_attention_weight_min": -56.394073486328125,
      "activations/layer6_attention_weight_max": 40.821006774902344,
      "activations/layer6_attention_weight_min": -44.63389205932617,
      "activations/layer7_attention_weight_max": 84.28779602050781,
      "activations/layer7_attention_weight_min": -92.25981903076172,
      "activations/layer8_attention_weight_max": 38.813358306884766,
      "activations/layer8_attention_weight_min": -43.08241653442383,
      "activations/layer9_attention_weight_max": 29.960193634033203,
      "activations/layer9_attention_weight_min": -32.44379425048828,
      "epoch": 17.51,
      "learning_rate": 3.7477272727272725e-05,
      "loss": 2.7353,
      "step": 301300
    },
    {
      "activations/layer0_attention_weight_max": 16.13427734375,
      "activations/layer0_attention_weight_min": -14.029096603393555,
      "activations/layer10_attention_weight_max": 34.891456604003906,
      "activations/layer10_attention_weight_min": -36.260009765625,
      "activations/layer11_attention_weight_max": 34.5135612487793,
      "activations/layer11_attention_weight_min": -38.4109001159668,
      "activations/layer12_attention_weight_max": 27.580909729003906,
      "activations/layer12_attention_weight_min": -28.092079162597656,
      "activations/layer13_attention_weight_max": 39.6633186340332,
      "activations/layer13_attention_weight_min": -36.7837028503418,
      "activations/layer14_attention_weight_max": 41.61408996582031,
      "activations/layer14_attention_weight_min": -40.11344909667969,
      "activations/layer15_attention_weight_max": 36.63988494873047,
      "activations/layer15_attention_weight_min": -35.403751373291016,
      "activations/layer16_attention_weight_max": 30.00359344482422,
      "activations/layer16_attention_weight_min": -27.670249938964844,
      "activations/layer17_attention_weight_max": 27.930723190307617,
      "activations/layer17_attention_weight_min": -25.437381744384766,
      "activations/layer18_attention_weight_max": 31.92375373840332,
      "activations/layer18_attention_weight_min": -26.25127410888672,
      "activations/layer19_attention_weight_max": 33.3402214050293,
      "activations/layer19_attention_weight_min": -31.84234046936035,
      "activations/layer1_attention_weight_max": 17.46478271484375,
      "activations/layer1_attention_weight_min": -14.423857688903809,
      "activations/layer20_attention_weight_max": 27.54543685913086,
      "activations/layer20_attention_weight_min": -25.334632873535156,
      "activations/layer21_attention_weight_max": 26.791263580322266,
      "activations/layer21_attention_weight_min": -26.050548553466797,
      "activations/layer22_attention_weight_max": 40.648109436035156,
      "activations/layer22_attention_weight_min": -33.209049224853516,
      "activations/layer23_attention_weight_max": 32.16405487060547,
      "activations/layer23_attention_weight_min": -27.251760482788086,
      "activations/layer2_attention_weight_max": 33.31636047363281,
      "activations/layer2_attention_weight_min": -33.017189025878906,
      "activations/layer3_attention_weight_max": 96.6583480834961,
      "activations/layer3_attention_weight_min": -99.23959350585938,
      "activations/layer4_attention_weight_max": 105.5066146850586,
      "activations/layer4_attention_weight_min": -109.79479217529297,
      "activations/layer5_attention_weight_max": 49.46079635620117,
      "activations/layer5_attention_weight_min": -60.5532341003418,
      "activations/layer6_attention_weight_max": 43.43645477294922,
      "activations/layer6_attention_weight_min": -45.6936149597168,
      "activations/layer7_attention_weight_max": 89.53707885742188,
      "activations/layer7_attention_weight_min": -91.30049896240234,
      "activations/layer8_attention_weight_max": 40.55928039550781,
      "activations/layer8_attention_weight_min": -43.04518127441406,
      "activations/layer9_attention_weight_max": 33.05715560913086,
      "activations/layer9_attention_weight_min": -34.74200439453125,
      "epoch": 17.51,
      "learning_rate": 3.7458333333333334e-05,
      "loss": 2.7331,
      "step": 301350
    },
    {
      "activations/layer0_attention_weight_max": 16.094038009643555,
      "activations/layer0_attention_weight_min": -12.209349632263184,
      "activations/layer10_attention_weight_max": 37.39029312133789,
      "activations/layer10_attention_weight_min": -37.41635513305664,
      "activations/layer11_attention_weight_max": 35.999359130859375,
      "activations/layer11_attention_weight_min": -35.190914154052734,
      "activations/layer12_attention_weight_max": 25.98655891418457,
      "activations/layer12_attention_weight_min": -26.029903411865234,
      "activations/layer13_attention_weight_max": 37.06345748901367,
      "activations/layer13_attention_weight_min": -37.935462951660156,
      "activations/layer14_attention_weight_max": 38.96446228027344,
      "activations/layer14_attention_weight_min": -40.2048454284668,
      "activations/layer15_attention_weight_max": 35.80044174194336,
      "activations/layer15_attention_weight_min": -35.76018524169922,
      "activations/layer16_attention_weight_max": 27.728242874145508,
      "activations/layer16_attention_weight_min": -28.199140548706055,
      "activations/layer17_attention_weight_max": 28.210514068603516,
      "activations/layer17_attention_weight_min": -26.28415298461914,
      "activations/layer18_attention_weight_max": 30.16897201538086,
      "activations/layer18_attention_weight_min": -26.031200408935547,
      "activations/layer19_attention_weight_max": 35.16687774658203,
      "activations/layer19_attention_weight_min": -34.1085090637207,
      "activations/layer1_attention_weight_max": 17.756242752075195,
      "activations/layer1_attention_weight_min": -14.261878967285156,
      "activations/layer20_attention_weight_max": 28.146900177001953,
      "activations/layer20_attention_weight_min": -25.607505798339844,
      "activations/layer21_attention_weight_max": 27.583465576171875,
      "activations/layer21_attention_weight_min": -26.24979591369629,
      "activations/layer22_attention_weight_max": 46.144325256347656,
      "activations/layer22_attention_weight_min": -33.309688568115234,
      "activations/layer23_attention_weight_max": 38.2507209777832,
      "activations/layer23_attention_weight_min": -29.60126304626465,
      "activations/layer2_attention_weight_max": 32.89119338989258,
      "activations/layer2_attention_weight_min": -34.13200378417969,
      "activations/layer3_attention_weight_max": 98.46366882324219,
      "activations/layer3_attention_weight_min": -97.38091278076172,
      "activations/layer4_attention_weight_max": 103.61231994628906,
      "activations/layer4_attention_weight_min": -107.49079895019531,
      "activations/layer5_attention_weight_max": 50.083396911621094,
      "activations/layer5_attention_weight_min": -58.97107696533203,
      "activations/layer6_attention_weight_max": 43.93346405029297,
      "activations/layer6_attention_weight_min": -48.54235076904297,
      "activations/layer7_attention_weight_max": 88.93220520019531,
      "activations/layer7_attention_weight_min": -99.25994110107422,
      "activations/layer8_attention_weight_max": 41.26634979248047,
      "activations/layer8_attention_weight_min": -42.33589553833008,
      "activations/layer9_attention_weight_max": 32.442726135253906,
      "activations/layer9_attention_weight_min": -33.86776351928711,
      "epoch": 17.51,
      "learning_rate": 3.7439393939393935e-05,
      "loss": 2.7323,
      "step": 301400
    },
    {
      "activations/layer0_attention_weight_max": 15.620163917541504,
      "activations/layer0_attention_weight_min": -12.943531036376953,
      "activations/layer10_attention_weight_max": 35.028472900390625,
      "activations/layer10_attention_weight_min": -35.81940460205078,
      "activations/layer11_attention_weight_max": 35.767295837402344,
      "activations/layer11_attention_weight_min": -36.90639877319336,
      "activations/layer12_attention_weight_max": 26.775447845458984,
      "activations/layer12_attention_weight_min": -27.935317993164062,
      "activations/layer13_attention_weight_max": 40.38411331176758,
      "activations/layer13_attention_weight_min": -39.900203704833984,
      "activations/layer14_attention_weight_max": 44.35577392578125,
      "activations/layer14_attention_weight_min": -39.72572326660156,
      "activations/layer15_attention_weight_max": 39.8731803894043,
      "activations/layer15_attention_weight_min": -35.14304733276367,
      "activations/layer16_attention_weight_max": 29.874469757080078,
      "activations/layer16_attention_weight_min": -28.279809951782227,
      "activations/layer17_attention_weight_max": 32.48030471801758,
      "activations/layer17_attention_weight_min": -26.21485710144043,
      "activations/layer18_attention_weight_max": 33.89444351196289,
      "activations/layer18_attention_weight_min": -25.18107795715332,
      "activations/layer19_attention_weight_max": 36.34595489501953,
      "activations/layer19_attention_weight_min": -30.737003326416016,
      "activations/layer1_attention_weight_max": 17.967153549194336,
      "activations/layer1_attention_weight_min": -18.074953079223633,
      "activations/layer20_attention_weight_max": 31.02028465270996,
      "activations/layer20_attention_weight_min": -25.444480895996094,
      "activations/layer21_attention_weight_max": 30.74228286743164,
      "activations/layer21_attention_weight_min": -25.26580238342285,
      "activations/layer22_attention_weight_max": 45.92102813720703,
      "activations/layer22_attention_weight_min": -35.266204833984375,
      "activations/layer23_attention_weight_max": 33.98912811279297,
      "activations/layer23_attention_weight_min": -25.240848541259766,
      "activations/layer2_attention_weight_max": 36.94390869140625,
      "activations/layer2_attention_weight_min": -36.219207763671875,
      "activations/layer3_attention_weight_max": 107.02947235107422,
      "activations/layer3_attention_weight_min": -107.27989959716797,
      "activations/layer4_attention_weight_max": 117.5127944946289,
      "activations/layer4_attention_weight_min": -115.01185607910156,
      "activations/layer5_attention_weight_max": 54.555076599121094,
      "activations/layer5_attention_weight_min": -69.74662780761719,
      "activations/layer6_attention_weight_max": 46.060218811035156,
      "activations/layer6_attention_weight_min": -51.03730773925781,
      "activations/layer7_attention_weight_max": 97.87413024902344,
      "activations/layer7_attention_weight_min": -98.73878479003906,
      "activations/layer8_attention_weight_max": 43.72787857055664,
      "activations/layer8_attention_weight_min": -45.2750358581543,
      "activations/layer9_attention_weight_max": 38.638038635253906,
      "activations/layer9_attention_weight_min": -36.45743942260742,
      "epoch": 17.52,
      "learning_rate": 3.742045454545454e-05,
      "loss": 2.7375,
      "step": 301450
    },
    {
      "activations/layer0_attention_weight_max": 16.761098861694336,
      "activations/layer0_attention_weight_min": -12.107820510864258,
      "activations/layer10_attention_weight_max": 32.56208038330078,
      "activations/layer10_attention_weight_min": -32.97749710083008,
      "activations/layer11_attention_weight_max": 31.942710876464844,
      "activations/layer11_attention_weight_min": -32.20594024658203,
      "activations/layer12_attention_weight_max": 24.361923217773438,
      "activations/layer12_attention_weight_min": -25.84579849243164,
      "activations/layer13_attention_weight_max": 36.85240173339844,
      "activations/layer13_attention_weight_min": -34.98555374145508,
      "activations/layer14_attention_weight_max": 39.360408782958984,
      "activations/layer14_attention_weight_min": -37.2170295715332,
      "activations/layer15_attention_weight_max": 36.443603515625,
      "activations/layer15_attention_weight_min": -34.24202346801758,
      "activations/layer16_attention_weight_max": 29.130895614624023,
      "activations/layer16_attention_weight_min": -26.635644912719727,
      "activations/layer17_attention_weight_max": 27.73881721496582,
      "activations/layer17_attention_weight_min": -25.236780166625977,
      "activations/layer18_attention_weight_max": 30.0092716217041,
      "activations/layer18_attention_weight_min": -25.106779098510742,
      "activations/layer19_attention_weight_max": 31.215076446533203,
      "activations/layer19_attention_weight_min": -29.35861587524414,
      "activations/layer1_attention_weight_max": 15.915406227111816,
      "activations/layer1_attention_weight_min": -13.73546028137207,
      "activations/layer20_attention_weight_max": 26.953411102294922,
      "activations/layer20_attention_weight_min": -26.53280258178711,
      "activations/layer21_attention_weight_max": 26.386356353759766,
      "activations/layer21_attention_weight_min": -24.50096321105957,
      "activations/layer22_attention_weight_max": 36.1894416809082,
      "activations/layer22_attention_weight_min": -30.331222534179688,
      "activations/layer23_attention_weight_max": 30.91864013671875,
      "activations/layer23_attention_weight_min": -25.837263107299805,
      "activations/layer2_attention_weight_max": 35.09482192993164,
      "activations/layer2_attention_weight_min": -32.376197814941406,
      "activations/layer3_attention_weight_max": 101.06603240966797,
      "activations/layer3_attention_weight_min": -97.45226287841797,
      "activations/layer4_attention_weight_max": 107.56231689453125,
      "activations/layer4_attention_weight_min": -104.60831451416016,
      "activations/layer5_attention_weight_max": 50.44770050048828,
      "activations/layer5_attention_weight_min": -58.42774200439453,
      "activations/layer6_attention_weight_max": 41.381919860839844,
      "activations/layer6_attention_weight_min": -44.368858337402344,
      "activations/layer7_attention_weight_max": 83.75125122070312,
      "activations/layer7_attention_weight_min": -85.8607177734375,
      "activations/layer8_attention_weight_max": 38.38218688964844,
      "activations/layer8_attention_weight_min": -39.788352966308594,
      "activations/layer9_attention_weight_max": 31.651880264282227,
      "activations/layer9_attention_weight_min": -31.4288272857666,
      "epoch": 17.52,
      "learning_rate": 3.7401515151515145e-05,
      "loss": 2.7238,
      "step": 301500
    },
    {
      "activations/layer0_attention_weight_max": 16.90056800842285,
      "activations/layer0_attention_weight_min": -12.94414234161377,
      "activations/layer10_attention_weight_max": 32.20627212524414,
      "activations/layer10_attention_weight_min": -33.253170013427734,
      "activations/layer11_attention_weight_max": 29.6553955078125,
      "activations/layer11_attention_weight_min": -31.912368774414062,
      "activations/layer12_attention_weight_max": 26.10912322998047,
      "activations/layer12_attention_weight_min": -25.655168533325195,
      "activations/layer13_attention_weight_max": 38.60599899291992,
      "activations/layer13_attention_weight_min": -35.669158935546875,
      "activations/layer14_attention_weight_max": 42.076385498046875,
      "activations/layer14_attention_weight_min": -36.702293395996094,
      "activations/layer15_attention_weight_max": 38.88484573364258,
      "activations/layer15_attention_weight_min": -32.030521392822266,
      "activations/layer16_attention_weight_max": 29.672679901123047,
      "activations/layer16_attention_weight_min": -27.23319435119629,
      "activations/layer17_attention_weight_max": 29.002336502075195,
      "activations/layer17_attention_weight_min": -24.394929885864258,
      "activations/layer18_attention_weight_max": 31.92672348022461,
      "activations/layer18_attention_weight_min": -26.951082229614258,
      "activations/layer19_attention_weight_max": 32.991451263427734,
      "activations/layer19_attention_weight_min": -32.10647964477539,
      "activations/layer1_attention_weight_max": 15.260697364807129,
      "activations/layer1_attention_weight_min": -15.459019660949707,
      "activations/layer20_attention_weight_max": 27.192615509033203,
      "activations/layer20_attention_weight_min": -25.490270614624023,
      "activations/layer21_attention_weight_max": 29.096012115478516,
      "activations/layer21_attention_weight_min": -25.067752838134766,
      "activations/layer22_attention_weight_max": 39.61988830566406,
      "activations/layer22_attention_weight_min": -31.58758544921875,
      "activations/layer23_attention_weight_max": 28.44672393798828,
      "activations/layer23_attention_weight_min": -27.875102996826172,
      "activations/layer2_attention_weight_max": 32.671546936035156,
      "activations/layer2_attention_weight_min": -31.130847930908203,
      "activations/layer3_attention_weight_max": 98.16011810302734,
      "activations/layer3_attention_weight_min": -97.89429473876953,
      "activations/layer4_attention_weight_max": 106.61815643310547,
      "activations/layer4_attention_weight_min": -111.58320617675781,
      "activations/layer5_attention_weight_max": 49.027496337890625,
      "activations/layer5_attention_weight_min": -65.65226745605469,
      "activations/layer6_attention_weight_max": 41.46014404296875,
      "activations/layer6_attention_weight_min": -45.00934600830078,
      "activations/layer7_attention_weight_max": 86.40013885498047,
      "activations/layer7_attention_weight_min": -91.74398803710938,
      "activations/layer8_attention_weight_max": 39.2857666015625,
      "activations/layer8_attention_weight_min": -40.19529724121094,
      "activations/layer9_attention_weight_max": 28.89105224609375,
      "activations/layer9_attention_weight_min": -31.34928321838379,
      "epoch": 17.52,
      "learning_rate": 3.738257575757575e-05,
      "loss": 2.7443,
      "step": 301550
    },
    {
      "activations/layer0_attention_weight_max": 15.611075401306152,
      "activations/layer0_attention_weight_min": -12.984674453735352,
      "activations/layer10_attention_weight_max": 39.178123474121094,
      "activations/layer10_attention_weight_min": -36.64628601074219,
      "activations/layer11_attention_weight_max": 37.1466064453125,
      "activations/layer11_attention_weight_min": -35.860225677490234,
      "activations/layer12_attention_weight_max": 28.968685150146484,
      "activations/layer12_attention_weight_min": -28.452295303344727,
      "activations/layer13_attention_weight_max": 41.893489837646484,
      "activations/layer13_attention_weight_min": -37.77306365966797,
      "activations/layer14_attention_weight_max": 42.55335235595703,
      "activations/layer14_attention_weight_min": -37.565269470214844,
      "activations/layer15_attention_weight_max": 42.407867431640625,
      "activations/layer15_attention_weight_min": -35.729652404785156,
      "activations/layer16_attention_weight_max": 30.025419235229492,
      "activations/layer16_attention_weight_min": -29.738758087158203,
      "activations/layer17_attention_weight_max": 29.51186752319336,
      "activations/layer17_attention_weight_min": -25.53728675842285,
      "activations/layer18_attention_weight_max": 30.145597457885742,
      "activations/layer18_attention_weight_min": -25.818193435668945,
      "activations/layer19_attention_weight_max": 33.79193878173828,
      "activations/layer19_attention_weight_min": -31.631906509399414,
      "activations/layer1_attention_weight_max": 15.75443172454834,
      "activations/layer1_attention_weight_min": -14.749743461608887,
      "activations/layer20_attention_weight_max": 27.572248458862305,
      "activations/layer20_attention_weight_min": -24.251768112182617,
      "activations/layer21_attention_weight_max": 26.44285011291504,
      "activations/layer21_attention_weight_min": -23.23992919921875,
      "activations/layer22_attention_weight_max": 42.05491256713867,
      "activations/layer22_attention_weight_min": -30.106021881103516,
      "activations/layer23_attention_weight_max": 33.7472038269043,
      "activations/layer23_attention_weight_min": -24.56914520263672,
      "activations/layer2_attention_weight_max": 31.335235595703125,
      "activations/layer2_attention_weight_min": -32.020973205566406,
      "activations/layer3_attention_weight_max": 100.74977111816406,
      "activations/layer3_attention_weight_min": -98.83898162841797,
      "activations/layer4_attention_weight_max": 106.0374526977539,
      "activations/layer4_attention_weight_min": -109.11236572265625,
      "activations/layer5_attention_weight_max": 50.82154083251953,
      "activations/layer5_attention_weight_min": -59.8862419128418,
      "activations/layer6_attention_weight_max": 44.01921463012695,
      "activations/layer6_attention_weight_min": -46.040061950683594,
      "activations/layer7_attention_weight_max": 96.95812225341797,
      "activations/layer7_attention_weight_min": -98.68070220947266,
      "activations/layer8_attention_weight_max": 43.62202072143555,
      "activations/layer8_attention_weight_min": -44.679603576660156,
      "activations/layer9_attention_weight_max": 35.65959167480469,
      "activations/layer9_attention_weight_min": -37.95717239379883,
      "epoch": 17.52,
      "learning_rate": 3.736363636363636e-05,
      "loss": 2.7572,
      "step": 301600
    },
    {
      "activations/layer0_attention_weight_max": 16.1417236328125,
      "activations/layer0_attention_weight_min": -12.570292472839355,
      "activations/layer10_attention_weight_max": 33.623382568359375,
      "activations/layer10_attention_weight_min": -33.83494186401367,
      "activations/layer11_attention_weight_max": 34.21637725830078,
      "activations/layer11_attention_weight_min": -35.06390380859375,
      "activations/layer12_attention_weight_max": 27.684070587158203,
      "activations/layer12_attention_weight_min": -28.616458892822266,
      "activations/layer13_attention_weight_max": 38.97502136230469,
      "activations/layer13_attention_weight_min": -38.08652877807617,
      "activations/layer14_attention_weight_max": 40.662208557128906,
      "activations/layer14_attention_weight_min": -37.50197219848633,
      "activations/layer15_attention_weight_max": 39.1127815246582,
      "activations/layer15_attention_weight_min": -34.52501678466797,
      "activations/layer16_attention_weight_max": 27.742570877075195,
      "activations/layer16_attention_weight_min": -29.716283798217773,
      "activations/layer17_attention_weight_max": 28.23680305480957,
      "activations/layer17_attention_weight_min": -26.209148406982422,
      "activations/layer18_attention_weight_max": 32.032344818115234,
      "activations/layer18_attention_weight_min": -22.66673469543457,
      "activations/layer19_attention_weight_max": 32.48724365234375,
      "activations/layer19_attention_weight_min": -31.371328353881836,
      "activations/layer1_attention_weight_max": 15.87023639678955,
      "activations/layer1_attention_weight_min": -13.864509582519531,
      "activations/layer20_attention_weight_max": 27.784250259399414,
      "activations/layer20_attention_weight_min": -24.594776153564453,
      "activations/layer21_attention_weight_max": 27.29056739807129,
      "activations/layer21_attention_weight_min": -22.961090087890625,
      "activations/layer22_attention_weight_max": 38.79839324951172,
      "activations/layer22_attention_weight_min": -27.97296905517578,
      "activations/layer23_attention_weight_max": 29.18661880493164,
      "activations/layer23_attention_weight_min": -24.1668701171875,
      "activations/layer2_attention_weight_max": 34.392581939697266,
      "activations/layer2_attention_weight_min": -32.8075065612793,
      "activations/layer3_attention_weight_max": 99.26704406738281,
      "activations/layer3_attention_weight_min": -98.70115661621094,
      "activations/layer4_attention_weight_max": 105.01679992675781,
      "activations/layer4_attention_weight_min": -105.03165435791016,
      "activations/layer5_attention_weight_max": 53.37537384033203,
      "activations/layer5_attention_weight_min": -60.2886962890625,
      "activations/layer6_attention_weight_max": 43.53684616088867,
      "activations/layer6_attention_weight_min": -45.2562141418457,
      "activations/layer7_attention_weight_max": 97.212890625,
      "activations/layer7_attention_weight_min": -91.83694458007812,
      "activations/layer8_attention_weight_max": 43.03645324707031,
      "activations/layer8_attention_weight_min": -44.45027160644531,
      "activations/layer9_attention_weight_max": 33.34387969970703,
      "activations/layer9_attention_weight_min": -33.19403839111328,
      "epoch": 17.53,
      "learning_rate": 3.734469696969697e-05,
      "loss": 2.7424,
      "step": 301650
    },
    {
      "activations/layer0_attention_weight_max": 15.711857795715332,
      "activations/layer0_attention_weight_min": -13.19163703918457,
      "activations/layer10_attention_weight_max": 36.09105682373047,
      "activations/layer10_attention_weight_min": -36.160552978515625,
      "activations/layer11_attention_weight_max": 35.03664779663086,
      "activations/layer11_attention_weight_min": -37.49676513671875,
      "activations/layer12_attention_weight_max": 27.523622512817383,
      "activations/layer12_attention_weight_min": -25.90686798095703,
      "activations/layer13_attention_weight_max": 39.69636917114258,
      "activations/layer13_attention_weight_min": -35.3558235168457,
      "activations/layer14_attention_weight_max": 46.03093719482422,
      "activations/layer14_attention_weight_min": -38.26463317871094,
      "activations/layer15_attention_weight_max": 38.37281799316406,
      "activations/layer15_attention_weight_min": -33.791168212890625,
      "activations/layer16_attention_weight_max": 29.844362258911133,
      "activations/layer16_attention_weight_min": -27.403539657592773,
      "activations/layer17_attention_weight_max": 30.32719612121582,
      "activations/layer17_attention_weight_min": -27.033771514892578,
      "activations/layer18_attention_weight_max": 31.11333656311035,
      "activations/layer18_attention_weight_min": -25.10022735595703,
      "activations/layer19_attention_weight_max": 40.1954231262207,
      "activations/layer19_attention_weight_min": -33.3201904296875,
      "activations/layer1_attention_weight_max": 16.73865509033203,
      "activations/layer1_attention_weight_min": -15.620872497558594,
      "activations/layer20_attention_weight_max": 33.23366165161133,
      "activations/layer20_attention_weight_min": -24.6678524017334,
      "activations/layer21_attention_weight_max": 31.188894271850586,
      "activations/layer21_attention_weight_min": -25.17911720275879,
      "activations/layer22_attention_weight_max": 44.46566390991211,
      "activations/layer22_attention_weight_min": -33.7133674621582,
      "activations/layer23_attention_weight_max": 33.38880157470703,
      "activations/layer23_attention_weight_min": -26.354244232177734,
      "activations/layer2_attention_weight_max": 34.283634185791016,
      "activations/layer2_attention_weight_min": -33.27277755737305,
      "activations/layer3_attention_weight_max": 106.567138671875,
      "activations/layer3_attention_weight_min": -101.60140228271484,
      "activations/layer4_attention_weight_max": 112.24665832519531,
      "activations/layer4_attention_weight_min": -111.71257781982422,
      "activations/layer5_attention_weight_max": 53.03770446777344,
      "activations/layer5_attention_weight_min": -63.011898040771484,
      "activations/layer6_attention_weight_max": 45.239418029785156,
      "activations/layer6_attention_weight_min": -47.57921600341797,
      "activations/layer7_attention_weight_max": 99.2073974609375,
      "activations/layer7_attention_weight_min": -96.57282257080078,
      "activations/layer8_attention_weight_max": 44.424556732177734,
      "activations/layer8_attention_weight_min": -45.51103973388672,
      "activations/layer9_attention_weight_max": 34.79808044433594,
      "activations/layer9_attention_weight_min": -35.9822998046875,
      "epoch": 17.53,
      "learning_rate": 3.732575757575757e-05,
      "loss": 2.7291,
      "step": 301700
    },
    {
      "activations/layer0_attention_weight_max": 16.59607696533203,
      "activations/layer0_attention_weight_min": -15.411296844482422,
      "activations/layer10_attention_weight_max": 34.306243896484375,
      "activations/layer10_attention_weight_min": -34.10923385620117,
      "activations/layer11_attention_weight_max": 32.900421142578125,
      "activations/layer11_attention_weight_min": -33.717613220214844,
      "activations/layer12_attention_weight_max": 26.280555725097656,
      "activations/layer12_attention_weight_min": -27.789735794067383,
      "activations/layer13_attention_weight_max": 39.440059661865234,
      "activations/layer13_attention_weight_min": -36.31550216674805,
      "activations/layer14_attention_weight_max": 41.99220275878906,
      "activations/layer14_attention_weight_min": -39.1152458190918,
      "activations/layer15_attention_weight_max": 38.03915786743164,
      "activations/layer15_attention_weight_min": -35.12862777709961,
      "activations/layer16_attention_weight_max": 28.236915588378906,
      "activations/layer16_attention_weight_min": -28.311355590820312,
      "activations/layer17_attention_weight_max": 28.869693756103516,
      "activations/layer17_attention_weight_min": -26.169740676879883,
      "activations/layer18_attention_weight_max": 31.630718231201172,
      "activations/layer18_attention_weight_min": -24.535051345825195,
      "activations/layer19_attention_weight_max": 33.96269989013672,
      "activations/layer19_attention_weight_min": -31.06922721862793,
      "activations/layer1_attention_weight_max": 16.03326988220215,
      "activations/layer1_attention_weight_min": -13.931809425354004,
      "activations/layer20_attention_weight_max": 29.959238052368164,
      "activations/layer20_attention_weight_min": -25.389677047729492,
      "activations/layer21_attention_weight_max": 28.274187088012695,
      "activations/layer21_attention_weight_min": -24.339136123657227,
      "activations/layer22_attention_weight_max": 46.24402618408203,
      "activations/layer22_attention_weight_min": -31.10272216796875,
      "activations/layer23_attention_weight_max": 32.70912551879883,
      "activations/layer23_attention_weight_min": -25.195117950439453,
      "activations/layer2_attention_weight_max": 32.37284851074219,
      "activations/layer2_attention_weight_min": -32.06108093261719,
      "activations/layer3_attention_weight_max": 99.35513305664062,
      "activations/layer3_attention_weight_min": -105.43215942382812,
      "activations/layer4_attention_weight_max": 110.03874969482422,
      "activations/layer4_attention_weight_min": -112.74710845947266,
      "activations/layer5_attention_weight_max": 50.3911247253418,
      "activations/layer5_attention_weight_min": -62.73217010498047,
      "activations/layer6_attention_weight_max": 43.929420471191406,
      "activations/layer6_attention_weight_min": -50.592857360839844,
      "activations/layer7_attention_weight_max": 94.4684066772461,
      "activations/layer7_attention_weight_min": -97.59471893310547,
      "activations/layer8_attention_weight_max": 41.83613586425781,
      "activations/layer8_attention_weight_min": -44.28144073486328,
      "activations/layer9_attention_weight_max": 32.13021469116211,
      "activations/layer9_attention_weight_min": -33.66941452026367,
      "epoch": 17.53,
      "learning_rate": 3.730681818181818e-05,
      "loss": 2.742,
      "step": 301750
    },
    {
      "activations/layer0_attention_weight_max": 15.901118278503418,
      "activations/layer0_attention_weight_min": -12.855713844299316,
      "activations/layer10_attention_weight_max": 33.70793151855469,
      "activations/layer10_attention_weight_min": -33.68128204345703,
      "activations/layer11_attention_weight_max": 34.91777420043945,
      "activations/layer11_attention_weight_min": -32.95728302001953,
      "activations/layer12_attention_weight_max": 37.760433197021484,
      "activations/layer12_attention_weight_min": -26.116641998291016,
      "activations/layer13_attention_weight_max": 45.70398712158203,
      "activations/layer13_attention_weight_min": -36.21914291381836,
      "activations/layer14_attention_weight_max": 41.371070861816406,
      "activations/layer14_attention_weight_min": -38.92777633666992,
      "activations/layer15_attention_weight_max": 40.5948371887207,
      "activations/layer15_attention_weight_min": -36.22159957885742,
      "activations/layer16_attention_weight_max": 31.017972946166992,
      "activations/layer16_attention_weight_min": -28.80213165283203,
      "activations/layer17_attention_weight_max": 30.0446720123291,
      "activations/layer17_attention_weight_min": -25.992021560668945,
      "activations/layer18_attention_weight_max": 32.1853141784668,
      "activations/layer18_attention_weight_min": -26.63580894470215,
      "activations/layer19_attention_weight_max": 36.566871643066406,
      "activations/layer19_attention_weight_min": -32.84352111816406,
      "activations/layer1_attention_weight_max": 16.913362503051758,
      "activations/layer1_attention_weight_min": -15.596585273742676,
      "activations/layer20_attention_weight_max": 28.088245391845703,
      "activations/layer20_attention_weight_min": -25.32597541809082,
      "activations/layer21_attention_weight_max": 29.2592830657959,
      "activations/layer21_attention_weight_min": -25.370758056640625,
      "activations/layer22_attention_weight_max": 42.14030838012695,
      "activations/layer22_attention_weight_min": -31.818696975708008,
      "activations/layer23_attention_weight_max": 33.8383903503418,
      "activations/layer23_attention_weight_min": -26.312467575073242,
      "activations/layer2_attention_weight_max": 32.25727844238281,
      "activations/layer2_attention_weight_min": -31.573822021484375,
      "activations/layer3_attention_weight_max": 95.8955307006836,
      "activations/layer3_attention_weight_min": -97.50302124023438,
      "activations/layer4_attention_weight_max": 104.59537506103516,
      "activations/layer4_attention_weight_min": -108.71907043457031,
      "activations/layer5_attention_weight_max": 49.30057907104492,
      "activations/layer5_attention_weight_min": -59.30178451538086,
      "activations/layer6_attention_weight_max": 42.775291442871094,
      "activations/layer6_attention_weight_min": -44.89118957519531,
      "activations/layer7_attention_weight_max": 89.55233001708984,
      "activations/layer7_attention_weight_min": -89.74486541748047,
      "activations/layer8_attention_weight_max": 39.725379943847656,
      "activations/layer8_attention_weight_min": -41.67036437988281,
      "activations/layer9_attention_weight_max": 32.421566009521484,
      "activations/layer9_attention_weight_min": -32.404903411865234,
      "epoch": 17.54,
      "learning_rate": 3.728787878787878e-05,
      "loss": 2.7307,
      "step": 301800
    },
    {
      "activations/layer0_attention_weight_max": 16.873815536499023,
      "activations/layer0_attention_weight_min": -12.690553665161133,
      "activations/layer10_attention_weight_max": 29.82600975036621,
      "activations/layer10_attention_weight_min": -32.23945999145508,
      "activations/layer11_attention_weight_max": 32.10440444946289,
      "activations/layer11_attention_weight_min": -32.1986083984375,
      "activations/layer12_attention_weight_max": 24.702890396118164,
      "activations/layer12_attention_weight_min": -26.56532096862793,
      "activations/layer13_attention_weight_max": 36.153446197509766,
      "activations/layer13_attention_weight_min": -33.67633056640625,
      "activations/layer14_attention_weight_max": 38.686466217041016,
      "activations/layer14_attention_weight_min": -35.54854202270508,
      "activations/layer15_attention_weight_max": 35.369659423828125,
      "activations/layer15_attention_weight_min": -33.971702575683594,
      "activations/layer16_attention_weight_max": 28.16455078125,
      "activations/layer16_attention_weight_min": -27.101083755493164,
      "activations/layer17_attention_weight_max": 27.031217575073242,
      "activations/layer17_attention_weight_min": -25.554418563842773,
      "activations/layer18_attention_weight_max": 29.762699127197266,
      "activations/layer18_attention_weight_min": -27.191640853881836,
      "activations/layer19_attention_weight_max": 29.879638671875,
      "activations/layer19_attention_weight_min": -29.872678756713867,
      "activations/layer1_attention_weight_max": 15.364850997924805,
      "activations/layer1_attention_weight_min": -14.441394805908203,
      "activations/layer20_attention_weight_max": 26.339561462402344,
      "activations/layer20_attention_weight_min": -25.842308044433594,
      "activations/layer21_attention_weight_max": 26.65509033203125,
      "activations/layer21_attention_weight_min": -23.24829864501953,
      "activations/layer22_attention_weight_max": 35.063026428222656,
      "activations/layer22_attention_weight_min": -31.101932525634766,
      "activations/layer23_attention_weight_max": 30.778255462646484,
      "activations/layer23_attention_weight_min": -25.21607780456543,
      "activations/layer2_attention_weight_max": 33.11770248413086,
      "activations/layer2_attention_weight_min": -31.473073959350586,
      "activations/layer3_attention_weight_max": 101.3086929321289,
      "activations/layer3_attention_weight_min": -100.50506591796875,
      "activations/layer4_attention_weight_max": 109.1978530883789,
      "activations/layer4_attention_weight_min": -111.17435455322266,
      "activations/layer5_attention_weight_max": 49.9708251953125,
      "activations/layer5_attention_weight_min": -59.59709167480469,
      "activations/layer6_attention_weight_max": 43.64219665527344,
      "activations/layer6_attention_weight_min": -47.421470642089844,
      "activations/layer7_attention_weight_max": 92.64104461669922,
      "activations/layer7_attention_weight_min": -88.93916320800781,
      "activations/layer8_attention_weight_max": 38.64665222167969,
      "activations/layer8_attention_weight_min": -41.21221160888672,
      "activations/layer9_attention_weight_max": 28.91064453125,
      "activations/layer9_attention_weight_min": -32.73965072631836,
      "epoch": 17.54,
      "learning_rate": 3.726893939393939e-05,
      "loss": 2.7256,
      "step": 301850
    },
    {
      "activations/layer0_attention_weight_max": 15.766454696655273,
      "activations/layer0_attention_weight_min": -12.634261131286621,
      "activations/layer10_attention_weight_max": 33.13956069946289,
      "activations/layer10_attention_weight_min": -33.283538818359375,
      "activations/layer11_attention_weight_max": 33.10468673706055,
      "activations/layer11_attention_weight_min": -33.31321716308594,
      "activations/layer12_attention_weight_max": 26.221586227416992,
      "activations/layer12_attention_weight_min": -26.717744827270508,
      "activations/layer13_attention_weight_max": 40.84893798828125,
      "activations/layer13_attention_weight_min": -37.08966064453125,
      "activations/layer14_attention_weight_max": 47.32024002075195,
      "activations/layer14_attention_weight_min": -38.35054016113281,
      "activations/layer15_attention_weight_max": 40.260860443115234,
      "activations/layer15_attention_weight_min": -34.39388656616211,
      "activations/layer16_attention_weight_max": 30.03388786315918,
      "activations/layer16_attention_weight_min": -29.142948150634766,
      "activations/layer17_attention_weight_max": 30.65083885192871,
      "activations/layer17_attention_weight_min": -26.938678741455078,
      "activations/layer18_attention_weight_max": 35.92724609375,
      "activations/layer18_attention_weight_min": -25.55275535583496,
      "activations/layer19_attention_weight_max": 36.8548698425293,
      "activations/layer19_attention_weight_min": -32.6721305847168,
      "activations/layer1_attention_weight_max": 16.38128089904785,
      "activations/layer1_attention_weight_min": -13.944856643676758,
      "activations/layer20_attention_weight_max": 30.37487030029297,
      "activations/layer20_attention_weight_min": -26.004247665405273,
      "activations/layer21_attention_weight_max": 31.538333892822266,
      "activations/layer21_attention_weight_min": -25.778942108154297,
      "activations/layer22_attention_weight_max": 48.64038848876953,
      "activations/layer22_attention_weight_min": -34.25291442871094,
      "activations/layer23_attention_weight_max": 37.78578186035156,
      "activations/layer23_attention_weight_min": -26.218671798706055,
      "activations/layer2_attention_weight_max": 33.123138427734375,
      "activations/layer2_attention_weight_min": -32.56731033325195,
      "activations/layer3_attention_weight_max": 102.78988647460938,
      "activations/layer3_attention_weight_min": -98.68650817871094,
      "activations/layer4_attention_weight_max": 111.00872039794922,
      "activations/layer4_attention_weight_min": -107.56092834472656,
      "activations/layer5_attention_weight_max": 51.13060760498047,
      "activations/layer5_attention_weight_min": -60.55546188354492,
      "activations/layer6_attention_weight_max": 43.93937301635742,
      "activations/layer6_attention_weight_min": -46.812259674072266,
      "activations/layer7_attention_weight_max": 96.3528060913086,
      "activations/layer7_attention_weight_min": -95.5613021850586,
      "activations/layer8_attention_weight_max": 41.29458999633789,
      "activations/layer8_attention_weight_min": -43.99049758911133,
      "activations/layer9_attention_weight_max": 31.111021041870117,
      "activations/layer9_attention_weight_min": -34.06818389892578,
      "epoch": 17.54,
      "learning_rate": 3.725e-05,
      "loss": 2.739,
      "step": 301900
    },
    {
      "activations/layer0_attention_weight_max": 16.044828414916992,
      "activations/layer0_attention_weight_min": -12.114115715026855,
      "activations/layer10_attention_weight_max": 36.490299224853516,
      "activations/layer10_attention_weight_min": -37.46448516845703,
      "activations/layer11_attention_weight_max": 33.90642547607422,
      "activations/layer11_attention_weight_min": -37.40752029418945,
      "activations/layer12_attention_weight_max": 25.343408584594727,
      "activations/layer12_attention_weight_min": -26.542673110961914,
      "activations/layer13_attention_weight_max": 38.78779602050781,
      "activations/layer13_attention_weight_min": -36.07229995727539,
      "activations/layer14_attention_weight_max": 42.76204299926758,
      "activations/layer14_attention_weight_min": -37.736549377441406,
      "activations/layer15_attention_weight_max": 37.33662796020508,
      "activations/layer15_attention_weight_min": -36.540767669677734,
      "activations/layer16_attention_weight_max": 28.540742874145508,
      "activations/layer16_attention_weight_min": -27.61355209350586,
      "activations/layer17_attention_weight_max": 29.39786148071289,
      "activations/layer17_attention_weight_min": -26.34893798828125,
      "activations/layer18_attention_weight_max": 29.946199417114258,
      "activations/layer18_attention_weight_min": -25.398454666137695,
      "activations/layer19_attention_weight_max": 36.51060104370117,
      "activations/layer19_attention_weight_min": -31.357471466064453,
      "activations/layer1_attention_weight_max": 17.30653953552246,
      "activations/layer1_attention_weight_min": -15.717220306396484,
      "activations/layer20_attention_weight_max": 30.036840438842773,
      "activations/layer20_attention_weight_min": -24.64885711669922,
      "activations/layer21_attention_weight_max": 29.79525375366211,
      "activations/layer21_attention_weight_min": -22.897306442260742,
      "activations/layer22_attention_weight_max": 39.784324645996094,
      "activations/layer22_attention_weight_min": -29.193952560424805,
      "activations/layer23_attention_weight_max": 31.20923614501953,
      "activations/layer23_attention_weight_min": -24.20887565612793,
      "activations/layer2_attention_weight_max": 33.17539978027344,
      "activations/layer2_attention_weight_min": -32.249107360839844,
      "activations/layer3_attention_weight_max": 102.01905059814453,
      "activations/layer3_attention_weight_min": -100.06708526611328,
      "activations/layer4_attention_weight_max": 110.32555389404297,
      "activations/layer4_attention_weight_min": -115.2426528930664,
      "activations/layer5_attention_weight_max": 50.081459045410156,
      "activations/layer5_attention_weight_min": -62.613861083984375,
      "activations/layer6_attention_weight_max": 45.22591781616211,
      "activations/layer6_attention_weight_min": -49.50159454345703,
      "activations/layer7_attention_weight_max": 97.75751495361328,
      "activations/layer7_attention_weight_min": -100.14678192138672,
      "activations/layer8_attention_weight_max": 42.55545425415039,
      "activations/layer8_attention_weight_min": -45.17283630371094,
      "activations/layer9_attention_weight_max": 33.13784408569336,
      "activations/layer9_attention_weight_min": -38.21455383300781,
      "epoch": 17.55,
      "learning_rate": 3.7231060606060606e-05,
      "loss": 2.7364,
      "step": 301950
    },
    {
      "activations/layer0_attention_weight_max": 15.029230117797852,
      "activations/layer0_attention_weight_min": -11.986990928649902,
      "activations/layer10_attention_weight_max": 34.7725715637207,
      "activations/layer10_attention_weight_min": -34.12633514404297,
      "activations/layer11_attention_weight_max": 33.236183166503906,
      "activations/layer11_attention_weight_min": -32.932830810546875,
      "activations/layer12_attention_weight_max": 26.703510284423828,
      "activations/layer12_attention_weight_min": -26.43755531311035,
      "activations/layer13_attention_weight_max": 42.41410827636719,
      "activations/layer13_attention_weight_min": -35.1154670715332,
      "activations/layer14_attention_weight_max": 44.30122756958008,
      "activations/layer14_attention_weight_min": -37.55861282348633,
      "activations/layer15_attention_weight_max": 43.5133056640625,
      "activations/layer15_attention_weight_min": -36.601749420166016,
      "activations/layer16_attention_weight_max": 31.76753807067871,
      "activations/layer16_attention_weight_min": -29.91877555847168,
      "activations/layer17_attention_weight_max": 31.296064376831055,
      "activations/layer17_attention_weight_min": -27.057445526123047,
      "activations/layer18_attention_weight_max": 33.64899826049805,
      "activations/layer18_attention_weight_min": -26.45403289794922,
      "activations/layer19_attention_weight_max": 33.95258331298828,
      "activations/layer19_attention_weight_min": -31.48372459411621,
      "activations/layer1_attention_weight_max": 17.039615631103516,
      "activations/layer1_attention_weight_min": -16.215219497680664,
      "activations/layer20_attention_weight_max": 28.27602767944336,
      "activations/layer20_attention_weight_min": -24.859230041503906,
      "activations/layer21_attention_weight_max": 29.096752166748047,
      "activations/layer21_attention_weight_min": -24.437145233154297,
      "activations/layer22_attention_weight_max": 42.542503356933594,
      "activations/layer22_attention_weight_min": -31.22028350830078,
      "activations/layer23_attention_weight_max": 31.405370712280273,
      "activations/layer23_attention_weight_min": -25.37390899658203,
      "activations/layer2_attention_weight_max": 33.84010314941406,
      "activations/layer2_attention_weight_min": -32.807823181152344,
      "activations/layer3_attention_weight_max": 101.71812438964844,
      "activations/layer3_attention_weight_min": -103.33650970458984,
      "activations/layer4_attention_weight_max": 108.71321868896484,
      "activations/layer4_attention_weight_min": -111.17110443115234,
      "activations/layer5_attention_weight_max": 49.30640411376953,
      "activations/layer5_attention_weight_min": -65.72264099121094,
      "activations/layer6_attention_weight_max": 43.72443389892578,
      "activations/layer6_attention_weight_min": -45.95845413208008,
      "activations/layer7_attention_weight_max": 90.43756866455078,
      "activations/layer7_attention_weight_min": -92.26544952392578,
      "activations/layer8_attention_weight_max": 43.2442741394043,
      "activations/layer8_attention_weight_min": -44.07132339477539,
      "activations/layer9_attention_weight_max": 33.80891418457031,
      "activations/layer9_attention_weight_min": -36.322662353515625,
      "epoch": 17.55,
      "learning_rate": 3.721212121212121e-05,
      "loss": 2.7321,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.5215,
      "eval_samples_per_second": 503.9,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.5215,
      "eval_openwebtext_samples_per_second": 503.9,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9423,
      "eval_wikitext_samples_per_second": 234.768,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_lambada_loss": 2.4609375,
      "eval_lambada_ppl": 11.715789946123383,
      "eval_lambada_runtime": 9.5472,
      "eval_lambada_samples_per_second": 509.992,
      "step": 302000
    },
    {
      "activations/layer0_attention_weight_max": 16.17827796936035,
      "activations/layer0_attention_weight_min": -13.549593925476074,
      "activations/layer10_attention_weight_max": 50.197052001953125,
      "activations/layer10_attention_weight_min": -50.353172302246094,
      "activations/layer11_attention_weight_max": 49.53535079956055,
      "activations/layer11_attention_weight_min": -49.548336029052734,
      "activations/layer12_attention_weight_max": 34.391029357910156,
      "activations/layer12_attention_weight_min": -32.39303970336914,
      "activations/layer13_attention_weight_max": 55.11554718017578,
      "activations/layer13_attention_weight_min": -47.05918884277344,
      "activations/layer14_attention_weight_max": 64.72880554199219,
      "activations/layer14_attention_weight_min": -57.81034469604492,
      "activations/layer15_attention_weight_max": 64.6662368774414,
      "activations/layer15_attention_weight_min": -51.78175735473633,
      "activations/layer16_attention_weight_max": 37.49806594848633,
      "activations/layer16_attention_weight_min": -29.133621215820312,
      "activations/layer17_attention_weight_max": 32.635005950927734,
      "activations/layer17_attention_weight_min": -31.18436622619629,
      "activations/layer18_attention_weight_max": 32.309329986572266,
      "activations/layer18_attention_weight_min": -28.073837280273438,
      "activations/layer19_attention_weight_max": 41.34251022338867,
      "activations/layer19_attention_weight_min": -30.836469650268555,
      "activations/layer1_attention_weight_max": 15.877055168151855,
      "activations/layer1_attention_weight_min": -15.150848388671875,
      "activations/layer20_attention_weight_max": 33.33955001831055,
      "activations/layer20_attention_weight_min": -23.64695930480957,
      "activations/layer21_attention_weight_max": 28.022884368896484,
      "activations/layer21_attention_weight_min": -23.545909881591797,
      "activations/layer22_attention_weight_max": 43.165348052978516,
      "activations/layer22_attention_weight_min": -32.143943786621094,
      "activations/layer23_attention_weight_max": 36.39368438720703,
      "activations/layer23_attention_weight_min": -24.347206115722656,
      "activations/layer2_attention_weight_max": 35.36332321166992,
      "activations/layer2_attention_weight_min": -34.534332275390625,
      "activations/layer3_attention_weight_max": 106.66448974609375,
      "activations/layer3_attention_weight_min": -111.5243148803711,
      "activations/layer4_attention_weight_max": 115.20146942138672,
      "activations/layer4_attention_weight_min": -108.13801574707031,
      "activations/layer5_attention_weight_max": 52.23200988769531,
      "activations/layer5_attention_weight_min": -60.02215576171875,
      "activations/layer6_attention_weight_max": 49.065731048583984,
      "activations/layer6_attention_weight_min": -48.59486389160156,
      "activations/layer7_attention_weight_max": 117.65365600585938,
      "activations/layer7_attention_weight_min": -111.97071838378906,
      "activations/layer8_attention_weight_max": 55.78402328491211,
      "activations/layer8_attention_weight_min": -56.149322509765625,
      "activations/layer9_attention_weight_max": 45.404624938964844,
      "activations/layer9_attention_weight_min": -47.15446853637695,
      "epoch": 17.55,
      "learning_rate": 3.7193181818181815e-05,
      "loss": 2.7484,
      "step": 302050
    },
    {
      "activations/layer0_attention_weight_max": 17.006254196166992,
      "activations/layer0_attention_weight_min": -12.78459358215332,
      "activations/layer10_attention_weight_max": 39.527923583984375,
      "activations/layer10_attention_weight_min": -33.36100387573242,
      "activations/layer11_attention_weight_max": 36.57170486450195,
      "activations/layer11_attention_weight_min": -33.24285125732422,
      "activations/layer12_attention_weight_max": 33.511749267578125,
      "activations/layer12_attention_weight_min": -25.051427841186523,
      "activations/layer13_attention_weight_max": 41.16284942626953,
      "activations/layer13_attention_weight_min": -33.16836166381836,
      "activations/layer14_attention_weight_max": 41.29490661621094,
      "activations/layer14_attention_weight_min": -36.09043884277344,
      "activations/layer15_attention_weight_max": 40.69975280761719,
      "activations/layer15_attention_weight_min": -36.03436279296875,
      "activations/layer16_attention_weight_max": 31.46308135986328,
      "activations/layer16_attention_weight_min": -30.687210083007812,
      "activations/layer17_attention_weight_max": 29.44339942932129,
      "activations/layer17_attention_weight_min": -26.65904426574707,
      "activations/layer18_attention_weight_max": 30.110492706298828,
      "activations/layer18_attention_weight_min": -27.007600784301758,
      "activations/layer19_attention_weight_max": 34.933311462402344,
      "activations/layer19_attention_weight_min": -33.27705001831055,
      "activations/layer1_attention_weight_max": 16.103574752807617,
      "activations/layer1_attention_weight_min": -16.3131046295166,
      "activations/layer20_attention_weight_max": 28.080705642700195,
      "activations/layer20_attention_weight_min": -28.35563850402832,
      "activations/layer21_attention_weight_max": 28.015506744384766,
      "activations/layer21_attention_weight_min": -27.01183319091797,
      "activations/layer22_attention_weight_max": 43.869407653808594,
      "activations/layer22_attention_weight_min": -33.81816101074219,
      "activations/layer23_attention_weight_max": 32.81794357299805,
      "activations/layer23_attention_weight_min": -27.917625427246094,
      "activations/layer2_attention_weight_max": 35.319610595703125,
      "activations/layer2_attention_weight_min": -33.804649353027344,
      "activations/layer3_attention_weight_max": 95.95616912841797,
      "activations/layer3_attention_weight_min": -99.88591766357422,
      "activations/layer4_attention_weight_max": 104.6792984008789,
      "activations/layer4_attention_weight_min": -106.09099578857422,
      "activations/layer5_attention_weight_max": 52.19164276123047,
      "activations/layer5_attention_weight_min": -61.34597396850586,
      "activations/layer6_attention_weight_max": 40.74517059326172,
      "activations/layer6_attention_weight_min": -43.42668533325195,
      "activations/layer7_attention_weight_max": 97.15238952636719,
      "activations/layer7_attention_weight_min": -94.193359375,
      "activations/layer8_attention_weight_max": 47.5500373840332,
      "activations/layer8_attention_weight_min": -44.07401657104492,
      "activations/layer9_attention_weight_max": 33.55792236328125,
      "activations/layer9_attention_weight_min": -33.779083251953125,
      "epoch": 17.55,
      "learning_rate": 3.7174242424242424e-05,
      "loss": 2.7417,
      "step": 302100
    },
    {
      "activations/layer0_attention_weight_max": 16.713977813720703,
      "activations/layer0_attention_weight_min": -12.50786018371582,
      "activations/layer10_attention_weight_max": 34.216556549072266,
      "activations/layer10_attention_weight_min": -35.46513748168945,
      "activations/layer11_attention_weight_max": 33.88347625732422,
      "activations/layer11_attention_weight_min": -34.446067810058594,
      "activations/layer12_attention_weight_max": 28.61573028564453,
      "activations/layer12_attention_weight_min": -28.2774715423584,
      "activations/layer13_attention_weight_max": 42.470367431640625,
      "activations/layer13_attention_weight_min": -36.73317337036133,
      "activations/layer14_attention_weight_max": 46.32234191894531,
      "activations/layer14_attention_weight_min": -37.79861831665039,
      "activations/layer15_attention_weight_max": 41.14488220214844,
      "activations/layer15_attention_weight_min": -36.12125015258789,
      "activations/layer16_attention_weight_max": 29.40726089477539,
      "activations/layer16_attention_weight_min": -31.88458251953125,
      "activations/layer17_attention_weight_max": 29.27220344543457,
      "activations/layer17_attention_weight_min": -26.62830924987793,
      "activations/layer18_attention_weight_max": 33.37225341796875,
      "activations/layer18_attention_weight_min": -25.96718978881836,
      "activations/layer19_attention_weight_max": 36.233394622802734,
      "activations/layer19_attention_weight_min": -33.154930114746094,
      "activations/layer1_attention_weight_max": 15.299549102783203,
      "activations/layer1_attention_weight_min": -14.771102905273438,
      "activations/layer20_attention_weight_max": 30.866487503051758,
      "activations/layer20_attention_weight_min": -27.7196102142334,
      "activations/layer21_attention_weight_max": 32.096134185791016,
      "activations/layer21_attention_weight_min": -26.51779556274414,
      "activations/layer22_attention_weight_max": 44.893882751464844,
      "activations/layer22_attention_weight_min": -37.54978561401367,
      "activations/layer23_attention_weight_max": 34.009117126464844,
      "activations/layer23_attention_weight_min": -25.184326171875,
      "activations/layer2_attention_weight_max": 32.55695343017578,
      "activations/layer2_attention_weight_min": -31.89828872680664,
      "activations/layer3_attention_weight_max": 100.49607849121094,
      "activations/layer3_attention_weight_min": -97.38805389404297,
      "activations/layer4_attention_weight_max": 110.18341064453125,
      "activations/layer4_attention_weight_min": -108.72139739990234,
      "activations/layer5_attention_weight_max": 51.18192672729492,
      "activations/layer5_attention_weight_min": -58.65378952026367,
      "activations/layer6_attention_weight_max": 45.775821685791016,
      "activations/layer6_attention_weight_min": -46.93265151977539,
      "activations/layer7_attention_weight_max": 95.67784118652344,
      "activations/layer7_attention_weight_min": -96.40963745117188,
      "activations/layer8_attention_weight_max": 45.68446350097656,
      "activations/layer8_attention_weight_min": -46.28767776489258,
      "activations/layer9_attention_weight_max": 34.94932556152344,
      "activations/layer9_attention_weight_min": -35.80337142944336,
      "epoch": 17.56,
      "learning_rate": 3.7155303030303025e-05,
      "loss": 2.7427,
      "step": 302150
    },
    {
      "activations/layer0_attention_weight_max": 15.432107925415039,
      "activations/layer0_attention_weight_min": -14.618474006652832,
      "activations/layer10_attention_weight_max": 35.4498176574707,
      "activations/layer10_attention_weight_min": -32.69013977050781,
      "activations/layer11_attention_weight_max": 37.035667419433594,
      "activations/layer11_attention_weight_min": -34.54996109008789,
      "activations/layer12_attention_weight_max": 30.2014217376709,
      "activations/layer12_attention_weight_min": -27.207534790039062,
      "activations/layer13_attention_weight_max": 40.03070831298828,
      "activations/layer13_attention_weight_min": -37.8441276550293,
      "activations/layer14_attention_weight_max": 47.23524856567383,
      "activations/layer14_attention_weight_min": -37.65180206298828,
      "activations/layer15_attention_weight_max": 40.097984313964844,
      "activations/layer15_attention_weight_min": -36.296932220458984,
      "activations/layer16_attention_weight_max": 30.524744033813477,
      "activations/layer16_attention_weight_min": -28.354509353637695,
      "activations/layer17_attention_weight_max": 31.626699447631836,
      "activations/layer17_attention_weight_min": -26.495664596557617,
      "activations/layer18_attention_weight_max": 31.087772369384766,
      "activations/layer18_attention_weight_min": -26.918384552001953,
      "activations/layer19_attention_weight_max": 36.36909484863281,
      "activations/layer19_attention_weight_min": -30.24530029296875,
      "activations/layer1_attention_weight_max": 16.76841926574707,
      "activations/layer1_attention_weight_min": -15.62753677368164,
      "activations/layer20_attention_weight_max": 31.442554473876953,
      "activations/layer20_attention_weight_min": -25.438671112060547,
      "activations/layer21_attention_weight_max": 29.52828025817871,
      "activations/layer21_attention_weight_min": -24.94707679748535,
      "activations/layer22_attention_weight_max": 44.82969284057617,
      "activations/layer22_attention_weight_min": -30.82441520690918,
      "activations/layer23_attention_weight_max": 37.068626403808594,
      "activations/layer23_attention_weight_min": -25.552480697631836,
      "activations/layer2_attention_weight_max": 31.391822814941406,
      "activations/layer2_attention_weight_min": -31.870542526245117,
      "activations/layer3_attention_weight_max": 95.8436050415039,
      "activations/layer3_attention_weight_min": -97.83199310302734,
      "activations/layer4_attention_weight_max": 104.6192626953125,
      "activations/layer4_attention_weight_min": -107.5088119506836,
      "activations/layer5_attention_weight_max": 51.37708282470703,
      "activations/layer5_attention_weight_min": -61.395565032958984,
      "activations/layer6_attention_weight_max": 46.1358528137207,
      "activations/layer6_attention_weight_min": -45.9059944152832,
      "activations/layer7_attention_weight_max": 91.12596130371094,
      "activations/layer7_attention_weight_min": -90.89815521240234,
      "activations/layer8_attention_weight_max": 41.53919982910156,
      "activations/layer8_attention_weight_min": -43.061641693115234,
      "activations/layer9_attention_weight_max": 32.86466979980469,
      "activations/layer9_attention_weight_min": -32.50857925415039,
      "epoch": 17.56,
      "learning_rate": 3.7136363636363633e-05,
      "loss": 2.7289,
      "step": 302200
    },
    {
      "activations/layer0_attention_weight_max": 16.40471839904785,
      "activations/layer0_attention_weight_min": -12.054685592651367,
      "activations/layer10_attention_weight_max": 32.7508430480957,
      "activations/layer10_attention_weight_min": -33.75241470336914,
      "activations/layer11_attention_weight_max": 31.66524314880371,
      "activations/layer11_attention_weight_min": -32.424625396728516,
      "activations/layer12_attention_weight_max": 25.002775192260742,
      "activations/layer12_attention_weight_min": -25.265172958374023,
      "activations/layer13_attention_weight_max": 38.30903244018555,
      "activations/layer13_attention_weight_min": -36.10368728637695,
      "activations/layer14_attention_weight_max": 41.70556640625,
      "activations/layer14_attention_weight_min": -37.743350982666016,
      "activations/layer15_attention_weight_max": 36.622947692871094,
      "activations/layer15_attention_weight_min": -35.5362548828125,
      "activations/layer16_attention_weight_max": 30.537683486938477,
      "activations/layer16_attention_weight_min": -29.497331619262695,
      "activations/layer17_attention_weight_max": 31.078638076782227,
      "activations/layer17_attention_weight_min": -26.00830078125,
      "activations/layer18_attention_weight_max": 32.24624252319336,
      "activations/layer18_attention_weight_min": -26.30623435974121,
      "activations/layer19_attention_weight_max": 34.5722541809082,
      "activations/layer19_attention_weight_min": -32.5012321472168,
      "activations/layer1_attention_weight_max": 16.271514892578125,
      "activations/layer1_attention_weight_min": -14.667340278625488,
      "activations/layer20_attention_weight_max": 28.673738479614258,
      "activations/layer20_attention_weight_min": -25.896263122558594,
      "activations/layer21_attention_weight_max": 28.215604782104492,
      "activations/layer21_attention_weight_min": -25.801658630371094,
      "activations/layer22_attention_weight_max": 37.81753921508789,
      "activations/layer22_attention_weight_min": -31.830596923828125,
      "activations/layer23_attention_weight_max": 30.87759017944336,
      "activations/layer23_attention_weight_min": -29.824583053588867,
      "activations/layer2_attention_weight_max": 32.270137786865234,
      "activations/layer2_attention_weight_min": -32.74366760253906,
      "activations/layer3_attention_weight_max": 96.38142395019531,
      "activations/layer3_attention_weight_min": -99.00727081298828,
      "activations/layer4_attention_weight_max": 106.13380432128906,
      "activations/layer4_attention_weight_min": -114.44945526123047,
      "activations/layer5_attention_weight_max": 47.79405975341797,
      "activations/layer5_attention_weight_min": -63.65218734741211,
      "activations/layer6_attention_weight_max": 42.525814056396484,
      "activations/layer6_attention_weight_min": -46.61556625366211,
      "activations/layer7_attention_weight_max": 90.6758804321289,
      "activations/layer7_attention_weight_min": -96.34906005859375,
      "activations/layer8_attention_weight_max": 41.95425033569336,
      "activations/layer8_attention_weight_min": -44.27029800415039,
      "activations/layer9_attention_weight_max": 30.60567283630371,
      "activations/layer9_attention_weight_min": -33.281681060791016,
      "epoch": 17.56,
      "learning_rate": 3.711742424242424e-05,
      "loss": 2.7385,
      "step": 302250
    },
    {
      "activations/layer0_attention_weight_max": 16.162424087524414,
      "activations/layer0_attention_weight_min": -12.4439058303833,
      "activations/layer10_attention_weight_max": 34.4525032043457,
      "activations/layer10_attention_weight_min": -34.68809509277344,
      "activations/layer11_attention_weight_max": 33.001991271972656,
      "activations/layer11_attention_weight_min": -34.46494674682617,
      "activations/layer12_attention_weight_max": 25.705949783325195,
      "activations/layer12_attention_weight_min": -26.538759231567383,
      "activations/layer13_attention_weight_max": 37.05995178222656,
      "activations/layer13_attention_weight_min": -36.68047332763672,
      "activations/layer14_attention_weight_max": 38.81606674194336,
      "activations/layer14_attention_weight_min": -36.25246047973633,
      "activations/layer15_attention_weight_max": 34.83669662475586,
      "activations/layer15_attention_weight_min": -35.86124801635742,
      "activations/layer16_attention_weight_max": 27.290599822998047,
      "activations/layer16_attention_weight_min": -27.402021408081055,
      "activations/layer17_attention_weight_max": 26.985057830810547,
      "activations/layer17_attention_weight_min": -26.307519912719727,
      "activations/layer18_attention_weight_max": 30.091976165771484,
      "activations/layer18_attention_weight_min": -26.75493621826172,
      "activations/layer19_attention_weight_max": 33.76886749267578,
      "activations/layer19_attention_weight_min": -29.69415855407715,
      "activations/layer1_attention_weight_max": 15.728453636169434,
      "activations/layer1_attention_weight_min": -14.5205659866333,
      "activations/layer20_attention_weight_max": 24.760923385620117,
      "activations/layer20_attention_weight_min": -25.51190185546875,
      "activations/layer21_attention_weight_max": 25.30763816833496,
      "activations/layer21_attention_weight_min": -24.29677391052246,
      "activations/layer22_attention_weight_max": 36.599605560302734,
      "activations/layer22_attention_weight_min": -29.46830940246582,
      "activations/layer23_attention_weight_max": 27.755531311035156,
      "activations/layer23_attention_weight_min": -25.923259735107422,
      "activations/layer2_attention_weight_max": 32.36583709716797,
      "activations/layer2_attention_weight_min": -33.727787017822266,
      "activations/layer3_attention_weight_max": 100.88875579833984,
      "activations/layer3_attention_weight_min": -100.42887878417969,
      "activations/layer4_attention_weight_max": 108.1437759399414,
      "activations/layer4_attention_weight_min": -113.46891021728516,
      "activations/layer5_attention_weight_max": 49.37229919433594,
      "activations/layer5_attention_weight_min": -62.223976135253906,
      "activations/layer6_attention_weight_max": 44.88288497924805,
      "activations/layer6_attention_weight_min": -46.80559158325195,
      "activations/layer7_attention_weight_max": 94.33744049072266,
      "activations/layer7_attention_weight_min": -95.98589324951172,
      "activations/layer8_attention_weight_max": 41.97662353515625,
      "activations/layer8_attention_weight_min": -45.11592483520508,
      "activations/layer9_attention_weight_max": 32.94002151489258,
      "activations/layer9_attention_weight_min": -35.0511360168457,
      "epoch": 17.57,
      "learning_rate": 3.709848484848485e-05,
      "loss": 2.7462,
      "step": 302300
    },
    {
      "activations/layer0_attention_weight_max": 15.678074836730957,
      "activations/layer0_attention_weight_min": -13.611937522888184,
      "activations/layer10_attention_weight_max": 37.03396987915039,
      "activations/layer10_attention_weight_min": -34.81703567504883,
      "activations/layer11_attention_weight_max": 39.27090072631836,
      "activations/layer11_attention_weight_min": -37.35279846191406,
      "activations/layer12_attention_weight_max": 28.39809799194336,
      "activations/layer12_attention_weight_min": -25.682554244995117,
      "activations/layer13_attention_weight_max": 38.73863983154297,
      "activations/layer13_attention_weight_min": -35.938873291015625,
      "activations/layer14_attention_weight_max": 39.26710891723633,
      "activations/layer14_attention_weight_min": -38.94327163696289,
      "activations/layer15_attention_weight_max": 38.37167739868164,
      "activations/layer15_attention_weight_min": -35.08030319213867,
      "activations/layer16_attention_weight_max": 30.59128761291504,
      "activations/layer16_attention_weight_min": -28.639545440673828,
      "activations/layer17_attention_weight_max": 32.4115104675293,
      "activations/layer17_attention_weight_min": -25.99626350402832,
      "activations/layer18_attention_weight_max": 31.203096389770508,
      "activations/layer18_attention_weight_min": -25.649127960205078,
      "activations/layer19_attention_weight_max": 31.52509880065918,
      "activations/layer19_attention_weight_min": -32.0223388671875,
      "activations/layer1_attention_weight_max": 16.66105842590332,
      "activations/layer1_attention_weight_min": -14.27433967590332,
      "activations/layer20_attention_weight_max": 27.869518280029297,
      "activations/layer20_attention_weight_min": -23.789461135864258,
      "activations/layer21_attention_weight_max": 27.029558181762695,
      "activations/layer21_attention_weight_min": -24.803258895874023,
      "activations/layer22_attention_weight_max": 41.15872573852539,
      "activations/layer22_attention_weight_min": -31.271806716918945,
      "activations/layer23_attention_weight_max": 31.02210235595703,
      "activations/layer23_attention_weight_min": -25.626888275146484,
      "activations/layer2_attention_weight_max": 35.771324157714844,
      "activations/layer2_attention_weight_min": -33.198368072509766,
      "activations/layer3_attention_weight_max": 105.46124267578125,
      "activations/layer3_attention_weight_min": -99.43460083007812,
      "activations/layer4_attention_weight_max": 104.78779602050781,
      "activations/layer4_attention_weight_min": -106.189453125,
      "activations/layer5_attention_weight_max": 49.008445739746094,
      "activations/layer5_attention_weight_min": -61.438812255859375,
      "activations/layer6_attention_weight_max": 43.26162338256836,
      "activations/layer6_attention_weight_min": -48.32167434692383,
      "activations/layer7_attention_weight_max": 96.39290618896484,
      "activations/layer7_attention_weight_min": -98.43915557861328,
      "activations/layer8_attention_weight_max": 45.00632095336914,
      "activations/layer8_attention_weight_min": -43.543922424316406,
      "activations/layer9_attention_weight_max": 33.239131927490234,
      "activations/layer9_attention_weight_min": -34.32682418823242,
      "epoch": 17.57,
      "learning_rate": 3.707954545454545e-05,
      "loss": 2.7257,
      "step": 302350
    },
    {
      "activations/layer0_attention_weight_max": 16.354604721069336,
      "activations/layer0_attention_weight_min": -13.818448066711426,
      "activations/layer10_attention_weight_max": 31.835905075073242,
      "activations/layer10_attention_weight_min": -32.567413330078125,
      "activations/layer11_attention_weight_max": 32.59992599487305,
      "activations/layer11_attention_weight_min": -32.676849365234375,
      "activations/layer12_attention_weight_max": 26.77465057373047,
      "activations/layer12_attention_weight_min": -26.042495727539062,
      "activations/layer13_attention_weight_max": 40.967735290527344,
      "activations/layer13_attention_weight_min": -35.19051742553711,
      "activations/layer14_attention_weight_max": 42.709964752197266,
      "activations/layer14_attention_weight_min": -38.679832458496094,
      "activations/layer15_attention_weight_max": 41.48014450073242,
      "activations/layer15_attention_weight_min": -35.48942184448242,
      "activations/layer16_attention_weight_max": 31.36089324951172,
      "activations/layer16_attention_weight_min": -29.946638107299805,
      "activations/layer17_attention_weight_max": 32.03289794921875,
      "activations/layer17_attention_weight_min": -26.477415084838867,
      "activations/layer18_attention_weight_max": 32.123931884765625,
      "activations/layer18_attention_weight_min": -25.829988479614258,
      "activations/layer19_attention_weight_max": 34.29433822631836,
      "activations/layer19_attention_weight_min": -32.776527404785156,
      "activations/layer1_attention_weight_max": 15.82145881652832,
      "activations/layer1_attention_weight_min": -14.4719877243042,
      "activations/layer20_attention_weight_max": 29.029417037963867,
      "activations/layer20_attention_weight_min": -25.028120040893555,
      "activations/layer21_attention_weight_max": 31.19939422607422,
      "activations/layer21_attention_weight_min": -24.661968231201172,
      "activations/layer22_attention_weight_max": 40.364402770996094,
      "activations/layer22_attention_weight_min": -32.833106994628906,
      "activations/layer23_attention_weight_max": 33.90122985839844,
      "activations/layer23_attention_weight_min": -27.075237274169922,
      "activations/layer2_attention_weight_max": 32.4411735534668,
      "activations/layer2_attention_weight_min": -32.51106262207031,
      "activations/layer3_attention_weight_max": 98.58675384521484,
      "activations/layer3_attention_weight_min": -100.53502655029297,
      "activations/layer4_attention_weight_max": 103.98600006103516,
      "activations/layer4_attention_weight_min": -106.91131591796875,
      "activations/layer5_attention_weight_max": 49.420772552490234,
      "activations/layer5_attention_weight_min": -57.63555908203125,
      "activations/layer6_attention_weight_max": 43.84994888305664,
      "activations/layer6_attention_weight_min": -46.04188537597656,
      "activations/layer7_attention_weight_max": 90.11579895019531,
      "activations/layer7_attention_weight_min": -91.89273834228516,
      "activations/layer8_attention_weight_max": 38.42610549926758,
      "activations/layer8_attention_weight_min": -43.17731857299805,
      "activations/layer9_attention_weight_max": 30.12284278869629,
      "activations/layer9_attention_weight_min": -33.382606506347656,
      "epoch": 17.57,
      "learning_rate": 3.706060606060606e-05,
      "loss": 2.7366,
      "step": 302400
    },
    {
      "activations/layer0_attention_weight_max": 16.342905044555664,
      "activations/layer0_attention_weight_min": -12.83836555480957,
      "activations/layer10_attention_weight_max": 37.37751388549805,
      "activations/layer10_attention_weight_min": -36.1658821105957,
      "activations/layer11_attention_weight_max": 35.677677154541016,
      "activations/layer11_attention_weight_min": -37.384002685546875,
      "activations/layer12_attention_weight_max": 29.34994888305664,
      "activations/layer12_attention_weight_min": -28.97303009033203,
      "activations/layer13_attention_weight_max": 44.470115661621094,
      "activations/layer13_attention_weight_min": -36.58915710449219,
      "activations/layer14_attention_weight_max": 46.72738265991211,
      "activations/layer14_attention_weight_min": -39.9249382019043,
      "activations/layer15_attention_weight_max": 46.15780258178711,
      "activations/layer15_attention_weight_min": -37.99941635131836,
      "activations/layer16_attention_weight_max": 31.991960525512695,
      "activations/layer16_attention_weight_min": -29.727787017822266,
      "activations/layer17_attention_weight_max": 31.39755630493164,
      "activations/layer17_attention_weight_min": -28.423547744750977,
      "activations/layer18_attention_weight_max": 33.96052551269531,
      "activations/layer18_attention_weight_min": -26.087739944458008,
      "activations/layer19_attention_weight_max": 41.94441223144531,
      "activations/layer19_attention_weight_min": -33.89176940917969,
      "activations/layer1_attention_weight_max": 17.59937286376953,
      "activations/layer1_attention_weight_min": -15.019217491149902,
      "activations/layer20_attention_weight_max": 31.516733169555664,
      "activations/layer20_attention_weight_min": -24.982887268066406,
      "activations/layer21_attention_weight_max": 31.832881927490234,
      "activations/layer21_attention_weight_min": -23.994157791137695,
      "activations/layer22_attention_weight_max": 45.519378662109375,
      "activations/layer22_attention_weight_min": -30.960750579833984,
      "activations/layer23_attention_weight_max": 36.20872116088867,
      "activations/layer23_attention_weight_min": -26.876773834228516,
      "activations/layer2_attention_weight_max": 35.92950439453125,
      "activations/layer2_attention_weight_min": -32.4708366394043,
      "activations/layer3_attention_weight_max": 105.4927749633789,
      "activations/layer3_attention_weight_min": -101.4274673461914,
      "activations/layer4_attention_weight_max": 114.53978729248047,
      "activations/layer4_attention_weight_min": -108.40301513671875,
      "activations/layer5_attention_weight_max": 52.9600944519043,
      "activations/layer5_attention_weight_min": -59.577667236328125,
      "activations/layer6_attention_weight_max": 45.450557708740234,
      "activations/layer6_attention_weight_min": -46.362606048583984,
      "activations/layer7_attention_weight_max": 95.75968933105469,
      "activations/layer7_attention_weight_min": -92.7891616821289,
      "activations/layer8_attention_weight_max": 44.04241943359375,
      "activations/layer8_attention_weight_min": -44.83565902709961,
      "activations/layer9_attention_weight_max": 35.193084716796875,
      "activations/layer9_attention_weight_min": -34.10150909423828,
      "epoch": 17.57,
      "learning_rate": 3.704166666666666e-05,
      "loss": 2.7419,
      "step": 302450
    },
    {
      "activations/layer0_attention_weight_max": 15.92216682434082,
      "activations/layer0_attention_weight_min": -14.509252548217773,
      "activations/layer10_attention_weight_max": 32.91290283203125,
      "activations/layer10_attention_weight_min": -33.255008697509766,
      "activations/layer11_attention_weight_max": 31.640806198120117,
      "activations/layer11_attention_weight_min": -35.122047424316406,
      "activations/layer12_attention_weight_max": 28.180923461914062,
      "activations/layer12_attention_weight_min": -26.990659713745117,
      "activations/layer13_attention_weight_max": 39.31747055053711,
      "activations/layer13_attention_weight_min": -34.31867599487305,
      "activations/layer14_attention_weight_max": 42.28099822998047,
      "activations/layer14_attention_weight_min": -36.622528076171875,
      "activations/layer15_attention_weight_max": 37.01136779785156,
      "activations/layer15_attention_weight_min": -35.48848342895508,
      "activations/layer16_attention_weight_max": 29.967662811279297,
      "activations/layer16_attention_weight_min": -29.73741340637207,
      "activations/layer17_attention_weight_max": 29.12668228149414,
      "activations/layer17_attention_weight_min": -25.91725730895996,
      "activations/layer18_attention_weight_max": 32.214569091796875,
      "activations/layer18_attention_weight_min": -25.273706436157227,
      "activations/layer19_attention_weight_max": 36.65631866455078,
      "activations/layer19_attention_weight_min": -36.25855255126953,
      "activations/layer1_attention_weight_max": 15.484941482543945,
      "activations/layer1_attention_weight_min": -13.532007217407227,
      "activations/layer20_attention_weight_max": 29.949954986572266,
      "activations/layer20_attention_weight_min": -26.754722595214844,
      "activations/layer21_attention_weight_max": 29.59195327758789,
      "activations/layer21_attention_weight_min": -27.440610885620117,
      "activations/layer22_attention_weight_max": 42.83798599243164,
      "activations/layer22_attention_weight_min": -31.73878288269043,
      "activations/layer23_attention_weight_max": 33.43369674682617,
      "activations/layer23_attention_weight_min": -28.79949188232422,
      "activations/layer2_attention_weight_max": 32.306190490722656,
      "activations/layer2_attention_weight_min": -31.5167236328125,
      "activations/layer3_attention_weight_max": 95.20045471191406,
      "activations/layer3_attention_weight_min": -95.55792236328125,
      "activations/layer4_attention_weight_max": 102.62529754638672,
      "activations/layer4_attention_weight_min": -109.6995620727539,
      "activations/layer5_attention_weight_max": 47.66517639160156,
      "activations/layer5_attention_weight_min": -63.06207275390625,
      "activations/layer6_attention_weight_max": 41.56325149536133,
      "activations/layer6_attention_weight_min": -45.1020622253418,
      "activations/layer7_attention_weight_max": 87.79734802246094,
      "activations/layer7_attention_weight_min": -86.78199005126953,
      "activations/layer8_attention_weight_max": 38.84678649902344,
      "activations/layer8_attention_weight_min": -41.084136962890625,
      "activations/layer9_attention_weight_max": 29.404088973999023,
      "activations/layer9_attention_weight_min": -32.65311813354492,
      "epoch": 17.58,
      "learning_rate": 3.702310606060606e-05,
      "loss": 2.7632,
      "step": 302500
    },
    {
      "activations/layer0_attention_weight_max": 16.684104919433594,
      "activations/layer0_attention_weight_min": -13.413536071777344,
      "activations/layer10_attention_weight_max": 32.2071533203125,
      "activations/layer10_attention_weight_min": -33.00244903564453,
      "activations/layer11_attention_weight_max": 32.9262809753418,
      "activations/layer11_attention_weight_min": -33.417320251464844,
      "activations/layer12_attention_weight_max": 25.81812286376953,
      "activations/layer12_attention_weight_min": -27.115310668945312,
      "activations/layer13_attention_weight_max": 40.104103088378906,
      "activations/layer13_attention_weight_min": -37.12825393676758,
      "activations/layer14_attention_weight_max": 40.026710510253906,
      "activations/layer14_attention_weight_min": -37.07870864868164,
      "activations/layer15_attention_weight_max": 42.46778869628906,
      "activations/layer15_attention_weight_min": -35.71839904785156,
      "activations/layer16_attention_weight_max": 31.706295013427734,
      "activations/layer16_attention_weight_min": -28.857866287231445,
      "activations/layer17_attention_weight_max": 29.75152015686035,
      "activations/layer17_attention_weight_min": -25.740617752075195,
      "activations/layer18_attention_weight_max": 32.8121223449707,
      "activations/layer18_attention_weight_min": -25.254552841186523,
      "activations/layer19_attention_weight_max": 33.02382278442383,
      "activations/layer19_attention_weight_min": -30.59990882873535,
      "activations/layer1_attention_weight_max": 15.829325675964355,
      "activations/layer1_attention_weight_min": -13.510245323181152,
      "activations/layer20_attention_weight_max": 27.618139266967773,
      "activations/layer20_attention_weight_min": -26.020601272583008,
      "activations/layer21_attention_weight_max": 26.974485397338867,
      "activations/layer21_attention_weight_min": -23.465970993041992,
      "activations/layer22_attention_weight_max": 40.71494674682617,
      "activations/layer22_attention_weight_min": -30.830684661865234,
      "activations/layer23_attention_weight_max": 31.692237854003906,
      "activations/layer23_attention_weight_min": -24.791479110717773,
      "activations/layer2_attention_weight_max": 33.36628341674805,
      "activations/layer2_attention_weight_min": -33.39081573486328,
      "activations/layer3_attention_weight_max": 95.50814056396484,
      "activations/layer3_attention_weight_min": -101.21572875976562,
      "activations/layer4_attention_weight_max": 102.64266204833984,
      "activations/layer4_attention_weight_min": -107.36959075927734,
      "activations/layer5_attention_weight_max": 50.804317474365234,
      "activations/layer5_attention_weight_min": -62.88349151611328,
      "activations/layer6_attention_weight_max": 42.16645431518555,
      "activations/layer6_attention_weight_min": -45.67829895019531,
      "activations/layer7_attention_weight_max": 87.76629638671875,
      "activations/layer7_attention_weight_min": -89.18072509765625,
      "activations/layer8_attention_weight_max": 39.563175201416016,
      "activations/layer8_attention_weight_min": -41.999935150146484,
      "activations/layer9_attention_weight_max": 31.497283935546875,
      "activations/layer9_attention_weight_min": -32.3756103515625,
      "epoch": 17.58,
      "learning_rate": 3.700416666666666e-05,
      "loss": 2.7223,
      "step": 302550
    },
    {
      "activations/layer0_attention_weight_max": 17.494050979614258,
      "activations/layer0_attention_weight_min": -13.20363998413086,
      "activations/layer10_attention_weight_max": 35.67721176147461,
      "activations/layer10_attention_weight_min": -35.797245025634766,
      "activations/layer11_attention_weight_max": 34.30584716796875,
      "activations/layer11_attention_weight_min": -35.8146858215332,
      "activations/layer12_attention_weight_max": 28.970840454101562,
      "activations/layer12_attention_weight_min": -28.996322631835938,
      "activations/layer13_attention_weight_max": 43.282615661621094,
      "activations/layer13_attention_weight_min": -38.884002685546875,
      "activations/layer14_attention_weight_max": 43.915931701660156,
      "activations/layer14_attention_weight_min": -39.02717590332031,
      "activations/layer15_attention_weight_max": 42.09080505371094,
      "activations/layer15_attention_weight_min": -36.94303894042969,
      "activations/layer16_attention_weight_max": 29.68513298034668,
      "activations/layer16_attention_weight_min": -28.72635841369629,
      "activations/layer17_attention_weight_max": 31.42330551147461,
      "activations/layer17_attention_weight_min": -29.331552505493164,
      "activations/layer18_attention_weight_max": 33.96106719970703,
      "activations/layer18_attention_weight_min": -27.48530387878418,
      "activations/layer19_attention_weight_max": 34.78123092651367,
      "activations/layer19_attention_weight_min": -31.194293975830078,
      "activations/layer1_attention_weight_max": 18.366859436035156,
      "activations/layer1_attention_weight_min": -15.05182933807373,
      "activations/layer20_attention_weight_max": 28.87604331970215,
      "activations/layer20_attention_weight_min": -26.456998825073242,
      "activations/layer21_attention_weight_max": 27.847585678100586,
      "activations/layer21_attention_weight_min": -25.183042526245117,
      "activations/layer22_attention_weight_max": 48.51655960083008,
      "activations/layer22_attention_weight_min": -33.480140686035156,
      "activations/layer23_attention_weight_max": 36.36079025268555,
      "activations/layer23_attention_weight_min": -27.082515716552734,
      "activations/layer2_attention_weight_max": 31.138111114501953,
      "activations/layer2_attention_weight_min": -31.998558044433594,
      "activations/layer3_attention_weight_max": 93.70909881591797,
      "activations/layer3_attention_weight_min": -97.3643798828125,
      "activations/layer4_attention_weight_max": 101.8592529296875,
      "activations/layer4_attention_weight_min": -108.70409393310547,
      "activations/layer5_attention_weight_max": 49.29948043823242,
      "activations/layer5_attention_weight_min": -60.6600341796875,
      "activations/layer6_attention_weight_max": 43.42048645019531,
      "activations/layer6_attention_weight_min": -47.74360656738281,
      "activations/layer7_attention_weight_max": 89.85514068603516,
      "activations/layer7_attention_weight_min": -100.61415100097656,
      "activations/layer8_attention_weight_max": 40.382606506347656,
      "activations/layer8_attention_weight_min": -44.83818435668945,
      "activations/layer9_attention_weight_max": 31.575820922851562,
      "activations/layer9_attention_weight_min": -34.370697021484375,
      "epoch": 17.58,
      "learning_rate": 3.698522727272727e-05,
      "loss": 2.7439,
      "step": 302600
    },
    {
      "activations/layer0_attention_weight_max": 16.244888305664062,
      "activations/layer0_attention_weight_min": -14.03744888305664,
      "activations/layer10_attention_weight_max": 34.43035888671875,
      "activations/layer10_attention_weight_min": -32.425167083740234,
      "activations/layer11_attention_weight_max": 33.86473846435547,
      "activations/layer11_attention_weight_min": -33.62022399902344,
      "activations/layer12_attention_weight_max": 28.836776733398438,
      "activations/layer12_attention_weight_min": -26.829015731811523,
      "activations/layer13_attention_weight_max": 40.012489318847656,
      "activations/layer13_attention_weight_min": -35.76536560058594,
      "activations/layer14_attention_weight_max": 42.58246994018555,
      "activations/layer14_attention_weight_min": -35.79431915283203,
      "activations/layer15_attention_weight_max": 37.45658493041992,
      "activations/layer15_attention_weight_min": -34.14042663574219,
      "activations/layer16_attention_weight_max": 28.452411651611328,
      "activations/layer16_attention_weight_min": -27.10055160522461,
      "activations/layer17_attention_weight_max": 30.609859466552734,
      "activations/layer17_attention_weight_min": -25.950389862060547,
      "activations/layer18_attention_weight_max": 33.737850189208984,
      "activations/layer18_attention_weight_min": -25.24100685119629,
      "activations/layer19_attention_weight_max": 33.32345199584961,
      "activations/layer19_attention_weight_min": -32.50004196166992,
      "activations/layer1_attention_weight_max": 16.750743865966797,
      "activations/layer1_attention_weight_min": -14.72529125213623,
      "activations/layer20_attention_weight_max": 27.751840591430664,
      "activations/layer20_attention_weight_min": -25.24599838256836,
      "activations/layer21_attention_weight_max": 27.78753662109375,
      "activations/layer21_attention_weight_min": -25.195098876953125,
      "activations/layer22_attention_weight_max": 39.29597854614258,
      "activations/layer22_attention_weight_min": -31.86163330078125,
      "activations/layer23_attention_weight_max": 29.68780517578125,
      "activations/layer23_attention_weight_min": -26.29969596862793,
      "activations/layer2_attention_weight_max": 32.02361297607422,
      "activations/layer2_attention_weight_min": -32.49940490722656,
      "activations/layer3_attention_weight_max": 99.37648010253906,
      "activations/layer3_attention_weight_min": -99.38337707519531,
      "activations/layer4_attention_weight_max": 114.33431243896484,
      "activations/layer4_attention_weight_min": -106.52125549316406,
      "activations/layer5_attention_weight_max": 52.020469665527344,
      "activations/layer5_attention_weight_min": -59.25285339355469,
      "activations/layer6_attention_weight_max": 45.395973205566406,
      "activations/layer6_attention_weight_min": -46.71321487426758,
      "activations/layer7_attention_weight_max": 97.14324188232422,
      "activations/layer7_attention_weight_min": -92.9424819946289,
      "activations/layer8_attention_weight_max": 41.18091583251953,
      "activations/layer8_attention_weight_min": -43.94606399536133,
      "activations/layer9_attention_weight_max": 32.08042526245117,
      "activations/layer9_attention_weight_min": -32.95988845825195,
      "epoch": 17.59,
      "learning_rate": 3.696628787878788e-05,
      "loss": 2.7447,
      "step": 302650
    },
    {
      "activations/layer0_attention_weight_max": 16.220321655273438,
      "activations/layer0_attention_weight_min": -12.263877868652344,
      "activations/layer10_attention_weight_max": 47.70853042602539,
      "activations/layer10_attention_weight_min": -47.87625503540039,
      "activations/layer11_attention_weight_max": 49.178916931152344,
      "activations/layer11_attention_weight_min": -50.013816833496094,
      "activations/layer12_attention_weight_max": 32.6883430480957,
      "activations/layer12_attention_weight_min": -32.25864791870117,
      "activations/layer13_attention_weight_max": 52.328678131103516,
      "activations/layer13_attention_weight_min": -44.16295623779297,
      "activations/layer14_attention_weight_max": 53.081085205078125,
      "activations/layer14_attention_weight_min": -46.737308502197266,
      "activations/layer15_attention_weight_max": 53.84579849243164,
      "activations/layer15_attention_weight_min": -43.217201232910156,
      "activations/layer16_attention_weight_max": 34.613826751708984,
      "activations/layer16_attention_weight_min": -29.180370330810547,
      "activations/layer17_attention_weight_max": 29.149038314819336,
      "activations/layer17_attention_weight_min": -30.43079948425293,
      "activations/layer18_attention_weight_max": 31.184696197509766,
      "activations/layer18_attention_weight_min": -29.989126205444336,
      "activations/layer19_attention_weight_max": 33.135223388671875,
      "activations/layer19_attention_weight_min": -34.707332611083984,
      "activations/layer1_attention_weight_max": 15.987299919128418,
      "activations/layer1_attention_weight_min": -14.404255867004395,
      "activations/layer20_attention_weight_max": 27.673595428466797,
      "activations/layer20_attention_weight_min": -27.900527954101562,
      "activations/layer21_attention_weight_max": 29.813467025756836,
      "activations/layer21_attention_weight_min": -25.833616256713867,
      "activations/layer22_attention_weight_max": 43.1428108215332,
      "activations/layer22_attention_weight_min": -32.14894104003906,
      "activations/layer23_attention_weight_max": 31.36913299560547,
      "activations/layer23_attention_weight_min": -27.068832397460938,
      "activations/layer2_attention_weight_max": 33.222557067871094,
      "activations/layer2_attention_weight_min": -32.053470611572266,
      "activations/layer3_attention_weight_max": 102.10407257080078,
      "activations/layer3_attention_weight_min": -100.5551986694336,
      "activations/layer4_attention_weight_max": 106.7503433227539,
      "activations/layer4_attention_weight_min": -106.81291961669922,
      "activations/layer5_attention_weight_max": 50.41599655151367,
      "activations/layer5_attention_weight_min": -60.38433074951172,
      "activations/layer6_attention_weight_max": 46.389896392822266,
      "activations/layer6_attention_weight_min": -47.62980270385742,
      "activations/layer7_attention_weight_max": 111.39167785644531,
      "activations/layer7_attention_weight_min": -101.25993347167969,
      "activations/layer8_attention_weight_max": 54.37213134765625,
      "activations/layer8_attention_weight_min": -53.625789642333984,
      "activations/layer9_attention_weight_max": 42.82955551147461,
      "activations/layer9_attention_weight_min": -45.60856628417969,
      "epoch": 17.59,
      "learning_rate": 3.6947348484848485e-05,
      "loss": 2.7455,
      "step": 302700
    },
    {
      "activations/layer0_attention_weight_max": 16.010515213012695,
      "activations/layer0_attention_weight_min": -11.945636749267578,
      "activations/layer10_attention_weight_max": 32.433326721191406,
      "activations/layer10_attention_weight_min": -33.95869064331055,
      "activations/layer11_attention_weight_max": 32.58889389038086,
      "activations/layer11_attention_weight_min": -34.10254669189453,
      "activations/layer12_attention_weight_max": 25.6149959564209,
      "activations/layer12_attention_weight_min": -25.78238868713379,
      "activations/layer13_attention_weight_max": 37.5821647644043,
      "activations/layer13_attention_weight_min": -34.55648422241211,
      "activations/layer14_attention_weight_max": 39.6646614074707,
      "activations/layer14_attention_weight_min": -36.67300033569336,
      "activations/layer15_attention_weight_max": 37.891963958740234,
      "activations/layer15_attention_weight_min": -34.0026969909668,
      "activations/layer16_attention_weight_max": 27.580522537231445,
      "activations/layer16_attention_weight_min": -27.815519332885742,
      "activations/layer17_attention_weight_max": 28.743154525756836,
      "activations/layer17_attention_weight_min": -25.46641731262207,
      "activations/layer18_attention_weight_max": 29.804067611694336,
      "activations/layer18_attention_weight_min": -27.833635330200195,
      "activations/layer19_attention_weight_max": 34.29685974121094,
      "activations/layer19_attention_weight_min": -29.91680335998535,
      "activations/layer1_attention_weight_max": 16.44129180908203,
      "activations/layer1_attention_weight_min": -15.047926902770996,
      "activations/layer20_attention_weight_max": 27.01654052734375,
      "activations/layer20_attention_weight_min": -24.910594940185547,
      "activations/layer21_attention_weight_max": 26.717178344726562,
      "activations/layer21_attention_weight_min": -25.638427734375,
      "activations/layer22_attention_weight_max": 40.629913330078125,
      "activations/layer22_attention_weight_min": -30.359540939331055,
      "activations/layer23_attention_weight_max": 30.18794822692871,
      "activations/layer23_attention_weight_min": -26.021099090576172,
      "activations/layer2_attention_weight_max": 32.13828659057617,
      "activations/layer2_attention_weight_min": -31.431303024291992,
      "activations/layer3_attention_weight_max": 97.4533462524414,
      "activations/layer3_attention_weight_min": -96.24652862548828,
      "activations/layer4_attention_weight_max": 102.44615936279297,
      "activations/layer4_attention_weight_min": -107.63903045654297,
      "activations/layer5_attention_weight_max": 46.53082275390625,
      "activations/layer5_attention_weight_min": -59.277095794677734,
      "activations/layer6_attention_weight_max": 41.92292785644531,
      "activations/layer6_attention_weight_min": -42.896907806396484,
      "activations/layer7_attention_weight_max": 88.56816864013672,
      "activations/layer7_attention_weight_min": -91.88575744628906,
      "activations/layer8_attention_weight_max": 39.645633697509766,
      "activations/layer8_attention_weight_min": -41.300193786621094,
      "activations/layer9_attention_weight_max": 32.532936096191406,
      "activations/layer9_attention_weight_min": -33.84410095214844,
      "epoch": 17.59,
      "learning_rate": 3.692840909090909e-05,
      "loss": 2.7285,
      "step": 302750
    },
    {
      "activations/layer0_attention_weight_max": 15.654094696044922,
      "activations/layer0_attention_weight_min": -12.817804336547852,
      "activations/layer10_attention_weight_max": 36.527252197265625,
      "activations/layer10_attention_weight_min": -34.334991455078125,
      "activations/layer11_attention_weight_max": 36.939170837402344,
      "activations/layer11_attention_weight_min": -34.13964080810547,
      "activations/layer12_attention_weight_max": 23.911258697509766,
      "activations/layer12_attention_weight_min": -25.905353546142578,
      "activations/layer13_attention_weight_max": 35.63011932373047,
      "activations/layer13_attention_weight_min": -34.938255310058594,
      "activations/layer14_attention_weight_max": 37.9010124206543,
      "activations/layer14_attention_weight_min": -36.93363952636719,
      "activations/layer15_attention_weight_max": 33.77443313598633,
      "activations/layer15_attention_weight_min": -34.123512268066406,
      "activations/layer16_attention_weight_max": 26.25505828857422,
      "activations/layer16_attention_weight_min": -27.98048973083496,
      "activations/layer17_attention_weight_max": 27.004344940185547,
      "activations/layer17_attention_weight_min": -25.489259719848633,
      "activations/layer18_attention_weight_max": 30.17536735534668,
      "activations/layer18_attention_weight_min": -26.136249542236328,
      "activations/layer19_attention_weight_max": 31.037338256835938,
      "activations/layer19_attention_weight_min": -30.843368530273438,
      "activations/layer1_attention_weight_max": 15.96597671508789,
      "activations/layer1_attention_weight_min": -13.74205207824707,
      "activations/layer20_attention_weight_max": 25.40517807006836,
      "activations/layer20_attention_weight_min": -25.39196014404297,
      "activations/layer21_attention_weight_max": 25.925443649291992,
      "activations/layer21_attention_weight_min": -25.718826293945312,
      "activations/layer22_attention_weight_max": 39.91211700439453,
      "activations/layer22_attention_weight_min": -31.467105865478516,
      "activations/layer23_attention_weight_max": 28.51012420654297,
      "activations/layer23_attention_weight_min": -25.535568237304688,
      "activations/layer2_attention_weight_max": 30.03046417236328,
      "activations/layer2_attention_weight_min": -29.409561157226562,
      "activations/layer3_attention_weight_max": 89.990966796875,
      "activations/layer3_attention_weight_min": -88.18974304199219,
      "activations/layer4_attention_weight_max": 95.56876373291016,
      "activations/layer4_attention_weight_min": -96.61376953125,
      "activations/layer5_attention_weight_max": 48.98426818847656,
      "activations/layer5_attention_weight_min": -56.4192008972168,
      "activations/layer6_attention_weight_max": 40.62657165527344,
      "activations/layer6_attention_weight_min": -45.54118728637695,
      "activations/layer7_attention_weight_max": 97.73932647705078,
      "activations/layer7_attention_weight_min": -88.26210021972656,
      "activations/layer8_attention_weight_max": 43.98198318481445,
      "activations/layer8_attention_weight_min": -42.541168212890625,
      "activations/layer9_attention_weight_max": 35.25851058959961,
      "activations/layer9_attention_weight_min": -33.46854782104492,
      "epoch": 17.59,
      "learning_rate": 3.6909469696969695e-05,
      "loss": 2.7323,
      "step": 302800
    },
    {
      "activations/layer0_attention_weight_max": 17.39298439025879,
      "activations/layer0_attention_weight_min": -13.542010307312012,
      "activations/layer10_attention_weight_max": 32.983062744140625,
      "activations/layer10_attention_weight_min": -33.65915298461914,
      "activations/layer11_attention_weight_max": 33.04940414428711,
      "activations/layer11_attention_weight_min": -34.5712776184082,
      "activations/layer12_attention_weight_max": 26.103408813476562,
      "activations/layer12_attention_weight_min": -27.85628890991211,
      "activations/layer13_attention_weight_max": 38.337764739990234,
      "activations/layer13_attention_weight_min": -35.96577835083008,
      "activations/layer14_attention_weight_max": 42.404197692871094,
      "activations/layer14_attention_weight_min": -38.34300994873047,
      "activations/layer15_attention_weight_max": 36.831844329833984,
      "activations/layer15_attention_weight_min": -35.44477081298828,
      "activations/layer16_attention_weight_max": 31.046268463134766,
      "activations/layer16_attention_weight_min": -28.528059005737305,
      "activations/layer17_attention_weight_max": 28.46402359008789,
      "activations/layer17_attention_weight_min": -26.368247985839844,
      "activations/layer18_attention_weight_max": 32.304141998291016,
      "activations/layer18_attention_weight_min": -28.6754207611084,
      "activations/layer19_attention_weight_max": 34.340938568115234,
      "activations/layer19_attention_weight_min": -31.845767974853516,
      "activations/layer1_attention_weight_max": 16.767179489135742,
      "activations/layer1_attention_weight_min": -15.383869171142578,
      "activations/layer20_attention_weight_max": 30.580026626586914,
      "activations/layer20_attention_weight_min": -26.50349998474121,
      "activations/layer21_attention_weight_max": 30.14364242553711,
      "activations/layer21_attention_weight_min": -27.08072853088379,
      "activations/layer22_attention_weight_max": 43.16138458251953,
      "activations/layer22_attention_weight_min": -34.549495697021484,
      "activations/layer23_attention_weight_max": 32.14558029174805,
      "activations/layer23_attention_weight_min": -25.4201717376709,
      "activations/layer2_attention_weight_max": 33.16334533691406,
      "activations/layer2_attention_weight_min": -31.911117553710938,
      "activations/layer3_attention_weight_max": 94.79193878173828,
      "activations/layer3_attention_weight_min": -98.42430114746094,
      "activations/layer4_attention_weight_max": 100.10405731201172,
      "activations/layer4_attention_weight_min": -102.53435516357422,
      "activations/layer5_attention_weight_max": 49.70852279663086,
      "activations/layer5_attention_weight_min": -60.62278366088867,
      "activations/layer6_attention_weight_max": 40.7364387512207,
      "activations/layer6_attention_weight_min": -44.58559036254883,
      "activations/layer7_attention_weight_max": 90.24534606933594,
      "activations/layer7_attention_weight_min": -93.91446685791016,
      "activations/layer8_attention_weight_max": 38.29356384277344,
      "activations/layer8_attention_weight_min": -41.714088439941406,
      "activations/layer9_attention_weight_max": 30.55096435546875,
      "activations/layer9_attention_weight_min": -32.93559646606445,
      "epoch": 17.6,
      "learning_rate": 3.6890530303030296e-05,
      "loss": 2.7535,
      "step": 302850
    },
    {
      "activations/layer0_attention_weight_max": 16.18307113647461,
      "activations/layer0_attention_weight_min": -13.848738670349121,
      "activations/layer10_attention_weight_max": 33.75385665893555,
      "activations/layer10_attention_weight_min": -34.761817932128906,
      "activations/layer11_attention_weight_max": 33.41261291503906,
      "activations/layer11_attention_weight_min": -34.92902374267578,
      "activations/layer12_attention_weight_max": 24.934616088867188,
      "activations/layer12_attention_weight_min": -25.67342758178711,
      "activations/layer13_attention_weight_max": 37.59545135498047,
      "activations/layer13_attention_weight_min": -34.12774658203125,
      "activations/layer14_attention_weight_max": 38.50275802612305,
      "activations/layer14_attention_weight_min": -37.663787841796875,
      "activations/layer15_attention_weight_max": 35.84246826171875,
      "activations/layer15_attention_weight_min": -35.66693878173828,
      "activations/layer16_attention_weight_max": 28.132240295410156,
      "activations/layer16_attention_weight_min": -28.25706672668457,
      "activations/layer17_attention_weight_max": 26.171653747558594,
      "activations/layer17_attention_weight_min": -25.341299057006836,
      "activations/layer18_attention_weight_max": 31.407554626464844,
      "activations/layer18_attention_weight_min": -27.605928421020508,
      "activations/layer19_attention_weight_max": 31.0446834564209,
      "activations/layer19_attention_weight_min": -30.8994197845459,
      "activations/layer1_attention_weight_max": 15.980552673339844,
      "activations/layer1_attention_weight_min": -14.997771263122559,
      "activations/layer20_attention_weight_max": 24.299047470092773,
      "activations/layer20_attention_weight_min": -25.347875595092773,
      "activations/layer21_attention_weight_max": 25.924596786499023,
      "activations/layer21_attention_weight_min": -25.87462043762207,
      "activations/layer22_attention_weight_max": 37.81631088256836,
      "activations/layer22_attention_weight_min": -31.655664443969727,
      "activations/layer23_attention_weight_max": 29.42230224609375,
      "activations/layer23_attention_weight_min": -27.179677963256836,
      "activations/layer2_attention_weight_max": 32.179718017578125,
      "activations/layer2_attention_weight_min": -33.30875015258789,
      "activations/layer3_attention_weight_max": 100.71621704101562,
      "activations/layer3_attention_weight_min": -102.90565490722656,
      "activations/layer4_attention_weight_max": 108.2635726928711,
      "activations/layer4_attention_weight_min": -107.29608154296875,
      "activations/layer5_attention_weight_max": 51.52895736694336,
      "activations/layer5_attention_weight_min": -60.37140655517578,
      "activations/layer6_attention_weight_max": 45.17528533935547,
      "activations/layer6_attention_weight_min": -46.532527923583984,
      "activations/layer7_attention_weight_max": 97.00863647460938,
      "activations/layer7_attention_weight_min": -98.06037902832031,
      "activations/layer8_attention_weight_max": 43.50811767578125,
      "activations/layer8_attention_weight_min": -45.68014144897461,
      "activations/layer9_attention_weight_max": 33.92204666137695,
      "activations/layer9_attention_weight_min": -35.18525314331055,
      "epoch": 17.6,
      "learning_rate": 3.6871590909090905e-05,
      "loss": 2.7312,
      "step": 302900
    },
    {
      "activations/layer0_attention_weight_max": 17.02664566040039,
      "activations/layer0_attention_weight_min": -12.257668495178223,
      "activations/layer10_attention_weight_max": 37.650150299072266,
      "activations/layer10_attention_weight_min": -34.326107025146484,
      "activations/layer11_attention_weight_max": 38.49760437011719,
      "activations/layer11_attention_weight_min": -35.9187126159668,
      "activations/layer12_attention_weight_max": 26.249174118041992,
      "activations/layer12_attention_weight_min": -26.09803581237793,
      "activations/layer13_attention_weight_max": 45.81441879272461,
      "activations/layer13_attention_weight_min": -35.98932647705078,
      "activations/layer14_attention_weight_max": 49.65623092651367,
      "activations/layer14_attention_weight_min": -37.653865814208984,
      "activations/layer15_attention_weight_max": 47.31330871582031,
      "activations/layer15_attention_weight_min": -37.73384475708008,
      "activations/layer16_attention_weight_max": 32.51515579223633,
      "activations/layer16_attention_weight_min": -27.510854721069336,
      "activations/layer17_attention_weight_max": 32.01796340942383,
      "activations/layer17_attention_weight_min": -27.869993209838867,
      "activations/layer18_attention_weight_max": 33.63017654418945,
      "activations/layer18_attention_weight_min": -27.189401626586914,
      "activations/layer19_attention_weight_max": 36.528812408447266,
      "activations/layer19_attention_weight_min": -29.161762237548828,
      "activations/layer1_attention_weight_max": 16.56842613220215,
      "activations/layer1_attention_weight_min": -15.329408645629883,
      "activations/layer20_attention_weight_max": 32.060455322265625,
      "activations/layer20_attention_weight_min": -23.905990600585938,
      "activations/layer21_attention_weight_max": 34.9459228515625,
      "activations/layer21_attention_weight_min": -24.084863662719727,
      "activations/layer22_attention_weight_max": 42.239322662353516,
      "activations/layer22_attention_weight_min": -32.57712936401367,
      "activations/layer23_attention_weight_max": 33.52699661254883,
      "activations/layer23_attention_weight_min": -24.1219539642334,
      "activations/layer2_attention_weight_max": 35.88947296142578,
      "activations/layer2_attention_weight_min": -33.06959915161133,
      "activations/layer3_attention_weight_max": 104.06670379638672,
      "activations/layer3_attention_weight_min": -100.4267578125,
      "activations/layer4_attention_weight_max": 105.84552001953125,
      "activations/layer4_attention_weight_min": -105.50390625,
      "activations/layer5_attention_weight_max": 50.89146041870117,
      "activations/layer5_attention_weight_min": -63.37461471557617,
      "activations/layer6_attention_weight_max": 42.21244812011719,
      "activations/layer6_attention_weight_min": -46.003299713134766,
      "activations/layer7_attention_weight_max": 91.45101928710938,
      "activations/layer7_attention_weight_min": -91.7456283569336,
      "activations/layer8_attention_weight_max": 41.62097930908203,
      "activations/layer8_attention_weight_min": -41.83616638183594,
      "activations/layer9_attention_weight_max": 32.55032730102539,
      "activations/layer9_attention_weight_min": -33.185245513916016,
      "epoch": 17.6,
      "learning_rate": 3.685265151515151e-05,
      "loss": 2.7248,
      "step": 302950
    },
    {
      "activations/layer0_attention_weight_max": 16.21811294555664,
      "activations/layer0_attention_weight_min": -12.680547714233398,
      "activations/layer10_attention_weight_max": 34.90903091430664,
      "activations/layer10_attention_weight_min": -36.72179412841797,
      "activations/layer11_attention_weight_max": 38.10874938964844,
      "activations/layer11_attention_weight_min": -36.611045837402344,
      "activations/layer12_attention_weight_max": 27.745765686035156,
      "activations/layer12_attention_weight_min": -26.840845108032227,
      "activations/layer13_attention_weight_max": 42.151634216308594,
      "activations/layer13_attention_weight_min": -36.84037780761719,
      "activations/layer14_attention_weight_max": 45.01866912841797,
      "activations/layer14_attention_weight_min": -39.85138702392578,
      "activations/layer15_attention_weight_max": 46.4734001159668,
      "activations/layer15_attention_weight_min": -37.809059143066406,
      "activations/layer16_attention_weight_max": 34.00667190551758,
      "activations/layer16_attention_weight_min": -28.83905029296875,
      "activations/layer17_attention_weight_max": 33.71731948852539,
      "activations/layer17_attention_weight_min": -26.46062469482422,
      "activations/layer18_attention_weight_max": 34.56575012207031,
      "activations/layer18_attention_weight_min": -26.512510299682617,
      "activations/layer19_attention_weight_max": 36.39762496948242,
      "activations/layer19_attention_weight_min": -31.592878341674805,
      "activations/layer1_attention_weight_max": 15.900080680847168,
      "activations/layer1_attention_weight_min": -14.613106727600098,
      "activations/layer20_attention_weight_max": 31.698240280151367,
      "activations/layer20_attention_weight_min": -26.758743286132812,
      "activations/layer21_attention_weight_max": 33.3121452331543,
      "activations/layer21_attention_weight_min": -27.048797607421875,
      "activations/layer22_attention_weight_max": 45.8398551940918,
      "activations/layer22_attention_weight_min": -32.183555603027344,
      "activations/layer23_attention_weight_max": 36.43888854980469,
      "activations/layer23_attention_weight_min": -26.04922866821289,
      "activations/layer2_attention_weight_max": 34.571533203125,
      "activations/layer2_attention_weight_min": -34.076499938964844,
      "activations/layer3_attention_weight_max": 104.82865142822266,
      "activations/layer3_attention_weight_min": -107.35725402832031,
      "activations/layer4_attention_weight_max": 113.66716766357422,
      "activations/layer4_attention_weight_min": -114.44005584716797,
      "activations/layer5_attention_weight_max": 52.4563102722168,
      "activations/layer5_attention_weight_min": -62.05366516113281,
      "activations/layer6_attention_weight_max": 46.64667510986328,
      "activations/layer6_attention_weight_min": -47.70465850830078,
      "activations/layer7_attention_weight_max": 98.64543914794922,
      "activations/layer7_attention_weight_min": -100.05410766601562,
      "activations/layer8_attention_weight_max": 43.59790802001953,
      "activations/layer8_attention_weight_min": -46.80998611450195,
      "activations/layer9_attention_weight_max": 33.79380416870117,
      "activations/layer9_attention_weight_min": -36.130489349365234,
      "epoch": 17.61,
      "learning_rate": 3.683371212121212e-05,
      "loss": 2.7215,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4653,
      "eval_samples_per_second": 507.245,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4653,
      "eval_openwebtext_samples_per_second": 507.245,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9597,
      "eval_wikitext_samples_per_second": 232.688,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_lambada_loss": 2.3984375,
      "eval_lambada_ppl": 11.005966116567095,
      "eval_lambada_runtime": 9.5374,
      "eval_lambada_samples_per_second": 510.514,
      "step": 303000
    },
    {
      "activations/layer0_attention_weight_max": 16.929841995239258,
      "activations/layer0_attention_weight_min": -14.050187110900879,
      "activations/layer10_attention_weight_max": 34.56208419799805,
      "activations/layer10_attention_weight_min": -34.88148498535156,
      "activations/layer11_attention_weight_max": 35.65238952636719,
      "activations/layer11_attention_weight_min": -35.11200714111328,
      "activations/layer12_attention_weight_max": 28.26158905029297,
      "activations/layer12_attention_weight_min": -27.297025680541992,
      "activations/layer13_attention_weight_max": 41.25025939941406,
      "activations/layer13_attention_weight_min": -35.68122100830078,
      "activations/layer14_attention_weight_max": 47.985897064208984,
      "activations/layer14_attention_weight_min": -39.02872848510742,
      "activations/layer15_attention_weight_max": 39.694984436035156,
      "activations/layer15_attention_weight_min": -36.57463073730469,
      "activations/layer16_attention_weight_max": 31.09621238708496,
      "activations/layer16_attention_weight_min": -30.514680862426758,
      "activations/layer17_attention_weight_max": 31.56839942932129,
      "activations/layer17_attention_weight_min": -26.934772491455078,
      "activations/layer18_attention_weight_max": 33.01890563964844,
      "activations/layer18_attention_weight_min": -28.775846481323242,
      "activations/layer19_attention_weight_max": 33.47257614135742,
      "activations/layer19_attention_weight_min": -33.173492431640625,
      "activations/layer1_attention_weight_max": 16.03661346435547,
      "activations/layer1_attention_weight_min": -14.67186164855957,
      "activations/layer20_attention_weight_max": 30.960824966430664,
      "activations/layer20_attention_weight_min": -25.339670181274414,
      "activations/layer21_attention_weight_max": 27.79056739807129,
      "activations/layer21_attention_weight_min": -25.70162010192871,
      "activations/layer22_attention_weight_max": 48.93212890625,
      "activations/layer22_attention_weight_min": -32.59761047363281,
      "activations/layer23_attention_weight_max": 34.78204345703125,
      "activations/layer23_attention_weight_min": -27.81122589111328,
      "activations/layer2_attention_weight_max": 30.964580535888672,
      "activations/layer2_attention_weight_min": -32.01056671142578,
      "activations/layer3_attention_weight_max": 91.08414459228516,
      "activations/layer3_attention_weight_min": -94.78703308105469,
      "activations/layer4_attention_weight_max": 103.01220703125,
      "activations/layer4_attention_weight_min": -108.82763671875,
      "activations/layer5_attention_weight_max": 51.25968551635742,
      "activations/layer5_attention_weight_min": -60.80084991455078,
      "activations/layer6_attention_weight_max": 43.614131927490234,
      "activations/layer6_attention_weight_min": -46.68925476074219,
      "activations/layer7_attention_weight_max": 94.14444732666016,
      "activations/layer7_attention_weight_min": -95.09339904785156,
      "activations/layer8_attention_weight_max": 40.13570785522461,
      "activations/layer8_attention_weight_min": -44.8651237487793,
      "activations/layer9_attention_weight_max": 34.039730072021484,
      "activations/layer9_attention_weight_min": -35.90821838378906,
      "epoch": 17.61,
      "learning_rate": 3.681477272727272e-05,
      "loss": 2.7409,
      "step": 303050
    },
    {
      "activations/layer0_attention_weight_max": 16.618732452392578,
      "activations/layer0_attention_weight_min": -15.780471801757812,
      "activations/layer10_attention_weight_max": 33.001556396484375,
      "activations/layer10_attention_weight_min": -34.4720573425293,
      "activations/layer11_attention_weight_max": 33.63787078857422,
      "activations/layer11_attention_weight_min": -34.18074035644531,
      "activations/layer12_attention_weight_max": 23.421422958374023,
      "activations/layer12_attention_weight_min": -24.583532333374023,
      "activations/layer13_attention_weight_max": 34.76880645751953,
      "activations/layer13_attention_weight_min": -33.48082733154297,
      "activations/layer14_attention_weight_max": 38.19181823730469,
      "activations/layer14_attention_weight_min": -33.676841735839844,
      "activations/layer15_attention_weight_max": 34.89755630493164,
      "activations/layer15_attention_weight_min": -32.64347457885742,
      "activations/layer16_attention_weight_max": 27.419559478759766,
      "activations/layer16_attention_weight_min": -27.850048065185547,
      "activations/layer17_attention_weight_max": 27.122102737426758,
      "activations/layer17_attention_weight_min": -25.603242874145508,
      "activations/layer18_attention_weight_max": 32.358177185058594,
      "activations/layer18_attention_weight_min": -25.670785903930664,
      "activations/layer19_attention_weight_max": 31.370405197143555,
      "activations/layer19_attention_weight_min": -30.51371192932129,
      "activations/layer1_attention_weight_max": 15.90821361541748,
      "activations/layer1_attention_weight_min": -14.339231491088867,
      "activations/layer20_attention_weight_max": 27.852787017822266,
      "activations/layer20_attention_weight_min": -25.335647583007812,
      "activations/layer21_attention_weight_max": 27.497390747070312,
      "activations/layer21_attention_weight_min": -27.035776138305664,
      "activations/layer22_attention_weight_max": 40.924739837646484,
      "activations/layer22_attention_weight_min": -31.41250991821289,
      "activations/layer23_attention_weight_max": 33.44105529785156,
      "activations/layer23_attention_weight_min": -27.421085357666016,
      "activations/layer2_attention_weight_max": 32.946163177490234,
      "activations/layer2_attention_weight_min": -32.06908416748047,
      "activations/layer3_attention_weight_max": 99.36345672607422,
      "activations/layer3_attention_weight_min": -100.19593811035156,
      "activations/layer4_attention_weight_max": 103.75838470458984,
      "activations/layer4_attention_weight_min": -107.47370910644531,
      "activations/layer5_attention_weight_max": 50.48655700683594,
      "activations/layer5_attention_weight_min": -63.43288803100586,
      "activations/layer6_attention_weight_max": 43.46913146972656,
      "activations/layer6_attention_weight_min": -45.565879821777344,
      "activations/layer7_attention_weight_max": 94.1825180053711,
      "activations/layer7_attention_weight_min": -92.59759521484375,
      "activations/layer8_attention_weight_max": 45.107643127441406,
      "activations/layer8_attention_weight_min": -40.7298583984375,
      "activations/layer9_attention_weight_max": 32.17070007324219,
      "activations/layer9_attention_weight_min": -32.655513763427734,
      "epoch": 17.61,
      "learning_rate": 3.679583333333333e-05,
      "loss": 2.7436,
      "step": 303100
    },
    {
      "activations/layer0_attention_weight_max": 16.87647819519043,
      "activations/layer0_attention_weight_min": -13.201512336730957,
      "activations/layer10_attention_weight_max": 33.291603088378906,
      "activations/layer10_attention_weight_min": -33.517860412597656,
      "activations/layer11_attention_weight_max": 32.94818115234375,
      "activations/layer11_attention_weight_min": -33.21253204345703,
      "activations/layer12_attention_weight_max": 24.511428833007812,
      "activations/layer12_attention_weight_min": -26.681503295898438,
      "activations/layer13_attention_weight_max": 37.439910888671875,
      "activations/layer13_attention_weight_min": -36.28481674194336,
      "activations/layer14_attention_weight_max": 40.293678283691406,
      "activations/layer14_attention_weight_min": -40.49275207519531,
      "activations/layer15_attention_weight_max": 36.34798812866211,
      "activations/layer15_attention_weight_min": -36.25780487060547,
      "activations/layer16_attention_weight_max": 29.500722885131836,
      "activations/layer16_attention_weight_min": -28.102901458740234,
      "activations/layer17_attention_weight_max": 29.63340950012207,
      "activations/layer17_attention_weight_min": -25.710800170898438,
      "activations/layer18_attention_weight_max": 31.462158203125,
      "activations/layer18_attention_weight_min": -25.158145904541016,
      "activations/layer19_attention_weight_max": 32.512603759765625,
      "activations/layer19_attention_weight_min": -32.1223258972168,
      "activations/layer1_attention_weight_max": 15.736016273498535,
      "activations/layer1_attention_weight_min": -14.982705116271973,
      "activations/layer20_attention_weight_max": 28.15779685974121,
      "activations/layer20_attention_weight_min": -26.552391052246094,
      "activations/layer21_attention_weight_max": 28.24479103088379,
      "activations/layer21_attention_weight_min": -26.404170989990234,
      "activations/layer22_attention_weight_max": 39.582698822021484,
      "activations/layer22_attention_weight_min": -32.812557220458984,
      "activations/layer23_attention_weight_max": 32.5402946472168,
      "activations/layer23_attention_weight_min": -26.707794189453125,
      "activations/layer2_attention_weight_max": 32.626808166503906,
      "activations/layer2_attention_weight_min": -33.178443908691406,
      "activations/layer3_attention_weight_max": 95.65885162353516,
      "activations/layer3_attention_weight_min": -100.3109130859375,
      "activations/layer4_attention_weight_max": 103.04021453857422,
      "activations/layer4_attention_weight_min": -111.37078857421875,
      "activations/layer5_attention_weight_max": 51.97886657714844,
      "activations/layer5_attention_weight_min": -62.33135986328125,
      "activations/layer6_attention_weight_max": 43.74264907836914,
      "activations/layer6_attention_weight_min": -47.9191780090332,
      "activations/layer7_attention_weight_max": 93.61357116699219,
      "activations/layer7_attention_weight_min": -97.92279052734375,
      "activations/layer8_attention_weight_max": 42.78675842285156,
      "activations/layer8_attention_weight_min": -44.35338592529297,
      "activations/layer9_attention_weight_max": 31.64715003967285,
      "activations/layer9_attention_weight_min": -34.3802375793457,
      "epoch": 17.61,
      "learning_rate": 3.677689393939394e-05,
      "loss": 2.7412,
      "step": 303150
    },
    {
      "activations/layer0_attention_weight_max": 17.274484634399414,
      "activations/layer0_attention_weight_min": -13.83271598815918,
      "activations/layer10_attention_weight_max": 35.48878860473633,
      "activations/layer10_attention_weight_min": -36.225440979003906,
      "activations/layer11_attention_weight_max": 35.20416259765625,
      "activations/layer11_attention_weight_min": -38.528900146484375,
      "activations/layer12_attention_weight_max": 26.548011779785156,
      "activations/layer12_attention_weight_min": -26.170503616333008,
      "activations/layer13_attention_weight_max": 37.06341552734375,
      "activations/layer13_attention_weight_min": -36.10320281982422,
      "activations/layer14_attention_weight_max": 41.855690002441406,
      "activations/layer14_attention_weight_min": -37.74939727783203,
      "activations/layer15_attention_weight_max": 36.11759948730469,
      "activations/layer15_attention_weight_min": -36.70367431640625,
      "activations/layer16_attention_weight_max": 28.856340408325195,
      "activations/layer16_attention_weight_min": -27.50433921813965,
      "activations/layer17_attention_weight_max": 27.83561134338379,
      "activations/layer17_attention_weight_min": -26.773954391479492,
      "activations/layer18_attention_weight_max": 31.2957820892334,
      "activations/layer18_attention_weight_min": -26.715364456176758,
      "activations/layer19_attention_weight_max": 33.13453674316406,
      "activations/layer19_attention_weight_min": -31.15604019165039,
      "activations/layer1_attention_weight_max": 18.771141052246094,
      "activations/layer1_attention_weight_min": -16.61573600769043,
      "activations/layer20_attention_weight_max": 29.902114868164062,
      "activations/layer20_attention_weight_min": -26.697364807128906,
      "activations/layer21_attention_weight_max": 30.466964721679688,
      "activations/layer21_attention_weight_min": -26.84359359741211,
      "activations/layer22_attention_weight_max": 42.74882125854492,
      "activations/layer22_attention_weight_min": -32.5203742980957,
      "activations/layer23_attention_weight_max": 33.720420837402344,
      "activations/layer23_attention_weight_min": -27.615310668945312,
      "activations/layer2_attention_weight_max": 34.86920928955078,
      "activations/layer2_attention_weight_min": -33.47007751464844,
      "activations/layer3_attention_weight_max": 107.61924743652344,
      "activations/layer3_attention_weight_min": -109.72876739501953,
      "activations/layer4_attention_weight_max": 110.04302215576172,
      "activations/layer4_attention_weight_min": -115.63793182373047,
      "activations/layer5_attention_weight_max": 54.34214782714844,
      "activations/layer5_attention_weight_min": -63.254268646240234,
      "activations/layer6_attention_weight_max": 49.170413970947266,
      "activations/layer6_attention_weight_min": -50.461910247802734,
      "activations/layer7_attention_weight_max": 95.3482666015625,
      "activations/layer7_attention_weight_min": -97.93470001220703,
      "activations/layer8_attention_weight_max": 42.48988723754883,
      "activations/layer8_attention_weight_min": -48.34490203857422,
      "activations/layer9_attention_weight_max": 33.41964340209961,
      "activations/layer9_attention_weight_min": -36.94764709472656,
      "epoch": 17.62,
      "learning_rate": 3.675795454545454e-05,
      "loss": 2.753,
      "step": 303200
    },
    {
      "activations/layer0_attention_weight_max": 15.502248764038086,
      "activations/layer0_attention_weight_min": -11.816060066223145,
      "activations/layer10_attention_weight_max": 34.30012512207031,
      "activations/layer10_attention_weight_min": -34.7769775390625,
      "activations/layer11_attention_weight_max": 35.00755310058594,
      "activations/layer11_attention_weight_min": -33.291446685791016,
      "activations/layer12_attention_weight_max": 24.514408111572266,
      "activations/layer12_attention_weight_min": -27.035343170166016,
      "activations/layer13_attention_weight_max": 36.72723388671875,
      "activations/layer13_attention_weight_min": -34.64378356933594,
      "activations/layer14_attention_weight_max": 39.380889892578125,
      "activations/layer14_attention_weight_min": -34.68675994873047,
      "activations/layer15_attention_weight_max": 34.93034362792969,
      "activations/layer15_attention_weight_min": -35.85160827636719,
      "activations/layer16_attention_weight_max": 26.718599319458008,
      "activations/layer16_attention_weight_min": -25.993165969848633,
      "activations/layer17_attention_weight_max": 29.07377815246582,
      "activations/layer17_attention_weight_min": -25.642812728881836,
      "activations/layer18_attention_weight_max": 30.73388671875,
      "activations/layer18_attention_weight_min": -26.92570686340332,
      "activations/layer19_attention_weight_max": 31.314712524414062,
      "activations/layer19_attention_weight_min": -30.725208282470703,
      "activations/layer1_attention_weight_max": 15.695298194885254,
      "activations/layer1_attention_weight_min": -14.709017753601074,
      "activations/layer20_attention_weight_max": 27.920745849609375,
      "activations/layer20_attention_weight_min": -25.61470603942871,
      "activations/layer21_attention_weight_max": 26.6322078704834,
      "activations/layer21_attention_weight_min": -24.302677154541016,
      "activations/layer22_attention_weight_max": 39.39918518066406,
      "activations/layer22_attention_weight_min": -31.64774513244629,
      "activations/layer23_attention_weight_max": 30.65249252319336,
      "activations/layer23_attention_weight_min": -26.49065399169922,
      "activations/layer2_attention_weight_max": 37.686683654785156,
      "activations/layer2_attention_weight_min": -34.55485153198242,
      "activations/layer3_attention_weight_max": 109.00891876220703,
      "activations/layer3_attention_weight_min": -103.30268096923828,
      "activations/layer4_attention_weight_max": 104.66988372802734,
      "activations/layer4_attention_weight_min": -108.17994689941406,
      "activations/layer5_attention_weight_max": 51.81235885620117,
      "activations/layer5_attention_weight_min": -66.39483642578125,
      "activations/layer6_attention_weight_max": 45.41415786743164,
      "activations/layer6_attention_weight_min": -49.7654914855957,
      "activations/layer7_attention_weight_max": 92.48954010009766,
      "activations/layer7_attention_weight_min": -97.89543914794922,
      "activations/layer8_attention_weight_max": 41.563682556152344,
      "activations/layer8_attention_weight_min": -43.83181381225586,
      "activations/layer9_attention_weight_max": 32.752376556396484,
      "activations/layer9_attention_weight_min": -34.58933639526367,
      "epoch": 17.62,
      "learning_rate": 3.673901515151515e-05,
      "loss": 2.7392,
      "step": 303250
    },
    {
      "activations/layer0_attention_weight_max": 16.800447463989258,
      "activations/layer0_attention_weight_min": -13.192305564880371,
      "activations/layer10_attention_weight_max": 31.425357818603516,
      "activations/layer10_attention_weight_min": -33.82196807861328,
      "activations/layer11_attention_weight_max": 32.2596435546875,
      "activations/layer11_attention_weight_min": -33.51389694213867,
      "activations/layer12_attention_weight_max": 26.252397537231445,
      "activations/layer12_attention_weight_min": -26.589378356933594,
      "activations/layer13_attention_weight_max": 38.04869079589844,
      "activations/layer13_attention_weight_min": -36.520782470703125,
      "activations/layer14_attention_weight_max": 43.044921875,
      "activations/layer14_attention_weight_min": -37.06483840942383,
      "activations/layer15_attention_weight_max": 38.78841018676758,
      "activations/layer15_attention_weight_min": -35.675113677978516,
      "activations/layer16_attention_weight_max": 30.40240478515625,
      "activations/layer16_attention_weight_min": -28.32768440246582,
      "activations/layer17_attention_weight_max": 29.948184967041016,
      "activations/layer17_attention_weight_min": -25.3705997467041,
      "activations/layer18_attention_weight_max": 31.15886878967285,
      "activations/layer18_attention_weight_min": -24.99085807800293,
      "activations/layer19_attention_weight_max": 34.96107482910156,
      "activations/layer19_attention_weight_min": -30.627546310424805,
      "activations/layer1_attention_weight_max": 16.13212776184082,
      "activations/layer1_attention_weight_min": -14.41435432434082,
      "activations/layer20_attention_weight_max": 29.361339569091797,
      "activations/layer20_attention_weight_min": -24.647865295410156,
      "activations/layer21_attention_weight_max": 30.546977996826172,
      "activations/layer21_attention_weight_min": -24.64257049560547,
      "activations/layer22_attention_weight_max": 40.350704193115234,
      "activations/layer22_attention_weight_min": -30.448835372924805,
      "activations/layer23_attention_weight_max": 32.05282974243164,
      "activations/layer23_attention_weight_min": -25.633420944213867,
      "activations/layer2_attention_weight_max": 32.16446304321289,
      "activations/layer2_attention_weight_min": -32.084510803222656,
      "activations/layer3_attention_weight_max": 98.91686248779297,
      "activations/layer3_attention_weight_min": -96.4169692993164,
      "activations/layer4_attention_weight_max": 105.0645523071289,
      "activations/layer4_attention_weight_min": -107.2468490600586,
      "activations/layer5_attention_weight_max": 49.77836608886719,
      "activations/layer5_attention_weight_min": -65.58647155761719,
      "activations/layer6_attention_weight_max": 41.75702667236328,
      "activations/layer6_attention_weight_min": -44.07986831665039,
      "activations/layer7_attention_weight_max": 88.8105239868164,
      "activations/layer7_attention_weight_min": -89.10560607910156,
      "activations/layer8_attention_weight_max": 38.87363052368164,
      "activations/layer8_attention_weight_min": -39.89641571044922,
      "activations/layer9_attention_weight_max": 28.73624038696289,
      "activations/layer9_attention_weight_min": -32.21371841430664,
      "epoch": 17.62,
      "learning_rate": 3.672007575757575e-05,
      "loss": 2.7401,
      "step": 303300
    },
    {
      "activations/layer0_attention_weight_max": 16.429256439208984,
      "activations/layer0_attention_weight_min": -13.481321334838867,
      "activations/layer10_attention_weight_max": 32.669349670410156,
      "activations/layer10_attention_weight_min": -34.60132598876953,
      "activations/layer11_attention_weight_max": 32.164424896240234,
      "activations/layer11_attention_weight_min": -34.42143249511719,
      "activations/layer12_attention_weight_max": 26.574106216430664,
      "activations/layer12_attention_weight_min": -25.650156021118164,
      "activations/layer13_attention_weight_max": 40.2932243347168,
      "activations/layer13_attention_weight_min": -36.07121658325195,
      "activations/layer14_attention_weight_max": 43.48868179321289,
      "activations/layer14_attention_weight_min": -37.71046829223633,
      "activations/layer15_attention_weight_max": 41.71528244018555,
      "activations/layer15_attention_weight_min": -35.134910583496094,
      "activations/layer16_attention_weight_max": 33.438331604003906,
      "activations/layer16_attention_weight_min": -27.96212387084961,
      "activations/layer17_attention_weight_max": 33.62761306762695,
      "activations/layer17_attention_weight_min": -27.828418731689453,
      "activations/layer18_attention_weight_max": 32.5258674621582,
      "activations/layer18_attention_weight_min": -24.50081443786621,
      "activations/layer19_attention_weight_max": 34.42756271362305,
      "activations/layer19_attention_weight_min": -32.27281188964844,
      "activations/layer1_attention_weight_max": 14.966715812683105,
      "activations/layer1_attention_weight_min": -14.287371635437012,
      "activations/layer20_attention_weight_max": 30.436500549316406,
      "activations/layer20_attention_weight_min": -26.833696365356445,
      "activations/layer21_attention_weight_max": 30.110488891601562,
      "activations/layer21_attention_weight_min": -26.485641479492188,
      "activations/layer22_attention_weight_max": 42.345542907714844,
      "activations/layer22_attention_weight_min": -34.122562408447266,
      "activations/layer23_attention_weight_max": 38.27715301513672,
      "activations/layer23_attention_weight_min": -27.96050262451172,
      "activations/layer2_attention_weight_max": 34.23362350463867,
      "activations/layer2_attention_weight_min": -32.14772033691406,
      "activations/layer3_attention_weight_max": 95.19377899169922,
      "activations/layer3_attention_weight_min": -99.02040100097656,
      "activations/layer4_attention_weight_max": 106.67598724365234,
      "activations/layer4_attention_weight_min": -109.9869155883789,
      "activations/layer5_attention_weight_max": 52.64908981323242,
      "activations/layer5_attention_weight_min": -61.042110443115234,
      "activations/layer6_attention_weight_max": 45.99388122558594,
      "activations/layer6_attention_weight_min": -46.46305847167969,
      "activations/layer7_attention_weight_max": 93.53150939941406,
      "activations/layer7_attention_weight_min": -93.90155029296875,
      "activations/layer8_attention_weight_max": 41.73775100708008,
      "activations/layer8_attention_weight_min": -42.51939392089844,
      "activations/layer9_attention_weight_max": 32.115989685058594,
      "activations/layer9_attention_weight_min": -33.949649810791016,
      "epoch": 17.63,
      "learning_rate": 3.670113636363636e-05,
      "loss": 2.7285,
      "step": 303350
    },
    {
      "activations/layer0_attention_weight_max": 16.038700103759766,
      "activations/layer0_attention_weight_min": -12.071831703186035,
      "activations/layer10_attention_weight_max": 32.79657745361328,
      "activations/layer10_attention_weight_min": -33.01991271972656,
      "activations/layer11_attention_weight_max": 32.340354919433594,
      "activations/layer11_attention_weight_min": -33.951904296875,
      "activations/layer12_attention_weight_max": 24.911102294921875,
      "activations/layer12_attention_weight_min": -26.902833938598633,
      "activations/layer13_attention_weight_max": 36.80057907104492,
      "activations/layer13_attention_weight_min": -35.636322021484375,
      "activations/layer14_attention_weight_max": 40.047672271728516,
      "activations/layer14_attention_weight_min": -36.32232666015625,
      "activations/layer15_attention_weight_max": 36.74180603027344,
      "activations/layer15_attention_weight_min": -35.418121337890625,
      "activations/layer16_attention_weight_max": 26.48122787475586,
      "activations/layer16_attention_weight_min": -28.584718704223633,
      "activations/layer17_attention_weight_max": 30.449411392211914,
      "activations/layer17_attention_weight_min": -26.877748489379883,
      "activations/layer18_attention_weight_max": 32.316383361816406,
      "activations/layer18_attention_weight_min": -26.511507034301758,
      "activations/layer19_attention_weight_max": 33.84885787963867,
      "activations/layer19_attention_weight_min": -32.06198501586914,
      "activations/layer1_attention_weight_max": 16.408828735351562,
      "activations/layer1_attention_weight_min": -14.08014965057373,
      "activations/layer20_attention_weight_max": 30.47494125366211,
      "activations/layer20_attention_weight_min": -26.883403778076172,
      "activations/layer21_attention_weight_max": 30.389080047607422,
      "activations/layer21_attention_weight_min": -26.800315856933594,
      "activations/layer22_attention_weight_max": 39.532588958740234,
      "activations/layer22_attention_weight_min": -34.56846237182617,
      "activations/layer23_attention_weight_max": 30.43764877319336,
      "activations/layer23_attention_weight_min": -27.489171981811523,
      "activations/layer2_attention_weight_max": 30.319488525390625,
      "activations/layer2_attention_weight_min": -30.836769104003906,
      "activations/layer3_attention_weight_max": 98.12753295898438,
      "activations/layer3_attention_weight_min": -95.72747039794922,
      "activations/layer4_attention_weight_max": 101.50208282470703,
      "activations/layer4_attention_weight_min": -103.73943328857422,
      "activations/layer5_attention_weight_max": 49.46291732788086,
      "activations/layer5_attention_weight_min": -64.19682312011719,
      "activations/layer6_attention_weight_max": 43.998802185058594,
      "activations/layer6_attention_weight_min": -46.013118743896484,
      "activations/layer7_attention_weight_max": 89.24970245361328,
      "activations/layer7_attention_weight_min": -93.8248062133789,
      "activations/layer8_attention_weight_max": 39.98637008666992,
      "activations/layer8_attention_weight_min": -42.98567199707031,
      "activations/layer9_attention_weight_max": 31.3472843170166,
      "activations/layer9_attention_weight_min": -33.60943603515625,
      "epoch": 17.63,
      "learning_rate": 3.668219696969697e-05,
      "loss": 2.7338,
      "step": 303400
    },
    {
      "activations/layer0_attention_weight_max": 15.63463020324707,
      "activations/layer0_attention_weight_min": -13.418346405029297,
      "activations/layer10_attention_weight_max": 34.0079345703125,
      "activations/layer10_attention_weight_min": -36.56840896606445,
      "activations/layer11_attention_weight_max": 31.63111686706543,
      "activations/layer11_attention_weight_min": -34.0579833984375,
      "activations/layer12_attention_weight_max": 28.05035972595215,
      "activations/layer12_attention_weight_min": -26.632766723632812,
      "activations/layer13_attention_weight_max": 40.80738067626953,
      "activations/layer13_attention_weight_min": -37.847591400146484,
      "activations/layer14_attention_weight_max": 43.04682540893555,
      "activations/layer14_attention_weight_min": -37.49435043334961,
      "activations/layer15_attention_weight_max": 40.11368179321289,
      "activations/layer15_attention_weight_min": -34.11281204223633,
      "activations/layer16_attention_weight_max": 30.97522735595703,
      "activations/layer16_attention_weight_min": -27.57914924621582,
      "activations/layer17_attention_weight_max": 32.762271881103516,
      "activations/layer17_attention_weight_min": -25.869808197021484,
      "activations/layer18_attention_weight_max": 36.569602966308594,
      "activations/layer18_attention_weight_min": -23.679643630981445,
      "activations/layer19_attention_weight_max": 35.5553092956543,
      "activations/layer19_attention_weight_min": -29.592742919921875,
      "activations/layer1_attention_weight_max": 16.806432723999023,
      "activations/layer1_attention_weight_min": -14.42073917388916,
      "activations/layer20_attention_weight_max": 33.29917907714844,
      "activations/layer20_attention_weight_min": -23.881805419921875,
      "activations/layer21_attention_weight_max": 33.77788162231445,
      "activations/layer21_attention_weight_min": -24.466833114624023,
      "activations/layer22_attention_weight_max": 47.0220832824707,
      "activations/layer22_attention_weight_min": -30.43614387512207,
      "activations/layer23_attention_weight_max": 38.248844146728516,
      "activations/layer23_attention_weight_min": -24.891324996948242,
      "activations/layer2_attention_weight_max": 33.50737762451172,
      "activations/layer2_attention_weight_min": -32.7325439453125,
      "activations/layer3_attention_weight_max": 97.45154571533203,
      "activations/layer3_attention_weight_min": -95.63504028320312,
      "activations/layer4_attention_weight_max": 101.6829833984375,
      "activations/layer4_attention_weight_min": -103.5177993774414,
      "activations/layer5_attention_weight_max": 48.98223114013672,
      "activations/layer5_attention_weight_min": -62.635643005371094,
      "activations/layer6_attention_weight_max": 42.70983123779297,
      "activations/layer6_attention_weight_min": -45.28226852416992,
      "activations/layer7_attention_weight_max": 89.62751007080078,
      "activations/layer7_attention_weight_min": -93.49794006347656,
      "activations/layer8_attention_weight_max": 43.34209060668945,
      "activations/layer8_attention_weight_min": -43.115482330322266,
      "activations/layer9_attention_weight_max": 32.0025749206543,
      "activations/layer9_attention_weight_min": -36.289939880371094,
      "epoch": 17.63,
      "learning_rate": 3.6663257575757575e-05,
      "loss": 2.7257,
      "step": 303450
    },
    {
      "activations/layer0_attention_weight_max": 16.60922622680664,
      "activations/layer0_attention_weight_min": -13.160409927368164,
      "activations/layer10_attention_weight_max": 31.745361328125,
      "activations/layer10_attention_weight_min": -32.010501861572266,
      "activations/layer11_attention_weight_max": 30.38287353515625,
      "activations/layer11_attention_weight_min": -32.71916198730469,
      "activations/layer12_attention_weight_max": 23.86384391784668,
      "activations/layer12_attention_weight_min": -24.734045028686523,
      "activations/layer13_attention_weight_max": 36.297576904296875,
      "activations/layer13_attention_weight_min": -36.6885986328125,
      "activations/layer14_attention_weight_max": 39.265464782714844,
      "activations/layer14_attention_weight_min": -37.24549865722656,
      "activations/layer15_attention_weight_max": 38.22129440307617,
      "activations/layer15_attention_weight_min": -32.85750961303711,
      "activations/layer16_attention_weight_max": 28.920106887817383,
      "activations/layer16_attention_weight_min": -28.51530647277832,
      "activations/layer17_attention_weight_max": 29.38327980041504,
      "activations/layer17_attention_weight_min": -25.337263107299805,
      "activations/layer18_attention_weight_max": 29.945085525512695,
      "activations/layer18_attention_weight_min": -23.559385299682617,
      "activations/layer19_attention_weight_max": 34.50324630737305,
      "activations/layer19_attention_weight_min": -29.80124282836914,
      "activations/layer1_attention_weight_max": 15.262673377990723,
      "activations/layer1_attention_weight_min": -13.480559349060059,
      "activations/layer20_attention_weight_max": 29.954587936401367,
      "activations/layer20_attention_weight_min": -24.12268829345703,
      "activations/layer21_attention_weight_max": 29.868282318115234,
      "activations/layer21_attention_weight_min": -23.559947967529297,
      "activations/layer22_attention_weight_max": 41.55160903930664,
      "activations/layer22_attention_weight_min": -30.15671157836914,
      "activations/layer23_attention_weight_max": 32.94100570678711,
      "activations/layer23_attention_weight_min": -25.289901733398438,
      "activations/layer2_attention_weight_max": 31.371829986572266,
      "activations/layer2_attention_weight_min": -31.79458999633789,
      "activations/layer3_attention_weight_max": 99.6257095336914,
      "activations/layer3_attention_weight_min": -96.75823211669922,
      "activations/layer4_attention_weight_max": 106.8608169555664,
      "activations/layer4_attention_weight_min": -105.81854248046875,
      "activations/layer5_attention_weight_max": 52.20854187011719,
      "activations/layer5_attention_weight_min": -61.57819366455078,
      "activations/layer6_attention_weight_max": 43.56534194946289,
      "activations/layer6_attention_weight_min": -46.15604782104492,
      "activations/layer7_attention_weight_max": 90.67211151123047,
      "activations/layer7_attention_weight_min": -92.59222412109375,
      "activations/layer8_attention_weight_max": 41.4362907409668,
      "activations/layer8_attention_weight_min": -40.20161437988281,
      "activations/layer9_attention_weight_max": 31.156269073486328,
      "activations/layer9_attention_weight_min": -31.84158706665039,
      "epoch": 17.64,
      "learning_rate": 3.664431818181818e-05,
      "loss": 2.7335,
      "step": 303500
    },
    {
      "activations/layer0_attention_weight_max": 16.105043411254883,
      "activations/layer0_attention_weight_min": -13.99795150756836,
      "activations/layer10_attention_weight_max": 32.63998031616211,
      "activations/layer10_attention_weight_min": -32.91578674316406,
      "activations/layer11_attention_weight_max": 32.87990951538086,
      "activations/layer11_attention_weight_min": -33.165130615234375,
      "activations/layer12_attention_weight_max": 25.99469566345215,
      "activations/layer12_attention_weight_min": -26.47549819946289,
      "activations/layer13_attention_weight_max": 39.11748123168945,
      "activations/layer13_attention_weight_min": -36.578941345214844,
      "activations/layer14_attention_weight_max": 40.53157043457031,
      "activations/layer14_attention_weight_min": -37.74692916870117,
      "activations/layer15_attention_weight_max": 36.974952697753906,
      "activations/layer15_attention_weight_min": -35.204917907714844,
      "activations/layer16_attention_weight_max": 26.76950454711914,
      "activations/layer16_attention_weight_min": -27.366458892822266,
      "activations/layer17_attention_weight_max": 29.26640510559082,
      "activations/layer17_attention_weight_min": -26.583356857299805,
      "activations/layer18_attention_weight_max": 30.223453521728516,
      "activations/layer18_attention_weight_min": -25.336000442504883,
      "activations/layer19_attention_weight_max": 31.445032119750977,
      "activations/layer19_attention_weight_min": -32.562191009521484,
      "activations/layer1_attention_weight_max": 16.308740615844727,
      "activations/layer1_attention_weight_min": -13.486952781677246,
      "activations/layer20_attention_weight_max": 25.453445434570312,
      "activations/layer20_attention_weight_min": -24.8160457611084,
      "activations/layer21_attention_weight_max": 25.829864501953125,
      "activations/layer21_attention_weight_min": -25.576969146728516,
      "activations/layer22_attention_weight_max": 35.8238639831543,
      "activations/layer22_attention_weight_min": -33.31885528564453,
      "activations/layer23_attention_weight_max": 30.067991256713867,
      "activations/layer23_attention_weight_min": -28.690305709838867,
      "activations/layer2_attention_weight_max": 33.34392547607422,
      "activations/layer2_attention_weight_min": -31.918498992919922,
      "activations/layer3_attention_weight_max": 97.66407775878906,
      "activations/layer3_attention_weight_min": -96.4919662475586,
      "activations/layer4_attention_weight_max": 102.96405029296875,
      "activations/layer4_attention_weight_min": -106.23457336425781,
      "activations/layer5_attention_weight_max": 49.10702133178711,
      "activations/layer5_attention_weight_min": -63.5352897644043,
      "activations/layer6_attention_weight_max": 44.68302917480469,
      "activations/layer6_attention_weight_min": -45.54119110107422,
      "activations/layer7_attention_weight_max": 89.73849487304688,
      "activations/layer7_attention_weight_min": -97.2786865234375,
      "activations/layer8_attention_weight_max": 42.3231086730957,
      "activations/layer8_attention_weight_min": -42.28005599975586,
      "activations/layer9_attention_weight_max": 32.32979202270508,
      "activations/layer9_attention_weight_min": -33.83613204956055,
      "epoch": 17.64,
      "learning_rate": 3.6625378787878785e-05,
      "loss": 2.747,
      "step": 303550
    },
    {
      "activations/layer0_attention_weight_max": 16.797025680541992,
      "activations/layer0_attention_weight_min": -14.818319320678711,
      "activations/layer10_attention_weight_max": 34.9254150390625,
      "activations/layer10_attention_weight_min": -35.96747589111328,
      "activations/layer11_attention_weight_max": 36.48149871826172,
      "activations/layer11_attention_weight_min": -34.24099349975586,
      "activations/layer12_attention_weight_max": 26.700956344604492,
      "activations/layer12_attention_weight_min": -26.359498977661133,
      "activations/layer13_attention_weight_max": 38.48493576049805,
      "activations/layer13_attention_weight_min": -35.56678009033203,
      "activations/layer14_attention_weight_max": 42.265602111816406,
      "activations/layer14_attention_weight_min": -37.679969787597656,
      "activations/layer15_attention_weight_max": 38.23246383666992,
      "activations/layer15_attention_weight_min": -34.68958282470703,
      "activations/layer16_attention_weight_max": 27.734643936157227,
      "activations/layer16_attention_weight_min": -27.224163055419922,
      "activations/layer17_attention_weight_max": 28.418813705444336,
      "activations/layer17_attention_weight_min": -24.335735321044922,
      "activations/layer18_attention_weight_max": 31.8815975189209,
      "activations/layer18_attention_weight_min": -24.402910232543945,
      "activations/layer19_attention_weight_max": 30.566726684570312,
      "activations/layer19_attention_weight_min": -30.639251708984375,
      "activations/layer1_attention_weight_max": 16.13714599609375,
      "activations/layer1_attention_weight_min": -14.132732391357422,
      "activations/layer20_attention_weight_max": 28.96221351623535,
      "activations/layer20_attention_weight_min": -25.84258270263672,
      "activations/layer21_attention_weight_max": 26.666837692260742,
      "activations/layer21_attention_weight_min": -25.892173767089844,
      "activations/layer22_attention_weight_max": 37.752620697021484,
      "activations/layer22_attention_weight_min": -29.69339370727539,
      "activations/layer23_attention_weight_max": 29.41699981689453,
      "activations/layer23_attention_weight_min": -24.677806854248047,
      "activations/layer2_attention_weight_max": 31.994009017944336,
      "activations/layer2_attention_weight_min": -33.225032806396484,
      "activations/layer3_attention_weight_max": 98.91917419433594,
      "activations/layer3_attention_weight_min": -98.61141967773438,
      "activations/layer4_attention_weight_max": 106.35237121582031,
      "activations/layer4_attention_weight_min": -107.80440521240234,
      "activations/layer5_attention_weight_max": 50.682430267333984,
      "activations/layer5_attention_weight_min": -60.818782806396484,
      "activations/layer6_attention_weight_max": 43.41781997680664,
      "activations/layer6_attention_weight_min": -45.96820831298828,
      "activations/layer7_attention_weight_max": 90.15518188476562,
      "activations/layer7_attention_weight_min": -96.86894226074219,
      "activations/layer8_attention_weight_max": 40.758941650390625,
      "activations/layer8_attention_weight_min": -44.42424392700195,
      "activations/layer9_attention_weight_max": 32.020530700683594,
      "activations/layer9_attention_weight_min": -34.311824798583984,
      "epoch": 17.64,
      "learning_rate": 3.6606439393939387e-05,
      "loss": 2.7454,
      "step": 303600
    },
    {
      "activations/layer0_attention_weight_max": 15.907384872436523,
      "activations/layer0_attention_weight_min": -12.43349838256836,
      "activations/layer10_attention_weight_max": 33.96159362792969,
      "activations/layer10_attention_weight_min": -34.01296615600586,
      "activations/layer11_attention_weight_max": 34.29005432128906,
      "activations/layer11_attention_weight_min": -32.987003326416016,
      "activations/layer12_attention_weight_max": 27.07022476196289,
      "activations/layer12_attention_weight_min": -29.18181610107422,
      "activations/layer13_attention_weight_max": 38.54113006591797,
      "activations/layer13_attention_weight_min": -37.15437316894531,
      "activations/layer14_attention_weight_max": 37.987510681152344,
      "activations/layer14_attention_weight_min": -39.6423454284668,
      "activations/layer15_attention_weight_max": 38.26400375366211,
      "activations/layer15_attention_weight_min": -37.06246566772461,
      "activations/layer16_attention_weight_max": 31.967695236206055,
      "activations/layer16_attention_weight_min": -28.07078742980957,
      "activations/layer17_attention_weight_max": 27.3831729888916,
      "activations/layer17_attention_weight_min": -25.345043182373047,
      "activations/layer18_attention_weight_max": 31.56401252746582,
      "activations/layer18_attention_weight_min": -24.708820343017578,
      "activations/layer19_attention_weight_max": 33.66535949707031,
      "activations/layer19_attention_weight_min": -31.008838653564453,
      "activations/layer1_attention_weight_max": 16.480979919433594,
      "activations/layer1_attention_weight_min": -14.882265090942383,
      "activations/layer20_attention_weight_max": 26.96422576904297,
      "activations/layer20_attention_weight_min": -25.493553161621094,
      "activations/layer21_attention_weight_max": 27.126554489135742,
      "activations/layer21_attention_weight_min": -24.77457046508789,
      "activations/layer22_attention_weight_max": 44.60840606689453,
      "activations/layer22_attention_weight_min": -31.05213165283203,
      "activations/layer23_attention_weight_max": 33.159027099609375,
      "activations/layer23_attention_weight_min": -25.399328231811523,
      "activations/layer2_attention_weight_max": 36.365966796875,
      "activations/layer2_attention_weight_min": -35.38841247558594,
      "activations/layer3_attention_weight_max": 113.86870574951172,
      "activations/layer3_attention_weight_min": -109.68778228759766,
      "activations/layer4_attention_weight_max": 115.04671478271484,
      "activations/layer4_attention_weight_min": -109.67889404296875,
      "activations/layer5_attention_weight_max": 48.82734298706055,
      "activations/layer5_attention_weight_min": -62.140953063964844,
      "activations/layer6_attention_weight_max": 43.7238655090332,
      "activations/layer6_attention_weight_min": -46.86452102661133,
      "activations/layer7_attention_weight_max": 91.70330047607422,
      "activations/layer7_attention_weight_min": -96.42527770996094,
      "activations/layer8_attention_weight_max": 41.58063888549805,
      "activations/layer8_attention_weight_min": -42.979469299316406,
      "activations/layer9_attention_weight_max": 31.52430534362793,
      "activations/layer9_attention_weight_min": -33.12070846557617,
      "epoch": 17.64,
      "learning_rate": 3.6587499999999995e-05,
      "loss": 2.7331,
      "step": 303650
    },
    {
      "activations/layer0_attention_weight_max": 16.467498779296875,
      "activations/layer0_attention_weight_min": -11.947961807250977,
      "activations/layer10_attention_weight_max": 41.02779769897461,
      "activations/layer10_attention_weight_min": -38.05048370361328,
      "activations/layer11_attention_weight_max": 37.85757064819336,
      "activations/layer11_attention_weight_min": -35.122352600097656,
      "activations/layer12_attention_weight_max": 26.843233108520508,
      "activations/layer12_attention_weight_min": -26.31969451904297,
      "activations/layer13_attention_weight_max": 38.53517150878906,
      "activations/layer13_attention_weight_min": -37.12501525878906,
      "activations/layer14_attention_weight_max": 45.104835510253906,
      "activations/layer14_attention_weight_min": -39.48745346069336,
      "activations/layer15_attention_weight_max": 38.3764533996582,
      "activations/layer15_attention_weight_min": -35.01628494262695,
      "activations/layer16_attention_weight_max": 30.70530891418457,
      "activations/layer16_attention_weight_min": -27.48704719543457,
      "activations/layer17_attention_weight_max": 29.74440574645996,
      "activations/layer17_attention_weight_min": -26.25716209411621,
      "activations/layer18_attention_weight_max": 34.120094299316406,
      "activations/layer18_attention_weight_min": -24.879209518432617,
      "activations/layer19_attention_weight_max": 37.74720764160156,
      "activations/layer19_attention_weight_min": -31.93912124633789,
      "activations/layer1_attention_weight_max": 16.993501663208008,
      "activations/layer1_attention_weight_min": -14.996597290039062,
      "activations/layer20_attention_weight_max": 30.71379280090332,
      "activations/layer20_attention_weight_min": -26.520790100097656,
      "activations/layer21_attention_weight_max": 29.51061248779297,
      "activations/layer21_attention_weight_min": -26.147123336791992,
      "activations/layer22_attention_weight_max": 46.4527587890625,
      "activations/layer22_attention_weight_min": -32.43808364868164,
      "activations/layer23_attention_weight_max": 35.15508270263672,
      "activations/layer23_attention_weight_min": -25.397306442260742,
      "activations/layer2_attention_weight_max": 33.21056365966797,
      "activations/layer2_attention_weight_min": -33.01276779174805,
      "activations/layer3_attention_weight_max": 99.5597915649414,
      "activations/layer3_attention_weight_min": -99.82449340820312,
      "activations/layer4_attention_weight_max": 105.18895721435547,
      "activations/layer4_attention_weight_min": -106.3558349609375,
      "activations/layer5_attention_weight_max": 48.2811279296875,
      "activations/layer5_attention_weight_min": -63.85917282104492,
      "activations/layer6_attention_weight_max": 45.5330696105957,
      "activations/layer6_attention_weight_min": -43.94137191772461,
      "activations/layer7_attention_weight_max": 97.89341735839844,
      "activations/layer7_attention_weight_min": -95.55184936523438,
      "activations/layer8_attention_weight_max": 46.6091423034668,
      "activations/layer8_attention_weight_min": -45.23521041870117,
      "activations/layer9_attention_weight_max": 35.9964714050293,
      "activations/layer9_attention_weight_min": -36.610435485839844,
      "epoch": 17.65,
      "learning_rate": 3.656893939393939e-05,
      "loss": 2.7194,
      "step": 303700
    },
    {
      "activations/layer0_attention_weight_max": 16.013568878173828,
      "activations/layer0_attention_weight_min": -12.695277214050293,
      "activations/layer10_attention_weight_max": 34.0359992980957,
      "activations/layer10_attention_weight_min": -34.242401123046875,
      "activations/layer11_attention_weight_max": 34.879981994628906,
      "activations/layer11_attention_weight_min": -35.95885467529297,
      "activations/layer12_attention_weight_max": 27.75432014465332,
      "activations/layer12_attention_weight_min": -27.459693908691406,
      "activations/layer13_attention_weight_max": 39.93706130981445,
      "activations/layer13_attention_weight_min": -35.386959075927734,
      "activations/layer14_attention_weight_max": 41.29829788208008,
      "activations/layer14_attention_weight_min": -38.27198791503906,
      "activations/layer15_attention_weight_max": 38.053958892822266,
      "activations/layer15_attention_weight_min": -35.36652755737305,
      "activations/layer16_attention_weight_max": 27.780241012573242,
      "activations/layer16_attention_weight_min": -27.42989730834961,
      "activations/layer17_attention_weight_max": 29.07916259765625,
      "activations/layer17_attention_weight_min": -27.692007064819336,
      "activations/layer18_attention_weight_max": 35.65396499633789,
      "activations/layer18_attention_weight_min": -26.732877731323242,
      "activations/layer19_attention_weight_max": 35.075523376464844,
      "activations/layer19_attention_weight_min": -31.792016983032227,
      "activations/layer1_attention_weight_max": 15.853431701660156,
      "activations/layer1_attention_weight_min": -14.713682174682617,
      "activations/layer20_attention_weight_max": 29.171159744262695,
      "activations/layer20_attention_weight_min": -25.511350631713867,
      "activations/layer21_attention_weight_max": 27.27758026123047,
      "activations/layer21_attention_weight_min": -25.219024658203125,
      "activations/layer22_attention_weight_max": 44.40483856201172,
      "activations/layer22_attention_weight_min": -31.532930374145508,
      "activations/layer23_attention_weight_max": 32.76808547973633,
      "activations/layer23_attention_weight_min": -24.495155334472656,
      "activations/layer2_attention_weight_max": 31.698331832885742,
      "activations/layer2_attention_weight_min": -33.986732482910156,
      "activations/layer3_attention_weight_max": 100.12120819091797,
      "activations/layer3_attention_weight_min": -100.99073028564453,
      "activations/layer4_attention_weight_max": 102.52880859375,
      "activations/layer4_attention_weight_min": -108.68885803222656,
      "activations/layer5_attention_weight_max": 47.70356750488281,
      "activations/layer5_attention_weight_min": -59.36328887939453,
      "activations/layer6_attention_weight_max": 41.38358688354492,
      "activations/layer6_attention_weight_min": -45.66874313354492,
      "activations/layer7_attention_weight_max": 82.7762451171875,
      "activations/layer7_attention_weight_min": -91.24240112304688,
      "activations/layer8_attention_weight_max": 40.65590286254883,
      "activations/layer8_attention_weight_min": -42.568458557128906,
      "activations/layer9_attention_weight_max": 32.539268493652344,
      "activations/layer9_attention_weight_min": -33.511348724365234,
      "epoch": 17.65,
      "learning_rate": 3.655e-05,
      "loss": 2.718,
      "step": 303750
    },
    {
      "activations/layer0_attention_weight_max": 16.02950096130371,
      "activations/layer0_attention_weight_min": -13.446207046508789,
      "activations/layer10_attention_weight_max": 37.76177215576172,
      "activations/layer10_attention_weight_min": -35.26996612548828,
      "activations/layer11_attention_weight_max": 37.57696533203125,
      "activations/layer11_attention_weight_min": -35.76898193359375,
      "activations/layer12_attention_weight_max": 27.6829776763916,
      "activations/layer12_attention_weight_min": -27.308292388916016,
      "activations/layer13_attention_weight_max": 42.01606369018555,
      "activations/layer13_attention_weight_min": -35.831382751464844,
      "activations/layer14_attention_weight_max": 44.26579666137695,
      "activations/layer14_attention_weight_min": -40.49978256225586,
      "activations/layer15_attention_weight_max": 39.632450103759766,
      "activations/layer15_attention_weight_min": -38.70376968383789,
      "activations/layer16_attention_weight_max": 29.169828414916992,
      "activations/layer16_attention_weight_min": -28.48430061340332,
      "activations/layer17_attention_weight_max": 27.191497802734375,
      "activations/layer17_attention_weight_min": -26.206771850585938,
      "activations/layer18_attention_weight_max": 29.3050537109375,
      "activations/layer18_attention_weight_min": -26.29637908935547,
      "activations/layer19_attention_weight_max": 31.90542984008789,
      "activations/layer19_attention_weight_min": -30.994659423828125,
      "activations/layer1_attention_weight_max": 16.109792709350586,
      "activations/layer1_attention_weight_min": -14.030607223510742,
      "activations/layer20_attention_weight_max": 26.97357749938965,
      "activations/layer20_attention_weight_min": -25.4228458404541,
      "activations/layer21_attention_weight_max": 25.27599334716797,
      "activations/layer21_attention_weight_min": -25.642803192138672,
      "activations/layer22_attention_weight_max": 36.93735122680664,
      "activations/layer22_attention_weight_min": -32.08460998535156,
      "activations/layer23_attention_weight_max": 29.177513122558594,
      "activations/layer23_attention_weight_min": -26.22541618347168,
      "activations/layer2_attention_weight_max": 31.578292846679688,
      "activations/layer2_attention_weight_min": -32.52354049682617,
      "activations/layer3_attention_weight_max": 100.6915283203125,
      "activations/layer3_attention_weight_min": -100.51229095458984,
      "activations/layer4_attention_weight_max": 107.142333984375,
      "activations/layer4_attention_weight_min": -116.9002456665039,
      "activations/layer5_attention_weight_max": 49.767913818359375,
      "activations/layer5_attention_weight_min": -64.63741302490234,
      "activations/layer6_attention_weight_max": 45.39157485961914,
      "activations/layer6_attention_weight_min": -47.396827697753906,
      "activations/layer7_attention_weight_max": 98.69730377197266,
      "activations/layer7_attention_weight_min": -98.4119644165039,
      "activations/layer8_attention_weight_max": 48.578094482421875,
      "activations/layer8_attention_weight_min": -45.53365707397461,
      "activations/layer9_attention_weight_max": 35.77803421020508,
      "activations/layer9_attention_weight_min": -37.2486457824707,
      "epoch": 17.65,
      "learning_rate": 3.65310606060606e-05,
      "loss": 2.7365,
      "step": 303800
    },
    {
      "activations/layer0_attention_weight_max": 15.968415260314941,
      "activations/layer0_attention_weight_min": -13.080378532409668,
      "activations/layer10_attention_weight_max": 41.83332443237305,
      "activations/layer10_attention_weight_min": -41.682960510253906,
      "activations/layer11_attention_weight_max": 45.44697189331055,
      "activations/layer11_attention_weight_min": -43.0002555847168,
      "activations/layer12_attention_weight_max": 26.673084259033203,
      "activations/layer12_attention_weight_min": -31.180936813354492,
      "activations/layer13_attention_weight_max": 38.471920013427734,
      "activations/layer13_attention_weight_min": -35.4666748046875,
      "activations/layer14_attention_weight_max": 40.207149505615234,
      "activations/layer14_attention_weight_min": -37.988914489746094,
      "activations/layer15_attention_weight_max": 38.73823547363281,
      "activations/layer15_attention_weight_min": -34.8942985534668,
      "activations/layer16_attention_weight_max": 30.202306747436523,
      "activations/layer16_attention_weight_min": -26.666271209716797,
      "activations/layer17_attention_weight_max": 30.638954162597656,
      "activations/layer17_attention_weight_min": -25.183197021484375,
      "activations/layer18_attention_weight_max": 33.901798248291016,
      "activations/layer18_attention_weight_min": -25.71672821044922,
      "activations/layer19_attention_weight_max": 33.99636459350586,
      "activations/layer19_attention_weight_min": -29.616971969604492,
      "activations/layer1_attention_weight_max": 16.778661727905273,
      "activations/layer1_attention_weight_min": -14.998733520507812,
      "activations/layer20_attention_weight_max": 30.208478927612305,
      "activations/layer20_attention_weight_min": -24.961429595947266,
      "activations/layer21_attention_weight_max": 29.181926727294922,
      "activations/layer21_attention_weight_min": -24.194372177124023,
      "activations/layer22_attention_weight_max": 42.739498138427734,
      "activations/layer22_attention_weight_min": -30.765058517456055,
      "activations/layer23_attention_weight_max": 33.705169677734375,
      "activations/layer23_attention_weight_min": -24.663166046142578,
      "activations/layer2_attention_weight_max": 31.678281784057617,
      "activations/layer2_attention_weight_min": -31.301433563232422,
      "activations/layer3_attention_weight_max": 98.23082733154297,
      "activations/layer3_attention_weight_min": -99.3283462524414,
      "activations/layer4_attention_weight_max": 104.13055419921875,
      "activations/layer4_attention_weight_min": -105.4483413696289,
      "activations/layer5_attention_weight_max": 55.51165771484375,
      "activations/layer5_attention_weight_min": -61.227718353271484,
      "activations/layer6_attention_weight_max": 42.26720428466797,
      "activations/layer6_attention_weight_min": -45.935752868652344,
      "activations/layer7_attention_weight_max": 108.44105529785156,
      "activations/layer7_attention_weight_min": -95.43266296386719,
      "activations/layer8_attention_weight_max": 50.6750602722168,
      "activations/layer8_attention_weight_min": -50.368675231933594,
      "activations/layer9_attention_weight_max": 36.49565124511719,
      "activations/layer9_attention_weight_min": -38.07039260864258,
      "epoch": 17.66,
      "learning_rate": 3.651212121212121e-05,
      "loss": 2.74,
      "step": 303850
    },
    {
      "activations/layer0_attention_weight_max": 15.9478759765625,
      "activations/layer0_attention_weight_min": -12.31501293182373,
      "activations/layer10_attention_weight_max": 50.81973648071289,
      "activations/layer10_attention_weight_min": -47.013763427734375,
      "activations/layer11_attention_weight_max": 50.40088653564453,
      "activations/layer11_attention_weight_min": -48.259521484375,
      "activations/layer12_attention_weight_max": 31.05615997314453,
      "activations/layer12_attention_weight_min": -34.70402145385742,
      "activations/layer13_attention_weight_max": 46.15352249145508,
      "activations/layer13_attention_weight_min": -44.431785583496094,
      "activations/layer14_attention_weight_max": 53.666419982910156,
      "activations/layer14_attention_weight_min": -51.90616989135742,
      "activations/layer15_attention_weight_max": 45.96255111694336,
      "activations/layer15_attention_weight_min": -47.027557373046875,
      "activations/layer16_attention_weight_max": 32.62763214111328,
      "activations/layer16_attention_weight_min": -27.679790496826172,
      "activations/layer17_attention_weight_max": 29.24336051940918,
      "activations/layer17_attention_weight_min": -27.336362838745117,
      "activations/layer18_attention_weight_max": 32.635684967041016,
      "activations/layer18_attention_weight_min": -27.0224552154541,
      "activations/layer19_attention_weight_max": 35.89083480834961,
      "activations/layer19_attention_weight_min": -29.59084129333496,
      "activations/layer1_attention_weight_max": 16.607572555541992,
      "activations/layer1_attention_weight_min": -15.715553283691406,
      "activations/layer20_attention_weight_max": 28.822032928466797,
      "activations/layer20_attention_weight_min": -25.256853103637695,
      "activations/layer21_attention_weight_max": 29.597557067871094,
      "activations/layer21_attention_weight_min": -25.74560546875,
      "activations/layer22_attention_weight_max": 39.83065414428711,
      "activations/layer22_attention_weight_min": -30.42473030090332,
      "activations/layer23_attention_weight_max": 30.74795150756836,
      "activations/layer23_attention_weight_min": -26.379009246826172,
      "activations/layer2_attention_weight_max": 34.70574951171875,
      "activations/layer2_attention_weight_min": -38.81946563720703,
      "activations/layer3_attention_weight_max": 105.51630401611328,
      "activations/layer3_attention_weight_min": -111.91183471679688,
      "activations/layer4_attention_weight_max": 114.4994888305664,
      "activations/layer4_attention_weight_min": -112.71569061279297,
      "activations/layer5_attention_weight_max": 59.235164642333984,
      "activations/layer5_attention_weight_min": -60.7122917175293,
      "activations/layer6_attention_weight_max": 51.32695007324219,
      "activations/layer6_attention_weight_min": -49.13044738769531,
      "activations/layer7_attention_weight_max": 121.65594482421875,
      "activations/layer7_attention_weight_min": -113.95855712890625,
      "activations/layer8_attention_weight_max": 57.27070617675781,
      "activations/layer8_attention_weight_min": -53.759735107421875,
      "activations/layer9_attention_weight_max": 43.27695846557617,
      "activations/layer9_attention_weight_min": -43.60749435424805,
      "epoch": 17.66,
      "learning_rate": 3.649318181818181e-05,
      "loss": 2.7361,
      "step": 303900
    },
    {
      "activations/layer0_attention_weight_max": 15.873921394348145,
      "activations/layer0_attention_weight_min": -13.079588890075684,
      "activations/layer10_attention_weight_max": 32.59621810913086,
      "activations/layer10_attention_weight_min": -33.10031509399414,
      "activations/layer11_attention_weight_max": 32.45379638671875,
      "activations/layer11_attention_weight_min": -32.57926940917969,
      "activations/layer12_attention_weight_max": 30.678062438964844,
      "activations/layer12_attention_weight_min": -27.454910278320312,
      "activations/layer13_attention_weight_max": 44.8484992980957,
      "activations/layer13_attention_weight_min": -39.02895736694336,
      "activations/layer14_attention_weight_max": 45.595237731933594,
      "activations/layer14_attention_weight_min": -43.12685012817383,
      "activations/layer15_attention_weight_max": 47.35277557373047,
      "activations/layer15_attention_weight_min": -42.803611755371094,
      "activations/layer16_attention_weight_max": 32.51907730102539,
      "activations/layer16_attention_weight_min": -28.92436981201172,
      "activations/layer17_attention_weight_max": 31.86910057067871,
      "activations/layer17_attention_weight_min": -27.369665145874023,
      "activations/layer18_attention_weight_max": 34.42542266845703,
      "activations/layer18_attention_weight_min": -26.500934600830078,
      "activations/layer19_attention_weight_max": 38.11775588989258,
      "activations/layer19_attention_weight_min": -35.34377670288086,
      "activations/layer1_attention_weight_max": 15.815370559692383,
      "activations/layer1_attention_weight_min": -16.029882431030273,
      "activations/layer20_attention_weight_max": 31.521099090576172,
      "activations/layer20_attention_weight_min": -25.953474044799805,
      "activations/layer21_attention_weight_max": 30.78031349182129,
      "activations/layer21_attention_weight_min": -25.05156898498535,
      "activations/layer22_attention_weight_max": 45.277069091796875,
      "activations/layer22_attention_weight_min": -35.02320861816406,
      "activations/layer23_attention_weight_max": 35.408836364746094,
      "activations/layer23_attention_weight_min": -28.377105712890625,
      "activations/layer2_attention_weight_max": 33.44267654418945,
      "activations/layer2_attention_weight_min": -33.14828109741211,
      "activations/layer3_attention_weight_max": 101.40105438232422,
      "activations/layer3_attention_weight_min": -100.88910675048828,
      "activations/layer4_attention_weight_max": 108.66999816894531,
      "activations/layer4_attention_weight_min": -108.50965881347656,
      "activations/layer5_attention_weight_max": 49.75726318359375,
      "activations/layer5_attention_weight_min": -60.26609420776367,
      "activations/layer6_attention_weight_max": 42.69570541381836,
      "activations/layer6_attention_weight_min": -44.37675857543945,
      "activations/layer7_attention_weight_max": 87.2132339477539,
      "activations/layer7_attention_weight_min": -92.81040954589844,
      "activations/layer8_attention_weight_max": 42.38377380371094,
      "activations/layer8_attention_weight_min": -43.1006965637207,
      "activations/layer9_attention_weight_max": 33.14778518676758,
      "activations/layer9_attention_weight_min": -32.67565155029297,
      "epoch": 17.66,
      "learning_rate": 3.647424242424242e-05,
      "loss": 2.728,
      "step": 303950
    },
    {
      "activations/layer0_attention_weight_max": 17.13808250427246,
      "activations/layer0_attention_weight_min": -12.934294700622559,
      "activations/layer10_attention_weight_max": 36.38594436645508,
      "activations/layer10_attention_weight_min": -33.7557373046875,
      "activations/layer11_attention_weight_max": 35.141082763671875,
      "activations/layer11_attention_weight_min": -34.165626525878906,
      "activations/layer12_attention_weight_max": 27.01736068725586,
      "activations/layer12_attention_weight_min": -26.558719635009766,
      "activations/layer13_attention_weight_max": 39.048622131347656,
      "activations/layer13_attention_weight_min": -35.443233489990234,
      "activations/layer14_attention_weight_max": 41.09821701049805,
      "activations/layer14_attention_weight_min": -38.72219467163086,
      "activations/layer15_attention_weight_max": 37.294376373291016,
      "activations/layer15_attention_weight_min": -34.713890075683594,
      "activations/layer16_attention_weight_max": 28.273765563964844,
      "activations/layer16_attention_weight_min": -28.53556251525879,
      "activations/layer17_attention_weight_max": 28.211496353149414,
      "activations/layer17_attention_weight_min": -28.49081039428711,
      "activations/layer18_attention_weight_max": 32.76498031616211,
      "activations/layer18_attention_weight_min": -26.686246871948242,
      "activations/layer19_attention_weight_max": 33.64747619628906,
      "activations/layer19_attention_weight_min": -30.897048950195312,
      "activations/layer1_attention_weight_max": 16.04113006591797,
      "activations/layer1_attention_weight_min": -15.259645462036133,
      "activations/layer20_attention_weight_max": 30.91264533996582,
      "activations/layer20_attention_weight_min": -25.537492752075195,
      "activations/layer21_attention_weight_max": 29.137046813964844,
      "activations/layer21_attention_weight_min": -24.28091812133789,
      "activations/layer22_attention_weight_max": 42.848819732666016,
      "activations/layer22_attention_weight_min": -32.02528762817383,
      "activations/layer23_attention_weight_max": 31.895992279052734,
      "activations/layer23_attention_weight_min": -30.54346466064453,
      "activations/layer2_attention_weight_max": 32.70515060424805,
      "activations/layer2_attention_weight_min": -33.72282791137695,
      "activations/layer3_attention_weight_max": 104.1166763305664,
      "activations/layer3_attention_weight_min": -105.37218475341797,
      "activations/layer4_attention_weight_max": 110.01253509521484,
      "activations/layer4_attention_weight_min": -113.2121810913086,
      "activations/layer5_attention_weight_max": 50.082786560058594,
      "activations/layer5_attention_weight_min": -60.496788024902344,
      "activations/layer6_attention_weight_max": 45.19913101196289,
      "activations/layer6_attention_weight_min": -47.44353103637695,
      "activations/layer7_attention_weight_max": 95.45744323730469,
      "activations/layer7_attention_weight_min": -99.070068359375,
      "activations/layer8_attention_weight_max": 43.34540557861328,
      "activations/layer8_attention_weight_min": -43.76713180541992,
      "activations/layer9_attention_weight_max": 32.63176345825195,
      "activations/layer9_attention_weight_min": -35.300743103027344,
      "epoch": 17.66,
      "learning_rate": 3.645530303030303e-05,
      "loss": 2.7331,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4645,
      "eval_samples_per_second": 507.292,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4645,
      "eval_openwebtext_samples_per_second": 507.292,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9786,
      "eval_wikitext_samples_per_second": 230.469,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_lambada_loss": 2.44140625,
      "eval_lambada_ppl": 11.489186055851967,
      "eval_lambada_runtime": 9.5644,
      "eval_lambada_samples_per_second": 509.075,
      "step": 304000
    },
    {
      "activations/layer0_attention_weight_max": 16.487245559692383,
      "activations/layer0_attention_weight_min": -13.846331596374512,
      "activations/layer10_attention_weight_max": 32.80376434326172,
      "activations/layer10_attention_weight_min": -34.147071838378906,
      "activations/layer11_attention_weight_max": 32.339263916015625,
      "activations/layer11_attention_weight_min": -35.36730194091797,
      "activations/layer12_attention_weight_max": 26.22240447998047,
      "activations/layer12_attention_weight_min": -26.1556453704834,
      "activations/layer13_attention_weight_max": 40.09803771972656,
      "activations/layer13_attention_weight_min": -36.80127716064453,
      "activations/layer14_attention_weight_max": 41.51251220703125,
      "activations/layer14_attention_weight_min": -38.188697814941406,
      "activations/layer15_attention_weight_max": 39.89505386352539,
      "activations/layer15_attention_weight_min": -35.116512298583984,
      "activations/layer16_attention_weight_max": 28.93781089782715,
      "activations/layer16_attention_weight_min": -28.56264305114746,
      "activations/layer17_attention_weight_max": 31.83254051208496,
      "activations/layer17_attention_weight_min": -28.33701515197754,
      "activations/layer18_attention_weight_max": 30.05140495300293,
      "activations/layer18_attention_weight_min": -27.246017456054688,
      "activations/layer19_attention_weight_max": 36.261722564697266,
      "activations/layer19_attention_weight_min": -31.46881675720215,
      "activations/layer1_attention_weight_max": 16.525798797607422,
      "activations/layer1_attention_weight_min": -15.761727333068848,
      "activations/layer20_attention_weight_max": 29.976146697998047,
      "activations/layer20_attention_weight_min": -27.24853515625,
      "activations/layer21_attention_weight_max": 27.601327896118164,
      "activations/layer21_attention_weight_min": -26.064971923828125,
      "activations/layer22_attention_weight_max": 45.61056900024414,
      "activations/layer22_attention_weight_min": -32.78998565673828,
      "activations/layer23_attention_weight_max": 33.317527770996094,
      "activations/layer23_attention_weight_min": -27.8582820892334,
      "activations/layer2_attention_weight_max": 31.980680465698242,
      "activations/layer2_attention_weight_min": -33.064430236816406,
      "activations/layer3_attention_weight_max": 95.54788208007812,
      "activations/layer3_attention_weight_min": -98.45404052734375,
      "activations/layer4_attention_weight_max": 99.19638061523438,
      "activations/layer4_attention_weight_min": -109.35919189453125,
      "activations/layer5_attention_weight_max": 48.36126708984375,
      "activations/layer5_attention_weight_min": -61.00019454956055,
      "activations/layer6_attention_weight_max": 43.41563034057617,
      "activations/layer6_attention_weight_min": -46.15671920776367,
      "activations/layer7_attention_weight_max": 90.1799087524414,
      "activations/layer7_attention_weight_min": -96.87126922607422,
      "activations/layer8_attention_weight_max": 39.932254791259766,
      "activations/layer8_attention_weight_min": -45.32674026489258,
      "activations/layer9_attention_weight_max": 32.159481048583984,
      "activations/layer9_attention_weight_min": -35.516841888427734,
      "epoch": 17.67,
      "learning_rate": 3.643636363636363e-05,
      "loss": 2.7422,
      "step": 304050
    },
    {
      "activations/layer0_attention_weight_max": 16.634977340698242,
      "activations/layer0_attention_weight_min": -12.775506019592285,
      "activations/layer10_attention_weight_max": 37.28042984008789,
      "activations/layer10_attention_weight_min": -33.2331428527832,
      "activations/layer11_attention_weight_max": 36.54138946533203,
      "activations/layer11_attention_weight_min": -35.302547454833984,
      "activations/layer12_attention_weight_max": 38.35496139526367,
      "activations/layer12_attention_weight_min": -30.800464630126953,
      "activations/layer13_attention_weight_max": 48.77826690673828,
      "activations/layer13_attention_weight_min": -34.61883544921875,
      "activations/layer14_attention_weight_max": 52.40311813354492,
      "activations/layer14_attention_weight_min": -39.598365783691406,
      "activations/layer15_attention_weight_max": 49.526527404785156,
      "activations/layer15_attention_weight_min": -40.86573028564453,
      "activations/layer16_attention_weight_max": 33.962127685546875,
      "activations/layer16_attention_weight_min": -26.999937057495117,
      "activations/layer17_attention_weight_max": 30.236793518066406,
      "activations/layer17_attention_weight_min": -26.78262710571289,
      "activations/layer18_attention_weight_max": 30.350242614746094,
      "activations/layer18_attention_weight_min": -27.372915267944336,
      "activations/layer19_attention_weight_max": 30.573196411132812,
      "activations/layer19_attention_weight_min": -33.55727005004883,
      "activations/layer1_attention_weight_max": 16.413686752319336,
      "activations/layer1_attention_weight_min": -14.866242408752441,
      "activations/layer20_attention_weight_max": 28.259624481201172,
      "activations/layer20_attention_weight_min": -24.81319236755371,
      "activations/layer21_attention_weight_max": 26.457517623901367,
      "activations/layer21_attention_weight_min": -27.906652450561523,
      "activations/layer22_attention_weight_max": 38.706912994384766,
      "activations/layer22_attention_weight_min": -32.62881851196289,
      "activations/layer23_attention_weight_max": 34.2309455871582,
      "activations/layer23_attention_weight_min": -26.332799911499023,
      "activations/layer2_attention_weight_max": 32.74903106689453,
      "activations/layer2_attention_weight_min": -31.68731689453125,
      "activations/layer3_attention_weight_max": 97.75249481201172,
      "activations/layer3_attention_weight_min": -92.85974884033203,
      "activations/layer4_attention_weight_max": 100.2081527709961,
      "activations/layer4_attention_weight_min": -99.63765716552734,
      "activations/layer5_attention_weight_max": 52.115028381347656,
      "activations/layer5_attention_weight_min": -58.36027526855469,
      "activations/layer6_attention_weight_max": 42.293312072753906,
      "activations/layer6_attention_weight_min": -46.54444885253906,
      "activations/layer7_attention_weight_max": 99.25560760498047,
      "activations/layer7_attention_weight_min": -91.6980209350586,
      "activations/layer8_attention_weight_max": 41.17351531982422,
      "activations/layer8_attention_weight_min": -42.28271484375,
      "activations/layer9_attention_weight_max": 34.00259017944336,
      "activations/layer9_attention_weight_min": -33.56877136230469,
      "epoch": 17.67,
      "learning_rate": 3.641742424242424e-05,
      "loss": 2.7317,
      "step": 304100
    },
    {
      "activations/layer0_attention_weight_max": 16.578916549682617,
      "activations/layer0_attention_weight_min": -12.8710298538208,
      "activations/layer10_attention_weight_max": 33.5931396484375,
      "activations/layer10_attention_weight_min": -36.21232986450195,
      "activations/layer11_attention_weight_max": 34.803749084472656,
      "activations/layer11_attention_weight_min": -36.6456184387207,
      "activations/layer12_attention_weight_max": 24.829076766967773,
      "activations/layer12_attention_weight_min": -27.237329483032227,
      "activations/layer13_attention_weight_max": 35.87077331542969,
      "activations/layer13_attention_weight_min": -36.3967399597168,
      "activations/layer14_attention_weight_max": 36.252784729003906,
      "activations/layer14_attention_weight_min": -38.95880889892578,
      "activations/layer15_attention_weight_max": 34.576114654541016,
      "activations/layer15_attention_weight_min": -36.948402404785156,
      "activations/layer16_attention_weight_max": 26.51816749572754,
      "activations/layer16_attention_weight_min": -29.441604614257812,
      "activations/layer17_attention_weight_max": 25.784666061401367,
      "activations/layer17_attention_weight_min": -27.8321590423584,
      "activations/layer18_attention_weight_max": 29.226045608520508,
      "activations/layer18_attention_weight_min": -28.413497924804688,
      "activations/layer19_attention_weight_max": 31.90142250061035,
      "activations/layer19_attention_weight_min": -33.97988510131836,
      "activations/layer1_attention_weight_max": 17.18793296813965,
      "activations/layer1_attention_weight_min": -15.812939643859863,
      "activations/layer20_attention_weight_max": 26.759416580200195,
      "activations/layer20_attention_weight_min": -28.157608032226562,
      "activations/layer21_attention_weight_max": 27.42731285095215,
      "activations/layer21_attention_weight_min": -27.037736892700195,
      "activations/layer22_attention_weight_max": 40.32331085205078,
      "activations/layer22_attention_weight_min": -33.44633102416992,
      "activations/layer23_attention_weight_max": 29.017642974853516,
      "activations/layer23_attention_weight_min": -27.237743377685547,
      "activations/layer2_attention_weight_max": 35.743080139160156,
      "activations/layer2_attention_weight_min": -34.77600860595703,
      "activations/layer3_attention_weight_max": 102.96851348876953,
      "activations/layer3_attention_weight_min": -100.5105972290039,
      "activations/layer4_attention_weight_max": 107.50127410888672,
      "activations/layer4_attention_weight_min": -109.1073226928711,
      "activations/layer5_attention_weight_max": 50.34056091308594,
      "activations/layer5_attention_weight_min": -61.3697509765625,
      "activations/layer6_attention_weight_max": 44.793846130371094,
      "activations/layer6_attention_weight_min": -46.67110061645508,
      "activations/layer7_attention_weight_max": 97.3056869506836,
      "activations/layer7_attention_weight_min": -101.52091979980469,
      "activations/layer8_attention_weight_max": 43.016395568847656,
      "activations/layer8_attention_weight_min": -44.220497131347656,
      "activations/layer9_attention_weight_max": 32.748634338378906,
      "activations/layer9_attention_weight_min": -35.3785514831543,
      "epoch": 17.67,
      "learning_rate": 3.6398484848484847e-05,
      "loss": 2.7522,
      "step": 304150
    },
    {
      "activations/layer0_attention_weight_max": 15.821593284606934,
      "activations/layer0_attention_weight_min": -12.256057739257812,
      "activations/layer10_attention_weight_max": 37.761207580566406,
      "activations/layer10_attention_weight_min": -35.48649978637695,
      "activations/layer11_attention_weight_max": 32.85009002685547,
      "activations/layer11_attention_weight_min": -35.81610107421875,
      "activations/layer12_attention_weight_max": 23.60037612915039,
      "activations/layer12_attention_weight_min": -26.618717193603516,
      "activations/layer13_attention_weight_max": 33.78312683105469,
      "activations/layer13_attention_weight_min": -36.30025863647461,
      "activations/layer14_attention_weight_max": 35.78757095336914,
      "activations/layer14_attention_weight_min": -40.414093017578125,
      "activations/layer15_attention_weight_max": 34.078975677490234,
      "activations/layer15_attention_weight_min": -38.128334045410156,
      "activations/layer16_attention_weight_max": 27.06551170349121,
      "activations/layer16_attention_weight_min": -29.14844512939453,
      "activations/layer17_attention_weight_max": 27.30030632019043,
      "activations/layer17_attention_weight_min": -27.29851531982422,
      "activations/layer18_attention_weight_max": 29.504789352416992,
      "activations/layer18_attention_weight_min": -26.306032180786133,
      "activations/layer19_attention_weight_max": 31.798355102539062,
      "activations/layer19_attention_weight_min": -33.281150817871094,
      "activations/layer1_attention_weight_max": 16.55063819885254,
      "activations/layer1_attention_weight_min": -15.747961044311523,
      "activations/layer20_attention_weight_max": 25.76934051513672,
      "activations/layer20_attention_weight_min": -25.872907638549805,
      "activations/layer21_attention_weight_max": 29.933469772338867,
      "activations/layer21_attention_weight_min": -26.59981346130371,
      "activations/layer22_attention_weight_max": 37.880165100097656,
      "activations/layer22_attention_weight_min": -32.70813751220703,
      "activations/layer23_attention_weight_max": 31.40468978881836,
      "activations/layer23_attention_weight_min": -27.977649688720703,
      "activations/layer2_attention_weight_max": 35.52372360229492,
      "activations/layer2_attention_weight_min": -32.638980865478516,
      "activations/layer3_attention_weight_max": 97.6178970336914,
      "activations/layer3_attention_weight_min": -99.81541442871094,
      "activations/layer4_attention_weight_max": 105.40704345703125,
      "activations/layer4_attention_weight_min": -113.62420654296875,
      "activations/layer5_attention_weight_max": 51.914031982421875,
      "activations/layer5_attention_weight_min": -63.091590881347656,
      "activations/layer6_attention_weight_max": 46.78731918334961,
      "activations/layer6_attention_weight_min": -48.751014709472656,
      "activations/layer7_attention_weight_max": 91.96952056884766,
      "activations/layer7_attention_weight_min": -97.23811340332031,
      "activations/layer8_attention_weight_max": 41.520286560058594,
      "activations/layer8_attention_weight_min": -45.51328659057617,
      "activations/layer9_attention_weight_max": 34.225948333740234,
      "activations/layer9_attention_weight_min": -35.68253707885742,
      "epoch": 17.68,
      "learning_rate": 3.6379545454545455e-05,
      "loss": 2.7596,
      "step": 304200
    },
    {
      "activations/layer0_attention_weight_max": 16.025638580322266,
      "activations/layer0_attention_weight_min": -12.75688362121582,
      "activations/layer10_attention_weight_max": 39.781593322753906,
      "activations/layer10_attention_weight_min": -40.137123107910156,
      "activations/layer11_attention_weight_max": 36.78850173950195,
      "activations/layer11_attention_weight_min": -38.261539459228516,
      "activations/layer12_attention_weight_max": 30.730548858642578,
      "activations/layer12_attention_weight_min": -27.19974708557129,
      "activations/layer13_attention_weight_max": 41.366363525390625,
      "activations/layer13_attention_weight_min": -37.72945785522461,
      "activations/layer14_attention_weight_max": 41.653839111328125,
      "activations/layer14_attention_weight_min": -40.42941665649414,
      "activations/layer15_attention_weight_max": 38.37434005737305,
      "activations/layer15_attention_weight_min": -39.17981719970703,
      "activations/layer16_attention_weight_max": 27.434253692626953,
      "activations/layer16_attention_weight_min": -29.7106876373291,
      "activations/layer17_attention_weight_max": 29.44675636291504,
      "activations/layer17_attention_weight_min": -26.556711196899414,
      "activations/layer18_attention_weight_max": 33.0470085144043,
      "activations/layer18_attention_weight_min": -27.332731246948242,
      "activations/layer19_attention_weight_max": 31.73219108581543,
      "activations/layer19_attention_weight_min": -32.380367279052734,
      "activations/layer1_attention_weight_max": 16.51491355895996,
      "activations/layer1_attention_weight_min": -14.733120918273926,
      "activations/layer20_attention_weight_max": 27.254838943481445,
      "activations/layer20_attention_weight_min": -26.348426818847656,
      "activations/layer21_attention_weight_max": 29.8615779876709,
      "activations/layer21_attention_weight_min": -27.19761848449707,
      "activations/layer22_attention_weight_max": 41.68125534057617,
      "activations/layer22_attention_weight_min": -34.21330642700195,
      "activations/layer23_attention_weight_max": 31.720905303955078,
      "activations/layer23_attention_weight_min": -28.098751068115234,
      "activations/layer2_attention_weight_max": 33.87120056152344,
      "activations/layer2_attention_weight_min": -34.39744186401367,
      "activations/layer3_attention_weight_max": 103.69882202148438,
      "activations/layer3_attention_weight_min": -104.47268676757812,
      "activations/layer4_attention_weight_max": 110.21583557128906,
      "activations/layer4_attention_weight_min": -117.19002532958984,
      "activations/layer5_attention_weight_max": 58.08814239501953,
      "activations/layer5_attention_weight_min": -64.01182556152344,
      "activations/layer6_attention_weight_max": 46.970314025878906,
      "activations/layer6_attention_weight_min": -51.5399055480957,
      "activations/layer7_attention_weight_max": 98.43037414550781,
      "activations/layer7_attention_weight_min": -105.82026672363281,
      "activations/layer8_attention_weight_max": 46.81209945678711,
      "activations/layer8_attention_weight_min": -49.007362365722656,
      "activations/layer9_attention_weight_max": 36.78701400756836,
      "activations/layer9_attention_weight_min": -37.74100875854492,
      "epoch": 17.68,
      "learning_rate": 3.6360606060606056e-05,
      "loss": 2.7453,
      "step": 304250
    },
    {
      "activations/layer0_attention_weight_max": 15.292529106140137,
      "activations/layer0_attention_weight_min": -12.981403350830078,
      "activations/layer10_attention_weight_max": 31.258487701416016,
      "activations/layer10_attention_weight_min": -34.563995361328125,
      "activations/layer11_attention_weight_max": 31.342695236206055,
      "activations/layer11_attention_weight_min": -34.173946380615234,
      "activations/layer12_attention_weight_max": 25.418607711791992,
      "activations/layer12_attention_weight_min": -27.039804458618164,
      "activations/layer13_attention_weight_max": 38.893150329589844,
      "activations/layer13_attention_weight_min": -36.6897087097168,
      "activations/layer14_attention_weight_max": 38.278255462646484,
      "activations/layer14_attention_weight_min": -39.97867965698242,
      "activations/layer15_attention_weight_max": 36.559471130371094,
      "activations/layer15_attention_weight_min": -37.613162994384766,
      "activations/layer16_attention_weight_max": 28.139362335205078,
      "activations/layer16_attention_weight_min": -28.43404197692871,
      "activations/layer17_attention_weight_max": 27.114057540893555,
      "activations/layer17_attention_weight_min": -27.186838150024414,
      "activations/layer18_attention_weight_max": 31.507244110107422,
      "activations/layer18_attention_weight_min": -26.816726684570312,
      "activations/layer19_attention_weight_max": 33.2183837890625,
      "activations/layer19_attention_weight_min": -32.823184967041016,
      "activations/layer1_attention_weight_max": 15.472896575927734,
      "activations/layer1_attention_weight_min": -14.667768478393555,
      "activations/layer20_attention_weight_max": 30.35982894897461,
      "activations/layer20_attention_weight_min": -27.326126098632812,
      "activations/layer21_attention_weight_max": 28.25034523010254,
      "activations/layer21_attention_weight_min": -26.04168701171875,
      "activations/layer22_attention_weight_max": 42.35167694091797,
      "activations/layer22_attention_weight_min": -34.34467315673828,
      "activations/layer23_attention_weight_max": 30.99646759033203,
      "activations/layer23_attention_weight_min": -27.875585556030273,
      "activations/layer2_attention_weight_max": 32.694557189941406,
      "activations/layer2_attention_weight_min": -32.50458526611328,
      "activations/layer3_attention_weight_max": 97.80904388427734,
      "activations/layer3_attention_weight_min": -99.13041687011719,
      "activations/layer4_attention_weight_max": 105.92303466796875,
      "activations/layer4_attention_weight_min": -106.75543212890625,
      "activations/layer5_attention_weight_max": 49.32752990722656,
      "activations/layer5_attention_weight_min": -58.76959228515625,
      "activations/layer6_attention_weight_max": 43.6662712097168,
      "activations/layer6_attention_weight_min": -45.664161682128906,
      "activations/layer7_attention_weight_max": 90.39576721191406,
      "activations/layer7_attention_weight_min": -100.550048828125,
      "activations/layer8_attention_weight_max": 39.99748992919922,
      "activations/layer8_attention_weight_min": -43.37798309326172,
      "activations/layer9_attention_weight_max": 31.282089233398438,
      "activations/layer9_attention_weight_min": -35.25654602050781,
      "epoch": 17.68,
      "learning_rate": 3.6341666666666665e-05,
      "loss": 2.7237,
      "step": 304300
    },
    {
      "activations/layer0_attention_weight_max": 15.729372024536133,
      "activations/layer0_attention_weight_min": -14.094277381896973,
      "activations/layer10_attention_weight_max": 32.68727111816406,
      "activations/layer10_attention_weight_min": -34.04301834106445,
      "activations/layer11_attention_weight_max": 31.295873641967773,
      "activations/layer11_attention_weight_min": -34.96990203857422,
      "activations/layer12_attention_weight_max": 26.255685806274414,
      "activations/layer12_attention_weight_min": -28.86582374572754,
      "activations/layer13_attention_weight_max": 38.08992385864258,
      "activations/layer13_attention_weight_min": -36.85227584838867,
      "activations/layer14_attention_weight_max": 41.07473373413086,
      "activations/layer14_attention_weight_min": -37.965126037597656,
      "activations/layer15_attention_weight_max": 38.6262092590332,
      "activations/layer15_attention_weight_min": -38.26237106323242,
      "activations/layer16_attention_weight_max": 29.733755111694336,
      "activations/layer16_attention_weight_min": -28.805660247802734,
      "activations/layer17_attention_weight_max": 30.9418888092041,
      "activations/layer17_attention_weight_min": -29.204524993896484,
      "activations/layer18_attention_weight_max": 30.276737213134766,
      "activations/layer18_attention_weight_min": -27.25056266784668,
      "activations/layer19_attention_weight_max": 34.03950119018555,
      "activations/layer19_attention_weight_min": -32.9583625793457,
      "activations/layer1_attention_weight_max": 15.721532821655273,
      "activations/layer1_attention_weight_min": -15.761409759521484,
      "activations/layer20_attention_weight_max": 30.5848445892334,
      "activations/layer20_attention_weight_min": -28.23744773864746,
      "activations/layer21_attention_weight_max": 28.88597869873047,
      "activations/layer21_attention_weight_min": -26.93282127380371,
      "activations/layer22_attention_weight_max": 41.43033218383789,
      "activations/layer22_attention_weight_min": -34.75510787963867,
      "activations/layer23_attention_weight_max": 31.77862548828125,
      "activations/layer23_attention_weight_min": -29.110868453979492,
      "activations/layer2_attention_weight_max": 32.97763442993164,
      "activations/layer2_attention_weight_min": -33.65205001831055,
      "activations/layer3_attention_weight_max": 100.1149673461914,
      "activations/layer3_attention_weight_min": -103.29808044433594,
      "activations/layer4_attention_weight_max": 103.31974029541016,
      "activations/layer4_attention_weight_min": -108.1932601928711,
      "activations/layer5_attention_weight_max": 47.56241989135742,
      "activations/layer5_attention_weight_min": -60.703575134277344,
      "activations/layer6_attention_weight_max": 43.20412826538086,
      "activations/layer6_attention_weight_min": -46.51620864868164,
      "activations/layer7_attention_weight_max": 91.17048645019531,
      "activations/layer7_attention_weight_min": -95.02593994140625,
      "activations/layer8_attention_weight_max": 40.82049560546875,
      "activations/layer8_attention_weight_min": -43.95901107788086,
      "activations/layer9_attention_weight_max": 31.301132202148438,
      "activations/layer9_attention_weight_min": -33.3218879699707,
      "epoch": 17.68,
      "learning_rate": 3.6322727272727266e-05,
      "loss": 2.7325,
      "step": 304350
    },
    {
      "activations/layer0_attention_weight_max": 16.281259536743164,
      "activations/layer0_attention_weight_min": -12.637401580810547,
      "activations/layer10_attention_weight_max": 33.83761978149414,
      "activations/layer10_attention_weight_min": -34.907779693603516,
      "activations/layer11_attention_weight_max": 33.02522659301758,
      "activations/layer11_attention_weight_min": -35.006282806396484,
      "activations/layer12_attention_weight_max": 30.59701919555664,
      "activations/layer12_attention_weight_min": -27.89211654663086,
      "activations/layer13_attention_weight_max": 42.76679611206055,
      "activations/layer13_attention_weight_min": -36.295196533203125,
      "activations/layer14_attention_weight_max": 45.61073684692383,
      "activations/layer14_attention_weight_min": -38.89559555053711,
      "activations/layer15_attention_weight_max": 39.20162582397461,
      "activations/layer15_attention_weight_min": -38.03196716308594,
      "activations/layer16_attention_weight_max": 27.7738037109375,
      "activations/layer16_attention_weight_min": -30.64586639404297,
      "activations/layer17_attention_weight_max": 30.67519760131836,
      "activations/layer17_attention_weight_min": -28.28917121887207,
      "activations/layer18_attention_weight_max": 32.112281799316406,
      "activations/layer18_attention_weight_min": -28.615449905395508,
      "activations/layer19_attention_weight_max": 36.17531967163086,
      "activations/layer19_attention_weight_min": -34.14131164550781,
      "activations/layer1_attention_weight_max": 16.70821189880371,
      "activations/layer1_attention_weight_min": -15.289695739746094,
      "activations/layer20_attention_weight_max": 30.83269500732422,
      "activations/layer20_attention_weight_min": -27.985107421875,
      "activations/layer21_attention_weight_max": 30.895793914794922,
      "activations/layer21_attention_weight_min": -27.347055435180664,
      "activations/layer22_attention_weight_max": 45.12592315673828,
      "activations/layer22_attention_weight_min": -34.18846893310547,
      "activations/layer23_attention_weight_max": 33.547027587890625,
      "activations/layer23_attention_weight_min": -29.670398712158203,
      "activations/layer2_attention_weight_max": 34.44025421142578,
      "activations/layer2_attention_weight_min": -35.20551681518555,
      "activations/layer3_attention_weight_max": 99.89362335205078,
      "activations/layer3_attention_weight_min": -102.8871078491211,
      "activations/layer4_attention_weight_max": 107.7123031616211,
      "activations/layer4_attention_weight_min": -109.89861297607422,
      "activations/layer5_attention_weight_max": 48.60749816894531,
      "activations/layer5_attention_weight_min": -70.66798400878906,
      "activations/layer6_attention_weight_max": 44.91984939575195,
      "activations/layer6_attention_weight_min": -46.18497085571289,
      "activations/layer7_attention_weight_max": 94.94084167480469,
      "activations/layer7_attention_weight_min": -101.13626861572266,
      "activations/layer8_attention_weight_max": 43.42859649658203,
      "activations/layer8_attention_weight_min": -45.227691650390625,
      "activations/layer9_attention_weight_max": 31.712919235229492,
      "activations/layer9_attention_weight_min": -34.94289779663086,
      "epoch": 17.69,
      "learning_rate": 3.6303787878787874e-05,
      "loss": 2.7498,
      "step": 304400
    },
    {
      "activations/layer0_attention_weight_max": 16.67867660522461,
      "activations/layer0_attention_weight_min": -13.09972095489502,
      "activations/layer10_attention_weight_max": 33.58026885986328,
      "activations/layer10_attention_weight_min": -36.556522369384766,
      "activations/layer11_attention_weight_max": 33.26771545410156,
      "activations/layer11_attention_weight_min": -36.24119567871094,
      "activations/layer12_attention_weight_max": 23.670780181884766,
      "activations/layer12_attention_weight_min": -28.378698348999023,
      "activations/layer13_attention_weight_max": 33.71027374267578,
      "activations/layer13_attention_weight_min": -37.74553680419922,
      "activations/layer14_attention_weight_max": 35.55120849609375,
      "activations/layer14_attention_weight_min": -38.93070983886719,
      "activations/layer15_attention_weight_max": 33.13904571533203,
      "activations/layer15_attention_weight_min": -38.94818878173828,
      "activations/layer16_attention_weight_max": 25.339311599731445,
      "activations/layer16_attention_weight_min": -28.652259826660156,
      "activations/layer17_attention_weight_max": 27.51412582397461,
      "activations/layer17_attention_weight_min": -26.97742462158203,
      "activations/layer18_attention_weight_max": 29.609188079833984,
      "activations/layer18_attention_weight_min": -27.01188850402832,
      "activations/layer19_attention_weight_max": 26.93317985534668,
      "activations/layer19_attention_weight_min": -32.74273681640625,
      "activations/layer1_attention_weight_max": 17.080036163330078,
      "activations/layer1_attention_weight_min": -15.458099365234375,
      "activations/layer20_attention_weight_max": 25.072751998901367,
      "activations/layer20_attention_weight_min": -26.729412078857422,
      "activations/layer21_attention_weight_max": 28.485300064086914,
      "activations/layer21_attention_weight_min": -25.052743911743164,
      "activations/layer22_attention_weight_max": 34.87516784667969,
      "activations/layer22_attention_weight_min": -31.224592208862305,
      "activations/layer23_attention_weight_max": 28.03333282470703,
      "activations/layer23_attention_weight_min": -25.625457763671875,
      "activations/layer2_attention_weight_max": 34.400089263916016,
      "activations/layer2_attention_weight_min": -33.80562973022461,
      "activations/layer3_attention_weight_max": 101.2087631225586,
      "activations/layer3_attention_weight_min": -102.26994323730469,
      "activations/layer4_attention_weight_max": 107.67049407958984,
      "activations/layer4_attention_weight_min": -112.53018951416016,
      "activations/layer5_attention_weight_max": 49.56376266479492,
      "activations/layer5_attention_weight_min": -61.360206604003906,
      "activations/layer6_attention_weight_max": 44.57549285888672,
      "activations/layer6_attention_weight_min": -49.54374313354492,
      "activations/layer7_attention_weight_max": 93.90338134765625,
      "activations/layer7_attention_weight_min": -103.5578384399414,
      "activations/layer8_attention_weight_max": 41.64168167114258,
      "activations/layer8_attention_weight_min": -48.04286575317383,
      "activations/layer9_attention_weight_max": 31.30936622619629,
      "activations/layer9_attention_weight_min": -35.126304626464844,
      "epoch": 17.69,
      "learning_rate": 3.628484848484848e-05,
      "loss": 2.7517,
      "step": 304450
    },
    {
      "activations/layer0_attention_weight_max": 16.18532371520996,
      "activations/layer0_attention_weight_min": -12.578591346740723,
      "activations/layer10_attention_weight_max": 34.34552764892578,
      "activations/layer10_attention_weight_min": -34.73493957519531,
      "activations/layer11_attention_weight_max": 31.492847442626953,
      "activations/layer11_attention_weight_min": -34.74824905395508,
      "activations/layer12_attention_weight_max": 26.553024291992188,
      "activations/layer12_attention_weight_min": -27.686145782470703,
      "activations/layer13_attention_weight_max": 36.41810989379883,
      "activations/layer13_attention_weight_min": -36.465980529785156,
      "activations/layer14_attention_weight_max": 38.589698791503906,
      "activations/layer14_attention_weight_min": -38.286617279052734,
      "activations/layer15_attention_weight_max": 36.29888153076172,
      "activations/layer15_attention_weight_min": -38.54417037963867,
      "activations/layer16_attention_weight_max": 30.548095703125,
      "activations/layer16_attention_weight_min": -29.08985710144043,
      "activations/layer17_attention_weight_max": 28.96932601928711,
      "activations/layer17_attention_weight_min": -28.6970157623291,
      "activations/layer18_attention_weight_max": 31.30035400390625,
      "activations/layer18_attention_weight_min": -29.918136596679688,
      "activations/layer19_attention_weight_max": 35.33579635620117,
      "activations/layer19_attention_weight_min": -34.5501823425293,
      "activations/layer1_attention_weight_max": 16.388883590698242,
      "activations/layer1_attention_weight_min": -15.424233436584473,
      "activations/layer20_attention_weight_max": 31.344541549682617,
      "activations/layer20_attention_weight_min": -29.92461585998535,
      "activations/layer21_attention_weight_max": 30.445602416992188,
      "activations/layer21_attention_weight_min": -29.408132553100586,
      "activations/layer22_attention_weight_max": 45.56460952758789,
      "activations/layer22_attention_weight_min": -37.53749465942383,
      "activations/layer23_attention_weight_max": 33.48546600341797,
      "activations/layer23_attention_weight_min": -30.892066955566406,
      "activations/layer2_attention_weight_max": 33.277435302734375,
      "activations/layer2_attention_weight_min": -34.98345947265625,
      "activations/layer3_attention_weight_max": 100.76895904541016,
      "activations/layer3_attention_weight_min": -104.28169250488281,
      "activations/layer4_attention_weight_max": 106.50593566894531,
      "activations/layer4_attention_weight_min": -106.71837615966797,
      "activations/layer5_attention_weight_max": 51.536415100097656,
      "activations/layer5_attention_weight_min": -59.248451232910156,
      "activations/layer6_attention_weight_max": 42.74666213989258,
      "activations/layer6_attention_weight_min": -47.30049133300781,
      "activations/layer7_attention_weight_max": 90.45170593261719,
      "activations/layer7_attention_weight_min": -105.8180160522461,
      "activations/layer8_attention_weight_max": 38.25654602050781,
      "activations/layer8_attention_weight_min": -43.59272003173828,
      "activations/layer9_attention_weight_max": 30.330610275268555,
      "activations/layer9_attention_weight_min": -35.93365478515625,
      "epoch": 17.69,
      "learning_rate": 3.626590909090909e-05,
      "loss": 2.7581,
      "step": 304500
    },
    {
      "activations/layer0_attention_weight_max": 15.901579856872559,
      "activations/layer0_attention_weight_min": -12.191754341125488,
      "activations/layer10_attention_weight_max": 37.49024200439453,
      "activations/layer10_attention_weight_min": -35.11009216308594,
      "activations/layer11_attention_weight_max": 35.28787612915039,
      "activations/layer11_attention_weight_min": -35.178619384765625,
      "activations/layer12_attention_weight_max": 28.05276107788086,
      "activations/layer12_attention_weight_min": -29.11868667602539,
      "activations/layer13_attention_weight_max": 39.64556884765625,
      "activations/layer13_attention_weight_min": -36.74329376220703,
      "activations/layer14_attention_weight_max": 42.233917236328125,
      "activations/layer14_attention_weight_min": -39.21223831176758,
      "activations/layer15_attention_weight_max": 40.2047004699707,
      "activations/layer15_attention_weight_min": -37.7006721496582,
      "activations/layer16_attention_weight_max": 30.769527435302734,
      "activations/layer16_attention_weight_min": -30.54205322265625,
      "activations/layer17_attention_weight_max": 28.437841415405273,
      "activations/layer17_attention_weight_min": -28.667070388793945,
      "activations/layer18_attention_weight_max": 28.76601791381836,
      "activations/layer18_attention_weight_min": -27.341697692871094,
      "activations/layer19_attention_weight_max": 32.16287612915039,
      "activations/layer19_attention_weight_min": -34.00999450683594,
      "activations/layer1_attention_weight_max": 16.108718872070312,
      "activations/layer1_attention_weight_min": -14.741206169128418,
      "activations/layer20_attention_weight_max": 29.450803756713867,
      "activations/layer20_attention_weight_min": -26.275798797607422,
      "activations/layer21_attention_weight_max": 28.59996795654297,
      "activations/layer21_attention_weight_min": -24.727886199951172,
      "activations/layer22_attention_weight_max": 40.543643951416016,
      "activations/layer22_attention_weight_min": -32.325077056884766,
      "activations/layer23_attention_weight_max": 30.592397689819336,
      "activations/layer23_attention_weight_min": -26.66329002380371,
      "activations/layer2_attention_weight_max": 35.15351867675781,
      "activations/layer2_attention_weight_min": -33.6688346862793,
      "activations/layer3_attention_weight_max": 102.62008666992188,
      "activations/layer3_attention_weight_min": -102.6832504272461,
      "activations/layer4_attention_weight_max": 108.39611053466797,
      "activations/layer4_attention_weight_min": -109.2868423461914,
      "activations/layer5_attention_weight_max": 49.0852165222168,
      "activations/layer5_attention_weight_min": -59.77677917480469,
      "activations/layer6_attention_weight_max": 45.20608139038086,
      "activations/layer6_attention_weight_min": -47.348567962646484,
      "activations/layer7_attention_weight_max": 89.74200439453125,
      "activations/layer7_attention_weight_min": -94.82368469238281,
      "activations/layer8_attention_weight_max": 41.5954704284668,
      "activations/layer8_attention_weight_min": -42.12757110595703,
      "activations/layer9_attention_weight_max": 35.80141067504883,
      "activations/layer9_attention_weight_min": -35.74774169921875,
      "epoch": 17.7,
      "learning_rate": 3.624696969696969e-05,
      "loss": 2.7358,
      "step": 304550
    },
    {
      "activations/layer0_attention_weight_max": 15.529447555541992,
      "activations/layer0_attention_weight_min": -12.12924861907959,
      "activations/layer10_attention_weight_max": 32.809810638427734,
      "activations/layer10_attention_weight_min": -35.5047607421875,
      "activations/layer11_attention_weight_max": 31.43037986755371,
      "activations/layer11_attention_weight_min": -34.552345275878906,
      "activations/layer12_attention_weight_max": 26.200342178344727,
      "activations/layer12_attention_weight_min": -27.049354553222656,
      "activations/layer13_attention_weight_max": 39.71625518798828,
      "activations/layer13_attention_weight_min": -38.19487380981445,
      "activations/layer14_attention_weight_max": 39.295711517333984,
      "activations/layer14_attention_weight_min": -38.006492614746094,
      "activations/layer15_attention_weight_max": 38.50334548950195,
      "activations/layer15_attention_weight_min": -38.45396041870117,
      "activations/layer16_attention_weight_max": 29.43693733215332,
      "activations/layer16_attention_weight_min": -30.721757888793945,
      "activations/layer17_attention_weight_max": 29.31483268737793,
      "activations/layer17_attention_weight_min": -27.805458068847656,
      "activations/layer18_attention_weight_max": 32.100189208984375,
      "activations/layer18_attention_weight_min": -26.66217613220215,
      "activations/layer19_attention_weight_max": 34.9235725402832,
      "activations/layer19_attention_weight_min": -34.804405212402344,
      "activations/layer1_attention_weight_max": 15.766550064086914,
      "activations/layer1_attention_weight_min": -14.294239044189453,
      "activations/layer20_attention_weight_max": 35.560306549072266,
      "activations/layer20_attention_weight_min": -28.3037052154541,
      "activations/layer21_attention_weight_max": 34.34349060058594,
      "activations/layer21_attention_weight_min": -29.365392684936523,
      "activations/layer22_attention_weight_max": 42.27513885498047,
      "activations/layer22_attention_weight_min": -35.7344970703125,
      "activations/layer23_attention_weight_max": 36.079952239990234,
      "activations/layer23_attention_weight_min": -29.977218627929688,
      "activations/layer2_attention_weight_max": 32.351016998291016,
      "activations/layer2_attention_weight_min": -34.17808532714844,
      "activations/layer3_attention_weight_max": 98.59609985351562,
      "activations/layer3_attention_weight_min": -102.65572357177734,
      "activations/layer4_attention_weight_max": 106.90218353271484,
      "activations/layer4_attention_weight_min": -109.63158416748047,
      "activations/layer5_attention_weight_max": 49.09659957885742,
      "activations/layer5_attention_weight_min": -62.21797180175781,
      "activations/layer6_attention_weight_max": 46.15894317626953,
      "activations/layer6_attention_weight_min": -47.20462417602539,
      "activations/layer7_attention_weight_max": 95.0227279663086,
      "activations/layer7_attention_weight_min": -103.35846710205078,
      "activations/layer8_attention_weight_max": 43.22032165527344,
      "activations/layer8_attention_weight_min": -47.857669830322266,
      "activations/layer9_attention_weight_max": 32.616058349609375,
      "activations/layer9_attention_weight_min": -34.61756896972656,
      "epoch": 17.7,
      "learning_rate": 3.62280303030303e-05,
      "loss": 2.7339,
      "step": 304600
    },
    {
      "activations/layer0_attention_weight_max": 15.883008003234863,
      "activations/layer0_attention_weight_min": -12.563189506530762,
      "activations/layer10_attention_weight_max": 33.104270935058594,
      "activations/layer10_attention_weight_min": -36.035926818847656,
      "activations/layer11_attention_weight_max": 33.636993408203125,
      "activations/layer11_attention_weight_min": -35.08619689941406,
      "activations/layer12_attention_weight_max": 26.08334732055664,
      "activations/layer12_attention_weight_min": -27.559850692749023,
      "activations/layer13_attention_weight_max": 37.030738830566406,
      "activations/layer13_attention_weight_min": -35.35035705566406,
      "activations/layer14_attention_weight_max": 40.541996002197266,
      "activations/layer14_attention_weight_min": -38.56290054321289,
      "activations/layer15_attention_weight_max": 38.47909927368164,
      "activations/layer15_attention_weight_min": -36.692623138427734,
      "activations/layer16_attention_weight_max": 26.178611755371094,
      "activations/layer16_attention_weight_min": -29.46064567565918,
      "activations/layer17_attention_weight_max": 28.392925262451172,
      "activations/layer17_attention_weight_min": -27.417442321777344,
      "activations/layer18_attention_weight_max": 29.632366180419922,
      "activations/layer18_attention_weight_min": -26.42985725402832,
      "activations/layer19_attention_weight_max": 29.13541030883789,
      "activations/layer19_attention_weight_min": -31.994863510131836,
      "activations/layer1_attention_weight_max": 15.575693130493164,
      "activations/layer1_attention_weight_min": -16.063796997070312,
      "activations/layer20_attention_weight_max": 25.255708694458008,
      "activations/layer20_attention_weight_min": -26.83074951171875,
      "activations/layer21_attention_weight_max": 26.038766860961914,
      "activations/layer21_attention_weight_min": -25.76262855529785,
      "activations/layer22_attention_weight_max": 36.9435920715332,
      "activations/layer22_attention_weight_min": -32.745872497558594,
      "activations/layer23_attention_weight_max": 28.69931411743164,
      "activations/layer23_attention_weight_min": -27.884626388549805,
      "activations/layer2_attention_weight_max": 33.9136962890625,
      "activations/layer2_attention_weight_min": -33.993385314941406,
      "activations/layer3_attention_weight_max": 99.85859680175781,
      "activations/layer3_attention_weight_min": -102.68232727050781,
      "activations/layer4_attention_weight_max": 105.9187240600586,
      "activations/layer4_attention_weight_min": -111.1187973022461,
      "activations/layer5_attention_weight_max": 49.679107666015625,
      "activations/layer5_attention_weight_min": -59.55621337890625,
      "activations/layer6_attention_weight_max": 45.15586471557617,
      "activations/layer6_attention_weight_min": -47.64348602294922,
      "activations/layer7_attention_weight_max": 89.68666076660156,
      "activations/layer7_attention_weight_min": -94.66970825195312,
      "activations/layer8_attention_weight_max": 42.89045333862305,
      "activations/layer8_attention_weight_min": -44.285743713378906,
      "activations/layer9_attention_weight_max": 32.05442428588867,
      "activations/layer9_attention_weight_min": -35.189266204833984,
      "epoch": 17.7,
      "learning_rate": 3.62090909090909e-05,
      "loss": 2.7308,
      "step": 304650
    },
    {
      "activations/layer0_attention_weight_max": 16.38519859313965,
      "activations/layer0_attention_weight_min": -12.350783348083496,
      "activations/layer10_attention_weight_max": 30.52695655822754,
      "activations/layer10_attention_weight_min": -32.424774169921875,
      "activations/layer11_attention_weight_max": 30.715221405029297,
      "activations/layer11_attention_weight_min": -34.63691711425781,
      "activations/layer12_attention_weight_max": 22.853450775146484,
      "activations/layer12_attention_weight_min": -25.947717666625977,
      "activations/layer13_attention_weight_max": 34.034812927246094,
      "activations/layer13_attention_weight_min": -35.8677978515625,
      "activations/layer14_attention_weight_max": 36.5405387878418,
      "activations/layer14_attention_weight_min": -36.74877166748047,
      "activations/layer15_attention_weight_max": 34.78145217895508,
      "activations/layer15_attention_weight_min": -35.791961669921875,
      "activations/layer16_attention_weight_max": 26.160659790039062,
      "activations/layer16_attention_weight_min": -26.730022430419922,
      "activations/layer17_attention_weight_max": 27.128520965576172,
      "activations/layer17_attention_weight_min": -26.097713470458984,
      "activations/layer18_attention_weight_max": 28.490495681762695,
      "activations/layer18_attention_weight_min": -25.598115921020508,
      "activations/layer19_attention_weight_max": 29.50721549987793,
      "activations/layer19_attention_weight_min": -30.892332077026367,
      "activations/layer1_attention_weight_max": 16.435237884521484,
      "activations/layer1_attention_weight_min": -14.241249084472656,
      "activations/layer20_attention_weight_max": 25.697248458862305,
      "activations/layer20_attention_weight_min": -24.92894744873047,
      "activations/layer21_attention_weight_max": 24.6673526763916,
      "activations/layer21_attention_weight_min": -24.654531478881836,
      "activations/layer22_attention_weight_max": 34.78130340576172,
      "activations/layer22_attention_weight_min": -31.06061553955078,
      "activations/layer23_attention_weight_max": 28.23297119140625,
      "activations/layer23_attention_weight_min": -27.308856964111328,
      "activations/layer2_attention_weight_max": 33.77735900878906,
      "activations/layer2_attention_weight_min": -33.92185974121094,
      "activations/layer3_attention_weight_max": 101.46167755126953,
      "activations/layer3_attention_weight_min": -101.61480712890625,
      "activations/layer4_attention_weight_max": 105.00011444091797,
      "activations/layer4_attention_weight_min": -106.4017105102539,
      "activations/layer5_attention_weight_max": 47.733192443847656,
      "activations/layer5_attention_weight_min": -61.90208435058594,
      "activations/layer6_attention_weight_max": 42.432373046875,
      "activations/layer6_attention_weight_min": -45.025230407714844,
      "activations/layer7_attention_weight_max": 84.77983093261719,
      "activations/layer7_attention_weight_min": -94.84334564208984,
      "activations/layer8_attention_weight_max": 40.001434326171875,
      "activations/layer8_attention_weight_min": -41.34799575805664,
      "activations/layer9_attention_weight_max": 29.857465744018555,
      "activations/layer9_attention_weight_min": -33.12407684326172,
      "epoch": 17.7,
      "learning_rate": 3.619015151515151e-05,
      "loss": 2.7413,
      "step": 304700
    },
    {
      "activations/layer0_attention_weight_max": 16.350744247436523,
      "activations/layer0_attention_weight_min": -12.683167457580566,
      "activations/layer10_attention_weight_max": 31.755619049072266,
      "activations/layer10_attention_weight_min": -33.54698944091797,
      "activations/layer11_attention_weight_max": 30.72578239440918,
      "activations/layer11_attention_weight_min": -35.93146514892578,
      "activations/layer12_attention_weight_max": 25.897062301635742,
      "activations/layer12_attention_weight_min": -27.97296905517578,
      "activations/layer13_attention_weight_max": 37.81938552856445,
      "activations/layer13_attention_weight_min": -39.66925811767578,
      "activations/layer14_attention_weight_max": 39.389801025390625,
      "activations/layer14_attention_weight_min": -39.56108856201172,
      "activations/layer15_attention_weight_max": 36.54007339477539,
      "activations/layer15_attention_weight_min": -37.523170471191406,
      "activations/layer16_attention_weight_max": 27.229055404663086,
      "activations/layer16_attention_weight_min": -30.661510467529297,
      "activations/layer17_attention_weight_max": 28.135799407958984,
      "activations/layer17_attention_weight_min": -27.33083152770996,
      "activations/layer18_attention_weight_max": 30.92632484436035,
      "activations/layer18_attention_weight_min": -26.77040672302246,
      "activations/layer19_attention_weight_max": 32.78770446777344,
      "activations/layer19_attention_weight_min": -35.19392013549805,
      "activations/layer1_attention_weight_max": 15.64082145690918,
      "activations/layer1_attention_weight_min": -14.66857624053955,
      "activations/layer20_attention_weight_max": 27.53583335876465,
      "activations/layer20_attention_weight_min": -26.2119083404541,
      "activations/layer21_attention_weight_max": 29.629669189453125,
      "activations/layer21_attention_weight_min": -27.731155395507812,
      "activations/layer22_attention_weight_max": 40.932281494140625,
      "activations/layer22_attention_weight_min": -33.89047622680664,
      "activations/layer23_attention_weight_max": 32.51365661621094,
      "activations/layer23_attention_weight_min": -28.187419891357422,
      "activations/layer2_attention_weight_max": 33.77643966674805,
      "activations/layer2_attention_weight_min": -34.763790130615234,
      "activations/layer3_attention_weight_max": 99.83824920654297,
      "activations/layer3_attention_weight_min": -106.23323059082031,
      "activations/layer4_attention_weight_max": 106.7898178100586,
      "activations/layer4_attention_weight_min": -112.5677719116211,
      "activations/layer5_attention_weight_max": 49.873779296875,
      "activations/layer5_attention_weight_min": -65.58659362792969,
      "activations/layer6_attention_weight_max": 44.43320846557617,
      "activations/layer6_attention_weight_min": -47.3259162902832,
      "activations/layer7_attention_weight_max": 88.97115325927734,
      "activations/layer7_attention_weight_min": -99.33929443359375,
      "activations/layer8_attention_weight_max": 40.63029861450195,
      "activations/layer8_attention_weight_min": -45.23103332519531,
      "activations/layer9_attention_weight_max": 30.304224014282227,
      "activations/layer9_attention_weight_min": -33.43708801269531,
      "epoch": 17.71,
      "learning_rate": 3.617121212121212e-05,
      "loss": 2.7517,
      "step": 304750
    },
    {
      "activations/layer0_attention_weight_max": 16.716087341308594,
      "activations/layer0_attention_weight_min": -12.969718933105469,
      "activations/layer10_attention_weight_max": 39.26150131225586,
      "activations/layer10_attention_weight_min": -37.44770431518555,
      "activations/layer11_attention_weight_max": 37.94469451904297,
      "activations/layer11_attention_weight_min": -37.20263671875,
      "activations/layer12_attention_weight_max": 28.314390182495117,
      "activations/layer12_attention_weight_min": -28.386390686035156,
      "activations/layer13_attention_weight_max": 39.010982513427734,
      "activations/layer13_attention_weight_min": -39.50172805786133,
      "activations/layer14_attention_weight_max": 43.94621658325195,
      "activations/layer14_attention_weight_min": -41.43290710449219,
      "activations/layer15_attention_weight_max": 42.59384536743164,
      "activations/layer15_attention_weight_min": -38.33980941772461,
      "activations/layer16_attention_weight_max": 33.38118362426758,
      "activations/layer16_attention_weight_min": -29.163143157958984,
      "activations/layer17_attention_weight_max": 28.81007957458496,
      "activations/layer17_attention_weight_min": -26.68044662475586,
      "activations/layer18_attention_weight_max": 29.594148635864258,
      "activations/layer18_attention_weight_min": -26.70819854736328,
      "activations/layer19_attention_weight_max": 42.01963424682617,
      "activations/layer19_attention_weight_min": -33.32572937011719,
      "activations/layer1_attention_weight_max": 15.677101135253906,
      "activations/layer1_attention_weight_min": -15.505611419677734,
      "activations/layer20_attention_weight_max": 29.655296325683594,
      "activations/layer20_attention_weight_min": -24.475934982299805,
      "activations/layer21_attention_weight_max": 28.94738006591797,
      "activations/layer21_attention_weight_min": -24.201169967651367,
      "activations/layer22_attention_weight_max": 42.434593200683594,
      "activations/layer22_attention_weight_min": -32.273887634277344,
      "activations/layer23_attention_weight_max": 37.956451416015625,
      "activations/layer23_attention_weight_min": -27.69655990600586,
      "activations/layer2_attention_weight_max": 34.04369354248047,
      "activations/layer2_attention_weight_min": -34.2437744140625,
      "activations/layer3_attention_weight_max": 108.9406509399414,
      "activations/layer3_attention_weight_min": -103.36492919921875,
      "activations/layer4_attention_weight_max": 108.0039291381836,
      "activations/layer4_attention_weight_min": -113.65803527832031,
      "activations/layer5_attention_weight_max": 51.987709045410156,
      "activations/layer5_attention_weight_min": -64.42526245117188,
      "activations/layer6_attention_weight_max": 46.02803039550781,
      "activations/layer6_attention_weight_min": -47.49783706665039,
      "activations/layer7_attention_weight_max": 98.17640686035156,
      "activations/layer7_attention_weight_min": -96.05012512207031,
      "activations/layer8_attention_weight_max": 43.48736572265625,
      "activations/layer8_attention_weight_min": -44.888671875,
      "activations/layer9_attention_weight_max": 35.5897102355957,
      "activations/layer9_attention_weight_min": -37.46603775024414,
      "epoch": 17.71,
      "learning_rate": 3.615227272727273e-05,
      "loss": 2.7331,
      "step": 304800
    },
    {
      "activations/layer0_attention_weight_max": 16.000316619873047,
      "activations/layer0_attention_weight_min": -13.918191909790039,
      "activations/layer10_attention_weight_max": 32.79863357543945,
      "activations/layer10_attention_weight_min": -34.961585998535156,
      "activations/layer11_attention_weight_max": 32.15971374511719,
      "activations/layer11_attention_weight_min": -33.67430114746094,
      "activations/layer12_attention_weight_max": 24.20769691467285,
      "activations/layer12_attention_weight_min": -27.652097702026367,
      "activations/layer13_attention_weight_max": 34.58592224121094,
      "activations/layer13_attention_weight_min": -35.66416931152344,
      "activations/layer14_attention_weight_max": 37.81904983520508,
      "activations/layer14_attention_weight_min": -37.974544525146484,
      "activations/layer15_attention_weight_max": 33.59376907348633,
      "activations/layer15_attention_weight_min": -35.33348083496094,
      "activations/layer16_attention_weight_max": 27.04974937438965,
      "activations/layer16_attention_weight_min": -29.138309478759766,
      "activations/layer17_attention_weight_max": 26.076812744140625,
      "activations/layer17_attention_weight_min": -27.15757179260254,
      "activations/layer18_attention_weight_max": 31.3974666595459,
      "activations/layer18_attention_weight_min": -27.922386169433594,
      "activations/layer19_attention_weight_max": 34.12865447998047,
      "activations/layer19_attention_weight_min": -33.07770538330078,
      "activations/layer1_attention_weight_max": 16.06336784362793,
      "activations/layer1_attention_weight_min": -15.491889953613281,
      "activations/layer20_attention_weight_max": 30.517831802368164,
      "activations/layer20_attention_weight_min": -29.225141525268555,
      "activations/layer21_attention_weight_max": 27.407018661499023,
      "activations/layer21_attention_weight_min": -28.038808822631836,
      "activations/layer22_attention_weight_max": 41.3312873840332,
      "activations/layer22_attention_weight_min": -33.391868591308594,
      "activations/layer23_attention_weight_max": 30.10433006286621,
      "activations/layer23_attention_weight_min": -28.874332427978516,
      "activations/layer2_attention_weight_max": 32.95598220825195,
      "activations/layer2_attention_weight_min": -33.81861114501953,
      "activations/layer3_attention_weight_max": 103.71700286865234,
      "activations/layer3_attention_weight_min": -106.16999053955078,
      "activations/layer4_attention_weight_max": 104.87623596191406,
      "activations/layer4_attention_weight_min": -114.95333099365234,
      "activations/layer5_attention_weight_max": 50.818477630615234,
      "activations/layer5_attention_weight_min": -65.71159362792969,
      "activations/layer6_attention_weight_max": 45.24097442626953,
      "activations/layer6_attention_weight_min": -48.58245086669922,
      "activations/layer7_attention_weight_max": 85.94544982910156,
      "activations/layer7_attention_weight_min": -97.69434356689453,
      "activations/layer8_attention_weight_max": 41.5668830871582,
      "activations/layer8_attention_weight_min": -44.4774284362793,
      "activations/layer9_attention_weight_max": 29.888242721557617,
      "activations/layer9_attention_weight_min": -34.978492736816406,
      "epoch": 17.71,
      "learning_rate": 3.6133333333333335e-05,
      "loss": 2.7273,
      "step": 304850
    },
    {
      "activations/layer0_attention_weight_max": 16.213361740112305,
      "activations/layer0_attention_weight_min": -12.956009864807129,
      "activations/layer10_attention_weight_max": 35.840553283691406,
      "activations/layer10_attention_weight_min": -34.735897064208984,
      "activations/layer11_attention_weight_max": 35.42007827758789,
      "activations/layer11_attention_weight_min": -34.88654327392578,
      "activations/layer12_attention_weight_max": 25.696836471557617,
      "activations/layer12_attention_weight_min": -27.592483520507812,
      "activations/layer13_attention_weight_max": 37.39379119873047,
      "activations/layer13_attention_weight_min": -35.93129348754883,
      "activations/layer14_attention_weight_max": 38.713226318359375,
      "activations/layer14_attention_weight_min": -37.231876373291016,
      "activations/layer15_attention_weight_max": 37.25210189819336,
      "activations/layer15_attention_weight_min": -38.76095199584961,
      "activations/layer16_attention_weight_max": 29.271102905273438,
      "activations/layer16_attention_weight_min": -29.679340362548828,
      "activations/layer17_attention_weight_max": 28.76346206665039,
      "activations/layer17_attention_weight_min": -29.178951263427734,
      "activations/layer18_attention_weight_max": 30.556743621826172,
      "activations/layer18_attention_weight_min": -28.531286239624023,
      "activations/layer19_attention_weight_max": 36.03261184692383,
      "activations/layer19_attention_weight_min": -34.01300811767578,
      "activations/layer1_attention_weight_max": 17.1430606842041,
      "activations/layer1_attention_weight_min": -14.877118110656738,
      "activations/layer20_attention_weight_max": 28.434070587158203,
      "activations/layer20_attention_weight_min": -28.456451416015625,
      "activations/layer21_attention_weight_max": 28.23843765258789,
      "activations/layer21_attention_weight_min": -27.790733337402344,
      "activations/layer22_attention_weight_max": 39.288902282714844,
      "activations/layer22_attention_weight_min": -35.35740661621094,
      "activations/layer23_attention_weight_max": 34.912574768066406,
      "activations/layer23_attention_weight_min": -30.895217895507812,
      "activations/layer2_attention_weight_max": 34.58165740966797,
      "activations/layer2_attention_weight_min": -34.60321807861328,
      "activations/layer3_attention_weight_max": 100.54850769042969,
      "activations/layer3_attention_weight_min": -105.90763854980469,
      "activations/layer4_attention_weight_max": 107.76055908203125,
      "activations/layer4_attention_weight_min": -107.86212158203125,
      "activations/layer5_attention_weight_max": 50.10071563720703,
      "activations/layer5_attention_weight_min": -61.19135665893555,
      "activations/layer6_attention_weight_max": 43.95295715332031,
      "activations/layer6_attention_weight_min": -46.14494323730469,
      "activations/layer7_attention_weight_max": 88.37196350097656,
      "activations/layer7_attention_weight_min": -95.42285919189453,
      "activations/layer8_attention_weight_max": 40.8588752746582,
      "activations/layer8_attention_weight_min": -47.44233703613281,
      "activations/layer9_attention_weight_max": 31.94736671447754,
      "activations/layer9_attention_weight_min": -36.965579986572266,
      "epoch": 17.72,
      "learning_rate": 3.611439393939394e-05,
      "loss": 2.7343,
      "step": 304900
    },
    {
      "activations/layer0_attention_weight_max": 16.111724853515625,
      "activations/layer0_attention_weight_min": -14.553393363952637,
      "activations/layer10_attention_weight_max": 33.89226150512695,
      "activations/layer10_attention_weight_min": -34.448585510253906,
      "activations/layer11_attention_weight_max": 35.073020935058594,
      "activations/layer11_attention_weight_min": -33.99894714355469,
      "activations/layer12_attention_weight_max": 26.60589027404785,
      "activations/layer12_attention_weight_min": -27.31183624267578,
      "activations/layer13_attention_weight_max": 37.64800262451172,
      "activations/layer13_attention_weight_min": -36.56523513793945,
      "activations/layer14_attention_weight_max": 40.280235290527344,
      "activations/layer14_attention_weight_min": -38.55849075317383,
      "activations/layer15_attention_weight_max": 36.90123748779297,
      "activations/layer15_attention_weight_min": -35.53713607788086,
      "activations/layer16_attention_weight_max": 28.108722686767578,
      "activations/layer16_attention_weight_min": -29.572853088378906,
      "activations/layer17_attention_weight_max": 30.62320327758789,
      "activations/layer17_attention_weight_min": -29.141265869140625,
      "activations/layer18_attention_weight_max": 30.428203582763672,
      "activations/layer18_attention_weight_min": -27.10235595703125,
      "activations/layer19_attention_weight_max": 33.10721969604492,
      "activations/layer19_attention_weight_min": -33.08224868774414,
      "activations/layer1_attention_weight_max": 17.07076644897461,
      "activations/layer1_attention_weight_min": -15.829564094543457,
      "activations/layer20_attention_weight_max": 27.887195587158203,
      "activations/layer20_attention_weight_min": -27.60763168334961,
      "activations/layer21_attention_weight_max": 27.213268280029297,
      "activations/layer21_attention_weight_min": -27.065587997436523,
      "activations/layer22_attention_weight_max": 40.17051696777344,
      "activations/layer22_attention_weight_min": -33.39311599731445,
      "activations/layer23_attention_weight_max": 34.12208938598633,
      "activations/layer23_attention_weight_min": -27.190670013427734,
      "activations/layer2_attention_weight_max": 33.28589630126953,
      "activations/layer2_attention_weight_min": -34.63157653808594,
      "activations/layer3_attention_weight_max": 102.43217468261719,
      "activations/layer3_attention_weight_min": -104.72044372558594,
      "activations/layer4_attention_weight_max": 105.0505599975586,
      "activations/layer4_attention_weight_min": -111.9936752319336,
      "activations/layer5_attention_weight_max": 50.7026481628418,
      "activations/layer5_attention_weight_min": -62.34530258178711,
      "activations/layer6_attention_weight_max": 45.430458068847656,
      "activations/layer6_attention_weight_min": -47.87321472167969,
      "activations/layer7_attention_weight_max": 94.47639465332031,
      "activations/layer7_attention_weight_min": -95.4017333984375,
      "activations/layer8_attention_weight_max": 40.83369827270508,
      "activations/layer8_attention_weight_min": -44.89213180541992,
      "activations/layer9_attention_weight_max": 31.591989517211914,
      "activations/layer9_attention_weight_min": -34.275150299072266,
      "epoch": 17.72,
      "learning_rate": 3.6095454545454545e-05,
      "loss": 2.7419,
      "step": 304950
    },
    {
      "activations/layer0_attention_weight_max": 16.015146255493164,
      "activations/layer0_attention_weight_min": -13.791081428527832,
      "activations/layer10_attention_weight_max": 32.73954391479492,
      "activations/layer10_attention_weight_min": -33.212646484375,
      "activations/layer11_attention_weight_max": 32.82332229614258,
      "activations/layer11_attention_weight_min": -34.646095275878906,
      "activations/layer12_attention_weight_max": 25.370887756347656,
      "activations/layer12_attention_weight_min": -27.810670852661133,
      "activations/layer13_attention_weight_max": 36.867347717285156,
      "activations/layer13_attention_weight_min": -36.49640655517578,
      "activations/layer14_attention_weight_max": 37.9890022277832,
      "activations/layer14_attention_weight_min": -39.087623596191406,
      "activations/layer15_attention_weight_max": 33.80336380004883,
      "activations/layer15_attention_weight_min": -35.855403900146484,
      "activations/layer16_attention_weight_max": 30.061269760131836,
      "activations/layer16_attention_weight_min": -29.54387664794922,
      "activations/layer17_attention_weight_max": 28.183473587036133,
      "activations/layer17_attention_weight_min": -28.745849609375,
      "activations/layer18_attention_weight_max": 29.951683044433594,
      "activations/layer18_attention_weight_min": -28.142452239990234,
      "activations/layer19_attention_weight_max": 31.1117000579834,
      "activations/layer19_attention_weight_min": -32.350502014160156,
      "activations/layer1_attention_weight_max": 15.997647285461426,
      "activations/layer1_attention_weight_min": -14.66659927368164,
      "activations/layer20_attention_weight_max": 26.5859317779541,
      "activations/layer20_attention_weight_min": -28.99125862121582,
      "activations/layer21_attention_weight_max": 25.372583389282227,
      "activations/layer21_attention_weight_min": -26.73869514465332,
      "activations/layer22_attention_weight_max": 39.451778411865234,
      "activations/layer22_attention_weight_min": -34.52177429199219,
      "activations/layer23_attention_weight_max": 30.86781120300293,
      "activations/layer23_attention_weight_min": -29.95730209350586,
      "activations/layer2_attention_weight_max": 36.44270324707031,
      "activations/layer2_attention_weight_min": -33.955257415771484,
      "activations/layer3_attention_weight_max": 99.45979309082031,
      "activations/layer3_attention_weight_min": -100.52557373046875,
      "activations/layer4_attention_weight_max": 104.76207733154297,
      "activations/layer4_attention_weight_min": -107.98199462890625,
      "activations/layer5_attention_weight_max": 50.083614349365234,
      "activations/layer5_attention_weight_min": -61.03805160522461,
      "activations/layer6_attention_weight_max": 44.6152458190918,
      "activations/layer6_attention_weight_min": -46.775577545166016,
      "activations/layer7_attention_weight_max": 89.09109497070312,
      "activations/layer7_attention_weight_min": -95.38063049316406,
      "activations/layer8_attention_weight_max": 41.850616455078125,
      "activations/layer8_attention_weight_min": -42.22599792480469,
      "activations/layer9_attention_weight_max": 31.09992027282715,
      "activations/layer9_attention_weight_min": -33.519927978515625,
      "epoch": 17.72,
      "learning_rate": 3.6076515151515146e-05,
      "loss": 2.7436,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4792,
      "eval_samples_per_second": 506.415,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4792,
      "eval_openwebtext_samples_per_second": 506.415,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9498,
      "eval_wikitext_samples_per_second": 233.874,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_lambada_loss": 2.38671875,
      "eval_lambada_ppl": 10.877742727456024,
      "eval_lambada_runtime": 9.5778,
      "eval_lambada_samples_per_second": 508.365,
      "step": 305000
    },
    {
      "activations/layer0_attention_weight_max": 16.31424903869629,
      "activations/layer0_attention_weight_min": -13.351827621459961,
      "activations/layer10_attention_weight_max": 33.38850021362305,
      "activations/layer10_attention_weight_min": -35.922359466552734,
      "activations/layer11_attention_weight_max": 34.149681091308594,
      "activations/layer11_attention_weight_min": -35.02454376220703,
      "activations/layer12_attention_weight_max": 29.25106430053711,
      "activations/layer12_attention_weight_min": -28.696725845336914,
      "activations/layer13_attention_weight_max": 43.8239860534668,
      "activations/layer13_attention_weight_min": -38.525428771972656,
      "activations/layer14_attention_weight_max": 48.67892837524414,
      "activations/layer14_attention_weight_min": -42.000431060791016,
      "activations/layer15_attention_weight_max": 43.486427307128906,
      "activations/layer15_attention_weight_min": -39.78973388671875,
      "activations/layer16_attention_weight_max": 30.09015655517578,
      "activations/layer16_attention_weight_min": -30.602046966552734,
      "activations/layer17_attention_weight_max": 30.17375373840332,
      "activations/layer17_attention_weight_min": -30.93476104736328,
      "activations/layer18_attention_weight_max": 33.828590393066406,
      "activations/layer18_attention_weight_min": -32.158050537109375,
      "activations/layer19_attention_weight_max": 38.15989303588867,
      "activations/layer19_attention_weight_min": -34.24692916870117,
      "activations/layer1_attention_weight_max": 18.617433547973633,
      "activations/layer1_attention_weight_min": -14.72131061553955,
      "activations/layer20_attention_weight_max": 32.16144561767578,
      "activations/layer20_attention_weight_min": -28.240922927856445,
      "activations/layer21_attention_weight_max": 31.148900985717773,
      "activations/layer21_attention_weight_min": -30.979223251342773,
      "activations/layer22_attention_weight_max": 44.480369567871094,
      "activations/layer22_attention_weight_min": -42.35932922363281,
      "activations/layer23_attention_weight_max": 32.91828155517578,
      "activations/layer23_attention_weight_min": -33.63080978393555,
      "activations/layer2_attention_weight_max": 33.157196044921875,
      "activations/layer2_attention_weight_min": -34.138465881347656,
      "activations/layer3_attention_weight_max": 99.81105041503906,
      "activations/layer3_attention_weight_min": -104.15786743164062,
      "activations/layer4_attention_weight_max": 109.3637466430664,
      "activations/layer4_attention_weight_min": -111.59378814697266,
      "activations/layer5_attention_weight_max": 52.4962158203125,
      "activations/layer5_attention_weight_min": -62.157012939453125,
      "activations/layer6_attention_weight_max": 44.88191223144531,
      "activations/layer6_attention_weight_min": -49.56721496582031,
      "activations/layer7_attention_weight_max": 93.79375457763672,
      "activations/layer7_attention_weight_min": -103.73695373535156,
      "activations/layer8_attention_weight_max": 42.04658508300781,
      "activations/layer8_attention_weight_min": -47.32783889770508,
      "activations/layer9_attention_weight_max": 31.843931198120117,
      "activations/layer9_attention_weight_min": -35.694091796875,
      "epoch": 17.73,
      "learning_rate": 3.6057575757575755e-05,
      "loss": 2.7455,
      "step": 305050
    },
    {
      "activations/layer0_attention_weight_max": 16.303333282470703,
      "activations/layer0_attention_weight_min": -13.000682830810547,
      "activations/layer10_attention_weight_max": 33.123939514160156,
      "activations/layer10_attention_weight_min": -33.68782424926758,
      "activations/layer11_attention_weight_max": 31.806224822998047,
      "activations/layer11_attention_weight_min": -34.20354080200195,
      "activations/layer12_attention_weight_max": 24.910865783691406,
      "activations/layer12_attention_weight_min": -27.936235427856445,
      "activations/layer13_attention_weight_max": 35.2800407409668,
      "activations/layer13_attention_weight_min": -39.86125946044922,
      "activations/layer14_attention_weight_max": 36.744022369384766,
      "activations/layer14_attention_weight_min": -43.55733108520508,
      "activations/layer15_attention_weight_max": 33.53727340698242,
      "activations/layer15_attention_weight_min": -37.86369323730469,
      "activations/layer16_attention_weight_max": 24.867904663085938,
      "activations/layer16_attention_weight_min": -29.90380859375,
      "activations/layer17_attention_weight_max": 25.818321228027344,
      "activations/layer17_attention_weight_min": -26.330020904541016,
      "activations/layer18_attention_weight_max": 29.233688354492188,
      "activations/layer18_attention_weight_min": -28.144052505493164,
      "activations/layer19_attention_weight_max": 29.75374412536621,
      "activations/layer19_attention_weight_min": -32.16524887084961,
      "activations/layer1_attention_weight_max": 16.569345474243164,
      "activations/layer1_attention_weight_min": -15.698192596435547,
      "activations/layer20_attention_weight_max": 25.208694458007812,
      "activations/layer20_attention_weight_min": -25.444608688354492,
      "activations/layer21_attention_weight_max": 26.13458251953125,
      "activations/layer21_attention_weight_min": -26.551437377929688,
      "activations/layer22_attention_weight_max": 36.11257553100586,
      "activations/layer22_attention_weight_min": -33.57593536376953,
      "activations/layer23_attention_weight_max": 28.836090087890625,
      "activations/layer23_attention_weight_min": -26.6433162689209,
      "activations/layer2_attention_weight_max": 34.223209381103516,
      "activations/layer2_attention_weight_min": -33.534664154052734,
      "activations/layer3_attention_weight_max": 103.9577865600586,
      "activations/layer3_attention_weight_min": -103.66361999511719,
      "activations/layer4_attention_weight_max": 112.299072265625,
      "activations/layer4_attention_weight_min": -108.7958755493164,
      "activations/layer5_attention_weight_max": 51.25614547729492,
      "activations/layer5_attention_weight_min": -60.75933074951172,
      "activations/layer6_attention_weight_max": 44.077701568603516,
      "activations/layer6_attention_weight_min": -46.53572463989258,
      "activations/layer7_attention_weight_max": 90.14768981933594,
      "activations/layer7_attention_weight_min": -96.16748046875,
      "activations/layer8_attention_weight_max": 38.563934326171875,
      "activations/layer8_attention_weight_min": -42.566707611083984,
      "activations/layer9_attention_weight_max": 31.309146881103516,
      "activations/layer9_attention_weight_min": -33.68419647216797,
      "epoch": 17.73,
      "learning_rate": 3.603863636363636e-05,
      "loss": 2.7663,
      "step": 305100
    },
    {
      "activations/layer0_attention_weight_max": 17.140230178833008,
      "activations/layer0_attention_weight_min": -12.564714431762695,
      "activations/layer10_attention_weight_max": 32.76778030395508,
      "activations/layer10_attention_weight_min": -32.76083755493164,
      "activations/layer11_attention_weight_max": 32.83891296386719,
      "activations/layer11_attention_weight_min": -33.00619125366211,
      "activations/layer12_attention_weight_max": 27.83071517944336,
      "activations/layer12_attention_weight_min": -27.70892906188965,
      "activations/layer13_attention_weight_max": 40.61232376098633,
      "activations/layer13_attention_weight_min": -36.485626220703125,
      "activations/layer14_attention_weight_max": 41.86164093017578,
      "activations/layer14_attention_weight_min": -40.37156677246094,
      "activations/layer15_attention_weight_max": 40.714996337890625,
      "activations/layer15_attention_weight_min": -36.939208984375,
      "activations/layer16_attention_weight_max": 30.366621017456055,
      "activations/layer16_attention_weight_min": -29.336122512817383,
      "activations/layer17_attention_weight_max": 28.623565673828125,
      "activations/layer17_attention_weight_min": -28.175172805786133,
      "activations/layer18_attention_weight_max": 32.1962890625,
      "activations/layer18_attention_weight_min": -26.49762725830078,
      "activations/layer19_attention_weight_max": 34.863136291503906,
      "activations/layer19_attention_weight_min": -33.79081344604492,
      "activations/layer1_attention_weight_max": 16.422378540039062,
      "activations/layer1_attention_weight_min": -17.04737663269043,
      "activations/layer20_attention_weight_max": 30.3767032623291,
      "activations/layer20_attention_weight_min": -28.47524642944336,
      "activations/layer21_attention_weight_max": 30.229549407958984,
      "activations/layer21_attention_weight_min": -26.785245895385742,
      "activations/layer22_attention_weight_max": 41.82611083984375,
      "activations/layer22_attention_weight_min": -32.23664474487305,
      "activations/layer23_attention_weight_max": 34.41678237915039,
      "activations/layer23_attention_weight_min": -26.97443962097168,
      "activations/layer2_attention_weight_max": 33.7160758972168,
      "activations/layer2_attention_weight_min": -33.10862350463867,
      "activations/layer3_attention_weight_max": 99.09817504882812,
      "activations/layer3_attention_weight_min": -98.6987533569336,
      "activations/layer4_attention_weight_max": 103.13032531738281,
      "activations/layer4_attention_weight_min": -109.86077880859375,
      "activations/layer5_attention_weight_max": 48.737579345703125,
      "activations/layer5_attention_weight_min": -60.18557357788086,
      "activations/layer6_attention_weight_max": 44.356468200683594,
      "activations/layer6_attention_weight_min": -47.66526794433594,
      "activations/layer7_attention_weight_max": 88.97605895996094,
      "activations/layer7_attention_weight_min": -91.78038024902344,
      "activations/layer8_attention_weight_max": 41.06942367553711,
      "activations/layer8_attention_weight_min": -44.650970458984375,
      "activations/layer9_attention_weight_max": 30.510513305664062,
      "activations/layer9_attention_weight_min": -34.68842697143555,
      "epoch": 17.73,
      "learning_rate": 3.6019696969696965e-05,
      "loss": 2.7498,
      "step": 305150
    },
    {
      "activations/layer0_attention_weight_max": 16.22934913635254,
      "activations/layer0_attention_weight_min": -13.743370056152344,
      "activations/layer10_attention_weight_max": 31.025266647338867,
      "activations/layer10_attention_weight_min": -33.82676315307617,
      "activations/layer11_attention_weight_max": 30.74270248413086,
      "activations/layer11_attention_weight_min": -33.3492431640625,
      "activations/layer12_attention_weight_max": 25.467491149902344,
      "activations/layer12_attention_weight_min": -26.25113296508789,
      "activations/layer13_attention_weight_max": 35.21351623535156,
      "activations/layer13_attention_weight_min": -35.363807678222656,
      "activations/layer14_attention_weight_max": 38.5748176574707,
      "activations/layer14_attention_weight_min": -36.88755416870117,
      "activations/layer15_attention_weight_max": 35.76502990722656,
      "activations/layer15_attention_weight_min": -36.399654388427734,
      "activations/layer16_attention_weight_max": 27.615127563476562,
      "activations/layer16_attention_weight_min": -28.19569206237793,
      "activations/layer17_attention_weight_max": 28.597063064575195,
      "activations/layer17_attention_weight_min": -27.948598861694336,
      "activations/layer18_attention_weight_max": 30.216562271118164,
      "activations/layer18_attention_weight_min": -26.610218048095703,
      "activations/layer19_attention_weight_max": 29.23600196838379,
      "activations/layer19_attention_weight_min": -31.040821075439453,
      "activations/layer1_attention_weight_max": 15.44782543182373,
      "activations/layer1_attention_weight_min": -15.385627746582031,
      "activations/layer20_attention_weight_max": 25.53400230407715,
      "activations/layer20_attention_weight_min": -25.011890411376953,
      "activations/layer21_attention_weight_max": 26.332998275756836,
      "activations/layer21_attention_weight_min": -25.01714324951172,
      "activations/layer22_attention_weight_max": 37.243621826171875,
      "activations/layer22_attention_weight_min": -31.42211151123047,
      "activations/layer23_attention_weight_max": 27.530805587768555,
      "activations/layer23_attention_weight_min": -29.00908660888672,
      "activations/layer2_attention_weight_max": 33.23527526855469,
      "activations/layer2_attention_weight_min": -34.78020477294922,
      "activations/layer3_attention_weight_max": 97.102294921875,
      "activations/layer3_attention_weight_min": -102.10057830810547,
      "activations/layer4_attention_weight_max": 106.19609832763672,
      "activations/layer4_attention_weight_min": -106.83269500732422,
      "activations/layer5_attention_weight_max": 48.35071563720703,
      "activations/layer5_attention_weight_min": -62.649696350097656,
      "activations/layer6_attention_weight_max": 41.37590026855469,
      "activations/layer6_attention_weight_min": -45.82865524291992,
      "activations/layer7_attention_weight_max": 85.69927215576172,
      "activations/layer7_attention_weight_min": -92.26687622070312,
      "activations/layer8_attention_weight_max": 37.82487487792969,
      "activations/layer8_attention_weight_min": -41.82258224487305,
      "activations/layer9_attention_weight_max": 27.977863311767578,
      "activations/layer9_attention_weight_min": -33.60978317260742,
      "epoch": 17.73,
      "learning_rate": 3.600075757575757e-05,
      "loss": 2.7256,
      "step": 305200
    },
    {
      "activations/layer0_attention_weight_max": 15.771577835083008,
      "activations/layer0_attention_weight_min": -12.506377220153809,
      "activations/layer10_attention_weight_max": 33.46912384033203,
      "activations/layer10_attention_weight_min": -36.577117919921875,
      "activations/layer11_attention_weight_max": 33.61020278930664,
      "activations/layer11_attention_weight_min": -35.92375946044922,
      "activations/layer12_attention_weight_max": 26.642919540405273,
      "activations/layer12_attention_weight_min": -28.62881088256836,
      "activations/layer13_attention_weight_max": 40.78053665161133,
      "activations/layer13_attention_weight_min": -38.513946533203125,
      "activations/layer14_attention_weight_max": 43.1624755859375,
      "activations/layer14_attention_weight_min": -39.54071807861328,
      "activations/layer15_attention_weight_max": 39.612152099609375,
      "activations/layer15_attention_weight_min": -36.52855682373047,
      "activations/layer16_attention_weight_max": 31.474267959594727,
      "activations/layer16_attention_weight_min": -29.266313552856445,
      "activations/layer17_attention_weight_max": 30.24909210205078,
      "activations/layer17_attention_weight_min": -27.80181884765625,
      "activations/layer18_attention_weight_max": 32.8403434753418,
      "activations/layer18_attention_weight_min": -26.164661407470703,
      "activations/layer19_attention_weight_max": 33.48982620239258,
      "activations/layer19_attention_weight_min": -32.19463348388672,
      "activations/layer1_attention_weight_max": 16.156923294067383,
      "activations/layer1_attention_weight_min": -14.559670448303223,
      "activations/layer20_attention_weight_max": 29.097166061401367,
      "activations/layer20_attention_weight_min": -27.620830535888672,
      "activations/layer21_attention_weight_max": 28.462886810302734,
      "activations/layer21_attention_weight_min": -24.965322494506836,
      "activations/layer22_attention_weight_max": 45.79173278808594,
      "activations/layer22_attention_weight_min": -31.766897201538086,
      "activations/layer23_attention_weight_max": 32.7575798034668,
      "activations/layer23_attention_weight_min": -27.202587127685547,
      "activations/layer2_attention_weight_max": 34.690818786621094,
      "activations/layer2_attention_weight_min": -33.30077362060547,
      "activations/layer3_attention_weight_max": 102.04511260986328,
      "activations/layer3_attention_weight_min": -102.51566314697266,
      "activations/layer4_attention_weight_max": 104.2362289428711,
      "activations/layer4_attention_weight_min": -112.4399185180664,
      "activations/layer5_attention_weight_max": 52.87821960449219,
      "activations/layer5_attention_weight_min": -68.34490966796875,
      "activations/layer6_attention_weight_max": 44.91352081298828,
      "activations/layer6_attention_weight_min": -47.65339279174805,
      "activations/layer7_attention_weight_max": 92.12003326416016,
      "activations/layer7_attention_weight_min": -97.48558044433594,
      "activations/layer8_attention_weight_max": 41.7523078918457,
      "activations/layer8_attention_weight_min": -46.95463180541992,
      "activations/layer9_attention_weight_max": 31.894824981689453,
      "activations/layer9_attention_weight_min": -34.9762077331543,
      "epoch": 17.74,
      "learning_rate": 3.598181818181818e-05,
      "loss": 2.7198,
      "step": 305250
    },
    {
      "activations/layer0_attention_weight_max": 16.658849716186523,
      "activations/layer0_attention_weight_min": -14.081182479858398,
      "activations/layer10_attention_weight_max": 36.21959686279297,
      "activations/layer10_attention_weight_min": -38.3334846496582,
      "activations/layer11_attention_weight_max": 40.25516891479492,
      "activations/layer11_attention_weight_min": -37.77044677734375,
      "activations/layer12_attention_weight_max": 30.108640670776367,
      "activations/layer12_attention_weight_min": -27.15329933166504,
      "activations/layer13_attention_weight_max": 43.3000373840332,
      "activations/layer13_attention_weight_min": -37.04961013793945,
      "activations/layer14_attention_weight_max": 53.906986236572266,
      "activations/layer14_attention_weight_min": -41.59748077392578,
      "activations/layer15_attention_weight_max": 43.930206298828125,
      "activations/layer15_attention_weight_min": -38.99778747558594,
      "activations/layer16_attention_weight_max": 34.212257385253906,
      "activations/layer16_attention_weight_min": -30.45660400390625,
      "activations/layer17_attention_weight_max": 30.909433364868164,
      "activations/layer17_attention_weight_min": -26.983015060424805,
      "activations/layer18_attention_weight_max": 37.002838134765625,
      "activations/layer18_attention_weight_min": -27.924057006835938,
      "activations/layer19_attention_weight_max": 39.7315788269043,
      "activations/layer19_attention_weight_min": -32.17337417602539,
      "activations/layer1_attention_weight_max": 17.277877807617188,
      "activations/layer1_attention_weight_min": -17.432689666748047,
      "activations/layer20_attention_weight_max": 30.75829315185547,
      "activations/layer20_attention_weight_min": -26.17193603515625,
      "activations/layer21_attention_weight_max": 30.22896385192871,
      "activations/layer21_attention_weight_min": -25.322368621826172,
      "activations/layer22_attention_weight_max": 45.56975173950195,
      "activations/layer22_attention_weight_min": -32.80741500854492,
      "activations/layer23_attention_weight_max": 35.924964904785156,
      "activations/layer23_attention_weight_min": -25.75640869140625,
      "activations/layer2_attention_weight_max": 35.94816589355469,
      "activations/layer2_attention_weight_min": -34.1465950012207,
      "activations/layer3_attention_weight_max": 101.0949478149414,
      "activations/layer3_attention_weight_min": -105.76451110839844,
      "activations/layer4_attention_weight_max": 106.76912689208984,
      "activations/layer4_attention_weight_min": -109.76067352294922,
      "activations/layer5_attention_weight_max": 51.21046447753906,
      "activations/layer5_attention_weight_min": -60.28650665283203,
      "activations/layer6_attention_weight_max": 45.81715774536133,
      "activations/layer6_attention_weight_min": -46.818660736083984,
      "activations/layer7_attention_weight_max": 102.42525482177734,
      "activations/layer7_attention_weight_min": -96.90287017822266,
      "activations/layer8_attention_weight_max": 47.627437591552734,
      "activations/layer8_attention_weight_min": -47.35689163208008,
      "activations/layer9_attention_weight_max": 36.152095794677734,
      "activations/layer9_attention_weight_min": -37.252201080322266,
      "epoch": 17.74,
      "learning_rate": 3.596287878787878e-05,
      "loss": 2.7354,
      "step": 305300
    },
    {
      "activations/layer0_attention_weight_max": 15.974387168884277,
      "activations/layer0_attention_weight_min": -13.844621658325195,
      "activations/layer10_attention_weight_max": 33.54195785522461,
      "activations/layer10_attention_weight_min": -36.51068878173828,
      "activations/layer11_attention_weight_max": 32.528438568115234,
      "activations/layer11_attention_weight_min": -34.85400390625,
      "activations/layer12_attention_weight_max": 31.353296279907227,
      "activations/layer12_attention_weight_min": -27.151050567626953,
      "activations/layer13_attention_weight_max": 43.10953140258789,
      "activations/layer13_attention_weight_min": -38.38338851928711,
      "activations/layer14_attention_weight_max": 43.65993881225586,
      "activations/layer14_attention_weight_min": -43.87406921386719,
      "activations/layer15_attention_weight_max": 40.11537170410156,
      "activations/layer15_attention_weight_min": -41.625083923339844,
      "activations/layer16_attention_weight_max": 30.110946655273438,
      "activations/layer16_attention_weight_min": -34.15179443359375,
      "activations/layer17_attention_weight_max": 29.9112491607666,
      "activations/layer17_attention_weight_min": -29.98213768005371,
      "activations/layer18_attention_weight_max": 29.806880950927734,
      "activations/layer18_attention_weight_min": -28.095191955566406,
      "activations/layer19_attention_weight_max": 35.94941329956055,
      "activations/layer19_attention_weight_min": -34.5264892578125,
      "activations/layer1_attention_weight_max": 16.84371566772461,
      "activations/layer1_attention_weight_min": -15.689040184020996,
      "activations/layer20_attention_weight_max": 30.4646053314209,
      "activations/layer20_attention_weight_min": -27.77301597595215,
      "activations/layer21_attention_weight_max": 27.453542709350586,
      "activations/layer21_attention_weight_min": -27.290828704833984,
      "activations/layer22_attention_weight_max": 46.56510543823242,
      "activations/layer22_attention_weight_min": -33.706974029541016,
      "activations/layer23_attention_weight_max": 33.176124572753906,
      "activations/layer23_attention_weight_min": -29.2015380859375,
      "activations/layer2_attention_weight_max": 34.154701232910156,
      "activations/layer2_attention_weight_min": -33.37836456298828,
      "activations/layer3_attention_weight_max": 101.32099151611328,
      "activations/layer3_attention_weight_min": -102.78936004638672,
      "activations/layer4_attention_weight_max": 104.33514404296875,
      "activations/layer4_attention_weight_min": -107.24947357177734,
      "activations/layer5_attention_weight_max": 49.37861633300781,
      "activations/layer5_attention_weight_min": -62.2126579284668,
      "activations/layer6_attention_weight_max": 42.0830192565918,
      "activations/layer6_attention_weight_min": -46.554603576660156,
      "activations/layer7_attention_weight_max": 91.71247100830078,
      "activations/layer7_attention_weight_min": -96.23049926757812,
      "activations/layer8_attention_weight_max": 40.184696197509766,
      "activations/layer8_attention_weight_min": -43.368045806884766,
      "activations/layer9_attention_weight_max": 33.684356689453125,
      "activations/layer9_attention_weight_min": -34.212215423583984,
      "epoch": 17.74,
      "learning_rate": 3.594393939393939e-05,
      "loss": 2.7465,
      "step": 305350
    },
    {
      "activations/layer0_attention_weight_max": 16.321182250976562,
      "activations/layer0_attention_weight_min": -13.40745735168457,
      "activations/layer10_attention_weight_max": 32.10642623901367,
      "activations/layer10_attention_weight_min": -33.88032150268555,
      "activations/layer11_attention_weight_max": 32.973060607910156,
      "activations/layer11_attention_weight_min": -34.576210021972656,
      "activations/layer12_attention_weight_max": 27.302248001098633,
      "activations/layer12_attention_weight_min": -27.012916564941406,
      "activations/layer13_attention_weight_max": 37.85334777832031,
      "activations/layer13_attention_weight_min": -37.68000030517578,
      "activations/layer14_attention_weight_max": 42.04126739501953,
      "activations/layer14_attention_weight_min": -41.59771728515625,
      "activations/layer15_attention_weight_max": 36.947635650634766,
      "activations/layer15_attention_weight_min": -40.34156036376953,
      "activations/layer16_attention_weight_max": 27.195276260375977,
      "activations/layer16_attention_weight_min": -30.135358810424805,
      "activations/layer17_attention_weight_max": 28.31571388244629,
      "activations/layer17_attention_weight_min": -29.616825103759766,
      "activations/layer18_attention_weight_max": 29.366008758544922,
      "activations/layer18_attention_weight_min": -32.04479217529297,
      "activations/layer19_attention_weight_max": 30.068328857421875,
      "activations/layer19_attention_weight_min": -34.90641403198242,
      "activations/layer1_attention_weight_max": 15.707534790039062,
      "activations/layer1_attention_weight_min": -13.817291259765625,
      "activations/layer20_attention_weight_max": 24.742002487182617,
      "activations/layer20_attention_weight_min": -27.809396743774414,
      "activations/layer21_attention_weight_max": 25.012474060058594,
      "activations/layer21_attention_weight_min": -28.999298095703125,
      "activations/layer22_attention_weight_max": 45.10383605957031,
      "activations/layer22_attention_weight_min": -38.01527786254883,
      "activations/layer23_attention_weight_max": 30.17560577392578,
      "activations/layer23_attention_weight_min": -29.701086044311523,
      "activations/layer2_attention_weight_max": 34.97876739501953,
      "activations/layer2_attention_weight_min": -31.66065788269043,
      "activations/layer3_attention_weight_max": 100.36125183105469,
      "activations/layer3_attention_weight_min": -97.65095520019531,
      "activations/layer4_attention_weight_max": 103.57506561279297,
      "activations/layer4_attention_weight_min": -110.85357666015625,
      "activations/layer5_attention_weight_max": 50.78746032714844,
      "activations/layer5_attention_weight_min": -61.087215423583984,
      "activations/layer6_attention_weight_max": 42.78221893310547,
      "activations/layer6_attention_weight_min": -46.99557876586914,
      "activations/layer7_attention_weight_max": 90.19974517822266,
      "activations/layer7_attention_weight_min": -94.84986114501953,
      "activations/layer8_attention_weight_max": 40.44864273071289,
      "activations/layer8_attention_weight_min": -44.286380767822266,
      "activations/layer9_attention_weight_max": 31.6622371673584,
      "activations/layer9_attention_weight_min": -35.667720794677734,
      "epoch": 17.75,
      "learning_rate": 3.592499999999999e-05,
      "loss": 2.7334,
      "step": 305400
    },
    {
      "activations/layer0_attention_weight_max": 16.235424041748047,
      "activations/layer0_attention_weight_min": -12.622740745544434,
      "activations/layer10_attention_weight_max": 30.74472427368164,
      "activations/layer10_attention_weight_min": -31.88961410522461,
      "activations/layer11_attention_weight_max": 31.990835189819336,
      "activations/layer11_attention_weight_min": -32.017059326171875,
      "activations/layer12_attention_weight_max": 25.536787033081055,
      "activations/layer12_attention_weight_min": -25.17105484008789,
      "activations/layer13_attention_weight_max": 35.72313690185547,
      "activations/layer13_attention_weight_min": -34.10737991333008,
      "activations/layer14_attention_weight_max": 38.52573776245117,
      "activations/layer14_attention_weight_min": -35.02998352050781,
      "activations/layer15_attention_weight_max": 36.16969680786133,
      "activations/layer15_attention_weight_min": -34.527442932128906,
      "activations/layer16_attention_weight_max": 27.601770401000977,
      "activations/layer16_attention_weight_min": -25.983034133911133,
      "activations/layer17_attention_weight_max": 28.366411209106445,
      "activations/layer17_attention_weight_min": -24.87880516052246,
      "activations/layer18_attention_weight_max": 29.584613800048828,
      "activations/layer18_attention_weight_min": -23.936466217041016,
      "activations/layer19_attention_weight_max": 30.794876098632812,
      "activations/layer19_attention_weight_min": -29.094507217407227,
      "activations/layer1_attention_weight_max": 15.953679084777832,
      "activations/layer1_attention_weight_min": -14.559947967529297,
      "activations/layer20_attention_weight_max": 27.211326599121094,
      "activations/layer20_attention_weight_min": -24.45695686340332,
      "activations/layer21_attention_weight_max": 25.834915161132812,
      "activations/layer21_attention_weight_min": -23.908201217651367,
      "activations/layer22_attention_weight_max": 38.92270278930664,
      "activations/layer22_attention_weight_min": -29.87289810180664,
      "activations/layer23_attention_weight_max": 27.92225456237793,
      "activations/layer23_attention_weight_min": -24.716087341308594,
      "activations/layer2_attention_weight_max": 32.38971710205078,
      "activations/layer2_attention_weight_min": -33.35107421875,
      "activations/layer3_attention_weight_max": 98.58927917480469,
      "activations/layer3_attention_weight_min": -99.33587646484375,
      "activations/layer4_attention_weight_max": 103.59620666503906,
      "activations/layer4_attention_weight_min": -104.21752166748047,
      "activations/layer5_attention_weight_max": 48.45985412597656,
      "activations/layer5_attention_weight_min": -60.45298767089844,
      "activations/layer6_attention_weight_max": 42.53398132324219,
      "activations/layer6_attention_weight_min": -45.4172248840332,
      "activations/layer7_attention_weight_max": 86.82624053955078,
      "activations/layer7_attention_weight_min": -95.4274673461914,
      "activations/layer8_attention_weight_max": 39.40816879272461,
      "activations/layer8_attention_weight_min": -43.413238525390625,
      "activations/layer9_attention_weight_max": 29.74930763244629,
      "activations/layer9_attention_weight_min": -32.48857879638672,
      "epoch": 17.75,
      "learning_rate": 3.59060606060606e-05,
      "loss": 2.7365,
      "step": 305450
    },
    {
      "activations/layer0_attention_weight_max": 16.54314422607422,
      "activations/layer0_attention_weight_min": -12.850698471069336,
      "activations/layer10_attention_weight_max": 32.793853759765625,
      "activations/layer10_attention_weight_min": -34.24673843383789,
      "activations/layer11_attention_weight_max": 33.379981994628906,
      "activations/layer11_attention_weight_min": -33.53473663330078,
      "activations/layer12_attention_weight_max": 30.607112884521484,
      "activations/layer12_attention_weight_min": -26.97440528869629,
      "activations/layer13_attention_weight_max": 44.682437896728516,
      "activations/layer13_attention_weight_min": -35.501869201660156,
      "activations/layer14_attention_weight_max": 45.79547882080078,
      "activations/layer14_attention_weight_min": -39.32094955444336,
      "activations/layer15_attention_weight_max": 41.49459457397461,
      "activations/layer15_attention_weight_min": -37.662193298339844,
      "activations/layer16_attention_weight_max": 27.526002883911133,
      "activations/layer16_attention_weight_min": -27.80492401123047,
      "activations/layer17_attention_weight_max": 27.430255889892578,
      "activations/layer17_attention_weight_min": -25.597923278808594,
      "activations/layer18_attention_weight_max": 29.780433654785156,
      "activations/layer18_attention_weight_min": -27.049930572509766,
      "activations/layer19_attention_weight_max": 32.45112991333008,
      "activations/layer19_attention_weight_min": -31.385162353515625,
      "activations/layer1_attention_weight_max": 15.575187683105469,
      "activations/layer1_attention_weight_min": -15.917057037353516,
      "activations/layer20_attention_weight_max": 27.485057830810547,
      "activations/layer20_attention_weight_min": -27.18013572692871,
      "activations/layer21_attention_weight_max": 29.34963035583496,
      "activations/layer21_attention_weight_min": -25.26271629333496,
      "activations/layer22_attention_weight_max": 38.41606521606445,
      "activations/layer22_attention_weight_min": -32.417232513427734,
      "activations/layer23_attention_weight_max": 29.145076751708984,
      "activations/layer23_attention_weight_min": -25.619884490966797,
      "activations/layer2_attention_weight_max": 33.85490798950195,
      "activations/layer2_attention_weight_min": -32.84864807128906,
      "activations/layer3_attention_weight_max": 100.3805923461914,
      "activations/layer3_attention_weight_min": -99.87971496582031,
      "activations/layer4_attention_weight_max": 109.22948455810547,
      "activations/layer4_attention_weight_min": -108.71638488769531,
      "activations/layer5_attention_weight_max": 49.41332244873047,
      "activations/layer5_attention_weight_min": -64.18789672851562,
      "activations/layer6_attention_weight_max": 43.17180633544922,
      "activations/layer6_attention_weight_min": -45.593265533447266,
      "activations/layer7_attention_weight_max": 90.63768768310547,
      "activations/layer7_attention_weight_min": -96.10441589355469,
      "activations/layer8_attention_weight_max": 40.04865264892578,
      "activations/layer8_attention_weight_min": -45.21453857421875,
      "activations/layer9_attention_weight_max": 31.323150634765625,
      "activations/layer9_attention_weight_min": -34.961448669433594,
      "epoch": 17.75,
      "learning_rate": 3.588712121212121e-05,
      "loss": 2.7367,
      "step": 305500
    },
    {
      "activations/layer0_attention_weight_max": 16.12786865234375,
      "activations/layer0_attention_weight_min": -12.373124122619629,
      "activations/layer10_attention_weight_max": 34.22261428833008,
      "activations/layer10_attention_weight_min": -34.78387451171875,
      "activations/layer11_attention_weight_max": 33.483009338378906,
      "activations/layer11_attention_weight_min": -35.563079833984375,
      "activations/layer12_attention_weight_max": 25.979022979736328,
      "activations/layer12_attention_weight_min": -27.668893814086914,
      "activations/layer13_attention_weight_max": 37.339752197265625,
      "activations/layer13_attention_weight_min": -36.17791748046875,
      "activations/layer14_attention_weight_max": 40.694244384765625,
      "activations/layer14_attention_weight_min": -39.09349822998047,
      "activations/layer15_attention_weight_max": 36.054222106933594,
      "activations/layer15_attention_weight_min": -34.170257568359375,
      "activations/layer16_attention_weight_max": 27.045164108276367,
      "activations/layer16_attention_weight_min": -27.74953842163086,
      "activations/layer17_attention_weight_max": 28.067522048950195,
      "activations/layer17_attention_weight_min": -25.176498413085938,
      "activations/layer18_attention_weight_max": 29.456533432006836,
      "activations/layer18_attention_weight_min": -24.836034774780273,
      "activations/layer19_attention_weight_max": 30.28934669494629,
      "activations/layer19_attention_weight_min": -31.276708602905273,
      "activations/layer1_attention_weight_max": 16.271926879882812,
      "activations/layer1_attention_weight_min": -14.645641326904297,
      "activations/layer20_attention_weight_max": 26.597318649291992,
      "activations/layer20_attention_weight_min": -25.162857055664062,
      "activations/layer21_attention_weight_max": 28.288522720336914,
      "activations/layer21_attention_weight_min": -23.86346435546875,
      "activations/layer22_attention_weight_max": 40.45199203491211,
      "activations/layer22_attention_weight_min": -29.68562126159668,
      "activations/layer23_attention_weight_max": 29.705028533935547,
      "activations/layer23_attention_weight_min": -26.09194564819336,
      "activations/layer2_attention_weight_max": 33.84585952758789,
      "activations/layer2_attention_weight_min": -33.173622131347656,
      "activations/layer3_attention_weight_max": 98.48441314697266,
      "activations/layer3_attention_weight_min": -101.02540588378906,
      "activations/layer4_attention_weight_max": 103.16683197021484,
      "activations/layer4_attention_weight_min": -107.80858612060547,
      "activations/layer5_attention_weight_max": 50.02488327026367,
      "activations/layer5_attention_weight_min": -66.11568450927734,
      "activations/layer6_attention_weight_max": 41.59529495239258,
      "activations/layer6_attention_weight_min": -45.14584732055664,
      "activations/layer7_attention_weight_max": 86.32357788085938,
      "activations/layer7_attention_weight_min": -93.30868530273438,
      "activations/layer8_attention_weight_max": 40.50288391113281,
      "activations/layer8_attention_weight_min": -42.9885139465332,
      "activations/layer9_attention_weight_max": 32.08589172363281,
      "activations/layer9_attention_weight_min": -32.55481719970703,
      "epoch": 17.75,
      "learning_rate": 3.586818181818182e-05,
      "loss": 2.7237,
      "step": 305550
    },
    {
      "activations/layer0_attention_weight_max": 16.426677703857422,
      "activations/layer0_attention_weight_min": -12.232266426086426,
      "activations/layer10_attention_weight_max": 35.136966705322266,
      "activations/layer10_attention_weight_min": -33.497169494628906,
      "activations/layer11_attention_weight_max": 34.244903564453125,
      "activations/layer11_attention_weight_min": -34.53296661376953,
      "activations/layer12_attention_weight_max": 26.613317489624023,
      "activations/layer12_attention_weight_min": -25.60337257385254,
      "activations/layer13_attention_weight_max": 38.492698669433594,
      "activations/layer13_attention_weight_min": -34.42184066772461,
      "activations/layer14_attention_weight_max": 46.53267288208008,
      "activations/layer14_attention_weight_min": -37.818050384521484,
      "activations/layer15_attention_weight_max": 41.12734603881836,
      "activations/layer15_attention_weight_min": -36.146568298339844,
      "activations/layer16_attention_weight_max": 31.30017852783203,
      "activations/layer16_attention_weight_min": -28.46624183654785,
      "activations/layer17_attention_weight_max": 29.571130752563477,
      "activations/layer17_attention_weight_min": -28.65144157409668,
      "activations/layer18_attention_weight_max": 31.416479110717773,
      "activations/layer18_attention_weight_min": -27.311359405517578,
      "activations/layer19_attention_weight_max": 30.328113555908203,
      "activations/layer19_attention_weight_min": -31.235782623291016,
      "activations/layer1_attention_weight_max": 17.74945640563965,
      "activations/layer1_attention_weight_min": -15.07857608795166,
      "activations/layer20_attention_weight_max": 26.873811721801758,
      "activations/layer20_attention_weight_min": -27.355134963989258,
      "activations/layer21_attention_weight_max": 28.527400970458984,
      "activations/layer21_attention_weight_min": -25.68699836730957,
      "activations/layer22_attention_weight_max": 39.38138198852539,
      "activations/layer22_attention_weight_min": -32.52369689941406,
      "activations/layer23_attention_weight_max": 29.40082550048828,
      "activations/layer23_attention_weight_min": -27.01678466796875,
      "activations/layer2_attention_weight_max": 36.80225372314453,
      "activations/layer2_attention_weight_min": -33.980533599853516,
      "activations/layer3_attention_weight_max": 107.28254699707031,
      "activations/layer3_attention_weight_min": -106.69987487792969,
      "activations/layer4_attention_weight_max": 107.76734924316406,
      "activations/layer4_attention_weight_min": -108.74382781982422,
      "activations/layer5_attention_weight_max": 50.61846160888672,
      "activations/layer5_attention_weight_min": -64.27832794189453,
      "activations/layer6_attention_weight_max": 42.03268051147461,
      "activations/layer6_attention_weight_min": -46.69758987426758,
      "activations/layer7_attention_weight_max": 90.7582778930664,
      "activations/layer7_attention_weight_min": -91.26432037353516,
      "activations/layer8_attention_weight_max": 40.72372055053711,
      "activations/layer8_attention_weight_min": -42.7424201965332,
      "activations/layer9_attention_weight_max": 31.891223907470703,
      "activations/layer9_attention_weight_min": -33.5627555847168,
      "epoch": 17.76,
      "learning_rate": 3.584924242424242e-05,
      "loss": 2.7481,
      "step": 305600
    },
    {
      "activations/layer0_attention_weight_max": 16.446996688842773,
      "activations/layer0_attention_weight_min": -13.917665481567383,
      "activations/layer10_attention_weight_max": 34.87083053588867,
      "activations/layer10_attention_weight_min": -34.528961181640625,
      "activations/layer11_attention_weight_max": 33.68276596069336,
      "activations/layer11_attention_weight_min": -33.68464660644531,
      "activations/layer12_attention_weight_max": 25.40507698059082,
      "activations/layer12_attention_weight_min": -25.684614181518555,
      "activations/layer13_attention_weight_max": 36.14618682861328,
      "activations/layer13_attention_weight_min": -37.54111862182617,
      "activations/layer14_attention_weight_max": 40.505470275878906,
      "activations/layer14_attention_weight_min": -38.4973030090332,
      "activations/layer15_attention_weight_max": 36.79017639160156,
      "activations/layer15_attention_weight_min": -33.022125244140625,
      "activations/layer16_attention_weight_max": 28.157182693481445,
      "activations/layer16_attention_weight_min": -28.845216751098633,
      "activations/layer17_attention_weight_max": 29.901376724243164,
      "activations/layer17_attention_weight_min": -26.39654541015625,
      "activations/layer18_attention_weight_max": 32.415794372558594,
      "activations/layer18_attention_weight_min": -26.96428108215332,
      "activations/layer19_attention_weight_max": 31.100543975830078,
      "activations/layer19_attention_weight_min": -32.22224426269531,
      "activations/layer1_attention_weight_max": 16.138919830322266,
      "activations/layer1_attention_weight_min": -14.233845710754395,
      "activations/layer20_attention_weight_max": 30.049560546875,
      "activations/layer20_attention_weight_min": -24.489971160888672,
      "activations/layer21_attention_weight_max": 28.91575813293457,
      "activations/layer21_attention_weight_min": -25.187816619873047,
      "activations/layer22_attention_weight_max": 40.916038513183594,
      "activations/layer22_attention_weight_min": -32.124595642089844,
      "activations/layer23_attention_weight_max": 31.770885467529297,
      "activations/layer23_attention_weight_min": -26.980270385742188,
      "activations/layer2_attention_weight_max": 32.23189163208008,
      "activations/layer2_attention_weight_min": -33.10969543457031,
      "activations/layer3_attention_weight_max": 103.485107421875,
      "activations/layer3_attention_weight_min": -101.76006317138672,
      "activations/layer4_attention_weight_max": 106.23785400390625,
      "activations/layer4_attention_weight_min": -107.15135955810547,
      "activations/layer5_attention_weight_max": 50.89338302612305,
      "activations/layer5_attention_weight_min": -68.94163513183594,
      "activations/layer6_attention_weight_max": 45.73207092285156,
      "activations/layer6_attention_weight_min": -47.39982986450195,
      "activations/layer7_attention_weight_max": 101.37229919433594,
      "activations/layer7_attention_weight_min": -97.12239074707031,
      "activations/layer8_attention_weight_max": 42.63770294189453,
      "activations/layer8_attention_weight_min": -45.72925567626953,
      "activations/layer9_attention_weight_max": 32.90533447265625,
      "activations/layer9_attention_weight_min": -34.72748947143555,
      "epoch": 17.76,
      "learning_rate": 3.583030303030303e-05,
      "loss": 2.7286,
      "step": 305650
    },
    {
      "activations/layer0_attention_weight_max": 16.47021484375,
      "activations/layer0_attention_weight_min": -13.170696258544922,
      "activations/layer10_attention_weight_max": 33.131534576416016,
      "activations/layer10_attention_weight_min": -32.483238220214844,
      "activations/layer11_attention_weight_max": 33.15534973144531,
      "activations/layer11_attention_weight_min": -33.49076843261719,
      "activations/layer12_attention_weight_max": 27.465669631958008,
      "activations/layer12_attention_weight_min": -26.9797306060791,
      "activations/layer13_attention_weight_max": 40.61445617675781,
      "activations/layer13_attention_weight_min": -36.83663558959961,
      "activations/layer14_attention_weight_max": 40.759620666503906,
      "activations/layer14_attention_weight_min": -38.540130615234375,
      "activations/layer15_attention_weight_max": 37.815277099609375,
      "activations/layer15_attention_weight_min": -34.917633056640625,
      "activations/layer16_attention_weight_max": 29.77692985534668,
      "activations/layer16_attention_weight_min": -28.477563858032227,
      "activations/layer17_attention_weight_max": 30.116254806518555,
      "activations/layer17_attention_weight_min": -25.925861358642578,
      "activations/layer18_attention_weight_max": 29.930612564086914,
      "activations/layer18_attention_weight_min": -26.277088165283203,
      "activations/layer19_attention_weight_max": 33.99625778198242,
      "activations/layer19_attention_weight_min": -31.86830711364746,
      "activations/layer1_attention_weight_max": 16.023378372192383,
      "activations/layer1_attention_weight_min": -15.318634033203125,
      "activations/layer20_attention_weight_max": 26.49904441833496,
      "activations/layer20_attention_weight_min": -25.585676193237305,
      "activations/layer21_attention_weight_max": 28.276811599731445,
      "activations/layer21_attention_weight_min": -27.095653533935547,
      "activations/layer22_attention_weight_max": 37.632266998291016,
      "activations/layer22_attention_weight_min": -33.31894302368164,
      "activations/layer23_attention_weight_max": 30.280141830444336,
      "activations/layer23_attention_weight_min": -26.509130477905273,
      "activations/layer2_attention_weight_max": 34.776824951171875,
      "activations/layer2_attention_weight_min": -35.95703125,
      "activations/layer3_attention_weight_max": 106.139892578125,
      "activations/layer3_attention_weight_min": -104.72245788574219,
      "activations/layer4_attention_weight_max": 109.23058319091797,
      "activations/layer4_attention_weight_min": -108.21187591552734,
      "activations/layer5_attention_weight_max": 51.43391418457031,
      "activations/layer5_attention_weight_min": -67.15392303466797,
      "activations/layer6_attention_weight_max": 46.37642288208008,
      "activations/layer6_attention_weight_min": -47.755577087402344,
      "activations/layer7_attention_weight_max": 93.76314544677734,
      "activations/layer7_attention_weight_min": -94.79185485839844,
      "activations/layer8_attention_weight_max": 42.624794006347656,
      "activations/layer8_attention_weight_min": -44.20724105834961,
      "activations/layer9_attention_weight_max": 32.474369049072266,
      "activations/layer9_attention_weight_min": -33.0137939453125,
      "epoch": 17.76,
      "learning_rate": 3.5811363636363635e-05,
      "loss": 2.7385,
      "step": 305700
    },
    {
      "activations/layer0_attention_weight_max": 16.00541114807129,
      "activations/layer0_attention_weight_min": -13.110258102416992,
      "activations/layer10_attention_weight_max": 34.609764099121094,
      "activations/layer10_attention_weight_min": -34.570823669433594,
      "activations/layer11_attention_weight_max": 35.04509735107422,
      "activations/layer11_attention_weight_min": -33.81420135498047,
      "activations/layer12_attention_weight_max": 27.06294059753418,
      "activations/layer12_attention_weight_min": -29.090946197509766,
      "activations/layer13_attention_weight_max": 40.56623458862305,
      "activations/layer13_attention_weight_min": -37.76829528808594,
      "activations/layer14_attention_weight_max": 45.034183502197266,
      "activations/layer14_attention_weight_min": -40.953460693359375,
      "activations/layer15_attention_weight_max": 39.83734130859375,
      "activations/layer15_attention_weight_min": -36.72897720336914,
      "activations/layer16_attention_weight_max": 29.85561180114746,
      "activations/layer16_attention_weight_min": -32.8353385925293,
      "activations/layer17_attention_weight_max": 30.13129425048828,
      "activations/layer17_attention_weight_min": -27.249752044677734,
      "activations/layer18_attention_weight_max": 34.133460998535156,
      "activations/layer18_attention_weight_min": -28.366674423217773,
      "activations/layer19_attention_weight_max": 32.98904037475586,
      "activations/layer19_attention_weight_min": -33.85047912597656,
      "activations/layer1_attention_weight_max": 17.641992568969727,
      "activations/layer1_attention_weight_min": -14.62590503692627,
      "activations/layer20_attention_weight_max": 29.03932762145996,
      "activations/layer20_attention_weight_min": -28.187244415283203,
      "activations/layer21_attention_weight_max": 28.99596405029297,
      "activations/layer21_attention_weight_min": -26.27104377746582,
      "activations/layer22_attention_weight_max": 40.88501739501953,
      "activations/layer22_attention_weight_min": -33.30692672729492,
      "activations/layer23_attention_weight_max": 28.97280502319336,
      "activations/layer23_attention_weight_min": -26.632076263427734,
      "activations/layer2_attention_weight_max": 34.5959587097168,
      "activations/layer2_attention_weight_min": -32.325469970703125,
      "activations/layer3_attention_weight_max": 103.31108856201172,
      "activations/layer3_attention_weight_min": -101.1020278930664,
      "activations/layer4_attention_weight_max": 106.6819076538086,
      "activations/layer4_attention_weight_min": -106.24786376953125,
      "activations/layer5_attention_weight_max": 51.74846649169922,
      "activations/layer5_attention_weight_min": -63.06380844116211,
      "activations/layer6_attention_weight_max": 46.57536697387695,
      "activations/layer6_attention_weight_min": -46.976219177246094,
      "activations/layer7_attention_weight_max": 94.89088439941406,
      "activations/layer7_attention_weight_min": -99.20913696289062,
      "activations/layer8_attention_weight_max": 42.860809326171875,
      "activations/layer8_attention_weight_min": -43.123451232910156,
      "activations/layer9_attention_weight_max": 33.09326171875,
      "activations/layer9_attention_weight_min": -33.8227653503418,
      "epoch": 17.77,
      "learning_rate": 3.579242424242424e-05,
      "loss": 2.737,
      "step": 305750
    },
    {
      "activations/layer0_attention_weight_max": 16.094377517700195,
      "activations/layer0_attention_weight_min": -14.00429916381836,
      "activations/layer10_attention_weight_max": 34.442955017089844,
      "activations/layer10_attention_weight_min": -36.300228118896484,
      "activations/layer11_attention_weight_max": 34.2581787109375,
      "activations/layer11_attention_weight_min": -37.36688995361328,
      "activations/layer12_attention_weight_max": 26.112218856811523,
      "activations/layer12_attention_weight_min": -25.276304244995117,
      "activations/layer13_attention_weight_max": 37.097923278808594,
      "activations/layer13_attention_weight_min": -34.46263122558594,
      "activations/layer14_attention_weight_max": 42.47703552246094,
      "activations/layer14_attention_weight_min": -36.280696868896484,
      "activations/layer15_attention_weight_max": 37.931331634521484,
      "activations/layer15_attention_weight_min": -33.648521423339844,
      "activations/layer16_attention_weight_max": 28.196054458618164,
      "activations/layer16_attention_weight_min": -27.433332443237305,
      "activations/layer17_attention_weight_max": 28.23496437072754,
      "activations/layer17_attention_weight_min": -25.874267578125,
      "activations/layer18_attention_weight_max": 31.524169921875,
      "activations/layer18_attention_weight_min": -26.097749710083008,
      "activations/layer19_attention_weight_max": 30.258655548095703,
      "activations/layer19_attention_weight_min": -30.84824562072754,
      "activations/layer1_attention_weight_max": 16.02309226989746,
      "activations/layer1_attention_weight_min": -14.927698135375977,
      "activations/layer20_attention_weight_max": 27.390878677368164,
      "activations/layer20_attention_weight_min": -25.64177703857422,
      "activations/layer21_attention_weight_max": 27.97793960571289,
      "activations/layer21_attention_weight_min": -25.489200592041016,
      "activations/layer22_attention_weight_max": 44.67217254638672,
      "activations/layer22_attention_weight_min": -30.872175216674805,
      "activations/layer23_attention_weight_max": 29.187131881713867,
      "activations/layer23_attention_weight_min": -26.847110748291016,
      "activations/layer2_attention_weight_max": 33.190757751464844,
      "activations/layer2_attention_weight_min": -33.71720886230469,
      "activations/layer3_attention_weight_max": 103.75563049316406,
      "activations/layer3_attention_weight_min": -100.00737762451172,
      "activations/layer4_attention_weight_max": 105.9365005493164,
      "activations/layer4_attention_weight_min": -106.63301086425781,
      "activations/layer5_attention_weight_max": 49.63630676269531,
      "activations/layer5_attention_weight_min": -61.04322052001953,
      "activations/layer6_attention_weight_max": 43.6357536315918,
      "activations/layer6_attention_weight_min": -45.752681732177734,
      "activations/layer7_attention_weight_max": 88.06010437011719,
      "activations/layer7_attention_weight_min": -94.6279525756836,
      "activations/layer8_attention_weight_max": 41.697906494140625,
      "activations/layer8_attention_weight_min": -41.235572814941406,
      "activations/layer9_attention_weight_max": 31.539676666259766,
      "activations/layer9_attention_weight_min": -34.75503921508789,
      "epoch": 17.77,
      "learning_rate": 3.5773863636363634e-05,
      "loss": 2.7278,
      "step": 305800
    },
    {
      "activations/layer0_attention_weight_max": 16.844348907470703,
      "activations/layer0_attention_weight_min": -13.019645690917969,
      "activations/layer10_attention_weight_max": 31.932262420654297,
      "activations/layer10_attention_weight_min": -33.612911224365234,
      "activations/layer11_attention_weight_max": 31.79688262939453,
      "activations/layer11_attention_weight_min": -33.642906188964844,
      "activations/layer12_attention_weight_max": 26.190454483032227,
      "activations/layer12_attention_weight_min": -25.846464157104492,
      "activations/layer13_attention_weight_max": 36.98420715332031,
      "activations/layer13_attention_weight_min": -34.97833251953125,
      "activations/layer14_attention_weight_max": 40.089927673339844,
      "activations/layer14_attention_weight_min": -36.12435531616211,
      "activations/layer15_attention_weight_max": 34.94314193725586,
      "activations/layer15_attention_weight_min": -33.36067581176758,
      "activations/layer16_attention_weight_max": 26.82701301574707,
      "activations/layer16_attention_weight_min": -28.052570343017578,
      "activations/layer17_attention_weight_max": 31.681692123413086,
      "activations/layer17_attention_weight_min": -27.009586334228516,
      "activations/layer18_attention_weight_max": 29.819974899291992,
      "activations/layer18_attention_weight_min": -25.039644241333008,
      "activations/layer19_attention_weight_max": 31.555768966674805,
      "activations/layer19_attention_weight_min": -32.97964859008789,
      "activations/layer1_attention_weight_max": 16.0274600982666,
      "activations/layer1_attention_weight_min": -15.271974563598633,
      "activations/layer20_attention_weight_max": 25.942153930664062,
      "activations/layer20_attention_weight_min": -25.38728904724121,
      "activations/layer21_attention_weight_max": 24.696664810180664,
      "activations/layer21_attention_weight_min": -24.541690826416016,
      "activations/layer22_attention_weight_max": 35.10834503173828,
      "activations/layer22_attention_weight_min": -32.518245697021484,
      "activations/layer23_attention_weight_max": 30.63581085205078,
      "activations/layer23_attention_weight_min": -26.86708641052246,
      "activations/layer2_attention_weight_max": 33.17969512939453,
      "activations/layer2_attention_weight_min": -34.32996368408203,
      "activations/layer3_attention_weight_max": 99.17247772216797,
      "activations/layer3_attention_weight_min": -105.06078338623047,
      "activations/layer4_attention_weight_max": 103.38832092285156,
      "activations/layer4_attention_weight_min": -105.65445709228516,
      "activations/layer5_attention_weight_max": 50.44855499267578,
      "activations/layer5_attention_weight_min": -62.01517868041992,
      "activations/layer6_attention_weight_max": 41.9353141784668,
      "activations/layer6_attention_weight_min": -44.78287124633789,
      "activations/layer7_attention_weight_max": 93.59147644042969,
      "activations/layer7_attention_weight_min": -97.12884521484375,
      "activations/layer8_attention_weight_max": 40.32426452636719,
      "activations/layer8_attention_weight_min": -42.411865234375,
      "activations/layer9_attention_weight_max": 29.916717529296875,
      "activations/layer9_attention_weight_min": -33.403717041015625,
      "epoch": 17.77,
      "learning_rate": 3.5754924242424236e-05,
      "loss": 2.7353,
      "step": 305850
    },
    {
      "activations/layer0_attention_weight_max": 15.854594230651855,
      "activations/layer0_attention_weight_min": -14.069987297058105,
      "activations/layer10_attention_weight_max": 34.141849517822266,
      "activations/layer10_attention_weight_min": -34.97184371948242,
      "activations/layer11_attention_weight_max": 34.53643035888672,
      "activations/layer11_attention_weight_min": -37.07555389404297,
      "activations/layer12_attention_weight_max": 27.045541763305664,
      "activations/layer12_attention_weight_min": -27.546085357666016,
      "activations/layer13_attention_weight_max": 40.49330520629883,
      "activations/layer13_attention_weight_min": -37.40373611450195,
      "activations/layer14_attention_weight_max": 44.84834289550781,
      "activations/layer14_attention_weight_min": -46.54266357421875,
      "activations/layer15_attention_weight_max": 40.0858039855957,
      "activations/layer15_attention_weight_min": -37.62411117553711,
      "activations/layer16_attention_weight_max": 32.21438217163086,
      "activations/layer16_attention_weight_min": -30.364728927612305,
      "activations/layer17_attention_weight_max": 31.60117530822754,
      "activations/layer17_attention_weight_min": -28.62405776977539,
      "activations/layer18_attention_weight_max": 33.05632400512695,
      "activations/layer18_attention_weight_min": -26.356285095214844,
      "activations/layer19_attention_weight_max": 35.31633758544922,
      "activations/layer19_attention_weight_min": -31.655317306518555,
      "activations/layer1_attention_weight_max": 16.172637939453125,
      "activations/layer1_attention_weight_min": -14.722549438476562,
      "activations/layer20_attention_weight_max": 31.174715042114258,
      "activations/layer20_attention_weight_min": -24.908897399902344,
      "activations/layer21_attention_weight_max": 30.314950942993164,
      "activations/layer21_attention_weight_min": -25.89427375793457,
      "activations/layer22_attention_weight_max": 44.09522247314453,
      "activations/layer22_attention_weight_min": -30.68208885192871,
      "activations/layer23_attention_weight_max": 32.387901306152344,
      "activations/layer23_attention_weight_min": -25.794599533081055,
      "activations/layer2_attention_weight_max": 33.065181732177734,
      "activations/layer2_attention_weight_min": -35.093177795410156,
      "activations/layer3_attention_weight_max": 103.05406188964844,
      "activations/layer3_attention_weight_min": -106.47515869140625,
      "activations/layer4_attention_weight_max": 107.115478515625,
      "activations/layer4_attention_weight_min": -112.19506072998047,
      "activations/layer5_attention_weight_max": 49.02981185913086,
      "activations/layer5_attention_weight_min": -64.5714340209961,
      "activations/layer6_attention_weight_max": 45.39161682128906,
      "activations/layer6_attention_weight_min": -47.53097152709961,
      "activations/layer7_attention_weight_max": 92.16181182861328,
      "activations/layer7_attention_weight_min": -93.75230407714844,
      "activations/layer8_attention_weight_max": 41.351646423339844,
      "activations/layer8_attention_weight_min": -43.42307662963867,
      "activations/layer9_attention_weight_max": 32.027137756347656,
      "activations/layer9_attention_weight_min": -33.6391716003418,
      "epoch": 17.77,
      "learning_rate": 3.5735984848484844e-05,
      "loss": 2.7316,
      "step": 305900
    },
    {
      "activations/layer0_attention_weight_max": 16.879728317260742,
      "activations/layer0_attention_weight_min": -14.5577974319458,
      "activations/layer10_attention_weight_max": 34.68476867675781,
      "activations/layer10_attention_weight_min": -35.27994918823242,
      "activations/layer11_attention_weight_max": 35.57863998413086,
      "activations/layer11_attention_weight_min": -36.7054557800293,
      "activations/layer12_attention_weight_max": 37.74301528930664,
      "activations/layer12_attention_weight_min": -26.990158081054688,
      "activations/layer13_attention_weight_max": 48.782859802246094,
      "activations/layer13_attention_weight_min": -34.81199645996094,
      "activations/layer14_attention_weight_max": 47.55881881713867,
      "activations/layer14_attention_weight_min": -39.394309997558594,
      "activations/layer15_attention_weight_max": 44.598506927490234,
      "activations/layer15_attention_weight_min": -39.78230285644531,
      "activations/layer16_attention_weight_max": 31.05141258239746,
      "activations/layer16_attention_weight_min": -27.30720329284668,
      "activations/layer17_attention_weight_max": 28.42052459716797,
      "activations/layer17_attention_weight_min": -26.368860244750977,
      "activations/layer18_attention_weight_max": 29.754549026489258,
      "activations/layer18_attention_weight_min": -25.857383728027344,
      "activations/layer19_attention_weight_max": 31.175504684448242,
      "activations/layer19_attention_weight_min": -30.852094650268555,
      "activations/layer1_attention_weight_max": 16.429975509643555,
      "activations/layer1_attention_weight_min": -13.85814380645752,
      "activations/layer20_attention_weight_max": 27.371076583862305,
      "activations/layer20_attention_weight_min": -25.284656524658203,
      "activations/layer21_attention_weight_max": 26.890560150146484,
      "activations/layer21_attention_weight_min": -24.46471405029297,
      "activations/layer22_attention_weight_max": 39.2624397277832,
      "activations/layer22_attention_weight_min": -32.611663818359375,
      "activations/layer23_attention_weight_max": 30.46236801147461,
      "activations/layer23_attention_weight_min": -24.046201705932617,
      "activations/layer2_attention_weight_max": 35.29145050048828,
      "activations/layer2_attention_weight_min": -33.416595458984375,
      "activations/layer3_attention_weight_max": 105.22013854980469,
      "activations/layer3_attention_weight_min": -104.53751373291016,
      "activations/layer4_attention_weight_max": 110.30789947509766,
      "activations/layer4_attention_weight_min": -118.62799072265625,
      "activations/layer5_attention_weight_max": 52.5096549987793,
      "activations/layer5_attention_weight_min": -63.088436126708984,
      "activations/layer6_attention_weight_max": 45.28065490722656,
      "activations/layer6_attention_weight_min": -49.6260871887207,
      "activations/layer7_attention_weight_max": 97.56694030761719,
      "activations/layer7_attention_weight_min": -100.09553527832031,
      "activations/layer8_attention_weight_max": 46.778480529785156,
      "activations/layer8_attention_weight_min": -47.742801666259766,
      "activations/layer9_attention_weight_max": 33.12101364135742,
      "activations/layer9_attention_weight_min": -33.882057189941406,
      "epoch": 17.78,
      "learning_rate": 3.571704545454545e-05,
      "loss": 2.7569,
      "step": 305950
    },
    {
      "activations/layer0_attention_weight_max": 16.678585052490234,
      "activations/layer0_attention_weight_min": -12.895638465881348,
      "activations/layer10_attention_weight_max": 35.65639877319336,
      "activations/layer10_attention_weight_min": -34.253787994384766,
      "activations/layer11_attention_weight_max": 34.63096618652344,
      "activations/layer11_attention_weight_min": -33.35334396362305,
      "activations/layer12_attention_weight_max": 27.372465133666992,
      "activations/layer12_attention_weight_min": -26.893388748168945,
      "activations/layer13_attention_weight_max": 40.36915588378906,
      "activations/layer13_attention_weight_min": -35.78800964355469,
      "activations/layer14_attention_weight_max": 42.29928207397461,
      "activations/layer14_attention_weight_min": -36.52511978149414,
      "activations/layer15_attention_weight_max": 41.432151794433594,
      "activations/layer15_attention_weight_min": -35.91366195678711,
      "activations/layer16_attention_weight_max": 29.35490608215332,
      "activations/layer16_attention_weight_min": -27.417932510375977,
      "activations/layer17_attention_weight_max": 31.537899017333984,
      "activations/layer17_attention_weight_min": -25.800783157348633,
      "activations/layer18_attention_weight_max": 32.17237091064453,
      "activations/layer18_attention_weight_min": -26.476369857788086,
      "activations/layer19_attention_weight_max": 30.806522369384766,
      "activations/layer19_attention_weight_min": -29.17315673828125,
      "activations/layer1_attention_weight_max": 16.99355125427246,
      "activations/layer1_attention_weight_min": -13.856913566589355,
      "activations/layer20_attention_weight_max": 28.56338119506836,
      "activations/layer20_attention_weight_min": -24.620555877685547,
      "activations/layer21_attention_weight_max": 28.39202117919922,
      "activations/layer21_attention_weight_min": -23.765520095825195,
      "activations/layer22_attention_weight_max": 39.08011245727539,
      "activations/layer22_attention_weight_min": -29.64792823791504,
      "activations/layer23_attention_weight_max": 30.701278686523438,
      "activations/layer23_attention_weight_min": -25.67990493774414,
      "activations/layer2_attention_weight_max": 32.61231994628906,
      "activations/layer2_attention_weight_min": -32.26676559448242,
      "activations/layer3_attention_weight_max": 97.88677978515625,
      "activations/layer3_attention_weight_min": -98.97823333740234,
      "activations/layer4_attention_weight_max": 104.71331787109375,
      "activations/layer4_attention_weight_min": -106.24613189697266,
      "activations/layer5_attention_weight_max": 49.54132843017578,
      "activations/layer5_attention_weight_min": -59.93929672241211,
      "activations/layer6_attention_weight_max": 42.850791931152344,
      "activations/layer6_attention_weight_min": -44.85234451293945,
      "activations/layer7_attention_weight_max": 89.8725814819336,
      "activations/layer7_attention_weight_min": -93.19194793701172,
      "activations/layer8_attention_weight_max": 40.5075569152832,
      "activations/layer8_attention_weight_min": -41.63643264770508,
      "activations/layer9_attention_weight_max": 30.765792846679688,
      "activations/layer9_attention_weight_min": -34.16684341430664,
      "epoch": 17.78,
      "learning_rate": 3.569810606060606e-05,
      "loss": 2.7368,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4488,
      "eval_samples_per_second": 508.24,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4488,
      "eval_openwebtext_samples_per_second": 508.24,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9398,
      "eval_wikitext_samples_per_second": 235.078,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_lambada_loss": 2.421875,
      "eval_lambada_ppl": 11.26696508157019,
      "eval_lambada_runtime": 9.5282,
      "eval_lambada_samples_per_second": 511.008,
      "step": 306000
    },
    {
      "activations/layer0_attention_weight_max": 15.73015022277832,
      "activations/layer0_attention_weight_min": -14.530755043029785,
      "activations/layer10_attention_weight_max": 36.60435104370117,
      "activations/layer10_attention_weight_min": -34.46419143676758,
      "activations/layer11_attention_weight_max": 35.51543426513672,
      "activations/layer11_attention_weight_min": -34.082252502441406,
      "activations/layer12_attention_weight_max": 29.707277297973633,
      "activations/layer12_attention_weight_min": -28.38286781311035,
      "activations/layer13_attention_weight_max": 43.09624099731445,
      "activations/layer13_attention_weight_min": -38.105743408203125,
      "activations/layer14_attention_weight_max": 43.88347244262695,
      "activations/layer14_attention_weight_min": -39.31969451904297,
      "activations/layer15_attention_weight_max": 40.29755401611328,
      "activations/layer15_attention_weight_min": -34.74840545654297,
      "activations/layer16_attention_weight_max": 33.232357025146484,
      "activations/layer16_attention_weight_min": -28.548917770385742,
      "activations/layer17_attention_weight_max": 31.647695541381836,
      "activations/layer17_attention_weight_min": -26.973247528076172,
      "activations/layer18_attention_weight_max": 33.97735595703125,
      "activations/layer18_attention_weight_min": -26.28681755065918,
      "activations/layer19_attention_weight_max": 36.18248748779297,
      "activations/layer19_attention_weight_min": -30.80381202697754,
      "activations/layer1_attention_weight_max": 15.630171775817871,
      "activations/layer1_attention_weight_min": -14.748815536499023,
      "activations/layer20_attention_weight_max": 31.525487899780273,
      "activations/layer20_attention_weight_min": -25.841983795166016,
      "activations/layer21_attention_weight_max": 29.79144287109375,
      "activations/layer21_attention_weight_min": -24.0604305267334,
      "activations/layer22_attention_weight_max": 53.167327880859375,
      "activations/layer22_attention_weight_min": -32.46631622314453,
      "activations/layer23_attention_weight_max": 33.350486755371094,
      "activations/layer23_attention_weight_min": -27.5665283203125,
      "activations/layer2_attention_weight_max": 33.73316955566406,
      "activations/layer2_attention_weight_min": -35.075496673583984,
      "activations/layer3_attention_weight_max": 106.6209716796875,
      "activations/layer3_attention_weight_min": -106.31407928466797,
      "activations/layer4_attention_weight_max": 104.37108612060547,
      "activations/layer4_attention_weight_min": -110.7399673461914,
      "activations/layer5_attention_weight_max": 49.12882614135742,
      "activations/layer5_attention_weight_min": -62.10687255859375,
      "activations/layer6_attention_weight_max": 43.72956466674805,
      "activations/layer6_attention_weight_min": -47.39468765258789,
      "activations/layer7_attention_weight_max": 90.33714294433594,
      "activations/layer7_attention_weight_min": -97.12649536132812,
      "activations/layer8_attention_weight_max": 43.410823822021484,
      "activations/layer8_attention_weight_min": -46.63978958129883,
      "activations/layer9_attention_weight_max": 33.93462371826172,
      "activations/layer9_attention_weight_min": -36.34873580932617,
      "epoch": 17.78,
      "learning_rate": 3.567954545454545e-05,
      "loss": 2.7429,
      "step": 306050
    },
    {
      "activations/layer0_attention_weight_max": 16.754589080810547,
      "activations/layer0_attention_weight_min": -13.204986572265625,
      "activations/layer10_attention_weight_max": 35.724952697753906,
      "activations/layer10_attention_weight_min": -34.660160064697266,
      "activations/layer11_attention_weight_max": 36.2928581237793,
      "activations/layer11_attention_weight_min": -34.48300552368164,
      "activations/layer12_attention_weight_max": 27.705469131469727,
      "activations/layer12_attention_weight_min": -26.986066818237305,
      "activations/layer13_attention_weight_max": 40.222679138183594,
      "activations/layer13_attention_weight_min": -38.86613082885742,
      "activations/layer14_attention_weight_max": 44.06322479248047,
      "activations/layer14_attention_weight_min": -39.301719665527344,
      "activations/layer15_attention_weight_max": 39.85174560546875,
      "activations/layer15_attention_weight_min": -35.28757858276367,
      "activations/layer16_attention_weight_max": 30.34581756591797,
      "activations/layer16_attention_weight_min": -28.622209548950195,
      "activations/layer17_attention_weight_max": 28.73231315612793,
      "activations/layer17_attention_weight_min": -25.455944061279297,
      "activations/layer18_attention_weight_max": 29.12940788269043,
      "activations/layer18_attention_weight_min": -23.644567489624023,
      "activations/layer19_attention_weight_max": 33.83746337890625,
      "activations/layer19_attention_weight_min": -29.580814361572266,
      "activations/layer1_attention_weight_max": 16.231351852416992,
      "activations/layer1_attention_weight_min": -14.856724739074707,
      "activations/layer20_attention_weight_max": 29.3221378326416,
      "activations/layer20_attention_weight_min": -24.881763458251953,
      "activations/layer21_attention_weight_max": 30.441661834716797,
      "activations/layer21_attention_weight_min": -23.684640884399414,
      "activations/layer22_attention_weight_max": 40.49983596801758,
      "activations/layer22_attention_weight_min": -29.906160354614258,
      "activations/layer23_attention_weight_max": 31.872114181518555,
      "activations/layer23_attention_weight_min": -24.162063598632812,
      "activations/layer2_attention_weight_max": 33.11360168457031,
      "activations/layer2_attention_weight_min": -35.331817626953125,
      "activations/layer3_attention_weight_max": 100.19126892089844,
      "activations/layer3_attention_weight_min": -106.78064727783203,
      "activations/layer4_attention_weight_max": 104.77471160888672,
      "activations/layer4_attention_weight_min": -110.88667297363281,
      "activations/layer5_attention_weight_max": 50.272911071777344,
      "activations/layer5_attention_weight_min": -64.46092224121094,
      "activations/layer6_attention_weight_max": 43.422176361083984,
      "activations/layer6_attention_weight_min": -46.797019958496094,
      "activations/layer7_attention_weight_max": 90.23104858398438,
      "activations/layer7_attention_weight_min": -98.25950622558594,
      "activations/layer8_attention_weight_max": 42.209129333496094,
      "activations/layer8_attention_weight_min": -44.68334197998047,
      "activations/layer9_attention_weight_max": 32.984291076660156,
      "activations/layer9_attention_weight_min": -35.80242156982422,
      "epoch": 17.79,
      "learning_rate": 3.566060606060606e-05,
      "loss": 2.7328,
      "step": 306100
    },
    {
      "activations/layer0_attention_weight_max": 16.430343627929688,
      "activations/layer0_attention_weight_min": -14.543476104736328,
      "activations/layer10_attention_weight_max": 37.102821350097656,
      "activations/layer10_attention_weight_min": -34.703880310058594,
      "activations/layer11_attention_weight_max": 38.191749572753906,
      "activations/layer11_attention_weight_min": -34.26797866821289,
      "activations/layer12_attention_weight_max": 26.889432907104492,
      "activations/layer12_attention_weight_min": -27.670642852783203,
      "activations/layer13_attention_weight_max": 39.82823181152344,
      "activations/layer13_attention_weight_min": -36.4935417175293,
      "activations/layer14_attention_weight_max": 41.06179428100586,
      "activations/layer14_attention_weight_min": -41.2586784362793,
      "activations/layer15_attention_weight_max": 38.86538314819336,
      "activations/layer15_attention_weight_min": -37.30290985107422,
      "activations/layer16_attention_weight_max": 30.861730575561523,
      "activations/layer16_attention_weight_min": -27.792009353637695,
      "activations/layer17_attention_weight_max": 33.50048065185547,
      "activations/layer17_attention_weight_min": -25.18177032470703,
      "activations/layer18_attention_weight_max": 32.741329193115234,
      "activations/layer18_attention_weight_min": -26.214454650878906,
      "activations/layer19_attention_weight_max": 38.438846588134766,
      "activations/layer19_attention_weight_min": -30.79863929748535,
      "activations/layer1_attention_weight_max": 15.739896774291992,
      "activations/layer1_attention_weight_min": -15.109147071838379,
      "activations/layer20_attention_weight_max": 29.888954162597656,
      "activations/layer20_attention_weight_min": -25.25978660583496,
      "activations/layer21_attention_weight_max": 25.59752082824707,
      "activations/layer21_attention_weight_min": -24.283620834350586,
      "activations/layer22_attention_weight_max": 39.24544143676758,
      "activations/layer22_attention_weight_min": -30.670042037963867,
      "activations/layer23_attention_weight_max": 29.450359344482422,
      "activations/layer23_attention_weight_min": -25.770301818847656,
      "activations/layer2_attention_weight_max": 35.001441955566406,
      "activations/layer2_attention_weight_min": -34.9324951171875,
      "activations/layer3_attention_weight_max": 102.346435546875,
      "activations/layer3_attention_weight_min": -101.7811279296875,
      "activations/layer4_attention_weight_max": 109.15956115722656,
      "activations/layer4_attention_weight_min": -108.84490966796875,
      "activations/layer5_attention_weight_max": 52.480369567871094,
      "activations/layer5_attention_weight_min": -60.46192932128906,
      "activations/layer6_attention_weight_max": 47.58767318725586,
      "activations/layer6_attention_weight_min": -46.332237243652344,
      "activations/layer7_attention_weight_max": 94.6008529663086,
      "activations/layer7_attention_weight_min": -94.67825317382812,
      "activations/layer8_attention_weight_max": 43.761295318603516,
      "activations/layer8_attention_weight_min": -45.674747467041016,
      "activations/layer9_attention_weight_max": 34.3211784362793,
      "activations/layer9_attention_weight_min": -35.81989669799805,
      "epoch": 17.79,
      "learning_rate": 3.564166666666666e-05,
      "loss": 2.7435,
      "step": 306150
    },
    {
      "activations/layer0_attention_weight_max": 16.3066463470459,
      "activations/layer0_attention_weight_min": -14.822931289672852,
      "activations/layer10_attention_weight_max": 32.04408645629883,
      "activations/layer10_attention_weight_min": -35.08073806762695,
      "activations/layer11_attention_weight_max": 34.79792022705078,
      "activations/layer11_attention_weight_min": -34.44300079345703,
      "activations/layer12_attention_weight_max": 25.891454696655273,
      "activations/layer12_attention_weight_min": -28.669015884399414,
      "activations/layer13_attention_weight_max": 36.97595977783203,
      "activations/layer13_attention_weight_min": -35.67695236206055,
      "activations/layer14_attention_weight_max": 41.077789306640625,
      "activations/layer14_attention_weight_min": -37.169158935546875,
      "activations/layer15_attention_weight_max": 35.841609954833984,
      "activations/layer15_attention_weight_min": -35.32971954345703,
      "activations/layer16_attention_weight_max": 29.29248046875,
      "activations/layer16_attention_weight_min": -28.904159545898438,
      "activations/layer17_attention_weight_max": 29.206071853637695,
      "activations/layer17_attention_weight_min": -26.46787452697754,
      "activations/layer18_attention_weight_max": 30.541488647460938,
      "activations/layer18_attention_weight_min": -26.99026107788086,
      "activations/layer19_attention_weight_max": 31.917158126831055,
      "activations/layer19_attention_weight_min": -32.9356803894043,
      "activations/layer1_attention_weight_max": 16.294858932495117,
      "activations/layer1_attention_weight_min": -14.049280166625977,
      "activations/layer20_attention_weight_max": 27.835742950439453,
      "activations/layer20_attention_weight_min": -24.9805908203125,
      "activations/layer21_attention_weight_max": 28.417190551757812,
      "activations/layer21_attention_weight_min": -22.89134407043457,
      "activations/layer22_attention_weight_max": 40.18932342529297,
      "activations/layer22_attention_weight_min": -30.42596435546875,
      "activations/layer23_attention_weight_max": 31.878276824951172,
      "activations/layer23_attention_weight_min": -24.659839630126953,
      "activations/layer2_attention_weight_max": 33.92216491699219,
      "activations/layer2_attention_weight_min": -34.15201950073242,
      "activations/layer3_attention_weight_max": 99.4815902709961,
      "activations/layer3_attention_weight_min": -101.65341186523438,
      "activations/layer4_attention_weight_max": 104.44917297363281,
      "activations/layer4_attention_weight_min": -114.44493865966797,
      "activations/layer5_attention_weight_max": 50.128578186035156,
      "activations/layer5_attention_weight_min": -60.56730270385742,
      "activations/layer6_attention_weight_max": 43.43582534790039,
      "activations/layer6_attention_weight_min": -45.92617416381836,
      "activations/layer7_attention_weight_max": 92.20308685302734,
      "activations/layer7_attention_weight_min": -91.1854019165039,
      "activations/layer8_attention_weight_max": 39.65853500366211,
      "activations/layer8_attention_weight_min": -42.97613525390625,
      "activations/layer9_attention_weight_max": 31.611135482788086,
      "activations/layer9_attention_weight_min": -36.20866775512695,
      "epoch": 17.79,
      "learning_rate": 3.562272727272727e-05,
      "loss": 2.7458,
      "step": 306200
    },
    {
      "activations/layer0_attention_weight_max": 15.836539268493652,
      "activations/layer0_attention_weight_min": -13.092242240905762,
      "activations/layer10_attention_weight_max": 37.06244659423828,
      "activations/layer10_attention_weight_min": -37.1448974609375,
      "activations/layer11_attention_weight_max": 37.09003448486328,
      "activations/layer11_attention_weight_min": -36.33134841918945,
      "activations/layer12_attention_weight_max": 28.458436965942383,
      "activations/layer12_attention_weight_min": -28.89733123779297,
      "activations/layer13_attention_weight_max": 42.023983001708984,
      "activations/layer13_attention_weight_min": -39.10423278808594,
      "activations/layer14_attention_weight_max": 43.961463928222656,
      "activations/layer14_attention_weight_min": -41.81850814819336,
      "activations/layer15_attention_weight_max": 39.73105239868164,
      "activations/layer15_attention_weight_min": -36.51337814331055,
      "activations/layer16_attention_weight_max": 28.779043197631836,
      "activations/layer16_attention_weight_min": -28.802621841430664,
      "activations/layer17_attention_weight_max": 30.57586669921875,
      "activations/layer17_attention_weight_min": -27.370521545410156,
      "activations/layer18_attention_weight_max": 32.061038970947266,
      "activations/layer18_attention_weight_min": -27.66718101501465,
      "activations/layer19_attention_weight_max": 33.719783782958984,
      "activations/layer19_attention_weight_min": -32.411922454833984,
      "activations/layer1_attention_weight_max": 16.167964935302734,
      "activations/layer1_attention_weight_min": -15.765966415405273,
      "activations/layer20_attention_weight_max": 26.249675750732422,
      "activations/layer20_attention_weight_min": -25.810258865356445,
      "activations/layer21_attention_weight_max": 26.668376922607422,
      "activations/layer21_attention_weight_min": -26.61855125427246,
      "activations/layer22_attention_weight_max": 43.670711517333984,
      "activations/layer22_attention_weight_min": -31.62661361694336,
      "activations/layer23_attention_weight_max": 31.59829330444336,
      "activations/layer23_attention_weight_min": -26.79894256591797,
      "activations/layer2_attention_weight_max": 34.10007095336914,
      "activations/layer2_attention_weight_min": -34.66329574584961,
      "activations/layer3_attention_weight_max": 103.95999908447266,
      "activations/layer3_attention_weight_min": -110.17625427246094,
      "activations/layer4_attention_weight_max": 109.151611328125,
      "activations/layer4_attention_weight_min": -113.69783020019531,
      "activations/layer5_attention_weight_max": 50.96391296386719,
      "activations/layer5_attention_weight_min": -63.999168395996094,
      "activations/layer6_attention_weight_max": 45.35593795776367,
      "activations/layer6_attention_weight_min": -48.43449783325195,
      "activations/layer7_attention_weight_max": 97.70122528076172,
      "activations/layer7_attention_weight_min": -101.56905364990234,
      "activations/layer8_attention_weight_max": 45.45866775512695,
      "activations/layer8_attention_weight_min": -46.889793395996094,
      "activations/layer9_attention_weight_max": 35.4980354309082,
      "activations/layer9_attention_weight_min": -36.18027877807617,
      "epoch": 17.79,
      "learning_rate": 3.560378787878788e-05,
      "loss": 2.7403,
      "step": 306250
    },
    {
      "activations/layer0_attention_weight_max": 16.932161331176758,
      "activations/layer0_attention_weight_min": -12.620427131652832,
      "activations/layer10_attention_weight_max": 34.785675048828125,
      "activations/layer10_attention_weight_min": -33.300601959228516,
      "activations/layer11_attention_weight_max": 33.879695892333984,
      "activations/layer11_attention_weight_min": -35.350372314453125,
      "activations/layer12_attention_weight_max": 26.273591995239258,
      "activations/layer12_attention_weight_min": -25.93887710571289,
      "activations/layer13_attention_weight_max": 37.85288619995117,
      "activations/layer13_attention_weight_min": -35.44367980957031,
      "activations/layer14_attention_weight_max": 43.44143295288086,
      "activations/layer14_attention_weight_min": -39.59326934814453,
      "activations/layer15_attention_weight_max": 38.17317581176758,
      "activations/layer15_attention_weight_min": -34.89915084838867,
      "activations/layer16_attention_weight_max": 31.033979415893555,
      "activations/layer16_attention_weight_min": -28.201078414916992,
      "activations/layer17_attention_weight_max": 28.535818099975586,
      "activations/layer17_attention_weight_min": -24.540231704711914,
      "activations/layer18_attention_weight_max": 31.381793975830078,
      "activations/layer18_attention_weight_min": -25.709396362304688,
      "activations/layer19_attention_weight_max": 32.135528564453125,
      "activations/layer19_attention_weight_min": -33.472591400146484,
      "activations/layer1_attention_weight_max": 15.515023231506348,
      "activations/layer1_attention_weight_min": -15.037111282348633,
      "activations/layer20_attention_weight_max": 30.886411666870117,
      "activations/layer20_attention_weight_min": -26.286853790283203,
      "activations/layer21_attention_weight_max": 30.199888229370117,
      "activations/layer21_attention_weight_min": -24.006662368774414,
      "activations/layer22_attention_weight_max": 40.116661071777344,
      "activations/layer22_attention_weight_min": -31.601282119750977,
      "activations/layer23_attention_weight_max": 30.413101196289062,
      "activations/layer23_attention_weight_min": -28.439186096191406,
      "activations/layer2_attention_weight_max": 35.81407928466797,
      "activations/layer2_attention_weight_min": -35.75751495361328,
      "activations/layer3_attention_weight_max": 107.62002563476562,
      "activations/layer3_attention_weight_min": -105.08128356933594,
      "activations/layer4_attention_weight_max": 112.14141082763672,
      "activations/layer4_attention_weight_min": -113.11588287353516,
      "activations/layer5_attention_weight_max": 50.83486557006836,
      "activations/layer5_attention_weight_min": -62.48194122314453,
      "activations/layer6_attention_weight_max": 45.339317321777344,
      "activations/layer6_attention_weight_min": -46.77982711791992,
      "activations/layer7_attention_weight_max": 90.97516632080078,
      "activations/layer7_attention_weight_min": -97.07986450195312,
      "activations/layer8_attention_weight_max": 39.97791290283203,
      "activations/layer8_attention_weight_min": -42.94204330444336,
      "activations/layer9_attention_weight_max": 31.272125244140625,
      "activations/layer9_attention_weight_min": -34.031890869140625,
      "epoch": 17.8,
      "learning_rate": 3.5584848484848486e-05,
      "loss": 2.7494,
      "step": 306300
    },
    {
      "activations/layer0_attention_weight_max": 16.35760498046875,
      "activations/layer0_attention_weight_min": -14.030498504638672,
      "activations/layer10_attention_weight_max": 32.39458084106445,
      "activations/layer10_attention_weight_min": -32.72502517700195,
      "activations/layer11_attention_weight_max": 32.819034576416016,
      "activations/layer11_attention_weight_min": -32.42443084716797,
      "activations/layer12_attention_weight_max": 27.588218688964844,
      "activations/layer12_attention_weight_min": -28.41018295288086,
      "activations/layer13_attention_weight_max": 37.403480529785156,
      "activations/layer13_attention_weight_min": -36.65458679199219,
      "activations/layer14_attention_weight_max": 39.18921661376953,
      "activations/layer14_attention_weight_min": -38.5536003112793,
      "activations/layer15_attention_weight_max": 36.8878173828125,
      "activations/layer15_attention_weight_min": -34.96205520629883,
      "activations/layer16_attention_weight_max": 30.457067489624023,
      "activations/layer16_attention_weight_min": -28.059904098510742,
      "activations/layer17_attention_weight_max": 29.210086822509766,
      "activations/layer17_attention_weight_min": -27.789865493774414,
      "activations/layer18_attention_weight_max": 31.202707290649414,
      "activations/layer18_attention_weight_min": -29.510507583618164,
      "activations/layer19_attention_weight_max": 35.30729293823242,
      "activations/layer19_attention_weight_min": -34.536048889160156,
      "activations/layer1_attention_weight_max": 16.46394157409668,
      "activations/layer1_attention_weight_min": -16.081985473632812,
      "activations/layer20_attention_weight_max": 29.48037338256836,
      "activations/layer20_attention_weight_min": -27.85496711730957,
      "activations/layer21_attention_weight_max": 31.473323822021484,
      "activations/layer21_attention_weight_min": -26.711576461791992,
      "activations/layer22_attention_weight_max": 48.93585205078125,
      "activations/layer22_attention_weight_min": -36.704383850097656,
      "activations/layer23_attention_weight_max": 35.095909118652344,
      "activations/layer23_attention_weight_min": -28.303192138671875,
      "activations/layer2_attention_weight_max": 32.756874084472656,
      "activations/layer2_attention_weight_min": -33.604591369628906,
      "activations/layer3_attention_weight_max": 104.4200439453125,
      "activations/layer3_attention_weight_min": -104.0799560546875,
      "activations/layer4_attention_weight_max": 104.00206756591797,
      "activations/layer4_attention_weight_min": -110.27257537841797,
      "activations/layer5_attention_weight_max": 47.9444580078125,
      "activations/layer5_attention_weight_min": -61.05668258666992,
      "activations/layer6_attention_weight_max": 44.719505310058594,
      "activations/layer6_attention_weight_min": -45.621437072753906,
      "activations/layer7_attention_weight_max": 89.49759674072266,
      "activations/layer7_attention_weight_min": -94.95109558105469,
      "activations/layer8_attention_weight_max": 40.776397705078125,
      "activations/layer8_attention_weight_min": -41.70637130737305,
      "activations/layer9_attention_weight_max": 33.04063034057617,
      "activations/layer9_attention_weight_min": -34.825374603271484,
      "epoch": 17.8,
      "learning_rate": 3.556590909090909e-05,
      "loss": 2.7384,
      "step": 306350
    },
    {
      "activations/layer0_attention_weight_max": 16.383596420288086,
      "activations/layer0_attention_weight_min": -13.278931617736816,
      "activations/layer10_attention_weight_max": 37.789520263671875,
      "activations/layer10_attention_weight_min": -37.36039733886719,
      "activations/layer11_attention_weight_max": 35.6242790222168,
      "activations/layer11_attention_weight_min": -38.60255813598633,
      "activations/layer12_attention_weight_max": 29.704635620117188,
      "activations/layer12_attention_weight_min": -27.99304962158203,
      "activations/layer13_attention_weight_max": 42.45840072631836,
      "activations/layer13_attention_weight_min": -38.524837493896484,
      "activations/layer14_attention_weight_max": 47.14870834350586,
      "activations/layer14_attention_weight_min": -39.45576095581055,
      "activations/layer15_attention_weight_max": 42.8438606262207,
      "activations/layer15_attention_weight_min": -36.418968200683594,
      "activations/layer16_attention_weight_max": 30.94689178466797,
      "activations/layer16_attention_weight_min": -31.51896858215332,
      "activations/layer17_attention_weight_max": 31.74962043762207,
      "activations/layer17_attention_weight_min": -27.082212448120117,
      "activations/layer18_attention_weight_max": 32.022377014160156,
      "activations/layer18_attention_weight_min": -25.93256187438965,
      "activations/layer19_attention_weight_max": 39.800811767578125,
      "activations/layer19_attention_weight_min": -32.86274337768555,
      "activations/layer1_attention_weight_max": 16.951547622680664,
      "activations/layer1_attention_weight_min": -15.528823852539062,
      "activations/layer20_attention_weight_max": 32.152130126953125,
      "activations/layer20_attention_weight_min": -24.825777053833008,
      "activations/layer21_attention_weight_max": 31.515518188476562,
      "activations/layer21_attention_weight_min": -23.799108505249023,
      "activations/layer22_attention_weight_max": 42.418052673339844,
      "activations/layer22_attention_weight_min": -30.375545501708984,
      "activations/layer23_attention_weight_max": 34.70224380493164,
      "activations/layer23_attention_weight_min": -25.69943618774414,
      "activations/layer2_attention_weight_max": 33.97251510620117,
      "activations/layer2_attention_weight_min": -33.57456588745117,
      "activations/layer3_attention_weight_max": 96.34661865234375,
      "activations/layer3_attention_weight_min": -100.99280548095703,
      "activations/layer4_attention_weight_max": 105.24134826660156,
      "activations/layer4_attention_weight_min": -109.207275390625,
      "activations/layer5_attention_weight_max": 51.75950622558594,
      "activations/layer5_attention_weight_min": -66.15897369384766,
      "activations/layer6_attention_weight_max": 45.290611267089844,
      "activations/layer6_attention_weight_min": -47.52645492553711,
      "activations/layer7_attention_weight_max": 96.29319763183594,
      "activations/layer7_attention_weight_min": -97.2531509399414,
      "activations/layer8_attention_weight_max": 41.63301467895508,
      "activations/layer8_attention_weight_min": -47.063716888427734,
      "activations/layer9_attention_weight_max": 34.67033004760742,
      "activations/layer9_attention_weight_min": -36.628841400146484,
      "epoch": 17.8,
      "learning_rate": 3.5547348484848485e-05,
      "loss": 2.7401,
      "step": 306400
    },
    {
      "activations/layer0_attention_weight_max": 16.564861297607422,
      "activations/layer0_attention_weight_min": -14.053736686706543,
      "activations/layer10_attention_weight_max": 32.98686599731445,
      "activations/layer10_attention_weight_min": -35.289222717285156,
      "activations/layer11_attention_weight_max": 33.037166595458984,
      "activations/layer11_attention_weight_min": -35.10516357421875,
      "activations/layer12_attention_weight_max": 27.32234001159668,
      "activations/layer12_attention_weight_min": -27.282325744628906,
      "activations/layer13_attention_weight_max": 41.17165756225586,
      "activations/layer13_attention_weight_min": -37.17430877685547,
      "activations/layer14_attention_weight_max": 45.555702209472656,
      "activations/layer14_attention_weight_min": -39.2978401184082,
      "activations/layer15_attention_weight_max": 41.42435836791992,
      "activations/layer15_attention_weight_min": -37.174312591552734,
      "activations/layer16_attention_weight_max": 28.772695541381836,
      "activations/layer16_attention_weight_min": -28.33364486694336,
      "activations/layer17_attention_weight_max": 30.210012435913086,
      "activations/layer17_attention_weight_min": -26.427452087402344,
      "activations/layer18_attention_weight_max": 29.206937789916992,
      "activations/layer18_attention_weight_min": -26.079036712646484,
      "activations/layer19_attention_weight_max": 29.744171142578125,
      "activations/layer19_attention_weight_min": -31.111709594726562,
      "activations/layer1_attention_weight_max": 16.067371368408203,
      "activations/layer1_attention_weight_min": -15.699231147766113,
      "activations/layer20_attention_weight_max": 27.2584171295166,
      "activations/layer20_attention_weight_min": -26.587961196899414,
      "activations/layer21_attention_weight_max": 28.065366744995117,
      "activations/layer21_attention_weight_min": -24.267309188842773,
      "activations/layer22_attention_weight_max": 37.18693161010742,
      "activations/layer22_attention_weight_min": -32.359981536865234,
      "activations/layer23_attention_weight_max": 28.725927352905273,
      "activations/layer23_attention_weight_min": -25.876785278320312,
      "activations/layer2_attention_weight_max": 35.26753234863281,
      "activations/layer2_attention_weight_min": -33.48188781738281,
      "activations/layer3_attention_weight_max": 97.50769805908203,
      "activations/layer3_attention_weight_min": -97.49666595458984,
      "activations/layer4_attention_weight_max": 103.70903778076172,
      "activations/layer4_attention_weight_min": -100.6990737915039,
      "activations/layer5_attention_weight_max": 50.41012191772461,
      "activations/layer5_attention_weight_min": -59.068275451660156,
      "activations/layer6_attention_weight_max": 44.23014831542969,
      "activations/layer6_attention_weight_min": -44.938472747802734,
      "activations/layer7_attention_weight_max": 94.88243103027344,
      "activations/layer7_attention_weight_min": -91.83735656738281,
      "activations/layer8_attention_weight_max": 39.96685028076172,
      "activations/layer8_attention_weight_min": -42.51421356201172,
      "activations/layer9_attention_weight_max": 32.45058822631836,
      "activations/layer9_attention_weight_min": -33.640804290771484,
      "epoch": 17.81,
      "learning_rate": 3.5528409090909087e-05,
      "loss": 2.7368,
      "step": 306450
    },
    {
      "activations/layer0_attention_weight_max": 16.27995491027832,
      "activations/layer0_attention_weight_min": -13.739273071289062,
      "activations/layer10_attention_weight_max": 34.48111343383789,
      "activations/layer10_attention_weight_min": -35.24264907836914,
      "activations/layer11_attention_weight_max": 33.718994140625,
      "activations/layer11_attention_weight_min": -35.428680419921875,
      "activations/layer12_attention_weight_max": 25.816572189331055,
      "activations/layer12_attention_weight_min": -26.527982711791992,
      "activations/layer13_attention_weight_max": 40.08332443237305,
      "activations/layer13_attention_weight_min": -37.00121307373047,
      "activations/layer14_attention_weight_max": 43.53483200073242,
      "activations/layer14_attention_weight_min": -40.81355667114258,
      "activations/layer15_attention_weight_max": 40.224876403808594,
      "activations/layer15_attention_weight_min": -36.658172607421875,
      "activations/layer16_attention_weight_max": 32.549110412597656,
      "activations/layer16_attention_weight_min": -28.214069366455078,
      "activations/layer17_attention_weight_max": 32.87062454223633,
      "activations/layer17_attention_weight_min": -27.6593074798584,
      "activations/layer18_attention_weight_max": 35.27013397216797,
      "activations/layer18_attention_weight_min": -25.38896942138672,
      "activations/layer19_attention_weight_max": 38.524654388427734,
      "activations/layer19_attention_weight_min": -33.86788558959961,
      "activations/layer1_attention_weight_max": 16.398012161254883,
      "activations/layer1_attention_weight_min": -14.039667129516602,
      "activations/layer20_attention_weight_max": 31.1436710357666,
      "activations/layer20_attention_weight_min": -27.512529373168945,
      "activations/layer21_attention_weight_max": 30.148061752319336,
      "activations/layer21_attention_weight_min": -27.19906234741211,
      "activations/layer22_attention_weight_max": 44.38045883178711,
      "activations/layer22_attention_weight_min": -34.80826950073242,
      "activations/layer23_attention_weight_max": 36.865325927734375,
      "activations/layer23_attention_weight_min": -28.184524536132812,
      "activations/layer2_attention_weight_max": 34.14434051513672,
      "activations/layer2_attention_weight_min": -33.73683166503906,
      "activations/layer3_attention_weight_max": 102.03338623046875,
      "activations/layer3_attention_weight_min": -104.4054183959961,
      "activations/layer4_attention_weight_max": 104.229248046875,
      "activations/layer4_attention_weight_min": -107.90850067138672,
      "activations/layer5_attention_weight_max": 49.79832458496094,
      "activations/layer5_attention_weight_min": -61.062339782714844,
      "activations/layer6_attention_weight_max": 44.14675521850586,
      "activations/layer6_attention_weight_min": -47.71697235107422,
      "activations/layer7_attention_weight_max": 95.49085998535156,
      "activations/layer7_attention_weight_min": -98.18485260009766,
      "activations/layer8_attention_weight_max": 42.13999938964844,
      "activations/layer8_attention_weight_min": -42.594696044921875,
      "activations/layer9_attention_weight_max": 32.83271408081055,
      "activations/layer9_attention_weight_min": -34.14992904663086,
      "epoch": 17.81,
      "learning_rate": 3.5509469696969695e-05,
      "loss": 2.7457,
      "step": 306500
    },
    {
      "activations/layer0_attention_weight_max": 16.84518051147461,
      "activations/layer0_attention_weight_min": -12.585780143737793,
      "activations/layer10_attention_weight_max": 33.80290222167969,
      "activations/layer10_attention_weight_min": -34.15266036987305,
      "activations/layer11_attention_weight_max": 33.65510940551758,
      "activations/layer11_attention_weight_min": -34.790428161621094,
      "activations/layer12_attention_weight_max": 26.414400100708008,
      "activations/layer12_attention_weight_min": -26.688852310180664,
      "activations/layer13_attention_weight_max": 40.859519958496094,
      "activations/layer13_attention_weight_min": -35.967498779296875,
      "activations/layer14_attention_weight_max": 43.846580505371094,
      "activations/layer14_attention_weight_min": -38.5045051574707,
      "activations/layer15_attention_weight_max": 38.47699737548828,
      "activations/layer15_attention_weight_min": -39.075965881347656,
      "activations/layer16_attention_weight_max": 29.245622634887695,
      "activations/layer16_attention_weight_min": -30.14020538330078,
      "activations/layer17_attention_weight_max": 30.248884201049805,
      "activations/layer17_attention_weight_min": -26.823057174682617,
      "activations/layer18_attention_weight_max": 31.295915603637695,
      "activations/layer18_attention_weight_min": -24.329835891723633,
      "activations/layer19_attention_weight_max": 33.871437072753906,
      "activations/layer19_attention_weight_min": -33.10772705078125,
      "activations/layer1_attention_weight_max": 15.439181327819824,
      "activations/layer1_attention_weight_min": -13.816967964172363,
      "activations/layer20_attention_weight_max": 29.654251098632812,
      "activations/layer20_attention_weight_min": -26.296939849853516,
      "activations/layer21_attention_weight_max": 28.867595672607422,
      "activations/layer21_attention_weight_min": -24.60251808166504,
      "activations/layer22_attention_weight_max": 38.79743576049805,
      "activations/layer22_attention_weight_min": -29.41890525817871,
      "activations/layer23_attention_weight_max": 27.963727951049805,
      "activations/layer23_attention_weight_min": -28.711484909057617,
      "activations/layer2_attention_weight_max": 33.35905838012695,
      "activations/layer2_attention_weight_min": -33.23299789428711,
      "activations/layer3_attention_weight_max": 100.05865478515625,
      "activations/layer3_attention_weight_min": -101.52752685546875,
      "activations/layer4_attention_weight_max": 108.39833068847656,
      "activations/layer4_attention_weight_min": -108.73827362060547,
      "activations/layer5_attention_weight_max": 53.588401794433594,
      "activations/layer5_attention_weight_min": -65.94697570800781,
      "activations/layer6_attention_weight_max": 46.809444427490234,
      "activations/layer6_attention_weight_min": -45.85395812988281,
      "activations/layer7_attention_weight_max": 97.744384765625,
      "activations/layer7_attention_weight_min": -93.78995513916016,
      "activations/layer8_attention_weight_max": 41.763118743896484,
      "activations/layer8_attention_weight_min": -43.8928108215332,
      "activations/layer9_attention_weight_max": 32.78052520751953,
      "activations/layer9_attention_weight_min": -33.517601013183594,
      "epoch": 17.81,
      "learning_rate": 3.5490530303030296e-05,
      "loss": 2.7342,
      "step": 306550
    },
    {
      "activations/layer0_attention_weight_max": 17.177839279174805,
      "activations/layer0_attention_weight_min": -12.961771965026855,
      "activations/layer10_attention_weight_max": 43.60361862182617,
      "activations/layer10_attention_weight_min": -39.99700927734375,
      "activations/layer11_attention_weight_max": 45.99452209472656,
      "activations/layer11_attention_weight_min": -42.610660552978516,
      "activations/layer12_attention_weight_max": 27.5401668548584,
      "activations/layer12_attention_weight_min": -34.468692779541016,
      "activations/layer13_attention_weight_max": 39.195552825927734,
      "activations/layer13_attention_weight_min": -34.82907485961914,
      "activations/layer14_attention_weight_max": 42.09477233886719,
      "activations/layer14_attention_weight_min": -37.955467224121094,
      "activations/layer15_attention_weight_max": 37.57538986206055,
      "activations/layer15_attention_weight_min": -37.717281341552734,
      "activations/layer16_attention_weight_max": 28.938783645629883,
      "activations/layer16_attention_weight_min": -27.042640686035156,
      "activations/layer17_attention_weight_max": 31.007415771484375,
      "activations/layer17_attention_weight_min": -29.57062530517578,
      "activations/layer18_attention_weight_max": 32.73695755004883,
      "activations/layer18_attention_weight_min": -26.258039474487305,
      "activations/layer19_attention_weight_max": 32.980003356933594,
      "activations/layer19_attention_weight_min": -31.674625396728516,
      "activations/layer1_attention_weight_max": 15.779623031616211,
      "activations/layer1_attention_weight_min": -15.789704322814941,
      "activations/layer20_attention_weight_max": 30.06636619567871,
      "activations/layer20_attention_weight_min": -25.946598052978516,
      "activations/layer21_attention_weight_max": 28.6837215423584,
      "activations/layer21_attention_weight_min": -25.20591163635254,
      "activations/layer22_attention_weight_max": 40.344200134277344,
      "activations/layer22_attention_weight_min": -32.2093505859375,
      "activations/layer23_attention_weight_max": 33.596710205078125,
      "activations/layer23_attention_weight_min": -25.22824478149414,
      "activations/layer2_attention_weight_max": 36.027626037597656,
      "activations/layer2_attention_weight_min": -34.737762451171875,
      "activations/layer3_attention_weight_max": 104.82735443115234,
      "activations/layer3_attention_weight_min": -103.29723358154297,
      "activations/layer4_attention_weight_max": 111.46910858154297,
      "activations/layer4_attention_weight_min": -106.6839828491211,
      "activations/layer5_attention_weight_max": 54.21568298339844,
      "activations/layer5_attention_weight_min": -58.6151123046875,
      "activations/layer6_attention_weight_max": 47.59888458251953,
      "activations/layer6_attention_weight_min": -47.8991584777832,
      "activations/layer7_attention_weight_max": 109.70475006103516,
      "activations/layer7_attention_weight_min": -101.65613555908203,
      "activations/layer8_attention_weight_max": 48.842041015625,
      "activations/layer8_attention_weight_min": -47.91652297973633,
      "activations/layer9_attention_weight_max": 37.71453857421875,
      "activations/layer9_attention_weight_min": -37.90468978881836,
      "epoch": 17.82,
      "learning_rate": 3.5471590909090905e-05,
      "loss": 2.7452,
      "step": 306600
    },
    {
      "activations/layer0_attention_weight_max": 16.67498207092285,
      "activations/layer0_attention_weight_min": -13.362492561340332,
      "activations/layer10_attention_weight_max": 34.140838623046875,
      "activations/layer10_attention_weight_min": -35.6474723815918,
      "activations/layer11_attention_weight_max": 34.35966110229492,
      "activations/layer11_attention_weight_min": -35.18512725830078,
      "activations/layer12_attention_weight_max": 26.281808853149414,
      "activations/layer12_attention_weight_min": -26.36229133605957,
      "activations/layer13_attention_weight_max": 40.86259841918945,
      "activations/layer13_attention_weight_min": -36.212425231933594,
      "activations/layer14_attention_weight_max": 43.6838264465332,
      "activations/layer14_attention_weight_min": -37.705928802490234,
      "activations/layer15_attention_weight_max": 40.42717742919922,
      "activations/layer15_attention_weight_min": -34.69797134399414,
      "activations/layer16_attention_weight_max": 33.12800979614258,
      "activations/layer16_attention_weight_min": -28.813995361328125,
      "activations/layer17_attention_weight_max": 30.935869216918945,
      "activations/layer17_attention_weight_min": -26.138687133789062,
      "activations/layer18_attention_weight_max": 33.03489685058594,
      "activations/layer18_attention_weight_min": -24.569530487060547,
      "activations/layer19_attention_weight_max": 34.71445846557617,
      "activations/layer19_attention_weight_min": -30.341012954711914,
      "activations/layer1_attention_weight_max": 16.12369155883789,
      "activations/layer1_attention_weight_min": -14.314650535583496,
      "activations/layer20_attention_weight_max": 29.469648361206055,
      "activations/layer20_attention_weight_min": -24.356945037841797,
      "activations/layer21_attention_weight_max": 29.287559509277344,
      "activations/layer21_attention_weight_min": -23.783763885498047,
      "activations/layer22_attention_weight_max": 41.0522346496582,
      "activations/layer22_attention_weight_min": -29.588459014892578,
      "activations/layer23_attention_weight_max": 33.36085510253906,
      "activations/layer23_attention_weight_min": -25.90212059020996,
      "activations/layer2_attention_weight_max": 34.52033996582031,
      "activations/layer2_attention_weight_min": -34.70386505126953,
      "activations/layer3_attention_weight_max": 105.03833770751953,
      "activations/layer3_attention_weight_min": -105.09346008300781,
      "activations/layer4_attention_weight_max": 107.0778579711914,
      "activations/layer4_attention_weight_min": -111.4942398071289,
      "activations/layer5_attention_weight_max": 50.795738220214844,
      "activations/layer5_attention_weight_min": -62.27870178222656,
      "activations/layer6_attention_weight_max": 44.805931091308594,
      "activations/layer6_attention_weight_min": -47.073524475097656,
      "activations/layer7_attention_weight_max": 94.49861145019531,
      "activations/layer7_attention_weight_min": -101.8965835571289,
      "activations/layer8_attention_weight_max": 43.32925033569336,
      "activations/layer8_attention_weight_min": -44.56143569946289,
      "activations/layer9_attention_weight_max": 33.2637825012207,
      "activations/layer9_attention_weight_min": -36.3684196472168,
      "epoch": 17.82,
      "learning_rate": 3.545265151515151e-05,
      "loss": 2.7288,
      "step": 306650
    },
    {
      "activations/layer0_attention_weight_max": 16.691898345947266,
      "activations/layer0_attention_weight_min": -13.049264907836914,
      "activations/layer10_attention_weight_max": 32.646697998046875,
      "activations/layer10_attention_weight_min": -34.31759262084961,
      "activations/layer11_attention_weight_max": 31.7572021484375,
      "activations/layer11_attention_weight_min": -35.261329650878906,
      "activations/layer12_attention_weight_max": 24.662546157836914,
      "activations/layer12_attention_weight_min": -25.78165626525879,
      "activations/layer13_attention_weight_max": 35.313541412353516,
      "activations/layer13_attention_weight_min": -37.15437698364258,
      "activations/layer14_attention_weight_max": 38.152347564697266,
      "activations/layer14_attention_weight_min": -36.29005432128906,
      "activations/layer15_attention_weight_max": 33.89388656616211,
      "activations/layer15_attention_weight_min": -34.271419525146484,
      "activations/layer16_attention_weight_max": 24.73015022277832,
      "activations/layer16_attention_weight_min": -26.342117309570312,
      "activations/layer17_attention_weight_max": 26.155750274658203,
      "activations/layer17_attention_weight_min": -24.963823318481445,
      "activations/layer18_attention_weight_max": 30.612098693847656,
      "activations/layer18_attention_weight_min": -24.94676971435547,
      "activations/layer19_attention_weight_max": 32.246456146240234,
      "activations/layer19_attention_weight_min": -30.63909912109375,
      "activations/layer1_attention_weight_max": 16.33677101135254,
      "activations/layer1_attention_weight_min": -14.57262897491455,
      "activations/layer20_attention_weight_max": 27.01518440246582,
      "activations/layer20_attention_weight_min": -25.162853240966797,
      "activations/layer21_attention_weight_max": 28.45493507385254,
      "activations/layer21_attention_weight_min": -25.823259353637695,
      "activations/layer22_attention_weight_max": 41.54893112182617,
      "activations/layer22_attention_weight_min": -32.30584716796875,
      "activations/layer23_attention_weight_max": 31.829925537109375,
      "activations/layer23_attention_weight_min": -26.867589950561523,
      "activations/layer2_attention_weight_max": 36.05329895019531,
      "activations/layer2_attention_weight_min": -34.08439636230469,
      "activations/layer3_attention_weight_max": 103.13072967529297,
      "activations/layer3_attention_weight_min": -100.00272369384766,
      "activations/layer4_attention_weight_max": 99.95269775390625,
      "activations/layer4_attention_weight_min": -105.6556167602539,
      "activations/layer5_attention_weight_max": 49.57195281982422,
      "activations/layer5_attention_weight_min": -61.16230010986328,
      "activations/layer6_attention_weight_max": 39.57316207885742,
      "activations/layer6_attention_weight_min": -44.56765365600586,
      "activations/layer7_attention_weight_max": 86.37313079833984,
      "activations/layer7_attention_weight_min": -94.3038330078125,
      "activations/layer8_attention_weight_max": 38.129425048828125,
      "activations/layer8_attention_weight_min": -44.57593536376953,
      "activations/layer9_attention_weight_max": 29.108503341674805,
      "activations/layer9_attention_weight_min": -33.99064636230469,
      "epoch": 17.82,
      "learning_rate": 3.543371212121212e-05,
      "loss": 2.742,
      "step": 306700
    },
    {
      "activations/layer0_attention_weight_max": 15.680092811584473,
      "activations/layer0_attention_weight_min": -12.795841217041016,
      "activations/layer10_attention_weight_max": 36.21807861328125,
      "activations/layer10_attention_weight_min": -35.8961067199707,
      "activations/layer11_attention_weight_max": 34.313270568847656,
      "activations/layer11_attention_weight_min": -36.10587692260742,
      "activations/layer12_attention_weight_max": 28.68517303466797,
      "activations/layer12_attention_weight_min": -29.10694694519043,
      "activations/layer13_attention_weight_max": 43.75373077392578,
      "activations/layer13_attention_weight_min": -37.64326858520508,
      "activations/layer14_attention_weight_max": 47.142154693603516,
      "activations/layer14_attention_weight_min": -41.30995559692383,
      "activations/layer15_attention_weight_max": 42.36698532104492,
      "activations/layer15_attention_weight_min": -37.334415435791016,
      "activations/layer16_attention_weight_max": 29.2087459564209,
      "activations/layer16_attention_weight_min": -28.275423049926758,
      "activations/layer17_attention_weight_max": 29.808984756469727,
      "activations/layer17_attention_weight_min": -27.107004165649414,
      "activations/layer18_attention_weight_max": 30.471162796020508,
      "activations/layer18_attention_weight_min": -25.787811279296875,
      "activations/layer19_attention_weight_max": 34.637367248535156,
      "activations/layer19_attention_weight_min": -31.5997257232666,
      "activations/layer1_attention_weight_max": 17.933422088623047,
      "activations/layer1_attention_weight_min": -17.526111602783203,
      "activations/layer20_attention_weight_max": 26.31038475036621,
      "activations/layer20_attention_weight_min": -25.986215591430664,
      "activations/layer21_attention_weight_max": 27.55765151977539,
      "activations/layer21_attention_weight_min": -25.227916717529297,
      "activations/layer22_attention_weight_max": 41.47903823852539,
      "activations/layer22_attention_weight_min": -33.935672760009766,
      "activations/layer23_attention_weight_max": 31.626617431640625,
      "activations/layer23_attention_weight_min": -27.75543975830078,
      "activations/layer2_attention_weight_max": 35.806739807128906,
      "activations/layer2_attention_weight_min": -34.68680953979492,
      "activations/layer3_attention_weight_max": 104.35074615478516,
      "activations/layer3_attention_weight_min": -103.20036315917969,
      "activations/layer4_attention_weight_max": 103.3553237915039,
      "activations/layer4_attention_weight_min": -106.6003189086914,
      "activations/layer5_attention_weight_max": 53.28091049194336,
      "activations/layer5_attention_weight_min": -62.24279022216797,
      "activations/layer6_attention_weight_max": 44.77965545654297,
      "activations/layer6_attention_weight_min": -46.90489959716797,
      "activations/layer7_attention_weight_max": 103.99691772460938,
      "activations/layer7_attention_weight_min": -102.20137786865234,
      "activations/layer8_attention_weight_max": 43.373329162597656,
      "activations/layer8_attention_weight_min": -46.41787338256836,
      "activations/layer9_attention_weight_max": 35.36368179321289,
      "activations/layer9_attention_weight_min": -37.62906265258789,
      "epoch": 17.82,
      "learning_rate": 3.541477272727272e-05,
      "loss": 2.745,
      "step": 306750
    },
    {
      "activations/layer0_attention_weight_max": 16.396947860717773,
      "activations/layer0_attention_weight_min": -14.567667961120605,
      "activations/layer10_attention_weight_max": 33.46567153930664,
      "activations/layer10_attention_weight_min": -35.43250274658203,
      "activations/layer11_attention_weight_max": 32.87468719482422,
      "activations/layer11_attention_weight_min": -35.9110107421875,
      "activations/layer12_attention_weight_max": 28.178218841552734,
      "activations/layer12_attention_weight_min": -27.01510238647461,
      "activations/layer13_attention_weight_max": 41.863338470458984,
      "activations/layer13_attention_weight_min": -38.098445892333984,
      "activations/layer14_attention_weight_max": 42.41718292236328,
      "activations/layer14_attention_weight_min": -42.67820358276367,
      "activations/layer15_attention_weight_max": 40.48582077026367,
      "activations/layer15_attention_weight_min": -37.63611602783203,
      "activations/layer16_attention_weight_max": 30.103097915649414,
      "activations/layer16_attention_weight_min": -29.212909698486328,
      "activations/layer17_attention_weight_max": 30.815711975097656,
      "activations/layer17_attention_weight_min": -25.786283493041992,
      "activations/layer18_attention_weight_max": 30.741619110107422,
      "activations/layer18_attention_weight_min": -25.356828689575195,
      "activations/layer19_attention_weight_max": 36.65675735473633,
      "activations/layer19_attention_weight_min": -32.83395004272461,
      "activations/layer1_attention_weight_max": 18.030927658081055,
      "activations/layer1_attention_weight_min": -17.460771560668945,
      "activations/layer20_attention_weight_max": 30.702608108520508,
      "activations/layer20_attention_weight_min": -26.781715393066406,
      "activations/layer21_attention_weight_max": 28.812658309936523,
      "activations/layer21_attention_weight_min": -25.888669967651367,
      "activations/layer22_attention_weight_max": 41.920806884765625,
      "activations/layer22_attention_weight_min": -32.72502136230469,
      "activations/layer23_attention_weight_max": 32.68223190307617,
      "activations/layer23_attention_weight_min": -27.298198699951172,
      "activations/layer2_attention_weight_max": 34.468177795410156,
      "activations/layer2_attention_weight_min": -34.41783142089844,
      "activations/layer3_attention_weight_max": 101.41755676269531,
      "activations/layer3_attention_weight_min": -102.80657196044922,
      "activations/layer4_attention_weight_max": 101.94251251220703,
      "activations/layer4_attention_weight_min": -107.92317199707031,
      "activations/layer5_attention_weight_max": 48.33490753173828,
      "activations/layer5_attention_weight_min": -61.5767936706543,
      "activations/layer6_attention_weight_max": 43.801151275634766,
      "activations/layer6_attention_weight_min": -45.7387809753418,
      "activations/layer7_attention_weight_max": 94.0639419555664,
      "activations/layer7_attention_weight_min": -95.21670532226562,
      "activations/layer8_attention_weight_max": 41.17334747314453,
      "activations/layer8_attention_weight_min": -44.15457534790039,
      "activations/layer9_attention_weight_max": 31.209671020507812,
      "activations/layer9_attention_weight_min": -33.7634391784668,
      "epoch": 17.83,
      "learning_rate": 3.539583333333333e-05,
      "loss": 2.733,
      "step": 306800
    },
    {
      "activations/layer0_attention_weight_max": 15.943643569946289,
      "activations/layer0_attention_weight_min": -14.049324035644531,
      "activations/layer10_attention_weight_max": 34.38017272949219,
      "activations/layer10_attention_weight_min": -33.68928909301758,
      "activations/layer11_attention_weight_max": 35.39082336425781,
      "activations/layer11_attention_weight_min": -36.11618423461914,
      "activations/layer12_attention_weight_max": 28.862720489501953,
      "activations/layer12_attention_weight_min": -26.8721866607666,
      "activations/layer13_attention_weight_max": 40.87700271606445,
      "activations/layer13_attention_weight_min": -35.51945877075195,
      "activations/layer14_attention_weight_max": 46.57229995727539,
      "activations/layer14_attention_weight_min": -38.831932067871094,
      "activations/layer15_attention_weight_max": 48.4726676940918,
      "activations/layer15_attention_weight_min": -36.8276481628418,
      "activations/layer16_attention_weight_max": 36.59856033325195,
      "activations/layer16_attention_weight_min": -28.276323318481445,
      "activations/layer17_attention_weight_max": 37.55888748168945,
      "activations/layer17_attention_weight_min": -26.146076202392578,
      "activations/layer18_attention_weight_max": 33.969024658203125,
      "activations/layer18_attention_weight_min": -25.491731643676758,
      "activations/layer19_attention_weight_max": 41.72468948364258,
      "activations/layer19_attention_weight_min": -35.77516174316406,
      "activations/layer1_attention_weight_max": 16.918781280517578,
      "activations/layer1_attention_weight_min": -15.571980476379395,
      "activations/layer20_attention_weight_max": 35.80939865112305,
      "activations/layer20_attention_weight_min": -26.710172653198242,
      "activations/layer21_attention_weight_max": 35.10709762573242,
      "activations/layer21_attention_weight_min": -25.384033203125,
      "activations/layer22_attention_weight_max": 46.24150848388672,
      "activations/layer22_attention_weight_min": -33.55908203125,
      "activations/layer23_attention_weight_max": 39.63253402709961,
      "activations/layer23_attention_weight_min": -26.412979125976562,
      "activations/layer2_attention_weight_max": 34.71735763549805,
      "activations/layer2_attention_weight_min": -34.08017349243164,
      "activations/layer3_attention_weight_max": 104.40727996826172,
      "activations/layer3_attention_weight_min": -101.39540100097656,
      "activations/layer4_attention_weight_max": 108.86537170410156,
      "activations/layer4_attention_weight_min": -114.04875946044922,
      "activations/layer5_attention_weight_max": 50.78190231323242,
      "activations/layer5_attention_weight_min": -66.09547424316406,
      "activations/layer6_attention_weight_max": 46.7795295715332,
      "activations/layer6_attention_weight_min": -50.06957244873047,
      "activations/layer7_attention_weight_max": 95.60453796386719,
      "activations/layer7_attention_weight_min": -96.052490234375,
      "activations/layer8_attention_weight_max": 43.41061782836914,
      "activations/layer8_attention_weight_min": -42.861759185791016,
      "activations/layer9_attention_weight_max": 33.695377349853516,
      "activations/layer9_attention_weight_min": -33.86665344238281,
      "epoch": 17.83,
      "learning_rate": 3.537689393939394e-05,
      "loss": 2.7322,
      "step": 306850
    },
    {
      "activations/layer0_attention_weight_max": 17.128524780273438,
      "activations/layer0_attention_weight_min": -14.400372505187988,
      "activations/layer10_attention_weight_max": 35.281307220458984,
      "activations/layer10_attention_weight_min": -32.89854431152344,
      "activations/layer11_attention_weight_max": 36.00381851196289,
      "activations/layer11_attention_weight_min": -35.47630310058594,
      "activations/layer12_attention_weight_max": 28.25648307800293,
      "activations/layer12_attention_weight_min": -26.979663848876953,
      "activations/layer13_attention_weight_max": 40.80033493041992,
      "activations/layer13_attention_weight_min": -34.920013427734375,
      "activations/layer14_attention_weight_max": 44.3070068359375,
      "activations/layer14_attention_weight_min": -40.058380126953125,
      "activations/layer15_attention_weight_max": 39.782752990722656,
      "activations/layer15_attention_weight_min": -38.054073333740234,
      "activations/layer16_attention_weight_max": 31.00212860107422,
      "activations/layer16_attention_weight_min": -26.800912857055664,
      "activations/layer17_attention_weight_max": 32.28139114379883,
      "activations/layer17_attention_weight_min": -27.101348876953125,
      "activations/layer18_attention_weight_max": 36.90083694458008,
      "activations/layer18_attention_weight_min": -27.00361442565918,
      "activations/layer19_attention_weight_max": 37.313106536865234,
      "activations/layer19_attention_weight_min": -31.081384658813477,
      "activations/layer1_attention_weight_max": 17.166288375854492,
      "activations/layer1_attention_weight_min": -14.287467002868652,
      "activations/layer20_attention_weight_max": 33.565067291259766,
      "activations/layer20_attention_weight_min": -27.2165470123291,
      "activations/layer21_attention_weight_max": 31.354705810546875,
      "activations/layer21_attention_weight_min": -26.0065860748291,
      "activations/layer22_attention_weight_max": 43.72544860839844,
      "activations/layer22_attention_weight_min": -33.495689392089844,
      "activations/layer23_attention_weight_max": 36.107666015625,
      "activations/layer23_attention_weight_min": -28.101383209228516,
      "activations/layer2_attention_weight_max": 36.05728530883789,
      "activations/layer2_attention_weight_min": -33.28668975830078,
      "activations/layer3_attention_weight_max": 98.43800354003906,
      "activations/layer3_attention_weight_min": -100.27229309082031,
      "activations/layer4_attention_weight_max": 102.898681640625,
      "activations/layer4_attention_weight_min": -109.9745864868164,
      "activations/layer5_attention_weight_max": 49.87316131591797,
      "activations/layer5_attention_weight_min": -64.82553100585938,
      "activations/layer6_attention_weight_max": 44.95471954345703,
      "activations/layer6_attention_weight_min": -46.87885284423828,
      "activations/layer7_attention_weight_max": 90.77367401123047,
      "activations/layer7_attention_weight_min": -92.44493103027344,
      "activations/layer8_attention_weight_max": 44.6852912902832,
      "activations/layer8_attention_weight_min": -42.65885543823242,
      "activations/layer9_attention_weight_max": 35.0773811340332,
      "activations/layer9_attention_weight_min": -34.14179611206055,
      "epoch": 17.83,
      "learning_rate": 3.535795454545454e-05,
      "loss": 2.725,
      "step": 306900
    },
    {
      "activations/layer0_attention_weight_max": 16.3956298828125,
      "activations/layer0_attention_weight_min": -13.097071647644043,
      "activations/layer10_attention_weight_max": 32.46674728393555,
      "activations/layer10_attention_weight_min": -33.0335807800293,
      "activations/layer11_attention_weight_max": 32.58238983154297,
      "activations/layer11_attention_weight_min": -34.002227783203125,
      "activations/layer12_attention_weight_max": 28.086103439331055,
      "activations/layer12_attention_weight_min": -27.160762786865234,
      "activations/layer13_attention_weight_max": 42.86039352416992,
      "activations/layer13_attention_weight_min": -35.91675567626953,
      "activations/layer14_attention_weight_max": 43.12683868408203,
      "activations/layer14_attention_weight_min": -35.92353439331055,
      "activations/layer15_attention_weight_max": 39.74325180053711,
      "activations/layer15_attention_weight_min": -35.85420608520508,
      "activations/layer16_attention_weight_max": 32.59130859375,
      "activations/layer16_attention_weight_min": -28.067758560180664,
      "activations/layer17_attention_weight_max": 33.76144027709961,
      "activations/layer17_attention_weight_min": -27.54556655883789,
      "activations/layer18_attention_weight_max": 34.33546829223633,
      "activations/layer18_attention_weight_min": -28.786588668823242,
      "activations/layer19_attention_weight_max": 33.091915130615234,
      "activations/layer19_attention_weight_min": -31.534137725830078,
      "activations/layer1_attention_weight_max": 16.49716567993164,
      "activations/layer1_attention_weight_min": -15.648674011230469,
      "activations/layer20_attention_weight_max": 30.769248962402344,
      "activations/layer20_attention_weight_min": -26.055370330810547,
      "activations/layer21_attention_weight_max": 31.272573471069336,
      "activations/layer21_attention_weight_min": -25.863767623901367,
      "activations/layer22_attention_weight_max": 39.01996994018555,
      "activations/layer22_attention_weight_min": -31.236909866333008,
      "activations/layer23_attention_weight_max": 30.648231506347656,
      "activations/layer23_attention_weight_min": -25.64662742614746,
      "activations/layer2_attention_weight_max": 33.07250213623047,
      "activations/layer2_attention_weight_min": -32.99113845825195,
      "activations/layer3_attention_weight_max": 96.93682098388672,
      "activations/layer3_attention_weight_min": -98.85450744628906,
      "activations/layer4_attention_weight_max": 102.78067779541016,
      "activations/layer4_attention_weight_min": -103.569580078125,
      "activations/layer5_attention_weight_max": 47.18893051147461,
      "activations/layer5_attention_weight_min": -63.448585510253906,
      "activations/layer6_attention_weight_max": 40.69829177856445,
      "activations/layer6_attention_weight_min": -44.34962844848633,
      "activations/layer7_attention_weight_max": 88.61805725097656,
      "activations/layer7_attention_weight_min": -91.11414337158203,
      "activations/layer8_attention_weight_max": 41.45357131958008,
      "activations/layer8_attention_weight_min": -44.69893264770508,
      "activations/layer9_attention_weight_max": 31.308202743530273,
      "activations/layer9_attention_weight_min": -34.21043014526367,
      "epoch": 17.84,
      "learning_rate": 3.533901515151515e-05,
      "loss": 2.7452,
      "step": 306950
    },
    {
      "activations/layer0_attention_weight_max": 15.713543891906738,
      "activations/layer0_attention_weight_min": -13.847273826599121,
      "activations/layer10_attention_weight_max": 37.01289749145508,
      "activations/layer10_attention_weight_min": -34.59938430786133,
      "activations/layer11_attention_weight_max": 36.17091369628906,
      "activations/layer11_attention_weight_min": -35.551448822021484,
      "activations/layer12_attention_weight_max": 27.75661849975586,
      "activations/layer12_attention_weight_min": -28.672822952270508,
      "activations/layer13_attention_weight_max": 40.20014953613281,
      "activations/layer13_attention_weight_min": -36.68974685668945,
      "activations/layer14_attention_weight_max": 42.742591857910156,
      "activations/layer14_attention_weight_min": -39.531715393066406,
      "activations/layer15_attention_weight_max": 37.4371223449707,
      "activations/layer15_attention_weight_min": -36.62371826171875,
      "activations/layer16_attention_weight_max": 31.80356788635254,
      "activations/layer16_attention_weight_min": -26.984636306762695,
      "activations/layer17_attention_weight_max": 34.99919509887695,
      "activations/layer17_attention_weight_min": -25.901498794555664,
      "activations/layer18_attention_weight_max": 33.526485443115234,
      "activations/layer18_attention_weight_min": -26.139328002929688,
      "activations/layer19_attention_weight_max": 36.37801742553711,
      "activations/layer19_attention_weight_min": -30.683496475219727,
      "activations/layer1_attention_weight_max": 17.844554901123047,
      "activations/layer1_attention_weight_min": -16.168254852294922,
      "activations/layer20_attention_weight_max": 33.0426025390625,
      "activations/layer20_attention_weight_min": -24.734851837158203,
      "activations/layer21_attention_weight_max": 33.84144592285156,
      "activations/layer21_attention_weight_min": -25.41044044494629,
      "activations/layer22_attention_weight_max": 42.994503021240234,
      "activations/layer22_attention_weight_min": -30.705936431884766,
      "activations/layer23_attention_weight_max": 34.388580322265625,
      "activations/layer23_attention_weight_min": -26.366249084472656,
      "activations/layer2_attention_weight_max": 33.80374526977539,
      "activations/layer2_attention_weight_min": -31.06569480895996,
      "activations/layer3_attention_weight_max": 103.38895416259766,
      "activations/layer3_attention_weight_min": -100.3144302368164,
      "activations/layer4_attention_weight_max": 108.22784423828125,
      "activations/layer4_attention_weight_min": -106.02064514160156,
      "activations/layer5_attention_weight_max": 52.75580596923828,
      "activations/layer5_attention_weight_min": -63.20262908935547,
      "activations/layer6_attention_weight_max": 45.20848846435547,
      "activations/layer6_attention_weight_min": -48.97694396972656,
      "activations/layer7_attention_weight_max": 95.82255554199219,
      "activations/layer7_attention_weight_min": -100.9985122680664,
      "activations/layer8_attention_weight_max": 43.44588088989258,
      "activations/layer8_attention_weight_min": -44.16954040527344,
      "activations/layer9_attention_weight_max": 34.231693267822266,
      "activations/layer9_attention_weight_min": -34.9134521484375,
      "epoch": 17.84,
      "learning_rate": 3.532007575757576e-05,
      "loss": 2.7393,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4797,
      "eval_samples_per_second": 506.387,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4797,
      "eval_openwebtext_samples_per_second": 506.387,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9877,
      "eval_wikitext_samples_per_second": 229.412,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.5896,
      "eval_lambada_samples_per_second": 507.737,
      "step": 307000
    },
    {
      "activations/layer0_attention_weight_max": 16.196666717529297,
      "activations/layer0_attention_weight_min": -13.128196716308594,
      "activations/layer10_attention_weight_max": 32.925697326660156,
      "activations/layer10_attention_weight_min": -33.82805252075195,
      "activations/layer11_attention_weight_max": 30.913564682006836,
      "activations/layer11_attention_weight_min": -34.28679656982422,
      "activations/layer12_attention_weight_max": 26.913124084472656,
      "activations/layer12_attention_weight_min": -24.721363067626953,
      "activations/layer13_attention_weight_max": 37.552955627441406,
      "activations/layer13_attention_weight_min": -34.385398864746094,
      "activations/layer14_attention_weight_max": 42.96177673339844,
      "activations/layer14_attention_weight_min": -36.25261306762695,
      "activations/layer15_attention_weight_max": 36.27678298950195,
      "activations/layer15_attention_weight_min": -33.11345291137695,
      "activations/layer16_attention_weight_max": 30.542150497436523,
      "activations/layer16_attention_weight_min": -28.83858871459961,
      "activations/layer17_attention_weight_max": 29.06085968017578,
      "activations/layer17_attention_weight_min": -25.87309455871582,
      "activations/layer18_attention_weight_max": 30.245229721069336,
      "activations/layer18_attention_weight_min": -25.54069709777832,
      "activations/layer19_attention_weight_max": 32.54507827758789,
      "activations/layer19_attention_weight_min": -30.93763542175293,
      "activations/layer1_attention_weight_max": 16.997215270996094,
      "activations/layer1_attention_weight_min": -13.986286163330078,
      "activations/layer20_attention_weight_max": 29.22283172607422,
      "activations/layer20_attention_weight_min": -26.562761306762695,
      "activations/layer21_attention_weight_max": 27.557662963867188,
      "activations/layer21_attention_weight_min": -24.56480598449707,
      "activations/layer22_attention_weight_max": 39.561363220214844,
      "activations/layer22_attention_weight_min": -30.981403350830078,
      "activations/layer23_attention_weight_max": 32.69990921020508,
      "activations/layer23_attention_weight_min": -26.492971420288086,
      "activations/layer2_attention_weight_max": 31.238285064697266,
      "activations/layer2_attention_weight_min": -33.342247009277344,
      "activations/layer3_attention_weight_max": 99.24747467041016,
      "activations/layer3_attention_weight_min": -103.10185241699219,
      "activations/layer4_attention_weight_max": 101.3351821899414,
      "activations/layer4_attention_weight_min": -105.7546157836914,
      "activations/layer5_attention_weight_max": 53.07872009277344,
      "activations/layer5_attention_weight_min": -63.131103515625,
      "activations/layer6_attention_weight_max": 45.177757263183594,
      "activations/layer6_attention_weight_min": -47.70108413696289,
      "activations/layer7_attention_weight_max": 89.86639404296875,
      "activations/layer7_attention_weight_min": -99.59361267089844,
      "activations/layer8_attention_weight_max": 40.57115173339844,
      "activations/layer8_attention_weight_min": -43.84268569946289,
      "activations/layer9_attention_weight_max": 31.743337631225586,
      "activations/layer9_attention_weight_min": -32.32216262817383,
      "epoch": 17.84,
      "learning_rate": 3.530113636363636e-05,
      "loss": 2.7438,
      "step": 307050
    },
    {
      "activations/layer0_attention_weight_max": 16.281232833862305,
      "activations/layer0_attention_weight_min": -12.624650001525879,
      "activations/layer10_attention_weight_max": 32.97801208496094,
      "activations/layer10_attention_weight_min": -34.718719482421875,
      "activations/layer11_attention_weight_max": 34.2014045715332,
      "activations/layer11_attention_weight_min": -35.470699310302734,
      "activations/layer12_attention_weight_max": 29.88849639892578,
      "activations/layer12_attention_weight_min": -27.087024688720703,
      "activations/layer13_attention_weight_max": 40.781776428222656,
      "activations/layer13_attention_weight_min": -37.099693298339844,
      "activations/layer14_attention_weight_max": 46.99971008300781,
      "activations/layer14_attention_weight_min": -39.71372604370117,
      "activations/layer15_attention_weight_max": 41.15351867675781,
      "activations/layer15_attention_weight_min": -37.262054443359375,
      "activations/layer16_attention_weight_max": 31.909984588623047,
      "activations/layer16_attention_weight_min": -28.253618240356445,
      "activations/layer17_attention_weight_max": 30.91143226623535,
      "activations/layer17_attention_weight_min": -28.484704971313477,
      "activations/layer18_attention_weight_max": 34.752376556396484,
      "activations/layer18_attention_weight_min": -27.240596771240234,
      "activations/layer19_attention_weight_max": 35.80758285522461,
      "activations/layer19_attention_weight_min": -31.614776611328125,
      "activations/layer1_attention_weight_max": 15.662568092346191,
      "activations/layer1_attention_weight_min": -14.849445343017578,
      "activations/layer20_attention_weight_max": 31.561256408691406,
      "activations/layer20_attention_weight_min": -26.776710510253906,
      "activations/layer21_attention_weight_max": 31.988407135009766,
      "activations/layer21_attention_weight_min": -24.9038028717041,
      "activations/layer22_attention_weight_max": 45.381019592285156,
      "activations/layer22_attention_weight_min": -33.83964538574219,
      "activations/layer23_attention_weight_max": 37.79594421386719,
      "activations/layer23_attention_weight_min": -27.75314712524414,
      "activations/layer2_attention_weight_max": 31.278369903564453,
      "activations/layer2_attention_weight_min": -31.22639274597168,
      "activations/layer3_attention_weight_max": 100.74419403076172,
      "activations/layer3_attention_weight_min": -96.93728637695312,
      "activations/layer4_attention_weight_max": 109.94146728515625,
      "activations/layer4_attention_weight_min": -107.82059478759766,
      "activations/layer5_attention_weight_max": 48.28916549682617,
      "activations/layer5_attention_weight_min": -60.366554260253906,
      "activations/layer6_attention_weight_max": 44.863582611083984,
      "activations/layer6_attention_weight_min": -46.312801361083984,
      "activations/layer7_attention_weight_max": 92.6602783203125,
      "activations/layer7_attention_weight_min": -92.49163818359375,
      "activations/layer8_attention_weight_max": 40.21132278442383,
      "activations/layer8_attention_weight_min": -44.154022216796875,
      "activations/layer9_attention_weight_max": 33.02804183959961,
      "activations/layer9_attention_weight_min": -34.877925872802734,
      "epoch": 17.84,
      "learning_rate": 3.528219696969697e-05,
      "loss": 2.7321,
      "step": 307100
    },
    {
      "activations/layer0_attention_weight_max": 16.39591407775879,
      "activations/layer0_attention_weight_min": -14.199824333190918,
      "activations/layer10_attention_weight_max": 35.20558547973633,
      "activations/layer10_attention_weight_min": -33.84663009643555,
      "activations/layer11_attention_weight_max": 34.949615478515625,
      "activations/layer11_attention_weight_min": -33.42194366455078,
      "activations/layer12_attention_weight_max": 27.26392936706543,
      "activations/layer12_attention_weight_min": -25.687911987304688,
      "activations/layer13_attention_weight_max": 37.56056594848633,
      "activations/layer13_attention_weight_min": -33.90532684326172,
      "activations/layer14_attention_weight_max": 41.24357986450195,
      "activations/layer14_attention_weight_min": -36.93429946899414,
      "activations/layer15_attention_weight_max": 39.39867401123047,
      "activations/layer15_attention_weight_min": -35.518699645996094,
      "activations/layer16_attention_weight_max": 29.821422576904297,
      "activations/layer16_attention_weight_min": -27.50255012512207,
      "activations/layer17_attention_weight_max": 29.15274429321289,
      "activations/layer17_attention_weight_min": -27.866872787475586,
      "activations/layer18_attention_weight_max": 32.017425537109375,
      "activations/layer18_attention_weight_min": -28.640226364135742,
      "activations/layer19_attention_weight_max": 31.614761352539062,
      "activations/layer19_attention_weight_min": -31.944589614868164,
      "activations/layer1_attention_weight_max": 15.12282943725586,
      "activations/layer1_attention_weight_min": -14.630749702453613,
      "activations/layer20_attention_weight_max": 28.21295738220215,
      "activations/layer20_attention_weight_min": -25.83732795715332,
      "activations/layer21_attention_weight_max": 28.061662673950195,
      "activations/layer21_attention_weight_min": -27.04939842224121,
      "activations/layer22_attention_weight_max": 41.20243453979492,
      "activations/layer22_attention_weight_min": -30.956005096435547,
      "activations/layer23_attention_weight_max": 30.641510009765625,
      "activations/layer23_attention_weight_min": -25.61042594909668,
      "activations/layer2_attention_weight_max": 32.346153259277344,
      "activations/layer2_attention_weight_min": -32.88873291015625,
      "activations/layer3_attention_weight_max": 99.63352966308594,
      "activations/layer3_attention_weight_min": -100.79969787597656,
      "activations/layer4_attention_weight_max": 105.10208892822266,
      "activations/layer4_attention_weight_min": -107.28343963623047,
      "activations/layer5_attention_weight_max": 49.359249114990234,
      "activations/layer5_attention_weight_min": -61.00934982299805,
      "activations/layer6_attention_weight_max": 45.06290054321289,
      "activations/layer6_attention_weight_min": -45.57516860961914,
      "activations/layer7_attention_weight_max": 90.89466094970703,
      "activations/layer7_attention_weight_min": -96.30728149414062,
      "activations/layer8_attention_weight_max": 43.05959701538086,
      "activations/layer8_attention_weight_min": -42.286651611328125,
      "activations/layer9_attention_weight_max": 32.10538864135742,
      "activations/layer9_attention_weight_min": -33.9077262878418,
      "epoch": 17.85,
      "learning_rate": 3.5263257575757575e-05,
      "loss": 2.7236,
      "step": 307150
    },
    {
      "activations/layer0_attention_weight_max": 16.329130172729492,
      "activations/layer0_attention_weight_min": -13.598085403442383,
      "activations/layer10_attention_weight_max": 33.54680252075195,
      "activations/layer10_attention_weight_min": -32.703006744384766,
      "activations/layer11_attention_weight_max": 34.29817581176758,
      "activations/layer11_attention_weight_min": -32.367652893066406,
      "activations/layer12_attention_weight_max": 26.514570236206055,
      "activations/layer12_attention_weight_min": -26.629091262817383,
      "activations/layer13_attention_weight_max": 39.22944259643555,
      "activations/layer13_attention_weight_min": -34.47397232055664,
      "activations/layer14_attention_weight_max": 41.130523681640625,
      "activations/layer14_attention_weight_min": -36.44224548339844,
      "activations/layer15_attention_weight_max": 38.10834503173828,
      "activations/layer15_attention_weight_min": -34.64857482910156,
      "activations/layer16_attention_weight_max": 28.107572555541992,
      "activations/layer16_attention_weight_min": -29.60340118408203,
      "activations/layer17_attention_weight_max": 30.329980850219727,
      "activations/layer17_attention_weight_min": -28.402917861938477,
      "activations/layer18_attention_weight_max": 33.633575439453125,
      "activations/layer18_attention_weight_min": -25.6096248626709,
      "activations/layer19_attention_weight_max": 33.96982192993164,
      "activations/layer19_attention_weight_min": -31.29749870300293,
      "activations/layer1_attention_weight_max": 15.603902816772461,
      "activations/layer1_attention_weight_min": -13.69001293182373,
      "activations/layer20_attention_weight_max": 26.860321044921875,
      "activations/layer20_attention_weight_min": -25.920276641845703,
      "activations/layer21_attention_weight_max": 29.654333114624023,
      "activations/layer21_attention_weight_min": -24.782861709594727,
      "activations/layer22_attention_weight_max": 38.46651077270508,
      "activations/layer22_attention_weight_min": -30.940128326416016,
      "activations/layer23_attention_weight_max": 31.84833526611328,
      "activations/layer23_attention_weight_min": -24.602216720581055,
      "activations/layer2_attention_weight_max": 32.74909591674805,
      "activations/layer2_attention_weight_min": -32.45634078979492,
      "activations/layer3_attention_weight_max": 101.46835327148438,
      "activations/layer3_attention_weight_min": -100.67638397216797,
      "activations/layer4_attention_weight_max": 109.24083709716797,
      "activations/layer4_attention_weight_min": -109.16857147216797,
      "activations/layer5_attention_weight_max": 49.78290939331055,
      "activations/layer5_attention_weight_min": -61.686073303222656,
      "activations/layer6_attention_weight_max": 43.981998443603516,
      "activations/layer6_attention_weight_min": -44.912681579589844,
      "activations/layer7_attention_weight_max": 91.48068237304688,
      "activations/layer7_attention_weight_min": -91.60600280761719,
      "activations/layer8_attention_weight_max": 42.116416931152344,
      "activations/layer8_attention_weight_min": -43.03133010864258,
      "activations/layer9_attention_weight_max": 32.299564361572266,
      "activations/layer9_attention_weight_min": -33.38816833496094,
      "epoch": 17.85,
      "learning_rate": 3.524431818181818e-05,
      "loss": 2.7223,
      "step": 307200
    },
    {
      "activations/layer0_attention_weight_max": 16.221237182617188,
      "activations/layer0_attention_weight_min": -13.358078002929688,
      "activations/layer10_attention_weight_max": 37.20171356201172,
      "activations/layer10_attention_weight_min": -36.898345947265625,
      "activations/layer11_attention_weight_max": 37.396358489990234,
      "activations/layer11_attention_weight_min": -36.8321418762207,
      "activations/layer12_attention_weight_max": 27.74598503112793,
      "activations/layer12_attention_weight_min": -25.89830780029297,
      "activations/layer13_attention_weight_max": 40.69448471069336,
      "activations/layer13_attention_weight_min": -36.0468864440918,
      "activations/layer14_attention_weight_max": 48.7965087890625,
      "activations/layer14_attention_weight_min": -38.93832778930664,
      "activations/layer15_attention_weight_max": 40.58863830566406,
      "activations/layer15_attention_weight_min": -33.84744644165039,
      "activations/layer16_attention_weight_max": 36.35312271118164,
      "activations/layer16_attention_weight_min": -28.785619735717773,
      "activations/layer17_attention_weight_max": 31.42740821838379,
      "activations/layer17_attention_weight_min": -26.3037166595459,
      "activations/layer18_attention_weight_max": 35.22066116333008,
      "activations/layer18_attention_weight_min": -28.485214233398438,
      "activations/layer19_attention_weight_max": 37.74872970581055,
      "activations/layer19_attention_weight_min": -36.05875015258789,
      "activations/layer1_attention_weight_max": 17.251768112182617,
      "activations/layer1_attention_weight_min": -18.601322174072266,
      "activations/layer20_attention_weight_max": 28.592817306518555,
      "activations/layer20_attention_weight_min": -26.65682029724121,
      "activations/layer21_attention_weight_max": 27.598161697387695,
      "activations/layer21_attention_weight_min": -26.98459243774414,
      "activations/layer22_attention_weight_max": 42.204872131347656,
      "activations/layer22_attention_weight_min": -35.386192321777344,
      "activations/layer23_attention_weight_max": 34.52180480957031,
      "activations/layer23_attention_weight_min": -28.47103500366211,
      "activations/layer2_attention_weight_max": 32.76906967163086,
      "activations/layer2_attention_weight_min": -32.79774475097656,
      "activations/layer3_attention_weight_max": 102.32949829101562,
      "activations/layer3_attention_weight_min": -105.77478790283203,
      "activations/layer4_attention_weight_max": 110.26546478271484,
      "activations/layer4_attention_weight_min": -107.95397186279297,
      "activations/layer5_attention_weight_max": 51.03951644897461,
      "activations/layer5_attention_weight_min": -61.287715911865234,
      "activations/layer6_attention_weight_max": 45.401248931884766,
      "activations/layer6_attention_weight_min": -46.7396354675293,
      "activations/layer7_attention_weight_max": 97.3517074584961,
      "activations/layer7_attention_weight_min": -95.52586364746094,
      "activations/layer8_attention_weight_max": 44.45478439331055,
      "activations/layer8_attention_weight_min": -47.75086212158203,
      "activations/layer9_attention_weight_max": 35.53795623779297,
      "activations/layer9_attention_weight_min": -37.860084533691406,
      "epoch": 17.85,
      "learning_rate": 3.5225378787878785e-05,
      "loss": 2.7423,
      "step": 307250
    },
    {
      "activations/layer0_attention_weight_max": 16.619098663330078,
      "activations/layer0_attention_weight_min": -14.471514701843262,
      "activations/layer10_attention_weight_max": 37.251522064208984,
      "activations/layer10_attention_weight_min": -36.529972076416016,
      "activations/layer11_attention_weight_max": 37.4002685546875,
      "activations/layer11_attention_weight_min": -36.89381408691406,
      "activations/layer12_attention_weight_max": 27.252708435058594,
      "activations/layer12_attention_weight_min": -26.8576717376709,
      "activations/layer13_attention_weight_max": 39.47364044189453,
      "activations/layer13_attention_weight_min": -35.734771728515625,
      "activations/layer14_attention_weight_max": 43.78780746459961,
      "activations/layer14_attention_weight_min": -38.24147033691406,
      "activations/layer15_attention_weight_max": 42.391178131103516,
      "activations/layer15_attention_weight_min": -36.1160888671875,
      "activations/layer16_attention_weight_max": 29.32465362548828,
      "activations/layer16_attention_weight_min": -29.850666046142578,
      "activations/layer17_attention_weight_max": 30.85000228881836,
      "activations/layer17_attention_weight_min": -26.467382431030273,
      "activations/layer18_attention_weight_max": 32.35985565185547,
      "activations/layer18_attention_weight_min": -27.657611846923828,
      "activations/layer19_attention_weight_max": 35.735774993896484,
      "activations/layer19_attention_weight_min": -32.21452331542969,
      "activations/layer1_attention_weight_max": 16.392213821411133,
      "activations/layer1_attention_weight_min": -14.344673156738281,
      "activations/layer20_attention_weight_max": 32.168827056884766,
      "activations/layer20_attention_weight_min": -26.072175979614258,
      "activations/layer21_attention_weight_max": 29.23077964782715,
      "activations/layer21_attention_weight_min": -24.94580078125,
      "activations/layer22_attention_weight_max": 47.06631088256836,
      "activations/layer22_attention_weight_min": -33.97666549682617,
      "activations/layer23_attention_weight_max": 34.93205261230469,
      "activations/layer23_attention_weight_min": -25.83241844177246,
      "activations/layer2_attention_weight_max": 31.913726806640625,
      "activations/layer2_attention_weight_min": -32.5525016784668,
      "activations/layer3_attention_weight_max": 100.1098403930664,
      "activations/layer3_attention_weight_min": -99.28942108154297,
      "activations/layer4_attention_weight_max": 105.5642318725586,
      "activations/layer4_attention_weight_min": -107.93131256103516,
      "activations/layer5_attention_weight_max": 50.45731735229492,
      "activations/layer5_attention_weight_min": -63.72294616699219,
      "activations/layer6_attention_weight_max": 44.32514953613281,
      "activations/layer6_attention_weight_min": -45.96005630493164,
      "activations/layer7_attention_weight_max": 93.97315979003906,
      "activations/layer7_attention_weight_min": -98.7306137084961,
      "activations/layer8_attention_weight_max": 42.95030975341797,
      "activations/layer8_attention_weight_min": -45.29045486450195,
      "activations/layer9_attention_weight_max": 33.123287200927734,
      "activations/layer9_attention_weight_min": -37.591182708740234,
      "epoch": 17.86,
      "learning_rate": 3.5206439393939386e-05,
      "loss": 2.7272,
      "step": 307300
    },
    {
      "activations/layer0_attention_weight_max": 15.853012084960938,
      "activations/layer0_attention_weight_min": -13.063908576965332,
      "activations/layer10_attention_weight_max": 34.002044677734375,
      "activations/layer10_attention_weight_min": -34.815521240234375,
      "activations/layer11_attention_weight_max": 34.36380386352539,
      "activations/layer11_attention_weight_min": -34.05305862426758,
      "activations/layer12_attention_weight_max": 28.166709899902344,
      "activations/layer12_attention_weight_min": -26.172399520874023,
      "activations/layer13_attention_weight_max": 38.08232498168945,
      "activations/layer13_attention_weight_min": -34.943016052246094,
      "activations/layer14_attention_weight_max": 42.27482223510742,
      "activations/layer14_attention_weight_min": -38.529563903808594,
      "activations/layer15_attention_weight_max": 38.10816955566406,
      "activations/layer15_attention_weight_min": -35.20083999633789,
      "activations/layer16_attention_weight_max": 32.344425201416016,
      "activations/layer16_attention_weight_min": -29.538000106811523,
      "activations/layer17_attention_weight_max": 33.14120101928711,
      "activations/layer17_attention_weight_min": -26.895708084106445,
      "activations/layer18_attention_weight_max": 32.248748779296875,
      "activations/layer18_attention_weight_min": -28.499618530273438,
      "activations/layer19_attention_weight_max": 34.61669921875,
      "activations/layer19_attention_weight_min": -34.655052185058594,
      "activations/layer1_attention_weight_max": 15.872420310974121,
      "activations/layer1_attention_weight_min": -13.841211318969727,
      "activations/layer20_attention_weight_max": 26.50189781188965,
      "activations/layer20_attention_weight_min": -27.852434158325195,
      "activations/layer21_attention_weight_max": 28.094280242919922,
      "activations/layer21_attention_weight_min": -25.75192642211914,
      "activations/layer22_attention_weight_max": 46.15998840332031,
      "activations/layer22_attention_weight_min": -31.48782730102539,
      "activations/layer23_attention_weight_max": 32.609642028808594,
      "activations/layer23_attention_weight_min": -26.79110336303711,
      "activations/layer2_attention_weight_max": 33.56122970581055,
      "activations/layer2_attention_weight_min": -34.201473236083984,
      "activations/layer3_attention_weight_max": 101.33135223388672,
      "activations/layer3_attention_weight_min": -103.39364624023438,
      "activations/layer4_attention_weight_max": 111.14522552490234,
      "activations/layer4_attention_weight_min": -113.92046356201172,
      "activations/layer5_attention_weight_max": 52.82422637939453,
      "activations/layer5_attention_weight_min": -69.65520477294922,
      "activations/layer6_attention_weight_max": 43.779457092285156,
      "activations/layer6_attention_weight_min": -46.495967864990234,
      "activations/layer7_attention_weight_max": 90.2229232788086,
      "activations/layer7_attention_weight_min": -97.60823059082031,
      "activations/layer8_attention_weight_max": 39.898590087890625,
      "activations/layer8_attention_weight_min": -42.04157638549805,
      "activations/layer9_attention_weight_max": 32.04159164428711,
      "activations/layer9_attention_weight_min": -33.312557220458984,
      "epoch": 17.86,
      "learning_rate": 3.5187499999999995e-05,
      "loss": 2.7313,
      "step": 307350
    },
    {
      "activations/layer0_attention_weight_max": 17.09398078918457,
      "activations/layer0_attention_weight_min": -13.93569564819336,
      "activations/layer10_attention_weight_max": 35.565040588378906,
      "activations/layer10_attention_weight_min": -36.47296905517578,
      "activations/layer11_attention_weight_max": 35.55171203613281,
      "activations/layer11_attention_weight_min": -34.86497497558594,
      "activations/layer12_attention_weight_max": 28.357908248901367,
      "activations/layer12_attention_weight_min": -28.099483489990234,
      "activations/layer13_attention_weight_max": 38.395103454589844,
      "activations/layer13_attention_weight_min": -38.29188919067383,
      "activations/layer14_attention_weight_max": 44.320953369140625,
      "activations/layer14_attention_weight_min": -40.51083755493164,
      "activations/layer15_attention_weight_max": 39.31352615356445,
      "activations/layer15_attention_weight_min": -37.14248275756836,
      "activations/layer16_attention_weight_max": 30.16510009765625,
      "activations/layer16_attention_weight_min": -28.322935104370117,
      "activations/layer17_attention_weight_max": 28.230552673339844,
      "activations/layer17_attention_weight_min": -26.876445770263672,
      "activations/layer18_attention_weight_max": 28.76298713684082,
      "activations/layer18_attention_weight_min": -25.9769229888916,
      "activations/layer19_attention_weight_max": 34.73299789428711,
      "activations/layer19_attention_weight_min": -34.49211120605469,
      "activations/layer1_attention_weight_max": 15.994524955749512,
      "activations/layer1_attention_weight_min": -15.237716674804688,
      "activations/layer20_attention_weight_max": 28.060457229614258,
      "activations/layer20_attention_weight_min": -27.81121826171875,
      "activations/layer21_attention_weight_max": 27.777006149291992,
      "activations/layer21_attention_weight_min": -25.140466690063477,
      "activations/layer22_attention_weight_max": 43.3657341003418,
      "activations/layer22_attention_weight_min": -34.49807357788086,
      "activations/layer23_attention_weight_max": 31.173126220703125,
      "activations/layer23_attention_weight_min": -28.165637969970703,
      "activations/layer2_attention_weight_max": 33.145748138427734,
      "activations/layer2_attention_weight_min": -33.2947883605957,
      "activations/layer3_attention_weight_max": 98.31414031982422,
      "activations/layer3_attention_weight_min": -99.8857192993164,
      "activations/layer4_attention_weight_max": 106.2302017211914,
      "activations/layer4_attention_weight_min": -114.52827453613281,
      "activations/layer5_attention_weight_max": 49.89386749267578,
      "activations/layer5_attention_weight_min": -59.954185485839844,
      "activations/layer6_attention_weight_max": 46.35698699951172,
      "activations/layer6_attention_weight_min": -50.360923767089844,
      "activations/layer7_attention_weight_max": 94.15736389160156,
      "activations/layer7_attention_weight_min": -102.87816619873047,
      "activations/layer8_attention_weight_max": 45.3161506652832,
      "activations/layer8_attention_weight_min": -47.829734802246094,
      "activations/layer9_attention_weight_max": 35.73219299316406,
      "activations/layer9_attention_weight_min": -35.297874450683594,
      "epoch": 17.86,
      "learning_rate": 3.51685606060606e-05,
      "loss": 2.7399,
      "step": 307400
    },
    {
      "activations/layer0_attention_weight_max": 16.47589683532715,
      "activations/layer0_attention_weight_min": -13.49608325958252,
      "activations/layer10_attention_weight_max": 33.663734436035156,
      "activations/layer10_attention_weight_min": -34.53289031982422,
      "activations/layer11_attention_weight_max": 34.589874267578125,
      "activations/layer11_attention_weight_min": -31.971057891845703,
      "activations/layer12_attention_weight_max": 26.955581665039062,
      "activations/layer12_attention_weight_min": -28.003787994384766,
      "activations/layer13_attention_weight_max": 39.624755859375,
      "activations/layer13_attention_weight_min": -36.86830520629883,
      "activations/layer14_attention_weight_max": 42.52842330932617,
      "activations/layer14_attention_weight_min": -40.081947326660156,
      "activations/layer15_attention_weight_max": 38.02103042602539,
      "activations/layer15_attention_weight_min": -35.78068161010742,
      "activations/layer16_attention_weight_max": 30.045385360717773,
      "activations/layer16_attention_weight_min": -28.554767608642578,
      "activations/layer17_attention_weight_max": 29.26576805114746,
      "activations/layer17_attention_weight_min": -25.50591278076172,
      "activations/layer18_attention_weight_max": 31.482397079467773,
      "activations/layer18_attention_weight_min": -25.981414794921875,
      "activations/layer19_attention_weight_max": 33.213401794433594,
      "activations/layer19_attention_weight_min": -30.81814956665039,
      "activations/layer1_attention_weight_max": 16.242189407348633,
      "activations/layer1_attention_weight_min": -14.455242156982422,
      "activations/layer20_attention_weight_max": 28.27197265625,
      "activations/layer20_attention_weight_min": -24.9685001373291,
      "activations/layer21_attention_weight_max": 28.779922485351562,
      "activations/layer21_attention_weight_min": -23.821971893310547,
      "activations/layer22_attention_weight_max": 41.132957458496094,
      "activations/layer22_attention_weight_min": -29.6256103515625,
      "activations/layer23_attention_weight_max": 33.69488525390625,
      "activations/layer23_attention_weight_min": -24.687551498413086,
      "activations/layer2_attention_weight_max": 31.999481201171875,
      "activations/layer2_attention_weight_min": -32.997032165527344,
      "activations/layer3_attention_weight_max": 94.38440704345703,
      "activations/layer3_attention_weight_min": -96.49213409423828,
      "activations/layer4_attention_weight_max": 103.10514068603516,
      "activations/layer4_attention_weight_min": -103.69908905029297,
      "activations/layer5_attention_weight_max": 46.94283676147461,
      "activations/layer5_attention_weight_min": -60.381866455078125,
      "activations/layer6_attention_weight_max": 42.525657653808594,
      "activations/layer6_attention_weight_min": -46.134639739990234,
      "activations/layer7_attention_weight_max": 95.8719253540039,
      "activations/layer7_attention_weight_min": -97.88081359863281,
      "activations/layer8_attention_weight_max": 43.15116500854492,
      "activations/layer8_attention_weight_min": -44.249786376953125,
      "activations/layer9_attention_weight_max": 32.6668815612793,
      "activations/layer9_attention_weight_min": -34.44375228881836,
      "epoch": 17.86,
      "learning_rate": 3.514962121212121e-05,
      "loss": 2.7245,
      "step": 307450
    },
    {
      "activations/layer0_attention_weight_max": 16.387298583984375,
      "activations/layer0_attention_weight_min": -14.475948333740234,
      "activations/layer10_attention_weight_max": 37.4842529296875,
      "activations/layer10_attention_weight_min": -36.08744812011719,
      "activations/layer11_attention_weight_max": 36.62905502319336,
      "activations/layer11_attention_weight_min": -36.43790817260742,
      "activations/layer12_attention_weight_max": 28.882055282592773,
      "activations/layer12_attention_weight_min": -28.49748992919922,
      "activations/layer13_attention_weight_max": 42.52944564819336,
      "activations/layer13_attention_weight_min": -37.79575729370117,
      "activations/layer14_attention_weight_max": 43.97093200683594,
      "activations/layer14_attention_weight_min": -40.81459426879883,
      "activations/layer15_attention_weight_max": 39.032958984375,
      "activations/layer15_attention_weight_min": -36.886531829833984,
      "activations/layer16_attention_weight_max": 30.64076805114746,
      "activations/layer16_attention_weight_min": -29.386709213256836,
      "activations/layer17_attention_weight_max": 32.79444122314453,
      "activations/layer17_attention_weight_min": -26.719459533691406,
      "activations/layer18_attention_weight_max": 36.43857192993164,
      "activations/layer18_attention_weight_min": -26.068416595458984,
      "activations/layer19_attention_weight_max": 37.78241729736328,
      "activations/layer19_attention_weight_min": -32.869388580322266,
      "activations/layer1_attention_weight_max": 18.294963836669922,
      "activations/layer1_attention_weight_min": -15.107213973999023,
      "activations/layer20_attention_weight_max": 32.3859977722168,
      "activations/layer20_attention_weight_min": -25.198820114135742,
      "activations/layer21_attention_weight_max": 31.12608528137207,
      "activations/layer21_attention_weight_min": -24.388187408447266,
      "activations/layer22_attention_weight_max": 47.09710693359375,
      "activations/layer22_attention_weight_min": -31.634458541870117,
      "activations/layer23_attention_weight_max": 35.860069274902344,
      "activations/layer23_attention_weight_min": -24.116580963134766,
      "activations/layer2_attention_weight_max": 33.10227966308594,
      "activations/layer2_attention_weight_min": -33.70873260498047,
      "activations/layer3_attention_weight_max": 99.74205780029297,
      "activations/layer3_attention_weight_min": -101.36778259277344,
      "activations/layer4_attention_weight_max": 104.19283294677734,
      "activations/layer4_attention_weight_min": -105.93763732910156,
      "activations/layer5_attention_weight_max": 49.21819305419922,
      "activations/layer5_attention_weight_min": -60.43091583251953,
      "activations/layer6_attention_weight_max": 47.06850051879883,
      "activations/layer6_attention_weight_min": -46.09428405761719,
      "activations/layer7_attention_weight_max": 97.93450164794922,
      "activations/layer7_attention_weight_min": -97.47933197021484,
      "activations/layer8_attention_weight_max": 45.9368782043457,
      "activations/layer8_attention_weight_min": -45.47825622558594,
      "activations/layer9_attention_weight_max": 34.231815338134766,
      "activations/layer9_attention_weight_min": -36.28079605102539,
      "epoch": 17.87,
      "learning_rate": 3.513068181818181e-05,
      "loss": 2.7307,
      "step": 307500
    },
    {
      "activations/layer0_attention_weight_max": 16.63078498840332,
      "activations/layer0_attention_weight_min": -13.612059593200684,
      "activations/layer10_attention_weight_max": 33.68950271606445,
      "activations/layer10_attention_weight_min": -35.028751373291016,
      "activations/layer11_attention_weight_max": 33.7684326171875,
      "activations/layer11_attention_weight_min": -35.984764099121094,
      "activations/layer12_attention_weight_max": 28.86111068725586,
      "activations/layer12_attention_weight_min": -28.985828399658203,
      "activations/layer13_attention_weight_max": 41.50801467895508,
      "activations/layer13_attention_weight_min": -41.165870666503906,
      "activations/layer14_attention_weight_max": 41.76063919067383,
      "activations/layer14_attention_weight_min": -41.742923736572266,
      "activations/layer15_attention_weight_max": 36.0914306640625,
      "activations/layer15_attention_weight_min": -37.14950180053711,
      "activations/layer16_attention_weight_max": 29.173112869262695,
      "activations/layer16_attention_weight_min": -30.59885597229004,
      "activations/layer17_attention_weight_max": 30.450408935546875,
      "activations/layer17_attention_weight_min": -26.414682388305664,
      "activations/layer18_attention_weight_max": 33.27099609375,
      "activations/layer18_attention_weight_min": -26.947458267211914,
      "activations/layer19_attention_weight_max": 32.28983688354492,
      "activations/layer19_attention_weight_min": -31.838682174682617,
      "activations/layer1_attention_weight_max": 15.807110786437988,
      "activations/layer1_attention_weight_min": -14.816993713378906,
      "activations/layer20_attention_weight_max": 29.61586570739746,
      "activations/layer20_attention_weight_min": -26.187847137451172,
      "activations/layer21_attention_weight_max": 30.048099517822266,
      "activations/layer21_attention_weight_min": -27.40533447265625,
      "activations/layer22_attention_weight_max": 44.587913513183594,
      "activations/layer22_attention_weight_min": -32.938011169433594,
      "activations/layer23_attention_weight_max": 32.43138885498047,
      "activations/layer23_attention_weight_min": -26.347806930541992,
      "activations/layer2_attention_weight_max": 33.28038787841797,
      "activations/layer2_attention_weight_min": -31.599576950073242,
      "activations/layer3_attention_weight_max": 99.54666900634766,
      "activations/layer3_attention_weight_min": -95.72937774658203,
      "activations/layer4_attention_weight_max": 106.06329345703125,
      "activations/layer4_attention_weight_min": -105.53588104248047,
      "activations/layer5_attention_weight_max": 52.709129333496094,
      "activations/layer5_attention_weight_min": -56.688133239746094,
      "activations/layer6_attention_weight_max": 46.65003204345703,
      "activations/layer6_attention_weight_min": -46.775306701660156,
      "activations/layer7_attention_weight_max": 97.40570831298828,
      "activations/layer7_attention_weight_min": -94.7722396850586,
      "activations/layer8_attention_weight_max": 42.68404006958008,
      "activations/layer8_attention_weight_min": -48.15062713623047,
      "activations/layer9_attention_weight_max": 32.005592346191406,
      "activations/layer9_attention_weight_min": -37.737972259521484,
      "epoch": 17.87,
      "learning_rate": 3.511174242424242e-05,
      "loss": 2.7592,
      "step": 307550
    },
    {
      "activations/layer0_attention_weight_max": 16.138769149780273,
      "activations/layer0_attention_weight_min": -13.528276443481445,
      "activations/layer10_attention_weight_max": 31.786483764648438,
      "activations/layer10_attention_weight_min": -32.69366455078125,
      "activations/layer11_attention_weight_max": 31.972820281982422,
      "activations/layer11_attention_weight_min": -31.791349411010742,
      "activations/layer12_attention_weight_max": 26.277042388916016,
      "activations/layer12_attention_weight_min": -25.170673370361328,
      "activations/layer13_attention_weight_max": 38.49284744262695,
      "activations/layer13_attention_weight_min": -34.930625915527344,
      "activations/layer14_attention_weight_max": 39.876888275146484,
      "activations/layer14_attention_weight_min": -35.14215850830078,
      "activations/layer15_attention_weight_max": 38.845279693603516,
      "activations/layer15_attention_weight_min": -36.01290512084961,
      "activations/layer16_attention_weight_max": 28.29768943786621,
      "activations/layer16_attention_weight_min": -27.53903579711914,
      "activations/layer17_attention_weight_max": 30.97096061706543,
      "activations/layer17_attention_weight_min": -26.386035919189453,
      "activations/layer18_attention_weight_max": 32.2220344543457,
      "activations/layer18_attention_weight_min": -26.979158401489258,
      "activations/layer19_attention_weight_max": 33.41183090209961,
      "activations/layer19_attention_weight_min": -33.83908462524414,
      "activations/layer1_attention_weight_max": 16.836774826049805,
      "activations/layer1_attention_weight_min": -13.687125205993652,
      "activations/layer20_attention_weight_max": 30.000505447387695,
      "activations/layer20_attention_weight_min": -29.17607879638672,
      "activations/layer21_attention_weight_max": 30.22441291809082,
      "activations/layer21_attention_weight_min": -27.443038940429688,
      "activations/layer22_attention_weight_max": 44.462467193603516,
      "activations/layer22_attention_weight_min": -35.327877044677734,
      "activations/layer23_attention_weight_max": 37.04888153076172,
      "activations/layer23_attention_weight_min": -27.75543785095215,
      "activations/layer2_attention_weight_max": 33.05773162841797,
      "activations/layer2_attention_weight_min": -34.23834991455078,
      "activations/layer3_attention_weight_max": 107.32100677490234,
      "activations/layer3_attention_weight_min": -102.77114868164062,
      "activations/layer4_attention_weight_max": 111.5650405883789,
      "activations/layer4_attention_weight_min": -111.90377044677734,
      "activations/layer5_attention_weight_max": 53.312252044677734,
      "activations/layer5_attention_weight_min": -64.05648803710938,
      "activations/layer6_attention_weight_max": 43.22248077392578,
      "activations/layer6_attention_weight_min": -47.57090377807617,
      "activations/layer7_attention_weight_max": 88.88029479980469,
      "activations/layer7_attention_weight_min": -96.44953155517578,
      "activations/layer8_attention_weight_max": 40.26753616333008,
      "activations/layer8_attention_weight_min": -41.629798889160156,
      "activations/layer9_attention_weight_max": 29.626657485961914,
      "activations/layer9_attention_weight_min": -31.704288482666016,
      "epoch": 17.87,
      "learning_rate": 3.509280303030302e-05,
      "loss": 2.738,
      "step": 307600
    },
    {
      "activations/layer0_attention_weight_max": 16.24772071838379,
      "activations/layer0_attention_weight_min": -13.193711280822754,
      "activations/layer10_attention_weight_max": 35.81582260131836,
      "activations/layer10_attention_weight_min": -36.16038513183594,
      "activations/layer11_attention_weight_max": 34.78363037109375,
      "activations/layer11_attention_weight_min": -34.61906433105469,
      "activations/layer12_attention_weight_max": 25.259742736816406,
      "activations/layer12_attention_weight_min": -27.31977653503418,
      "activations/layer13_attention_weight_max": 38.09477996826172,
      "activations/layer13_attention_weight_min": -35.59436798095703,
      "activations/layer14_attention_weight_max": 40.77402114868164,
      "activations/layer14_attention_weight_min": -35.78605651855469,
      "activations/layer15_attention_weight_max": 37.12236785888672,
      "activations/layer15_attention_weight_min": -34.34083938598633,
      "activations/layer16_attention_weight_max": 29.01556968688965,
      "activations/layer16_attention_weight_min": -29.508037567138672,
      "activations/layer17_attention_weight_max": 27.641572952270508,
      "activations/layer17_attention_weight_min": -26.838651657104492,
      "activations/layer18_attention_weight_max": 31.5738525390625,
      "activations/layer18_attention_weight_min": -26.38678550720215,
      "activations/layer19_attention_weight_max": 33.19884490966797,
      "activations/layer19_attention_weight_min": -32.27945327758789,
      "activations/layer1_attention_weight_max": 16.18947410583496,
      "activations/layer1_attention_weight_min": -15.587262153625488,
      "activations/layer20_attention_weight_max": 28.153762817382812,
      "activations/layer20_attention_weight_min": -25.417407989501953,
      "activations/layer21_attention_weight_max": 27.860843658447266,
      "activations/layer21_attention_weight_min": -26.003704071044922,
      "activations/layer22_attention_weight_max": 40.88911437988281,
      "activations/layer22_attention_weight_min": -31.538286209106445,
      "activations/layer23_attention_weight_max": 30.60778045654297,
      "activations/layer23_attention_weight_min": -25.316823959350586,
      "activations/layer2_attention_weight_max": 34.59462356567383,
      "activations/layer2_attention_weight_min": -33.10071563720703,
      "activations/layer3_attention_weight_max": 98.10001373291016,
      "activations/layer3_attention_weight_min": -103.9217529296875,
      "activations/layer4_attention_weight_max": 104.46414947509766,
      "activations/layer4_attention_weight_min": -106.01142883300781,
      "activations/layer5_attention_weight_max": 50.438255310058594,
      "activations/layer5_attention_weight_min": -60.60101318359375,
      "activations/layer6_attention_weight_max": 44.58994674682617,
      "activations/layer6_attention_weight_min": -47.037288665771484,
      "activations/layer7_attention_weight_max": 97.55573272705078,
      "activations/layer7_attention_weight_min": -99.1838607788086,
      "activations/layer8_attention_weight_max": 42.348690032958984,
      "activations/layer8_attention_weight_min": -43.8159065246582,
      "activations/layer9_attention_weight_max": 34.402339935302734,
      "activations/layer9_attention_weight_min": -36.16903305053711,
      "epoch": 17.88,
      "learning_rate": 3.507386363636363e-05,
      "loss": 2.7278,
      "step": 307650
    },
    {
      "activations/layer0_attention_weight_max": 16.316776275634766,
      "activations/layer0_attention_weight_min": -13.226627349853516,
      "activations/layer10_attention_weight_max": 35.9439811706543,
      "activations/layer10_attention_weight_min": -35.38367462158203,
      "activations/layer11_attention_weight_max": 35.73751449584961,
      "activations/layer11_attention_weight_min": -34.361629486083984,
      "activations/layer12_attention_weight_max": 25.695039749145508,
      "activations/layer12_attention_weight_min": -26.873859405517578,
      "activations/layer13_attention_weight_max": 38.35431671142578,
      "activations/layer13_attention_weight_min": -37.8148193359375,
      "activations/layer14_attention_weight_max": 42.049842834472656,
      "activations/layer14_attention_weight_min": -40.01789474487305,
      "activations/layer15_attention_weight_max": 35.52627944946289,
      "activations/layer15_attention_weight_min": -36.56991195678711,
      "activations/layer16_attention_weight_max": 28.356647491455078,
      "activations/layer16_attention_weight_min": -27.977689743041992,
      "activations/layer17_attention_weight_max": 32.85541534423828,
      "activations/layer17_attention_weight_min": -26.67976951599121,
      "activations/layer18_attention_weight_max": 35.17936706542969,
      "activations/layer18_attention_weight_min": -28.807668685913086,
      "activations/layer19_attention_weight_max": 33.4205207824707,
      "activations/layer19_attention_weight_min": -33.898380279541016,
      "activations/layer1_attention_weight_max": 17.245763778686523,
      "activations/layer1_attention_weight_min": -15.181230545043945,
      "activations/layer20_attention_weight_max": 29.34001922607422,
      "activations/layer20_attention_weight_min": -27.054553985595703,
      "activations/layer21_attention_weight_max": 28.933700561523438,
      "activations/layer21_attention_weight_min": -26.162107467651367,
      "activations/layer22_attention_weight_max": 48.42979431152344,
      "activations/layer22_attention_weight_min": -34.51539993286133,
      "activations/layer23_attention_weight_max": 36.07343673706055,
      "activations/layer23_attention_weight_min": -26.959373474121094,
      "activations/layer2_attention_weight_max": 35.27857971191406,
      "activations/layer2_attention_weight_min": -33.78318786621094,
      "activations/layer3_attention_weight_max": 97.79434204101562,
      "activations/layer3_attention_weight_min": -99.26652526855469,
      "activations/layer4_attention_weight_max": 105.5451431274414,
      "activations/layer4_attention_weight_min": -107.73397064208984,
      "activations/layer5_attention_weight_max": 48.71501159667969,
      "activations/layer5_attention_weight_min": -58.23577117919922,
      "activations/layer6_attention_weight_max": 44.23324966430664,
      "activations/layer6_attention_weight_min": -46.724586486816406,
      "activations/layer7_attention_weight_max": 91.8324966430664,
      "activations/layer7_attention_weight_min": -93.0397720336914,
      "activations/layer8_attention_weight_max": 45.04977035522461,
      "activations/layer8_attention_weight_min": -46.03902053833008,
      "activations/layer9_attention_weight_max": 34.52008056640625,
      "activations/layer9_attention_weight_min": -35.889617919921875,
      "epoch": 17.88,
      "learning_rate": 3.505492424242424e-05,
      "loss": 2.7356,
      "step": 307700
    },
    {
      "activations/layer0_attention_weight_max": 16.765518188476562,
      "activations/layer0_attention_weight_min": -13.462347030639648,
      "activations/layer10_attention_weight_max": 34.526283264160156,
      "activations/layer10_attention_weight_min": -35.08436965942383,
      "activations/layer11_attention_weight_max": 35.21257019042969,
      "activations/layer11_attention_weight_min": -34.47434616088867,
      "activations/layer12_attention_weight_max": 27.58379554748535,
      "activations/layer12_attention_weight_min": -27.825016021728516,
      "activations/layer13_attention_weight_max": 38.79719161987305,
      "activations/layer13_attention_weight_min": -37.29059600830078,
      "activations/layer14_attention_weight_max": 43.78945541381836,
      "activations/layer14_attention_weight_min": -38.38166427612305,
      "activations/layer15_attention_weight_max": 37.66838836669922,
      "activations/layer15_attention_weight_min": -37.461524963378906,
      "activations/layer16_attention_weight_max": 27.787214279174805,
      "activations/layer16_attention_weight_min": -27.409116744995117,
      "activations/layer17_attention_weight_max": 29.45616340637207,
      "activations/layer17_attention_weight_min": -26.00066566467285,
      "activations/layer18_attention_weight_max": 31.73040771484375,
      "activations/layer18_attention_weight_min": -26.846473693847656,
      "activations/layer19_attention_weight_max": 33.49195861816406,
      "activations/layer19_attention_weight_min": -31.987512588500977,
      "activations/layer1_attention_weight_max": 16.442256927490234,
      "activations/layer1_attention_weight_min": -14.410240173339844,
      "activations/layer20_attention_weight_max": 29.8531436920166,
      "activations/layer20_attention_weight_min": -26.497591018676758,
      "activations/layer21_attention_weight_max": 31.281272888183594,
      "activations/layer21_attention_weight_min": -25.04532241821289,
      "activations/layer22_attention_weight_max": 38.59225082397461,
      "activations/layer22_attention_weight_min": -29.662248611450195,
      "activations/layer23_attention_weight_max": 30.100364685058594,
      "activations/layer23_attention_weight_min": -26.618152618408203,
      "activations/layer2_attention_weight_max": 34.82964324951172,
      "activations/layer2_attention_weight_min": -32.48065948486328,
      "activations/layer3_attention_weight_max": 100.58399200439453,
      "activations/layer3_attention_weight_min": -98.74129486083984,
      "activations/layer4_attention_weight_max": 109.86849212646484,
      "activations/layer4_attention_weight_min": -110.1415023803711,
      "activations/layer5_attention_weight_max": 49.547969818115234,
      "activations/layer5_attention_weight_min": -58.06232452392578,
      "activations/layer6_attention_weight_max": 44.55660629272461,
      "activations/layer6_attention_weight_min": -48.96947479248047,
      "activations/layer7_attention_weight_max": 94.83100128173828,
      "activations/layer7_attention_weight_min": -98.66072845458984,
      "activations/layer8_attention_weight_max": 42.064449310302734,
      "activations/layer8_attention_weight_min": -46.05636978149414,
      "activations/layer9_attention_weight_max": 31.736242294311523,
      "activations/layer9_attention_weight_min": -34.26359176635742,
      "epoch": 17.88,
      "learning_rate": 3.503598484848485e-05,
      "loss": 2.739,
      "step": 307750
    },
    {
      "activations/layer0_attention_weight_max": 16.26380157470703,
      "activations/layer0_attention_weight_min": -12.957924842834473,
      "activations/layer10_attention_weight_max": 35.861419677734375,
      "activations/layer10_attention_weight_min": -34.52388381958008,
      "activations/layer11_attention_weight_max": 36.470157623291016,
      "activations/layer11_attention_weight_min": -35.47304153442383,
      "activations/layer12_attention_weight_max": 27.34770965576172,
      "activations/layer12_attention_weight_min": -27.1185359954834,
      "activations/layer13_attention_weight_max": 38.35261535644531,
      "activations/layer13_attention_weight_min": -37.17689514160156,
      "activations/layer14_attention_weight_max": 41.62683868408203,
      "activations/layer14_attention_weight_min": -41.648292541503906,
      "activations/layer15_attention_weight_max": 37.629093170166016,
      "activations/layer15_attention_weight_min": -38.88555908203125,
      "activations/layer16_attention_weight_max": 28.174943923950195,
      "activations/layer16_attention_weight_min": -28.584760665893555,
      "activations/layer17_attention_weight_max": 29.05638313293457,
      "activations/layer17_attention_weight_min": -25.120576858520508,
      "activations/layer18_attention_weight_max": 31.14968490600586,
      "activations/layer18_attention_weight_min": -27.709125518798828,
      "activations/layer19_attention_weight_max": 35.115596771240234,
      "activations/layer19_attention_weight_min": -31.142194747924805,
      "activations/layer1_attention_weight_max": 16.116701126098633,
      "activations/layer1_attention_weight_min": -15.157533645629883,
      "activations/layer20_attention_weight_max": 27.98137664794922,
      "activations/layer20_attention_weight_min": -25.7292423248291,
      "activations/layer21_attention_weight_max": 28.08139991760254,
      "activations/layer21_attention_weight_min": -24.312259674072266,
      "activations/layer22_attention_weight_max": 41.50832748413086,
      "activations/layer22_attention_weight_min": -33.2120361328125,
      "activations/layer23_attention_weight_max": 30.86396598815918,
      "activations/layer23_attention_weight_min": -29.46150016784668,
      "activations/layer2_attention_weight_max": 32.53571319580078,
      "activations/layer2_attention_weight_min": -32.7739143371582,
      "activations/layer3_attention_weight_max": 97.97782897949219,
      "activations/layer3_attention_weight_min": -99.06685638427734,
      "activations/layer4_attention_weight_max": 106.42000579833984,
      "activations/layer4_attention_weight_min": -109.16310119628906,
      "activations/layer5_attention_weight_max": 50.5307731628418,
      "activations/layer5_attention_weight_min": -60.95472717285156,
      "activations/layer6_attention_weight_max": 43.90852737426758,
      "activations/layer6_attention_weight_min": -47.58229064941406,
      "activations/layer7_attention_weight_max": 96.29658508300781,
      "activations/layer7_attention_weight_min": -94.81218719482422,
      "activations/layer8_attention_weight_max": 42.295623779296875,
      "activations/layer8_attention_weight_min": -43.98933792114258,
      "activations/layer9_attention_weight_max": 35.66348648071289,
      "activations/layer9_attention_weight_min": -33.63187026977539,
      "epoch": 17.88,
      "learning_rate": 3.5017045454545456e-05,
      "loss": 2.7444,
      "step": 307800
    },
    {
      "activations/layer0_attention_weight_max": 16.027301788330078,
      "activations/layer0_attention_weight_min": -14.77885627746582,
      "activations/layer10_attention_weight_max": 31.15797996520996,
      "activations/layer10_attention_weight_min": -35.467308044433594,
      "activations/layer11_attention_weight_max": 31.0238094329834,
      "activations/layer11_attention_weight_min": -32.74907302856445,
      "activations/layer12_attention_weight_max": 28.77640724182129,
      "activations/layer12_attention_weight_min": -28.211227416992188,
      "activations/layer13_attention_weight_max": 41.790401458740234,
      "activations/layer13_attention_weight_min": -37.97990417480469,
      "activations/layer14_attention_weight_max": 43.21435546875,
      "activations/layer14_attention_weight_min": -40.02980041503906,
      "activations/layer15_attention_weight_max": 39.06941604614258,
      "activations/layer15_attention_weight_min": -37.379093170166016,
      "activations/layer16_attention_weight_max": 28.31369972229004,
      "activations/layer16_attention_weight_min": -29.228710174560547,
      "activations/layer17_attention_weight_max": 28.950830459594727,
      "activations/layer17_attention_weight_min": -27.7576961517334,
      "activations/layer18_attention_weight_max": 34.97484588623047,
      "activations/layer18_attention_weight_min": -27.648433685302734,
      "activations/layer19_attention_weight_max": 40.28643798828125,
      "activations/layer19_attention_weight_min": -32.81163787841797,
      "activations/layer1_attention_weight_max": 19.1107120513916,
      "activations/layer1_attention_weight_min": -15.36444091796875,
      "activations/layer20_attention_weight_max": 36.61143493652344,
      "activations/layer20_attention_weight_min": -27.31468391418457,
      "activations/layer21_attention_weight_max": 31.822912216186523,
      "activations/layer21_attention_weight_min": -26.22246551513672,
      "activations/layer22_attention_weight_max": 42.052940368652344,
      "activations/layer22_attention_weight_min": -32.8485107421875,
      "activations/layer23_attention_weight_max": 34.54850769042969,
      "activations/layer23_attention_weight_min": -28.470218658447266,
      "activations/layer2_attention_weight_max": 34.118003845214844,
      "activations/layer2_attention_weight_min": -31.175466537475586,
      "activations/layer3_attention_weight_max": 102.374267578125,
      "activations/layer3_attention_weight_min": -98.12492370605469,
      "activations/layer4_attention_weight_max": 107.12001037597656,
      "activations/layer4_attention_weight_min": -104.82537078857422,
      "activations/layer5_attention_weight_max": 50.6834831237793,
      "activations/layer5_attention_weight_min": -59.015174865722656,
      "activations/layer6_attention_weight_max": 44.40479278564453,
      "activations/layer6_attention_weight_min": -45.106224060058594,
      "activations/layer7_attention_weight_max": 90.56272888183594,
      "activations/layer7_attention_weight_min": -101.74722290039062,
      "activations/layer8_attention_weight_max": 39.18019485473633,
      "activations/layer8_attention_weight_min": -43.16679000854492,
      "activations/layer9_attention_weight_max": 30.410734176635742,
      "activations/layer9_attention_weight_min": -36.69822311401367,
      "epoch": 17.89,
      "learning_rate": 3.499810606060606e-05,
      "loss": 2.7349,
      "step": 307850
    },
    {
      "activations/layer0_attention_weight_max": 15.934402465820312,
      "activations/layer0_attention_weight_min": -12.819726943969727,
      "activations/layer10_attention_weight_max": 32.61204147338867,
      "activations/layer10_attention_weight_min": -34.11175537109375,
      "activations/layer11_attention_weight_max": 35.40473175048828,
      "activations/layer11_attention_weight_min": -34.429649353027344,
      "activations/layer12_attention_weight_max": 28.86083221435547,
      "activations/layer12_attention_weight_min": -28.49382781982422,
      "activations/layer13_attention_weight_max": 43.24824905395508,
      "activations/layer13_attention_weight_min": -39.05862045288086,
      "activations/layer14_attention_weight_max": 50.0072021484375,
      "activations/layer14_attention_weight_min": -39.98100280761719,
      "activations/layer15_attention_weight_max": 46.64912033081055,
      "activations/layer15_attention_weight_min": -39.91999053955078,
      "activations/layer16_attention_weight_max": 33.6879997253418,
      "activations/layer16_attention_weight_min": -30.68100929260254,
      "activations/layer17_attention_weight_max": 32.6124382019043,
      "activations/layer17_attention_weight_min": -28.579843521118164,
      "activations/layer18_attention_weight_max": 35.96015548706055,
      "activations/layer18_attention_weight_min": -28.782079696655273,
      "activations/layer19_attention_weight_max": 36.213565826416016,
      "activations/layer19_attention_weight_min": -32.953121185302734,
      "activations/layer1_attention_weight_max": 17.901304244995117,
      "activations/layer1_attention_weight_min": -16.020906448364258,
      "activations/layer20_attention_weight_max": 29.21050262451172,
      "activations/layer20_attention_weight_min": -26.60707664489746,
      "activations/layer21_attention_weight_max": 28.832265853881836,
      "activations/layer21_attention_weight_min": -26.171415328979492,
      "activations/layer22_attention_weight_max": 48.70853805541992,
      "activations/layer22_attention_weight_min": -35.204036712646484,
      "activations/layer23_attention_weight_max": 37.19955062866211,
      "activations/layer23_attention_weight_min": -27.20100975036621,
      "activations/layer2_attention_weight_max": 34.338096618652344,
      "activations/layer2_attention_weight_min": -31.728519439697266,
      "activations/layer3_attention_weight_max": 98.74858856201172,
      "activations/layer3_attention_weight_min": -100.21282196044922,
      "activations/layer4_attention_weight_max": 104.2271499633789,
      "activations/layer4_attention_weight_min": -107.18951416015625,
      "activations/layer5_attention_weight_max": 50.14018249511719,
      "activations/layer5_attention_weight_min": -62.67378234863281,
      "activations/layer6_attention_weight_max": 42.929908752441406,
      "activations/layer6_attention_weight_min": -46.43635177612305,
      "activations/layer7_attention_weight_max": 93.01740264892578,
      "activations/layer7_attention_weight_min": -94.79566192626953,
      "activations/layer8_attention_weight_max": 42.943206787109375,
      "activations/layer8_attention_weight_min": -43.89485549926758,
      "activations/layer9_attention_weight_max": 33.071598052978516,
      "activations/layer9_attention_weight_min": -34.050575256347656,
      "epoch": 17.89,
      "learning_rate": 3.4979166666666665e-05,
      "loss": 2.7264,
      "step": 307900
    },
    {
      "activations/layer0_attention_weight_max": 16.148401260375977,
      "activations/layer0_attention_weight_min": -13.55770492553711,
      "activations/layer10_attention_weight_max": 33.16074752807617,
      "activations/layer10_attention_weight_min": -33.92345428466797,
      "activations/layer11_attention_weight_max": 34.29652404785156,
      "activations/layer11_attention_weight_min": -33.64922332763672,
      "activations/layer12_attention_weight_max": 25.760347366333008,
      "activations/layer12_attention_weight_min": -27.397470474243164,
      "activations/layer13_attention_weight_max": 37.32228469848633,
      "activations/layer13_attention_weight_min": -34.411949157714844,
      "activations/layer14_attention_weight_max": 40.66790008544922,
      "activations/layer14_attention_weight_min": -37.770259857177734,
      "activations/layer15_attention_weight_max": 37.33803176879883,
      "activations/layer15_attention_weight_min": -36.2541389465332,
      "activations/layer16_attention_weight_max": 29.926403045654297,
      "activations/layer16_attention_weight_min": -30.55530548095703,
      "activations/layer17_attention_weight_max": 29.16162109375,
      "activations/layer17_attention_weight_min": -27.767650604248047,
      "activations/layer18_attention_weight_max": 31.860275268554688,
      "activations/layer18_attention_weight_min": -29.65850830078125,
      "activations/layer19_attention_weight_max": 32.76459884643555,
      "activations/layer19_attention_weight_min": -31.303518295288086,
      "activations/layer1_attention_weight_max": 16.222335815429688,
      "activations/layer1_attention_weight_min": -14.374512672424316,
      "activations/layer20_attention_weight_max": 29.920610427856445,
      "activations/layer20_attention_weight_min": -26.533214569091797,
      "activations/layer21_attention_weight_max": 29.52761459350586,
      "activations/layer21_attention_weight_min": -25.4477481842041,
      "activations/layer22_attention_weight_max": 39.523555755615234,
      "activations/layer22_attention_weight_min": -33.90092468261719,
      "activations/layer23_attention_weight_max": 31.000289916992188,
      "activations/layer23_attention_weight_min": -27.807682037353516,
      "activations/layer2_attention_weight_max": 31.594974517822266,
      "activations/layer2_attention_weight_min": -32.241695404052734,
      "activations/layer3_attention_weight_max": 98.50391387939453,
      "activations/layer3_attention_weight_min": -97.4351806640625,
      "activations/layer4_attention_weight_max": 106.5400390625,
      "activations/layer4_attention_weight_min": -108.16499328613281,
      "activations/layer5_attention_weight_max": 47.591121673583984,
      "activations/layer5_attention_weight_min": -58.68226623535156,
      "activations/layer6_attention_weight_max": 43.29352951049805,
      "activations/layer6_attention_weight_min": -45.354915618896484,
      "activations/layer7_attention_weight_max": 92.63352966308594,
      "activations/layer7_attention_weight_min": -90.97633361816406,
      "activations/layer8_attention_weight_max": 43.19905471801758,
      "activations/layer8_attention_weight_min": -43.26243209838867,
      "activations/layer9_attention_weight_max": 33.29092788696289,
      "activations/layer9_attention_weight_min": -34.03200149536133,
      "epoch": 17.89,
      "learning_rate": 3.496022727272727e-05,
      "loss": 2.7416,
      "step": 307950
    },
    {
      "activations/layer0_attention_weight_max": 15.259275436401367,
      "activations/layer0_attention_weight_min": -13.025894165039062,
      "activations/layer10_attention_weight_max": 33.81647872924805,
      "activations/layer10_attention_weight_min": -38.46145248413086,
      "activations/layer11_attention_weight_max": 34.64363098144531,
      "activations/layer11_attention_weight_min": -38.52483367919922,
      "activations/layer12_attention_weight_max": 27.87071990966797,
      "activations/layer12_attention_weight_min": -28.69188117980957,
      "activations/layer13_attention_weight_max": 43.69704055786133,
      "activations/layer13_attention_weight_min": -37.08262634277344,
      "activations/layer14_attention_weight_max": 45.15475845336914,
      "activations/layer14_attention_weight_min": -42.997196197509766,
      "activations/layer15_attention_weight_max": 42.38164138793945,
      "activations/layer15_attention_weight_min": -38.88261795043945,
      "activations/layer16_attention_weight_max": 29.54705238342285,
      "activations/layer16_attention_weight_min": -29.746904373168945,
      "activations/layer17_attention_weight_max": 31.3223876953125,
      "activations/layer17_attention_weight_min": -27.233423233032227,
      "activations/layer18_attention_weight_max": 30.902816772460938,
      "activations/layer18_attention_weight_min": -26.43840217590332,
      "activations/layer19_attention_weight_max": 34.62702178955078,
      "activations/layer19_attention_weight_min": -33.37624740600586,
      "activations/layer1_attention_weight_max": 15.071319580078125,
      "activations/layer1_attention_weight_min": -13.507197380065918,
      "activations/layer20_attention_weight_max": 30.330753326416016,
      "activations/layer20_attention_weight_min": -25.959569931030273,
      "activations/layer21_attention_weight_max": 29.31463623046875,
      "activations/layer21_attention_weight_min": -26.363567352294922,
      "activations/layer22_attention_weight_max": 49.43507385253906,
      "activations/layer22_attention_weight_min": -34.51231002807617,
      "activations/layer23_attention_weight_max": 33.19709014892578,
      "activations/layer23_attention_weight_min": -27.086071014404297,
      "activations/layer2_attention_weight_max": 33.465240478515625,
      "activations/layer2_attention_weight_min": -32.68336486816406,
      "activations/layer3_attention_weight_max": 102.54252624511719,
      "activations/layer3_attention_weight_min": -103.40213775634766,
      "activations/layer4_attention_weight_max": 107.63834381103516,
      "activations/layer4_attention_weight_min": -112.85383605957031,
      "activations/layer5_attention_weight_max": 52.354248046875,
      "activations/layer5_attention_weight_min": -59.348976135253906,
      "activations/layer6_attention_weight_max": 44.11217498779297,
      "activations/layer6_attention_weight_min": -48.08698272705078,
      "activations/layer7_attention_weight_max": 91.64897155761719,
      "activations/layer7_attention_weight_min": -94.39193725585938,
      "activations/layer8_attention_weight_max": 41.38871765136719,
      "activations/layer8_attention_weight_min": -45.51729202270508,
      "activations/layer9_attention_weight_max": 32.76248550415039,
      "activations/layer9_attention_weight_min": -36.74333572387695,
      "epoch": 17.9,
      "learning_rate": 3.4941287878787875e-05,
      "loss": 2.7481,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4677,
      "eval_samples_per_second": 507.106,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4677,
      "eval_openwebtext_samples_per_second": 507.106,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9677,
      "eval_wikitext_samples_per_second": 231.743,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_lambada_loss": 2.43359375,
      "eval_lambada_ppl": 11.399776500625777,
      "eval_lambada_runtime": 9.5127,
      "eval_lambada_samples_per_second": 511.84,
      "step": 308000
    },
    {
      "activations/layer0_attention_weight_max": 16.121150970458984,
      "activations/layer0_attention_weight_min": -12.870016098022461,
      "activations/layer10_attention_weight_max": 34.35316467285156,
      "activations/layer10_attention_weight_min": -35.562496185302734,
      "activations/layer11_attention_weight_max": 37.25740051269531,
      "activations/layer11_attention_weight_min": -35.320457458496094,
      "activations/layer12_attention_weight_max": 28.467138290405273,
      "activations/layer12_attention_weight_min": -27.458337783813477,
      "activations/layer13_attention_weight_max": 41.360286712646484,
      "activations/layer13_attention_weight_min": -39.4409294128418,
      "activations/layer14_attention_weight_max": 44.85990905761719,
      "activations/layer14_attention_weight_min": -41.65778350830078,
      "activations/layer15_attention_weight_max": 45.35238265991211,
      "activations/layer15_attention_weight_min": -38.52641677856445,
      "activations/layer16_attention_weight_max": 30.319351196289062,
      "activations/layer16_attention_weight_min": -28.589426040649414,
      "activations/layer17_attention_weight_max": 31.64642333984375,
      "activations/layer17_attention_weight_min": -25.80443000793457,
      "activations/layer18_attention_weight_max": 31.213825225830078,
      "activations/layer18_attention_weight_min": -27.277217864990234,
      "activations/layer19_attention_weight_max": 37.64884567260742,
      "activations/layer19_attention_weight_min": -33.13582229614258,
      "activations/layer1_attention_weight_max": 16.028560638427734,
      "activations/layer1_attention_weight_min": -14.91435432434082,
      "activations/layer20_attention_weight_max": 30.4295597076416,
      "activations/layer20_attention_weight_min": -25.658517837524414,
      "activations/layer21_attention_weight_max": 30.832015991210938,
      "activations/layer21_attention_weight_min": -23.967119216918945,
      "activations/layer22_attention_weight_max": 44.30860137939453,
      "activations/layer22_attention_weight_min": -31.735980987548828,
      "activations/layer23_attention_weight_max": 32.092899322509766,
      "activations/layer23_attention_weight_min": -25.39752769470215,
      "activations/layer2_attention_weight_max": 32.564659118652344,
      "activations/layer2_attention_weight_min": -33.59395980834961,
      "activations/layer3_attention_weight_max": 103.85694122314453,
      "activations/layer3_attention_weight_min": -103.87904357910156,
      "activations/layer4_attention_weight_max": 110.36006927490234,
      "activations/layer4_attention_weight_min": -113.64349365234375,
      "activations/layer5_attention_weight_max": 51.22313690185547,
      "activations/layer5_attention_weight_min": -64.42245483398438,
      "activations/layer6_attention_weight_max": 46.65625762939453,
      "activations/layer6_attention_weight_min": -48.26201248168945,
      "activations/layer7_attention_weight_max": 92.61237335205078,
      "activations/layer7_attention_weight_min": -95.69914245605469,
      "activations/layer8_attention_weight_max": 42.6944465637207,
      "activations/layer8_attention_weight_min": -44.337005615234375,
      "activations/layer9_attention_weight_max": 32.730201721191406,
      "activations/layer9_attention_weight_min": -36.10856246948242,
      "epoch": 17.9,
      "learning_rate": 3.492234848484848e-05,
      "loss": 2.7227,
      "step": 308050
    },
    {
      "activations/layer0_attention_weight_max": 16.38636016845703,
      "activations/layer0_attention_weight_min": -13.228553771972656,
      "activations/layer10_attention_weight_max": 33.34444046020508,
      "activations/layer10_attention_weight_min": -33.502811431884766,
      "activations/layer11_attention_weight_max": 33.76285934448242,
      "activations/layer11_attention_weight_min": -32.96064758300781,
      "activations/layer12_attention_weight_max": 28.653011322021484,
      "activations/layer12_attention_weight_min": -26.97541046142578,
      "activations/layer13_attention_weight_max": 39.45741271972656,
      "activations/layer13_attention_weight_min": -36.78148651123047,
      "activations/layer14_attention_weight_max": 44.59005355834961,
      "activations/layer14_attention_weight_min": -38.606075286865234,
      "activations/layer15_attention_weight_max": 40.87681198120117,
      "activations/layer15_attention_weight_min": -35.982723236083984,
      "activations/layer16_attention_weight_max": 30.298681259155273,
      "activations/layer16_attention_weight_min": -29.414358139038086,
      "activations/layer17_attention_weight_max": 31.38398551940918,
      "activations/layer17_attention_weight_min": -27.414167404174805,
      "activations/layer18_attention_weight_max": 33.06989669799805,
      "activations/layer18_attention_weight_min": -28.002033233642578,
      "activations/layer19_attention_weight_max": 36.740760803222656,
      "activations/layer19_attention_weight_min": -35.948673248291016,
      "activations/layer1_attention_weight_max": 17.78935432434082,
      "activations/layer1_attention_weight_min": -15.427986145019531,
      "activations/layer20_attention_weight_max": 29.143815994262695,
      "activations/layer20_attention_weight_min": -26.5655460357666,
      "activations/layer21_attention_weight_max": 28.83695411682129,
      "activations/layer21_attention_weight_min": -27.18206024169922,
      "activations/layer22_attention_weight_max": 45.262596130371094,
      "activations/layer22_attention_weight_min": -34.23212432861328,
      "activations/layer23_attention_weight_max": 35.99339294433594,
      "activations/layer23_attention_weight_min": -28.15638542175293,
      "activations/layer2_attention_weight_max": 33.56380844116211,
      "activations/layer2_attention_weight_min": -33.42650604248047,
      "activations/layer3_attention_weight_max": 99.24967193603516,
      "activations/layer3_attention_weight_min": -101.03886413574219,
      "activations/layer4_attention_weight_max": 107.5521011352539,
      "activations/layer4_attention_weight_min": -110.21671295166016,
      "activations/layer5_attention_weight_max": 50.56636047363281,
      "activations/layer5_attention_weight_min": -61.111812591552734,
      "activations/layer6_attention_weight_max": 45.48876190185547,
      "activations/layer6_attention_weight_min": -45.95602798461914,
      "activations/layer7_attention_weight_max": 93.22927856445312,
      "activations/layer7_attention_weight_min": -97.38715362548828,
      "activations/layer8_attention_weight_max": 41.5006217956543,
      "activations/layer8_attention_weight_min": -43.958595275878906,
      "activations/layer9_attention_weight_max": 31.926340103149414,
      "activations/layer9_attention_weight_min": -32.74653244018555,
      "epoch": 17.9,
      "learning_rate": 3.490340909090909e-05,
      "loss": 2.7346,
      "step": 308100
    },
    {
      "activations/layer0_attention_weight_max": 16.498661041259766,
      "activations/layer0_attention_weight_min": -12.751608848571777,
      "activations/layer10_attention_weight_max": 32.52544403076172,
      "activations/layer10_attention_weight_min": -34.59886169433594,
      "activations/layer11_attention_weight_max": 29.06372833251953,
      "activations/layer11_attention_weight_min": -32.03545379638672,
      "activations/layer12_attention_weight_max": 25.21600341796875,
      "activations/layer12_attention_weight_min": -24.36284637451172,
      "activations/layer13_attention_weight_max": 36.396724700927734,
      "activations/layer13_attention_weight_min": -33.546478271484375,
      "activations/layer14_attention_weight_max": 43.75236892700195,
      "activations/layer14_attention_weight_min": -39.45576095581055,
      "activations/layer15_attention_weight_max": 39.64410400390625,
      "activations/layer15_attention_weight_min": -36.123966217041016,
      "activations/layer16_attention_weight_max": 29.319272994995117,
      "activations/layer16_attention_weight_min": -28.04086685180664,
      "activations/layer17_attention_weight_max": 29.497549057006836,
      "activations/layer17_attention_weight_min": -25.88642120361328,
      "activations/layer18_attention_weight_max": 31.215246200561523,
      "activations/layer18_attention_weight_min": -28.036128997802734,
      "activations/layer19_attention_weight_max": 32.22507858276367,
      "activations/layer19_attention_weight_min": -34.14651107788086,
      "activations/layer1_attention_weight_max": 15.558712005615234,
      "activations/layer1_attention_weight_min": -14.23261547088623,
      "activations/layer20_attention_weight_max": 27.57295799255371,
      "activations/layer20_attention_weight_min": -25.81834602355957,
      "activations/layer21_attention_weight_max": 27.710220336914062,
      "activations/layer21_attention_weight_min": -26.64683723449707,
      "activations/layer22_attention_weight_max": 38.186302185058594,
      "activations/layer22_attention_weight_min": -35.528350830078125,
      "activations/layer23_attention_weight_max": 32.13983917236328,
      "activations/layer23_attention_weight_min": -30.742225646972656,
      "activations/layer2_attention_weight_max": 32.0708122253418,
      "activations/layer2_attention_weight_min": -30.947566986083984,
      "activations/layer3_attention_weight_max": 95.64722442626953,
      "activations/layer3_attention_weight_min": -96.09967803955078,
      "activations/layer4_attention_weight_max": 99.54324340820312,
      "activations/layer4_attention_weight_min": -109.93656158447266,
      "activations/layer5_attention_weight_max": 46.540496826171875,
      "activations/layer5_attention_weight_min": -57.915321350097656,
      "activations/layer6_attention_weight_max": 40.943572998046875,
      "activations/layer6_attention_weight_min": -45.6239128112793,
      "activations/layer7_attention_weight_max": 86.50814819335938,
      "activations/layer7_attention_weight_min": -97.52957916259766,
      "activations/layer8_attention_weight_max": 41.42938995361328,
      "activations/layer8_attention_weight_min": -42.19245147705078,
      "activations/layer9_attention_weight_max": 30.28378677368164,
      "activations/layer9_attention_weight_min": -32.53716278076172,
      "epoch": 17.91,
      "learning_rate": 3.488446969696969e-05,
      "loss": 2.7391,
      "step": 308150
    },
    {
      "activations/layer0_attention_weight_max": 15.999670028686523,
      "activations/layer0_attention_weight_min": -12.743021011352539,
      "activations/layer10_attention_weight_max": 37.01641845703125,
      "activations/layer10_attention_weight_min": -36.259376525878906,
      "activations/layer11_attention_weight_max": 36.03608703613281,
      "activations/layer11_attention_weight_min": -36.320098876953125,
      "activations/layer12_attention_weight_max": 28.232267379760742,
      "activations/layer12_attention_weight_min": -28.00418472290039,
      "activations/layer13_attention_weight_max": 38.9319953918457,
      "activations/layer13_attention_weight_min": -37.3078727722168,
      "activations/layer14_attention_weight_max": 42.97398376464844,
      "activations/layer14_attention_weight_min": -39.897674560546875,
      "activations/layer15_attention_weight_max": 41.196380615234375,
      "activations/layer15_attention_weight_min": -36.395992279052734,
      "activations/layer16_attention_weight_max": 29.764034271240234,
      "activations/layer16_attention_weight_min": -26.661943435668945,
      "activations/layer17_attention_weight_max": 28.87449836730957,
      "activations/layer17_attention_weight_min": -26.91048240661621,
      "activations/layer18_attention_weight_max": 30.733144760131836,
      "activations/layer18_attention_weight_min": -25.001605987548828,
      "activations/layer19_attention_weight_max": 36.13694381713867,
      "activations/layer19_attention_weight_min": -30.225208282470703,
      "activations/layer1_attention_weight_max": 16.84128761291504,
      "activations/layer1_attention_weight_min": -14.063236236572266,
      "activations/layer20_attention_weight_max": 29.401071548461914,
      "activations/layer20_attention_weight_min": -26.418821334838867,
      "activations/layer21_attention_weight_max": 26.849485397338867,
      "activations/layer21_attention_weight_min": -26.211233139038086,
      "activations/layer22_attention_weight_max": 42.53343963623047,
      "activations/layer22_attention_weight_min": -30.775285720825195,
      "activations/layer23_attention_weight_max": 31.192916870117188,
      "activations/layer23_attention_weight_min": -23.988956451416016,
      "activations/layer2_attention_weight_max": 33.08353805541992,
      "activations/layer2_attention_weight_min": -33.54330825805664,
      "activations/layer3_attention_weight_max": 105.75948333740234,
      "activations/layer3_attention_weight_min": -105.35382843017578,
      "activations/layer4_attention_weight_max": 114.8713607788086,
      "activations/layer4_attention_weight_min": -117.4819564819336,
      "activations/layer5_attention_weight_max": 51.363792419433594,
      "activations/layer5_attention_weight_min": -62.72739791870117,
      "activations/layer6_attention_weight_max": 43.95375442504883,
      "activations/layer6_attention_weight_min": -48.907833099365234,
      "activations/layer7_attention_weight_max": 96.39591979980469,
      "activations/layer7_attention_weight_min": -105.38844299316406,
      "activations/layer8_attention_weight_max": 43.986778259277344,
      "activations/layer8_attention_weight_min": -45.933963775634766,
      "activations/layer9_attention_weight_max": 34.48845672607422,
      "activations/layer9_attention_weight_min": -36.28772735595703,
      "epoch": 17.91,
      "learning_rate": 3.48655303030303e-05,
      "loss": 2.728,
      "step": 308200
    },
    {
      "activations/layer0_attention_weight_max": 15.582468032836914,
      "activations/layer0_attention_weight_min": -13.831114768981934,
      "activations/layer10_attention_weight_max": 32.999454498291016,
      "activations/layer10_attention_weight_min": -34.501800537109375,
      "activations/layer11_attention_weight_max": 33.83794021606445,
      "activations/layer11_attention_weight_min": -36.29302978515625,
      "activations/layer12_attention_weight_max": 28.832040786743164,
      "activations/layer12_attention_weight_min": -26.743619918823242,
      "activations/layer13_attention_weight_max": 44.23644256591797,
      "activations/layer13_attention_weight_min": -38.53756332397461,
      "activations/layer14_attention_weight_max": 45.22014236450195,
      "activations/layer14_attention_weight_min": -37.75517272949219,
      "activations/layer15_attention_weight_max": 41.731353759765625,
      "activations/layer15_attention_weight_min": -37.21824645996094,
      "activations/layer16_attention_weight_max": 32.94662094116211,
      "activations/layer16_attention_weight_min": -28.454526901245117,
      "activations/layer17_attention_weight_max": 33.405357360839844,
      "activations/layer17_attention_weight_min": -27.041051864624023,
      "activations/layer18_attention_weight_max": 33.83197784423828,
      "activations/layer18_attention_weight_min": -25.91132164001465,
      "activations/layer19_attention_weight_max": 36.171382904052734,
      "activations/layer19_attention_weight_min": -33.26288986206055,
      "activations/layer1_attention_weight_max": 16.224327087402344,
      "activations/layer1_attention_weight_min": -14.760836601257324,
      "activations/layer20_attention_weight_max": 33.823028564453125,
      "activations/layer20_attention_weight_min": -25.861467361450195,
      "activations/layer21_attention_weight_max": 31.27237892150879,
      "activations/layer21_attention_weight_min": -27.070547103881836,
      "activations/layer22_attention_weight_max": 50.47425079345703,
      "activations/layer22_attention_weight_min": -35.83881759643555,
      "activations/layer23_attention_weight_max": 36.86035919189453,
      "activations/layer23_attention_weight_min": -28.172958374023438,
      "activations/layer2_attention_weight_max": 33.63011932373047,
      "activations/layer2_attention_weight_min": -31.82288360595703,
      "activations/layer3_attention_weight_max": 100.70946502685547,
      "activations/layer3_attention_weight_min": -99.8764419555664,
      "activations/layer4_attention_weight_max": 109.08602142333984,
      "activations/layer4_attention_weight_min": -109.40152740478516,
      "activations/layer5_attention_weight_max": 54.10984802246094,
      "activations/layer5_attention_weight_min": -62.01522445678711,
      "activations/layer6_attention_weight_max": 45.976219177246094,
      "activations/layer6_attention_weight_min": -51.93806838989258,
      "activations/layer7_attention_weight_max": 96.22704315185547,
      "activations/layer7_attention_weight_min": -104.82423400878906,
      "activations/layer8_attention_weight_max": 41.64922332763672,
      "activations/layer8_attention_weight_min": -44.79353713989258,
      "activations/layer9_attention_weight_max": 33.852046966552734,
      "activations/layer9_attention_weight_min": -35.49923324584961,
      "epoch": 17.91,
      "learning_rate": 3.48465909090909e-05,
      "loss": 2.7457,
      "step": 308250
    },
    {
      "activations/layer0_attention_weight_max": 16.141143798828125,
      "activations/layer0_attention_weight_min": -12.863436698913574,
      "activations/layer10_attention_weight_max": 32.77212905883789,
      "activations/layer10_attention_weight_min": -33.72880172729492,
      "activations/layer11_attention_weight_max": 32.402931213378906,
      "activations/layer11_attention_weight_min": -33.68259048461914,
      "activations/layer12_attention_weight_max": 25.275928497314453,
      "activations/layer12_attention_weight_min": -25.643787384033203,
      "activations/layer13_attention_weight_max": 38.4753303527832,
      "activations/layer13_attention_weight_min": -37.82535934448242,
      "activations/layer14_attention_weight_max": 42.05951690673828,
      "activations/layer14_attention_weight_min": -37.58918380737305,
      "activations/layer15_attention_weight_max": 37.068538665771484,
      "activations/layer15_attention_weight_min": -35.342681884765625,
      "activations/layer16_attention_weight_max": 28.9825382232666,
      "activations/layer16_attention_weight_min": -30.179553985595703,
      "activations/layer17_attention_weight_max": 30.526758193969727,
      "activations/layer17_attention_weight_min": -26.42115020751953,
      "activations/layer18_attention_weight_max": 31.300521850585938,
      "activations/layer18_attention_weight_min": -24.771686553955078,
      "activations/layer19_attention_weight_max": 33.221534729003906,
      "activations/layer19_attention_weight_min": -31.786813735961914,
      "activations/layer1_attention_weight_max": 15.600993156433105,
      "activations/layer1_attention_weight_min": -14.906360626220703,
      "activations/layer20_attention_weight_max": 32.87736892700195,
      "activations/layer20_attention_weight_min": -23.745071411132812,
      "activations/layer21_attention_weight_max": 29.39440155029297,
      "activations/layer21_attention_weight_min": -23.598649978637695,
      "activations/layer22_attention_weight_max": 40.924835205078125,
      "activations/layer22_attention_weight_min": -31.25505256652832,
      "activations/layer23_attention_weight_max": 32.358116149902344,
      "activations/layer23_attention_weight_min": -25.860464096069336,
      "activations/layer2_attention_weight_max": 32.869930267333984,
      "activations/layer2_attention_weight_min": -32.14731979370117,
      "activations/layer3_attention_weight_max": 98.83187103271484,
      "activations/layer3_attention_weight_min": -99.16242980957031,
      "activations/layer4_attention_weight_max": 108.2484130859375,
      "activations/layer4_attention_weight_min": -105.27020263671875,
      "activations/layer5_attention_weight_max": 53.32184600830078,
      "activations/layer5_attention_weight_min": -61.15234375,
      "activations/layer6_attention_weight_max": 42.227027893066406,
      "activations/layer6_attention_weight_min": -43.87019729614258,
      "activations/layer7_attention_weight_max": 90.35610961914062,
      "activations/layer7_attention_weight_min": -92.96577453613281,
      "activations/layer8_attention_weight_max": 40.94972229003906,
      "activations/layer8_attention_weight_min": -43.79854202270508,
      "activations/layer9_attention_weight_max": 32.94515609741211,
      "activations/layer9_attention_weight_min": -33.06102752685547,
      "epoch": 17.91,
      "learning_rate": 3.482765151515151e-05,
      "loss": 2.7377,
      "step": 308300
    },
    {
      "activations/layer0_attention_weight_max": 16.201045989990234,
      "activations/layer0_attention_weight_min": -12.735450744628906,
      "activations/layer10_attention_weight_max": 34.455291748046875,
      "activations/layer10_attention_weight_min": -34.60516357421875,
      "activations/layer11_attention_weight_max": 33.38422393798828,
      "activations/layer11_attention_weight_min": -33.68527603149414,
      "activations/layer12_attention_weight_max": 28.057518005371094,
      "activations/layer12_attention_weight_min": -25.158355712890625,
      "activations/layer13_attention_weight_max": 39.98201370239258,
      "activations/layer13_attention_weight_min": -35.46988296508789,
      "activations/layer14_attention_weight_max": 43.15071487426758,
      "activations/layer14_attention_weight_min": -39.18602752685547,
      "activations/layer15_attention_weight_max": 40.503822326660156,
      "activations/layer15_attention_weight_min": -35.867530822753906,
      "activations/layer16_attention_weight_max": 30.073938369750977,
      "activations/layer16_attention_weight_min": -28.00141716003418,
      "activations/layer17_attention_weight_max": 29.50396156311035,
      "activations/layer17_attention_weight_min": -25.656841278076172,
      "activations/layer18_attention_weight_max": 31.906774520874023,
      "activations/layer18_attention_weight_min": -28.079120635986328,
      "activations/layer19_attention_weight_max": 36.063987731933594,
      "activations/layer19_attention_weight_min": -34.256736755371094,
      "activations/layer1_attention_weight_max": 17.257675170898438,
      "activations/layer1_attention_weight_min": -14.826539993286133,
      "activations/layer20_attention_weight_max": 30.137428283691406,
      "activations/layer20_attention_weight_min": -27.509765625,
      "activations/layer21_attention_weight_max": 28.373044967651367,
      "activations/layer21_attention_weight_min": -26.747074127197266,
      "activations/layer22_attention_weight_max": 42.05396270751953,
      "activations/layer22_attention_weight_min": -32.253780364990234,
      "activations/layer23_attention_weight_max": 30.960187911987305,
      "activations/layer23_attention_weight_min": -26.678892135620117,
      "activations/layer2_attention_weight_max": 34.248130798339844,
      "activations/layer2_attention_weight_min": -32.24207305908203,
      "activations/layer3_attention_weight_max": 103.70903015136719,
      "activations/layer3_attention_weight_min": -100.44925689697266,
      "activations/layer4_attention_weight_max": 108.33855438232422,
      "activations/layer4_attention_weight_min": -106.36772918701172,
      "activations/layer5_attention_weight_max": 50.185821533203125,
      "activations/layer5_attention_weight_min": -58.69843673706055,
      "activations/layer6_attention_weight_max": 43.869285583496094,
      "activations/layer6_attention_weight_min": -46.8661994934082,
      "activations/layer7_attention_weight_max": 92.58372497558594,
      "activations/layer7_attention_weight_min": -93.36061096191406,
      "activations/layer8_attention_weight_max": 40.52032470703125,
      "activations/layer8_attention_weight_min": -42.80402374267578,
      "activations/layer9_attention_weight_max": 31.875410079956055,
      "activations/layer9_attention_weight_min": -33.8365364074707,
      "epoch": 17.92,
      "learning_rate": 3.480871212121212e-05,
      "loss": 2.7307,
      "step": 308350
    },
    {
      "activations/layer0_attention_weight_max": 15.345624923706055,
      "activations/layer0_attention_weight_min": -14.2205228805542,
      "activations/layer10_attention_weight_max": 37.824462890625,
      "activations/layer10_attention_weight_min": -35.13153076171875,
      "activations/layer11_attention_weight_max": 36.17609405517578,
      "activations/layer11_attention_weight_min": -33.664833068847656,
      "activations/layer12_attention_weight_max": 28.541654586791992,
      "activations/layer12_attention_weight_min": -26.759971618652344,
      "activations/layer13_attention_weight_max": 40.97048568725586,
      "activations/layer13_attention_weight_min": -35.048797607421875,
      "activations/layer14_attention_weight_max": 41.74504852294922,
      "activations/layer14_attention_weight_min": -38.457984924316406,
      "activations/layer15_attention_weight_max": 36.803218841552734,
      "activations/layer15_attention_weight_min": -34.430702209472656,
      "activations/layer16_attention_weight_max": 27.95231819152832,
      "activations/layer16_attention_weight_min": -27.05805778503418,
      "activations/layer17_attention_weight_max": 29.710161209106445,
      "activations/layer17_attention_weight_min": -27.27716636657715,
      "activations/layer18_attention_weight_max": 33.20281982421875,
      "activations/layer18_attention_weight_min": -27.092302322387695,
      "activations/layer19_attention_weight_max": 29.998239517211914,
      "activations/layer19_attention_weight_min": -33.8343505859375,
      "activations/layer1_attention_weight_max": 16.691858291625977,
      "activations/layer1_attention_weight_min": -15.24797248840332,
      "activations/layer20_attention_weight_max": 26.240577697753906,
      "activations/layer20_attention_weight_min": -28.768035888671875,
      "activations/layer21_attention_weight_max": 28.64713478088379,
      "activations/layer21_attention_weight_min": -27.320907592773438,
      "activations/layer22_attention_weight_max": 40.70840835571289,
      "activations/layer22_attention_weight_min": -35.23154830932617,
      "activations/layer23_attention_weight_max": 31.98328971862793,
      "activations/layer23_attention_weight_min": -28.447240829467773,
      "activations/layer2_attention_weight_max": 31.720006942749023,
      "activations/layer2_attention_weight_min": -32.07118606567383,
      "activations/layer3_attention_weight_max": 101.2844467163086,
      "activations/layer3_attention_weight_min": -97.52925872802734,
      "activations/layer4_attention_weight_max": 107.2597885131836,
      "activations/layer4_attention_weight_min": -105.49468994140625,
      "activations/layer5_attention_weight_max": 50.987548828125,
      "activations/layer5_attention_weight_min": -60.445213317871094,
      "activations/layer6_attention_weight_max": 43.92223358154297,
      "activations/layer6_attention_weight_min": -46.47056579589844,
      "activations/layer7_attention_weight_max": 97.18846130371094,
      "activations/layer7_attention_weight_min": -90.15059661865234,
      "activations/layer8_attention_weight_max": 42.56156539916992,
      "activations/layer8_attention_weight_min": -42.784095764160156,
      "activations/layer9_attention_weight_max": 34.19933319091797,
      "activations/layer9_attention_weight_min": -37.0969352722168,
      "epoch": 17.92,
      "learning_rate": 3.478977272727273e-05,
      "loss": 2.746,
      "step": 308400
    },
    {
      "activations/layer0_attention_weight_max": 15.515924453735352,
      "activations/layer0_attention_weight_min": -13.73727798461914,
      "activations/layer10_attention_weight_max": 31.121049880981445,
      "activations/layer10_attention_weight_min": -32.22291946411133,
      "activations/layer11_attention_weight_max": 32.48789596557617,
      "activations/layer11_attention_weight_min": -32.135154724121094,
      "activations/layer12_attention_weight_max": 24.7755126953125,
      "activations/layer12_attention_weight_min": -25.892534255981445,
      "activations/layer13_attention_weight_max": 35.98396301269531,
      "activations/layer13_attention_weight_min": -35.68563461303711,
      "activations/layer14_attention_weight_max": 41.476348876953125,
      "activations/layer14_attention_weight_min": -37.196502685546875,
      "activations/layer15_attention_weight_max": 36.53873825073242,
      "activations/layer15_attention_weight_min": -36.81566619873047,
      "activations/layer16_attention_weight_max": 29.65839958190918,
      "activations/layer16_attention_weight_min": -27.75114631652832,
      "activations/layer17_attention_weight_max": 30.17057991027832,
      "activations/layer17_attention_weight_min": -27.07421875,
      "activations/layer18_attention_weight_max": 30.08342933654785,
      "activations/layer18_attention_weight_min": -25.393978118896484,
      "activations/layer19_attention_weight_max": 32.878021240234375,
      "activations/layer19_attention_weight_min": -32.15149688720703,
      "activations/layer1_attention_weight_max": 15.644848823547363,
      "activations/layer1_attention_weight_min": -14.614462852478027,
      "activations/layer20_attention_weight_max": 27.983665466308594,
      "activations/layer20_attention_weight_min": -25.935752868652344,
      "activations/layer21_attention_weight_max": 28.765743255615234,
      "activations/layer21_attention_weight_min": -23.88395118713379,
      "activations/layer22_attention_weight_max": 38.366424560546875,
      "activations/layer22_attention_weight_min": -32.5468864440918,
      "activations/layer23_attention_weight_max": 30.600372314453125,
      "activations/layer23_attention_weight_min": -26.051624298095703,
      "activations/layer2_attention_weight_max": 30.991661071777344,
      "activations/layer2_attention_weight_min": -31.811134338378906,
      "activations/layer3_attention_weight_max": 97.01943969726562,
      "activations/layer3_attention_weight_min": -102.95811462402344,
      "activations/layer4_attention_weight_max": 106.19625091552734,
      "activations/layer4_attention_weight_min": -110.26409149169922,
      "activations/layer5_attention_weight_max": 49.896583557128906,
      "activations/layer5_attention_weight_min": -64.3573989868164,
      "activations/layer6_attention_weight_max": 44.36954879760742,
      "activations/layer6_attention_weight_min": -45.87697219848633,
      "activations/layer7_attention_weight_max": 85.90709686279297,
      "activations/layer7_attention_weight_min": -92.37909698486328,
      "activations/layer8_attention_weight_max": 40.221683502197266,
      "activations/layer8_attention_weight_min": -42.30679702758789,
      "activations/layer9_attention_weight_max": 30.985769271850586,
      "activations/layer9_attention_weight_min": -34.426517486572266,
      "epoch": 17.92,
      "learning_rate": 3.477083333333333e-05,
      "loss": 2.7318,
      "step": 308450
    },
    {
      "activations/layer0_attention_weight_max": 15.302506446838379,
      "activations/layer0_attention_weight_min": -13.186299324035645,
      "activations/layer10_attention_weight_max": 31.23379135131836,
      "activations/layer10_attention_weight_min": -31.524089813232422,
      "activations/layer11_attention_weight_max": 32.861053466796875,
      "activations/layer11_attention_weight_min": -32.34546661376953,
      "activations/layer12_attention_weight_max": 26.41118812561035,
      "activations/layer12_attention_weight_min": -27.658906936645508,
      "activations/layer13_attention_weight_max": 38.75400924682617,
      "activations/layer13_attention_weight_min": -35.25798416137695,
      "activations/layer14_attention_weight_max": 40.369773864746094,
      "activations/layer14_attention_weight_min": -37.400413513183594,
      "activations/layer15_attention_weight_max": 38.03937530517578,
      "activations/layer15_attention_weight_min": -34.69413757324219,
      "activations/layer16_attention_weight_max": 29.7750301361084,
      "activations/layer16_attention_weight_min": -27.26872444152832,
      "activations/layer17_attention_weight_max": 28.13003158569336,
      "activations/layer17_attention_weight_min": -27.156888961791992,
      "activations/layer18_attention_weight_max": 35.3004264831543,
      "activations/layer18_attention_weight_min": -28.244680404663086,
      "activations/layer19_attention_weight_max": 35.57579803466797,
      "activations/layer19_attention_weight_min": -32.53810501098633,
      "activations/layer1_attention_weight_max": 15.760502815246582,
      "activations/layer1_attention_weight_min": -15.24487590789795,
      "activations/layer20_attention_weight_max": 30.561634063720703,
      "activations/layer20_attention_weight_min": -26.826292037963867,
      "activations/layer21_attention_weight_max": 33.341102600097656,
      "activations/layer21_attention_weight_min": -25.740421295166016,
      "activations/layer22_attention_weight_max": 42.2864990234375,
      "activations/layer22_attention_weight_min": -33.890506744384766,
      "activations/layer23_attention_weight_max": 32.60969161987305,
      "activations/layer23_attention_weight_min": -24.439918518066406,
      "activations/layer2_attention_weight_max": 33.02510070800781,
      "activations/layer2_attention_weight_min": -31.742515563964844,
      "activations/layer3_attention_weight_max": 97.72882843017578,
      "activations/layer3_attention_weight_min": -98.41224670410156,
      "activations/layer4_attention_weight_max": 103.52135467529297,
      "activations/layer4_attention_weight_min": -107.06758880615234,
      "activations/layer5_attention_weight_max": 48.16484832763672,
      "activations/layer5_attention_weight_min": -60.69647216796875,
      "activations/layer6_attention_weight_max": 43.416221618652344,
      "activations/layer6_attention_weight_min": -49.40665054321289,
      "activations/layer7_attention_weight_max": 89.86119842529297,
      "activations/layer7_attention_weight_min": -95.02899169921875,
      "activations/layer8_attention_weight_max": 40.282222747802734,
      "activations/layer8_attention_weight_min": -43.879150390625,
      "activations/layer9_attention_weight_max": 31.229700088500977,
      "activations/layer9_attention_weight_min": -32.24789810180664,
      "epoch": 17.93,
      "learning_rate": 3.475189393939394e-05,
      "loss": 2.747,
      "step": 308500
    },
    {
      "activations/layer0_attention_weight_max": 15.583121299743652,
      "activations/layer0_attention_weight_min": -13.305061340332031,
      "activations/layer10_attention_weight_max": 32.157066345214844,
      "activations/layer10_attention_weight_min": -32.686893463134766,
      "activations/layer11_attention_weight_max": 32.47487258911133,
      "activations/layer11_attention_weight_min": -31.924177169799805,
      "activations/layer12_attention_weight_max": 25.513715744018555,
      "activations/layer12_attention_weight_min": -25.88068389892578,
      "activations/layer13_attention_weight_max": 37.34352493286133,
      "activations/layer13_attention_weight_min": -37.545772552490234,
      "activations/layer14_attention_weight_max": 39.772274017333984,
      "activations/layer14_attention_weight_min": -38.672935485839844,
      "activations/layer15_attention_weight_max": 36.42302322387695,
      "activations/layer15_attention_weight_min": -36.879878997802734,
      "activations/layer16_attention_weight_max": 27.4877872467041,
      "activations/layer16_attention_weight_min": -28.314556121826172,
      "activations/layer17_attention_weight_max": 28.81342315673828,
      "activations/layer17_attention_weight_min": -26.13574981689453,
      "activations/layer18_attention_weight_max": 33.12089538574219,
      "activations/layer18_attention_weight_min": -26.477981567382812,
      "activations/layer19_attention_weight_max": 31.479766845703125,
      "activations/layer19_attention_weight_min": -30.038969039916992,
      "activations/layer1_attention_weight_max": 16.29310417175293,
      "activations/layer1_attention_weight_min": -14.443669319152832,
      "activations/layer20_attention_weight_max": 28.824132919311523,
      "activations/layer20_attention_weight_min": -24.440441131591797,
      "activations/layer21_attention_weight_max": 30.133255004882812,
      "activations/layer21_attention_weight_min": -23.98026466369629,
      "activations/layer22_attention_weight_max": 43.60007095336914,
      "activations/layer22_attention_weight_min": -30.937511444091797,
      "activations/layer23_attention_weight_max": 30.023052215576172,
      "activations/layer23_attention_weight_min": -25.550512313842773,
      "activations/layer2_attention_weight_max": 35.10493469238281,
      "activations/layer2_attention_weight_min": -32.973182678222656,
      "activations/layer3_attention_weight_max": 106.37406158447266,
      "activations/layer3_attention_weight_min": -103.59793853759766,
      "activations/layer4_attention_weight_max": 108.86665344238281,
      "activations/layer4_attention_weight_min": -113.17585754394531,
      "activations/layer5_attention_weight_max": 49.20561218261719,
      "activations/layer5_attention_weight_min": -60.98078918457031,
      "activations/layer6_attention_weight_max": 44.853267669677734,
      "activations/layer6_attention_weight_min": -45.09461975097656,
      "activations/layer7_attention_weight_max": 87.9635238647461,
      "activations/layer7_attention_weight_min": -92.90133666992188,
      "activations/layer8_attention_weight_max": 41.557373046875,
      "activations/layer8_attention_weight_min": -41.24402618408203,
      "activations/layer9_attention_weight_max": 31.87412452697754,
      "activations/layer9_attention_weight_min": -31.34075355529785,
      "epoch": 17.93,
      "learning_rate": 3.4732954545454546e-05,
      "loss": 2.7354,
      "step": 308550
    },
    {
      "activations/layer0_attention_weight_max": 15.917460441589355,
      "activations/layer0_attention_weight_min": -13.801722526550293,
      "activations/layer10_attention_weight_max": 36.841182708740234,
      "activations/layer10_attention_weight_min": -36.75126647949219,
      "activations/layer11_attention_weight_max": 34.35284423828125,
      "activations/layer11_attention_weight_min": -35.03377151489258,
      "activations/layer12_attention_weight_max": 26.304733276367188,
      "activations/layer12_attention_weight_min": -27.685983657836914,
      "activations/layer13_attention_weight_max": 38.663719177246094,
      "activations/layer13_attention_weight_min": -36.47394561767578,
      "activations/layer14_attention_weight_max": 41.99492645263672,
      "activations/layer14_attention_weight_min": -40.0257682800293,
      "activations/layer15_attention_weight_max": 38.315406799316406,
      "activations/layer15_attention_weight_min": -38.097843170166016,
      "activations/layer16_attention_weight_max": 29.72270965576172,
      "activations/layer16_attention_weight_min": -29.454538345336914,
      "activations/layer17_attention_weight_max": 30.176753997802734,
      "activations/layer17_attention_weight_min": -26.82904624938965,
      "activations/layer18_attention_weight_max": 32.82293701171875,
      "activations/layer18_attention_weight_min": -27.381332397460938,
      "activations/layer19_attention_weight_max": 34.15884017944336,
      "activations/layer19_attention_weight_min": -32.28510665893555,
      "activations/layer1_attention_weight_max": 16.180282592773438,
      "activations/layer1_attention_weight_min": -14.633082389831543,
      "activations/layer20_attention_weight_max": 28.653709411621094,
      "activations/layer20_attention_weight_min": -25.187009811401367,
      "activations/layer21_attention_weight_max": 32.14259719848633,
      "activations/layer21_attention_weight_min": -27.560100555419922,
      "activations/layer22_attention_weight_max": 42.81917953491211,
      "activations/layer22_attention_weight_min": -32.97637176513672,
      "activations/layer23_attention_weight_max": 30.769081115722656,
      "activations/layer23_attention_weight_min": -25.70429801940918,
      "activations/layer2_attention_weight_max": 34.296566009521484,
      "activations/layer2_attention_weight_min": -33.67243194580078,
      "activations/layer3_attention_weight_max": 105.40379333496094,
      "activations/layer3_attention_weight_min": -103.11983489990234,
      "activations/layer4_attention_weight_max": 106.6011734008789,
      "activations/layer4_attention_weight_min": -109.96905517578125,
      "activations/layer5_attention_weight_max": 50.127044677734375,
      "activations/layer5_attention_weight_min": -65.17877960205078,
      "activations/layer6_attention_weight_max": 44.866050720214844,
      "activations/layer6_attention_weight_min": -49.55862045288086,
      "activations/layer7_attention_weight_max": 93.19145202636719,
      "activations/layer7_attention_weight_min": -100.16357421875,
      "activations/layer8_attention_weight_max": 42.82723617553711,
      "activations/layer8_attention_weight_min": -43.921016693115234,
      "activations/layer9_attention_weight_max": 33.240638732910156,
      "activations/layer9_attention_weight_min": -36.06697463989258,
      "epoch": 17.93,
      "learning_rate": 3.471401515151515e-05,
      "loss": 2.7318,
      "step": 308600
    },
    {
      "activations/layer0_attention_weight_max": 15.883136749267578,
      "activations/layer0_attention_weight_min": -11.886358261108398,
      "activations/layer10_attention_weight_max": 33.727088928222656,
      "activations/layer10_attention_weight_min": -33.51004409790039,
      "activations/layer11_attention_weight_max": 32.62248229980469,
      "activations/layer11_attention_weight_min": -32.564476013183594,
      "activations/layer12_attention_weight_max": 25.184803009033203,
      "activations/layer12_attention_weight_min": -27.20730209350586,
      "activations/layer13_attention_weight_max": 35.686519622802734,
      "activations/layer13_attention_weight_min": -36.305702209472656,
      "activations/layer14_attention_weight_max": 36.7331428527832,
      "activations/layer14_attention_weight_min": -37.42960739135742,
      "activations/layer15_attention_weight_max": 38.68205261230469,
      "activations/layer15_attention_weight_min": -35.74451446533203,
      "activations/layer16_attention_weight_max": 27.603378295898438,
      "activations/layer16_attention_weight_min": -29.204832077026367,
      "activations/layer17_attention_weight_max": 30.13404083251953,
      "activations/layer17_attention_weight_min": -29.43400001525879,
      "activations/layer18_attention_weight_max": 28.030508041381836,
      "activations/layer18_attention_weight_min": -25.872432708740234,
      "activations/layer19_attention_weight_max": 30.484617233276367,
      "activations/layer19_attention_weight_min": -30.747135162353516,
      "activations/layer1_attention_weight_max": 17.836402893066406,
      "activations/layer1_attention_weight_min": -17.390926361083984,
      "activations/layer20_attention_weight_max": 27.13090705871582,
      "activations/layer20_attention_weight_min": -25.995943069458008,
      "activations/layer21_attention_weight_max": 26.79710578918457,
      "activations/layer21_attention_weight_min": -25.694286346435547,
      "activations/layer22_attention_weight_max": 35.16059112548828,
      "activations/layer22_attention_weight_min": -30.476810455322266,
      "activations/layer23_attention_weight_max": 29.282970428466797,
      "activations/layer23_attention_weight_min": -24.650009155273438,
      "activations/layer2_attention_weight_max": 32.52599334716797,
      "activations/layer2_attention_weight_min": -33.07530975341797,
      "activations/layer3_attention_weight_max": 101.74181365966797,
      "activations/layer3_attention_weight_min": -105.03734588623047,
      "activations/layer4_attention_weight_max": 105.1075439453125,
      "activations/layer4_attention_weight_min": -109.30201721191406,
      "activations/layer5_attention_weight_max": 52.36717224121094,
      "activations/layer5_attention_weight_min": -59.51719284057617,
      "activations/layer6_attention_weight_max": 43.10649490356445,
      "activations/layer6_attention_weight_min": -46.502628326416016,
      "activations/layer7_attention_weight_max": 90.30687713623047,
      "activations/layer7_attention_weight_min": -92.48490142822266,
      "activations/layer8_attention_weight_max": 40.626705169677734,
      "activations/layer8_attention_weight_min": -42.483070373535156,
      "activations/layer9_attention_weight_max": 30.84625244140625,
      "activations/layer9_attention_weight_min": -32.925350189208984,
      "epoch": 17.93,
      "learning_rate": 3.4695075757575756e-05,
      "loss": 2.7303,
      "step": 308650
    },
    {
      "activations/layer0_attention_weight_max": 16.11674690246582,
      "activations/layer0_attention_weight_min": -12.712483406066895,
      "activations/layer10_attention_weight_max": 31.977184295654297,
      "activations/layer10_attention_weight_min": -34.714019775390625,
      "activations/layer11_attention_weight_max": 36.0653076171875,
      "activations/layer11_attention_weight_min": -36.452491760253906,
      "activations/layer12_attention_weight_max": 26.988996505737305,
      "activations/layer12_attention_weight_min": -26.533649444580078,
      "activations/layer13_attention_weight_max": 41.40296173095703,
      "activations/layer13_attention_weight_min": -35.06602478027344,
      "activations/layer14_attention_weight_max": 41.92251968383789,
      "activations/layer14_attention_weight_min": -37.62270736694336,
      "activations/layer15_attention_weight_max": 39.91796875,
      "activations/layer15_attention_weight_min": -36.55745315551758,
      "activations/layer16_attention_weight_max": 30.144075393676758,
      "activations/layer16_attention_weight_min": -29.647294998168945,
      "activations/layer17_attention_weight_max": 28.208770751953125,
      "activations/layer17_attention_weight_min": -27.852588653564453,
      "activations/layer18_attention_weight_max": 31.346763610839844,
      "activations/layer18_attention_weight_min": -27.050926208496094,
      "activations/layer19_attention_weight_max": 35.535865783691406,
      "activations/layer19_attention_weight_min": -35.396728515625,
      "activations/layer1_attention_weight_max": 17.84231185913086,
      "activations/layer1_attention_weight_min": -15.431830406188965,
      "activations/layer20_attention_weight_max": 30.990692138671875,
      "activations/layer20_attention_weight_min": -28.084644317626953,
      "activations/layer21_attention_weight_max": 28.629018783569336,
      "activations/layer21_attention_weight_min": -26.604570388793945,
      "activations/layer22_attention_weight_max": 39.70438003540039,
      "activations/layer22_attention_weight_min": -33.0038948059082,
      "activations/layer23_attention_weight_max": 33.23320770263672,
      "activations/layer23_attention_weight_min": -27.1842041015625,
      "activations/layer2_attention_weight_max": 32.830238342285156,
      "activations/layer2_attention_weight_min": -32.54718017578125,
      "activations/layer3_attention_weight_max": 101.04927062988281,
      "activations/layer3_attention_weight_min": -100.9528579711914,
      "activations/layer4_attention_weight_max": 106.81412506103516,
      "activations/layer4_attention_weight_min": -109.80897521972656,
      "activations/layer5_attention_weight_max": 53.000022888183594,
      "activations/layer5_attention_weight_min": -61.092742919921875,
      "activations/layer6_attention_weight_max": 43.403053283691406,
      "activations/layer6_attention_weight_min": -45.156227111816406,
      "activations/layer7_attention_weight_max": 88.40109252929688,
      "activations/layer7_attention_weight_min": -91.8134536743164,
      "activations/layer8_attention_weight_max": 39.575050354003906,
      "activations/layer8_attention_weight_min": -41.62403869628906,
      "activations/layer9_attention_weight_max": 31.733102798461914,
      "activations/layer9_attention_weight_min": -34.06672668457031,
      "epoch": 17.94,
      "learning_rate": 3.467613636363636e-05,
      "loss": 2.7299,
      "step": 308700
    },
    {
      "activations/layer0_attention_weight_max": 16.974227905273438,
      "activations/layer0_attention_weight_min": -12.288477897644043,
      "activations/layer10_attention_weight_max": 32.685813903808594,
      "activations/layer10_attention_weight_min": -35.03554916381836,
      "activations/layer11_attention_weight_max": 33.873111724853516,
      "activations/layer11_attention_weight_min": -36.187705993652344,
      "activations/layer12_attention_weight_max": 27.442527770996094,
      "activations/layer12_attention_weight_min": -28.02529525756836,
      "activations/layer13_attention_weight_max": 38.19777297973633,
      "activations/layer13_attention_weight_min": -38.599422454833984,
      "activations/layer14_attention_weight_max": 42.02708053588867,
      "activations/layer14_attention_weight_min": -39.19369125366211,
      "activations/layer15_attention_weight_max": 38.54109191894531,
      "activations/layer15_attention_weight_min": -38.24711608886719,
      "activations/layer16_attention_weight_max": 30.33232879638672,
      "activations/layer16_attention_weight_min": -29.662090301513672,
      "activations/layer17_attention_weight_max": 29.51784324645996,
      "activations/layer17_attention_weight_min": -27.494468688964844,
      "activations/layer18_attention_weight_max": 33.80099868774414,
      "activations/layer18_attention_weight_min": -28.174312591552734,
      "activations/layer19_attention_weight_max": 32.72846984863281,
      "activations/layer19_attention_weight_min": -35.94862747192383,
      "activations/layer1_attention_weight_max": 16.779264450073242,
      "activations/layer1_attention_weight_min": -14.453604698181152,
      "activations/layer20_attention_weight_max": 28.298011779785156,
      "activations/layer20_attention_weight_min": -29.000398635864258,
      "activations/layer21_attention_weight_max": 28.778209686279297,
      "activations/layer21_attention_weight_min": -27.103530883789062,
      "activations/layer22_attention_weight_max": 40.56766128540039,
      "activations/layer22_attention_weight_min": -35.59742736816406,
      "activations/layer23_attention_weight_max": 30.275222778320312,
      "activations/layer23_attention_weight_min": -27.28937530517578,
      "activations/layer2_attention_weight_max": 33.37843322753906,
      "activations/layer2_attention_weight_min": -32.90766143798828,
      "activations/layer3_attention_weight_max": 97.5347900390625,
      "activations/layer3_attention_weight_min": -99.3045883178711,
      "activations/layer4_attention_weight_max": 102.0473861694336,
      "activations/layer4_attention_weight_min": -105.6474609375,
      "activations/layer5_attention_weight_max": 47.46813201904297,
      "activations/layer5_attention_weight_min": -61.91529846191406,
      "activations/layer6_attention_weight_max": 44.889686584472656,
      "activations/layer6_attention_weight_min": -47.49361038208008,
      "activations/layer7_attention_weight_max": 95.36788177490234,
      "activations/layer7_attention_weight_min": -95.02698516845703,
      "activations/layer8_attention_weight_max": 39.541072845458984,
      "activations/layer8_attention_weight_min": -42.05100631713867,
      "activations/layer9_attention_weight_max": 30.284311294555664,
      "activations/layer9_attention_weight_min": -33.36798858642578,
      "epoch": 17.94,
      "learning_rate": 3.4657196969696965e-05,
      "loss": 2.7259,
      "step": 308750
    },
    {
      "activations/layer0_attention_weight_max": 15.561515808105469,
      "activations/layer0_attention_weight_min": -12.804594993591309,
      "activations/layer10_attention_weight_max": 37.74095916748047,
      "activations/layer10_attention_weight_min": -36.1232795715332,
      "activations/layer11_attention_weight_max": 38.555667877197266,
      "activations/layer11_attention_weight_min": -37.55744552612305,
      "activations/layer12_attention_weight_max": 31.593076705932617,
      "activations/layer12_attention_weight_min": -32.145103454589844,
      "activations/layer13_attention_weight_max": 47.71756362915039,
      "activations/layer13_attention_weight_min": -41.0901985168457,
      "activations/layer14_attention_weight_max": 51.50930404663086,
      "activations/layer14_attention_weight_min": -44.591800689697266,
      "activations/layer15_attention_weight_max": 48.44628143310547,
      "activations/layer15_attention_weight_min": -39.97158432006836,
      "activations/layer16_attention_weight_max": 33.65192413330078,
      "activations/layer16_attention_weight_min": -28.92574119567871,
      "activations/layer17_attention_weight_max": 30.121978759765625,
      "activations/layer17_attention_weight_min": -27.28862762451172,
      "activations/layer18_attention_weight_max": 33.3999137878418,
      "activations/layer18_attention_weight_min": -25.9755916595459,
      "activations/layer19_attention_weight_max": 34.97945785522461,
      "activations/layer19_attention_weight_min": -30.82077407836914,
      "activations/layer1_attention_weight_max": 16.604145050048828,
      "activations/layer1_attention_weight_min": -14.6928071975708,
      "activations/layer20_attention_weight_max": 32.67496109008789,
      "activations/layer20_attention_weight_min": -25.83946990966797,
      "activations/layer21_attention_weight_max": 29.16361427307129,
      "activations/layer21_attention_weight_min": -25.3913516998291,
      "activations/layer22_attention_weight_max": 41.32041549682617,
      "activations/layer22_attention_weight_min": -33.65808868408203,
      "activations/layer23_attention_weight_max": 32.311458587646484,
      "activations/layer23_attention_weight_min": -26.873342514038086,
      "activations/layer2_attention_weight_max": 32.68114471435547,
      "activations/layer2_attention_weight_min": -33.672176361083984,
      "activations/layer3_attention_weight_max": 100.02511596679688,
      "activations/layer3_attention_weight_min": -102.2436752319336,
      "activations/layer4_attention_weight_max": 107.5456314086914,
      "activations/layer4_attention_weight_min": -111.9222412109375,
      "activations/layer5_attention_weight_max": 55.01403045654297,
      "activations/layer5_attention_weight_min": -62.388092041015625,
      "activations/layer6_attention_weight_max": 44.96116638183594,
      "activations/layer6_attention_weight_min": -46.48440933227539,
      "activations/layer7_attention_weight_max": 93.34457397460938,
      "activations/layer7_attention_weight_min": -98.3495101928711,
      "activations/layer8_attention_weight_max": 42.97930145263672,
      "activations/layer8_attention_weight_min": -46.49885177612305,
      "activations/layer9_attention_weight_max": 33.7276611328125,
      "activations/layer9_attention_weight_min": -35.54630661010742,
      "epoch": 17.94,
      "learning_rate": 3.4638257575757574e-05,
      "loss": 2.746,
      "step": 308800
    },
    {
      "activations/layer0_attention_weight_max": 15.918148040771484,
      "activations/layer0_attention_weight_min": -13.5127534866333,
      "activations/layer10_attention_weight_max": 30.760616302490234,
      "activations/layer10_attention_weight_min": -31.401412963867188,
      "activations/layer11_attention_weight_max": 31.704856872558594,
      "activations/layer11_attention_weight_min": -31.613630294799805,
      "activations/layer12_attention_weight_max": 25.041427612304688,
      "activations/layer12_attention_weight_min": -25.473773956298828,
      "activations/layer13_attention_weight_max": 36.201839447021484,
      "activations/layer13_attention_weight_min": -34.36725997924805,
      "activations/layer14_attention_weight_max": 41.137691497802734,
      "activations/layer14_attention_weight_min": -35.47770309448242,
      "activations/layer15_attention_weight_max": 38.678977966308594,
      "activations/layer15_attention_weight_min": -35.52351760864258,
      "activations/layer16_attention_weight_max": 28.029874801635742,
      "activations/layer16_attention_weight_min": -25.77585220336914,
      "activations/layer17_attention_weight_max": 27.013641357421875,
      "activations/layer17_attention_weight_min": -25.727663040161133,
      "activations/layer18_attention_weight_max": 30.4837703704834,
      "activations/layer18_attention_weight_min": -24.401315689086914,
      "activations/layer19_attention_weight_max": 31.70709800720215,
      "activations/layer19_attention_weight_min": -30.205310821533203,
      "activations/layer1_attention_weight_max": 15.454296112060547,
      "activations/layer1_attention_weight_min": -15.637691497802734,
      "activations/layer20_attention_weight_max": 26.047527313232422,
      "activations/layer20_attention_weight_min": -24.78121566772461,
      "activations/layer21_attention_weight_max": 25.339336395263672,
      "activations/layer21_attention_weight_min": -24.698856353759766,
      "activations/layer22_attention_weight_max": 40.07878112792969,
      "activations/layer22_attention_weight_min": -29.51974868774414,
      "activations/layer23_attention_weight_max": 30.224876403808594,
      "activations/layer23_attention_weight_min": -26.06963348388672,
      "activations/layer2_attention_weight_max": 34.915924072265625,
      "activations/layer2_attention_weight_min": -32.230045318603516,
      "activations/layer3_attention_weight_max": 103.31951141357422,
      "activations/layer3_attention_weight_min": -99.67832946777344,
      "activations/layer4_attention_weight_max": 104.30867767333984,
      "activations/layer4_attention_weight_min": -108.32246398925781,
      "activations/layer5_attention_weight_max": 47.999916076660156,
      "activations/layer5_attention_weight_min": -60.1827507019043,
      "activations/layer6_attention_weight_max": 42.14045715332031,
      "activations/layer6_attention_weight_min": -46.24744415283203,
      "activations/layer7_attention_weight_max": 89.77314758300781,
      "activations/layer7_attention_weight_min": -93.1084213256836,
      "activations/layer8_attention_weight_max": 38.98868942260742,
      "activations/layer8_attention_weight_min": -39.60435104370117,
      "activations/layer9_attention_weight_max": 30.143722534179688,
      "activations/layer9_attention_weight_min": -32.635746002197266,
      "epoch": 17.95,
      "learning_rate": 3.461931818181818e-05,
      "loss": 2.7299,
      "step": 308850
    },
    {
      "activations/layer0_attention_weight_max": 15.45167350769043,
      "activations/layer0_attention_weight_min": -11.801383972167969,
      "activations/layer10_attention_weight_max": 32.2381591796875,
      "activations/layer10_attention_weight_min": -34.54701232910156,
      "activations/layer11_attention_weight_max": 31.92542839050293,
      "activations/layer11_attention_weight_min": -34.20986557006836,
      "activations/layer12_attention_weight_max": 25.136884689331055,
      "activations/layer12_attention_weight_min": -26.843381881713867,
      "activations/layer13_attention_weight_max": 38.6993293762207,
      "activations/layer13_attention_weight_min": -37.925880432128906,
      "activations/layer14_attention_weight_max": 41.85520553588867,
      "activations/layer14_attention_weight_min": -39.62464141845703,
      "activations/layer15_attention_weight_max": 39.930633544921875,
      "activations/layer15_attention_weight_min": -38.400917053222656,
      "activations/layer16_attention_weight_max": 33.87068557739258,
      "activations/layer16_attention_weight_min": -28.855623245239258,
      "activations/layer17_attention_weight_max": 28.803117752075195,
      "activations/layer17_attention_weight_min": -26.328149795532227,
      "activations/layer18_attention_weight_max": 30.77016830444336,
      "activations/layer18_attention_weight_min": -27.588228225708008,
      "activations/layer19_attention_weight_max": 31.76816749572754,
      "activations/layer19_attention_weight_min": -31.364734649658203,
      "activations/layer1_attention_weight_max": 16.62250328063965,
      "activations/layer1_attention_weight_min": -15.256316184997559,
      "activations/layer20_attention_weight_max": 28.46321678161621,
      "activations/layer20_attention_weight_min": -24.651582717895508,
      "activations/layer21_attention_weight_max": 29.310056686401367,
      "activations/layer21_attention_weight_min": -23.24037742614746,
      "activations/layer22_attention_weight_max": 39.09050369262695,
      "activations/layer22_attention_weight_min": -30.844478607177734,
      "activations/layer23_attention_weight_max": 30.261356353759766,
      "activations/layer23_attention_weight_min": -25.622875213623047,
      "activations/layer2_attention_weight_max": 32.89692306518555,
      "activations/layer2_attention_weight_min": -33.243324279785156,
      "activations/layer3_attention_weight_max": 99.09896850585938,
      "activations/layer3_attention_weight_min": -99.87952423095703,
      "activations/layer4_attention_weight_max": 106.80281829833984,
      "activations/layer4_attention_weight_min": -107.14703369140625,
      "activations/layer5_attention_weight_max": 48.916542053222656,
      "activations/layer5_attention_weight_min": -63.79742431640625,
      "activations/layer6_attention_weight_max": 44.38579177856445,
      "activations/layer6_attention_weight_min": -48.108375549316406,
      "activations/layer7_attention_weight_max": 91.48786163330078,
      "activations/layer7_attention_weight_min": -94.72015380859375,
      "activations/layer8_attention_weight_max": 41.52614212036133,
      "activations/layer8_attention_weight_min": -43.84679412841797,
      "activations/layer9_attention_weight_max": 31.02558708190918,
      "activations/layer9_attention_weight_min": -32.817466735839844,
      "epoch": 17.95,
      "learning_rate": 3.460037878787878e-05,
      "loss": 2.7315,
      "step": 308900
    },
    {
      "activations/layer0_attention_weight_max": 16.211549758911133,
      "activations/layer0_attention_weight_min": -15.183486938476562,
      "activations/layer10_attention_weight_max": 35.30903625488281,
      "activations/layer10_attention_weight_min": -34.41218948364258,
      "activations/layer11_attention_weight_max": 35.62818908691406,
      "activations/layer11_attention_weight_min": -35.94536209106445,
      "activations/layer12_attention_weight_max": 26.989490509033203,
      "activations/layer12_attention_weight_min": -26.900806427001953,
      "activations/layer13_attention_weight_max": 39.25584411621094,
      "activations/layer13_attention_weight_min": -35.07230758666992,
      "activations/layer14_attention_weight_max": 40.830482482910156,
      "activations/layer14_attention_weight_min": -37.18624496459961,
      "activations/layer15_attention_weight_max": 37.491241455078125,
      "activations/layer15_attention_weight_min": -34.55678939819336,
      "activations/layer16_attention_weight_max": 29.729736328125,
      "activations/layer16_attention_weight_min": -27.235599517822266,
      "activations/layer17_attention_weight_max": 28.7318058013916,
      "activations/layer17_attention_weight_min": -25.69227409362793,
      "activations/layer18_attention_weight_max": 28.800540924072266,
      "activations/layer18_attention_weight_min": -24.89276885986328,
      "activations/layer19_attention_weight_max": 33.9107666015625,
      "activations/layer19_attention_weight_min": -32.602664947509766,
      "activations/layer1_attention_weight_max": 15.91631031036377,
      "activations/layer1_attention_weight_min": -14.51000690460205,
      "activations/layer20_attention_weight_max": 30.121124267578125,
      "activations/layer20_attention_weight_min": -25.621477127075195,
      "activations/layer21_attention_weight_max": 30.11135482788086,
      "activations/layer21_attention_weight_min": -25.964447021484375,
      "activations/layer22_attention_weight_max": 39.67332077026367,
      "activations/layer22_attention_weight_min": -33.01203536987305,
      "activations/layer23_attention_weight_max": 32.15618896484375,
      "activations/layer23_attention_weight_min": -25.011632919311523,
      "activations/layer2_attention_weight_max": 32.232643127441406,
      "activations/layer2_attention_weight_min": -34.04407501220703,
      "activations/layer3_attention_weight_max": 100.91040802001953,
      "activations/layer3_attention_weight_min": -106.02338409423828,
      "activations/layer4_attention_weight_max": 107.80704498291016,
      "activations/layer4_attention_weight_min": -110.9115982055664,
      "activations/layer5_attention_weight_max": 51.985595703125,
      "activations/layer5_attention_weight_min": -61.910560607910156,
      "activations/layer6_attention_weight_max": 45.53142166137695,
      "activations/layer6_attention_weight_min": -48.813716888427734,
      "activations/layer7_attention_weight_max": 94.21053314208984,
      "activations/layer7_attention_weight_min": -98.78240966796875,
      "activations/layer8_attention_weight_max": 45.71388626098633,
      "activations/layer8_attention_weight_min": -47.670860290527344,
      "activations/layer9_attention_weight_max": 32.4897346496582,
      "activations/layer9_attention_weight_min": -34.33659362792969,
      "epoch": 17.95,
      "learning_rate": 3.458143939393939e-05,
      "loss": 2.7382,
      "step": 308950
    },
    {
      "activations/layer0_attention_weight_max": 16.279075622558594,
      "activations/layer0_attention_weight_min": -15.051328659057617,
      "activations/layer10_attention_weight_max": 36.92926788330078,
      "activations/layer10_attention_weight_min": -32.49455261230469,
      "activations/layer11_attention_weight_max": 36.211585998535156,
      "activations/layer11_attention_weight_min": -33.328086853027344,
      "activations/layer12_attention_weight_max": 27.50851058959961,
      "activations/layer12_attention_weight_min": -26.72081184387207,
      "activations/layer13_attention_weight_max": 39.340755462646484,
      "activations/layer13_attention_weight_min": -36.53739547729492,
      "activations/layer14_attention_weight_max": 45.179847717285156,
      "activations/layer14_attention_weight_min": -42.36104965209961,
      "activations/layer15_attention_weight_max": 41.9705924987793,
      "activations/layer15_attention_weight_min": -36.89208221435547,
      "activations/layer16_attention_weight_max": 31.527246475219727,
      "activations/layer16_attention_weight_min": -28.791086196899414,
      "activations/layer17_attention_weight_max": 32.865142822265625,
      "activations/layer17_attention_weight_min": -29.74991226196289,
      "activations/layer18_attention_weight_max": 33.148345947265625,
      "activations/layer18_attention_weight_min": -27.274179458618164,
      "activations/layer19_attention_weight_max": 36.63914108276367,
      "activations/layer19_attention_weight_min": -33.59040832519531,
      "activations/layer1_attention_weight_max": 16.19364356994629,
      "activations/layer1_attention_weight_min": -15.333877563476562,
      "activations/layer20_attention_weight_max": 32.14295196533203,
      "activations/layer20_attention_weight_min": -26.523887634277344,
      "activations/layer21_attention_weight_max": 31.442277908325195,
      "activations/layer21_attention_weight_min": -25.6375789642334,
      "activations/layer22_attention_weight_max": 41.41080093383789,
      "activations/layer22_attention_weight_min": -34.6220588684082,
      "activations/layer23_attention_weight_max": 34.99933624267578,
      "activations/layer23_attention_weight_min": -29.70438003540039,
      "activations/layer2_attention_weight_max": 34.11204528808594,
      "activations/layer2_attention_weight_min": -32.43483352661133,
      "activations/layer3_attention_weight_max": 100.11386108398438,
      "activations/layer3_attention_weight_min": -101.9171371459961,
      "activations/layer4_attention_weight_max": 104.9386978149414,
      "activations/layer4_attention_weight_min": -104.40423583984375,
      "activations/layer5_attention_weight_max": 49.69605255126953,
      "activations/layer5_attention_weight_min": -62.765254974365234,
      "activations/layer6_attention_weight_max": 47.612937927246094,
      "activations/layer6_attention_weight_min": -47.843528747558594,
      "activations/layer7_attention_weight_max": 94.24029541015625,
      "activations/layer7_attention_weight_min": -94.7000961303711,
      "activations/layer8_attention_weight_max": 45.37310028076172,
      "activations/layer8_attention_weight_min": -46.82634735107422,
      "activations/layer9_attention_weight_max": 34.554603576660156,
      "activations/layer9_attention_weight_min": -33.871280670166016,
      "epoch": 17.95,
      "learning_rate": 3.456287878787878e-05,
      "loss": 2.7403,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.5233,
      "eval_samples_per_second": 503.797,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.5233,
      "eval_openwebtext_samples_per_second": 503.797,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.97,
      "eval_wikitext_samples_per_second": 231.471,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_lambada_loss": 2.373046875,
      "eval_lambada_ppl": 10.730035606033903,
      "eval_lambada_runtime": 9.5156,
      "eval_lambada_samples_per_second": 511.684,
      "step": 309000
    },
    {
      "activations/layer0_attention_weight_max": 15.623828887939453,
      "activations/layer0_attention_weight_min": -13.49911880493164,
      "activations/layer10_attention_weight_max": 37.83936309814453,
      "activations/layer10_attention_weight_min": -36.0112419128418,
      "activations/layer11_attention_weight_max": 37.96677017211914,
      "activations/layer11_attention_weight_min": -37.03547286987305,
      "activations/layer12_attention_weight_max": 28.92092514038086,
      "activations/layer12_attention_weight_min": -28.264909744262695,
      "activations/layer13_attention_weight_max": 40.807037353515625,
      "activations/layer13_attention_weight_min": -36.463226318359375,
      "activations/layer14_attention_weight_max": 43.442413330078125,
      "activations/layer14_attention_weight_min": -40.464385986328125,
      "activations/layer15_attention_weight_max": 38.774070739746094,
      "activations/layer15_attention_weight_min": -37.08098220825195,
      "activations/layer16_attention_weight_max": 26.87030601501465,
      "activations/layer16_attention_weight_min": -27.318212509155273,
      "activations/layer17_attention_weight_max": 27.99003028869629,
      "activations/layer17_attention_weight_min": -27.158130645751953,
      "activations/layer18_attention_weight_max": 29.58539390563965,
      "activations/layer18_attention_weight_min": -26.444522857666016,
      "activations/layer19_attention_weight_max": 31.33827781677246,
      "activations/layer19_attention_weight_min": -32.93764114379883,
      "activations/layer1_attention_weight_max": 15.573885917663574,
      "activations/layer1_attention_weight_min": -15.226028442382812,
      "activations/layer20_attention_weight_max": 27.437088012695312,
      "activations/layer20_attention_weight_min": -25.859956741333008,
      "activations/layer21_attention_weight_max": 27.6094970703125,
      "activations/layer21_attention_weight_min": -26.33597183227539,
      "activations/layer22_attention_weight_max": 37.23984909057617,
      "activations/layer22_attention_weight_min": -32.87826156616211,
      "activations/layer23_attention_weight_max": 30.46237564086914,
      "activations/layer23_attention_weight_min": -26.677976608276367,
      "activations/layer2_attention_weight_max": 33.307010650634766,
      "activations/layer2_attention_weight_min": -33.80974578857422,
      "activations/layer3_attention_weight_max": 99.12300109863281,
      "activations/layer3_attention_weight_min": -102.99697875976562,
      "activations/layer4_attention_weight_max": 106.46864318847656,
      "activations/layer4_attention_weight_min": -105.37147521972656,
      "activations/layer5_attention_weight_max": 49.82896041870117,
      "activations/layer5_attention_weight_min": -61.67036056518555,
      "activations/layer6_attention_weight_max": 46.210384368896484,
      "activations/layer6_attention_weight_min": -46.805294036865234,
      "activations/layer7_attention_weight_max": 93.15558624267578,
      "activations/layer7_attention_weight_min": -95.03337860107422,
      "activations/layer8_attention_weight_max": 43.04988098144531,
      "activations/layer8_attention_weight_min": -43.9628791809082,
      "activations/layer9_attention_weight_max": 35.35674285888672,
      "activations/layer9_attention_weight_min": -35.06267166137695,
      "epoch": 17.96,
      "learning_rate": 3.454393939393939e-05,
      "loss": 2.7357,
      "step": 309050
    },
    {
      "activations/layer0_attention_weight_max": 16.198081970214844,
      "activations/layer0_attention_weight_min": -12.847820281982422,
      "activations/layer10_attention_weight_max": 37.77885055541992,
      "activations/layer10_attention_weight_min": -35.7629280090332,
      "activations/layer11_attention_weight_max": 37.41182327270508,
      "activations/layer11_attention_weight_min": -37.700469970703125,
      "activations/layer12_attention_weight_max": 33.091636657714844,
      "activations/layer12_attention_weight_min": -30.786958694458008,
      "activations/layer13_attention_weight_max": 48.85293197631836,
      "activations/layer13_attention_weight_min": -38.36528396606445,
      "activations/layer14_attention_weight_max": 52.94001770019531,
      "activations/layer14_attention_weight_min": -43.245731353759766,
      "activations/layer15_attention_weight_max": 46.34952163696289,
      "activations/layer15_attention_weight_min": -40.16365432739258,
      "activations/layer16_attention_weight_max": 33.823211669921875,
      "activations/layer16_attention_weight_min": -28.973623275756836,
      "activations/layer17_attention_weight_max": 34.27602005004883,
      "activations/layer17_attention_weight_min": -28.230344772338867,
      "activations/layer18_attention_weight_max": 34.926109313964844,
      "activations/layer18_attention_weight_min": -28.021398544311523,
      "activations/layer19_attention_weight_max": 36.18991470336914,
      "activations/layer19_attention_weight_min": -35.58640670776367,
      "activations/layer1_attention_weight_max": 16.2724609375,
      "activations/layer1_attention_weight_min": -16.04582977294922,
      "activations/layer20_attention_weight_max": 32.48992156982422,
      "activations/layer20_attention_weight_min": -29.19390296936035,
      "activations/layer21_attention_weight_max": 31.288700103759766,
      "activations/layer21_attention_weight_min": -26.808645248413086,
      "activations/layer22_attention_weight_max": 48.05654525756836,
      "activations/layer22_attention_weight_min": -36.06337356567383,
      "activations/layer23_attention_weight_max": 33.524051666259766,
      "activations/layer23_attention_weight_min": -27.524181365966797,
      "activations/layer2_attention_weight_max": 35.489105224609375,
      "activations/layer2_attention_weight_min": -34.06842041015625,
      "activations/layer3_attention_weight_max": 105.2509765625,
      "activations/layer3_attention_weight_min": -107.43321990966797,
      "activations/layer4_attention_weight_max": 108.45726776123047,
      "activations/layer4_attention_weight_min": -109.7929916381836,
      "activations/layer5_attention_weight_max": 49.99604415893555,
      "activations/layer5_attention_weight_min": -61.50543975830078,
      "activations/layer6_attention_weight_max": 44.789608001708984,
      "activations/layer6_attention_weight_min": -48.65717697143555,
      "activations/layer7_attention_weight_max": 95.84505462646484,
      "activations/layer7_attention_weight_min": -109.04798889160156,
      "activations/layer8_attention_weight_max": 44.260433197021484,
      "activations/layer8_attention_weight_min": -45.80738067626953,
      "activations/layer9_attention_weight_max": 35.310516357421875,
      "activations/layer9_attention_weight_min": -36.027950286865234,
      "epoch": 17.96,
      "learning_rate": 3.4525e-05,
      "loss": 2.7305,
      "step": 309100
    },
    {
      "activations/layer0_attention_weight_max": 16.681442260742188,
      "activations/layer0_attention_weight_min": -15.936159133911133,
      "activations/layer10_attention_weight_max": 42.446807861328125,
      "activations/layer10_attention_weight_min": -34.538368225097656,
      "activations/layer11_attention_weight_max": 40.23252868652344,
      "activations/layer11_attention_weight_min": -36.176361083984375,
      "activations/layer12_attention_weight_max": 26.24724769592285,
      "activations/layer12_attention_weight_min": -26.313844680786133,
      "activations/layer13_attention_weight_max": 37.23878860473633,
      "activations/layer13_attention_weight_min": -34.84298324584961,
      "activations/layer14_attention_weight_max": 42.92317199707031,
      "activations/layer14_attention_weight_min": -39.380165100097656,
      "activations/layer15_attention_weight_max": 40.04169464111328,
      "activations/layer15_attention_weight_min": -37.68770217895508,
      "activations/layer16_attention_weight_max": 31.12481689453125,
      "activations/layer16_attention_weight_min": -29.67704963684082,
      "activations/layer17_attention_weight_max": 27.686471939086914,
      "activations/layer17_attention_weight_min": -25.568950653076172,
      "activations/layer18_attention_weight_max": 28.8453426361084,
      "activations/layer18_attention_weight_min": -26.21995735168457,
      "activations/layer19_attention_weight_max": 33.10189437866211,
      "activations/layer19_attention_weight_min": -30.72641944885254,
      "activations/layer1_attention_weight_max": 16.898305892944336,
      "activations/layer1_attention_weight_min": -14.193196296691895,
      "activations/layer20_attention_weight_max": 28.073978424072266,
      "activations/layer20_attention_weight_min": -24.030941009521484,
      "activations/layer21_attention_weight_max": 26.712657928466797,
      "activations/layer21_attention_weight_min": -23.79744529724121,
      "activations/layer22_attention_weight_max": 37.34766387939453,
      "activations/layer22_attention_weight_min": -29.65951919555664,
      "activations/layer23_attention_weight_max": 32.138099670410156,
      "activations/layer23_attention_weight_min": -25.510955810546875,
      "activations/layer2_attention_weight_max": 32.55873107910156,
      "activations/layer2_attention_weight_min": -31.976730346679688,
      "activations/layer3_attention_weight_max": 98.04100799560547,
      "activations/layer3_attention_weight_min": -95.07620239257812,
      "activations/layer4_attention_weight_max": 106.53614807128906,
      "activations/layer4_attention_weight_min": -103.4542236328125,
      "activations/layer5_attention_weight_max": 51.56919479370117,
      "activations/layer5_attention_weight_min": -62.705726623535156,
      "activations/layer6_attention_weight_max": 44.170963287353516,
      "activations/layer6_attention_weight_min": -45.73524475097656,
      "activations/layer7_attention_weight_max": 105.00140380859375,
      "activations/layer7_attention_weight_min": -97.66808319091797,
      "activations/layer8_attention_weight_max": 46.33050537109375,
      "activations/layer8_attention_weight_min": -45.26932907104492,
      "activations/layer9_attention_weight_max": 35.727630615234375,
      "activations/layer9_attention_weight_min": -34.8966178894043,
      "epoch": 17.96,
      "learning_rate": 3.45060606060606e-05,
      "loss": 2.7462,
      "step": 309150
    },
    {
      "activations/layer0_attention_weight_max": 16.004392623901367,
      "activations/layer0_attention_weight_min": -13.710448265075684,
      "activations/layer10_attention_weight_max": 34.16773986816406,
      "activations/layer10_attention_weight_min": -36.868038177490234,
      "activations/layer11_attention_weight_max": 33.98069763183594,
      "activations/layer11_attention_weight_min": -35.07148742675781,
      "activations/layer12_attention_weight_max": 25.82805061340332,
      "activations/layer12_attention_weight_min": -26.670000076293945,
      "activations/layer13_attention_weight_max": 40.75448226928711,
      "activations/layer13_attention_weight_min": -37.3453254699707,
      "activations/layer14_attention_weight_max": 42.73455810546875,
      "activations/layer14_attention_weight_min": -40.26051712036133,
      "activations/layer15_attention_weight_max": 39.44792175292969,
      "activations/layer15_attention_weight_min": -36.21690368652344,
      "activations/layer16_attention_weight_max": 29.499385833740234,
      "activations/layer16_attention_weight_min": -28.25126838684082,
      "activations/layer17_attention_weight_max": 28.4766788482666,
      "activations/layer17_attention_weight_min": -27.38311195373535,
      "activations/layer18_attention_weight_max": 30.015666961669922,
      "activations/layer18_attention_weight_min": -25.540353775024414,
      "activations/layer19_attention_weight_max": 34.50309753417969,
      "activations/layer19_attention_weight_min": -32.514732360839844,
      "activations/layer1_attention_weight_max": 16.425683975219727,
      "activations/layer1_attention_weight_min": -14.388004302978516,
      "activations/layer20_attention_weight_max": 28.795635223388672,
      "activations/layer20_attention_weight_min": -26.02297019958496,
      "activations/layer21_attention_weight_max": 32.184234619140625,
      "activations/layer21_attention_weight_min": -27.008207321166992,
      "activations/layer22_attention_weight_max": 43.095420837402344,
      "activations/layer22_attention_weight_min": -33.17831802368164,
      "activations/layer23_attention_weight_max": 33.87007522583008,
      "activations/layer23_attention_weight_min": -27.845611572265625,
      "activations/layer2_attention_weight_max": 35.06964874267578,
      "activations/layer2_attention_weight_min": -31.710554122924805,
      "activations/layer3_attention_weight_max": 103.30008697509766,
      "activations/layer3_attention_weight_min": -101.90631103515625,
      "activations/layer4_attention_weight_max": 106.57691955566406,
      "activations/layer4_attention_weight_min": -110.55609130859375,
      "activations/layer5_attention_weight_max": 48.53008270263672,
      "activations/layer5_attention_weight_min": -62.83488082885742,
      "activations/layer6_attention_weight_max": 45.29065704345703,
      "activations/layer6_attention_weight_min": -46.344688415527344,
      "activations/layer7_attention_weight_max": 94.4072494506836,
      "activations/layer7_attention_weight_min": -94.5207290649414,
      "activations/layer8_attention_weight_max": 40.60136032104492,
      "activations/layer8_attention_weight_min": -44.05196762084961,
      "activations/layer9_attention_weight_max": 32.745540618896484,
      "activations/layer9_attention_weight_min": -35.471553802490234,
      "epoch": 17.97,
      "learning_rate": 3.448712121212121e-05,
      "loss": 2.7412,
      "step": 309200
    },
    {
      "activations/layer0_attention_weight_max": 17.819175720214844,
      "activations/layer0_attention_weight_min": -12.837635040283203,
      "activations/layer10_attention_weight_max": 32.64292907714844,
      "activations/layer10_attention_weight_min": -33.28179931640625,
      "activations/layer11_attention_weight_max": 33.389286041259766,
      "activations/layer11_attention_weight_min": -33.08449172973633,
      "activations/layer12_attention_weight_max": 26.487375259399414,
      "activations/layer12_attention_weight_min": -27.85123062133789,
      "activations/layer13_attention_weight_max": 38.0358772277832,
      "activations/layer13_attention_weight_min": -37.18223571777344,
      "activations/layer14_attention_weight_max": 43.886138916015625,
      "activations/layer14_attention_weight_min": -40.17158889770508,
      "activations/layer15_attention_weight_max": 39.15465545654297,
      "activations/layer15_attention_weight_min": -38.79914855957031,
      "activations/layer16_attention_weight_max": 31.982860565185547,
      "activations/layer16_attention_weight_min": -30.86064910888672,
      "activations/layer17_attention_weight_max": 29.85279655456543,
      "activations/layer17_attention_weight_min": -31.83513832092285,
      "activations/layer18_attention_weight_max": 33.85057830810547,
      "activations/layer18_attention_weight_min": -29.513710021972656,
      "activations/layer19_attention_weight_max": 36.083438873291016,
      "activations/layer19_attention_weight_min": -33.21430587768555,
      "activations/layer1_attention_weight_max": 16.4044246673584,
      "activations/layer1_attention_weight_min": -14.021559715270996,
      "activations/layer20_attention_weight_max": 30.663408279418945,
      "activations/layer20_attention_weight_min": -26.696321487426758,
      "activations/layer21_attention_weight_max": 33.8804817199707,
      "activations/layer21_attention_weight_min": -26.571462631225586,
      "activations/layer22_attention_weight_max": 46.09841537475586,
      "activations/layer22_attention_weight_min": -33.232357025146484,
      "activations/layer23_attention_weight_max": 33.26515197753906,
      "activations/layer23_attention_weight_min": -26.816856384277344,
      "activations/layer2_attention_weight_max": 33.90858459472656,
      "activations/layer2_attention_weight_min": -31.92047691345215,
      "activations/layer3_attention_weight_max": 100.80526733398438,
      "activations/layer3_attention_weight_min": -99.32347869873047,
      "activations/layer4_attention_weight_max": 110.49040985107422,
      "activations/layer4_attention_weight_min": -107.07411193847656,
      "activations/layer5_attention_weight_max": 48.758567810058594,
      "activations/layer5_attention_weight_min": -61.90755844116211,
      "activations/layer6_attention_weight_max": 42.595760345458984,
      "activations/layer6_attention_weight_min": -47.76663589477539,
      "activations/layer7_attention_weight_max": 92.18141174316406,
      "activations/layer7_attention_weight_min": -92.61766052246094,
      "activations/layer8_attention_weight_max": 39.7714729309082,
      "activations/layer8_attention_weight_min": -40.84257507324219,
      "activations/layer9_attention_weight_max": 31.269765853881836,
      "activations/layer9_attention_weight_min": -32.72451400756836,
      "epoch": 17.97,
      "learning_rate": 3.446818181818182e-05,
      "loss": 2.7532,
      "step": 309250
    },
    {
      "activations/layer0_attention_weight_max": 15.798993110656738,
      "activations/layer0_attention_weight_min": -12.869245529174805,
      "activations/layer10_attention_weight_max": 31.467647552490234,
      "activations/layer10_attention_weight_min": -34.37095260620117,
      "activations/layer11_attention_weight_max": 32.485050201416016,
      "activations/layer11_attention_weight_min": -33.79729461669922,
      "activations/layer12_attention_weight_max": 26.652997970581055,
      "activations/layer12_attention_weight_min": -25.934640884399414,
      "activations/layer13_attention_weight_max": 37.592010498046875,
      "activations/layer13_attention_weight_min": -33.733489990234375,
      "activations/layer14_attention_weight_max": 38.992252349853516,
      "activations/layer14_attention_weight_min": -36.914180755615234,
      "activations/layer15_attention_weight_max": 36.02299118041992,
      "activations/layer15_attention_weight_min": -35.61821365356445,
      "activations/layer16_attention_weight_max": 27.14816665649414,
      "activations/layer16_attention_weight_min": -28.12226104736328,
      "activations/layer17_attention_weight_max": 32.22964859008789,
      "activations/layer17_attention_weight_min": -26.89006805419922,
      "activations/layer18_attention_weight_max": 32.987548828125,
      "activations/layer18_attention_weight_min": -27.4173526763916,
      "activations/layer19_attention_weight_max": 31.48948097229004,
      "activations/layer19_attention_weight_min": -32.06360626220703,
      "activations/layer1_attention_weight_max": 17.856121063232422,
      "activations/layer1_attention_weight_min": -14.596776008605957,
      "activations/layer20_attention_weight_max": 26.937633514404297,
      "activations/layer20_attention_weight_min": -27.732114791870117,
      "activations/layer21_attention_weight_max": 28.189218521118164,
      "activations/layer21_attention_weight_min": -26.305124282836914,
      "activations/layer22_attention_weight_max": 39.30941390991211,
      "activations/layer22_attention_weight_min": -30.397417068481445,
      "activations/layer23_attention_weight_max": 32.476558685302734,
      "activations/layer23_attention_weight_min": -25.42749786376953,
      "activations/layer2_attention_weight_max": 31.09089469909668,
      "activations/layer2_attention_weight_min": -30.859272003173828,
      "activations/layer3_attention_weight_max": 99.82689666748047,
      "activations/layer3_attention_weight_min": -96.14169311523438,
      "activations/layer4_attention_weight_max": 101.38336181640625,
      "activations/layer4_attention_weight_min": -100.03630828857422,
      "activations/layer5_attention_weight_max": 45.74050521850586,
      "activations/layer5_attention_weight_min": -60.859615325927734,
      "activations/layer6_attention_weight_max": 40.752315521240234,
      "activations/layer6_attention_weight_min": -43.339576721191406,
      "activations/layer7_attention_weight_max": 85.17289733886719,
      "activations/layer7_attention_weight_min": -86.96176147460938,
      "activations/layer8_attention_weight_max": 40.004432678222656,
      "activations/layer8_attention_weight_min": -43.07292556762695,
      "activations/layer9_attention_weight_max": 31.1866397857666,
      "activations/layer9_attention_weight_min": -35.42771530151367,
      "epoch": 17.97,
      "learning_rate": 3.444924242424242e-05,
      "loss": 2.7171,
      "step": 309300
    },
    {
      "activations/layer0_attention_weight_max": 16.736528396606445,
      "activations/layer0_attention_weight_min": -12.79727840423584,
      "activations/layer10_attention_weight_max": 37.13434982299805,
      "activations/layer10_attention_weight_min": -36.85016632080078,
      "activations/layer11_attention_weight_max": 37.822540283203125,
      "activations/layer11_attention_weight_min": -36.862876892089844,
      "activations/layer12_attention_weight_max": 31.08498764038086,
      "activations/layer12_attention_weight_min": -28.353952407836914,
      "activations/layer13_attention_weight_max": 47.62100601196289,
      "activations/layer13_attention_weight_min": -37.86511993408203,
      "activations/layer14_attention_weight_max": 51.270999908447266,
      "activations/layer14_attention_weight_min": -40.69820785522461,
      "activations/layer15_attention_weight_max": 51.23657989501953,
      "activations/layer15_attention_weight_min": -35.57075881958008,
      "activations/layer16_attention_weight_max": 33.80138397216797,
      "activations/layer16_attention_weight_min": -30.290969848632812,
      "activations/layer17_attention_weight_max": 32.87599563598633,
      "activations/layer17_attention_weight_min": -25.897907257080078,
      "activations/layer18_attention_weight_max": 30.3145809173584,
      "activations/layer18_attention_weight_min": -26.51913070678711,
      "activations/layer19_attention_weight_max": 31.192197799682617,
      "activations/layer19_attention_weight_min": -33.1955680847168,
      "activations/layer1_attention_weight_max": 17.271217346191406,
      "activations/layer1_attention_weight_min": -15.944979667663574,
      "activations/layer20_attention_weight_max": 31.10192108154297,
      "activations/layer20_attention_weight_min": -27.11251449584961,
      "activations/layer21_attention_weight_max": 27.00740623474121,
      "activations/layer21_attention_weight_min": -27.972082138061523,
      "activations/layer22_attention_weight_max": 40.38991928100586,
      "activations/layer22_attention_weight_min": -37.362464904785156,
      "activations/layer23_attention_weight_max": 28.891185760498047,
      "activations/layer23_attention_weight_min": -26.5148983001709,
      "activations/layer2_attention_weight_max": 35.313987731933594,
      "activations/layer2_attention_weight_min": -34.862525939941406,
      "activations/layer3_attention_weight_max": 106.40555572509766,
      "activations/layer3_attention_weight_min": -108.17220306396484,
      "activations/layer4_attention_weight_max": 104.06719207763672,
      "activations/layer4_attention_weight_min": -111.44073486328125,
      "activations/layer5_attention_weight_max": 50.188438415527344,
      "activations/layer5_attention_weight_min": -59.912227630615234,
      "activations/layer6_attention_weight_max": 45.076393127441406,
      "activations/layer6_attention_weight_min": -47.800270080566406,
      "activations/layer7_attention_weight_max": 95.2103271484375,
      "activations/layer7_attention_weight_min": -98.86478424072266,
      "activations/layer8_attention_weight_max": 44.31980895996094,
      "activations/layer8_attention_weight_min": -47.44076919555664,
      "activations/layer9_attention_weight_max": 35.50275802612305,
      "activations/layer9_attention_weight_min": -36.17388153076172,
      "epoch": 17.97,
      "learning_rate": 3.443030303030303e-05,
      "loss": 2.7328,
      "step": 309350
    },
    {
      "activations/layer0_attention_weight_max": 16.029193878173828,
      "activations/layer0_attention_weight_min": -13.346169471740723,
      "activations/layer10_attention_weight_max": 32.298866271972656,
      "activations/layer10_attention_weight_min": -35.976234436035156,
      "activations/layer11_attention_weight_max": 33.45824432373047,
      "activations/layer11_attention_weight_min": -35.8087158203125,
      "activations/layer12_attention_weight_max": 27.08161735534668,
      "activations/layer12_attention_weight_min": -29.548717498779297,
      "activations/layer13_attention_weight_max": 41.49531555175781,
      "activations/layer13_attention_weight_min": -38.48961639404297,
      "activations/layer14_attention_weight_max": 43.689125061035156,
      "activations/layer14_attention_weight_min": -42.27204513549805,
      "activations/layer15_attention_weight_max": 40.65575408935547,
      "activations/layer15_attention_weight_min": -35.702754974365234,
      "activations/layer16_attention_weight_max": 31.374723434448242,
      "activations/layer16_attention_weight_min": -27.658226013183594,
      "activations/layer17_attention_weight_max": 32.35362243652344,
      "activations/layer17_attention_weight_min": -25.04073143005371,
      "activations/layer18_attention_weight_max": 30.564409255981445,
      "activations/layer18_attention_weight_min": -24.918323516845703,
      "activations/layer19_attention_weight_max": 35.4957389831543,
      "activations/layer19_attention_weight_min": -31.157032012939453,
      "activations/layer1_attention_weight_max": 20.080596923828125,
      "activations/layer1_attention_weight_min": -17.974328994750977,
      "activations/layer20_attention_weight_max": 30.919042587280273,
      "activations/layer20_attention_weight_min": -26.78506088256836,
      "activations/layer21_attention_weight_max": 29.408016204833984,
      "activations/layer21_attention_weight_min": -24.967533111572266,
      "activations/layer22_attention_weight_max": 44.02388000488281,
      "activations/layer22_attention_weight_min": -33.830623626708984,
      "activations/layer23_attention_weight_max": 37.61255645751953,
      "activations/layer23_attention_weight_min": -27.95665740966797,
      "activations/layer2_attention_weight_max": 33.61278533935547,
      "activations/layer2_attention_weight_min": -33.61186981201172,
      "activations/layer3_attention_weight_max": 99.70025634765625,
      "activations/layer3_attention_weight_min": -108.87183380126953,
      "activations/layer4_attention_weight_max": 104.73673248291016,
      "activations/layer4_attention_weight_min": -109.05000305175781,
      "activations/layer5_attention_weight_max": 50.04637908935547,
      "activations/layer5_attention_weight_min": -61.573272705078125,
      "activations/layer6_attention_weight_max": 44.36940002441406,
      "activations/layer6_attention_weight_min": -47.216468811035156,
      "activations/layer7_attention_weight_max": 93.84098052978516,
      "activations/layer7_attention_weight_min": -101.11900329589844,
      "activations/layer8_attention_weight_max": 41.645408630371094,
      "activations/layer8_attention_weight_min": -47.202457427978516,
      "activations/layer9_attention_weight_max": 32.01565170288086,
      "activations/layer9_attention_weight_min": -37.08980941772461,
      "epoch": 17.98,
      "learning_rate": 3.441136363636363e-05,
      "loss": 2.7264,
      "step": 309400
    },
    {
      "activations/layer0_attention_weight_max": 16.09598159790039,
      "activations/layer0_attention_weight_min": -13.196857452392578,
      "activations/layer10_attention_weight_max": 35.57748794555664,
      "activations/layer10_attention_weight_min": -35.01634216308594,
      "activations/layer11_attention_weight_max": 37.36041259765625,
      "activations/layer11_attention_weight_min": -33.097023010253906,
      "activations/layer12_attention_weight_max": 29.12602424621582,
      "activations/layer12_attention_weight_min": -26.69632339477539,
      "activations/layer13_attention_weight_max": 40.695350646972656,
      "activations/layer13_attention_weight_min": -35.922080993652344,
      "activations/layer14_attention_weight_max": 42.78348159790039,
      "activations/layer14_attention_weight_min": -39.036102294921875,
      "activations/layer15_attention_weight_max": 39.046478271484375,
      "activations/layer15_attention_weight_min": -35.6124382019043,
      "activations/layer16_attention_weight_max": 29.641347885131836,
      "activations/layer16_attention_weight_min": -28.85879898071289,
      "activations/layer17_attention_weight_max": 30.151649475097656,
      "activations/layer17_attention_weight_min": -28.09984016418457,
      "activations/layer18_attention_weight_max": 31.6849422454834,
      "activations/layer18_attention_weight_min": -27.01491928100586,
      "activations/layer19_attention_weight_max": 34.534332275390625,
      "activations/layer19_attention_weight_min": -31.162200927734375,
      "activations/layer1_attention_weight_max": 18.16617774963379,
      "activations/layer1_attention_weight_min": -16.36128044128418,
      "activations/layer20_attention_weight_max": 28.827674865722656,
      "activations/layer20_attention_weight_min": -24.473514556884766,
      "activations/layer21_attention_weight_max": 28.905508041381836,
      "activations/layer21_attention_weight_min": -25.63626480102539,
      "activations/layer22_attention_weight_max": 46.01224899291992,
      "activations/layer22_attention_weight_min": -32.03321838378906,
      "activations/layer23_attention_weight_max": 32.34734344482422,
      "activations/layer23_attention_weight_min": -28.503284454345703,
      "activations/layer2_attention_weight_max": 34.951805114746094,
      "activations/layer2_attention_weight_min": -32.871761322021484,
      "activations/layer3_attention_weight_max": 99.00456237792969,
      "activations/layer3_attention_weight_min": -100.93504333496094,
      "activations/layer4_attention_weight_max": 102.56591796875,
      "activations/layer4_attention_weight_min": -106.98198699951172,
      "activations/layer5_attention_weight_max": 48.43020248413086,
      "activations/layer5_attention_weight_min": -65.05766296386719,
      "activations/layer6_attention_weight_max": 43.775245666503906,
      "activations/layer6_attention_weight_min": -45.56433868408203,
      "activations/layer7_attention_weight_max": 96.71652221679688,
      "activations/layer7_attention_weight_min": -95.01275634765625,
      "activations/layer8_attention_weight_max": 41.25237274169922,
      "activations/layer8_attention_weight_min": -43.11653518676758,
      "activations/layer9_attention_weight_max": 36.01454162597656,
      "activations/layer9_attention_weight_min": -35.438228607177734,
      "epoch": 17.98,
      "learning_rate": 3.4392424242424236e-05,
      "loss": 2.7303,
      "step": 309450
    },
    {
      "activations/layer0_attention_weight_max": 15.63460636138916,
      "activations/layer0_attention_weight_min": -13.401346206665039,
      "activations/layer10_attention_weight_max": 38.368568420410156,
      "activations/layer10_attention_weight_min": -37.933631896972656,
      "activations/layer11_attention_weight_max": 35.96827697753906,
      "activations/layer11_attention_weight_min": -35.378395080566406,
      "activations/layer12_attention_weight_max": 33.575706481933594,
      "activations/layer12_attention_weight_min": -27.000934600830078,
      "activations/layer13_attention_weight_max": 42.05510330200195,
      "activations/layer13_attention_weight_min": -36.46340560913086,
      "activations/layer14_attention_weight_max": 42.469703674316406,
      "activations/layer14_attention_weight_min": -38.33669662475586,
      "activations/layer15_attention_weight_max": 39.06156921386719,
      "activations/layer15_attention_weight_min": -34.58154296875,
      "activations/layer16_attention_weight_max": 29.709692001342773,
      "activations/layer16_attention_weight_min": -29.98181915283203,
      "activations/layer17_attention_weight_max": 28.767805099487305,
      "activations/layer17_attention_weight_min": -26.821603775024414,
      "activations/layer18_attention_weight_max": 31.038175582885742,
      "activations/layer18_attention_weight_min": -25.779560089111328,
      "activations/layer19_attention_weight_max": 31.630680084228516,
      "activations/layer19_attention_weight_min": -31.83614158630371,
      "activations/layer1_attention_weight_max": 18.198429107666016,
      "activations/layer1_attention_weight_min": -17.484338760375977,
      "activations/layer20_attention_weight_max": 28.093259811401367,
      "activations/layer20_attention_weight_min": -25.991798400878906,
      "activations/layer21_attention_weight_max": 28.717777252197266,
      "activations/layer21_attention_weight_min": -24.77274513244629,
      "activations/layer22_attention_weight_max": 41.196083068847656,
      "activations/layer22_attention_weight_min": -30.993751525878906,
      "activations/layer23_attention_weight_max": 31.42723846435547,
      "activations/layer23_attention_weight_min": -25.49608612060547,
      "activations/layer2_attention_weight_max": 32.65129089355469,
      "activations/layer2_attention_weight_min": -30.91352081298828,
      "activations/layer3_attention_weight_max": 99.56588745117188,
      "activations/layer3_attention_weight_min": -99.21441650390625,
      "activations/layer4_attention_weight_max": 108.344482421875,
      "activations/layer4_attention_weight_min": -110.23457336425781,
      "activations/layer5_attention_weight_max": 49.93370056152344,
      "activations/layer5_attention_weight_min": -60.49711227416992,
      "activations/layer6_attention_weight_max": 45.31401443481445,
      "activations/layer6_attention_weight_min": -47.99197006225586,
      "activations/layer7_attention_weight_max": 99.33911895751953,
      "activations/layer7_attention_weight_min": -101.3424301147461,
      "activations/layer8_attention_weight_max": 45.56545639038086,
      "activations/layer8_attention_weight_min": -44.276790618896484,
      "activations/layer9_attention_weight_max": 36.05113220214844,
      "activations/layer9_attention_weight_min": -36.40897750854492,
      "epoch": 17.98,
      "learning_rate": 3.4373484848484845e-05,
      "loss": 2.7077,
      "step": 309500
    },
    {
      "activations/layer0_attention_weight_max": 16.460729598999023,
      "activations/layer0_attention_weight_min": -13.151321411132812,
      "activations/layer10_attention_weight_max": 34.59479904174805,
      "activations/layer10_attention_weight_min": -37.40180587768555,
      "activations/layer11_attention_weight_max": 36.56711196899414,
      "activations/layer11_attention_weight_min": -33.824951171875,
      "activations/layer12_attention_weight_max": 29.94680404663086,
      "activations/layer12_attention_weight_min": -29.76580047607422,
      "activations/layer13_attention_weight_max": 44.40721130371094,
      "activations/layer13_attention_weight_min": -39.07746124267578,
      "activations/layer14_attention_weight_max": 47.041664123535156,
      "activations/layer14_attention_weight_min": -39.4254264831543,
      "activations/layer15_attention_weight_max": 41.96553039550781,
      "activations/layer15_attention_weight_min": -35.98596954345703,
      "activations/layer16_attention_weight_max": 29.159502029418945,
      "activations/layer16_attention_weight_min": -29.953083038330078,
      "activations/layer17_attention_weight_max": 28.131296157836914,
      "activations/layer17_attention_weight_min": -25.991086959838867,
      "activations/layer18_attention_weight_max": 31.027803421020508,
      "activations/layer18_attention_weight_min": -26.439788818359375,
      "activations/layer19_attention_weight_max": 35.81710433959961,
      "activations/layer19_attention_weight_min": -32.98629379272461,
      "activations/layer1_attention_weight_max": 17.476757049560547,
      "activations/layer1_attention_weight_min": -16.215579986572266,
      "activations/layer20_attention_weight_max": 27.48442840576172,
      "activations/layer20_attention_weight_min": -25.015867233276367,
      "activations/layer21_attention_weight_max": 27.2576961517334,
      "activations/layer21_attention_weight_min": -24.8109073638916,
      "activations/layer22_attention_weight_max": 43.168128967285156,
      "activations/layer22_attention_weight_min": -32.95560836791992,
      "activations/layer23_attention_weight_max": 31.235687255859375,
      "activations/layer23_attention_weight_min": -26.9443416595459,
      "activations/layer2_attention_weight_max": 32.695499420166016,
      "activations/layer2_attention_weight_min": -33.399784088134766,
      "activations/layer3_attention_weight_max": 99.1830825805664,
      "activations/layer3_attention_weight_min": -102.30008697509766,
      "activations/layer4_attention_weight_max": 106.91157531738281,
      "activations/layer4_attention_weight_min": -108.27233123779297,
      "activations/layer5_attention_weight_max": 50.5160026550293,
      "activations/layer5_attention_weight_min": -63.2503776550293,
      "activations/layer6_attention_weight_max": 44.8170051574707,
      "activations/layer6_attention_weight_min": -49.21652603149414,
      "activations/layer7_attention_weight_max": 97.3265609741211,
      "activations/layer7_attention_weight_min": -99.28349304199219,
      "activations/layer8_attention_weight_max": 41.30910110473633,
      "activations/layer8_attention_weight_min": -48.75127029418945,
      "activations/layer9_attention_weight_max": 35.55373764038086,
      "activations/layer9_attention_weight_min": -38.350643157958984,
      "epoch": 17.99,
      "learning_rate": 3.435454545454545e-05,
      "loss": 2.7512,
      "step": 309550
    },
    {
      "activations/layer0_attention_weight_max": 16.635404586791992,
      "activations/layer0_attention_weight_min": -13.767991065979004,
      "activations/layer10_attention_weight_max": 37.813926696777344,
      "activations/layer10_attention_weight_min": -35.1651725769043,
      "activations/layer11_attention_weight_max": 38.75382995605469,
      "activations/layer11_attention_weight_min": -36.728607177734375,
      "activations/layer12_attention_weight_max": 29.726228713989258,
      "activations/layer12_attention_weight_min": -26.821327209472656,
      "activations/layer13_attention_weight_max": 44.777992248535156,
      "activations/layer13_attention_weight_min": -35.905799865722656,
      "activations/layer14_attention_weight_max": 43.44792556762695,
      "activations/layer14_attention_weight_min": -40.287391662597656,
      "activations/layer15_attention_weight_max": 39.56308364868164,
      "activations/layer15_attention_weight_min": -35.28304672241211,
      "activations/layer16_attention_weight_max": 28.840227127075195,
      "activations/layer16_attention_weight_min": -27.992387771606445,
      "activations/layer17_attention_weight_max": 32.59622573852539,
      "activations/layer17_attention_weight_min": -27.12238121032715,
      "activations/layer18_attention_weight_max": 32.17850112915039,
      "activations/layer18_attention_weight_min": -29.285226821899414,
      "activations/layer19_attention_weight_max": 33.90297317504883,
      "activations/layer19_attention_weight_min": -36.396358489990234,
      "activations/layer1_attention_weight_max": 17.370532989501953,
      "activations/layer1_attention_weight_min": -19.1704044342041,
      "activations/layer20_attention_weight_max": 30.002635955810547,
      "activations/layer20_attention_weight_min": -28.483470916748047,
      "activations/layer21_attention_weight_max": 29.89291000366211,
      "activations/layer21_attention_weight_min": -28.35861587524414,
      "activations/layer22_attention_weight_max": 46.18132781982422,
      "activations/layer22_attention_weight_min": -37.51707077026367,
      "activations/layer23_attention_weight_max": 35.02104187011719,
      "activations/layer23_attention_weight_min": -30.881559371948242,
      "activations/layer2_attention_weight_max": 34.45981216430664,
      "activations/layer2_attention_weight_min": -33.680076599121094,
      "activations/layer3_attention_weight_max": 102.20890045166016,
      "activations/layer3_attention_weight_min": -102.2242202758789,
      "activations/layer4_attention_weight_max": 108.1493911743164,
      "activations/layer4_attention_weight_min": -107.60469055175781,
      "activations/layer5_attention_weight_max": 50.123748779296875,
      "activations/layer5_attention_weight_min": -61.91709899902344,
      "activations/layer6_attention_weight_max": 45.77263641357422,
      "activations/layer6_attention_weight_min": -45.25708770751953,
      "activations/layer7_attention_weight_max": 96.76478576660156,
      "activations/layer7_attention_weight_min": -94.39500427246094,
      "activations/layer8_attention_weight_max": 44.50311279296875,
      "activations/layer8_attention_weight_min": -42.96754837036133,
      "activations/layer9_attention_weight_max": 34.649749755859375,
      "activations/layer9_attention_weight_min": -33.76239776611328,
      "epoch": 17.99,
      "learning_rate": 3.433560606060606e-05,
      "loss": 2.7216,
      "step": 309600
    },
    {
      "activations/layer0_attention_weight_max": 16.21937370300293,
      "activations/layer0_attention_weight_min": -11.915733337402344,
      "activations/layer10_attention_weight_max": 38.5250358581543,
      "activations/layer10_attention_weight_min": -36.29690933227539,
      "activations/layer11_attention_weight_max": 35.41557312011719,
      "activations/layer11_attention_weight_min": -36.29818344116211,
      "activations/layer12_attention_weight_max": 29.373409271240234,
      "activations/layer12_attention_weight_min": -28.172927856445312,
      "activations/layer13_attention_weight_max": 44.287452697753906,
      "activations/layer13_attention_weight_min": -40.3288459777832,
      "activations/layer14_attention_weight_max": 44.90312957763672,
      "activations/layer14_attention_weight_min": -44.05609130859375,
      "activations/layer15_attention_weight_max": 43.53034591674805,
      "activations/layer15_attention_weight_min": -37.24753189086914,
      "activations/layer16_attention_weight_max": 32.2752571105957,
      "activations/layer16_attention_weight_min": -28.104366302490234,
      "activations/layer17_attention_weight_max": 32.00926208496094,
      "activations/layer17_attention_weight_min": -27.6160945892334,
      "activations/layer18_attention_weight_max": 32.52726745605469,
      "activations/layer18_attention_weight_min": -26.09752082824707,
      "activations/layer19_attention_weight_max": 36.64994430541992,
      "activations/layer19_attention_weight_min": -32.7159538269043,
      "activations/layer1_attention_weight_max": 19.12143898010254,
      "activations/layer1_attention_weight_min": -17.622018814086914,
      "activations/layer20_attention_weight_max": 32.5296745300293,
      "activations/layer20_attention_weight_min": -25.0729923248291,
      "activations/layer21_attention_weight_max": 31.420324325561523,
      "activations/layer21_attention_weight_min": -24.271526336669922,
      "activations/layer22_attention_weight_max": 47.82743453979492,
      "activations/layer22_attention_weight_min": -33.349365234375,
      "activations/layer23_attention_weight_max": 38.0954475402832,
      "activations/layer23_attention_weight_min": -26.130741119384766,
      "activations/layer2_attention_weight_max": 34.11072540283203,
      "activations/layer2_attention_weight_min": -32.81517028808594,
      "activations/layer3_attention_weight_max": 100.90583038330078,
      "activations/layer3_attention_weight_min": -103.70173645019531,
      "activations/layer4_attention_weight_max": 112.08843231201172,
      "activations/layer4_attention_weight_min": -111.49049377441406,
      "activations/layer5_attention_weight_max": 51.77762985229492,
      "activations/layer5_attention_weight_min": -60.25361251831055,
      "activations/layer6_attention_weight_max": 45.89906311035156,
      "activations/layer6_attention_weight_min": -50.31169891357422,
      "activations/layer7_attention_weight_max": 99.07672882080078,
      "activations/layer7_attention_weight_min": -99.26235961914062,
      "activations/layer8_attention_weight_max": 44.7336540222168,
      "activations/layer8_attention_weight_min": -46.563201904296875,
      "activations/layer9_attention_weight_max": 36.05012893676758,
      "activations/layer9_attention_weight_min": -36.995811462402344,
      "epoch": 17.99,
      "learning_rate": 3.431666666666666e-05,
      "loss": 2.7403,
      "step": 309650
    },
    {
      "activations/layer0_attention_weight_max": 15.501794815063477,
      "activations/layer0_attention_weight_min": -14.096399307250977,
      "activations/layer10_attention_weight_max": 34.31659698486328,
      "activations/layer10_attention_weight_min": -34.54655456542969,
      "activations/layer11_attention_weight_max": 33.47993850708008,
      "activations/layer11_attention_weight_min": -35.45250701904297,
      "activations/layer12_attention_weight_max": 26.90361213684082,
      "activations/layer12_attention_weight_min": -27.404340744018555,
      "activations/layer13_attention_weight_max": 39.5821647644043,
      "activations/layer13_attention_weight_min": -36.41703414916992,
      "activations/layer14_attention_weight_max": 41.091957092285156,
      "activations/layer14_attention_weight_min": -39.079551696777344,
      "activations/layer15_attention_weight_max": 37.23664474487305,
      "activations/layer15_attention_weight_min": -35.628543853759766,
      "activations/layer16_attention_weight_max": 31.416988372802734,
      "activations/layer16_attention_weight_min": -28.932769775390625,
      "activations/layer17_attention_weight_max": 31.203866958618164,
      "activations/layer17_attention_weight_min": -28.49032211303711,
      "activations/layer18_attention_weight_max": 33.003814697265625,
      "activations/layer18_attention_weight_min": -26.35321044921875,
      "activations/layer19_attention_weight_max": 35.145442962646484,
      "activations/layer19_attention_weight_min": -30.878803253173828,
      "activations/layer1_attention_weight_max": 16.645803451538086,
      "activations/layer1_attention_weight_min": -16.530065536499023,
      "activations/layer20_attention_weight_max": 32.812557220458984,
      "activations/layer20_attention_weight_min": -26.176990509033203,
      "activations/layer21_attention_weight_max": 28.497432708740234,
      "activations/layer21_attention_weight_min": -25.881187438964844,
      "activations/layer22_attention_weight_max": 42.427978515625,
      "activations/layer22_attention_weight_min": -30.885129928588867,
      "activations/layer23_attention_weight_max": 33.73525619506836,
      "activations/layer23_attention_weight_min": -28.780332565307617,
      "activations/layer2_attention_weight_max": 32.57299041748047,
      "activations/layer2_attention_weight_min": -31.818180084228516,
      "activations/layer3_attention_weight_max": 100.45928955078125,
      "activations/layer3_attention_weight_min": -100.32598876953125,
      "activations/layer4_attention_weight_max": 105.32808685302734,
      "activations/layer4_attention_weight_min": -107.44022369384766,
      "activations/layer5_attention_weight_max": 49.755611419677734,
      "activations/layer5_attention_weight_min": -61.18744659423828,
      "activations/layer6_attention_weight_max": 43.504615783691406,
      "activations/layer6_attention_weight_min": -46.77933883666992,
      "activations/layer7_attention_weight_max": 93.07693481445312,
      "activations/layer7_attention_weight_min": -95.92417907714844,
      "activations/layer8_attention_weight_max": 40.95316696166992,
      "activations/layer8_attention_weight_min": -44.31772232055664,
      "activations/layer9_attention_weight_max": 30.863101959228516,
      "activations/layer9_attention_weight_min": -35.653465270996094,
      "epoch": 18.0,
      "learning_rate": 3.429772727272727e-05,
      "loss": 2.7278,
      "step": 309700
    },
    {
      "activations/layer0_attention_weight_max": 16.269474029541016,
      "activations/layer0_attention_weight_min": -13.332443237304688,
      "activations/layer10_attention_weight_max": 35.31889343261719,
      "activations/layer10_attention_weight_min": -37.204383850097656,
      "activations/layer11_attention_weight_max": 35.099796295166016,
      "activations/layer11_attention_weight_min": -36.27241516113281,
      "activations/layer12_attention_weight_max": 27.218271255493164,
      "activations/layer12_attention_weight_min": -27.718124389648438,
      "activations/layer13_attention_weight_max": 39.13072967529297,
      "activations/layer13_attention_weight_min": -36.54216384887695,
      "activations/layer14_attention_weight_max": 40.66895294189453,
      "activations/layer14_attention_weight_min": -41.463932037353516,
      "activations/layer15_attention_weight_max": 35.841732025146484,
      "activations/layer15_attention_weight_min": -38.435115814208984,
      "activations/layer16_attention_weight_max": 28.655649185180664,
      "activations/layer16_attention_weight_min": -29.211849212646484,
      "activations/layer17_attention_weight_max": 28.237512588500977,
      "activations/layer17_attention_weight_min": -26.887969970703125,
      "activations/layer18_attention_weight_max": 30.115192413330078,
      "activations/layer18_attention_weight_min": -25.912704467773438,
      "activations/layer19_attention_weight_max": 37.52724838256836,
      "activations/layer19_attention_weight_min": -33.24456024169922,
      "activations/layer1_attention_weight_max": 15.951770782470703,
      "activations/layer1_attention_weight_min": -15.820873260498047,
      "activations/layer20_attention_weight_max": 32.83020782470703,
      "activations/layer20_attention_weight_min": -27.035404205322266,
      "activations/layer21_attention_weight_max": 31.156204223632812,
      "activations/layer21_attention_weight_min": -25.771574020385742,
      "activations/layer22_attention_weight_max": 43.7411994934082,
      "activations/layer22_attention_weight_min": -32.027896881103516,
      "activations/layer23_attention_weight_max": 35.3409423828125,
      "activations/layer23_attention_weight_min": -26.107391357421875,
      "activations/layer2_attention_weight_max": 32.5786247253418,
      "activations/layer2_attention_weight_min": -32.236793518066406,
      "activations/layer3_attention_weight_max": 99.89817810058594,
      "activations/layer3_attention_weight_min": -101.32633209228516,
      "activations/layer4_attention_weight_max": 108.80387115478516,
      "activations/layer4_attention_weight_min": -109.26140594482422,
      "activations/layer5_attention_weight_max": 52.75755310058594,
      "activations/layer5_attention_weight_min": -64.07521057128906,
      "activations/layer6_attention_weight_max": 45.65121841430664,
      "activations/layer6_attention_weight_min": -47.79560852050781,
      "activations/layer7_attention_weight_max": 93.8162612915039,
      "activations/layer7_attention_weight_min": -97.34636688232422,
      "activations/layer8_attention_weight_max": 42.24544143676758,
      "activations/layer8_attention_weight_min": -45.09400939941406,
      "activations/layer9_attention_weight_max": 34.064979553222656,
      "activations/layer9_attention_weight_min": -34.61100387573242,
      "epoch": 18.0,
      "learning_rate": 3.427878787878787e-05,
      "loss": 2.7434,
      "step": 309750
    },
    {
      "activations/layer0_attention_weight_max": 16.16564178466797,
      "activations/layer0_attention_weight_min": -12.3196439743042,
      "activations/layer10_attention_weight_max": 36.00334930419922,
      "activations/layer10_attention_weight_min": -36.65024185180664,
      "activations/layer11_attention_weight_max": 35.39733123779297,
      "activations/layer11_attention_weight_min": -35.55281448364258,
      "activations/layer12_attention_weight_max": 29.963701248168945,
      "activations/layer12_attention_weight_min": -29.646467208862305,
      "activations/layer13_attention_weight_max": 40.63216781616211,
      "activations/layer13_attention_weight_min": -36.158409118652344,
      "activations/layer14_attention_weight_max": 41.79280090332031,
      "activations/layer14_attention_weight_min": -40.849098205566406,
      "activations/layer15_attention_weight_max": 37.42232131958008,
      "activations/layer15_attention_weight_min": -37.229042053222656,
      "activations/layer16_attention_weight_max": 28.767593383789062,
      "activations/layer16_attention_weight_min": -29.358034133911133,
      "activations/layer17_attention_weight_max": 27.89111328125,
      "activations/layer17_attention_weight_min": -27.01026725769043,
      "activations/layer18_attention_weight_max": 29.028278350830078,
      "activations/layer18_attention_weight_min": -26.01018524169922,
      "activations/layer19_attention_weight_max": 41.91580581665039,
      "activations/layer19_attention_weight_min": -33.03721237182617,
      "activations/layer1_attention_weight_max": 15.943648338317871,
      "activations/layer1_attention_weight_min": -14.645236015319824,
      "activations/layer20_attention_weight_max": 29.385295867919922,
      "activations/layer20_attention_weight_min": -28.153274536132812,
      "activations/layer21_attention_weight_max": 28.2990779876709,
      "activations/layer21_attention_weight_min": -26.239574432373047,
      "activations/layer22_attention_weight_max": 48.17607498168945,
      "activations/layer22_attention_weight_min": -33.43205642700195,
      "activations/layer23_attention_weight_max": 32.55073547363281,
      "activations/layer23_attention_weight_min": -27.049718856811523,
      "activations/layer2_attention_weight_max": 32.80377960205078,
      "activations/layer2_attention_weight_min": -31.77898406982422,
      "activations/layer3_attention_weight_max": 97.75155639648438,
      "activations/layer3_attention_weight_min": -99.38751983642578,
      "activations/layer4_attention_weight_max": 105.29605865478516,
      "activations/layer4_attention_weight_min": -109.56587982177734,
      "activations/layer5_attention_weight_max": 49.09344482421875,
      "activations/layer5_attention_weight_min": -62.448158264160156,
      "activations/layer6_attention_weight_max": 42.19559860229492,
      "activations/layer6_attention_weight_min": -45.49781799316406,
      "activations/layer7_attention_weight_max": 101.44026947021484,
      "activations/layer7_attention_weight_min": -92.44854736328125,
      "activations/layer8_attention_weight_max": 42.4096565246582,
      "activations/layer8_attention_weight_min": -44.01591110229492,
      "activations/layer9_attention_weight_max": 31.76218032836914,
      "activations/layer9_attention_weight_min": -36.046085357666016,
      "epoch": 18.0,
      "learning_rate": 3.425984848484848e-05,
      "loss": 2.7974,
      "step": 309800
    },
    {
      "activations/layer0_attention_weight_max": 16.231237411499023,
      "activations/layer0_attention_weight_min": -13.533404350280762,
      "activations/layer10_attention_weight_max": 42.17711639404297,
      "activations/layer10_attention_weight_min": -37.91364669799805,
      "activations/layer11_attention_weight_max": 41.80170440673828,
      "activations/layer11_attention_weight_min": -38.94544219970703,
      "activations/layer12_attention_weight_max": 28.43623161315918,
      "activations/layer12_attention_weight_min": -27.883384704589844,
      "activations/layer13_attention_weight_max": 39.28849792480469,
      "activations/layer13_attention_weight_min": -37.60636901855469,
      "activations/layer14_attention_weight_max": 42.66537857055664,
      "activations/layer14_attention_weight_min": -38.902896881103516,
      "activations/layer15_attention_weight_max": 45.77032470703125,
      "activations/layer15_attention_weight_min": -41.526920318603516,
      "activations/layer16_attention_weight_max": 31.175691604614258,
      "activations/layer16_attention_weight_min": -28.90782356262207,
      "activations/layer17_attention_weight_max": 29.16615867614746,
      "activations/layer17_attention_weight_min": -32.401832580566406,
      "activations/layer18_attention_weight_max": 28.82066535949707,
      "activations/layer18_attention_weight_min": -30.258285522460938,
      "activations/layer19_attention_weight_max": 30.226388931274414,
      "activations/layer19_attention_weight_min": -29.216598510742188,
      "activations/layer1_attention_weight_max": 16.4554500579834,
      "activations/layer1_attention_weight_min": -15.535091400146484,
      "activations/layer20_attention_weight_max": 27.045337677001953,
      "activations/layer20_attention_weight_min": -23.9124813079834,
      "activations/layer21_attention_weight_max": 24.9626522064209,
      "activations/layer21_attention_weight_min": -23.78609275817871,
      "activations/layer22_attention_weight_max": 35.81856918334961,
      "activations/layer22_attention_weight_min": -30.46858024597168,
      "activations/layer23_attention_weight_max": 30.41744041442871,
      "activations/layer23_attention_weight_min": -24.998449325561523,
      "activations/layer2_attention_weight_max": 34.63380432128906,
      "activations/layer2_attention_weight_min": -34.311546325683594,
      "activations/layer3_attention_weight_max": 101.75780487060547,
      "activations/layer3_attention_weight_min": -105.01521301269531,
      "activations/layer4_attention_weight_max": 108.45832824707031,
      "activations/layer4_attention_weight_min": -108.60601043701172,
      "activations/layer5_attention_weight_max": 60.84382629394531,
      "activations/layer5_attention_weight_min": -59.13853073120117,
      "activations/layer6_attention_weight_max": 43.55691909790039,
      "activations/layer6_attention_weight_min": -44.27250289916992,
      "activations/layer7_attention_weight_max": 110.03108978271484,
      "activations/layer7_attention_weight_min": -98.35449981689453,
      "activations/layer8_attention_weight_max": 51.22783279418945,
      "activations/layer8_attention_weight_min": -47.87791442871094,
      "activations/layer9_attention_weight_max": 36.142574310302734,
      "activations/layer9_attention_weight_min": -36.80377197265625,
      "epoch": 18.0,
      "learning_rate": 3.424090909090909e-05,
      "loss": 2.7111,
      "step": 309850
    },
    {
      "activations/layer0_attention_weight_max": 15.682600021362305,
      "activations/layer0_attention_weight_min": -14.504623413085938,
      "activations/layer10_attention_weight_max": 40.17873001098633,
      "activations/layer10_attention_weight_min": -36.33831787109375,
      "activations/layer11_attention_weight_max": 38.482505798339844,
      "activations/layer11_attention_weight_min": -35.596866607666016,
      "activations/layer12_attention_weight_max": 27.852388381958008,
      "activations/layer12_attention_weight_min": -27.416553497314453,
      "activations/layer13_attention_weight_max": 40.93818664550781,
      "activations/layer13_attention_weight_min": -36.55842208862305,
      "activations/layer14_attention_weight_max": 44.1084098815918,
      "activations/layer14_attention_weight_min": -41.71689987182617,
      "activations/layer15_attention_weight_max": 37.69231414794922,
      "activations/layer15_attention_weight_min": -36.525142669677734,
      "activations/layer16_attention_weight_max": 29.230417251586914,
      "activations/layer16_attention_weight_min": -31.066944122314453,
      "activations/layer17_attention_weight_max": 27.39165687561035,
      "activations/layer17_attention_weight_min": -27.120935440063477,
      "activations/layer18_attention_weight_max": 30.83416748046875,
      "activations/layer18_attention_weight_min": -26.969491958618164,
      "activations/layer19_attention_weight_max": 35.32512283325195,
      "activations/layer19_attention_weight_min": -31.83905601501465,
      "activations/layer1_attention_weight_max": 18.352235794067383,
      "activations/layer1_attention_weight_min": -16.55980682373047,
      "activations/layer20_attention_weight_max": 28.550363540649414,
      "activations/layer20_attention_weight_min": -25.020519256591797,
      "activations/layer21_attention_weight_max": 26.679729461669922,
      "activations/layer21_attention_weight_min": -25.076568603515625,
      "activations/layer22_attention_weight_max": 40.20987319946289,
      "activations/layer22_attention_weight_min": -30.301485061645508,
      "activations/layer23_attention_weight_max": 36.450706481933594,
      "activations/layer23_attention_weight_min": -25.904991149902344,
      "activations/layer2_attention_weight_max": 33.61228561401367,
      "activations/layer2_attention_weight_min": -33.02186965942383,
      "activations/layer3_attention_weight_max": 101.0,
      "activations/layer3_attention_weight_min": -99.02232360839844,
      "activations/layer4_attention_weight_max": 107.76058197021484,
      "activations/layer4_attention_weight_min": -109.04888916015625,
      "activations/layer5_attention_weight_max": 51.23554992675781,
      "activations/layer5_attention_weight_min": -63.53997039794922,
      "activations/layer6_attention_weight_max": 44.57645034790039,
      "activations/layer6_attention_weight_min": -46.85068893432617,
      "activations/layer7_attention_weight_max": 105.62122344970703,
      "activations/layer7_attention_weight_min": -97.33045959472656,
      "activations/layer8_attention_weight_max": 46.863525390625,
      "activations/layer8_attention_weight_min": -45.680381774902344,
      "activations/layer9_attention_weight_max": 36.52212142944336,
      "activations/layer9_attention_weight_min": -38.2135124206543,
      "epoch": 18.01,
      "learning_rate": 3.42219696969697e-05,
      "loss": 2.7288,
      "step": 309900
    },
    {
      "activations/layer0_attention_weight_max": 15.736274719238281,
      "activations/layer0_attention_weight_min": -12.501880645751953,
      "activations/layer10_attention_weight_max": 34.05820846557617,
      "activations/layer10_attention_weight_min": -34.14048385620117,
      "activations/layer11_attention_weight_max": 33.00253677368164,
      "activations/layer11_attention_weight_min": -36.42808532714844,
      "activations/layer12_attention_weight_max": 25.608760833740234,
      "activations/layer12_attention_weight_min": -26.259244918823242,
      "activations/layer13_attention_weight_max": 37.26192092895508,
      "activations/layer13_attention_weight_min": -36.557533264160156,
      "activations/layer14_attention_weight_max": 40.44558334350586,
      "activations/layer14_attention_weight_min": -35.71879196166992,
      "activations/layer15_attention_weight_max": 39.71761703491211,
      "activations/layer15_attention_weight_min": -35.28955841064453,
      "activations/layer16_attention_weight_max": 29.65484046936035,
      "activations/layer16_attention_weight_min": -28.145751953125,
      "activations/layer17_attention_weight_max": 28.320430755615234,
      "activations/layer17_attention_weight_min": -27.66135597229004,
      "activations/layer18_attention_weight_max": 30.890323638916016,
      "activations/layer18_attention_weight_min": -27.446035385131836,
      "activations/layer19_attention_weight_max": 35.979618072509766,
      "activations/layer19_attention_weight_min": -32.10477828979492,
      "activations/layer1_attention_weight_max": 16.836679458618164,
      "activations/layer1_attention_weight_min": -14.224099159240723,
      "activations/layer20_attention_weight_max": 33.15695571899414,
      "activations/layer20_attention_weight_min": -25.58940887451172,
      "activations/layer21_attention_weight_max": 31.116649627685547,
      "activations/layer21_attention_weight_min": -25.353307723999023,
      "activations/layer22_attention_weight_max": 41.45245361328125,
      "activations/layer22_attention_weight_min": -33.44776153564453,
      "activations/layer23_attention_weight_max": 34.359962463378906,
      "activations/layer23_attention_weight_min": -27.45502281188965,
      "activations/layer2_attention_weight_max": 33.808990478515625,
      "activations/layer2_attention_weight_min": -34.06172180175781,
      "activations/layer3_attention_weight_max": 97.90998840332031,
      "activations/layer3_attention_weight_min": -101.60545349121094,
      "activations/layer4_attention_weight_max": 103.62560272216797,
      "activations/layer4_attention_weight_min": -105.31075286865234,
      "activations/layer5_attention_weight_max": 52.583946228027344,
      "activations/layer5_attention_weight_min": -62.984825134277344,
      "activations/layer6_attention_weight_max": 42.50140380859375,
      "activations/layer6_attention_weight_min": -45.01359176635742,
      "activations/layer7_attention_weight_max": 100.0970458984375,
      "activations/layer7_attention_weight_min": -98.70071411132812,
      "activations/layer8_attention_weight_max": 40.123626708984375,
      "activations/layer8_attention_weight_min": -42.06137466430664,
      "activations/layer9_attention_weight_max": 31.3692684173584,
      "activations/layer9_attention_weight_min": -34.190914154052734,
      "epoch": 18.01,
      "learning_rate": 3.42030303030303e-05,
      "loss": 2.7288,
      "step": 309950
    },
    {
      "activations/layer0_attention_weight_max": 16.572080612182617,
      "activations/layer0_attention_weight_min": -12.406667709350586,
      "activations/layer10_attention_weight_max": 38.24086380004883,
      "activations/layer10_attention_weight_min": -39.42311477661133,
      "activations/layer11_attention_weight_max": 37.15351486206055,
      "activations/layer11_attention_weight_min": -40.46287155151367,
      "activations/layer12_attention_weight_max": 29.355716705322266,
      "activations/layer12_attention_weight_min": -29.122617721557617,
      "activations/layer13_attention_weight_max": 43.0882453918457,
      "activations/layer13_attention_weight_min": -38.61454391479492,
      "activations/layer14_attention_weight_max": 48.40773010253906,
      "activations/layer14_attention_weight_min": -41.39632797241211,
      "activations/layer15_attention_weight_max": 46.90413284301758,
      "activations/layer15_attention_weight_min": -41.4453010559082,
      "activations/layer16_attention_weight_max": 36.11578369140625,
      "activations/layer16_attention_weight_min": -29.03217315673828,
      "activations/layer17_attention_weight_max": 30.285329818725586,
      "activations/layer17_attention_weight_min": -30.94191551208496,
      "activations/layer18_attention_weight_max": 30.026151657104492,
      "activations/layer18_attention_weight_min": -29.389074325561523,
      "activations/layer19_attention_weight_max": 35.51898956298828,
      "activations/layer19_attention_weight_min": -31.590396881103516,
      "activations/layer1_attention_weight_max": 17.338621139526367,
      "activations/layer1_attention_weight_min": -15.849218368530273,
      "activations/layer20_attention_weight_max": 29.646623611450195,
      "activations/layer20_attention_weight_min": -25.19066619873047,
      "activations/layer21_attention_weight_max": 28.279674530029297,
      "activations/layer21_attention_weight_min": -26.20170783996582,
      "activations/layer22_attention_weight_max": 41.42039108276367,
      "activations/layer22_attention_weight_min": -35.22212219238281,
      "activations/layer23_attention_weight_max": 30.091575622558594,
      "activations/layer23_attention_weight_min": -24.871633529663086,
      "activations/layer2_attention_weight_max": 34.716060638427734,
      "activations/layer2_attention_weight_min": -33.9059944152832,
      "activations/layer3_attention_weight_max": 102.13493347167969,
      "activations/layer3_attention_weight_min": -100.41526794433594,
      "activations/layer4_attention_weight_max": 109.54095458984375,
      "activations/layer4_attention_weight_min": -112.9535140991211,
      "activations/layer5_attention_weight_max": 51.61377716064453,
      "activations/layer5_attention_weight_min": -59.45506286621094,
      "activations/layer6_attention_weight_max": 44.4228630065918,
      "activations/layer6_attention_weight_min": -48.38267135620117,
      "activations/layer7_attention_weight_max": 107.4290771484375,
      "activations/layer7_attention_weight_min": -97.0143814086914,
      "activations/layer8_attention_weight_max": 45.075279235839844,
      "activations/layer8_attention_weight_min": -47.362060546875,
      "activations/layer9_attention_weight_max": 33.1178092956543,
      "activations/layer9_attention_weight_min": -37.7027702331543,
      "epoch": 18.01,
      "learning_rate": 3.4184469696969696e-05,
      "loss": 2.7301,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_loss": 2.6875,
      "eval_runtime": 8.5685,
      "eval_samples_per_second": 501.14,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.5685,
      "eval_openwebtext_samples_per_second": 501.14,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.005,
      "eval_wikitext_samples_per_second": 227.429,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.6235,
      "eval_lambada_samples_per_second": 505.948,
      "step": 310000
    },
    {
      "activations/layer0_attention_weight_max": 15.619354248046875,
      "activations/layer0_attention_weight_min": -12.67550277709961,
      "activations/layer10_attention_weight_max": 33.97218322753906,
      "activations/layer10_attention_weight_min": -35.86960220336914,
      "activations/layer11_attention_weight_max": 34.5797233581543,
      "activations/layer11_attention_weight_min": -35.248382568359375,
      "activations/layer12_attention_weight_max": 28.50847625732422,
      "activations/layer12_attention_weight_min": -28.33878517150879,
      "activations/layer13_attention_weight_max": 39.79420852661133,
      "activations/layer13_attention_weight_min": -39.11848831176758,
      "activations/layer14_attention_weight_max": 41.40957260131836,
      "activations/layer14_attention_weight_min": -38.24665451049805,
      "activations/layer15_attention_weight_max": 37.84685516357422,
      "activations/layer15_attention_weight_min": -35.98027420043945,
      "activations/layer16_attention_weight_max": 29.332855224609375,
      "activations/layer16_attention_weight_min": -29.340551376342773,
      "activations/layer17_attention_weight_max": 29.131996154785156,
      "activations/layer17_attention_weight_min": -27.518396377563477,
      "activations/layer18_attention_weight_max": 31.49449348449707,
      "activations/layer18_attention_weight_min": -24.537450790405273,
      "activations/layer19_attention_weight_max": 32.86082077026367,
      "activations/layer19_attention_weight_min": -33.5527458190918,
      "activations/layer1_attention_weight_max": 15.962190628051758,
      "activations/layer1_attention_weight_min": -15.907812118530273,
      "activations/layer20_attention_weight_max": 28.83685874938965,
      "activations/layer20_attention_weight_min": -27.598411560058594,
      "activations/layer21_attention_weight_max": 27.254772186279297,
      "activations/layer21_attention_weight_min": -27.2889461517334,
      "activations/layer22_attention_weight_max": 47.264713287353516,
      "activations/layer22_attention_weight_min": -31.98811912536621,
      "activations/layer23_attention_weight_max": 32.29351806640625,
      "activations/layer23_attention_weight_min": -27.82085418701172,
      "activations/layer2_attention_weight_max": 32.430545806884766,
      "activations/layer2_attention_weight_min": -32.263389587402344,
      "activations/layer3_attention_weight_max": 101.04069519042969,
      "activations/layer3_attention_weight_min": -96.90644073486328,
      "activations/layer4_attention_weight_max": 104.57193756103516,
      "activations/layer4_attention_weight_min": -109.4003677368164,
      "activations/layer5_attention_weight_max": 50.12579345703125,
      "activations/layer5_attention_weight_min": -61.95940399169922,
      "activations/layer6_attention_weight_max": 42.95540237426758,
      "activations/layer6_attention_weight_min": -45.937068939208984,
      "activations/layer7_attention_weight_max": 94.25508880615234,
      "activations/layer7_attention_weight_min": -98.01538848876953,
      "activations/layer8_attention_weight_max": 43.97957229614258,
      "activations/layer8_attention_weight_min": -45.098690032958984,
      "activations/layer9_attention_weight_max": 33.07590866088867,
      "activations/layer9_attention_weight_min": -36.63206481933594,
      "epoch": 18.02,
      "learning_rate": 3.41655303030303e-05,
      "loss": 2.7218,
      "step": 310050
    },
    {
      "activations/layer0_attention_weight_max": 16.173158645629883,
      "activations/layer0_attention_weight_min": -12.277129173278809,
      "activations/layer10_attention_weight_max": 32.56108093261719,
      "activations/layer10_attention_weight_min": -33.810218811035156,
      "activations/layer11_attention_weight_max": 32.50737380981445,
      "activations/layer11_attention_weight_min": -34.8129997253418,
      "activations/layer12_attention_weight_max": 27.935604095458984,
      "activations/layer12_attention_weight_min": -26.841569900512695,
      "activations/layer13_attention_weight_max": 42.17254638671875,
      "activations/layer13_attention_weight_min": -39.4015998840332,
      "activations/layer14_attention_weight_max": 43.343666076660156,
      "activations/layer14_attention_weight_min": -38.812461853027344,
      "activations/layer15_attention_weight_max": 46.29826354980469,
      "activations/layer15_attention_weight_min": -37.6236686706543,
      "activations/layer16_attention_weight_max": 32.80509948730469,
      "activations/layer16_attention_weight_min": -28.830617904663086,
      "activations/layer17_attention_weight_max": 30.903690338134766,
      "activations/layer17_attention_weight_min": -27.66645622253418,
      "activations/layer18_attention_weight_max": 32.42211151123047,
      "activations/layer18_attention_weight_min": -26.1754207611084,
      "activations/layer19_attention_weight_max": 38.42157745361328,
      "activations/layer19_attention_weight_min": -32.9160041809082,
      "activations/layer1_attention_weight_max": 18.969951629638672,
      "activations/layer1_attention_weight_min": -14.620464324951172,
      "activations/layer20_attention_weight_max": 32.23405456542969,
      "activations/layer20_attention_weight_min": -28.13884925842285,
      "activations/layer21_attention_weight_max": 34.31406021118164,
      "activations/layer21_attention_weight_min": -26.100339889526367,
      "activations/layer22_attention_weight_max": 46.73033142089844,
      "activations/layer22_attention_weight_min": -34.41115951538086,
      "activations/layer23_attention_weight_max": 34.004783630371094,
      "activations/layer23_attention_weight_min": -27.22566795349121,
      "activations/layer2_attention_weight_max": 35.07038879394531,
      "activations/layer2_attention_weight_min": -30.294193267822266,
      "activations/layer3_attention_weight_max": 96.47584533691406,
      "activations/layer3_attention_weight_min": -100.43429565429688,
      "activations/layer4_attention_weight_max": 98.52671813964844,
      "activations/layer4_attention_weight_min": -101.12125396728516,
      "activations/layer5_attention_weight_max": 45.96094512939453,
      "activations/layer5_attention_weight_min": -58.52507781982422,
      "activations/layer6_attention_weight_max": 41.18782424926758,
      "activations/layer6_attention_weight_min": -44.40037536621094,
      "activations/layer7_attention_weight_max": 93.19792938232422,
      "activations/layer7_attention_weight_min": -89.98753356933594,
      "activations/layer8_attention_weight_max": 42.469173431396484,
      "activations/layer8_attention_weight_min": -43.69287109375,
      "activations/layer9_attention_weight_max": 30.601911544799805,
      "activations/layer9_attention_weight_min": -33.94796371459961,
      "epoch": 18.02,
      "learning_rate": 3.4146590909090906e-05,
      "loss": 2.7336,
      "step": 310100
    },
    {
      "activations/layer0_attention_weight_max": 15.827530860900879,
      "activations/layer0_attention_weight_min": -12.909059524536133,
      "activations/layer10_attention_weight_max": 33.10948181152344,
      "activations/layer10_attention_weight_min": -32.88657760620117,
      "activations/layer11_attention_weight_max": 34.843299865722656,
      "activations/layer11_attention_weight_min": -34.768531799316406,
      "activations/layer12_attention_weight_max": 26.497364044189453,
      "activations/layer12_attention_weight_min": -26.457597732543945,
      "activations/layer13_attention_weight_max": 38.89689636230469,
      "activations/layer13_attention_weight_min": -36.65566635131836,
      "activations/layer14_attention_weight_max": 39.67189025878906,
      "activations/layer14_attention_weight_min": -38.01445388793945,
      "activations/layer15_attention_weight_max": 37.99978256225586,
      "activations/layer15_attention_weight_min": -34.81380081176758,
      "activations/layer16_attention_weight_max": 28.569406509399414,
      "activations/layer16_attention_weight_min": -27.96653938293457,
      "activations/layer17_attention_weight_max": 29.47322654724121,
      "activations/layer17_attention_weight_min": -26.81721305847168,
      "activations/layer18_attention_weight_max": 33.19389343261719,
      "activations/layer18_attention_weight_min": -25.00565528869629,
      "activations/layer19_attention_weight_max": 34.29961395263672,
      "activations/layer19_attention_weight_min": -31.8135986328125,
      "activations/layer1_attention_weight_max": 16.521404266357422,
      "activations/layer1_attention_weight_min": -13.923558235168457,
      "activations/layer20_attention_weight_max": 29.623544692993164,
      "activations/layer20_attention_weight_min": -25.332609176635742,
      "activations/layer21_attention_weight_max": 30.34661102294922,
      "activations/layer21_attention_weight_min": -26.042057037353516,
      "activations/layer22_attention_weight_max": 42.58179473876953,
      "activations/layer22_attention_weight_min": -31.31013298034668,
      "activations/layer23_attention_weight_max": 35.7880859375,
      "activations/layer23_attention_weight_min": -26.122901916503906,
      "activations/layer2_attention_weight_max": 35.46983337402344,
      "activations/layer2_attention_weight_min": -33.81014633178711,
      "activations/layer3_attention_weight_max": 103.56480407714844,
      "activations/layer3_attention_weight_min": -107.6041030883789,
      "activations/layer4_attention_weight_max": 108.1902847290039,
      "activations/layer4_attention_weight_min": -108.03932189941406,
      "activations/layer5_attention_weight_max": 51.28331756591797,
      "activations/layer5_attention_weight_min": -63.102943420410156,
      "activations/layer6_attention_weight_max": 46.76712417602539,
      "activations/layer6_attention_weight_min": -47.88953399658203,
      "activations/layer7_attention_weight_max": 91.22325897216797,
      "activations/layer7_attention_weight_min": -98.53921508789062,
      "activations/layer8_attention_weight_max": 41.306636810302734,
      "activations/layer8_attention_weight_min": -43.88538360595703,
      "activations/layer9_attention_weight_max": 32.11227035522461,
      "activations/layer9_attention_weight_min": -36.21670150756836,
      "epoch": 18.02,
      "learning_rate": 3.412765151515151e-05,
      "loss": 2.738,
      "step": 310150
    },
    {
      "activations/layer0_attention_weight_max": 15.405494689941406,
      "activations/layer0_attention_weight_min": -12.925558090209961,
      "activations/layer10_attention_weight_max": 32.90160369873047,
      "activations/layer10_attention_weight_min": -37.15625762939453,
      "activations/layer11_attention_weight_max": 33.56187438964844,
      "activations/layer11_attention_weight_min": -38.20769119262695,
      "activations/layer12_attention_weight_max": 25.686237335205078,
      "activations/layer12_attention_weight_min": -28.482215881347656,
      "activations/layer13_attention_weight_max": 37.982940673828125,
      "activations/layer13_attention_weight_min": -39.51367950439453,
      "activations/layer14_attention_weight_max": 40.45149612426758,
      "activations/layer14_attention_weight_min": -42.243778228759766,
      "activations/layer15_attention_weight_max": 37.81090545654297,
      "activations/layer15_attention_weight_min": -38.87663269042969,
      "activations/layer16_attention_weight_max": 28.577903747558594,
      "activations/layer16_attention_weight_min": -31.213085174560547,
      "activations/layer17_attention_weight_max": 28.810638427734375,
      "activations/layer17_attention_weight_min": -28.247751235961914,
      "activations/layer18_attention_weight_max": 31.41911506652832,
      "activations/layer18_attention_weight_min": -27.123310089111328,
      "activations/layer19_attention_weight_max": 34.7745475769043,
      "activations/layer19_attention_weight_min": -31.645849227905273,
      "activations/layer1_attention_weight_max": 16.234130859375,
      "activations/layer1_attention_weight_min": -14.910445213317871,
      "activations/layer20_attention_weight_max": 28.753190994262695,
      "activations/layer20_attention_weight_min": -25.8592586517334,
      "activations/layer21_attention_weight_max": 30.322221755981445,
      "activations/layer21_attention_weight_min": -26.122787475585938,
      "activations/layer22_attention_weight_max": 43.98457336425781,
      "activations/layer22_attention_weight_min": -30.727415084838867,
      "activations/layer23_attention_weight_max": 36.38184356689453,
      "activations/layer23_attention_weight_min": -27.63758087158203,
      "activations/layer2_attention_weight_max": 34.03486633300781,
      "activations/layer2_attention_weight_min": -31.973522186279297,
      "activations/layer3_attention_weight_max": 100.35687255859375,
      "activations/layer3_attention_weight_min": -100.06937408447266,
      "activations/layer4_attention_weight_max": 106.7872314453125,
      "activations/layer4_attention_weight_min": -107.52828216552734,
      "activations/layer5_attention_weight_max": 53.215965270996094,
      "activations/layer5_attention_weight_min": -62.067237854003906,
      "activations/layer6_attention_weight_max": 42.50718307495117,
      "activations/layer6_attention_weight_min": -45.3669548034668,
      "activations/layer7_attention_weight_max": 87.95269775390625,
      "activations/layer7_attention_weight_min": -93.79126739501953,
      "activations/layer8_attention_weight_max": 38.86357498168945,
      "activations/layer8_attention_weight_min": -44.3759765625,
      "activations/layer9_attention_weight_max": 30.939346313476562,
      "activations/layer9_attention_weight_min": -35.08219909667969,
      "epoch": 18.02,
      "learning_rate": 3.4108712121212116e-05,
      "loss": 2.7346,
      "step": 310200
    },
    {
      "activations/layer0_attention_weight_max": 15.782354354858398,
      "activations/layer0_attention_weight_min": -12.91426944732666,
      "activations/layer10_attention_weight_max": 41.87823486328125,
      "activations/layer10_attention_weight_min": -36.38931655883789,
      "activations/layer11_attention_weight_max": 42.368324279785156,
      "activations/layer11_attention_weight_min": -37.810482025146484,
      "activations/layer12_attention_weight_max": 29.134870529174805,
      "activations/layer12_attention_weight_min": -28.075130462646484,
      "activations/layer13_attention_weight_max": 43.36625671386719,
      "activations/layer13_attention_weight_min": -35.732845306396484,
      "activations/layer14_attention_weight_max": 47.234161376953125,
      "activations/layer14_attention_weight_min": -39.76736068725586,
      "activations/layer15_attention_weight_max": 44.23433303833008,
      "activations/layer15_attention_weight_min": -38.79948806762695,
      "activations/layer16_attention_weight_max": 32.52906036376953,
      "activations/layer16_attention_weight_min": -31.20261573791504,
      "activations/layer17_attention_weight_max": 34.96998977661133,
      "activations/layer17_attention_weight_min": -27.87340545654297,
      "activations/layer18_attention_weight_max": 35.3039665222168,
      "activations/layer18_attention_weight_min": -26.42375373840332,
      "activations/layer19_attention_weight_max": 35.913169860839844,
      "activations/layer19_attention_weight_min": -31.594589233398438,
      "activations/layer1_attention_weight_max": 17.863487243652344,
      "activations/layer1_attention_weight_min": -15.92041301727295,
      "activations/layer20_attention_weight_max": 30.278873443603516,
      "activations/layer20_attention_weight_min": -27.120361328125,
      "activations/layer21_attention_weight_max": 30.848276138305664,
      "activations/layer21_attention_weight_min": -27.276870727539062,
      "activations/layer22_attention_weight_max": 47.30329895019531,
      "activations/layer22_attention_weight_min": -34.003822326660156,
      "activations/layer23_attention_weight_max": 37.14546203613281,
      "activations/layer23_attention_weight_min": -26.99075698852539,
      "activations/layer2_attention_weight_max": 35.54789733886719,
      "activations/layer2_attention_weight_min": -33.45261001586914,
      "activations/layer3_attention_weight_max": 105.58336639404297,
      "activations/layer3_attention_weight_min": -99.79248809814453,
      "activations/layer4_attention_weight_max": 110.08538818359375,
      "activations/layer4_attention_weight_min": -107.07173919677734,
      "activations/layer5_attention_weight_max": 53.54206466674805,
      "activations/layer5_attention_weight_min": -59.29795837402344,
      "activations/layer6_attention_weight_max": 43.816200256347656,
      "activations/layer6_attention_weight_min": -46.30595016479492,
      "activations/layer7_attention_weight_max": 100.84671020507812,
      "activations/layer7_attention_weight_min": -95.8878173828125,
      "activations/layer8_attention_weight_max": 46.551876068115234,
      "activations/layer8_attention_weight_min": -46.01353073120117,
      "activations/layer9_attention_weight_max": 35.54490661621094,
      "activations/layer9_attention_weight_min": -38.94464874267578,
      "epoch": 18.03,
      "learning_rate": 3.4089772727272724e-05,
      "loss": 2.7103,
      "step": 310250
    },
    {
      "activations/layer0_attention_weight_max": 16.383113861083984,
      "activations/layer0_attention_weight_min": -13.162944793701172,
      "activations/layer10_attention_weight_max": 34.536041259765625,
      "activations/layer10_attention_weight_min": -33.22209548950195,
      "activations/layer11_attention_weight_max": 34.90032196044922,
      "activations/layer11_attention_weight_min": -33.893035888671875,
      "activations/layer12_attention_weight_max": 29.285463333129883,
      "activations/layer12_attention_weight_min": -26.769638061523438,
      "activations/layer13_attention_weight_max": 41.836265563964844,
      "activations/layer13_attention_weight_min": -36.32499313354492,
      "activations/layer14_attention_weight_max": 46.20614242553711,
      "activations/layer14_attention_weight_min": -38.11940383911133,
      "activations/layer15_attention_weight_max": 41.68890380859375,
      "activations/layer15_attention_weight_min": -34.859169006347656,
      "activations/layer16_attention_weight_max": 33.52754592895508,
      "activations/layer16_attention_weight_min": -28.01860237121582,
      "activations/layer17_attention_weight_max": 29.219045639038086,
      "activations/layer17_attention_weight_min": -26.351634979248047,
      "activations/layer18_attention_weight_max": 33.528961181640625,
      "activations/layer18_attention_weight_min": -27.42202377319336,
      "activations/layer19_attention_weight_max": 36.19164276123047,
      "activations/layer19_attention_weight_min": -33.820621490478516,
      "activations/layer1_attention_weight_max": 16.655323028564453,
      "activations/layer1_attention_weight_min": -15.666574478149414,
      "activations/layer20_attention_weight_max": 30.340551376342773,
      "activations/layer20_attention_weight_min": -28.4970645904541,
      "activations/layer21_attention_weight_max": 29.461210250854492,
      "activations/layer21_attention_weight_min": -25.430253982543945,
      "activations/layer22_attention_weight_max": 44.31037902832031,
      "activations/layer22_attention_weight_min": -30.581697463989258,
      "activations/layer23_attention_weight_max": 31.89594268798828,
      "activations/layer23_attention_weight_min": -25.931140899658203,
      "activations/layer2_attention_weight_max": 35.15795135498047,
      "activations/layer2_attention_weight_min": -33.37558364868164,
      "activations/layer3_attention_weight_max": 107.23351287841797,
      "activations/layer3_attention_weight_min": -109.25013732910156,
      "activations/layer4_attention_weight_max": 108.88658905029297,
      "activations/layer4_attention_weight_min": -111.00875091552734,
      "activations/layer5_attention_weight_max": 51.611507415771484,
      "activations/layer5_attention_weight_min": -65.81228637695312,
      "activations/layer6_attention_weight_max": 46.341224670410156,
      "activations/layer6_attention_weight_min": -49.30077362060547,
      "activations/layer7_attention_weight_max": 95.8451156616211,
      "activations/layer7_attention_weight_min": -101.86841583251953,
      "activations/layer8_attention_weight_max": 43.60649108886719,
      "activations/layer8_attention_weight_min": -44.84455108642578,
      "activations/layer9_attention_weight_max": 33.4603271484375,
      "activations/layer9_attention_weight_min": -34.0697021484375,
      "epoch": 18.03,
      "learning_rate": 3.407083333333333e-05,
      "loss": 2.722,
      "step": 310300
    },
    {
      "activations/layer0_attention_weight_max": 16.00686264038086,
      "activations/layer0_attention_weight_min": -12.347939491271973,
      "activations/layer10_attention_weight_max": 32.85595703125,
      "activations/layer10_attention_weight_min": -33.722537994384766,
      "activations/layer11_attention_weight_max": 32.291866302490234,
      "activations/layer11_attention_weight_min": -34.648319244384766,
      "activations/layer12_attention_weight_max": 24.95574951171875,
      "activations/layer12_attention_weight_min": -25.40849494934082,
      "activations/layer13_attention_weight_max": 37.3955192565918,
      "activations/layer13_attention_weight_min": -36.12742233276367,
      "activations/layer14_attention_weight_max": 40.70679473876953,
      "activations/layer14_attention_weight_min": -39.880706787109375,
      "activations/layer15_attention_weight_max": 36.59035873413086,
      "activations/layer15_attention_weight_min": -37.802406311035156,
      "activations/layer16_attention_weight_max": 28.378568649291992,
      "activations/layer16_attention_weight_min": -28.288570404052734,
      "activations/layer17_attention_weight_max": 30.84079360961914,
      "activations/layer17_attention_weight_min": -24.967506408691406,
      "activations/layer18_attention_weight_max": 29.709884643554688,
      "activations/layer18_attention_weight_min": -25.36763572692871,
      "activations/layer19_attention_weight_max": 32.47422409057617,
      "activations/layer19_attention_weight_min": -30.949020385742188,
      "activations/layer1_attention_weight_max": 15.493488311767578,
      "activations/layer1_attention_weight_min": -15.858607292175293,
      "activations/layer20_attention_weight_max": 29.31462287902832,
      "activations/layer20_attention_weight_min": -26.747791290283203,
      "activations/layer21_attention_weight_max": 27.535520553588867,
      "activations/layer21_attention_weight_min": -24.715068817138672,
      "activations/layer22_attention_weight_max": 42.048667907714844,
      "activations/layer22_attention_weight_min": -32.24324417114258,
      "activations/layer23_attention_weight_max": 34.43529510498047,
      "activations/layer23_attention_weight_min": -25.713199615478516,
      "activations/layer2_attention_weight_max": 32.44818115234375,
      "activations/layer2_attention_weight_min": -31.7876033782959,
      "activations/layer3_attention_weight_max": 98.52129364013672,
      "activations/layer3_attention_weight_min": -101.65914916992188,
      "activations/layer4_attention_weight_max": 105.74208068847656,
      "activations/layer4_attention_weight_min": -109.30194091796875,
      "activations/layer5_attention_weight_max": 48.96489715576172,
      "activations/layer5_attention_weight_min": -64.34891510009766,
      "activations/layer6_attention_weight_max": 42.39009094238281,
      "activations/layer6_attention_weight_min": -45.40132522583008,
      "activations/layer7_attention_weight_max": 88.99398040771484,
      "activations/layer7_attention_weight_min": -93.6192626953125,
      "activations/layer8_attention_weight_max": 38.872554779052734,
      "activations/layer8_attention_weight_min": -42.47482681274414,
      "activations/layer9_attention_weight_max": 32.355472564697266,
      "activations/layer9_attention_weight_min": -34.218807220458984,
      "epoch": 18.03,
      "learning_rate": 3.4051893939393934e-05,
      "loss": 2.7226,
      "step": 310350
    },
    {
      "activations/layer0_attention_weight_max": 15.973102569580078,
      "activations/layer0_attention_weight_min": -13.82280445098877,
      "activations/layer10_attention_weight_max": 34.3713264465332,
      "activations/layer10_attention_weight_min": -34.80462646484375,
      "activations/layer11_attention_weight_max": 34.658565521240234,
      "activations/layer11_attention_weight_min": -33.899505615234375,
      "activations/layer12_attention_weight_max": 27.171566009521484,
      "activations/layer12_attention_weight_min": -27.34587860107422,
      "activations/layer13_attention_weight_max": 37.54777145385742,
      "activations/layer13_attention_weight_min": -35.681922912597656,
      "activations/layer14_attention_weight_max": 41.729042053222656,
      "activations/layer14_attention_weight_min": -39.03261947631836,
      "activations/layer15_attention_weight_max": 37.124290466308594,
      "activations/layer15_attention_weight_min": -35.97050857543945,
      "activations/layer16_attention_weight_max": 28.92476463317871,
      "activations/layer16_attention_weight_min": -27.747982025146484,
      "activations/layer17_attention_weight_max": 30.67713165283203,
      "activations/layer17_attention_weight_min": -26.045724868774414,
      "activations/layer18_attention_weight_max": 32.52869415283203,
      "activations/layer18_attention_weight_min": -26.436403274536133,
      "activations/layer19_attention_weight_max": 34.025203704833984,
      "activations/layer19_attention_weight_min": -31.400455474853516,
      "activations/layer1_attention_weight_max": 17.205991744995117,
      "activations/layer1_attention_weight_min": -14.922999382019043,
      "activations/layer20_attention_weight_max": 29.292320251464844,
      "activations/layer20_attention_weight_min": -25.98812484741211,
      "activations/layer21_attention_weight_max": 30.30583953857422,
      "activations/layer21_attention_weight_min": -25.89373207092285,
      "activations/layer22_attention_weight_max": 38.32543182373047,
      "activations/layer22_attention_weight_min": -32.72759246826172,
      "activations/layer23_attention_weight_max": 30.138967514038086,
      "activations/layer23_attention_weight_min": -26.67220115661621,
      "activations/layer2_attention_weight_max": 34.155487060546875,
      "activations/layer2_attention_weight_min": -31.427343368530273,
      "activations/layer3_attention_weight_max": 104.12434387207031,
      "activations/layer3_attention_weight_min": -101.87854766845703,
      "activations/layer4_attention_weight_max": 111.64002990722656,
      "activations/layer4_attention_weight_min": -110.12383270263672,
      "activations/layer5_attention_weight_max": 51.54132080078125,
      "activations/layer5_attention_weight_min": -61.305660247802734,
      "activations/layer6_attention_weight_max": 45.63010787963867,
      "activations/layer6_attention_weight_min": -48.1241569519043,
      "activations/layer7_attention_weight_max": 97.62987518310547,
      "activations/layer7_attention_weight_min": -95.56932067871094,
      "activations/layer8_attention_weight_max": 42.30915451049805,
      "activations/layer8_attention_weight_min": -41.736568450927734,
      "activations/layer9_attention_weight_max": 32.073726654052734,
      "activations/layer9_attention_weight_min": -34.24111557006836,
      "epoch": 18.04,
      "learning_rate": 3.403295454545454e-05,
      "loss": 2.726,
      "step": 310400
    },
    {
      "activations/layer0_attention_weight_max": 16.26416778564453,
      "activations/layer0_attention_weight_min": -14.966597557067871,
      "activations/layer10_attention_weight_max": 36.65642166137695,
      "activations/layer10_attention_weight_min": -35.93971633911133,
      "activations/layer11_attention_weight_max": 36.45930862426758,
      "activations/layer11_attention_weight_min": -35.260292053222656,
      "activations/layer12_attention_weight_max": 29.023591995239258,
      "activations/layer12_attention_weight_min": -31.194869995117188,
      "activations/layer13_attention_weight_max": 41.422664642333984,
      "activations/layer13_attention_weight_min": -38.59730911254883,
      "activations/layer14_attention_weight_max": 41.856163024902344,
      "activations/layer14_attention_weight_min": -39.417381286621094,
      "activations/layer15_attention_weight_max": 40.5696907043457,
      "activations/layer15_attention_weight_min": -36.58805465698242,
      "activations/layer16_attention_weight_max": 31.702905654907227,
      "activations/layer16_attention_weight_min": -28.17082977294922,
      "activations/layer17_attention_weight_max": 31.5299015045166,
      "activations/layer17_attention_weight_min": -27.1842098236084,
      "activations/layer18_attention_weight_max": 31.302377700805664,
      "activations/layer18_attention_weight_min": -25.57542610168457,
      "activations/layer19_attention_weight_max": 36.785316467285156,
      "activations/layer19_attention_weight_min": -32.59067916870117,
      "activations/layer1_attention_weight_max": 16.111656188964844,
      "activations/layer1_attention_weight_min": -14.818758010864258,
      "activations/layer20_attention_weight_max": 27.759498596191406,
      "activations/layer20_attention_weight_min": -26.712411880493164,
      "activations/layer21_attention_weight_max": 28.450210571289062,
      "activations/layer21_attention_weight_min": -25.11564064025879,
      "activations/layer22_attention_weight_max": 43.53324890136719,
      "activations/layer22_attention_weight_min": -31.90241050720215,
      "activations/layer23_attention_weight_max": 29.836894989013672,
      "activations/layer23_attention_weight_min": -26.210590362548828,
      "activations/layer2_attention_weight_max": 34.718345642089844,
      "activations/layer2_attention_weight_min": -32.897674560546875,
      "activations/layer3_attention_weight_max": 99.30655670166016,
      "activations/layer3_attention_weight_min": -100.40090942382812,
      "activations/layer4_attention_weight_max": 107.53734588623047,
      "activations/layer4_attention_weight_min": -109.38526916503906,
      "activations/layer5_attention_weight_max": 48.96550750732422,
      "activations/layer5_attention_weight_min": -57.444419860839844,
      "activations/layer6_attention_weight_max": 46.10796356201172,
      "activations/layer6_attention_weight_min": -47.90503692626953,
      "activations/layer7_attention_weight_max": 94.18826293945312,
      "activations/layer7_attention_weight_min": -95.57500457763672,
      "activations/layer8_attention_weight_max": 42.02494812011719,
      "activations/layer8_attention_weight_min": -44.1175422668457,
      "activations/layer9_attention_weight_max": 33.490291595458984,
      "activations/layer9_attention_weight_min": -35.16263198852539,
      "epoch": 18.04,
      "learning_rate": 3.401401515151515e-05,
      "loss": 2.7132,
      "step": 310450
    },
    {
      "activations/layer0_attention_weight_max": 16.531002044677734,
      "activations/layer0_attention_weight_min": -13.688313484191895,
      "activations/layer10_attention_weight_max": 32.981422424316406,
      "activations/layer10_attention_weight_min": -33.826499938964844,
      "activations/layer11_attention_weight_max": 34.84654235839844,
      "activations/layer11_attention_weight_min": -36.108253479003906,
      "activations/layer12_attention_weight_max": 25.550004959106445,
      "activations/layer12_attention_weight_min": -25.57648277282715,
      "activations/layer13_attention_weight_max": 37.245460510253906,
      "activations/layer13_attention_weight_min": -34.831336975097656,
      "activations/layer14_attention_weight_max": 40.15475845336914,
      "activations/layer14_attention_weight_min": -36.20510482788086,
      "activations/layer15_attention_weight_max": 34.79984664916992,
      "activations/layer15_attention_weight_min": -36.05781173706055,
      "activations/layer16_attention_weight_max": 26.695362091064453,
      "activations/layer16_attention_weight_min": -28.93927574157715,
      "activations/layer17_attention_weight_max": 29.817834854125977,
      "activations/layer17_attention_weight_min": -26.54973602294922,
      "activations/layer18_attention_weight_max": 29.282548904418945,
      "activations/layer18_attention_weight_min": -27.839689254760742,
      "activations/layer19_attention_weight_max": 32.14276123046875,
      "activations/layer19_attention_weight_min": -33.567447662353516,
      "activations/layer1_attention_weight_max": 17.416715621948242,
      "activations/layer1_attention_weight_min": -15.626083374023438,
      "activations/layer20_attention_weight_max": 29.3703670501709,
      "activations/layer20_attention_weight_min": -27.32821273803711,
      "activations/layer21_attention_weight_max": 27.72658348083496,
      "activations/layer21_attention_weight_min": -26.494169235229492,
      "activations/layer22_attention_weight_max": 37.19628143310547,
      "activations/layer22_attention_weight_min": -32.885982513427734,
      "activations/layer23_attention_weight_max": 30.272113800048828,
      "activations/layer23_attention_weight_min": -25.080474853515625,
      "activations/layer2_attention_weight_max": 31.42015838623047,
      "activations/layer2_attention_weight_min": -30.607168197631836,
      "activations/layer3_attention_weight_max": 97.5858154296875,
      "activations/layer3_attention_weight_min": -101.19334411621094,
      "activations/layer4_attention_weight_max": 106.08135223388672,
      "activations/layer4_attention_weight_min": -109.64818572998047,
      "activations/layer5_attention_weight_max": 50.79393005371094,
      "activations/layer5_attention_weight_min": -61.450660705566406,
      "activations/layer6_attention_weight_max": 41.861602783203125,
      "activations/layer6_attention_weight_min": -45.912288665771484,
      "activations/layer7_attention_weight_max": 90.49090576171875,
      "activations/layer7_attention_weight_min": -95.18775939941406,
      "activations/layer8_attention_weight_max": 39.47239303588867,
      "activations/layer8_attention_weight_min": -41.79834747314453,
      "activations/layer9_attention_weight_max": 31.50278663635254,
      "activations/layer9_attention_weight_min": -35.922393798828125,
      "epoch": 18.04,
      "learning_rate": 3.399507575757575e-05,
      "loss": 2.7129,
      "step": 310500
    },
    {
      "activations/layer0_attention_weight_max": 16.613800048828125,
      "activations/layer0_attention_weight_min": -14.11736011505127,
      "activations/layer10_attention_weight_max": 32.915252685546875,
      "activations/layer10_attention_weight_min": -35.321441650390625,
      "activations/layer11_attention_weight_max": 32.097686767578125,
      "activations/layer11_attention_weight_min": -34.854827880859375,
      "activations/layer12_attention_weight_max": 25.90756607055664,
      "activations/layer12_attention_weight_min": -28.94926643371582,
      "activations/layer13_attention_weight_max": 38.5628547668457,
      "activations/layer13_attention_weight_min": -37.99525833129883,
      "activations/layer14_attention_weight_max": 39.35359191894531,
      "activations/layer14_attention_weight_min": -40.48012924194336,
      "activations/layer15_attention_weight_max": 34.6413459777832,
      "activations/layer15_attention_weight_min": -36.40410614013672,
      "activations/layer16_attention_weight_max": 27.94874382019043,
      "activations/layer16_attention_weight_min": -29.474153518676758,
      "activations/layer17_attention_weight_max": 31.46973991394043,
      "activations/layer17_attention_weight_min": -26.07727813720703,
      "activations/layer18_attention_weight_max": 30.5595703125,
      "activations/layer18_attention_weight_min": -26.97854232788086,
      "activations/layer19_attention_weight_max": 32.59647750854492,
      "activations/layer19_attention_weight_min": -32.38503646850586,
      "activations/layer1_attention_weight_max": 16.998903274536133,
      "activations/layer1_attention_weight_min": -15.69721508026123,
      "activations/layer20_attention_weight_max": 31.90633201599121,
      "activations/layer20_attention_weight_min": -27.679414749145508,
      "activations/layer21_attention_weight_max": 29.340227127075195,
      "activations/layer21_attention_weight_min": -26.18583869934082,
      "activations/layer22_attention_weight_max": 38.35884094238281,
      "activations/layer22_attention_weight_min": -33.26478958129883,
      "activations/layer23_attention_weight_max": 32.12983703613281,
      "activations/layer23_attention_weight_min": -26.48165512084961,
      "activations/layer2_attention_weight_max": 33.39057159423828,
      "activations/layer2_attention_weight_min": -32.72526550292969,
      "activations/layer3_attention_weight_max": 103.10665130615234,
      "activations/layer3_attention_weight_min": -101.29102325439453,
      "activations/layer4_attention_weight_max": 107.16202545166016,
      "activations/layer4_attention_weight_min": -109.65654754638672,
      "activations/layer5_attention_weight_max": 52.87774658203125,
      "activations/layer5_attention_weight_min": -61.17753601074219,
      "activations/layer6_attention_weight_max": 43.701175689697266,
      "activations/layer6_attention_weight_min": -46.939449310302734,
      "activations/layer7_attention_weight_max": 88.69921112060547,
      "activations/layer7_attention_weight_min": -104.33039093017578,
      "activations/layer8_attention_weight_max": 39.28589630126953,
      "activations/layer8_attention_weight_min": -46.51956558227539,
      "activations/layer9_attention_weight_max": 31.664676666259766,
      "activations/layer9_attention_weight_min": -34.308040618896484,
      "epoch": 18.04,
      "learning_rate": 3.397613636363636e-05,
      "loss": 2.7335,
      "step": 310550
    },
    {
      "activations/layer0_attention_weight_max": 17.264259338378906,
      "activations/layer0_attention_weight_min": -12.789319038391113,
      "activations/layer10_attention_weight_max": 43.130950927734375,
      "activations/layer10_attention_weight_min": -38.568458557128906,
      "activations/layer11_attention_weight_max": 44.28428649902344,
      "activations/layer11_attention_weight_min": -41.16677474975586,
      "activations/layer12_attention_weight_max": 25.286396026611328,
      "activations/layer12_attention_weight_min": -26.583091735839844,
      "activations/layer13_attention_weight_max": 38.16045379638672,
      "activations/layer13_attention_weight_min": -35.815589904785156,
      "activations/layer14_attention_weight_max": 41.52641296386719,
      "activations/layer14_attention_weight_min": -38.08405685424805,
      "activations/layer15_attention_weight_max": 38.11201095581055,
      "activations/layer15_attention_weight_min": -36.530517578125,
      "activations/layer16_attention_weight_max": 29.675907135009766,
      "activations/layer16_attention_weight_min": -29.208696365356445,
      "activations/layer17_attention_weight_max": 32.51221466064453,
      "activations/layer17_attention_weight_min": -23.79568099975586,
      "activations/layer18_attention_weight_max": 28.765188217163086,
      "activations/layer18_attention_weight_min": -24.219858169555664,
      "activations/layer19_attention_weight_max": 34.064937591552734,
      "activations/layer19_attention_weight_min": -29.72629737854004,
      "activations/layer1_attention_weight_max": 17.603450775146484,
      "activations/layer1_attention_weight_min": -14.946130752563477,
      "activations/layer20_attention_weight_max": 26.126970291137695,
      "activations/layer20_attention_weight_min": -23.435686111450195,
      "activations/layer21_attention_weight_max": 27.16133689880371,
      "activations/layer21_attention_weight_min": -22.689905166625977,
      "activations/layer22_attention_weight_max": 37.78474426269531,
      "activations/layer22_attention_weight_min": -28.12262535095215,
      "activations/layer23_attention_weight_max": 29.725173950195312,
      "activations/layer23_attention_weight_min": -24.22701072692871,
      "activations/layer2_attention_weight_max": 38.85272216796875,
      "activations/layer2_attention_weight_min": -33.80158615112305,
      "activations/layer3_attention_weight_max": 104.43865966796875,
      "activations/layer3_attention_weight_min": -103.03498077392578,
      "activations/layer4_attention_weight_max": 110.5522689819336,
      "activations/layer4_attention_weight_min": -109.3121337890625,
      "activations/layer5_attention_weight_max": 55.333831787109375,
      "activations/layer5_attention_weight_min": -57.47241973876953,
      "activations/layer6_attention_weight_max": 45.56968688964844,
      "activations/layer6_attention_weight_min": -47.279048919677734,
      "activations/layer7_attention_weight_max": 109.70121765136719,
      "activations/layer7_attention_weight_min": -104.14287567138672,
      "activations/layer8_attention_weight_max": 52.459964752197266,
      "activations/layer8_attention_weight_min": -50.5119514465332,
      "activations/layer9_attention_weight_max": 38.36189270019531,
      "activations/layer9_attention_weight_min": -37.688697814941406,
      "epoch": 18.05,
      "learning_rate": 3.395719696969697e-05,
      "loss": 2.7406,
      "step": 310600
    },
    {
      "activations/layer0_attention_weight_max": 16.002307891845703,
      "activations/layer0_attention_weight_min": -14.123289108276367,
      "activations/layer10_attention_weight_max": 32.890953063964844,
      "activations/layer10_attention_weight_min": -33.5606575012207,
      "activations/layer11_attention_weight_max": 34.50876998901367,
      "activations/layer11_attention_weight_min": -33.35344314575195,
      "activations/layer12_attention_weight_max": 26.920652389526367,
      "activations/layer12_attention_weight_min": -26.036142349243164,
      "activations/layer13_attention_weight_max": 39.91176223754883,
      "activations/layer13_attention_weight_min": -34.463279724121094,
      "activations/layer14_attention_weight_max": 40.621700286865234,
      "activations/layer14_attention_weight_min": -36.56166076660156,
      "activations/layer15_attention_weight_max": 38.60896682739258,
      "activations/layer15_attention_weight_min": -34.72195053100586,
      "activations/layer16_attention_weight_max": 32.491905212402344,
      "activations/layer16_attention_weight_min": -28.640056610107422,
      "activations/layer17_attention_weight_max": 29.82008171081543,
      "activations/layer17_attention_weight_min": -28.080106735229492,
      "activations/layer18_attention_weight_max": 34.290733337402344,
      "activations/layer18_attention_weight_min": -27.783615112304688,
      "activations/layer19_attention_weight_max": 36.01704406738281,
      "activations/layer19_attention_weight_min": -33.865718841552734,
      "activations/layer1_attention_weight_max": 15.927648544311523,
      "activations/layer1_attention_weight_min": -15.408875465393066,
      "activations/layer20_attention_weight_max": 30.92946434020996,
      "activations/layer20_attention_weight_min": -27.520517349243164,
      "activations/layer21_attention_weight_max": 31.387975692749023,
      "activations/layer21_attention_weight_min": -26.332134246826172,
      "activations/layer22_attention_weight_max": 44.55185317993164,
      "activations/layer22_attention_weight_min": -33.48793029785156,
      "activations/layer23_attention_weight_max": 33.728511810302734,
      "activations/layer23_attention_weight_min": -28.61301040649414,
      "activations/layer2_attention_weight_max": 34.12333297729492,
      "activations/layer2_attention_weight_min": -32.881011962890625,
      "activations/layer3_attention_weight_max": 107.50743103027344,
      "activations/layer3_attention_weight_min": -104.13850402832031,
      "activations/layer4_attention_weight_max": 112.54296112060547,
      "activations/layer4_attention_weight_min": -110.22924041748047,
      "activations/layer5_attention_weight_max": 51.99747848510742,
      "activations/layer5_attention_weight_min": -63.048641204833984,
      "activations/layer6_attention_weight_max": 45.188987731933594,
      "activations/layer6_attention_weight_min": -47.973289489746094,
      "activations/layer7_attention_weight_max": 98.56407165527344,
      "activations/layer7_attention_weight_min": -100.08651733398438,
      "activations/layer8_attention_weight_max": 46.09522247314453,
      "activations/layer8_attention_weight_min": -45.122222900390625,
      "activations/layer9_attention_weight_max": 34.104339599609375,
      "activations/layer9_attention_weight_min": -35.326725006103516,
      "epoch": 18.05,
      "learning_rate": 3.393825757575758e-05,
      "loss": 2.7266,
      "step": 310650
    },
    {
      "activations/layer0_attention_weight_max": 17.138473510742188,
      "activations/layer0_attention_weight_min": -15.39728832244873,
      "activations/layer10_attention_weight_max": 33.818580627441406,
      "activations/layer10_attention_weight_min": -33.45304870605469,
      "activations/layer11_attention_weight_max": 34.120697021484375,
      "activations/layer11_attention_weight_min": -34.178565979003906,
      "activations/layer12_attention_weight_max": 27.79560089111328,
      "activations/layer12_attention_weight_min": -26.40579605102539,
      "activations/layer13_attention_weight_max": 40.60837936401367,
      "activations/layer13_attention_weight_min": -34.56683349609375,
      "activations/layer14_attention_weight_max": 42.4549446105957,
      "activations/layer14_attention_weight_min": -37.430416107177734,
      "activations/layer15_attention_weight_max": 39.92192077636719,
      "activations/layer15_attention_weight_min": -35.264217376708984,
      "activations/layer16_attention_weight_max": 32.505645751953125,
      "activations/layer16_attention_weight_min": -27.049917221069336,
      "activations/layer17_attention_weight_max": 31.96017837524414,
      "activations/layer17_attention_weight_min": -25.196752548217773,
      "activations/layer18_attention_weight_max": 31.6509952545166,
      "activations/layer18_attention_weight_min": -25.6274471282959,
      "activations/layer19_attention_weight_max": 35.82767105102539,
      "activations/layer19_attention_weight_min": -31.575443267822266,
      "activations/layer1_attention_weight_max": 17.332042694091797,
      "activations/layer1_attention_weight_min": -14.692300796508789,
      "activations/layer20_attention_weight_max": 29.213132858276367,
      "activations/layer20_attention_weight_min": -25.387426376342773,
      "activations/layer21_attention_weight_max": 30.88456153869629,
      "activations/layer21_attention_weight_min": -25.99351692199707,
      "activations/layer22_attention_weight_max": 41.8513069152832,
      "activations/layer22_attention_weight_min": -34.954673767089844,
      "activations/layer23_attention_weight_max": 35.30107879638672,
      "activations/layer23_attention_weight_min": -25.560224533081055,
      "activations/layer2_attention_weight_max": 35.47239685058594,
      "activations/layer2_attention_weight_min": -32.164058685302734,
      "activations/layer3_attention_weight_max": 104.87964630126953,
      "activations/layer3_attention_weight_min": -97.86708068847656,
      "activations/layer4_attention_weight_max": 108.04996490478516,
      "activations/layer4_attention_weight_min": -102.3698959350586,
      "activations/layer5_attention_weight_max": 50.223934173583984,
      "activations/layer5_attention_weight_min": -59.383758544921875,
      "activations/layer6_attention_weight_max": 42.20781326293945,
      "activations/layer6_attention_weight_min": -44.55806350708008,
      "activations/layer7_attention_weight_max": 96.08715057373047,
      "activations/layer7_attention_weight_min": -85.8268051147461,
      "activations/layer8_attention_weight_max": 43.998111724853516,
      "activations/layer8_attention_weight_min": -40.44514846801758,
      "activations/layer9_attention_weight_max": 30.807586669921875,
      "activations/layer9_attention_weight_min": -32.11674499511719,
      "epoch": 18.05,
      "learning_rate": 3.391931818181818e-05,
      "loss": 2.7348,
      "step": 310700
    },
    {
      "activations/layer0_attention_weight_max": 15.746244430541992,
      "activations/layer0_attention_weight_min": -13.919669151306152,
      "activations/layer10_attention_weight_max": 34.2612419128418,
      "activations/layer10_attention_weight_min": -35.96637725830078,
      "activations/layer11_attention_weight_max": 33.70925521850586,
      "activations/layer11_attention_weight_min": -36.34507751464844,
      "activations/layer12_attention_weight_max": 28.824228286743164,
      "activations/layer12_attention_weight_min": -27.48567008972168,
      "activations/layer13_attention_weight_max": 41.16634750366211,
      "activations/layer13_attention_weight_min": -36.81752014160156,
      "activations/layer14_attention_weight_max": 43.728973388671875,
      "activations/layer14_attention_weight_min": -40.21664810180664,
      "activations/layer15_attention_weight_max": 42.80216598510742,
      "activations/layer15_attention_weight_min": -37.63264465332031,
      "activations/layer16_attention_weight_max": 29.883268356323242,
      "activations/layer16_attention_weight_min": -27.574771881103516,
      "activations/layer17_attention_weight_max": 30.27142333984375,
      "activations/layer17_attention_weight_min": -27.422021865844727,
      "activations/layer18_attention_weight_max": 31.33367347717285,
      "activations/layer18_attention_weight_min": -26.72507095336914,
      "activations/layer19_attention_weight_max": 34.570377349853516,
      "activations/layer19_attention_weight_min": -33.94025802612305,
      "activations/layer1_attention_weight_max": 17.33915138244629,
      "activations/layer1_attention_weight_min": -14.979345321655273,
      "activations/layer20_attention_weight_max": 29.826416015625,
      "activations/layer20_attention_weight_min": -27.853702545166016,
      "activations/layer21_attention_weight_max": 28.681039810180664,
      "activations/layer21_attention_weight_min": -27.86916160583496,
      "activations/layer22_attention_weight_max": 43.045997619628906,
      "activations/layer22_attention_weight_min": -36.33498764038086,
      "activations/layer23_attention_weight_max": 31.218311309814453,
      "activations/layer23_attention_weight_min": -29.194705963134766,
      "activations/layer2_attention_weight_max": 32.53639602661133,
      "activations/layer2_attention_weight_min": -31.94073486328125,
      "activations/layer3_attention_weight_max": 97.0888900756836,
      "activations/layer3_attention_weight_min": -99.86334991455078,
      "activations/layer4_attention_weight_max": 105.00008392333984,
      "activations/layer4_attention_weight_min": -113.01921844482422,
      "activations/layer5_attention_weight_max": 49.41905975341797,
      "activations/layer5_attention_weight_min": -62.32341766357422,
      "activations/layer6_attention_weight_max": 44.65825271606445,
      "activations/layer6_attention_weight_min": -46.61398696899414,
      "activations/layer7_attention_weight_max": 91.67626190185547,
      "activations/layer7_attention_weight_min": -95.26604461669922,
      "activations/layer8_attention_weight_max": 45.502906799316406,
      "activations/layer8_attention_weight_min": -44.67418670654297,
      "activations/layer9_attention_weight_max": 35.34961700439453,
      "activations/layer9_attention_weight_min": -36.25246047973633,
      "epoch": 18.06,
      "learning_rate": 3.390037878787879e-05,
      "loss": 2.7225,
      "step": 310750
    },
    {
      "activations/layer0_attention_weight_max": 15.708038330078125,
      "activations/layer0_attention_weight_min": -13.123527526855469,
      "activations/layer10_attention_weight_max": 37.61282730102539,
      "activations/layer10_attention_weight_min": -36.4119758605957,
      "activations/layer11_attention_weight_max": 38.01090621948242,
      "activations/layer11_attention_weight_min": -38.208587646484375,
      "activations/layer12_attention_weight_max": 30.23577308654785,
      "activations/layer12_attention_weight_min": -29.54879379272461,
      "activations/layer13_attention_weight_max": 45.30358123779297,
      "activations/layer13_attention_weight_min": -40.33303451538086,
      "activations/layer14_attention_weight_max": 50.775821685791016,
      "activations/layer14_attention_weight_min": -43.07651901245117,
      "activations/layer15_attention_weight_max": 44.83193588256836,
      "activations/layer15_attention_weight_min": -40.82151794433594,
      "activations/layer16_attention_weight_max": 36.34354019165039,
      "activations/layer16_attention_weight_min": -29.611106872558594,
      "activations/layer17_attention_weight_max": 36.256290435791016,
      "activations/layer17_attention_weight_min": -28.53852653503418,
      "activations/layer18_attention_weight_max": 37.235599517822266,
      "activations/layer18_attention_weight_min": -28.12727165222168,
      "activations/layer19_attention_weight_max": 39.76387023925781,
      "activations/layer19_attention_weight_min": -35.14521408081055,
      "activations/layer1_attention_weight_max": 18.91448211669922,
      "activations/layer1_attention_weight_min": -15.876107215881348,
      "activations/layer20_attention_weight_max": 33.86001968383789,
      "activations/layer20_attention_weight_min": -27.083927154541016,
      "activations/layer21_attention_weight_max": 34.98249816894531,
      "activations/layer21_attention_weight_min": -27.11229705810547,
      "activations/layer22_attention_weight_max": 52.149696350097656,
      "activations/layer22_attention_weight_min": -37.046226501464844,
      "activations/layer23_attention_weight_max": 37.63514709472656,
      "activations/layer23_attention_weight_min": -27.216724395751953,
      "activations/layer2_attention_weight_max": 34.41080856323242,
      "activations/layer2_attention_weight_min": -33.072547912597656,
      "activations/layer3_attention_weight_max": 102.84518432617188,
      "activations/layer3_attention_weight_min": -100.10400390625,
      "activations/layer4_attention_weight_max": 107.06922912597656,
      "activations/layer4_attention_weight_min": -108.39629364013672,
      "activations/layer5_attention_weight_max": 51.23739242553711,
      "activations/layer5_attention_weight_min": -59.859642028808594,
      "activations/layer6_attention_weight_max": 48.120174407958984,
      "activations/layer6_attention_weight_min": -46.749637603759766,
      "activations/layer7_attention_weight_max": 94.99948120117188,
      "activations/layer7_attention_weight_min": -96.9126968383789,
      "activations/layer8_attention_weight_max": 45.386741638183594,
      "activations/layer8_attention_weight_min": -43.88776397705078,
      "activations/layer9_attention_weight_max": 35.65751266479492,
      "activations/layer9_attention_weight_min": -36.410552978515625,
      "epoch": 18.06,
      "learning_rate": 3.388143939393939e-05,
      "loss": 2.7451,
      "step": 310800
    },
    {
      "activations/layer0_attention_weight_max": 15.636784553527832,
      "activations/layer0_attention_weight_min": -13.391107559204102,
      "activations/layer10_attention_weight_max": 34.74220657348633,
      "activations/layer10_attention_weight_min": -34.4887580871582,
      "activations/layer11_attention_weight_max": 33.90382385253906,
      "activations/layer11_attention_weight_min": -35.33025360107422,
      "activations/layer12_attention_weight_max": 27.14171600341797,
      "activations/layer12_attention_weight_min": -28.030725479125977,
      "activations/layer13_attention_weight_max": 40.248565673828125,
      "activations/layer13_attention_weight_min": -39.332763671875,
      "activations/layer14_attention_weight_max": 42.711097717285156,
      "activations/layer14_attention_weight_min": -39.49523162841797,
      "activations/layer15_attention_weight_max": 38.292781829833984,
      "activations/layer15_attention_weight_min": -36.09092712402344,
      "activations/layer16_attention_weight_max": 27.778573989868164,
      "activations/layer16_attention_weight_min": -27.935819625854492,
      "activations/layer17_attention_weight_max": 31.65650177001953,
      "activations/layer17_attention_weight_min": -26.3352108001709,
      "activations/layer18_attention_weight_max": 28.147140502929688,
      "activations/layer18_attention_weight_min": -26.71666145324707,
      "activations/layer19_attention_weight_max": 32.07354736328125,
      "activations/layer19_attention_weight_min": -31.510517120361328,
      "activations/layer1_attention_weight_max": 17.478355407714844,
      "activations/layer1_attention_weight_min": -14.961830139160156,
      "activations/layer20_attention_weight_max": 28.36251449584961,
      "activations/layer20_attention_weight_min": -26.611072540283203,
      "activations/layer21_attention_weight_max": 29.50406265258789,
      "activations/layer21_attention_weight_min": -25.881240844726562,
      "activations/layer22_attention_weight_max": 38.6310920715332,
      "activations/layer22_attention_weight_min": -32.38346862792969,
      "activations/layer23_attention_weight_max": 32.14790344238281,
      "activations/layer23_attention_weight_min": -26.752689361572266,
      "activations/layer2_attention_weight_max": 33.360496520996094,
      "activations/layer2_attention_weight_min": -30.59273338317871,
      "activations/layer3_attention_weight_max": 100.25434112548828,
      "activations/layer3_attention_weight_min": -100.17241668701172,
      "activations/layer4_attention_weight_max": 107.1790771484375,
      "activations/layer4_attention_weight_min": -108.49993896484375,
      "activations/layer5_attention_weight_max": 50.71780014038086,
      "activations/layer5_attention_weight_min": -59.86603546142578,
      "activations/layer6_attention_weight_max": 45.60226058959961,
      "activations/layer6_attention_weight_min": -46.833763122558594,
      "activations/layer7_attention_weight_max": 90.86859893798828,
      "activations/layer7_attention_weight_min": -96.75997924804688,
      "activations/layer8_attention_weight_max": 45.88677215576172,
      "activations/layer8_attention_weight_min": -47.08904266357422,
      "activations/layer9_attention_weight_max": 37.589847564697266,
      "activations/layer9_attention_weight_min": -36.262733459472656,
      "epoch": 18.06,
      "learning_rate": 3.3862499999999996e-05,
      "loss": 2.7298,
      "step": 310850
    },
    {
      "activations/layer0_attention_weight_max": 16.90297508239746,
      "activations/layer0_attention_weight_min": -13.260729789733887,
      "activations/layer10_attention_weight_max": 33.70024490356445,
      "activations/layer10_attention_weight_min": -34.7091178894043,
      "activations/layer11_attention_weight_max": 34.59503173828125,
      "activations/layer11_attention_weight_min": -36.282432556152344,
      "activations/layer12_attention_weight_max": 26.867904663085938,
      "activations/layer12_attention_weight_min": -27.513023376464844,
      "activations/layer13_attention_weight_max": 40.20685577392578,
      "activations/layer13_attention_weight_min": -36.54338073730469,
      "activations/layer14_attention_weight_max": 43.395042419433594,
      "activations/layer14_attention_weight_min": -38.60206604003906,
      "activations/layer15_attention_weight_max": 43.16847229003906,
      "activations/layer15_attention_weight_min": -35.880332946777344,
      "activations/layer16_attention_weight_max": 29.640687942504883,
      "activations/layer16_attention_weight_min": -27.91179084777832,
      "activations/layer17_attention_weight_max": 30.55942153930664,
      "activations/layer17_attention_weight_min": -27.31804084777832,
      "activations/layer18_attention_weight_max": 31.96381187438965,
      "activations/layer18_attention_weight_min": -26.504209518432617,
      "activations/layer19_attention_weight_max": 32.621315002441406,
      "activations/layer19_attention_weight_min": -31.174306869506836,
      "activations/layer1_attention_weight_max": 16.241140365600586,
      "activations/layer1_attention_weight_min": -15.389632225036621,
      "activations/layer20_attention_weight_max": 27.791202545166016,
      "activations/layer20_attention_weight_min": -26.56713104248047,
      "activations/layer21_attention_weight_max": 26.420902252197266,
      "activations/layer21_attention_weight_min": -25.86643409729004,
      "activations/layer22_attention_weight_max": 38.91535949707031,
      "activations/layer22_attention_weight_min": -33.94772720336914,
      "activations/layer23_attention_weight_max": 31.325597763061523,
      "activations/layer23_attention_weight_min": -29.47842025756836,
      "activations/layer2_attention_weight_max": 31.489173889160156,
      "activations/layer2_attention_weight_min": -31.995777130126953,
      "activations/layer3_attention_weight_max": 99.52079010009766,
      "activations/layer3_attention_weight_min": -98.36216735839844,
      "activations/layer4_attention_weight_max": 107.1480712890625,
      "activations/layer4_attention_weight_min": -108.70159149169922,
      "activations/layer5_attention_weight_max": 52.76036834716797,
      "activations/layer5_attention_weight_min": -62.97181701660156,
      "activations/layer6_attention_weight_max": 45.63252258300781,
      "activations/layer6_attention_weight_min": -46.19839096069336,
      "activations/layer7_attention_weight_max": 95.37947082519531,
      "activations/layer7_attention_weight_min": -93.82601928710938,
      "activations/layer8_attention_weight_max": 43.2630500793457,
      "activations/layer8_attention_weight_min": -43.28657913208008,
      "activations/layer9_attention_weight_max": 35.31751251220703,
      "activations/layer9_attention_weight_min": -34.338436126708984,
      "epoch": 18.07,
      "learning_rate": 3.3843560606060605e-05,
      "loss": 2.7254,
      "step": 310900
    },
    {
      "activations/layer0_attention_weight_max": 16.16313362121582,
      "activations/layer0_attention_weight_min": -13.784719467163086,
      "activations/layer10_attention_weight_max": 31.240360260009766,
      "activations/layer10_attention_weight_min": -32.39115905761719,
      "activations/layer11_attention_weight_max": 33.61654281616211,
      "activations/layer11_attention_weight_min": -32.97089385986328,
      "activations/layer12_attention_weight_max": 27.827299118041992,
      "activations/layer12_attention_weight_min": -26.46759796142578,
      "activations/layer13_attention_weight_max": 42.917659759521484,
      "activations/layer13_attention_weight_min": -37.896583557128906,
      "activations/layer14_attention_weight_max": 48.17441177368164,
      "activations/layer14_attention_weight_min": -38.70209503173828,
      "activations/layer15_attention_weight_max": 43.12995529174805,
      "activations/layer15_attention_weight_min": -36.72520065307617,
      "activations/layer16_attention_weight_max": 33.684486389160156,
      "activations/layer16_attention_weight_min": -30.21500587463379,
      "activations/layer17_attention_weight_max": 30.35661506652832,
      "activations/layer17_attention_weight_min": -26.372217178344727,
      "activations/layer18_attention_weight_max": 35.32319259643555,
      "activations/layer18_attention_weight_min": -26.84519386291504,
      "activations/layer19_attention_weight_max": 34.53196716308594,
      "activations/layer19_attention_weight_min": -32.90598678588867,
      "activations/layer1_attention_weight_max": 16.724470138549805,
      "activations/layer1_attention_weight_min": -12.500221252441406,
      "activations/layer20_attention_weight_max": 28.574687957763672,
      "activations/layer20_attention_weight_min": -25.242429733276367,
      "activations/layer21_attention_weight_max": 28.86655044555664,
      "activations/layer21_attention_weight_min": -26.441503524780273,
      "activations/layer22_attention_weight_max": 47.20686340332031,
      "activations/layer22_attention_weight_min": -30.72789192199707,
      "activations/layer23_attention_weight_max": 32.29825210571289,
      "activations/layer23_attention_weight_min": -25.794591903686523,
      "activations/layer2_attention_weight_max": 29.581457138061523,
      "activations/layer2_attention_weight_min": -29.282875061035156,
      "activations/layer3_attention_weight_max": 88.28689575195312,
      "activations/layer3_attention_weight_min": -90.303955078125,
      "activations/layer4_attention_weight_max": 100.02949523925781,
      "activations/layer4_attention_weight_min": -108.8917465209961,
      "activations/layer5_attention_weight_max": 48.71403503417969,
      "activations/layer5_attention_weight_min": -61.869285583496094,
      "activations/layer6_attention_weight_max": 40.68217468261719,
      "activations/layer6_attention_weight_min": -44.4290657043457,
      "activations/layer7_attention_weight_max": 91.16075134277344,
      "activations/layer7_attention_weight_min": -90.36150360107422,
      "activations/layer8_attention_weight_max": 38.41423797607422,
      "activations/layer8_attention_weight_min": -41.44277572631836,
      "activations/layer9_attention_weight_max": 29.13655662536621,
      "activations/layer9_attention_weight_min": -32.57493591308594,
      "epoch": 18.07,
      "learning_rate": 3.382462121212121e-05,
      "loss": 2.7288,
      "step": 310950
    },
    {
      "activations/layer0_attention_weight_max": 17.504133224487305,
      "activations/layer0_attention_weight_min": -13.566634178161621,
      "activations/layer10_attention_weight_max": 37.35293960571289,
      "activations/layer10_attention_weight_min": -33.46744918823242,
      "activations/layer11_attention_weight_max": 36.12660598754883,
      "activations/layer11_attention_weight_min": -35.651824951171875,
      "activations/layer12_attention_weight_max": 32.90431594848633,
      "activations/layer12_attention_weight_min": -28.70707130432129,
      "activations/layer13_attention_weight_max": 43.37755584716797,
      "activations/layer13_attention_weight_min": -38.63022994995117,
      "activations/layer14_attention_weight_max": 46.32972717285156,
      "activations/layer14_attention_weight_min": -42.17620849609375,
      "activations/layer15_attention_weight_max": 42.19838333129883,
      "activations/layer15_attention_weight_min": -39.7307014465332,
      "activations/layer16_attention_weight_max": 33.28704833984375,
      "activations/layer16_attention_weight_min": -28.22614288330078,
      "activations/layer17_attention_weight_max": 31.246070861816406,
      "activations/layer17_attention_weight_min": -27.53887367248535,
      "activations/layer18_attention_weight_max": 33.14820861816406,
      "activations/layer18_attention_weight_min": -26.43683624267578,
      "activations/layer19_attention_weight_max": 37.426395416259766,
      "activations/layer19_attention_weight_min": -33.40385055541992,
      "activations/layer1_attention_weight_max": 15.60076904296875,
      "activations/layer1_attention_weight_min": -15.51528549194336,
      "activations/layer20_attention_weight_max": 33.33952713012695,
      "activations/layer20_attention_weight_min": -25.887123107910156,
      "activations/layer21_attention_weight_max": 31.30661392211914,
      "activations/layer21_attention_weight_min": -25.061742782592773,
      "activations/layer22_attention_weight_max": 43.59302520751953,
      "activations/layer22_attention_weight_min": -32.758174896240234,
      "activations/layer23_attention_weight_max": 34.44442367553711,
      "activations/layer23_attention_weight_min": -26.736469268798828,
      "activations/layer2_attention_weight_max": 34.64305114746094,
      "activations/layer2_attention_weight_min": -31.5743408203125,
      "activations/layer3_attention_weight_max": 103.03801727294922,
      "activations/layer3_attention_weight_min": -99.4549331665039,
      "activations/layer4_attention_weight_max": 103.8702392578125,
      "activations/layer4_attention_weight_min": -105.98870086669922,
      "activations/layer5_attention_weight_max": 50.34259033203125,
      "activations/layer5_attention_weight_min": -61.00624084472656,
      "activations/layer6_attention_weight_max": 44.39079284667969,
      "activations/layer6_attention_weight_min": -44.855220794677734,
      "activations/layer7_attention_weight_max": 91.35689544677734,
      "activations/layer7_attention_weight_min": -91.80368041992188,
      "activations/layer8_attention_weight_max": 42.3889274597168,
      "activations/layer8_attention_weight_min": -44.20925521850586,
      "activations/layer9_attention_weight_max": 33.14704513549805,
      "activations/layer9_attention_weight_min": -35.69765853881836,
      "epoch": 18.07,
      "learning_rate": 3.3805681818181814e-05,
      "loss": 2.7291,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.3736,
      "eval_samples_per_second": 512.804,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.3736,
      "eval_openwebtext_samples_per_second": 512.804,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.7065,
      "eval_wikitext_samples_per_second": 267.208,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.6012,
      "eval_lambada_samples_per_second": 507.123,
      "step": 311000
    },
    {
      "activations/layer0_attention_weight_max": 16.404142379760742,
      "activations/layer0_attention_weight_min": -12.355602264404297,
      "activations/layer10_attention_weight_max": 36.74605178833008,
      "activations/layer10_attention_weight_min": -38.95317077636719,
      "activations/layer11_attention_weight_max": 36.00758361816406,
      "activations/layer11_attention_weight_min": -35.43317794799805,
      "activations/layer12_attention_weight_max": 26.735572814941406,
      "activations/layer12_attention_weight_min": -27.596471786499023,
      "activations/layer13_attention_weight_max": 37.1712532043457,
      "activations/layer13_attention_weight_min": -37.025978088378906,
      "activations/layer14_attention_weight_max": 44.595855712890625,
      "activations/layer14_attention_weight_min": -39.646217346191406,
      "activations/layer15_attention_weight_max": 37.02579879760742,
      "activations/layer15_attention_weight_min": -36.52187728881836,
      "activations/layer16_attention_weight_max": 32.96561050415039,
      "activations/layer16_attention_weight_min": -27.892486572265625,
      "activations/layer17_attention_weight_max": 31.2220401763916,
      "activations/layer17_attention_weight_min": -25.76222038269043,
      "activations/layer18_attention_weight_max": 31.674720764160156,
      "activations/layer18_attention_weight_min": -25.88628387451172,
      "activations/layer19_attention_weight_max": 40.4805908203125,
      "activations/layer19_attention_weight_min": -34.796348571777344,
      "activations/layer1_attention_weight_max": 15.951180458068848,
      "activations/layer1_attention_weight_min": -15.10952377319336,
      "activations/layer20_attention_weight_max": 32.718441009521484,
      "activations/layer20_attention_weight_min": -27.24062156677246,
      "activations/layer21_attention_weight_max": 31.981218338012695,
      "activations/layer21_attention_weight_min": -26.6295166015625,
      "activations/layer22_attention_weight_max": 43.20308303833008,
      "activations/layer22_attention_weight_min": -35.10121154785156,
      "activations/layer23_attention_weight_max": 32.712162017822266,
      "activations/layer23_attention_weight_min": -26.37965965270996,
      "activations/layer2_attention_weight_max": 33.374534606933594,
      "activations/layer2_attention_weight_min": -32.11946487426758,
      "activations/layer3_attention_weight_max": 108.08091735839844,
      "activations/layer3_attention_weight_min": -106.27232360839844,
      "activations/layer4_attention_weight_max": 113.74816131591797,
      "activations/layer4_attention_weight_min": -113.60585021972656,
      "activations/layer5_attention_weight_max": 52.52446746826172,
      "activations/layer5_attention_weight_min": -61.75727844238281,
      "activations/layer6_attention_weight_max": 47.397483825683594,
      "activations/layer6_attention_weight_min": -47.263633728027344,
      "activations/layer7_attention_weight_max": 94.14639282226562,
      "activations/layer7_attention_weight_min": -104.75603485107422,
      "activations/layer8_attention_weight_max": 43.24427795410156,
      "activations/layer8_attention_weight_min": -50.255428314208984,
      "activations/layer9_attention_weight_max": 34.29956817626953,
      "activations/layer9_attention_weight_min": -39.931976318359375,
      "epoch": 18.07,
      "learning_rate": 3.378674242424242e-05,
      "loss": 2.7268,
      "step": 311050
    },
    {
      "activations/layer0_attention_weight_max": 16.526325225830078,
      "activations/layer0_attention_weight_min": -12.749839782714844,
      "activations/layer10_attention_weight_max": 37.11187744140625,
      "activations/layer10_attention_weight_min": -36.3552360534668,
      "activations/layer11_attention_weight_max": 37.229488372802734,
      "activations/layer11_attention_weight_min": -35.17219161987305,
      "activations/layer12_attention_weight_max": 32.22245407104492,
      "activations/layer12_attention_weight_min": -28.32700538635254,
      "activations/layer13_attention_weight_max": 46.938392639160156,
      "activations/layer13_attention_weight_min": -39.53071594238281,
      "activations/layer14_attention_weight_max": 49.07455825805664,
      "activations/layer14_attention_weight_min": -41.976173400878906,
      "activations/layer15_attention_weight_max": 46.60916519165039,
      "activations/layer15_attention_weight_min": -37.49193572998047,
      "activations/layer16_attention_weight_max": 33.436187744140625,
      "activations/layer16_attention_weight_min": -28.211355209350586,
      "activations/layer17_attention_weight_max": 35.360347747802734,
      "activations/layer17_attention_weight_min": -26.71174430847168,
      "activations/layer18_attention_weight_max": 36.0162467956543,
      "activations/layer18_attention_weight_min": -26.182146072387695,
      "activations/layer19_attention_weight_max": 42.92238998413086,
      "activations/layer19_attention_weight_min": -34.48087692260742,
      "activations/layer1_attention_weight_max": 16.64059829711914,
      "activations/layer1_attention_weight_min": -14.488677024841309,
      "activations/layer20_attention_weight_max": 36.82517623901367,
      "activations/layer20_attention_weight_min": -28.87004852294922,
      "activations/layer21_attention_weight_max": 33.01447296142578,
      "activations/layer21_attention_weight_min": -26.888885498046875,
      "activations/layer22_attention_weight_max": 44.72160720825195,
      "activations/layer22_attention_weight_min": -33.69425582885742,
      "activations/layer23_attention_weight_max": 35.137107849121094,
      "activations/layer23_attention_weight_min": -26.07234001159668,
      "activations/layer2_attention_weight_max": 34.771785736083984,
      "activations/layer2_attention_weight_min": -32.6292610168457,
      "activations/layer3_attention_weight_max": 103.07404327392578,
      "activations/layer3_attention_weight_min": -106.8740463256836,
      "activations/layer4_attention_weight_max": 108.5853042602539,
      "activations/layer4_attention_weight_min": -110.78316497802734,
      "activations/layer5_attention_weight_max": 54.831825256347656,
      "activations/layer5_attention_weight_min": -63.86511993408203,
      "activations/layer6_attention_weight_max": 44.41307067871094,
      "activations/layer6_attention_weight_min": -46.798213958740234,
      "activations/layer7_attention_weight_max": 98.05351257324219,
      "activations/layer7_attention_weight_min": -97.35540771484375,
      "activations/layer8_attention_weight_max": 44.1688117980957,
      "activations/layer8_attention_weight_min": -46.16481018066406,
      "activations/layer9_attention_weight_max": 35.92251205444336,
      "activations/layer9_attention_weight_min": -36.88139343261719,
      "epoch": 18.08,
      "learning_rate": 3.3767803030303024e-05,
      "loss": 2.7163,
      "step": 311100
    },
    {
      "activations/layer0_attention_weight_max": 16.980440139770508,
      "activations/layer0_attention_weight_min": -12.622605323791504,
      "activations/layer10_attention_weight_max": 34.63105773925781,
      "activations/layer10_attention_weight_min": -35.070899963378906,
      "activations/layer11_attention_weight_max": 34.7003173828125,
      "activations/layer11_attention_weight_min": -34.773860931396484,
      "activations/layer12_attention_weight_max": 31.009191513061523,
      "activations/layer12_attention_weight_min": -27.931428909301758,
      "activations/layer13_attention_weight_max": 40.87994384765625,
      "activations/layer13_attention_weight_min": -38.303489685058594,
      "activations/layer14_attention_weight_max": 44.37175750732422,
      "activations/layer14_attention_weight_min": -38.047855377197266,
      "activations/layer15_attention_weight_max": 40.5192756652832,
      "activations/layer15_attention_weight_min": -35.61871337890625,
      "activations/layer16_attention_weight_max": 30.341833114624023,
      "activations/layer16_attention_weight_min": -31.117029190063477,
      "activations/layer17_attention_weight_max": 31.483469009399414,
      "activations/layer17_attention_weight_min": -26.923044204711914,
      "activations/layer18_attention_weight_max": 32.05768585205078,
      "activations/layer18_attention_weight_min": -27.30023193359375,
      "activations/layer19_attention_weight_max": 36.09492111206055,
      "activations/layer19_attention_weight_min": -33.000850677490234,
      "activations/layer1_attention_weight_max": 15.755915641784668,
      "activations/layer1_attention_weight_min": -15.059683799743652,
      "activations/layer20_attention_weight_max": 30.809513092041016,
      "activations/layer20_attention_weight_min": -28.07228660583496,
      "activations/layer21_attention_weight_max": 30.790658950805664,
      "activations/layer21_attention_weight_min": -26.035497665405273,
      "activations/layer22_attention_weight_max": 53.68748092651367,
      "activations/layer22_attention_weight_min": -32.7535514831543,
      "activations/layer23_attention_weight_max": 36.4447021484375,
      "activations/layer23_attention_weight_min": -26.19253921508789,
      "activations/layer2_attention_weight_max": 34.70768356323242,
      "activations/layer2_attention_weight_min": -33.509639739990234,
      "activations/layer3_attention_weight_max": 101.73847198486328,
      "activations/layer3_attention_weight_min": -103.49286651611328,
      "activations/layer4_attention_weight_max": 110.17595672607422,
      "activations/layer4_attention_weight_min": -115.5644302368164,
      "activations/layer5_attention_weight_max": 51.91278076171875,
      "activations/layer5_attention_weight_min": -66.81586456298828,
      "activations/layer6_attention_weight_max": 47.600467681884766,
      "activations/layer6_attention_weight_min": -49.38501739501953,
      "activations/layer7_attention_weight_max": 100.4274673461914,
      "activations/layer7_attention_weight_min": -101.8442153930664,
      "activations/layer8_attention_weight_max": 43.27363204956055,
      "activations/layer8_attention_weight_min": -45.667964935302734,
      "activations/layer9_attention_weight_max": 34.87160873413086,
      "activations/layer9_attention_weight_min": -35.465694427490234,
      "epoch": 18.08,
      "learning_rate": 3.374886363636363e-05,
      "loss": 2.7297,
      "step": 311150
    },
    {
      "activations/layer0_attention_weight_max": 16.694347381591797,
      "activations/layer0_attention_weight_min": -12.2611722946167,
      "activations/layer10_attention_weight_max": 34.015838623046875,
      "activations/layer10_attention_weight_min": -34.47850036621094,
      "activations/layer11_attention_weight_max": 33.63311767578125,
      "activations/layer11_attention_weight_min": -33.11076354980469,
      "activations/layer12_attention_weight_max": 25.786314010620117,
      "activations/layer12_attention_weight_min": -27.267745971679688,
      "activations/layer13_attention_weight_max": 38.97822570800781,
      "activations/layer13_attention_weight_min": -35.66450119018555,
      "activations/layer14_attention_weight_max": 40.421409606933594,
      "activations/layer14_attention_weight_min": -38.318275451660156,
      "activations/layer15_attention_weight_max": 38.332923889160156,
      "activations/layer15_attention_weight_min": -34.67740249633789,
      "activations/layer16_attention_weight_max": 28.78131675720215,
      "activations/layer16_attention_weight_min": -27.011802673339844,
      "activations/layer17_attention_weight_max": 29.79501724243164,
      "activations/layer17_attention_weight_min": -25.237022399902344,
      "activations/layer18_attention_weight_max": 30.590158462524414,
      "activations/layer18_attention_weight_min": -25.3754940032959,
      "activations/layer19_attention_weight_max": 33.21627426147461,
      "activations/layer19_attention_weight_min": -31.81214714050293,
      "activations/layer1_attention_weight_max": 15.903717994689941,
      "activations/layer1_attention_weight_min": -13.882993698120117,
      "activations/layer20_attention_weight_max": 29.718215942382812,
      "activations/layer20_attention_weight_min": -26.635986328125,
      "activations/layer21_attention_weight_max": 28.04158592224121,
      "activations/layer21_attention_weight_min": -24.87858009338379,
      "activations/layer22_attention_weight_max": 41.3907356262207,
      "activations/layer22_attention_weight_min": -31.05947494506836,
      "activations/layer23_attention_weight_max": 30.860519409179688,
      "activations/layer23_attention_weight_min": -25.118465423583984,
      "activations/layer2_attention_weight_max": 34.135040283203125,
      "activations/layer2_attention_weight_min": -31.486265182495117,
      "activations/layer3_attention_weight_max": 97.66352081298828,
      "activations/layer3_attention_weight_min": -98.41927337646484,
      "activations/layer4_attention_weight_max": 103.51251220703125,
      "activations/layer4_attention_weight_min": -107.29937744140625,
      "activations/layer5_attention_weight_max": 47.83354949951172,
      "activations/layer5_attention_weight_min": -58.64366912841797,
      "activations/layer6_attention_weight_max": 43.20667266845703,
      "activations/layer6_attention_weight_min": -46.1330451965332,
      "activations/layer7_attention_weight_max": 93.38651275634766,
      "activations/layer7_attention_weight_min": -101.71085357666016,
      "activations/layer8_attention_weight_max": 42.5722541809082,
      "activations/layer8_attention_weight_min": -45.10938262939453,
      "activations/layer9_attention_weight_max": 32.121803283691406,
      "activations/layer9_attention_weight_min": -34.120635986328125,
      "epoch": 18.08,
      "learning_rate": 3.372992424242424e-05,
      "loss": 2.7203,
      "step": 311200
    },
    {
      "activations/layer0_attention_weight_max": 15.70280647277832,
      "activations/layer0_attention_weight_min": -12.439513206481934,
      "activations/layer10_attention_weight_max": 35.40523147583008,
      "activations/layer10_attention_weight_min": -35.67116165161133,
      "activations/layer11_attention_weight_max": 35.38330841064453,
      "activations/layer11_attention_weight_min": -37.023216247558594,
      "activations/layer12_attention_weight_max": 26.60474395751953,
      "activations/layer12_attention_weight_min": -27.11834144592285,
      "activations/layer13_attention_weight_max": 42.05818176269531,
      "activations/layer13_attention_weight_min": -35.515899658203125,
      "activations/layer14_attention_weight_max": 46.2920036315918,
      "activations/layer14_attention_weight_min": -38.02993392944336,
      "activations/layer15_attention_weight_max": 38.382568359375,
      "activations/layer15_attention_weight_min": -36.51971435546875,
      "activations/layer16_attention_weight_max": 34.34307098388672,
      "activations/layer16_attention_weight_min": -29.536849975585938,
      "activations/layer17_attention_weight_max": 28.829864501953125,
      "activations/layer17_attention_weight_min": -25.109560012817383,
      "activations/layer18_attention_weight_max": 30.609840393066406,
      "activations/layer18_attention_weight_min": -26.426692962646484,
      "activations/layer19_attention_weight_max": 39.0412712097168,
      "activations/layer19_attention_weight_min": -34.57292938232422,
      "activations/layer1_attention_weight_max": 15.997519493103027,
      "activations/layer1_attention_weight_min": -14.08848762512207,
      "activations/layer20_attention_weight_max": 31.126113891601562,
      "activations/layer20_attention_weight_min": -26.954296112060547,
      "activations/layer21_attention_weight_max": 29.50139617919922,
      "activations/layer21_attention_weight_min": -25.95299530029297,
      "activations/layer22_attention_weight_max": 51.701438903808594,
      "activations/layer22_attention_weight_min": -32.81788635253906,
      "activations/layer23_attention_weight_max": 37.59089660644531,
      "activations/layer23_attention_weight_min": -26.584928512573242,
      "activations/layer2_attention_weight_max": 34.020904541015625,
      "activations/layer2_attention_weight_min": -31.34279441833496,
      "activations/layer3_attention_weight_max": 99.59536743164062,
      "activations/layer3_attention_weight_min": -101.06671142578125,
      "activations/layer4_attention_weight_max": 107.81206512451172,
      "activations/layer4_attention_weight_min": -109.31459045410156,
      "activations/layer5_attention_weight_max": 52.25382614135742,
      "activations/layer5_attention_weight_min": -63.94707489013672,
      "activations/layer6_attention_weight_max": 43.72866439819336,
      "activations/layer6_attention_weight_min": -46.17355728149414,
      "activations/layer7_attention_weight_max": 92.4783706665039,
      "activations/layer7_attention_weight_min": -99.3902359008789,
      "activations/layer8_attention_weight_max": 42.41188430786133,
      "activations/layer8_attention_weight_min": -44.420223236083984,
      "activations/layer9_attention_weight_max": 31.9495906829834,
      "activations/layer9_attention_weight_min": -36.181087493896484,
      "epoch": 18.09,
      "learning_rate": 3.371098484848484e-05,
      "loss": 2.7214,
      "step": 311250
    },
    {
      "activations/layer0_attention_weight_max": 15.862665176391602,
      "activations/layer0_attention_weight_min": -12.976953506469727,
      "activations/layer10_attention_weight_max": 33.0077018737793,
      "activations/layer10_attention_weight_min": -34.61748123168945,
      "activations/layer11_attention_weight_max": 35.73870086669922,
      "activations/layer11_attention_weight_min": -34.46350860595703,
      "activations/layer12_attention_weight_max": 27.04027557373047,
      "activations/layer12_attention_weight_min": -27.605546951293945,
      "activations/layer13_attention_weight_max": 39.99385452270508,
      "activations/layer13_attention_weight_min": -35.34821701049805,
      "activations/layer14_attention_weight_max": 42.23332595825195,
      "activations/layer14_attention_weight_min": -36.10209655761719,
      "activations/layer15_attention_weight_max": 38.4640998840332,
      "activations/layer15_attention_weight_min": -33.94136428833008,
      "activations/layer16_attention_weight_max": 28.542274475097656,
      "activations/layer16_attention_weight_min": -26.943693161010742,
      "activations/layer17_attention_weight_max": 27.860637664794922,
      "activations/layer17_attention_weight_min": -26.195560455322266,
      "activations/layer18_attention_weight_max": 32.24757766723633,
      "activations/layer18_attention_weight_min": -27.572772979736328,
      "activations/layer19_attention_weight_max": 32.458736419677734,
      "activations/layer19_attention_weight_min": -32.9053840637207,
      "activations/layer1_attention_weight_max": 15.913999557495117,
      "activations/layer1_attention_weight_min": -15.543607711791992,
      "activations/layer20_attention_weight_max": 27.7542781829834,
      "activations/layer20_attention_weight_min": -25.020095825195312,
      "activations/layer21_attention_weight_max": 27.929698944091797,
      "activations/layer21_attention_weight_min": -25.375228881835938,
      "activations/layer22_attention_weight_max": 42.357322692871094,
      "activations/layer22_attention_weight_min": -33.75282287597656,
      "activations/layer23_attention_weight_max": 32.60783004760742,
      "activations/layer23_attention_weight_min": -25.595325469970703,
      "activations/layer2_attention_weight_max": 33.0354118347168,
      "activations/layer2_attention_weight_min": -32.531471252441406,
      "activations/layer3_attention_weight_max": 95.91928100585938,
      "activations/layer3_attention_weight_min": -96.81517028808594,
      "activations/layer4_attention_weight_max": 105.48632049560547,
      "activations/layer4_attention_weight_min": -114.26073455810547,
      "activations/layer5_attention_weight_max": 49.88309860229492,
      "activations/layer5_attention_weight_min": -62.942195892333984,
      "activations/layer6_attention_weight_max": 43.09217834472656,
      "activations/layer6_attention_weight_min": -45.893550872802734,
      "activations/layer7_attention_weight_max": 87.93753814697266,
      "activations/layer7_attention_weight_min": -93.53520202636719,
      "activations/layer8_attention_weight_max": 39.53218460083008,
      "activations/layer8_attention_weight_min": -43.33793258666992,
      "activations/layer9_attention_weight_max": 31.794235229492188,
      "activations/layer9_attention_weight_min": -32.65407943725586,
      "epoch": 18.09,
      "learning_rate": 3.369204545454545e-05,
      "loss": 2.7389,
      "step": 311300
    },
    {
      "activations/layer0_attention_weight_max": 16.248098373413086,
      "activations/layer0_attention_weight_min": -13.333131790161133,
      "activations/layer10_attention_weight_max": 32.16978073120117,
      "activations/layer10_attention_weight_min": -32.6362419128418,
      "activations/layer11_attention_weight_max": 31.748271942138672,
      "activations/layer11_attention_weight_min": -32.597694396972656,
      "activations/layer12_attention_weight_max": 25.610214233398438,
      "activations/layer12_attention_weight_min": -26.977670669555664,
      "activations/layer13_attention_weight_max": 37.459808349609375,
      "activations/layer13_attention_weight_min": -33.41062927246094,
      "activations/layer14_attention_weight_max": 38.46855163574219,
      "activations/layer14_attention_weight_min": -35.88591384887695,
      "activations/layer15_attention_weight_max": 34.417335510253906,
      "activations/layer15_attention_weight_min": -35.070091247558594,
      "activations/layer16_attention_weight_max": 28.390064239501953,
      "activations/layer16_attention_weight_min": -26.763822555541992,
      "activations/layer17_attention_weight_max": 26.991357803344727,
      "activations/layer17_attention_weight_min": -25.766096115112305,
      "activations/layer18_attention_weight_max": 30.3514347076416,
      "activations/layer18_attention_weight_min": -23.869140625,
      "activations/layer19_attention_weight_max": 30.904056549072266,
      "activations/layer19_attention_weight_min": -32.706207275390625,
      "activations/layer1_attention_weight_max": 15.877991676330566,
      "activations/layer1_attention_weight_min": -14.657854080200195,
      "activations/layer20_attention_weight_max": 27.526885986328125,
      "activations/layer20_attention_weight_min": -26.059904098510742,
      "activations/layer21_attention_weight_max": 25.673521041870117,
      "activations/layer21_attention_weight_min": -24.97432518005371,
      "activations/layer22_attention_weight_max": 34.939361572265625,
      "activations/layer22_attention_weight_min": -29.78059196472168,
      "activations/layer23_attention_weight_max": 30.353527069091797,
      "activations/layer23_attention_weight_min": -24.09814453125,
      "activations/layer2_attention_weight_max": 33.82076644897461,
      "activations/layer2_attention_weight_min": -32.61327362060547,
      "activations/layer3_attention_weight_max": 102.11788940429688,
      "activations/layer3_attention_weight_min": -102.20916748046875,
      "activations/layer4_attention_weight_max": 108.7889175415039,
      "activations/layer4_attention_weight_min": -107.2231674194336,
      "activations/layer5_attention_weight_max": 55.71364974975586,
      "activations/layer5_attention_weight_min": -60.93281555175781,
      "activations/layer6_attention_weight_max": 43.91868209838867,
      "activations/layer6_attention_weight_min": -46.72539520263672,
      "activations/layer7_attention_weight_max": 90.69937896728516,
      "activations/layer7_attention_weight_min": -94.31389617919922,
      "activations/layer8_attention_weight_max": 39.92938995361328,
      "activations/layer8_attention_weight_min": -41.62384033203125,
      "activations/layer9_attention_weight_max": 31.069076538085938,
      "activations/layer9_attention_weight_min": -32.4163703918457,
      "epoch": 18.09,
      "learning_rate": 3.367310606060606e-05,
      "loss": 2.7226,
      "step": 311350
    },
    {
      "activations/layer0_attention_weight_max": 17.60944175720215,
      "activations/layer0_attention_weight_min": -13.029720306396484,
      "activations/layer10_attention_weight_max": 45.4715461730957,
      "activations/layer10_attention_weight_min": -40.276458740234375,
      "activations/layer11_attention_weight_max": 45.36723327636719,
      "activations/layer11_attention_weight_min": -42.174686431884766,
      "activations/layer12_attention_weight_max": 26.23836898803711,
      "activations/layer12_attention_weight_min": -28.292909622192383,
      "activations/layer13_attention_weight_max": 36.57231140136719,
      "activations/layer13_attention_weight_min": -33.992645263671875,
      "activations/layer14_attention_weight_max": 41.34370422363281,
      "activations/layer14_attention_weight_min": -37.03001022338867,
      "activations/layer15_attention_weight_max": 34.84457015991211,
      "activations/layer15_attention_weight_min": -35.235713958740234,
      "activations/layer16_attention_weight_max": 27.22087860107422,
      "activations/layer16_attention_weight_min": -29.09313201904297,
      "activations/layer17_attention_weight_max": 28.41301918029785,
      "activations/layer17_attention_weight_min": -26.7532901763916,
      "activations/layer18_attention_weight_max": 29.705669403076172,
      "activations/layer18_attention_weight_min": -25.853939056396484,
      "activations/layer19_attention_weight_max": 30.696792602539062,
      "activations/layer19_attention_weight_min": -31.29791259765625,
      "activations/layer1_attention_weight_max": 16.381391525268555,
      "activations/layer1_attention_weight_min": -15.111842155456543,
      "activations/layer20_attention_weight_max": 25.317502975463867,
      "activations/layer20_attention_weight_min": -26.396467208862305,
      "activations/layer21_attention_weight_max": 27.325698852539062,
      "activations/layer21_attention_weight_min": -24.727834701538086,
      "activations/layer22_attention_weight_max": 40.16352844238281,
      "activations/layer22_attention_weight_min": -32.409263610839844,
      "activations/layer23_attention_weight_max": 30.867109298706055,
      "activations/layer23_attention_weight_min": -26.369359970092773,
      "activations/layer2_attention_weight_max": 33.03043746948242,
      "activations/layer2_attention_weight_min": -34.350860595703125,
      "activations/layer3_attention_weight_max": 101.44132232666016,
      "activations/layer3_attention_weight_min": -102.76790618896484,
      "activations/layer4_attention_weight_max": 112.68770599365234,
      "activations/layer4_attention_weight_min": -107.23270416259766,
      "activations/layer5_attention_weight_max": 53.1464958190918,
      "activations/layer5_attention_weight_min": -60.281646728515625,
      "activations/layer6_attention_weight_max": 43.85935592651367,
      "activations/layer6_attention_weight_min": -43.76992416381836,
      "activations/layer7_attention_weight_max": 114.3321533203125,
      "activations/layer7_attention_weight_min": -95.4720687866211,
      "activations/layer8_attention_weight_max": 53.563804626464844,
      "activations/layer8_attention_weight_min": -49.26643753051758,
      "activations/layer9_attention_weight_max": 40.62091064453125,
      "activations/layer9_attention_weight_min": -44.44392013549805,
      "epoch": 18.09,
      "learning_rate": 3.365416666666667e-05,
      "loss": 2.7366,
      "step": 311400
    },
    {
      "activations/layer0_attention_weight_max": 15.880003929138184,
      "activations/layer0_attention_weight_min": -13.277724266052246,
      "activations/layer10_attention_weight_max": 36.59477615356445,
      "activations/layer10_attention_weight_min": -35.73346710205078,
      "activations/layer11_attention_weight_max": 35.946868896484375,
      "activations/layer11_attention_weight_min": -35.25355529785156,
      "activations/layer12_attention_weight_max": 29.744239807128906,
      "activations/layer12_attention_weight_min": -27.789352416992188,
      "activations/layer13_attention_weight_max": 43.14336395263672,
      "activations/layer13_attention_weight_min": -37.60933303833008,
      "activations/layer14_attention_weight_max": 50.64448928833008,
      "activations/layer14_attention_weight_min": -41.68724060058594,
      "activations/layer15_attention_weight_max": 44.88053512573242,
      "activations/layer15_attention_weight_min": -38.86471939086914,
      "activations/layer16_attention_weight_max": 34.62934494018555,
      "activations/layer16_attention_weight_min": -31.385271072387695,
      "activations/layer17_attention_weight_max": 34.191898345947266,
      "activations/layer17_attention_weight_min": -29.242082595825195,
      "activations/layer18_attention_weight_max": 32.42958068847656,
      "activations/layer18_attention_weight_min": -26.579097747802734,
      "activations/layer19_attention_weight_max": 38.59413146972656,
      "activations/layer19_attention_weight_min": -32.14236068725586,
      "activations/layer1_attention_weight_max": 16.20254898071289,
      "activations/layer1_attention_weight_min": -14.980708122253418,
      "activations/layer20_attention_weight_max": 33.422298431396484,
      "activations/layer20_attention_weight_min": -26.723804473876953,
      "activations/layer21_attention_weight_max": 30.38224983215332,
      "activations/layer21_attention_weight_min": -24.539762496948242,
      "activations/layer22_attention_weight_max": 45.478759765625,
      "activations/layer22_attention_weight_min": -32.435672760009766,
      "activations/layer23_attention_weight_max": 34.78998565673828,
      "activations/layer23_attention_weight_min": -25.475685119628906,
      "activations/layer2_attention_weight_max": 35.28795623779297,
      "activations/layer2_attention_weight_min": -34.48681640625,
      "activations/layer3_attention_weight_max": 101.72905731201172,
      "activations/layer3_attention_weight_min": -104.65037536621094,
      "activations/layer4_attention_weight_max": 109.96012115478516,
      "activations/layer4_attention_weight_min": -109.23683166503906,
      "activations/layer5_attention_weight_max": 52.49168014526367,
      "activations/layer5_attention_weight_min": -59.016902923583984,
      "activations/layer6_attention_weight_max": 43.730106353759766,
      "activations/layer6_attention_weight_min": -45.077857971191406,
      "activations/layer7_attention_weight_max": 98.52151489257812,
      "activations/layer7_attention_weight_min": -101.90601348876953,
      "activations/layer8_attention_weight_max": 42.9049186706543,
      "activations/layer8_attention_weight_min": -45.85994338989258,
      "activations/layer9_attention_weight_max": 33.88865661621094,
      "activations/layer9_attention_weight_min": -37.175228118896484,
      "epoch": 18.1,
      "learning_rate": 3.363522727272727e-05,
      "loss": 2.7313,
      "step": 311450
    },
    {
      "activations/layer0_attention_weight_max": 17.48343849182129,
      "activations/layer0_attention_weight_min": -12.54029369354248,
      "activations/layer10_attention_weight_max": 33.16611099243164,
      "activations/layer10_attention_weight_min": -33.25052261352539,
      "activations/layer11_attention_weight_max": 32.29046630859375,
      "activations/layer11_attention_weight_min": -33.64056396484375,
      "activations/layer12_attention_weight_max": 27.70642852783203,
      "activations/layer12_attention_weight_min": -26.525373458862305,
      "activations/layer13_attention_weight_max": 39.44086837768555,
      "activations/layer13_attention_weight_min": -37.76007843017578,
      "activations/layer14_attention_weight_max": 50.71561813354492,
      "activations/layer14_attention_weight_min": -41.84694290161133,
      "activations/layer15_attention_weight_max": 44.638919830322266,
      "activations/layer15_attention_weight_min": -37.528385162353516,
      "activations/layer16_attention_weight_max": 35.72941970825195,
      "activations/layer16_attention_weight_min": -29.35936737060547,
      "activations/layer17_attention_weight_max": 33.53230667114258,
      "activations/layer17_attention_weight_min": -27.47966194152832,
      "activations/layer18_attention_weight_max": 33.51340866088867,
      "activations/layer18_attention_weight_min": -26.243587493896484,
      "activations/layer19_attention_weight_max": 39.33677673339844,
      "activations/layer19_attention_weight_min": -32.80015182495117,
      "activations/layer1_attention_weight_max": 16.235837936401367,
      "activations/layer1_attention_weight_min": -14.492177963256836,
      "activations/layer20_attention_weight_max": 36.26569366455078,
      "activations/layer20_attention_weight_min": -28.993085861206055,
      "activations/layer21_attention_weight_max": 33.259437561035156,
      "activations/layer21_attention_weight_min": -26.593130111694336,
      "activations/layer22_attention_weight_max": 48.636810302734375,
      "activations/layer22_attention_weight_min": -34.344017028808594,
      "activations/layer23_attention_weight_max": 40.58787536621094,
      "activations/layer23_attention_weight_min": -28.122852325439453,
      "activations/layer2_attention_weight_max": 32.92513656616211,
      "activations/layer2_attention_weight_min": -32.330326080322266,
      "activations/layer3_attention_weight_max": 101.34700775146484,
      "activations/layer3_attention_weight_min": -98.26382446289062,
      "activations/layer4_attention_weight_max": 106.65376281738281,
      "activations/layer4_attention_weight_min": -107.68172454833984,
      "activations/layer5_attention_weight_max": 50.658782958984375,
      "activations/layer5_attention_weight_min": -63.26329040527344,
      "activations/layer6_attention_weight_max": 41.59226608276367,
      "activations/layer6_attention_weight_min": -46.104896545410156,
      "activations/layer7_attention_weight_max": 93.19700622558594,
      "activations/layer7_attention_weight_min": -92.30110931396484,
      "activations/layer8_attention_weight_max": 39.31621170043945,
      "activations/layer8_attention_weight_min": -41.742286682128906,
      "activations/layer9_attention_weight_max": 31.758529663085938,
      "activations/layer9_attention_weight_min": -33.31280517578125,
      "epoch": 18.1,
      "learning_rate": 3.361628787878788e-05,
      "loss": 2.7289,
      "step": 311500
    },
    {
      "activations/layer0_attention_weight_max": 16.070669174194336,
      "activations/layer0_attention_weight_min": -14.100634574890137,
      "activations/layer10_attention_weight_max": 43.098907470703125,
      "activations/layer10_attention_weight_min": -38.7257080078125,
      "activations/layer11_attention_weight_max": 43.14817428588867,
      "activations/layer11_attention_weight_min": -41.04384231567383,
      "activations/layer12_attention_weight_max": 29.535341262817383,
      "activations/layer12_attention_weight_min": -28.72164535522461,
      "activations/layer13_attention_weight_max": 41.9276123046875,
      "activations/layer13_attention_weight_min": -40.75425720214844,
      "activations/layer14_attention_weight_max": 47.91397476196289,
      "activations/layer14_attention_weight_min": -46.07017135620117,
      "activations/layer15_attention_weight_max": 42.916229248046875,
      "activations/layer15_attention_weight_min": -40.1026496887207,
      "activations/layer16_attention_weight_max": 35.52915954589844,
      "activations/layer16_attention_weight_min": -29.322736740112305,
      "activations/layer17_attention_weight_max": 28.783395767211914,
      "activations/layer17_attention_weight_min": -27.70574378967285,
      "activations/layer18_attention_weight_max": 33.080570220947266,
      "activations/layer18_attention_weight_min": -29.53584098815918,
      "activations/layer19_attention_weight_max": 40.7721061706543,
      "activations/layer19_attention_weight_min": -37.83904266357422,
      "activations/layer1_attention_weight_max": 16.939573287963867,
      "activations/layer1_attention_weight_min": -14.55509090423584,
      "activations/layer20_attention_weight_max": 31.3104248046875,
      "activations/layer20_attention_weight_min": -28.505104064941406,
      "activations/layer21_attention_weight_max": 31.417190551757812,
      "activations/layer21_attention_weight_min": -27.80954360961914,
      "activations/layer22_attention_weight_max": 44.05084991455078,
      "activations/layer22_attention_weight_min": -34.682647705078125,
      "activations/layer23_attention_weight_max": 33.41536331176758,
      "activations/layer23_attention_weight_min": -25.693471908569336,
      "activations/layer2_attention_weight_max": 35.58802795410156,
      "activations/layer2_attention_weight_min": -32.371803283691406,
      "activations/layer3_attention_weight_max": 97.89124298095703,
      "activations/layer3_attention_weight_min": -98.15388488769531,
      "activations/layer4_attention_weight_max": 102.40657806396484,
      "activations/layer4_attention_weight_min": -102.09838104248047,
      "activations/layer5_attention_weight_max": 48.3052978515625,
      "activations/layer5_attention_weight_min": -58.939857482910156,
      "activations/layer6_attention_weight_max": 42.57918930053711,
      "activations/layer6_attention_weight_min": -46.493919372558594,
      "activations/layer7_attention_weight_max": 101.8616714477539,
      "activations/layer7_attention_weight_min": -95.67032623291016,
      "activations/layer8_attention_weight_max": 50.19020462036133,
      "activations/layer8_attention_weight_min": -47.08367156982422,
      "activations/layer9_attention_weight_max": 39.02748489379883,
      "activations/layer9_attention_weight_min": -38.18638229370117,
      "epoch": 18.1,
      "learning_rate": 3.359734848484848e-05,
      "loss": 2.7239,
      "step": 311550
    },
    {
      "activations/layer0_attention_weight_max": 16.30557632446289,
      "activations/layer0_attention_weight_min": -13.855186462402344,
      "activations/layer10_attention_weight_max": 34.35954666137695,
      "activations/layer10_attention_weight_min": -34.420448303222656,
      "activations/layer11_attention_weight_max": 36.747833251953125,
      "activations/layer11_attention_weight_min": -36.19054412841797,
      "activations/layer12_attention_weight_max": 28.173898696899414,
      "activations/layer12_attention_weight_min": -28.01228141784668,
      "activations/layer13_attention_weight_max": 42.715824127197266,
      "activations/layer13_attention_weight_min": -37.9179801940918,
      "activations/layer14_attention_weight_max": 46.27727127075195,
      "activations/layer14_attention_weight_min": -38.432552337646484,
      "activations/layer15_attention_weight_max": 40.518226623535156,
      "activations/layer15_attention_weight_min": -37.17144775390625,
      "activations/layer16_attention_weight_max": 31.919246673583984,
      "activations/layer16_attention_weight_min": -27.794137954711914,
      "activations/layer17_attention_weight_max": 35.47139358520508,
      "activations/layer17_attention_weight_min": -27.326189041137695,
      "activations/layer18_attention_weight_max": 34.9105339050293,
      "activations/layer18_attention_weight_min": -26.2867488861084,
      "activations/layer19_attention_weight_max": 39.31813049316406,
      "activations/layer19_attention_weight_min": -33.91997528076172,
      "activations/layer1_attention_weight_max": 15.461832046508789,
      "activations/layer1_attention_weight_min": -13.810379981994629,
      "activations/layer20_attention_weight_max": 32.122047424316406,
      "activations/layer20_attention_weight_min": -26.505247116088867,
      "activations/layer21_attention_weight_max": 31.807321548461914,
      "activations/layer21_attention_weight_min": -25.976699829101562,
      "activations/layer22_attention_weight_max": 48.34922790527344,
      "activations/layer22_attention_weight_min": -34.789817810058594,
      "activations/layer23_attention_weight_max": 35.1839714050293,
      "activations/layer23_attention_weight_min": -27.085262298583984,
      "activations/layer2_attention_weight_max": 32.281036376953125,
      "activations/layer2_attention_weight_min": -32.31487274169922,
      "activations/layer3_attention_weight_max": 101.35591888427734,
      "activations/layer3_attention_weight_min": -99.05797576904297,
      "activations/layer4_attention_weight_max": 110.87164306640625,
      "activations/layer4_attention_weight_min": -111.76240539550781,
      "activations/layer5_attention_weight_max": 51.566768646240234,
      "activations/layer5_attention_weight_min": -66.52582550048828,
      "activations/layer6_attention_weight_max": 45.2493782043457,
      "activations/layer6_attention_weight_min": -48.20170211791992,
      "activations/layer7_attention_weight_max": 97.61137390136719,
      "activations/layer7_attention_weight_min": -100.63591003417969,
      "activations/layer8_attention_weight_max": 44.6050910949707,
      "activations/layer8_attention_weight_min": -44.77239990234375,
      "activations/layer9_attention_weight_max": 33.45763397216797,
      "activations/layer9_attention_weight_min": -37.97658920288086,
      "epoch": 18.11,
      "learning_rate": 3.3578409090909087e-05,
      "loss": 2.7273,
      "step": 311600
    },
    {
      "activations/layer0_attention_weight_max": 16.50774574279785,
      "activations/layer0_attention_weight_min": -13.529571533203125,
      "activations/layer10_attention_weight_max": 32.09600830078125,
      "activations/layer10_attention_weight_min": -35.318241119384766,
      "activations/layer11_attention_weight_max": 32.40991973876953,
      "activations/layer11_attention_weight_min": -31.975528717041016,
      "activations/layer12_attention_weight_max": 24.08575439453125,
      "activations/layer12_attention_weight_min": -27.27625274658203,
      "activations/layer13_attention_weight_max": 35.45640563964844,
      "activations/layer13_attention_weight_min": -35.60315704345703,
      "activations/layer14_attention_weight_max": 42.11084747314453,
      "activations/layer14_attention_weight_min": -36.22224426269531,
      "activations/layer15_attention_weight_max": 36.485816955566406,
      "activations/layer15_attention_weight_min": -31.817333221435547,
      "activations/layer16_attention_weight_max": 27.90293312072754,
      "activations/layer16_attention_weight_min": -28.425004959106445,
      "activations/layer17_attention_weight_max": 26.3819522857666,
      "activations/layer17_attention_weight_min": -25.340744018554688,
      "activations/layer18_attention_weight_max": 33.297210693359375,
      "activations/layer18_attention_weight_min": -25.30705451965332,
      "activations/layer19_attention_weight_max": 35.61756134033203,
      "activations/layer19_attention_weight_min": -31.121051788330078,
      "activations/layer1_attention_weight_max": 15.854737281799316,
      "activations/layer1_attention_weight_min": -15.351828575134277,
      "activations/layer20_attention_weight_max": 30.880653381347656,
      "activations/layer20_attention_weight_min": -26.27812957763672,
      "activations/layer21_attention_weight_max": 28.825803756713867,
      "activations/layer21_attention_weight_min": -25.72519874572754,
      "activations/layer22_attention_weight_max": 38.84198760986328,
      "activations/layer22_attention_weight_min": -32.22752380371094,
      "activations/layer23_attention_weight_max": 33.4072265625,
      "activations/layer23_attention_weight_min": -26.738033294677734,
      "activations/layer2_attention_weight_max": 34.36567687988281,
      "activations/layer2_attention_weight_min": -32.101593017578125,
      "activations/layer3_attention_weight_max": 96.63194274902344,
      "activations/layer3_attention_weight_min": -98.25291442871094,
      "activations/layer4_attention_weight_max": 104.29054260253906,
      "activations/layer4_attention_weight_min": -108.90544891357422,
      "activations/layer5_attention_weight_max": 47.91764450073242,
      "activations/layer5_attention_weight_min": -59.50995635986328,
      "activations/layer6_attention_weight_max": 42.683876037597656,
      "activations/layer6_attention_weight_min": -45.66679382324219,
      "activations/layer7_attention_weight_max": 86.598876953125,
      "activations/layer7_attention_weight_min": -93.34097290039062,
      "activations/layer8_attention_weight_max": 38.31409454345703,
      "activations/layer8_attention_weight_min": -40.97306442260742,
      "activations/layer9_attention_weight_max": 30.117956161499023,
      "activations/layer9_attention_weight_min": -33.14222717285156,
      "epoch": 18.11,
      "learning_rate": 3.3559469696969695e-05,
      "loss": 2.7393,
      "step": 311650
    },
    {
      "activations/layer0_attention_weight_max": 17.312849044799805,
      "activations/layer0_attention_weight_min": -12.664816856384277,
      "activations/layer10_attention_weight_max": 35.9955940246582,
      "activations/layer10_attention_weight_min": -32.56971740722656,
      "activations/layer11_attention_weight_max": 34.9337158203125,
      "activations/layer11_attention_weight_min": -33.360599517822266,
      "activations/layer12_attention_weight_max": 24.832685470581055,
      "activations/layer12_attention_weight_min": -25.219562530517578,
      "activations/layer13_attention_weight_max": 38.836181640625,
      "activations/layer13_attention_weight_min": -35.323760986328125,
      "activations/layer14_attention_weight_max": 42.6099967956543,
      "activations/layer14_attention_weight_min": -37.241600036621094,
      "activations/layer15_attention_weight_max": 37.724124908447266,
      "activations/layer15_attention_weight_min": -35.78835678100586,
      "activations/layer16_attention_weight_max": 34.25185775756836,
      "activations/layer16_attention_weight_min": -31.04445457458496,
      "activations/layer17_attention_weight_max": 29.02913475036621,
      "activations/layer17_attention_weight_min": -26.0441837310791,
      "activations/layer18_attention_weight_max": 31.192081451416016,
      "activations/layer18_attention_weight_min": -25.65076446533203,
      "activations/layer19_attention_weight_max": 33.17186737060547,
      "activations/layer19_attention_weight_min": -33.135414123535156,
      "activations/layer1_attention_weight_max": 15.273730278015137,
      "activations/layer1_attention_weight_min": -15.145533561706543,
      "activations/layer20_attention_weight_max": 28.39449691772461,
      "activations/layer20_attention_weight_min": -26.17443084716797,
      "activations/layer21_attention_weight_max": 28.22222900390625,
      "activations/layer21_attention_weight_min": -25.59781837463379,
      "activations/layer22_attention_weight_max": 40.75639724731445,
      "activations/layer22_attention_weight_min": -32.47114181518555,
      "activations/layer23_attention_weight_max": 36.71619415283203,
      "activations/layer23_attention_weight_min": -25.81783103942871,
      "activations/layer2_attention_weight_max": 34.97625732421875,
      "activations/layer2_attention_weight_min": -33.737091064453125,
      "activations/layer3_attention_weight_max": 104.6341323852539,
      "activations/layer3_attention_weight_min": -106.00665283203125,
      "activations/layer4_attention_weight_max": 109.30516815185547,
      "activations/layer4_attention_weight_min": -112.5028076171875,
      "activations/layer5_attention_weight_max": 52.47761535644531,
      "activations/layer5_attention_weight_min": -59.85041046142578,
      "activations/layer6_attention_weight_max": 43.99970626831055,
      "activations/layer6_attention_weight_min": -44.59543228149414,
      "activations/layer7_attention_weight_max": 90.16534423828125,
      "activations/layer7_attention_weight_min": -91.31426239013672,
      "activations/layer8_attention_weight_max": 42.72157669067383,
      "activations/layer8_attention_weight_min": -41.30552291870117,
      "activations/layer9_attention_weight_max": 31.285085678100586,
      "activations/layer9_attention_weight_min": -32.624027252197266,
      "epoch": 18.11,
      "learning_rate": 3.35405303030303e-05,
      "loss": 2.725,
      "step": 311700
    },
    {
      "activations/layer0_attention_weight_max": 16.312915802001953,
      "activations/layer0_attention_weight_min": -12.887224197387695,
      "activations/layer10_attention_weight_max": 34.04234313964844,
      "activations/layer10_attention_weight_min": -34.73602294921875,
      "activations/layer11_attention_weight_max": 34.0284423828125,
      "activations/layer11_attention_weight_min": -35.33318328857422,
      "activations/layer12_attention_weight_max": 26.377662658691406,
      "activations/layer12_attention_weight_min": -28.554283142089844,
      "activations/layer13_attention_weight_max": 40.71025085449219,
      "activations/layer13_attention_weight_min": -37.437252044677734,
      "activations/layer14_attention_weight_max": 43.283203125,
      "activations/layer14_attention_weight_min": -37.80817413330078,
      "activations/layer15_attention_weight_max": 39.2930793762207,
      "activations/layer15_attention_weight_min": -37.028751373291016,
      "activations/layer16_attention_weight_max": 31.724014282226562,
      "activations/layer16_attention_weight_min": -28.961732864379883,
      "activations/layer17_attention_weight_max": 30.727975845336914,
      "activations/layer17_attention_weight_min": -26.121322631835938,
      "activations/layer18_attention_weight_max": 34.39133071899414,
      "activations/layer18_attention_weight_min": -25.856950759887695,
      "activations/layer19_attention_weight_max": 33.49003982543945,
      "activations/layer19_attention_weight_min": -32.745140075683594,
      "activations/layer1_attention_weight_max": 15.877546310424805,
      "activations/layer1_attention_weight_min": -14.161128997802734,
      "activations/layer20_attention_weight_max": 29.656919479370117,
      "activations/layer20_attention_weight_min": -25.590246200561523,
      "activations/layer21_attention_weight_max": 30.084325790405273,
      "activations/layer21_attention_weight_min": -25.67546272277832,
      "activations/layer22_attention_weight_max": 51.13909912109375,
      "activations/layer22_attention_weight_min": -34.521018981933594,
      "activations/layer23_attention_weight_max": 34.89552307128906,
      "activations/layer23_attention_weight_min": -29.776968002319336,
      "activations/layer2_attention_weight_max": 32.633235931396484,
      "activations/layer2_attention_weight_min": -31.08444595336914,
      "activations/layer3_attention_weight_max": 99.61656951904297,
      "activations/layer3_attention_weight_min": -100.56100463867188,
      "activations/layer4_attention_weight_max": 107.00030517578125,
      "activations/layer4_attention_weight_min": -104.59626770019531,
      "activations/layer5_attention_weight_max": 52.152008056640625,
      "activations/layer5_attention_weight_min": -60.28770065307617,
      "activations/layer6_attention_weight_max": 43.70316696166992,
      "activations/layer6_attention_weight_min": -46.79667663574219,
      "activations/layer7_attention_weight_max": 87.60128021240234,
      "activations/layer7_attention_weight_min": -97.47036743164062,
      "activations/layer8_attention_weight_max": 41.27540969848633,
      "activations/layer8_attention_weight_min": -43.95015335083008,
      "activations/layer9_attention_weight_max": 32.8831901550293,
      "activations/layer9_attention_weight_min": -36.66107940673828,
      "epoch": 18.11,
      "learning_rate": 3.3521590909090905e-05,
      "loss": 2.7236,
      "step": 311750
    },
    {
      "activations/layer0_attention_weight_max": 17.637283325195312,
      "activations/layer0_attention_weight_min": -13.317075729370117,
      "activations/layer10_attention_weight_max": 35.10186767578125,
      "activations/layer10_attention_weight_min": -33.72024154663086,
      "activations/layer11_attention_weight_max": 33.06529998779297,
      "activations/layer11_attention_weight_min": -32.678733825683594,
      "activations/layer12_attention_weight_max": 26.539270401000977,
      "activations/layer12_attention_weight_min": -28.998987197875977,
      "activations/layer13_attention_weight_max": 39.46516418457031,
      "activations/layer13_attention_weight_min": -34.00825500488281,
      "activations/layer14_attention_weight_max": 42.28377914428711,
      "activations/layer14_attention_weight_min": -37.210365295410156,
      "activations/layer15_attention_weight_max": 38.9460563659668,
      "activations/layer15_attention_weight_min": -34.952728271484375,
      "activations/layer16_attention_weight_max": 30.77689552307129,
      "activations/layer16_attention_weight_min": -29.47457504272461,
      "activations/layer17_attention_weight_max": 28.371931076049805,
      "activations/layer17_attention_weight_min": -25.754953384399414,
      "activations/layer18_attention_weight_max": 30.789569854736328,
      "activations/layer18_attention_weight_min": -27.4588565826416,
      "activations/layer19_attention_weight_max": 31.76435661315918,
      "activations/layer19_attention_weight_min": -31.737796783447266,
      "activations/layer1_attention_weight_max": 16.191608428955078,
      "activations/layer1_attention_weight_min": -14.803250312805176,
      "activations/layer20_attention_weight_max": 31.066587448120117,
      "activations/layer20_attention_weight_min": -26.4102725982666,
      "activations/layer21_attention_weight_max": 30.897323608398438,
      "activations/layer21_attention_weight_min": -28.613035202026367,
      "activations/layer22_attention_weight_max": 45.92224884033203,
      "activations/layer22_attention_weight_min": -32.7685661315918,
      "activations/layer23_attention_weight_max": 32.90552520751953,
      "activations/layer23_attention_weight_min": -25.997570037841797,
      "activations/layer2_attention_weight_max": 32.18156433105469,
      "activations/layer2_attention_weight_min": -31.261573791503906,
      "activations/layer3_attention_weight_max": 101.10649871826172,
      "activations/layer3_attention_weight_min": -99.73413848876953,
      "activations/layer4_attention_weight_max": 109.30322265625,
      "activations/layer4_attention_weight_min": -110.0957260131836,
      "activations/layer5_attention_weight_max": 56.32644271850586,
      "activations/layer5_attention_weight_min": -62.33308410644531,
      "activations/layer6_attention_weight_max": 44.52790069580078,
      "activations/layer6_attention_weight_min": -47.62274932861328,
      "activations/layer7_attention_weight_max": 92.4002914428711,
      "activations/layer7_attention_weight_min": -95.01161193847656,
      "activations/layer8_attention_weight_max": 40.966224670410156,
      "activations/layer8_attention_weight_min": -46.016395568847656,
      "activations/layer9_attention_weight_max": 31.252628326416016,
      "activations/layer9_attention_weight_min": -35.57093811035156,
      "epoch": 18.12,
      "learning_rate": 3.350265151515151e-05,
      "loss": 2.7234,
      "step": 311800
    },
    {
      "activations/layer0_attention_weight_max": 16.575950622558594,
      "activations/layer0_attention_weight_min": -13.293777465820312,
      "activations/layer10_attention_weight_max": 34.23173522949219,
      "activations/layer10_attention_weight_min": -32.48789596557617,
      "activations/layer11_attention_weight_max": 34.45366668701172,
      "activations/layer11_attention_weight_min": -32.00874328613281,
      "activations/layer12_attention_weight_max": 28.620540618896484,
      "activations/layer12_attention_weight_min": -25.65605354309082,
      "activations/layer13_attention_weight_max": 40.56507873535156,
      "activations/layer13_attention_weight_min": -36.09326171875,
      "activations/layer14_attention_weight_max": 42.00749969482422,
      "activations/layer14_attention_weight_min": -38.78822708129883,
      "activations/layer15_attention_weight_max": 38.593994140625,
      "activations/layer15_attention_weight_min": -35.77692413330078,
      "activations/layer16_attention_weight_max": 30.002513885498047,
      "activations/layer16_attention_weight_min": -29.946435928344727,
      "activations/layer17_attention_weight_max": 29.87640380859375,
      "activations/layer17_attention_weight_min": -26.56914710998535,
      "activations/layer18_attention_weight_max": 30.940692901611328,
      "activations/layer18_attention_weight_min": -27.180721282958984,
      "activations/layer19_attention_weight_max": 34.66925811767578,
      "activations/layer19_attention_weight_min": -35.17622375488281,
      "activations/layer1_attention_weight_max": 17.806884765625,
      "activations/layer1_attention_weight_min": -14.627070426940918,
      "activations/layer20_attention_weight_max": 27.540599822998047,
      "activations/layer20_attention_weight_min": -25.948692321777344,
      "activations/layer21_attention_weight_max": 28.270206451416016,
      "activations/layer21_attention_weight_min": -25.56424903869629,
      "activations/layer22_attention_weight_max": 38.34420394897461,
      "activations/layer22_attention_weight_min": -32.531070709228516,
      "activations/layer23_attention_weight_max": 33.30018615722656,
      "activations/layer23_attention_weight_min": -25.988357543945312,
      "activations/layer2_attention_weight_max": 32.41707229614258,
      "activations/layer2_attention_weight_min": -31.537229537963867,
      "activations/layer3_attention_weight_max": 99.69758605957031,
      "activations/layer3_attention_weight_min": -96.78341674804688,
      "activations/layer4_attention_weight_max": 104.20906066894531,
      "activations/layer4_attention_weight_min": -107.24742889404297,
      "activations/layer5_attention_weight_max": 50.004798889160156,
      "activations/layer5_attention_weight_min": -59.887062072753906,
      "activations/layer6_attention_weight_max": 41.780181884765625,
      "activations/layer6_attention_weight_min": -44.33537292480469,
      "activations/layer7_attention_weight_max": 89.44762420654297,
      "activations/layer7_attention_weight_min": -98.28345489501953,
      "activations/layer8_attention_weight_max": 45.52165222167969,
      "activations/layer8_attention_weight_min": -44.28414535522461,
      "activations/layer9_attention_weight_max": 33.59321594238281,
      "activations/layer9_attention_weight_min": -32.793617248535156,
      "epoch": 18.12,
      "learning_rate": 3.3483712121212114e-05,
      "loss": 2.743,
      "step": 311850
    },
    {
      "activations/layer0_attention_weight_max": 16.850202560424805,
      "activations/layer0_attention_weight_min": -13.0012788772583,
      "activations/layer10_attention_weight_max": 33.8001823425293,
      "activations/layer10_attention_weight_min": -33.6683235168457,
      "activations/layer11_attention_weight_max": 32.91180419921875,
      "activations/layer11_attention_weight_min": -33.02265167236328,
      "activations/layer12_attention_weight_max": 28.846281051635742,
      "activations/layer12_attention_weight_min": -25.935373306274414,
      "activations/layer13_attention_weight_max": 42.7779655456543,
      "activations/layer13_attention_weight_min": -35.28081512451172,
      "activations/layer14_attention_weight_max": 41.46342086791992,
      "activations/layer14_attention_weight_min": -38.54045867919922,
      "activations/layer15_attention_weight_max": 37.58553695678711,
      "activations/layer15_attention_weight_min": -34.36601638793945,
      "activations/layer16_attention_weight_max": 29.506122589111328,
      "activations/layer16_attention_weight_min": -27.447751998901367,
      "activations/layer17_attention_weight_max": 29.734127044677734,
      "activations/layer17_attention_weight_min": -25.333885192871094,
      "activations/layer18_attention_weight_max": 31.113643646240234,
      "activations/layer18_attention_weight_min": -24.70217514038086,
      "activations/layer19_attention_weight_max": 42.218509674072266,
      "activations/layer19_attention_weight_min": -33.73139572143555,
      "activations/layer1_attention_weight_max": 17.098766326904297,
      "activations/layer1_attention_weight_min": -14.166361808776855,
      "activations/layer20_attention_weight_max": 30.81581687927246,
      "activations/layer20_attention_weight_min": -27.137550354003906,
      "activations/layer21_attention_weight_max": 28.733715057373047,
      "activations/layer21_attention_weight_min": -26.820430755615234,
      "activations/layer22_attention_weight_max": 43.15326690673828,
      "activations/layer22_attention_weight_min": -30.973068237304688,
      "activations/layer23_attention_weight_max": 33.66209030151367,
      "activations/layer23_attention_weight_min": -25.80928611755371,
      "activations/layer2_attention_weight_max": 33.45186233520508,
      "activations/layer2_attention_weight_min": -32.050010681152344,
      "activations/layer3_attention_weight_max": 104.0221176147461,
      "activations/layer3_attention_weight_min": -102.37517547607422,
      "activations/layer4_attention_weight_max": 111.36616516113281,
      "activations/layer4_attention_weight_min": -110.41130828857422,
      "activations/layer5_attention_weight_max": 50.090118408203125,
      "activations/layer5_attention_weight_min": -62.17949676513672,
      "activations/layer6_attention_weight_max": 45.89750671386719,
      "activations/layer6_attention_weight_min": -47.500301361083984,
      "activations/layer7_attention_weight_max": 100.42412567138672,
      "activations/layer7_attention_weight_min": -95.54194641113281,
      "activations/layer8_attention_weight_max": 41.170372009277344,
      "activations/layer8_attention_weight_min": -42.60918045043945,
      "activations/layer9_attention_weight_max": 33.02402877807617,
      "activations/layer9_attention_weight_min": -34.37889862060547,
      "epoch": 18.12,
      "learning_rate": 3.346477272727272e-05,
      "loss": 2.7244,
      "step": 311900
    },
    {
      "activations/layer0_attention_weight_max": 16.460405349731445,
      "activations/layer0_attention_weight_min": -13.942498207092285,
      "activations/layer10_attention_weight_max": 31.15971565246582,
      "activations/layer10_attention_weight_min": -32.312896728515625,
      "activations/layer11_attention_weight_max": 31.667333602905273,
      "activations/layer11_attention_weight_min": -34.16090393066406,
      "activations/layer12_attention_weight_max": 26.566774368286133,
      "activations/layer12_attention_weight_min": -27.02240753173828,
      "activations/layer13_attention_weight_max": 39.662200927734375,
      "activations/layer13_attention_weight_min": -34.27216720581055,
      "activations/layer14_attention_weight_max": 41.40673065185547,
      "activations/layer14_attention_weight_min": -37.3076057434082,
      "activations/layer15_attention_weight_max": 39.788475036621094,
      "activations/layer15_attention_weight_min": -35.64712905883789,
      "activations/layer16_attention_weight_max": 29.581438064575195,
      "activations/layer16_attention_weight_min": -27.53801155090332,
      "activations/layer17_attention_weight_max": 28.50379753112793,
      "activations/layer17_attention_weight_min": -25.014781951904297,
      "activations/layer18_attention_weight_max": 33.71466064453125,
      "activations/layer18_attention_weight_min": -25.490346908569336,
      "activations/layer19_attention_weight_max": 32.54903030395508,
      "activations/layer19_attention_weight_min": -31.235809326171875,
      "activations/layer1_attention_weight_max": 16.391429901123047,
      "activations/layer1_attention_weight_min": -14.187979698181152,
      "activations/layer20_attention_weight_max": 28.408296585083008,
      "activations/layer20_attention_weight_min": -25.61155891418457,
      "activations/layer21_attention_weight_max": 28.715829849243164,
      "activations/layer21_attention_weight_min": -25.855632781982422,
      "activations/layer22_attention_weight_max": 40.27569580078125,
      "activations/layer22_attention_weight_min": -33.60403060913086,
      "activations/layer23_attention_weight_max": 33.2240104675293,
      "activations/layer23_attention_weight_min": -27.388822555541992,
      "activations/layer2_attention_weight_max": 33.071197509765625,
      "activations/layer2_attention_weight_min": -32.38084030151367,
      "activations/layer3_attention_weight_max": 97.83122253417969,
      "activations/layer3_attention_weight_min": -100.43606567382812,
      "activations/layer4_attention_weight_max": 105.25038146972656,
      "activations/layer4_attention_weight_min": -114.9476318359375,
      "activations/layer5_attention_weight_max": 51.47642517089844,
      "activations/layer5_attention_weight_min": -65.0986557006836,
      "activations/layer6_attention_weight_max": 41.48177719116211,
      "activations/layer6_attention_weight_min": -44.86172103881836,
      "activations/layer7_attention_weight_max": 86.95170593261719,
      "activations/layer7_attention_weight_min": -92.7496109008789,
      "activations/layer8_attention_weight_max": 38.4395637512207,
      "activations/layer8_attention_weight_min": -41.6702766418457,
      "activations/layer9_attention_weight_max": 30.507827758789062,
      "activations/layer9_attention_weight_min": -32.63520431518555,
      "epoch": 18.13,
      "learning_rate": 3.344583333333333e-05,
      "loss": 2.7252,
      "step": 311950
    },
    {
      "activations/layer0_attention_weight_max": 16.972761154174805,
      "activations/layer0_attention_weight_min": -12.830645561218262,
      "activations/layer10_attention_weight_max": 31.83658790588379,
      "activations/layer10_attention_weight_min": -31.830551147460938,
      "activations/layer11_attention_weight_max": 34.10499954223633,
      "activations/layer11_attention_weight_min": -31.128036499023438,
      "activations/layer12_attention_weight_max": 24.76995277404785,
      "activations/layer12_attention_weight_min": -26.634506225585938,
      "activations/layer13_attention_weight_max": 36.38009262084961,
      "activations/layer13_attention_weight_min": -35.655853271484375,
      "activations/layer14_attention_weight_max": 40.736968994140625,
      "activations/layer14_attention_weight_min": -36.93070602416992,
      "activations/layer15_attention_weight_max": 35.2867546081543,
      "activations/layer15_attention_weight_min": -33.905879974365234,
      "activations/layer16_attention_weight_max": 30.420927047729492,
      "activations/layer16_attention_weight_min": -29.07977294921875,
      "activations/layer17_attention_weight_max": 28.470760345458984,
      "activations/layer17_attention_weight_min": -26.509544372558594,
      "activations/layer18_attention_weight_max": 29.472238540649414,
      "activations/layer18_attention_weight_min": -26.697038650512695,
      "activations/layer19_attention_weight_max": 31.85780906677246,
      "activations/layer19_attention_weight_min": -35.3050651550293,
      "activations/layer1_attention_weight_max": 16.1156005859375,
      "activations/layer1_attention_weight_min": -14.524563789367676,
      "activations/layer20_attention_weight_max": 28.36556053161621,
      "activations/layer20_attention_weight_min": -27.37870979309082,
      "activations/layer21_attention_weight_max": 27.528915405273438,
      "activations/layer21_attention_weight_min": -26.931310653686523,
      "activations/layer22_attention_weight_max": 41.16543197631836,
      "activations/layer22_attention_weight_min": -35.723751068115234,
      "activations/layer23_attention_weight_max": 31.31080436706543,
      "activations/layer23_attention_weight_min": -26.468894958496094,
      "activations/layer2_attention_weight_max": 33.37921905517578,
      "activations/layer2_attention_weight_min": -31.794170379638672,
      "activations/layer3_attention_weight_max": 98.60684967041016,
      "activations/layer3_attention_weight_min": -102.36001586914062,
      "activations/layer4_attention_weight_max": 104.98858642578125,
      "activations/layer4_attention_weight_min": -107.2915267944336,
      "activations/layer5_attention_weight_max": 51.20841598510742,
      "activations/layer5_attention_weight_min": -60.202117919921875,
      "activations/layer6_attention_weight_max": 44.30610656738281,
      "activations/layer6_attention_weight_min": -44.87209701538086,
      "activations/layer7_attention_weight_max": 90.89176177978516,
      "activations/layer7_attention_weight_min": -90.35142517089844,
      "activations/layer8_attention_weight_max": 38.75473403930664,
      "activations/layer8_attention_weight_min": -42.63302230834961,
      "activations/layer9_attention_weight_max": 30.97273826599121,
      "activations/layer9_attention_weight_min": -32.8337287902832,
      "epoch": 18.13,
      "learning_rate": 3.342689393939394e-05,
      "loss": 2.725,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_loss": 2.6875,
      "eval_runtime": 8.4807,
      "eval_samples_per_second": 506.327,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.4807,
      "eval_openwebtext_samples_per_second": 506.327,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9944,
      "eval_wikitext_samples_per_second": 228.637,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.5668,
      "eval_lambada_samples_per_second": 508.947,
      "step": 312000
    },
    {
      "activations/layer0_attention_weight_max": 16.582483291625977,
      "activations/layer0_attention_weight_min": -13.174464225769043,
      "activations/layer10_attention_weight_max": 32.83643341064453,
      "activations/layer10_attention_weight_min": -33.393959045410156,
      "activations/layer11_attention_weight_max": 32.434791564941406,
      "activations/layer11_attention_weight_min": -33.21344757080078,
      "activations/layer12_attention_weight_max": 25.922035217285156,
      "activations/layer12_attention_weight_min": -27.841880798339844,
      "activations/layer13_attention_weight_max": 39.32966995239258,
      "activations/layer13_attention_weight_min": -35.68195724487305,
      "activations/layer14_attention_weight_max": 42.55445861816406,
      "activations/layer14_attention_weight_min": -38.188819885253906,
      "activations/layer15_attention_weight_max": 37.87052536010742,
      "activations/layer15_attention_weight_min": -34.561279296875,
      "activations/layer16_attention_weight_max": 29.57615089416504,
      "activations/layer16_attention_weight_min": -27.54323959350586,
      "activations/layer17_attention_weight_max": 29.48410987854004,
      "activations/layer17_attention_weight_min": -26.743623733520508,
      "activations/layer18_attention_weight_max": 32.74283218383789,
      "activations/layer18_attention_weight_min": -26.940322875976562,
      "activations/layer19_attention_weight_max": 32.2248649597168,
      "activations/layer19_attention_weight_min": -31.351455688476562,
      "activations/layer1_attention_weight_max": 16.02027130126953,
      "activations/layer1_attention_weight_min": -14.53935718536377,
      "activations/layer20_attention_weight_max": 28.89794921875,
      "activations/layer20_attention_weight_min": -25.558576583862305,
      "activations/layer21_attention_weight_max": 27.49941635131836,
      "activations/layer21_attention_weight_min": -24.487308502197266,
      "activations/layer22_attention_weight_max": 39.67695236206055,
      "activations/layer22_attention_weight_min": -31.306257247924805,
      "activations/layer23_attention_weight_max": 31.88996696472168,
      "activations/layer23_attention_weight_min": -24.673725128173828,
      "activations/layer2_attention_weight_max": 33.76250457763672,
      "activations/layer2_attention_weight_min": -31.168128967285156,
      "activations/layer3_attention_weight_max": 96.4598617553711,
      "activations/layer3_attention_weight_min": -98.06040954589844,
      "activations/layer4_attention_weight_max": 103.93541717529297,
      "activations/layer4_attention_weight_min": -105.04769134521484,
      "activations/layer5_attention_weight_max": 50.7292594909668,
      "activations/layer5_attention_weight_min": -60.00999450683594,
      "activations/layer6_attention_weight_max": 44.36531448364258,
      "activations/layer6_attention_weight_min": -46.74465560913086,
      "activations/layer7_attention_weight_max": 91.07635498046875,
      "activations/layer7_attention_weight_min": -94.53795623779297,
      "activations/layer8_attention_weight_max": 42.35078430175781,
      "activations/layer8_attention_weight_min": -43.04364776611328,
      "activations/layer9_attention_weight_max": 30.595388412475586,
      "activations/layer9_attention_weight_min": -32.929847717285156,
      "epoch": 18.13,
      "learning_rate": 3.340795454545454e-05,
      "loss": 2.7207,
      "step": 312050
    },
    {
      "activations/layer0_attention_weight_max": 16.465957641601562,
      "activations/layer0_attention_weight_min": -12.518892288208008,
      "activations/layer10_attention_weight_max": 43.3536262512207,
      "activations/layer10_attention_weight_min": -40.215911865234375,
      "activations/layer11_attention_weight_max": 41.80414581298828,
      "activations/layer11_attention_weight_min": -41.49736785888672,
      "activations/layer12_attention_weight_max": 28.903730392456055,
      "activations/layer12_attention_weight_min": -27.449064254760742,
      "activations/layer13_attention_weight_max": 43.07856369018555,
      "activations/layer13_attention_weight_min": -36.18669509887695,
      "activations/layer14_attention_weight_max": 43.18062210083008,
      "activations/layer14_attention_weight_min": -39.527896881103516,
      "activations/layer15_attention_weight_max": 40.60624694824219,
      "activations/layer15_attention_weight_min": -36.39300537109375,
      "activations/layer16_attention_weight_max": 28.743024826049805,
      "activations/layer16_attention_weight_min": -27.43165397644043,
      "activations/layer17_attention_weight_max": 30.78936767578125,
      "activations/layer17_attention_weight_min": -25.393287658691406,
      "activations/layer18_attention_weight_max": 32.959014892578125,
      "activations/layer18_attention_weight_min": -24.624757766723633,
      "activations/layer19_attention_weight_max": 32.708335876464844,
      "activations/layer19_attention_weight_min": -31.19785499572754,
      "activations/layer1_attention_weight_max": 17.541704177856445,
      "activations/layer1_attention_weight_min": -15.982027053833008,
      "activations/layer20_attention_weight_max": 28.975067138671875,
      "activations/layer20_attention_weight_min": -23.53362464904785,
      "activations/layer21_attention_weight_max": 29.751054763793945,
      "activations/layer21_attention_weight_min": -25.268218994140625,
      "activations/layer22_attention_weight_max": 42.85684585571289,
      "activations/layer22_attention_weight_min": -29.373676300048828,
      "activations/layer23_attention_weight_max": 31.888591766357422,
      "activations/layer23_attention_weight_min": -25.126508712768555,
      "activations/layer2_attention_weight_max": 35.38044357299805,
      "activations/layer2_attention_weight_min": -35.11180114746094,
      "activations/layer3_attention_weight_max": 112.45004272460938,
      "activations/layer3_attention_weight_min": -111.68455505371094,
      "activations/layer4_attention_weight_max": 121.91808319091797,
      "activations/layer4_attention_weight_min": -110.17768859863281,
      "activations/layer5_attention_weight_max": 61.57926559448242,
      "activations/layer5_attention_weight_min": -59.149166107177734,
      "activations/layer6_attention_weight_max": 48.25429153442383,
      "activations/layer6_attention_weight_min": -47.67036819458008,
      "activations/layer7_attention_weight_max": 116.39494323730469,
      "activations/layer7_attention_weight_min": -101.11029052734375,
      "activations/layer8_attention_weight_max": 49.87222671508789,
      "activations/layer8_attention_weight_min": -46.70667266845703,
      "activations/layer9_attention_weight_max": 35.79488754272461,
      "activations/layer9_attention_weight_min": -38.35036087036133,
      "epoch": 18.13,
      "learning_rate": 3.338901515151515e-05,
      "loss": 2.7185,
      "step": 312100
    },
    {
      "activations/layer0_attention_weight_max": 15.658493041992188,
      "activations/layer0_attention_weight_min": -12.929694175720215,
      "activations/layer10_attention_weight_max": 32.57329559326172,
      "activations/layer10_attention_weight_min": -33.037471771240234,
      "activations/layer11_attention_weight_max": 34.753204345703125,
      "activations/layer11_attention_weight_min": -33.889930725097656,
      "activations/layer12_attention_weight_max": 27.224056243896484,
      "activations/layer12_attention_weight_min": -28.73172378540039,
      "activations/layer13_attention_weight_max": 37.875179290771484,
      "activations/layer13_attention_weight_min": -35.93846130371094,
      "activations/layer14_attention_weight_max": 42.133201599121094,
      "activations/layer14_attention_weight_min": -37.67613220214844,
      "activations/layer15_attention_weight_max": 39.69280242919922,
      "activations/layer15_attention_weight_min": -38.33232116699219,
      "activations/layer16_attention_weight_max": 29.14484405517578,
      "activations/layer16_attention_weight_min": -28.391542434692383,
      "activations/layer17_attention_weight_max": 28.97283363342285,
      "activations/layer17_attention_weight_min": -27.394285202026367,
      "activations/layer18_attention_weight_max": 31.757028579711914,
      "activations/layer18_attention_weight_min": -27.533973693847656,
      "activations/layer19_attention_weight_max": 31.5496883392334,
      "activations/layer19_attention_weight_min": -30.16448974609375,
      "activations/layer1_attention_weight_max": 16.77126693725586,
      "activations/layer1_attention_weight_min": -14.64083194732666,
      "activations/layer20_attention_weight_max": 30.900732040405273,
      "activations/layer20_attention_weight_min": -25.38289451599121,
      "activations/layer21_attention_weight_max": 31.144540786743164,
      "activations/layer21_attention_weight_min": -24.595212936401367,
      "activations/layer22_attention_weight_max": 40.008094787597656,
      "activations/layer22_attention_weight_min": -33.69331359863281,
      "activations/layer23_attention_weight_max": 34.32585906982422,
      "activations/layer23_attention_weight_min": -27.516067504882812,
      "activations/layer2_attention_weight_max": 37.638145446777344,
      "activations/layer2_attention_weight_min": -34.84410858154297,
      "activations/layer3_attention_weight_max": 103.35282135009766,
      "activations/layer3_attention_weight_min": -100.28614807128906,
      "activations/layer4_attention_weight_max": 107.1676025390625,
      "activations/layer4_attention_weight_min": -103.35457611083984,
      "activations/layer5_attention_weight_max": 51.492122650146484,
      "activations/layer5_attention_weight_min": -60.205867767333984,
      "activations/layer6_attention_weight_max": 43.881019592285156,
      "activations/layer6_attention_weight_min": -44.46137619018555,
      "activations/layer7_attention_weight_max": 97.37242126464844,
      "activations/layer7_attention_weight_min": -92.24075317382812,
      "activations/layer8_attention_weight_max": 41.13478088378906,
      "activations/layer8_attention_weight_min": -42.271881103515625,
      "activations/layer9_attention_weight_max": 30.916501998901367,
      "activations/layer9_attention_weight_min": -33.86910629272461,
      "epoch": 18.14,
      "learning_rate": 3.337007575757576e-05,
      "loss": 2.7458,
      "step": 312150
    },
    {
      "activations/layer0_attention_weight_max": 17.05413818359375,
      "activations/layer0_attention_weight_min": -12.875967979431152,
      "activations/layer10_attention_weight_max": 37.465728759765625,
      "activations/layer10_attention_weight_min": -36.58908462524414,
      "activations/layer11_attention_weight_max": 39.63762664794922,
      "activations/layer11_attention_weight_min": -36.130149841308594,
      "activations/layer12_attention_weight_max": 28.997486114501953,
      "activations/layer12_attention_weight_min": -27.063623428344727,
      "activations/layer13_attention_weight_max": 40.501155853271484,
      "activations/layer13_attention_weight_min": -36.673946380615234,
      "activations/layer14_attention_weight_max": 42.77616882324219,
      "activations/layer14_attention_weight_min": -40.011444091796875,
      "activations/layer15_attention_weight_max": 39.658443450927734,
      "activations/layer15_attention_weight_min": -36.98420715332031,
      "activations/layer16_attention_weight_max": 36.091583251953125,
      "activations/layer16_attention_weight_min": -27.9420166015625,
      "activations/layer17_attention_weight_max": 31.427377700805664,
      "activations/layer17_attention_weight_min": -26.699609756469727,
      "activations/layer18_attention_weight_max": 32.79698181152344,
      "activations/layer18_attention_weight_min": -29.126558303833008,
      "activations/layer19_attention_weight_max": 36.73040771484375,
      "activations/layer19_attention_weight_min": -31.543119430541992,
      "activations/layer1_attention_weight_max": 17.034543991088867,
      "activations/layer1_attention_weight_min": -14.80675220489502,
      "activations/layer20_attention_weight_max": 30.246904373168945,
      "activations/layer20_attention_weight_min": -26.713829040527344,
      "activations/layer21_attention_weight_max": 28.70303726196289,
      "activations/layer21_attention_weight_min": -27.113725662231445,
      "activations/layer22_attention_weight_max": 42.078514099121094,
      "activations/layer22_attention_weight_min": -33.74324035644531,
      "activations/layer23_attention_weight_max": 32.875892639160156,
      "activations/layer23_attention_weight_min": -25.994335174560547,
      "activations/layer2_attention_weight_max": 34.8635139465332,
      "activations/layer2_attention_weight_min": -32.62452697753906,
      "activations/layer3_attention_weight_max": 105.5791244506836,
      "activations/layer3_attention_weight_min": -99.46246337890625,
      "activations/layer4_attention_weight_max": 108.91029357910156,
      "activations/layer4_attention_weight_min": -108.9445571899414,
      "activations/layer5_attention_weight_max": 52.7049674987793,
      "activations/layer5_attention_weight_min": -59.087501525878906,
      "activations/layer6_attention_weight_max": 48.33120346069336,
      "activations/layer6_attention_weight_min": -48.9103889465332,
      "activations/layer7_attention_weight_max": 95.5345230102539,
      "activations/layer7_attention_weight_min": -103.83076477050781,
      "activations/layer8_attention_weight_max": 45.740901947021484,
      "activations/layer8_attention_weight_min": -46.5358772277832,
      "activations/layer9_attention_weight_max": 34.282161712646484,
      "activations/layer9_attention_weight_min": -36.61177062988281,
      "epoch": 18.14,
      "learning_rate": 3.335113636363636e-05,
      "loss": 2.7405,
      "step": 312200
    },
    {
      "activations/layer0_attention_weight_max": 15.878317832946777,
      "activations/layer0_attention_weight_min": -13.506044387817383,
      "activations/layer10_attention_weight_max": 40.78899002075195,
      "activations/layer10_attention_weight_min": -35.469478607177734,
      "activations/layer11_attention_weight_max": 39.475223541259766,
      "activations/layer11_attention_weight_min": -37.64539337158203,
      "activations/layer12_attention_weight_max": 28.438169479370117,
      "activations/layer12_attention_weight_min": -25.968276977539062,
      "activations/layer13_attention_weight_max": 40.62052536010742,
      "activations/layer13_attention_weight_min": -34.33460998535156,
      "activations/layer14_attention_weight_max": 40.019493103027344,
      "activations/layer14_attention_weight_min": -35.007686614990234,
      "activations/layer15_attention_weight_max": 37.16646194458008,
      "activations/layer15_attention_weight_min": -34.36440658569336,
      "activations/layer16_attention_weight_max": 30.148283004760742,
      "activations/layer16_attention_weight_min": -28.25698471069336,
      "activations/layer17_attention_weight_max": 28.303510665893555,
      "activations/layer17_attention_weight_min": -26.56546974182129,
      "activations/layer18_attention_weight_max": 31.017494201660156,
      "activations/layer18_attention_weight_min": -26.75806999206543,
      "activations/layer19_attention_weight_max": 34.480995178222656,
      "activations/layer19_attention_weight_min": -33.77204132080078,
      "activations/layer1_attention_weight_max": 16.87834358215332,
      "activations/layer1_attention_weight_min": -16.211502075195312,
      "activations/layer20_attention_weight_max": 29.045181274414062,
      "activations/layer20_attention_weight_min": -27.3764591217041,
      "activations/layer21_attention_weight_max": 27.562541961669922,
      "activations/layer21_attention_weight_min": -27.724945068359375,
      "activations/layer22_attention_weight_max": 40.46379470825195,
      "activations/layer22_attention_weight_min": -32.99919891357422,
      "activations/layer23_attention_weight_max": 32.61509704589844,
      "activations/layer23_attention_weight_min": -25.784835815429688,
      "activations/layer2_attention_weight_max": 33.653465270996094,
      "activations/layer2_attention_weight_min": -32.748085021972656,
      "activations/layer3_attention_weight_max": 104.2007827758789,
      "activations/layer3_attention_weight_min": -103.006103515625,
      "activations/layer4_attention_weight_max": 110.3714599609375,
      "activations/layer4_attention_weight_min": -105.9669418334961,
      "activations/layer5_attention_weight_max": 50.14745330810547,
      "activations/layer5_attention_weight_min": -63.69285583496094,
      "activations/layer6_attention_weight_max": 45.546607971191406,
      "activations/layer6_attention_weight_min": -45.282691955566406,
      "activations/layer7_attention_weight_max": 91.11903381347656,
      "activations/layer7_attention_weight_min": -94.87425994873047,
      "activations/layer8_attention_weight_max": 43.45695877075195,
      "activations/layer8_attention_weight_min": -42.467308044433594,
      "activations/layer9_attention_weight_max": 36.39934158325195,
      "activations/layer9_attention_weight_min": -35.17058563232422,
      "epoch": 18.14,
      "learning_rate": 3.333219696969697e-05,
      "loss": 2.7444,
      "step": 312250
    },
    {
      "activations/layer0_attention_weight_max": 15.991214752197266,
      "activations/layer0_attention_weight_min": -12.501423835754395,
      "activations/layer10_attention_weight_max": 35.40657043457031,
      "activations/layer10_attention_weight_min": -37.42595672607422,
      "activations/layer11_attention_weight_max": 35.756649017333984,
      "activations/layer11_attention_weight_min": -37.012142181396484,
      "activations/layer12_attention_weight_max": 27.441041946411133,
      "activations/layer12_attention_weight_min": -28.566707611083984,
      "activations/layer13_attention_weight_max": 42.37590408325195,
      "activations/layer13_attention_weight_min": -37.03233337402344,
      "activations/layer14_attention_weight_max": 42.47044372558594,
      "activations/layer14_attention_weight_min": -40.992557525634766,
      "activations/layer15_attention_weight_max": 43.27859115600586,
      "activations/layer15_attention_weight_min": -35.967864990234375,
      "activations/layer16_attention_weight_max": 31.23777198791504,
      "activations/layer16_attention_weight_min": -29.22719955444336,
      "activations/layer17_attention_weight_max": 30.119775772094727,
      "activations/layer17_attention_weight_min": -26.696556091308594,
      "activations/layer18_attention_weight_max": 32.327781677246094,
      "activations/layer18_attention_weight_min": -25.38247299194336,
      "activations/layer19_attention_weight_max": 34.446441650390625,
      "activations/layer19_attention_weight_min": -32.0722541809082,
      "activations/layer1_attention_weight_max": 16.73560905456543,
      "activations/layer1_attention_weight_min": -14.943010330200195,
      "activations/layer20_attention_weight_max": 29.329776763916016,
      "activations/layer20_attention_weight_min": -25.01665687561035,
      "activations/layer21_attention_weight_max": 29.10716438293457,
      "activations/layer21_attention_weight_min": -25.329801559448242,
      "activations/layer22_attention_weight_max": 44.89080047607422,
      "activations/layer22_attention_weight_min": -30.305641174316406,
      "activations/layer23_attention_weight_max": 32.586280822753906,
      "activations/layer23_attention_weight_min": -25.7568416595459,
      "activations/layer2_attention_weight_max": 31.43483543395996,
      "activations/layer2_attention_weight_min": -32.60612487792969,
      "activations/layer3_attention_weight_max": 100.37997436523438,
      "activations/layer3_attention_weight_min": -103.82978057861328,
      "activations/layer4_attention_weight_max": 108.64656066894531,
      "activations/layer4_attention_weight_min": -112.6829833984375,
      "activations/layer5_attention_weight_max": 52.604637145996094,
      "activations/layer5_attention_weight_min": -62.93462371826172,
      "activations/layer6_attention_weight_max": 44.24235534667969,
      "activations/layer6_attention_weight_min": -47.40739059448242,
      "activations/layer7_attention_weight_max": 97.04190826416016,
      "activations/layer7_attention_weight_min": -100.64655303955078,
      "activations/layer8_attention_weight_max": 41.97380065917969,
      "activations/layer8_attention_weight_min": -44.69362258911133,
      "activations/layer9_attention_weight_max": 32.9150276184082,
      "activations/layer9_attention_weight_min": -37.938209533691406,
      "epoch": 18.15,
      "learning_rate": 3.3313257575757575e-05,
      "loss": 2.7256,
      "step": 312300
    },
    {
      "activations/layer0_attention_weight_max": 16.889495849609375,
      "activations/layer0_attention_weight_min": -13.973356246948242,
      "activations/layer10_attention_weight_max": 35.533443450927734,
      "activations/layer10_attention_weight_min": -33.444549560546875,
      "activations/layer11_attention_weight_max": 32.51168441772461,
      "activations/layer11_attention_weight_min": -32.508697509765625,
      "activations/layer12_attention_weight_max": 32.1449089050293,
      "activations/layer12_attention_weight_min": -28.947818756103516,
      "activations/layer13_attention_weight_max": 45.954429626464844,
      "activations/layer13_attention_weight_min": -37.692806243896484,
      "activations/layer14_attention_weight_max": 46.099342346191406,
      "activations/layer14_attention_weight_min": -38.75777053833008,
      "activations/layer15_attention_weight_max": 39.91094207763672,
      "activations/layer15_attention_weight_min": -35.79911422729492,
      "activations/layer16_attention_weight_max": 32.42604064941406,
      "activations/layer16_attention_weight_min": -28.924057006835938,
      "activations/layer17_attention_weight_max": 32.62083053588867,
      "activations/layer17_attention_weight_min": -27.23786735534668,
      "activations/layer18_attention_weight_max": 36.02342224121094,
      "activations/layer18_attention_weight_min": -25.56998634338379,
      "activations/layer19_attention_weight_max": 38.3559455871582,
      "activations/layer19_attention_weight_min": -32.85739517211914,
      "activations/layer1_attention_weight_max": 15.91905403137207,
      "activations/layer1_attention_weight_min": -16.075729370117188,
      "activations/layer20_attention_weight_max": 36.241634368896484,
      "activations/layer20_attention_weight_min": -26.951709747314453,
      "activations/layer21_attention_weight_max": 36.4171028137207,
      "activations/layer21_attention_weight_min": -27.230979919433594,
      "activations/layer22_attention_weight_max": 48.54981231689453,
      "activations/layer22_attention_weight_min": -34.61842727661133,
      "activations/layer23_attention_weight_max": 36.29070281982422,
      "activations/layer23_attention_weight_min": -27.232519149780273,
      "activations/layer2_attention_weight_max": 33.975555419921875,
      "activations/layer2_attention_weight_min": -33.20073318481445,
      "activations/layer3_attention_weight_max": 101.52079010009766,
      "activations/layer3_attention_weight_min": -104.58100891113281,
      "activations/layer4_attention_weight_max": 107.4214096069336,
      "activations/layer4_attention_weight_min": -109.7996597290039,
      "activations/layer5_attention_weight_max": 51.14435577392578,
      "activations/layer5_attention_weight_min": -62.342079162597656,
      "activations/layer6_attention_weight_max": 44.63704299926758,
      "activations/layer6_attention_weight_min": -46.60857391357422,
      "activations/layer7_attention_weight_max": 92.10433959960938,
      "activations/layer7_attention_weight_min": -91.5549087524414,
      "activations/layer8_attention_weight_max": 41.27507400512695,
      "activations/layer8_attention_weight_min": -42.75416564941406,
      "activations/layer9_attention_weight_max": 32.4788932800293,
      "activations/layer9_attention_weight_min": -33.09741973876953,
      "epoch": 18.15,
      "learning_rate": 3.3294318181818183e-05,
      "loss": 2.7119,
      "step": 312350
    },
    {
      "activations/layer0_attention_weight_max": 17.58355140686035,
      "activations/layer0_attention_weight_min": -13.585372924804688,
      "activations/layer10_attention_weight_max": 35.14228057861328,
      "activations/layer10_attention_weight_min": -34.20459747314453,
      "activations/layer11_attention_weight_max": 35.4570426940918,
      "activations/layer11_attention_weight_min": -33.91849136352539,
      "activations/layer12_attention_weight_max": 26.457294464111328,
      "activations/layer12_attention_weight_min": -26.75765609741211,
      "activations/layer13_attention_weight_max": 40.3283576965332,
      "activations/layer13_attention_weight_min": -35.70359802246094,
      "activations/layer14_attention_weight_max": 46.116111755371094,
      "activations/layer14_attention_weight_min": -38.45285415649414,
      "activations/layer15_attention_weight_max": 38.94215393066406,
      "activations/layer15_attention_weight_min": -36.3045768737793,
      "activations/layer16_attention_weight_max": 31.00960350036621,
      "activations/layer16_attention_weight_min": -29.336273193359375,
      "activations/layer17_attention_weight_max": 32.368350982666016,
      "activations/layer17_attention_weight_min": -27.34756088256836,
      "activations/layer18_attention_weight_max": 32.638851165771484,
      "activations/layer18_attention_weight_min": -27.797578811645508,
      "activations/layer19_attention_weight_max": 34.699501037597656,
      "activations/layer19_attention_weight_min": -32.254608154296875,
      "activations/layer1_attention_weight_max": 17.1012020111084,
      "activations/layer1_attention_weight_min": -16.359228134155273,
      "activations/layer20_attention_weight_max": 31.611337661743164,
      "activations/layer20_attention_weight_min": -25.893163681030273,
      "activations/layer21_attention_weight_max": 30.658157348632812,
      "activations/layer21_attention_weight_min": -25.28356170654297,
      "activations/layer22_attention_weight_max": 44.6513557434082,
      "activations/layer22_attention_weight_min": -33.79315948486328,
      "activations/layer23_attention_weight_max": 33.59899139404297,
      "activations/layer23_attention_weight_min": -29.119976043701172,
      "activations/layer2_attention_weight_max": 33.395782470703125,
      "activations/layer2_attention_weight_min": -34.707786560058594,
      "activations/layer3_attention_weight_max": 101.5606460571289,
      "activations/layer3_attention_weight_min": -102.82499694824219,
      "activations/layer4_attention_weight_max": 108.46977996826172,
      "activations/layer4_attention_weight_min": -114.78645324707031,
      "activations/layer5_attention_weight_max": 52.00477600097656,
      "activations/layer5_attention_weight_min": -70.06062316894531,
      "activations/layer6_attention_weight_max": 44.78976821899414,
      "activations/layer6_attention_weight_min": -48.087215423583984,
      "activations/layer7_attention_weight_max": 101.39126586914062,
      "activations/layer7_attention_weight_min": -99.77777862548828,
      "activations/layer8_attention_weight_max": 43.24851989746094,
      "activations/layer8_attention_weight_min": -45.820579528808594,
      "activations/layer9_attention_weight_max": 32.574771881103516,
      "activations/layer9_attention_weight_min": -34.86438751220703,
      "epoch": 18.15,
      "learning_rate": 3.3275378787878785e-05,
      "loss": 2.7204,
      "step": 312400
    },
    {
      "activations/layer0_attention_weight_max": 15.550244331359863,
      "activations/layer0_attention_weight_min": -12.848676681518555,
      "activations/layer10_attention_weight_max": 35.298988342285156,
      "activations/layer10_attention_weight_min": -36.46925735473633,
      "activations/layer11_attention_weight_max": 37.90940856933594,
      "activations/layer11_attention_weight_min": -38.43968963623047,
      "activations/layer12_attention_weight_max": 29.282651901245117,
      "activations/layer12_attention_weight_min": -29.589988708496094,
      "activations/layer13_attention_weight_max": 42.19701385498047,
      "activations/layer13_attention_weight_min": -39.7220344543457,
      "activations/layer14_attention_weight_max": 48.18229293823242,
      "activations/layer14_attention_weight_min": -44.13352584838867,
      "activations/layer15_attention_weight_max": 42.6173095703125,
      "activations/layer15_attention_weight_min": -39.77715301513672,
      "activations/layer16_attention_weight_max": 30.606855392456055,
      "activations/layer16_attention_weight_min": -29.325393676757812,
      "activations/layer17_attention_weight_max": 31.778329849243164,
      "activations/layer17_attention_weight_min": -25.83124351501465,
      "activations/layer18_attention_weight_max": 30.530672073364258,
      "activations/layer18_attention_weight_min": -26.644250869750977,
      "activations/layer19_attention_weight_max": 32.311012268066406,
      "activations/layer19_attention_weight_min": -29.894357681274414,
      "activations/layer1_attention_weight_max": 16.233196258544922,
      "activations/layer1_attention_weight_min": -14.880062103271484,
      "activations/layer20_attention_weight_max": 27.745161056518555,
      "activations/layer20_attention_weight_min": -25.6188907623291,
      "activations/layer21_attention_weight_max": 28.836471557617188,
      "activations/layer21_attention_weight_min": -24.653661727905273,
      "activations/layer22_attention_weight_max": 40.68118667602539,
      "activations/layer22_attention_weight_min": -30.848285675048828,
      "activations/layer23_attention_weight_max": 32.665443420410156,
      "activations/layer23_attention_weight_min": -25.46890640258789,
      "activations/layer2_attention_weight_max": 34.01437759399414,
      "activations/layer2_attention_weight_min": -32.4996223449707,
      "activations/layer3_attention_weight_max": 102.5185317993164,
      "activations/layer3_attention_weight_min": -101.94026184082031,
      "activations/layer4_attention_weight_max": 111.3947525024414,
      "activations/layer4_attention_weight_min": -113.98895263671875,
      "activations/layer5_attention_weight_max": 51.783409118652344,
      "activations/layer5_attention_weight_min": -64.78321075439453,
      "activations/layer6_attention_weight_max": 46.0102424621582,
      "activations/layer6_attention_weight_min": -48.31730270385742,
      "activations/layer7_attention_weight_max": 92.54188537597656,
      "activations/layer7_attention_weight_min": -97.86430358886719,
      "activations/layer8_attention_weight_max": 43.835636138916016,
      "activations/layer8_attention_weight_min": -44.55101776123047,
      "activations/layer9_attention_weight_max": 33.63689422607422,
      "activations/layer9_attention_weight_min": -36.42245864868164,
      "epoch": 18.16,
      "learning_rate": 3.325643939393939e-05,
      "loss": 2.7288,
      "step": 312450
    },
    {
      "activations/layer0_attention_weight_max": 16.796056747436523,
      "activations/layer0_attention_weight_min": -13.575827598571777,
      "activations/layer10_attention_weight_max": 34.41393280029297,
      "activations/layer10_attention_weight_min": -33.42481994628906,
      "activations/layer11_attention_weight_max": 32.7386360168457,
      "activations/layer11_attention_weight_min": -32.31201934814453,
      "activations/layer12_attention_weight_max": 25.07069206237793,
      "activations/layer12_attention_weight_min": -24.600431442260742,
      "activations/layer13_attention_weight_max": 37.12630081176758,
      "activations/layer13_attention_weight_min": -33.888916015625,
      "activations/layer14_attention_weight_max": 42.06248092651367,
      "activations/layer14_attention_weight_min": -33.89354705810547,
      "activations/layer15_attention_weight_max": 38.3484992980957,
      "activations/layer15_attention_weight_min": -33.99024200439453,
      "activations/layer16_attention_weight_max": 28.670488357543945,
      "activations/layer16_attention_weight_min": -27.79300880432129,
      "activations/layer17_attention_weight_max": 28.06724739074707,
      "activations/layer17_attention_weight_min": -26.740299224853516,
      "activations/layer18_attention_weight_max": 30.427133560180664,
      "activations/layer18_attention_weight_min": -24.792346954345703,
      "activations/layer19_attention_weight_max": 32.184364318847656,
      "activations/layer19_attention_weight_min": -32.57253646850586,
      "activations/layer1_attention_weight_max": 15.716207504272461,
      "activations/layer1_attention_weight_min": -14.918594360351562,
      "activations/layer20_attention_weight_max": 27.088945388793945,
      "activations/layer20_attention_weight_min": -24.876384735107422,
      "activations/layer21_attention_weight_max": 29.028074264526367,
      "activations/layer21_attention_weight_min": -24.596399307250977,
      "activations/layer22_attention_weight_max": 38.37194061279297,
      "activations/layer22_attention_weight_min": -31.67568588256836,
      "activations/layer23_attention_weight_max": 29.34100914001465,
      "activations/layer23_attention_weight_min": -27.70197296142578,
      "activations/layer2_attention_weight_max": 33.41941833496094,
      "activations/layer2_attention_weight_min": -31.693763732910156,
      "activations/layer3_attention_weight_max": 100.42013549804688,
      "activations/layer3_attention_weight_min": -99.4205551147461,
      "activations/layer4_attention_weight_max": 107.6784896850586,
      "activations/layer4_attention_weight_min": -106.3556900024414,
      "activations/layer5_attention_weight_max": 52.4288444519043,
      "activations/layer5_attention_weight_min": -62.97689437866211,
      "activations/layer6_attention_weight_max": 43.16538619995117,
      "activations/layer6_attention_weight_min": -44.30373764038086,
      "activations/layer7_attention_weight_max": 89.56134033203125,
      "activations/layer7_attention_weight_min": -87.29240417480469,
      "activations/layer8_attention_weight_max": 40.060855865478516,
      "activations/layer8_attention_weight_min": -40.77632522583008,
      "activations/layer9_attention_weight_max": 32.56180953979492,
      "activations/layer9_attention_weight_min": -33.77553176879883,
      "epoch": 18.16,
      "learning_rate": 3.3237499999999995e-05,
      "loss": 2.7246,
      "step": 312500
    },
    {
      "activations/layer0_attention_weight_max": 15.773298263549805,
      "activations/layer0_attention_weight_min": -13.89417552947998,
      "activations/layer10_attention_weight_max": 31.182809829711914,
      "activations/layer10_attention_weight_min": -34.34040451049805,
      "activations/layer11_attention_weight_max": 31.413898468017578,
      "activations/layer11_attention_weight_min": -34.18913269042969,
      "activations/layer12_attention_weight_max": 26.41033172607422,
      "activations/layer12_attention_weight_min": -25.654523849487305,
      "activations/layer13_attention_weight_max": 41.33811569213867,
      "activations/layer13_attention_weight_min": -33.513832092285156,
      "activations/layer14_attention_weight_max": 43.24106979370117,
      "activations/layer14_attention_weight_min": -36.24692916870117,
      "activations/layer15_attention_weight_max": 38.85456085205078,
      "activations/layer15_attention_weight_min": -37.252628326416016,
      "activations/layer16_attention_weight_max": 28.015695571899414,
      "activations/layer16_attention_weight_min": -26.370582580566406,
      "activations/layer17_attention_weight_max": 28.09197425842285,
      "activations/layer17_attention_weight_min": -26.88482093811035,
      "activations/layer18_attention_weight_max": 32.73601531982422,
      "activations/layer18_attention_weight_min": -27.33619499206543,
      "activations/layer19_attention_weight_max": 32.0265007019043,
      "activations/layer19_attention_weight_min": -30.862003326416016,
      "activations/layer1_attention_weight_max": 15.71168327331543,
      "activations/layer1_attention_weight_min": -15.240981101989746,
      "activations/layer20_attention_weight_max": 28.60728645324707,
      "activations/layer20_attention_weight_min": -26.197343826293945,
      "activations/layer21_attention_weight_max": 27.0745849609375,
      "activations/layer21_attention_weight_min": -26.271800994873047,
      "activations/layer22_attention_weight_max": 48.11094284057617,
      "activations/layer22_attention_weight_min": -31.67776870727539,
      "activations/layer23_attention_weight_max": 31.813077926635742,
      "activations/layer23_attention_weight_min": -26.140090942382812,
      "activations/layer2_attention_weight_max": 32.47099304199219,
      "activations/layer2_attention_weight_min": -33.105369567871094,
      "activations/layer3_attention_weight_max": 101.44705200195312,
      "activations/layer3_attention_weight_min": -102.12220001220703,
      "activations/layer4_attention_weight_max": 105.6507339477539,
      "activations/layer4_attention_weight_min": -106.12200164794922,
      "activations/layer5_attention_weight_max": 51.56041717529297,
      "activations/layer5_attention_weight_min": -65.38324737548828,
      "activations/layer6_attention_weight_max": 42.889320373535156,
      "activations/layer6_attention_weight_min": -44.677371978759766,
      "activations/layer7_attention_weight_max": 91.7695541381836,
      "activations/layer7_attention_weight_min": -94.96971130371094,
      "activations/layer8_attention_weight_max": 39.957794189453125,
      "activations/layer8_attention_weight_min": -41.921409606933594,
      "activations/layer9_attention_weight_max": 29.3184757232666,
      "activations/layer9_attention_weight_min": -33.87258529663086,
      "epoch": 18.16,
      "learning_rate": 3.32185606060606e-05,
      "loss": 2.7262,
      "step": 312550
    },
    {
      "activations/layer0_attention_weight_max": 16.600414276123047,
      "activations/layer0_attention_weight_min": -13.673843383789062,
      "activations/layer10_attention_weight_max": 35.08258056640625,
      "activations/layer10_attention_weight_min": -34.26764678955078,
      "activations/layer11_attention_weight_max": 36.53514862060547,
      "activations/layer11_attention_weight_min": -34.44947814941406,
      "activations/layer12_attention_weight_max": 26.82160758972168,
      "activations/layer12_attention_weight_min": -26.5587100982666,
      "activations/layer13_attention_weight_max": 37.94118881225586,
      "activations/layer13_attention_weight_min": -35.90414047241211,
      "activations/layer14_attention_weight_max": 41.37077331542969,
      "activations/layer14_attention_weight_min": -37.85933303833008,
      "activations/layer15_attention_weight_max": 38.57564163208008,
      "activations/layer15_attention_weight_min": -36.14109420776367,
      "activations/layer16_attention_weight_max": 27.63538360595703,
      "activations/layer16_attention_weight_min": -29.365215301513672,
      "activations/layer17_attention_weight_max": 28.7414493560791,
      "activations/layer17_attention_weight_min": -26.867433547973633,
      "activations/layer18_attention_weight_max": 29.067964553833008,
      "activations/layer18_attention_weight_min": -25.23666763305664,
      "activations/layer19_attention_weight_max": 30.425846099853516,
      "activations/layer19_attention_weight_min": -32.19533920288086,
      "activations/layer1_attention_weight_max": 15.313085556030273,
      "activations/layer1_attention_weight_min": -13.958475112915039,
      "activations/layer20_attention_weight_max": 27.911043167114258,
      "activations/layer20_attention_weight_min": -26.521060943603516,
      "activations/layer21_attention_weight_max": 29.267375946044922,
      "activations/layer21_attention_weight_min": -26.383895874023438,
      "activations/layer22_attention_weight_max": 39.353816986083984,
      "activations/layer22_attention_weight_min": -31.47457504272461,
      "activations/layer23_attention_weight_max": 29.638240814208984,
      "activations/layer23_attention_weight_min": -26.135000228881836,
      "activations/layer2_attention_weight_max": 30.99275016784668,
      "activations/layer2_attention_weight_min": -31.891477584838867,
      "activations/layer3_attention_weight_max": 96.54570007324219,
      "activations/layer3_attention_weight_min": -98.14190673828125,
      "activations/layer4_attention_weight_max": 105.86917877197266,
      "activations/layer4_attention_weight_min": -106.1186752319336,
      "activations/layer5_attention_weight_max": 48.77864074707031,
      "activations/layer5_attention_weight_min": -61.11920928955078,
      "activations/layer6_attention_weight_max": 43.53886795043945,
      "activations/layer6_attention_weight_min": -46.42387771606445,
      "activations/layer7_attention_weight_max": 91.5309829711914,
      "activations/layer7_attention_weight_min": -97.59645080566406,
      "activations/layer8_attention_weight_max": 41.7640266418457,
      "activations/layer8_attention_weight_min": -42.680660247802734,
      "activations/layer9_attention_weight_max": 32.766361236572266,
      "activations/layer9_attention_weight_min": -34.04188919067383,
      "epoch": 18.16,
      "learning_rate": 3.319962121212121e-05,
      "loss": 2.7451,
      "step": 312600
    },
    {
      "activations/layer0_attention_weight_max": 16.357398986816406,
      "activations/layer0_attention_weight_min": -13.77319049835205,
      "activations/layer10_attention_weight_max": 31.46769142150879,
      "activations/layer10_attention_weight_min": -32.83475875854492,
      "activations/layer11_attention_weight_max": 32.208656311035156,
      "activations/layer11_attention_weight_min": -33.68193435668945,
      "activations/layer12_attention_weight_max": 24.73774528503418,
      "activations/layer12_attention_weight_min": -26.345014572143555,
      "activations/layer13_attention_weight_max": 37.22827911376953,
      "activations/layer13_attention_weight_min": -34.79269027709961,
      "activations/layer14_attention_weight_max": 40.14192581176758,
      "activations/layer14_attention_weight_min": -37.993892669677734,
      "activations/layer15_attention_weight_max": 34.89211654663086,
      "activations/layer15_attention_weight_min": -35.14829635620117,
      "activations/layer16_attention_weight_max": 28.343769073486328,
      "activations/layer16_attention_weight_min": -28.728979110717773,
      "activations/layer17_attention_weight_max": 30.03011703491211,
      "activations/layer17_attention_weight_min": -26.071088790893555,
      "activations/layer18_attention_weight_max": 29.814556121826172,
      "activations/layer18_attention_weight_min": -26.144596099853516,
      "activations/layer19_attention_weight_max": 31.768573760986328,
      "activations/layer19_attention_weight_min": -32.8967399597168,
      "activations/layer1_attention_weight_max": 15.790505409240723,
      "activations/layer1_attention_weight_min": -14.646944046020508,
      "activations/layer20_attention_weight_max": 27.147504806518555,
      "activations/layer20_attention_weight_min": -25.339555740356445,
      "activations/layer21_attention_weight_max": 28.140947341918945,
      "activations/layer21_attention_weight_min": -26.730466842651367,
      "activations/layer22_attention_weight_max": 41.196678161621094,
      "activations/layer22_attention_weight_min": -30.8597469329834,
      "activations/layer23_attention_weight_max": 33.631256103515625,
      "activations/layer23_attention_weight_min": -25.110389709472656,
      "activations/layer2_attention_weight_max": 34.21711349487305,
      "activations/layer2_attention_weight_min": -32.734413146972656,
      "activations/layer3_attention_weight_max": 102.15705108642578,
      "activations/layer3_attention_weight_min": -100.36860656738281,
      "activations/layer4_attention_weight_max": 106.77980041503906,
      "activations/layer4_attention_weight_min": -106.68981170654297,
      "activations/layer5_attention_weight_max": 47.576454162597656,
      "activations/layer5_attention_weight_min": -59.806636810302734,
      "activations/layer6_attention_weight_max": 43.13785171508789,
      "activations/layer6_attention_weight_min": -48.61701583862305,
      "activations/layer7_attention_weight_max": 92.78881072998047,
      "activations/layer7_attention_weight_min": -96.19740295410156,
      "activations/layer8_attention_weight_max": 42.79690170288086,
      "activations/layer8_attention_weight_min": -43.15217208862305,
      "activations/layer9_attention_weight_max": 32.426334381103516,
      "activations/layer9_attention_weight_min": -35.51610565185547,
      "epoch": 18.17,
      "learning_rate": 3.318068181818181e-05,
      "loss": 2.7365,
      "step": 312650
    },
    {
      "activations/layer0_attention_weight_max": 16.97319984436035,
      "activations/layer0_attention_weight_min": -12.310969352722168,
      "activations/layer10_attention_weight_max": 36.46087646484375,
      "activations/layer10_attention_weight_min": -37.964073181152344,
      "activations/layer11_attention_weight_max": 35.83510971069336,
      "activations/layer11_attention_weight_min": -37.18533706665039,
      "activations/layer12_attention_weight_max": 29.473161697387695,
      "activations/layer12_attention_weight_min": -27.165546417236328,
      "activations/layer13_attention_weight_max": 41.39337921142578,
      "activations/layer13_attention_weight_min": -36.485958099365234,
      "activations/layer14_attention_weight_max": 44.22841262817383,
      "activations/layer14_attention_weight_min": -38.923187255859375,
      "activations/layer15_attention_weight_max": 40.917442321777344,
      "activations/layer15_attention_weight_min": -39.34614562988281,
      "activations/layer16_attention_weight_max": 31.28399658203125,
      "activations/layer16_attention_weight_min": -29.213781356811523,
      "activations/layer17_attention_weight_max": 32.55766677856445,
      "activations/layer17_attention_weight_min": -27.90191078186035,
      "activations/layer18_attention_weight_max": 32.61244583129883,
      "activations/layer18_attention_weight_min": -26.697803497314453,
      "activations/layer19_attention_weight_max": 35.08010482788086,
      "activations/layer19_attention_weight_min": -33.62150192260742,
      "activations/layer1_attention_weight_max": 16.49315071105957,
      "activations/layer1_attention_weight_min": -14.1835355758667,
      "activations/layer20_attention_weight_max": 28.739660263061523,
      "activations/layer20_attention_weight_min": -25.977018356323242,
      "activations/layer21_attention_weight_max": 28.79340171813965,
      "activations/layer21_attention_weight_min": -24.91105079650879,
      "activations/layer22_attention_weight_max": 41.966400146484375,
      "activations/layer22_attention_weight_min": -34.004188537597656,
      "activations/layer23_attention_weight_max": 33.326271057128906,
      "activations/layer23_attention_weight_min": -27.16347885131836,
      "activations/layer2_attention_weight_max": 34.71234893798828,
      "activations/layer2_attention_weight_min": -35.07972717285156,
      "activations/layer3_attention_weight_max": 106.838623046875,
      "activations/layer3_attention_weight_min": -110.93323516845703,
      "activations/layer4_attention_weight_max": 111.64159393310547,
      "activations/layer4_attention_weight_min": -115.38651275634766,
      "activations/layer5_attention_weight_max": 52.42597961425781,
      "activations/layer5_attention_weight_min": -59.863319396972656,
      "activations/layer6_attention_weight_max": 47.92713928222656,
      "activations/layer6_attention_weight_min": -49.543296813964844,
      "activations/layer7_attention_weight_max": 103.26651000976562,
      "activations/layer7_attention_weight_min": -99.89997863769531,
      "activations/layer8_attention_weight_max": 46.37394332885742,
      "activations/layer8_attention_weight_min": -47.66005325317383,
      "activations/layer9_attention_weight_max": 36.33793258666992,
      "activations/layer9_attention_weight_min": -36.31781005859375,
      "epoch": 18.17,
      "learning_rate": 3.316174242424242e-05,
      "loss": 2.7373,
      "step": 312700
    },
    {
      "activations/layer0_attention_weight_max": 16.6453800201416,
      "activations/layer0_attention_weight_min": -12.45595645904541,
      "activations/layer10_attention_weight_max": 39.68333053588867,
      "activations/layer10_attention_weight_min": -35.41713333129883,
      "activations/layer11_attention_weight_max": 41.63003921508789,
      "activations/layer11_attention_weight_min": -38.423858642578125,
      "activations/layer12_attention_weight_max": 25.487533569335938,
      "activations/layer12_attention_weight_min": -27.75261116027832,
      "activations/layer13_attention_weight_max": 38.295021057128906,
      "activations/layer13_attention_weight_min": -36.608314514160156,
      "activations/layer14_attention_weight_max": 40.82368469238281,
      "activations/layer14_attention_weight_min": -41.15554428100586,
      "activations/layer15_attention_weight_max": 41.440147399902344,
      "activations/layer15_attention_weight_min": -37.72092819213867,
      "activations/layer16_attention_weight_max": 31.579116821289062,
      "activations/layer16_attention_weight_min": -28.70209503173828,
      "activations/layer17_attention_weight_max": 29.96538734436035,
      "activations/layer17_attention_weight_min": -28.99802589416504,
      "activations/layer18_attention_weight_max": 29.864225387573242,
      "activations/layer18_attention_weight_min": -27.23997688293457,
      "activations/layer19_attention_weight_max": 31.631122589111328,
      "activations/layer19_attention_weight_min": -33.935123443603516,
      "activations/layer1_attention_weight_max": 16.404441833496094,
      "activations/layer1_attention_weight_min": -15.454485893249512,
      "activations/layer20_attention_weight_max": 27.630592346191406,
      "activations/layer20_attention_weight_min": -28.254959106445312,
      "activations/layer21_attention_weight_max": 27.67778205871582,
      "activations/layer21_attention_weight_min": -26.393949508666992,
      "activations/layer22_attention_weight_max": 41.22248458862305,
      "activations/layer22_attention_weight_min": -32.06174087524414,
      "activations/layer23_attention_weight_max": 34.27510070800781,
      "activations/layer23_attention_weight_min": -24.88301658630371,
      "activations/layer2_attention_weight_max": 33.24091720581055,
      "activations/layer2_attention_weight_min": -32.582313537597656,
      "activations/layer3_attention_weight_max": 99.1422119140625,
      "activations/layer3_attention_weight_min": -105.95233917236328,
      "activations/layer4_attention_weight_max": 110.11287689208984,
      "activations/layer4_attention_weight_min": -109.47069549560547,
      "activations/layer5_attention_weight_max": 50.59294128417969,
      "activations/layer5_attention_weight_min": -63.528297424316406,
      "activations/layer6_attention_weight_max": 44.47182083129883,
      "activations/layer6_attention_weight_min": -45.48020935058594,
      "activations/layer7_attention_weight_max": 102.09281158447266,
      "activations/layer7_attention_weight_min": -92.80047607421875,
      "activations/layer8_attention_weight_max": 44.985260009765625,
      "activations/layer8_attention_weight_min": -42.67067337036133,
      "activations/layer9_attention_weight_max": 33.54331970214844,
      "activations/layer9_attention_weight_min": -36.01754379272461,
      "epoch": 18.17,
      "learning_rate": 3.314318181818182e-05,
      "loss": 2.7272,
      "step": 312750
    },
    {
      "activations/layer0_attention_weight_max": 16.11865234375,
      "activations/layer0_attention_weight_min": -14.65723705291748,
      "activations/layer10_attention_weight_max": 31.70992660522461,
      "activations/layer10_attention_weight_min": -34.65090560913086,
      "activations/layer11_attention_weight_max": 34.731449127197266,
      "activations/layer11_attention_weight_min": -34.74310302734375,
      "activations/layer12_attention_weight_max": 27.753421783447266,
      "activations/layer12_attention_weight_min": -29.11577606201172,
      "activations/layer13_attention_weight_max": 39.758663177490234,
      "activations/layer13_attention_weight_min": -40.10770797729492,
      "activations/layer14_attention_weight_max": 41.88381576538086,
      "activations/layer14_attention_weight_min": -38.35431671142578,
      "activations/layer15_attention_weight_max": 38.17155838012695,
      "activations/layer15_attention_weight_min": -35.855220794677734,
      "activations/layer16_attention_weight_max": 29.118324279785156,
      "activations/layer16_attention_weight_min": -27.55244255065918,
      "activations/layer17_attention_weight_max": 30.00168800354004,
      "activations/layer17_attention_weight_min": -28.365943908691406,
      "activations/layer18_attention_weight_max": 31.67462730407715,
      "activations/layer18_attention_weight_min": -26.045392990112305,
      "activations/layer19_attention_weight_max": 35.12080383300781,
      "activations/layer19_attention_weight_min": -34.092010498046875,
      "activations/layer1_attention_weight_max": 15.574188232421875,
      "activations/layer1_attention_weight_min": -14.608713150024414,
      "activations/layer20_attention_weight_max": 30.45311164855957,
      "activations/layer20_attention_weight_min": -26.997434616088867,
      "activations/layer21_attention_weight_max": 28.26528549194336,
      "activations/layer21_attention_weight_min": -27.249141693115234,
      "activations/layer22_attention_weight_max": 43.95556640625,
      "activations/layer22_attention_weight_min": -33.64918899536133,
      "activations/layer23_attention_weight_max": 31.841102600097656,
      "activations/layer23_attention_weight_min": -25.597553253173828,
      "activations/layer2_attention_weight_max": 31.880786895751953,
      "activations/layer2_attention_weight_min": -30.45261001586914,
      "activations/layer3_attention_weight_max": 96.94681549072266,
      "activations/layer3_attention_weight_min": -96.55281066894531,
      "activations/layer4_attention_weight_max": 109.94792938232422,
      "activations/layer4_attention_weight_min": -105.8293685913086,
      "activations/layer5_attention_weight_max": 50.41259765625,
      "activations/layer5_attention_weight_min": -63.29893112182617,
      "activations/layer6_attention_weight_max": 46.189876556396484,
      "activations/layer6_attention_weight_min": -50.193172454833984,
      "activations/layer7_attention_weight_max": 104.15826416015625,
      "activations/layer7_attention_weight_min": -102.38941192626953,
      "activations/layer8_attention_weight_max": 40.45771789550781,
      "activations/layer8_attention_weight_min": -47.175445556640625,
      "activations/layer9_attention_weight_max": 32.113582611083984,
      "activations/layer9_attention_weight_min": -35.90134811401367,
      "epoch": 18.18,
      "learning_rate": 3.312462121212121e-05,
      "loss": 2.735,
      "step": 312800
    },
    {
      "activations/layer0_attention_weight_max": 16.113679885864258,
      "activations/layer0_attention_weight_min": -13.344046592712402,
      "activations/layer10_attention_weight_max": 31.23944664001465,
      "activations/layer10_attention_weight_min": -34.64681625366211,
      "activations/layer11_attention_weight_max": 32.0848388671875,
      "activations/layer11_attention_weight_min": -34.37113571166992,
      "activations/layer12_attention_weight_max": 26.795883178710938,
      "activations/layer12_attention_weight_min": -28.6270809173584,
      "activations/layer13_attention_weight_max": 41.06171417236328,
      "activations/layer13_attention_weight_min": -38.7555046081543,
      "activations/layer14_attention_weight_max": 44.098533630371094,
      "activations/layer14_attention_weight_min": -41.69002914428711,
      "activations/layer15_attention_weight_max": 39.418304443359375,
      "activations/layer15_attention_weight_min": -38.58035659790039,
      "activations/layer16_attention_weight_max": 29.96824073791504,
      "activations/layer16_attention_weight_min": -29.380599975585938,
      "activations/layer17_attention_weight_max": 29.095712661743164,
      "activations/layer17_attention_weight_min": -27.377010345458984,
      "activations/layer18_attention_weight_max": 33.13153839111328,
      "activations/layer18_attention_weight_min": -27.410619735717773,
      "activations/layer19_attention_weight_max": 36.892513275146484,
      "activations/layer19_attention_weight_min": -36.10794448852539,
      "activations/layer1_attention_weight_max": 15.666210174560547,
      "activations/layer1_attention_weight_min": -15.085392951965332,
      "activations/layer20_attention_weight_max": 31.474924087524414,
      "activations/layer20_attention_weight_min": -28.74959373474121,
      "activations/layer21_attention_weight_max": 31.071584701538086,
      "activations/layer21_attention_weight_min": -28.564443588256836,
      "activations/layer22_attention_weight_max": 45.36396789550781,
      "activations/layer22_attention_weight_min": -33.81264114379883,
      "activations/layer23_attention_weight_max": 34.392478942871094,
      "activations/layer23_attention_weight_min": -29.235641479492188,
      "activations/layer2_attention_weight_max": 34.229217529296875,
      "activations/layer2_attention_weight_min": -33.98625183105469,
      "activations/layer3_attention_weight_max": 101.77902221679688,
      "activations/layer3_attention_weight_min": -102.62520599365234,
      "activations/layer4_attention_weight_max": 111.12499237060547,
      "activations/layer4_attention_weight_min": -113.4883804321289,
      "activations/layer5_attention_weight_max": 49.80324935913086,
      "activations/layer5_attention_weight_min": -63.276668548583984,
      "activations/layer6_attention_weight_max": 45.32551574707031,
      "activations/layer6_attention_weight_min": -46.99415588378906,
      "activations/layer7_attention_weight_max": 91.71675872802734,
      "activations/layer7_attention_weight_min": -96.1827621459961,
      "activations/layer8_attention_weight_max": 40.60311508178711,
      "activations/layer8_attention_weight_min": -46.91197204589844,
      "activations/layer9_attention_weight_max": 30.541934967041016,
      "activations/layer9_attention_weight_min": -36.07404708862305,
      "epoch": 18.18,
      "learning_rate": 3.310568181818182e-05,
      "loss": 2.7354,
      "step": 312850
    },
    {
      "activations/layer0_attention_weight_max": 16.345523834228516,
      "activations/layer0_attention_weight_min": -13.643976211547852,
      "activations/layer10_attention_weight_max": 31.0289363861084,
      "activations/layer10_attention_weight_min": -33.245216369628906,
      "activations/layer11_attention_weight_max": 31.659019470214844,
      "activations/layer11_attention_weight_min": -32.461952209472656,
      "activations/layer12_attention_weight_max": 25.21026611328125,
      "activations/layer12_attention_weight_min": -27.823570251464844,
      "activations/layer13_attention_weight_max": 39.897918701171875,
      "activations/layer13_attention_weight_min": -37.82172393798828,
      "activations/layer14_attention_weight_max": 39.51596450805664,
      "activations/layer14_attention_weight_min": -38.70204544067383,
      "activations/layer15_attention_weight_max": 36.30620193481445,
      "activations/layer15_attention_weight_min": -39.36018371582031,
      "activations/layer16_attention_weight_max": 27.341819763183594,
      "activations/layer16_attention_weight_min": -29.52836036682129,
      "activations/layer17_attention_weight_max": 28.226974487304688,
      "activations/layer17_attention_weight_min": -27.263349533081055,
      "activations/layer18_attention_weight_max": 33.24380111694336,
      "activations/layer18_attention_weight_min": -26.664974212646484,
      "activations/layer19_attention_weight_max": 30.985233306884766,
      "activations/layer19_attention_weight_min": -32.9753532409668,
      "activations/layer1_attention_weight_max": 16.231027603149414,
      "activations/layer1_attention_weight_min": -15.002004623413086,
      "activations/layer20_attention_weight_max": 30.11507797241211,
      "activations/layer20_attention_weight_min": -27.30436897277832,
      "activations/layer21_attention_weight_max": 25.597869873046875,
      "activations/layer21_attention_weight_min": -25.310453414916992,
      "activations/layer22_attention_weight_max": 40.643341064453125,
      "activations/layer22_attention_weight_min": -31.60406494140625,
      "activations/layer23_attention_weight_max": 29.879653930664062,
      "activations/layer23_attention_weight_min": -27.02912712097168,
      "activations/layer2_attention_weight_max": 34.70933532714844,
      "activations/layer2_attention_weight_min": -31.923484802246094,
      "activations/layer3_attention_weight_max": 94.66677856445312,
      "activations/layer3_attention_weight_min": -96.38099670410156,
      "activations/layer4_attention_weight_max": 98.7594223022461,
      "activations/layer4_attention_weight_min": -102.2683334350586,
      "activations/layer5_attention_weight_max": 48.679500579833984,
      "activations/layer5_attention_weight_min": -62.49834442138672,
      "activations/layer6_attention_weight_max": 43.036502838134766,
      "activations/layer6_attention_weight_min": -46.143333435058594,
      "activations/layer7_attention_weight_max": 87.2083969116211,
      "activations/layer7_attention_weight_min": -97.35250854492188,
      "activations/layer8_attention_weight_max": 43.26433563232422,
      "activations/layer8_attention_weight_min": -45.06953811645508,
      "activations/layer9_attention_weight_max": 30.31284523010254,
      "activations/layer9_attention_weight_min": -34.550472259521484,
      "epoch": 18.18,
      "learning_rate": 3.308674242424242e-05,
      "loss": 2.7222,
      "step": 312900
    },
    {
      "activations/layer0_attention_weight_max": 17.736669540405273,
      "activations/layer0_attention_weight_min": -13.678131103515625,
      "activations/layer10_attention_weight_max": 40.23267364501953,
      "activations/layer10_attention_weight_min": -35.7543830871582,
      "activations/layer11_attention_weight_max": 39.588775634765625,
      "activations/layer11_attention_weight_min": -37.18688201904297,
      "activations/layer12_attention_weight_max": 26.43412208557129,
      "activations/layer12_attention_weight_min": -29.315805435180664,
      "activations/layer13_attention_weight_max": 39.53296661376953,
      "activations/layer13_attention_weight_min": -35.327606201171875,
      "activations/layer14_attention_weight_max": 40.76549530029297,
      "activations/layer14_attention_weight_min": -36.33229064941406,
      "activations/layer15_attention_weight_max": 41.96791458129883,
      "activations/layer15_attention_weight_min": -36.490447998046875,
      "activations/layer16_attention_weight_max": 32.82297134399414,
      "activations/layer16_attention_weight_min": -27.08791732788086,
      "activations/layer17_attention_weight_max": 29.334049224853516,
      "activations/layer17_attention_weight_min": -24.85020637512207,
      "activations/layer18_attention_weight_max": 31.825355529785156,
      "activations/layer18_attention_weight_min": -26.55621910095215,
      "activations/layer19_attention_weight_max": 36.96543502807617,
      "activations/layer19_attention_weight_min": -30.7393798828125,
      "activations/layer1_attention_weight_max": 16.235488891601562,
      "activations/layer1_attention_weight_min": -15.907119750976562,
      "activations/layer20_attention_weight_max": 29.0903263092041,
      "activations/layer20_attention_weight_min": -25.235261917114258,
      "activations/layer21_attention_weight_max": 30.24388313293457,
      "activations/layer21_attention_weight_min": -24.459564208984375,
      "activations/layer22_attention_weight_max": 42.826416015625,
      "activations/layer22_attention_weight_min": -31.22572898864746,
      "activations/layer23_attention_weight_max": 35.139183044433594,
      "activations/layer23_attention_weight_min": -24.324665069580078,
      "activations/layer2_attention_weight_max": 32.90892028808594,
      "activations/layer2_attention_weight_min": -33.273902893066406,
      "activations/layer3_attention_weight_max": 96.94207000732422,
      "activations/layer3_attention_weight_min": -101.90220642089844,
      "activations/layer4_attention_weight_max": 100.40235137939453,
      "activations/layer4_attention_weight_min": -104.1007308959961,
      "activations/layer5_attention_weight_max": 47.7061767578125,
      "activations/layer5_attention_weight_min": -61.01496505737305,
      "activations/layer6_attention_weight_max": 43.748775482177734,
      "activations/layer6_attention_weight_min": -44.41474533081055,
      "activations/layer7_attention_weight_max": 101.88954162597656,
      "activations/layer7_attention_weight_min": -98.78174591064453,
      "activations/layer8_attention_weight_max": 46.14916229248047,
      "activations/layer8_attention_weight_min": -43.05485153198242,
      "activations/layer9_attention_weight_max": 32.41519546508789,
      "activations/layer9_attention_weight_min": -35.35981750488281,
      "epoch": 18.18,
      "learning_rate": 3.306780303030303e-05,
      "loss": 2.7354,
      "step": 312950
    },
    {
      "activations/layer0_attention_weight_max": 16.56532096862793,
      "activations/layer0_attention_weight_min": -12.86711597442627,
      "activations/layer10_attention_weight_max": 34.027587890625,
      "activations/layer10_attention_weight_min": -32.985965728759766,
      "activations/layer11_attention_weight_max": 33.83095932006836,
      "activations/layer11_attention_weight_min": -33.65153503417969,
      "activations/layer12_attention_weight_max": 27.01224136352539,
      "activations/layer12_attention_weight_min": -25.820907592773438,
      "activations/layer13_attention_weight_max": 37.72799301147461,
      "activations/layer13_attention_weight_min": -33.83773422241211,
      "activations/layer14_attention_weight_max": 42.393104553222656,
      "activations/layer14_attention_weight_min": -36.096923828125,
      "activations/layer15_attention_weight_max": 37.470741271972656,
      "activations/layer15_attention_weight_min": -35.791465759277344,
      "activations/layer16_attention_weight_max": 29.056211471557617,
      "activations/layer16_attention_weight_min": -29.573209762573242,
      "activations/layer17_attention_weight_max": 30.622095108032227,
      "activations/layer17_attention_weight_min": -27.439048767089844,
      "activations/layer18_attention_weight_max": 31.53211784362793,
      "activations/layer18_attention_weight_min": -28.8304500579834,
      "activations/layer19_attention_weight_max": 31.712570190429688,
      "activations/layer19_attention_weight_min": -32.18699645996094,
      "activations/layer1_attention_weight_max": 16.23038673400879,
      "activations/layer1_attention_weight_min": -15.563790321350098,
      "activations/layer20_attention_weight_max": 27.05739974975586,
      "activations/layer20_attention_weight_min": -26.5334415435791,
      "activations/layer21_attention_weight_max": 28.07311248779297,
      "activations/layer21_attention_weight_min": -26.18998146057129,
      "activations/layer22_attention_weight_max": 42.96381378173828,
      "activations/layer22_attention_weight_min": -31.68651008605957,
      "activations/layer23_attention_weight_max": 30.67522430419922,
      "activations/layer23_attention_weight_min": -25.611988067626953,
      "activations/layer2_attention_weight_max": 31.251667022705078,
      "activations/layer2_attention_weight_min": -30.52956199645996,
      "activations/layer3_attention_weight_max": 88.61170196533203,
      "activations/layer3_attention_weight_min": -96.53240203857422,
      "activations/layer4_attention_weight_max": 99.46459197998047,
      "activations/layer4_attention_weight_min": -104.2225341796875,
      "activations/layer5_attention_weight_max": 46.35798645019531,
      "activations/layer5_attention_weight_min": -59.529720306396484,
      "activations/layer6_attention_weight_max": 42.017066955566406,
      "activations/layer6_attention_weight_min": -45.57408142089844,
      "activations/layer7_attention_weight_max": 89.7228775024414,
      "activations/layer7_attention_weight_min": -91.36906433105469,
      "activations/layer8_attention_weight_max": 40.35432434082031,
      "activations/layer8_attention_weight_min": -45.68727111816406,
      "activations/layer9_attention_weight_max": 31.29754638671875,
      "activations/layer9_attention_weight_min": -34.038021087646484,
      "epoch": 18.19,
      "learning_rate": 3.304886363636363e-05,
      "loss": 2.721,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_loss": 2.6875,
      "eval_runtime": 8.5244,
      "eval_samples_per_second": 503.732,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.5244,
      "eval_openwebtext_samples_per_second": 503.732,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9818,
      "eval_wikitext_samples_per_second": 230.089,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_lambada_loss": 2.388671875,
      "eval_lambada_ppl": 10.899009079882486,
      "eval_lambada_runtime": 9.585,
      "eval_lambada_samples_per_second": 507.982,
      "step": 313000
    },
    {
      "activations/layer0_attention_weight_max": 15.374624252319336,
      "activations/layer0_attention_weight_min": -13.438325881958008,
      "activations/layer10_attention_weight_max": 33.926239013671875,
      "activations/layer10_attention_weight_min": -33.599609375,
      "activations/layer11_attention_weight_max": 33.787139892578125,
      "activations/layer11_attention_weight_min": -33.64012908935547,
      "activations/layer12_attention_weight_max": 27.888755798339844,
      "activations/layer12_attention_weight_min": -26.979507446289062,
      "activations/layer13_attention_weight_max": 40.46989822387695,
      "activations/layer13_attention_weight_min": -37.31977462768555,
      "activations/layer14_attention_weight_max": 42.32884216308594,
      "activations/layer14_attention_weight_min": -37.39535140991211,
      "activations/layer15_attention_weight_max": 44.426513671875,
      "activations/layer15_attention_weight_min": -38.94658660888672,
      "activations/layer16_attention_weight_max": 33.39035415649414,
      "activations/layer16_attention_weight_min": -29.36253547668457,
      "activations/layer17_attention_weight_max": 34.04069519042969,
      "activations/layer17_attention_weight_min": -27.13564682006836,
      "activations/layer18_attention_weight_max": 35.010623931884766,
      "activations/layer18_attention_weight_min": -28.205974578857422,
      "activations/layer19_attention_weight_max": 42.81718826293945,
      "activations/layer19_attention_weight_min": -34.76235580444336,
      "activations/layer1_attention_weight_max": 15.72840690612793,
      "activations/layer1_attention_weight_min": -14.29649829864502,
      "activations/layer20_attention_weight_max": 31.380409240722656,
      "activations/layer20_attention_weight_min": -26.699495315551758,
      "activations/layer21_attention_weight_max": 31.442150115966797,
      "activations/layer21_attention_weight_min": -25.616458892822266,
      "activations/layer22_attention_weight_max": 42.34309768676758,
      "activations/layer22_attention_weight_min": -33.00979995727539,
      "activations/layer23_attention_weight_max": 37.67240524291992,
      "activations/layer23_attention_weight_min": -28.368253707885742,
      "activations/layer2_attention_weight_max": 31.295942306518555,
      "activations/layer2_attention_weight_min": -31.456993103027344,
      "activations/layer3_attention_weight_max": 93.2081069946289,
      "activations/layer3_attention_weight_min": -96.08822631835938,
      "activations/layer4_attention_weight_max": 107.2021713256836,
      "activations/layer4_attention_weight_min": -105.37931060791016,
      "activations/layer5_attention_weight_max": 48.15016174316406,
      "activations/layer5_attention_weight_min": -59.22669219970703,
      "activations/layer6_attention_weight_max": 42.35072708129883,
      "activations/layer6_attention_weight_min": -46.67448425292969,
      "activations/layer7_attention_weight_max": 91.11746978759766,
      "activations/layer7_attention_weight_min": -96.07381439208984,
      "activations/layer8_attention_weight_max": 40.2308235168457,
      "activations/layer8_attention_weight_min": -43.59771728515625,
      "activations/layer9_attention_weight_max": 30.770004272460938,
      "activations/layer9_attention_weight_min": -33.09877395629883,
      "epoch": 18.19,
      "learning_rate": 3.302992424242424e-05,
      "loss": 2.7241,
      "step": 313050
    },
    {
      "activations/layer0_attention_weight_max": 16.380908966064453,
      "activations/layer0_attention_weight_min": -12.725554466247559,
      "activations/layer10_attention_weight_max": 32.279205322265625,
      "activations/layer10_attention_weight_min": -32.360939025878906,
      "activations/layer11_attention_weight_max": 32.21044158935547,
      "activations/layer11_attention_weight_min": -34.388755798339844,
      "activations/layer12_attention_weight_max": 24.42078971862793,
      "activations/layer12_attention_weight_min": -25.607324600219727,
      "activations/layer13_attention_weight_max": 36.4556999206543,
      "activations/layer13_attention_weight_min": -35.3187370300293,
      "activations/layer14_attention_weight_max": 39.58189010620117,
      "activations/layer14_attention_weight_min": -37.88216018676758,
      "activations/layer15_attention_weight_max": 35.2484016418457,
      "activations/layer15_attention_weight_min": -37.21796798706055,
      "activations/layer16_attention_weight_max": 27.15664291381836,
      "activations/layer16_attention_weight_min": -29.609045028686523,
      "activations/layer17_attention_weight_max": 29.887224197387695,
      "activations/layer17_attention_weight_min": -26.813159942626953,
      "activations/layer18_attention_weight_max": 31.548686981201172,
      "activations/layer18_attention_weight_min": -27.042865753173828,
      "activations/layer19_attention_weight_max": 32.751625061035156,
      "activations/layer19_attention_weight_min": -31.510385513305664,
      "activations/layer1_attention_weight_max": 16.598491668701172,
      "activations/layer1_attention_weight_min": -15.378364562988281,
      "activations/layer20_attention_weight_max": 28.17207908630371,
      "activations/layer20_attention_weight_min": -27.7340145111084,
      "activations/layer21_attention_weight_max": 32.34898376464844,
      "activations/layer21_attention_weight_min": -26.31696128845215,
      "activations/layer22_attention_weight_max": 42.038330078125,
      "activations/layer22_attention_weight_min": -34.25141143798828,
      "activations/layer23_attention_weight_max": 33.13597869873047,
      "activations/layer23_attention_weight_min": -24.544509887695312,
      "activations/layer2_attention_weight_max": 33.84807586669922,
      "activations/layer2_attention_weight_min": -33.082679748535156,
      "activations/layer3_attention_weight_max": 102.24778747558594,
      "activations/layer3_attention_weight_min": -101.68839263916016,
      "activations/layer4_attention_weight_max": 101.78582000732422,
      "activations/layer4_attention_weight_min": -105.0921401977539,
      "activations/layer5_attention_weight_max": 49.425621032714844,
      "activations/layer5_attention_weight_min": -62.74054718017578,
      "activations/layer6_attention_weight_max": 41.95750427246094,
      "activations/layer6_attention_weight_min": -43.6218376159668,
      "activations/layer7_attention_weight_max": 90.8831787109375,
      "activations/layer7_attention_weight_min": -89.01372528076172,
      "activations/layer8_attention_weight_max": 40.15752410888672,
      "activations/layer8_attention_weight_min": -40.97104263305664,
      "activations/layer9_attention_weight_max": 29.764570236206055,
      "activations/layer9_attention_weight_min": -32.90562438964844,
      "epoch": 18.19,
      "learning_rate": 3.3010984848484846e-05,
      "loss": 2.7469,
      "step": 313100
    },
    {
      "activations/layer0_attention_weight_max": 16.458099365234375,
      "activations/layer0_attention_weight_min": -12.232705116271973,
      "activations/layer10_attention_weight_max": 32.71112060546875,
      "activations/layer10_attention_weight_min": -36.0076789855957,
      "activations/layer11_attention_weight_max": 33.56879806518555,
      "activations/layer11_attention_weight_min": -33.068359375,
      "activations/layer12_attention_weight_max": 25.60417366027832,
      "activations/layer12_attention_weight_min": -26.285017013549805,
      "activations/layer13_attention_weight_max": 39.328765869140625,
      "activations/layer13_attention_weight_min": -35.669246673583984,
      "activations/layer14_attention_weight_max": 39.75461196899414,
      "activations/layer14_attention_weight_min": -36.09471130371094,
      "activations/layer15_attention_weight_max": 36.103721618652344,
      "activations/layer15_attention_weight_min": -35.05281448364258,
      "activations/layer16_attention_weight_max": 31.243175506591797,
      "activations/layer16_attention_weight_min": -28.85915184020996,
      "activations/layer17_attention_weight_max": 30.848440170288086,
      "activations/layer17_attention_weight_min": -26.31995964050293,
      "activations/layer18_attention_weight_max": 33.25913619995117,
      "activations/layer18_attention_weight_min": -25.896705627441406,
      "activations/layer19_attention_weight_max": 34.93104553222656,
      "activations/layer19_attention_weight_min": -30.857412338256836,
      "activations/layer1_attention_weight_max": 16.477466583251953,
      "activations/layer1_attention_weight_min": -15.42690372467041,
      "activations/layer20_attention_weight_max": 26.934892654418945,
      "activations/layer20_attention_weight_min": -27.976261138916016,
      "activations/layer21_attention_weight_max": 27.205419540405273,
      "activations/layer21_attention_weight_min": -27.461257934570312,
      "activations/layer22_attention_weight_max": 37.93623733520508,
      "activations/layer22_attention_weight_min": -32.155250549316406,
      "activations/layer23_attention_weight_max": 28.8420467376709,
      "activations/layer23_attention_weight_min": -27.173429489135742,
      "activations/layer2_attention_weight_max": 31.398014068603516,
      "activations/layer2_attention_weight_min": -30.492292404174805,
      "activations/layer3_attention_weight_max": 95.70831298828125,
      "activations/layer3_attention_weight_min": -94.31282043457031,
      "activations/layer4_attention_weight_max": 103.00882720947266,
      "activations/layer4_attention_weight_min": -105.46392822265625,
      "activations/layer5_attention_weight_max": 48.386470794677734,
      "activations/layer5_attention_weight_min": -62.492652893066406,
      "activations/layer6_attention_weight_max": 42.43857955932617,
      "activations/layer6_attention_weight_min": -45.519004821777344,
      "activations/layer7_attention_weight_max": 88.00615692138672,
      "activations/layer7_attention_weight_min": -96.14254760742188,
      "activations/layer8_attention_weight_max": 40.31142044067383,
      "activations/layer8_attention_weight_min": -42.20973587036133,
      "activations/layer9_attention_weight_max": 30.47511863708496,
      "activations/layer9_attention_weight_min": -34.41236114501953,
      "epoch": 18.2,
      "learning_rate": 3.2992045454545454e-05,
      "loss": 2.7214,
      "step": 313150
    },
    {
      "activations/layer0_attention_weight_max": 15.945669174194336,
      "activations/layer0_attention_weight_min": -13.290416717529297,
      "activations/layer10_attention_weight_max": 30.97215461730957,
      "activations/layer10_attention_weight_min": -31.99485206604004,
      "activations/layer11_attention_weight_max": 32.729164123535156,
      "activations/layer11_attention_weight_min": -31.43126678466797,
      "activations/layer12_attention_weight_max": 25.858997344970703,
      "activations/layer12_attention_weight_min": -27.641355514526367,
      "activations/layer13_attention_weight_max": 38.21690368652344,
      "activations/layer13_attention_weight_min": -35.68031311035156,
      "activations/layer14_attention_weight_max": 40.786712646484375,
      "activations/layer14_attention_weight_min": -35.643890380859375,
      "activations/layer15_attention_weight_max": 41.31597900390625,
      "activations/layer15_attention_weight_min": -35.59388732910156,
      "activations/layer16_attention_weight_max": 30.133838653564453,
      "activations/layer16_attention_weight_min": -27.364849090576172,
      "activations/layer17_attention_weight_max": 29.282602310180664,
      "activations/layer17_attention_weight_min": -25.492040634155273,
      "activations/layer18_attention_weight_max": 34.411094665527344,
      "activations/layer18_attention_weight_min": -25.897624969482422,
      "activations/layer19_attention_weight_max": 36.265838623046875,
      "activations/layer19_attention_weight_min": -31.94317626953125,
      "activations/layer1_attention_weight_max": 17.16058921813965,
      "activations/layer1_attention_weight_min": -14.07534122467041,
      "activations/layer20_attention_weight_max": 29.525707244873047,
      "activations/layer20_attention_weight_min": -26.045215606689453,
      "activations/layer21_attention_weight_max": 32.68501281738281,
      "activations/layer21_attention_weight_min": -25.695817947387695,
      "activations/layer22_attention_weight_max": 44.66518783569336,
      "activations/layer22_attention_weight_min": -31.573524475097656,
      "activations/layer23_attention_weight_max": 32.68702697753906,
      "activations/layer23_attention_weight_min": -26.976659774780273,
      "activations/layer2_attention_weight_max": 32.18855285644531,
      "activations/layer2_attention_weight_min": -32.89754104614258,
      "activations/layer3_attention_weight_max": 95.77015686035156,
      "activations/layer3_attention_weight_min": -98.1755142211914,
      "activations/layer4_attention_weight_max": 99.06807708740234,
      "activations/layer4_attention_weight_min": -103.47566986083984,
      "activations/layer5_attention_weight_max": 46.72343063354492,
      "activations/layer5_attention_weight_min": -59.23065948486328,
      "activations/layer6_attention_weight_max": 42.85556411743164,
      "activations/layer6_attention_weight_min": -45.78209686279297,
      "activations/layer7_attention_weight_max": 86.27828216552734,
      "activations/layer7_attention_weight_min": -95.2535400390625,
      "activations/layer8_attention_weight_max": 39.36919403076172,
      "activations/layer8_attention_weight_min": -43.01667785644531,
      "activations/layer9_attention_weight_max": 31.239782333374023,
      "activations/layer9_attention_weight_min": -34.568321228027344,
      "epoch": 18.2,
      "learning_rate": 3.297310606060606e-05,
      "loss": 2.721,
      "step": 313200
    },
    {
      "activations/layer0_attention_weight_max": 16.431692123413086,
      "activations/layer0_attention_weight_min": -13.025710105895996,
      "activations/layer10_attention_weight_max": 33.02708053588867,
      "activations/layer10_attention_weight_min": -32.828250885009766,
      "activations/layer11_attention_weight_max": 33.625240325927734,
      "activations/layer11_attention_weight_min": -33.00651550292969,
      "activations/layer12_attention_weight_max": 26.380701065063477,
      "activations/layer12_attention_weight_min": -27.511796951293945,
      "activations/layer13_attention_weight_max": 37.35197830200195,
      "activations/layer13_attention_weight_min": -36.402671813964844,
      "activations/layer14_attention_weight_max": 40.30121994018555,
      "activations/layer14_attention_weight_min": -38.606510162353516,
      "activations/layer15_attention_weight_max": 36.17137145996094,
      "activations/layer15_attention_weight_min": -34.84727478027344,
      "activations/layer16_attention_weight_max": 28.28728485107422,
      "activations/layer16_attention_weight_min": -26.645055770874023,
      "activations/layer17_attention_weight_max": 28.237764358520508,
      "activations/layer17_attention_weight_min": -26.6376953125,
      "activations/layer18_attention_weight_max": 30.725582122802734,
      "activations/layer18_attention_weight_min": -26.246309280395508,
      "activations/layer19_attention_weight_max": 30.009204864501953,
      "activations/layer19_attention_weight_min": -31.36154556274414,
      "activations/layer1_attention_weight_max": 17.794822692871094,
      "activations/layer1_attention_weight_min": -15.20145320892334,
      "activations/layer20_attention_weight_max": 26.783830642700195,
      "activations/layer20_attention_weight_min": -27.169780731201172,
      "activations/layer21_attention_weight_max": 26.568424224853516,
      "activations/layer21_attention_weight_min": -26.764976501464844,
      "activations/layer22_attention_weight_max": 37.195823669433594,
      "activations/layer22_attention_weight_min": -34.25584030151367,
      "activations/layer23_attention_weight_max": 30.66242218017578,
      "activations/layer23_attention_weight_min": -28.06206512451172,
      "activations/layer2_attention_weight_max": 32.55756378173828,
      "activations/layer2_attention_weight_min": -29.47686767578125,
      "activations/layer3_attention_weight_max": 94.02954864501953,
      "activations/layer3_attention_weight_min": -96.31916809082031,
      "activations/layer4_attention_weight_max": 101.80699157714844,
      "activations/layer4_attention_weight_min": -101.19690704345703,
      "activations/layer5_attention_weight_max": 51.96950912475586,
      "activations/layer5_attention_weight_min": -61.39918518066406,
      "activations/layer6_attention_weight_max": 43.00461959838867,
      "activations/layer6_attention_weight_min": -46.22085189819336,
      "activations/layer7_attention_weight_max": 92.00598907470703,
      "activations/layer7_attention_weight_min": -91.51514434814453,
      "activations/layer8_attention_weight_max": 39.23553466796875,
      "activations/layer8_attention_weight_min": -41.405181884765625,
      "activations/layer9_attention_weight_max": 33.30855178833008,
      "activations/layer9_attention_weight_min": -33.53437423706055,
      "epoch": 18.2,
      "learning_rate": 3.2954166666666664e-05,
      "loss": 2.7282,
      "step": 313250
    },
    {
      "activations/layer0_attention_weight_max": 15.516529083251953,
      "activations/layer0_attention_weight_min": -12.490378379821777,
      "activations/layer10_attention_weight_max": 32.5646858215332,
      "activations/layer10_attention_weight_min": -34.82979965209961,
      "activations/layer11_attention_weight_max": 32.207908630371094,
      "activations/layer11_attention_weight_min": -33.04827880859375,
      "activations/layer12_attention_weight_max": 29.212024688720703,
      "activations/layer12_attention_weight_min": -27.38188362121582,
      "activations/layer13_attention_weight_max": 38.97222137451172,
      "activations/layer13_attention_weight_min": -35.687496185302734,
      "activations/layer14_attention_weight_max": 44.43225860595703,
      "activations/layer14_attention_weight_min": -39.33549118041992,
      "activations/layer15_attention_weight_max": 38.7010612487793,
      "activations/layer15_attention_weight_min": -35.77142333984375,
      "activations/layer16_attention_weight_max": 30.046812057495117,
      "activations/layer16_attention_weight_min": -28.606430053710938,
      "activations/layer17_attention_weight_max": 30.16944122314453,
      "activations/layer17_attention_weight_min": -27.48165512084961,
      "activations/layer18_attention_weight_max": 33.33735275268555,
      "activations/layer18_attention_weight_min": -26.104278564453125,
      "activations/layer19_attention_weight_max": 37.35844802856445,
      "activations/layer19_attention_weight_min": -34.64609909057617,
      "activations/layer1_attention_weight_max": 17.62870979309082,
      "activations/layer1_attention_weight_min": -14.270503997802734,
      "activations/layer20_attention_weight_max": 32.19284439086914,
      "activations/layer20_attention_weight_min": -26.89964485168457,
      "activations/layer21_attention_weight_max": 32.04429626464844,
      "activations/layer21_attention_weight_min": -25.69334602355957,
      "activations/layer22_attention_weight_max": 50.79642105102539,
      "activations/layer22_attention_weight_min": -32.60557174682617,
      "activations/layer23_attention_weight_max": 34.735389709472656,
      "activations/layer23_attention_weight_min": -26.149375915527344,
      "activations/layer2_attention_weight_max": 33.2783317565918,
      "activations/layer2_attention_weight_min": -32.77577209472656,
      "activations/layer3_attention_weight_max": 99.04476928710938,
      "activations/layer3_attention_weight_min": -97.91798400878906,
      "activations/layer4_attention_weight_max": 108.48480224609375,
      "activations/layer4_attention_weight_min": -110.8006820678711,
      "activations/layer5_attention_weight_max": 53.265846252441406,
      "activations/layer5_attention_weight_min": -65.02519989013672,
      "activations/layer6_attention_weight_max": 43.158470153808594,
      "activations/layer6_attention_weight_min": -47.42012023925781,
      "activations/layer7_attention_weight_max": 88.52386474609375,
      "activations/layer7_attention_weight_min": -95.80443572998047,
      "activations/layer8_attention_weight_max": 40.91963577270508,
      "activations/layer8_attention_weight_min": -43.54667282104492,
      "activations/layer9_attention_weight_max": 32.47359085083008,
      "activations/layer9_attention_weight_min": -33.08934783935547,
      "epoch": 18.2,
      "learning_rate": 3.293522727272727e-05,
      "loss": 2.7273,
      "step": 313300
    },
    {
      "activations/layer0_attention_weight_max": 16.31977081298828,
      "activations/layer0_attention_weight_min": -12.539037704467773,
      "activations/layer10_attention_weight_max": 33.001487731933594,
      "activations/layer10_attention_weight_min": -33.73188018798828,
      "activations/layer11_attention_weight_max": 33.70573806762695,
      "activations/layer11_attention_weight_min": -34.4818115234375,
      "activations/layer12_attention_weight_max": 25.373205184936523,
      "activations/layer12_attention_weight_min": -25.859115600585938,
      "activations/layer13_attention_weight_max": 36.142486572265625,
      "activations/layer13_attention_weight_min": -35.42376708984375,
      "activations/layer14_attention_weight_max": 39.86505889892578,
      "activations/layer14_attention_weight_min": -37.08707809448242,
      "activations/layer15_attention_weight_max": 38.24526596069336,
      "activations/layer15_attention_weight_min": -33.902549743652344,
      "activations/layer16_attention_weight_max": 29.204383850097656,
      "activations/layer16_attention_weight_min": -28.395647048950195,
      "activations/layer17_attention_weight_max": 31.341779708862305,
      "activations/layer17_attention_weight_min": -25.816864013671875,
      "activations/layer18_attention_weight_max": 32.24852752685547,
      "activations/layer18_attention_weight_min": -26.031911849975586,
      "activations/layer19_attention_weight_max": 32.659889221191406,
      "activations/layer19_attention_weight_min": -34.845603942871094,
      "activations/layer1_attention_weight_max": 16.763160705566406,
      "activations/layer1_attention_weight_min": -14.41553020477295,
      "activations/layer20_attention_weight_max": 30.056562423706055,
      "activations/layer20_attention_weight_min": -27.203643798828125,
      "activations/layer21_attention_weight_max": 28.88578987121582,
      "activations/layer21_attention_weight_min": -25.94981575012207,
      "activations/layer22_attention_weight_max": 41.343528747558594,
      "activations/layer22_attention_weight_min": -32.94375991821289,
      "activations/layer23_attention_weight_max": 32.61823654174805,
      "activations/layer23_attention_weight_min": -28.02068328857422,
      "activations/layer2_attention_weight_max": 30.003034591674805,
      "activations/layer2_attention_weight_min": -30.374988555908203,
      "activations/layer3_attention_weight_max": 93.11339569091797,
      "activations/layer3_attention_weight_min": -94.91937255859375,
      "activations/layer4_attention_weight_max": 103.2731704711914,
      "activations/layer4_attention_weight_min": -101.70736694335938,
      "activations/layer5_attention_weight_max": 50.375946044921875,
      "activations/layer5_attention_weight_min": -59.334632873535156,
      "activations/layer6_attention_weight_max": 41.69167709350586,
      "activations/layer6_attention_weight_min": -44.25704574584961,
      "activations/layer7_attention_weight_max": 94.4752426147461,
      "activations/layer7_attention_weight_min": -94.65462493896484,
      "activations/layer8_attention_weight_max": 39.474788665771484,
      "activations/layer8_attention_weight_min": -40.19464874267578,
      "activations/layer9_attention_weight_max": 31.24329948425293,
      "activations/layer9_attention_weight_min": -32.218021392822266,
      "epoch": 18.21,
      "learning_rate": 3.291628787878787e-05,
      "loss": 2.7127,
      "step": 313350
    },
    {
      "activations/layer0_attention_weight_max": 16.575786590576172,
      "activations/layer0_attention_weight_min": -13.117796897888184,
      "activations/layer10_attention_weight_max": 37.272342681884766,
      "activations/layer10_attention_weight_min": -37.155601501464844,
      "activations/layer11_attention_weight_max": 36.86909484863281,
      "activations/layer11_attention_weight_min": -36.32966232299805,
      "activations/layer12_attention_weight_max": 27.2310733795166,
      "activations/layer12_attention_weight_min": -26.959733963012695,
      "activations/layer13_attention_weight_max": 39.06096649169922,
      "activations/layer13_attention_weight_min": -34.64406967163086,
      "activations/layer14_attention_weight_max": 41.277069091796875,
      "activations/layer14_attention_weight_min": -35.46643829345703,
      "activations/layer15_attention_weight_max": 40.85330581665039,
      "activations/layer15_attention_weight_min": -36.33824157714844,
      "activations/layer16_attention_weight_max": 31.777759552001953,
      "activations/layer16_attention_weight_min": -26.91791534423828,
      "activations/layer17_attention_weight_max": 30.09517478942871,
      "activations/layer17_attention_weight_min": -26.115488052368164,
      "activations/layer18_attention_weight_max": 30.8604679107666,
      "activations/layer18_attention_weight_min": -24.50832748413086,
      "activations/layer19_attention_weight_max": 32.4761962890625,
      "activations/layer19_attention_weight_min": -30.347614288330078,
      "activations/layer1_attention_weight_max": 16.684978485107422,
      "activations/layer1_attention_weight_min": -14.510944366455078,
      "activations/layer20_attention_weight_max": 28.7203369140625,
      "activations/layer20_attention_weight_min": -23.607213973999023,
      "activations/layer21_attention_weight_max": 28.474136352539062,
      "activations/layer21_attention_weight_min": -24.3005428314209,
      "activations/layer22_attention_weight_max": 43.03577423095703,
      "activations/layer22_attention_weight_min": -30.793298721313477,
      "activations/layer23_attention_weight_max": 34.780059814453125,
      "activations/layer23_attention_weight_min": -25.776844024658203,
      "activations/layer2_attention_weight_max": 31.173137664794922,
      "activations/layer2_attention_weight_min": -31.899028778076172,
      "activations/layer3_attention_weight_max": 96.23741149902344,
      "activations/layer3_attention_weight_min": -98.114013671875,
      "activations/layer4_attention_weight_max": 104.94123840332031,
      "activations/layer4_attention_weight_min": -107.5450210571289,
      "activations/layer5_attention_weight_max": 51.333229064941406,
      "activations/layer5_attention_weight_min": -60.600791931152344,
      "activations/layer6_attention_weight_max": 44.10689926147461,
      "activations/layer6_attention_weight_min": -44.47075653076172,
      "activations/layer7_attention_weight_max": 95.11663818359375,
      "activations/layer7_attention_weight_min": -107.43898010253906,
      "activations/layer8_attention_weight_max": 45.1236457824707,
      "activations/layer8_attention_weight_min": -48.113407135009766,
      "activations/layer9_attention_weight_max": 34.156402587890625,
      "activations/layer9_attention_weight_min": -38.126258850097656,
      "epoch": 18.21,
      "learning_rate": 3.289734848484848e-05,
      "loss": 2.7277,
      "step": 313400
    },
    {
      "activations/layer0_attention_weight_max": 16.52279281616211,
      "activations/layer0_attention_weight_min": -12.565696716308594,
      "activations/layer10_attention_weight_max": 36.82965087890625,
      "activations/layer10_attention_weight_min": -37.7485237121582,
      "activations/layer11_attention_weight_max": 36.31911849975586,
      "activations/layer11_attention_weight_min": -36.265052795410156,
      "activations/layer12_attention_weight_max": 27.121801376342773,
      "activations/layer12_attention_weight_min": -28.17386245727539,
      "activations/layer13_attention_weight_max": 41.87052917480469,
      "activations/layer13_attention_weight_min": -40.081172943115234,
      "activations/layer14_attention_weight_max": 43.93525314331055,
      "activations/layer14_attention_weight_min": -41.711082458496094,
      "activations/layer15_attention_weight_max": 40.48616409301758,
      "activations/layer15_attention_weight_min": -39.1767692565918,
      "activations/layer16_attention_weight_max": 29.904314041137695,
      "activations/layer16_attention_weight_min": -29.24960708618164,
      "activations/layer17_attention_weight_max": 33.343257904052734,
      "activations/layer17_attention_weight_min": -28.000993728637695,
      "activations/layer18_attention_weight_max": 37.980804443359375,
      "activations/layer18_attention_weight_min": -27.45394515991211,
      "activations/layer19_attention_weight_max": 37.263118743896484,
      "activations/layer19_attention_weight_min": -32.19197082519531,
      "activations/layer1_attention_weight_max": 15.866788864135742,
      "activations/layer1_attention_weight_min": -14.69473648071289,
      "activations/layer20_attention_weight_max": 32.146427154541016,
      "activations/layer20_attention_weight_min": -26.1419734954834,
      "activations/layer21_attention_weight_max": 30.885557174682617,
      "activations/layer21_attention_weight_min": -26.403146743774414,
      "activations/layer22_attention_weight_max": 43.903717041015625,
      "activations/layer22_attention_weight_min": -34.185462951660156,
      "activations/layer23_attention_weight_max": 33.57305145263672,
      "activations/layer23_attention_weight_min": -28.07497787475586,
      "activations/layer2_attention_weight_max": 32.758785247802734,
      "activations/layer2_attention_weight_min": -33.522579193115234,
      "activations/layer3_attention_weight_max": 98.83169555664062,
      "activations/layer3_attention_weight_min": -97.86959075927734,
      "activations/layer4_attention_weight_max": 107.23070526123047,
      "activations/layer4_attention_weight_min": -108.99037170410156,
      "activations/layer5_attention_weight_max": 51.561614990234375,
      "activations/layer5_attention_weight_min": -63.772735595703125,
      "activations/layer6_attention_weight_max": 45.36246871948242,
      "activations/layer6_attention_weight_min": -48.2064094543457,
      "activations/layer7_attention_weight_max": 92.30203247070312,
      "activations/layer7_attention_weight_min": -98.62739562988281,
      "activations/layer8_attention_weight_max": 42.49211502075195,
      "activations/layer8_attention_weight_min": -45.52744674682617,
      "activations/layer9_attention_weight_max": 34.99226379394531,
      "activations/layer9_attention_weight_min": -37.75507354736328,
      "epoch": 18.21,
      "learning_rate": 3.287840909090909e-05,
      "loss": 2.7354,
      "step": 313450
    },
    {
      "activations/layer0_attention_weight_max": 16.1309814453125,
      "activations/layer0_attention_weight_min": -13.100932121276855,
      "activations/layer10_attention_weight_max": 32.8678092956543,
      "activations/layer10_attention_weight_min": -37.50336456298828,
      "activations/layer11_attention_weight_max": 36.12346649169922,
      "activations/layer11_attention_weight_min": -34.21232604980469,
      "activations/layer12_attention_weight_max": 27.58317756652832,
      "activations/layer12_attention_weight_min": -28.727331161499023,
      "activations/layer13_attention_weight_max": 41.606605529785156,
      "activations/layer13_attention_weight_min": -38.57155990600586,
      "activations/layer14_attention_weight_max": 47.04177474975586,
      "activations/layer14_attention_weight_min": -41.715545654296875,
      "activations/layer15_attention_weight_max": 42.12656021118164,
      "activations/layer15_attention_weight_min": -37.92289352416992,
      "activations/layer16_attention_weight_max": 33.505393981933594,
      "activations/layer16_attention_weight_min": -29.386682510375977,
      "activations/layer17_attention_weight_max": 29.62200355529785,
      "activations/layer17_attention_weight_min": -28.654314041137695,
      "activations/layer18_attention_weight_max": 32.556983947753906,
      "activations/layer18_attention_weight_min": -27.949745178222656,
      "activations/layer19_attention_weight_max": 39.17434310913086,
      "activations/layer19_attention_weight_min": -32.728981018066406,
      "activations/layer1_attention_weight_max": 16.334815979003906,
      "activations/layer1_attention_weight_min": -15.497835159301758,
      "activations/layer20_attention_weight_max": 33.42218017578125,
      "activations/layer20_attention_weight_min": -27.285459518432617,
      "activations/layer21_attention_weight_max": 31.765483856201172,
      "activations/layer21_attention_weight_min": -26.42732810974121,
      "activations/layer22_attention_weight_max": 44.54682922363281,
      "activations/layer22_attention_weight_min": -35.01266860961914,
      "activations/layer23_attention_weight_max": 34.0851936340332,
      "activations/layer23_attention_weight_min": -26.91693878173828,
      "activations/layer2_attention_weight_max": 33.222145080566406,
      "activations/layer2_attention_weight_min": -31.54488182067871,
      "activations/layer3_attention_weight_max": 95.49014282226562,
      "activations/layer3_attention_weight_min": -96.63395690917969,
      "activations/layer4_attention_weight_max": 104.59024810791016,
      "activations/layer4_attention_weight_min": -105.60099029541016,
      "activations/layer5_attention_weight_max": 49.27134323120117,
      "activations/layer5_attention_weight_min": -59.53379440307617,
      "activations/layer6_attention_weight_max": 43.62028503417969,
      "activations/layer6_attention_weight_min": -44.99184036254883,
      "activations/layer7_attention_weight_max": 94.28997802734375,
      "activations/layer7_attention_weight_min": -93.0882797241211,
      "activations/layer8_attention_weight_max": 40.88860321044922,
      "activations/layer8_attention_weight_min": -43.55337142944336,
      "activations/layer9_attention_weight_max": 33.50072479248047,
      "activations/layer9_attention_weight_min": -34.88071823120117,
      "epoch": 18.22,
      "learning_rate": 3.28594696969697e-05,
      "loss": 2.7361,
      "step": 313500
    },
    {
      "activations/layer0_attention_weight_max": 16.113088607788086,
      "activations/layer0_attention_weight_min": -13.209882736206055,
      "activations/layer10_attention_weight_max": 32.395748138427734,
      "activations/layer10_attention_weight_min": -32.35184097290039,
      "activations/layer11_attention_weight_max": 34.24087142944336,
      "activations/layer11_attention_weight_min": -32.54874801635742,
      "activations/layer12_attention_weight_max": 25.958459854125977,
      "activations/layer12_attention_weight_min": -26.745943069458008,
      "activations/layer13_attention_weight_max": 37.147274017333984,
      "activations/layer13_attention_weight_min": -37.15581130981445,
      "activations/layer14_attention_weight_max": 38.59526824951172,
      "activations/layer14_attention_weight_min": -37.42374038696289,
      "activations/layer15_attention_weight_max": 36.44798278808594,
      "activations/layer15_attention_weight_min": -35.485103607177734,
      "activations/layer16_attention_weight_max": 28.973234176635742,
      "activations/layer16_attention_weight_min": -29.742307662963867,
      "activations/layer17_attention_weight_max": 28.58008575439453,
      "activations/layer17_attention_weight_min": -26.99363899230957,
      "activations/layer18_attention_weight_max": 32.04264831542969,
      "activations/layer18_attention_weight_min": -28.080768585205078,
      "activations/layer19_attention_weight_max": 33.704830169677734,
      "activations/layer19_attention_weight_min": -33.951560974121094,
      "activations/layer1_attention_weight_max": 16.75248146057129,
      "activations/layer1_attention_weight_min": -14.43743896484375,
      "activations/layer20_attention_weight_max": 31.138181686401367,
      "activations/layer20_attention_weight_min": -24.740489959716797,
      "activations/layer21_attention_weight_max": 29.894914627075195,
      "activations/layer21_attention_weight_min": -25.327470779418945,
      "activations/layer22_attention_weight_max": 38.01517105102539,
      "activations/layer22_attention_weight_min": -32.10211944580078,
      "activations/layer23_attention_weight_max": 28.53970718383789,
      "activations/layer23_attention_weight_min": -28.590457916259766,
      "activations/layer2_attention_weight_max": 31.39784049987793,
      "activations/layer2_attention_weight_min": -30.752269744873047,
      "activations/layer3_attention_weight_max": 93.8438491821289,
      "activations/layer3_attention_weight_min": -95.98035430908203,
      "activations/layer4_attention_weight_max": 103.5322494506836,
      "activations/layer4_attention_weight_min": -105.28126525878906,
      "activations/layer5_attention_weight_max": 49.40441131591797,
      "activations/layer5_attention_weight_min": -67.51593017578125,
      "activations/layer6_attention_weight_max": 41.93574142456055,
      "activations/layer6_attention_weight_min": -44.20695495605469,
      "activations/layer7_attention_weight_max": 87.525390625,
      "activations/layer7_attention_weight_min": -88.97403717041016,
      "activations/layer8_attention_weight_max": 39.998291015625,
      "activations/layer8_attention_weight_min": -41.78173828125,
      "activations/layer9_attention_weight_max": 30.50996971130371,
      "activations/layer9_attention_weight_min": -32.7745361328125,
      "epoch": 18.22,
      "learning_rate": 3.28405303030303e-05,
      "loss": 2.7194,
      "step": 313550
    },
    {
      "activations/layer0_attention_weight_max": 15.893177032470703,
      "activations/layer0_attention_weight_min": -12.955863952636719,
      "activations/layer10_attention_weight_max": 37.4808464050293,
      "activations/layer10_attention_weight_min": -36.035396575927734,
      "activations/layer11_attention_weight_max": 33.49765396118164,
      "activations/layer11_attention_weight_min": -35.37926483154297,
      "activations/layer12_attention_weight_max": 27.70566749572754,
      "activations/layer12_attention_weight_min": -29.990800857543945,
      "activations/layer13_attention_weight_max": 42.007911682128906,
      "activations/layer13_attention_weight_min": -39.61713409423828,
      "activations/layer14_attention_weight_max": 41.260276794433594,
      "activations/layer14_attention_weight_min": -38.96414566040039,
      "activations/layer15_attention_weight_max": 37.89171600341797,
      "activations/layer15_attention_weight_min": -37.74092483520508,
      "activations/layer16_attention_weight_max": 30.176219940185547,
      "activations/layer16_attention_weight_min": -28.721025466918945,
      "activations/layer17_attention_weight_max": 30.14418601989746,
      "activations/layer17_attention_weight_min": -26.6953182220459,
      "activations/layer18_attention_weight_max": 33.19176483154297,
      "activations/layer18_attention_weight_min": -27.545269012451172,
      "activations/layer19_attention_weight_max": 32.3248405456543,
      "activations/layer19_attention_weight_min": -31.03278923034668,
      "activations/layer1_attention_weight_max": 15.736583709716797,
      "activations/layer1_attention_weight_min": -13.294564247131348,
      "activations/layer20_attention_weight_max": 30.688053131103516,
      "activations/layer20_attention_weight_min": -25.61550521850586,
      "activations/layer21_attention_weight_max": 33.668243408203125,
      "activations/layer21_attention_weight_min": -24.753585815429688,
      "activations/layer22_attention_weight_max": 43.13684844970703,
      "activations/layer22_attention_weight_min": -32.96970748901367,
      "activations/layer23_attention_weight_max": 35.77305221557617,
      "activations/layer23_attention_weight_min": -25.80824089050293,
      "activations/layer2_attention_weight_max": 32.78517150878906,
      "activations/layer2_attention_weight_min": -31.804641723632812,
      "activations/layer3_attention_weight_max": 94.26882934570312,
      "activations/layer3_attention_weight_min": -93.71887969970703,
      "activations/layer4_attention_weight_max": 99.94416809082031,
      "activations/layer4_attention_weight_min": -106.25091552734375,
      "activations/layer5_attention_weight_max": 50.13740539550781,
      "activations/layer5_attention_weight_min": -63.412559509277344,
      "activations/layer6_attention_weight_max": 42.391963958740234,
      "activations/layer6_attention_weight_min": -47.008811950683594,
      "activations/layer7_attention_weight_max": 88.30931091308594,
      "activations/layer7_attention_weight_min": -99.76428985595703,
      "activations/layer8_attention_weight_max": 41.70035171508789,
      "activations/layer8_attention_weight_min": -42.711280822753906,
      "activations/layer9_attention_weight_max": 32.47710037231445,
      "activations/layer9_attention_weight_min": -34.554988861083984,
      "epoch": 18.22,
      "learning_rate": 3.282159090909091e-05,
      "loss": 2.7233,
      "step": 313600
    },
    {
      "activations/layer0_attention_weight_max": 16.188982009887695,
      "activations/layer0_attention_weight_min": -12.813468933105469,
      "activations/layer10_attention_weight_max": 36.16679763793945,
      "activations/layer10_attention_weight_min": -36.43777847290039,
      "activations/layer11_attention_weight_max": 35.66532516479492,
      "activations/layer11_attention_weight_min": -34.906105041503906,
      "activations/layer12_attention_weight_max": 28.79252815246582,
      "activations/layer12_attention_weight_min": -29.35053825378418,
      "activations/layer13_attention_weight_max": 44.55843734741211,
      "activations/layer13_attention_weight_min": -39.365699768066406,
      "activations/layer14_attention_weight_max": 44.55593490600586,
      "activations/layer14_attention_weight_min": -41.23750686645508,
      "activations/layer15_attention_weight_max": 39.65010070800781,
      "activations/layer15_attention_weight_min": -38.93115997314453,
      "activations/layer16_attention_weight_max": 30.557308197021484,
      "activations/layer16_attention_weight_min": -27.5225830078125,
      "activations/layer17_attention_weight_max": 29.920745849609375,
      "activations/layer17_attention_weight_min": -24.743371963500977,
      "activations/layer18_attention_weight_max": 32.53395462036133,
      "activations/layer18_attention_weight_min": -25.76141929626465,
      "activations/layer19_attention_weight_max": 32.83555221557617,
      "activations/layer19_attention_weight_min": -29.940576553344727,
      "activations/layer1_attention_weight_max": 16.091907501220703,
      "activations/layer1_attention_weight_min": -13.77015495300293,
      "activations/layer20_attention_weight_max": 28.28703498840332,
      "activations/layer20_attention_weight_min": -23.808120727539062,
      "activations/layer21_attention_weight_max": 28.394845962524414,
      "activations/layer21_attention_weight_min": -24.79205322265625,
      "activations/layer22_attention_weight_max": 41.28892135620117,
      "activations/layer22_attention_weight_min": -29.543264389038086,
      "activations/layer23_attention_weight_max": 30.285682678222656,
      "activations/layer23_attention_weight_min": -25.8800048828125,
      "activations/layer2_attention_weight_max": 30.567060470581055,
      "activations/layer2_attention_weight_min": -30.73202133178711,
      "activations/layer3_attention_weight_max": 96.2364501953125,
      "activations/layer3_attention_weight_min": -98.9339828491211,
      "activations/layer4_attention_weight_max": 108.1224594116211,
      "activations/layer4_attention_weight_min": -111.79566955566406,
      "activations/layer5_attention_weight_max": 50.164669036865234,
      "activations/layer5_attention_weight_min": -63.681556701660156,
      "activations/layer6_attention_weight_max": 46.46778869628906,
      "activations/layer6_attention_weight_min": -49.2563362121582,
      "activations/layer7_attention_weight_max": 103.7743148803711,
      "activations/layer7_attention_weight_min": -103.59195709228516,
      "activations/layer8_attention_weight_max": 43.441017150878906,
      "activations/layer8_attention_weight_min": -45.73197937011719,
      "activations/layer9_attention_weight_max": 34.32424545288086,
      "activations/layer9_attention_weight_min": -36.9896354675293,
      "epoch": 18.22,
      "learning_rate": 3.280265151515151e-05,
      "loss": 2.7417,
      "step": 313650
    },
    {
      "activations/layer0_attention_weight_max": 15.445406913757324,
      "activations/layer0_attention_weight_min": -12.840503692626953,
      "activations/layer10_attention_weight_max": 32.793678283691406,
      "activations/layer10_attention_weight_min": -34.5798454284668,
      "activations/layer11_attention_weight_max": 31.161914825439453,
      "activations/layer11_attention_weight_min": -33.933998107910156,
      "activations/layer12_attention_weight_max": 24.344913482666016,
      "activations/layer12_attention_weight_min": -28.44392204284668,
      "activations/layer13_attention_weight_max": 36.91090393066406,
      "activations/layer13_attention_weight_min": -35.958961486816406,
      "activations/layer14_attention_weight_max": 40.301151275634766,
      "activations/layer14_attention_weight_min": -39.265140533447266,
      "activations/layer15_attention_weight_max": 37.082401275634766,
      "activations/layer15_attention_weight_min": -36.663177490234375,
      "activations/layer16_attention_weight_max": 31.86046600341797,
      "activations/layer16_attention_weight_min": -28.37221908569336,
      "activations/layer17_attention_weight_max": 28.224321365356445,
      "activations/layer17_attention_weight_min": -26.854990005493164,
      "activations/layer18_attention_weight_max": 31.039409637451172,
      "activations/layer18_attention_weight_min": -28.639360427856445,
      "activations/layer19_attention_weight_max": 31.0137996673584,
      "activations/layer19_attention_weight_min": -30.6068058013916,
      "activations/layer1_attention_weight_max": 15.913592338562012,
      "activations/layer1_attention_weight_min": -12.716303825378418,
      "activations/layer20_attention_weight_max": 26.827306747436523,
      "activations/layer20_attention_weight_min": -25.759008407592773,
      "activations/layer21_attention_weight_max": 28.49802589416504,
      "activations/layer21_attention_weight_min": -24.023094177246094,
      "activations/layer22_attention_weight_max": 39.3869514465332,
      "activations/layer22_attention_weight_min": -29.92120361328125,
      "activations/layer23_attention_weight_max": 30.451366424560547,
      "activations/layer23_attention_weight_min": -24.298503875732422,
      "activations/layer2_attention_weight_max": 29.53302574157715,
      "activations/layer2_attention_weight_min": -30.513723373413086,
      "activations/layer3_attention_weight_max": 95.27921295166016,
      "activations/layer3_attention_weight_min": -97.26362609863281,
      "activations/layer4_attention_weight_max": 105.12828826904297,
      "activations/layer4_attention_weight_min": -105.84732818603516,
      "activations/layer5_attention_weight_max": 48.00910949707031,
      "activations/layer5_attention_weight_min": -59.770198822021484,
      "activations/layer6_attention_weight_max": 41.4882926940918,
      "activations/layer6_attention_weight_min": -44.97712707519531,
      "activations/layer7_attention_weight_max": 88.05523681640625,
      "activations/layer7_attention_weight_min": -88.06771850585938,
      "activations/layer8_attention_weight_max": 38.44968795776367,
      "activations/layer8_attention_weight_min": -42.4097900390625,
      "activations/layer9_attention_weight_max": 30.741422653198242,
      "activations/layer9_attention_weight_min": -33.04107666015625,
      "epoch": 18.23,
      "learning_rate": 3.278371212121212e-05,
      "loss": 2.7311,
      "step": 313700
    },
    {
      "activations/layer0_attention_weight_max": 16.25166893005371,
      "activations/layer0_attention_weight_min": -12.967479705810547,
      "activations/layer10_attention_weight_max": 34.69725799560547,
      "activations/layer10_attention_weight_min": -34.091678619384766,
      "activations/layer11_attention_weight_max": 33.87582778930664,
      "activations/layer11_attention_weight_min": -33.03396987915039,
      "activations/layer12_attention_weight_max": 26.853425979614258,
      "activations/layer12_attention_weight_min": -26.62786293029785,
      "activations/layer13_attention_weight_max": 41.11610412597656,
      "activations/layer13_attention_weight_min": -35.461143493652344,
      "activations/layer14_attention_weight_max": 41.781646728515625,
      "activations/layer14_attention_weight_min": -40.286033630371094,
      "activations/layer15_attention_weight_max": 40.07168197631836,
      "activations/layer15_attention_weight_min": -36.55310821533203,
      "activations/layer16_attention_weight_max": 28.95954704284668,
      "activations/layer16_attention_weight_min": -29.985570907592773,
      "activations/layer17_attention_weight_max": 29.35480499267578,
      "activations/layer17_attention_weight_min": -26.96489906311035,
      "activations/layer18_attention_weight_max": 31.252656936645508,
      "activations/layer18_attention_weight_min": -26.527563095092773,
      "activations/layer19_attention_weight_max": 33.24131393432617,
      "activations/layer19_attention_weight_min": -33.80662536621094,
      "activations/layer1_attention_weight_max": 15.982758522033691,
      "activations/layer1_attention_weight_min": -15.620461463928223,
      "activations/layer20_attention_weight_max": 29.036792755126953,
      "activations/layer20_attention_weight_min": -27.852022171020508,
      "activations/layer21_attention_weight_max": 28.203716278076172,
      "activations/layer21_attention_weight_min": -26.3945369720459,
      "activations/layer22_attention_weight_max": 40.21330261230469,
      "activations/layer22_attention_weight_min": -32.00160598754883,
      "activations/layer23_attention_weight_max": 29.517032623291016,
      "activations/layer23_attention_weight_min": -25.28180694580078,
      "activations/layer2_attention_weight_max": 33.544403076171875,
      "activations/layer2_attention_weight_min": -32.412574768066406,
      "activations/layer3_attention_weight_max": 100.02984619140625,
      "activations/layer3_attention_weight_min": -103.58152770996094,
      "activations/layer4_attention_weight_max": 108.24584197998047,
      "activations/layer4_attention_weight_min": -105.92818450927734,
      "activations/layer5_attention_weight_max": 49.62024688720703,
      "activations/layer5_attention_weight_min": -62.9881706237793,
      "activations/layer6_attention_weight_max": 44.07481384277344,
      "activations/layer6_attention_weight_min": -45.513370513916016,
      "activations/layer7_attention_weight_max": 92.34729766845703,
      "activations/layer7_attention_weight_min": -94.32088470458984,
      "activations/layer8_attention_weight_max": 40.699764251708984,
      "activations/layer8_attention_weight_min": -42.19264602661133,
      "activations/layer9_attention_weight_max": 32.24020004272461,
      "activations/layer9_attention_weight_min": -33.43549728393555,
      "epoch": 18.23,
      "learning_rate": 3.2764772727272726e-05,
      "loss": 2.7539,
      "step": 313750
    },
    {
      "activations/layer0_attention_weight_max": 15.957245826721191,
      "activations/layer0_attention_weight_min": -14.371826171875,
      "activations/layer10_attention_weight_max": 33.08281326293945,
      "activations/layer10_attention_weight_min": -32.64018249511719,
      "activations/layer11_attention_weight_max": 35.03561019897461,
      "activations/layer11_attention_weight_min": -33.26401138305664,
      "activations/layer12_attention_weight_max": 26.57724380493164,
      "activations/layer12_attention_weight_min": -26.594480514526367,
      "activations/layer13_attention_weight_max": 40.83619689941406,
      "activations/layer13_attention_weight_min": -37.511009216308594,
      "activations/layer14_attention_weight_max": 42.13740921020508,
      "activations/layer14_attention_weight_min": -40.65840148925781,
      "activations/layer15_attention_weight_max": 40.12456130981445,
      "activations/layer15_attention_weight_min": -37.34029769897461,
      "activations/layer16_attention_weight_max": 28.674297332763672,
      "activations/layer16_attention_weight_min": -29.230714797973633,
      "activations/layer17_attention_weight_max": 31.11717414855957,
      "activations/layer17_attention_weight_min": -27.89484977722168,
      "activations/layer18_attention_weight_max": 35.181793212890625,
      "activations/layer18_attention_weight_min": -25.553483963012695,
      "activations/layer19_attention_weight_max": 34.359073638916016,
      "activations/layer19_attention_weight_min": -31.2534236907959,
      "activations/layer1_attention_weight_max": 15.579360008239746,
      "activations/layer1_attention_weight_min": -14.123147964477539,
      "activations/layer20_attention_weight_max": 30.06634521484375,
      "activations/layer20_attention_weight_min": -25.71019744873047,
      "activations/layer21_attention_weight_max": 32.58790969848633,
      "activations/layer21_attention_weight_min": -28.312101364135742,
      "activations/layer22_attention_weight_max": 42.58811569213867,
      "activations/layer22_attention_weight_min": -31.18498992919922,
      "activations/layer23_attention_weight_max": 33.479034423828125,
      "activations/layer23_attention_weight_min": -27.233131408691406,
      "activations/layer2_attention_weight_max": 31.790592193603516,
      "activations/layer2_attention_weight_min": -31.527456283569336,
      "activations/layer3_attention_weight_max": 97.92784118652344,
      "activations/layer3_attention_weight_min": -102.28791046142578,
      "activations/layer4_attention_weight_max": 104.9189453125,
      "activations/layer4_attention_weight_min": -109.77201080322266,
      "activations/layer5_attention_weight_max": 52.66291046142578,
      "activations/layer5_attention_weight_min": -61.10555648803711,
      "activations/layer6_attention_weight_max": 42.93718719482422,
      "activations/layer6_attention_weight_min": -46.606929779052734,
      "activations/layer7_attention_weight_max": 89.59774780273438,
      "activations/layer7_attention_weight_min": -98.2180404663086,
      "activations/layer8_attention_weight_max": 40.04914474487305,
      "activations/layer8_attention_weight_min": -43.50551223754883,
      "activations/layer9_attention_weight_max": 32.67387771606445,
      "activations/layer9_attention_weight_min": -34.75445556640625,
      "epoch": 18.23,
      "learning_rate": 3.2745833333333334e-05,
      "loss": 2.7011,
      "step": 313800
    },
    {
      "activations/layer0_attention_weight_max": 16.316064834594727,
      "activations/layer0_attention_weight_min": -12.844428062438965,
      "activations/layer10_attention_weight_max": 31.29207420349121,
      "activations/layer10_attention_weight_min": -34.15239715576172,
      "activations/layer11_attention_weight_max": 32.621402740478516,
      "activations/layer11_attention_weight_min": -34.217124938964844,
      "activations/layer12_attention_weight_max": 28.556474685668945,
      "activations/layer12_attention_weight_min": -26.98336410522461,
      "activations/layer13_attention_weight_max": 40.97772216796875,
      "activations/layer13_attention_weight_min": -38.53112030029297,
      "activations/layer14_attention_weight_max": 49.441890716552734,
      "activations/layer14_attention_weight_min": -43.088890075683594,
      "activations/layer15_attention_weight_max": 40.99238204956055,
      "activations/layer15_attention_weight_min": -38.67221450805664,
      "activations/layer16_attention_weight_max": 30.300796508789062,
      "activations/layer16_attention_weight_min": -30.92053985595703,
      "activations/layer17_attention_weight_max": 31.985599517822266,
      "activations/layer17_attention_weight_min": -26.610523223876953,
      "activations/layer18_attention_weight_max": 32.31135177612305,
      "activations/layer18_attention_weight_min": -26.708166122436523,
      "activations/layer19_attention_weight_max": 34.400535583496094,
      "activations/layer19_attention_weight_min": -32.64370346069336,
      "activations/layer1_attention_weight_max": 15.598946571350098,
      "activations/layer1_attention_weight_min": -13.024033546447754,
      "activations/layer20_attention_weight_max": 32.204036712646484,
      "activations/layer20_attention_weight_min": -26.585973739624023,
      "activations/layer21_attention_weight_max": 30.696481704711914,
      "activations/layer21_attention_weight_min": -24.749521255493164,
      "activations/layer22_attention_weight_max": 44.41740417480469,
      "activations/layer22_attention_weight_min": -32.73954391479492,
      "activations/layer23_attention_weight_max": 35.66265106201172,
      "activations/layer23_attention_weight_min": -27.37331199645996,
      "activations/layer2_attention_weight_max": 30.297313690185547,
      "activations/layer2_attention_weight_min": -31.542739868164062,
      "activations/layer3_attention_weight_max": 96.38394165039062,
      "activations/layer3_attention_weight_min": -99.26551055908203,
      "activations/layer4_attention_weight_max": 108.66810607910156,
      "activations/layer4_attention_weight_min": -113.35770416259766,
      "activations/layer5_attention_weight_max": 52.036354064941406,
      "activations/layer5_attention_weight_min": -65.15796661376953,
      "activations/layer6_attention_weight_max": 43.66640853881836,
      "activations/layer6_attention_weight_min": -47.447044372558594,
      "activations/layer7_attention_weight_max": 93.03063201904297,
      "activations/layer7_attention_weight_min": -98.0353775024414,
      "activations/layer8_attention_weight_max": 39.32735061645508,
      "activations/layer8_attention_weight_min": -43.748538970947266,
      "activations/layer9_attention_weight_max": 30.48853874206543,
      "activations/layer9_attention_weight_min": -33.970088958740234,
      "epoch": 18.24,
      "learning_rate": 3.2726893939393936e-05,
      "loss": 2.7243,
      "step": 313850
    },
    {
      "activations/layer0_attention_weight_max": 18.111682891845703,
      "activations/layer0_attention_weight_min": -13.917977333068848,
      "activations/layer10_attention_weight_max": 37.8599739074707,
      "activations/layer10_attention_weight_min": -39.29298782348633,
      "activations/layer11_attention_weight_max": 38.236942291259766,
      "activations/layer11_attention_weight_min": -38.34353256225586,
      "activations/layer12_attention_weight_max": 28.986286163330078,
      "activations/layer12_attention_weight_min": -31.159584045410156,
      "activations/layer13_attention_weight_max": 42.06908416748047,
      "activations/layer13_attention_weight_min": -42.429412841796875,
      "activations/layer14_attention_weight_max": 46.9675407409668,
      "activations/layer14_attention_weight_min": -40.89100646972656,
      "activations/layer15_attention_weight_max": 42.631736755371094,
      "activations/layer15_attention_weight_min": -38.64586639404297,
      "activations/layer16_attention_weight_max": 30.689470291137695,
      "activations/layer16_attention_weight_min": -28.449254989624023,
      "activations/layer17_attention_weight_max": 33.37400817871094,
      "activations/layer17_attention_weight_min": -28.95159339904785,
      "activations/layer18_attention_weight_max": 35.084041595458984,
      "activations/layer18_attention_weight_min": -27.89410400390625,
      "activations/layer19_attention_weight_max": 39.23478698730469,
      "activations/layer19_attention_weight_min": -35.212581634521484,
      "activations/layer1_attention_weight_max": 16.100278854370117,
      "activations/layer1_attention_weight_min": -14.068720817565918,
      "activations/layer20_attention_weight_max": 37.28977966308594,
      "activations/layer20_attention_weight_min": -28.051069259643555,
      "activations/layer21_attention_weight_max": 36.35491943359375,
      "activations/layer21_attention_weight_min": -27.478160858154297,
      "activations/layer22_attention_weight_max": 54.75944900512695,
      "activations/layer22_attention_weight_min": -35.8752326965332,
      "activations/layer23_attention_weight_max": 37.75938415527344,
      "activations/layer23_attention_weight_min": -27.5048828125,
      "activations/layer2_attention_weight_max": 32.01764678955078,
      "activations/layer2_attention_weight_min": -32.10332107543945,
      "activations/layer3_attention_weight_max": 99.4717788696289,
      "activations/layer3_attention_weight_min": -103.0088119506836,
      "activations/layer4_attention_weight_max": 109.16987609863281,
      "activations/layer4_attention_weight_min": -109.76165771484375,
      "activations/layer5_attention_weight_max": 53.22074508666992,
      "activations/layer5_attention_weight_min": -61.5979118347168,
      "activations/layer6_attention_weight_max": 45.809974670410156,
      "activations/layer6_attention_weight_min": -47.9920539855957,
      "activations/layer7_attention_weight_max": 98.8604507446289,
      "activations/layer7_attention_weight_min": -109.30813598632812,
      "activations/layer8_attention_weight_max": 45.431556701660156,
      "activations/layer8_attention_weight_min": -49.084251403808594,
      "activations/layer9_attention_weight_max": 34.39894485473633,
      "activations/layer9_attention_weight_min": -36.82878875732422,
      "epoch": 18.24,
      "learning_rate": 3.2707954545454544e-05,
      "loss": 2.7306,
      "step": 313900
    },
    {
      "activations/layer0_attention_weight_max": 16.430070877075195,
      "activations/layer0_attention_weight_min": -12.495634078979492,
      "activations/layer10_attention_weight_max": 38.08221435546875,
      "activations/layer10_attention_weight_min": -36.044464111328125,
      "activations/layer11_attention_weight_max": 37.82759475708008,
      "activations/layer11_attention_weight_min": -37.9582405090332,
      "activations/layer12_attention_weight_max": 30.032636642456055,
      "activations/layer12_attention_weight_min": -26.92574691772461,
      "activations/layer13_attention_weight_max": 40.2741813659668,
      "activations/layer13_attention_weight_min": -37.249176025390625,
      "activations/layer14_attention_weight_max": 44.5528450012207,
      "activations/layer14_attention_weight_min": -40.508262634277344,
      "activations/layer15_attention_weight_max": 41.862815856933594,
      "activations/layer15_attention_weight_min": -37.00278091430664,
      "activations/layer16_attention_weight_max": 31.885404586791992,
      "activations/layer16_attention_weight_min": -29.659271240234375,
      "activations/layer17_attention_weight_max": 28.730939865112305,
      "activations/layer17_attention_weight_min": -25.033557891845703,
      "activations/layer18_attention_weight_max": 32.526851654052734,
      "activations/layer18_attention_weight_min": -27.10296630859375,
      "activations/layer19_attention_weight_max": 40.980098724365234,
      "activations/layer19_attention_weight_min": -32.033443450927734,
      "activations/layer1_attention_weight_max": 16.43447494506836,
      "activations/layer1_attention_weight_min": -14.33858871459961,
      "activations/layer20_attention_weight_max": 33.18723678588867,
      "activations/layer20_attention_weight_min": -25.22972297668457,
      "activations/layer21_attention_weight_max": 34.05188751220703,
      "activations/layer21_attention_weight_min": -25.993675231933594,
      "activations/layer22_attention_weight_max": 43.75651168823242,
      "activations/layer22_attention_weight_min": -32.943565368652344,
      "activations/layer23_attention_weight_max": 34.503475189208984,
      "activations/layer23_attention_weight_min": -25.593830108642578,
      "activations/layer2_attention_weight_max": 32.9451904296875,
      "activations/layer2_attention_weight_min": -31.497516632080078,
      "activations/layer3_attention_weight_max": 102.76908874511719,
      "activations/layer3_attention_weight_min": -100.60508728027344,
      "activations/layer4_attention_weight_max": 103.48697662353516,
      "activations/layer4_attention_weight_min": -103.20166015625,
      "activations/layer5_attention_weight_max": 49.235809326171875,
      "activations/layer5_attention_weight_min": -58.151710510253906,
      "activations/layer6_attention_weight_max": 42.61749267578125,
      "activations/layer6_attention_weight_min": -44.18783950805664,
      "activations/layer7_attention_weight_max": 98.34557342529297,
      "activations/layer7_attention_weight_min": -90.85326385498047,
      "activations/layer8_attention_weight_max": 41.9461555480957,
      "activations/layer8_attention_weight_min": -44.6721076965332,
      "activations/layer9_attention_weight_max": 34.29441452026367,
      "activations/layer9_attention_weight_min": -36.503936767578125,
      "epoch": 18.24,
      "learning_rate": 3.2689015151515145e-05,
      "loss": 2.7248,
      "step": 313950
    },
    {
      "activations/layer0_attention_weight_max": 15.573474884033203,
      "activations/layer0_attention_weight_min": -14.551786422729492,
      "activations/layer10_attention_weight_max": 33.095924377441406,
      "activations/layer10_attention_weight_min": -34.96744918823242,
      "activations/layer11_attention_weight_max": 33.446876525878906,
      "activations/layer11_attention_weight_min": -35.34025192260742,
      "activations/layer12_attention_weight_max": 27.22107696533203,
      "activations/layer12_attention_weight_min": -28.601612091064453,
      "activations/layer13_attention_weight_max": 40.269737243652344,
      "activations/layer13_attention_weight_min": -37.4076042175293,
      "activations/layer14_attention_weight_max": 45.24171829223633,
      "activations/layer14_attention_weight_min": -40.380462646484375,
      "activations/layer15_attention_weight_max": 41.077640533447266,
      "activations/layer15_attention_weight_min": -38.655548095703125,
      "activations/layer16_attention_weight_max": 30.22144317626953,
      "activations/layer16_attention_weight_min": -29.13651466369629,
      "activations/layer17_attention_weight_max": 29.861080169677734,
      "activations/layer17_attention_weight_min": -27.09864044189453,
      "activations/layer18_attention_weight_max": 34.697120666503906,
      "activations/layer18_attention_weight_min": -25.09347152709961,
      "activations/layer19_attention_weight_max": 32.953556060791016,
      "activations/layer19_attention_weight_min": -32.90387725830078,
      "activations/layer1_attention_weight_max": 16.00301742553711,
      "activations/layer1_attention_weight_min": -14.605021476745605,
      "activations/layer20_attention_weight_max": 29.866737365722656,
      "activations/layer20_attention_weight_min": -26.77406883239746,
      "activations/layer21_attention_weight_max": 30.28390884399414,
      "activations/layer21_attention_weight_min": -25.258432388305664,
      "activations/layer22_attention_weight_max": 45.27521514892578,
      "activations/layer22_attention_weight_min": -32.30610656738281,
      "activations/layer23_attention_weight_max": 35.41065216064453,
      "activations/layer23_attention_weight_min": -24.853958129882812,
      "activations/layer2_attention_weight_max": 32.20060729980469,
      "activations/layer2_attention_weight_min": -32.106624603271484,
      "activations/layer3_attention_weight_max": 102.25308227539062,
      "activations/layer3_attention_weight_min": -102.69026947021484,
      "activations/layer4_attention_weight_max": 110.540283203125,
      "activations/layer4_attention_weight_min": -112.0791244506836,
      "activations/layer5_attention_weight_max": 52.491146087646484,
      "activations/layer5_attention_weight_min": -65.117431640625,
      "activations/layer6_attention_weight_max": 45.0026741027832,
      "activations/layer6_attention_weight_min": -47.79133224487305,
      "activations/layer7_attention_weight_max": 94.14318084716797,
      "activations/layer7_attention_weight_min": -98.11161041259766,
      "activations/layer8_attention_weight_max": 40.93841552734375,
      "activations/layer8_attention_weight_min": -43.502662658691406,
      "activations/layer9_attention_weight_max": 33.63839340209961,
      "activations/layer9_attention_weight_min": -34.112457275390625,
      "epoch": 18.25,
      "learning_rate": 3.267045454545454e-05,
      "loss": 2.7218,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_loss": 2.6875,
      "eval_runtime": 8.5,
      "eval_samples_per_second": 505.176,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.5,
      "eval_openwebtext_samples_per_second": 505.176,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9842,
      "eval_wikitext_samples_per_second": 229.811,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_lambada_loss": 2.466796875,
      "eval_lambada_ppl": 11.784638661080912,
      "eval_lambada_runtime": 9.5969,
      "eval_lambada_samples_per_second": 507.352,
      "step": 314000
    },
    {
      "activations/layer0_attention_weight_max": 16.188861846923828,
      "activations/layer0_attention_weight_min": -12.184530258178711,
      "activations/layer10_attention_weight_max": 35.501216888427734,
      "activations/layer10_attention_weight_min": -35.57384490966797,
      "activations/layer11_attention_weight_max": 36.09138488769531,
      "activations/layer11_attention_weight_min": -35.651634216308594,
      "activations/layer12_attention_weight_max": 26.007461547851562,
      "activations/layer12_attention_weight_min": -27.01990509033203,
      "activations/layer13_attention_weight_max": 38.260406494140625,
      "activations/layer13_attention_weight_min": -34.871543884277344,
      "activations/layer14_attention_weight_max": 42.141014099121094,
      "activations/layer14_attention_weight_min": -38.27555847167969,
      "activations/layer15_attention_weight_max": 41.41730499267578,
      "activations/layer15_attention_weight_min": -37.57106018066406,
      "activations/layer16_attention_weight_max": 31.38134765625,
      "activations/layer16_attention_weight_min": -27.734970092773438,
      "activations/layer17_attention_weight_max": 30.60696029663086,
      "activations/layer17_attention_weight_min": -26.857458114624023,
      "activations/layer18_attention_weight_max": 29.777626037597656,
      "activations/layer18_attention_weight_min": -26.51351547241211,
      "activations/layer19_attention_weight_max": 33.07554244995117,
      "activations/layer19_attention_weight_min": -32.937015533447266,
      "activations/layer1_attention_weight_max": 17.137474060058594,
      "activations/layer1_attention_weight_min": -14.440725326538086,
      "activations/layer20_attention_weight_max": 28.795249938964844,
      "activations/layer20_attention_weight_min": -25.664348602294922,
      "activations/layer21_attention_weight_max": 26.840904235839844,
      "activations/layer21_attention_weight_min": -24.382108688354492,
      "activations/layer22_attention_weight_max": 42.75400924682617,
      "activations/layer22_attention_weight_min": -33.88609313964844,
      "activations/layer23_attention_weight_max": 31.317607879638672,
      "activations/layer23_attention_weight_min": -27.07845687866211,
      "activations/layer2_attention_weight_max": 33.443931579589844,
      "activations/layer2_attention_weight_min": -33.816429138183594,
      "activations/layer3_attention_weight_max": 99.54313659667969,
      "activations/layer3_attention_weight_min": -101.10204315185547,
      "activations/layer4_attention_weight_max": 106.71553039550781,
      "activations/layer4_attention_weight_min": -105.08098602294922,
      "activations/layer5_attention_weight_max": 50.32455062866211,
      "activations/layer5_attention_weight_min": -65.43771362304688,
      "activations/layer6_attention_weight_max": 43.0794792175293,
      "activations/layer6_attention_weight_min": -45.65808868408203,
      "activations/layer7_attention_weight_max": 95.61414337158203,
      "activations/layer7_attention_weight_min": -98.22562408447266,
      "activations/layer8_attention_weight_max": 44.30209732055664,
      "activations/layer8_attention_weight_min": -43.4017219543457,
      "activations/layer9_attention_weight_max": 32.374488830566406,
      "activations/layer9_attention_weight_min": -34.06121826171875,
      "epoch": 18.25,
      "learning_rate": 3.2651515151515145e-05,
      "loss": 2.7226,
      "step": 314050
    },
    {
      "activations/layer0_attention_weight_max": 16.18012809753418,
      "activations/layer0_attention_weight_min": -12.78695297241211,
      "activations/layer10_attention_weight_max": 33.5618782043457,
      "activations/layer10_attention_weight_min": -34.7850227355957,
      "activations/layer11_attention_weight_max": 33.89706802368164,
      "activations/layer11_attention_weight_min": -35.72062683105469,
      "activations/layer12_attention_weight_max": 23.909561157226562,
      "activations/layer12_attention_weight_min": -26.77947235107422,
      "activations/layer13_attention_weight_max": 35.383331298828125,
      "activations/layer13_attention_weight_min": -36.31525802612305,
      "activations/layer14_attention_weight_max": 38.618038177490234,
      "activations/layer14_attention_weight_min": -37.7082405090332,
      "activations/layer15_attention_weight_max": 34.91427993774414,
      "activations/layer15_attention_weight_min": -36.457847595214844,
      "activations/layer16_attention_weight_max": 26.15435218811035,
      "activations/layer16_attention_weight_min": -27.683868408203125,
      "activations/layer17_attention_weight_max": 29.51331329345703,
      "activations/layer17_attention_weight_min": -26.28575325012207,
      "activations/layer18_attention_weight_max": 29.678062438964844,
      "activations/layer18_attention_weight_min": -25.582000732421875,
      "activations/layer19_attention_weight_max": 29.2225399017334,
      "activations/layer19_attention_weight_min": -30.5003719329834,
      "activations/layer1_attention_weight_max": 15.559525489807129,
      "activations/layer1_attention_weight_min": -13.116178512573242,
      "activations/layer20_attention_weight_max": 26.26397132873535,
      "activations/layer20_attention_weight_min": -25.480491638183594,
      "activations/layer21_attention_weight_max": 25.440013885498047,
      "activations/layer21_attention_weight_min": -25.11442756652832,
      "activations/layer22_attention_weight_max": 36.91798400878906,
      "activations/layer22_attention_weight_min": -30.44491958618164,
      "activations/layer23_attention_weight_max": 29.182395935058594,
      "activations/layer23_attention_weight_min": -26.12799644470215,
      "activations/layer2_attention_weight_max": 30.6571044921875,
      "activations/layer2_attention_weight_min": -32.07456588745117,
      "activations/layer3_attention_weight_max": 97.74755096435547,
      "activations/layer3_attention_weight_min": -99.05821228027344,
      "activations/layer4_attention_weight_max": 105.51908111572266,
      "activations/layer4_attention_weight_min": -107.97603607177734,
      "activations/layer5_attention_weight_max": 48.7261962890625,
      "activations/layer5_attention_weight_min": -58.79412841796875,
      "activations/layer6_attention_weight_max": 41.90871047973633,
      "activations/layer6_attention_weight_min": -44.92671203613281,
      "activations/layer7_attention_weight_max": 87.5969009399414,
      "activations/layer7_attention_weight_min": -95.47246551513672,
      "activations/layer8_attention_weight_max": 40.961360931396484,
      "activations/layer8_attention_weight_min": -45.50936508178711,
      "activations/layer9_attention_weight_max": 30.650836944580078,
      "activations/layer9_attention_weight_min": -34.58249282836914,
      "epoch": 18.25,
      "learning_rate": 3.263295454545454e-05,
      "loss": 2.7288,
      "step": 314100
    },
    {
      "activations/layer0_attention_weight_max": 17.9479923248291,
      "activations/layer0_attention_weight_min": -12.490105628967285,
      "activations/layer10_attention_weight_max": 32.77840042114258,
      "activations/layer10_attention_weight_min": -33.39918518066406,
      "activations/layer11_attention_weight_max": 33.847923278808594,
      "activations/layer11_attention_weight_min": -36.1234016418457,
      "activations/layer12_attention_weight_max": 27.827116012573242,
      "activations/layer12_attention_weight_min": -26.429197311401367,
      "activations/layer13_attention_weight_max": 37.9184455871582,
      "activations/layer13_attention_weight_min": -35.56663131713867,
      "activations/layer14_attention_weight_max": 39.368038177490234,
      "activations/layer14_attention_weight_min": -35.90471649169922,
      "activations/layer15_attention_weight_max": 35.959190368652344,
      "activations/layer15_attention_weight_min": -35.21382522583008,
      "activations/layer16_attention_weight_max": 27.98157501220703,
      "activations/layer16_attention_weight_min": -27.0521183013916,
      "activations/layer17_attention_weight_max": 28.52109146118164,
      "activations/layer17_attention_weight_min": -26.165639877319336,
      "activations/layer18_attention_weight_max": 30.428695678710938,
      "activations/layer18_attention_weight_min": -28.490819931030273,
      "activations/layer19_attention_weight_max": 33.079917907714844,
      "activations/layer19_attention_weight_min": -29.656021118164062,
      "activations/layer1_attention_weight_max": 16.32161521911621,
      "activations/layer1_attention_weight_min": -13.524073600769043,
      "activations/layer20_attention_weight_max": 27.443553924560547,
      "activations/layer20_attention_weight_min": -26.211679458618164,
      "activations/layer21_attention_weight_max": 27.577451705932617,
      "activations/layer21_attention_weight_min": -23.902130126953125,
      "activations/layer22_attention_weight_max": 39.238609313964844,
      "activations/layer22_attention_weight_min": -30.56232261657715,
      "activations/layer23_attention_weight_max": 30.58636474609375,
      "activations/layer23_attention_weight_min": -26.429906845092773,
      "activations/layer2_attention_weight_max": 30.7032527923584,
      "activations/layer2_attention_weight_min": -31.20650863647461,
      "activations/layer3_attention_weight_max": 93.17603302001953,
      "activations/layer3_attention_weight_min": -96.33192443847656,
      "activations/layer4_attention_weight_max": 101.59920501708984,
      "activations/layer4_attention_weight_min": -103.38082122802734,
      "activations/layer5_attention_weight_max": 50.19845199584961,
      "activations/layer5_attention_weight_min": -59.558082580566406,
      "activations/layer6_attention_weight_max": 42.86756896972656,
      "activations/layer6_attention_weight_min": -45.35409164428711,
      "activations/layer7_attention_weight_max": 94.2871322631836,
      "activations/layer7_attention_weight_min": -95.19383239746094,
      "activations/layer8_attention_weight_max": 40.80122756958008,
      "activations/layer8_attention_weight_min": -42.6118049621582,
      "activations/layer9_attention_weight_max": 30.92055320739746,
      "activations/layer9_attention_weight_min": -33.337337493896484,
      "epoch": 18.25,
      "learning_rate": 3.261401515151515e-05,
      "loss": 2.7277,
      "step": 314150
    },
    {
      "activations/layer0_attention_weight_max": 16.95463752746582,
      "activations/layer0_attention_weight_min": -13.455018997192383,
      "activations/layer10_attention_weight_max": 35.56103515625,
      "activations/layer10_attention_weight_min": -34.516727447509766,
      "activations/layer11_attention_weight_max": 35.74832534790039,
      "activations/layer11_attention_weight_min": -37.14445877075195,
      "activations/layer12_attention_weight_max": 27.467243194580078,
      "activations/layer12_attention_weight_min": -27.67180824279785,
      "activations/layer13_attention_weight_max": 40.67159652709961,
      "activations/layer13_attention_weight_min": -38.491851806640625,
      "activations/layer14_attention_weight_max": 42.46112060546875,
      "activations/layer14_attention_weight_min": -38.523040771484375,
      "activations/layer15_attention_weight_max": 37.83952331542969,
      "activations/layer15_attention_weight_min": -36.53877639770508,
      "activations/layer16_attention_weight_max": 28.981725692749023,
      "activations/layer16_attention_weight_min": -28.615070343017578,
      "activations/layer17_attention_weight_max": 29.652978897094727,
      "activations/layer17_attention_weight_min": -27.16367530822754,
      "activations/layer18_attention_weight_max": 30.086278915405273,
      "activations/layer18_attention_weight_min": -28.917015075683594,
      "activations/layer19_attention_weight_max": 32.48530197143555,
      "activations/layer19_attention_weight_min": -35.82310485839844,
      "activations/layer1_attention_weight_max": 16.33702850341797,
      "activations/layer1_attention_weight_min": -13.922353744506836,
      "activations/layer20_attention_weight_max": 29.584123611450195,
      "activations/layer20_attention_weight_min": -27.53875732421875,
      "activations/layer21_attention_weight_max": 30.436946868896484,
      "activations/layer21_attention_weight_min": -25.849924087524414,
      "activations/layer22_attention_weight_max": 45.73388671875,
      "activations/layer22_attention_weight_min": -32.786197662353516,
      "activations/layer23_attention_weight_max": 30.081218719482422,
      "activations/layer23_attention_weight_min": -25.454788208007812,
      "activations/layer2_attention_weight_max": 32.06431198120117,
      "activations/layer2_attention_weight_min": -33.389461517333984,
      "activations/layer3_attention_weight_max": 95.41512298583984,
      "activations/layer3_attention_weight_min": -100.73755645751953,
      "activations/layer4_attention_weight_max": 104.44493865966797,
      "activations/layer4_attention_weight_min": -106.06725311279297,
      "activations/layer5_attention_weight_max": 50.45056915283203,
      "activations/layer5_attention_weight_min": -61.663509368896484,
      "activations/layer6_attention_weight_max": 44.07004165649414,
      "activations/layer6_attention_weight_min": -48.474796295166016,
      "activations/layer7_attention_weight_max": 92.97539520263672,
      "activations/layer7_attention_weight_min": -97.4781265258789,
      "activations/layer8_attention_weight_max": 44.360313415527344,
      "activations/layer8_attention_weight_min": -43.89579391479492,
      "activations/layer9_attention_weight_max": 36.05076599121094,
      "activations/layer9_attention_weight_min": -35.11196517944336,
      "epoch": 18.26,
      "learning_rate": 3.259507575757575e-05,
      "loss": 2.7235,
      "step": 314200
    },
    {
      "activations/layer0_attention_weight_max": 16.117084503173828,
      "activations/layer0_attention_weight_min": -13.474417686462402,
      "activations/layer10_attention_weight_max": 33.930999755859375,
      "activations/layer10_attention_weight_min": -36.08216094970703,
      "activations/layer11_attention_weight_max": 33.239715576171875,
      "activations/layer11_attention_weight_min": -35.958553314208984,
      "activations/layer12_attention_weight_max": 29.241268157958984,
      "activations/layer12_attention_weight_min": -30.797510147094727,
      "activations/layer13_attention_weight_max": 43.6404914855957,
      "activations/layer13_attention_weight_min": -38.764686584472656,
      "activations/layer14_attention_weight_max": 46.01416778564453,
      "activations/layer14_attention_weight_min": -38.565147399902344,
      "activations/layer15_attention_weight_max": 42.03978729248047,
      "activations/layer15_attention_weight_min": -37.919803619384766,
      "activations/layer16_attention_weight_max": 28.555953979492188,
      "activations/layer16_attention_weight_min": -28.845294952392578,
      "activations/layer17_attention_weight_max": 28.751953125,
      "activations/layer17_attention_weight_min": -25.29228973388672,
      "activations/layer18_attention_weight_max": 31.28167724609375,
      "activations/layer18_attention_weight_min": -27.82842445373535,
      "activations/layer19_attention_weight_max": 31.361835479736328,
      "activations/layer19_attention_weight_min": -30.305374145507812,
      "activations/layer1_attention_weight_max": 16.3658504486084,
      "activations/layer1_attention_weight_min": -13.90270709991455,
      "activations/layer20_attention_weight_max": 27.65764045715332,
      "activations/layer20_attention_weight_min": -27.141183853149414,
      "activations/layer21_attention_weight_max": 27.515300750732422,
      "activations/layer21_attention_weight_min": -25.428878784179688,
      "activations/layer22_attention_weight_max": 38.315467834472656,
      "activations/layer22_attention_weight_min": -32.07718276977539,
      "activations/layer23_attention_weight_max": 30.87482452392578,
      "activations/layer23_attention_weight_min": -26.24562644958496,
      "activations/layer2_attention_weight_max": 33.009605407714844,
      "activations/layer2_attention_weight_min": -34.3665657043457,
      "activations/layer3_attention_weight_max": 96.9571762084961,
      "activations/layer3_attention_weight_min": -101.1627197265625,
      "activations/layer4_attention_weight_max": 103.75799560546875,
      "activations/layer4_attention_weight_min": -110.9549331665039,
      "activations/layer5_attention_weight_max": 49.763397216796875,
      "activations/layer5_attention_weight_min": -59.189842224121094,
      "activations/layer6_attention_weight_max": 43.61368942260742,
      "activations/layer6_attention_weight_min": -46.08884811401367,
      "activations/layer7_attention_weight_max": 89.2470474243164,
      "activations/layer7_attention_weight_min": -98.98340606689453,
      "activations/layer8_attention_weight_max": 40.72909164428711,
      "activations/layer8_attention_weight_min": -44.2453727722168,
      "activations/layer9_attention_weight_max": 32.679012298583984,
      "activations/layer9_attention_weight_min": -33.84663391113281,
      "epoch": 18.26,
      "learning_rate": 3.257613636363636e-05,
      "loss": 2.7123,
      "step": 314250
    },
    {
      "activations/layer0_attention_weight_max": 16.31249237060547,
      "activations/layer0_attention_weight_min": -12.958028793334961,
      "activations/layer10_attention_weight_max": 35.30752944946289,
      "activations/layer10_attention_weight_min": -34.76124572753906,
      "activations/layer11_attention_weight_max": 37.02760314941406,
      "activations/layer11_attention_weight_min": -34.965492248535156,
      "activations/layer12_attention_weight_max": 26.761741638183594,
      "activations/layer12_attention_weight_min": -26.12650489807129,
      "activations/layer13_attention_weight_max": 38.64039611816406,
      "activations/layer13_attention_weight_min": -35.685089111328125,
      "activations/layer14_attention_weight_max": 41.44222640991211,
      "activations/layer14_attention_weight_min": -36.62803268432617,
      "activations/layer15_attention_weight_max": 40.85269546508789,
      "activations/layer15_attention_weight_min": -35.67963790893555,
      "activations/layer16_attention_weight_max": 30.084699630737305,
      "activations/layer16_attention_weight_min": -26.81363296508789,
      "activations/layer17_attention_weight_max": 28.81292724609375,
      "activations/layer17_attention_weight_min": -26.086008071899414,
      "activations/layer18_attention_weight_max": 30.068880081176758,
      "activations/layer18_attention_weight_min": -25.635892868041992,
      "activations/layer19_attention_weight_max": 35.245704650878906,
      "activations/layer19_attention_weight_min": -31.735891342163086,
      "activations/layer1_attention_weight_max": 16.47315788269043,
      "activations/layer1_attention_weight_min": -14.784175872802734,
      "activations/layer20_attention_weight_max": 30.19869613647461,
      "activations/layer20_attention_weight_min": -25.908178329467773,
      "activations/layer21_attention_weight_max": 28.492191314697266,
      "activations/layer21_attention_weight_min": -24.93033790588379,
      "activations/layer22_attention_weight_max": 39.875572204589844,
      "activations/layer22_attention_weight_min": -30.069124221801758,
      "activations/layer23_attention_weight_max": 30.525772094726562,
      "activations/layer23_attention_weight_min": -25.885982513427734,
      "activations/layer2_attention_weight_max": 34.72419738769531,
      "activations/layer2_attention_weight_min": -34.38715362548828,
      "activations/layer3_attention_weight_max": 103.2563247680664,
      "activations/layer3_attention_weight_min": -105.82957458496094,
      "activations/layer4_attention_weight_max": 110.20320129394531,
      "activations/layer4_attention_weight_min": -115.33690643310547,
      "activations/layer5_attention_weight_max": 53.641929626464844,
      "activations/layer5_attention_weight_min": -62.097625732421875,
      "activations/layer6_attention_weight_max": 46.45112609863281,
      "activations/layer6_attention_weight_min": -49.07413864135742,
      "activations/layer7_attention_weight_max": 99.83163452148438,
      "activations/layer7_attention_weight_min": -101.25890350341797,
      "activations/layer8_attention_weight_max": 43.34095764160156,
      "activations/layer8_attention_weight_min": -46.230892181396484,
      "activations/layer9_attention_weight_max": 32.99528503417969,
      "activations/layer9_attention_weight_min": -35.88125228881836,
      "epoch": 18.26,
      "learning_rate": 3.255719696969697e-05,
      "loss": 2.7231,
      "step": 314300
    },
    {
      "activations/layer0_attention_weight_max": 16.986040115356445,
      "activations/layer0_attention_weight_min": -12.817140579223633,
      "activations/layer10_attention_weight_max": 30.987022399902344,
      "activations/layer10_attention_weight_min": -32.08028030395508,
      "activations/layer11_attention_weight_max": 32.98168182373047,
      "activations/layer11_attention_weight_min": -33.05126190185547,
      "activations/layer12_attention_weight_max": 28.397375106811523,
      "activations/layer12_attention_weight_min": -25.65648078918457,
      "activations/layer13_attention_weight_max": 40.376670837402344,
      "activations/layer13_attention_weight_min": -34.774017333984375,
      "activations/layer14_attention_weight_max": 43.73218536376953,
      "activations/layer14_attention_weight_min": -36.18134689331055,
      "activations/layer15_attention_weight_max": 41.41899871826172,
      "activations/layer15_attention_weight_min": -34.44196701049805,
      "activations/layer16_attention_weight_max": 29.557186126708984,
      "activations/layer16_attention_weight_min": -26.848709106445312,
      "activations/layer17_attention_weight_max": 29.74150276184082,
      "activations/layer17_attention_weight_min": -26.030872344970703,
      "activations/layer18_attention_weight_max": 30.266380310058594,
      "activations/layer18_attention_weight_min": -25.584827423095703,
      "activations/layer19_attention_weight_max": 38.83449935913086,
      "activations/layer19_attention_weight_min": -36.473758697509766,
      "activations/layer1_attention_weight_max": 15.715644836425781,
      "activations/layer1_attention_weight_min": -13.624361991882324,
      "activations/layer20_attention_weight_max": 26.899290084838867,
      "activations/layer20_attention_weight_min": -27.84249496459961,
      "activations/layer21_attention_weight_max": 27.191513061523438,
      "activations/layer21_attention_weight_min": -27.709495544433594,
      "activations/layer22_attention_weight_max": 39.702266693115234,
      "activations/layer22_attention_weight_min": -34.79574966430664,
      "activations/layer23_attention_weight_max": 32.28875732421875,
      "activations/layer23_attention_weight_min": -28.58074188232422,
      "activations/layer2_attention_weight_max": 31.337379455566406,
      "activations/layer2_attention_weight_min": -30.828853607177734,
      "activations/layer3_attention_weight_max": 94.53768157958984,
      "activations/layer3_attention_weight_min": -96.26506805419922,
      "activations/layer4_attention_weight_max": 100.25300598144531,
      "activations/layer4_attention_weight_min": -106.86174774169922,
      "activations/layer5_attention_weight_max": 49.3684196472168,
      "activations/layer5_attention_weight_min": -64.22747802734375,
      "activations/layer6_attention_weight_max": 42.48997497558594,
      "activations/layer6_attention_weight_min": -43.90154266357422,
      "activations/layer7_attention_weight_max": 87.60789489746094,
      "activations/layer7_attention_weight_min": -87.19513702392578,
      "activations/layer8_attention_weight_max": 37.490257263183594,
      "activations/layer8_attention_weight_min": -40.42683792114258,
      "activations/layer9_attention_weight_max": 29.90494728088379,
      "activations/layer9_attention_weight_min": -31.821279525756836,
      "epoch": 18.27,
      "learning_rate": 3.253825757575758e-05,
      "loss": 2.7216,
      "step": 314350
    },
    {
      "activations/layer0_attention_weight_max": 16.407468795776367,
      "activations/layer0_attention_weight_min": -13.685548782348633,
      "activations/layer10_attention_weight_max": 35.716835021972656,
      "activations/layer10_attention_weight_min": -34.1307258605957,
      "activations/layer11_attention_weight_max": 35.40060806274414,
      "activations/layer11_attention_weight_min": -35.30361557006836,
      "activations/layer12_attention_weight_max": 28.015153884887695,
      "activations/layer12_attention_weight_min": -26.714982986450195,
      "activations/layer13_attention_weight_max": 39.999183654785156,
      "activations/layer13_attention_weight_min": -35.980438232421875,
      "activations/layer14_attention_weight_max": 41.27943420410156,
      "activations/layer14_attention_weight_min": -36.729042053222656,
      "activations/layer15_attention_weight_max": 40.20780563354492,
      "activations/layer15_attention_weight_min": -34.968013763427734,
      "activations/layer16_attention_weight_max": 30.31242561340332,
      "activations/layer16_attention_weight_min": -26.57992172241211,
      "activations/layer17_attention_weight_max": 30.39034080505371,
      "activations/layer17_attention_weight_min": -25.02288246154785,
      "activations/layer18_attention_weight_max": 35.26817321777344,
      "activations/layer18_attention_weight_min": -24.36196517944336,
      "activations/layer19_attention_weight_max": 34.12343215942383,
      "activations/layer19_attention_weight_min": -29.691268920898438,
      "activations/layer1_attention_weight_max": 16.00033950805664,
      "activations/layer1_attention_weight_min": -14.529952049255371,
      "activations/layer20_attention_weight_max": 29.30736541748047,
      "activations/layer20_attention_weight_min": -25.44249153137207,
      "activations/layer21_attention_weight_max": 28.99538803100586,
      "activations/layer21_attention_weight_min": -24.96957778930664,
      "activations/layer22_attention_weight_max": 42.815093994140625,
      "activations/layer22_attention_weight_min": -28.76088523864746,
      "activations/layer23_attention_weight_max": 30.518272399902344,
      "activations/layer23_attention_weight_min": -26.419391632080078,
      "activations/layer2_attention_weight_max": 31.57288932800293,
      "activations/layer2_attention_weight_min": -32.04021072387695,
      "activations/layer3_attention_weight_max": 99.49732208251953,
      "activations/layer3_attention_weight_min": -100.21005249023438,
      "activations/layer4_attention_weight_max": 105.56526947021484,
      "activations/layer4_attention_weight_min": -104.66991424560547,
      "activations/layer5_attention_weight_max": 49.66181182861328,
      "activations/layer5_attention_weight_min": -61.64585494995117,
      "activations/layer6_attention_weight_max": 43.04301452636719,
      "activations/layer6_attention_weight_min": -44.54906463623047,
      "activations/layer7_attention_weight_max": 93.5951156616211,
      "activations/layer7_attention_weight_min": -96.06945037841797,
      "activations/layer8_attention_weight_max": 43.31380844116211,
      "activations/layer8_attention_weight_min": -43.69778060913086,
      "activations/layer9_attention_weight_max": 33.70677185058594,
      "activations/layer9_attention_weight_min": -35.657047271728516,
      "epoch": 18.27,
      "learning_rate": 3.251931818181818e-05,
      "loss": 2.7381,
      "step": 314400
    },
    {
      "activations/layer0_attention_weight_max": 16.488567352294922,
      "activations/layer0_attention_weight_min": -13.4628267288208,
      "activations/layer10_attention_weight_max": 34.15089797973633,
      "activations/layer10_attention_weight_min": -36.361846923828125,
      "activations/layer11_attention_weight_max": 35.01696014404297,
      "activations/layer11_attention_weight_min": -34.04838943481445,
      "activations/layer12_attention_weight_max": 28.452787399291992,
      "activations/layer12_attention_weight_min": -28.110286712646484,
      "activations/layer13_attention_weight_max": 43.96544647216797,
      "activations/layer13_attention_weight_min": -37.43149948120117,
      "activations/layer14_attention_weight_max": 49.868804931640625,
      "activations/layer14_attention_weight_min": -42.889366149902344,
      "activations/layer15_attention_weight_max": 42.673004150390625,
      "activations/layer15_attention_weight_min": -37.01148223876953,
      "activations/layer16_attention_weight_max": 32.781063079833984,
      "activations/layer16_attention_weight_min": -27.841779708862305,
      "activations/layer17_attention_weight_max": 34.85567855834961,
      "activations/layer17_attention_weight_min": -27.39942169189453,
      "activations/layer18_attention_weight_max": 33.80428695678711,
      "activations/layer18_attention_weight_min": -27.812362670898438,
      "activations/layer19_attention_weight_max": 37.603702545166016,
      "activations/layer19_attention_weight_min": -34.69705581665039,
      "activations/layer1_attention_weight_max": 15.822566032409668,
      "activations/layer1_attention_weight_min": -14.986659049987793,
      "activations/layer20_attention_weight_max": 31.140483856201172,
      "activations/layer20_attention_weight_min": -26.60503387451172,
      "activations/layer21_attention_weight_max": 31.718700408935547,
      "activations/layer21_attention_weight_min": -27.356847763061523,
      "activations/layer22_attention_weight_max": 47.65290832519531,
      "activations/layer22_attention_weight_min": -36.13450241088867,
      "activations/layer23_attention_weight_max": 35.6898078918457,
      "activations/layer23_attention_weight_min": -27.821319580078125,
      "activations/layer2_attention_weight_max": 31.881391525268555,
      "activations/layer2_attention_weight_min": -32.581443786621094,
      "activations/layer3_attention_weight_max": 99.06761932373047,
      "activations/layer3_attention_weight_min": -99.9655990600586,
      "activations/layer4_attention_weight_max": 105.39470672607422,
      "activations/layer4_attention_weight_min": -110.96697235107422,
      "activations/layer5_attention_weight_max": 54.00824737548828,
      "activations/layer5_attention_weight_min": -59.40660095214844,
      "activations/layer6_attention_weight_max": 44.67085266113281,
      "activations/layer6_attention_weight_min": -46.38814163208008,
      "activations/layer7_attention_weight_max": 92.75946807861328,
      "activations/layer7_attention_weight_min": -93.89610290527344,
      "activations/layer8_attention_weight_max": 42.52016830444336,
      "activations/layer8_attention_weight_min": -45.086421966552734,
      "activations/layer9_attention_weight_max": 32.65208053588867,
      "activations/layer9_attention_weight_min": -34.7367057800293,
      "epoch": 18.27,
      "learning_rate": 3.2500378787878786e-05,
      "loss": 2.7231,
      "step": 314450
    },
    {
      "activations/layer0_attention_weight_max": 16.076013565063477,
      "activations/layer0_attention_weight_min": -13.572681427001953,
      "activations/layer10_attention_weight_max": 35.279335021972656,
      "activations/layer10_attention_weight_min": -35.888671875,
      "activations/layer11_attention_weight_max": 35.177513122558594,
      "activations/layer11_attention_weight_min": -34.79660415649414,
      "activations/layer12_attention_weight_max": 25.677595138549805,
      "activations/layer12_attention_weight_min": -26.19636344909668,
      "activations/layer13_attention_weight_max": 37.92378234863281,
      "activations/layer13_attention_weight_min": -36.67122268676758,
      "activations/layer14_attention_weight_max": 40.9833869934082,
      "activations/layer14_attention_weight_min": -35.95102310180664,
      "activations/layer15_attention_weight_max": 37.67424774169922,
      "activations/layer15_attention_weight_min": -34.45497131347656,
      "activations/layer16_attention_weight_max": 29.67692756652832,
      "activations/layer16_attention_weight_min": -27.769954681396484,
      "activations/layer17_attention_weight_max": 30.07703399658203,
      "activations/layer17_attention_weight_min": -26.782033920288086,
      "activations/layer18_attention_weight_max": 31.94963264465332,
      "activations/layer18_attention_weight_min": -25.66309356689453,
      "activations/layer19_attention_weight_max": 34.5023307800293,
      "activations/layer19_attention_weight_min": -31.223468780517578,
      "activations/layer1_attention_weight_max": 16.402151107788086,
      "activations/layer1_attention_weight_min": -14.958779335021973,
      "activations/layer20_attention_weight_max": 29.498367309570312,
      "activations/layer20_attention_weight_min": -24.50265121459961,
      "activations/layer21_attention_weight_max": 28.801448822021484,
      "activations/layer21_attention_weight_min": -24.619102478027344,
      "activations/layer22_attention_weight_max": 42.920692443847656,
      "activations/layer22_attention_weight_min": -30.90302276611328,
      "activations/layer23_attention_weight_max": 31.49138641357422,
      "activations/layer23_attention_weight_min": -25.670774459838867,
      "activations/layer2_attention_weight_max": 31.70236587524414,
      "activations/layer2_attention_weight_min": -32.54230880737305,
      "activations/layer3_attention_weight_max": 99.64114379882812,
      "activations/layer3_attention_weight_min": -99.24676513671875,
      "activations/layer4_attention_weight_max": 102.71932220458984,
      "activations/layer4_attention_weight_min": -108.23280334472656,
      "activations/layer5_attention_weight_max": 47.95574188232422,
      "activations/layer5_attention_weight_min": -63.238563537597656,
      "activations/layer6_attention_weight_max": 44.03835678100586,
      "activations/layer6_attention_weight_min": -46.6147346496582,
      "activations/layer7_attention_weight_max": 88.85153198242188,
      "activations/layer7_attention_weight_min": -99.02507019042969,
      "activations/layer8_attention_weight_max": 41.04692840576172,
      "activations/layer8_attention_weight_min": -43.5990104675293,
      "activations/layer9_attention_weight_max": 33.15721893310547,
      "activations/layer9_attention_weight_min": -34.17157745361328,
      "epoch": 18.27,
      "learning_rate": 3.248143939393939e-05,
      "loss": 2.7412,
      "step": 314500
    },
    {
      "activations/layer0_attention_weight_max": 16.315549850463867,
      "activations/layer0_attention_weight_min": -13.17873764038086,
      "activations/layer10_attention_weight_max": 37.42539978027344,
      "activations/layer10_attention_weight_min": -39.591800689697266,
      "activations/layer11_attention_weight_max": 34.719261169433594,
      "activations/layer11_attention_weight_min": -35.121456146240234,
      "activations/layer12_attention_weight_max": 25.713300704956055,
      "activations/layer12_attention_weight_min": -25.762104034423828,
      "activations/layer13_attention_weight_max": 35.94647216796875,
      "activations/layer13_attention_weight_min": -35.677978515625,
      "activations/layer14_attention_weight_max": 39.79069900512695,
      "activations/layer14_attention_weight_min": -36.91104507446289,
      "activations/layer15_attention_weight_max": 35.14883041381836,
      "activations/layer15_attention_weight_min": -35.782981872558594,
      "activations/layer16_attention_weight_max": 27.306894302368164,
      "activations/layer16_attention_weight_min": -27.988258361816406,
      "activations/layer17_attention_weight_max": 27.45130157470703,
      "activations/layer17_attention_weight_min": -25.565677642822266,
      "activations/layer18_attention_weight_max": 32.50395965576172,
      "activations/layer18_attention_weight_min": -26.480640411376953,
      "activations/layer19_attention_weight_max": 31.157743453979492,
      "activations/layer19_attention_weight_min": -33.01394271850586,
      "activations/layer1_attention_weight_max": 15.917187690734863,
      "activations/layer1_attention_weight_min": -16.866004943847656,
      "activations/layer20_attention_weight_max": 24.735008239746094,
      "activations/layer20_attention_weight_min": -25.507211685180664,
      "activations/layer21_attention_weight_max": 25.1180362701416,
      "activations/layer21_attention_weight_min": -25.312759399414062,
      "activations/layer22_attention_weight_max": 40.843162536621094,
      "activations/layer22_attention_weight_min": -32.321407318115234,
      "activations/layer23_attention_weight_max": 33.95799255371094,
      "activations/layer23_attention_weight_min": -27.25177574157715,
      "activations/layer2_attention_weight_max": 31.167461395263672,
      "activations/layer2_attention_weight_min": -31.971675872802734,
      "activations/layer3_attention_weight_max": 96.6052017211914,
      "activations/layer3_attention_weight_min": -99.0501708984375,
      "activations/layer4_attention_weight_max": 103.12801361083984,
      "activations/layer4_attention_weight_min": -106.17919921875,
      "activations/layer5_attention_weight_max": 49.279476165771484,
      "activations/layer5_attention_weight_min": -59.07460021972656,
      "activations/layer6_attention_weight_max": 43.280479431152344,
      "activations/layer6_attention_weight_min": -44.96881866455078,
      "activations/layer7_attention_weight_max": 89.32369995117188,
      "activations/layer7_attention_weight_min": -96.44903564453125,
      "activations/layer8_attention_weight_max": 42.19987487792969,
      "activations/layer8_attention_weight_min": -46.56666946411133,
      "activations/layer9_attention_weight_max": 32.79330062866211,
      "activations/layer9_attention_weight_min": -36.84227752685547,
      "epoch": 18.28,
      "learning_rate": 3.2462499999999996e-05,
      "loss": 2.7428,
      "step": 314550
    },
    {
      "activations/layer0_attention_weight_max": 15.748059272766113,
      "activations/layer0_attention_weight_min": -12.089064598083496,
      "activations/layer10_attention_weight_max": 36.27193832397461,
      "activations/layer10_attention_weight_min": -36.566566467285156,
      "activations/layer11_attention_weight_max": 37.07204055786133,
      "activations/layer11_attention_weight_min": -35.814476013183594,
      "activations/layer12_attention_weight_max": 28.485721588134766,
      "activations/layer12_attention_weight_min": -27.18450164794922,
      "activations/layer13_attention_weight_max": 43.02030944824219,
      "activations/layer13_attention_weight_min": -37.433860778808594,
      "activations/layer14_attention_weight_max": 43.989078521728516,
      "activations/layer14_attention_weight_min": -39.973270416259766,
      "activations/layer15_attention_weight_max": 41.71701431274414,
      "activations/layer15_attention_weight_min": -38.53483200073242,
      "activations/layer16_attention_weight_max": 28.83536148071289,
      "activations/layer16_attention_weight_min": -26.984434127807617,
      "activations/layer17_attention_weight_max": 30.713491439819336,
      "activations/layer17_attention_weight_min": -25.029003143310547,
      "activations/layer18_attention_weight_max": 31.328022003173828,
      "activations/layer18_attention_weight_min": -23.890398025512695,
      "activations/layer19_attention_weight_max": 34.71860122680664,
      "activations/layer19_attention_weight_min": -31.976621627807617,
      "activations/layer1_attention_weight_max": 16.945056915283203,
      "activations/layer1_attention_weight_min": -15.120942115783691,
      "activations/layer20_attention_weight_max": 28.03325843811035,
      "activations/layer20_attention_weight_min": -25.274873733520508,
      "activations/layer21_attention_weight_max": 29.83361053466797,
      "activations/layer21_attention_weight_min": -25.09926414489746,
      "activations/layer22_attention_weight_max": 39.60750961303711,
      "activations/layer22_attention_weight_min": -33.49793243408203,
      "activations/layer23_attention_weight_max": 32.161590576171875,
      "activations/layer23_attention_weight_min": -26.767822265625,
      "activations/layer2_attention_weight_max": 31.998062133789062,
      "activations/layer2_attention_weight_min": -31.80977439880371,
      "activations/layer3_attention_weight_max": 101.13953399658203,
      "activations/layer3_attention_weight_min": -100.96261596679688,
      "activations/layer4_attention_weight_max": 107.19940185546875,
      "activations/layer4_attention_weight_min": -109.7904052734375,
      "activations/layer5_attention_weight_max": 53.8753776550293,
      "activations/layer5_attention_weight_min": -63.614715576171875,
      "activations/layer6_attention_weight_max": 44.58895492553711,
      "activations/layer6_attention_weight_min": -47.64531326293945,
      "activations/layer7_attention_weight_max": 96.34819793701172,
      "activations/layer7_attention_weight_min": -96.83203887939453,
      "activations/layer8_attention_weight_max": 44.985984802246094,
      "activations/layer8_attention_weight_min": -43.90558624267578,
      "activations/layer9_attention_weight_max": 34.79608917236328,
      "activations/layer9_attention_weight_min": -36.828269958496094,
      "epoch": 18.28,
      "learning_rate": 3.2443560606060605e-05,
      "loss": 2.7249,
      "step": 314600
    },
    {
      "activations/layer0_attention_weight_max": 17.114961624145508,
      "activations/layer0_attention_weight_min": -12.607172966003418,
      "activations/layer10_attention_weight_max": 35.723838806152344,
      "activations/layer10_attention_weight_min": -35.29127883911133,
      "activations/layer11_attention_weight_max": 34.552574157714844,
      "activations/layer11_attention_weight_min": -34.09889221191406,
      "activations/layer12_attention_weight_max": 28.019596099853516,
      "activations/layer12_attention_weight_min": -27.18357276916504,
      "activations/layer13_attention_weight_max": 41.48175811767578,
      "activations/layer13_attention_weight_min": -38.09975051879883,
      "activations/layer14_attention_weight_max": 44.95185089111328,
      "activations/layer14_attention_weight_min": -40.430782318115234,
      "activations/layer15_attention_weight_max": 39.1981201171875,
      "activations/layer15_attention_weight_min": -35.759395599365234,
      "activations/layer16_attention_weight_max": 30.896520614624023,
      "activations/layer16_attention_weight_min": -27.68213653564453,
      "activations/layer17_attention_weight_max": 31.411699295043945,
      "activations/layer17_attention_weight_min": -25.60101318359375,
      "activations/layer18_attention_weight_max": 33.87770080566406,
      "activations/layer18_attention_weight_min": -25.277034759521484,
      "activations/layer19_attention_weight_max": 33.89878845214844,
      "activations/layer19_attention_weight_min": -32.70130920410156,
      "activations/layer1_attention_weight_max": 15.552535057067871,
      "activations/layer1_attention_weight_min": -14.208868026733398,
      "activations/layer20_attention_weight_max": 27.84984588623047,
      "activations/layer20_attention_weight_min": -28.48571014404297,
      "activations/layer21_attention_weight_max": 30.506216049194336,
      "activations/layer21_attention_weight_min": -25.93699836730957,
      "activations/layer22_attention_weight_max": 43.54273223876953,
      "activations/layer22_attention_weight_min": -30.285005569458008,
      "activations/layer23_attention_weight_max": 30.417919158935547,
      "activations/layer23_attention_weight_min": -25.15326499938965,
      "activations/layer2_attention_weight_max": 32.88262176513672,
      "activations/layer2_attention_weight_min": -32.94608688354492,
      "activations/layer3_attention_weight_max": 99.13810729980469,
      "activations/layer3_attention_weight_min": -101.76591491699219,
      "activations/layer4_attention_weight_max": 106.13385009765625,
      "activations/layer4_attention_weight_min": -116.2643814086914,
      "activations/layer5_attention_weight_max": 49.511016845703125,
      "activations/layer5_attention_weight_min": -65.34671783447266,
      "activations/layer6_attention_weight_max": 46.56870651245117,
      "activations/layer6_attention_weight_min": -50.37226104736328,
      "activations/layer7_attention_weight_max": 97.76471710205078,
      "activations/layer7_attention_weight_min": -106.70482635498047,
      "activations/layer8_attention_weight_max": 42.462318420410156,
      "activations/layer8_attention_weight_min": -46.49595642089844,
      "activations/layer9_attention_weight_max": 33.499210357666016,
      "activations/layer9_attention_weight_min": -35.61035919189453,
      "epoch": 18.28,
      "learning_rate": 3.242462121212121e-05,
      "loss": 2.7372,
      "step": 314650
    },
    {
      "activations/layer0_attention_weight_max": 17.40792465209961,
      "activations/layer0_attention_weight_min": -15.417819023132324,
      "activations/layer10_attention_weight_max": 34.39107894897461,
      "activations/layer10_attention_weight_min": -33.76723098754883,
      "activations/layer11_attention_weight_max": 33.211212158203125,
      "activations/layer11_attention_weight_min": -34.62512969970703,
      "activations/layer12_attention_weight_max": 24.445032119750977,
      "activations/layer12_attention_weight_min": -29.001178741455078,
      "activations/layer13_attention_weight_max": 37.1370849609375,
      "activations/layer13_attention_weight_min": -35.98342514038086,
      "activations/layer14_attention_weight_max": 40.172760009765625,
      "activations/layer14_attention_weight_min": -41.13246536254883,
      "activations/layer15_attention_weight_max": 36.24040985107422,
      "activations/layer15_attention_weight_min": -36.87519836425781,
      "activations/layer16_attention_weight_max": 25.862470626831055,
      "activations/layer16_attention_weight_min": -26.52475929260254,
      "activations/layer17_attention_weight_max": 29.291364669799805,
      "activations/layer17_attention_weight_min": -26.448333740234375,
      "activations/layer18_attention_weight_max": 33.036285400390625,
      "activations/layer18_attention_weight_min": -26.67255210876465,
      "activations/layer19_attention_weight_max": 33.147586822509766,
      "activations/layer19_attention_weight_min": -30.74232292175293,
      "activations/layer1_attention_weight_max": 17.10143280029297,
      "activations/layer1_attention_weight_min": -15.832566261291504,
      "activations/layer20_attention_weight_max": 30.112321853637695,
      "activations/layer20_attention_weight_min": -26.083267211914062,
      "activations/layer21_attention_weight_max": 28.390789031982422,
      "activations/layer21_attention_weight_min": -27.36766815185547,
      "activations/layer22_attention_weight_max": 36.99794387817383,
      "activations/layer22_attention_weight_min": -32.498619079589844,
      "activations/layer23_attention_weight_max": 31.670089721679688,
      "activations/layer23_attention_weight_min": -25.944068908691406,
      "activations/layer2_attention_weight_max": 32.27287292480469,
      "activations/layer2_attention_weight_min": -33.87936782836914,
      "activations/layer3_attention_weight_max": 99.73164367675781,
      "activations/layer3_attention_weight_min": -101.33641052246094,
      "activations/layer4_attention_weight_max": 103.50776672363281,
      "activations/layer4_attention_weight_min": -107.40852355957031,
      "activations/layer5_attention_weight_max": 52.43024444580078,
      "activations/layer5_attention_weight_min": -59.9312744140625,
      "activations/layer6_attention_weight_max": 43.671592712402344,
      "activations/layer6_attention_weight_min": -46.70206832885742,
      "activations/layer7_attention_weight_max": 93.17724609375,
      "activations/layer7_attention_weight_min": -99.31560516357422,
      "activations/layer8_attention_weight_max": 40.4969367980957,
      "activations/layer8_attention_weight_min": -42.768646240234375,
      "activations/layer9_attention_weight_max": 31.970434188842773,
      "activations/layer9_attention_weight_min": -34.072845458984375,
      "epoch": 18.29,
      "learning_rate": 3.2405681818181814e-05,
      "loss": 2.7233,
      "step": 314700
    },
    {
      "activations/layer0_attention_weight_max": 16.097919464111328,
      "activations/layer0_attention_weight_min": -11.971757888793945,
      "activations/layer10_attention_weight_max": 35.69747543334961,
      "activations/layer10_attention_weight_min": -33.95114517211914,
      "activations/layer11_attention_weight_max": 34.32545852661133,
      "activations/layer11_attention_weight_min": -34.9370002746582,
      "activations/layer12_attention_weight_max": 24.905237197875977,
      "activations/layer12_attention_weight_min": -26.38068389892578,
      "activations/layer13_attention_weight_max": 34.7522087097168,
      "activations/layer13_attention_weight_min": -35.68649673461914,
      "activations/layer14_attention_weight_max": 38.86090850830078,
      "activations/layer14_attention_weight_min": -39.44918441772461,
      "activations/layer15_attention_weight_max": 35.785587310791016,
      "activations/layer15_attention_weight_min": -35.42862319946289,
      "activations/layer16_attention_weight_max": 26.781797409057617,
      "activations/layer16_attention_weight_min": -26.813764572143555,
      "activations/layer17_attention_weight_max": 25.76214027404785,
      "activations/layer17_attention_weight_min": -26.33920669555664,
      "activations/layer18_attention_weight_max": 27.448823928833008,
      "activations/layer18_attention_weight_min": -27.85907554626465,
      "activations/layer19_attention_weight_max": 30.783674240112305,
      "activations/layer19_attention_weight_min": -29.795270919799805,
      "activations/layer1_attention_weight_max": 17.459814071655273,
      "activations/layer1_attention_weight_min": -15.76248550415039,
      "activations/layer20_attention_weight_max": 28.416946411132812,
      "activations/layer20_attention_weight_min": -24.375865936279297,
      "activations/layer21_attention_weight_max": 26.68084716796875,
      "activations/layer21_attention_weight_min": -24.544389724731445,
      "activations/layer22_attention_weight_max": 35.226505279541016,
      "activations/layer22_attention_weight_min": -31.143524169921875,
      "activations/layer23_attention_weight_max": 27.600194931030273,
      "activations/layer23_attention_weight_min": -26.479530334472656,
      "activations/layer2_attention_weight_max": 34.43450927734375,
      "activations/layer2_attention_weight_min": -33.187255859375,
      "activations/layer3_attention_weight_max": 99.84394073486328,
      "activations/layer3_attention_weight_min": -97.33708953857422,
      "activations/layer4_attention_weight_max": 105.5461196899414,
      "activations/layer4_attention_weight_min": -104.35345458984375,
      "activations/layer5_attention_weight_max": 52.124610900878906,
      "activations/layer5_attention_weight_min": -60.95261764526367,
      "activations/layer6_attention_weight_max": 43.06193542480469,
      "activations/layer6_attention_weight_min": -47.57117462158203,
      "activations/layer7_attention_weight_max": 103.1451187133789,
      "activations/layer7_attention_weight_min": -97.1277847290039,
      "activations/layer8_attention_weight_max": 44.10573196411133,
      "activations/layer8_attention_weight_min": -44.224090576171875,
      "activations/layer9_attention_weight_max": 30.0246639251709,
      "activations/layer9_attention_weight_min": -32.706607818603516,
      "epoch": 18.29,
      "learning_rate": 3.238674242424242e-05,
      "loss": 2.7324,
      "step": 314750
    },
    {
      "activations/layer0_attention_weight_max": 16.179676055908203,
      "activations/layer0_attention_weight_min": -11.863788604736328,
      "activations/layer10_attention_weight_max": 33.22689437866211,
      "activations/layer10_attention_weight_min": -34.707733154296875,
      "activations/layer11_attention_weight_max": 34.381263732910156,
      "activations/layer11_attention_weight_min": -34.92063903808594,
      "activations/layer12_attention_weight_max": 28.534011840820312,
      "activations/layer12_attention_weight_min": -26.82134246826172,
      "activations/layer13_attention_weight_max": 44.51322555541992,
      "activations/layer13_attention_weight_min": -35.51131820678711,
      "activations/layer14_attention_weight_max": 43.37544250488281,
      "activations/layer14_attention_weight_min": -39.243751525878906,
      "activations/layer15_attention_weight_max": 42.48160934448242,
      "activations/layer15_attention_weight_min": -35.64231491088867,
      "activations/layer16_attention_weight_max": 28.477737426757812,
      "activations/layer16_attention_weight_min": -29.14914894104004,
      "activations/layer17_attention_weight_max": 31.14893913269043,
      "activations/layer17_attention_weight_min": -27.6485538482666,
      "activations/layer18_attention_weight_max": 30.218069076538086,
      "activations/layer18_attention_weight_min": -24.960521697998047,
      "activations/layer19_attention_weight_max": 32.52490234375,
      "activations/layer19_attention_weight_min": -32.22669219970703,
      "activations/layer1_attention_weight_max": 15.449753761291504,
      "activations/layer1_attention_weight_min": -13.899515151977539,
      "activations/layer20_attention_weight_max": 28.926462173461914,
      "activations/layer20_attention_weight_min": -25.46916961669922,
      "activations/layer21_attention_weight_max": 28.628860473632812,
      "activations/layer21_attention_weight_min": -24.53973960876465,
      "activations/layer22_attention_weight_max": 41.845252990722656,
      "activations/layer22_attention_weight_min": -31.82814598083496,
      "activations/layer23_attention_weight_max": 32.42863845825195,
      "activations/layer23_attention_weight_min": -26.130447387695312,
      "activations/layer2_attention_weight_max": 28.687545776367188,
      "activations/layer2_attention_weight_min": -30.220502853393555,
      "activations/layer3_attention_weight_max": 94.35859680175781,
      "activations/layer3_attention_weight_min": -96.97113037109375,
      "activations/layer4_attention_weight_max": 101.41707611083984,
      "activations/layer4_attention_weight_min": -106.16756439208984,
      "activations/layer5_attention_weight_max": 49.20258331298828,
      "activations/layer5_attention_weight_min": -58.97732162475586,
      "activations/layer6_attention_weight_max": 43.53011703491211,
      "activations/layer6_attention_weight_min": -44.85464096069336,
      "activations/layer7_attention_weight_max": 92.5697250366211,
      "activations/layer7_attention_weight_min": -96.2533187866211,
      "activations/layer8_attention_weight_max": 39.32101058959961,
      "activations/layer8_attention_weight_min": -41.82251739501953,
      "activations/layer9_attention_weight_max": 32.07748031616211,
      "activations/layer9_attention_weight_min": -34.8479118347168,
      "epoch": 18.29,
      "learning_rate": 3.2367803030303024e-05,
      "loss": 2.7369,
      "step": 314800
    },
    {
      "activations/layer0_attention_weight_max": 15.980345726013184,
      "activations/layer0_attention_weight_min": -12.747092247009277,
      "activations/layer10_attention_weight_max": 36.45817184448242,
      "activations/layer10_attention_weight_min": -38.06269073486328,
      "activations/layer11_attention_weight_max": 35.19514083862305,
      "activations/layer11_attention_weight_min": -36.27878189086914,
      "activations/layer12_attention_weight_max": 27.300315856933594,
      "activations/layer12_attention_weight_min": -27.758670806884766,
      "activations/layer13_attention_weight_max": 39.97557830810547,
      "activations/layer13_attention_weight_min": -37.2861328125,
      "activations/layer14_attention_weight_max": 42.77650451660156,
      "activations/layer14_attention_weight_min": -39.11861038208008,
      "activations/layer15_attention_weight_max": 41.067142486572266,
      "activations/layer15_attention_weight_min": -38.85353469848633,
      "activations/layer16_attention_weight_max": 30.415861129760742,
      "activations/layer16_attention_weight_min": -28.617828369140625,
      "activations/layer17_attention_weight_max": 30.729835510253906,
      "activations/layer17_attention_weight_min": -27.32022476196289,
      "activations/layer18_attention_weight_max": 30.469467163085938,
      "activations/layer18_attention_weight_min": -26.423810958862305,
      "activations/layer19_attention_weight_max": 32.904117584228516,
      "activations/layer19_attention_weight_min": -30.94099998474121,
      "activations/layer1_attention_weight_max": 15.951305389404297,
      "activations/layer1_attention_weight_min": -13.869731903076172,
      "activations/layer20_attention_weight_max": 29.40682029724121,
      "activations/layer20_attention_weight_min": -24.745542526245117,
      "activations/layer21_attention_weight_max": 28.4299373626709,
      "activations/layer21_attention_weight_min": -23.817840576171875,
      "activations/layer22_attention_weight_max": 39.994327545166016,
      "activations/layer22_attention_weight_min": -29.58028793334961,
      "activations/layer23_attention_weight_max": 30.890239715576172,
      "activations/layer23_attention_weight_min": -24.73271942138672,
      "activations/layer2_attention_weight_max": 32.673179626464844,
      "activations/layer2_attention_weight_min": -33.433712005615234,
      "activations/layer3_attention_weight_max": 104.08950805664062,
      "activations/layer3_attention_weight_min": -105.02651977539062,
      "activations/layer4_attention_weight_max": 113.2594223022461,
      "activations/layer4_attention_weight_min": -110.03474426269531,
      "activations/layer5_attention_weight_max": 52.11426544189453,
      "activations/layer5_attention_weight_min": -65.0133285522461,
      "activations/layer6_attention_weight_max": 46.63231658935547,
      "activations/layer6_attention_weight_min": -49.00688171386719,
      "activations/layer7_attention_weight_max": 95.85480499267578,
      "activations/layer7_attention_weight_min": -101.16768646240234,
      "activations/layer8_attention_weight_max": 43.97944259643555,
      "activations/layer8_attention_weight_min": -45.5803108215332,
      "activations/layer9_attention_weight_max": 33.53409194946289,
      "activations/layer9_attention_weight_min": -36.58525466918945,
      "epoch": 18.29,
      "learning_rate": 3.234886363636363e-05,
      "loss": 2.7409,
      "step": 314850
    },
    {
      "activations/layer0_attention_weight_max": 16.238012313842773,
      "activations/layer0_attention_weight_min": -12.964116096496582,
      "activations/layer10_attention_weight_max": 33.137454986572266,
      "activations/layer10_attention_weight_min": -34.363037109375,
      "activations/layer11_attention_weight_max": 32.76285934448242,
      "activations/layer11_attention_weight_min": -34.718631744384766,
      "activations/layer12_attention_weight_max": 28.615982055664062,
      "activations/layer12_attention_weight_min": -27.871671676635742,
      "activations/layer13_attention_weight_max": 42.87155532836914,
      "activations/layer13_attention_weight_min": -38.29307556152344,
      "activations/layer14_attention_weight_max": 44.98981857299805,
      "activations/layer14_attention_weight_min": -39.524452209472656,
      "activations/layer15_attention_weight_max": 42.59366989135742,
      "activations/layer15_attention_weight_min": -37.069217681884766,
      "activations/layer16_attention_weight_max": 32.46297836303711,
      "activations/layer16_attention_weight_min": -29.252456665039062,
      "activations/layer17_attention_weight_max": 29.61334228515625,
      "activations/layer17_attention_weight_min": -26.89754295349121,
      "activations/layer18_attention_weight_max": 31.414569854736328,
      "activations/layer18_attention_weight_min": -26.705575942993164,
      "activations/layer19_attention_weight_max": 37.613277435302734,
      "activations/layer19_attention_weight_min": -33.15205001831055,
      "activations/layer1_attention_weight_max": 15.82719898223877,
      "activations/layer1_attention_weight_min": -13.3908109664917,
      "activations/layer20_attention_weight_max": 30.10710334777832,
      "activations/layer20_attention_weight_min": -28.252777099609375,
      "activations/layer21_attention_weight_max": 30.91010093688965,
      "activations/layer21_attention_weight_min": -27.988901138305664,
      "activations/layer22_attention_weight_max": 46.40113067626953,
      "activations/layer22_attention_weight_min": -36.30253219604492,
      "activations/layer23_attention_weight_max": 33.38007736206055,
      "activations/layer23_attention_weight_min": -26.522781372070312,
      "activations/layer2_attention_weight_max": 32.51750946044922,
      "activations/layer2_attention_weight_min": -32.097259521484375,
      "activations/layer3_attention_weight_max": 98.13156127929688,
      "activations/layer3_attention_weight_min": -96.99356079101562,
      "activations/layer4_attention_weight_max": 102.5103988647461,
      "activations/layer4_attention_weight_min": -105.34725189208984,
      "activations/layer5_attention_weight_max": 48.14641571044922,
      "activations/layer5_attention_weight_min": -59.797943115234375,
      "activations/layer6_attention_weight_max": 44.25813674926758,
      "activations/layer6_attention_weight_min": -46.87468338012695,
      "activations/layer7_attention_weight_max": 89.5302963256836,
      "activations/layer7_attention_weight_min": -102.8726577758789,
      "activations/layer8_attention_weight_max": 40.816314697265625,
      "activations/layer8_attention_weight_min": -44.815364837646484,
      "activations/layer9_attention_weight_max": 32.099552154541016,
      "activations/layer9_attention_weight_min": -35.08098602294922,
      "epoch": 18.3,
      "learning_rate": 3.232992424242424e-05,
      "loss": 2.747,
      "step": 314900
    },
    {
      "activations/layer0_attention_weight_max": 15.71119213104248,
      "activations/layer0_attention_weight_min": -13.19787311553955,
      "activations/layer10_attention_weight_max": 33.470096588134766,
      "activations/layer10_attention_weight_min": -33.84599685668945,
      "activations/layer11_attention_weight_max": 32.87853240966797,
      "activations/layer11_attention_weight_min": -35.274879455566406,
      "activations/layer12_attention_weight_max": 28.051301956176758,
      "activations/layer12_attention_weight_min": -28.534826278686523,
      "activations/layer13_attention_weight_max": 42.504669189453125,
      "activations/layer13_attention_weight_min": -39.125587463378906,
      "activations/layer14_attention_weight_max": 42.23323059082031,
      "activations/layer14_attention_weight_min": -41.7830810546875,
      "activations/layer15_attention_weight_max": 42.21964645385742,
      "activations/layer15_attention_weight_min": -38.56039810180664,
      "activations/layer16_attention_weight_max": 29.857086181640625,
      "activations/layer16_attention_weight_min": -29.055757522583008,
      "activations/layer17_attention_weight_max": 27.05891990661621,
      "activations/layer17_attention_weight_min": -27.99935531616211,
      "activations/layer18_attention_weight_max": 33.44031524658203,
      "activations/layer18_attention_weight_min": -29.54205322265625,
      "activations/layer19_attention_weight_max": 33.55119705200195,
      "activations/layer19_attention_weight_min": -31.46278953552246,
      "activations/layer1_attention_weight_max": 15.889874458312988,
      "activations/layer1_attention_weight_min": -14.205998420715332,
      "activations/layer20_attention_weight_max": 26.87299156188965,
      "activations/layer20_attention_weight_min": -25.31344223022461,
      "activations/layer21_attention_weight_max": 27.277633666992188,
      "activations/layer21_attention_weight_min": -25.18565559387207,
      "activations/layer22_attention_weight_max": 39.39384078979492,
      "activations/layer22_attention_weight_min": -32.77347183227539,
      "activations/layer23_attention_weight_max": 30.089628219604492,
      "activations/layer23_attention_weight_min": -27.906972885131836,
      "activations/layer2_attention_weight_max": 33.92295837402344,
      "activations/layer2_attention_weight_min": -32.31700134277344,
      "activations/layer3_attention_weight_max": 93.62808990478516,
      "activations/layer3_attention_weight_min": -95.674072265625,
      "activations/layer4_attention_weight_max": 100.7591323852539,
      "activations/layer4_attention_weight_min": -107.81922912597656,
      "activations/layer5_attention_weight_max": 47.15079879760742,
      "activations/layer5_attention_weight_min": -58.399009704589844,
      "activations/layer6_attention_weight_max": 44.54871368408203,
      "activations/layer6_attention_weight_min": -46.133140563964844,
      "activations/layer7_attention_weight_max": 87.71757507324219,
      "activations/layer7_attention_weight_min": -98.3900375366211,
      "activations/layer8_attention_weight_max": 39.49324035644531,
      "activations/layer8_attention_weight_min": -43.34566116333008,
      "activations/layer9_attention_weight_max": 31.929580688476562,
      "activations/layer9_attention_weight_min": -34.21501159667969,
      "epoch": 18.3,
      "learning_rate": 3.231098484848485e-05,
      "loss": 2.7179,
      "step": 314950
    },
    {
      "activations/layer0_attention_weight_max": 16.233388900756836,
      "activations/layer0_attention_weight_min": -14.14719009399414,
      "activations/layer10_attention_weight_max": 32.11396026611328,
      "activations/layer10_attention_weight_min": -31.948532104492188,
      "activations/layer11_attention_weight_max": 31.66195297241211,
      "activations/layer11_attention_weight_min": -32.62895965576172,
      "activations/layer12_attention_weight_max": 27.8756160736084,
      "activations/layer12_attention_weight_min": -25.536521911621094,
      "activations/layer13_attention_weight_max": 38.351470947265625,
      "activations/layer13_attention_weight_min": -35.60258102416992,
      "activations/layer14_attention_weight_max": 43.321231842041016,
      "activations/layer14_attention_weight_min": -37.09189987182617,
      "activations/layer15_attention_weight_max": 40.92643737792969,
      "activations/layer15_attention_weight_min": -34.125244140625,
      "activations/layer16_attention_weight_max": 29.98525047302246,
      "activations/layer16_attention_weight_min": -28.205032348632812,
      "activations/layer17_attention_weight_max": 29.099817276000977,
      "activations/layer17_attention_weight_min": -25.89015769958496,
      "activations/layer18_attention_weight_max": 32.48861312866211,
      "activations/layer18_attention_weight_min": -24.344512939453125,
      "activations/layer19_attention_weight_max": 34.17875671386719,
      "activations/layer19_attention_weight_min": -30.81141471862793,
      "activations/layer1_attention_weight_max": 15.950502395629883,
      "activations/layer1_attention_weight_min": -15.181645393371582,
      "activations/layer20_attention_weight_max": 29.405611038208008,
      "activations/layer20_attention_weight_min": -24.709978103637695,
      "activations/layer21_attention_weight_max": 28.91439437866211,
      "activations/layer21_attention_weight_min": -25.391128540039062,
      "activations/layer22_attention_weight_max": 41.482872009277344,
      "activations/layer22_attention_weight_min": -31.68374252319336,
      "activations/layer23_attention_weight_max": 34.2343864440918,
      "activations/layer23_attention_weight_min": -27.494911193847656,
      "activations/layer2_attention_weight_max": 32.75117874145508,
      "activations/layer2_attention_weight_min": -32.07616424560547,
      "activations/layer3_attention_weight_max": 96.36158752441406,
      "activations/layer3_attention_weight_min": -95.87989044189453,
      "activations/layer4_attention_weight_max": 103.34415435791016,
      "activations/layer4_attention_weight_min": -103.2214126586914,
      "activations/layer5_attention_weight_max": 49.208980560302734,
      "activations/layer5_attention_weight_min": -62.295127868652344,
      "activations/layer6_attention_weight_max": 43.87273406982422,
      "activations/layer6_attention_weight_min": -46.14931106567383,
      "activations/layer7_attention_weight_max": 91.21737670898438,
      "activations/layer7_attention_weight_min": -90.35674285888672,
      "activations/layer8_attention_weight_max": 39.88026809692383,
      "activations/layer8_attention_weight_min": -41.436004638671875,
      "activations/layer9_attention_weight_max": 31.4560604095459,
      "activations/layer9_attention_weight_min": -34.73836898803711,
      "epoch": 18.3,
      "learning_rate": 3.229204545454546e-05,
      "loss": 2.7185,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_loss": 2.6875,
      "eval_runtime": 8.4862,
      "eval_samples_per_second": 505.997,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.4862,
      "eval_openwebtext_samples_per_second": 505.997,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.0237,
      "eval_wikitext_samples_per_second": 225.332,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_lambada_loss": 2.353515625,
      "eval_lambada_ppl": 10.522497930618114,
      "eval_lambada_runtime": 9.5898,
      "eval_lambada_samples_per_second": 507.729,
      "step": 315000
    },
    {
      "activations/layer0_attention_weight_max": 16.716150283813477,
      "activations/layer0_attention_weight_min": -13.210050582885742,
      "activations/layer10_attention_weight_max": 34.35490036010742,
      "activations/layer10_attention_weight_min": -36.3210334777832,
      "activations/layer11_attention_weight_max": 35.42670440673828,
      "activations/layer11_attention_weight_min": -34.3055534362793,
      "activations/layer12_attention_weight_max": 27.771459579467773,
      "activations/layer12_attention_weight_min": -27.259565353393555,
      "activations/layer13_attention_weight_max": 44.18667984008789,
      "activations/layer13_attention_weight_min": -36.9046630859375,
      "activations/layer14_attention_weight_max": 49.440860748291016,
      "activations/layer14_attention_weight_min": -39.653839111328125,
      "activations/layer15_attention_weight_max": 42.994667053222656,
      "activations/layer15_attention_weight_min": -38.24980545043945,
      "activations/layer16_attention_weight_max": 35.150474548339844,
      "activations/layer16_attention_weight_min": -28.894670486450195,
      "activations/layer17_attention_weight_max": 29.455121994018555,
      "activations/layer17_attention_weight_min": -28.538923263549805,
      "activations/layer18_attention_weight_max": 30.18177604675293,
      "activations/layer18_attention_weight_min": -26.052461624145508,
      "activations/layer19_attention_weight_max": 34.28182601928711,
      "activations/layer19_attention_weight_min": -32.91653823852539,
      "activations/layer1_attention_weight_max": 16.269149780273438,
      "activations/layer1_attention_weight_min": -15.111781120300293,
      "activations/layer20_attention_weight_max": 31.075788497924805,
      "activations/layer20_attention_weight_min": -24.658184051513672,
      "activations/layer21_attention_weight_max": 29.691646575927734,
      "activations/layer21_attention_weight_min": -26.295913696289062,
      "activations/layer22_attention_weight_max": 41.94776153564453,
      "activations/layer22_attention_weight_min": -34.012203216552734,
      "activations/layer23_attention_weight_max": 29.845420837402344,
      "activations/layer23_attention_weight_min": -27.112091064453125,
      "activations/layer2_attention_weight_max": 30.924978256225586,
      "activations/layer2_attention_weight_min": -30.702619552612305,
      "activations/layer3_attention_weight_max": 94.96095275878906,
      "activations/layer3_attention_weight_min": -94.96305084228516,
      "activations/layer4_attention_weight_max": 105.46520233154297,
      "activations/layer4_attention_weight_min": -103.1837158203125,
      "activations/layer5_attention_weight_max": 50.659271240234375,
      "activations/layer5_attention_weight_min": -59.490509033203125,
      "activations/layer6_attention_weight_max": 44.73762512207031,
      "activations/layer6_attention_weight_min": -46.980140686035156,
      "activations/layer7_attention_weight_max": 93.19890594482422,
      "activations/layer7_attention_weight_min": -89.28419494628906,
      "activations/layer8_attention_weight_max": 42.851783752441406,
      "activations/layer8_attention_weight_min": -43.993961334228516,
      "activations/layer9_attention_weight_max": 33.639862060546875,
      "activations/layer9_attention_weight_min": -34.39010238647461,
      "epoch": 18.31,
      "learning_rate": 3.227310606060606e-05,
      "loss": 2.7164,
      "step": 315050
    },
    {
      "activations/layer0_attention_weight_max": 16.920879364013672,
      "activations/layer0_attention_weight_min": -14.03255844116211,
      "activations/layer10_attention_weight_max": 32.219451904296875,
      "activations/layer10_attention_weight_min": -34.59626007080078,
      "activations/layer11_attention_weight_max": 34.41199493408203,
      "activations/layer11_attention_weight_min": -36.163841247558594,
      "activations/layer12_attention_weight_max": 27.224470138549805,
      "activations/layer12_attention_weight_min": -24.7545223236084,
      "activations/layer13_attention_weight_max": 37.43935775756836,
      "activations/layer13_attention_weight_min": -33.146793365478516,
      "activations/layer14_attention_weight_max": 42.374210357666016,
      "activations/layer14_attention_weight_min": -35.14723205566406,
      "activations/layer15_attention_weight_max": 38.629302978515625,
      "activations/layer15_attention_weight_min": -35.036067962646484,
      "activations/layer16_attention_weight_max": 34.11457443237305,
      "activations/layer16_attention_weight_min": -28.29428482055664,
      "activations/layer17_attention_weight_max": 33.90060043334961,
      "activations/layer17_attention_weight_min": -25.169466018676758,
      "activations/layer18_attention_weight_max": 36.77784729003906,
      "activations/layer18_attention_weight_min": -26.165096282958984,
      "activations/layer19_attention_weight_max": 39.25942611694336,
      "activations/layer19_attention_weight_min": -31.24567222595215,
      "activations/layer1_attention_weight_max": 16.57027816772461,
      "activations/layer1_attention_weight_min": -15.671491622924805,
      "activations/layer20_attention_weight_max": 35.782493591308594,
      "activations/layer20_attention_weight_min": -25.1239013671875,
      "activations/layer21_attention_weight_max": 33.79600143432617,
      "activations/layer21_attention_weight_min": -25.595815658569336,
      "activations/layer22_attention_weight_max": 51.394039154052734,
      "activations/layer22_attention_weight_min": -31.926057815551758,
      "activations/layer23_attention_weight_max": 37.57331466674805,
      "activations/layer23_attention_weight_min": -26.673988342285156,
      "activations/layer2_attention_weight_max": 30.95162010192871,
      "activations/layer2_attention_weight_min": -31.338157653808594,
      "activations/layer3_attention_weight_max": 96.10100555419922,
      "activations/layer3_attention_weight_min": -99.84104919433594,
      "activations/layer4_attention_weight_max": 103.3458480834961,
      "activations/layer4_attention_weight_min": -103.16766357421875,
      "activations/layer5_attention_weight_max": 47.574981689453125,
      "activations/layer5_attention_weight_min": -60.7022705078125,
      "activations/layer6_attention_weight_max": 42.97688293457031,
      "activations/layer6_attention_weight_min": -46.829463958740234,
      "activations/layer7_attention_weight_max": 91.7990951538086,
      "activations/layer7_attention_weight_min": -93.0739974975586,
      "activations/layer8_attention_weight_max": 39.37957763671875,
      "activations/layer8_attention_weight_min": -42.6733283996582,
      "activations/layer9_attention_weight_max": 29.886850357055664,
      "activations/layer9_attention_weight_min": -33.46371078491211,
      "epoch": 18.31,
      "learning_rate": 3.225416666666667e-05,
      "loss": 2.7261,
      "step": 315100
    },
    {
      "activations/layer0_attention_weight_max": 16.531848907470703,
      "activations/layer0_attention_weight_min": -12.802531242370605,
      "activations/layer10_attention_weight_max": 36.247596740722656,
      "activations/layer10_attention_weight_min": -34.26613235473633,
      "activations/layer11_attention_weight_max": 38.28623962402344,
      "activations/layer11_attention_weight_min": -34.35498046875,
      "activations/layer12_attention_weight_max": 25.808597564697266,
      "activations/layer12_attention_weight_min": -26.926071166992188,
      "activations/layer13_attention_weight_max": 37.92292404174805,
      "activations/layer13_attention_weight_min": -38.27162170410156,
      "activations/layer14_attention_weight_max": 40.43214797973633,
      "activations/layer14_attention_weight_min": -41.52519989013672,
      "activations/layer15_attention_weight_max": 42.10121154785156,
      "activations/layer15_attention_weight_min": -40.02705383300781,
      "activations/layer16_attention_weight_max": 27.279680252075195,
      "activations/layer16_attention_weight_min": -27.798418045043945,
      "activations/layer17_attention_weight_max": 26.147573471069336,
      "activations/layer17_attention_weight_min": -25.545391082763672,
      "activations/layer18_attention_weight_max": 29.667152404785156,
      "activations/layer18_attention_weight_min": -26.374778747558594,
      "activations/layer19_attention_weight_max": 33.30557632446289,
      "activations/layer19_attention_weight_min": -30.454971313476562,
      "activations/layer1_attention_weight_max": 15.913022994995117,
      "activations/layer1_attention_weight_min": -13.468466758728027,
      "activations/layer20_attention_weight_max": 26.271377563476562,
      "activations/layer20_attention_weight_min": -24.65462303161621,
      "activations/layer21_attention_weight_max": 24.409685134887695,
      "activations/layer21_attention_weight_min": -24.23134422302246,
      "activations/layer22_attention_weight_max": 38.63920974731445,
      "activations/layer22_attention_weight_min": -31.874895095825195,
      "activations/layer23_attention_weight_max": 36.11519241333008,
      "activations/layer23_attention_weight_min": -25.223257064819336,
      "activations/layer2_attention_weight_max": 34.2000617980957,
      "activations/layer2_attention_weight_min": -30.900203704833984,
      "activations/layer3_attention_weight_max": 93.47640991210938,
      "activations/layer3_attention_weight_min": -96.76017761230469,
      "activations/layer4_attention_weight_max": 104.31102752685547,
      "activations/layer4_attention_weight_min": -104.36518859863281,
      "activations/layer5_attention_weight_max": 50.4210319519043,
      "activations/layer5_attention_weight_min": -58.40663146972656,
      "activations/layer6_attention_weight_max": 47.6799430847168,
      "activations/layer6_attention_weight_min": -46.75822067260742,
      "activations/layer7_attention_weight_max": 95.89503479003906,
      "activations/layer7_attention_weight_min": -95.9305191040039,
      "activations/layer8_attention_weight_max": 43.4337043762207,
      "activations/layer8_attention_weight_min": -42.34282302856445,
      "activations/layer9_attention_weight_max": 31.641088485717773,
      "activations/layer9_attention_weight_min": -33.759544372558594,
      "epoch": 18.31,
      "learning_rate": 3.223560606060606e-05,
      "loss": 2.7291,
      "step": 315150
    },
    {
      "activations/layer0_attention_weight_max": 15.900361061096191,
      "activations/layer0_attention_weight_min": -12.310373306274414,
      "activations/layer10_attention_weight_max": 32.31382751464844,
      "activations/layer10_attention_weight_min": -35.11990737915039,
      "activations/layer11_attention_weight_max": 32.345611572265625,
      "activations/layer11_attention_weight_min": -34.43724060058594,
      "activations/layer12_attention_weight_max": 25.95208168029785,
      "activations/layer12_attention_weight_min": -30.61090660095215,
      "activations/layer13_attention_weight_max": 36.50139236450195,
      "activations/layer13_attention_weight_min": -34.71204376220703,
      "activations/layer14_attention_weight_max": 40.495391845703125,
      "activations/layer14_attention_weight_min": -37.395286560058594,
      "activations/layer15_attention_weight_max": 35.88718795776367,
      "activations/layer15_attention_weight_min": -36.27467727661133,
      "activations/layer16_attention_weight_max": 27.658447265625,
      "activations/layer16_attention_weight_min": -27.654876708984375,
      "activations/layer17_attention_weight_max": 29.31377601623535,
      "activations/layer17_attention_weight_min": -25.396392822265625,
      "activations/layer18_attention_weight_max": 30.81928825378418,
      "activations/layer18_attention_weight_min": -26.198312759399414,
      "activations/layer19_attention_weight_max": 31.628646850585938,
      "activations/layer19_attention_weight_min": -32.093563079833984,
      "activations/layer1_attention_weight_max": 15.946131706237793,
      "activations/layer1_attention_weight_min": -14.793066024780273,
      "activations/layer20_attention_weight_max": 26.508527755737305,
      "activations/layer20_attention_weight_min": -25.569095611572266,
      "activations/layer21_attention_weight_max": 26.619543075561523,
      "activations/layer21_attention_weight_min": -25.578758239746094,
      "activations/layer22_attention_weight_max": 40.54695510864258,
      "activations/layer22_attention_weight_min": -32.77075958251953,
      "activations/layer23_attention_weight_max": 31.10736083984375,
      "activations/layer23_attention_weight_min": -28.56552505493164,
      "activations/layer2_attention_weight_max": 29.676822662353516,
      "activations/layer2_attention_weight_min": -30.824451446533203,
      "activations/layer3_attention_weight_max": 91.22611999511719,
      "activations/layer3_attention_weight_min": -99.64871215820312,
      "activations/layer4_attention_weight_max": 99.19672393798828,
      "activations/layer4_attention_weight_min": -101.41925048828125,
      "activations/layer5_attention_weight_max": 47.25889205932617,
      "activations/layer5_attention_weight_min": -62.2498893737793,
      "activations/layer6_attention_weight_max": 43.9431266784668,
      "activations/layer6_attention_weight_min": -46.42024230957031,
      "activations/layer7_attention_weight_max": 91.35376739501953,
      "activations/layer7_attention_weight_min": -93.52542114257812,
      "activations/layer8_attention_weight_max": 42.4589729309082,
      "activations/layer8_attention_weight_min": -45.59596252441406,
      "activations/layer9_attention_weight_max": 31.003507614135742,
      "activations/layer9_attention_weight_min": -35.57050704956055,
      "epoch": 18.31,
      "learning_rate": 3.2216666666666666e-05,
      "loss": 2.7328,
      "step": 315200
    },
    {
      "activations/layer0_attention_weight_max": 17.25445556640625,
      "activations/layer0_attention_weight_min": -12.435070991516113,
      "activations/layer10_attention_weight_max": 35.150577545166016,
      "activations/layer10_attention_weight_min": -34.82271194458008,
      "activations/layer11_attention_weight_max": 35.69064712524414,
      "activations/layer11_attention_weight_min": -35.008453369140625,
      "activations/layer12_attention_weight_max": 26.477754592895508,
      "activations/layer12_attention_weight_min": -27.28392219543457,
      "activations/layer13_attention_weight_max": 40.33723449707031,
      "activations/layer13_attention_weight_min": -35.51556396484375,
      "activations/layer14_attention_weight_max": 43.42223358154297,
      "activations/layer14_attention_weight_min": -38.239654541015625,
      "activations/layer15_attention_weight_max": 39.81769943237305,
      "activations/layer15_attention_weight_min": -36.91913604736328,
      "activations/layer16_attention_weight_max": 31.52741241455078,
      "activations/layer16_attention_weight_min": -29.733261108398438,
      "activations/layer17_attention_weight_max": 32.888526916503906,
      "activations/layer17_attention_weight_min": -26.774311065673828,
      "activations/layer18_attention_weight_max": 30.881832122802734,
      "activations/layer18_attention_weight_min": -26.981605529785156,
      "activations/layer19_attention_weight_max": 31.485010147094727,
      "activations/layer19_attention_weight_min": -34.11635971069336,
      "activations/layer1_attention_weight_max": 17.87060546875,
      "activations/layer1_attention_weight_min": -13.646060943603516,
      "activations/layer20_attention_weight_max": 27.36687469482422,
      "activations/layer20_attention_weight_min": -27.295082092285156,
      "activations/layer21_attention_weight_max": 29.47810935974121,
      "activations/layer21_attention_weight_min": -27.867910385131836,
      "activations/layer22_attention_weight_max": 39.581844329833984,
      "activations/layer22_attention_weight_min": -32.43274688720703,
      "activations/layer23_attention_weight_max": 31.30414390563965,
      "activations/layer23_attention_weight_min": -26.96866226196289,
      "activations/layer2_attention_weight_max": 33.74392318725586,
      "activations/layer2_attention_weight_min": -31.873214721679688,
      "activations/layer3_attention_weight_max": 95.94285583496094,
      "activations/layer3_attention_weight_min": -97.56861877441406,
      "activations/layer4_attention_weight_max": 109.22350311279297,
      "activations/layer4_attention_weight_min": -102.71162414550781,
      "activations/layer5_attention_weight_max": 51.900146484375,
      "activations/layer5_attention_weight_min": -58.06202697753906,
      "activations/layer6_attention_weight_max": 44.94108200073242,
      "activations/layer6_attention_weight_min": -47.389930725097656,
      "activations/layer7_attention_weight_max": 95.56315612792969,
      "activations/layer7_attention_weight_min": -93.20152282714844,
      "activations/layer8_attention_weight_max": 42.85579299926758,
      "activations/layer8_attention_weight_min": -43.60371780395508,
      "activations/layer9_attention_weight_max": 33.21002960205078,
      "activations/layer9_attention_weight_min": -34.42512130737305,
      "epoch": 18.32,
      "learning_rate": 3.219772727272727e-05,
      "loss": 2.7156,
      "step": 315250
    },
    {
      "activations/layer0_attention_weight_max": 16.451398849487305,
      "activations/layer0_attention_weight_min": -13.633106231689453,
      "activations/layer10_attention_weight_max": 36.765838623046875,
      "activations/layer10_attention_weight_min": -35.5690803527832,
      "activations/layer11_attention_weight_max": 35.966739654541016,
      "activations/layer11_attention_weight_min": -35.87565994262695,
      "activations/layer12_attention_weight_max": 28.671323776245117,
      "activations/layer12_attention_weight_min": -28.495737075805664,
      "activations/layer13_attention_weight_max": 40.9735107421875,
      "activations/layer13_attention_weight_min": -36.16843795776367,
      "activations/layer14_attention_weight_max": 44.38547897338867,
      "activations/layer14_attention_weight_min": -39.074920654296875,
      "activations/layer15_attention_weight_max": 42.02927017211914,
      "activations/layer15_attention_weight_min": -39.71923065185547,
      "activations/layer16_attention_weight_max": 28.44108772277832,
      "activations/layer16_attention_weight_min": -27.59381866455078,
      "activations/layer17_attention_weight_max": 28.712480545043945,
      "activations/layer17_attention_weight_min": -26.20075798034668,
      "activations/layer18_attention_weight_max": 29.761381149291992,
      "activations/layer18_attention_weight_min": -25.940876007080078,
      "activations/layer19_attention_weight_max": 33.18794631958008,
      "activations/layer19_attention_weight_min": -34.337059020996094,
      "activations/layer1_attention_weight_max": 15.987540245056152,
      "activations/layer1_attention_weight_min": -14.364192008972168,
      "activations/layer20_attention_weight_max": 27.20348358154297,
      "activations/layer20_attention_weight_min": -25.429336547851562,
      "activations/layer21_attention_weight_max": 28.71157455444336,
      "activations/layer21_attention_weight_min": -26.165557861328125,
      "activations/layer22_attention_weight_max": 43.80882263183594,
      "activations/layer22_attention_weight_min": -32.95598220825195,
      "activations/layer23_attention_weight_max": 33.251487731933594,
      "activations/layer23_attention_weight_min": -25.872915267944336,
      "activations/layer2_attention_weight_max": 31.888809204101562,
      "activations/layer2_attention_weight_min": -32.58837127685547,
      "activations/layer3_attention_weight_max": 98.30989837646484,
      "activations/layer3_attention_weight_min": -100.36283874511719,
      "activations/layer4_attention_weight_max": 107.7781753540039,
      "activations/layer4_attention_weight_min": -108.87743377685547,
      "activations/layer5_attention_weight_max": 51.69991683959961,
      "activations/layer5_attention_weight_min": -65.75729370117188,
      "activations/layer6_attention_weight_max": 44.2839469909668,
      "activations/layer6_attention_weight_min": -48.11091995239258,
      "activations/layer7_attention_weight_max": 96.19799041748047,
      "activations/layer7_attention_weight_min": -97.55656433105469,
      "activations/layer8_attention_weight_max": 42.7261962890625,
      "activations/layer8_attention_weight_min": -44.35698318481445,
      "activations/layer9_attention_weight_max": 33.92374038696289,
      "activations/layer9_attention_weight_min": -34.67108154296875,
      "epoch": 18.32,
      "learning_rate": 3.2178787878787876e-05,
      "loss": 2.7385,
      "step": 315300
    },
    {
      "activations/layer0_attention_weight_max": 16.96266746520996,
      "activations/layer0_attention_weight_min": -13.335079193115234,
      "activations/layer10_attention_weight_max": 33.72495651245117,
      "activations/layer10_attention_weight_min": -36.10840606689453,
      "activations/layer11_attention_weight_max": 33.95428466796875,
      "activations/layer11_attention_weight_min": -37.06658172607422,
      "activations/layer12_attention_weight_max": 26.94154167175293,
      "activations/layer12_attention_weight_min": -26.43341827392578,
      "activations/layer13_attention_weight_max": 41.33720779418945,
      "activations/layer13_attention_weight_min": -34.959449768066406,
      "activations/layer14_attention_weight_max": 42.87993240356445,
      "activations/layer14_attention_weight_min": -37.255470275878906,
      "activations/layer15_attention_weight_max": 38.176422119140625,
      "activations/layer15_attention_weight_min": -35.307857513427734,
      "activations/layer16_attention_weight_max": 31.86308479309082,
      "activations/layer16_attention_weight_min": -26.497234344482422,
      "activations/layer17_attention_weight_max": 30.777517318725586,
      "activations/layer17_attention_weight_min": -25.371173858642578,
      "activations/layer18_attention_weight_max": 32.1021614074707,
      "activations/layer18_attention_weight_min": -24.631458282470703,
      "activations/layer19_attention_weight_max": 36.33951950073242,
      "activations/layer19_attention_weight_min": -30.5199031829834,
      "activations/layer1_attention_weight_max": 17.19108009338379,
      "activations/layer1_attention_weight_min": -13.97011661529541,
      "activations/layer20_attention_weight_max": 31.564189910888672,
      "activations/layer20_attention_weight_min": -26.727848052978516,
      "activations/layer21_attention_weight_max": 32.060516357421875,
      "activations/layer21_attention_weight_min": -23.786178588867188,
      "activations/layer22_attention_weight_max": 47.75395584106445,
      "activations/layer22_attention_weight_min": -30.048303604125977,
      "activations/layer23_attention_weight_max": 37.59943389892578,
      "activations/layer23_attention_weight_min": -25.562253952026367,
      "activations/layer2_attention_weight_max": 34.978492736816406,
      "activations/layer2_attention_weight_min": -34.57262420654297,
      "activations/layer3_attention_weight_max": 100.72142028808594,
      "activations/layer3_attention_weight_min": -98.53742980957031,
      "activations/layer4_attention_weight_max": 107.23516082763672,
      "activations/layer4_attention_weight_min": -106.24676513671875,
      "activations/layer5_attention_weight_max": 53.088470458984375,
      "activations/layer5_attention_weight_min": -60.69670104980469,
      "activations/layer6_attention_weight_max": 45.95978927612305,
      "activations/layer6_attention_weight_min": -48.410823822021484,
      "activations/layer7_attention_weight_max": 95.9718246459961,
      "activations/layer7_attention_weight_min": -96.27100372314453,
      "activations/layer8_attention_weight_max": 42.985313415527344,
      "activations/layer8_attention_weight_min": -46.00483703613281,
      "activations/layer9_attention_weight_max": 33.46242141723633,
      "activations/layer9_attention_weight_min": -37.731136322021484,
      "epoch": 18.32,
      "learning_rate": 3.2159848484848484e-05,
      "loss": 2.7346,
      "step": 315350
    },
    {
      "activations/layer0_attention_weight_max": 16.01181983947754,
      "activations/layer0_attention_weight_min": -13.304511070251465,
      "activations/layer10_attention_weight_max": 32.185306549072266,
      "activations/layer10_attention_weight_min": -32.121238708496094,
      "activations/layer11_attention_weight_max": 32.1694221496582,
      "activations/layer11_attention_weight_min": -32.13766098022461,
      "activations/layer12_attention_weight_max": 25.86241912841797,
      "activations/layer12_attention_weight_min": -27.637788772583008,
      "activations/layer13_attention_weight_max": 37.70119857788086,
      "activations/layer13_attention_weight_min": -35.782291412353516,
      "activations/layer14_attention_weight_max": 39.3701286315918,
      "activations/layer14_attention_weight_min": -35.860809326171875,
      "activations/layer15_attention_weight_max": 38.08805465698242,
      "activations/layer15_attention_weight_min": -33.8784065246582,
      "activations/layer16_attention_weight_max": 28.68230438232422,
      "activations/layer16_attention_weight_min": -28.045713424682617,
      "activations/layer17_attention_weight_max": 30.774967193603516,
      "activations/layer17_attention_weight_min": -27.140806198120117,
      "activations/layer18_attention_weight_max": 31.444692611694336,
      "activations/layer18_attention_weight_min": -27.474876403808594,
      "activations/layer19_attention_weight_max": 31.82681655883789,
      "activations/layer19_attention_weight_min": -32.30201721191406,
      "activations/layer1_attention_weight_max": 15.618328094482422,
      "activations/layer1_attention_weight_min": -13.837507247924805,
      "activations/layer20_attention_weight_max": 25.406991958618164,
      "activations/layer20_attention_weight_min": -25.30483055114746,
      "activations/layer21_attention_weight_max": 26.11374855041504,
      "activations/layer21_attention_weight_min": -24.23663330078125,
      "activations/layer22_attention_weight_max": 35.67490768432617,
      "activations/layer22_attention_weight_min": -32.49668502807617,
      "activations/layer23_attention_weight_max": 28.927627563476562,
      "activations/layer23_attention_weight_min": -27.010215759277344,
      "activations/layer2_attention_weight_max": 31.449718475341797,
      "activations/layer2_attention_weight_min": -31.237783432006836,
      "activations/layer3_attention_weight_max": 100.484619140625,
      "activations/layer3_attention_weight_min": -101.90133666992188,
      "activations/layer4_attention_weight_max": 106.0144271850586,
      "activations/layer4_attention_weight_min": -107.07592010498047,
      "activations/layer5_attention_weight_max": 49.548545837402344,
      "activations/layer5_attention_weight_min": -64.31619262695312,
      "activations/layer6_attention_weight_max": 41.20238494873047,
      "activations/layer6_attention_weight_min": -46.82168960571289,
      "activations/layer7_attention_weight_max": 86.13968658447266,
      "activations/layer7_attention_weight_min": -92.71736145019531,
      "activations/layer8_attention_weight_max": 40.25130844116211,
      "activations/layer8_attention_weight_min": -42.010894775390625,
      "activations/layer9_attention_weight_max": 30.017667770385742,
      "activations/layer9_attention_weight_min": -32.84001159667969,
      "epoch": 18.33,
      "learning_rate": 3.214090909090909e-05,
      "loss": 2.7349,
      "step": 315400
    },
    {
      "activations/layer0_attention_weight_max": 16.390256881713867,
      "activations/layer0_attention_weight_min": -14.390642166137695,
      "activations/layer10_attention_weight_max": 35.957252502441406,
      "activations/layer10_attention_weight_min": -36.88212966918945,
      "activations/layer11_attention_weight_max": 35.40034103393555,
      "activations/layer11_attention_weight_min": -37.17201232910156,
      "activations/layer12_attention_weight_max": 28.67608070373535,
      "activations/layer12_attention_weight_min": -28.150148391723633,
      "activations/layer13_attention_weight_max": 43.925575256347656,
      "activations/layer13_attention_weight_min": -38.404842376708984,
      "activations/layer14_attention_weight_max": 46.94776153564453,
      "activations/layer14_attention_weight_min": -42.933433532714844,
      "activations/layer15_attention_weight_max": 43.615074157714844,
      "activations/layer15_attention_weight_min": -37.782352447509766,
      "activations/layer16_attention_weight_max": 32.3037109375,
      "activations/layer16_attention_weight_min": -27.89333152770996,
      "activations/layer17_attention_weight_max": 31.23710823059082,
      "activations/layer17_attention_weight_min": -26.660327911376953,
      "activations/layer18_attention_weight_max": 30.289710998535156,
      "activations/layer18_attention_weight_min": -25.59191131591797,
      "activations/layer19_attention_weight_max": 32.13637924194336,
      "activations/layer19_attention_weight_min": -32.614784240722656,
      "activations/layer1_attention_weight_max": 16.114469528198242,
      "activations/layer1_attention_weight_min": -14.175041198730469,
      "activations/layer20_attention_weight_max": 28.04817771911621,
      "activations/layer20_attention_weight_min": -25.768709182739258,
      "activations/layer21_attention_weight_max": 27.794445037841797,
      "activations/layer21_attention_weight_min": -25.783349990844727,
      "activations/layer22_attention_weight_max": 46.1983757019043,
      "activations/layer22_attention_weight_min": -31.383766174316406,
      "activations/layer23_attention_weight_max": 32.14674377441406,
      "activations/layer23_attention_weight_min": -25.672805786132812,
      "activations/layer2_attention_weight_max": 35.06168746948242,
      "activations/layer2_attention_weight_min": -32.65685272216797,
      "activations/layer3_attention_weight_max": 93.60143280029297,
      "activations/layer3_attention_weight_min": -96.62361907958984,
      "activations/layer4_attention_weight_max": 101.70272827148438,
      "activations/layer4_attention_weight_min": -103.87920379638672,
      "activations/layer5_attention_weight_max": 49.23603439331055,
      "activations/layer5_attention_weight_min": -62.584495544433594,
      "activations/layer6_attention_weight_max": 42.99413299560547,
      "activations/layer6_attention_weight_min": -44.16267395019531,
      "activations/layer7_attention_weight_max": 91.85742950439453,
      "activations/layer7_attention_weight_min": -93.096923828125,
      "activations/layer8_attention_weight_max": 43.83002471923828,
      "activations/layer8_attention_weight_min": -43.70646667480469,
      "activations/layer9_attention_weight_max": 33.23806381225586,
      "activations/layer9_attention_weight_min": -37.39559555053711,
      "epoch": 18.33,
      "learning_rate": 3.2121969696969694e-05,
      "loss": 2.741,
      "step": 315450
    },
    {
      "activations/layer0_attention_weight_max": 15.922070503234863,
      "activations/layer0_attention_weight_min": -12.392731666564941,
      "activations/layer10_attention_weight_max": 31.984758377075195,
      "activations/layer10_attention_weight_min": -33.60894775390625,
      "activations/layer11_attention_weight_max": 33.813072204589844,
      "activations/layer11_attention_weight_min": -35.14254379272461,
      "activations/layer12_attention_weight_max": 26.76877212524414,
      "activations/layer12_attention_weight_min": -25.816768646240234,
      "activations/layer13_attention_weight_max": 38.593353271484375,
      "activations/layer13_attention_weight_min": -35.40502166748047,
      "activations/layer14_attention_weight_max": 43.64072799682617,
      "activations/layer14_attention_weight_min": -38.45030975341797,
      "activations/layer15_attention_weight_max": 39.01942825317383,
      "activations/layer15_attention_weight_min": -35.52662658691406,
      "activations/layer16_attention_weight_max": 29.372848510742188,
      "activations/layer16_attention_weight_min": -27.633594512939453,
      "activations/layer17_attention_weight_max": 28.020265579223633,
      "activations/layer17_attention_weight_min": -27.470008850097656,
      "activations/layer18_attention_weight_max": 29.93878746032715,
      "activations/layer18_attention_weight_min": -28.187850952148438,
      "activations/layer19_attention_weight_max": 29.890167236328125,
      "activations/layer19_attention_weight_min": -30.751758575439453,
      "activations/layer1_attention_weight_max": 15.904977798461914,
      "activations/layer1_attention_weight_min": -14.18891429901123,
      "activations/layer20_attention_weight_max": 25.186002731323242,
      "activations/layer20_attention_weight_min": -25.099708557128906,
      "activations/layer21_attention_weight_max": 27.309734344482422,
      "activations/layer21_attention_weight_min": -25.139549255371094,
      "activations/layer22_attention_weight_max": 43.4495849609375,
      "activations/layer22_attention_weight_min": -34.190895080566406,
      "activations/layer23_attention_weight_max": 29.79157066345215,
      "activations/layer23_attention_weight_min": -26.774808883666992,
      "activations/layer2_attention_weight_max": 32.429561614990234,
      "activations/layer2_attention_weight_min": -31.641719818115234,
      "activations/layer3_attention_weight_max": 100.10462951660156,
      "activations/layer3_attention_weight_min": -97.38957214355469,
      "activations/layer4_attention_weight_max": 106.66229248046875,
      "activations/layer4_attention_weight_min": -108.63582611083984,
      "activations/layer5_attention_weight_max": 53.157859802246094,
      "activations/layer5_attention_weight_min": -62.280574798583984,
      "activations/layer6_attention_weight_max": 46.68034744262695,
      "activations/layer6_attention_weight_min": -46.083377838134766,
      "activations/layer7_attention_weight_max": 95.54483032226562,
      "activations/layer7_attention_weight_min": -96.86111450195312,
      "activations/layer8_attention_weight_max": 46.0688362121582,
      "activations/layer8_attention_weight_min": -44.891517639160156,
      "activations/layer9_attention_weight_max": 33.556724548339844,
      "activations/layer9_attention_weight_min": -33.69277572631836,
      "epoch": 18.33,
      "learning_rate": 3.21030303030303e-05,
      "loss": 2.7187,
      "step": 315500
    },
    {
      "activations/layer0_attention_weight_max": 16.04308319091797,
      "activations/layer0_attention_weight_min": -12.592026710510254,
      "activations/layer10_attention_weight_max": 34.516300201416016,
      "activations/layer10_attention_weight_min": -35.3490104675293,
      "activations/layer11_attention_weight_max": 34.72670364379883,
      "activations/layer11_attention_weight_min": -33.671226501464844,
      "activations/layer12_attention_weight_max": 30.55734634399414,
      "activations/layer12_attention_weight_min": -26.275856018066406,
      "activations/layer13_attention_weight_max": 44.90330505371094,
      "activations/layer13_attention_weight_min": -34.09870147705078,
      "activations/layer14_attention_weight_max": 41.62936019897461,
      "activations/layer14_attention_weight_min": -38.735939025878906,
      "activations/layer15_attention_weight_max": 37.045654296875,
      "activations/layer15_attention_weight_min": -34.322452545166016,
      "activations/layer16_attention_weight_max": 27.43475341796875,
      "activations/layer16_attention_weight_min": -26.758136749267578,
      "activations/layer17_attention_weight_max": 27.620532989501953,
      "activations/layer17_attention_weight_min": -23.943445205688477,
      "activations/layer18_attention_weight_max": 29.75799560546875,
      "activations/layer18_attention_weight_min": -25.0816650390625,
      "activations/layer19_attention_weight_max": 31.955224990844727,
      "activations/layer19_attention_weight_min": -29.54145622253418,
      "activations/layer1_attention_weight_max": 15.715004920959473,
      "activations/layer1_attention_weight_min": -13.59803295135498,
      "activations/layer20_attention_weight_max": 25.876279830932617,
      "activations/layer20_attention_weight_min": -25.878761291503906,
      "activations/layer21_attention_weight_max": 25.65049171447754,
      "activations/layer21_attention_weight_min": -24.335704803466797,
      "activations/layer22_attention_weight_max": 35.16804122924805,
      "activations/layer22_attention_weight_min": -29.94245719909668,
      "activations/layer23_attention_weight_max": 27.917404174804688,
      "activations/layer23_attention_weight_min": -25.487730026245117,
      "activations/layer2_attention_weight_max": 31.061599731445312,
      "activations/layer2_attention_weight_min": -30.440250396728516,
      "activations/layer3_attention_weight_max": 97.51545715332031,
      "activations/layer3_attention_weight_min": -94.7326889038086,
      "activations/layer4_attention_weight_max": 105.7118148803711,
      "activations/layer4_attention_weight_min": -106.33831787109375,
      "activations/layer5_attention_weight_max": 47.9516716003418,
      "activations/layer5_attention_weight_min": -60.43292236328125,
      "activations/layer6_attention_weight_max": 42.52980041503906,
      "activations/layer6_attention_weight_min": -44.49940490722656,
      "activations/layer7_attention_weight_max": 92.3482666015625,
      "activations/layer7_attention_weight_min": -94.75442504882812,
      "activations/layer8_attention_weight_max": 42.601585388183594,
      "activations/layer8_attention_weight_min": -43.09001159667969,
      "activations/layer9_attention_weight_max": 34.088584899902344,
      "activations/layer9_attention_weight_min": -35.6419677734375,
      "epoch": 18.34,
      "learning_rate": 3.2084090909090904e-05,
      "loss": 2.748,
      "step": 315550
    },
    {
      "activations/layer0_attention_weight_max": 15.820663452148438,
      "activations/layer0_attention_weight_min": -14.307555198669434,
      "activations/layer10_attention_weight_max": 31.194175720214844,
      "activations/layer10_attention_weight_min": -34.39677047729492,
      "activations/layer11_attention_weight_max": 30.26119613647461,
      "activations/layer11_attention_weight_min": -32.8447151184082,
      "activations/layer12_attention_weight_max": 24.847124099731445,
      "activations/layer12_attention_weight_min": -26.731647491455078,
      "activations/layer13_attention_weight_max": 39.24536895751953,
      "activations/layer13_attention_weight_min": -34.39268493652344,
      "activations/layer14_attention_weight_max": 42.168094635009766,
      "activations/layer14_attention_weight_min": -37.907997131347656,
      "activations/layer15_attention_weight_max": 38.155113220214844,
      "activations/layer15_attention_weight_min": -35.43537139892578,
      "activations/layer16_attention_weight_max": 30.213594436645508,
      "activations/layer16_attention_weight_min": -28.575326919555664,
      "activations/layer17_attention_weight_max": 30.7042179107666,
      "activations/layer17_attention_weight_min": -26.36789894104004,
      "activations/layer18_attention_weight_max": 33.11791229248047,
      "activations/layer18_attention_weight_min": -29.234004974365234,
      "activations/layer19_attention_weight_max": 37.94368362426758,
      "activations/layer19_attention_weight_min": -32.98809051513672,
      "activations/layer1_attention_weight_max": 16.114280700683594,
      "activations/layer1_attention_weight_min": -14.540301322937012,
      "activations/layer20_attention_weight_max": 30.71797752380371,
      "activations/layer20_attention_weight_min": -26.239274978637695,
      "activations/layer21_attention_weight_max": 31.52676010131836,
      "activations/layer21_attention_weight_min": -26.697866439819336,
      "activations/layer22_attention_weight_max": 44.52638244628906,
      "activations/layer22_attention_weight_min": -33.37980270385742,
      "activations/layer23_attention_weight_max": 35.62276840209961,
      "activations/layer23_attention_weight_min": -28.897680282592773,
      "activations/layer2_attention_weight_max": 31.64375877380371,
      "activations/layer2_attention_weight_min": -28.878936767578125,
      "activations/layer3_attention_weight_max": 93.39362335205078,
      "activations/layer3_attention_weight_min": -91.39891815185547,
      "activations/layer4_attention_weight_max": 102.44404602050781,
      "activations/layer4_attention_weight_min": -103.61602020263672,
      "activations/layer5_attention_weight_max": 48.99267578125,
      "activations/layer5_attention_weight_min": -60.675106048583984,
      "activations/layer6_attention_weight_max": 41.17589569091797,
      "activations/layer6_attention_weight_min": -43.70450210571289,
      "activations/layer7_attention_weight_max": 86.63273620605469,
      "activations/layer7_attention_weight_min": -87.56393432617188,
      "activations/layer8_attention_weight_max": 39.465087890625,
      "activations/layer8_attention_weight_min": -41.055362701416016,
      "activations/layer9_attention_weight_max": 28.995298385620117,
      "activations/layer9_attention_weight_min": -32.849308013916016,
      "epoch": 18.34,
      "learning_rate": 3.206515151515151e-05,
      "loss": 2.7314,
      "step": 315600
    },
    {
      "activations/layer0_attention_weight_max": 15.943413734436035,
      "activations/layer0_attention_weight_min": -12.455842018127441,
      "activations/layer10_attention_weight_max": 34.30680847167969,
      "activations/layer10_attention_weight_min": -35.60270690917969,
      "activations/layer11_attention_weight_max": 34.55095672607422,
      "activations/layer11_attention_weight_min": -34.19720458984375,
      "activations/layer12_attention_weight_max": 31.44344711303711,
      "activations/layer12_attention_weight_min": -28.666574478149414,
      "activations/layer13_attention_weight_max": 44.662784576416016,
      "activations/layer13_attention_weight_min": -37.04767990112305,
      "activations/layer14_attention_weight_max": 48.85722351074219,
      "activations/layer14_attention_weight_min": -41.87446975708008,
      "activations/layer15_attention_weight_max": 46.07365036010742,
      "activations/layer15_attention_weight_min": -44.3184928894043,
      "activations/layer16_attention_weight_max": 36.97883605957031,
      "activations/layer16_attention_weight_min": -32.11869430541992,
      "activations/layer17_attention_weight_max": 35.54605484008789,
      "activations/layer17_attention_weight_min": -30.02968406677246,
      "activations/layer18_attention_weight_max": 39.767059326171875,
      "activations/layer18_attention_weight_min": -28.854890823364258,
      "activations/layer19_attention_weight_max": 42.12870788574219,
      "activations/layer19_attention_weight_min": -34.04832077026367,
      "activations/layer1_attention_weight_max": 17.28724479675293,
      "activations/layer1_attention_weight_min": -14.408870697021484,
      "activations/layer20_attention_weight_max": 37.90153503417969,
      "activations/layer20_attention_weight_min": -29.331113815307617,
      "activations/layer21_attention_weight_max": 37.23772048950195,
      "activations/layer21_attention_weight_min": -27.480613708496094,
      "activations/layer22_attention_weight_max": 50.67048645019531,
      "activations/layer22_attention_weight_min": -36.109703063964844,
      "activations/layer23_attention_weight_max": 39.27223587036133,
      "activations/layer23_attention_weight_min": -28.539562225341797,
      "activations/layer2_attention_weight_max": 31.631872177124023,
      "activations/layer2_attention_weight_min": -30.93027114868164,
      "activations/layer3_attention_weight_max": 92.33049774169922,
      "activations/layer3_attention_weight_min": -100.67389678955078,
      "activations/layer4_attention_weight_max": 101.69105529785156,
      "activations/layer4_attention_weight_min": -107.24470520019531,
      "activations/layer5_attention_weight_max": 49.96483612060547,
      "activations/layer5_attention_weight_min": -61.87016296386719,
      "activations/layer6_attention_weight_max": 44.547386169433594,
      "activations/layer6_attention_weight_min": -45.65827941894531,
      "activations/layer7_attention_weight_max": 92.78650665283203,
      "activations/layer7_attention_weight_min": -91.57437896728516,
      "activations/layer8_attention_weight_max": 40.73981475830078,
      "activations/layer8_attention_weight_min": -42.193058013916016,
      "activations/layer9_attention_weight_max": 33.65145492553711,
      "activations/layer9_attention_weight_min": -34.44738006591797,
      "epoch": 18.34,
      "learning_rate": 3.204621212121212e-05,
      "loss": 2.7336,
      "step": 315650
    },
    {
      "activations/layer0_attention_weight_max": 16.829435348510742,
      "activations/layer0_attention_weight_min": -12.974514961242676,
      "activations/layer10_attention_weight_max": 33.336395263671875,
      "activations/layer10_attention_weight_min": -34.89213943481445,
      "activations/layer11_attention_weight_max": 33.202850341796875,
      "activations/layer11_attention_weight_min": -34.23603820800781,
      "activations/layer12_attention_weight_max": 24.965145111083984,
      "activations/layer12_attention_weight_min": -25.682357788085938,
      "activations/layer13_attention_weight_max": 35.70951843261719,
      "activations/layer13_attention_weight_min": -35.29735565185547,
      "activations/layer14_attention_weight_max": 38.707420349121094,
      "activations/layer14_attention_weight_min": -38.266109466552734,
      "activations/layer15_attention_weight_max": 34.99805450439453,
      "activations/layer15_attention_weight_min": -34.50019836425781,
      "activations/layer16_attention_weight_max": 27.848886489868164,
      "activations/layer16_attention_weight_min": -28.15563201904297,
      "activations/layer17_attention_weight_max": 29.387672424316406,
      "activations/layer17_attention_weight_min": -26.053783416748047,
      "activations/layer18_attention_weight_max": 32.38937759399414,
      "activations/layer18_attention_weight_min": -26.37615203857422,
      "activations/layer19_attention_weight_max": 36.715579986572266,
      "activations/layer19_attention_weight_min": -31.72854995727539,
      "activations/layer1_attention_weight_max": 15.49887752532959,
      "activations/layer1_attention_weight_min": -13.850910186767578,
      "activations/layer20_attention_weight_max": 31.534210205078125,
      "activations/layer20_attention_weight_min": -26.4553279876709,
      "activations/layer21_attention_weight_max": 31.292667388916016,
      "activations/layer21_attention_weight_min": -24.810800552368164,
      "activations/layer22_attention_weight_max": 42.051692962646484,
      "activations/layer22_attention_weight_min": -32.509483337402344,
      "activations/layer23_attention_weight_max": 32.39754867553711,
      "activations/layer23_attention_weight_min": -27.146289825439453,
      "activations/layer2_attention_weight_max": 29.743696212768555,
      "activations/layer2_attention_weight_min": -30.789165496826172,
      "activations/layer3_attention_weight_max": 93.23606872558594,
      "activations/layer3_attention_weight_min": -97.44864654541016,
      "activations/layer4_attention_weight_max": 104.61724090576172,
      "activations/layer4_attention_weight_min": -111.45291900634766,
      "activations/layer5_attention_weight_max": 48.92289352416992,
      "activations/layer5_attention_weight_min": -62.99824523925781,
      "activations/layer6_attention_weight_max": 40.24506759643555,
      "activations/layer6_attention_weight_min": -46.855308532714844,
      "activations/layer7_attention_weight_max": 91.70793151855469,
      "activations/layer7_attention_weight_min": -91.88899230957031,
      "activations/layer8_attention_weight_max": 39.08061599731445,
      "activations/layer8_attention_weight_min": -41.34745788574219,
      "activations/layer9_attention_weight_max": 32.29523849487305,
      "activations/layer9_attention_weight_min": -32.94683837890625,
      "epoch": 18.34,
      "learning_rate": 3.202727272727273e-05,
      "loss": 2.7166,
      "step": 315700
    },
    {
      "activations/layer0_attention_weight_max": 17.22919464111328,
      "activations/layer0_attention_weight_min": -13.071938514709473,
      "activations/layer10_attention_weight_max": 35.77058029174805,
      "activations/layer10_attention_weight_min": -35.24663162231445,
      "activations/layer11_attention_weight_max": 36.158302307128906,
      "activations/layer11_attention_weight_min": -35.782920837402344,
      "activations/layer12_attention_weight_max": 26.089937210083008,
      "activations/layer12_attention_weight_min": -28.111608505249023,
      "activations/layer13_attention_weight_max": 38.4729118347168,
      "activations/layer13_attention_weight_min": -37.49357223510742,
      "activations/layer14_attention_weight_max": 41.4384651184082,
      "activations/layer14_attention_weight_min": -40.59743881225586,
      "activations/layer15_attention_weight_max": 39.21068572998047,
      "activations/layer15_attention_weight_min": -35.26320266723633,
      "activations/layer16_attention_weight_max": 30.464839935302734,
      "activations/layer16_attention_weight_min": -27.947362899780273,
      "activations/layer17_attention_weight_max": 28.871484756469727,
      "activations/layer17_attention_weight_min": -25.814523696899414,
      "activations/layer18_attention_weight_max": 31.163433074951172,
      "activations/layer18_attention_weight_min": -25.36693000793457,
      "activations/layer19_attention_weight_max": 33.8354377746582,
      "activations/layer19_attention_weight_min": -30.99384880065918,
      "activations/layer1_attention_weight_max": 16.027137756347656,
      "activations/layer1_attention_weight_min": -15.166278839111328,
      "activations/layer20_attention_weight_max": 28.53301239013672,
      "activations/layer20_attention_weight_min": -27.197608947753906,
      "activations/layer21_attention_weight_max": 27.759519577026367,
      "activations/layer21_attention_weight_min": -27.278844833374023,
      "activations/layer22_attention_weight_max": 42.95512771606445,
      "activations/layer22_attention_weight_min": -31.47719955444336,
      "activations/layer23_attention_weight_max": 34.48565673828125,
      "activations/layer23_attention_weight_min": -26.52967643737793,
      "activations/layer2_attention_weight_max": 29.842525482177734,
      "activations/layer2_attention_weight_min": -29.829397201538086,
      "activations/layer3_attention_weight_max": 95.61679077148438,
      "activations/layer3_attention_weight_min": -95.50970458984375,
      "activations/layer4_attention_weight_max": 110.78954315185547,
      "activations/layer4_attention_weight_min": -108.10589599609375,
      "activations/layer5_attention_weight_max": 53.39546203613281,
      "activations/layer5_attention_weight_min": -59.66984558105469,
      "activations/layer6_attention_weight_max": 45.10853958129883,
      "activations/layer6_attention_weight_min": -48.30518341064453,
      "activations/layer7_attention_weight_max": 97.45462036132812,
      "activations/layer7_attention_weight_min": -94.40275573730469,
      "activations/layer8_attention_weight_max": 42.63385009765625,
      "activations/layer8_attention_weight_min": -44.327728271484375,
      "activations/layer9_attention_weight_max": 32.91520309448242,
      "activations/layer9_attention_weight_min": -34.881168365478516,
      "epoch": 18.35,
      "learning_rate": 3.200833333333333e-05,
      "loss": 2.7206,
      "step": 315750
    },
    {
      "activations/layer0_attention_weight_max": 15.415878295898438,
      "activations/layer0_attention_weight_min": -13.475909233093262,
      "activations/layer10_attention_weight_max": 54.4471549987793,
      "activations/layer10_attention_weight_min": -50.54489517211914,
      "activations/layer11_attention_weight_max": 57.40961837768555,
      "activations/layer11_attention_weight_min": -53.671852111816406,
      "activations/layer12_attention_weight_max": 33.12343978881836,
      "activations/layer12_attention_weight_min": -40.53234100341797,
      "activations/layer13_attention_weight_max": 47.533077239990234,
      "activations/layer13_attention_weight_min": -39.889312744140625,
      "activations/layer14_attention_weight_max": 53.20446014404297,
      "activations/layer14_attention_weight_min": -44.201416015625,
      "activations/layer15_attention_weight_max": 55.483123779296875,
      "activations/layer15_attention_weight_min": -45.86932373046875,
      "activations/layer16_attention_weight_max": 36.83980178833008,
      "activations/layer16_attention_weight_min": -29.436189651489258,
      "activations/layer17_attention_weight_max": 30.431650161743164,
      "activations/layer17_attention_weight_min": -29.295644760131836,
      "activations/layer18_attention_weight_max": 32.17336654663086,
      "activations/layer18_attention_weight_min": -28.42119026184082,
      "activations/layer19_attention_weight_max": 39.92186737060547,
      "activations/layer19_attention_weight_min": -34.527278900146484,
      "activations/layer1_attention_weight_max": 16.89337158203125,
      "activations/layer1_attention_weight_min": -14.290882110595703,
      "activations/layer20_attention_weight_max": 31.832006454467773,
      "activations/layer20_attention_weight_min": -24.8724365234375,
      "activations/layer21_attention_weight_max": 29.254716873168945,
      "activations/layer21_attention_weight_min": -24.414018630981445,
      "activations/layer22_attention_weight_max": 43.004737854003906,
      "activations/layer22_attention_weight_min": -33.13599395751953,
      "activations/layer23_attention_weight_max": 30.453895568847656,
      "activations/layer23_attention_weight_min": -25.47774314880371,
      "activations/layer2_attention_weight_max": 32.11609649658203,
      "activations/layer2_attention_weight_min": -34.90286636352539,
      "activations/layer3_attention_weight_max": 101.4412612915039,
      "activations/layer3_attention_weight_min": -109.242431640625,
      "activations/layer4_attention_weight_max": 110.72185516357422,
      "activations/layer4_attention_weight_min": -109.66017150878906,
      "activations/layer5_attention_weight_max": 54.24078369140625,
      "activations/layer5_attention_weight_min": -65.5147476196289,
      "activations/layer6_attention_weight_max": 46.179481506347656,
      "activations/layer6_attention_weight_min": -49.725929260253906,
      "activations/layer7_attention_weight_max": 117.69685363769531,
      "activations/layer7_attention_weight_min": -109.17975616455078,
      "activations/layer8_attention_weight_max": 55.636592864990234,
      "activations/layer8_attention_weight_min": -54.393489837646484,
      "activations/layer9_attention_weight_max": 45.29281997680664,
      "activations/layer9_attention_weight_min": -46.41566467285156,
      "epoch": 18.35,
      "learning_rate": 3.198939393939394e-05,
      "loss": 2.7464,
      "step": 315800
    },
    {
      "activations/layer0_attention_weight_max": 16.870315551757812,
      "activations/layer0_attention_weight_min": -13.229715347290039,
      "activations/layer10_attention_weight_max": 34.80347442626953,
      "activations/layer10_attention_weight_min": -35.267425537109375,
      "activations/layer11_attention_weight_max": 33.89911651611328,
      "activations/layer11_attention_weight_min": -34.67945098876953,
      "activations/layer12_attention_weight_max": 26.92066192626953,
      "activations/layer12_attention_weight_min": -26.883153915405273,
      "activations/layer13_attention_weight_max": 40.6944694519043,
      "activations/layer13_attention_weight_min": -36.751461029052734,
      "activations/layer14_attention_weight_max": 41.75028610229492,
      "activations/layer14_attention_weight_min": -39.00346374511719,
      "activations/layer15_attention_weight_max": 38.90211868286133,
      "activations/layer15_attention_weight_min": -37.064605712890625,
      "activations/layer16_attention_weight_max": 28.146697998046875,
      "activations/layer16_attention_weight_min": -31.441434860229492,
      "activations/layer17_attention_weight_max": 27.517274856567383,
      "activations/layer17_attention_weight_min": -26.84993553161621,
      "activations/layer18_attention_weight_max": 31.086959838867188,
      "activations/layer18_attention_weight_min": -25.393545150756836,
      "activations/layer19_attention_weight_max": 31.35181999206543,
      "activations/layer19_attention_weight_min": -31.52925682067871,
      "activations/layer1_attention_weight_max": 16.747478485107422,
      "activations/layer1_attention_weight_min": -14.50815200805664,
      "activations/layer20_attention_weight_max": 26.001474380493164,
      "activations/layer20_attention_weight_min": -25.350069046020508,
      "activations/layer21_attention_weight_max": 25.756261825561523,
      "activations/layer21_attention_weight_min": -25.363204956054688,
      "activations/layer22_attention_weight_max": 37.40152359008789,
      "activations/layer22_attention_weight_min": -30.315656661987305,
      "activations/layer23_attention_weight_max": 31.911334991455078,
      "activations/layer23_attention_weight_min": -25.76521873474121,
      "activations/layer2_attention_weight_max": 32.444679260253906,
      "activations/layer2_attention_weight_min": -32.06366729736328,
      "activations/layer3_attention_weight_max": 96.15126037597656,
      "activations/layer3_attention_weight_min": -102.41016387939453,
      "activations/layer4_attention_weight_max": 107.50331115722656,
      "activations/layer4_attention_weight_min": -106.8762435913086,
      "activations/layer5_attention_weight_max": 50.366371154785156,
      "activations/layer5_attention_weight_min": -60.592132568359375,
      "activations/layer6_attention_weight_max": 45.16748809814453,
      "activations/layer6_attention_weight_min": -47.950645446777344,
      "activations/layer7_attention_weight_max": 95.16313934326172,
      "activations/layer7_attention_weight_min": -99.81097412109375,
      "activations/layer8_attention_weight_max": 43.8858642578125,
      "activations/layer8_attention_weight_min": -45.46360778808594,
      "activations/layer9_attention_weight_max": 33.14832305908203,
      "activations/layer9_attention_weight_min": -36.08397674560547,
      "epoch": 18.35,
      "learning_rate": 3.197045454545454e-05,
      "loss": 2.7447,
      "step": 315850
    },
    {
      "activations/layer0_attention_weight_max": 17.050918579101562,
      "activations/layer0_attention_weight_min": -14.339956283569336,
      "activations/layer10_attention_weight_max": 35.32975387573242,
      "activations/layer10_attention_weight_min": -36.14021682739258,
      "activations/layer11_attention_weight_max": 36.17550277709961,
      "activations/layer11_attention_weight_min": -36.75217056274414,
      "activations/layer12_attention_weight_max": 27.822532653808594,
      "activations/layer12_attention_weight_min": -26.781892776489258,
      "activations/layer13_attention_weight_max": 40.714351654052734,
      "activations/layer13_attention_weight_min": -36.88689422607422,
      "activations/layer14_attention_weight_max": 43.213958740234375,
      "activations/layer14_attention_weight_min": -36.4046630859375,
      "activations/layer15_attention_weight_max": 41.549903869628906,
      "activations/layer15_attention_weight_min": -36.27872085571289,
      "activations/layer16_attention_weight_max": 28.790714263916016,
      "activations/layer16_attention_weight_min": -27.372087478637695,
      "activations/layer17_attention_weight_max": 27.70745277404785,
      "activations/layer17_attention_weight_min": -25.843875885009766,
      "activations/layer18_attention_weight_max": 30.629060745239258,
      "activations/layer18_attention_weight_min": -25.820556640625,
      "activations/layer19_attention_weight_max": 32.22176742553711,
      "activations/layer19_attention_weight_min": -30.12684440612793,
      "activations/layer1_attention_weight_max": 17.591144561767578,
      "activations/layer1_attention_weight_min": -15.49905776977539,
      "activations/layer20_attention_weight_max": 26.707963943481445,
      "activations/layer20_attention_weight_min": -25.479337692260742,
      "activations/layer21_attention_weight_max": 26.92388153076172,
      "activations/layer21_attention_weight_min": -24.421661376953125,
      "activations/layer22_attention_weight_max": 37.904178619384766,
      "activations/layer22_attention_weight_min": -30.32077980041504,
      "activations/layer23_attention_weight_max": 33.1329345703125,
      "activations/layer23_attention_weight_min": -25.367080688476562,
      "activations/layer2_attention_weight_max": 33.765411376953125,
      "activations/layer2_attention_weight_min": -32.34547424316406,
      "activations/layer3_attention_weight_max": 103.0241928100586,
      "activations/layer3_attention_weight_min": -101.88654327392578,
      "activations/layer4_attention_weight_max": 112.78743743896484,
      "activations/layer4_attention_weight_min": -109.91395568847656,
      "activations/layer5_attention_weight_max": 51.51396942138672,
      "activations/layer5_attention_weight_min": -63.00002670288086,
      "activations/layer6_attention_weight_max": 46.70689392089844,
      "activations/layer6_attention_weight_min": -49.934146881103516,
      "activations/layer7_attention_weight_max": 97.98655700683594,
      "activations/layer7_attention_weight_min": -97.49701690673828,
      "activations/layer8_attention_weight_max": 43.701656341552734,
      "activations/layer8_attention_weight_min": -45.548927307128906,
      "activations/layer9_attention_weight_max": 34.431880950927734,
      "activations/layer9_attention_weight_min": -36.71710968017578,
      "epoch": 18.36,
      "learning_rate": 3.195151515151515e-05,
      "loss": 2.7278,
      "step": 315900
    },
    {
      "activations/layer0_attention_weight_max": 16.860994338989258,
      "activations/layer0_attention_weight_min": -12.306327819824219,
      "activations/layer10_attention_weight_max": 32.73877716064453,
      "activations/layer10_attention_weight_min": -32.93196105957031,
      "activations/layer11_attention_weight_max": 31.621280670166016,
      "activations/layer11_attention_weight_min": -33.755821228027344,
      "activations/layer12_attention_weight_max": 23.995431900024414,
      "activations/layer12_attention_weight_min": -25.5436954498291,
      "activations/layer13_attention_weight_max": 36.1485710144043,
      "activations/layer13_attention_weight_min": -33.224605560302734,
      "activations/layer14_attention_weight_max": 37.69684982299805,
      "activations/layer14_attention_weight_min": -34.57283020019531,
      "activations/layer15_attention_weight_max": 31.444063186645508,
      "activations/layer15_attention_weight_min": -33.45576858520508,
      "activations/layer16_attention_weight_max": 25.838115692138672,
      "activations/layer16_attention_weight_min": -29.244033813476562,
      "activations/layer17_attention_weight_max": 27.960214614868164,
      "activations/layer17_attention_weight_min": -25.831892013549805,
      "activations/layer18_attention_weight_max": 28.987430572509766,
      "activations/layer18_attention_weight_min": -25.324813842773438,
      "activations/layer19_attention_weight_max": 29.54119300842285,
      "activations/layer19_attention_weight_min": -30.405593872070312,
      "activations/layer1_attention_weight_max": 16.323169708251953,
      "activations/layer1_attention_weight_min": -12.575366973876953,
      "activations/layer20_attention_weight_max": 27.150407791137695,
      "activations/layer20_attention_weight_min": -24.27886962890625,
      "activations/layer21_attention_weight_max": 26.60333824157715,
      "activations/layer21_attention_weight_min": -24.94601821899414,
      "activations/layer22_attention_weight_max": 40.371726989746094,
      "activations/layer22_attention_weight_min": -31.861881256103516,
      "activations/layer23_attention_weight_max": 30.999252319335938,
      "activations/layer23_attention_weight_min": -25.703222274780273,
      "activations/layer2_attention_weight_max": 31.945287704467773,
      "activations/layer2_attention_weight_min": -31.68435287475586,
      "activations/layer3_attention_weight_max": 96.42806243896484,
      "activations/layer3_attention_weight_min": -93.32103729248047,
      "activations/layer4_attention_weight_max": 103.05033111572266,
      "activations/layer4_attention_weight_min": -104.72584533691406,
      "activations/layer5_attention_weight_max": 49.06644821166992,
      "activations/layer5_attention_weight_min": -59.96291732788086,
      "activations/layer6_attention_weight_max": 43.43424606323242,
      "activations/layer6_attention_weight_min": -44.16703796386719,
      "activations/layer7_attention_weight_max": 83.22212219238281,
      "activations/layer7_attention_weight_min": -89.79271697998047,
      "activations/layer8_attention_weight_max": 38.15262985229492,
      "activations/layer8_attention_weight_min": -42.93901824951172,
      "activations/layer9_attention_weight_max": 29.339521408081055,
      "activations/layer9_attention_weight_min": -32.75126266479492,
      "epoch": 18.36,
      "learning_rate": 3.1932575757575756e-05,
      "loss": 2.7311,
      "step": 315950
    },
    {
      "activations/layer0_attention_weight_max": 16.48705291748047,
      "activations/layer0_attention_weight_min": -12.244876861572266,
      "activations/layer10_attention_weight_max": 34.04433059692383,
      "activations/layer10_attention_weight_min": -33.38960647583008,
      "activations/layer11_attention_weight_max": 33.34889221191406,
      "activations/layer11_attention_weight_min": -33.38838195800781,
      "activations/layer12_attention_weight_max": 28.099321365356445,
      "activations/layer12_attention_weight_min": -26.259403228759766,
      "activations/layer13_attention_weight_max": 43.272335052490234,
      "activations/layer13_attention_weight_min": -36.308013916015625,
      "activations/layer14_attention_weight_max": 45.48627853393555,
      "activations/layer14_attention_weight_min": -39.64742660522461,
      "activations/layer15_attention_weight_max": 41.481414794921875,
      "activations/layer15_attention_weight_min": -37.421241760253906,
      "activations/layer16_attention_weight_max": 30.72552490234375,
      "activations/layer16_attention_weight_min": -28.884613037109375,
      "activations/layer17_attention_weight_max": 29.53399085998535,
      "activations/layer17_attention_weight_min": -26.54360580444336,
      "activations/layer18_attention_weight_max": 30.650590896606445,
      "activations/layer18_attention_weight_min": -25.566118240356445,
      "activations/layer19_attention_weight_max": 34.65262222290039,
      "activations/layer19_attention_weight_min": -32.224822998046875,
      "activations/layer1_attention_weight_max": 17.011449813842773,
      "activations/layer1_attention_weight_min": -14.625506401062012,
      "activations/layer20_attention_weight_max": 28.908336639404297,
      "activations/layer20_attention_weight_min": -27.426950454711914,
      "activations/layer21_attention_weight_max": 28.705583572387695,
      "activations/layer21_attention_weight_min": -25.37234878540039,
      "activations/layer22_attention_weight_max": 41.106197357177734,
      "activations/layer22_attention_weight_min": -32.78290557861328,
      "activations/layer23_attention_weight_max": 31.12706756591797,
      "activations/layer23_attention_weight_min": -24.879796981811523,
      "activations/layer2_attention_weight_max": 29.469432830810547,
      "activations/layer2_attention_weight_min": -30.79999351501465,
      "activations/layer3_attention_weight_max": 97.31521606445312,
      "activations/layer3_attention_weight_min": -98.48150634765625,
      "activations/layer4_attention_weight_max": 105.67064666748047,
      "activations/layer4_attention_weight_min": -104.1122055053711,
      "activations/layer5_attention_weight_max": 50.76310348510742,
      "activations/layer5_attention_weight_min": -61.15745544433594,
      "activations/layer6_attention_weight_max": 44.44603729248047,
      "activations/layer6_attention_weight_min": -47.6865119934082,
      "activations/layer7_attention_weight_max": 91.54661560058594,
      "activations/layer7_attention_weight_min": -97.172119140625,
      "activations/layer8_attention_weight_max": 40.82183837890625,
      "activations/layer8_attention_weight_min": -43.089935302734375,
      "activations/layer9_attention_weight_max": 33.23857879638672,
      "activations/layer9_attention_weight_min": -33.3282585144043,
      "epoch": 18.36,
      "learning_rate": 3.191363636363636e-05,
      "loss": 2.7291,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_loss": 2.6875,
      "eval_runtime": 8.5167,
      "eval_samples_per_second": 504.184,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.5167,
      "eval_openwebtext_samples_per_second": 504.184,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9817,
      "eval_wikitext_samples_per_second": 230.109,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_lambada_loss": 2.3359375,
      "eval_lambada_ppl": 10.33914833878563,
      "eval_lambada_runtime": 9.5579,
      "eval_lambada_samples_per_second": 509.421,
      "step": 316000
    },
    {
      "activations/layer0_attention_weight_max": 16.084707260131836,
      "activations/layer0_attention_weight_min": -13.34518051147461,
      "activations/layer10_attention_weight_max": 38.182098388671875,
      "activations/layer10_attention_weight_min": -36.50968551635742,
      "activations/layer11_attention_weight_max": 38.192283630371094,
      "activations/layer11_attention_weight_min": -35.99087142944336,
      "activations/layer12_attention_weight_max": 28.5212459564209,
      "activations/layer12_attention_weight_min": -27.46207046508789,
      "activations/layer13_attention_weight_max": 41.89474105834961,
      "activations/layer13_attention_weight_min": -37.6641731262207,
      "activations/layer14_attention_weight_max": 45.593658447265625,
      "activations/layer14_attention_weight_min": -40.8907356262207,
      "activations/layer15_attention_weight_max": 44.116153717041016,
      "activations/layer15_attention_weight_min": -37.410865783691406,
      "activations/layer16_attention_weight_max": 31.62233543395996,
      "activations/layer16_attention_weight_min": -26.671518325805664,
      "activations/layer17_attention_weight_max": 32.46233367919922,
      "activations/layer17_attention_weight_min": -25.84614372253418,
      "activations/layer18_attention_weight_max": 31.38823127746582,
      "activations/layer18_attention_weight_min": -27.168216705322266,
      "activations/layer19_attention_weight_max": 38.98723602294922,
      "activations/layer19_attention_weight_min": -33.556976318359375,
      "activations/layer1_attention_weight_max": 16.88495445251465,
      "activations/layer1_attention_weight_min": -15.176027297973633,
      "activations/layer20_attention_weight_max": 30.940828323364258,
      "activations/layer20_attention_weight_min": -25.8919677734375,
      "activations/layer21_attention_weight_max": 30.30386734008789,
      "activations/layer21_attention_weight_min": -25.137760162353516,
      "activations/layer22_attention_weight_max": 44.62896728515625,
      "activations/layer22_attention_weight_min": -34.48938751220703,
      "activations/layer23_attention_weight_max": 35.01654815673828,
      "activations/layer23_attention_weight_min": -27.704418182373047,
      "activations/layer2_attention_weight_max": 33.46727752685547,
      "activations/layer2_attention_weight_min": -33.93202209472656,
      "activations/layer3_attention_weight_max": 97.49677276611328,
      "activations/layer3_attention_weight_min": -99.2418441772461,
      "activations/layer4_attention_weight_max": 108.0843505859375,
      "activations/layer4_attention_weight_min": -114.31744384765625,
      "activations/layer5_attention_weight_max": 51.45713424682617,
      "activations/layer5_attention_weight_min": -64.13814544677734,
      "activations/layer6_attention_weight_max": 44.597469329833984,
      "activations/layer6_attention_weight_min": -48.423622131347656,
      "activations/layer7_attention_weight_max": 93.75743865966797,
      "activations/layer7_attention_weight_min": -96.18244171142578,
      "activations/layer8_attention_weight_max": 42.535606384277344,
      "activations/layer8_attention_weight_min": -44.725624084472656,
      "activations/layer9_attention_weight_max": 31.70192527770996,
      "activations/layer9_attention_weight_min": -35.299800872802734,
      "epoch": 18.36,
      "learning_rate": 3.1894696969696966e-05,
      "loss": 2.7234,
      "step": 316050
    },
    {
      "activations/layer0_attention_weight_max": 17.200807571411133,
      "activations/layer0_attention_weight_min": -12.533988952636719,
      "activations/layer10_attention_weight_max": 49.19516372680664,
      "activations/layer10_attention_weight_min": -45.04102325439453,
      "activations/layer11_attention_weight_max": 50.097381591796875,
      "activations/layer11_attention_weight_min": -42.813026428222656,
      "activations/layer12_attention_weight_max": 34.4570198059082,
      "activations/layer12_attention_weight_min": -28.527725219726562,
      "activations/layer13_attention_weight_max": 49.1381721496582,
      "activations/layer13_attention_weight_min": -40.03852462768555,
      "activations/layer14_attention_weight_max": 59.5887336730957,
      "activations/layer14_attention_weight_min": -44.09346389770508,
      "activations/layer15_attention_weight_max": 58.51667404174805,
      "activations/layer15_attention_weight_min": -43.447574615478516,
      "activations/layer16_attention_weight_max": 41.26049041748047,
      "activations/layer16_attention_weight_min": -30.11817169189453,
      "activations/layer17_attention_weight_max": 37.76655960083008,
      "activations/layer17_attention_weight_min": -30.618595123291016,
      "activations/layer18_attention_weight_max": 39.35871124267578,
      "activations/layer18_attention_weight_min": -32.50847244262695,
      "activations/layer19_attention_weight_max": 50.425437927246094,
      "activations/layer19_attention_weight_min": -35.291805267333984,
      "activations/layer1_attention_weight_max": 18.076372146606445,
      "activations/layer1_attention_weight_min": -16.30771827697754,
      "activations/layer20_attention_weight_max": 40.51058578491211,
      "activations/layer20_attention_weight_min": -27.15593147277832,
      "activations/layer21_attention_weight_max": 39.98862075805664,
      "activations/layer21_attention_weight_min": -30.628286361694336,
      "activations/layer22_attention_weight_max": 64.1160659790039,
      "activations/layer22_attention_weight_min": -40.88351058959961,
      "activations/layer23_attention_weight_max": 43.99374008178711,
      "activations/layer23_attention_weight_min": -29.729366302490234,
      "activations/layer2_attention_weight_max": 38.34515380859375,
      "activations/layer2_attention_weight_min": -32.590911865234375,
      "activations/layer3_attention_weight_max": 110.86211395263672,
      "activations/layer3_attention_weight_min": -103.66746520996094,
      "activations/layer4_attention_weight_max": 112.1749267578125,
      "activations/layer4_attention_weight_min": -109.7056655883789,
      "activations/layer5_attention_weight_max": 52.65303039550781,
      "activations/layer5_attention_weight_min": -59.20881652832031,
      "activations/layer6_attention_weight_max": 44.4481315612793,
      "activations/layer6_attention_weight_min": -45.26398468017578,
      "activations/layer7_attention_weight_max": 109.94898223876953,
      "activations/layer7_attention_weight_min": -93.77845001220703,
      "activations/layer8_attention_weight_max": 54.85085678100586,
      "activations/layer8_attention_weight_min": -47.98960494995117,
      "activations/layer9_attention_weight_max": 41.02426528930664,
      "activations/layer9_attention_weight_min": -40.70013427734375,
      "epoch": 18.37,
      "learning_rate": 3.1875757575757574e-05,
      "loss": 2.7258,
      "step": 316100
    },
    {
      "activations/layer0_attention_weight_max": 16.492464065551758,
      "activations/layer0_attention_weight_min": -14.802841186523438,
      "activations/layer10_attention_weight_max": 34.55305862426758,
      "activations/layer10_attention_weight_min": -33.78435516357422,
      "activations/layer11_attention_weight_max": 32.477760314941406,
      "activations/layer11_attention_weight_min": -33.298213958740234,
      "activations/layer12_attention_weight_max": 26.10739517211914,
      "activations/layer12_attention_weight_min": -27.612653732299805,
      "activations/layer13_attention_weight_max": 38.48857498168945,
      "activations/layer13_attention_weight_min": -36.557044982910156,
      "activations/layer14_attention_weight_max": 46.23457336425781,
      "activations/layer14_attention_weight_min": -38.16842269897461,
      "activations/layer15_attention_weight_max": 38.88035583496094,
      "activations/layer15_attention_weight_min": -34.697933197021484,
      "activations/layer16_attention_weight_max": 29.819488525390625,
      "activations/layer16_attention_weight_min": -29.958253860473633,
      "activations/layer17_attention_weight_max": 32.142173767089844,
      "activations/layer17_attention_weight_min": -27.348493576049805,
      "activations/layer18_attention_weight_max": 33.8990592956543,
      "activations/layer18_attention_weight_min": -29.506052017211914,
      "activations/layer19_attention_weight_max": 33.58292770385742,
      "activations/layer19_attention_weight_min": -33.00653076171875,
      "activations/layer1_attention_weight_max": 17.178009033203125,
      "activations/layer1_attention_weight_min": -14.6201810836792,
      "activations/layer20_attention_weight_max": 27.81065559387207,
      "activations/layer20_attention_weight_min": -25.48663330078125,
      "activations/layer21_attention_weight_max": 28.691951751708984,
      "activations/layer21_attention_weight_min": -25.742782592773438,
      "activations/layer22_attention_weight_max": 44.634925842285156,
      "activations/layer22_attention_weight_min": -31.644678115844727,
      "activations/layer23_attention_weight_max": 32.21919631958008,
      "activations/layer23_attention_weight_min": -27.495311737060547,
      "activations/layer2_attention_weight_max": 30.486656188964844,
      "activations/layer2_attention_weight_min": -30.97567367553711,
      "activations/layer3_attention_weight_max": 95.67815399169922,
      "activations/layer3_attention_weight_min": -93.56826782226562,
      "activations/layer4_attention_weight_max": 105.13384246826172,
      "activations/layer4_attention_weight_min": -105.0908203125,
      "activations/layer5_attention_weight_max": 48.762969970703125,
      "activations/layer5_attention_weight_min": -61.52610778808594,
      "activations/layer6_attention_weight_max": 43.61398696899414,
      "activations/layer6_attention_weight_min": -45.71649932861328,
      "activations/layer7_attention_weight_max": 91.62117004394531,
      "activations/layer7_attention_weight_min": -95.2399673461914,
      "activations/layer8_attention_weight_max": 40.542118072509766,
      "activations/layer8_attention_weight_min": -42.707725524902344,
      "activations/layer9_attention_weight_max": 31.053104400634766,
      "activations/layer9_attention_weight_min": -33.401275634765625,
      "epoch": 18.37,
      "learning_rate": 3.185681818181818e-05,
      "loss": 2.7324,
      "step": 316150
    },
    {
      "activations/layer0_attention_weight_max": 17.189016342163086,
      "activations/layer0_attention_weight_min": -12.859436988830566,
      "activations/layer10_attention_weight_max": 33.12458801269531,
      "activations/layer10_attention_weight_min": -37.61082077026367,
      "activations/layer11_attention_weight_max": 34.604576110839844,
      "activations/layer11_attention_weight_min": -37.64625549316406,
      "activations/layer12_attention_weight_max": 24.970117568969727,
      "activations/layer12_attention_weight_min": -26.76563835144043,
      "activations/layer13_attention_weight_max": 39.805110931396484,
      "activations/layer13_attention_weight_min": -36.38690948486328,
      "activations/layer14_attention_weight_max": 42.213829040527344,
      "activations/layer14_attention_weight_min": -36.693058013916016,
      "activations/layer15_attention_weight_max": 36.88416290283203,
      "activations/layer15_attention_weight_min": -35.44396209716797,
      "activations/layer16_attention_weight_max": 27.83985710144043,
      "activations/layer16_attention_weight_min": -26.04979133605957,
      "activations/layer17_attention_weight_max": 29.848249435424805,
      "activations/layer17_attention_weight_min": -25.502626419067383,
      "activations/layer18_attention_weight_max": 32.87611770629883,
      "activations/layer18_attention_weight_min": -26.3917293548584,
      "activations/layer19_attention_weight_max": 34.81462097167969,
      "activations/layer19_attention_weight_min": -31.083740234375,
      "activations/layer1_attention_weight_max": 17.072017669677734,
      "activations/layer1_attention_weight_min": -14.805154800415039,
      "activations/layer20_attention_weight_max": 27.28447151184082,
      "activations/layer20_attention_weight_min": -24.663700103759766,
      "activations/layer21_attention_weight_max": 27.734725952148438,
      "activations/layer21_attention_weight_min": -23.81557273864746,
      "activations/layer22_attention_weight_max": 38.83626937866211,
      "activations/layer22_attention_weight_min": -33.0407600402832,
      "activations/layer23_attention_weight_max": 31.830501556396484,
      "activations/layer23_attention_weight_min": -26.591726303100586,
      "activations/layer2_attention_weight_max": 33.783973693847656,
      "activations/layer2_attention_weight_min": -32.88605499267578,
      "activations/layer3_attention_weight_max": 100.88624572753906,
      "activations/layer3_attention_weight_min": -99.14665985107422,
      "activations/layer4_attention_weight_max": 107.794921875,
      "activations/layer4_attention_weight_min": -106.85030364990234,
      "activations/layer5_attention_weight_max": 48.28486251831055,
      "activations/layer5_attention_weight_min": -62.021697998046875,
      "activations/layer6_attention_weight_max": 45.48782730102539,
      "activations/layer6_attention_weight_min": -46.64592742919922,
      "activations/layer7_attention_weight_max": 94.91171264648438,
      "activations/layer7_attention_weight_min": -98.13619232177734,
      "activations/layer8_attention_weight_max": 40.89741516113281,
      "activations/layer8_attention_weight_min": -45.77827453613281,
      "activations/layer9_attention_weight_max": 33.57988739013672,
      "activations/layer9_attention_weight_min": -37.63364791870117,
      "epoch": 18.37,
      "learning_rate": 3.1837878787878784e-05,
      "loss": 2.7431,
      "step": 316200
    },
    {
      "activations/layer0_attention_weight_max": 16.172225952148438,
      "activations/layer0_attention_weight_min": -12.276301383972168,
      "activations/layer10_attention_weight_max": 31.859317779541016,
      "activations/layer10_attention_weight_min": -34.08143997192383,
      "activations/layer11_attention_weight_max": 34.317604064941406,
      "activations/layer11_attention_weight_min": -33.54742431640625,
      "activations/layer12_attention_weight_max": 25.543787002563477,
      "activations/layer12_attention_weight_min": -32.03963088989258,
      "activations/layer13_attention_weight_max": 39.767574310302734,
      "activations/layer13_attention_weight_min": -40.94901657104492,
      "activations/layer14_attention_weight_max": 43.27671813964844,
      "activations/layer14_attention_weight_min": -40.48771667480469,
      "activations/layer15_attention_weight_max": 39.59036636352539,
      "activations/layer15_attention_weight_min": -38.72377014160156,
      "activations/layer16_attention_weight_max": 31.780574798583984,
      "activations/layer16_attention_weight_min": -28.05040740966797,
      "activations/layer17_attention_weight_max": 30.178646087646484,
      "activations/layer17_attention_weight_min": -26.73532485961914,
      "activations/layer18_attention_weight_max": 31.96468734741211,
      "activations/layer18_attention_weight_min": -25.819164276123047,
      "activations/layer19_attention_weight_max": 34.961856842041016,
      "activations/layer19_attention_weight_min": -33.1759033203125,
      "activations/layer1_attention_weight_max": 16.791746139526367,
      "activations/layer1_attention_weight_min": -14.777213096618652,
      "activations/layer20_attention_weight_max": 25.90459632873535,
      "activations/layer20_attention_weight_min": -25.567781448364258,
      "activations/layer21_attention_weight_max": 25.890758514404297,
      "activations/layer21_attention_weight_min": -25.376285552978516,
      "activations/layer22_attention_weight_max": 45.2398567199707,
      "activations/layer22_attention_weight_min": -33.19358444213867,
      "activations/layer23_attention_weight_max": 30.149139404296875,
      "activations/layer23_attention_weight_min": -27.42345428466797,
      "activations/layer2_attention_weight_max": 31.257131576538086,
      "activations/layer2_attention_weight_min": -31.775646209716797,
      "activations/layer3_attention_weight_max": 95.1485824584961,
      "activations/layer3_attention_weight_min": -93.38829040527344,
      "activations/layer4_attention_weight_max": 102.63802337646484,
      "activations/layer4_attention_weight_min": -104.0606918334961,
      "activations/layer5_attention_weight_max": 48.58259201049805,
      "activations/layer5_attention_weight_min": -62.070045471191406,
      "activations/layer6_attention_weight_max": 42.18830490112305,
      "activations/layer6_attention_weight_min": -45.197059631347656,
      "activations/layer7_attention_weight_max": 88.39350891113281,
      "activations/layer7_attention_weight_min": -89.8559799194336,
      "activations/layer8_attention_weight_max": 39.69267654418945,
      "activations/layer8_attention_weight_min": -42.4169807434082,
      "activations/layer9_attention_weight_max": 30.29693031311035,
      "activations/layer9_attention_weight_min": -33.75248336791992,
      "epoch": 18.38,
      "learning_rate": 3.181893939393939e-05,
      "loss": 2.7235,
      "step": 316250
    },
    {
      "activations/layer0_attention_weight_max": 16.42059898376465,
      "activations/layer0_attention_weight_min": -13.110918045043945,
      "activations/layer10_attention_weight_max": 35.21120071411133,
      "activations/layer10_attention_weight_min": -35.585304260253906,
      "activations/layer11_attention_weight_max": 35.94452667236328,
      "activations/layer11_attention_weight_min": -36.98667907714844,
      "activations/layer12_attention_weight_max": 27.07288360595703,
      "activations/layer12_attention_weight_min": -28.092893600463867,
      "activations/layer13_attention_weight_max": 38.373348236083984,
      "activations/layer13_attention_weight_min": -37.37010955810547,
      "activations/layer14_attention_weight_max": 41.802757263183594,
      "activations/layer14_attention_weight_min": -40.661678314208984,
      "activations/layer15_attention_weight_max": 36.702903747558594,
      "activations/layer15_attention_weight_min": -36.62836456298828,
      "activations/layer16_attention_weight_max": 27.15196418762207,
      "activations/layer16_attention_weight_min": -28.80434799194336,
      "activations/layer17_attention_weight_max": 28.95501708984375,
      "activations/layer17_attention_weight_min": -28.501949310302734,
      "activations/layer18_attention_weight_max": 32.31186294555664,
      "activations/layer18_attention_weight_min": -26.66566276550293,
      "activations/layer19_attention_weight_max": 33.51148986816406,
      "activations/layer19_attention_weight_min": -33.01714324951172,
      "activations/layer1_attention_weight_max": 17.335054397583008,
      "activations/layer1_attention_weight_min": -13.794438362121582,
      "activations/layer20_attention_weight_max": 27.83180809020996,
      "activations/layer20_attention_weight_min": -24.384069442749023,
      "activations/layer21_attention_weight_max": 27.248960494995117,
      "activations/layer21_attention_weight_min": -25.01865005493164,
      "activations/layer22_attention_weight_max": 41.46923065185547,
      "activations/layer22_attention_weight_min": -32.11720275878906,
      "activations/layer23_attention_weight_max": 32.090248107910156,
      "activations/layer23_attention_weight_min": -28.230995178222656,
      "activations/layer2_attention_weight_max": 33.21479797363281,
      "activations/layer2_attention_weight_min": -30.356605529785156,
      "activations/layer3_attention_weight_max": 96.95380401611328,
      "activations/layer3_attention_weight_min": -97.42790222167969,
      "activations/layer4_attention_weight_max": 104.98665618896484,
      "activations/layer4_attention_weight_min": -110.02397918701172,
      "activations/layer5_attention_weight_max": 50.22130584716797,
      "activations/layer5_attention_weight_min": -64.72158813476562,
      "activations/layer6_attention_weight_max": 46.02076721191406,
      "activations/layer6_attention_weight_min": -47.87008285522461,
      "activations/layer7_attention_weight_max": 98.20052337646484,
      "activations/layer7_attention_weight_min": -103.03746795654297,
      "activations/layer8_attention_weight_max": 41.61579132080078,
      "activations/layer8_attention_weight_min": -45.61272048950195,
      "activations/layer9_attention_weight_max": 32.75560760498047,
      "activations/layer9_attention_weight_min": -37.03840637207031,
      "epoch": 18.38,
      "learning_rate": 3.1799999999999994e-05,
      "loss": 2.7274,
      "step": 316300
    },
    {
      "activations/layer0_attention_weight_max": 16.547531127929688,
      "activations/layer0_attention_weight_min": -13.260589599609375,
      "activations/layer10_attention_weight_max": 30.96514892578125,
      "activations/layer10_attention_weight_min": -32.730350494384766,
      "activations/layer11_attention_weight_max": 32.363258361816406,
      "activations/layer11_attention_weight_min": -33.275299072265625,
      "activations/layer12_attention_weight_max": 25.662973403930664,
      "activations/layer12_attention_weight_min": -25.959522247314453,
      "activations/layer13_attention_weight_max": 35.42912292480469,
      "activations/layer13_attention_weight_min": -32.75209426879883,
      "activations/layer14_attention_weight_max": 41.348724365234375,
      "activations/layer14_attention_weight_min": -36.881378173828125,
      "activations/layer15_attention_weight_max": 38.735477447509766,
      "activations/layer15_attention_weight_min": -34.660552978515625,
      "activations/layer16_attention_weight_max": 28.923173904418945,
      "activations/layer16_attention_weight_min": -27.081235885620117,
      "activations/layer17_attention_weight_max": 27.070837020874023,
      "activations/layer17_attention_weight_min": -25.280109405517578,
      "activations/layer18_attention_weight_max": 30.469472885131836,
      "activations/layer18_attention_weight_min": -25.731639862060547,
      "activations/layer19_attention_weight_max": 34.36567687988281,
      "activations/layer19_attention_weight_min": -32.472625732421875,
      "activations/layer1_attention_weight_max": 16.789613723754883,
      "activations/layer1_attention_weight_min": -16.645217895507812,
      "activations/layer20_attention_weight_max": 27.572423934936523,
      "activations/layer20_attention_weight_min": -24.357023239135742,
      "activations/layer21_attention_weight_max": 25.2392635345459,
      "activations/layer21_attention_weight_min": -24.14158821105957,
      "activations/layer22_attention_weight_max": 38.168392181396484,
      "activations/layer22_attention_weight_min": -31.144290924072266,
      "activations/layer23_attention_weight_max": 33.18467712402344,
      "activations/layer23_attention_weight_min": -25.88568878173828,
      "activations/layer2_attention_weight_max": 30.652862548828125,
      "activations/layer2_attention_weight_min": -28.54511260986328,
      "activations/layer3_attention_weight_max": 93.66759490966797,
      "activations/layer3_attention_weight_min": -95.22088623046875,
      "activations/layer4_attention_weight_max": 98.86963653564453,
      "activations/layer4_attention_weight_min": -104.1494140625,
      "activations/layer5_attention_weight_max": 48.6471061706543,
      "activations/layer5_attention_weight_min": -63.76924133300781,
      "activations/layer6_attention_weight_max": 43.33426284790039,
      "activations/layer6_attention_weight_min": -44.43688201904297,
      "activations/layer7_attention_weight_max": 85.5964584350586,
      "activations/layer7_attention_weight_min": -90.83909606933594,
      "activations/layer8_attention_weight_max": 37.730140686035156,
      "activations/layer8_attention_weight_min": -41.43967056274414,
      "activations/layer9_attention_weight_max": 28.939924240112305,
      "activations/layer9_attention_weight_min": -33.17403793334961,
      "epoch": 18.38,
      "learning_rate": 3.17810606060606e-05,
      "loss": 2.718,
      "step": 316350
    },
    {
      "activations/layer0_attention_weight_max": 16.029369354248047,
      "activations/layer0_attention_weight_min": -12.290452003479004,
      "activations/layer10_attention_weight_max": 32.932777404785156,
      "activations/layer10_attention_weight_min": -35.80064010620117,
      "activations/layer11_attention_weight_max": 32.89529800415039,
      "activations/layer11_attention_weight_min": -34.01405334472656,
      "activations/layer12_attention_weight_max": 28.773014068603516,
      "activations/layer12_attention_weight_min": -28.007728576660156,
      "activations/layer13_attention_weight_max": 43.050724029541016,
      "activations/layer13_attention_weight_min": -36.13957214355469,
      "activations/layer14_attention_weight_max": 43.8895263671875,
      "activations/layer14_attention_weight_min": -39.68016815185547,
      "activations/layer15_attention_weight_max": 43.64634323120117,
      "activations/layer15_attention_weight_min": -37.87694549560547,
      "activations/layer16_attention_weight_max": 30.965654373168945,
      "activations/layer16_attention_weight_min": -30.273820877075195,
      "activations/layer17_attention_weight_max": 30.36995506286621,
      "activations/layer17_attention_weight_min": -28.28200340270996,
      "activations/layer18_attention_weight_max": 35.53526306152344,
      "activations/layer18_attention_weight_min": -26.620052337646484,
      "activations/layer19_attention_weight_max": 36.650146484375,
      "activations/layer19_attention_weight_min": -33.30037307739258,
      "activations/layer1_attention_weight_max": 17.836336135864258,
      "activations/layer1_attention_weight_min": -13.199597358703613,
      "activations/layer20_attention_weight_max": 31.139802932739258,
      "activations/layer20_attention_weight_min": -26.67242431640625,
      "activations/layer21_attention_weight_max": 31.242483139038086,
      "activations/layer21_attention_weight_min": -27.20446014404297,
      "activations/layer22_attention_weight_max": 44.10673904418945,
      "activations/layer22_attention_weight_min": -32.49534606933594,
      "activations/layer23_attention_weight_max": 32.416133880615234,
      "activations/layer23_attention_weight_min": -25.759984970092773,
      "activations/layer2_attention_weight_max": 32.356597900390625,
      "activations/layer2_attention_weight_min": -30.60658073425293,
      "activations/layer3_attention_weight_max": 93.9014663696289,
      "activations/layer3_attention_weight_min": -92.54857635498047,
      "activations/layer4_attention_weight_max": 99.92980194091797,
      "activations/layer4_attention_weight_min": -106.74991607666016,
      "activations/layer5_attention_weight_max": 47.121517181396484,
      "activations/layer5_attention_weight_min": -62.27107238769531,
      "activations/layer6_attention_weight_max": 43.28752136230469,
      "activations/layer6_attention_weight_min": -47.4396858215332,
      "activations/layer7_attention_weight_max": 91.50070190429688,
      "activations/layer7_attention_weight_min": -96.81640625,
      "activations/layer8_attention_weight_max": 40.26885223388672,
      "activations/layer8_attention_weight_min": -43.270957946777344,
      "activations/layer9_attention_weight_max": 31.33990478515625,
      "activations/layer9_attention_weight_min": -34.54160690307617,
      "epoch": 18.38,
      "learning_rate": 3.176212121212121e-05,
      "loss": 2.7409,
      "step": 316400
    },
    {
      "activations/layer0_attention_weight_max": 16.248449325561523,
      "activations/layer0_attention_weight_min": -12.534388542175293,
      "activations/layer10_attention_weight_max": 36.51901626586914,
      "activations/layer10_attention_weight_min": -34.723358154296875,
      "activations/layer11_attention_weight_max": 35.71779251098633,
      "activations/layer11_attention_weight_min": -35.859619140625,
      "activations/layer12_attention_weight_max": 26.225492477416992,
      "activations/layer12_attention_weight_min": -27.276025772094727,
      "activations/layer13_attention_weight_max": 40.245330810546875,
      "activations/layer13_attention_weight_min": -34.98394775390625,
      "activations/layer14_attention_weight_max": 42.246856689453125,
      "activations/layer14_attention_weight_min": -38.67819595336914,
      "activations/layer15_attention_weight_max": 37.03356170654297,
      "activations/layer15_attention_weight_min": -37.64670181274414,
      "activations/layer16_attention_weight_max": 29.213787078857422,
      "activations/layer16_attention_weight_min": -27.533998489379883,
      "activations/layer17_attention_weight_max": 29.21575355529785,
      "activations/layer17_attention_weight_min": -26.439016342163086,
      "activations/layer18_attention_weight_max": 30.565181732177734,
      "activations/layer18_attention_weight_min": -24.894861221313477,
      "activations/layer19_attention_weight_max": 32.6450309753418,
      "activations/layer19_attention_weight_min": -31.78238868713379,
      "activations/layer1_attention_weight_max": 15.870985984802246,
      "activations/layer1_attention_weight_min": -14.198227882385254,
      "activations/layer20_attention_weight_max": 30.265878677368164,
      "activations/layer20_attention_weight_min": -26.251380920410156,
      "activations/layer21_attention_weight_max": 29.252147674560547,
      "activations/layer21_attention_weight_min": -25.486520767211914,
      "activations/layer22_attention_weight_max": 40.105567932128906,
      "activations/layer22_attention_weight_min": -32.323543548583984,
      "activations/layer23_attention_weight_max": 29.79440689086914,
      "activations/layer23_attention_weight_min": -27.758703231811523,
      "activations/layer2_attention_weight_max": 31.787086486816406,
      "activations/layer2_attention_weight_min": -31.00412368774414,
      "activations/layer3_attention_weight_max": 95.6422348022461,
      "activations/layer3_attention_weight_min": -95.85751342773438,
      "activations/layer4_attention_weight_max": 103.21331024169922,
      "activations/layer4_attention_weight_min": -107.68590545654297,
      "activations/layer5_attention_weight_max": 49.99290466308594,
      "activations/layer5_attention_weight_min": -60.51521301269531,
      "activations/layer6_attention_weight_max": 44.06711959838867,
      "activations/layer6_attention_weight_min": -45.32053756713867,
      "activations/layer7_attention_weight_max": 94.23856353759766,
      "activations/layer7_attention_weight_min": -90.93085479736328,
      "activations/layer8_attention_weight_max": 42.1331787109375,
      "activations/layer8_attention_weight_min": -42.77062225341797,
      "activations/layer9_attention_weight_max": 33.24673080444336,
      "activations/layer9_attention_weight_min": -36.3651123046875,
      "epoch": 18.39,
      "learning_rate": 3.174318181818182e-05,
      "loss": 2.7351,
      "step": 316450
    },
    {
      "activations/layer0_attention_weight_max": 17.289670944213867,
      "activations/layer0_attention_weight_min": -12.748651504516602,
      "activations/layer10_attention_weight_max": 32.4753303527832,
      "activations/layer10_attention_weight_min": -33.726898193359375,
      "activations/layer11_attention_weight_max": 33.73346710205078,
      "activations/layer11_attention_weight_min": -33.9822998046875,
      "activations/layer12_attention_weight_max": 27.09762191772461,
      "activations/layer12_attention_weight_min": -26.486982345581055,
      "activations/layer13_attention_weight_max": 38.77622604370117,
      "activations/layer13_attention_weight_min": -37.63606262207031,
      "activations/layer14_attention_weight_max": 42.653594970703125,
      "activations/layer14_attention_weight_min": -38.449214935302734,
      "activations/layer15_attention_weight_max": 39.953250885009766,
      "activations/layer15_attention_weight_min": -36.14298629760742,
      "activations/layer16_attention_weight_max": 32.23728561401367,
      "activations/layer16_attention_weight_min": -28.182302474975586,
      "activations/layer17_attention_weight_max": 30.33974838256836,
      "activations/layer17_attention_weight_min": -27.73497772216797,
      "activations/layer18_attention_weight_max": 32.082069396972656,
      "activations/layer18_attention_weight_min": -27.221769332885742,
      "activations/layer19_attention_weight_max": 35.296295166015625,
      "activations/layer19_attention_weight_min": -32.80459213256836,
      "activations/layer1_attention_weight_max": 16.09408950805664,
      "activations/layer1_attention_weight_min": -16.83461570739746,
      "activations/layer20_attention_weight_max": 31.047996520996094,
      "activations/layer20_attention_weight_min": -26.729219436645508,
      "activations/layer21_attention_weight_max": 30.20902442932129,
      "activations/layer21_attention_weight_min": -24.922937393188477,
      "activations/layer22_attention_weight_max": 45.86106491088867,
      "activations/layer22_attention_weight_min": -33.58802795410156,
      "activations/layer23_attention_weight_max": 33.83218765258789,
      "activations/layer23_attention_weight_min": -25.567649841308594,
      "activations/layer2_attention_weight_max": 34.05171203613281,
      "activations/layer2_attention_weight_min": -32.62832260131836,
      "activations/layer3_attention_weight_max": 102.31523132324219,
      "activations/layer3_attention_weight_min": -98.85741424560547,
      "activations/layer4_attention_weight_max": 109.4189682006836,
      "activations/layer4_attention_weight_min": -108.11101531982422,
      "activations/layer5_attention_weight_max": 50.62623596191406,
      "activations/layer5_attention_weight_min": -65.27336120605469,
      "activations/layer6_attention_weight_max": 44.60010528564453,
      "activations/layer6_attention_weight_min": -47.6134147644043,
      "activations/layer7_attention_weight_max": 93.17473602294922,
      "activations/layer7_attention_weight_min": -95.40410614013672,
      "activations/layer8_attention_weight_max": 40.6653938293457,
      "activations/layer8_attention_weight_min": -45.85050964355469,
      "activations/layer9_attention_weight_max": 30.883718490600586,
      "activations/layer9_attention_weight_min": -35.66451644897461,
      "epoch": 18.39,
      "learning_rate": 3.172424242424242e-05,
      "loss": 2.7325,
      "step": 316500
    },
    {
      "activations/layer0_attention_weight_max": 16.315275192260742,
      "activations/layer0_attention_weight_min": -13.03276538848877,
      "activations/layer10_attention_weight_max": 32.023529052734375,
      "activations/layer10_attention_weight_min": -33.00712966918945,
      "activations/layer11_attention_weight_max": 34.03520584106445,
      "activations/layer11_attention_weight_min": -34.179988861083984,
      "activations/layer12_attention_weight_max": 25.187833786010742,
      "activations/layer12_attention_weight_min": -26.019697189331055,
      "activations/layer13_attention_weight_max": 36.64469528198242,
      "activations/layer13_attention_weight_min": -34.661781311035156,
      "activations/layer14_attention_weight_max": 43.64201736450195,
      "activations/layer14_attention_weight_min": -36.85883331298828,
      "activations/layer15_attention_weight_max": 36.56889343261719,
      "activations/layer15_attention_weight_min": -33.95448684692383,
      "activations/layer16_attention_weight_max": 28.673870086669922,
      "activations/layer16_attention_weight_min": -28.219573974609375,
      "activations/layer17_attention_weight_max": 29.056543350219727,
      "activations/layer17_attention_weight_min": -25.42251968383789,
      "activations/layer18_attention_weight_max": 33.847469329833984,
      "activations/layer18_attention_weight_min": -25.768306732177734,
      "activations/layer19_attention_weight_max": 33.699623107910156,
      "activations/layer19_attention_weight_min": -31.966140747070312,
      "activations/layer1_attention_weight_max": 15.648356437683105,
      "activations/layer1_attention_weight_min": -13.678365707397461,
      "activations/layer20_attention_weight_max": 29.03902816772461,
      "activations/layer20_attention_weight_min": -25.089338302612305,
      "activations/layer21_attention_weight_max": 29.78555679321289,
      "activations/layer21_attention_weight_min": -24.351449966430664,
      "activations/layer22_attention_weight_max": 41.389366149902344,
      "activations/layer22_attention_weight_min": -35.31589889526367,
      "activations/layer23_attention_weight_max": 31.979450225830078,
      "activations/layer23_attention_weight_min": -27.364471435546875,
      "activations/layer2_attention_weight_max": 31.356306076049805,
      "activations/layer2_attention_weight_min": -30.838924407958984,
      "activations/layer3_attention_weight_max": 95.54314422607422,
      "activations/layer3_attention_weight_min": -95.310302734375,
      "activations/layer4_attention_weight_max": 104.33074188232422,
      "activations/layer4_attention_weight_min": -103.66156005859375,
      "activations/layer5_attention_weight_max": 48.70220947265625,
      "activations/layer5_attention_weight_min": -58.69401168823242,
      "activations/layer6_attention_weight_max": 43.52104949951172,
      "activations/layer6_attention_weight_min": -46.578556060791016,
      "activations/layer7_attention_weight_max": 94.89602661132812,
      "activations/layer7_attention_weight_min": -95.5530014038086,
      "activations/layer8_attention_weight_max": 41.289127349853516,
      "activations/layer8_attention_weight_min": -42.565208435058594,
      "activations/layer9_attention_weight_max": 32.21256637573242,
      "activations/layer9_attention_weight_min": -35.18826675415039,
      "epoch": 18.39,
      "learning_rate": 3.170530303030303e-05,
      "loss": 2.7344,
      "step": 316550
    },
    {
      "activations/layer0_attention_weight_max": 16.257110595703125,
      "activations/layer0_attention_weight_min": -12.41214656829834,
      "activations/layer10_attention_weight_max": 33.707977294921875,
      "activations/layer10_attention_weight_min": -31.45332908630371,
      "activations/layer11_attention_weight_max": 34.05390167236328,
      "activations/layer11_attention_weight_min": -33.39314651489258,
      "activations/layer12_attention_weight_max": 26.84261703491211,
      "activations/layer12_attention_weight_min": -27.330562591552734,
      "activations/layer13_attention_weight_max": 38.01701736450195,
      "activations/layer13_attention_weight_min": -34.46662902832031,
      "activations/layer14_attention_weight_max": 40.7719612121582,
      "activations/layer14_attention_weight_min": -36.659061431884766,
      "activations/layer15_attention_weight_max": 38.1624755859375,
      "activations/layer15_attention_weight_min": -35.460052490234375,
      "activations/layer16_attention_weight_max": 28.56662940979004,
      "activations/layer16_attention_weight_min": -27.628101348876953,
      "activations/layer17_attention_weight_max": 29.868520736694336,
      "activations/layer17_attention_weight_min": -27.465070724487305,
      "activations/layer18_attention_weight_max": 31.445730209350586,
      "activations/layer18_attention_weight_min": -25.649362564086914,
      "activations/layer19_attention_weight_max": 36.820308685302734,
      "activations/layer19_attention_weight_min": -32.709468841552734,
      "activations/layer1_attention_weight_max": 16.223873138427734,
      "activations/layer1_attention_weight_min": -14.164351463317871,
      "activations/layer20_attention_weight_max": 30.06294822692871,
      "activations/layer20_attention_weight_min": -26.58329963684082,
      "activations/layer21_attention_weight_max": 31.398723602294922,
      "activations/layer21_attention_weight_min": -26.07010841369629,
      "activations/layer22_attention_weight_max": 40.439815521240234,
      "activations/layer22_attention_weight_min": -31.693662643432617,
      "activations/layer23_attention_weight_max": 31.06520652770996,
      "activations/layer23_attention_weight_min": -26.81511688232422,
      "activations/layer2_attention_weight_max": 30.691747665405273,
      "activations/layer2_attention_weight_min": -30.644750595092773,
      "activations/layer3_attention_weight_max": 96.6832504272461,
      "activations/layer3_attention_weight_min": -97.5805435180664,
      "activations/layer4_attention_weight_max": 105.24470520019531,
      "activations/layer4_attention_weight_min": -107.2957534790039,
      "activations/layer5_attention_weight_max": 51.550453186035156,
      "activations/layer5_attention_weight_min": -59.532135009765625,
      "activations/layer6_attention_weight_max": 43.26813888549805,
      "activations/layer6_attention_weight_min": -45.61625289916992,
      "activations/layer7_attention_weight_max": 91.5990219116211,
      "activations/layer7_attention_weight_min": -92.8428726196289,
      "activations/layer8_attention_weight_max": 39.79413604736328,
      "activations/layer8_attention_weight_min": -44.737403869628906,
      "activations/layer9_attention_weight_max": 30.278417587280273,
      "activations/layer9_attention_weight_min": -32.54926681518555,
      "epoch": 18.4,
      "learning_rate": 3.168636363636363e-05,
      "loss": 2.7357,
      "step": 316600
    },
    {
      "activations/layer0_attention_weight_max": 17.436704635620117,
      "activations/layer0_attention_weight_min": -12.684906959533691,
      "activations/layer10_attention_weight_max": 31.544876098632812,
      "activations/layer10_attention_weight_min": -31.146379470825195,
      "activations/layer11_attention_weight_max": 32.00284957885742,
      "activations/layer11_attention_weight_min": -32.22663879394531,
      "activations/layer12_attention_weight_max": 25.65485382080078,
      "activations/layer12_attention_weight_min": -26.25522804260254,
      "activations/layer13_attention_weight_max": 38.9426155090332,
      "activations/layer13_attention_weight_min": -36.28188705444336,
      "activations/layer14_attention_weight_max": 41.76485824584961,
      "activations/layer14_attention_weight_min": -38.660274505615234,
      "activations/layer15_attention_weight_max": 36.78206253051758,
      "activations/layer15_attention_weight_min": -37.86605453491211,
      "activations/layer16_attention_weight_max": 27.31956672668457,
      "activations/layer16_attention_weight_min": -28.96311378479004,
      "activations/layer17_attention_weight_max": 26.724164962768555,
      "activations/layer17_attention_weight_min": -25.631017684936523,
      "activations/layer18_attention_weight_max": 30.657489776611328,
      "activations/layer18_attention_weight_min": -25.30884552001953,
      "activations/layer19_attention_weight_max": 32.979469299316406,
      "activations/layer19_attention_weight_min": -32.09559631347656,
      "activations/layer1_attention_weight_max": 15.485541343688965,
      "activations/layer1_attention_weight_min": -14.704893112182617,
      "activations/layer20_attention_weight_max": 26.262117385864258,
      "activations/layer20_attention_weight_min": -25.88375473022461,
      "activations/layer21_attention_weight_max": 26.45217514038086,
      "activations/layer21_attention_weight_min": -26.487552642822266,
      "activations/layer22_attention_weight_max": 37.86770248413086,
      "activations/layer22_attention_weight_min": -29.38385581970215,
      "activations/layer23_attention_weight_max": 34.10555648803711,
      "activations/layer23_attention_weight_min": -25.424942016601562,
      "activations/layer2_attention_weight_max": 31.111526489257812,
      "activations/layer2_attention_weight_min": -30.68482208251953,
      "activations/layer3_attention_weight_max": 103.72300720214844,
      "activations/layer3_attention_weight_min": -101.52057647705078,
      "activations/layer4_attention_weight_max": 106.54436492919922,
      "activations/layer4_attention_weight_min": -107.33345031738281,
      "activations/layer5_attention_weight_max": 50.731178283691406,
      "activations/layer5_attention_weight_min": -64.47892761230469,
      "activations/layer6_attention_weight_max": 44.52942657470703,
      "activations/layer6_attention_weight_min": -46.514461517333984,
      "activations/layer7_attention_weight_max": 100.10155487060547,
      "activations/layer7_attention_weight_min": -97.52820587158203,
      "activations/layer8_attention_weight_max": 38.01286697387695,
      "activations/layer8_attention_weight_min": -41.98184585571289,
      "activations/layer9_attention_weight_max": 31.662494659423828,
      "activations/layer9_attention_weight_min": -30.745868682861328,
      "epoch": 18.4,
      "learning_rate": 3.166742424242424e-05,
      "loss": 2.7388,
      "step": 316650
    },
    {
      "activations/layer0_attention_weight_max": 16.72841453552246,
      "activations/layer0_attention_weight_min": -12.982535362243652,
      "activations/layer10_attention_weight_max": 33.59025955200195,
      "activations/layer10_attention_weight_min": -34.940433502197266,
      "activations/layer11_attention_weight_max": 35.41802978515625,
      "activations/layer11_attention_weight_min": -33.457359313964844,
      "activations/layer12_attention_weight_max": 27.701644897460938,
      "activations/layer12_attention_weight_min": -26.13700294494629,
      "activations/layer13_attention_weight_max": 41.46511459350586,
      "activations/layer13_attention_weight_min": -36.613677978515625,
      "activations/layer14_attention_weight_max": 44.66884231567383,
      "activations/layer14_attention_weight_min": -39.39775085449219,
      "activations/layer15_attention_weight_max": 42.4721565246582,
      "activations/layer15_attention_weight_min": -36.5885124206543,
      "activations/layer16_attention_weight_max": 30.05582618713379,
      "activations/layer16_attention_weight_min": -27.69677734375,
      "activations/layer17_attention_weight_max": 30.21029281616211,
      "activations/layer17_attention_weight_min": -28.11475372314453,
      "activations/layer18_attention_weight_max": 31.975120544433594,
      "activations/layer18_attention_weight_min": -26.21017074584961,
      "activations/layer19_attention_weight_max": 31.313344955444336,
      "activations/layer19_attention_weight_min": -31.17438316345215,
      "activations/layer1_attention_weight_max": 15.876579284667969,
      "activations/layer1_attention_weight_min": -14.673999786376953,
      "activations/layer20_attention_weight_max": 27.535802841186523,
      "activations/layer20_attention_weight_min": -27.05501365661621,
      "activations/layer21_attention_weight_max": 27.926998138427734,
      "activations/layer21_attention_weight_min": -24.632097244262695,
      "activations/layer22_attention_weight_max": 41.2945671081543,
      "activations/layer22_attention_weight_min": -31.76875877380371,
      "activations/layer23_attention_weight_max": 31.895835876464844,
      "activations/layer23_attention_weight_min": -26.23955535888672,
      "activations/layer2_attention_weight_max": 30.46430015563965,
      "activations/layer2_attention_weight_min": -29.195690155029297,
      "activations/layer3_attention_weight_max": 95.25691986083984,
      "activations/layer3_attention_weight_min": -97.1217041015625,
      "activations/layer4_attention_weight_max": 101.99065399169922,
      "activations/layer4_attention_weight_min": -103.54243469238281,
      "activations/layer5_attention_weight_max": 47.77184295654297,
      "activations/layer5_attention_weight_min": -60.66039276123047,
      "activations/layer6_attention_weight_max": 43.44832229614258,
      "activations/layer6_attention_weight_min": -45.630393981933594,
      "activations/layer7_attention_weight_max": 95.27478790283203,
      "activations/layer7_attention_weight_min": -94.49198150634766,
      "activations/layer8_attention_weight_max": 43.10521697998047,
      "activations/layer8_attention_weight_min": -44.93831253051758,
      "activations/layer9_attention_weight_max": 33.22805404663086,
      "activations/layer9_attention_weight_min": -35.28356170654297,
      "epoch": 18.4,
      "learning_rate": 3.1648484848484846e-05,
      "loss": 2.727,
      "step": 316700
    },
    {
      "activations/layer0_attention_weight_max": 16.26097869873047,
      "activations/layer0_attention_weight_min": -12.448246955871582,
      "activations/layer10_attention_weight_max": 45.22528076171875,
      "activations/layer10_attention_weight_min": -43.010108947753906,
      "activations/layer11_attention_weight_max": 50.72367858886719,
      "activations/layer11_attention_weight_min": -43.63853454589844,
      "activations/layer12_attention_weight_max": 31.091815948486328,
      "activations/layer12_attention_weight_min": -33.3319091796875,
      "activations/layer13_attention_weight_max": 47.12482833862305,
      "activations/layer13_attention_weight_min": -41.054412841796875,
      "activations/layer14_attention_weight_max": 47.35432052612305,
      "activations/layer14_attention_weight_min": -49.07047653198242,
      "activations/layer15_attention_weight_max": 44.70629119873047,
      "activations/layer15_attention_weight_min": -43.36141586303711,
      "activations/layer16_attention_weight_max": 35.90021514892578,
      "activations/layer16_attention_weight_min": -29.393123626708984,
      "activations/layer17_attention_weight_max": 33.71675491333008,
      "activations/layer17_attention_weight_min": -28.956865310668945,
      "activations/layer18_attention_weight_max": 32.19626998901367,
      "activations/layer18_attention_weight_min": -29.609670639038086,
      "activations/layer19_attention_weight_max": 36.54038619995117,
      "activations/layer19_attention_weight_min": -35.21665954589844,
      "activations/layer1_attention_weight_max": 17.129905700683594,
      "activations/layer1_attention_weight_min": -16.40324592590332,
      "activations/layer20_attention_weight_max": 27.77532958984375,
      "activations/layer20_attention_weight_min": -28.770679473876953,
      "activations/layer21_attention_weight_max": 28.042789459228516,
      "activations/layer21_attention_weight_min": -27.856182098388672,
      "activations/layer22_attention_weight_max": 38.751827239990234,
      "activations/layer22_attention_weight_min": -36.213775634765625,
      "activations/layer23_attention_weight_max": 32.77165222167969,
      "activations/layer23_attention_weight_min": -29.856067657470703,
      "activations/layer2_attention_weight_max": 37.458099365234375,
      "activations/layer2_attention_weight_min": -32.89142608642578,
      "activations/layer3_attention_weight_max": 105.73668670654297,
      "activations/layer3_attention_weight_min": -102.48787689208984,
      "activations/layer4_attention_weight_max": 110.86055755615234,
      "activations/layer4_attention_weight_min": -106.80731201171875,
      "activations/layer5_attention_weight_max": 49.7839469909668,
      "activations/layer5_attention_weight_min": -59.329647064208984,
      "activations/layer6_attention_weight_max": 49.40035629272461,
      "activations/layer6_attention_weight_min": -47.76362228393555,
      "activations/layer7_attention_weight_max": 111.69779968261719,
      "activations/layer7_attention_weight_min": -101.17053985595703,
      "activations/layer8_attention_weight_max": 50.55885696411133,
      "activations/layer8_attention_weight_min": -49.43077087402344,
      "activations/layer9_attention_weight_max": 39.35385513305664,
      "activations/layer9_attention_weight_min": -39.78558349609375,
      "epoch": 18.4,
      "learning_rate": 3.1629545454545455e-05,
      "loss": 2.7442,
      "step": 316750
    },
    {
      "activations/layer0_attention_weight_max": 15.381978034973145,
      "activations/layer0_attention_weight_min": -12.2650146484375,
      "activations/layer10_attention_weight_max": 36.7803955078125,
      "activations/layer10_attention_weight_min": -35.26914596557617,
      "activations/layer11_attention_weight_max": 37.3377799987793,
      "activations/layer11_attention_weight_min": -35.92937088012695,
      "activations/layer12_attention_weight_max": 30.245731353759766,
      "activations/layer12_attention_weight_min": -27.9962100982666,
      "activations/layer13_attention_weight_max": 42.82636642456055,
      "activations/layer13_attention_weight_min": -38.99653244018555,
      "activations/layer14_attention_weight_max": 50.62282943725586,
      "activations/layer14_attention_weight_min": -41.5652961730957,
      "activations/layer15_attention_weight_max": 42.65003204345703,
      "activations/layer15_attention_weight_min": -37.60983657836914,
      "activations/layer16_attention_weight_max": 30.870616912841797,
      "activations/layer16_attention_weight_min": -27.30478858947754,
      "activations/layer17_attention_weight_max": 30.14466094970703,
      "activations/layer17_attention_weight_min": -25.845834732055664,
      "activations/layer18_attention_weight_max": 30.25422477722168,
      "activations/layer18_attention_weight_min": -24.572206497192383,
      "activations/layer19_attention_weight_max": 33.50639343261719,
      "activations/layer19_attention_weight_min": -31.800230026245117,
      "activations/layer1_attention_weight_max": 16.30352020263672,
      "activations/layer1_attention_weight_min": -15.279547691345215,
      "activations/layer20_attention_weight_max": 28.458688735961914,
      "activations/layer20_attention_weight_min": -26.22475814819336,
      "activations/layer21_attention_weight_max": 28.64165496826172,
      "activations/layer21_attention_weight_min": -26.929716110229492,
      "activations/layer22_attention_weight_max": 45.389930725097656,
      "activations/layer22_attention_weight_min": -32.72969055175781,
      "activations/layer23_attention_weight_max": 32.96068572998047,
      "activations/layer23_attention_weight_min": -24.210830688476562,
      "activations/layer2_attention_weight_max": 30.768627166748047,
      "activations/layer2_attention_weight_min": -29.902603149414062,
      "activations/layer3_attention_weight_max": 101.26061248779297,
      "activations/layer3_attention_weight_min": -99.18536376953125,
      "activations/layer4_attention_weight_max": 108.3868179321289,
      "activations/layer4_attention_weight_min": -112.53179168701172,
      "activations/layer5_attention_weight_max": 50.08457565307617,
      "activations/layer5_attention_weight_min": -61.76890563964844,
      "activations/layer6_attention_weight_max": 47.134437561035156,
      "activations/layer6_attention_weight_min": -48.362247467041016,
      "activations/layer7_attention_weight_max": 97.48658752441406,
      "activations/layer7_attention_weight_min": -97.72960662841797,
      "activations/layer8_attention_weight_max": 46.377750396728516,
      "activations/layer8_attention_weight_min": -46.257205963134766,
      "activations/layer9_attention_weight_max": 36.59749984741211,
      "activations/layer9_attention_weight_min": -37.35873031616211,
      "epoch": 18.41,
      "learning_rate": 3.1610984848484845e-05,
      "loss": 2.7239,
      "step": 316800
    },
    {
      "activations/layer0_attention_weight_max": 16.25029754638672,
      "activations/layer0_attention_weight_min": -13.77659797668457,
      "activations/layer10_attention_weight_max": 32.8724250793457,
      "activations/layer10_attention_weight_min": -34.22993469238281,
      "activations/layer11_attention_weight_max": 34.39122772216797,
      "activations/layer11_attention_weight_min": -34.609535217285156,
      "activations/layer12_attention_weight_max": 26.44251251220703,
      "activations/layer12_attention_weight_min": -25.554044723510742,
      "activations/layer13_attention_weight_max": 39.363975524902344,
      "activations/layer13_attention_weight_min": -34.99775314331055,
      "activations/layer14_attention_weight_max": 42.275543212890625,
      "activations/layer14_attention_weight_min": -37.53361129760742,
      "activations/layer15_attention_weight_max": 38.34071731567383,
      "activations/layer15_attention_weight_min": -35.62675476074219,
      "activations/layer16_attention_weight_max": 28.912878036499023,
      "activations/layer16_attention_weight_min": -27.27529525756836,
      "activations/layer17_attention_weight_max": 28.62709617614746,
      "activations/layer17_attention_weight_min": -25.273977279663086,
      "activations/layer18_attention_weight_max": 30.513479232788086,
      "activations/layer18_attention_weight_min": -24.521467208862305,
      "activations/layer19_attention_weight_max": 34.055538177490234,
      "activations/layer19_attention_weight_min": -30.94013023376465,
      "activations/layer1_attention_weight_max": 16.50016975402832,
      "activations/layer1_attention_weight_min": -15.830212593078613,
      "activations/layer20_attention_weight_max": 28.956602096557617,
      "activations/layer20_attention_weight_min": -26.913867950439453,
      "activations/layer21_attention_weight_max": 29.3466796875,
      "activations/layer21_attention_weight_min": -25.37140464782715,
      "activations/layer22_attention_weight_max": 40.57807922363281,
      "activations/layer22_attention_weight_min": -30.43730354309082,
      "activations/layer23_attention_weight_max": 30.28711700439453,
      "activations/layer23_attention_weight_min": -26.780797958374023,
      "activations/layer2_attention_weight_max": 32.96739196777344,
      "activations/layer2_attention_weight_min": -29.75914764404297,
      "activations/layer3_attention_weight_max": 95.64906311035156,
      "activations/layer3_attention_weight_min": -96.2789535522461,
      "activations/layer4_attention_weight_max": 99.46554565429688,
      "activations/layer4_attention_weight_min": -102.61720275878906,
      "activations/layer5_attention_weight_max": 53.80962371826172,
      "activations/layer5_attention_weight_min": -62.86048126220703,
      "activations/layer6_attention_weight_max": 44.94198989868164,
      "activations/layer6_attention_weight_min": -44.8613166809082,
      "activations/layer7_attention_weight_max": 95.69812774658203,
      "activations/layer7_attention_weight_min": -95.782958984375,
      "activations/layer8_attention_weight_max": 40.02170181274414,
      "activations/layer8_attention_weight_min": -42.58952713012695,
      "activations/layer9_attention_weight_max": 32.89149475097656,
      "activations/layer9_attention_weight_min": -33.87248992919922,
      "epoch": 18.41,
      "learning_rate": 3.1592045454545454e-05,
      "loss": 2.7144,
      "step": 316850
    },
    {
      "activations/layer0_attention_weight_max": 17.204540252685547,
      "activations/layer0_attention_weight_min": -11.665389060974121,
      "activations/layer10_attention_weight_max": 34.93840789794922,
      "activations/layer10_attention_weight_min": -33.779457092285156,
      "activations/layer11_attention_weight_max": 37.5320930480957,
      "activations/layer11_attention_weight_min": -33.73111343383789,
      "activations/layer12_attention_weight_max": 27.986366271972656,
      "activations/layer12_attention_weight_min": -24.123044967651367,
      "activations/layer13_attention_weight_max": 39.20033645629883,
      "activations/layer13_attention_weight_min": -34.2384147644043,
      "activations/layer14_attention_weight_max": 45.104820251464844,
      "activations/layer14_attention_weight_min": -35.06744384765625,
      "activations/layer15_attention_weight_max": 42.82494354248047,
      "activations/layer15_attention_weight_min": -34.979976654052734,
      "activations/layer16_attention_weight_max": 32.993324279785156,
      "activations/layer16_attention_weight_min": -28.463945388793945,
      "activations/layer17_attention_weight_max": 29.83710479736328,
      "activations/layer17_attention_weight_min": -25.10643768310547,
      "activations/layer18_attention_weight_max": 30.07691764831543,
      "activations/layer18_attention_weight_min": -25.99397850036621,
      "activations/layer19_attention_weight_max": 33.890846252441406,
      "activations/layer19_attention_weight_min": -31.7526798248291,
      "activations/layer1_attention_weight_max": 16.43918228149414,
      "activations/layer1_attention_weight_min": -15.137995719909668,
      "activations/layer20_attention_weight_max": 27.982158660888672,
      "activations/layer20_attention_weight_min": -24.56966209411621,
      "activations/layer21_attention_weight_max": 27.467926025390625,
      "activations/layer21_attention_weight_min": -26.03240394592285,
      "activations/layer22_attention_weight_max": 39.05970001220703,
      "activations/layer22_attention_weight_min": -31.8203125,
      "activations/layer23_attention_weight_max": 28.02823257446289,
      "activations/layer23_attention_weight_min": -25.349363327026367,
      "activations/layer2_attention_weight_max": 32.89425277709961,
      "activations/layer2_attention_weight_min": -31.380329132080078,
      "activations/layer3_attention_weight_max": 99.73888397216797,
      "activations/layer3_attention_weight_min": -98.72640991210938,
      "activations/layer4_attention_weight_max": 109.46501922607422,
      "activations/layer4_attention_weight_min": -110.65498352050781,
      "activations/layer5_attention_weight_max": 52.142696380615234,
      "activations/layer5_attention_weight_min": -62.99059295654297,
      "activations/layer6_attention_weight_max": 44.083744049072266,
      "activations/layer6_attention_weight_min": -48.21297836303711,
      "activations/layer7_attention_weight_max": 93.4834213256836,
      "activations/layer7_attention_weight_min": -92.71434783935547,
      "activations/layer8_attention_weight_max": 41.69271469116211,
      "activations/layer8_attention_weight_min": -41.99198913574219,
      "activations/layer9_attention_weight_max": 32.943180084228516,
      "activations/layer9_attention_weight_min": -33.28824996948242,
      "epoch": 18.41,
      "learning_rate": 3.157310606060606e-05,
      "loss": 2.7286,
      "step": 316900
    },
    {
      "activations/layer0_attention_weight_max": 15.853431701660156,
      "activations/layer0_attention_weight_min": -12.638019561767578,
      "activations/layer10_attention_weight_max": 33.859458923339844,
      "activations/layer10_attention_weight_min": -35.014076232910156,
      "activations/layer11_attention_weight_max": 35.370948791503906,
      "activations/layer11_attention_weight_min": -35.32720184326172,
      "activations/layer12_attention_weight_max": 30.14092254638672,
      "activations/layer12_attention_weight_min": -29.06314468383789,
      "activations/layer13_attention_weight_max": 42.672542572021484,
      "activations/layer13_attention_weight_min": -38.19517135620117,
      "activations/layer14_attention_weight_max": 48.37448501586914,
      "activations/layer14_attention_weight_min": -40.14119338989258,
      "activations/layer15_attention_weight_max": 44.06040954589844,
      "activations/layer15_attention_weight_min": -37.75471878051758,
      "activations/layer16_attention_weight_max": 31.99243927001953,
      "activations/layer16_attention_weight_min": -29.140539169311523,
      "activations/layer17_attention_weight_max": 31.67616081237793,
      "activations/layer17_attention_weight_min": -28.105308532714844,
      "activations/layer18_attention_weight_max": 33.67116928100586,
      "activations/layer18_attention_weight_min": -29.156057357788086,
      "activations/layer19_attention_weight_max": 37.50279235839844,
      "activations/layer19_attention_weight_min": -34.90522384643555,
      "activations/layer1_attention_weight_max": 18.60669708251953,
      "activations/layer1_attention_weight_min": -15.556527137756348,
      "activations/layer20_attention_weight_max": 32.12082290649414,
      "activations/layer20_attention_weight_min": -28.2380313873291,
      "activations/layer21_attention_weight_max": 29.618722915649414,
      "activations/layer21_attention_weight_min": -26.865392684936523,
      "activations/layer22_attention_weight_max": 43.023990631103516,
      "activations/layer22_attention_weight_min": -35.07234191894531,
      "activations/layer23_attention_weight_max": 35.237422943115234,
      "activations/layer23_attention_weight_min": -27.223615646362305,
      "activations/layer2_attention_weight_max": 30.418325424194336,
      "activations/layer2_attention_weight_min": -29.4014949798584,
      "activations/layer3_attention_weight_max": 91.89568328857422,
      "activations/layer3_attention_weight_min": -93.5853042602539,
      "activations/layer4_attention_weight_max": 99.13591003417969,
      "activations/layer4_attention_weight_min": -101.20574188232422,
      "activations/layer5_attention_weight_max": 47.83917236328125,
      "activations/layer5_attention_weight_min": -58.00836181640625,
      "activations/layer6_attention_weight_max": 42.676456451416016,
      "activations/layer6_attention_weight_min": -45.214515686035156,
      "activations/layer7_attention_weight_max": 94.93692779541016,
      "activations/layer7_attention_weight_min": -91.57652282714844,
      "activations/layer8_attention_weight_max": 42.44740676879883,
      "activations/layer8_attention_weight_min": -42.91547393798828,
      "activations/layer9_attention_weight_max": 32.8641242980957,
      "activations/layer9_attention_weight_min": -35.63393783569336,
      "epoch": 18.42,
      "learning_rate": 3.1554166666666663e-05,
      "loss": 2.728,
      "step": 316950
    },
    {
      "activations/layer0_attention_weight_max": 16.23758316040039,
      "activations/layer0_attention_weight_min": -13.51749038696289,
      "activations/layer10_attention_weight_max": 32.49475860595703,
      "activations/layer10_attention_weight_min": -32.78511428833008,
      "activations/layer11_attention_weight_max": 32.291419982910156,
      "activations/layer11_attention_weight_min": -33.90270233154297,
      "activations/layer12_attention_weight_max": 26.157011032104492,
      "activations/layer12_attention_weight_min": -26.94089698791504,
      "activations/layer13_attention_weight_max": 38.815895080566406,
      "activations/layer13_attention_weight_min": -35.45652389526367,
      "activations/layer14_attention_weight_max": 40.229103088378906,
      "activations/layer14_attention_weight_min": -37.50386047363281,
      "activations/layer15_attention_weight_max": 39.44554138183594,
      "activations/layer15_attention_weight_min": -35.09030532836914,
      "activations/layer16_attention_weight_max": 31.017871856689453,
      "activations/layer16_attention_weight_min": -29.76798439025879,
      "activations/layer17_attention_weight_max": 29.14017105102539,
      "activations/layer17_attention_weight_min": -26.08414649963379,
      "activations/layer18_attention_weight_max": 33.00914001464844,
      "activations/layer18_attention_weight_min": -25.954124450683594,
      "activations/layer19_attention_weight_max": 32.00621795654297,
      "activations/layer19_attention_weight_min": -32.01017379760742,
      "activations/layer1_attention_weight_max": 17.018491744995117,
      "activations/layer1_attention_weight_min": -16.586320877075195,
      "activations/layer20_attention_weight_max": 27.656639099121094,
      "activations/layer20_attention_weight_min": -24.96100425720215,
      "activations/layer21_attention_weight_max": 29.133378982543945,
      "activations/layer21_attention_weight_min": -24.619413375854492,
      "activations/layer22_attention_weight_max": 38.13975143432617,
      "activations/layer22_attention_weight_min": -31.27313995361328,
      "activations/layer23_attention_weight_max": 31.074552536010742,
      "activations/layer23_attention_weight_min": -25.743452072143555,
      "activations/layer2_attention_weight_max": 30.549787521362305,
      "activations/layer2_attention_weight_min": -31.519775390625,
      "activations/layer3_attention_weight_max": 97.93096160888672,
      "activations/layer3_attention_weight_min": -97.24085998535156,
      "activations/layer4_attention_weight_max": 105.93120574951172,
      "activations/layer4_attention_weight_min": -105.4588394165039,
      "activations/layer5_attention_weight_max": 53.37739181518555,
      "activations/layer5_attention_weight_min": -59.01801300048828,
      "activations/layer6_attention_weight_max": 45.9311408996582,
      "activations/layer6_attention_weight_min": -47.856258392333984,
      "activations/layer7_attention_weight_max": 94.68212127685547,
      "activations/layer7_attention_weight_min": -92.38967895507812,
      "activations/layer8_attention_weight_max": 41.385704040527344,
      "activations/layer8_attention_weight_min": -44.254005432128906,
      "activations/layer9_attention_weight_max": 31.19999122619629,
      "activations/layer9_attention_weight_min": -33.76666259765625,
      "epoch": 18.42,
      "learning_rate": 3.153522727272727e-05,
      "loss": 2.716,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_loss": 2.6875,
      "eval_runtime": 8.5069,
      "eval_samples_per_second": 504.768,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.5069,
      "eval_openwebtext_samples_per_second": 504.768,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 2.0102,
      "eval_wikitext_samples_per_second": 226.844,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_lambada_loss": 2.37109375,
      "eval_lambada_ppl": 10.709098957841949,
      "eval_lambada_runtime": 9.5858,
      "eval_lambada_samples_per_second": 507.939,
      "step": 317000
    },
    {
      "activations/layer0_attention_weight_max": 16.078866958618164,
      "activations/layer0_attention_weight_min": -14.211994171142578,
      "activations/layer10_attention_weight_max": 32.2330436706543,
      "activations/layer10_attention_weight_min": -32.761688232421875,
      "activations/layer11_attention_weight_max": 31.437788009643555,
      "activations/layer11_attention_weight_min": -34.485801696777344,
      "activations/layer12_attention_weight_max": 26.79841423034668,
      "activations/layer12_attention_weight_min": -28.681686401367188,
      "activations/layer13_attention_weight_max": 36.78007507324219,
      "activations/layer13_attention_weight_min": -35.913063049316406,
      "activations/layer14_attention_weight_max": 43.20033264160156,
      "activations/layer14_attention_weight_min": -38.339839935302734,
      "activations/layer15_attention_weight_max": 36.28921890258789,
      "activations/layer15_attention_weight_min": -34.38078689575195,
      "activations/layer16_attention_weight_max": 30.250871658325195,
      "activations/layer16_attention_weight_min": -29.394582748413086,
      "activations/layer17_attention_weight_max": 30.132965087890625,
      "activations/layer17_attention_weight_min": -26.04791259765625,
      "activations/layer18_attention_weight_max": 30.700538635253906,
      "activations/layer18_attention_weight_min": -28.000003814697266,
      "activations/layer19_attention_weight_max": 33.7740364074707,
      "activations/layer19_attention_weight_min": -33.5594367980957,
      "activations/layer1_attention_weight_max": 17.75346565246582,
      "activations/layer1_attention_weight_min": -14.272978782653809,
      "activations/layer20_attention_weight_max": 30.883277893066406,
      "activations/layer20_attention_weight_min": -26.483503341674805,
      "activations/layer21_attention_weight_max": 30.074520111083984,
      "activations/layer21_attention_weight_min": -27.837627410888672,
      "activations/layer22_attention_weight_max": 42.72946548461914,
      "activations/layer22_attention_weight_min": -35.158111572265625,
      "activations/layer23_attention_weight_max": 34.73309326171875,
      "activations/layer23_attention_weight_min": -27.664108276367188,
      "activations/layer2_attention_weight_max": 31.107498168945312,
      "activations/layer2_attention_weight_min": -28.013774871826172,
      "activations/layer3_attention_weight_max": 92.74771118164062,
      "activations/layer3_attention_weight_min": -92.16736602783203,
      "activations/layer4_attention_weight_max": 99.52621459960938,
      "activations/layer4_attention_weight_min": -102.80208587646484,
      "activations/layer5_attention_weight_max": 48.826377868652344,
      "activations/layer5_attention_weight_min": -63.491600036621094,
      "activations/layer6_attention_weight_max": 43.60103988647461,
      "activations/layer6_attention_weight_min": -45.582340240478516,
      "activations/layer7_attention_weight_max": 90.10256958007812,
      "activations/layer7_attention_weight_min": -97.04634857177734,
      "activations/layer8_attention_weight_max": 40.10618209838867,
      "activations/layer8_attention_weight_min": -42.63734436035156,
      "activations/layer9_attention_weight_max": 32.06071853637695,
      "activations/layer9_attention_weight_min": -33.926212310791016,
      "epoch": 18.42,
      "learning_rate": 3.151628787878787e-05,
      "loss": 2.7318,
      "step": 317050
    },
    {
      "activations/layer0_attention_weight_max": 16.039676666259766,
      "activations/layer0_attention_weight_min": -13.009533882141113,
      "activations/layer10_attention_weight_max": 34.131927490234375,
      "activations/layer10_attention_weight_min": -32.94683837890625,
      "activations/layer11_attention_weight_max": 33.02201461791992,
      "activations/layer11_attention_weight_min": -34.09834671020508,
      "activations/layer12_attention_weight_max": 26.691362380981445,
      "activations/layer12_attention_weight_min": -26.44371223449707,
      "activations/layer13_attention_weight_max": 38.9813346862793,
      "activations/layer13_attention_weight_min": -37.3284912109375,
      "activations/layer14_attention_weight_max": 42.91493225097656,
      "activations/layer14_attention_weight_min": -39.34752655029297,
      "activations/layer15_attention_weight_max": 37.61750793457031,
      "activations/layer15_attention_weight_min": -37.715782165527344,
      "activations/layer16_attention_weight_max": 30.565534591674805,
      "activations/layer16_attention_weight_min": -28.100427627563477,
      "activations/layer17_attention_weight_max": 30.04083251953125,
      "activations/layer17_attention_weight_min": -26.21799087524414,
      "activations/layer18_attention_weight_max": 30.101402282714844,
      "activations/layer18_attention_weight_min": -25.84373664855957,
      "activations/layer19_attention_weight_max": 33.93210983276367,
      "activations/layer19_attention_weight_min": -31.7614803314209,
      "activations/layer1_attention_weight_max": 17.48854637145996,
      "activations/layer1_attention_weight_min": -15.055193901062012,
      "activations/layer20_attention_weight_max": 29.27610969543457,
      "activations/layer20_attention_weight_min": -26.326913833618164,
      "activations/layer21_attention_weight_max": 29.227689743041992,
      "activations/layer21_attention_weight_min": -26.46341323852539,
      "activations/layer22_attention_weight_max": 41.262168884277344,
      "activations/layer22_attention_weight_min": -33.183372497558594,
      "activations/layer23_attention_weight_max": 32.481014251708984,
      "activations/layer23_attention_weight_min": -27.538183212280273,
      "activations/layer2_attention_weight_max": 32.58256149291992,
      "activations/layer2_attention_weight_min": -28.613452911376953,
      "activations/layer3_attention_weight_max": 98.35901641845703,
      "activations/layer3_attention_weight_min": -95.30081176757812,
      "activations/layer4_attention_weight_max": 100.4422378540039,
      "activations/layer4_attention_weight_min": -101.9110336303711,
      "activations/layer5_attention_weight_max": 50.0538330078125,
      "activations/layer5_attention_weight_min": -58.997039794921875,
      "activations/layer6_attention_weight_max": 44.11845016479492,
      "activations/layer6_attention_weight_min": -44.558616638183594,
      "activations/layer7_attention_weight_max": 87.14808654785156,
      "activations/layer7_attention_weight_min": -93.65193176269531,
      "activations/layer8_attention_weight_max": 41.0219841003418,
      "activations/layer8_attention_weight_min": -42.68208694458008,
      "activations/layer9_attention_weight_max": 30.797977447509766,
      "activations/layer9_attention_weight_min": -33.09048080444336,
      "epoch": 18.43,
      "learning_rate": 3.149734848484848e-05,
      "loss": 2.7163,
      "step": 317100
    },
    {
      "activations/layer0_attention_weight_max": 17.442991256713867,
      "activations/layer0_attention_weight_min": -13.51259994506836,
      "activations/layer10_attention_weight_max": 34.65437316894531,
      "activations/layer10_attention_weight_min": -34.419090270996094,
      "activations/layer11_attention_weight_max": 32.94901657104492,
      "activations/layer11_attention_weight_min": -35.222503662109375,
      "activations/layer12_attention_weight_max": 26.574478149414062,
      "activations/layer12_attention_weight_min": -25.891504287719727,
      "activations/layer13_attention_weight_max": 39.203609466552734,
      "activations/layer13_attention_weight_min": -34.84590530395508,
      "activations/layer14_attention_weight_max": 41.61613082885742,
      "activations/layer14_attention_weight_min": -39.42961120605469,
      "activations/layer15_attention_weight_max": 37.870906829833984,
      "activations/layer15_attention_weight_min": -35.65726852416992,
      "activations/layer16_attention_weight_max": 30.698522567749023,
      "activations/layer16_attention_weight_min": -28.0084285736084,
      "activations/layer17_attention_weight_max": 28.50950813293457,
      "activations/layer17_attention_weight_min": -26.03827667236328,
      "activations/layer18_attention_weight_max": 32.69158935546875,
      "activations/layer18_attention_weight_min": -28.52678871154785,
      "activations/layer19_attention_weight_max": 34.04133605957031,
      "activations/layer19_attention_weight_min": -33.913883209228516,
      "activations/layer1_attention_weight_max": 16.887109756469727,
      "activations/layer1_attention_weight_min": -13.433771133422852,
      "activations/layer20_attention_weight_max": 30.833824157714844,
      "activations/layer20_attention_weight_min": -26.489871978759766,
      "activations/layer21_attention_weight_max": 29.385576248168945,
      "activations/layer21_attention_weight_min": -27.628751754760742,
      "activations/layer22_attention_weight_max": 41.226051330566406,
      "activations/layer22_attention_weight_min": -36.211578369140625,
      "activations/layer23_attention_weight_max": 34.94804000854492,
      "activations/layer23_attention_weight_min": -28.158309936523438,
      "activations/layer2_attention_weight_max": 30.86433982849121,
      "activations/layer2_attention_weight_min": -29.917686462402344,
      "activations/layer3_attention_weight_max": 96.27886962890625,
      "activations/layer3_attention_weight_min": -95.85749053955078,
      "activations/layer4_attention_weight_max": 102.0273208618164,
      "activations/layer4_attention_weight_min": -107.5862808227539,
      "activations/layer5_attention_weight_max": 50.17803192138672,
      "activations/layer5_attention_weight_min": -59.99620056152344,
      "activations/layer6_attention_weight_max": 44.39695739746094,
      "activations/layer6_attention_weight_min": -45.56850814819336,
      "activations/layer7_attention_weight_max": 89.0041275024414,
      "activations/layer7_attention_weight_min": -95.89427185058594,
      "activations/layer8_attention_weight_max": 39.84109878540039,
      "activations/layer8_attention_weight_min": -42.00062942504883,
      "activations/layer9_attention_weight_max": 30.11847496032715,
      "activations/layer9_attention_weight_min": -34.69281768798828,
      "epoch": 18.43,
      "learning_rate": 3.147840909090909e-05,
      "loss": 2.7311,
      "step": 317150
    },
    {
      "activations/layer0_attention_weight_max": 16.102970123291016,
      "activations/layer0_attention_weight_min": -12.455564498901367,
      "activations/layer10_attention_weight_max": 30.864896774291992,
      "activations/layer10_attention_weight_min": -34.25385665893555,
      "activations/layer11_attention_weight_max": 33.23295211791992,
      "activations/layer11_attention_weight_min": -34.03673553466797,
      "activations/layer12_attention_weight_max": 26.308237075805664,
      "activations/layer12_attention_weight_min": -26.286338806152344,
      "activations/layer13_attention_weight_max": 36.422298431396484,
      "activations/layer13_attention_weight_min": -35.93730545043945,
      "activations/layer14_attention_weight_max": 38.62974548339844,
      "activations/layer14_attention_weight_min": -38.51548385620117,
      "activations/layer15_attention_weight_max": 36.64592361450195,
      "activations/layer15_attention_weight_min": -36.53720474243164,
      "activations/layer16_attention_weight_max": 30.739139556884766,
      "activations/layer16_attention_weight_min": -26.91671371459961,
      "activations/layer17_attention_weight_max": 29.734148025512695,
      "activations/layer17_attention_weight_min": -25.943958282470703,
      "activations/layer18_attention_weight_max": 31.14594078063965,
      "activations/layer18_attention_weight_min": -26.212339401245117,
      "activations/layer19_attention_weight_max": 34.63460159301758,
      "activations/layer19_attention_weight_min": -30.946508407592773,
      "activations/layer1_attention_weight_max": 17.061243057250977,
      "activations/layer1_attention_weight_min": -14.990286827087402,
      "activations/layer20_attention_weight_max": 27.8173828125,
      "activations/layer20_attention_weight_min": -26.188371658325195,
      "activations/layer21_attention_weight_max": 27.037612915039062,
      "activations/layer21_attention_weight_min": -25.351646423339844,
      "activations/layer22_attention_weight_max": 37.32355499267578,
      "activations/layer22_attention_weight_min": -31.89179801940918,
      "activations/layer23_attention_weight_max": 30.400100708007812,
      "activations/layer23_attention_weight_min": -25.757061004638672,
      "activations/layer2_attention_weight_max": 29.91144371032715,
      "activations/layer2_attention_weight_min": -29.111190795898438,
      "activations/layer3_attention_weight_max": 87.4729995727539,
      "activations/layer3_attention_weight_min": -88.70115661621094,
      "activations/layer4_attention_weight_max": 96.37085723876953,
      "activations/layer4_attention_weight_min": -99.55132293701172,
      "activations/layer5_attention_weight_max": 48.45178985595703,
      "activations/layer5_attention_weight_min": -59.57404327392578,
      "activations/layer6_attention_weight_max": 43.04521942138672,
      "activations/layer6_attention_weight_min": -47.608924865722656,
      "activations/layer7_attention_weight_max": 91.98632049560547,
      "activations/layer7_attention_weight_min": -94.52566528320312,
      "activations/layer8_attention_weight_max": 38.834320068359375,
      "activations/layer8_attention_weight_min": -43.01742935180664,
      "activations/layer9_attention_weight_max": 28.72649574279785,
      "activations/layer9_attention_weight_min": -33.60062789916992,
      "epoch": 18.43,
      "learning_rate": 3.14594696969697e-05,
      "loss": 2.7304,
      "step": 317200
    },
    {
      "activations/layer0_attention_weight_max": 16.61074447631836,
      "activations/layer0_attention_weight_min": -13.500016212463379,
      "activations/layer10_attention_weight_max": 35.169471740722656,
      "activations/layer10_attention_weight_min": -34.096824645996094,
      "activations/layer11_attention_weight_max": 33.428653717041016,
      "activations/layer11_attention_weight_min": -33.84414291381836,
      "activations/layer12_attention_weight_max": 28.187049865722656,
      "activations/layer12_attention_weight_min": -28.201744079589844,
      "activations/layer13_attention_weight_max": 42.84667205810547,
      "activations/layer13_attention_weight_min": -38.44384002685547,
      "activations/layer14_attention_weight_max": 44.15784454345703,
      "activations/layer14_attention_weight_min": -37.42213439941406,
      "activations/layer15_attention_weight_max": 43.430824279785156,
      "activations/layer15_attention_weight_min": -36.31071472167969,
      "activations/layer16_attention_weight_max": 30.05348014831543,
      "activations/layer16_attention_weight_min": -28.485265731811523,
      "activations/layer17_attention_weight_max": 28.46827507019043,
      "activations/layer17_attention_weight_min": -27.562166213989258,
      "activations/layer18_attention_weight_max": 31.645620346069336,
      "activations/layer18_attention_weight_min": -27.849088668823242,
      "activations/layer19_attention_weight_max": 31.98117446899414,
      "activations/layer19_attention_weight_min": -32.31917190551758,
      "activations/layer1_attention_weight_max": 15.649176597595215,
      "activations/layer1_attention_weight_min": -13.138442993164062,
      "activations/layer20_attention_weight_max": 27.52049446105957,
      "activations/layer20_attention_weight_min": -28.05929183959961,
      "activations/layer21_attention_weight_max": 29.78453826904297,
      "activations/layer21_attention_weight_min": -28.823745727539062,
      "activations/layer22_attention_weight_max": 36.40827941894531,
      "activations/layer22_attention_weight_min": -34.09642028808594,
      "activations/layer23_attention_weight_max": 29.351341247558594,
      "activations/layer23_attention_weight_min": -29.407154083251953,
      "activations/layer2_attention_weight_max": 28.789745330810547,
      "activations/layer2_attention_weight_min": -28.45106315612793,
      "activations/layer3_attention_weight_max": 93.83418273925781,
      "activations/layer3_attention_weight_min": -96.13639831542969,
      "activations/layer4_attention_weight_max": 105.98329162597656,
      "activations/layer4_attention_weight_min": -107.10796356201172,
      "activations/layer5_attention_weight_max": 50.440460205078125,
      "activations/layer5_attention_weight_min": -68.5107421875,
      "activations/layer6_attention_weight_max": 43.55911636352539,
      "activations/layer6_attention_weight_min": -46.907127380371094,
      "activations/layer7_attention_weight_max": 91.84156036376953,
      "activations/layer7_attention_weight_min": -95.80023193359375,
      "activations/layer8_attention_weight_max": 40.35969543457031,
      "activations/layer8_attention_weight_min": -44.21469497680664,
      "activations/layer9_attention_weight_max": 31.122451782226562,
      "activations/layer9_attention_weight_min": -33.97652816772461,
      "epoch": 18.43,
      "learning_rate": 3.14405303030303e-05,
      "loss": 2.7222,
      "step": 317250
    },
    {
      "activations/layer0_attention_weight_max": 15.84019660949707,
      "activations/layer0_attention_weight_min": -12.479568481445312,
      "activations/layer10_attention_weight_max": 36.34634780883789,
      "activations/layer10_attention_weight_min": -33.8956184387207,
      "activations/layer11_attention_weight_max": 34.4841194152832,
      "activations/layer11_attention_weight_min": -33.66127395629883,
      "activations/layer12_attention_weight_max": 27.703750610351562,
      "activations/layer12_attention_weight_min": -27.308170318603516,
      "activations/layer13_attention_weight_max": 40.22256088256836,
      "activations/layer13_attention_weight_min": -37.16291046142578,
      "activations/layer14_attention_weight_max": 44.192848205566406,
      "activations/layer14_attention_weight_min": -39.68457794189453,
      "activations/layer15_attention_weight_max": 40.224822998046875,
      "activations/layer15_attention_weight_min": -38.35661315917969,
      "activations/layer16_attention_weight_max": 29.606975555419922,
      "activations/layer16_attention_weight_min": -31.555700302124023,
      "activations/layer17_attention_weight_max": 30.040828704833984,
      "activations/layer17_attention_weight_min": -27.763748168945312,
      "activations/layer18_attention_weight_max": 32.10991668701172,
      "activations/layer18_attention_weight_min": -27.93758773803711,
      "activations/layer19_attention_weight_max": 36.36488342285156,
      "activations/layer19_attention_weight_min": -37.99656677246094,
      "activations/layer1_attention_weight_max": 17.33567237854004,
      "activations/layer1_attention_weight_min": -15.807525634765625,
      "activations/layer20_attention_weight_max": 30.3384952545166,
      "activations/layer20_attention_weight_min": -26.97307586669922,
      "activations/layer21_attention_weight_max": 31.079259872436523,
      "activations/layer21_attention_weight_min": -27.13616943359375,
      "activations/layer22_attention_weight_max": 47.46748733520508,
      "activations/layer22_attention_weight_min": -34.81606674194336,
      "activations/layer23_attention_weight_max": 36.31749725341797,
      "activations/layer23_attention_weight_min": -28.619701385498047,
      "activations/layer2_attention_weight_max": 32.061012268066406,
      "activations/layer2_attention_weight_min": -29.799930572509766,
      "activations/layer3_attention_weight_max": 101.49942779541016,
      "activations/layer3_attention_weight_min": -100.1019287109375,
      "activations/layer4_attention_weight_max": 100.9753646850586,
      "activations/layer4_attention_weight_min": -106.86622619628906,
      "activations/layer5_attention_weight_max": 49.714630126953125,
      "activations/layer5_attention_weight_min": -61.42171859741211,
      "activations/layer6_attention_weight_max": 43.36225128173828,
      "activations/layer6_attention_weight_min": -45.286537170410156,
      "activations/layer7_attention_weight_max": 91.6681900024414,
      "activations/layer7_attention_weight_min": -100.5262680053711,
      "activations/layer8_attention_weight_max": 40.01701736450195,
      "activations/layer8_attention_weight_min": -46.13583755493164,
      "activations/layer9_attention_weight_max": 30.595945358276367,
      "activations/layer9_attention_weight_min": -33.46906661987305,
      "epoch": 18.44,
      "learning_rate": 3.142159090909091e-05,
      "loss": 2.7506,
      "step": 317300
    },
    {
      "activations/layer0_attention_weight_max": 16.231760025024414,
      "activations/layer0_attention_weight_min": -12.172757148742676,
      "activations/layer10_attention_weight_max": 34.51380157470703,
      "activations/layer10_attention_weight_min": -35.17250442504883,
      "activations/layer11_attention_weight_max": 33.45542907714844,
      "activations/layer11_attention_weight_min": -34.301029205322266,
      "activations/layer12_attention_weight_max": 26.075071334838867,
      "activations/layer12_attention_weight_min": -27.454118728637695,
      "activations/layer13_attention_weight_max": 38.316871643066406,
      "activations/layer13_attention_weight_min": -36.47256088256836,
      "activations/layer14_attention_weight_max": 44.66929244995117,
      "activations/layer14_attention_weight_min": -40.296043395996094,
      "activations/layer15_attention_weight_max": 42.77958297729492,
      "activations/layer15_attention_weight_min": -41.01625442504883,
      "activations/layer16_attention_weight_max": 31.440576553344727,
      "activations/layer16_attention_weight_min": -27.70693588256836,
      "activations/layer17_attention_weight_max": 30.843801498413086,
      "activations/layer17_attention_weight_min": -27.30965805053711,
      "activations/layer18_attention_weight_max": 31.517240524291992,
      "activations/layer18_attention_weight_min": -27.93943977355957,
      "activations/layer19_attention_weight_max": 34.18960189819336,
      "activations/layer19_attention_weight_min": -29.393598556518555,
      "activations/layer1_attention_weight_max": 19.31580924987793,
      "activations/layer1_attention_weight_min": -15.433647155761719,
      "activations/layer20_attention_weight_max": 29.947906494140625,
      "activations/layer20_attention_weight_min": -23.71721649169922,
      "activations/layer21_attention_weight_max": 26.61148452758789,
      "activations/layer21_attention_weight_min": -22.8502140045166,
      "activations/layer22_attention_weight_max": 39.25373840332031,
      "activations/layer22_attention_weight_min": -31.424701690673828,
      "activations/layer23_attention_weight_max": 31.948589324951172,
      "activations/layer23_attention_weight_min": -25.595706939697266,
      "activations/layer2_attention_weight_max": 32.35597610473633,
      "activations/layer2_attention_weight_min": -30.560253143310547,
      "activations/layer3_attention_weight_max": 101.01478576660156,
      "activations/layer3_attention_weight_min": -96.7864761352539,
      "activations/layer4_attention_weight_max": 104.20574951171875,
      "activations/layer4_attention_weight_min": -100.63660430908203,
      "activations/layer5_attention_weight_max": 53.0516357421875,
      "activations/layer5_attention_weight_min": -60.170379638671875,
      "activations/layer6_attention_weight_max": 44.77974319458008,
      "activations/layer6_attention_weight_min": -45.43323516845703,
      "activations/layer7_attention_weight_max": 102.33118438720703,
      "activations/layer7_attention_weight_min": -92.89238739013672,
      "activations/layer8_attention_weight_max": 43.00920486450195,
      "activations/layer8_attention_weight_min": -43.5828742980957,
      "activations/layer9_attention_weight_max": 33.312747955322266,
      "activations/layer9_attention_weight_min": -35.31256103515625,
      "epoch": 18.44,
      "learning_rate": 3.140265151515151e-05,
      "loss": 2.7374,
      "step": 317350
    },
    {
      "activations/layer0_attention_weight_max": 16.397960662841797,
      "activations/layer0_attention_weight_min": -12.585976600646973,
      "activations/layer10_attention_weight_max": 39.88623809814453,
      "activations/layer10_attention_weight_min": -35.70466613769531,
      "activations/layer11_attention_weight_max": 34.9110107421875,
      "activations/layer11_attention_weight_min": -34.0515022277832,
      "activations/layer12_attention_weight_max": 26.612756729125977,
      "activations/layer12_attention_weight_min": -26.59760856628418,
      "activations/layer13_attention_weight_max": 39.74085235595703,
      "activations/layer13_attention_weight_min": -36.032135009765625,
      "activations/layer14_attention_weight_max": 45.39302444458008,
      "activations/layer14_attention_weight_min": -37.90945053100586,
      "activations/layer15_attention_weight_max": 38.422386169433594,
      "activations/layer15_attention_weight_min": -38.80525207519531,
      "activations/layer16_attention_weight_max": 30.339771270751953,
      "activations/layer16_attention_weight_min": -29.971271514892578,
      "activations/layer17_attention_weight_max": 28.502424240112305,
      "activations/layer17_attention_weight_min": -27.127010345458984,
      "activations/layer18_attention_weight_max": 31.23797035217285,
      "activations/layer18_attention_weight_min": -28.568340301513672,
      "activations/layer19_attention_weight_max": 36.288299560546875,
      "activations/layer19_attention_weight_min": -32.79146194458008,
      "activations/layer1_attention_weight_max": 16.878826141357422,
      "activations/layer1_attention_weight_min": -15.811676979064941,
      "activations/layer20_attention_weight_max": 29.925424575805664,
      "activations/layer20_attention_weight_min": -26.244182586669922,
      "activations/layer21_attention_weight_max": 31.64480209350586,
      "activations/layer21_attention_weight_min": -26.327571868896484,
      "activations/layer22_attention_weight_max": 43.85377502441406,
      "activations/layer22_attention_weight_min": -35.92052459716797,
      "activations/layer23_attention_weight_max": 34.29730987548828,
      "activations/layer23_attention_weight_min": -28.441274642944336,
      "activations/layer2_attention_weight_max": 31.694059371948242,
      "activations/layer2_attention_weight_min": -30.579490661621094,
      "activations/layer3_attention_weight_max": 95.2183837890625,
      "activations/layer3_attention_weight_min": -96.2010726928711,
      "activations/layer4_attention_weight_max": 101.67113494873047,
      "activations/layer4_attention_weight_min": -100.32451629638672,
      "activations/layer5_attention_weight_max": 49.98124313354492,
      "activations/layer5_attention_weight_min": -59.37067413330078,
      "activations/layer6_attention_weight_max": 42.88096237182617,
      "activations/layer6_attention_weight_min": -45.137638092041016,
      "activations/layer7_attention_weight_max": 94.60851287841797,
      "activations/layer7_attention_weight_min": -100.8841323852539,
      "activations/layer8_attention_weight_max": 42.93910217285156,
      "activations/layer8_attention_weight_min": -46.42458724975586,
      "activations/layer9_attention_weight_max": 34.60114669799805,
      "activations/layer9_attention_weight_min": -35.804012298583984,
      "epoch": 18.44,
      "learning_rate": 3.138371212121212e-05,
      "loss": 2.7423,
      "step": 317400
    },
    {
      "activations/layer0_attention_weight_max": 15.825296401977539,
      "activations/layer0_attention_weight_min": -13.520429611206055,
      "activations/layer10_attention_weight_max": 34.9691047668457,
      "activations/layer10_attention_weight_min": -35.28221893310547,
      "activations/layer11_attention_weight_max": 35.14552307128906,
      "activations/layer11_attention_weight_min": -35.44804763793945,
      "activations/layer12_attention_weight_max": 27.80717658996582,
      "activations/layer12_attention_weight_min": -26.730817794799805,
      "activations/layer13_attention_weight_max": 40.20241928100586,
      "activations/layer13_attention_weight_min": -35.3118896484375,
      "activations/layer14_attention_weight_max": 43.98480987548828,
      "activations/layer14_attention_weight_min": -38.3034782409668,
      "activations/layer15_attention_weight_max": 38.06509017944336,
      "activations/layer15_attention_weight_min": -35.982208251953125,
      "activations/layer16_attention_weight_max": 29.58197593688965,
      "activations/layer16_attention_weight_min": -30.510828018188477,
      "activations/layer17_attention_weight_max": 29.117794036865234,
      "activations/layer17_attention_weight_min": -26.328596115112305,
      "activations/layer18_attention_weight_max": 30.727645874023438,
      "activations/layer18_attention_weight_min": -24.751684188842773,
      "activations/layer19_attention_weight_max": 32.41059875488281,
      "activations/layer19_attention_weight_min": -32.18207931518555,
      "activations/layer1_attention_weight_max": 16.598432540893555,
      "activations/layer1_attention_weight_min": -12.83765983581543,
      "activations/layer20_attention_weight_max": 27.2346248626709,
      "activations/layer20_attention_weight_min": -26.732250213623047,
      "activations/layer21_attention_weight_max": 27.39017677307129,
      "activations/layer21_attention_weight_min": -26.568248748779297,
      "activations/layer22_attention_weight_max": 39.96040344238281,
      "activations/layer22_attention_weight_min": -31.76487922668457,
      "activations/layer23_attention_weight_max": 32.065284729003906,
      "activations/layer23_attention_weight_min": -28.935941696166992,
      "activations/layer2_attention_weight_max": 30.248149871826172,
      "activations/layer2_attention_weight_min": -29.897499084472656,
      "activations/layer3_attention_weight_max": 95.558837890625,
      "activations/layer3_attention_weight_min": -97.36962890625,
      "activations/layer4_attention_weight_max": 104.78661346435547,
      "activations/layer4_attention_weight_min": -104.70345306396484,
      "activations/layer5_attention_weight_max": 50.22831726074219,
      "activations/layer5_attention_weight_min": -60.144283294677734,
      "activations/layer6_attention_weight_max": 44.405147552490234,
      "activations/layer6_attention_weight_min": -47.65961837768555,
      "activations/layer7_attention_weight_max": 89.52051544189453,
      "activations/layer7_attention_weight_min": -95.18431091308594,
      "activations/layer8_attention_weight_max": 40.901981353759766,
      "activations/layer8_attention_weight_min": -43.83089828491211,
      "activations/layer9_attention_weight_max": 32.110511779785156,
      "activations/layer9_attention_weight_min": -34.61864471435547,
      "epoch": 18.45,
      "learning_rate": 3.1364772727272726e-05,
      "loss": 2.7278,
      "step": 317450
    },
    {
      "activations/layer0_attention_weight_max": 15.959818840026855,
      "activations/layer0_attention_weight_min": -13.02479362487793,
      "activations/layer10_attention_weight_max": 34.052703857421875,
      "activations/layer10_attention_weight_min": -33.846893310546875,
      "activations/layer11_attention_weight_max": 34.71818923950195,
      "activations/layer11_attention_weight_min": -34.61164474487305,
      "activations/layer12_attention_weight_max": 29.88927459716797,
      "activations/layer12_attention_weight_min": -29.621845245361328,
      "activations/layer13_attention_weight_max": 45.79773712158203,
      "activations/layer13_attention_weight_min": -39.726348876953125,
      "activations/layer14_attention_weight_max": 52.284183502197266,
      "activations/layer14_attention_weight_min": -44.191558837890625,
      "activations/layer15_attention_weight_max": 46.972198486328125,
      "activations/layer15_attention_weight_min": -40.356292724609375,
      "activations/layer16_attention_weight_max": 34.24767303466797,
      "activations/layer16_attention_weight_min": -31.214187622070312,
      "activations/layer17_attention_weight_max": 31.758249282836914,
      "activations/layer17_attention_weight_min": -27.49381446838379,
      "activations/layer18_attention_weight_max": 35.91242218017578,
      "activations/layer18_attention_weight_min": -27.76044273376465,
      "activations/layer19_attention_weight_max": 41.98017883300781,
      "activations/layer19_attention_weight_min": -33.563716888427734,
      "activations/layer1_attention_weight_max": 16.961448669433594,
      "activations/layer1_attention_weight_min": -13.095941543579102,
      "activations/layer20_attention_weight_max": 34.05197525024414,
      "activations/layer20_attention_weight_min": -26.657224655151367,
      "activations/layer21_attention_weight_max": 31.322851181030273,
      "activations/layer21_attention_weight_min": -25.861169815063477,
      "activations/layer22_attention_weight_max": 50.59431838989258,
      "activations/layer22_attention_weight_min": -33.88289260864258,
      "activations/layer23_attention_weight_max": 37.200775146484375,
      "activations/layer23_attention_weight_min": -26.72418975830078,
      "activations/layer2_attention_weight_max": 32.016014099121094,
      "activations/layer2_attention_weight_min": -30.368343353271484,
      "activations/layer3_attention_weight_max": 95.11795043945312,
      "activations/layer3_attention_weight_min": -97.0321273803711,
      "activations/layer4_attention_weight_max": 103.30936431884766,
      "activations/layer4_attention_weight_min": -106.24217987060547,
      "activations/layer5_attention_weight_max": 51.176998138427734,
      "activations/layer5_attention_weight_min": -67.53965759277344,
      "activations/layer6_attention_weight_max": 43.90846252441406,
      "activations/layer6_attention_weight_min": -46.58491134643555,
      "activations/layer7_attention_weight_max": 97.18477630615234,
      "activations/layer7_attention_weight_min": -94.66663360595703,
      "activations/layer8_attention_weight_max": 43.18072509765625,
      "activations/layer8_attention_weight_min": -42.72760009765625,
      "activations/layer9_attention_weight_max": 31.31670570373535,
      "activations/layer9_attention_weight_min": -32.193382263183594,
      "epoch": 18.45,
      "learning_rate": 3.1345833333333334e-05,
      "loss": 2.7117,
      "step": 317500
    },
    {
      "activations/layer0_attention_weight_max": 15.637500762939453,
      "activations/layer0_attention_weight_min": -12.071004867553711,
      "activations/layer10_attention_weight_max": 40.7132568359375,
      "activations/layer10_attention_weight_min": -35.5241584777832,
      "activations/layer11_attention_weight_max": 44.13578796386719,
      "activations/layer11_attention_weight_min": -39.58427429199219,
      "activations/layer12_attention_weight_max": 31.838472366333008,
      "activations/layer12_attention_weight_min": -29.134464263916016,
      "activations/layer13_attention_weight_max": 47.84233093261719,
      "activations/layer13_attention_weight_min": -38.32019805908203,
      "activations/layer14_attention_weight_max": 54.00947189331055,
      "activations/layer14_attention_weight_min": -42.88924789428711,
      "activations/layer15_attention_weight_max": 61.031986236572266,
      "activations/layer15_attention_weight_min": -43.8804931640625,
      "activations/layer16_attention_weight_max": 42.95780563354492,
      "activations/layer16_attention_weight_min": -30.648591995239258,
      "activations/layer17_attention_weight_max": 36.25817108154297,
      "activations/layer17_attention_weight_min": -30.170846939086914,
      "activations/layer18_attention_weight_max": 34.221126556396484,
      "activations/layer18_attention_weight_min": -29.7230167388916,
      "activations/layer19_attention_weight_max": 40.02716827392578,
      "activations/layer19_attention_weight_min": -33.4249153137207,
      "activations/layer1_attention_weight_max": 19.053531646728516,
      "activations/layer1_attention_weight_min": -15.818702697753906,
      "activations/layer20_attention_weight_max": 34.82610321044922,
      "activations/layer20_attention_weight_min": -25.222078323364258,
      "activations/layer21_attention_weight_max": 36.27340316772461,
      "activations/layer21_attention_weight_min": -26.53607749938965,
      "activations/layer22_attention_weight_max": 55.880069732666016,
      "activations/layer22_attention_weight_min": -35.076473236083984,
      "activations/layer23_attention_weight_max": 35.21300506591797,
      "activations/layer23_attention_weight_min": -24.384733200073242,
      "activations/layer2_attention_weight_max": 30.384944915771484,
      "activations/layer2_attention_weight_min": -31.44159698486328,
      "activations/layer3_attention_weight_max": 94.45455932617188,
      "activations/layer3_attention_weight_min": -95.40911865234375,
      "activations/layer4_attention_weight_max": 103.8349838256836,
      "activations/layer4_attention_weight_min": -105.0409927368164,
      "activations/layer5_attention_weight_max": 50.73080062866211,
      "activations/layer5_attention_weight_min": -61.012428283691406,
      "activations/layer6_attention_weight_max": 45.559139251708984,
      "activations/layer6_attention_weight_min": -49.150611877441406,
      "activations/layer7_attention_weight_max": 98.77206420898438,
      "activations/layer7_attention_weight_min": -96.18159484863281,
      "activations/layer8_attention_weight_max": 45.614585876464844,
      "activations/layer8_attention_weight_min": -47.73829650878906,
      "activations/layer9_attention_weight_max": 33.17947006225586,
      "activations/layer9_attention_weight_min": -35.64725112915039,
      "epoch": 18.45,
      "learning_rate": 3.1326893939393936e-05,
      "loss": 2.7472,
      "step": 317550
    },
    {
      "activations/layer0_attention_weight_max": 16.609268188476562,
      "activations/layer0_attention_weight_min": -13.206707000732422,
      "activations/layer10_attention_weight_max": 34.43885040283203,
      "activations/layer10_attention_weight_min": -35.3773078918457,
      "activations/layer11_attention_weight_max": 35.32880401611328,
      "activations/layer11_attention_weight_min": -36.335819244384766,
      "activations/layer12_attention_weight_max": 28.236616134643555,
      "activations/layer12_attention_weight_min": -26.965103149414062,
      "activations/layer13_attention_weight_max": 41.56840133666992,
      "activations/layer13_attention_weight_min": -35.251686096191406,
      "activations/layer14_attention_weight_max": 42.98862075805664,
      "activations/layer14_attention_weight_min": -39.446773529052734,
      "activations/layer15_attention_weight_max": 38.09920883178711,
      "activations/layer15_attention_weight_min": -34.16828918457031,
      "activations/layer16_attention_weight_max": 30.907625198364258,
      "activations/layer16_attention_weight_min": -29.491792678833008,
      "activations/layer17_attention_weight_max": 27.780887603759766,
      "activations/layer17_attention_weight_min": -27.478906631469727,
      "activations/layer18_attention_weight_max": 31.8061466217041,
      "activations/layer18_attention_weight_min": -26.427732467651367,
      "activations/layer19_attention_weight_max": 33.353763580322266,
      "activations/layer19_attention_weight_min": -30.39910316467285,
      "activations/layer1_attention_weight_max": 16.981924057006836,
      "activations/layer1_attention_weight_min": -14.356084823608398,
      "activations/layer20_attention_weight_max": 27.588762283325195,
      "activations/layer20_attention_weight_min": -26.64715576171875,
      "activations/layer21_attention_weight_max": 25.85724449157715,
      "activations/layer21_attention_weight_min": -25.007776260375977,
      "activations/layer22_attention_weight_max": 40.59991455078125,
      "activations/layer22_attention_weight_min": -32.147335052490234,
      "activations/layer23_attention_weight_max": 30.132041931152344,
      "activations/layer23_attention_weight_min": -25.770179748535156,
      "activations/layer2_attention_weight_max": 30.007884979248047,
      "activations/layer2_attention_weight_min": -29.27534294128418,
      "activations/layer3_attention_weight_max": 96.95183563232422,
      "activations/layer3_attention_weight_min": -98.40774536132812,
      "activations/layer4_attention_weight_max": 110.1249008178711,
      "activations/layer4_attention_weight_min": -108.53125,
      "activations/layer5_attention_weight_max": 51.50459289550781,
      "activations/layer5_attention_weight_min": -61.44044494628906,
      "activations/layer6_attention_weight_max": 45.38092803955078,
      "activations/layer6_attention_weight_min": -46.243709564208984,
      "activations/layer7_attention_weight_max": 94.28142547607422,
      "activations/layer7_attention_weight_min": -95.39543914794922,
      "activations/layer8_attention_weight_max": 44.02941131591797,
      "activations/layer8_attention_weight_min": -43.84382247924805,
      "activations/layer9_attention_weight_max": 34.503658294677734,
      "activations/layer9_attention_weight_min": -35.384456634521484,
      "epoch": 18.45,
      "learning_rate": 3.1307954545454544e-05,
      "loss": 2.7295,
      "step": 317600
    },
    {
      "activations/layer0_attention_weight_max": 15.83591079711914,
      "activations/layer0_attention_weight_min": -13.305919647216797,
      "activations/layer10_attention_weight_max": 36.33913040161133,
      "activations/layer10_attention_weight_min": -37.47964859008789,
      "activations/layer11_attention_weight_max": 34.26589584350586,
      "activations/layer11_attention_weight_min": -36.51982116699219,
      "activations/layer12_attention_weight_max": 28.831336975097656,
      "activations/layer12_attention_weight_min": -27.178836822509766,
      "activations/layer13_attention_weight_max": 37.565032958984375,
      "activations/layer13_attention_weight_min": -36.54822540283203,
      "activations/layer14_attention_weight_max": 39.534706115722656,
      "activations/layer14_attention_weight_min": -39.03519821166992,
      "activations/layer15_attention_weight_max": 37.563392639160156,
      "activations/layer15_attention_weight_min": -38.70518493652344,
      "activations/layer16_attention_weight_max": 28.427337646484375,
      "activations/layer16_attention_weight_min": -32.031803131103516,
      "activations/layer17_attention_weight_max": 29.276819229125977,
      "activations/layer17_attention_weight_min": -25.58989906311035,
      "activations/layer18_attention_weight_max": 30.226552963256836,
      "activations/layer18_attention_weight_min": -25.34162139892578,
      "activations/layer19_attention_weight_max": 32.11128234863281,
      "activations/layer19_attention_weight_min": -31.119449615478516,
      "activations/layer1_attention_weight_max": 17.488201141357422,
      "activations/layer1_attention_weight_min": -13.377846717834473,
      "activations/layer20_attention_weight_max": 26.379581451416016,
      "activations/layer20_attention_weight_min": -24.42021942138672,
      "activations/layer21_attention_weight_max": 27.329586029052734,
      "activations/layer21_attention_weight_min": -25.511058807373047,
      "activations/layer22_attention_weight_max": 38.603878021240234,
      "activations/layer22_attention_weight_min": -31.611005783081055,
      "activations/layer23_attention_weight_max": 29.622570037841797,
      "activations/layer23_attention_weight_min": -26.8865966796875,
      "activations/layer2_attention_weight_max": 31.330915451049805,
      "activations/layer2_attention_weight_min": -31.196311950683594,
      "activations/layer3_attention_weight_max": 99.2997817993164,
      "activations/layer3_attention_weight_min": -101.73054504394531,
      "activations/layer4_attention_weight_max": 100.72164154052734,
      "activations/layer4_attention_weight_min": -102.8170166015625,
      "activations/layer5_attention_weight_max": 47.94857406616211,
      "activations/layer5_attention_weight_min": -58.24125671386719,
      "activations/layer6_attention_weight_max": 44.525577545166016,
      "activations/layer6_attention_weight_min": -45.847618103027344,
      "activations/layer7_attention_weight_max": 95.6661148071289,
      "activations/layer7_attention_weight_min": -100.25161743164062,
      "activations/layer8_attention_weight_max": 44.831512451171875,
      "activations/layer8_attention_weight_min": -46.8079719543457,
      "activations/layer9_attention_weight_max": 31.888540267944336,
      "activations/layer9_attention_weight_min": -37.56963348388672,
      "epoch": 18.46,
      "learning_rate": 3.1289015151515145e-05,
      "loss": 2.7303,
      "step": 317650
    },
    {
      "activations/layer0_attention_weight_max": 17.250802993774414,
      "activations/layer0_attention_weight_min": -12.661380767822266,
      "activations/layer10_attention_weight_max": 38.064117431640625,
      "activations/layer10_attention_weight_min": -34.65386962890625,
      "activations/layer11_attention_weight_max": 41.124656677246094,
      "activations/layer11_attention_weight_min": -38.298667907714844,
      "activations/layer12_attention_weight_max": 29.052824020385742,
      "activations/layer12_attention_weight_min": -29.173946380615234,
      "activations/layer13_attention_weight_max": 42.61454772949219,
      "activations/layer13_attention_weight_min": -42.55436325073242,
      "activations/layer14_attention_weight_max": 49.84941482543945,
      "activations/layer14_attention_weight_min": -44.286651611328125,
      "activations/layer15_attention_weight_max": 44.089656829833984,
      "activations/layer15_attention_weight_min": -39.50315475463867,
      "activations/layer16_attention_weight_max": 31.946069717407227,
      "activations/layer16_attention_weight_min": -30.924291610717773,
      "activations/layer17_attention_weight_max": 32.03337860107422,
      "activations/layer17_attention_weight_min": -27.59190559387207,
      "activations/layer18_attention_weight_max": 33.96061706542969,
      "activations/layer18_attention_weight_min": -28.505756378173828,
      "activations/layer19_attention_weight_max": 39.834861755371094,
      "activations/layer19_attention_weight_min": -31.778066635131836,
      "activations/layer1_attention_weight_max": 17.011646270751953,
      "activations/layer1_attention_weight_min": -14.99032211303711,
      "activations/layer20_attention_weight_max": 33.55197525024414,
      "activations/layer20_attention_weight_min": -28.1019287109375,
      "activations/layer21_attention_weight_max": 30.123769760131836,
      "activations/layer21_attention_weight_min": -26.354198455810547,
      "activations/layer22_attention_weight_max": 44.97039794921875,
      "activations/layer22_attention_weight_min": -34.564910888671875,
      "activations/layer23_attention_weight_max": 34.78453826904297,
      "activations/layer23_attention_weight_min": -28.753986358642578,
      "activations/layer2_attention_weight_max": 34.13817596435547,
      "activations/layer2_attention_weight_min": -30.675518035888672,
      "activations/layer3_attention_weight_max": 100.62279510498047,
      "activations/layer3_attention_weight_min": -104.82368469238281,
      "activations/layer4_attention_weight_max": 108.26875305175781,
      "activations/layer4_attention_weight_min": -112.70556640625,
      "activations/layer5_attention_weight_max": 50.44868087768555,
      "activations/layer5_attention_weight_min": -61.25395965576172,
      "activations/layer6_attention_weight_max": 43.87428665161133,
      "activations/layer6_attention_weight_min": -45.514427185058594,
      "activations/layer7_attention_weight_max": 97.2864990234375,
      "activations/layer7_attention_weight_min": -99.24329376220703,
      "activations/layer8_attention_weight_max": 43.00332260131836,
      "activations/layer8_attention_weight_min": -45.954017639160156,
      "activations/layer9_attention_weight_max": 34.6169548034668,
      "activations/layer9_attention_weight_min": -33.98970413208008,
      "epoch": 18.46,
      "learning_rate": 3.1270075757575754e-05,
      "loss": 2.7339,
      "step": 317700
    },
    {
      "activations/layer0_attention_weight_max": 16.340869903564453,
      "activations/layer0_attention_weight_min": -13.870969772338867,
      "activations/layer10_attention_weight_max": 33.20534896850586,
      "activations/layer10_attention_weight_min": -33.546043395996094,
      "activations/layer11_attention_weight_max": 33.84626007080078,
      "activations/layer11_attention_weight_min": -34.798187255859375,
      "activations/layer12_attention_weight_max": 26.55112648010254,
      "activations/layer12_attention_weight_min": -27.443668365478516,
      "activations/layer13_attention_weight_max": 41.66764831542969,
      "activations/layer13_attention_weight_min": -36.4715690612793,
      "activations/layer14_attention_weight_max": 41.970191955566406,
      "activations/layer14_attention_weight_min": -41.18412780761719,
      "activations/layer15_attention_weight_max": 40.26348876953125,
      "activations/layer15_attention_weight_min": -36.24357604980469,
      "activations/layer16_attention_weight_max": 29.41507339477539,
      "activations/layer16_attention_weight_min": -28.75173568725586,
      "activations/layer17_attention_weight_max": 30.154170989990234,
      "activations/layer17_attention_weight_min": -27.032236099243164,
      "activations/layer18_attention_weight_max": 31.64794921875,
      "activations/layer18_attention_weight_min": -28.48243522644043,
      "activations/layer19_attention_weight_max": 34.09577560424805,
      "activations/layer19_attention_weight_min": -32.38372039794922,
      "activations/layer1_attention_weight_max": 16.57215690612793,
      "activations/layer1_attention_weight_min": -15.168105125427246,
      "activations/layer20_attention_weight_max": 28.725440979003906,
      "activations/layer20_attention_weight_min": -26.7332763671875,
      "activations/layer21_attention_weight_max": 28.35527801513672,
      "activations/layer21_attention_weight_min": -25.688711166381836,
      "activations/layer22_attention_weight_max": 42.37943649291992,
      "activations/layer22_attention_weight_min": -32.33351135253906,
      "activations/layer23_attention_weight_max": 32.05595016479492,
      "activations/layer23_attention_weight_min": -27.973241806030273,
      "activations/layer2_attention_weight_max": 30.108924865722656,
      "activations/layer2_attention_weight_min": -29.16925048828125,
      "activations/layer3_attention_weight_max": 97.1723861694336,
      "activations/layer3_attention_weight_min": -96.25880432128906,
      "activations/layer4_attention_weight_max": 105.5744400024414,
      "activations/layer4_attention_weight_min": -109.18243408203125,
      "activations/layer5_attention_weight_max": 52.31422424316406,
      "activations/layer5_attention_weight_min": -65.44921875,
      "activations/layer6_attention_weight_max": 42.69016647338867,
      "activations/layer6_attention_weight_min": -47.794761657714844,
      "activations/layer7_attention_weight_max": 92.3303451538086,
      "activations/layer7_attention_weight_min": -98.85179901123047,
      "activations/layer8_attention_weight_max": 40.94325637817383,
      "activations/layer8_attention_weight_min": -46.62086486816406,
      "activations/layer9_attention_weight_max": 32.63655090332031,
      "activations/layer9_attention_weight_min": -33.664581298828125,
      "epoch": 18.46,
      "learning_rate": 3.125113636363636e-05,
      "loss": 2.7261,
      "step": 317750
    },
    {
      "activations/layer0_attention_weight_max": 16.62483787536621,
      "activations/layer0_attention_weight_min": -13.065579414367676,
      "activations/layer10_attention_weight_max": 36.14892578125,
      "activations/layer10_attention_weight_min": -35.89776611328125,
      "activations/layer11_attention_weight_max": 37.514381408691406,
      "activations/layer11_attention_weight_min": -36.51216125488281,
      "activations/layer12_attention_weight_max": 27.913965225219727,
      "activations/layer12_attention_weight_min": -27.472461700439453,
      "activations/layer13_attention_weight_max": 43.46813201904297,
      "activations/layer13_attention_weight_min": -36.501495361328125,
      "activations/layer14_attention_weight_max": 44.66648864746094,
      "activations/layer14_attention_weight_min": -40.15398025512695,
      "activations/layer15_attention_weight_max": 40.96791458129883,
      "activations/layer15_attention_weight_min": -36.894161224365234,
      "activations/layer16_attention_weight_max": 32.36525344848633,
      "activations/layer16_attention_weight_min": -28.513713836669922,
      "activations/layer17_attention_weight_max": 30.768415451049805,
      "activations/layer17_attention_weight_min": -25.682497024536133,
      "activations/layer18_attention_weight_max": 34.300559997558594,
      "activations/layer18_attention_weight_min": -24.87543487548828,
      "activations/layer19_attention_weight_max": 34.99061965942383,
      "activations/layer19_attention_weight_min": -30.646818161010742,
      "activations/layer1_attention_weight_max": 16.61366081237793,
      "activations/layer1_attention_weight_min": -15.429959297180176,
      "activations/layer20_attention_weight_max": 29.7758731842041,
      "activations/layer20_attention_weight_min": -25.37079429626465,
      "activations/layer21_attention_weight_max": 28.8148193359375,
      "activations/layer21_attention_weight_min": -25.4095401763916,
      "activations/layer22_attention_weight_max": 39.55271530151367,
      "activations/layer22_attention_weight_min": -32.540122985839844,
      "activations/layer23_attention_weight_max": 31.946144104003906,
      "activations/layer23_attention_weight_min": -25.467254638671875,
      "activations/layer2_attention_weight_max": 30.537738800048828,
      "activations/layer2_attention_weight_min": -30.30592155456543,
      "activations/layer3_attention_weight_max": 99.17097473144531,
      "activations/layer3_attention_weight_min": -99.14979553222656,
      "activations/layer4_attention_weight_max": 107.22723388671875,
      "activations/layer4_attention_weight_min": -108.85517883300781,
      "activations/layer5_attention_weight_max": 51.13889694213867,
      "activations/layer5_attention_weight_min": -61.51034164428711,
      "activations/layer6_attention_weight_max": 42.926063537597656,
      "activations/layer6_attention_weight_min": -46.748043060302734,
      "activations/layer7_attention_weight_max": 97.19388580322266,
      "activations/layer7_attention_weight_min": -94.27468872070312,
      "activations/layer8_attention_weight_max": 42.62213134765625,
      "activations/layer8_attention_weight_min": -45.53904342651367,
      "activations/layer9_attention_weight_max": 33.403472900390625,
      "activations/layer9_attention_weight_min": -36.24293899536133,
      "epoch": 18.47,
      "learning_rate": 3.123219696969697e-05,
      "loss": 2.7154,
      "step": 317800
    },
    {
      "activations/layer0_attention_weight_max": 15.764008522033691,
      "activations/layer0_attention_weight_min": -12.28736400604248,
      "activations/layer10_attention_weight_max": 40.293216705322266,
      "activations/layer10_attention_weight_min": -36.1489143371582,
      "activations/layer11_attention_weight_max": 42.57947540283203,
      "activations/layer11_attention_weight_min": -39.437965393066406,
      "activations/layer12_attention_weight_max": 26.338924407958984,
      "activations/layer12_attention_weight_min": -27.393003463745117,
      "activations/layer13_attention_weight_max": 38.33056640625,
      "activations/layer13_attention_weight_min": -37.71550750732422,
      "activations/layer14_attention_weight_max": 42.91823196411133,
      "activations/layer14_attention_weight_min": -38.53263473510742,
      "activations/layer15_attention_weight_max": 39.559024810791016,
      "activations/layer15_attention_weight_min": -37.82668685913086,
      "activations/layer16_attention_weight_max": 29.949129104614258,
      "activations/layer16_attention_weight_min": -27.997770309448242,
      "activations/layer17_attention_weight_max": 29.181434631347656,
      "activations/layer17_attention_weight_min": -28.29030418395996,
      "activations/layer18_attention_weight_max": 34.8979606628418,
      "activations/layer18_attention_weight_min": -25.999296188354492,
      "activations/layer19_attention_weight_max": 35.06325149536133,
      "activations/layer19_attention_weight_min": -34.51571273803711,
      "activations/layer1_attention_weight_max": 16.3678035736084,
      "activations/layer1_attention_weight_min": -14.913202285766602,
      "activations/layer20_attention_weight_max": 28.9859619140625,
      "activations/layer20_attention_weight_min": -27.129459381103516,
      "activations/layer21_attention_weight_max": 31.935636520385742,
      "activations/layer21_attention_weight_min": -27.909122467041016,
      "activations/layer22_attention_weight_max": 41.85196304321289,
      "activations/layer22_attention_weight_min": -31.739749908447266,
      "activations/layer23_attention_weight_max": 31.34490203857422,
      "activations/layer23_attention_weight_min": -25.897693634033203,
      "activations/layer2_attention_weight_max": 32.32085418701172,
      "activations/layer2_attention_weight_min": -30.56673812866211,
      "activations/layer3_attention_weight_max": 99.3023910522461,
      "activations/layer3_attention_weight_min": -94.28919219970703,
      "activations/layer4_attention_weight_max": 106.85746002197266,
      "activations/layer4_attention_weight_min": -110.03376770019531,
      "activations/layer5_attention_weight_max": 54.707008361816406,
      "activations/layer5_attention_weight_min": -60.557647705078125,
      "activations/layer6_attention_weight_max": 45.838600158691406,
      "activations/layer6_attention_weight_min": -46.39887619018555,
      "activations/layer7_attention_weight_max": 97.5984115600586,
      "activations/layer7_attention_weight_min": -94.75010681152344,
      "activations/layer8_attention_weight_max": 47.01787185668945,
      "activations/layer8_attention_weight_min": -43.55461502075195,
      "activations/layer9_attention_weight_max": 34.6805534362793,
      "activations/layer9_attention_weight_min": -35.69224548339844,
      "epoch": 18.47,
      "learning_rate": 3.121363636363636e-05,
      "loss": 2.7274,
      "step": 317850
    },
    {
      "activations/layer0_attention_weight_max": 18.25098991394043,
      "activations/layer0_attention_weight_min": -13.407946586608887,
      "activations/layer10_attention_weight_max": 41.75139236450195,
      "activations/layer10_attention_weight_min": -35.11837387084961,
      "activations/layer11_attention_weight_max": 40.73136901855469,
      "activations/layer11_attention_weight_min": -36.424495697021484,
      "activations/layer12_attention_weight_max": 28.06808853149414,
      "activations/layer12_attention_weight_min": -28.509078979492188,
      "activations/layer13_attention_weight_max": 40.94743728637695,
      "activations/layer13_attention_weight_min": -38.3962516784668,
      "activations/layer14_attention_weight_max": 45.12849044799805,
      "activations/layer14_attention_weight_min": -38.09577178955078,
      "activations/layer15_attention_weight_max": 38.39178466796875,
      "activations/layer15_attention_weight_min": -36.47661590576172,
      "activations/layer16_attention_weight_max": 29.354673385620117,
      "activations/layer16_attention_weight_min": -30.36106300354004,
      "activations/layer17_attention_weight_max": 30.52701187133789,
      "activations/layer17_attention_weight_min": -27.11567497253418,
      "activations/layer18_attention_weight_max": 28.8560848236084,
      "activations/layer18_attention_weight_min": -26.44546127319336,
      "activations/layer19_attention_weight_max": 37.652076721191406,
      "activations/layer19_attention_weight_min": -33.12800598144531,
      "activations/layer1_attention_weight_max": 18.37818717956543,
      "activations/layer1_attention_weight_min": -15.615903854370117,
      "activations/layer20_attention_weight_max": 30.52142333984375,
      "activations/layer20_attention_weight_min": -26.2751522064209,
      "activations/layer21_attention_weight_max": 29.539169311523438,
      "activations/layer21_attention_weight_min": -25.25877571105957,
      "activations/layer22_attention_weight_max": 41.198516845703125,
      "activations/layer22_attention_weight_min": -33.822471618652344,
      "activations/layer23_attention_weight_max": 33.66278076171875,
      "activations/layer23_attention_weight_min": -28.29463005065918,
      "activations/layer2_attention_weight_max": 31.469144821166992,
      "activations/layer2_attention_weight_min": -31.72272491455078,
      "activations/layer3_attention_weight_max": 99.52933502197266,
      "activations/layer3_attention_weight_min": -96.85881805419922,
      "activations/layer4_attention_weight_max": 104.16510009765625,
      "activations/layer4_attention_weight_min": -105.92589569091797,
      "activations/layer5_attention_weight_max": 50.166839599609375,
      "activations/layer5_attention_weight_min": -63.784446716308594,
      "activations/layer6_attention_weight_max": 46.04598617553711,
      "activations/layer6_attention_weight_min": -47.7092170715332,
      "activations/layer7_attention_weight_max": 103.8515396118164,
      "activations/layer7_attention_weight_min": -101.86720275878906,
      "activations/layer8_attention_weight_max": 49.666404724121094,
      "activations/layer8_attention_weight_min": -46.352134704589844,
      "activations/layer9_attention_weight_max": 37.630775451660156,
      "activations/layer9_attention_weight_min": -36.012577056884766,
      "epoch": 18.47,
      "learning_rate": 3.119469696969697e-05,
      "loss": 2.73,
      "step": 317900
    },
    {
      "activations/layer0_attention_weight_max": 16.83547592163086,
      "activations/layer0_attention_weight_min": -12.288111686706543,
      "activations/layer10_attention_weight_max": 33.02510452270508,
      "activations/layer10_attention_weight_min": -36.30270004272461,
      "activations/layer11_attention_weight_max": 33.680973052978516,
      "activations/layer11_attention_weight_min": -35.108585357666016,
      "activations/layer12_attention_weight_max": 28.835025787353516,
      "activations/layer12_attention_weight_min": -29.798128128051758,
      "activations/layer13_attention_weight_max": 39.55947494506836,
      "activations/layer13_attention_weight_min": -35.3863525390625,
      "activations/layer14_attention_weight_max": 48.00806427001953,
      "activations/layer14_attention_weight_min": -39.04307174682617,
      "activations/layer15_attention_weight_max": 41.08540725708008,
      "activations/layer15_attention_weight_min": -38.14292526245117,
      "activations/layer16_attention_weight_max": 29.163957595825195,
      "activations/layer16_attention_weight_min": -28.114694595336914,
      "activations/layer17_attention_weight_max": 28.98526954650879,
      "activations/layer17_attention_weight_min": -27.887989044189453,
      "activations/layer18_attention_weight_max": 31.853267669677734,
      "activations/layer18_attention_weight_min": -27.73304557800293,
      "activations/layer19_attention_weight_max": 32.00227355957031,
      "activations/layer19_attention_weight_min": -30.083393096923828,
      "activations/layer1_attention_weight_max": 17.627674102783203,
      "activations/layer1_attention_weight_min": -16.095746994018555,
      "activations/layer20_attention_weight_max": 29.485706329345703,
      "activations/layer20_attention_weight_min": -24.464216232299805,
      "activations/layer21_attention_weight_max": 27.510250091552734,
      "activations/layer21_attention_weight_min": -25.0918025970459,
      "activations/layer22_attention_weight_max": 45.47012710571289,
      "activations/layer22_attention_weight_min": -31.612648010253906,
      "activations/layer23_attention_weight_max": 32.6063232421875,
      "activations/layer23_attention_weight_min": -27.0528564453125,
      "activations/layer2_attention_weight_max": 33.79169464111328,
      "activations/layer2_attention_weight_min": -31.358917236328125,
      "activations/layer3_attention_weight_max": 98.28861236572266,
      "activations/layer3_attention_weight_min": -100.71583557128906,
      "activations/layer4_attention_weight_max": 107.179931640625,
      "activations/layer4_attention_weight_min": -107.51299285888672,
      "activations/layer5_attention_weight_max": 52.546783447265625,
      "activations/layer5_attention_weight_min": -58.34931564331055,
      "activations/layer6_attention_weight_max": 43.94404983520508,
      "activations/layer6_attention_weight_min": -46.91487503051758,
      "activations/layer7_attention_weight_max": 92.20936584472656,
      "activations/layer7_attention_weight_min": -90.90887451171875,
      "activations/layer8_attention_weight_max": 41.41943359375,
      "activations/layer8_attention_weight_min": -45.5573616027832,
      "activations/layer9_attention_weight_max": 32.38538360595703,
      "activations/layer9_attention_weight_min": -34.77814483642578,
      "epoch": 18.47,
      "learning_rate": 3.117613636363636e-05,
      "loss": 2.7251,
      "step": 317950
    },
    {
      "activations/layer0_attention_weight_max": 16.997764587402344,
      "activations/layer0_attention_weight_min": -12.082097053527832,
      "activations/layer10_attention_weight_max": 32.72919464111328,
      "activations/layer10_attention_weight_min": -35.110774993896484,
      "activations/layer11_attention_weight_max": 33.2602653503418,
      "activations/layer11_attention_weight_min": -35.290653228759766,
      "activations/layer12_attention_weight_max": 27.46328353881836,
      "activations/layer12_attention_weight_min": -25.20527458190918,
      "activations/layer13_attention_weight_max": 40.73568344116211,
      "activations/layer13_attention_weight_min": -34.84648895263672,
      "activations/layer14_attention_weight_max": 42.41831588745117,
      "activations/layer14_attention_weight_min": -35.66194534301758,
      "activations/layer15_attention_weight_max": 38.903682708740234,
      "activations/layer15_attention_weight_min": -35.20906448364258,
      "activations/layer16_attention_weight_max": 31.30611228942871,
      "activations/layer16_attention_weight_min": -27.58441734313965,
      "activations/layer17_attention_weight_max": 29.190242767333984,
      "activations/layer17_attention_weight_min": -25.863880157470703,
      "activations/layer18_attention_weight_max": 29.805166244506836,
      "activations/layer18_attention_weight_min": -25.327556610107422,
      "activations/layer19_attention_weight_max": 31.554702758789062,
      "activations/layer19_attention_weight_min": -31.14822769165039,
      "activations/layer1_attention_weight_max": 16.49993133544922,
      "activations/layer1_attention_weight_min": -15.31440544128418,
      "activations/layer20_attention_weight_max": 29.13456916809082,
      "activations/layer20_attention_weight_min": -26.76186752319336,
      "activations/layer21_attention_weight_max": 27.64370346069336,
      "activations/layer21_attention_weight_min": -25.6066837310791,
      "activations/layer22_attention_weight_max": 40.93989562988281,
      "activations/layer22_attention_weight_min": -31.47096061706543,
      "activations/layer23_attention_weight_max": 31.388439178466797,
      "activations/layer23_attention_weight_min": -26.001644134521484,
      "activations/layer2_attention_weight_max": 30.695796966552734,
      "activations/layer2_attention_weight_min": -33.89320373535156,
      "activations/layer3_attention_weight_max": 101.25106811523438,
      "activations/layer3_attention_weight_min": -106.0599594116211,
      "activations/layer4_attention_weight_max": 108.34896087646484,
      "activations/layer4_attention_weight_min": -115.27562713623047,
      "activations/layer5_attention_weight_max": 50.34988021850586,
      "activations/layer5_attention_weight_min": -68.55329895019531,
      "activations/layer6_attention_weight_max": 46.76834487915039,
      "activations/layer6_attention_weight_min": -49.092899322509766,
      "activations/layer7_attention_weight_max": 93.84610748291016,
      "activations/layer7_attention_weight_min": -95.93251037597656,
      "activations/layer8_attention_weight_max": 41.283050537109375,
      "activations/layer8_attention_weight_min": -45.708133697509766,
      "activations/layer9_attention_weight_max": 33.04236602783203,
      "activations/layer9_attention_weight_min": -34.67246627807617,
      "epoch": 18.48,
      "learning_rate": 3.115719696969697e-05,
      "loss": 2.7129,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.4808,
      "eval_samples_per_second": 506.318,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.4808,
      "eval_openwebtext_samples_per_second": 506.318,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 2.0091,
      "eval_wikitext_samples_per_second": 226.963,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.5724,
      "eval_lambada_samples_per_second": 508.649,
      "step": 318000
    },
    {
      "activations/layer0_attention_weight_max": 16.35916519165039,
      "activations/layer0_attention_weight_min": -12.887579917907715,
      "activations/layer10_attention_weight_max": 32.31776809692383,
      "activations/layer10_attention_weight_min": -33.421607971191406,
      "activations/layer11_attention_weight_max": 31.495664596557617,
      "activations/layer11_attention_weight_min": -35.09961700439453,
      "activations/layer12_attention_weight_max": 27.250120162963867,
      "activations/layer12_attention_weight_min": -25.855375289916992,
      "activations/layer13_attention_weight_max": 39.715877532958984,
      "activations/layer13_attention_weight_min": -36.18049240112305,
      "activations/layer14_attention_weight_max": 45.389312744140625,
      "activations/layer14_attention_weight_min": -38.71015548706055,
      "activations/layer15_attention_weight_max": 39.37012481689453,
      "activations/layer15_attention_weight_min": -36.77788162231445,
      "activations/layer16_attention_weight_max": 28.951183319091797,
      "activations/layer16_attention_weight_min": -28.254688262939453,
      "activations/layer17_attention_weight_max": 36.14052200317383,
      "activations/layer17_attention_weight_min": -25.96859359741211,
      "activations/layer18_attention_weight_max": 31.001487731933594,
      "activations/layer18_attention_weight_min": -26.692941665649414,
      "activations/layer19_attention_weight_max": 34.669979095458984,
      "activations/layer19_attention_weight_min": -32.79681396484375,
      "activations/layer1_attention_weight_max": 17.459299087524414,
      "activations/layer1_attention_weight_min": -14.44079303741455,
      "activations/layer20_attention_weight_max": 29.841411590576172,
      "activations/layer20_attention_weight_min": -26.556621551513672,
      "activations/layer21_attention_weight_max": 31.17494773864746,
      "activations/layer21_attention_weight_min": -28.02402687072754,
      "activations/layer22_attention_weight_max": 42.37826156616211,
      "activations/layer22_attention_weight_min": -34.60621643066406,
      "activations/layer23_attention_weight_max": 33.930992126464844,
      "activations/layer23_attention_weight_min": -25.445236206054688,
      "activations/layer2_attention_weight_max": 29.45977020263672,
      "activations/layer2_attention_weight_min": -29.404800415039062,
      "activations/layer3_attention_weight_max": 93.92568969726562,
      "activations/layer3_attention_weight_min": -94.76423645019531,
      "activations/layer4_attention_weight_max": 103.1424331665039,
      "activations/layer4_attention_weight_min": -108.55635070800781,
      "activations/layer5_attention_weight_max": 48.50028610229492,
      "activations/layer5_attention_weight_min": -63.60614776611328,
      "activations/layer6_attention_weight_max": 43.64601135253906,
      "activations/layer6_attention_weight_min": -45.641841888427734,
      "activations/layer7_attention_weight_max": 90.93618774414062,
      "activations/layer7_attention_weight_min": -94.157958984375,
      "activations/layer8_attention_weight_max": 38.967647552490234,
      "activations/layer8_attention_weight_min": -43.47532653808594,
      "activations/layer9_attention_weight_max": 33.02402114868164,
      "activations/layer9_attention_weight_min": -34.74915313720703,
      "epoch": 18.48,
      "learning_rate": 3.1138257575757577e-05,
      "loss": 2.7297,
      "step": 318050
    },
    {
      "activations/layer0_attention_weight_max": 15.48743724822998,
      "activations/layer0_attention_weight_min": -14.446944236755371,
      "activations/layer10_attention_weight_max": 33.887725830078125,
      "activations/layer10_attention_weight_min": -34.61161804199219,
      "activations/layer11_attention_weight_max": 34.582828521728516,
      "activations/layer11_attention_weight_min": -34.14404296875,
      "activations/layer12_attention_weight_max": 28.973840713500977,
      "activations/layer12_attention_weight_min": -27.137964248657227,
      "activations/layer13_attention_weight_max": 40.92771911621094,
      "activations/layer13_attention_weight_min": -36.93555450439453,
      "activations/layer14_attention_weight_max": 41.51791000366211,
      "activations/layer14_attention_weight_min": -39.70840835571289,
      "activations/layer15_attention_weight_max": 36.19569396972656,
      "activations/layer15_attention_weight_min": -36.997772216796875,
      "activations/layer16_attention_weight_max": 29.06252098083496,
      "activations/layer16_attention_weight_min": -31.281400680541992,
      "activations/layer17_attention_weight_max": 27.618593215942383,
      "activations/layer17_attention_weight_min": -27.634807586669922,
      "activations/layer18_attention_weight_max": 32.27677917480469,
      "activations/layer18_attention_weight_min": -27.345443725585938,
      "activations/layer19_attention_weight_max": 34.079246520996094,
      "activations/layer19_attention_weight_min": -35.029197692871094,
      "activations/layer1_attention_weight_max": 15.770267486572266,
      "activations/layer1_attention_weight_min": -16.413339614868164,
      "activations/layer20_attention_weight_max": 29.4775333404541,
      "activations/layer20_attention_weight_min": -26.518505096435547,
      "activations/layer21_attention_weight_max": 29.765920639038086,
      "activations/layer21_attention_weight_min": -25.684967041015625,
      "activations/layer22_attention_weight_max": 41.94109344482422,
      "activations/layer22_attention_weight_min": -33.379146575927734,
      "activations/layer23_attention_weight_max": 31.97158432006836,
      "activations/layer23_attention_weight_min": -28.52735710144043,
      "activations/layer2_attention_weight_max": 32.119476318359375,
      "activations/layer2_attention_weight_min": -29.721830368041992,
      "activations/layer3_attention_weight_max": 96.3708724975586,
      "activations/layer3_attention_weight_min": -95.32450866699219,
      "activations/layer4_attention_weight_max": 102.83061981201172,
      "activations/layer4_attention_weight_min": -103.26192474365234,
      "activations/layer5_attention_weight_max": 57.00749969482422,
      "activations/layer5_attention_weight_min": -60.270294189453125,
      "activations/layer6_attention_weight_max": 47.73109817504883,
      "activations/layer6_attention_weight_min": -45.28827667236328,
      "activations/layer7_attention_weight_max": 94.82852935791016,
      "activations/layer7_attention_weight_min": -99.68157196044922,
      "activations/layer8_attention_weight_max": 45.58781814575195,
      "activations/layer8_attention_weight_min": -42.68879318237305,
      "activations/layer9_attention_weight_max": 31.864782333374023,
      "activations/layer9_attention_weight_min": -32.73101043701172,
      "epoch": 18.48,
      "learning_rate": 3.111931818181818e-05,
      "loss": 2.7351,
      "step": 318100
    },
    {
      "activations/layer0_attention_weight_max": 16.613283157348633,
      "activations/layer0_attention_weight_min": -12.798213958740234,
      "activations/layer10_attention_weight_max": 35.903785705566406,
      "activations/layer10_attention_weight_min": -35.343482971191406,
      "activations/layer11_attention_weight_max": 40.044883728027344,
      "activations/layer11_attention_weight_min": -37.59370422363281,
      "activations/layer12_attention_weight_max": 26.06206703186035,
      "activations/layer12_attention_weight_min": -26.840328216552734,
      "activations/layer13_attention_weight_max": 38.17274856567383,
      "activations/layer13_attention_weight_min": -36.39569854736328,
      "activations/layer14_attention_weight_max": 40.643653869628906,
      "activations/layer14_attention_weight_min": -39.32304000854492,
      "activations/layer15_attention_weight_max": 39.73394012451172,
      "activations/layer15_attention_weight_min": -36.06182098388672,
      "activations/layer16_attention_weight_max": 28.82141876220703,
      "activations/layer16_attention_weight_min": -27.961606979370117,
      "activations/layer17_attention_weight_max": 31.028432846069336,
      "activations/layer17_attention_weight_min": -26.413238525390625,
      "activations/layer18_attention_weight_max": 35.134090423583984,
      "activations/layer18_attention_weight_min": -25.91446876525879,
      "activations/layer19_attention_weight_max": 32.542449951171875,
      "activations/layer19_attention_weight_min": -30.495397567749023,
      "activations/layer1_attention_weight_max": 17.341245651245117,
      "activations/layer1_attention_weight_min": -14.490194320678711,
      "activations/layer20_attention_weight_max": 26.718994140625,
      "activations/layer20_attention_weight_min": -26.363407135009766,
      "activations/layer21_attention_weight_max": 27.18558692932129,
      "activations/layer21_attention_weight_min": -25.3931884765625,
      "activations/layer22_attention_weight_max": 39.63439178466797,
      "activations/layer22_attention_weight_min": -30.8241024017334,
      "activations/layer23_attention_weight_max": 31.97952651977539,
      "activations/layer23_attention_weight_min": -24.99774932861328,
      "activations/layer2_attention_weight_max": 31.14065933227539,
      "activations/layer2_attention_weight_min": -30.84964370727539,
      "activations/layer3_attention_weight_max": 99.8931884765625,
      "activations/layer3_attention_weight_min": -99.03913879394531,
      "activations/layer4_attention_weight_max": 104.6212158203125,
      "activations/layer4_attention_weight_min": -108.05670166015625,
      "activations/layer5_attention_weight_max": 48.754722595214844,
      "activations/layer5_attention_weight_min": -63.211692810058594,
      "activations/layer6_attention_weight_max": 44.724525451660156,
      "activations/layer6_attention_weight_min": -49.3896598815918,
      "activations/layer7_attention_weight_max": 97.56507873535156,
      "activations/layer7_attention_weight_min": -99.175048828125,
      "activations/layer8_attention_weight_max": 44.39589309692383,
      "activations/layer8_attention_weight_min": -46.52299880981445,
      "activations/layer9_attention_weight_max": 33.0888786315918,
      "activations/layer9_attention_weight_min": -35.85956573486328,
      "epoch": 18.49,
      "learning_rate": 3.1100378787878786e-05,
      "loss": 2.7353,
      "step": 318150
    },
    {
      "activations/layer0_attention_weight_max": 16.541942596435547,
      "activations/layer0_attention_weight_min": -12.545432090759277,
      "activations/layer10_attention_weight_max": 31.839305877685547,
      "activations/layer10_attention_weight_min": -34.16925811767578,
      "activations/layer11_attention_weight_max": 33.212738037109375,
      "activations/layer11_attention_weight_min": -32.97010040283203,
      "activations/layer12_attention_weight_max": 26.964168548583984,
      "activations/layer12_attention_weight_min": -28.01062774658203,
      "activations/layer13_attention_weight_max": 39.6212272644043,
      "activations/layer13_attention_weight_min": -36.95884323120117,
      "activations/layer14_attention_weight_max": 44.99128723144531,
      "activations/layer14_attention_weight_min": -37.46139144897461,
      "activations/layer15_attention_weight_max": 39.62099838256836,
      "activations/layer15_attention_weight_min": -36.7908935546875,
      "activations/layer16_attention_weight_max": 33.18220138549805,
      "activations/layer16_attention_weight_min": -27.46143341064453,
      "activations/layer17_attention_weight_max": 32.60078811645508,
      "activations/layer17_attention_weight_min": -25.67107391357422,
      "activations/layer18_attention_weight_max": 35.11906814575195,
      "activations/layer18_attention_weight_min": -26.720041275024414,
      "activations/layer19_attention_weight_max": 40.28483963012695,
      "activations/layer19_attention_weight_min": -33.281593322753906,
      "activations/layer1_attention_weight_max": 16.012243270874023,
      "activations/layer1_attention_weight_min": -13.322052001953125,
      "activations/layer20_attention_weight_max": 37.049198150634766,
      "activations/layer20_attention_weight_min": -27.265857696533203,
      "activations/layer21_attention_weight_max": 33.09554672241211,
      "activations/layer21_attention_weight_min": -26.68242073059082,
      "activations/layer22_attention_weight_max": 54.76077651977539,
      "activations/layer22_attention_weight_min": -35.80325698852539,
      "activations/layer23_attention_weight_max": 36.6357421875,
      "activations/layer23_attention_weight_min": -27.01563835144043,
      "activations/layer2_attention_weight_max": 28.883298873901367,
      "activations/layer2_attention_weight_min": -29.069433212280273,
      "activations/layer3_attention_weight_max": 94.8447265625,
      "activations/layer3_attention_weight_min": -95.60134887695312,
      "activations/layer4_attention_weight_max": 105.52127838134766,
      "activations/layer4_attention_weight_min": -101.75646209716797,
      "activations/layer5_attention_weight_max": 49.6372184753418,
      "activations/layer5_attention_weight_min": -61.82964324951172,
      "activations/layer6_attention_weight_max": 45.165306091308594,
      "activations/layer6_attention_weight_min": -48.63428497314453,
      "activations/layer7_attention_weight_max": 89.4914321899414,
      "activations/layer7_attention_weight_min": -97.43412017822266,
      "activations/layer8_attention_weight_max": 40.02394104003906,
      "activations/layer8_attention_weight_min": -44.2952880859375,
      "activations/layer9_attention_weight_max": 30.9232120513916,
      "activations/layer9_attention_weight_min": -34.46903610229492,
      "epoch": 18.49,
      "learning_rate": 3.108143939393939e-05,
      "loss": 2.7345,
      "step": 318200
    },
    {
      "activations/layer0_attention_weight_max": 17.300151824951172,
      "activations/layer0_attention_weight_min": -12.070432662963867,
      "activations/layer10_attention_weight_max": 33.850555419921875,
      "activations/layer10_attention_weight_min": -34.698760986328125,
      "activations/layer11_attention_weight_max": 34.071617126464844,
      "activations/layer11_attention_weight_min": -35.08671951293945,
      "activations/layer12_attention_weight_max": 26.454011917114258,
      "activations/layer12_attention_weight_min": -27.08771324157715,
      "activations/layer13_attention_weight_max": 39.74470138549805,
      "activations/layer13_attention_weight_min": -38.22117233276367,
      "activations/layer14_attention_weight_max": 44.70420455932617,
      "activations/layer14_attention_weight_min": -41.15079879760742,
      "activations/layer15_attention_weight_max": 40.109474182128906,
      "activations/layer15_attention_weight_min": -38.92466354370117,
      "activations/layer16_attention_weight_max": 28.26683235168457,
      "activations/layer16_attention_weight_min": -29.387422561645508,
      "activations/layer17_attention_weight_max": 28.592451095581055,
      "activations/layer17_attention_weight_min": -27.712438583374023,
      "activations/layer18_attention_weight_max": 31.24795150756836,
      "activations/layer18_attention_weight_min": -27.35167121887207,
      "activations/layer19_attention_weight_max": 35.16238021850586,
      "activations/layer19_attention_weight_min": -32.37928009033203,
      "activations/layer1_attention_weight_max": 16.042627334594727,
      "activations/layer1_attention_weight_min": -14.042271614074707,
      "activations/layer20_attention_weight_max": 28.896615982055664,
      "activations/layer20_attention_weight_min": -24.69036293029785,
      "activations/layer21_attention_weight_max": 30.252822875976562,
      "activations/layer21_attention_weight_min": -25.110870361328125,
      "activations/layer22_attention_weight_max": 42.12844467163086,
      "activations/layer22_attention_weight_min": -32.1578254699707,
      "activations/layer23_attention_weight_max": 32.57514953613281,
      "activations/layer23_attention_weight_min": -24.91769027709961,
      "activations/layer2_attention_weight_max": 29.97156524658203,
      "activations/layer2_attention_weight_min": -31.08112335205078,
      "activations/layer3_attention_weight_max": 96.07771301269531,
      "activations/layer3_attention_weight_min": -100.36699676513672,
      "activations/layer4_attention_weight_max": 107.39261627197266,
      "activations/layer4_attention_weight_min": -114.23799133300781,
      "activations/layer5_attention_weight_max": 56.683265686035156,
      "activations/layer5_attention_weight_min": -65.14849853515625,
      "activations/layer6_attention_weight_max": 47.29536056518555,
      "activations/layer6_attention_weight_min": -51.33418655395508,
      "activations/layer7_attention_weight_max": 94.43150329589844,
      "activations/layer7_attention_weight_min": -99.05216217041016,
      "activations/layer8_attention_weight_max": 40.63044738769531,
      "activations/layer8_attention_weight_min": -45.146705627441406,
      "activations/layer9_attention_weight_max": 32.269325256347656,
      "activations/layer9_attention_weight_min": -34.82639694213867,
      "epoch": 18.49,
      "learning_rate": 3.1062499999999996e-05,
      "loss": 2.726,
      "step": 318250
    },
    {
      "activations/layer0_attention_weight_max": 16.476896286010742,
      "activations/layer0_attention_weight_min": -13.000533103942871,
      "activations/layer10_attention_weight_max": 35.487430572509766,
      "activations/layer10_attention_weight_min": -36.59865951538086,
      "activations/layer11_attention_weight_max": 37.95305252075195,
      "activations/layer11_attention_weight_min": -37.77109909057617,
      "activations/layer12_attention_weight_max": 26.01915168762207,
      "activations/layer12_attention_weight_min": -28.0223445892334,
      "activations/layer13_attention_weight_max": 38.67000198364258,
      "activations/layer13_attention_weight_min": -36.83324432373047,
      "activations/layer14_attention_weight_max": 43.118927001953125,
      "activations/layer14_attention_weight_min": -37.80055236816406,
      "activations/layer15_attention_weight_max": 40.17247772216797,
      "activations/layer15_attention_weight_min": -37.66853332519531,
      "activations/layer16_attention_weight_max": 36.26818084716797,
      "activations/layer16_attention_weight_min": -28.271560668945312,
      "activations/layer17_attention_weight_max": 29.358530044555664,
      "activations/layer17_attention_weight_min": -26.576833724975586,
      "activations/layer18_attention_weight_max": 29.722145080566406,
      "activations/layer18_attention_weight_min": -26.08479118347168,
      "activations/layer19_attention_weight_max": 32.87277603149414,
      "activations/layer19_attention_weight_min": -29.07647705078125,
      "activations/layer1_attention_weight_max": 16.13446807861328,
      "activations/layer1_attention_weight_min": -15.150511741638184,
      "activations/layer20_attention_weight_max": 27.916702270507812,
      "activations/layer20_attention_weight_min": -24.253284454345703,
      "activations/layer21_attention_weight_max": 26.733367919921875,
      "activations/layer21_attention_weight_min": -26.404830932617188,
      "activations/layer22_attention_weight_max": 41.637184143066406,
      "activations/layer22_attention_weight_min": -31.151458740234375,
      "activations/layer23_attention_weight_max": 32.2012939453125,
      "activations/layer23_attention_weight_min": -25.814558029174805,
      "activations/layer2_attention_weight_max": 32.154151916503906,
      "activations/layer2_attention_weight_min": -30.280933380126953,
      "activations/layer3_attention_weight_max": 95.33448791503906,
      "activations/layer3_attention_weight_min": -98.41358947753906,
      "activations/layer4_attention_weight_max": 102.39849090576172,
      "activations/layer4_attention_weight_min": -106.4950180053711,
      "activations/layer5_attention_weight_max": 51.225135803222656,
      "activations/layer5_attention_weight_min": -62.898719787597656,
      "activations/layer6_attention_weight_max": 43.31420135498047,
      "activations/layer6_attention_weight_min": -45.95331573486328,
      "activations/layer7_attention_weight_max": 94.575927734375,
      "activations/layer7_attention_weight_min": -94.880126953125,
      "activations/layer8_attention_weight_max": 45.81542205810547,
      "activations/layer8_attention_weight_min": -45.68117141723633,
      "activations/layer9_attention_weight_max": 33.68705368041992,
      "activations/layer9_attention_weight_min": -33.780517578125,
      "epoch": 18.5,
      "learning_rate": 3.1043560606060604e-05,
      "loss": 2.7387,
      "step": 318300
    },
    {
      "activations/layer0_attention_weight_max": 17.216388702392578,
      "activations/layer0_attention_weight_min": -12.878679275512695,
      "activations/layer10_attention_weight_max": 40.849754333496094,
      "activations/layer10_attention_weight_min": -36.68413162231445,
      "activations/layer11_attention_weight_max": 41.42619323730469,
      "activations/layer11_attention_weight_min": -35.51905822753906,
      "activations/layer12_attention_weight_max": 28.508787155151367,
      "activations/layer12_attention_weight_min": -27.42837142944336,
      "activations/layer13_attention_weight_max": 39.840293884277344,
      "activations/layer13_attention_weight_min": -35.526832580566406,
      "activations/layer14_attention_weight_max": 45.09081268310547,
      "activations/layer14_attention_weight_min": -38.724002838134766,
      "activations/layer15_attention_weight_max": 44.41594696044922,
      "activations/layer15_attention_weight_min": -34.99148178100586,
      "activations/layer16_attention_weight_max": 30.460865020751953,
      "activations/layer16_attention_weight_min": -27.727672576904297,
      "activations/layer17_attention_weight_max": 28.843130111694336,
      "activations/layer17_attention_weight_min": -25.580856323242188,
      "activations/layer18_attention_weight_max": 32.49919891357422,
      "activations/layer18_attention_weight_min": -26.34264373779297,
      "activations/layer19_attention_weight_max": 34.516353607177734,
      "activations/layer19_attention_weight_min": -33.336448669433594,
      "activations/layer1_attention_weight_max": 17.133939743041992,
      "activations/layer1_attention_weight_min": -16.339441299438477,
      "activations/layer20_attention_weight_max": 29.08196258544922,
      "activations/layer20_attention_weight_min": -24.02994155883789,
      "activations/layer21_attention_weight_max": 28.766475677490234,
      "activations/layer21_attention_weight_min": -23.610965728759766,
      "activations/layer22_attention_weight_max": 44.50244140625,
      "activations/layer22_attention_weight_min": -33.11868667602539,
      "activations/layer23_attention_weight_max": 35.67072296142578,
      "activations/layer23_attention_weight_min": -25.066511154174805,
      "activations/layer2_attention_weight_max": 30.898483276367188,
      "activations/layer2_attention_weight_min": -31.774799346923828,
      "activations/layer3_attention_weight_max": 103.11788177490234,
      "activations/layer3_attention_weight_min": -106.51119995117188,
      "activations/layer4_attention_weight_max": 110.28035736083984,
      "activations/layer4_attention_weight_min": -115.9892578125,
      "activations/layer5_attention_weight_max": 52.52864074707031,
      "activations/layer5_attention_weight_min": -58.950984954833984,
      "activations/layer6_attention_weight_max": 46.06174087524414,
      "activations/layer6_attention_weight_min": -48.27254867553711,
      "activations/layer7_attention_weight_max": 100.30542755126953,
      "activations/layer7_attention_weight_min": -102.32897186279297,
      "activations/layer8_attention_weight_max": 48.279048919677734,
      "activations/layer8_attention_weight_min": -46.14069366455078,
      "activations/layer9_attention_weight_max": 35.3263053894043,
      "activations/layer9_attention_weight_min": -35.58591842651367,
      "epoch": 18.5,
      "learning_rate": 3.102462121212121e-05,
      "loss": 2.7313,
      "step": 318350
    },
    {
      "activations/layer0_attention_weight_max": 15.947407722473145,
      "activations/layer0_attention_weight_min": -14.17515754699707,
      "activations/layer10_attention_weight_max": 33.92324447631836,
      "activations/layer10_attention_weight_min": -36.280792236328125,
      "activations/layer11_attention_weight_max": 32.35974884033203,
      "activations/layer11_attention_weight_min": -34.418731689453125,
      "activations/layer12_attention_weight_max": 25.91282844543457,
      "activations/layer12_attention_weight_min": -27.00350570678711,
      "activations/layer13_attention_weight_max": 38.15595626831055,
      "activations/layer13_attention_weight_min": -36.36674499511719,
      "activations/layer14_attention_weight_max": 42.47126388549805,
      "activations/layer14_attention_weight_min": -38.63164138793945,
      "activations/layer15_attention_weight_max": 36.91991424560547,
      "activations/layer15_attention_weight_min": -35.502906799316406,
      "activations/layer16_attention_weight_max": 31.32208251953125,
      "activations/layer16_attention_weight_min": -27.97698211669922,
      "activations/layer17_attention_weight_max": 29.282440185546875,
      "activations/layer17_attention_weight_min": -26.2225341796875,
      "activations/layer18_attention_weight_max": 35.80895233154297,
      "activations/layer18_attention_weight_min": -28.841327667236328,
      "activations/layer19_attention_weight_max": 33.9984130859375,
      "activations/layer19_attention_weight_min": -34.737937927246094,
      "activations/layer1_attention_weight_max": 15.878705024719238,
      "activations/layer1_attention_weight_min": -14.944231986999512,
      "activations/layer20_attention_weight_max": 28.666513442993164,
      "activations/layer20_attention_weight_min": -25.701017379760742,
      "activations/layer21_attention_weight_max": 30.071195602416992,
      "activations/layer21_attention_weight_min": -25.377403259277344,
      "activations/layer22_attention_weight_max": 41.02663803100586,
      "activations/layer22_attention_weight_min": -33.787471771240234,
      "activations/layer23_attention_weight_max": 33.98810577392578,
      "activations/layer23_attention_weight_min": -28.612516403198242,
      "activations/layer2_attention_weight_max": 32.133155822753906,
      "activations/layer2_attention_weight_min": -32.153480529785156,
      "activations/layer3_attention_weight_max": 100.4020767211914,
      "activations/layer3_attention_weight_min": -96.8389892578125,
      "activations/layer4_attention_weight_max": 104.19972229003906,
      "activations/layer4_attention_weight_min": -110.81196594238281,
      "activations/layer5_attention_weight_max": 49.86137390136719,
      "activations/layer5_attention_weight_min": -63.63227844238281,
      "activations/layer6_attention_weight_max": 46.39148712158203,
      "activations/layer6_attention_weight_min": -49.07919692993164,
      "activations/layer7_attention_weight_max": 88.27184295654297,
      "activations/layer7_attention_weight_min": -102.75334167480469,
      "activations/layer8_attention_weight_max": 42.527225494384766,
      "activations/layer8_attention_weight_min": -46.61969757080078,
      "activations/layer9_attention_weight_max": 32.20042419433594,
      "activations/layer9_attention_weight_min": -34.37106704711914,
      "epoch": 18.5,
      "learning_rate": 3.1005681818181814e-05,
      "loss": 2.7179,
      "step": 318400
    },
    {
      "activations/layer0_attention_weight_max": 16.85894012451172,
      "activations/layer0_attention_weight_min": -11.693436622619629,
      "activations/layer10_attention_weight_max": 35.92206573486328,
      "activations/layer10_attention_weight_min": -36.149253845214844,
      "activations/layer11_attention_weight_max": 38.6106071472168,
      "activations/layer11_attention_weight_min": -36.144989013671875,
      "activations/layer12_attention_weight_max": 26.673751831054688,
      "activations/layer12_attention_weight_min": -25.709842681884766,
      "activations/layer13_attention_weight_max": 37.232967376708984,
      "activations/layer13_attention_weight_min": -36.589080810546875,
      "activations/layer14_attention_weight_max": 48.599979400634766,
      "activations/layer14_attention_weight_min": -38.536956787109375,
      "activations/layer15_attention_weight_max": 44.16194534301758,
      "activations/layer15_attention_weight_min": -37.24985885620117,
      "activations/layer16_attention_weight_max": 32.54480743408203,
      "activations/layer16_attention_weight_min": -28.941144943237305,
      "activations/layer17_attention_weight_max": 32.22055435180664,
      "activations/layer17_attention_weight_min": -26.011310577392578,
      "activations/layer18_attention_weight_max": 30.904611587524414,
      "activations/layer18_attention_weight_min": -26.999332427978516,
      "activations/layer19_attention_weight_max": 34.194583892822266,
      "activations/layer19_attention_weight_min": -31.055063247680664,
      "activations/layer1_attention_weight_max": 16.994558334350586,
      "activations/layer1_attention_weight_min": -14.478520393371582,
      "activations/layer20_attention_weight_max": 28.879594802856445,
      "activations/layer20_attention_weight_min": -24.62087059020996,
      "activations/layer21_attention_weight_max": 28.120309829711914,
      "activations/layer21_attention_weight_min": -24.403532028198242,
      "activations/layer22_attention_weight_max": 43.611846923828125,
      "activations/layer22_attention_weight_min": -29.300106048583984,
      "activations/layer23_attention_weight_max": 34.1676025390625,
      "activations/layer23_attention_weight_min": -23.895605087280273,
      "activations/layer2_attention_weight_max": 29.44003677368164,
      "activations/layer2_attention_weight_min": -28.01222038269043,
      "activations/layer3_attention_weight_max": 95.8218994140625,
      "activations/layer3_attention_weight_min": -99.13871002197266,
      "activations/layer4_attention_weight_max": 105.3390121459961,
      "activations/layer4_attention_weight_min": -107.31395721435547,
      "activations/layer5_attention_weight_max": 49.70201110839844,
      "activations/layer5_attention_weight_min": -65.205078125,
      "activations/layer6_attention_weight_max": 45.37218475341797,
      "activations/layer6_attention_weight_min": -47.15768814086914,
      "activations/layer7_attention_weight_max": 104.57392883300781,
      "activations/layer7_attention_weight_min": -98.8910140991211,
      "activations/layer8_attention_weight_max": 45.562984466552734,
      "activations/layer8_attention_weight_min": -44.5258903503418,
      "activations/layer9_attention_weight_max": 34.450565338134766,
      "activations/layer9_attention_weight_min": -37.354427337646484,
      "epoch": 18.5,
      "learning_rate": 3.098674242424242e-05,
      "loss": 2.7258,
      "step": 318450
    },
    {
      "activations/layer0_attention_weight_max": 16.779062271118164,
      "activations/layer0_attention_weight_min": -12.822014808654785,
      "activations/layer10_attention_weight_max": 35.01516342163086,
      "activations/layer10_attention_weight_min": -36.64238357543945,
      "activations/layer11_attention_weight_max": 33.93035125732422,
      "activations/layer11_attention_weight_min": -35.53828430175781,
      "activations/layer12_attention_weight_max": 28.647470474243164,
      "activations/layer12_attention_weight_min": -26.59866714477539,
      "activations/layer13_attention_weight_max": 39.70128631591797,
      "activations/layer13_attention_weight_min": -35.90947341918945,
      "activations/layer14_attention_weight_max": 42.29229736328125,
      "activations/layer14_attention_weight_min": -37.181827545166016,
      "activations/layer15_attention_weight_max": 37.36654281616211,
      "activations/layer15_attention_weight_min": -34.73625946044922,
      "activations/layer16_attention_weight_max": 30.53092384338379,
      "activations/layer16_attention_weight_min": -28.1090087890625,
      "activations/layer17_attention_weight_max": 29.129804611206055,
      "activations/layer17_attention_weight_min": -26.718303680419922,
      "activations/layer18_attention_weight_max": 32.12057876586914,
      "activations/layer18_attention_weight_min": -25.594377517700195,
      "activations/layer19_attention_weight_max": 35.96295928955078,
      "activations/layer19_attention_weight_min": -34.19021224975586,
      "activations/layer1_attention_weight_max": 16.893386840820312,
      "activations/layer1_attention_weight_min": -15.87830638885498,
      "activations/layer20_attention_weight_max": 28.498062133789062,
      "activations/layer20_attention_weight_min": -28.224613189697266,
      "activations/layer21_attention_weight_max": 26.8569393157959,
      "activations/layer21_attention_weight_min": -25.106456756591797,
      "activations/layer22_attention_weight_max": 41.61054992675781,
      "activations/layer22_attention_weight_min": -34.64370346069336,
      "activations/layer23_attention_weight_max": 36.95984649658203,
      "activations/layer23_attention_weight_min": -27.495206832885742,
      "activations/layer2_attention_weight_max": 29.750385284423828,
      "activations/layer2_attention_weight_min": -29.24770736694336,
      "activations/layer3_attention_weight_max": 98.13629150390625,
      "activations/layer3_attention_weight_min": -101.9926986694336,
      "activations/layer4_attention_weight_max": 107.47930145263672,
      "activations/layer4_attention_weight_min": -111.65486907958984,
      "activations/layer5_attention_weight_max": 52.29247283935547,
      "activations/layer5_attention_weight_min": -62.0221061706543,
      "activations/layer6_attention_weight_max": 45.77799606323242,
      "activations/layer6_attention_weight_min": -46.42912673950195,
      "activations/layer7_attention_weight_max": 92.24020385742188,
      "activations/layer7_attention_weight_min": -95.24579620361328,
      "activations/layer8_attention_weight_max": 40.05801010131836,
      "activations/layer8_attention_weight_min": -44.418453216552734,
      "activations/layer9_attention_weight_max": 32.74451446533203,
      "activations/layer9_attention_weight_min": -35.58565139770508,
      "epoch": 18.51,
      "learning_rate": 3.0967803030303024e-05,
      "loss": 2.7468,
      "step": 318500
    },
    {
      "activations/layer0_attention_weight_max": 16.786603927612305,
      "activations/layer0_attention_weight_min": -12.854400634765625,
      "activations/layer10_attention_weight_max": 36.30889892578125,
      "activations/layer10_attention_weight_min": -33.02266311645508,
      "activations/layer11_attention_weight_max": 36.48235321044922,
      "activations/layer11_attention_weight_min": -34.39213562011719,
      "activations/layer12_attention_weight_max": 26.8671817779541,
      "activations/layer12_attention_weight_min": -26.18808364868164,
      "activations/layer13_attention_weight_max": 38.419761657714844,
      "activations/layer13_attention_weight_min": -34.3597297668457,
      "activations/layer14_attention_weight_max": 42.481510162353516,
      "activations/layer14_attention_weight_min": -37.466644287109375,
      "activations/layer15_attention_weight_max": 38.79696273803711,
      "activations/layer15_attention_weight_min": -34.83626937866211,
      "activations/layer16_attention_weight_max": 28.975128173828125,
      "activations/layer16_attention_weight_min": -29.413009643554688,
      "activations/layer17_attention_weight_max": 31.08746337890625,
      "activations/layer17_attention_weight_min": -28.1934814453125,
      "activations/layer18_attention_weight_max": 31.537696838378906,
      "activations/layer18_attention_weight_min": -27.96337890625,
      "activations/layer19_attention_weight_max": 34.338619232177734,
      "activations/layer19_attention_weight_min": -33.368343353271484,
      "activations/layer1_attention_weight_max": 17.64366912841797,
      "activations/layer1_attention_weight_min": -15.764732360839844,
      "activations/layer20_attention_weight_max": 30.382585525512695,
      "activations/layer20_attention_weight_min": -26.9738712310791,
      "activations/layer21_attention_weight_max": 28.972789764404297,
      "activations/layer21_attention_weight_min": -26.081016540527344,
      "activations/layer22_attention_weight_max": 50.17159652709961,
      "activations/layer22_attention_weight_min": -32.493141174316406,
      "activations/layer23_attention_weight_max": 31.57107162475586,
      "activations/layer23_attention_weight_min": -28.525768280029297,
      "activations/layer2_attention_weight_max": 34.39011764526367,
      "activations/layer2_attention_weight_min": -35.58403778076172,
      "activations/layer3_attention_weight_max": 94.87413787841797,
      "activations/layer3_attention_weight_min": -98.96611785888672,
      "activations/layer4_attention_weight_max": 105.66228485107422,
      "activations/layer4_attention_weight_min": -99.14486694335938,
      "activations/layer5_attention_weight_max": 50.968990325927734,
      "activations/layer5_attention_weight_min": -62.265560150146484,
      "activations/layer6_attention_weight_max": 45.15824890136719,
      "activations/layer6_attention_weight_min": -44.78971481323242,
      "activations/layer7_attention_weight_max": 97.6904525756836,
      "activations/layer7_attention_weight_min": -91.88636779785156,
      "activations/layer8_attention_weight_max": 43.90949630737305,
      "activations/layer8_attention_weight_min": -43.746604919433594,
      "activations/layer9_attention_weight_max": 31.523569107055664,
      "activations/layer9_attention_weight_min": -34.15431213378906,
      "epoch": 18.51,
      "learning_rate": 3.094886363636363e-05,
      "loss": 2.7299,
      "step": 318550
    },
    {
      "activations/layer0_attention_weight_max": 16.565258026123047,
      "activations/layer0_attention_weight_min": -11.91612720489502,
      "activations/layer10_attention_weight_max": 33.78864669799805,
      "activations/layer10_attention_weight_min": -37.42454528808594,
      "activations/layer11_attention_weight_max": 35.41699981689453,
      "activations/layer11_attention_weight_min": -35.80181121826172,
      "activations/layer12_attention_weight_max": 27.62900161743164,
      "activations/layer12_attention_weight_min": -27.473922729492188,
      "activations/layer13_attention_weight_max": 41.78962707519531,
      "activations/layer13_attention_weight_min": -34.65525436401367,
      "activations/layer14_attention_weight_max": 39.99405288696289,
      "activations/layer14_attention_weight_min": -36.386985778808594,
      "activations/layer15_attention_weight_max": 35.09782791137695,
      "activations/layer15_attention_weight_min": -36.22406005859375,
      "activations/layer16_attention_weight_max": 29.901103973388672,
      "activations/layer16_attention_weight_min": -28.70328712463379,
      "activations/layer17_attention_weight_max": 29.27699851989746,
      "activations/layer17_attention_weight_min": -29.608436584472656,
      "activations/layer18_attention_weight_max": 31.36947250366211,
      "activations/layer18_attention_weight_min": -28.385459899902344,
      "activations/layer19_attention_weight_max": 34.38507843017578,
      "activations/layer19_attention_weight_min": -33.73340606689453,
      "activations/layer1_attention_weight_max": 16.031082153320312,
      "activations/layer1_attention_weight_min": -15.112260818481445,
      "activations/layer20_attention_weight_max": 27.893049240112305,
      "activations/layer20_attention_weight_min": -28.722137451171875,
      "activations/layer21_attention_weight_max": 27.385177612304688,
      "activations/layer21_attention_weight_min": -28.667593002319336,
      "activations/layer22_attention_weight_max": 43.83890151977539,
      "activations/layer22_attention_weight_min": -35.680389404296875,
      "activations/layer23_attention_weight_max": 30.51021957397461,
      "activations/layer23_attention_weight_min": -28.232845306396484,
      "activations/layer2_attention_weight_max": 31.193212509155273,
      "activations/layer2_attention_weight_min": -30.79739761352539,
      "activations/layer3_attention_weight_max": 97.53584289550781,
      "activations/layer3_attention_weight_min": -98.56246948242188,
      "activations/layer4_attention_weight_max": 104.65926361083984,
      "activations/layer4_attention_weight_min": -111.85665130615234,
      "activations/layer5_attention_weight_max": 53.89691162109375,
      "activations/layer5_attention_weight_min": -61.68095016479492,
      "activations/layer6_attention_weight_max": 43.88166809082031,
      "activations/layer6_attention_weight_min": -47.12564468383789,
      "activations/layer7_attention_weight_max": 92.87085723876953,
      "activations/layer7_attention_weight_min": -99.4791259765625,
      "activations/layer8_attention_weight_max": 43.28554153442383,
      "activations/layer8_attention_weight_min": -47.755584716796875,
      "activations/layer9_attention_weight_max": 31.9677791595459,
      "activations/layer9_attention_weight_min": -36.63874816894531,
      "epoch": 18.51,
      "learning_rate": 3.092992424242424e-05,
      "loss": 2.746,
      "step": 318600
    },
    {
      "activations/layer0_attention_weight_max": 16.170860290527344,
      "activations/layer0_attention_weight_min": -11.924406051635742,
      "activations/layer10_attention_weight_max": 33.8832893371582,
      "activations/layer10_attention_weight_min": -37.61943817138672,
      "activations/layer11_attention_weight_max": 35.56781768798828,
      "activations/layer11_attention_weight_min": -40.196224212646484,
      "activations/layer12_attention_weight_max": 26.799781799316406,
      "activations/layer12_attention_weight_min": -29.535770416259766,
      "activations/layer13_attention_weight_max": 38.17946243286133,
      "activations/layer13_attention_weight_min": -37.23060607910156,
      "activations/layer14_attention_weight_max": 41.68962860107422,
      "activations/layer14_attention_weight_min": -38.5318489074707,
      "activations/layer15_attention_weight_max": 43.97343063354492,
      "activations/layer15_attention_weight_min": -36.70234680175781,
      "activations/layer16_attention_weight_max": 34.95526885986328,
      "activations/layer16_attention_weight_min": -28.619062423706055,
      "activations/layer17_attention_weight_max": 33.59929656982422,
      "activations/layer17_attention_weight_min": -28.03527069091797,
      "activations/layer18_attention_weight_max": 31.262779235839844,
      "activations/layer18_attention_weight_min": -24.691137313842773,
      "activations/layer19_attention_weight_max": 33.77295684814453,
      "activations/layer19_attention_weight_min": -31.916202545166016,
      "activations/layer1_attention_weight_max": 16.953266143798828,
      "activations/layer1_attention_weight_min": -13.905733108520508,
      "activations/layer20_attention_weight_max": 29.55804443359375,
      "activations/layer20_attention_weight_min": -26.35176658630371,
      "activations/layer21_attention_weight_max": 27.80051612854004,
      "activations/layer21_attention_weight_min": -24.714244842529297,
      "activations/layer22_attention_weight_max": 39.97439956665039,
      "activations/layer22_attention_weight_min": -30.454322814941406,
      "activations/layer23_attention_weight_max": 33.76565170288086,
      "activations/layer23_attention_weight_min": -23.94539451599121,
      "activations/layer2_attention_weight_max": 29.470603942871094,
      "activations/layer2_attention_weight_min": -30.215259552001953,
      "activations/layer3_attention_weight_max": 94.27928161621094,
      "activations/layer3_attention_weight_min": -98.15803527832031,
      "activations/layer4_attention_weight_max": 106.24769592285156,
      "activations/layer4_attention_weight_min": -112.93924713134766,
      "activations/layer5_attention_weight_max": 49.93547058105469,
      "activations/layer5_attention_weight_min": -66.68543243408203,
      "activations/layer6_attention_weight_max": 43.793392181396484,
      "activations/layer6_attention_weight_min": -49.77536392211914,
      "activations/layer7_attention_weight_max": 96.54560089111328,
      "activations/layer7_attention_weight_min": -98.5301513671875,
      "activations/layer8_attention_weight_max": 40.93600845336914,
      "activations/layer8_attention_weight_min": -44.36476516723633,
      "activations/layer9_attention_weight_max": 32.88125228881836,
      "activations/layer9_attention_weight_min": -36.87917709350586,
      "epoch": 18.52,
      "learning_rate": 3.091098484848485e-05,
      "loss": 2.7265,
      "step": 318650
    },
    {
      "activations/layer0_attention_weight_max": 16.015100479125977,
      "activations/layer0_attention_weight_min": -13.589179992675781,
      "activations/layer10_attention_weight_max": 33.622196197509766,
      "activations/layer10_attention_weight_min": -34.421180725097656,
      "activations/layer11_attention_weight_max": 35.99687194824219,
      "activations/layer11_attention_weight_min": -33.474403381347656,
      "activations/layer12_attention_weight_max": 24.80607795715332,
      "activations/layer12_attention_weight_min": -26.45989418029785,
      "activations/layer13_attention_weight_max": 35.53593063354492,
      "activations/layer13_attention_weight_min": -34.66099166870117,
      "activations/layer14_attention_weight_max": 38.10612869262695,
      "activations/layer14_attention_weight_min": -37.952232360839844,
      "activations/layer15_attention_weight_max": 38.24803161621094,
      "activations/layer15_attention_weight_min": -34.495174407958984,
      "activations/layer16_attention_weight_max": 32.1107292175293,
      "activations/layer16_attention_weight_min": -29.534242630004883,
      "activations/layer17_attention_weight_max": 29.497066497802734,
      "activations/layer17_attention_weight_min": -25.445419311523438,
      "activations/layer18_attention_weight_max": 29.870393753051758,
      "activations/layer18_attention_weight_min": -26.68745231628418,
      "activations/layer19_attention_weight_max": 38.61304473876953,
      "activations/layer19_attention_weight_min": -30.689193725585938,
      "activations/layer1_attention_weight_max": 16.303436279296875,
      "activations/layer1_attention_weight_min": -14.094364166259766,
      "activations/layer20_attention_weight_max": 29.75063705444336,
      "activations/layer20_attention_weight_min": -25.930065155029297,
      "activations/layer21_attention_weight_max": 28.83659553527832,
      "activations/layer21_attention_weight_min": -25.151443481445312,
      "activations/layer22_attention_weight_max": 41.003543853759766,
      "activations/layer22_attention_weight_min": -30.625513076782227,
      "activations/layer23_attention_weight_max": 36.87519454956055,
      "activations/layer23_attention_weight_min": -25.579524993896484,
      "activations/layer2_attention_weight_max": 30.47334861755371,
      "activations/layer2_attention_weight_min": -28.646427154541016,
      "activations/layer3_attention_weight_max": 94.47925567626953,
      "activations/layer3_attention_weight_min": -93.70500946044922,
      "activations/layer4_attention_weight_max": 106.13787841796875,
      "activations/layer4_attention_weight_min": -106.91609191894531,
      "activations/layer5_attention_weight_max": 49.53636932373047,
      "activations/layer5_attention_weight_min": -63.753265380859375,
      "activations/layer6_attention_weight_max": 43.916053771972656,
      "activations/layer6_attention_weight_min": -45.55947494506836,
      "activations/layer7_attention_weight_max": 89.90951538085938,
      "activations/layer7_attention_weight_min": -92.47689819335938,
      "activations/layer8_attention_weight_max": 39.5633430480957,
      "activations/layer8_attention_weight_min": -44.87222671508789,
      "activations/layer9_attention_weight_max": 30.92923927307129,
      "activations/layer9_attention_weight_min": -34.3044319152832,
      "epoch": 18.52,
      "learning_rate": 3.089204545454545e-05,
      "loss": 2.7286,
      "step": 318700
    },
    {
      "activations/layer0_attention_weight_max": 16.936948776245117,
      "activations/layer0_attention_weight_min": -11.982217788696289,
      "activations/layer10_attention_weight_max": 35.94422912597656,
      "activations/layer10_attention_weight_min": -35.23328399658203,
      "activations/layer11_attention_weight_max": 34.744590759277344,
      "activations/layer11_attention_weight_min": -34.59379577636719,
      "activations/layer12_attention_weight_max": 30.1397762298584,
      "activations/layer12_attention_weight_min": -27.2899227142334,
      "activations/layer13_attention_weight_max": 44.60048294067383,
      "activations/layer13_attention_weight_min": -37.426170349121094,
      "activations/layer14_attention_weight_max": 46.31239700317383,
      "activations/layer14_attention_weight_min": -39.82384490966797,
      "activations/layer15_attention_weight_max": 40.669281005859375,
      "activations/layer15_attention_weight_min": -38.52060317993164,
      "activations/layer16_attention_weight_max": 32.59709930419922,
      "activations/layer16_attention_weight_min": -28.99190330505371,
      "activations/layer17_attention_weight_max": 30.37299919128418,
      "activations/layer17_attention_weight_min": -26.8084659576416,
      "activations/layer18_attention_weight_max": 33.286739349365234,
      "activations/layer18_attention_weight_min": -24.808923721313477,
      "activations/layer19_attention_weight_max": 35.269020080566406,
      "activations/layer19_attention_weight_min": -30.78956413269043,
      "activations/layer1_attention_weight_max": 15.41952133178711,
      "activations/layer1_attention_weight_min": -13.065695762634277,
      "activations/layer20_attention_weight_max": 28.66704559326172,
      "activations/layer20_attention_weight_min": -25.10183334350586,
      "activations/layer21_attention_weight_max": 28.869382858276367,
      "activations/layer21_attention_weight_min": -25.14604949951172,
      "activations/layer22_attention_weight_max": 39.25758361816406,
      "activations/layer22_attention_weight_min": -31.05328369140625,
      "activations/layer23_attention_weight_max": 35.56636047363281,
      "activations/layer23_attention_weight_min": -25.37373924255371,
      "activations/layer2_attention_weight_max": 29.69158935546875,
      "activations/layer2_attention_weight_min": -30.299230575561523,
      "activations/layer3_attention_weight_max": 97.13981628417969,
      "activations/layer3_attention_weight_min": -98.16129302978516,
      "activations/layer4_attention_weight_max": 106.62389373779297,
      "activations/layer4_attention_weight_min": -109.79963684082031,
      "activations/layer5_attention_weight_max": 54.41231918334961,
      "activations/layer5_attention_weight_min": -64.21200561523438,
      "activations/layer6_attention_weight_max": 43.91016387939453,
      "activations/layer6_attention_weight_min": -47.2553596496582,
      "activations/layer7_attention_weight_max": 91.94775390625,
      "activations/layer7_attention_weight_min": -93.46751403808594,
      "activations/layer8_attention_weight_max": 40.29695510864258,
      "activations/layer8_attention_weight_min": -43.17667770385742,
      "activations/layer9_attention_weight_max": 33.32993698120117,
      "activations/layer9_attention_weight_min": -33.97151184082031,
      "epoch": 18.52,
      "learning_rate": 3.087310606060606e-05,
      "loss": 2.7312,
      "step": 318750
    },
    {
      "activations/layer0_attention_weight_max": 15.896446228027344,
      "activations/layer0_attention_weight_min": -12.065074920654297,
      "activations/layer10_attention_weight_max": 38.22809982299805,
      "activations/layer10_attention_weight_min": -36.702056884765625,
      "activations/layer11_attention_weight_max": 38.79197692871094,
      "activations/layer11_attention_weight_min": -37.538848876953125,
      "activations/layer12_attention_weight_max": 29.15604019165039,
      "activations/layer12_attention_weight_min": -29.16950798034668,
      "activations/layer13_attention_weight_max": 42.870887756347656,
      "activations/layer13_attention_weight_min": -39.76035690307617,
      "activations/layer14_attention_weight_max": 42.71926498413086,
      "activations/layer14_attention_weight_min": -41.36397171020508,
      "activations/layer15_attention_weight_max": 40.556419372558594,
      "activations/layer15_attention_weight_min": -39.91462707519531,
      "activations/layer16_attention_weight_max": 29.484846115112305,
      "activations/layer16_attention_weight_min": -28.004350662231445,
      "activations/layer17_attention_weight_max": 29.444091796875,
      "activations/layer17_attention_weight_min": -28.32882308959961,
      "activations/layer18_attention_weight_max": 29.286935806274414,
      "activations/layer18_attention_weight_min": -26.142684936523438,
      "activations/layer19_attention_weight_max": 30.541147232055664,
      "activations/layer19_attention_weight_min": -33.30400466918945,
      "activations/layer1_attention_weight_max": 16.871183395385742,
      "activations/layer1_attention_weight_min": -14.090890884399414,
      "activations/layer20_attention_weight_max": 27.579145431518555,
      "activations/layer20_attention_weight_min": -27.751585006713867,
      "activations/layer21_attention_weight_max": 26.275278091430664,
      "activations/layer21_attention_weight_min": -25.932010650634766,
      "activations/layer22_attention_weight_max": 41.79102325439453,
      "activations/layer22_attention_weight_min": -33.94154357910156,
      "activations/layer23_attention_weight_max": 29.874174118041992,
      "activations/layer23_attention_weight_min": -29.140769958496094,
      "activations/layer2_attention_weight_max": 30.7482967376709,
      "activations/layer2_attention_weight_min": -29.94076919555664,
      "activations/layer3_attention_weight_max": 95.25498962402344,
      "activations/layer3_attention_weight_min": -95.68991088867188,
      "activations/layer4_attention_weight_max": 106.77059173583984,
      "activations/layer4_attention_weight_min": -105.89238739013672,
      "activations/layer5_attention_weight_max": 50.39336395263672,
      "activations/layer5_attention_weight_min": -60.102088928222656,
      "activations/layer6_attention_weight_max": 46.26259994506836,
      "activations/layer6_attention_weight_min": -48.319068908691406,
      "activations/layer7_attention_weight_max": 98.06681823730469,
      "activations/layer7_attention_weight_min": -104.68402099609375,
      "activations/layer8_attention_weight_max": 43.51556396484375,
      "activations/layer8_attention_weight_min": -48.792484283447266,
      "activations/layer9_attention_weight_max": 34.64320755004883,
      "activations/layer9_attention_weight_min": -37.447654724121094,
      "epoch": 18.52,
      "learning_rate": 3.085416666666667e-05,
      "loss": 2.7363,
      "step": 318800
    },
    {
      "activations/layer0_attention_weight_max": 16.090574264526367,
      "activations/layer0_attention_weight_min": -13.443076133728027,
      "activations/layer10_attention_weight_max": 36.178016662597656,
      "activations/layer10_attention_weight_min": -34.30195617675781,
      "activations/layer11_attention_weight_max": 35.75908660888672,
      "activations/layer11_attention_weight_min": -35.48457336425781,
      "activations/layer12_attention_weight_max": 31.495925903320312,
      "activations/layer12_attention_weight_min": -28.938425064086914,
      "activations/layer13_attention_weight_max": 46.97056198120117,
      "activations/layer13_attention_weight_min": -39.997493743896484,
      "activations/layer14_attention_weight_max": 51.226646423339844,
      "activations/layer14_attention_weight_min": -41.60556411743164,
      "activations/layer15_attention_weight_max": 49.81393051147461,
      "activations/layer15_attention_weight_min": -41.019386291503906,
      "activations/layer16_attention_weight_max": 34.13186264038086,
      "activations/layer16_attention_weight_min": -30.08798599243164,
      "activations/layer17_attention_weight_max": 33.7978515625,
      "activations/layer17_attention_weight_min": -27.739723205566406,
      "activations/layer18_attention_weight_max": 33.753299713134766,
      "activations/layer18_attention_weight_min": -26.03252601623535,
      "activations/layer19_attention_weight_max": 38.79493713378906,
      "activations/layer19_attention_weight_min": -34.37800216674805,
      "activations/layer1_attention_weight_max": 16.29269790649414,
      "activations/layer1_attention_weight_min": -14.066601753234863,
      "activations/layer20_attention_weight_max": 32.140995025634766,
      "activations/layer20_attention_weight_min": -27.228858947753906,
      "activations/layer21_attention_weight_max": 31.353158950805664,
      "activations/layer21_attention_weight_min": -25.781259536743164,
      "activations/layer22_attention_weight_max": 41.56089401245117,
      "activations/layer22_attention_weight_min": -34.431392669677734,
      "activations/layer23_attention_weight_max": 35.196510314941406,
      "activations/layer23_attention_weight_min": -26.04456901550293,
      "activations/layer2_attention_weight_max": 29.559795379638672,
      "activations/layer2_attention_weight_min": -29.902942657470703,
      "activations/layer3_attention_weight_max": 94.34996795654297,
      "activations/layer3_attention_weight_min": -96.88976287841797,
      "activations/layer4_attention_weight_max": 105.80785369873047,
      "activations/layer4_attention_weight_min": -108.37812805175781,
      "activations/layer5_attention_weight_max": 51.58774948120117,
      "activations/layer5_attention_weight_min": -62.31020736694336,
      "activations/layer6_attention_weight_max": 45.07480239868164,
      "activations/layer6_attention_weight_min": -47.37167739868164,
      "activations/layer7_attention_weight_max": 95.59807586669922,
      "activations/layer7_attention_weight_min": -97.32501983642578,
      "activations/layer8_attention_weight_max": 43.76912307739258,
      "activations/layer8_attention_weight_min": -44.35157012939453,
      "activations/layer9_attention_weight_max": 34.2421989440918,
      "activations/layer9_attention_weight_min": -35.852474212646484,
      "epoch": 18.53,
      "learning_rate": 3.083522727272727e-05,
      "loss": 2.7211,
      "step": 318850
    },
    {
      "activations/layer0_attention_weight_max": 16.163909912109375,
      "activations/layer0_attention_weight_min": -12.490585327148438,
      "activations/layer10_attention_weight_max": 37.1513671875,
      "activations/layer10_attention_weight_min": -33.57506561279297,
      "activations/layer11_attention_weight_max": 35.898094177246094,
      "activations/layer11_attention_weight_min": -36.150146484375,
      "activations/layer12_attention_weight_max": 25.0429744720459,
      "activations/layer12_attention_weight_min": -25.752079010009766,
      "activations/layer13_attention_weight_max": 36.03424835205078,
      "activations/layer13_attention_weight_min": -34.348995208740234,
      "activations/layer14_attention_weight_max": 39.03481674194336,
      "activations/layer14_attention_weight_min": -35.59016418457031,
      "activations/layer15_attention_weight_max": 35.740753173828125,
      "activations/layer15_attention_weight_min": -35.72068405151367,
      "activations/layer16_attention_weight_max": 26.992639541625977,
      "activations/layer16_attention_weight_min": -27.768686294555664,
      "activations/layer17_attention_weight_max": 28.13275718688965,
      "activations/layer17_attention_weight_min": -24.79509925842285,
      "activations/layer18_attention_weight_max": 30.5074462890625,
      "activations/layer18_attention_weight_min": -26.201295852661133,
      "activations/layer19_attention_weight_max": 30.483957290649414,
      "activations/layer19_attention_weight_min": -29.8496036529541,
      "activations/layer1_attention_weight_max": 15.764993667602539,
      "activations/layer1_attention_weight_min": -13.883955955505371,
      "activations/layer20_attention_weight_max": 27.639041900634766,
      "activations/layer20_attention_weight_min": -23.988767623901367,
      "activations/layer21_attention_weight_max": 27.439550399780273,
      "activations/layer21_attention_weight_min": -24.102243423461914,
      "activations/layer22_attention_weight_max": 41.53223419189453,
      "activations/layer22_attention_weight_min": -29.404205322265625,
      "activations/layer23_attention_weight_max": 30.946815490722656,
      "activations/layer23_attention_weight_min": -24.129560470581055,
      "activations/layer2_attention_weight_max": 30.26517105102539,
      "activations/layer2_attention_weight_min": -31.834033966064453,
      "activations/layer3_attention_weight_max": 100.70536041259766,
      "activations/layer3_attention_weight_min": -100.98445892333984,
      "activations/layer4_attention_weight_max": 109.4045639038086,
      "activations/layer4_attention_weight_min": -106.0710678100586,
      "activations/layer5_attention_weight_max": 51.507720947265625,
      "activations/layer5_attention_weight_min": -61.24947738647461,
      "activations/layer6_attention_weight_max": 44.90219497680664,
      "activations/layer6_attention_weight_min": -48.87991714477539,
      "activations/layer7_attention_weight_max": 92.32767486572266,
      "activations/layer7_attention_weight_min": -104.72859191894531,
      "activations/layer8_attention_weight_max": 41.01444625854492,
      "activations/layer8_attention_weight_min": -42.9096565246582,
      "activations/layer9_attention_weight_max": 30.640369415283203,
      "activations/layer9_attention_weight_min": -34.12098693847656,
      "epoch": 18.53,
      "learning_rate": 3.0816287878787876e-05,
      "loss": 2.7473,
      "step": 318900
    },
    {
      "activations/layer0_attention_weight_max": 15.960060119628906,
      "activations/layer0_attention_weight_min": -12.545023918151855,
      "activations/layer10_attention_weight_max": 34.604618072509766,
      "activations/layer10_attention_weight_min": -36.67366409301758,
      "activations/layer11_attention_weight_max": 35.55246353149414,
      "activations/layer11_attention_weight_min": -36.095428466796875,
      "activations/layer12_attention_weight_max": 25.50080108642578,
      "activations/layer12_attention_weight_min": -25.51926612854004,
      "activations/layer13_attention_weight_max": 38.99663162231445,
      "activations/layer13_attention_weight_min": -34.43784713745117,
      "activations/layer14_attention_weight_max": 41.41543197631836,
      "activations/layer14_attention_weight_min": -41.35921096801758,
      "activations/layer15_attention_weight_max": 39.983917236328125,
      "activations/layer15_attention_weight_min": -37.7465934753418,
      "activations/layer16_attention_weight_max": 29.692821502685547,
      "activations/layer16_attention_weight_min": -27.169551849365234,
      "activations/layer17_attention_weight_max": 29.322452545166016,
      "activations/layer17_attention_weight_min": -26.72332191467285,
      "activations/layer18_attention_weight_max": 28.169103622436523,
      "activations/layer18_attention_weight_min": -27.657394409179688,
      "activations/layer19_attention_weight_max": 35.3236198425293,
      "activations/layer19_attention_weight_min": -31.99198341369629,
      "activations/layer1_attention_weight_max": 16.35256576538086,
      "activations/layer1_attention_weight_min": -13.53102970123291,
      "activations/layer20_attention_weight_max": 30.028154373168945,
      "activations/layer20_attention_weight_min": -25.989534378051758,
      "activations/layer21_attention_weight_max": 29.395156860351562,
      "activations/layer21_attention_weight_min": -26.619365692138672,
      "activations/layer22_attention_weight_max": 40.64421463012695,
      "activations/layer22_attention_weight_min": -34.225975036621094,
      "activations/layer23_attention_weight_max": 34.7257194519043,
      "activations/layer23_attention_weight_min": -27.112991333007812,
      "activations/layer2_attention_weight_max": 29.900503158569336,
      "activations/layer2_attention_weight_min": -29.724281311035156,
      "activations/layer3_attention_weight_max": 98.3248519897461,
      "activations/layer3_attention_weight_min": -102.01130676269531,
      "activations/layer4_attention_weight_max": 105.40107727050781,
      "activations/layer4_attention_weight_min": -110.08260345458984,
      "activations/layer5_attention_weight_max": 48.675537109375,
      "activations/layer5_attention_weight_min": -66.97917175292969,
      "activations/layer6_attention_weight_max": 43.71784210205078,
      "activations/layer6_attention_weight_min": -45.77698516845703,
      "activations/layer7_attention_weight_max": 99.78707122802734,
      "activations/layer7_attention_weight_min": -97.55619049072266,
      "activations/layer8_attention_weight_max": 41.74361038208008,
      "activations/layer8_attention_weight_min": -44.84494400024414,
      "activations/layer9_attention_weight_max": 33.167213439941406,
      "activations/layer9_attention_weight_min": -34.46601104736328,
      "epoch": 18.53,
      "learning_rate": 3.0797348484848485e-05,
      "loss": 2.7383,
      "step": 318950
    },
    {
      "activations/layer0_attention_weight_max": 16.469886779785156,
      "activations/layer0_attention_weight_min": -12.226350784301758,
      "activations/layer10_attention_weight_max": 33.38254928588867,
      "activations/layer10_attention_weight_min": -34.47374725341797,
      "activations/layer11_attention_weight_max": 32.79420471191406,
      "activations/layer11_attention_weight_min": -32.674476623535156,
      "activations/layer12_attention_weight_max": 24.666702270507812,
      "activations/layer12_attention_weight_min": -26.523841857910156,
      "activations/layer13_attention_weight_max": 36.71559524536133,
      "activations/layer13_attention_weight_min": -35.77558135986328,
      "activations/layer14_attention_weight_max": 37.82870864868164,
      "activations/layer14_attention_weight_min": -39.03995132446289,
      "activations/layer15_attention_weight_max": 34.259132385253906,
      "activations/layer15_attention_weight_min": -36.074214935302734,
      "activations/layer16_attention_weight_max": 27.156896591186523,
      "activations/layer16_attention_weight_min": -29.781497955322266,
      "activations/layer17_attention_weight_max": 26.46783447265625,
      "activations/layer17_attention_weight_min": -27.02479362487793,
      "activations/layer18_attention_weight_max": 30.777286529541016,
      "activations/layer18_attention_weight_min": -26.5161190032959,
      "activations/layer19_attention_weight_max": 29.90458106994629,
      "activations/layer19_attention_weight_min": -33.48447799682617,
      "activations/layer1_attention_weight_max": 16.260263442993164,
      "activations/layer1_attention_weight_min": -13.849160194396973,
      "activations/layer20_attention_weight_max": 27.139745712280273,
      "activations/layer20_attention_weight_min": -26.781206130981445,
      "activations/layer21_attention_weight_max": 26.39967918395996,
      "activations/layer21_attention_weight_min": -26.1457576751709,
      "activations/layer22_attention_weight_max": 36.92387771606445,
      "activations/layer22_attention_weight_min": -31.69791030883789,
      "activations/layer23_attention_weight_max": 30.851486206054688,
      "activations/layer23_attention_weight_min": -26.55145263671875,
      "activations/layer2_attention_weight_max": 29.99358367919922,
      "activations/layer2_attention_weight_min": -28.94615936279297,
      "activations/layer3_attention_weight_max": 95.7423095703125,
      "activations/layer3_attention_weight_min": -95.29740905761719,
      "activations/layer4_attention_weight_max": 105.04682922363281,
      "activations/layer4_attention_weight_min": -101.96997833251953,
      "activations/layer5_attention_weight_max": 51.37739181518555,
      "activations/layer5_attention_weight_min": -64.44327545166016,
      "activations/layer6_attention_weight_max": 42.85612869262695,
      "activations/layer6_attention_weight_min": -45.92854309082031,
      "activations/layer7_attention_weight_max": 90.56021881103516,
      "activations/layer7_attention_weight_min": -91.34640502929688,
      "activations/layer8_attention_weight_max": 40.879722595214844,
      "activations/layer8_attention_weight_min": -43.93803787231445,
      "activations/layer9_attention_weight_max": 32.724117279052734,
      "activations/layer9_attention_weight_min": -33.32366180419922,
      "epoch": 18.54,
      "learning_rate": 3.077840909090909e-05,
      "loss": 2.7291,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.4883,
      "eval_samples_per_second": 505.873,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.4883,
      "eval_openwebtext_samples_per_second": 505.873,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9834,
      "eval_wikitext_samples_per_second": 229.911,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_lambada_loss": 2.369140625,
      "eval_lambada_ppl": 10.688203161633492,
      "eval_lambada_runtime": 9.5735,
      "eval_lambada_samples_per_second": 508.593,
      "step": 319000
    },
    {
      "activations/layer0_attention_weight_max": 16.21659278869629,
      "activations/layer0_attention_weight_min": -13.358985900878906,
      "activations/layer10_attention_weight_max": 36.78773880004883,
      "activations/layer10_attention_weight_min": -37.76747131347656,
      "activations/layer11_attention_weight_max": 34.706573486328125,
      "activations/layer11_attention_weight_min": -37.230567932128906,
      "activations/layer12_attention_weight_max": 34.00257110595703,
      "activations/layer12_attention_weight_min": -32.16688537597656,
      "activations/layer13_attention_weight_max": 47.94166564941406,
      "activations/layer13_attention_weight_min": -41.76849365234375,
      "activations/layer14_attention_weight_max": 52.85438537597656,
      "activations/layer14_attention_weight_min": -44.16267776489258,
      "activations/layer15_attention_weight_max": 47.00532913208008,
      "activations/layer15_attention_weight_min": -40.05759811401367,
      "activations/layer16_attention_weight_max": 34.81327438354492,
      "activations/layer16_attention_weight_min": -27.60025405883789,
      "activations/layer17_attention_weight_max": 32.65190124511719,
      "activations/layer17_attention_weight_min": -26.492639541625977,
      "activations/layer18_attention_weight_max": 33.9144287109375,
      "activations/layer18_attention_weight_min": -25.53237533569336,
      "activations/layer19_attention_weight_max": 36.1496467590332,
      "activations/layer19_attention_weight_min": -30.1324405670166,
      "activations/layer1_attention_weight_max": 20.12746810913086,
      "activations/layer1_attention_weight_min": -12.340811729431152,
      "activations/layer20_attention_weight_max": 29.90062713623047,
      "activations/layer20_attention_weight_min": -24.845352172851562,
      "activations/layer21_attention_weight_max": 29.990903854370117,
      "activations/layer21_attention_weight_min": -24.020248413085938,
      "activations/layer22_attention_weight_max": 46.61286163330078,
      "activations/layer22_attention_weight_min": -30.07508087158203,
      "activations/layer23_attention_weight_max": 33.83540725708008,
      "activations/layer23_attention_weight_min": -25.069473266601562,
      "activations/layer2_attention_weight_max": 28.50495719909668,
      "activations/layer2_attention_weight_min": -30.078794479370117,
      "activations/layer3_attention_weight_max": 92.12136840820312,
      "activations/layer3_attention_weight_min": -99.1825180053711,
      "activations/layer4_attention_weight_max": 104.8320083618164,
      "activations/layer4_attention_weight_min": -107.05843353271484,
      "activations/layer5_attention_weight_max": 49.0334587097168,
      "activations/layer5_attention_weight_min": -61.2213249206543,
      "activations/layer6_attention_weight_max": 43.78196716308594,
      "activations/layer6_attention_weight_min": -47.00351333618164,
      "activations/layer7_attention_weight_max": 92.04569244384766,
      "activations/layer7_attention_weight_min": -96.00345611572266,
      "activations/layer8_attention_weight_max": 44.43763732910156,
      "activations/layer8_attention_weight_min": -45.9909553527832,
      "activations/layer9_attention_weight_max": 34.27553939819336,
      "activations/layer9_attention_weight_min": -35.8612060546875,
      "epoch": 18.54,
      "learning_rate": 3.0759848484848484e-05,
      "loss": 2.7226,
      "step": 319050
    },
    {
      "activations/layer0_attention_weight_max": 16.126150131225586,
      "activations/layer0_attention_weight_min": -13.075418472290039,
      "activations/layer10_attention_weight_max": 47.78055953979492,
      "activations/layer10_attention_weight_min": -42.75746536254883,
      "activations/layer11_attention_weight_max": 48.27102279663086,
      "activations/layer11_attention_weight_min": -44.13871383666992,
      "activations/layer12_attention_weight_max": 31.195615768432617,
      "activations/layer12_attention_weight_min": -29.264448165893555,
      "activations/layer13_attention_weight_max": 43.977962493896484,
      "activations/layer13_attention_weight_min": -39.27335739135742,
      "activations/layer14_attention_weight_max": 45.891937255859375,
      "activations/layer14_attention_weight_min": -40.747352600097656,
      "activations/layer15_attention_weight_max": 43.363765716552734,
      "activations/layer15_attention_weight_min": -41.497230529785156,
      "activations/layer16_attention_weight_max": 31.483936309814453,
      "activations/layer16_attention_weight_min": -29.6679630279541,
      "activations/layer17_attention_weight_max": 31.987390518188477,
      "activations/layer17_attention_weight_min": -29.0508975982666,
      "activations/layer18_attention_weight_max": 34.810333251953125,
      "activations/layer18_attention_weight_min": -28.924715042114258,
      "activations/layer19_attention_weight_max": 31.37118911743164,
      "activations/layer19_attention_weight_min": -33.43722152709961,
      "activations/layer1_attention_weight_max": 16.236299514770508,
      "activations/layer1_attention_weight_min": -13.457032203674316,
      "activations/layer20_attention_weight_max": 26.210918426513672,
      "activations/layer20_attention_weight_min": -26.524309158325195,
      "activations/layer21_attention_weight_max": 26.24275779724121,
      "activations/layer21_attention_weight_min": -28.237619400024414,
      "activations/layer22_attention_weight_max": 42.3955078125,
      "activations/layer22_attention_weight_min": -34.78621292114258,
      "activations/layer23_attention_weight_max": 31.64739990234375,
      "activations/layer23_attention_weight_min": -31.110837936401367,
      "activations/layer2_attention_weight_max": 29.525516510009766,
      "activations/layer2_attention_weight_min": -30.96971893310547,
      "activations/layer3_attention_weight_max": 94.06352233886719,
      "activations/layer3_attention_weight_min": -96.69461059570312,
      "activations/layer4_attention_weight_max": 104.93343353271484,
      "activations/layer4_attention_weight_min": -107.4982681274414,
      "activations/layer5_attention_weight_max": 48.6527214050293,
      "activations/layer5_attention_weight_min": -60.449214935302734,
      "activations/layer6_attention_weight_max": 43.52260971069336,
      "activations/layer6_attention_weight_min": -47.03053283691406,
      "activations/layer7_attention_weight_max": 107.51058197021484,
      "activations/layer7_attention_weight_min": -97.8813705444336,
      "activations/layer8_attention_weight_max": 52.18059539794922,
      "activations/layer8_attention_weight_min": -45.76383972167969,
      "activations/layer9_attention_weight_max": 42.09016418457031,
      "activations/layer9_attention_weight_min": -39.656429290771484,
      "epoch": 18.54,
      "learning_rate": 3.074090909090909e-05,
      "loss": 2.7528,
      "step": 319100
    },
    {
      "activations/layer0_attention_weight_max": 16.464937210083008,
      "activations/layer0_attention_weight_min": -11.987411499023438,
      "activations/layer10_attention_weight_max": 44.00381851196289,
      "activations/layer10_attention_weight_min": -45.12601089477539,
      "activations/layer11_attention_weight_max": 41.50304412841797,
      "activations/layer11_attention_weight_min": -43.4615592956543,
      "activations/layer12_attention_weight_max": 37.666236877441406,
      "activations/layer12_attention_weight_min": -32.85307312011719,
      "activations/layer13_attention_weight_max": 55.86006546020508,
      "activations/layer13_attention_weight_min": -46.93164825439453,
      "activations/layer14_attention_weight_max": 68.97246551513672,
      "activations/layer14_attention_weight_min": -53.28482437133789,
      "activations/layer15_attention_weight_max": 66.22030639648438,
      "activations/layer15_attention_weight_min": -51.59225082397461,
      "activations/layer16_attention_weight_max": 38.327606201171875,
      "activations/layer16_attention_weight_min": -31.026212692260742,
      "activations/layer17_attention_weight_max": 36.23259735107422,
      "activations/layer17_attention_weight_min": -33.65004348754883,
      "activations/layer18_attention_weight_max": 37.37232971191406,
      "activations/layer18_attention_weight_min": -35.12123489379883,
      "activations/layer19_attention_weight_max": 45.71714401245117,
      "activations/layer19_attention_weight_min": -35.03295135498047,
      "activations/layer1_attention_weight_max": 19.996618270874023,
      "activations/layer1_attention_weight_min": -14.440805435180664,
      "activations/layer20_attention_weight_max": 37.90949630737305,
      "activations/layer20_attention_weight_min": -27.25591278076172,
      "activations/layer21_attention_weight_max": 35.220672607421875,
      "activations/layer21_attention_weight_min": -29.692241668701172,
      "activations/layer22_attention_weight_max": 56.59750747680664,
      "activations/layer22_attention_weight_min": -40.6237907409668,
      "activations/layer23_attention_weight_max": 35.99740982055664,
      "activations/layer23_attention_weight_min": -26.93625259399414,
      "activations/layer2_attention_weight_max": 33.606971740722656,
      "activations/layer2_attention_weight_min": -33.40045928955078,
      "activations/layer3_attention_weight_max": 106.03632354736328,
      "activations/layer3_attention_weight_min": -105.2885971069336,
      "activations/layer4_attention_weight_max": 114.70674133300781,
      "activations/layer4_attention_weight_min": -111.26168060302734,
      "activations/layer5_attention_weight_max": 52.69316864013672,
      "activations/layer5_attention_weight_min": -60.924095153808594,
      "activations/layer6_attention_weight_max": 45.22018051147461,
      "activations/layer6_attention_weight_min": -45.97676467895508,
      "activations/layer7_attention_weight_max": 102.16706085205078,
      "activations/layer7_attention_weight_min": -106.9967041015625,
      "activations/layer8_attention_weight_max": 50.47617721557617,
      "activations/layer8_attention_weight_min": -51.600181579589844,
      "activations/layer9_attention_weight_max": 38.163143157958984,
      "activations/layer9_attention_weight_min": -41.16324996948242,
      "epoch": 18.54,
      "learning_rate": 3.072234848484848e-05,
      "loss": 2.7432,
      "step": 319150
    },
    {
      "activations/layer0_attention_weight_max": 16.394577026367188,
      "activations/layer0_attention_weight_min": -12.570455551147461,
      "activations/layer10_attention_weight_max": 32.24725341796875,
      "activations/layer10_attention_weight_min": -33.69740676879883,
      "activations/layer11_attention_weight_max": 34.053955078125,
      "activations/layer11_attention_weight_min": -34.34130859375,
      "activations/layer12_attention_weight_max": 25.00078582763672,
      "activations/layer12_attention_weight_min": -26.075571060180664,
      "activations/layer13_attention_weight_max": 37.61941146850586,
      "activations/layer13_attention_weight_min": -32.97340393066406,
      "activations/layer14_attention_weight_max": 39.226104736328125,
      "activations/layer14_attention_weight_min": -37.96595001220703,
      "activations/layer15_attention_weight_max": 34.43669891357422,
      "activations/layer15_attention_weight_min": -34.992469787597656,
      "activations/layer16_attention_weight_max": 27.961790084838867,
      "activations/layer16_attention_weight_min": -29.40276336669922,
      "activations/layer17_attention_weight_max": 28.299102783203125,
      "activations/layer17_attention_weight_min": -26.276315689086914,
      "activations/layer18_attention_weight_max": 29.42243766784668,
      "activations/layer18_attention_weight_min": -26.09400177001953,
      "activations/layer19_attention_weight_max": 29.30234146118164,
      "activations/layer19_attention_weight_min": -31.379886627197266,
      "activations/layer1_attention_weight_max": 15.531509399414062,
      "activations/layer1_attention_weight_min": -13.254045486450195,
      "activations/layer20_attention_weight_max": 26.694595336914062,
      "activations/layer20_attention_weight_min": -25.801074981689453,
      "activations/layer21_attention_weight_max": 27.982694625854492,
      "activations/layer21_attention_weight_min": -25.80636978149414,
      "activations/layer22_attention_weight_max": 37.33961486816406,
      "activations/layer22_attention_weight_min": -31.326534271240234,
      "activations/layer23_attention_weight_max": 31.14105796813965,
      "activations/layer23_attention_weight_min": -26.377552032470703,
      "activations/layer2_attention_weight_max": 29.426864624023438,
      "activations/layer2_attention_weight_min": -27.952062606811523,
      "activations/layer3_attention_weight_max": 89.96632385253906,
      "activations/layer3_attention_weight_min": -92.19867706298828,
      "activations/layer4_attention_weight_max": 100.99948120117188,
      "activations/layer4_attention_weight_min": -103.2706527709961,
      "activations/layer5_attention_weight_max": 47.229042053222656,
      "activations/layer5_attention_weight_min": -58.680442810058594,
      "activations/layer6_attention_weight_max": 41.21390914916992,
      "activations/layer6_attention_weight_min": -43.1922721862793,
      "activations/layer7_attention_weight_max": 86.28414916992188,
      "activations/layer7_attention_weight_min": -86.37666320800781,
      "activations/layer8_attention_weight_max": 39.29042434692383,
      "activations/layer8_attention_weight_min": -40.69798278808594,
      "activations/layer9_attention_weight_max": 29.138654708862305,
      "activations/layer9_attention_weight_min": -31.99933433532715,
      "epoch": 18.55,
      "learning_rate": 3.070340909090909e-05,
      "loss": 2.7377,
      "step": 319200
    },
    {
      "activations/layer0_attention_weight_max": 16.816884994506836,
      "activations/layer0_attention_weight_min": -12.428773880004883,
      "activations/layer10_attention_weight_max": 40.9851188659668,
      "activations/layer10_attention_weight_min": -41.08599853515625,
      "activations/layer11_attention_weight_max": 41.46744918823242,
      "activations/layer11_attention_weight_min": -41.11296844482422,
      "activations/layer12_attention_weight_max": 31.62004852294922,
      "activations/layer12_attention_weight_min": -29.94973373413086,
      "activations/layer13_attention_weight_max": 43.630775451660156,
      "activations/layer13_attention_weight_min": -39.15569305419922,
      "activations/layer14_attention_weight_max": 45.936370849609375,
      "activations/layer14_attention_weight_min": -43.039825439453125,
      "activations/layer15_attention_weight_max": 50.430747985839844,
      "activations/layer15_attention_weight_min": -41.7485237121582,
      "activations/layer16_attention_weight_max": 33.06259536743164,
      "activations/layer16_attention_weight_min": -27.5773868560791,
      "activations/layer17_attention_weight_max": 29.770715713500977,
      "activations/layer17_attention_weight_min": -27.12201499938965,
      "activations/layer18_attention_weight_max": 31.635896682739258,
      "activations/layer18_attention_weight_min": -25.849300384521484,
      "activations/layer19_attention_weight_max": 37.45781326293945,
      "activations/layer19_attention_weight_min": -32.5123405456543,
      "activations/layer1_attention_weight_max": 16.222871780395508,
      "activations/layer1_attention_weight_min": -13.310235977172852,
      "activations/layer20_attention_weight_max": 28.956096649169922,
      "activations/layer20_attention_weight_min": -25.719188690185547,
      "activations/layer21_attention_weight_max": 31.208017349243164,
      "activations/layer21_attention_weight_min": -24.731719970703125,
      "activations/layer22_attention_weight_max": 40.173583984375,
      "activations/layer22_attention_weight_min": -32.83867263793945,
      "activations/layer23_attention_weight_max": 36.16918182373047,
      "activations/layer23_attention_weight_min": -26.198680877685547,
      "activations/layer2_attention_weight_max": 32.233917236328125,
      "activations/layer2_attention_weight_min": -33.31764221191406,
      "activations/layer3_attention_weight_max": 101.60231018066406,
      "activations/layer3_attention_weight_min": -99.9153060913086,
      "activations/layer4_attention_weight_max": 107.07439422607422,
      "activations/layer4_attention_weight_min": -106.1520767211914,
      "activations/layer5_attention_weight_max": 50.14433288574219,
      "activations/layer5_attention_weight_min": -63.24198532104492,
      "activations/layer6_attention_weight_max": 45.68297576904297,
      "activations/layer6_attention_weight_min": -46.02352523803711,
      "activations/layer7_attention_weight_max": 101.61561584472656,
      "activations/layer7_attention_weight_min": -96.42234802246094,
      "activations/layer8_attention_weight_max": 48.300418853759766,
      "activations/layer8_attention_weight_min": -48.9140739440918,
      "activations/layer9_attention_weight_max": 37.006553649902344,
      "activations/layer9_attention_weight_min": -39.4709358215332,
      "epoch": 18.55,
      "learning_rate": 3.068446969696969e-05,
      "loss": 2.7146,
      "step": 319250
    },
    {
      "activations/layer0_attention_weight_max": 15.847406387329102,
      "activations/layer0_attention_weight_min": -12.594246864318848,
      "activations/layer10_attention_weight_max": 32.84429168701172,
      "activations/layer10_attention_weight_min": -34.11799621582031,
      "activations/layer11_attention_weight_max": 33.81753921508789,
      "activations/layer11_attention_weight_min": -35.45236587524414,
      "activations/layer12_attention_weight_max": 26.209169387817383,
      "activations/layer12_attention_weight_min": -29.176685333251953,
      "activations/layer13_attention_weight_max": 37.83120346069336,
      "activations/layer13_attention_weight_min": -37.04848861694336,
      "activations/layer14_attention_weight_max": 41.72235107421875,
      "activations/layer14_attention_weight_min": -41.83858108520508,
      "activations/layer15_attention_weight_max": 38.3685302734375,
      "activations/layer15_attention_weight_min": -36.90420150756836,
      "activations/layer16_attention_weight_max": 28.787250518798828,
      "activations/layer16_attention_weight_min": -29.58341407775879,
      "activations/layer17_attention_weight_max": 28.86093521118164,
      "activations/layer17_attention_weight_min": -28.04865074157715,
      "activations/layer18_attention_weight_max": 33.27260971069336,
      "activations/layer18_attention_weight_min": -29.04948616027832,
      "activations/layer19_attention_weight_max": 33.70448303222656,
      "activations/layer19_attention_weight_min": -33.93553161621094,
      "activations/layer1_attention_weight_max": 15.627044677734375,
      "activations/layer1_attention_weight_min": -14.189282417297363,
      "activations/layer20_attention_weight_max": 28.96637535095215,
      "activations/layer20_attention_weight_min": -28.013994216918945,
      "activations/layer21_attention_weight_max": 29.467330932617188,
      "activations/layer21_attention_weight_min": -25.576574325561523,
      "activations/layer22_attention_weight_max": 44.0338020324707,
      "activations/layer22_attention_weight_min": -34.35094451904297,
      "activations/layer23_attention_weight_max": 32.49711227416992,
      "activations/layer23_attention_weight_min": -28.72553825378418,
      "activations/layer2_attention_weight_max": 30.870513916015625,
      "activations/layer2_attention_weight_min": -30.04395866394043,
      "activations/layer3_attention_weight_max": 96.26322174072266,
      "activations/layer3_attention_weight_min": -93.29515838623047,
      "activations/layer4_attention_weight_max": 101.22228240966797,
      "activations/layer4_attention_weight_min": -107.04353332519531,
      "activations/layer5_attention_weight_max": 51.674312591552734,
      "activations/layer5_attention_weight_min": -65.76964569091797,
      "activations/layer6_attention_weight_max": 45.12925338745117,
      "activations/layer6_attention_weight_min": -48.813819885253906,
      "activations/layer7_attention_weight_max": 91.49267578125,
      "activations/layer7_attention_weight_min": -98.87054443359375,
      "activations/layer8_attention_weight_max": 41.07875442504883,
      "activations/layer8_attention_weight_min": -45.369789123535156,
      "activations/layer9_attention_weight_max": 30.43739128112793,
      "activations/layer9_attention_weight_min": -33.558753967285156,
      "epoch": 18.55,
      "learning_rate": 3.06655303030303e-05,
      "loss": 2.7197,
      "step": 319300
    },
    {
      "activations/layer0_attention_weight_max": 17.19847869873047,
      "activations/layer0_attention_weight_min": -13.578126907348633,
      "activations/layer10_attention_weight_max": 32.684112548828125,
      "activations/layer10_attention_weight_min": -32.5900764465332,
      "activations/layer11_attention_weight_max": 30.996482849121094,
      "activations/layer11_attention_weight_min": -34.355804443359375,
      "activations/layer12_attention_weight_max": 29.92312240600586,
      "activations/layer12_attention_weight_min": -27.28984260559082,
      "activations/layer13_attention_weight_max": 38.12453079223633,
      "activations/layer13_attention_weight_min": -34.393611907958984,
      "activations/layer14_attention_weight_max": 44.02621078491211,
      "activations/layer14_attention_weight_min": -36.271270751953125,
      "activations/layer15_attention_weight_max": 40.27351760864258,
      "activations/layer15_attention_weight_min": -34.31660842895508,
      "activations/layer16_attention_weight_max": 29.117509841918945,
      "activations/layer16_attention_weight_min": -28.266298294067383,
      "activations/layer17_attention_weight_max": 29.63181495666504,
      "activations/layer17_attention_weight_min": -25.406085968017578,
      "activations/layer18_attention_weight_max": 33.80561828613281,
      "activations/layer18_attention_weight_min": -28.168020248413086,
      "activations/layer19_attention_weight_max": 34.205177307128906,
      "activations/layer19_attention_weight_min": -32.491493225097656,
      "activations/layer1_attention_weight_max": 16.101648330688477,
      "activations/layer1_attention_weight_min": -13.761844635009766,
      "activations/layer20_attention_weight_max": 29.345252990722656,
      "activations/layer20_attention_weight_min": -27.054847717285156,
      "activations/layer21_attention_weight_max": 32.425743103027344,
      "activations/layer21_attention_weight_min": -26.6372127532959,
      "activations/layer22_attention_weight_max": 45.1680908203125,
      "activations/layer22_attention_weight_min": -34.25662612915039,
      "activations/layer23_attention_weight_max": 34.94029235839844,
      "activations/layer23_attention_weight_min": -28.312841415405273,
      "activations/layer2_attention_weight_max": 30.30038070678711,
      "activations/layer2_attention_weight_min": -28.739744186401367,
      "activations/layer3_attention_weight_max": 92.83269500732422,
      "activations/layer3_attention_weight_min": -93.67157745361328,
      "activations/layer4_attention_weight_max": 103.92635345458984,
      "activations/layer4_attention_weight_min": -99.89904022216797,
      "activations/layer5_attention_weight_max": 51.089534759521484,
      "activations/layer5_attention_weight_min": -59.76929473876953,
      "activations/layer6_attention_weight_max": 42.41279602050781,
      "activations/layer6_attention_weight_min": -44.99065399169922,
      "activations/layer7_attention_weight_max": 86.9560317993164,
      "activations/layer7_attention_weight_min": -94.35211944580078,
      "activations/layer8_attention_weight_max": 39.99540710449219,
      "activations/layer8_attention_weight_min": -43.84020233154297,
      "activations/layer9_attention_weight_max": 29.427488327026367,
      "activations/layer9_attention_weight_min": -33.73814392089844,
      "epoch": 18.56,
      "learning_rate": 3.06465909090909e-05,
      "loss": 2.732,
      "step": 319350
    },
    {
      "activations/layer0_attention_weight_max": 15.875832557678223,
      "activations/layer0_attention_weight_min": -12.042816162109375,
      "activations/layer10_attention_weight_max": 31.923973083496094,
      "activations/layer10_attention_weight_min": -34.30208206176758,
      "activations/layer11_attention_weight_max": 35.69321823120117,
      "activations/layer11_attention_weight_min": -35.373329162597656,
      "activations/layer12_attention_weight_max": 25.179473876953125,
      "activations/layer12_attention_weight_min": -27.466588973999023,
      "activations/layer13_attention_weight_max": 37.72529220581055,
      "activations/layer13_attention_weight_min": -37.110107421875,
      "activations/layer14_attention_weight_max": 40.6280632019043,
      "activations/layer14_attention_weight_min": -40.36001968383789,
      "activations/layer15_attention_weight_max": 37.014549255371094,
      "activations/layer15_attention_weight_min": -39.71517562866211,
      "activations/layer16_attention_weight_max": 27.709482192993164,
      "activations/layer16_attention_weight_min": -31.322338104248047,
      "activations/layer17_attention_weight_max": 32.3570556640625,
      "activations/layer17_attention_weight_min": -27.02457046508789,
      "activations/layer18_attention_weight_max": 32.482608795166016,
      "activations/layer18_attention_weight_min": -29.127599716186523,
      "activations/layer19_attention_weight_max": 37.67742156982422,
      "activations/layer19_attention_weight_min": -33.53044128417969,
      "activations/layer1_attention_weight_max": 16.929290771484375,
      "activations/layer1_attention_weight_min": -14.356247901916504,
      "activations/layer20_attention_weight_max": 33.15782165527344,
      "activations/layer20_attention_weight_min": -25.809642791748047,
      "activations/layer21_attention_weight_max": 32.35022735595703,
      "activations/layer21_attention_weight_min": -29.950345993041992,
      "activations/layer22_attention_weight_max": 41.627159118652344,
      "activations/layer22_attention_weight_min": -36.042850494384766,
      "activations/layer23_attention_weight_max": 31.942989349365234,
      "activations/layer23_attention_weight_min": -28.046894073486328,
      "activations/layer2_attention_weight_max": 29.435680389404297,
      "activations/layer2_attention_weight_min": -28.691619873046875,
      "activations/layer3_attention_weight_max": 94.20292663574219,
      "activations/layer3_attention_weight_min": -94.01629638671875,
      "activations/layer4_attention_weight_max": 103.57006072998047,
      "activations/layer4_attention_weight_min": -104.53712463378906,
      "activations/layer5_attention_weight_max": 50.18436813354492,
      "activations/layer5_attention_weight_min": -62.287044525146484,
      "activations/layer6_attention_weight_max": 41.75361633300781,
      "activations/layer6_attention_weight_min": -46.32917022705078,
      "activations/layer7_attention_weight_max": 91.86112213134766,
      "activations/layer7_attention_weight_min": -92.39527130126953,
      "activations/layer8_attention_weight_max": 40.14795684814453,
      "activations/layer8_attention_weight_min": -42.733699798583984,
      "activations/layer9_attention_weight_max": 32.2446403503418,
      "activations/layer9_attention_weight_min": -33.698482513427734,
      "epoch": 18.56,
      "learning_rate": 3.062765151515151e-05,
      "loss": 2.7298,
      "step": 319400
    },
    {
      "activations/layer0_attention_weight_max": 15.991308212280273,
      "activations/layer0_attention_weight_min": -12.18981647491455,
      "activations/layer10_attention_weight_max": 32.25078201293945,
      "activations/layer10_attention_weight_min": -34.26038360595703,
      "activations/layer11_attention_weight_max": 32.0838508605957,
      "activations/layer11_attention_weight_min": -33.26087188720703,
      "activations/layer12_attention_weight_max": 26.544795989990234,
      "activations/layer12_attention_weight_min": -26.703893661499023,
      "activations/layer13_attention_weight_max": 41.84632110595703,
      "activations/layer13_attention_weight_min": -36.42132568359375,
      "activations/layer14_attention_weight_max": 44.43373107910156,
      "activations/layer14_attention_weight_min": -41.13037872314453,
      "activations/layer15_attention_weight_max": 40.744903564453125,
      "activations/layer15_attention_weight_min": -36.93869400024414,
      "activations/layer16_attention_weight_max": 29.969541549682617,
      "activations/layer16_attention_weight_min": -27.67059898376465,
      "activations/layer17_attention_weight_max": 28.81326675415039,
      "activations/layer17_attention_weight_min": -25.422372817993164,
      "activations/layer18_attention_weight_max": 29.92475700378418,
      "activations/layer18_attention_weight_min": -25.490585327148438,
      "activations/layer19_attention_weight_max": 31.430776596069336,
      "activations/layer19_attention_weight_min": -31.2559757232666,
      "activations/layer1_attention_weight_max": 16.514739990234375,
      "activations/layer1_attention_weight_min": -14.377358436584473,
      "activations/layer20_attention_weight_max": 27.916027069091797,
      "activations/layer20_attention_weight_min": -25.23268699645996,
      "activations/layer21_attention_weight_max": 25.64861488342285,
      "activations/layer21_attention_weight_min": -24.732192993164062,
      "activations/layer22_attention_weight_max": 38.67643356323242,
      "activations/layer22_attention_weight_min": -30.95509147644043,
      "activations/layer23_attention_weight_max": 29.6356258392334,
      "activations/layer23_attention_weight_min": -24.59698486328125,
      "activations/layer2_attention_weight_max": 31.476713180541992,
      "activations/layer2_attention_weight_min": -30.57010269165039,
      "activations/layer3_attention_weight_max": 92.97252655029297,
      "activations/layer3_attention_weight_min": -99.83138275146484,
      "activations/layer4_attention_weight_max": 101.72467803955078,
      "activations/layer4_attention_weight_min": -104.6184310913086,
      "activations/layer5_attention_weight_max": 48.53125,
      "activations/layer5_attention_weight_min": -60.576194763183594,
      "activations/layer6_attention_weight_max": 41.381736755371094,
      "activations/layer6_attention_weight_min": -44.83956527709961,
      "activations/layer7_attention_weight_max": 89.67603302001953,
      "activations/layer7_attention_weight_min": -91.18889617919922,
      "activations/layer8_attention_weight_max": 39.88681411743164,
      "activations/layer8_attention_weight_min": -43.51303482055664,
      "activations/layer9_attention_weight_max": 30.967899322509766,
      "activations/layer9_attention_weight_min": -32.415771484375,
      "epoch": 18.56,
      "learning_rate": 3.060871212121212e-05,
      "loss": 2.7349,
      "step": 319450
    },
    {
      "activations/layer0_attention_weight_max": 16.003149032592773,
      "activations/layer0_attention_weight_min": -12.722808837890625,
      "activations/layer10_attention_weight_max": 38.85725784301758,
      "activations/layer10_attention_weight_min": -40.08535385131836,
      "activations/layer11_attention_weight_max": 38.727882385253906,
      "activations/layer11_attention_weight_min": -40.379905700683594,
      "activations/layer12_attention_weight_max": 29.85074234008789,
      "activations/layer12_attention_weight_min": -28.898439407348633,
      "activations/layer13_attention_weight_max": 40.66509246826172,
      "activations/layer13_attention_weight_min": -39.80229187011719,
      "activations/layer14_attention_weight_max": 43.640174865722656,
      "activations/layer14_attention_weight_min": -40.773765563964844,
      "activations/layer15_attention_weight_max": 38.92878341674805,
      "activations/layer15_attention_weight_min": -38.76323699951172,
      "activations/layer16_attention_weight_max": 29.969316482543945,
      "activations/layer16_attention_weight_min": -30.836673736572266,
      "activations/layer17_attention_weight_max": 31.14450454711914,
      "activations/layer17_attention_weight_min": -29.708200454711914,
      "activations/layer18_attention_weight_max": 31.577077865600586,
      "activations/layer18_attention_weight_min": -28.25712776184082,
      "activations/layer19_attention_weight_max": 32.0955810546875,
      "activations/layer19_attention_weight_min": -32.61421585083008,
      "activations/layer1_attention_weight_max": 15.689695358276367,
      "activations/layer1_attention_weight_min": -14.85847282409668,
      "activations/layer20_attention_weight_max": 30.621335983276367,
      "activations/layer20_attention_weight_min": -28.561737060546875,
      "activations/layer21_attention_weight_max": 30.905771255493164,
      "activations/layer21_attention_weight_min": -26.899124145507812,
      "activations/layer22_attention_weight_max": 42.38220977783203,
      "activations/layer22_attention_weight_min": -33.576515197753906,
      "activations/layer23_attention_weight_max": 33.89228820800781,
      "activations/layer23_attention_weight_min": -26.09632110595703,
      "activations/layer2_attention_weight_max": 31.883380889892578,
      "activations/layer2_attention_weight_min": -31.342559814453125,
      "activations/layer3_attention_weight_max": 103.02041625976562,
      "activations/layer3_attention_weight_min": -102.75797271728516,
      "activations/layer4_attention_weight_max": 112.90303039550781,
      "activations/layer4_attention_weight_min": -113.61981201171875,
      "activations/layer5_attention_weight_max": 52.06639862060547,
      "activations/layer5_attention_weight_min": -63.46601486206055,
      "activations/layer6_attention_weight_max": 47.46129608154297,
      "activations/layer6_attention_weight_min": -47.289310455322266,
      "activations/layer7_attention_weight_max": 102.55145263671875,
      "activations/layer7_attention_weight_min": -104.70994567871094,
      "activations/layer8_attention_weight_max": 46.618507385253906,
      "activations/layer8_attention_weight_min": -50.156986236572266,
      "activations/layer9_attention_weight_max": 35.256683349609375,
      "activations/layer9_attention_weight_min": -37.32142639160156,
      "epoch": 18.56,
      "learning_rate": 3.058977272727273e-05,
      "loss": 2.7511,
      "step": 319500
    },
    {
      "activations/layer0_attention_weight_max": 16.132856369018555,
      "activations/layer0_attention_weight_min": -12.352592468261719,
      "activations/layer10_attention_weight_max": 33.094703674316406,
      "activations/layer10_attention_weight_min": -33.406829833984375,
      "activations/layer11_attention_weight_max": 32.660484313964844,
      "activations/layer11_attention_weight_min": -32.63468933105469,
      "activations/layer12_attention_weight_max": 25.96963882446289,
      "activations/layer12_attention_weight_min": -26.328636169433594,
      "activations/layer13_attention_weight_max": 39.26476287841797,
      "activations/layer13_attention_weight_min": -35.96763610839844,
      "activations/layer14_attention_weight_max": 42.396339416503906,
      "activations/layer14_attention_weight_min": -39.28227996826172,
      "activations/layer15_attention_weight_max": 41.834686279296875,
      "activations/layer15_attention_weight_min": -37.10639572143555,
      "activations/layer16_attention_weight_max": 29.865163803100586,
      "activations/layer16_attention_weight_min": -27.73847198486328,
      "activations/layer17_attention_weight_max": 29.622297286987305,
      "activations/layer17_attention_weight_min": -25.53338623046875,
      "activations/layer18_attention_weight_max": 33.52687454223633,
      "activations/layer18_attention_weight_min": -25.01360511779785,
      "activations/layer19_attention_weight_max": 33.36838912963867,
      "activations/layer19_attention_weight_min": -31.207672119140625,
      "activations/layer1_attention_weight_max": 15.568499565124512,
      "activations/layer1_attention_weight_min": -14.755912780761719,
      "activations/layer20_attention_weight_max": 30.006168365478516,
      "activations/layer20_attention_weight_min": -26.434545516967773,
      "activations/layer21_attention_weight_max": 29.09658432006836,
      "activations/layer21_attention_weight_min": -23.916305541992188,
      "activations/layer22_attention_weight_max": 39.975128173828125,
      "activations/layer22_attention_weight_min": -29.656232833862305,
      "activations/layer23_attention_weight_max": 33.33271789550781,
      "activations/layer23_attention_weight_min": -24.868045806884766,
      "activations/layer2_attention_weight_max": 33.1951904296875,
      "activations/layer2_attention_weight_min": -31.300357818603516,
      "activations/layer3_attention_weight_max": 96.4878158569336,
      "activations/layer3_attention_weight_min": -95.63630676269531,
      "activations/layer4_attention_weight_max": 104.14790344238281,
      "activations/layer4_attention_weight_min": -104.26173400878906,
      "activations/layer5_attention_weight_max": 50.337181091308594,
      "activations/layer5_attention_weight_min": -62.39812469482422,
      "activations/layer6_attention_weight_max": 43.87627410888672,
      "activations/layer6_attention_weight_min": -46.591453552246094,
      "activations/layer7_attention_weight_max": 93.5035400390625,
      "activations/layer7_attention_weight_min": -100.6127700805664,
      "activations/layer8_attention_weight_max": 39.984703063964844,
      "activations/layer8_attention_weight_min": -43.01658248901367,
      "activations/layer9_attention_weight_max": 33.271392822265625,
      "activations/layer9_attention_weight_min": -34.07615280151367,
      "epoch": 18.57,
      "learning_rate": 3.057083333333333e-05,
      "loss": 2.7442,
      "step": 319550
    },
    {
      "activations/layer0_attention_weight_max": 16.77616310119629,
      "activations/layer0_attention_weight_min": -13.568382263183594,
      "activations/layer10_attention_weight_max": 36.6878547668457,
      "activations/layer10_attention_weight_min": -35.7100830078125,
      "activations/layer11_attention_weight_max": 35.91727828979492,
      "activations/layer11_attention_weight_min": -36.051673889160156,
      "activations/layer12_attention_weight_max": 31.706058502197266,
      "activations/layer12_attention_weight_min": -27.954151153564453,
      "activations/layer13_attention_weight_max": 48.283748626708984,
      "activations/layer13_attention_weight_min": -39.57460021972656,
      "activations/layer14_attention_weight_max": 50.977081298828125,
      "activations/layer14_attention_weight_min": -40.9278678894043,
      "activations/layer15_attention_weight_max": 47.2495231628418,
      "activations/layer15_attention_weight_min": -41.747901916503906,
      "activations/layer16_attention_weight_max": 32.422603607177734,
      "activations/layer16_attention_weight_min": -29.974529266357422,
      "activations/layer17_attention_weight_max": 29.55734634399414,
      "activations/layer17_attention_weight_min": -28.862504959106445,
      "activations/layer18_attention_weight_max": 33.80501937866211,
      "activations/layer18_attention_weight_min": -27.929521560668945,
      "activations/layer19_attention_weight_max": 37.358360290527344,
      "activations/layer19_attention_weight_min": -31.992923736572266,
      "activations/layer1_attention_weight_max": 16.53217887878418,
      "activations/layer1_attention_weight_min": -13.752264022827148,
      "activations/layer20_attention_weight_max": 29.547311782836914,
      "activations/layer20_attention_weight_min": -24.932111740112305,
      "activations/layer21_attention_weight_max": 28.55267333984375,
      "activations/layer21_attention_weight_min": -26.072738647460938,
      "activations/layer22_attention_weight_max": 44.592041015625,
      "activations/layer22_attention_weight_min": -33.88907241821289,
      "activations/layer23_attention_weight_max": 33.550682067871094,
      "activations/layer23_attention_weight_min": -26.099943161010742,
      "activations/layer2_attention_weight_max": 31.60066795349121,
      "activations/layer2_attention_weight_min": -30.291248321533203,
      "activations/layer3_attention_weight_max": 93.94985961914062,
      "activations/layer3_attention_weight_min": -97.87371063232422,
      "activations/layer4_attention_weight_max": 108.1960220336914,
      "activations/layer4_attention_weight_min": -107.66442108154297,
      "activations/layer5_attention_weight_max": 50.59185028076172,
      "activations/layer5_attention_weight_min": -63.91108322143555,
      "activations/layer6_attention_weight_max": 43.081974029541016,
      "activations/layer6_attention_weight_min": -46.949337005615234,
      "activations/layer7_attention_weight_max": 92.10513305664062,
      "activations/layer7_attention_weight_min": -94.94190216064453,
      "activations/layer8_attention_weight_max": 41.250022888183594,
      "activations/layer8_attention_weight_min": -43.7346076965332,
      "activations/layer9_attention_weight_max": 34.39434051513672,
      "activations/layer9_attention_weight_min": -36.19857406616211,
      "epoch": 18.57,
      "learning_rate": 3.055189393939394e-05,
      "loss": 2.7252,
      "step": 319600
    },
    {
      "activations/layer0_attention_weight_max": 15.929656982421875,
      "activations/layer0_attention_weight_min": -13.344655990600586,
      "activations/layer10_attention_weight_max": 37.6744384765625,
      "activations/layer10_attention_weight_min": -37.30891036987305,
      "activations/layer11_attention_weight_max": 35.816810607910156,
      "activations/layer11_attention_weight_min": -38.749088287353516,
      "activations/layer12_attention_weight_max": 29.15556526184082,
      "activations/layer12_attention_weight_min": -28.379514694213867,
      "activations/layer13_attention_weight_max": 45.03703689575195,
      "activations/layer13_attention_weight_min": -37.26555252075195,
      "activations/layer14_attention_weight_max": 48.11964416503906,
      "activations/layer14_attention_weight_min": -40.399112701416016,
      "activations/layer15_attention_weight_max": 40.562400817871094,
      "activations/layer15_attention_weight_min": -37.77196502685547,
      "activations/layer16_attention_weight_max": 31.183475494384766,
      "activations/layer16_attention_weight_min": -29.556285858154297,
      "activations/layer17_attention_weight_max": 32.07756042480469,
      "activations/layer17_attention_weight_min": -28.659467697143555,
      "activations/layer18_attention_weight_max": 31.534526824951172,
      "activations/layer18_attention_weight_min": -27.215795516967773,
      "activations/layer19_attention_weight_max": 32.50992965698242,
      "activations/layer19_attention_weight_min": -32.61192321777344,
      "activations/layer1_attention_weight_max": 16.142406463623047,
      "activations/layer1_attention_weight_min": -15.054726600646973,
      "activations/layer20_attention_weight_max": 29.007543563842773,
      "activations/layer20_attention_weight_min": -26.393442153930664,
      "activations/layer21_attention_weight_max": 28.710416793823242,
      "activations/layer21_attention_weight_min": -25.273557662963867,
      "activations/layer22_attention_weight_max": 43.201080322265625,
      "activations/layer22_attention_weight_min": -31.930757522583008,
      "activations/layer23_attention_weight_max": 30.769338607788086,
      "activations/layer23_attention_weight_min": -25.875850677490234,
      "activations/layer2_attention_weight_max": 31.630165100097656,
      "activations/layer2_attention_weight_min": -29.273239135742188,
      "activations/layer3_attention_weight_max": 99.58993530273438,
      "activations/layer3_attention_weight_min": -96.97257232666016,
      "activations/layer4_attention_weight_max": 105.6844482421875,
      "activations/layer4_attention_weight_min": -107.9839859008789,
      "activations/layer5_attention_weight_max": 50.59661865234375,
      "activations/layer5_attention_weight_min": -62.661399841308594,
      "activations/layer6_attention_weight_max": 46.72783279418945,
      "activations/layer6_attention_weight_min": -46.74750518798828,
      "activations/layer7_attention_weight_max": 95.86278533935547,
      "activations/layer7_attention_weight_min": -92.11547088623047,
      "activations/layer8_attention_weight_max": 44.2362060546875,
      "activations/layer8_attention_weight_min": -46.681217193603516,
      "activations/layer9_attention_weight_max": 37.66802215576172,
      "activations/layer9_attention_weight_min": -38.570465087890625,
      "epoch": 18.57,
      "learning_rate": 3.053295454545454e-05,
      "loss": 2.7344,
      "step": 319650
    },
    {
      "activations/layer0_attention_weight_max": 16.48048973083496,
      "activations/layer0_attention_weight_min": -13.07157039642334,
      "activations/layer10_attention_weight_max": 37.021568298339844,
      "activations/layer10_attention_weight_min": -35.27985382080078,
      "activations/layer11_attention_weight_max": 37.276546478271484,
      "activations/layer11_attention_weight_min": -37.74174118041992,
      "activations/layer12_attention_weight_max": 25.37413787841797,
      "activations/layer12_attention_weight_min": -25.615623474121094,
      "activations/layer13_attention_weight_max": 37.96150588989258,
      "activations/layer13_attention_weight_min": -33.859622955322266,
      "activations/layer14_attention_weight_max": 41.127105712890625,
      "activations/layer14_attention_weight_min": -37.12250518798828,
      "activations/layer15_attention_weight_max": 37.62310028076172,
      "activations/layer15_attention_weight_min": -35.288177490234375,
      "activations/layer16_attention_weight_max": 29.258508682250977,
      "activations/layer16_attention_weight_min": -26.646623611450195,
      "activations/layer17_attention_weight_max": 27.667095184326172,
      "activations/layer17_attention_weight_min": -23.71982765197754,
      "activations/layer18_attention_weight_max": 30.967191696166992,
      "activations/layer18_attention_weight_min": -24.97598648071289,
      "activations/layer19_attention_weight_max": 31.577306747436523,
      "activations/layer19_attention_weight_min": -28.943618774414062,
      "activations/layer1_attention_weight_max": 16.169971466064453,
      "activations/layer1_attention_weight_min": -14.15550422668457,
      "activations/layer20_attention_weight_max": 26.806644439697266,
      "activations/layer20_attention_weight_min": -23.4740047454834,
      "activations/layer21_attention_weight_max": 26.978208541870117,
      "activations/layer21_attention_weight_min": -24.77008628845215,
      "activations/layer22_attention_weight_max": 36.46613311767578,
      "activations/layer22_attention_weight_min": -31.37894058227539,
      "activations/layer23_attention_weight_max": 31.335182189941406,
      "activations/layer23_attention_weight_min": -24.415607452392578,
      "activations/layer2_attention_weight_max": 29.856544494628906,
      "activations/layer2_attention_weight_min": -31.933673858642578,
      "activations/layer3_attention_weight_max": 94.06758117675781,
      "activations/layer3_attention_weight_min": -101.18595123291016,
      "activations/layer4_attention_weight_max": 103.39937591552734,
      "activations/layer4_attention_weight_min": -102.99674224853516,
      "activations/layer5_attention_weight_max": 51.25849914550781,
      "activations/layer5_attention_weight_min": -63.199825286865234,
      "activations/layer6_attention_weight_max": 44.96096420288086,
      "activations/layer6_attention_weight_min": -47.52660369873047,
      "activations/layer7_attention_weight_max": 92.0863265991211,
      "activations/layer7_attention_weight_min": -94.57575988769531,
      "activations/layer8_attention_weight_max": 45.08610153198242,
      "activations/layer8_attention_weight_min": -43.31767654418945,
      "activations/layer9_attention_weight_max": 33.9835319519043,
      "activations/layer9_attention_weight_min": -35.867919921875,
      "epoch": 18.58,
      "learning_rate": 3.051401515151515e-05,
      "loss": 2.7359,
      "step": 319700
    },
    {
      "activations/layer0_attention_weight_max": 16.63661003112793,
      "activations/layer0_attention_weight_min": -13.507611274719238,
      "activations/layer10_attention_weight_max": 35.460384368896484,
      "activations/layer10_attention_weight_min": -35.34986114501953,
      "activations/layer11_attention_weight_max": 35.17098617553711,
      "activations/layer11_attention_weight_min": -35.66995620727539,
      "activations/layer12_attention_weight_max": 26.930482864379883,
      "activations/layer12_attention_weight_min": -26.776931762695312,
      "activations/layer13_attention_weight_max": 40.759090423583984,
      "activations/layer13_attention_weight_min": -36.403587341308594,
      "activations/layer14_attention_weight_max": 45.7345085144043,
      "activations/layer14_attention_weight_min": -39.083736419677734,
      "activations/layer15_attention_weight_max": 41.193965911865234,
      "activations/layer15_attention_weight_min": -35.7132453918457,
      "activations/layer16_attention_weight_max": 28.816875457763672,
      "activations/layer16_attention_weight_min": -27.505571365356445,
      "activations/layer17_attention_weight_max": 29.450172424316406,
      "activations/layer17_attention_weight_min": -25.365150451660156,
      "activations/layer18_attention_weight_max": 29.79119873046875,
      "activations/layer18_attention_weight_min": -25.120210647583008,
      "activations/layer19_attention_weight_max": 32.236045837402344,
      "activations/layer19_attention_weight_min": -31.368459701538086,
      "activations/layer1_attention_weight_max": 17.049293518066406,
      "activations/layer1_attention_weight_min": -14.538329124450684,
      "activations/layer20_attention_weight_max": 28.568273544311523,
      "activations/layer20_attention_weight_min": -24.647249221801758,
      "activations/layer21_attention_weight_max": 25.81748390197754,
      "activations/layer21_attention_weight_min": -24.258007049560547,
      "activations/layer22_attention_weight_max": 37.641883850097656,
      "activations/layer22_attention_weight_min": -31.86376190185547,
      "activations/layer23_attention_weight_max": 30.6040096282959,
      "activations/layer23_attention_weight_min": -25.100772857666016,
      "activations/layer2_attention_weight_max": 29.67913055419922,
      "activations/layer2_attention_weight_min": -29.53142547607422,
      "activations/layer3_attention_weight_max": 94.24601745605469,
      "activations/layer3_attention_weight_min": -95.86591339111328,
      "activations/layer4_attention_weight_max": 104.40401458740234,
      "activations/layer4_attention_weight_min": -111.568359375,
      "activations/layer5_attention_weight_max": 49.48710250854492,
      "activations/layer5_attention_weight_min": -62.42738342285156,
      "activations/layer6_attention_weight_max": 44.508094787597656,
      "activations/layer6_attention_weight_min": -49.759342193603516,
      "activations/layer7_attention_weight_max": 93.3515853881836,
      "activations/layer7_attention_weight_min": -105.88438415527344,
      "activations/layer8_attention_weight_max": 42.220890045166016,
      "activations/layer8_attention_weight_min": -45.613121032714844,
      "activations/layer9_attention_weight_max": 34.779380798339844,
      "activations/layer9_attention_weight_min": -35.32769012451172,
      "epoch": 18.58,
      "learning_rate": 3.049507575757575e-05,
      "loss": 2.7299,
      "step": 319750
    },
    {
      "activations/layer0_attention_weight_max": 15.666629791259766,
      "activations/layer0_attention_weight_min": -12.163238525390625,
      "activations/layer10_attention_weight_max": 41.53398513793945,
      "activations/layer10_attention_weight_min": -40.15269470214844,
      "activations/layer11_attention_weight_max": 37.533935546875,
      "activations/layer11_attention_weight_min": -38.38700866699219,
      "activations/layer12_attention_weight_max": 26.921510696411133,
      "activations/layer12_attention_weight_min": -27.018526077270508,
      "activations/layer13_attention_weight_max": 39.918983459472656,
      "activations/layer13_attention_weight_min": -36.67293167114258,
      "activations/layer14_attention_weight_max": 42.78271484375,
      "activations/layer14_attention_weight_min": -38.43617630004883,
      "activations/layer15_attention_weight_max": 40.112571716308594,
      "activations/layer15_attention_weight_min": -36.35469055175781,
      "activations/layer16_attention_weight_max": 30.22788429260254,
      "activations/layer16_attention_weight_min": -29.0218505859375,
      "activations/layer17_attention_weight_max": 28.914560317993164,
      "activations/layer17_attention_weight_min": -27.756853103637695,
      "activations/layer18_attention_weight_max": 31.381816864013672,
      "activations/layer18_attention_weight_min": -26.323177337646484,
      "activations/layer19_attention_weight_max": 37.924320220947266,
      "activations/layer19_attention_weight_min": -31.112844467163086,
      "activations/layer1_attention_weight_max": 16.42064094543457,
      "activations/layer1_attention_weight_min": -14.839987754821777,
      "activations/layer20_attention_weight_max": 30.02257537841797,
      "activations/layer20_attention_weight_min": -25.144712448120117,
      "activations/layer21_attention_weight_max": 31.80368995666504,
      "activations/layer21_attention_weight_min": -25.318553924560547,
      "activations/layer22_attention_weight_max": 45.052635192871094,
      "activations/layer22_attention_weight_min": -33.01009750366211,
      "activations/layer23_attention_weight_max": 35.24688720703125,
      "activations/layer23_attention_weight_min": -27.654592514038086,
      "activations/layer2_attention_weight_max": 31.113401412963867,
      "activations/layer2_attention_weight_min": -32.13754653930664,
      "activations/layer3_attention_weight_max": 94.45975494384766,
      "activations/layer3_attention_weight_min": -98.32979583740234,
      "activations/layer4_attention_weight_max": 101.7239761352539,
      "activations/layer4_attention_weight_min": -109.0543212890625,
      "activations/layer5_attention_weight_max": 48.765235900878906,
      "activations/layer5_attention_weight_min": -62.5102653503418,
      "activations/layer6_attention_weight_max": 43.714805603027344,
      "activations/layer6_attention_weight_min": -46.63343811035156,
      "activations/layer7_attention_weight_max": 102.10325622558594,
      "activations/layer7_attention_weight_min": -95.98162078857422,
      "activations/layer8_attention_weight_max": 46.28922653198242,
      "activations/layer8_attention_weight_min": -47.01875305175781,
      "activations/layer9_attention_weight_max": 35.203304290771484,
      "activations/layer9_attention_weight_min": -38.82444381713867,
      "epoch": 18.58,
      "learning_rate": 3.047613636363636e-05,
      "loss": 2.741,
      "step": 319800
    },
    {
      "activations/layer0_attention_weight_max": 15.708818435668945,
      "activations/layer0_attention_weight_min": -12.564682006835938,
      "activations/layer10_attention_weight_max": 33.91884231567383,
      "activations/layer10_attention_weight_min": -35.574676513671875,
      "activations/layer11_attention_weight_max": 32.76718521118164,
      "activations/layer11_attention_weight_min": -34.667572021484375,
      "activations/layer12_attention_weight_max": 27.203649520874023,
      "activations/layer12_attention_weight_min": -32.521453857421875,
      "activations/layer13_attention_weight_max": 38.51057815551758,
      "activations/layer13_attention_weight_min": -36.19736862182617,
      "activations/layer14_attention_weight_max": 42.13706588745117,
      "activations/layer14_attention_weight_min": -38.32729721069336,
      "activations/layer15_attention_weight_max": 38.46648025512695,
      "activations/layer15_attention_weight_min": -36.646034240722656,
      "activations/layer16_attention_weight_max": 30.19841194152832,
      "activations/layer16_attention_weight_min": -30.259557723999023,
      "activations/layer17_attention_weight_max": 26.793977737426758,
      "activations/layer17_attention_weight_min": -27.819337844848633,
      "activations/layer18_attention_weight_max": 29.930612564086914,
      "activations/layer18_attention_weight_min": -27.394933700561523,
      "activations/layer19_attention_weight_max": 30.53519630432129,
      "activations/layer19_attention_weight_min": -32.00539779663086,
      "activations/layer1_attention_weight_max": 16.91655731201172,
      "activations/layer1_attention_weight_min": -14.8062744140625,
      "activations/layer20_attention_weight_max": 26.894439697265625,
      "activations/layer20_attention_weight_min": -25.795225143432617,
      "activations/layer21_attention_weight_max": 25.257034301757812,
      "activations/layer21_attention_weight_min": -25.740453720092773,
      "activations/layer22_attention_weight_max": 41.115501403808594,
      "activations/layer22_attention_weight_min": -34.90403747558594,
      "activations/layer23_attention_weight_max": 28.386390686035156,
      "activations/layer23_attention_weight_min": -26.157690048217773,
      "activations/layer2_attention_weight_max": 30.02920150756836,
      "activations/layer2_attention_weight_min": -28.555143356323242,
      "activations/layer3_attention_weight_max": 94.46514892578125,
      "activations/layer3_attention_weight_min": -93.28742980957031,
      "activations/layer4_attention_weight_max": 105.35367584228516,
      "activations/layer4_attention_weight_min": -102.93785858154297,
      "activations/layer5_attention_weight_max": 49.44867706298828,
      "activations/layer5_attention_weight_min": -60.307525634765625,
      "activations/layer6_attention_weight_max": 43.25407028198242,
      "activations/layer6_attention_weight_min": -43.985679626464844,
      "activations/layer7_attention_weight_max": 94.10469818115234,
      "activations/layer7_attention_weight_min": -95.56330871582031,
      "activations/layer8_attention_weight_max": 43.29668045043945,
      "activations/layer8_attention_weight_min": -43.91150665283203,
      "activations/layer9_attention_weight_max": 30.421222686767578,
      "activations/layer9_attention_weight_min": -33.272518157958984,
      "epoch": 18.59,
      "learning_rate": 3.0457196969696968e-05,
      "loss": 2.7337,
      "step": 319850
    },
    {
      "activations/layer0_attention_weight_max": 16.509811401367188,
      "activations/layer0_attention_weight_min": -11.77868366241455,
      "activations/layer10_attention_weight_max": 45.9012336730957,
      "activations/layer10_attention_weight_min": -39.96011734008789,
      "activations/layer11_attention_weight_max": 44.49326705932617,
      "activations/layer11_attention_weight_min": -42.41886901855469,
      "activations/layer12_attention_weight_max": 29.153507232666016,
      "activations/layer12_attention_weight_min": -28.835548400878906,
      "activations/layer13_attention_weight_max": 41.29581832885742,
      "activations/layer13_attention_weight_min": -37.7923698425293,
      "activations/layer14_attention_weight_max": 45.50339126586914,
      "activations/layer14_attention_weight_min": -38.692806243896484,
      "activations/layer15_attention_weight_max": 39.70737075805664,
      "activations/layer15_attention_weight_min": -37.65784454345703,
      "activations/layer16_attention_weight_max": 30.478376388549805,
      "activations/layer16_attention_weight_min": -27.056310653686523,
      "activations/layer17_attention_weight_max": 32.158321380615234,
      "activations/layer17_attention_weight_min": -26.030786514282227,
      "activations/layer18_attention_weight_max": 35.715023040771484,
      "activations/layer18_attention_weight_min": -26.702022552490234,
      "activations/layer19_attention_weight_max": 37.50948715209961,
      "activations/layer19_attention_weight_min": -30.674697875976562,
      "activations/layer1_attention_weight_max": 17.608928680419922,
      "activations/layer1_attention_weight_min": -14.638998031616211,
      "activations/layer20_attention_weight_max": 31.270151138305664,
      "activations/layer20_attention_weight_min": -24.760353088378906,
      "activations/layer21_attention_weight_max": 29.867538452148438,
      "activations/layer21_attention_weight_min": -26.957042694091797,
      "activations/layer22_attention_weight_max": 49.56450271606445,
      "activations/layer22_attention_weight_min": -32.93001937866211,
      "activations/layer23_attention_weight_max": 38.36913299560547,
      "activations/layer23_attention_weight_min": -27.878265380859375,
      "activations/layer2_attention_weight_max": 29.68256378173828,
      "activations/layer2_attention_weight_min": -29.158226013183594,
      "activations/layer3_attention_weight_max": 95.18585968017578,
      "activations/layer3_attention_weight_min": -99.55365753173828,
      "activations/layer4_attention_weight_max": 108.65068817138672,
      "activations/layer4_attention_weight_min": -108.91666412353516,
      "activations/layer5_attention_weight_max": 60.052520751953125,
      "activations/layer5_attention_weight_min": -60.6031494140625,
      "activations/layer6_attention_weight_max": 46.742103576660156,
      "activations/layer6_attention_weight_min": -47.54360580444336,
      "activations/layer7_attention_weight_max": 111.47248840332031,
      "activations/layer7_attention_weight_min": -101.85371398925781,
      "activations/layer8_attention_weight_max": 48.26524353027344,
      "activations/layer8_attention_weight_min": -46.79496765136719,
      "activations/layer9_attention_weight_max": 34.919410705566406,
      "activations/layer9_attention_weight_min": -38.92958068847656,
      "epoch": 18.59,
      "learning_rate": 3.0438257575757573e-05,
      "loss": 2.7299,
      "step": 319900
    },
    {
      "activations/layer0_attention_weight_max": 16.225095748901367,
      "activations/layer0_attention_weight_min": -13.328126907348633,
      "activations/layer10_attention_weight_max": 33.81193923950195,
      "activations/layer10_attention_weight_min": -33.76569366455078,
      "activations/layer11_attention_weight_max": 36.20592498779297,
      "activations/layer11_attention_weight_min": -33.50653076171875,
      "activations/layer12_attention_weight_max": 25.77848243713379,
      "activations/layer12_attention_weight_min": -27.135095596313477,
      "activations/layer13_attention_weight_max": 36.75594711303711,
      "activations/layer13_attention_weight_min": -36.820796966552734,
      "activations/layer14_attention_weight_max": 44.58375549316406,
      "activations/layer14_attention_weight_min": -38.02273178100586,
      "activations/layer15_attention_weight_max": 39.724327087402344,
      "activations/layer15_attention_weight_min": -36.430564880371094,
      "activations/layer16_attention_weight_max": 29.103559494018555,
      "activations/layer16_attention_weight_min": -27.303190231323242,
      "activations/layer17_attention_weight_max": 27.76044273376465,
      "activations/layer17_attention_weight_min": -25.37841796875,
      "activations/layer18_attention_weight_max": 28.153409957885742,
      "activations/layer18_attention_weight_min": -25.183475494384766,
      "activations/layer19_attention_weight_max": 35.09978103637695,
      "activations/layer19_attention_weight_min": -33.569828033447266,
      "activations/layer1_attention_weight_max": 16.685794830322266,
      "activations/layer1_attention_weight_min": -13.18499755859375,
      "activations/layer20_attention_weight_max": 27.964853286743164,
      "activations/layer20_attention_weight_min": -25.84133529663086,
      "activations/layer21_attention_weight_max": 28.873918533325195,
      "activations/layer21_attention_weight_min": -26.336973190307617,
      "activations/layer22_attention_weight_max": 38.156349182128906,
      "activations/layer22_attention_weight_min": -30.274303436279297,
      "activations/layer23_attention_weight_max": 32.718936920166016,
      "activations/layer23_attention_weight_min": -26.754301071166992,
      "activations/layer2_attention_weight_max": 32.10601043701172,
      "activations/layer2_attention_weight_min": -29.571937561035156,
      "activations/layer3_attention_weight_max": 96.6008071899414,
      "activations/layer3_attention_weight_min": -93.4062271118164,
      "activations/layer4_attention_weight_max": 107.08967590332031,
      "activations/layer4_attention_weight_min": -106.22191619873047,
      "activations/layer5_attention_weight_max": 51.56473922729492,
      "activations/layer5_attention_weight_min": -58.6956901550293,
      "activations/layer6_attention_weight_max": 43.2909049987793,
      "activations/layer6_attention_weight_min": -46.75865173339844,
      "activations/layer7_attention_weight_max": 104.13482666015625,
      "activations/layer7_attention_weight_min": -96.57400512695312,
      "activations/layer8_attention_weight_max": 43.297996520996094,
      "activations/layer8_attention_weight_min": -43.92892837524414,
      "activations/layer9_attention_weight_max": 33.247318267822266,
      "activations/layer9_attention_weight_min": -34.21562957763672,
      "epoch": 18.59,
      "learning_rate": 3.041931818181818e-05,
      "loss": 2.7408,
      "step": 319950
    },
    {
      "activations/layer0_attention_weight_max": 16.129074096679688,
      "activations/layer0_attention_weight_min": -12.867081642150879,
      "activations/layer10_attention_weight_max": 33.429725646972656,
      "activations/layer10_attention_weight_min": -33.08394241333008,
      "activations/layer11_attention_weight_max": 32.35875701904297,
      "activations/layer11_attention_weight_min": -33.456077575683594,
      "activations/layer12_attention_weight_max": 27.235570907592773,
      "activations/layer12_attention_weight_min": -26.69495964050293,
      "activations/layer13_attention_weight_max": 40.477108001708984,
      "activations/layer13_attention_weight_min": -36.48844909667969,
      "activations/layer14_attention_weight_max": 42.76624298095703,
      "activations/layer14_attention_weight_min": -38.06541061401367,
      "activations/layer15_attention_weight_max": 40.33464431762695,
      "activations/layer15_attention_weight_min": -37.970375061035156,
      "activations/layer16_attention_weight_max": 30.600671768188477,
      "activations/layer16_attention_weight_min": -30.05174446105957,
      "activations/layer17_attention_weight_max": 32.089717864990234,
      "activations/layer17_attention_weight_min": -26.92484474182129,
      "activations/layer18_attention_weight_max": 31.949373245239258,
      "activations/layer18_attention_weight_min": -27.052188873291016,
      "activations/layer19_attention_weight_max": 33.1113395690918,
      "activations/layer19_attention_weight_min": -32.99407196044922,
      "activations/layer1_attention_weight_max": 17.04922103881836,
      "activations/layer1_attention_weight_min": -12.763914108276367,
      "activations/layer20_attention_weight_max": 29.789400100708008,
      "activations/layer20_attention_weight_min": -27.709115982055664,
      "activations/layer21_attention_weight_max": 31.11064910888672,
      "activations/layer21_attention_weight_min": -27.31170654296875,
      "activations/layer22_attention_weight_max": 42.83927536010742,
      "activations/layer22_attention_weight_min": -34.336143493652344,
      "activations/layer23_attention_weight_max": 33.29463195800781,
      "activations/layer23_attention_weight_min": -26.176326751708984,
      "activations/layer2_attention_weight_max": 29.339256286621094,
      "activations/layer2_attention_weight_min": -28.461750030517578,
      "activations/layer3_attention_weight_max": 95.33232116699219,
      "activations/layer3_attention_weight_min": -91.91234588623047,
      "activations/layer4_attention_weight_max": 103.86433410644531,
      "activations/layer4_attention_weight_min": -104.89705657958984,
      "activations/layer5_attention_weight_max": 48.19464111328125,
      "activations/layer5_attention_weight_min": -61.785972595214844,
      "activations/layer6_attention_weight_max": 47.124332427978516,
      "activations/layer6_attention_weight_min": -44.243309020996094,
      "activations/layer7_attention_weight_max": 89.62783813476562,
      "activations/layer7_attention_weight_min": -95.78866577148438,
      "activations/layer8_attention_weight_max": 38.546356201171875,
      "activations/layer8_attention_weight_min": -43.04997634887695,
      "activations/layer9_attention_weight_max": 30.83343505859375,
      "activations/layer9_attention_weight_min": -31.601255416870117,
      "epoch": 18.59,
      "learning_rate": 3.0400378787878786e-05,
      "loss": 2.7369,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.4418,
      "eval_samples_per_second": 508.662,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.4418,
      "eval_openwebtext_samples_per_second": 508.662,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9784,
      "eval_wikitext_samples_per_second": 230.491,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_lambada_loss": 2.330078125,
      "eval_lambada_ppl": 10.278744528592322,
      "eval_lambada_runtime": 9.6301,
      "eval_lambada_samples_per_second": 505.603,
      "step": 320000
    },
    {
      "activations/layer0_attention_weight_max": 16.39327621459961,
      "activations/layer0_attention_weight_min": -12.611815452575684,
      "activations/layer10_attention_weight_max": 33.09366989135742,
      "activations/layer10_attention_weight_min": -35.619842529296875,
      "activations/layer11_attention_weight_max": 33.09523010253906,
      "activations/layer11_attention_weight_min": -35.54222869873047,
      "activations/layer12_attention_weight_max": 29.880958557128906,
      "activations/layer12_attention_weight_min": -28.01496696472168,
      "activations/layer13_attention_weight_max": 42.31553268432617,
      "activations/layer13_attention_weight_min": -36.501129150390625,
      "activations/layer14_attention_weight_max": 44.97751998901367,
      "activations/layer14_attention_weight_min": -37.53945541381836,
      "activations/layer15_attention_weight_max": 42.60557556152344,
      "activations/layer15_attention_weight_min": -35.49750518798828,
      "activations/layer16_attention_weight_max": 33.2744026184082,
      "activations/layer16_attention_weight_min": -29.86989974975586,
      "activations/layer17_attention_weight_max": 36.33931350708008,
      "activations/layer17_attention_weight_min": -28.344697952270508,
      "activations/layer18_attention_weight_max": 34.729862213134766,
      "activations/layer18_attention_weight_min": -25.337852478027344,
      "activations/layer19_attention_weight_max": 36.46603012084961,
      "activations/layer19_attention_weight_min": -32.912445068359375,
      "activations/layer1_attention_weight_max": 15.541205406188965,
      "activations/layer1_attention_weight_min": -14.324223518371582,
      "activations/layer20_attention_weight_max": 30.63555908203125,
      "activations/layer20_attention_weight_min": -25.665172576904297,
      "activations/layer21_attention_weight_max": 31.218917846679688,
      "activations/layer21_attention_weight_min": -25.685571670532227,
      "activations/layer22_attention_weight_max": 46.349552154541016,
      "activations/layer22_attention_weight_min": -31.592004776000977,
      "activations/layer23_attention_weight_max": 37.70956802368164,
      "activations/layer23_attention_weight_min": -26.955570220947266,
      "activations/layer2_attention_weight_max": 31.703922271728516,
      "activations/layer2_attention_weight_min": -30.149351119995117,
      "activations/layer3_attention_weight_max": 93.19884490966797,
      "activations/layer3_attention_weight_min": -93.3743667602539,
      "activations/layer4_attention_weight_max": 102.26787567138672,
      "activations/layer4_attention_weight_min": -107.5400390625,
      "activations/layer5_attention_weight_max": 51.583187103271484,
      "activations/layer5_attention_weight_min": -63.98479461669922,
      "activations/layer6_attention_weight_max": 42.981971740722656,
      "activations/layer6_attention_weight_min": -45.44074630737305,
      "activations/layer7_attention_weight_max": 88.37503051757812,
      "activations/layer7_attention_weight_min": -93.83472442626953,
      "activations/layer8_attention_weight_max": 40.04943084716797,
      "activations/layer8_attention_weight_min": -43.73317337036133,
      "activations/layer9_attention_weight_max": 33.37416458129883,
      "activations/layer9_attention_weight_min": -36.920143127441406,
      "epoch": 18.6,
      "learning_rate": 3.038143939393939e-05,
      "loss": 2.7323,
      "step": 320050
    },
    {
      "activations/layer0_attention_weight_max": 15.82007884979248,
      "activations/layer0_attention_weight_min": -11.992816925048828,
      "activations/layer10_attention_weight_max": 33.17567825317383,
      "activations/layer10_attention_weight_min": -36.07685089111328,
      "activations/layer11_attention_weight_max": 32.48158264160156,
      "activations/layer11_attention_weight_min": -35.65577697753906,
      "activations/layer12_attention_weight_max": 27.537302017211914,
      "activations/layer12_attention_weight_min": -26.8895320892334,
      "activations/layer13_attention_weight_max": 39.45759201049805,
      "activations/layer13_attention_weight_min": -35.2382698059082,
      "activations/layer14_attention_weight_max": 42.82036209106445,
      "activations/layer14_attention_weight_min": -38.74333190917969,
      "activations/layer15_attention_weight_max": 38.8371696472168,
      "activations/layer15_attention_weight_min": -34.977203369140625,
      "activations/layer16_attention_weight_max": 30.367704391479492,
      "activations/layer16_attention_weight_min": -28.81201171875,
      "activations/layer17_attention_weight_max": 30.230852127075195,
      "activations/layer17_attention_weight_min": -25.679302215576172,
      "activations/layer18_attention_weight_max": 37.347537994384766,
      "activations/layer18_attention_weight_min": -25.772899627685547,
      "activations/layer19_attention_weight_max": 34.09623336791992,
      "activations/layer19_attention_weight_min": -30.646520614624023,
      "activations/layer1_attention_weight_max": 16.70575523376465,
      "activations/layer1_attention_weight_min": -14.22899055480957,
      "activations/layer20_attention_weight_max": 29.888160705566406,
      "activations/layer20_attention_weight_min": -24.99159812927246,
      "activations/layer21_attention_weight_max": 29.814977645874023,
      "activations/layer21_attention_weight_min": -26.70193099975586,
      "activations/layer22_attention_weight_max": 44.29444122314453,
      "activations/layer22_attention_weight_min": -34.64352798461914,
      "activations/layer23_attention_weight_max": 35.21027755737305,
      "activations/layer23_attention_weight_min": -26.74049186706543,
      "activations/layer2_attention_weight_max": 30.37750816345215,
      "activations/layer2_attention_weight_min": -30.732824325561523,
      "activations/layer3_attention_weight_max": 96.56199645996094,
      "activations/layer3_attention_weight_min": -101.39640045166016,
      "activations/layer4_attention_weight_max": 107.11315155029297,
      "activations/layer4_attention_weight_min": -112.42127990722656,
      "activations/layer5_attention_weight_max": 52.685462951660156,
      "activations/layer5_attention_weight_min": -66.95903778076172,
      "activations/layer6_attention_weight_max": 44.733238220214844,
      "activations/layer6_attention_weight_min": -47.37064743041992,
      "activations/layer7_attention_weight_max": 94.29474639892578,
      "activations/layer7_attention_weight_min": -99.70551300048828,
      "activations/layer8_attention_weight_max": 41.94450759887695,
      "activations/layer8_attention_weight_min": -44.8331298828125,
      "activations/layer9_attention_weight_max": 33.28451156616211,
      "activations/layer9_attention_weight_min": -36.20004653930664,
      "epoch": 18.6,
      "learning_rate": 3.0362499999999996e-05,
      "loss": 2.724,
      "step": 320100
    },
    {
      "activations/layer0_attention_weight_max": 16.000286102294922,
      "activations/layer0_attention_weight_min": -12.043184280395508,
      "activations/layer10_attention_weight_max": 36.138702392578125,
      "activations/layer10_attention_weight_min": -36.47476577758789,
      "activations/layer11_attention_weight_max": 36.813682556152344,
      "activations/layer11_attention_weight_min": -35.01953125,
      "activations/layer12_attention_weight_max": 29.333637237548828,
      "activations/layer12_attention_weight_min": -28.054513931274414,
      "activations/layer13_attention_weight_max": 42.7121467590332,
      "activations/layer13_attention_weight_min": -36.53623962402344,
      "activations/layer14_attention_weight_max": 47.4617805480957,
      "activations/layer14_attention_weight_min": -38.42043685913086,
      "activations/layer15_attention_weight_max": 41.908931732177734,
      "activations/layer15_attention_weight_min": -37.10022735595703,
      "activations/layer16_attention_weight_max": 30.02720832824707,
      "activations/layer16_attention_weight_min": -28.846717834472656,
      "activations/layer17_attention_weight_max": 30.200387954711914,
      "activations/layer17_attention_weight_min": -26.301349639892578,
      "activations/layer18_attention_weight_max": 34.31352996826172,
      "activations/layer18_attention_weight_min": -30.393531799316406,
      "activations/layer19_attention_weight_max": 41.88335037231445,
      "activations/layer19_attention_weight_min": -31.257339477539062,
      "activations/layer1_attention_weight_max": 15.844833374023438,
      "activations/layer1_attention_weight_min": -13.372262001037598,
      "activations/layer20_attention_weight_max": 33.52785873413086,
      "activations/layer20_attention_weight_min": -27.734895706176758,
      "activations/layer21_attention_weight_max": 33.860382080078125,
      "activations/layer21_attention_weight_min": -26.67570686340332,
      "activations/layer22_attention_weight_max": 45.284996032714844,
      "activations/layer22_attention_weight_min": -34.7016716003418,
      "activations/layer23_attention_weight_max": 36.000205993652344,
      "activations/layer23_attention_weight_min": -29.528865814208984,
      "activations/layer2_attention_weight_max": 31.58791160583496,
      "activations/layer2_attention_weight_min": -30.364246368408203,
      "activations/layer3_attention_weight_max": 94.6583251953125,
      "activations/layer3_attention_weight_min": -98.345458984375,
      "activations/layer4_attention_weight_max": 104.5382308959961,
      "activations/layer4_attention_weight_min": -107.4919204711914,
      "activations/layer5_attention_weight_max": 49.8952751159668,
      "activations/layer5_attention_weight_min": -63.23149108886719,
      "activations/layer6_attention_weight_max": 47.610572814941406,
      "activations/layer6_attention_weight_min": -47.441871643066406,
      "activations/layer7_attention_weight_max": 98.38525390625,
      "activations/layer7_attention_weight_min": -98.32447814941406,
      "activations/layer8_attention_weight_max": 44.65595626831055,
      "activations/layer8_attention_weight_min": -46.55424118041992,
      "activations/layer9_attention_weight_max": 34.078468322753906,
      "activations/layer9_attention_weight_min": -36.0516242980957,
      "epoch": 18.6,
      "learning_rate": 3.0343560606060604e-05,
      "loss": 2.737,
      "step": 320150
    },
    {
      "activations/layer0_attention_weight_max": 17.74080467224121,
      "activations/layer0_attention_weight_min": -12.09772777557373,
      "activations/layer10_attention_weight_max": 33.195762634277344,
      "activations/layer10_attention_weight_min": -34.78013610839844,
      "activations/layer11_attention_weight_max": 32.69546890258789,
      "activations/layer11_attention_weight_min": -34.077903747558594,
      "activations/layer12_attention_weight_max": 27.497865676879883,
      "activations/layer12_attention_weight_min": -28.01211929321289,
      "activations/layer13_attention_weight_max": 39.602291107177734,
      "activations/layer13_attention_weight_min": -36.493656158447266,
      "activations/layer14_attention_weight_max": 40.33022689819336,
      "activations/layer14_attention_weight_min": -38.06008529663086,
      "activations/layer15_attention_weight_max": 39.63662338256836,
      "activations/layer15_attention_weight_min": -36.210933685302734,
      "activations/layer16_attention_weight_max": 30.084060668945312,
      "activations/layer16_attention_weight_min": -27.397663116455078,
      "activations/layer17_attention_weight_max": 30.744049072265625,
      "activations/layer17_attention_weight_min": -26.717681884765625,
      "activations/layer18_attention_weight_max": 33.64826583862305,
      "activations/layer18_attention_weight_min": -29.13848304748535,
      "activations/layer19_attention_weight_max": 37.182769775390625,
      "activations/layer19_attention_weight_min": -31.627531051635742,
      "activations/layer1_attention_weight_max": 15.462852478027344,
      "activations/layer1_attention_weight_min": -13.3768949508667,
      "activations/layer20_attention_weight_max": 32.42361068725586,
      "activations/layer20_attention_weight_min": -26.101848602294922,
      "activations/layer21_attention_weight_max": 32.47875213623047,
      "activations/layer21_attention_weight_min": -26.08197021484375,
      "activations/layer22_attention_weight_max": 45.026336669921875,
      "activations/layer22_attention_weight_min": -32.96296691894531,
      "activations/layer23_attention_weight_max": 35.57776641845703,
      "activations/layer23_attention_weight_min": -28.117780685424805,
      "activations/layer2_attention_weight_max": 30.13488006591797,
      "activations/layer2_attention_weight_min": -29.165340423583984,
      "activations/layer3_attention_weight_max": 95.04560089111328,
      "activations/layer3_attention_weight_min": -94.99041748046875,
      "activations/layer4_attention_weight_max": 108.90819549560547,
      "activations/layer4_attention_weight_min": -111.51937103271484,
      "activations/layer5_attention_weight_max": 52.48960876464844,
      "activations/layer5_attention_weight_min": -62.87781524658203,
      "activations/layer6_attention_weight_max": 45.54105758666992,
      "activations/layer6_attention_weight_min": -47.869991302490234,
      "activations/layer7_attention_weight_max": 90.95694732666016,
      "activations/layer7_attention_weight_min": -93.50023651123047,
      "activations/layer8_attention_weight_max": 39.70763397216797,
      "activations/layer8_attention_weight_min": -43.38566589355469,
      "activations/layer9_attention_weight_max": 30.560134887695312,
      "activations/layer9_attention_weight_min": -35.20996856689453,
      "epoch": 18.61,
      "learning_rate": 3.032462121212121e-05,
      "loss": 2.719,
      "step": 320200
    },
    {
      "activations/layer0_attention_weight_max": 16.716489791870117,
      "activations/layer0_attention_weight_min": -11.869384765625,
      "activations/layer10_attention_weight_max": 33.393348693847656,
      "activations/layer10_attention_weight_min": -35.686912536621094,
      "activations/layer11_attention_weight_max": 35.38822555541992,
      "activations/layer11_attention_weight_min": -34.98923110961914,
      "activations/layer12_attention_weight_max": 26.299596786499023,
      "activations/layer12_attention_weight_min": -31.219179153442383,
      "activations/layer13_attention_weight_max": 36.194610595703125,
      "activations/layer13_attention_weight_min": -41.1825065612793,
      "activations/layer14_attention_weight_max": 44.22945022583008,
      "activations/layer14_attention_weight_min": -44.87215805053711,
      "activations/layer15_attention_weight_max": 40.42549514770508,
      "activations/layer15_attention_weight_min": -43.55613708496094,
      "activations/layer16_attention_weight_max": 30.932050704956055,
      "activations/layer16_attention_weight_min": -32.84779739379883,
      "activations/layer17_attention_weight_max": 28.88691520690918,
      "activations/layer17_attention_weight_min": -28.41638946533203,
      "activations/layer18_attention_weight_max": 30.305063247680664,
      "activations/layer18_attention_weight_min": -28.686073303222656,
      "activations/layer19_attention_weight_max": 32.7989616394043,
      "activations/layer19_attention_weight_min": -31.31511878967285,
      "activations/layer1_attention_weight_max": 16.261404037475586,
      "activations/layer1_attention_weight_min": -14.56419849395752,
      "activations/layer20_attention_weight_max": 26.11363410949707,
      "activations/layer20_attention_weight_min": -24.220882415771484,
      "activations/layer21_attention_weight_max": 27.012662887573242,
      "activations/layer21_attention_weight_min": -24.81120491027832,
      "activations/layer22_attention_weight_max": 40.81422805786133,
      "activations/layer22_attention_weight_min": -32.109718322753906,
      "activations/layer23_attention_weight_max": 29.521209716796875,
      "activations/layer23_attention_weight_min": -26.220388412475586,
      "activations/layer2_attention_weight_max": 30.13617706298828,
      "activations/layer2_attention_weight_min": -29.992862701416016,
      "activations/layer3_attention_weight_max": 94.29381561279297,
      "activations/layer3_attention_weight_min": -99.34130859375,
      "activations/layer4_attention_weight_max": 99.90792083740234,
      "activations/layer4_attention_weight_min": -111.28746795654297,
      "activations/layer5_attention_weight_max": 49.091896057128906,
      "activations/layer5_attention_weight_min": -62.594688415527344,
      "activations/layer6_attention_weight_max": 43.20530700683594,
      "activations/layer6_attention_weight_min": -48.99366760253906,
      "activations/layer7_attention_weight_max": 88.78418731689453,
      "activations/layer7_attention_weight_min": -100.24220275878906,
      "activations/layer8_attention_weight_max": 40.503334045410156,
      "activations/layer8_attention_weight_min": -43.38743591308594,
      "activations/layer9_attention_weight_max": 31.492752075195312,
      "activations/layer9_attention_weight_min": -34.618736267089844,
      "epoch": 18.61,
      "learning_rate": 3.0305681818181817e-05,
      "loss": 2.7159,
      "step": 320250
    },
    {
      "activations/layer0_attention_weight_max": 16.170455932617188,
      "activations/layer0_attention_weight_min": -12.374283790588379,
      "activations/layer10_attention_weight_max": 34.33954620361328,
      "activations/layer10_attention_weight_min": -33.33405685424805,
      "activations/layer11_attention_weight_max": 34.77241516113281,
      "activations/layer11_attention_weight_min": -35.767616271972656,
      "activations/layer12_attention_weight_max": 29.239070892333984,
      "activations/layer12_attention_weight_min": -28.374860763549805,
      "activations/layer13_attention_weight_max": 41.60021209716797,
      "activations/layer13_attention_weight_min": -37.73945236206055,
      "activations/layer14_attention_weight_max": 49.449501037597656,
      "activations/layer14_attention_weight_min": -40.966026306152344,
      "activations/layer15_attention_weight_max": 43.6861457824707,
      "activations/layer15_attention_weight_min": -40.44343185424805,
      "activations/layer16_attention_weight_max": 32.03531265258789,
      "activations/layer16_attention_weight_min": -29.486919403076172,
      "activations/layer17_attention_weight_max": 31.814321517944336,
      "activations/layer17_attention_weight_min": -27.86666488647461,
      "activations/layer18_attention_weight_max": 36.869937896728516,
      "activations/layer18_attention_weight_min": -27.06745719909668,
      "activations/layer19_attention_weight_max": 38.81826400756836,
      "activations/layer19_attention_weight_min": -29.70918083190918,
      "activations/layer1_attention_weight_max": 16.190500259399414,
      "activations/layer1_attention_weight_min": -12.781081199645996,
      "activations/layer20_attention_weight_max": 30.856489181518555,
      "activations/layer20_attention_weight_min": -25.894996643066406,
      "activations/layer21_attention_weight_max": 29.143705368041992,
      "activations/layer21_attention_weight_min": -24.981496810913086,
      "activations/layer22_attention_weight_max": 44.32286071777344,
      "activations/layer22_attention_weight_min": -31.507991790771484,
      "activations/layer23_attention_weight_max": 36.481563568115234,
      "activations/layer23_attention_weight_min": -25.680809020996094,
      "activations/layer2_attention_weight_max": 29.15150260925293,
      "activations/layer2_attention_weight_min": -27.969877243041992,
      "activations/layer3_attention_weight_max": 91.8115005493164,
      "activations/layer3_attention_weight_min": -93.21930694580078,
      "activations/layer4_attention_weight_max": 105.61671447753906,
      "activations/layer4_attention_weight_min": -106.96473693847656,
      "activations/layer5_attention_weight_max": 49.24172592163086,
      "activations/layer5_attention_weight_min": -63.156044006347656,
      "activations/layer6_attention_weight_max": 43.35018539428711,
      "activations/layer6_attention_weight_min": -44.98814010620117,
      "activations/layer7_attention_weight_max": 90.62804412841797,
      "activations/layer7_attention_weight_min": -91.73812866210938,
      "activations/layer8_attention_weight_max": 41.35601043701172,
      "activations/layer8_attention_weight_min": -42.44853591918945,
      "activations/layer9_attention_weight_max": 33.859066009521484,
      "activations/layer9_attention_weight_min": -33.2135124206543,
      "epoch": 18.61,
      "learning_rate": 3.028674242424242e-05,
      "loss": 2.7425,
      "step": 320300
    },
    {
      "activations/layer0_attention_weight_max": 15.948101997375488,
      "activations/layer0_attention_weight_min": -12.244551658630371,
      "activations/layer10_attention_weight_max": 31.852397918701172,
      "activations/layer10_attention_weight_min": -31.600431442260742,
      "activations/layer11_attention_weight_max": 32.333648681640625,
      "activations/layer11_attention_weight_min": -32.64830017089844,
      "activations/layer12_attention_weight_max": 27.931583404541016,
      "activations/layer12_attention_weight_min": -25.5950870513916,
      "activations/layer13_attention_weight_max": 40.7578010559082,
      "activations/layer13_attention_weight_min": -34.130043029785156,
      "activations/layer14_attention_weight_max": 43.46162414550781,
      "activations/layer14_attention_weight_min": -37.36157989501953,
      "activations/layer15_attention_weight_max": 38.34454345703125,
      "activations/layer15_attention_weight_min": -35.422733306884766,
      "activations/layer16_attention_weight_max": 29.29193878173828,
      "activations/layer16_attention_weight_min": -27.699634552001953,
      "activations/layer17_attention_weight_max": 28.63786506652832,
      "activations/layer17_attention_weight_min": -26.74167823791504,
      "activations/layer18_attention_weight_max": 32.38286590576172,
      "activations/layer18_attention_weight_min": -25.474552154541016,
      "activations/layer19_attention_weight_max": 30.798263549804688,
      "activations/layer19_attention_weight_min": -30.882476806640625,
      "activations/layer1_attention_weight_max": 17.407835006713867,
      "activations/layer1_attention_weight_min": -13.06920337677002,
      "activations/layer20_attention_weight_max": 27.548572540283203,
      "activations/layer20_attention_weight_min": -25.20599937438965,
      "activations/layer21_attention_weight_max": 27.373870849609375,
      "activations/layer21_attention_weight_min": -26.88909912109375,
      "activations/layer22_attention_weight_max": 39.39432144165039,
      "activations/layer22_attention_weight_min": -32.13197326660156,
      "activations/layer23_attention_weight_max": 34.28670120239258,
      "activations/layer23_attention_weight_min": -25.988990783691406,
      "activations/layer2_attention_weight_max": 28.966108322143555,
      "activations/layer2_attention_weight_min": -28.777164459228516,
      "activations/layer3_attention_weight_max": 91.47857666015625,
      "activations/layer3_attention_weight_min": -94.49445343017578,
      "activations/layer4_attention_weight_max": 96.3944320678711,
      "activations/layer4_attention_weight_min": -102.84111785888672,
      "activations/layer5_attention_weight_max": 46.73385238647461,
      "activations/layer5_attention_weight_min": -61.746002197265625,
      "activations/layer6_attention_weight_max": 41.21660232543945,
      "activations/layer6_attention_weight_min": -43.69834518432617,
      "activations/layer7_attention_weight_max": 85.48829650878906,
      "activations/layer7_attention_weight_min": -90.6904067993164,
      "activations/layer8_attention_weight_max": 38.6800422668457,
      "activations/layer8_attention_weight_min": -41.1089973449707,
      "activations/layer9_attention_weight_max": 30.342130661010742,
      "activations/layer9_attention_weight_min": -34.22355270385742,
      "epoch": 18.61,
      "learning_rate": 3.0267803030303027e-05,
      "loss": 2.7171,
      "step": 320350
    },
    {
      "activations/layer0_attention_weight_max": 16.89307403564453,
      "activations/layer0_attention_weight_min": -12.604288101196289,
      "activations/layer10_attention_weight_max": 36.74079132080078,
      "activations/layer10_attention_weight_min": -36.58756637573242,
      "activations/layer11_attention_weight_max": 35.89514923095703,
      "activations/layer11_attention_weight_min": -36.4897346496582,
      "activations/layer12_attention_weight_max": 26.722511291503906,
      "activations/layer12_attention_weight_min": -28.424577713012695,
      "activations/layer13_attention_weight_max": 41.494720458984375,
      "activations/layer13_attention_weight_min": -40.57074737548828,
      "activations/layer14_attention_weight_max": 43.78651809692383,
      "activations/layer14_attention_weight_min": -42.36613082885742,
      "activations/layer15_attention_weight_max": 39.889739990234375,
      "activations/layer15_attention_weight_min": -38.29195022583008,
      "activations/layer16_attention_weight_max": 30.568614959716797,
      "activations/layer16_attention_weight_min": -28.620376586914062,
      "activations/layer17_attention_weight_max": 32.61723709106445,
      "activations/layer17_attention_weight_min": -26.171714782714844,
      "activations/layer18_attention_weight_max": 30.660152435302734,
      "activations/layer18_attention_weight_min": -25.54397201538086,
      "activations/layer19_attention_weight_max": 35.71796798706055,
      "activations/layer19_attention_weight_min": -31.46439552307129,
      "activations/layer1_attention_weight_max": 15.666790962219238,
      "activations/layer1_attention_weight_min": -13.067048072814941,
      "activations/layer20_attention_weight_max": 26.948156356811523,
      "activations/layer20_attention_weight_min": -25.428804397583008,
      "activations/layer21_attention_weight_max": 27.919647216796875,
      "activations/layer21_attention_weight_min": -25.65866470336914,
      "activations/layer22_attention_weight_max": 41.201255798339844,
      "activations/layer22_attention_weight_min": -34.51597595214844,
      "activations/layer23_attention_weight_max": 34.02658462524414,
      "activations/layer23_attention_weight_min": -27.863689422607422,
      "activations/layer2_attention_weight_max": 31.782072067260742,
      "activations/layer2_attention_weight_min": -28.960479736328125,
      "activations/layer3_attention_weight_max": 94.7693099975586,
      "activations/layer3_attention_weight_min": -99.64897918701172,
      "activations/layer4_attention_weight_max": 106.80266571044922,
      "activations/layer4_attention_weight_min": -112.77589416503906,
      "activations/layer5_attention_weight_max": 51.17523956298828,
      "activations/layer5_attention_weight_min": -63.250396728515625,
      "activations/layer6_attention_weight_max": 46.17323303222656,
      "activations/layer6_attention_weight_min": -50.84397506713867,
      "activations/layer7_attention_weight_max": 92.67221069335938,
      "activations/layer7_attention_weight_min": -106.96642303466797,
      "activations/layer8_attention_weight_max": 42.65351867675781,
      "activations/layer8_attention_weight_min": -45.97233581542969,
      "activations/layer9_attention_weight_max": 33.23577880859375,
      "activations/layer9_attention_weight_min": -35.85773849487305,
      "epoch": 18.62,
      "learning_rate": 3.0248863636363632e-05,
      "loss": 2.7237,
      "step": 320400
    },
    {
      "activations/layer0_attention_weight_max": 16.386486053466797,
      "activations/layer0_attention_weight_min": -11.882953643798828,
      "activations/layer10_attention_weight_max": 46.35824966430664,
      "activations/layer10_attention_weight_min": -38.34144592285156,
      "activations/layer11_attention_weight_max": 47.11261749267578,
      "activations/layer11_attention_weight_min": -43.13318634033203,
      "activations/layer12_attention_weight_max": 26.910486221313477,
      "activations/layer12_attention_weight_min": -28.480680465698242,
      "activations/layer13_attention_weight_max": 39.31612777709961,
      "activations/layer13_attention_weight_min": -38.091094970703125,
      "activations/layer14_attention_weight_max": 44.888092041015625,
      "activations/layer14_attention_weight_min": -41.5775260925293,
      "activations/layer15_attention_weight_max": 41.68540573120117,
      "activations/layer15_attention_weight_min": -37.80703353881836,
      "activations/layer16_attention_weight_max": 29.828386306762695,
      "activations/layer16_attention_weight_min": -29.240964889526367,
      "activations/layer17_attention_weight_max": 27.311330795288086,
      "activations/layer17_attention_weight_min": -27.5767879486084,
      "activations/layer18_attention_weight_max": 30.2761173248291,
      "activations/layer18_attention_weight_min": -25.117286682128906,
      "activations/layer19_attention_weight_max": 31.879995346069336,
      "activations/layer19_attention_weight_min": -31.71944236755371,
      "activations/layer1_attention_weight_max": 16.243614196777344,
      "activations/layer1_attention_weight_min": -14.493786811828613,
      "activations/layer20_attention_weight_max": 26.147560119628906,
      "activations/layer20_attention_weight_min": -25.086702346801758,
      "activations/layer21_attention_weight_max": 27.651897430419922,
      "activations/layer21_attention_weight_min": -24.5289306640625,
      "activations/layer22_attention_weight_max": 36.46192932128906,
      "activations/layer22_attention_weight_min": -29.548330307006836,
      "activations/layer23_attention_weight_max": 32.3804817199707,
      "activations/layer23_attention_weight_min": -23.06220054626465,
      "activations/layer2_attention_weight_max": 31.581907272338867,
      "activations/layer2_attention_weight_min": -30.14593505859375,
      "activations/layer3_attention_weight_max": 105.38191986083984,
      "activations/layer3_attention_weight_min": -106.05770111083984,
      "activations/layer4_attention_weight_max": 117.91969299316406,
      "activations/layer4_attention_weight_min": -114.8531494140625,
      "activations/layer5_attention_weight_max": 58.386817932128906,
      "activations/layer5_attention_weight_min": -63.46289825439453,
      "activations/layer6_attention_weight_max": 48.888450622558594,
      "activations/layer6_attention_weight_min": -49.00737380981445,
      "activations/layer7_attention_weight_max": 110.74685668945312,
      "activations/layer7_attention_weight_min": -98.41456604003906,
      "activations/layer8_attention_weight_max": 51.93873977661133,
      "activations/layer8_attention_weight_min": -47.196022033691406,
      "activations/layer9_attention_weight_max": 37.81187057495117,
      "activations/layer9_attention_weight_min": -39.28164291381836,
      "epoch": 18.62,
      "learning_rate": 3.022992424242424e-05,
      "loss": 2.7388,
      "step": 320450
    },
    {
      "activations/layer0_attention_weight_max": 16.498830795288086,
      "activations/layer0_attention_weight_min": -12.35478401184082,
      "activations/layer10_attention_weight_max": 33.08936309814453,
      "activations/layer10_attention_weight_min": -35.90196990966797,
      "activations/layer11_attention_weight_max": 32.523468017578125,
      "activations/layer11_attention_weight_min": -33.77120590209961,
      "activations/layer12_attention_weight_max": 26.985807418823242,
      "activations/layer12_attention_weight_min": -26.55514907836914,
      "activations/layer13_attention_weight_max": 38.99211120605469,
      "activations/layer13_attention_weight_min": -36.0522346496582,
      "activations/layer14_attention_weight_max": 42.64940643310547,
      "activations/layer14_attention_weight_min": -38.25184631347656,
      "activations/layer15_attention_weight_max": 38.93283462524414,
      "activations/layer15_attention_weight_min": -37.108985900878906,
      "activations/layer16_attention_weight_max": 28.5263614654541,
      "activations/layer16_attention_weight_min": -28.804630279541016,
      "activations/layer17_attention_weight_max": 30.00589942932129,
      "activations/layer17_attention_weight_min": -26.57944107055664,
      "activations/layer18_attention_weight_max": 33.935577392578125,
      "activations/layer18_attention_weight_min": -28.218955993652344,
      "activations/layer19_attention_weight_max": 35.28904342651367,
      "activations/layer19_attention_weight_min": -35.939979553222656,
      "activations/layer1_attention_weight_max": 16.09824562072754,
      "activations/layer1_attention_weight_min": -13.713420867919922,
      "activations/layer20_attention_weight_max": 33.05188751220703,
      "activations/layer20_attention_weight_min": -27.900249481201172,
      "activations/layer21_attention_weight_max": 29.22640609741211,
      "activations/layer21_attention_weight_min": -27.715267181396484,
      "activations/layer22_attention_weight_max": 42.32975769042969,
      "activations/layer22_attention_weight_min": -34.78742218017578,
      "activations/layer23_attention_weight_max": 36.36293029785156,
      "activations/layer23_attention_weight_min": -26.80138397216797,
      "activations/layer2_attention_weight_max": 28.83364486694336,
      "activations/layer2_attention_weight_min": -30.803050994873047,
      "activations/layer3_attention_weight_max": 94.0157241821289,
      "activations/layer3_attention_weight_min": -101.66265869140625,
      "activations/layer4_attention_weight_max": 103.29327392578125,
      "activations/layer4_attention_weight_min": -108.28570556640625,
      "activations/layer5_attention_weight_max": 49.73246383666992,
      "activations/layer5_attention_weight_min": -62.63618087768555,
      "activations/layer6_attention_weight_max": 44.11434555053711,
      "activations/layer6_attention_weight_min": -47.24300765991211,
      "activations/layer7_attention_weight_max": 92.79765319824219,
      "activations/layer7_attention_weight_min": -94.89324951171875,
      "activations/layer8_attention_weight_max": 42.624942779541016,
      "activations/layer8_attention_weight_min": -42.74006271362305,
      "activations/layer9_attention_weight_max": 33.62384033203125,
      "activations/layer9_attention_weight_min": -35.89870834350586,
      "epoch": 18.62,
      "learning_rate": 3.0210984848484845e-05,
      "loss": 2.7212,
      "step": 320500
    },
    {
      "activations/layer0_attention_weight_max": 16.163602828979492,
      "activations/layer0_attention_weight_min": -11.688857078552246,
      "activations/layer10_attention_weight_max": 37.508663177490234,
      "activations/layer10_attention_weight_min": -34.50094223022461,
      "activations/layer11_attention_weight_max": 37.786983489990234,
      "activations/layer11_attention_weight_min": -36.10624694824219,
      "activations/layer12_attention_weight_max": 26.65274429321289,
      "activations/layer12_attention_weight_min": -26.123531341552734,
      "activations/layer13_attention_weight_max": 38.452632904052734,
      "activations/layer13_attention_weight_min": -34.242462158203125,
      "activations/layer14_attention_weight_max": 40.85245132446289,
      "activations/layer14_attention_weight_min": -37.56581115722656,
      "activations/layer15_attention_weight_max": 40.65559768676758,
      "activations/layer15_attention_weight_min": -35.772186279296875,
      "activations/layer16_attention_weight_max": 30.86128807067871,
      "activations/layer16_attention_weight_min": -28.086748123168945,
      "activations/layer17_attention_weight_max": 32.2576789855957,
      "activations/layer17_attention_weight_min": -25.98857879638672,
      "activations/layer18_attention_weight_max": 31.443681716918945,
      "activations/layer18_attention_weight_min": -26.652759552001953,
      "activations/layer19_attention_weight_max": 30.605955123901367,
      "activations/layer19_attention_weight_min": -30.080617904663086,
      "activations/layer1_attention_weight_max": 16.539806365966797,
      "activations/layer1_attention_weight_min": -13.683432579040527,
      "activations/layer20_attention_weight_max": 27.244462966918945,
      "activations/layer20_attention_weight_min": -24.322046279907227,
      "activations/layer21_attention_weight_max": 26.139616012573242,
      "activations/layer21_attention_weight_min": -25.254772186279297,
      "activations/layer22_attention_weight_max": 41.19733428955078,
      "activations/layer22_attention_weight_min": -33.07893371582031,
      "activations/layer23_attention_weight_max": 30.952566146850586,
      "activations/layer23_attention_weight_min": -25.013442993164062,
      "activations/layer2_attention_weight_max": 31.848485946655273,
      "activations/layer2_attention_weight_min": -30.95025634765625,
      "activations/layer3_attention_weight_max": 99.57791137695312,
      "activations/layer3_attention_weight_min": -97.24365234375,
      "activations/layer4_attention_weight_max": 110.66905975341797,
      "activations/layer4_attention_weight_min": -104.30731201171875,
      "activations/layer5_attention_weight_max": 54.19867706298828,
      "activations/layer5_attention_weight_min": -63.57334518432617,
      "activations/layer6_attention_weight_max": 45.39205551147461,
      "activations/layer6_attention_weight_min": -47.296566009521484,
      "activations/layer7_attention_weight_max": 103.13572692871094,
      "activations/layer7_attention_weight_min": -91.8487319946289,
      "activations/layer8_attention_weight_max": 41.88340377807617,
      "activations/layer8_attention_weight_min": -44.50860595703125,
      "activations/layer9_attention_weight_max": 33.6922721862793,
      "activations/layer9_attention_weight_min": -33.709957122802734,
      "epoch": 18.63,
      "learning_rate": 3.0192045454545454e-05,
      "loss": 2.7297,
      "step": 320550
    },
    {
      "activations/layer0_attention_weight_max": 16.39196014404297,
      "activations/layer0_attention_weight_min": -13.113422393798828,
      "activations/layer10_attention_weight_max": 34.486576080322266,
      "activations/layer10_attention_weight_min": -35.760738372802734,
      "activations/layer11_attention_weight_max": 34.04196548461914,
      "activations/layer11_attention_weight_min": -36.075767517089844,
      "activations/layer12_attention_weight_max": 26.620548248291016,
      "activations/layer12_attention_weight_min": -26.509138107299805,
      "activations/layer13_attention_weight_max": 40.01661682128906,
      "activations/layer13_attention_weight_min": -36.78207015991211,
      "activations/layer14_attention_weight_max": 43.05747985839844,
      "activations/layer14_attention_weight_min": -39.616912841796875,
      "activations/layer15_attention_weight_max": 39.22023010253906,
      "activations/layer15_attention_weight_min": -35.911163330078125,
      "activations/layer16_attention_weight_max": 29.694169998168945,
      "activations/layer16_attention_weight_min": -28.75273323059082,
      "activations/layer17_attention_weight_max": 32.08796691894531,
      "activations/layer17_attention_weight_min": -27.253128051757812,
      "activations/layer18_attention_weight_max": 32.186676025390625,
      "activations/layer18_attention_weight_min": -29.052141189575195,
      "activations/layer19_attention_weight_max": 33.3199577331543,
      "activations/layer19_attention_weight_min": -33.60611343383789,
      "activations/layer1_attention_weight_max": 15.59774112701416,
      "activations/layer1_attention_weight_min": -14.597235679626465,
      "activations/layer20_attention_weight_max": 29.544151306152344,
      "activations/layer20_attention_weight_min": -26.6964111328125,
      "activations/layer21_attention_weight_max": 29.98405647277832,
      "activations/layer21_attention_weight_min": -26.12352752685547,
      "activations/layer22_attention_weight_max": 43.106101989746094,
      "activations/layer22_attention_weight_min": -32.37184524536133,
      "activations/layer23_attention_weight_max": 31.401485443115234,
      "activations/layer23_attention_weight_min": -25.88497543334961,
      "activations/layer2_attention_weight_max": 31.498701095581055,
      "activations/layer2_attention_weight_min": -30.566287994384766,
      "activations/layer3_attention_weight_max": 93.72489166259766,
      "activations/layer3_attention_weight_min": -95.29907989501953,
      "activations/layer4_attention_weight_max": 101.02950286865234,
      "activations/layer4_attention_weight_min": -106.21524810791016,
      "activations/layer5_attention_weight_max": 49.33784484863281,
      "activations/layer5_attention_weight_min": -62.552268981933594,
      "activations/layer6_attention_weight_max": 44.78939437866211,
      "activations/layer6_attention_weight_min": -48.42811584472656,
      "activations/layer7_attention_weight_max": 93.5290756225586,
      "activations/layer7_attention_weight_min": -104.09126281738281,
      "activations/layer8_attention_weight_max": 41.34209060668945,
      "activations/layer8_attention_weight_min": -47.99748229980469,
      "activations/layer9_attention_weight_max": 32.435115814208984,
      "activations/layer9_attention_weight_min": -34.78749465942383,
      "epoch": 18.63,
      "learning_rate": 3.0173106060606055e-05,
      "loss": 2.7381,
      "step": 320600
    },
    {
      "activations/layer0_attention_weight_max": 15.90750503540039,
      "activations/layer0_attention_weight_min": -12.600695610046387,
      "activations/layer10_attention_weight_max": 37.9517707824707,
      "activations/layer10_attention_weight_min": -38.66644287109375,
      "activations/layer11_attention_weight_max": 36.46925354003906,
      "activations/layer11_attention_weight_min": -37.74257278442383,
      "activations/layer12_attention_weight_max": 32.312705993652344,
      "activations/layer12_attention_weight_min": -30.886350631713867,
      "activations/layer13_attention_weight_max": 49.13731002807617,
      "activations/layer13_attention_weight_min": -43.12822341918945,
      "activations/layer14_attention_weight_max": 54.70490646362305,
      "activations/layer14_attention_weight_min": -45.342716217041016,
      "activations/layer15_attention_weight_max": 51.76519775390625,
      "activations/layer15_attention_weight_min": -41.3485221862793,
      "activations/layer16_attention_weight_max": 40.6231803894043,
      "activations/layer16_attention_weight_min": -30.145658493041992,
      "activations/layer17_attention_weight_max": 38.05232620239258,
      "activations/layer17_attention_weight_min": -29.905805587768555,
      "activations/layer18_attention_weight_max": 39.95995330810547,
      "activations/layer18_attention_weight_min": -31.86385726928711,
      "activations/layer19_attention_weight_max": 53.5352783203125,
      "activations/layer19_attention_weight_min": -36.04872512817383,
      "activations/layer1_attention_weight_max": 16.093374252319336,
      "activations/layer1_attention_weight_min": -13.308588027954102,
      "activations/layer20_attention_weight_max": 40.11189270019531,
      "activations/layer20_attention_weight_min": -25.804454803466797,
      "activations/layer21_attention_weight_max": 39.809608459472656,
      "activations/layer21_attention_weight_min": -28.226409912109375,
      "activations/layer22_attention_weight_max": 61.6225700378418,
      "activations/layer22_attention_weight_min": -39.107093811035156,
      "activations/layer23_attention_weight_max": 47.95754623413086,
      "activations/layer23_attention_weight_min": -29.527057647705078,
      "activations/layer2_attention_weight_max": 32.10496520996094,
      "activations/layer2_attention_weight_min": -28.61766815185547,
      "activations/layer3_attention_weight_max": 93.5912094116211,
      "activations/layer3_attention_weight_min": -94.44330596923828,
      "activations/layer4_attention_weight_max": 106.39697265625,
      "activations/layer4_attention_weight_min": -107.04534149169922,
      "activations/layer5_attention_weight_max": 50.10517501831055,
      "activations/layer5_attention_weight_min": -66.65989685058594,
      "activations/layer6_attention_weight_max": 46.00143051147461,
      "activations/layer6_attention_weight_min": -46.213504791259766,
      "activations/layer7_attention_weight_max": 98.1828842163086,
      "activations/layer7_attention_weight_min": -97.47608947753906,
      "activations/layer8_attention_weight_max": 41.60707473754883,
      "activations/layer8_attention_weight_min": -46.293312072753906,
      "activations/layer9_attention_weight_max": 35.10301971435547,
      "activations/layer9_attention_weight_min": -37.23614501953125,
      "epoch": 18.63,
      "learning_rate": 3.0154166666666663e-05,
      "loss": 2.7325,
      "step": 320650
    },
    {
      "activations/layer0_attention_weight_max": 15.85690689086914,
      "activations/layer0_attention_weight_min": -11.929353713989258,
      "activations/layer10_attention_weight_max": 34.681121826171875,
      "activations/layer10_attention_weight_min": -35.534210205078125,
      "activations/layer11_attention_weight_max": 32.82188034057617,
      "activations/layer11_attention_weight_min": -37.656497955322266,
      "activations/layer12_attention_weight_max": 31.226154327392578,
      "activations/layer12_attention_weight_min": -30.01080894470215,
      "activations/layer13_attention_weight_max": 47.85698318481445,
      "activations/layer13_attention_weight_min": -40.11174011230469,
      "activations/layer14_attention_weight_max": 53.52337646484375,
      "activations/layer14_attention_weight_min": -42.66722869873047,
      "activations/layer15_attention_weight_max": 47.88507080078125,
      "activations/layer15_attention_weight_min": -38.37570571899414,
      "activations/layer16_attention_weight_max": 36.56406021118164,
      "activations/layer16_attention_weight_min": -29.760547637939453,
      "activations/layer17_attention_weight_max": 36.094200134277344,
      "activations/layer17_attention_weight_min": -27.17315673828125,
      "activations/layer18_attention_weight_max": 36.85178756713867,
      "activations/layer18_attention_weight_min": -27.185617446899414,
      "activations/layer19_attention_weight_max": 38.26102828979492,
      "activations/layer19_attention_weight_min": -30.77467155456543,
      "activations/layer1_attention_weight_max": 16.853553771972656,
      "activations/layer1_attention_weight_min": -15.1116361618042,
      "activations/layer20_attention_weight_max": 33.246639251708984,
      "activations/layer20_attention_weight_min": -24.09838104248047,
      "activations/layer21_attention_weight_max": 35.34079360961914,
      "activations/layer21_attention_weight_min": -24.772476196289062,
      "activations/layer22_attention_weight_max": 56.5738410949707,
      "activations/layer22_attention_weight_min": -33.19623565673828,
      "activations/layer23_attention_weight_max": 37.325016021728516,
      "activations/layer23_attention_weight_min": -26.126575469970703,
      "activations/layer2_attention_weight_max": 29.276660919189453,
      "activations/layer2_attention_weight_min": -29.791532516479492,
      "activations/layer3_attention_weight_max": 97.74816131591797,
      "activations/layer3_attention_weight_min": -95.03284454345703,
      "activations/layer4_attention_weight_max": 109.0091323852539,
      "activations/layer4_attention_weight_min": -109.24983215332031,
      "activations/layer5_attention_weight_max": 52.8140869140625,
      "activations/layer5_attention_weight_min": -62.90655517578125,
      "activations/layer6_attention_weight_max": 43.40883255004883,
      "activations/layer6_attention_weight_min": -47.36062240600586,
      "activations/layer7_attention_weight_max": 92.0914077758789,
      "activations/layer7_attention_weight_min": -97.68592834472656,
      "activations/layer8_attention_weight_max": 44.04921340942383,
      "activations/layer8_attention_weight_min": -46.01613235473633,
      "activations/layer9_attention_weight_max": 31.712284088134766,
      "activations/layer9_attention_weight_min": -36.66673278808594,
      "epoch": 18.63,
      "learning_rate": 3.013522727272727e-05,
      "loss": 2.7306,
      "step": 320700
    },
    {
      "activations/layer0_attention_weight_max": 15.951112747192383,
      "activations/layer0_attention_weight_min": -12.209458351135254,
      "activations/layer10_attention_weight_max": 32.706031799316406,
      "activations/layer10_attention_weight_min": -33.345218658447266,
      "activations/layer11_attention_weight_max": 32.172794342041016,
      "activations/layer11_attention_weight_min": -34.69354248046875,
      "activations/layer12_attention_weight_max": 27.50812339782715,
      "activations/layer12_attention_weight_min": -26.29177474975586,
      "activations/layer13_attention_weight_max": 40.709754943847656,
      "activations/layer13_attention_weight_min": -35.07855987548828,
      "activations/layer14_attention_weight_max": 45.7888298034668,
      "activations/layer14_attention_weight_min": -38.84803771972656,
      "activations/layer15_attention_weight_max": 39.15420913696289,
      "activations/layer15_attention_weight_min": -37.40843963623047,
      "activations/layer16_attention_weight_max": 29.889965057373047,
      "activations/layer16_attention_weight_min": -30.27309226989746,
      "activations/layer17_attention_weight_max": 27.903268814086914,
      "activations/layer17_attention_weight_min": -27.51449203491211,
      "activations/layer18_attention_weight_max": 31.948518753051758,
      "activations/layer18_attention_weight_min": -28.798221588134766,
      "activations/layer19_attention_weight_max": 32.01551055908203,
      "activations/layer19_attention_weight_min": -33.115020751953125,
      "activations/layer1_attention_weight_max": 16.40766143798828,
      "activations/layer1_attention_weight_min": -13.076034545898438,
      "activations/layer20_attention_weight_max": 30.079776763916016,
      "activations/layer20_attention_weight_min": -25.579166412353516,
      "activations/layer21_attention_weight_max": 30.169857025146484,
      "activations/layer21_attention_weight_min": -26.235857009887695,
      "activations/layer22_attention_weight_max": 38.652801513671875,
      "activations/layer22_attention_weight_min": -33.61640930175781,
      "activations/layer23_attention_weight_max": 31.36041831970215,
      "activations/layer23_attention_weight_min": -27.262008666992188,
      "activations/layer2_attention_weight_max": 30.8802547454834,
      "activations/layer2_attention_weight_min": -29.852352142333984,
      "activations/layer3_attention_weight_max": 88.20146942138672,
      "activations/layer3_attention_weight_min": -96.89520263671875,
      "activations/layer4_attention_weight_max": 102.0519027709961,
      "activations/layer4_attention_weight_min": -106.97438049316406,
      "activations/layer5_attention_weight_max": 50.5903434753418,
      "activations/layer5_attention_weight_min": -62.83362579345703,
      "activations/layer6_attention_weight_max": 42.035762786865234,
      "activations/layer6_attention_weight_min": -44.92612838745117,
      "activations/layer7_attention_weight_max": 89.66060638427734,
      "activations/layer7_attention_weight_min": -94.89348602294922,
      "activations/layer8_attention_weight_max": 39.46590042114258,
      "activations/layer8_attention_weight_min": -43.875030517578125,
      "activations/layer9_attention_weight_max": 30.8720703125,
      "activations/layer9_attention_weight_min": -34.62123107910156,
      "epoch": 18.64,
      "learning_rate": 3.0116287878787876e-05,
      "loss": 2.7368,
      "step": 320750
    },
    {
      "activations/layer0_attention_weight_max": 15.953973770141602,
      "activations/layer0_attention_weight_min": -12.914960861206055,
      "activations/layer10_attention_weight_max": 38.72233200073242,
      "activations/layer10_attention_weight_min": -37.61669158935547,
      "activations/layer11_attention_weight_max": 38.480186462402344,
      "activations/layer11_attention_weight_min": -35.44991683959961,
      "activations/layer12_attention_weight_max": 29.432727813720703,
      "activations/layer12_attention_weight_min": -28.88873863220215,
      "activations/layer13_attention_weight_max": 40.795772552490234,
      "activations/layer13_attention_weight_min": -37.72590637207031,
      "activations/layer14_attention_weight_max": 44.01656723022461,
      "activations/layer14_attention_weight_min": -38.84128952026367,
      "activations/layer15_attention_weight_max": 40.95602035522461,
      "activations/layer15_attention_weight_min": -36.697017669677734,
      "activations/layer16_attention_weight_max": 28.06338882446289,
      "activations/layer16_attention_weight_min": -29.226320266723633,
      "activations/layer17_attention_weight_max": 28.98188018798828,
      "activations/layer17_attention_weight_min": -29.200448989868164,
      "activations/layer18_attention_weight_max": 33.4241943359375,
      "activations/layer18_attention_weight_min": -26.475814819335938,
      "activations/layer19_attention_weight_max": 37.41771697998047,
      "activations/layer19_attention_weight_min": -35.17289352416992,
      "activations/layer1_attention_weight_max": 17.020124435424805,
      "activations/layer1_attention_weight_min": -18.147310256958008,
      "activations/layer20_attention_weight_max": 34.13523483276367,
      "activations/layer20_attention_weight_min": -29.904706954956055,
      "activations/layer21_attention_weight_max": 35.011474609375,
      "activations/layer21_attention_weight_min": -28.410179138183594,
      "activations/layer22_attention_weight_max": 44.55167007446289,
      "activations/layer22_attention_weight_min": -33.22662353515625,
      "activations/layer23_attention_weight_max": 36.46934509277344,
      "activations/layer23_attention_weight_min": -28.34518814086914,
      "activations/layer2_attention_weight_max": 30.80231475830078,
      "activations/layer2_attention_weight_min": -28.626174926757812,
      "activations/layer3_attention_weight_max": 89.240234375,
      "activations/layer3_attention_weight_min": -87.19873809814453,
      "activations/layer4_attention_weight_max": 101.06007385253906,
      "activations/layer4_attention_weight_min": -108.82405853271484,
      "activations/layer5_attention_weight_max": 48.21533203125,
      "activations/layer5_attention_weight_min": -63.32378387451172,
      "activations/layer6_attention_weight_max": 43.1969108581543,
      "activations/layer6_attention_weight_min": -46.17667770385742,
      "activations/layer7_attention_weight_max": 94.5918197631836,
      "activations/layer7_attention_weight_min": -93.40110778808594,
      "activations/layer8_attention_weight_max": 43.907371520996094,
      "activations/layer8_attention_weight_min": -46.780616760253906,
      "activations/layer9_attention_weight_max": 34.656925201416016,
      "activations/layer9_attention_weight_min": -36.57979202270508,
      "epoch": 18.64,
      "learning_rate": 3.0097348484848485e-05,
      "loss": 2.7115,
      "step": 320800
    },
    {
      "activations/layer0_attention_weight_max": 16.164287567138672,
      "activations/layer0_attention_weight_min": -12.997618675231934,
      "activations/layer10_attention_weight_max": 36.46143341064453,
      "activations/layer10_attention_weight_min": -34.27732849121094,
      "activations/layer11_attention_weight_max": 35.13993835449219,
      "activations/layer11_attention_weight_min": -34.64714813232422,
      "activations/layer12_attention_weight_max": 27.973743438720703,
      "activations/layer12_attention_weight_min": -26.696191787719727,
      "activations/layer13_attention_weight_max": 38.233551025390625,
      "activations/layer13_attention_weight_min": -34.75033187866211,
      "activations/layer14_attention_weight_max": 41.59511184692383,
      "activations/layer14_attention_weight_min": -36.75737380981445,
      "activations/layer15_attention_weight_max": 40.05384063720703,
      "activations/layer15_attention_weight_min": -35.755245208740234,
      "activations/layer16_attention_weight_max": 29.703214645385742,
      "activations/layer16_attention_weight_min": -30.15146827697754,
      "activations/layer17_attention_weight_max": 29.06043243408203,
      "activations/layer17_attention_weight_min": -27.000316619873047,
      "activations/layer18_attention_weight_max": 31.282493591308594,
      "activations/layer18_attention_weight_min": -25.90131187438965,
      "activations/layer19_attention_weight_max": 31.644901275634766,
      "activations/layer19_attention_weight_min": -30.94569969177246,
      "activations/layer1_attention_weight_max": 16.50609588623047,
      "activations/layer1_attention_weight_min": -14.53126049041748,
      "activations/layer20_attention_weight_max": 27.22783851623535,
      "activations/layer20_attention_weight_min": -25.84225082397461,
      "activations/layer21_attention_weight_max": 28.252273559570312,
      "activations/layer21_attention_weight_min": -25.077550888061523,
      "activations/layer22_attention_weight_max": 39.13010025024414,
      "activations/layer22_attention_weight_min": -32.805625915527344,
      "activations/layer23_attention_weight_max": 32.21698760986328,
      "activations/layer23_attention_weight_min": -27.554370880126953,
      "activations/layer2_attention_weight_max": 30.221302032470703,
      "activations/layer2_attention_weight_min": -29.402570724487305,
      "activations/layer3_attention_weight_max": 95.58281707763672,
      "activations/layer3_attention_weight_min": -96.98947143554688,
      "activations/layer4_attention_weight_max": 103.79180908203125,
      "activations/layer4_attention_weight_min": -112.40348052978516,
      "activations/layer5_attention_weight_max": 46.93901824951172,
      "activations/layer5_attention_weight_min": -65.34574890136719,
      "activations/layer6_attention_weight_max": 43.68376541137695,
      "activations/layer6_attention_weight_min": -47.2974967956543,
      "activations/layer7_attention_weight_max": 91.30491638183594,
      "activations/layer7_attention_weight_min": -96.3138198852539,
      "activations/layer8_attention_weight_max": 43.688289642333984,
      "activations/layer8_attention_weight_min": -44.110286712646484,
      "activations/layer9_attention_weight_max": 32.47254180908203,
      "activations/layer9_attention_weight_min": -34.00794982910156,
      "epoch": 18.64,
      "learning_rate": 3.0078409090909086e-05,
      "loss": 2.7029,
      "step": 320850
    },
    {
      "activations/layer0_attention_weight_max": 15.270896911621094,
      "activations/layer0_attention_weight_min": -13.412590980529785,
      "activations/layer10_attention_weight_max": 34.167022705078125,
      "activations/layer10_attention_weight_min": -34.682151794433594,
      "activations/layer11_attention_weight_max": 35.449974060058594,
      "activations/layer11_attention_weight_min": -36.82328796386719,
      "activations/layer12_attention_weight_max": 26.286531448364258,
      "activations/layer12_attention_weight_min": -31.350879669189453,
      "activations/layer13_attention_weight_max": 43.71157455444336,
      "activations/layer13_attention_weight_min": -35.20978927612305,
      "activations/layer14_attention_weight_max": 51.03469467163086,
      "activations/layer14_attention_weight_min": -39.88032913208008,
      "activations/layer15_attention_weight_max": 44.39596939086914,
      "activations/layer15_attention_weight_min": -36.79600143432617,
      "activations/layer16_attention_weight_max": 32.37519836425781,
      "activations/layer16_attention_weight_min": -30.312158584594727,
      "activations/layer17_attention_weight_max": 29.46170425415039,
      "activations/layer17_attention_weight_min": -26.5816707611084,
      "activations/layer18_attention_weight_max": 31.117238998413086,
      "activations/layer18_attention_weight_min": -26.189424514770508,
      "activations/layer19_attention_weight_max": 33.49968719482422,
      "activations/layer19_attention_weight_min": -33.57059860229492,
      "activations/layer1_attention_weight_max": 16.230138778686523,
      "activations/layer1_attention_weight_min": -13.861052513122559,
      "activations/layer20_attention_weight_max": 29.399269104003906,
      "activations/layer20_attention_weight_min": -26.298843383789062,
      "activations/layer21_attention_weight_max": 31.147573471069336,
      "activations/layer21_attention_weight_min": -26.366849899291992,
      "activations/layer22_attention_weight_max": 42.465232849121094,
      "activations/layer22_attention_weight_min": -33.61797332763672,
      "activations/layer23_attention_weight_max": 31.45588493347168,
      "activations/layer23_attention_weight_min": -28.63197898864746,
      "activations/layer2_attention_weight_max": 31.243404388427734,
      "activations/layer2_attention_weight_min": -30.431941986083984,
      "activations/layer3_attention_weight_max": 94.12467193603516,
      "activations/layer3_attention_weight_min": -96.79846954345703,
      "activations/layer4_attention_weight_max": 104.49165344238281,
      "activations/layer4_attention_weight_min": -102.39005279541016,
      "activations/layer5_attention_weight_max": 50.94071960449219,
      "activations/layer5_attention_weight_min": -61.5128288269043,
      "activations/layer6_attention_weight_max": 46.44826126098633,
      "activations/layer6_attention_weight_min": -47.36091613769531,
      "activations/layer7_attention_weight_max": 91.98906707763672,
      "activations/layer7_attention_weight_min": -95.51744079589844,
      "activations/layer8_attention_weight_max": 43.23662567138672,
      "activations/layer8_attention_weight_min": -43.96943283081055,
      "activations/layer9_attention_weight_max": 32.6146354675293,
      "activations/layer9_attention_weight_min": -35.51442337036133,
      "epoch": 18.65,
      "learning_rate": 3.0059469696969694e-05,
      "loss": 2.7362,
      "step": 320900
    },
    {
      "activations/layer0_attention_weight_max": 15.940486907958984,
      "activations/layer0_attention_weight_min": -12.39501667022705,
      "activations/layer10_attention_weight_max": 33.60118865966797,
      "activations/layer10_attention_weight_min": -35.55029296875,
      "activations/layer11_attention_weight_max": 34.90324401855469,
      "activations/layer11_attention_weight_min": -34.930450439453125,
      "activations/layer12_attention_weight_max": 28.092557907104492,
      "activations/layer12_attention_weight_min": -27.50474739074707,
      "activations/layer13_attention_weight_max": 41.31618118286133,
      "activations/layer13_attention_weight_min": -36.58790969848633,
      "activations/layer14_attention_weight_max": 44.39971923828125,
      "activations/layer14_attention_weight_min": -39.11363983154297,
      "activations/layer15_attention_weight_max": 41.33514404296875,
      "activations/layer15_attention_weight_min": -36.66179275512695,
      "activations/layer16_attention_weight_max": 30.209182739257812,
      "activations/layer16_attention_weight_min": -28.554685592651367,
      "activations/layer17_attention_weight_max": 30.171213150024414,
      "activations/layer17_attention_weight_min": -28.974525451660156,
      "activations/layer18_attention_weight_max": 34.219139099121094,
      "activations/layer18_attention_weight_min": -27.347692489624023,
      "activations/layer19_attention_weight_max": 35.33925247192383,
      "activations/layer19_attention_weight_min": -32.080474853515625,
      "activations/layer1_attention_weight_max": 15.49955940246582,
      "activations/layer1_attention_weight_min": -13.948163032531738,
      "activations/layer20_attention_weight_max": 27.91844367980957,
      "activations/layer20_attention_weight_min": -28.16515350341797,
      "activations/layer21_attention_weight_max": 30.23738670349121,
      "activations/layer21_attention_weight_min": -26.196136474609375,
      "activations/layer22_attention_weight_max": 49.1236457824707,
      "activations/layer22_attention_weight_min": -32.347068786621094,
      "activations/layer23_attention_weight_max": 32.903717041015625,
      "activations/layer23_attention_weight_min": -26.422447204589844,
      "activations/layer2_attention_weight_max": 31.634870529174805,
      "activations/layer2_attention_weight_min": -29.843765258789062,
      "activations/layer3_attention_weight_max": 101.31075286865234,
      "activations/layer3_attention_weight_min": -97.91049194335938,
      "activations/layer4_attention_weight_max": 106.73140716552734,
      "activations/layer4_attention_weight_min": -105.70987701416016,
      "activations/layer5_attention_weight_max": 50.889495849609375,
      "activations/layer5_attention_weight_min": -63.176361083984375,
      "activations/layer6_attention_weight_max": 44.7165641784668,
      "activations/layer6_attention_weight_min": -46.341793060302734,
      "activations/layer7_attention_weight_max": 97.83592224121094,
      "activations/layer7_attention_weight_min": -97.2226333618164,
      "activations/layer8_attention_weight_max": 45.28102493286133,
      "activations/layer8_attention_weight_min": -44.91499710083008,
      "activations/layer9_attention_weight_max": 33.10888671875,
      "activations/layer9_attention_weight_min": -34.458038330078125,
      "epoch": 18.65,
      "learning_rate": 3.00405303030303e-05,
      "loss": 2.74,
      "step": 320950
    },
    {
      "activations/layer0_attention_weight_max": 16.08726692199707,
      "activations/layer0_attention_weight_min": -11.9705228805542,
      "activations/layer10_attention_weight_max": 34.73761749267578,
      "activations/layer10_attention_weight_min": -36.907142639160156,
      "activations/layer11_attention_weight_max": 33.13905715942383,
      "activations/layer11_attention_weight_min": -35.40026092529297,
      "activations/layer12_attention_weight_max": 28.42247772216797,
      "activations/layer12_attention_weight_min": -29.17160415649414,
      "activations/layer13_attention_weight_max": 41.675350189208984,
      "activations/layer13_attention_weight_min": -38.31260681152344,
      "activations/layer14_attention_weight_max": 43.817237854003906,
      "activations/layer14_attention_weight_min": -39.05546951293945,
      "activations/layer15_attention_weight_max": 41.52495574951172,
      "activations/layer15_attention_weight_min": -36.569210052490234,
      "activations/layer16_attention_weight_max": 30.774381637573242,
      "activations/layer16_attention_weight_min": -28.760026931762695,
      "activations/layer17_attention_weight_max": 29.2707462310791,
      "activations/layer17_attention_weight_min": -26.2891788482666,
      "activations/layer18_attention_weight_max": 27.5506649017334,
      "activations/layer18_attention_weight_min": -26.15536117553711,
      "activations/layer19_attention_weight_max": 31.03763771057129,
      "activations/layer19_attention_weight_min": -30.564748764038086,
      "activations/layer1_attention_weight_max": 16.82236099243164,
      "activations/layer1_attention_weight_min": -13.152735710144043,
      "activations/layer20_attention_weight_max": 25.49382781982422,
      "activations/layer20_attention_weight_min": -25.92222785949707,
      "activations/layer21_attention_weight_max": 24.611454010009766,
      "activations/layer21_attention_weight_min": -24.444211959838867,
      "activations/layer22_attention_weight_max": 39.497276306152344,
      "activations/layer22_attention_weight_min": -29.203414916992188,
      "activations/layer23_attention_weight_max": 30.34148406982422,
      "activations/layer23_attention_weight_min": -24.235538482666016,
      "activations/layer2_attention_weight_max": 29.713504791259766,
      "activations/layer2_attention_weight_min": -28.90423583984375,
      "activations/layer3_attention_weight_max": 96.1416015625,
      "activations/layer3_attention_weight_min": -92.6325454711914,
      "activations/layer4_attention_weight_max": 105.1420669555664,
      "activations/layer4_attention_weight_min": -101.77415466308594,
      "activations/layer5_attention_weight_max": 49.422889709472656,
      "activations/layer5_attention_weight_min": -59.37638854980469,
      "activations/layer6_attention_weight_max": 44.7397575378418,
      "activations/layer6_attention_weight_min": -48.07024383544922,
      "activations/layer7_attention_weight_max": 95.69368743896484,
      "activations/layer7_attention_weight_min": -97.98066711425781,
      "activations/layer8_attention_weight_max": 42.32435607910156,
      "activations/layer8_attention_weight_min": -45.81404113769531,
      "activations/layer9_attention_weight_max": 32.087615966796875,
      "activations/layer9_attention_weight_min": -34.63508987426758,
      "epoch": 18.65,
      "learning_rate": 3.0021590909090908e-05,
      "loss": 2.7208,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.4967,
      "eval_samples_per_second": 505.375,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.4967,
      "eval_openwebtext_samples_per_second": 505.375,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9803,
      "eval_wikitext_samples_per_second": 230.263,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_lambada_loss": 2.359375,
      "eval_lambada_ppl": 10.584334175890335,
      "eval_lambada_runtime": 9.5552,
      "eval_lambada_samples_per_second": 509.566,
      "step": 321000
    },
    {
      "activations/layer0_attention_weight_max": 15.556170463562012,
      "activations/layer0_attention_weight_min": -12.648455619812012,
      "activations/layer10_attention_weight_max": 38.59107208251953,
      "activations/layer10_attention_weight_min": -37.28132629394531,
      "activations/layer11_attention_weight_max": 38.37980270385742,
      "activations/layer11_attention_weight_min": -37.3296012878418,
      "activations/layer12_attention_weight_max": 28.960281372070312,
      "activations/layer12_attention_weight_min": -27.39206314086914,
      "activations/layer13_attention_weight_max": 46.18305587768555,
      "activations/layer13_attention_weight_min": -37.89619064331055,
      "activations/layer14_attention_weight_max": 47.55713653564453,
      "activations/layer14_attention_weight_min": -42.06531524658203,
      "activations/layer15_attention_weight_max": 47.80867385864258,
      "activations/layer15_attention_weight_min": -37.35388946533203,
      "activations/layer16_attention_weight_max": 32.17301940917969,
      "activations/layer16_attention_weight_min": -27.87453842163086,
      "activations/layer17_attention_weight_max": 32.93629455566406,
      "activations/layer17_attention_weight_min": -27.749406814575195,
      "activations/layer18_attention_weight_max": 34.89411926269531,
      "activations/layer18_attention_weight_min": -27.89558219909668,
      "activations/layer19_attention_weight_max": 37.18428421020508,
      "activations/layer19_attention_weight_min": -32.96833801269531,
      "activations/layer1_attention_weight_max": 17.35915756225586,
      "activations/layer1_attention_weight_min": -14.393068313598633,
      "activations/layer20_attention_weight_max": 29.6837158203125,
      "activations/layer20_attention_weight_min": -27.23700523376465,
      "activations/layer21_attention_weight_max": 32.29997634887695,
      "activations/layer21_attention_weight_min": -26.10190200805664,
      "activations/layer22_attention_weight_max": 42.7744140625,
      "activations/layer22_attention_weight_min": -33.148948669433594,
      "activations/layer23_attention_weight_max": 34.34721374511719,
      "activations/layer23_attention_weight_min": -29.325592041015625,
      "activations/layer2_attention_weight_max": 31.36635971069336,
      "activations/layer2_attention_weight_min": -29.231956481933594,
      "activations/layer3_attention_weight_max": 97.3273696899414,
      "activations/layer3_attention_weight_min": -99.36280059814453,
      "activations/layer4_attention_weight_max": 103.617919921875,
      "activations/layer4_attention_weight_min": -113.65630340576172,
      "activations/layer5_attention_weight_max": 48.50764465332031,
      "activations/layer5_attention_weight_min": -65.15545654296875,
      "activations/layer6_attention_weight_max": 44.28383255004883,
      "activations/layer6_attention_weight_min": -48.682682037353516,
      "activations/layer7_attention_weight_max": 93.14505767822266,
      "activations/layer7_attention_weight_min": -98.07158660888672,
      "activations/layer8_attention_weight_max": 44.95882034301758,
      "activations/layer8_attention_weight_min": -46.128055572509766,
      "activations/layer9_attention_weight_max": 36.648406982421875,
      "activations/layer9_attention_weight_min": -35.991695404052734,
      "epoch": 18.65,
      "learning_rate": 3.0002651515151512e-05,
      "loss": 2.7289,
      "step": 321050
    },
    {
      "activations/layer0_attention_weight_max": 16.16380500793457,
      "activations/layer0_attention_weight_min": -12.449911117553711,
      "activations/layer10_attention_weight_max": 31.920330047607422,
      "activations/layer10_attention_weight_min": -35.16350173950195,
      "activations/layer11_attention_weight_max": 33.65711975097656,
      "activations/layer11_attention_weight_min": -32.92249298095703,
      "activations/layer12_attention_weight_max": 24.526966094970703,
      "activations/layer12_attention_weight_min": -26.226009368896484,
      "activations/layer13_attention_weight_max": 37.34782409667969,
      "activations/layer13_attention_weight_min": -34.98937225341797,
      "activations/layer14_attention_weight_max": 40.977806091308594,
      "activations/layer14_attention_weight_min": -36.61000442504883,
      "activations/layer15_attention_weight_max": 38.87725830078125,
      "activations/layer15_attention_weight_min": -35.517730712890625,
      "activations/layer16_attention_weight_max": 29.516416549682617,
      "activations/layer16_attention_weight_min": -29.539640426635742,
      "activations/layer17_attention_weight_max": 29.605295181274414,
      "activations/layer17_attention_weight_min": -27.40833854675293,
      "activations/layer18_attention_weight_max": 35.991641998291016,
      "activations/layer18_attention_weight_min": -27.697105407714844,
      "activations/layer19_attention_weight_max": 32.474082946777344,
      "activations/layer19_attention_weight_min": -32.626792907714844,
      "activations/layer1_attention_weight_max": 16.658554077148438,
      "activations/layer1_attention_weight_min": -14.26345157623291,
      "activations/layer20_attention_weight_max": 28.824134826660156,
      "activations/layer20_attention_weight_min": -25.773714065551758,
      "activations/layer21_attention_weight_max": 27.39860725402832,
      "activations/layer21_attention_weight_min": -28.14714813232422,
      "activations/layer22_attention_weight_max": 42.92151641845703,
      "activations/layer22_attention_weight_min": -35.952476501464844,
      "activations/layer23_attention_weight_max": 30.613388061523438,
      "activations/layer23_attention_weight_min": -25.966293334960938,
      "activations/layer2_attention_weight_max": 30.391157150268555,
      "activations/layer2_attention_weight_min": -29.10708999633789,
      "activations/layer3_attention_weight_max": 94.21334075927734,
      "activations/layer3_attention_weight_min": -96.45491790771484,
      "activations/layer4_attention_weight_max": 101.88133239746094,
      "activations/layer4_attention_weight_min": -104.93627166748047,
      "activations/layer5_attention_weight_max": 49.43781661987305,
      "activations/layer5_attention_weight_min": -59.93642807006836,
      "activations/layer6_attention_weight_max": 41.272361755371094,
      "activations/layer6_attention_weight_min": -44.59159469604492,
      "activations/layer7_attention_weight_max": 87.8410415649414,
      "activations/layer7_attention_weight_min": -95.66136932373047,
      "activations/layer8_attention_weight_max": 38.63267135620117,
      "activations/layer8_attention_weight_min": -43.07866287231445,
      "activations/layer9_attention_weight_max": 30.65060806274414,
      "activations/layer9_attention_weight_min": -33.82320022583008,
      "epoch": 18.66,
      "learning_rate": 2.998371212121212e-05,
      "loss": 2.7356,
      "step": 321100
    },
    {
      "activations/layer0_attention_weight_max": 16.5211181640625,
      "activations/layer0_attention_weight_min": -12.480399131774902,
      "activations/layer10_attention_weight_max": 33.9101676940918,
      "activations/layer10_attention_weight_min": -34.49617385864258,
      "activations/layer11_attention_weight_max": 34.268611907958984,
      "activations/layer11_attention_weight_min": -33.96712112426758,
      "activations/layer12_attention_weight_max": 27.388669967651367,
      "activations/layer12_attention_weight_min": -27.329883575439453,
      "activations/layer13_attention_weight_max": 43.39744186401367,
      "activations/layer13_attention_weight_min": -36.398170471191406,
      "activations/layer14_attention_weight_max": 42.873191833496094,
      "activations/layer14_attention_weight_min": -37.51270294189453,
      "activations/layer15_attention_weight_max": 39.33475875854492,
      "activations/layer15_attention_weight_min": -35.77432632446289,
      "activations/layer16_attention_weight_max": 30.560087203979492,
      "activations/layer16_attention_weight_min": -30.308671951293945,
      "activations/layer17_attention_weight_max": 29.21334457397461,
      "activations/layer17_attention_weight_min": -27.615079879760742,
      "activations/layer18_attention_weight_max": 35.30562973022461,
      "activations/layer18_attention_weight_min": -27.398473739624023,
      "activations/layer19_attention_weight_max": 38.491573333740234,
      "activations/layer19_attention_weight_min": -33.31874084472656,
      "activations/layer1_attention_weight_max": 15.682506561279297,
      "activations/layer1_attention_weight_min": -13.970650672912598,
      "activations/layer20_attention_weight_max": 32.58989334106445,
      "activations/layer20_attention_weight_min": -26.984844207763672,
      "activations/layer21_attention_weight_max": 31.32645034790039,
      "activations/layer21_attention_weight_min": -26.7113037109375,
      "activations/layer22_attention_weight_max": 43.156253814697266,
      "activations/layer22_attention_weight_min": -34.62270736694336,
      "activations/layer23_attention_weight_max": 34.395328521728516,
      "activations/layer23_attention_weight_min": -29.94644546508789,
      "activations/layer2_attention_weight_max": 29.802356719970703,
      "activations/layer2_attention_weight_min": -30.81641387939453,
      "activations/layer3_attention_weight_max": 95.78780364990234,
      "activations/layer3_attention_weight_min": -95.45880889892578,
      "activations/layer4_attention_weight_max": 100.70101165771484,
      "activations/layer4_attention_weight_min": -104.11849212646484,
      "activations/layer5_attention_weight_max": 48.475624084472656,
      "activations/layer5_attention_weight_min": -61.788780212402344,
      "activations/layer6_attention_weight_max": 44.42860412597656,
      "activations/layer6_attention_weight_min": -46.74503707885742,
      "activations/layer7_attention_weight_max": 88.83575439453125,
      "activations/layer7_attention_weight_min": -96.14488220214844,
      "activations/layer8_attention_weight_max": 41.237060546875,
      "activations/layer8_attention_weight_min": -43.98497772216797,
      "activations/layer9_attention_weight_max": 33.698707580566406,
      "activations/layer9_attention_weight_min": -34.1519889831543,
      "epoch": 18.66,
      "learning_rate": 2.9964772727272722e-05,
      "loss": 2.7219,
      "step": 321150
    },
    {
      "activations/layer0_attention_weight_max": 15.953868865966797,
      "activations/layer0_attention_weight_min": -14.679946899414062,
      "activations/layer10_attention_weight_max": 33.280128479003906,
      "activations/layer10_attention_weight_min": -34.83174514770508,
      "activations/layer11_attention_weight_max": 32.298004150390625,
      "activations/layer11_attention_weight_min": -33.767066955566406,
      "activations/layer12_attention_weight_max": 26.533809661865234,
      "activations/layer12_attention_weight_min": -26.011384963989258,
      "activations/layer13_attention_weight_max": 37.95014572143555,
      "activations/layer13_attention_weight_min": -37.57612228393555,
      "activations/layer14_attention_weight_max": 41.28679275512695,
      "activations/layer14_attention_weight_min": -38.547550201416016,
      "activations/layer15_attention_weight_max": 40.138038635253906,
      "activations/layer15_attention_weight_min": -35.195491790771484,
      "activations/layer16_attention_weight_max": 29.123451232910156,
      "activations/layer16_attention_weight_min": -29.264671325683594,
      "activations/layer17_attention_weight_max": 31.423526763916016,
      "activations/layer17_attention_weight_min": -27.886436462402344,
      "activations/layer18_attention_weight_max": 31.57525634765625,
      "activations/layer18_attention_weight_min": -28.54657554626465,
      "activations/layer19_attention_weight_max": 36.43900680541992,
      "activations/layer19_attention_weight_min": -34.133296966552734,
      "activations/layer1_attention_weight_max": 15.97752571105957,
      "activations/layer1_attention_weight_min": -16.553388595581055,
      "activations/layer20_attention_weight_max": 29.782896041870117,
      "activations/layer20_attention_weight_min": -26.515806198120117,
      "activations/layer21_attention_weight_max": 26.809030532836914,
      "activations/layer21_attention_weight_min": -27.191864013671875,
      "activations/layer22_attention_weight_max": 43.99905776977539,
      "activations/layer22_attention_weight_min": -33.27872848510742,
      "activations/layer23_attention_weight_max": 35.55004119873047,
      "activations/layer23_attention_weight_min": -27.718128204345703,
      "activations/layer2_attention_weight_max": 29.979736328125,
      "activations/layer2_attention_weight_min": -29.315412521362305,
      "activations/layer3_attention_weight_max": 94.24156951904297,
      "activations/layer3_attention_weight_min": -90.30043029785156,
      "activations/layer4_attention_weight_max": 102.437255859375,
      "activations/layer4_attention_weight_min": -103.35965728759766,
      "activations/layer5_attention_weight_max": 51.01627731323242,
      "activations/layer5_attention_weight_min": -61.736114501953125,
      "activations/layer6_attention_weight_max": 43.38010787963867,
      "activations/layer6_attention_weight_min": -47.71747589111328,
      "activations/layer7_attention_weight_max": 97.04583740234375,
      "activations/layer7_attention_weight_min": -93.94493865966797,
      "activations/layer8_attention_weight_max": 40.7161750793457,
      "activations/layer8_attention_weight_min": -44.19818115234375,
      "activations/layer9_attention_weight_max": 33.5721549987793,
      "activations/layer9_attention_weight_min": -34.60210037231445,
      "epoch": 18.66,
      "learning_rate": 2.994583333333333e-05,
      "loss": 2.7532,
      "step": 321200
    },
    {
      "activations/layer0_attention_weight_max": 16.068424224853516,
      "activations/layer0_attention_weight_min": -12.694799423217773,
      "activations/layer10_attention_weight_max": 35.087005615234375,
      "activations/layer10_attention_weight_min": -37.626346588134766,
      "activations/layer11_attention_weight_max": 31.89687156677246,
      "activations/layer11_attention_weight_min": -34.32915115356445,
      "activations/layer12_attention_weight_max": 27.5572452545166,
      "activations/layer12_attention_weight_min": -25.861112594604492,
      "activations/layer13_attention_weight_max": 41.889617919921875,
      "activations/layer13_attention_weight_min": -35.55727767944336,
      "activations/layer14_attention_weight_max": 44.392295837402344,
      "activations/layer14_attention_weight_min": -37.55580139160156,
      "activations/layer15_attention_weight_max": 40.64088439941406,
      "activations/layer15_attention_weight_min": -36.88698959350586,
      "activations/layer16_attention_weight_max": 31.631071090698242,
      "activations/layer16_attention_weight_min": -28.77865219116211,
      "activations/layer17_attention_weight_max": 30.40218162536621,
      "activations/layer17_attention_weight_min": -25.045345306396484,
      "activations/layer18_attention_weight_max": 29.981693267822266,
      "activations/layer18_attention_weight_min": -25.70779800415039,
      "activations/layer19_attention_weight_max": 32.86380386352539,
      "activations/layer19_attention_weight_min": -32.82463455200195,
      "activations/layer1_attention_weight_max": 16.19838523864746,
      "activations/layer1_attention_weight_min": -15.25788688659668,
      "activations/layer20_attention_weight_max": 28.105358123779297,
      "activations/layer20_attention_weight_min": -27.373756408691406,
      "activations/layer21_attention_weight_max": 28.101558685302734,
      "activations/layer21_attention_weight_min": -27.102657318115234,
      "activations/layer22_attention_weight_max": 40.99348831176758,
      "activations/layer22_attention_weight_min": -31.076677322387695,
      "activations/layer23_attention_weight_max": 33.375877380371094,
      "activations/layer23_attention_weight_min": -25.20357322692871,
      "activations/layer2_attention_weight_max": 29.45362091064453,
      "activations/layer2_attention_weight_min": -29.125898361206055,
      "activations/layer3_attention_weight_max": 91.94005584716797,
      "activations/layer3_attention_weight_min": -94.50106048583984,
      "activations/layer4_attention_weight_max": 101.64885711669922,
      "activations/layer4_attention_weight_min": -102.72914123535156,
      "activations/layer5_attention_weight_max": 50.25916290283203,
      "activations/layer5_attention_weight_min": -61.255409240722656,
      "activations/layer6_attention_weight_max": 44.76863098144531,
      "activations/layer6_attention_weight_min": -46.58961486816406,
      "activations/layer7_attention_weight_max": 88.72848510742188,
      "activations/layer7_attention_weight_min": -100.48676300048828,
      "activations/layer8_attention_weight_max": 39.92034912109375,
      "activations/layer8_attention_weight_min": -46.2172737121582,
      "activations/layer9_attention_weight_max": 31.791183471679688,
      "activations/layer9_attention_weight_min": -35.2110710144043,
      "epoch": 18.67,
      "learning_rate": 2.9926893939393935e-05,
      "loss": 2.7472,
      "step": 321250
    },
    {
      "activations/layer0_attention_weight_max": 16.18977165222168,
      "activations/layer0_attention_weight_min": -13.085463523864746,
      "activations/layer10_attention_weight_max": 32.74994659423828,
      "activations/layer10_attention_weight_min": -35.05580520629883,
      "activations/layer11_attention_weight_max": 33.15107727050781,
      "activations/layer11_attention_weight_min": -33.95549011230469,
      "activations/layer12_attention_weight_max": 25.707853317260742,
      "activations/layer12_attention_weight_min": -26.446216583251953,
      "activations/layer13_attention_weight_max": 39.15980911254883,
      "activations/layer13_attention_weight_min": -36.283390045166016,
      "activations/layer14_attention_weight_max": 39.59130096435547,
      "activations/layer14_attention_weight_min": -38.77851867675781,
      "activations/layer15_attention_weight_max": 35.11473846435547,
      "activations/layer15_attention_weight_min": -35.523399353027344,
      "activations/layer16_attention_weight_max": 28.519060134887695,
      "activations/layer16_attention_weight_min": -27.881303787231445,
      "activations/layer17_attention_weight_max": 28.697717666625977,
      "activations/layer17_attention_weight_min": -26.136621475219727,
      "activations/layer18_attention_weight_max": 29.5284481048584,
      "activations/layer18_attention_weight_min": -24.403614044189453,
      "activations/layer19_attention_weight_max": 30.466344833374023,
      "activations/layer19_attention_weight_min": -33.52693176269531,
      "activations/layer1_attention_weight_max": 15.962905883789062,
      "activations/layer1_attention_weight_min": -13.971443176269531,
      "activations/layer20_attention_weight_max": 25.093156814575195,
      "activations/layer20_attention_weight_min": -25.810884475708008,
      "activations/layer21_attention_weight_max": 26.99242401123047,
      "activations/layer21_attention_weight_min": -24.131338119506836,
      "activations/layer22_attention_weight_max": 38.52488327026367,
      "activations/layer22_attention_weight_min": -30.940887451171875,
      "activations/layer23_attention_weight_max": 30.623592376708984,
      "activations/layer23_attention_weight_min": -23.633718490600586,
      "activations/layer2_attention_weight_max": 31.59395408630371,
      "activations/layer2_attention_weight_min": -30.40445327758789,
      "activations/layer3_attention_weight_max": 92.12954711914062,
      "activations/layer3_attention_weight_min": -92.1406478881836,
      "activations/layer4_attention_weight_max": 98.5593490600586,
      "activations/layer4_attention_weight_min": -103.69637298583984,
      "activations/layer5_attention_weight_max": 49.2884521484375,
      "activations/layer5_attention_weight_min": -59.119964599609375,
      "activations/layer6_attention_weight_max": 42.318538665771484,
      "activations/layer6_attention_weight_min": -45.814414978027344,
      "activations/layer7_attention_weight_max": 86.69491577148438,
      "activations/layer7_attention_weight_min": -97.56513977050781,
      "activations/layer8_attention_weight_max": 38.94377899169922,
      "activations/layer8_attention_weight_min": -42.78715515136719,
      "activations/layer9_attention_weight_max": 32.1681022644043,
      "activations/layer9_attention_weight_min": -33.598506927490234,
      "epoch": 18.67,
      "learning_rate": 2.9907954545454544e-05,
      "loss": 2.721,
      "step": 321300
    },
    {
      "activations/layer0_attention_weight_max": 15.877979278564453,
      "activations/layer0_attention_weight_min": -13.271617889404297,
      "activations/layer10_attention_weight_max": 32.627891540527344,
      "activations/layer10_attention_weight_min": -34.489295959472656,
      "activations/layer11_attention_weight_max": 32.47602844238281,
      "activations/layer11_attention_weight_min": -33.30253219604492,
      "activations/layer12_attention_weight_max": 26.19931983947754,
      "activations/layer12_attention_weight_min": -25.463254928588867,
      "activations/layer13_attention_weight_max": 39.93489456176758,
      "activations/layer13_attention_weight_min": -34.50452423095703,
      "activations/layer14_attention_weight_max": 42.55978012084961,
      "activations/layer14_attention_weight_min": -38.54677963256836,
      "activations/layer15_attention_weight_max": 39.21888732910156,
      "activations/layer15_attention_weight_min": -37.37266159057617,
      "activations/layer16_attention_weight_max": 30.784683227539062,
      "activations/layer16_attention_weight_min": -28.987401962280273,
      "activations/layer17_attention_weight_max": 29.77503204345703,
      "activations/layer17_attention_weight_min": -28.29718780517578,
      "activations/layer18_attention_weight_max": 32.20105743408203,
      "activations/layer18_attention_weight_min": -26.469356536865234,
      "activations/layer19_attention_weight_max": 32.97047805786133,
      "activations/layer19_attention_weight_min": -32.02400207519531,
      "activations/layer1_attention_weight_max": 16.250356674194336,
      "activations/layer1_attention_weight_min": -15.227679252624512,
      "activations/layer20_attention_weight_max": 29.227827072143555,
      "activations/layer20_attention_weight_min": -25.602266311645508,
      "activations/layer21_attention_weight_max": 27.926387786865234,
      "activations/layer21_attention_weight_min": -24.865707397460938,
      "activations/layer22_attention_weight_max": 40.65314865112305,
      "activations/layer22_attention_weight_min": -32.39611053466797,
      "activations/layer23_attention_weight_max": 31.417919158935547,
      "activations/layer23_attention_weight_min": -28.44042205810547,
      "activations/layer2_attention_weight_max": 29.928722381591797,
      "activations/layer2_attention_weight_min": -30.19038963317871,
      "activations/layer3_attention_weight_max": 98.10626983642578,
      "activations/layer3_attention_weight_min": -98.64105987548828,
      "activations/layer4_attention_weight_max": 103.6934814453125,
      "activations/layer4_attention_weight_min": -110.30950927734375,
      "activations/layer5_attention_weight_max": 48.91714096069336,
      "activations/layer5_attention_weight_min": -60.57725524902344,
      "activations/layer6_attention_weight_max": 43.12201690673828,
      "activations/layer6_attention_weight_min": -45.08533477783203,
      "activations/layer7_attention_weight_max": 89.43484497070312,
      "activations/layer7_attention_weight_min": -94.25794219970703,
      "activations/layer8_attention_weight_max": 40.56693649291992,
      "activations/layer8_attention_weight_min": -41.49753952026367,
      "activations/layer9_attention_weight_max": 30.707916259765625,
      "activations/layer9_attention_weight_min": -34.67723083496094,
      "epoch": 18.67,
      "learning_rate": 2.988901515151515e-05,
      "loss": 2.7435,
      "step": 321350
    },
    {
      "activations/layer0_attention_weight_max": 14.96912670135498,
      "activations/layer0_attention_weight_min": -11.815811157226562,
      "activations/layer10_attention_weight_max": 34.992897033691406,
      "activations/layer10_attention_weight_min": -36.40706253051758,
      "activations/layer11_attention_weight_max": 35.869937896728516,
      "activations/layer11_attention_weight_min": -36.531307220458984,
      "activations/layer12_attention_weight_max": 28.986906051635742,
      "activations/layer12_attention_weight_min": -30.768421173095703,
      "activations/layer13_attention_weight_max": 43.784202575683594,
      "activations/layer13_attention_weight_min": -39.96488571166992,
      "activations/layer14_attention_weight_max": 48.30299377441406,
      "activations/layer14_attention_weight_min": -43.951995849609375,
      "activations/layer15_attention_weight_max": 44.64005661010742,
      "activations/layer15_attention_weight_min": -39.59635543823242,
      "activations/layer16_attention_weight_max": 36.33294677734375,
      "activations/layer16_attention_weight_min": -31.923246383666992,
      "activations/layer17_attention_weight_max": 32.775596618652344,
      "activations/layer17_attention_weight_min": -26.332244873046875,
      "activations/layer18_attention_weight_max": 35.32975387573242,
      "activations/layer18_attention_weight_min": -26.60781478881836,
      "activations/layer19_attention_weight_max": 38.27572250366211,
      "activations/layer19_attention_weight_min": -33.64496994018555,
      "activations/layer1_attention_weight_max": 16.050601959228516,
      "activations/layer1_attention_weight_min": -15.095130920410156,
      "activations/layer20_attention_weight_max": 32.02212905883789,
      "activations/layer20_attention_weight_min": -25.595458984375,
      "activations/layer21_attention_weight_max": 32.18109130859375,
      "activations/layer21_attention_weight_min": -24.520170211791992,
      "activations/layer22_attention_weight_max": 41.38816452026367,
      "activations/layer22_attention_weight_min": -32.846065521240234,
      "activations/layer23_attention_weight_max": 36.55390930175781,
      "activations/layer23_attention_weight_min": -26.31832504272461,
      "activations/layer2_attention_weight_max": 28.42329978942871,
      "activations/layer2_attention_weight_min": -30.38491439819336,
      "activations/layer3_attention_weight_max": 94.28559875488281,
      "activations/layer3_attention_weight_min": -100.69756317138672,
      "activations/layer4_attention_weight_max": 106.41462707519531,
      "activations/layer4_attention_weight_min": -106.16854095458984,
      "activations/layer5_attention_weight_max": 50.426658630371094,
      "activations/layer5_attention_weight_min": -63.11848068237305,
      "activations/layer6_attention_weight_max": 44.11354446411133,
      "activations/layer6_attention_weight_min": -45.751319885253906,
      "activations/layer7_attention_weight_max": 95.51026916503906,
      "activations/layer7_attention_weight_min": -92.32301330566406,
      "activations/layer8_attention_weight_max": 42.21598815917969,
      "activations/layer8_attention_weight_min": -45.15333938598633,
      "activations/layer9_attention_weight_max": 33.038177490234375,
      "activations/layer9_attention_weight_min": -37.22320556640625,
      "epoch": 18.68,
      "learning_rate": 2.9870075757575757e-05,
      "loss": 2.725,
      "step": 321400
    },
    {
      "activations/layer0_attention_weight_max": 16.112178802490234,
      "activations/layer0_attention_weight_min": -12.06273365020752,
      "activations/layer10_attention_weight_max": 35.900840759277344,
      "activations/layer10_attention_weight_min": -34.80752944946289,
      "activations/layer11_attention_weight_max": 34.65628433227539,
      "activations/layer11_attention_weight_min": -34.52949523925781,
      "activations/layer12_attention_weight_max": 27.029598236083984,
      "activations/layer12_attention_weight_min": -27.077112197875977,
      "activations/layer13_attention_weight_max": 40.72703552246094,
      "activations/layer13_attention_weight_min": -35.36798095703125,
      "activations/layer14_attention_weight_max": 48.285770416259766,
      "activations/layer14_attention_weight_min": -37.39716720581055,
      "activations/layer15_attention_weight_max": 40.952423095703125,
      "activations/layer15_attention_weight_min": -37.59107208251953,
      "activations/layer16_attention_weight_max": 30.63762092590332,
      "activations/layer16_attention_weight_min": -26.908031463623047,
      "activations/layer17_attention_weight_max": 30.87507438659668,
      "activations/layer17_attention_weight_min": -26.41243553161621,
      "activations/layer18_attention_weight_max": 33.971370697021484,
      "activations/layer18_attention_weight_min": -29.843647003173828,
      "activations/layer19_attention_weight_max": 34.852333068847656,
      "activations/layer19_attention_weight_min": -32.10285568237305,
      "activations/layer1_attention_weight_max": 15.806163787841797,
      "activations/layer1_attention_weight_min": -15.12613296508789,
      "activations/layer20_attention_weight_max": 30.54509162902832,
      "activations/layer20_attention_weight_min": -24.917367935180664,
      "activations/layer21_attention_weight_max": 30.395299911499023,
      "activations/layer21_attention_weight_min": -23.976572036743164,
      "activations/layer22_attention_weight_max": 47.750946044921875,
      "activations/layer22_attention_weight_min": -31.33872413635254,
      "activations/layer23_attention_weight_max": 37.4061279296875,
      "activations/layer23_attention_weight_min": -26.18604850769043,
      "activations/layer2_attention_weight_max": 31.113920211791992,
      "activations/layer2_attention_weight_min": -31.105758666992188,
      "activations/layer3_attention_weight_max": 97.90312957763672,
      "activations/layer3_attention_weight_min": -99.53599548339844,
      "activations/layer4_attention_weight_max": 105.1020278930664,
      "activations/layer4_attention_weight_min": -110.83756256103516,
      "activations/layer5_attention_weight_max": 51.66717529296875,
      "activations/layer5_attention_weight_min": -59.85590362548828,
      "activations/layer6_attention_weight_max": 45.55373764038086,
      "activations/layer6_attention_weight_min": -46.55128479003906,
      "activations/layer7_attention_weight_max": 99.37504577636719,
      "activations/layer7_attention_weight_min": -98.10763549804688,
      "activations/layer8_attention_weight_max": 47.3923225402832,
      "activations/layer8_attention_weight_min": -50.06621170043945,
      "activations/layer9_attention_weight_max": 35.34511184692383,
      "activations/layer9_attention_weight_min": -35.65313720703125,
      "epoch": 18.68,
      "learning_rate": 2.985113636363636e-05,
      "loss": 2.7363,
      "step": 321450
    },
    {
      "activations/layer0_attention_weight_max": 15.55456829071045,
      "activations/layer0_attention_weight_min": -12.910350799560547,
      "activations/layer10_attention_weight_max": 36.979705810546875,
      "activations/layer10_attention_weight_min": -36.892757415771484,
      "activations/layer11_attention_weight_max": 34.81071472167969,
      "activations/layer11_attention_weight_min": -35.492340087890625,
      "activations/layer12_attention_weight_max": 25.201963424682617,
      "activations/layer12_attention_weight_min": -27.255014419555664,
      "activations/layer13_attention_weight_max": 37.453033447265625,
      "activations/layer13_attention_weight_min": -37.189048767089844,
      "activations/layer14_attention_weight_max": 40.413936614990234,
      "activations/layer14_attention_weight_min": -38.92708969116211,
      "activations/layer15_attention_weight_max": 37.665771484375,
      "activations/layer15_attention_weight_min": -39.256587982177734,
      "activations/layer16_attention_weight_max": 28.199054718017578,
      "activations/layer16_attention_weight_min": -30.547134399414062,
      "activations/layer17_attention_weight_max": 31.45386505126953,
      "activations/layer17_attention_weight_min": -28.54465675354004,
      "activations/layer18_attention_weight_max": 31.3039493560791,
      "activations/layer18_attention_weight_min": -27.836030960083008,
      "activations/layer19_attention_weight_max": 31.932279586791992,
      "activations/layer19_attention_weight_min": -33.50730895996094,
      "activations/layer1_attention_weight_max": 15.507771492004395,
      "activations/layer1_attention_weight_min": -14.804929733276367,
      "activations/layer20_attention_weight_max": 28.04010581970215,
      "activations/layer20_attention_weight_min": -26.708316802978516,
      "activations/layer21_attention_weight_max": 27.726821899414062,
      "activations/layer21_attention_weight_min": -27.329998016357422,
      "activations/layer22_attention_weight_max": 38.60991287231445,
      "activations/layer22_attention_weight_min": -34.689300537109375,
      "activations/layer23_attention_weight_max": 28.245548248291016,
      "activations/layer23_attention_weight_min": -29.26349449157715,
      "activations/layer2_attention_weight_max": 30.105802536010742,
      "activations/layer2_attention_weight_min": -28.662738800048828,
      "activations/layer3_attention_weight_max": 95.87435150146484,
      "activations/layer3_attention_weight_min": -95.50691223144531,
      "activations/layer4_attention_weight_max": 103.75386810302734,
      "activations/layer4_attention_weight_min": -107.31182861328125,
      "activations/layer5_attention_weight_max": 55.58204650878906,
      "activations/layer5_attention_weight_min": -59.606773376464844,
      "activations/layer6_attention_weight_max": 44.06614303588867,
      "activations/layer6_attention_weight_min": -48.41637420654297,
      "activations/layer7_attention_weight_max": 93.70885467529297,
      "activations/layer7_attention_weight_min": -97.8569107055664,
      "activations/layer8_attention_weight_max": 45.365257263183594,
      "activations/layer8_attention_weight_min": -50.24521255493164,
      "activations/layer9_attention_weight_max": 37.12246322631836,
      "activations/layer9_attention_weight_min": -39.8446159362793,
      "epoch": 18.68,
      "learning_rate": 2.9832196969696967e-05,
      "loss": 2.7383,
      "step": 321500
    },
    {
      "activations/layer0_attention_weight_max": 16.144020080566406,
      "activations/layer0_attention_weight_min": -12.031120300292969,
      "activations/layer10_attention_weight_max": 39.108360290527344,
      "activations/layer10_attention_weight_min": -37.89557647705078,
      "activations/layer11_attention_weight_max": 38.888465881347656,
      "activations/layer11_attention_weight_min": -37.607887268066406,
      "activations/layer12_attention_weight_max": 33.3734245300293,
      "activations/layer12_attention_weight_min": -31.113752365112305,
      "activations/layer13_attention_weight_max": 51.291378021240234,
      "activations/layer13_attention_weight_min": -42.54829025268555,
      "activations/layer14_attention_weight_max": 51.86764144897461,
      "activations/layer14_attention_weight_min": -46.75397872924805,
      "activations/layer15_attention_weight_max": 51.142574310302734,
      "activations/layer15_attention_weight_min": -41.99839782714844,
      "activations/layer16_attention_weight_max": 33.597843170166016,
      "activations/layer16_attention_weight_min": -28.599645614624023,
      "activations/layer17_attention_weight_max": 31.811662673950195,
      "activations/layer17_attention_weight_min": -27.127864837646484,
      "activations/layer18_attention_weight_max": 34.625213623046875,
      "activations/layer18_attention_weight_min": -27.73784065246582,
      "activations/layer19_attention_weight_max": 34.01882553100586,
      "activations/layer19_attention_weight_min": -33.15606689453125,
      "activations/layer1_attention_weight_max": 16.495676040649414,
      "activations/layer1_attention_weight_min": -14.648804664611816,
      "activations/layer20_attention_weight_max": 30.906919479370117,
      "activations/layer20_attention_weight_min": -25.970718383789062,
      "activations/layer21_attention_weight_max": 28.876745223999023,
      "activations/layer21_attention_weight_min": -26.115245819091797,
      "activations/layer22_attention_weight_max": 42.33626174926758,
      "activations/layer22_attention_weight_min": -33.67396545410156,
      "activations/layer23_attention_weight_max": 33.23728942871094,
      "activations/layer23_attention_weight_min": -26.556087493896484,
      "activations/layer2_attention_weight_max": 32.238441467285156,
      "activations/layer2_attention_weight_min": -31.948623657226562,
      "activations/layer3_attention_weight_max": 102.13249206542969,
      "activations/layer3_attention_weight_min": -108.1095199584961,
      "activations/layer4_attention_weight_max": 111.09435272216797,
      "activations/layer4_attention_weight_min": -112.05623626708984,
      "activations/layer5_attention_weight_max": 52.7756462097168,
      "activations/layer5_attention_weight_min": -63.184364318847656,
      "activations/layer6_attention_weight_max": 44.614383697509766,
      "activations/layer6_attention_weight_min": -48.7671012878418,
      "activations/layer7_attention_weight_max": 101.4290542602539,
      "activations/layer7_attention_weight_min": -103.38975524902344,
      "activations/layer8_attention_weight_max": 44.4267578125,
      "activations/layer8_attention_weight_min": -47.49989700317383,
      "activations/layer9_attention_weight_max": 34.483036041259766,
      "activations/layer9_attention_weight_min": -37.24671936035156,
      "epoch": 18.68,
      "learning_rate": 2.9813257575757575e-05,
      "loss": 2.7279,
      "step": 321550
    },
    {
      "activations/layer0_attention_weight_max": 16.520851135253906,
      "activations/layer0_attention_weight_min": -12.261425018310547,
      "activations/layer10_attention_weight_max": 34.22751235961914,
      "activations/layer10_attention_weight_min": -32.98944854736328,
      "activations/layer11_attention_weight_max": 34.212432861328125,
      "activations/layer11_attention_weight_min": -33.160282135009766,
      "activations/layer12_attention_weight_max": 28.140506744384766,
      "activations/layer12_attention_weight_min": -28.153671264648438,
      "activations/layer13_attention_weight_max": 41.652618408203125,
      "activations/layer13_attention_weight_min": -39.25871658325195,
      "activations/layer14_attention_weight_max": 42.467803955078125,
      "activations/layer14_attention_weight_min": -40.076473236083984,
      "activations/layer15_attention_weight_max": 38.88042068481445,
      "activations/layer15_attention_weight_min": -38.296810150146484,
      "activations/layer16_attention_weight_max": 33.08472442626953,
      "activations/layer16_attention_weight_min": -29.708600997924805,
      "activations/layer17_attention_weight_max": 31.567626953125,
      "activations/layer17_attention_weight_min": -27.389938354492188,
      "activations/layer18_attention_weight_max": 32.62335205078125,
      "activations/layer18_attention_weight_min": -27.649616241455078,
      "activations/layer19_attention_weight_max": 39.31058120727539,
      "activations/layer19_attention_weight_min": -33.482479095458984,
      "activations/layer1_attention_weight_max": 17.267314910888672,
      "activations/layer1_attention_weight_min": -14.137103080749512,
      "activations/layer20_attention_weight_max": 30.205686569213867,
      "activations/layer20_attention_weight_min": -26.226350784301758,
      "activations/layer21_attention_weight_max": 29.718503952026367,
      "activations/layer21_attention_weight_min": -24.368144989013672,
      "activations/layer22_attention_weight_max": 46.00482177734375,
      "activations/layer22_attention_weight_min": -30.829113006591797,
      "activations/layer23_attention_weight_max": 36.73347473144531,
      "activations/layer23_attention_weight_min": -27.452123641967773,
      "activations/layer2_attention_weight_max": 30.7117977142334,
      "activations/layer2_attention_weight_min": -30.058773040771484,
      "activations/layer3_attention_weight_max": 96.72183227539062,
      "activations/layer3_attention_weight_min": -95.25741577148438,
      "activations/layer4_attention_weight_max": 104.53364562988281,
      "activations/layer4_attention_weight_min": -109.52137756347656,
      "activations/layer5_attention_weight_max": 51.63957977294922,
      "activations/layer5_attention_weight_min": -59.828956604003906,
      "activations/layer6_attention_weight_max": 46.06150436401367,
      "activations/layer6_attention_weight_min": -45.28238296508789,
      "activations/layer7_attention_weight_max": 93.11754608154297,
      "activations/layer7_attention_weight_min": -94.0661849975586,
      "activations/layer8_attention_weight_max": 42.17818832397461,
      "activations/layer8_attention_weight_min": -43.27383804321289,
      "activations/layer9_attention_weight_max": 31.907917022705078,
      "activations/layer9_attention_weight_min": -32.3469123840332,
      "epoch": 18.69,
      "learning_rate": 2.979431818181818e-05,
      "loss": 2.7139,
      "step": 321600
    },
    {
      "activations/layer0_attention_weight_max": 16.856355667114258,
      "activations/layer0_attention_weight_min": -12.663121223449707,
      "activations/layer10_attention_weight_max": 34.0467414855957,
      "activations/layer10_attention_weight_min": -35.96965408325195,
      "activations/layer11_attention_weight_max": 33.81200408935547,
      "activations/layer11_attention_weight_min": -34.46497344970703,
      "activations/layer12_attention_weight_max": 27.32717514038086,
      "activations/layer12_attention_weight_min": -29.637733459472656,
      "activations/layer13_attention_weight_max": 42.12282943725586,
      "activations/layer13_attention_weight_min": -36.95072937011719,
      "activations/layer14_attention_weight_max": 43.58330535888672,
      "activations/layer14_attention_weight_min": -39.86947250366211,
      "activations/layer15_attention_weight_max": 41.176063537597656,
      "activations/layer15_attention_weight_min": -38.30952072143555,
      "activations/layer16_attention_weight_max": 29.82912254333496,
      "activations/layer16_attention_weight_min": -28.093734741210938,
      "activations/layer17_attention_weight_max": 30.57821273803711,
      "activations/layer17_attention_weight_min": -26.614336013793945,
      "activations/layer18_attention_weight_max": 31.029296875,
      "activations/layer18_attention_weight_min": -25.545602798461914,
      "activations/layer19_attention_weight_max": 35.766841888427734,
      "activations/layer19_attention_weight_min": -31.826719284057617,
      "activations/layer1_attention_weight_max": 15.48938274383545,
      "activations/layer1_attention_weight_min": -13.530923843383789,
      "activations/layer20_attention_weight_max": 28.05156707763672,
      "activations/layer20_attention_weight_min": -25.990842819213867,
      "activations/layer21_attention_weight_max": 28.833181381225586,
      "activations/layer21_attention_weight_min": -25.308897018432617,
      "activations/layer22_attention_weight_max": 45.74562072753906,
      "activations/layer22_attention_weight_min": -31.791107177734375,
      "activations/layer23_attention_weight_max": 33.89215087890625,
      "activations/layer23_attention_weight_min": -28.550107955932617,
      "activations/layer2_attention_weight_max": 29.071128845214844,
      "activations/layer2_attention_weight_min": -29.613216400146484,
      "activations/layer3_attention_weight_max": 92.6663818359375,
      "activations/layer3_attention_weight_min": -94.476806640625,
      "activations/layer4_attention_weight_max": 99.961181640625,
      "activations/layer4_attention_weight_min": -106.37584686279297,
      "activations/layer5_attention_weight_max": 47.138916015625,
      "activations/layer5_attention_weight_min": -60.05120086669922,
      "activations/layer6_attention_weight_max": 43.77478790283203,
      "activations/layer6_attention_weight_min": -46.79584884643555,
      "activations/layer7_attention_weight_max": 92.17342376708984,
      "activations/layer7_attention_weight_min": -105.96158599853516,
      "activations/layer8_attention_weight_max": 40.3072509765625,
      "activations/layer8_attention_weight_min": -43.97541809082031,
      "activations/layer9_attention_weight_max": 32.02848815917969,
      "activations/layer9_attention_weight_min": -34.538047790527344,
      "epoch": 18.69,
      "learning_rate": 2.9775378787878788e-05,
      "loss": 2.7262,
      "step": 321650
    },
    {
      "activations/layer0_attention_weight_max": 15.244365692138672,
      "activations/layer0_attention_weight_min": -11.884540557861328,
      "activations/layer10_attention_weight_max": 34.76984405517578,
      "activations/layer10_attention_weight_min": -34.923500061035156,
      "activations/layer11_attention_weight_max": 37.47028732299805,
      "activations/layer11_attention_weight_min": -35.449737548828125,
      "activations/layer12_attention_weight_max": 29.139087677001953,
      "activations/layer12_attention_weight_min": -29.263999938964844,
      "activations/layer13_attention_weight_max": 43.549171447753906,
      "activations/layer13_attention_weight_min": -39.94724655151367,
      "activations/layer14_attention_weight_max": 44.36934280395508,
      "activations/layer14_attention_weight_min": -41.781776428222656,
      "activations/layer15_attention_weight_max": 43.78046798706055,
      "activations/layer15_attention_weight_min": -41.87077713012695,
      "activations/layer16_attention_weight_max": 33.54827880859375,
      "activations/layer16_attention_weight_min": -27.47549057006836,
      "activations/layer17_attention_weight_max": 31.1968994140625,
      "activations/layer17_attention_weight_min": -27.347116470336914,
      "activations/layer18_attention_weight_max": 33.41584396362305,
      "activations/layer18_attention_weight_min": -29.694583892822266,
      "activations/layer19_attention_weight_max": 38.50058364868164,
      "activations/layer19_attention_weight_min": -32.669124603271484,
      "activations/layer1_attention_weight_max": 15.428230285644531,
      "activations/layer1_attention_weight_min": -13.937255859375,
      "activations/layer20_attention_weight_max": 32.65031433105469,
      "activations/layer20_attention_weight_min": -26.534151077270508,
      "activations/layer21_attention_weight_max": 31.649856567382812,
      "activations/layer21_attention_weight_min": -26.32438087463379,
      "activations/layer22_attention_weight_max": 43.7271728515625,
      "activations/layer22_attention_weight_min": -31.89816665649414,
      "activations/layer23_attention_weight_max": 38.159767150878906,
      "activations/layer23_attention_weight_min": -27.731651306152344,
      "activations/layer2_attention_weight_max": 30.642948150634766,
      "activations/layer2_attention_weight_min": -31.33968734741211,
      "activations/layer3_attention_weight_max": 97.27342987060547,
      "activations/layer3_attention_weight_min": -99.02332305908203,
      "activations/layer4_attention_weight_max": 106.31096649169922,
      "activations/layer4_attention_weight_min": -105.91332244873047,
      "activations/layer5_attention_weight_max": 52.04810333251953,
      "activations/layer5_attention_weight_min": -62.0125732421875,
      "activations/layer6_attention_weight_max": 43.93089294433594,
      "activations/layer6_attention_weight_min": -48.162391662597656,
      "activations/layer7_attention_weight_max": 96.22233581542969,
      "activations/layer7_attention_weight_min": -98.7457275390625,
      "activations/layer8_attention_weight_max": 41.38310241699219,
      "activations/layer8_attention_weight_min": -46.1624641418457,
      "activations/layer9_attention_weight_max": 35.10939407348633,
      "activations/layer9_attention_weight_min": -34.0911750793457,
      "epoch": 18.69,
      "learning_rate": 2.975643939393939e-05,
      "loss": 2.7351,
      "step": 321700
    },
    {
      "activations/layer0_attention_weight_max": 15.94619369506836,
      "activations/layer0_attention_weight_min": -12.212302207946777,
      "activations/layer10_attention_weight_max": 46.395938873291016,
      "activations/layer10_attention_weight_min": -42.75135040283203,
      "activations/layer11_attention_weight_max": 44.97312545776367,
      "activations/layer11_attention_weight_min": -44.256649017333984,
      "activations/layer12_attention_weight_max": 26.68102264404297,
      "activations/layer12_attention_weight_min": -27.459917068481445,
      "activations/layer13_attention_weight_max": 40.86301803588867,
      "activations/layer13_attention_weight_min": -36.05933380126953,
      "activations/layer14_attention_weight_max": 43.158180236816406,
      "activations/layer14_attention_weight_min": -38.92631912231445,
      "activations/layer15_attention_weight_max": 41.48141098022461,
      "activations/layer15_attention_weight_min": -36.13251495361328,
      "activations/layer16_attention_weight_max": 28.73235511779785,
      "activations/layer16_attention_weight_min": -28.049043655395508,
      "activations/layer17_attention_weight_max": 32.98440170288086,
      "activations/layer17_attention_weight_min": -25.891883850097656,
      "activations/layer18_attention_weight_max": 32.04808807373047,
      "activations/layer18_attention_weight_min": -25.47310447692871,
      "activations/layer19_attention_weight_max": 29.343536376953125,
      "activations/layer19_attention_weight_min": -29.86543846130371,
      "activations/layer1_attention_weight_max": 16.884584426879883,
      "activations/layer1_attention_weight_min": -14.133648872375488,
      "activations/layer20_attention_weight_max": 25.497907638549805,
      "activations/layer20_attention_weight_min": -25.634693145751953,
      "activations/layer21_attention_weight_max": 23.962297439575195,
      "activations/layer21_attention_weight_min": -24.944265365600586,
      "activations/layer22_attention_weight_max": 37.37310028076172,
      "activations/layer22_attention_weight_min": -30.29343605041504,
      "activations/layer23_attention_weight_max": 28.020999908447266,
      "activations/layer23_attention_weight_min": -24.736143112182617,
      "activations/layer2_attention_weight_max": 33.60667419433594,
      "activations/layer2_attention_weight_min": -34.55653381347656,
      "activations/layer3_attention_weight_max": 98.2545166015625,
      "activations/layer3_attention_weight_min": -100.7269058227539,
      "activations/layer4_attention_weight_max": 106.67891693115234,
      "activations/layer4_attention_weight_min": -107.59526824951172,
      "activations/layer5_attention_weight_max": 55.955284118652344,
      "activations/layer5_attention_weight_min": -62.768821716308594,
      "activations/layer6_attention_weight_max": 44.51947784423828,
      "activations/layer6_attention_weight_min": -47.36445236206055,
      "activations/layer7_attention_weight_max": 111.27766418457031,
      "activations/layer7_attention_weight_min": -98.4534912109375,
      "activations/layer8_attention_weight_max": 54.08565902709961,
      "activations/layer8_attention_weight_min": -47.55125427246094,
      "activations/layer9_attention_weight_max": 38.96024703979492,
      "activations/layer9_attention_weight_min": -42.125938415527344,
      "epoch": 18.7,
      "learning_rate": 2.9737499999999998e-05,
      "loss": 2.7587,
      "step": 321750
    },
    {
      "activations/layer0_attention_weight_max": 15.84408950805664,
      "activations/layer0_attention_weight_min": -13.513846397399902,
      "activations/layer10_attention_weight_max": 35.10870361328125,
      "activations/layer10_attention_weight_min": -34.446956634521484,
      "activations/layer11_attention_weight_max": 33.14476013183594,
      "activations/layer11_attention_weight_min": -35.810394287109375,
      "activations/layer12_attention_weight_max": 28.086488723754883,
      "activations/layer12_attention_weight_min": -27.31096839904785,
      "activations/layer13_attention_weight_max": 40.659645080566406,
      "activations/layer13_attention_weight_min": -35.71656799316406,
      "activations/layer14_attention_weight_max": 45.20830154418945,
      "activations/layer14_attention_weight_min": -38.56150817871094,
      "activations/layer15_attention_weight_max": 39.71205139160156,
      "activations/layer15_attention_weight_min": -35.550838470458984,
      "activations/layer16_attention_weight_max": 31.103803634643555,
      "activations/layer16_attention_weight_min": -28.829524993896484,
      "activations/layer17_attention_weight_max": 31.710575103759766,
      "activations/layer17_attention_weight_min": -28.000164031982422,
      "activations/layer18_attention_weight_max": 33.91391372680664,
      "activations/layer18_attention_weight_min": -24.3542537689209,
      "activations/layer19_attention_weight_max": 32.904720306396484,
      "activations/layer19_attention_weight_min": -31.171924591064453,
      "activations/layer1_attention_weight_max": 16.374303817749023,
      "activations/layer1_attention_weight_min": -14.41240406036377,
      "activations/layer20_attention_weight_max": 28.634033203125,
      "activations/layer20_attention_weight_min": -25.749027252197266,
      "activations/layer21_attention_weight_max": 28.45408058166504,
      "activations/layer21_attention_weight_min": -24.85074806213379,
      "activations/layer22_attention_weight_max": 41.79984664916992,
      "activations/layer22_attention_weight_min": -30.3790340423584,
      "activations/layer23_attention_weight_max": 35.63719177246094,
      "activations/layer23_attention_weight_min": -26.085050582885742,
      "activations/layer2_attention_weight_max": 28.959266662597656,
      "activations/layer2_attention_weight_min": -29.53347396850586,
      "activations/layer3_attention_weight_max": 91.62664794921875,
      "activations/layer3_attention_weight_min": -96.74386596679688,
      "activations/layer4_attention_weight_max": 100.1976547241211,
      "activations/layer4_attention_weight_min": -104.51616668701172,
      "activations/layer5_attention_weight_max": 49.339393615722656,
      "activations/layer5_attention_weight_min": -64.15328979492188,
      "activations/layer6_attention_weight_max": 44.3808708190918,
      "activations/layer6_attention_weight_min": -46.022884368896484,
      "activations/layer7_attention_weight_max": 92.20026397705078,
      "activations/layer7_attention_weight_min": -92.25887298583984,
      "activations/layer8_attention_weight_max": 41.41371536254883,
      "activations/layer8_attention_weight_min": -42.96928787231445,
      "activations/layer9_attention_weight_max": 33.681663513183594,
      "activations/layer9_attention_weight_min": -34.793914794921875,
      "epoch": 18.7,
      "learning_rate": 2.9718560606060603e-05,
      "loss": 2.73,
      "step": 321800
    },
    {
      "activations/layer0_attention_weight_max": 15.680014610290527,
      "activations/layer0_attention_weight_min": -12.020661354064941,
      "activations/layer10_attention_weight_max": 39.407493591308594,
      "activations/layer10_attention_weight_min": -34.19856643676758,
      "activations/layer11_attention_weight_max": 39.514060974121094,
      "activations/layer11_attention_weight_min": -35.37468338012695,
      "activations/layer12_attention_weight_max": 25.18187713623047,
      "activations/layer12_attention_weight_min": -29.043306350708008,
      "activations/layer13_attention_weight_max": 37.12152099609375,
      "activations/layer13_attention_weight_min": -33.64727783203125,
      "activations/layer14_attention_weight_max": 41.3946418762207,
      "activations/layer14_attention_weight_min": -36.848140716552734,
      "activations/layer15_attention_weight_max": 40.753746032714844,
      "activations/layer15_attention_weight_min": -37.095054626464844,
      "activations/layer16_attention_weight_max": 27.573806762695312,
      "activations/layer16_attention_weight_min": -27.840248107910156,
      "activations/layer17_attention_weight_max": 27.85173797607422,
      "activations/layer17_attention_weight_min": -26.396350860595703,
      "activations/layer18_attention_weight_max": 30.731361389160156,
      "activations/layer18_attention_weight_min": -27.027183532714844,
      "activations/layer19_attention_weight_max": 33.25946807861328,
      "activations/layer19_attention_weight_min": -33.00111770629883,
      "activations/layer1_attention_weight_max": 16.205408096313477,
      "activations/layer1_attention_weight_min": -13.764763832092285,
      "activations/layer20_attention_weight_max": 28.885467529296875,
      "activations/layer20_attention_weight_min": -25.72146224975586,
      "activations/layer21_attention_weight_max": 28.543106079101562,
      "activations/layer21_attention_weight_min": -25.657751083374023,
      "activations/layer22_attention_weight_max": 41.50117492675781,
      "activations/layer22_attention_weight_min": -33.8781623840332,
      "activations/layer23_attention_weight_max": 31.634389877319336,
      "activations/layer23_attention_weight_min": -26.847074508666992,
      "activations/layer2_attention_weight_max": 29.81641387939453,
      "activations/layer2_attention_weight_min": -30.131460189819336,
      "activations/layer3_attention_weight_max": 95.23800659179688,
      "activations/layer3_attention_weight_min": -95.81962585449219,
      "activations/layer4_attention_weight_max": 103.32537841796875,
      "activations/layer4_attention_weight_min": -103.30450439453125,
      "activations/layer5_attention_weight_max": 50.25690841674805,
      "activations/layer5_attention_weight_min": -61.39617919921875,
      "activations/layer6_attention_weight_max": 42.15983200073242,
      "activations/layer6_attention_weight_min": -45.30126190185547,
      "activations/layer7_attention_weight_max": 92.20250701904297,
      "activations/layer7_attention_weight_min": -90.59652709960938,
      "activations/layer8_attention_weight_max": 41.49271011352539,
      "activations/layer8_attention_weight_min": -43.58277893066406,
      "activations/layer9_attention_weight_max": 32.8421745300293,
      "activations/layer9_attention_weight_min": -33.78908920288086,
      "epoch": 18.7,
      "learning_rate": 2.969962121212121e-05,
      "loss": 2.7205,
      "step": 321850
    },
    {
      "activations/layer0_attention_weight_max": 16.4967098236084,
      "activations/layer0_attention_weight_min": -12.294458389282227,
      "activations/layer10_attention_weight_max": 33.83940887451172,
      "activations/layer10_attention_weight_min": -34.68909454345703,
      "activations/layer11_attention_weight_max": 34.95752716064453,
      "activations/layer11_attention_weight_min": -32.995697021484375,
      "activations/layer12_attention_weight_max": 26.177392959594727,
      "activations/layer12_attention_weight_min": -25.841325759887695,
      "activations/layer13_attention_weight_max": 39.681396484375,
      "activations/layer13_attention_weight_min": -34.68204879760742,
      "activations/layer14_attention_weight_max": 44.594974517822266,
      "activations/layer14_attention_weight_min": -38.51179122924805,
      "activations/layer15_attention_weight_max": 38.99369430541992,
      "activations/layer15_attention_weight_min": -34.97004318237305,
      "activations/layer16_attention_weight_max": 30.4263916015625,
      "activations/layer16_attention_weight_min": -27.63591194152832,
      "activations/layer17_attention_weight_max": 29.554311752319336,
      "activations/layer17_attention_weight_min": -25.977571487426758,
      "activations/layer18_attention_weight_max": 32.3781852722168,
      "activations/layer18_attention_weight_min": -25.267183303833008,
      "activations/layer19_attention_weight_max": 34.84710693359375,
      "activations/layer19_attention_weight_min": -30.992834091186523,
      "activations/layer1_attention_weight_max": 15.972403526306152,
      "activations/layer1_attention_weight_min": -15.426689147949219,
      "activations/layer20_attention_weight_max": 28.649396896362305,
      "activations/layer20_attention_weight_min": -24.895740509033203,
      "activations/layer21_attention_weight_max": 29.084075927734375,
      "activations/layer21_attention_weight_min": -24.667131423950195,
      "activations/layer22_attention_weight_max": 47.26860809326172,
      "activations/layer22_attention_weight_min": -31.598079681396484,
      "activations/layer23_attention_weight_max": 34.74354553222656,
      "activations/layer23_attention_weight_min": -25.965816497802734,
      "activations/layer2_attention_weight_max": 31.54828643798828,
      "activations/layer2_attention_weight_min": -30.403738021850586,
      "activations/layer3_attention_weight_max": 97.53377532958984,
      "activations/layer3_attention_weight_min": -97.83025360107422,
      "activations/layer4_attention_weight_max": 105.4715805053711,
      "activations/layer4_attention_weight_min": -104.947265625,
      "activations/layer5_attention_weight_max": 49.92118835449219,
      "activations/layer5_attention_weight_min": -61.887813568115234,
      "activations/layer6_attention_weight_max": 45.13539123535156,
      "activations/layer6_attention_weight_min": -47.90456008911133,
      "activations/layer7_attention_weight_max": 93.9276351928711,
      "activations/layer7_attention_weight_min": -96.70148468017578,
      "activations/layer8_attention_weight_max": 42.18974304199219,
      "activations/layer8_attention_weight_min": -45.42270278930664,
      "activations/layer9_attention_weight_max": 34.12638473510742,
      "activations/layer9_attention_weight_min": -34.66300582885742,
      "epoch": 18.7,
      "learning_rate": 2.9680681818181816e-05,
      "loss": 2.7305,
      "step": 321900
    },
    {
      "activations/layer0_attention_weight_max": 16.03150177001953,
      "activations/layer0_attention_weight_min": -12.733019828796387,
      "activations/layer10_attention_weight_max": 36.63819885253906,
      "activations/layer10_attention_weight_min": -34.35470199584961,
      "activations/layer11_attention_weight_max": 34.79633331298828,
      "activations/layer11_attention_weight_min": -36.046417236328125,
      "activations/layer12_attention_weight_max": 28.769046783447266,
      "activations/layer12_attention_weight_min": -27.260269165039062,
      "activations/layer13_attention_weight_max": 39.17008972167969,
      "activations/layer13_attention_weight_min": -37.08864974975586,
      "activations/layer14_attention_weight_max": 42.043922424316406,
      "activations/layer14_attention_weight_min": -37.4185676574707,
      "activations/layer15_attention_weight_max": 39.809173583984375,
      "activations/layer15_attention_weight_min": -36.82290267944336,
      "activations/layer16_attention_weight_max": 28.859819412231445,
      "activations/layer16_attention_weight_min": -29.102998733520508,
      "activations/layer17_attention_weight_max": 32.106380462646484,
      "activations/layer17_attention_weight_min": -27.564167022705078,
      "activations/layer18_attention_weight_max": 34.07360076904297,
      "activations/layer18_attention_weight_min": -27.581134796142578,
      "activations/layer19_attention_weight_max": 36.346248626708984,
      "activations/layer19_attention_weight_min": -35.51192092895508,
      "activations/layer1_attention_weight_max": 16.983150482177734,
      "activations/layer1_attention_weight_min": -14.310317039489746,
      "activations/layer20_attention_weight_max": 31.32073211669922,
      "activations/layer20_attention_weight_min": -28.308414459228516,
      "activations/layer21_attention_weight_max": 31.202041625976562,
      "activations/layer21_attention_weight_min": -27.582717895507812,
      "activations/layer22_attention_weight_max": 43.21640396118164,
      "activations/layer22_attention_weight_min": -35.9238166809082,
      "activations/layer23_attention_weight_max": 33.528114318847656,
      "activations/layer23_attention_weight_min": -27.60953140258789,
      "activations/layer2_attention_weight_max": 29.259449005126953,
      "activations/layer2_attention_weight_min": -30.7371883392334,
      "activations/layer3_attention_weight_max": 97.63848876953125,
      "activations/layer3_attention_weight_min": -101.05912017822266,
      "activations/layer4_attention_weight_max": 106.75865936279297,
      "activations/layer4_attention_weight_min": -109.06450653076172,
      "activations/layer5_attention_weight_max": 54.64862060546875,
      "activations/layer5_attention_weight_min": -61.5758056640625,
      "activations/layer6_attention_weight_max": 44.79330825805664,
      "activations/layer6_attention_weight_min": -47.8000602722168,
      "activations/layer7_attention_weight_max": 92.88002014160156,
      "activations/layer7_attention_weight_min": -99.41414642333984,
      "activations/layer8_attention_weight_max": 42.15260696411133,
      "activations/layer8_attention_weight_min": -44.12732696533203,
      "activations/layer9_attention_weight_max": 33.83020782470703,
      "activations/layer9_attention_weight_min": -35.29623031616211,
      "epoch": 18.71,
      "learning_rate": 2.9661742424242424e-05,
      "loss": 2.7365,
      "step": 321950
    },
    {
      "activations/layer0_attention_weight_max": 15.76569652557373,
      "activations/layer0_attention_weight_min": -12.144976615905762,
      "activations/layer10_attention_weight_max": 33.65534210205078,
      "activations/layer10_attention_weight_min": -33.575950622558594,
      "activations/layer11_attention_weight_max": 33.7255859375,
      "activations/layer11_attention_weight_min": -33.65744400024414,
      "activations/layer12_attention_weight_max": 28.29468536376953,
      "activations/layer12_attention_weight_min": -28.01276969909668,
      "activations/layer13_attention_weight_max": 43.59891891479492,
      "activations/layer13_attention_weight_min": -36.87908935546875,
      "activations/layer14_attention_weight_max": 44.13821029663086,
      "activations/layer14_attention_weight_min": -40.6511116027832,
      "activations/layer15_attention_weight_max": 41.809818267822266,
      "activations/layer15_attention_weight_min": -36.74632263183594,
      "activations/layer16_attention_weight_max": 29.97728157043457,
      "activations/layer16_attention_weight_min": -28.313371658325195,
      "activations/layer17_attention_weight_max": 29.919376373291016,
      "activations/layer17_attention_weight_min": -26.990869522094727,
      "activations/layer18_attention_weight_max": 31.977157592773438,
      "activations/layer18_attention_weight_min": -26.249252319335938,
      "activations/layer19_attention_weight_max": 32.61977767944336,
      "activations/layer19_attention_weight_min": -31.37601661682129,
      "activations/layer1_attention_weight_max": 15.044057846069336,
      "activations/layer1_attention_weight_min": -14.484387397766113,
      "activations/layer20_attention_weight_max": 28.33917808532715,
      "activations/layer20_attention_weight_min": -26.373699188232422,
      "activations/layer21_attention_weight_max": 28.138439178466797,
      "activations/layer21_attention_weight_min": -25.174184799194336,
      "activations/layer22_attention_weight_max": 44.06877899169922,
      "activations/layer22_attention_weight_min": -32.43960189819336,
      "activations/layer23_attention_weight_max": 31.95992660522461,
      "activations/layer23_attention_weight_min": -25.06390953063965,
      "activations/layer2_attention_weight_max": 33.04876708984375,
      "activations/layer2_attention_weight_min": -29.808683395385742,
      "activations/layer3_attention_weight_max": 94.28825378417969,
      "activations/layer3_attention_weight_min": -100.24972534179688,
      "activations/layer4_attention_weight_max": 103.69010162353516,
      "activations/layer4_attention_weight_min": -109.6974105834961,
      "activations/layer5_attention_weight_max": 49.150611877441406,
      "activations/layer5_attention_weight_min": -62.200706481933594,
      "activations/layer6_attention_weight_max": 43.14045715332031,
      "activations/layer6_attention_weight_min": -47.6125602722168,
      "activations/layer7_attention_weight_max": 87.18610382080078,
      "activations/layer7_attention_weight_min": -93.23657989501953,
      "activations/layer8_attention_weight_max": 40.32243728637695,
      "activations/layer8_attention_weight_min": -43.10371017456055,
      "activations/layer9_attention_weight_max": 31.293821334838867,
      "activations/layer9_attention_weight_min": -33.51058578491211,
      "epoch": 18.71,
      "learning_rate": 2.9642803030303026e-05,
      "loss": 2.7428,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.4906,
      "eval_samples_per_second": 505.735,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.4906,
      "eval_openwebtext_samples_per_second": 505.735,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9768,
      "eval_wikitext_samples_per_second": 230.67,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_lambada_loss": 2.306640625,
      "eval_lambada_ppl": 10.040637668042795,
      "eval_lambada_runtime": 9.5809,
      "eval_lambada_samples_per_second": 508.197,
      "step": 322000
    },
    {
      "activations/layer0_attention_weight_max": 17.35478401184082,
      "activations/layer0_attention_weight_min": -12.095112800598145,
      "activations/layer10_attention_weight_max": 30.337919235229492,
      "activations/layer10_attention_weight_min": -35.74147415161133,
      "activations/layer11_attention_weight_max": 31.828702926635742,
      "activations/layer11_attention_weight_min": -32.77791976928711,
      "activations/layer12_attention_weight_max": 26.202774047851562,
      "activations/layer12_attention_weight_min": -26.1396427154541,
      "activations/layer13_attention_weight_max": 39.95771789550781,
      "activations/layer13_attention_weight_min": -34.90412902832031,
      "activations/layer14_attention_weight_max": 41.221412658691406,
      "activations/layer14_attention_weight_min": -36.24382019042969,
      "activations/layer15_attention_weight_max": 40.963191986083984,
      "activations/layer15_attention_weight_min": -35.584999084472656,
      "activations/layer16_attention_weight_max": 27.085308074951172,
      "activations/layer16_attention_weight_min": -28.391984939575195,
      "activations/layer17_attention_weight_max": 28.34332847595215,
      "activations/layer17_attention_weight_min": -27.027080535888672,
      "activations/layer18_attention_weight_max": 31.28693199157715,
      "activations/layer18_attention_weight_min": -28.48699951171875,
      "activations/layer19_attention_weight_max": 32.047607421875,
      "activations/layer19_attention_weight_min": -32.87504577636719,
      "activations/layer1_attention_weight_max": 16.10045623779297,
      "activations/layer1_attention_weight_min": -13.936223983764648,
      "activations/layer20_attention_weight_max": 27.788894653320312,
      "activations/layer20_attention_weight_min": -26.945295333862305,
      "activations/layer21_attention_weight_max": 29.90665626525879,
      "activations/layer21_attention_weight_min": -25.135936737060547,
      "activations/layer22_attention_weight_max": 41.54928207397461,
      "activations/layer22_attention_weight_min": -33.78812026977539,
      "activations/layer23_attention_weight_max": 31.88551902770996,
      "activations/layer23_attention_weight_min": -26.39751434326172,
      "activations/layer2_attention_weight_max": 29.676692962646484,
      "activations/layer2_attention_weight_min": -32.27817153930664,
      "activations/layer3_attention_weight_max": 97.4345932006836,
      "activations/layer3_attention_weight_min": -94.82511138916016,
      "activations/layer4_attention_weight_max": 100.93135833740234,
      "activations/layer4_attention_weight_min": -106.97948455810547,
      "activations/layer5_attention_weight_max": 50.93659973144531,
      "activations/layer5_attention_weight_min": -59.090049743652344,
      "activations/layer6_attention_weight_max": 43.15088653564453,
      "activations/layer6_attention_weight_min": -46.48683547973633,
      "activations/layer7_attention_weight_max": 86.15757751464844,
      "activations/layer7_attention_weight_min": -92.81881713867188,
      "activations/layer8_attention_weight_max": 37.952884674072266,
      "activations/layer8_attention_weight_min": -46.82108688354492,
      "activations/layer9_attention_weight_max": 30.571847915649414,
      "activations/layer9_attention_weight_min": -36.523399353027344,
      "epoch": 18.71,
      "learning_rate": 2.9623863636363634e-05,
      "loss": 2.7392,
      "step": 322050
    },
    {
      "activations/layer0_attention_weight_max": 15.897834777832031,
      "activations/layer0_attention_weight_min": -13.19413948059082,
      "activations/layer10_attention_weight_max": 33.160011291503906,
      "activations/layer10_attention_weight_min": -33.82680130004883,
      "activations/layer11_attention_weight_max": 33.414608001708984,
      "activations/layer11_attention_weight_min": -33.64104080200195,
      "activations/layer12_attention_weight_max": 25.993242263793945,
      "activations/layer12_attention_weight_min": -25.909164428710938,
      "activations/layer13_attention_weight_max": 38.50655746459961,
      "activations/layer13_attention_weight_min": -34.701759338378906,
      "activations/layer14_attention_weight_max": 40.47108459472656,
      "activations/layer14_attention_weight_min": -39.155826568603516,
      "activations/layer15_attention_weight_max": 42.15754318237305,
      "activations/layer15_attention_weight_min": -35.745391845703125,
      "activations/layer16_attention_weight_max": 29.377023696899414,
      "activations/layer16_attention_weight_min": -27.494218826293945,
      "activations/layer17_attention_weight_max": 31.80826187133789,
      "activations/layer17_attention_weight_min": -24.64267349243164,
      "activations/layer18_attention_weight_max": 31.480998992919922,
      "activations/layer18_attention_weight_min": -24.82552719116211,
      "activations/layer19_attention_weight_max": 34.652618408203125,
      "activations/layer19_attention_weight_min": -29.29908561706543,
      "activations/layer1_attention_weight_max": 15.603816032409668,
      "activations/layer1_attention_weight_min": -13.883838653564453,
      "activations/layer20_attention_weight_max": 34.96477127075195,
      "activations/layer20_attention_weight_min": -24.141050338745117,
      "activations/layer21_attention_weight_max": 33.25059509277344,
      "activations/layer21_attention_weight_min": -24.096216201782227,
      "activations/layer22_attention_weight_max": 41.69911193847656,
      "activations/layer22_attention_weight_min": -31.097286224365234,
      "activations/layer23_attention_weight_max": 32.805824279785156,
      "activations/layer23_attention_weight_min": -24.968948364257812,
      "activations/layer2_attention_weight_max": 30.62982177734375,
      "activations/layer2_attention_weight_min": -28.125654220581055,
      "activations/layer3_attention_weight_max": 91.76526641845703,
      "activations/layer3_attention_weight_min": -92.87618255615234,
      "activations/layer4_attention_weight_max": 101.40428161621094,
      "activations/layer4_attention_weight_min": -110.7593765258789,
      "activations/layer5_attention_weight_max": 47.5445671081543,
      "activations/layer5_attention_weight_min": -62.826507568359375,
      "activations/layer6_attention_weight_max": 42.74592590332031,
      "activations/layer6_attention_weight_min": -46.203819274902344,
      "activations/layer7_attention_weight_max": 92.22905731201172,
      "activations/layer7_attention_weight_min": -96.36343383789062,
      "activations/layer8_attention_weight_max": 40.24724197387695,
      "activations/layer8_attention_weight_min": -42.25477600097656,
      "activations/layer9_attention_weight_max": 32.741065979003906,
      "activations/layer9_attention_weight_min": -33.41073226928711,
      "epoch": 18.72,
      "learning_rate": 2.960492424242424e-05,
      "loss": 2.7408,
      "step": 322100
    },
    {
      "activations/layer0_attention_weight_max": 16.110143661499023,
      "activations/layer0_attention_weight_min": -12.528830528259277,
      "activations/layer10_attention_weight_max": 35.56350326538086,
      "activations/layer10_attention_weight_min": -35.14594268798828,
      "activations/layer11_attention_weight_max": 37.30064392089844,
      "activations/layer11_attention_weight_min": -38.31182098388672,
      "activations/layer12_attention_weight_max": 30.43415641784668,
      "activations/layer12_attention_weight_min": -28.10859489440918,
      "activations/layer13_attention_weight_max": 45.16181564331055,
      "activations/layer13_attention_weight_min": -37.466304779052734,
      "activations/layer14_attention_weight_max": 50.07283020019531,
      "activations/layer14_attention_weight_min": -41.57806396484375,
      "activations/layer15_attention_weight_max": 47.36632537841797,
      "activations/layer15_attention_weight_min": -39.02745819091797,
      "activations/layer16_attention_weight_max": 36.07607650756836,
      "activations/layer16_attention_weight_min": -28.708452224731445,
      "activations/layer17_attention_weight_max": 30.955902099609375,
      "activations/layer17_attention_weight_min": -26.207826614379883,
      "activations/layer18_attention_weight_max": 31.628671646118164,
      "activations/layer18_attention_weight_min": -26.415800094604492,
      "activations/layer19_attention_weight_max": 37.06745529174805,
      "activations/layer19_attention_weight_min": -33.57160568237305,
      "activations/layer1_attention_weight_max": 16.00906753540039,
      "activations/layer1_attention_weight_min": -14.86984634399414,
      "activations/layer20_attention_weight_max": 31.502063751220703,
      "activations/layer20_attention_weight_min": -26.097116470336914,
      "activations/layer21_attention_weight_max": 28.351774215698242,
      "activations/layer21_attention_weight_min": -24.655996322631836,
      "activations/layer22_attention_weight_max": 42.14872360229492,
      "activations/layer22_attention_weight_min": -32.96425247192383,
      "activations/layer23_attention_weight_max": 34.42671203613281,
      "activations/layer23_attention_weight_min": -25.30171775817871,
      "activations/layer2_attention_weight_max": 31.779998779296875,
      "activations/layer2_attention_weight_min": -30.395370483398438,
      "activations/layer3_attention_weight_max": 91.63340759277344,
      "activations/layer3_attention_weight_min": -95.49024963378906,
      "activations/layer4_attention_weight_max": 100.98450469970703,
      "activations/layer4_attention_weight_min": -105.58656311035156,
      "activations/layer5_attention_weight_max": 48.91093063354492,
      "activations/layer5_attention_weight_min": -62.718421936035156,
      "activations/layer6_attention_weight_max": 42.15644836425781,
      "activations/layer6_attention_weight_min": -44.64371871948242,
      "activations/layer7_attention_weight_max": 98.57201385498047,
      "activations/layer7_attention_weight_min": -93.7693862915039,
      "activations/layer8_attention_weight_max": 42.141273498535156,
      "activations/layer8_attention_weight_min": -44.1306037902832,
      "activations/layer9_attention_weight_max": 30.360055923461914,
      "activations/layer9_attention_weight_min": -35.59484100341797,
      "epoch": 18.72,
      "learning_rate": 2.9585984848484847e-05,
      "loss": 2.7175,
      "step": 322150
    },
    {
      "activations/layer0_attention_weight_max": 16.054805755615234,
      "activations/layer0_attention_weight_min": -12.651698112487793,
      "activations/layer10_attention_weight_max": 37.72246551513672,
      "activations/layer10_attention_weight_min": -36.7474250793457,
      "activations/layer11_attention_weight_max": 37.022518157958984,
      "activations/layer11_attention_weight_min": -36.808135986328125,
      "activations/layer12_attention_weight_max": 31.541399002075195,
      "activations/layer12_attention_weight_min": -28.012863159179688,
      "activations/layer13_attention_weight_max": 43.681068420410156,
      "activations/layer13_attention_weight_min": -38.1200065612793,
      "activations/layer14_attention_weight_max": 51.63888931274414,
      "activations/layer14_attention_weight_min": -39.32931900024414,
      "activations/layer15_attention_weight_max": 45.5727424621582,
      "activations/layer15_attention_weight_min": -37.975807189941406,
      "activations/layer16_attention_weight_max": 35.8730354309082,
      "activations/layer16_attention_weight_min": -28.529436111450195,
      "activations/layer17_attention_weight_max": 32.28847122192383,
      "activations/layer17_attention_weight_min": -26.062772750854492,
      "activations/layer18_attention_weight_max": 34.55656051635742,
      "activations/layer18_attention_weight_min": -26.573226928710938,
      "activations/layer19_attention_weight_max": 35.19670104980469,
      "activations/layer19_attention_weight_min": -32.64677810668945,
      "activations/layer1_attention_weight_max": 16.54793357849121,
      "activations/layer1_attention_weight_min": -15.832712173461914,
      "activations/layer20_attention_weight_max": 30.9414005279541,
      "activations/layer20_attention_weight_min": -26.377958297729492,
      "activations/layer21_attention_weight_max": 29.793899536132812,
      "activations/layer21_attention_weight_min": -25.41623306274414,
      "activations/layer22_attention_weight_max": 47.729408264160156,
      "activations/layer22_attention_weight_min": -34.07345962524414,
      "activations/layer23_attention_weight_max": 32.87019348144531,
      "activations/layer23_attention_weight_min": -27.24799156188965,
      "activations/layer2_attention_weight_max": 32.20780563354492,
      "activations/layer2_attention_weight_min": -30.496397018432617,
      "activations/layer3_attention_weight_max": 100.39482879638672,
      "activations/layer3_attention_weight_min": -101.45575714111328,
      "activations/layer4_attention_weight_max": 107.67732238769531,
      "activations/layer4_attention_weight_min": -110.59099578857422,
      "activations/layer5_attention_weight_max": 53.7251091003418,
      "activations/layer5_attention_weight_min": -59.584388732910156,
      "activations/layer6_attention_weight_max": 47.14448165893555,
      "activations/layer6_attention_weight_min": -47.855308532714844,
      "activations/layer7_attention_weight_max": 108.99092102050781,
      "activations/layer7_attention_weight_min": -103.02106475830078,
      "activations/layer8_attention_weight_max": 49.147239685058594,
      "activations/layer8_attention_weight_min": -49.43483352661133,
      "activations/layer9_attention_weight_max": 36.47622299194336,
      "activations/layer9_attention_weight_min": -38.94588851928711,
      "epoch": 18.72,
      "learning_rate": 2.9567045454545452e-05,
      "loss": 2.7148,
      "step": 322200
    },
    {
      "activations/layer0_attention_weight_max": 15.783146858215332,
      "activations/layer0_attention_weight_min": -12.409441947937012,
      "activations/layer10_attention_weight_max": 36.43390655517578,
      "activations/layer10_attention_weight_min": -36.15583419799805,
      "activations/layer11_attention_weight_max": 37.04750442504883,
      "activations/layer11_attention_weight_min": -37.09888458251953,
      "activations/layer12_attention_weight_max": 29.348533630371094,
      "activations/layer12_attention_weight_min": -29.597492218017578,
      "activations/layer13_attention_weight_max": 43.08810043334961,
      "activations/layer13_attention_weight_min": -40.52595520019531,
      "activations/layer14_attention_weight_max": 44.96015548706055,
      "activations/layer14_attention_weight_min": -42.86649703979492,
      "activations/layer15_attention_weight_max": 43.052940368652344,
      "activations/layer15_attention_weight_min": -37.33727264404297,
      "activations/layer16_attention_weight_max": 31.546207427978516,
      "activations/layer16_attention_weight_min": -30.57482147216797,
      "activations/layer17_attention_weight_max": 31.23851203918457,
      "activations/layer17_attention_weight_min": -26.029983520507812,
      "activations/layer18_attention_weight_max": 33.610347747802734,
      "activations/layer18_attention_weight_min": -24.969449996948242,
      "activations/layer19_attention_weight_max": 38.464141845703125,
      "activations/layer19_attention_weight_min": -33.0362434387207,
      "activations/layer1_attention_weight_max": 16.177810668945312,
      "activations/layer1_attention_weight_min": -15.093408584594727,
      "activations/layer20_attention_weight_max": 30.692636489868164,
      "activations/layer20_attention_weight_min": -26.059419631958008,
      "activations/layer21_attention_weight_max": 30.949031829833984,
      "activations/layer21_attention_weight_min": -25.1193790435791,
      "activations/layer22_attention_weight_max": 47.93131637573242,
      "activations/layer22_attention_weight_min": -32.06291580200195,
      "activations/layer23_attention_weight_max": 41.04212951660156,
      "activations/layer23_attention_weight_min": -25.92761993408203,
      "activations/layer2_attention_weight_max": 30.991552352905273,
      "activations/layer2_attention_weight_min": -30.448413848876953,
      "activations/layer3_attention_weight_max": 97.51365661621094,
      "activations/layer3_attention_weight_min": -99.03400421142578,
      "activations/layer4_attention_weight_max": 107.41802215576172,
      "activations/layer4_attention_weight_min": -109.91143798828125,
      "activations/layer5_attention_weight_max": 51.390830993652344,
      "activations/layer5_attention_weight_min": -62.417510986328125,
      "activations/layer6_attention_weight_max": 45.89486312866211,
      "activations/layer6_attention_weight_min": -47.21491241455078,
      "activations/layer7_attention_weight_max": 93.3333740234375,
      "activations/layer7_attention_weight_min": -98.25240325927734,
      "activations/layer8_attention_weight_max": 43.00135803222656,
      "activations/layer8_attention_weight_min": -46.48147201538086,
      "activations/layer9_attention_weight_max": 34.75336837768555,
      "activations/layer9_attention_weight_min": -36.53517532348633,
      "epoch": 18.72,
      "learning_rate": 2.9548484848484846e-05,
      "loss": 2.739,
      "step": 322250
    },
    {
      "activations/layer0_attention_weight_max": 16.11610221862793,
      "activations/layer0_attention_weight_min": -13.131054878234863,
      "activations/layer10_attention_weight_max": 30.005693435668945,
      "activations/layer10_attention_weight_min": -32.59989929199219,
      "activations/layer11_attention_weight_max": 32.15926742553711,
      "activations/layer11_attention_weight_min": -35.578125,
      "activations/layer12_attention_weight_max": 25.110334396362305,
      "activations/layer12_attention_weight_min": -26.11032485961914,
      "activations/layer13_attention_weight_max": 39.38279724121094,
      "activations/layer13_attention_weight_min": -36.037200927734375,
      "activations/layer14_attention_weight_max": 41.7679443359375,
      "activations/layer14_attention_weight_min": -38.26181411743164,
      "activations/layer15_attention_weight_max": 36.60113525390625,
      "activations/layer15_attention_weight_min": -36.71964645385742,
      "activations/layer16_attention_weight_max": 29.575307846069336,
      "activations/layer16_attention_weight_min": -27.29048728942871,
      "activations/layer17_attention_weight_max": 30.589162826538086,
      "activations/layer17_attention_weight_min": -25.62920570373535,
      "activations/layer18_attention_weight_max": 32.697208404541016,
      "activations/layer18_attention_weight_min": -27.906612396240234,
      "activations/layer19_attention_weight_max": 34.23441696166992,
      "activations/layer19_attention_weight_min": -32.55887222290039,
      "activations/layer1_attention_weight_max": 16.167884826660156,
      "activations/layer1_attention_weight_min": -16.2301082611084,
      "activations/layer20_attention_weight_max": 28.308698654174805,
      "activations/layer20_attention_weight_min": -25.645122528076172,
      "activations/layer21_attention_weight_max": 27.936290740966797,
      "activations/layer21_attention_weight_min": -24.712331771850586,
      "activations/layer22_attention_weight_max": 43.74689865112305,
      "activations/layer22_attention_weight_min": -34.174381256103516,
      "activations/layer23_attention_weight_max": 31.85834312438965,
      "activations/layer23_attention_weight_min": -28.221778869628906,
      "activations/layer2_attention_weight_max": 29.62799644470215,
      "activations/layer2_attention_weight_min": -30.56392478942871,
      "activations/layer3_attention_weight_max": 92.05562591552734,
      "activations/layer3_attention_weight_min": -94.48799133300781,
      "activations/layer4_attention_weight_max": 103.08971405029297,
      "activations/layer4_attention_weight_min": -105.60316467285156,
      "activations/layer5_attention_weight_max": 47.875946044921875,
      "activations/layer5_attention_weight_min": -58.919647216796875,
      "activations/layer6_attention_weight_max": 43.0102424621582,
      "activations/layer6_attention_weight_min": -47.206138610839844,
      "activations/layer7_attention_weight_max": 86.89157104492188,
      "activations/layer7_attention_weight_min": -95.8157958984375,
      "activations/layer8_attention_weight_max": 40.689544677734375,
      "activations/layer8_attention_weight_min": -42.74012756347656,
      "activations/layer9_attention_weight_max": 29.230497360229492,
      "activations/layer9_attention_weight_min": -33.0525016784668,
      "epoch": 18.73,
      "learning_rate": 2.952954545454545e-05,
      "loss": 2.7384,
      "step": 322300
    },
    {
      "activations/layer0_attention_weight_max": 16.042604446411133,
      "activations/layer0_attention_weight_min": -11.771791458129883,
      "activations/layer10_attention_weight_max": 39.059051513671875,
      "activations/layer10_attention_weight_min": -37.68408203125,
      "activations/layer11_attention_weight_max": 37.26036834716797,
      "activations/layer11_attention_weight_min": -37.55810546875,
      "activations/layer12_attention_weight_max": 28.00836944580078,
      "activations/layer12_attention_weight_min": -27.611129760742188,
      "activations/layer13_attention_weight_max": 40.425376892089844,
      "activations/layer13_attention_weight_min": -36.983680725097656,
      "activations/layer14_attention_weight_max": 42.767189025878906,
      "activations/layer14_attention_weight_min": -38.51997375488281,
      "activations/layer15_attention_weight_max": 38.6373405456543,
      "activations/layer15_attention_weight_min": -34.23568344116211,
      "activations/layer16_attention_weight_max": 28.991418838500977,
      "activations/layer16_attention_weight_min": -27.51663589477539,
      "activations/layer17_attention_weight_max": 30.114709854125977,
      "activations/layer17_attention_weight_min": -25.40744972229004,
      "activations/layer18_attention_weight_max": 31.413753509521484,
      "activations/layer18_attention_weight_min": -25.891761779785156,
      "activations/layer19_attention_weight_max": 31.6184139251709,
      "activations/layer19_attention_weight_min": -31.199050903320312,
      "activations/layer1_attention_weight_max": 15.814485549926758,
      "activations/layer1_attention_weight_min": -14.297449111938477,
      "activations/layer20_attention_weight_max": 28.2883243560791,
      "activations/layer20_attention_weight_min": -23.614879608154297,
      "activations/layer21_attention_weight_max": 25.999267578125,
      "activations/layer21_attention_weight_min": -23.580791473388672,
      "activations/layer22_attention_weight_max": 42.94328308105469,
      "activations/layer22_attention_weight_min": -30.26294708251953,
      "activations/layer23_attention_weight_max": 32.14364242553711,
      "activations/layer23_attention_weight_min": -25.25400161743164,
      "activations/layer2_attention_weight_max": 32.828399658203125,
      "activations/layer2_attention_weight_min": -30.133502960205078,
      "activations/layer3_attention_weight_max": 98.58061218261719,
      "activations/layer3_attention_weight_min": -97.16171264648438,
      "activations/layer4_attention_weight_max": 107.56817626953125,
      "activations/layer4_attention_weight_min": -107.51789855957031,
      "activations/layer5_attention_weight_max": 50.67097091674805,
      "activations/layer5_attention_weight_min": -63.76689910888672,
      "activations/layer6_attention_weight_max": 45.97770690917969,
      "activations/layer6_attention_weight_min": -50.016544342041016,
      "activations/layer7_attention_weight_max": 94.96275329589844,
      "activations/layer7_attention_weight_min": -104.90934753417969,
      "activations/layer8_attention_weight_max": 44.47605514526367,
      "activations/layer8_attention_weight_min": -46.87192916870117,
      "activations/layer9_attention_weight_max": 36.69640350341797,
      "activations/layer9_attention_weight_min": -37.771881103515625,
      "epoch": 18.73,
      "learning_rate": 2.951060606060606e-05,
      "loss": 2.7122,
      "step": 322350
    },
    {
      "activations/layer0_attention_weight_max": 15.583207130432129,
      "activations/layer0_attention_weight_min": -12.517906188964844,
      "activations/layer10_attention_weight_max": 33.81270217895508,
      "activations/layer10_attention_weight_min": -36.3358039855957,
      "activations/layer11_attention_weight_max": 35.89330291748047,
      "activations/layer11_attention_weight_min": -35.48869323730469,
      "activations/layer12_attention_weight_max": 28.148609161376953,
      "activations/layer12_attention_weight_min": -28.265846252441406,
      "activations/layer13_attention_weight_max": 45.02554702758789,
      "activations/layer13_attention_weight_min": -37.42799377441406,
      "activations/layer14_attention_weight_max": 46.22766876220703,
      "activations/layer14_attention_weight_min": -39.66903305053711,
      "activations/layer15_attention_weight_max": 44.368595123291016,
      "activations/layer15_attention_weight_min": -37.05107879638672,
      "activations/layer16_attention_weight_max": 32.15986633300781,
      "activations/layer16_attention_weight_min": -34.19532012939453,
      "activations/layer17_attention_weight_max": 29.771827697753906,
      "activations/layer17_attention_weight_min": -29.143869400024414,
      "activations/layer18_attention_weight_max": 30.24664878845215,
      "activations/layer18_attention_weight_min": -25.576208114624023,
      "activations/layer19_attention_weight_max": 30.79854393005371,
      "activations/layer19_attention_weight_min": -30.352920532226562,
      "activations/layer1_attention_weight_max": 16.085006713867188,
      "activations/layer1_attention_weight_min": -15.604734420776367,
      "activations/layer20_attention_weight_max": 29.414520263671875,
      "activations/layer20_attention_weight_min": -26.835498809814453,
      "activations/layer21_attention_weight_max": 29.126262664794922,
      "activations/layer21_attention_weight_min": -26.290292739868164,
      "activations/layer22_attention_weight_max": 40.792198181152344,
      "activations/layer22_attention_weight_min": -32.49012756347656,
      "activations/layer23_attention_weight_max": 30.77832794189453,
      "activations/layer23_attention_weight_min": -25.207866668701172,
      "activations/layer2_attention_weight_max": 30.862478256225586,
      "activations/layer2_attention_weight_min": -30.117813110351562,
      "activations/layer3_attention_weight_max": 95.53016662597656,
      "activations/layer3_attention_weight_min": -100.54358673095703,
      "activations/layer4_attention_weight_max": 108.99714660644531,
      "activations/layer4_attention_weight_min": -119.14418029785156,
      "activations/layer5_attention_weight_max": 51.76445770263672,
      "activations/layer5_attention_weight_min": -61.741024017333984,
      "activations/layer6_attention_weight_max": 46.04564666748047,
      "activations/layer6_attention_weight_min": -46.7520866394043,
      "activations/layer7_attention_weight_max": 101.3260726928711,
      "activations/layer7_attention_weight_min": -99.34331512451172,
      "activations/layer8_attention_weight_max": 44.46409606933594,
      "activations/layer8_attention_weight_min": -44.8007698059082,
      "activations/layer9_attention_weight_max": 30.85914421081543,
      "activations/layer9_attention_weight_min": -34.819950103759766,
      "epoch": 18.73,
      "learning_rate": 2.949166666666666e-05,
      "loss": 2.7135,
      "step": 322400
    },
    {
      "activations/layer0_attention_weight_max": 16.986276626586914,
      "activations/layer0_attention_weight_min": -12.34156608581543,
      "activations/layer10_attention_weight_max": 34.437461853027344,
      "activations/layer10_attention_weight_min": -34.50611114501953,
      "activations/layer11_attention_weight_max": 34.58367919921875,
      "activations/layer11_attention_weight_min": -36.82441329956055,
      "activations/layer12_attention_weight_max": 25.735254287719727,
      "activations/layer12_attention_weight_min": -27.25164031982422,
      "activations/layer13_attention_weight_max": 37.82893371582031,
      "activations/layer13_attention_weight_min": -36.694175720214844,
      "activations/layer14_attention_weight_max": 44.75125503540039,
      "activations/layer14_attention_weight_min": -38.87007522583008,
      "activations/layer15_attention_weight_max": 36.550899505615234,
      "activations/layer15_attention_weight_min": -37.39170837402344,
      "activations/layer16_attention_weight_max": 27.114429473876953,
      "activations/layer16_attention_weight_min": -27.282468795776367,
      "activations/layer17_attention_weight_max": 27.65947151184082,
      "activations/layer17_attention_weight_min": -25.96186637878418,
      "activations/layer18_attention_weight_max": 28.45890998840332,
      "activations/layer18_attention_weight_min": -25.85765266418457,
      "activations/layer19_attention_weight_max": 28.538076400756836,
      "activations/layer19_attention_weight_min": -31.81583023071289,
      "activations/layer1_attention_weight_max": 17.59658432006836,
      "activations/layer1_attention_weight_min": -16.247989654541016,
      "activations/layer20_attention_weight_max": 23.5599365234375,
      "activations/layer20_attention_weight_min": -24.430706024169922,
      "activations/layer21_attention_weight_max": 26.919301986694336,
      "activations/layer21_attention_weight_min": -24.60723876953125,
      "activations/layer22_attention_weight_max": 36.24866485595703,
      "activations/layer22_attention_weight_min": -30.642473220825195,
      "activations/layer23_attention_weight_max": 27.836559295654297,
      "activations/layer23_attention_weight_min": -27.403385162353516,
      "activations/layer2_attention_weight_max": 31.380781173706055,
      "activations/layer2_attention_weight_min": -29.69525146484375,
      "activations/layer3_attention_weight_max": 97.526123046875,
      "activations/layer3_attention_weight_min": -97.82166290283203,
      "activations/layer4_attention_weight_max": 105.7952880859375,
      "activations/layer4_attention_weight_min": -107.40313720703125,
      "activations/layer5_attention_weight_max": 51.37146759033203,
      "activations/layer5_attention_weight_min": -63.0221061706543,
      "activations/layer6_attention_weight_max": 45.158939361572266,
      "activations/layer6_attention_weight_min": -46.6924934387207,
      "activations/layer7_attention_weight_max": 93.5685043334961,
      "activations/layer7_attention_weight_min": -98.69851684570312,
      "activations/layer8_attention_weight_max": 43.220401763916016,
      "activations/layer8_attention_weight_min": -48.16925048828125,
      "activations/layer9_attention_weight_max": 32.522361755371094,
      "activations/layer9_attention_weight_min": -36.93333053588867,
      "epoch": 18.74,
      "learning_rate": 2.947272727272727e-05,
      "loss": 2.7281,
      "step": 322450
    },
    {
      "activations/layer0_attention_weight_max": 16.321624755859375,
      "activations/layer0_attention_weight_min": -12.393136024475098,
      "activations/layer10_attention_weight_max": 31.4168758392334,
      "activations/layer10_attention_weight_min": -32.31156921386719,
      "activations/layer11_attention_weight_max": 31.400588989257812,
      "activations/layer11_attention_weight_min": -30.68226432800293,
      "activations/layer12_attention_weight_max": 26.914709091186523,
      "activations/layer12_attention_weight_min": -26.135250091552734,
      "activations/layer13_attention_weight_max": 38.256202697753906,
      "activations/layer13_attention_weight_min": -36.46421813964844,
      "activations/layer14_attention_weight_max": 41.30061340332031,
      "activations/layer14_attention_weight_min": -36.59498596191406,
      "activations/layer15_attention_weight_max": 37.966896057128906,
      "activations/layer15_attention_weight_min": -34.780128479003906,
      "activations/layer16_attention_weight_max": 28.852876663208008,
      "activations/layer16_attention_weight_min": -28.00166130065918,
      "activations/layer17_attention_weight_max": 28.77210235595703,
      "activations/layer17_attention_weight_min": -27.584421157836914,
      "activations/layer18_attention_weight_max": 30.51348304748535,
      "activations/layer18_attention_weight_min": -25.088159561157227,
      "activations/layer19_attention_weight_max": 34.37234878540039,
      "activations/layer19_attention_weight_min": -31.410226821899414,
      "activations/layer1_attention_weight_max": 15.91494083404541,
      "activations/layer1_attention_weight_min": -13.645782470703125,
      "activations/layer20_attention_weight_max": 28.189912796020508,
      "activations/layer20_attention_weight_min": -25.9139347076416,
      "activations/layer21_attention_weight_max": 27.26964569091797,
      "activations/layer21_attention_weight_min": -25.114933013916016,
      "activations/layer22_attention_weight_max": 39.838958740234375,
      "activations/layer22_attention_weight_min": -34.28734588623047,
      "activations/layer23_attention_weight_max": 34.269012451171875,
      "activations/layer23_attention_weight_min": -28.837799072265625,
      "activations/layer2_attention_weight_max": 31.381887435913086,
      "activations/layer2_attention_weight_min": -28.264095306396484,
      "activations/layer3_attention_weight_max": 88.95096588134766,
      "activations/layer3_attention_weight_min": -88.9906997680664,
      "activations/layer4_attention_weight_max": 99.96150207519531,
      "activations/layer4_attention_weight_min": -100.36544799804688,
      "activations/layer5_attention_weight_max": 53.91697311401367,
      "activations/layer5_attention_weight_min": -62.53832244873047,
      "activations/layer6_attention_weight_max": 42.25283432006836,
      "activations/layer6_attention_weight_min": -44.01667022705078,
      "activations/layer7_attention_weight_max": 87.97746276855469,
      "activations/layer7_attention_weight_min": -92.42012786865234,
      "activations/layer8_attention_weight_max": 37.92052459716797,
      "activations/layer8_attention_weight_min": -40.50098419189453,
      "activations/layer9_attention_weight_max": 29.46174430847168,
      "activations/layer9_attention_weight_min": -32.57021713256836,
      "epoch": 18.74,
      "learning_rate": 2.9453787878787877e-05,
      "loss": 2.7357,
      "step": 322500
    },
    {
      "activations/layer0_attention_weight_max": 16.5484676361084,
      "activations/layer0_attention_weight_min": -13.585942268371582,
      "activations/layer10_attention_weight_max": 35.43852615356445,
      "activations/layer10_attention_weight_min": -33.83060836791992,
      "activations/layer11_attention_weight_max": 35.564022064208984,
      "activations/layer11_attention_weight_min": -35.07496643066406,
      "activations/layer12_attention_weight_max": 27.350540161132812,
      "activations/layer12_attention_weight_min": -27.245437622070312,
      "activations/layer13_attention_weight_max": 40.11717224121094,
      "activations/layer13_attention_weight_min": -36.819698333740234,
      "activations/layer14_attention_weight_max": 43.1568603515625,
      "activations/layer14_attention_weight_min": -39.57712173461914,
      "activations/layer15_attention_weight_max": 40.75912094116211,
      "activations/layer15_attention_weight_min": -36.1419792175293,
      "activations/layer16_attention_weight_max": 29.572284698486328,
      "activations/layer16_attention_weight_min": -29.21208381652832,
      "activations/layer17_attention_weight_max": 29.8658447265625,
      "activations/layer17_attention_weight_min": -26.1231632232666,
      "activations/layer18_attention_weight_max": 32.63309097290039,
      "activations/layer18_attention_weight_min": -27.185752868652344,
      "activations/layer19_attention_weight_max": 33.19004440307617,
      "activations/layer19_attention_weight_min": -36.1455192565918,
      "activations/layer1_attention_weight_max": 19.33259391784668,
      "activations/layer1_attention_weight_min": -15.844372749328613,
      "activations/layer20_attention_weight_max": 27.752779006958008,
      "activations/layer20_attention_weight_min": -29.456857681274414,
      "activations/layer21_attention_weight_max": 30.861364364624023,
      "activations/layer21_attention_weight_min": -29.00238037109375,
      "activations/layer22_attention_weight_max": 40.80287170410156,
      "activations/layer22_attention_weight_min": -39.26670455932617,
      "activations/layer23_attention_weight_max": 35.582061767578125,
      "activations/layer23_attention_weight_min": -28.65815544128418,
      "activations/layer2_attention_weight_max": 30.384693145751953,
      "activations/layer2_attention_weight_min": -29.86891746520996,
      "activations/layer3_attention_weight_max": 99.21134948730469,
      "activations/layer3_attention_weight_min": -93.75115203857422,
      "activations/layer4_attention_weight_max": 111.82750701904297,
      "activations/layer4_attention_weight_min": -104.88482666015625,
      "activations/layer5_attention_weight_max": 53.1872673034668,
      "activations/layer5_attention_weight_min": -62.32929992675781,
      "activations/layer6_attention_weight_max": 43.64704132080078,
      "activations/layer6_attention_weight_min": -46.03628921508789,
      "activations/layer7_attention_weight_max": 93.18658447265625,
      "activations/layer7_attention_weight_min": -96.42758178710938,
      "activations/layer8_attention_weight_max": 42.709022521972656,
      "activations/layer8_attention_weight_min": -43.3709602355957,
      "activations/layer9_attention_weight_max": 33.36854553222656,
      "activations/layer9_attention_weight_min": -33.59409713745117,
      "epoch": 18.74,
      "learning_rate": 2.9434848484848482e-05,
      "loss": 2.7397,
      "step": 322550
    },
    {
      "activations/layer0_attention_weight_max": 16.447423934936523,
      "activations/layer0_attention_weight_min": -11.895458221435547,
      "activations/layer10_attention_weight_max": 39.282318115234375,
      "activations/layer10_attention_weight_min": -37.263458251953125,
      "activations/layer11_attention_weight_max": 39.63074493408203,
      "activations/layer11_attention_weight_min": -37.59612274169922,
      "activations/layer12_attention_weight_max": 28.846073150634766,
      "activations/layer12_attention_weight_min": -30.854022979736328,
      "activations/layer13_attention_weight_max": 44.03945541381836,
      "activations/layer13_attention_weight_min": -38.9324836730957,
      "activations/layer14_attention_weight_max": 52.404056549072266,
      "activations/layer14_attention_weight_min": -46.51288604736328,
      "activations/layer15_attention_weight_max": 45.51845932006836,
      "activations/layer15_attention_weight_min": -42.68564224243164,
      "activations/layer16_attention_weight_max": 34.53247833251953,
      "activations/layer16_attention_weight_min": -28.189767837524414,
      "activations/layer17_attention_weight_max": 32.49271011352539,
      "activations/layer17_attention_weight_min": -29.748178482055664,
      "activations/layer18_attention_weight_max": 34.97820281982422,
      "activations/layer18_attention_weight_min": -29.706819534301758,
      "activations/layer19_attention_weight_max": 36.84726333618164,
      "activations/layer19_attention_weight_min": -30.851621627807617,
      "activations/layer1_attention_weight_max": 15.698750495910645,
      "activations/layer1_attention_weight_min": -15.326302528381348,
      "activations/layer20_attention_weight_max": 32.280853271484375,
      "activations/layer20_attention_weight_min": -25.59678840637207,
      "activations/layer21_attention_weight_max": 26.636144638061523,
      "activations/layer21_attention_weight_min": -25.26415252685547,
      "activations/layer22_attention_weight_max": 40.48930358886719,
      "activations/layer22_attention_weight_min": -33.9736442565918,
      "activations/layer23_attention_weight_max": 35.49460983276367,
      "activations/layer23_attention_weight_min": -26.079086303710938,
      "activations/layer2_attention_weight_max": 31.828052520751953,
      "activations/layer2_attention_weight_min": -29.956205368041992,
      "activations/layer3_attention_weight_max": 94.57041931152344,
      "activations/layer3_attention_weight_min": -99.1696548461914,
      "activations/layer4_attention_weight_max": 105.95291900634766,
      "activations/layer4_attention_weight_min": -110.94268035888672,
      "activations/layer5_attention_weight_max": 53.01005554199219,
      "activations/layer5_attention_weight_min": -67.98716735839844,
      "activations/layer6_attention_weight_max": 43.52458572387695,
      "activations/layer6_attention_weight_min": -46.1762809753418,
      "activations/layer7_attention_weight_max": 100.38758850097656,
      "activations/layer7_attention_weight_min": -99.9429931640625,
      "activations/layer8_attention_weight_max": 46.12825393676758,
      "activations/layer8_attention_weight_min": -46.8006591796875,
      "activations/layer9_attention_weight_max": 33.95235061645508,
      "activations/layer9_attention_weight_min": -36.99781036376953,
      "epoch": 18.74,
      "learning_rate": 2.941590909090909e-05,
      "loss": 2.7492,
      "step": 322600
    },
    {
      "activations/layer0_attention_weight_max": 15.573251724243164,
      "activations/layer0_attention_weight_min": -13.7483549118042,
      "activations/layer10_attention_weight_max": 37.802154541015625,
      "activations/layer10_attention_weight_min": -34.988094329833984,
      "activations/layer11_attention_weight_max": 37.1702880859375,
      "activations/layer11_attention_weight_min": -36.62421417236328,
      "activations/layer12_attention_weight_max": 27.215253829956055,
      "activations/layer12_attention_weight_min": -32.299560546875,
      "activations/layer13_attention_weight_max": 40.29726791381836,
      "activations/layer13_attention_weight_min": -36.083839416503906,
      "activations/layer14_attention_weight_max": 42.82306671142578,
      "activations/layer14_attention_weight_min": -41.034847259521484,
      "activations/layer15_attention_weight_max": 39.84149932861328,
      "activations/layer15_attention_weight_min": -38.27743911743164,
      "activations/layer16_attention_weight_max": 28.403743743896484,
      "activations/layer16_attention_weight_min": -28.330917358398438,
      "activations/layer17_attention_weight_max": 30.24285316467285,
      "activations/layer17_attention_weight_min": -26.750558853149414,
      "activations/layer18_attention_weight_max": 29.63178825378418,
      "activations/layer18_attention_weight_min": -25.327939987182617,
      "activations/layer19_attention_weight_max": 32.25768280029297,
      "activations/layer19_attention_weight_min": -30.988393783569336,
      "activations/layer1_attention_weight_max": 17.889293670654297,
      "activations/layer1_attention_weight_min": -16.456968307495117,
      "activations/layer20_attention_weight_max": 27.015579223632812,
      "activations/layer20_attention_weight_min": -24.69957160949707,
      "activations/layer21_attention_weight_max": 28.00472068786621,
      "activations/layer21_attention_weight_min": -24.944061279296875,
      "activations/layer22_attention_weight_max": 41.65193176269531,
      "activations/layer22_attention_weight_min": -30.43000030517578,
      "activations/layer23_attention_weight_max": 31.751279830932617,
      "activations/layer23_attention_weight_min": -25.721885681152344,
      "activations/layer2_attention_weight_max": 33.167503356933594,
      "activations/layer2_attention_weight_min": -32.02821350097656,
      "activations/layer3_attention_weight_max": 100.82875061035156,
      "activations/layer3_attention_weight_min": -103.1531982421875,
      "activations/layer4_attention_weight_max": 111.71500396728516,
      "activations/layer4_attention_weight_min": -112.56621551513672,
      "activations/layer5_attention_weight_max": 56.00147247314453,
      "activations/layer5_attention_weight_min": -62.67424011230469,
      "activations/layer6_attention_weight_max": 48.27351760864258,
      "activations/layer6_attention_weight_min": -49.83670425415039,
      "activations/layer7_attention_weight_max": 104.95726776123047,
      "activations/layer7_attention_weight_min": -102.47454071044922,
      "activations/layer8_attention_weight_max": 44.12751770019531,
      "activations/layer8_attention_weight_min": -46.65629577636719,
      "activations/layer9_attention_weight_max": 35.02891159057617,
      "activations/layer9_attention_weight_min": -35.64905548095703,
      "epoch": 18.75,
      "learning_rate": 2.9396969696969695e-05,
      "loss": 2.7312,
      "step": 322650
    },
    {
      "activations/layer0_attention_weight_max": 15.91364860534668,
      "activations/layer0_attention_weight_min": -13.024049758911133,
      "activations/layer10_attention_weight_max": 34.378665924072266,
      "activations/layer10_attention_weight_min": -33.48424530029297,
      "activations/layer11_attention_weight_max": 36.15550994873047,
      "activations/layer11_attention_weight_min": -34.11128616333008,
      "activations/layer12_attention_weight_max": 26.47599220275879,
      "activations/layer12_attention_weight_min": -26.2711124420166,
      "activations/layer13_attention_weight_max": 40.201560974121094,
      "activations/layer13_attention_weight_min": -35.743255615234375,
      "activations/layer14_attention_weight_max": 43.795597076416016,
      "activations/layer14_attention_weight_min": -37.23942565917969,
      "activations/layer15_attention_weight_max": 36.881526947021484,
      "activations/layer15_attention_weight_min": -35.75009536743164,
      "activations/layer16_attention_weight_max": 30.65999412536621,
      "activations/layer16_attention_weight_min": -30.17997169494629,
      "activations/layer17_attention_weight_max": 30.814565658569336,
      "activations/layer17_attention_weight_min": -27.242441177368164,
      "activations/layer18_attention_weight_max": 30.18584632873535,
      "activations/layer18_attention_weight_min": -27.47918701171875,
      "activations/layer19_attention_weight_max": 33.58016586303711,
      "activations/layer19_attention_weight_min": -31.567480087280273,
      "activations/layer1_attention_weight_max": 16.17669105529785,
      "activations/layer1_attention_weight_min": -15.229023933410645,
      "activations/layer20_attention_weight_max": 32.57257843017578,
      "activations/layer20_attention_weight_min": -26.65399742126465,
      "activations/layer21_attention_weight_max": 27.98986053466797,
      "activations/layer21_attention_weight_min": -26.688364028930664,
      "activations/layer22_attention_weight_max": 40.351863861083984,
      "activations/layer22_attention_weight_min": -33.48822784423828,
      "activations/layer23_attention_weight_max": 33.27459716796875,
      "activations/layer23_attention_weight_min": -27.94046401977539,
      "activations/layer2_attention_weight_max": 30.22028923034668,
      "activations/layer2_attention_weight_min": -29.558265686035156,
      "activations/layer3_attention_weight_max": 91.95967864990234,
      "activations/layer3_attention_weight_min": -93.19255828857422,
      "activations/layer4_attention_weight_max": 101.71994018554688,
      "activations/layer4_attention_weight_min": -103.61186981201172,
      "activations/layer5_attention_weight_max": 51.83824157714844,
      "activations/layer5_attention_weight_min": -60.845130920410156,
      "activations/layer6_attention_weight_max": 43.08271408081055,
      "activations/layer6_attention_weight_min": -44.73117446899414,
      "activations/layer7_attention_weight_max": 91.28656005859375,
      "activations/layer7_attention_weight_min": -90.22984313964844,
      "activations/layer8_attention_weight_max": 42.74869918823242,
      "activations/layer8_attention_weight_min": -41.51348876953125,
      "activations/layer9_attention_weight_max": 31.962080001831055,
      "activations/layer9_attention_weight_min": -33.2089958190918,
      "epoch": 18.75,
      "learning_rate": 2.93780303030303e-05,
      "loss": 2.7233,
      "step": 322700
    },
    {
      "activations/layer0_attention_weight_max": 16.49354362487793,
      "activations/layer0_attention_weight_min": -13.118521690368652,
      "activations/layer10_attention_weight_max": 35.70896911621094,
      "activations/layer10_attention_weight_min": -36.23627853393555,
      "activations/layer11_attention_weight_max": 34.59382247924805,
      "activations/layer11_attention_weight_min": -35.73307800292969,
      "activations/layer12_attention_weight_max": 27.6431827545166,
      "activations/layer12_attention_weight_min": -26.96849250793457,
      "activations/layer13_attention_weight_max": 38.66372299194336,
      "activations/layer13_attention_weight_min": -35.33561325073242,
      "activations/layer14_attention_weight_max": 41.820152282714844,
      "activations/layer14_attention_weight_min": -37.10658264160156,
      "activations/layer15_attention_weight_max": 37.58252716064453,
      "activations/layer15_attention_weight_min": -36.55062484741211,
      "activations/layer16_attention_weight_max": 30.39440155029297,
      "activations/layer16_attention_weight_min": -28.679527282714844,
      "activations/layer17_attention_weight_max": 30.27981185913086,
      "activations/layer17_attention_weight_min": -25.99516487121582,
      "activations/layer18_attention_weight_max": 29.5518741607666,
      "activations/layer18_attention_weight_min": -26.810165405273438,
      "activations/layer19_attention_weight_max": 32.0338134765625,
      "activations/layer19_attention_weight_min": -32.01958465576172,
      "activations/layer1_attention_weight_max": 16.21160888671875,
      "activations/layer1_attention_weight_min": -14.325088500976562,
      "activations/layer20_attention_weight_max": 29.081464767456055,
      "activations/layer20_attention_weight_min": -24.50172996520996,
      "activations/layer21_attention_weight_max": 28.92599868774414,
      "activations/layer21_attention_weight_min": -24.740848541259766,
      "activations/layer22_attention_weight_max": 44.383018493652344,
      "activations/layer22_attention_weight_min": -30.91832733154297,
      "activations/layer23_attention_weight_max": 34.969669342041016,
      "activations/layer23_attention_weight_min": -25.133525848388672,
      "activations/layer2_attention_weight_max": 30.482481002807617,
      "activations/layer2_attention_weight_min": -28.978422164916992,
      "activations/layer3_attention_weight_max": 94.44456481933594,
      "activations/layer3_attention_weight_min": -97.32963562011719,
      "activations/layer4_attention_weight_max": 102.1753158569336,
      "activations/layer4_attention_weight_min": -108.61651611328125,
      "activations/layer5_attention_weight_max": 52.51357650756836,
      "activations/layer5_attention_weight_min": -63.76559066772461,
      "activations/layer6_attention_weight_max": 45.97184371948242,
      "activations/layer6_attention_weight_min": -46.3920783996582,
      "activations/layer7_attention_weight_max": 99.3412857055664,
      "activations/layer7_attention_weight_min": -101.19195556640625,
      "activations/layer8_attention_weight_max": 43.329586029052734,
      "activations/layer8_attention_weight_min": -43.6544189453125,
      "activations/layer9_attention_weight_max": 33.8869743347168,
      "activations/layer9_attention_weight_min": -35.04464340209961,
      "epoch": 18.75,
      "learning_rate": 2.9359090909090905e-05,
      "loss": 2.7335,
      "step": 322750
    },
    {
      "activations/layer0_attention_weight_max": 16.370840072631836,
      "activations/layer0_attention_weight_min": -12.037833213806152,
      "activations/layer10_attention_weight_max": 40.09712219238281,
      "activations/layer10_attention_weight_min": -36.011043548583984,
      "activations/layer11_attention_weight_max": 37.412757873535156,
      "activations/layer11_attention_weight_min": -35.78645324707031,
      "activations/layer12_attention_weight_max": 30.120851516723633,
      "activations/layer12_attention_weight_min": -28.8253231048584,
      "activations/layer13_attention_weight_max": 43.51784133911133,
      "activations/layer13_attention_weight_min": -39.234230041503906,
      "activations/layer14_attention_weight_max": 46.835567474365234,
      "activations/layer14_attention_weight_min": -45.16593933105469,
      "activations/layer15_attention_weight_max": 46.1633415222168,
      "activations/layer15_attention_weight_min": -42.96803283691406,
      "activations/layer16_attention_weight_max": 30.426530838012695,
      "activations/layer16_attention_weight_min": -28.500629425048828,
      "activations/layer17_attention_weight_max": 34.6202278137207,
      "activations/layer17_attention_weight_min": -27.07293128967285,
      "activations/layer18_attention_weight_max": 30.515390396118164,
      "activations/layer18_attention_weight_min": -26.62016487121582,
      "activations/layer19_attention_weight_max": 34.0361442565918,
      "activations/layer19_attention_weight_min": -31.461408615112305,
      "activations/layer1_attention_weight_max": 17.74360466003418,
      "activations/layer1_attention_weight_min": -16.404560089111328,
      "activations/layer20_attention_weight_max": 30.972000122070312,
      "activations/layer20_attention_weight_min": -24.730911254882812,
      "activations/layer21_attention_weight_max": 28.927776336669922,
      "activations/layer21_attention_weight_min": -24.530357360839844,
      "activations/layer22_attention_weight_max": 44.04400634765625,
      "activations/layer22_attention_weight_min": -31.385074615478516,
      "activations/layer23_attention_weight_max": 31.803024291992188,
      "activations/layer23_attention_weight_min": -25.945045471191406,
      "activations/layer2_attention_weight_max": 32.30613708496094,
      "activations/layer2_attention_weight_min": -32.9856071472168,
      "activations/layer3_attention_weight_max": 99.86358642578125,
      "activations/layer3_attention_weight_min": -103.77793884277344,
      "activations/layer4_attention_weight_max": 106.2052993774414,
      "activations/layer4_attention_weight_min": -108.17205047607422,
      "activations/layer5_attention_weight_max": 49.909202575683594,
      "activations/layer5_attention_weight_min": -63.53538131713867,
      "activations/layer6_attention_weight_max": 47.50025939941406,
      "activations/layer6_attention_weight_min": -50.94111251831055,
      "activations/layer7_attention_weight_max": 101.44564819335938,
      "activations/layer7_attention_weight_min": -109.36915588378906,
      "activations/layer8_attention_weight_max": 50.123077392578125,
      "activations/layer8_attention_weight_min": -53.353797912597656,
      "activations/layer9_attention_weight_max": 35.366851806640625,
      "activations/layer9_attention_weight_min": -38.16263961791992,
      "epoch": 18.76,
      "learning_rate": 2.9340151515151513e-05,
      "loss": 2.7469,
      "step": 322800
    },
    {
      "activations/layer0_attention_weight_max": 16.079330444335938,
      "activations/layer0_attention_weight_min": -12.586344718933105,
      "activations/layer10_attention_weight_max": 33.542232513427734,
      "activations/layer10_attention_weight_min": -33.40023422241211,
      "activations/layer11_attention_weight_max": 33.40932846069336,
      "activations/layer11_attention_weight_min": -31.99317741394043,
      "activations/layer12_attention_weight_max": 26.24543571472168,
      "activations/layer12_attention_weight_min": -26.83734893798828,
      "activations/layer13_attention_weight_max": 41.05329132080078,
      "activations/layer13_attention_weight_min": -35.96724319458008,
      "activations/layer14_attention_weight_max": 41.30952072143555,
      "activations/layer14_attention_weight_min": -36.75343322753906,
      "activations/layer15_attention_weight_max": 39.79207229614258,
      "activations/layer15_attention_weight_min": -35.054561614990234,
      "activations/layer16_attention_weight_max": 29.815153121948242,
      "activations/layer16_attention_weight_min": -29.13959503173828,
      "activations/layer17_attention_weight_max": 30.63848876953125,
      "activations/layer17_attention_weight_min": -26.22489356994629,
      "activations/layer18_attention_weight_max": 30.191129684448242,
      "activations/layer18_attention_weight_min": -25.91342544555664,
      "activations/layer19_attention_weight_max": 31.216842651367188,
      "activations/layer19_attention_weight_min": -29.35542869567871,
      "activations/layer1_attention_weight_max": 15.859713554382324,
      "activations/layer1_attention_weight_min": -14.512274742126465,
      "activations/layer20_attention_weight_max": 24.40884017944336,
      "activations/layer20_attention_weight_min": -24.190650939941406,
      "activations/layer21_attention_weight_max": 25.526878356933594,
      "activations/layer21_attention_weight_min": -24.05497932434082,
      "activations/layer22_attention_weight_max": 38.42644500732422,
      "activations/layer22_attention_weight_min": -30.171573638916016,
      "activations/layer23_attention_weight_max": 29.26346206665039,
      "activations/layer23_attention_weight_min": -25.90322494506836,
      "activations/layer2_attention_weight_max": 30.95513916015625,
      "activations/layer2_attention_weight_min": -29.611431121826172,
      "activations/layer3_attention_weight_max": 96.51097869873047,
      "activations/layer3_attention_weight_min": -97.08837127685547,
      "activations/layer4_attention_weight_max": 100.02652740478516,
      "activations/layer4_attention_weight_min": -103.28895568847656,
      "activations/layer5_attention_weight_max": 48.10453796386719,
      "activations/layer5_attention_weight_min": -60.56220626831055,
      "activations/layer6_attention_weight_max": 42.240142822265625,
      "activations/layer6_attention_weight_min": -46.47844314575195,
      "activations/layer7_attention_weight_max": 86.81695556640625,
      "activations/layer7_attention_weight_min": -95.5491943359375,
      "activations/layer8_attention_weight_max": 42.53445816040039,
      "activations/layer8_attention_weight_min": -43.10171890258789,
      "activations/layer9_attention_weight_max": 31.651931762695312,
      "activations/layer9_attention_weight_min": -33.05091857910156,
      "epoch": 18.76,
      "learning_rate": 2.9321212121212118e-05,
      "loss": 2.7258,
      "step": 322850
    },
    {
      "activations/layer0_attention_weight_max": 16.1568546295166,
      "activations/layer0_attention_weight_min": -13.216104507446289,
      "activations/layer10_attention_weight_max": 33.1644172668457,
      "activations/layer10_attention_weight_min": -33.98814392089844,
      "activations/layer11_attention_weight_max": 34.35141372680664,
      "activations/layer11_attention_weight_min": -35.227657318115234,
      "activations/layer12_attention_weight_max": 25.746366500854492,
      "activations/layer12_attention_weight_min": -26.011377334594727,
      "activations/layer13_attention_weight_max": 37.82166290283203,
      "activations/layer13_attention_weight_min": -34.38350296020508,
      "activations/layer14_attention_weight_max": 39.98896789550781,
      "activations/layer14_attention_weight_min": -37.535919189453125,
      "activations/layer15_attention_weight_max": 38.67449188232422,
      "activations/layer15_attention_weight_min": -36.155452728271484,
      "activations/layer16_attention_weight_max": 31.264171600341797,
      "activations/layer16_attention_weight_min": -28.607378005981445,
      "activations/layer17_attention_weight_max": 29.452009201049805,
      "activations/layer17_attention_weight_min": -26.966426849365234,
      "activations/layer18_attention_weight_max": 31.637868881225586,
      "activations/layer18_attention_weight_min": -26.58055305480957,
      "activations/layer19_attention_weight_max": 34.012447357177734,
      "activations/layer19_attention_weight_min": -31.407922744750977,
      "activations/layer1_attention_weight_max": 15.136594772338867,
      "activations/layer1_attention_weight_min": -13.902965545654297,
      "activations/layer20_attention_weight_max": 27.217384338378906,
      "activations/layer20_attention_weight_min": -26.525291442871094,
      "activations/layer21_attention_weight_max": 29.003355026245117,
      "activations/layer21_attention_weight_min": -25.36411476135254,
      "activations/layer22_attention_weight_max": 38.85340118408203,
      "activations/layer22_attention_weight_min": -31.253150939941406,
      "activations/layer23_attention_weight_max": 36.87007522583008,
      "activations/layer23_attention_weight_min": -26.8115234375,
      "activations/layer2_attention_weight_max": 30.905330657958984,
      "activations/layer2_attention_weight_min": -29.024633407592773,
      "activations/layer3_attention_weight_max": 93.78633117675781,
      "activations/layer3_attention_weight_min": -95.47488403320312,
      "activations/layer4_attention_weight_max": 101.72770690917969,
      "activations/layer4_attention_weight_min": -105.95903778076172,
      "activations/layer5_attention_weight_max": 54.114097595214844,
      "activations/layer5_attention_weight_min": -61.54494857788086,
      "activations/layer6_attention_weight_max": 43.260616302490234,
      "activations/layer6_attention_weight_min": -46.02607345581055,
      "activations/layer7_attention_weight_max": 92.4354248046875,
      "activations/layer7_attention_weight_min": -95.1010513305664,
      "activations/layer8_attention_weight_max": 41.4465446472168,
      "activations/layer8_attention_weight_min": -44.27333450317383,
      "activations/layer9_attention_weight_max": 31.998489379882812,
      "activations/layer9_attention_weight_min": -34.84650802612305,
      "epoch": 18.76,
      "learning_rate": 2.9302272727272726e-05,
      "loss": 2.736,
      "step": 322900
    },
    {
      "activations/layer0_attention_weight_max": 15.547602653503418,
      "activations/layer0_attention_weight_min": -13.720373153686523,
      "activations/layer10_attention_weight_max": 35.66259002685547,
      "activations/layer10_attention_weight_min": -38.589080810546875,
      "activations/layer11_attention_weight_max": 36.767066955566406,
      "activations/layer11_attention_weight_min": -37.105262756347656,
      "activations/layer12_attention_weight_max": 27.499759674072266,
      "activations/layer12_attention_weight_min": -28.263263702392578,
      "activations/layer13_attention_weight_max": 42.63764572143555,
      "activations/layer13_attention_weight_min": -36.7149772644043,
      "activations/layer14_attention_weight_max": 44.96717834472656,
      "activations/layer14_attention_weight_min": -39.51164245605469,
      "activations/layer15_attention_weight_max": 42.053775787353516,
      "activations/layer15_attention_weight_min": -35.51049041748047,
      "activations/layer16_attention_weight_max": 30.054580688476562,
      "activations/layer16_attention_weight_min": -26.565608978271484,
      "activations/layer17_attention_weight_max": 31.068336486816406,
      "activations/layer17_attention_weight_min": -25.6508731842041,
      "activations/layer18_attention_weight_max": 29.944725036621094,
      "activations/layer18_attention_weight_min": -25.3359432220459,
      "activations/layer19_attention_weight_max": 36.14729690551758,
      "activations/layer19_attention_weight_min": -31.41770362854004,
      "activations/layer1_attention_weight_max": 16.92544937133789,
      "activations/layer1_attention_weight_min": -15.945691108703613,
      "activations/layer20_attention_weight_max": 29.795562744140625,
      "activations/layer20_attention_weight_min": -28.02205467224121,
      "activations/layer21_attention_weight_max": 28.77020263671875,
      "activations/layer21_attention_weight_min": -24.96807289123535,
      "activations/layer22_attention_weight_max": 41.439544677734375,
      "activations/layer22_attention_weight_min": -33.17242431640625,
      "activations/layer23_attention_weight_max": 32.93805694580078,
      "activations/layer23_attention_weight_min": -25.246076583862305,
      "activations/layer2_attention_weight_max": 31.91677474975586,
      "activations/layer2_attention_weight_min": -31.199810028076172,
      "activations/layer3_attention_weight_max": 95.10029602050781,
      "activations/layer3_attention_weight_min": -96.75359344482422,
      "activations/layer4_attention_weight_max": 106.90565490722656,
      "activations/layer4_attention_weight_min": -107.42120361328125,
      "activations/layer5_attention_weight_max": 51.93728256225586,
      "activations/layer5_attention_weight_min": -59.946189880371094,
      "activations/layer6_attention_weight_max": 45.111900329589844,
      "activations/layer6_attention_weight_min": -45.89897918701172,
      "activations/layer7_attention_weight_max": 96.31482696533203,
      "activations/layer7_attention_weight_min": -98.90829467773438,
      "activations/layer8_attention_weight_max": 44.4564094543457,
      "activations/layer8_attention_weight_min": -45.57645034790039,
      "activations/layer9_attention_weight_max": 35.665443420410156,
      "activations/layer9_attention_weight_min": -36.81276321411133,
      "epoch": 18.77,
      "learning_rate": 2.9283333333333328e-05,
      "loss": 2.7148,
      "step": 322950
    },
    {
      "activations/layer0_attention_weight_max": 15.292798042297363,
      "activations/layer0_attention_weight_min": -12.82554817199707,
      "activations/layer10_attention_weight_max": 33.244171142578125,
      "activations/layer10_attention_weight_min": -34.51351547241211,
      "activations/layer11_attention_weight_max": 33.56489944458008,
      "activations/layer11_attention_weight_min": -36.792091369628906,
      "activations/layer12_attention_weight_max": 29.139816284179688,
      "activations/layer12_attention_weight_min": -28.425752639770508,
      "activations/layer13_attention_weight_max": 41.78694152832031,
      "activations/layer13_attention_weight_min": -35.130455017089844,
      "activations/layer14_attention_weight_max": 44.84530258178711,
      "activations/layer14_attention_weight_min": -39.0639762878418,
      "activations/layer15_attention_weight_max": 42.823421478271484,
      "activations/layer15_attention_weight_min": -36.71693801879883,
      "activations/layer16_attention_weight_max": 32.01896286010742,
      "activations/layer16_attention_weight_min": -30.386804580688477,
      "activations/layer17_attention_weight_max": 29.59404945373535,
      "activations/layer17_attention_weight_min": -27.58470916748047,
      "activations/layer18_attention_weight_max": 30.775474548339844,
      "activations/layer18_attention_weight_min": -27.665842056274414,
      "activations/layer19_attention_weight_max": 36.286537170410156,
      "activations/layer19_attention_weight_min": -35.81417465209961,
      "activations/layer1_attention_weight_max": 15.518685340881348,
      "activations/layer1_attention_weight_min": -12.780765533447266,
      "activations/layer20_attention_weight_max": 30.447099685668945,
      "activations/layer20_attention_weight_min": -28.102630615234375,
      "activations/layer21_attention_weight_max": 29.01436996459961,
      "activations/layer21_attention_weight_min": -27.261838912963867,
      "activations/layer22_attention_weight_max": 41.23337173461914,
      "activations/layer22_attention_weight_min": -35.699153900146484,
      "activations/layer23_attention_weight_max": 33.711517333984375,
      "activations/layer23_attention_weight_min": -29.483503341674805,
      "activations/layer2_attention_weight_max": 28.51818084716797,
      "activations/layer2_attention_weight_min": -28.3298397064209,
      "activations/layer3_attention_weight_max": 89.1427230834961,
      "activations/layer3_attention_weight_min": -94.18208312988281,
      "activations/layer4_attention_weight_max": 99.3113021850586,
      "activations/layer4_attention_weight_min": -101.7464370727539,
      "activations/layer5_attention_weight_max": 47.812225341796875,
      "activations/layer5_attention_weight_min": -62.802791595458984,
      "activations/layer6_attention_weight_max": 42.80845260620117,
      "activations/layer6_attention_weight_min": -46.70732116699219,
      "activations/layer7_attention_weight_max": 91.99327850341797,
      "activations/layer7_attention_weight_min": -100.4398422241211,
      "activations/layer8_attention_weight_max": 42.67512893676758,
      "activations/layer8_attention_weight_min": -45.41204071044922,
      "activations/layer9_attention_weight_max": 31.051084518432617,
      "activations/layer9_attention_weight_min": -33.90815353393555,
      "epoch": 18.77,
      "learning_rate": 2.9264393939393936e-05,
      "loss": 2.7318,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.544,
      "eval_samples_per_second": 502.577,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.544,
      "eval_openwebtext_samples_per_second": 502.577,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0245,
      "eval_wikitext_samples_per_second": 225.242,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_lambada_loss": 2.421875,
      "eval_lambada_ppl": 11.26696508157019,
      "eval_lambada_runtime": 9.5625,
      "eval_lambada_samples_per_second": 509.175,
      "step": 323000
    },
    {
      "activations/layer0_attention_weight_max": 15.500146865844727,
      "activations/layer0_attention_weight_min": -12.152802467346191,
      "activations/layer10_attention_weight_max": 37.48982238769531,
      "activations/layer10_attention_weight_min": -34.782752990722656,
      "activations/layer11_attention_weight_max": 35.60747528076172,
      "activations/layer11_attention_weight_min": -34.47502136230469,
      "activations/layer12_attention_weight_max": 26.44845962524414,
      "activations/layer12_attention_weight_min": -28.699308395385742,
      "activations/layer13_attention_weight_max": 40.58819580078125,
      "activations/layer13_attention_weight_min": -37.13661575317383,
      "activations/layer14_attention_weight_max": 42.24725341796875,
      "activations/layer14_attention_weight_min": -39.7893180847168,
      "activations/layer15_attention_weight_max": 42.001808166503906,
      "activations/layer15_attention_weight_min": -36.99110794067383,
      "activations/layer16_attention_weight_max": 35.44100570678711,
      "activations/layer16_attention_weight_min": -29.595897674560547,
      "activations/layer17_attention_weight_max": 33.08503341674805,
      "activations/layer17_attention_weight_min": -26.708343505859375,
      "activations/layer18_attention_weight_max": 34.091087341308594,
      "activations/layer18_attention_weight_min": -26.434484481811523,
      "activations/layer19_attention_weight_max": 32.81120300292969,
      "activations/layer19_attention_weight_min": -31.945068359375,
      "activations/layer1_attention_weight_max": 16.776704788208008,
      "activations/layer1_attention_weight_min": -15.449392318725586,
      "activations/layer20_attention_weight_max": 28.51268768310547,
      "activations/layer20_attention_weight_min": -24.514713287353516,
      "activations/layer21_attention_weight_max": 27.490360260009766,
      "activations/layer21_attention_weight_min": -24.39469337463379,
      "activations/layer22_attention_weight_max": 46.32999038696289,
      "activations/layer22_attention_weight_min": -32.219058990478516,
      "activations/layer23_attention_weight_max": 33.77201461791992,
      "activations/layer23_attention_weight_min": -28.753402709960938,
      "activations/layer2_attention_weight_max": 31.452970504760742,
      "activations/layer2_attention_weight_min": -30.15697479248047,
      "activations/layer3_attention_weight_max": 96.17919921875,
      "activations/layer3_attention_weight_min": -99.6332778930664,
      "activations/layer4_attention_weight_max": 105.54541015625,
      "activations/layer4_attention_weight_min": -109.38702392578125,
      "activations/layer5_attention_weight_max": 53.27722930908203,
      "activations/layer5_attention_weight_min": -62.85942077636719,
      "activations/layer6_attention_weight_max": 47.060367584228516,
      "activations/layer6_attention_weight_min": -48.17805480957031,
      "activations/layer7_attention_weight_max": 97.9628677368164,
      "activations/layer7_attention_weight_min": -100.60782623291016,
      "activations/layer8_attention_weight_max": 43.08964538574219,
      "activations/layer8_attention_weight_min": -45.84749221801758,
      "activations/layer9_attention_weight_max": 32.50394058227539,
      "activations/layer9_attention_weight_min": -34.974281311035156,
      "epoch": 18.77,
      "learning_rate": 2.924545454545454e-05,
      "loss": 2.727,
      "step": 323050
    },
    {
      "activations/layer0_attention_weight_max": 17.125402450561523,
      "activations/layer0_attention_weight_min": -12.828365325927734,
      "activations/layer10_attention_weight_max": 34.37779998779297,
      "activations/layer10_attention_weight_min": -35.340087890625,
      "activations/layer11_attention_weight_max": 35.86196517944336,
      "activations/layer11_attention_weight_min": -35.24981689453125,
      "activations/layer12_attention_weight_max": 30.09125328063965,
      "activations/layer12_attention_weight_min": -27.574684143066406,
      "activations/layer13_attention_weight_max": 44.701438903808594,
      "activations/layer13_attention_weight_min": -36.492652893066406,
      "activations/layer14_attention_weight_max": 49.4209098815918,
      "activations/layer14_attention_weight_min": -38.26192855834961,
      "activations/layer15_attention_weight_max": 45.14268112182617,
      "activations/layer15_attention_weight_min": -36.441368103027344,
      "activations/layer16_attention_weight_max": 33.198387145996094,
      "activations/layer16_attention_weight_min": -28.3548641204834,
      "activations/layer17_attention_weight_max": 31.847782135009766,
      "activations/layer17_attention_weight_min": -26.90509033203125,
      "activations/layer18_attention_weight_max": 35.22086715698242,
      "activations/layer18_attention_weight_min": -24.660768508911133,
      "activations/layer19_attention_weight_max": 36.47567367553711,
      "activations/layer19_attention_weight_min": -32.57210922241211,
      "activations/layer1_attention_weight_max": 15.864662170410156,
      "activations/layer1_attention_weight_min": -15.531743049621582,
      "activations/layer20_attention_weight_max": 32.034175872802734,
      "activations/layer20_attention_weight_min": -25.979618072509766,
      "activations/layer21_attention_weight_max": 32.45410919189453,
      "activations/layer21_attention_weight_min": -23.863262176513672,
      "activations/layer22_attention_weight_max": 47.42111587524414,
      "activations/layer22_attention_weight_min": -32.2033805847168,
      "activations/layer23_attention_weight_max": 34.288726806640625,
      "activations/layer23_attention_weight_min": -26.872716903686523,
      "activations/layer2_attention_weight_max": 29.076635360717773,
      "activations/layer2_attention_weight_min": -29.638442993164062,
      "activations/layer3_attention_weight_max": 93.23905181884766,
      "activations/layer3_attention_weight_min": -93.28807067871094,
      "activations/layer4_attention_weight_max": 104.17793273925781,
      "activations/layer4_attention_weight_min": -104.50858306884766,
      "activations/layer5_attention_weight_max": 50.725704193115234,
      "activations/layer5_attention_weight_min": -62.11812973022461,
      "activations/layer6_attention_weight_max": 44.20228576660156,
      "activations/layer6_attention_weight_min": -45.571815490722656,
      "activations/layer7_attention_weight_max": 94.89423370361328,
      "activations/layer7_attention_weight_min": -93.81786346435547,
      "activations/layer8_attention_weight_max": 39.839569091796875,
      "activations/layer8_attention_weight_min": -42.96577835083008,
      "activations/layer9_attention_weight_max": 31.0994873046875,
      "activations/layer9_attention_weight_min": -35.06843185424805,
      "epoch": 18.77,
      "learning_rate": 2.922651515151515e-05,
      "loss": 2.7145,
      "step": 323100
    },
    {
      "activations/layer0_attention_weight_max": 18.306734085083008,
      "activations/layer0_attention_weight_min": -11.900137901306152,
      "activations/layer10_attention_weight_max": 32.56212615966797,
      "activations/layer10_attention_weight_min": -35.38056182861328,
      "activations/layer11_attention_weight_max": 34.66388702392578,
      "activations/layer11_attention_weight_min": -35.406517028808594,
      "activations/layer12_attention_weight_max": 27.06881332397461,
      "activations/layer12_attention_weight_min": -28.447532653808594,
      "activations/layer13_attention_weight_max": 38.24325180053711,
      "activations/layer13_attention_weight_min": -37.21943283081055,
      "activations/layer14_attention_weight_max": 43.60076904296875,
      "activations/layer14_attention_weight_min": -40.110145568847656,
      "activations/layer15_attention_weight_max": 39.74115753173828,
      "activations/layer15_attention_weight_min": -38.52288055419922,
      "activations/layer16_attention_weight_max": 29.11690902709961,
      "activations/layer16_attention_weight_min": -31.061389923095703,
      "activations/layer17_attention_weight_max": 29.99772834777832,
      "activations/layer17_attention_weight_min": -28.678945541381836,
      "activations/layer18_attention_weight_max": 36.94292449951172,
      "activations/layer18_attention_weight_min": -29.82391357421875,
      "activations/layer19_attention_weight_max": 35.542423248291016,
      "activations/layer19_attention_weight_min": -33.43877410888672,
      "activations/layer1_attention_weight_max": 16.172060012817383,
      "activations/layer1_attention_weight_min": -14.324002265930176,
      "activations/layer20_attention_weight_max": 28.329633712768555,
      "activations/layer20_attention_weight_min": -26.081138610839844,
      "activations/layer21_attention_weight_max": 28.90036392211914,
      "activations/layer21_attention_weight_min": -27.320219039916992,
      "activations/layer22_attention_weight_max": 40.42504119873047,
      "activations/layer22_attention_weight_min": -35.51216506958008,
      "activations/layer23_attention_weight_max": 32.08564758300781,
      "activations/layer23_attention_weight_min": -28.963911056518555,
      "activations/layer2_attention_weight_max": 32.18669128417969,
      "activations/layer2_attention_weight_min": -30.240428924560547,
      "activations/layer3_attention_weight_max": 94.08316802978516,
      "activations/layer3_attention_weight_min": -96.58332061767578,
      "activations/layer4_attention_weight_max": 106.409423828125,
      "activations/layer4_attention_weight_min": -105.06971740722656,
      "activations/layer5_attention_weight_max": 51.58245086669922,
      "activations/layer5_attention_weight_min": -63.78759002685547,
      "activations/layer6_attention_weight_max": 43.7641716003418,
      "activations/layer6_attention_weight_min": -48.743778228759766,
      "activations/layer7_attention_weight_max": 90.64291381835938,
      "activations/layer7_attention_weight_min": -99.31558227539062,
      "activations/layer8_attention_weight_max": 40.44279479980469,
      "activations/layer8_attention_weight_min": -44.68646240234375,
      "activations/layer9_attention_weight_max": 31.8726863861084,
      "activations/layer9_attention_weight_min": -34.57730484008789,
      "epoch": 18.78,
      "learning_rate": 2.9207575757575754e-05,
      "loss": 2.731,
      "step": 323150
    },
    {
      "activations/layer0_attention_weight_max": 16.000146865844727,
      "activations/layer0_attention_weight_min": -12.073673248291016,
      "activations/layer10_attention_weight_max": 35.25514602661133,
      "activations/layer10_attention_weight_min": -35.498992919921875,
      "activations/layer11_attention_weight_max": 36.07955551147461,
      "activations/layer11_attention_weight_min": -37.46699523925781,
      "activations/layer12_attention_weight_max": 27.6719913482666,
      "activations/layer12_attention_weight_min": -28.358701705932617,
      "activations/layer13_attention_weight_max": 45.02144241333008,
      "activations/layer13_attention_weight_min": -39.96684265136719,
      "activations/layer14_attention_weight_max": 44.88425827026367,
      "activations/layer14_attention_weight_min": -41.47462463378906,
      "activations/layer15_attention_weight_max": 43.32086944580078,
      "activations/layer15_attention_weight_min": -37.72905349731445,
      "activations/layer16_attention_weight_max": 32.991661071777344,
      "activations/layer16_attention_weight_min": -30.378564834594727,
      "activations/layer17_attention_weight_max": 31.685312271118164,
      "activations/layer17_attention_weight_min": -27.16386604309082,
      "activations/layer18_attention_weight_max": 31.066757202148438,
      "activations/layer18_attention_weight_min": -24.654857635498047,
      "activations/layer19_attention_weight_max": 35.641719818115234,
      "activations/layer19_attention_weight_min": -31.243207931518555,
      "activations/layer1_attention_weight_max": 16.03081703186035,
      "activations/layer1_attention_weight_min": -14.206463813781738,
      "activations/layer20_attention_weight_max": 30.34569549560547,
      "activations/layer20_attention_weight_min": -25.41073226928711,
      "activations/layer21_attention_weight_max": 29.601566314697266,
      "activations/layer21_attention_weight_min": -25.804569244384766,
      "activations/layer22_attention_weight_max": 42.759674072265625,
      "activations/layer22_attention_weight_min": -31.060319900512695,
      "activations/layer23_attention_weight_max": 33.761634826660156,
      "activations/layer23_attention_weight_min": -25.659391403198242,
      "activations/layer2_attention_weight_max": 32.649452209472656,
      "activations/layer2_attention_weight_min": -33.10920715332031,
      "activations/layer3_attention_weight_max": 98.3081283569336,
      "activations/layer3_attention_weight_min": -104.39449310302734,
      "activations/layer4_attention_weight_max": 108.73945617675781,
      "activations/layer4_attention_weight_min": -116.68157958984375,
      "activations/layer5_attention_weight_max": 49.50405502319336,
      "activations/layer5_attention_weight_min": -61.8175163269043,
      "activations/layer6_attention_weight_max": 45.88654327392578,
      "activations/layer6_attention_weight_min": -49.62181091308594,
      "activations/layer7_attention_weight_max": 97.10999298095703,
      "activations/layer7_attention_weight_min": -100.40179443359375,
      "activations/layer8_attention_weight_max": 43.45566940307617,
      "activations/layer8_attention_weight_min": -44.78468704223633,
      "activations/layer9_attention_weight_max": 33.368797302246094,
      "activations/layer9_attention_weight_min": -35.79283905029297,
      "epoch": 18.78,
      "learning_rate": 2.9188636363636363e-05,
      "loss": 2.7303,
      "step": 323200
    },
    {
      "activations/layer0_attention_weight_max": 15.728072166442871,
      "activations/layer0_attention_weight_min": -13.081623077392578,
      "activations/layer10_attention_weight_max": 35.66286087036133,
      "activations/layer10_attention_weight_min": -34.64745330810547,
      "activations/layer11_attention_weight_max": 34.627017974853516,
      "activations/layer11_attention_weight_min": -34.92930603027344,
      "activations/layer12_attention_weight_max": 26.824752807617188,
      "activations/layer12_attention_weight_min": -26.804636001586914,
      "activations/layer13_attention_weight_max": 38.21936798095703,
      "activations/layer13_attention_weight_min": -35.97652053833008,
      "activations/layer14_attention_weight_max": 43.048614501953125,
      "activations/layer14_attention_weight_min": -37.54643249511719,
      "activations/layer15_attention_weight_max": 39.64103317260742,
      "activations/layer15_attention_weight_min": -35.0460090637207,
      "activations/layer16_attention_weight_max": 33.794681549072266,
      "activations/layer16_attention_weight_min": -29.038854598999023,
      "activations/layer17_attention_weight_max": 31.45704460144043,
      "activations/layer17_attention_weight_min": -28.602304458618164,
      "activations/layer18_attention_weight_max": 31.90008544921875,
      "activations/layer18_attention_weight_min": -27.75777816772461,
      "activations/layer19_attention_weight_max": 38.60194778442383,
      "activations/layer19_attention_weight_min": -31.7898006439209,
      "activations/layer1_attention_weight_max": 16.2380428314209,
      "activations/layer1_attention_weight_min": -15.683353424072266,
      "activations/layer20_attention_weight_max": 30.46578598022461,
      "activations/layer20_attention_weight_min": -26.681978225708008,
      "activations/layer21_attention_weight_max": 29.744749069213867,
      "activations/layer21_attention_weight_min": -25.626760482788086,
      "activations/layer22_attention_weight_max": 42.394439697265625,
      "activations/layer22_attention_weight_min": -32.000450134277344,
      "activations/layer23_attention_weight_max": 34.88981628417969,
      "activations/layer23_attention_weight_min": -26.804889678955078,
      "activations/layer2_attention_weight_max": 31.413738250732422,
      "activations/layer2_attention_weight_min": -30.783084869384766,
      "activations/layer3_attention_weight_max": 95.67235565185547,
      "activations/layer3_attention_weight_min": -99.08893585205078,
      "activations/layer4_attention_weight_max": 104.81269836425781,
      "activations/layer4_attention_weight_min": -106.8656997680664,
      "activations/layer5_attention_weight_max": 50.70500183105469,
      "activations/layer5_attention_weight_min": -64.20234680175781,
      "activations/layer6_attention_weight_max": 45.13447952270508,
      "activations/layer6_attention_weight_min": -47.15176010131836,
      "activations/layer7_attention_weight_max": 94.1112060546875,
      "activations/layer7_attention_weight_min": -98.28973388671875,
      "activations/layer8_attention_weight_max": 43.304256439208984,
      "activations/layer8_attention_weight_min": -44.36264419555664,
      "activations/layer9_attention_weight_max": 34.71030044555664,
      "activations/layer9_attention_weight_min": -35.806243896484375,
      "epoch": 18.78,
      "learning_rate": 2.9169696969696964e-05,
      "loss": 2.7179,
      "step": 323250
    },
    {
      "activations/layer0_attention_weight_max": 16.198043823242188,
      "activations/layer0_attention_weight_min": -12.905759811401367,
      "activations/layer10_attention_weight_max": 33.45185470581055,
      "activations/layer10_attention_weight_min": -33.45484924316406,
      "activations/layer11_attention_weight_max": 34.02349090576172,
      "activations/layer11_attention_weight_min": -33.14971923828125,
      "activations/layer12_attention_weight_max": 28.64752960205078,
      "activations/layer12_attention_weight_min": -26.92595100402832,
      "activations/layer13_attention_weight_max": 41.877052307128906,
      "activations/layer13_attention_weight_min": -35.676353454589844,
      "activations/layer14_attention_weight_max": 44.34153747558594,
      "activations/layer14_attention_weight_min": -40.39783477783203,
      "activations/layer15_attention_weight_max": 38.89103698730469,
      "activations/layer15_attention_weight_min": -35.58035659790039,
      "activations/layer16_attention_weight_max": 29.864728927612305,
      "activations/layer16_attention_weight_min": -29.894893646240234,
      "activations/layer17_attention_weight_max": 31.743425369262695,
      "activations/layer17_attention_weight_min": -25.444820404052734,
      "activations/layer18_attention_weight_max": 31.482450485229492,
      "activations/layer18_attention_weight_min": -24.96603012084961,
      "activations/layer19_attention_weight_max": 35.72343826293945,
      "activations/layer19_attention_weight_min": -33.629878997802734,
      "activations/layer1_attention_weight_max": 15.710777282714844,
      "activations/layer1_attention_weight_min": -13.506790161132812,
      "activations/layer20_attention_weight_max": 29.137672424316406,
      "activations/layer20_attention_weight_min": -26.84147834777832,
      "activations/layer21_attention_weight_max": 30.12405776977539,
      "activations/layer21_attention_weight_min": -24.857938766479492,
      "activations/layer22_attention_weight_max": 43.22107696533203,
      "activations/layer22_attention_weight_min": -32.17456817626953,
      "activations/layer23_attention_weight_max": 32.47844696044922,
      "activations/layer23_attention_weight_min": -25.816556930541992,
      "activations/layer2_attention_weight_max": 30.86025047302246,
      "activations/layer2_attention_weight_min": -29.546127319335938,
      "activations/layer3_attention_weight_max": 96.42611694335938,
      "activations/layer3_attention_weight_min": -93.43871307373047,
      "activations/layer4_attention_weight_max": 102.6874771118164,
      "activations/layer4_attention_weight_min": -102.54695129394531,
      "activations/layer5_attention_weight_max": 54.0640983581543,
      "activations/layer5_attention_weight_min": -61.36054992675781,
      "activations/layer6_attention_weight_max": 41.739437103271484,
      "activations/layer6_attention_weight_min": -45.07046127319336,
      "activations/layer7_attention_weight_max": 86.9259033203125,
      "activations/layer7_attention_weight_min": -100.64866638183594,
      "activations/layer8_attention_weight_max": 41.32627487182617,
      "activations/layer8_attention_weight_min": -41.38832473754883,
      "activations/layer9_attention_weight_max": 32.023677825927734,
      "activations/layer9_attention_weight_min": -32.169010162353516,
      "epoch": 18.79,
      "learning_rate": 2.9150757575757572e-05,
      "loss": 2.7219,
      "step": 323300
    },
    {
      "activations/layer0_attention_weight_max": 16.003040313720703,
      "activations/layer0_attention_weight_min": -12.321830749511719,
      "activations/layer10_attention_weight_max": 34.96982192993164,
      "activations/layer10_attention_weight_min": -33.645263671875,
      "activations/layer11_attention_weight_max": 34.028533935546875,
      "activations/layer11_attention_weight_min": -34.82096481323242,
      "activations/layer12_attention_weight_max": 25.931289672851562,
      "activations/layer12_attention_weight_min": -27.719205856323242,
      "activations/layer13_attention_weight_max": 38.17766571044922,
      "activations/layer13_attention_weight_min": -35.67524337768555,
      "activations/layer14_attention_weight_max": 43.161903381347656,
      "activations/layer14_attention_weight_min": -38.22836685180664,
      "activations/layer15_attention_weight_max": 37.48066329956055,
      "activations/layer15_attention_weight_min": -35.933815002441406,
      "activations/layer16_attention_weight_max": 30.313501358032227,
      "activations/layer16_attention_weight_min": -28.050981521606445,
      "activations/layer17_attention_weight_max": 28.866201400756836,
      "activations/layer17_attention_weight_min": -27.074535369873047,
      "activations/layer18_attention_weight_max": 30.744176864624023,
      "activations/layer18_attention_weight_min": -27.310556411743164,
      "activations/layer19_attention_weight_max": 32.65593338012695,
      "activations/layer19_attention_weight_min": -33.856719970703125,
      "activations/layer1_attention_weight_max": 16.486196517944336,
      "activations/layer1_attention_weight_min": -14.516545295715332,
      "activations/layer20_attention_weight_max": 28.700681686401367,
      "activations/layer20_attention_weight_min": -26.46526336669922,
      "activations/layer21_attention_weight_max": 26.202682495117188,
      "activations/layer21_attention_weight_min": -25.327585220336914,
      "activations/layer22_attention_weight_max": 44.51018524169922,
      "activations/layer22_attention_weight_min": -32.213951110839844,
      "activations/layer23_attention_weight_max": 32.80466842651367,
      "activations/layer23_attention_weight_min": -26.915111541748047,
      "activations/layer2_attention_weight_max": 32.215694427490234,
      "activations/layer2_attention_weight_min": -31.917844772338867,
      "activations/layer3_attention_weight_max": 98.66648864746094,
      "activations/layer3_attention_weight_min": -103.18136596679688,
      "activations/layer4_attention_weight_max": 106.34124755859375,
      "activations/layer4_attention_weight_min": -110.17894744873047,
      "activations/layer5_attention_weight_max": 50.96331787109375,
      "activations/layer5_attention_weight_min": -63.546077728271484,
      "activations/layer6_attention_weight_max": 46.44033432006836,
      "activations/layer6_attention_weight_min": -47.621559143066406,
      "activations/layer7_attention_weight_max": 95.11861419677734,
      "activations/layer7_attention_weight_min": -99.23612976074219,
      "activations/layer8_attention_weight_max": 46.02400588989258,
      "activations/layer8_attention_weight_min": -45.18764114379883,
      "activations/layer9_attention_weight_max": 34.98918151855469,
      "activations/layer9_attention_weight_min": -34.03194808959961,
      "epoch": 18.79,
      "learning_rate": 2.913181818181818e-05,
      "loss": 2.7114,
      "step": 323350
    },
    {
      "activations/layer0_attention_weight_max": 16.430315017700195,
      "activations/layer0_attention_weight_min": -13.076787948608398,
      "activations/layer10_attention_weight_max": 32.21620178222656,
      "activations/layer10_attention_weight_min": -33.912254333496094,
      "activations/layer11_attention_weight_max": 32.900299072265625,
      "activations/layer11_attention_weight_min": -32.100250244140625,
      "activations/layer12_attention_weight_max": 25.42892074584961,
      "activations/layer12_attention_weight_min": -24.78888702392578,
      "activations/layer13_attention_weight_max": 38.189117431640625,
      "activations/layer13_attention_weight_min": -34.06782913208008,
      "activations/layer14_attention_weight_max": 42.52891159057617,
      "activations/layer14_attention_weight_min": -36.68224334716797,
      "activations/layer15_attention_weight_max": 37.31670379638672,
      "activations/layer15_attention_weight_min": -35.55951690673828,
      "activations/layer16_attention_weight_max": 30.911577224731445,
      "activations/layer16_attention_weight_min": -25.991668701171875,
      "activations/layer17_attention_weight_max": 27.46950912475586,
      "activations/layer17_attention_weight_min": -24.6279296875,
      "activations/layer18_attention_weight_max": 30.640426635742188,
      "activations/layer18_attention_weight_min": -24.398313522338867,
      "activations/layer19_attention_weight_max": 35.075355529785156,
      "activations/layer19_attention_weight_min": -30.129093170166016,
      "activations/layer1_attention_weight_max": 15.387460708618164,
      "activations/layer1_attention_weight_min": -14.120431900024414,
      "activations/layer20_attention_weight_max": 29.65859603881836,
      "activations/layer20_attention_weight_min": -23.72783660888672,
      "activations/layer21_attention_weight_max": 29.17894744873047,
      "activations/layer21_attention_weight_min": -24.074722290039062,
      "activations/layer22_attention_weight_max": 38.94261169433594,
      "activations/layer22_attention_weight_min": -30.489349365234375,
      "activations/layer23_attention_weight_max": 33.39635467529297,
      "activations/layer23_attention_weight_min": -24.610782623291016,
      "activations/layer2_attention_weight_max": 30.823776245117188,
      "activations/layer2_attention_weight_min": -31.566038131713867,
      "activations/layer3_attention_weight_max": 96.21807861328125,
      "activations/layer3_attention_weight_min": -99.72443389892578,
      "activations/layer4_attention_weight_max": 106.79991912841797,
      "activations/layer4_attention_weight_min": -110.1265640258789,
      "activations/layer5_attention_weight_max": 49.7235221862793,
      "activations/layer5_attention_weight_min": -61.894065856933594,
      "activations/layer6_attention_weight_max": 44.535823822021484,
      "activations/layer6_attention_weight_min": -47.6733512878418,
      "activations/layer7_attention_weight_max": 91.42498779296875,
      "activations/layer7_attention_weight_min": -96.26402282714844,
      "activations/layer8_attention_weight_max": 41.32801818847656,
      "activations/layer8_attention_weight_min": -44.985042572021484,
      "activations/layer9_attention_weight_max": 30.672744750976562,
      "activations/layer9_attention_weight_min": -34.313472747802734,
      "epoch": 18.79,
      "learning_rate": 2.9112878787878785e-05,
      "loss": 2.7311,
      "step": 323400
    },
    {
      "activations/layer0_attention_weight_max": 16.141651153564453,
      "activations/layer0_attention_weight_min": -12.521180152893066,
      "activations/layer10_attention_weight_max": 35.915130615234375,
      "activations/layer10_attention_weight_min": -35.243408203125,
      "activations/layer11_attention_weight_max": 37.30110549926758,
      "activations/layer11_attention_weight_min": -34.04755401611328,
      "activations/layer12_attention_weight_max": 28.36550521850586,
      "activations/layer12_attention_weight_min": -27.771501541137695,
      "activations/layer13_attention_weight_max": 41.06281661987305,
      "activations/layer13_attention_weight_min": -37.4350700378418,
      "activations/layer14_attention_weight_max": 44.8833122253418,
      "activations/layer14_attention_weight_min": -40.326942443847656,
      "activations/layer15_attention_weight_max": 43.60430145263672,
      "activations/layer15_attention_weight_min": -38.7191162109375,
      "activations/layer16_attention_weight_max": 31.004711151123047,
      "activations/layer16_attention_weight_min": -28.98903465270996,
      "activations/layer17_attention_weight_max": 33.30415344238281,
      "activations/layer17_attention_weight_min": -26.340696334838867,
      "activations/layer18_attention_weight_max": 34.627899169921875,
      "activations/layer18_attention_weight_min": -26.707948684692383,
      "activations/layer19_attention_weight_max": 37.11674118041992,
      "activations/layer19_attention_weight_min": -33.1500129699707,
      "activations/layer1_attention_weight_max": 17.00029182434082,
      "activations/layer1_attention_weight_min": -14.31855583190918,
      "activations/layer20_attention_weight_max": 32.2033805847168,
      "activations/layer20_attention_weight_min": -25.50445556640625,
      "activations/layer21_attention_weight_max": 30.258590698242188,
      "activations/layer21_attention_weight_min": -25.231481552124023,
      "activations/layer22_attention_weight_max": 43.218685150146484,
      "activations/layer22_attention_weight_min": -31.75399398803711,
      "activations/layer23_attention_weight_max": 36.30165481567383,
      "activations/layer23_attention_weight_min": -26.105609893798828,
      "activations/layer2_attention_weight_max": 31.348167419433594,
      "activations/layer2_attention_weight_min": -32.48851776123047,
      "activations/layer3_attention_weight_max": 97.08326721191406,
      "activations/layer3_attention_weight_min": -99.30055236816406,
      "activations/layer4_attention_weight_max": 108.57173156738281,
      "activations/layer4_attention_weight_min": -108.89647674560547,
      "activations/layer5_attention_weight_max": 50.360069274902344,
      "activations/layer5_attention_weight_min": -61.50126647949219,
      "activations/layer6_attention_weight_max": 43.26165008544922,
      "activations/layer6_attention_weight_min": -47.440895080566406,
      "activations/layer7_attention_weight_max": 96.36760711669922,
      "activations/layer7_attention_weight_min": -99.30056762695312,
      "activations/layer8_attention_weight_max": 44.4498176574707,
      "activations/layer8_attention_weight_min": -45.29184341430664,
      "activations/layer9_attention_weight_max": 32.837703704833984,
      "activations/layer9_attention_weight_min": -33.95071029663086,
      "epoch": 18.79,
      "learning_rate": 2.9093939393939394e-05,
      "loss": 2.7276,
      "step": 323450
    },
    {
      "activations/layer0_attention_weight_max": 15.987052917480469,
      "activations/layer0_attention_weight_min": -12.396418571472168,
      "activations/layer10_attention_weight_max": 39.7421760559082,
      "activations/layer10_attention_weight_min": -39.440589904785156,
      "activations/layer11_attention_weight_max": 38.236419677734375,
      "activations/layer11_attention_weight_min": -37.49148941040039,
      "activations/layer12_attention_weight_max": 29.85454559326172,
      "activations/layer12_attention_weight_min": -27.869434356689453,
      "activations/layer13_attention_weight_max": 44.39621353149414,
      "activations/layer13_attention_weight_min": -38.71699142456055,
      "activations/layer14_attention_weight_max": 48.959205627441406,
      "activations/layer14_attention_weight_min": -43.162208557128906,
      "activations/layer15_attention_weight_max": 45.841190338134766,
      "activations/layer15_attention_weight_min": -41.31282043457031,
      "activations/layer16_attention_weight_max": 39.23896789550781,
      "activations/layer16_attention_weight_min": -29.410503387451172,
      "activations/layer17_attention_weight_max": 33.4906120300293,
      "activations/layer17_attention_weight_min": -29.36492347717285,
      "activations/layer18_attention_weight_max": 35.76947784423828,
      "activations/layer18_attention_weight_min": -28.625864028930664,
      "activations/layer19_attention_weight_max": 38.25606155395508,
      "activations/layer19_attention_weight_min": -32.900917053222656,
      "activations/layer1_attention_weight_max": 15.673660278320312,
      "activations/layer1_attention_weight_min": -15.111030578613281,
      "activations/layer20_attention_weight_max": 32.15031433105469,
      "activations/layer20_attention_weight_min": -25.25847053527832,
      "activations/layer21_attention_weight_max": 35.95306396484375,
      "activations/layer21_attention_weight_min": -26.416439056396484,
      "activations/layer22_attention_weight_max": 47.71990966796875,
      "activations/layer22_attention_weight_min": -37.30595397949219,
      "activations/layer23_attention_weight_max": 36.50032043457031,
      "activations/layer23_attention_weight_min": -28.322568893432617,
      "activations/layer2_attention_weight_max": 31.340009689331055,
      "activations/layer2_attention_weight_min": -30.354965209960938,
      "activations/layer3_attention_weight_max": 97.27922058105469,
      "activations/layer3_attention_weight_min": -102.78046417236328,
      "activations/layer4_attention_weight_max": 106.18145751953125,
      "activations/layer4_attention_weight_min": -109.78655242919922,
      "activations/layer5_attention_weight_max": 51.5238151550293,
      "activations/layer5_attention_weight_min": -60.949493408203125,
      "activations/layer6_attention_weight_max": 47.2484016418457,
      "activations/layer6_attention_weight_min": -47.332576751708984,
      "activations/layer7_attention_weight_max": 92.46052551269531,
      "activations/layer7_attention_weight_min": -97.96513366699219,
      "activations/layer8_attention_weight_max": 45.35780334472656,
      "activations/layer8_attention_weight_min": -48.16057586669922,
      "activations/layer9_attention_weight_max": 37.583412170410156,
      "activations/layer9_attention_weight_min": -38.323890686035156,
      "epoch": 18.8,
      "learning_rate": 2.9074999999999995e-05,
      "loss": 2.7553,
      "step": 323500
    },
    {
      "activations/layer0_attention_weight_max": 16.24925422668457,
      "activations/layer0_attention_weight_min": -12.1517333984375,
      "activations/layer10_attention_weight_max": 35.25075149536133,
      "activations/layer10_attention_weight_min": -35.27018356323242,
      "activations/layer11_attention_weight_max": 32.89846420288086,
      "activations/layer11_attention_weight_min": -33.9881706237793,
      "activations/layer12_attention_weight_max": 26.953548431396484,
      "activations/layer12_attention_weight_min": -26.80350112915039,
      "activations/layer13_attention_weight_max": 38.847496032714844,
      "activations/layer13_attention_weight_min": -36.852989196777344,
      "activations/layer14_attention_weight_max": 49.844642639160156,
      "activations/layer14_attention_weight_min": -39.91301345825195,
      "activations/layer15_attention_weight_max": 38.445980072021484,
      "activations/layer15_attention_weight_min": -37.404075622558594,
      "activations/layer16_attention_weight_max": 29.849349975585938,
      "activations/layer16_attention_weight_min": -28.931743621826172,
      "activations/layer17_attention_weight_max": 28.076492309570312,
      "activations/layer17_attention_weight_min": -27.9298152923584,
      "activations/layer18_attention_weight_max": 29.674287796020508,
      "activations/layer18_attention_weight_min": -28.37112045288086,
      "activations/layer19_attention_weight_max": 33.029624938964844,
      "activations/layer19_attention_weight_min": -30.914072036743164,
      "activations/layer1_attention_weight_max": 17.96180534362793,
      "activations/layer1_attention_weight_min": -14.74413776397705,
      "activations/layer20_attention_weight_max": 28.057525634765625,
      "activations/layer20_attention_weight_min": -24.154306411743164,
      "activations/layer21_attention_weight_max": 25.7872371673584,
      "activations/layer21_attention_weight_min": -24.306028366088867,
      "activations/layer22_attention_weight_max": 38.456939697265625,
      "activations/layer22_attention_weight_min": -30.070966720581055,
      "activations/layer23_attention_weight_max": 33.82813262939453,
      "activations/layer23_attention_weight_min": -25.034154891967773,
      "activations/layer2_attention_weight_max": 31.236740112304688,
      "activations/layer2_attention_weight_min": -30.99687957763672,
      "activations/layer3_attention_weight_max": 98.39292907714844,
      "activations/layer3_attention_weight_min": -100.57154846191406,
      "activations/layer4_attention_weight_max": 102.70999908447266,
      "activations/layer4_attention_weight_min": -104.60738372802734,
      "activations/layer5_attention_weight_max": 51.532135009765625,
      "activations/layer5_attention_weight_min": -59.62403869628906,
      "activations/layer6_attention_weight_max": 42.12165451049805,
      "activations/layer6_attention_weight_min": -47.634273529052734,
      "activations/layer7_attention_weight_max": 90.24861907958984,
      "activations/layer7_attention_weight_min": -97.84352111816406,
      "activations/layer8_attention_weight_max": 40.209999084472656,
      "activations/layer8_attention_weight_min": -43.88428497314453,
      "activations/layer9_attention_weight_max": 30.825834274291992,
      "activations/layer9_attention_weight_min": -35.16527557373047,
      "epoch": 18.8,
      "learning_rate": 2.9056060606060603e-05,
      "loss": 2.7342,
      "step": 323550
    },
    {
      "activations/layer0_attention_weight_max": 16.526742935180664,
      "activations/layer0_attention_weight_min": -12.74695873260498,
      "activations/layer10_attention_weight_max": 33.150447845458984,
      "activations/layer10_attention_weight_min": -35.401004791259766,
      "activations/layer11_attention_weight_max": 35.36674499511719,
      "activations/layer11_attention_weight_min": -34.34208679199219,
      "activations/layer12_attention_weight_max": 28.234668731689453,
      "activations/layer12_attention_weight_min": -28.33426856994629,
      "activations/layer13_attention_weight_max": 40.706329345703125,
      "activations/layer13_attention_weight_min": -39.490203857421875,
      "activations/layer14_attention_weight_max": 47.991554260253906,
      "activations/layer14_attention_weight_min": -41.05309295654297,
      "activations/layer15_attention_weight_max": 44.27479934692383,
      "activations/layer15_attention_weight_min": -36.891902923583984,
      "activations/layer16_attention_weight_max": 33.233524322509766,
      "activations/layer16_attention_weight_min": -29.674495697021484,
      "activations/layer17_attention_weight_max": 32.338687896728516,
      "activations/layer17_attention_weight_min": -25.926334381103516,
      "activations/layer18_attention_weight_max": 33.868587493896484,
      "activations/layer18_attention_weight_min": -27.690326690673828,
      "activations/layer19_attention_weight_max": 33.514156341552734,
      "activations/layer19_attention_weight_min": -33.11833953857422,
      "activations/layer1_attention_weight_max": 15.93682861328125,
      "activations/layer1_attention_weight_min": -13.943553924560547,
      "activations/layer20_attention_weight_max": 29.220169067382812,
      "activations/layer20_attention_weight_min": -24.5869083404541,
      "activations/layer21_attention_weight_max": 28.84587860107422,
      "activations/layer21_attention_weight_min": -25.07006072998047,
      "activations/layer22_attention_weight_max": 46.29663848876953,
      "activations/layer22_attention_weight_min": -32.80542755126953,
      "activations/layer23_attention_weight_max": 35.252323150634766,
      "activations/layer23_attention_weight_min": -25.41230583190918,
      "activations/layer2_attention_weight_max": 31.153579711914062,
      "activations/layer2_attention_weight_min": -30.304344177246094,
      "activations/layer3_attention_weight_max": 95.01752471923828,
      "activations/layer3_attention_weight_min": -98.09029388427734,
      "activations/layer4_attention_weight_max": 99.62397766113281,
      "activations/layer4_attention_weight_min": -110.64115142822266,
      "activations/layer5_attention_weight_max": 49.3801383972168,
      "activations/layer5_attention_weight_min": -64.0982437133789,
      "activations/layer6_attention_weight_max": 42.34151077270508,
      "activations/layer6_attention_weight_min": -46.58833312988281,
      "activations/layer7_attention_weight_max": 95.5370864868164,
      "activations/layer7_attention_weight_min": -94.52190399169922,
      "activations/layer8_attention_weight_max": 39.643699645996094,
      "activations/layer8_attention_weight_min": -43.788448333740234,
      "activations/layer9_attention_weight_max": 32.24365997314453,
      "activations/layer9_attention_weight_min": -35.28642654418945,
      "epoch": 18.8,
      "learning_rate": 2.903712121212121e-05,
      "loss": 2.717,
      "step": 323600
    },
    {
      "activations/layer0_attention_weight_max": 16.095626831054688,
      "activations/layer0_attention_weight_min": -13.396574020385742,
      "activations/layer10_attention_weight_max": 35.79228591918945,
      "activations/layer10_attention_weight_min": -36.16727828979492,
      "activations/layer11_attention_weight_max": 36.00638198852539,
      "activations/layer11_attention_weight_min": -36.312625885009766,
      "activations/layer12_attention_weight_max": 27.834077835083008,
      "activations/layer12_attention_weight_min": -26.75153160095215,
      "activations/layer13_attention_weight_max": 43.396671295166016,
      "activations/layer13_attention_weight_min": -38.95087432861328,
      "activations/layer14_attention_weight_max": 45.50677490234375,
      "activations/layer14_attention_weight_min": -40.94173049926758,
      "activations/layer15_attention_weight_max": 41.7953987121582,
      "activations/layer15_attention_weight_min": -37.82831954956055,
      "activations/layer16_attention_weight_max": 30.912137985229492,
      "activations/layer16_attention_weight_min": -27.53826332092285,
      "activations/layer17_attention_weight_max": 34.32642364501953,
      "activations/layer17_attention_weight_min": -26.10987091064453,
      "activations/layer18_attention_weight_max": 34.6142463684082,
      "activations/layer18_attention_weight_min": -26.591909408569336,
      "activations/layer19_attention_weight_max": 36.334529876708984,
      "activations/layer19_attention_weight_min": -33.09000015258789,
      "activations/layer1_attention_weight_max": 15.57751750946045,
      "activations/layer1_attention_weight_min": -15.003576278686523,
      "activations/layer20_attention_weight_max": 29.28365707397461,
      "activations/layer20_attention_weight_min": -24.965654373168945,
      "activations/layer21_attention_weight_max": 30.845535278320312,
      "activations/layer21_attention_weight_min": -24.275787353515625,
      "activations/layer22_attention_weight_max": 40.14152908325195,
      "activations/layer22_attention_weight_min": -31.51749038696289,
      "activations/layer23_attention_weight_max": 31.08241844177246,
      "activations/layer23_attention_weight_min": -24.161914825439453,
      "activations/layer2_attention_weight_max": 30.524015426635742,
      "activations/layer2_attention_weight_min": -30.632383346557617,
      "activations/layer3_attention_weight_max": 97.35533142089844,
      "activations/layer3_attention_weight_min": -97.54373931884766,
      "activations/layer4_attention_weight_max": 104.0782241821289,
      "activations/layer4_attention_weight_min": -112.46651458740234,
      "activations/layer5_attention_weight_max": 49.23820114135742,
      "activations/layer5_attention_weight_min": -61.667381286621094,
      "activations/layer6_attention_weight_max": 43.61598587036133,
      "activations/layer6_attention_weight_min": -45.987892150878906,
      "activations/layer7_attention_weight_max": 94.38636779785156,
      "activations/layer7_attention_weight_min": -94.75468444824219,
      "activations/layer8_attention_weight_max": 44.3137092590332,
      "activations/layer8_attention_weight_min": -44.36964797973633,
      "activations/layer9_attention_weight_max": 34.946083068847656,
      "activations/layer9_attention_weight_min": -36.2283821105957,
      "epoch": 18.81,
      "learning_rate": 2.9018181818181817e-05,
      "loss": 2.7213,
      "step": 323650
    },
    {
      "activations/layer0_attention_weight_max": 15.580155372619629,
      "activations/layer0_attention_weight_min": -13.181129455566406,
      "activations/layer10_attention_weight_max": 32.216583251953125,
      "activations/layer10_attention_weight_min": -35.10947036743164,
      "activations/layer11_attention_weight_max": 34.777435302734375,
      "activations/layer11_attention_weight_min": -33.74571990966797,
      "activations/layer12_attention_weight_max": 28.62730598449707,
      "activations/layer12_attention_weight_min": -29.23897933959961,
      "activations/layer13_attention_weight_max": 42.129207611083984,
      "activations/layer13_attention_weight_min": -36.55735778808594,
      "activations/layer14_attention_weight_max": 43.1610221862793,
      "activations/layer14_attention_weight_min": -40.089229583740234,
      "activations/layer15_attention_weight_max": 39.482330322265625,
      "activations/layer15_attention_weight_min": -36.9993896484375,
      "activations/layer16_attention_weight_max": 30.545576095581055,
      "activations/layer16_attention_weight_min": -29.72951889038086,
      "activations/layer17_attention_weight_max": 29.86269187927246,
      "activations/layer17_attention_weight_min": -26.822574615478516,
      "activations/layer18_attention_weight_max": 33.747440338134766,
      "activations/layer18_attention_weight_min": -25.305418014526367,
      "activations/layer19_attention_weight_max": 34.715728759765625,
      "activations/layer19_attention_weight_min": -32.350196838378906,
      "activations/layer1_attention_weight_max": 15.416068077087402,
      "activations/layer1_attention_weight_min": -13.948432922363281,
      "activations/layer20_attention_weight_max": 28.678638458251953,
      "activations/layer20_attention_weight_min": -24.49703025817871,
      "activations/layer21_attention_weight_max": 28.999481201171875,
      "activations/layer21_attention_weight_min": -24.767019271850586,
      "activations/layer22_attention_weight_max": 42.1778564453125,
      "activations/layer22_attention_weight_min": -30.841386795043945,
      "activations/layer23_attention_weight_max": 31.66312026977539,
      "activations/layer23_attention_weight_min": -25.725568771362305,
      "activations/layer2_attention_weight_max": 29.707489013671875,
      "activations/layer2_attention_weight_min": -29.19827651977539,
      "activations/layer3_attention_weight_max": 98.32749938964844,
      "activations/layer3_attention_weight_min": -97.05025482177734,
      "activations/layer4_attention_weight_max": 107.57604217529297,
      "activations/layer4_attention_weight_min": -110.34442138671875,
      "activations/layer5_attention_weight_max": 50.556182861328125,
      "activations/layer5_attention_weight_min": -63.58126449584961,
      "activations/layer6_attention_weight_max": 46.96641159057617,
      "activations/layer6_attention_weight_min": -50.99775314331055,
      "activations/layer7_attention_weight_max": 92.25054168701172,
      "activations/layer7_attention_weight_min": -99.34046173095703,
      "activations/layer8_attention_weight_max": 43.24745559692383,
      "activations/layer8_attention_weight_min": -46.837669372558594,
      "activations/layer9_attention_weight_max": 33.22368240356445,
      "activations/layer9_attention_weight_min": -37.14614486694336,
      "epoch": 18.81,
      "learning_rate": 2.899924242424242e-05,
      "loss": 2.7336,
      "step": 323700
    },
    {
      "activations/layer0_attention_weight_max": 16.955860137939453,
      "activations/layer0_attention_weight_min": -12.344230651855469,
      "activations/layer10_attention_weight_max": 32.881446838378906,
      "activations/layer10_attention_weight_min": -35.12133026123047,
      "activations/layer11_attention_weight_max": 32.83905792236328,
      "activations/layer11_attention_weight_min": -33.854881286621094,
      "activations/layer12_attention_weight_max": 27.15148162841797,
      "activations/layer12_attention_weight_min": -26.85051727294922,
      "activations/layer13_attention_weight_max": 36.89257049560547,
      "activations/layer13_attention_weight_min": -34.425506591796875,
      "activations/layer14_attention_weight_max": 41.56581497192383,
      "activations/layer14_attention_weight_min": -37.55231475830078,
      "activations/layer15_attention_weight_max": 37.777183532714844,
      "activations/layer15_attention_weight_min": -34.42969512939453,
      "activations/layer16_attention_weight_max": 26.78837776184082,
      "activations/layer16_attention_weight_min": -30.833145141601562,
      "activations/layer17_attention_weight_max": 28.99155616760254,
      "activations/layer17_attention_weight_min": -26.87931251525879,
      "activations/layer18_attention_weight_max": 30.545211791992188,
      "activations/layer18_attention_weight_min": -28.363603591918945,
      "activations/layer19_attention_weight_max": 32.531803131103516,
      "activations/layer19_attention_weight_min": -34.48319625854492,
      "activations/layer1_attention_weight_max": 17.30896759033203,
      "activations/layer1_attention_weight_min": -14.406344413757324,
      "activations/layer20_attention_weight_max": 27.64419174194336,
      "activations/layer20_attention_weight_min": -26.128379821777344,
      "activations/layer21_attention_weight_max": 27.091217041015625,
      "activations/layer21_attention_weight_min": -26.283681869506836,
      "activations/layer22_attention_weight_max": 42.49565505981445,
      "activations/layer22_attention_weight_min": -35.48646545410156,
      "activations/layer23_attention_weight_max": 31.919174194335938,
      "activations/layer23_attention_weight_min": -28.47301483154297,
      "activations/layer2_attention_weight_max": 31.017301559448242,
      "activations/layer2_attention_weight_min": -30.662384033203125,
      "activations/layer3_attention_weight_max": 94.3348617553711,
      "activations/layer3_attention_weight_min": -99.08109283447266,
      "activations/layer4_attention_weight_max": 106.7779769897461,
      "activations/layer4_attention_weight_min": -109.82286834716797,
      "activations/layer5_attention_weight_max": 49.3695068359375,
      "activations/layer5_attention_weight_min": -63.99451446533203,
      "activations/layer6_attention_weight_max": 44.68189239501953,
      "activations/layer6_attention_weight_min": -48.64220428466797,
      "activations/layer7_attention_weight_max": 94.62484741210938,
      "activations/layer7_attention_weight_min": -100.79964447021484,
      "activations/layer8_attention_weight_max": 42.397274017333984,
      "activations/layer8_attention_weight_min": -45.07624053955078,
      "activations/layer9_attention_weight_max": 30.46417808532715,
      "activations/layer9_attention_weight_min": -34.178707122802734,
      "epoch": 18.81,
      "learning_rate": 2.898030303030303e-05,
      "loss": 2.7285,
      "step": 323750
    },
    {
      "activations/layer0_attention_weight_max": 15.626327514648438,
      "activations/layer0_attention_weight_min": -12.623132705688477,
      "activations/layer10_attention_weight_max": 37.134765625,
      "activations/layer10_attention_weight_min": -34.57522201538086,
      "activations/layer11_attention_weight_max": 34.95091247558594,
      "activations/layer11_attention_weight_min": -33.53706359863281,
      "activations/layer12_attention_weight_max": 27.85926628112793,
      "activations/layer12_attention_weight_min": -28.355302810668945,
      "activations/layer13_attention_weight_max": 41.621559143066406,
      "activations/layer13_attention_weight_min": -38.26930618286133,
      "activations/layer14_attention_weight_max": 46.205474853515625,
      "activations/layer14_attention_weight_min": -38.345359802246094,
      "activations/layer15_attention_weight_max": 40.94192123413086,
      "activations/layer15_attention_weight_min": -35.879356384277344,
      "activations/layer16_attention_weight_max": 31.613203048706055,
      "activations/layer16_attention_weight_min": -29.827529907226562,
      "activations/layer17_attention_weight_max": 31.952260971069336,
      "activations/layer17_attention_weight_min": -26.447202682495117,
      "activations/layer18_attention_weight_max": 32.143402099609375,
      "activations/layer18_attention_weight_min": -28.297725677490234,
      "activations/layer19_attention_weight_max": 36.867164611816406,
      "activations/layer19_attention_weight_min": -32.15501022338867,
      "activations/layer1_attention_weight_max": 15.653564453125,
      "activations/layer1_attention_weight_min": -14.055298805236816,
      "activations/layer20_attention_weight_max": 31.809192657470703,
      "activations/layer20_attention_weight_min": -27.137577056884766,
      "activations/layer21_attention_weight_max": 31.855606079101562,
      "activations/layer21_attention_weight_min": -26.159048080444336,
      "activations/layer22_attention_weight_max": 44.08680725097656,
      "activations/layer22_attention_weight_min": -32.8036003112793,
      "activations/layer23_attention_weight_max": 36.097389221191406,
      "activations/layer23_attention_weight_min": -26.892501831054688,
      "activations/layer2_attention_weight_max": 31.23907470703125,
      "activations/layer2_attention_weight_min": -30.49243927001953,
      "activations/layer3_attention_weight_max": 99.72035217285156,
      "activations/layer3_attention_weight_min": -99.8680648803711,
      "activations/layer4_attention_weight_max": 106.29048919677734,
      "activations/layer4_attention_weight_min": -109.10929107666016,
      "activations/layer5_attention_weight_max": 50.40563201904297,
      "activations/layer5_attention_weight_min": -61.33547592163086,
      "activations/layer6_attention_weight_max": 43.95286178588867,
      "activations/layer6_attention_weight_min": -45.47901916503906,
      "activations/layer7_attention_weight_max": 93.43338012695312,
      "activations/layer7_attention_weight_min": -94.28852844238281,
      "activations/layer8_attention_weight_max": 41.86962890625,
      "activations/layer8_attention_weight_min": -45.737937927246094,
      "activations/layer9_attention_weight_max": 33.33511734008789,
      "activations/layer9_attention_weight_min": -35.16420364379883,
      "epoch": 18.81,
      "learning_rate": 2.896136363636363e-05,
      "loss": 2.729,
      "step": 323800
    },
    {
      "activations/layer0_attention_weight_max": 15.488356590270996,
      "activations/layer0_attention_weight_min": -12.209724426269531,
      "activations/layer10_attention_weight_max": 33.477394104003906,
      "activations/layer10_attention_weight_min": -34.762847900390625,
      "activations/layer11_attention_weight_max": 33.76039123535156,
      "activations/layer11_attention_weight_min": -35.96498107910156,
      "activations/layer12_attention_weight_max": 26.29738998413086,
      "activations/layer12_attention_weight_min": -29.616056442260742,
      "activations/layer13_attention_weight_max": 39.43351364135742,
      "activations/layer13_attention_weight_min": -40.223793029785156,
      "activations/layer14_attention_weight_max": 45.084449768066406,
      "activations/layer14_attention_weight_min": -42.07511520385742,
      "activations/layer15_attention_weight_max": 38.64266586303711,
      "activations/layer15_attention_weight_min": -35.628108978271484,
      "activations/layer16_attention_weight_max": 31.885265350341797,
      "activations/layer16_attention_weight_min": -30.48407745361328,
      "activations/layer17_attention_weight_max": 30.30640411376953,
      "activations/layer17_attention_weight_min": -26.88614845275879,
      "activations/layer18_attention_weight_max": 32.719268798828125,
      "activations/layer18_attention_weight_min": -27.898868560791016,
      "activations/layer19_attention_weight_max": 35.815528869628906,
      "activations/layer19_attention_weight_min": -31.94795799255371,
      "activations/layer1_attention_weight_max": 15.997336387634277,
      "activations/layer1_attention_weight_min": -15.34377670288086,
      "activations/layer20_attention_weight_max": 28.40410041809082,
      "activations/layer20_attention_weight_min": -26.841901779174805,
      "activations/layer21_attention_weight_max": 28.72884178161621,
      "activations/layer21_attention_weight_min": -25.774578094482422,
      "activations/layer22_attention_weight_max": 41.48302459716797,
      "activations/layer22_attention_weight_min": -32.84185028076172,
      "activations/layer23_attention_weight_max": 31.966312408447266,
      "activations/layer23_attention_weight_min": -27.989734649658203,
      "activations/layer2_attention_weight_max": 30.963863372802734,
      "activations/layer2_attention_weight_min": -30.477767944335938,
      "activations/layer3_attention_weight_max": 97.54983520507812,
      "activations/layer3_attention_weight_min": -99.34722900390625,
      "activations/layer4_attention_weight_max": 109.8709945678711,
      "activations/layer4_attention_weight_min": -108.37086486816406,
      "activations/layer5_attention_weight_max": 53.39335250854492,
      "activations/layer5_attention_weight_min": -61.475914001464844,
      "activations/layer6_attention_weight_max": 45.02013397216797,
      "activations/layer6_attention_weight_min": -44.97626495361328,
      "activations/layer7_attention_weight_max": 94.77389526367188,
      "activations/layer7_attention_weight_min": -93.6275405883789,
      "activations/layer8_attention_weight_max": 46.22021484375,
      "activations/layer8_attention_weight_min": -43.754737854003906,
      "activations/layer9_attention_weight_max": 32.443885803222656,
      "activations/layer9_attention_weight_min": -35.93736267089844,
      "epoch": 18.82,
      "learning_rate": 2.894242424242424e-05,
      "loss": 2.7299,
      "step": 323850
    },
    {
      "activations/layer0_attention_weight_max": 16.453208923339844,
      "activations/layer0_attention_weight_min": -13.418322563171387,
      "activations/layer10_attention_weight_max": 32.97903823852539,
      "activations/layer10_attention_weight_min": -35.817684173583984,
      "activations/layer11_attention_weight_max": 32.02086639404297,
      "activations/layer11_attention_weight_min": -34.556575775146484,
      "activations/layer12_attention_weight_max": 25.883150100708008,
      "activations/layer12_attention_weight_min": -27.012331008911133,
      "activations/layer13_attention_weight_max": 36.56540298461914,
      "activations/layer13_attention_weight_min": -34.620872497558594,
      "activations/layer14_attention_weight_max": 39.181373596191406,
      "activations/layer14_attention_weight_min": -38.782310485839844,
      "activations/layer15_attention_weight_max": 36.04806137084961,
      "activations/layer15_attention_weight_min": -34.95513916015625,
      "activations/layer16_attention_weight_max": 28.97458267211914,
      "activations/layer16_attention_weight_min": -27.277320861816406,
      "activations/layer17_attention_weight_max": 29.62018585205078,
      "activations/layer17_attention_weight_min": -27.656230926513672,
      "activations/layer18_attention_weight_max": 30.971553802490234,
      "activations/layer18_attention_weight_min": -28.258596420288086,
      "activations/layer19_attention_weight_max": 33.373165130615234,
      "activations/layer19_attention_weight_min": -30.35833168029785,
      "activations/layer1_attention_weight_max": 16.176794052124023,
      "activations/layer1_attention_weight_min": -15.053930282592773,
      "activations/layer20_attention_weight_max": 31.353591918945312,
      "activations/layer20_attention_weight_min": -25.6840763092041,
      "activations/layer21_attention_weight_max": 30.541379928588867,
      "activations/layer21_attention_weight_min": -25.952880859375,
      "activations/layer22_attention_weight_max": 41.32707595825195,
      "activations/layer22_attention_weight_min": -30.960020065307617,
      "activations/layer23_attention_weight_max": 33.527748107910156,
      "activations/layer23_attention_weight_min": -26.62521743774414,
      "activations/layer2_attention_weight_max": 30.305191040039062,
      "activations/layer2_attention_weight_min": -30.868125915527344,
      "activations/layer3_attention_weight_max": 91.86190795898438,
      "activations/layer3_attention_weight_min": -98.68048858642578,
      "activations/layer4_attention_weight_max": 100.23106384277344,
      "activations/layer4_attention_weight_min": -106.36394500732422,
      "activations/layer5_attention_weight_max": 49.15464782714844,
      "activations/layer5_attention_weight_min": -61.047847747802734,
      "activations/layer6_attention_weight_max": 41.368980407714844,
      "activations/layer6_attention_weight_min": -45.48951721191406,
      "activations/layer7_attention_weight_max": 87.89881134033203,
      "activations/layer7_attention_weight_min": -97.26316833496094,
      "activations/layer8_attention_weight_max": 40.43206024169922,
      "activations/layer8_attention_weight_min": -43.22267150878906,
      "activations/layer9_attention_weight_max": 30.705331802368164,
      "activations/layer9_attention_weight_min": -33.326080322265625,
      "epoch": 18.82,
      "learning_rate": 2.8923484848484844e-05,
      "loss": 2.72,
      "step": 323900
    },
    {
      "activations/layer0_attention_weight_max": 15.88716983795166,
      "activations/layer0_attention_weight_min": -13.461820602416992,
      "activations/layer10_attention_weight_max": 35.98076629638672,
      "activations/layer10_attention_weight_min": -34.18761444091797,
      "activations/layer11_attention_weight_max": 34.70491027832031,
      "activations/layer11_attention_weight_min": -34.50669860839844,
      "activations/layer12_attention_weight_max": 24.662315368652344,
      "activations/layer12_attention_weight_min": -25.36907386779785,
      "activations/layer13_attention_weight_max": 35.932010650634766,
      "activations/layer13_attention_weight_min": -34.030006408691406,
      "activations/layer14_attention_weight_max": 38.89235305786133,
      "activations/layer14_attention_weight_min": -36.60127258300781,
      "activations/layer15_attention_weight_max": 38.663211822509766,
      "activations/layer15_attention_weight_min": -36.39858627319336,
      "activations/layer16_attention_weight_max": 26.26380157470703,
      "activations/layer16_attention_weight_min": -29.24307632446289,
      "activations/layer17_attention_weight_max": 27.389238357543945,
      "activations/layer17_attention_weight_min": -26.386402130126953,
      "activations/layer18_attention_weight_max": 28.177181243896484,
      "activations/layer18_attention_weight_min": -27.585369110107422,
      "activations/layer19_attention_weight_max": 29.939346313476562,
      "activations/layer19_attention_weight_min": -32.24823760986328,
      "activations/layer1_attention_weight_max": 15.669782638549805,
      "activations/layer1_attention_weight_min": -15.536370277404785,
      "activations/layer20_attention_weight_max": 25.725902557373047,
      "activations/layer20_attention_weight_min": -26.242774963378906,
      "activations/layer21_attention_weight_max": 27.25731086730957,
      "activations/layer21_attention_weight_min": -27.16567611694336,
      "activations/layer22_attention_weight_max": 37.41862106323242,
      "activations/layer22_attention_weight_min": -32.03445053100586,
      "activations/layer23_attention_weight_max": 28.93161392211914,
      "activations/layer23_attention_weight_min": -28.02685546875,
      "activations/layer2_attention_weight_max": 30.234630584716797,
      "activations/layer2_attention_weight_min": -29.943601608276367,
      "activations/layer3_attention_weight_max": 95.65805053710938,
      "activations/layer3_attention_weight_min": -97.76957702636719,
      "activations/layer4_attention_weight_max": 106.45963287353516,
      "activations/layer4_attention_weight_min": -106.90447998046875,
      "activations/layer5_attention_weight_max": 49.20421600341797,
      "activations/layer5_attention_weight_min": -64.31849670410156,
      "activations/layer6_attention_weight_max": 42.67469787597656,
      "activations/layer6_attention_weight_min": -46.24614715576172,
      "activations/layer7_attention_weight_max": 95.42427825927734,
      "activations/layer7_attention_weight_min": -99.0491714477539,
      "activations/layer8_attention_weight_max": 42.316471099853516,
      "activations/layer8_attention_weight_min": -42.93115997314453,
      "activations/layer9_attention_weight_max": 33.013118743896484,
      "activations/layer9_attention_weight_min": -34.36518478393555,
      "epoch": 18.82,
      "learning_rate": 2.8904545454545453e-05,
      "loss": 2.738,
      "step": 323950
    },
    {
      "activations/layer0_attention_weight_max": 16.012537002563477,
      "activations/layer0_attention_weight_min": -13.55826187133789,
      "activations/layer10_attention_weight_max": 35.84796142578125,
      "activations/layer10_attention_weight_min": -34.58452606201172,
      "activations/layer11_attention_weight_max": 35.69975662231445,
      "activations/layer11_attention_weight_min": -33.86714172363281,
      "activations/layer12_attention_weight_max": 28.199527740478516,
      "activations/layer12_attention_weight_min": -28.22646713256836,
      "activations/layer13_attention_weight_max": 42.70778274536133,
      "activations/layer13_attention_weight_min": -37.33772277832031,
      "activations/layer14_attention_weight_max": 44.50031661987305,
      "activations/layer14_attention_weight_min": -38.02839660644531,
      "activations/layer15_attention_weight_max": 41.8878173828125,
      "activations/layer15_attention_weight_min": -35.76536178588867,
      "activations/layer16_attention_weight_max": 31.363574981689453,
      "activations/layer16_attention_weight_min": -28.20358657836914,
      "activations/layer17_attention_weight_max": 31.46148109436035,
      "activations/layer17_attention_weight_min": -28.88752555847168,
      "activations/layer18_attention_weight_max": 36.91743469238281,
      "activations/layer18_attention_weight_min": -27.626440048217773,
      "activations/layer19_attention_weight_max": 40.30242919921875,
      "activations/layer19_attention_weight_min": -33.205055236816406,
      "activations/layer1_attention_weight_max": 16.38492774963379,
      "activations/layer1_attention_weight_min": -15.07485580444336,
      "activations/layer20_attention_weight_max": 37.06346893310547,
      "activations/layer20_attention_weight_min": -27.770933151245117,
      "activations/layer21_attention_weight_max": 35.30702209472656,
      "activations/layer21_attention_weight_min": -26.327850341796875,
      "activations/layer22_attention_weight_max": 50.21213150024414,
      "activations/layer22_attention_weight_min": -34.45413589477539,
      "activations/layer23_attention_weight_max": 42.833473205566406,
      "activations/layer23_attention_weight_min": -25.513761520385742,
      "activations/layer2_attention_weight_max": 30.471160888671875,
      "activations/layer2_attention_weight_min": -29.317947387695312,
      "activations/layer3_attention_weight_max": 95.43698120117188,
      "activations/layer3_attention_weight_min": -94.25210571289062,
      "activations/layer4_attention_weight_max": 105.12316131591797,
      "activations/layer4_attention_weight_min": -109.4957504272461,
      "activations/layer5_attention_weight_max": 50.34618377685547,
      "activations/layer5_attention_weight_min": -62.735496520996094,
      "activations/layer6_attention_weight_max": 46.19160461425781,
      "activations/layer6_attention_weight_min": -46.90443420410156,
      "activations/layer7_attention_weight_max": 99.58904266357422,
      "activations/layer7_attention_weight_min": -101.679443359375,
      "activations/layer8_attention_weight_max": 44.353275299072266,
      "activations/layer8_attention_weight_min": -45.7242317199707,
      "activations/layer9_attention_weight_max": 33.572086334228516,
      "activations/layer9_attention_weight_min": -35.850894927978516,
      "epoch": 18.83,
      "learning_rate": 2.8885606060606058e-05,
      "loss": 2.7367,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.5573,
      "eval_samples_per_second": 501.795,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.5573,
      "eval_openwebtext_samples_per_second": 501.795,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9916,
      "eval_wikitext_samples_per_second": 228.961,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_lambada_loss": 2.423828125,
      "eval_lambada_ppl": 11.288992376773315,
      "eval_lambada_runtime": 9.5505,
      "eval_lambada_samples_per_second": 509.817,
      "step": 324000
    },
    {
      "activations/layer0_attention_weight_max": 15.857060432434082,
      "activations/layer0_attention_weight_min": -12.356622695922852,
      "activations/layer10_attention_weight_max": 32.59914016723633,
      "activations/layer10_attention_weight_min": -34.99250030517578,
      "activations/layer11_attention_weight_max": 32.405818939208984,
      "activations/layer11_attention_weight_min": -32.71933364868164,
      "activations/layer12_attention_weight_max": 25.000240325927734,
      "activations/layer12_attention_weight_min": -26.87807846069336,
      "activations/layer13_attention_weight_max": 35.570587158203125,
      "activations/layer13_attention_weight_min": -36.621253967285156,
      "activations/layer14_attention_weight_max": 38.99618148803711,
      "activations/layer14_attention_weight_min": -38.274200439453125,
      "activations/layer15_attention_weight_max": 34.34171676635742,
      "activations/layer15_attention_weight_min": -37.17955780029297,
      "activations/layer16_attention_weight_max": 28.103498458862305,
      "activations/layer16_attention_weight_min": -28.943490982055664,
      "activations/layer17_attention_weight_max": 28.13151741027832,
      "activations/layer17_attention_weight_min": -25.15757942199707,
      "activations/layer18_attention_weight_max": 29.91726303100586,
      "activations/layer18_attention_weight_min": -25.77450180053711,
      "activations/layer19_attention_weight_max": 34.33126449584961,
      "activations/layer19_attention_weight_min": -31.4252872467041,
      "activations/layer1_attention_weight_max": 15.403729438781738,
      "activations/layer1_attention_weight_min": -15.406854629516602,
      "activations/layer20_attention_weight_max": 31.18526840209961,
      "activations/layer20_attention_weight_min": -27.083667755126953,
      "activations/layer21_attention_weight_max": 28.827688217163086,
      "activations/layer21_attention_weight_min": -26.07289695739746,
      "activations/layer22_attention_weight_max": 41.546661376953125,
      "activations/layer22_attention_weight_min": -35.16843032836914,
      "activations/layer23_attention_weight_max": 31.006450653076172,
      "activations/layer23_attention_weight_min": -28.42597007751465,
      "activations/layer2_attention_weight_max": 29.630298614501953,
      "activations/layer2_attention_weight_min": -31.02568817138672,
      "activations/layer3_attention_weight_max": 97.60809326171875,
      "activations/layer3_attention_weight_min": -97.95886993408203,
      "activations/layer4_attention_weight_max": 105.31349182128906,
      "activations/layer4_attention_weight_min": -104.25782775878906,
      "activations/layer5_attention_weight_max": 51.536865234375,
      "activations/layer5_attention_weight_min": -62.36811065673828,
      "activations/layer6_attention_weight_max": 44.81476593017578,
      "activations/layer6_attention_weight_min": -46.3335075378418,
      "activations/layer7_attention_weight_max": 94.7327651977539,
      "activations/layer7_attention_weight_min": -95.00322723388672,
      "activations/layer8_attention_weight_max": 43.21357727050781,
      "activations/layer8_attention_weight_min": -43.561241149902344,
      "activations/layer9_attention_weight_max": 31.418970108032227,
      "activations/layer9_attention_weight_min": -34.509090423583984,
      "epoch": 18.83,
      "learning_rate": 2.8866666666666666e-05,
      "loss": 2.7222,
      "step": 324050
    },
    {
      "activations/layer0_attention_weight_max": 16.807100296020508,
      "activations/layer0_attention_weight_min": -13.117050170898438,
      "activations/layer10_attention_weight_max": 48.49554443359375,
      "activations/layer10_attention_weight_min": -42.9819450378418,
      "activations/layer11_attention_weight_max": 47.489173889160156,
      "activations/layer11_attention_weight_min": -41.80176544189453,
      "activations/layer12_attention_weight_max": 35.96802520751953,
      "activations/layer12_attention_weight_min": -31.669479370117188,
      "activations/layer13_attention_weight_max": 55.496070861816406,
      "activations/layer13_attention_weight_min": -41.74200439453125,
      "activations/layer14_attention_weight_max": 58.293819427490234,
      "activations/layer14_attention_weight_min": -45.44337463378906,
      "activations/layer15_attention_weight_max": 64.52582550048828,
      "activations/layer15_attention_weight_min": -42.392906188964844,
      "activations/layer16_attention_weight_max": 42.64696502685547,
      "activations/layer16_attention_weight_min": -30.174837112426758,
      "activations/layer17_attention_weight_max": 35.03571319580078,
      "activations/layer17_attention_weight_min": -27.6970272064209,
      "activations/layer18_attention_weight_max": 32.8302001953125,
      "activations/layer18_attention_weight_min": -28.121807098388672,
      "activations/layer19_attention_weight_max": 36.193359375,
      "activations/layer19_attention_weight_min": -33.26220703125,
      "activations/layer1_attention_weight_max": 15.56139087677002,
      "activations/layer1_attention_weight_min": -15.39659309387207,
      "activations/layer20_attention_weight_max": 30.790637969970703,
      "activations/layer20_attention_weight_min": -27.89944076538086,
      "activations/layer21_attention_weight_max": 28.97223472595215,
      "activations/layer21_attention_weight_min": -27.814315795898438,
      "activations/layer22_attention_weight_max": 38.82109832763672,
      "activations/layer22_attention_weight_min": -34.16997528076172,
      "activations/layer23_attention_weight_max": 30.321002960205078,
      "activations/layer23_attention_weight_min": -27.270618438720703,
      "activations/layer2_attention_weight_max": 33.77981948852539,
      "activations/layer2_attention_weight_min": -33.23741912841797,
      "activations/layer3_attention_weight_max": 94.25394439697266,
      "activations/layer3_attention_weight_min": -99.25944519042969,
      "activations/layer4_attention_weight_max": 102.97975158691406,
      "activations/layer4_attention_weight_min": -110.2237319946289,
      "activations/layer5_attention_weight_max": 48.979488372802734,
      "activations/layer5_attention_weight_min": -64.95309448242188,
      "activations/layer6_attention_weight_max": 47.31930923461914,
      "activations/layer6_attention_weight_min": -48.679195404052734,
      "activations/layer7_attention_weight_max": 121.01388549804688,
      "activations/layer7_attention_weight_min": -106.39742279052734,
      "activations/layer8_attention_weight_max": 54.52689743041992,
      "activations/layer8_attention_weight_min": -55.873374938964844,
      "activations/layer9_attention_weight_max": 41.69084930419922,
      "activations/layer9_attention_weight_min": -41.963958740234375,
      "epoch": 18.83,
      "learning_rate": 2.8847727272727267e-05,
      "loss": 2.7333,
      "step": 324100
    },
    {
      "activations/layer0_attention_weight_max": 16.236225128173828,
      "activations/layer0_attention_weight_min": -12.549671173095703,
      "activations/layer10_attention_weight_max": 33.505916595458984,
      "activations/layer10_attention_weight_min": -34.15974044799805,
      "activations/layer11_attention_weight_max": 34.237545013427734,
      "activations/layer11_attention_weight_min": -35.210289001464844,
      "activations/layer12_attention_weight_max": 27.5026912689209,
      "activations/layer12_attention_weight_min": -27.60086441040039,
      "activations/layer13_attention_weight_max": 39.19297409057617,
      "activations/layer13_attention_weight_min": -34.65116500854492,
      "activations/layer14_attention_weight_max": 41.60637283325195,
      "activations/layer14_attention_weight_min": -36.74113845825195,
      "activations/layer15_attention_weight_max": 38.38376235961914,
      "activations/layer15_attention_weight_min": -36.01403045654297,
      "activations/layer16_attention_weight_max": 29.9050350189209,
      "activations/layer16_attention_weight_min": -26.469745635986328,
      "activations/layer17_attention_weight_max": 30.44046401977539,
      "activations/layer17_attention_weight_min": -26.301042556762695,
      "activations/layer18_attention_weight_max": 30.14238166809082,
      "activations/layer18_attention_weight_min": -26.232345581054688,
      "activations/layer19_attention_weight_max": 32.373008728027344,
      "activations/layer19_attention_weight_min": -32.36175537109375,
      "activations/layer1_attention_weight_max": 15.679025650024414,
      "activations/layer1_attention_weight_min": -13.75367259979248,
      "activations/layer20_attention_weight_max": 28.828044891357422,
      "activations/layer20_attention_weight_min": -26.72599983215332,
      "activations/layer21_attention_weight_max": 28.540578842163086,
      "activations/layer21_attention_weight_min": -25.554471969604492,
      "activations/layer22_attention_weight_max": 38.345420837402344,
      "activations/layer22_attention_weight_min": -33.02765655517578,
      "activations/layer23_attention_weight_max": 30.642040252685547,
      "activations/layer23_attention_weight_min": -26.965412139892578,
      "activations/layer2_attention_weight_max": 32.18675231933594,
      "activations/layer2_attention_weight_min": -30.101661682128906,
      "activations/layer3_attention_weight_max": 94.74784088134766,
      "activations/layer3_attention_weight_min": -95.5315933227539,
      "activations/layer4_attention_weight_max": 102.74590301513672,
      "activations/layer4_attention_weight_min": -100.52948760986328,
      "activations/layer5_attention_weight_max": 48.08588790893555,
      "activations/layer5_attention_weight_min": -58.027252197265625,
      "activations/layer6_attention_weight_max": 42.63184356689453,
      "activations/layer6_attention_weight_min": -43.838802337646484,
      "activations/layer7_attention_weight_max": 99.98217010498047,
      "activations/layer7_attention_weight_min": -92.47655487060547,
      "activations/layer8_attention_weight_max": 40.9522819519043,
      "activations/layer8_attention_weight_min": -41.62367630004883,
      "activations/layer9_attention_weight_max": 31.912137985229492,
      "activations/layer9_attention_weight_min": -32.873714447021484,
      "epoch": 18.83,
      "learning_rate": 2.8828787878787876e-05,
      "loss": 2.7345,
      "step": 324150
    },
    {
      "activations/layer0_attention_weight_max": 14.95720386505127,
      "activations/layer0_attention_weight_min": -13.007756233215332,
      "activations/layer10_attention_weight_max": 36.05649185180664,
      "activations/layer10_attention_weight_min": -36.192283630371094,
      "activations/layer11_attention_weight_max": 37.270103454589844,
      "activations/layer11_attention_weight_min": -35.39623260498047,
      "activations/layer12_attention_weight_max": 27.956539154052734,
      "activations/layer12_attention_weight_min": -27.598398208618164,
      "activations/layer13_attention_weight_max": 42.216590881347656,
      "activations/layer13_attention_weight_min": -37.51649475097656,
      "activations/layer14_attention_weight_max": 44.762001037597656,
      "activations/layer14_attention_weight_min": -43.27421951293945,
      "activations/layer15_attention_weight_max": 42.591800689697266,
      "activations/layer15_attention_weight_min": -37.580135345458984,
      "activations/layer16_attention_weight_max": 29.34733772277832,
      "activations/layer16_attention_weight_min": -29.224239349365234,
      "activations/layer17_attention_weight_max": 31.705078125,
      "activations/layer17_attention_weight_min": -26.495628356933594,
      "activations/layer18_attention_weight_max": 30.256628036499023,
      "activations/layer18_attention_weight_min": -24.3447322845459,
      "activations/layer19_attention_weight_max": 35.40508270263672,
      "activations/layer19_attention_weight_min": -31.60623550415039,
      "activations/layer1_attention_weight_max": 16.901336669921875,
      "activations/layer1_attention_weight_min": -16.15262794494629,
      "activations/layer20_attention_weight_max": 28.983503341674805,
      "activations/layer20_attention_weight_min": -24.29327964782715,
      "activations/layer21_attention_weight_max": 28.2127742767334,
      "activations/layer21_attention_weight_min": -26.645483016967773,
      "activations/layer22_attention_weight_max": 42.733253479003906,
      "activations/layer22_attention_weight_min": -31.65465545654297,
      "activations/layer23_attention_weight_max": 32.7564811706543,
      "activations/layer23_attention_weight_min": -24.681720733642578,
      "activations/layer2_attention_weight_max": 33.078643798828125,
      "activations/layer2_attention_weight_min": -30.121498107910156,
      "activations/layer3_attention_weight_max": 95.23055267333984,
      "activations/layer3_attention_weight_min": -95.38877868652344,
      "activations/layer4_attention_weight_max": 104.6722640991211,
      "activations/layer4_attention_weight_min": -110.7269287109375,
      "activations/layer5_attention_weight_max": 48.9222526550293,
      "activations/layer5_attention_weight_min": -61.63709259033203,
      "activations/layer6_attention_weight_max": 44.295204162597656,
      "activations/layer6_attention_weight_min": -46.25931930541992,
      "activations/layer7_attention_weight_max": 92.30315399169922,
      "activations/layer7_attention_weight_min": -94.48047637939453,
      "activations/layer8_attention_weight_max": 45.12245559692383,
      "activations/layer8_attention_weight_min": -43.50529479980469,
      "activations/layer9_attention_weight_max": 33.314125061035156,
      "activations/layer9_attention_weight_min": -35.68458938598633,
      "epoch": 18.84,
      "learning_rate": 2.8809848484848484e-05,
      "loss": 2.7347,
      "step": 324200
    },
    {
      "activations/layer0_attention_weight_max": 16.241098403930664,
      "activations/layer0_attention_weight_min": -12.34069538116455,
      "activations/layer10_attention_weight_max": 39.52548599243164,
      "activations/layer10_attention_weight_min": -39.20466232299805,
      "activations/layer11_attention_weight_max": 38.670719146728516,
      "activations/layer11_attention_weight_min": -38.840476989746094,
      "activations/layer12_attention_weight_max": 31.09630012512207,
      "activations/layer12_attention_weight_min": -32.42973709106445,
      "activations/layer13_attention_weight_max": 49.0411491394043,
      "activations/layer13_attention_weight_min": -44.06631088256836,
      "activations/layer14_attention_weight_max": 54.04808044433594,
      "activations/layer14_attention_weight_min": -45.701988220214844,
      "activations/layer15_attention_weight_max": 45.54077911376953,
      "activations/layer15_attention_weight_min": -41.04603576660156,
      "activations/layer16_attention_weight_max": 33.18270492553711,
      "activations/layer16_attention_weight_min": -29.409812927246094,
      "activations/layer17_attention_weight_max": 34.4842414855957,
      "activations/layer17_attention_weight_min": -27.29311752319336,
      "activations/layer18_attention_weight_max": 33.73758316040039,
      "activations/layer18_attention_weight_min": -29.787668228149414,
      "activations/layer19_attention_weight_max": 41.32737350463867,
      "activations/layer19_attention_weight_min": -35.3142204284668,
      "activations/layer1_attention_weight_max": 16.521297454833984,
      "activations/layer1_attention_weight_min": -15.684561729431152,
      "activations/layer20_attention_weight_max": 33.90351104736328,
      "activations/layer20_attention_weight_min": -27.18488121032715,
      "activations/layer21_attention_weight_max": 36.228031158447266,
      "activations/layer21_attention_weight_min": -28.157365798950195,
      "activations/layer22_attention_weight_max": 50.22901153564453,
      "activations/layer22_attention_weight_min": -33.78044128417969,
      "activations/layer23_attention_weight_max": 41.33252716064453,
      "activations/layer23_attention_weight_min": -28.711729049682617,
      "activations/layer2_attention_weight_max": 32.56916046142578,
      "activations/layer2_attention_weight_min": -31.320327758789062,
      "activations/layer3_attention_weight_max": 98.94674682617188,
      "activations/layer3_attention_weight_min": -100.47453308105469,
      "activations/layer4_attention_weight_max": 110.04353332519531,
      "activations/layer4_attention_weight_min": -116.2608413696289,
      "activations/layer5_attention_weight_max": 53.621192932128906,
      "activations/layer5_attention_weight_min": -63.048248291015625,
      "activations/layer6_attention_weight_max": 46.968589782714844,
      "activations/layer6_attention_weight_min": -50.10084533691406,
      "activations/layer7_attention_weight_max": 95.67462158203125,
      "activations/layer7_attention_weight_min": -109.55553436279297,
      "activations/layer8_attention_weight_max": 45.3332633972168,
      "activations/layer8_attention_weight_min": -47.56219482421875,
      "activations/layer9_attention_weight_max": 35.85115432739258,
      "activations/layer9_attention_weight_min": -38.04020309448242,
      "epoch": 18.84,
      "learning_rate": 2.879090909090909e-05,
      "loss": 2.7146,
      "step": 324250
    },
    {
      "activations/layer0_attention_weight_max": 16.30405616760254,
      "activations/layer0_attention_weight_min": -12.815443992614746,
      "activations/layer10_attention_weight_max": 31.223594665527344,
      "activations/layer10_attention_weight_min": -33.02098846435547,
      "activations/layer11_attention_weight_max": 31.179746627807617,
      "activations/layer11_attention_weight_min": -34.648704528808594,
      "activations/layer12_attention_weight_max": 25.378210067749023,
      "activations/layer12_attention_weight_min": -27.47821807861328,
      "activations/layer13_attention_weight_max": 35.842491149902344,
      "activations/layer13_attention_weight_min": -37.434932708740234,
      "activations/layer14_attention_weight_max": 41.08637619018555,
      "activations/layer14_attention_weight_min": -39.37390899658203,
      "activations/layer15_attention_weight_max": 35.11287307739258,
      "activations/layer15_attention_weight_min": -36.217105865478516,
      "activations/layer16_attention_weight_max": 29.119333267211914,
      "activations/layer16_attention_weight_min": -31.320775985717773,
      "activations/layer17_attention_weight_max": 26.83316993713379,
      "activations/layer17_attention_weight_min": -31.041114807128906,
      "activations/layer18_attention_weight_max": 29.82619857788086,
      "activations/layer18_attention_weight_min": -27.617168426513672,
      "activations/layer19_attention_weight_max": 31.049497604370117,
      "activations/layer19_attention_weight_min": -33.12214279174805,
      "activations/layer1_attention_weight_max": 15.75481128692627,
      "activations/layer1_attention_weight_min": -14.678471565246582,
      "activations/layer20_attention_weight_max": 25.48250961303711,
      "activations/layer20_attention_weight_min": -26.845312118530273,
      "activations/layer21_attention_weight_max": 27.302526473999023,
      "activations/layer21_attention_weight_min": -26.266796112060547,
      "activations/layer22_attention_weight_max": 39.860008239746094,
      "activations/layer22_attention_weight_min": -32.30122375488281,
      "activations/layer23_attention_weight_max": 31.31447982788086,
      "activations/layer23_attention_weight_min": -26.772884368896484,
      "activations/layer2_attention_weight_max": 31.671890258789062,
      "activations/layer2_attention_weight_min": -31.20452117919922,
      "activations/layer3_attention_weight_max": 97.8259506225586,
      "activations/layer3_attention_weight_min": -98.78580474853516,
      "activations/layer4_attention_weight_max": 102.79113006591797,
      "activations/layer4_attention_weight_min": -107.82246398925781,
      "activations/layer5_attention_weight_max": 50.395851135253906,
      "activations/layer5_attention_weight_min": -60.74286651611328,
      "activations/layer6_attention_weight_max": 42.593544006347656,
      "activations/layer6_attention_weight_min": -44.70830154418945,
      "activations/layer7_attention_weight_max": 89.45578002929688,
      "activations/layer7_attention_weight_min": -94.19351196289062,
      "activations/layer8_attention_weight_max": 40.04969024658203,
      "activations/layer8_attention_weight_min": -43.45008850097656,
      "activations/layer9_attention_weight_max": 30.89975357055664,
      "activations/layer9_attention_weight_min": -32.218318939208984,
      "epoch": 18.84,
      "learning_rate": 2.8771969696969697e-05,
      "loss": 2.7185,
      "step": 324300
    },
    {
      "activations/layer0_attention_weight_max": 16.01754379272461,
      "activations/layer0_attention_weight_min": -12.674783706665039,
      "activations/layer10_attention_weight_max": 44.5162239074707,
      "activations/layer10_attention_weight_min": -42.30023956298828,
      "activations/layer11_attention_weight_max": 45.259944915771484,
      "activations/layer11_attention_weight_min": -44.50898742675781,
      "activations/layer12_attention_weight_max": 28.100053787231445,
      "activations/layer12_attention_weight_min": -29.927715301513672,
      "activations/layer13_attention_weight_max": 41.570892333984375,
      "activations/layer13_attention_weight_min": -42.509456634521484,
      "activations/layer14_attention_weight_max": 47.442806243896484,
      "activations/layer14_attention_weight_min": -47.52981948852539,
      "activations/layer15_attention_weight_max": 43.10051727294922,
      "activations/layer15_attention_weight_min": -45.1571044921875,
      "activations/layer16_attention_weight_max": 30.505887985229492,
      "activations/layer16_attention_weight_min": -31.25522804260254,
      "activations/layer17_attention_weight_max": 33.18075942993164,
      "activations/layer17_attention_weight_min": -27.44629669189453,
      "activations/layer18_attention_weight_max": 34.19388198852539,
      "activations/layer18_attention_weight_min": -28.52850341796875,
      "activations/layer19_attention_weight_max": 35.04279327392578,
      "activations/layer19_attention_weight_min": -32.78404235839844,
      "activations/layer1_attention_weight_max": 16.35578155517578,
      "activations/layer1_attention_weight_min": -16.400920867919922,
      "activations/layer20_attention_weight_max": 31.529556274414062,
      "activations/layer20_attention_weight_min": -24.420438766479492,
      "activations/layer21_attention_weight_max": 33.410030364990234,
      "activations/layer21_attention_weight_min": -25.025222778320312,
      "activations/layer22_attention_weight_max": 43.29339599609375,
      "activations/layer22_attention_weight_min": -31.1624813079834,
      "activations/layer23_attention_weight_max": 31.699642181396484,
      "activations/layer23_attention_weight_min": -25.44131088256836,
      "activations/layer2_attention_weight_max": 35.5546875,
      "activations/layer2_attention_weight_min": -36.9749641418457,
      "activations/layer3_attention_weight_max": 114.26625061035156,
      "activations/layer3_attention_weight_min": -116.19681549072266,
      "activations/layer4_attention_weight_max": 112.56761932373047,
      "activations/layer4_attention_weight_min": -109.45977783203125,
      "activations/layer5_attention_weight_max": 52.58869552612305,
      "activations/layer5_attention_weight_min": -60.4812126159668,
      "activations/layer6_attention_weight_max": 43.28695297241211,
      "activations/layer6_attention_weight_min": -48.40822982788086,
      "activations/layer7_attention_weight_max": 114.86091613769531,
      "activations/layer7_attention_weight_min": -101.43679809570312,
      "activations/layer8_attention_weight_max": 52.51762390136719,
      "activations/layer8_attention_weight_min": -50.19883728027344,
      "activations/layer9_attention_weight_max": 39.82978439331055,
      "activations/layer9_attention_weight_min": -42.695003509521484,
      "epoch": 18.85,
      "learning_rate": 2.87530303030303e-05,
      "loss": 2.7245,
      "step": 324350
    },
    {
      "activations/layer0_attention_weight_max": 16.349044799804688,
      "activations/layer0_attention_weight_min": -14.098738670349121,
      "activations/layer10_attention_weight_max": 38.772064208984375,
      "activations/layer10_attention_weight_min": -37.528968811035156,
      "activations/layer11_attention_weight_max": 40.77534866333008,
      "activations/layer11_attention_weight_min": -38.19209671020508,
      "activations/layer12_attention_weight_max": 28.019533157348633,
      "activations/layer12_attention_weight_min": -27.89678192138672,
      "activations/layer13_attention_weight_max": 41.1401481628418,
      "activations/layer13_attention_weight_min": -36.8967170715332,
      "activations/layer14_attention_weight_max": 45.16441345214844,
      "activations/layer14_attention_weight_min": -40.39442443847656,
      "activations/layer15_attention_weight_max": 39.36283493041992,
      "activations/layer15_attention_weight_min": -35.38948059082031,
      "activations/layer16_attention_weight_max": 28.693273544311523,
      "activations/layer16_attention_weight_min": -27.986417770385742,
      "activations/layer17_attention_weight_max": 27.987686157226562,
      "activations/layer17_attention_weight_min": -25.553709030151367,
      "activations/layer18_attention_weight_max": 31.118276596069336,
      "activations/layer18_attention_weight_min": -24.580116271972656,
      "activations/layer19_attention_weight_max": 33.03889465332031,
      "activations/layer19_attention_weight_min": -31.732934951782227,
      "activations/layer1_attention_weight_max": 15.424469947814941,
      "activations/layer1_attention_weight_min": -14.722321510314941,
      "activations/layer20_attention_weight_max": 29.40346908569336,
      "activations/layer20_attention_weight_min": -24.698408126831055,
      "activations/layer21_attention_weight_max": 29.98667335510254,
      "activations/layer21_attention_weight_min": -24.008630752563477,
      "activations/layer22_attention_weight_max": 45.10942840576172,
      "activations/layer22_attention_weight_min": -30.82535743713379,
      "activations/layer23_attention_weight_max": 31.346710205078125,
      "activations/layer23_attention_weight_min": -25.270427703857422,
      "activations/layer2_attention_weight_max": 31.644004821777344,
      "activations/layer2_attention_weight_min": -32.50103759765625,
      "activations/layer3_attention_weight_max": 96.2078857421875,
      "activations/layer3_attention_weight_min": -102.69223022460938,
      "activations/layer4_attention_weight_max": 105.35450744628906,
      "activations/layer4_attention_weight_min": -109.56478118896484,
      "activations/layer5_attention_weight_max": 51.49694061279297,
      "activations/layer5_attention_weight_min": -63.91618347167969,
      "activations/layer6_attention_weight_max": 47.018184661865234,
      "activations/layer6_attention_weight_min": -46.51296615600586,
      "activations/layer7_attention_weight_max": 96.31566619873047,
      "activations/layer7_attention_weight_min": -102.50697326660156,
      "activations/layer8_attention_weight_max": 45.86968231201172,
      "activations/layer8_attention_weight_min": -48.147830963134766,
      "activations/layer9_attention_weight_max": 36.7546272277832,
      "activations/layer9_attention_weight_min": -36.50519943237305,
      "epoch": 18.85,
      "learning_rate": 2.8734090909090907e-05,
      "loss": 2.7292,
      "step": 324400
    },
    {
      "activations/layer0_attention_weight_max": 15.870353698730469,
      "activations/layer0_attention_weight_min": -11.839831352233887,
      "activations/layer10_attention_weight_max": 37.343841552734375,
      "activations/layer10_attention_weight_min": -38.33810043334961,
      "activations/layer11_attention_weight_max": 37.787635803222656,
      "activations/layer11_attention_weight_min": -37.08279037475586,
      "activations/layer12_attention_weight_max": 26.66655158996582,
      "activations/layer12_attention_weight_min": -28.208999633789062,
      "activations/layer13_attention_weight_max": 37.97225570678711,
      "activations/layer13_attention_weight_min": -37.79682159423828,
      "activations/layer14_attention_weight_max": 42.19112014770508,
      "activations/layer14_attention_weight_min": -41.588584899902344,
      "activations/layer15_attention_weight_max": 39.51420593261719,
      "activations/layer15_attention_weight_min": -40.16676712036133,
      "activations/layer16_attention_weight_max": 30.053007125854492,
      "activations/layer16_attention_weight_min": -28.87598419189453,
      "activations/layer17_attention_weight_max": 29.67630386352539,
      "activations/layer17_attention_weight_min": -29.060096740722656,
      "activations/layer18_attention_weight_max": 33.37120819091797,
      "activations/layer18_attention_weight_min": -26.999778747558594,
      "activations/layer19_attention_weight_max": 34.34334182739258,
      "activations/layer19_attention_weight_min": -30.90299415588379,
      "activations/layer1_attention_weight_max": 15.946553230285645,
      "activations/layer1_attention_weight_min": -15.07621955871582,
      "activations/layer20_attention_weight_max": 27.44013023376465,
      "activations/layer20_attention_weight_min": -25.218860626220703,
      "activations/layer21_attention_weight_max": 28.469072341918945,
      "activations/layer21_attention_weight_min": -24.630285263061523,
      "activations/layer22_attention_weight_max": 43.30928421020508,
      "activations/layer22_attention_weight_min": -31.60260009765625,
      "activations/layer23_attention_weight_max": 34.9812126159668,
      "activations/layer23_attention_weight_min": -25.962200164794922,
      "activations/layer2_attention_weight_max": 31.82248306274414,
      "activations/layer2_attention_weight_min": -32.482078552246094,
      "activations/layer3_attention_weight_max": 99.67447662353516,
      "activations/layer3_attention_weight_min": -103.83251190185547,
      "activations/layer4_attention_weight_max": 109.56522369384766,
      "activations/layer4_attention_weight_min": -107.29723358154297,
      "activations/layer5_attention_weight_max": 51.85528564453125,
      "activations/layer5_attention_weight_min": -63.76116180419922,
      "activations/layer6_attention_weight_max": 46.006263732910156,
      "activations/layer6_attention_weight_min": -47.02075958251953,
      "activations/layer7_attention_weight_max": 100.82678985595703,
      "activations/layer7_attention_weight_min": -105.10965728759766,
      "activations/layer8_attention_weight_max": 47.899078369140625,
      "activations/layer8_attention_weight_min": -49.51215362548828,
      "activations/layer9_attention_weight_max": 35.315303802490234,
      "activations/layer9_attention_weight_min": -37.6316032409668,
      "epoch": 18.85,
      "learning_rate": 2.871515151515151e-05,
      "loss": 2.7245,
      "step": 324450
    },
    {
      "activations/layer0_attention_weight_max": 15.648451805114746,
      "activations/layer0_attention_weight_min": -12.786418914794922,
      "activations/layer10_attention_weight_max": 35.55442810058594,
      "activations/layer10_attention_weight_min": -34.0479850769043,
      "activations/layer11_attention_weight_max": 34.463985443115234,
      "activations/layer11_attention_weight_min": -34.62884521484375,
      "activations/layer12_attention_weight_max": 27.862384796142578,
      "activations/layer12_attention_weight_min": -27.872074127197266,
      "activations/layer13_attention_weight_max": 40.20491409301758,
      "activations/layer13_attention_weight_min": -35.747337341308594,
      "activations/layer14_attention_weight_max": 41.72978210449219,
      "activations/layer14_attention_weight_min": -38.278221130371094,
      "activations/layer15_attention_weight_max": 39.52690124511719,
      "activations/layer15_attention_weight_min": -37.91513442993164,
      "activations/layer16_attention_weight_max": 29.031038284301758,
      "activations/layer16_attention_weight_min": -29.404191970825195,
      "activations/layer17_attention_weight_max": 28.628456115722656,
      "activations/layer17_attention_weight_min": -26.6571102142334,
      "activations/layer18_attention_weight_max": 31.063114166259766,
      "activations/layer18_attention_weight_min": -26.08808135986328,
      "activations/layer19_attention_weight_max": 33.321022033691406,
      "activations/layer19_attention_weight_min": -30.57377052307129,
      "activations/layer1_attention_weight_max": 16.30559730529785,
      "activations/layer1_attention_weight_min": -14.87933349609375,
      "activations/layer20_attention_weight_max": 29.48678207397461,
      "activations/layer20_attention_weight_min": -24.745798110961914,
      "activations/layer21_attention_weight_max": 27.55425262451172,
      "activations/layer21_attention_weight_min": -23.415908813476562,
      "activations/layer22_attention_weight_max": 41.14310073852539,
      "activations/layer22_attention_weight_min": -30.58181381225586,
      "activations/layer23_attention_weight_max": 29.204509735107422,
      "activations/layer23_attention_weight_min": -25.341411590576172,
      "activations/layer2_attention_weight_max": 31.278274536132812,
      "activations/layer2_attention_weight_min": -31.152816772460938,
      "activations/layer3_attention_weight_max": 97.23214721679688,
      "activations/layer3_attention_weight_min": -97.53097534179688,
      "activations/layer4_attention_weight_max": 105.83131408691406,
      "activations/layer4_attention_weight_min": -106.00919342041016,
      "activations/layer5_attention_weight_max": 49.791805267333984,
      "activations/layer5_attention_weight_min": -61.645870208740234,
      "activations/layer6_attention_weight_max": 43.36445999145508,
      "activations/layer6_attention_weight_min": -45.55133819580078,
      "activations/layer7_attention_weight_max": 91.37503051757812,
      "activations/layer7_attention_weight_min": -97.31244659423828,
      "activations/layer8_attention_weight_max": 41.783329010009766,
      "activations/layer8_attention_weight_min": -43.21887969970703,
      "activations/layer9_attention_weight_max": 32.13776397705078,
      "activations/layer9_attention_weight_min": -33.6178092956543,
      "epoch": 18.86,
      "learning_rate": 2.869621212121212e-05,
      "loss": 2.7175,
      "step": 324500
    },
    {
      "activations/layer0_attention_weight_max": 15.380852699279785,
      "activations/layer0_attention_weight_min": -12.911688804626465,
      "activations/layer10_attention_weight_max": 40.26763916015625,
      "activations/layer10_attention_weight_min": -40.51105499267578,
      "activations/layer11_attention_weight_max": 39.795162200927734,
      "activations/layer11_attention_weight_min": -40.68449401855469,
      "activations/layer12_attention_weight_max": 32.62843704223633,
      "activations/layer12_attention_weight_min": -30.529016494750977,
      "activations/layer13_attention_weight_max": 48.639549255371094,
      "activations/layer13_attention_weight_min": -40.57172775268555,
      "activations/layer14_attention_weight_max": 49.54469680786133,
      "activations/layer14_attention_weight_min": -41.40712356567383,
      "activations/layer15_attention_weight_max": 50.562644958496094,
      "activations/layer15_attention_weight_min": -40.781219482421875,
      "activations/layer16_attention_weight_max": 34.84174728393555,
      "activations/layer16_attention_weight_min": -29.56887435913086,
      "activations/layer17_attention_weight_max": 33.37726593017578,
      "activations/layer17_attention_weight_min": -31.865447998046875,
      "activations/layer18_attention_weight_max": 34.29494857788086,
      "activations/layer18_attention_weight_min": -30.08697509765625,
      "activations/layer19_attention_weight_max": 36.93254089355469,
      "activations/layer19_attention_weight_min": -32.382076263427734,
      "activations/layer1_attention_weight_max": 16.020177841186523,
      "activations/layer1_attention_weight_min": -14.445108413696289,
      "activations/layer20_attention_weight_max": 27.869260787963867,
      "activations/layer20_attention_weight_min": -26.474802017211914,
      "activations/layer21_attention_weight_max": 28.584518432617188,
      "activations/layer21_attention_weight_min": -27.302230834960938,
      "activations/layer22_attention_weight_max": 41.743717193603516,
      "activations/layer22_attention_weight_min": -36.99506378173828,
      "activations/layer23_attention_weight_max": 31.73175048828125,
      "activations/layer23_attention_weight_min": -27.53573989868164,
      "activations/layer2_attention_weight_max": 33.270389556884766,
      "activations/layer2_attention_weight_min": -32.368194580078125,
      "activations/layer3_attention_weight_max": 103.9027328491211,
      "activations/layer3_attention_weight_min": -107.75675964355469,
      "activations/layer4_attention_weight_max": 114.7872543334961,
      "activations/layer4_attention_weight_min": -112.9360122680664,
      "activations/layer5_attention_weight_max": 50.97346115112305,
      "activations/layer5_attention_weight_min": -60.932106018066406,
      "activations/layer6_attention_weight_max": 48.40843200683594,
      "activations/layer6_attention_weight_min": -50.64167785644531,
      "activations/layer7_attention_weight_max": 102.2395248413086,
      "activations/layer7_attention_weight_min": -108.01165008544922,
      "activations/layer8_attention_weight_max": 46.88693618774414,
      "activations/layer8_attention_weight_min": -50.69711685180664,
      "activations/layer9_attention_weight_max": 39.652889251708984,
      "activations/layer9_attention_weight_min": -39.35417938232422,
      "epoch": 18.86,
      "learning_rate": 2.8677272727272725e-05,
      "loss": 2.7322,
      "step": 324550
    },
    {
      "activations/layer0_attention_weight_max": 16.292184829711914,
      "activations/layer0_attention_weight_min": -13.0216646194458,
      "activations/layer10_attention_weight_max": 31.331100463867188,
      "activations/layer10_attention_weight_min": -32.94873046875,
      "activations/layer11_attention_weight_max": 31.885183334350586,
      "activations/layer11_attention_weight_min": -32.8775634765625,
      "activations/layer12_attention_weight_max": 25.311756134033203,
      "activations/layer12_attention_weight_min": -25.87411117553711,
      "activations/layer13_attention_weight_max": 37.661476135253906,
      "activations/layer13_attention_weight_min": -35.451900482177734,
      "activations/layer14_attention_weight_max": 38.2690544128418,
      "activations/layer14_attention_weight_min": -37.03363800048828,
      "activations/layer15_attention_weight_max": 37.635223388671875,
      "activations/layer15_attention_weight_min": -35.001930236816406,
      "activations/layer16_attention_weight_max": 26.42344856262207,
      "activations/layer16_attention_weight_min": -27.639118194580078,
      "activations/layer17_attention_weight_max": 29.174177169799805,
      "activations/layer17_attention_weight_min": -25.387845993041992,
      "activations/layer18_attention_weight_max": 29.536930084228516,
      "activations/layer18_attention_weight_min": -26.93991470336914,
      "activations/layer19_attention_weight_max": 31.935977935791016,
      "activations/layer19_attention_weight_min": -32.999908447265625,
      "activations/layer1_attention_weight_max": 16.155813217163086,
      "activations/layer1_attention_weight_min": -13.649924278259277,
      "activations/layer20_attention_weight_max": 26.632909774780273,
      "activations/layer20_attention_weight_min": -25.275962829589844,
      "activations/layer21_attention_weight_max": 27.054447174072266,
      "activations/layer21_attention_weight_min": -25.658376693725586,
      "activations/layer22_attention_weight_max": 37.9553108215332,
      "activations/layer22_attention_weight_min": -32.6309814453125,
      "activations/layer23_attention_weight_max": 31.29328155517578,
      "activations/layer23_attention_weight_min": -27.064495086669922,
      "activations/layer2_attention_weight_max": 30.333885192871094,
      "activations/layer2_attention_weight_min": -29.99663543701172,
      "activations/layer3_attention_weight_max": 93.63108825683594,
      "activations/layer3_attention_weight_min": -94.50035858154297,
      "activations/layer4_attention_weight_max": 100.13543701171875,
      "activations/layer4_attention_weight_min": -101.67741394042969,
      "activations/layer5_attention_weight_max": 49.77480697631836,
      "activations/layer5_attention_weight_min": -59.81163787841797,
      "activations/layer6_attention_weight_max": 41.5426025390625,
      "activations/layer6_attention_weight_min": -45.791446685791016,
      "activations/layer7_attention_weight_max": 86.29798889160156,
      "activations/layer7_attention_weight_min": -96.19729614257812,
      "activations/layer8_attention_weight_max": 39.17107391357422,
      "activations/layer8_attention_weight_min": -43.19829177856445,
      "activations/layer9_attention_weight_max": 30.61679458618164,
      "activations/layer9_attention_weight_min": -32.84211349487305,
      "epoch": 18.86,
      "learning_rate": 2.8658333333333333e-05,
      "loss": 2.7287,
      "step": 324600
    },
    {
      "activations/layer0_attention_weight_max": 16.157167434692383,
      "activations/layer0_attention_weight_min": -11.999659538269043,
      "activations/layer10_attention_weight_max": 36.1335563659668,
      "activations/layer10_attention_weight_min": -34.60844039916992,
      "activations/layer11_attention_weight_max": 35.89785385131836,
      "activations/layer11_attention_weight_min": -35.44681930541992,
      "activations/layer12_attention_weight_max": 27.804819107055664,
      "activations/layer12_attention_weight_min": -30.30561637878418,
      "activations/layer13_attention_weight_max": 40.51019287109375,
      "activations/layer13_attention_weight_min": -36.2822380065918,
      "activations/layer14_attention_weight_max": 42.325706481933594,
      "activations/layer14_attention_weight_min": -38.89707946777344,
      "activations/layer15_attention_weight_max": 40.86434555053711,
      "activations/layer15_attention_weight_min": -37.58313751220703,
      "activations/layer16_attention_weight_max": 30.748926162719727,
      "activations/layer16_attention_weight_min": -27.591068267822266,
      "activations/layer17_attention_weight_max": 28.80892562866211,
      "activations/layer17_attention_weight_min": -27.951780319213867,
      "activations/layer18_attention_weight_max": 32.383026123046875,
      "activations/layer18_attention_weight_min": -26.25187873840332,
      "activations/layer19_attention_weight_max": 31.754547119140625,
      "activations/layer19_attention_weight_min": -31.92329978942871,
      "activations/layer1_attention_weight_max": 16.163835525512695,
      "activations/layer1_attention_weight_min": -14.351055145263672,
      "activations/layer20_attention_weight_max": 27.815895080566406,
      "activations/layer20_attention_weight_min": -27.268287658691406,
      "activations/layer21_attention_weight_max": 28.719444274902344,
      "activations/layer21_attention_weight_min": -28.01525115966797,
      "activations/layer22_attention_weight_max": 42.50712966918945,
      "activations/layer22_attention_weight_min": -32.799888610839844,
      "activations/layer23_attention_weight_max": 33.30413818359375,
      "activations/layer23_attention_weight_min": -27.196273803710938,
      "activations/layer2_attention_weight_max": 30.904293060302734,
      "activations/layer2_attention_weight_min": -31.035430908203125,
      "activations/layer3_attention_weight_max": 93.81793975830078,
      "activations/layer3_attention_weight_min": -96.96609497070312,
      "activations/layer4_attention_weight_max": 103.47554779052734,
      "activations/layer4_attention_weight_min": -108.87477111816406,
      "activations/layer5_attention_weight_max": 48.59637451171875,
      "activations/layer5_attention_weight_min": -62.02421188354492,
      "activations/layer6_attention_weight_max": 43.58906936645508,
      "activations/layer6_attention_weight_min": -47.9011344909668,
      "activations/layer7_attention_weight_max": 92.21898651123047,
      "activations/layer7_attention_weight_min": -94.61128997802734,
      "activations/layer8_attention_weight_max": 42.46565246582031,
      "activations/layer8_attention_weight_min": -45.314697265625,
      "activations/layer9_attention_weight_max": 34.247886657714844,
      "activations/layer9_attention_weight_min": -35.700782775878906,
      "epoch": 18.86,
      "learning_rate": 2.8639772727272724e-05,
      "loss": 2.7214,
      "step": 324650
    },
    {
      "activations/layer0_attention_weight_max": 15.731952667236328,
      "activations/layer0_attention_weight_min": -12.411474227905273,
      "activations/layer10_attention_weight_max": 43.881378173828125,
      "activations/layer10_attention_weight_min": -38.139434814453125,
      "activations/layer11_attention_weight_max": 44.33258056640625,
      "activations/layer11_attention_weight_min": -42.441978454589844,
      "activations/layer12_attention_weight_max": 28.3848934173584,
      "activations/layer12_attention_weight_min": -28.100723266601562,
      "activations/layer13_attention_weight_max": 38.5341682434082,
      "activations/layer13_attention_weight_min": -36.62870788574219,
      "activations/layer14_attention_weight_max": 45.08743667602539,
      "activations/layer14_attention_weight_min": -39.67562484741211,
      "activations/layer15_attention_weight_max": 45.50825500488281,
      "activations/layer15_attention_weight_min": -39.8695068359375,
      "activations/layer16_attention_weight_max": 31.906335830688477,
      "activations/layer16_attention_weight_min": -27.43602180480957,
      "activations/layer17_attention_weight_max": 30.768335342407227,
      "activations/layer17_attention_weight_min": -29.275264739990234,
      "activations/layer18_attention_weight_max": 30.664907455444336,
      "activations/layer18_attention_weight_min": -28.80009651184082,
      "activations/layer19_attention_weight_max": 34.571353912353516,
      "activations/layer19_attention_weight_min": -33.226768493652344,
      "activations/layer1_attention_weight_max": 17.71302032470703,
      "activations/layer1_attention_weight_min": -14.110886573791504,
      "activations/layer20_attention_weight_max": 29.958580017089844,
      "activations/layer20_attention_weight_min": -26.950542449951172,
      "activations/layer21_attention_weight_max": 29.8643798828125,
      "activations/layer21_attention_weight_min": -28.835472106933594,
      "activations/layer22_attention_weight_max": 47.159141540527344,
      "activations/layer22_attention_weight_min": -38.97343826293945,
      "activations/layer23_attention_weight_max": 34.195987701416016,
      "activations/layer23_attention_weight_min": -28.47010040283203,
      "activations/layer2_attention_weight_max": 31.718021392822266,
      "activations/layer2_attention_weight_min": -30.611736297607422,
      "activations/layer3_attention_weight_max": 93.91835021972656,
      "activations/layer3_attention_weight_min": -96.94950103759766,
      "activations/layer4_attention_weight_max": 100.04891967773438,
      "activations/layer4_attention_weight_min": -97.84210968017578,
      "activations/layer5_attention_weight_max": 48.17875671386719,
      "activations/layer5_attention_weight_min": -61.37574768066406,
      "activations/layer6_attention_weight_max": 46.40754699707031,
      "activations/layer6_attention_weight_min": -45.233341217041016,
      "activations/layer7_attention_weight_max": 99.54219818115234,
      "activations/layer7_attention_weight_min": -92.98564147949219,
      "activations/layer8_attention_weight_max": 45.41712188720703,
      "activations/layer8_attention_weight_min": -44.61731719970703,
      "activations/layer9_attention_weight_max": 37.080589294433594,
      "activations/layer9_attention_weight_min": -38.46548080444336,
      "epoch": 18.87,
      "learning_rate": 2.8620833333333332e-05,
      "loss": 2.7103,
      "step": 324700
    },
    {
      "activations/layer0_attention_weight_max": 16.18955421447754,
      "activations/layer0_attention_weight_min": -12.18799114227295,
      "activations/layer10_attention_weight_max": 42.286659240722656,
      "activations/layer10_attention_weight_min": -38.699195861816406,
      "activations/layer11_attention_weight_max": 44.51751708984375,
      "activations/layer11_attention_weight_min": -40.80332946777344,
      "activations/layer12_attention_weight_max": 36.0916633605957,
      "activations/layer12_attention_weight_min": -31.03071403503418,
      "activations/layer13_attention_weight_max": 52.52848434448242,
      "activations/layer13_attention_weight_min": -42.200653076171875,
      "activations/layer14_attention_weight_max": 62.84039306640625,
      "activations/layer14_attention_weight_min": -45.99250411987305,
      "activations/layer15_attention_weight_max": 55.00445556640625,
      "activations/layer15_attention_weight_min": -47.95101547241211,
      "activations/layer16_attention_weight_max": 41.1273307800293,
      "activations/layer16_attention_weight_min": -30.351160049438477,
      "activations/layer17_attention_weight_max": 45.46245574951172,
      "activations/layer17_attention_weight_min": -31.021745681762695,
      "activations/layer18_attention_weight_max": 39.03646469116211,
      "activations/layer18_attention_weight_min": -29.82918930053711,
      "activations/layer19_attention_weight_max": 46.885746002197266,
      "activations/layer19_attention_weight_min": -33.8868522644043,
      "activations/layer1_attention_weight_max": 16.32542610168457,
      "activations/layer1_attention_weight_min": -16.746240615844727,
      "activations/layer20_attention_weight_max": 39.289772033691406,
      "activations/layer20_attention_weight_min": -26.8922061920166,
      "activations/layer21_attention_weight_max": 35.153900146484375,
      "activations/layer21_attention_weight_min": -26.931758880615234,
      "activations/layer22_attention_weight_max": 53.21387481689453,
      "activations/layer22_attention_weight_min": -34.01825714111328,
      "activations/layer23_attention_weight_max": 40.0501708984375,
      "activations/layer23_attention_weight_min": -28.158672332763672,
      "activations/layer2_attention_weight_max": 33.75005340576172,
      "activations/layer2_attention_weight_min": -30.782255172729492,
      "activations/layer3_attention_weight_max": 93.84038543701172,
      "activations/layer3_attention_weight_min": -95.5329818725586,
      "activations/layer4_attention_weight_max": 101.23578643798828,
      "activations/layer4_attention_weight_min": -108.2270278930664,
      "activations/layer5_attention_weight_max": 50.958473205566406,
      "activations/layer5_attention_weight_min": -63.9864501953125,
      "activations/layer6_attention_weight_max": 44.63823699951172,
      "activations/layer6_attention_weight_min": -49.4112434387207,
      "activations/layer7_attention_weight_max": 102.41446685791016,
      "activations/layer7_attention_weight_min": -100.39938354492188,
      "activations/layer8_attention_weight_max": 46.692073822021484,
      "activations/layer8_attention_weight_min": -46.12745666503906,
      "activations/layer9_attention_weight_max": 38.37899398803711,
      "activations/layer9_attention_weight_min": -38.96824264526367,
      "epoch": 18.87,
      "learning_rate": 2.8601893939393937e-05,
      "loss": 2.7143,
      "step": 324750
    },
    {
      "activations/layer0_attention_weight_max": 16.19157600402832,
      "activations/layer0_attention_weight_min": -12.92906379699707,
      "activations/layer10_attention_weight_max": 40.6469612121582,
      "activations/layer10_attention_weight_min": -37.804405212402344,
      "activations/layer11_attention_weight_max": 40.018531799316406,
      "activations/layer11_attention_weight_min": -36.95979309082031,
      "activations/layer12_attention_weight_max": 36.82210922241211,
      "activations/layer12_attention_weight_min": -31.41600227355957,
      "activations/layer13_attention_weight_max": 46.45673751831055,
      "activations/layer13_attention_weight_min": -40.04114532470703,
      "activations/layer14_attention_weight_max": 46.52619934082031,
      "activations/layer14_attention_weight_min": -41.405635833740234,
      "activations/layer15_attention_weight_max": 44.83045959472656,
      "activations/layer15_attention_weight_min": -40.861270904541016,
      "activations/layer16_attention_weight_max": 29.75792121887207,
      "activations/layer16_attention_weight_min": -29.918424606323242,
      "activations/layer17_attention_weight_max": 30.42823028564453,
      "activations/layer17_attention_weight_min": -27.250160217285156,
      "activations/layer18_attention_weight_max": 33.4383659362793,
      "activations/layer18_attention_weight_min": -26.9605655670166,
      "activations/layer19_attention_weight_max": 35.3139762878418,
      "activations/layer19_attention_weight_min": -33.52032470703125,
      "activations/layer1_attention_weight_max": 15.595088005065918,
      "activations/layer1_attention_weight_min": -15.040739059448242,
      "activations/layer20_attention_weight_max": 30.208410263061523,
      "activations/layer20_attention_weight_min": -26.555601119995117,
      "activations/layer21_attention_weight_max": 29.90494728088379,
      "activations/layer21_attention_weight_min": -25.624370574951172,
      "activations/layer22_attention_weight_max": 42.272186279296875,
      "activations/layer22_attention_weight_min": -32.379024505615234,
      "activations/layer23_attention_weight_max": 34.96608352661133,
      "activations/layer23_attention_weight_min": -29.20122528076172,
      "activations/layer2_attention_weight_max": 31.733501434326172,
      "activations/layer2_attention_weight_min": -31.061779022216797,
      "activations/layer3_attention_weight_max": 103.53208923339844,
      "activations/layer3_attention_weight_min": -105.20951843261719,
      "activations/layer4_attention_weight_max": 111.14306640625,
      "activations/layer4_attention_weight_min": -110.83658599853516,
      "activations/layer5_attention_weight_max": 50.98940658569336,
      "activations/layer5_attention_weight_min": -61.70085906982422,
      "activations/layer6_attention_weight_max": 46.772865295410156,
      "activations/layer6_attention_weight_min": -47.94849395751953,
      "activations/layer7_attention_weight_max": 98.02938079833984,
      "activations/layer7_attention_weight_min": -101.07624053955078,
      "activations/layer8_attention_weight_max": 46.489349365234375,
      "activations/layer8_attention_weight_min": -46.5933723449707,
      "activations/layer9_attention_weight_max": 36.0429801940918,
      "activations/layer9_attention_weight_min": -38.465484619140625,
      "epoch": 18.87,
      "learning_rate": 2.8582954545454542e-05,
      "loss": 2.722,
      "step": 324800
    },
    {
      "activations/layer0_attention_weight_max": 16.097949981689453,
      "activations/layer0_attention_weight_min": -12.93730640411377,
      "activations/layer10_attention_weight_max": 35.67403030395508,
      "activations/layer10_attention_weight_min": -35.9195442199707,
      "activations/layer11_attention_weight_max": 32.98756408691406,
      "activations/layer11_attention_weight_min": -34.6810302734375,
      "activations/layer12_attention_weight_max": 28.739633560180664,
      "activations/layer12_attention_weight_min": -28.2144832611084,
      "activations/layer13_attention_weight_max": 42.09211730957031,
      "activations/layer13_attention_weight_min": -37.890037536621094,
      "activations/layer14_attention_weight_max": 46.59366989135742,
      "activations/layer14_attention_weight_min": -40.050865173339844,
      "activations/layer15_attention_weight_max": 41.11875915527344,
      "activations/layer15_attention_weight_min": -38.863739013671875,
      "activations/layer16_attention_weight_max": 29.114057540893555,
      "activations/layer16_attention_weight_min": -28.179140090942383,
      "activations/layer17_attention_weight_max": 31.56231689453125,
      "activations/layer17_attention_weight_min": -26.410667419433594,
      "activations/layer18_attention_weight_max": 34.28152084350586,
      "activations/layer18_attention_weight_min": -26.440153121948242,
      "activations/layer19_attention_weight_max": 36.14201354980469,
      "activations/layer19_attention_weight_min": -33.990360260009766,
      "activations/layer1_attention_weight_max": 16.041004180908203,
      "activations/layer1_attention_weight_min": -15.159845352172852,
      "activations/layer20_attention_weight_max": 32.604591369628906,
      "activations/layer20_attention_weight_min": -26.840105056762695,
      "activations/layer21_attention_weight_max": 29.543132781982422,
      "activations/layer21_attention_weight_min": -25.922832489013672,
      "activations/layer22_attention_weight_max": 44.1546516418457,
      "activations/layer22_attention_weight_min": -33.49663543701172,
      "activations/layer23_attention_weight_max": 35.77364730834961,
      "activations/layer23_attention_weight_min": -26.9206600189209,
      "activations/layer2_attention_weight_max": 29.776992797851562,
      "activations/layer2_attention_weight_min": -29.581100463867188,
      "activations/layer3_attention_weight_max": 96.12772369384766,
      "activations/layer3_attention_weight_min": -97.6400146484375,
      "activations/layer4_attention_weight_max": 104.59099578857422,
      "activations/layer4_attention_weight_min": -106.0882568359375,
      "activations/layer5_attention_weight_max": 48.49985885620117,
      "activations/layer5_attention_weight_min": -62.27752685546875,
      "activations/layer6_attention_weight_max": 44.0769157409668,
      "activations/layer6_attention_weight_min": -46.16154861450195,
      "activations/layer7_attention_weight_max": 93.74610900878906,
      "activations/layer7_attention_weight_min": -103.19148254394531,
      "activations/layer8_attention_weight_max": 42.86927032470703,
      "activations/layer8_attention_weight_min": -44.009803771972656,
      "activations/layer9_attention_weight_max": 33.680782318115234,
      "activations/layer9_attention_weight_min": -34.66189193725586,
      "epoch": 18.88,
      "learning_rate": 2.8564015151515147e-05,
      "loss": 2.7241,
      "step": 324850
    },
    {
      "activations/layer0_attention_weight_max": 16.217166900634766,
      "activations/layer0_attention_weight_min": -12.154380798339844,
      "activations/layer10_attention_weight_max": 32.29106521606445,
      "activations/layer10_attention_weight_min": -31.62108039855957,
      "activations/layer11_attention_weight_max": 34.39976119995117,
      "activations/layer11_attention_weight_min": -31.416622161865234,
      "activations/layer12_attention_weight_max": 27.521385192871094,
      "activations/layer12_attention_weight_min": -27.607450485229492,
      "activations/layer13_attention_weight_max": 39.8513298034668,
      "activations/layer13_attention_weight_min": -39.07068634033203,
      "activations/layer14_attention_weight_max": 45.37464141845703,
      "activations/layer14_attention_weight_min": -39.680877685546875,
      "activations/layer15_attention_weight_max": 40.64454650878906,
      "activations/layer15_attention_weight_min": -37.087738037109375,
      "activations/layer16_attention_weight_max": 32.64614486694336,
      "activations/layer16_attention_weight_min": -28.712636947631836,
      "activations/layer17_attention_weight_max": 31.66976547241211,
      "activations/layer17_attention_weight_min": -26.263803482055664,
      "activations/layer18_attention_weight_max": 32.4830436706543,
      "activations/layer18_attention_weight_min": -26.034122467041016,
      "activations/layer19_attention_weight_max": 35.885581970214844,
      "activations/layer19_attention_weight_min": -31.78606605529785,
      "activations/layer1_attention_weight_max": 17.532161712646484,
      "activations/layer1_attention_weight_min": -13.90626335144043,
      "activations/layer20_attention_weight_max": 30.840286254882812,
      "activations/layer20_attention_weight_min": -26.05242347717285,
      "activations/layer21_attention_weight_max": 31.638351440429688,
      "activations/layer21_attention_weight_min": -25.2957706451416,
      "activations/layer22_attention_weight_max": 42.25975799560547,
      "activations/layer22_attention_weight_min": -31.49580192565918,
      "activations/layer23_attention_weight_max": 35.259830474853516,
      "activations/layer23_attention_weight_min": -28.845577239990234,
      "activations/layer2_attention_weight_max": 29.85993766784668,
      "activations/layer2_attention_weight_min": -31.25069808959961,
      "activations/layer3_attention_weight_max": 88.79458618164062,
      "activations/layer3_attention_weight_min": -97.49226379394531,
      "activations/layer4_attention_weight_max": 95.79308319091797,
      "activations/layer4_attention_weight_min": -103.2444076538086,
      "activations/layer5_attention_weight_max": 46.11640167236328,
      "activations/layer5_attention_weight_min": -57.77558898925781,
      "activations/layer6_attention_weight_max": 41.73517990112305,
      "activations/layer6_attention_weight_min": -45.14431381225586,
      "activations/layer7_attention_weight_max": 90.49927520751953,
      "activations/layer7_attention_weight_min": -92.35997009277344,
      "activations/layer8_attention_weight_max": 41.342647552490234,
      "activations/layer8_attention_weight_min": -41.728363037109375,
      "activations/layer9_attention_weight_max": 32.34170150756836,
      "activations/layer9_attention_weight_min": -32.206661224365234,
      "epoch": 18.88,
      "learning_rate": 2.8545075757575755e-05,
      "loss": 2.729,
      "step": 324900
    },
    {
      "activations/layer0_attention_weight_max": 16.173402786254883,
      "activations/layer0_attention_weight_min": -12.998956680297852,
      "activations/layer10_attention_weight_max": 35.96116256713867,
      "activations/layer10_attention_weight_min": -34.2936897277832,
      "activations/layer11_attention_weight_max": 34.741634368896484,
      "activations/layer11_attention_weight_min": -34.446353912353516,
      "activations/layer12_attention_weight_max": 27.478986740112305,
      "activations/layer12_attention_weight_min": -26.607704162597656,
      "activations/layer13_attention_weight_max": 41.65320587158203,
      "activations/layer13_attention_weight_min": -35.4926872253418,
      "activations/layer14_attention_weight_max": 41.9776725769043,
      "activations/layer14_attention_weight_min": -38.424190521240234,
      "activations/layer15_attention_weight_max": 41.85348892211914,
      "activations/layer15_attention_weight_min": -35.934532165527344,
      "activations/layer16_attention_weight_max": 31.260147094726562,
      "activations/layer16_attention_weight_min": -29.92052459716797,
      "activations/layer17_attention_weight_max": 29.52417755126953,
      "activations/layer17_attention_weight_min": -26.313194274902344,
      "activations/layer18_attention_weight_max": 33.72399139404297,
      "activations/layer18_attention_weight_min": -28.43107795715332,
      "activations/layer19_attention_weight_max": 31.95240592956543,
      "activations/layer19_attention_weight_min": -32.993431091308594,
      "activations/layer1_attention_weight_max": 17.26263427734375,
      "activations/layer1_attention_weight_min": -13.638790130615234,
      "activations/layer20_attention_weight_max": 28.379165649414062,
      "activations/layer20_attention_weight_min": -28.66115951538086,
      "activations/layer21_attention_weight_max": 29.5002498626709,
      "activations/layer21_attention_weight_min": -25.842458724975586,
      "activations/layer22_attention_weight_max": 39.262962341308594,
      "activations/layer22_attention_weight_min": -33.13713455200195,
      "activations/layer23_attention_weight_max": 31.397987365722656,
      "activations/layer23_attention_weight_min": -27.604902267456055,
      "activations/layer2_attention_weight_max": 30.18548583984375,
      "activations/layer2_attention_weight_min": -31.045963287353516,
      "activations/layer3_attention_weight_max": 94.69097137451172,
      "activations/layer3_attention_weight_min": -96.0168228149414,
      "activations/layer4_attention_weight_max": 103.89139556884766,
      "activations/layer4_attention_weight_min": -99.11085510253906,
      "activations/layer5_attention_weight_max": 48.0885124206543,
      "activations/layer5_attention_weight_min": -62.93025207519531,
      "activations/layer6_attention_weight_max": 45.349830627441406,
      "activations/layer6_attention_weight_min": -46.651100158691406,
      "activations/layer7_attention_weight_max": 94.26940155029297,
      "activations/layer7_attention_weight_min": -95.5267562866211,
      "activations/layer8_attention_weight_max": 42.855316162109375,
      "activations/layer8_attention_weight_min": -42.44362258911133,
      "activations/layer9_attention_weight_max": 34.33568572998047,
      "activations/layer9_attention_weight_min": -34.29352951049805,
      "epoch": 18.88,
      "learning_rate": 2.852613636363636e-05,
      "loss": 2.7251,
      "step": 324950
    },
    {
      "activations/layer0_attention_weight_max": 15.349414825439453,
      "activations/layer0_attention_weight_min": -12.08896541595459,
      "activations/layer10_attention_weight_max": 34.84230422973633,
      "activations/layer10_attention_weight_min": -36.47847366333008,
      "activations/layer11_attention_weight_max": 34.43972396850586,
      "activations/layer11_attention_weight_min": -36.453758239746094,
      "activations/layer12_attention_weight_max": 30.0258731842041,
      "activations/layer12_attention_weight_min": -28.527326583862305,
      "activations/layer13_attention_weight_max": 44.62853240966797,
      "activations/layer13_attention_weight_min": -40.13371658325195,
      "activations/layer14_attention_weight_max": 52.31727600097656,
      "activations/layer14_attention_weight_min": -44.58052062988281,
      "activations/layer15_attention_weight_max": 45.419708251953125,
      "activations/layer15_attention_weight_min": -39.762699127197266,
      "activations/layer16_attention_weight_max": 34.754478454589844,
      "activations/layer16_attention_weight_min": -28.92486000061035,
      "activations/layer17_attention_weight_max": 31.276044845581055,
      "activations/layer17_attention_weight_min": -26.931987762451172,
      "activations/layer18_attention_weight_max": 35.22602462768555,
      "activations/layer18_attention_weight_min": -26.73163414001465,
      "activations/layer19_attention_weight_max": 37.2020378112793,
      "activations/layer19_attention_weight_min": -32.980491638183594,
      "activations/layer1_attention_weight_max": 16.708032608032227,
      "activations/layer1_attention_weight_min": -14.909027099609375,
      "activations/layer20_attention_weight_max": 31.16949462890625,
      "activations/layer20_attention_weight_min": -29.158153533935547,
      "activations/layer21_attention_weight_max": 30.25478172302246,
      "activations/layer21_attention_weight_min": -29.0118408203125,
      "activations/layer22_attention_weight_max": 47.706565856933594,
      "activations/layer22_attention_weight_min": -33.009300231933594,
      "activations/layer23_attention_weight_max": 35.67503356933594,
      "activations/layer23_attention_weight_min": -29.54967498779297,
      "activations/layer2_attention_weight_max": 33.073692321777344,
      "activations/layer2_attention_weight_min": -32.46571350097656,
      "activations/layer3_attention_weight_max": 103.80198669433594,
      "activations/layer3_attention_weight_min": -103.16155242919922,
      "activations/layer4_attention_weight_max": 108.09870910644531,
      "activations/layer4_attention_weight_min": -105.93155670166016,
      "activations/layer5_attention_weight_max": 50.084224700927734,
      "activations/layer5_attention_weight_min": -60.20368957519531,
      "activations/layer6_attention_weight_max": 45.311241149902344,
      "activations/layer6_attention_weight_min": -46.892765045166016,
      "activations/layer7_attention_weight_max": 90.97005462646484,
      "activations/layer7_attention_weight_min": -100.7530517578125,
      "activations/layer8_attention_weight_max": 42.3135986328125,
      "activations/layer8_attention_weight_min": -45.367549896240234,
      "activations/layer9_attention_weight_max": 31.11653709411621,
      "activations/layer9_attention_weight_min": -34.74502182006836,
      "epoch": 18.88,
      "learning_rate": 2.8507196969696968e-05,
      "loss": 2.7167,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4832,
      "eval_samples_per_second": 506.178,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4832,
      "eval_openwebtext_samples_per_second": 506.178,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9389,
      "eval_wikitext_samples_per_second": 235.185,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_lambada_loss": 2.40234375,
      "eval_lambada_ppl": 11.049042250008942,
      "eval_lambada_runtime": 9.534,
      "eval_lambada_samples_per_second": 510.696,
      "step": 325000
    },
    {
      "activations/layer0_attention_weight_max": 16.968950271606445,
      "activations/layer0_attention_weight_min": -12.578339576721191,
      "activations/layer10_attention_weight_max": 34.08338165283203,
      "activations/layer10_attention_weight_min": -34.52699661254883,
      "activations/layer11_attention_weight_max": 35.704593658447266,
      "activations/layer11_attention_weight_min": -36.0753288269043,
      "activations/layer12_attention_weight_max": 26.47678565979004,
      "activations/layer12_attention_weight_min": -25.12554168701172,
      "activations/layer13_attention_weight_max": 37.05590057373047,
      "activations/layer13_attention_weight_min": -36.164886474609375,
      "activations/layer14_attention_weight_max": 40.73017883300781,
      "activations/layer14_attention_weight_min": -36.30714416503906,
      "activations/layer15_attention_weight_max": 36.919952392578125,
      "activations/layer15_attention_weight_min": -35.611297607421875,
      "activations/layer16_attention_weight_max": 27.23310089111328,
      "activations/layer16_attention_weight_min": -27.964820861816406,
      "activations/layer17_attention_weight_max": 28.38067626953125,
      "activations/layer17_attention_weight_min": -26.16523551940918,
      "activations/layer18_attention_weight_max": 31.752017974853516,
      "activations/layer18_attention_weight_min": -24.934553146362305,
      "activations/layer19_attention_weight_max": 33.8597526550293,
      "activations/layer19_attention_weight_min": -30.619037628173828,
      "activations/layer1_attention_weight_max": 15.841289520263672,
      "activations/layer1_attention_weight_min": -14.474445343017578,
      "activations/layer20_attention_weight_max": 26.186315536499023,
      "activations/layer20_attention_weight_min": -24.64313316345215,
      "activations/layer21_attention_weight_max": 25.81636619567871,
      "activations/layer21_attention_weight_min": -25.210033416748047,
      "activations/layer22_attention_weight_max": 38.55116271972656,
      "activations/layer22_attention_weight_min": -32.8128547668457,
      "activations/layer23_attention_weight_max": 31.425968170166016,
      "activations/layer23_attention_weight_min": -25.904645919799805,
      "activations/layer2_attention_weight_max": 30.46804428100586,
      "activations/layer2_attention_weight_min": -29.52901840209961,
      "activations/layer3_attention_weight_max": 95.896240234375,
      "activations/layer3_attention_weight_min": -96.28545379638672,
      "activations/layer4_attention_weight_max": 105.84459686279297,
      "activations/layer4_attention_weight_min": -106.4261474609375,
      "activations/layer5_attention_weight_max": 50.455078125,
      "activations/layer5_attention_weight_min": -62.97550582885742,
      "activations/layer6_attention_weight_max": 46.91285705566406,
      "activations/layer6_attention_weight_min": -47.050235748291016,
      "activations/layer7_attention_weight_max": 92.97911071777344,
      "activations/layer7_attention_weight_min": -96.09819793701172,
      "activations/layer8_attention_weight_max": 42.714820861816406,
      "activations/layer8_attention_weight_min": -44.635555267333984,
      "activations/layer9_attention_weight_max": 32.44041442871094,
      "activations/layer9_attention_weight_min": -35.47537612915039,
      "epoch": 18.89,
      "learning_rate": 2.848825757575757e-05,
      "loss": 2.7308,
      "step": 325050
    },
    {
      "activations/layer0_attention_weight_max": 17.245765686035156,
      "activations/layer0_attention_weight_min": -12.539276123046875,
      "activations/layer10_attention_weight_max": 40.00181579589844,
      "activations/layer10_attention_weight_min": -40.12276077270508,
      "activations/layer11_attention_weight_max": 39.650794982910156,
      "activations/layer11_attention_weight_min": -39.57080078125,
      "activations/layer12_attention_weight_max": 28.97759246826172,
      "activations/layer12_attention_weight_min": -26.415950775146484,
      "activations/layer13_attention_weight_max": 41.6845588684082,
      "activations/layer13_attention_weight_min": -35.69906234741211,
      "activations/layer14_attention_weight_max": 41.376739501953125,
      "activations/layer14_attention_weight_min": -37.73728561401367,
      "activations/layer15_attention_weight_max": 38.67620849609375,
      "activations/layer15_attention_weight_min": -37.69981002807617,
      "activations/layer16_attention_weight_max": 26.58059310913086,
      "activations/layer16_attention_weight_min": -27.011445999145508,
      "activations/layer17_attention_weight_max": 27.782001495361328,
      "activations/layer17_attention_weight_min": -24.73748207092285,
      "activations/layer18_attention_weight_max": 31.275630950927734,
      "activations/layer18_attention_weight_min": -26.29524803161621,
      "activations/layer19_attention_weight_max": 30.51429557800293,
      "activations/layer19_attention_weight_min": -31.443822860717773,
      "activations/layer1_attention_weight_max": 16.117807388305664,
      "activations/layer1_attention_weight_min": -14.226853370666504,
      "activations/layer20_attention_weight_max": 25.574047088623047,
      "activations/layer20_attention_weight_min": -24.604137420654297,
      "activations/layer21_attention_weight_max": 26.480363845825195,
      "activations/layer21_attention_weight_min": -25.5654354095459,
      "activations/layer22_attention_weight_max": 39.12201690673828,
      "activations/layer22_attention_weight_min": -30.35858726501465,
      "activations/layer23_attention_weight_max": 31.37385368347168,
      "activations/layer23_attention_weight_min": -25.871450424194336,
      "activations/layer2_attention_weight_max": 32.88279342651367,
      "activations/layer2_attention_weight_min": -33.48274230957031,
      "activations/layer3_attention_weight_max": 99.34695434570312,
      "activations/layer3_attention_weight_min": -100.3019790649414,
      "activations/layer4_attention_weight_max": 106.05366516113281,
      "activations/layer4_attention_weight_min": -107.31654357910156,
      "activations/layer5_attention_weight_max": 49.95073699951172,
      "activations/layer5_attention_weight_min": -64.66328430175781,
      "activations/layer6_attention_weight_max": 44.12407684326172,
      "activations/layer6_attention_weight_min": -48.48876190185547,
      "activations/layer7_attention_weight_max": 98.66213989257812,
      "activations/layer7_attention_weight_min": -98.29370880126953,
      "activations/layer8_attention_weight_max": 45.37776184082031,
      "activations/layer8_attention_weight_min": -44.85741424560547,
      "activations/layer9_attention_weight_max": 35.27062225341797,
      "activations/layer9_attention_weight_min": -37.69009780883789,
      "epoch": 18.89,
      "learning_rate": 2.8469318181818178e-05,
      "loss": 2.711,
      "step": 325100
    },
    {
      "activations/layer0_attention_weight_max": 15.74637222290039,
      "activations/layer0_attention_weight_min": -13.019488334655762,
      "activations/layer10_attention_weight_max": 33.820648193359375,
      "activations/layer10_attention_weight_min": -34.94907760620117,
      "activations/layer11_attention_weight_max": 35.83549880981445,
      "activations/layer11_attention_weight_min": -33.89012145996094,
      "activations/layer12_attention_weight_max": 29.4656925201416,
      "activations/layer12_attention_weight_min": -26.00454330444336,
      "activations/layer13_attention_weight_max": 44.78522872924805,
      "activations/layer13_attention_weight_min": -35.8861083984375,
      "activations/layer14_attention_weight_max": 47.22486877441406,
      "activations/layer14_attention_weight_min": -38.44385528564453,
      "activations/layer15_attention_weight_max": 42.119422912597656,
      "activations/layer15_attention_weight_min": -36.946590423583984,
      "activations/layer16_attention_weight_max": 32.76805114746094,
      "activations/layer16_attention_weight_min": -28.582233428955078,
      "activations/layer17_attention_weight_max": 35.69533920288086,
      "activations/layer17_attention_weight_min": -25.632564544677734,
      "activations/layer18_attention_weight_max": 33.676300048828125,
      "activations/layer18_attention_weight_min": -26.91071128845215,
      "activations/layer19_attention_weight_max": 40.14897918701172,
      "activations/layer19_attention_weight_min": -32.63364791870117,
      "activations/layer1_attention_weight_max": 16.547834396362305,
      "activations/layer1_attention_weight_min": -14.844106674194336,
      "activations/layer20_attention_weight_max": 34.34865951538086,
      "activations/layer20_attention_weight_min": -26.358671188354492,
      "activations/layer21_attention_weight_max": 32.1964111328125,
      "activations/layer21_attention_weight_min": -25.557165145874023,
      "activations/layer22_attention_weight_max": 47.78009796142578,
      "activations/layer22_attention_weight_min": -32.12105178833008,
      "activations/layer23_attention_weight_max": 36.43876266479492,
      "activations/layer23_attention_weight_min": -27.29084014892578,
      "activations/layer2_attention_weight_max": 30.95821762084961,
      "activations/layer2_attention_weight_min": -30.717239379882812,
      "activations/layer3_attention_weight_max": 97.19912719726562,
      "activations/layer3_attention_weight_min": -101.14114379882812,
      "activations/layer4_attention_weight_max": 108.04365539550781,
      "activations/layer4_attention_weight_min": -108.27684783935547,
      "activations/layer5_attention_weight_max": 49.81481170654297,
      "activations/layer5_attention_weight_min": -62.202205657958984,
      "activations/layer6_attention_weight_max": 43.75584411621094,
      "activations/layer6_attention_weight_min": -47.76667785644531,
      "activations/layer7_attention_weight_max": 95.15785217285156,
      "activations/layer7_attention_weight_min": -100.75776672363281,
      "activations/layer8_attention_weight_max": 42.79957962036133,
      "activations/layer8_attention_weight_min": -46.011993408203125,
      "activations/layer9_attention_weight_max": 33.886837005615234,
      "activations/layer9_attention_weight_min": -35.73269271850586,
      "epoch": 18.89,
      "learning_rate": 2.8450378787878786e-05,
      "loss": 2.7307,
      "step": 325150
    },
    {
      "activations/layer0_attention_weight_max": 16.281469345092773,
      "activations/layer0_attention_weight_min": -13.245344161987305,
      "activations/layer10_attention_weight_max": 36.559906005859375,
      "activations/layer10_attention_weight_min": -36.07634353637695,
      "activations/layer11_attention_weight_max": 37.85188674926758,
      "activations/layer11_attention_weight_min": -36.53351593017578,
      "activations/layer12_attention_weight_max": 27.130325317382812,
      "activations/layer12_attention_weight_min": -28.112394332885742,
      "activations/layer13_attention_weight_max": 41.00221633911133,
      "activations/layer13_attention_weight_min": -36.11970520019531,
      "activations/layer14_attention_weight_max": 42.09560012817383,
      "activations/layer14_attention_weight_min": -39.22673416137695,
      "activations/layer15_attention_weight_max": 43.41292190551758,
      "activations/layer15_attention_weight_min": -38.98181915283203,
      "activations/layer16_attention_weight_max": 32.442378997802734,
      "activations/layer16_attention_weight_min": -30.95790672302246,
      "activations/layer17_attention_weight_max": 30.693912506103516,
      "activations/layer17_attention_weight_min": -28.243160247802734,
      "activations/layer18_attention_weight_max": 32.53958511352539,
      "activations/layer18_attention_weight_min": -25.870967864990234,
      "activations/layer19_attention_weight_max": 36.457088470458984,
      "activations/layer19_attention_weight_min": -32.57809066772461,
      "activations/layer1_attention_weight_max": 15.180540084838867,
      "activations/layer1_attention_weight_min": -15.513919830322266,
      "activations/layer20_attention_weight_max": 31.445138931274414,
      "activations/layer20_attention_weight_min": -25.353303909301758,
      "activations/layer21_attention_weight_max": 28.246780395507812,
      "activations/layer21_attention_weight_min": -23.469322204589844,
      "activations/layer22_attention_weight_max": 40.86570358276367,
      "activations/layer22_attention_weight_min": -33.73842239379883,
      "activations/layer23_attention_weight_max": 30.535520553588867,
      "activations/layer23_attention_weight_min": -26.794004440307617,
      "activations/layer2_attention_weight_max": 31.555835723876953,
      "activations/layer2_attention_weight_min": -30.85822868347168,
      "activations/layer3_attention_weight_max": 100.10352325439453,
      "activations/layer3_attention_weight_min": -98.46898651123047,
      "activations/layer4_attention_weight_max": 114.7423095703125,
      "activations/layer4_attention_weight_min": -109.99486541748047,
      "activations/layer5_attention_weight_max": 54.858917236328125,
      "activations/layer5_attention_weight_min": -64.16506958007812,
      "activations/layer6_attention_weight_max": 46.063072204589844,
      "activations/layer6_attention_weight_min": -47.7423210144043,
      "activations/layer7_attention_weight_max": 100.1766586303711,
      "activations/layer7_attention_weight_min": -103.07892608642578,
      "activations/layer8_attention_weight_max": 43.63249969482422,
      "activations/layer8_attention_weight_min": -45.586509704589844,
      "activations/layer9_attention_weight_max": 34.56095886230469,
      "activations/layer9_attention_weight_min": -35.73191833496094,
      "epoch": 18.9,
      "learning_rate": 2.843143939393939e-05,
      "loss": 2.7328,
      "step": 325200
    },
    {
      "activations/layer0_attention_weight_max": 16.861207962036133,
      "activations/layer0_attention_weight_min": -13.0903959274292,
      "activations/layer10_attention_weight_max": 34.348243713378906,
      "activations/layer10_attention_weight_min": -34.87861633300781,
      "activations/layer11_attention_weight_max": 35.699886322021484,
      "activations/layer11_attention_weight_min": -34.55078125,
      "activations/layer12_attention_weight_max": 26.070585250854492,
      "activations/layer12_attention_weight_min": -25.897769927978516,
      "activations/layer13_attention_weight_max": 38.284423828125,
      "activations/layer13_attention_weight_min": -34.4385871887207,
      "activations/layer14_attention_weight_max": 43.532920837402344,
      "activations/layer14_attention_weight_min": -36.979515075683594,
      "activations/layer15_attention_weight_max": 39.071632385253906,
      "activations/layer15_attention_weight_min": -36.193511962890625,
      "activations/layer16_attention_weight_max": 28.988056182861328,
      "activations/layer16_attention_weight_min": -28.477264404296875,
      "activations/layer17_attention_weight_max": 29.48723030090332,
      "activations/layer17_attention_weight_min": -25.749326705932617,
      "activations/layer18_attention_weight_max": 34.53523254394531,
      "activations/layer18_attention_weight_min": -27.021873474121094,
      "activations/layer19_attention_weight_max": 32.94840621948242,
      "activations/layer19_attention_weight_min": -32.54079818725586,
      "activations/layer1_attention_weight_max": 16.834402084350586,
      "activations/layer1_attention_weight_min": -15.427854537963867,
      "activations/layer20_attention_weight_max": 29.82233238220215,
      "activations/layer20_attention_weight_min": -26.685558319091797,
      "activations/layer21_attention_weight_max": 27.727476119995117,
      "activations/layer21_attention_weight_min": -25.973228454589844,
      "activations/layer22_attention_weight_max": 37.56572341918945,
      "activations/layer22_attention_weight_min": -33.3568000793457,
      "activations/layer23_attention_weight_max": 29.558380126953125,
      "activations/layer23_attention_weight_min": -26.644607543945312,
      "activations/layer2_attention_weight_max": 31.34166717529297,
      "activations/layer2_attention_weight_min": -32.05613327026367,
      "activations/layer3_attention_weight_max": 95.18807983398438,
      "activations/layer3_attention_weight_min": -99.75590515136719,
      "activations/layer4_attention_weight_max": 105.9256820678711,
      "activations/layer4_attention_weight_min": -106.3419189453125,
      "activations/layer5_attention_weight_max": 49.67347717285156,
      "activations/layer5_attention_weight_min": -62.63784408569336,
      "activations/layer6_attention_weight_max": 43.08530044555664,
      "activations/layer6_attention_weight_min": -45.786808013916016,
      "activations/layer7_attention_weight_max": 91.97815704345703,
      "activations/layer7_attention_weight_min": -94.91842651367188,
      "activations/layer8_attention_weight_max": 42.00018310546875,
      "activations/layer8_attention_weight_min": -42.9348030090332,
      "activations/layer9_attention_weight_max": 31.96189308166504,
      "activations/layer9_attention_weight_min": -34.17359924316406,
      "epoch": 18.9,
      "learning_rate": 2.84125e-05,
      "loss": 2.7161,
      "step": 325250
    },
    {
      "activations/layer0_attention_weight_max": 15.569353103637695,
      "activations/layer0_attention_weight_min": -12.574776649475098,
      "activations/layer10_attention_weight_max": 34.239784240722656,
      "activations/layer10_attention_weight_min": -35.52425003051758,
      "activations/layer11_attention_weight_max": 35.25123977661133,
      "activations/layer11_attention_weight_min": -37.035888671875,
      "activations/layer12_attention_weight_max": 30.062177658081055,
      "activations/layer12_attention_weight_min": -27.580245971679688,
      "activations/layer13_attention_weight_max": 43.02408218383789,
      "activations/layer13_attention_weight_min": -37.8540153503418,
      "activations/layer14_attention_weight_max": 48.744205474853516,
      "activations/layer14_attention_weight_min": -40.51909255981445,
      "activations/layer15_attention_weight_max": 44.97763442993164,
      "activations/layer15_attention_weight_min": -37.56222915649414,
      "activations/layer16_attention_weight_max": 33.03418731689453,
      "activations/layer16_attention_weight_min": -28.006059646606445,
      "activations/layer17_attention_weight_max": 30.395206451416016,
      "activations/layer17_attention_weight_min": -26.113670349121094,
      "activations/layer18_attention_weight_max": 32.41230392456055,
      "activations/layer18_attention_weight_min": -26.161035537719727,
      "activations/layer19_attention_weight_max": 37.2242546081543,
      "activations/layer19_attention_weight_min": -31.936721801757812,
      "activations/layer1_attention_weight_max": 16.34374237060547,
      "activations/layer1_attention_weight_min": -13.687792778015137,
      "activations/layer20_attention_weight_max": 30.658977508544922,
      "activations/layer20_attention_weight_min": -25.62441062927246,
      "activations/layer21_attention_weight_max": 32.628753662109375,
      "activations/layer21_attention_weight_min": -25.669519424438477,
      "activations/layer22_attention_weight_max": 42.92421340942383,
      "activations/layer22_attention_weight_min": -31.880733489990234,
      "activations/layer23_attention_weight_max": 36.884368896484375,
      "activations/layer23_attention_weight_min": -28.148040771484375,
      "activations/layer2_attention_weight_max": 31.670913696289062,
      "activations/layer2_attention_weight_min": -30.247085571289062,
      "activations/layer3_attention_weight_max": 94.53861236572266,
      "activations/layer3_attention_weight_min": -98.06951141357422,
      "activations/layer4_attention_weight_max": 102.83717346191406,
      "activations/layer4_attention_weight_min": -103.600830078125,
      "activations/layer5_attention_weight_max": 48.533138275146484,
      "activations/layer5_attention_weight_min": -61.76173400878906,
      "activations/layer6_attention_weight_max": 42.5561408996582,
      "activations/layer6_attention_weight_min": -48.126625061035156,
      "activations/layer7_attention_weight_max": 93.3824462890625,
      "activations/layer7_attention_weight_min": -94.75492858886719,
      "activations/layer8_attention_weight_max": 43.5435676574707,
      "activations/layer8_attention_weight_min": -43.575111389160156,
      "activations/layer9_attention_weight_max": 35.4339485168457,
      "activations/layer9_attention_weight_min": -35.158119201660156,
      "epoch": 18.9,
      "learning_rate": 2.8393560606060604e-05,
      "loss": 2.7349,
      "step": 325300
    },
    {
      "activations/layer0_attention_weight_max": 16.226186752319336,
      "activations/layer0_attention_weight_min": -12.309459686279297,
      "activations/layer10_attention_weight_max": 34.63119888305664,
      "activations/layer10_attention_weight_min": -34.66092300415039,
      "activations/layer11_attention_weight_max": 36.78668975830078,
      "activations/layer11_attention_weight_min": -33.112972259521484,
      "activations/layer12_attention_weight_max": 28.14675521850586,
      "activations/layer12_attention_weight_min": -27.347593307495117,
      "activations/layer13_attention_weight_max": 39.97477340698242,
      "activations/layer13_attention_weight_min": -36.86122512817383,
      "activations/layer14_attention_weight_max": 44.111534118652344,
      "activations/layer14_attention_weight_min": -38.74692916870117,
      "activations/layer15_attention_weight_max": 41.897422790527344,
      "activations/layer15_attention_weight_min": -36.09473419189453,
      "activations/layer16_attention_weight_max": 31.98040008544922,
      "activations/layer16_attention_weight_min": -28.505388259887695,
      "activations/layer17_attention_weight_max": 29.9951114654541,
      "activations/layer17_attention_weight_min": -26.793506622314453,
      "activations/layer18_attention_weight_max": 34.40864181518555,
      "activations/layer18_attention_weight_min": -28.187580108642578,
      "activations/layer19_attention_weight_max": 33.48577117919922,
      "activations/layer19_attention_weight_min": -33.08282470703125,
      "activations/layer1_attention_weight_max": 16.23175811767578,
      "activations/layer1_attention_weight_min": -16.10440444946289,
      "activations/layer20_attention_weight_max": 31.164201736450195,
      "activations/layer20_attention_weight_min": -25.189558029174805,
      "activations/layer21_attention_weight_max": 29.83781623840332,
      "activations/layer21_attention_weight_min": -24.28841209411621,
      "activations/layer22_attention_weight_max": 48.10218048095703,
      "activations/layer22_attention_weight_min": -31.79176139831543,
      "activations/layer23_attention_weight_max": 34.946292877197266,
      "activations/layer23_attention_weight_min": -27.189315795898438,
      "activations/layer2_attention_weight_max": 30.809154510498047,
      "activations/layer2_attention_weight_min": -30.531890869140625,
      "activations/layer3_attention_weight_max": 101.46508026123047,
      "activations/layer3_attention_weight_min": -101.1529541015625,
      "activations/layer4_attention_weight_max": 107.6559066772461,
      "activations/layer4_attention_weight_min": -108.571044921875,
      "activations/layer5_attention_weight_max": 51.04214096069336,
      "activations/layer5_attention_weight_min": -61.07518768310547,
      "activations/layer6_attention_weight_max": 43.616825103759766,
      "activations/layer6_attention_weight_min": -47.22658920288086,
      "activations/layer7_attention_weight_max": 101.385498046875,
      "activations/layer7_attention_weight_min": -98.63121795654297,
      "activations/layer8_attention_weight_max": 42.6888427734375,
      "activations/layer8_attention_weight_min": -43.8339958190918,
      "activations/layer9_attention_weight_max": 33.28815460205078,
      "activations/layer9_attention_weight_min": -34.491485595703125,
      "epoch": 18.9,
      "learning_rate": 2.8374621212121213e-05,
      "loss": 2.7261,
      "step": 325350
    },
    {
      "activations/layer0_attention_weight_max": 16.051963806152344,
      "activations/layer0_attention_weight_min": -12.866497993469238,
      "activations/layer10_attention_weight_max": 40.5087890625,
      "activations/layer10_attention_weight_min": -38.95780944824219,
      "activations/layer11_attention_weight_max": 40.88453674316406,
      "activations/layer11_attention_weight_min": -42.51680374145508,
      "activations/layer12_attention_weight_max": 32.25926208496094,
      "activations/layer12_attention_weight_min": -30.143884658813477,
      "activations/layer13_attention_weight_max": 47.94407653808594,
      "activations/layer13_attention_weight_min": -39.39862823486328,
      "activations/layer14_attention_weight_max": 48.05792236328125,
      "activations/layer14_attention_weight_min": -41.793766021728516,
      "activations/layer15_attention_weight_max": 47.62202453613281,
      "activations/layer15_attention_weight_min": -41.65932083129883,
      "activations/layer16_attention_weight_max": 33.465904235839844,
      "activations/layer16_attention_weight_min": -29.301279067993164,
      "activations/layer17_attention_weight_max": 31.146833419799805,
      "activations/layer17_attention_weight_min": -27.178354263305664,
      "activations/layer18_attention_weight_max": 32.388763427734375,
      "activations/layer18_attention_weight_min": -26.818574905395508,
      "activations/layer19_attention_weight_max": 33.90449523925781,
      "activations/layer19_attention_weight_min": -34.694637298583984,
      "activations/layer1_attention_weight_max": 17.766210556030273,
      "activations/layer1_attention_weight_min": -14.432207107543945,
      "activations/layer20_attention_weight_max": 30.005809783935547,
      "activations/layer20_attention_weight_min": -27.385435104370117,
      "activations/layer21_attention_weight_max": 30.978336334228516,
      "activations/layer21_attention_weight_min": -25.60744285583496,
      "activations/layer22_attention_weight_max": 44.10953140258789,
      "activations/layer22_attention_weight_min": -32.96662139892578,
      "activations/layer23_attention_weight_max": 32.76264572143555,
      "activations/layer23_attention_weight_min": -27.639623641967773,
      "activations/layer2_attention_weight_max": 32.194068908691406,
      "activations/layer2_attention_weight_min": -31.70486068725586,
      "activations/layer3_attention_weight_max": 100.84207153320312,
      "activations/layer3_attention_weight_min": -103.52090454101562,
      "activations/layer4_attention_weight_max": 103.15034484863281,
      "activations/layer4_attention_weight_min": -111.3102035522461,
      "activations/layer5_attention_weight_max": 50.04017639160156,
      "activations/layer5_attention_weight_min": -62.27754211425781,
      "activations/layer6_attention_weight_max": 45.31736373901367,
      "activations/layer6_attention_weight_min": -48.179969787597656,
      "activations/layer7_attention_weight_max": 97.54222106933594,
      "activations/layer7_attention_weight_min": -100.05980682373047,
      "activations/layer8_attention_weight_max": 45.88928985595703,
      "activations/layer8_attention_weight_min": -46.26002883911133,
      "activations/layer9_attention_weight_max": 35.767581939697266,
      "activations/layer9_attention_weight_min": -38.407066345214844,
      "epoch": 18.91,
      "learning_rate": 2.8355681818181814e-05,
      "loss": 2.731,
      "step": 325400
    },
    {
      "activations/layer0_attention_weight_max": 16.950565338134766,
      "activations/layer0_attention_weight_min": -12.704153060913086,
      "activations/layer10_attention_weight_max": 31.034757614135742,
      "activations/layer10_attention_weight_min": -32.93251037597656,
      "activations/layer11_attention_weight_max": 31.97325897216797,
      "activations/layer11_attention_weight_min": -33.768470764160156,
      "activations/layer12_attention_weight_max": 26.69796371459961,
      "activations/layer12_attention_weight_min": -26.64934730529785,
      "activations/layer13_attention_weight_max": 38.80809020996094,
      "activations/layer13_attention_weight_min": -35.21942138671875,
      "activations/layer14_attention_weight_max": 41.00187683105469,
      "activations/layer14_attention_weight_min": -40.16317367553711,
      "activations/layer15_attention_weight_max": 40.11876678466797,
      "activations/layer15_attention_weight_min": -38.10054397583008,
      "activations/layer16_attention_weight_max": 29.188066482543945,
      "activations/layer16_attention_weight_min": -28.033550262451172,
      "activations/layer17_attention_weight_max": 29.04185676574707,
      "activations/layer17_attention_weight_min": -26.5200138092041,
      "activations/layer18_attention_weight_max": 31.571626663208008,
      "activations/layer18_attention_weight_min": -26.242427825927734,
      "activations/layer19_attention_weight_max": 34.18170166015625,
      "activations/layer19_attention_weight_min": -30.715988159179688,
      "activations/layer1_attention_weight_max": 17.861074447631836,
      "activations/layer1_attention_weight_min": -15.901683807373047,
      "activations/layer20_attention_weight_max": 27.987205505371094,
      "activations/layer20_attention_weight_min": -24.945030212402344,
      "activations/layer21_attention_weight_max": 27.886734008789062,
      "activations/layer21_attention_weight_min": -25.563573837280273,
      "activations/layer22_attention_weight_max": 40.60452651977539,
      "activations/layer22_attention_weight_min": -30.91633415222168,
      "activations/layer23_attention_weight_max": 30.23347282409668,
      "activations/layer23_attention_weight_min": -24.89069175720215,
      "activations/layer2_attention_weight_max": 30.37154197692871,
      "activations/layer2_attention_weight_min": -30.016029357910156,
      "activations/layer3_attention_weight_max": 95.19425201416016,
      "activations/layer3_attention_weight_min": -95.69274139404297,
      "activations/layer4_attention_weight_max": 103.632080078125,
      "activations/layer4_attention_weight_min": -107.2889404296875,
      "activations/layer5_attention_weight_max": 47.67810821533203,
      "activations/layer5_attention_weight_min": -64.53671264648438,
      "activations/layer6_attention_weight_max": 42.02996826171875,
      "activations/layer6_attention_weight_min": -44.19088363647461,
      "activations/layer7_attention_weight_max": 96.0020980834961,
      "activations/layer7_attention_weight_min": -94.3100357055664,
      "activations/layer8_attention_weight_max": 39.65924835205078,
      "activations/layer8_attention_weight_min": -42.964385986328125,
      "activations/layer9_attention_weight_max": 31.919631958007812,
      "activations/layer9_attention_weight_min": -33.42875289916992,
      "epoch": 18.91,
      "learning_rate": 2.8336742424242422e-05,
      "loss": 2.7292,
      "step": 325450
    },
    {
      "activations/layer0_attention_weight_max": 17.17200469970703,
      "activations/layer0_attention_weight_min": -12.326311111450195,
      "activations/layer10_attention_weight_max": 32.66972351074219,
      "activations/layer10_attention_weight_min": -34.24872589111328,
      "activations/layer11_attention_weight_max": 33.957763671875,
      "activations/layer11_attention_weight_min": -33.895957946777344,
      "activations/layer12_attention_weight_max": 29.695751190185547,
      "activations/layer12_attention_weight_min": -28.100143432617188,
      "activations/layer13_attention_weight_max": 43.23655319213867,
      "activations/layer13_attention_weight_min": -36.56085205078125,
      "activations/layer14_attention_weight_max": 45.17202377319336,
      "activations/layer14_attention_weight_min": -38.814884185791016,
      "activations/layer15_attention_weight_max": 43.75364303588867,
      "activations/layer15_attention_weight_min": -35.536659240722656,
      "activations/layer16_attention_weight_max": 28.878826141357422,
      "activations/layer16_attention_weight_min": -26.9776668548584,
      "activations/layer17_attention_weight_max": 33.01460647583008,
      "activations/layer17_attention_weight_min": -25.950590133666992,
      "activations/layer18_attention_weight_max": 33.8647575378418,
      "activations/layer18_attention_weight_min": -27.9838809967041,
      "activations/layer19_attention_weight_max": 35.48613357543945,
      "activations/layer19_attention_weight_min": -33.45205307006836,
      "activations/layer1_attention_weight_max": 14.914053916931152,
      "activations/layer1_attention_weight_min": -15.909599304199219,
      "activations/layer20_attention_weight_max": 31.408491134643555,
      "activations/layer20_attention_weight_min": -25.565601348876953,
      "activations/layer21_attention_weight_max": 30.329763412475586,
      "activations/layer21_attention_weight_min": -24.782501220703125,
      "activations/layer22_attention_weight_max": 47.19046401977539,
      "activations/layer22_attention_weight_min": -32.533470153808594,
      "activations/layer23_attention_weight_max": 33.535343170166016,
      "activations/layer23_attention_weight_min": -25.892868041992188,
      "activations/layer2_attention_weight_max": 30.522602081298828,
      "activations/layer2_attention_weight_min": -30.270526885986328,
      "activations/layer3_attention_weight_max": 99.61637115478516,
      "activations/layer3_attention_weight_min": -97.6178207397461,
      "activations/layer4_attention_weight_max": 104.38607025146484,
      "activations/layer4_attention_weight_min": -103.13775634765625,
      "activations/layer5_attention_weight_max": 50.10460662841797,
      "activations/layer5_attention_weight_min": -63.41400146484375,
      "activations/layer6_attention_weight_max": 44.9638786315918,
      "activations/layer6_attention_weight_min": -47.85429763793945,
      "activations/layer7_attention_weight_max": 97.10397338867188,
      "activations/layer7_attention_weight_min": -96.8102035522461,
      "activations/layer8_attention_weight_max": 40.79160690307617,
      "activations/layer8_attention_weight_min": -42.69175720214844,
      "activations/layer9_attention_weight_max": 31.84059715270996,
      "activations/layer9_attention_weight_min": -33.81563949584961,
      "epoch": 18.91,
      "learning_rate": 2.8317803030303027e-05,
      "loss": 2.7308,
      "step": 325500
    },
    {
      "activations/layer0_attention_weight_max": 16.19821548461914,
      "activations/layer0_attention_weight_min": -12.379804611206055,
      "activations/layer10_attention_weight_max": 36.144859313964844,
      "activations/layer10_attention_weight_min": -36.54244613647461,
      "activations/layer11_attention_weight_max": 37.08256530761719,
      "activations/layer11_attention_weight_min": -35.40227508544922,
      "activations/layer12_attention_weight_max": 30.27018928527832,
      "activations/layer12_attention_weight_min": -26.806270599365234,
      "activations/layer13_attention_weight_max": 45.63907241821289,
      "activations/layer13_attention_weight_min": -36.73786926269531,
      "activations/layer14_attention_weight_max": 48.522422790527344,
      "activations/layer14_attention_weight_min": -38.42757034301758,
      "activations/layer15_attention_weight_max": 45.19234848022461,
      "activations/layer15_attention_weight_min": -37.062564849853516,
      "activations/layer16_attention_weight_max": 32.85282516479492,
      "activations/layer16_attention_weight_min": -29.511903762817383,
      "activations/layer17_attention_weight_max": 33.295833587646484,
      "activations/layer17_attention_weight_min": -26.03778839111328,
      "activations/layer18_attention_weight_max": 30.80517578125,
      "activations/layer18_attention_weight_min": -26.84971809387207,
      "activations/layer19_attention_weight_max": 34.00998306274414,
      "activations/layer19_attention_weight_min": -32.60288619995117,
      "activations/layer1_attention_weight_max": 15.377553939819336,
      "activations/layer1_attention_weight_min": -14.633072853088379,
      "activations/layer20_attention_weight_max": 33.04587936401367,
      "activations/layer20_attention_weight_min": -25.602764129638672,
      "activations/layer21_attention_weight_max": 29.84737205505371,
      "activations/layer21_attention_weight_min": -25.21678352355957,
      "activations/layer22_attention_weight_max": 43.77199935913086,
      "activations/layer22_attention_weight_min": -32.85063552856445,
      "activations/layer23_attention_weight_max": 31.855300903320312,
      "activations/layer23_attention_weight_min": -27.142410278320312,
      "activations/layer2_attention_weight_max": 31.670324325561523,
      "activations/layer2_attention_weight_min": -30.194137573242188,
      "activations/layer3_attention_weight_max": 99.00545501708984,
      "activations/layer3_attention_weight_min": -96.35953521728516,
      "activations/layer4_attention_weight_max": 105.822021484375,
      "activations/layer4_attention_weight_min": -106.56194305419922,
      "activations/layer5_attention_weight_max": 51.08169174194336,
      "activations/layer5_attention_weight_min": -62.29651641845703,
      "activations/layer6_attention_weight_max": 45.93161392211914,
      "activations/layer6_attention_weight_min": -47.889583587646484,
      "activations/layer7_attention_weight_max": 96.83307647705078,
      "activations/layer7_attention_weight_min": -102.95127868652344,
      "activations/layer8_attention_weight_max": 43.5079460144043,
      "activations/layer8_attention_weight_min": -47.87762451171875,
      "activations/layer9_attention_weight_max": 34.19533157348633,
      "activations/layer9_attention_weight_min": -36.84980392456055,
      "epoch": 18.92,
      "learning_rate": 2.8298863636363636e-05,
      "loss": 2.7434,
      "step": 325550
    },
    {
      "activations/layer0_attention_weight_max": 16.445087432861328,
      "activations/layer0_attention_weight_min": -13.548728942871094,
      "activations/layer10_attention_weight_max": 37.8727912902832,
      "activations/layer10_attention_weight_min": -37.11155700683594,
      "activations/layer11_attention_weight_max": 36.732627868652344,
      "activations/layer11_attention_weight_min": -36.92140197753906,
      "activations/layer12_attention_weight_max": 26.444948196411133,
      "activations/layer12_attention_weight_min": -27.30062484741211,
      "activations/layer13_attention_weight_max": 41.52584457397461,
      "activations/layer13_attention_weight_min": -38.85729217529297,
      "activations/layer14_attention_weight_max": 47.93941879272461,
      "activations/layer14_attention_weight_min": -42.2326545715332,
      "activations/layer15_attention_weight_max": 42.36333465576172,
      "activations/layer15_attention_weight_min": -38.48102569580078,
      "activations/layer16_attention_weight_max": 33.37550354003906,
      "activations/layer16_attention_weight_min": -27.692039489746094,
      "activations/layer17_attention_weight_max": 29.04990005493164,
      "activations/layer17_attention_weight_min": -25.56731414794922,
      "activations/layer18_attention_weight_max": 34.96449279785156,
      "activations/layer18_attention_weight_min": -28.062496185302734,
      "activations/layer19_attention_weight_max": 31.289031982421875,
      "activations/layer19_attention_weight_min": -32.28547286987305,
      "activations/layer1_attention_weight_max": 18.084449768066406,
      "activations/layer1_attention_weight_min": -15.710700988769531,
      "activations/layer20_attention_weight_max": 27.829227447509766,
      "activations/layer20_attention_weight_min": -25.03817367553711,
      "activations/layer21_attention_weight_max": 28.141353607177734,
      "activations/layer21_attention_weight_min": -24.34238624572754,
      "activations/layer22_attention_weight_max": 46.11381149291992,
      "activations/layer22_attention_weight_min": -30.09766960144043,
      "activations/layer23_attention_weight_max": 30.016164779663086,
      "activations/layer23_attention_weight_min": -25.035045623779297,
      "activations/layer2_attention_weight_max": 34.60523986816406,
      "activations/layer2_attention_weight_min": -32.13096237182617,
      "activations/layer3_attention_weight_max": 96.28729248046875,
      "activations/layer3_attention_weight_min": -99.1513900756836,
      "activations/layer4_attention_weight_max": 99.84835052490234,
      "activations/layer4_attention_weight_min": -106.62090301513672,
      "activations/layer5_attention_weight_max": 51.84275436401367,
      "activations/layer5_attention_weight_min": -63.083770751953125,
      "activations/layer6_attention_weight_max": 44.9864387512207,
      "activations/layer6_attention_weight_min": -46.756248474121094,
      "activations/layer7_attention_weight_max": 101.62591552734375,
      "activations/layer7_attention_weight_min": -97.48701477050781,
      "activations/layer8_attention_weight_max": 49.28988265991211,
      "activations/layer8_attention_weight_min": -48.5297737121582,
      "activations/layer9_attention_weight_max": 35.7382698059082,
      "activations/layer9_attention_weight_min": -37.45292282104492,
      "epoch": 18.92,
      "learning_rate": 2.8279924242424237e-05,
      "loss": 2.7259,
      "step": 325600
    },
    {
      "activations/layer0_attention_weight_max": 16.0616512298584,
      "activations/layer0_attention_weight_min": -14.862628936767578,
      "activations/layer10_attention_weight_max": 31.929702758789062,
      "activations/layer10_attention_weight_min": -33.45225143432617,
      "activations/layer11_attention_weight_max": 31.9036865234375,
      "activations/layer11_attention_weight_min": -31.800060272216797,
      "activations/layer12_attention_weight_max": 26.089969635009766,
      "activations/layer12_attention_weight_min": -27.14996910095215,
      "activations/layer13_attention_weight_max": 36.2999382019043,
      "activations/layer13_attention_weight_min": -33.44477462768555,
      "activations/layer14_attention_weight_max": 40.777976989746094,
      "activations/layer14_attention_weight_min": -35.55388259887695,
      "activations/layer15_attention_weight_max": 36.36654281616211,
      "activations/layer15_attention_weight_min": -34.58634567260742,
      "activations/layer16_attention_weight_max": 26.244861602783203,
      "activations/layer16_attention_weight_min": -27.758773803710938,
      "activations/layer17_attention_weight_max": 28.71096420288086,
      "activations/layer17_attention_weight_min": -24.78394889831543,
      "activations/layer18_attention_weight_max": 31.682809829711914,
      "activations/layer18_attention_weight_min": -27.453453063964844,
      "activations/layer19_attention_weight_max": 33.7064323425293,
      "activations/layer19_attention_weight_min": -31.905202865600586,
      "activations/layer1_attention_weight_max": 15.279711723327637,
      "activations/layer1_attention_weight_min": -15.291085243225098,
      "activations/layer20_attention_weight_max": 24.727432250976562,
      "activations/layer20_attention_weight_min": -25.433958053588867,
      "activations/layer21_attention_weight_max": 26.20843505859375,
      "activations/layer21_attention_weight_min": -25.959941864013672,
      "activations/layer22_attention_weight_max": 37.055259704589844,
      "activations/layer22_attention_weight_min": -31.98100471496582,
      "activations/layer23_attention_weight_max": 29.28042984008789,
      "activations/layer23_attention_weight_min": -25.898984909057617,
      "activations/layer2_attention_weight_max": 30.501365661621094,
      "activations/layer2_attention_weight_min": -30.074569702148438,
      "activations/layer3_attention_weight_max": 96.0527114868164,
      "activations/layer3_attention_weight_min": -95.02086639404297,
      "activations/layer4_attention_weight_max": 101.9383316040039,
      "activations/layer4_attention_weight_min": -104.51622772216797,
      "activations/layer5_attention_weight_max": 47.61115264892578,
      "activations/layer5_attention_weight_min": -61.79878234863281,
      "activations/layer6_attention_weight_max": 43.99031448364258,
      "activations/layer6_attention_weight_min": -45.30290603637695,
      "activations/layer7_attention_weight_max": 90.19248962402344,
      "activations/layer7_attention_weight_min": -94.63513946533203,
      "activations/layer8_attention_weight_max": 40.44799041748047,
      "activations/layer8_attention_weight_min": -42.075138092041016,
      "activations/layer9_attention_weight_max": 29.782581329345703,
      "activations/layer9_attention_weight_min": -31.71626091003418,
      "epoch": 18.92,
      "learning_rate": 2.8260984848484845e-05,
      "loss": 2.7282,
      "step": 325650
    },
    {
      "activations/layer0_attention_weight_max": 16.194236755371094,
      "activations/layer0_attention_weight_min": -11.683794975280762,
      "activations/layer10_attention_weight_max": 31.394033432006836,
      "activations/layer10_attention_weight_min": -34.27073669433594,
      "activations/layer11_attention_weight_max": 33.21099090576172,
      "activations/layer11_attention_weight_min": -34.89442443847656,
      "activations/layer12_attention_weight_max": 25.07457160949707,
      "activations/layer12_attention_weight_min": -27.546144485473633,
      "activations/layer13_attention_weight_max": 39.0089225769043,
      "activations/layer13_attention_weight_min": -37.17790603637695,
      "activations/layer14_attention_weight_max": 41.2738151550293,
      "activations/layer14_attention_weight_min": -38.89923858642578,
      "activations/layer15_attention_weight_max": 37.29997253417969,
      "activations/layer15_attention_weight_min": -37.3252067565918,
      "activations/layer16_attention_weight_max": 28.63981819152832,
      "activations/layer16_attention_weight_min": -28.53215789794922,
      "activations/layer17_attention_weight_max": 29.977380752563477,
      "activations/layer17_attention_weight_min": -28.07599639892578,
      "activations/layer18_attention_weight_max": 32.60920333862305,
      "activations/layer18_attention_weight_min": -30.135238647460938,
      "activations/layer19_attention_weight_max": 34.34495162963867,
      "activations/layer19_attention_weight_min": -34.80860900878906,
      "activations/layer1_attention_weight_max": 16.762165069580078,
      "activations/layer1_attention_weight_min": -15.432907104492188,
      "activations/layer20_attention_weight_max": 29.959814071655273,
      "activations/layer20_attention_weight_min": -30.348852157592773,
      "activations/layer21_attention_weight_max": 30.08524513244629,
      "activations/layer21_attention_weight_min": -27.859922409057617,
      "activations/layer22_attention_weight_max": 41.05350112915039,
      "activations/layer22_attention_weight_min": -34.298484802246094,
      "activations/layer23_attention_weight_max": 32.540374755859375,
      "activations/layer23_attention_weight_min": -26.857189178466797,
      "activations/layer2_attention_weight_max": 34.181480407714844,
      "activations/layer2_attention_weight_min": -30.558013916015625,
      "activations/layer3_attention_weight_max": 95.0277328491211,
      "activations/layer3_attention_weight_min": -95.71717834472656,
      "activations/layer4_attention_weight_max": 104.46014404296875,
      "activations/layer4_attention_weight_min": -103.0813217163086,
      "activations/layer5_attention_weight_max": 48.434959411621094,
      "activations/layer5_attention_weight_min": -67.00016784667969,
      "activations/layer6_attention_weight_max": 41.16371536254883,
      "activations/layer6_attention_weight_min": -47.11878204345703,
      "activations/layer7_attention_weight_max": 93.66523742675781,
      "activations/layer7_attention_weight_min": -93.07057189941406,
      "activations/layer8_attention_weight_max": 40.96287155151367,
      "activations/layer8_attention_weight_min": -43.759307861328125,
      "activations/layer9_attention_weight_max": 30.175296783447266,
      "activations/layer9_attention_weight_min": -33.479488372802734,
      "epoch": 18.92,
      "learning_rate": 2.824204545454545e-05,
      "loss": 2.7146,
      "step": 325700
    },
    {
      "activations/layer0_attention_weight_max": 16.070112228393555,
      "activations/layer0_attention_weight_min": -12.381522178649902,
      "activations/layer10_attention_weight_max": 33.498775482177734,
      "activations/layer10_attention_weight_min": -38.65373611450195,
      "activations/layer11_attention_weight_max": 32.562828063964844,
      "activations/layer11_attention_weight_min": -36.935508728027344,
      "activations/layer12_attention_weight_max": 26.15041732788086,
      "activations/layer12_attention_weight_min": -26.33981704711914,
      "activations/layer13_attention_weight_max": 36.16285705566406,
      "activations/layer13_attention_weight_min": -37.26338195800781,
      "activations/layer14_attention_weight_max": 40.52714157104492,
      "activations/layer14_attention_weight_min": -37.37675857543945,
      "activations/layer15_attention_weight_max": 37.06502914428711,
      "activations/layer15_attention_weight_min": -35.0941276550293,
      "activations/layer16_attention_weight_max": 27.27533531188965,
      "activations/layer16_attention_weight_min": -28.34170913696289,
      "activations/layer17_attention_weight_max": 30.394569396972656,
      "activations/layer17_attention_weight_min": -25.212467193603516,
      "activations/layer18_attention_weight_max": 31.40804672241211,
      "activations/layer18_attention_weight_min": -26.93643569946289,
      "activations/layer19_attention_weight_max": 31.16766357421875,
      "activations/layer19_attention_weight_min": -34.40025329589844,
      "activations/layer1_attention_weight_max": 17.287702560424805,
      "activations/layer1_attention_weight_min": -15.49710464477539,
      "activations/layer20_attention_weight_max": 27.37787437438965,
      "activations/layer20_attention_weight_min": -26.494945526123047,
      "activations/layer21_attention_weight_max": 26.899883270263672,
      "activations/layer21_attention_weight_min": -25.7164249420166,
      "activations/layer22_attention_weight_max": 40.156700134277344,
      "activations/layer22_attention_weight_min": -32.79140090942383,
      "activations/layer23_attention_weight_max": 29.878660202026367,
      "activations/layer23_attention_weight_min": -26.625988006591797,
      "activations/layer2_attention_weight_max": 31.771713256835938,
      "activations/layer2_attention_weight_min": -29.943700790405273,
      "activations/layer3_attention_weight_max": 96.18515014648438,
      "activations/layer3_attention_weight_min": -94.36976623535156,
      "activations/layer4_attention_weight_max": 102.1085433959961,
      "activations/layer4_attention_weight_min": -103.85151672363281,
      "activations/layer5_attention_weight_max": 49.925567626953125,
      "activations/layer5_attention_weight_min": -62.48948287963867,
      "activations/layer6_attention_weight_max": 42.802696228027344,
      "activations/layer6_attention_weight_min": -45.932090759277344,
      "activations/layer7_attention_weight_max": 90.18572235107422,
      "activations/layer7_attention_weight_min": -97.2151107788086,
      "activations/layer8_attention_weight_max": 39.99020004272461,
      "activations/layer8_attention_weight_min": -44.53071594238281,
      "activations/layer9_attention_weight_max": 31.26729393005371,
      "activations/layer9_attention_weight_min": -35.12775421142578,
      "epoch": 18.93,
      "learning_rate": 2.8223484848484848e-05,
      "loss": 2.7267,
      "step": 325750
    },
    {
      "activations/layer0_attention_weight_max": 17.085254669189453,
      "activations/layer0_attention_weight_min": -12.338266372680664,
      "activations/layer10_attention_weight_max": 31.524553298950195,
      "activations/layer10_attention_weight_min": -33.35529327392578,
      "activations/layer11_attention_weight_max": 34.619728088378906,
      "activations/layer11_attention_weight_min": -31.768348693847656,
      "activations/layer12_attention_weight_max": 28.223299026489258,
      "activations/layer12_attention_weight_min": -27.16904067993164,
      "activations/layer13_attention_weight_max": 41.36489486694336,
      "activations/layer13_attention_weight_min": -36.97871780395508,
      "activations/layer14_attention_weight_max": 47.623538970947266,
      "activations/layer14_attention_weight_min": -39.03492736816406,
      "activations/layer15_attention_weight_max": 42.70094680786133,
      "activations/layer15_attention_weight_min": -37.287540435791016,
      "activations/layer16_attention_weight_max": 32.6780891418457,
      "activations/layer16_attention_weight_min": -29.699724197387695,
      "activations/layer17_attention_weight_max": 30.261634826660156,
      "activations/layer17_attention_weight_min": -26.368480682373047,
      "activations/layer18_attention_weight_max": 31.598852157592773,
      "activations/layer18_attention_weight_min": -28.76400375366211,
      "activations/layer19_attention_weight_max": 33.761348724365234,
      "activations/layer19_attention_weight_min": -33.06989288330078,
      "activations/layer1_attention_weight_max": 18.127206802368164,
      "activations/layer1_attention_weight_min": -15.69553279876709,
      "activations/layer20_attention_weight_max": 28.460275650024414,
      "activations/layer20_attention_weight_min": -25.87285614013672,
      "activations/layer21_attention_weight_max": 28.63422203063965,
      "activations/layer21_attention_weight_min": -25.42580223083496,
      "activations/layer22_attention_weight_max": 42.221126556396484,
      "activations/layer22_attention_weight_min": -31.789249420166016,
      "activations/layer23_attention_weight_max": 34.696109771728516,
      "activations/layer23_attention_weight_min": -27.228206634521484,
      "activations/layer2_attention_weight_max": 33.16994857788086,
      "activations/layer2_attention_weight_min": -31.908390045166016,
      "activations/layer3_attention_weight_max": 96.93900299072266,
      "activations/layer3_attention_weight_min": -98.90491485595703,
      "activations/layer4_attention_weight_max": 102.94463348388672,
      "activations/layer4_attention_weight_min": -105.22529602050781,
      "activations/layer5_attention_weight_max": 48.12016296386719,
      "activations/layer5_attention_weight_min": -59.917869567871094,
      "activations/layer6_attention_weight_max": 42.729164123535156,
      "activations/layer6_attention_weight_min": -45.968482971191406,
      "activations/layer7_attention_weight_max": 87.95415496826172,
      "activations/layer7_attention_weight_min": -92.19151306152344,
      "activations/layer8_attention_weight_max": 39.964691162109375,
      "activations/layer8_attention_weight_min": -44.7337532043457,
      "activations/layer9_attention_weight_max": 30.906789779663086,
      "activations/layer9_attention_weight_min": -33.61790466308594,
      "epoch": 18.93,
      "learning_rate": 2.820454545454545e-05,
      "loss": 2.7284,
      "step": 325800
    },
    {
      "activations/layer0_attention_weight_max": 16.34401512145996,
      "activations/layer0_attention_weight_min": -12.196352005004883,
      "activations/layer10_attention_weight_max": 34.0021858215332,
      "activations/layer10_attention_weight_min": -35.798736572265625,
      "activations/layer11_attention_weight_max": 36.116790771484375,
      "activations/layer11_attention_weight_min": -35.09696578979492,
      "activations/layer12_attention_weight_max": 29.589025497436523,
      "activations/layer12_attention_weight_min": -28.066761016845703,
      "activations/layer13_attention_weight_max": 42.18732833862305,
      "activations/layer13_attention_weight_min": -36.696327209472656,
      "activations/layer14_attention_weight_max": 43.2440185546875,
      "activations/layer14_attention_weight_min": -39.520660400390625,
      "activations/layer15_attention_weight_max": 43.212833404541016,
      "activations/layer15_attention_weight_min": -36.17551040649414,
      "activations/layer16_attention_weight_max": 32.92255401611328,
      "activations/layer16_attention_weight_min": -28.49251937866211,
      "activations/layer17_attention_weight_max": 29.277008056640625,
      "activations/layer17_attention_weight_min": -26.348939895629883,
      "activations/layer18_attention_weight_max": 32.228336334228516,
      "activations/layer18_attention_weight_min": -29.15581703186035,
      "activations/layer19_attention_weight_max": 38.03676986694336,
      "activations/layer19_attention_weight_min": -33.73484420776367,
      "activations/layer1_attention_weight_max": 18.44171714782715,
      "activations/layer1_attention_weight_min": -15.220751762390137,
      "activations/layer20_attention_weight_max": 32.50599670410156,
      "activations/layer20_attention_weight_min": -26.65790367126465,
      "activations/layer21_attention_weight_max": 33.85762023925781,
      "activations/layer21_attention_weight_min": -29.757274627685547,
      "activations/layer22_attention_weight_max": 46.57219696044922,
      "activations/layer22_attention_weight_min": -34.49366760253906,
      "activations/layer23_attention_weight_max": 36.436832427978516,
      "activations/layer23_attention_weight_min": -27.308345794677734,
      "activations/layer2_attention_weight_max": 33.57441329956055,
      "activations/layer2_attention_weight_min": -31.219327926635742,
      "activations/layer3_attention_weight_max": 99.66914367675781,
      "activations/layer3_attention_weight_min": -98.4728012084961,
      "activations/layer4_attention_weight_max": 102.47655487060547,
      "activations/layer4_attention_weight_min": -103.92066955566406,
      "activations/layer5_attention_weight_max": 51.73387145996094,
      "activations/layer5_attention_weight_min": -63.260406494140625,
      "activations/layer6_attention_weight_max": 45.61904525756836,
      "activations/layer6_attention_weight_min": -46.29304504394531,
      "activations/layer7_attention_weight_max": 92.44867706298828,
      "activations/layer7_attention_weight_min": -97.49684143066406,
      "activations/layer8_attention_weight_max": 43.20073318481445,
      "activations/layer8_attention_weight_min": -45.635250091552734,
      "activations/layer9_attention_weight_max": 32.1175651550293,
      "activations/layer9_attention_weight_min": -36.437400817871094,
      "epoch": 18.93,
      "learning_rate": 2.8185606060606058e-05,
      "loss": 2.7246,
      "step": 325850
    },
    {
      "activations/layer0_attention_weight_max": 16.418285369873047,
      "activations/layer0_attention_weight_min": -11.943290710449219,
      "activations/layer10_attention_weight_max": 39.88858413696289,
      "activations/layer10_attention_weight_min": -36.71202087402344,
      "activations/layer11_attention_weight_max": 38.81853103637695,
      "activations/layer11_attention_weight_min": -39.70220947265625,
      "activations/layer12_attention_weight_max": 25.586591720581055,
      "activations/layer12_attention_weight_min": -25.38999366760254,
      "activations/layer13_attention_weight_max": 40.068817138671875,
      "activations/layer13_attention_weight_min": -33.926361083984375,
      "activations/layer14_attention_weight_max": 43.11564254760742,
      "activations/layer14_attention_weight_min": -35.80604934692383,
      "activations/layer15_attention_weight_max": 40.573097229003906,
      "activations/layer15_attention_weight_min": -35.99118423461914,
      "activations/layer16_attention_weight_max": 29.77421760559082,
      "activations/layer16_attention_weight_min": -29.97455596923828,
      "activations/layer17_attention_weight_max": 30.497615814208984,
      "activations/layer17_attention_weight_min": -26.552579879760742,
      "activations/layer18_attention_weight_max": 36.59777069091797,
      "activations/layer18_attention_weight_min": -25.984695434570312,
      "activations/layer19_attention_weight_max": 37.30247497558594,
      "activations/layer19_attention_weight_min": -33.94053649902344,
      "activations/layer1_attention_weight_max": 16.653818130493164,
      "activations/layer1_attention_weight_min": -15.153962135314941,
      "activations/layer20_attention_weight_max": 31.096433639526367,
      "activations/layer20_attention_weight_min": -26.436084747314453,
      "activations/layer21_attention_weight_max": 30.08907127380371,
      "activations/layer21_attention_weight_min": -25.436918258666992,
      "activations/layer22_attention_weight_max": 40.34457015991211,
      "activations/layer22_attention_weight_min": -32.41952896118164,
      "activations/layer23_attention_weight_max": 33.81755447387695,
      "activations/layer23_attention_weight_min": -26.930858612060547,
      "activations/layer2_attention_weight_max": 33.12586212158203,
      "activations/layer2_attention_weight_min": -31.581663131713867,
      "activations/layer3_attention_weight_max": 103.6992416381836,
      "activations/layer3_attention_weight_min": -99.82183074951172,
      "activations/layer4_attention_weight_max": 105.50154113769531,
      "activations/layer4_attention_weight_min": -105.24230194091797,
      "activations/layer5_attention_weight_max": 52.7591552734375,
      "activations/layer5_attention_weight_min": -61.54789352416992,
      "activations/layer6_attention_weight_max": 46.523963928222656,
      "activations/layer6_attention_weight_min": -48.40448760986328,
      "activations/layer7_attention_weight_max": 93.99531555175781,
      "activations/layer7_attention_weight_min": -100.5120849609375,
      "activations/layer8_attention_weight_max": 45.185367584228516,
      "activations/layer8_attention_weight_min": -47.971134185791016,
      "activations/layer9_attention_weight_max": 34.51508712768555,
      "activations/layer9_attention_weight_min": -35.19171142578125,
      "epoch": 18.94,
      "learning_rate": 2.8166666666666662e-05,
      "loss": 2.7178,
      "step": 325900
    },
    {
      "activations/layer0_attention_weight_max": 16.671424865722656,
      "activations/layer0_attention_weight_min": -12.16418743133545,
      "activations/layer10_attention_weight_max": 35.688724517822266,
      "activations/layer10_attention_weight_min": -34.27283477783203,
      "activations/layer11_attention_weight_max": 37.71165466308594,
      "activations/layer11_attention_weight_min": -34.95011901855469,
      "activations/layer12_attention_weight_max": 30.442729949951172,
      "activations/layer12_attention_weight_min": -27.288068771362305,
      "activations/layer13_attention_weight_max": 43.07353210449219,
      "activations/layer13_attention_weight_min": -37.36663055419922,
      "activations/layer14_attention_weight_max": 45.904151916503906,
      "activations/layer14_attention_weight_min": -38.24821472167969,
      "activations/layer15_attention_weight_max": 42.048095703125,
      "activations/layer15_attention_weight_min": -37.908531188964844,
      "activations/layer16_attention_weight_max": 32.18843078613281,
      "activations/layer16_attention_weight_min": -29.24915885925293,
      "activations/layer17_attention_weight_max": 31.048547744750977,
      "activations/layer17_attention_weight_min": -26.912504196166992,
      "activations/layer18_attention_weight_max": 34.34536361694336,
      "activations/layer18_attention_weight_min": -26.353464126586914,
      "activations/layer19_attention_weight_max": 35.031227111816406,
      "activations/layer19_attention_weight_min": -32.419273376464844,
      "activations/layer1_attention_weight_max": 15.69115161895752,
      "activations/layer1_attention_weight_min": -14.593055725097656,
      "activations/layer20_attention_weight_max": 30.049264907836914,
      "activations/layer20_attention_weight_min": -26.696399688720703,
      "activations/layer21_attention_weight_max": 30.52488899230957,
      "activations/layer21_attention_weight_min": -25.525516510009766,
      "activations/layer22_attention_weight_max": 41.44142532348633,
      "activations/layer22_attention_weight_min": -33.57978057861328,
      "activations/layer23_attention_weight_max": 34.02330017089844,
      "activations/layer23_attention_weight_min": -26.638893127441406,
      "activations/layer2_attention_weight_max": 32.979576110839844,
      "activations/layer2_attention_weight_min": -30.24982452392578,
      "activations/layer3_attention_weight_max": 91.75959014892578,
      "activations/layer3_attention_weight_min": -95.80126953125,
      "activations/layer4_attention_weight_max": 101.9354248046875,
      "activations/layer4_attention_weight_min": -103.04454803466797,
      "activations/layer5_attention_weight_max": 49.374351501464844,
      "activations/layer5_attention_weight_min": -61.64072799682617,
      "activations/layer6_attention_weight_max": 44.7429313659668,
      "activations/layer6_attention_weight_min": -46.24809646606445,
      "activations/layer7_attention_weight_max": 93.18720245361328,
      "activations/layer7_attention_weight_min": -94.43008422851562,
      "activations/layer8_attention_weight_max": 42.78690719604492,
      "activations/layer8_attention_weight_min": -44.17481994628906,
      "activations/layer9_attention_weight_max": 34.15758514404297,
      "activations/layer9_attention_weight_min": -34.491641998291016,
      "epoch": 18.94,
      "learning_rate": 2.814772727272727e-05,
      "loss": 2.7364,
      "step": 325950
    },
    {
      "activations/layer0_attention_weight_max": 16.602684020996094,
      "activations/layer0_attention_weight_min": -12.429710388183594,
      "activations/layer10_attention_weight_max": 34.8732795715332,
      "activations/layer10_attention_weight_min": -32.47715377807617,
      "activations/layer11_attention_weight_max": 33.103694915771484,
      "activations/layer11_attention_weight_min": -33.76390075683594,
      "activations/layer12_attention_weight_max": 26.131004333496094,
      "activations/layer12_attention_weight_min": -27.664155960083008,
      "activations/layer13_attention_weight_max": 39.98198318481445,
      "activations/layer13_attention_weight_min": -36.193302154541016,
      "activations/layer14_attention_weight_max": 41.78511428833008,
      "activations/layer14_attention_weight_min": -38.31222915649414,
      "activations/layer15_attention_weight_max": 39.990421295166016,
      "activations/layer15_attention_weight_min": -36.10368728637695,
      "activations/layer16_attention_weight_max": 31.36955451965332,
      "activations/layer16_attention_weight_min": -29.1460018157959,
      "activations/layer17_attention_weight_max": 29.796878814697266,
      "activations/layer17_attention_weight_min": -25.931129455566406,
      "activations/layer18_attention_weight_max": 32.609397888183594,
      "activations/layer18_attention_weight_min": -25.40225601196289,
      "activations/layer19_attention_weight_max": 39.90475082397461,
      "activations/layer19_attention_weight_min": -32.02216720581055,
      "activations/layer1_attention_weight_max": 16.86286163330078,
      "activations/layer1_attention_weight_min": -15.866186141967773,
      "activations/layer20_attention_weight_max": 30.1054744720459,
      "activations/layer20_attention_weight_min": -25.506736755371094,
      "activations/layer21_attention_weight_max": 29.24515151977539,
      "activations/layer21_attention_weight_min": -25.361204147338867,
      "activations/layer22_attention_weight_max": 44.540428161621094,
      "activations/layer22_attention_weight_min": -33.23860168457031,
      "activations/layer23_attention_weight_max": 31.119062423706055,
      "activations/layer23_attention_weight_min": -25.612041473388672,
      "activations/layer2_attention_weight_max": 32.88014221191406,
      "activations/layer2_attention_weight_min": -32.63698959350586,
      "activations/layer3_attention_weight_max": 107.8636245727539,
      "activations/layer3_attention_weight_min": -105.26864624023438,
      "activations/layer4_attention_weight_max": 113.32366180419922,
      "activations/layer4_attention_weight_min": -114.463623046875,
      "activations/layer5_attention_weight_max": 52.81458282470703,
      "activations/layer5_attention_weight_min": -60.085567474365234,
      "activations/layer6_attention_weight_max": 47.07035446166992,
      "activations/layer6_attention_weight_min": -50.275142669677734,
      "activations/layer7_attention_weight_max": 95.24618530273438,
      "activations/layer7_attention_weight_min": -103.94922637939453,
      "activations/layer8_attention_weight_max": 41.01431655883789,
      "activations/layer8_attention_weight_min": -44.731075286865234,
      "activations/layer9_attention_weight_max": 32.450252532958984,
      "activations/layer9_attention_weight_min": -34.73378372192383,
      "epoch": 18.94,
      "learning_rate": 2.8128787878787876e-05,
      "loss": 2.73,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4447,
      "eval_samples_per_second": 508.483,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4447,
      "eval_openwebtext_samples_per_second": 508.483,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9401,
      "eval_wikitext_samples_per_second": 235.044,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_lambada_loss": 2.419921875,
      "eval_lambada_ppl": 11.244980766441616,
      "eval_lambada_runtime": 9.5026,
      "eval_lambada_samples_per_second": 512.388,
      "step": 326000
    },
    {
      "activations/layer0_attention_weight_max": 16.080820083618164,
      "activations/layer0_attention_weight_min": -12.302529335021973,
      "activations/layer10_attention_weight_max": 34.60456848144531,
      "activations/layer10_attention_weight_min": -36.124664306640625,
      "activations/layer11_attention_weight_max": 35.354896545410156,
      "activations/layer11_attention_weight_min": -33.29941177368164,
      "activations/layer12_attention_weight_max": 26.38359832763672,
      "activations/layer12_attention_weight_min": -26.339094161987305,
      "activations/layer13_attention_weight_max": 39.875030517578125,
      "activations/layer13_attention_weight_min": -34.055450439453125,
      "activations/layer14_attention_weight_max": 43.16880416870117,
      "activations/layer14_attention_weight_min": -37.19285583496094,
      "activations/layer15_attention_weight_max": 39.6251106262207,
      "activations/layer15_attention_weight_min": -35.7275390625,
      "activations/layer16_attention_weight_max": 31.123958587646484,
      "activations/layer16_attention_weight_min": -29.030811309814453,
      "activations/layer17_attention_weight_max": 30.86668586730957,
      "activations/layer17_attention_weight_min": -25.67290687561035,
      "activations/layer18_attention_weight_max": 31.652854919433594,
      "activations/layer18_attention_weight_min": -26.01268196105957,
      "activations/layer19_attention_weight_max": 32.30423355102539,
      "activations/layer19_attention_weight_min": -33.44041442871094,
      "activations/layer1_attention_weight_max": 16.275514602661133,
      "activations/layer1_attention_weight_min": -15.095534324645996,
      "activations/layer20_attention_weight_max": 30.322185516357422,
      "activations/layer20_attention_weight_min": -26.151878356933594,
      "activations/layer21_attention_weight_max": 29.979461669921875,
      "activations/layer21_attention_weight_min": -25.977874755859375,
      "activations/layer22_attention_weight_max": 42.02439498901367,
      "activations/layer22_attention_weight_min": -32.3805046081543,
      "activations/layer23_attention_weight_max": 30.356122970581055,
      "activations/layer23_attention_weight_min": -26.117849349975586,
      "activations/layer2_attention_weight_max": 32.30210876464844,
      "activations/layer2_attention_weight_min": -32.95905303955078,
      "activations/layer3_attention_weight_max": 99.73680114746094,
      "activations/layer3_attention_weight_min": -99.48078918457031,
      "activations/layer4_attention_weight_max": 104.930419921875,
      "activations/layer4_attention_weight_min": -104.90685272216797,
      "activations/layer5_attention_weight_max": 49.18410873413086,
      "activations/layer5_attention_weight_min": -63.391048431396484,
      "activations/layer6_attention_weight_max": 44.30544662475586,
      "activations/layer6_attention_weight_min": -46.83803176879883,
      "activations/layer7_attention_weight_max": 94.09225463867188,
      "activations/layer7_attention_weight_min": -102.09015655517578,
      "activations/layer8_attention_weight_max": 44.241668701171875,
      "activations/layer8_attention_weight_min": -44.0287971496582,
      "activations/layer9_attention_weight_max": 33.620304107666016,
      "activations/layer9_attention_weight_min": -34.639610290527344,
      "epoch": 18.95,
      "learning_rate": 2.8109848484848484e-05,
      "loss": 2.7214,
      "step": 326050
    },
    {
      "activations/layer0_attention_weight_max": 16.823198318481445,
      "activations/layer0_attention_weight_min": -13.814724922180176,
      "activations/layer10_attention_weight_max": 38.06332778930664,
      "activations/layer10_attention_weight_min": -36.46326446533203,
      "activations/layer11_attention_weight_max": 39.240745544433594,
      "activations/layer11_attention_weight_min": -39.526588439941406,
      "activations/layer12_attention_weight_max": 28.624515533447266,
      "activations/layer12_attention_weight_min": -28.452716827392578,
      "activations/layer13_attention_weight_max": 43.68058776855469,
      "activations/layer13_attention_weight_min": -39.389495849609375,
      "activations/layer14_attention_weight_max": 46.97275161743164,
      "activations/layer14_attention_weight_min": -44.745548248291016,
      "activations/layer15_attention_weight_max": 44.86669921875,
      "activations/layer15_attention_weight_min": -39.279056549072266,
      "activations/layer16_attention_weight_max": 32.80134963989258,
      "activations/layer16_attention_weight_min": -28.559207916259766,
      "activations/layer17_attention_weight_max": 34.880611419677734,
      "activations/layer17_attention_weight_min": -28.205482482910156,
      "activations/layer18_attention_weight_max": 35.13045120239258,
      "activations/layer18_attention_weight_min": -27.875886917114258,
      "activations/layer19_attention_weight_max": 37.9604377746582,
      "activations/layer19_attention_weight_min": -31.84828758239746,
      "activations/layer1_attention_weight_max": 16.898256301879883,
      "activations/layer1_attention_weight_min": -15.192937850952148,
      "activations/layer20_attention_weight_max": 31.499176025390625,
      "activations/layer20_attention_weight_min": -26.332040786743164,
      "activations/layer21_attention_weight_max": 29.926326751708984,
      "activations/layer21_attention_weight_min": -25.83218765258789,
      "activations/layer22_attention_weight_max": 46.719085693359375,
      "activations/layer22_attention_weight_min": -35.055519104003906,
      "activations/layer23_attention_weight_max": 34.45906448364258,
      "activations/layer23_attention_weight_min": -27.240711212158203,
      "activations/layer2_attention_weight_max": 31.98151397705078,
      "activations/layer2_attention_weight_min": -30.885374069213867,
      "activations/layer3_attention_weight_max": 98.76791381835938,
      "activations/layer3_attention_weight_min": -100.06404876708984,
      "activations/layer4_attention_weight_max": 105.9089126586914,
      "activations/layer4_attention_weight_min": -105.19023895263672,
      "activations/layer5_attention_weight_max": 48.47694396972656,
      "activations/layer5_attention_weight_min": -60.19415283203125,
      "activations/layer6_attention_weight_max": 44.49794387817383,
      "activations/layer6_attention_weight_min": -46.33342361450195,
      "activations/layer7_attention_weight_max": 93.63744354248047,
      "activations/layer7_attention_weight_min": -94.88528442382812,
      "activations/layer8_attention_weight_max": 43.76460266113281,
      "activations/layer8_attention_weight_min": -44.032630920410156,
      "activations/layer9_attention_weight_max": 33.96602249145508,
      "activations/layer9_attention_weight_min": -35.14696502685547,
      "epoch": 18.95,
      "learning_rate": 2.809090909090909e-05,
      "loss": 2.7284,
      "step": 326100
    },
    {
      "activations/layer0_attention_weight_max": 15.56198787689209,
      "activations/layer0_attention_weight_min": -12.811561584472656,
      "activations/layer10_attention_weight_max": 37.61833572387695,
      "activations/layer10_attention_weight_min": -37.693260192871094,
      "activations/layer11_attention_weight_max": 37.67398452758789,
      "activations/layer11_attention_weight_min": -36.43495559692383,
      "activations/layer12_attention_weight_max": 25.33271598815918,
      "activations/layer12_attention_weight_min": -27.12747573852539,
      "activations/layer13_attention_weight_max": 38.49397277832031,
      "activations/layer13_attention_weight_min": -36.73661804199219,
      "activations/layer14_attention_weight_max": 39.51594924926758,
      "activations/layer14_attention_weight_min": -39.05288314819336,
      "activations/layer15_attention_weight_max": 37.457550048828125,
      "activations/layer15_attention_weight_min": -38.60960006713867,
      "activations/layer16_attention_weight_max": 31.254865646362305,
      "activations/layer16_attention_weight_min": -29.74392318725586,
      "activations/layer17_attention_weight_max": 30.016849517822266,
      "activations/layer17_attention_weight_min": -26.182071685791016,
      "activations/layer18_attention_weight_max": 29.98543930053711,
      "activations/layer18_attention_weight_min": -25.16938018798828,
      "activations/layer19_attention_weight_max": 33.27626419067383,
      "activations/layer19_attention_weight_min": -32.31547164916992,
      "activations/layer1_attention_weight_max": 14.261687278747559,
      "activations/layer1_attention_weight_min": -14.152068138122559,
      "activations/layer20_attention_weight_max": 30.253833770751953,
      "activations/layer20_attention_weight_min": -24.435352325439453,
      "activations/layer21_attention_weight_max": 27.33657455444336,
      "activations/layer21_attention_weight_min": -24.858449935913086,
      "activations/layer22_attention_weight_max": 39.0580940246582,
      "activations/layer22_attention_weight_min": -29.647764205932617,
      "activations/layer23_attention_weight_max": 32.918190002441406,
      "activations/layer23_attention_weight_min": -25.317167282104492,
      "activations/layer2_attention_weight_max": 29.9437313079834,
      "activations/layer2_attention_weight_min": -30.89162254333496,
      "activations/layer3_attention_weight_max": 100.93243408203125,
      "activations/layer3_attention_weight_min": -97.72781372070312,
      "activations/layer4_attention_weight_max": 103.7721939086914,
      "activations/layer4_attention_weight_min": -105.9754638671875,
      "activations/layer5_attention_weight_max": 48.762081146240234,
      "activations/layer5_attention_weight_min": -61.22205352783203,
      "activations/layer6_attention_weight_max": 43.5208625793457,
      "activations/layer6_attention_weight_min": -47.95149230957031,
      "activations/layer7_attention_weight_max": 97.71664428710938,
      "activations/layer7_attention_weight_min": -101.30607604980469,
      "activations/layer8_attention_weight_max": 42.78615188598633,
      "activations/layer8_attention_weight_min": -43.11490249633789,
      "activations/layer9_attention_weight_max": 31.51258659362793,
      "activations/layer9_attention_weight_min": -35.24494171142578,
      "epoch": 18.95,
      "learning_rate": 2.8071969696969694e-05,
      "loss": 2.7228,
      "step": 326150
    },
    {
      "activations/layer0_attention_weight_max": 17.28160285949707,
      "activations/layer0_attention_weight_min": -12.92591381072998,
      "activations/layer10_attention_weight_max": 35.51491928100586,
      "activations/layer10_attention_weight_min": -34.75593948364258,
      "activations/layer11_attention_weight_max": 36.833045959472656,
      "activations/layer11_attention_weight_min": -37.028480529785156,
      "activations/layer12_attention_weight_max": 30.122377395629883,
      "activations/layer12_attention_weight_min": -28.00147819519043,
      "activations/layer13_attention_weight_max": 44.63117218017578,
      "activations/layer13_attention_weight_min": -39.23577117919922,
      "activations/layer14_attention_weight_max": 49.41033935546875,
      "activations/layer14_attention_weight_min": -39.8231315612793,
      "activations/layer15_attention_weight_max": 44.30083084106445,
      "activations/layer15_attention_weight_min": -37.059566497802734,
      "activations/layer16_attention_weight_max": 31.04863166809082,
      "activations/layer16_attention_weight_min": -28.029956817626953,
      "activations/layer17_attention_weight_max": 32.111778259277344,
      "activations/layer17_attention_weight_min": -26.354923248291016,
      "activations/layer18_attention_weight_max": 31.341716766357422,
      "activations/layer18_attention_weight_min": -27.040145874023438,
      "activations/layer19_attention_weight_max": 33.7471809387207,
      "activations/layer19_attention_weight_min": -31.96058464050293,
      "activations/layer1_attention_weight_max": 15.2575044631958,
      "activations/layer1_attention_weight_min": -15.318317413330078,
      "activations/layer20_attention_weight_max": 28.02067756652832,
      "activations/layer20_attention_weight_min": -25.159584045410156,
      "activations/layer21_attention_weight_max": 27.749908447265625,
      "activations/layer21_attention_weight_min": -25.902385711669922,
      "activations/layer22_attention_weight_max": 40.143798828125,
      "activations/layer22_attention_weight_min": -33.46275329589844,
      "activations/layer23_attention_weight_max": 30.317899703979492,
      "activations/layer23_attention_weight_min": -28.454410552978516,
      "activations/layer2_attention_weight_max": 30.7910213470459,
      "activations/layer2_attention_weight_min": -31.420520782470703,
      "activations/layer3_attention_weight_max": 99.65535736083984,
      "activations/layer3_attention_weight_min": -101.89202880859375,
      "activations/layer4_attention_weight_max": 105.51854705810547,
      "activations/layer4_attention_weight_min": -108.38176727294922,
      "activations/layer5_attention_weight_max": 49.51104736328125,
      "activations/layer5_attention_weight_min": -63.31194305419922,
      "activations/layer6_attention_weight_max": 46.66968536376953,
      "activations/layer6_attention_weight_min": -48.23563766479492,
      "activations/layer7_attention_weight_max": 101.17024993896484,
      "activations/layer7_attention_weight_min": -99.65953826904297,
      "activations/layer8_attention_weight_max": 43.7830696105957,
      "activations/layer8_attention_weight_min": -48.18586730957031,
      "activations/layer9_attention_weight_max": 34.015804290771484,
      "activations/layer9_attention_weight_min": -35.923587799072266,
      "epoch": 18.95,
      "learning_rate": 2.8053030303030302e-05,
      "loss": 2.7396,
      "step": 326200
    },
    {
      "activations/layer0_attention_weight_max": 16.164018630981445,
      "activations/layer0_attention_weight_min": -13.088824272155762,
      "activations/layer10_attention_weight_max": 34.03902816772461,
      "activations/layer10_attention_weight_min": -34.7313117980957,
      "activations/layer11_attention_weight_max": 34.907569885253906,
      "activations/layer11_attention_weight_min": -34.74958038330078,
      "activations/layer12_attention_weight_max": 26.547853469848633,
      "activations/layer12_attention_weight_min": -26.98405647277832,
      "activations/layer13_attention_weight_max": 42.47389602661133,
      "activations/layer13_attention_weight_min": -36.7444953918457,
      "activations/layer14_attention_weight_max": 42.40607452392578,
      "activations/layer14_attention_weight_min": -37.26710510253906,
      "activations/layer15_attention_weight_max": 39.654422760009766,
      "activations/layer15_attention_weight_min": -37.76224899291992,
      "activations/layer16_attention_weight_max": 29.339210510253906,
      "activations/layer16_attention_weight_min": -30.78556251525879,
      "activations/layer17_attention_weight_max": 31.842844009399414,
      "activations/layer17_attention_weight_min": -27.548137664794922,
      "activations/layer18_attention_weight_max": 30.434099197387695,
      "activations/layer18_attention_weight_min": -26.38195037841797,
      "activations/layer19_attention_weight_max": 33.16842269897461,
      "activations/layer19_attention_weight_min": -33.00745391845703,
      "activations/layer1_attention_weight_max": 15.025879859924316,
      "activations/layer1_attention_weight_min": -14.736305236816406,
      "activations/layer20_attention_weight_max": 30.67629623413086,
      "activations/layer20_attention_weight_min": -26.437963485717773,
      "activations/layer21_attention_weight_max": 29.337799072265625,
      "activations/layer21_attention_weight_min": -26.464139938354492,
      "activations/layer22_attention_weight_max": 41.60833740234375,
      "activations/layer22_attention_weight_min": -33.273780822753906,
      "activations/layer23_attention_weight_max": 31.283870697021484,
      "activations/layer23_attention_weight_min": -27.20858383178711,
      "activations/layer2_attention_weight_max": 30.572500228881836,
      "activations/layer2_attention_weight_min": -31.099937438964844,
      "activations/layer3_attention_weight_max": 94.75971984863281,
      "activations/layer3_attention_weight_min": -98.15037536621094,
      "activations/layer4_attention_weight_max": 103.01421356201172,
      "activations/layer4_attention_weight_min": -111.19392395019531,
      "activations/layer5_attention_weight_max": 50.9854736328125,
      "activations/layer5_attention_weight_min": -63.17570495605469,
      "activations/layer6_attention_weight_max": 44.166481018066406,
      "activations/layer6_attention_weight_min": -47.7580680847168,
      "activations/layer7_attention_weight_max": 97.32061004638672,
      "activations/layer7_attention_weight_min": -101.11808013916016,
      "activations/layer8_attention_weight_max": 45.28230667114258,
      "activations/layer8_attention_weight_min": -44.6472282409668,
      "activations/layer9_attention_weight_max": 31.015684127807617,
      "activations/layer9_attention_weight_min": -35.78534698486328,
      "epoch": 18.96,
      "learning_rate": 2.8034090909090907e-05,
      "loss": 2.7437,
      "step": 326250
    },
    {
      "activations/layer0_attention_weight_max": 16.73685646057129,
      "activations/layer0_attention_weight_min": -11.992558479309082,
      "activations/layer10_attention_weight_max": 34.18886184692383,
      "activations/layer10_attention_weight_min": -33.95512390136719,
      "activations/layer11_attention_weight_max": 35.47156524658203,
      "activations/layer11_attention_weight_min": -34.146240234375,
      "activations/layer12_attention_weight_max": 26.847017288208008,
      "activations/layer12_attention_weight_min": -26.23760223388672,
      "activations/layer13_attention_weight_max": 38.2393913269043,
      "activations/layer13_attention_weight_min": -38.15659713745117,
      "activations/layer14_attention_weight_max": 41.03312683105469,
      "activations/layer14_attention_weight_min": -38.74468231201172,
      "activations/layer15_attention_weight_max": 38.52878189086914,
      "activations/layer15_attention_weight_min": -38.36075210571289,
      "activations/layer16_attention_weight_max": 29.73699378967285,
      "activations/layer16_attention_weight_min": -29.43671417236328,
      "activations/layer17_attention_weight_max": 28.76423454284668,
      "activations/layer17_attention_weight_min": -26.143470764160156,
      "activations/layer18_attention_weight_max": 37.193233489990234,
      "activations/layer18_attention_weight_min": -27.8730525970459,
      "activations/layer19_attention_weight_max": 35.82003402709961,
      "activations/layer19_attention_weight_min": -32.97004699707031,
      "activations/layer1_attention_weight_max": 15.962449073791504,
      "activations/layer1_attention_weight_min": -14.528730392456055,
      "activations/layer20_attention_weight_max": 28.714954376220703,
      "activations/layer20_attention_weight_min": -25.629383087158203,
      "activations/layer21_attention_weight_max": 28.8942928314209,
      "activations/layer21_attention_weight_min": -27.807035446166992,
      "activations/layer22_attention_weight_max": 44.286285400390625,
      "activations/layer22_attention_weight_min": -36.00828552246094,
      "activations/layer23_attention_weight_max": 36.45559310913086,
      "activations/layer23_attention_weight_min": -30.373708724975586,
      "activations/layer2_attention_weight_max": 30.67218780517578,
      "activations/layer2_attention_weight_min": -30.788833618164062,
      "activations/layer3_attention_weight_max": 98.16201782226562,
      "activations/layer3_attention_weight_min": -98.7015151977539,
      "activations/layer4_attention_weight_max": 102.39299011230469,
      "activations/layer4_attention_weight_min": -107.00819396972656,
      "activations/layer5_attention_weight_max": 48.36228942871094,
      "activations/layer5_attention_weight_min": -59.80830383300781,
      "activations/layer6_attention_weight_max": 45.01841354370117,
      "activations/layer6_attention_weight_min": -48.12175750732422,
      "activations/layer7_attention_weight_max": 88.45329284667969,
      "activations/layer7_attention_weight_min": -98.26749420166016,
      "activations/layer8_attention_weight_max": 42.59992218017578,
      "activations/layer8_attention_weight_min": -46.163780212402344,
      "activations/layer9_attention_weight_max": 31.986974716186523,
      "activations/layer9_attention_weight_min": -35.346153259277344,
      "epoch": 18.96,
      "learning_rate": 2.80155303030303e-05,
      "loss": 2.7113,
      "step": 326300
    },
    {
      "activations/layer0_attention_weight_max": 17.222946166992188,
      "activations/layer0_attention_weight_min": -13.042725563049316,
      "activations/layer10_attention_weight_max": 32.19404602050781,
      "activations/layer10_attention_weight_min": -34.9876708984375,
      "activations/layer11_attention_weight_max": 33.14836502075195,
      "activations/layer11_attention_weight_min": -35.49337387084961,
      "activations/layer12_attention_weight_max": 26.44725799560547,
      "activations/layer12_attention_weight_min": -27.481733322143555,
      "activations/layer13_attention_weight_max": 38.72002029418945,
      "activations/layer13_attention_weight_min": -38.48187255859375,
      "activations/layer14_attention_weight_max": 44.77973937988281,
      "activations/layer14_attention_weight_min": -41.41337585449219,
      "activations/layer15_attention_weight_max": 40.23907470703125,
      "activations/layer15_attention_weight_min": -39.74654006958008,
      "activations/layer16_attention_weight_max": 31.637109756469727,
      "activations/layer16_attention_weight_min": -30.179821014404297,
      "activations/layer17_attention_weight_max": 35.60002517700195,
      "activations/layer17_attention_weight_min": -26.393529891967773,
      "activations/layer18_attention_weight_max": 35.21265411376953,
      "activations/layer18_attention_weight_min": -25.994997024536133,
      "activations/layer19_attention_weight_max": 40.52792739868164,
      "activations/layer19_attention_weight_min": -33.23094177246094,
      "activations/layer1_attention_weight_max": 15.652337074279785,
      "activations/layer1_attention_weight_min": -15.014235496520996,
      "activations/layer20_attention_weight_max": 31.07085418701172,
      "activations/layer20_attention_weight_min": -27.065244674682617,
      "activations/layer21_attention_weight_max": 30.537059783935547,
      "activations/layer21_attention_weight_min": -25.188066482543945,
      "activations/layer22_attention_weight_max": 41.84919738769531,
      "activations/layer22_attention_weight_min": -32.55173110961914,
      "activations/layer23_attention_weight_max": 35.60913848876953,
      "activations/layer23_attention_weight_min": -27.651473999023438,
      "activations/layer2_attention_weight_max": 31.121910095214844,
      "activations/layer2_attention_weight_min": -29.880508422851562,
      "activations/layer3_attention_weight_max": 94.62762451171875,
      "activations/layer3_attention_weight_min": -94.8467025756836,
      "activations/layer4_attention_weight_max": 105.28372955322266,
      "activations/layer4_attention_weight_min": -109.5379638671875,
      "activations/layer5_attention_weight_max": 52.229034423828125,
      "activations/layer5_attention_weight_min": -64.2152099609375,
      "activations/layer6_attention_weight_max": 42.98434829711914,
      "activations/layer6_attention_weight_min": -46.695213317871094,
      "activations/layer7_attention_weight_max": 88.23910522460938,
      "activations/layer7_attention_weight_min": -95.93373107910156,
      "activations/layer8_attention_weight_max": 41.12855911254883,
      "activations/layer8_attention_weight_min": -45.49394607543945,
      "activations/layer9_attention_weight_max": 31.598356246948242,
      "activations/layer9_attention_weight_min": -34.364070892333984,
      "epoch": 18.96,
      "learning_rate": 2.7996590909090906e-05,
      "loss": 2.713,
      "step": 326350
    },
    {
      "activations/layer0_attention_weight_max": 16.390470504760742,
      "activations/layer0_attention_weight_min": -13.411540031433105,
      "activations/layer10_attention_weight_max": 36.47822952270508,
      "activations/layer10_attention_weight_min": -38.72465515136719,
      "activations/layer11_attention_weight_max": 34.53759002685547,
      "activations/layer11_attention_weight_min": -37.32604217529297,
      "activations/layer12_attention_weight_max": 29.50650405883789,
      "activations/layer12_attention_weight_min": -31.09621810913086,
      "activations/layer13_attention_weight_max": 44.46311569213867,
      "activations/layer13_attention_weight_min": -40.69728088378906,
      "activations/layer14_attention_weight_max": 44.63762283325195,
      "activations/layer14_attention_weight_min": -40.62029266357422,
      "activations/layer15_attention_weight_max": 40.533348083496094,
      "activations/layer15_attention_weight_min": -36.856136322021484,
      "activations/layer16_attention_weight_max": 29.975727081298828,
      "activations/layer16_attention_weight_min": -30.444608688354492,
      "activations/layer17_attention_weight_max": 31.277076721191406,
      "activations/layer17_attention_weight_min": -25.53837013244629,
      "activations/layer18_attention_weight_max": 31.094003677368164,
      "activations/layer18_attention_weight_min": -24.44976043701172,
      "activations/layer19_attention_weight_max": 32.97638702392578,
      "activations/layer19_attention_weight_min": -31.18182945251465,
      "activations/layer1_attention_weight_max": 15.936677932739258,
      "activations/layer1_attention_weight_min": -14.790658950805664,
      "activations/layer20_attention_weight_max": 27.716575622558594,
      "activations/layer20_attention_weight_min": -24.897653579711914,
      "activations/layer21_attention_weight_max": 28.6466064453125,
      "activations/layer21_attention_weight_min": -24.53623390197754,
      "activations/layer22_attention_weight_max": 42.24346160888672,
      "activations/layer22_attention_weight_min": -30.593673706054688,
      "activations/layer23_attention_weight_max": 31.49043846130371,
      "activations/layer23_attention_weight_min": -26.64076805114746,
      "activations/layer2_attention_weight_max": 30.58875274658203,
      "activations/layer2_attention_weight_min": -30.48323631286621,
      "activations/layer3_attention_weight_max": 100.046142578125,
      "activations/layer3_attention_weight_min": -104.05561065673828,
      "activations/layer4_attention_weight_max": 109.1957778930664,
      "activations/layer4_attention_weight_min": -107.32291412353516,
      "activations/layer5_attention_weight_max": 53.236263275146484,
      "activations/layer5_attention_weight_min": -63.468196868896484,
      "activations/layer6_attention_weight_max": 45.43742752075195,
      "activations/layer6_attention_weight_min": -45.34947967529297,
      "activations/layer7_attention_weight_max": 97.3802719116211,
      "activations/layer7_attention_weight_min": -100.93609619140625,
      "activations/layer8_attention_weight_max": 43.78887176513672,
      "activations/layer8_attention_weight_min": -46.00687026977539,
      "activations/layer9_attention_weight_max": 33.879207611083984,
      "activations/layer9_attention_weight_min": -35.59319305419922,
      "epoch": 18.97,
      "learning_rate": 2.7977651515151514e-05,
      "loss": 2.743,
      "step": 326400
    },
    {
      "activations/layer0_attention_weight_max": 16.233861923217773,
      "activations/layer0_attention_weight_min": -12.143644332885742,
      "activations/layer10_attention_weight_max": 33.09560012817383,
      "activations/layer10_attention_weight_min": -36.25815963745117,
      "activations/layer11_attention_weight_max": 33.09087371826172,
      "activations/layer11_attention_weight_min": -35.60077667236328,
      "activations/layer12_attention_weight_max": 27.16683578491211,
      "activations/layer12_attention_weight_min": -26.088159561157227,
      "activations/layer13_attention_weight_max": 43.002689361572266,
      "activations/layer13_attention_weight_min": -35.765235900878906,
      "activations/layer14_attention_weight_max": 47.77452087402344,
      "activations/layer14_attention_weight_min": -40.69449996948242,
      "activations/layer15_attention_weight_max": 40.36969757080078,
      "activations/layer15_attention_weight_min": -36.603111267089844,
      "activations/layer16_attention_weight_max": 28.90690040588379,
      "activations/layer16_attention_weight_min": -27.410900115966797,
      "activations/layer17_attention_weight_max": 32.271915435791016,
      "activations/layer17_attention_weight_min": -25.105470657348633,
      "activations/layer18_attention_weight_max": 33.25212478637695,
      "activations/layer18_attention_weight_min": -25.98565101623535,
      "activations/layer19_attention_weight_max": 35.156429290771484,
      "activations/layer19_attention_weight_min": -30.881711959838867,
      "activations/layer1_attention_weight_max": 15.403714179992676,
      "activations/layer1_attention_weight_min": -14.340991973876953,
      "activations/layer20_attention_weight_max": 31.485477447509766,
      "activations/layer20_attention_weight_min": -27.245452880859375,
      "activations/layer21_attention_weight_max": 33.49347686767578,
      "activations/layer21_attention_weight_min": -24.62749671936035,
      "activations/layer22_attention_weight_max": 47.985172271728516,
      "activations/layer22_attention_weight_min": -32.246334075927734,
      "activations/layer23_attention_weight_max": 36.235191345214844,
      "activations/layer23_attention_weight_min": -26.985435485839844,
      "activations/layer2_attention_weight_max": 31.852001190185547,
      "activations/layer2_attention_weight_min": -30.734134674072266,
      "activations/layer3_attention_weight_max": 95.71607971191406,
      "activations/layer3_attention_weight_min": -96.895263671875,
      "activations/layer4_attention_weight_max": 103.39580535888672,
      "activations/layer4_attention_weight_min": -106.4493179321289,
      "activations/layer5_attention_weight_max": 48.68117904663086,
      "activations/layer5_attention_weight_min": -63.14024353027344,
      "activations/layer6_attention_weight_max": 42.438987731933594,
      "activations/layer6_attention_weight_min": -48.620304107666016,
      "activations/layer7_attention_weight_max": 93.17777252197266,
      "activations/layer7_attention_weight_min": -99.80860900878906,
      "activations/layer8_attention_weight_max": 41.03434753417969,
      "activations/layer8_attention_weight_min": -44.6434211730957,
      "activations/layer9_attention_weight_max": 33.08256530761719,
      "activations/layer9_attention_weight_min": -35.64947509765625,
      "epoch": 18.97,
      "learning_rate": 2.795871212121212e-05,
      "loss": 2.7429,
      "step": 326450
    },
    {
      "activations/layer0_attention_weight_max": 17.44692039489746,
      "activations/layer0_attention_weight_min": -12.500945091247559,
      "activations/layer10_attention_weight_max": 34.61273956298828,
      "activations/layer10_attention_weight_min": -35.25449752807617,
      "activations/layer11_attention_weight_max": 34.610694885253906,
      "activations/layer11_attention_weight_min": -35.400413513183594,
      "activations/layer12_attention_weight_max": 27.182064056396484,
      "activations/layer12_attention_weight_min": -27.132001876831055,
      "activations/layer13_attention_weight_max": 38.0345458984375,
      "activations/layer13_attention_weight_min": -36.169029235839844,
      "activations/layer14_attention_weight_max": 42.65835952758789,
      "activations/layer14_attention_weight_min": -37.52177810668945,
      "activations/layer15_attention_weight_max": 36.81611633300781,
      "activations/layer15_attention_weight_min": -38.15934371948242,
      "activations/layer16_attention_weight_max": 28.01064109802246,
      "activations/layer16_attention_weight_min": -29.519695281982422,
      "activations/layer17_attention_weight_max": 29.91965675354004,
      "activations/layer17_attention_weight_min": -27.0346622467041,
      "activations/layer18_attention_weight_max": 34.21636199951172,
      "activations/layer18_attention_weight_min": -26.523881912231445,
      "activations/layer19_attention_weight_max": 33.384952545166016,
      "activations/layer19_attention_weight_min": -32.73421859741211,
      "activations/layer1_attention_weight_max": 17.405540466308594,
      "activations/layer1_attention_weight_min": -17.93871307373047,
      "activations/layer20_attention_weight_max": 26.339387893676758,
      "activations/layer20_attention_weight_min": -24.79142951965332,
      "activations/layer21_attention_weight_max": 27.381999969482422,
      "activations/layer21_attention_weight_min": -24.42394256591797,
      "activations/layer22_attention_weight_max": 40.70219039916992,
      "activations/layer22_attention_weight_min": -33.61305618286133,
      "activations/layer23_attention_weight_max": 32.82948684692383,
      "activations/layer23_attention_weight_min": -26.85724449157715,
      "activations/layer2_attention_weight_max": 31.876785278320312,
      "activations/layer2_attention_weight_min": -31.741260528564453,
      "activations/layer3_attention_weight_max": 98.53919982910156,
      "activations/layer3_attention_weight_min": -100.63044738769531,
      "activations/layer4_attention_weight_max": 107.0719985961914,
      "activations/layer4_attention_weight_min": -111.16043853759766,
      "activations/layer5_attention_weight_max": 49.54841613769531,
      "activations/layer5_attention_weight_min": -62.13987350463867,
      "activations/layer6_attention_weight_max": 42.14524459838867,
      "activations/layer6_attention_weight_min": -46.360836029052734,
      "activations/layer7_attention_weight_max": 91.2397232055664,
      "activations/layer7_attention_weight_min": -94.75640106201172,
      "activations/layer8_attention_weight_max": 40.20982360839844,
      "activations/layer8_attention_weight_min": -42.21897506713867,
      "activations/layer9_attention_weight_max": 29.973831176757812,
      "activations/layer9_attention_weight_min": -33.00365447998047,
      "epoch": 18.97,
      "learning_rate": 2.7939772727272727e-05,
      "loss": 2.7075,
      "step": 326500
    },
    {
      "activations/layer0_attention_weight_max": 16.0003604888916,
      "activations/layer0_attention_weight_min": -11.845834732055664,
      "activations/layer10_attention_weight_max": 32.09912109375,
      "activations/layer10_attention_weight_min": -33.497501373291016,
      "activations/layer11_attention_weight_max": 33.34279251098633,
      "activations/layer11_attention_weight_min": -32.48646545410156,
      "activations/layer12_attention_weight_max": 26.231157302856445,
      "activations/layer12_attention_weight_min": -26.248844146728516,
      "activations/layer13_attention_weight_max": 39.43335723876953,
      "activations/layer13_attention_weight_min": -35.829620361328125,
      "activations/layer14_attention_weight_max": 41.50532913208008,
      "activations/layer14_attention_weight_min": -37.60041046142578,
      "activations/layer15_attention_weight_max": 36.74497985839844,
      "activations/layer15_attention_weight_min": -36.930843353271484,
      "activations/layer16_attention_weight_max": 26.434051513671875,
      "activations/layer16_attention_weight_min": -28.75754165649414,
      "activations/layer17_attention_weight_max": 29.1529483795166,
      "activations/layer17_attention_weight_min": -24.82168960571289,
      "activations/layer18_attention_weight_max": 31.61260986328125,
      "activations/layer18_attention_weight_min": -26.242345809936523,
      "activations/layer19_attention_weight_max": 33.1922607421875,
      "activations/layer19_attention_weight_min": -31.725187301635742,
      "activations/layer1_attention_weight_max": 16.10325050354004,
      "activations/layer1_attention_weight_min": -16.900230407714844,
      "activations/layer20_attention_weight_max": 29.535993576049805,
      "activations/layer20_attention_weight_min": -25.984901428222656,
      "activations/layer21_attention_weight_max": 29.059707641601562,
      "activations/layer21_attention_weight_min": -27.166881561279297,
      "activations/layer22_attention_weight_max": 42.201534271240234,
      "activations/layer22_attention_weight_min": -33.020565032958984,
      "activations/layer23_attention_weight_max": 31.606616973876953,
      "activations/layer23_attention_weight_min": -29.509197235107422,
      "activations/layer2_attention_weight_max": 31.05293083190918,
      "activations/layer2_attention_weight_min": -29.632896423339844,
      "activations/layer3_attention_weight_max": 96.79473114013672,
      "activations/layer3_attention_weight_min": -99.31209564208984,
      "activations/layer4_attention_weight_max": 105.2643814086914,
      "activations/layer4_attention_weight_min": -106.9761962890625,
      "activations/layer5_attention_weight_max": 52.44131088256836,
      "activations/layer5_attention_weight_min": -60.15214538574219,
      "activations/layer6_attention_weight_max": 44.66450119018555,
      "activations/layer6_attention_weight_min": -46.11534118652344,
      "activations/layer7_attention_weight_max": 98.01469421386719,
      "activations/layer7_attention_weight_min": -92.42454528808594,
      "activations/layer8_attention_weight_max": 39.77680969238281,
      "activations/layer8_attention_weight_min": -42.07474136352539,
      "activations/layer9_attention_weight_max": 30.54451560974121,
      "activations/layer9_attention_weight_min": -33.99744415283203,
      "epoch": 18.97,
      "learning_rate": 2.792083333333333e-05,
      "loss": 2.7488,
      "step": 326550
    },
    {
      "activations/layer0_attention_weight_max": 16.00460433959961,
      "activations/layer0_attention_weight_min": -12.15810775756836,
      "activations/layer10_attention_weight_max": 36.9274787902832,
      "activations/layer10_attention_weight_min": -35.23435974121094,
      "activations/layer11_attention_weight_max": 36.256561279296875,
      "activations/layer11_attention_weight_min": -35.53541564941406,
      "activations/layer12_attention_weight_max": 24.80112648010254,
      "activations/layer12_attention_weight_min": -25.03181266784668,
      "activations/layer13_attention_weight_max": 39.81291580200195,
      "activations/layer13_attention_weight_min": -35.69877243041992,
      "activations/layer14_attention_weight_max": 51.01713180541992,
      "activations/layer14_attention_weight_min": -39.14773178100586,
      "activations/layer15_attention_weight_max": 40.85371780395508,
      "activations/layer15_attention_weight_min": -37.9409294128418,
      "activations/layer16_attention_weight_max": 28.55561065673828,
      "activations/layer16_attention_weight_min": -29.03026008605957,
      "activations/layer17_attention_weight_max": 27.561737060546875,
      "activations/layer17_attention_weight_min": -26.38831901550293,
      "activations/layer18_attention_weight_max": 29.733957290649414,
      "activations/layer18_attention_weight_min": -26.909465789794922,
      "activations/layer19_attention_weight_max": 32.93619918823242,
      "activations/layer19_attention_weight_min": -31.42813491821289,
      "activations/layer1_attention_weight_max": 17.363605499267578,
      "activations/layer1_attention_weight_min": -15.451414108276367,
      "activations/layer20_attention_weight_max": 28.892749786376953,
      "activations/layer20_attention_weight_min": -24.155874252319336,
      "activations/layer21_attention_weight_max": 28.106510162353516,
      "activations/layer21_attention_weight_min": -25.13010597229004,
      "activations/layer22_attention_weight_max": 40.80259704589844,
      "activations/layer22_attention_weight_min": -30.967256546020508,
      "activations/layer23_attention_weight_max": 30.587665557861328,
      "activations/layer23_attention_weight_min": -25.38996124267578,
      "activations/layer2_attention_weight_max": 31.405006408691406,
      "activations/layer2_attention_weight_min": -30.47726058959961,
      "activations/layer3_attention_weight_max": 97.62022399902344,
      "activations/layer3_attention_weight_min": -99.26461029052734,
      "activations/layer4_attention_weight_max": 105.12983703613281,
      "activations/layer4_attention_weight_min": -106.32209777832031,
      "activations/layer5_attention_weight_max": 49.17478561401367,
      "activations/layer5_attention_weight_min": -61.87784957885742,
      "activations/layer6_attention_weight_max": 44.02668380737305,
      "activations/layer6_attention_weight_min": -45.007545471191406,
      "activations/layer7_attention_weight_max": 96.48457336425781,
      "activations/layer7_attention_weight_min": -96.5163803100586,
      "activations/layer8_attention_weight_max": 42.32926940917969,
      "activations/layer8_attention_weight_min": -43.475547790527344,
      "activations/layer9_attention_weight_max": 33.4510612487793,
      "activations/layer9_attention_weight_min": -35.84565353393555,
      "epoch": 18.98,
      "learning_rate": 2.7901893939393937e-05,
      "loss": 2.7095,
      "step": 326600
    },
    {
      "activations/layer0_attention_weight_max": 15.674967765808105,
      "activations/layer0_attention_weight_min": -12.819528579711914,
      "activations/layer10_attention_weight_max": 36.48566818237305,
      "activations/layer10_attention_weight_min": -35.87940216064453,
      "activations/layer11_attention_weight_max": 38.124351501464844,
      "activations/layer11_attention_weight_min": -35.18757247924805,
      "activations/layer12_attention_weight_max": 29.643932342529297,
      "activations/layer12_attention_weight_min": -29.06208038330078,
      "activations/layer13_attention_weight_max": 45.32217788696289,
      "activations/layer13_attention_weight_min": -37.57412338256836,
      "activations/layer14_attention_weight_max": 47.15478515625,
      "activations/layer14_attention_weight_min": -40.1677131652832,
      "activations/layer15_attention_weight_max": 42.45307922363281,
      "activations/layer15_attention_weight_min": -38.34465026855469,
      "activations/layer16_attention_weight_max": 29.76603889465332,
      "activations/layer16_attention_weight_min": -29.02610969543457,
      "activations/layer17_attention_weight_max": 32.746429443359375,
      "activations/layer17_attention_weight_min": -28.330244064331055,
      "activations/layer18_attention_weight_max": 34.346797943115234,
      "activations/layer18_attention_weight_min": -26.044158935546875,
      "activations/layer19_attention_weight_max": 34.681156158447266,
      "activations/layer19_attention_weight_min": -32.73637008666992,
      "activations/layer1_attention_weight_max": 16.969924926757812,
      "activations/layer1_attention_weight_min": -15.31798267364502,
      "activations/layer20_attention_weight_max": 30.262866973876953,
      "activations/layer20_attention_weight_min": -27.834714889526367,
      "activations/layer21_attention_weight_max": 30.185394287109375,
      "activations/layer21_attention_weight_min": -26.684682846069336,
      "activations/layer22_attention_weight_max": 45.40825271606445,
      "activations/layer22_attention_weight_min": -34.64433670043945,
      "activations/layer23_attention_weight_max": 33.83220291137695,
      "activations/layer23_attention_weight_min": -25.659561157226562,
      "activations/layer2_attention_weight_max": 32.24671936035156,
      "activations/layer2_attention_weight_min": -31.04438018798828,
      "activations/layer3_attention_weight_max": 99.59003448486328,
      "activations/layer3_attention_weight_min": -98.59060668945312,
      "activations/layer4_attention_weight_max": 106.4980697631836,
      "activations/layer4_attention_weight_min": -109.08086395263672,
      "activations/layer5_attention_weight_max": 50.80486297607422,
      "activations/layer5_attention_weight_min": -64.70491790771484,
      "activations/layer6_attention_weight_max": 45.96872329711914,
      "activations/layer6_attention_weight_min": -46.76774215698242,
      "activations/layer7_attention_weight_max": 96.10037994384766,
      "activations/layer7_attention_weight_min": -100.97595977783203,
      "activations/layer8_attention_weight_max": 42.936973571777344,
      "activations/layer8_attention_weight_min": -45.15270233154297,
      "activations/layer9_attention_weight_max": 33.009952545166016,
      "activations/layer9_attention_weight_min": -35.878299713134766,
      "epoch": 18.98,
      "learning_rate": 2.7882954545454542e-05,
      "loss": 2.7264,
      "step": 326650
    },
    {
      "activations/layer0_attention_weight_max": 17.266277313232422,
      "activations/layer0_attention_weight_min": -13.136573791503906,
      "activations/layer10_attention_weight_max": 34.14653396606445,
      "activations/layer10_attention_weight_min": -32.95185852050781,
      "activations/layer11_attention_weight_max": 35.04279708862305,
      "activations/layer11_attention_weight_min": -33.5190544128418,
      "activations/layer12_attention_weight_max": 31.503843307495117,
      "activations/layer12_attention_weight_min": -28.200794219970703,
      "activations/layer13_attention_weight_max": 46.48649978637695,
      "activations/layer13_attention_weight_min": -39.870540618896484,
      "activations/layer14_attention_weight_max": 51.46177291870117,
      "activations/layer14_attention_weight_min": -43.61777114868164,
      "activations/layer15_attention_weight_max": 49.01198959350586,
      "activations/layer15_attention_weight_min": -41.643348693847656,
      "activations/layer16_attention_weight_max": 31.233675003051758,
      "activations/layer16_attention_weight_min": -28.438434600830078,
      "activations/layer17_attention_weight_max": 31.392818450927734,
      "activations/layer17_attention_weight_min": -25.976287841796875,
      "activations/layer18_attention_weight_max": 31.83501625061035,
      "activations/layer18_attention_weight_min": -25.516069412231445,
      "activations/layer19_attention_weight_max": 36.359649658203125,
      "activations/layer19_attention_weight_min": -31.378904342651367,
      "activations/layer1_attention_weight_max": 16.20364761352539,
      "activations/layer1_attention_weight_min": -15.215807914733887,
      "activations/layer20_attention_weight_max": 30.672441482543945,
      "activations/layer20_attention_weight_min": -25.307289123535156,
      "activations/layer21_attention_weight_max": 29.82958221435547,
      "activations/layer21_attention_weight_min": -23.37225341796875,
      "activations/layer22_attention_weight_max": 41.591407775878906,
      "activations/layer22_attention_weight_min": -32.51057434082031,
      "activations/layer23_attention_weight_max": 32.30793762207031,
      "activations/layer23_attention_weight_min": -24.016448974609375,
      "activations/layer2_attention_weight_max": 33.077362060546875,
      "activations/layer2_attention_weight_min": -32.22174072265625,
      "activations/layer3_attention_weight_max": 103.62665557861328,
      "activations/layer3_attention_weight_min": -105.485595703125,
      "activations/layer4_attention_weight_max": 105.48514556884766,
      "activations/layer4_attention_weight_min": -110.60259246826172,
      "activations/layer5_attention_weight_max": 47.64522933959961,
      "activations/layer5_attention_weight_min": -61.74262237548828,
      "activations/layer6_attention_weight_max": 43.07209777832031,
      "activations/layer6_attention_weight_min": -46.454124450683594,
      "activations/layer7_attention_weight_max": 96.63131713867188,
      "activations/layer7_attention_weight_min": -94.09017181396484,
      "activations/layer8_attention_weight_max": 44.008766174316406,
      "activations/layer8_attention_weight_min": -43.784202575683594,
      "activations/layer9_attention_weight_max": 32.0627555847168,
      "activations/layer9_attention_weight_min": -34.071556091308594,
      "epoch": 18.98,
      "learning_rate": 2.786401515151515e-05,
      "loss": 2.722,
      "step": 326700
    },
    {
      "activations/layer0_attention_weight_max": 16.71887969970703,
      "activations/layer0_attention_weight_min": -12.311651229858398,
      "activations/layer10_attention_weight_max": 33.85834503173828,
      "activations/layer10_attention_weight_min": -34.39625549316406,
      "activations/layer11_attention_weight_max": 35.7548713684082,
      "activations/layer11_attention_weight_min": -34.84578323364258,
      "activations/layer12_attention_weight_max": 28.112768173217773,
      "activations/layer12_attention_weight_min": -28.123233795166016,
      "activations/layer13_attention_weight_max": 42.56699752807617,
      "activations/layer13_attention_weight_min": -37.39130401611328,
      "activations/layer14_attention_weight_max": 43.16305160522461,
      "activations/layer14_attention_weight_min": -42.37643814086914,
      "activations/layer15_attention_weight_max": 44.257484436035156,
      "activations/layer15_attention_weight_min": -35.521270751953125,
      "activations/layer16_attention_weight_max": 29.92539405822754,
      "activations/layer16_attention_weight_min": -29.699464797973633,
      "activations/layer17_attention_weight_max": 33.280120849609375,
      "activations/layer17_attention_weight_min": -27.51798439025879,
      "activations/layer18_attention_weight_max": 33.89377975463867,
      "activations/layer18_attention_weight_min": -28.041982650756836,
      "activations/layer19_attention_weight_max": 38.45012664794922,
      "activations/layer19_attention_weight_min": -33.714019775390625,
      "activations/layer1_attention_weight_max": 16.769132614135742,
      "activations/layer1_attention_weight_min": -15.25389289855957,
      "activations/layer20_attention_weight_max": 34.4833869934082,
      "activations/layer20_attention_weight_min": -25.760807037353516,
      "activations/layer21_attention_weight_max": 35.52724075317383,
      "activations/layer21_attention_weight_min": -25.763500213623047,
      "activations/layer22_attention_weight_max": 49.8734245300293,
      "activations/layer22_attention_weight_min": -32.49592590332031,
      "activations/layer23_attention_weight_max": 36.240264892578125,
      "activations/layer23_attention_weight_min": -25.71027946472168,
      "activations/layer2_attention_weight_max": 31.987396240234375,
      "activations/layer2_attention_weight_min": -30.700851440429688,
      "activations/layer3_attention_weight_max": 98.07823944091797,
      "activations/layer3_attention_weight_min": -100.65522003173828,
      "activations/layer4_attention_weight_max": 105.9576187133789,
      "activations/layer4_attention_weight_min": -109.16487121582031,
      "activations/layer5_attention_weight_max": 49.90743637084961,
      "activations/layer5_attention_weight_min": -66.69091796875,
      "activations/layer6_attention_weight_max": 44.865989685058594,
      "activations/layer6_attention_weight_min": -47.12632369995117,
      "activations/layer7_attention_weight_max": 92.85070037841797,
      "activations/layer7_attention_weight_min": -99.08231353759766,
      "activations/layer8_attention_weight_max": 41.854549407958984,
      "activations/layer8_attention_weight_min": -42.56753158569336,
      "activations/layer9_attention_weight_max": 31.818323135375977,
      "activations/layer9_attention_weight_min": -34.50989532470703,
      "epoch": 18.99,
      "learning_rate": 2.7845075757575755e-05,
      "loss": 2.7273,
      "step": 326750
    },
    {
      "activations/layer0_attention_weight_max": 17.162822723388672,
      "activations/layer0_attention_weight_min": -12.429826736450195,
      "activations/layer10_attention_weight_max": 36.776634216308594,
      "activations/layer10_attention_weight_min": -34.83618927001953,
      "activations/layer11_attention_weight_max": 36.65629196166992,
      "activations/layer11_attention_weight_min": -35.90617370605469,
      "activations/layer12_attention_weight_max": 28.76467514038086,
      "activations/layer12_attention_weight_min": -26.05415153503418,
      "activations/layer13_attention_weight_max": 43.70563888549805,
      "activations/layer13_attention_weight_min": -37.48424530029297,
      "activations/layer14_attention_weight_max": 43.76569747924805,
      "activations/layer14_attention_weight_min": -38.15450668334961,
      "activations/layer15_attention_weight_max": 37.75060272216797,
      "activations/layer15_attention_weight_min": -37.418697357177734,
      "activations/layer16_attention_weight_max": 29.097198486328125,
      "activations/layer16_attention_weight_min": -29.421518325805664,
      "activations/layer17_attention_weight_max": 28.52588653564453,
      "activations/layer17_attention_weight_min": -28.325960159301758,
      "activations/layer18_attention_weight_max": 31.357059478759766,
      "activations/layer18_attention_weight_min": -26.08306312561035,
      "activations/layer19_attention_weight_max": 34.3447265625,
      "activations/layer19_attention_weight_min": -34.20939636230469,
      "activations/layer1_attention_weight_max": 15.943388938903809,
      "activations/layer1_attention_weight_min": -15.867009162902832,
      "activations/layer20_attention_weight_max": 29.897581100463867,
      "activations/layer20_attention_weight_min": -27.24669647216797,
      "activations/layer21_attention_weight_max": 28.183523178100586,
      "activations/layer21_attention_weight_min": -27.668033599853516,
      "activations/layer22_attention_weight_max": 36.884151458740234,
      "activations/layer22_attention_weight_min": -33.060150146484375,
      "activations/layer23_attention_weight_max": 30.137557983398438,
      "activations/layer23_attention_weight_min": -26.606977462768555,
      "activations/layer2_attention_weight_max": 32.47025680541992,
      "activations/layer2_attention_weight_min": -30.84066390991211,
      "activations/layer3_attention_weight_max": 95.66149139404297,
      "activations/layer3_attention_weight_min": -98.75151062011719,
      "activations/layer4_attention_weight_max": 103.15065002441406,
      "activations/layer4_attention_weight_min": -106.8081283569336,
      "activations/layer5_attention_weight_max": 49.464012145996094,
      "activations/layer5_attention_weight_min": -59.2105712890625,
      "activations/layer6_attention_weight_max": 41.66803741455078,
      "activations/layer6_attention_weight_min": -44.6959114074707,
      "activations/layer7_attention_weight_max": 91.03501892089844,
      "activations/layer7_attention_weight_min": -99.07096099853516,
      "activations/layer8_attention_weight_max": 42.59821319580078,
      "activations/layer8_attention_weight_min": -44.86347198486328,
      "activations/layer9_attention_weight_max": 33.5026741027832,
      "activations/layer9_attention_weight_min": -34.64692306518555,
      "epoch": 18.99,
      "learning_rate": 2.782613636363636e-05,
      "loss": 2.7151,
      "step": 326800
    },
    {
      "activations/layer0_attention_weight_max": 15.96351432800293,
      "activations/layer0_attention_weight_min": -12.909213066101074,
      "activations/layer10_attention_weight_max": 38.14308547973633,
      "activations/layer10_attention_weight_min": -36.84561538696289,
      "activations/layer11_attention_weight_max": 39.334068298339844,
      "activations/layer11_attention_weight_min": -36.76390075683594,
      "activations/layer12_attention_weight_max": 27.908370971679688,
      "activations/layer12_attention_weight_min": -25.79046058654785,
      "activations/layer13_attention_weight_max": 41.91341781616211,
      "activations/layer13_attention_weight_min": -35.16157150268555,
      "activations/layer14_attention_weight_max": 40.96394729614258,
      "activations/layer14_attention_weight_min": -37.805091857910156,
      "activations/layer15_attention_weight_max": 40.04227066040039,
      "activations/layer15_attention_weight_min": -36.04582214355469,
      "activations/layer16_attention_weight_max": 31.095369338989258,
      "activations/layer16_attention_weight_min": -27.395944595336914,
      "activations/layer17_attention_weight_max": 28.960975646972656,
      "activations/layer17_attention_weight_min": -25.834810256958008,
      "activations/layer18_attention_weight_max": 30.037635803222656,
      "activations/layer18_attention_weight_min": -25.7474308013916,
      "activations/layer19_attention_weight_max": 33.185890197753906,
      "activations/layer19_attention_weight_min": -31.57687759399414,
      "activations/layer1_attention_weight_max": 15.971322059631348,
      "activations/layer1_attention_weight_min": -15.356651306152344,
      "activations/layer20_attention_weight_max": 27.901016235351562,
      "activations/layer20_attention_weight_min": -24.700162887573242,
      "activations/layer21_attention_weight_max": 27.528465270996094,
      "activations/layer21_attention_weight_min": -24.59511375427246,
      "activations/layer22_attention_weight_max": 39.3499870300293,
      "activations/layer22_attention_weight_min": -32.536827087402344,
      "activations/layer23_attention_weight_max": 37.10245132446289,
      "activations/layer23_attention_weight_min": -27.331130981445312,
      "activations/layer2_attention_weight_max": 32.0810432434082,
      "activations/layer2_attention_weight_min": -31.67617416381836,
      "activations/layer3_attention_weight_max": 98.8757553100586,
      "activations/layer3_attention_weight_min": -97.0785903930664,
      "activations/layer4_attention_weight_max": 107.94799041748047,
      "activations/layer4_attention_weight_min": -107.49337005615234,
      "activations/layer5_attention_weight_max": 48.101234436035156,
      "activations/layer5_attention_weight_min": -64.96011352539062,
      "activations/layer6_attention_weight_max": 44.10408401489258,
      "activations/layer6_attention_weight_min": -46.38176727294922,
      "activations/layer7_attention_weight_max": 94.71974182128906,
      "activations/layer7_attention_weight_min": -97.40277862548828,
      "activations/layer8_attention_weight_max": 42.69511032104492,
      "activations/layer8_attention_weight_min": -43.342891693115234,
      "activations/layer9_attention_weight_max": 33.85382080078125,
      "activations/layer9_attention_weight_min": -33.85416793823242,
      "epoch": 18.99,
      "learning_rate": 2.7807196969696965e-05,
      "loss": 2.7289,
      "step": 326850
    },
    {
      "activations/layer0_attention_weight_max": 16.045984268188477,
      "activations/layer0_attention_weight_min": -12.040675163269043,
      "activations/layer10_attention_weight_max": 38.55431365966797,
      "activations/layer10_attention_weight_min": -37.19267654418945,
      "activations/layer11_attention_weight_max": 38.83946990966797,
      "activations/layer11_attention_weight_min": -37.89398193359375,
      "activations/layer12_attention_weight_max": 26.521547317504883,
      "activations/layer12_attention_weight_min": -27.033388137817383,
      "activations/layer13_attention_weight_max": 38.114356994628906,
      "activations/layer13_attention_weight_min": -35.82632064819336,
      "activations/layer14_attention_weight_max": 40.02102279663086,
      "activations/layer14_attention_weight_min": -38.16682052612305,
      "activations/layer15_attention_weight_max": 37.391963958740234,
      "activations/layer15_attention_weight_min": -35.462738037109375,
      "activations/layer16_attention_weight_max": 28.75801658630371,
      "activations/layer16_attention_weight_min": -28.008413314819336,
      "activations/layer17_attention_weight_max": 29.055532455444336,
      "activations/layer17_attention_weight_min": -26.05718231201172,
      "activations/layer18_attention_weight_max": 30.66631507873535,
      "activations/layer18_attention_weight_min": -27.298654556274414,
      "activations/layer19_attention_weight_max": 34.087646484375,
      "activations/layer19_attention_weight_min": -32.74650192260742,
      "activations/layer1_attention_weight_max": 16.03654670715332,
      "activations/layer1_attention_weight_min": -14.448006629943848,
      "activations/layer20_attention_weight_max": 28.292722702026367,
      "activations/layer20_attention_weight_min": -26.401382446289062,
      "activations/layer21_attention_weight_max": 30.122140884399414,
      "activations/layer21_attention_weight_min": -27.945816040039062,
      "activations/layer22_attention_weight_max": 39.06415557861328,
      "activations/layer22_attention_weight_min": -33.43557357788086,
      "activations/layer23_attention_weight_max": 32.48666000366211,
      "activations/layer23_attention_weight_min": -27.072660446166992,
      "activations/layer2_attention_weight_max": 31.66610336303711,
      "activations/layer2_attention_weight_min": -31.694843292236328,
      "activations/layer3_attention_weight_max": 97.28559875488281,
      "activations/layer3_attention_weight_min": -100.73688507080078,
      "activations/layer4_attention_weight_max": 105.07868957519531,
      "activations/layer4_attention_weight_min": -109.04444885253906,
      "activations/layer5_attention_weight_max": 52.4212646484375,
      "activations/layer5_attention_weight_min": -60.83024597167969,
      "activations/layer6_attention_weight_max": 42.3661994934082,
      "activations/layer6_attention_weight_min": -46.8845329284668,
      "activations/layer7_attention_weight_max": 101.41714477539062,
      "activations/layer7_attention_weight_min": -96.17627716064453,
      "activations/layer8_attention_weight_max": 44.34123992919922,
      "activations/layer8_attention_weight_min": -44.57920455932617,
      "activations/layer9_attention_weight_max": 32.246917724609375,
      "activations/layer9_attention_weight_min": -36.139427185058594,
      "epoch": 18.99,
      "learning_rate": 2.7788257575757573e-05,
      "loss": 2.7099,
      "step": 326900
    },
    {
      "activations/layer0_attention_weight_max": 15.927721977233887,
      "activations/layer0_attention_weight_min": -12.361089706420898,
      "activations/layer10_attention_weight_max": 34.39604568481445,
      "activations/layer10_attention_weight_min": -35.769508361816406,
      "activations/layer11_attention_weight_max": 34.69503402709961,
      "activations/layer11_attention_weight_min": -34.2131233215332,
      "activations/layer12_attention_weight_max": 28.274187088012695,
      "activations/layer12_attention_weight_min": -28.19146156311035,
      "activations/layer13_attention_weight_max": 39.94071960449219,
      "activations/layer13_attention_weight_min": -37.16419982910156,
      "activations/layer14_attention_weight_max": 42.85544204711914,
      "activations/layer14_attention_weight_min": -38.28024673461914,
      "activations/layer15_attention_weight_max": 37.8271484375,
      "activations/layer15_attention_weight_min": -36.030033111572266,
      "activations/layer16_attention_weight_max": 30.833654403686523,
      "activations/layer16_attention_weight_min": -28.18569564819336,
      "activations/layer17_attention_weight_max": 29.0174560546875,
      "activations/layer17_attention_weight_min": -26.279516220092773,
      "activations/layer18_attention_weight_max": 29.16009521484375,
      "activations/layer18_attention_weight_min": -26.482553482055664,
      "activations/layer19_attention_weight_max": 32.3692741394043,
      "activations/layer19_attention_weight_min": -30.036590576171875,
      "activations/layer1_attention_weight_max": 15.754631996154785,
      "activations/layer1_attention_weight_min": -14.701805114746094,
      "activations/layer20_attention_weight_max": 29.087902069091797,
      "activations/layer20_attention_weight_min": -24.97052574157715,
      "activations/layer21_attention_weight_max": 27.74357032775879,
      "activations/layer21_attention_weight_min": -25.51970672607422,
      "activations/layer22_attention_weight_max": 38.3529052734375,
      "activations/layer22_attention_weight_min": -31.280879974365234,
      "activations/layer23_attention_weight_max": 31.003875732421875,
      "activations/layer23_attention_weight_min": -25.313932418823242,
      "activations/layer2_attention_weight_max": 32.43903350830078,
      "activations/layer2_attention_weight_min": -32.21839141845703,
      "activations/layer3_attention_weight_max": 99.6053695678711,
      "activations/layer3_attention_weight_min": -98.74523162841797,
      "activations/layer4_attention_weight_max": 107.00411224365234,
      "activations/layer4_attention_weight_min": -110.96443939208984,
      "activations/layer5_attention_weight_max": 50.57914733886719,
      "activations/layer5_attention_weight_min": -62.37464141845703,
      "activations/layer6_attention_weight_max": 44.622581481933594,
      "activations/layer6_attention_weight_min": -48.683311462402344,
      "activations/layer7_attention_weight_max": 97.84902954101562,
      "activations/layer7_attention_weight_min": -99.74566650390625,
      "activations/layer8_attention_weight_max": 44.53519821166992,
      "activations/layer8_attention_weight_min": -46.518890380859375,
      "activations/layer9_attention_weight_max": 34.5187873840332,
      "activations/layer9_attention_weight_min": -36.80624771118164,
      "epoch": 19.0,
      "learning_rate": 2.776931818181818e-05,
      "loss": 2.7226,
      "step": 326950
    },
    {
      "activations/layer0_attention_weight_max": 15.75150203704834,
      "activations/layer0_attention_weight_min": -13.213235855102539,
      "activations/layer10_attention_weight_max": 33.96905517578125,
      "activations/layer10_attention_weight_min": -34.20042037963867,
      "activations/layer11_attention_weight_max": 34.81463623046875,
      "activations/layer11_attention_weight_min": -34.256195068359375,
      "activations/layer12_attention_weight_max": 26.79613494873047,
      "activations/layer12_attention_weight_min": -27.30460548400879,
      "activations/layer13_attention_weight_max": 38.279476165771484,
      "activations/layer13_attention_weight_min": -34.004066467285156,
      "activations/layer14_attention_weight_max": 41.29753875732422,
      "activations/layer14_attention_weight_min": -35.835365295410156,
      "activations/layer15_attention_weight_max": 37.161659240722656,
      "activations/layer15_attention_weight_min": -33.83467483520508,
      "activations/layer16_attention_weight_max": 27.58966636657715,
      "activations/layer16_attention_weight_min": -30.69293975830078,
      "activations/layer17_attention_weight_max": 27.413854598999023,
      "activations/layer17_attention_weight_min": -25.64749526977539,
      "activations/layer18_attention_weight_max": 30.050148010253906,
      "activations/layer18_attention_weight_min": -25.740766525268555,
      "activations/layer19_attention_weight_max": 32.6956672668457,
      "activations/layer19_attention_weight_min": -33.304988861083984,
      "activations/layer1_attention_weight_max": 15.908574104309082,
      "activations/layer1_attention_weight_min": -15.389233589172363,
      "activations/layer20_attention_weight_max": 29.803239822387695,
      "activations/layer20_attention_weight_min": -26.112646102905273,
      "activations/layer21_attention_weight_max": 29.053991317749023,
      "activations/layer21_attention_weight_min": -25.655914306640625,
      "activations/layer22_attention_weight_max": 41.9224853515625,
      "activations/layer22_attention_weight_min": -31.230798721313477,
      "activations/layer23_attention_weight_max": 31.88524627685547,
      "activations/layer23_attention_weight_min": -26.054128646850586,
      "activations/layer2_attention_weight_max": 31.318973541259766,
      "activations/layer2_attention_weight_min": -31.187271118164062,
      "activations/layer3_attention_weight_max": 98.70149230957031,
      "activations/layer3_attention_weight_min": -98.3025894165039,
      "activations/layer4_attention_weight_max": 105.0567855834961,
      "activations/layer4_attention_weight_min": -105.58111572265625,
      "activations/layer5_attention_weight_max": 49.99702453613281,
      "activations/layer5_attention_weight_min": -58.3702507019043,
      "activations/layer6_attention_weight_max": 42.1353874206543,
      "activations/layer6_attention_weight_min": -46.10257339477539,
      "activations/layer7_attention_weight_max": 96.2435302734375,
      "activations/layer7_attention_weight_min": -94.56112670898438,
      "activations/layer8_attention_weight_max": 40.74837112426758,
      "activations/layer8_attention_weight_min": -41.80215835571289,
      "activations/layer9_attention_weight_max": 31.531522750854492,
      "activations/layer9_attention_weight_min": -34.24717330932617,
      "epoch": 19.0,
      "learning_rate": 2.7750378787878786e-05,
      "loss": 2.7711,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.1353,
      "eval_samples_per_second": 527.821,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.1353,
      "eval_openwebtext_samples_per_second": 527.821,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.95,
      "eval_wikitext_samples_per_second": 233.843,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.5442,
      "eval_lambada_samples_per_second": 510.151,
      "step": 327000
    },
    {
      "activations/layer0_attention_weight_max": 16.180187225341797,
      "activations/layer0_attention_weight_min": -13.210054397583008,
      "activations/layer10_attention_weight_max": 31.6339111328125,
      "activations/layer10_attention_weight_min": -32.24407958984375,
      "activations/layer11_attention_weight_max": 31.95301628112793,
      "activations/layer11_attention_weight_min": -34.090797424316406,
      "activations/layer12_attention_weight_max": 25.458637237548828,
      "activations/layer12_attention_weight_min": -27.611648559570312,
      "activations/layer13_attention_weight_max": 36.971397399902344,
      "activations/layer13_attention_weight_min": -36.37890625,
      "activations/layer14_attention_weight_max": 39.253684997558594,
      "activations/layer14_attention_weight_min": -40.15159225463867,
      "activations/layer15_attention_weight_max": 35.62888717651367,
      "activations/layer15_attention_weight_min": -37.513938903808594,
      "activations/layer16_attention_weight_max": 30.03969383239746,
      "activations/layer16_attention_weight_min": -29.222700119018555,
      "activations/layer17_attention_weight_max": 28.390830993652344,
      "activations/layer17_attention_weight_min": -25.329980850219727,
      "activations/layer18_attention_weight_max": 28.9593563079834,
      "activations/layer18_attention_weight_min": -26.552753448486328,
      "activations/layer19_attention_weight_max": 30.681507110595703,
      "activations/layer19_attention_weight_min": -31.75977897644043,
      "activations/layer1_attention_weight_max": 14.705595970153809,
      "activations/layer1_attention_weight_min": -14.386066436767578,
      "activations/layer20_attention_weight_max": 26.56622314453125,
      "activations/layer20_attention_weight_min": -24.936500549316406,
      "activations/layer21_attention_weight_max": 25.005746841430664,
      "activations/layer21_attention_weight_min": -25.225444793701172,
      "activations/layer22_attention_weight_max": 38.90806198120117,
      "activations/layer22_attention_weight_min": -34.420440673828125,
      "activations/layer23_attention_weight_max": 29.729961395263672,
      "activations/layer23_attention_weight_min": -28.316463470458984,
      "activations/layer2_attention_weight_max": 31.062698364257812,
      "activations/layer2_attention_weight_min": -29.865079879760742,
      "activations/layer3_attention_weight_max": 102.5208969116211,
      "activations/layer3_attention_weight_min": -98.32901000976562,
      "activations/layer4_attention_weight_max": 104.02320861816406,
      "activations/layer4_attention_weight_min": -105.0311508178711,
      "activations/layer5_attention_weight_max": 49.47454071044922,
      "activations/layer5_attention_weight_min": -63.62311553955078,
      "activations/layer6_attention_weight_max": 43.07112503051758,
      "activations/layer6_attention_weight_min": -46.357181549072266,
      "activations/layer7_attention_weight_max": 93.44058990478516,
      "activations/layer7_attention_weight_min": -94.01484680175781,
      "activations/layer8_attention_weight_max": 40.81697082519531,
      "activations/layer8_attention_weight_min": -41.89418029785156,
      "activations/layer9_attention_weight_max": 30.847431182861328,
      "activations/layer9_attention_weight_min": -33.49750518798828,
      "epoch": 19.0,
      "learning_rate": 2.7731439393939394e-05,
      "loss": 2.7314,
      "step": 327050
    },
    {
      "activations/layer0_attention_weight_max": 16.365070343017578,
      "activations/layer0_attention_weight_min": -12.110313415527344,
      "activations/layer10_attention_weight_max": 34.04153823852539,
      "activations/layer10_attention_weight_min": -37.200504302978516,
      "activations/layer11_attention_weight_max": 34.625579833984375,
      "activations/layer11_attention_weight_min": -38.55238723754883,
      "activations/layer12_attention_weight_max": 25.97455406188965,
      "activations/layer12_attention_weight_min": -26.440902709960938,
      "activations/layer13_attention_weight_max": 38.02836608886719,
      "activations/layer13_attention_weight_min": -35.41006088256836,
      "activations/layer14_attention_weight_max": 41.5757942199707,
      "activations/layer14_attention_weight_min": -40.316463470458984,
      "activations/layer15_attention_weight_max": 38.57072830200195,
      "activations/layer15_attention_weight_min": -37.80868148803711,
      "activations/layer16_attention_weight_max": 29.8255615234375,
      "activations/layer16_attention_weight_min": -29.682477951049805,
      "activations/layer17_attention_weight_max": 27.940235137939453,
      "activations/layer17_attention_weight_min": -27.732709884643555,
      "activations/layer18_attention_weight_max": 29.36016273498535,
      "activations/layer18_attention_weight_min": -27.315946578979492,
      "activations/layer19_attention_weight_max": 33.150901794433594,
      "activations/layer19_attention_weight_min": -33.05999755859375,
      "activations/layer1_attention_weight_max": 16.867523193359375,
      "activations/layer1_attention_weight_min": -14.697364807128906,
      "activations/layer20_attention_weight_max": 26.523847579956055,
      "activations/layer20_attention_weight_min": -26.620309829711914,
      "activations/layer21_attention_weight_max": 27.779056549072266,
      "activations/layer21_attention_weight_min": -28.380868911743164,
      "activations/layer22_attention_weight_max": 41.620059967041016,
      "activations/layer22_attention_weight_min": -42.17673873901367,
      "activations/layer23_attention_weight_max": 32.35772705078125,
      "activations/layer23_attention_weight_min": -33.80181884765625,
      "activations/layer2_attention_weight_max": 31.252613067626953,
      "activations/layer2_attention_weight_min": -31.24372100830078,
      "activations/layer3_attention_weight_max": 96.16361999511719,
      "activations/layer3_attention_weight_min": -98.78244018554688,
      "activations/layer4_attention_weight_max": 106.06224822998047,
      "activations/layer4_attention_weight_min": -109.78736877441406,
      "activations/layer5_attention_weight_max": 49.77886199951172,
      "activations/layer5_attention_weight_min": -61.146095275878906,
      "activations/layer6_attention_weight_max": 43.98651885986328,
      "activations/layer6_attention_weight_min": -47.78581619262695,
      "activations/layer7_attention_weight_max": 91.13925170898438,
      "activations/layer7_attention_weight_min": -103.03014373779297,
      "activations/layer8_attention_weight_max": 42.26979446411133,
      "activations/layer8_attention_weight_min": -45.053504943847656,
      "activations/layer9_attention_weight_max": 31.60309410095215,
      "activations/layer9_attention_weight_min": -36.36472702026367,
      "epoch": 19.01,
      "learning_rate": 2.7712499999999996e-05,
      "loss": 2.7133,
      "step": 327100
    },
    {
      "activations/layer0_attention_weight_max": 16.11915397644043,
      "activations/layer0_attention_weight_min": -12.039617538452148,
      "activations/layer10_attention_weight_max": 32.390254974365234,
      "activations/layer10_attention_weight_min": -33.35970687866211,
      "activations/layer11_attention_weight_max": 32.958770751953125,
      "activations/layer11_attention_weight_min": -34.22480010986328,
      "activations/layer12_attention_weight_max": 26.774885177612305,
      "activations/layer12_attention_weight_min": -26.235231399536133,
      "activations/layer13_attention_weight_max": 40.21513748168945,
      "activations/layer13_attention_weight_min": -36.12520217895508,
      "activations/layer14_attention_weight_max": 46.48724365234375,
      "activations/layer14_attention_weight_min": -37.384403228759766,
      "activations/layer15_attention_weight_max": 44.75416564941406,
      "activations/layer15_attention_weight_min": -35.085105895996094,
      "activations/layer16_attention_weight_max": 33.324954986572266,
      "activations/layer16_attention_weight_min": -28.71591567993164,
      "activations/layer17_attention_weight_max": 30.24502944946289,
      "activations/layer17_attention_weight_min": -26.48685073852539,
      "activations/layer18_attention_weight_max": 34.706573486328125,
      "activations/layer18_attention_weight_min": -27.024662017822266,
      "activations/layer19_attention_weight_max": 37.0102653503418,
      "activations/layer19_attention_weight_min": -33.015769958496094,
      "activations/layer1_attention_weight_max": 16.427837371826172,
      "activations/layer1_attention_weight_min": -14.486194610595703,
      "activations/layer20_attention_weight_max": 34.47257614135742,
      "activations/layer20_attention_weight_min": -25.37362289428711,
      "activations/layer21_attention_weight_max": 33.65816116333008,
      "activations/layer21_attention_weight_min": -23.830955505371094,
      "activations/layer22_attention_weight_max": 46.437129974365234,
      "activations/layer22_attention_weight_min": -31.00508689880371,
      "activations/layer23_attention_weight_max": 37.18241882324219,
      "activations/layer23_attention_weight_min": -26.934843063354492,
      "activations/layer2_attention_weight_max": 31.269529342651367,
      "activations/layer2_attention_weight_min": -32.22932434082031,
      "activations/layer3_attention_weight_max": 98.80895233154297,
      "activations/layer3_attention_weight_min": -97.12055969238281,
      "activations/layer4_attention_weight_max": 102.84307861328125,
      "activations/layer4_attention_weight_min": -105.23091125488281,
      "activations/layer5_attention_weight_max": 47.7730712890625,
      "activations/layer5_attention_weight_min": -59.99687957763672,
      "activations/layer6_attention_weight_max": 42.59831237792969,
      "activations/layer6_attention_weight_min": -46.70872497558594,
      "activations/layer7_attention_weight_max": 93.69976806640625,
      "activations/layer7_attention_weight_min": -95.7494125366211,
      "activations/layer8_attention_weight_max": 40.88714599609375,
      "activations/layer8_attention_weight_min": -43.89821243286133,
      "activations/layer9_attention_weight_max": 31.4757137298584,
      "activations/layer9_attention_weight_min": -33.16950607299805,
      "epoch": 19.01,
      "learning_rate": 2.7693560606060604e-05,
      "loss": 2.7072,
      "step": 327150
    },
    {
      "activations/layer0_attention_weight_max": 17.179330825805664,
      "activations/layer0_attention_weight_min": -12.554868698120117,
      "activations/layer10_attention_weight_max": 35.010955810546875,
      "activations/layer10_attention_weight_min": -37.24600601196289,
      "activations/layer11_attention_weight_max": 35.837318420410156,
      "activations/layer11_attention_weight_min": -37.24126434326172,
      "activations/layer12_attention_weight_max": 29.138172149658203,
      "activations/layer12_attention_weight_min": -28.272418975830078,
      "activations/layer13_attention_weight_max": 44.65825271606445,
      "activations/layer13_attention_weight_min": -37.03330612182617,
      "activations/layer14_attention_weight_max": 46.58146286010742,
      "activations/layer14_attention_weight_min": -39.32099533081055,
      "activations/layer15_attention_weight_max": 43.658302307128906,
      "activations/layer15_attention_weight_min": -38.80354690551758,
      "activations/layer16_attention_weight_max": 33.88437271118164,
      "activations/layer16_attention_weight_min": -29.507490158081055,
      "activations/layer17_attention_weight_max": 31.45382308959961,
      "activations/layer17_attention_weight_min": -26.66225814819336,
      "activations/layer18_attention_weight_max": 37.78180694580078,
      "activations/layer18_attention_weight_min": -27.330123901367188,
      "activations/layer19_attention_weight_max": 37.7082633972168,
      "activations/layer19_attention_weight_min": -32.38040542602539,
      "activations/layer1_attention_weight_max": 16.06621551513672,
      "activations/layer1_attention_weight_min": -15.147729873657227,
      "activations/layer20_attention_weight_max": 29.488508224487305,
      "activations/layer20_attention_weight_min": -24.39717674255371,
      "activations/layer21_attention_weight_max": 30.190147399902344,
      "activations/layer21_attention_weight_min": -24.375144958496094,
      "activations/layer22_attention_weight_max": 48.01856994628906,
      "activations/layer22_attention_weight_min": -33.252952575683594,
      "activations/layer23_attention_weight_max": 35.736366271972656,
      "activations/layer23_attention_weight_min": -26.166976928710938,
      "activations/layer2_attention_weight_max": 31.469799041748047,
      "activations/layer2_attention_weight_min": -31.84418296813965,
      "activations/layer3_attention_weight_max": 99.82980346679688,
      "activations/layer3_attention_weight_min": -103.92982482910156,
      "activations/layer4_attention_weight_max": 106.4136962890625,
      "activations/layer4_attention_weight_min": -111.2078857421875,
      "activations/layer5_attention_weight_max": 50.420589447021484,
      "activations/layer5_attention_weight_min": -59.804443359375,
      "activations/layer6_attention_weight_max": 44.00895309448242,
      "activations/layer6_attention_weight_min": -46.33806610107422,
      "activations/layer7_attention_weight_max": 93.17890167236328,
      "activations/layer7_attention_weight_min": -95.5866928100586,
      "activations/layer8_attention_weight_max": 42.72162628173828,
      "activations/layer8_attention_weight_min": -46.58749008178711,
      "activations/layer9_attention_weight_max": 33.728153228759766,
      "activations/layer9_attention_weight_min": -36.06173324584961,
      "epoch": 19.01,
      "learning_rate": 2.767462121212121e-05,
      "loss": 2.7033,
      "step": 327200
    },
    {
      "activations/layer0_attention_weight_max": 16.189388275146484,
      "activations/layer0_attention_weight_min": -12.464616775512695,
      "activations/layer10_attention_weight_max": 37.3133544921875,
      "activations/layer10_attention_weight_min": -38.294830322265625,
      "activations/layer11_attention_weight_max": 35.799522399902344,
      "activations/layer11_attention_weight_min": -36.48634338378906,
      "activations/layer12_attention_weight_max": 27.486061096191406,
      "activations/layer12_attention_weight_min": -29.788978576660156,
      "activations/layer13_attention_weight_max": 43.01137161254883,
      "activations/layer13_attention_weight_min": -37.6939582824707,
      "activations/layer14_attention_weight_max": 45.63473892211914,
      "activations/layer14_attention_weight_min": -40.39738082885742,
      "activations/layer15_attention_weight_max": 44.43659210205078,
      "activations/layer15_attention_weight_min": -36.565032958984375,
      "activations/layer16_attention_weight_max": 32.88678741455078,
      "activations/layer16_attention_weight_min": -29.87429428100586,
      "activations/layer17_attention_weight_max": 27.97685432434082,
      "activations/layer17_attention_weight_min": -26.369266510009766,
      "activations/layer18_attention_weight_max": 31.545066833496094,
      "activations/layer18_attention_weight_min": -25.906394958496094,
      "activations/layer19_attention_weight_max": 31.790241241455078,
      "activations/layer19_attention_weight_min": -33.3023796081543,
      "activations/layer1_attention_weight_max": 15.235966682434082,
      "activations/layer1_attention_weight_min": -14.329168319702148,
      "activations/layer20_attention_weight_max": 26.576038360595703,
      "activations/layer20_attention_weight_min": -25.758129119873047,
      "activations/layer21_attention_weight_max": 27.919607162475586,
      "activations/layer21_attention_weight_min": -25.741580963134766,
      "activations/layer22_attention_weight_max": 39.58035659790039,
      "activations/layer22_attention_weight_min": -32.02568435668945,
      "activations/layer23_attention_weight_max": 31.86481285095215,
      "activations/layer23_attention_weight_min": -24.579069137573242,
      "activations/layer2_attention_weight_max": 30.560489654541016,
      "activations/layer2_attention_weight_min": -31.51434326171875,
      "activations/layer3_attention_weight_max": 101.18508911132812,
      "activations/layer3_attention_weight_min": -101.52924346923828,
      "activations/layer4_attention_weight_max": 108.83361053466797,
      "activations/layer4_attention_weight_min": -113.78474426269531,
      "activations/layer5_attention_weight_max": 50.41240692138672,
      "activations/layer5_attention_weight_min": -61.92249298095703,
      "activations/layer6_attention_weight_max": 44.27739715576172,
      "activations/layer6_attention_weight_min": -47.1965446472168,
      "activations/layer7_attention_weight_max": 94.74537658691406,
      "activations/layer7_attention_weight_min": -98.96995544433594,
      "activations/layer8_attention_weight_max": 46.7944221496582,
      "activations/layer8_attention_weight_min": -44.58974075317383,
      "activations/layer9_attention_weight_max": 35.14920425415039,
      "activations/layer9_attention_weight_min": -37.13255310058594,
      "epoch": 19.02,
      "learning_rate": 2.7655681818181817e-05,
      "loss": 2.7162,
      "step": 327250
    },
    {
      "activations/layer0_attention_weight_max": 16.022300720214844,
      "activations/layer0_attention_weight_min": -13.80903148651123,
      "activations/layer10_attention_weight_max": 35.96112060546875,
      "activations/layer10_attention_weight_min": -40.51409912109375,
      "activations/layer11_attention_weight_max": 34.530517578125,
      "activations/layer11_attention_weight_min": -36.07501983642578,
      "activations/layer12_attention_weight_max": 29.239429473876953,
      "activations/layer12_attention_weight_min": -29.0929012298584,
      "activations/layer13_attention_weight_max": 42.51971435546875,
      "activations/layer13_attention_weight_min": -37.83045196533203,
      "activations/layer14_attention_weight_max": 44.8277473449707,
      "activations/layer14_attention_weight_min": -39.21058654785156,
      "activations/layer15_attention_weight_max": 43.56128692626953,
      "activations/layer15_attention_weight_min": -35.89838409423828,
      "activations/layer16_attention_weight_max": 30.427892684936523,
      "activations/layer16_attention_weight_min": -28.482568740844727,
      "activations/layer17_attention_weight_max": 31.062150955200195,
      "activations/layer17_attention_weight_min": -26.74831199645996,
      "activations/layer18_attention_weight_max": 33.259239196777344,
      "activations/layer18_attention_weight_min": -26.487247467041016,
      "activations/layer19_attention_weight_max": 36.69196701049805,
      "activations/layer19_attention_weight_min": -32.563472747802734,
      "activations/layer1_attention_weight_max": 16.63022804260254,
      "activations/layer1_attention_weight_min": -15.593475341796875,
      "activations/layer20_attention_weight_max": 32.53122329711914,
      "activations/layer20_attention_weight_min": -27.129581451416016,
      "activations/layer21_attention_weight_max": 31.09183120727539,
      "activations/layer21_attention_weight_min": -25.858060836791992,
      "activations/layer22_attention_weight_max": 51.330101013183594,
      "activations/layer22_attention_weight_min": -34.56824493408203,
      "activations/layer23_attention_weight_max": 35.25928497314453,
      "activations/layer23_attention_weight_min": -27.975446701049805,
      "activations/layer2_attention_weight_max": 33.311065673828125,
      "activations/layer2_attention_weight_min": -31.239849090576172,
      "activations/layer3_attention_weight_max": 99.98561096191406,
      "activations/layer3_attention_weight_min": -99.69380950927734,
      "activations/layer4_attention_weight_max": 109.48314666748047,
      "activations/layer4_attention_weight_min": -109.17137908935547,
      "activations/layer5_attention_weight_max": 51.43508529663086,
      "activations/layer5_attention_weight_min": -63.48192596435547,
      "activations/layer6_attention_weight_max": 46.65543746948242,
      "activations/layer6_attention_weight_min": -48.458133697509766,
      "activations/layer7_attention_weight_max": 100.62507629394531,
      "activations/layer7_attention_weight_min": -99.81626892089844,
      "activations/layer8_attention_weight_max": 44.689456939697266,
      "activations/layer8_attention_weight_min": -46.22037887573242,
      "activations/layer9_attention_weight_max": 33.699954986572266,
      "activations/layer9_attention_weight_min": -36.594844818115234,
      "epoch": 19.02,
      "learning_rate": 2.7636742424242422e-05,
      "loss": 2.718,
      "step": 327300
    },
    {
      "activations/layer0_attention_weight_max": 16.464252471923828,
      "activations/layer0_attention_weight_min": -14.691508293151855,
      "activations/layer10_attention_weight_max": 34.503578186035156,
      "activations/layer10_attention_weight_min": -34.50669479370117,
      "activations/layer11_attention_weight_max": 35.60005187988281,
      "activations/layer11_attention_weight_min": -34.444618225097656,
      "activations/layer12_attention_weight_max": 27.206527709960938,
      "activations/layer12_attention_weight_min": -26.784765243530273,
      "activations/layer13_attention_weight_max": 36.49613952636719,
      "activations/layer13_attention_weight_min": -35.451416015625,
      "activations/layer14_attention_weight_max": 45.198997497558594,
      "activations/layer14_attention_weight_min": -40.14921569824219,
      "activations/layer15_attention_weight_max": 38.939815521240234,
      "activations/layer15_attention_weight_min": -37.081993103027344,
      "activations/layer16_attention_weight_max": 29.631376266479492,
      "activations/layer16_attention_weight_min": -27.967370986938477,
      "activations/layer17_attention_weight_max": 29.66465187072754,
      "activations/layer17_attention_weight_min": -26.876258850097656,
      "activations/layer18_attention_weight_max": 31.610868453979492,
      "activations/layer18_attention_weight_min": -25.78400993347168,
      "activations/layer19_attention_weight_max": 35.89829635620117,
      "activations/layer19_attention_weight_min": -32.71751022338867,
      "activations/layer1_attention_weight_max": 16.092342376708984,
      "activations/layer1_attention_weight_min": -14.168234825134277,
      "activations/layer20_attention_weight_max": 28.313629150390625,
      "activations/layer20_attention_weight_min": -27.347734451293945,
      "activations/layer21_attention_weight_max": 26.951847076416016,
      "activations/layer21_attention_weight_min": -26.805206298828125,
      "activations/layer22_attention_weight_max": 45.50592041015625,
      "activations/layer22_attention_weight_min": -32.4737548828125,
      "activations/layer23_attention_weight_max": 30.77627182006836,
      "activations/layer23_attention_weight_min": -27.178449630737305,
      "activations/layer2_attention_weight_max": 31.251487731933594,
      "activations/layer2_attention_weight_min": -30.605453491210938,
      "activations/layer3_attention_weight_max": 98.45166015625,
      "activations/layer3_attention_weight_min": -100.6852035522461,
      "activations/layer4_attention_weight_max": 105.78372955322266,
      "activations/layer4_attention_weight_min": -110.96004486083984,
      "activations/layer5_attention_weight_max": 50.37624740600586,
      "activations/layer5_attention_weight_min": -64.78944396972656,
      "activations/layer6_attention_weight_max": 44.87804412841797,
      "activations/layer6_attention_weight_min": -48.46818923950195,
      "activations/layer7_attention_weight_max": 93.05760955810547,
      "activations/layer7_attention_weight_min": -100.03865814208984,
      "activations/layer8_attention_weight_max": 43.40010070800781,
      "activations/layer8_attention_weight_min": -45.81201171875,
      "activations/layer9_attention_weight_max": 32.28761291503906,
      "activations/layer9_attention_weight_min": -34.83174514770508,
      "epoch": 19.02,
      "learning_rate": 2.761780303030303e-05,
      "loss": 2.7211,
      "step": 327350
    },
    {
      "activations/layer0_attention_weight_max": 16.35287094116211,
      "activations/layer0_attention_weight_min": -14.274515151977539,
      "activations/layer10_attention_weight_max": 33.81299591064453,
      "activations/layer10_attention_weight_min": -35.494564056396484,
      "activations/layer11_attention_weight_max": 33.566497802734375,
      "activations/layer11_attention_weight_min": -33.82190704345703,
      "activations/layer12_attention_weight_max": 26.202476501464844,
      "activations/layer12_attention_weight_min": -27.6622257232666,
      "activations/layer13_attention_weight_max": 38.17034149169922,
      "activations/layer13_attention_weight_min": -35.99662399291992,
      "activations/layer14_attention_weight_max": 40.10316467285156,
      "activations/layer14_attention_weight_min": -37.354209899902344,
      "activations/layer15_attention_weight_max": 35.74563217163086,
      "activations/layer15_attention_weight_min": -35.86367416381836,
      "activations/layer16_attention_weight_max": 26.95859146118164,
      "activations/layer16_attention_weight_min": -28.25929832458496,
      "activations/layer17_attention_weight_max": 29.18164825439453,
      "activations/layer17_attention_weight_min": -24.26251220703125,
      "activations/layer18_attention_weight_max": 34.298213958740234,
      "activations/layer18_attention_weight_min": -26.424333572387695,
      "activations/layer19_attention_weight_max": 31.10548973083496,
      "activations/layer19_attention_weight_min": -31.744598388671875,
      "activations/layer1_attention_weight_max": 16.20064926147461,
      "activations/layer1_attention_weight_min": -14.326093673706055,
      "activations/layer20_attention_weight_max": 26.374181747436523,
      "activations/layer20_attention_weight_min": -24.223230361938477,
      "activations/layer21_attention_weight_max": 26.867446899414062,
      "activations/layer21_attention_weight_min": -23.44138526916504,
      "activations/layer22_attention_weight_max": 38.36567687988281,
      "activations/layer22_attention_weight_min": -32.82571792602539,
      "activations/layer23_attention_weight_max": 31.755897521972656,
      "activations/layer23_attention_weight_min": -25.68035125732422,
      "activations/layer2_attention_weight_max": 32.43581008911133,
      "activations/layer2_attention_weight_min": -30.47967529296875,
      "activations/layer3_attention_weight_max": 101.88496398925781,
      "activations/layer3_attention_weight_min": -100.63516235351562,
      "activations/layer4_attention_weight_max": 110.55321502685547,
      "activations/layer4_attention_weight_min": -114.60597229003906,
      "activations/layer5_attention_weight_max": 54.94089889526367,
      "activations/layer5_attention_weight_min": -63.35005187988281,
      "activations/layer6_attention_weight_max": 45.87984848022461,
      "activations/layer6_attention_weight_min": -51.57219314575195,
      "activations/layer7_attention_weight_max": 96.59391784667969,
      "activations/layer7_attention_weight_min": -101.50226593017578,
      "activations/layer8_attention_weight_max": 44.56218338012695,
      "activations/layer8_attention_weight_min": -45.55344009399414,
      "activations/layer9_attention_weight_max": 32.366458892822266,
      "activations/layer9_attention_weight_min": -35.94524002075195,
      "epoch": 19.02,
      "learning_rate": 2.759924242424242e-05,
      "loss": 2.7231,
      "step": 327400
    },
    {
      "activations/layer0_attention_weight_max": 15.903058052062988,
      "activations/layer0_attention_weight_min": -13.514718055725098,
      "activations/layer10_attention_weight_max": 44.7412109375,
      "activations/layer10_attention_weight_min": -38.704368591308594,
      "activations/layer11_attention_weight_max": 41.00456237792969,
      "activations/layer11_attention_weight_min": -38.29055404663086,
      "activations/layer12_attention_weight_max": 33.053855895996094,
      "activations/layer12_attention_weight_min": -28.518394470214844,
      "activations/layer13_attention_weight_max": 47.97645950317383,
      "activations/layer13_attention_weight_min": -37.844261169433594,
      "activations/layer14_attention_weight_max": 52.33018493652344,
      "activations/layer14_attention_weight_min": -43.45348358154297,
      "activations/layer15_attention_weight_max": 49.12955856323242,
      "activations/layer15_attention_weight_min": -42.25605010986328,
      "activations/layer16_attention_weight_max": 39.05668258666992,
      "activations/layer16_attention_weight_min": -28.3944034576416,
      "activations/layer17_attention_weight_max": 33.99538803100586,
      "activations/layer17_attention_weight_min": -30.494646072387695,
      "activations/layer18_attention_weight_max": 32.90147018432617,
      "activations/layer18_attention_weight_min": -28.239660263061523,
      "activations/layer19_attention_weight_max": 35.296260833740234,
      "activations/layer19_attention_weight_min": -31.682300567626953,
      "activations/layer1_attention_weight_max": 15.20198917388916,
      "activations/layer1_attention_weight_min": -14.567323684692383,
      "activations/layer20_attention_weight_max": 29.04632568359375,
      "activations/layer20_attention_weight_min": -27.063018798828125,
      "activations/layer21_attention_weight_max": 28.59800148010254,
      "activations/layer21_attention_weight_min": -28.962846755981445,
      "activations/layer22_attention_weight_max": 41.27503967285156,
      "activations/layer22_attention_weight_min": -33.102046966552734,
      "activations/layer23_attention_weight_max": 32.16961669921875,
      "activations/layer23_attention_weight_min": -25.614233016967773,
      "activations/layer2_attention_weight_max": 32.090641021728516,
      "activations/layer2_attention_weight_min": -31.697860717773438,
      "activations/layer3_attention_weight_max": 102.34984588623047,
      "activations/layer3_attention_weight_min": -101.82662200927734,
      "activations/layer4_attention_weight_max": 105.44905853271484,
      "activations/layer4_attention_weight_min": -108.56587982177734,
      "activations/layer5_attention_weight_max": 52.980552673339844,
      "activations/layer5_attention_weight_min": -60.45152282714844,
      "activations/layer6_attention_weight_max": 46.179359436035156,
      "activations/layer6_attention_weight_min": -46.94578170776367,
      "activations/layer7_attention_weight_max": 106.1993408203125,
      "activations/layer7_attention_weight_min": -99.30361938476562,
      "activations/layer8_attention_weight_max": 45.796146392822266,
      "activations/layer8_attention_weight_min": -46.914306640625,
      "activations/layer9_attention_weight_max": 37.47101593017578,
      "activations/layer9_attention_weight_min": -36.65061950683594,
      "epoch": 19.03,
      "learning_rate": 2.758030303030303e-05,
      "loss": 2.7099,
      "step": 327450
    },
    {
      "activations/layer0_attention_weight_max": 16.22694969177246,
      "activations/layer0_attention_weight_min": -12.910889625549316,
      "activations/layer10_attention_weight_max": 43.8417854309082,
      "activations/layer10_attention_weight_min": -36.860328674316406,
      "activations/layer11_attention_weight_max": 42.922576904296875,
      "activations/layer11_attention_weight_min": -37.675228118896484,
      "activations/layer12_attention_weight_max": 35.4393196105957,
      "activations/layer12_attention_weight_min": -26.62214469909668,
      "activations/layer13_attention_weight_max": 43.941429138183594,
      "activations/layer13_attention_weight_min": -33.6505241394043,
      "activations/layer14_attention_weight_max": 42.64107131958008,
      "activations/layer14_attention_weight_min": -37.88286209106445,
      "activations/layer15_attention_weight_max": 48.91642761230469,
      "activations/layer15_attention_weight_min": -39.873905181884766,
      "activations/layer16_attention_weight_max": 32.62915802001953,
      "activations/layer16_attention_weight_min": -27.62645149230957,
      "activations/layer17_attention_weight_max": 29.83922004699707,
      "activations/layer17_attention_weight_min": -28.30555534362793,
      "activations/layer18_attention_weight_max": 31.557809829711914,
      "activations/layer18_attention_weight_min": -25.02146339416504,
      "activations/layer19_attention_weight_max": 31.02676773071289,
      "activations/layer19_attention_weight_min": -29.888751983642578,
      "activations/layer1_attention_weight_max": 15.89724349975586,
      "activations/layer1_attention_weight_min": -14.732885360717773,
      "activations/layer20_attention_weight_max": 29.6755428314209,
      "activations/layer20_attention_weight_min": -25.03118133544922,
      "activations/layer21_attention_weight_max": 31.956005096435547,
      "activations/layer21_attention_weight_min": -27.21512222290039,
      "activations/layer22_attention_weight_max": 39.95355224609375,
      "activations/layer22_attention_weight_min": -30.098785400390625,
      "activations/layer23_attention_weight_max": 29.993017196655273,
      "activations/layer23_attention_weight_min": -25.68488311767578,
      "activations/layer2_attention_weight_max": 33.57040023803711,
      "activations/layer2_attention_weight_min": -33.59046173095703,
      "activations/layer3_attention_weight_max": 105.81118774414062,
      "activations/layer3_attention_weight_min": -103.16390228271484,
      "activations/layer4_attention_weight_max": 112.63551330566406,
      "activations/layer4_attention_weight_min": -105.6377182006836,
      "activations/layer5_attention_weight_max": 56.062965393066406,
      "activations/layer5_attention_weight_min": -59.89153289794922,
      "activations/layer6_attention_weight_max": 45.17357635498047,
      "activations/layer6_attention_weight_min": -46.38973617553711,
      "activations/layer7_attention_weight_max": 108.60508728027344,
      "activations/layer7_attention_weight_min": -102.06304931640625,
      "activations/layer8_attention_weight_max": 48.820831298828125,
      "activations/layer8_attention_weight_min": -48.668731689453125,
      "activations/layer9_attention_weight_max": 39.18940734863281,
      "activations/layer9_attention_weight_min": -37.09343338012695,
      "epoch": 19.03,
      "learning_rate": 2.756136363636363e-05,
      "loss": 2.6968,
      "step": 327500
    },
    {
      "activations/layer0_attention_weight_max": 16.047414779663086,
      "activations/layer0_attention_weight_min": -13.248137474060059,
      "activations/layer10_attention_weight_max": 34.26398468017578,
      "activations/layer10_attention_weight_min": -33.5509147644043,
      "activations/layer11_attention_weight_max": 32.88580322265625,
      "activations/layer11_attention_weight_min": -34.551509857177734,
      "activations/layer12_attention_weight_max": 24.991857528686523,
      "activations/layer12_attention_weight_min": -26.618860244750977,
      "activations/layer13_attention_weight_max": 36.10746383666992,
      "activations/layer13_attention_weight_min": -33.83338165283203,
      "activations/layer14_attention_weight_max": 39.08827209472656,
      "activations/layer14_attention_weight_min": -37.3264274597168,
      "activations/layer15_attention_weight_max": 35.80804443359375,
      "activations/layer15_attention_weight_min": -36.76618957519531,
      "activations/layer16_attention_weight_max": 25.989173889160156,
      "activations/layer16_attention_weight_min": -27.053104400634766,
      "activations/layer17_attention_weight_max": 26.10414695739746,
      "activations/layer17_attention_weight_min": -26.08881950378418,
      "activations/layer18_attention_weight_max": 32.9347038269043,
      "activations/layer18_attention_weight_min": -28.861299514770508,
      "activations/layer19_attention_weight_max": 29.46721839904785,
      "activations/layer19_attention_weight_min": -35.632442474365234,
      "activations/layer1_attention_weight_max": 15.360541343688965,
      "activations/layer1_attention_weight_min": -14.792903900146484,
      "activations/layer20_attention_weight_max": 26.272354125976562,
      "activations/layer20_attention_weight_min": -25.815879821777344,
      "activations/layer21_attention_weight_max": 25.822656631469727,
      "activations/layer21_attention_weight_min": -25.614892959594727,
      "activations/layer22_attention_weight_max": 42.38290023803711,
      "activations/layer22_attention_weight_min": -33.71271514892578,
      "activations/layer23_attention_weight_max": 30.204425811767578,
      "activations/layer23_attention_weight_min": -27.71475601196289,
      "activations/layer2_attention_weight_max": 32.92826843261719,
      "activations/layer2_attention_weight_min": -31.560462951660156,
      "activations/layer3_attention_weight_max": 99.94666290283203,
      "activations/layer3_attention_weight_min": -101.66937255859375,
      "activations/layer4_attention_weight_max": 104.8351058959961,
      "activations/layer4_attention_weight_min": -104.10247039794922,
      "activations/layer5_attention_weight_max": 49.7456169128418,
      "activations/layer5_attention_weight_min": -59.90970993041992,
      "activations/layer6_attention_weight_max": 43.820674896240234,
      "activations/layer6_attention_weight_min": -47.1504020690918,
      "activations/layer7_attention_weight_max": 92.94522094726562,
      "activations/layer7_attention_weight_min": -96.45914459228516,
      "activations/layer8_attention_weight_max": 40.71469497680664,
      "activations/layer8_attention_weight_min": -43.823280334472656,
      "activations/layer9_attention_weight_max": 30.899198532104492,
      "activations/layer9_attention_weight_min": -33.52294158935547,
      "epoch": 19.03,
      "learning_rate": 2.754242424242424e-05,
      "loss": 2.7164,
      "step": 327550
    },
    {
      "activations/layer0_attention_weight_max": 16.647645950317383,
      "activations/layer0_attention_weight_min": -12.187470436096191,
      "activations/layer10_attention_weight_max": 37.521236419677734,
      "activations/layer10_attention_weight_min": -36.692405700683594,
      "activations/layer11_attention_weight_max": 35.49010467529297,
      "activations/layer11_attention_weight_min": -34.918701171875,
      "activations/layer12_attention_weight_max": 27.89614486694336,
      "activations/layer12_attention_weight_min": -27.265878677368164,
      "activations/layer13_attention_weight_max": 40.00853729248047,
      "activations/layer13_attention_weight_min": -36.514583587646484,
      "activations/layer14_attention_weight_max": 43.77848434448242,
      "activations/layer14_attention_weight_min": -39.1332893371582,
      "activations/layer15_attention_weight_max": 41.33465576171875,
      "activations/layer15_attention_weight_min": -35.713504791259766,
      "activations/layer16_attention_weight_max": 28.53679084777832,
      "activations/layer16_attention_weight_min": -27.478717803955078,
      "activations/layer17_attention_weight_max": 27.268909454345703,
      "activations/layer17_attention_weight_min": -24.090137481689453,
      "activations/layer18_attention_weight_max": 30.679744720458984,
      "activations/layer18_attention_weight_min": -25.25050163269043,
      "activations/layer19_attention_weight_max": 33.27157211303711,
      "activations/layer19_attention_weight_min": -30.734619140625,
      "activations/layer1_attention_weight_max": 16.377330780029297,
      "activations/layer1_attention_weight_min": -15.6963472366333,
      "activations/layer20_attention_weight_max": 26.000341415405273,
      "activations/layer20_attention_weight_min": -25.00264549255371,
      "activations/layer21_attention_weight_max": 29.102554321289062,
      "activations/layer21_attention_weight_min": -24.07137107849121,
      "activations/layer22_attention_weight_max": 38.65818405151367,
      "activations/layer22_attention_weight_min": -30.864919662475586,
      "activations/layer23_attention_weight_max": 30.11822509765625,
      "activations/layer23_attention_weight_min": -24.997425079345703,
      "activations/layer2_attention_weight_max": 31.682292938232422,
      "activations/layer2_attention_weight_min": -32.261531829833984,
      "activations/layer3_attention_weight_max": 101.7170639038086,
      "activations/layer3_attention_weight_min": -102.92375946044922,
      "activations/layer4_attention_weight_max": 109.6071548461914,
      "activations/layer4_attention_weight_min": -112.36921691894531,
      "activations/layer5_attention_weight_max": 52.418174743652344,
      "activations/layer5_attention_weight_min": -62.73789596557617,
      "activations/layer6_attention_weight_max": 46.15041732788086,
      "activations/layer6_attention_weight_min": -47.891685485839844,
      "activations/layer7_attention_weight_max": 97.77250671386719,
      "activations/layer7_attention_weight_min": -102.2120132446289,
      "activations/layer8_attention_weight_max": 44.647926330566406,
      "activations/layer8_attention_weight_min": -45.466644287109375,
      "activations/layer9_attention_weight_max": 35.01583480834961,
      "activations/layer9_attention_weight_min": -37.35300827026367,
      "epoch": 19.04,
      "learning_rate": 2.7523484848484844e-05,
      "loss": 2.7081,
      "step": 327600
    },
    {
      "activations/layer0_attention_weight_max": 16.874252319335938,
      "activations/layer0_attention_weight_min": -11.651960372924805,
      "activations/layer10_attention_weight_max": 35.56099319458008,
      "activations/layer10_attention_weight_min": -36.75667190551758,
      "activations/layer11_attention_weight_max": 35.58513641357422,
      "activations/layer11_attention_weight_min": -36.842071533203125,
      "activations/layer12_attention_weight_max": 27.23615074157715,
      "activations/layer12_attention_weight_min": -28.90933609008789,
      "activations/layer13_attention_weight_max": 39.01984405517578,
      "activations/layer13_attention_weight_min": -39.8402214050293,
      "activations/layer14_attention_weight_max": 41.48453903198242,
      "activations/layer14_attention_weight_min": -45.589176177978516,
      "activations/layer15_attention_weight_max": 38.02043914794922,
      "activations/layer15_attention_weight_min": -38.94735336303711,
      "activations/layer16_attention_weight_max": 28.81070327758789,
      "activations/layer16_attention_weight_min": -29.34296417236328,
      "activations/layer17_attention_weight_max": 28.454118728637695,
      "activations/layer17_attention_weight_min": -29.924861907958984,
      "activations/layer18_attention_weight_max": 33.3400993347168,
      "activations/layer18_attention_weight_min": -27.833255767822266,
      "activations/layer19_attention_weight_max": 31.794498443603516,
      "activations/layer19_attention_weight_min": -35.00490188598633,
      "activations/layer1_attention_weight_max": 16.003890991210938,
      "activations/layer1_attention_weight_min": -14.86139965057373,
      "activations/layer20_attention_weight_max": 27.044910430908203,
      "activations/layer20_attention_weight_min": -27.817886352539062,
      "activations/layer21_attention_weight_max": 28.4427547454834,
      "activations/layer21_attention_weight_min": -27.843278884887695,
      "activations/layer22_attention_weight_max": 42.265655517578125,
      "activations/layer22_attention_weight_min": -36.86935806274414,
      "activations/layer23_attention_weight_max": 32.634300231933594,
      "activations/layer23_attention_weight_min": -28.835289001464844,
      "activations/layer2_attention_weight_max": 33.18533706665039,
      "activations/layer2_attention_weight_min": -32.40174865722656,
      "activations/layer3_attention_weight_max": 100.46248626708984,
      "activations/layer3_attention_weight_min": -98.99887084960938,
      "activations/layer4_attention_weight_max": 107.80718994140625,
      "activations/layer4_attention_weight_min": -106.75418853759766,
      "activations/layer5_attention_weight_max": 50.26036071777344,
      "activations/layer5_attention_weight_min": -63.354774475097656,
      "activations/layer6_attention_weight_max": 45.21926498413086,
      "activations/layer6_attention_weight_min": -45.90090560913086,
      "activations/layer7_attention_weight_max": 95.06263732910156,
      "activations/layer7_attention_weight_min": -97.56012725830078,
      "activations/layer8_attention_weight_max": 42.99946975708008,
      "activations/layer8_attention_weight_min": -45.13977813720703,
      "activations/layer9_attention_weight_max": 33.23622512817383,
      "activations/layer9_attention_weight_min": -37.05204772949219,
      "epoch": 19.04,
      "learning_rate": 2.7504545454545453e-05,
      "loss": 2.7283,
      "step": 327650
    },
    {
      "activations/layer0_attention_weight_max": 17.57403564453125,
      "activations/layer0_attention_weight_min": -14.116673469543457,
      "activations/layer10_attention_weight_max": 32.401336669921875,
      "activations/layer10_attention_weight_min": -32.638946533203125,
      "activations/layer11_attention_weight_max": 33.14162063598633,
      "activations/layer11_attention_weight_min": -33.11134719848633,
      "activations/layer12_attention_weight_max": 25.429235458374023,
      "activations/layer12_attention_weight_min": -25.660093307495117,
      "activations/layer13_attention_weight_max": 37.978858947753906,
      "activations/layer13_attention_weight_min": -34.616817474365234,
      "activations/layer14_attention_weight_max": 39.299983978271484,
      "activations/layer14_attention_weight_min": -36.22306823730469,
      "activations/layer15_attention_weight_max": 36.76316452026367,
      "activations/layer15_attention_weight_min": -35.851783752441406,
      "activations/layer16_attention_weight_max": 29.651887893676758,
      "activations/layer16_attention_weight_min": -33.495426177978516,
      "activations/layer17_attention_weight_max": 28.69062042236328,
      "activations/layer17_attention_weight_min": -25.44622230529785,
      "activations/layer18_attention_weight_max": 31.862300872802734,
      "activations/layer18_attention_weight_min": -25.524267196655273,
      "activations/layer19_attention_weight_max": 29.201602935791016,
      "activations/layer19_attention_weight_min": -30.435880661010742,
      "activations/layer1_attention_weight_max": 16.632516860961914,
      "activations/layer1_attention_weight_min": -15.007420539855957,
      "activations/layer20_attention_weight_max": 25.163667678833008,
      "activations/layer20_attention_weight_min": -24.442581176757812,
      "activations/layer21_attention_weight_max": 24.807571411132812,
      "activations/layer21_attention_weight_min": -24.367883682250977,
      "activations/layer22_attention_weight_max": 36.70766830444336,
      "activations/layer22_attention_weight_min": -31.429779052734375,
      "activations/layer23_attention_weight_max": 29.382177352905273,
      "activations/layer23_attention_weight_min": -26.656827926635742,
      "activations/layer2_attention_weight_max": 31.69216537475586,
      "activations/layer2_attention_weight_min": -31.31671142578125,
      "activations/layer3_attention_weight_max": 98.12523651123047,
      "activations/layer3_attention_weight_min": -100.12765502929688,
      "activations/layer4_attention_weight_max": 103.25139617919922,
      "activations/layer4_attention_weight_min": -110.14985656738281,
      "activations/layer5_attention_weight_max": 48.75318145751953,
      "activations/layer5_attention_weight_min": -59.939205169677734,
      "activations/layer6_attention_weight_max": 41.367889404296875,
      "activations/layer6_attention_weight_min": -44.78849792480469,
      "activations/layer7_attention_weight_max": 89.95255279541016,
      "activations/layer7_attention_weight_min": -92.99739074707031,
      "activations/layer8_attention_weight_max": 38.85250473022461,
      "activations/layer8_attention_weight_min": -41.25777053833008,
      "activations/layer9_attention_weight_max": 29.33449363708496,
      "activations/layer9_attention_weight_min": -33.4144287109375,
      "epoch": 19.04,
      "learning_rate": 2.7485606060606057e-05,
      "loss": 2.7154,
      "step": 327700
    },
    {
      "activations/layer0_attention_weight_max": 16.33684539794922,
      "activations/layer0_attention_weight_min": -12.141831398010254,
      "activations/layer10_attention_weight_max": 32.2636604309082,
      "activations/layer10_attention_weight_min": -33.0455322265625,
      "activations/layer11_attention_weight_max": 32.41168975830078,
      "activations/layer11_attention_weight_min": -32.782161712646484,
      "activations/layer12_attention_weight_max": 28.094301223754883,
      "activations/layer12_attention_weight_min": -25.990482330322266,
      "activations/layer13_attention_weight_max": 38.93949508666992,
      "activations/layer13_attention_weight_min": -34.71251678466797,
      "activations/layer14_attention_weight_max": 43.546051025390625,
      "activations/layer14_attention_weight_min": -36.71582794189453,
      "activations/layer15_attention_weight_max": 40.64566421508789,
      "activations/layer15_attention_weight_min": -36.02772903442383,
      "activations/layer16_attention_weight_max": 31.55791473388672,
      "activations/layer16_attention_weight_min": -27.345211029052734,
      "activations/layer17_attention_weight_max": 28.443653106689453,
      "activations/layer17_attention_weight_min": -27.32733917236328,
      "activations/layer18_attention_weight_max": 32.38860321044922,
      "activations/layer18_attention_weight_min": -28.347759246826172,
      "activations/layer19_attention_weight_max": 33.75901412963867,
      "activations/layer19_attention_weight_min": -30.696191787719727,
      "activations/layer1_attention_weight_max": 15.541465759277344,
      "activations/layer1_attention_weight_min": -14.19044017791748,
      "activations/layer20_attention_weight_max": 29.22795867919922,
      "activations/layer20_attention_weight_min": -24.95452117919922,
      "activations/layer21_attention_weight_max": 28.99176025390625,
      "activations/layer21_attention_weight_min": -25.338104248046875,
      "activations/layer22_attention_weight_max": 39.784732818603516,
      "activations/layer22_attention_weight_min": -31.015676498413086,
      "activations/layer23_attention_weight_max": 30.099842071533203,
      "activations/layer23_attention_weight_min": -27.362714767456055,
      "activations/layer2_attention_weight_max": 31.367813110351562,
      "activations/layer2_attention_weight_min": -30.586132049560547,
      "activations/layer3_attention_weight_max": 96.76730346679688,
      "activations/layer3_attention_weight_min": -97.48367309570312,
      "activations/layer4_attention_weight_max": 102.48876190185547,
      "activations/layer4_attention_weight_min": -108.01861572265625,
      "activations/layer5_attention_weight_max": 46.57189178466797,
      "activations/layer5_attention_weight_min": -62.410335540771484,
      "activations/layer6_attention_weight_max": 42.132076263427734,
      "activations/layer6_attention_weight_min": -44.47075653076172,
      "activations/layer7_attention_weight_max": 86.915283203125,
      "activations/layer7_attention_weight_min": -93.1327133178711,
      "activations/layer8_attention_weight_max": 41.7000846862793,
      "activations/layer8_attention_weight_min": -41.67805862426758,
      "activations/layer9_attention_weight_max": 32.160499572753906,
      "activations/layer9_attention_weight_min": -34.91797637939453,
      "epoch": 19.04,
      "learning_rate": 2.7466666666666666e-05,
      "loss": 2.7139,
      "step": 327750
    },
    {
      "activations/layer0_attention_weight_max": 16.514354705810547,
      "activations/layer0_attention_weight_min": -12.06771469116211,
      "activations/layer10_attention_weight_max": 35.75037384033203,
      "activations/layer10_attention_weight_min": -36.282318115234375,
      "activations/layer11_attention_weight_max": 37.39546203613281,
      "activations/layer11_attention_weight_min": -36.686187744140625,
      "activations/layer12_attention_weight_max": 28.23716926574707,
      "activations/layer12_attention_weight_min": -27.8255558013916,
      "activations/layer13_attention_weight_max": 42.45222473144531,
      "activations/layer13_attention_weight_min": -37.134033203125,
      "activations/layer14_attention_weight_max": 42.257938385009766,
      "activations/layer14_attention_weight_min": -37.8379020690918,
      "activations/layer15_attention_weight_max": 40.61165237426758,
      "activations/layer15_attention_weight_min": -35.71074676513672,
      "activations/layer16_attention_weight_max": 30.237329483032227,
      "activations/layer16_attention_weight_min": -27.44521141052246,
      "activations/layer17_attention_weight_max": 30.07704734802246,
      "activations/layer17_attention_weight_min": -24.99454689025879,
      "activations/layer18_attention_weight_max": 31.78518295288086,
      "activations/layer18_attention_weight_min": -25.084177017211914,
      "activations/layer19_attention_weight_max": 37.1488151550293,
      "activations/layer19_attention_weight_min": -32.34469985961914,
      "activations/layer1_attention_weight_max": 15.796693801879883,
      "activations/layer1_attention_weight_min": -13.26524543762207,
      "activations/layer20_attention_weight_max": 31.80845832824707,
      "activations/layer20_attention_weight_min": -24.771928787231445,
      "activations/layer21_attention_weight_max": 30.311559677124023,
      "activations/layer21_attention_weight_min": -23.932151794433594,
      "activations/layer22_attention_weight_max": 39.202144622802734,
      "activations/layer22_attention_weight_min": -31.09768295288086,
      "activations/layer23_attention_weight_max": 34.88671875,
      "activations/layer23_attention_weight_min": -24.745384216308594,
      "activations/layer2_attention_weight_max": 31.3284969329834,
      "activations/layer2_attention_weight_min": -30.12714958190918,
      "activations/layer3_attention_weight_max": 93.9216079711914,
      "activations/layer3_attention_weight_min": -94.81814575195312,
      "activations/layer4_attention_weight_max": 102.29338836669922,
      "activations/layer4_attention_weight_min": -105.13866424560547,
      "activations/layer5_attention_weight_max": 49.87527084350586,
      "activations/layer5_attention_weight_min": -64.61964416503906,
      "activations/layer6_attention_weight_max": 46.24734115600586,
      "activations/layer6_attention_weight_min": -48.342098236083984,
      "activations/layer7_attention_weight_max": 94.73729705810547,
      "activations/layer7_attention_weight_min": -94.47677612304688,
      "activations/layer8_attention_weight_max": 42.69024658203125,
      "activations/layer8_attention_weight_min": -44.081764221191406,
      "activations/layer9_attention_weight_max": 31.991718292236328,
      "activations/layer9_attention_weight_min": -33.09785461425781,
      "epoch": 19.05,
      "learning_rate": 2.7447727272727267e-05,
      "loss": 2.733,
      "step": 327800
    },
    {
      "activations/layer0_attention_weight_max": 16.956851959228516,
      "activations/layer0_attention_weight_min": -13.098740577697754,
      "activations/layer10_attention_weight_max": 39.3769416809082,
      "activations/layer10_attention_weight_min": -37.44659423828125,
      "activations/layer11_attention_weight_max": 39.51617431640625,
      "activations/layer11_attention_weight_min": -37.07511901855469,
      "activations/layer12_attention_weight_max": 26.280187606811523,
      "activations/layer12_attention_weight_min": -27.551855087280273,
      "activations/layer13_attention_weight_max": 39.513267517089844,
      "activations/layer13_attention_weight_min": -35.829620361328125,
      "activations/layer14_attention_weight_max": 41.74477767944336,
      "activations/layer14_attention_weight_min": -37.68311309814453,
      "activations/layer15_attention_weight_max": 36.736175537109375,
      "activations/layer15_attention_weight_min": -35.108917236328125,
      "activations/layer16_attention_weight_max": 27.73693084716797,
      "activations/layer16_attention_weight_min": -30.62114715576172,
      "activations/layer17_attention_weight_max": 26.762086868286133,
      "activations/layer17_attention_weight_min": -26.414918899536133,
      "activations/layer18_attention_weight_max": 29.34587287902832,
      "activations/layer18_attention_weight_min": -25.4598388671875,
      "activations/layer19_attention_weight_max": 31.683496475219727,
      "activations/layer19_attention_weight_min": -31.567453384399414,
      "activations/layer1_attention_weight_max": 15.577268600463867,
      "activations/layer1_attention_weight_min": -15.4570951461792,
      "activations/layer20_attention_weight_max": 28.32518768310547,
      "activations/layer20_attention_weight_min": -26.205547332763672,
      "activations/layer21_attention_weight_max": 29.69529151916504,
      "activations/layer21_attention_weight_min": -23.6779727935791,
      "activations/layer22_attention_weight_max": 41.00141906738281,
      "activations/layer22_attention_weight_min": -30.041793823242188,
      "activations/layer23_attention_weight_max": 31.999866485595703,
      "activations/layer23_attention_weight_min": -24.154897689819336,
      "activations/layer2_attention_weight_max": 33.497314453125,
      "activations/layer2_attention_weight_min": -32.716346740722656,
      "activations/layer3_attention_weight_max": 101.68233489990234,
      "activations/layer3_attention_weight_min": -102.7893295288086,
      "activations/layer4_attention_weight_max": 107.3929672241211,
      "activations/layer4_attention_weight_min": -107.0815200805664,
      "activations/layer5_attention_weight_max": 49.82786560058594,
      "activations/layer5_attention_weight_min": -61.64826583862305,
      "activations/layer6_attention_weight_max": 49.049354553222656,
      "activations/layer6_attention_weight_min": -48.42005157470703,
      "activations/layer7_attention_weight_max": 96.20726013183594,
      "activations/layer7_attention_weight_min": -98.54824829101562,
      "activations/layer8_attention_weight_max": 45.26765060424805,
      "activations/layer8_attention_weight_min": -47.62152862548828,
      "activations/layer9_attention_weight_max": 36.11993408203125,
      "activations/layer9_attention_weight_min": -35.48014831542969,
      "epoch": 19.05,
      "learning_rate": 2.7428787878787875e-05,
      "loss": 2.7293,
      "step": 327850
    },
    {
      "activations/layer0_attention_weight_max": 16.51742172241211,
      "activations/layer0_attention_weight_min": -13.758036613464355,
      "activations/layer10_attention_weight_max": 38.4951286315918,
      "activations/layer10_attention_weight_min": -35.557552337646484,
      "activations/layer11_attention_weight_max": 35.797481536865234,
      "activations/layer11_attention_weight_min": -36.162803649902344,
      "activations/layer12_attention_weight_max": 29.906099319458008,
      "activations/layer12_attention_weight_min": -27.259727478027344,
      "activations/layer13_attention_weight_max": 43.935035705566406,
      "activations/layer13_attention_weight_min": -38.28202438354492,
      "activations/layer14_attention_weight_max": 48.089107513427734,
      "activations/layer14_attention_weight_min": -41.42959976196289,
      "activations/layer15_attention_weight_max": 42.82392501831055,
      "activations/layer15_attention_weight_min": -37.506961822509766,
      "activations/layer16_attention_weight_max": 34.57514190673828,
      "activations/layer16_attention_weight_min": -29.2285213470459,
      "activations/layer17_attention_weight_max": 34.87458419799805,
      "activations/layer17_attention_weight_min": -28.281198501586914,
      "activations/layer18_attention_weight_max": 33.53208923339844,
      "activations/layer18_attention_weight_min": -26.687667846679688,
      "activations/layer19_attention_weight_max": 32.90176010131836,
      "activations/layer19_attention_weight_min": -33.03086471557617,
      "activations/layer1_attention_weight_max": 16.457096099853516,
      "activations/layer1_attention_weight_min": -15.762636184692383,
      "activations/layer20_attention_weight_max": 29.27901840209961,
      "activations/layer20_attention_weight_min": -25.84428596496582,
      "activations/layer21_attention_weight_max": 29.508495330810547,
      "activations/layer21_attention_weight_min": -26.948562622070312,
      "activations/layer22_attention_weight_max": 41.667816162109375,
      "activations/layer22_attention_weight_min": -33.7192497253418,
      "activations/layer23_attention_weight_max": 30.486717224121094,
      "activations/layer23_attention_weight_min": -27.218120574951172,
      "activations/layer2_attention_weight_max": 33.204933166503906,
      "activations/layer2_attention_weight_min": -32.255958557128906,
      "activations/layer3_attention_weight_max": 98.62300109863281,
      "activations/layer3_attention_weight_min": -100.27961730957031,
      "activations/layer4_attention_weight_max": 105.83870697021484,
      "activations/layer4_attention_weight_min": -111.8972396850586,
      "activations/layer5_attention_weight_max": 50.31821823120117,
      "activations/layer5_attention_weight_min": -63.10905075073242,
      "activations/layer6_attention_weight_max": 44.87788009643555,
      "activations/layer6_attention_weight_min": -49.2324333190918,
      "activations/layer7_attention_weight_max": 93.74602508544922,
      "activations/layer7_attention_weight_min": -102.2528305053711,
      "activations/layer8_attention_weight_max": 45.80607986450195,
      "activations/layer8_attention_weight_min": -47.02524948120117,
      "activations/layer9_attention_weight_max": 34.77724075317383,
      "activations/layer9_attention_weight_min": -36.28142547607422,
      "epoch": 19.05,
      "learning_rate": 2.7409848484848484e-05,
      "loss": 2.7201,
      "step": 327900
    },
    {
      "activations/layer0_attention_weight_max": 17.57740020751953,
      "activations/layer0_attention_weight_min": -11.84495735168457,
      "activations/layer10_attention_weight_max": 40.977142333984375,
      "activations/layer10_attention_weight_min": -38.19070816040039,
      "activations/layer11_attention_weight_max": 38.581443786621094,
      "activations/layer11_attention_weight_min": -37.33982849121094,
      "activations/layer12_attention_weight_max": 31.194843292236328,
      "activations/layer12_attention_weight_min": -28.515018463134766,
      "activations/layer13_attention_weight_max": 45.92217254638672,
      "activations/layer13_attention_weight_min": -39.3863410949707,
      "activations/layer14_attention_weight_max": 49.10219955444336,
      "activations/layer14_attention_weight_min": -43.68041229248047,
      "activations/layer15_attention_weight_max": 45.55588150024414,
      "activations/layer15_attention_weight_min": -37.6270637512207,
      "activations/layer16_attention_weight_max": 32.39394760131836,
      "activations/layer16_attention_weight_min": -27.349767684936523,
      "activations/layer17_attention_weight_max": 31.32208251953125,
      "activations/layer17_attention_weight_min": -28.10995101928711,
      "activations/layer18_attention_weight_max": 30.59709358215332,
      "activations/layer18_attention_weight_min": -26.88811683654785,
      "activations/layer19_attention_weight_max": 34.555912017822266,
      "activations/layer19_attention_weight_min": -30.833471298217773,
      "activations/layer1_attention_weight_max": 15.939990997314453,
      "activations/layer1_attention_weight_min": -13.973922729492188,
      "activations/layer20_attention_weight_max": 27.305335998535156,
      "activations/layer20_attention_weight_min": -24.176502227783203,
      "activations/layer21_attention_weight_max": 28.335006713867188,
      "activations/layer21_attention_weight_min": -23.76409339904785,
      "activations/layer22_attention_weight_max": 44.726951599121094,
      "activations/layer22_attention_weight_min": -31.485553741455078,
      "activations/layer23_attention_weight_max": 35.45333480834961,
      "activations/layer23_attention_weight_min": -22.86639976501465,
      "activations/layer2_attention_weight_max": 36.43436813354492,
      "activations/layer2_attention_weight_min": -32.63850402832031,
      "activations/layer3_attention_weight_max": 107.91925811767578,
      "activations/layer3_attention_weight_min": -106.94384765625,
      "activations/layer4_attention_weight_max": 109.47112274169922,
      "activations/layer4_attention_weight_min": -111.32159423828125,
      "activations/layer5_attention_weight_max": 52.17782211303711,
      "activations/layer5_attention_weight_min": -61.722408294677734,
      "activations/layer6_attention_weight_max": 44.34162521362305,
      "activations/layer6_attention_weight_min": -46.853519439697266,
      "activations/layer7_attention_weight_max": 96.4823989868164,
      "activations/layer7_attention_weight_min": -96.25550079345703,
      "activations/layer8_attention_weight_max": 46.86917495727539,
      "activations/layer8_attention_weight_min": -49.8991584777832,
      "activations/layer9_attention_weight_max": 36.3504638671875,
      "activations/layer9_attention_weight_min": -38.63740158081055,
      "epoch": 19.06,
      "learning_rate": 2.739090909090909e-05,
      "loss": 2.736,
      "step": 327950
    },
    {
      "activations/layer0_attention_weight_max": 16.45117950439453,
      "activations/layer0_attention_weight_min": -12.493926048278809,
      "activations/layer10_attention_weight_max": 52.980430603027344,
      "activations/layer10_attention_weight_min": -46.7551383972168,
      "activations/layer11_attention_weight_max": 55.15638732910156,
      "activations/layer11_attention_weight_min": -50.6815299987793,
      "activations/layer12_attention_weight_max": 33.16136169433594,
      "activations/layer12_attention_weight_min": -30.819087982177734,
      "activations/layer13_attention_weight_max": 47.723087310791016,
      "activations/layer13_attention_weight_min": -42.04469680786133,
      "activations/layer14_attention_weight_max": 53.363677978515625,
      "activations/layer14_attention_weight_min": -44.298282623291016,
      "activations/layer15_attention_weight_max": 53.7501220703125,
      "activations/layer15_attention_weight_min": -47.49749755859375,
      "activations/layer16_attention_weight_max": 34.711952209472656,
      "activations/layer16_attention_weight_min": -31.419437408447266,
      "activations/layer17_attention_weight_max": 35.684486389160156,
      "activations/layer17_attention_weight_min": -32.76222610473633,
      "activations/layer18_attention_weight_max": 32.82467269897461,
      "activations/layer18_attention_weight_min": -29.8768367767334,
      "activations/layer19_attention_weight_max": 32.00858688354492,
      "activations/layer19_attention_weight_min": -30.889596939086914,
      "activations/layer1_attention_weight_max": 16.271427154541016,
      "activations/layer1_attention_weight_min": -14.647953987121582,
      "activations/layer20_attention_weight_max": 27.334880828857422,
      "activations/layer20_attention_weight_min": -25.273380279541016,
      "activations/layer21_attention_weight_max": 27.04775047302246,
      "activations/layer21_attention_weight_min": -24.814512252807617,
      "activations/layer22_attention_weight_max": 43.2914924621582,
      "activations/layer22_attention_weight_min": -32.627437591552734,
      "activations/layer23_attention_weight_max": 29.649066925048828,
      "activations/layer23_attention_weight_min": -25.902114868164062,
      "activations/layer2_attention_weight_max": 35.74257278442383,
      "activations/layer2_attention_weight_min": -33.83511734008789,
      "activations/layer3_attention_weight_max": 105.3915786743164,
      "activations/layer3_attention_weight_min": -106.85173034667969,
      "activations/layer4_attention_weight_max": 117.36185455322266,
      "activations/layer4_attention_weight_min": -106.21388244628906,
      "activations/layer5_attention_weight_max": 57.895626068115234,
      "activations/layer5_attention_weight_min": -63.9448356628418,
      "activations/layer6_attention_weight_max": 50.10348892211914,
      "activations/layer6_attention_weight_min": -46.105525970458984,
      "activations/layer7_attention_weight_max": 119.25504302978516,
      "activations/layer7_attention_weight_min": -110.61457824707031,
      "activations/layer8_attention_weight_max": 55.466766357421875,
      "activations/layer8_attention_weight_min": -51.15279006958008,
      "activations/layer9_attention_weight_max": 43.28785705566406,
      "activations/layer9_attention_weight_min": -41.68287658691406,
      "epoch": 19.06,
      "learning_rate": 2.7371969696969697e-05,
      "loss": 2.7193,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.4669,
      "eval_samples_per_second": 507.153,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.4669,
      "eval_openwebtext_samples_per_second": 507.153,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9512,
      "eval_wikitext_samples_per_second": 233.702,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_lambada_loss": 2.380859375,
      "eval_lambada_ppl": 10.814192318320675,
      "eval_lambada_runtime": 9.5559,
      "eval_lambada_samples_per_second": 509.527,
      "step": 328000
    },
    {
      "activations/layer0_attention_weight_max": 16.36548614501953,
      "activations/layer0_attention_weight_min": -14.040595054626465,
      "activations/layer10_attention_weight_max": 35.87004470825195,
      "activations/layer10_attention_weight_min": -39.43397903442383,
      "activations/layer11_attention_weight_max": 37.036277770996094,
      "activations/layer11_attention_weight_min": -37.83964538574219,
      "activations/layer12_attention_weight_max": 28.743427276611328,
      "activations/layer12_attention_weight_min": -27.576013565063477,
      "activations/layer13_attention_weight_max": 41.5043830871582,
      "activations/layer13_attention_weight_min": -36.360687255859375,
      "activations/layer14_attention_weight_max": 42.67782974243164,
      "activations/layer14_attention_weight_min": -37.22929000854492,
      "activations/layer15_attention_weight_max": 42.63886642456055,
      "activations/layer15_attention_weight_min": -37.55775451660156,
      "activations/layer16_attention_weight_max": 32.408172607421875,
      "activations/layer16_attention_weight_min": -28.41813087463379,
      "activations/layer17_attention_weight_max": 34.02840805053711,
      "activations/layer17_attention_weight_min": -27.88124656677246,
      "activations/layer18_attention_weight_max": 29.918746948242188,
      "activations/layer18_attention_weight_min": -27.331518173217773,
      "activations/layer19_attention_weight_max": 37.84511184692383,
      "activations/layer19_attention_weight_min": -32.22905731201172,
      "activations/layer1_attention_weight_max": 16.2408504486084,
      "activations/layer1_attention_weight_min": -14.402976989746094,
      "activations/layer20_attention_weight_max": 29.30879783630371,
      "activations/layer20_attention_weight_min": -27.05455207824707,
      "activations/layer21_attention_weight_max": 30.32613182067871,
      "activations/layer21_attention_weight_min": -26.175304412841797,
      "activations/layer22_attention_weight_max": 38.22197341918945,
      "activations/layer22_attention_weight_min": -31.309600830078125,
      "activations/layer23_attention_weight_max": 29.738868713378906,
      "activations/layer23_attention_weight_min": -24.71622085571289,
      "activations/layer2_attention_weight_max": 33.25648498535156,
      "activations/layer2_attention_weight_min": -31.756467819213867,
      "activations/layer3_attention_weight_max": 99.37679290771484,
      "activations/layer3_attention_weight_min": -105.4172134399414,
      "activations/layer4_attention_weight_max": 107.19606018066406,
      "activations/layer4_attention_weight_min": -110.2012939453125,
      "activations/layer5_attention_weight_max": 49.435020446777344,
      "activations/layer5_attention_weight_min": -67.15423583984375,
      "activations/layer6_attention_weight_max": 43.290016174316406,
      "activations/layer6_attention_weight_min": -46.26259994506836,
      "activations/layer7_attention_weight_max": 94.4352798461914,
      "activations/layer7_attention_weight_min": -100.9159927368164,
      "activations/layer8_attention_weight_max": 44.40406799316406,
      "activations/layer8_attention_weight_min": -49.856727600097656,
      "activations/layer9_attention_weight_max": 32.68320846557617,
      "activations/layer9_attention_weight_min": -38.226253509521484,
      "epoch": 19.06,
      "learning_rate": 2.7353030303030302e-05,
      "loss": 2.7125,
      "step": 328050
    },
    {
      "activations/layer0_attention_weight_max": 16.171640396118164,
      "activations/layer0_attention_weight_min": -12.257759094238281,
      "activations/layer10_attention_weight_max": 42.4064826965332,
      "activations/layer10_attention_weight_min": -38.074424743652344,
      "activations/layer11_attention_weight_max": 41.476558685302734,
      "activations/layer11_attention_weight_min": -42.88778305053711,
      "activations/layer12_attention_weight_max": 33.89195251464844,
      "activations/layer12_attention_weight_min": -28.305055618286133,
      "activations/layer13_attention_weight_max": 47.376163482666016,
      "activations/layer13_attention_weight_min": -37.85792922973633,
      "activations/layer14_attention_weight_max": 42.69581604003906,
      "activations/layer14_attention_weight_min": -42.0777702331543,
      "activations/layer15_attention_weight_max": 39.819374084472656,
      "activations/layer15_attention_weight_min": -43.45317459106445,
      "activations/layer16_attention_weight_max": 28.99258041381836,
      "activations/layer16_attention_weight_min": -28.462356567382812,
      "activations/layer17_attention_weight_max": 28.80703353881836,
      "activations/layer17_attention_weight_min": -29.478736877441406,
      "activations/layer18_attention_weight_max": 31.273521423339844,
      "activations/layer18_attention_weight_min": -30.910551071166992,
      "activations/layer19_attention_weight_max": 31.79075050354004,
      "activations/layer19_attention_weight_min": -33.55895233154297,
      "activations/layer1_attention_weight_max": 15.619217872619629,
      "activations/layer1_attention_weight_min": -14.502792358398438,
      "activations/layer20_attention_weight_max": 26.972820281982422,
      "activations/layer20_attention_weight_min": -27.456785202026367,
      "activations/layer21_attention_weight_max": 28.603124618530273,
      "activations/layer21_attention_weight_min": -26.99875831604004,
      "activations/layer22_attention_weight_max": 36.30433654785156,
      "activations/layer22_attention_weight_min": -33.49687576293945,
      "activations/layer23_attention_weight_max": 29.641475677490234,
      "activations/layer23_attention_weight_min": -26.4736328125,
      "activations/layer2_attention_weight_max": 29.889877319335938,
      "activations/layer2_attention_weight_min": -29.741300582885742,
      "activations/layer3_attention_weight_max": 92.90155029296875,
      "activations/layer3_attention_weight_min": -94.2774658203125,
      "activations/layer4_attention_weight_max": 97.51178741455078,
      "activations/layer4_attention_weight_min": -104.4430923461914,
      "activations/layer5_attention_weight_max": 47.61606979370117,
      "activations/layer5_attention_weight_min": -61.294776916503906,
      "activations/layer6_attention_weight_max": 43.630123138427734,
      "activations/layer6_attention_weight_min": -44.34939193725586,
      "activations/layer7_attention_weight_max": 99.17687225341797,
      "activations/layer7_attention_weight_min": -91.65316772460938,
      "activations/layer8_attention_weight_max": 44.423641204833984,
      "activations/layer8_attention_weight_min": -44.27975082397461,
      "activations/layer9_attention_weight_max": 35.41203308105469,
      "activations/layer9_attention_weight_min": -37.007652282714844,
      "epoch": 19.06,
      "learning_rate": 2.7334090909090907e-05,
      "loss": 2.7292,
      "step": 328100
    },
    {
      "activations/layer0_attention_weight_max": 16.30171012878418,
      "activations/layer0_attention_weight_min": -14.599663734436035,
      "activations/layer10_attention_weight_max": 36.46491241455078,
      "activations/layer10_attention_weight_min": -37.305641174316406,
      "activations/layer11_attention_weight_max": 37.56482696533203,
      "activations/layer11_attention_weight_min": -39.56487274169922,
      "activations/layer12_attention_weight_max": 26.867862701416016,
      "activations/layer12_attention_weight_min": -27.2841739654541,
      "activations/layer13_attention_weight_max": 38.98146438598633,
      "activations/layer13_attention_weight_min": -37.66566848754883,
      "activations/layer14_attention_weight_max": 41.571937561035156,
      "activations/layer14_attention_weight_min": -38.088199615478516,
      "activations/layer15_attention_weight_max": 40.27691650390625,
      "activations/layer15_attention_weight_min": -35.36845779418945,
      "activations/layer16_attention_weight_max": 30.294763565063477,
      "activations/layer16_attention_weight_min": -30.371965408325195,
      "activations/layer17_attention_weight_max": 31.50754737854004,
      "activations/layer17_attention_weight_min": -27.49788475036621,
      "activations/layer18_attention_weight_max": 32.96446228027344,
      "activations/layer18_attention_weight_min": -26.271398544311523,
      "activations/layer19_attention_weight_max": 34.67159652709961,
      "activations/layer19_attention_weight_min": -34.80352783203125,
      "activations/layer1_attention_weight_max": 15.898008346557617,
      "activations/layer1_attention_weight_min": -14.625374794006348,
      "activations/layer20_attention_weight_max": 29.750442504882812,
      "activations/layer20_attention_weight_min": -26.94902229309082,
      "activations/layer21_attention_weight_max": 30.638132095336914,
      "activations/layer21_attention_weight_min": -26.72098159790039,
      "activations/layer22_attention_weight_max": 47.24224853515625,
      "activations/layer22_attention_weight_min": -36.214622497558594,
      "activations/layer23_attention_weight_max": 34.25745391845703,
      "activations/layer23_attention_weight_min": -26.44321060180664,
      "activations/layer2_attention_weight_max": 30.825529098510742,
      "activations/layer2_attention_weight_min": -31.973798751831055,
      "activations/layer3_attention_weight_max": 100.7580337524414,
      "activations/layer3_attention_weight_min": -107.20706176757812,
      "activations/layer4_attention_weight_max": 101.62114715576172,
      "activations/layer4_attention_weight_min": -106.69415283203125,
      "activations/layer5_attention_weight_max": 49.53800582885742,
      "activations/layer5_attention_weight_min": -59.797218322753906,
      "activations/layer6_attention_weight_max": 43.21949005126953,
      "activations/layer6_attention_weight_min": -47.83650588989258,
      "activations/layer7_attention_weight_max": 90.52513122558594,
      "activations/layer7_attention_weight_min": -100.93257904052734,
      "activations/layer8_attention_weight_max": 41.69136047363281,
      "activations/layer8_attention_weight_min": -44.532291412353516,
      "activations/layer9_attention_weight_max": 34.110923767089844,
      "activations/layer9_attention_weight_min": -37.04165267944336,
      "epoch": 19.07,
      "learning_rate": 2.731515151515151e-05,
      "loss": 2.7231,
      "step": 328150
    },
    {
      "activations/layer0_attention_weight_max": 16.116579055786133,
      "activations/layer0_attention_weight_min": -12.268023490905762,
      "activations/layer10_attention_weight_max": 33.245826721191406,
      "activations/layer10_attention_weight_min": -33.80576705932617,
      "activations/layer11_attention_weight_max": 34.34100341796875,
      "activations/layer11_attention_weight_min": -33.81616973876953,
      "activations/layer12_attention_weight_max": 27.593294143676758,
      "activations/layer12_attention_weight_min": -26.96617317199707,
      "activations/layer13_attention_weight_max": 41.99348068237305,
      "activations/layer13_attention_weight_min": -34.74885177612305,
      "activations/layer14_attention_weight_max": 45.517112731933594,
      "activations/layer14_attention_weight_min": -37.42536544799805,
      "activations/layer15_attention_weight_max": 41.931705474853516,
      "activations/layer15_attention_weight_min": -37.52089309692383,
      "activations/layer16_attention_weight_max": 33.33637619018555,
      "activations/layer16_attention_weight_min": -28.413997650146484,
      "activations/layer17_attention_weight_max": 31.620609283447266,
      "activations/layer17_attention_weight_min": -26.953519821166992,
      "activations/layer18_attention_weight_max": 33.96883010864258,
      "activations/layer18_attention_weight_min": -28.05647087097168,
      "activations/layer19_attention_weight_max": 34.51167678833008,
      "activations/layer19_attention_weight_min": -31.30171775817871,
      "activations/layer1_attention_weight_max": 16.18359375,
      "activations/layer1_attention_weight_min": -14.660087585449219,
      "activations/layer20_attention_weight_max": 29.56911277770996,
      "activations/layer20_attention_weight_min": -23.962411880493164,
      "activations/layer21_attention_weight_max": 29.956863403320312,
      "activations/layer21_attention_weight_min": -24.340513229370117,
      "activations/layer22_attention_weight_max": 45.817447662353516,
      "activations/layer22_attention_weight_min": -31.39455223083496,
      "activations/layer23_attention_weight_max": 32.541900634765625,
      "activations/layer23_attention_weight_min": -25.58812713623047,
      "activations/layer2_attention_weight_max": 31.5073299407959,
      "activations/layer2_attention_weight_min": -31.373729705810547,
      "activations/layer3_attention_weight_max": 94.55889892578125,
      "activations/layer3_attention_weight_min": -99.06957244873047,
      "activations/layer4_attention_weight_max": 104.9972152709961,
      "activations/layer4_attention_weight_min": -101.63150024414062,
      "activations/layer5_attention_weight_max": 50.48487091064453,
      "activations/layer5_attention_weight_min": -60.847023010253906,
      "activations/layer6_attention_weight_max": 42.998924255371094,
      "activations/layer6_attention_weight_min": -46.11884307861328,
      "activations/layer7_attention_weight_max": 92.61402130126953,
      "activations/layer7_attention_weight_min": -92.54104614257812,
      "activations/layer8_attention_weight_max": 42.69790267944336,
      "activations/layer8_attention_weight_min": -43.56937026977539,
      "activations/layer9_attention_weight_max": 33.72665786743164,
      "activations/layer9_attention_weight_min": -34.99329376220703,
      "epoch": 19.07,
      "learning_rate": 2.729621212121212e-05,
      "loss": 2.7297,
      "step": 328200
    },
    {
      "activations/layer0_attention_weight_max": 16.882261276245117,
      "activations/layer0_attention_weight_min": -12.491693496704102,
      "activations/layer10_attention_weight_max": 39.426856994628906,
      "activations/layer10_attention_weight_min": -37.6229133605957,
      "activations/layer11_attention_weight_max": 42.670406341552734,
      "activations/layer11_attention_weight_min": -38.411476135253906,
      "activations/layer12_attention_weight_max": 28.16340446472168,
      "activations/layer12_attention_weight_min": -27.974672317504883,
      "activations/layer13_attention_weight_max": 40.4176139831543,
      "activations/layer13_attention_weight_min": -35.67768096923828,
      "activations/layer14_attention_weight_max": 45.922386169433594,
      "activations/layer14_attention_weight_min": -38.898590087890625,
      "activations/layer15_attention_weight_max": 38.47383117675781,
      "activations/layer15_attention_weight_min": -35.039833068847656,
      "activations/layer16_attention_weight_max": 32.36136245727539,
      "activations/layer16_attention_weight_min": -32.34761428833008,
      "activations/layer17_attention_weight_max": 31.654659271240234,
      "activations/layer17_attention_weight_min": -25.616992950439453,
      "activations/layer18_attention_weight_max": 34.960453033447266,
      "activations/layer18_attention_weight_min": -25.417760848999023,
      "activations/layer19_attention_weight_max": 38.47156524658203,
      "activations/layer19_attention_weight_min": -32.15887451171875,
      "activations/layer1_attention_weight_max": 15.564033508300781,
      "activations/layer1_attention_weight_min": -16.20014762878418,
      "activations/layer20_attention_weight_max": 30.27107810974121,
      "activations/layer20_attention_weight_min": -25.885501861572266,
      "activations/layer21_attention_weight_max": 30.183225631713867,
      "activations/layer21_attention_weight_min": -26.308828353881836,
      "activations/layer22_attention_weight_max": 47.92627716064453,
      "activations/layer22_attention_weight_min": -33.720542907714844,
      "activations/layer23_attention_weight_max": 34.43421173095703,
      "activations/layer23_attention_weight_min": -26.27153778076172,
      "activations/layer2_attention_weight_max": 30.71902084350586,
      "activations/layer2_attention_weight_min": -33.16202926635742,
      "activations/layer3_attention_weight_max": 99.67991638183594,
      "activations/layer3_attention_weight_min": -105.76256561279297,
      "activations/layer4_attention_weight_max": 108.6205062866211,
      "activations/layer4_attention_weight_min": -108.486572265625,
      "activations/layer5_attention_weight_max": 51.45201110839844,
      "activations/layer5_attention_weight_min": -63.62022018432617,
      "activations/layer6_attention_weight_max": 47.29442596435547,
      "activations/layer6_attention_weight_min": -50.69755935668945,
      "activations/layer7_attention_weight_max": 100.80036926269531,
      "activations/layer7_attention_weight_min": -105.31871032714844,
      "activations/layer8_attention_weight_max": 44.11314392089844,
      "activations/layer8_attention_weight_min": -47.09014129638672,
      "activations/layer9_attention_weight_max": 34.138343811035156,
      "activations/layer9_attention_weight_min": -36.84491729736328,
      "epoch": 19.07,
      "learning_rate": 2.7277272727272725e-05,
      "loss": 2.7255,
      "step": 328250
    },
    {
      "activations/layer0_attention_weight_max": 16.1092586517334,
      "activations/layer0_attention_weight_min": -13.308416366577148,
      "activations/layer10_attention_weight_max": 35.705177307128906,
      "activations/layer10_attention_weight_min": -36.820350646972656,
      "activations/layer11_attention_weight_max": 36.18327713012695,
      "activations/layer11_attention_weight_min": -38.58428955078125,
      "activations/layer12_attention_weight_max": 31.7122745513916,
      "activations/layer12_attention_weight_min": -28.424558639526367,
      "activations/layer13_attention_weight_max": 45.74378967285156,
      "activations/layer13_attention_weight_min": -40.035316467285156,
      "activations/layer14_attention_weight_max": 46.8835334777832,
      "activations/layer14_attention_weight_min": -40.86365509033203,
      "activations/layer15_attention_weight_max": 39.96799850463867,
      "activations/layer15_attention_weight_min": -41.05968475341797,
      "activations/layer16_attention_weight_max": 30.963041305541992,
      "activations/layer16_attention_weight_min": -27.891849517822266,
      "activations/layer17_attention_weight_max": 30.61834716796875,
      "activations/layer17_attention_weight_min": -26.256113052368164,
      "activations/layer18_attention_weight_max": 33.931732177734375,
      "activations/layer18_attention_weight_min": -28.075786590576172,
      "activations/layer19_attention_weight_max": 37.308570861816406,
      "activations/layer19_attention_weight_min": -31.886991500854492,
      "activations/layer1_attention_weight_max": 16.79730224609375,
      "activations/layer1_attention_weight_min": -14.780057907104492,
      "activations/layer20_attention_weight_max": 33.24546813964844,
      "activations/layer20_attention_weight_min": -25.4273624420166,
      "activations/layer21_attention_weight_max": 30.671798706054688,
      "activations/layer21_attention_weight_min": -25.504837036132812,
      "activations/layer22_attention_weight_max": 46.77876663208008,
      "activations/layer22_attention_weight_min": -33.38394546508789,
      "activations/layer23_attention_weight_max": 37.120174407958984,
      "activations/layer23_attention_weight_min": -27.47726058959961,
      "activations/layer2_attention_weight_max": 32.48898696899414,
      "activations/layer2_attention_weight_min": -31.218198776245117,
      "activations/layer3_attention_weight_max": 100.68121337890625,
      "activations/layer3_attention_weight_min": -98.1097412109375,
      "activations/layer4_attention_weight_max": 106.8643798828125,
      "activations/layer4_attention_weight_min": -104.80806732177734,
      "activations/layer5_attention_weight_max": 55.813392639160156,
      "activations/layer5_attention_weight_min": -62.3040885925293,
      "activations/layer6_attention_weight_max": 44.286685943603516,
      "activations/layer6_attention_weight_min": -47.63788986206055,
      "activations/layer7_attention_weight_max": 92.29541778564453,
      "activations/layer7_attention_weight_min": -94.21073150634766,
      "activations/layer8_attention_weight_max": 42.90275573730469,
      "activations/layer8_attention_weight_min": -44.41725540161133,
      "activations/layer9_attention_weight_max": 33.63831329345703,
      "activations/layer9_attention_weight_min": -35.000484466552734,
      "epoch": 19.08,
      "learning_rate": 2.7258333333333333e-05,
      "loss": 2.7085,
      "step": 328300
    },
    {
      "activations/layer0_attention_weight_max": 15.97424602508545,
      "activations/layer0_attention_weight_min": -12.321024894714355,
      "activations/layer10_attention_weight_max": 34.33955764770508,
      "activations/layer10_attention_weight_min": -34.1682014465332,
      "activations/layer11_attention_weight_max": 35.124717712402344,
      "activations/layer11_attention_weight_min": -33.4520378112793,
      "activations/layer12_attention_weight_max": 28.736360549926758,
      "activations/layer12_attention_weight_min": -27.603134155273438,
      "activations/layer13_attention_weight_max": 42.55901336669922,
      "activations/layer13_attention_weight_min": -35.58892059326172,
      "activations/layer14_attention_weight_max": 47.35051345825195,
      "activations/layer14_attention_weight_min": -39.92011260986328,
      "activations/layer15_attention_weight_max": 42.467376708984375,
      "activations/layer15_attention_weight_min": -36.79648971557617,
      "activations/layer16_attention_weight_max": 31.139921188354492,
      "activations/layer16_attention_weight_min": -30.65947723388672,
      "activations/layer17_attention_weight_max": 29.508386611938477,
      "activations/layer17_attention_weight_min": -27.453079223632812,
      "activations/layer18_attention_weight_max": 32.833316802978516,
      "activations/layer18_attention_weight_min": -29.321378707885742,
      "activations/layer19_attention_weight_max": 35.1416130065918,
      "activations/layer19_attention_weight_min": -35.278682708740234,
      "activations/layer1_attention_weight_max": 15.564046859741211,
      "activations/layer1_attention_weight_min": -14.61750602722168,
      "activations/layer20_attention_weight_max": 29.107311248779297,
      "activations/layer20_attention_weight_min": -25.50090217590332,
      "activations/layer21_attention_weight_max": 29.254661560058594,
      "activations/layer21_attention_weight_min": -25.541532516479492,
      "activations/layer22_attention_weight_max": 44.559078216552734,
      "activations/layer22_attention_weight_min": -35.324581146240234,
      "activations/layer23_attention_weight_max": 34.963661193847656,
      "activations/layer23_attention_weight_min": -27.99886131286621,
      "activations/layer2_attention_weight_max": 31.7818660736084,
      "activations/layer2_attention_weight_min": -32.01454162597656,
      "activations/layer3_attention_weight_max": 97.32198333740234,
      "activations/layer3_attention_weight_min": -98.9758529663086,
      "activations/layer4_attention_weight_max": 100.88937377929688,
      "activations/layer4_attention_weight_min": -105.9817886352539,
      "activations/layer5_attention_weight_max": 48.636077880859375,
      "activations/layer5_attention_weight_min": -62.11591339111328,
      "activations/layer6_attention_weight_max": 45.37522888183594,
      "activations/layer6_attention_weight_min": -46.85102844238281,
      "activations/layer7_attention_weight_max": 97.09831237792969,
      "activations/layer7_attention_weight_min": -96.64250946044922,
      "activations/layer8_attention_weight_max": 43.94914627075195,
      "activations/layer8_attention_weight_min": -43.25175857543945,
      "activations/layer9_attention_weight_max": 32.10942459106445,
      "activations/layer9_attention_weight_min": -33.63978958129883,
      "epoch": 19.08,
      "learning_rate": 2.7239393939393934e-05,
      "loss": 2.7214,
      "step": 328350
    },
    {
      "activations/layer0_attention_weight_max": 16.220041275024414,
      "activations/layer0_attention_weight_min": -13.905863761901855,
      "activations/layer10_attention_weight_max": 42.68449401855469,
      "activations/layer10_attention_weight_min": -43.18075180053711,
      "activations/layer11_attention_weight_max": 45.89335250854492,
      "activations/layer11_attention_weight_min": -45.056785583496094,
      "activations/layer12_attention_weight_max": 29.220922470092773,
      "activations/layer12_attention_weight_min": -27.380508422851562,
      "activations/layer13_attention_weight_max": 39.6176872253418,
      "activations/layer13_attention_weight_min": -36.181640625,
      "activations/layer14_attention_weight_max": 49.81443786621094,
      "activations/layer14_attention_weight_min": -39.520484924316406,
      "activations/layer15_attention_weight_max": 46.26271057128906,
      "activations/layer15_attention_weight_min": -40.84799575805664,
      "activations/layer16_attention_weight_max": 36.363243103027344,
      "activations/layer16_attention_weight_min": -30.537927627563477,
      "activations/layer17_attention_weight_max": 32.298866271972656,
      "activations/layer17_attention_weight_min": -31.424503326416016,
      "activations/layer18_attention_weight_max": 33.625831604003906,
      "activations/layer18_attention_weight_min": -30.83562660217285,
      "activations/layer19_attention_weight_max": 38.92441940307617,
      "activations/layer19_attention_weight_min": -35.98472213745117,
      "activations/layer1_attention_weight_max": 15.922004699707031,
      "activations/layer1_attention_weight_min": -15.096927642822266,
      "activations/layer20_attention_weight_max": 34.25621032714844,
      "activations/layer20_attention_weight_min": -26.324260711669922,
      "activations/layer21_attention_weight_max": 33.36019515991211,
      "activations/layer21_attention_weight_min": -27.35833740234375,
      "activations/layer22_attention_weight_max": 49.47615051269531,
      "activations/layer22_attention_weight_min": -37.554595947265625,
      "activations/layer23_attention_weight_max": 33.97928237915039,
      "activations/layer23_attention_weight_min": -27.128875732421875,
      "activations/layer2_attention_weight_max": 33.140281677246094,
      "activations/layer2_attention_weight_min": -32.73835754394531,
      "activations/layer3_attention_weight_max": 100.62017059326172,
      "activations/layer3_attention_weight_min": -108.75867462158203,
      "activations/layer4_attention_weight_max": 109.04036712646484,
      "activations/layer4_attention_weight_min": -111.1761703491211,
      "activations/layer5_attention_weight_max": 49.39484786987305,
      "activations/layer5_attention_weight_min": -64.23038482666016,
      "activations/layer6_attention_weight_max": 47.106788635253906,
      "activations/layer6_attention_weight_min": -48.2564811706543,
      "activations/layer7_attention_weight_max": 99.09591674804688,
      "activations/layer7_attention_weight_min": -105.18285369873047,
      "activations/layer8_attention_weight_max": 46.82540512084961,
      "activations/layer8_attention_weight_min": -48.74913024902344,
      "activations/layer9_attention_weight_max": 36.288150787353516,
      "activations/layer9_attention_weight_min": -40.45710372924805,
      "epoch": 19.08,
      "learning_rate": 2.7220454545454543e-05,
      "loss": 2.7345,
      "step": 328400
    },
    {
      "activations/layer0_attention_weight_max": 15.524942398071289,
      "activations/layer0_attention_weight_min": -13.54136848449707,
      "activations/layer10_attention_weight_max": 44.53730773925781,
      "activations/layer10_attention_weight_min": -43.304447174072266,
      "activations/layer11_attention_weight_max": 45.71688461303711,
      "activations/layer11_attention_weight_min": -47.67495346069336,
      "activations/layer12_attention_weight_max": 31.299636840820312,
      "activations/layer12_attention_weight_min": -31.833906173706055,
      "activations/layer13_attention_weight_max": 47.70984649658203,
      "activations/layer13_attention_weight_min": -45.15647506713867,
      "activations/layer14_attention_weight_max": 55.22991180419922,
      "activations/layer14_attention_weight_min": -50.65924072265625,
      "activations/layer15_attention_weight_max": 52.140254974365234,
      "activations/layer15_attention_weight_min": -48.85103225708008,
      "activations/layer16_attention_weight_max": 40.466957092285156,
      "activations/layer16_attention_weight_min": -32.15623474121094,
      "activations/layer17_attention_weight_max": 41.53766632080078,
      "activations/layer17_attention_weight_min": -35.81914520263672,
      "activations/layer18_attention_weight_max": 42.55262756347656,
      "activations/layer18_attention_weight_min": -33.648902893066406,
      "activations/layer19_attention_weight_max": 41.23008728027344,
      "activations/layer19_attention_weight_min": -34.6768913269043,
      "activations/layer1_attention_weight_max": 15.49874210357666,
      "activations/layer1_attention_weight_min": -15.122773170471191,
      "activations/layer20_attention_weight_max": 35.37831497192383,
      "activations/layer20_attention_weight_min": -27.638275146484375,
      "activations/layer21_attention_weight_max": 32.57539749145508,
      "activations/layer21_attention_weight_min": -28.292753219604492,
      "activations/layer22_attention_weight_max": 49.50400161743164,
      "activations/layer22_attention_weight_min": -40.596412658691406,
      "activations/layer23_attention_weight_max": 35.94084930419922,
      "activations/layer23_attention_weight_min": -30.105484008789062,
      "activations/layer2_attention_weight_max": 32.47210693359375,
      "activations/layer2_attention_weight_min": -31.58571434020996,
      "activations/layer3_attention_weight_max": 98.69529724121094,
      "activations/layer3_attention_weight_min": -100.36360168457031,
      "activations/layer4_attention_weight_max": 105.18556213378906,
      "activations/layer4_attention_weight_min": -104.42321014404297,
      "activations/layer5_attention_weight_max": 51.55836486816406,
      "activations/layer5_attention_weight_min": -66.98104858398438,
      "activations/layer6_attention_weight_max": 45.53209686279297,
      "activations/layer6_attention_weight_min": -48.00983810424805,
      "activations/layer7_attention_weight_max": 101.89959716796875,
      "activations/layer7_attention_weight_min": -99.32398223876953,
      "activations/layer8_attention_weight_max": 45.92405700683594,
      "activations/layer8_attention_weight_min": -45.83555221557617,
      "activations/layer9_attention_weight_max": 39.62130355834961,
      "activations/layer9_attention_weight_min": -39.406734466552734,
      "epoch": 19.08,
      "learning_rate": 2.7201515151515148e-05,
      "loss": 2.7153,
      "step": 328450
    },
    {
      "activations/layer0_attention_weight_max": 16.030113220214844,
      "activations/layer0_attention_weight_min": -12.898234367370605,
      "activations/layer10_attention_weight_max": 46.927764892578125,
      "activations/layer10_attention_weight_min": -42.25214385986328,
      "activations/layer11_attention_weight_max": 49.182395935058594,
      "activations/layer11_attention_weight_min": -44.32395553588867,
      "activations/layer12_attention_weight_max": 40.53144454956055,
      "activations/layer12_attention_weight_min": -34.57386779785156,
      "activations/layer13_attention_weight_max": 58.71712112426758,
      "activations/layer13_attention_weight_min": -49.90449523925781,
      "activations/layer14_attention_weight_max": 75.86213684082031,
      "activations/layer14_attention_weight_min": -54.16114044189453,
      "activations/layer15_attention_weight_max": 67.18836975097656,
      "activations/layer15_attention_weight_min": -54.21662139892578,
      "activations/layer16_attention_weight_max": 51.33099365234375,
      "activations/layer16_attention_weight_min": -35.262813568115234,
      "activations/layer17_attention_weight_max": 53.0185661315918,
      "activations/layer17_attention_weight_min": -37.229461669921875,
      "activations/layer18_attention_weight_max": 48.563743591308594,
      "activations/layer18_attention_weight_min": -36.336387634277344,
      "activations/layer19_attention_weight_max": 52.626441955566406,
      "activations/layer19_attention_weight_min": -37.03032302856445,
      "activations/layer1_attention_weight_max": 15.358795166015625,
      "activations/layer1_attention_weight_min": -14.697905540466309,
      "activations/layer20_attention_weight_max": 42.9182243347168,
      "activations/layer20_attention_weight_min": -28.340232849121094,
      "activations/layer21_attention_weight_max": 40.97211837768555,
      "activations/layer21_attention_weight_min": -28.836584091186523,
      "activations/layer22_attention_weight_max": 70.87862396240234,
      "activations/layer22_attention_weight_min": -43.65003967285156,
      "activations/layer23_attention_weight_max": 53.85447311401367,
      "activations/layer23_attention_weight_min": -30.74203872680664,
      "activations/layer2_attention_weight_max": 35.44132995605469,
      "activations/layer2_attention_weight_min": -31.68465805053711,
      "activations/layer3_attention_weight_max": 106.24947357177734,
      "activations/layer3_attention_weight_min": -102.31880950927734,
      "activations/layer4_attention_weight_max": 110.1452865600586,
      "activations/layer4_attention_weight_min": -110.44222259521484,
      "activations/layer5_attention_weight_max": 51.94613265991211,
      "activations/layer5_attention_weight_min": -63.448760986328125,
      "activations/layer6_attention_weight_max": 45.6726188659668,
      "activations/layer6_attention_weight_min": -46.3599739074707,
      "activations/layer7_attention_weight_max": 100.10226440429688,
      "activations/layer7_attention_weight_min": -103.4933853149414,
      "activations/layer8_attention_weight_max": 52.1483039855957,
      "activations/layer8_attention_weight_min": -48.41101837158203,
      "activations/layer9_attention_weight_max": 40.96759796142578,
      "activations/layer9_attention_weight_min": -39.72145080566406,
      "epoch": 19.09,
      "learning_rate": 2.7182575757575756e-05,
      "loss": 2.7266,
      "step": 328500
    },
    {
      "activations/layer0_attention_weight_max": 16.23142433166504,
      "activations/layer0_attention_weight_min": -12.848965644836426,
      "activations/layer10_attention_weight_max": 35.241310119628906,
      "activations/layer10_attention_weight_min": -34.532981872558594,
      "activations/layer11_attention_weight_max": 37.976287841796875,
      "activations/layer11_attention_weight_min": -35.08030700683594,
      "activations/layer12_attention_weight_max": 27.806095123291016,
      "activations/layer12_attention_weight_min": -27.487810134887695,
      "activations/layer13_attention_weight_max": 44.535194396972656,
      "activations/layer13_attention_weight_min": -36.61921310424805,
      "activations/layer14_attention_weight_max": 45.94284439086914,
      "activations/layer14_attention_weight_min": -39.3970832824707,
      "activations/layer15_attention_weight_max": 40.08733367919922,
      "activations/layer15_attention_weight_min": -37.106632232666016,
      "activations/layer16_attention_weight_max": 30.1054630279541,
      "activations/layer16_attention_weight_min": -28.56093406677246,
      "activations/layer17_attention_weight_max": 33.25437927246094,
      "activations/layer17_attention_weight_min": -29.46482276916504,
      "activations/layer18_attention_weight_max": 35.10933303833008,
      "activations/layer18_attention_weight_min": -27.235126495361328,
      "activations/layer19_attention_weight_max": 35.02433395385742,
      "activations/layer19_attention_weight_min": -32.424110412597656,
      "activations/layer1_attention_weight_max": 16.2164363861084,
      "activations/layer1_attention_weight_min": -13.404339790344238,
      "activations/layer20_attention_weight_max": 30.004638671875,
      "activations/layer20_attention_weight_min": -24.700471878051758,
      "activations/layer21_attention_weight_max": 31.176422119140625,
      "activations/layer21_attention_weight_min": -26.12559700012207,
      "activations/layer22_attention_weight_max": 45.49217224121094,
      "activations/layer22_attention_weight_min": -34.53257369995117,
      "activations/layer23_attention_weight_max": 36.953887939453125,
      "activations/layer23_attention_weight_min": -28.297992706298828,
      "activations/layer2_attention_weight_max": 30.25555992126465,
      "activations/layer2_attention_weight_min": -30.7690486907959,
      "activations/layer3_attention_weight_max": 94.04601287841797,
      "activations/layer3_attention_weight_min": -98.00109100341797,
      "activations/layer4_attention_weight_max": 102.93060302734375,
      "activations/layer4_attention_weight_min": -106.63740539550781,
      "activations/layer5_attention_weight_max": 52.82514572143555,
      "activations/layer5_attention_weight_min": -70.4531021118164,
      "activations/layer6_attention_weight_max": 46.69297409057617,
      "activations/layer6_attention_weight_min": -47.15288162231445,
      "activations/layer7_attention_weight_max": 96.26521301269531,
      "activations/layer7_attention_weight_min": -98.89636993408203,
      "activations/layer8_attention_weight_max": 43.83329772949219,
      "activations/layer8_attention_weight_min": -45.51852035522461,
      "activations/layer9_attention_weight_max": 34.52653121948242,
      "activations/layer9_attention_weight_min": -34.09559631347656,
      "epoch": 19.09,
      "learning_rate": 2.716363636363636e-05,
      "loss": 2.7306,
      "step": 328550
    },
    {
      "activations/layer0_attention_weight_max": 17.35637855529785,
      "activations/layer0_attention_weight_min": -13.374467849731445,
      "activations/layer10_attention_weight_max": 35.583984375,
      "activations/layer10_attention_weight_min": -34.77593231201172,
      "activations/layer11_attention_weight_max": 34.71907043457031,
      "activations/layer11_attention_weight_min": -35.84324645996094,
      "activations/layer12_attention_weight_max": 26.510786056518555,
      "activations/layer12_attention_weight_min": -27.523561477661133,
      "activations/layer13_attention_weight_max": 38.06319808959961,
      "activations/layer13_attention_weight_min": -36.04973602294922,
      "activations/layer14_attention_weight_max": 41.75114822387695,
      "activations/layer14_attention_weight_min": -37.86050033569336,
      "activations/layer15_attention_weight_max": 38.62864685058594,
      "activations/layer15_attention_weight_min": -36.08127212524414,
      "activations/layer16_attention_weight_max": 29.859464645385742,
      "activations/layer16_attention_weight_min": -27.985332489013672,
      "activations/layer17_attention_weight_max": 32.20780563354492,
      "activations/layer17_attention_weight_min": -26.36355209350586,
      "activations/layer18_attention_weight_max": 34.82240676879883,
      "activations/layer18_attention_weight_min": -26.135467529296875,
      "activations/layer19_attention_weight_max": 32.62813949584961,
      "activations/layer19_attention_weight_min": -30.9971866607666,
      "activations/layer1_attention_weight_max": 15.344661712646484,
      "activations/layer1_attention_weight_min": -13.563617706298828,
      "activations/layer20_attention_weight_max": 26.546430587768555,
      "activations/layer20_attention_weight_min": -25.131803512573242,
      "activations/layer21_attention_weight_max": 28.118791580200195,
      "activations/layer21_attention_weight_min": -24.9086856842041,
      "activations/layer22_attention_weight_max": 42.58521270751953,
      "activations/layer22_attention_weight_min": -31.39647102355957,
      "activations/layer23_attention_weight_max": 38.1517333984375,
      "activations/layer23_attention_weight_min": -26.121784210205078,
      "activations/layer2_attention_weight_max": 28.586029052734375,
      "activations/layer2_attention_weight_min": -29.51265525817871,
      "activations/layer3_attention_weight_max": 95.76622009277344,
      "activations/layer3_attention_weight_min": -97.853759765625,
      "activations/layer4_attention_weight_max": 104.84891510009766,
      "activations/layer4_attention_weight_min": -110.1313247680664,
      "activations/layer5_attention_weight_max": 52.03779602050781,
      "activations/layer5_attention_weight_min": -61.898643493652344,
      "activations/layer6_attention_weight_max": 44.915523529052734,
      "activations/layer6_attention_weight_min": -47.610843658447266,
      "activations/layer7_attention_weight_max": 97.99773406982422,
      "activations/layer7_attention_weight_min": -97.23175811767578,
      "activations/layer8_attention_weight_max": 46.77837371826172,
      "activations/layer8_attention_weight_min": -48.909542083740234,
      "activations/layer9_attention_weight_max": 32.452491760253906,
      "activations/layer9_attention_weight_min": -35.023887634277344,
      "epoch": 19.09,
      "learning_rate": 2.7145075757575755e-05,
      "loss": 2.7198,
      "step": 328600
    },
    {
      "activations/layer0_attention_weight_max": 17.392581939697266,
      "activations/layer0_attention_weight_min": -13.318506240844727,
      "activations/layer10_attention_weight_max": 34.43646240234375,
      "activations/layer10_attention_weight_min": -33.7791862487793,
      "activations/layer11_attention_weight_max": 34.89759826660156,
      "activations/layer11_attention_weight_min": -35.204689025878906,
      "activations/layer12_attention_weight_max": 28.70781135559082,
      "activations/layer12_attention_weight_min": -27.11069107055664,
      "activations/layer13_attention_weight_max": 40.850337982177734,
      "activations/layer13_attention_weight_min": -38.51317596435547,
      "activations/layer14_attention_weight_max": 43.97367477416992,
      "activations/layer14_attention_weight_min": -40.479087829589844,
      "activations/layer15_attention_weight_max": 42.202396392822266,
      "activations/layer15_attention_weight_min": -35.56999969482422,
      "activations/layer16_attention_weight_max": 33.15864181518555,
      "activations/layer16_attention_weight_min": -28.80331802368164,
      "activations/layer17_attention_weight_max": 29.53553581237793,
      "activations/layer17_attention_weight_min": -27.2014102935791,
      "activations/layer18_attention_weight_max": 34.07183074951172,
      "activations/layer18_attention_weight_min": -27.68203353881836,
      "activations/layer19_attention_weight_max": 35.83937454223633,
      "activations/layer19_attention_weight_min": -31.093679428100586,
      "activations/layer1_attention_weight_max": 15.825765609741211,
      "activations/layer1_attention_weight_min": -13.77373218536377,
      "activations/layer20_attention_weight_max": 30.85307502746582,
      "activations/layer20_attention_weight_min": -25.869661331176758,
      "activations/layer21_attention_weight_max": 29.26959800720215,
      "activations/layer21_attention_weight_min": -24.67828369140625,
      "activations/layer22_attention_weight_max": 41.119903564453125,
      "activations/layer22_attention_weight_min": -31.40410041809082,
      "activations/layer23_attention_weight_max": 31.62006187438965,
      "activations/layer23_attention_weight_min": -25.900556564331055,
      "activations/layer2_attention_weight_max": 29.51207733154297,
      "activations/layer2_attention_weight_min": -29.526832580566406,
      "activations/layer3_attention_weight_max": 95.86467742919922,
      "activations/layer3_attention_weight_min": -95.41155242919922,
      "activations/layer4_attention_weight_max": 100.77526092529297,
      "activations/layer4_attention_weight_min": -104.01493072509766,
      "activations/layer5_attention_weight_max": 50.97663497924805,
      "activations/layer5_attention_weight_min": -59.30389404296875,
      "activations/layer6_attention_weight_max": 43.54350280761719,
      "activations/layer6_attention_weight_min": -45.9625129699707,
      "activations/layer7_attention_weight_max": 91.70712280273438,
      "activations/layer7_attention_weight_min": -93.77804565429688,
      "activations/layer8_attention_weight_max": 40.88150405883789,
      "activations/layer8_attention_weight_min": -42.7612419128418,
      "activations/layer9_attention_weight_max": 32.20730209350586,
      "activations/layer9_attention_weight_min": -35.01676559448242,
      "epoch": 19.1,
      "learning_rate": 2.712651515151515e-05,
      "loss": 2.7096,
      "step": 328650
    },
    {
      "activations/layer0_attention_weight_max": 16.167255401611328,
      "activations/layer0_attention_weight_min": -11.711146354675293,
      "activations/layer10_attention_weight_max": 32.2158088684082,
      "activations/layer10_attention_weight_min": -32.89339828491211,
      "activations/layer11_attention_weight_max": 34.996002197265625,
      "activations/layer11_attention_weight_min": -32.9157600402832,
      "activations/layer12_attention_weight_max": 24.63913345336914,
      "activations/layer12_attention_weight_min": -24.270299911499023,
      "activations/layer13_attention_weight_max": 36.935142517089844,
      "activations/layer13_attention_weight_min": -33.48646926879883,
      "activations/layer14_attention_weight_max": 42.62382888793945,
      "activations/layer14_attention_weight_min": -37.98705291748047,
      "activations/layer15_attention_weight_max": 40.834144592285156,
      "activations/layer15_attention_weight_min": -38.64659118652344,
      "activations/layer16_attention_weight_max": 36.17937469482422,
      "activations/layer16_attention_weight_min": -27.9262752532959,
      "activations/layer17_attention_weight_max": 31.682519912719727,
      "activations/layer17_attention_weight_min": -26.68381690979004,
      "activations/layer18_attention_weight_max": 37.97808074951172,
      "activations/layer18_attention_weight_min": -32.4803581237793,
      "activations/layer19_attention_weight_max": 40.54831314086914,
      "activations/layer19_attention_weight_min": -33.57823944091797,
      "activations/layer1_attention_weight_max": 15.991888046264648,
      "activations/layer1_attention_weight_min": -15.347275733947754,
      "activations/layer20_attention_weight_max": 31.0450382232666,
      "activations/layer20_attention_weight_min": -26.870269775390625,
      "activations/layer21_attention_weight_max": 31.717227935791016,
      "activations/layer21_attention_weight_min": -26.549922943115234,
      "activations/layer22_attention_weight_max": 50.81244659423828,
      "activations/layer22_attention_weight_min": -34.72819137573242,
      "activations/layer23_attention_weight_max": 31.40904998779297,
      "activations/layer23_attention_weight_min": -26.30414581298828,
      "activations/layer2_attention_weight_max": 34.315895080566406,
      "activations/layer2_attention_weight_min": -32.28549575805664,
      "activations/layer3_attention_weight_max": 98.35612487792969,
      "activations/layer3_attention_weight_min": -101.55683898925781,
      "activations/layer4_attention_weight_max": 109.48895263671875,
      "activations/layer4_attention_weight_min": -104.72328186035156,
      "activations/layer5_attention_weight_max": 51.60597229003906,
      "activations/layer5_attention_weight_min": -62.27220916748047,
      "activations/layer6_attention_weight_max": 43.511131286621094,
      "activations/layer6_attention_weight_min": -46.757164001464844,
      "activations/layer7_attention_weight_max": 91.39396667480469,
      "activations/layer7_attention_weight_min": -93.4857406616211,
      "activations/layer8_attention_weight_max": 40.17643356323242,
      "activations/layer8_attention_weight_min": -41.32811737060547,
      "activations/layer9_attention_weight_max": 29.98176383972168,
      "activations/layer9_attention_weight_min": -32.779930114746094,
      "epoch": 19.1,
      "learning_rate": 2.7107575757575754e-05,
      "loss": 2.7336,
      "step": 328700
    },
    {
      "activations/layer0_attention_weight_max": 15.822434425354004,
      "activations/layer0_attention_weight_min": -13.024932861328125,
      "activations/layer10_attention_weight_max": 30.99386215209961,
      "activations/layer10_attention_weight_min": -33.11241149902344,
      "activations/layer11_attention_weight_max": 32.41423797607422,
      "activations/layer11_attention_weight_min": -33.95383834838867,
      "activations/layer12_attention_weight_max": 26.394197463989258,
      "activations/layer12_attention_weight_min": -27.188194274902344,
      "activations/layer13_attention_weight_max": 38.45246887207031,
      "activations/layer13_attention_weight_min": -37.06537628173828,
      "activations/layer14_attention_weight_max": 38.98750686645508,
      "activations/layer14_attention_weight_min": -37.03415298461914,
      "activations/layer15_attention_weight_max": 38.68050765991211,
      "activations/layer15_attention_weight_min": -35.28847885131836,
      "activations/layer16_attention_weight_max": 32.86232376098633,
      "activations/layer16_attention_weight_min": -28.394573211669922,
      "activations/layer17_attention_weight_max": 29.962324142456055,
      "activations/layer17_attention_weight_min": -27.46332359313965,
      "activations/layer18_attention_weight_max": 32.261138916015625,
      "activations/layer18_attention_weight_min": -28.116497039794922,
      "activations/layer19_attention_weight_max": 34.85451889038086,
      "activations/layer19_attention_weight_min": -32.14360046386719,
      "activations/layer1_attention_weight_max": 15.359869956970215,
      "activations/layer1_attention_weight_min": -14.321255683898926,
      "activations/layer20_attention_weight_max": 26.769010543823242,
      "activations/layer20_attention_weight_min": -24.672428131103516,
      "activations/layer21_attention_weight_max": 27.821147918701172,
      "activations/layer21_attention_weight_min": -24.61015510559082,
      "activations/layer22_attention_weight_max": 41.83880615234375,
      "activations/layer22_attention_weight_min": -31.491914749145508,
      "activations/layer23_attention_weight_max": 36.1011848449707,
      "activations/layer23_attention_weight_min": -27.137775421142578,
      "activations/layer2_attention_weight_max": 29.381229400634766,
      "activations/layer2_attention_weight_min": -29.753585815429688,
      "activations/layer3_attention_weight_max": 92.44803619384766,
      "activations/layer3_attention_weight_min": -94.72144317626953,
      "activations/layer4_attention_weight_max": 99.73651885986328,
      "activations/layer4_attention_weight_min": -103.1066665649414,
      "activations/layer5_attention_weight_max": 49.60838317871094,
      "activations/layer5_attention_weight_min": -63.602088928222656,
      "activations/layer6_attention_weight_max": 42.038604736328125,
      "activations/layer6_attention_weight_min": -44.868656158447266,
      "activations/layer7_attention_weight_max": 85.37872314453125,
      "activations/layer7_attention_weight_min": -92.35913848876953,
      "activations/layer8_attention_weight_max": 39.038028717041016,
      "activations/layer8_attention_weight_min": -42.575889587402344,
      "activations/layer9_attention_weight_max": 29.688034057617188,
      "activations/layer9_attention_weight_min": -34.01939010620117,
      "epoch": 19.1,
      "learning_rate": 2.708863636363636e-05,
      "loss": 2.7341,
      "step": 328750
    },
    {
      "activations/layer0_attention_weight_max": 15.89391040802002,
      "activations/layer0_attention_weight_min": -12.923186302185059,
      "activations/layer10_attention_weight_max": 34.604522705078125,
      "activations/layer10_attention_weight_min": -36.9555549621582,
      "activations/layer11_attention_weight_max": 35.125205993652344,
      "activations/layer11_attention_weight_min": -37.21778869628906,
      "activations/layer12_attention_weight_max": 26.5478572845459,
      "activations/layer12_attention_weight_min": -27.0676212310791,
      "activations/layer13_attention_weight_max": 39.10363006591797,
      "activations/layer13_attention_weight_min": -34.94165802001953,
      "activations/layer14_attention_weight_max": 42.439353942871094,
      "activations/layer14_attention_weight_min": -38.22199249267578,
      "activations/layer15_attention_weight_max": 39.8442497253418,
      "activations/layer15_attention_weight_min": -37.396583557128906,
      "activations/layer16_attention_weight_max": 30.19892120361328,
      "activations/layer16_attention_weight_min": -29.250158309936523,
      "activations/layer17_attention_weight_max": 29.09510040283203,
      "activations/layer17_attention_weight_min": -26.532798767089844,
      "activations/layer18_attention_weight_max": 33.864383697509766,
      "activations/layer18_attention_weight_min": -26.930580139160156,
      "activations/layer19_attention_weight_max": 34.333946228027344,
      "activations/layer19_attention_weight_min": -30.786163330078125,
      "activations/layer1_attention_weight_max": 16.543231964111328,
      "activations/layer1_attention_weight_min": -15.35616683959961,
      "activations/layer20_attention_weight_max": 27.69403076171875,
      "activations/layer20_attention_weight_min": -25.345081329345703,
      "activations/layer21_attention_weight_max": 26.232763290405273,
      "activations/layer21_attention_weight_min": -24.3597354888916,
      "activations/layer22_attention_weight_max": 41.94285583496094,
      "activations/layer22_attention_weight_min": -29.84478759765625,
      "activations/layer23_attention_weight_max": 31.72859764099121,
      "activations/layer23_attention_weight_min": -26.982969284057617,
      "activations/layer2_attention_weight_max": 34.17889404296875,
      "activations/layer2_attention_weight_min": -31.950597763061523,
      "activations/layer3_attention_weight_max": 103.2422866821289,
      "activations/layer3_attention_weight_min": -99.0257568359375,
      "activations/layer4_attention_weight_max": 112.8035888671875,
      "activations/layer4_attention_weight_min": -108.36702728271484,
      "activations/layer5_attention_weight_max": 52.07487487792969,
      "activations/layer5_attention_weight_min": -67.8267593383789,
      "activations/layer6_attention_weight_max": 45.0100212097168,
      "activations/layer6_attention_weight_min": -47.723838806152344,
      "activations/layer7_attention_weight_max": 94.72002410888672,
      "activations/layer7_attention_weight_min": -98.80807495117188,
      "activations/layer8_attention_weight_max": 44.77123260498047,
      "activations/layer8_attention_weight_min": -47.399658203125,
      "activations/layer9_attention_weight_max": 33.56709289550781,
      "activations/layer9_attention_weight_min": -36.670799255371094,
      "epoch": 19.11,
      "learning_rate": 2.7069696969696967e-05,
      "loss": 2.7237,
      "step": 328800
    },
    {
      "activations/layer0_attention_weight_max": 17.631017684936523,
      "activations/layer0_attention_weight_min": -12.218103408813477,
      "activations/layer10_attention_weight_max": 33.0814094543457,
      "activations/layer10_attention_weight_min": -34.616703033447266,
      "activations/layer11_attention_weight_max": 32.64015197753906,
      "activations/layer11_attention_weight_min": -34.430084228515625,
      "activations/layer12_attention_weight_max": 28.319787979125977,
      "activations/layer12_attention_weight_min": -28.98196029663086,
      "activations/layer13_attention_weight_max": 40.609718322753906,
      "activations/layer13_attention_weight_min": -36.085208892822266,
      "activations/layer14_attention_weight_max": 46.828819274902344,
      "activations/layer14_attention_weight_min": -40.58046340942383,
      "activations/layer15_attention_weight_max": 41.257503509521484,
      "activations/layer15_attention_weight_min": -36.646480560302734,
      "activations/layer16_attention_weight_max": 35.06663513183594,
      "activations/layer16_attention_weight_min": -30.935192108154297,
      "activations/layer17_attention_weight_max": 31.670719146728516,
      "activations/layer17_attention_weight_min": -26.084388732910156,
      "activations/layer18_attention_weight_max": 33.98014450073242,
      "activations/layer18_attention_weight_min": -26.72049903869629,
      "activations/layer19_attention_weight_max": 38.552608489990234,
      "activations/layer19_attention_weight_min": -32.21976852416992,
      "activations/layer1_attention_weight_max": 15.739466667175293,
      "activations/layer1_attention_weight_min": -14.61626148223877,
      "activations/layer20_attention_weight_max": 31.172130584716797,
      "activations/layer20_attention_weight_min": -26.16087532043457,
      "activations/layer21_attention_weight_max": 29.848827362060547,
      "activations/layer21_attention_weight_min": -26.73025894165039,
      "activations/layer22_attention_weight_max": 39.90635299682617,
      "activations/layer22_attention_weight_min": -33.5612678527832,
      "activations/layer23_attention_weight_max": 30.764623641967773,
      "activations/layer23_attention_weight_min": -27.839977264404297,
      "activations/layer2_attention_weight_max": 30.89046287536621,
      "activations/layer2_attention_weight_min": -30.020652770996094,
      "activations/layer3_attention_weight_max": 96.68553161621094,
      "activations/layer3_attention_weight_min": -94.44276428222656,
      "activations/layer4_attention_weight_max": 100.60101318359375,
      "activations/layer4_attention_weight_min": -101.78514099121094,
      "activations/layer5_attention_weight_max": 49.731712341308594,
      "activations/layer5_attention_weight_min": -58.13243103027344,
      "activations/layer6_attention_weight_max": 41.990203857421875,
      "activations/layer6_attention_weight_min": -44.83983612060547,
      "activations/layer7_attention_weight_max": 89.8398208618164,
      "activations/layer7_attention_weight_min": -93.72692108154297,
      "activations/layer8_attention_weight_max": 40.37471008300781,
      "activations/layer8_attention_weight_min": -42.82815170288086,
      "activations/layer9_attention_weight_max": 30.64137077331543,
      "activations/layer9_attention_weight_min": -33.8140754699707,
      "epoch": 19.11,
      "learning_rate": 2.7050757575757572e-05,
      "loss": 2.7241,
      "step": 328850
    },
    {
      "activations/layer0_attention_weight_max": 16.525876998901367,
      "activations/layer0_attention_weight_min": -13.670745849609375,
      "activations/layer10_attention_weight_max": 38.67464065551758,
      "activations/layer10_attention_weight_min": -37.70387649536133,
      "activations/layer11_attention_weight_max": 36.51359939575195,
      "activations/layer11_attention_weight_min": -37.486961364746094,
      "activations/layer12_attention_weight_max": 27.59307098388672,
      "activations/layer12_attention_weight_min": -29.505762100219727,
      "activations/layer13_attention_weight_max": 40.10358810424805,
      "activations/layer13_attention_weight_min": -37.550262451171875,
      "activations/layer14_attention_weight_max": 40.624507904052734,
      "activations/layer14_attention_weight_min": -38.71961975097656,
      "activations/layer15_attention_weight_max": 40.300899505615234,
      "activations/layer15_attention_weight_min": -37.15066146850586,
      "activations/layer16_attention_weight_max": 30.957706451416016,
      "activations/layer16_attention_weight_min": -29.44939422607422,
      "activations/layer17_attention_weight_max": 28.886865615844727,
      "activations/layer17_attention_weight_min": -25.868343353271484,
      "activations/layer18_attention_weight_max": 32.363426208496094,
      "activations/layer18_attention_weight_min": -26.79384422302246,
      "activations/layer19_attention_weight_max": 37.23241424560547,
      "activations/layer19_attention_weight_min": -31.829099655151367,
      "activations/layer1_attention_weight_max": 16.618684768676758,
      "activations/layer1_attention_weight_min": -14.816670417785645,
      "activations/layer20_attention_weight_max": 31.9898738861084,
      "activations/layer20_attention_weight_min": -27.79808235168457,
      "activations/layer21_attention_weight_max": 28.880544662475586,
      "activations/layer21_attention_weight_min": -26.319076538085938,
      "activations/layer22_attention_weight_max": 41.63779830932617,
      "activations/layer22_attention_weight_min": -32.27299880981445,
      "activations/layer23_attention_weight_max": 33.581295013427734,
      "activations/layer23_attention_weight_min": -24.901363372802734,
      "activations/layer2_attention_weight_max": 33.03948211669922,
      "activations/layer2_attention_weight_min": -31.7218017578125,
      "activations/layer3_attention_weight_max": 100.7120590209961,
      "activations/layer3_attention_weight_min": -99.10491943359375,
      "activations/layer4_attention_weight_max": 108.41661834716797,
      "activations/layer4_attention_weight_min": -109.4927749633789,
      "activations/layer5_attention_weight_max": 52.6486930847168,
      "activations/layer5_attention_weight_min": -61.77193832397461,
      "activations/layer6_attention_weight_max": 44.215614318847656,
      "activations/layer6_attention_weight_min": -45.884498596191406,
      "activations/layer7_attention_weight_max": 97.75165557861328,
      "activations/layer7_attention_weight_min": -96.41256713867188,
      "activations/layer8_attention_weight_max": 45.1531867980957,
      "activations/layer8_attention_weight_min": -45.83032989501953,
      "activations/layer9_attention_weight_max": 35.70594787597656,
      "activations/layer9_attention_weight_min": -37.73291015625,
      "epoch": 19.11,
      "learning_rate": 2.703181818181818e-05,
      "loss": 2.7025,
      "step": 328900
    },
    {
      "activations/layer0_attention_weight_max": 15.34142017364502,
      "activations/layer0_attention_weight_min": -12.396363258361816,
      "activations/layer10_attention_weight_max": 34.536842346191406,
      "activations/layer10_attention_weight_min": -34.496986389160156,
      "activations/layer11_attention_weight_max": 36.020904541015625,
      "activations/layer11_attention_weight_min": -34.58473587036133,
      "activations/layer12_attention_weight_max": 25.708839416503906,
      "activations/layer12_attention_weight_min": -25.67060661315918,
      "activations/layer13_attention_weight_max": 39.48328399658203,
      "activations/layer13_attention_weight_min": -36.054290771484375,
      "activations/layer14_attention_weight_max": 43.857093811035156,
      "activations/layer14_attention_weight_min": -39.76227951049805,
      "activations/layer15_attention_weight_max": 40.52790832519531,
      "activations/layer15_attention_weight_min": -37.48441696166992,
      "activations/layer16_attention_weight_max": 30.198856353759766,
      "activations/layer16_attention_weight_min": -29.06072235107422,
      "activations/layer17_attention_weight_max": 30.738080978393555,
      "activations/layer17_attention_weight_min": -27.20458221435547,
      "activations/layer18_attention_weight_max": 33.51224899291992,
      "activations/layer18_attention_weight_min": -26.71101951599121,
      "activations/layer19_attention_weight_max": 32.8631706237793,
      "activations/layer19_attention_weight_min": -30.89206314086914,
      "activations/layer1_attention_weight_max": 16.945959091186523,
      "activations/layer1_attention_weight_min": -14.125998497009277,
      "activations/layer20_attention_weight_max": 28.392887115478516,
      "activations/layer20_attention_weight_min": -23.56256675720215,
      "activations/layer21_attention_weight_max": 28.26972007751465,
      "activations/layer21_attention_weight_min": -23.716522216796875,
      "activations/layer22_attention_weight_max": 40.189430236816406,
      "activations/layer22_attention_weight_min": -30.46268653869629,
      "activations/layer23_attention_weight_max": 31.971160888671875,
      "activations/layer23_attention_weight_min": -24.785228729248047,
      "activations/layer2_attention_weight_max": 32.180274963378906,
      "activations/layer2_attention_weight_min": -30.520727157592773,
      "activations/layer3_attention_weight_max": 98.47930908203125,
      "activations/layer3_attention_weight_min": -97.89647674560547,
      "activations/layer4_attention_weight_max": 109.06896209716797,
      "activations/layer4_attention_weight_min": -107.83951568603516,
      "activations/layer5_attention_weight_max": 52.978240966796875,
      "activations/layer5_attention_weight_min": -63.8316650390625,
      "activations/layer6_attention_weight_max": 44.334999084472656,
      "activations/layer6_attention_weight_min": -47.83494567871094,
      "activations/layer7_attention_weight_max": 95.89612579345703,
      "activations/layer7_attention_weight_min": -98.59918975830078,
      "activations/layer8_attention_weight_max": 43.68501663208008,
      "activations/layer8_attention_weight_min": -43.36624526977539,
      "activations/layer9_attention_weight_max": 31.738006591796875,
      "activations/layer9_attention_weight_min": -34.846126556396484,
      "epoch": 19.11,
      "learning_rate": 2.701287878787879e-05,
      "loss": 2.727,
      "step": 328950
    },
    {
      "activations/layer0_attention_weight_max": 15.341048240661621,
      "activations/layer0_attention_weight_min": -12.823102951049805,
      "activations/layer10_attention_weight_max": 32.642269134521484,
      "activations/layer10_attention_weight_min": -34.01721954345703,
      "activations/layer11_attention_weight_max": 34.373531341552734,
      "activations/layer11_attention_weight_min": -32.7371940612793,
      "activations/layer12_attention_weight_max": 25.978656768798828,
      "activations/layer12_attention_weight_min": -26.855478286743164,
      "activations/layer13_attention_weight_max": 36.899070739746094,
      "activations/layer13_attention_weight_min": -35.14060592651367,
      "activations/layer14_attention_weight_max": 44.4200439453125,
      "activations/layer14_attention_weight_min": -36.88117218017578,
      "activations/layer15_attention_weight_max": 36.68398666381836,
      "activations/layer15_attention_weight_min": -34.78609848022461,
      "activations/layer16_attention_weight_max": 28.496702194213867,
      "activations/layer16_attention_weight_min": -27.93864631652832,
      "activations/layer17_attention_weight_max": 32.642051696777344,
      "activations/layer17_attention_weight_min": -25.898332595825195,
      "activations/layer18_attention_weight_max": 32.962310791015625,
      "activations/layer18_attention_weight_min": -26.44466781616211,
      "activations/layer19_attention_weight_max": 39.9697380065918,
      "activations/layer19_attention_weight_min": -34.144195556640625,
      "activations/layer1_attention_weight_max": 15.801148414611816,
      "activations/layer1_attention_weight_min": -14.649646759033203,
      "activations/layer20_attention_weight_max": 34.71497344970703,
      "activations/layer20_attention_weight_min": -25.46527671813965,
      "activations/layer21_attention_weight_max": 31.353666305541992,
      "activations/layer21_attention_weight_min": -27.70078468322754,
      "activations/layer22_attention_weight_max": 42.62944412231445,
      "activations/layer22_attention_weight_min": -32.475608825683594,
      "activations/layer23_attention_weight_max": 32.19629669189453,
      "activations/layer23_attention_weight_min": -28.90985870361328,
      "activations/layer2_attention_weight_max": 31.937496185302734,
      "activations/layer2_attention_weight_min": -31.801219940185547,
      "activations/layer3_attention_weight_max": 96.45663452148438,
      "activations/layer3_attention_weight_min": -97.31818389892578,
      "activations/layer4_attention_weight_max": 102.76004791259766,
      "activations/layer4_attention_weight_min": -108.22161102294922,
      "activations/layer5_attention_weight_max": 49.903255462646484,
      "activations/layer5_attention_weight_min": -63.478179931640625,
      "activations/layer6_attention_weight_max": 42.480770111083984,
      "activations/layer6_attention_weight_min": -45.73627853393555,
      "activations/layer7_attention_weight_max": 92.21807861328125,
      "activations/layer7_attention_weight_min": -92.39030456542969,
      "activations/layer8_attention_weight_max": 39.63959503173828,
      "activations/layer8_attention_weight_min": -42.67655563354492,
      "activations/layer9_attention_weight_max": 30.623071670532227,
      "activations/layer9_attention_weight_min": -32.530696868896484,
      "epoch": 19.12,
      "learning_rate": 2.699393939393939e-05,
      "loss": 2.7209,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4902,
      "eval_samples_per_second": 505.762,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4902,
      "eval_openwebtext_samples_per_second": 505.762,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9754,
      "eval_wikitext_samples_per_second": 230.838,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_lambada_loss": 2.3125,
      "eval_lambada_ppl": 10.099642225480054,
      "eval_lambada_runtime": 9.5937,
      "eval_lambada_samples_per_second": 507.52,
      "step": 329000
    },
    {
      "activations/layer0_attention_weight_max": 15.91386890411377,
      "activations/layer0_attention_weight_min": -12.993951797485352,
      "activations/layer10_attention_weight_max": 36.5668830871582,
      "activations/layer10_attention_weight_min": -33.652252197265625,
      "activations/layer11_attention_weight_max": 35.756919860839844,
      "activations/layer11_attention_weight_min": -35.031742095947266,
      "activations/layer12_attention_weight_max": 27.587120056152344,
      "activations/layer12_attention_weight_min": -26.17717933654785,
      "activations/layer13_attention_weight_max": 41.482173919677734,
      "activations/layer13_attention_weight_min": -35.724693298339844,
      "activations/layer14_attention_weight_max": 43.63637924194336,
      "activations/layer14_attention_weight_min": -39.441566467285156,
      "activations/layer15_attention_weight_max": 44.620513916015625,
      "activations/layer15_attention_weight_min": -37.689109802246094,
      "activations/layer16_attention_weight_max": 37.373146057128906,
      "activations/layer16_attention_weight_min": -29.741771697998047,
      "activations/layer17_attention_weight_max": 32.01958084106445,
      "activations/layer17_attention_weight_min": -27.565706253051758,
      "activations/layer18_attention_weight_max": 33.38641357421875,
      "activations/layer18_attention_weight_min": -26.338592529296875,
      "activations/layer19_attention_weight_max": 34.01805877685547,
      "activations/layer19_attention_weight_min": -35.05808639526367,
      "activations/layer1_attention_weight_max": 15.419150352478027,
      "activations/layer1_attention_weight_min": -14.037440299987793,
      "activations/layer20_attention_weight_max": 31.696420669555664,
      "activations/layer20_attention_weight_min": -26.682456970214844,
      "activations/layer21_attention_weight_max": 28.130107879638672,
      "activations/layer21_attention_weight_min": -25.78369140625,
      "activations/layer22_attention_weight_max": 41.466064453125,
      "activations/layer22_attention_weight_min": -32.08222579956055,
      "activations/layer23_attention_weight_max": 32.002052307128906,
      "activations/layer23_attention_weight_min": -24.620697021484375,
      "activations/layer2_attention_weight_max": 30.805646896362305,
      "activations/layer2_attention_weight_min": -29.838848114013672,
      "activations/layer3_attention_weight_max": 95.46066284179688,
      "activations/layer3_attention_weight_min": -95.02006530761719,
      "activations/layer4_attention_weight_max": 102.08670806884766,
      "activations/layer4_attention_weight_min": -104.84571838378906,
      "activations/layer5_attention_weight_max": 48.974159240722656,
      "activations/layer5_attention_weight_min": -62.77809143066406,
      "activations/layer6_attention_weight_max": 42.73198699951172,
      "activations/layer6_attention_weight_min": -46.62632369995117,
      "activations/layer7_attention_weight_max": 94.14877319335938,
      "activations/layer7_attention_weight_min": -96.69454193115234,
      "activations/layer8_attention_weight_max": 43.75297927856445,
      "activations/layer8_attention_weight_min": -43.98601150512695,
      "activations/layer9_attention_weight_max": 33.58939743041992,
      "activations/layer9_attention_weight_min": -32.977027893066406,
      "epoch": 19.12,
      "learning_rate": 2.6975e-05,
      "loss": 2.7221,
      "step": 329050
    },
    {
      "activations/layer0_attention_weight_max": 16.777908325195312,
      "activations/layer0_attention_weight_min": -13.052494049072266,
      "activations/layer10_attention_weight_max": 35.73976135253906,
      "activations/layer10_attention_weight_min": -35.5871696472168,
      "activations/layer11_attention_weight_max": 35.97031784057617,
      "activations/layer11_attention_weight_min": -34.558658599853516,
      "activations/layer12_attention_weight_max": 26.36430549621582,
      "activations/layer12_attention_weight_min": -27.556650161743164,
      "activations/layer13_attention_weight_max": 38.919769287109375,
      "activations/layer13_attention_weight_min": -36.21038818359375,
      "activations/layer14_attention_weight_max": 40.45333480834961,
      "activations/layer14_attention_weight_min": -38.822566986083984,
      "activations/layer15_attention_weight_max": 36.50244903564453,
      "activations/layer15_attention_weight_min": -35.40331268310547,
      "activations/layer16_attention_weight_max": 27.33751106262207,
      "activations/layer16_attention_weight_min": -28.730514526367188,
      "activations/layer17_attention_weight_max": 29.14521026611328,
      "activations/layer17_attention_weight_min": -25.574975967407227,
      "activations/layer18_attention_weight_max": 31.835205078125,
      "activations/layer18_attention_weight_min": -27.59764289855957,
      "activations/layer19_attention_weight_max": 35.120460510253906,
      "activations/layer19_attention_weight_min": -32.641422271728516,
      "activations/layer1_attention_weight_max": 15.87852668762207,
      "activations/layer1_attention_weight_min": -14.84460735321045,
      "activations/layer20_attention_weight_max": 29.725400924682617,
      "activations/layer20_attention_weight_min": -25.040470123291016,
      "activations/layer21_attention_weight_max": 30.045194625854492,
      "activations/layer21_attention_weight_min": -23.87764549255371,
      "activations/layer22_attention_weight_max": 44.15290451049805,
      "activations/layer22_attention_weight_min": -32.22674560546875,
      "activations/layer23_attention_weight_max": 34.85728073120117,
      "activations/layer23_attention_weight_min": -26.507308959960938,
      "activations/layer2_attention_weight_max": 31.289579391479492,
      "activations/layer2_attention_weight_min": -30.246562957763672,
      "activations/layer3_attention_weight_max": 95.18476867675781,
      "activations/layer3_attention_weight_min": -95.78471374511719,
      "activations/layer4_attention_weight_max": 102.97212982177734,
      "activations/layer4_attention_weight_min": -107.24885559082031,
      "activations/layer5_attention_weight_max": 49.725921630859375,
      "activations/layer5_attention_weight_min": -61.394222259521484,
      "activations/layer6_attention_weight_max": 45.95528793334961,
      "activations/layer6_attention_weight_min": -46.54721450805664,
      "activations/layer7_attention_weight_max": 102.6331787109375,
      "activations/layer7_attention_weight_min": -99.32685089111328,
      "activations/layer8_attention_weight_max": 48.97492218017578,
      "activations/layer8_attention_weight_min": -46.59892654418945,
      "activations/layer9_attention_weight_max": 34.54573440551758,
      "activations/layer9_attention_weight_min": -34.46703338623047,
      "epoch": 19.12,
      "learning_rate": 2.6956060606060603e-05,
      "loss": 2.7299,
      "step": 329100
    },
    {
      "activations/layer0_attention_weight_max": 16.856163024902344,
      "activations/layer0_attention_weight_min": -12.840790748596191,
      "activations/layer10_attention_weight_max": 32.953067779541016,
      "activations/layer10_attention_weight_min": -34.01311111450195,
      "activations/layer11_attention_weight_max": 32.98298645019531,
      "activations/layer11_attention_weight_min": -33.699974060058594,
      "activations/layer12_attention_weight_max": 27.92650032043457,
      "activations/layer12_attention_weight_min": -26.056222915649414,
      "activations/layer13_attention_weight_max": 40.530757904052734,
      "activations/layer13_attention_weight_min": -34.61167526245117,
      "activations/layer14_attention_weight_max": 45.80845642089844,
      "activations/layer14_attention_weight_min": -38.20061492919922,
      "activations/layer15_attention_weight_max": 40.869873046875,
      "activations/layer15_attention_weight_min": -36.213829040527344,
      "activations/layer16_attention_weight_max": 30.70110321044922,
      "activations/layer16_attention_weight_min": -27.761653900146484,
      "activations/layer17_attention_weight_max": 31.710567474365234,
      "activations/layer17_attention_weight_min": -25.10144805908203,
      "activations/layer18_attention_weight_max": 34.2794303894043,
      "activations/layer18_attention_weight_min": -26.201871871948242,
      "activations/layer19_attention_weight_max": 34.75836944580078,
      "activations/layer19_attention_weight_min": -30.894155502319336,
      "activations/layer1_attention_weight_max": 15.897106170654297,
      "activations/layer1_attention_weight_min": -14.936200141906738,
      "activations/layer20_attention_weight_max": 29.11623764038086,
      "activations/layer20_attention_weight_min": -24.758268356323242,
      "activations/layer21_attention_weight_max": 27.554861068725586,
      "activations/layer21_attention_weight_min": -25.008264541625977,
      "activations/layer22_attention_weight_max": 42.34685134887695,
      "activations/layer22_attention_weight_min": -30.470172882080078,
      "activations/layer23_attention_weight_max": 30.649417877197266,
      "activations/layer23_attention_weight_min": -26.633346557617188,
      "activations/layer2_attention_weight_max": 33.60857391357422,
      "activations/layer2_attention_weight_min": -32.060691833496094,
      "activations/layer3_attention_weight_max": 98.55279541015625,
      "activations/layer3_attention_weight_min": -100.64500427246094,
      "activations/layer4_attention_weight_max": 106.20426177978516,
      "activations/layer4_attention_weight_min": -109.45586395263672,
      "activations/layer5_attention_weight_max": 51.039546966552734,
      "activations/layer5_attention_weight_min": -62.19715118408203,
      "activations/layer6_attention_weight_max": 43.98310852050781,
      "activations/layer6_attention_weight_min": -47.25122833251953,
      "activations/layer7_attention_weight_max": 94.64804077148438,
      "activations/layer7_attention_weight_min": -94.82281494140625,
      "activations/layer8_attention_weight_max": 40.5897331237793,
      "activations/layer8_attention_weight_min": -42.15023422241211,
      "activations/layer9_attention_weight_max": 32.359920501708984,
      "activations/layer9_attention_weight_min": -34.764007568359375,
      "epoch": 19.13,
      "learning_rate": 2.693712121212121e-05,
      "loss": 2.7132,
      "step": 329150
    },
    {
      "activations/layer0_attention_weight_max": 17.52513313293457,
      "activations/layer0_attention_weight_min": -12.448058128356934,
      "activations/layer10_attention_weight_max": 34.402347564697266,
      "activations/layer10_attention_weight_min": -35.018943786621094,
      "activations/layer11_attention_weight_max": 35.588687896728516,
      "activations/layer11_attention_weight_min": -36.06909942626953,
      "activations/layer12_attention_weight_max": 26.78082847595215,
      "activations/layer12_attention_weight_min": -28.806007385253906,
      "activations/layer13_attention_weight_max": 40.800880432128906,
      "activations/layer13_attention_weight_min": -38.58262252807617,
      "activations/layer14_attention_weight_max": 44.15705490112305,
      "activations/layer14_attention_weight_min": -41.8944091796875,
      "activations/layer15_attention_weight_max": 39.08137130737305,
      "activations/layer15_attention_weight_min": -39.03814697265625,
      "activations/layer16_attention_weight_max": 31.534011840820312,
      "activations/layer16_attention_weight_min": -28.73653221130371,
      "activations/layer17_attention_weight_max": 34.24471664428711,
      "activations/layer17_attention_weight_min": -26.07106590270996,
      "activations/layer18_attention_weight_max": 33.04533767700195,
      "activations/layer18_attention_weight_min": -27.092744827270508,
      "activations/layer19_attention_weight_max": 35.44080352783203,
      "activations/layer19_attention_weight_min": -31.91763687133789,
      "activations/layer1_attention_weight_max": 15.916309356689453,
      "activations/layer1_attention_weight_min": -14.885111808776855,
      "activations/layer20_attention_weight_max": 30.73732566833496,
      "activations/layer20_attention_weight_min": -28.221622467041016,
      "activations/layer21_attention_weight_max": 31.11144256591797,
      "activations/layer21_attention_weight_min": -25.8757266998291,
      "activations/layer22_attention_weight_max": 40.29322052001953,
      "activations/layer22_attention_weight_min": -30.269351959228516,
      "activations/layer23_attention_weight_max": 30.746625900268555,
      "activations/layer23_attention_weight_min": -26.67359161376953,
      "activations/layer2_attention_weight_max": 33.02219772338867,
      "activations/layer2_attention_weight_min": -31.16994285583496,
      "activations/layer3_attention_weight_max": 105.32748413085938,
      "activations/layer3_attention_weight_min": -101.63140106201172,
      "activations/layer4_attention_weight_max": 114.12263488769531,
      "activations/layer4_attention_weight_min": -107.0523452758789,
      "activations/layer5_attention_weight_max": 51.92060470581055,
      "activations/layer5_attention_weight_min": -62.18757629394531,
      "activations/layer6_attention_weight_max": 46.72562789916992,
      "activations/layer6_attention_weight_min": -46.47261428833008,
      "activations/layer7_attention_weight_max": 91.16356658935547,
      "activations/layer7_attention_weight_min": -96.62718963623047,
      "activations/layer8_attention_weight_max": 42.41188049316406,
      "activations/layer8_attention_weight_min": -42.417179107666016,
      "activations/layer9_attention_weight_max": 33.09148025512695,
      "activations/layer9_attention_weight_min": -34.16944122314453,
      "epoch": 19.13,
      "learning_rate": 2.6918181818181816e-05,
      "loss": 2.7216,
      "step": 329200
    },
    {
      "activations/layer0_attention_weight_max": 16.58608055114746,
      "activations/layer0_attention_weight_min": -13.17304515838623,
      "activations/layer10_attention_weight_max": 34.445472717285156,
      "activations/layer10_attention_weight_min": -33.93555450439453,
      "activations/layer11_attention_weight_max": 34.76026916503906,
      "activations/layer11_attention_weight_min": -35.75440216064453,
      "activations/layer12_attention_weight_max": 30.01651382446289,
      "activations/layer12_attention_weight_min": -27.29163932800293,
      "activations/layer13_attention_weight_max": 44.49082565307617,
      "activations/layer13_attention_weight_min": -36.5634765625,
      "activations/layer14_attention_weight_max": 47.45407485961914,
      "activations/layer14_attention_weight_min": -40.05119323730469,
      "activations/layer15_attention_weight_max": 45.874107360839844,
      "activations/layer15_attention_weight_min": -37.51985549926758,
      "activations/layer16_attention_weight_max": 31.085065841674805,
      "activations/layer16_attention_weight_min": -29.902734756469727,
      "activations/layer17_attention_weight_max": 32.797218322753906,
      "activations/layer17_attention_weight_min": -29.738149642944336,
      "activations/layer18_attention_weight_max": 31.951236724853516,
      "activations/layer18_attention_weight_min": -27.699893951416016,
      "activations/layer19_attention_weight_max": 34.3441047668457,
      "activations/layer19_attention_weight_min": -32.10667419433594,
      "activations/layer1_attention_weight_max": 16.63573455810547,
      "activations/layer1_attention_weight_min": -15.481379508972168,
      "activations/layer20_attention_weight_max": 30.16732406616211,
      "activations/layer20_attention_weight_min": -26.247459411621094,
      "activations/layer21_attention_weight_max": 29.869247436523438,
      "activations/layer21_attention_weight_min": -25.23626136779785,
      "activations/layer22_attention_weight_max": 40.46014404296875,
      "activations/layer22_attention_weight_min": -33.0339241027832,
      "activations/layer23_attention_weight_max": 30.932228088378906,
      "activations/layer23_attention_weight_min": -24.73286247253418,
      "activations/layer2_attention_weight_max": 31.63679313659668,
      "activations/layer2_attention_weight_min": -30.6401424407959,
      "activations/layer3_attention_weight_max": 95.54153442382812,
      "activations/layer3_attention_weight_min": -93.66009521484375,
      "activations/layer4_attention_weight_max": 105.23640441894531,
      "activations/layer4_attention_weight_min": -102.56926727294922,
      "activations/layer5_attention_weight_max": 48.30723190307617,
      "activations/layer5_attention_weight_min": -62.632286071777344,
      "activations/layer6_attention_weight_max": 41.275611877441406,
      "activations/layer6_attention_weight_min": -45.0330924987793,
      "activations/layer7_attention_weight_max": 91.72360229492188,
      "activations/layer7_attention_weight_min": -93.70061492919922,
      "activations/layer8_attention_weight_max": 38.5545654296875,
      "activations/layer8_attention_weight_min": -42.02056121826172,
      "activations/layer9_attention_weight_max": 29.815887451171875,
      "activations/layer9_attention_weight_min": -33.69402313232422,
      "epoch": 19.13,
      "learning_rate": 2.6899242424242425e-05,
      "loss": 2.7183,
      "step": 329250
    },
    {
      "activations/layer0_attention_weight_max": 16.79866600036621,
      "activations/layer0_attention_weight_min": -13.299181938171387,
      "activations/layer10_attention_weight_max": 33.8294792175293,
      "activations/layer10_attention_weight_min": -35.49847412109375,
      "activations/layer11_attention_weight_max": 36.3166389465332,
      "activations/layer11_attention_weight_min": -35.671051025390625,
      "activations/layer12_attention_weight_max": 27.0374813079834,
      "activations/layer12_attention_weight_min": -26.701278686523438,
      "activations/layer13_attention_weight_max": 41.04133605957031,
      "activations/layer13_attention_weight_min": -35.62916946411133,
      "activations/layer14_attention_weight_max": 41.75624465942383,
      "activations/layer14_attention_weight_min": -38.10703659057617,
      "activations/layer15_attention_weight_max": 39.3203010559082,
      "activations/layer15_attention_weight_min": -36.208953857421875,
      "activations/layer16_attention_weight_max": 27.41827964782715,
      "activations/layer16_attention_weight_min": -30.22428321838379,
      "activations/layer17_attention_weight_max": 29.875545501708984,
      "activations/layer17_attention_weight_min": -27.22035789489746,
      "activations/layer18_attention_weight_max": 32.19058609008789,
      "activations/layer18_attention_weight_min": -26.018306732177734,
      "activations/layer19_attention_weight_max": 35.87382125854492,
      "activations/layer19_attention_weight_min": -32.38540267944336,
      "activations/layer1_attention_weight_max": 17.06751823425293,
      "activations/layer1_attention_weight_min": -16.211124420166016,
      "activations/layer20_attention_weight_max": 30.728513717651367,
      "activations/layer20_attention_weight_min": -26.600234985351562,
      "activations/layer21_attention_weight_max": 31.97998046875,
      "activations/layer21_attention_weight_min": -26.580480575561523,
      "activations/layer22_attention_weight_max": 43.71738815307617,
      "activations/layer22_attention_weight_min": -32.80156326293945,
      "activations/layer23_attention_weight_max": 34.056358337402344,
      "activations/layer23_attention_weight_min": -26.55071258544922,
      "activations/layer2_attention_weight_max": 31.43292808532715,
      "activations/layer2_attention_weight_min": -31.51071548461914,
      "activations/layer3_attention_weight_max": 98.11602783203125,
      "activations/layer3_attention_weight_min": -98.92069244384766,
      "activations/layer4_attention_weight_max": 105.31441497802734,
      "activations/layer4_attention_weight_min": -109.59619140625,
      "activations/layer5_attention_weight_max": 49.44145965576172,
      "activations/layer5_attention_weight_min": -63.588706970214844,
      "activations/layer6_attention_weight_max": 43.621341705322266,
      "activations/layer6_attention_weight_min": -47.82749557495117,
      "activations/layer7_attention_weight_max": 91.9267807006836,
      "activations/layer7_attention_weight_min": -90.45954895019531,
      "activations/layer8_attention_weight_max": 41.14101028442383,
      "activations/layer8_attention_weight_min": -42.235572814941406,
      "activations/layer9_attention_weight_max": 31.51590347290039,
      "activations/layer9_attention_weight_min": -33.072784423828125,
      "epoch": 19.13,
      "learning_rate": 2.6880303030303026e-05,
      "loss": 2.7331,
      "step": 329300
    },
    {
      "activations/layer0_attention_weight_max": 16.472545623779297,
      "activations/layer0_attention_weight_min": -13.677167892456055,
      "activations/layer10_attention_weight_max": 34.306495666503906,
      "activations/layer10_attention_weight_min": -34.099430084228516,
      "activations/layer11_attention_weight_max": 31.903270721435547,
      "activations/layer11_attention_weight_min": -33.24266052246094,
      "activations/layer12_attention_weight_max": 26.16558074951172,
      "activations/layer12_attention_weight_min": -27.255598068237305,
      "activations/layer13_attention_weight_max": 38.31237030029297,
      "activations/layer13_attention_weight_min": -34.892250061035156,
      "activations/layer14_attention_weight_max": 39.25305938720703,
      "activations/layer14_attention_weight_min": -36.84858322143555,
      "activations/layer15_attention_weight_max": 36.35210418701172,
      "activations/layer15_attention_weight_min": -36.11805725097656,
      "activations/layer16_attention_weight_max": 27.558881759643555,
      "activations/layer16_attention_weight_min": -31.045528411865234,
      "activations/layer17_attention_weight_max": 28.299612045288086,
      "activations/layer17_attention_weight_min": -26.57676887512207,
      "activations/layer18_attention_weight_max": 30.241741180419922,
      "activations/layer18_attention_weight_min": -26.610517501831055,
      "activations/layer19_attention_weight_max": 32.103458404541016,
      "activations/layer19_attention_weight_min": -31.581531524658203,
      "activations/layer1_attention_weight_max": 15.985724449157715,
      "activations/layer1_attention_weight_min": -15.590530395507812,
      "activations/layer20_attention_weight_max": 26.640640258789062,
      "activations/layer20_attention_weight_min": -25.618141174316406,
      "activations/layer21_attention_weight_max": 25.936431884765625,
      "activations/layer21_attention_weight_min": -25.298837661743164,
      "activations/layer22_attention_weight_max": 39.38874053955078,
      "activations/layer22_attention_weight_min": -31.468631744384766,
      "activations/layer23_attention_weight_max": 31.034366607666016,
      "activations/layer23_attention_weight_min": -25.706342697143555,
      "activations/layer2_attention_weight_max": 33.24335479736328,
      "activations/layer2_attention_weight_min": -31.689008712768555,
      "activations/layer3_attention_weight_max": 98.912353515625,
      "activations/layer3_attention_weight_min": -104.42523956298828,
      "activations/layer4_attention_weight_max": 106.4730453491211,
      "activations/layer4_attention_weight_min": -107.6947021484375,
      "activations/layer5_attention_weight_max": 51.82183074951172,
      "activations/layer5_attention_weight_min": -62.60142517089844,
      "activations/layer6_attention_weight_max": 47.75577926635742,
      "activations/layer6_attention_weight_min": -47.882469177246094,
      "activations/layer7_attention_weight_max": 91.89713287353516,
      "activations/layer7_attention_weight_min": -97.40438079833984,
      "activations/layer8_attention_weight_max": 43.50198745727539,
      "activations/layer8_attention_weight_min": -44.434852600097656,
      "activations/layer9_attention_weight_max": 32.96775817871094,
      "activations/layer9_attention_weight_min": -35.06093978881836,
      "epoch": 19.14,
      "learning_rate": 2.6861363636363634e-05,
      "loss": 2.7219,
      "step": 329350
    },
    {
      "activations/layer0_attention_weight_max": 15.561186790466309,
      "activations/layer0_attention_weight_min": -12.997108459472656,
      "activations/layer10_attention_weight_max": 34.44917297363281,
      "activations/layer10_attention_weight_min": -34.795143127441406,
      "activations/layer11_attention_weight_max": 34.12006759643555,
      "activations/layer11_attention_weight_min": -34.2703857421875,
      "activations/layer12_attention_weight_max": 26.663644790649414,
      "activations/layer12_attention_weight_min": -26.505674362182617,
      "activations/layer13_attention_weight_max": 39.31052780151367,
      "activations/layer13_attention_weight_min": -35.503395080566406,
      "activations/layer14_attention_weight_max": 42.978363037109375,
      "activations/layer14_attention_weight_min": -37.53628158569336,
      "activations/layer15_attention_weight_max": 38.69342041015625,
      "activations/layer15_attention_weight_min": -35.10523986816406,
      "activations/layer16_attention_weight_max": 29.14716148376465,
      "activations/layer16_attention_weight_min": -27.527408599853516,
      "activations/layer17_attention_weight_max": 30.162410736083984,
      "activations/layer17_attention_weight_min": -25.97287368774414,
      "activations/layer18_attention_weight_max": 33.7998161315918,
      "activations/layer18_attention_weight_min": -26.43385124206543,
      "activations/layer19_attention_weight_max": 34.87598419189453,
      "activations/layer19_attention_weight_min": -30.813278198242188,
      "activations/layer1_attention_weight_max": 15.890349388122559,
      "activations/layer1_attention_weight_min": -14.968634605407715,
      "activations/layer20_attention_weight_max": 29.22826385498047,
      "activations/layer20_attention_weight_min": -25.145854949951172,
      "activations/layer21_attention_weight_max": 31.16683006286621,
      "activations/layer21_attention_weight_min": -25.306921005249023,
      "activations/layer22_attention_weight_max": 41.8947639465332,
      "activations/layer22_attention_weight_min": -30.94740104675293,
      "activations/layer23_attention_weight_max": 32.62230682373047,
      "activations/layer23_attention_weight_min": -25.484935760498047,
      "activations/layer2_attention_weight_max": 31.498748779296875,
      "activations/layer2_attention_weight_min": -32.12933349609375,
      "activations/layer3_attention_weight_max": 103.08212280273438,
      "activations/layer3_attention_weight_min": -102.20689392089844,
      "activations/layer4_attention_weight_max": 107.517333984375,
      "activations/layer4_attention_weight_min": -108.71809387207031,
      "activations/layer5_attention_weight_max": 51.81501770019531,
      "activations/layer5_attention_weight_min": -62.59943389892578,
      "activations/layer6_attention_weight_max": 44.135704040527344,
      "activations/layer6_attention_weight_min": -50.39206314086914,
      "activations/layer7_attention_weight_max": 92.7676773071289,
      "activations/layer7_attention_weight_min": -104.47508239746094,
      "activations/layer8_attention_weight_max": 41.60106658935547,
      "activations/layer8_attention_weight_min": -44.183597564697266,
      "activations/layer9_attention_weight_max": 31.511499404907227,
      "activations/layer9_attention_weight_min": -34.29877471923828,
      "epoch": 19.14,
      "learning_rate": 2.684242424242424e-05,
      "loss": 2.7197,
      "step": 329400
    },
    {
      "activations/layer0_attention_weight_max": 16.024065017700195,
      "activations/layer0_attention_weight_min": -13.781559944152832,
      "activations/layer10_attention_weight_max": 31.785465240478516,
      "activations/layer10_attention_weight_min": -32.798377990722656,
      "activations/layer11_attention_weight_max": 31.271976470947266,
      "activations/layer11_attention_weight_min": -31.209224700927734,
      "activations/layer12_attention_weight_max": 29.32158851623535,
      "activations/layer12_attention_weight_min": -25.846664428710938,
      "activations/layer13_attention_weight_max": 41.380943298339844,
      "activations/layer13_attention_weight_min": -34.15826416015625,
      "activations/layer14_attention_weight_max": 45.888343811035156,
      "activations/layer14_attention_weight_min": -37.17987823486328,
      "activations/layer15_attention_weight_max": 40.66761779785156,
      "activations/layer15_attention_weight_min": -37.10652542114258,
      "activations/layer16_attention_weight_max": 29.885723114013672,
      "activations/layer16_attention_weight_min": -27.32968521118164,
      "activations/layer17_attention_weight_max": 32.310970306396484,
      "activations/layer17_attention_weight_min": -26.781389236450195,
      "activations/layer18_attention_weight_max": 34.12763214111328,
      "activations/layer18_attention_weight_min": -26.50748634338379,
      "activations/layer19_attention_weight_max": 30.730239868164062,
      "activations/layer19_attention_weight_min": -30.38443946838379,
      "activations/layer1_attention_weight_max": 16.366464614868164,
      "activations/layer1_attention_weight_min": -14.238831520080566,
      "activations/layer20_attention_weight_max": 26.497785568237305,
      "activations/layer20_attention_weight_min": -24.01038932800293,
      "activations/layer21_attention_weight_max": 33.639339447021484,
      "activations/layer21_attention_weight_min": -22.872705459594727,
      "activations/layer22_attention_weight_max": 42.05376434326172,
      "activations/layer22_attention_weight_min": -31.862533569335938,
      "activations/layer23_attention_weight_max": 32.10369873046875,
      "activations/layer23_attention_weight_min": -28.367664337158203,
      "activations/layer2_attention_weight_max": 30.959882736206055,
      "activations/layer2_attention_weight_min": -30.344932556152344,
      "activations/layer3_attention_weight_max": 96.24667358398438,
      "activations/layer3_attention_weight_min": -94.9352035522461,
      "activations/layer4_attention_weight_max": 100.99498748779297,
      "activations/layer4_attention_weight_min": -99.18986511230469,
      "activations/layer5_attention_weight_max": 48.389923095703125,
      "activations/layer5_attention_weight_min": -59.81788635253906,
      "activations/layer6_attention_weight_max": 43.54054260253906,
      "activations/layer6_attention_weight_min": -47.142860412597656,
      "activations/layer7_attention_weight_max": 89.84774780273438,
      "activations/layer7_attention_weight_min": -91.81908416748047,
      "activations/layer8_attention_weight_max": 44.170372009277344,
      "activations/layer8_attention_weight_min": -47.02507781982422,
      "activations/layer9_attention_weight_max": 31.536605834960938,
      "activations/layer9_attention_weight_min": -32.73194885253906,
      "epoch": 19.14,
      "learning_rate": 2.6823484848484848e-05,
      "loss": 2.715,
      "step": 329450
    },
    {
      "activations/layer0_attention_weight_max": 15.92895221710205,
      "activations/layer0_attention_weight_min": -12.70511245727539,
      "activations/layer10_attention_weight_max": 35.33852767944336,
      "activations/layer10_attention_weight_min": -35.952415466308594,
      "activations/layer11_attention_weight_max": 34.598289489746094,
      "activations/layer11_attention_weight_min": -35.750675201416016,
      "activations/layer12_attention_weight_max": 28.13619613647461,
      "activations/layer12_attention_weight_min": -26.542646408081055,
      "activations/layer13_attention_weight_max": 43.976505279541016,
      "activations/layer13_attention_weight_min": -36.59640121459961,
      "activations/layer14_attention_weight_max": 45.347904205322266,
      "activations/layer14_attention_weight_min": -38.00156021118164,
      "activations/layer15_attention_weight_max": 43.453975677490234,
      "activations/layer15_attention_weight_min": -36.63675308227539,
      "activations/layer16_attention_weight_max": 33.12932586669922,
      "activations/layer16_attention_weight_min": -28.782081604003906,
      "activations/layer17_attention_weight_max": 31.866464614868164,
      "activations/layer17_attention_weight_min": -28.287639617919922,
      "activations/layer18_attention_weight_max": 37.607425689697266,
      "activations/layer18_attention_weight_min": -27.519216537475586,
      "activations/layer19_attention_weight_max": 41.56376266479492,
      "activations/layer19_attention_weight_min": -32.272823333740234,
      "activations/layer1_attention_weight_max": 16.39474105834961,
      "activations/layer1_attention_weight_min": -13.522838592529297,
      "activations/layer20_attention_weight_max": 33.20882034301758,
      "activations/layer20_attention_weight_min": -27.906442642211914,
      "activations/layer21_attention_weight_max": 33.866798400878906,
      "activations/layer21_attention_weight_min": -25.8184757232666,
      "activations/layer22_attention_weight_max": 49.97566223144531,
      "activations/layer22_attention_weight_min": -31.872541427612305,
      "activations/layer23_attention_weight_max": 38.55376052856445,
      "activations/layer23_attention_weight_min": -26.470016479492188,
      "activations/layer2_attention_weight_max": 31.647001266479492,
      "activations/layer2_attention_weight_min": -31.06978988647461,
      "activations/layer3_attention_weight_max": 95.3876953125,
      "activations/layer3_attention_weight_min": -97.06356811523438,
      "activations/layer4_attention_weight_max": 103.05448150634766,
      "activations/layer4_attention_weight_min": -104.39686584472656,
      "activations/layer5_attention_weight_max": 49.16383361816406,
      "activations/layer5_attention_weight_min": -62.43662643432617,
      "activations/layer6_attention_weight_max": 47.73589324951172,
      "activations/layer6_attention_weight_min": -48.84178161621094,
      "activations/layer7_attention_weight_max": 98.93124389648438,
      "activations/layer7_attention_weight_min": -95.94578552246094,
      "activations/layer8_attention_weight_max": 41.947872161865234,
      "activations/layer8_attention_weight_min": -44.1559944152832,
      "activations/layer9_attention_weight_max": 33.27524948120117,
      "activations/layer9_attention_weight_min": -34.288909912109375,
      "epoch": 19.15,
      "learning_rate": 2.680454545454545e-05,
      "loss": 2.7215,
      "step": 329500
    },
    {
      "activations/layer0_attention_weight_max": 15.797064781188965,
      "activations/layer0_attention_weight_min": -13.346970558166504,
      "activations/layer10_attention_weight_max": 35.283226013183594,
      "activations/layer10_attention_weight_min": -34.28257751464844,
      "activations/layer11_attention_weight_max": 37.87046432495117,
      "activations/layer11_attention_weight_min": -33.87245178222656,
      "activations/layer12_attention_weight_max": 25.948732376098633,
      "activations/layer12_attention_weight_min": -27.966068267822266,
      "activations/layer13_attention_weight_max": 39.250999450683594,
      "activations/layer13_attention_weight_min": -38.1419792175293,
      "activations/layer14_attention_weight_max": 42.06356430053711,
      "activations/layer14_attention_weight_min": -41.36945343017578,
      "activations/layer15_attention_weight_max": 40.91060256958008,
      "activations/layer15_attention_weight_min": -39.39167022705078,
      "activations/layer16_attention_weight_max": 32.39268493652344,
      "activations/layer16_attention_weight_min": -28.691984176635742,
      "activations/layer17_attention_weight_max": 30.511905670166016,
      "activations/layer17_attention_weight_min": -26.21198081970215,
      "activations/layer18_attention_weight_max": 33.43062973022461,
      "activations/layer18_attention_weight_min": -26.764543533325195,
      "activations/layer19_attention_weight_max": 32.876705169677734,
      "activations/layer19_attention_weight_min": -30.2607479095459,
      "activations/layer1_attention_weight_max": 15.83126163482666,
      "activations/layer1_attention_weight_min": -14.830864906311035,
      "activations/layer20_attention_weight_max": 27.66725730895996,
      "activations/layer20_attention_weight_min": -25.587919235229492,
      "activations/layer21_attention_weight_max": 25.665441513061523,
      "activations/layer21_attention_weight_min": -24.256103515625,
      "activations/layer22_attention_weight_max": 41.29534912109375,
      "activations/layer22_attention_weight_min": -29.89175033569336,
      "activations/layer23_attention_weight_max": 28.14520835876465,
      "activations/layer23_attention_weight_min": -27.15180778503418,
      "activations/layer2_attention_weight_max": 32.38751983642578,
      "activations/layer2_attention_weight_min": -31.569332122802734,
      "activations/layer3_attention_weight_max": 99.40123748779297,
      "activations/layer3_attention_weight_min": -102.35394287109375,
      "activations/layer4_attention_weight_max": 108.29402160644531,
      "activations/layer4_attention_weight_min": -107.62506103515625,
      "activations/layer5_attention_weight_max": 48.20204162597656,
      "activations/layer5_attention_weight_min": -61.640926361083984,
      "activations/layer6_attention_weight_max": 45.20079040527344,
      "activations/layer6_attention_weight_min": -48.44816970825195,
      "activations/layer7_attention_weight_max": 92.14920043945312,
      "activations/layer7_attention_weight_min": -100.02120971679688,
      "activations/layer8_attention_weight_max": 43.135013580322266,
      "activations/layer8_attention_weight_min": -43.46980667114258,
      "activations/layer9_attention_weight_max": 32.406585693359375,
      "activations/layer9_attention_weight_min": -34.27799987792969,
      "epoch": 19.15,
      "learning_rate": 2.6785606060606057e-05,
      "loss": 2.718,
      "step": 329550
    },
    {
      "activations/layer0_attention_weight_max": 15.993156433105469,
      "activations/layer0_attention_weight_min": -12.71896743774414,
      "activations/layer10_attention_weight_max": 33.942413330078125,
      "activations/layer10_attention_weight_min": -32.2222900390625,
      "activations/layer11_attention_weight_max": 34.66035842895508,
      "activations/layer11_attention_weight_min": -32.80702209472656,
      "activations/layer12_attention_weight_max": 27.056568145751953,
      "activations/layer12_attention_weight_min": -26.042770385742188,
      "activations/layer13_attention_weight_max": 39.43165969848633,
      "activations/layer13_attention_weight_min": -34.589073181152344,
      "activations/layer14_attention_weight_max": 41.338958740234375,
      "activations/layer14_attention_weight_min": -37.899383544921875,
      "activations/layer15_attention_weight_max": 41.28238296508789,
      "activations/layer15_attention_weight_min": -35.81058120727539,
      "activations/layer16_attention_weight_max": 28.067119598388672,
      "activations/layer16_attention_weight_min": -27.429983139038086,
      "activations/layer17_attention_weight_max": 28.83424949645996,
      "activations/layer17_attention_weight_min": -25.574533462524414,
      "activations/layer18_attention_weight_max": 31.24986457824707,
      "activations/layer18_attention_weight_min": -25.45171546936035,
      "activations/layer19_attention_weight_max": 32.64207458496094,
      "activations/layer19_attention_weight_min": -33.50971221923828,
      "activations/layer1_attention_weight_max": 16.704265594482422,
      "activations/layer1_attention_weight_min": -15.105758666992188,
      "activations/layer20_attention_weight_max": 29.481332778930664,
      "activations/layer20_attention_weight_min": -26.51445960998535,
      "activations/layer21_attention_weight_max": 30.375350952148438,
      "activations/layer21_attention_weight_min": -25.1320743560791,
      "activations/layer22_attention_weight_max": 38.8004035949707,
      "activations/layer22_attention_weight_min": -30.764638900756836,
      "activations/layer23_attention_weight_max": 32.86189270019531,
      "activations/layer23_attention_weight_min": -25.139890670776367,
      "activations/layer2_attention_weight_max": 32.30149459838867,
      "activations/layer2_attention_weight_min": -30.644832611083984,
      "activations/layer3_attention_weight_max": 97.0401382446289,
      "activations/layer3_attention_weight_min": -93.90084075927734,
      "activations/layer4_attention_weight_max": 105.00389862060547,
      "activations/layer4_attention_weight_min": -103.6849136352539,
      "activations/layer5_attention_weight_max": 54.39220428466797,
      "activations/layer5_attention_weight_min": -61.35020446777344,
      "activations/layer6_attention_weight_max": 44.43973922729492,
      "activations/layer6_attention_weight_min": -47.37446975708008,
      "activations/layer7_attention_weight_max": 97.11700439453125,
      "activations/layer7_attention_weight_min": -95.33216857910156,
      "activations/layer8_attention_weight_max": 43.201316833496094,
      "activations/layer8_attention_weight_min": -42.87697982788086,
      "activations/layer9_attention_weight_max": 31.382644653320312,
      "activations/layer9_attention_weight_min": -33.7484130859375,
      "epoch": 19.15,
      "learning_rate": 2.6766666666666662e-05,
      "loss": 2.7161,
      "step": 329600
    },
    {
      "activations/layer0_attention_weight_max": 16.510286331176758,
      "activations/layer0_attention_weight_min": -13.231830596923828,
      "activations/layer10_attention_weight_max": 36.4318733215332,
      "activations/layer10_attention_weight_min": -38.62113952636719,
      "activations/layer11_attention_weight_max": 35.340667724609375,
      "activations/layer11_attention_weight_min": -38.35249328613281,
      "activations/layer12_attention_weight_max": 26.4836483001709,
      "activations/layer12_attention_weight_min": -28.25984001159668,
      "activations/layer13_attention_weight_max": 40.745296478271484,
      "activations/layer13_attention_weight_min": -38.479366302490234,
      "activations/layer14_attention_weight_max": 43.53103256225586,
      "activations/layer14_attention_weight_min": -41.43486404418945,
      "activations/layer15_attention_weight_max": 39.92597198486328,
      "activations/layer15_attention_weight_min": -39.073814392089844,
      "activations/layer16_attention_weight_max": 30.74088478088379,
      "activations/layer16_attention_weight_min": -28.708555221557617,
      "activations/layer17_attention_weight_max": 31.46586036682129,
      "activations/layer17_attention_weight_min": -25.984445571899414,
      "activations/layer18_attention_weight_max": 32.00481414794922,
      "activations/layer18_attention_weight_min": -26.532176971435547,
      "activations/layer19_attention_weight_max": 32.528560638427734,
      "activations/layer19_attention_weight_min": -32.05067443847656,
      "activations/layer1_attention_weight_max": 16.083484649658203,
      "activations/layer1_attention_weight_min": -14.19597053527832,
      "activations/layer20_attention_weight_max": 29.805076599121094,
      "activations/layer20_attention_weight_min": -26.14264488220215,
      "activations/layer21_attention_weight_max": 29.56491470336914,
      "activations/layer21_attention_weight_min": -26.248435974121094,
      "activations/layer22_attention_weight_max": 39.07078170776367,
      "activations/layer22_attention_weight_min": -31.951950073242188,
      "activations/layer23_attention_weight_max": 31.680591583251953,
      "activations/layer23_attention_weight_min": -26.95400047302246,
      "activations/layer2_attention_weight_max": 31.54749298095703,
      "activations/layer2_attention_weight_min": -31.13411521911621,
      "activations/layer3_attention_weight_max": 98.26368713378906,
      "activations/layer3_attention_weight_min": -101.31365203857422,
      "activations/layer4_attention_weight_max": 107.30594635009766,
      "activations/layer4_attention_weight_min": -108.30599212646484,
      "activations/layer5_attention_weight_max": 49.37590026855469,
      "activations/layer5_attention_weight_min": -61.72538757324219,
      "activations/layer6_attention_weight_max": 45.545127868652344,
      "activations/layer6_attention_weight_min": -48.34398651123047,
      "activations/layer7_attention_weight_max": 97.98614501953125,
      "activations/layer7_attention_weight_min": -103.42642211914062,
      "activations/layer8_attention_weight_max": 43.79637145996094,
      "activations/layer8_attention_weight_min": -47.9817008972168,
      "activations/layer9_attention_weight_max": 34.191097259521484,
      "activations/layer9_attention_weight_min": -37.92805099487305,
      "epoch": 19.15,
      "learning_rate": 2.674772727272727e-05,
      "loss": 2.7285,
      "step": 329650
    },
    {
      "activations/layer0_attention_weight_max": 16.13465118408203,
      "activations/layer0_attention_weight_min": -12.652341842651367,
      "activations/layer10_attention_weight_max": 34.768089294433594,
      "activations/layer10_attention_weight_min": -34.81037902832031,
      "activations/layer11_attention_weight_max": 35.12025833129883,
      "activations/layer11_attention_weight_min": -34.19000244140625,
      "activations/layer12_attention_weight_max": 27.309938430786133,
      "activations/layer12_attention_weight_min": -27.251075744628906,
      "activations/layer13_attention_weight_max": 38.63618087768555,
      "activations/layer13_attention_weight_min": -37.725616455078125,
      "activations/layer14_attention_weight_max": 41.55182647705078,
      "activations/layer14_attention_weight_min": -39.756771087646484,
      "activations/layer15_attention_weight_max": 37.39165115356445,
      "activations/layer15_attention_weight_min": -35.19834899902344,
      "activations/layer16_attention_weight_max": 28.222402572631836,
      "activations/layer16_attention_weight_min": -27.865365982055664,
      "activations/layer17_attention_weight_max": 32.968021392822266,
      "activations/layer17_attention_weight_min": -25.188955307006836,
      "activations/layer18_attention_weight_max": 32.90651321411133,
      "activations/layer18_attention_weight_min": -26.13336753845215,
      "activations/layer19_attention_weight_max": 32.6976432800293,
      "activations/layer19_attention_weight_min": -30.780675888061523,
      "activations/layer1_attention_weight_max": 15.781537055969238,
      "activations/layer1_attention_weight_min": -15.445520401000977,
      "activations/layer20_attention_weight_max": 27.651643753051758,
      "activations/layer20_attention_weight_min": -25.274234771728516,
      "activations/layer21_attention_weight_max": 27.349327087402344,
      "activations/layer21_attention_weight_min": -24.857696533203125,
      "activations/layer22_attention_weight_max": 40.97041702270508,
      "activations/layer22_attention_weight_min": -31.23845672607422,
      "activations/layer23_attention_weight_max": 30.559825897216797,
      "activations/layer23_attention_weight_min": -24.05794906616211,
      "activations/layer2_attention_weight_max": 32.615806579589844,
      "activations/layer2_attention_weight_min": -31.27713394165039,
      "activations/layer3_attention_weight_max": 99.44654083251953,
      "activations/layer3_attention_weight_min": -100.00135040283203,
      "activations/layer4_attention_weight_max": 108.18646240234375,
      "activations/layer4_attention_weight_min": -107.07586669921875,
      "activations/layer5_attention_weight_max": 51.481563568115234,
      "activations/layer5_attention_weight_min": -60.993247985839844,
      "activations/layer6_attention_weight_max": 45.520606994628906,
      "activations/layer6_attention_weight_min": -46.51961898803711,
      "activations/layer7_attention_weight_max": 97.1805648803711,
      "activations/layer7_attention_weight_min": -99.62908935546875,
      "activations/layer8_attention_weight_max": 42.10668182373047,
      "activations/layer8_attention_weight_min": -45.2639045715332,
      "activations/layer9_attention_weight_max": 33.624549865722656,
      "activations/layer9_attention_weight_min": -33.70454025268555,
      "epoch": 19.16,
      "learning_rate": 2.6728787878787875e-05,
      "loss": 2.7297,
      "step": 329700
    },
    {
      "activations/layer0_attention_weight_max": 15.386983871459961,
      "activations/layer0_attention_weight_min": -13.35464859008789,
      "activations/layer10_attention_weight_max": 34.9552001953125,
      "activations/layer10_attention_weight_min": -34.12180709838867,
      "activations/layer11_attention_weight_max": 38.2496337890625,
      "activations/layer11_attention_weight_min": -34.830814361572266,
      "activations/layer12_attention_weight_max": 28.267488479614258,
      "activations/layer12_attention_weight_min": -28.0598201751709,
      "activations/layer13_attention_weight_max": 44.48418045043945,
      "activations/layer13_attention_weight_min": -38.15914535522461,
      "activations/layer14_attention_weight_max": 48.1887092590332,
      "activations/layer14_attention_weight_min": -41.131954193115234,
      "activations/layer15_attention_weight_max": 41.83119583129883,
      "activations/layer15_attention_weight_min": -37.417476654052734,
      "activations/layer16_attention_weight_max": 30.113285064697266,
      "activations/layer16_attention_weight_min": -27.95447540283203,
      "activations/layer17_attention_weight_max": 32.10353469848633,
      "activations/layer17_attention_weight_min": -26.77151870727539,
      "activations/layer18_attention_weight_max": 32.079627990722656,
      "activations/layer18_attention_weight_min": -27.394386291503906,
      "activations/layer19_attention_weight_max": 33.68747329711914,
      "activations/layer19_attention_weight_min": -31.070959091186523,
      "activations/layer1_attention_weight_max": 15.68843936920166,
      "activations/layer1_attention_weight_min": -14.743416786193848,
      "activations/layer20_attention_weight_max": 30.72930908203125,
      "activations/layer20_attention_weight_min": -25.9287166595459,
      "activations/layer21_attention_weight_max": 29.04876708984375,
      "activations/layer21_attention_weight_min": -24.675933837890625,
      "activations/layer22_attention_weight_max": 40.458824157714844,
      "activations/layer22_attention_weight_min": -31.533203125,
      "activations/layer23_attention_weight_max": 34.30668258666992,
      "activations/layer23_attention_weight_min": -26.657264709472656,
      "activations/layer2_attention_weight_max": 30.438695907592773,
      "activations/layer2_attention_weight_min": -29.88593101501465,
      "activations/layer3_attention_weight_max": 94.55085754394531,
      "activations/layer3_attention_weight_min": -95.69921112060547,
      "activations/layer4_attention_weight_max": 102.36284637451172,
      "activations/layer4_attention_weight_min": -102.11327362060547,
      "activations/layer5_attention_weight_max": 49.9842529296875,
      "activations/layer5_attention_weight_min": -62.754207611083984,
      "activations/layer6_attention_weight_max": 43.45199203491211,
      "activations/layer6_attention_weight_min": -47.076419830322266,
      "activations/layer7_attention_weight_max": 94.08514404296875,
      "activations/layer7_attention_weight_min": -94.59973907470703,
      "activations/layer8_attention_weight_max": 40.789093017578125,
      "activations/layer8_attention_weight_min": -44.24728775024414,
      "activations/layer9_attention_weight_max": 32.3764533996582,
      "activations/layer9_attention_weight_min": -34.7691764831543,
      "epoch": 19.16,
      "learning_rate": 2.6709848484848484e-05,
      "loss": 2.7423,
      "step": 329750
    },
    {
      "activations/layer0_attention_weight_max": 15.44600772857666,
      "activations/layer0_attention_weight_min": -13.856912612915039,
      "activations/layer10_attention_weight_max": 36.22583770751953,
      "activations/layer10_attention_weight_min": -33.7685546875,
      "activations/layer11_attention_weight_max": 36.21335983276367,
      "activations/layer11_attention_weight_min": -36.183563232421875,
      "activations/layer12_attention_weight_max": 28.02004623413086,
      "activations/layer12_attention_weight_min": -27.5970401763916,
      "activations/layer13_attention_weight_max": 41.04891586303711,
      "activations/layer13_attention_weight_min": -37.33912658691406,
      "activations/layer14_attention_weight_max": 45.201454162597656,
      "activations/layer14_attention_weight_min": -38.5615119934082,
      "activations/layer15_attention_weight_max": 42.8283576965332,
      "activations/layer15_attention_weight_min": -35.56828689575195,
      "activations/layer16_attention_weight_max": 32.300052642822266,
      "activations/layer16_attention_weight_min": -28.71520233154297,
      "activations/layer17_attention_weight_max": 30.610464096069336,
      "activations/layer17_attention_weight_min": -26.46260643005371,
      "activations/layer18_attention_weight_max": 32.49139404296875,
      "activations/layer18_attention_weight_min": -25.832921981811523,
      "activations/layer19_attention_weight_max": 36.66926574707031,
      "activations/layer19_attention_weight_min": -30.628347396850586,
      "activations/layer1_attention_weight_max": 15.980486869812012,
      "activations/layer1_attention_weight_min": -14.680288314819336,
      "activations/layer20_attention_weight_max": 29.35238265991211,
      "activations/layer20_attention_weight_min": -23.707921981811523,
      "activations/layer21_attention_weight_max": 26.74895477294922,
      "activations/layer21_attention_weight_min": -24.35483741760254,
      "activations/layer22_attention_weight_max": 39.8446159362793,
      "activations/layer22_attention_weight_min": -31.403934478759766,
      "activations/layer23_attention_weight_max": 31.856624603271484,
      "activations/layer23_attention_weight_min": -24.371166229248047,
      "activations/layer2_attention_weight_max": 32.266326904296875,
      "activations/layer2_attention_weight_min": -31.195188522338867,
      "activations/layer3_attention_weight_max": 99.76382446289062,
      "activations/layer3_attention_weight_min": -100.50565338134766,
      "activations/layer4_attention_weight_max": 106.29139709472656,
      "activations/layer4_attention_weight_min": -107.45235443115234,
      "activations/layer5_attention_weight_max": 51.037532806396484,
      "activations/layer5_attention_weight_min": -62.6562614440918,
      "activations/layer6_attention_weight_max": 43.43539810180664,
      "activations/layer6_attention_weight_min": -48.957435607910156,
      "activations/layer7_attention_weight_max": 97.45050811767578,
      "activations/layer7_attention_weight_min": -94.81541442871094,
      "activations/layer8_attention_weight_max": 42.41169738769531,
      "activations/layer8_attention_weight_min": -42.784034729003906,
      "activations/layer9_attention_weight_max": 32.34568786621094,
      "activations/layer9_attention_weight_min": -33.558128356933594,
      "epoch": 19.16,
      "learning_rate": 2.6690909090909092e-05,
      "loss": 2.7194,
      "step": 329800
    },
    {
      "activations/layer0_attention_weight_max": 16.02863883972168,
      "activations/layer0_attention_weight_min": -12.895575523376465,
      "activations/layer10_attention_weight_max": 33.4481201171875,
      "activations/layer10_attention_weight_min": -34.70654296875,
      "activations/layer11_attention_weight_max": 33.56112289428711,
      "activations/layer11_attention_weight_min": -34.84335708618164,
      "activations/layer12_attention_weight_max": 25.19601821899414,
      "activations/layer12_attention_weight_min": -25.797779083251953,
      "activations/layer13_attention_weight_max": 37.133094787597656,
      "activations/layer13_attention_weight_min": -34.07596969604492,
      "activations/layer14_attention_weight_max": 39.081233978271484,
      "activations/layer14_attention_weight_min": -36.485931396484375,
      "activations/layer15_attention_weight_max": 35.742610931396484,
      "activations/layer15_attention_weight_min": -33.6103630065918,
      "activations/layer16_attention_weight_max": 28.608537673950195,
      "activations/layer16_attention_weight_min": -28.390796661376953,
      "activations/layer17_attention_weight_max": 30.038354873657227,
      "activations/layer17_attention_weight_min": -25.533245086669922,
      "activations/layer18_attention_weight_max": 31.04026985168457,
      "activations/layer18_attention_weight_min": -23.972925186157227,
      "activations/layer19_attention_weight_max": 33.95432662963867,
      "activations/layer19_attention_weight_min": -32.80341339111328,
      "activations/layer1_attention_weight_max": 15.261456489562988,
      "activations/layer1_attention_weight_min": -15.245264053344727,
      "activations/layer20_attention_weight_max": 29.215036392211914,
      "activations/layer20_attention_weight_min": -25.858352661132812,
      "activations/layer21_attention_weight_max": 28.715499877929688,
      "activations/layer21_attention_weight_min": -25.112640380859375,
      "activations/layer22_attention_weight_max": 45.425743103027344,
      "activations/layer22_attention_weight_min": -32.7331657409668,
      "activations/layer23_attention_weight_max": 34.975982666015625,
      "activations/layer23_attention_weight_min": -25.796194076538086,
      "activations/layer2_attention_weight_max": 31.728988647460938,
      "activations/layer2_attention_weight_min": -30.962303161621094,
      "activations/layer3_attention_weight_max": 95.84944915771484,
      "activations/layer3_attention_weight_min": -100.5248794555664,
      "activations/layer4_attention_weight_max": 109.00152587890625,
      "activations/layer4_attention_weight_min": -110.47090148925781,
      "activations/layer5_attention_weight_max": 52.472103118896484,
      "activations/layer5_attention_weight_min": -63.63810348510742,
      "activations/layer6_attention_weight_max": 43.735069274902344,
      "activations/layer6_attention_weight_min": -48.204872131347656,
      "activations/layer7_attention_weight_max": 94.40690612792969,
      "activations/layer7_attention_weight_min": -102.73302459716797,
      "activations/layer8_attention_weight_max": 42.603355407714844,
      "activations/layer8_attention_weight_min": -45.53173828125,
      "activations/layer9_attention_weight_max": 31.56912612915039,
      "activations/layer9_attention_weight_min": -35.19045639038086,
      "epoch": 19.17,
      "learning_rate": 2.6671969696969693e-05,
      "loss": 2.7265,
      "step": 329850
    },
    {
      "activations/layer0_attention_weight_max": 16.250349044799805,
      "activations/layer0_attention_weight_min": -12.915377616882324,
      "activations/layer10_attention_weight_max": 38.744171142578125,
      "activations/layer10_attention_weight_min": -39.41795349121094,
      "activations/layer11_attention_weight_max": 38.80252456665039,
      "activations/layer11_attention_weight_min": -37.7546501159668,
      "activations/layer12_attention_weight_max": 32.97807693481445,
      "activations/layer12_attention_weight_min": -30.03679084777832,
      "activations/layer13_attention_weight_max": 49.23221206665039,
      "activations/layer13_attention_weight_min": -39.72393035888672,
      "activations/layer14_attention_weight_max": 50.43280792236328,
      "activations/layer14_attention_weight_min": -41.856475830078125,
      "activations/layer15_attention_weight_max": 45.612640380859375,
      "activations/layer15_attention_weight_min": -38.12830352783203,
      "activations/layer16_attention_weight_max": 34.8547248840332,
      "activations/layer16_attention_weight_min": -28.82197380065918,
      "activations/layer17_attention_weight_max": 34.02788162231445,
      "activations/layer17_attention_weight_min": -25.86811065673828,
      "activations/layer18_attention_weight_max": 35.95748519897461,
      "activations/layer18_attention_weight_min": -25.892282485961914,
      "activations/layer19_attention_weight_max": 39.40172576904297,
      "activations/layer19_attention_weight_min": -31.586389541625977,
      "activations/layer1_attention_weight_max": 15.959568977355957,
      "activations/layer1_attention_weight_min": -13.940276145935059,
      "activations/layer20_attention_weight_max": 33.50364303588867,
      "activations/layer20_attention_weight_min": -25.211851119995117,
      "activations/layer21_attention_weight_max": 32.02042007446289,
      "activations/layer21_attention_weight_min": -26.365697860717773,
      "activations/layer22_attention_weight_max": 47.11626434326172,
      "activations/layer22_attention_weight_min": -32.185096740722656,
      "activations/layer23_attention_weight_max": 34.889373779296875,
      "activations/layer23_attention_weight_min": -26.874618530273438,
      "activations/layer2_attention_weight_max": 30.892990112304688,
      "activations/layer2_attention_weight_min": -30.70626449584961,
      "activations/layer3_attention_weight_max": 97.51677703857422,
      "activations/layer3_attention_weight_min": -99.6278305053711,
      "activations/layer4_attention_weight_max": 105.4072265625,
      "activations/layer4_attention_weight_min": -109.8376693725586,
      "activations/layer5_attention_weight_max": 51.0452995300293,
      "activations/layer5_attention_weight_min": -61.38098907470703,
      "activations/layer6_attention_weight_max": 44.345767974853516,
      "activations/layer6_attention_weight_min": -48.12962341308594,
      "activations/layer7_attention_weight_max": 97.0093994140625,
      "activations/layer7_attention_weight_min": -103.13780975341797,
      "activations/layer8_attention_weight_max": 46.013450622558594,
      "activations/layer8_attention_weight_min": -46.45280456542969,
      "activations/layer9_attention_weight_max": 35.78470993041992,
      "activations/layer9_attention_weight_min": -38.13663101196289,
      "epoch": 19.17,
      "learning_rate": 2.6653030303030302e-05,
      "loss": 2.7158,
      "step": 329900
    },
    {
      "activations/layer0_attention_weight_max": 15.33584213256836,
      "activations/layer0_attention_weight_min": -13.262260437011719,
      "activations/layer10_attention_weight_max": 37.131248474121094,
      "activations/layer10_attention_weight_min": -36.127716064453125,
      "activations/layer11_attention_weight_max": 38.63116455078125,
      "activations/layer11_attention_weight_min": -37.53049087524414,
      "activations/layer12_attention_weight_max": 28.367576599121094,
      "activations/layer12_attention_weight_min": -29.051103591918945,
      "activations/layer13_attention_weight_max": 43.50328826904297,
      "activations/layer13_attention_weight_min": -38.9382209777832,
      "activations/layer14_attention_weight_max": 47.6050910949707,
      "activations/layer14_attention_weight_min": -43.370216369628906,
      "activations/layer15_attention_weight_max": 43.41482162475586,
      "activations/layer15_attention_weight_min": -39.181209564208984,
      "activations/layer16_attention_weight_max": 33.02561569213867,
      "activations/layer16_attention_weight_min": -31.31858253479004,
      "activations/layer17_attention_weight_max": 30.92518424987793,
      "activations/layer17_attention_weight_min": -26.90591812133789,
      "activations/layer18_attention_weight_max": 33.77294158935547,
      "activations/layer18_attention_weight_min": -27.4902400970459,
      "activations/layer19_attention_weight_max": 34.59097671508789,
      "activations/layer19_attention_weight_min": -33.32429122924805,
      "activations/layer1_attention_weight_max": 16.409412384033203,
      "activations/layer1_attention_weight_min": -14.530905723571777,
      "activations/layer20_attention_weight_max": 32.21510696411133,
      "activations/layer20_attention_weight_min": -25.724836349487305,
      "activations/layer21_attention_weight_max": 30.616851806640625,
      "activations/layer21_attention_weight_min": -25.380229949951172,
      "activations/layer22_attention_weight_max": 46.59270095825195,
      "activations/layer22_attention_weight_min": -32.951805114746094,
      "activations/layer23_attention_weight_max": 31.61212158203125,
      "activations/layer23_attention_weight_min": -26.537466049194336,
      "activations/layer2_attention_weight_max": 31.38941192626953,
      "activations/layer2_attention_weight_min": -33.113807678222656,
      "activations/layer3_attention_weight_max": 99.86151885986328,
      "activations/layer3_attention_weight_min": -103.3216781616211,
      "activations/layer4_attention_weight_max": 107.29106903076172,
      "activations/layer4_attention_weight_min": -110.10819244384766,
      "activations/layer5_attention_weight_max": 51.25706481933594,
      "activations/layer5_attention_weight_min": -62.7248649597168,
      "activations/layer6_attention_weight_max": 44.8729133605957,
      "activations/layer6_attention_weight_min": -46.704463958740234,
      "activations/layer7_attention_weight_max": 92.8818588256836,
      "activations/layer7_attention_weight_min": -96.43537902832031,
      "activations/layer8_attention_weight_max": 45.6804084777832,
      "activations/layer8_attention_weight_min": -44.6789665222168,
      "activations/layer9_attention_weight_max": 33.2639274597168,
      "activations/layer9_attention_weight_min": -35.002323150634766,
      "epoch": 19.17,
      "learning_rate": 2.6634090909090907e-05,
      "loss": 2.7223,
      "step": 329950
    },
    {
      "activations/layer0_attention_weight_max": 16.27128791809082,
      "activations/layer0_attention_weight_min": -13.445953369140625,
      "activations/layer10_attention_weight_max": 36.181034088134766,
      "activations/layer10_attention_weight_min": -37.05557632446289,
      "activations/layer11_attention_weight_max": 34.71958541870117,
      "activations/layer11_attention_weight_min": -36.69182586669922,
      "activations/layer12_attention_weight_max": 28.446256637573242,
      "activations/layer12_attention_weight_min": -26.571372985839844,
      "activations/layer13_attention_weight_max": 39.970489501953125,
      "activations/layer13_attention_weight_min": -35.14015579223633,
      "activations/layer14_attention_weight_max": 44.53330612182617,
      "activations/layer14_attention_weight_min": -36.24003601074219,
      "activations/layer15_attention_weight_max": 38.113182067871094,
      "activations/layer15_attention_weight_min": -35.71928405761719,
      "activations/layer16_attention_weight_max": 30.437744140625,
      "activations/layer16_attention_weight_min": -30.34562873840332,
      "activations/layer17_attention_weight_max": 30.361223220825195,
      "activations/layer17_attention_weight_min": -27.042734146118164,
      "activations/layer18_attention_weight_max": 32.372215270996094,
      "activations/layer18_attention_weight_min": -26.011144638061523,
      "activations/layer19_attention_weight_max": 33.8016471862793,
      "activations/layer19_attention_weight_min": -33.011260986328125,
      "activations/layer1_attention_weight_max": 15.4965181350708,
      "activations/layer1_attention_weight_min": -16.421478271484375,
      "activations/layer20_attention_weight_max": 28.833377838134766,
      "activations/layer20_attention_weight_min": -24.966218948364258,
      "activations/layer21_attention_weight_max": 27.824687957763672,
      "activations/layer21_attention_weight_min": -25.290740966796875,
      "activations/layer22_attention_weight_max": 42.0513916015625,
      "activations/layer22_attention_weight_min": -31.003217697143555,
      "activations/layer23_attention_weight_max": 33.13182830810547,
      "activations/layer23_attention_weight_min": -25.68364715576172,
      "activations/layer2_attention_weight_max": 31.38849639892578,
      "activations/layer2_attention_weight_min": -31.255186080932617,
      "activations/layer3_attention_weight_max": 97.9833755493164,
      "activations/layer3_attention_weight_min": -101.68650817871094,
      "activations/layer4_attention_weight_max": 106.4167251586914,
      "activations/layer4_attention_weight_min": -111.93666076660156,
      "activations/layer5_attention_weight_max": 50.68564987182617,
      "activations/layer5_attention_weight_min": -62.459693908691406,
      "activations/layer6_attention_weight_max": 43.46120071411133,
      "activations/layer6_attention_weight_min": -46.98515319824219,
      "activations/layer7_attention_weight_max": 90.48140716552734,
      "activations/layer7_attention_weight_min": -97.04617309570312,
      "activations/layer8_attention_weight_max": 41.851810455322266,
      "activations/layer8_attention_weight_min": -42.87447738647461,
      "activations/layer9_attention_weight_max": 32.83378219604492,
      "activations/layer9_attention_weight_min": -36.01129150390625,
      "epoch": 19.17,
      "learning_rate": 2.6615151515151515e-05,
      "loss": 2.7059,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4793,
      "eval_samples_per_second": 506.409,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4793,
      "eval_openwebtext_samples_per_second": 506.409,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9686,
      "eval_wikitext_samples_per_second": 231.64,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_lambada_loss": 2.314453125,
      "eval_lambada_ppl": 10.119387365287977,
      "eval_lambada_runtime": 9.5546,
      "eval_lambada_samples_per_second": 509.598,
      "step": 330000
    },
    {
      "activations/layer0_attention_weight_max": 16.0245304107666,
      "activations/layer0_attention_weight_min": -12.793259620666504,
      "activations/layer10_attention_weight_max": 36.483306884765625,
      "activations/layer10_attention_weight_min": -36.189598083496094,
      "activations/layer11_attention_weight_max": 36.3041877746582,
      "activations/layer11_attention_weight_min": -36.0558967590332,
      "activations/layer12_attention_weight_max": 29.16978645324707,
      "activations/layer12_attention_weight_min": -27.250192642211914,
      "activations/layer13_attention_weight_max": 42.37489700317383,
      "activations/layer13_attention_weight_min": -36.93191146850586,
      "activations/layer14_attention_weight_max": 47.328392028808594,
      "activations/layer14_attention_weight_min": -39.13747024536133,
      "activations/layer15_attention_weight_max": 42.39371871948242,
      "activations/layer15_attention_weight_min": -39.58392333984375,
      "activations/layer16_attention_weight_max": 31.242578506469727,
      "activations/layer16_attention_weight_min": -28.70555877685547,
      "activations/layer17_attention_weight_max": 31.718692779541016,
      "activations/layer17_attention_weight_min": -27.851964950561523,
      "activations/layer18_attention_weight_max": 33.241947174072266,
      "activations/layer18_attention_weight_min": -26.85380744934082,
      "activations/layer19_attention_weight_max": 39.45473098754883,
      "activations/layer19_attention_weight_min": -32.08436965942383,
      "activations/layer1_attention_weight_max": 15.619970321655273,
      "activations/layer1_attention_weight_min": -14.634689331054688,
      "activations/layer20_attention_weight_max": 31.02916717529297,
      "activations/layer20_attention_weight_min": -25.839509963989258,
      "activations/layer21_attention_weight_max": 30.65476417541504,
      "activations/layer21_attention_weight_min": -24.434635162353516,
      "activations/layer22_attention_weight_max": 45.66533279418945,
      "activations/layer22_attention_weight_min": -31.25506591796875,
      "activations/layer23_attention_weight_max": 30.990272521972656,
      "activations/layer23_attention_weight_min": -25.99434471130371,
      "activations/layer2_attention_weight_max": 30.939823150634766,
      "activations/layer2_attention_weight_min": -30.38386344909668,
      "activations/layer3_attention_weight_max": 94.6326675415039,
      "activations/layer3_attention_weight_min": -95.77156066894531,
      "activations/layer4_attention_weight_max": 99.29146575927734,
      "activations/layer4_attention_weight_min": -102.06048583984375,
      "activations/layer5_attention_weight_max": 49.42285919189453,
      "activations/layer5_attention_weight_min": -61.10471725463867,
      "activations/layer6_attention_weight_max": 42.48875045776367,
      "activations/layer6_attention_weight_min": -45.57373809814453,
      "activations/layer7_attention_weight_max": 93.18109130859375,
      "activations/layer7_attention_weight_min": -93.57523345947266,
      "activations/layer8_attention_weight_max": 42.61463165283203,
      "activations/layer8_attention_weight_min": -44.146568298339844,
      "activations/layer9_attention_weight_max": 31.59286880493164,
      "activations/layer9_attention_weight_min": -36.76395797729492,
      "epoch": 19.18,
      "learning_rate": 2.659621212121212e-05,
      "loss": 2.719,
      "step": 330050
    },
    {
      "activations/layer0_attention_weight_max": 16.12421226501465,
      "activations/layer0_attention_weight_min": -13.767623901367188,
      "activations/layer10_attention_weight_max": 33.36442947387695,
      "activations/layer10_attention_weight_min": -33.822811126708984,
      "activations/layer11_attention_weight_max": 34.60542297363281,
      "activations/layer11_attention_weight_min": -33.137962341308594,
      "activations/layer12_attention_weight_max": 25.094196319580078,
      "activations/layer12_attention_weight_min": -28.21820640563965,
      "activations/layer13_attention_weight_max": 39.46220016479492,
      "activations/layer13_attention_weight_min": -37.902618408203125,
      "activations/layer14_attention_weight_max": 42.207035064697266,
      "activations/layer14_attention_weight_min": -38.32065963745117,
      "activations/layer15_attention_weight_max": 37.01152420043945,
      "activations/layer15_attention_weight_min": -34.835147857666016,
      "activations/layer16_attention_weight_max": 28.119415283203125,
      "activations/layer16_attention_weight_min": -27.6564884185791,
      "activations/layer17_attention_weight_max": 28.615394592285156,
      "activations/layer17_attention_weight_min": -27.130887985229492,
      "activations/layer18_attention_weight_max": 33.09109878540039,
      "activations/layer18_attention_weight_min": -27.082942962646484,
      "activations/layer19_attention_weight_max": 35.54384994506836,
      "activations/layer19_attention_weight_min": -33.07002258300781,
      "activations/layer1_attention_weight_max": 16.7899169921875,
      "activations/layer1_attention_weight_min": -13.397941589355469,
      "activations/layer20_attention_weight_max": 29.066755294799805,
      "activations/layer20_attention_weight_min": -25.609195709228516,
      "activations/layer21_attention_weight_max": 29.049123764038086,
      "activations/layer21_attention_weight_min": -25.243547439575195,
      "activations/layer22_attention_weight_max": 40.840396881103516,
      "activations/layer22_attention_weight_min": -33.0673942565918,
      "activations/layer23_attention_weight_max": 35.68537139892578,
      "activations/layer23_attention_weight_min": -28.443401336669922,
      "activations/layer2_attention_weight_max": 32.20891571044922,
      "activations/layer2_attention_weight_min": -32.649471282958984,
      "activations/layer3_attention_weight_max": 96.84558868408203,
      "activations/layer3_attention_weight_min": -101.47438049316406,
      "activations/layer4_attention_weight_max": 99.14533996582031,
      "activations/layer4_attention_weight_min": -102.59651947021484,
      "activations/layer5_attention_weight_max": 49.824310302734375,
      "activations/layer5_attention_weight_min": -64.03863525390625,
      "activations/layer6_attention_weight_max": 42.67843246459961,
      "activations/layer6_attention_weight_min": -45.4371223449707,
      "activations/layer7_attention_weight_max": 94.07878875732422,
      "activations/layer7_attention_weight_min": -93.24991607666016,
      "activations/layer8_attention_weight_max": 40.81471633911133,
      "activations/layer8_attention_weight_min": -45.1828498840332,
      "activations/layer9_attention_weight_max": 31.816701889038086,
      "activations/layer9_attention_weight_min": -35.49855041503906,
      "epoch": 19.18,
      "learning_rate": 2.6577272727272725e-05,
      "loss": 2.7295,
      "step": 330100
    },
    {
      "activations/layer0_attention_weight_max": 15.765509605407715,
      "activations/layer0_attention_weight_min": -15.02469539642334,
      "activations/layer10_attention_weight_max": 36.40578079223633,
      "activations/layer10_attention_weight_min": -36.17047119140625,
      "activations/layer11_attention_weight_max": 34.43701171875,
      "activations/layer11_attention_weight_min": -35.72060775756836,
      "activations/layer12_attention_weight_max": 26.057064056396484,
      "activations/layer12_attention_weight_min": -27.687902450561523,
      "activations/layer13_attention_weight_max": 37.83649444580078,
      "activations/layer13_attention_weight_min": -37.32112121582031,
      "activations/layer14_attention_weight_max": 40.34657287597656,
      "activations/layer14_attention_weight_min": -38.673851013183594,
      "activations/layer15_attention_weight_max": 38.56077194213867,
      "activations/layer15_attention_weight_min": -40.32988739013672,
      "activations/layer16_attention_weight_max": 27.730287551879883,
      "activations/layer16_attention_weight_min": -27.78573226928711,
      "activations/layer17_attention_weight_max": 28.914167404174805,
      "activations/layer17_attention_weight_min": -26.577625274658203,
      "activations/layer18_attention_weight_max": 32.16897964477539,
      "activations/layer18_attention_weight_min": -26.01200294494629,
      "activations/layer19_attention_weight_max": 31.993377685546875,
      "activations/layer19_attention_weight_min": -30.570648193359375,
      "activations/layer1_attention_weight_max": 15.994542121887207,
      "activations/layer1_attention_weight_min": -15.690662384033203,
      "activations/layer20_attention_weight_max": 25.821327209472656,
      "activations/layer20_attention_weight_min": -26.83001708984375,
      "activations/layer21_attention_weight_max": 24.378326416015625,
      "activations/layer21_attention_weight_min": -25.152347564697266,
      "activations/layer22_attention_weight_max": 38.82624053955078,
      "activations/layer22_attention_weight_min": -31.85242462158203,
      "activations/layer23_attention_weight_max": 32.2424430847168,
      "activations/layer23_attention_weight_min": -26.16726303100586,
      "activations/layer2_attention_weight_max": 34.70880889892578,
      "activations/layer2_attention_weight_min": -32.58415222167969,
      "activations/layer3_attention_weight_max": 102.14492797851562,
      "activations/layer3_attention_weight_min": -102.73734283447266,
      "activations/layer4_attention_weight_max": 107.2332763671875,
      "activations/layer4_attention_weight_min": -112.2105712890625,
      "activations/layer5_attention_weight_max": 52.48783874511719,
      "activations/layer5_attention_weight_min": -65.81278991699219,
      "activations/layer6_attention_weight_max": 45.96200942993164,
      "activations/layer6_attention_weight_min": -47.46715545654297,
      "activations/layer7_attention_weight_max": 97.4906005859375,
      "activations/layer7_attention_weight_min": -101.51334381103516,
      "activations/layer8_attention_weight_max": 42.83481216430664,
      "activations/layer8_attention_weight_min": -44.444766998291016,
      "activations/layer9_attention_weight_max": 32.41495132446289,
      "activations/layer9_attention_weight_min": -33.651580810546875,
      "epoch": 19.18,
      "learning_rate": 2.655833333333333e-05,
      "loss": 2.7108,
      "step": 330150
    },
    {
      "activations/layer0_attention_weight_max": 16.848201751708984,
      "activations/layer0_attention_weight_min": -13.185617446899414,
      "activations/layer10_attention_weight_max": 37.106040954589844,
      "activations/layer10_attention_weight_min": -36.94545364379883,
      "activations/layer11_attention_weight_max": 38.52702713012695,
      "activations/layer11_attention_weight_min": -36.30265808105469,
      "activations/layer12_attention_weight_max": 30.468229293823242,
      "activations/layer12_attention_weight_min": -27.163040161132812,
      "activations/layer13_attention_weight_max": 41.9237060546875,
      "activations/layer13_attention_weight_min": -37.45186996459961,
      "activations/layer14_attention_weight_max": 46.623260498046875,
      "activations/layer14_attention_weight_min": -38.76496124267578,
      "activations/layer15_attention_weight_max": 42.175437927246094,
      "activations/layer15_attention_weight_min": -36.574127197265625,
      "activations/layer16_attention_weight_max": 31.5516357421875,
      "activations/layer16_attention_weight_min": -27.587366104125977,
      "activations/layer17_attention_weight_max": 32.875946044921875,
      "activations/layer17_attention_weight_min": -24.886669158935547,
      "activations/layer18_attention_weight_max": 32.689754486083984,
      "activations/layer18_attention_weight_min": -25.469934463500977,
      "activations/layer19_attention_weight_max": 34.080196380615234,
      "activations/layer19_attention_weight_min": -31.758886337280273,
      "activations/layer1_attention_weight_max": 15.930468559265137,
      "activations/layer1_attention_weight_min": -14.248571395874023,
      "activations/layer20_attention_weight_max": 30.872657775878906,
      "activations/layer20_attention_weight_min": -24.28851890563965,
      "activations/layer21_attention_weight_max": 28.108901977539062,
      "activations/layer21_attention_weight_min": -24.67970848083496,
      "activations/layer22_attention_weight_max": 47.65557098388672,
      "activations/layer22_attention_weight_min": -32.8480110168457,
      "activations/layer23_attention_weight_max": 35.667579650878906,
      "activations/layer23_attention_weight_min": -26.21564292907715,
      "activations/layer2_attention_weight_max": 32.62204360961914,
      "activations/layer2_attention_weight_min": -32.41600799560547,
      "activations/layer3_attention_weight_max": 101.35187530517578,
      "activations/layer3_attention_weight_min": -101.00184631347656,
      "activations/layer4_attention_weight_max": 110.27772521972656,
      "activations/layer4_attention_weight_min": -112.51163482666016,
      "activations/layer5_attention_weight_max": 52.45359802246094,
      "activations/layer5_attention_weight_min": -61.553382873535156,
      "activations/layer6_attention_weight_max": 45.17292404174805,
      "activations/layer6_attention_weight_min": -47.164390563964844,
      "activations/layer7_attention_weight_max": 104.69776916503906,
      "activations/layer7_attention_weight_min": -105.87290954589844,
      "activations/layer8_attention_weight_max": 46.95204544067383,
      "activations/layer8_attention_weight_min": -47.84560012817383,
      "activations/layer9_attention_weight_max": 36.469600677490234,
      "activations/layer9_attention_weight_min": -36.43198776245117,
      "epoch": 19.19,
      "learning_rate": 2.6539393939393938e-05,
      "loss": 2.6874,
      "step": 330200
    },
    {
      "activations/layer0_attention_weight_max": 16.357906341552734,
      "activations/layer0_attention_weight_min": -14.161486625671387,
      "activations/layer10_attention_weight_max": 46.65421676635742,
      "activations/layer10_attention_weight_min": -45.934173583984375,
      "activations/layer11_attention_weight_max": 48.67149353027344,
      "activations/layer11_attention_weight_min": -46.947261810302734,
      "activations/layer12_attention_weight_max": 35.780609130859375,
      "activations/layer12_attention_weight_min": -32.51273727416992,
      "activations/layer13_attention_weight_max": 44.27113723754883,
      "activations/layer13_attention_weight_min": -42.84299087524414,
      "activations/layer14_attention_weight_max": 48.78816223144531,
      "activations/layer14_attention_weight_min": -47.12144088745117,
      "activations/layer15_attention_weight_max": 46.390865325927734,
      "activations/layer15_attention_weight_min": -44.60460662841797,
      "activations/layer16_attention_weight_max": 35.3931884765625,
      "activations/layer16_attention_weight_min": -28.482742309570312,
      "activations/layer17_attention_weight_max": 30.937076568603516,
      "activations/layer17_attention_weight_min": -28.173410415649414,
      "activations/layer18_attention_weight_max": 31.554819107055664,
      "activations/layer18_attention_weight_min": -29.081504821777344,
      "activations/layer19_attention_weight_max": 34.91032028198242,
      "activations/layer19_attention_weight_min": -31.880002975463867,
      "activations/layer1_attention_weight_max": 16.274202346801758,
      "activations/layer1_attention_weight_min": -14.399605751037598,
      "activations/layer20_attention_weight_max": 27.355749130249023,
      "activations/layer20_attention_weight_min": -25.968608856201172,
      "activations/layer21_attention_weight_max": 28.71590232849121,
      "activations/layer21_attention_weight_min": -26.402999877929688,
      "activations/layer22_attention_weight_max": 38.74641036987305,
      "activations/layer22_attention_weight_min": -36.74905014038086,
      "activations/layer23_attention_weight_max": 31.218528747558594,
      "activations/layer23_attention_weight_min": -27.846769332885742,
      "activations/layer2_attention_weight_max": 33.8582649230957,
      "activations/layer2_attention_weight_min": -33.26458740234375,
      "activations/layer3_attention_weight_max": 104.93878936767578,
      "activations/layer3_attention_weight_min": -107.29205322265625,
      "activations/layer4_attention_weight_max": 110.63520812988281,
      "activations/layer4_attention_weight_min": -109.4644775390625,
      "activations/layer5_attention_weight_max": 52.286861419677734,
      "activations/layer5_attention_weight_min": -61.0009765625,
      "activations/layer6_attention_weight_max": 49.99135971069336,
      "activations/layer6_attention_weight_min": -50.085731506347656,
      "activations/layer7_attention_weight_max": 119.47334289550781,
      "activations/layer7_attention_weight_min": -114.49652099609375,
      "activations/layer8_attention_weight_max": 53.773101806640625,
      "activations/layer8_attention_weight_min": -54.687705993652344,
      "activations/layer9_attention_weight_max": 42.32667922973633,
      "activations/layer9_attention_weight_min": -43.83002853393555,
      "epoch": 19.19,
      "learning_rate": 2.6520454545454543e-05,
      "loss": 2.7202,
      "step": 330250
    },
    {
      "activations/layer0_attention_weight_max": 16.24437713623047,
      "activations/layer0_attention_weight_min": -13.57699203491211,
      "activations/layer10_attention_weight_max": 34.74794387817383,
      "activations/layer10_attention_weight_min": -36.58740234375,
      "activations/layer11_attention_weight_max": 34.92901611328125,
      "activations/layer11_attention_weight_min": -35.55663299560547,
      "activations/layer12_attention_weight_max": 27.7180118560791,
      "activations/layer12_attention_weight_min": -26.42119026184082,
      "activations/layer13_attention_weight_max": 39.70536804199219,
      "activations/layer13_attention_weight_min": -36.87151336669922,
      "activations/layer14_attention_weight_max": 40.804832458496094,
      "activations/layer14_attention_weight_min": -37.56382369995117,
      "activations/layer15_attention_weight_max": 39.13767623901367,
      "activations/layer15_attention_weight_min": -35.451438903808594,
      "activations/layer16_attention_weight_max": 32.01482009887695,
      "activations/layer16_attention_weight_min": -29.62100601196289,
      "activations/layer17_attention_weight_max": 30.737640380859375,
      "activations/layer17_attention_weight_min": -26.563678741455078,
      "activations/layer18_attention_weight_max": 34.4810676574707,
      "activations/layer18_attention_weight_min": -25.801673889160156,
      "activations/layer19_attention_weight_max": 34.64895248413086,
      "activations/layer19_attention_weight_min": -31.2594051361084,
      "activations/layer1_attention_weight_max": 15.973921775817871,
      "activations/layer1_attention_weight_min": -15.316228866577148,
      "activations/layer20_attention_weight_max": 29.47162437438965,
      "activations/layer20_attention_weight_min": -24.17803192138672,
      "activations/layer21_attention_weight_max": 30.845428466796875,
      "activations/layer21_attention_weight_min": -24.4390869140625,
      "activations/layer22_attention_weight_max": 39.36587905883789,
      "activations/layer22_attention_weight_min": -30.368608474731445,
      "activations/layer23_attention_weight_max": 31.282270431518555,
      "activations/layer23_attention_weight_min": -27.151691436767578,
      "activations/layer2_attention_weight_max": 32.67979431152344,
      "activations/layer2_attention_weight_min": -31.529075622558594,
      "activations/layer3_attention_weight_max": 97.01585388183594,
      "activations/layer3_attention_weight_min": -98.04865264892578,
      "activations/layer4_attention_weight_max": 106.38581848144531,
      "activations/layer4_attention_weight_min": -115.83782196044922,
      "activations/layer5_attention_weight_max": 52.976898193359375,
      "activations/layer5_attention_weight_min": -64.78471374511719,
      "activations/layer6_attention_weight_max": 45.67728805541992,
      "activations/layer6_attention_weight_min": -50.36485290527344,
      "activations/layer7_attention_weight_max": 96.67110443115234,
      "activations/layer7_attention_weight_min": -99.89259338378906,
      "activations/layer8_attention_weight_max": 42.840511322021484,
      "activations/layer8_attention_weight_min": -45.481231689453125,
      "activations/layer9_attention_weight_max": 33.830562591552734,
      "activations/layer9_attention_weight_min": -36.389678955078125,
      "epoch": 19.19,
      "learning_rate": 2.650151515151515e-05,
      "loss": 2.7092,
      "step": 330300
    },
    {
      "activations/layer0_attention_weight_max": 15.900439262390137,
      "activations/layer0_attention_weight_min": -13.068402290344238,
      "activations/layer10_attention_weight_max": 34.17389678955078,
      "activations/layer10_attention_weight_min": -34.860374450683594,
      "activations/layer11_attention_weight_max": 34.187232971191406,
      "activations/layer11_attention_weight_min": -35.66200637817383,
      "activations/layer12_attention_weight_max": 27.76012420654297,
      "activations/layer12_attention_weight_min": -28.799861907958984,
      "activations/layer13_attention_weight_max": 40.96921920776367,
      "activations/layer13_attention_weight_min": -36.83073043823242,
      "activations/layer14_attention_weight_max": 43.57432174682617,
      "activations/layer14_attention_weight_min": -39.3292121887207,
      "activations/layer15_attention_weight_max": 39.93354415893555,
      "activations/layer15_attention_weight_min": -36.2510871887207,
      "activations/layer16_attention_weight_max": 31.860506057739258,
      "activations/layer16_attention_weight_min": -26.900705337524414,
      "activations/layer17_attention_weight_max": 31.491737365722656,
      "activations/layer17_attention_weight_min": -26.418745040893555,
      "activations/layer18_attention_weight_max": 32.552764892578125,
      "activations/layer18_attention_weight_min": -25.554927825927734,
      "activations/layer19_attention_weight_max": 38.314208984375,
      "activations/layer19_attention_weight_min": -31.3226261138916,
      "activations/layer1_attention_weight_max": 15.13007640838623,
      "activations/layer1_attention_weight_min": -14.1654634475708,
      "activations/layer20_attention_weight_max": 32.16349792480469,
      "activations/layer20_attention_weight_min": -25.67412757873535,
      "activations/layer21_attention_weight_max": 31.01654624938965,
      "activations/layer21_attention_weight_min": -25.6638126373291,
      "activations/layer22_attention_weight_max": 42.222816467285156,
      "activations/layer22_attention_weight_min": -33.4362907409668,
      "activations/layer23_attention_weight_max": 37.15406036376953,
      "activations/layer23_attention_weight_min": -27.15671157836914,
      "activations/layer2_attention_weight_max": 32.949317932128906,
      "activations/layer2_attention_weight_min": -34.74510955810547,
      "activations/layer3_attention_weight_max": 101.24547576904297,
      "activations/layer3_attention_weight_min": -107.7217788696289,
      "activations/layer4_attention_weight_max": 106.61571502685547,
      "activations/layer4_attention_weight_min": -110.8226547241211,
      "activations/layer5_attention_weight_max": 50.90885543823242,
      "activations/layer5_attention_weight_min": -61.36446762084961,
      "activations/layer6_attention_weight_max": 45.03785705566406,
      "activations/layer6_attention_weight_min": -48.29673385620117,
      "activations/layer7_attention_weight_max": 96.23387145996094,
      "activations/layer7_attention_weight_min": -98.10842895507812,
      "activations/layer8_attention_weight_max": 41.63121032714844,
      "activations/layer8_attention_weight_min": -46.30147171020508,
      "activations/layer9_attention_weight_max": 33.13625717163086,
      "activations/layer9_attention_weight_min": -35.1888542175293,
      "epoch": 19.2,
      "learning_rate": 2.6482575757575752e-05,
      "loss": 2.702,
      "step": 330350
    },
    {
      "activations/layer0_attention_weight_max": 16.195981979370117,
      "activations/layer0_attention_weight_min": -12.033690452575684,
      "activations/layer10_attention_weight_max": 33.856292724609375,
      "activations/layer10_attention_weight_min": -32.82992172241211,
      "activations/layer11_attention_weight_max": 34.15266036987305,
      "activations/layer11_attention_weight_min": -33.10992431640625,
      "activations/layer12_attention_weight_max": 24.98122787475586,
      "activations/layer12_attention_weight_min": -26.509111404418945,
      "activations/layer13_attention_weight_max": 37.857872009277344,
      "activations/layer13_attention_weight_min": -35.99396514892578,
      "activations/layer14_attention_weight_max": 40.820350646972656,
      "activations/layer14_attention_weight_min": -40.545169830322266,
      "activations/layer15_attention_weight_max": 39.36195755004883,
      "activations/layer15_attention_weight_min": -36.33914566040039,
      "activations/layer16_attention_weight_max": 29.323223114013672,
      "activations/layer16_attention_weight_min": -28.88237762451172,
      "activations/layer17_attention_weight_max": 28.620065689086914,
      "activations/layer17_attention_weight_min": -25.88581657409668,
      "activations/layer18_attention_weight_max": 33.72472381591797,
      "activations/layer18_attention_weight_min": -27.1870174407959,
      "activations/layer19_attention_weight_max": 33.48611068725586,
      "activations/layer19_attention_weight_min": -33.00419235229492,
      "activations/layer1_attention_weight_max": 17.32672882080078,
      "activations/layer1_attention_weight_min": -16.200881958007812,
      "activations/layer20_attention_weight_max": 27.94914436340332,
      "activations/layer20_attention_weight_min": -26.633216857910156,
      "activations/layer21_attention_weight_max": 26.767074584960938,
      "activations/layer21_attention_weight_min": -25.288724899291992,
      "activations/layer22_attention_weight_max": 39.3392448425293,
      "activations/layer22_attention_weight_min": -30.96369743347168,
      "activations/layer23_attention_weight_max": 33.625892639160156,
      "activations/layer23_attention_weight_min": -24.978872299194336,
      "activations/layer2_attention_weight_max": 33.905906677246094,
      "activations/layer2_attention_weight_min": -33.55830383300781,
      "activations/layer3_attention_weight_max": 99.73291778564453,
      "activations/layer3_attention_weight_min": -104.86898040771484,
      "activations/layer4_attention_weight_max": 104.64434051513672,
      "activations/layer4_attention_weight_min": -108.73966217041016,
      "activations/layer5_attention_weight_max": 50.20912170410156,
      "activations/layer5_attention_weight_min": -63.386898040771484,
      "activations/layer6_attention_weight_max": 45.324790954589844,
      "activations/layer6_attention_weight_min": -46.681522369384766,
      "activations/layer7_attention_weight_max": 94.37044525146484,
      "activations/layer7_attention_weight_min": -95.0597152709961,
      "activations/layer8_attention_weight_max": 39.8162727355957,
      "activations/layer8_attention_weight_min": -43.93385314941406,
      "activations/layer9_attention_weight_max": 31.680490493774414,
      "activations/layer9_attention_weight_min": -33.83464050292969,
      "epoch": 19.2,
      "learning_rate": 2.646363636363636e-05,
      "loss": 2.718,
      "step": 330400
    },
    {
      "activations/layer0_attention_weight_max": 16.59777069091797,
      "activations/layer0_attention_weight_min": -13.507208824157715,
      "activations/layer10_attention_weight_max": 35.35081100463867,
      "activations/layer10_attention_weight_min": -38.826480865478516,
      "activations/layer11_attention_weight_max": 35.55049514770508,
      "activations/layer11_attention_weight_min": -33.946685791015625,
      "activations/layer12_attention_weight_max": 28.07949447631836,
      "activations/layer12_attention_weight_min": -27.23249626159668,
      "activations/layer13_attention_weight_max": 40.97567367553711,
      "activations/layer13_attention_weight_min": -38.13904571533203,
      "activations/layer14_attention_weight_max": 45.122169494628906,
      "activations/layer14_attention_weight_min": -39.77536392211914,
      "activations/layer15_attention_weight_max": 41.793296813964844,
      "activations/layer15_attention_weight_min": -35.76884841918945,
      "activations/layer16_attention_weight_max": 30.786588668823242,
      "activations/layer16_attention_weight_min": -29.308853149414062,
      "activations/layer17_attention_weight_max": 32.9373664855957,
      "activations/layer17_attention_weight_min": -28.67583656311035,
      "activations/layer18_attention_weight_max": 34.94817352294922,
      "activations/layer18_attention_weight_min": -26.095291137695312,
      "activations/layer19_attention_weight_max": 36.737972259521484,
      "activations/layer19_attention_weight_min": -32.036094665527344,
      "activations/layer1_attention_weight_max": 16.382234573364258,
      "activations/layer1_attention_weight_min": -13.969354629516602,
      "activations/layer20_attention_weight_max": 30.186355590820312,
      "activations/layer20_attention_weight_min": -26.357908248901367,
      "activations/layer21_attention_weight_max": 30.224193572998047,
      "activations/layer21_attention_weight_min": -25.852991104125977,
      "activations/layer22_attention_weight_max": 43.46323776245117,
      "activations/layer22_attention_weight_min": -34.20185089111328,
      "activations/layer23_attention_weight_max": 34.30224609375,
      "activations/layer23_attention_weight_min": -25.422916412353516,
      "activations/layer2_attention_weight_max": 30.577980041503906,
      "activations/layer2_attention_weight_min": -31.06186294555664,
      "activations/layer3_attention_weight_max": 96.66703033447266,
      "activations/layer3_attention_weight_min": -99.55217742919922,
      "activations/layer4_attention_weight_max": 100.81298828125,
      "activations/layer4_attention_weight_min": -108.74165344238281,
      "activations/layer5_attention_weight_max": 48.39048767089844,
      "activations/layer5_attention_weight_min": -65.06654357910156,
      "activations/layer6_attention_weight_max": 42.86063766479492,
      "activations/layer6_attention_weight_min": -47.28361129760742,
      "activations/layer7_attention_weight_max": 93.37403869628906,
      "activations/layer7_attention_weight_min": -98.1988296508789,
      "activations/layer8_attention_weight_max": 41.97099685668945,
      "activations/layer8_attention_weight_min": -43.306888580322266,
      "activations/layer9_attention_weight_max": 32.78569030761719,
      "activations/layer9_attention_weight_min": -35.44960021972656,
      "epoch": 19.2,
      "learning_rate": 2.6444696969696966e-05,
      "loss": 2.7285,
      "step": 330450
    },
    {
      "activations/layer0_attention_weight_max": 16.60306739807129,
      "activations/layer0_attention_weight_min": -12.448187828063965,
      "activations/layer10_attention_weight_max": 39.52219009399414,
      "activations/layer10_attention_weight_min": -36.41117858886719,
      "activations/layer11_attention_weight_max": 36.0177001953125,
      "activations/layer11_attention_weight_min": -35.713802337646484,
      "activations/layer12_attention_weight_max": 27.530332565307617,
      "activations/layer12_attention_weight_min": -27.33204460144043,
      "activations/layer13_attention_weight_max": 40.157073974609375,
      "activations/layer13_attention_weight_min": -36.08003616333008,
      "activations/layer14_attention_weight_max": 44.50162124633789,
      "activations/layer14_attention_weight_min": -37.81440353393555,
      "activations/layer15_attention_weight_max": 41.72162628173828,
      "activations/layer15_attention_weight_min": -36.226165771484375,
      "activations/layer16_attention_weight_max": 32.55126190185547,
      "activations/layer16_attention_weight_min": -27.718473434448242,
      "activations/layer17_attention_weight_max": 34.97194290161133,
      "activations/layer17_attention_weight_min": -26.493154525756836,
      "activations/layer18_attention_weight_max": 35.565155029296875,
      "activations/layer18_attention_weight_min": -25.739301681518555,
      "activations/layer19_attention_weight_max": 36.537540435791016,
      "activations/layer19_attention_weight_min": -30.708784103393555,
      "activations/layer1_attention_weight_max": 16.208866119384766,
      "activations/layer1_attention_weight_min": -15.75478744506836,
      "activations/layer20_attention_weight_max": 30.01302719116211,
      "activations/layer20_attention_weight_min": -25.315616607666016,
      "activations/layer21_attention_weight_max": 32.560813903808594,
      "activations/layer21_attention_weight_min": -26.1282958984375,
      "activations/layer22_attention_weight_max": 43.8942756652832,
      "activations/layer22_attention_weight_min": -30.65849494934082,
      "activations/layer23_attention_weight_max": 37.27099609375,
      "activations/layer23_attention_weight_min": -25.318052291870117,
      "activations/layer2_attention_weight_max": 32.1954345703125,
      "activations/layer2_attention_weight_min": -31.22705078125,
      "activations/layer3_attention_weight_max": 99.27474975585938,
      "activations/layer3_attention_weight_min": -103.0467758178711,
      "activations/layer4_attention_weight_max": 105.0373764038086,
      "activations/layer4_attention_weight_min": -109.18360900878906,
      "activations/layer5_attention_weight_max": 50.25248718261719,
      "activations/layer5_attention_weight_min": -63.69047546386719,
      "activations/layer6_attention_weight_max": 44.57356262207031,
      "activations/layer6_attention_weight_min": -46.52507019042969,
      "activations/layer7_attention_weight_max": 95.258544921875,
      "activations/layer7_attention_weight_min": -98.28919982910156,
      "activations/layer8_attention_weight_max": 44.15672302246094,
      "activations/layer8_attention_weight_min": -45.080081939697266,
      "activations/layer9_attention_weight_max": 36.02523422241211,
      "activations/layer9_attention_weight_min": -35.1677131652832,
      "epoch": 19.2,
      "learning_rate": 2.6425757575757574e-05,
      "loss": 2.7137,
      "step": 330500
    },
    {
      "activations/layer0_attention_weight_max": 16.169918060302734,
      "activations/layer0_attention_weight_min": -13.704785346984863,
      "activations/layer10_attention_weight_max": 36.17442321777344,
      "activations/layer10_attention_weight_min": -36.15123748779297,
      "activations/layer11_attention_weight_max": 35.08021926879883,
      "activations/layer11_attention_weight_min": -35.928428649902344,
      "activations/layer12_attention_weight_max": 27.624160766601562,
      "activations/layer12_attention_weight_min": -27.563343048095703,
      "activations/layer13_attention_weight_max": 39.83112716674805,
      "activations/layer13_attention_weight_min": -36.470855712890625,
      "activations/layer14_attention_weight_max": 41.8094482421875,
      "activations/layer14_attention_weight_min": -37.81929016113281,
      "activations/layer15_attention_weight_max": 39.08296585083008,
      "activations/layer15_attention_weight_min": -35.85285568237305,
      "activations/layer16_attention_weight_max": 30.817745208740234,
      "activations/layer16_attention_weight_min": -27.448583602905273,
      "activations/layer17_attention_weight_max": 28.70496940612793,
      "activations/layer17_attention_weight_min": -25.01543426513672,
      "activations/layer18_attention_weight_max": 33.04320526123047,
      "activations/layer18_attention_weight_min": -26.406612396240234,
      "activations/layer19_attention_weight_max": 32.94087219238281,
      "activations/layer19_attention_weight_min": -30.98899269104004,
      "activations/layer1_attention_weight_max": 16.3784122467041,
      "activations/layer1_attention_weight_min": -16.42177391052246,
      "activations/layer20_attention_weight_max": 26.579912185668945,
      "activations/layer20_attention_weight_min": -24.93831443786621,
      "activations/layer21_attention_weight_max": 26.832988739013672,
      "activations/layer21_attention_weight_min": -25.565414428710938,
      "activations/layer22_attention_weight_max": 43.07959747314453,
      "activations/layer22_attention_weight_min": -32.3961067199707,
      "activations/layer23_attention_weight_max": 28.375621795654297,
      "activations/layer23_attention_weight_min": -28.254125595092773,
      "activations/layer2_attention_weight_max": 31.245033264160156,
      "activations/layer2_attention_weight_min": -31.65005111694336,
      "activations/layer3_attention_weight_max": 95.57547760009766,
      "activations/layer3_attention_weight_min": -103.98614501953125,
      "activations/layer4_attention_weight_max": 101.4001693725586,
      "activations/layer4_attention_weight_min": -108.21829986572266,
      "activations/layer5_attention_weight_max": 50.6418571472168,
      "activations/layer5_attention_weight_min": -58.630897521972656,
      "activations/layer6_attention_weight_max": 43.70176315307617,
      "activations/layer6_attention_weight_min": -48.048179626464844,
      "activations/layer7_attention_weight_max": 88.57926940917969,
      "activations/layer7_attention_weight_min": -96.79405212402344,
      "activations/layer8_attention_weight_max": 43.49660110473633,
      "activations/layer8_attention_weight_min": -46.27499008178711,
      "activations/layer9_attention_weight_max": 31.507421493530273,
      "activations/layer9_attention_weight_min": -34.84008026123047,
      "epoch": 19.21,
      "learning_rate": 2.640681818181818e-05,
      "loss": 2.7196,
      "step": 330550
    },
    {
      "activations/layer0_attention_weight_max": 15.388652801513672,
      "activations/layer0_attention_weight_min": -13.57906436920166,
      "activations/layer10_attention_weight_max": 32.42036056518555,
      "activations/layer10_attention_weight_min": -33.312034606933594,
      "activations/layer11_attention_weight_max": 35.06768035888672,
      "activations/layer11_attention_weight_min": -34.58082580566406,
      "activations/layer12_attention_weight_max": 27.37558937072754,
      "activations/layer12_attention_weight_min": -27.052335739135742,
      "activations/layer13_attention_weight_max": 40.5340461730957,
      "activations/layer13_attention_weight_min": -35.82113265991211,
      "activations/layer14_attention_weight_max": 42.45779037475586,
      "activations/layer14_attention_weight_min": -37.36389923095703,
      "activations/layer15_attention_weight_max": 38.870731353759766,
      "activations/layer15_attention_weight_min": -34.86005783081055,
      "activations/layer16_attention_weight_max": 30.704830169677734,
      "activations/layer16_attention_weight_min": -29.96133804321289,
      "activations/layer17_attention_weight_max": 31.148677825927734,
      "activations/layer17_attention_weight_min": -26.348621368408203,
      "activations/layer18_attention_weight_max": 31.315040588378906,
      "activations/layer18_attention_weight_min": -27.1984920501709,
      "activations/layer19_attention_weight_max": 34.53396224975586,
      "activations/layer19_attention_weight_min": -32.72721481323242,
      "activations/layer1_attention_weight_max": 15.629727363586426,
      "activations/layer1_attention_weight_min": -14.491491317749023,
      "activations/layer20_attention_weight_max": 31.613229751586914,
      "activations/layer20_attention_weight_min": -25.09624671936035,
      "activations/layer21_attention_weight_max": 29.784175872802734,
      "activations/layer21_attention_weight_min": -24.94955825805664,
      "activations/layer22_attention_weight_max": 44.590816497802734,
      "activations/layer22_attention_weight_min": -30.901290893554688,
      "activations/layer23_attention_weight_max": 33.51288986206055,
      "activations/layer23_attention_weight_min": -26.601390838623047,
      "activations/layer2_attention_weight_max": 30.39828109741211,
      "activations/layer2_attention_weight_min": -30.440231323242188,
      "activations/layer3_attention_weight_max": 96.29463958740234,
      "activations/layer3_attention_weight_min": -99.78173828125,
      "activations/layer4_attention_weight_max": 103.2755126953125,
      "activations/layer4_attention_weight_min": -109.2081298828125,
      "activations/layer5_attention_weight_max": 49.0628776550293,
      "activations/layer5_attention_weight_min": -64.94327545166016,
      "activations/layer6_attention_weight_max": 43.0142936706543,
      "activations/layer6_attention_weight_min": -47.84562301635742,
      "activations/layer7_attention_weight_max": 94.05915832519531,
      "activations/layer7_attention_weight_min": -98.82791900634766,
      "activations/layer8_attention_weight_max": 40.89925003051758,
      "activations/layer8_attention_weight_min": -43.91414260864258,
      "activations/layer9_attention_weight_max": 31.242551803588867,
      "activations/layer9_attention_weight_min": -33.95993423461914,
      "epoch": 19.21,
      "learning_rate": 2.6387878787878787e-05,
      "loss": 2.7307,
      "step": 330600
    },
    {
      "activations/layer0_attention_weight_max": 17.03402328491211,
      "activations/layer0_attention_weight_min": -13.641459465026855,
      "activations/layer10_attention_weight_max": 39.2406005859375,
      "activations/layer10_attention_weight_min": -35.74214172363281,
      "activations/layer11_attention_weight_max": 34.932334899902344,
      "activations/layer11_attention_weight_min": -33.78705978393555,
      "activations/layer12_attention_weight_max": 25.3431453704834,
      "activations/layer12_attention_weight_min": -27.046274185180664,
      "activations/layer13_attention_weight_max": 38.69475173950195,
      "activations/layer13_attention_weight_min": -34.156333923339844,
      "activations/layer14_attention_weight_max": 39.53995132446289,
      "activations/layer14_attention_weight_min": -36.26133346557617,
      "activations/layer15_attention_weight_max": 36.594364166259766,
      "activations/layer15_attention_weight_min": -36.147010803222656,
      "activations/layer16_attention_weight_max": 28.17977523803711,
      "activations/layer16_attention_weight_min": -28.420806884765625,
      "activations/layer17_attention_weight_max": 26.768781661987305,
      "activations/layer17_attention_weight_min": -26.256330490112305,
      "activations/layer18_attention_weight_max": 32.03586959838867,
      "activations/layer18_attention_weight_min": -24.93303680419922,
      "activations/layer19_attention_weight_max": 33.39669418334961,
      "activations/layer19_attention_weight_min": -29.449161529541016,
      "activations/layer1_attention_weight_max": 15.960295677185059,
      "activations/layer1_attention_weight_min": -14.81223201751709,
      "activations/layer20_attention_weight_max": 28.002605438232422,
      "activations/layer20_attention_weight_min": -24.366056442260742,
      "activations/layer21_attention_weight_max": 25.78301429748535,
      "activations/layer21_attention_weight_min": -22.76021385192871,
      "activations/layer22_attention_weight_max": 41.47293472290039,
      "activations/layer22_attention_weight_min": -29.68995475769043,
      "activations/layer23_attention_weight_max": 28.936904907226562,
      "activations/layer23_attention_weight_min": -24.4719295501709,
      "activations/layer2_attention_weight_max": 34.66790008544922,
      "activations/layer2_attention_weight_min": -30.45033073425293,
      "activations/layer3_attention_weight_max": 100.7877197265625,
      "activations/layer3_attention_weight_min": -95.49017333984375,
      "activations/layer4_attention_weight_max": 109.4127197265625,
      "activations/layer4_attention_weight_min": -105.98274230957031,
      "activations/layer5_attention_weight_max": 52.34966278076172,
      "activations/layer5_attention_weight_min": -61.595306396484375,
      "activations/layer6_attention_weight_max": 44.07306671142578,
      "activations/layer6_attention_weight_min": -48.34275436401367,
      "activations/layer7_attention_weight_max": 97.11358642578125,
      "activations/layer7_attention_weight_min": -100.15042114257812,
      "activations/layer8_attention_weight_max": 44.2603645324707,
      "activations/layer8_attention_weight_min": -43.130638122558594,
      "activations/layer9_attention_weight_max": 32.129581451416016,
      "activations/layer9_attention_weight_min": -33.396244049072266,
      "epoch": 19.21,
      "learning_rate": 2.6369318181818178e-05,
      "loss": 2.7183,
      "step": 330650
    },
    {
      "activations/layer0_attention_weight_max": 16.85603141784668,
      "activations/layer0_attention_weight_min": -12.307841300964355,
      "activations/layer10_attention_weight_max": 33.584754943847656,
      "activations/layer10_attention_weight_min": -34.02287292480469,
      "activations/layer11_attention_weight_max": 32.33094024658203,
      "activations/layer11_attention_weight_min": -34.14025115966797,
      "activations/layer12_attention_weight_max": 24.660404205322266,
      "activations/layer12_attention_weight_min": -26.143390655517578,
      "activations/layer13_attention_weight_max": 35.94361877441406,
      "activations/layer13_attention_weight_min": -34.53834533691406,
      "activations/layer14_attention_weight_max": 40.258426666259766,
      "activations/layer14_attention_weight_min": -37.88846969604492,
      "activations/layer15_attention_weight_max": 37.716514587402344,
      "activations/layer15_attention_weight_min": -35.9426383972168,
      "activations/layer16_attention_weight_max": 29.101003646850586,
      "activations/layer16_attention_weight_min": -27.740949630737305,
      "activations/layer17_attention_weight_max": 31.16242790222168,
      "activations/layer17_attention_weight_min": -26.986202239990234,
      "activations/layer18_attention_weight_max": 28.807884216308594,
      "activations/layer18_attention_weight_min": -25.008317947387695,
      "activations/layer19_attention_weight_max": 35.14005661010742,
      "activations/layer19_attention_weight_min": -31.10576820373535,
      "activations/layer1_attention_weight_max": 15.705728530883789,
      "activations/layer1_attention_weight_min": -13.562033653259277,
      "activations/layer20_attention_weight_max": 28.723031997680664,
      "activations/layer20_attention_weight_min": -25.568784713745117,
      "activations/layer21_attention_weight_max": 27.279191970825195,
      "activations/layer21_attention_weight_min": -25.0334415435791,
      "activations/layer22_attention_weight_max": 39.67679214477539,
      "activations/layer22_attention_weight_min": -30.08734703063965,
      "activations/layer23_attention_weight_max": 29.693744659423828,
      "activations/layer23_attention_weight_min": -26.22609519958496,
      "activations/layer2_attention_weight_max": 31.124008178710938,
      "activations/layer2_attention_weight_min": -30.27381134033203,
      "activations/layer3_attention_weight_max": 94.31204223632812,
      "activations/layer3_attention_weight_min": -98.30213165283203,
      "activations/layer4_attention_weight_max": 102.68922424316406,
      "activations/layer4_attention_weight_min": -109.2771224975586,
      "activations/layer5_attention_weight_max": 51.50079345703125,
      "activations/layer5_attention_weight_min": -62.24346160888672,
      "activations/layer6_attention_weight_max": 45.62870788574219,
      "activations/layer6_attention_weight_min": -46.75413513183594,
      "activations/layer7_attention_weight_max": 95.7783432006836,
      "activations/layer7_attention_weight_min": -96.62796783447266,
      "activations/layer8_attention_weight_max": 40.87727355957031,
      "activations/layer8_attention_weight_min": -45.24666213989258,
      "activations/layer9_attention_weight_max": 31.294687271118164,
      "activations/layer9_attention_weight_min": -36.116031646728516,
      "epoch": 19.22,
      "learning_rate": 2.6350378787878786e-05,
      "loss": 2.7257,
      "step": 330700
    },
    {
      "activations/layer0_attention_weight_max": 18.00254249572754,
      "activations/layer0_attention_weight_min": -13.688782691955566,
      "activations/layer10_attention_weight_max": 38.674720764160156,
      "activations/layer10_attention_weight_min": -35.79240798950195,
      "activations/layer11_attention_weight_max": 36.89219284057617,
      "activations/layer11_attention_weight_min": -37.95592498779297,
      "activations/layer12_attention_weight_max": 28.26260757446289,
      "activations/layer12_attention_weight_min": -28.470975875854492,
      "activations/layer13_attention_weight_max": 41.82631301879883,
      "activations/layer13_attention_weight_min": -36.405555725097656,
      "activations/layer14_attention_weight_max": 45.16610336303711,
      "activations/layer14_attention_weight_min": -37.00847244262695,
      "activations/layer15_attention_weight_max": 48.05401611328125,
      "activations/layer15_attention_weight_min": -38.0264892578125,
      "activations/layer16_attention_weight_max": 31.751726150512695,
      "activations/layer16_attention_weight_min": -28.95433235168457,
      "activations/layer17_attention_weight_max": 29.63779067993164,
      "activations/layer17_attention_weight_min": -27.0776309967041,
      "activations/layer18_attention_weight_max": 33.68622970581055,
      "activations/layer18_attention_weight_min": -26.15599822998047,
      "activations/layer19_attention_weight_max": 34.49068069458008,
      "activations/layer19_attention_weight_min": -34.23369216918945,
      "activations/layer1_attention_weight_max": 16.828487396240234,
      "activations/layer1_attention_weight_min": -15.317978858947754,
      "activations/layer20_attention_weight_max": 31.003755569458008,
      "activations/layer20_attention_weight_min": -27.825666427612305,
      "activations/layer21_attention_weight_max": 34.0204963684082,
      "activations/layer21_attention_weight_min": -27.926021575927734,
      "activations/layer22_attention_weight_max": 41.60908126831055,
      "activations/layer22_attention_weight_min": -31.48340606689453,
      "activations/layer23_attention_weight_max": 32.5374870300293,
      "activations/layer23_attention_weight_min": -25.38722038269043,
      "activations/layer2_attention_weight_max": 31.607275009155273,
      "activations/layer2_attention_weight_min": -31.91969108581543,
      "activations/layer3_attention_weight_max": 95.1204833984375,
      "activations/layer3_attention_weight_min": -96.47660827636719,
      "activations/layer4_attention_weight_max": 102.6413345336914,
      "activations/layer4_attention_weight_min": -101.29827880859375,
      "activations/layer5_attention_weight_max": 47.796783447265625,
      "activations/layer5_attention_weight_min": -62.04972457885742,
      "activations/layer6_attention_weight_max": 43.25232696533203,
      "activations/layer6_attention_weight_min": -45.59452438354492,
      "activations/layer7_attention_weight_max": 94.47407531738281,
      "activations/layer7_attention_weight_min": -98.48165130615234,
      "activations/layer8_attention_weight_max": 43.23853302001953,
      "activations/layer8_attention_weight_min": -43.54231643676758,
      "activations/layer9_attention_weight_max": 32.421077728271484,
      "activations/layer9_attention_weight_min": -35.58951187133789,
      "epoch": 19.22,
      "learning_rate": 2.6331439393939394e-05,
      "loss": 2.7419,
      "step": 330750
    },
    {
      "activations/layer0_attention_weight_max": 15.64124584197998,
      "activations/layer0_attention_weight_min": -13.467253684997559,
      "activations/layer10_attention_weight_max": 41.8880729675293,
      "activations/layer10_attention_weight_min": -37.50167465209961,
      "activations/layer11_attention_weight_max": 37.859832763671875,
      "activations/layer11_attention_weight_min": -38.031532287597656,
      "activations/layer12_attention_weight_max": 27.906896591186523,
      "activations/layer12_attention_weight_min": -28.109657287597656,
      "activations/layer13_attention_weight_max": 41.25554275512695,
      "activations/layer13_attention_weight_min": -37.83290481567383,
      "activations/layer14_attention_weight_max": 46.20981216430664,
      "activations/layer14_attention_weight_min": -39.389461517333984,
      "activations/layer15_attention_weight_max": 39.55402374267578,
      "activations/layer15_attention_weight_min": -37.839359283447266,
      "activations/layer16_attention_weight_max": 30.951738357543945,
      "activations/layer16_attention_weight_min": -29.028608322143555,
      "activations/layer17_attention_weight_max": 29.969308853149414,
      "activations/layer17_attention_weight_min": -26.243621826171875,
      "activations/layer18_attention_weight_max": 31.787113189697266,
      "activations/layer18_attention_weight_min": -26.71287727355957,
      "activations/layer19_attention_weight_max": 33.77108383178711,
      "activations/layer19_attention_weight_min": -33.8726806640625,
      "activations/layer1_attention_weight_max": 15.036625862121582,
      "activations/layer1_attention_weight_min": -13.98775577545166,
      "activations/layer20_attention_weight_max": 30.423837661743164,
      "activations/layer20_attention_weight_min": -25.430644989013672,
      "activations/layer21_attention_weight_max": 29.78961181640625,
      "activations/layer21_attention_weight_min": -23.80150604248047,
      "activations/layer22_attention_weight_max": 45.39767837524414,
      "activations/layer22_attention_weight_min": -31.0609130859375,
      "activations/layer23_attention_weight_max": 33.83576965332031,
      "activations/layer23_attention_weight_min": -25.26239585876465,
      "activations/layer2_attention_weight_max": 31.955915451049805,
      "activations/layer2_attention_weight_min": -31.41065788269043,
      "activations/layer3_attention_weight_max": 94.53012084960938,
      "activations/layer3_attention_weight_min": -99.6700439453125,
      "activations/layer4_attention_weight_max": 102.4847640991211,
      "activations/layer4_attention_weight_min": -109.12495422363281,
      "activations/layer5_attention_weight_max": 50.23760223388672,
      "activations/layer5_attention_weight_min": -64.19715881347656,
      "activations/layer6_attention_weight_max": 45.63640594482422,
      "activations/layer6_attention_weight_min": -48.718990325927734,
      "activations/layer7_attention_weight_max": 96.92955017089844,
      "activations/layer7_attention_weight_min": -102.29924774169922,
      "activations/layer8_attention_weight_max": 45.552364349365234,
      "activations/layer8_attention_weight_min": -46.533119201660156,
      "activations/layer9_attention_weight_max": 36.794395446777344,
      "activations/layer9_attention_weight_min": -36.79705047607422,
      "epoch": 19.22,
      "learning_rate": 2.6312499999999996e-05,
      "loss": 2.7232,
      "step": 330800
    },
    {
      "activations/layer0_attention_weight_max": 16.013580322265625,
      "activations/layer0_attention_weight_min": -12.683443069458008,
      "activations/layer10_attention_weight_max": 32.22542190551758,
      "activations/layer10_attention_weight_min": -34.95258331298828,
      "activations/layer11_attention_weight_max": 31.688983917236328,
      "activations/layer11_attention_weight_min": -34.215362548828125,
      "activations/layer12_attention_weight_max": 27.319639205932617,
      "activations/layer12_attention_weight_min": -26.592548370361328,
      "activations/layer13_attention_weight_max": 40.04587936401367,
      "activations/layer13_attention_weight_min": -36.72467041015625,
      "activations/layer14_attention_weight_max": 46.25826644897461,
      "activations/layer14_attention_weight_min": -38.82160568237305,
      "activations/layer15_attention_weight_max": 40.4005126953125,
      "activations/layer15_attention_weight_min": -36.06786346435547,
      "activations/layer16_attention_weight_max": 31.702869415283203,
      "activations/layer16_attention_weight_min": -29.80708122253418,
      "activations/layer17_attention_weight_max": 32.66805648803711,
      "activations/layer17_attention_weight_min": -27.191328048706055,
      "activations/layer18_attention_weight_max": 32.74003219604492,
      "activations/layer18_attention_weight_min": -27.727333068847656,
      "activations/layer19_attention_weight_max": 34.460411071777344,
      "activations/layer19_attention_weight_min": -32.15385437011719,
      "activations/layer1_attention_weight_max": 16.698955535888672,
      "activations/layer1_attention_weight_min": -13.802102088928223,
      "activations/layer20_attention_weight_max": 29.4952449798584,
      "activations/layer20_attention_weight_min": -24.241287231445312,
      "activations/layer21_attention_weight_max": 30.527074813842773,
      "activations/layer21_attention_weight_min": -25.7146053314209,
      "activations/layer22_attention_weight_max": 46.05186080932617,
      "activations/layer22_attention_weight_min": -32.091243743896484,
      "activations/layer23_attention_weight_max": 35.343238830566406,
      "activations/layer23_attention_weight_min": -26.918701171875,
      "activations/layer2_attention_weight_max": 31.078306198120117,
      "activations/layer2_attention_weight_min": -29.54151725769043,
      "activations/layer3_attention_weight_max": 91.45692443847656,
      "activations/layer3_attention_weight_min": -92.53963470458984,
      "activations/layer4_attention_weight_max": 96.95952606201172,
      "activations/layer4_attention_weight_min": -100.504150390625,
      "activations/layer5_attention_weight_max": 47.766571044921875,
      "activations/layer5_attention_weight_min": -60.08329772949219,
      "activations/layer6_attention_weight_max": 41.04486846923828,
      "activations/layer6_attention_weight_min": -48.27067565917969,
      "activations/layer7_attention_weight_max": 86.42660522460938,
      "activations/layer7_attention_weight_min": -94.1134262084961,
      "activations/layer8_attention_weight_max": 39.761383056640625,
      "activations/layer8_attention_weight_min": -42.65732192993164,
      "activations/layer9_attention_weight_max": 29.786108016967773,
      "activations/layer9_attention_weight_min": -33.61896896362305,
      "epoch": 19.22,
      "learning_rate": 2.6293560606060604e-05,
      "loss": 2.719,
      "step": 330850
    },
    {
      "activations/layer0_attention_weight_max": 15.743372917175293,
      "activations/layer0_attention_weight_min": -13.210468292236328,
      "activations/layer10_attention_weight_max": 33.891090393066406,
      "activations/layer10_attention_weight_min": -35.92584228515625,
      "activations/layer11_attention_weight_max": 33.54150390625,
      "activations/layer11_attention_weight_min": -34.90857696533203,
      "activations/layer12_attention_weight_max": 26.193557739257812,
      "activations/layer12_attention_weight_min": -25.982009887695312,
      "activations/layer13_attention_weight_max": 40.0638542175293,
      "activations/layer13_attention_weight_min": -34.296390533447266,
      "activations/layer14_attention_weight_max": 40.956268310546875,
      "activations/layer14_attention_weight_min": -36.756927490234375,
      "activations/layer15_attention_weight_max": 37.78915786743164,
      "activations/layer15_attention_weight_min": -34.90602111816406,
      "activations/layer16_attention_weight_max": 28.52734375,
      "activations/layer16_attention_weight_min": -28.782562255859375,
      "activations/layer17_attention_weight_max": 31.40749168395996,
      "activations/layer17_attention_weight_min": -26.091920852661133,
      "activations/layer18_attention_weight_max": 31.335453033447266,
      "activations/layer18_attention_weight_min": -25.944368362426758,
      "activations/layer19_attention_weight_max": 32.22884750366211,
      "activations/layer19_attention_weight_min": -35.56473159790039,
      "activations/layer1_attention_weight_max": 15.955989837646484,
      "activations/layer1_attention_weight_min": -13.672321319580078,
      "activations/layer20_attention_weight_max": 30.892650604248047,
      "activations/layer20_attention_weight_min": -26.390647888183594,
      "activations/layer21_attention_weight_max": 26.666934967041016,
      "activations/layer21_attention_weight_min": -26.13016700744629,
      "activations/layer22_attention_weight_max": 41.69972229003906,
      "activations/layer22_attention_weight_min": -33.68574523925781,
      "activations/layer23_attention_weight_max": 32.472511291503906,
      "activations/layer23_attention_weight_min": -26.89452362060547,
      "activations/layer2_attention_weight_max": 31.33456039428711,
      "activations/layer2_attention_weight_min": -30.892370223999023,
      "activations/layer3_attention_weight_max": 98.5892333984375,
      "activations/layer3_attention_weight_min": -97.1564712524414,
      "activations/layer4_attention_weight_max": 104.927001953125,
      "activations/layer4_attention_weight_min": -105.03739166259766,
      "activations/layer5_attention_weight_max": 49.21805953979492,
      "activations/layer5_attention_weight_min": -64.88832092285156,
      "activations/layer6_attention_weight_max": 44.08000564575195,
      "activations/layer6_attention_weight_min": -46.18278884887695,
      "activations/layer7_attention_weight_max": 90.0827407836914,
      "activations/layer7_attention_weight_min": -98.59451293945312,
      "activations/layer8_attention_weight_max": 40.699615478515625,
      "activations/layer8_attention_weight_min": -42.56635665893555,
      "activations/layer9_attention_weight_max": 32.366878509521484,
      "activations/layer9_attention_weight_min": -33.21049499511719,
      "epoch": 19.23,
      "learning_rate": 2.627462121212121e-05,
      "loss": 2.7298,
      "step": 330900
    },
    {
      "activations/layer0_attention_weight_max": 16.531705856323242,
      "activations/layer0_attention_weight_min": -14.060110092163086,
      "activations/layer10_attention_weight_max": 34.31906509399414,
      "activations/layer10_attention_weight_min": -36.78334045410156,
      "activations/layer11_attention_weight_max": 33.39850997924805,
      "activations/layer11_attention_weight_min": -35.029747009277344,
      "activations/layer12_attention_weight_max": 28.554967880249023,
      "activations/layer12_attention_weight_min": -29.75708770751953,
      "activations/layer13_attention_weight_max": 41.30231857299805,
      "activations/layer13_attention_weight_min": -37.41854476928711,
      "activations/layer14_attention_weight_max": 43.09381103515625,
      "activations/layer14_attention_weight_min": -39.68777084350586,
      "activations/layer15_attention_weight_max": 40.03764724731445,
      "activations/layer15_attention_weight_min": -37.748905181884766,
      "activations/layer16_attention_weight_max": 30.44994354248047,
      "activations/layer16_attention_weight_min": -26.745386123657227,
      "activations/layer17_attention_weight_max": 30.636484146118164,
      "activations/layer17_attention_weight_min": -25.256338119506836,
      "activations/layer18_attention_weight_max": 35.43049240112305,
      "activations/layer18_attention_weight_min": -25.27092170715332,
      "activations/layer19_attention_weight_max": 34.545310974121094,
      "activations/layer19_attention_weight_min": -31.476764678955078,
      "activations/layer1_attention_weight_max": 15.404608726501465,
      "activations/layer1_attention_weight_min": -14.544508934020996,
      "activations/layer20_attention_weight_max": 31.1947021484375,
      "activations/layer20_attention_weight_min": -25.27815818786621,
      "activations/layer21_attention_weight_max": 29.915014266967773,
      "activations/layer21_attention_weight_min": -25.15799331665039,
      "activations/layer22_attention_weight_max": 44.38813781738281,
      "activations/layer22_attention_weight_min": -34.37116241455078,
      "activations/layer23_attention_weight_max": 32.871639251708984,
      "activations/layer23_attention_weight_min": -27.40181541442871,
      "activations/layer2_attention_weight_max": 33.764862060546875,
      "activations/layer2_attention_weight_min": -33.70962142944336,
      "activations/layer3_attention_weight_max": 98.2099380493164,
      "activations/layer3_attention_weight_min": -102.97333526611328,
      "activations/layer4_attention_weight_max": 104.2278823852539,
      "activations/layer4_attention_weight_min": -111.8326187133789,
      "activations/layer5_attention_weight_max": 50.05744934082031,
      "activations/layer5_attention_weight_min": -69.62641143798828,
      "activations/layer6_attention_weight_max": 44.31092834472656,
      "activations/layer6_attention_weight_min": -47.91749954223633,
      "activations/layer7_attention_weight_max": 91.98152160644531,
      "activations/layer7_attention_weight_min": -99.08614349365234,
      "activations/layer8_attention_weight_max": 43.003902435302734,
      "activations/layer8_attention_weight_min": -45.53719711303711,
      "activations/layer9_attention_weight_max": 33.73293685913086,
      "activations/layer9_attention_weight_min": -35.917091369628906,
      "epoch": 19.23,
      "learning_rate": 2.6255681818181817e-05,
      "loss": 2.7272,
      "step": 330950
    },
    {
      "activations/layer0_attention_weight_max": 15.870319366455078,
      "activations/layer0_attention_weight_min": -12.291760444641113,
      "activations/layer10_attention_weight_max": 35.34571075439453,
      "activations/layer10_attention_weight_min": -37.0954475402832,
      "activations/layer11_attention_weight_max": 34.32319641113281,
      "activations/layer11_attention_weight_min": -35.96015930175781,
      "activations/layer12_attention_weight_max": 27.802675247192383,
      "activations/layer12_attention_weight_min": -27.140897750854492,
      "activations/layer13_attention_weight_max": 42.82240295410156,
      "activations/layer13_attention_weight_min": -36.7958869934082,
      "activations/layer14_attention_weight_max": 46.25059127807617,
      "activations/layer14_attention_weight_min": -39.02415084838867,
      "activations/layer15_attention_weight_max": 45.068878173828125,
      "activations/layer15_attention_weight_min": -39.82228469848633,
      "activations/layer16_attention_weight_max": 33.10956954956055,
      "activations/layer16_attention_weight_min": -29.40241241455078,
      "activations/layer17_attention_weight_max": 32.97220230102539,
      "activations/layer17_attention_weight_min": -25.900550842285156,
      "activations/layer18_attention_weight_max": 33.309478759765625,
      "activations/layer18_attention_weight_min": -26.559049606323242,
      "activations/layer19_attention_weight_max": 36.56037521362305,
      "activations/layer19_attention_weight_min": -30.20111656188965,
      "activations/layer1_attention_weight_max": 15.669577598571777,
      "activations/layer1_attention_weight_min": -13.902498245239258,
      "activations/layer20_attention_weight_max": 29.350339889526367,
      "activations/layer20_attention_weight_min": -24.543907165527344,
      "activations/layer21_attention_weight_max": 27.531185150146484,
      "activations/layer21_attention_weight_min": -24.972671508789062,
      "activations/layer22_attention_weight_max": 45.60322570800781,
      "activations/layer22_attention_weight_min": -34.83409118652344,
      "activations/layer23_attention_weight_max": 35.628814697265625,
      "activations/layer23_attention_weight_min": -27.468717575073242,
      "activations/layer2_attention_weight_max": 33.34667205810547,
      "activations/layer2_attention_weight_min": -30.290180206298828,
      "activations/layer3_attention_weight_max": 95.03636169433594,
      "activations/layer3_attention_weight_min": -96.07479858398438,
      "activations/layer4_attention_weight_max": 104.2727279663086,
      "activations/layer4_attention_weight_min": -105.90472412109375,
      "activations/layer5_attention_weight_max": 49.377342224121094,
      "activations/layer5_attention_weight_min": -62.33369445800781,
      "activations/layer6_attention_weight_max": 43.84617233276367,
      "activations/layer6_attention_weight_min": -47.37259292602539,
      "activations/layer7_attention_weight_max": 94.89921569824219,
      "activations/layer7_attention_weight_min": -97.57233428955078,
      "activations/layer8_attention_weight_max": 41.85515594482422,
      "activations/layer8_attention_weight_min": -45.111080169677734,
      "activations/layer9_attention_weight_max": 33.25468444824219,
      "activations/layer9_attention_weight_min": -37.9071159362793,
      "epoch": 19.23,
      "learning_rate": 2.6236742424242422e-05,
      "loss": 2.7086,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4867,
      "eval_samples_per_second": 505.969,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4867,
      "eval_openwebtext_samples_per_second": 505.969,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9861,
      "eval_wikitext_samples_per_second": 229.597,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_lambada_loss": 2.3515625,
      "eval_lambada_ppl": 10.501966233859342,
      "eval_lambada_runtime": 9.5767,
      "eval_lambada_samples_per_second": 508.421,
      "step": 331000
    },
    {
      "activations/layer0_attention_weight_max": 16.48177146911621,
      "activations/layer0_attention_weight_min": -13.890186309814453,
      "activations/layer10_attention_weight_max": 37.888465881347656,
      "activations/layer10_attention_weight_min": -38.63843536376953,
      "activations/layer11_attention_weight_max": 37.640262603759766,
      "activations/layer11_attention_weight_min": -39.569793701171875,
      "activations/layer12_attention_weight_max": 29.318063735961914,
      "activations/layer12_attention_weight_min": -29.311885833740234,
      "activations/layer13_attention_weight_max": 40.16646194458008,
      "activations/layer13_attention_weight_min": -39.67879867553711,
      "activations/layer14_attention_weight_max": 43.44745635986328,
      "activations/layer14_attention_weight_min": -41.538856506347656,
      "activations/layer15_attention_weight_max": 40.99397277832031,
      "activations/layer15_attention_weight_min": -39.9841423034668,
      "activations/layer16_attention_weight_max": 31.240205764770508,
      "activations/layer16_attention_weight_min": -28.762712478637695,
      "activations/layer17_attention_weight_max": 31.46121597290039,
      "activations/layer17_attention_weight_min": -27.147367477416992,
      "activations/layer18_attention_weight_max": 29.467628479003906,
      "activations/layer18_attention_weight_min": -24.466243743896484,
      "activations/layer19_attention_weight_max": 34.37031555175781,
      "activations/layer19_attention_weight_min": -32.00510787963867,
      "activations/layer1_attention_weight_max": 15.300763130187988,
      "activations/layer1_attention_weight_min": -13.966763496398926,
      "activations/layer20_attention_weight_max": 28.1604061126709,
      "activations/layer20_attention_weight_min": -24.785112380981445,
      "activations/layer21_attention_weight_max": 28.316585540771484,
      "activations/layer21_attention_weight_min": -24.957107543945312,
      "activations/layer22_attention_weight_max": 40.55229568481445,
      "activations/layer22_attention_weight_min": -32.12654113769531,
      "activations/layer23_attention_weight_max": 31.722782135009766,
      "activations/layer23_attention_weight_min": -26.423429489135742,
      "activations/layer2_attention_weight_max": 33.942665100097656,
      "activations/layer2_attention_weight_min": -31.61151123046875,
      "activations/layer3_attention_weight_max": 96.84693908691406,
      "activations/layer3_attention_weight_min": -98.53709411621094,
      "activations/layer4_attention_weight_max": 108.45328521728516,
      "activations/layer4_attention_weight_min": -104.9278793334961,
      "activations/layer5_attention_weight_max": 49.17069625854492,
      "activations/layer5_attention_weight_min": -59.51786804199219,
      "activations/layer6_attention_weight_max": 44.18252944946289,
      "activations/layer6_attention_weight_min": -46.72433090209961,
      "activations/layer7_attention_weight_max": 96.57244110107422,
      "activations/layer7_attention_weight_min": -96.81988525390625,
      "activations/layer8_attention_weight_max": 43.39189910888672,
      "activations/layer8_attention_weight_min": -46.081844329833984,
      "activations/layer9_attention_weight_max": 34.669166564941406,
      "activations/layer9_attention_weight_min": -39.179805755615234,
      "epoch": 19.24,
      "learning_rate": 2.621780303030303e-05,
      "loss": 2.7371,
      "step": 331050
    },
    {
      "activations/layer0_attention_weight_max": 16.116443634033203,
      "activations/layer0_attention_weight_min": -13.777764320373535,
      "activations/layer10_attention_weight_max": 32.39787292480469,
      "activations/layer10_attention_weight_min": -33.03326416015625,
      "activations/layer11_attention_weight_max": 34.019168853759766,
      "activations/layer11_attention_weight_min": -33.64105987548828,
      "activations/layer12_attention_weight_max": 26.409074783325195,
      "activations/layer12_attention_weight_min": -27.72882843017578,
      "activations/layer13_attention_weight_max": 39.997413635253906,
      "activations/layer13_attention_weight_min": -38.16349411010742,
      "activations/layer14_attention_weight_max": 43.36738204956055,
      "activations/layer14_attention_weight_min": -40.644317626953125,
      "activations/layer15_attention_weight_max": 40.760581970214844,
      "activations/layer15_attention_weight_min": -36.844608306884766,
      "activations/layer16_attention_weight_max": 31.878589630126953,
      "activations/layer16_attention_weight_min": -29.7740535736084,
      "activations/layer17_attention_weight_max": 30.563730239868164,
      "activations/layer17_attention_weight_min": -27.12875747680664,
      "activations/layer18_attention_weight_max": 33.97716522216797,
      "activations/layer18_attention_weight_min": -25.54802894592285,
      "activations/layer19_attention_weight_max": 37.770668029785156,
      "activations/layer19_attention_weight_min": -32.03281784057617,
      "activations/layer1_attention_weight_max": 15.611032485961914,
      "activations/layer1_attention_weight_min": -14.324983596801758,
      "activations/layer20_attention_weight_max": 32.66446304321289,
      "activations/layer20_attention_weight_min": -24.716655731201172,
      "activations/layer21_attention_weight_max": 32.38690948486328,
      "activations/layer21_attention_weight_min": -25.345518112182617,
      "activations/layer22_attention_weight_max": 43.44492721557617,
      "activations/layer22_attention_weight_min": -32.09628677368164,
      "activations/layer23_attention_weight_max": 36.262367248535156,
      "activations/layer23_attention_weight_min": -25.7749080657959,
      "activations/layer2_attention_weight_max": 33.293270111083984,
      "activations/layer2_attention_weight_min": -32.32599639892578,
      "activations/layer3_attention_weight_max": 96.29878234863281,
      "activations/layer3_attention_weight_min": -100.72838592529297,
      "activations/layer4_attention_weight_max": 103.44966888427734,
      "activations/layer4_attention_weight_min": -104.94874572753906,
      "activations/layer5_attention_weight_max": 51.43354415893555,
      "activations/layer5_attention_weight_min": -58.542335510253906,
      "activations/layer6_attention_weight_max": 42.24238204956055,
      "activations/layer6_attention_weight_min": -46.588199615478516,
      "activations/layer7_attention_weight_max": 90.2956771850586,
      "activations/layer7_attention_weight_min": -93.22282409667969,
      "activations/layer8_attention_weight_max": 40.06336212158203,
      "activations/layer8_attention_weight_min": -41.99638748168945,
      "activations/layer9_attention_weight_max": 32.458839416503906,
      "activations/layer9_attention_weight_min": -32.42341613769531,
      "epoch": 19.24,
      "learning_rate": 2.6198863636363632e-05,
      "loss": 2.7126,
      "step": 331100
    },
    {
      "activations/layer0_attention_weight_max": 17.629236221313477,
      "activations/layer0_attention_weight_min": -13.840310096740723,
      "activations/layer10_attention_weight_max": 37.853885650634766,
      "activations/layer10_attention_weight_min": -36.49318313598633,
      "activations/layer11_attention_weight_max": 37.67323684692383,
      "activations/layer11_attention_weight_min": -37.94501876831055,
      "activations/layer12_attention_weight_max": 26.757057189941406,
      "activations/layer12_attention_weight_min": -26.340354919433594,
      "activations/layer13_attention_weight_max": 38.49152755737305,
      "activations/layer13_attention_weight_min": -36.130611419677734,
      "activations/layer14_attention_weight_max": 45.13187026977539,
      "activations/layer14_attention_weight_min": -39.325740814208984,
      "activations/layer15_attention_weight_max": 39.71173095703125,
      "activations/layer15_attention_weight_min": -38.54499435424805,
      "activations/layer16_attention_weight_max": 32.82297134399414,
      "activations/layer16_attention_weight_min": -27.8474063873291,
      "activations/layer17_attention_weight_max": 30.68115234375,
      "activations/layer17_attention_weight_min": -26.239646911621094,
      "activations/layer18_attention_weight_max": 30.72091293334961,
      "activations/layer18_attention_weight_min": -25.19065284729004,
      "activations/layer19_attention_weight_max": 37.3720588684082,
      "activations/layer19_attention_weight_min": -32.0372428894043,
      "activations/layer1_attention_weight_max": 16.530811309814453,
      "activations/layer1_attention_weight_min": -15.620964050292969,
      "activations/layer20_attention_weight_max": 31.012760162353516,
      "activations/layer20_attention_weight_min": -24.032819747924805,
      "activations/layer21_attention_weight_max": 33.33893966674805,
      "activations/layer21_attention_weight_min": -24.66690444946289,
      "activations/layer22_attention_weight_max": 40.39143371582031,
      "activations/layer22_attention_weight_min": -30.251056671142578,
      "activations/layer23_attention_weight_max": 31.20503807067871,
      "activations/layer23_attention_weight_min": -27.848434448242188,
      "activations/layer2_attention_weight_max": 32.70551681518555,
      "activations/layer2_attention_weight_min": -30.739341735839844,
      "activations/layer3_attention_weight_max": 97.2318115234375,
      "activations/layer3_attention_weight_min": -95.31737518310547,
      "activations/layer4_attention_weight_max": 104.57913970947266,
      "activations/layer4_attention_weight_min": -103.38927459716797,
      "activations/layer5_attention_weight_max": 48.63257598876953,
      "activations/layer5_attention_weight_min": -58.25979232788086,
      "activations/layer6_attention_weight_max": 43.02698516845703,
      "activations/layer6_attention_weight_min": -45.75095748901367,
      "activations/layer7_attention_weight_max": 94.29507446289062,
      "activations/layer7_attention_weight_min": -93.83851623535156,
      "activations/layer8_attention_weight_max": 42.35038375854492,
      "activations/layer8_attention_weight_min": -44.616817474365234,
      "activations/layer9_attention_weight_max": 33.30130386352539,
      "activations/layer9_attention_weight_min": -35.88422775268555,
      "epoch": 19.24,
      "learning_rate": 2.617992424242424e-05,
      "loss": 2.7094,
      "step": 331150
    },
    {
      "activations/layer0_attention_weight_max": 15.992513656616211,
      "activations/layer0_attention_weight_min": -13.85607624053955,
      "activations/layer10_attention_weight_max": 42.04277420043945,
      "activations/layer10_attention_weight_min": -40.747032165527344,
      "activations/layer11_attention_weight_max": 44.43266296386719,
      "activations/layer11_attention_weight_min": -44.12568283081055,
      "activations/layer12_attention_weight_max": 33.582725524902344,
      "activations/layer12_attention_weight_min": -30.486555099487305,
      "activations/layer13_attention_weight_max": 50.29756546020508,
      "activations/layer13_attention_weight_min": -42.35622024536133,
      "activations/layer14_attention_weight_max": 62.353702545166016,
      "activations/layer14_attention_weight_min": -48.78392028808594,
      "activations/layer15_attention_weight_max": 54.336570739746094,
      "activations/layer15_attention_weight_min": -47.143348693847656,
      "activations/layer16_attention_weight_max": 45.66849136352539,
      "activations/layer16_attention_weight_min": -34.92959976196289,
      "activations/layer17_attention_weight_max": 51.42531204223633,
      "activations/layer17_attention_weight_min": -33.24600601196289,
      "activations/layer18_attention_weight_max": 39.88756561279297,
      "activations/layer18_attention_weight_min": -33.53020477294922,
      "activations/layer19_attention_weight_max": 44.673580169677734,
      "activations/layer19_attention_weight_min": -40.76554489135742,
      "activations/layer1_attention_weight_max": 15.53611946105957,
      "activations/layer1_attention_weight_min": -16.31772232055664,
      "activations/layer20_attention_weight_max": 37.87199401855469,
      "activations/layer20_attention_weight_min": -31.999317169189453,
      "activations/layer21_attention_weight_max": 33.96431350708008,
      "activations/layer21_attention_weight_min": -31.369930267333984,
      "activations/layer22_attention_weight_max": 57.49322509765625,
      "activations/layer22_attention_weight_min": -42.973880767822266,
      "activations/layer23_attention_weight_max": 43.63274383544922,
      "activations/layer23_attention_weight_min": -31.18216323852539,
      "activations/layer2_attention_weight_max": 30.592384338378906,
      "activations/layer2_attention_weight_min": -31.379894256591797,
      "activations/layer3_attention_weight_max": 95.955078125,
      "activations/layer3_attention_weight_min": -100.69048309326172,
      "activations/layer4_attention_weight_max": 105.26981353759766,
      "activations/layer4_attention_weight_min": -106.91483306884766,
      "activations/layer5_attention_weight_max": 51.134483337402344,
      "activations/layer5_attention_weight_min": -60.971900939941406,
      "activations/layer6_attention_weight_max": 48.61127853393555,
      "activations/layer6_attention_weight_min": -47.30522537231445,
      "activations/layer7_attention_weight_max": 107.2994613647461,
      "activations/layer7_attention_weight_min": -101.95270538330078,
      "activations/layer8_attention_weight_max": 48.156742095947266,
      "activations/layer8_attention_weight_min": -46.38630294799805,
      "activations/layer9_attention_weight_max": 40.37944793701172,
      "activations/layer9_attention_weight_min": -40.36479187011719,
      "epoch": 19.24,
      "learning_rate": 2.6160984848484845e-05,
      "loss": 2.7137,
      "step": 331200
    },
    {
      "activations/layer0_attention_weight_max": 16.24199676513672,
      "activations/layer0_attention_weight_min": -13.224235534667969,
      "activations/layer10_attention_weight_max": 33.05146026611328,
      "activations/layer10_attention_weight_min": -33.67630386352539,
      "activations/layer11_attention_weight_max": 32.20851516723633,
      "activations/layer11_attention_weight_min": -32.3964729309082,
      "activations/layer12_attention_weight_max": 25.96454429626465,
      "activations/layer12_attention_weight_min": -26.029390335083008,
      "activations/layer13_attention_weight_max": 38.68495178222656,
      "activations/layer13_attention_weight_min": -35.807621002197266,
      "activations/layer14_attention_weight_max": 41.55624771118164,
      "activations/layer14_attention_weight_min": -37.49275588989258,
      "activations/layer15_attention_weight_max": 39.77920913696289,
      "activations/layer15_attention_weight_min": -36.6350212097168,
      "activations/layer16_attention_weight_max": 29.14937973022461,
      "activations/layer16_attention_weight_min": -27.488250732421875,
      "activations/layer17_attention_weight_max": 29.38974380493164,
      "activations/layer17_attention_weight_min": -25.80980682373047,
      "activations/layer18_attention_weight_max": 31.797382354736328,
      "activations/layer18_attention_weight_min": -25.31808090209961,
      "activations/layer19_attention_weight_max": 31.561521530151367,
      "activations/layer19_attention_weight_min": -31.1473388671875,
      "activations/layer1_attention_weight_max": 15.131993293762207,
      "activations/layer1_attention_weight_min": -14.310003280639648,
      "activations/layer20_attention_weight_max": 29.430511474609375,
      "activations/layer20_attention_weight_min": -24.375465393066406,
      "activations/layer21_attention_weight_max": 26.753507614135742,
      "activations/layer21_attention_weight_min": -25.13022232055664,
      "activations/layer22_attention_weight_max": 39.22574234008789,
      "activations/layer22_attention_weight_min": -30.26784896850586,
      "activations/layer23_attention_weight_max": 30.111003875732422,
      "activations/layer23_attention_weight_min": -24.787572860717773,
      "activations/layer2_attention_weight_max": 32.2855224609375,
      "activations/layer2_attention_weight_min": -30.477413177490234,
      "activations/layer3_attention_weight_max": 94.94685363769531,
      "activations/layer3_attention_weight_min": -102.67817687988281,
      "activations/layer4_attention_weight_max": 103.37300872802734,
      "activations/layer4_attention_weight_min": -107.2496337890625,
      "activations/layer5_attention_weight_max": 47.82750701904297,
      "activations/layer5_attention_weight_min": -63.553768157958984,
      "activations/layer6_attention_weight_max": 42.81035614013672,
      "activations/layer6_attention_weight_min": -45.04713439941406,
      "activations/layer7_attention_weight_max": 92.72526550292969,
      "activations/layer7_attention_weight_min": -96.36536407470703,
      "activations/layer8_attention_weight_max": 40.90080261230469,
      "activations/layer8_attention_weight_min": -42.14031982421875,
      "activations/layer9_attention_weight_max": 32.81109619140625,
      "activations/layer9_attention_weight_min": -34.222591400146484,
      "epoch": 19.25,
      "learning_rate": 2.6142045454545453e-05,
      "loss": 2.7274,
      "step": 331250
    },
    {
      "activations/layer0_attention_weight_max": 16.518556594848633,
      "activations/layer0_attention_weight_min": -12.360247611999512,
      "activations/layer10_attention_weight_max": 31.329727172851562,
      "activations/layer10_attention_weight_min": -33.676513671875,
      "activations/layer11_attention_weight_max": 32.603271484375,
      "activations/layer11_attention_weight_min": -32.792381286621094,
      "activations/layer12_attention_weight_max": 27.591520309448242,
      "activations/layer12_attention_weight_min": -26.630390167236328,
      "activations/layer13_attention_weight_max": 42.48900604248047,
      "activations/layer13_attention_weight_min": -37.3419075012207,
      "activations/layer14_attention_weight_max": 45.35593795776367,
      "activations/layer14_attention_weight_min": -38.684715270996094,
      "activations/layer15_attention_weight_max": 39.177310943603516,
      "activations/layer15_attention_weight_min": -34.998416900634766,
      "activations/layer16_attention_weight_max": 29.259613037109375,
      "activations/layer16_attention_weight_min": -27.56160545349121,
      "activations/layer17_attention_weight_max": 28.8616943359375,
      "activations/layer17_attention_weight_min": -26.739160537719727,
      "activations/layer18_attention_weight_max": 31.17666244506836,
      "activations/layer18_attention_weight_min": -25.550823211669922,
      "activations/layer19_attention_weight_max": 32.01586151123047,
      "activations/layer19_attention_weight_min": -31.730146408081055,
      "activations/layer1_attention_weight_max": 15.779688835144043,
      "activations/layer1_attention_weight_min": -16.88458251953125,
      "activations/layer20_attention_weight_max": 29.004165649414062,
      "activations/layer20_attention_weight_min": -26.590282440185547,
      "activations/layer21_attention_weight_max": 29.22105598449707,
      "activations/layer21_attention_weight_min": -26.77800750732422,
      "activations/layer22_attention_weight_max": 40.6878547668457,
      "activations/layer22_attention_weight_min": -32.6871223449707,
      "activations/layer23_attention_weight_max": 31.60866928100586,
      "activations/layer23_attention_weight_min": -24.752822875976562,
      "activations/layer2_attention_weight_max": 35.003173828125,
      "activations/layer2_attention_weight_min": -31.917240142822266,
      "activations/layer3_attention_weight_max": 92.36849212646484,
      "activations/layer3_attention_weight_min": -89.1122055053711,
      "activations/layer4_attention_weight_max": 96.84539794921875,
      "activations/layer4_attention_weight_min": -95.7544937133789,
      "activations/layer5_attention_weight_max": 48.26313400268555,
      "activations/layer5_attention_weight_min": -63.40946578979492,
      "activations/layer6_attention_weight_max": 44.171749114990234,
      "activations/layer6_attention_weight_min": -43.41376495361328,
      "activations/layer7_attention_weight_max": 90.69593811035156,
      "activations/layer7_attention_weight_min": -89.9519271850586,
      "activations/layer8_attention_weight_max": 44.97991180419922,
      "activations/layer8_attention_weight_min": -40.954612731933594,
      "activations/layer9_attention_weight_max": 30.944692611694336,
      "activations/layer9_attention_weight_min": -31.3234920501709,
      "epoch": 19.25,
      "learning_rate": 2.6123106060606058e-05,
      "loss": 2.7221,
      "step": 331300
    },
    {
      "activations/layer0_attention_weight_max": 16.595043182373047,
      "activations/layer0_attention_weight_min": -13.526823043823242,
      "activations/layer10_attention_weight_max": 34.32025146484375,
      "activations/layer10_attention_weight_min": -35.31585693359375,
      "activations/layer11_attention_weight_max": 34.76057434082031,
      "activations/layer11_attention_weight_min": -34.53950500488281,
      "activations/layer12_attention_weight_max": 26.736263275146484,
      "activations/layer12_attention_weight_min": -26.238197326660156,
      "activations/layer13_attention_weight_max": 38.66667175292969,
      "activations/layer13_attention_weight_min": -35.38849639892578,
      "activations/layer14_attention_weight_max": 42.528907775878906,
      "activations/layer14_attention_weight_min": -37.747676849365234,
      "activations/layer15_attention_weight_max": 41.69652557373047,
      "activations/layer15_attention_weight_min": -35.32902908325195,
      "activations/layer16_attention_weight_max": 29.866575241088867,
      "activations/layer16_attention_weight_min": -28.27471160888672,
      "activations/layer17_attention_weight_max": 28.64971351623535,
      "activations/layer17_attention_weight_min": -26.305757522583008,
      "activations/layer18_attention_weight_max": 30.75457763671875,
      "activations/layer18_attention_weight_min": -26.023513793945312,
      "activations/layer19_attention_weight_max": 36.98997116088867,
      "activations/layer19_attention_weight_min": -32.505523681640625,
      "activations/layer1_attention_weight_max": 16.62315559387207,
      "activations/layer1_attention_weight_min": -14.061958312988281,
      "activations/layer20_attention_weight_max": 32.51887893676758,
      "activations/layer20_attention_weight_min": -25.8188533782959,
      "activations/layer21_attention_weight_max": 30.88597869873047,
      "activations/layer21_attention_weight_min": -25.4229793548584,
      "activations/layer22_attention_weight_max": 45.725337982177734,
      "activations/layer22_attention_weight_min": -32.76710510253906,
      "activations/layer23_attention_weight_max": 39.709449768066406,
      "activations/layer23_attention_weight_min": -27.807523727416992,
      "activations/layer2_attention_weight_max": 30.22762107849121,
      "activations/layer2_attention_weight_min": -29.842252731323242,
      "activations/layer3_attention_weight_max": 92.8987045288086,
      "activations/layer3_attention_weight_min": -95.18951416015625,
      "activations/layer4_attention_weight_max": 100.31694793701172,
      "activations/layer4_attention_weight_min": -105.14569091796875,
      "activations/layer5_attention_weight_max": 50.159690856933594,
      "activations/layer5_attention_weight_min": -60.90476989746094,
      "activations/layer6_attention_weight_max": 43.58427047729492,
      "activations/layer6_attention_weight_min": -47.45490264892578,
      "activations/layer7_attention_weight_max": 96.30828094482422,
      "activations/layer7_attention_weight_min": -93.62310028076172,
      "activations/layer8_attention_weight_max": 43.687538146972656,
      "activations/layer8_attention_weight_min": -45.57878494262695,
      "activations/layer9_attention_weight_max": 31.29636573791504,
      "activations/layer9_attention_weight_min": -37.6395378112793,
      "epoch": 19.25,
      "learning_rate": 2.6104166666666666e-05,
      "loss": 2.7207,
      "step": 331350
    },
    {
      "activations/layer0_attention_weight_max": 16.386110305786133,
      "activations/layer0_attention_weight_min": -13.54541015625,
      "activations/layer10_attention_weight_max": 34.21115493774414,
      "activations/layer10_attention_weight_min": -34.454044342041016,
      "activations/layer11_attention_weight_max": 34.272491455078125,
      "activations/layer11_attention_weight_min": -33.82936477661133,
      "activations/layer12_attention_weight_max": 30.828554153442383,
      "activations/layer12_attention_weight_min": -28.00497055053711,
      "activations/layer13_attention_weight_max": 43.65840530395508,
      "activations/layer13_attention_weight_min": -38.42918014526367,
      "activations/layer14_attention_weight_max": 48.963382720947266,
      "activations/layer14_attention_weight_min": -40.105979919433594,
      "activations/layer15_attention_weight_max": 42.69143295288086,
      "activations/layer15_attention_weight_min": -38.27412796020508,
      "activations/layer16_attention_weight_max": 31.28075408935547,
      "activations/layer16_attention_weight_min": -27.285465240478516,
      "activations/layer17_attention_weight_max": 29.922632217407227,
      "activations/layer17_attention_weight_min": -25.47994041442871,
      "activations/layer18_attention_weight_max": 33.528656005859375,
      "activations/layer18_attention_weight_min": -24.790523529052734,
      "activations/layer19_attention_weight_max": 36.62776565551758,
      "activations/layer19_attention_weight_min": -31.088788986206055,
      "activations/layer1_attention_weight_max": 16.43425178527832,
      "activations/layer1_attention_weight_min": -14.412019729614258,
      "activations/layer20_attention_weight_max": 32.5365104675293,
      "activations/layer20_attention_weight_min": -25.553930282592773,
      "activations/layer21_attention_weight_max": 30.917953491210938,
      "activations/layer21_attention_weight_min": -24.83806610107422,
      "activations/layer22_attention_weight_max": 44.42717361450195,
      "activations/layer22_attention_weight_min": -30.909564971923828,
      "activations/layer23_attention_weight_max": 34.577518463134766,
      "activations/layer23_attention_weight_min": -24.92817497253418,
      "activations/layer2_attention_weight_max": 31.862964630126953,
      "activations/layer2_attention_weight_min": -29.890010833740234,
      "activations/layer3_attention_weight_max": 94.36688232421875,
      "activations/layer3_attention_weight_min": -96.0281982421875,
      "activations/layer4_attention_weight_max": 104.68865966796875,
      "activations/layer4_attention_weight_min": -108.56710052490234,
      "activations/layer5_attention_weight_max": 49.30823516845703,
      "activations/layer5_attention_weight_min": -60.93457794189453,
      "activations/layer6_attention_weight_max": 44.21304702758789,
      "activations/layer6_attention_weight_min": -46.52566146850586,
      "activations/layer7_attention_weight_max": 91.71603393554688,
      "activations/layer7_attention_weight_min": -100.22442626953125,
      "activations/layer8_attention_weight_max": 41.19276428222656,
      "activations/layer8_attention_weight_min": -45.40422439575195,
      "activations/layer9_attention_weight_max": 32.555908203125,
      "activations/layer9_attention_weight_min": -36.68734359741211,
      "epoch": 19.26,
      "learning_rate": 2.6085227272727268e-05,
      "loss": 2.713,
      "step": 331400
    },
    {
      "activations/layer0_attention_weight_max": 16.69008445739746,
      "activations/layer0_attention_weight_min": -12.745803833007812,
      "activations/layer10_attention_weight_max": 41.66557312011719,
      "activations/layer10_attention_weight_min": -41.13140869140625,
      "activations/layer11_attention_weight_max": 39.462738037109375,
      "activations/layer11_attention_weight_min": -41.52360534667969,
      "activations/layer12_attention_weight_max": 31.40604019165039,
      "activations/layer12_attention_weight_min": -30.280818939208984,
      "activations/layer13_attention_weight_max": 40.82145309448242,
      "activations/layer13_attention_weight_min": -40.263694763183594,
      "activations/layer14_attention_weight_max": 45.821128845214844,
      "activations/layer14_attention_weight_min": -42.91929626464844,
      "activations/layer15_attention_weight_max": 41.79817581176758,
      "activations/layer15_attention_weight_min": -44.610740661621094,
      "activations/layer16_attention_weight_max": 31.277721405029297,
      "activations/layer16_attention_weight_min": -29.780149459838867,
      "activations/layer17_attention_weight_max": 32.20524597167969,
      "activations/layer17_attention_weight_min": -29.55386734008789,
      "activations/layer18_attention_weight_max": 34.54441452026367,
      "activations/layer18_attention_weight_min": -29.507539749145508,
      "activations/layer19_attention_weight_max": 35.849918365478516,
      "activations/layer19_attention_weight_min": -35.38541793823242,
      "activations/layer1_attention_weight_max": 15.891128540039062,
      "activations/layer1_attention_weight_min": -15.376595497131348,
      "activations/layer20_attention_weight_max": 30.42254638671875,
      "activations/layer20_attention_weight_min": -25.713134765625,
      "activations/layer21_attention_weight_max": 31.284072875976562,
      "activations/layer21_attention_weight_min": -26.23986053466797,
      "activations/layer22_attention_weight_max": 48.150360107421875,
      "activations/layer22_attention_weight_min": -35.174007415771484,
      "activations/layer23_attention_weight_max": 35.286033630371094,
      "activations/layer23_attention_weight_min": -27.14334487915039,
      "activations/layer2_attention_weight_max": 33.40706253051758,
      "activations/layer2_attention_weight_min": -31.243183135986328,
      "activations/layer3_attention_weight_max": 97.8663558959961,
      "activations/layer3_attention_weight_min": -97.49542236328125,
      "activations/layer4_attention_weight_max": 103.5044937133789,
      "activations/layer4_attention_weight_min": -106.02471923828125,
      "activations/layer5_attention_weight_max": 51.65177917480469,
      "activations/layer5_attention_weight_min": -60.82424545288086,
      "activations/layer6_attention_weight_max": 44.68650436401367,
      "activations/layer6_attention_weight_min": -48.31007766723633,
      "activations/layer7_attention_weight_max": 95.75250244140625,
      "activations/layer7_attention_weight_min": -103.76738739013672,
      "activations/layer8_attention_weight_max": 46.203006744384766,
      "activations/layer8_attention_weight_min": -47.68803787231445,
      "activations/layer9_attention_weight_max": 36.386146545410156,
      "activations/layer9_attention_weight_min": -39.34607696533203,
      "epoch": 19.26,
      "learning_rate": 2.6066287878787876e-05,
      "loss": 2.7266,
      "step": 331450
    },
    {
      "activations/layer0_attention_weight_max": 15.345930099487305,
      "activations/layer0_attention_weight_min": -12.491795539855957,
      "activations/layer10_attention_weight_max": 34.40650177001953,
      "activations/layer10_attention_weight_min": -34.53715133666992,
      "activations/layer11_attention_weight_max": 33.88794708251953,
      "activations/layer11_attention_weight_min": -34.02497482299805,
      "activations/layer12_attention_weight_max": 29.523277282714844,
      "activations/layer12_attention_weight_min": -27.074668884277344,
      "activations/layer13_attention_weight_max": 38.64331817626953,
      "activations/layer13_attention_weight_min": -36.56681442260742,
      "activations/layer14_attention_weight_max": 40.76435470581055,
      "activations/layer14_attention_weight_min": -39.10649490356445,
      "activations/layer15_attention_weight_max": 36.50638198852539,
      "activations/layer15_attention_weight_min": -36.51622009277344,
      "activations/layer16_attention_weight_max": 29.187355041503906,
      "activations/layer16_attention_weight_min": -27.899503707885742,
      "activations/layer17_attention_weight_max": 29.413339614868164,
      "activations/layer17_attention_weight_min": -25.060443878173828,
      "activations/layer18_attention_weight_max": 31.05232048034668,
      "activations/layer18_attention_weight_min": -24.813045501708984,
      "activations/layer19_attention_weight_max": 34.513816833496094,
      "activations/layer19_attention_weight_min": -30.290424346923828,
      "activations/layer1_attention_weight_max": 15.233843803405762,
      "activations/layer1_attention_weight_min": -14.147980690002441,
      "activations/layer20_attention_weight_max": 27.715513229370117,
      "activations/layer20_attention_weight_min": -26.298625946044922,
      "activations/layer21_attention_weight_max": 25.761547088623047,
      "activations/layer21_attention_weight_min": -25.675365447998047,
      "activations/layer22_attention_weight_max": 41.624977111816406,
      "activations/layer22_attention_weight_min": -31.48832130432129,
      "activations/layer23_attention_weight_max": 32.64753341674805,
      "activations/layer23_attention_weight_min": -25.262510299682617,
      "activations/layer2_attention_weight_max": 32.72572326660156,
      "activations/layer2_attention_weight_min": -32.559913635253906,
      "activations/layer3_attention_weight_max": 99.17753601074219,
      "activations/layer3_attention_weight_min": -101.64508819580078,
      "activations/layer4_attention_weight_max": 104.8440170288086,
      "activations/layer4_attention_weight_min": -108.66539764404297,
      "activations/layer5_attention_weight_max": 49.91216278076172,
      "activations/layer5_attention_weight_min": -62.62777328491211,
      "activations/layer6_attention_weight_max": 46.95955276489258,
      "activations/layer6_attention_weight_min": -49.734066009521484,
      "activations/layer7_attention_weight_max": 96.82545471191406,
      "activations/layer7_attention_weight_min": -100.76652526855469,
      "activations/layer8_attention_weight_max": 46.15147018432617,
      "activations/layer8_attention_weight_min": -46.75247573852539,
      "activations/layer9_attention_weight_max": 34.37644577026367,
      "activations/layer9_attention_weight_min": -34.84028244018555,
      "epoch": 19.26,
      "learning_rate": 2.604734848484848e-05,
      "loss": 2.7311,
      "step": 331500
    },
    {
      "activations/layer0_attention_weight_max": 17.57207679748535,
      "activations/layer0_attention_weight_min": -13.380615234375,
      "activations/layer10_attention_weight_max": 33.78805160522461,
      "activations/layer10_attention_weight_min": -32.78023147583008,
      "activations/layer11_attention_weight_max": 35.36284255981445,
      "activations/layer11_attention_weight_min": -32.853450775146484,
      "activations/layer12_attention_weight_max": 28.17227554321289,
      "activations/layer12_attention_weight_min": -25.999656677246094,
      "activations/layer13_attention_weight_max": 41.62250518798828,
      "activations/layer13_attention_weight_min": -34.28485870361328,
      "activations/layer14_attention_weight_max": 42.457000732421875,
      "activations/layer14_attention_weight_min": -36.90937423706055,
      "activations/layer15_attention_weight_max": 40.462982177734375,
      "activations/layer15_attention_weight_min": -34.483489990234375,
      "activations/layer16_attention_weight_max": 29.746257781982422,
      "activations/layer16_attention_weight_min": -28.462312698364258,
      "activations/layer17_attention_weight_max": 30.9945011138916,
      "activations/layer17_attention_weight_min": -25.387117385864258,
      "activations/layer18_attention_weight_max": 31.631694793701172,
      "activations/layer18_attention_weight_min": -26.74951934814453,
      "activations/layer19_attention_weight_max": 31.70709800720215,
      "activations/layer19_attention_weight_min": -30.58772087097168,
      "activations/layer1_attention_weight_max": 15.914794921875,
      "activations/layer1_attention_weight_min": -13.957398414611816,
      "activations/layer20_attention_weight_max": 29.581132888793945,
      "activations/layer20_attention_weight_min": -25.28884506225586,
      "activations/layer21_attention_weight_max": 28.792003631591797,
      "activations/layer21_attention_weight_min": -25.154037475585938,
      "activations/layer22_attention_weight_max": 41.29037094116211,
      "activations/layer22_attention_weight_min": -33.87533950805664,
      "activations/layer23_attention_weight_max": 31.793254852294922,
      "activations/layer23_attention_weight_min": -26.951839447021484,
      "activations/layer2_attention_weight_max": 31.4533748626709,
      "activations/layer2_attention_weight_min": -31.61732292175293,
      "activations/layer3_attention_weight_max": 89.1488037109375,
      "activations/layer3_attention_weight_min": -91.74429321289062,
      "activations/layer4_attention_weight_max": 98.5496826171875,
      "activations/layer4_attention_weight_min": -98.25518035888672,
      "activations/layer5_attention_weight_max": 50.62623596191406,
      "activations/layer5_attention_weight_min": -59.01890182495117,
      "activations/layer6_attention_weight_max": 43.79426574707031,
      "activations/layer6_attention_weight_min": -44.83637619018555,
      "activations/layer7_attention_weight_max": 95.2956771850586,
      "activations/layer7_attention_weight_min": -91.0733642578125,
      "activations/layer8_attention_weight_max": 44.356536865234375,
      "activations/layer8_attention_weight_min": -41.971256256103516,
      "activations/layer9_attention_weight_max": 31.760961532592773,
      "activations/layer9_attention_weight_min": -32.94453811645508,
      "epoch": 19.26,
      "learning_rate": 2.602840909090909e-05,
      "loss": 2.7176,
      "step": 331550
    },
    {
      "activations/layer0_attention_weight_max": 15.571757316589355,
      "activations/layer0_attention_weight_min": -12.153388977050781,
      "activations/layer10_attention_weight_max": 36.199642181396484,
      "activations/layer10_attention_weight_min": -34.02989196777344,
      "activations/layer11_attention_weight_max": 34.49439239501953,
      "activations/layer11_attention_weight_min": -35.01768493652344,
      "activations/layer12_attention_weight_max": 26.604616165161133,
      "activations/layer12_attention_weight_min": -31.88892936706543,
      "activations/layer13_attention_weight_max": 38.889583587646484,
      "activations/layer13_attention_weight_min": -35.79991149902344,
      "activations/layer14_attention_weight_max": 40.388275146484375,
      "activations/layer14_attention_weight_min": -38.13159942626953,
      "activations/layer15_attention_weight_max": 37.726837158203125,
      "activations/layer15_attention_weight_min": -35.572776794433594,
      "activations/layer16_attention_weight_max": 29.073501586914062,
      "activations/layer16_attention_weight_min": -28.361814498901367,
      "activations/layer17_attention_weight_max": 30.773645401000977,
      "activations/layer17_attention_weight_min": -26.16958236694336,
      "activations/layer18_attention_weight_max": 31.092388153076172,
      "activations/layer18_attention_weight_min": -24.991649627685547,
      "activations/layer19_attention_weight_max": 36.28377914428711,
      "activations/layer19_attention_weight_min": -31.290563583374023,
      "activations/layer1_attention_weight_max": 16.135643005371094,
      "activations/layer1_attention_weight_min": -14.014494895935059,
      "activations/layer20_attention_weight_max": 28.728912353515625,
      "activations/layer20_attention_weight_min": -25.584754943847656,
      "activations/layer21_attention_weight_max": 27.75835418701172,
      "activations/layer21_attention_weight_min": -23.112045288085938,
      "activations/layer22_attention_weight_max": 41.47296905517578,
      "activations/layer22_attention_weight_min": -31.212589263916016,
      "activations/layer23_attention_weight_max": 34.5235481262207,
      "activations/layer23_attention_weight_min": -25.7636775970459,
      "activations/layer2_attention_weight_max": 33.150604248046875,
      "activations/layer2_attention_weight_min": -32.54180908203125,
      "activations/layer3_attention_weight_max": 98.42211151123047,
      "activations/layer3_attention_weight_min": -103.07970428466797,
      "activations/layer4_attention_weight_max": 107.50016021728516,
      "activations/layer4_attention_weight_min": -110.25565338134766,
      "activations/layer5_attention_weight_max": 51.94778060913086,
      "activations/layer5_attention_weight_min": -61.789306640625,
      "activations/layer6_attention_weight_max": 44.53911590576172,
      "activations/layer6_attention_weight_min": -49.92304992675781,
      "activations/layer7_attention_weight_max": 99.37033081054688,
      "activations/layer7_attention_weight_min": -98.57398986816406,
      "activations/layer8_attention_weight_max": 42.898353576660156,
      "activations/layer8_attention_weight_min": -44.157283782958984,
      "activations/layer9_attention_weight_max": 32.57622528076172,
      "activations/layer9_attention_weight_min": -34.496212005615234,
      "epoch": 19.27,
      "learning_rate": 2.6009469696969698e-05,
      "loss": 2.7101,
      "step": 331600
    },
    {
      "activations/layer0_attention_weight_max": 16.30333709716797,
      "activations/layer0_attention_weight_min": -12.956265449523926,
      "activations/layer10_attention_weight_max": 39.075477600097656,
      "activations/layer10_attention_weight_min": -38.86098098754883,
      "activations/layer11_attention_weight_max": 39.74149703979492,
      "activations/layer11_attention_weight_min": -40.18223571777344,
      "activations/layer12_attention_weight_max": 25.18990707397461,
      "activations/layer12_attention_weight_min": -26.930540084838867,
      "activations/layer13_attention_weight_max": 34.93526840209961,
      "activations/layer13_attention_weight_min": -33.64020919799805,
      "activations/layer14_attention_weight_max": 39.547569274902344,
      "activations/layer14_attention_weight_min": -36.17770767211914,
      "activations/layer15_attention_weight_max": 35.13998794555664,
      "activations/layer15_attention_weight_min": -35.638240814208984,
      "activations/layer16_attention_weight_max": 26.75360870361328,
      "activations/layer16_attention_weight_min": -27.740291595458984,
      "activations/layer17_attention_weight_max": 27.06900405883789,
      "activations/layer17_attention_weight_min": -25.379465103149414,
      "activations/layer18_attention_weight_max": 28.85927963256836,
      "activations/layer18_attention_weight_min": -24.612680435180664,
      "activations/layer19_attention_weight_max": 30.959035873413086,
      "activations/layer19_attention_weight_min": -28.922101974487305,
      "activations/layer1_attention_weight_max": 15.917713165283203,
      "activations/layer1_attention_weight_min": -14.383413314819336,
      "activations/layer20_attention_weight_max": 24.720603942871094,
      "activations/layer20_attention_weight_min": -24.046875,
      "activations/layer21_attention_weight_max": 24.2567138671875,
      "activations/layer21_attention_weight_min": -23.407188415527344,
      "activations/layer22_attention_weight_max": 35.097198486328125,
      "activations/layer22_attention_weight_min": -28.91928482055664,
      "activations/layer23_attention_weight_max": 27.567230224609375,
      "activations/layer23_attention_weight_min": -26.345701217651367,
      "activations/layer2_attention_weight_max": 31.72511863708496,
      "activations/layer2_attention_weight_min": -31.918746948242188,
      "activations/layer3_attention_weight_max": 96.7112045288086,
      "activations/layer3_attention_weight_min": -98.28872680664062,
      "activations/layer4_attention_weight_max": 105.44164276123047,
      "activations/layer4_attention_weight_min": -107.25089263916016,
      "activations/layer5_attention_weight_max": 52.44745635986328,
      "activations/layer5_attention_weight_min": -61.64652633666992,
      "activations/layer6_attention_weight_max": 46.389915466308594,
      "activations/layer6_attention_weight_min": -47.35719680786133,
      "activations/layer7_attention_weight_max": 97.54839324951172,
      "activations/layer7_attention_weight_min": -100.57725524902344,
      "activations/layer8_attention_weight_max": 44.62148666381836,
      "activations/layer8_attention_weight_min": -45.64241027832031,
      "activations/layer9_attention_weight_max": 34.86921310424805,
      "activations/layer9_attention_weight_min": -36.95943832397461,
      "epoch": 19.27,
      "learning_rate": 2.59905303030303e-05,
      "loss": 2.7162,
      "step": 331650
    },
    {
      "activations/layer0_attention_weight_max": 15.71279525756836,
      "activations/layer0_attention_weight_min": -12.473529815673828,
      "activations/layer10_attention_weight_max": 35.295223236083984,
      "activations/layer10_attention_weight_min": -37.150390625,
      "activations/layer11_attention_weight_max": 35.87512969970703,
      "activations/layer11_attention_weight_min": -36.51671600341797,
      "activations/layer12_attention_weight_max": 29.7888240814209,
      "activations/layer12_attention_weight_min": -28.97024154663086,
      "activations/layer13_attention_weight_max": 43.56924819946289,
      "activations/layer13_attention_weight_min": -39.50811004638672,
      "activations/layer14_attention_weight_max": 43.53750991821289,
      "activations/layer14_attention_weight_min": -39.986351013183594,
      "activations/layer15_attention_weight_max": 42.35383224487305,
      "activations/layer15_attention_weight_min": -37.73097610473633,
      "activations/layer16_attention_weight_max": 31.95363426208496,
      "activations/layer16_attention_weight_min": -29.87874412536621,
      "activations/layer17_attention_weight_max": 32.3021125793457,
      "activations/layer17_attention_weight_min": -27.099445343017578,
      "activations/layer18_attention_weight_max": 34.49219512939453,
      "activations/layer18_attention_weight_min": -26.127187728881836,
      "activations/layer19_attention_weight_max": 34.95656967163086,
      "activations/layer19_attention_weight_min": -33.44706344604492,
      "activations/layer1_attention_weight_max": 17.219696044921875,
      "activations/layer1_attention_weight_min": -13.971263885498047,
      "activations/layer20_attention_weight_max": 31.31569480895996,
      "activations/layer20_attention_weight_min": -26.647010803222656,
      "activations/layer21_attention_weight_max": 32.32864761352539,
      "activations/layer21_attention_weight_min": -26.368297576904297,
      "activations/layer22_attention_weight_max": 41.52435302734375,
      "activations/layer22_attention_weight_min": -33.73104476928711,
      "activations/layer23_attention_weight_max": 32.41958999633789,
      "activations/layer23_attention_weight_min": -27.113574981689453,
      "activations/layer2_attention_weight_max": 31.43550682067871,
      "activations/layer2_attention_weight_min": -31.664104461669922,
      "activations/layer3_attention_weight_max": 98.38530731201172,
      "activations/layer3_attention_weight_min": -100.8853530883789,
      "activations/layer4_attention_weight_max": 103.43267822265625,
      "activations/layer4_attention_weight_min": -108.13703918457031,
      "activations/layer5_attention_weight_max": 50.50712585449219,
      "activations/layer5_attention_weight_min": -60.5445671081543,
      "activations/layer6_attention_weight_max": 43.949100494384766,
      "activations/layer6_attention_weight_min": -47.88185119628906,
      "activations/layer7_attention_weight_max": 93.74332427978516,
      "activations/layer7_attention_weight_min": -100.33053588867188,
      "activations/layer8_attention_weight_max": 42.21643829345703,
      "activations/layer8_attention_weight_min": -45.89786148071289,
      "activations/layer9_attention_weight_max": 32.63462829589844,
      "activations/layer9_attention_weight_min": -37.209877014160156,
      "epoch": 19.27,
      "learning_rate": 2.5971590909090907e-05,
      "loss": 2.7241,
      "step": 331700
    },
    {
      "activations/layer0_attention_weight_max": 16.765174865722656,
      "activations/layer0_attention_weight_min": -12.85030746459961,
      "activations/layer10_attention_weight_max": 36.00624084472656,
      "activations/layer10_attention_weight_min": -34.934608459472656,
      "activations/layer11_attention_weight_max": 35.20820236206055,
      "activations/layer11_attention_weight_min": -34.61429214477539,
      "activations/layer12_attention_weight_max": 28.490638732910156,
      "activations/layer12_attention_weight_min": -28.155611038208008,
      "activations/layer13_attention_weight_max": 40.2901611328125,
      "activations/layer13_attention_weight_min": -36.68538284301758,
      "activations/layer14_attention_weight_max": 42.872398376464844,
      "activations/layer14_attention_weight_min": -38.178497314453125,
      "activations/layer15_attention_weight_max": 38.53526306152344,
      "activations/layer15_attention_weight_min": -35.550235748291016,
      "activations/layer16_attention_weight_max": 28.79007911682129,
      "activations/layer16_attention_weight_min": -29.99667739868164,
      "activations/layer17_attention_weight_max": 30.49905776977539,
      "activations/layer17_attention_weight_min": -25.80645751953125,
      "activations/layer18_attention_weight_max": 32.57509231567383,
      "activations/layer18_attention_weight_min": -25.390310287475586,
      "activations/layer19_attention_weight_max": 32.072940826416016,
      "activations/layer19_attention_weight_min": -34.315948486328125,
      "activations/layer1_attention_weight_max": 15.958030700683594,
      "activations/layer1_attention_weight_min": -15.187280654907227,
      "activations/layer20_attention_weight_max": 27.645261764526367,
      "activations/layer20_attention_weight_min": -25.500255584716797,
      "activations/layer21_attention_weight_max": 28.79396629333496,
      "activations/layer21_attention_weight_min": -24.49315643310547,
      "activations/layer22_attention_weight_max": 46.415035247802734,
      "activations/layer22_attention_weight_min": -32.109066009521484,
      "activations/layer23_attention_weight_max": 31.393516540527344,
      "activations/layer23_attention_weight_min": -27.35811996459961,
      "activations/layer2_attention_weight_max": 31.54117202758789,
      "activations/layer2_attention_weight_min": -31.098241806030273,
      "activations/layer3_attention_weight_max": 96.7594223022461,
      "activations/layer3_attention_weight_min": -98.30912017822266,
      "activations/layer4_attention_weight_max": 107.4128189086914,
      "activations/layer4_attention_weight_min": -106.9036636352539,
      "activations/layer5_attention_weight_max": 51.99176025390625,
      "activations/layer5_attention_weight_min": -60.876766204833984,
      "activations/layer6_attention_weight_max": 43.093238830566406,
      "activations/layer6_attention_weight_min": -47.53432846069336,
      "activations/layer7_attention_weight_max": 94.45223999023438,
      "activations/layer7_attention_weight_min": -96.45559692382812,
      "activations/layer8_attention_weight_max": 40.1669807434082,
      "activations/layer8_attention_weight_min": -42.54767990112305,
      "activations/layer9_attention_weight_max": 33.599998474121094,
      "activations/layer9_attention_weight_min": -33.810882568359375,
      "epoch": 19.28,
      "learning_rate": 2.5952651515151512e-05,
      "loss": 2.7279,
      "step": 331750
    },
    {
      "activations/layer0_attention_weight_max": 16.6086483001709,
      "activations/layer0_attention_weight_min": -12.482218742370605,
      "activations/layer10_attention_weight_max": 36.2534065246582,
      "activations/layer10_attention_weight_min": -36.434730529785156,
      "activations/layer11_attention_weight_max": 34.80342102050781,
      "activations/layer11_attention_weight_min": -33.675926208496094,
      "activations/layer12_attention_weight_max": 28.55307388305664,
      "activations/layer12_attention_weight_min": -27.524248123168945,
      "activations/layer13_attention_weight_max": 41.46880340576172,
      "activations/layer13_attention_weight_min": -37.534950256347656,
      "activations/layer14_attention_weight_max": 45.39243698120117,
      "activations/layer14_attention_weight_min": -41.34671401977539,
      "activations/layer15_attention_weight_max": 40.126251220703125,
      "activations/layer15_attention_weight_min": -41.90302658081055,
      "activations/layer16_attention_weight_max": 29.655107498168945,
      "activations/layer16_attention_weight_min": -27.906625747680664,
      "activations/layer17_attention_weight_max": 29.931108474731445,
      "activations/layer17_attention_weight_min": -27.256366729736328,
      "activations/layer18_attention_weight_max": 32.913734436035156,
      "activations/layer18_attention_weight_min": -26.19487190246582,
      "activations/layer19_attention_weight_max": 32.91020965576172,
      "activations/layer19_attention_weight_min": -31.339563369750977,
      "activations/layer1_attention_weight_max": 15.634334564208984,
      "activations/layer1_attention_weight_min": -15.051280975341797,
      "activations/layer20_attention_weight_max": 29.012529373168945,
      "activations/layer20_attention_weight_min": -24.60597801208496,
      "activations/layer21_attention_weight_max": 34.21589279174805,
      "activations/layer21_attention_weight_min": -25.631332397460938,
      "activations/layer22_attention_weight_max": 39.74496841430664,
      "activations/layer22_attention_weight_min": -30.95296287536621,
      "activations/layer23_attention_weight_max": 32.05691146850586,
      "activations/layer23_attention_weight_min": -25.00720977783203,
      "activations/layer2_attention_weight_max": 31.016990661621094,
      "activations/layer2_attention_weight_min": -31.657054901123047,
      "activations/layer3_attention_weight_max": 98.08086395263672,
      "activations/layer3_attention_weight_min": -98.4495620727539,
      "activations/layer4_attention_weight_max": 104.13744354248047,
      "activations/layer4_attention_weight_min": -106.14128875732422,
      "activations/layer5_attention_weight_max": 50.557830810546875,
      "activations/layer5_attention_weight_min": -62.06993103027344,
      "activations/layer6_attention_weight_max": 42.71678924560547,
      "activations/layer6_attention_weight_min": -46.74515151977539,
      "activations/layer7_attention_weight_max": 93.78024291992188,
      "activations/layer7_attention_weight_min": -94.34262084960938,
      "activations/layer8_attention_weight_max": 42.20550537109375,
      "activations/layer8_attention_weight_min": -44.55263137817383,
      "activations/layer9_attention_weight_max": 32.3226432800293,
      "activations/layer9_attention_weight_min": -34.957176208496094,
      "epoch": 19.28,
      "learning_rate": 2.593371212121212e-05,
      "loss": 2.7054,
      "step": 331800
    },
    {
      "activations/layer0_attention_weight_max": 15.707579612731934,
      "activations/layer0_attention_weight_min": -12.795705795288086,
      "activations/layer10_attention_weight_max": 34.328521728515625,
      "activations/layer10_attention_weight_min": -33.42417526245117,
      "activations/layer11_attention_weight_max": 33.56127166748047,
      "activations/layer11_attention_weight_min": -34.56019592285156,
      "activations/layer12_attention_weight_max": 26.8287296295166,
      "activations/layer12_attention_weight_min": -27.164743423461914,
      "activations/layer13_attention_weight_max": 40.282745361328125,
      "activations/layer13_attention_weight_min": -36.063880920410156,
      "activations/layer14_attention_weight_max": 40.952266693115234,
      "activations/layer14_attention_weight_min": -39.35386276245117,
      "activations/layer15_attention_weight_max": 38.39207077026367,
      "activations/layer15_attention_weight_min": -35.93099594116211,
      "activations/layer16_attention_weight_max": 26.003738403320312,
      "activations/layer16_attention_weight_min": -28.177806854248047,
      "activations/layer17_attention_weight_max": 27.595783233642578,
      "activations/layer17_attention_weight_min": -24.796428680419922,
      "activations/layer18_attention_weight_max": 31.096364974975586,
      "activations/layer18_attention_weight_min": -25.16411781311035,
      "activations/layer19_attention_weight_max": 31.41351890563965,
      "activations/layer19_attention_weight_min": -32.01723861694336,
      "activations/layer1_attention_weight_max": 15.80556869506836,
      "activations/layer1_attention_weight_min": -14.34221363067627,
      "activations/layer20_attention_weight_max": 27.583263397216797,
      "activations/layer20_attention_weight_min": -24.633411407470703,
      "activations/layer21_attention_weight_max": 30.757829666137695,
      "activations/layer21_attention_weight_min": -24.35079002380371,
      "activations/layer22_attention_weight_max": 41.19350051879883,
      "activations/layer22_attention_weight_min": -35.283878326416016,
      "activations/layer23_attention_weight_max": 34.867950439453125,
      "activations/layer23_attention_weight_min": -27.795913696289062,
      "activations/layer2_attention_weight_max": 32.35565948486328,
      "activations/layer2_attention_weight_min": -31.669103622436523,
      "activations/layer3_attention_weight_max": 97.2155532836914,
      "activations/layer3_attention_weight_min": -101.67804718017578,
      "activations/layer4_attention_weight_max": 105.44095611572266,
      "activations/layer4_attention_weight_min": -104.72705078125,
      "activations/layer5_attention_weight_max": 52.049110412597656,
      "activations/layer5_attention_weight_min": -61.96898651123047,
      "activations/layer6_attention_weight_max": 44.89992141723633,
      "activations/layer6_attention_weight_min": -45.677974700927734,
      "activations/layer7_attention_weight_max": 93.0696792602539,
      "activations/layer7_attention_weight_min": -93.87987518310547,
      "activations/layer8_attention_weight_max": 42.239990234375,
      "activations/layer8_attention_weight_min": -44.7956657409668,
      "activations/layer9_attention_weight_max": 33.07172393798828,
      "activations/layer9_attention_weight_min": -33.816829681396484,
      "epoch": 19.28,
      "learning_rate": 2.5914772727272725e-05,
      "loss": 2.697,
      "step": 331850
    },
    {
      "activations/layer0_attention_weight_max": 16.52989959716797,
      "activations/layer0_attention_weight_min": -13.927663803100586,
      "activations/layer10_attention_weight_max": 37.7811164855957,
      "activations/layer10_attention_weight_min": -35.071659088134766,
      "activations/layer11_attention_weight_max": 35.60190963745117,
      "activations/layer11_attention_weight_min": -35.43461990356445,
      "activations/layer12_attention_weight_max": 27.234079360961914,
      "activations/layer12_attention_weight_min": -26.078733444213867,
      "activations/layer13_attention_weight_max": 39.218624114990234,
      "activations/layer13_attention_weight_min": -34.25967788696289,
      "activations/layer14_attention_weight_max": 41.255035400390625,
      "activations/layer14_attention_weight_min": -35.46879959106445,
      "activations/layer15_attention_weight_max": 35.01853561401367,
      "activations/layer15_attention_weight_min": -34.53089904785156,
      "activations/layer16_attention_weight_max": 30.155025482177734,
      "activations/layer16_attention_weight_min": -26.669429779052734,
      "activations/layer17_attention_weight_max": 29.014339447021484,
      "activations/layer17_attention_weight_min": -24.11663246154785,
      "activations/layer18_attention_weight_max": 31.69146156311035,
      "activations/layer18_attention_weight_min": -24.945703506469727,
      "activations/layer19_attention_weight_max": 31.906335830688477,
      "activations/layer19_attention_weight_min": -29.512710571289062,
      "activations/layer1_attention_weight_max": 15.612150192260742,
      "activations/layer1_attention_weight_min": -14.235127449035645,
      "activations/layer20_attention_weight_max": 26.918529510498047,
      "activations/layer20_attention_weight_min": -26.840923309326172,
      "activations/layer21_attention_weight_max": 28.99032211303711,
      "activations/layer21_attention_weight_min": -25.50625991821289,
      "activations/layer22_attention_weight_max": 41.33844757080078,
      "activations/layer22_attention_weight_min": -29.63284683227539,
      "activations/layer23_attention_weight_max": 33.21731185913086,
      "activations/layer23_attention_weight_min": -25.01681137084961,
      "activations/layer2_attention_weight_max": 33.65727996826172,
      "activations/layer2_attention_weight_min": -33.998538970947266,
      "activations/layer3_attention_weight_max": 96.74234008789062,
      "activations/layer3_attention_weight_min": -101.90071868896484,
      "activations/layer4_attention_weight_max": 102.9012222290039,
      "activations/layer4_attention_weight_min": -103.02599334716797,
      "activations/layer5_attention_weight_max": 47.84797286987305,
      "activations/layer5_attention_weight_min": -60.11467742919922,
      "activations/layer6_attention_weight_max": 42.58509826660156,
      "activations/layer6_attention_weight_min": -47.11935806274414,
      "activations/layer7_attention_weight_max": 96.32365417480469,
      "activations/layer7_attention_weight_min": -96.30384063720703,
      "activations/layer8_attention_weight_max": 41.86973190307617,
      "activations/layer8_attention_weight_min": -43.228240966796875,
      "activations/layer9_attention_weight_max": 31.379840850830078,
      "activations/layer9_attention_weight_min": -33.96474075317383,
      "epoch": 19.29,
      "learning_rate": 2.5895833333333334e-05,
      "loss": 2.72,
      "step": 331900
    },
    {
      "activations/layer0_attention_weight_max": 16.136940002441406,
      "activations/layer0_attention_weight_min": -12.352208137512207,
      "activations/layer10_attention_weight_max": 39.21155548095703,
      "activations/layer10_attention_weight_min": -39.28168487548828,
      "activations/layer11_attention_weight_max": 37.55098342895508,
      "activations/layer11_attention_weight_min": -40.249244689941406,
      "activations/layer12_attention_weight_max": 29.21135902404785,
      "activations/layer12_attention_weight_min": -29.13970184326172,
      "activations/layer13_attention_weight_max": 43.96647644042969,
      "activations/layer13_attention_weight_min": -38.52251434326172,
      "activations/layer14_attention_weight_max": 42.67673873901367,
      "activations/layer14_attention_weight_min": -40.28780746459961,
      "activations/layer15_attention_weight_max": 41.84819793701172,
      "activations/layer15_attention_weight_min": -39.034114837646484,
      "activations/layer16_attention_weight_max": 30.568984985351562,
      "activations/layer16_attention_weight_min": -30.870365142822266,
      "activations/layer17_attention_weight_max": 31.676984786987305,
      "activations/layer17_attention_weight_min": -25.574975967407227,
      "activations/layer18_attention_weight_max": 30.81049156188965,
      "activations/layer18_attention_weight_min": -26.089021682739258,
      "activations/layer19_attention_weight_max": 33.84712600708008,
      "activations/layer19_attention_weight_min": -31.522085189819336,
      "activations/layer1_attention_weight_max": 15.846246719360352,
      "activations/layer1_attention_weight_min": -13.906929969787598,
      "activations/layer20_attention_weight_max": 27.799217224121094,
      "activations/layer20_attention_weight_min": -24.347856521606445,
      "activations/layer21_attention_weight_max": 27.19356918334961,
      "activations/layer21_attention_weight_min": -23.934438705444336,
      "activations/layer22_attention_weight_max": 37.1940803527832,
      "activations/layer22_attention_weight_min": -31.275354385375977,
      "activations/layer23_attention_weight_max": 30.330291748046875,
      "activations/layer23_attention_weight_min": -26.016021728515625,
      "activations/layer2_attention_weight_max": 31.841964721679688,
      "activations/layer2_attention_weight_min": -31.615032196044922,
      "activations/layer3_attention_weight_max": 97.43470001220703,
      "activations/layer3_attention_weight_min": -96.98473358154297,
      "activations/layer4_attention_weight_max": 101.07555389404297,
      "activations/layer4_attention_weight_min": -107.44097137451172,
      "activations/layer5_attention_weight_max": 52.12409973144531,
      "activations/layer5_attention_weight_min": -63.21424865722656,
      "activations/layer6_attention_weight_max": 43.78428268432617,
      "activations/layer6_attention_weight_min": -46.50400924682617,
      "activations/layer7_attention_weight_max": 96.7763442993164,
      "activations/layer7_attention_weight_min": -97.29481506347656,
      "activations/layer8_attention_weight_max": 44.26426696777344,
      "activations/layer8_attention_weight_min": -46.19734573364258,
      "activations/layer9_attention_weight_max": 34.31315231323242,
      "activations/layer9_attention_weight_min": -37.64408493041992,
      "epoch": 19.29,
      "learning_rate": 2.5876893939393935e-05,
      "loss": 2.7192,
      "step": 331950
    },
    {
      "activations/layer0_attention_weight_max": 16.572298049926758,
      "activations/layer0_attention_weight_min": -13.496199607849121,
      "activations/layer10_attention_weight_max": 33.319217681884766,
      "activations/layer10_attention_weight_min": -34.12372970581055,
      "activations/layer11_attention_weight_max": 33.834693908691406,
      "activations/layer11_attention_weight_min": -33.821075439453125,
      "activations/layer12_attention_weight_max": 27.654199600219727,
      "activations/layer12_attention_weight_min": -28.756120681762695,
      "activations/layer13_attention_weight_max": 41.11018753051758,
      "activations/layer13_attention_weight_min": -39.77205276489258,
      "activations/layer14_attention_weight_max": 47.59296798706055,
      "activations/layer14_attention_weight_min": -42.111541748046875,
      "activations/layer15_attention_weight_max": 43.481956481933594,
      "activations/layer15_attention_weight_min": -38.79096984863281,
      "activations/layer16_attention_weight_max": 30.95563507080078,
      "activations/layer16_attention_weight_min": -28.52901840209961,
      "activations/layer17_attention_weight_max": 33.45411682128906,
      "activations/layer17_attention_weight_min": -26.578577041625977,
      "activations/layer18_attention_weight_max": 34.71052932739258,
      "activations/layer18_attention_weight_min": -27.412405014038086,
      "activations/layer19_attention_weight_max": 41.240997314453125,
      "activations/layer19_attention_weight_min": -33.19466781616211,
      "activations/layer1_attention_weight_max": 15.303208351135254,
      "activations/layer1_attention_weight_min": -14.913286209106445,
      "activations/layer20_attention_weight_max": 37.32400131225586,
      "activations/layer20_attention_weight_min": -27.1097469329834,
      "activations/layer21_attention_weight_max": 33.49130630493164,
      "activations/layer21_attention_weight_min": -26.428531646728516,
      "activations/layer22_attention_weight_max": 48.369110107421875,
      "activations/layer22_attention_weight_min": -34.229881286621094,
      "activations/layer23_attention_weight_max": 37.54224395751953,
      "activations/layer23_attention_weight_min": -27.38569450378418,
      "activations/layer2_attention_weight_max": 30.845050811767578,
      "activations/layer2_attention_weight_min": -31.07765007019043,
      "activations/layer3_attention_weight_max": 96.33668518066406,
      "activations/layer3_attention_weight_min": -97.29227447509766,
      "activations/layer4_attention_weight_max": 106.13665008544922,
      "activations/layer4_attention_weight_min": -107.50007629394531,
      "activations/layer5_attention_weight_max": 49.52289581298828,
      "activations/layer5_attention_weight_min": -64.66921997070312,
      "activations/layer6_attention_weight_max": 44.60826110839844,
      "activations/layer6_attention_weight_min": -48.11894989013672,
      "activations/layer7_attention_weight_max": 94.65637969970703,
      "activations/layer7_attention_weight_min": -98.64561462402344,
      "activations/layer8_attention_weight_max": 44.31361770629883,
      "activations/layer8_attention_weight_min": -44.13499069213867,
      "activations/layer9_attention_weight_max": 32.281044006347656,
      "activations/layer9_attention_weight_min": -34.3280143737793,
      "epoch": 19.29,
      "learning_rate": 2.5857954545454543e-05,
      "loss": 2.7214,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4975,
      "eval_samples_per_second": 505.324,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4975,
      "eval_openwebtext_samples_per_second": 505.324,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9988,
      "eval_wikitext_samples_per_second": 228.139,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_lambada_loss": 2.349609375,
      "eval_lambada_ppl": 10.481474598935181,
      "eval_lambada_runtime": 9.5571,
      "eval_lambada_samples_per_second": 509.466,
      "step": 332000
    },
    {
      "activations/layer0_attention_weight_max": 16.211936950683594,
      "activations/layer0_attention_weight_min": -13.924811363220215,
      "activations/layer10_attention_weight_max": 35.25000762939453,
      "activations/layer10_attention_weight_min": -34.4846305847168,
      "activations/layer11_attention_weight_max": 33.47595977783203,
      "activations/layer11_attention_weight_min": -34.1568717956543,
      "activations/layer12_attention_weight_max": 26.6995906829834,
      "activations/layer12_attention_weight_min": -27.839445114135742,
      "activations/layer13_attention_weight_max": 37.6903190612793,
      "activations/layer13_attention_weight_min": -36.73931884765625,
      "activations/layer14_attention_weight_max": 41.9261474609375,
      "activations/layer14_attention_weight_min": -39.12556838989258,
      "activations/layer15_attention_weight_max": 36.61421585083008,
      "activations/layer15_attention_weight_min": -36.572914123535156,
      "activations/layer16_attention_weight_max": 27.869792938232422,
      "activations/layer16_attention_weight_min": -28.187204360961914,
      "activations/layer17_attention_weight_max": 31.120527267456055,
      "activations/layer17_attention_weight_min": -26.8516902923584,
      "activations/layer18_attention_weight_max": 31.409591674804688,
      "activations/layer18_attention_weight_min": -28.40847396850586,
      "activations/layer19_attention_weight_max": 32.449764251708984,
      "activations/layer19_attention_weight_min": -32.741451263427734,
      "activations/layer1_attention_weight_max": 15.919390678405762,
      "activations/layer1_attention_weight_min": -14.685879707336426,
      "activations/layer20_attention_weight_max": 29.607221603393555,
      "activations/layer20_attention_weight_min": -25.951475143432617,
      "activations/layer21_attention_weight_max": 30.6663818359375,
      "activations/layer21_attention_weight_min": -26.796037673950195,
      "activations/layer22_attention_weight_max": 41.78273010253906,
      "activations/layer22_attention_weight_min": -35.79856491088867,
      "activations/layer23_attention_weight_max": 31.792957305908203,
      "activations/layer23_attention_weight_min": -28.390464782714844,
      "activations/layer2_attention_weight_max": 32.309181213378906,
      "activations/layer2_attention_weight_min": -31.293502807617188,
      "activations/layer3_attention_weight_max": 100.79014587402344,
      "activations/layer3_attention_weight_min": -100.45748901367188,
      "activations/layer4_attention_weight_max": 107.92742919921875,
      "activations/layer4_attention_weight_min": -108.154541015625,
      "activations/layer5_attention_weight_max": 50.306068420410156,
      "activations/layer5_attention_weight_min": -62.29777526855469,
      "activations/layer6_attention_weight_max": 45.50956726074219,
      "activations/layer6_attention_weight_min": -47.04013442993164,
      "activations/layer7_attention_weight_max": 92.88494110107422,
      "activations/layer7_attention_weight_min": -93.23737335205078,
      "activations/layer8_attention_weight_max": 45.08480453491211,
      "activations/layer8_attention_weight_min": -43.644264221191406,
      "activations/layer9_attention_weight_max": 37.228981018066406,
      "activations/layer9_attention_weight_min": -35.36747741699219,
      "epoch": 19.29,
      "learning_rate": 2.583901515151515e-05,
      "loss": 2.7267,
      "step": 332050
    },
    {
      "activations/layer0_attention_weight_max": 16.499414443969727,
      "activations/layer0_attention_weight_min": -12.803284645080566,
      "activations/layer10_attention_weight_max": 41.631439208984375,
      "activations/layer10_attention_weight_min": -37.931148529052734,
      "activations/layer11_attention_weight_max": 42.35927963256836,
      "activations/layer11_attention_weight_min": -41.121559143066406,
      "activations/layer12_attention_weight_max": 27.306751251220703,
      "activations/layer12_attention_weight_min": -26.86411476135254,
      "activations/layer13_attention_weight_max": 40.16002655029297,
      "activations/layer13_attention_weight_min": -35.521629333496094,
      "activations/layer14_attention_weight_max": 44.90352249145508,
      "activations/layer14_attention_weight_min": -39.22759246826172,
      "activations/layer15_attention_weight_max": 40.46479034423828,
      "activations/layer15_attention_weight_min": -37.80899429321289,
      "activations/layer16_attention_weight_max": 34.31972122192383,
      "activations/layer16_attention_weight_min": -28.880582809448242,
      "activations/layer17_attention_weight_max": 32.66162872314453,
      "activations/layer17_attention_weight_min": -28.060720443725586,
      "activations/layer18_attention_weight_max": 31.017000198364258,
      "activations/layer18_attention_weight_min": -27.09221076965332,
      "activations/layer19_attention_weight_max": 38.16265106201172,
      "activations/layer19_attention_weight_min": -32.51087951660156,
      "activations/layer1_attention_weight_max": 17.35774040222168,
      "activations/layer1_attention_weight_min": -15.062544822692871,
      "activations/layer20_attention_weight_max": 31.600786209106445,
      "activations/layer20_attention_weight_min": -24.063508987426758,
      "activations/layer21_attention_weight_max": 30.38228988647461,
      "activations/layer21_attention_weight_min": -22.87933349609375,
      "activations/layer22_attention_weight_max": 42.05820083618164,
      "activations/layer22_attention_weight_min": -31.42995262145996,
      "activations/layer23_attention_weight_max": 34.212188720703125,
      "activations/layer23_attention_weight_min": -27.97937774658203,
      "activations/layer2_attention_weight_max": 33.399864196777344,
      "activations/layer2_attention_weight_min": -31.984752655029297,
      "activations/layer3_attention_weight_max": 99.99650573730469,
      "activations/layer3_attention_weight_min": -100.13621520996094,
      "activations/layer4_attention_weight_max": 108.76773834228516,
      "activations/layer4_attention_weight_min": -104.58387756347656,
      "activations/layer5_attention_weight_max": 55.80304718017578,
      "activations/layer5_attention_weight_min": -59.991111755371094,
      "activations/layer6_attention_weight_max": 47.56190872192383,
      "activations/layer6_attention_weight_min": -47.94404983520508,
      "activations/layer7_attention_weight_max": 118.40943908691406,
      "activations/layer7_attention_weight_min": -98.6529769897461,
      "activations/layer8_attention_weight_max": 50.80253219604492,
      "activations/layer8_attention_weight_min": -45.689910888671875,
      "activations/layer9_attention_weight_max": 36.1264533996582,
      "activations/layer9_attention_weight_min": -37.12736129760742,
      "epoch": 19.3,
      "learning_rate": 2.5820075757575757e-05,
      "loss": 2.7175,
      "step": 332100
    },
    {
      "activations/layer0_attention_weight_max": 17.047996520996094,
      "activations/layer0_attention_weight_min": -15.085204124450684,
      "activations/layer10_attention_weight_max": 31.997617721557617,
      "activations/layer10_attention_weight_min": -32.8985710144043,
      "activations/layer11_attention_weight_max": 32.60802459716797,
      "activations/layer11_attention_weight_min": -33.481788635253906,
      "activations/layer12_attention_weight_max": 30.378467559814453,
      "activations/layer12_attention_weight_min": -27.87360191345215,
      "activations/layer13_attention_weight_max": 44.415077209472656,
      "activations/layer13_attention_weight_min": -38.03525924682617,
      "activations/layer14_attention_weight_max": 47.389888763427734,
      "activations/layer14_attention_weight_min": -37.37538528442383,
      "activations/layer15_attention_weight_max": 41.68996810913086,
      "activations/layer15_attention_weight_min": -34.44487380981445,
      "activations/layer16_attention_weight_max": 30.59986114501953,
      "activations/layer16_attention_weight_min": -27.226699829101562,
      "activations/layer17_attention_weight_max": 31.520803451538086,
      "activations/layer17_attention_weight_min": -26.352405548095703,
      "activations/layer18_attention_weight_max": 34.07909393310547,
      "activations/layer18_attention_weight_min": -27.700407028198242,
      "activations/layer19_attention_weight_max": 37.15293884277344,
      "activations/layer19_attention_weight_min": -29.618820190429688,
      "activations/layer1_attention_weight_max": 16.347448348999023,
      "activations/layer1_attention_weight_min": -13.236128807067871,
      "activations/layer20_attention_weight_max": 29.08241844177246,
      "activations/layer20_attention_weight_min": -23.845306396484375,
      "activations/layer21_attention_weight_max": 30.653972625732422,
      "activations/layer21_attention_weight_min": -23.70245361328125,
      "activations/layer22_attention_weight_max": 48.94955825805664,
      "activations/layer22_attention_weight_min": -32.199974060058594,
      "activations/layer23_attention_weight_max": 36.667755126953125,
      "activations/layer23_attention_weight_min": -25.312009811401367,
      "activations/layer2_attention_weight_max": 33.789852142333984,
      "activations/layer2_attention_weight_min": -32.931827545166016,
      "activations/layer3_attention_weight_max": 101.76187133789062,
      "activations/layer3_attention_weight_min": -100.1246337890625,
      "activations/layer4_attention_weight_max": 105.2894058227539,
      "activations/layer4_attention_weight_min": -107.85389709472656,
      "activations/layer5_attention_weight_max": 52.134605407714844,
      "activations/layer5_attention_weight_min": -64.04236602783203,
      "activations/layer6_attention_weight_max": 44.566890716552734,
      "activations/layer6_attention_weight_min": -47.91169357299805,
      "activations/layer7_attention_weight_max": 94.12873840332031,
      "activations/layer7_attention_weight_min": -95.93611907958984,
      "activations/layer8_attention_weight_max": 40.061920166015625,
      "activations/layer8_attention_weight_min": -43.19948959350586,
      "activations/layer9_attention_weight_max": 29.950855255126953,
      "activations/layer9_attention_weight_min": -32.52046585083008,
      "epoch": 19.3,
      "learning_rate": 2.580113636363636e-05,
      "loss": 2.7307,
      "step": 332150
    },
    {
      "activations/layer0_attention_weight_max": 16.970245361328125,
      "activations/layer0_attention_weight_min": -12.392457962036133,
      "activations/layer10_attention_weight_max": 32.467594146728516,
      "activations/layer10_attention_weight_min": -33.83095169067383,
      "activations/layer11_attention_weight_max": 32.78539276123047,
      "activations/layer11_attention_weight_min": -35.348995208740234,
      "activations/layer12_attention_weight_max": 25.9910831451416,
      "activations/layer12_attention_weight_min": -27.681310653686523,
      "activations/layer13_attention_weight_max": 37.511512756347656,
      "activations/layer13_attention_weight_min": -36.76211166381836,
      "activations/layer14_attention_weight_max": 43.10982131958008,
      "activations/layer14_attention_weight_min": -39.5205192565918,
      "activations/layer15_attention_weight_max": 39.4290771484375,
      "activations/layer15_attention_weight_min": -37.4506721496582,
      "activations/layer16_attention_weight_max": 29.65053939819336,
      "activations/layer16_attention_weight_min": -28.171279907226562,
      "activations/layer17_attention_weight_max": 35.75971984863281,
      "activations/layer17_attention_weight_min": -26.038591384887695,
      "activations/layer18_attention_weight_max": 32.9692268371582,
      "activations/layer18_attention_weight_min": -27.310283660888672,
      "activations/layer19_attention_weight_max": 34.918968200683594,
      "activations/layer19_attention_weight_min": -31.954492568969727,
      "activations/layer1_attention_weight_max": 15.62321949005127,
      "activations/layer1_attention_weight_min": -14.208812713623047,
      "activations/layer20_attention_weight_max": 29.751359939575195,
      "activations/layer20_attention_weight_min": -26.414705276489258,
      "activations/layer21_attention_weight_max": 31.356901168823242,
      "activations/layer21_attention_weight_min": -25.135387420654297,
      "activations/layer22_attention_weight_max": 43.484413146972656,
      "activations/layer22_attention_weight_min": -32.80946731567383,
      "activations/layer23_attention_weight_max": 35.281124114990234,
      "activations/layer23_attention_weight_min": -26.774030685424805,
      "activations/layer2_attention_weight_max": 32.050376892089844,
      "activations/layer2_attention_weight_min": -31.724735260009766,
      "activations/layer3_attention_weight_max": 94.78201293945312,
      "activations/layer3_attention_weight_min": -97.50566864013672,
      "activations/layer4_attention_weight_max": 102.0532455444336,
      "activations/layer4_attention_weight_min": -106.23054504394531,
      "activations/layer5_attention_weight_max": 51.73151779174805,
      "activations/layer5_attention_weight_min": -63.490478515625,
      "activations/layer6_attention_weight_max": 45.13869094848633,
      "activations/layer6_attention_weight_min": -47.57585906982422,
      "activations/layer7_attention_weight_max": 92.05059814453125,
      "activations/layer7_attention_weight_min": -94.71856689453125,
      "activations/layer8_attention_weight_max": 39.93961715698242,
      "activations/layer8_attention_weight_min": -45.69634246826172,
      "activations/layer9_attention_weight_max": 33.70888900756836,
      "activations/layer9_attention_weight_min": -33.880855560302734,
      "epoch": 19.3,
      "learning_rate": 2.5782196969696966e-05,
      "loss": 2.7166,
      "step": 332200
    },
    {
      "activations/layer0_attention_weight_max": 15.359450340270996,
      "activations/layer0_attention_weight_min": -11.941179275512695,
      "activations/layer10_attention_weight_max": 33.11600875854492,
      "activations/layer10_attention_weight_min": -36.71919631958008,
      "activations/layer11_attention_weight_max": 34.76237106323242,
      "activations/layer11_attention_weight_min": -37.05140686035156,
      "activations/layer12_attention_weight_max": 32.781639099121094,
      "activations/layer12_attention_weight_min": -28.47028160095215,
      "activations/layer13_attention_weight_max": 44.79993438720703,
      "activations/layer13_attention_weight_min": -37.98079299926758,
      "activations/layer14_attention_weight_max": 48.28680419921875,
      "activations/layer14_attention_weight_min": -41.97330856323242,
      "activations/layer15_attention_weight_max": 47.66480255126953,
      "activations/layer15_attention_weight_min": -40.13506317138672,
      "activations/layer16_attention_weight_max": 38.14616775512695,
      "activations/layer16_attention_weight_min": -32.660404205322266,
      "activations/layer17_attention_weight_max": 38.48978042602539,
      "activations/layer17_attention_weight_min": -32.03419494628906,
      "activations/layer18_attention_weight_max": 36.85273361206055,
      "activations/layer18_attention_weight_min": -30.48261833190918,
      "activations/layer19_attention_weight_max": 41.87786865234375,
      "activations/layer19_attention_weight_min": -35.588253021240234,
      "activations/layer1_attention_weight_max": 15.006436347961426,
      "activations/layer1_attention_weight_min": -14.230104446411133,
      "activations/layer20_attention_weight_max": 37.15846252441406,
      "activations/layer20_attention_weight_min": -27.8482666015625,
      "activations/layer21_attention_weight_max": 36.13819122314453,
      "activations/layer21_attention_weight_min": -26.250736236572266,
      "activations/layer22_attention_weight_max": 49.34603500366211,
      "activations/layer22_attention_weight_min": -35.19120407104492,
      "activations/layer23_attention_weight_max": 34.653926849365234,
      "activations/layer23_attention_weight_min": -26.086774826049805,
      "activations/layer2_attention_weight_max": 31.613128662109375,
      "activations/layer2_attention_weight_min": -30.480581283569336,
      "activations/layer3_attention_weight_max": 95.6447525024414,
      "activations/layer3_attention_weight_min": -97.16990661621094,
      "activations/layer4_attention_weight_max": 103.90081024169922,
      "activations/layer4_attention_weight_min": -104.1085433959961,
      "activations/layer5_attention_weight_max": 48.87431335449219,
      "activations/layer5_attention_weight_min": -61.60292053222656,
      "activations/layer6_attention_weight_max": 42.88853073120117,
      "activations/layer6_attention_weight_min": -47.476627349853516,
      "activations/layer7_attention_weight_max": 90.40074157714844,
      "activations/layer7_attention_weight_min": -95.70391082763672,
      "activations/layer8_attention_weight_max": 41.48381805419922,
      "activations/layer8_attention_weight_min": -44.524906158447266,
      "activations/layer9_attention_weight_max": 32.390769958496094,
      "activations/layer9_attention_weight_min": -35.693599700927734,
      "epoch": 19.31,
      "learning_rate": 2.576325757575757e-05,
      "loss": 2.7217,
      "step": 332250
    },
    {
      "activations/layer0_attention_weight_max": 17.25847816467285,
      "activations/layer0_attention_weight_min": -13.035640716552734,
      "activations/layer10_attention_weight_max": 33.36079025268555,
      "activations/layer10_attention_weight_min": -37.93159866333008,
      "activations/layer11_attention_weight_max": 34.45854187011719,
      "activations/layer11_attention_weight_min": -37.35224914550781,
      "activations/layer12_attention_weight_max": 27.638460159301758,
      "activations/layer12_attention_weight_min": -28.592111587524414,
      "activations/layer13_attention_weight_max": 41.055328369140625,
      "activations/layer13_attention_weight_min": -38.34114456176758,
      "activations/layer14_attention_weight_max": 43.87408447265625,
      "activations/layer14_attention_weight_min": -39.24275588989258,
      "activations/layer15_attention_weight_max": 38.59855270385742,
      "activations/layer15_attention_weight_min": -35.78110122680664,
      "activations/layer16_attention_weight_max": 27.24755096435547,
      "activations/layer16_attention_weight_min": -29.688879013061523,
      "activations/layer17_attention_weight_max": 28.638309478759766,
      "activations/layer17_attention_weight_min": -25.990859985351562,
      "activations/layer18_attention_weight_max": 30.626323699951172,
      "activations/layer18_attention_weight_min": -26.19293212890625,
      "activations/layer19_attention_weight_max": 31.741809844970703,
      "activations/layer19_attention_weight_min": -31.33949089050293,
      "activations/layer1_attention_weight_max": 16.078706741333008,
      "activations/layer1_attention_weight_min": -14.358210563659668,
      "activations/layer20_attention_weight_max": 29.69619369506836,
      "activations/layer20_attention_weight_min": -25.127574920654297,
      "activations/layer21_attention_weight_max": 29.381601333618164,
      "activations/layer21_attention_weight_min": -25.33391761779785,
      "activations/layer22_attention_weight_max": 43.5367431640625,
      "activations/layer22_attention_weight_min": -31.734683990478516,
      "activations/layer23_attention_weight_max": 32.293174743652344,
      "activations/layer23_attention_weight_min": -27.279083251953125,
      "activations/layer2_attention_weight_max": 31.758625030517578,
      "activations/layer2_attention_weight_min": -32.465187072753906,
      "activations/layer3_attention_weight_max": 97.50772857666016,
      "activations/layer3_attention_weight_min": -96.37187194824219,
      "activations/layer4_attention_weight_max": 105.2258529663086,
      "activations/layer4_attention_weight_min": -109.6103515625,
      "activations/layer5_attention_weight_max": 49.75925827026367,
      "activations/layer5_attention_weight_min": -63.68099594116211,
      "activations/layer6_attention_weight_max": 45.26698303222656,
      "activations/layer6_attention_weight_min": -48.02445983886719,
      "activations/layer7_attention_weight_max": 95.93278503417969,
      "activations/layer7_attention_weight_min": -101.23564147949219,
      "activations/layer8_attention_weight_max": 41.8260383605957,
      "activations/layer8_attention_weight_min": -45.339027404785156,
      "activations/layer9_attention_weight_max": 32.29983139038086,
      "activations/layer9_attention_weight_min": -35.97242736816406,
      "epoch": 19.31,
      "learning_rate": 2.574431818181818e-05,
      "loss": 2.7222,
      "step": 332300
    },
    {
      "activations/layer0_attention_weight_max": 16.396759033203125,
      "activations/layer0_attention_weight_min": -12.670308113098145,
      "activations/layer10_attention_weight_max": 36.25684356689453,
      "activations/layer10_attention_weight_min": -37.845218658447266,
      "activations/layer11_attention_weight_max": 37.82765579223633,
      "activations/layer11_attention_weight_min": -36.99760818481445,
      "activations/layer12_attention_weight_max": 29.926456451416016,
      "activations/layer12_attention_weight_min": -27.676340103149414,
      "activations/layer13_attention_weight_max": 44.19184494018555,
      "activations/layer13_attention_weight_min": -40.29426574707031,
      "activations/layer14_attention_weight_max": 44.78394317626953,
      "activations/layer14_attention_weight_min": -38.821346282958984,
      "activations/layer15_attention_weight_max": 41.11246109008789,
      "activations/layer15_attention_weight_min": -35.089141845703125,
      "activations/layer16_attention_weight_max": 32.942039489746094,
      "activations/layer16_attention_weight_min": -30.35346221923828,
      "activations/layer17_attention_weight_max": 31.6021728515625,
      "activations/layer17_attention_weight_min": -27.922821044921875,
      "activations/layer18_attention_weight_max": 38.53887939453125,
      "activations/layer18_attention_weight_min": -26.143701553344727,
      "activations/layer19_attention_weight_max": 40.52674865722656,
      "activations/layer19_attention_weight_min": -32.51568603515625,
      "activations/layer1_attention_weight_max": 16.04093360900879,
      "activations/layer1_attention_weight_min": -13.468799591064453,
      "activations/layer20_attention_weight_max": 34.175926208496094,
      "activations/layer20_attention_weight_min": -25.99406623840332,
      "activations/layer21_attention_weight_max": 33.426029205322266,
      "activations/layer21_attention_weight_min": -24.84902000427246,
      "activations/layer22_attention_weight_max": 47.64756774902344,
      "activations/layer22_attention_weight_min": -32.7568244934082,
      "activations/layer23_attention_weight_max": 39.39532470703125,
      "activations/layer23_attention_weight_min": -26.69283676147461,
      "activations/layer2_attention_weight_max": 31.864179611206055,
      "activations/layer2_attention_weight_min": -31.078166961669922,
      "activations/layer3_attention_weight_max": 97.61214447021484,
      "activations/layer3_attention_weight_min": -99.76451110839844,
      "activations/layer4_attention_weight_max": 108.5348129272461,
      "activations/layer4_attention_weight_min": -109.69734191894531,
      "activations/layer5_attention_weight_max": 52.04317855834961,
      "activations/layer5_attention_weight_min": -62.61484909057617,
      "activations/layer6_attention_weight_max": 45.66836929321289,
      "activations/layer6_attention_weight_min": -49.37346649169922,
      "activations/layer7_attention_weight_max": 101.50723266601562,
      "activations/layer7_attention_weight_min": -105.76053619384766,
      "activations/layer8_attention_weight_max": 47.99085998535156,
      "activations/layer8_attention_weight_min": -47.94792175292969,
      "activations/layer9_attention_weight_max": 37.62274169921875,
      "activations/layer9_attention_weight_min": -38.100494384765625,
      "epoch": 19.31,
      "learning_rate": 2.5725378787878784e-05,
      "loss": 2.7099,
      "step": 332350
    },
    {
      "activations/layer0_attention_weight_max": 17.093904495239258,
      "activations/layer0_attention_weight_min": -13.111395835876465,
      "activations/layer10_attention_weight_max": 36.32627868652344,
      "activations/layer10_attention_weight_min": -35.541011810302734,
      "activations/layer11_attention_weight_max": 36.56201934814453,
      "activations/layer11_attention_weight_min": -35.23398208618164,
      "activations/layer12_attention_weight_max": 26.71273422241211,
      "activations/layer12_attention_weight_min": -30.42375946044922,
      "activations/layer13_attention_weight_max": 40.78413009643555,
      "activations/layer13_attention_weight_min": -39.807518005371094,
      "activations/layer14_attention_weight_max": 42.885337829589844,
      "activations/layer14_attention_weight_min": -40.67245101928711,
      "activations/layer15_attention_weight_max": 41.04182434082031,
      "activations/layer15_attention_weight_min": -38.5965461730957,
      "activations/layer16_attention_weight_max": 28.874319076538086,
      "activations/layer16_attention_weight_min": -28.788843154907227,
      "activations/layer17_attention_weight_max": 28.051258087158203,
      "activations/layer17_attention_weight_min": -25.547609329223633,
      "activations/layer18_attention_weight_max": 30.6969051361084,
      "activations/layer18_attention_weight_min": -25.7932186126709,
      "activations/layer19_attention_weight_max": 30.695592880249023,
      "activations/layer19_attention_weight_min": -32.60744094848633,
      "activations/layer1_attention_weight_max": 16.582176208496094,
      "activations/layer1_attention_weight_min": -13.509519577026367,
      "activations/layer20_attention_weight_max": 27.72402000427246,
      "activations/layer20_attention_weight_min": -26.355812072753906,
      "activations/layer21_attention_weight_max": 25.488866806030273,
      "activations/layer21_attention_weight_min": -26.746164321899414,
      "activations/layer22_attention_weight_max": 38.51603698730469,
      "activations/layer22_attention_weight_min": -31.118892669677734,
      "activations/layer23_attention_weight_max": 29.305923461914062,
      "activations/layer23_attention_weight_min": -26.242504119873047,
      "activations/layer2_attention_weight_max": 30.661245346069336,
      "activations/layer2_attention_weight_min": -32.26033401489258,
      "activations/layer3_attention_weight_max": 94.92134857177734,
      "activations/layer3_attention_weight_min": -98.30673217773438,
      "activations/layer4_attention_weight_max": 105.75971984863281,
      "activations/layer4_attention_weight_min": -109.84920501708984,
      "activations/layer5_attention_weight_max": 54.0399284362793,
      "activations/layer5_attention_weight_min": -61.847328186035156,
      "activations/layer6_attention_weight_max": 45.58528137207031,
      "activations/layer6_attention_weight_min": -45.33838653564453,
      "activations/layer7_attention_weight_max": 97.61677551269531,
      "activations/layer7_attention_weight_min": -92.42385864257812,
      "activations/layer8_attention_weight_max": 41.67897415161133,
      "activations/layer8_attention_weight_min": -43.39126968383789,
      "activations/layer9_attention_weight_max": 31.29030418395996,
      "activations/layer9_attention_weight_min": -32.69371032714844,
      "epoch": 19.31,
      "learning_rate": 2.5706439393939393e-05,
      "loss": 2.7113,
      "step": 332400
    },
    {
      "activations/layer0_attention_weight_max": 16.600072860717773,
      "activations/layer0_attention_weight_min": -12.52274227142334,
      "activations/layer10_attention_weight_max": 35.043827056884766,
      "activations/layer10_attention_weight_min": -33.52545166015625,
      "activations/layer11_attention_weight_max": 35.249961853027344,
      "activations/layer11_attention_weight_min": -33.562923431396484,
      "activations/layer12_attention_weight_max": 26.087366104125977,
      "activations/layer12_attention_weight_min": -27.340351104736328,
      "activations/layer13_attention_weight_max": 40.7416877746582,
      "activations/layer13_attention_weight_min": -37.66503143310547,
      "activations/layer14_attention_weight_max": 42.83433532714844,
      "activations/layer14_attention_weight_min": -38.84195327758789,
      "activations/layer15_attention_weight_max": 43.41013717651367,
      "activations/layer15_attention_weight_min": -37.86992645263672,
      "activations/layer16_attention_weight_max": 28.73526954650879,
      "activations/layer16_attention_weight_min": -28.737943649291992,
      "activations/layer17_attention_weight_max": 30.080026626586914,
      "activations/layer17_attention_weight_min": -25.699684143066406,
      "activations/layer18_attention_weight_max": 34.45880889892578,
      "activations/layer18_attention_weight_min": -26.405635833740234,
      "activations/layer19_attention_weight_max": 34.29902267456055,
      "activations/layer19_attention_weight_min": -33.322818756103516,
      "activations/layer1_attention_weight_max": 15.539876937866211,
      "activations/layer1_attention_weight_min": -14.729656219482422,
      "activations/layer20_attention_weight_max": 30.43046760559082,
      "activations/layer20_attention_weight_min": -26.79268455505371,
      "activations/layer21_attention_weight_max": 30.1595458984375,
      "activations/layer21_attention_weight_min": -25.701622009277344,
      "activations/layer22_attention_weight_max": 46.504066467285156,
      "activations/layer22_attention_weight_min": -33.64044952392578,
      "activations/layer23_attention_weight_max": 34.64759063720703,
      "activations/layer23_attention_weight_min": -27.112403869628906,
      "activations/layer2_attention_weight_max": 31.686058044433594,
      "activations/layer2_attention_weight_min": -29.116153717041016,
      "activations/layer3_attention_weight_max": 100.14734649658203,
      "activations/layer3_attention_weight_min": -96.22061157226562,
      "activations/layer4_attention_weight_max": 107.62287902832031,
      "activations/layer4_attention_weight_min": -107.30751037597656,
      "activations/layer5_attention_weight_max": 50.97113037109375,
      "activations/layer5_attention_weight_min": -60.04867172241211,
      "activations/layer6_attention_weight_max": 42.6206169128418,
      "activations/layer6_attention_weight_min": -45.95400619506836,
      "activations/layer7_attention_weight_max": 88.56272888183594,
      "activations/layer7_attention_weight_min": -91.13605499267578,
      "activations/layer8_attention_weight_max": 40.663551330566406,
      "activations/layer8_attention_weight_min": -41.04144287109375,
      "activations/layer9_attention_weight_max": 31.601877212524414,
      "activations/layer9_attention_weight_min": -32.8497314453125,
      "epoch": 19.32,
      "learning_rate": 2.56875e-05,
      "loss": 2.7246,
      "step": 332450
    },
    {
      "activations/layer0_attention_weight_max": 17.50816535949707,
      "activations/layer0_attention_weight_min": -12.832592964172363,
      "activations/layer10_attention_weight_max": 34.28079605102539,
      "activations/layer10_attention_weight_min": -36.827152252197266,
      "activations/layer11_attention_weight_max": 35.073421478271484,
      "activations/layer11_attention_weight_min": -37.99533462524414,
      "activations/layer12_attention_weight_max": 26.770423889160156,
      "activations/layer12_attention_weight_min": -27.484312057495117,
      "activations/layer13_attention_weight_max": 39.06869125366211,
      "activations/layer13_attention_weight_min": -37.5858039855957,
      "activations/layer14_attention_weight_max": 44.07402801513672,
      "activations/layer14_attention_weight_min": -40.48543930053711,
      "activations/layer15_attention_weight_max": 37.960941314697266,
      "activations/layer15_attention_weight_min": -37.0024528503418,
      "activations/layer16_attention_weight_max": 30.742647171020508,
      "activations/layer16_attention_weight_min": -28.872163772583008,
      "activations/layer17_attention_weight_max": 30.14286231994629,
      "activations/layer17_attention_weight_min": -26.8797550201416,
      "activations/layer18_attention_weight_max": 29.657785415649414,
      "activations/layer18_attention_weight_min": -25.27894401550293,
      "activations/layer19_attention_weight_max": 32.74967575073242,
      "activations/layer19_attention_weight_min": -30.749975204467773,
      "activations/layer1_attention_weight_max": 15.020730018615723,
      "activations/layer1_attention_weight_min": -15.082983016967773,
      "activations/layer20_attention_weight_max": 27.819561004638672,
      "activations/layer20_attention_weight_min": -23.990663528442383,
      "activations/layer21_attention_weight_max": 28.319833755493164,
      "activations/layer21_attention_weight_min": -24.016530990600586,
      "activations/layer22_attention_weight_max": 41.506473541259766,
      "activations/layer22_attention_weight_min": -31.3060302734375,
      "activations/layer23_attention_weight_max": 31.46417808532715,
      "activations/layer23_attention_weight_min": -25.87397003173828,
      "activations/layer2_attention_weight_max": 30.3371639251709,
      "activations/layer2_attention_weight_min": -31.829662322998047,
      "activations/layer3_attention_weight_max": 94.81246185302734,
      "activations/layer3_attention_weight_min": -100.55693817138672,
      "activations/layer4_attention_weight_max": 101.65237426757812,
      "activations/layer4_attention_weight_min": -109.64765930175781,
      "activations/layer5_attention_weight_max": 47.557037353515625,
      "activations/layer5_attention_weight_min": -62.38407897949219,
      "activations/layer6_attention_weight_max": 44.11484909057617,
      "activations/layer6_attention_weight_min": -48.895992279052734,
      "activations/layer7_attention_weight_max": 92.21888732910156,
      "activations/layer7_attention_weight_min": -99.99510955810547,
      "activations/layer8_attention_weight_max": 43.18085479736328,
      "activations/layer8_attention_weight_min": -44.81491470336914,
      "activations/layer9_attention_weight_max": 33.167049407958984,
      "activations/layer9_attention_weight_min": -36.250396728515625,
      "epoch": 19.32,
      "learning_rate": 2.5668560606060602e-05,
      "loss": 2.7214,
      "step": 332500
    },
    {
      "activations/layer0_attention_weight_max": 16.357328414916992,
      "activations/layer0_attention_weight_min": -14.087400436401367,
      "activations/layer10_attention_weight_max": 32.850738525390625,
      "activations/layer10_attention_weight_min": -33.903663635253906,
      "activations/layer11_attention_weight_max": 34.641700744628906,
      "activations/layer11_attention_weight_min": -35.01018524169922,
      "activations/layer12_attention_weight_max": 26.858800888061523,
      "activations/layer12_attention_weight_min": -26.890989303588867,
      "activations/layer13_attention_weight_max": 39.18130111694336,
      "activations/layer13_attention_weight_min": -36.81646728515625,
      "activations/layer14_attention_weight_max": 43.97243881225586,
      "activations/layer14_attention_weight_min": -40.287635803222656,
      "activations/layer15_attention_weight_max": 40.42121505737305,
      "activations/layer15_attention_weight_min": -38.9155158996582,
      "activations/layer16_attention_weight_max": 30.281164169311523,
      "activations/layer16_attention_weight_min": -29.379467010498047,
      "activations/layer17_attention_weight_max": 32.4280891418457,
      "activations/layer17_attention_weight_min": -29.379743576049805,
      "activations/layer18_attention_weight_max": 35.437774658203125,
      "activations/layer18_attention_weight_min": -27.81745719909668,
      "activations/layer19_attention_weight_max": 39.36419677734375,
      "activations/layer19_attention_weight_min": -34.936912536621094,
      "activations/layer1_attention_weight_max": 15.614303588867188,
      "activations/layer1_attention_weight_min": -14.581710815429688,
      "activations/layer20_attention_weight_max": 33.121219635009766,
      "activations/layer20_attention_weight_min": -25.672372817993164,
      "activations/layer21_attention_weight_max": 31.866836547851562,
      "activations/layer21_attention_weight_min": -26.373538970947266,
      "activations/layer22_attention_weight_max": 44.904541015625,
      "activations/layer22_attention_weight_min": -33.74540328979492,
      "activations/layer23_attention_weight_max": 34.712215423583984,
      "activations/layer23_attention_weight_min": -27.011276245117188,
      "activations/layer2_attention_weight_max": 31.97667694091797,
      "activations/layer2_attention_weight_min": -32.91557312011719,
      "activations/layer3_attention_weight_max": 101.05950927734375,
      "activations/layer3_attention_weight_min": -101.0602035522461,
      "activations/layer4_attention_weight_max": 106.57042694091797,
      "activations/layer4_attention_weight_min": -107.03742980957031,
      "activations/layer5_attention_weight_max": 51.34953308105469,
      "activations/layer5_attention_weight_min": -61.6552734375,
      "activations/layer6_attention_weight_max": 42.994712829589844,
      "activations/layer6_attention_weight_min": -46.457664489746094,
      "activations/layer7_attention_weight_max": 97.38823699951172,
      "activations/layer7_attention_weight_min": -92.31877899169922,
      "activations/layer8_attention_weight_max": 42.28632354736328,
      "activations/layer8_attention_weight_min": -43.4417610168457,
      "activations/layer9_attention_weight_max": 32.65806198120117,
      "activations/layer9_attention_weight_min": -34.27460861206055,
      "epoch": 19.32,
      "learning_rate": 2.564962121212121e-05,
      "loss": 2.7283,
      "step": 332550
    },
    {
      "activations/layer0_attention_weight_max": 16.440305709838867,
      "activations/layer0_attention_weight_min": -12.548444747924805,
      "activations/layer10_attention_weight_max": 34.79393768310547,
      "activations/layer10_attention_weight_min": -33.87247848510742,
      "activations/layer11_attention_weight_max": 33.9182243347168,
      "activations/layer11_attention_weight_min": -37.02053451538086,
      "activations/layer12_attention_weight_max": 24.319290161132812,
      "activations/layer12_attention_weight_min": -26.355003356933594,
      "activations/layer13_attention_weight_max": 36.370601654052734,
      "activations/layer13_attention_weight_min": -33.96555709838867,
      "activations/layer14_attention_weight_max": 39.02975082397461,
      "activations/layer14_attention_weight_min": -37.01252746582031,
      "activations/layer15_attention_weight_max": 40.23807907104492,
      "activations/layer15_attention_weight_min": -35.764915466308594,
      "activations/layer16_attention_weight_max": 29.669879913330078,
      "activations/layer16_attention_weight_min": -27.917417526245117,
      "activations/layer17_attention_weight_max": 28.063932418823242,
      "activations/layer17_attention_weight_min": -26.812389373779297,
      "activations/layer18_attention_weight_max": 29.603078842163086,
      "activations/layer18_attention_weight_min": -25.68427848815918,
      "activations/layer19_attention_weight_max": 32.445560455322266,
      "activations/layer19_attention_weight_min": -33.52891159057617,
      "activations/layer1_attention_weight_max": 16.497140884399414,
      "activations/layer1_attention_weight_min": -14.151535987854004,
      "activations/layer20_attention_weight_max": 25.836891174316406,
      "activations/layer20_attention_weight_min": -24.206613540649414,
      "activations/layer21_attention_weight_max": 27.43994140625,
      "activations/layer21_attention_weight_min": -25.417068481445312,
      "activations/layer22_attention_weight_max": 36.345359802246094,
      "activations/layer22_attention_weight_min": -30.749048233032227,
      "activations/layer23_attention_weight_max": 28.410537719726562,
      "activations/layer23_attention_weight_min": -25.184776306152344,
      "activations/layer2_attention_weight_max": 35.597557067871094,
      "activations/layer2_attention_weight_min": -33.26130676269531,
      "activations/layer3_attention_weight_max": 105.25558471679688,
      "activations/layer3_attention_weight_min": -103.60083770751953,
      "activations/layer4_attention_weight_max": 106.97358703613281,
      "activations/layer4_attention_weight_min": -102.63854217529297,
      "activations/layer5_attention_weight_max": 47.95793914794922,
      "activations/layer5_attention_weight_min": -61.85123825073242,
      "activations/layer6_attention_weight_max": 43.20201873779297,
      "activations/layer6_attention_weight_min": -46.41571807861328,
      "activations/layer7_attention_weight_max": 96.36363983154297,
      "activations/layer7_attention_weight_min": -94.65254211425781,
      "activations/layer8_attention_weight_max": 43.35808181762695,
      "activations/layer8_attention_weight_min": -43.60570526123047,
      "activations/layer9_attention_weight_max": 30.95209312438965,
      "activations/layer9_attention_weight_min": -35.72178268432617,
      "epoch": 19.33,
      "learning_rate": 2.5630681818181816e-05,
      "loss": 2.7294,
      "step": 332600
    },
    {
      "activations/layer0_attention_weight_max": 15.765976905822754,
      "activations/layer0_attention_weight_min": -12.424485206604004,
      "activations/layer10_attention_weight_max": 34.5075569152832,
      "activations/layer10_attention_weight_min": -36.34856033325195,
      "activations/layer11_attention_weight_max": 35.208255767822266,
      "activations/layer11_attention_weight_min": -33.58696746826172,
      "activations/layer12_attention_weight_max": 27.778366088867188,
      "activations/layer12_attention_weight_min": -26.577768325805664,
      "activations/layer13_attention_weight_max": 41.29133605957031,
      "activations/layer13_attention_weight_min": -34.690101623535156,
      "activations/layer14_attention_weight_max": 43.79283142089844,
      "activations/layer14_attention_weight_min": -36.879085540771484,
      "activations/layer15_attention_weight_max": 40.967105865478516,
      "activations/layer15_attention_weight_min": -35.538333892822266,
      "activations/layer16_attention_weight_max": 37.616512298583984,
      "activations/layer16_attention_weight_min": -28.652713775634766,
      "activations/layer17_attention_weight_max": 33.53278350830078,
      "activations/layer17_attention_weight_min": -25.815326690673828,
      "activations/layer18_attention_weight_max": 38.70771026611328,
      "activations/layer18_attention_weight_min": -26.417638778686523,
      "activations/layer19_attention_weight_max": 39.81755447387695,
      "activations/layer19_attention_weight_min": -32.57270431518555,
      "activations/layer1_attention_weight_max": 16.14419174194336,
      "activations/layer1_attention_weight_min": -14.701573371887207,
      "activations/layer20_attention_weight_max": 33.547821044921875,
      "activations/layer20_attention_weight_min": -27.003509521484375,
      "activations/layer21_attention_weight_max": 32.34258270263672,
      "activations/layer21_attention_weight_min": -24.07171058654785,
      "activations/layer22_attention_weight_max": 48.0278205871582,
      "activations/layer22_attention_weight_min": -32.029197692871094,
      "activations/layer23_attention_weight_max": 36.09273910522461,
      "activations/layer23_attention_weight_min": -26.283742904663086,
      "activations/layer2_attention_weight_max": 33.784759521484375,
      "activations/layer2_attention_weight_min": -32.302913665771484,
      "activations/layer3_attention_weight_max": 101.62032318115234,
      "activations/layer3_attention_weight_min": -100.77291870117188,
      "activations/layer4_attention_weight_max": 109.95068359375,
      "activations/layer4_attention_weight_min": -111.35343170166016,
      "activations/layer5_attention_weight_max": 50.53102111816406,
      "activations/layer5_attention_weight_min": -63.45137023925781,
      "activations/layer6_attention_weight_max": 46.084659576416016,
      "activations/layer6_attention_weight_min": -47.64373016357422,
      "activations/layer7_attention_weight_max": 96.02506256103516,
      "activations/layer7_attention_weight_min": -99.97154235839844,
      "activations/layer8_attention_weight_max": 42.28792190551758,
      "activations/layer8_attention_weight_min": -41.809120178222656,
      "activations/layer9_attention_weight_max": 32.64081954956055,
      "activations/layer9_attention_weight_min": -34.48497009277344,
      "epoch": 19.33,
      "learning_rate": 2.5611742424242424e-05,
      "loss": 2.7204,
      "step": 332650
    },
    {
      "activations/layer0_attention_weight_max": 16.542987823486328,
      "activations/layer0_attention_weight_min": -12.817444801330566,
      "activations/layer10_attention_weight_max": 37.26405715942383,
      "activations/layer10_attention_weight_min": -36.96990966796875,
      "activations/layer11_attention_weight_max": 38.705894470214844,
      "activations/layer11_attention_weight_min": -38.34654235839844,
      "activations/layer12_attention_weight_max": 28.455095291137695,
      "activations/layer12_attention_weight_min": -28.410083770751953,
      "activations/layer13_attention_weight_max": 45.174137115478516,
      "activations/layer13_attention_weight_min": -38.951316833496094,
      "activations/layer14_attention_weight_max": 45.44487380981445,
      "activations/layer14_attention_weight_min": -41.156375885009766,
      "activations/layer15_attention_weight_max": 46.11289596557617,
      "activations/layer15_attention_weight_min": -39.18550109863281,
      "activations/layer16_attention_weight_max": 32.158878326416016,
      "activations/layer16_attention_weight_min": -29.913246154785156,
      "activations/layer17_attention_weight_max": 35.894813537597656,
      "activations/layer17_attention_weight_min": -27.034461975097656,
      "activations/layer18_attention_weight_max": 33.509666442871094,
      "activations/layer18_attention_weight_min": -27.627397537231445,
      "activations/layer19_attention_weight_max": 36.11582565307617,
      "activations/layer19_attention_weight_min": -31.464162826538086,
      "activations/layer1_attention_weight_max": 15.540238380432129,
      "activations/layer1_attention_weight_min": -14.717266082763672,
      "activations/layer20_attention_weight_max": 29.93707847595215,
      "activations/layer20_attention_weight_min": -25.19350814819336,
      "activations/layer21_attention_weight_max": 30.0366268157959,
      "activations/layer21_attention_weight_min": -24.90887451171875,
      "activations/layer22_attention_weight_max": 40.142215728759766,
      "activations/layer22_attention_weight_min": -32.242698669433594,
      "activations/layer23_attention_weight_max": 32.165733337402344,
      "activations/layer23_attention_weight_min": -25.591693878173828,
      "activations/layer2_attention_weight_max": 31.08495330810547,
      "activations/layer2_attention_weight_min": -29.669729232788086,
      "activations/layer3_attention_weight_max": 92.7262191772461,
      "activations/layer3_attention_weight_min": -95.14623260498047,
      "activations/layer4_attention_weight_max": 97.52455139160156,
      "activations/layer4_attention_weight_min": -105.96356201171875,
      "activations/layer5_attention_weight_max": 51.08600616455078,
      "activations/layer5_attention_weight_min": -59.87394332885742,
      "activations/layer6_attention_weight_max": 43.94046401977539,
      "activations/layer6_attention_weight_min": -48.024559020996094,
      "activations/layer7_attention_weight_max": 88.22402954101562,
      "activations/layer7_attention_weight_min": -98.77925872802734,
      "activations/layer8_attention_weight_max": 42.069881439208984,
      "activations/layer8_attention_weight_min": -44.23025894165039,
      "activations/layer9_attention_weight_max": 35.80194854736328,
      "activations/layer9_attention_weight_min": -36.5804557800293,
      "epoch": 19.33,
      "learning_rate": 2.559280303030303e-05,
      "loss": 2.7029,
      "step": 332700
    },
    {
      "activations/layer0_attention_weight_max": 15.836563110351562,
      "activations/layer0_attention_weight_min": -13.183258056640625,
      "activations/layer10_attention_weight_max": 34.30970001220703,
      "activations/layer10_attention_weight_min": -33.63020706176758,
      "activations/layer11_attention_weight_max": 35.01111602783203,
      "activations/layer11_attention_weight_min": -35.65300750732422,
      "activations/layer12_attention_weight_max": 27.646297454833984,
      "activations/layer12_attention_weight_min": -32.075096130371094,
      "activations/layer13_attention_weight_max": 41.346458435058594,
      "activations/layer13_attention_weight_min": -39.90660858154297,
      "activations/layer14_attention_weight_max": 42.37289047241211,
      "activations/layer14_attention_weight_min": -41.00409698486328,
      "activations/layer15_attention_weight_max": 39.5003662109375,
      "activations/layer15_attention_weight_min": -37.25230026245117,
      "activations/layer16_attention_weight_max": 30.495182037353516,
      "activations/layer16_attention_weight_min": -30.267248153686523,
      "activations/layer17_attention_weight_max": 29.449554443359375,
      "activations/layer17_attention_weight_min": -25.92340850830078,
      "activations/layer18_attention_weight_max": 30.62079429626465,
      "activations/layer18_attention_weight_min": -25.243057250976562,
      "activations/layer19_attention_weight_max": 30.168882369995117,
      "activations/layer19_attention_weight_min": -33.79545593261719,
      "activations/layer1_attention_weight_max": 15.912074089050293,
      "activations/layer1_attention_weight_min": -13.463775634765625,
      "activations/layer20_attention_weight_max": 27.70494842529297,
      "activations/layer20_attention_weight_min": -25.404827117919922,
      "activations/layer21_attention_weight_max": 28.128215789794922,
      "activations/layer21_attention_weight_min": -25.927358627319336,
      "activations/layer22_attention_weight_max": 40.621986389160156,
      "activations/layer22_attention_weight_min": -33.262630462646484,
      "activations/layer23_attention_weight_max": 29.845083236694336,
      "activations/layer23_attention_weight_min": -26.138385772705078,
      "activations/layer2_attention_weight_max": 31.402347564697266,
      "activations/layer2_attention_weight_min": -30.220478057861328,
      "activations/layer3_attention_weight_max": 96.8592529296875,
      "activations/layer3_attention_weight_min": -93.3717041015625,
      "activations/layer4_attention_weight_max": 103.19793701171875,
      "activations/layer4_attention_weight_min": -100.41817474365234,
      "activations/layer5_attention_weight_max": 49.63922882080078,
      "activations/layer5_attention_weight_min": -59.41386032104492,
      "activations/layer6_attention_weight_max": 46.09230041503906,
      "activations/layer6_attention_weight_min": -47.28675079345703,
      "activations/layer7_attention_weight_max": 99.56425476074219,
      "activations/layer7_attention_weight_min": -99.87464904785156,
      "activations/layer8_attention_weight_max": 40.96656799316406,
      "activations/layer8_attention_weight_min": -42.79896545410156,
      "activations/layer9_attention_weight_max": 32.84841537475586,
      "activations/layer9_attention_weight_min": -33.75777816772461,
      "epoch": 19.33,
      "learning_rate": 2.5573863636363637e-05,
      "loss": 2.7213,
      "step": 332750
    },
    {
      "activations/layer0_attention_weight_max": 16.32840347290039,
      "activations/layer0_attention_weight_min": -12.289827346801758,
      "activations/layer10_attention_weight_max": 31.840612411499023,
      "activations/layer10_attention_weight_min": -33.15094757080078,
      "activations/layer11_attention_weight_max": 32.26509475708008,
      "activations/layer11_attention_weight_min": -34.18506622314453,
      "activations/layer12_attention_weight_max": 24.660125732421875,
      "activations/layer12_attention_weight_min": -25.82224464416504,
      "activations/layer13_attention_weight_max": 36.159645080566406,
      "activations/layer13_attention_weight_min": -34.9486198425293,
      "activations/layer14_attention_weight_max": 37.76091766357422,
      "activations/layer14_attention_weight_min": -37.769874572753906,
      "activations/layer15_attention_weight_max": 35.04930877685547,
      "activations/layer15_attention_weight_min": -35.88336181640625,
      "activations/layer16_attention_weight_max": 27.177799224853516,
      "activations/layer16_attention_weight_min": -27.992042541503906,
      "activations/layer17_attention_weight_max": 29.718366622924805,
      "activations/layer17_attention_weight_min": -25.91904067993164,
      "activations/layer18_attention_weight_max": 32.95445251464844,
      "activations/layer18_attention_weight_min": -26.183000564575195,
      "activations/layer19_attention_weight_max": 32.0130729675293,
      "activations/layer19_attention_weight_min": -32.24309539794922,
      "activations/layer1_attention_weight_max": 15.861946105957031,
      "activations/layer1_attention_weight_min": -14.485280990600586,
      "activations/layer20_attention_weight_max": 30.929758071899414,
      "activations/layer20_attention_weight_min": -24.663684844970703,
      "activations/layer21_attention_weight_max": 30.83580780029297,
      "activations/layer21_attention_weight_min": -24.76277732849121,
      "activations/layer22_attention_weight_max": 42.39936828613281,
      "activations/layer22_attention_weight_min": -33.681236267089844,
      "activations/layer23_attention_weight_max": 32.23528289794922,
      "activations/layer23_attention_weight_min": -26.86270523071289,
      "activations/layer2_attention_weight_max": 33.037353515625,
      "activations/layer2_attention_weight_min": -32.71941375732422,
      "activations/layer3_attention_weight_max": 99.40697479248047,
      "activations/layer3_attention_weight_min": -105.56912994384766,
      "activations/layer4_attention_weight_max": 104.60652160644531,
      "activations/layer4_attention_weight_min": -110.33735656738281,
      "activations/layer5_attention_weight_max": 49.17829513549805,
      "activations/layer5_attention_weight_min": -61.0476188659668,
      "activations/layer6_attention_weight_max": 45.36159896850586,
      "activations/layer6_attention_weight_min": -49.521888732910156,
      "activations/layer7_attention_weight_max": 92.01380157470703,
      "activations/layer7_attention_weight_min": -93.63870239257812,
      "activations/layer8_attention_weight_max": 39.96352767944336,
      "activations/layer8_attention_weight_min": -42.23070526123047,
      "activations/layer9_attention_weight_max": 29.667072296142578,
      "activations/layer9_attention_weight_min": -33.57109832763672,
      "epoch": 19.34,
      "learning_rate": 2.555492424242424e-05,
      "loss": 2.7231,
      "step": 332800
    },
    {
      "activations/layer0_attention_weight_max": 15.856477737426758,
      "activations/layer0_attention_weight_min": -13.188530921936035,
      "activations/layer10_attention_weight_max": 32.6536750793457,
      "activations/layer10_attention_weight_min": -33.80321502685547,
      "activations/layer11_attention_weight_max": 34.08534622192383,
      "activations/layer11_attention_weight_min": -34.663307189941406,
      "activations/layer12_attention_weight_max": 28.076400756835938,
      "activations/layer12_attention_weight_min": -26.52460479736328,
      "activations/layer13_attention_weight_max": 41.052337646484375,
      "activations/layer13_attention_weight_min": -35.58231735229492,
      "activations/layer14_attention_weight_max": 45.35146713256836,
      "activations/layer14_attention_weight_min": -37.46732711791992,
      "activations/layer15_attention_weight_max": 42.345523834228516,
      "activations/layer15_attention_weight_min": -39.84916687011719,
      "activations/layer16_attention_weight_max": 33.682373046875,
      "activations/layer16_attention_weight_min": -27.805986404418945,
      "activations/layer17_attention_weight_max": 31.76148223876953,
      "activations/layer17_attention_weight_min": -26.07283592224121,
      "activations/layer18_attention_weight_max": 30.806169509887695,
      "activations/layer18_attention_weight_min": -25.345455169677734,
      "activations/layer19_attention_weight_max": 38.32833480834961,
      "activations/layer19_attention_weight_min": -32.950233459472656,
      "activations/layer1_attention_weight_max": 15.592632293701172,
      "activations/layer1_attention_weight_min": -15.37679386138916,
      "activations/layer20_attention_weight_max": 30.888633728027344,
      "activations/layer20_attention_weight_min": -25.50615119934082,
      "activations/layer21_attention_weight_max": 28.147924423217773,
      "activations/layer21_attention_weight_min": -24.83806037902832,
      "activations/layer22_attention_weight_max": 39.95868682861328,
      "activations/layer22_attention_weight_min": -31.20469093322754,
      "activations/layer23_attention_weight_max": 30.147296905517578,
      "activations/layer23_attention_weight_min": -26.473310470581055,
      "activations/layer2_attention_weight_max": 32.548458099365234,
      "activations/layer2_attention_weight_min": -32.70882034301758,
      "activations/layer3_attention_weight_max": 97.88127136230469,
      "activations/layer3_attention_weight_min": -100.0770034790039,
      "activations/layer4_attention_weight_max": 99.10780334472656,
      "activations/layer4_attention_weight_min": -105.87914276123047,
      "activations/layer5_attention_weight_max": 47.75178527832031,
      "activations/layer5_attention_weight_min": -58.3333740234375,
      "activations/layer6_attention_weight_max": 42.44373321533203,
      "activations/layer6_attention_weight_min": -44.94533157348633,
      "activations/layer7_attention_weight_max": 87.71124267578125,
      "activations/layer7_attention_weight_min": -92.53675842285156,
      "activations/layer8_attention_weight_max": 41.67633819580078,
      "activations/layer8_attention_weight_min": -43.86225509643555,
      "activations/layer9_attention_weight_max": 31.66340446472168,
      "activations/layer9_attention_weight_min": -34.21363830566406,
      "epoch": 19.34,
      "learning_rate": 2.5535984848484847e-05,
      "loss": 2.7361,
      "step": 332850
    },
    {
      "activations/layer0_attention_weight_max": 16.201906204223633,
      "activations/layer0_attention_weight_min": -12.741575241088867,
      "activations/layer10_attention_weight_max": 33.33026123046875,
      "activations/layer10_attention_weight_min": -34.647216796875,
      "activations/layer11_attention_weight_max": 32.983638763427734,
      "activations/layer11_attention_weight_min": -36.29694366455078,
      "activations/layer12_attention_weight_max": 25.063928604125977,
      "activations/layer12_attention_weight_min": -27.46640396118164,
      "activations/layer13_attention_weight_max": 35.480953216552734,
      "activations/layer13_attention_weight_min": -35.17937088012695,
      "activations/layer14_attention_weight_max": 39.967613220214844,
      "activations/layer14_attention_weight_min": -39.2120475769043,
      "activations/layer15_attention_weight_max": 41.15121078491211,
      "activations/layer15_attention_weight_min": -36.815128326416016,
      "activations/layer16_attention_weight_max": 29.557321548461914,
      "activations/layer16_attention_weight_min": -28.025697708129883,
      "activations/layer17_attention_weight_max": 27.56034278869629,
      "activations/layer17_attention_weight_min": -27.94172477722168,
      "activations/layer18_attention_weight_max": 34.125282287597656,
      "activations/layer18_attention_weight_min": -28.269174575805664,
      "activations/layer19_attention_weight_max": 35.49880599975586,
      "activations/layer19_attention_weight_min": -31.22836685180664,
      "activations/layer1_attention_weight_max": 15.775928497314453,
      "activations/layer1_attention_weight_min": -15.173826217651367,
      "activations/layer20_attention_weight_max": 26.999486923217773,
      "activations/layer20_attention_weight_min": -25.77895736694336,
      "activations/layer21_attention_weight_max": 27.759916305541992,
      "activations/layer21_attention_weight_min": -26.407060623168945,
      "activations/layer22_attention_weight_max": 40.646366119384766,
      "activations/layer22_attention_weight_min": -34.29255676269531,
      "activations/layer23_attention_weight_max": 34.16295623779297,
      "activations/layer23_attention_weight_min": -28.7382869720459,
      "activations/layer2_attention_weight_max": 33.436424255371094,
      "activations/layer2_attention_weight_min": -32.774169921875,
      "activations/layer3_attention_weight_max": 99.39686584472656,
      "activations/layer3_attention_weight_min": -100.42053985595703,
      "activations/layer4_attention_weight_max": 105.2108383178711,
      "activations/layer4_attention_weight_min": -104.9894790649414,
      "activations/layer5_attention_weight_max": 48.11185836791992,
      "activations/layer5_attention_weight_min": -59.85947799682617,
      "activations/layer6_attention_weight_max": 44.983455657958984,
      "activations/layer6_attention_weight_min": -47.96882247924805,
      "activations/layer7_attention_weight_max": 88.79303741455078,
      "activations/layer7_attention_weight_min": -95.43144989013672,
      "activations/layer8_attention_weight_max": 40.665321350097656,
      "activations/layer8_attention_weight_min": -43.00395584106445,
      "activations/layer9_attention_weight_max": 31.337106704711914,
      "activations/layer9_attention_weight_min": -35.06443405151367,
      "epoch": 19.34,
      "learning_rate": 2.551704545454545e-05,
      "loss": 2.7167,
      "step": 332900
    },
    {
      "activations/layer0_attention_weight_max": 16.518598556518555,
      "activations/layer0_attention_weight_min": -13.582734107971191,
      "activations/layer10_attention_weight_max": 42.33929443359375,
      "activations/layer10_attention_weight_min": -41.825042724609375,
      "activations/layer11_attention_weight_max": 41.9112548828125,
      "activations/layer11_attention_weight_min": -42.89257049560547,
      "activations/layer12_attention_weight_max": 31.27560043334961,
      "activations/layer12_attention_weight_min": -30.541500091552734,
      "activations/layer13_attention_weight_max": 43.39360427856445,
      "activations/layer13_attention_weight_min": -40.78654098510742,
      "activations/layer14_attention_weight_max": 45.691505432128906,
      "activations/layer14_attention_weight_min": -43.09006118774414,
      "activations/layer15_attention_weight_max": 47.78325653076172,
      "activations/layer15_attention_weight_min": -37.94703674316406,
      "activations/layer16_attention_weight_max": 30.916959762573242,
      "activations/layer16_attention_weight_min": -28.21021842956543,
      "activations/layer17_attention_weight_max": 30.43606948852539,
      "activations/layer17_attention_weight_min": -26.61703872680664,
      "activations/layer18_attention_weight_max": 34.807708740234375,
      "activations/layer18_attention_weight_min": -25.639883041381836,
      "activations/layer19_attention_weight_max": 34.206966400146484,
      "activations/layer19_attention_weight_min": -33.85557174682617,
      "activations/layer1_attention_weight_max": 15.047125816345215,
      "activations/layer1_attention_weight_min": -14.843137741088867,
      "activations/layer20_attention_weight_max": 28.885251998901367,
      "activations/layer20_attention_weight_min": -26.035133361816406,
      "activations/layer21_attention_weight_max": 28.008071899414062,
      "activations/layer21_attention_weight_min": -26.6451473236084,
      "activations/layer22_attention_weight_max": 42.951351165771484,
      "activations/layer22_attention_weight_min": -33.09262466430664,
      "activations/layer23_attention_weight_max": 31.14980697631836,
      "activations/layer23_attention_weight_min": -27.19098663330078,
      "activations/layer2_attention_weight_max": 33.33761978149414,
      "activations/layer2_attention_weight_min": -32.790367126464844,
      "activations/layer3_attention_weight_max": 101.25285339355469,
      "activations/layer3_attention_weight_min": -103.14747619628906,
      "activations/layer4_attention_weight_max": 111.03248596191406,
      "activations/layer4_attention_weight_min": -108.42935943603516,
      "activations/layer5_attention_weight_max": 50.106178283691406,
      "activations/layer5_attention_weight_min": -62.364349365234375,
      "activations/layer6_attention_weight_max": 47.15792465209961,
      "activations/layer6_attention_weight_min": -48.07955551147461,
      "activations/layer7_attention_weight_max": 95.55657958984375,
      "activations/layer7_attention_weight_min": -99.62397003173828,
      "activations/layer8_attention_weight_max": 44.79077911376953,
      "activations/layer8_attention_weight_min": -44.996158599853516,
      "activations/layer9_attention_weight_max": 37.45323181152344,
      "activations/layer9_attention_weight_min": -38.14787292480469,
      "epoch": 19.35,
      "learning_rate": 2.549810606060606e-05,
      "loss": 2.7263,
      "step": 332950
    },
    {
      "activations/layer0_attention_weight_max": 16.639873504638672,
      "activations/layer0_attention_weight_min": -12.166909217834473,
      "activations/layer10_attention_weight_max": 33.85157775878906,
      "activations/layer10_attention_weight_min": -33.36503601074219,
      "activations/layer11_attention_weight_max": 33.01542663574219,
      "activations/layer11_attention_weight_min": -32.363521575927734,
      "activations/layer12_attention_weight_max": 24.198116302490234,
      "activations/layer12_attention_weight_min": -27.070114135742188,
      "activations/layer13_attention_weight_max": 34.33103561401367,
      "activations/layer13_attention_weight_min": -36.733543395996094,
      "activations/layer14_attention_weight_max": 38.33602523803711,
      "activations/layer14_attention_weight_min": -37.250972747802734,
      "activations/layer15_attention_weight_max": 35.10502243041992,
      "activations/layer15_attention_weight_min": -36.30087661743164,
      "activations/layer16_attention_weight_max": 26.495555877685547,
      "activations/layer16_attention_weight_min": -27.660085678100586,
      "activations/layer17_attention_weight_max": 27.992992401123047,
      "activations/layer17_attention_weight_min": -24.90920639038086,
      "activations/layer18_attention_weight_max": 30.071367263793945,
      "activations/layer18_attention_weight_min": -25.18179702758789,
      "activations/layer19_attention_weight_max": 34.530582427978516,
      "activations/layer19_attention_weight_min": -31.69329833984375,
      "activations/layer1_attention_weight_max": 15.840728759765625,
      "activations/layer1_attention_weight_min": -15.734623908996582,
      "activations/layer20_attention_weight_max": 30.71441078186035,
      "activations/layer20_attention_weight_min": -24.050493240356445,
      "activations/layer21_attention_weight_max": 27.359344482421875,
      "activations/layer21_attention_weight_min": -25.87520408630371,
      "activations/layer22_attention_weight_max": 38.380210876464844,
      "activations/layer22_attention_weight_min": -33.33633041381836,
      "activations/layer23_attention_weight_max": 29.691137313842773,
      "activations/layer23_attention_weight_min": -26.300350189208984,
      "activations/layer2_attention_weight_max": 34.53907775878906,
      "activations/layer2_attention_weight_min": -31.315120697021484,
      "activations/layer3_attention_weight_max": 98.39910888671875,
      "activations/layer3_attention_weight_min": -97.5943374633789,
      "activations/layer4_attention_weight_max": 101.94896697998047,
      "activations/layer4_attention_weight_min": -103.53292083740234,
      "activations/layer5_attention_weight_max": 49.2999267578125,
      "activations/layer5_attention_weight_min": -60.82664489746094,
      "activations/layer6_attention_weight_max": 41.48956298828125,
      "activations/layer6_attention_weight_min": -45.691200256347656,
      "activations/layer7_attention_weight_max": 87.60393524169922,
      "activations/layer7_attention_weight_min": -94.55704498291016,
      "activations/layer8_attention_weight_max": 38.70643615722656,
      "activations/layer8_attention_weight_min": -41.645118713378906,
      "activations/layer9_attention_weight_max": 30.61500358581543,
      "activations/layer9_attention_weight_min": -34.12358474731445,
      "epoch": 19.35,
      "learning_rate": 2.547916666666666e-05,
      "loss": 2.7156,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4601,
      "eval_samples_per_second": 507.559,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4601,
      "eval_openwebtext_samples_per_second": 507.559,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9588,
      "eval_wikitext_samples_per_second": 232.795,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_lambada_loss": 2.341796875,
      "eval_lambada_ppl": 10.399907116482579,
      "eval_lambada_runtime": 9.5494,
      "eval_lambada_samples_per_second": 509.876,
      "step": 333000
    },
    {
      "activations/layer0_attention_weight_max": 14.95535659790039,
      "activations/layer0_attention_weight_min": -14.096517562866211,
      "activations/layer10_attention_weight_max": 33.707420349121094,
      "activations/layer10_attention_weight_min": -34.854766845703125,
      "activations/layer11_attention_weight_max": 32.502349853515625,
      "activations/layer11_attention_weight_min": -35.06718063354492,
      "activations/layer12_attention_weight_max": 28.99065399169922,
      "activations/layer12_attention_weight_min": -26.744712829589844,
      "activations/layer13_attention_weight_max": 45.10356521606445,
      "activations/layer13_attention_weight_min": -36.45612335205078,
      "activations/layer14_attention_weight_max": 49.34164047241211,
      "activations/layer14_attention_weight_min": -39.653438568115234,
      "activations/layer15_attention_weight_max": 44.331321716308594,
      "activations/layer15_attention_weight_min": -36.14971160888672,
      "activations/layer16_attention_weight_max": 31.60401153564453,
      "activations/layer16_attention_weight_min": -28.49260711669922,
      "activations/layer17_attention_weight_max": 33.11806106567383,
      "activations/layer17_attention_weight_min": -27.165802001953125,
      "activations/layer18_attention_weight_max": 36.08244323730469,
      "activations/layer18_attention_weight_min": -27.93537712097168,
      "activations/layer19_attention_weight_max": 45.77553939819336,
      "activations/layer19_attention_weight_min": -34.58229446411133,
      "activations/layer1_attention_weight_max": 15.710003852844238,
      "activations/layer1_attention_weight_min": -15.44196605682373,
      "activations/layer20_attention_weight_max": 38.45243835449219,
      "activations/layer20_attention_weight_min": -27.3286075592041,
      "activations/layer21_attention_weight_max": 35.0146484375,
      "activations/layer21_attention_weight_min": -27.342056274414062,
      "activations/layer22_attention_weight_max": 47.05129623413086,
      "activations/layer22_attention_weight_min": -36.693904876708984,
      "activations/layer23_attention_weight_max": 36.33279037475586,
      "activations/layer23_attention_weight_min": -29.322917938232422,
      "activations/layer2_attention_weight_max": 32.52056121826172,
      "activations/layer2_attention_weight_min": -31.59966278076172,
      "activations/layer3_attention_weight_max": 104.55989074707031,
      "activations/layer3_attention_weight_min": -101.1307144165039,
      "activations/layer4_attention_weight_max": 108.01839447021484,
      "activations/layer4_attention_weight_min": -108.5704116821289,
      "activations/layer5_attention_weight_max": 49.04594421386719,
      "activations/layer5_attention_weight_min": -61.58362579345703,
      "activations/layer6_attention_weight_max": 43.2054557800293,
      "activations/layer6_attention_weight_min": -46.247230529785156,
      "activations/layer7_attention_weight_max": 94.07727813720703,
      "activations/layer7_attention_weight_min": -96.22367858886719,
      "activations/layer8_attention_weight_max": 39.5558967590332,
      "activations/layer8_attention_weight_min": -45.46903610229492,
      "activations/layer9_attention_weight_max": 31.477380752563477,
      "activations/layer9_attention_weight_min": -35.09321212768555,
      "epoch": 19.35,
      "learning_rate": 2.546022727272727e-05,
      "loss": 2.7243,
      "step": 333050
    },
    {
      "activations/layer0_attention_weight_max": 17.176259994506836,
      "activations/layer0_attention_weight_min": -13.391317367553711,
      "activations/layer10_attention_weight_max": 29.7803955078125,
      "activations/layer10_attention_weight_min": -32.93212890625,
      "activations/layer11_attention_weight_max": 30.48346710205078,
      "activations/layer11_attention_weight_min": -30.437454223632812,
      "activations/layer12_attention_weight_max": 28.916231155395508,
      "activations/layer12_attention_weight_min": -26.357343673706055,
      "activations/layer13_attention_weight_max": 43.0328483581543,
      "activations/layer13_attention_weight_min": -36.73759460449219,
      "activations/layer14_attention_weight_max": 46.183448791503906,
      "activations/layer14_attention_weight_min": -39.1059455871582,
      "activations/layer15_attention_weight_max": 43.17481231689453,
      "activations/layer15_attention_weight_min": -35.0950813293457,
      "activations/layer16_attention_weight_max": 33.037269592285156,
      "activations/layer16_attention_weight_min": -28.69292449951172,
      "activations/layer17_attention_weight_max": 32.16454315185547,
      "activations/layer17_attention_weight_min": -25.90423011779785,
      "activations/layer18_attention_weight_max": 32.292083740234375,
      "activations/layer18_attention_weight_min": -27.091739654541016,
      "activations/layer19_attention_weight_max": 37.434844970703125,
      "activations/layer19_attention_weight_min": -31.541736602783203,
      "activations/layer1_attention_weight_max": 15.202434539794922,
      "activations/layer1_attention_weight_min": -15.022135734558105,
      "activations/layer20_attention_weight_max": 32.55521774291992,
      "activations/layer20_attention_weight_min": -25.652610778808594,
      "activations/layer21_attention_weight_max": 33.31331253051758,
      "activations/layer21_attention_weight_min": -24.251943588256836,
      "activations/layer22_attention_weight_max": 43.21541976928711,
      "activations/layer22_attention_weight_min": -32.00416946411133,
      "activations/layer23_attention_weight_max": 33.79330825805664,
      "activations/layer23_attention_weight_min": -26.17061996459961,
      "activations/layer2_attention_weight_max": 30.342132568359375,
      "activations/layer2_attention_weight_min": -29.823387145996094,
      "activations/layer3_attention_weight_max": 93.85863494873047,
      "activations/layer3_attention_weight_min": -96.40496826171875,
      "activations/layer4_attention_weight_max": 99.70044708251953,
      "activations/layer4_attention_weight_min": -107.88834381103516,
      "activations/layer5_attention_weight_max": 47.84765625,
      "activations/layer5_attention_weight_min": -60.98431396484375,
      "activations/layer6_attention_weight_max": 41.128662109375,
      "activations/layer6_attention_weight_min": -45.22677230834961,
      "activations/layer7_attention_weight_max": 90.93107604980469,
      "activations/layer7_attention_weight_min": -90.94805908203125,
      "activations/layer8_attention_weight_max": 40.13017272949219,
      "activations/layer8_attention_weight_min": -41.05149841308594,
      "activations/layer9_attention_weight_max": 30.948989868164062,
      "activations/layer9_attention_weight_min": -34.75878143310547,
      "epoch": 19.36,
      "learning_rate": 2.5441287878787875e-05,
      "loss": 2.7388,
      "step": 333100
    },
    {
      "activations/layer0_attention_weight_max": 16.083654403686523,
      "activations/layer0_attention_weight_min": -13.131027221679688,
      "activations/layer10_attention_weight_max": 35.112159729003906,
      "activations/layer10_attention_weight_min": -37.268516540527344,
      "activations/layer11_attention_weight_max": 33.23585510253906,
      "activations/layer11_attention_weight_min": -34.64033508300781,
      "activations/layer12_attention_weight_max": 27.095584869384766,
      "activations/layer12_attention_weight_min": -27.54519271850586,
      "activations/layer13_attention_weight_max": 39.9649543762207,
      "activations/layer13_attention_weight_min": -38.66263961791992,
      "activations/layer14_attention_weight_max": 41.12857437133789,
      "activations/layer14_attention_weight_min": -40.29859924316406,
      "activations/layer15_attention_weight_max": 44.435394287109375,
      "activations/layer15_attention_weight_min": -36.56755828857422,
      "activations/layer16_attention_weight_max": 28.342693328857422,
      "activations/layer16_attention_weight_min": -27.57023048400879,
      "activations/layer17_attention_weight_max": 26.67824935913086,
      "activations/layer17_attention_weight_min": -25.99920082092285,
      "activations/layer18_attention_weight_max": 30.915573120117188,
      "activations/layer18_attention_weight_min": -27.42534065246582,
      "activations/layer19_attention_weight_max": 32.42107009887695,
      "activations/layer19_attention_weight_min": -31.797714233398438,
      "activations/layer1_attention_weight_max": 16.464996337890625,
      "activations/layer1_attention_weight_min": -15.213971138000488,
      "activations/layer20_attention_weight_max": 29.288022994995117,
      "activations/layer20_attention_weight_min": -25.73479652404785,
      "activations/layer21_attention_weight_max": 28.563962936401367,
      "activations/layer21_attention_weight_min": -25.488554000854492,
      "activations/layer22_attention_weight_max": 43.117977142333984,
      "activations/layer22_attention_weight_min": -36.6662483215332,
      "activations/layer23_attention_weight_max": 35.71117401123047,
      "activations/layer23_attention_weight_min": -28.257186889648438,
      "activations/layer2_attention_weight_max": 31.648880004882812,
      "activations/layer2_attention_weight_min": -30.97995376586914,
      "activations/layer3_attention_weight_max": 97.08784484863281,
      "activations/layer3_attention_weight_min": -98.90927124023438,
      "activations/layer4_attention_weight_max": 105.7815170288086,
      "activations/layer4_attention_weight_min": -106.48359680175781,
      "activations/layer5_attention_weight_max": 47.98441696166992,
      "activations/layer5_attention_weight_min": -64.99144744873047,
      "activations/layer6_attention_weight_max": 43.83662796020508,
      "activations/layer6_attention_weight_min": -48.368141174316406,
      "activations/layer7_attention_weight_max": 91.4494857788086,
      "activations/layer7_attention_weight_min": -100.83338928222656,
      "activations/layer8_attention_weight_max": 41.22046661376953,
      "activations/layer8_attention_weight_min": -44.6408576965332,
      "activations/layer9_attention_weight_max": 31.823575973510742,
      "activations/layer9_attention_weight_min": -36.02016830444336,
      "epoch": 19.36,
      "learning_rate": 2.5422348484848483e-05,
      "loss": 2.708,
      "step": 333150
    },
    {
      "activations/layer0_attention_weight_max": 16.843997955322266,
      "activations/layer0_attention_weight_min": -12.5640287399292,
      "activations/layer10_attention_weight_max": 37.87239456176758,
      "activations/layer10_attention_weight_min": -36.63379669189453,
      "activations/layer11_attention_weight_max": 35.54203796386719,
      "activations/layer11_attention_weight_min": -38.826499938964844,
      "activations/layer12_attention_weight_max": 29.9624080657959,
      "activations/layer12_attention_weight_min": -29.708904266357422,
      "activations/layer13_attention_weight_max": 45.400691986083984,
      "activations/layer13_attention_weight_min": -38.73310852050781,
      "activations/layer14_attention_weight_max": 49.615352630615234,
      "activations/layer14_attention_weight_min": -40.91109848022461,
      "activations/layer15_attention_weight_max": 46.06412887573242,
      "activations/layer15_attention_weight_min": -39.88646697998047,
      "activations/layer16_attention_weight_max": 35.59770965576172,
      "activations/layer16_attention_weight_min": -27.83749008178711,
      "activations/layer17_attention_weight_max": 37.171470642089844,
      "activations/layer17_attention_weight_min": -26.54656410217285,
      "activations/layer18_attention_weight_max": 34.3139762878418,
      "activations/layer18_attention_weight_min": -26.669551849365234,
      "activations/layer19_attention_weight_max": 41.73912048339844,
      "activations/layer19_attention_weight_min": -35.24007797241211,
      "activations/layer1_attention_weight_max": 16.33318519592285,
      "activations/layer1_attention_weight_min": -14.63061237335205,
      "activations/layer20_attention_weight_max": 36.80133056640625,
      "activations/layer20_attention_weight_min": -26.593177795410156,
      "activations/layer21_attention_weight_max": 34.980777740478516,
      "activations/layer21_attention_weight_min": -26.222427368164062,
      "activations/layer22_attention_weight_max": 46.12925338745117,
      "activations/layer22_attention_weight_min": -32.100223541259766,
      "activations/layer23_attention_weight_max": 37.81233215332031,
      "activations/layer23_attention_weight_min": -26.84532928466797,
      "activations/layer2_attention_weight_max": 31.323402404785156,
      "activations/layer2_attention_weight_min": -30.277433395385742,
      "activations/layer3_attention_weight_max": 97.75274658203125,
      "activations/layer3_attention_weight_min": -100.95318603515625,
      "activations/layer4_attention_weight_max": 105.50439453125,
      "activations/layer4_attention_weight_min": -106.33740234375,
      "activations/layer5_attention_weight_max": 48.921043395996094,
      "activations/layer5_attention_weight_min": -61.42644500732422,
      "activations/layer6_attention_weight_max": 42.59209060668945,
      "activations/layer6_attention_weight_min": -46.98555374145508,
      "activations/layer7_attention_weight_max": 91.79183197021484,
      "activations/layer7_attention_weight_min": -97.7352066040039,
      "activations/layer8_attention_weight_max": 43.15846633911133,
      "activations/layer8_attention_weight_min": -44.567626953125,
      "activations/layer9_attention_weight_max": 34.81844711303711,
      "activations/layer9_attention_weight_min": -35.00492858886719,
      "epoch": 19.36,
      "learning_rate": 2.5403409090909088e-05,
      "loss": 2.7147,
      "step": 333200
    },
    {
      "activations/layer0_attention_weight_max": 16.242900848388672,
      "activations/layer0_attention_weight_min": -12.773184776306152,
      "activations/layer10_attention_weight_max": 30.982723236083984,
      "activations/layer10_attention_weight_min": -32.72608184814453,
      "activations/layer11_attention_weight_max": 33.53436279296875,
      "activations/layer11_attention_weight_min": -34.04010009765625,
      "activations/layer12_attention_weight_max": 24.558881759643555,
      "activations/layer12_attention_weight_min": -26.79491424560547,
      "activations/layer13_attention_weight_max": 36.7513427734375,
      "activations/layer13_attention_weight_min": -36.63221740722656,
      "activations/layer14_attention_weight_max": 38.763267517089844,
      "activations/layer14_attention_weight_min": -37.489498138427734,
      "activations/layer15_attention_weight_max": 35.63849639892578,
      "activations/layer15_attention_weight_min": -33.43815994262695,
      "activations/layer16_attention_weight_max": 26.429603576660156,
      "activations/layer16_attention_weight_min": -27.53463363647461,
      "activations/layer17_attention_weight_max": 26.864736557006836,
      "activations/layer17_attention_weight_min": -25.7401123046875,
      "activations/layer18_attention_weight_max": 30.897705078125,
      "activations/layer18_attention_weight_min": -25.607650756835938,
      "activations/layer19_attention_weight_max": 30.087682723999023,
      "activations/layer19_attention_weight_min": -31.778371810913086,
      "activations/layer1_attention_weight_max": 15.494156837463379,
      "activations/layer1_attention_weight_min": -15.182238578796387,
      "activations/layer20_attention_weight_max": 27.726879119873047,
      "activations/layer20_attention_weight_min": -24.827978134155273,
      "activations/layer21_attention_weight_max": 29.126052856445312,
      "activations/layer21_attention_weight_min": -26.620622634887695,
      "activations/layer22_attention_weight_max": 38.783660888671875,
      "activations/layer22_attention_weight_min": -31.21903419494629,
      "activations/layer23_attention_weight_max": 31.253433227539062,
      "activations/layer23_attention_weight_min": -25.538969039916992,
      "activations/layer2_attention_weight_max": 32.34869384765625,
      "activations/layer2_attention_weight_min": -30.320127487182617,
      "activations/layer3_attention_weight_max": 95.502197265625,
      "activations/layer3_attention_weight_min": -97.41513061523438,
      "activations/layer4_attention_weight_max": 99.03167724609375,
      "activations/layer4_attention_weight_min": -104.51004791259766,
      "activations/layer5_attention_weight_max": 48.78245544433594,
      "activations/layer5_attention_weight_min": -58.18803405761719,
      "activations/layer6_attention_weight_max": 43.20255661010742,
      "activations/layer6_attention_weight_min": -44.94550704956055,
      "activations/layer7_attention_weight_max": 97.13914489746094,
      "activations/layer7_attention_weight_min": -94.46149444580078,
      "activations/layer8_attention_weight_max": 39.020267486572266,
      "activations/layer8_attention_weight_min": -43.53181076049805,
      "activations/layer9_attention_weight_max": 31.923261642456055,
      "activations/layer9_attention_weight_min": -32.73505401611328,
      "epoch": 19.36,
      "learning_rate": 2.5384469696969696e-05,
      "loss": 2.7082,
      "step": 333250
    },
    {
      "activations/layer0_attention_weight_max": 16.561443328857422,
      "activations/layer0_attention_weight_min": -13.432279586791992,
      "activations/layer10_attention_weight_max": 40.19074249267578,
      "activations/layer10_attention_weight_min": -39.0307502746582,
      "activations/layer11_attention_weight_max": 37.28664016723633,
      "activations/layer11_attention_weight_min": -36.557979583740234,
      "activations/layer12_attention_weight_max": 27.62445831298828,
      "activations/layer12_attention_weight_min": -27.229820251464844,
      "activations/layer13_attention_weight_max": 39.407867431640625,
      "activations/layer13_attention_weight_min": -37.66744613647461,
      "activations/layer14_attention_weight_max": 43.34756851196289,
      "activations/layer14_attention_weight_min": -39.411705017089844,
      "activations/layer15_attention_weight_max": 40.50062942504883,
      "activations/layer15_attention_weight_min": -37.458744049072266,
      "activations/layer16_attention_weight_max": 32.010841369628906,
      "activations/layer16_attention_weight_min": -30.971324920654297,
      "activations/layer17_attention_weight_max": 31.060075759887695,
      "activations/layer17_attention_weight_min": -27.08451271057129,
      "activations/layer18_attention_weight_max": 33.2674446105957,
      "activations/layer18_attention_weight_min": -27.723783493041992,
      "activations/layer19_attention_weight_max": 37.52366638183594,
      "activations/layer19_attention_weight_min": -33.74396896362305,
      "activations/layer1_attention_weight_max": 16.55512237548828,
      "activations/layer1_attention_weight_min": -15.395557403564453,
      "activations/layer20_attention_weight_max": 28.957796096801758,
      "activations/layer20_attention_weight_min": -28.068828582763672,
      "activations/layer21_attention_weight_max": 28.68416976928711,
      "activations/layer21_attention_weight_min": -24.78984832763672,
      "activations/layer22_attention_weight_max": 41.73550033569336,
      "activations/layer22_attention_weight_min": -34.57239532470703,
      "activations/layer23_attention_weight_max": 31.109683990478516,
      "activations/layer23_attention_weight_min": -27.124324798583984,
      "activations/layer2_attention_weight_max": 34.76462173461914,
      "activations/layer2_attention_weight_min": -31.903106689453125,
      "activations/layer3_attention_weight_max": 98.86962127685547,
      "activations/layer3_attention_weight_min": -98.06500244140625,
      "activations/layer4_attention_weight_max": 107.15425872802734,
      "activations/layer4_attention_weight_min": -106.06511688232422,
      "activations/layer5_attention_weight_max": 50.923431396484375,
      "activations/layer5_attention_weight_min": -61.089107513427734,
      "activations/layer6_attention_weight_max": 47.441287994384766,
      "activations/layer6_attention_weight_min": -45.8385124206543,
      "activations/layer7_attention_weight_max": 98.66703033447266,
      "activations/layer7_attention_weight_min": -101.16094970703125,
      "activations/layer8_attention_weight_max": 46.27006530761719,
      "activations/layer8_attention_weight_min": -47.273284912109375,
      "activations/layer9_attention_weight_max": 36.19281005859375,
      "activations/layer9_attention_weight_min": -35.80231475830078,
      "epoch": 19.37,
      "learning_rate": 2.5365530303030304e-05,
      "loss": 2.7222,
      "step": 333300
    },
    {
      "activations/layer0_attention_weight_max": 16.0419979095459,
      "activations/layer0_attention_weight_min": -12.892326354980469,
      "activations/layer10_attention_weight_max": 36.210906982421875,
      "activations/layer10_attention_weight_min": -38.374908447265625,
      "activations/layer11_attention_weight_max": 36.50346755981445,
      "activations/layer11_attention_weight_min": -38.637474060058594,
      "activations/layer12_attention_weight_max": 28.48891830444336,
      "activations/layer12_attention_weight_min": -28.365501403808594,
      "activations/layer13_attention_weight_max": 41.266876220703125,
      "activations/layer13_attention_weight_min": -38.73052215576172,
      "activations/layer14_attention_weight_max": 47.29254150390625,
      "activations/layer14_attention_weight_min": -41.70708084106445,
      "activations/layer15_attention_weight_max": 38.35343551635742,
      "activations/layer15_attention_weight_min": -37.024749755859375,
      "activations/layer16_attention_weight_max": 29.019824981689453,
      "activations/layer16_attention_weight_min": -29.700864791870117,
      "activations/layer17_attention_weight_max": 29.34462547302246,
      "activations/layer17_attention_weight_min": -28.243301391601562,
      "activations/layer18_attention_weight_max": 32.18122863769531,
      "activations/layer18_attention_weight_min": -26.153947830200195,
      "activations/layer19_attention_weight_max": 34.09288024902344,
      "activations/layer19_attention_weight_min": -34.23577880859375,
      "activations/layer1_attention_weight_max": 15.39064884185791,
      "activations/layer1_attention_weight_min": -14.87098503112793,
      "activations/layer20_attention_weight_max": 31.627084732055664,
      "activations/layer20_attention_weight_min": -26.130064010620117,
      "activations/layer21_attention_weight_max": 29.521465301513672,
      "activations/layer21_attention_weight_min": -26.335771560668945,
      "activations/layer22_attention_weight_max": 44.8389892578125,
      "activations/layer22_attention_weight_min": -32.408206939697266,
      "activations/layer23_attention_weight_max": 33.55731964111328,
      "activations/layer23_attention_weight_min": -25.477758407592773,
      "activations/layer2_attention_weight_max": 31.73705291748047,
      "activations/layer2_attention_weight_min": -31.819486618041992,
      "activations/layer3_attention_weight_max": 100.72804260253906,
      "activations/layer3_attention_weight_min": -99.535400390625,
      "activations/layer4_attention_weight_max": 107.69210052490234,
      "activations/layer4_attention_weight_min": -109.58503723144531,
      "activations/layer5_attention_weight_max": 53.29913330078125,
      "activations/layer5_attention_weight_min": -63.795501708984375,
      "activations/layer6_attention_weight_max": 45.61870574951172,
      "activations/layer6_attention_weight_min": -47.136844635009766,
      "activations/layer7_attention_weight_max": 97.32500457763672,
      "activations/layer7_attention_weight_min": -100.57728576660156,
      "activations/layer8_attention_weight_max": 45.57758712768555,
      "activations/layer8_attention_weight_min": -45.455970764160156,
      "activations/layer9_attention_weight_max": 35.594234466552734,
      "activations/layer9_attention_weight_min": -37.001731872558594,
      "epoch": 19.37,
      "learning_rate": 2.5346590909090906e-05,
      "loss": 2.7331,
      "step": 333350
    },
    {
      "activations/layer0_attention_weight_max": 16.677085876464844,
      "activations/layer0_attention_weight_min": -13.653350830078125,
      "activations/layer10_attention_weight_max": 33.66606903076172,
      "activations/layer10_attention_weight_min": -34.15562057495117,
      "activations/layer11_attention_weight_max": 35.05070495605469,
      "activations/layer11_attention_weight_min": -34.139766693115234,
      "activations/layer12_attention_weight_max": 26.649429321289062,
      "activations/layer12_attention_weight_min": -25.774314880371094,
      "activations/layer13_attention_weight_max": 38.85951614379883,
      "activations/layer13_attention_weight_min": -35.924530029296875,
      "activations/layer14_attention_weight_max": 42.69511032104492,
      "activations/layer14_attention_weight_min": -38.409820556640625,
      "activations/layer15_attention_weight_max": 38.13050079345703,
      "activations/layer15_attention_weight_min": -35.81734848022461,
      "activations/layer16_attention_weight_max": 30.249507904052734,
      "activations/layer16_attention_weight_min": -28.256669998168945,
      "activations/layer17_attention_weight_max": 29.52488136291504,
      "activations/layer17_attention_weight_min": -27.301761627197266,
      "activations/layer18_attention_weight_max": 31.79580307006836,
      "activations/layer18_attention_weight_min": -26.029939651489258,
      "activations/layer19_attention_weight_max": 34.850608825683594,
      "activations/layer19_attention_weight_min": -33.438114166259766,
      "activations/layer1_attention_weight_max": 15.810532569885254,
      "activations/layer1_attention_weight_min": -14.681130409240723,
      "activations/layer20_attention_weight_max": 32.69697952270508,
      "activations/layer20_attention_weight_min": -26.375564575195312,
      "activations/layer21_attention_weight_max": 34.863033294677734,
      "activations/layer21_attention_weight_min": -25.28171730041504,
      "activations/layer22_attention_weight_max": 44.4675407409668,
      "activations/layer22_attention_weight_min": -32.95077133178711,
      "activations/layer23_attention_weight_max": 34.258888244628906,
      "activations/layer23_attention_weight_min": -26.630279541015625,
      "activations/layer2_attention_weight_max": 32.76172637939453,
      "activations/layer2_attention_weight_min": -31.6081600189209,
      "activations/layer3_attention_weight_max": 94.21994018554688,
      "activations/layer3_attention_weight_min": -96.12397766113281,
      "activations/layer4_attention_weight_max": 105.82606506347656,
      "activations/layer4_attention_weight_min": -109.13338470458984,
      "activations/layer5_attention_weight_max": 49.38604736328125,
      "activations/layer5_attention_weight_min": -60.892913818359375,
      "activations/layer6_attention_weight_max": 42.8231315612793,
      "activations/layer6_attention_weight_min": -46.09663391113281,
      "activations/layer7_attention_weight_max": 91.9571762084961,
      "activations/layer7_attention_weight_min": -94.74794006347656,
      "activations/layer8_attention_weight_max": 41.519813537597656,
      "activations/layer8_attention_weight_min": -43.304752349853516,
      "activations/layer9_attention_weight_max": 33.00516128540039,
      "activations/layer9_attention_weight_min": -33.3841667175293,
      "epoch": 19.37,
      "learning_rate": 2.5328030303030303e-05,
      "loss": 2.7191,
      "step": 333400
    },
    {
      "activations/layer0_attention_weight_max": 16.716737747192383,
      "activations/layer0_attention_weight_min": -13.38824462890625,
      "activations/layer10_attention_weight_max": 33.041717529296875,
      "activations/layer10_attention_weight_min": -35.2789421081543,
      "activations/layer11_attention_weight_max": 33.13220977783203,
      "activations/layer11_attention_weight_min": -34.66453552246094,
      "activations/layer12_attention_weight_max": 25.67020606994629,
      "activations/layer12_attention_weight_min": -26.549150466918945,
      "activations/layer13_attention_weight_max": 36.79905700683594,
      "activations/layer13_attention_weight_min": -38.42019271850586,
      "activations/layer14_attention_weight_max": 41.516944885253906,
      "activations/layer14_attention_weight_min": -40.50598907470703,
      "activations/layer15_attention_weight_max": 36.991207122802734,
      "activations/layer15_attention_weight_min": -34.022491455078125,
      "activations/layer16_attention_weight_max": 29.317529678344727,
      "activations/layer16_attention_weight_min": -26.95657730102539,
      "activations/layer17_attention_weight_max": 29.675540924072266,
      "activations/layer17_attention_weight_min": -24.797534942626953,
      "activations/layer18_attention_weight_max": 32.09843063354492,
      "activations/layer18_attention_weight_min": -26.36363983154297,
      "activations/layer19_attention_weight_max": 32.20729064941406,
      "activations/layer19_attention_weight_min": -32.039222717285156,
      "activations/layer1_attention_weight_max": 16.20435905456543,
      "activations/layer1_attention_weight_min": -13.972426414489746,
      "activations/layer20_attention_weight_max": 29.41742515563965,
      "activations/layer20_attention_weight_min": -24.812238693237305,
      "activations/layer21_attention_weight_max": 28.727428436279297,
      "activations/layer21_attention_weight_min": -23.98552131652832,
      "activations/layer22_attention_weight_max": 38.89706802368164,
      "activations/layer22_attention_weight_min": -30.156585693359375,
      "activations/layer23_attention_weight_max": 29.41745376586914,
      "activations/layer23_attention_weight_min": -24.342121124267578,
      "activations/layer2_attention_weight_max": 31.926734924316406,
      "activations/layer2_attention_weight_min": -31.529987335205078,
      "activations/layer3_attention_weight_max": 99.61425018310547,
      "activations/layer3_attention_weight_min": -98.43668365478516,
      "activations/layer4_attention_weight_max": 105.60076141357422,
      "activations/layer4_attention_weight_min": -108.55338287353516,
      "activations/layer5_attention_weight_max": 48.65602111816406,
      "activations/layer5_attention_weight_min": -65.11844635009766,
      "activations/layer6_attention_weight_max": 45.67024612426758,
      "activations/layer6_attention_weight_min": -47.497276306152344,
      "activations/layer7_attention_weight_max": 96.22709655761719,
      "activations/layer7_attention_weight_min": -100.67095947265625,
      "activations/layer8_attention_weight_max": 40.62525177001953,
      "activations/layer8_attention_weight_min": -44.666561126708984,
      "activations/layer9_attention_weight_max": 34.5869255065918,
      "activations/layer9_attention_weight_min": -35.168392181396484,
      "epoch": 19.38,
      "learning_rate": 2.5309090909090908e-05,
      "loss": 2.7215,
      "step": 333450
    },
    {
      "activations/layer0_attention_weight_max": 16.62095832824707,
      "activations/layer0_attention_weight_min": -13.213468551635742,
      "activations/layer10_attention_weight_max": 46.25376510620117,
      "activations/layer10_attention_weight_min": -40.46736526489258,
      "activations/layer11_attention_weight_max": 44.777320861816406,
      "activations/layer11_attention_weight_min": -38.708431243896484,
      "activations/layer12_attention_weight_max": 30.043073654174805,
      "activations/layer12_attention_weight_min": -30.567365646362305,
      "activations/layer13_attention_weight_max": 44.35363006591797,
      "activations/layer13_attention_weight_min": -37.84754943847656,
      "activations/layer14_attention_weight_max": 46.805908203125,
      "activations/layer14_attention_weight_min": -41.81807327270508,
      "activations/layer15_attention_weight_max": 50.415855407714844,
      "activations/layer15_attention_weight_min": -43.753089904785156,
      "activations/layer16_attention_weight_max": 33.9122428894043,
      "activations/layer16_attention_weight_min": -28.35941505432129,
      "activations/layer17_attention_weight_max": 34.639991760253906,
      "activations/layer17_attention_weight_min": -29.090450286865234,
      "activations/layer18_attention_weight_max": 32.494651794433594,
      "activations/layer18_attention_weight_min": -27.597017288208008,
      "activations/layer19_attention_weight_max": 35.49176788330078,
      "activations/layer19_attention_weight_min": -34.24750900268555,
      "activations/layer1_attention_weight_max": 15.77082633972168,
      "activations/layer1_attention_weight_min": -15.992086410522461,
      "activations/layer20_attention_weight_max": 32.38762664794922,
      "activations/layer20_attention_weight_min": -26.140235900878906,
      "activations/layer21_attention_weight_max": 30.597984313964844,
      "activations/layer21_attention_weight_min": -24.51609992980957,
      "activations/layer22_attention_weight_max": 41.91735076904297,
      "activations/layer22_attention_weight_min": -32.79755783081055,
      "activations/layer23_attention_weight_max": 32.34174728393555,
      "activations/layer23_attention_weight_min": -28.430755615234375,
      "activations/layer2_attention_weight_max": 32.30694580078125,
      "activations/layer2_attention_weight_min": -32.194129943847656,
      "activations/layer3_attention_weight_max": 98.26680755615234,
      "activations/layer3_attention_weight_min": -97.87262725830078,
      "activations/layer4_attention_weight_max": 104.9665756225586,
      "activations/layer4_attention_weight_min": -110.48774719238281,
      "activations/layer5_attention_weight_max": 48.73735809326172,
      "activations/layer5_attention_weight_min": -61.64457702636719,
      "activations/layer6_attention_weight_max": 44.504940032958984,
      "activations/layer6_attention_weight_min": -48.177040100097656,
      "activations/layer7_attention_weight_max": 101.03401184082031,
      "activations/layer7_attention_weight_min": -99.57340240478516,
      "activations/layer8_attention_weight_max": 50.393714904785156,
      "activations/layer8_attention_weight_min": -48.36536407470703,
      "activations/layer9_attention_weight_max": 38.19118881225586,
      "activations/layer9_attention_weight_min": -39.00365447998047,
      "epoch": 19.38,
      "learning_rate": 2.5290151515151513e-05,
      "loss": 2.7236,
      "step": 333500
    },
    {
      "activations/layer0_attention_weight_max": 16.122583389282227,
      "activations/layer0_attention_weight_min": -12.394262313842773,
      "activations/layer10_attention_weight_max": 36.063358306884766,
      "activations/layer10_attention_weight_min": -36.74803161621094,
      "activations/layer11_attention_weight_max": 36.544925689697266,
      "activations/layer11_attention_weight_min": -37.466087341308594,
      "activations/layer12_attention_weight_max": 29.818147659301758,
      "activations/layer12_attention_weight_min": -30.44853973388672,
      "activations/layer13_attention_weight_max": 42.90684127807617,
      "activations/layer13_attention_weight_min": -37.017250061035156,
      "activations/layer14_attention_weight_max": 41.74869155883789,
      "activations/layer14_attention_weight_min": -41.79618835449219,
      "activations/layer15_attention_weight_max": 41.50181579589844,
      "activations/layer15_attention_weight_min": -37.996795654296875,
      "activations/layer16_attention_weight_max": 28.949459075927734,
      "activations/layer16_attention_weight_min": -27.889205932617188,
      "activations/layer17_attention_weight_max": 29.954389572143555,
      "activations/layer17_attention_weight_min": -24.992353439331055,
      "activations/layer18_attention_weight_max": 31.462886810302734,
      "activations/layer18_attention_weight_min": -26.199560165405273,
      "activations/layer19_attention_weight_max": 33.4400520324707,
      "activations/layer19_attention_weight_min": -33.3540153503418,
      "activations/layer1_attention_weight_max": 15.059438705444336,
      "activations/layer1_attention_weight_min": -14.872464179992676,
      "activations/layer20_attention_weight_max": 29.328100204467773,
      "activations/layer20_attention_weight_min": -27.055999755859375,
      "activations/layer21_attention_weight_max": 29.959793090820312,
      "activations/layer21_attention_weight_min": -25.09330940246582,
      "activations/layer22_attention_weight_max": 42.34422302246094,
      "activations/layer22_attention_weight_min": -33.72966384887695,
      "activations/layer23_attention_weight_max": 32.99967956542969,
      "activations/layer23_attention_weight_min": -28.040969848632812,
      "activations/layer2_attention_weight_max": 31.55934715270996,
      "activations/layer2_attention_weight_min": -31.22650718688965,
      "activations/layer3_attention_weight_max": 98.93937683105469,
      "activations/layer3_attention_weight_min": -103.2953109741211,
      "activations/layer4_attention_weight_max": 102.06566619873047,
      "activations/layer4_attention_weight_min": -107.1146240234375,
      "activations/layer5_attention_weight_max": 48.93897247314453,
      "activations/layer5_attention_weight_min": -63.130638122558594,
      "activations/layer6_attention_weight_max": 44.35350036621094,
      "activations/layer6_attention_weight_min": -50.23540496826172,
      "activations/layer7_attention_weight_max": 92.93811798095703,
      "activations/layer7_attention_weight_min": -101.50448608398438,
      "activations/layer8_attention_weight_max": 45.30509567260742,
      "activations/layer8_attention_weight_min": -46.1334342956543,
      "activations/layer9_attention_weight_max": 33.46693420410156,
      "activations/layer9_attention_weight_min": -35.051815032958984,
      "epoch": 19.38,
      "learning_rate": 2.5271212121212118e-05,
      "loss": 2.7128,
      "step": 333550
    },
    {
      "activations/layer0_attention_weight_max": 16.285558700561523,
      "activations/layer0_attention_weight_min": -13.061601638793945,
      "activations/layer10_attention_weight_max": 32.755401611328125,
      "activations/layer10_attention_weight_min": -32.97331237792969,
      "activations/layer11_attention_weight_max": 33.94371795654297,
      "activations/layer11_attention_weight_min": -33.968505859375,
      "activations/layer12_attention_weight_max": 27.46683692932129,
      "activations/layer12_attention_weight_min": -28.16006851196289,
      "activations/layer13_attention_weight_max": 39.42890930175781,
      "activations/layer13_attention_weight_min": -37.24859619140625,
      "activations/layer14_attention_weight_max": 39.60364532470703,
      "activations/layer14_attention_weight_min": -41.95035934448242,
      "activations/layer15_attention_weight_max": 40.09360122680664,
      "activations/layer15_attention_weight_min": -39.2935905456543,
      "activations/layer16_attention_weight_max": 29.56755828857422,
      "activations/layer16_attention_weight_min": -28.398143768310547,
      "activations/layer17_attention_weight_max": 29.81020164489746,
      "activations/layer17_attention_weight_min": -25.118181228637695,
      "activations/layer18_attention_weight_max": 31.257097244262695,
      "activations/layer18_attention_weight_min": -26.582786560058594,
      "activations/layer19_attention_weight_max": 34.493629455566406,
      "activations/layer19_attention_weight_min": -30.68385124206543,
      "activations/layer1_attention_weight_max": 15.745253562927246,
      "activations/layer1_attention_weight_min": -14.034879684448242,
      "activations/layer20_attention_weight_max": 28.26586151123047,
      "activations/layer20_attention_weight_min": -25.27846908569336,
      "activations/layer21_attention_weight_max": 29.0682430267334,
      "activations/layer21_attention_weight_min": -27.103744506835938,
      "activations/layer22_attention_weight_max": 40.628501892089844,
      "activations/layer22_attention_weight_min": -30.994754791259766,
      "activations/layer23_attention_weight_max": 34.789283752441406,
      "activations/layer23_attention_weight_min": -26.211769104003906,
      "activations/layer2_attention_weight_max": 32.402000427246094,
      "activations/layer2_attention_weight_min": -30.32118034362793,
      "activations/layer3_attention_weight_max": 98.37134552001953,
      "activations/layer3_attention_weight_min": -97.3253402709961,
      "activations/layer4_attention_weight_max": 101.17606353759766,
      "activations/layer4_attention_weight_min": -104.50630950927734,
      "activations/layer5_attention_weight_max": 49.91374969482422,
      "activations/layer5_attention_weight_min": -61.63939666748047,
      "activations/layer6_attention_weight_max": 42.195919036865234,
      "activations/layer6_attention_weight_min": -46.94573211669922,
      "activations/layer7_attention_weight_max": 91.16187286376953,
      "activations/layer7_attention_weight_min": -91.42984771728516,
      "activations/layer8_attention_weight_max": 40.746131896972656,
      "activations/layer8_attention_weight_min": -41.92443084716797,
      "activations/layer9_attention_weight_max": 30.622900009155273,
      "activations/layer9_attention_weight_min": -34.04638671875,
      "epoch": 19.38,
      "learning_rate": 2.5252272727272726e-05,
      "loss": 2.7122,
      "step": 333600
    },
    {
      "activations/layer0_attention_weight_max": 15.776065826416016,
      "activations/layer0_attention_weight_min": -13.641533851623535,
      "activations/layer10_attention_weight_max": 32.75651550292969,
      "activations/layer10_attention_weight_min": -35.01449203491211,
      "activations/layer11_attention_weight_max": 34.243831634521484,
      "activations/layer11_attention_weight_min": -33.99977493286133,
      "activations/layer12_attention_weight_max": 26.64769744873047,
      "activations/layer12_attention_weight_min": -26.616762161254883,
      "activations/layer13_attention_weight_max": 38.34819412231445,
      "activations/layer13_attention_weight_min": -35.181121826171875,
      "activations/layer14_attention_weight_max": 41.822959899902344,
      "activations/layer14_attention_weight_min": -39.5958366394043,
      "activations/layer15_attention_weight_max": 37.46974182128906,
      "activations/layer15_attention_weight_min": -37.90303421020508,
      "activations/layer16_attention_weight_max": 29.295372009277344,
      "activations/layer16_attention_weight_min": -29.87978172302246,
      "activations/layer17_attention_weight_max": 30.785369873046875,
      "activations/layer17_attention_weight_min": -26.799739837646484,
      "activations/layer18_attention_weight_max": 31.710590362548828,
      "activations/layer18_attention_weight_min": -28.827449798583984,
      "activations/layer19_attention_weight_max": 33.2699089050293,
      "activations/layer19_attention_weight_min": -32.80815887451172,
      "activations/layer1_attention_weight_max": 15.573643684387207,
      "activations/layer1_attention_weight_min": -14.17075252532959,
      "activations/layer20_attention_weight_max": 29.85033416748047,
      "activations/layer20_attention_weight_min": -26.357938766479492,
      "activations/layer21_attention_weight_max": 30.487865447998047,
      "activations/layer21_attention_weight_min": -24.754745483398438,
      "activations/layer22_attention_weight_max": 42.539554595947266,
      "activations/layer22_attention_weight_min": -31.050365447998047,
      "activations/layer23_attention_weight_max": 36.98529815673828,
      "activations/layer23_attention_weight_min": -24.641889572143555,
      "activations/layer2_attention_weight_max": 31.795570373535156,
      "activations/layer2_attention_weight_min": -30.597412109375,
      "activations/layer3_attention_weight_max": 97.07148742675781,
      "activations/layer3_attention_weight_min": -100.19033813476562,
      "activations/layer4_attention_weight_max": 100.38037872314453,
      "activations/layer4_attention_weight_min": -104.5266342163086,
      "activations/layer5_attention_weight_max": 48.829444885253906,
      "activations/layer5_attention_weight_min": -60.35215377807617,
      "activations/layer6_attention_weight_max": 41.9503288269043,
      "activations/layer6_attention_weight_min": -45.98714065551758,
      "activations/layer7_attention_weight_max": 88.58741760253906,
      "activations/layer7_attention_weight_min": -98.96575927734375,
      "activations/layer8_attention_weight_max": 40.03078842163086,
      "activations/layer8_attention_weight_min": -43.98368453979492,
      "activations/layer9_attention_weight_max": 31.008955001831055,
      "activations/layer9_attention_weight_min": -35.42782974243164,
      "epoch": 19.39,
      "learning_rate": 2.523333333333333e-05,
      "loss": 2.7262,
      "step": 333650
    },
    {
      "activations/layer0_attention_weight_max": 16.558134078979492,
      "activations/layer0_attention_weight_min": -13.240612030029297,
      "activations/layer10_attention_weight_max": 32.72926330566406,
      "activations/layer10_attention_weight_min": -34.39921951293945,
      "activations/layer11_attention_weight_max": 33.362060546875,
      "activations/layer11_attention_weight_min": -34.7270622253418,
      "activations/layer12_attention_weight_max": 27.266977310180664,
      "activations/layer12_attention_weight_min": -27.75981330871582,
      "activations/layer13_attention_weight_max": 43.38047790527344,
      "activations/layer13_attention_weight_min": -36.72340774536133,
      "activations/layer14_attention_weight_max": 46.529579162597656,
      "activations/layer14_attention_weight_min": -38.573692321777344,
      "activations/layer15_attention_weight_max": 44.71229553222656,
      "activations/layer15_attention_weight_min": -39.80946350097656,
      "activations/layer16_attention_weight_max": 33.288509368896484,
      "activations/layer16_attention_weight_min": -30.30738067626953,
      "activations/layer17_attention_weight_max": 33.35164260864258,
      "activations/layer17_attention_weight_min": -27.890520095825195,
      "activations/layer18_attention_weight_max": 33.32905197143555,
      "activations/layer18_attention_weight_min": -26.01323127746582,
      "activations/layer19_attention_weight_max": 39.31196212768555,
      "activations/layer19_attention_weight_min": -32.41209030151367,
      "activations/layer1_attention_weight_max": 15.240866661071777,
      "activations/layer1_attention_weight_min": -15.11125659942627,
      "activations/layer20_attention_weight_max": 29.769319534301758,
      "activations/layer20_attention_weight_min": -26.001663208007812,
      "activations/layer21_attention_weight_max": 29.35764503479004,
      "activations/layer21_attention_weight_min": -26.187255859375,
      "activations/layer22_attention_weight_max": 42.78754425048828,
      "activations/layer22_attention_weight_min": -32.3275032043457,
      "activations/layer23_attention_weight_max": 35.85835266113281,
      "activations/layer23_attention_weight_min": -25.61701011657715,
      "activations/layer2_attention_weight_max": 30.69729232788086,
      "activations/layer2_attention_weight_min": -29.89400863647461,
      "activations/layer3_attention_weight_max": 99.69026184082031,
      "activations/layer3_attention_weight_min": -97.79419708251953,
      "activations/layer4_attention_weight_max": 106.02823638916016,
      "activations/layer4_attention_weight_min": -106.07881927490234,
      "activations/layer5_attention_weight_max": 49.071861267089844,
      "activations/layer5_attention_weight_min": -65.02705383300781,
      "activations/layer6_attention_weight_max": 46.119380950927734,
      "activations/layer6_attention_weight_min": -49.152034759521484,
      "activations/layer7_attention_weight_max": 95.73441314697266,
      "activations/layer7_attention_weight_min": -98.2002944946289,
      "activations/layer8_attention_weight_max": 40.61940383911133,
      "activations/layer8_attention_weight_min": -44.100337982177734,
      "activations/layer9_attention_weight_max": 31.278766632080078,
      "activations/layer9_attention_weight_min": -33.688228607177734,
      "epoch": 19.39,
      "learning_rate": 2.521439393939394e-05,
      "loss": 2.7246,
      "step": 333700
    },
    {
      "activations/layer0_attention_weight_max": 16.41727638244629,
      "activations/layer0_attention_weight_min": -13.594195365905762,
      "activations/layer10_attention_weight_max": 33.46998596191406,
      "activations/layer10_attention_weight_min": -32.72804260253906,
      "activations/layer11_attention_weight_max": 33.19017028808594,
      "activations/layer11_attention_weight_min": -33.584693908691406,
      "activations/layer12_attention_weight_max": 26.97329330444336,
      "activations/layer12_attention_weight_min": -27.135562896728516,
      "activations/layer13_attention_weight_max": 37.75679016113281,
      "activations/layer13_attention_weight_min": -36.84022903442383,
      "activations/layer14_attention_weight_max": 40.29983901977539,
      "activations/layer14_attention_weight_min": -39.41110610961914,
      "activations/layer15_attention_weight_max": 38.86900329589844,
      "activations/layer15_attention_weight_min": -36.7572135925293,
      "activations/layer16_attention_weight_max": 28.574085235595703,
      "activations/layer16_attention_weight_min": -27.323043823242188,
      "activations/layer17_attention_weight_max": 29.794715881347656,
      "activations/layer17_attention_weight_min": -27.575510025024414,
      "activations/layer18_attention_weight_max": 29.399883270263672,
      "activations/layer18_attention_weight_min": -25.024396896362305,
      "activations/layer19_attention_weight_max": 31.528989791870117,
      "activations/layer19_attention_weight_min": -31.90132713317871,
      "activations/layer1_attention_weight_max": 15.85554027557373,
      "activations/layer1_attention_weight_min": -14.191086769104004,
      "activations/layer20_attention_weight_max": 28.820781707763672,
      "activations/layer20_attention_weight_min": -24.309452056884766,
      "activations/layer21_attention_weight_max": 29.839017868041992,
      "activations/layer21_attention_weight_min": -24.838815689086914,
      "activations/layer22_attention_weight_max": 39.268157958984375,
      "activations/layer22_attention_weight_min": -32.19354248046875,
      "activations/layer23_attention_weight_max": 30.190458297729492,
      "activations/layer23_attention_weight_min": -25.480695724487305,
      "activations/layer2_attention_weight_max": 30.985746383666992,
      "activations/layer2_attention_weight_min": -30.136463165283203,
      "activations/layer3_attention_weight_max": 98.6396713256836,
      "activations/layer3_attention_weight_min": -97.67488861083984,
      "activations/layer4_attention_weight_max": 105.33021545410156,
      "activations/layer4_attention_weight_min": -104.3326644897461,
      "activations/layer5_attention_weight_max": 49.12577438354492,
      "activations/layer5_attention_weight_min": -59.76689910888672,
      "activations/layer6_attention_weight_max": 43.09321212768555,
      "activations/layer6_attention_weight_min": -47.064144134521484,
      "activations/layer7_attention_weight_max": 89.36369323730469,
      "activations/layer7_attention_weight_min": -91.24205017089844,
      "activations/layer8_attention_weight_max": 39.420928955078125,
      "activations/layer8_attention_weight_min": -40.89876937866211,
      "activations/layer9_attention_weight_max": 32.40098190307617,
      "activations/layer9_attention_weight_min": -32.37966537475586,
      "epoch": 19.39,
      "learning_rate": 2.519545454545454e-05,
      "loss": 2.7173,
      "step": 333750
    },
    {
      "activations/layer0_attention_weight_max": 16.68695068359375,
      "activations/layer0_attention_weight_min": -13.817872047424316,
      "activations/layer10_attention_weight_max": 31.485258102416992,
      "activations/layer10_attention_weight_min": -33.4532470703125,
      "activations/layer11_attention_weight_max": 32.04873275756836,
      "activations/layer11_attention_weight_min": -32.95311737060547,
      "activations/layer12_attention_weight_max": 26.45787811279297,
      "activations/layer12_attention_weight_min": -27.321853637695312,
      "activations/layer13_attention_weight_max": 37.459877014160156,
      "activations/layer13_attention_weight_min": -36.66168975830078,
      "activations/layer14_attention_weight_max": 45.107425689697266,
      "activations/layer14_attention_weight_min": -36.67960739135742,
      "activations/layer15_attention_weight_max": 37.696529388427734,
      "activations/layer15_attention_weight_min": -35.290016174316406,
      "activations/layer16_attention_weight_max": 28.824581146240234,
      "activations/layer16_attention_weight_min": -29.753997802734375,
      "activations/layer17_attention_weight_max": 29.067922592163086,
      "activations/layer17_attention_weight_min": -26.489404678344727,
      "activations/layer18_attention_weight_max": 30.763822555541992,
      "activations/layer18_attention_weight_min": -26.299123764038086,
      "activations/layer19_attention_weight_max": 30.95210838317871,
      "activations/layer19_attention_weight_min": -31.74228286743164,
      "activations/layer1_attention_weight_max": 15.790550231933594,
      "activations/layer1_attention_weight_min": -15.06298828125,
      "activations/layer20_attention_weight_max": 26.9427547454834,
      "activations/layer20_attention_weight_min": -26.054508209228516,
      "activations/layer21_attention_weight_max": 26.4548397064209,
      "activations/layer21_attention_weight_min": -24.57259178161621,
      "activations/layer22_attention_weight_max": 38.3419303894043,
      "activations/layer22_attention_weight_min": -32.25242614746094,
      "activations/layer23_attention_weight_max": 30.064682006835938,
      "activations/layer23_attention_weight_min": -26.487232208251953,
      "activations/layer2_attention_weight_max": 31.435672760009766,
      "activations/layer2_attention_weight_min": -31.27594566345215,
      "activations/layer3_attention_weight_max": 96.26649475097656,
      "activations/layer3_attention_weight_min": -98.96257019042969,
      "activations/layer4_attention_weight_max": 104.0604248046875,
      "activations/layer4_attention_weight_min": -107.31197357177734,
      "activations/layer5_attention_weight_max": 51.91852569580078,
      "activations/layer5_attention_weight_min": -61.50190353393555,
      "activations/layer6_attention_weight_max": 42.28532028198242,
      "activations/layer6_attention_weight_min": -44.10540008544922,
      "activations/layer7_attention_weight_max": 91.42597198486328,
      "activations/layer7_attention_weight_min": -92.40650939941406,
      "activations/layer8_attention_weight_max": 39.33374786376953,
      "activations/layer8_attention_weight_min": -41.91191482543945,
      "activations/layer9_attention_weight_max": 29.69431495666504,
      "activations/layer9_attention_weight_min": -32.798309326171875,
      "epoch": 19.4,
      "learning_rate": 2.517651515151515e-05,
      "loss": 2.7189,
      "step": 333800
    },
    {
      "activations/layer0_attention_weight_max": 15.713869094848633,
      "activations/layer0_attention_weight_min": -13.00466251373291,
      "activations/layer10_attention_weight_max": 34.11427688598633,
      "activations/layer10_attention_weight_min": -37.343238830566406,
      "activations/layer11_attention_weight_max": 32.73935317993164,
      "activations/layer11_attention_weight_min": -35.26287078857422,
      "activations/layer12_attention_weight_max": 26.082460403442383,
      "activations/layer12_attention_weight_min": -27.144412994384766,
      "activations/layer13_attention_weight_max": 37.618560791015625,
      "activations/layer13_attention_weight_min": -35.053321838378906,
      "activations/layer14_attention_weight_max": 41.72030258178711,
      "activations/layer14_attention_weight_min": -36.74950408935547,
      "activations/layer15_attention_weight_max": 36.66835021972656,
      "activations/layer15_attention_weight_min": -37.29390335083008,
      "activations/layer16_attention_weight_max": 28.755290985107422,
      "activations/layer16_attention_weight_min": -28.111116409301758,
      "activations/layer17_attention_weight_max": 29.036224365234375,
      "activations/layer17_attention_weight_min": -26.8421573638916,
      "activations/layer18_attention_weight_max": 28.835588455200195,
      "activations/layer18_attention_weight_min": -26.105226516723633,
      "activations/layer19_attention_weight_max": 33.59593963623047,
      "activations/layer19_attention_weight_min": -32.38015365600586,
      "activations/layer1_attention_weight_max": 15.534348487854004,
      "activations/layer1_attention_weight_min": -14.772926330566406,
      "activations/layer20_attention_weight_max": 28.350786209106445,
      "activations/layer20_attention_weight_min": -26.816123962402344,
      "activations/layer21_attention_weight_max": 28.283565521240234,
      "activations/layer21_attention_weight_min": -27.111276626586914,
      "activations/layer22_attention_weight_max": 38.08794021606445,
      "activations/layer22_attention_weight_min": -33.13898468017578,
      "activations/layer23_attention_weight_max": 33.078643798828125,
      "activations/layer23_attention_weight_min": -29.304405212402344,
      "activations/layer2_attention_weight_max": 31.490001678466797,
      "activations/layer2_attention_weight_min": -32.079620361328125,
      "activations/layer3_attention_weight_max": 100.42214965820312,
      "activations/layer3_attention_weight_min": -101.35039520263672,
      "activations/layer4_attention_weight_max": 102.73225402832031,
      "activations/layer4_attention_weight_min": -111.76049041748047,
      "activations/layer5_attention_weight_max": 48.5424919128418,
      "activations/layer5_attention_weight_min": -59.23706817626953,
      "activations/layer6_attention_weight_max": 42.93936538696289,
      "activations/layer6_attention_weight_min": -48.966636657714844,
      "activations/layer7_attention_weight_max": 93.86890411376953,
      "activations/layer7_attention_weight_min": -107.90756225585938,
      "activations/layer8_attention_weight_max": 39.18365478515625,
      "activations/layer8_attention_weight_min": -43.210941314697266,
      "activations/layer9_attention_weight_max": 31.01316261291504,
      "activations/layer9_attention_weight_min": -34.29993438720703,
      "epoch": 19.4,
      "learning_rate": 2.5157575757575754e-05,
      "loss": 2.7104,
      "step": 333850
    },
    {
      "activations/layer0_attention_weight_max": 15.587502479553223,
      "activations/layer0_attention_weight_min": -13.35599136352539,
      "activations/layer10_attention_weight_max": 35.56488037109375,
      "activations/layer10_attention_weight_min": -37.22576904296875,
      "activations/layer11_attention_weight_max": 36.795013427734375,
      "activations/layer11_attention_weight_min": -34.938690185546875,
      "activations/layer12_attention_weight_max": 26.42107582092285,
      "activations/layer12_attention_weight_min": -25.457304000854492,
      "activations/layer13_attention_weight_max": 40.06827926635742,
      "activations/layer13_attention_weight_min": -35.484825134277344,
      "activations/layer14_attention_weight_max": 41.76350021362305,
      "activations/layer14_attention_weight_min": -37.066856384277344,
      "activations/layer15_attention_weight_max": 39.76786804199219,
      "activations/layer15_attention_weight_min": -37.9636344909668,
      "activations/layer16_attention_weight_max": 32.325138092041016,
      "activations/layer16_attention_weight_min": -31.07320213317871,
      "activations/layer17_attention_weight_max": 28.8087100982666,
      "activations/layer17_attention_weight_min": -27.970542907714844,
      "activations/layer18_attention_weight_max": 31.02739143371582,
      "activations/layer18_attention_weight_min": -26.995473861694336,
      "activations/layer19_attention_weight_max": 30.729490280151367,
      "activations/layer19_attention_weight_min": -30.99302101135254,
      "activations/layer1_attention_weight_max": 17.220149993896484,
      "activations/layer1_attention_weight_min": -12.812402725219727,
      "activations/layer20_attention_weight_max": 26.700023651123047,
      "activations/layer20_attention_weight_min": -26.20162582397461,
      "activations/layer21_attention_weight_max": 27.7908935546875,
      "activations/layer21_attention_weight_min": -25.288599014282227,
      "activations/layer22_attention_weight_max": 37.75453186035156,
      "activations/layer22_attention_weight_min": -32.26456069946289,
      "activations/layer23_attention_weight_max": 30.083972930908203,
      "activations/layer23_attention_weight_min": -27.174367904663086,
      "activations/layer2_attention_weight_max": 34.24900436401367,
      "activations/layer2_attention_weight_min": -32.76319885253906,
      "activations/layer3_attention_weight_max": 100.16111755371094,
      "activations/layer3_attention_weight_min": -99.38363647460938,
      "activations/layer4_attention_weight_max": 105.0125961303711,
      "activations/layer4_attention_weight_min": -109.17911529541016,
      "activations/layer5_attention_weight_max": 49.06792449951172,
      "activations/layer5_attention_weight_min": -66.85065460205078,
      "activations/layer6_attention_weight_max": 44.357826232910156,
      "activations/layer6_attention_weight_min": -48.28953552246094,
      "activations/layer7_attention_weight_max": 89.8138656616211,
      "activations/layer7_attention_weight_min": -95.85926055908203,
      "activations/layer8_attention_weight_max": 43.721588134765625,
      "activations/layer8_attention_weight_min": -44.519325256347656,
      "activations/layer9_attention_weight_max": 30.38334083557129,
      "activations/layer9_attention_weight_min": -33.34091567993164,
      "epoch": 19.4,
      "learning_rate": 2.5138636363636362e-05,
      "loss": 2.7322,
      "step": 333900
    },
    {
      "activations/layer0_attention_weight_max": 16.30105209350586,
      "activations/layer0_attention_weight_min": -14.524149894714355,
      "activations/layer10_attention_weight_max": 33.687591552734375,
      "activations/layer10_attention_weight_min": -35.854251861572266,
      "activations/layer11_attention_weight_max": 34.315467834472656,
      "activations/layer11_attention_weight_min": -37.29853057861328,
      "activations/layer12_attention_weight_max": 28.275087356567383,
      "activations/layer12_attention_weight_min": -28.014772415161133,
      "activations/layer13_attention_weight_max": 42.52120590209961,
      "activations/layer13_attention_weight_min": -35.41263961791992,
      "activations/layer14_attention_weight_max": 51.37762451171875,
      "activations/layer14_attention_weight_min": -38.68857955932617,
      "activations/layer15_attention_weight_max": 42.32957458496094,
      "activations/layer15_attention_weight_min": -37.178279876708984,
      "activations/layer16_attention_weight_max": 34.62588882446289,
      "activations/layer16_attention_weight_min": -29.092626571655273,
      "activations/layer17_attention_weight_max": 31.81670570373535,
      "activations/layer17_attention_weight_min": -27.79060173034668,
      "activations/layer18_attention_weight_max": 36.54576110839844,
      "activations/layer18_attention_weight_min": -26.498064041137695,
      "activations/layer19_attention_weight_max": 37.78129959106445,
      "activations/layer19_attention_weight_min": -32.0272331237793,
      "activations/layer1_attention_weight_max": 15.908276557922363,
      "activations/layer1_attention_weight_min": -13.374052047729492,
      "activations/layer20_attention_weight_max": 30.83446502685547,
      "activations/layer20_attention_weight_min": -25.364734649658203,
      "activations/layer21_attention_weight_max": 32.38935852050781,
      "activations/layer21_attention_weight_min": -25.189964294433594,
      "activations/layer22_attention_weight_max": 49.375511169433594,
      "activations/layer22_attention_weight_min": -32.075714111328125,
      "activations/layer23_attention_weight_max": 35.7859992980957,
      "activations/layer23_attention_weight_min": -26.646564483642578,
      "activations/layer2_attention_weight_max": 30.45899772644043,
      "activations/layer2_attention_weight_min": -30.36770248413086,
      "activations/layer3_attention_weight_max": 97.90699768066406,
      "activations/layer3_attention_weight_min": -102.55243682861328,
      "activations/layer4_attention_weight_max": 106.7928466796875,
      "activations/layer4_attention_weight_min": -106.40990447998047,
      "activations/layer5_attention_weight_max": 48.34113311767578,
      "activations/layer5_attention_weight_min": -62.192665100097656,
      "activations/layer6_attention_weight_max": 44.0926628112793,
      "activations/layer6_attention_weight_min": -47.259605407714844,
      "activations/layer7_attention_weight_max": 91.24131774902344,
      "activations/layer7_attention_weight_min": -99.5291519165039,
      "activations/layer8_attention_weight_max": 42.532737731933594,
      "activations/layer8_attention_weight_min": -45.415916442871094,
      "activations/layer9_attention_weight_max": 33.027347564697266,
      "activations/layer9_attention_weight_min": -35.56049346923828,
      "epoch": 19.4,
      "learning_rate": 2.5120075757575753e-05,
      "loss": 2.7186,
      "step": 333950
    },
    {
      "activations/layer0_attention_weight_max": 15.725822448730469,
      "activations/layer0_attention_weight_min": -12.940970420837402,
      "activations/layer10_attention_weight_max": 37.77805709838867,
      "activations/layer10_attention_weight_min": -34.87205123901367,
      "activations/layer11_attention_weight_max": 35.04509353637695,
      "activations/layer11_attention_weight_min": -34.37030792236328,
      "activations/layer12_attention_weight_max": 29.381010055541992,
      "activations/layer12_attention_weight_min": -26.755306243896484,
      "activations/layer13_attention_weight_max": 45.04043960571289,
      "activations/layer13_attention_weight_min": -35.521217346191406,
      "activations/layer14_attention_weight_max": 46.66168212890625,
      "activations/layer14_attention_weight_min": -38.749900817871094,
      "activations/layer15_attention_weight_max": 44.71905517578125,
      "activations/layer15_attention_weight_min": -36.62036895751953,
      "activations/layer16_attention_weight_max": 30.659923553466797,
      "activations/layer16_attention_weight_min": -28.021160125732422,
      "activations/layer17_attention_weight_max": 31.01808738708496,
      "activations/layer17_attention_weight_min": -25.945148468017578,
      "activations/layer18_attention_weight_max": 34.19160079956055,
      "activations/layer18_attention_weight_min": -25.582218170166016,
      "activations/layer19_attention_weight_max": 38.535858154296875,
      "activations/layer19_attention_weight_min": -31.243759155273438,
      "activations/layer1_attention_weight_max": 15.36057186126709,
      "activations/layer1_attention_weight_min": -15.857839584350586,
      "activations/layer20_attention_weight_max": 30.1263427734375,
      "activations/layer20_attention_weight_min": -25.69220542907715,
      "activations/layer21_attention_weight_max": 30.136688232421875,
      "activations/layer21_attention_weight_min": -24.493301391601562,
      "activations/layer22_attention_weight_max": 46.0304069519043,
      "activations/layer22_attention_weight_min": -32.61061096191406,
      "activations/layer23_attention_weight_max": 32.84028625488281,
      "activations/layer23_attention_weight_min": -26.97838592529297,
      "activations/layer2_attention_weight_max": 30.841018676757812,
      "activations/layer2_attention_weight_min": -31.469533920288086,
      "activations/layer3_attention_weight_max": 94.98005676269531,
      "activations/layer3_attention_weight_min": -99.07545471191406,
      "activations/layer4_attention_weight_max": 102.35832977294922,
      "activations/layer4_attention_weight_min": -108.509765625,
      "activations/layer5_attention_weight_max": 52.34763717651367,
      "activations/layer5_attention_weight_min": -62.57587814331055,
      "activations/layer6_attention_weight_max": 43.141090393066406,
      "activations/layer6_attention_weight_min": -46.36396026611328,
      "activations/layer7_attention_weight_max": 90.66327667236328,
      "activations/layer7_attention_weight_min": -95.43641662597656,
      "activations/layer8_attention_weight_max": 41.60954284667969,
      "activations/layer8_attention_weight_min": -44.37158966064453,
      "activations/layer9_attention_weight_max": 33.09959030151367,
      "activations/layer9_attention_weight_min": -34.82847213745117,
      "epoch": 19.41,
      "learning_rate": 2.510113636363636e-05,
      "loss": 2.7314,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4585,
      "eval_samples_per_second": 507.652,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4585,
      "eval_openwebtext_samples_per_second": 507.652,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9352,
      "eval_wikitext_samples_per_second": 235.639,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_lambada_loss": 2.3515625,
      "eval_lambada_ppl": 10.501966233859342,
      "eval_lambada_runtime": 9.5216,
      "eval_lambada_samples_per_second": 511.364,
      "step": 334000
    },
    {
      "activations/layer0_attention_weight_max": 16.275959014892578,
      "activations/layer0_attention_weight_min": -12.610722541809082,
      "activations/layer10_attention_weight_max": 34.61974334716797,
      "activations/layer10_attention_weight_min": -34.56999588012695,
      "activations/layer11_attention_weight_max": 34.7420539855957,
      "activations/layer11_attention_weight_min": -34.67023468017578,
      "activations/layer12_attention_weight_max": 30.541332244873047,
      "activations/layer12_attention_weight_min": -29.35320472717285,
      "activations/layer13_attention_weight_max": 44.51301956176758,
      "activations/layer13_attention_weight_min": -39.562286376953125,
      "activations/layer14_attention_weight_max": 46.98084259033203,
      "activations/layer14_attention_weight_min": -43.203338623046875,
      "activations/layer15_attention_weight_max": 41.41653060913086,
      "activations/layer15_attention_weight_min": -38.210906982421875,
      "activations/layer16_attention_weight_max": 29.457170486450195,
      "activations/layer16_attention_weight_min": -28.964351654052734,
      "activations/layer17_attention_weight_max": 29.86629867553711,
      "activations/layer17_attention_weight_min": -26.94587516784668,
      "activations/layer18_attention_weight_max": 32.76174545288086,
      "activations/layer18_attention_weight_min": -24.9152889251709,
      "activations/layer19_attention_weight_max": 31.134775161743164,
      "activations/layer19_attention_weight_min": -30.76261329650879,
      "activations/layer1_attention_weight_max": 15.30562686920166,
      "activations/layer1_attention_weight_min": -14.073927879333496,
      "activations/layer20_attention_weight_max": 28.659523010253906,
      "activations/layer20_attention_weight_min": -26.1351261138916,
      "activations/layer21_attention_weight_max": 27.47650718688965,
      "activations/layer21_attention_weight_min": -24.4432315826416,
      "activations/layer22_attention_weight_max": 39.60309600830078,
      "activations/layer22_attention_weight_min": -31.39324188232422,
      "activations/layer23_attention_weight_max": 30.38482666015625,
      "activations/layer23_attention_weight_min": -24.4017333984375,
      "activations/layer2_attention_weight_max": 31.298992156982422,
      "activations/layer2_attention_weight_min": -30.376327514648438,
      "activations/layer3_attention_weight_max": 97.52241516113281,
      "activations/layer3_attention_weight_min": -94.85305786132812,
      "activations/layer4_attention_weight_max": 102.62548065185547,
      "activations/layer4_attention_weight_min": -105.09474182128906,
      "activations/layer5_attention_weight_max": 50.7387809753418,
      "activations/layer5_attention_weight_min": -60.53765869140625,
      "activations/layer6_attention_weight_max": 43.81808853149414,
      "activations/layer6_attention_weight_min": -46.830894470214844,
      "activations/layer7_attention_weight_max": 88.48997497558594,
      "activations/layer7_attention_weight_min": -96.66557312011719,
      "activations/layer8_attention_weight_max": 40.982826232910156,
      "activations/layer8_attention_weight_min": -43.923789978027344,
      "activations/layer9_attention_weight_max": 32.49475860595703,
      "activations/layer9_attention_weight_min": -34.78007125854492,
      "epoch": 19.41,
      "learning_rate": 2.5082196969696966e-05,
      "loss": 2.7404,
      "step": 334050
    },
    {
      "activations/layer0_attention_weight_max": 16.276765823364258,
      "activations/layer0_attention_weight_min": -12.720418930053711,
      "activations/layer10_attention_weight_max": 35.4798469543457,
      "activations/layer10_attention_weight_min": -37.01603698730469,
      "activations/layer11_attention_weight_max": 33.82368469238281,
      "activations/layer11_attention_weight_min": -36.155826568603516,
      "activations/layer12_attention_weight_max": 27.449316024780273,
      "activations/layer12_attention_weight_min": -27.621395111083984,
      "activations/layer13_attention_weight_max": 41.174224853515625,
      "activations/layer13_attention_weight_min": -35.84589767456055,
      "activations/layer14_attention_weight_max": 44.263267517089844,
      "activations/layer14_attention_weight_min": -38.99055480957031,
      "activations/layer15_attention_weight_max": 43.34666442871094,
      "activations/layer15_attention_weight_min": -36.304622650146484,
      "activations/layer16_attention_weight_max": 30.934782028198242,
      "activations/layer16_attention_weight_min": -28.45005989074707,
      "activations/layer17_attention_weight_max": 31.06888771057129,
      "activations/layer17_attention_weight_min": -25.368005752563477,
      "activations/layer18_attention_weight_max": 35.118717193603516,
      "activations/layer18_attention_weight_min": -27.705955505371094,
      "activations/layer19_attention_weight_max": 34.327396392822266,
      "activations/layer19_attention_weight_min": -31.079940795898438,
      "activations/layer1_attention_weight_max": 15.394580841064453,
      "activations/layer1_attention_weight_min": -13.816205978393555,
      "activations/layer20_attention_weight_max": 31.703899383544922,
      "activations/layer20_attention_weight_min": -25.54290008544922,
      "activations/layer21_attention_weight_max": 31.05168914794922,
      "activations/layer21_attention_weight_min": -24.84145164489746,
      "activations/layer22_attention_weight_max": 49.22078323364258,
      "activations/layer22_attention_weight_min": -30.750160217285156,
      "activations/layer23_attention_weight_max": 36.11137771606445,
      "activations/layer23_attention_weight_min": -25.41632080078125,
      "activations/layer2_attention_weight_max": 31.4124813079834,
      "activations/layer2_attention_weight_min": -31.388988494873047,
      "activations/layer3_attention_weight_max": 95.7338638305664,
      "activations/layer3_attention_weight_min": -97.71851348876953,
      "activations/layer4_attention_weight_max": 102.72113037109375,
      "activations/layer4_attention_weight_min": -108.3788070678711,
      "activations/layer5_attention_weight_max": 49.471431732177734,
      "activations/layer5_attention_weight_min": -59.32600021362305,
      "activations/layer6_attention_weight_max": 46.50883483886719,
      "activations/layer6_attention_weight_min": -47.67623519897461,
      "activations/layer7_attention_weight_max": 94.17417907714844,
      "activations/layer7_attention_weight_min": -103.54346466064453,
      "activations/layer8_attention_weight_max": 43.24726867675781,
      "activations/layer8_attention_weight_min": -48.26175308227539,
      "activations/layer9_attention_weight_max": 32.805908203125,
      "activations/layer9_attention_weight_min": -35.70060348510742,
      "epoch": 19.41,
      "learning_rate": 2.5063257575757575e-05,
      "loss": 2.741,
      "step": 334100
    },
    {
      "activations/layer0_attention_weight_max": 17.496644973754883,
      "activations/layer0_attention_weight_min": -13.0178804397583,
      "activations/layer10_attention_weight_max": 33.34864044189453,
      "activations/layer10_attention_weight_min": -35.09970474243164,
      "activations/layer11_attention_weight_max": 32.85454177856445,
      "activations/layer11_attention_weight_min": -35.621856689453125,
      "activations/layer12_attention_weight_max": 29.077266693115234,
      "activations/layer12_attention_weight_min": -28.44070053100586,
      "activations/layer13_attention_weight_max": 42.32935333251953,
      "activations/layer13_attention_weight_min": -37.040504455566406,
      "activations/layer14_attention_weight_max": 45.102638244628906,
      "activations/layer14_attention_weight_min": -39.52711868286133,
      "activations/layer15_attention_weight_max": 40.396366119384766,
      "activations/layer15_attention_weight_min": -38.31271743774414,
      "activations/layer16_attention_weight_max": 32.45801544189453,
      "activations/layer16_attention_weight_min": -29.23658561706543,
      "activations/layer17_attention_weight_max": 29.86298370361328,
      "activations/layer17_attention_weight_min": -26.17426109313965,
      "activations/layer18_attention_weight_max": 33.3732795715332,
      "activations/layer18_attention_weight_min": -26.10114288330078,
      "activations/layer19_attention_weight_max": 35.55526351928711,
      "activations/layer19_attention_weight_min": -30.717355728149414,
      "activations/layer1_attention_weight_max": 15.691217422485352,
      "activations/layer1_attention_weight_min": -13.34858512878418,
      "activations/layer20_attention_weight_max": 29.740459442138672,
      "activations/layer20_attention_weight_min": -25.62999153137207,
      "activations/layer21_attention_weight_max": 31.87162208557129,
      "activations/layer21_attention_weight_min": -24.352262496948242,
      "activations/layer22_attention_weight_max": 48.40825653076172,
      "activations/layer22_attention_weight_min": -31.46107292175293,
      "activations/layer23_attention_weight_max": 35.1492919921875,
      "activations/layer23_attention_weight_min": -28.571887969970703,
      "activations/layer2_attention_weight_max": 32.382965087890625,
      "activations/layer2_attention_weight_min": -30.811614990234375,
      "activations/layer3_attention_weight_max": 94.58285522460938,
      "activations/layer3_attention_weight_min": -99.71670532226562,
      "activations/layer4_attention_weight_max": 104.9483642578125,
      "activations/layer4_attention_weight_min": -108.05907440185547,
      "activations/layer5_attention_weight_max": 54.099342346191406,
      "activations/layer5_attention_weight_min": -64.14896392822266,
      "activations/layer6_attention_weight_max": 43.52396011352539,
      "activations/layer6_attention_weight_min": -45.88172149658203,
      "activations/layer7_attention_weight_max": 89.62850189208984,
      "activations/layer7_attention_weight_min": -98.99864196777344,
      "activations/layer8_attention_weight_max": 41.4015998840332,
      "activations/layer8_attention_weight_min": -42.77118682861328,
      "activations/layer9_attention_weight_max": 32.2475471496582,
      "activations/layer9_attention_weight_min": -34.71471405029297,
      "epoch": 19.42,
      "learning_rate": 2.504431818181818e-05,
      "loss": 2.7078,
      "step": 334150
    },
    {
      "activations/layer0_attention_weight_max": 16.924379348754883,
      "activations/layer0_attention_weight_min": -14.724729537963867,
      "activations/layer10_attention_weight_max": 33.97982406616211,
      "activations/layer10_attention_weight_min": -32.903480529785156,
      "activations/layer11_attention_weight_max": 34.540077209472656,
      "activations/layer11_attention_weight_min": -31.85947036743164,
      "activations/layer12_attention_weight_max": 25.721744537353516,
      "activations/layer12_attention_weight_min": -26.350130081176758,
      "activations/layer13_attention_weight_max": 39.6595458984375,
      "activations/layer13_attention_weight_min": -37.09846115112305,
      "activations/layer14_attention_weight_max": 41.8023681640625,
      "activations/layer14_attention_weight_min": -38.36487579345703,
      "activations/layer15_attention_weight_max": 39.658302307128906,
      "activations/layer15_attention_weight_min": -36.81255340576172,
      "activations/layer16_attention_weight_max": 29.88690948486328,
      "activations/layer16_attention_weight_min": -28.939865112304688,
      "activations/layer17_attention_weight_max": 33.76218032836914,
      "activations/layer17_attention_weight_min": -25.883962631225586,
      "activations/layer18_attention_weight_max": 33.364501953125,
      "activations/layer18_attention_weight_min": -26.832197189331055,
      "activations/layer19_attention_weight_max": 35.34857940673828,
      "activations/layer19_attention_weight_min": -32.90810775756836,
      "activations/layer1_attention_weight_max": 16.370431900024414,
      "activations/layer1_attention_weight_min": -14.19192123413086,
      "activations/layer20_attention_weight_max": 30.17091941833496,
      "activations/layer20_attention_weight_min": -28.22267723083496,
      "activations/layer21_attention_weight_max": 30.21506690979004,
      "activations/layer21_attention_weight_min": -27.54378318786621,
      "activations/layer22_attention_weight_max": 40.97385787963867,
      "activations/layer22_attention_weight_min": -35.13938522338867,
      "activations/layer23_attention_weight_max": 31.220726013183594,
      "activations/layer23_attention_weight_min": -27.30440902709961,
      "activations/layer2_attention_weight_max": 30.98480796813965,
      "activations/layer2_attention_weight_min": -30.77790069580078,
      "activations/layer3_attention_weight_max": 95.92952728271484,
      "activations/layer3_attention_weight_min": -97.16204833984375,
      "activations/layer4_attention_weight_max": 103.60872650146484,
      "activations/layer4_attention_weight_min": -110.9778823852539,
      "activations/layer5_attention_weight_max": 48.957027435302734,
      "activations/layer5_attention_weight_min": -60.54670715332031,
      "activations/layer6_attention_weight_max": 43.53966522216797,
      "activations/layer6_attention_weight_min": -46.722023010253906,
      "activations/layer7_attention_weight_max": 94.24259185791016,
      "activations/layer7_attention_weight_min": -94.97059631347656,
      "activations/layer8_attention_weight_max": 40.82048416137695,
      "activations/layer8_attention_weight_min": -45.61359786987305,
      "activations/layer9_attention_weight_max": 30.82559585571289,
      "activations/layer9_attention_weight_min": -33.22787094116211,
      "epoch": 19.42,
      "learning_rate": 2.5025378787878784e-05,
      "loss": 2.7238,
      "step": 334200
    },
    {
      "activations/layer0_attention_weight_max": 16.47954750061035,
      "activations/layer0_attention_weight_min": -12.606527328491211,
      "activations/layer10_attention_weight_max": 34.54250717163086,
      "activations/layer10_attention_weight_min": -35.67103576660156,
      "activations/layer11_attention_weight_max": 35.028602600097656,
      "activations/layer11_attention_weight_min": -36.20133590698242,
      "activations/layer12_attention_weight_max": 32.580081939697266,
      "activations/layer12_attention_weight_min": -29.110410690307617,
      "activations/layer13_attention_weight_max": 53.31990432739258,
      "activations/layer13_attention_weight_min": -40.2177848815918,
      "activations/layer14_attention_weight_max": 53.79812240600586,
      "activations/layer14_attention_weight_min": -43.53768539428711,
      "activations/layer15_attention_weight_max": 49.896339416503906,
      "activations/layer15_attention_weight_min": -39.41050720214844,
      "activations/layer16_attention_weight_max": 37.867942810058594,
      "activations/layer16_attention_weight_min": -29.2824764251709,
      "activations/layer17_attention_weight_max": 36.22827911376953,
      "activations/layer17_attention_weight_min": -26.595256805419922,
      "activations/layer18_attention_weight_max": 35.73198318481445,
      "activations/layer18_attention_weight_min": -28.094331741333008,
      "activations/layer19_attention_weight_max": 35.558292388916016,
      "activations/layer19_attention_weight_min": -33.497711181640625,
      "activations/layer1_attention_weight_max": 16.13770866394043,
      "activations/layer1_attention_weight_min": -15.298277854919434,
      "activations/layer20_attention_weight_max": 31.33822250366211,
      "activations/layer20_attention_weight_min": -26.814523696899414,
      "activations/layer21_attention_weight_max": 31.63472557067871,
      "activations/layer21_attention_weight_min": -26.303287506103516,
      "activations/layer22_attention_weight_max": 42.1957893371582,
      "activations/layer22_attention_weight_min": -33.93820571899414,
      "activations/layer23_attention_weight_max": 32.49281692504883,
      "activations/layer23_attention_weight_min": -28.072067260742188,
      "activations/layer2_attention_weight_max": 31.638273239135742,
      "activations/layer2_attention_weight_min": -32.69412612915039,
      "activations/layer3_attention_weight_max": 95.27239227294922,
      "activations/layer3_attention_weight_min": -101.07329559326172,
      "activations/layer4_attention_weight_max": 105.48933410644531,
      "activations/layer4_attention_weight_min": -106.60011291503906,
      "activations/layer5_attention_weight_max": 52.57932662963867,
      "activations/layer5_attention_weight_min": -63.15364074707031,
      "activations/layer6_attention_weight_max": 45.85817337036133,
      "activations/layer6_attention_weight_min": -48.471778869628906,
      "activations/layer7_attention_weight_max": 99.30699157714844,
      "activations/layer7_attention_weight_min": -104.88996124267578,
      "activations/layer8_attention_weight_max": 44.02725601196289,
      "activations/layer8_attention_weight_min": -46.432769775390625,
      "activations/layer9_attention_weight_max": 33.28801345825195,
      "activations/layer9_attention_weight_min": -36.20512771606445,
      "epoch": 19.42,
      "learning_rate": 2.500643939393939e-05,
      "loss": 2.7275,
      "step": 334250
    },
    {
      "activations/layer0_attention_weight_max": 16.52455711364746,
      "activations/layer0_attention_weight_min": -13.903474807739258,
      "activations/layer10_attention_weight_max": 32.612342834472656,
      "activations/layer10_attention_weight_min": -32.47936248779297,
      "activations/layer11_attention_weight_max": 33.211483001708984,
      "activations/layer11_attention_weight_min": -32.45245361328125,
      "activations/layer12_attention_weight_max": 25.454980850219727,
      "activations/layer12_attention_weight_min": -26.691043853759766,
      "activations/layer13_attention_weight_max": 37.22776794433594,
      "activations/layer13_attention_weight_min": -35.125648498535156,
      "activations/layer14_attention_weight_max": 40.02688980102539,
      "activations/layer14_attention_weight_min": -37.33232498168945,
      "activations/layer15_attention_weight_max": 38.15767288208008,
      "activations/layer15_attention_weight_min": -34.913299560546875,
      "activations/layer16_attention_weight_max": 28.785968780517578,
      "activations/layer16_attention_weight_min": -31.04506492614746,
      "activations/layer17_attention_weight_max": 30.51922607421875,
      "activations/layer17_attention_weight_min": -25.398351669311523,
      "activations/layer18_attention_weight_max": 36.43115997314453,
      "activations/layer18_attention_weight_min": -27.396047592163086,
      "activations/layer19_attention_weight_max": 34.82905197143555,
      "activations/layer19_attention_weight_min": -32.6380615234375,
      "activations/layer1_attention_weight_max": 16.309736251831055,
      "activations/layer1_attention_weight_min": -14.503863334655762,
      "activations/layer20_attention_weight_max": 30.062759399414062,
      "activations/layer20_attention_weight_min": -25.666730880737305,
      "activations/layer21_attention_weight_max": 31.210861206054688,
      "activations/layer21_attention_weight_min": -23.993284225463867,
      "activations/layer22_attention_weight_max": 43.477298736572266,
      "activations/layer22_attention_weight_min": -32.084678649902344,
      "activations/layer23_attention_weight_max": 30.61794090270996,
      "activations/layer23_attention_weight_min": -25.717544555664062,
      "activations/layer2_attention_weight_max": 31.02350616455078,
      "activations/layer2_attention_weight_min": -31.086660385131836,
      "activations/layer3_attention_weight_max": 94.28025817871094,
      "activations/layer3_attention_weight_min": -98.82694244384766,
      "activations/layer4_attention_weight_max": 101.03952026367188,
      "activations/layer4_attention_weight_min": -108.04979705810547,
      "activations/layer5_attention_weight_max": 49.34504699707031,
      "activations/layer5_attention_weight_min": -62.34175109863281,
      "activations/layer6_attention_weight_max": 44.328243255615234,
      "activations/layer6_attention_weight_min": -46.81117248535156,
      "activations/layer7_attention_weight_max": 95.51016998291016,
      "activations/layer7_attention_weight_min": -105.16798400878906,
      "activations/layer8_attention_weight_max": 40.52874755859375,
      "activations/layer8_attention_weight_min": -46.65163040161133,
      "activations/layer9_attention_weight_max": 32.329097747802734,
      "activations/layer9_attention_weight_min": -32.69005584716797,
      "epoch": 19.42,
      "learning_rate": 2.4987499999999998e-05,
      "loss": 2.7134,
      "step": 334300
    },
    {
      "activations/layer0_attention_weight_max": 14.918560981750488,
      "activations/layer0_attention_weight_min": -12.761192321777344,
      "activations/layer10_attention_weight_max": 33.42057800292969,
      "activations/layer10_attention_weight_min": -33.95679473876953,
      "activations/layer11_attention_weight_max": 34.376686096191406,
      "activations/layer11_attention_weight_min": -34.85020065307617,
      "activations/layer12_attention_weight_max": 27.358659744262695,
      "activations/layer12_attention_weight_min": -26.652742385864258,
      "activations/layer13_attention_weight_max": 41.9788703918457,
      "activations/layer13_attention_weight_min": -36.00126266479492,
      "activations/layer14_attention_weight_max": 47.42264175415039,
      "activations/layer14_attention_weight_min": -39.061370849609375,
      "activations/layer15_attention_weight_max": 44.71629333496094,
      "activations/layer15_attention_weight_min": -37.07316589355469,
      "activations/layer16_attention_weight_max": 33.348655700683594,
      "activations/layer16_attention_weight_min": -29.872943878173828,
      "activations/layer17_attention_weight_max": 32.632102966308594,
      "activations/layer17_attention_weight_min": -28.30179786682129,
      "activations/layer18_attention_weight_max": 32.699974060058594,
      "activations/layer18_attention_weight_min": -25.528295516967773,
      "activations/layer19_attention_weight_max": 36.84317398071289,
      "activations/layer19_attention_weight_min": -33.32868576049805,
      "activations/layer1_attention_weight_max": 15.139275550842285,
      "activations/layer1_attention_weight_min": -14.166544914245605,
      "activations/layer20_attention_weight_max": 32.67728805541992,
      "activations/layer20_attention_weight_min": -26.911081314086914,
      "activations/layer21_attention_weight_max": 29.62188720703125,
      "activations/layer21_attention_weight_min": -25.6507625579834,
      "activations/layer22_attention_weight_max": 46.67937469482422,
      "activations/layer22_attention_weight_min": -33.27544021606445,
      "activations/layer23_attention_weight_max": 33.83988571166992,
      "activations/layer23_attention_weight_min": -27.21417808532715,
      "activations/layer2_attention_weight_max": 31.594879150390625,
      "activations/layer2_attention_weight_min": -30.728010177612305,
      "activations/layer3_attention_weight_max": 97.3987808227539,
      "activations/layer3_attention_weight_min": -95.64098358154297,
      "activations/layer4_attention_weight_max": 103.37761688232422,
      "activations/layer4_attention_weight_min": -110.23502349853516,
      "activations/layer5_attention_weight_max": 50.92220687866211,
      "activations/layer5_attention_weight_min": -61.395957946777344,
      "activations/layer6_attention_weight_max": 46.288856506347656,
      "activations/layer6_attention_weight_min": -47.23435974121094,
      "activations/layer7_attention_weight_max": 93.95808410644531,
      "activations/layer7_attention_weight_min": -98.90323638916016,
      "activations/layer8_attention_weight_max": 43.22499465942383,
      "activations/layer8_attention_weight_min": -44.57939529418945,
      "activations/layer9_attention_weight_max": 32.38478088378906,
      "activations/layer9_attention_weight_min": -36.339874267578125,
      "epoch": 19.43,
      "learning_rate": 2.4968560606060606e-05,
      "loss": 2.7187,
      "step": 334350
    },
    {
      "activations/layer0_attention_weight_max": 16.703184127807617,
      "activations/layer0_attention_weight_min": -13.337666511535645,
      "activations/layer10_attention_weight_max": 32.19160842895508,
      "activations/layer10_attention_weight_min": -33.28194046020508,
      "activations/layer11_attention_weight_max": 33.015220642089844,
      "activations/layer11_attention_weight_min": -33.63958740234375,
      "activations/layer12_attention_weight_max": 26.491762161254883,
      "activations/layer12_attention_weight_min": -27.467220306396484,
      "activations/layer13_attention_weight_max": 38.575286865234375,
      "activations/layer13_attention_weight_min": -36.72673416137695,
      "activations/layer14_attention_weight_max": 41.316593170166016,
      "activations/layer14_attention_weight_min": -37.14719772338867,
      "activations/layer15_attention_weight_max": 38.51384735107422,
      "activations/layer15_attention_weight_min": -34.668739318847656,
      "activations/layer16_attention_weight_max": 30.191524505615234,
      "activations/layer16_attention_weight_min": -28.468416213989258,
      "activations/layer17_attention_weight_max": 29.78268051147461,
      "activations/layer17_attention_weight_min": -27.610363006591797,
      "activations/layer18_attention_weight_max": 34.53852462768555,
      "activations/layer18_attention_weight_min": -26.726459503173828,
      "activations/layer19_attention_weight_max": 34.09489440917969,
      "activations/layer19_attention_weight_min": -32.615440368652344,
      "activations/layer1_attention_weight_max": 16.945728302001953,
      "activations/layer1_attention_weight_min": -17.011821746826172,
      "activations/layer20_attention_weight_max": 27.374366760253906,
      "activations/layer20_attention_weight_min": -24.848875045776367,
      "activations/layer21_attention_weight_max": 28.767864227294922,
      "activations/layer21_attention_weight_min": -25.36838150024414,
      "activations/layer22_attention_weight_max": 42.602745056152344,
      "activations/layer22_attention_weight_min": -33.59265899658203,
      "activations/layer23_attention_weight_max": 33.83475112915039,
      "activations/layer23_attention_weight_min": -27.23379135131836,
      "activations/layer2_attention_weight_max": 31.31873893737793,
      "activations/layer2_attention_weight_min": -31.772071838378906,
      "activations/layer3_attention_weight_max": 98.35883331298828,
      "activations/layer3_attention_weight_min": -101.05602264404297,
      "activations/layer4_attention_weight_max": 105.60314178466797,
      "activations/layer4_attention_weight_min": -107.26106262207031,
      "activations/layer5_attention_weight_max": 50.25244903564453,
      "activations/layer5_attention_weight_min": -60.818565368652344,
      "activations/layer6_attention_weight_max": 44.7459602355957,
      "activations/layer6_attention_weight_min": -48.03280258178711,
      "activations/layer7_attention_weight_max": 93.36109161376953,
      "activations/layer7_attention_weight_min": -97.958740234375,
      "activations/layer8_attention_weight_max": 41.5088996887207,
      "activations/layer8_attention_weight_min": -43.72200393676758,
      "activations/layer9_attention_weight_max": 33.55793380737305,
      "activations/layer9_attention_weight_min": -33.14921569824219,
      "epoch": 19.43,
      "learning_rate": 2.494962121212121e-05,
      "loss": 2.7323,
      "step": 334400
    },
    {
      "activations/layer0_attention_weight_max": 16.77423095703125,
      "activations/layer0_attention_weight_min": -13.085538864135742,
      "activations/layer10_attention_weight_max": 32.22612762451172,
      "activations/layer10_attention_weight_min": -33.7492561340332,
      "activations/layer11_attention_weight_max": 33.768550872802734,
      "activations/layer11_attention_weight_min": -34.636940002441406,
      "activations/layer12_attention_weight_max": 26.85283851623535,
      "activations/layer12_attention_weight_min": -25.894994735717773,
      "activations/layer13_attention_weight_max": 37.951351165771484,
      "activations/layer13_attention_weight_min": -35.18636703491211,
      "activations/layer14_attention_weight_max": 41.50104904174805,
      "activations/layer14_attention_weight_min": -38.14072036743164,
      "activations/layer15_attention_weight_max": 44.02364730834961,
      "activations/layer15_attention_weight_min": -35.57601547241211,
      "activations/layer16_attention_weight_max": 32.32936096191406,
      "activations/layer16_attention_weight_min": -26.573076248168945,
      "activations/layer17_attention_weight_max": 31.056095123291016,
      "activations/layer17_attention_weight_min": -25.894479751586914,
      "activations/layer18_attention_weight_max": 31.846410751342773,
      "activations/layer18_attention_weight_min": -27.097936630249023,
      "activations/layer19_attention_weight_max": 45.46089553833008,
      "activations/layer19_attention_weight_min": -33.773563385009766,
      "activations/layer1_attention_weight_max": 15.781460762023926,
      "activations/layer1_attention_weight_min": -15.359695434570312,
      "activations/layer20_attention_weight_max": 35.84440612792969,
      "activations/layer20_attention_weight_min": -26.81960105895996,
      "activations/layer21_attention_weight_max": 32.652339935302734,
      "activations/layer21_attention_weight_min": -25.16049575805664,
      "activations/layer22_attention_weight_max": 44.05437469482422,
      "activations/layer22_attention_weight_min": -32.85590362548828,
      "activations/layer23_attention_weight_max": 35.64768981933594,
      "activations/layer23_attention_weight_min": -25.747299194335938,
      "activations/layer2_attention_weight_max": 32.010223388671875,
      "activations/layer2_attention_weight_min": -30.87350082397461,
      "activations/layer3_attention_weight_max": 96.11141967773438,
      "activations/layer3_attention_weight_min": -96.9865493774414,
      "activations/layer4_attention_weight_max": 103.62481689453125,
      "activations/layer4_attention_weight_min": -106.02301025390625,
      "activations/layer5_attention_weight_max": 49.49106979370117,
      "activations/layer5_attention_weight_min": -61.31634521484375,
      "activations/layer6_attention_weight_max": 42.34003448486328,
      "activations/layer6_attention_weight_min": -47.10618591308594,
      "activations/layer7_attention_weight_max": 94.09916687011719,
      "activations/layer7_attention_weight_min": -92.91603088378906,
      "activations/layer8_attention_weight_max": 37.83139419555664,
      "activations/layer8_attention_weight_min": -41.914512634277344,
      "activations/layer9_attention_weight_max": 29.385923385620117,
      "activations/layer9_attention_weight_min": -33.02634811401367,
      "epoch": 19.43,
      "learning_rate": 2.493068181818182e-05,
      "loss": 2.7324,
      "step": 334450
    },
    {
      "activations/layer0_attention_weight_max": 16.102590560913086,
      "activations/layer0_attention_weight_min": -12.070948600769043,
      "activations/layer10_attention_weight_max": 32.18680953979492,
      "activations/layer10_attention_weight_min": -31.901235580444336,
      "activations/layer11_attention_weight_max": 33.09172821044922,
      "activations/layer11_attention_weight_min": -33.43006896972656,
      "activations/layer12_attention_weight_max": 27.18623161315918,
      "activations/layer12_attention_weight_min": -26.214927673339844,
      "activations/layer13_attention_weight_max": 38.40498352050781,
      "activations/layer13_attention_weight_min": -34.837196350097656,
      "activations/layer14_attention_weight_max": 42.58157730102539,
      "activations/layer14_attention_weight_min": -38.799503326416016,
      "activations/layer15_attention_weight_max": 38.02224349975586,
      "activations/layer15_attention_weight_min": -39.753482818603516,
      "activations/layer16_attention_weight_max": 28.160097122192383,
      "activations/layer16_attention_weight_min": -29.29307746887207,
      "activations/layer17_attention_weight_max": 27.283784866333008,
      "activations/layer17_attention_weight_min": -26.409326553344727,
      "activations/layer18_attention_weight_max": 30.139949798583984,
      "activations/layer18_attention_weight_min": -27.083255767822266,
      "activations/layer19_attention_weight_max": 32.35559844970703,
      "activations/layer19_attention_weight_min": -32.29421615600586,
      "activations/layer1_attention_weight_max": 15.592164039611816,
      "activations/layer1_attention_weight_min": -14.940603256225586,
      "activations/layer20_attention_weight_max": 27.356201171875,
      "activations/layer20_attention_weight_min": -26.271623611450195,
      "activations/layer21_attention_weight_max": 28.40629005432129,
      "activations/layer21_attention_weight_min": -24.86728286743164,
      "activations/layer22_attention_weight_max": 41.412315368652344,
      "activations/layer22_attention_weight_min": -36.375022888183594,
      "activations/layer23_attention_weight_max": 35.918312072753906,
      "activations/layer23_attention_weight_min": -27.43330192565918,
      "activations/layer2_attention_weight_max": 32.038124084472656,
      "activations/layer2_attention_weight_min": -30.034446716308594,
      "activations/layer3_attention_weight_max": 99.11898040771484,
      "activations/layer3_attention_weight_min": -95.8506088256836,
      "activations/layer4_attention_weight_max": 108.73287200927734,
      "activations/layer4_attention_weight_min": -105.6927719116211,
      "activations/layer5_attention_weight_max": 53.32487487792969,
      "activations/layer5_attention_weight_min": -65.25196075439453,
      "activations/layer6_attention_weight_max": 42.6505241394043,
      "activations/layer6_attention_weight_min": -45.6556510925293,
      "activations/layer7_attention_weight_max": 93.10594940185547,
      "activations/layer7_attention_weight_min": -95.37734985351562,
      "activations/layer8_attention_weight_max": 39.51957702636719,
      "activations/layer8_attention_weight_min": -41.97554397583008,
      "activations/layer9_attention_weight_max": 31.62810707092285,
      "activations/layer9_attention_weight_min": -31.856122970581055,
      "epoch": 19.44,
      "learning_rate": 2.491174242424242e-05,
      "loss": 2.7321,
      "step": 334500
    },
    {
      "activations/layer0_attention_weight_max": 16.387208938598633,
      "activations/layer0_attention_weight_min": -14.628390312194824,
      "activations/layer10_attention_weight_max": 34.6152458190918,
      "activations/layer10_attention_weight_min": -34.65477752685547,
      "activations/layer11_attention_weight_max": 34.70536422729492,
      "activations/layer11_attention_weight_min": -35.032814025878906,
      "activations/layer12_attention_weight_max": 29.458271026611328,
      "activations/layer12_attention_weight_min": -28.2636775970459,
      "activations/layer13_attention_weight_max": 40.51097869873047,
      "activations/layer13_attention_weight_min": -36.956295013427734,
      "activations/layer14_attention_weight_max": 44.9715461730957,
      "activations/layer14_attention_weight_min": -38.492706298828125,
      "activations/layer15_attention_weight_max": 41.24260330200195,
      "activations/layer15_attention_weight_min": -36.51106262207031,
      "activations/layer16_attention_weight_max": 30.726993560791016,
      "activations/layer16_attention_weight_min": -28.5460262298584,
      "activations/layer17_attention_weight_max": 31.169477462768555,
      "activations/layer17_attention_weight_min": -28.76023292541504,
      "activations/layer18_attention_weight_max": 31.27956771850586,
      "activations/layer18_attention_weight_min": -28.26417350769043,
      "activations/layer19_attention_weight_max": 34.8951301574707,
      "activations/layer19_attention_weight_min": -37.1234245300293,
      "activations/layer1_attention_weight_max": 16.072193145751953,
      "activations/layer1_attention_weight_min": -16.039602279663086,
      "activations/layer20_attention_weight_max": 31.66014289855957,
      "activations/layer20_attention_weight_min": -28.805368423461914,
      "activations/layer21_attention_weight_max": 29.399810791015625,
      "activations/layer21_attention_weight_min": -28.272153854370117,
      "activations/layer22_attention_weight_max": 44.697471618652344,
      "activations/layer22_attention_weight_min": -34.650020599365234,
      "activations/layer23_attention_weight_max": 35.476219177246094,
      "activations/layer23_attention_weight_min": -27.406700134277344,
      "activations/layer2_attention_weight_max": 31.935178756713867,
      "activations/layer2_attention_weight_min": -30.933990478515625,
      "activations/layer3_attention_weight_max": 100.25272369384766,
      "activations/layer3_attention_weight_min": -98.56378936767578,
      "activations/layer4_attention_weight_max": 102.9040756225586,
      "activations/layer4_attention_weight_min": -106.4180679321289,
      "activations/layer5_attention_weight_max": 52.344234466552734,
      "activations/layer5_attention_weight_min": -60.673683166503906,
      "activations/layer6_attention_weight_max": 44.350624084472656,
      "activations/layer6_attention_weight_min": -46.739200592041016,
      "activations/layer7_attention_weight_max": 94.33275604248047,
      "activations/layer7_attention_weight_min": -97.03500366210938,
      "activations/layer8_attention_weight_max": 41.95561218261719,
      "activations/layer8_attention_weight_min": -43.06890106201172,
      "activations/layer9_attention_weight_max": 31.362720489501953,
      "activations/layer9_attention_weight_min": -34.89741134643555,
      "epoch": 19.44,
      "learning_rate": 2.489280303030303e-05,
      "loss": 2.7147,
      "step": 334550
    },
    {
      "activations/layer0_attention_weight_max": 15.984786987304688,
      "activations/layer0_attention_weight_min": -12.94328498840332,
      "activations/layer10_attention_weight_max": 35.23223876953125,
      "activations/layer10_attention_weight_min": -36.700923919677734,
      "activations/layer11_attention_weight_max": 36.95520782470703,
      "activations/layer11_attention_weight_min": -36.76426696777344,
      "activations/layer12_attention_weight_max": 27.81454086303711,
      "activations/layer12_attention_weight_min": -27.877277374267578,
      "activations/layer13_attention_weight_max": 45.196468353271484,
      "activations/layer13_attention_weight_min": -37.01947784423828,
      "activations/layer14_attention_weight_max": 48.589996337890625,
      "activations/layer14_attention_weight_min": -39.58689498901367,
      "activations/layer15_attention_weight_max": 42.73261642456055,
      "activations/layer15_attention_weight_min": -37.43876647949219,
      "activations/layer16_attention_weight_max": 32.28628158569336,
      "activations/layer16_attention_weight_min": -27.854990005493164,
      "activations/layer17_attention_weight_max": 30.5832462310791,
      "activations/layer17_attention_weight_min": -25.477598190307617,
      "activations/layer18_attention_weight_max": 31.951086044311523,
      "activations/layer18_attention_weight_min": -26.15256118774414,
      "activations/layer19_attention_weight_max": 32.24769592285156,
      "activations/layer19_attention_weight_min": -30.671838760375977,
      "activations/layer1_attention_weight_max": 16.541915893554688,
      "activations/layer1_attention_weight_min": -16.068519592285156,
      "activations/layer20_attention_weight_max": 27.484241485595703,
      "activations/layer20_attention_weight_min": -25.69249725341797,
      "activations/layer21_attention_weight_max": 28.118331909179688,
      "activations/layer21_attention_weight_min": -24.908477783203125,
      "activations/layer22_attention_weight_max": 45.074039459228516,
      "activations/layer22_attention_weight_min": -31.88214111328125,
      "activations/layer23_attention_weight_max": 33.53334426879883,
      "activations/layer23_attention_weight_min": -26.361373901367188,
      "activations/layer2_attention_weight_max": 33.48548126220703,
      "activations/layer2_attention_weight_min": -29.47873306274414,
      "activations/layer3_attention_weight_max": 95.26720428466797,
      "activations/layer3_attention_weight_min": -97.28978729248047,
      "activations/layer4_attention_weight_max": 103.39517974853516,
      "activations/layer4_attention_weight_min": -102.56078338623047,
      "activations/layer5_attention_weight_max": 50.849693298339844,
      "activations/layer5_attention_weight_min": -61.106666564941406,
      "activations/layer6_attention_weight_max": 42.212608337402344,
      "activations/layer6_attention_weight_min": -46.73722839355469,
      "activations/layer7_attention_weight_max": 93.26605987548828,
      "activations/layer7_attention_weight_min": -96.88093566894531,
      "activations/layer8_attention_weight_max": 41.53254318237305,
      "activations/layer8_attention_weight_min": -45.93107604980469,
      "activations/layer9_attention_weight_max": 33.21114730834961,
      "activations/layer9_attention_weight_min": -34.52935028076172,
      "epoch": 19.44,
      "learning_rate": 2.4873863636363634e-05,
      "loss": 2.7335,
      "step": 334600
    },
    {
      "activations/layer0_attention_weight_max": 16.532873153686523,
      "activations/layer0_attention_weight_min": -12.193881034851074,
      "activations/layer10_attention_weight_max": 33.32761764526367,
      "activations/layer10_attention_weight_min": -33.61225128173828,
      "activations/layer11_attention_weight_max": 32.561065673828125,
      "activations/layer11_attention_weight_min": -34.60710525512695,
      "activations/layer12_attention_weight_max": 25.105466842651367,
      "activations/layer12_attention_weight_min": -27.136009216308594,
      "activations/layer13_attention_weight_max": 36.06858444213867,
      "activations/layer13_attention_weight_min": -36.029685974121094,
      "activations/layer14_attention_weight_max": 40.55998229980469,
      "activations/layer14_attention_weight_min": -37.371009826660156,
      "activations/layer15_attention_weight_max": 36.65132522583008,
      "activations/layer15_attention_weight_min": -34.57422637939453,
      "activations/layer16_attention_weight_max": 26.858369827270508,
      "activations/layer16_attention_weight_min": -29.17653465270996,
      "activations/layer17_attention_weight_max": 29.30160903930664,
      "activations/layer17_attention_weight_min": -26.59053611755371,
      "activations/layer18_attention_weight_max": 32.90309524536133,
      "activations/layer18_attention_weight_min": -27.500261306762695,
      "activations/layer19_attention_weight_max": 34.584327697753906,
      "activations/layer19_attention_weight_min": -32.084957122802734,
      "activations/layer1_attention_weight_max": 15.553911209106445,
      "activations/layer1_attention_weight_min": -14.063545227050781,
      "activations/layer20_attention_weight_max": 28.98162841796875,
      "activations/layer20_attention_weight_min": -26.75405502319336,
      "activations/layer21_attention_weight_max": 27.915483474731445,
      "activations/layer21_attention_weight_min": -24.89544105529785,
      "activations/layer22_attention_weight_max": 41.90398025512695,
      "activations/layer22_attention_weight_min": -31.962522506713867,
      "activations/layer23_attention_weight_max": 31.836746215820312,
      "activations/layer23_attention_weight_min": -25.563535690307617,
      "activations/layer2_attention_weight_max": 29.91103744506836,
      "activations/layer2_attention_weight_min": -31.627857208251953,
      "activations/layer3_attention_weight_max": 96.1731948852539,
      "activations/layer3_attention_weight_min": -95.8614730834961,
      "activations/layer4_attention_weight_max": 102.97945404052734,
      "activations/layer4_attention_weight_min": -106.34252166748047,
      "activations/layer5_attention_weight_max": 51.69009017944336,
      "activations/layer5_attention_weight_min": -61.79322814941406,
      "activations/layer6_attention_weight_max": 44.00503921508789,
      "activations/layer6_attention_weight_min": -49.77664566040039,
      "activations/layer7_attention_weight_max": 95.16968536376953,
      "activations/layer7_attention_weight_min": -102.45046997070312,
      "activations/layer8_attention_weight_max": 40.995540618896484,
      "activations/layer8_attention_weight_min": -45.50410461425781,
      "activations/layer9_attention_weight_max": 30.774246215820312,
      "activations/layer9_attention_weight_min": -33.88805389404297,
      "epoch": 19.45,
      "learning_rate": 2.4854924242424242e-05,
      "loss": 2.7419,
      "step": 334650
    },
    {
      "activations/layer0_attention_weight_max": 17.107908248901367,
      "activations/layer0_attention_weight_min": -12.097831726074219,
      "activations/layer10_attention_weight_max": 34.64912796020508,
      "activations/layer10_attention_weight_min": -37.17224884033203,
      "activations/layer11_attention_weight_max": 33.05595397949219,
      "activations/layer11_attention_weight_min": -35.183982849121094,
      "activations/layer12_attention_weight_max": 27.360933303833008,
      "activations/layer12_attention_weight_min": -27.316118240356445,
      "activations/layer13_attention_weight_max": 40.2760009765625,
      "activations/layer13_attention_weight_min": -36.88739776611328,
      "activations/layer14_attention_weight_max": 43.49317932128906,
      "activations/layer14_attention_weight_min": -38.30251693725586,
      "activations/layer15_attention_weight_max": 39.18773651123047,
      "activations/layer15_attention_weight_min": -36.788394927978516,
      "activations/layer16_attention_weight_max": 29.55297088623047,
      "activations/layer16_attention_weight_min": -28.523983001708984,
      "activations/layer17_attention_weight_max": 31.491729736328125,
      "activations/layer17_attention_weight_min": -28.144201278686523,
      "activations/layer18_attention_weight_max": 33.937110900878906,
      "activations/layer18_attention_weight_min": -26.326881408691406,
      "activations/layer19_attention_weight_max": 35.16870880126953,
      "activations/layer19_attention_weight_min": -31.82322120666504,
      "activations/layer1_attention_weight_max": 15.403410911560059,
      "activations/layer1_attention_weight_min": -13.91514778137207,
      "activations/layer20_attention_weight_max": 30.484006881713867,
      "activations/layer20_attention_weight_min": -26.22378158569336,
      "activations/layer21_attention_weight_max": 28.48978614807129,
      "activations/layer21_attention_weight_min": -26.139720916748047,
      "activations/layer22_attention_weight_max": 42.66554641723633,
      "activations/layer22_attention_weight_min": -33.119319915771484,
      "activations/layer23_attention_weight_max": 34.059425354003906,
      "activations/layer23_attention_weight_min": -26.372928619384766,
      "activations/layer2_attention_weight_max": 31.209678649902344,
      "activations/layer2_attention_weight_min": -30.692928314208984,
      "activations/layer3_attention_weight_max": 96.56244659423828,
      "activations/layer3_attention_weight_min": -97.4545669555664,
      "activations/layer4_attention_weight_max": 103.6161117553711,
      "activations/layer4_attention_weight_min": -106.82640838623047,
      "activations/layer5_attention_weight_max": 48.91959762573242,
      "activations/layer5_attention_weight_min": -65.84196472167969,
      "activations/layer6_attention_weight_max": 45.38998031616211,
      "activations/layer6_attention_weight_min": -47.835533142089844,
      "activations/layer7_attention_weight_max": 95.31922912597656,
      "activations/layer7_attention_weight_min": -103.78480529785156,
      "activations/layer8_attention_weight_max": 41.27216720581055,
      "activations/layer8_attention_weight_min": -45.8213005065918,
      "activations/layer9_attention_weight_max": 33.65729522705078,
      "activations/layer9_attention_weight_min": -36.10832595825195,
      "epoch": 19.45,
      "learning_rate": 2.4835984848484847e-05,
      "loss": 2.723,
      "step": 334700
    },
    {
      "activations/layer0_attention_weight_max": 16.06173324584961,
      "activations/layer0_attention_weight_min": -13.715365409851074,
      "activations/layer10_attention_weight_max": 38.036041259765625,
      "activations/layer10_attention_weight_min": -37.56856918334961,
      "activations/layer11_attention_weight_max": 38.944393157958984,
      "activations/layer11_attention_weight_min": -36.79472732543945,
      "activations/layer12_attention_weight_max": 28.818790435791016,
      "activations/layer12_attention_weight_min": -26.913715362548828,
      "activations/layer13_attention_weight_max": 43.708431243896484,
      "activations/layer13_attention_weight_min": -36.31665802001953,
      "activations/layer14_attention_weight_max": 47.46622085571289,
      "activations/layer14_attention_weight_min": -37.590370178222656,
      "activations/layer15_attention_weight_max": 45.811824798583984,
      "activations/layer15_attention_weight_min": -34.87745666503906,
      "activations/layer16_attention_weight_max": 31.737720489501953,
      "activations/layer16_attention_weight_min": -28.918777465820312,
      "activations/layer17_attention_weight_max": 31.88831901550293,
      "activations/layer17_attention_weight_min": -25.86678695678711,
      "activations/layer18_attention_weight_max": 34.12142562866211,
      "activations/layer18_attention_weight_min": -25.6393985748291,
      "activations/layer19_attention_weight_max": 33.88053512573242,
      "activations/layer19_attention_weight_min": -29.88677978515625,
      "activations/layer1_attention_weight_max": 15.45266342163086,
      "activations/layer1_attention_weight_min": -15.250611305236816,
      "activations/layer20_attention_weight_max": 31.502002716064453,
      "activations/layer20_attention_weight_min": -25.33490562438965,
      "activations/layer21_attention_weight_max": 31.302059173583984,
      "activations/layer21_attention_weight_min": -23.287099838256836,
      "activations/layer22_attention_weight_max": 47.32401657104492,
      "activations/layer22_attention_weight_min": -34.790443420410156,
      "activations/layer23_attention_weight_max": 32.9097900390625,
      "activations/layer23_attention_weight_min": -26.093006134033203,
      "activations/layer2_attention_weight_max": 32.00847625732422,
      "activations/layer2_attention_weight_min": -31.91352081298828,
      "activations/layer3_attention_weight_max": 103.58634185791016,
      "activations/layer3_attention_weight_min": -103.12642669677734,
      "activations/layer4_attention_weight_max": 112.73065185546875,
      "activations/layer4_attention_weight_min": -112.71058654785156,
      "activations/layer5_attention_weight_max": 52.83034896850586,
      "activations/layer5_attention_weight_min": -65.68492126464844,
      "activations/layer6_attention_weight_max": 46.38856506347656,
      "activations/layer6_attention_weight_min": -52.091087341308594,
      "activations/layer7_attention_weight_max": 101.5374755859375,
      "activations/layer7_attention_weight_min": -109.21129608154297,
      "activations/layer8_attention_weight_max": 45.5212287902832,
      "activations/layer8_attention_weight_min": -51.166046142578125,
      "activations/layer9_attention_weight_max": 35.148860931396484,
      "activations/layer9_attention_weight_min": -36.780113220214844,
      "epoch": 19.45,
      "learning_rate": 2.4817045454545455e-05,
      "loss": 2.7354,
      "step": 334750
    },
    {
      "activations/layer0_attention_weight_max": 15.846511840820312,
      "activations/layer0_attention_weight_min": -13.1666898727417,
      "activations/layer10_attention_weight_max": 36.83113479614258,
      "activations/layer10_attention_weight_min": -37.15961456298828,
      "activations/layer11_attention_weight_max": 37.65160369873047,
      "activations/layer11_attention_weight_min": -34.44619369506836,
      "activations/layer12_attention_weight_max": 25.90353012084961,
      "activations/layer12_attention_weight_min": -25.807462692260742,
      "activations/layer13_attention_weight_max": 37.259212493896484,
      "activations/layer13_attention_weight_min": -34.708465576171875,
      "activations/layer14_attention_weight_max": 40.44424819946289,
      "activations/layer14_attention_weight_min": -35.797760009765625,
      "activations/layer15_attention_weight_max": 38.08037185668945,
      "activations/layer15_attention_weight_min": -34.62327194213867,
      "activations/layer16_attention_weight_max": 29.71042251586914,
      "activations/layer16_attention_weight_min": -27.210832595825195,
      "activations/layer17_attention_weight_max": 28.06420135498047,
      "activations/layer17_attention_weight_min": -25.199058532714844,
      "activations/layer18_attention_weight_max": 32.72901153564453,
      "activations/layer18_attention_weight_min": -26.259279251098633,
      "activations/layer19_attention_weight_max": 31.999465942382812,
      "activations/layer19_attention_weight_min": -30.15983009338379,
      "activations/layer1_attention_weight_max": 15.573118209838867,
      "activations/layer1_attention_weight_min": -14.053756713867188,
      "activations/layer20_attention_weight_max": 28.0013370513916,
      "activations/layer20_attention_weight_min": -25.044925689697266,
      "activations/layer21_attention_weight_max": 28.251110076904297,
      "activations/layer21_attention_weight_min": -23.792104721069336,
      "activations/layer22_attention_weight_max": 40.15530776977539,
      "activations/layer22_attention_weight_min": -30.887001037597656,
      "activations/layer23_attention_weight_max": 32.56751251220703,
      "activations/layer23_attention_weight_min": -26.230655670166016,
      "activations/layer2_attention_weight_max": 32.955528259277344,
      "activations/layer2_attention_weight_min": -30.652189254760742,
      "activations/layer3_attention_weight_max": 100.70153045654297,
      "activations/layer3_attention_weight_min": -96.60074615478516,
      "activations/layer4_attention_weight_max": 106.65023040771484,
      "activations/layer4_attention_weight_min": -106.5572280883789,
      "activations/layer5_attention_weight_max": 51.12543487548828,
      "activations/layer5_attention_weight_min": -62.62281036376953,
      "activations/layer6_attention_weight_max": 45.89645004272461,
      "activations/layer6_attention_weight_min": -48.21934127807617,
      "activations/layer7_attention_weight_max": 94.54322052001953,
      "activations/layer7_attention_weight_min": -105.88715362548828,
      "activations/layer8_attention_weight_max": 43.22785568237305,
      "activations/layer8_attention_weight_min": -43.9854621887207,
      "activations/layer9_attention_weight_max": 34.38450241088867,
      "activations/layer9_attention_weight_min": -35.708675384521484,
      "epoch": 19.45,
      "learning_rate": 2.4798106060606056e-05,
      "loss": 2.7245,
      "step": 334800
    },
    {
      "activations/layer0_attention_weight_max": 16.74666404724121,
      "activations/layer0_attention_weight_min": -12.74055290222168,
      "activations/layer10_attention_weight_max": 34.06642532348633,
      "activations/layer10_attention_weight_min": -36.68709182739258,
      "activations/layer11_attention_weight_max": 34.6768913269043,
      "activations/layer11_attention_weight_min": -36.752593994140625,
      "activations/layer12_attention_weight_max": 25.258378982543945,
      "activations/layer12_attention_weight_min": -27.298460006713867,
      "activations/layer13_attention_weight_max": 38.407142639160156,
      "activations/layer13_attention_weight_min": -38.27924728393555,
      "activations/layer14_attention_weight_max": 41.14918899536133,
      "activations/layer14_attention_weight_min": -42.54765701293945,
      "activations/layer15_attention_weight_max": 38.59400177001953,
      "activations/layer15_attention_weight_min": -41.585662841796875,
      "activations/layer16_attention_weight_max": 27.838565826416016,
      "activations/layer16_attention_weight_min": -27.601282119750977,
      "activations/layer17_attention_weight_max": 27.82977867126465,
      "activations/layer17_attention_weight_min": -25.304819107055664,
      "activations/layer18_attention_weight_max": 28.538799285888672,
      "activations/layer18_attention_weight_min": -25.197864532470703,
      "activations/layer19_attention_weight_max": 31.528156280517578,
      "activations/layer19_attention_weight_min": -32.13703155517578,
      "activations/layer1_attention_weight_max": 14.784858703613281,
      "activations/layer1_attention_weight_min": -14.158187866210938,
      "activations/layer20_attention_weight_max": 28.347183227539062,
      "activations/layer20_attention_weight_min": -24.933250427246094,
      "activations/layer21_attention_weight_max": 26.25822639465332,
      "activations/layer21_attention_weight_min": -22.842130661010742,
      "activations/layer22_attention_weight_max": 39.674461364746094,
      "activations/layer22_attention_weight_min": -28.82154655456543,
      "activations/layer23_attention_weight_max": 28.94786834716797,
      "activations/layer23_attention_weight_min": -24.834989547729492,
      "activations/layer2_attention_weight_max": 32.685569763183594,
      "activations/layer2_attention_weight_min": -30.115985870361328,
      "activations/layer3_attention_weight_max": 98.60332489013672,
      "activations/layer3_attention_weight_min": -100.12074279785156,
      "activations/layer4_attention_weight_max": 101.81807708740234,
      "activations/layer4_attention_weight_min": -107.503173828125,
      "activations/layer5_attention_weight_max": 51.609657287597656,
      "activations/layer5_attention_weight_min": -60.830806732177734,
      "activations/layer6_attention_weight_max": 43.17842102050781,
      "activations/layer6_attention_weight_min": -46.1602897644043,
      "activations/layer7_attention_weight_max": 90.35688781738281,
      "activations/layer7_attention_weight_min": -95.38317108154297,
      "activations/layer8_attention_weight_max": 38.44612503051758,
      "activations/layer8_attention_weight_min": -44.220882415771484,
      "activations/layer9_attention_weight_max": 31.82791519165039,
      "activations/layer9_attention_weight_min": -34.13251495361328,
      "epoch": 19.46,
      "learning_rate": 2.4779166666666665e-05,
      "loss": 2.7191,
      "step": 334850
    },
    {
      "activations/layer0_attention_weight_max": 17.061222076416016,
      "activations/layer0_attention_weight_min": -13.676263809204102,
      "activations/layer10_attention_weight_max": 34.81404113769531,
      "activations/layer10_attention_weight_min": -35.31788635253906,
      "activations/layer11_attention_weight_max": 33.864601135253906,
      "activations/layer11_attention_weight_min": -37.15019989013672,
      "activations/layer12_attention_weight_max": 26.535863876342773,
      "activations/layer12_attention_weight_min": -25.642990112304688,
      "activations/layer13_attention_weight_max": 37.975852966308594,
      "activations/layer13_attention_weight_min": -34.20155715942383,
      "activations/layer14_attention_weight_max": 38.656463623046875,
      "activations/layer14_attention_weight_min": -36.374122619628906,
      "activations/layer15_attention_weight_max": 34.059181213378906,
      "activations/layer15_attention_weight_min": -34.67824172973633,
      "activations/layer16_attention_weight_max": 26.92816925048828,
      "activations/layer16_attention_weight_min": -26.166440963745117,
      "activations/layer17_attention_weight_max": 28.098398208618164,
      "activations/layer17_attention_weight_min": -23.888805389404297,
      "activations/layer18_attention_weight_max": 30.609638214111328,
      "activations/layer18_attention_weight_min": -23.973899841308594,
      "activations/layer19_attention_weight_max": 32.02839660644531,
      "activations/layer19_attention_weight_min": -29.986875534057617,
      "activations/layer1_attention_weight_max": 16.016836166381836,
      "activations/layer1_attention_weight_min": -15.220893859863281,
      "activations/layer20_attention_weight_max": 27.708499908447266,
      "activations/layer20_attention_weight_min": -24.516155242919922,
      "activations/layer21_attention_weight_max": 26.10538101196289,
      "activations/layer21_attention_weight_min": -23.011247634887695,
      "activations/layer22_attention_weight_max": 39.08932113647461,
      "activations/layer22_attention_weight_min": -30.612417221069336,
      "activations/layer23_attention_weight_max": 31.896236419677734,
      "activations/layer23_attention_weight_min": -25.606342315673828,
      "activations/layer2_attention_weight_max": 32.55847930908203,
      "activations/layer2_attention_weight_min": -31.233325958251953,
      "activations/layer3_attention_weight_max": 99.25325775146484,
      "activations/layer3_attention_weight_min": -97.41156005859375,
      "activations/layer4_attention_weight_max": 100.93680572509766,
      "activations/layer4_attention_weight_min": -102.75225830078125,
      "activations/layer5_attention_weight_max": 51.316741943359375,
      "activations/layer5_attention_weight_min": -61.087425231933594,
      "activations/layer6_attention_weight_max": 42.390464782714844,
      "activations/layer6_attention_weight_min": -45.660728454589844,
      "activations/layer7_attention_weight_max": 88.57478332519531,
      "activations/layer7_attention_weight_min": -93.87039184570312,
      "activations/layer8_attention_weight_max": 38.367637634277344,
      "activations/layer8_attention_weight_min": -44.64708709716797,
      "activations/layer9_attention_weight_max": 31.951801300048828,
      "activations/layer9_attention_weight_min": -34.78872299194336,
      "epoch": 19.46,
      "learning_rate": 2.476022727272727e-05,
      "loss": 2.714,
      "step": 334900
    },
    {
      "activations/layer0_attention_weight_max": 16.40038299560547,
      "activations/layer0_attention_weight_min": -12.887198448181152,
      "activations/layer10_attention_weight_max": 32.20880126953125,
      "activations/layer10_attention_weight_min": -33.54084014892578,
      "activations/layer11_attention_weight_max": 31.57001495361328,
      "activations/layer11_attention_weight_min": -33.56353759765625,
      "activations/layer12_attention_weight_max": 26.040925979614258,
      "activations/layer12_attention_weight_min": -25.256750106811523,
      "activations/layer13_attention_weight_max": 38.63956832885742,
      "activations/layer13_attention_weight_min": -34.32635498046875,
      "activations/layer14_attention_weight_max": 40.49976348876953,
      "activations/layer14_attention_weight_min": -37.47972106933594,
      "activations/layer15_attention_weight_max": 35.325992584228516,
      "activations/layer15_attention_weight_min": -33.20133590698242,
      "activations/layer16_attention_weight_max": 29.00417709350586,
      "activations/layer16_attention_weight_min": -30.09446144104004,
      "activations/layer17_attention_weight_max": 31.263832092285156,
      "activations/layer17_attention_weight_min": -26.818077087402344,
      "activations/layer18_attention_weight_max": 30.562606811523438,
      "activations/layer18_attention_weight_min": -25.475175857543945,
      "activations/layer19_attention_weight_max": 35.9796142578125,
      "activations/layer19_attention_weight_min": -31.347213745117188,
      "activations/layer1_attention_weight_max": 15.816207885742188,
      "activations/layer1_attention_weight_min": -14.720903396606445,
      "activations/layer20_attention_weight_max": 29.241899490356445,
      "activations/layer20_attention_weight_min": -26.61955451965332,
      "activations/layer21_attention_weight_max": 28.05190658569336,
      "activations/layer21_attention_weight_min": -25.900466918945312,
      "activations/layer22_attention_weight_max": 44.627647399902344,
      "activations/layer22_attention_weight_min": -31.594730377197266,
      "activations/layer23_attention_weight_max": 33.83381271362305,
      "activations/layer23_attention_weight_min": -27.175838470458984,
      "activations/layer2_attention_weight_max": 31.498931884765625,
      "activations/layer2_attention_weight_min": -32.342716217041016,
      "activations/layer3_attention_weight_max": 100.71414184570312,
      "activations/layer3_attention_weight_min": -100.43002319335938,
      "activations/layer4_attention_weight_max": 106.65882873535156,
      "activations/layer4_attention_weight_min": -108.64332580566406,
      "activations/layer5_attention_weight_max": 50.812644958496094,
      "activations/layer5_attention_weight_min": -64.84355163574219,
      "activations/layer6_attention_weight_max": 45.259944915771484,
      "activations/layer6_attention_weight_min": -45.158451080322266,
      "activations/layer7_attention_weight_max": 95.54100036621094,
      "activations/layer7_attention_weight_min": -106.27576446533203,
      "activations/layer8_attention_weight_max": 42.48054885864258,
      "activations/layer8_attention_weight_min": -43.816925048828125,
      "activations/layer9_attention_weight_max": 30.968252182006836,
      "activations/layer9_attention_weight_min": -35.21152877807617,
      "epoch": 19.46,
      "learning_rate": 2.4741287878787878e-05,
      "loss": 2.7133,
      "step": 334950
    },
    {
      "activations/layer0_attention_weight_max": 16.0625057220459,
      "activations/layer0_attention_weight_min": -13.864151000976562,
      "activations/layer10_attention_weight_max": 35.33699035644531,
      "activations/layer10_attention_weight_min": -35.57093811035156,
      "activations/layer11_attention_weight_max": 33.76871871948242,
      "activations/layer11_attention_weight_min": -35.201786041259766,
      "activations/layer12_attention_weight_max": 27.547704696655273,
      "activations/layer12_attention_weight_min": -27.242088317871094,
      "activations/layer13_attention_weight_max": 42.4875373840332,
      "activations/layer13_attention_weight_min": -35.63587188720703,
      "activations/layer14_attention_weight_max": 42.396949768066406,
      "activations/layer14_attention_weight_min": -39.115806579589844,
      "activations/layer15_attention_weight_max": 39.38615036010742,
      "activations/layer15_attention_weight_min": -36.76720428466797,
      "activations/layer16_attention_weight_max": 32.044151306152344,
      "activations/layer16_attention_weight_min": -28.899959564208984,
      "activations/layer17_attention_weight_max": 29.990739822387695,
      "activations/layer17_attention_weight_min": -27.58959197998047,
      "activations/layer18_attention_weight_max": 30.905614852905273,
      "activations/layer18_attention_weight_min": -25.350830078125,
      "activations/layer19_attention_weight_max": 35.29233932495117,
      "activations/layer19_attention_weight_min": -31.381561279296875,
      "activations/layer1_attention_weight_max": 15.360184669494629,
      "activations/layer1_attention_weight_min": -13.753077507019043,
      "activations/layer20_attention_weight_max": 30.445371627807617,
      "activations/layer20_attention_weight_min": -25.4812068939209,
      "activations/layer21_attention_weight_max": 29.960281372070312,
      "activations/layer21_attention_weight_min": -25.348766326904297,
      "activations/layer22_attention_weight_max": 46.00527572631836,
      "activations/layer22_attention_weight_min": -36.0989875793457,
      "activations/layer23_attention_weight_max": 35.44915771484375,
      "activations/layer23_attention_weight_min": -27.807279586791992,
      "activations/layer2_attention_weight_max": 32.66862106323242,
      "activations/layer2_attention_weight_min": -31.45795440673828,
      "activations/layer3_attention_weight_max": 101.63448333740234,
      "activations/layer3_attention_weight_min": -101.20560455322266,
      "activations/layer4_attention_weight_max": 106.6589584350586,
      "activations/layer4_attention_weight_min": -115.44474029541016,
      "activations/layer5_attention_weight_max": 52.46946334838867,
      "activations/layer5_attention_weight_min": -63.006591796875,
      "activations/layer6_attention_weight_max": 45.42280960083008,
      "activations/layer6_attention_weight_min": -48.74561309814453,
      "activations/layer7_attention_weight_max": 99.40117645263672,
      "activations/layer7_attention_weight_min": -97.63119506835938,
      "activations/layer8_attention_weight_max": 42.39400863647461,
      "activations/layer8_attention_weight_min": -45.4249267578125,
      "activations/layer9_attention_weight_max": 33.921791076660156,
      "activations/layer9_attention_weight_min": -35.58369064331055,
      "epoch": 19.47,
      "learning_rate": 2.472234848484848e-05,
      "loss": 2.7162,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4531,
      "eval_samples_per_second": 507.98,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4531,
      "eval_openwebtext_samples_per_second": 507.98,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.953,
      "eval_wikitext_samples_per_second": 233.493,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_lambada_loss": 2.341796875,
      "eval_lambada_ppl": 10.399907116482579,
      "eval_lambada_runtime": 9.547,
      "eval_lambada_samples_per_second": 510.005,
      "step": 335000
    },
    {
      "activations/layer0_attention_weight_max": 16.788835525512695,
      "activations/layer0_attention_weight_min": -14.266969680786133,
      "activations/layer10_attention_weight_max": 41.83905029296875,
      "activations/layer10_attention_weight_min": -38.466651916503906,
      "activations/layer11_attention_weight_max": 40.425384521484375,
      "activations/layer11_attention_weight_min": -36.21879959106445,
      "activations/layer12_attention_weight_max": 30.62833023071289,
      "activations/layer12_attention_weight_min": -32.04948425292969,
      "activations/layer13_attention_weight_max": 53.59978485107422,
      "activations/layer13_attention_weight_min": -39.7652473449707,
      "activations/layer14_attention_weight_max": 53.67053985595703,
      "activations/layer14_attention_weight_min": -42.89509582519531,
      "activations/layer15_attention_weight_max": 46.510650634765625,
      "activations/layer15_attention_weight_min": -39.29545211791992,
      "activations/layer16_attention_weight_max": 33.07453536987305,
      "activations/layer16_attention_weight_min": -27.184587478637695,
      "activations/layer17_attention_weight_max": 28.920997619628906,
      "activations/layer17_attention_weight_min": -27.491010665893555,
      "activations/layer18_attention_weight_max": 31.589317321777344,
      "activations/layer18_attention_weight_min": -26.35506248474121,
      "activations/layer19_attention_weight_max": 32.63372802734375,
      "activations/layer19_attention_weight_min": -31.950414657592773,
      "activations/layer1_attention_weight_max": 16.347614288330078,
      "activations/layer1_attention_weight_min": -13.715402603149414,
      "activations/layer20_attention_weight_max": 28.040355682373047,
      "activations/layer20_attention_weight_min": -26.0407772064209,
      "activations/layer21_attention_weight_max": 28.055587768554688,
      "activations/layer21_attention_weight_min": -25.907865524291992,
      "activations/layer22_attention_weight_max": 41.76154327392578,
      "activations/layer22_attention_weight_min": -35.850589752197266,
      "activations/layer23_attention_weight_max": 32.00946044921875,
      "activations/layer23_attention_weight_min": -28.94025421142578,
      "activations/layer2_attention_weight_max": 33.488059997558594,
      "activations/layer2_attention_weight_min": -32.82183837890625,
      "activations/layer3_attention_weight_max": 104.54450988769531,
      "activations/layer3_attention_weight_min": -105.68182373046875,
      "activations/layer4_attention_weight_max": 109.03753662109375,
      "activations/layer4_attention_weight_min": -109.05860900878906,
      "activations/layer5_attention_weight_max": 53.11046600341797,
      "activations/layer5_attention_weight_min": -61.6291618347168,
      "activations/layer6_attention_weight_max": 45.15467071533203,
      "activations/layer6_attention_weight_min": -49.08506774902344,
      "activations/layer7_attention_weight_max": 101.41841888427734,
      "activations/layer7_attention_weight_min": -104.29302978515625,
      "activations/layer8_attention_weight_max": 44.41050720214844,
      "activations/layer8_attention_weight_min": -48.51041030883789,
      "activations/layer9_attention_weight_max": 34.16123580932617,
      "activations/layer9_attention_weight_min": -36.21512985229492,
      "epoch": 19.47,
      "learning_rate": 2.4703409090909088e-05,
      "loss": 2.7306,
      "step": 335050
    },
    {
      "activations/layer0_attention_weight_max": 16.917322158813477,
      "activations/layer0_attention_weight_min": -12.2576265335083,
      "activations/layer10_attention_weight_max": 33.61041259765625,
      "activations/layer10_attention_weight_min": -35.546051025390625,
      "activations/layer11_attention_weight_max": 34.060951232910156,
      "activations/layer11_attention_weight_min": -36.09090042114258,
      "activations/layer12_attention_weight_max": 27.3918514251709,
      "activations/layer12_attention_weight_min": -28.40045928955078,
      "activations/layer13_attention_weight_max": 39.91204071044922,
      "activations/layer13_attention_weight_min": -38.3131103515625,
      "activations/layer14_attention_weight_max": 43.28827667236328,
      "activations/layer14_attention_weight_min": -39.17034149169922,
      "activations/layer15_attention_weight_max": 43.718116760253906,
      "activations/layer15_attention_weight_min": -35.274173736572266,
      "activations/layer16_attention_weight_max": 30.904157638549805,
      "activations/layer16_attention_weight_min": -28.192012786865234,
      "activations/layer17_attention_weight_max": 29.69123077392578,
      "activations/layer17_attention_weight_min": -26.07267189025879,
      "activations/layer18_attention_weight_max": 31.399154663085938,
      "activations/layer18_attention_weight_min": -26.906551361083984,
      "activations/layer19_attention_weight_max": 33.64543151855469,
      "activations/layer19_attention_weight_min": -32.5700798034668,
      "activations/layer1_attention_weight_max": 15.446758270263672,
      "activations/layer1_attention_weight_min": -14.556299209594727,
      "activations/layer20_attention_weight_max": 28.214513778686523,
      "activations/layer20_attention_weight_min": -24.58440589904785,
      "activations/layer21_attention_weight_max": 28.895259857177734,
      "activations/layer21_attention_weight_min": -25.208213806152344,
      "activations/layer22_attention_weight_max": 40.21371078491211,
      "activations/layer22_attention_weight_min": -33.032997131347656,
      "activations/layer23_attention_weight_max": 30.83434295654297,
      "activations/layer23_attention_weight_min": -27.17047882080078,
      "activations/layer2_attention_weight_max": 31.85860824584961,
      "activations/layer2_attention_weight_min": -31.475589752197266,
      "activations/layer3_attention_weight_max": 95.81309509277344,
      "activations/layer3_attention_weight_min": -97.10852813720703,
      "activations/layer4_attention_weight_max": 101.52978515625,
      "activations/layer4_attention_weight_min": -105.06367492675781,
      "activations/layer5_attention_weight_max": 52.471649169921875,
      "activations/layer5_attention_weight_min": -60.63993453979492,
      "activations/layer6_attention_weight_max": 44.19768524169922,
      "activations/layer6_attention_weight_min": -45.86812973022461,
      "activations/layer7_attention_weight_max": 93.0606918334961,
      "activations/layer7_attention_weight_min": -97.3955078125,
      "activations/layer8_attention_weight_max": 40.58427047729492,
      "activations/layer8_attention_weight_min": -42.66204833984375,
      "activations/layer9_attention_weight_max": 32.30950164794922,
      "activations/layer9_attention_weight_min": -32.98348617553711,
      "epoch": 19.47,
      "learning_rate": 2.4684469696969693e-05,
      "loss": 2.7031,
      "step": 335100
    },
    {
      "activations/layer0_attention_weight_max": 16.30875587463379,
      "activations/layer0_attention_weight_min": -12.870940208435059,
      "activations/layer10_attention_weight_max": 36.372737884521484,
      "activations/layer10_attention_weight_min": -36.34210968017578,
      "activations/layer11_attention_weight_max": 35.0789680480957,
      "activations/layer11_attention_weight_min": -34.58721160888672,
      "activations/layer12_attention_weight_max": 27.834957122802734,
      "activations/layer12_attention_weight_min": -27.316322326660156,
      "activations/layer13_attention_weight_max": 41.33672332763672,
      "activations/layer13_attention_weight_min": -36.60335159301758,
      "activations/layer14_attention_weight_max": 39.90570831298828,
      "activations/layer14_attention_weight_min": -40.10209274291992,
      "activations/layer15_attention_weight_max": 37.20964050292969,
      "activations/layer15_attention_weight_min": -36.38232421875,
      "activations/layer16_attention_weight_max": 27.666641235351562,
      "activations/layer16_attention_weight_min": -28.100038528442383,
      "activations/layer17_attention_weight_max": 29.838876724243164,
      "activations/layer17_attention_weight_min": -27.002391815185547,
      "activations/layer18_attention_weight_max": 33.626529693603516,
      "activations/layer18_attention_weight_min": -28.412071228027344,
      "activations/layer19_attention_weight_max": 34.79969787597656,
      "activations/layer19_attention_weight_min": -32.1364860534668,
      "activations/layer1_attention_weight_max": 16.391250610351562,
      "activations/layer1_attention_weight_min": -13.897941589355469,
      "activations/layer20_attention_weight_max": 29.314828872680664,
      "activations/layer20_attention_weight_min": -26.153718948364258,
      "activations/layer21_attention_weight_max": 29.166118621826172,
      "activations/layer21_attention_weight_min": -25.322546005249023,
      "activations/layer22_attention_weight_max": 38.85007858276367,
      "activations/layer22_attention_weight_min": -32.575679779052734,
      "activations/layer23_attention_weight_max": 34.48889923095703,
      "activations/layer23_attention_weight_min": -25.451080322265625,
      "activations/layer2_attention_weight_max": 30.61487579345703,
      "activations/layer2_attention_weight_min": -29.247159957885742,
      "activations/layer3_attention_weight_max": 96.89794921875,
      "activations/layer3_attention_weight_min": -96.90485382080078,
      "activations/layer4_attention_weight_max": 103.01007080078125,
      "activations/layer4_attention_weight_min": -110.73848724365234,
      "activations/layer5_attention_weight_max": 47.942657470703125,
      "activations/layer5_attention_weight_min": -62.486976623535156,
      "activations/layer6_attention_weight_max": 44.155677795410156,
      "activations/layer6_attention_weight_min": -46.37716293334961,
      "activations/layer7_attention_weight_max": 91.62820434570312,
      "activations/layer7_attention_weight_min": -92.77839660644531,
      "activations/layer8_attention_weight_max": 42.04926681518555,
      "activations/layer8_attention_weight_min": -44.76445388793945,
      "activations/layer9_attention_weight_max": 33.19111251831055,
      "activations/layer9_attention_weight_min": -36.73046875,
      "epoch": 19.47,
      "learning_rate": 2.46655303030303e-05,
      "loss": 2.714,
      "step": 335150
    },
    {
      "activations/layer0_attention_weight_max": 17.466482162475586,
      "activations/layer0_attention_weight_min": -13.2275390625,
      "activations/layer10_attention_weight_max": 33.58644485473633,
      "activations/layer10_attention_weight_min": -34.77714920043945,
      "activations/layer11_attention_weight_max": 33.21430206298828,
      "activations/layer11_attention_weight_min": -34.87826919555664,
      "activations/layer12_attention_weight_max": 25.88434410095215,
      "activations/layer12_attention_weight_min": -31.25757598876953,
      "activations/layer13_attention_weight_max": 41.001583099365234,
      "activations/layer13_attention_weight_min": -35.28437042236328,
      "activations/layer14_attention_weight_max": 40.927730560302734,
      "activations/layer14_attention_weight_min": -38.01573944091797,
      "activations/layer15_attention_weight_max": 37.36060333251953,
      "activations/layer15_attention_weight_min": -35.05860137939453,
      "activations/layer16_attention_weight_max": 28.840681076049805,
      "activations/layer16_attention_weight_min": -27.022926330566406,
      "activations/layer17_attention_weight_max": 32.96443557739258,
      "activations/layer17_attention_weight_min": -26.138019561767578,
      "activations/layer18_attention_weight_max": 31.5617733001709,
      "activations/layer18_attention_weight_min": -25.91790199279785,
      "activations/layer19_attention_weight_max": 34.06883239746094,
      "activations/layer19_attention_weight_min": -30.813100814819336,
      "activations/layer1_attention_weight_max": 15.804632186889648,
      "activations/layer1_attention_weight_min": -14.059248924255371,
      "activations/layer20_attention_weight_max": 29.427043914794922,
      "activations/layer20_attention_weight_min": -25.315507888793945,
      "activations/layer21_attention_weight_max": 30.68062973022461,
      "activations/layer21_attention_weight_min": -24.56341552734375,
      "activations/layer22_attention_weight_max": 42.04889678955078,
      "activations/layer22_attention_weight_min": -31.727434158325195,
      "activations/layer23_attention_weight_max": 32.66144561767578,
      "activations/layer23_attention_weight_min": -28.623647689819336,
      "activations/layer2_attention_weight_max": 31.509811401367188,
      "activations/layer2_attention_weight_min": -30.370006561279297,
      "activations/layer3_attention_weight_max": 96.5341567993164,
      "activations/layer3_attention_weight_min": -98.16643524169922,
      "activations/layer4_attention_weight_max": 104.77803802490234,
      "activations/layer4_attention_weight_min": -107.16156005859375,
      "activations/layer5_attention_weight_max": 49.588741302490234,
      "activations/layer5_attention_weight_min": -64.28003692626953,
      "activations/layer6_attention_weight_max": 43.17435836791992,
      "activations/layer6_attention_weight_min": -47.76622009277344,
      "activations/layer7_attention_weight_max": 93.76060485839844,
      "activations/layer7_attention_weight_min": -95.48543548583984,
      "activations/layer8_attention_weight_max": 40.55974197387695,
      "activations/layer8_attention_weight_min": -46.57235336303711,
      "activations/layer9_attention_weight_max": 31.425222396850586,
      "activations/layer9_attention_weight_min": -34.80478286743164,
      "epoch": 19.48,
      "learning_rate": 2.464659090909091e-05,
      "loss": 2.7173,
      "step": 335200
    },
    {
      "activations/layer0_attention_weight_max": 17.139753341674805,
      "activations/layer0_attention_weight_min": -12.995386123657227,
      "activations/layer10_attention_weight_max": 31.85667610168457,
      "activations/layer10_attention_weight_min": -33.15937042236328,
      "activations/layer11_attention_weight_max": 33.364715576171875,
      "activations/layer11_attention_weight_min": -33.607295989990234,
      "activations/layer12_attention_weight_max": 25.719093322753906,
      "activations/layer12_attention_weight_min": -27.696062088012695,
      "activations/layer13_attention_weight_max": 39.32506561279297,
      "activations/layer13_attention_weight_min": -35.69425964355469,
      "activations/layer14_attention_weight_max": 39.791778564453125,
      "activations/layer14_attention_weight_min": -38.71792221069336,
      "activations/layer15_attention_weight_max": 37.184417724609375,
      "activations/layer15_attention_weight_min": -34.09079360961914,
      "activations/layer16_attention_weight_max": 29.711706161499023,
      "activations/layer16_attention_weight_min": -29.871694564819336,
      "activations/layer17_attention_weight_max": 27.020662307739258,
      "activations/layer17_attention_weight_min": -26.06127166748047,
      "activations/layer18_attention_weight_max": 30.14076042175293,
      "activations/layer18_attention_weight_min": -26.02519989013672,
      "activations/layer19_attention_weight_max": 32.50609588623047,
      "activations/layer19_attention_weight_min": -31.50962257385254,
      "activations/layer1_attention_weight_max": 16.180986404418945,
      "activations/layer1_attention_weight_min": -14.486136436462402,
      "activations/layer20_attention_weight_max": 26.3604736328125,
      "activations/layer20_attention_weight_min": -24.828527450561523,
      "activations/layer21_attention_weight_max": 28.33354377746582,
      "activations/layer21_attention_weight_min": -24.946212768554688,
      "activations/layer22_attention_weight_max": 36.45469284057617,
      "activations/layer22_attention_weight_min": -31.19624900817871,
      "activations/layer23_attention_weight_max": 29.653898239135742,
      "activations/layer23_attention_weight_min": -26.286922454833984,
      "activations/layer2_attention_weight_max": 30.271032333374023,
      "activations/layer2_attention_weight_min": -32.46129608154297,
      "activations/layer3_attention_weight_max": 93.18663787841797,
      "activations/layer3_attention_weight_min": -99.84568786621094,
      "activations/layer4_attention_weight_max": 99.35257720947266,
      "activations/layer4_attention_weight_min": -102.50780487060547,
      "activations/layer5_attention_weight_max": 47.86602020263672,
      "activations/layer5_attention_weight_min": -59.82676696777344,
      "activations/layer6_attention_weight_max": 44.41604995727539,
      "activations/layer6_attention_weight_min": -48.00774383544922,
      "activations/layer7_attention_weight_max": 90.90455627441406,
      "activations/layer7_attention_weight_min": -102.45970153808594,
      "activations/layer8_attention_weight_max": 42.13003158569336,
      "activations/layer8_attention_weight_min": -46.38869857788086,
      "activations/layer9_attention_weight_max": 30.959304809570312,
      "activations/layer9_attention_weight_min": -35.73946762084961,
      "epoch": 19.48,
      "learning_rate": 2.4627651515151514e-05,
      "loss": 2.7158,
      "step": 335250
    },
    {
      "activations/layer0_attention_weight_max": 16.249149322509766,
      "activations/layer0_attention_weight_min": -13.07208251953125,
      "activations/layer10_attention_weight_max": 34.82269287109375,
      "activations/layer10_attention_weight_min": -34.48377990722656,
      "activations/layer11_attention_weight_max": 37.8865852355957,
      "activations/layer11_attention_weight_min": -34.45470428466797,
      "activations/layer12_attention_weight_max": 27.21662712097168,
      "activations/layer12_attention_weight_min": -27.55204200744629,
      "activations/layer13_attention_weight_max": 39.20819091796875,
      "activations/layer13_attention_weight_min": -36.389404296875,
      "activations/layer14_attention_weight_max": 40.7739143371582,
      "activations/layer14_attention_weight_min": -36.81134796142578,
      "activations/layer15_attention_weight_max": 38.22086715698242,
      "activations/layer15_attention_weight_min": -37.429100036621094,
      "activations/layer16_attention_weight_max": 30.584749221801758,
      "activations/layer16_attention_weight_min": -29.742572784423828,
      "activations/layer17_attention_weight_max": 29.420291900634766,
      "activations/layer17_attention_weight_min": -27.006845474243164,
      "activations/layer18_attention_weight_max": 34.5621337890625,
      "activations/layer18_attention_weight_min": -27.827457427978516,
      "activations/layer19_attention_weight_max": 35.15400314331055,
      "activations/layer19_attention_weight_min": -33.03612518310547,
      "activations/layer1_attention_weight_max": 15.364176750183105,
      "activations/layer1_attention_weight_min": -13.812058448791504,
      "activations/layer20_attention_weight_max": 28.349895477294922,
      "activations/layer20_attention_weight_min": -26.571744918823242,
      "activations/layer21_attention_weight_max": 27.31732177734375,
      "activations/layer21_attention_weight_min": -25.90636444091797,
      "activations/layer22_attention_weight_max": 40.63456344604492,
      "activations/layer22_attention_weight_min": -33.86662292480469,
      "activations/layer23_attention_weight_max": 31.801490783691406,
      "activations/layer23_attention_weight_min": -26.72594451904297,
      "activations/layer2_attention_weight_max": 32.825416564941406,
      "activations/layer2_attention_weight_min": -32.048946380615234,
      "activations/layer3_attention_weight_max": 99.90898895263672,
      "activations/layer3_attention_weight_min": -99.65000915527344,
      "activations/layer4_attention_weight_max": 107.03983306884766,
      "activations/layer4_attention_weight_min": -113.99311828613281,
      "activations/layer5_attention_weight_max": 50.63658905029297,
      "activations/layer5_attention_weight_min": -62.90741729736328,
      "activations/layer6_attention_weight_max": 45.952877044677734,
      "activations/layer6_attention_weight_min": -48.66862869262695,
      "activations/layer7_attention_weight_max": 97.1897964477539,
      "activations/layer7_attention_weight_min": -99.2830810546875,
      "activations/layer8_attention_weight_max": 41.12868118286133,
      "activations/layer8_attention_weight_min": -43.92495346069336,
      "activations/layer9_attention_weight_max": 32.05076599121094,
      "activations/layer9_attention_weight_min": -33.39327621459961,
      "epoch": 19.48,
      "learning_rate": 2.4608712121212122e-05,
      "loss": 2.7234,
      "step": 335300
    },
    {
      "activations/layer0_attention_weight_max": 17.017330169677734,
      "activations/layer0_attention_weight_min": -13.411680221557617,
      "activations/layer10_attention_weight_max": 32.92267608642578,
      "activations/layer10_attention_weight_min": -33.525665283203125,
      "activations/layer11_attention_weight_max": 34.17271041870117,
      "activations/layer11_attention_weight_min": -33.92887878417969,
      "activations/layer12_attention_weight_max": 23.541183471679688,
      "activations/layer12_attention_weight_min": -26.321626663208008,
      "activations/layer13_attention_weight_max": 36.860023498535156,
      "activations/layer13_attention_weight_min": -37.66801834106445,
      "activations/layer14_attention_weight_max": 39.29213333129883,
      "activations/layer14_attention_weight_min": -39.88285827636719,
      "activations/layer15_attention_weight_max": 37.46986389160156,
      "activations/layer15_attention_weight_min": -38.665889739990234,
      "activations/layer16_attention_weight_max": 30.238697052001953,
      "activations/layer16_attention_weight_min": -27.350072860717773,
      "activations/layer17_attention_weight_max": 28.477352142333984,
      "activations/layer17_attention_weight_min": -24.2154541015625,
      "activations/layer18_attention_weight_max": 30.610261917114258,
      "activations/layer18_attention_weight_min": -24.792591094970703,
      "activations/layer19_attention_weight_max": 33.6005973815918,
      "activations/layer19_attention_weight_min": -31.06180191040039,
      "activations/layer1_attention_weight_max": 17.420602798461914,
      "activations/layer1_attention_weight_min": -15.847306251525879,
      "activations/layer20_attention_weight_max": 27.26702117919922,
      "activations/layer20_attention_weight_min": -25.075294494628906,
      "activations/layer21_attention_weight_max": 28.732555389404297,
      "activations/layer21_attention_weight_min": -23.898324966430664,
      "activations/layer22_attention_weight_max": 37.458065032958984,
      "activations/layer22_attention_weight_min": -30.369964599609375,
      "activations/layer23_attention_weight_max": 30.907394409179688,
      "activations/layer23_attention_weight_min": -23.737483978271484,
      "activations/layer2_attention_weight_max": 30.70015525817871,
      "activations/layer2_attention_weight_min": -29.818470001220703,
      "activations/layer3_attention_weight_max": 102.99685668945312,
      "activations/layer3_attention_weight_min": -100.72908782958984,
      "activations/layer4_attention_weight_max": 107.82518768310547,
      "activations/layer4_attention_weight_min": -107.556640625,
      "activations/layer5_attention_weight_max": 50.561397552490234,
      "activations/layer5_attention_weight_min": -60.280174255371094,
      "activations/layer6_attention_weight_max": 43.3691291809082,
      "activations/layer6_attention_weight_min": -44.26363754272461,
      "activations/layer7_attention_weight_max": 95.0339126586914,
      "activations/layer7_attention_weight_min": -95.81172180175781,
      "activations/layer8_attention_weight_max": 41.21823501586914,
      "activations/layer8_attention_weight_min": -42.423240661621094,
      "activations/layer9_attention_weight_max": 31.997314453125,
      "activations/layer9_attention_weight_min": -33.577274322509766,
      "epoch": 19.49,
      "learning_rate": 2.4589772727272724e-05,
      "loss": 2.7225,
      "step": 335350
    },
    {
      "activations/layer0_attention_weight_max": 16.195463180541992,
      "activations/layer0_attention_weight_min": -13.13545036315918,
      "activations/layer10_attention_weight_max": 34.882057189941406,
      "activations/layer10_attention_weight_min": -35.14823532104492,
      "activations/layer11_attention_weight_max": 33.04690933227539,
      "activations/layer11_attention_weight_min": -34.48952102661133,
      "activations/layer12_attention_weight_max": 25.1611385345459,
      "activations/layer12_attention_weight_min": -26.839527130126953,
      "activations/layer13_attention_weight_max": 35.62325668334961,
      "activations/layer13_attention_weight_min": -35.42452621459961,
      "activations/layer14_attention_weight_max": 41.870269775390625,
      "activations/layer14_attention_weight_min": -38.93001937866211,
      "activations/layer15_attention_weight_max": 38.19341278076172,
      "activations/layer15_attention_weight_min": -38.375770568847656,
      "activations/layer16_attention_weight_max": 33.04507827758789,
      "activations/layer16_attention_weight_min": -27.297115325927734,
      "activations/layer17_attention_weight_max": 31.264535903930664,
      "activations/layer17_attention_weight_min": -26.167531967163086,
      "activations/layer18_attention_weight_max": 33.686126708984375,
      "activations/layer18_attention_weight_min": -30.3468074798584,
      "activations/layer19_attention_weight_max": 32.77298355102539,
      "activations/layer19_attention_weight_min": -30.83182144165039,
      "activations/layer1_attention_weight_max": 16.381820678710938,
      "activations/layer1_attention_weight_min": -14.704156875610352,
      "activations/layer20_attention_weight_max": 27.855024337768555,
      "activations/layer20_attention_weight_min": -25.33986473083496,
      "activations/layer21_attention_weight_max": 26.932418823242188,
      "activations/layer21_attention_weight_min": -23.489940643310547,
      "activations/layer22_attention_weight_max": 36.98563766479492,
      "activations/layer22_attention_weight_min": -30.59036636352539,
      "activations/layer23_attention_weight_max": 28.820632934570312,
      "activations/layer23_attention_weight_min": -26.742841720581055,
      "activations/layer2_attention_weight_max": 30.081707000732422,
      "activations/layer2_attention_weight_min": -32.18958282470703,
      "activations/layer3_attention_weight_max": 100.18695831298828,
      "activations/layer3_attention_weight_min": -101.62178802490234,
      "activations/layer4_attention_weight_max": 105.85063934326172,
      "activations/layer4_attention_weight_min": -105.66643524169922,
      "activations/layer5_attention_weight_max": 48.4744987487793,
      "activations/layer5_attention_weight_min": -60.978939056396484,
      "activations/layer6_attention_weight_max": 42.424774169921875,
      "activations/layer6_attention_weight_min": -46.53298568725586,
      "activations/layer7_attention_weight_max": 88.45777893066406,
      "activations/layer7_attention_weight_min": -93.11213684082031,
      "activations/layer8_attention_weight_max": 40.149986267089844,
      "activations/layer8_attention_weight_min": -43.78714370727539,
      "activations/layer9_attention_weight_max": 31.36147117614746,
      "activations/layer9_attention_weight_min": -33.42084503173828,
      "epoch": 19.49,
      "learning_rate": 2.4570833333333332e-05,
      "loss": 2.721,
      "step": 335400
    },
    {
      "activations/layer0_attention_weight_max": 17.037534713745117,
      "activations/layer0_attention_weight_min": -12.632171630859375,
      "activations/layer10_attention_weight_max": 34.604305267333984,
      "activations/layer10_attention_weight_min": -34.63083267211914,
      "activations/layer11_attention_weight_max": 34.33164978027344,
      "activations/layer11_attention_weight_min": -37.02768325805664,
      "activations/layer12_attention_weight_max": 28.91808319091797,
      "activations/layer12_attention_weight_min": -31.27529525756836,
      "activations/layer13_attention_weight_max": 41.3033332824707,
      "activations/layer13_attention_weight_min": -39.91460037231445,
      "activations/layer14_attention_weight_max": 42.244293212890625,
      "activations/layer14_attention_weight_min": -41.29779815673828,
      "activations/layer15_attention_weight_max": 41.04307556152344,
      "activations/layer15_attention_weight_min": -36.727752685546875,
      "activations/layer16_attention_weight_max": 31.704797744750977,
      "activations/layer16_attention_weight_min": -30.467761993408203,
      "activations/layer17_attention_weight_max": 30.66240119934082,
      "activations/layer17_attention_weight_min": -29.62257194519043,
      "activations/layer18_attention_weight_max": 34.3547477722168,
      "activations/layer18_attention_weight_min": -25.81241798400879,
      "activations/layer19_attention_weight_max": 35.050838470458984,
      "activations/layer19_attention_weight_min": -33.224910736083984,
      "activations/layer1_attention_weight_max": 15.780132293701172,
      "activations/layer1_attention_weight_min": -14.060059547424316,
      "activations/layer20_attention_weight_max": 28.6549072265625,
      "activations/layer20_attention_weight_min": -26.93267059326172,
      "activations/layer21_attention_weight_max": 28.955821990966797,
      "activations/layer21_attention_weight_min": -26.191513061523438,
      "activations/layer22_attention_weight_max": 41.089962005615234,
      "activations/layer22_attention_weight_min": -31.804943084716797,
      "activations/layer23_attention_weight_max": 35.173519134521484,
      "activations/layer23_attention_weight_min": -25.763011932373047,
      "activations/layer2_attention_weight_max": 31.576757431030273,
      "activations/layer2_attention_weight_min": -30.217544555664062,
      "activations/layer3_attention_weight_max": 98.86239624023438,
      "activations/layer3_attention_weight_min": -98.74766540527344,
      "activations/layer4_attention_weight_max": 104.92574310302734,
      "activations/layer4_attention_weight_min": -109.44502258300781,
      "activations/layer5_attention_weight_max": 51.20634460449219,
      "activations/layer5_attention_weight_min": -63.205345153808594,
      "activations/layer6_attention_weight_max": 45.91389465332031,
      "activations/layer6_attention_weight_min": -47.22377014160156,
      "activations/layer7_attention_weight_max": 95.65848541259766,
      "activations/layer7_attention_weight_min": -98.00274658203125,
      "activations/layer8_attention_weight_max": 44.1505012512207,
      "activations/layer8_attention_weight_min": -44.6629524230957,
      "activations/layer9_attention_weight_max": 32.73977279663086,
      "activations/layer9_attention_weight_min": -34.86899185180664,
      "epoch": 19.49,
      "learning_rate": 2.4551893939393937e-05,
      "loss": 2.7252,
      "step": 335450
    },
    {
      "activations/layer0_attention_weight_max": 16.4119930267334,
      "activations/layer0_attention_weight_min": -12.563409805297852,
      "activations/layer10_attention_weight_max": 33.95845413208008,
      "activations/layer10_attention_weight_min": -34.84898376464844,
      "activations/layer11_attention_weight_max": 34.27078628540039,
      "activations/layer11_attention_weight_min": -34.913063049316406,
      "activations/layer12_attention_weight_max": 30.094072341918945,
      "activations/layer12_attention_weight_min": -27.603696823120117,
      "activations/layer13_attention_weight_max": 41.32880783081055,
      "activations/layer13_attention_weight_min": -34.63031005859375,
      "activations/layer14_attention_weight_max": 42.27595520019531,
      "activations/layer14_attention_weight_min": -35.737300872802734,
      "activations/layer15_attention_weight_max": 39.7517204284668,
      "activations/layer15_attention_weight_min": -35.82273483276367,
      "activations/layer16_attention_weight_max": 28.01910972595215,
      "activations/layer16_attention_weight_min": -28.367307662963867,
      "activations/layer17_attention_weight_max": 30.87448501586914,
      "activations/layer17_attention_weight_min": -27.612096786499023,
      "activations/layer18_attention_weight_max": 29.57801628112793,
      "activations/layer18_attention_weight_min": -26.86190414428711,
      "activations/layer19_attention_weight_max": 33.046382904052734,
      "activations/layer19_attention_weight_min": -32.539283752441406,
      "activations/layer1_attention_weight_max": 15.17251968383789,
      "activations/layer1_attention_weight_min": -14.050270080566406,
      "activations/layer20_attention_weight_max": 27.511362075805664,
      "activations/layer20_attention_weight_min": -24.081335067749023,
      "activations/layer21_attention_weight_max": 26.24253273010254,
      "activations/layer21_attention_weight_min": -24.935245513916016,
      "activations/layer22_attention_weight_max": 41.21308135986328,
      "activations/layer22_attention_weight_min": -32.77075958251953,
      "activations/layer23_attention_weight_max": 30.854232788085938,
      "activations/layer23_attention_weight_min": -24.491771697998047,
      "activations/layer2_attention_weight_max": 31.43862533569336,
      "activations/layer2_attention_weight_min": -30.102035522460938,
      "activations/layer3_attention_weight_max": 98.31539916992188,
      "activations/layer3_attention_weight_min": -101.52030944824219,
      "activations/layer4_attention_weight_max": 105.67218017578125,
      "activations/layer4_attention_weight_min": -107.6707992553711,
      "activations/layer5_attention_weight_max": 49.24004364013672,
      "activations/layer5_attention_weight_min": -59.87190246582031,
      "activations/layer6_attention_weight_max": 43.8311882019043,
      "activations/layer6_attention_weight_min": -46.92282485961914,
      "activations/layer7_attention_weight_max": 95.07488250732422,
      "activations/layer7_attention_weight_min": -95.5529556274414,
      "activations/layer8_attention_weight_max": 40.78372573852539,
      "activations/layer8_attention_weight_min": -44.3055419921875,
      "activations/layer9_attention_weight_max": 32.700077056884766,
      "activations/layer9_attention_weight_min": -34.55567932128906,
      "epoch": 19.49,
      "learning_rate": 2.4532954545454545e-05,
      "loss": 2.7216,
      "step": 335500
    },
    {
      "activations/layer0_attention_weight_max": 16.11170196533203,
      "activations/layer0_attention_weight_min": -12.87821102142334,
      "activations/layer10_attention_weight_max": 36.511573791503906,
      "activations/layer10_attention_weight_min": -34.02901840209961,
      "activations/layer11_attention_weight_max": 34.04957580566406,
      "activations/layer11_attention_weight_min": -34.052467346191406,
      "activations/layer12_attention_weight_max": 27.880434036254883,
      "activations/layer12_attention_weight_min": -26.82918930053711,
      "activations/layer13_attention_weight_max": 40.96378707885742,
      "activations/layer13_attention_weight_min": -35.806148529052734,
      "activations/layer14_attention_weight_max": 42.420108795166016,
      "activations/layer14_attention_weight_min": -39.87720489501953,
      "activations/layer15_attention_weight_max": 39.71894073486328,
      "activations/layer15_attention_weight_min": -38.475223541259766,
      "activations/layer16_attention_weight_max": 30.764949798583984,
      "activations/layer16_attention_weight_min": -28.57201385498047,
      "activations/layer17_attention_weight_max": 30.374114990234375,
      "activations/layer17_attention_weight_min": -27.128965377807617,
      "activations/layer18_attention_weight_max": 32.521392822265625,
      "activations/layer18_attention_weight_min": -29.05340576171875,
      "activations/layer19_attention_weight_max": 35.53080368041992,
      "activations/layer19_attention_weight_min": -32.22396469116211,
      "activations/layer1_attention_weight_max": 15.42546272277832,
      "activations/layer1_attention_weight_min": -14.168042182922363,
      "activations/layer20_attention_weight_max": 29.6516056060791,
      "activations/layer20_attention_weight_min": -26.1667423248291,
      "activations/layer21_attention_weight_max": 28.43758201599121,
      "activations/layer21_attention_weight_min": -25.26073455810547,
      "activations/layer22_attention_weight_max": 40.08282470703125,
      "activations/layer22_attention_weight_min": -32.280738830566406,
      "activations/layer23_attention_weight_max": 34.38437271118164,
      "activations/layer23_attention_weight_min": -26.427295684814453,
      "activations/layer2_attention_weight_max": 32.44883346557617,
      "activations/layer2_attention_weight_min": -31.261363983154297,
      "activations/layer3_attention_weight_max": 101.98464965820312,
      "activations/layer3_attention_weight_min": -99.21401977539062,
      "activations/layer4_attention_weight_max": 109.21392059326172,
      "activations/layer4_attention_weight_min": -111.57161712646484,
      "activations/layer5_attention_weight_max": 52.022274017333984,
      "activations/layer5_attention_weight_min": -64.11207580566406,
      "activations/layer6_attention_weight_max": 43.40395736694336,
      "activations/layer6_attention_weight_min": -46.6147575378418,
      "activations/layer7_attention_weight_max": 99.9070816040039,
      "activations/layer7_attention_weight_min": -103.00920104980469,
      "activations/layer8_attention_weight_max": 42.42240905761719,
      "activations/layer8_attention_weight_min": -45.850189208984375,
      "activations/layer9_attention_weight_max": 32.60169982910156,
      "activations/layer9_attention_weight_min": -33.766502380371094,
      "epoch": 19.5,
      "learning_rate": 2.451401515151515e-05,
      "loss": 2.7233,
      "step": 335550
    },
    {
      "activations/layer0_attention_weight_max": 15.089269638061523,
      "activations/layer0_attention_weight_min": -12.390742301940918,
      "activations/layer10_attention_weight_max": 38.6010627746582,
      "activations/layer10_attention_weight_min": -34.96926498413086,
      "activations/layer11_attention_weight_max": 35.222679138183594,
      "activations/layer11_attention_weight_min": -36.88054656982422,
      "activations/layer12_attention_weight_max": 26.32748031616211,
      "activations/layer12_attention_weight_min": -27.661447525024414,
      "activations/layer13_attention_weight_max": 38.93642044067383,
      "activations/layer13_attention_weight_min": -38.979331970214844,
      "activations/layer14_attention_weight_max": 49.68595504760742,
      "activations/layer14_attention_weight_min": -39.87657928466797,
      "activations/layer15_attention_weight_max": 40.97024154663086,
      "activations/layer15_attention_weight_min": -35.98278045654297,
      "activations/layer16_attention_weight_max": 32.445068359375,
      "activations/layer16_attention_weight_min": -26.75445556640625,
      "activations/layer17_attention_weight_max": 32.37051773071289,
      "activations/layer17_attention_weight_min": -25.756183624267578,
      "activations/layer18_attention_weight_max": 31.068944931030273,
      "activations/layer18_attention_weight_min": -25.442630767822266,
      "activations/layer19_attention_weight_max": 36.59421157836914,
      "activations/layer19_attention_weight_min": -31.227365493774414,
      "activations/layer1_attention_weight_max": 15.996916770935059,
      "activations/layer1_attention_weight_min": -16.17238426208496,
      "activations/layer20_attention_weight_max": 30.42633628845215,
      "activations/layer20_attention_weight_min": -23.994163513183594,
      "activations/layer21_attention_weight_max": 29.896665573120117,
      "activations/layer21_attention_weight_min": -24.048635482788086,
      "activations/layer22_attention_weight_max": 42.86921310424805,
      "activations/layer22_attention_weight_min": -29.302867889404297,
      "activations/layer23_attention_weight_max": 35.111122131347656,
      "activations/layer23_attention_weight_min": -24.345510482788086,
      "activations/layer2_attention_weight_max": 30.586063385009766,
      "activations/layer2_attention_weight_min": -31.114179611206055,
      "activations/layer3_attention_weight_max": 97.83829498291016,
      "activations/layer3_attention_weight_min": -96.404296875,
      "activations/layer4_attention_weight_max": 106.58458709716797,
      "activations/layer4_attention_weight_min": -106.25422668457031,
      "activations/layer5_attention_weight_max": 54.82538986206055,
      "activations/layer5_attention_weight_min": -64.69522857666016,
      "activations/layer6_attention_weight_max": 48.553672790527344,
      "activations/layer6_attention_weight_min": -47.3735237121582,
      "activations/layer7_attention_weight_max": 95.11485290527344,
      "activations/layer7_attention_weight_min": -99.02983856201172,
      "activations/layer8_attention_weight_max": 43.065086364746094,
      "activations/layer8_attention_weight_min": -44.50014114379883,
      "activations/layer9_attention_weight_max": 33.00107192993164,
      "activations/layer9_attention_weight_min": -35.29589080810547,
      "epoch": 19.5,
      "learning_rate": 2.4495075757575755e-05,
      "loss": 2.7212,
      "step": 335600
    },
    {
      "activations/layer0_attention_weight_max": 16.26042366027832,
      "activations/layer0_attention_weight_min": -12.230709075927734,
      "activations/layer10_attention_weight_max": 35.777591705322266,
      "activations/layer10_attention_weight_min": -35.32435607910156,
      "activations/layer11_attention_weight_max": 34.266990661621094,
      "activations/layer11_attention_weight_min": -34.52589416503906,
      "activations/layer12_attention_weight_max": 28.038835525512695,
      "activations/layer12_attention_weight_min": -27.034151077270508,
      "activations/layer13_attention_weight_max": 38.434669494628906,
      "activations/layer13_attention_weight_min": -36.49569320678711,
      "activations/layer14_attention_weight_max": 41.29999923706055,
      "activations/layer14_attention_weight_min": -36.01142120361328,
      "activations/layer15_attention_weight_max": 39.954917907714844,
      "activations/layer15_attention_weight_min": -35.680660247802734,
      "activations/layer16_attention_weight_max": 28.642702102661133,
      "activations/layer16_attention_weight_min": -27.6924991607666,
      "activations/layer17_attention_weight_max": 29.05514907836914,
      "activations/layer17_attention_weight_min": -25.696203231811523,
      "activations/layer18_attention_weight_max": 34.840057373046875,
      "activations/layer18_attention_weight_min": -27.333782196044922,
      "activations/layer19_attention_weight_max": 35.49116134643555,
      "activations/layer19_attention_weight_min": -32.312469482421875,
      "activations/layer1_attention_weight_max": 15.7230806350708,
      "activations/layer1_attention_weight_min": -14.558328628540039,
      "activations/layer20_attention_weight_max": 27.676097869873047,
      "activations/layer20_attention_weight_min": -24.216503143310547,
      "activations/layer21_attention_weight_max": 26.45281982421875,
      "activations/layer21_attention_weight_min": -25.66617202758789,
      "activations/layer22_attention_weight_max": 39.80855178833008,
      "activations/layer22_attention_weight_min": -31.36870574951172,
      "activations/layer23_attention_weight_max": 31.885658264160156,
      "activations/layer23_attention_weight_min": -25.9947509765625,
      "activations/layer2_attention_weight_max": 30.847501754760742,
      "activations/layer2_attention_weight_min": -31.505325317382812,
      "activations/layer3_attention_weight_max": 98.12935638427734,
      "activations/layer3_attention_weight_min": -100.64849853515625,
      "activations/layer4_attention_weight_max": 105.15657043457031,
      "activations/layer4_attention_weight_min": -107.85975646972656,
      "activations/layer5_attention_weight_max": 51.94788360595703,
      "activations/layer5_attention_weight_min": -61.04454803466797,
      "activations/layer6_attention_weight_max": 43.295066833496094,
      "activations/layer6_attention_weight_min": -46.22054672241211,
      "activations/layer7_attention_weight_max": 95.81055450439453,
      "activations/layer7_attention_weight_min": -96.84480285644531,
      "activations/layer8_attention_weight_max": 43.524635314941406,
      "activations/layer8_attention_weight_min": -43.944419860839844,
      "activations/layer9_attention_weight_max": 34.02827072143555,
      "activations/layer9_attention_weight_min": -35.45570755004883,
      "epoch": 19.5,
      "learning_rate": 2.447613636363636e-05,
      "loss": 2.7336,
      "step": 335650
    },
    {
      "activations/layer0_attention_weight_max": 16.63422966003418,
      "activations/layer0_attention_weight_min": -12.835679054260254,
      "activations/layer10_attention_weight_max": 36.56431198120117,
      "activations/layer10_attention_weight_min": -36.78271484375,
      "activations/layer11_attention_weight_max": 34.99708557128906,
      "activations/layer11_attention_weight_min": -36.35831069946289,
      "activations/layer12_attention_weight_max": 29.111555099487305,
      "activations/layer12_attention_weight_min": -28.438291549682617,
      "activations/layer13_attention_weight_max": 42.879432678222656,
      "activations/layer13_attention_weight_min": -37.92312240600586,
      "activations/layer14_attention_weight_max": 44.841949462890625,
      "activations/layer14_attention_weight_min": -41.457340240478516,
      "activations/layer15_attention_weight_max": 45.30475997924805,
      "activations/layer15_attention_weight_min": -38.6159553527832,
      "activations/layer16_attention_weight_max": 30.726879119873047,
      "activations/layer16_attention_weight_min": -27.541452407836914,
      "activations/layer17_attention_weight_max": 29.35096549987793,
      "activations/layer17_attention_weight_min": -25.462432861328125,
      "activations/layer18_attention_weight_max": 31.52788734436035,
      "activations/layer18_attention_weight_min": -25.491392135620117,
      "activations/layer19_attention_weight_max": 36.260032653808594,
      "activations/layer19_attention_weight_min": -30.3231143951416,
      "activations/layer1_attention_weight_max": 17.295595169067383,
      "activations/layer1_attention_weight_min": -14.663661003112793,
      "activations/layer20_attention_weight_max": 29.554393768310547,
      "activations/layer20_attention_weight_min": -23.685333251953125,
      "activations/layer21_attention_weight_max": 29.474130630493164,
      "activations/layer21_attention_weight_min": -23.0155086517334,
      "activations/layer22_attention_weight_max": 45.05997085571289,
      "activations/layer22_attention_weight_min": -32.48824691772461,
      "activations/layer23_attention_weight_max": 35.737022399902344,
      "activations/layer23_attention_weight_min": -26.675207138061523,
      "activations/layer2_attention_weight_max": 31.09052085876465,
      "activations/layer2_attention_weight_min": -29.667598724365234,
      "activations/layer3_attention_weight_max": 92.88684844970703,
      "activations/layer3_attention_weight_min": -95.5726547241211,
      "activations/layer4_attention_weight_max": 101.81400299072266,
      "activations/layer4_attention_weight_min": -100.90906524658203,
      "activations/layer5_attention_weight_max": 48.33483123779297,
      "activations/layer5_attention_weight_min": -60.243019104003906,
      "activations/layer6_attention_weight_max": 44.4619026184082,
      "activations/layer6_attention_weight_min": -46.87639236450195,
      "activations/layer7_attention_weight_max": 93.21237182617188,
      "activations/layer7_attention_weight_min": -100.93582916259766,
      "activations/layer8_attention_weight_max": 44.321346282958984,
      "activations/layer8_attention_weight_min": -45.540069580078125,
      "activations/layer9_attention_weight_max": 35.55390167236328,
      "activations/layer9_attention_weight_min": -37.153831481933594,
      "epoch": 19.51,
      "learning_rate": 2.4457196969696968e-05,
      "loss": 2.738,
      "step": 335700
    },
    {
      "activations/layer0_attention_weight_max": 15.502370834350586,
      "activations/layer0_attention_weight_min": -12.465046882629395,
      "activations/layer10_attention_weight_max": 34.806884765625,
      "activations/layer10_attention_weight_min": -35.40207290649414,
      "activations/layer11_attention_weight_max": 34.88921356201172,
      "activations/layer11_attention_weight_min": -36.798091888427734,
      "activations/layer12_attention_weight_max": 27.228931427001953,
      "activations/layer12_attention_weight_min": -25.9687442779541,
      "activations/layer13_attention_weight_max": 40.121334075927734,
      "activations/layer13_attention_weight_min": -34.651161193847656,
      "activations/layer14_attention_weight_max": 40.66013717651367,
      "activations/layer14_attention_weight_min": -37.97632598876953,
      "activations/layer15_attention_weight_max": 38.21643829345703,
      "activations/layer15_attention_weight_min": -36.246707916259766,
      "activations/layer16_attention_weight_max": 27.454652786254883,
      "activations/layer16_attention_weight_min": -28.096633911132812,
      "activations/layer17_attention_weight_max": 27.28322410583496,
      "activations/layer17_attention_weight_min": -25.01557731628418,
      "activations/layer18_attention_weight_max": 34.51350402832031,
      "activations/layer18_attention_weight_min": -26.856399536132812,
      "activations/layer19_attention_weight_max": 32.046077728271484,
      "activations/layer19_attention_weight_min": -31.25979995727539,
      "activations/layer1_attention_weight_max": 16.82647705078125,
      "activations/layer1_attention_weight_min": -15.481101989746094,
      "activations/layer20_attention_weight_max": 26.307048797607422,
      "activations/layer20_attention_weight_min": -26.10319709777832,
      "activations/layer21_attention_weight_max": 27.073429107666016,
      "activations/layer21_attention_weight_min": -25.240604400634766,
      "activations/layer22_attention_weight_max": 40.89295196533203,
      "activations/layer22_attention_weight_min": -32.161617279052734,
      "activations/layer23_attention_weight_max": 33.4290885925293,
      "activations/layer23_attention_weight_min": -26.287458419799805,
      "activations/layer2_attention_weight_max": 30.913206100463867,
      "activations/layer2_attention_weight_min": -30.251693725585938,
      "activations/layer3_attention_weight_max": 97.79248809814453,
      "activations/layer3_attention_weight_min": -102.87299346923828,
      "activations/layer4_attention_weight_max": 103.7999267578125,
      "activations/layer4_attention_weight_min": -106.5128402709961,
      "activations/layer5_attention_weight_max": 47.674713134765625,
      "activations/layer5_attention_weight_min": -61.974037170410156,
      "activations/layer6_attention_weight_max": 42.5287971496582,
      "activations/layer6_attention_weight_min": -47.8880729675293,
      "activations/layer7_attention_weight_max": 91.54437255859375,
      "activations/layer7_attention_weight_min": -101.00029754638672,
      "activations/layer8_attention_weight_max": 41.587913513183594,
      "activations/layer8_attention_weight_min": -45.41423034667969,
      "activations/layer9_attention_weight_max": 31.47536277770996,
      "activations/layer9_attention_weight_min": -33.54975509643555,
      "epoch": 19.51,
      "learning_rate": 2.4438257575757573e-05,
      "loss": 2.7299,
      "step": 335750
    },
    {
      "activations/layer0_attention_weight_max": 16.627046585083008,
      "activations/layer0_attention_weight_min": -13.634313583374023,
      "activations/layer10_attention_weight_max": 37.095703125,
      "activations/layer10_attention_weight_min": -39.22124099731445,
      "activations/layer11_attention_weight_max": 38.346031188964844,
      "activations/layer11_attention_weight_min": -37.50782775878906,
      "activations/layer12_attention_weight_max": 30.55482292175293,
      "activations/layer12_attention_weight_min": -29.488338470458984,
      "activations/layer13_attention_weight_max": 41.80522155761719,
      "activations/layer13_attention_weight_min": -38.00212860107422,
      "activations/layer14_attention_weight_max": 45.424888610839844,
      "activations/layer14_attention_weight_min": -38.8333625793457,
      "activations/layer15_attention_weight_max": 44.933956146240234,
      "activations/layer15_attention_weight_min": -38.54364013671875,
      "activations/layer16_attention_weight_max": 33.700801849365234,
      "activations/layer16_attention_weight_min": -29.286827087402344,
      "activations/layer17_attention_weight_max": 30.320810317993164,
      "activations/layer17_attention_weight_min": -29.44134521484375,
      "activations/layer18_attention_weight_max": 36.9748420715332,
      "activations/layer18_attention_weight_min": -28.044931411743164,
      "activations/layer19_attention_weight_max": 36.44088363647461,
      "activations/layer19_attention_weight_min": -32.69511795043945,
      "activations/layer1_attention_weight_max": 15.166115760803223,
      "activations/layer1_attention_weight_min": -13.71222972869873,
      "activations/layer20_attention_weight_max": 31.745037078857422,
      "activations/layer20_attention_weight_min": -24.71747398376465,
      "activations/layer21_attention_weight_max": 30.53261375427246,
      "activations/layer21_attention_weight_min": -24.230093002319336,
      "activations/layer22_attention_weight_max": 40.64570617675781,
      "activations/layer22_attention_weight_min": -31.158920288085938,
      "activations/layer23_attention_weight_max": 30.232749938964844,
      "activations/layer23_attention_weight_min": -24.928443908691406,
      "activations/layer2_attention_weight_max": 31.786893844604492,
      "activations/layer2_attention_weight_min": -29.939298629760742,
      "activations/layer3_attention_weight_max": 95.60673522949219,
      "activations/layer3_attention_weight_min": -98.50564575195312,
      "activations/layer4_attention_weight_max": 106.60367584228516,
      "activations/layer4_attention_weight_min": -103.81497955322266,
      "activations/layer5_attention_weight_max": 47.97875213623047,
      "activations/layer5_attention_weight_min": -61.495704650878906,
      "activations/layer6_attention_weight_max": 42.80656433105469,
      "activations/layer6_attention_weight_min": -47.12624740600586,
      "activations/layer7_attention_weight_max": 91.52652740478516,
      "activations/layer7_attention_weight_min": -98.14371490478516,
      "activations/layer8_attention_weight_max": 42.599639892578125,
      "activations/layer8_attention_weight_min": -45.461952209472656,
      "activations/layer9_attention_weight_max": 33.112850189208984,
      "activations/layer9_attention_weight_min": -36.2408332824707,
      "epoch": 19.51,
      "learning_rate": 2.441931818181818e-05,
      "loss": 2.7224,
      "step": 335800
    },
    {
      "activations/layer0_attention_weight_max": 16.70055389404297,
      "activations/layer0_attention_weight_min": -13.206855773925781,
      "activations/layer10_attention_weight_max": 33.89956283569336,
      "activations/layer10_attention_weight_min": -33.97871017456055,
      "activations/layer11_attention_weight_max": 33.66295623779297,
      "activations/layer11_attention_weight_min": -34.00205993652344,
      "activations/layer12_attention_weight_max": 26.984230041503906,
      "activations/layer12_attention_weight_min": -26.64763832092285,
      "activations/layer13_attention_weight_max": 40.11445617675781,
      "activations/layer13_attention_weight_min": -36.22178649902344,
      "activations/layer14_attention_weight_max": 44.58037185668945,
      "activations/layer14_attention_weight_min": -40.28178405761719,
      "activations/layer15_attention_weight_max": 39.2108039855957,
      "activations/layer15_attention_weight_min": -35.697471618652344,
      "activations/layer16_attention_weight_max": 31.631757736206055,
      "activations/layer16_attention_weight_min": -28.632747650146484,
      "activations/layer17_attention_weight_max": 31.41441535949707,
      "activations/layer17_attention_weight_min": -27.191186904907227,
      "activations/layer18_attention_weight_max": 35.289085388183594,
      "activations/layer18_attention_weight_min": -26.07774543762207,
      "activations/layer19_attention_weight_max": 35.451175689697266,
      "activations/layer19_attention_weight_min": -32.340782165527344,
      "activations/layer1_attention_weight_max": 15.041857719421387,
      "activations/layer1_attention_weight_min": -15.9752836227417,
      "activations/layer20_attention_weight_max": 30.295169830322266,
      "activations/layer20_attention_weight_min": -27.074678421020508,
      "activations/layer21_attention_weight_max": 32.03474807739258,
      "activations/layer21_attention_weight_min": -26.325931549072266,
      "activations/layer22_attention_weight_max": 45.03241729736328,
      "activations/layer22_attention_weight_min": -36.22256851196289,
      "activations/layer23_attention_weight_max": 34.36764144897461,
      "activations/layer23_attention_weight_min": -27.95442008972168,
      "activations/layer2_attention_weight_max": 33.17322540283203,
      "activations/layer2_attention_weight_min": -32.97846984863281,
      "activations/layer3_attention_weight_max": 99.45638275146484,
      "activations/layer3_attention_weight_min": -102.41725158691406,
      "activations/layer4_attention_weight_max": 106.8050537109375,
      "activations/layer4_attention_weight_min": -106.35640716552734,
      "activations/layer5_attention_weight_max": 52.77262878417969,
      "activations/layer5_attention_weight_min": -62.507286071777344,
      "activations/layer6_attention_weight_max": 44.44786071777344,
      "activations/layer6_attention_weight_min": -47.70439147949219,
      "activations/layer7_attention_weight_max": 93.96829986572266,
      "activations/layer7_attention_weight_min": -100.77812957763672,
      "activations/layer8_attention_weight_max": 42.16846466064453,
      "activations/layer8_attention_weight_min": -45.0590705871582,
      "activations/layer9_attention_weight_max": 32.854705810546875,
      "activations/layer9_attention_weight_min": -35.68663787841797,
      "epoch": 19.51,
      "learning_rate": 2.4400378787878783e-05,
      "loss": 2.7263,
      "step": 335850
    },
    {
      "activations/layer0_attention_weight_max": 15.80510425567627,
      "activations/layer0_attention_weight_min": -14.013218879699707,
      "activations/layer10_attention_weight_max": 35.569358825683594,
      "activations/layer10_attention_weight_min": -32.703330993652344,
      "activations/layer11_attention_weight_max": 34.654380798339844,
      "activations/layer11_attention_weight_min": -34.54507064819336,
      "activations/layer12_attention_weight_max": 27.791950225830078,
      "activations/layer12_attention_weight_min": -28.493173599243164,
      "activations/layer13_attention_weight_max": 43.22127914428711,
      "activations/layer13_attention_weight_min": -39.21794891357422,
      "activations/layer14_attention_weight_max": 42.970069885253906,
      "activations/layer14_attention_weight_min": -39.62337112426758,
      "activations/layer15_attention_weight_max": 40.75041580200195,
      "activations/layer15_attention_weight_min": -35.33998489379883,
      "activations/layer16_attention_weight_max": 34.735538482666016,
      "activations/layer16_attention_weight_min": -30.5985107421875,
      "activations/layer17_attention_weight_max": 29.43668556213379,
      "activations/layer17_attention_weight_min": -25.919038772583008,
      "activations/layer18_attention_weight_max": 31.219295501708984,
      "activations/layer18_attention_weight_min": -26.86774253845215,
      "activations/layer19_attention_weight_max": 45.4195442199707,
      "activations/layer19_attention_weight_min": -33.14498519897461,
      "activations/layer1_attention_weight_max": 15.628899574279785,
      "activations/layer1_attention_weight_min": -14.1000394821167,
      "activations/layer20_attention_weight_max": 33.16869354248047,
      "activations/layer20_attention_weight_min": -25.502605438232422,
      "activations/layer21_attention_weight_max": 30.725664138793945,
      "activations/layer21_attention_weight_min": -25.61655044555664,
      "activations/layer22_attention_weight_max": 45.326507568359375,
      "activations/layer22_attention_weight_min": -32.837520599365234,
      "activations/layer23_attention_weight_max": 39.14395523071289,
      "activations/layer23_attention_weight_min": -26.38190269470215,
      "activations/layer2_attention_weight_max": 31.883155822753906,
      "activations/layer2_attention_weight_min": -30.14897918701172,
      "activations/layer3_attention_weight_max": 102.25900268554688,
      "activations/layer3_attention_weight_min": -100.27631378173828,
      "activations/layer4_attention_weight_max": 109.94673156738281,
      "activations/layer4_attention_weight_min": -107.2579116821289,
      "activations/layer5_attention_weight_max": 50.861854553222656,
      "activations/layer5_attention_weight_min": -61.922603607177734,
      "activations/layer6_attention_weight_max": 43.64468002319336,
      "activations/layer6_attention_weight_min": -45.71949005126953,
      "activations/layer7_attention_weight_max": 96.13835906982422,
      "activations/layer7_attention_weight_min": -95.54354095458984,
      "activations/layer8_attention_weight_max": 42.13499450683594,
      "activations/layer8_attention_weight_min": -44.97166442871094,
      "activations/layer9_attention_weight_max": 32.352535247802734,
      "activations/layer9_attention_weight_min": -33.9004020690918,
      "epoch": 19.52,
      "learning_rate": 2.438143939393939e-05,
      "loss": 2.7141,
      "step": 335900
    },
    {
      "activations/layer0_attention_weight_max": 15.948131561279297,
      "activations/layer0_attention_weight_min": -12.3114595413208,
      "activations/layer10_attention_weight_max": 34.30354309082031,
      "activations/layer10_attention_weight_min": -36.001121520996094,
      "activations/layer11_attention_weight_max": 36.71345138549805,
      "activations/layer11_attention_weight_min": -35.945472717285156,
      "activations/layer12_attention_weight_max": 29.034582138061523,
      "activations/layer12_attention_weight_min": -28.461013793945312,
      "activations/layer13_attention_weight_max": 41.24697494506836,
      "activations/layer13_attention_weight_min": -37.92647933959961,
      "activations/layer14_attention_weight_max": 43.35667037963867,
      "activations/layer14_attention_weight_min": -37.76231384277344,
      "activations/layer15_attention_weight_max": 38.62491226196289,
      "activations/layer15_attention_weight_min": -36.06037521362305,
      "activations/layer16_attention_weight_max": 29.128660202026367,
      "activations/layer16_attention_weight_min": -27.98577880859375,
      "activations/layer17_attention_weight_max": 28.60715675354004,
      "activations/layer17_attention_weight_min": -26.659677505493164,
      "activations/layer18_attention_weight_max": 34.178550720214844,
      "activations/layer18_attention_weight_min": -26.3211612701416,
      "activations/layer19_attention_weight_max": 35.289432525634766,
      "activations/layer19_attention_weight_min": -34.71345901489258,
      "activations/layer1_attention_weight_max": 15.857852935791016,
      "activations/layer1_attention_weight_min": -13.144739151000977,
      "activations/layer20_attention_weight_max": 30.257266998291016,
      "activations/layer20_attention_weight_min": -26.799131393432617,
      "activations/layer21_attention_weight_max": 31.36836814880371,
      "activations/layer21_attention_weight_min": -26.08949851989746,
      "activations/layer22_attention_weight_max": 43.792049407958984,
      "activations/layer22_attention_weight_min": -33.032257080078125,
      "activations/layer23_attention_weight_max": 34.300514221191406,
      "activations/layer23_attention_weight_min": -26.48102378845215,
      "activations/layer2_attention_weight_max": 31.22443962097168,
      "activations/layer2_attention_weight_min": -31.168088912963867,
      "activations/layer3_attention_weight_max": 98.84577941894531,
      "activations/layer3_attention_weight_min": -99.74925231933594,
      "activations/layer4_attention_weight_max": 112.40752410888672,
      "activations/layer4_attention_weight_min": -115.21820068359375,
      "activations/layer5_attention_weight_max": 53.82389831542969,
      "activations/layer5_attention_weight_min": -64.35475158691406,
      "activations/layer6_attention_weight_max": 45.09025573730469,
      "activations/layer6_attention_weight_min": -48.38882064819336,
      "activations/layer7_attention_weight_max": 100.78339385986328,
      "activations/layer7_attention_weight_min": -95.82170867919922,
      "activations/layer8_attention_weight_max": 42.89606475830078,
      "activations/layer8_attention_weight_min": -43.53883743286133,
      "activations/layer9_attention_weight_max": 32.55294418334961,
      "activations/layer9_attention_weight_min": -34.35244369506836,
      "epoch": 19.52,
      "learning_rate": 2.4362499999999996e-05,
      "loss": 2.7217,
      "step": 335950
    },
    {
      "activations/layer0_attention_weight_max": 16.321868896484375,
      "activations/layer0_attention_weight_min": -12.809199333190918,
      "activations/layer10_attention_weight_max": 32.51286315917969,
      "activations/layer10_attention_weight_min": -32.09546661376953,
      "activations/layer11_attention_weight_max": 31.9046688079834,
      "activations/layer11_attention_weight_min": -32.69013214111328,
      "activations/layer12_attention_weight_max": 27.253416061401367,
      "activations/layer12_attention_weight_min": -27.371814727783203,
      "activations/layer13_attention_weight_max": 42.61481475830078,
      "activations/layer13_attention_weight_min": -35.994842529296875,
      "activations/layer14_attention_weight_max": 46.87101745605469,
      "activations/layer14_attention_weight_min": -39.942543029785156,
      "activations/layer15_attention_weight_max": 45.45689392089844,
      "activations/layer15_attention_weight_min": -40.5798454284668,
      "activations/layer16_attention_weight_max": 33.02523422241211,
      "activations/layer16_attention_weight_min": -28.46768569946289,
      "activations/layer17_attention_weight_max": 30.499080657958984,
      "activations/layer17_attention_weight_min": -27.003889083862305,
      "activations/layer18_attention_weight_max": 36.6743278503418,
      "activations/layer18_attention_weight_min": -25.266576766967773,
      "activations/layer19_attention_weight_max": 41.03655242919922,
      "activations/layer19_attention_weight_min": -30.090927124023438,
      "activations/layer1_attention_weight_max": 15.223051071166992,
      "activations/layer1_attention_weight_min": -13.430255889892578,
      "activations/layer20_attention_weight_max": 33.701908111572266,
      "activations/layer20_attention_weight_min": -25.587106704711914,
      "activations/layer21_attention_weight_max": 33.36287307739258,
      "activations/layer21_attention_weight_min": -23.925268173217773,
      "activations/layer22_attention_weight_max": 44.6291389465332,
      "activations/layer22_attention_weight_min": -32.96385955810547,
      "activations/layer23_attention_weight_max": 36.535675048828125,
      "activations/layer23_attention_weight_min": -26.444530487060547,
      "activations/layer2_attention_weight_max": 32.10620880126953,
      "activations/layer2_attention_weight_min": -29.951276779174805,
      "activations/layer3_attention_weight_max": 99.16130828857422,
      "activations/layer3_attention_weight_min": -98.73628997802734,
      "activations/layer4_attention_weight_max": 100.92523956298828,
      "activations/layer4_attention_weight_min": -105.51256561279297,
      "activations/layer5_attention_weight_max": 50.171966552734375,
      "activations/layer5_attention_weight_min": -59.800716400146484,
      "activations/layer6_attention_weight_max": 41.408294677734375,
      "activations/layer6_attention_weight_min": -43.77782440185547,
      "activations/layer7_attention_weight_max": 86.67597198486328,
      "activations/layer7_attention_weight_min": -87.90662384033203,
      "activations/layer8_attention_weight_max": 38.67444610595703,
      "activations/layer8_attention_weight_min": -40.587284088134766,
      "activations/layer9_attention_weight_max": 28.213703155517578,
      "activations/layer9_attention_weight_min": -31.442840576171875,
      "epoch": 19.52,
      "learning_rate": 2.4343560606060604e-05,
      "loss": 2.719,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.4644,
      "eval_samples_per_second": 507.304,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.4644,
      "eval_openwebtext_samples_per_second": 507.304,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9481,
      "eval_wikitext_samples_per_second": 234.072,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_lambada_loss": 2.376953125,
      "eval_lambada_ppl": 10.77203177799245,
      "eval_lambada_runtime": 9.5699,
      "eval_lambada_samples_per_second": 508.782,
      "step": 336000
    },
    {
      "activations/layer0_attention_weight_max": 16.49567413330078,
      "activations/layer0_attention_weight_min": -12.42314338684082,
      "activations/layer10_attention_weight_max": 32.0746955871582,
      "activations/layer10_attention_weight_min": -35.00812530517578,
      "activations/layer11_attention_weight_max": 33.851436614990234,
      "activations/layer11_attention_weight_min": -35.33634948730469,
      "activations/layer12_attention_weight_max": 27.691635131835938,
      "activations/layer12_attention_weight_min": -26.45206642150879,
      "activations/layer13_attention_weight_max": 37.6582145690918,
      "activations/layer13_attention_weight_min": -37.42976760864258,
      "activations/layer14_attention_weight_max": 41.12394332885742,
      "activations/layer14_attention_weight_min": -38.79588317871094,
      "activations/layer15_attention_weight_max": 39.978370666503906,
      "activations/layer15_attention_weight_min": -35.42000198364258,
      "activations/layer16_attention_weight_max": 30.49552345275879,
      "activations/layer16_attention_weight_min": -28.369386672973633,
      "activations/layer17_attention_weight_max": 30.74138832092285,
      "activations/layer17_attention_weight_min": -26.602577209472656,
      "activations/layer18_attention_weight_max": 33.54587936401367,
      "activations/layer18_attention_weight_min": -26.2251033782959,
      "activations/layer19_attention_weight_max": 32.793785095214844,
      "activations/layer19_attention_weight_min": -32.482608795166016,
      "activations/layer1_attention_weight_max": 15.613972663879395,
      "activations/layer1_attention_weight_min": -14.430349349975586,
      "activations/layer20_attention_weight_max": 28.141084671020508,
      "activations/layer20_attention_weight_min": -27.599674224853516,
      "activations/layer21_attention_weight_max": 26.406339645385742,
      "activations/layer21_attention_weight_min": -25.702001571655273,
      "activations/layer22_attention_weight_max": 45.066566467285156,
      "activations/layer22_attention_weight_min": -36.059417724609375,
      "activations/layer23_attention_weight_max": 31.704809188842773,
      "activations/layer23_attention_weight_min": -29.858518600463867,
      "activations/layer2_attention_weight_max": 33.17381286621094,
      "activations/layer2_attention_weight_min": -31.369754791259766,
      "activations/layer3_attention_weight_max": 100.47735595703125,
      "activations/layer3_attention_weight_min": -102.6327133178711,
      "activations/layer4_attention_weight_max": 105.81294250488281,
      "activations/layer4_attention_weight_min": -114.15929412841797,
      "activations/layer5_attention_weight_max": 48.99736785888672,
      "activations/layer5_attention_weight_min": -62.96680450439453,
      "activations/layer6_attention_weight_max": 46.97231674194336,
      "activations/layer6_attention_weight_min": -48.95246887207031,
      "activations/layer7_attention_weight_max": 93.5456771850586,
      "activations/layer7_attention_weight_min": -102.08889770507812,
      "activations/layer8_attention_weight_max": 41.61226272583008,
      "activations/layer8_attention_weight_min": -43.47121810913086,
      "activations/layer9_attention_weight_max": 30.86384391784668,
      "activations/layer9_attention_weight_min": -33.716705322265625,
      "epoch": 19.53,
      "learning_rate": 2.4324999999999995e-05,
      "loss": 2.7057,
      "step": 336050
    },
    {
      "activations/layer0_attention_weight_max": 16.06951141357422,
      "activations/layer0_attention_weight_min": -14.488043785095215,
      "activations/layer10_attention_weight_max": 34.342376708984375,
      "activations/layer10_attention_weight_min": -36.254127502441406,
      "activations/layer11_attention_weight_max": 34.05204772949219,
      "activations/layer11_attention_weight_min": -37.01063537597656,
      "activations/layer12_attention_weight_max": 27.04629135131836,
      "activations/layer12_attention_weight_min": -27.278230667114258,
      "activations/layer13_attention_weight_max": 41.34959030151367,
      "activations/layer13_attention_weight_min": -37.171112060546875,
      "activations/layer14_attention_weight_max": 44.3278923034668,
      "activations/layer14_attention_weight_min": -38.05381774902344,
      "activations/layer15_attention_weight_max": 43.690513610839844,
      "activations/layer15_attention_weight_min": -36.66954803466797,
      "activations/layer16_attention_weight_max": 31.640270233154297,
      "activations/layer16_attention_weight_min": -28.549192428588867,
      "activations/layer17_attention_weight_max": 30.295068740844727,
      "activations/layer17_attention_weight_min": -27.494911193847656,
      "activations/layer18_attention_weight_max": 39.53693771362305,
      "activations/layer18_attention_weight_min": -29.608604431152344,
      "activations/layer19_attention_weight_max": 41.431949615478516,
      "activations/layer19_attention_weight_min": -31.931018829345703,
      "activations/layer1_attention_weight_max": 15.71294116973877,
      "activations/layer1_attention_weight_min": -14.400677680969238,
      "activations/layer20_attention_weight_max": 33.072757720947266,
      "activations/layer20_attention_weight_min": -25.069889068603516,
      "activations/layer21_attention_weight_max": 34.62830352783203,
      "activations/layer21_attention_weight_min": -25.67116355895996,
      "activations/layer22_attention_weight_max": 47.93903350830078,
      "activations/layer22_attention_weight_min": -31.182918548583984,
      "activations/layer23_attention_weight_max": 38.74601745605469,
      "activations/layer23_attention_weight_min": -24.29471206665039,
      "activations/layer2_attention_weight_max": 33.276947021484375,
      "activations/layer2_attention_weight_min": -31.76232147216797,
      "activations/layer3_attention_weight_max": 97.89185333251953,
      "activations/layer3_attention_weight_min": -102.90994262695312,
      "activations/layer4_attention_weight_max": 107.09773254394531,
      "activations/layer4_attention_weight_min": -114.01335906982422,
      "activations/layer5_attention_weight_max": 51.28166198730469,
      "activations/layer5_attention_weight_min": -63.72391891479492,
      "activations/layer6_attention_weight_max": 46.234989166259766,
      "activations/layer6_attention_weight_min": -47.1396598815918,
      "activations/layer7_attention_weight_max": 97.26182556152344,
      "activations/layer7_attention_weight_min": -95.7385482788086,
      "activations/layer8_attention_weight_max": 41.45814895629883,
      "activations/layer8_attention_weight_min": -45.640602111816406,
      "activations/layer9_attention_weight_max": 32.16181564331055,
      "activations/layer9_attention_weight_min": -34.97011184692383,
      "epoch": 19.53,
      "learning_rate": 2.4306060606060603e-05,
      "loss": 2.7174,
      "step": 336100
    },
    {
      "activations/layer0_attention_weight_max": 16.307958602905273,
      "activations/layer0_attention_weight_min": -13.388460159301758,
      "activations/layer10_attention_weight_max": 37.28303527832031,
      "activations/layer10_attention_weight_min": -36.52476119995117,
      "activations/layer11_attention_weight_max": 36.82770538330078,
      "activations/layer11_attention_weight_min": -35.050498962402344,
      "activations/layer12_attention_weight_max": 29.130634307861328,
      "activations/layer12_attention_weight_min": -26.560976028442383,
      "activations/layer13_attention_weight_max": 40.81068420410156,
      "activations/layer13_attention_weight_min": -35.33309555053711,
      "activations/layer14_attention_weight_max": 45.57952880859375,
      "activations/layer14_attention_weight_min": -39.48554229736328,
      "activations/layer15_attention_weight_max": 39.124019622802734,
      "activations/layer15_attention_weight_min": -40.36787414550781,
      "activations/layer16_attention_weight_max": 33.666263580322266,
      "activations/layer16_attention_weight_min": -29.71303939819336,
      "activations/layer17_attention_weight_max": 28.56647491455078,
      "activations/layer17_attention_weight_min": -27.799264907836914,
      "activations/layer18_attention_weight_max": 29.59393882751465,
      "activations/layer18_attention_weight_min": -26.778045654296875,
      "activations/layer19_attention_weight_max": 30.858905792236328,
      "activations/layer19_attention_weight_min": -32.91783905029297,
      "activations/layer1_attention_weight_max": 15.773211479187012,
      "activations/layer1_attention_weight_min": -13.720991134643555,
      "activations/layer20_attention_weight_max": 27.12185287475586,
      "activations/layer20_attention_weight_min": -26.010873794555664,
      "activations/layer21_attention_weight_max": 26.750890731811523,
      "activations/layer21_attention_weight_min": -25.865875244140625,
      "activations/layer22_attention_weight_max": 40.5096435546875,
      "activations/layer22_attention_weight_min": -35.399925231933594,
      "activations/layer23_attention_weight_max": 31.644834518432617,
      "activations/layer23_attention_weight_min": -27.322261810302734,
      "activations/layer2_attention_weight_max": 31.747417449951172,
      "activations/layer2_attention_weight_min": -32.79496765136719,
      "activations/layer3_attention_weight_max": 99.95597839355469,
      "activations/layer3_attention_weight_min": -96.96996307373047,
      "activations/layer4_attention_weight_max": 107.05284881591797,
      "activations/layer4_attention_weight_min": -103.08971405029297,
      "activations/layer5_attention_weight_max": 53.77936553955078,
      "activations/layer5_attention_weight_min": -58.73781967163086,
      "activations/layer6_attention_weight_max": 45.98112487792969,
      "activations/layer6_attention_weight_min": -48.03596496582031,
      "activations/layer7_attention_weight_max": 94.80911254882812,
      "activations/layer7_attention_weight_min": -101.97970581054688,
      "activations/layer8_attention_weight_max": 46.45668411254883,
      "activations/layer8_attention_weight_min": -47.99850082397461,
      "activations/layer9_attention_weight_max": 35.42882537841797,
      "activations/layer9_attention_weight_min": -36.983116149902344,
      "epoch": 19.53,
      "learning_rate": 2.428712121212121e-05,
      "loss": 2.7109,
      "step": 336150
    },
    {
      "activations/layer0_attention_weight_max": 16.451019287109375,
      "activations/layer0_attention_weight_min": -14.271173477172852,
      "activations/layer10_attention_weight_max": 36.22922897338867,
      "activations/layer10_attention_weight_min": -38.35321044921875,
      "activations/layer11_attention_weight_max": 34.23405075073242,
      "activations/layer11_attention_weight_min": -36.52605438232422,
      "activations/layer12_attention_weight_max": 25.05795669555664,
      "activations/layer12_attention_weight_min": -27.216039657592773,
      "activations/layer13_attention_weight_max": 36.62944793701172,
      "activations/layer13_attention_weight_min": -35.27562713623047,
      "activations/layer14_attention_weight_max": 39.95146179199219,
      "activations/layer14_attention_weight_min": -38.31941604614258,
      "activations/layer15_attention_weight_max": 35.84623336791992,
      "activations/layer15_attention_weight_min": -34.368099212646484,
      "activations/layer16_attention_weight_max": 29.48113250732422,
      "activations/layer16_attention_weight_min": -28.72842025756836,
      "activations/layer17_attention_weight_max": 29.284000396728516,
      "activations/layer17_attention_weight_min": -26.524559020996094,
      "activations/layer18_attention_weight_max": 29.978744506835938,
      "activations/layer18_attention_weight_min": -26.19532012939453,
      "activations/layer19_attention_weight_max": 30.273324966430664,
      "activations/layer19_attention_weight_min": -30.261341094970703,
      "activations/layer1_attention_weight_max": 15.34699821472168,
      "activations/layer1_attention_weight_min": -14.703866004943848,
      "activations/layer20_attention_weight_max": 26.39937400817871,
      "activations/layer20_attention_weight_min": -25.7489013671875,
      "activations/layer21_attention_weight_max": 27.77615737915039,
      "activations/layer21_attention_weight_min": -25.933115005493164,
      "activations/layer22_attention_weight_max": 39.86160659790039,
      "activations/layer22_attention_weight_min": -32.386009216308594,
      "activations/layer23_attention_weight_max": 29.573848724365234,
      "activations/layer23_attention_weight_min": -25.09876823425293,
      "activations/layer2_attention_weight_max": 30.999191284179688,
      "activations/layer2_attention_weight_min": -31.350215911865234,
      "activations/layer3_attention_weight_max": 97.61421203613281,
      "activations/layer3_attention_weight_min": -97.09371948242188,
      "activations/layer4_attention_weight_max": 104.85489654541016,
      "activations/layer4_attention_weight_min": -103.39763641357422,
      "activations/layer5_attention_weight_max": 53.855255126953125,
      "activations/layer5_attention_weight_min": -62.083412170410156,
      "activations/layer6_attention_weight_max": 44.89539337158203,
      "activations/layer6_attention_weight_min": -46.666175842285156,
      "activations/layer7_attention_weight_max": 92.31986999511719,
      "activations/layer7_attention_weight_min": -98.82955932617188,
      "activations/layer8_attention_weight_max": 42.164241790771484,
      "activations/layer8_attention_weight_min": -44.02978515625,
      "activations/layer9_attention_weight_max": 33.219932556152344,
      "activations/layer9_attention_weight_min": -34.599971771240234,
      "epoch": 19.54,
      "learning_rate": 2.4268181818181816e-05,
      "loss": 2.7327,
      "step": 336200
    },
    {
      "activations/layer0_attention_weight_max": 16.772275924682617,
      "activations/layer0_attention_weight_min": -12.263960838317871,
      "activations/layer10_attention_weight_max": 49.69690704345703,
      "activations/layer10_attention_weight_min": -46.30329132080078,
      "activations/layer11_attention_weight_max": 46.067893981933594,
      "activations/layer11_attention_weight_min": -41.82100296020508,
      "activations/layer12_attention_weight_max": 32.373695373535156,
      "activations/layer12_attention_weight_min": -30.918285369873047,
      "activations/layer13_attention_weight_max": 45.41770935058594,
      "activations/layer13_attention_weight_min": -40.04020690917969,
      "activations/layer14_attention_weight_max": 56.23487091064453,
      "activations/layer14_attention_weight_min": -41.65398406982422,
      "activations/layer15_attention_weight_max": 54.749324798583984,
      "activations/layer15_attention_weight_min": -41.01152038574219,
      "activations/layer16_attention_weight_max": 41.448211669921875,
      "activations/layer16_attention_weight_min": -31.812883377075195,
      "activations/layer17_attention_weight_max": 38.695499420166016,
      "activations/layer17_attention_weight_min": -31.23874282836914,
      "activations/layer18_attention_weight_max": 37.25226593017578,
      "activations/layer18_attention_weight_min": -32.066532135009766,
      "activations/layer19_attention_weight_max": 37.441287994384766,
      "activations/layer19_attention_weight_min": -34.79500961303711,
      "activations/layer1_attention_weight_max": 15.890765190124512,
      "activations/layer1_attention_weight_min": -14.783154487609863,
      "activations/layer20_attention_weight_max": 30.661531448364258,
      "activations/layer20_attention_weight_min": -25.854984283447266,
      "activations/layer21_attention_weight_max": 31.979368209838867,
      "activations/layer21_attention_weight_min": -25.490915298461914,
      "activations/layer22_attention_weight_max": 49.78176498413086,
      "activations/layer22_attention_weight_min": -35.18790054321289,
      "activations/layer23_attention_weight_max": 35.23321533203125,
      "activations/layer23_attention_weight_min": -28.05849838256836,
      "activations/layer2_attention_weight_max": 32.93995666503906,
      "activations/layer2_attention_weight_min": -34.794185638427734,
      "activations/layer3_attention_weight_max": 102.2806167602539,
      "activations/layer3_attention_weight_min": -108.99728393554688,
      "activations/layer4_attention_weight_max": 112.55049896240234,
      "activations/layer4_attention_weight_min": -116.0545883178711,
      "activations/layer5_attention_weight_max": 56.96266174316406,
      "activations/layer5_attention_weight_min": -67.97021484375,
      "activations/layer6_attention_weight_max": 48.53785705566406,
      "activations/layer6_attention_weight_min": -51.74399185180664,
      "activations/layer7_attention_weight_max": 112.296875,
      "activations/layer7_attention_weight_min": -111.8513412475586,
      "activations/layer8_attention_weight_max": 53.3759651184082,
      "activations/layer8_attention_weight_min": -52.22246170043945,
      "activations/layer9_attention_weight_max": 44.7100944519043,
      "activations/layer9_attention_weight_min": -44.67715835571289,
      "epoch": 19.54,
      "learning_rate": 2.4249242424242425e-05,
      "loss": 2.7173,
      "step": 336250
    },
    {
      "activations/layer0_attention_weight_max": 16.906951904296875,
      "activations/layer0_attention_weight_min": -13.386946678161621,
      "activations/layer10_attention_weight_max": 37.96928787231445,
      "activations/layer10_attention_weight_min": -36.851844787597656,
      "activations/layer11_attention_weight_max": 36.70320129394531,
      "activations/layer11_attention_weight_min": -36.11594772338867,
      "activations/layer12_attention_weight_max": 28.184389114379883,
      "activations/layer12_attention_weight_min": -26.30471420288086,
      "activations/layer13_attention_weight_max": 40.35003662109375,
      "activations/layer13_attention_weight_min": -36.319881439208984,
      "activations/layer14_attention_weight_max": 43.010986328125,
      "activations/layer14_attention_weight_min": -39.01931381225586,
      "activations/layer15_attention_weight_max": 37.736167907714844,
      "activations/layer15_attention_weight_min": -36.2956657409668,
      "activations/layer16_attention_weight_max": 29.30939483642578,
      "activations/layer16_attention_weight_min": -27.139429092407227,
      "activations/layer17_attention_weight_max": 29.430519104003906,
      "activations/layer17_attention_weight_min": -27.923070907592773,
      "activations/layer18_attention_weight_max": 35.6971549987793,
      "activations/layer18_attention_weight_min": -28.97320556640625,
      "activations/layer19_attention_weight_max": 33.02983856201172,
      "activations/layer19_attention_weight_min": -31.096948623657227,
      "activations/layer1_attention_weight_max": 16.34080696105957,
      "activations/layer1_attention_weight_min": -14.93944263458252,
      "activations/layer20_attention_weight_max": 28.01940155029297,
      "activations/layer20_attention_weight_min": -24.102556228637695,
      "activations/layer21_attention_weight_max": 27.09842300415039,
      "activations/layer21_attention_weight_min": -24.75119400024414,
      "activations/layer22_attention_weight_max": 40.861595153808594,
      "activations/layer22_attention_weight_min": -33.321773529052734,
      "activations/layer23_attention_weight_max": 33.67048263549805,
      "activations/layer23_attention_weight_min": -25.07086181640625,
      "activations/layer2_attention_weight_max": 32.91649627685547,
      "activations/layer2_attention_weight_min": -32.51205825805664,
      "activations/layer3_attention_weight_max": 102.96980285644531,
      "activations/layer3_attention_weight_min": -103.42548370361328,
      "activations/layer4_attention_weight_max": 108.44857025146484,
      "activations/layer4_attention_weight_min": -109.8583984375,
      "activations/layer5_attention_weight_max": 57.421302795410156,
      "activations/layer5_attention_weight_min": -60.71012496948242,
      "activations/layer6_attention_weight_max": 46.05617904663086,
      "activations/layer6_attention_weight_min": -48.60722351074219,
      "activations/layer7_attention_weight_max": 99.83238983154297,
      "activations/layer7_attention_weight_min": -101.05583953857422,
      "activations/layer8_attention_weight_max": 47.1747932434082,
      "activations/layer8_attention_weight_min": -46.037078857421875,
      "activations/layer9_attention_weight_max": 34.77733612060547,
      "activations/layer9_attention_weight_min": -34.84940719604492,
      "epoch": 19.54,
      "learning_rate": 2.4230303030303026e-05,
      "loss": 2.7233,
      "step": 336300
    },
    {
      "activations/layer0_attention_weight_max": 16.255809783935547,
      "activations/layer0_attention_weight_min": -12.415552139282227,
      "activations/layer10_attention_weight_max": 35.31021499633789,
      "activations/layer10_attention_weight_min": -35.604923248291016,
      "activations/layer11_attention_weight_max": 34.652366638183594,
      "activations/layer11_attention_weight_min": -36.54842758178711,
      "activations/layer12_attention_weight_max": 28.7451114654541,
      "activations/layer12_attention_weight_min": -26.990047454833984,
      "activations/layer13_attention_weight_max": 43.256690979003906,
      "activations/layer13_attention_weight_min": -38.5222282409668,
      "activations/layer14_attention_weight_max": 44.178466796875,
      "activations/layer14_attention_weight_min": -39.64832305908203,
      "activations/layer15_attention_weight_max": 39.64392852783203,
      "activations/layer15_attention_weight_min": -36.303184509277344,
      "activations/layer16_attention_weight_max": 29.51114273071289,
      "activations/layer16_attention_weight_min": -27.73943328857422,
      "activations/layer17_attention_weight_max": 29.61160659790039,
      "activations/layer17_attention_weight_min": -26.983869552612305,
      "activations/layer18_attention_weight_max": 34.24103927612305,
      "activations/layer18_attention_weight_min": -25.693151473999023,
      "activations/layer19_attention_weight_max": 36.565486907958984,
      "activations/layer19_attention_weight_min": -31.06333351135254,
      "activations/layer1_attention_weight_max": 16.173866271972656,
      "activations/layer1_attention_weight_min": -13.893824577331543,
      "activations/layer20_attention_weight_max": 29.243562698364258,
      "activations/layer20_attention_weight_min": -23.98954963684082,
      "activations/layer21_attention_weight_max": 30.1231689453125,
      "activations/layer21_attention_weight_min": -23.970972061157227,
      "activations/layer22_attention_weight_max": 42.13079833984375,
      "activations/layer22_attention_weight_min": -31.593904495239258,
      "activations/layer23_attention_weight_max": 33.756797790527344,
      "activations/layer23_attention_weight_min": -25.435264587402344,
      "activations/layer2_attention_weight_max": 33.3858757019043,
      "activations/layer2_attention_weight_min": -31.824766159057617,
      "activations/layer3_attention_weight_max": 99.2848892211914,
      "activations/layer3_attention_weight_min": -103.64667510986328,
      "activations/layer4_attention_weight_max": 107.67317199707031,
      "activations/layer4_attention_weight_min": -109.2191162109375,
      "activations/layer5_attention_weight_max": 52.07648468017578,
      "activations/layer5_attention_weight_min": -58.62461853027344,
      "activations/layer6_attention_weight_max": 46.174800872802734,
      "activations/layer6_attention_weight_min": -48.66382598876953,
      "activations/layer7_attention_weight_max": 93.63525390625,
      "activations/layer7_attention_weight_min": -105.33000946044922,
      "activations/layer8_attention_weight_max": 42.5090217590332,
      "activations/layer8_attention_weight_min": -45.888248443603516,
      "activations/layer9_attention_weight_max": 33.69471740722656,
      "activations/layer9_attention_weight_min": -35.5514030456543,
      "epoch": 19.54,
      "learning_rate": 2.4211363636363634e-05,
      "loss": 2.7287,
      "step": 336350
    },
    {
      "activations/layer0_attention_weight_max": 15.973172187805176,
      "activations/layer0_attention_weight_min": -13.802155494689941,
      "activations/layer10_attention_weight_max": 39.18564987182617,
      "activations/layer10_attention_weight_min": -36.20395278930664,
      "activations/layer11_attention_weight_max": 39.28668212890625,
      "activations/layer11_attention_weight_min": -35.665435791015625,
      "activations/layer12_attention_weight_max": 28.555356979370117,
      "activations/layer12_attention_weight_min": -29.134510040283203,
      "activations/layer13_attention_weight_max": 39.911712646484375,
      "activations/layer13_attention_weight_min": -41.63843536376953,
      "activations/layer14_attention_weight_max": 46.914817810058594,
      "activations/layer14_attention_weight_min": -44.95867156982422,
      "activations/layer15_attention_weight_max": 43.34324645996094,
      "activations/layer15_attention_weight_min": -43.57122802734375,
      "activations/layer16_attention_weight_max": 33.25040054321289,
      "activations/layer16_attention_weight_min": -28.21014404296875,
      "activations/layer17_attention_weight_max": 29.176862716674805,
      "activations/layer17_attention_weight_min": -27.39100456237793,
      "activations/layer18_attention_weight_max": 28.76848030090332,
      "activations/layer18_attention_weight_min": -27.52214241027832,
      "activations/layer19_attention_weight_max": 34.08895492553711,
      "activations/layer19_attention_weight_min": -35.79280471801758,
      "activations/layer1_attention_weight_max": 15.656593322753906,
      "activations/layer1_attention_weight_min": -15.00069808959961,
      "activations/layer20_attention_weight_max": 28.801536560058594,
      "activations/layer20_attention_weight_min": -28.9472599029541,
      "activations/layer21_attention_weight_max": 26.52602767944336,
      "activations/layer21_attention_weight_min": -29.259366989135742,
      "activations/layer22_attention_weight_max": 41.73126983642578,
      "activations/layer22_attention_weight_min": -36.017555236816406,
      "activations/layer23_attention_weight_max": 30.420631408691406,
      "activations/layer23_attention_weight_min": -31.418813705444336,
      "activations/layer2_attention_weight_max": 32.32374954223633,
      "activations/layer2_attention_weight_min": -31.85585594177246,
      "activations/layer3_attention_weight_max": 99.1181411743164,
      "activations/layer3_attention_weight_min": -98.43965148925781,
      "activations/layer4_attention_weight_max": 106.749267578125,
      "activations/layer4_attention_weight_min": -108.7094497680664,
      "activations/layer5_attention_weight_max": 54.987998962402344,
      "activations/layer5_attention_weight_min": -62.785682678222656,
      "activations/layer6_attention_weight_max": 45.34718322753906,
      "activations/layer6_attention_weight_min": -46.32819747924805,
      "activations/layer7_attention_weight_max": 93.92023468017578,
      "activations/layer7_attention_weight_min": -102.7933120727539,
      "activations/layer8_attention_weight_max": 44.55525588989258,
      "activations/layer8_attention_weight_min": -43.684085845947266,
      "activations/layer9_attention_weight_max": 36.580467224121094,
      "activations/layer9_attention_weight_min": -36.22754669189453,
      "epoch": 19.55,
      "learning_rate": 2.419242424242424e-05,
      "loss": 2.7338,
      "step": 336400
    },
    {
      "activations/layer0_attention_weight_max": 16.3649845123291,
      "activations/layer0_attention_weight_min": -12.74583911895752,
      "activations/layer10_attention_weight_max": 37.12372970581055,
      "activations/layer10_attention_weight_min": -39.05982971191406,
      "activations/layer11_attention_weight_max": 38.78502655029297,
      "activations/layer11_attention_weight_min": -38.460487365722656,
      "activations/layer12_attention_weight_max": 30.662029266357422,
      "activations/layer12_attention_weight_min": -31.47415542602539,
      "activations/layer13_attention_weight_max": 48.31282424926758,
      "activations/layer13_attention_weight_min": -41.43345260620117,
      "activations/layer14_attention_weight_max": 50.527626037597656,
      "activations/layer14_attention_weight_min": -45.89220428466797,
      "activations/layer15_attention_weight_max": 47.30744552612305,
      "activations/layer15_attention_weight_min": -41.18095397949219,
      "activations/layer16_attention_weight_max": 34.657169342041016,
      "activations/layer16_attention_weight_min": -28.926963806152344,
      "activations/layer17_attention_weight_max": 35.12874221801758,
      "activations/layer17_attention_weight_min": -29.159175872802734,
      "activations/layer18_attention_weight_max": 35.63775634765625,
      "activations/layer18_attention_weight_min": -27.471590042114258,
      "activations/layer19_attention_weight_max": 44.11882781982422,
      "activations/layer19_attention_weight_min": -35.17003631591797,
      "activations/layer1_attention_weight_max": 15.766396522521973,
      "activations/layer1_attention_weight_min": -13.8187894821167,
      "activations/layer20_attention_weight_max": 34.11951446533203,
      "activations/layer20_attention_weight_min": -28.40145492553711,
      "activations/layer21_attention_weight_max": 33.154937744140625,
      "activations/layer21_attention_weight_min": -29.031295776367188,
      "activations/layer22_attention_weight_max": 48.95673370361328,
      "activations/layer22_attention_weight_min": -35.58384323120117,
      "activations/layer23_attention_weight_max": 40.9893798828125,
      "activations/layer23_attention_weight_min": -28.435001373291016,
      "activations/layer2_attention_weight_max": 33.10624313354492,
      "activations/layer2_attention_weight_min": -32.492523193359375,
      "activations/layer3_attention_weight_max": 102.39058685302734,
      "activations/layer3_attention_weight_min": -101.52225494384766,
      "activations/layer4_attention_weight_max": 111.8666000366211,
      "activations/layer4_attention_weight_min": -114.83125305175781,
      "activations/layer5_attention_weight_max": 51.25678253173828,
      "activations/layer5_attention_weight_min": -66.87899017333984,
      "activations/layer6_attention_weight_max": 46.27668762207031,
      "activations/layer6_attention_weight_min": -49.90802764892578,
      "activations/layer7_attention_weight_max": 103.36947631835938,
      "activations/layer7_attention_weight_min": -108.82817840576172,
      "activations/layer8_attention_weight_max": 47.17595291137695,
      "activations/layer8_attention_weight_min": -49.058677673339844,
      "activations/layer9_attention_weight_max": 34.38778305053711,
      "activations/layer9_attention_weight_min": -37.41923141479492,
      "epoch": 19.55,
      "learning_rate": 2.4173484848484848e-05,
      "loss": 2.7323,
      "step": 336450
    },
    {
      "activations/layer0_attention_weight_max": 16.001428604125977,
      "activations/layer0_attention_weight_min": -13.377445220947266,
      "activations/layer10_attention_weight_max": 38.397064208984375,
      "activations/layer10_attention_weight_min": -36.26015090942383,
      "activations/layer11_attention_weight_max": 38.053768157958984,
      "activations/layer11_attention_weight_min": -37.07524871826172,
      "activations/layer12_attention_weight_max": 30.234678268432617,
      "activations/layer12_attention_weight_min": -27.482803344726562,
      "activations/layer13_attention_weight_max": 39.47291564941406,
      "activations/layer13_attention_weight_min": -34.803890228271484,
      "activations/layer14_attention_weight_max": 40.769187927246094,
      "activations/layer14_attention_weight_min": -37.20756530761719,
      "activations/layer15_attention_weight_max": 39.43611145019531,
      "activations/layer15_attention_weight_min": -35.71879577636719,
      "activations/layer16_attention_weight_max": 28.15745735168457,
      "activations/layer16_attention_weight_min": -28.30437660217285,
      "activations/layer17_attention_weight_max": 28.56978988647461,
      "activations/layer17_attention_weight_min": -26.51683235168457,
      "activations/layer18_attention_weight_max": 31.716693878173828,
      "activations/layer18_attention_weight_min": -27.472270965576172,
      "activations/layer19_attention_weight_max": 31.593900680541992,
      "activations/layer19_attention_weight_min": -31.002517700195312,
      "activations/layer1_attention_weight_max": 15.753714561462402,
      "activations/layer1_attention_weight_min": -15.85689926147461,
      "activations/layer20_attention_weight_max": 26.907602310180664,
      "activations/layer20_attention_weight_min": -24.473526000976562,
      "activations/layer21_attention_weight_max": 27.85103416442871,
      "activations/layer21_attention_weight_min": -24.96611976623535,
      "activations/layer22_attention_weight_max": 40.611244201660156,
      "activations/layer22_attention_weight_min": -31.870595932006836,
      "activations/layer23_attention_weight_max": 29.898502349853516,
      "activations/layer23_attention_weight_min": -24.592649459838867,
      "activations/layer2_attention_weight_max": 31.332199096679688,
      "activations/layer2_attention_weight_min": -29.779380798339844,
      "activations/layer3_attention_weight_max": 93.53669738769531,
      "activations/layer3_attention_weight_min": -94.92140197753906,
      "activations/layer4_attention_weight_max": 101.03890228271484,
      "activations/layer4_attention_weight_min": -102.95552825927734,
      "activations/layer5_attention_weight_max": 48.539974212646484,
      "activations/layer5_attention_weight_min": -65.08634948730469,
      "activations/layer6_attention_weight_max": 43.936012268066406,
      "activations/layer6_attention_weight_min": -46.53286361694336,
      "activations/layer7_attention_weight_max": 102.37527465820312,
      "activations/layer7_attention_weight_min": -95.314697265625,
      "activations/layer8_attention_weight_max": 43.39239501953125,
      "activations/layer8_attention_weight_min": -43.50672912597656,
      "activations/layer9_attention_weight_max": 35.85548400878906,
      "activations/layer9_attention_weight_min": -35.414337158203125,
      "epoch": 19.55,
      "learning_rate": 2.4154545454545452e-05,
      "loss": 2.7126,
      "step": 336500
    },
    {
      "activations/layer0_attention_weight_max": 16.066171646118164,
      "activations/layer0_attention_weight_min": -12.77091121673584,
      "activations/layer10_attention_weight_max": 34.59758377075195,
      "activations/layer10_attention_weight_min": -36.42988586425781,
      "activations/layer11_attention_weight_max": 34.32511520385742,
      "activations/layer11_attention_weight_min": -35.34804916381836,
      "activations/layer12_attention_weight_max": 26.610774993896484,
      "activations/layer12_attention_weight_min": -27.308734893798828,
      "activations/layer13_attention_weight_max": 39.87782669067383,
      "activations/layer13_attention_weight_min": -39.19962692260742,
      "activations/layer14_attention_weight_max": 42.771751403808594,
      "activations/layer14_attention_weight_min": -40.53066635131836,
      "activations/layer15_attention_weight_max": 38.39495086669922,
      "activations/layer15_attention_weight_min": -38.18035888671875,
      "activations/layer16_attention_weight_max": 29.068370819091797,
      "activations/layer16_attention_weight_min": -29.216520309448242,
      "activations/layer17_attention_weight_max": 28.029991149902344,
      "activations/layer17_attention_weight_min": -28.46375846862793,
      "activations/layer18_attention_weight_max": 29.553104400634766,
      "activations/layer18_attention_weight_min": -27.7626953125,
      "activations/layer19_attention_weight_max": 31.639822006225586,
      "activations/layer19_attention_weight_min": -32.03654098510742,
      "activations/layer1_attention_weight_max": 15.392111778259277,
      "activations/layer1_attention_weight_min": -13.927789688110352,
      "activations/layer20_attention_weight_max": 26.998361587524414,
      "activations/layer20_attention_weight_min": -25.46553611755371,
      "activations/layer21_attention_weight_max": 27.077922821044922,
      "activations/layer21_attention_weight_min": -25.54674530029297,
      "activations/layer22_attention_weight_max": 39.58205795288086,
      "activations/layer22_attention_weight_min": -31.63776969909668,
      "activations/layer23_attention_weight_max": 33.243370056152344,
      "activations/layer23_attention_weight_min": -25.242551803588867,
      "activations/layer2_attention_weight_max": 30.843069076538086,
      "activations/layer2_attention_weight_min": -31.177005767822266,
      "activations/layer3_attention_weight_max": 99.61677551269531,
      "activations/layer3_attention_weight_min": -98.36897277832031,
      "activations/layer4_attention_weight_max": 103.68607330322266,
      "activations/layer4_attention_weight_min": -107.49431610107422,
      "activations/layer5_attention_weight_max": 50.223106384277344,
      "activations/layer5_attention_weight_min": -63.700111389160156,
      "activations/layer6_attention_weight_max": 45.40890884399414,
      "activations/layer6_attention_weight_min": -48.59095764160156,
      "activations/layer7_attention_weight_max": 97.27290344238281,
      "activations/layer7_attention_weight_min": -100.5929183959961,
      "activations/layer8_attention_weight_max": 43.89584732055664,
      "activations/layer8_attention_weight_min": -45.37295913696289,
      "activations/layer9_attention_weight_max": 31.816455841064453,
      "activations/layer9_attention_weight_min": -35.26133346557617,
      "epoch": 19.56,
      "learning_rate": 2.413560606060606e-05,
      "loss": 2.7307,
      "step": 336550
    },
    {
      "activations/layer0_attention_weight_max": 15.913817405700684,
      "activations/layer0_attention_weight_min": -12.38072395324707,
      "activations/layer10_attention_weight_max": 49.6104736328125,
      "activations/layer10_attention_weight_min": -47.68098831176758,
      "activations/layer11_attention_weight_max": 49.219322204589844,
      "activations/layer11_attention_weight_min": -49.04218292236328,
      "activations/layer12_attention_weight_max": 32.87422180175781,
      "activations/layer12_attention_weight_min": -33.347816467285156,
      "activations/layer13_attention_weight_max": 49.990440368652344,
      "activations/layer13_attention_weight_min": -43.999454498291016,
      "activations/layer14_attention_weight_max": 54.118621826171875,
      "activations/layer14_attention_weight_min": -47.20866012573242,
      "activations/layer15_attention_weight_max": 48.97214126586914,
      "activations/layer15_attention_weight_min": -45.78308868408203,
      "activations/layer16_attention_weight_max": 31.294937133789062,
      "activations/layer16_attention_weight_min": -31.050174713134766,
      "activations/layer17_attention_weight_max": 30.03263282775879,
      "activations/layer17_attention_weight_min": -28.21476173400879,
      "activations/layer18_attention_weight_max": 31.80286979675293,
      "activations/layer18_attention_weight_min": -28.26618766784668,
      "activations/layer19_attention_weight_max": 32.4077262878418,
      "activations/layer19_attention_weight_min": -31.950891494750977,
      "activations/layer1_attention_weight_max": 17.43584442138672,
      "activations/layer1_attention_weight_min": -14.614117622375488,
      "activations/layer20_attention_weight_max": 25.347637176513672,
      "activations/layer20_attention_weight_min": -24.024869918823242,
      "activations/layer21_attention_weight_max": 25.510160446166992,
      "activations/layer21_attention_weight_min": -23.58129119873047,
      "activations/layer22_attention_weight_max": 41.2846794128418,
      "activations/layer22_attention_weight_min": -30.791337966918945,
      "activations/layer23_attention_weight_max": 32.92389678955078,
      "activations/layer23_attention_weight_min": -25.249197006225586,
      "activations/layer2_attention_weight_max": 32.079193115234375,
      "activations/layer2_attention_weight_min": -32.49669647216797,
      "activations/layer3_attention_weight_max": 105.29711151123047,
      "activations/layer3_attention_weight_min": -108.32259368896484,
      "activations/layer4_attention_weight_max": 110.83061981201172,
      "activations/layer4_attention_weight_min": -104.64896392822266,
      "activations/layer5_attention_weight_max": 52.26923370361328,
      "activations/layer5_attention_weight_min": -60.77050018310547,
      "activations/layer6_attention_weight_max": 49.74417495727539,
      "activations/layer6_attention_weight_min": -47.83036422729492,
      "activations/layer7_attention_weight_max": 118.74642181396484,
      "activations/layer7_attention_weight_min": -112.38614654541016,
      "activations/layer8_attention_weight_max": 55.703819274902344,
      "activations/layer8_attention_weight_min": -53.816165924072266,
      "activations/layer9_attention_weight_max": 43.837974548339844,
      "activations/layer9_attention_weight_min": -45.72233200073242,
      "epoch": 19.56,
      "learning_rate": 2.4116666666666662e-05,
      "loss": 2.7234,
      "step": 336600
    },
    {
      "activations/layer0_attention_weight_max": 16.02497100830078,
      "activations/layer0_attention_weight_min": -14.148118019104004,
      "activations/layer10_attention_weight_max": 33.36383819580078,
      "activations/layer10_attention_weight_min": -35.98527526855469,
      "activations/layer11_attention_weight_max": 33.51581954956055,
      "activations/layer11_attention_weight_min": -35.32610321044922,
      "activations/layer12_attention_weight_max": 26.061506271362305,
      "activations/layer12_attention_weight_min": -26.332393646240234,
      "activations/layer13_attention_weight_max": 38.31183624267578,
      "activations/layer13_attention_weight_min": -36.62382888793945,
      "activations/layer14_attention_weight_max": 40.65110397338867,
      "activations/layer14_attention_weight_min": -37.0444221496582,
      "activations/layer15_attention_weight_max": 37.526309967041016,
      "activations/layer15_attention_weight_min": -36.94819641113281,
      "activations/layer16_attention_weight_max": 28.77983283996582,
      "activations/layer16_attention_weight_min": -28.29649543762207,
      "activations/layer17_attention_weight_max": 29.467193603515625,
      "activations/layer17_attention_weight_min": -26.35899543762207,
      "activations/layer18_attention_weight_max": 30.272733688354492,
      "activations/layer18_attention_weight_min": -26.464256286621094,
      "activations/layer19_attention_weight_max": 34.04887771606445,
      "activations/layer19_attention_weight_min": -31.64822769165039,
      "activations/layer1_attention_weight_max": 15.58592414855957,
      "activations/layer1_attention_weight_min": -13.72047233581543,
      "activations/layer20_attention_weight_max": 31.783496856689453,
      "activations/layer20_attention_weight_min": -24.471580505371094,
      "activations/layer21_attention_weight_max": 28.544662475585938,
      "activations/layer21_attention_weight_min": -25.244821548461914,
      "activations/layer22_attention_weight_max": 43.263423919677734,
      "activations/layer22_attention_weight_min": -30.78208351135254,
      "activations/layer23_attention_weight_max": 30.993741989135742,
      "activations/layer23_attention_weight_min": -26.15821647644043,
      "activations/layer2_attention_weight_max": 30.83174705505371,
      "activations/layer2_attention_weight_min": -30.764591217041016,
      "activations/layer3_attention_weight_max": 100.96525573730469,
      "activations/layer3_attention_weight_min": -96.33822631835938,
      "activations/layer4_attention_weight_max": 110.9258041381836,
      "activations/layer4_attention_weight_min": -104.43727111816406,
      "activations/layer5_attention_weight_max": 56.836524963378906,
      "activations/layer5_attention_weight_min": -62.86022186279297,
      "activations/layer6_attention_weight_max": 42.649070739746094,
      "activations/layer6_attention_weight_min": -46.917659759521484,
      "activations/layer7_attention_weight_max": 94.97687530517578,
      "activations/layer7_attention_weight_min": -95.19589233398438,
      "activations/layer8_attention_weight_max": 41.54629135131836,
      "activations/layer8_attention_weight_min": -45.27915573120117,
      "activations/layer9_attention_weight_max": 31.315855026245117,
      "activations/layer9_attention_weight_min": -36.1649284362793,
      "epoch": 19.56,
      "learning_rate": 2.409772727272727e-05,
      "loss": 2.715,
      "step": 336650
    },
    {
      "activations/layer0_attention_weight_max": 16.277130126953125,
      "activations/layer0_attention_weight_min": -13.16979694366455,
      "activations/layer10_attention_weight_max": 35.56934356689453,
      "activations/layer10_attention_weight_min": -34.722564697265625,
      "activations/layer11_attention_weight_max": 34.783180236816406,
      "activations/layer11_attention_weight_min": -33.658782958984375,
      "activations/layer12_attention_weight_max": 26.90989112854004,
      "activations/layer12_attention_weight_min": -27.443832397460938,
      "activations/layer13_attention_weight_max": 41.446773529052734,
      "activations/layer13_attention_weight_min": -38.30180358886719,
      "activations/layer14_attention_weight_max": 42.01009750366211,
      "activations/layer14_attention_weight_min": -40.80449295043945,
      "activations/layer15_attention_weight_max": 38.833885192871094,
      "activations/layer15_attention_weight_min": -36.90318298339844,
      "activations/layer16_attention_weight_max": 29.516908645629883,
      "activations/layer16_attention_weight_min": -28.134294509887695,
      "activations/layer17_attention_weight_max": 29.52989387512207,
      "activations/layer17_attention_weight_min": -27.490633010864258,
      "activations/layer18_attention_weight_max": 32.573509216308594,
      "activations/layer18_attention_weight_min": -27.137174606323242,
      "activations/layer19_attention_weight_max": 32.57210922241211,
      "activations/layer19_attention_weight_min": -34.27609634399414,
      "activations/layer1_attention_weight_max": 15.58851146697998,
      "activations/layer1_attention_weight_min": -13.86067008972168,
      "activations/layer20_attention_weight_max": 28.862768173217773,
      "activations/layer20_attention_weight_min": -26.39464569091797,
      "activations/layer21_attention_weight_max": 28.525474548339844,
      "activations/layer21_attention_weight_min": -25.332469940185547,
      "activations/layer22_attention_weight_max": 40.17103958129883,
      "activations/layer22_attention_weight_min": -33.07905578613281,
      "activations/layer23_attention_weight_max": 33.0260124206543,
      "activations/layer23_attention_weight_min": -28.258705139160156,
      "activations/layer2_attention_weight_max": 32.45883560180664,
      "activations/layer2_attention_weight_min": -31.385351181030273,
      "activations/layer3_attention_weight_max": 98.77297973632812,
      "activations/layer3_attention_weight_min": -103.08926391601562,
      "activations/layer4_attention_weight_max": 106.6506576538086,
      "activations/layer4_attention_weight_min": -116.4579849243164,
      "activations/layer5_attention_weight_max": 50.00788116455078,
      "activations/layer5_attention_weight_min": -61.044677734375,
      "activations/layer6_attention_weight_max": 43.88290786743164,
      "activations/layer6_attention_weight_min": -46.21982955932617,
      "activations/layer7_attention_weight_max": 95.09632873535156,
      "activations/layer7_attention_weight_min": -100.86866760253906,
      "activations/layer8_attention_weight_max": 43.00520324707031,
      "activations/layer8_attention_weight_min": -44.26355743408203,
      "activations/layer9_attention_weight_max": 32.34769821166992,
      "activations/layer9_attention_weight_min": -33.7291374206543,
      "epoch": 19.56,
      "learning_rate": 2.4078787878787875e-05,
      "loss": 2.7151,
      "step": 336700
    },
    {
      "activations/layer0_attention_weight_max": 17.793752670288086,
      "activations/layer0_attention_weight_min": -14.199223518371582,
      "activations/layer10_attention_weight_max": 33.03010940551758,
      "activations/layer10_attention_weight_min": -34.37849807739258,
      "activations/layer11_attention_weight_max": 32.59967041015625,
      "activations/layer11_attention_weight_min": -34.87805938720703,
      "activations/layer12_attention_weight_max": 28.15958023071289,
      "activations/layer12_attention_weight_min": -27.66403579711914,
      "activations/layer13_attention_weight_max": 40.05891799926758,
      "activations/layer13_attention_weight_min": -36.58085250854492,
      "activations/layer14_attention_weight_max": 45.01022720336914,
      "activations/layer14_attention_weight_min": -40.641212463378906,
      "activations/layer15_attention_weight_max": 39.96583557128906,
      "activations/layer15_attention_weight_min": -36.857200622558594,
      "activations/layer16_attention_weight_max": 29.19225311279297,
      "activations/layer16_attention_weight_min": -27.997440338134766,
      "activations/layer17_attention_weight_max": 30.06796646118164,
      "activations/layer17_attention_weight_min": -25.641414642333984,
      "activations/layer18_attention_weight_max": 31.625253677368164,
      "activations/layer18_attention_weight_min": -27.070199966430664,
      "activations/layer19_attention_weight_max": 33.05526351928711,
      "activations/layer19_attention_weight_min": -33.624488830566406,
      "activations/layer1_attention_weight_max": 15.259275436401367,
      "activations/layer1_attention_weight_min": -15.001168251037598,
      "activations/layer20_attention_weight_max": 26.297067642211914,
      "activations/layer20_attention_weight_min": -26.67928123474121,
      "activations/layer21_attention_weight_max": 27.854412078857422,
      "activations/layer21_attention_weight_min": -25.27287483215332,
      "activations/layer22_attention_weight_max": 42.0277099609375,
      "activations/layer22_attention_weight_min": -32.32050704956055,
      "activations/layer23_attention_weight_max": 32.1859130859375,
      "activations/layer23_attention_weight_min": -28.528730392456055,
      "activations/layer2_attention_weight_max": 30.580915451049805,
      "activations/layer2_attention_weight_min": -30.89794158935547,
      "activations/layer3_attention_weight_max": 99.41742706298828,
      "activations/layer3_attention_weight_min": -96.13704681396484,
      "activations/layer4_attention_weight_max": 103.23563385009766,
      "activations/layer4_attention_weight_min": -108.6399154663086,
      "activations/layer5_attention_weight_max": 50.455406188964844,
      "activations/layer5_attention_weight_min": -64.68406677246094,
      "activations/layer6_attention_weight_max": 45.69004821777344,
      "activations/layer6_attention_weight_min": -45.84968948364258,
      "activations/layer7_attention_weight_max": 91.11890411376953,
      "activations/layer7_attention_weight_min": -94.67173767089844,
      "activations/layer8_attention_weight_max": 40.586727142333984,
      "activations/layer8_attention_weight_min": -43.177215576171875,
      "activations/layer9_attention_weight_max": 31.418869018554688,
      "activations/layer9_attention_weight_min": -35.683536529541016,
      "epoch": 19.57,
      "learning_rate": 2.4059848484848484e-05,
      "loss": 2.6981,
      "step": 336750
    },
    {
      "activations/layer0_attention_weight_max": 16.54838752746582,
      "activations/layer0_attention_weight_min": -13.315003395080566,
      "activations/layer10_attention_weight_max": 32.145225524902344,
      "activations/layer10_attention_weight_min": -33.700927734375,
      "activations/layer11_attention_weight_max": 32.955135345458984,
      "activations/layer11_attention_weight_min": -36.056026458740234,
      "activations/layer12_attention_weight_max": 29.188962936401367,
      "activations/layer12_attention_weight_min": -26.50968360900879,
      "activations/layer13_attention_weight_max": 41.48511505126953,
      "activations/layer13_attention_weight_min": -36.80479049682617,
      "activations/layer14_attention_weight_max": 42.46929931640625,
      "activations/layer14_attention_weight_min": -38.791072845458984,
      "activations/layer15_attention_weight_max": 38.59543991088867,
      "activations/layer15_attention_weight_min": -36.76437759399414,
      "activations/layer16_attention_weight_max": 30.720781326293945,
      "activations/layer16_attention_weight_min": -28.776323318481445,
      "activations/layer17_attention_weight_max": 29.964685440063477,
      "activations/layer17_attention_weight_min": -24.616615295410156,
      "activations/layer18_attention_weight_max": 29.303884506225586,
      "activations/layer18_attention_weight_min": -25.562850952148438,
      "activations/layer19_attention_weight_max": 34.4367561340332,
      "activations/layer19_attention_weight_min": -31.407394409179688,
      "activations/layer1_attention_weight_max": 15.995962142944336,
      "activations/layer1_attention_weight_min": -14.014900207519531,
      "activations/layer20_attention_weight_max": 27.977405548095703,
      "activations/layer20_attention_weight_min": -25.895112991333008,
      "activations/layer21_attention_weight_max": 28.903329849243164,
      "activations/layer21_attention_weight_min": -24.222030639648438,
      "activations/layer22_attention_weight_max": 41.11406707763672,
      "activations/layer22_attention_weight_min": -32.70090103149414,
      "activations/layer23_attention_weight_max": 34.230098724365234,
      "activations/layer23_attention_weight_min": -25.0632381439209,
      "activations/layer2_attention_weight_max": 31.474637985229492,
      "activations/layer2_attention_weight_min": -31.04730987548828,
      "activations/layer3_attention_weight_max": 102.23592376708984,
      "activations/layer3_attention_weight_min": -99.99950408935547,
      "activations/layer4_attention_weight_max": 105.79619598388672,
      "activations/layer4_attention_weight_min": -111.87611389160156,
      "activations/layer5_attention_weight_max": 49.624229431152344,
      "activations/layer5_attention_weight_min": -63.5022087097168,
      "activations/layer6_attention_weight_max": 43.16627883911133,
      "activations/layer6_attention_weight_min": -46.29008865356445,
      "activations/layer7_attention_weight_max": 94.54615783691406,
      "activations/layer7_attention_weight_min": -97.56256866455078,
      "activations/layer8_attention_weight_max": 42.49372100830078,
      "activations/layer8_attention_weight_min": -43.61425018310547,
      "activations/layer9_attention_weight_max": 31.72176170349121,
      "activations/layer9_attention_weight_min": -34.42974090576172,
      "epoch": 19.57,
      "learning_rate": 2.404090909090909e-05,
      "loss": 2.7296,
      "step": 336800
    },
    {
      "activations/layer0_attention_weight_max": 16.82855224609375,
      "activations/layer0_attention_weight_min": -12.795953750610352,
      "activations/layer10_attention_weight_max": 46.42702102661133,
      "activations/layer10_attention_weight_min": -40.71259307861328,
      "activations/layer11_attention_weight_max": 42.80402755737305,
      "activations/layer11_attention_weight_min": -43.144866943359375,
      "activations/layer12_attention_weight_max": 37.33930206298828,
      "activations/layer12_attention_weight_min": -32.79733657836914,
      "activations/layer13_attention_weight_max": 56.09920120239258,
      "activations/layer13_attention_weight_min": -49.28819274902344,
      "activations/layer14_attention_weight_max": 65.30813598632812,
      "activations/layer14_attention_weight_min": -56.492225646972656,
      "activations/layer15_attention_weight_max": 59.96827697753906,
      "activations/layer15_attention_weight_min": -53.071685791015625,
      "activations/layer16_attention_weight_max": 37.47783279418945,
      "activations/layer16_attention_weight_min": -30.154674530029297,
      "activations/layer17_attention_weight_max": 35.891876220703125,
      "activations/layer17_attention_weight_min": -31.82089614868164,
      "activations/layer18_attention_weight_max": 37.140289306640625,
      "activations/layer18_attention_weight_min": -31.597496032714844,
      "activations/layer19_attention_weight_max": 44.00913619995117,
      "activations/layer19_attention_weight_min": -33.659812927246094,
      "activations/layer1_attention_weight_max": 16.085355758666992,
      "activations/layer1_attention_weight_min": -13.617683410644531,
      "activations/layer20_attention_weight_max": 35.958099365234375,
      "activations/layer20_attention_weight_min": -27.145214080810547,
      "activations/layer21_attention_weight_max": 33.722328186035156,
      "activations/layer21_attention_weight_min": -28.03996467590332,
      "activations/layer22_attention_weight_max": 60.0467643737793,
      "activations/layer22_attention_weight_min": -40.571563720703125,
      "activations/layer23_attention_weight_max": 41.15362548828125,
      "activations/layer23_attention_weight_min": -32.33415222167969,
      "activations/layer2_attention_weight_max": 33.33056640625,
      "activations/layer2_attention_weight_min": -32.557701110839844,
      "activations/layer3_attention_weight_max": 105.02201843261719,
      "activations/layer3_attention_weight_min": -103.56974792480469,
      "activations/layer4_attention_weight_max": 112.97350311279297,
      "activations/layer4_attention_weight_min": -106.33959197998047,
      "activations/layer5_attention_weight_max": 54.90100860595703,
      "activations/layer5_attention_weight_min": -60.45625305175781,
      "activations/layer6_attention_weight_max": 44.69375991821289,
      "activations/layer6_attention_weight_min": -44.51865005493164,
      "activations/layer7_attention_weight_max": 104.6998519897461,
      "activations/layer7_attention_weight_min": -98.76898193359375,
      "activations/layer8_attention_weight_max": 50.10430145263672,
      "activations/layer8_attention_weight_min": -47.964576721191406,
      "activations/layer9_attention_weight_max": 38.754722595214844,
      "activations/layer9_attention_weight_min": -39.66518783569336,
      "epoch": 19.57,
      "learning_rate": 2.4021969696969697e-05,
      "loss": 2.7173,
      "step": 336850
    },
    {
      "activations/layer0_attention_weight_max": 15.663497924804688,
      "activations/layer0_attention_weight_min": -12.759801864624023,
      "activations/layer10_attention_weight_max": 35.2571907043457,
      "activations/layer10_attention_weight_min": -35.590576171875,
      "activations/layer11_attention_weight_max": 36.902462005615234,
      "activations/layer11_attention_weight_min": -36.50659942626953,
      "activations/layer12_attention_weight_max": 26.748554229736328,
      "activations/layer12_attention_weight_min": -29.005813598632812,
      "activations/layer13_attention_weight_max": 40.95500183105469,
      "activations/layer13_attention_weight_min": -38.8200569152832,
      "activations/layer14_attention_weight_max": 43.14214324951172,
      "activations/layer14_attention_weight_min": -41.834842681884766,
      "activations/layer15_attention_weight_max": 40.82856369018555,
      "activations/layer15_attention_weight_min": -39.935455322265625,
      "activations/layer16_attention_weight_max": 28.974437713623047,
      "activations/layer16_attention_weight_min": -29.032711029052734,
      "activations/layer17_attention_weight_max": 29.026147842407227,
      "activations/layer17_attention_weight_min": -27.61154556274414,
      "activations/layer18_attention_weight_max": 32.959442138671875,
      "activations/layer18_attention_weight_min": -25.216358184814453,
      "activations/layer19_attention_weight_max": 32.307830810546875,
      "activations/layer19_attention_weight_min": -30.220401763916016,
      "activations/layer1_attention_weight_max": 15.696720123291016,
      "activations/layer1_attention_weight_min": -14.178877830505371,
      "activations/layer20_attention_weight_max": 27.519725799560547,
      "activations/layer20_attention_weight_min": -25.032554626464844,
      "activations/layer21_attention_weight_max": 27.22422218322754,
      "activations/layer21_attention_weight_min": -23.76422119140625,
      "activations/layer22_attention_weight_max": 45.03937530517578,
      "activations/layer22_attention_weight_min": -30.97017478942871,
      "activations/layer23_attention_weight_max": 29.564239501953125,
      "activations/layer23_attention_weight_min": -25.398151397705078,
      "activations/layer2_attention_weight_max": 31.904769897460938,
      "activations/layer2_attention_weight_min": -30.745298385620117,
      "activations/layer3_attention_weight_max": 100.8242416381836,
      "activations/layer3_attention_weight_min": -98.10938262939453,
      "activations/layer4_attention_weight_max": 106.86231994628906,
      "activations/layer4_attention_weight_min": -107.8875732421875,
      "activations/layer5_attention_weight_max": 49.384613037109375,
      "activations/layer5_attention_weight_min": -61.64806365966797,
      "activations/layer6_attention_weight_max": 44.56804656982422,
      "activations/layer6_attention_weight_min": -48.0897331237793,
      "activations/layer7_attention_weight_max": 91.3194580078125,
      "activations/layer7_attention_weight_min": -103.96499633789062,
      "activations/layer8_attention_weight_max": 43.79787063598633,
      "activations/layer8_attention_weight_min": -47.77523422241211,
      "activations/layer9_attention_weight_max": 32.44153594970703,
      "activations/layer9_attention_weight_min": -35.1153564453125,
      "epoch": 19.58,
      "learning_rate": 2.4003030303030298e-05,
      "loss": 2.7304,
      "step": 336900
    },
    {
      "activations/layer0_attention_weight_max": 17.101343154907227,
      "activations/layer0_attention_weight_min": -13.774110794067383,
      "activations/layer10_attention_weight_max": 34.36838912963867,
      "activations/layer10_attention_weight_min": -35.41449737548828,
      "activations/layer11_attention_weight_max": 35.27385330200195,
      "activations/layer11_attention_weight_min": -35.74652862548828,
      "activations/layer12_attention_weight_max": 27.820396423339844,
      "activations/layer12_attention_weight_min": -27.108951568603516,
      "activations/layer13_attention_weight_max": 40.69622039794922,
      "activations/layer13_attention_weight_min": -37.10404968261719,
      "activations/layer14_attention_weight_max": 50.456050872802734,
      "activations/layer14_attention_weight_min": -41.279563903808594,
      "activations/layer15_attention_weight_max": 45.028785705566406,
      "activations/layer15_attention_weight_min": -38.889434814453125,
      "activations/layer16_attention_weight_max": 34.634666442871094,
      "activations/layer16_attention_weight_min": -28.47523307800293,
      "activations/layer17_attention_weight_max": 31.31635856628418,
      "activations/layer17_attention_weight_min": -28.300628662109375,
      "activations/layer18_attention_weight_max": 33.35558319091797,
      "activations/layer18_attention_weight_min": -26.99578285217285,
      "activations/layer19_attention_weight_max": 38.26339340209961,
      "activations/layer19_attention_weight_min": -34.42151641845703,
      "activations/layer1_attention_weight_max": 15.123046875,
      "activations/layer1_attention_weight_min": -15.875064849853516,
      "activations/layer20_attention_weight_max": 32.57794952392578,
      "activations/layer20_attention_weight_min": -26.075298309326172,
      "activations/layer21_attention_weight_max": 33.7359504699707,
      "activations/layer21_attention_weight_min": -26.43556785583496,
      "activations/layer22_attention_weight_max": 44.87490463256836,
      "activations/layer22_attention_weight_min": -34.209983825683594,
      "activations/layer23_attention_weight_max": 33.38359069824219,
      "activations/layer23_attention_weight_min": -26.498432159423828,
      "activations/layer2_attention_weight_max": 34.0849494934082,
      "activations/layer2_attention_weight_min": -34.57369613647461,
      "activations/layer3_attention_weight_max": 98.17066955566406,
      "activations/layer3_attention_weight_min": -106.6611099243164,
      "activations/layer4_attention_weight_max": 101.8182601928711,
      "activations/layer4_attention_weight_min": -110.17082977294922,
      "activations/layer5_attention_weight_max": 50.21125793457031,
      "activations/layer5_attention_weight_min": -61.8929443359375,
      "activations/layer6_attention_weight_max": 43.048316955566406,
      "activations/layer6_attention_weight_min": -49.36288070678711,
      "activations/layer7_attention_weight_max": 92.93815612792969,
      "activations/layer7_attention_weight_min": -96.27010345458984,
      "activations/layer8_attention_weight_max": 42.671119689941406,
      "activations/layer8_attention_weight_min": -45.71708679199219,
      "activations/layer9_attention_weight_max": 32.40452575683594,
      "activations/layer9_attention_weight_min": -36.40584182739258,
      "epoch": 19.58,
      "learning_rate": 2.3984090909090907e-05,
      "loss": 2.7051,
      "step": 336950
    },
    {
      "activations/layer0_attention_weight_max": 16.29345703125,
      "activations/layer0_attention_weight_min": -12.875223159790039,
      "activations/layer10_attention_weight_max": 33.78837585449219,
      "activations/layer10_attention_weight_min": -34.9823112487793,
      "activations/layer11_attention_weight_max": 32.95455551147461,
      "activations/layer11_attention_weight_min": -37.58251190185547,
      "activations/layer12_attention_weight_max": 28.483198165893555,
      "activations/layer12_attention_weight_min": -28.13040542602539,
      "activations/layer13_attention_weight_max": 43.266292572021484,
      "activations/layer13_attention_weight_min": -37.294219970703125,
      "activations/layer14_attention_weight_max": 42.95371627807617,
      "activations/layer14_attention_weight_min": -39.961708068847656,
      "activations/layer15_attention_weight_max": 44.253150939941406,
      "activations/layer15_attention_weight_min": -39.84938049316406,
      "activations/layer16_attention_weight_max": 34.03438186645508,
      "activations/layer16_attention_weight_min": -29.54118537902832,
      "activations/layer17_attention_weight_max": 35.23234939575195,
      "activations/layer17_attention_weight_min": -28.518579483032227,
      "activations/layer18_attention_weight_max": 36.52278518676758,
      "activations/layer18_attention_weight_min": -28.292436599731445,
      "activations/layer19_attention_weight_max": 39.514671325683594,
      "activations/layer19_attention_weight_min": -32.68250274658203,
      "activations/layer1_attention_weight_max": 16.691558837890625,
      "activations/layer1_attention_weight_min": -14.925091743469238,
      "activations/layer20_attention_weight_max": 32.28889465332031,
      "activations/layer20_attention_weight_min": -26.154338836669922,
      "activations/layer21_attention_weight_max": 30.396738052368164,
      "activations/layer21_attention_weight_min": -25.24034881591797,
      "activations/layer22_attention_weight_max": 42.02839660644531,
      "activations/layer22_attention_weight_min": -35.44312286376953,
      "activations/layer23_attention_weight_max": 32.20109558105469,
      "activations/layer23_attention_weight_min": -28.184158325195312,
      "activations/layer2_attention_weight_max": 31.776390075683594,
      "activations/layer2_attention_weight_min": -31.618194580078125,
      "activations/layer3_attention_weight_max": 101.90525817871094,
      "activations/layer3_attention_weight_min": -101.11810302734375,
      "activations/layer4_attention_weight_max": 106.958984375,
      "activations/layer4_attention_weight_min": -104.96357727050781,
      "activations/layer5_attention_weight_max": 48.35692596435547,
      "activations/layer5_attention_weight_min": -64.25105285644531,
      "activations/layer6_attention_weight_max": 42.569091796875,
      "activations/layer6_attention_weight_min": -46.388633728027344,
      "activations/layer7_attention_weight_max": 93.73950958251953,
      "activations/layer7_attention_weight_min": -90.71273040771484,
      "activations/layer8_attention_weight_max": 41.9560661315918,
      "activations/layer8_attention_weight_min": -43.82517623901367,
      "activations/layer9_attention_weight_max": 32.9771842956543,
      "activations/layer9_attention_weight_min": -36.03910827636719,
      "epoch": 19.58,
      "learning_rate": 2.3965151515151515e-05,
      "loss": 2.7241,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4842,
      "eval_samples_per_second": 506.115,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4842,
      "eval_openwebtext_samples_per_second": 506.115,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9407,
      "eval_wikitext_samples_per_second": 234.968,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_lambada_loss": 2.369140625,
      "eval_lambada_ppl": 10.688203161633492,
      "eval_lambada_runtime": 9.5509,
      "eval_lambada_samples_per_second": 509.796,
      "step": 337000
    },
    {
      "activations/layer0_attention_weight_max": 15.925848007202148,
      "activations/layer0_attention_weight_min": -13.995882034301758,
      "activations/layer10_attention_weight_max": 33.11003494262695,
      "activations/layer10_attention_weight_min": -35.014034271240234,
      "activations/layer11_attention_weight_max": 32.16362762451172,
      "activations/layer11_attention_weight_min": -36.47459030151367,
      "activations/layer12_attention_weight_max": 27.67412567138672,
      "activations/layer12_attention_weight_min": -26.641746520996094,
      "activations/layer13_attention_weight_max": 36.93866729736328,
      "activations/layer13_attention_weight_min": -36.16331481933594,
      "activations/layer14_attention_weight_max": 40.702823638916016,
      "activations/layer14_attention_weight_min": -36.916419982910156,
      "activations/layer15_attention_weight_max": 37.309452056884766,
      "activations/layer15_attention_weight_min": -34.545814514160156,
      "activations/layer16_attention_weight_max": 29.176836013793945,
      "activations/layer16_attention_weight_min": -28.406978607177734,
      "activations/layer17_attention_weight_max": 29.00894546508789,
      "activations/layer17_attention_weight_min": -26.76560401916504,
      "activations/layer18_attention_weight_max": 32.12933349609375,
      "activations/layer18_attention_weight_min": -25.970579147338867,
      "activations/layer19_attention_weight_max": 34.286468505859375,
      "activations/layer19_attention_weight_min": -31.230960845947266,
      "activations/layer1_attention_weight_max": 15.435633659362793,
      "activations/layer1_attention_weight_min": -14.674297332763672,
      "activations/layer20_attention_weight_max": 31.284284591674805,
      "activations/layer20_attention_weight_min": -26.78691864013672,
      "activations/layer21_attention_weight_max": 29.941091537475586,
      "activations/layer21_attention_weight_min": -25.49974822998047,
      "activations/layer22_attention_weight_max": 45.139793395996094,
      "activations/layer22_attention_weight_min": -31.86626434326172,
      "activations/layer23_attention_weight_max": 33.003990173339844,
      "activations/layer23_attention_weight_min": -25.08340072631836,
      "activations/layer2_attention_weight_max": 30.728940963745117,
      "activations/layer2_attention_weight_min": -31.127477645874023,
      "activations/layer3_attention_weight_max": 98.8002700805664,
      "activations/layer3_attention_weight_min": -99.2288589477539,
      "activations/layer4_attention_weight_max": 104.61443328857422,
      "activations/layer4_attention_weight_min": -103.57353973388672,
      "activations/layer5_attention_weight_max": 47.32373046875,
      "activations/layer5_attention_weight_min": -63.78474044799805,
      "activations/layer6_attention_weight_max": 41.92925262451172,
      "activations/layer6_attention_weight_min": -47.49253463745117,
      "activations/layer7_attention_weight_max": 91.78099822998047,
      "activations/layer7_attention_weight_min": -96.55335998535156,
      "activations/layer8_attention_weight_max": 39.557891845703125,
      "activations/layer8_attention_weight_min": -42.29707717895508,
      "activations/layer9_attention_weight_max": 30.428024291992188,
      "activations/layer9_attention_weight_min": -33.823116302490234,
      "epoch": 19.58,
      "learning_rate": 2.394621212121212e-05,
      "loss": 2.7073,
      "step": 337050
    },
    {
      "activations/layer0_attention_weight_max": 16.124486923217773,
      "activations/layer0_attention_weight_min": -13.49593734741211,
      "activations/layer10_attention_weight_max": 36.894100189208984,
      "activations/layer10_attention_weight_min": -36.1822395324707,
      "activations/layer11_attention_weight_max": 36.1212158203125,
      "activations/layer11_attention_weight_min": -38.243988037109375,
      "activations/layer12_attention_weight_max": 26.34178352355957,
      "activations/layer12_attention_weight_min": -26.624286651611328,
      "activations/layer13_attention_weight_max": 40.57789993286133,
      "activations/layer13_attention_weight_min": -36.57487106323242,
      "activations/layer14_attention_weight_max": 39.74703598022461,
      "activations/layer14_attention_weight_min": -37.769927978515625,
      "activations/layer15_attention_weight_max": 37.66657257080078,
      "activations/layer15_attention_weight_min": -35.20012283325195,
      "activations/layer16_attention_weight_max": 28.310033798217773,
      "activations/layer16_attention_weight_min": -29.375837326049805,
      "activations/layer17_attention_weight_max": 30.808427810668945,
      "activations/layer17_attention_weight_min": -26.34442138671875,
      "activations/layer18_attention_weight_max": 33.042537689208984,
      "activations/layer18_attention_weight_min": -27.971927642822266,
      "activations/layer19_attention_weight_max": 37.075157165527344,
      "activations/layer19_attention_weight_min": -30.55557632446289,
      "activations/layer1_attention_weight_max": 15.786009788513184,
      "activations/layer1_attention_weight_min": -14.623272895812988,
      "activations/layer20_attention_weight_max": 28.745065689086914,
      "activations/layer20_attention_weight_min": -24.593006134033203,
      "activations/layer21_attention_weight_max": 28.489194869995117,
      "activations/layer21_attention_weight_min": -25.30803680419922,
      "activations/layer22_attention_weight_max": 42.48937225341797,
      "activations/layer22_attention_weight_min": -30.920543670654297,
      "activations/layer23_attention_weight_max": 32.63889694213867,
      "activations/layer23_attention_weight_min": -26.139060974121094,
      "activations/layer2_attention_weight_max": 32.167354583740234,
      "activations/layer2_attention_weight_min": -31.36531639099121,
      "activations/layer3_attention_weight_max": 98.945068359375,
      "activations/layer3_attention_weight_min": -103.49626159667969,
      "activations/layer4_attention_weight_max": 107.10216522216797,
      "activations/layer4_attention_weight_min": -107.6278305053711,
      "activations/layer5_attention_weight_max": 50.425811767578125,
      "activations/layer5_attention_weight_min": -61.438209533691406,
      "activations/layer6_attention_weight_max": 44.36544418334961,
      "activations/layer6_attention_weight_min": -48.76609420776367,
      "activations/layer7_attention_weight_max": 102.4888687133789,
      "activations/layer7_attention_weight_min": -100.79076385498047,
      "activations/layer8_attention_weight_max": 44.0471305847168,
      "activations/layer8_attention_weight_min": -46.58924865722656,
      "activations/layer9_attention_weight_max": 31.9305419921875,
      "activations/layer9_attention_weight_min": -36.30510711669922,
      "epoch": 19.59,
      "learning_rate": 2.3927272727272728e-05,
      "loss": 2.7329,
      "step": 337100
    },
    {
      "activations/layer0_attention_weight_max": 16.09970474243164,
      "activations/layer0_attention_weight_min": -13.02578353881836,
      "activations/layer10_attention_weight_max": 36.84407424926758,
      "activations/layer10_attention_weight_min": -35.644874572753906,
      "activations/layer11_attention_weight_max": 37.84300994873047,
      "activations/layer11_attention_weight_min": -36.04179382324219,
      "activations/layer12_attention_weight_max": 34.68653869628906,
      "activations/layer12_attention_weight_min": -28.749324798583984,
      "activations/layer13_attention_weight_max": 47.419010162353516,
      "activations/layer13_attention_weight_min": -37.26442337036133,
      "activations/layer14_attention_weight_max": 48.36212921142578,
      "activations/layer14_attention_weight_min": -39.69904327392578,
      "activations/layer15_attention_weight_max": 51.42172622680664,
      "activations/layer15_attention_weight_min": -41.62091064453125,
      "activations/layer16_attention_weight_max": 31.930252075195312,
      "activations/layer16_attention_weight_min": -27.2043399810791,
      "activations/layer17_attention_weight_max": 31.481868743896484,
      "activations/layer17_attention_weight_min": -26.098060607910156,
      "activations/layer18_attention_weight_max": 32.72824478149414,
      "activations/layer18_attention_weight_min": -26.322969436645508,
      "activations/layer19_attention_weight_max": 34.1876220703125,
      "activations/layer19_attention_weight_min": -29.845752716064453,
      "activations/layer1_attention_weight_max": 15.927192687988281,
      "activations/layer1_attention_weight_min": -12.58080768585205,
      "activations/layer20_attention_weight_max": 28.707279205322266,
      "activations/layer20_attention_weight_min": -23.115983963012695,
      "activations/layer21_attention_weight_max": 28.965255737304688,
      "activations/layer21_attention_weight_min": -25.26408576965332,
      "activations/layer22_attention_weight_max": 43.89985656738281,
      "activations/layer22_attention_weight_min": -32.635833740234375,
      "activations/layer23_attention_weight_max": 28.9731502532959,
      "activations/layer23_attention_weight_min": -27.15778160095215,
      "activations/layer2_attention_weight_max": 32.00788116455078,
      "activations/layer2_attention_weight_min": -32.36072540283203,
      "activations/layer3_attention_weight_max": 99.573486328125,
      "activations/layer3_attention_weight_min": -99.76097106933594,
      "activations/layer4_attention_weight_max": 104.85530853271484,
      "activations/layer4_attention_weight_min": -110.3493423461914,
      "activations/layer5_attention_weight_max": 52.804054260253906,
      "activations/layer5_attention_weight_min": -64.52088928222656,
      "activations/layer6_attention_weight_max": 44.71904754638672,
      "activations/layer6_attention_weight_min": -46.82292175292969,
      "activations/layer7_attention_weight_max": 97.2796401977539,
      "activations/layer7_attention_weight_min": -96.43134307861328,
      "activations/layer8_attention_weight_max": 44.769126892089844,
      "activations/layer8_attention_weight_min": -44.848812103271484,
      "activations/layer9_attention_weight_max": 32.53104782104492,
      "activations/layer9_attention_weight_min": -34.51531219482422,
      "epoch": 19.59,
      "learning_rate": 2.390833333333333e-05,
      "loss": 2.7289,
      "step": 337150
    },
    {
      "activations/layer0_attention_weight_max": 16.364431381225586,
      "activations/layer0_attention_weight_min": -14.399017333984375,
      "activations/layer10_attention_weight_max": 41.70459747314453,
      "activations/layer10_attention_weight_min": -36.06654739379883,
      "activations/layer11_attention_weight_max": 38.02622604370117,
      "activations/layer11_attention_weight_min": -36.881385803222656,
      "activations/layer12_attention_weight_max": 27.595335006713867,
      "activations/layer12_attention_weight_min": -34.99169921875,
      "activations/layer13_attention_weight_max": 43.19342803955078,
      "activations/layer13_attention_weight_min": -41.305599212646484,
      "activations/layer14_attention_weight_max": 48.868858337402344,
      "activations/layer14_attention_weight_min": -42.8250846862793,
      "activations/layer15_attention_weight_max": 42.95905303955078,
      "activations/layer15_attention_weight_min": -41.234188079833984,
      "activations/layer16_attention_weight_max": 36.535003662109375,
      "activations/layer16_attention_weight_min": -31.00940704345703,
      "activations/layer17_attention_weight_max": 33.858192443847656,
      "activations/layer17_attention_weight_min": -27.846729278564453,
      "activations/layer18_attention_weight_max": 35.49958801269531,
      "activations/layer18_attention_weight_min": -26.736833572387695,
      "activations/layer19_attention_weight_max": 40.04957580566406,
      "activations/layer19_attention_weight_min": -32.31214904785156,
      "activations/layer1_attention_weight_max": 15.5789155960083,
      "activations/layer1_attention_weight_min": -14.400731086730957,
      "activations/layer20_attention_weight_max": 33.78894805908203,
      "activations/layer20_attention_weight_min": -24.6292781829834,
      "activations/layer21_attention_weight_max": 32.70036697387695,
      "activations/layer21_attention_weight_min": -24.699663162231445,
      "activations/layer22_attention_weight_max": 52.61945343017578,
      "activations/layer22_attention_weight_min": -32.61450958251953,
      "activations/layer23_attention_weight_max": 35.57548522949219,
      "activations/layer23_attention_weight_min": -25.341533660888672,
      "activations/layer2_attention_weight_max": 33.82471466064453,
      "activations/layer2_attention_weight_min": -32.05970764160156,
      "activations/layer3_attention_weight_max": 99.52713012695312,
      "activations/layer3_attention_weight_min": -101.38924407958984,
      "activations/layer4_attention_weight_max": 103.2411117553711,
      "activations/layer4_attention_weight_min": -104.22845458984375,
      "activations/layer5_attention_weight_max": 50.77611541748047,
      "activations/layer5_attention_weight_min": -60.5087890625,
      "activations/layer6_attention_weight_max": 43.33580780029297,
      "activations/layer6_attention_weight_min": -45.550575256347656,
      "activations/layer7_attention_weight_max": 97.48504638671875,
      "activations/layer7_attention_weight_min": -98.64973449707031,
      "activations/layer8_attention_weight_max": 45.81281661987305,
      "activations/layer8_attention_weight_min": -47.48628616333008,
      "activations/layer9_attention_weight_max": 34.34964370727539,
      "activations/layer9_attention_weight_min": -36.74098587036133,
      "epoch": 19.59,
      "learning_rate": 2.3889393939393938e-05,
      "loss": 2.7128,
      "step": 337200
    },
    {
      "activations/layer0_attention_weight_max": 16.701215744018555,
      "activations/layer0_attention_weight_min": -12.015412330627441,
      "activations/layer10_attention_weight_max": 39.19105911254883,
      "activations/layer10_attention_weight_min": -36.67970657348633,
      "activations/layer11_attention_weight_max": 41.0404052734375,
      "activations/layer11_attention_weight_min": -36.91657638549805,
      "activations/layer12_attention_weight_max": 34.356353759765625,
      "activations/layer12_attention_weight_min": -29.6187801361084,
      "activations/layer13_attention_weight_max": 49.60393524169922,
      "activations/layer13_attention_weight_min": -39.65418243408203,
      "activations/layer14_attention_weight_max": 51.18180465698242,
      "activations/layer14_attention_weight_min": -43.3129768371582,
      "activations/layer15_attention_weight_max": 45.664302825927734,
      "activations/layer15_attention_weight_min": -38.935298919677734,
      "activations/layer16_attention_weight_max": 35.62228775024414,
      "activations/layer16_attention_weight_min": -28.252134323120117,
      "activations/layer17_attention_weight_max": 34.61067581176758,
      "activations/layer17_attention_weight_min": -28.19244384765625,
      "activations/layer18_attention_weight_max": 36.5306510925293,
      "activations/layer18_attention_weight_min": -26.77669906616211,
      "activations/layer19_attention_weight_max": 41.24544906616211,
      "activations/layer19_attention_weight_min": -32.75141906738281,
      "activations/layer1_attention_weight_max": 16.405914306640625,
      "activations/layer1_attention_weight_min": -14.533912658691406,
      "activations/layer20_attention_weight_max": 34.02775955200195,
      "activations/layer20_attention_weight_min": -25.483642578125,
      "activations/layer21_attention_weight_max": 32.49347686767578,
      "activations/layer21_attention_weight_min": -25.074203491210938,
      "activations/layer22_attention_weight_max": 55.18551254272461,
      "activations/layer22_attention_weight_min": -33.870723724365234,
      "activations/layer23_attention_weight_max": 39.75860595703125,
      "activations/layer23_attention_weight_min": -26.54816436767578,
      "activations/layer2_attention_weight_max": 33.713592529296875,
      "activations/layer2_attention_weight_min": -31.941661834716797,
      "activations/layer3_attention_weight_max": 101.3008041381836,
      "activations/layer3_attention_weight_min": -103.92790985107422,
      "activations/layer4_attention_weight_max": 105.7169189453125,
      "activations/layer4_attention_weight_min": -110.28702545166016,
      "activations/layer5_attention_weight_max": 51.397308349609375,
      "activations/layer5_attention_weight_min": -62.727508544921875,
      "activations/layer6_attention_weight_max": 45.45784378051758,
      "activations/layer6_attention_weight_min": -46.512901306152344,
      "activations/layer7_attention_weight_max": 99.89420318603516,
      "activations/layer7_attention_weight_min": -103.13117980957031,
      "activations/layer8_attention_weight_max": 46.37166213989258,
      "activations/layer8_attention_weight_min": -49.5110969543457,
      "activations/layer9_attention_weight_max": 34.176177978515625,
      "activations/layer9_attention_weight_min": -36.36671829223633,
      "epoch": 19.6,
      "learning_rate": 2.3870833333333332e-05,
      "loss": 2.7225,
      "step": 337250
    },
    {
      "activations/layer0_attention_weight_max": 16.299083709716797,
      "activations/layer0_attention_weight_min": -12.975833892822266,
      "activations/layer10_attention_weight_max": 35.238311767578125,
      "activations/layer10_attention_weight_min": -34.982566833496094,
      "activations/layer11_attention_weight_max": 33.757102966308594,
      "activations/layer11_attention_weight_min": -36.24506759643555,
      "activations/layer12_attention_weight_max": 28.21768569946289,
      "activations/layer12_attention_weight_min": -27.619918823242188,
      "activations/layer13_attention_weight_max": 38.1844596862793,
      "activations/layer13_attention_weight_min": -35.124900817871094,
      "activations/layer14_attention_weight_max": 39.611473083496094,
      "activations/layer14_attention_weight_min": -36.64853286743164,
      "activations/layer15_attention_weight_max": 37.939701080322266,
      "activations/layer15_attention_weight_min": -34.6845817565918,
      "activations/layer16_attention_weight_max": 27.75143814086914,
      "activations/layer16_attention_weight_min": -29.932619094848633,
      "activations/layer17_attention_weight_max": 28.12374496459961,
      "activations/layer17_attention_weight_min": -26.31494140625,
      "activations/layer18_attention_weight_max": 31.15031623840332,
      "activations/layer18_attention_weight_min": -24.8963623046875,
      "activations/layer19_attention_weight_max": 33.39781188964844,
      "activations/layer19_attention_weight_min": -32.2447624206543,
      "activations/layer1_attention_weight_max": 15.184280395507812,
      "activations/layer1_attention_weight_min": -13.48446273803711,
      "activations/layer20_attention_weight_max": 29.388565063476562,
      "activations/layer20_attention_weight_min": -25.646533966064453,
      "activations/layer21_attention_weight_max": 29.197845458984375,
      "activations/layer21_attention_weight_min": -24.98539924621582,
      "activations/layer22_attention_weight_max": 44.221832275390625,
      "activations/layer22_attention_weight_min": -35.03770065307617,
      "activations/layer23_attention_weight_max": 35.14227294921875,
      "activations/layer23_attention_weight_min": -25.587261199951172,
      "activations/layer2_attention_weight_max": 30.42976951599121,
      "activations/layer2_attention_weight_min": -31.555240631103516,
      "activations/layer3_attention_weight_max": 96.19165802001953,
      "activations/layer3_attention_weight_min": -97.19288635253906,
      "activations/layer4_attention_weight_max": 103.54473876953125,
      "activations/layer4_attention_weight_min": -105.19779205322266,
      "activations/layer5_attention_weight_max": 51.56151580810547,
      "activations/layer5_attention_weight_min": -61.90837860107422,
      "activations/layer6_attention_weight_max": 45.23849105834961,
      "activations/layer6_attention_weight_min": -47.11422348022461,
      "activations/layer7_attention_weight_max": 99.61376953125,
      "activations/layer7_attention_weight_min": -98.72354888916016,
      "activations/layer8_attention_weight_max": 42.48662185668945,
      "activations/layer8_attention_weight_min": -43.272544860839844,
      "activations/layer9_attention_weight_max": 32.84235763549805,
      "activations/layer9_attention_weight_min": -35.03289031982422,
      "epoch": 19.6,
      "learning_rate": 2.385189393939394e-05,
      "loss": 2.7232,
      "step": 337300
    },
    {
      "activations/layer0_attention_weight_max": 16.543983459472656,
      "activations/layer0_attention_weight_min": -12.258859634399414,
      "activations/layer10_attention_weight_max": 35.978981018066406,
      "activations/layer10_attention_weight_min": -36.54730987548828,
      "activations/layer11_attention_weight_max": 32.76853942871094,
      "activations/layer11_attention_weight_min": -34.123809814453125,
      "activations/layer12_attention_weight_max": 26.89299774169922,
      "activations/layer12_attention_weight_min": -26.591196060180664,
      "activations/layer13_attention_weight_max": 39.90128707885742,
      "activations/layer13_attention_weight_min": -35.706886291503906,
      "activations/layer14_attention_weight_max": 42.645751953125,
      "activations/layer14_attention_weight_min": -36.325645446777344,
      "activations/layer15_attention_weight_max": 40.89339065551758,
      "activations/layer15_attention_weight_min": -36.223915100097656,
      "activations/layer16_attention_weight_max": 30.539806365966797,
      "activations/layer16_attention_weight_min": -28.903705596923828,
      "activations/layer17_attention_weight_max": 29.326223373413086,
      "activations/layer17_attention_weight_min": -26.383378982543945,
      "activations/layer18_attention_weight_max": 35.38615798950195,
      "activations/layer18_attention_weight_min": -27.850383758544922,
      "activations/layer19_attention_weight_max": 38.110740661621094,
      "activations/layer19_attention_weight_min": -32.890743255615234,
      "activations/layer1_attention_weight_max": 15.677544593811035,
      "activations/layer1_attention_weight_min": -14.656275749206543,
      "activations/layer20_attention_weight_max": 31.576108932495117,
      "activations/layer20_attention_weight_min": -27.819828033447266,
      "activations/layer21_attention_weight_max": 28.322616577148438,
      "activations/layer21_attention_weight_min": -25.572998046875,
      "activations/layer22_attention_weight_max": 47.06587219238281,
      "activations/layer22_attention_weight_min": -33.5118408203125,
      "activations/layer23_attention_weight_max": 33.603538513183594,
      "activations/layer23_attention_weight_min": -26.11744499206543,
      "activations/layer2_attention_weight_max": 32.39806365966797,
      "activations/layer2_attention_weight_min": -30.91998863220215,
      "activations/layer3_attention_weight_max": 96.54749298095703,
      "activations/layer3_attention_weight_min": -96.67569732666016,
      "activations/layer4_attention_weight_max": 102.02534484863281,
      "activations/layer4_attention_weight_min": -111.06563568115234,
      "activations/layer5_attention_weight_max": 49.305503845214844,
      "activations/layer5_attention_weight_min": -63.483741760253906,
      "activations/layer6_attention_weight_max": 45.06303787231445,
      "activations/layer6_attention_weight_min": -48.236820220947266,
      "activations/layer7_attention_weight_max": 94.82100677490234,
      "activations/layer7_attention_weight_min": -99.84969329833984,
      "activations/layer8_attention_weight_max": 43.96356201171875,
      "activations/layer8_attention_weight_min": -47.96274185180664,
      "activations/layer9_attention_weight_max": 33.45869445800781,
      "activations/layer9_attention_weight_min": -36.63506317138672,
      "epoch": 19.6,
      "learning_rate": 2.3832954545454542e-05,
      "loss": 2.7191,
      "step": 337350
    },
    {
      "activations/layer0_attention_weight_max": 17.466318130493164,
      "activations/layer0_attention_weight_min": -12.630583763122559,
      "activations/layer10_attention_weight_max": 36.511714935302734,
      "activations/layer10_attention_weight_min": -36.233070373535156,
      "activations/layer11_attention_weight_max": 36.96879577636719,
      "activations/layer11_attention_weight_min": -38.75924301147461,
      "activations/layer12_attention_weight_max": 28.625890731811523,
      "activations/layer12_attention_weight_min": -28.701496124267578,
      "activations/layer13_attention_weight_max": 42.0077018737793,
      "activations/layer13_attention_weight_min": -38.70502471923828,
      "activations/layer14_attention_weight_max": 45.14646530151367,
      "activations/layer14_attention_weight_min": -39.0156364440918,
      "activations/layer15_attention_weight_max": 41.9296760559082,
      "activations/layer15_attention_weight_min": -37.97573471069336,
      "activations/layer16_attention_weight_max": 31.396907806396484,
      "activations/layer16_attention_weight_min": -28.387130737304688,
      "activations/layer17_attention_weight_max": 30.72519302368164,
      "activations/layer17_attention_weight_min": -26.415611267089844,
      "activations/layer18_attention_weight_max": 31.846908569335938,
      "activations/layer18_attention_weight_min": -26.188907623291016,
      "activations/layer19_attention_weight_max": 35.85465621948242,
      "activations/layer19_attention_weight_min": -31.678953170776367,
      "activations/layer1_attention_weight_max": 15.636682510375977,
      "activations/layer1_attention_weight_min": -14.665447235107422,
      "activations/layer20_attention_weight_max": 30.892417907714844,
      "activations/layer20_attention_weight_min": -24.6309871673584,
      "activations/layer21_attention_weight_max": 28.86296272277832,
      "activations/layer21_attention_weight_min": -25.26165008544922,
      "activations/layer22_attention_weight_max": 45.13059616088867,
      "activations/layer22_attention_weight_min": -32.3238410949707,
      "activations/layer23_attention_weight_max": 34.92779541015625,
      "activations/layer23_attention_weight_min": -24.77285385131836,
      "activations/layer2_attention_weight_max": 34.12823486328125,
      "activations/layer2_attention_weight_min": -31.186826705932617,
      "activations/layer3_attention_weight_max": 98.46817779541016,
      "activations/layer3_attention_weight_min": -100.48088073730469,
      "activations/layer4_attention_weight_max": 107.02911376953125,
      "activations/layer4_attention_weight_min": -109.87259674072266,
      "activations/layer5_attention_weight_max": 51.094993591308594,
      "activations/layer5_attention_weight_min": -62.67549133300781,
      "activations/layer6_attention_weight_max": 44.920284271240234,
      "activations/layer6_attention_weight_min": -50.873043060302734,
      "activations/layer7_attention_weight_max": 97.65435028076172,
      "activations/layer7_attention_weight_min": -108.87808990478516,
      "activations/layer8_attention_weight_max": 43.93589401245117,
      "activations/layer8_attention_weight_min": -48.69911193847656,
      "activations/layer9_attention_weight_max": 33.4769172668457,
      "activations/layer9_attention_weight_min": -37.86019515991211,
      "epoch": 19.6,
      "learning_rate": 2.381401515151515e-05,
      "loss": 2.7381,
      "step": 337400
    },
    {
      "activations/layer0_attention_weight_max": 15.645339012145996,
      "activations/layer0_attention_weight_min": -13.840882301330566,
      "activations/layer10_attention_weight_max": 33.6021728515625,
      "activations/layer10_attention_weight_min": -33.93637466430664,
      "activations/layer11_attention_weight_max": 32.93390655517578,
      "activations/layer11_attention_weight_min": -33.527225494384766,
      "activations/layer12_attention_weight_max": 27.73563575744629,
      "activations/layer12_attention_weight_min": -27.27277946472168,
      "activations/layer13_attention_weight_max": 40.903831481933594,
      "activations/layer13_attention_weight_min": -37.0238151550293,
      "activations/layer14_attention_weight_max": 44.50788497924805,
      "activations/layer14_attention_weight_min": -38.344932556152344,
      "activations/layer15_attention_weight_max": 41.685394287109375,
      "activations/layer15_attention_weight_min": -37.97162628173828,
      "activations/layer16_attention_weight_max": 31.01781463623047,
      "activations/layer16_attention_weight_min": -29.503442764282227,
      "activations/layer17_attention_weight_max": 33.80640411376953,
      "activations/layer17_attention_weight_min": -26.04169273376465,
      "activations/layer18_attention_weight_max": 32.82575988769531,
      "activations/layer18_attention_weight_min": -25.378559112548828,
      "activations/layer19_attention_weight_max": 40.079795837402344,
      "activations/layer19_attention_weight_min": -31.135013580322266,
      "activations/layer1_attention_weight_max": 15.59940242767334,
      "activations/layer1_attention_weight_min": -13.866939544677734,
      "activations/layer20_attention_weight_max": 36.25616455078125,
      "activations/layer20_attention_weight_min": -26.549875259399414,
      "activations/layer21_attention_weight_max": 35.177459716796875,
      "activations/layer21_attention_weight_min": -25.857778549194336,
      "activations/layer22_attention_weight_max": 46.266441345214844,
      "activations/layer22_attention_weight_min": -33.165653228759766,
      "activations/layer23_attention_weight_max": 37.03374099731445,
      "activations/layer23_attention_weight_min": -25.60666275024414,
      "activations/layer2_attention_weight_max": 30.796499252319336,
      "activations/layer2_attention_weight_min": -30.09041976928711,
      "activations/layer3_attention_weight_max": 94.47628021240234,
      "activations/layer3_attention_weight_min": -93.79804992675781,
      "activations/layer4_attention_weight_max": 103.09199523925781,
      "activations/layer4_attention_weight_min": -104.78028869628906,
      "activations/layer5_attention_weight_max": 53.157264709472656,
      "activations/layer5_attention_weight_min": -59.878658294677734,
      "activations/layer6_attention_weight_max": 41.309505462646484,
      "activations/layer6_attention_weight_min": -46.747196197509766,
      "activations/layer7_attention_weight_max": 86.50154876708984,
      "activations/layer7_attention_weight_min": -96.0215835571289,
      "activations/layer8_attention_weight_max": 40.735774993896484,
      "activations/layer8_attention_weight_min": -46.38709259033203,
      "activations/layer9_attention_weight_max": 30.660816192626953,
      "activations/layer9_attention_weight_min": -33.756614685058594,
      "epoch": 19.61,
      "learning_rate": 2.3795075757575755e-05,
      "loss": 2.7254,
      "step": 337450
    },
    {
      "activations/layer0_attention_weight_max": 16.330463409423828,
      "activations/layer0_attention_weight_min": -13.507338523864746,
      "activations/layer10_attention_weight_max": 34.66852569580078,
      "activations/layer10_attention_weight_min": -34.96025848388672,
      "activations/layer11_attention_weight_max": 34.592308044433594,
      "activations/layer11_attention_weight_min": -34.25792694091797,
      "activations/layer12_attention_weight_max": 26.991775512695312,
      "activations/layer12_attention_weight_min": -27.44623565673828,
      "activations/layer13_attention_weight_max": 38.8336181640625,
      "activations/layer13_attention_weight_min": -38.053855895996094,
      "activations/layer14_attention_weight_max": 43.245018005371094,
      "activations/layer14_attention_weight_min": -38.880340576171875,
      "activations/layer15_attention_weight_max": 40.58769226074219,
      "activations/layer15_attention_weight_min": -38.76643371582031,
      "activations/layer16_attention_weight_max": 31.152950286865234,
      "activations/layer16_attention_weight_min": -29.695571899414062,
      "activations/layer17_attention_weight_max": 28.84181022644043,
      "activations/layer17_attention_weight_min": -28.95893669128418,
      "activations/layer18_attention_weight_max": 31.698659896850586,
      "activations/layer18_attention_weight_min": -29.304912567138672,
      "activations/layer19_attention_weight_max": 33.79598617553711,
      "activations/layer19_attention_weight_min": -33.56439208984375,
      "activations/layer1_attention_weight_max": 15.50241470336914,
      "activations/layer1_attention_weight_min": -14.611549377441406,
      "activations/layer20_attention_weight_max": 29.702919006347656,
      "activations/layer20_attention_weight_min": -26.220569610595703,
      "activations/layer21_attention_weight_max": 29.087783813476562,
      "activations/layer21_attention_weight_min": -26.037395477294922,
      "activations/layer22_attention_weight_max": 40.76541519165039,
      "activations/layer22_attention_weight_min": -33.698097229003906,
      "activations/layer23_attention_weight_max": 33.1656494140625,
      "activations/layer23_attention_weight_min": -28.365806579589844,
      "activations/layer2_attention_weight_max": 31.59793472290039,
      "activations/layer2_attention_weight_min": -30.908519744873047,
      "activations/layer3_attention_weight_max": 99.10982513427734,
      "activations/layer3_attention_weight_min": -98.87464904785156,
      "activations/layer4_attention_weight_max": 106.0597152709961,
      "activations/layer4_attention_weight_min": -104.48419189453125,
      "activations/layer5_attention_weight_max": 49.64670944213867,
      "activations/layer5_attention_weight_min": -62.05125045776367,
      "activations/layer6_attention_weight_max": 46.22745132446289,
      "activations/layer6_attention_weight_min": -51.7407112121582,
      "activations/layer7_attention_weight_max": 96.56084442138672,
      "activations/layer7_attention_weight_min": -103.60037231445312,
      "activations/layer8_attention_weight_max": 42.917396545410156,
      "activations/layer8_attention_weight_min": -44.267391204833984,
      "activations/layer9_attention_weight_max": 31.145009994506836,
      "activations/layer9_attention_weight_min": -34.57495880126953,
      "epoch": 19.61,
      "learning_rate": 2.3776136363636363e-05,
      "loss": 2.7323,
      "step": 337500
    },
    {
      "activations/layer0_attention_weight_max": 16.179704666137695,
      "activations/layer0_attention_weight_min": -12.541064262390137,
      "activations/layer10_attention_weight_max": 32.663753509521484,
      "activations/layer10_attention_weight_min": -33.00593948364258,
      "activations/layer11_attention_weight_max": 30.815153121948242,
      "activations/layer11_attention_weight_min": -34.836483001708984,
      "activations/layer12_attention_weight_max": 28.260446548461914,
      "activations/layer12_attention_weight_min": -24.994840621948242,
      "activations/layer13_attention_weight_max": 41.892608642578125,
      "activations/layer13_attention_weight_min": -34.61360168457031,
      "activations/layer14_attention_weight_max": 45.58424377441406,
      "activations/layer14_attention_weight_min": -38.020572662353516,
      "activations/layer15_attention_weight_max": 37.99312973022461,
      "activations/layer15_attention_weight_min": -34.7535400390625,
      "activations/layer16_attention_weight_max": 29.45741844177246,
      "activations/layer16_attention_weight_min": -27.754770278930664,
      "activations/layer17_attention_weight_max": 28.76017189025879,
      "activations/layer17_attention_weight_min": -26.030702590942383,
      "activations/layer18_attention_weight_max": 30.156925201416016,
      "activations/layer18_attention_weight_min": -26.27828598022461,
      "activations/layer19_attention_weight_max": 32.564815521240234,
      "activations/layer19_attention_weight_min": -32.75386428833008,
      "activations/layer1_attention_weight_max": 15.541376113891602,
      "activations/layer1_attention_weight_min": -15.151647567749023,
      "activations/layer20_attention_weight_max": 26.4299373626709,
      "activations/layer20_attention_weight_min": -27.396638870239258,
      "activations/layer21_attention_weight_max": 28.179973602294922,
      "activations/layer21_attention_weight_min": -26.79408836364746,
      "activations/layer22_attention_weight_max": 40.6203727722168,
      "activations/layer22_attention_weight_min": -32.203121185302734,
      "activations/layer23_attention_weight_max": 30.5516414642334,
      "activations/layer23_attention_weight_min": -25.949424743652344,
      "activations/layer2_attention_weight_max": 32.02729415893555,
      "activations/layer2_attention_weight_min": -31.102781295776367,
      "activations/layer3_attention_weight_max": 98.13714599609375,
      "activations/layer3_attention_weight_min": -99.82537841796875,
      "activations/layer4_attention_weight_max": 106.38270568847656,
      "activations/layer4_attention_weight_min": -107.26981353759766,
      "activations/layer5_attention_weight_max": 50.88072204589844,
      "activations/layer5_attention_weight_min": -64.62133026123047,
      "activations/layer6_attention_weight_max": 43.96086502075195,
      "activations/layer6_attention_weight_min": -46.123741149902344,
      "activations/layer7_attention_weight_max": 94.613037109375,
      "activations/layer7_attention_weight_min": -106.58927917480469,
      "activations/layer8_attention_weight_max": 41.13555145263672,
      "activations/layer8_attention_weight_min": -45.27743148803711,
      "activations/layer9_attention_weight_max": 30.589231491088867,
      "activations/layer9_attention_weight_min": -33.46928024291992,
      "epoch": 19.61,
      "learning_rate": 2.3757196969696968e-05,
      "loss": 2.7194,
      "step": 337550
    },
    {
      "activations/layer0_attention_weight_max": 16.48140525817871,
      "activations/layer0_attention_weight_min": -12.251476287841797,
      "activations/layer10_attention_weight_max": 32.68804931640625,
      "activations/layer10_attention_weight_min": -35.03327178955078,
      "activations/layer11_attention_weight_max": 33.15168762207031,
      "activations/layer11_attention_weight_min": -33.64560317993164,
      "activations/layer12_attention_weight_max": 25.746231079101562,
      "activations/layer12_attention_weight_min": -28.80878257751465,
      "activations/layer13_attention_weight_max": 39.754913330078125,
      "activations/layer13_attention_weight_min": -35.92045593261719,
      "activations/layer14_attention_weight_max": 41.923805236816406,
      "activations/layer14_attention_weight_min": -38.641212463378906,
      "activations/layer15_attention_weight_max": 38.93532180786133,
      "activations/layer15_attention_weight_min": -34.818885803222656,
      "activations/layer16_attention_weight_max": 30.632173538208008,
      "activations/layer16_attention_weight_min": -28.78217124938965,
      "activations/layer17_attention_weight_max": 28.50235939025879,
      "activations/layer17_attention_weight_min": -26.646377563476562,
      "activations/layer18_attention_weight_max": 32.220306396484375,
      "activations/layer18_attention_weight_min": -25.987382888793945,
      "activations/layer19_attention_weight_max": 34.50167465209961,
      "activations/layer19_attention_weight_min": -33.007137298583984,
      "activations/layer1_attention_weight_max": 15.86107349395752,
      "activations/layer1_attention_weight_min": -14.687049865722656,
      "activations/layer20_attention_weight_max": 31.040687561035156,
      "activations/layer20_attention_weight_min": -25.059354782104492,
      "activations/layer21_attention_weight_max": 33.07212829589844,
      "activations/layer21_attention_weight_min": -24.990489959716797,
      "activations/layer22_attention_weight_max": 45.50836181640625,
      "activations/layer22_attention_weight_min": -30.131032943725586,
      "activations/layer23_attention_weight_max": 31.878873825073242,
      "activations/layer23_attention_weight_min": -26.44904327392578,
      "activations/layer2_attention_weight_max": 31.091054916381836,
      "activations/layer2_attention_weight_min": -32.104896545410156,
      "activations/layer3_attention_weight_max": 96.92110443115234,
      "activations/layer3_attention_weight_min": -97.7245864868164,
      "activations/layer4_attention_weight_max": 104.13323974609375,
      "activations/layer4_attention_weight_min": -110.4765853881836,
      "activations/layer5_attention_weight_max": 49.266990661621094,
      "activations/layer5_attention_weight_min": -60.982704162597656,
      "activations/layer6_attention_weight_max": 43.19575500488281,
      "activations/layer6_attention_weight_min": -46.78163528442383,
      "activations/layer7_attention_weight_max": 94.89997863769531,
      "activations/layer7_attention_weight_min": -96.06160736083984,
      "activations/layer8_attention_weight_max": 42.043216705322266,
      "activations/layer8_attention_weight_min": -45.22848129272461,
      "activations/layer9_attention_weight_max": 32.1005973815918,
      "activations/layer9_attention_weight_min": -34.25924301147461,
      "epoch": 19.62,
      "learning_rate": 2.3738257575757573e-05,
      "loss": 2.7249,
      "step": 337600
    },
    {
      "activations/layer0_attention_weight_max": 16.448312759399414,
      "activations/layer0_attention_weight_min": -13.116110801696777,
      "activations/layer10_attention_weight_max": 35.4788932800293,
      "activations/layer10_attention_weight_min": -35.49260711669922,
      "activations/layer11_attention_weight_max": 33.5888557434082,
      "activations/layer11_attention_weight_min": -34.941246032714844,
      "activations/layer12_attention_weight_max": 28.23883819580078,
      "activations/layer12_attention_weight_min": -27.647445678710938,
      "activations/layer13_attention_weight_max": 38.96372985839844,
      "activations/layer13_attention_weight_min": -36.80205535888672,
      "activations/layer14_attention_weight_max": 40.008575439453125,
      "activations/layer14_attention_weight_min": -39.2054557800293,
      "activations/layer15_attention_weight_max": 37.681156158447266,
      "activations/layer15_attention_weight_min": -36.10239791870117,
      "activations/layer16_attention_weight_max": 28.480405807495117,
      "activations/layer16_attention_weight_min": -27.420940399169922,
      "activations/layer17_attention_weight_max": 28.81815528869629,
      "activations/layer17_attention_weight_min": -27.4199161529541,
      "activations/layer18_attention_weight_max": 32.0123405456543,
      "activations/layer18_attention_weight_min": -26.58822250366211,
      "activations/layer19_attention_weight_max": 33.65937042236328,
      "activations/layer19_attention_weight_min": -31.891706466674805,
      "activations/layer1_attention_weight_max": 16.547555923461914,
      "activations/layer1_attention_weight_min": -14.515992164611816,
      "activations/layer20_attention_weight_max": 28.871978759765625,
      "activations/layer20_attention_weight_min": -25.793277740478516,
      "activations/layer21_attention_weight_max": 27.867788314819336,
      "activations/layer21_attention_weight_min": -24.486053466796875,
      "activations/layer22_attention_weight_max": 43.10731506347656,
      "activations/layer22_attention_weight_min": -31.947587966918945,
      "activations/layer23_attention_weight_max": 36.782711029052734,
      "activations/layer23_attention_weight_min": -26.270856857299805,
      "activations/layer2_attention_weight_max": 30.740461349487305,
      "activations/layer2_attention_weight_min": -29.586454391479492,
      "activations/layer3_attention_weight_max": 96.63996887207031,
      "activations/layer3_attention_weight_min": -94.4216537475586,
      "activations/layer4_attention_weight_max": 99.75472259521484,
      "activations/layer4_attention_weight_min": -102.9428939819336,
      "activations/layer5_attention_weight_max": 49.631935119628906,
      "activations/layer5_attention_weight_min": -61.03380584716797,
      "activations/layer6_attention_weight_max": 42.646392822265625,
      "activations/layer6_attention_weight_min": -46.33055114746094,
      "activations/layer7_attention_weight_max": 94.00985717773438,
      "activations/layer7_attention_weight_min": -101.42787170410156,
      "activations/layer8_attention_weight_max": 41.2114143371582,
      "activations/layer8_attention_weight_min": -43.8211555480957,
      "activations/layer9_attention_weight_max": 31.85664939880371,
      "activations/layer9_attention_weight_min": -34.867523193359375,
      "epoch": 19.62,
      "learning_rate": 2.3719318181818178e-05,
      "loss": 2.727,
      "step": 337650
    },
    {
      "activations/layer0_attention_weight_max": 16.107772827148438,
      "activations/layer0_attention_weight_min": -13.242633819580078,
      "activations/layer10_attention_weight_max": 33.647804260253906,
      "activations/layer10_attention_weight_min": -34.871246337890625,
      "activations/layer11_attention_weight_max": 34.86490249633789,
      "activations/layer11_attention_weight_min": -34.90163803100586,
      "activations/layer12_attention_weight_max": 29.215065002441406,
      "activations/layer12_attention_weight_min": -27.86489486694336,
      "activations/layer13_attention_weight_max": 45.74213790893555,
      "activations/layer13_attention_weight_min": -38.798011779785156,
      "activations/layer14_attention_weight_max": 46.655487060546875,
      "activations/layer14_attention_weight_min": -40.22236633300781,
      "activations/layer15_attention_weight_max": 42.265987396240234,
      "activations/layer15_attention_weight_min": -34.47249221801758,
      "activations/layer16_attention_weight_max": 30.222993850708008,
      "activations/layer16_attention_weight_min": -30.45024871826172,
      "activations/layer17_attention_weight_max": 31.00587272644043,
      "activations/layer17_attention_weight_min": -28.71050453186035,
      "activations/layer18_attention_weight_max": 33.589263916015625,
      "activations/layer18_attention_weight_min": -26.108341217041016,
      "activations/layer19_attention_weight_max": 32.96247482299805,
      "activations/layer19_attention_weight_min": -33.45320129394531,
      "activations/layer1_attention_weight_max": 15.826452255249023,
      "activations/layer1_attention_weight_min": -15.769818305969238,
      "activations/layer20_attention_weight_max": 26.918262481689453,
      "activations/layer20_attention_weight_min": -28.431516647338867,
      "activations/layer21_attention_weight_max": 28.545774459838867,
      "activations/layer21_attention_weight_min": -26.61180877685547,
      "activations/layer22_attention_weight_max": 38.298789978027344,
      "activations/layer22_attention_weight_min": -31.459232330322266,
      "activations/layer23_attention_weight_max": 30.130325317382812,
      "activations/layer23_attention_weight_min": -25.307022094726562,
      "activations/layer2_attention_weight_max": 34.20538330078125,
      "activations/layer2_attention_weight_min": -32.9871940612793,
      "activations/layer3_attention_weight_max": 102.77835083007812,
      "activations/layer3_attention_weight_min": -99.8439712524414,
      "activations/layer4_attention_weight_max": 107.12743377685547,
      "activations/layer4_attention_weight_min": -111.99452209472656,
      "activations/layer5_attention_weight_max": 51.691036224365234,
      "activations/layer5_attention_weight_min": -61.52305221557617,
      "activations/layer6_attention_weight_max": 44.92904281616211,
      "activations/layer6_attention_weight_min": -47.763404846191406,
      "activations/layer7_attention_weight_max": 99.41329193115234,
      "activations/layer7_attention_weight_min": -104.13727569580078,
      "activations/layer8_attention_weight_max": 43.94316482543945,
      "activations/layer8_attention_weight_min": -45.40437698364258,
      "activations/layer9_attention_weight_max": 33.32143020629883,
      "activations/layer9_attention_weight_min": -35.29275894165039,
      "epoch": 19.62,
      "learning_rate": 2.3700757575757575e-05,
      "loss": 2.7136,
      "step": 337700
    },
    {
      "activations/layer0_attention_weight_max": 17.109424591064453,
      "activations/layer0_attention_weight_min": -13.688838005065918,
      "activations/layer10_attention_weight_max": 33.251007080078125,
      "activations/layer10_attention_weight_min": -36.61159133911133,
      "activations/layer11_attention_weight_max": 34.82389450073242,
      "activations/layer11_attention_weight_min": -36.945289611816406,
      "activations/layer12_attention_weight_max": 37.19011306762695,
      "activations/layer12_attention_weight_min": -28.127405166625977,
      "activations/layer13_attention_weight_max": 49.69082260131836,
      "activations/layer13_attention_weight_min": -39.147926330566406,
      "activations/layer14_attention_weight_max": 51.79949188232422,
      "activations/layer14_attention_weight_min": -41.60594940185547,
      "activations/layer15_attention_weight_max": 47.17827224731445,
      "activations/layer15_attention_weight_min": -39.128807067871094,
      "activations/layer16_attention_weight_max": 38.63580322265625,
      "activations/layer16_attention_weight_min": -30.246191024780273,
      "activations/layer17_attention_weight_max": 35.570552825927734,
      "activations/layer17_attention_weight_min": -27.75029945373535,
      "activations/layer18_attention_weight_max": 36.06806182861328,
      "activations/layer18_attention_weight_min": -27.004491806030273,
      "activations/layer19_attention_weight_max": 36.85002517700195,
      "activations/layer19_attention_weight_min": -32.986881256103516,
      "activations/layer1_attention_weight_max": 16.359954833984375,
      "activations/layer1_attention_weight_min": -15.368547439575195,
      "activations/layer20_attention_weight_max": 31.212533950805664,
      "activations/layer20_attention_weight_min": -27.3829402923584,
      "activations/layer21_attention_weight_max": 31.466136932373047,
      "activations/layer21_attention_weight_min": -26.253124237060547,
      "activations/layer22_attention_weight_max": 46.71358108520508,
      "activations/layer22_attention_weight_min": -32.39461898803711,
      "activations/layer23_attention_weight_max": 35.72172546386719,
      "activations/layer23_attention_weight_min": -25.917455673217773,
      "activations/layer2_attention_weight_max": 32.01408004760742,
      "activations/layer2_attention_weight_min": -30.795089721679688,
      "activations/layer3_attention_weight_max": 96.91165924072266,
      "activations/layer3_attention_weight_min": -100.68634033203125,
      "activations/layer4_attention_weight_max": 104.56599426269531,
      "activations/layer4_attention_weight_min": -108.95023345947266,
      "activations/layer5_attention_weight_max": 49.560699462890625,
      "activations/layer5_attention_weight_min": -62.547080993652344,
      "activations/layer6_attention_weight_max": 42.19667053222656,
      "activations/layer6_attention_weight_min": -46.66057586669922,
      "activations/layer7_attention_weight_max": 85.91661834716797,
      "activations/layer7_attention_weight_min": -97.12971496582031,
      "activations/layer8_attention_weight_max": 41.74220657348633,
      "activations/layer8_attention_weight_min": -46.06875991821289,
      "activations/layer9_attention_weight_max": 32.47634506225586,
      "activations/layer9_attention_weight_min": -33.405574798583984,
      "epoch": 19.63,
      "learning_rate": 2.3681818181818177e-05,
      "loss": 2.7281,
      "step": 337750
    },
    {
      "activations/layer0_attention_weight_max": 15.865137100219727,
      "activations/layer0_attention_weight_min": -13.93982219696045,
      "activations/layer10_attention_weight_max": 34.46400833129883,
      "activations/layer10_attention_weight_min": -35.60437774658203,
      "activations/layer11_attention_weight_max": 38.02288055419922,
      "activations/layer11_attention_weight_min": -35.08549118041992,
      "activations/layer12_attention_weight_max": 30.277515411376953,
      "activations/layer12_attention_weight_min": -26.85294532775879,
      "activations/layer13_attention_weight_max": 43.5928840637207,
      "activations/layer13_attention_weight_min": -36.40925598144531,
      "activations/layer14_attention_weight_max": 47.847293853759766,
      "activations/layer14_attention_weight_min": -37.50990676879883,
      "activations/layer15_attention_weight_max": 45.11866760253906,
      "activations/layer15_attention_weight_min": -38.70372772216797,
      "activations/layer16_attention_weight_max": 34.838645935058594,
      "activations/layer16_attention_weight_min": -27.628780364990234,
      "activations/layer17_attention_weight_max": 32.42557144165039,
      "activations/layer17_attention_weight_min": -29.907060623168945,
      "activations/layer18_attention_weight_max": 31.41390609741211,
      "activations/layer18_attention_weight_min": -28.333969116210938,
      "activations/layer19_attention_weight_max": 46.13974380493164,
      "activations/layer19_attention_weight_min": -34.92795944213867,
      "activations/layer1_attention_weight_max": 17.250377655029297,
      "activations/layer1_attention_weight_min": -15.719225883483887,
      "activations/layer20_attention_weight_max": 32.31404113769531,
      "activations/layer20_attention_weight_min": -25.796953201293945,
      "activations/layer21_attention_weight_max": 29.374338150024414,
      "activations/layer21_attention_weight_min": -26.54623031616211,
      "activations/layer22_attention_weight_max": 42.054161071777344,
      "activations/layer22_attention_weight_min": -30.56076431274414,
      "activations/layer23_attention_weight_max": 34.044219970703125,
      "activations/layer23_attention_weight_min": -25.46571922302246,
      "activations/layer2_attention_weight_max": 35.132774353027344,
      "activations/layer2_attention_weight_min": -33.88432312011719,
      "activations/layer3_attention_weight_max": 113.28836059570312,
      "activations/layer3_attention_weight_min": -105.52418518066406,
      "activations/layer4_attention_weight_max": 120.21880340576172,
      "activations/layer4_attention_weight_min": -107.93968963623047,
      "activations/layer5_attention_weight_max": 53.03045654296875,
      "activations/layer5_attention_weight_min": -65.98707580566406,
      "activations/layer6_attention_weight_max": 46.119930267333984,
      "activations/layer6_attention_weight_min": -48.118717193603516,
      "activations/layer7_attention_weight_max": 101.4845962524414,
      "activations/layer7_attention_weight_min": -95.08478546142578,
      "activations/layer8_attention_weight_max": 42.29821014404297,
      "activations/layer8_attention_weight_min": -45.5103645324707,
      "activations/layer9_attention_weight_max": 31.771677017211914,
      "activations/layer9_attention_weight_min": -34.569332122802734,
      "epoch": 19.63,
      "learning_rate": 2.3662878787878785e-05,
      "loss": 2.7233,
      "step": 337800
    },
    {
      "activations/layer0_attention_weight_max": 17.633098602294922,
      "activations/layer0_attention_weight_min": -13.643686294555664,
      "activations/layer10_attention_weight_max": 32.69044876098633,
      "activations/layer10_attention_weight_min": -33.83905792236328,
      "activations/layer11_attention_weight_max": 31.726608276367188,
      "activations/layer11_attention_weight_min": -34.752220153808594,
      "activations/layer12_attention_weight_max": 25.98267364501953,
      "activations/layer12_attention_weight_min": -26.9887752532959,
      "activations/layer13_attention_weight_max": 36.8885612487793,
      "activations/layer13_attention_weight_min": -38.46879196166992,
      "activations/layer14_attention_weight_max": 40.37810134887695,
      "activations/layer14_attention_weight_min": -38.882171630859375,
      "activations/layer15_attention_weight_max": 39.39495086669922,
      "activations/layer15_attention_weight_min": -37.4633903503418,
      "activations/layer16_attention_weight_max": 28.150548934936523,
      "activations/layer16_attention_weight_min": -27.77737808227539,
      "activations/layer17_attention_weight_max": 29.915693283081055,
      "activations/layer17_attention_weight_min": -26.332660675048828,
      "activations/layer18_attention_weight_max": 32.649803161621094,
      "activations/layer18_attention_weight_min": -25.91050910949707,
      "activations/layer19_attention_weight_max": 31.665021896362305,
      "activations/layer19_attention_weight_min": -33.3802490234375,
      "activations/layer1_attention_weight_max": 17.206607818603516,
      "activations/layer1_attention_weight_min": -14.172567367553711,
      "activations/layer20_attention_weight_max": 28.235645294189453,
      "activations/layer20_attention_weight_min": -24.971994400024414,
      "activations/layer21_attention_weight_max": 30.2318172454834,
      "activations/layer21_attention_weight_min": -25.63938331604004,
      "activations/layer22_attention_weight_max": 40.972137451171875,
      "activations/layer22_attention_weight_min": -32.17735290527344,
      "activations/layer23_attention_weight_max": 33.564178466796875,
      "activations/layer23_attention_weight_min": -26.69015884399414,
      "activations/layer2_attention_weight_max": 32.19526672363281,
      "activations/layer2_attention_weight_min": -30.869773864746094,
      "activations/layer3_attention_weight_max": 98.99822235107422,
      "activations/layer3_attention_weight_min": -98.33953857421875,
      "activations/layer4_attention_weight_max": 103.7487564086914,
      "activations/layer4_attention_weight_min": -106.92805480957031,
      "activations/layer5_attention_weight_max": 50.39085006713867,
      "activations/layer5_attention_weight_min": -62.026161193847656,
      "activations/layer6_attention_weight_max": 43.016197204589844,
      "activations/layer6_attention_weight_min": -47.641441345214844,
      "activations/layer7_attention_weight_max": 92.63520812988281,
      "activations/layer7_attention_weight_min": -95.44825744628906,
      "activations/layer8_attention_weight_max": 39.8211784362793,
      "activations/layer8_attention_weight_min": -41.1390266418457,
      "activations/layer9_attention_weight_max": 30.69803237915039,
      "activations/layer9_attention_weight_min": -32.15465545654297,
      "epoch": 19.63,
      "learning_rate": 2.364393939393939e-05,
      "loss": 2.7212,
      "step": 337850
    },
    {
      "activations/layer0_attention_weight_max": 16.169876098632812,
      "activations/layer0_attention_weight_min": -13.301189422607422,
      "activations/layer10_attention_weight_max": 37.20174789428711,
      "activations/layer10_attention_weight_min": -38.8453369140625,
      "activations/layer11_attention_weight_max": 35.99104309082031,
      "activations/layer11_attention_weight_min": -37.52075958251953,
      "activations/layer12_attention_weight_max": 28.569839477539062,
      "activations/layer12_attention_weight_min": -27.516260147094727,
      "activations/layer13_attention_weight_max": 43.933815002441406,
      "activations/layer13_attention_weight_min": -36.0538444519043,
      "activations/layer14_attention_weight_max": 46.654170989990234,
      "activations/layer14_attention_weight_min": -38.12071990966797,
      "activations/layer15_attention_weight_max": 43.91217803955078,
      "activations/layer15_attention_weight_min": -40.76311111450195,
      "activations/layer16_attention_weight_max": 32.78544616699219,
      "activations/layer16_attention_weight_min": -27.546621322631836,
      "activations/layer17_attention_weight_max": 31.158926010131836,
      "activations/layer17_attention_weight_min": -29.19898796081543,
      "activations/layer18_attention_weight_max": 34.406463623046875,
      "activations/layer18_attention_weight_min": -30.4732723236084,
      "activations/layer19_attention_weight_max": 34.9620361328125,
      "activations/layer19_attention_weight_min": -33.92170333862305,
      "activations/layer1_attention_weight_max": 15.467883110046387,
      "activations/layer1_attention_weight_min": -14.537069320678711,
      "activations/layer20_attention_weight_max": 29.932456970214844,
      "activations/layer20_attention_weight_min": -27.078174591064453,
      "activations/layer21_attention_weight_max": 29.220396041870117,
      "activations/layer21_attention_weight_min": -25.928359985351562,
      "activations/layer22_attention_weight_max": 40.06514358520508,
      "activations/layer22_attention_weight_min": -37.09029769897461,
      "activations/layer23_attention_weight_max": 31.676143646240234,
      "activations/layer23_attention_weight_min": -26.860279083251953,
      "activations/layer2_attention_weight_max": 31.968542098999023,
      "activations/layer2_attention_weight_min": -31.754804611206055,
      "activations/layer3_attention_weight_max": 98.27464294433594,
      "activations/layer3_attention_weight_min": -98.72552490234375,
      "activations/layer4_attention_weight_max": 105.4177017211914,
      "activations/layer4_attention_weight_min": -105.67522430419922,
      "activations/layer5_attention_weight_max": 51.093597412109375,
      "activations/layer5_attention_weight_min": -63.80448913574219,
      "activations/layer6_attention_weight_max": 45.939170837402344,
      "activations/layer6_attention_weight_min": -45.648197174072266,
      "activations/layer7_attention_weight_max": 98.0845718383789,
      "activations/layer7_attention_weight_min": -98.00309753417969,
      "activations/layer8_attention_weight_max": 43.1380500793457,
      "activations/layer8_attention_weight_min": -46.688209533691406,
      "activations/layer9_attention_weight_max": 33.05836486816406,
      "activations/layer9_attention_weight_min": -38.279964447021484,
      "epoch": 19.63,
      "learning_rate": 2.3624999999999998e-05,
      "loss": 2.6984,
      "step": 337900
    },
    {
      "activations/layer0_attention_weight_max": 15.85618782043457,
      "activations/layer0_attention_weight_min": -12.482961654663086,
      "activations/layer10_attention_weight_max": 34.63374328613281,
      "activations/layer10_attention_weight_min": -34.455596923828125,
      "activations/layer11_attention_weight_max": 32.92760467529297,
      "activations/layer11_attention_weight_min": -33.629783630371094,
      "activations/layer12_attention_weight_max": 27.019142150878906,
      "activations/layer12_attention_weight_min": -28.007938385009766,
      "activations/layer13_attention_weight_max": 40.02972412109375,
      "activations/layer13_attention_weight_min": -37.67245101928711,
      "activations/layer14_attention_weight_max": 43.625518798828125,
      "activations/layer14_attention_weight_min": -37.58600616455078,
      "activations/layer15_attention_weight_max": 38.31772232055664,
      "activations/layer15_attention_weight_min": -35.67867660522461,
      "activations/layer16_attention_weight_max": 28.868610382080078,
      "activations/layer16_attention_weight_min": -28.07229995727539,
      "activations/layer17_attention_weight_max": 29.92066764831543,
      "activations/layer17_attention_weight_min": -27.041475296020508,
      "activations/layer18_attention_weight_max": 31.614261627197266,
      "activations/layer18_attention_weight_min": -25.21587562561035,
      "activations/layer19_attention_weight_max": 34.479183197021484,
      "activations/layer19_attention_weight_min": -34.24691390991211,
      "activations/layer1_attention_weight_max": 15.363408088684082,
      "activations/layer1_attention_weight_min": -14.356892585754395,
      "activations/layer20_attention_weight_max": 29.439790725708008,
      "activations/layer20_attention_weight_min": -27.438491821289062,
      "activations/layer21_attention_weight_max": 30.366622924804688,
      "activations/layer21_attention_weight_min": -26.798723220825195,
      "activations/layer22_attention_weight_max": 38.79294204711914,
      "activations/layer22_attention_weight_min": -33.06779479980469,
      "activations/layer23_attention_weight_max": 31.32213592529297,
      "activations/layer23_attention_weight_min": -28.04983139038086,
      "activations/layer2_attention_weight_max": 31.234434127807617,
      "activations/layer2_attention_weight_min": -32.46947479248047,
      "activations/layer3_attention_weight_max": 95.56165313720703,
      "activations/layer3_attention_weight_min": -98.86974334716797,
      "activations/layer4_attention_weight_max": 100.394287109375,
      "activations/layer4_attention_weight_min": -108.7619857788086,
      "activations/layer5_attention_weight_max": 48.27278137207031,
      "activations/layer5_attention_weight_min": -60.24036407470703,
      "activations/layer6_attention_weight_max": 45.529109954833984,
      "activations/layer6_attention_weight_min": -45.23587417602539,
      "activations/layer7_attention_weight_max": 91.77456665039062,
      "activations/layer7_attention_weight_min": -96.75579071044922,
      "activations/layer8_attention_weight_max": 41.993919372558594,
      "activations/layer8_attention_weight_min": -43.06310272216797,
      "activations/layer9_attention_weight_max": 32.379207611083984,
      "activations/layer9_attention_weight_min": -34.531558990478516,
      "epoch": 19.64,
      "learning_rate": 2.3606060606060603e-05,
      "loss": 2.7168,
      "step": 337950
    },
    {
      "activations/layer0_attention_weight_max": 15.864489555358887,
      "activations/layer0_attention_weight_min": -12.837357521057129,
      "activations/layer10_attention_weight_max": 35.70280838012695,
      "activations/layer10_attention_weight_min": -35.23429870605469,
      "activations/layer11_attention_weight_max": 36.174503326416016,
      "activations/layer11_attention_weight_min": -36.21221160888672,
      "activations/layer12_attention_weight_max": 28.257638931274414,
      "activations/layer12_attention_weight_min": -28.428621292114258,
      "activations/layer13_attention_weight_max": 40.29349899291992,
      "activations/layer13_attention_weight_min": -40.79905319213867,
      "activations/layer14_attention_weight_max": 46.05434799194336,
      "activations/layer14_attention_weight_min": -41.02733612060547,
      "activations/layer15_attention_weight_max": 40.830970764160156,
      "activations/layer15_attention_weight_min": -37.88408660888672,
      "activations/layer16_attention_weight_max": 28.03626823425293,
      "activations/layer16_attention_weight_min": -29.345746994018555,
      "activations/layer17_attention_weight_max": 30.36838150024414,
      "activations/layer17_attention_weight_min": -29.52943229675293,
      "activations/layer18_attention_weight_max": 30.862045288085938,
      "activations/layer18_attention_weight_min": -28.453542709350586,
      "activations/layer19_attention_weight_max": 35.15921401977539,
      "activations/layer19_attention_weight_min": -33.83358383178711,
      "activations/layer1_attention_weight_max": 15.052430152893066,
      "activations/layer1_attention_weight_min": -14.081964492797852,
      "activations/layer20_attention_weight_max": 30.44517707824707,
      "activations/layer20_attention_weight_min": -26.145761489868164,
      "activations/layer21_attention_weight_max": 28.461936950683594,
      "activations/layer21_attention_weight_min": -26.062705993652344,
      "activations/layer22_attention_weight_max": 40.36124801635742,
      "activations/layer22_attention_weight_min": -33.67294692993164,
      "activations/layer23_attention_weight_max": 32.075897216796875,
      "activations/layer23_attention_weight_min": -27.56966781616211,
      "activations/layer2_attention_weight_max": 30.488767623901367,
      "activations/layer2_attention_weight_min": -30.01436996459961,
      "activations/layer3_attention_weight_max": 97.63945770263672,
      "activations/layer3_attention_weight_min": -98.1430435180664,
      "activations/layer4_attention_weight_max": 106.3159408569336,
      "activations/layer4_attention_weight_min": -110.57828521728516,
      "activations/layer5_attention_weight_max": 50.82861328125,
      "activations/layer5_attention_weight_min": -63.195091247558594,
      "activations/layer6_attention_weight_max": 44.043190002441406,
      "activations/layer6_attention_weight_min": -48.67049789428711,
      "activations/layer7_attention_weight_max": 96.2072982788086,
      "activations/layer7_attention_weight_min": -102.80425262451172,
      "activations/layer8_attention_weight_max": 43.730411529541016,
      "activations/layer8_attention_weight_min": -45.44350051879883,
      "activations/layer9_attention_weight_max": 33.726009368896484,
      "activations/layer9_attention_weight_min": -34.940574645996094,
      "epoch": 19.64,
      "learning_rate": 2.358712121212121e-05,
      "loss": 2.7225,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4828,
      "eval_samples_per_second": 506.199,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4828,
      "eval_openwebtext_samples_per_second": 506.199,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9714,
      "eval_wikitext_samples_per_second": 231.302,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_lambada_loss": 2.314453125,
      "eval_lambada_ppl": 10.119387365287977,
      "eval_lambada_runtime": 9.583,
      "eval_lambada_samples_per_second": 508.089,
      "step": 338000
    },
    {
      "activations/layer0_attention_weight_max": 15.926288604736328,
      "activations/layer0_attention_weight_min": -12.986954689025879,
      "activations/layer10_attention_weight_max": 38.033119201660156,
      "activations/layer10_attention_weight_min": -33.573875427246094,
      "activations/layer11_attention_weight_max": 36.521873474121094,
      "activations/layer11_attention_weight_min": -33.755409240722656,
      "activations/layer12_attention_weight_max": 29.564228057861328,
      "activations/layer12_attention_weight_min": -27.214311599731445,
      "activations/layer13_attention_weight_max": 42.23870849609375,
      "activations/layer13_attention_weight_min": -37.483604431152344,
      "activations/layer14_attention_weight_max": 44.01849365234375,
      "activations/layer14_attention_weight_min": -38.845218658447266,
      "activations/layer15_attention_weight_max": 41.19061279296875,
      "activations/layer15_attention_weight_min": -37.48359680175781,
      "activations/layer16_attention_weight_max": 29.772615432739258,
      "activations/layer16_attention_weight_min": -27.641252517700195,
      "activations/layer17_attention_weight_max": 26.901330947875977,
      "activations/layer17_attention_weight_min": -27.66615104675293,
      "activations/layer18_attention_weight_max": 29.87986946105957,
      "activations/layer18_attention_weight_min": -28.811824798583984,
      "activations/layer19_attention_weight_max": 31.5768985748291,
      "activations/layer19_attention_weight_min": -34.17118453979492,
      "activations/layer1_attention_weight_max": 17.62081527709961,
      "activations/layer1_attention_weight_min": -16.362014770507812,
      "activations/layer20_attention_weight_max": 28.831674575805664,
      "activations/layer20_attention_weight_min": -28.816871643066406,
      "activations/layer21_attention_weight_max": 27.992177963256836,
      "activations/layer21_attention_weight_min": -29.64493179321289,
      "activations/layer22_attention_weight_max": 40.62898635864258,
      "activations/layer22_attention_weight_min": -36.44274139404297,
      "activations/layer23_attention_weight_max": 30.168676376342773,
      "activations/layer23_attention_weight_min": -30.06573486328125,
      "activations/layer2_attention_weight_max": 33.51850128173828,
      "activations/layer2_attention_weight_min": -33.00324249267578,
      "activations/layer3_attention_weight_max": 102.73297882080078,
      "activations/layer3_attention_weight_min": -106.28996276855469,
      "activations/layer4_attention_weight_max": 102.6487045288086,
      "activations/layer4_attention_weight_min": -106.67325592041016,
      "activations/layer5_attention_weight_max": 48.51057052612305,
      "activations/layer5_attention_weight_min": -60.355525970458984,
      "activations/layer6_attention_weight_max": 43.52577590942383,
      "activations/layer6_attention_weight_min": -48.100975036621094,
      "activations/layer7_attention_weight_max": 96.50743865966797,
      "activations/layer7_attention_weight_min": -97.85574340820312,
      "activations/layer8_attention_weight_max": 46.25102615356445,
      "activations/layer8_attention_weight_min": -43.35344314575195,
      "activations/layer9_attention_weight_max": 32.6936149597168,
      "activations/layer9_attention_weight_min": -34.248565673828125,
      "epoch": 19.64,
      "learning_rate": 2.356818181818182e-05,
      "loss": 2.7102,
      "step": 338050
    },
    {
      "activations/layer0_attention_weight_max": 16.679662704467773,
      "activations/layer0_attention_weight_min": -13.973075866699219,
      "activations/layer10_attention_weight_max": 32.712615966796875,
      "activations/layer10_attention_weight_min": -33.65443801879883,
      "activations/layer11_attention_weight_max": 35.05193328857422,
      "activations/layer11_attention_weight_min": -35.948768615722656,
      "activations/layer12_attention_weight_max": 27.435575485229492,
      "activations/layer12_attention_weight_min": -27.661291122436523,
      "activations/layer13_attention_weight_max": 38.47721481323242,
      "activations/layer13_attention_weight_min": -36.74446105957031,
      "activations/layer14_attention_weight_max": 48.63380432128906,
      "activations/layer14_attention_weight_min": -38.91023635864258,
      "activations/layer15_attention_weight_max": 40.52254104614258,
      "activations/layer15_attention_weight_min": -37.75458526611328,
      "activations/layer16_attention_weight_max": 35.78654861450195,
      "activations/layer16_attention_weight_min": -28.744115829467773,
      "activations/layer17_attention_weight_max": 32.012939453125,
      "activations/layer17_attention_weight_min": -27.647388458251953,
      "activations/layer18_attention_weight_max": 34.15450668334961,
      "activations/layer18_attention_weight_min": -26.74989128112793,
      "activations/layer19_attention_weight_max": 40.001991271972656,
      "activations/layer19_attention_weight_min": -30.41650390625,
      "activations/layer1_attention_weight_max": 15.111988067626953,
      "activations/layer1_attention_weight_min": -14.678385734558105,
      "activations/layer20_attention_weight_max": 31.827007293701172,
      "activations/layer20_attention_weight_min": -25.501052856445312,
      "activations/layer21_attention_weight_max": 30.9512939453125,
      "activations/layer21_attention_weight_min": -24.9133358001709,
      "activations/layer22_attention_weight_max": 44.483524322509766,
      "activations/layer22_attention_weight_min": -32.57083511352539,
      "activations/layer23_attention_weight_max": 34.53289031982422,
      "activations/layer23_attention_weight_min": -25.772655487060547,
      "activations/layer2_attention_weight_max": 31.644363403320312,
      "activations/layer2_attention_weight_min": -27.737125396728516,
      "activations/layer3_attention_weight_max": 95.32106018066406,
      "activations/layer3_attention_weight_min": -93.96125030517578,
      "activations/layer4_attention_weight_max": 104.0408935546875,
      "activations/layer4_attention_weight_min": -105.26979064941406,
      "activations/layer5_attention_weight_max": 53.54515075683594,
      "activations/layer5_attention_weight_min": -61.355613708496094,
      "activations/layer6_attention_weight_max": 47.122562408447266,
      "activations/layer6_attention_weight_min": -48.2916145324707,
      "activations/layer7_attention_weight_max": 96.30286407470703,
      "activations/layer7_attention_weight_min": -96.1025161743164,
      "activations/layer8_attention_weight_max": 43.4183464050293,
      "activations/layer8_attention_weight_min": -42.8022575378418,
      "activations/layer9_attention_weight_max": 33.23610305786133,
      "activations/layer9_attention_weight_min": -34.76719284057617,
      "epoch": 19.65,
      "learning_rate": 2.354924242424242e-05,
      "loss": 2.7369,
      "step": 338100
    },
    {
      "activations/layer0_attention_weight_max": 16.153079986572266,
      "activations/layer0_attention_weight_min": -13.306286811828613,
      "activations/layer10_attention_weight_max": 31.474239349365234,
      "activations/layer10_attention_weight_min": -32.24791717529297,
      "activations/layer11_attention_weight_max": 32.111122131347656,
      "activations/layer11_attention_weight_min": -32.377838134765625,
      "activations/layer12_attention_weight_max": 27.319442749023438,
      "activations/layer12_attention_weight_min": -27.358375549316406,
      "activations/layer13_attention_weight_max": 37.40789794921875,
      "activations/layer13_attention_weight_min": -35.54963684082031,
      "activations/layer14_attention_weight_max": 40.63595199584961,
      "activations/layer14_attention_weight_min": -38.660789489746094,
      "activations/layer15_attention_weight_max": 36.54851150512695,
      "activations/layer15_attention_weight_min": -36.66248321533203,
      "activations/layer16_attention_weight_max": 28.616363525390625,
      "activations/layer16_attention_weight_min": -30.157258987426758,
      "activations/layer17_attention_weight_max": 30.345951080322266,
      "activations/layer17_attention_weight_min": -26.78294563293457,
      "activations/layer18_attention_weight_max": 33.51164245605469,
      "activations/layer18_attention_weight_min": -27.602148056030273,
      "activations/layer19_attention_weight_max": 31.066970825195312,
      "activations/layer19_attention_weight_min": -34.30170822143555,
      "activations/layer1_attention_weight_max": 16.46525001525879,
      "activations/layer1_attention_weight_min": -15.721090316772461,
      "activations/layer20_attention_weight_max": 27.076683044433594,
      "activations/layer20_attention_weight_min": -26.22319793701172,
      "activations/layer21_attention_weight_max": 26.523942947387695,
      "activations/layer21_attention_weight_min": -26.15338134765625,
      "activations/layer22_attention_weight_max": 37.4113883972168,
      "activations/layer22_attention_weight_min": -35.71103286743164,
      "activations/layer23_attention_weight_max": 32.563594818115234,
      "activations/layer23_attention_weight_min": -30.825366973876953,
      "activations/layer2_attention_weight_max": 30.36980438232422,
      "activations/layer2_attention_weight_min": -30.351512908935547,
      "activations/layer3_attention_weight_max": 97.41383361816406,
      "activations/layer3_attention_weight_min": -96.25057220458984,
      "activations/layer4_attention_weight_max": 105.5766372680664,
      "activations/layer4_attention_weight_min": -104.79957580566406,
      "activations/layer5_attention_weight_max": 50.15542221069336,
      "activations/layer5_attention_weight_min": -63.15682601928711,
      "activations/layer6_attention_weight_max": 43.90681838989258,
      "activations/layer6_attention_weight_min": -47.48271560668945,
      "activations/layer7_attention_weight_max": 87.53241729736328,
      "activations/layer7_attention_weight_min": -96.84951782226562,
      "activations/layer8_attention_weight_max": 40.77676773071289,
      "activations/layer8_attention_weight_min": -43.14253234863281,
      "activations/layer9_attention_weight_max": 29.241613388061523,
      "activations/layer9_attention_weight_min": -31.96602439880371,
      "epoch": 19.65,
      "learning_rate": 2.353030303030303e-05,
      "loss": 2.718,
      "step": 338150
    },
    {
      "activations/layer0_attention_weight_max": 16.26207160949707,
      "activations/layer0_attention_weight_min": -12.517075538635254,
      "activations/layer10_attention_weight_max": 36.67823028564453,
      "activations/layer10_attention_weight_min": -37.65919876098633,
      "activations/layer11_attention_weight_max": 38.02143096923828,
      "activations/layer11_attention_weight_min": -34.72392272949219,
      "activations/layer12_attention_weight_max": 31.056129455566406,
      "activations/layer12_attention_weight_min": -29.003440856933594,
      "activations/layer13_attention_weight_max": 47.9754524230957,
      "activations/layer13_attention_weight_min": -39.91716003417969,
      "activations/layer14_attention_weight_max": 49.36250305175781,
      "activations/layer14_attention_weight_min": -40.55509948730469,
      "activations/layer15_attention_weight_max": 47.6452522277832,
      "activations/layer15_attention_weight_min": -37.90926742553711,
      "activations/layer16_attention_weight_max": 31.972509384155273,
      "activations/layer16_attention_weight_min": -29.601625442504883,
      "activations/layer17_attention_weight_max": 33.32424545288086,
      "activations/layer17_attention_weight_min": -26.99618911743164,
      "activations/layer18_attention_weight_max": 34.198814392089844,
      "activations/layer18_attention_weight_min": -25.077428817749023,
      "activations/layer19_attention_weight_max": 37.20315170288086,
      "activations/layer19_attention_weight_min": -32.4849739074707,
      "activations/layer1_attention_weight_max": 15.351558685302734,
      "activations/layer1_attention_weight_min": -13.075840950012207,
      "activations/layer20_attention_weight_max": 30.742597579956055,
      "activations/layer20_attention_weight_min": -24.999589920043945,
      "activations/layer21_attention_weight_max": 30.782005310058594,
      "activations/layer21_attention_weight_min": -25.19377899169922,
      "activations/layer22_attention_weight_max": 46.07752990722656,
      "activations/layer22_attention_weight_min": -32.435081481933594,
      "activations/layer23_attention_weight_max": 35.94675064086914,
      "activations/layer23_attention_weight_min": -27.396926879882812,
      "activations/layer2_attention_weight_max": 29.22002410888672,
      "activations/layer2_attention_weight_min": -29.68648338317871,
      "activations/layer3_attention_weight_max": 94.25516510009766,
      "activations/layer3_attention_weight_min": -94.33746337890625,
      "activations/layer4_attention_weight_max": 104.03722381591797,
      "activations/layer4_attention_weight_min": -104.6993179321289,
      "activations/layer5_attention_weight_max": 49.368499755859375,
      "activations/layer5_attention_weight_min": -63.562522888183594,
      "activations/layer6_attention_weight_max": 44.7084846496582,
      "activations/layer6_attention_weight_min": -48.18904113769531,
      "activations/layer7_attention_weight_max": 99.34451293945312,
      "activations/layer7_attention_weight_min": -98.49217987060547,
      "activations/layer8_attention_weight_max": 43.8807487487793,
      "activations/layer8_attention_weight_min": -46.55326461791992,
      "activations/layer9_attention_weight_max": 34.12813186645508,
      "activations/layer9_attention_weight_min": -37.70232009887695,
      "epoch": 19.65,
      "learning_rate": 2.3511363636363634e-05,
      "loss": 2.731,
      "step": 338200
    },
    {
      "activations/layer0_attention_weight_max": 16.51568031311035,
      "activations/layer0_attention_weight_min": -13.078181266784668,
      "activations/layer10_attention_weight_max": 34.389915466308594,
      "activations/layer10_attention_weight_min": -34.16217041015625,
      "activations/layer11_attention_weight_max": 36.124759674072266,
      "activations/layer11_attention_weight_min": -35.783695220947266,
      "activations/layer12_attention_weight_max": 27.71739387512207,
      "activations/layer12_attention_weight_min": -26.62421989440918,
      "activations/layer13_attention_weight_max": 39.67634963989258,
      "activations/layer13_attention_weight_min": -37.142967224121094,
      "activations/layer14_attention_weight_max": 41.354366302490234,
      "activations/layer14_attention_weight_min": -39.04703140258789,
      "activations/layer15_attention_weight_max": 35.33012771606445,
      "activations/layer15_attention_weight_min": -35.74798583984375,
      "activations/layer16_attention_weight_max": 27.110488891601562,
      "activations/layer16_attention_weight_min": -28.906930923461914,
      "activations/layer17_attention_weight_max": 30.57849884033203,
      "activations/layer17_attention_weight_min": -26.861841201782227,
      "activations/layer18_attention_weight_max": 30.35455322265625,
      "activations/layer18_attention_weight_min": -25.76898193359375,
      "activations/layer19_attention_weight_max": 35.595130920410156,
      "activations/layer19_attention_weight_min": -34.686744689941406,
      "activations/layer1_attention_weight_max": 16.19121551513672,
      "activations/layer1_attention_weight_min": -15.738917350769043,
      "activations/layer20_attention_weight_max": 30.1917724609375,
      "activations/layer20_attention_weight_min": -28.233285903930664,
      "activations/layer21_attention_weight_max": 30.857498168945312,
      "activations/layer21_attention_weight_min": -26.647930145263672,
      "activations/layer22_attention_weight_max": 43.29018020629883,
      "activations/layer22_attention_weight_min": -33.129642486572266,
      "activations/layer23_attention_weight_max": 33.67863082885742,
      "activations/layer23_attention_weight_min": -27.274328231811523,
      "activations/layer2_attention_weight_max": 31.307737350463867,
      "activations/layer2_attention_weight_min": -31.146177291870117,
      "activations/layer3_attention_weight_max": 97.04804229736328,
      "activations/layer3_attention_weight_min": -97.88784790039062,
      "activations/layer4_attention_weight_max": 103.83599853515625,
      "activations/layer4_attention_weight_min": -106.65331268310547,
      "activations/layer5_attention_weight_max": 48.68998718261719,
      "activations/layer5_attention_weight_min": -64.20512390136719,
      "activations/layer6_attention_weight_max": 43.72517776489258,
      "activations/layer6_attention_weight_min": -46.0482063293457,
      "activations/layer7_attention_weight_max": 93.72966766357422,
      "activations/layer7_attention_weight_min": -95.76254272460938,
      "activations/layer8_attention_weight_max": 42.825687408447266,
      "activations/layer8_attention_weight_min": -44.81643295288086,
      "activations/layer9_attention_weight_max": 33.47090530395508,
      "activations/layer9_attention_weight_min": -33.70329284667969,
      "epoch": 19.65,
      "learning_rate": 2.3492424242424243e-05,
      "loss": 2.718,
      "step": 338250
    },
    {
      "activations/layer0_attention_weight_max": 16.109376907348633,
      "activations/layer0_attention_weight_min": -12.788786888122559,
      "activations/layer10_attention_weight_max": 35.452030181884766,
      "activations/layer10_attention_weight_min": -36.65290069580078,
      "activations/layer11_attention_weight_max": 35.220516204833984,
      "activations/layer11_attention_weight_min": -36.482276916503906,
      "activations/layer12_attention_weight_max": 29.781917572021484,
      "activations/layer12_attention_weight_min": -31.322107315063477,
      "activations/layer13_attention_weight_max": 44.2154426574707,
      "activations/layer13_attention_weight_min": -41.61155319213867,
      "activations/layer14_attention_weight_max": 47.81368637084961,
      "activations/layer14_attention_weight_min": -45.11191940307617,
      "activations/layer15_attention_weight_max": 42.54450988769531,
      "activations/layer15_attention_weight_min": -39.54465103149414,
      "activations/layer16_attention_weight_max": 32.79880905151367,
      "activations/layer16_attention_weight_min": -29.693103790283203,
      "activations/layer17_attention_weight_max": 31.91608428955078,
      "activations/layer17_attention_weight_min": -27.691598892211914,
      "activations/layer18_attention_weight_max": 35.231910705566406,
      "activations/layer18_attention_weight_min": -28.423763275146484,
      "activations/layer19_attention_weight_max": 36.7811393737793,
      "activations/layer19_attention_weight_min": -33.29774475097656,
      "activations/layer1_attention_weight_max": 15.888717651367188,
      "activations/layer1_attention_weight_min": -14.013903617858887,
      "activations/layer20_attention_weight_max": 30.992420196533203,
      "activations/layer20_attention_weight_min": -26.980144500732422,
      "activations/layer21_attention_weight_max": 29.89431381225586,
      "activations/layer21_attention_weight_min": -25.69353485107422,
      "activations/layer22_attention_weight_max": 48.849388122558594,
      "activations/layer22_attention_weight_min": -34.547142028808594,
      "activations/layer23_attention_weight_max": 34.794795989990234,
      "activations/layer23_attention_weight_min": -29.516075134277344,
      "activations/layer2_attention_weight_max": 30.238195419311523,
      "activations/layer2_attention_weight_min": -29.94461441040039,
      "activations/layer3_attention_weight_max": 95.86412811279297,
      "activations/layer3_attention_weight_min": -100.06639099121094,
      "activations/layer4_attention_weight_max": 103.43541717529297,
      "activations/layer4_attention_weight_min": -104.9617919921875,
      "activations/layer5_attention_weight_max": 52.70658874511719,
      "activations/layer5_attention_weight_min": -62.968074798583984,
      "activations/layer6_attention_weight_max": 46.0406379699707,
      "activations/layer6_attention_weight_min": -49.926082611083984,
      "activations/layer7_attention_weight_max": 99.81105041503906,
      "activations/layer7_attention_weight_min": -99.45120239257812,
      "activations/layer8_attention_weight_max": 42.753334045410156,
      "activations/layer8_attention_weight_min": -44.47538375854492,
      "activations/layer9_attention_weight_max": 34.62852096557617,
      "activations/layer9_attention_weight_min": -36.17112731933594,
      "epoch": 19.66,
      "learning_rate": 2.3473484848484844e-05,
      "loss": 2.7196,
      "step": 338300
    },
    {
      "activations/layer0_attention_weight_max": 15.92343807220459,
      "activations/layer0_attention_weight_min": -12.56667709350586,
      "activations/layer10_attention_weight_max": 34.94740295410156,
      "activations/layer10_attention_weight_min": -36.20444107055664,
      "activations/layer11_attention_weight_max": 32.413883209228516,
      "activations/layer11_attention_weight_min": -34.58961868286133,
      "activations/layer12_attention_weight_max": 27.524202346801758,
      "activations/layer12_attention_weight_min": -27.979507446289062,
      "activations/layer13_attention_weight_max": 40.90438461303711,
      "activations/layer13_attention_weight_min": -38.96015167236328,
      "activations/layer14_attention_weight_max": 42.297340393066406,
      "activations/layer14_attention_weight_min": -42.82059097290039,
      "activations/layer15_attention_weight_max": 42.17738342285156,
      "activations/layer15_attention_weight_min": -39.80381393432617,
      "activations/layer16_attention_weight_max": 33.994632720947266,
      "activations/layer16_attention_weight_min": -29.81781578063965,
      "activations/layer17_attention_weight_max": 32.11211013793945,
      "activations/layer17_attention_weight_min": -26.68495750427246,
      "activations/layer18_attention_weight_max": 32.05730438232422,
      "activations/layer18_attention_weight_min": -26.83839988708496,
      "activations/layer19_attention_weight_max": 39.06760787963867,
      "activations/layer19_attention_weight_min": -33.33600997924805,
      "activations/layer1_attention_weight_max": 15.839323997497559,
      "activations/layer1_attention_weight_min": -14.66896915435791,
      "activations/layer20_attention_weight_max": 32.961177825927734,
      "activations/layer20_attention_weight_min": -25.820035934448242,
      "activations/layer21_attention_weight_max": 32.95964431762695,
      "activations/layer21_attention_weight_min": -25.22991371154785,
      "activations/layer22_attention_weight_max": 44.42373275756836,
      "activations/layer22_attention_weight_min": -33.069271087646484,
      "activations/layer23_attention_weight_max": 35.037391662597656,
      "activations/layer23_attention_weight_min": -26.90831756591797,
      "activations/layer2_attention_weight_max": 31.147993087768555,
      "activations/layer2_attention_weight_min": -31.00636100769043,
      "activations/layer3_attention_weight_max": 97.74329376220703,
      "activations/layer3_attention_weight_min": -97.88147735595703,
      "activations/layer4_attention_weight_max": 102.79605865478516,
      "activations/layer4_attention_weight_min": -107.61800384521484,
      "activations/layer5_attention_weight_max": 49.707969665527344,
      "activations/layer5_attention_weight_min": -62.20075225830078,
      "activations/layer6_attention_weight_max": 42.72928237915039,
      "activations/layer6_attention_weight_min": -48.238014221191406,
      "activations/layer7_attention_weight_max": 91.95281982421875,
      "activations/layer7_attention_weight_min": -98.08824920654297,
      "activations/layer8_attention_weight_max": 41.516902923583984,
      "activations/layer8_attention_weight_min": -44.02385330200195,
      "activations/layer9_attention_weight_max": 30.876707077026367,
      "activations/layer9_attention_weight_min": -33.498512268066406,
      "epoch": 19.66,
      "learning_rate": 2.3454545454545452e-05,
      "loss": 2.7207,
      "step": 338350
    },
    {
      "activations/layer0_attention_weight_max": 16.297134399414062,
      "activations/layer0_attention_weight_min": -12.818514823913574,
      "activations/layer10_attention_weight_max": 33.0804328918457,
      "activations/layer10_attention_weight_min": -34.98051834106445,
      "activations/layer11_attention_weight_max": 35.12525177001953,
      "activations/layer11_attention_weight_min": -34.733036041259766,
      "activations/layer12_attention_weight_max": 27.007736206054688,
      "activations/layer12_attention_weight_min": -27.040546417236328,
      "activations/layer13_attention_weight_max": 39.48006057739258,
      "activations/layer13_attention_weight_min": -36.979270935058594,
      "activations/layer14_attention_weight_max": 45.51841735839844,
      "activations/layer14_attention_weight_min": -38.017723083496094,
      "activations/layer15_attention_weight_max": 38.63918685913086,
      "activations/layer15_attention_weight_min": -35.14356994628906,
      "activations/layer16_attention_weight_max": 29.54252052307129,
      "activations/layer16_attention_weight_min": -29.592851638793945,
      "activations/layer17_attention_weight_max": 31.390676498413086,
      "activations/layer17_attention_weight_min": -26.635042190551758,
      "activations/layer18_attention_weight_max": 31.602468490600586,
      "activations/layer18_attention_weight_min": -26.4418888092041,
      "activations/layer19_attention_weight_max": 35.75132369995117,
      "activations/layer19_attention_weight_min": -33.999610900878906,
      "activations/layer1_attention_weight_max": 17.809274673461914,
      "activations/layer1_attention_weight_min": -14.66550064086914,
      "activations/layer20_attention_weight_max": 30.095014572143555,
      "activations/layer20_attention_weight_min": -25.835777282714844,
      "activations/layer21_attention_weight_max": 28.855802536010742,
      "activations/layer21_attention_weight_min": -25.202945709228516,
      "activations/layer22_attention_weight_max": 44.80252456665039,
      "activations/layer22_attention_weight_min": -32.48084259033203,
      "activations/layer23_attention_weight_max": 33.66399002075195,
      "activations/layer23_attention_weight_min": -27.736400604248047,
      "activations/layer2_attention_weight_max": 30.45977020263672,
      "activations/layer2_attention_weight_min": -30.0260066986084,
      "activations/layer3_attention_weight_max": 94.8357162475586,
      "activations/layer3_attention_weight_min": -98.25765991210938,
      "activations/layer4_attention_weight_max": 103.66143798828125,
      "activations/layer4_attention_weight_min": -108.026123046875,
      "activations/layer5_attention_weight_max": 47.72351837158203,
      "activations/layer5_attention_weight_min": -63.31652069091797,
      "activations/layer6_attention_weight_max": 42.61279296875,
      "activations/layer6_attention_weight_min": -47.372047424316406,
      "activations/layer7_attention_weight_max": 91.51547241210938,
      "activations/layer7_attention_weight_min": -95.8223876953125,
      "activations/layer8_attention_weight_max": 41.14103698730469,
      "activations/layer8_attention_weight_min": -43.41166687011719,
      "activations/layer9_attention_weight_max": 31.975339889526367,
      "activations/layer9_attention_weight_min": -33.699485778808594,
      "epoch": 19.66,
      "learning_rate": 2.3435606060606057e-05,
      "loss": 2.7223,
      "step": 338400
    },
    {
      "activations/layer0_attention_weight_max": 17.107473373413086,
      "activations/layer0_attention_weight_min": -13.762738227844238,
      "activations/layer10_attention_weight_max": 35.305809020996094,
      "activations/layer10_attention_weight_min": -34.821285247802734,
      "activations/layer11_attention_weight_max": 34.45184326171875,
      "activations/layer11_attention_weight_min": -35.49372863769531,
      "activations/layer12_attention_weight_max": 24.24604606628418,
      "activations/layer12_attention_weight_min": -25.832298278808594,
      "activations/layer13_attention_weight_max": 36.61307907104492,
      "activations/layer13_attention_weight_min": -34.74108123779297,
      "activations/layer14_attention_weight_max": 44.579559326171875,
      "activations/layer14_attention_weight_min": -39.51382827758789,
      "activations/layer15_attention_weight_max": 37.45018005371094,
      "activations/layer15_attention_weight_min": -37.03107452392578,
      "activations/layer16_attention_weight_max": 29.987369537353516,
      "activations/layer16_attention_weight_min": -28.970600128173828,
      "activations/layer17_attention_weight_max": 28.915021896362305,
      "activations/layer17_attention_weight_min": -27.527978897094727,
      "activations/layer18_attention_weight_max": 32.8989372253418,
      "activations/layer18_attention_weight_min": -27.476224899291992,
      "activations/layer19_attention_weight_max": 32.102603912353516,
      "activations/layer19_attention_weight_min": -31.40204429626465,
      "activations/layer1_attention_weight_max": 16.439966201782227,
      "activations/layer1_attention_weight_min": -15.320371627807617,
      "activations/layer20_attention_weight_max": 33.92601776123047,
      "activations/layer20_attention_weight_min": -23.80509376525879,
      "activations/layer21_attention_weight_max": 27.49041748046875,
      "activations/layer21_attention_weight_min": -24.43931007385254,
      "activations/layer22_attention_weight_max": 39.72017288208008,
      "activations/layer22_attention_weight_min": -32.25971603393555,
      "activations/layer23_attention_weight_max": 31.00433349609375,
      "activations/layer23_attention_weight_min": -25.762237548828125,
      "activations/layer2_attention_weight_max": 31.287372589111328,
      "activations/layer2_attention_weight_min": -32.7692756652832,
      "activations/layer3_attention_weight_max": 99.57112884521484,
      "activations/layer3_attention_weight_min": -101.1908950805664,
      "activations/layer4_attention_weight_max": 102.56278228759766,
      "activations/layer4_attention_weight_min": -108.77642059326172,
      "activations/layer5_attention_weight_max": 50.01509094238281,
      "activations/layer5_attention_weight_min": -59.4429931640625,
      "activations/layer6_attention_weight_max": 43.857460021972656,
      "activations/layer6_attention_weight_min": -45.028343200683594,
      "activations/layer7_attention_weight_max": 94.96675109863281,
      "activations/layer7_attention_weight_min": -95.92064666748047,
      "activations/layer8_attention_weight_max": 43.73167419433594,
      "activations/layer8_attention_weight_min": -44.664371490478516,
      "activations/layer9_attention_weight_max": 31.864282608032227,
      "activations/layer9_attention_weight_min": -36.79118347167969,
      "epoch": 19.67,
      "learning_rate": 2.3416666666666666e-05,
      "loss": 2.7253,
      "step": 338450
    },
    {
      "activations/layer0_attention_weight_max": 15.815483093261719,
      "activations/layer0_attention_weight_min": -14.136080741882324,
      "activations/layer10_attention_weight_max": 32.6508674621582,
      "activations/layer10_attention_weight_min": -35.06786346435547,
      "activations/layer11_attention_weight_max": 34.571746826171875,
      "activations/layer11_attention_weight_min": -36.55982971191406,
      "activations/layer12_attention_weight_max": 28.02207374572754,
      "activations/layer12_attention_weight_min": -28.767038345336914,
      "activations/layer13_attention_weight_max": 41.4576416015625,
      "activations/layer13_attention_weight_min": -38.94523620605469,
      "activations/layer14_attention_weight_max": 46.460506439208984,
      "activations/layer14_attention_weight_min": -39.4620246887207,
      "activations/layer15_attention_weight_max": 39.50922775268555,
      "activations/layer15_attention_weight_min": -36.769134521484375,
      "activations/layer16_attention_weight_max": 31.089370727539062,
      "activations/layer16_attention_weight_min": -30.857540130615234,
      "activations/layer17_attention_weight_max": 29.448877334594727,
      "activations/layer17_attention_weight_min": -26.466394424438477,
      "activations/layer18_attention_weight_max": 31.34304428100586,
      "activations/layer18_attention_weight_min": -25.84747886657715,
      "activations/layer19_attention_weight_max": 33.330169677734375,
      "activations/layer19_attention_weight_min": -32.36000442504883,
      "activations/layer1_attention_weight_max": 15.737634658813477,
      "activations/layer1_attention_weight_min": -13.2767972946167,
      "activations/layer20_attention_weight_max": 30.437314987182617,
      "activations/layer20_attention_weight_min": -24.995914459228516,
      "activations/layer21_attention_weight_max": 31.281993865966797,
      "activations/layer21_attention_weight_min": -24.206689834594727,
      "activations/layer22_attention_weight_max": 42.806495666503906,
      "activations/layer22_attention_weight_min": -32.09438705444336,
      "activations/layer23_attention_weight_max": 33.28632736206055,
      "activations/layer23_attention_weight_min": -27.612882614135742,
      "activations/layer2_attention_weight_max": 28.981098175048828,
      "activations/layer2_attention_weight_min": -31.27297592163086,
      "activations/layer3_attention_weight_max": 95.67224884033203,
      "activations/layer3_attention_weight_min": -96.89546203613281,
      "activations/layer4_attention_weight_max": 101.71544647216797,
      "activations/layer4_attention_weight_min": -102.6347427368164,
      "activations/layer5_attention_weight_max": 49.347496032714844,
      "activations/layer5_attention_weight_min": -62.871089935302734,
      "activations/layer6_attention_weight_max": 42.0455436706543,
      "activations/layer6_attention_weight_min": -44.885746002197266,
      "activations/layer7_attention_weight_max": 91.54791259765625,
      "activations/layer7_attention_weight_min": -99.52227783203125,
      "activations/layer8_attention_weight_max": 42.93557357788086,
      "activations/layer8_attention_weight_min": -44.620296478271484,
      "activations/layer9_attention_weight_max": 33.63922882080078,
      "activations/layer9_attention_weight_min": -34.31507110595703,
      "epoch": 19.67,
      "learning_rate": 2.339772727272727e-05,
      "loss": 2.7233,
      "step": 338500
    },
    {
      "activations/layer0_attention_weight_max": 15.658797264099121,
      "activations/layer0_attention_weight_min": -13.246563911437988,
      "activations/layer10_attention_weight_max": 34.958251953125,
      "activations/layer10_attention_weight_min": -36.102935791015625,
      "activations/layer11_attention_weight_max": 33.71324920654297,
      "activations/layer11_attention_weight_min": -35.85279846191406,
      "activations/layer12_attention_weight_max": 28.130502700805664,
      "activations/layer12_attention_weight_min": -26.637832641601562,
      "activations/layer13_attention_weight_max": 40.713863372802734,
      "activations/layer13_attention_weight_min": -37.334075927734375,
      "activations/layer14_attention_weight_max": 40.96788787841797,
      "activations/layer14_attention_weight_min": -39.46867370605469,
      "activations/layer15_attention_weight_max": 40.21570587158203,
      "activations/layer15_attention_weight_min": -36.553897857666016,
      "activations/layer16_attention_weight_max": 31.116897583007812,
      "activations/layer16_attention_weight_min": -29.246854782104492,
      "activations/layer17_attention_weight_max": 31.912012100219727,
      "activations/layer17_attention_weight_min": -25.994487762451172,
      "activations/layer18_attention_weight_max": 32.211814880371094,
      "activations/layer18_attention_weight_min": -26.273372650146484,
      "activations/layer19_attention_weight_max": 36.117855072021484,
      "activations/layer19_attention_weight_min": -32.24302291870117,
      "activations/layer1_attention_weight_max": 15.737634658813477,
      "activations/layer1_attention_weight_min": -15.541855812072754,
      "activations/layer20_attention_weight_max": 31.056291580200195,
      "activations/layer20_attention_weight_min": -24.883272171020508,
      "activations/layer21_attention_weight_max": 29.358640670776367,
      "activations/layer21_attention_weight_min": -24.998947143554688,
      "activations/layer22_attention_weight_max": 42.57950973510742,
      "activations/layer22_attention_weight_min": -31.90456771850586,
      "activations/layer23_attention_weight_max": 35.169212341308594,
      "activations/layer23_attention_weight_min": -25.677162170410156,
      "activations/layer2_attention_weight_max": 30.359806060791016,
      "activations/layer2_attention_weight_min": -29.43695640563965,
      "activations/layer3_attention_weight_max": 98.10700225830078,
      "activations/layer3_attention_weight_min": -98.3575439453125,
      "activations/layer4_attention_weight_max": 104.67121887207031,
      "activations/layer4_attention_weight_min": -112.0249252319336,
      "activations/layer5_attention_weight_max": 49.32768249511719,
      "activations/layer5_attention_weight_min": -62.36178207397461,
      "activations/layer6_attention_weight_max": 43.80404281616211,
      "activations/layer6_attention_weight_min": -46.61943054199219,
      "activations/layer7_attention_weight_max": 95.63557434082031,
      "activations/layer7_attention_weight_min": -97.66429901123047,
      "activations/layer8_attention_weight_max": 41.86726760864258,
      "activations/layer8_attention_weight_min": -42.705787658691406,
      "activations/layer9_attention_weight_max": 31.66056251525879,
      "activations/layer9_attention_weight_min": -33.32943344116211,
      "epoch": 19.67,
      "learning_rate": 2.337878787878788e-05,
      "loss": 2.7102,
      "step": 338550
    },
    {
      "activations/layer0_attention_weight_max": 16.114389419555664,
      "activations/layer0_attention_weight_min": -11.950709342956543,
      "activations/layer10_attention_weight_max": 37.206539154052734,
      "activations/layer10_attention_weight_min": -35.09141540527344,
      "activations/layer11_attention_weight_max": 35.932350158691406,
      "activations/layer11_attention_weight_min": -34.19017791748047,
      "activations/layer12_attention_weight_max": 28.658430099487305,
      "activations/layer12_attention_weight_min": -30.619850158691406,
      "activations/layer13_attention_weight_max": 40.397430419921875,
      "activations/layer13_attention_weight_min": -37.12054443359375,
      "activations/layer14_attention_weight_max": 43.6952018737793,
      "activations/layer14_attention_weight_min": -36.188148498535156,
      "activations/layer15_attention_weight_max": 39.85744857788086,
      "activations/layer15_attention_weight_min": -35.44378662109375,
      "activations/layer16_attention_weight_max": 30.734432220458984,
      "activations/layer16_attention_weight_min": -27.315996170043945,
      "activations/layer17_attention_weight_max": 28.905879974365234,
      "activations/layer17_attention_weight_min": -26.380525588989258,
      "activations/layer18_attention_weight_max": 34.84056854248047,
      "activations/layer18_attention_weight_min": -27.94384765625,
      "activations/layer19_attention_weight_max": 34.88485336303711,
      "activations/layer19_attention_weight_min": -31.177616119384766,
      "activations/layer1_attention_weight_max": 16.206762313842773,
      "activations/layer1_attention_weight_min": -14.597023963928223,
      "activations/layer20_attention_weight_max": 28.554901123046875,
      "activations/layer20_attention_weight_min": -26.559226989746094,
      "activations/layer21_attention_weight_max": 32.51941680908203,
      "activations/layer21_attention_weight_min": -25.806926727294922,
      "activations/layer22_attention_weight_max": 47.85200500488281,
      "activations/layer22_attention_weight_min": -31.671493530273438,
      "activations/layer23_attention_weight_max": 35.3607177734375,
      "activations/layer23_attention_weight_min": -25.927534103393555,
      "activations/layer2_attention_weight_max": 32.28892517089844,
      "activations/layer2_attention_weight_min": -31.290822982788086,
      "activations/layer3_attention_weight_max": 95.12369537353516,
      "activations/layer3_attention_weight_min": -96.03583526611328,
      "activations/layer4_attention_weight_max": 101.49177551269531,
      "activations/layer4_attention_weight_min": -105.92900848388672,
      "activations/layer5_attention_weight_max": 50.13819885253906,
      "activations/layer5_attention_weight_min": -60.561763763427734,
      "activations/layer6_attention_weight_max": 44.64765167236328,
      "activations/layer6_attention_weight_min": -45.58223342895508,
      "activations/layer7_attention_weight_max": 99.3380126953125,
      "activations/layer7_attention_weight_min": -94.22795867919922,
      "activations/layer8_attention_weight_max": 43.49541091918945,
      "activations/layer8_attention_weight_min": -44.50897979736328,
      "activations/layer9_attention_weight_max": 31.12067222595215,
      "activations/layer9_attention_weight_min": -33.63837814331055,
      "epoch": 19.67,
      "learning_rate": 2.335984848484848e-05,
      "loss": 2.7193,
      "step": 338600
    },
    {
      "activations/layer0_attention_weight_max": 16.65810775756836,
      "activations/layer0_attention_weight_min": -13.01317310333252,
      "activations/layer10_attention_weight_max": 35.14548873901367,
      "activations/layer10_attention_weight_min": -35.2052001953125,
      "activations/layer11_attention_weight_max": 34.61329650878906,
      "activations/layer11_attention_weight_min": -36.19072723388672,
      "activations/layer12_attention_weight_max": 25.720720291137695,
      "activations/layer12_attention_weight_min": -29.032163619995117,
      "activations/layer13_attention_weight_max": 39.3254508972168,
      "activations/layer13_attention_weight_min": -37.52959442138672,
      "activations/layer14_attention_weight_max": 42.59175491333008,
      "activations/layer14_attention_weight_min": -39.78477478027344,
      "activations/layer15_attention_weight_max": 38.69232177734375,
      "activations/layer15_attention_weight_min": -42.031394958496094,
      "activations/layer16_attention_weight_max": 30.980655670166016,
      "activations/layer16_attention_weight_min": -29.126237869262695,
      "activations/layer17_attention_weight_max": 32.47544479370117,
      "activations/layer17_attention_weight_min": -26.541568756103516,
      "activations/layer18_attention_weight_max": 34.66618347167969,
      "activations/layer18_attention_weight_min": -27.769990921020508,
      "activations/layer19_attention_weight_max": 39.53373336791992,
      "activations/layer19_attention_weight_min": -32.92599105834961,
      "activations/layer1_attention_weight_max": 17.214210510253906,
      "activations/layer1_attention_weight_min": -16.78417205810547,
      "activations/layer20_attention_weight_max": 34.63345718383789,
      "activations/layer20_attention_weight_min": -28.325387954711914,
      "activations/layer21_attention_weight_max": 33.770263671875,
      "activations/layer21_attention_weight_min": -26.643693923950195,
      "activations/layer22_attention_weight_max": 45.20949935913086,
      "activations/layer22_attention_weight_min": -34.45952224731445,
      "activations/layer23_attention_weight_max": 36.609466552734375,
      "activations/layer23_attention_weight_min": -27.837461471557617,
      "activations/layer2_attention_weight_max": 29.080068588256836,
      "activations/layer2_attention_weight_min": -28.95246696472168,
      "activations/layer3_attention_weight_max": 95.11274719238281,
      "activations/layer3_attention_weight_min": -96.95785522460938,
      "activations/layer4_attention_weight_max": 98.62679290771484,
      "activations/layer4_attention_weight_min": -102.05840301513672,
      "activations/layer5_attention_weight_max": 49.3116569519043,
      "activations/layer5_attention_weight_min": -61.93498992919922,
      "activations/layer6_attention_weight_max": 41.97547149658203,
      "activations/layer6_attention_weight_min": -45.70779800415039,
      "activations/layer7_attention_weight_max": 91.85076141357422,
      "activations/layer7_attention_weight_min": -98.19049835205078,
      "activations/layer8_attention_weight_max": 42.60801696777344,
      "activations/layer8_attention_weight_min": -44.39905548095703,
      "activations/layer9_attention_weight_max": 31.825048446655273,
      "activations/layer9_attention_weight_min": -34.09819412231445,
      "epoch": 19.68,
      "learning_rate": 2.334090909090909e-05,
      "loss": 2.7261,
      "step": 338650
    },
    {
      "activations/layer0_attention_weight_max": 15.945752143859863,
      "activations/layer0_attention_weight_min": -12.712075233459473,
      "activations/layer10_attention_weight_max": 34.703575134277344,
      "activations/layer10_attention_weight_min": -35.43687057495117,
      "activations/layer11_attention_weight_max": 35.044761657714844,
      "activations/layer11_attention_weight_min": -35.900177001953125,
      "activations/layer12_attention_weight_max": 27.66291046142578,
      "activations/layer12_attention_weight_min": -26.077478408813477,
      "activations/layer13_attention_weight_max": 41.66530227661133,
      "activations/layer13_attention_weight_min": -36.96455383300781,
      "activations/layer14_attention_weight_max": 43.49654006958008,
      "activations/layer14_attention_weight_min": -37.12574005126953,
      "activations/layer15_attention_weight_max": 41.08140563964844,
      "activations/layer15_attention_weight_min": -35.47909164428711,
      "activations/layer16_attention_weight_max": 31.211380004882812,
      "activations/layer16_attention_weight_min": -26.80406379699707,
      "activations/layer17_attention_weight_max": 29.2972354888916,
      "activations/layer17_attention_weight_min": -27.66607666015625,
      "activations/layer18_attention_weight_max": 31.578954696655273,
      "activations/layer18_attention_weight_min": -27.292329788208008,
      "activations/layer19_attention_weight_max": 32.347015380859375,
      "activations/layer19_attention_weight_min": -31.319135665893555,
      "activations/layer1_attention_weight_max": 17.053321838378906,
      "activations/layer1_attention_weight_min": -15.772974014282227,
      "activations/layer20_attention_weight_max": 31.231582641601562,
      "activations/layer20_attention_weight_min": -28.35197639465332,
      "activations/layer21_attention_weight_max": 30.403451919555664,
      "activations/layer21_attention_weight_min": -26.17044448852539,
      "activations/layer22_attention_weight_max": 37.3468132019043,
      "activations/layer22_attention_weight_min": -31.21394157409668,
      "activations/layer23_attention_weight_max": 30.392086029052734,
      "activations/layer23_attention_weight_min": -25.973739624023438,
      "activations/layer2_attention_weight_max": 30.39150047302246,
      "activations/layer2_attention_weight_min": -30.169151306152344,
      "activations/layer3_attention_weight_max": 100.07221221923828,
      "activations/layer3_attention_weight_min": -98.65157318115234,
      "activations/layer4_attention_weight_max": 106.13941955566406,
      "activations/layer4_attention_weight_min": -107.0417709350586,
      "activations/layer5_attention_weight_max": 49.32787322998047,
      "activations/layer5_attention_weight_min": -64.57763671875,
      "activations/layer6_attention_weight_max": 46.133216857910156,
      "activations/layer6_attention_weight_min": -45.56583023071289,
      "activations/layer7_attention_weight_max": 94.17410278320312,
      "activations/layer7_attention_weight_min": -100.67977905273438,
      "activations/layer8_attention_weight_max": 44.21907424926758,
      "activations/layer8_attention_weight_min": -44.59817123413086,
      "activations/layer9_attention_weight_max": 33.73355484008789,
      "activations/layer9_attention_weight_min": -34.56154251098633,
      "epoch": 19.68,
      "learning_rate": 2.3321969696969693e-05,
      "loss": 2.7426,
      "step": 338700
    },
    {
      "activations/layer0_attention_weight_max": 17.058208465576172,
      "activations/layer0_attention_weight_min": -13.816862106323242,
      "activations/layer10_attention_weight_max": 34.97672653198242,
      "activations/layer10_attention_weight_min": -34.74733352661133,
      "activations/layer11_attention_weight_max": 32.63452911376953,
      "activations/layer11_attention_weight_min": -34.18434143066406,
      "activations/layer12_attention_weight_max": 25.33539581298828,
      "activations/layer12_attention_weight_min": -26.60038185119629,
      "activations/layer13_attention_weight_max": 37.11933898925781,
      "activations/layer13_attention_weight_min": -35.08135223388672,
      "activations/layer14_attention_weight_max": 37.911746978759766,
      "activations/layer14_attention_weight_min": -36.430381774902344,
      "activations/layer15_attention_weight_max": 36.948707580566406,
      "activations/layer15_attention_weight_min": -35.58953094482422,
      "activations/layer16_attention_weight_max": 27.93743133544922,
      "activations/layer16_attention_weight_min": -29.224454879760742,
      "activations/layer17_attention_weight_max": 27.192033767700195,
      "activations/layer17_attention_weight_min": -27.379484176635742,
      "activations/layer18_attention_weight_max": 31.001201629638672,
      "activations/layer18_attention_weight_min": -27.29937744140625,
      "activations/layer19_attention_weight_max": 37.07008361816406,
      "activations/layer19_attention_weight_min": -34.304866790771484,
      "activations/layer1_attention_weight_max": 16.936477661132812,
      "activations/layer1_attention_weight_min": -16.60811424255371,
      "activations/layer20_attention_weight_max": 27.563026428222656,
      "activations/layer20_attention_weight_min": -29.273681640625,
      "activations/layer21_attention_weight_max": 27.793560028076172,
      "activations/layer21_attention_weight_min": -28.68312644958496,
      "activations/layer22_attention_weight_max": 38.057342529296875,
      "activations/layer22_attention_weight_min": -35.72626876831055,
      "activations/layer23_attention_weight_max": 31.38947296142578,
      "activations/layer23_attention_weight_min": -29.001813888549805,
      "activations/layer2_attention_weight_max": 30.640289306640625,
      "activations/layer2_attention_weight_min": -30.483158111572266,
      "activations/layer3_attention_weight_max": 103.62132263183594,
      "activations/layer3_attention_weight_min": -102.18099975585938,
      "activations/layer4_attention_weight_max": 107.81243133544922,
      "activations/layer4_attention_weight_min": -110.36624908447266,
      "activations/layer5_attention_weight_max": 49.04003143310547,
      "activations/layer5_attention_weight_min": -68.73216247558594,
      "activations/layer6_attention_weight_max": 44.205387115478516,
      "activations/layer6_attention_weight_min": -48.91376876831055,
      "activations/layer7_attention_weight_max": 89.92752075195312,
      "activations/layer7_attention_weight_min": -105.0393295288086,
      "activations/layer8_attention_weight_max": 41.759342193603516,
      "activations/layer8_attention_weight_min": -44.75959777832031,
      "activations/layer9_attention_weight_max": 32.210289001464844,
      "activations/layer9_attention_weight_min": -35.79629135131836,
      "epoch": 19.68,
      "learning_rate": 2.33030303030303e-05,
      "loss": 2.7144,
      "step": 338750
    },
    {
      "activations/layer0_attention_weight_max": 17.33955192565918,
      "activations/layer0_attention_weight_min": -13.360575675964355,
      "activations/layer10_attention_weight_max": 35.30998611450195,
      "activations/layer10_attention_weight_min": -34.908992767333984,
      "activations/layer11_attention_weight_max": 32.542606353759766,
      "activations/layer11_attention_weight_min": -34.154232025146484,
      "activations/layer12_attention_weight_max": 28.881128311157227,
      "activations/layer12_attention_weight_min": -25.477252960205078,
      "activations/layer13_attention_weight_max": 40.85319900512695,
      "activations/layer13_attention_weight_min": -33.551025390625,
      "activations/layer14_attention_weight_max": 44.91816711425781,
      "activations/layer14_attention_weight_min": -36.63480758666992,
      "activations/layer15_attention_weight_max": 39.90488052368164,
      "activations/layer15_attention_weight_min": -35.97936248779297,
      "activations/layer16_attention_weight_max": 29.901966094970703,
      "activations/layer16_attention_weight_min": -27.26671028137207,
      "activations/layer17_attention_weight_max": 30.690507888793945,
      "activations/layer17_attention_weight_min": -25.734521865844727,
      "activations/layer18_attention_weight_max": 30.961994171142578,
      "activations/layer18_attention_weight_min": -24.665218353271484,
      "activations/layer19_attention_weight_max": 36.801025390625,
      "activations/layer19_attention_weight_min": -32.38900375366211,
      "activations/layer1_attention_weight_max": 16.13978385925293,
      "activations/layer1_attention_weight_min": -14.537446022033691,
      "activations/layer20_attention_weight_max": 32.93577194213867,
      "activations/layer20_attention_weight_min": -26.817602157592773,
      "activations/layer21_attention_weight_max": 34.24455261230469,
      "activations/layer21_attention_weight_min": -27.14871597290039,
      "activations/layer22_attention_weight_max": 40.5820426940918,
      "activations/layer22_attention_weight_min": -33.156734466552734,
      "activations/layer23_attention_weight_max": 31.628990173339844,
      "activations/layer23_attention_weight_min": -26.913848876953125,
      "activations/layer2_attention_weight_max": 29.450719833374023,
      "activations/layer2_attention_weight_min": -30.99748992919922,
      "activations/layer3_attention_weight_max": 94.31982421875,
      "activations/layer3_attention_weight_min": -99.1437759399414,
      "activations/layer4_attention_weight_max": 101.64339447021484,
      "activations/layer4_attention_weight_min": -105.05885314941406,
      "activations/layer5_attention_weight_max": 46.50222396850586,
      "activations/layer5_attention_weight_min": -60.978939056396484,
      "activations/layer6_attention_weight_max": 41.89442825317383,
      "activations/layer6_attention_weight_min": -44.53486251831055,
      "activations/layer7_attention_weight_max": 88.34829711914062,
      "activations/layer7_attention_weight_min": -96.50837707519531,
      "activations/layer8_attention_weight_max": 39.87656784057617,
      "activations/layer8_attention_weight_min": -43.71889114379883,
      "activations/layer9_attention_weight_max": 32.093116760253906,
      "activations/layer9_attention_weight_min": -34.663753509521484,
      "epoch": 19.69,
      "learning_rate": 2.3284090909090906e-05,
      "loss": 2.7315,
      "step": 338800
    },
    {
      "activations/layer0_attention_weight_max": 16.883304595947266,
      "activations/layer0_attention_weight_min": -12.859109878540039,
      "activations/layer10_attention_weight_max": 35.9734992980957,
      "activations/layer10_attention_weight_min": -35.385658264160156,
      "activations/layer11_attention_weight_max": 38.71920394897461,
      "activations/layer11_attention_weight_min": -39.59123992919922,
      "activations/layer12_attention_weight_max": 25.470550537109375,
      "activations/layer12_attention_weight_min": -25.410728454589844,
      "activations/layer13_attention_weight_max": 36.99982452392578,
      "activations/layer13_attention_weight_min": -35.73096466064453,
      "activations/layer14_attention_weight_max": 41.56544876098633,
      "activations/layer14_attention_weight_min": -37.90341567993164,
      "activations/layer15_attention_weight_max": 38.44877243041992,
      "activations/layer15_attention_weight_min": -37.8489875793457,
      "activations/layer16_attention_weight_max": 27.58885383605957,
      "activations/layer16_attention_weight_min": -27.396949768066406,
      "activations/layer17_attention_weight_max": 30.48232078552246,
      "activations/layer17_attention_weight_min": -26.75538444519043,
      "activations/layer18_attention_weight_max": 32.08918380737305,
      "activations/layer18_attention_weight_min": -27.423540115356445,
      "activations/layer19_attention_weight_max": 33.69069290161133,
      "activations/layer19_attention_weight_min": -31.477680206298828,
      "activations/layer1_attention_weight_max": 18.327030181884766,
      "activations/layer1_attention_weight_min": -16.139034271240234,
      "activations/layer20_attention_weight_max": 30.616802215576172,
      "activations/layer20_attention_weight_min": -26.662153244018555,
      "activations/layer21_attention_weight_max": 29.411771774291992,
      "activations/layer21_attention_weight_min": -25.48355484008789,
      "activations/layer22_attention_weight_max": 40.50789260864258,
      "activations/layer22_attention_weight_min": -31.916107177734375,
      "activations/layer23_attention_weight_max": 28.898906707763672,
      "activations/layer23_attention_weight_min": -27.80765151977539,
      "activations/layer2_attention_weight_max": 31.628202438354492,
      "activations/layer2_attention_weight_min": -32.27275466918945,
      "activations/layer3_attention_weight_max": 96.78755187988281,
      "activations/layer3_attention_weight_min": -99.2132568359375,
      "activations/layer4_attention_weight_max": 102.04869842529297,
      "activations/layer4_attention_weight_min": -105.29711151123047,
      "activations/layer5_attention_weight_max": 47.94367599487305,
      "activations/layer5_attention_weight_min": -60.773704528808594,
      "activations/layer6_attention_weight_max": 42.333595275878906,
      "activations/layer6_attention_weight_min": -43.86044692993164,
      "activations/layer7_attention_weight_max": 90.91206359863281,
      "activations/layer7_attention_weight_min": -97.56879425048828,
      "activations/layer8_attention_weight_max": 43.84657669067383,
      "activations/layer8_attention_weight_min": -46.014102935791016,
      "activations/layer9_attention_weight_max": 33.02092361450195,
      "activations/layer9_attention_weight_min": -37.15445327758789,
      "epoch": 19.69,
      "learning_rate": 2.3265151515151515e-05,
      "loss": 2.7169,
      "step": 338850
    },
    {
      "activations/layer0_attention_weight_max": 16.51533317565918,
      "activations/layer0_attention_weight_min": -12.784159660339355,
      "activations/layer10_attention_weight_max": 34.095741271972656,
      "activations/layer10_attention_weight_min": -32.91424560546875,
      "activations/layer11_attention_weight_max": 35.490726470947266,
      "activations/layer11_attention_weight_min": -35.13853454589844,
      "activations/layer12_attention_weight_max": 32.7003288269043,
      "activations/layer12_attention_weight_min": -25.8625545501709,
      "activations/layer13_attention_weight_max": 38.786376953125,
      "activations/layer13_attention_weight_min": -35.29483413696289,
      "activations/layer14_attention_weight_max": 45.53792190551758,
      "activations/layer14_attention_weight_min": -38.32347106933594,
      "activations/layer15_attention_weight_max": 39.726356506347656,
      "activations/layer15_attention_weight_min": -36.616493225097656,
      "activations/layer16_attention_weight_max": 30.264123916625977,
      "activations/layer16_attention_weight_min": -28.188844680786133,
      "activations/layer17_attention_weight_max": 29.941951751708984,
      "activations/layer17_attention_weight_min": -25.35274314880371,
      "activations/layer18_attention_weight_max": 34.36800765991211,
      "activations/layer18_attention_weight_min": -27.21784782409668,
      "activations/layer19_attention_weight_max": 43.0279541015625,
      "activations/layer19_attention_weight_min": -33.6373405456543,
      "activations/layer1_attention_weight_max": 15.227950096130371,
      "activations/layer1_attention_weight_min": -14.456972122192383,
      "activations/layer20_attention_weight_max": 34.706626892089844,
      "activations/layer20_attention_weight_min": -25.277257919311523,
      "activations/layer21_attention_weight_max": 33.248748779296875,
      "activations/layer21_attention_weight_min": -24.925743103027344,
      "activations/layer22_attention_weight_max": 46.65135192871094,
      "activations/layer22_attention_weight_min": -32.4637336730957,
      "activations/layer23_attention_weight_max": 40.29998779296875,
      "activations/layer23_attention_weight_min": -26.614158630371094,
      "activations/layer2_attention_weight_max": 32.762939453125,
      "activations/layer2_attention_weight_min": -32.71783447265625,
      "activations/layer3_attention_weight_max": 101.44469451904297,
      "activations/layer3_attention_weight_min": -98.18727111816406,
      "activations/layer4_attention_weight_max": 104.03303527832031,
      "activations/layer4_attention_weight_min": -106.61368560791016,
      "activations/layer5_attention_weight_max": 51.854209899902344,
      "activations/layer5_attention_weight_min": -58.6468620300293,
      "activations/layer6_attention_weight_max": 42.18402099609375,
      "activations/layer6_attention_weight_min": -47.02132797241211,
      "activations/layer7_attention_weight_max": 90.5338363647461,
      "activations/layer7_attention_weight_min": -98.40869903564453,
      "activations/layer8_attention_weight_max": 41.1871223449707,
      "activations/layer8_attention_weight_min": -43.11591339111328,
      "activations/layer9_attention_weight_max": 30.782697677612305,
      "activations/layer9_attention_weight_min": -33.78761672973633,
      "epoch": 19.69,
      "learning_rate": 2.324621212121212e-05,
      "loss": 2.7154,
      "step": 338900
    },
    {
      "activations/layer0_attention_weight_max": 15.997472763061523,
      "activations/layer0_attention_weight_min": -13.2342529296875,
      "activations/layer10_attention_weight_max": 33.6609992980957,
      "activations/layer10_attention_weight_min": -34.48436737060547,
      "activations/layer11_attention_weight_max": 33.396236419677734,
      "activations/layer11_attention_weight_min": -35.013343811035156,
      "activations/layer12_attention_weight_max": 27.799341201782227,
      "activations/layer12_attention_weight_min": -28.633827209472656,
      "activations/layer13_attention_weight_max": 40.7813835144043,
      "activations/layer13_attention_weight_min": -36.98802947998047,
      "activations/layer14_attention_weight_max": 42.1605110168457,
      "activations/layer14_attention_weight_min": -37.614810943603516,
      "activations/layer15_attention_weight_max": 38.92809295654297,
      "activations/layer15_attention_weight_min": -35.648372650146484,
      "activations/layer16_attention_weight_max": 30.777116775512695,
      "activations/layer16_attention_weight_min": -29.493366241455078,
      "activations/layer17_attention_weight_max": 31.68609046936035,
      "activations/layer17_attention_weight_min": -25.822572708129883,
      "activations/layer18_attention_weight_max": 34.92494583129883,
      "activations/layer18_attention_weight_min": -27.88348960876465,
      "activations/layer19_attention_weight_max": 34.43275451660156,
      "activations/layer19_attention_weight_min": -31.47418212890625,
      "activations/layer1_attention_weight_max": 15.608631134033203,
      "activations/layer1_attention_weight_min": -14.971746444702148,
      "activations/layer20_attention_weight_max": 31.815093994140625,
      "activations/layer20_attention_weight_min": -25.860151290893555,
      "activations/layer21_attention_weight_max": 30.82109260559082,
      "activations/layer21_attention_weight_min": -25.79837417602539,
      "activations/layer22_attention_weight_max": 42.83928680419922,
      "activations/layer22_attention_weight_min": -31.85194206237793,
      "activations/layer23_attention_weight_max": 32.87948989868164,
      "activations/layer23_attention_weight_min": -25.594663619995117,
      "activations/layer2_attention_weight_max": 29.924442291259766,
      "activations/layer2_attention_weight_min": -30.300750732421875,
      "activations/layer3_attention_weight_max": 98.60163116455078,
      "activations/layer3_attention_weight_min": -98.0796127319336,
      "activations/layer4_attention_weight_max": 104.93038177490234,
      "activations/layer4_attention_weight_min": -108.56915283203125,
      "activations/layer5_attention_weight_max": 51.83032989501953,
      "activations/layer5_attention_weight_min": -63.216609954833984,
      "activations/layer6_attention_weight_max": 42.82516860961914,
      "activations/layer6_attention_weight_min": -45.794002532958984,
      "activations/layer7_attention_weight_max": 90.20852661132812,
      "activations/layer7_attention_weight_min": -95.99829864501953,
      "activations/layer8_attention_weight_max": 39.99633026123047,
      "activations/layer8_attention_weight_min": -43.119937896728516,
      "activations/layer9_attention_weight_max": 32.09635543823242,
      "activations/layer9_attention_weight_min": -35.01945877075195,
      "epoch": 19.69,
      "learning_rate": 2.3227272727272724e-05,
      "loss": 2.7325,
      "step": 338950
    },
    {
      "activations/layer0_attention_weight_max": 16.109474182128906,
      "activations/layer0_attention_weight_min": -13.758389472961426,
      "activations/layer10_attention_weight_max": 34.238319396972656,
      "activations/layer10_attention_weight_min": -35.884151458740234,
      "activations/layer11_attention_weight_max": 33.93617630004883,
      "activations/layer11_attention_weight_min": -34.37840270996094,
      "activations/layer12_attention_weight_max": 27.846525192260742,
      "activations/layer12_attention_weight_min": -27.0999813079834,
      "activations/layer13_attention_weight_max": 40.32524108886719,
      "activations/layer13_attention_weight_min": -36.86030960083008,
      "activations/layer14_attention_weight_max": 46.20933151245117,
      "activations/layer14_attention_weight_min": -40.06599044799805,
      "activations/layer15_attention_weight_max": 41.56679916381836,
      "activations/layer15_attention_weight_min": -38.327308654785156,
      "activations/layer16_attention_weight_max": 30.269807815551758,
      "activations/layer16_attention_weight_min": -30.8043212890625,
      "activations/layer17_attention_weight_max": 30.279138565063477,
      "activations/layer17_attention_weight_min": -26.020252227783203,
      "activations/layer18_attention_weight_max": 31.50892448425293,
      "activations/layer18_attention_weight_min": -26.88106346130371,
      "activations/layer19_attention_weight_max": 32.311683654785156,
      "activations/layer19_attention_weight_min": -30.3104248046875,
      "activations/layer1_attention_weight_max": 16.177337646484375,
      "activations/layer1_attention_weight_min": -14.123674392700195,
      "activations/layer20_attention_weight_max": 27.45654296875,
      "activations/layer20_attention_weight_min": -25.301286697387695,
      "activations/layer21_attention_weight_max": 29.064104080200195,
      "activations/layer21_attention_weight_min": -24.810760498046875,
      "activations/layer22_attention_weight_max": 39.27656936645508,
      "activations/layer22_attention_weight_min": -33.287899017333984,
      "activations/layer23_attention_weight_max": 29.691511154174805,
      "activations/layer23_attention_weight_min": -27.80177879333496,
      "activations/layer2_attention_weight_max": 30.590248107910156,
      "activations/layer2_attention_weight_min": -31.019298553466797,
      "activations/layer3_attention_weight_max": 98.92887115478516,
      "activations/layer3_attention_weight_min": -97.04432678222656,
      "activations/layer4_attention_weight_max": 103.5573501586914,
      "activations/layer4_attention_weight_min": -108.49653625488281,
      "activations/layer5_attention_weight_max": 49.68378448486328,
      "activations/layer5_attention_weight_min": -63.7569694519043,
      "activations/layer6_attention_weight_max": 43.93520736694336,
      "activations/layer6_attention_weight_min": -46.80130386352539,
      "activations/layer7_attention_weight_max": 88.74776458740234,
      "activations/layer7_attention_weight_min": -99.64912414550781,
      "activations/layer8_attention_weight_max": 42.253684997558594,
      "activations/layer8_attention_weight_min": -45.81282424926758,
      "activations/layer9_attention_weight_max": 33.50343322753906,
      "activations/layer9_attention_weight_min": -35.95369338989258,
      "epoch": 19.7,
      "learning_rate": 2.3208333333333333e-05,
      "loss": 2.7229,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4457,
      "eval_samples_per_second": 508.425,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4457,
      "eval_openwebtext_samples_per_second": 508.425,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9502,
      "eval_wikitext_samples_per_second": 233.824,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_lambada_loss": 2.33203125,
      "eval_lambada_ppl": 10.298839819419056,
      "eval_lambada_runtime": 9.573,
      "eval_lambada_samples_per_second": 508.62,
      "step": 339000
    },
    {
      "activations/layer0_attention_weight_max": 16.871740341186523,
      "activations/layer0_attention_weight_min": -13.040103912353516,
      "activations/layer10_attention_weight_max": 34.9385871887207,
      "activations/layer10_attention_weight_min": -34.111183166503906,
      "activations/layer11_attention_weight_max": 35.109832763671875,
      "activations/layer11_attention_weight_min": -34.13789367675781,
      "activations/layer12_attention_weight_max": 27.102338790893555,
      "activations/layer12_attention_weight_min": -27.773494720458984,
      "activations/layer13_attention_weight_max": 39.45130157470703,
      "activations/layer13_attention_weight_min": -38.14397430419922,
      "activations/layer14_attention_weight_max": 41.92774200439453,
      "activations/layer14_attention_weight_min": -39.38716125488281,
      "activations/layer15_attention_weight_max": 39.53586196899414,
      "activations/layer15_attention_weight_min": -38.172019958496094,
      "activations/layer16_attention_weight_max": 30.443408966064453,
      "activations/layer16_attention_weight_min": -28.12092399597168,
      "activations/layer17_attention_weight_max": 31.079872131347656,
      "activations/layer17_attention_weight_min": -28.45656394958496,
      "activations/layer18_attention_weight_max": 33.882537841796875,
      "activations/layer18_attention_weight_min": -26.115962982177734,
      "activations/layer19_attention_weight_max": 36.60134506225586,
      "activations/layer19_attention_weight_min": -37.14101028442383,
      "activations/layer1_attention_weight_max": 16.716337203979492,
      "activations/layer1_attention_weight_min": -13.77591609954834,
      "activations/layer20_attention_weight_max": 32.02290725708008,
      "activations/layer20_attention_weight_min": -27.491981506347656,
      "activations/layer21_attention_weight_max": 29.20339584350586,
      "activations/layer21_attention_weight_min": -26.7653865814209,
      "activations/layer22_attention_weight_max": 43.622802734375,
      "activations/layer22_attention_weight_min": -35.26649475097656,
      "activations/layer23_attention_weight_max": 35.362327575683594,
      "activations/layer23_attention_weight_min": -26.547338485717773,
      "activations/layer2_attention_weight_max": 32.322120666503906,
      "activations/layer2_attention_weight_min": -31.88448715209961,
      "activations/layer3_attention_weight_max": 97.3086929321289,
      "activations/layer3_attention_weight_min": -98.35140991210938,
      "activations/layer4_attention_weight_max": 106.0355224609375,
      "activations/layer4_attention_weight_min": -108.2476577758789,
      "activations/layer5_attention_weight_max": 55.820186614990234,
      "activations/layer5_attention_weight_min": -69.90862274169922,
      "activations/layer6_attention_weight_max": 43.60243606567383,
      "activations/layer6_attention_weight_min": -47.61309051513672,
      "activations/layer7_attention_weight_max": 93.26079559326172,
      "activations/layer7_attention_weight_min": -102.6572265625,
      "activations/layer8_attention_weight_max": 43.08617401123047,
      "activations/layer8_attention_weight_min": -43.77204132080078,
      "activations/layer9_attention_weight_max": 33.38338851928711,
      "activations/layer9_attention_weight_min": -34.31584930419922,
      "epoch": 19.7,
      "learning_rate": 2.3189393939393938e-05,
      "loss": 2.7111,
      "step": 339050
    },
    {
      "activations/layer0_attention_weight_max": 15.822471618652344,
      "activations/layer0_attention_weight_min": -12.715597152709961,
      "activations/layer10_attention_weight_max": 34.75239944458008,
      "activations/layer10_attention_weight_min": -36.117881774902344,
      "activations/layer11_attention_weight_max": 36.03033447265625,
      "activations/layer11_attention_weight_min": -35.465843200683594,
      "activations/layer12_attention_weight_max": 27.47747230529785,
      "activations/layer12_attention_weight_min": -26.975496292114258,
      "activations/layer13_attention_weight_max": 42.50812911987305,
      "activations/layer13_attention_weight_min": -38.51378631591797,
      "activations/layer14_attention_weight_max": 43.61544418334961,
      "activations/layer14_attention_weight_min": -40.49858093261719,
      "activations/layer15_attention_weight_max": 39.28532028198242,
      "activations/layer15_attention_weight_min": -36.63426971435547,
      "activations/layer16_attention_weight_max": 30.104690551757812,
      "activations/layer16_attention_weight_min": -27.51727294921875,
      "activations/layer17_attention_weight_max": 32.73209762573242,
      "activations/layer17_attention_weight_min": -28.111162185668945,
      "activations/layer18_attention_weight_max": 33.10764694213867,
      "activations/layer18_attention_weight_min": -26.800783157348633,
      "activations/layer19_attention_weight_max": 32.253658294677734,
      "activations/layer19_attention_weight_min": -31.874927520751953,
      "activations/layer1_attention_weight_max": 15.565696716308594,
      "activations/layer1_attention_weight_min": -13.710641860961914,
      "activations/layer20_attention_weight_max": 29.50299644470215,
      "activations/layer20_attention_weight_min": -26.83641242980957,
      "activations/layer21_attention_weight_max": 30.002859115600586,
      "activations/layer21_attention_weight_min": -25.086599349975586,
      "activations/layer22_attention_weight_max": 40.501564025878906,
      "activations/layer22_attention_weight_min": -34.0314826965332,
      "activations/layer23_attention_weight_max": 34.0086669921875,
      "activations/layer23_attention_weight_min": -29.958290100097656,
      "activations/layer2_attention_weight_max": 28.8254451751709,
      "activations/layer2_attention_weight_min": -28.109134674072266,
      "activations/layer3_attention_weight_max": 93.89178466796875,
      "activations/layer3_attention_weight_min": -95.13936614990234,
      "activations/layer4_attention_weight_max": 103.32757568359375,
      "activations/layer4_attention_weight_min": -101.4346923828125,
      "activations/layer5_attention_weight_max": 52.032997131347656,
      "activations/layer5_attention_weight_min": -68.75303649902344,
      "activations/layer6_attention_weight_max": 46.41057205200195,
      "activations/layer6_attention_weight_min": -47.11045455932617,
      "activations/layer7_attention_weight_max": 92.17938232421875,
      "activations/layer7_attention_weight_min": -98.20294189453125,
      "activations/layer8_attention_weight_max": 43.95353698730469,
      "activations/layer8_attention_weight_min": -45.694114685058594,
      "activations/layer9_attention_weight_max": 34.02582550048828,
      "activations/layer9_attention_weight_min": -35.13353729248047,
      "epoch": 19.7,
      "learning_rate": 2.3170454545454546e-05,
      "loss": 2.7213,
      "step": 339100
    },
    {
      "activations/layer0_attention_weight_max": 16.05946922302246,
      "activations/layer0_attention_weight_min": -12.927281379699707,
      "activations/layer10_attention_weight_max": 31.391597747802734,
      "activations/layer10_attention_weight_min": -32.64361572265625,
      "activations/layer11_attention_weight_max": 33.612857818603516,
      "activations/layer11_attention_weight_min": -33.24835968017578,
      "activations/layer12_attention_weight_max": 26.61425018310547,
      "activations/layer12_attention_weight_min": -25.704370498657227,
      "activations/layer13_attention_weight_max": 41.25797653198242,
      "activations/layer13_attention_weight_min": -36.68041229248047,
      "activations/layer14_attention_weight_max": 48.169921875,
      "activations/layer14_attention_weight_min": -39.18660354614258,
      "activations/layer15_attention_weight_max": 46.06474304199219,
      "activations/layer15_attention_weight_min": -40.998165130615234,
      "activations/layer16_attention_weight_max": 35.970252990722656,
      "activations/layer16_attention_weight_min": -30.9290771484375,
      "activations/layer17_attention_weight_max": 30.744855880737305,
      "activations/layer17_attention_weight_min": -28.33685874938965,
      "activations/layer18_attention_weight_max": 32.0103874206543,
      "activations/layer18_attention_weight_min": -27.910892486572266,
      "activations/layer19_attention_weight_max": 35.59772491455078,
      "activations/layer19_attention_weight_min": -31.743606567382812,
      "activations/layer1_attention_weight_max": 16.43558120727539,
      "activations/layer1_attention_weight_min": -14.815293312072754,
      "activations/layer20_attention_weight_max": 31.157825469970703,
      "activations/layer20_attention_weight_min": -26.759355545043945,
      "activations/layer21_attention_weight_max": 29.432668685913086,
      "activations/layer21_attention_weight_min": -25.2466983795166,
      "activations/layer22_attention_weight_max": 44.83943557739258,
      "activations/layer22_attention_weight_min": -32.75944137573242,
      "activations/layer23_attention_weight_max": 38.15028762817383,
      "activations/layer23_attention_weight_min": -26.132835388183594,
      "activations/layer2_attention_weight_max": 30.22437858581543,
      "activations/layer2_attention_weight_min": -30.08283042907715,
      "activations/layer3_attention_weight_max": 94.7047348022461,
      "activations/layer3_attention_weight_min": -94.95799255371094,
      "activations/layer4_attention_weight_max": 100.04010009765625,
      "activations/layer4_attention_weight_min": -103.94084930419922,
      "activations/layer5_attention_weight_max": 50.48637008666992,
      "activations/layer5_attention_weight_min": -59.470436096191406,
      "activations/layer6_attention_weight_max": 41.01244354248047,
      "activations/layer6_attention_weight_min": -43.30289840698242,
      "activations/layer7_attention_weight_max": 87.69646453857422,
      "activations/layer7_attention_weight_min": -93.0535888671875,
      "activations/layer8_attention_weight_max": 41.345401763916016,
      "activations/layer8_attention_weight_min": -42.483394622802734,
      "activations/layer9_attention_weight_max": 30.381534576416016,
      "activations/layer9_attention_weight_min": -32.806209564208984,
      "epoch": 19.71,
      "learning_rate": 2.3151515151515147e-05,
      "loss": 2.7203,
      "step": 339150
    },
    {
      "activations/layer0_attention_weight_max": 15.892064094543457,
      "activations/layer0_attention_weight_min": -14.142810821533203,
      "activations/layer10_attention_weight_max": 34.889678955078125,
      "activations/layer10_attention_weight_min": -34.89925765991211,
      "activations/layer11_attention_weight_max": 33.567298889160156,
      "activations/layer11_attention_weight_min": -34.63759231567383,
      "activations/layer12_attention_weight_max": 26.772846221923828,
      "activations/layer12_attention_weight_min": -26.683053970336914,
      "activations/layer13_attention_weight_max": 41.14697265625,
      "activations/layer13_attention_weight_min": -38.29249572753906,
      "activations/layer14_attention_weight_max": 43.069217681884766,
      "activations/layer14_attention_weight_min": -41.35297393798828,
      "activations/layer15_attention_weight_max": 39.54876708984375,
      "activations/layer15_attention_weight_min": -35.851707458496094,
      "activations/layer16_attention_weight_max": 33.466041564941406,
      "activations/layer16_attention_weight_min": -29.541135787963867,
      "activations/layer17_attention_weight_max": 30.64472770690918,
      "activations/layer17_attention_weight_min": -27.05430030822754,
      "activations/layer18_attention_weight_max": 32.153343200683594,
      "activations/layer18_attention_weight_min": -26.3458309173584,
      "activations/layer19_attention_weight_max": 36.334625244140625,
      "activations/layer19_attention_weight_min": -32.18106460571289,
      "activations/layer1_attention_weight_max": 15.379902839660645,
      "activations/layer1_attention_weight_min": -13.963761329650879,
      "activations/layer20_attention_weight_max": 31.897668838500977,
      "activations/layer20_attention_weight_min": -25.156164169311523,
      "activations/layer21_attention_weight_max": 30.10007667541504,
      "activations/layer21_attention_weight_min": -25.18369483947754,
      "activations/layer22_attention_weight_max": 44.94235610961914,
      "activations/layer22_attention_weight_min": -31.93031883239746,
      "activations/layer23_attention_weight_max": 34.59925079345703,
      "activations/layer23_attention_weight_min": -27.820316314697266,
      "activations/layer2_attention_weight_max": 32.683624267578125,
      "activations/layer2_attention_weight_min": -30.58917999267578,
      "activations/layer3_attention_weight_max": 97.2369155883789,
      "activations/layer3_attention_weight_min": -100.43212127685547,
      "activations/layer4_attention_weight_max": 104.1019515991211,
      "activations/layer4_attention_weight_min": -108.62784576416016,
      "activations/layer5_attention_weight_max": 53.13460922241211,
      "activations/layer5_attention_weight_min": -63.475990295410156,
      "activations/layer6_attention_weight_max": 43.94618225097656,
      "activations/layer6_attention_weight_min": -46.920902252197266,
      "activations/layer7_attention_weight_max": 93.00833129882812,
      "activations/layer7_attention_weight_min": -96.27952575683594,
      "activations/layer8_attention_weight_max": 41.349700927734375,
      "activations/layer8_attention_weight_min": -45.70671081542969,
      "activations/layer9_attention_weight_max": 32.98441696166992,
      "activations/layer9_attention_weight_min": -36.48113250732422,
      "epoch": 19.71,
      "learning_rate": 2.3132575757575756e-05,
      "loss": 2.7343,
      "step": 339200
    },
    {
      "activations/layer0_attention_weight_max": 16.00849151611328,
      "activations/layer0_attention_weight_min": -13.887420654296875,
      "activations/layer10_attention_weight_max": 35.55577850341797,
      "activations/layer10_attention_weight_min": -37.14381408691406,
      "activations/layer11_attention_weight_max": 38.1490478515625,
      "activations/layer11_attention_weight_min": -35.9727897644043,
      "activations/layer12_attention_weight_max": 29.332643508911133,
      "activations/layer12_attention_weight_min": -27.929859161376953,
      "activations/layer13_attention_weight_max": 42.17243194580078,
      "activations/layer13_attention_weight_min": -38.85356521606445,
      "activations/layer14_attention_weight_max": 42.264556884765625,
      "activations/layer14_attention_weight_min": -39.392433166503906,
      "activations/layer15_attention_weight_max": 45.11097717285156,
      "activations/layer15_attention_weight_min": -36.05462646484375,
      "activations/layer16_attention_weight_max": 30.61114501953125,
      "activations/layer16_attention_weight_min": -28.349794387817383,
      "activations/layer17_attention_weight_max": 30.448076248168945,
      "activations/layer17_attention_weight_min": -26.022701263427734,
      "activations/layer18_attention_weight_max": 33.07316970825195,
      "activations/layer18_attention_weight_min": -28.738718032836914,
      "activations/layer19_attention_weight_max": 33.30918502807617,
      "activations/layer19_attention_weight_min": -29.930692672729492,
      "activations/layer1_attention_weight_max": 15.621156692504883,
      "activations/layer1_attention_weight_min": -15.033025741577148,
      "activations/layer20_attention_weight_max": 27.992664337158203,
      "activations/layer20_attention_weight_min": -24.49627685546875,
      "activations/layer21_attention_weight_max": 31.489530563354492,
      "activations/layer21_attention_weight_min": -24.192947387695312,
      "activations/layer22_attention_weight_max": 44.069244384765625,
      "activations/layer22_attention_weight_min": -31.089284896850586,
      "activations/layer23_attention_weight_max": 31.39229393005371,
      "activations/layer23_attention_weight_min": -25.89215660095215,
      "activations/layer2_attention_weight_max": 30.428394317626953,
      "activations/layer2_attention_weight_min": -30.284896850585938,
      "activations/layer3_attention_weight_max": 102.53005981445312,
      "activations/layer3_attention_weight_min": -100.27311706542969,
      "activations/layer4_attention_weight_max": 108.84004211425781,
      "activations/layer4_attention_weight_min": -111.28594207763672,
      "activations/layer5_attention_weight_max": 49.487754821777344,
      "activations/layer5_attention_weight_min": -64.63653564453125,
      "activations/layer6_attention_weight_max": 44.95406723022461,
      "activations/layer6_attention_weight_min": -46.977054595947266,
      "activations/layer7_attention_weight_max": 99.73065948486328,
      "activations/layer7_attention_weight_min": -98.33619689941406,
      "activations/layer8_attention_weight_max": 43.048858642578125,
      "activations/layer8_attention_weight_min": -47.67426300048828,
      "activations/layer9_attention_weight_max": 32.89756774902344,
      "activations/layer9_attention_weight_min": -35.03691101074219,
      "epoch": 19.71,
      "learning_rate": 2.311363636363636e-05,
      "loss": 2.7318,
      "step": 339250
    },
    {
      "activations/layer0_attention_weight_max": 15.920086860656738,
      "activations/layer0_attention_weight_min": -12.872941970825195,
      "activations/layer10_attention_weight_max": 37.27836608886719,
      "activations/layer10_attention_weight_min": -37.699371337890625,
      "activations/layer11_attention_weight_max": 37.49989700317383,
      "activations/layer11_attention_weight_min": -36.532386779785156,
      "activations/layer12_attention_weight_max": 28.85029411315918,
      "activations/layer12_attention_weight_min": -28.500614166259766,
      "activations/layer13_attention_weight_max": 42.65950012207031,
      "activations/layer13_attention_weight_min": -40.62650680541992,
      "activations/layer14_attention_weight_max": 44.58566665649414,
      "activations/layer14_attention_weight_min": -41.10615921020508,
      "activations/layer15_attention_weight_max": 38.05713653564453,
      "activations/layer15_attention_weight_min": -40.03548812866211,
      "activations/layer16_attention_weight_max": 31.43509864807129,
      "activations/layer16_attention_weight_min": -28.286569595336914,
      "activations/layer17_attention_weight_max": 28.773353576660156,
      "activations/layer17_attention_weight_min": -26.869365692138672,
      "activations/layer18_attention_weight_max": 31.4875545501709,
      "activations/layer18_attention_weight_min": -29.682531356811523,
      "activations/layer19_attention_weight_max": 35.992286682128906,
      "activations/layer19_attention_weight_min": -32.48463439941406,
      "activations/layer1_attention_weight_max": 15.254167556762695,
      "activations/layer1_attention_weight_min": -15.450937271118164,
      "activations/layer20_attention_weight_max": 28.62575340270996,
      "activations/layer20_attention_weight_min": -26.174419403076172,
      "activations/layer21_attention_weight_max": 27.82512092590332,
      "activations/layer21_attention_weight_min": -24.52037239074707,
      "activations/layer22_attention_weight_max": 46.56715393066406,
      "activations/layer22_attention_weight_min": -32.80236053466797,
      "activations/layer23_attention_weight_max": 33.51045227050781,
      "activations/layer23_attention_weight_min": -28.539932250976562,
      "activations/layer2_attention_weight_max": 32.98899841308594,
      "activations/layer2_attention_weight_min": -32.287986755371094,
      "activations/layer3_attention_weight_max": 102.98638916015625,
      "activations/layer3_attention_weight_min": -102.94757080078125,
      "activations/layer4_attention_weight_max": 109.89818572998047,
      "activations/layer4_attention_weight_min": -111.63072967529297,
      "activations/layer5_attention_weight_max": 54.78911590576172,
      "activations/layer5_attention_weight_min": -65.86534881591797,
      "activations/layer6_attention_weight_max": 45.35764694213867,
      "activations/layer6_attention_weight_min": -48.11121368408203,
      "activations/layer7_attention_weight_max": 102.12164306640625,
      "activations/layer7_attention_weight_min": -101.53717041015625,
      "activations/layer8_attention_weight_max": 45.640804290771484,
      "activations/layer8_attention_weight_min": -45.003517150878906,
      "activations/layer9_attention_weight_max": 34.376163482666016,
      "activations/layer9_attention_weight_min": -36.28672790527344,
      "epoch": 19.72,
      "learning_rate": 2.309469696969697e-05,
      "loss": 2.6986,
      "step": 339300
    },
    {
      "activations/layer0_attention_weight_max": 15.677230834960938,
      "activations/layer0_attention_weight_min": -14.140704154968262,
      "activations/layer10_attention_weight_max": 34.964229583740234,
      "activations/layer10_attention_weight_min": -36.122650146484375,
      "activations/layer11_attention_weight_max": 38.58750534057617,
      "activations/layer11_attention_weight_min": -39.57746505737305,
      "activations/layer12_attention_weight_max": 28.986881256103516,
      "activations/layer12_attention_weight_min": -28.862598419189453,
      "activations/layer13_attention_weight_max": 44.3103141784668,
      "activations/layer13_attention_weight_min": -37.66827392578125,
      "activations/layer14_attention_weight_max": 46.975730895996094,
      "activations/layer14_attention_weight_min": -41.245574951171875,
      "activations/layer15_attention_weight_max": 42.98708724975586,
      "activations/layer15_attention_weight_min": -38.4205436706543,
      "activations/layer16_attention_weight_max": 33.47236251831055,
      "activations/layer16_attention_weight_min": -28.760894775390625,
      "activations/layer17_attention_weight_max": 33.51865005493164,
      "activations/layer17_attention_weight_min": -27.41398048400879,
      "activations/layer18_attention_weight_max": 34.81822967529297,
      "activations/layer18_attention_weight_min": -29.64801025390625,
      "activations/layer19_attention_weight_max": 37.228271484375,
      "activations/layer19_attention_weight_min": -34.46332931518555,
      "activations/layer1_attention_weight_max": 16.2491397857666,
      "activations/layer1_attention_weight_min": -14.295967102050781,
      "activations/layer20_attention_weight_max": 33.51578140258789,
      "activations/layer20_attention_weight_min": -25.30828094482422,
      "activations/layer21_attention_weight_max": 32.225616455078125,
      "activations/layer21_attention_weight_min": -25.551706314086914,
      "activations/layer22_attention_weight_max": 45.906368255615234,
      "activations/layer22_attention_weight_min": -33.14104080200195,
      "activations/layer23_attention_weight_max": 36.7020263671875,
      "activations/layer23_attention_weight_min": -27.27950668334961,
      "activations/layer2_attention_weight_max": 30.507545471191406,
      "activations/layer2_attention_weight_min": -30.071044921875,
      "activations/layer3_attention_weight_max": 101.85845947265625,
      "activations/layer3_attention_weight_min": -98.46502685546875,
      "activations/layer4_attention_weight_max": 112.04796600341797,
      "activations/layer4_attention_weight_min": -104.90245819091797,
      "activations/layer5_attention_weight_max": 50.09110641479492,
      "activations/layer5_attention_weight_min": -63.9421501159668,
      "activations/layer6_attention_weight_max": 43.83074188232422,
      "activations/layer6_attention_weight_min": -46.21751022338867,
      "activations/layer7_attention_weight_max": 94.48529052734375,
      "activations/layer7_attention_weight_min": -100.63101959228516,
      "activations/layer8_attention_weight_max": 41.218135833740234,
      "activations/layer8_attention_weight_min": -46.44854736328125,
      "activations/layer9_attention_weight_max": 32.08962631225586,
      "activations/layer9_attention_weight_min": -37.521297454833984,
      "epoch": 19.72,
      "learning_rate": 2.3075757575757574e-05,
      "loss": 2.7136,
      "step": 339350
    },
    {
      "activations/layer0_attention_weight_max": 15.720630645751953,
      "activations/layer0_attention_weight_min": -14.203147888183594,
      "activations/layer10_attention_weight_max": 33.5361328125,
      "activations/layer10_attention_weight_min": -33.47510528564453,
      "activations/layer11_attention_weight_max": 34.31208419799805,
      "activations/layer11_attention_weight_min": -34.574378967285156,
      "activations/layer12_attention_weight_max": 26.815439224243164,
      "activations/layer12_attention_weight_min": -27.033044815063477,
      "activations/layer13_attention_weight_max": 39.136138916015625,
      "activations/layer13_attention_weight_min": -36.35569763183594,
      "activations/layer14_attention_weight_max": 43.312644958496094,
      "activations/layer14_attention_weight_min": -38.00796890258789,
      "activations/layer15_attention_weight_max": 38.96529006958008,
      "activations/layer15_attention_weight_min": -37.44642639160156,
      "activations/layer16_attention_weight_max": 29.370697021484375,
      "activations/layer16_attention_weight_min": -29.056386947631836,
      "activations/layer17_attention_weight_max": 33.70015335083008,
      "activations/layer17_attention_weight_min": -28.889177322387695,
      "activations/layer18_attention_weight_max": 36.813926696777344,
      "activations/layer18_attention_weight_min": -26.644269943237305,
      "activations/layer19_attention_weight_max": 34.42841339111328,
      "activations/layer19_attention_weight_min": -33.854766845703125,
      "activations/layer1_attention_weight_max": 14.5618257522583,
      "activations/layer1_attention_weight_min": -13.865196228027344,
      "activations/layer20_attention_weight_max": 31.795984268188477,
      "activations/layer20_attention_weight_min": -27.689878463745117,
      "activations/layer21_attention_weight_max": 29.882116317749023,
      "activations/layer21_attention_weight_min": -25.934860229492188,
      "activations/layer22_attention_weight_max": 44.9713249206543,
      "activations/layer22_attention_weight_min": -34.24315643310547,
      "activations/layer23_attention_weight_max": 32.4931755065918,
      "activations/layer23_attention_weight_min": -28.92917251586914,
      "activations/layer2_attention_weight_max": 31.07927703857422,
      "activations/layer2_attention_weight_min": -29.673603057861328,
      "activations/layer3_attention_weight_max": 99.39044952392578,
      "activations/layer3_attention_weight_min": -93.98406219482422,
      "activations/layer4_attention_weight_max": 108.70882415771484,
      "activations/layer4_attention_weight_min": -105.93277740478516,
      "activations/layer5_attention_weight_max": 50.66671371459961,
      "activations/layer5_attention_weight_min": -62.47724533081055,
      "activations/layer6_attention_weight_max": 43.40317916870117,
      "activations/layer6_attention_weight_min": -46.114871978759766,
      "activations/layer7_attention_weight_max": 95.57828521728516,
      "activations/layer7_attention_weight_min": -92.88260650634766,
      "activations/layer8_attention_weight_max": 43.046600341796875,
      "activations/layer8_attention_weight_min": -44.69752883911133,
      "activations/layer9_attention_weight_max": 32.95648193359375,
      "activations/layer9_attention_weight_min": -34.4526252746582,
      "epoch": 19.72,
      "learning_rate": 2.3056818181818182e-05,
      "loss": 2.7187,
      "step": 339400
    },
    {
      "activations/layer0_attention_weight_max": 16.562725067138672,
      "activations/layer0_attention_weight_min": -14.378211975097656,
      "activations/layer10_attention_weight_max": 36.32657241821289,
      "activations/layer10_attention_weight_min": -37.658939361572266,
      "activations/layer11_attention_weight_max": 33.5767936706543,
      "activations/layer11_attention_weight_min": -36.12065887451172,
      "activations/layer12_attention_weight_max": 29.289596557617188,
      "activations/layer12_attention_weight_min": -27.939191818237305,
      "activations/layer13_attention_weight_max": 41.96859359741211,
      "activations/layer13_attention_weight_min": -37.26450729370117,
      "activations/layer14_attention_weight_max": 43.20766067504883,
      "activations/layer14_attention_weight_min": -39.618743896484375,
      "activations/layer15_attention_weight_max": 39.50904846191406,
      "activations/layer15_attention_weight_min": -36.55142593383789,
      "activations/layer16_attention_weight_max": 29.20856475830078,
      "activations/layer16_attention_weight_min": -27.545333862304688,
      "activations/layer17_attention_weight_max": 30.58878517150879,
      "activations/layer17_attention_weight_min": -25.937055587768555,
      "activations/layer18_attention_weight_max": 34.076995849609375,
      "activations/layer18_attention_weight_min": -29.745426177978516,
      "activations/layer19_attention_weight_max": 31.351511001586914,
      "activations/layer19_attention_weight_min": -30.360946655273438,
      "activations/layer1_attention_weight_max": 15.766365051269531,
      "activations/layer1_attention_weight_min": -13.859857559204102,
      "activations/layer20_attention_weight_max": 29.43788719177246,
      "activations/layer20_attention_weight_min": -25.236186981201172,
      "activations/layer21_attention_weight_max": 28.971200942993164,
      "activations/layer21_attention_weight_min": -24.987510681152344,
      "activations/layer22_attention_weight_max": 39.97932815551758,
      "activations/layer22_attention_weight_min": -33.38737487792969,
      "activations/layer23_attention_weight_max": 30.55986785888672,
      "activations/layer23_attention_weight_min": -25.649425506591797,
      "activations/layer2_attention_weight_max": 30.673259735107422,
      "activations/layer2_attention_weight_min": -29.9315242767334,
      "activations/layer3_attention_weight_max": 94.98346710205078,
      "activations/layer3_attention_weight_min": -98.59481811523438,
      "activations/layer4_attention_weight_max": 106.26014709472656,
      "activations/layer4_attention_weight_min": -107.70867919921875,
      "activations/layer5_attention_weight_max": 50.720619201660156,
      "activations/layer5_attention_weight_min": -60.1483154296875,
      "activations/layer6_attention_weight_max": 43.70030212402344,
      "activations/layer6_attention_weight_min": -48.12309646606445,
      "activations/layer7_attention_weight_max": 92.22896575927734,
      "activations/layer7_attention_weight_min": -111.77188110351562,
      "activations/layer8_attention_weight_max": 41.930145263671875,
      "activations/layer8_attention_weight_min": -45.79767990112305,
      "activations/layer9_attention_weight_max": 31.328649520874023,
      "activations/layer9_attention_weight_min": -35.12272262573242,
      "epoch": 19.72,
      "learning_rate": 2.3037878787878783e-05,
      "loss": 2.7158,
      "step": 339450
    },
    {
      "activations/layer0_attention_weight_max": 15.95652961730957,
      "activations/layer0_attention_weight_min": -13.040441513061523,
      "activations/layer10_attention_weight_max": 34.1417236328125,
      "activations/layer10_attention_weight_min": -35.6336555480957,
      "activations/layer11_attention_weight_max": 35.01493835449219,
      "activations/layer11_attention_weight_min": -36.35248947143555,
      "activations/layer12_attention_weight_max": 30.522571563720703,
      "activations/layer12_attention_weight_min": -27.54542350769043,
      "activations/layer13_attention_weight_max": 45.61967849731445,
      "activations/layer13_attention_weight_min": -38.331390380859375,
      "activations/layer14_attention_weight_max": 47.67360305786133,
      "activations/layer14_attention_weight_min": -41.77865219116211,
      "activations/layer15_attention_weight_max": 44.750633239746094,
      "activations/layer15_attention_weight_min": -42.13231658935547,
      "activations/layer16_attention_weight_max": 35.107086181640625,
      "activations/layer16_attention_weight_min": -29.84425163269043,
      "activations/layer17_attention_weight_max": 33.22958755493164,
      "activations/layer17_attention_weight_min": -28.71302604675293,
      "activations/layer18_attention_weight_max": 37.452484130859375,
      "activations/layer18_attention_weight_min": -25.655012130737305,
      "activations/layer19_attention_weight_max": 40.44102096557617,
      "activations/layer19_attention_weight_min": -32.31350326538086,
      "activations/layer1_attention_weight_max": 15.277570724487305,
      "activations/layer1_attention_weight_min": -14.253238677978516,
      "activations/layer20_attention_weight_max": 34.04159927368164,
      "activations/layer20_attention_weight_min": -25.038118362426758,
      "activations/layer21_attention_weight_max": 33.60167694091797,
      "activations/layer21_attention_weight_min": -26.212072372436523,
      "activations/layer22_attention_weight_max": 50.92629623413086,
      "activations/layer22_attention_weight_min": -33.024803161621094,
      "activations/layer23_attention_weight_max": 39.67622375488281,
      "activations/layer23_attention_weight_min": -25.803159713745117,
      "activations/layer2_attention_weight_max": 30.762775421142578,
      "activations/layer2_attention_weight_min": -28.9362850189209,
      "activations/layer3_attention_weight_max": 93.14930725097656,
      "activations/layer3_attention_weight_min": -92.10908508300781,
      "activations/layer4_attention_weight_max": 103.17704010009766,
      "activations/layer4_attention_weight_min": -105.18464660644531,
      "activations/layer5_attention_weight_max": 50.528892517089844,
      "activations/layer5_attention_weight_min": -63.56388854980469,
      "activations/layer6_attention_weight_max": 44.49028778076172,
      "activations/layer6_attention_weight_min": -48.474239349365234,
      "activations/layer7_attention_weight_max": 96.58113098144531,
      "activations/layer7_attention_weight_min": -101.40746307373047,
      "activations/layer8_attention_weight_max": 44.330142974853516,
      "activations/layer8_attention_weight_min": -44.83549880981445,
      "activations/layer9_attention_weight_max": 32.55548858642578,
      "activations/layer9_attention_weight_min": -35.319149017333984,
      "epoch": 19.73,
      "learning_rate": 2.3018939393939392e-05,
      "loss": 2.7314,
      "step": 339500
    },
    {
      "activations/layer0_attention_weight_max": 16.21521759033203,
      "activations/layer0_attention_weight_min": -13.01716136932373,
      "activations/layer10_attention_weight_max": 37.40920639038086,
      "activations/layer10_attention_weight_min": -35.3753662109375,
      "activations/layer11_attention_weight_max": 38.63581848144531,
      "activations/layer11_attention_weight_min": -36.82391357421875,
      "activations/layer12_attention_weight_max": 26.844867706298828,
      "activations/layer12_attention_weight_min": -25.603092193603516,
      "activations/layer13_attention_weight_max": 37.916683197021484,
      "activations/layer13_attention_weight_min": -35.56406784057617,
      "activations/layer14_attention_weight_max": 40.192264556884766,
      "activations/layer14_attention_weight_min": -36.7880973815918,
      "activations/layer15_attention_weight_max": 39.82268524169922,
      "activations/layer15_attention_weight_min": -36.0120735168457,
      "activations/layer16_attention_weight_max": 28.220199584960938,
      "activations/layer16_attention_weight_min": -28.149200439453125,
      "activations/layer17_attention_weight_max": 33.39468002319336,
      "activations/layer17_attention_weight_min": -24.8437442779541,
      "activations/layer18_attention_weight_max": 34.974853515625,
      "activations/layer18_attention_weight_min": -27.189531326293945,
      "activations/layer19_attention_weight_max": 36.11644744873047,
      "activations/layer19_attention_weight_min": -31.67097282409668,
      "activations/layer1_attention_weight_max": 16.09880256652832,
      "activations/layer1_attention_weight_min": -14.094353675842285,
      "activations/layer20_attention_weight_max": 29.96027183532715,
      "activations/layer20_attention_weight_min": -24.793846130371094,
      "activations/layer21_attention_weight_max": 29.61313819885254,
      "activations/layer21_attention_weight_min": -23.944808959960938,
      "activations/layer22_attention_weight_max": 42.66851806640625,
      "activations/layer22_attention_weight_min": -30.8450984954834,
      "activations/layer23_attention_weight_max": 34.743709564208984,
      "activations/layer23_attention_weight_min": -26.384521484375,
      "activations/layer2_attention_weight_max": 31.883161544799805,
      "activations/layer2_attention_weight_min": -31.813533782958984,
      "activations/layer3_attention_weight_max": 98.17298126220703,
      "activations/layer3_attention_weight_min": -100.62593841552734,
      "activations/layer4_attention_weight_max": 105.1552963256836,
      "activations/layer4_attention_weight_min": -111.29888153076172,
      "activations/layer5_attention_weight_max": 52.603694915771484,
      "activations/layer5_attention_weight_min": -63.33800506591797,
      "activations/layer6_attention_weight_max": 45.072425842285156,
      "activations/layer6_attention_weight_min": -49.505924224853516,
      "activations/layer7_attention_weight_max": 95.49232482910156,
      "activations/layer7_attention_weight_min": -100.13692474365234,
      "activations/layer8_attention_weight_max": 45.3828239440918,
      "activations/layer8_attention_weight_min": -44.07878494262695,
      "activations/layer9_attention_weight_max": 34.29991149902344,
      "activations/layer9_attention_weight_min": -36.26490020751953,
      "epoch": 19.73,
      "learning_rate": 2.2999999999999997e-05,
      "loss": 2.7052,
      "step": 339550
    },
    {
      "activations/layer0_attention_weight_max": 15.821857452392578,
      "activations/layer0_attention_weight_min": -12.156721115112305,
      "activations/layer10_attention_weight_max": 57.191097259521484,
      "activations/layer10_attention_weight_min": -56.96519470214844,
      "activations/layer11_attention_weight_max": 57.434059143066406,
      "activations/layer11_attention_weight_min": -58.029266357421875,
      "activations/layer12_attention_weight_max": 40.25843811035156,
      "activations/layer12_attention_weight_min": -37.08203887939453,
      "activations/layer13_attention_weight_max": 51.169742584228516,
      "activations/layer13_attention_weight_min": -46.530296325683594,
      "activations/layer14_attention_weight_max": 57.09477615356445,
      "activations/layer14_attention_weight_min": -50.319313049316406,
      "activations/layer15_attention_weight_max": 55.80133056640625,
      "activations/layer15_attention_weight_min": -49.067222595214844,
      "activations/layer16_attention_weight_max": 37.316349029541016,
      "activations/layer16_attention_weight_min": -31.180118560791016,
      "activations/layer17_attention_weight_max": 32.297462463378906,
      "activations/layer17_attention_weight_min": -31.319801330566406,
      "activations/layer18_attention_weight_max": 32.63843536376953,
      "activations/layer18_attention_weight_min": -31.115285873413086,
      "activations/layer19_attention_weight_max": 34.53757858276367,
      "activations/layer19_attention_weight_min": -33.02313995361328,
      "activations/layer1_attention_weight_max": 16.028873443603516,
      "activations/layer1_attention_weight_min": -14.640527725219727,
      "activations/layer20_attention_weight_max": 32.02338409423828,
      "activations/layer20_attention_weight_min": -27.784500122070312,
      "activations/layer21_attention_weight_max": 31.98929214477539,
      "activations/layer21_attention_weight_min": -28.317440032958984,
      "activations/layer22_attention_weight_max": 45.50754928588867,
      "activations/layer22_attention_weight_min": -36.81071090698242,
      "activations/layer23_attention_weight_max": 32.93719482421875,
      "activations/layer23_attention_weight_min": -27.927326202392578,
      "activations/layer2_attention_weight_max": 34.5418701171875,
      "activations/layer2_attention_weight_min": -34.118995666503906,
      "activations/layer3_attention_weight_max": 106.31583404541016,
      "activations/layer3_attention_weight_min": -108.95692443847656,
      "activations/layer4_attention_weight_max": 113.87080383300781,
      "activations/layer4_attention_weight_min": -115.6735610961914,
      "activations/layer5_attention_weight_max": 55.69396209716797,
      "activations/layer5_attention_weight_min": -64.47312927246094,
      "activations/layer6_attention_weight_max": 53.1212043762207,
      "activations/layer6_attention_weight_min": -53.36375427246094,
      "activations/layer7_attention_weight_max": 126.31388854980469,
      "activations/layer7_attention_weight_min": -117.73088073730469,
      "activations/layer8_attention_weight_max": 61.87050247192383,
      "activations/layer8_attention_weight_min": -62.26946258544922,
      "activations/layer9_attention_weight_max": 48.28867721557617,
      "activations/layer9_attention_weight_min": -51.33864974975586,
      "epoch": 19.73,
      "learning_rate": 2.2981060606060605e-05,
      "loss": 2.7176,
      "step": 339600
    },
    {
      "activations/layer0_attention_weight_max": 16.963403701782227,
      "activations/layer0_attention_weight_min": -12.103044509887695,
      "activations/layer10_attention_weight_max": 34.02366256713867,
      "activations/layer10_attention_weight_min": -33.54485321044922,
      "activations/layer11_attention_weight_max": 33.26163864135742,
      "activations/layer11_attention_weight_min": -34.18293380737305,
      "activations/layer12_attention_weight_max": 27.31852149963379,
      "activations/layer12_attention_weight_min": -27.013105392456055,
      "activations/layer13_attention_weight_max": 42.06817626953125,
      "activations/layer13_attention_weight_min": -36.32142639160156,
      "activations/layer14_attention_weight_max": 43.095848083496094,
      "activations/layer14_attention_weight_min": -37.870460510253906,
      "activations/layer15_attention_weight_max": 43.18735885620117,
      "activations/layer15_attention_weight_min": -41.56487274169922,
      "activations/layer16_attention_weight_max": 32.37854766845703,
      "activations/layer16_attention_weight_min": -29.516733169555664,
      "activations/layer17_attention_weight_max": 33.449920654296875,
      "activations/layer17_attention_weight_min": -30.13212013244629,
      "activations/layer18_attention_weight_max": 32.398399353027344,
      "activations/layer18_attention_weight_min": -30.59132194519043,
      "activations/layer19_attention_weight_max": 36.1130485534668,
      "activations/layer19_attention_weight_min": -34.41958236694336,
      "activations/layer1_attention_weight_max": 15.413468360900879,
      "activations/layer1_attention_weight_min": -13.995851516723633,
      "activations/layer20_attention_weight_max": 33.15829849243164,
      "activations/layer20_attention_weight_min": -27.470613479614258,
      "activations/layer21_attention_weight_max": 33.55573654174805,
      "activations/layer21_attention_weight_min": -25.572973251342773,
      "activations/layer22_attention_weight_max": 42.672706604003906,
      "activations/layer22_attention_weight_min": -33.22995376586914,
      "activations/layer23_attention_weight_max": 37.290740966796875,
      "activations/layer23_attention_weight_min": -26.072263717651367,
      "activations/layer2_attention_weight_max": 31.091018676757812,
      "activations/layer2_attention_weight_min": -30.720176696777344,
      "activations/layer3_attention_weight_max": 101.90914916992188,
      "activations/layer3_attention_weight_min": -98.42327117919922,
      "activations/layer4_attention_weight_max": 108.20861053466797,
      "activations/layer4_attention_weight_min": -108.48854064941406,
      "activations/layer5_attention_weight_max": 50.51134490966797,
      "activations/layer5_attention_weight_min": -65.42267608642578,
      "activations/layer6_attention_weight_max": 46.41889953613281,
      "activations/layer6_attention_weight_min": -46.54662322998047,
      "activations/layer7_attention_weight_max": 95.01899719238281,
      "activations/layer7_attention_weight_min": -96.09650421142578,
      "activations/layer8_attention_weight_max": 41.670387268066406,
      "activations/layer8_attention_weight_min": -43.26371765136719,
      "activations/layer9_attention_weight_max": 32.85298538208008,
      "activations/layer9_attention_weight_min": -33.47609329223633,
      "epoch": 19.74,
      "learning_rate": 2.296212121212121e-05,
      "loss": 2.7108,
      "step": 339650
    },
    {
      "activations/layer0_attention_weight_max": 17.216510772705078,
      "activations/layer0_attention_weight_min": -12.990823745727539,
      "activations/layer10_attention_weight_max": 32.45484924316406,
      "activations/layer10_attention_weight_min": -34.5216064453125,
      "activations/layer11_attention_weight_max": 32.81376647949219,
      "activations/layer11_attention_weight_min": -36.4501953125,
      "activations/layer12_attention_weight_max": 25.147457122802734,
      "activations/layer12_attention_weight_min": -26.364704132080078,
      "activations/layer13_attention_weight_max": 41.672080993652344,
      "activations/layer13_attention_weight_min": -34.053157806396484,
      "activations/layer14_attention_weight_max": 49.75784683227539,
      "activations/layer14_attention_weight_min": -39.87068176269531,
      "activations/layer15_attention_weight_max": 42.81844711303711,
      "activations/layer15_attention_weight_min": -36.57775115966797,
      "activations/layer16_attention_weight_max": 33.76947784423828,
      "activations/layer16_attention_weight_min": -29.021682739257812,
      "activations/layer17_attention_weight_max": 35.098106384277344,
      "activations/layer17_attention_weight_min": -26.705219268798828,
      "activations/layer18_attention_weight_max": 34.17174530029297,
      "activations/layer18_attention_weight_min": -27.73822021484375,
      "activations/layer19_attention_weight_max": 34.99711227416992,
      "activations/layer19_attention_weight_min": -32.23269271850586,
      "activations/layer1_attention_weight_max": 15.74916934967041,
      "activations/layer1_attention_weight_min": -14.588262557983398,
      "activations/layer20_attention_weight_max": 29.923227310180664,
      "activations/layer20_attention_weight_min": -24.75762367248535,
      "activations/layer21_attention_weight_max": 30.34732437133789,
      "activations/layer21_attention_weight_min": -24.515262603759766,
      "activations/layer22_attention_weight_max": 42.524681091308594,
      "activations/layer22_attention_weight_min": -31.137563705444336,
      "activations/layer23_attention_weight_max": 30.431638717651367,
      "activations/layer23_attention_weight_min": -24.51667594909668,
      "activations/layer2_attention_weight_max": 30.184803009033203,
      "activations/layer2_attention_weight_min": -29.462095260620117,
      "activations/layer3_attention_weight_max": 98.0555648803711,
      "activations/layer3_attention_weight_min": -97.4203109741211,
      "activations/layer4_attention_weight_max": 105.64464569091797,
      "activations/layer4_attention_weight_min": -105.239013671875,
      "activations/layer5_attention_weight_max": 50.20848083496094,
      "activations/layer5_attention_weight_min": -58.85259246826172,
      "activations/layer6_attention_weight_max": 42.936256408691406,
      "activations/layer6_attention_weight_min": -45.8895378112793,
      "activations/layer7_attention_weight_max": 95.2899398803711,
      "activations/layer7_attention_weight_min": -95.22421264648438,
      "activations/layer8_attention_weight_max": 41.58787536621094,
      "activations/layer8_attention_weight_min": -44.32661437988281,
      "activations/layer9_attention_weight_max": 30.752628326416016,
      "activations/layer9_attention_weight_min": -34.531490325927734,
      "epoch": 19.74,
      "learning_rate": 2.2943181818181815e-05,
      "loss": 2.7266,
      "step": 339700
    },
    {
      "activations/layer0_attention_weight_max": 16.846399307250977,
      "activations/layer0_attention_weight_min": -13.098581314086914,
      "activations/layer10_attention_weight_max": 50.01826095581055,
      "activations/layer10_attention_weight_min": -48.78422927856445,
      "activations/layer11_attention_weight_max": 48.540679931640625,
      "activations/layer11_attention_weight_min": -50.30492401123047,
      "activations/layer12_attention_weight_max": 32.047935485839844,
      "activations/layer12_attention_weight_min": -32.39023971557617,
      "activations/layer13_attention_weight_max": 44.87545394897461,
      "activations/layer13_attention_weight_min": -45.01317596435547,
      "activations/layer14_attention_weight_max": 48.29486846923828,
      "activations/layer14_attention_weight_min": -49.43899154663086,
      "activations/layer15_attention_weight_max": 48.47835922241211,
      "activations/layer15_attention_weight_min": -43.75243377685547,
      "activations/layer16_attention_weight_max": 33.00271224975586,
      "activations/layer16_attention_weight_min": -28.71493148803711,
      "activations/layer17_attention_weight_max": 31.55137062072754,
      "activations/layer17_attention_weight_min": -28.450931549072266,
      "activations/layer18_attention_weight_max": 31.189109802246094,
      "activations/layer18_attention_weight_min": -26.26317024230957,
      "activations/layer19_attention_weight_max": 31.94759178161621,
      "activations/layer19_attention_weight_min": -29.87384605407715,
      "activations/layer1_attention_weight_max": 15.695301055908203,
      "activations/layer1_attention_weight_min": -14.0972261428833,
      "activations/layer20_attention_weight_max": 28.27049446105957,
      "activations/layer20_attention_weight_min": -24.18174171447754,
      "activations/layer21_attention_weight_max": 27.740901947021484,
      "activations/layer21_attention_weight_min": -24.898448944091797,
      "activations/layer22_attention_weight_max": 40.151512145996094,
      "activations/layer22_attention_weight_min": -31.18539810180664,
      "activations/layer23_attention_weight_max": 30.11638641357422,
      "activations/layer23_attention_weight_min": -24.60348129272461,
      "activations/layer2_attention_weight_max": 35.01614761352539,
      "activations/layer2_attention_weight_min": -35.71540832519531,
      "activations/layer3_attention_weight_max": 110.08201599121094,
      "activations/layer3_attention_weight_min": -111.68682861328125,
      "activations/layer4_attention_weight_max": 115.7960205078125,
      "activations/layer4_attention_weight_min": -111.8019790649414,
      "activations/layer5_attention_weight_max": 53.0604248046875,
      "activations/layer5_attention_weight_min": -60.1241569519043,
      "activations/layer6_attention_weight_max": 48.73387908935547,
      "activations/layer6_attention_weight_min": -47.96949005126953,
      "activations/layer7_attention_weight_max": 121.37408447265625,
      "activations/layer7_attention_weight_min": -116.77042388916016,
      "activations/layer8_attention_weight_max": 54.81850051879883,
      "activations/layer8_attention_weight_min": -55.01057815551758,
      "activations/layer9_attention_weight_max": 41.93175506591797,
      "activations/layer9_attention_weight_min": -45.4478759765625,
      "epoch": 19.74,
      "learning_rate": 2.2924242424242423e-05,
      "loss": 2.7323,
      "step": 339750
    },
    {
      "activations/layer0_attention_weight_max": 15.803790092468262,
      "activations/layer0_attention_weight_min": -12.929670333862305,
      "activations/layer10_attention_weight_max": 35.809078216552734,
      "activations/layer10_attention_weight_min": -35.6844482421875,
      "activations/layer11_attention_weight_max": 36.406158447265625,
      "activations/layer11_attention_weight_min": -35.844024658203125,
      "activations/layer12_attention_weight_max": 28.322834014892578,
      "activations/layer12_attention_weight_min": -27.324111938476562,
      "activations/layer13_attention_weight_max": 40.621212005615234,
      "activations/layer13_attention_weight_min": -38.04362487792969,
      "activations/layer14_attention_weight_max": 46.83258819580078,
      "activations/layer14_attention_weight_min": -39.927764892578125,
      "activations/layer15_attention_weight_max": 39.2880973815918,
      "activations/layer15_attention_weight_min": -39.44428253173828,
      "activations/layer16_attention_weight_max": 34.25693130493164,
      "activations/layer16_attention_weight_min": -29.270553588867188,
      "activations/layer17_attention_weight_max": 34.1758918762207,
      "activations/layer17_attention_weight_min": -27.741493225097656,
      "activations/layer18_attention_weight_max": 35.573089599609375,
      "activations/layer18_attention_weight_min": -25.558992385864258,
      "activations/layer19_attention_weight_max": 37.37499237060547,
      "activations/layer19_attention_weight_min": -34.3835334777832,
      "activations/layer1_attention_weight_max": 15.95887565612793,
      "activations/layer1_attention_weight_min": -14.233474731445312,
      "activations/layer20_attention_weight_max": 30.112401962280273,
      "activations/layer20_attention_weight_min": -29.331554412841797,
      "activations/layer21_attention_weight_max": 31.467227935791016,
      "activations/layer21_attention_weight_min": -28.826366424560547,
      "activations/layer22_attention_weight_max": 44.30122375488281,
      "activations/layer22_attention_weight_min": -35.45004653930664,
      "activations/layer23_attention_weight_max": 34.835140228271484,
      "activations/layer23_attention_weight_min": -26.029312133789062,
      "activations/layer2_attention_weight_max": 31.021085739135742,
      "activations/layer2_attention_weight_min": -29.91084861755371,
      "activations/layer3_attention_weight_max": 94.97427368164062,
      "activations/layer3_attention_weight_min": -97.79751586914062,
      "activations/layer4_attention_weight_max": 105.33602142333984,
      "activations/layer4_attention_weight_min": -107.47737121582031,
      "activations/layer5_attention_weight_max": 48.979286193847656,
      "activations/layer5_attention_weight_min": -62.040672302246094,
      "activations/layer6_attention_weight_max": 47.236488342285156,
      "activations/layer6_attention_weight_min": -49.256752014160156,
      "activations/layer7_attention_weight_max": 96.41645812988281,
      "activations/layer7_attention_weight_min": -108.8510971069336,
      "activations/layer8_attention_weight_max": 42.61400604248047,
      "activations/layer8_attention_weight_min": -45.86231994628906,
      "activations/layer9_attention_weight_max": 33.774532318115234,
      "activations/layer9_attention_weight_min": -35.1026496887207,
      "epoch": 19.74,
      "learning_rate": 2.2905303030303028e-05,
      "loss": 2.7235,
      "step": 339800
    },
    {
      "activations/layer0_attention_weight_max": 15.76125717163086,
      "activations/layer0_attention_weight_min": -12.670595169067383,
      "activations/layer10_attention_weight_max": 32.38325119018555,
      "activations/layer10_attention_weight_min": -34.28864288330078,
      "activations/layer11_attention_weight_max": 32.3546257019043,
      "activations/layer11_attention_weight_min": -33.92424392700195,
      "activations/layer12_attention_weight_max": 27.099441528320312,
      "activations/layer12_attention_weight_min": -26.985021591186523,
      "activations/layer13_attention_weight_max": 40.68205642700195,
      "activations/layer13_attention_weight_min": -39.9802131652832,
      "activations/layer14_attention_weight_max": 42.509891510009766,
      "activations/layer14_attention_weight_min": -41.484622955322266,
      "activations/layer15_attention_weight_max": 39.17038345336914,
      "activations/layer15_attention_weight_min": -35.85092544555664,
      "activations/layer16_attention_weight_max": 29.484758377075195,
      "activations/layer16_attention_weight_min": -28.447317123413086,
      "activations/layer17_attention_weight_max": 28.91638946533203,
      "activations/layer17_attention_weight_min": -27.55366325378418,
      "activations/layer18_attention_weight_max": 30.340557098388672,
      "activations/layer18_attention_weight_min": -24.842029571533203,
      "activations/layer19_attention_weight_max": 31.60995864868164,
      "activations/layer19_attention_weight_min": -31.972335815429688,
      "activations/layer1_attention_weight_max": 15.535161018371582,
      "activations/layer1_attention_weight_min": -15.45564079284668,
      "activations/layer20_attention_weight_max": 28.1310977935791,
      "activations/layer20_attention_weight_min": -25.567026138305664,
      "activations/layer21_attention_weight_max": 27.432592391967773,
      "activations/layer21_attention_weight_min": -26.939817428588867,
      "activations/layer22_attention_weight_max": 41.50400924682617,
      "activations/layer22_attention_weight_min": -31.244136810302734,
      "activations/layer23_attention_weight_max": 30.20721435546875,
      "activations/layer23_attention_weight_min": -26.38684844970703,
      "activations/layer2_attention_weight_max": 32.6905517578125,
      "activations/layer2_attention_weight_min": -30.12527847290039,
      "activations/layer3_attention_weight_max": 98.11385345458984,
      "activations/layer3_attention_weight_min": -97.69342803955078,
      "activations/layer4_attention_weight_max": 104.20550537109375,
      "activations/layer4_attention_weight_min": -105.95672607421875,
      "activations/layer5_attention_weight_max": 51.669105529785156,
      "activations/layer5_attention_weight_min": -61.796661376953125,
      "activations/layer6_attention_weight_max": 43.593284606933594,
      "activations/layer6_attention_weight_min": -46.310123443603516,
      "activations/layer7_attention_weight_max": 94.86083984375,
      "activations/layer7_attention_weight_min": -95.24861907958984,
      "activations/layer8_attention_weight_max": 41.261600494384766,
      "activations/layer8_attention_weight_min": -43.298439025878906,
      "activations/layer9_attention_weight_max": 32.7451057434082,
      "activations/layer9_attention_weight_min": -34.69480895996094,
      "epoch": 19.75,
      "learning_rate": 2.2886363636363636e-05,
      "loss": 2.7187,
      "step": 339850
    },
    {
      "activations/layer0_attention_weight_max": 15.948235511779785,
      "activations/layer0_attention_weight_min": -12.73696231842041,
      "activations/layer10_attention_weight_max": 36.25577163696289,
      "activations/layer10_attention_weight_min": -36.76543045043945,
      "activations/layer11_attention_weight_max": 36.0299186706543,
      "activations/layer11_attention_weight_min": -36.354103088378906,
      "activations/layer12_attention_weight_max": 30.755720138549805,
      "activations/layer12_attention_weight_min": -30.504194259643555,
      "activations/layer13_attention_weight_max": 48.04555130004883,
      "activations/layer13_attention_weight_min": -42.160804748535156,
      "activations/layer14_attention_weight_max": 45.66343688964844,
      "activations/layer14_attention_weight_min": -41.91856384277344,
      "activations/layer15_attention_weight_max": 45.983856201171875,
      "activations/layer15_attention_weight_min": -37.38855743408203,
      "activations/layer16_attention_weight_max": 32.334434509277344,
      "activations/layer16_attention_weight_min": -30.266077041625977,
      "activations/layer17_attention_weight_max": 34.37763214111328,
      "activations/layer17_attention_weight_min": -25.946460723876953,
      "activations/layer18_attention_weight_max": 34.63608932495117,
      "activations/layer18_attention_weight_min": -25.753231048583984,
      "activations/layer19_attention_weight_max": 37.39434051513672,
      "activations/layer19_attention_weight_min": -34.57514190673828,
      "activations/layer1_attention_weight_max": 16.32069206237793,
      "activations/layer1_attention_weight_min": -14.6654691696167,
      "activations/layer20_attention_weight_max": 32.61772155761719,
      "activations/layer20_attention_weight_min": -29.824499130249023,
      "activations/layer21_attention_weight_max": 31.961761474609375,
      "activations/layer21_attention_weight_min": -27.597652435302734,
      "activations/layer22_attention_weight_max": 47.20096206665039,
      "activations/layer22_attention_weight_min": -33.550384521484375,
      "activations/layer23_attention_weight_max": 35.383567810058594,
      "activations/layer23_attention_weight_min": -25.72549819946289,
      "activations/layer2_attention_weight_max": 32.24164581298828,
      "activations/layer2_attention_weight_min": -30.387100219726562,
      "activations/layer3_attention_weight_max": 97.53196716308594,
      "activations/layer3_attention_weight_min": -98.053466796875,
      "activations/layer4_attention_weight_max": 110.2572021484375,
      "activations/layer4_attention_weight_min": -108.1533432006836,
      "activations/layer5_attention_weight_max": 51.409114837646484,
      "activations/layer5_attention_weight_min": -62.7118034362793,
      "activations/layer6_attention_weight_max": 44.20338439941406,
      "activations/layer6_attention_weight_min": -48.50355529785156,
      "activations/layer7_attention_weight_max": 98.38944244384766,
      "activations/layer7_attention_weight_min": -98.92269897460938,
      "activations/layer8_attention_weight_max": 42.703575134277344,
      "activations/layer8_attention_weight_min": -47.538211822509766,
      "activations/layer9_attention_weight_max": 35.018497467041016,
      "activations/layer9_attention_weight_min": -36.763221740722656,
      "epoch": 19.75,
      "learning_rate": 2.286742424242424e-05,
      "loss": 2.738,
      "step": 339900
    },
    {
      "activations/layer0_attention_weight_max": 16.656618118286133,
      "activations/layer0_attention_weight_min": -12.510167121887207,
      "activations/layer10_attention_weight_max": 40.2140998840332,
      "activations/layer10_attention_weight_min": -37.20492172241211,
      "activations/layer11_attention_weight_max": 39.47385787963867,
      "activations/layer11_attention_weight_min": -38.181434631347656,
      "activations/layer12_attention_weight_max": 28.81711196899414,
      "activations/layer12_attention_weight_min": -28.731021881103516,
      "activations/layer13_attention_weight_max": 42.71902847290039,
      "activations/layer13_attention_weight_min": -38.029380798339844,
      "activations/layer14_attention_weight_max": 48.39198684692383,
      "activations/layer14_attention_weight_min": -41.00424575805664,
      "activations/layer15_attention_weight_max": 41.82788848876953,
      "activations/layer15_attention_weight_min": -37.08820724487305,
      "activations/layer16_attention_weight_max": 29.690000534057617,
      "activations/layer16_attention_weight_min": -29.383991241455078,
      "activations/layer17_attention_weight_max": 31.87417221069336,
      "activations/layer17_attention_weight_min": -26.101364135742188,
      "activations/layer18_attention_weight_max": 35.96691131591797,
      "activations/layer18_attention_weight_min": -25.946311950683594,
      "activations/layer19_attention_weight_max": 35.0269889831543,
      "activations/layer19_attention_weight_min": -30.23845863342285,
      "activations/layer1_attention_weight_max": 15.365928649902344,
      "activations/layer1_attention_weight_min": -15.034512519836426,
      "activations/layer20_attention_weight_max": 31.017406463623047,
      "activations/layer20_attention_weight_min": -23.929264068603516,
      "activations/layer21_attention_weight_max": 31.319377899169922,
      "activations/layer21_attention_weight_min": -24.914907455444336,
      "activations/layer22_attention_weight_max": 45.14654541015625,
      "activations/layer22_attention_weight_min": -30.334484100341797,
      "activations/layer23_attention_weight_max": 33.34178924560547,
      "activations/layer23_attention_weight_min": -26.3538875579834,
      "activations/layer2_attention_weight_max": 32.211578369140625,
      "activations/layer2_attention_weight_min": -30.929969787597656,
      "activations/layer3_attention_weight_max": 98.11141967773438,
      "activations/layer3_attention_weight_min": -99.5945816040039,
      "activations/layer4_attention_weight_max": 106.46858215332031,
      "activations/layer4_attention_weight_min": -110.9247817993164,
      "activations/layer5_attention_weight_max": 50.45481491088867,
      "activations/layer5_attention_weight_min": -61.62884521484375,
      "activations/layer6_attention_weight_max": 47.073692321777344,
      "activations/layer6_attention_weight_min": -47.17805862426758,
      "activations/layer7_attention_weight_max": 97.45278930664062,
      "activations/layer7_attention_weight_min": -104.76715087890625,
      "activations/layer8_attention_weight_max": 46.42378616333008,
      "activations/layer8_attention_weight_min": -48.11602020263672,
      "activations/layer9_attention_weight_max": 35.40484619140625,
      "activations/layer9_attention_weight_min": -37.495365142822266,
      "epoch": 19.75,
      "learning_rate": 2.284848484848485e-05,
      "loss": 2.7248,
      "step": 339950
    },
    {
      "activations/layer0_attention_weight_max": 16.320009231567383,
      "activations/layer0_attention_weight_min": -12.68468952178955,
      "activations/layer10_attention_weight_max": 33.814613342285156,
      "activations/layer10_attention_weight_min": -34.73701095581055,
      "activations/layer11_attention_weight_max": 34.138343811035156,
      "activations/layer11_attention_weight_min": -33.84230422973633,
      "activations/layer12_attention_weight_max": 28.29169464111328,
      "activations/layer12_attention_weight_min": -28.40380096435547,
      "activations/layer13_attention_weight_max": 40.756011962890625,
      "activations/layer13_attention_weight_min": -37.449459075927734,
      "activations/layer14_attention_weight_max": 41.737388610839844,
      "activations/layer14_attention_weight_min": -40.32482147216797,
      "activations/layer15_attention_weight_max": 39.24293518066406,
      "activations/layer15_attention_weight_min": -36.66263198852539,
      "activations/layer16_attention_weight_max": 35.57136917114258,
      "activations/layer16_attention_weight_min": -28.595125198364258,
      "activations/layer17_attention_weight_max": 30.27947425842285,
      "activations/layer17_attention_weight_min": -25.30342674255371,
      "activations/layer18_attention_weight_max": 33.42859649658203,
      "activations/layer18_attention_weight_min": -26.496021270751953,
      "activations/layer19_attention_weight_max": 32.96697998046875,
      "activations/layer19_attention_weight_min": -30.05610466003418,
      "activations/layer1_attention_weight_max": 15.882020950317383,
      "activations/layer1_attention_weight_min": -13.847334861755371,
      "activations/layer20_attention_weight_max": 28.149553298950195,
      "activations/layer20_attention_weight_min": -25.84657859802246,
      "activations/layer21_attention_weight_max": 27.645641326904297,
      "activations/layer21_attention_weight_min": -25.584165573120117,
      "activations/layer22_attention_weight_max": 41.892822265625,
      "activations/layer22_attention_weight_min": -29.768999099731445,
      "activations/layer23_attention_weight_max": 28.976545333862305,
      "activations/layer23_attention_weight_min": -26.313232421875,
      "activations/layer2_attention_weight_max": 30.303844451904297,
      "activations/layer2_attention_weight_min": -30.476699829101562,
      "activations/layer3_attention_weight_max": 96.41607666015625,
      "activations/layer3_attention_weight_min": -96.82485961914062,
      "activations/layer4_attention_weight_max": 104.8143539428711,
      "activations/layer4_attention_weight_min": -107.72798156738281,
      "activations/layer5_attention_weight_max": 50.27471923828125,
      "activations/layer5_attention_weight_min": -66.68933868408203,
      "activations/layer6_attention_weight_max": 44.01322937011719,
      "activations/layer6_attention_weight_min": -46.059593200683594,
      "activations/layer7_attention_weight_max": 94.3509292602539,
      "activations/layer7_attention_weight_min": -96.44458770751953,
      "activations/layer8_attention_weight_max": 43.889801025390625,
      "activations/layer8_attention_weight_min": -44.97038650512695,
      "activations/layer9_attention_weight_max": 33.425350189208984,
      "activations/layer9_attention_weight_min": -35.893836975097656,
      "epoch": 19.76,
      "learning_rate": 2.282954545454545e-05,
      "loss": 2.7382,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.4451,
      "eval_samples_per_second": 508.462,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.4451,
      "eval_openwebtext_samples_per_second": 508.462,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9595,
      "eval_wikitext_samples_per_second": 232.71,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_lambada_loss": 2.345703125,
      "eval_lambada_ppl": 10.440611202065584,
      "eval_lambada_runtime": 9.5601,
      "eval_lambada_samples_per_second": 509.306,
      "step": 340000
    },
    {
      "activations/layer0_attention_weight_max": 16.022014617919922,
      "activations/layer0_attention_weight_min": -13.943458557128906,
      "activations/layer10_attention_weight_max": 34.94636917114258,
      "activations/layer10_attention_weight_min": -36.183990478515625,
      "activations/layer11_attention_weight_max": 36.59698486328125,
      "activations/layer11_attention_weight_min": -37.37513732910156,
      "activations/layer12_attention_weight_max": 27.956851959228516,
      "activations/layer12_attention_weight_min": -28.081113815307617,
      "activations/layer13_attention_weight_max": 41.68849182128906,
      "activations/layer13_attention_weight_min": -36.43333435058594,
      "activations/layer14_attention_weight_max": 42.925662994384766,
      "activations/layer14_attention_weight_min": -38.01887130737305,
      "activations/layer15_attention_weight_max": 48.34111022949219,
      "activations/layer15_attention_weight_min": -35.692142486572266,
      "activations/layer16_attention_weight_max": 31.093778610229492,
      "activations/layer16_attention_weight_min": -28.437768936157227,
      "activations/layer17_attention_weight_max": 30.97201156616211,
      "activations/layer17_attention_weight_min": -27.60796546936035,
      "activations/layer18_attention_weight_max": 34.46626663208008,
      "activations/layer18_attention_weight_min": -29.441936492919922,
      "activations/layer19_attention_weight_max": 33.60755157470703,
      "activations/layer19_attention_weight_min": -36.18007278442383,
      "activations/layer1_attention_weight_max": 15.552323341369629,
      "activations/layer1_attention_weight_min": -14.541872024536133,
      "activations/layer20_attention_weight_max": 28.83592414855957,
      "activations/layer20_attention_weight_min": -29.58205223083496,
      "activations/layer21_attention_weight_max": 27.70244026184082,
      "activations/layer21_attention_weight_min": -27.054513931274414,
      "activations/layer22_attention_weight_max": 43.51216125488281,
      "activations/layer22_attention_weight_min": -35.15827560424805,
      "activations/layer23_attention_weight_max": 31.338857650756836,
      "activations/layer23_attention_weight_min": -27.978046417236328,
      "activations/layer2_attention_weight_max": 31.790367126464844,
      "activations/layer2_attention_weight_min": -30.37398910522461,
      "activations/layer3_attention_weight_max": 98.28968811035156,
      "activations/layer3_attention_weight_min": -95.7702407836914,
      "activations/layer4_attention_weight_max": 103.76016998291016,
      "activations/layer4_attention_weight_min": -108.432373046875,
      "activations/layer5_attention_weight_max": 48.807106018066406,
      "activations/layer5_attention_weight_min": -62.12324142456055,
      "activations/layer6_attention_weight_max": 43.16667556762695,
      "activations/layer6_attention_weight_min": -48.09465026855469,
      "activations/layer7_attention_weight_max": 99.67801666259766,
      "activations/layer7_attention_weight_min": -95.31709289550781,
      "activations/layer8_attention_weight_max": 42.610748291015625,
      "activations/layer8_attention_weight_min": -46.0565071105957,
      "activations/layer9_attention_weight_max": 35.426692962646484,
      "activations/layer9_attention_weight_min": -36.8067626953125,
      "epoch": 19.76,
      "learning_rate": 2.281060606060606e-05,
      "loss": 2.7128,
      "step": 340050
    },
    {
      "activations/layer0_attention_weight_max": 16.438541412353516,
      "activations/layer0_attention_weight_min": -13.011218070983887,
      "activations/layer10_attention_weight_max": 36.365840911865234,
      "activations/layer10_attention_weight_min": -37.87603759765625,
      "activations/layer11_attention_weight_max": 37.100547790527344,
      "activations/layer11_attention_weight_min": -37.830841064453125,
      "activations/layer12_attention_weight_max": 28.669387817382812,
      "activations/layer12_attention_weight_min": -28.16160774230957,
      "activations/layer13_attention_weight_max": 39.8448486328125,
      "activations/layer13_attention_weight_min": -37.668800354003906,
      "activations/layer14_attention_weight_max": 42.77731704711914,
      "activations/layer14_attention_weight_min": -39.54141616821289,
      "activations/layer15_attention_weight_max": 40.81218338012695,
      "activations/layer15_attention_weight_min": -38.035919189453125,
      "activations/layer16_attention_weight_max": 30.369083404541016,
      "activations/layer16_attention_weight_min": -28.782686233520508,
      "activations/layer17_attention_weight_max": 28.433664321899414,
      "activations/layer17_attention_weight_min": -26.58988380432129,
      "activations/layer18_attention_weight_max": 33.70339584350586,
      "activations/layer18_attention_weight_min": -25.243743896484375,
      "activations/layer19_attention_weight_max": 35.0081672668457,
      "activations/layer19_attention_weight_min": -31.087934494018555,
      "activations/layer1_attention_weight_max": 16.1604061126709,
      "activations/layer1_attention_weight_min": -14.68109130859375,
      "activations/layer20_attention_weight_max": 26.7142391204834,
      "activations/layer20_attention_weight_min": -24.753206253051758,
      "activations/layer21_attention_weight_max": 28.554216384887695,
      "activations/layer21_attention_weight_min": -24.07745361328125,
      "activations/layer22_attention_weight_max": 42.6679573059082,
      "activations/layer22_attention_weight_min": -31.440814971923828,
      "activations/layer23_attention_weight_max": 32.26130676269531,
      "activations/layer23_attention_weight_min": -28.27477264404297,
      "activations/layer2_attention_weight_max": 32.132408142089844,
      "activations/layer2_attention_weight_min": -31.47711944580078,
      "activations/layer3_attention_weight_max": 96.47589111328125,
      "activations/layer3_attention_weight_min": -103.16380310058594,
      "activations/layer4_attention_weight_max": 104.55494689941406,
      "activations/layer4_attention_weight_min": -108.7980728149414,
      "activations/layer5_attention_weight_max": 50.108516693115234,
      "activations/layer5_attention_weight_min": -59.361385345458984,
      "activations/layer6_attention_weight_max": 47.66147232055664,
      "activations/layer6_attention_weight_min": -50.585819244384766,
      "activations/layer7_attention_weight_max": 96.07288360595703,
      "activations/layer7_attention_weight_min": -109.12220001220703,
      "activations/layer8_attention_weight_max": 45.08620071411133,
      "activations/layer8_attention_weight_min": -49.47473907470703,
      "activations/layer9_attention_weight_max": 34.47875213623047,
      "activations/layer9_attention_weight_min": -38.34156036376953,
      "epoch": 19.76,
      "learning_rate": 2.2791666666666664e-05,
      "loss": 2.7356,
      "step": 340100
    },
    {
      "activations/layer0_attention_weight_max": 17.84649085998535,
      "activations/layer0_attention_weight_min": -12.718451499938965,
      "activations/layer10_attention_weight_max": 39.063114166259766,
      "activations/layer10_attention_weight_min": -35.35712432861328,
      "activations/layer11_attention_weight_max": 37.74184036254883,
      "activations/layer11_attention_weight_min": -33.911048889160156,
      "activations/layer12_attention_weight_max": 31.734115600585938,
      "activations/layer12_attention_weight_min": -27.723243713378906,
      "activations/layer13_attention_weight_max": 45.70869445800781,
      "activations/layer13_attention_weight_min": -39.211700439453125,
      "activations/layer14_attention_weight_max": 48.43973922729492,
      "activations/layer14_attention_weight_min": -40.57789611816406,
      "activations/layer15_attention_weight_max": 43.923118591308594,
      "activations/layer15_attention_weight_min": -36.84450149536133,
      "activations/layer16_attention_weight_max": 33.51095962524414,
      "activations/layer16_attention_weight_min": -28.759878158569336,
      "activations/layer17_attention_weight_max": 30.99650001525879,
      "activations/layer17_attention_weight_min": -26.1825008392334,
      "activations/layer18_attention_weight_max": 30.663602828979492,
      "activations/layer18_attention_weight_min": -27.961238861083984,
      "activations/layer19_attention_weight_max": 32.61435317993164,
      "activations/layer19_attention_weight_min": -30.491565704345703,
      "activations/layer1_attention_weight_max": 15.143778800964355,
      "activations/layer1_attention_weight_min": -13.264839172363281,
      "activations/layer20_attention_weight_max": 28.48196029663086,
      "activations/layer20_attention_weight_min": -24.66081428527832,
      "activations/layer21_attention_weight_max": 30.650115966796875,
      "activations/layer21_attention_weight_min": -24.83254051208496,
      "activations/layer22_attention_weight_max": 37.69565963745117,
      "activations/layer22_attention_weight_min": -30.852306365966797,
      "activations/layer23_attention_weight_max": 29.461124420166016,
      "activations/layer23_attention_weight_min": -25.510696411132812,
      "activations/layer2_attention_weight_max": 32.53767013549805,
      "activations/layer2_attention_weight_min": -30.572769165039062,
      "activations/layer3_attention_weight_max": 101.63404083251953,
      "activations/layer3_attention_weight_min": -100.93075561523438,
      "activations/layer4_attention_weight_max": 111.56377410888672,
      "activations/layer4_attention_weight_min": -107.53752899169922,
      "activations/layer5_attention_weight_max": 50.6068115234375,
      "activations/layer5_attention_weight_min": -58.85708236694336,
      "activations/layer6_attention_weight_max": 43.09461212158203,
      "activations/layer6_attention_weight_min": -46.19032669067383,
      "activations/layer7_attention_weight_max": 101.21623229980469,
      "activations/layer7_attention_weight_min": -94.95069122314453,
      "activations/layer8_attention_weight_max": 48.52145767211914,
      "activations/layer8_attention_weight_min": -46.106597900390625,
      "activations/layer9_attention_weight_max": 33.818153381347656,
      "activations/layer9_attention_weight_min": -35.14267349243164,
      "epoch": 19.76,
      "learning_rate": 2.2772727272727272e-05,
      "loss": 2.7318,
      "step": 340150
    },
    {
      "activations/layer0_attention_weight_max": 16.301708221435547,
      "activations/layer0_attention_weight_min": -12.692519187927246,
      "activations/layer10_attention_weight_max": 34.56050491333008,
      "activations/layer10_attention_weight_min": -36.0516242980957,
      "activations/layer11_attention_weight_max": 36.75421142578125,
      "activations/layer11_attention_weight_min": -36.74628448486328,
      "activations/layer12_attention_weight_max": 28.946378707885742,
      "activations/layer12_attention_weight_min": -28.10785675048828,
      "activations/layer13_attention_weight_max": 41.4649772644043,
      "activations/layer13_attention_weight_min": -36.91184997558594,
      "activations/layer14_attention_weight_max": 44.08888244628906,
      "activations/layer14_attention_weight_min": -38.687686920166016,
      "activations/layer15_attention_weight_max": 39.638301849365234,
      "activations/layer15_attention_weight_min": -38.116477966308594,
      "activations/layer16_attention_weight_max": 29.94137954711914,
      "activations/layer16_attention_weight_min": -28.498577117919922,
      "activations/layer17_attention_weight_max": 30.632110595703125,
      "activations/layer17_attention_weight_min": -26.42012596130371,
      "activations/layer18_attention_weight_max": 35.6619873046875,
      "activations/layer18_attention_weight_min": -27.42645263671875,
      "activations/layer19_attention_weight_max": 38.40800094604492,
      "activations/layer19_attention_weight_min": -30.92661476135254,
      "activations/layer1_attention_weight_max": 16.522581100463867,
      "activations/layer1_attention_weight_min": -15.143889427185059,
      "activations/layer20_attention_weight_max": 30.761144638061523,
      "activations/layer20_attention_weight_min": -24.677621841430664,
      "activations/layer21_attention_weight_max": 29.270713806152344,
      "activations/layer21_attention_weight_min": -23.96875,
      "activations/layer22_attention_weight_max": 42.42698669433594,
      "activations/layer22_attention_weight_min": -31.26724624633789,
      "activations/layer23_attention_weight_max": 33.48252487182617,
      "activations/layer23_attention_weight_min": -26.612377166748047,
      "activations/layer2_attention_weight_max": 30.269933700561523,
      "activations/layer2_attention_weight_min": -29.588932037353516,
      "activations/layer3_attention_weight_max": 99.76002502441406,
      "activations/layer3_attention_weight_min": -97.5561294555664,
      "activations/layer4_attention_weight_max": 107.88761138916016,
      "activations/layer4_attention_weight_min": -106.7582015991211,
      "activations/layer5_attention_weight_max": 49.74119186401367,
      "activations/layer5_attention_weight_min": -61.76469421386719,
      "activations/layer6_attention_weight_max": 44.28559112548828,
      "activations/layer6_attention_weight_min": -49.05051040649414,
      "activations/layer7_attention_weight_max": 95.4805679321289,
      "activations/layer7_attention_weight_min": -102.10733032226562,
      "activations/layer8_attention_weight_max": 43.53253173828125,
      "activations/layer8_attention_weight_min": -45.15264129638672,
      "activations/layer9_attention_weight_max": 33.097869873046875,
      "activations/layer9_attention_weight_min": -35.94602966308594,
      "epoch": 19.77,
      "learning_rate": 2.2754166666666663e-05,
      "loss": 2.7099,
      "step": 340200
    },
    {
      "activations/layer0_attention_weight_max": 15.911794662475586,
      "activations/layer0_attention_weight_min": -12.465036392211914,
      "activations/layer10_attention_weight_max": 40.50431823730469,
      "activations/layer10_attention_weight_min": -36.15066146850586,
      "activations/layer11_attention_weight_max": 39.71478271484375,
      "activations/layer11_attention_weight_min": -40.62159729003906,
      "activations/layer12_attention_weight_max": 27.271238327026367,
      "activations/layer12_attention_weight_min": -25.53068733215332,
      "activations/layer13_attention_weight_max": 39.888671875,
      "activations/layer13_attention_weight_min": -35.59538650512695,
      "activations/layer14_attention_weight_max": 45.29035949707031,
      "activations/layer14_attention_weight_min": -40.000267028808594,
      "activations/layer15_attention_weight_max": 45.47687530517578,
      "activations/layer15_attention_weight_min": -36.52201461791992,
      "activations/layer16_attention_weight_max": 32.63319778442383,
      "activations/layer16_attention_weight_min": -28.01432991027832,
      "activations/layer17_attention_weight_max": 31.047189712524414,
      "activations/layer17_attention_weight_min": -26.139190673828125,
      "activations/layer18_attention_weight_max": 32.765838623046875,
      "activations/layer18_attention_weight_min": -27.110332489013672,
      "activations/layer19_attention_weight_max": 38.997432708740234,
      "activations/layer19_attention_weight_min": -32.24917221069336,
      "activations/layer1_attention_weight_max": 17.36429786682129,
      "activations/layer1_attention_weight_min": -14.951621055603027,
      "activations/layer20_attention_weight_max": 33.08341979980469,
      "activations/layer20_attention_weight_min": -27.83153533935547,
      "activations/layer21_attention_weight_max": 29.150707244873047,
      "activations/layer21_attention_weight_min": -26.60935401916504,
      "activations/layer22_attention_weight_max": 43.576236724853516,
      "activations/layer22_attention_weight_min": -34.75551986694336,
      "activations/layer23_attention_weight_max": 33.7574462890625,
      "activations/layer23_attention_weight_min": -28.44021224975586,
      "activations/layer2_attention_weight_max": 31.78851318359375,
      "activations/layer2_attention_weight_min": -32.75641632080078,
      "activations/layer3_attention_weight_max": 100.88150024414062,
      "activations/layer3_attention_weight_min": -103.48928833007812,
      "activations/layer4_attention_weight_max": 103.66961669921875,
      "activations/layer4_attention_weight_min": -110.854736328125,
      "activations/layer5_attention_weight_max": 51.09016036987305,
      "activations/layer5_attention_weight_min": -58.5682373046875,
      "activations/layer6_attention_weight_max": 42.837364196777344,
      "activations/layer6_attention_weight_min": -47.1067008972168,
      "activations/layer7_attention_weight_max": 101.3022232055664,
      "activations/layer7_attention_weight_min": -92.94309997558594,
      "activations/layer8_attention_weight_max": 47.61592483520508,
      "activations/layer8_attention_weight_min": -42.40084457397461,
      "activations/layer9_attention_weight_max": 31.7623291015625,
      "activations/layer9_attention_weight_min": -35.03648376464844,
      "epoch": 19.77,
      "learning_rate": 2.273522727272727e-05,
      "loss": 2.7115,
      "step": 340250
    },
    {
      "activations/layer0_attention_weight_max": 15.784048080444336,
      "activations/layer0_attention_weight_min": -12.472492218017578,
      "activations/layer10_attention_weight_max": 31.850154876708984,
      "activations/layer10_attention_weight_min": -33.96976089477539,
      "activations/layer11_attention_weight_max": 32.41797637939453,
      "activations/layer11_attention_weight_min": -33.80634307861328,
      "activations/layer12_attention_weight_max": 25.858652114868164,
      "activations/layer12_attention_weight_min": -26.184236526489258,
      "activations/layer13_attention_weight_max": 41.54780578613281,
      "activations/layer13_attention_weight_min": -35.37028121948242,
      "activations/layer14_attention_weight_max": 42.87459945678711,
      "activations/layer14_attention_weight_min": -38.53092956542969,
      "activations/layer15_attention_weight_max": 40.85374450683594,
      "activations/layer15_attention_weight_min": -37.64344024658203,
      "activations/layer16_attention_weight_max": 27.654006958007812,
      "activations/layer16_attention_weight_min": -28.740617752075195,
      "activations/layer17_attention_weight_max": 30.63770866394043,
      "activations/layer17_attention_weight_min": -26.873117446899414,
      "activations/layer18_attention_weight_max": 30.160879135131836,
      "activations/layer18_attention_weight_min": -28.04867172241211,
      "activations/layer19_attention_weight_max": 32.67110824584961,
      "activations/layer19_attention_weight_min": -32.2386360168457,
      "activations/layer1_attention_weight_max": 14.93166732788086,
      "activations/layer1_attention_weight_min": -14.39993953704834,
      "activations/layer20_attention_weight_max": 26.970767974853516,
      "activations/layer20_attention_weight_min": -27.665386199951172,
      "activations/layer21_attention_weight_max": 26.403457641601562,
      "activations/layer21_attention_weight_min": -25.875768661499023,
      "activations/layer22_attention_weight_max": 36.56260681152344,
      "activations/layer22_attention_weight_min": -31.034347534179688,
      "activations/layer23_attention_weight_max": 28.62824821472168,
      "activations/layer23_attention_weight_min": -26.781543731689453,
      "activations/layer2_attention_weight_max": 32.156166076660156,
      "activations/layer2_attention_weight_min": -30.99464225769043,
      "activations/layer3_attention_weight_max": 98.0564193725586,
      "activations/layer3_attention_weight_min": -99.67410278320312,
      "activations/layer4_attention_weight_max": 103.9660415649414,
      "activations/layer4_attention_weight_min": -106.6808853149414,
      "activations/layer5_attention_weight_max": 49.626991271972656,
      "activations/layer5_attention_weight_min": -60.730690002441406,
      "activations/layer6_attention_weight_max": 43.95735168457031,
      "activations/layer6_attention_weight_min": -46.699398040771484,
      "activations/layer7_attention_weight_max": 88.76715087890625,
      "activations/layer7_attention_weight_min": -96.34129333496094,
      "activations/layer8_attention_weight_max": 41.40275955200195,
      "activations/layer8_attention_weight_min": -43.20634841918945,
      "activations/layer9_attention_weight_max": 31.331098556518555,
      "activations/layer9_attention_weight_min": -34.477935791015625,
      "epoch": 19.77,
      "learning_rate": 2.2716287878787876e-05,
      "loss": 2.7243,
      "step": 340300
    },
    {
      "activations/layer0_attention_weight_max": 15.89795207977295,
      "activations/layer0_attention_weight_min": -13.075194358825684,
      "activations/layer10_attention_weight_max": 37.27681350708008,
      "activations/layer10_attention_weight_min": -36.69172286987305,
      "activations/layer11_attention_weight_max": 36.198272705078125,
      "activations/layer11_attention_weight_min": -36.66741180419922,
      "activations/layer12_attention_weight_max": 28.302879333496094,
      "activations/layer12_attention_weight_min": -28.24671173095703,
      "activations/layer13_attention_weight_max": 40.71620559692383,
      "activations/layer13_attention_weight_min": -35.72946548461914,
      "activations/layer14_attention_weight_max": 43.53362274169922,
      "activations/layer14_attention_weight_min": -37.20784378051758,
      "activations/layer15_attention_weight_max": 41.17912673950195,
      "activations/layer15_attention_weight_min": -35.68668746948242,
      "activations/layer16_attention_weight_max": 29.354629516601562,
      "activations/layer16_attention_weight_min": -28.07847785949707,
      "activations/layer17_attention_weight_max": 30.28481674194336,
      "activations/layer17_attention_weight_min": -26.647836685180664,
      "activations/layer18_attention_weight_max": 32.70344161987305,
      "activations/layer18_attention_weight_min": -27.32758140563965,
      "activations/layer19_attention_weight_max": 34.150020599365234,
      "activations/layer19_attention_weight_min": -31.792943954467773,
      "activations/layer1_attention_weight_max": 15.714439392089844,
      "activations/layer1_attention_weight_min": -14.588274955749512,
      "activations/layer20_attention_weight_max": 30.207805633544922,
      "activations/layer20_attention_weight_min": -27.93810272216797,
      "activations/layer21_attention_weight_max": 30.319000244140625,
      "activations/layer21_attention_weight_min": -27.378759384155273,
      "activations/layer22_attention_weight_max": 43.08000183105469,
      "activations/layer22_attention_weight_min": -35.29425048828125,
      "activations/layer23_attention_weight_max": 33.686668395996094,
      "activations/layer23_attention_weight_min": -28.07357406616211,
      "activations/layer2_attention_weight_max": 31.063762664794922,
      "activations/layer2_attention_weight_min": -30.50958824157715,
      "activations/layer3_attention_weight_max": 98.8567123413086,
      "activations/layer3_attention_weight_min": -100.67772674560547,
      "activations/layer4_attention_weight_max": 102.9922866821289,
      "activations/layer4_attention_weight_min": -108.09720611572266,
      "activations/layer5_attention_weight_max": 48.26371383666992,
      "activations/layer5_attention_weight_min": -61.766090393066406,
      "activations/layer6_attention_weight_max": 45.648197174072266,
      "activations/layer6_attention_weight_min": -47.07664489746094,
      "activations/layer7_attention_weight_max": 98.93312072753906,
      "activations/layer7_attention_weight_min": -98.21881103515625,
      "activations/layer8_attention_weight_max": 44.2902717590332,
      "activations/layer8_attention_weight_min": -47.3088264465332,
      "activations/layer9_attention_weight_max": 35.84757995605469,
      "activations/layer9_attention_weight_min": -37.05929183959961,
      "epoch": 19.78,
      "learning_rate": 2.2697348484848484e-05,
      "loss": 2.7393,
      "step": 340350
    },
    {
      "activations/layer0_attention_weight_max": 18.0014705657959,
      "activations/layer0_attention_weight_min": -14.891170501708984,
      "activations/layer10_attention_weight_max": 42.14936065673828,
      "activations/layer10_attention_weight_min": -41.1854362487793,
      "activations/layer11_attention_weight_max": 43.59089279174805,
      "activations/layer11_attention_weight_min": -39.971641540527344,
      "activations/layer12_attention_weight_max": 29.033239364624023,
      "activations/layer12_attention_weight_min": -28.203317642211914,
      "activations/layer13_attention_weight_max": 42.12403869628906,
      "activations/layer13_attention_weight_min": -39.22272872924805,
      "activations/layer14_attention_weight_max": 44.508384704589844,
      "activations/layer14_attention_weight_min": -41.3530158996582,
      "activations/layer15_attention_weight_max": 40.42975997924805,
      "activations/layer15_attention_weight_min": -37.10470962524414,
      "activations/layer16_attention_weight_max": 29.339210510253906,
      "activations/layer16_attention_weight_min": -28.85847282409668,
      "activations/layer17_attention_weight_max": 31.33713722229004,
      "activations/layer17_attention_weight_min": -24.963207244873047,
      "activations/layer18_attention_weight_max": 36.23766326904297,
      "activations/layer18_attention_weight_min": -25.956096649169922,
      "activations/layer19_attention_weight_max": 37.53182601928711,
      "activations/layer19_attention_weight_min": -30.173053741455078,
      "activations/layer1_attention_weight_max": 15.43581771850586,
      "activations/layer1_attention_weight_min": -16.05720329284668,
      "activations/layer20_attention_weight_max": 31.921518325805664,
      "activations/layer20_attention_weight_min": -25.806201934814453,
      "activations/layer21_attention_weight_max": 31.79543685913086,
      "activations/layer21_attention_weight_min": -24.409317016601562,
      "activations/layer22_attention_weight_max": 51.26033401489258,
      "activations/layer22_attention_weight_min": -32.63252639770508,
      "activations/layer23_attention_weight_max": 34.55446243286133,
      "activations/layer23_attention_weight_min": -25.243141174316406,
      "activations/layer2_attention_weight_max": 31.91507339477539,
      "activations/layer2_attention_weight_min": -30.23188591003418,
      "activations/layer3_attention_weight_max": 102.23575592041016,
      "activations/layer3_attention_weight_min": -100.55805969238281,
      "activations/layer4_attention_weight_max": 110.27862548828125,
      "activations/layer4_attention_weight_min": -108.5564956665039,
      "activations/layer5_attention_weight_max": 51.9229736328125,
      "activations/layer5_attention_weight_min": -62.71562957763672,
      "activations/layer6_attention_weight_max": 45.622764587402344,
      "activations/layer6_attention_weight_min": -50.17833709716797,
      "activations/layer7_attention_weight_max": 99.6334457397461,
      "activations/layer7_attention_weight_min": -103.45584869384766,
      "activations/layer8_attention_weight_max": 47.951263427734375,
      "activations/layer8_attention_weight_min": -46.51303482055664,
      "activations/layer9_attention_weight_max": 37.62925338745117,
      "activations/layer9_attention_weight_min": -39.10801696777344,
      "epoch": 19.78,
      "learning_rate": 2.2678409090909086e-05,
      "loss": 2.7156,
      "step": 340400
    },
    {
      "activations/layer0_attention_weight_max": 17.501338958740234,
      "activations/layer0_attention_weight_min": -13.0564603805542,
      "activations/layer10_attention_weight_max": 36.90922164916992,
      "activations/layer10_attention_weight_min": -35.7079963684082,
      "activations/layer11_attention_weight_max": 34.368896484375,
      "activations/layer11_attention_weight_min": -33.87065505981445,
      "activations/layer12_attention_weight_max": 29.41819190979004,
      "activations/layer12_attention_weight_min": -28.105810165405273,
      "activations/layer13_attention_weight_max": 42.96450424194336,
      "activations/layer13_attention_weight_min": -37.14158248901367,
      "activations/layer14_attention_weight_max": 47.438148498535156,
      "activations/layer14_attention_weight_min": -40.353172302246094,
      "activations/layer15_attention_weight_max": 43.22708511352539,
      "activations/layer15_attention_weight_min": -38.556636810302734,
      "activations/layer16_attention_weight_max": 29.83133888244629,
      "activations/layer16_attention_weight_min": -27.3150577545166,
      "activations/layer17_attention_weight_max": 30.46057891845703,
      "activations/layer17_attention_weight_min": -25.85137367248535,
      "activations/layer18_attention_weight_max": 30.007144927978516,
      "activations/layer18_attention_weight_min": -25.281368255615234,
      "activations/layer19_attention_weight_max": 33.322669982910156,
      "activations/layer19_attention_weight_min": -30.60955810546875,
      "activations/layer1_attention_weight_max": 15.76215934753418,
      "activations/layer1_attention_weight_min": -14.462891578674316,
      "activations/layer20_attention_weight_max": 29.917205810546875,
      "activations/layer20_attention_weight_min": -24.995210647583008,
      "activations/layer21_attention_weight_max": 29.839275360107422,
      "activations/layer21_attention_weight_min": -23.917736053466797,
      "activations/layer22_attention_weight_max": 48.1470832824707,
      "activations/layer22_attention_weight_min": -30.42821502685547,
      "activations/layer23_attention_weight_max": 34.040557861328125,
      "activations/layer23_attention_weight_min": -24.727354049682617,
      "activations/layer2_attention_weight_max": 29.72806167602539,
      "activations/layer2_attention_weight_min": -30.73226547241211,
      "activations/layer3_attention_weight_max": 94.53094482421875,
      "activations/layer3_attention_weight_min": -95.83200073242188,
      "activations/layer4_attention_weight_max": 101.58043670654297,
      "activations/layer4_attention_weight_min": -105.179443359375,
      "activations/layer5_attention_weight_max": 48.476417541503906,
      "activations/layer5_attention_weight_min": -62.54132843017578,
      "activations/layer6_attention_weight_max": 45.72507858276367,
      "activations/layer6_attention_weight_min": -45.68240737915039,
      "activations/layer7_attention_weight_max": 94.57703399658203,
      "activations/layer7_attention_weight_min": -95.71232604980469,
      "activations/layer8_attention_weight_max": 45.05887222290039,
      "activations/layer8_attention_weight_min": -45.284690856933594,
      "activations/layer9_attention_weight_max": 32.49433517456055,
      "activations/layer9_attention_weight_min": -34.76402282714844,
      "epoch": 19.78,
      "learning_rate": 2.2659469696969694e-05,
      "loss": 2.7244,
      "step": 340450
    },
    {
      "activations/layer0_attention_weight_max": 16.232412338256836,
      "activations/layer0_attention_weight_min": -12.87104320526123,
      "activations/layer10_attention_weight_max": 34.9382438659668,
      "activations/layer10_attention_weight_min": -37.83843231201172,
      "activations/layer11_attention_weight_max": 33.91899871826172,
      "activations/layer11_attention_weight_min": -35.522918701171875,
      "activations/layer12_attention_weight_max": 27.7698917388916,
      "activations/layer12_attention_weight_min": -30.10847282409668,
      "activations/layer13_attention_weight_max": 42.24597930908203,
      "activations/layer13_attention_weight_min": -35.71328353881836,
      "activations/layer14_attention_weight_max": 43.21240997314453,
      "activations/layer14_attention_weight_min": -37.675018310546875,
      "activations/layer15_attention_weight_max": 41.95875930786133,
      "activations/layer15_attention_weight_min": -36.23277282714844,
      "activations/layer16_attention_weight_max": 32.91091537475586,
      "activations/layer16_attention_weight_min": -29.910104751586914,
      "activations/layer17_attention_weight_max": 33.57315444946289,
      "activations/layer17_attention_weight_min": -26.428808212280273,
      "activations/layer18_attention_weight_max": 31.46827507019043,
      "activations/layer18_attention_weight_min": -24.71143341064453,
      "activations/layer19_attention_weight_max": 32.005001068115234,
      "activations/layer19_attention_weight_min": -30.916784286499023,
      "activations/layer1_attention_weight_max": 14.613666534423828,
      "activations/layer1_attention_weight_min": -13.843152046203613,
      "activations/layer20_attention_weight_max": 28.990406036376953,
      "activations/layer20_attention_weight_min": -25.19249153137207,
      "activations/layer21_attention_weight_max": 29.945180892944336,
      "activations/layer21_attention_weight_min": -24.926895141601562,
      "activations/layer22_attention_weight_max": 41.66567611694336,
      "activations/layer22_attention_weight_min": -30.012929916381836,
      "activations/layer23_attention_weight_max": 31.17706298828125,
      "activations/layer23_attention_weight_min": -23.248348236083984,
      "activations/layer2_attention_weight_max": 30.974592208862305,
      "activations/layer2_attention_weight_min": -30.513370513916016,
      "activations/layer3_attention_weight_max": 98.20410919189453,
      "activations/layer3_attention_weight_min": -98.68773651123047,
      "activations/layer4_attention_weight_max": 105.42485809326172,
      "activations/layer4_attention_weight_min": -103.83708953857422,
      "activations/layer5_attention_weight_max": 51.080623626708984,
      "activations/layer5_attention_weight_min": -61.931602478027344,
      "activations/layer6_attention_weight_max": 44.36177444458008,
      "activations/layer6_attention_weight_min": -45.8151741027832,
      "activations/layer7_attention_weight_max": 96.91484069824219,
      "activations/layer7_attention_weight_min": -93.48417663574219,
      "activations/layer8_attention_weight_max": 42.12397384643555,
      "activations/layer8_attention_weight_min": -45.33551788330078,
      "activations/layer9_attention_weight_max": 32.710933685302734,
      "activations/layer9_attention_weight_min": -33.93787384033203,
      "epoch": 19.78,
      "learning_rate": 2.26405303030303e-05,
      "loss": 2.727,
      "step": 340500
    },
    {
      "activations/layer0_attention_weight_max": 15.861518859863281,
      "activations/layer0_attention_weight_min": -12.4409818649292,
      "activations/layer10_attention_weight_max": 36.2250862121582,
      "activations/layer10_attention_weight_min": -37.09479522705078,
      "activations/layer11_attention_weight_max": 35.09939193725586,
      "activations/layer11_attention_weight_min": -36.54212188720703,
      "activations/layer12_attention_weight_max": 30.963008880615234,
      "activations/layer12_attention_weight_min": -29.02652931213379,
      "activations/layer13_attention_weight_max": 46.80928421020508,
      "activations/layer13_attention_weight_min": -41.01523971557617,
      "activations/layer14_attention_weight_max": 45.61161422729492,
      "activations/layer14_attention_weight_min": -43.122222900390625,
      "activations/layer15_attention_weight_max": 48.20893859863281,
      "activations/layer15_attention_weight_min": -40.37346649169922,
      "activations/layer16_attention_weight_max": 31.2109375,
      "activations/layer16_attention_weight_min": -31.575611114501953,
      "activations/layer17_attention_weight_max": 31.08592414855957,
      "activations/layer17_attention_weight_min": -31.801809310913086,
      "activations/layer18_attention_weight_max": 36.466331481933594,
      "activations/layer18_attention_weight_min": -29.489299774169922,
      "activations/layer19_attention_weight_max": 35.754024505615234,
      "activations/layer19_attention_weight_min": -36.680660247802734,
      "activations/layer1_attention_weight_max": 16.28157615661621,
      "activations/layer1_attention_weight_min": -15.927510261535645,
      "activations/layer20_attention_weight_max": 31.383975982666016,
      "activations/layer20_attention_weight_min": -27.21999740600586,
      "activations/layer21_attention_weight_max": 30.745981216430664,
      "activations/layer21_attention_weight_min": -27.93747329711914,
      "activations/layer22_attention_weight_max": 44.40824508666992,
      "activations/layer22_attention_weight_min": -34.67646789550781,
      "activations/layer23_attention_weight_max": 35.445960998535156,
      "activations/layer23_attention_weight_min": -27.276283264160156,
      "activations/layer2_attention_weight_max": 32.78810119628906,
      "activations/layer2_attention_weight_min": -31.562307357788086,
      "activations/layer3_attention_weight_max": 97.31239318847656,
      "activations/layer3_attention_weight_min": -100.12733459472656,
      "activations/layer4_attention_weight_max": 102.57347869873047,
      "activations/layer4_attention_weight_min": -107.69982147216797,
      "activations/layer5_attention_weight_max": 50.056983947753906,
      "activations/layer5_attention_weight_min": -60.50339889526367,
      "activations/layer6_attention_weight_max": 44.832969665527344,
      "activations/layer6_attention_weight_min": -49.802852630615234,
      "activations/layer7_attention_weight_max": 95.94110107421875,
      "activations/layer7_attention_weight_min": -101.65025329589844,
      "activations/layer8_attention_weight_max": 44.267906188964844,
      "activations/layer8_attention_weight_min": -46.752220153808594,
      "activations/layer9_attention_weight_max": 33.58742141723633,
      "activations/layer9_attention_weight_min": -36.922515869140625,
      "epoch": 19.79,
      "learning_rate": 2.2621590909090907e-05,
      "loss": 2.711,
      "step": 340550
    },
    {
      "activations/layer0_attention_weight_max": 16.40732765197754,
      "activations/layer0_attention_weight_min": -12.353538513183594,
      "activations/layer10_attention_weight_max": 34.492088317871094,
      "activations/layer10_attention_weight_min": -34.5767822265625,
      "activations/layer11_attention_weight_max": 34.438228607177734,
      "activations/layer11_attention_weight_min": -34.051292419433594,
      "activations/layer12_attention_weight_max": 27.819746017456055,
      "activations/layer12_attention_weight_min": -26.793739318847656,
      "activations/layer13_attention_weight_max": 40.75703430175781,
      "activations/layer13_attention_weight_min": -37.53272247314453,
      "activations/layer14_attention_weight_max": 41.92854690551758,
      "activations/layer14_attention_weight_min": -40.7661018371582,
      "activations/layer15_attention_weight_max": 41.207115173339844,
      "activations/layer15_attention_weight_min": -36.8977165222168,
      "activations/layer16_attention_weight_max": 29.611093521118164,
      "activations/layer16_attention_weight_min": -29.16099739074707,
      "activations/layer17_attention_weight_max": 31.300607681274414,
      "activations/layer17_attention_weight_min": -27.33176612854004,
      "activations/layer18_attention_weight_max": 36.6209831237793,
      "activations/layer18_attention_weight_min": -25.9206485748291,
      "activations/layer19_attention_weight_max": 38.2718620300293,
      "activations/layer19_attention_weight_min": -32.337642669677734,
      "activations/layer1_attention_weight_max": 16.047142028808594,
      "activations/layer1_attention_weight_min": -15.992255210876465,
      "activations/layer20_attention_weight_max": 30.317699432373047,
      "activations/layer20_attention_weight_min": -26.054153442382812,
      "activations/layer21_attention_weight_max": 30.546884536743164,
      "activations/layer21_attention_weight_min": -26.239744186401367,
      "activations/layer22_attention_weight_max": 47.73213577270508,
      "activations/layer22_attention_weight_min": -33.78177261352539,
      "activations/layer23_attention_weight_max": 34.2164192199707,
      "activations/layer23_attention_weight_min": -27.41694450378418,
      "activations/layer2_attention_weight_max": 29.959564208984375,
      "activations/layer2_attention_weight_min": -28.228179931640625,
      "activations/layer3_attention_weight_max": 92.66950225830078,
      "activations/layer3_attention_weight_min": -89.77386474609375,
      "activations/layer4_attention_weight_max": 100.99869537353516,
      "activations/layer4_attention_weight_min": -100.30056762695312,
      "activations/layer5_attention_weight_max": 47.01929473876953,
      "activations/layer5_attention_weight_min": -58.653289794921875,
      "activations/layer6_attention_weight_max": 44.32852554321289,
      "activations/layer6_attention_weight_min": -45.38438034057617,
      "activations/layer7_attention_weight_max": 94.7124252319336,
      "activations/layer7_attention_weight_min": -94.09599304199219,
      "activations/layer8_attention_weight_max": 42.797828674316406,
      "activations/layer8_attention_weight_min": -43.472896575927734,
      "activations/layer9_attention_weight_max": 31.804269790649414,
      "activations/layer9_attention_weight_min": -34.03432846069336,
      "epoch": 19.79,
      "learning_rate": 2.2602651515151512e-05,
      "loss": 2.7036,
      "step": 340600
    },
    {
      "activations/layer0_attention_weight_max": 16.189098358154297,
      "activations/layer0_attention_weight_min": -13.560805320739746,
      "activations/layer10_attention_weight_max": 40.45299530029297,
      "activations/layer10_attention_weight_min": -38.179901123046875,
      "activations/layer11_attention_weight_max": 37.744285583496094,
      "activations/layer11_attention_weight_min": -40.31523132324219,
      "activations/layer12_attention_weight_max": 31.2778263092041,
      "activations/layer12_attention_weight_min": -28.587617874145508,
      "activations/layer13_attention_weight_max": 42.80883026123047,
      "activations/layer13_attention_weight_min": -38.99799728393555,
      "activations/layer14_attention_weight_max": 52.283668518066406,
      "activations/layer14_attention_weight_min": -39.727291107177734,
      "activations/layer15_attention_weight_max": 48.36716842651367,
      "activations/layer15_attention_weight_min": -41.42127990722656,
      "activations/layer16_attention_weight_max": 31.36712646484375,
      "activations/layer16_attention_weight_min": -29.423538208007812,
      "activations/layer17_attention_weight_max": 29.880950927734375,
      "activations/layer17_attention_weight_min": -27.002817153930664,
      "activations/layer18_attention_weight_max": 32.313865661621094,
      "activations/layer18_attention_weight_min": -26.41454315185547,
      "activations/layer19_attention_weight_max": 33.3795280456543,
      "activations/layer19_attention_weight_min": -33.985286712646484,
      "activations/layer1_attention_weight_max": 17.267147064208984,
      "activations/layer1_attention_weight_min": -15.968839645385742,
      "activations/layer20_attention_weight_max": 29.262248992919922,
      "activations/layer20_attention_weight_min": -25.745159149169922,
      "activations/layer21_attention_weight_max": 29.299108505249023,
      "activations/layer21_attention_weight_min": -24.751806259155273,
      "activations/layer22_attention_weight_max": 49.21013641357422,
      "activations/layer22_attention_weight_min": -33.230533599853516,
      "activations/layer23_attention_weight_max": 32.6832160949707,
      "activations/layer23_attention_weight_min": -26.55698013305664,
      "activations/layer2_attention_weight_max": 32.35673522949219,
      "activations/layer2_attention_weight_min": -31.210933685302734,
      "activations/layer3_attention_weight_max": 95.45777130126953,
      "activations/layer3_attention_weight_min": -98.18205261230469,
      "activations/layer4_attention_weight_max": 103.3093490600586,
      "activations/layer4_attention_weight_min": -106.42394256591797,
      "activations/layer5_attention_weight_max": 50.544429779052734,
      "activations/layer5_attention_weight_min": -59.37847137451172,
      "activations/layer6_attention_weight_max": 46.76032257080078,
      "activations/layer6_attention_weight_min": -48.42961120605469,
      "activations/layer7_attention_weight_max": 101.88420104980469,
      "activations/layer7_attention_weight_min": -104.94094848632812,
      "activations/layer8_attention_weight_max": 45.59654235839844,
      "activations/layer8_attention_weight_min": -49.864383697509766,
      "activations/layer9_attention_weight_max": 35.480831146240234,
      "activations/layer9_attention_weight_min": -39.213096618652344,
      "epoch": 19.79,
      "learning_rate": 2.258371212121212e-05,
      "loss": 2.7258,
      "step": 340650
    },
    {
      "activations/layer0_attention_weight_max": 16.178756713867188,
      "activations/layer0_attention_weight_min": -12.787202835083008,
      "activations/layer10_attention_weight_max": 32.063026428222656,
      "activations/layer10_attention_weight_min": -35.399349212646484,
      "activations/layer11_attention_weight_max": 32.86907196044922,
      "activations/layer11_attention_weight_min": -33.73088836669922,
      "activations/layer12_attention_weight_max": 25.999820709228516,
      "activations/layer12_attention_weight_min": -27.061176300048828,
      "activations/layer13_attention_weight_max": 37.406578063964844,
      "activations/layer13_attention_weight_min": -36.062232971191406,
      "activations/layer14_attention_weight_max": 41.678009033203125,
      "activations/layer14_attention_weight_min": -39.49808120727539,
      "activations/layer15_attention_weight_max": 37.822418212890625,
      "activations/layer15_attention_weight_min": -35.937294006347656,
      "activations/layer16_attention_weight_max": 28.408954620361328,
      "activations/layer16_attention_weight_min": -26.96306610107422,
      "activations/layer17_attention_weight_max": 28.72934913635254,
      "activations/layer17_attention_weight_min": -24.280431747436523,
      "activations/layer18_attention_weight_max": 31.52601432800293,
      "activations/layer18_attention_weight_min": -24.777454376220703,
      "activations/layer19_attention_weight_max": 36.477867126464844,
      "activations/layer19_attention_weight_min": -31.5771427154541,
      "activations/layer1_attention_weight_max": 15.414143562316895,
      "activations/layer1_attention_weight_min": -13.996469497680664,
      "activations/layer20_attention_weight_max": 30.55965232849121,
      "activations/layer20_attention_weight_min": -25.315019607543945,
      "activations/layer21_attention_weight_max": 31.50023078918457,
      "activations/layer21_attention_weight_min": -26.53102684020996,
      "activations/layer22_attention_weight_max": 43.1771125793457,
      "activations/layer22_attention_weight_min": -30.696592330932617,
      "activations/layer23_attention_weight_max": 33.0015869140625,
      "activations/layer23_attention_weight_min": -24.598297119140625,
      "activations/layer2_attention_weight_max": 31.96487808227539,
      "activations/layer2_attention_weight_min": -30.718976974487305,
      "activations/layer3_attention_weight_max": 100.20407104492188,
      "activations/layer3_attention_weight_min": -100.19586944580078,
      "activations/layer4_attention_weight_max": 107.8049087524414,
      "activations/layer4_attention_weight_min": -107.50086975097656,
      "activations/layer5_attention_weight_max": 49.587562561035156,
      "activations/layer5_attention_weight_min": -61.767913818359375,
      "activations/layer6_attention_weight_max": 43.46546173095703,
      "activations/layer6_attention_weight_min": -47.03190231323242,
      "activations/layer7_attention_weight_max": 92.38351440429688,
      "activations/layer7_attention_weight_min": -99.27061462402344,
      "activations/layer8_attention_weight_max": 41.4902458190918,
      "activations/layer8_attention_weight_min": -44.420467376708984,
      "activations/layer9_attention_weight_max": 31.2192440032959,
      "activations/layer9_attention_weight_min": -32.51607894897461,
      "epoch": 19.8,
      "learning_rate": 2.256477272727273e-05,
      "loss": 2.7219,
      "step": 340700
    },
    {
      "activations/layer0_attention_weight_max": 16.566265106201172,
      "activations/layer0_attention_weight_min": -12.776345252990723,
      "activations/layer10_attention_weight_max": 38.579986572265625,
      "activations/layer10_attention_weight_min": -38.677528381347656,
      "activations/layer11_attention_weight_max": 37.055030822753906,
      "activations/layer11_attention_weight_min": -35.09536361694336,
      "activations/layer12_attention_weight_max": 27.535926818847656,
      "activations/layer12_attention_weight_min": -29.582279205322266,
      "activations/layer13_attention_weight_max": 39.355098724365234,
      "activations/layer13_attention_weight_min": -40.37804412841797,
      "activations/layer14_attention_weight_max": 44.81113052368164,
      "activations/layer14_attention_weight_min": -46.37655258178711,
      "activations/layer15_attention_weight_max": 42.21089172363281,
      "activations/layer15_attention_weight_min": -44.686988830566406,
      "activations/layer16_attention_weight_max": 34.05169677734375,
      "activations/layer16_attention_weight_min": -27.163570404052734,
      "activations/layer17_attention_weight_max": 32.37868118286133,
      "activations/layer17_attention_weight_min": -29.51005744934082,
      "activations/layer18_attention_weight_max": 32.06172561645508,
      "activations/layer18_attention_weight_min": -27.983144760131836,
      "activations/layer19_attention_weight_max": 37.03519821166992,
      "activations/layer19_attention_weight_min": -32.735713958740234,
      "activations/layer1_attention_weight_max": 16.297054290771484,
      "activations/layer1_attention_weight_min": -15.21318244934082,
      "activations/layer20_attention_weight_max": 31.422163009643555,
      "activations/layer20_attention_weight_min": -26.900489807128906,
      "activations/layer21_attention_weight_max": 29.782840728759766,
      "activations/layer21_attention_weight_min": -24.8678035736084,
      "activations/layer22_attention_weight_max": 41.9951057434082,
      "activations/layer22_attention_weight_min": -33.20634841918945,
      "activations/layer23_attention_weight_max": 33.430381774902344,
      "activations/layer23_attention_weight_min": -24.339263916015625,
      "activations/layer2_attention_weight_max": 31.664823532104492,
      "activations/layer2_attention_weight_min": -32.026580810546875,
      "activations/layer3_attention_weight_max": 97.28685760498047,
      "activations/layer3_attention_weight_min": -98.52973175048828,
      "activations/layer4_attention_weight_max": 107.31172943115234,
      "activations/layer4_attention_weight_min": -108.90357971191406,
      "activations/layer5_attention_weight_max": 51.31893539428711,
      "activations/layer5_attention_weight_min": -63.40068054199219,
      "activations/layer6_attention_weight_max": 45.393184661865234,
      "activations/layer6_attention_weight_min": -47.8951530456543,
      "activations/layer7_attention_weight_max": 98.40299987792969,
      "activations/layer7_attention_weight_min": -101.65406036376953,
      "activations/layer8_attention_weight_max": 45.26969528198242,
      "activations/layer8_attention_weight_min": -45.29107666015625,
      "activations/layer9_attention_weight_max": 34.2048225402832,
      "activations/layer9_attention_weight_min": -36.25034713745117,
      "epoch": 19.8,
      "learning_rate": 2.254583333333333e-05,
      "loss": 2.7176,
      "step": 340750
    },
    {
      "activations/layer0_attention_weight_max": 16.63463020324707,
      "activations/layer0_attention_weight_min": -13.157133102416992,
      "activations/layer10_attention_weight_max": 37.83497619628906,
      "activations/layer10_attention_weight_min": -37.26786422729492,
      "activations/layer11_attention_weight_max": 35.00444412231445,
      "activations/layer11_attention_weight_min": -35.310585021972656,
      "activations/layer12_attention_weight_max": 27.8129940032959,
      "activations/layer12_attention_weight_min": -28.232908248901367,
      "activations/layer13_attention_weight_max": 42.1533203125,
      "activations/layer13_attention_weight_min": -37.606605529785156,
      "activations/layer14_attention_weight_max": 45.00178146362305,
      "activations/layer14_attention_weight_min": -38.49543762207031,
      "activations/layer15_attention_weight_max": 40.7025146484375,
      "activations/layer15_attention_weight_min": -36.44307327270508,
      "activations/layer16_attention_weight_max": 28.599782943725586,
      "activations/layer16_attention_weight_min": -29.15346908569336,
      "activations/layer17_attention_weight_max": 28.021705627441406,
      "activations/layer17_attention_weight_min": -26.094884872436523,
      "activations/layer18_attention_weight_max": 31.768112182617188,
      "activations/layer18_attention_weight_min": -25.606874465942383,
      "activations/layer19_attention_weight_max": 32.91721725463867,
      "activations/layer19_attention_weight_min": -31.66431999206543,
      "activations/layer1_attention_weight_max": 15.854195594787598,
      "activations/layer1_attention_weight_min": -15.531513214111328,
      "activations/layer20_attention_weight_max": 26.653230667114258,
      "activations/layer20_attention_weight_min": -24.777847290039062,
      "activations/layer21_attention_weight_max": 26.017459869384766,
      "activations/layer21_attention_weight_min": -24.015838623046875,
      "activations/layer22_attention_weight_max": 39.59599685668945,
      "activations/layer22_attention_weight_min": -30.271745681762695,
      "activations/layer23_attention_weight_max": 32.53581237792969,
      "activations/layer23_attention_weight_min": -25.586469650268555,
      "activations/layer2_attention_weight_max": 30.420013427734375,
      "activations/layer2_attention_weight_min": -31.8526668548584,
      "activations/layer3_attention_weight_max": 94.50959014892578,
      "activations/layer3_attention_weight_min": -99.54877471923828,
      "activations/layer4_attention_weight_max": 102.72977447509766,
      "activations/layer4_attention_weight_min": -105.08821105957031,
      "activations/layer5_attention_weight_max": 46.567115783691406,
      "activations/layer5_attention_weight_min": -60.48762512207031,
      "activations/layer6_attention_weight_max": 41.634803771972656,
      "activations/layer6_attention_weight_min": -47.576576232910156,
      "activations/layer7_attention_weight_max": 94.9638442993164,
      "activations/layer7_attention_weight_min": -96.96017456054688,
      "activations/layer8_attention_weight_max": 42.34334182739258,
      "activations/layer8_attention_weight_min": -43.4793701171875,
      "activations/layer9_attention_weight_max": 33.149940490722656,
      "activations/layer9_attention_weight_min": -36.00613784790039,
      "epoch": 19.8,
      "learning_rate": 2.252689393939394e-05,
      "loss": 2.7301,
      "step": 340800
    },
    {
      "activations/layer0_attention_weight_max": 15.803964614868164,
      "activations/layer0_attention_weight_min": -12.005661964416504,
      "activations/layer10_attention_weight_max": 45.8557014465332,
      "activations/layer10_attention_weight_min": -40.289588928222656,
      "activations/layer11_attention_weight_max": 42.39977264404297,
      "activations/layer11_attention_weight_min": -38.69770050048828,
      "activations/layer12_attention_weight_max": 30.847139358520508,
      "activations/layer12_attention_weight_min": -30.632797241210938,
      "activations/layer13_attention_weight_max": 47.92186737060547,
      "activations/layer13_attention_weight_min": -43.183292388916016,
      "activations/layer14_attention_weight_max": 50.18122100830078,
      "activations/layer14_attention_weight_min": -48.130393981933594,
      "activations/layer15_attention_weight_max": 51.331932067871094,
      "activations/layer15_attention_weight_min": -45.60797882080078,
      "activations/layer16_attention_weight_max": 32.43684005737305,
      "activations/layer16_attention_weight_min": -32.54173278808594,
      "activations/layer17_attention_weight_max": 31.00983428955078,
      "activations/layer17_attention_weight_min": -29.865617752075195,
      "activations/layer18_attention_weight_max": 34.272281646728516,
      "activations/layer18_attention_weight_min": -28.024826049804688,
      "activations/layer19_attention_weight_max": 35.405452728271484,
      "activations/layer19_attention_weight_min": -31.755268096923828,
      "activations/layer1_attention_weight_max": 15.036774635314941,
      "activations/layer1_attention_weight_min": -14.561180114746094,
      "activations/layer20_attention_weight_max": 29.81334114074707,
      "activations/layer20_attention_weight_min": -25.951927185058594,
      "activations/layer21_attention_weight_max": 29.662681579589844,
      "activations/layer21_attention_weight_min": -24.545095443725586,
      "activations/layer22_attention_weight_max": 40.85837173461914,
      "activations/layer22_attention_weight_min": -33.72941970825195,
      "activations/layer23_attention_weight_max": 30.667354583740234,
      "activations/layer23_attention_weight_min": -26.831073760986328,
      "activations/layer2_attention_weight_max": 32.28322219848633,
      "activations/layer2_attention_weight_min": -31.744110107421875,
      "activations/layer3_attention_weight_max": 103.1520767211914,
      "activations/layer3_attention_weight_min": -103.32091522216797,
      "activations/layer4_attention_weight_max": 113.3203125,
      "activations/layer4_attention_weight_min": -113.8463134765625,
      "activations/layer5_attention_weight_max": 53.29948425292969,
      "activations/layer5_attention_weight_min": -61.860015869140625,
      "activations/layer6_attention_weight_max": 49.69707107543945,
      "activations/layer6_attention_weight_min": -52.71845626831055,
      "activations/layer7_attention_weight_max": 108.21174621582031,
      "activations/layer7_attention_weight_min": -112.9639663696289,
      "activations/layer8_attention_weight_max": 47.40004348754883,
      "activations/layer8_attention_weight_min": -52.7221565246582,
      "activations/layer9_attention_weight_max": 41.29507827758789,
      "activations/layer9_attention_weight_min": -38.3933219909668,
      "epoch": 19.81,
      "learning_rate": 2.2507954545454543e-05,
      "loss": 2.7083,
      "step": 340850
    },
    {
      "activations/layer0_attention_weight_max": 17.279523849487305,
      "activations/layer0_attention_weight_min": -12.62729263305664,
      "activations/layer10_attention_weight_max": 35.18748474121094,
      "activations/layer10_attention_weight_min": -35.1472282409668,
      "activations/layer11_attention_weight_max": 35.15020751953125,
      "activations/layer11_attention_weight_min": -33.42205047607422,
      "activations/layer12_attention_weight_max": 27.309110641479492,
      "activations/layer12_attention_weight_min": -27.507362365722656,
      "activations/layer13_attention_weight_max": 38.43415069580078,
      "activations/layer13_attention_weight_min": -36.24563980102539,
      "activations/layer14_attention_weight_max": 39.23204803466797,
      "activations/layer14_attention_weight_min": -39.613399505615234,
      "activations/layer15_attention_weight_max": 37.83918762207031,
      "activations/layer15_attention_weight_min": -37.29867935180664,
      "activations/layer16_attention_weight_max": 29.996213912963867,
      "activations/layer16_attention_weight_min": -28.360471725463867,
      "activations/layer17_attention_weight_max": 29.743404388427734,
      "activations/layer17_attention_weight_min": -26.00213623046875,
      "activations/layer18_attention_weight_max": 33.0076789855957,
      "activations/layer18_attention_weight_min": -26.230722427368164,
      "activations/layer19_attention_weight_max": 30.682859420776367,
      "activations/layer19_attention_weight_min": -30.555089950561523,
      "activations/layer1_attention_weight_max": 15.470519065856934,
      "activations/layer1_attention_weight_min": -14.282060623168945,
      "activations/layer20_attention_weight_max": 27.649145126342773,
      "activations/layer20_attention_weight_min": -25.30430793762207,
      "activations/layer21_attention_weight_max": 27.302602767944336,
      "activations/layer21_attention_weight_min": -25.065231323242188,
      "activations/layer22_attention_weight_max": 36.65852355957031,
      "activations/layer22_attention_weight_min": -31.140832901000977,
      "activations/layer23_attention_weight_max": 28.499616622924805,
      "activations/layer23_attention_weight_min": -27.041358947753906,
      "activations/layer2_attention_weight_max": 30.967817306518555,
      "activations/layer2_attention_weight_min": -29.229694366455078,
      "activations/layer3_attention_weight_max": 95.43902587890625,
      "activations/layer3_attention_weight_min": -102.30284118652344,
      "activations/layer4_attention_weight_max": 101.4065170288086,
      "activations/layer4_attention_weight_min": -111.4792709350586,
      "activations/layer5_attention_weight_max": 50.61450958251953,
      "activations/layer5_attention_weight_min": -60.95032501220703,
      "activations/layer6_attention_weight_max": 43.380455017089844,
      "activations/layer6_attention_weight_min": -45.15005111694336,
      "activations/layer7_attention_weight_max": 93.69937896728516,
      "activations/layer7_attention_weight_min": -93.06779479980469,
      "activations/layer8_attention_weight_max": 39.435394287109375,
      "activations/layer8_attention_weight_min": -42.21384811401367,
      "activations/layer9_attention_weight_max": 31.202407836914062,
      "activations/layer9_attention_weight_min": -33.48250198364258,
      "epoch": 19.81,
      "learning_rate": 2.248901515151515e-05,
      "loss": 2.7221,
      "step": 340900
    },
    {
      "activations/layer0_attention_weight_max": 16.080589294433594,
      "activations/layer0_attention_weight_min": -14.057273864746094,
      "activations/layer10_attention_weight_max": 31.923093795776367,
      "activations/layer10_attention_weight_min": -32.93947219848633,
      "activations/layer11_attention_weight_max": 31.37674331665039,
      "activations/layer11_attention_weight_min": -32.63432312011719,
      "activations/layer12_attention_weight_max": 24.49308967590332,
      "activations/layer12_attention_weight_min": -25.9205379486084,
      "activations/layer13_attention_weight_max": 36.012351989746094,
      "activations/layer13_attention_weight_min": -35.62260437011719,
      "activations/layer14_attention_weight_max": 39.725650787353516,
      "activations/layer14_attention_weight_min": -38.94570541381836,
      "activations/layer15_attention_weight_max": 35.14812469482422,
      "activations/layer15_attention_weight_min": -37.29090881347656,
      "activations/layer16_attention_weight_max": 26.03053092956543,
      "activations/layer16_attention_weight_min": -28.63966941833496,
      "activations/layer17_attention_weight_max": 25.556161880493164,
      "activations/layer17_attention_weight_min": -25.606801986694336,
      "activations/layer18_attention_weight_max": 31.713701248168945,
      "activations/layer18_attention_weight_min": -26.32122230529785,
      "activations/layer19_attention_weight_max": 31.86394691467285,
      "activations/layer19_attention_weight_min": -31.277219772338867,
      "activations/layer1_attention_weight_max": 15.808670043945312,
      "activations/layer1_attention_weight_min": -14.088441848754883,
      "activations/layer20_attention_weight_max": 29.705032348632812,
      "activations/layer20_attention_weight_min": -24.25139617919922,
      "activations/layer21_attention_weight_max": 29.41513442993164,
      "activations/layer21_attention_weight_min": -25.582382202148438,
      "activations/layer22_attention_weight_max": 38.32162094116211,
      "activations/layer22_attention_weight_min": -32.48501968383789,
      "activations/layer23_attention_weight_max": 32.729278564453125,
      "activations/layer23_attention_weight_min": -28.009693145751953,
      "activations/layer2_attention_weight_max": 31.476619720458984,
      "activations/layer2_attention_weight_min": -28.875883102416992,
      "activations/layer3_attention_weight_max": 92.34122467041016,
      "activations/layer3_attention_weight_min": -92.81055450439453,
      "activations/layer4_attention_weight_max": 102.2353286743164,
      "activations/layer4_attention_weight_min": -105.54839324951172,
      "activations/layer5_attention_weight_max": 52.00153732299805,
      "activations/layer5_attention_weight_min": -61.61589813232422,
      "activations/layer6_attention_weight_max": 41.58770751953125,
      "activations/layer6_attention_weight_min": -45.13351058959961,
      "activations/layer7_attention_weight_max": 90.7549057006836,
      "activations/layer7_attention_weight_min": -92.22079467773438,
      "activations/layer8_attention_weight_max": 39.9465446472168,
      "activations/layer8_attention_weight_min": -43.38811111450195,
      "activations/layer9_attention_weight_max": 31.231046676635742,
      "activations/layer9_attention_weight_min": -33.79315185546875,
      "epoch": 19.81,
      "learning_rate": 2.2470075757575753e-05,
      "loss": 2.7208,
      "step": 340950
    },
    {
      "activations/layer0_attention_weight_max": 15.677799224853516,
      "activations/layer0_attention_weight_min": -13.074687957763672,
      "activations/layer10_attention_weight_max": 30.764965057373047,
      "activations/layer10_attention_weight_min": -33.61658477783203,
      "activations/layer11_attention_weight_max": 34.644874572753906,
      "activations/layer11_attention_weight_min": -34.39434051513672,
      "activations/layer12_attention_weight_max": 25.63300132751465,
      "activations/layer12_attention_weight_min": -26.739423751831055,
      "activations/layer13_attention_weight_max": 39.81825256347656,
      "activations/layer13_attention_weight_min": -34.91938781738281,
      "activations/layer14_attention_weight_max": 39.871604919433594,
      "activations/layer14_attention_weight_min": -35.60406494140625,
      "activations/layer15_attention_weight_max": 36.98127746582031,
      "activations/layer15_attention_weight_min": -33.47178649902344,
      "activations/layer16_attention_weight_max": 28.326927185058594,
      "activations/layer16_attention_weight_min": -30.52903175354004,
      "activations/layer17_attention_weight_max": 28.214242935180664,
      "activations/layer17_attention_weight_min": -25.86908531188965,
      "activations/layer18_attention_weight_max": 34.042327880859375,
      "activations/layer18_attention_weight_min": -24.774944305419922,
      "activations/layer19_attention_weight_max": 30.5887451171875,
      "activations/layer19_attention_weight_min": -31.545392990112305,
      "activations/layer1_attention_weight_max": 15.317497253417969,
      "activations/layer1_attention_weight_min": -14.457880020141602,
      "activations/layer20_attention_weight_max": 28.634429931640625,
      "activations/layer20_attention_weight_min": -24.519302368164062,
      "activations/layer21_attention_weight_max": 29.945438385009766,
      "activations/layer21_attention_weight_min": -24.0365047454834,
      "activations/layer22_attention_weight_max": 37.05724334716797,
      "activations/layer22_attention_weight_min": -30.87514305114746,
      "activations/layer23_attention_weight_max": 30.065969467163086,
      "activations/layer23_attention_weight_min": -26.08418846130371,
      "activations/layer2_attention_weight_max": 30.58318328857422,
      "activations/layer2_attention_weight_min": -30.765193939208984,
      "activations/layer3_attention_weight_max": 97.58074188232422,
      "activations/layer3_attention_weight_min": -95.44898986816406,
      "activations/layer4_attention_weight_max": 103.25011444091797,
      "activations/layer4_attention_weight_min": -109.66926574707031,
      "activations/layer5_attention_weight_max": 49.5963249206543,
      "activations/layer5_attention_weight_min": -61.729835510253906,
      "activations/layer6_attention_weight_max": 42.538204193115234,
      "activations/layer6_attention_weight_min": -48.215702056884766,
      "activations/layer7_attention_weight_max": 93.05309295654297,
      "activations/layer7_attention_weight_min": -97.49246978759766,
      "activations/layer8_attention_weight_max": 41.533050537109375,
      "activations/layer8_attention_weight_min": -46.13351821899414,
      "activations/layer9_attention_weight_max": 29.792938232421875,
      "activations/layer9_attention_weight_min": -32.201168060302734,
      "epoch": 19.81,
      "learning_rate": 2.245151515151515e-05,
      "loss": 2.7261,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4439,
      "eval_samples_per_second": 508.534,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4439,
      "eval_openwebtext_samples_per_second": 508.534,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9629,
      "eval_wikitext_samples_per_second": 232.311,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_lambada_loss": 2.34765625,
      "eval_lambada_ppl": 10.461022947676224,
      "eval_lambada_runtime": 9.5614,
      "eval_lambada_samples_per_second": 509.235,
      "step": 341000
    },
    {
      "activations/layer0_attention_weight_max": 16.121047973632812,
      "activations/layer0_attention_weight_min": -14.163660049438477,
      "activations/layer10_attention_weight_max": 35.643558502197266,
      "activations/layer10_attention_weight_min": -37.20393371582031,
      "activations/layer11_attention_weight_max": 37.222511291503906,
      "activations/layer11_attention_weight_min": -38.32428741455078,
      "activations/layer12_attention_weight_max": 27.551836013793945,
      "activations/layer12_attention_weight_min": -29.577360153198242,
      "activations/layer13_attention_weight_max": 40.45681381225586,
      "activations/layer13_attention_weight_min": -36.63151931762695,
      "activations/layer14_attention_weight_max": 43.2075080871582,
      "activations/layer14_attention_weight_min": -40.307342529296875,
      "activations/layer15_attention_weight_max": 40.61907196044922,
      "activations/layer15_attention_weight_min": -35.683494567871094,
      "activations/layer16_attention_weight_max": 30.697315216064453,
      "activations/layer16_attention_weight_min": -27.389331817626953,
      "activations/layer17_attention_weight_max": 30.353511810302734,
      "activations/layer17_attention_weight_min": -26.113969802856445,
      "activations/layer18_attention_weight_max": 32.04468536376953,
      "activations/layer18_attention_weight_min": -26.29262924194336,
      "activations/layer19_attention_weight_max": 39.07136154174805,
      "activations/layer19_attention_weight_min": -31.337650299072266,
      "activations/layer1_attention_weight_max": 16.675846099853516,
      "activations/layer1_attention_weight_min": -15.116671562194824,
      "activations/layer20_attention_weight_max": 28.12382698059082,
      "activations/layer20_attention_weight_min": -25.961252212524414,
      "activations/layer21_attention_weight_max": 29.668197631835938,
      "activations/layer21_attention_weight_min": -25.4830322265625,
      "activations/layer22_attention_weight_max": 41.61241149902344,
      "activations/layer22_attention_weight_min": -33.930259704589844,
      "activations/layer23_attention_weight_max": 34.59501647949219,
      "activations/layer23_attention_weight_min": -25.04720687866211,
      "activations/layer2_attention_weight_max": 32.81999206542969,
      "activations/layer2_attention_weight_min": -33.034217834472656,
      "activations/layer3_attention_weight_max": 98.41404724121094,
      "activations/layer3_attention_weight_min": -102.31480407714844,
      "activations/layer4_attention_weight_max": 108.9455795288086,
      "activations/layer4_attention_weight_min": -109.26301574707031,
      "activations/layer5_attention_weight_max": 52.974082946777344,
      "activations/layer5_attention_weight_min": -63.322784423828125,
      "activations/layer6_attention_weight_max": 44.193138122558594,
      "activations/layer6_attention_weight_min": -46.38492965698242,
      "activations/layer7_attention_weight_max": 103.3472900390625,
      "activations/layer7_attention_weight_min": -97.40755462646484,
      "activations/layer8_attention_weight_max": 43.6928825378418,
      "activations/layer8_attention_weight_min": -43.708465576171875,
      "activations/layer9_attention_weight_max": 31.542036056518555,
      "activations/layer9_attention_weight_min": -35.96762466430664,
      "epoch": 19.82,
      "learning_rate": 2.2432575757575756e-05,
      "loss": 2.7291,
      "step": 341050
    },
    {
      "activations/layer0_attention_weight_max": 17.098556518554688,
      "activations/layer0_attention_weight_min": -12.17052173614502,
      "activations/layer10_attention_weight_max": 38.646671295166016,
      "activations/layer10_attention_weight_min": -37.925758361816406,
      "activations/layer11_attention_weight_max": 37.41057586669922,
      "activations/layer11_attention_weight_min": -37.490074157714844,
      "activations/layer12_attention_weight_max": 28.78142547607422,
      "activations/layer12_attention_weight_min": -30.053054809570312,
      "activations/layer13_attention_weight_max": 41.077880859375,
      "activations/layer13_attention_weight_min": -40.31999969482422,
      "activations/layer14_attention_weight_max": 42.32424545288086,
      "activations/layer14_attention_weight_min": -41.93080139160156,
      "activations/layer15_attention_weight_max": 38.623985290527344,
      "activations/layer15_attention_weight_min": -40.4901237487793,
      "activations/layer16_attention_weight_max": 31.801061630249023,
      "activations/layer16_attention_weight_min": -30.060279846191406,
      "activations/layer17_attention_weight_max": 33.43368148803711,
      "activations/layer17_attention_weight_min": -30.043033599853516,
      "activations/layer18_attention_weight_max": 34.55071258544922,
      "activations/layer18_attention_weight_min": -27.70284652709961,
      "activations/layer19_attention_weight_max": 36.05815505981445,
      "activations/layer19_attention_weight_min": -33.37976837158203,
      "activations/layer1_attention_weight_max": 15.99868106842041,
      "activations/layer1_attention_weight_min": -16.1821346282959,
      "activations/layer20_attention_weight_max": 31.659343719482422,
      "activations/layer20_attention_weight_min": -26.443632125854492,
      "activations/layer21_attention_weight_max": 30.986135482788086,
      "activations/layer21_attention_weight_min": -26.921470642089844,
      "activations/layer22_attention_weight_max": 44.82981491088867,
      "activations/layer22_attention_weight_min": -33.72349166870117,
      "activations/layer23_attention_weight_max": 34.18726348876953,
      "activations/layer23_attention_weight_min": -26.401126861572266,
      "activations/layer2_attention_weight_max": 32.47537612915039,
      "activations/layer2_attention_weight_min": -32.085960388183594,
      "activations/layer3_attention_weight_max": 104.56505584716797,
      "activations/layer3_attention_weight_min": -103.0892562866211,
      "activations/layer4_attention_weight_max": 116.59774017333984,
      "activations/layer4_attention_weight_min": -114.08656311035156,
      "activations/layer5_attention_weight_max": 53.89038848876953,
      "activations/layer5_attention_weight_min": -64.0138168334961,
      "activations/layer6_attention_weight_max": 48.62820816040039,
      "activations/layer6_attention_weight_min": -49.21321105957031,
      "activations/layer7_attention_weight_max": 97.79145050048828,
      "activations/layer7_attention_weight_min": -107.5202407836914,
      "activations/layer8_attention_weight_max": 45.26603698730469,
      "activations/layer8_attention_weight_min": -49.50414276123047,
      "activations/layer9_attention_weight_max": 34.6300163269043,
      "activations/layer9_attention_weight_min": -37.379417419433594,
      "epoch": 19.82,
      "learning_rate": 2.2413636363636364e-05,
      "loss": 2.7204,
      "step": 341100
    },
    {
      "activations/layer0_attention_weight_max": 16.423664093017578,
      "activations/layer0_attention_weight_min": -13.067633628845215,
      "activations/layer10_attention_weight_max": 33.90293884277344,
      "activations/layer10_attention_weight_min": -34.885711669921875,
      "activations/layer11_attention_weight_max": 33.436336517333984,
      "activations/layer11_attention_weight_min": -36.252098083496094,
      "activations/layer12_attention_weight_max": 27.35853385925293,
      "activations/layer12_attention_weight_min": -27.570213317871094,
      "activations/layer13_attention_weight_max": 39.39183807373047,
      "activations/layer13_attention_weight_min": -37.65057373046875,
      "activations/layer14_attention_weight_max": 42.09501266479492,
      "activations/layer14_attention_weight_min": -38.58002853393555,
      "activations/layer15_attention_weight_max": 37.89976501464844,
      "activations/layer15_attention_weight_min": -39.350582122802734,
      "activations/layer16_attention_weight_max": 30.274417877197266,
      "activations/layer16_attention_weight_min": -32.041996002197266,
      "activations/layer17_attention_weight_max": 32.906036376953125,
      "activations/layer17_attention_weight_min": -29.79463005065918,
      "activations/layer18_attention_weight_max": 33.734004974365234,
      "activations/layer18_attention_weight_min": -29.538021087646484,
      "activations/layer19_attention_weight_max": 32.71006393432617,
      "activations/layer19_attention_weight_min": -35.99442672729492,
      "activations/layer1_attention_weight_max": 15.450786590576172,
      "activations/layer1_attention_weight_min": -14.684270858764648,
      "activations/layer20_attention_weight_max": 30.6802978515625,
      "activations/layer20_attention_weight_min": -29.17916488647461,
      "activations/layer21_attention_weight_max": 28.70201301574707,
      "activations/layer21_attention_weight_min": -29.984298706054688,
      "activations/layer22_attention_weight_max": 42.09368133544922,
      "activations/layer22_attention_weight_min": -36.43516159057617,
      "activations/layer23_attention_weight_max": 31.91724967956543,
      "activations/layer23_attention_weight_min": -29.32063102722168,
      "activations/layer2_attention_weight_max": 31.359554290771484,
      "activations/layer2_attention_weight_min": -29.625782012939453,
      "activations/layer3_attention_weight_max": 98.76802062988281,
      "activations/layer3_attention_weight_min": -101.35933685302734,
      "activations/layer4_attention_weight_max": 104.61602020263672,
      "activations/layer4_attention_weight_min": -109.2751693725586,
      "activations/layer5_attention_weight_max": 51.21251678466797,
      "activations/layer5_attention_weight_min": -59.944236755371094,
      "activations/layer6_attention_weight_max": 45.06289291381836,
      "activations/layer6_attention_weight_min": -46.37969207763672,
      "activations/layer7_attention_weight_max": 94.58765411376953,
      "activations/layer7_attention_weight_min": -98.82440185546875,
      "activations/layer8_attention_weight_max": 41.67040252685547,
      "activations/layer8_attention_weight_min": -45.15367126464844,
      "activations/layer9_attention_weight_max": 32.06057357788086,
      "activations/layer9_attention_weight_min": -34.237213134765625,
      "epoch": 19.82,
      "learning_rate": 2.2394696969696965e-05,
      "loss": 2.7198,
      "step": 341150
    },
    {
      "activations/layer0_attention_weight_max": 16.826744079589844,
      "activations/layer0_attention_weight_min": -14.592550277709961,
      "activations/layer10_attention_weight_max": 33.543392181396484,
      "activations/layer10_attention_weight_min": -34.161476135253906,
      "activations/layer11_attention_weight_max": 32.138084411621094,
      "activations/layer11_attention_weight_min": -33.63840103149414,
      "activations/layer12_attention_weight_max": 24.569583892822266,
      "activations/layer12_attention_weight_min": -27.69464874267578,
      "activations/layer13_attention_weight_max": 36.68920135498047,
      "activations/layer13_attention_weight_min": -38.5128059387207,
      "activations/layer14_attention_weight_max": 39.747650146484375,
      "activations/layer14_attention_weight_min": -39.09242630004883,
      "activations/layer15_attention_weight_max": 39.597511291503906,
      "activations/layer15_attention_weight_min": -39.57514953613281,
      "activations/layer16_attention_weight_max": 28.872758865356445,
      "activations/layer16_attention_weight_min": -30.801700592041016,
      "activations/layer17_attention_weight_max": 30.363101959228516,
      "activations/layer17_attention_weight_min": -29.36274528503418,
      "activations/layer18_attention_weight_max": 31.326982498168945,
      "activations/layer18_attention_weight_min": -27.920175552368164,
      "activations/layer19_attention_weight_max": 33.78839111328125,
      "activations/layer19_attention_weight_min": -31.308307647705078,
      "activations/layer1_attention_weight_max": 14.726873397827148,
      "activations/layer1_attention_weight_min": -15.767422676086426,
      "activations/layer20_attention_weight_max": 26.753637313842773,
      "activations/layer20_attention_weight_min": -27.458972930908203,
      "activations/layer21_attention_weight_max": 25.32932472229004,
      "activations/layer21_attention_weight_min": -24.163713455200195,
      "activations/layer22_attention_weight_max": 37.643272399902344,
      "activations/layer22_attention_weight_min": -30.74502944946289,
      "activations/layer23_attention_weight_max": 30.52513885498047,
      "activations/layer23_attention_weight_min": -25.64119529724121,
      "activations/layer2_attention_weight_max": 30.103958129882812,
      "activations/layer2_attention_weight_min": -28.924482345581055,
      "activations/layer3_attention_weight_max": 94.15123748779297,
      "activations/layer3_attention_weight_min": -95.37919616699219,
      "activations/layer4_attention_weight_max": 101.5321273803711,
      "activations/layer4_attention_weight_min": -107.4112777709961,
      "activations/layer5_attention_weight_max": 47.76780700683594,
      "activations/layer5_attention_weight_min": -59.34172058105469,
      "activations/layer6_attention_weight_max": 41.879913330078125,
      "activations/layer6_attention_weight_min": -44.84657669067383,
      "activations/layer7_attention_weight_max": 91.61341094970703,
      "activations/layer7_attention_weight_min": -97.35185241699219,
      "activations/layer8_attention_weight_max": 40.03602981567383,
      "activations/layer8_attention_weight_min": -41.76264190673828,
      "activations/layer9_attention_weight_max": 31.107257843017578,
      "activations/layer9_attention_weight_min": -33.004825592041016,
      "epoch": 19.83,
      "learning_rate": 2.2375757575757574e-05,
      "loss": 2.7112,
      "step": 341200
    },
    {
      "activations/layer0_attention_weight_max": 16.336030960083008,
      "activations/layer0_attention_weight_min": -12.297223091125488,
      "activations/layer10_attention_weight_max": 35.67700958251953,
      "activations/layer10_attention_weight_min": -36.61664581298828,
      "activations/layer11_attention_weight_max": 34.707576751708984,
      "activations/layer11_attention_weight_min": -35.499473571777344,
      "activations/layer12_attention_weight_max": 27.578279495239258,
      "activations/layer12_attention_weight_min": -27.63432502746582,
      "activations/layer13_attention_weight_max": 42.40687561035156,
      "activations/layer13_attention_weight_min": -36.64073181152344,
      "activations/layer14_attention_weight_max": 46.31472396850586,
      "activations/layer14_attention_weight_min": -41.89787673950195,
      "activations/layer15_attention_weight_max": 41.059757232666016,
      "activations/layer15_attention_weight_min": -40.03110885620117,
      "activations/layer16_attention_weight_max": 29.699010848999023,
      "activations/layer16_attention_weight_min": -28.508636474609375,
      "activations/layer17_attention_weight_max": 29.703250885009766,
      "activations/layer17_attention_weight_min": -28.822383880615234,
      "activations/layer18_attention_weight_max": 33.5081672668457,
      "activations/layer18_attention_weight_min": -28.77039909362793,
      "activations/layer19_attention_weight_max": 35.07026290893555,
      "activations/layer19_attention_weight_min": -35.25719451904297,
      "activations/layer1_attention_weight_max": 15.296823501586914,
      "activations/layer1_attention_weight_min": -14.898411750793457,
      "activations/layer20_attention_weight_max": 28.6116943359375,
      "activations/layer20_attention_weight_min": -26.6370792388916,
      "activations/layer21_attention_weight_max": 29.41171646118164,
      "activations/layer21_attention_weight_min": -27.750280380249023,
      "activations/layer22_attention_weight_max": 41.764591217041016,
      "activations/layer22_attention_weight_min": -35.47663879394531,
      "activations/layer23_attention_weight_max": 34.88813781738281,
      "activations/layer23_attention_weight_min": -27.455305099487305,
      "activations/layer2_attention_weight_max": 32.1554069519043,
      "activations/layer2_attention_weight_min": -31.29132843017578,
      "activations/layer3_attention_weight_max": 98.41181945800781,
      "activations/layer3_attention_weight_min": -101.13343048095703,
      "activations/layer4_attention_weight_max": 104.68318176269531,
      "activations/layer4_attention_weight_min": -108.37438201904297,
      "activations/layer5_attention_weight_max": 49.00808334350586,
      "activations/layer5_attention_weight_min": -66.52743530273438,
      "activations/layer6_attention_weight_max": 45.44347381591797,
      "activations/layer6_attention_weight_min": -53.13351058959961,
      "activations/layer7_attention_weight_max": 96.5353012084961,
      "activations/layer7_attention_weight_min": -105.32054901123047,
      "activations/layer8_attention_weight_max": 44.76033020019531,
      "activations/layer8_attention_weight_min": -48.398773193359375,
      "activations/layer9_attention_weight_max": 35.033607482910156,
      "activations/layer9_attention_weight_min": -37.741573333740234,
      "epoch": 19.83,
      "learning_rate": 2.235681818181818e-05,
      "loss": 2.7209,
      "step": 341250
    },
    {
      "activations/layer0_attention_weight_max": 15.275886535644531,
      "activations/layer0_attention_weight_min": -12.704004287719727,
      "activations/layer10_attention_weight_max": 34.1043701171875,
      "activations/layer10_attention_weight_min": -37.23175048828125,
      "activations/layer11_attention_weight_max": 33.73256301879883,
      "activations/layer11_attention_weight_min": -35.97246551513672,
      "activations/layer12_attention_weight_max": 26.500431060791016,
      "activations/layer12_attention_weight_min": -29.74264144897461,
      "activations/layer13_attention_weight_max": 37.789119720458984,
      "activations/layer13_attention_weight_min": -38.332454681396484,
      "activations/layer14_attention_weight_max": 42.48310470581055,
      "activations/layer14_attention_weight_min": -40.29117202758789,
      "activations/layer15_attention_weight_max": 39.71254348754883,
      "activations/layer15_attention_weight_min": -39.312801361083984,
      "activations/layer16_attention_weight_max": 30.963451385498047,
      "activations/layer16_attention_weight_min": -28.256208419799805,
      "activations/layer17_attention_weight_max": 28.80942726135254,
      "activations/layer17_attention_weight_min": -27.246347427368164,
      "activations/layer18_attention_weight_max": 30.252925872802734,
      "activations/layer18_attention_weight_min": -29.39387321472168,
      "activations/layer19_attention_weight_max": 30.990554809570312,
      "activations/layer19_attention_weight_min": -32.25580978393555,
      "activations/layer1_attention_weight_max": 15.847841262817383,
      "activations/layer1_attention_weight_min": -14.974225044250488,
      "activations/layer20_attention_weight_max": 26.161399841308594,
      "activations/layer20_attention_weight_min": -27.115501403808594,
      "activations/layer21_attention_weight_max": 25.495037078857422,
      "activations/layer21_attention_weight_min": -25.843490600585938,
      "activations/layer22_attention_weight_max": 41.807411193847656,
      "activations/layer22_attention_weight_min": -37.37353515625,
      "activations/layer23_attention_weight_max": 34.57099533081055,
      "activations/layer23_attention_weight_min": -28.151912689208984,
      "activations/layer2_attention_weight_max": 30.85680389404297,
      "activations/layer2_attention_weight_min": -30.74808120727539,
      "activations/layer3_attention_weight_max": 97.698974609375,
      "activations/layer3_attention_weight_min": -99.63937377929688,
      "activations/layer4_attention_weight_max": 103.35298156738281,
      "activations/layer4_attention_weight_min": -106.2728500366211,
      "activations/layer5_attention_weight_max": 50.35113525390625,
      "activations/layer5_attention_weight_min": -62.465633392333984,
      "activations/layer6_attention_weight_max": 44.641395568847656,
      "activations/layer6_attention_weight_min": -48.283695220947266,
      "activations/layer7_attention_weight_max": 92.09228515625,
      "activations/layer7_attention_weight_min": -98.85729217529297,
      "activations/layer8_attention_weight_max": 40.718955993652344,
      "activations/layer8_attention_weight_min": -43.05379867553711,
      "activations/layer9_attention_weight_max": 31.96317481994629,
      "activations/layer9_attention_weight_min": -33.94619369506836,
      "epoch": 19.83,
      "learning_rate": 2.2337878787878787e-05,
      "loss": 2.7133,
      "step": 341300
    },
    {
      "activations/layer0_attention_weight_max": 17.50984001159668,
      "activations/layer0_attention_weight_min": -12.84149169921875,
      "activations/layer10_attention_weight_max": 40.244258880615234,
      "activations/layer10_attention_weight_min": -39.82634353637695,
      "activations/layer11_attention_weight_max": 38.45457458496094,
      "activations/layer11_attention_weight_min": -40.557945251464844,
      "activations/layer12_attention_weight_max": 32.14970779418945,
      "activations/layer12_attention_weight_min": -30.06562042236328,
      "activations/layer13_attention_weight_max": 48.23479461669922,
      "activations/layer13_attention_weight_min": -40.19350051879883,
      "activations/layer14_attention_weight_max": 53.81875228881836,
      "activations/layer14_attention_weight_min": -43.61611557006836,
      "activations/layer15_attention_weight_max": 45.79710388183594,
      "activations/layer15_attention_weight_min": -41.35265350341797,
      "activations/layer16_attention_weight_max": 35.901344299316406,
      "activations/layer16_attention_weight_min": -30.796724319458008,
      "activations/layer17_attention_weight_max": 32.535953521728516,
      "activations/layer17_attention_weight_min": -27.189096450805664,
      "activations/layer18_attention_weight_max": 35.627681732177734,
      "activations/layer18_attention_weight_min": -29.47494125366211,
      "activations/layer19_attention_weight_max": 41.604976654052734,
      "activations/layer19_attention_weight_min": -35.80348587036133,
      "activations/layer1_attention_weight_max": 15.536592483520508,
      "activations/layer1_attention_weight_min": -14.288890838623047,
      "activations/layer20_attention_weight_max": 33.85368728637695,
      "activations/layer20_attention_weight_min": -27.89971160888672,
      "activations/layer21_attention_weight_max": 34.24519348144531,
      "activations/layer21_attention_weight_min": -27.578960418701172,
      "activations/layer22_attention_weight_max": 49.7145881652832,
      "activations/layer22_attention_weight_min": -37.70838928222656,
      "activations/layer23_attention_weight_max": 44.04038619995117,
      "activations/layer23_attention_weight_min": -28.71803855895996,
      "activations/layer2_attention_weight_max": 30.743894577026367,
      "activations/layer2_attention_weight_min": -29.73941421508789,
      "activations/layer3_attention_weight_max": 99.83236694335938,
      "activations/layer3_attention_weight_min": -100.89160919189453,
      "activations/layer4_attention_weight_max": 106.94679260253906,
      "activations/layer4_attention_weight_min": -109.84342193603516,
      "activations/layer5_attention_weight_max": 52.352291107177734,
      "activations/layer5_attention_weight_min": -66.11770629882812,
      "activations/layer6_attention_weight_max": 45.68122482299805,
      "activations/layer6_attention_weight_min": -49.828067779541016,
      "activations/layer7_attention_weight_max": 97.26747131347656,
      "activations/layer7_attention_weight_min": -105.05648803710938,
      "activations/layer8_attention_weight_max": 45.2276496887207,
      "activations/layer8_attention_weight_min": -48.32322692871094,
      "activations/layer9_attention_weight_max": 35.48946762084961,
      "activations/layer9_attention_weight_min": -35.74009704589844,
      "epoch": 19.83,
      "learning_rate": 2.231893939393939e-05,
      "loss": 2.7294,
      "step": 341350
    },
    {
      "activations/layer0_attention_weight_max": 15.601017951965332,
      "activations/layer0_attention_weight_min": -11.890218734741211,
      "activations/layer10_attention_weight_max": 35.04004669189453,
      "activations/layer10_attention_weight_min": -37.74205017089844,
      "activations/layer11_attention_weight_max": 35.57273864746094,
      "activations/layer11_attention_weight_min": -38.058189392089844,
      "activations/layer12_attention_weight_max": 26.80911636352539,
      "activations/layer12_attention_weight_min": -26.72635841369629,
      "activations/layer13_attention_weight_max": 40.03936004638672,
      "activations/layer13_attention_weight_min": -35.58818435668945,
      "activations/layer14_attention_weight_max": 44.07767868041992,
      "activations/layer14_attention_weight_min": -39.81270980834961,
      "activations/layer15_attention_weight_max": 40.58948516845703,
      "activations/layer15_attention_weight_min": -35.87183380126953,
      "activations/layer16_attention_weight_max": 30.055435180664062,
      "activations/layer16_attention_weight_min": -28.082773208618164,
      "activations/layer17_attention_weight_max": 30.93709945678711,
      "activations/layer17_attention_weight_min": -26.166034698486328,
      "activations/layer18_attention_weight_max": 31.286155700683594,
      "activations/layer18_attention_weight_min": -25.285030364990234,
      "activations/layer19_attention_weight_max": 35.995994567871094,
      "activations/layer19_attention_weight_min": -30.534597396850586,
      "activations/layer1_attention_weight_max": 15.542802810668945,
      "activations/layer1_attention_weight_min": -14.187955856323242,
      "activations/layer20_attention_weight_max": 31.495038986206055,
      "activations/layer20_attention_weight_min": -24.439342498779297,
      "activations/layer21_attention_weight_max": 31.58531379699707,
      "activations/layer21_attention_weight_min": -25.239599227905273,
      "activations/layer22_attention_weight_max": 43.990745544433594,
      "activations/layer22_attention_weight_min": -30.935380935668945,
      "activations/layer23_attention_weight_max": 34.578956604003906,
      "activations/layer23_attention_weight_min": -25.414457321166992,
      "activations/layer2_attention_weight_max": 31.156003952026367,
      "activations/layer2_attention_weight_min": -30.125158309936523,
      "activations/layer3_attention_weight_max": 102.26449584960938,
      "activations/layer3_attention_weight_min": -99.07164764404297,
      "activations/layer4_attention_weight_max": 109.5123291015625,
      "activations/layer4_attention_weight_min": -108.56563568115234,
      "activations/layer5_attention_weight_max": 52.25592041015625,
      "activations/layer5_attention_weight_min": -61.54356384277344,
      "activations/layer6_attention_weight_max": 45.42134094238281,
      "activations/layer6_attention_weight_min": -47.904258728027344,
      "activations/layer7_attention_weight_max": 100.4404067993164,
      "activations/layer7_attention_weight_min": -102.7033462524414,
      "activations/layer8_attention_weight_max": 43.78876876831055,
      "activations/layer8_attention_weight_min": -47.155033111572266,
      "activations/layer9_attention_weight_max": 33.520442962646484,
      "activations/layer9_attention_weight_min": -36.67197799682617,
      "epoch": 19.84,
      "learning_rate": 2.23e-05,
      "loss": 2.7313,
      "step": 341400
    },
    {
      "activations/layer0_attention_weight_max": 16.49510955810547,
      "activations/layer0_attention_weight_min": -14.05455207824707,
      "activations/layer10_attention_weight_max": 33.843788146972656,
      "activations/layer10_attention_weight_min": -35.54533386230469,
      "activations/layer11_attention_weight_max": 32.21751403808594,
      "activations/layer11_attention_weight_min": -37.27980422973633,
      "activations/layer12_attention_weight_max": 30.431238174438477,
      "activations/layer12_attention_weight_min": -26.902921676635742,
      "activations/layer13_attention_weight_max": 39.96307373046875,
      "activations/layer13_attention_weight_min": -37.61518096923828,
      "activations/layer14_attention_weight_max": 43.528160095214844,
      "activations/layer14_attention_weight_min": -38.822547912597656,
      "activations/layer15_attention_weight_max": 39.081214904785156,
      "activations/layer15_attention_weight_min": -37.078468322753906,
      "activations/layer16_attention_weight_max": 28.302392959594727,
      "activations/layer16_attention_weight_min": -29.132347106933594,
      "activations/layer17_attention_weight_max": 32.36150360107422,
      "activations/layer17_attention_weight_min": -29.57267951965332,
      "activations/layer18_attention_weight_max": 32.2026481628418,
      "activations/layer18_attention_weight_min": -26.90206527709961,
      "activations/layer19_attention_weight_max": 33.621639251708984,
      "activations/layer19_attention_weight_min": -32.111820220947266,
      "activations/layer1_attention_weight_max": 15.691356658935547,
      "activations/layer1_attention_weight_min": -15.457894325256348,
      "activations/layer20_attention_weight_max": 28.796634674072266,
      "activations/layer20_attention_weight_min": -25.799583435058594,
      "activations/layer21_attention_weight_max": 27.257665634155273,
      "activations/layer21_attention_weight_min": -25.0531063079834,
      "activations/layer22_attention_weight_max": 43.944000244140625,
      "activations/layer22_attention_weight_min": -33.73118591308594,
      "activations/layer23_attention_weight_max": 31.733089447021484,
      "activations/layer23_attention_weight_min": -27.375572204589844,
      "activations/layer2_attention_weight_max": 32.052978515625,
      "activations/layer2_attention_weight_min": -30.099157333374023,
      "activations/layer3_attention_weight_max": 97.7123794555664,
      "activations/layer3_attention_weight_min": -99.21758270263672,
      "activations/layer4_attention_weight_max": 109.02430725097656,
      "activations/layer4_attention_weight_min": -111.1946792602539,
      "activations/layer5_attention_weight_max": 50.63157272338867,
      "activations/layer5_attention_weight_min": -60.04758834838867,
      "activations/layer6_attention_weight_max": 44.16807556152344,
      "activations/layer6_attention_weight_min": -45.92622375488281,
      "activations/layer7_attention_weight_max": 91.77560424804688,
      "activations/layer7_attention_weight_min": -94.8927001953125,
      "activations/layer8_attention_weight_max": 43.455406188964844,
      "activations/layer8_attention_weight_min": -45.220951080322266,
      "activations/layer9_attention_weight_max": 33.85013961791992,
      "activations/layer9_attention_weight_min": -35.37704086303711,
      "epoch": 19.84,
      "learning_rate": 2.22810606060606e-05,
      "loss": 2.7161,
      "step": 341450
    },
    {
      "activations/layer0_attention_weight_max": 17.467470169067383,
      "activations/layer0_attention_weight_min": -12.082775115966797,
      "activations/layer10_attention_weight_max": 37.81182098388672,
      "activations/layer10_attention_weight_min": -38.02191925048828,
      "activations/layer11_attention_weight_max": 38.69208526611328,
      "activations/layer11_attention_weight_min": -40.19270324707031,
      "activations/layer12_attention_weight_max": 27.627792358398438,
      "activations/layer12_attention_weight_min": -27.438289642333984,
      "activations/layer13_attention_weight_max": 43.3585090637207,
      "activations/layer13_attention_weight_min": -36.75416946411133,
      "activations/layer14_attention_weight_max": 42.849571228027344,
      "activations/layer14_attention_weight_min": -39.33544158935547,
      "activations/layer15_attention_weight_max": 41.009742736816406,
      "activations/layer15_attention_weight_min": -35.67560577392578,
      "activations/layer16_attention_weight_max": 29.58871078491211,
      "activations/layer16_attention_weight_min": -26.94175910949707,
      "activations/layer17_attention_weight_max": 28.02403450012207,
      "activations/layer17_attention_weight_min": -25.09748077392578,
      "activations/layer18_attention_weight_max": 33.27217102050781,
      "activations/layer18_attention_weight_min": -25.43307113647461,
      "activations/layer19_attention_weight_max": 36.136226654052734,
      "activations/layer19_attention_weight_min": -30.495010375976562,
      "activations/layer1_attention_weight_max": 16.776952743530273,
      "activations/layer1_attention_weight_min": -16.228118896484375,
      "activations/layer20_attention_weight_max": 26.613927841186523,
      "activations/layer20_attention_weight_min": -24.240875244140625,
      "activations/layer21_attention_weight_max": 27.927141189575195,
      "activations/layer21_attention_weight_min": -23.852590560913086,
      "activations/layer22_attention_weight_max": 41.11884307861328,
      "activations/layer22_attention_weight_min": -31.27486801147461,
      "activations/layer23_attention_weight_max": 33.047767639160156,
      "activations/layer23_attention_weight_min": -26.58865737915039,
      "activations/layer2_attention_weight_max": 34.74751281738281,
      "activations/layer2_attention_weight_min": -33.18852233886719,
      "activations/layer3_attention_weight_max": 105.04544830322266,
      "activations/layer3_attention_weight_min": -110.0376205444336,
      "activations/layer4_attention_weight_max": 111.54618072509766,
      "activations/layer4_attention_weight_min": -114.20903778076172,
      "activations/layer5_attention_weight_max": 52.452484130859375,
      "activations/layer5_attention_weight_min": -62.67869186401367,
      "activations/layer6_attention_weight_max": 46.77730941772461,
      "activations/layer6_attention_weight_min": -49.77293014526367,
      "activations/layer7_attention_weight_max": 99.0109634399414,
      "activations/layer7_attention_weight_min": -104.40724182128906,
      "activations/layer8_attention_weight_max": 45.46067428588867,
      "activations/layer8_attention_weight_min": -46.50950241088867,
      "activations/layer9_attention_weight_max": 35.40138244628906,
      "activations/layer9_attention_weight_min": -36.44561004638672,
      "epoch": 19.84,
      "learning_rate": 2.226212121212121e-05,
      "loss": 2.714,
      "step": 341500
    },
    {
      "activations/layer0_attention_weight_max": 16.304443359375,
      "activations/layer0_attention_weight_min": -12.246845245361328,
      "activations/layer10_attention_weight_max": 33.322418212890625,
      "activations/layer10_attention_weight_min": -35.38717269897461,
      "activations/layer11_attention_weight_max": 35.94573211669922,
      "activations/layer11_attention_weight_min": -37.1044921875,
      "activations/layer12_attention_weight_max": 28.171878814697266,
      "activations/layer12_attention_weight_min": -30.420530319213867,
      "activations/layer13_attention_weight_max": 39.2830696105957,
      "activations/layer13_attention_weight_min": -35.76615524291992,
      "activations/layer14_attention_weight_max": 44.06135559082031,
      "activations/layer14_attention_weight_min": -37.7709846496582,
      "activations/layer15_attention_weight_max": 41.204673767089844,
      "activations/layer15_attention_weight_min": -36.82036209106445,
      "activations/layer16_attention_weight_max": 31.164079666137695,
      "activations/layer16_attention_weight_min": -28.175676345825195,
      "activations/layer17_attention_weight_max": 36.375972747802734,
      "activations/layer17_attention_weight_min": -26.28232765197754,
      "activations/layer18_attention_weight_max": 34.79124450683594,
      "activations/layer18_attention_weight_min": -26.405380249023438,
      "activations/layer19_attention_weight_max": 36.63810348510742,
      "activations/layer19_attention_weight_min": -30.814453125,
      "activations/layer1_attention_weight_max": 16.722726821899414,
      "activations/layer1_attention_weight_min": -13.684192657470703,
      "activations/layer20_attention_weight_max": 32.80241012573242,
      "activations/layer20_attention_weight_min": -25.237215042114258,
      "activations/layer21_attention_weight_max": 34.46197509765625,
      "activations/layer21_attention_weight_min": -24.410778045654297,
      "activations/layer22_attention_weight_max": 46.19160842895508,
      "activations/layer22_attention_weight_min": -29.82276725769043,
      "activations/layer23_attention_weight_max": 34.1727294921875,
      "activations/layer23_attention_weight_min": -24.889755249023438,
      "activations/layer2_attention_weight_max": 30.88222312927246,
      "activations/layer2_attention_weight_min": -31.254371643066406,
      "activations/layer3_attention_weight_max": 98.39054870605469,
      "activations/layer3_attention_weight_min": -98.63166046142578,
      "activations/layer4_attention_weight_max": 102.99747467041016,
      "activations/layer4_attention_weight_min": -107.85619354248047,
      "activations/layer5_attention_weight_max": 50.169395446777344,
      "activations/layer5_attention_weight_min": -58.83999252319336,
      "activations/layer6_attention_weight_max": 43.66549301147461,
      "activations/layer6_attention_weight_min": -48.83384323120117,
      "activations/layer7_attention_weight_max": 90.96733856201172,
      "activations/layer7_attention_weight_min": -101.17790222167969,
      "activations/layer8_attention_weight_max": 41.538028717041016,
      "activations/layer8_attention_weight_min": -45.61366653442383,
      "activations/layer9_attention_weight_max": 30.797773361206055,
      "activations/layer9_attention_weight_min": -35.08692169189453,
      "epoch": 19.85,
      "learning_rate": 2.2243181818181815e-05,
      "loss": 2.7347,
      "step": 341550
    },
    {
      "activations/layer0_attention_weight_max": 15.895034790039062,
      "activations/layer0_attention_weight_min": -12.978615760803223,
      "activations/layer10_attention_weight_max": 33.378780364990234,
      "activations/layer10_attention_weight_min": -34.80876159667969,
      "activations/layer11_attention_weight_max": 34.65011978149414,
      "activations/layer11_attention_weight_min": -36.38364791870117,
      "activations/layer12_attention_weight_max": 27.077003479003906,
      "activations/layer12_attention_weight_min": -26.62077522277832,
      "activations/layer13_attention_weight_max": 39.55523681640625,
      "activations/layer13_attention_weight_min": -36.513214111328125,
      "activations/layer14_attention_weight_max": 42.520023345947266,
      "activations/layer14_attention_weight_min": -38.959983825683594,
      "activations/layer15_attention_weight_max": 41.387149810791016,
      "activations/layer15_attention_weight_min": -36.56985092163086,
      "activations/layer16_attention_weight_max": 32.154666900634766,
      "activations/layer16_attention_weight_min": -29.406681060791016,
      "activations/layer17_attention_weight_max": 29.20633316040039,
      "activations/layer17_attention_weight_min": -28.049217224121094,
      "activations/layer18_attention_weight_max": 31.256803512573242,
      "activations/layer18_attention_weight_min": -26.405595779418945,
      "activations/layer19_attention_weight_max": 34.59258270263672,
      "activations/layer19_attention_weight_min": -31.16886329650879,
      "activations/layer1_attention_weight_max": 15.928577423095703,
      "activations/layer1_attention_weight_min": -15.430765151977539,
      "activations/layer20_attention_weight_max": 27.715024948120117,
      "activations/layer20_attention_weight_min": -24.03466033935547,
      "activations/layer21_attention_weight_max": 30.313278198242188,
      "activations/layer21_attention_weight_min": -24.353946685791016,
      "activations/layer22_attention_weight_max": 44.51938247680664,
      "activations/layer22_attention_weight_min": -32.22832107543945,
      "activations/layer23_attention_weight_max": 31.34341049194336,
      "activations/layer23_attention_weight_min": -26.4560604095459,
      "activations/layer2_attention_weight_max": 31.563459396362305,
      "activations/layer2_attention_weight_min": -30.44196891784668,
      "activations/layer3_attention_weight_max": 102.05731964111328,
      "activations/layer3_attention_weight_min": -98.55630493164062,
      "activations/layer4_attention_weight_max": 108.3194580078125,
      "activations/layer4_attention_weight_min": -108.61085510253906,
      "activations/layer5_attention_weight_max": 51.14263916015625,
      "activations/layer5_attention_weight_min": -62.39824676513672,
      "activations/layer6_attention_weight_max": 44.6668815612793,
      "activations/layer6_attention_weight_min": -47.35707473754883,
      "activations/layer7_attention_weight_max": 91.61116027832031,
      "activations/layer7_attention_weight_min": -104.98025512695312,
      "activations/layer8_attention_weight_max": 41.55895233154297,
      "activations/layer8_attention_weight_min": -45.94445037841797,
      "activations/layer9_attention_weight_max": 30.588850021362305,
      "activations/layer9_attention_weight_min": -35.76340103149414,
      "epoch": 19.85,
      "learning_rate": 2.2224242424242423e-05,
      "loss": 2.7274,
      "step": 341600
    },
    {
      "activations/layer0_attention_weight_max": 15.908430099487305,
      "activations/layer0_attention_weight_min": -13.32668399810791,
      "activations/layer10_attention_weight_max": 53.3829345703125,
      "activations/layer10_attention_weight_min": -51.654022216796875,
      "activations/layer11_attention_weight_max": 51.15602111816406,
      "activations/layer11_attention_weight_min": -54.106964111328125,
      "activations/layer12_attention_weight_max": 39.68146896362305,
      "activations/layer12_attention_weight_min": -34.60508346557617,
      "activations/layer13_attention_weight_max": 59.399906158447266,
      "activations/layer13_attention_weight_min": -47.145408630371094,
      "activations/layer14_attention_weight_max": 67.76113891601562,
      "activations/layer14_attention_weight_min": -54.56263732910156,
      "activations/layer15_attention_weight_max": 67.07437133789062,
      "activations/layer15_attention_weight_min": -58.784603118896484,
      "activations/layer16_attention_weight_max": 48.8116569519043,
      "activations/layer16_attention_weight_min": -36.798179626464844,
      "activations/layer17_attention_weight_max": 42.747745513916016,
      "activations/layer17_attention_weight_min": -38.75373077392578,
      "activations/layer18_attention_weight_max": 40.94636154174805,
      "activations/layer18_attention_weight_min": -37.57415008544922,
      "activations/layer19_attention_weight_max": 48.02895736694336,
      "activations/layer19_attention_weight_min": -37.55544662475586,
      "activations/layer1_attention_weight_max": 15.740731239318848,
      "activations/layer1_attention_weight_min": -14.707778930664062,
      "activations/layer20_attention_weight_max": 40.18692398071289,
      "activations/layer20_attention_weight_min": -28.9942626953125,
      "activations/layer21_attention_weight_max": 37.750144958496094,
      "activations/layer21_attention_weight_min": -27.826080322265625,
      "activations/layer22_attention_weight_max": 60.00453186035156,
      "activations/layer22_attention_weight_min": -38.88703155517578,
      "activations/layer23_attention_weight_max": 41.922950744628906,
      "activations/layer23_attention_weight_min": -28.23353385925293,
      "activations/layer2_attention_weight_max": 36.58777618408203,
      "activations/layer2_attention_weight_min": -34.45100784301758,
      "activations/layer3_attention_weight_max": 115.71941375732422,
      "activations/layer3_attention_weight_min": -114.56707000732422,
      "activations/layer4_attention_weight_max": 122.28059387207031,
      "activations/layer4_attention_weight_min": -115.51158905029297,
      "activations/layer5_attention_weight_max": 58.38923263549805,
      "activations/layer5_attention_weight_min": -60.706695556640625,
      "activations/layer6_attention_weight_max": 49.486873626708984,
      "activations/layer6_attention_weight_min": -47.891483306884766,
      "activations/layer7_attention_weight_max": 113.37446594238281,
      "activations/layer7_attention_weight_min": -119.65168762207031,
      "activations/layer8_attention_weight_max": 55.905120849609375,
      "activations/layer8_attention_weight_min": -55.99708557128906,
      "activations/layer9_attention_weight_max": 44.0580940246582,
      "activations/layer9_attention_weight_min": -47.16092300415039,
      "epoch": 19.85,
      "learning_rate": 2.220530303030303e-05,
      "loss": 2.7137,
      "step": 341650
    },
    {
      "activations/layer0_attention_weight_max": 16.71147346496582,
      "activations/layer0_attention_weight_min": -13.037771224975586,
      "activations/layer10_attention_weight_max": 35.24787902832031,
      "activations/layer10_attention_weight_min": -36.232513427734375,
      "activations/layer11_attention_weight_max": 36.6607551574707,
      "activations/layer11_attention_weight_min": -37.23078155517578,
      "activations/layer12_attention_weight_max": 28.929237365722656,
      "activations/layer12_attention_weight_min": -28.219179153442383,
      "activations/layer13_attention_weight_max": 43.07731628417969,
      "activations/layer13_attention_weight_min": -37.39381790161133,
      "activations/layer14_attention_weight_max": 44.21487045288086,
      "activations/layer14_attention_weight_min": -42.136234283447266,
      "activations/layer15_attention_weight_max": 44.13030242919922,
      "activations/layer15_attention_weight_min": -38.39128875732422,
      "activations/layer16_attention_weight_max": 31.263195037841797,
      "activations/layer16_attention_weight_min": -33.49467849731445,
      "activations/layer17_attention_weight_max": 34.64631652832031,
      "activations/layer17_attention_weight_min": -29.938623428344727,
      "activations/layer18_attention_weight_max": 35.550506591796875,
      "activations/layer18_attention_weight_min": -29.411710739135742,
      "activations/layer19_attention_weight_max": 35.86481475830078,
      "activations/layer19_attention_weight_min": -33.690216064453125,
      "activations/layer1_attention_weight_max": 16.57364845275879,
      "activations/layer1_attention_weight_min": -14.04230785369873,
      "activations/layer20_attention_weight_max": 31.081525802612305,
      "activations/layer20_attention_weight_min": -28.738048553466797,
      "activations/layer21_attention_weight_max": 30.7916316986084,
      "activations/layer21_attention_weight_min": -27.90755271911621,
      "activations/layer22_attention_weight_max": 47.29704666137695,
      "activations/layer22_attention_weight_min": -38.81880187988281,
      "activations/layer23_attention_weight_max": 35.20415496826172,
      "activations/layer23_attention_weight_min": -28.504650115966797,
      "activations/layer2_attention_weight_max": 31.61504364013672,
      "activations/layer2_attention_weight_min": -32.0413932800293,
      "activations/layer3_attention_weight_max": 100.97962951660156,
      "activations/layer3_attention_weight_min": -102.89081573486328,
      "activations/layer4_attention_weight_max": 105.69378662109375,
      "activations/layer4_attention_weight_min": -112.16817474365234,
      "activations/layer5_attention_weight_max": 50.01150131225586,
      "activations/layer5_attention_weight_min": -63.518821716308594,
      "activations/layer6_attention_weight_max": 44.645084381103516,
      "activations/layer6_attention_weight_min": -47.92512130737305,
      "activations/layer7_attention_weight_max": 95.03268432617188,
      "activations/layer7_attention_weight_min": -104.84356689453125,
      "activations/layer8_attention_weight_max": 42.23091506958008,
      "activations/layer8_attention_weight_min": -47.22145462036133,
      "activations/layer9_attention_weight_max": 32.8968620300293,
      "activations/layer9_attention_weight_min": -35.659236907958984,
      "epoch": 19.85,
      "learning_rate": 2.2186363636363633e-05,
      "loss": 2.7305,
      "step": 341700
    },
    {
      "activations/layer0_attention_weight_max": 15.741689682006836,
      "activations/layer0_attention_weight_min": -12.091741561889648,
      "activations/layer10_attention_weight_max": 32.92625427246094,
      "activations/layer10_attention_weight_min": -33.98883819580078,
      "activations/layer11_attention_weight_max": 32.45050811767578,
      "activations/layer11_attention_weight_min": -32.6735725402832,
      "activations/layer12_attention_weight_max": 27.430173873901367,
      "activations/layer12_attention_weight_min": -27.95884132385254,
      "activations/layer13_attention_weight_max": 39.587886810302734,
      "activations/layer13_attention_weight_min": -35.88771057128906,
      "activations/layer14_attention_weight_max": 40.3416633605957,
      "activations/layer14_attention_weight_min": -37.449623107910156,
      "activations/layer15_attention_weight_max": 37.18097686767578,
      "activations/layer15_attention_weight_min": -35.87348175048828,
      "activations/layer16_attention_weight_max": 28.23652458190918,
      "activations/layer16_attention_weight_min": -28.523731231689453,
      "activations/layer17_attention_weight_max": 30.195627212524414,
      "activations/layer17_attention_weight_min": -26.359920501708984,
      "activations/layer18_attention_weight_max": 31.126188278198242,
      "activations/layer18_attention_weight_min": -25.470863342285156,
      "activations/layer19_attention_weight_max": 34.83903121948242,
      "activations/layer19_attention_weight_min": -31.741535186767578,
      "activations/layer1_attention_weight_max": 16.733911514282227,
      "activations/layer1_attention_weight_min": -17.77096176147461,
      "activations/layer20_attention_weight_max": 29.65947723388672,
      "activations/layer20_attention_weight_min": -25.179397583007812,
      "activations/layer21_attention_weight_max": 30.019222259521484,
      "activations/layer21_attention_weight_min": -25.935409545898438,
      "activations/layer22_attention_weight_max": 41.539798736572266,
      "activations/layer22_attention_weight_min": -31.54084587097168,
      "activations/layer23_attention_weight_max": 31.359424591064453,
      "activations/layer23_attention_weight_min": -26.04290008544922,
      "activations/layer2_attention_weight_max": 30.914775848388672,
      "activations/layer2_attention_weight_min": -31.03466033935547,
      "activations/layer3_attention_weight_max": 97.3536148071289,
      "activations/layer3_attention_weight_min": -98.94985961914062,
      "activations/layer4_attention_weight_max": 107.47613525390625,
      "activations/layer4_attention_weight_min": -111.1121597290039,
      "activations/layer5_attention_weight_max": 48.84870147705078,
      "activations/layer5_attention_weight_min": -67.52920532226562,
      "activations/layer6_attention_weight_max": 44.99170684814453,
      "activations/layer6_attention_weight_min": -49.272254943847656,
      "activations/layer7_attention_weight_max": 93.54942321777344,
      "activations/layer7_attention_weight_min": -97.67874145507812,
      "activations/layer8_attention_weight_max": 42.72896957397461,
      "activations/layer8_attention_weight_min": -43.70127487182617,
      "activations/layer9_attention_weight_max": 31.64723014831543,
      "activations/layer9_attention_weight_min": -33.526798248291016,
      "epoch": 19.86,
      "learning_rate": 2.216742424242424e-05,
      "loss": 2.7148,
      "step": 341750
    },
    {
      "activations/layer0_attention_weight_max": 17.452749252319336,
      "activations/layer0_attention_weight_min": -13.096097946166992,
      "activations/layer10_attention_weight_max": 32.709007263183594,
      "activations/layer10_attention_weight_min": -34.51008605957031,
      "activations/layer11_attention_weight_max": 32.78997039794922,
      "activations/layer11_attention_weight_min": -35.36925506591797,
      "activations/layer12_attention_weight_max": 23.744916915893555,
      "activations/layer12_attention_weight_min": -26.06736946105957,
      "activations/layer13_attention_weight_max": 36.357723236083984,
      "activations/layer13_attention_weight_min": -33.633235931396484,
      "activations/layer14_attention_weight_max": 38.443424224853516,
      "activations/layer14_attention_weight_min": -37.49436950683594,
      "activations/layer15_attention_weight_max": 35.053524017333984,
      "activations/layer15_attention_weight_min": -33.86795425415039,
      "activations/layer16_attention_weight_max": 26.7216796875,
      "activations/layer16_attention_weight_min": -28.17893409729004,
      "activations/layer17_attention_weight_max": 30.30646514892578,
      "activations/layer17_attention_weight_min": -25.028976440429688,
      "activations/layer18_attention_weight_max": 30.038864135742188,
      "activations/layer18_attention_weight_min": -25.656070709228516,
      "activations/layer19_attention_weight_max": 30.673425674438477,
      "activations/layer19_attention_weight_min": -30.590900421142578,
      "activations/layer1_attention_weight_max": 16.434043884277344,
      "activations/layer1_attention_weight_min": -15.122319221496582,
      "activations/layer20_attention_weight_max": 26.566871643066406,
      "activations/layer20_attention_weight_min": -25.1626033782959,
      "activations/layer21_attention_weight_max": 26.537689208984375,
      "activations/layer21_attention_weight_min": -24.700719833374023,
      "activations/layer22_attention_weight_max": 38.91547393798828,
      "activations/layer22_attention_weight_min": -32.642818450927734,
      "activations/layer23_attention_weight_max": 31.067665100097656,
      "activations/layer23_attention_weight_min": -27.144073486328125,
      "activations/layer2_attention_weight_max": 31.494426727294922,
      "activations/layer2_attention_weight_min": -31.738309860229492,
      "activations/layer3_attention_weight_max": 99.25763702392578,
      "activations/layer3_attention_weight_min": -100.86546325683594,
      "activations/layer4_attention_weight_max": 103.32154846191406,
      "activations/layer4_attention_weight_min": -108.43453216552734,
      "activations/layer5_attention_weight_max": 50.72080993652344,
      "activations/layer5_attention_weight_min": -65.51264953613281,
      "activations/layer6_attention_weight_max": 44.412254333496094,
      "activations/layer6_attention_weight_min": -49.7315788269043,
      "activations/layer7_attention_weight_max": 93.35382843017578,
      "activations/layer7_attention_weight_min": -99.16634368896484,
      "activations/layer8_attention_weight_max": 40.0555534362793,
      "activations/layer8_attention_weight_min": -44.47172927856445,
      "activations/layer9_attention_weight_max": 31.12076187133789,
      "activations/layer9_attention_weight_min": -35.53070068359375,
      "epoch": 19.86,
      "learning_rate": 2.2148484848484846e-05,
      "loss": 2.7001,
      "step": 341800
    },
    {
      "activations/layer0_attention_weight_max": 16.107271194458008,
      "activations/layer0_attention_weight_min": -12.892427444458008,
      "activations/layer10_attention_weight_max": 34.213783264160156,
      "activations/layer10_attention_weight_min": -34.194766998291016,
      "activations/layer11_attention_weight_max": 34.2579460144043,
      "activations/layer11_attention_weight_min": -36.174415588378906,
      "activations/layer12_attention_weight_max": 29.439393997192383,
      "activations/layer12_attention_weight_min": -28.614852905273438,
      "activations/layer13_attention_weight_max": 40.978660583496094,
      "activations/layer13_attention_weight_min": -37.7503662109375,
      "activations/layer14_attention_weight_max": 45.87971115112305,
      "activations/layer14_attention_weight_min": -39.83390426635742,
      "activations/layer15_attention_weight_max": 39.45939254760742,
      "activations/layer15_attention_weight_min": -35.83467102050781,
      "activations/layer16_attention_weight_max": 29.77580451965332,
      "activations/layer16_attention_weight_min": -27.225839614868164,
      "activations/layer17_attention_weight_max": 29.857492446899414,
      "activations/layer17_attention_weight_min": -26.272993087768555,
      "activations/layer18_attention_weight_max": 33.335567474365234,
      "activations/layer18_attention_weight_min": -24.647624969482422,
      "activations/layer19_attention_weight_max": 33.61516189575195,
      "activations/layer19_attention_weight_min": -34.88950729370117,
      "activations/layer1_attention_weight_max": 16.009023666381836,
      "activations/layer1_attention_weight_min": -13.812935829162598,
      "activations/layer20_attention_weight_max": 29.17730712890625,
      "activations/layer20_attention_weight_min": -26.488685607910156,
      "activations/layer21_attention_weight_max": 30.28621482849121,
      "activations/layer21_attention_weight_min": -25.63275718688965,
      "activations/layer22_attention_weight_max": 43.1905632019043,
      "activations/layer22_attention_weight_min": -31.791990280151367,
      "activations/layer23_attention_weight_max": 34.968162536621094,
      "activations/layer23_attention_weight_min": -26.713565826416016,
      "activations/layer2_attention_weight_max": 31.04155731201172,
      "activations/layer2_attention_weight_min": -31.70557403564453,
      "activations/layer3_attention_weight_max": 97.34241485595703,
      "activations/layer3_attention_weight_min": -99.24728393554688,
      "activations/layer4_attention_weight_max": 102.29122161865234,
      "activations/layer4_attention_weight_min": -108.11016845703125,
      "activations/layer5_attention_weight_max": 49.79869842529297,
      "activations/layer5_attention_weight_min": -59.59466552734375,
      "activations/layer6_attention_weight_max": 46.02127456665039,
      "activations/layer6_attention_weight_min": -48.21355438232422,
      "activations/layer7_attention_weight_max": 94.64569091796875,
      "activations/layer7_attention_weight_min": -102.75840759277344,
      "activations/layer8_attention_weight_max": 41.318233489990234,
      "activations/layer8_attention_weight_min": -47.35199737548828,
      "activations/layer9_attention_weight_max": 33.50749969482422,
      "activations/layer9_attention_weight_min": -34.75010681152344,
      "epoch": 19.86,
      "learning_rate": 2.2129545454545454e-05,
      "loss": 2.7427,
      "step": 341850
    },
    {
      "activations/layer0_attention_weight_max": 17.23097038269043,
      "activations/layer0_attention_weight_min": -13.270242691040039,
      "activations/layer10_attention_weight_max": 35.055118560791016,
      "activations/layer10_attention_weight_min": -35.905418395996094,
      "activations/layer11_attention_weight_max": 35.0760612487793,
      "activations/layer11_attention_weight_min": -36.5022087097168,
      "activations/layer12_attention_weight_max": 27.52861213684082,
      "activations/layer12_attention_weight_min": -26.278963088989258,
      "activations/layer13_attention_weight_max": 40.1533317565918,
      "activations/layer13_attention_weight_min": -36.31108474731445,
      "activations/layer14_attention_weight_max": 41.778221130371094,
      "activations/layer14_attention_weight_min": -38.19234085083008,
      "activations/layer15_attention_weight_max": 38.1863899230957,
      "activations/layer15_attention_weight_min": -38.96197509765625,
      "activations/layer16_attention_weight_max": 28.750993728637695,
      "activations/layer16_attention_weight_min": -30.44908905029297,
      "activations/layer17_attention_weight_max": 30.42131233215332,
      "activations/layer17_attention_weight_min": -27.837732315063477,
      "activations/layer18_attention_weight_max": 32.85969161987305,
      "activations/layer18_attention_weight_min": -27.35565185546875,
      "activations/layer19_attention_weight_max": 33.48861312866211,
      "activations/layer19_attention_weight_min": -31.999677658081055,
      "activations/layer1_attention_weight_max": 15.526920318603516,
      "activations/layer1_attention_weight_min": -13.833033561706543,
      "activations/layer20_attention_weight_max": 30.057199478149414,
      "activations/layer20_attention_weight_min": -27.122955322265625,
      "activations/layer21_attention_weight_max": 29.744171142578125,
      "activations/layer21_attention_weight_min": -26.593441009521484,
      "activations/layer22_attention_weight_max": 38.824066162109375,
      "activations/layer22_attention_weight_min": -35.09822082519531,
      "activations/layer23_attention_weight_max": 30.091087341308594,
      "activations/layer23_attention_weight_min": -27.538070678710938,
      "activations/layer2_attention_weight_max": 30.471302032470703,
      "activations/layer2_attention_weight_min": -30.690542221069336,
      "activations/layer3_attention_weight_max": 100.08882904052734,
      "activations/layer3_attention_weight_min": -101.17322540283203,
      "activations/layer4_attention_weight_max": 108.13795471191406,
      "activations/layer4_attention_weight_min": -110.06639862060547,
      "activations/layer5_attention_weight_max": 50.275596618652344,
      "activations/layer5_attention_weight_min": -61.69074249267578,
      "activations/layer6_attention_weight_max": 45.03911209106445,
      "activations/layer6_attention_weight_min": -49.63545227050781,
      "activations/layer7_attention_weight_max": 96.62252807617188,
      "activations/layer7_attention_weight_min": -101.1545181274414,
      "activations/layer8_attention_weight_max": 42.906856536865234,
      "activations/layer8_attention_weight_min": -45.846412658691406,
      "activations/layer9_attention_weight_max": 33.24094772338867,
      "activations/layer9_attention_weight_min": -35.288230895996094,
      "epoch": 19.87,
      "learning_rate": 2.211060606060606e-05,
      "loss": 2.7247,
      "step": 341900
    },
    {
      "activations/layer0_attention_weight_max": 16.320859909057617,
      "activations/layer0_attention_weight_min": -12.666728019714355,
      "activations/layer10_attention_weight_max": 36.21331024169922,
      "activations/layer10_attention_weight_min": -36.1485595703125,
      "activations/layer11_attention_weight_max": 35.72538375854492,
      "activations/layer11_attention_weight_min": -36.45100784301758,
      "activations/layer12_attention_weight_max": 33.49378967285156,
      "activations/layer12_attention_weight_min": -28.049842834472656,
      "activations/layer13_attention_weight_max": 45.906715393066406,
      "activations/layer13_attention_weight_min": -38.652549743652344,
      "activations/layer14_attention_weight_max": 50.85744857788086,
      "activations/layer14_attention_weight_min": -41.05417251586914,
      "activations/layer15_attention_weight_max": 47.00271987915039,
      "activations/layer15_attention_weight_min": -37.743316650390625,
      "activations/layer16_attention_weight_max": 32.78395462036133,
      "activations/layer16_attention_weight_min": -29.780010223388672,
      "activations/layer17_attention_weight_max": 33.96907043457031,
      "activations/layer17_attention_weight_min": -26.51990509033203,
      "activations/layer18_attention_weight_max": 33.08416748046875,
      "activations/layer18_attention_weight_min": -26.536937713623047,
      "activations/layer19_attention_weight_max": 42.02322006225586,
      "activations/layer19_attention_weight_min": -34.0468635559082,
      "activations/layer1_attention_weight_max": 15.435980796813965,
      "activations/layer1_attention_weight_min": -14.027970314025879,
      "activations/layer20_attention_weight_max": 34.505733489990234,
      "activations/layer20_attention_weight_min": -27.5291805267334,
      "activations/layer21_attention_weight_max": 32.8837776184082,
      "activations/layer21_attention_weight_min": -26.398773193359375,
      "activations/layer22_attention_weight_max": 47.80073928833008,
      "activations/layer22_attention_weight_min": -33.07950210571289,
      "activations/layer23_attention_weight_max": 40.36467742919922,
      "activations/layer23_attention_weight_min": -30.451780319213867,
      "activations/layer2_attention_weight_max": 32.112640380859375,
      "activations/layer2_attention_weight_min": -31.998111724853516,
      "activations/layer3_attention_weight_max": 101.12659454345703,
      "activations/layer3_attention_weight_min": -99.29792022705078,
      "activations/layer4_attention_weight_max": 109.4090805053711,
      "activations/layer4_attention_weight_min": -107.29042053222656,
      "activations/layer5_attention_weight_max": 50.474571228027344,
      "activations/layer5_attention_weight_min": -61.724422454833984,
      "activations/layer6_attention_weight_max": 47.48357009887695,
      "activations/layer6_attention_weight_min": -46.7017936706543,
      "activations/layer7_attention_weight_max": 103.56103515625,
      "activations/layer7_attention_weight_min": -100.12820434570312,
      "activations/layer8_attention_weight_max": 45.0585823059082,
      "activations/layer8_attention_weight_min": -45.53246307373047,
      "activations/layer9_attention_weight_max": 34.80263900756836,
      "activations/layer9_attention_weight_min": -36.295684814453125,
      "epoch": 19.87,
      "learning_rate": 2.2091666666666667e-05,
      "loss": 2.7207,
      "step": 341950
    },
    {
      "activations/layer0_attention_weight_max": 15.814498901367188,
      "activations/layer0_attention_weight_min": -12.494511604309082,
      "activations/layer10_attention_weight_max": 34.35289764404297,
      "activations/layer10_attention_weight_min": -35.17478561401367,
      "activations/layer11_attention_weight_max": 32.80895233154297,
      "activations/layer11_attention_weight_min": -33.85409164428711,
      "activations/layer12_attention_weight_max": 26.537172317504883,
      "activations/layer12_attention_weight_min": -26.535402297973633,
      "activations/layer13_attention_weight_max": 38.68403625488281,
      "activations/layer13_attention_weight_min": -36.272613525390625,
      "activations/layer14_attention_weight_max": 42.0639762878418,
      "activations/layer14_attention_weight_min": -39.15851974487305,
      "activations/layer15_attention_weight_max": 38.96245193481445,
      "activations/layer15_attention_weight_min": -36.949764251708984,
      "activations/layer16_attention_weight_max": 29.91594886779785,
      "activations/layer16_attention_weight_min": -28.58985710144043,
      "activations/layer17_attention_weight_max": 31.301313400268555,
      "activations/layer17_attention_weight_min": -26.475595474243164,
      "activations/layer18_attention_weight_max": 31.99996566772461,
      "activations/layer18_attention_weight_min": -25.58016014099121,
      "activations/layer19_attention_weight_max": 34.97733688354492,
      "activations/layer19_attention_weight_min": -31.980260848999023,
      "activations/layer1_attention_weight_max": 15.998361587524414,
      "activations/layer1_attention_weight_min": -14.576606750488281,
      "activations/layer20_attention_weight_max": 29.454593658447266,
      "activations/layer20_attention_weight_min": -24.490703582763672,
      "activations/layer21_attention_weight_max": 30.62548065185547,
      "activations/layer21_attention_weight_min": -24.152263641357422,
      "activations/layer22_attention_weight_max": 45.73667526245117,
      "activations/layer22_attention_weight_min": -34.20621109008789,
      "activations/layer23_attention_weight_max": 36.054168701171875,
      "activations/layer23_attention_weight_min": -24.576536178588867,
      "activations/layer2_attention_weight_max": 30.310226440429688,
      "activations/layer2_attention_weight_min": -30.69655418395996,
      "activations/layer3_attention_weight_max": 98.28919982910156,
      "activations/layer3_attention_weight_min": -99.9209213256836,
      "activations/layer4_attention_weight_max": 105.4634017944336,
      "activations/layer4_attention_weight_min": -110.29627990722656,
      "activations/layer5_attention_weight_max": 48.72367858886719,
      "activations/layer5_attention_weight_min": -60.41343688964844,
      "activations/layer6_attention_weight_max": 44.40226745605469,
      "activations/layer6_attention_weight_min": -47.637062072753906,
      "activations/layer7_attention_weight_max": 91.56586456298828,
      "activations/layer7_attention_weight_min": -98.19062042236328,
      "activations/layer8_attention_weight_max": 40.850013732910156,
      "activations/layer8_attention_weight_min": -44.54538345336914,
      "activations/layer9_attention_weight_max": 31.820770263671875,
      "activations/layer9_attention_weight_min": -33.6967887878418,
      "epoch": 19.87,
      "learning_rate": 2.207272727272727e-05,
      "loss": 2.7361,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.4736,
      "eval_samples_per_second": 506.75,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.4736,
      "eval_openwebtext_samples_per_second": 506.75,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9442,
      "eval_wikitext_samples_per_second": 234.548,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_lambada_loss": 2.376953125,
      "eval_lambada_ppl": 10.77203177799245,
      "eval_lambada_runtime": 9.5354,
      "eval_lambada_samples_per_second": 510.624,
      "step": 342000
    },
    {
      "activations/layer0_attention_weight_max": 15.644993782043457,
      "activations/layer0_attention_weight_min": -13.083954811096191,
      "activations/layer10_attention_weight_max": 32.33319854736328,
      "activations/layer10_attention_weight_min": -34.66786575317383,
      "activations/layer11_attention_weight_max": 34.51742935180664,
      "activations/layer11_attention_weight_min": -35.70042419433594,
      "activations/layer12_attention_weight_max": 28.283227920532227,
      "activations/layer12_attention_weight_min": -26.07871437072754,
      "activations/layer13_attention_weight_max": 39.70342254638672,
      "activations/layer13_attention_weight_min": -34.832679748535156,
      "activations/layer14_attention_weight_max": 42.101470947265625,
      "activations/layer14_attention_weight_min": -35.40544509887695,
      "activations/layer15_attention_weight_max": 39.52423095703125,
      "activations/layer15_attention_weight_min": -36.35865783691406,
      "activations/layer16_attention_weight_max": 29.376222610473633,
      "activations/layer16_attention_weight_min": -28.47439193725586,
      "activations/layer17_attention_weight_max": 30.183284759521484,
      "activations/layer17_attention_weight_min": -25.201868057250977,
      "activations/layer18_attention_weight_max": 32.31532669067383,
      "activations/layer18_attention_weight_min": -25.844274520874023,
      "activations/layer19_attention_weight_max": 34.08454513549805,
      "activations/layer19_attention_weight_min": -31.089887619018555,
      "activations/layer1_attention_weight_max": 16.440853118896484,
      "activations/layer1_attention_weight_min": -14.504956245422363,
      "activations/layer20_attention_weight_max": 30.031404495239258,
      "activations/layer20_attention_weight_min": -26.800731658935547,
      "activations/layer21_attention_weight_max": 28.29513168334961,
      "activations/layer21_attention_weight_min": -23.888458251953125,
      "activations/layer22_attention_weight_max": 41.56248092651367,
      "activations/layer22_attention_weight_min": -31.23417091369629,
      "activations/layer23_attention_weight_max": 31.17559814453125,
      "activations/layer23_attention_weight_min": -26.080936431884766,
      "activations/layer2_attention_weight_max": 30.938899993896484,
      "activations/layer2_attention_weight_min": -31.250812530517578,
      "activations/layer3_attention_weight_max": 95.55194854736328,
      "activations/layer3_attention_weight_min": -99.9235610961914,
      "activations/layer4_attention_weight_max": 106.32582092285156,
      "activations/layer4_attention_weight_min": -112.2789077758789,
      "activations/layer5_attention_weight_max": 48.24848937988281,
      "activations/layer5_attention_weight_min": -59.828338623046875,
      "activations/layer6_attention_weight_max": 41.862911224365234,
      "activations/layer6_attention_weight_min": -47.206417083740234,
      "activations/layer7_attention_weight_max": 91.92182159423828,
      "activations/layer7_attention_weight_min": -93.42495727539062,
      "activations/layer8_attention_weight_max": 41.4835319519043,
      "activations/layer8_attention_weight_min": -43.28569793701172,
      "activations/layer9_attention_weight_max": 31.629215240478516,
      "activations/layer9_attention_weight_min": -34.0221061706543,
      "epoch": 19.88,
      "learning_rate": 2.2053787878787877e-05,
      "loss": 2.7117,
      "step": 342050
    },
    {
      "activations/layer0_attention_weight_max": 17.252017974853516,
      "activations/layer0_attention_weight_min": -12.894049644470215,
      "activations/layer10_attention_weight_max": 38.82401657104492,
      "activations/layer10_attention_weight_min": -35.99820327758789,
      "activations/layer11_attention_weight_max": 38.32463455200195,
      "activations/layer11_attention_weight_min": -34.54726791381836,
      "activations/layer12_attention_weight_max": 29.42058563232422,
      "activations/layer12_attention_weight_min": -26.974475860595703,
      "activations/layer13_attention_weight_max": 46.90849304199219,
      "activations/layer13_attention_weight_min": -36.21488571166992,
      "activations/layer14_attention_weight_max": 47.88608169555664,
      "activations/layer14_attention_weight_min": -39.43574523925781,
      "activations/layer15_attention_weight_max": 43.09317398071289,
      "activations/layer15_attention_weight_min": -37.400474548339844,
      "activations/layer16_attention_weight_max": 30.9180908203125,
      "activations/layer16_attention_weight_min": -28.786752700805664,
      "activations/layer17_attention_weight_max": 31.771419525146484,
      "activations/layer17_attention_weight_min": -26.969892501831055,
      "activations/layer18_attention_weight_max": 30.388530731201172,
      "activations/layer18_attention_weight_min": -28.564376831054688,
      "activations/layer19_attention_weight_max": 33.240272521972656,
      "activations/layer19_attention_weight_min": -31.47938346862793,
      "activations/layer1_attention_weight_max": 16.19352149963379,
      "activations/layer1_attention_weight_min": -14.974806785583496,
      "activations/layer20_attention_weight_max": 28.322099685668945,
      "activations/layer20_attention_weight_min": -26.331880569458008,
      "activations/layer21_attention_weight_max": 26.544530868530273,
      "activations/layer21_attention_weight_min": -25.18051528930664,
      "activations/layer22_attention_weight_max": 40.96897506713867,
      "activations/layer22_attention_weight_min": -32.50944900512695,
      "activations/layer23_attention_weight_max": 30.374120712280273,
      "activations/layer23_attention_weight_min": -25.183544158935547,
      "activations/layer2_attention_weight_max": 33.30750274658203,
      "activations/layer2_attention_weight_min": -29.930767059326172,
      "activations/layer3_attention_weight_max": 102.3171157836914,
      "activations/layer3_attention_weight_min": -100.59330749511719,
      "activations/layer4_attention_weight_max": 113.25533294677734,
      "activations/layer4_attention_weight_min": -108.1727294921875,
      "activations/layer5_attention_weight_max": 51.1009521484375,
      "activations/layer5_attention_weight_min": -64.3165283203125,
      "activations/layer6_attention_weight_max": 46.52267074584961,
      "activations/layer6_attention_weight_min": -47.60489273071289,
      "activations/layer7_attention_weight_max": 101.10477447509766,
      "activations/layer7_attention_weight_min": -95.88797760009766,
      "activations/layer8_attention_weight_max": 41.998477935791016,
      "activations/layer8_attention_weight_min": -45.94544219970703,
      "activations/layer9_attention_weight_max": 32.206661224365234,
      "activations/layer9_attention_weight_min": -35.47866439819336,
      "epoch": 19.88,
      "learning_rate": 2.2034848484848482e-05,
      "loss": 2.7243,
      "step": 342100
    },
    {
      "activations/layer0_attention_weight_max": 15.885846138000488,
      "activations/layer0_attention_weight_min": -12.041021347045898,
      "activations/layer10_attention_weight_max": 32.384376525878906,
      "activations/layer10_attention_weight_min": -33.350433349609375,
      "activations/layer11_attention_weight_max": 33.31639862060547,
      "activations/layer11_attention_weight_min": -32.12474822998047,
      "activations/layer12_attention_weight_max": 28.777870178222656,
      "activations/layer12_attention_weight_min": -27.55931282043457,
      "activations/layer13_attention_weight_max": 42.0061149597168,
      "activations/layer13_attention_weight_min": -37.36289596557617,
      "activations/layer14_attention_weight_max": 42.56081008911133,
      "activations/layer14_attention_weight_min": -39.09660339355469,
      "activations/layer15_attention_weight_max": 39.86982727050781,
      "activations/layer15_attention_weight_min": -36.54839324951172,
      "activations/layer16_attention_weight_max": 29.837305068969727,
      "activations/layer16_attention_weight_min": -28.394025802612305,
      "activations/layer17_attention_weight_max": 27.85462760925293,
      "activations/layer17_attention_weight_min": -26.39751625061035,
      "activations/layer18_attention_weight_max": 32.36610794067383,
      "activations/layer18_attention_weight_min": -25.821224212646484,
      "activations/layer19_attention_weight_max": 33.22410202026367,
      "activations/layer19_attention_weight_min": -31.071760177612305,
      "activations/layer1_attention_weight_max": 16.337181091308594,
      "activations/layer1_attention_weight_min": -13.95862865447998,
      "activations/layer20_attention_weight_max": 27.196014404296875,
      "activations/layer20_attention_weight_min": -25.00275230407715,
      "activations/layer21_attention_weight_max": 27.397518157958984,
      "activations/layer21_attention_weight_min": -23.88353157043457,
      "activations/layer22_attention_weight_max": 46.5401496887207,
      "activations/layer22_attention_weight_min": -31.382980346679688,
      "activations/layer23_attention_weight_max": 32.45303726196289,
      "activations/layer23_attention_weight_min": -25.26582145690918,
      "activations/layer2_attention_weight_max": 31.20178985595703,
      "activations/layer2_attention_weight_min": -31.733139038085938,
      "activations/layer3_attention_weight_max": 98.74923706054688,
      "activations/layer3_attention_weight_min": -99.37677001953125,
      "activations/layer4_attention_weight_max": 107.2825698852539,
      "activations/layer4_attention_weight_min": -104.7589111328125,
      "activations/layer5_attention_weight_max": 49.147613525390625,
      "activations/layer5_attention_weight_min": -61.71295166015625,
      "activations/layer6_attention_weight_max": 44.244239807128906,
      "activations/layer6_attention_weight_min": -46.96928024291992,
      "activations/layer7_attention_weight_max": 97.52330017089844,
      "activations/layer7_attention_weight_min": -95.82176971435547,
      "activations/layer8_attention_weight_max": 41.19768142700195,
      "activations/layer8_attention_weight_min": -44.52498245239258,
      "activations/layer9_attention_weight_max": 31.669727325439453,
      "activations/layer9_attention_weight_min": -33.42810821533203,
      "epoch": 19.88,
      "learning_rate": 2.201590909090909e-05,
      "loss": 2.7254,
      "step": 342150
    },
    {
      "activations/layer0_attention_weight_max": 16.77585220336914,
      "activations/layer0_attention_weight_min": -13.895831108093262,
      "activations/layer10_attention_weight_max": 36.08412170410156,
      "activations/layer10_attention_weight_min": -36.6624755859375,
      "activations/layer11_attention_weight_max": 35.38707733154297,
      "activations/layer11_attention_weight_min": -37.100067138671875,
      "activations/layer12_attention_weight_max": 29.501319885253906,
      "activations/layer12_attention_weight_min": -29.67437171936035,
      "activations/layer13_attention_weight_max": 39.86732482910156,
      "activations/layer13_attention_weight_min": -34.4908561706543,
      "activations/layer14_attention_weight_max": 43.98845291137695,
      "activations/layer14_attention_weight_min": -36.99043655395508,
      "activations/layer15_attention_weight_max": 37.75821304321289,
      "activations/layer15_attention_weight_min": -34.32609558105469,
      "activations/layer16_attention_weight_max": 29.52857208251953,
      "activations/layer16_attention_weight_min": -29.456480026245117,
      "activations/layer17_attention_weight_max": 28.806055068969727,
      "activations/layer17_attention_weight_min": -25.30344581604004,
      "activations/layer18_attention_weight_max": 33.542083740234375,
      "activations/layer18_attention_weight_min": -24.64370346069336,
      "activations/layer19_attention_weight_max": 31.077451705932617,
      "activations/layer19_attention_weight_min": -31.273969650268555,
      "activations/layer1_attention_weight_max": 15.111214637756348,
      "activations/layer1_attention_weight_min": -14.111656188964844,
      "activations/layer20_attention_weight_max": 27.12877082824707,
      "activations/layer20_attention_weight_min": -25.87592124938965,
      "activations/layer21_attention_weight_max": 28.012004852294922,
      "activations/layer21_attention_weight_min": -25.73748016357422,
      "activations/layer22_attention_weight_max": 45.34071350097656,
      "activations/layer22_attention_weight_min": -31.439496994018555,
      "activations/layer23_attention_weight_max": 30.91040802001953,
      "activations/layer23_attention_weight_min": -26.217376708984375,
      "activations/layer2_attention_weight_max": 30.370288848876953,
      "activations/layer2_attention_weight_min": -30.23018455505371,
      "activations/layer3_attention_weight_max": 98.94200897216797,
      "activations/layer3_attention_weight_min": -99.58262634277344,
      "activations/layer4_attention_weight_max": 105.5015640258789,
      "activations/layer4_attention_weight_min": -113.09742736816406,
      "activations/layer5_attention_weight_max": 50.064186096191406,
      "activations/layer5_attention_weight_min": -65.46045684814453,
      "activations/layer6_attention_weight_max": 44.2946662902832,
      "activations/layer6_attention_weight_min": -48.84249496459961,
      "activations/layer7_attention_weight_max": 96.33718872070312,
      "activations/layer7_attention_weight_min": -100.4281005859375,
      "activations/layer8_attention_weight_max": 42.99661636352539,
      "activations/layer8_attention_weight_min": -44.530391693115234,
      "activations/layer9_attention_weight_max": 32.864784240722656,
      "activations/layer9_attention_weight_min": -35.463199615478516,
      "epoch": 19.88,
      "learning_rate": 2.1996969696969695e-05,
      "loss": 2.7033,
      "step": 342200
    },
    {
      "activations/layer0_attention_weight_max": 16.146484375,
      "activations/layer0_attention_weight_min": -11.901934623718262,
      "activations/layer10_attention_weight_max": 37.133583068847656,
      "activations/layer10_attention_weight_min": -39.40372848510742,
      "activations/layer11_attention_weight_max": 35.13034439086914,
      "activations/layer11_attention_weight_min": -37.917354583740234,
      "activations/layer12_attention_weight_max": 30.421201705932617,
      "activations/layer12_attention_weight_min": -27.904199600219727,
      "activations/layer13_attention_weight_max": 44.734169006347656,
      "activations/layer13_attention_weight_min": -38.181175231933594,
      "activations/layer14_attention_weight_max": 46.18804931640625,
      "activations/layer14_attention_weight_min": -40.08750534057617,
      "activations/layer15_attention_weight_max": 41.118465423583984,
      "activations/layer15_attention_weight_min": -37.804141998291016,
      "activations/layer16_attention_weight_max": 31.573331832885742,
      "activations/layer16_attention_weight_min": -28.860490798950195,
      "activations/layer17_attention_weight_max": 32.960121154785156,
      "activations/layer17_attention_weight_min": -25.829391479492188,
      "activations/layer18_attention_weight_max": 31.776811599731445,
      "activations/layer18_attention_weight_min": -27.192440032958984,
      "activations/layer19_attention_weight_max": 37.816139221191406,
      "activations/layer19_attention_weight_min": -34.65995407104492,
      "activations/layer1_attention_weight_max": 16.097421646118164,
      "activations/layer1_attention_weight_min": -16.86517906188965,
      "activations/layer20_attention_weight_max": 33.76127624511719,
      "activations/layer20_attention_weight_min": -25.702428817749023,
      "activations/layer21_attention_weight_max": 31.51910400390625,
      "activations/layer21_attention_weight_min": -26.279373168945312,
      "activations/layer22_attention_weight_max": 45.0657844543457,
      "activations/layer22_attention_weight_min": -35.05665588378906,
      "activations/layer23_attention_weight_max": 34.180198669433594,
      "activations/layer23_attention_weight_min": -30.256147384643555,
      "activations/layer2_attention_weight_max": 34.118045806884766,
      "activations/layer2_attention_weight_min": -34.70865249633789,
      "activations/layer3_attention_weight_max": 104.7005615234375,
      "activations/layer3_attention_weight_min": -106.46855163574219,
      "activations/layer4_attention_weight_max": 113.181884765625,
      "activations/layer4_attention_weight_min": -116.38106536865234,
      "activations/layer5_attention_weight_max": 52.197444915771484,
      "activations/layer5_attention_weight_min": -62.65119171142578,
      "activations/layer6_attention_weight_max": 45.97998809814453,
      "activations/layer6_attention_weight_min": -48.974178314208984,
      "activations/layer7_attention_weight_max": 97.78182983398438,
      "activations/layer7_attention_weight_min": -103.50669860839844,
      "activations/layer8_attention_weight_max": 44.78649139404297,
      "activations/layer8_attention_weight_min": -46.274112701416016,
      "activations/layer9_attention_weight_max": 33.85796356201172,
      "activations/layer9_attention_weight_min": -36.94672775268555,
      "epoch": 19.89,
      "learning_rate": 2.19780303030303e-05,
      "loss": 2.7209,
      "step": 342250
    },
    {
      "activations/layer0_attention_weight_max": 16.487709045410156,
      "activations/layer0_attention_weight_min": -11.665305137634277,
      "activations/layer10_attention_weight_max": 35.18072509765625,
      "activations/layer10_attention_weight_min": -35.06364822387695,
      "activations/layer11_attention_weight_max": 35.627071380615234,
      "activations/layer11_attention_weight_min": -34.59139633178711,
      "activations/layer12_attention_weight_max": 25.29987907409668,
      "activations/layer12_attention_weight_min": -25.330652236938477,
      "activations/layer13_attention_weight_max": 40.3291015625,
      "activations/layer13_attention_weight_min": -35.43301773071289,
      "activations/layer14_attention_weight_max": 43.617950439453125,
      "activations/layer14_attention_weight_min": -38.66425704956055,
      "activations/layer15_attention_weight_max": 38.2142219543457,
      "activations/layer15_attention_weight_min": -36.1920051574707,
      "activations/layer16_attention_weight_max": 28.601259231567383,
      "activations/layer16_attention_weight_min": -28.3560848236084,
      "activations/layer17_attention_weight_max": 28.54417610168457,
      "activations/layer17_attention_weight_min": -26.37679672241211,
      "activations/layer18_attention_weight_max": 33.43614959716797,
      "activations/layer18_attention_weight_min": -26.075641632080078,
      "activations/layer19_attention_weight_max": 33.50029754638672,
      "activations/layer19_attention_weight_min": -30.8444881439209,
      "activations/layer1_attention_weight_max": 15.172123908996582,
      "activations/layer1_attention_weight_min": -14.760567665100098,
      "activations/layer20_attention_weight_max": 29.885480880737305,
      "activations/layer20_attention_weight_min": -25.219615936279297,
      "activations/layer21_attention_weight_max": 28.333599090576172,
      "activations/layer21_attention_weight_min": -23.54038429260254,
      "activations/layer22_attention_weight_max": 38.157562255859375,
      "activations/layer22_attention_weight_min": -31.993486404418945,
      "activations/layer23_attention_weight_max": 31.973831176757812,
      "activations/layer23_attention_weight_min": -24.945173263549805,
      "activations/layer2_attention_weight_max": 32.65816879272461,
      "activations/layer2_attention_weight_min": -30.750152587890625,
      "activations/layer3_attention_weight_max": 102.29747009277344,
      "activations/layer3_attention_weight_min": -100.22885131835938,
      "activations/layer4_attention_weight_max": 111.71620178222656,
      "activations/layer4_attention_weight_min": -111.99298095703125,
      "activations/layer5_attention_weight_max": 50.501319885253906,
      "activations/layer5_attention_weight_min": -60.945411682128906,
      "activations/layer6_attention_weight_max": 45.40934753417969,
      "activations/layer6_attention_weight_min": -47.538028717041016,
      "activations/layer7_attention_weight_max": 95.32144165039062,
      "activations/layer7_attention_weight_min": -94.84839630126953,
      "activations/layer8_attention_weight_max": 43.667327880859375,
      "activations/layer8_attention_weight_min": -44.23491668701172,
      "activations/layer9_attention_weight_max": 34.11288070678711,
      "activations/layer9_attention_weight_min": -35.559471130371094,
      "epoch": 19.89,
      "learning_rate": 2.1959090909090905e-05,
      "loss": 2.7243,
      "step": 342300
    },
    {
      "activations/layer0_attention_weight_max": 15.607635498046875,
      "activations/layer0_attention_weight_min": -13.774103164672852,
      "activations/layer10_attention_weight_max": 33.329490661621094,
      "activations/layer10_attention_weight_min": -34.57960891723633,
      "activations/layer11_attention_weight_max": 33.959678649902344,
      "activations/layer11_attention_weight_min": -35.15572738647461,
      "activations/layer12_attention_weight_max": 26.43085479736328,
      "activations/layer12_attention_weight_min": -26.29757308959961,
      "activations/layer13_attention_weight_max": 38.26167678833008,
      "activations/layer13_attention_weight_min": -34.929988861083984,
      "activations/layer14_attention_weight_max": 41.096317291259766,
      "activations/layer14_attention_weight_min": -36.153717041015625,
      "activations/layer15_attention_weight_max": 37.04013442993164,
      "activations/layer15_attention_weight_min": -34.62594985961914,
      "activations/layer16_attention_weight_max": 28.482818603515625,
      "activations/layer16_attention_weight_min": -28.437416076660156,
      "activations/layer17_attention_weight_max": 29.55327796936035,
      "activations/layer17_attention_weight_min": -24.67024040222168,
      "activations/layer18_attention_weight_max": 30.136985778808594,
      "activations/layer18_attention_weight_min": -24.951602935791016,
      "activations/layer19_attention_weight_max": 33.455448150634766,
      "activations/layer19_attention_weight_min": -30.079076766967773,
      "activations/layer1_attention_weight_max": 16.160457611083984,
      "activations/layer1_attention_weight_min": -14.499241828918457,
      "activations/layer20_attention_weight_max": 28.380353927612305,
      "activations/layer20_attention_weight_min": -22.891435623168945,
      "activations/layer21_attention_weight_max": 27.926389694213867,
      "activations/layer21_attention_weight_min": -23.86602020263672,
      "activations/layer22_attention_weight_max": 43.05634307861328,
      "activations/layer22_attention_weight_min": -29.032508850097656,
      "activations/layer23_attention_weight_max": 32.73377990722656,
      "activations/layer23_attention_weight_min": -24.368118286132812,
      "activations/layer2_attention_weight_max": 32.2828369140625,
      "activations/layer2_attention_weight_min": -31.23874282836914,
      "activations/layer3_attention_weight_max": 102.09844207763672,
      "activations/layer3_attention_weight_min": -103.60663604736328,
      "activations/layer4_attention_weight_max": 108.42366790771484,
      "activations/layer4_attention_weight_min": -114.2673568725586,
      "activations/layer5_attention_weight_max": 52.125545501708984,
      "activations/layer5_attention_weight_min": -63.97294616699219,
      "activations/layer6_attention_weight_max": 44.83546829223633,
      "activations/layer6_attention_weight_min": -49.53162384033203,
      "activations/layer7_attention_weight_max": 90.93296813964844,
      "activations/layer7_attention_weight_min": -99.06256103515625,
      "activations/layer8_attention_weight_max": 42.553070068359375,
      "activations/layer8_attention_weight_min": -46.91264724731445,
      "activations/layer9_attention_weight_max": 32.23162078857422,
      "activations/layer9_attention_weight_min": -34.47626495361328,
      "epoch": 19.89,
      "learning_rate": 2.1940151515151513e-05,
      "loss": 2.7252,
      "step": 342350
    },
    {
      "activations/layer0_attention_weight_max": 15.766318321228027,
      "activations/layer0_attention_weight_min": -12.68209457397461,
      "activations/layer10_attention_weight_max": 36.17420959472656,
      "activations/layer10_attention_weight_min": -36.673301696777344,
      "activations/layer11_attention_weight_max": 33.295318603515625,
      "activations/layer11_attention_weight_min": -36.27648162841797,
      "activations/layer12_attention_weight_max": 26.457685470581055,
      "activations/layer12_attention_weight_min": -25.564453125,
      "activations/layer13_attention_weight_max": 38.46211624145508,
      "activations/layer13_attention_weight_min": -35.22837829589844,
      "activations/layer14_attention_weight_max": 44.17127990722656,
      "activations/layer14_attention_weight_min": -38.03168487548828,
      "activations/layer15_attention_weight_max": 39.741214752197266,
      "activations/layer15_attention_weight_min": -35.530574798583984,
      "activations/layer16_attention_weight_max": 28.96329689025879,
      "activations/layer16_attention_weight_min": -27.56174659729004,
      "activations/layer17_attention_weight_max": 31.03982162475586,
      "activations/layer17_attention_weight_min": -25.634815216064453,
      "activations/layer18_attention_weight_max": 34.790157318115234,
      "activations/layer18_attention_weight_min": -27.479267120361328,
      "activations/layer19_attention_weight_max": 36.03950119018555,
      "activations/layer19_attention_weight_min": -35.232337951660156,
      "activations/layer1_attention_weight_max": 15.117814064025879,
      "activations/layer1_attention_weight_min": -14.284703254699707,
      "activations/layer20_attention_weight_max": 29.570409774780273,
      "activations/layer20_attention_weight_min": -28.847776412963867,
      "activations/layer21_attention_weight_max": 33.776397705078125,
      "activations/layer21_attention_weight_min": -27.722728729248047,
      "activations/layer22_attention_weight_max": 42.93466567993164,
      "activations/layer22_attention_weight_min": -34.96980285644531,
      "activations/layer23_attention_weight_max": 33.04388427734375,
      "activations/layer23_attention_weight_min": -25.997251510620117,
      "activations/layer2_attention_weight_max": 31.51972007751465,
      "activations/layer2_attention_weight_min": -30.57332420349121,
      "activations/layer3_attention_weight_max": 100.55553436279297,
      "activations/layer3_attention_weight_min": -100.71375274658203,
      "activations/layer4_attention_weight_max": 104.21721649169922,
      "activations/layer4_attention_weight_min": -108.32694244384766,
      "activations/layer5_attention_weight_max": 50.458152770996094,
      "activations/layer5_attention_weight_min": -62.17987060546875,
      "activations/layer6_attention_weight_max": 44.5340461730957,
      "activations/layer6_attention_weight_min": -48.767024993896484,
      "activations/layer7_attention_weight_max": 92.90055084228516,
      "activations/layer7_attention_weight_min": -103.10359954833984,
      "activations/layer8_attention_weight_max": 40.82757568359375,
      "activations/layer8_attention_weight_min": -44.33716583251953,
      "activations/layer9_attention_weight_max": 31.092864990234375,
      "activations/layer9_attention_weight_min": -34.68842315673828,
      "epoch": 19.9,
      "learning_rate": 2.1921212121212118e-05,
      "loss": 2.7287,
      "step": 342400
    },
    {
      "activations/layer0_attention_weight_max": 15.776765823364258,
      "activations/layer0_attention_weight_min": -13.024474143981934,
      "activations/layer10_attention_weight_max": 38.675376892089844,
      "activations/layer10_attention_weight_min": -36.77878952026367,
      "activations/layer11_attention_weight_max": 39.65931701660156,
      "activations/layer11_attention_weight_min": -39.62383270263672,
      "activations/layer12_attention_weight_max": 26.109851837158203,
      "activations/layer12_attention_weight_min": -28.824827194213867,
      "activations/layer13_attention_weight_max": 38.57161331176758,
      "activations/layer13_attention_weight_min": -38.32304000854492,
      "activations/layer14_attention_weight_max": 41.33869552612305,
      "activations/layer14_attention_weight_min": -41.50192642211914,
      "activations/layer15_attention_weight_max": 39.5661506652832,
      "activations/layer15_attention_weight_min": -40.07650375366211,
      "activations/layer16_attention_weight_max": 31.82991600036621,
      "activations/layer16_attention_weight_min": -27.969131469726562,
      "activations/layer17_attention_weight_max": 29.95405387878418,
      "activations/layer17_attention_weight_min": -26.155460357666016,
      "activations/layer18_attention_weight_max": 30.66176986694336,
      "activations/layer18_attention_weight_min": -26.069713592529297,
      "activations/layer19_attention_weight_max": 33.60634994506836,
      "activations/layer19_attention_weight_min": -31.9217529296875,
      "activations/layer1_attention_weight_max": 16.517562866210938,
      "activations/layer1_attention_weight_min": -15.988139152526855,
      "activations/layer20_attention_weight_max": 29.02071189880371,
      "activations/layer20_attention_weight_min": -25.291147232055664,
      "activations/layer21_attention_weight_max": 28.543169021606445,
      "activations/layer21_attention_weight_min": -25.40610694885254,
      "activations/layer22_attention_weight_max": 36.21879196166992,
      "activations/layer22_attention_weight_min": -30.01740264892578,
      "activations/layer23_attention_weight_max": 30.410497665405273,
      "activations/layer23_attention_weight_min": -26.372549057006836,
      "activations/layer2_attention_weight_max": 34.53601837158203,
      "activations/layer2_attention_weight_min": -32.872650146484375,
      "activations/layer3_attention_weight_max": 104.43804168701172,
      "activations/layer3_attention_weight_min": -104.28773498535156,
      "activations/layer4_attention_weight_max": 104.92115783691406,
      "activations/layer4_attention_weight_min": -108.4666519165039,
      "activations/layer5_attention_weight_max": 51.01481246948242,
      "activations/layer5_attention_weight_min": -63.792198181152344,
      "activations/layer6_attention_weight_max": 45.266056060791016,
      "activations/layer6_attention_weight_min": -47.083595275878906,
      "activations/layer7_attention_weight_max": 99.77196502685547,
      "activations/layer7_attention_weight_min": -101.29177856445312,
      "activations/layer8_attention_weight_max": 41.4183464050293,
      "activations/layer8_attention_weight_min": -43.463863372802734,
      "activations/layer9_attention_weight_max": 31.311817169189453,
      "activations/layer9_attention_weight_min": -33.802066802978516,
      "epoch": 19.9,
      "learning_rate": 2.1902272727272726e-05,
      "loss": 2.7234,
      "step": 342450
    },
    {
      "activations/layer0_attention_weight_max": 16.778745651245117,
      "activations/layer0_attention_weight_min": -13.689960479736328,
      "activations/layer10_attention_weight_max": 39.505470275878906,
      "activations/layer10_attention_weight_min": -39.81262969970703,
      "activations/layer11_attention_weight_max": 38.784881591796875,
      "activations/layer11_attention_weight_min": -37.461490631103516,
      "activations/layer12_attention_weight_max": 31.157014846801758,
      "activations/layer12_attention_weight_min": -27.368112564086914,
      "activations/layer13_attention_weight_max": 46.794559478759766,
      "activations/layer13_attention_weight_min": -36.35030746459961,
      "activations/layer14_attention_weight_max": 50.21381759643555,
      "activations/layer14_attention_weight_min": -36.78349304199219,
      "activations/layer15_attention_weight_max": 46.55463409423828,
      "activations/layer15_attention_weight_min": -37.62192153930664,
      "activations/layer16_attention_weight_max": 37.696922302246094,
      "activations/layer16_attention_weight_min": -27.85750389099121,
      "activations/layer17_attention_weight_max": 34.49687576293945,
      "activations/layer17_attention_weight_min": -27.52040672302246,
      "activations/layer18_attention_weight_max": 34.842124938964844,
      "activations/layer18_attention_weight_min": -27.751501083374023,
      "activations/layer19_attention_weight_max": 33.18686294555664,
      "activations/layer19_attention_weight_min": -30.98813819885254,
      "activations/layer1_attention_weight_max": 15.698813438415527,
      "activations/layer1_attention_weight_min": -13.805390357971191,
      "activations/layer20_attention_weight_max": 28.807449340820312,
      "activations/layer20_attention_weight_min": -24.771400451660156,
      "activations/layer21_attention_weight_max": 26.481857299804688,
      "activations/layer21_attention_weight_min": -24.17509651184082,
      "activations/layer22_attention_weight_max": 47.663787841796875,
      "activations/layer22_attention_weight_min": -32.6368408203125,
      "activations/layer23_attention_weight_max": 32.64045715332031,
      "activations/layer23_attention_weight_min": -27.145048141479492,
      "activations/layer2_attention_weight_max": 31.11640739440918,
      "activations/layer2_attention_weight_min": -29.541667938232422,
      "activations/layer3_attention_weight_max": 98.13722229003906,
      "activations/layer3_attention_weight_min": -97.47888946533203,
      "activations/layer4_attention_weight_max": 108.2268295288086,
      "activations/layer4_attention_weight_min": -108.01460266113281,
      "activations/layer5_attention_weight_max": 49.736934661865234,
      "activations/layer5_attention_weight_min": -59.38706588745117,
      "activations/layer6_attention_weight_max": 49.18165588378906,
      "activations/layer6_attention_weight_min": -50.08430480957031,
      "activations/layer7_attention_weight_max": 101.11603546142578,
      "activations/layer7_attention_weight_min": -102.00584411621094,
      "activations/layer8_attention_weight_max": 47.63685607910156,
      "activations/layer8_attention_weight_min": -49.57651138305664,
      "activations/layer9_attention_weight_max": 34.8195915222168,
      "activations/layer9_attention_weight_min": -38.58340072631836,
      "epoch": 19.9,
      "learning_rate": 2.1883333333333334e-05,
      "loss": 2.7108,
      "step": 342500
    },
    {
      "activations/layer0_attention_weight_max": 16.218751907348633,
      "activations/layer0_attention_weight_min": -12.908614158630371,
      "activations/layer10_attention_weight_max": 41.43678283691406,
      "activations/layer10_attention_weight_min": -42.26134490966797,
      "activations/layer11_attention_weight_max": 42.22328186035156,
      "activations/layer11_attention_weight_min": -37.89257049560547,
      "activations/layer12_attention_weight_max": 32.73340606689453,
      "activations/layer12_attention_weight_min": -29.68976402282715,
      "activations/layer13_attention_weight_max": 45.57234191894531,
      "activations/layer13_attention_weight_min": -40.1888427734375,
      "activations/layer14_attention_weight_max": 48.59648513793945,
      "activations/layer14_attention_weight_min": -43.79083251953125,
      "activations/layer15_attention_weight_max": 42.245330810546875,
      "activations/layer15_attention_weight_min": -40.40250778198242,
      "activations/layer16_attention_weight_max": 28.98138427734375,
      "activations/layer16_attention_weight_min": -29.03357696533203,
      "activations/layer17_attention_weight_max": 29.522525787353516,
      "activations/layer17_attention_weight_min": -27.592451095581055,
      "activations/layer18_attention_weight_max": 32.53181076049805,
      "activations/layer18_attention_weight_min": -26.078523635864258,
      "activations/layer19_attention_weight_max": 32.860572814941406,
      "activations/layer19_attention_weight_min": -33.327083587646484,
      "activations/layer1_attention_weight_max": 15.423046112060547,
      "activations/layer1_attention_weight_min": -13.723219871520996,
      "activations/layer20_attention_weight_max": 28.924779891967773,
      "activations/layer20_attention_weight_min": -27.4899845123291,
      "activations/layer21_attention_weight_max": 28.821075439453125,
      "activations/layer21_attention_weight_min": -25.997684478759766,
      "activations/layer22_attention_weight_max": 43.91800308227539,
      "activations/layer22_attention_weight_min": -34.690120697021484,
      "activations/layer23_attention_weight_max": 29.645904541015625,
      "activations/layer23_attention_weight_min": -28.005435943603516,
      "activations/layer2_attention_weight_max": 31.59259796142578,
      "activations/layer2_attention_weight_min": -32.30973815917969,
      "activations/layer3_attention_weight_max": 111.3983154296875,
      "activations/layer3_attention_weight_min": -108.08407592773438,
      "activations/layer4_attention_weight_max": 115.1387710571289,
      "activations/layer4_attention_weight_min": -114.13134765625,
      "activations/layer5_attention_weight_max": 50.60831832885742,
      "activations/layer5_attention_weight_min": -61.46073913574219,
      "activations/layer6_attention_weight_max": 45.57460403442383,
      "activations/layer6_attention_weight_min": -47.7043571472168,
      "activations/layer7_attention_weight_max": 102.841796875,
      "activations/layer7_attention_weight_min": -102.50732421875,
      "activations/layer8_attention_weight_max": 44.86281204223633,
      "activations/layer8_attention_weight_min": -47.56801223754883,
      "activations/layer9_attention_weight_max": 35.33852005004883,
      "activations/layer9_attention_weight_min": -38.32521438598633,
      "epoch": 19.9,
      "learning_rate": 2.1864393939393936e-05,
      "loss": 2.7085,
      "step": 342550
    },
    {
      "activations/layer0_attention_weight_max": 16.09575843811035,
      "activations/layer0_attention_weight_min": -12.850650787353516,
      "activations/layer10_attention_weight_max": 35.61195755004883,
      "activations/layer10_attention_weight_min": -35.587284088134766,
      "activations/layer11_attention_weight_max": 35.43570327758789,
      "activations/layer11_attention_weight_min": -34.774696350097656,
      "activations/layer12_attention_weight_max": 26.65001106262207,
      "activations/layer12_attention_weight_min": -26.06364631652832,
      "activations/layer13_attention_weight_max": 38.18367385864258,
      "activations/layer13_attention_weight_min": -37.23878860473633,
      "activations/layer14_attention_weight_max": 41.80377960205078,
      "activations/layer14_attention_weight_min": -39.50627136230469,
      "activations/layer15_attention_weight_max": 38.0597038269043,
      "activations/layer15_attention_weight_min": -38.48366928100586,
      "activations/layer16_attention_weight_max": 27.077043533325195,
      "activations/layer16_attention_weight_min": -30.550548553466797,
      "activations/layer17_attention_weight_max": 29.0244083404541,
      "activations/layer17_attention_weight_min": -25.906871795654297,
      "activations/layer18_attention_weight_max": 31.226322174072266,
      "activations/layer18_attention_weight_min": -27.474018096923828,
      "activations/layer19_attention_weight_max": 32.9658317565918,
      "activations/layer19_attention_weight_min": -31.71103286743164,
      "activations/layer1_attention_weight_max": 15.515000343322754,
      "activations/layer1_attention_weight_min": -14.409765243530273,
      "activations/layer20_attention_weight_max": 26.381818771362305,
      "activations/layer20_attention_weight_min": -27.4378662109375,
      "activations/layer21_attention_weight_max": 26.96746063232422,
      "activations/layer21_attention_weight_min": -28.031450271606445,
      "activations/layer22_attention_weight_max": 40.56715393066406,
      "activations/layer22_attention_weight_min": -32.15610122680664,
      "activations/layer23_attention_weight_max": 30.70400047302246,
      "activations/layer23_attention_weight_min": -27.850906372070312,
      "activations/layer2_attention_weight_max": 32.77922439575195,
      "activations/layer2_attention_weight_min": -31.46310043334961,
      "activations/layer3_attention_weight_max": 100.2576904296875,
      "activations/layer3_attention_weight_min": -103.1270980834961,
      "activations/layer4_attention_weight_max": 105.24361419677734,
      "activations/layer4_attention_weight_min": -111.9535903930664,
      "activations/layer5_attention_weight_max": 49.97746276855469,
      "activations/layer5_attention_weight_min": -59.18703842163086,
      "activations/layer6_attention_weight_max": 45.778507232666016,
      "activations/layer6_attention_weight_min": -49.155338287353516,
      "activations/layer7_attention_weight_max": 93.18299865722656,
      "activations/layer7_attention_weight_min": -95.92936706542969,
      "activations/layer8_attention_weight_max": 43.34396743774414,
      "activations/layer8_attention_weight_min": -45.8953742980957,
      "activations/layer9_attention_weight_max": 32.72608947753906,
      "activations/layer9_attention_weight_min": -36.06092834472656,
      "epoch": 19.91,
      "learning_rate": 2.1845833333333334e-05,
      "loss": 2.7291,
      "step": 342600
    },
    {
      "activations/layer0_attention_weight_max": 15.741443634033203,
      "activations/layer0_attention_weight_min": -11.905675888061523,
      "activations/layer10_attention_weight_max": 36.762752532958984,
      "activations/layer10_attention_weight_min": -37.385536193847656,
      "activations/layer11_attention_weight_max": 36.911231994628906,
      "activations/layer11_attention_weight_min": -35.47041320800781,
      "activations/layer12_attention_weight_max": 27.068138122558594,
      "activations/layer12_attention_weight_min": -27.57916831970215,
      "activations/layer13_attention_weight_max": 42.3638801574707,
      "activations/layer13_attention_weight_min": -37.36973571777344,
      "activations/layer14_attention_weight_max": 48.132511138916016,
      "activations/layer14_attention_weight_min": -41.44050598144531,
      "activations/layer15_attention_weight_max": 42.043880462646484,
      "activations/layer15_attention_weight_min": -40.06647491455078,
      "activations/layer16_attention_weight_max": 29.522539138793945,
      "activations/layer16_attention_weight_min": -29.238344192504883,
      "activations/layer17_attention_weight_max": 27.917678833007812,
      "activations/layer17_attention_weight_min": -25.47199821472168,
      "activations/layer18_attention_weight_max": 31.434982299804688,
      "activations/layer18_attention_weight_min": -24.834012985229492,
      "activations/layer19_attention_weight_max": 33.492618560791016,
      "activations/layer19_attention_weight_min": -31.518869400024414,
      "activations/layer1_attention_weight_max": 15.593734741210938,
      "activations/layer1_attention_weight_min": -13.3464937210083,
      "activations/layer20_attention_weight_max": 28.257152557373047,
      "activations/layer20_attention_weight_min": -24.859750747680664,
      "activations/layer21_attention_weight_max": 27.598325729370117,
      "activations/layer21_attention_weight_min": -24.31839942932129,
      "activations/layer22_attention_weight_max": 38.62961959838867,
      "activations/layer22_attention_weight_min": -31.241167068481445,
      "activations/layer23_attention_weight_max": 29.78692626953125,
      "activations/layer23_attention_weight_min": -24.4715576171875,
      "activations/layer2_attention_weight_max": 30.427244186401367,
      "activations/layer2_attention_weight_min": -30.10234832763672,
      "activations/layer3_attention_weight_max": 95.77516174316406,
      "activations/layer3_attention_weight_min": -97.6749038696289,
      "activations/layer4_attention_weight_max": 101.87113189697266,
      "activations/layer4_attention_weight_min": -108.01824188232422,
      "activations/layer5_attention_weight_max": 48.67040252685547,
      "activations/layer5_attention_weight_min": -60.37588119506836,
      "activations/layer6_attention_weight_max": 43.10338592529297,
      "activations/layer6_attention_weight_min": -47.49932098388672,
      "activations/layer7_attention_weight_max": 101.65217590332031,
      "activations/layer7_attention_weight_min": -95.2717514038086,
      "activations/layer8_attention_weight_max": 43.771846771240234,
      "activations/layer8_attention_weight_min": -46.331329345703125,
      "activations/layer9_attention_weight_max": 32.50639724731445,
      "activations/layer9_attention_weight_min": -35.750343322753906,
      "epoch": 19.91,
      "learning_rate": 2.182689393939394e-05,
      "loss": 2.7095,
      "step": 342650
    },
    {
      "activations/layer0_attention_weight_max": 16.83445167541504,
      "activations/layer0_attention_weight_min": -13.315357208251953,
      "activations/layer10_attention_weight_max": 33.51722717285156,
      "activations/layer10_attention_weight_min": -35.2947883605957,
      "activations/layer11_attention_weight_max": 34.39690399169922,
      "activations/layer11_attention_weight_min": -37.40176773071289,
      "activations/layer12_attention_weight_max": 25.957319259643555,
      "activations/layer12_attention_weight_min": -26.146501541137695,
      "activations/layer13_attention_weight_max": 39.13417053222656,
      "activations/layer13_attention_weight_min": -36.610435485839844,
      "activations/layer14_attention_weight_max": 42.99874496459961,
      "activations/layer14_attention_weight_min": -38.03428268432617,
      "activations/layer15_attention_weight_max": 39.45222473144531,
      "activations/layer15_attention_weight_min": -36.385215759277344,
      "activations/layer16_attention_weight_max": 28.104999542236328,
      "activations/layer16_attention_weight_min": -27.345138549804688,
      "activations/layer17_attention_weight_max": 29.386661529541016,
      "activations/layer17_attention_weight_min": -26.371225357055664,
      "activations/layer18_attention_weight_max": 31.0334529876709,
      "activations/layer18_attention_weight_min": -27.922161102294922,
      "activations/layer19_attention_weight_max": 33.006744384765625,
      "activations/layer19_attention_weight_min": -29.981555938720703,
      "activations/layer1_attention_weight_max": 15.805435180664062,
      "activations/layer1_attention_weight_min": -13.893902778625488,
      "activations/layer20_attention_weight_max": 27.734052658081055,
      "activations/layer20_attention_weight_min": -24.947702407836914,
      "activations/layer21_attention_weight_max": 27.759613037109375,
      "activations/layer21_attention_weight_min": -24.656328201293945,
      "activations/layer22_attention_weight_max": 37.09589385986328,
      "activations/layer22_attention_weight_min": -30.815773010253906,
      "activations/layer23_attention_weight_max": 30.99184799194336,
      "activations/layer23_attention_weight_min": -26.583255767822266,
      "activations/layer2_attention_weight_max": 32.519561767578125,
      "activations/layer2_attention_weight_min": -29.74508285522461,
      "activations/layer3_attention_weight_max": 97.5724868774414,
      "activations/layer3_attention_weight_min": -97.25910949707031,
      "activations/layer4_attention_weight_max": 102.06784057617188,
      "activations/layer4_attention_weight_min": -107.5934829711914,
      "activations/layer5_attention_weight_max": 50.38739776611328,
      "activations/layer5_attention_weight_min": -57.28324890136719,
      "activations/layer6_attention_weight_max": 43.15542221069336,
      "activations/layer6_attention_weight_min": -46.42176818847656,
      "activations/layer7_attention_weight_max": 94.30874633789062,
      "activations/layer7_attention_weight_min": -91.568359375,
      "activations/layer8_attention_weight_max": 39.88367462158203,
      "activations/layer8_attention_weight_min": -42.85025405883789,
      "activations/layer9_attention_weight_max": 31.522085189819336,
      "activations/layer9_attention_weight_min": -34.3515510559082,
      "epoch": 19.91,
      "learning_rate": 2.1807954545454547e-05,
      "loss": 2.7168,
      "step": 342700
    },
    {
      "activations/layer0_attention_weight_max": 15.83022403717041,
      "activations/layer0_attention_weight_min": -13.499255180358887,
      "activations/layer10_attention_weight_max": 35.53718185424805,
      "activations/layer10_attention_weight_min": -34.149505615234375,
      "activations/layer11_attention_weight_max": 36.15144348144531,
      "activations/layer11_attention_weight_min": -34.19134521484375,
      "activations/layer12_attention_weight_max": 26.574003219604492,
      "activations/layer12_attention_weight_min": -26.81998634338379,
      "activations/layer13_attention_weight_max": 39.77444076538086,
      "activations/layer13_attention_weight_min": -34.583534240722656,
      "activations/layer14_attention_weight_max": 42.43006134033203,
      "activations/layer14_attention_weight_min": -39.87046813964844,
      "activations/layer15_attention_weight_max": 39.59150695800781,
      "activations/layer15_attention_weight_min": -34.92155075073242,
      "activations/layer16_attention_weight_max": 28.1353759765625,
      "activations/layer16_attention_weight_min": -28.0030517578125,
      "activations/layer17_attention_weight_max": 29.45623207092285,
      "activations/layer17_attention_weight_min": -26.26653480529785,
      "activations/layer18_attention_weight_max": 34.225215911865234,
      "activations/layer18_attention_weight_min": -26.395292282104492,
      "activations/layer19_attention_weight_max": 35.21377944946289,
      "activations/layer19_attention_weight_min": -32.475807189941406,
      "activations/layer1_attention_weight_max": 15.75284481048584,
      "activations/layer1_attention_weight_min": -13.761335372924805,
      "activations/layer20_attention_weight_max": 29.53225326538086,
      "activations/layer20_attention_weight_min": -25.474693298339844,
      "activations/layer21_attention_weight_max": 28.84737777709961,
      "activations/layer21_attention_weight_min": -24.856924057006836,
      "activations/layer22_attention_weight_max": 40.812232971191406,
      "activations/layer22_attention_weight_min": -31.855098724365234,
      "activations/layer23_attention_weight_max": 32.533203125,
      "activations/layer23_attention_weight_min": -27.284027099609375,
      "activations/layer2_attention_weight_max": 30.121570587158203,
      "activations/layer2_attention_weight_min": -30.067123413085938,
      "activations/layer3_attention_weight_max": 99.6051254272461,
      "activations/layer3_attention_weight_min": -98.8050308227539,
      "activations/layer4_attention_weight_max": 103.63191223144531,
      "activations/layer4_attention_weight_min": -108.38165283203125,
      "activations/layer5_attention_weight_max": 48.977752685546875,
      "activations/layer5_attention_weight_min": -64.122314453125,
      "activations/layer6_attention_weight_max": 42.81391525268555,
      "activations/layer6_attention_weight_min": -46.943824768066406,
      "activations/layer7_attention_weight_max": 93.83438110351562,
      "activations/layer7_attention_weight_min": -100.47982788085938,
      "activations/layer8_attention_weight_max": 41.571048736572266,
      "activations/layer8_attention_weight_min": -44.054325103759766,
      "activations/layer9_attention_weight_max": 31.119497299194336,
      "activations/layer9_attention_weight_min": -34.137325286865234,
      "epoch": 19.92,
      "learning_rate": 2.1789015151515148e-05,
      "loss": 2.7137,
      "step": 342750
    },
    {
      "activations/layer0_attention_weight_max": 16.42284393310547,
      "activations/layer0_attention_weight_min": -12.577035903930664,
      "activations/layer10_attention_weight_max": 37.81745147705078,
      "activations/layer10_attention_weight_min": -39.45526885986328,
      "activations/layer11_attention_weight_max": 39.15888977050781,
      "activations/layer11_attention_weight_min": -35.531341552734375,
      "activations/layer12_attention_weight_max": 32.44866943359375,
      "activations/layer12_attention_weight_min": -29.24546241760254,
      "activations/layer13_attention_weight_max": 48.87626647949219,
      "activations/layer13_attention_weight_min": -38.427024841308594,
      "activations/layer14_attention_weight_max": 51.76056671142578,
      "activations/layer14_attention_weight_min": -41.11990737915039,
      "activations/layer15_attention_weight_max": 51.04686737060547,
      "activations/layer15_attention_weight_min": -40.15439224243164,
      "activations/layer16_attention_weight_max": 35.90370559692383,
      "activations/layer16_attention_weight_min": -29.462434768676758,
      "activations/layer17_attention_weight_max": 39.526878356933594,
      "activations/layer17_attention_weight_min": -27.218063354492188,
      "activations/layer18_attention_weight_max": 42.42030715942383,
      "activations/layer18_attention_weight_min": -27.384782791137695,
      "activations/layer19_attention_weight_max": 45.96332931518555,
      "activations/layer19_attention_weight_min": -33.41111373901367,
      "activations/layer1_attention_weight_max": 15.349611282348633,
      "activations/layer1_attention_weight_min": -14.257472038269043,
      "activations/layer20_attention_weight_max": 39.714900970458984,
      "activations/layer20_attention_weight_min": -26.78070068359375,
      "activations/layer21_attention_weight_max": 38.258670806884766,
      "activations/layer21_attention_weight_min": -26.389036178588867,
      "activations/layer22_attention_weight_max": 62.4237174987793,
      "activations/layer22_attention_weight_min": -37.57780456542969,
      "activations/layer23_attention_weight_max": 41.88380432128906,
      "activations/layer23_attention_weight_min": -27.681257247924805,
      "activations/layer2_attention_weight_max": 31.19509506225586,
      "activations/layer2_attention_weight_min": -30.17487907409668,
      "activations/layer3_attention_weight_max": 101.08531951904297,
      "activations/layer3_attention_weight_min": -98.72229766845703,
      "activations/layer4_attention_weight_max": 107.62407684326172,
      "activations/layer4_attention_weight_min": -110.50508117675781,
      "activations/layer5_attention_weight_max": 52.70978546142578,
      "activations/layer5_attention_weight_min": -59.22260284423828,
      "activations/layer6_attention_weight_max": 44.903377532958984,
      "activations/layer6_attention_weight_min": -49.854915618896484,
      "activations/layer7_attention_weight_max": 98.8057632446289,
      "activations/layer7_attention_weight_min": -103.50193786621094,
      "activations/layer8_attention_weight_max": 49.02937698364258,
      "activations/layer8_attention_weight_min": -45.33961868286133,
      "activations/layer9_attention_weight_max": 36.07876205444336,
      "activations/layer9_attention_weight_min": -37.92325210571289,
      "epoch": 19.92,
      "learning_rate": 2.1770075757575756e-05,
      "loss": 2.7073,
      "step": 342800
    },
    {
      "activations/layer0_attention_weight_max": 16.201679229736328,
      "activations/layer0_attention_weight_min": -14.252230644226074,
      "activations/layer10_attention_weight_max": 33.365169525146484,
      "activations/layer10_attention_weight_min": -33.89385986328125,
      "activations/layer11_attention_weight_max": 34.30651092529297,
      "activations/layer11_attention_weight_min": -33.8631591796875,
      "activations/layer12_attention_weight_max": 25.17188835144043,
      "activations/layer12_attention_weight_min": -26.245437622070312,
      "activations/layer13_attention_weight_max": 38.57617950439453,
      "activations/layer13_attention_weight_min": -35.70212936401367,
      "activations/layer14_attention_weight_max": 40.827762603759766,
      "activations/layer14_attention_weight_min": -39.3447380065918,
      "activations/layer15_attention_weight_max": 35.75078201293945,
      "activations/layer15_attention_weight_min": -36.27103805541992,
      "activations/layer16_attention_weight_max": 28.936792373657227,
      "activations/layer16_attention_weight_min": -29.158336639404297,
      "activations/layer17_attention_weight_max": 26.584993362426758,
      "activations/layer17_attention_weight_min": -25.858617782592773,
      "activations/layer18_attention_weight_max": 31.095430374145508,
      "activations/layer18_attention_weight_min": -26.44252586364746,
      "activations/layer19_attention_weight_max": 32.13794708251953,
      "activations/layer19_attention_weight_min": -31.369338989257812,
      "activations/layer1_attention_weight_max": 16.48760986328125,
      "activations/layer1_attention_weight_min": -14.87541389465332,
      "activations/layer20_attention_weight_max": 26.467771530151367,
      "activations/layer20_attention_weight_min": -25.432085037231445,
      "activations/layer21_attention_weight_max": 27.47777557373047,
      "activations/layer21_attention_weight_min": -24.72825050354004,
      "activations/layer22_attention_weight_max": 41.131038665771484,
      "activations/layer22_attention_weight_min": -30.832536697387695,
      "activations/layer23_attention_weight_max": 30.97267723083496,
      "activations/layer23_attention_weight_min": -26.07921600341797,
      "activations/layer2_attention_weight_max": 31.13983154296875,
      "activations/layer2_attention_weight_min": -29.41036033630371,
      "activations/layer3_attention_weight_max": 101.65259552001953,
      "activations/layer3_attention_weight_min": -99.1867446899414,
      "activations/layer4_attention_weight_max": 108.35884094238281,
      "activations/layer4_attention_weight_min": -108.42921447753906,
      "activations/layer5_attention_weight_max": 51.607322692871094,
      "activations/layer5_attention_weight_min": -62.56031799316406,
      "activations/layer6_attention_weight_max": 45.06987762451172,
      "activations/layer6_attention_weight_min": -46.039493560791016,
      "activations/layer7_attention_weight_max": 95.24320220947266,
      "activations/layer7_attention_weight_min": -101.82518005371094,
      "activations/layer8_attention_weight_max": 43.69845962524414,
      "activations/layer8_attention_weight_min": -45.37364959716797,
      "activations/layer9_attention_weight_max": 31.64216423034668,
      "activations/layer9_attention_weight_min": -36.42951583862305,
      "epoch": 19.92,
      "learning_rate": 2.175113636363636e-05,
      "loss": 2.7232,
      "step": 342850
    },
    {
      "activations/layer0_attention_weight_max": 15.931648254394531,
      "activations/layer0_attention_weight_min": -14.002054214477539,
      "activations/layer10_attention_weight_max": 30.90207862854004,
      "activations/layer10_attention_weight_min": -34.209144592285156,
      "activations/layer11_attention_weight_max": 33.274627685546875,
      "activations/layer11_attention_weight_min": -36.417396545410156,
      "activations/layer12_attention_weight_max": 26.391639709472656,
      "activations/layer12_attention_weight_min": -27.810413360595703,
      "activations/layer13_attention_weight_max": 38.41997146606445,
      "activations/layer13_attention_weight_min": -37.12736892700195,
      "activations/layer14_attention_weight_max": 42.062034606933594,
      "activations/layer14_attention_weight_min": -38.64170455932617,
      "activations/layer15_attention_weight_max": 39.3264045715332,
      "activations/layer15_attention_weight_min": -35.439029693603516,
      "activations/layer16_attention_weight_max": 37.64582061767578,
      "activations/layer16_attention_weight_min": -28.44627571105957,
      "activations/layer17_attention_weight_max": 30.80230140686035,
      "activations/layer17_attention_weight_min": -26.891359329223633,
      "activations/layer18_attention_weight_max": 32.35006332397461,
      "activations/layer18_attention_weight_min": -26.87432289123535,
      "activations/layer19_attention_weight_max": 45.68507385253906,
      "activations/layer19_attention_weight_min": -31.520246505737305,
      "activations/layer1_attention_weight_max": 15.967313766479492,
      "activations/layer1_attention_weight_min": -13.465463638305664,
      "activations/layer20_attention_weight_max": 33.306156158447266,
      "activations/layer20_attention_weight_min": -27.644174575805664,
      "activations/layer21_attention_weight_max": 33.229652404785156,
      "activations/layer21_attention_weight_min": -24.908300399780273,
      "activations/layer22_attention_weight_max": 48.15467834472656,
      "activations/layer22_attention_weight_min": -32.00834274291992,
      "activations/layer23_attention_weight_max": 38.93315124511719,
      "activations/layer23_attention_weight_min": -27.09148406982422,
      "activations/layer2_attention_weight_max": 31.075862884521484,
      "activations/layer2_attention_weight_min": -29.896108627319336,
      "activations/layer3_attention_weight_max": 99.62480163574219,
      "activations/layer3_attention_weight_min": -97.97393035888672,
      "activations/layer4_attention_weight_max": 105.56490325927734,
      "activations/layer4_attention_weight_min": -108.62737274169922,
      "activations/layer5_attention_weight_max": 49.644039154052734,
      "activations/layer5_attention_weight_min": -61.571495056152344,
      "activations/layer6_attention_weight_max": 44.26150131225586,
      "activations/layer6_attention_weight_min": -49.390689849853516,
      "activations/layer7_attention_weight_max": 92.58941650390625,
      "activations/layer7_attention_weight_min": -99.04610443115234,
      "activations/layer8_attention_weight_max": 41.7678108215332,
      "activations/layer8_attention_weight_min": -44.415916442871094,
      "activations/layer9_attention_weight_max": 30.246784210205078,
      "activations/layer9_attention_weight_min": -33.54648971557617,
      "epoch": 19.92,
      "learning_rate": 2.173219696969697e-05,
      "loss": 2.7338,
      "step": 342900
    },
    {
      "activations/layer0_attention_weight_max": 15.707083702087402,
      "activations/layer0_attention_weight_min": -13.673483848571777,
      "activations/layer10_attention_weight_max": 33.49213790893555,
      "activations/layer10_attention_weight_min": -34.53095245361328,
      "activations/layer11_attention_weight_max": 35.365806579589844,
      "activations/layer11_attention_weight_min": -35.55126190185547,
      "activations/layer12_attention_weight_max": 26.449073791503906,
      "activations/layer12_attention_weight_min": -28.875375747680664,
      "activations/layer13_attention_weight_max": 39.399078369140625,
      "activations/layer13_attention_weight_min": -37.697933197021484,
      "activations/layer14_attention_weight_max": 43.498268127441406,
      "activations/layer14_attention_weight_min": -39.87962341308594,
      "activations/layer15_attention_weight_max": 39.89488220214844,
      "activations/layer15_attention_weight_min": -36.7852668762207,
      "activations/layer16_attention_weight_max": 30.626052856445312,
      "activations/layer16_attention_weight_min": -28.408634185791016,
      "activations/layer17_attention_weight_max": 30.839590072631836,
      "activations/layer17_attention_weight_min": -27.7471866607666,
      "activations/layer18_attention_weight_max": 36.004371643066406,
      "activations/layer18_attention_weight_min": -27.709177017211914,
      "activations/layer19_attention_weight_max": 38.33376693725586,
      "activations/layer19_attention_weight_min": -35.77082061767578,
      "activations/layer1_attention_weight_max": 16.06513214111328,
      "activations/layer1_attention_weight_min": -13.766353607177734,
      "activations/layer20_attention_weight_max": 35.40727615356445,
      "activations/layer20_attention_weight_min": -26.45697021484375,
      "activations/layer21_attention_weight_max": 34.329193115234375,
      "activations/layer21_attention_weight_min": -25.714706420898438,
      "activations/layer22_attention_weight_max": 54.04607391357422,
      "activations/layer22_attention_weight_min": -35.54981994628906,
      "activations/layer23_attention_weight_max": 38.79130554199219,
      "activations/layer23_attention_weight_min": -27.025259017944336,
      "activations/layer2_attention_weight_max": 30.33279037475586,
      "activations/layer2_attention_weight_min": -28.028018951416016,
      "activations/layer3_attention_weight_max": 92.22467803955078,
      "activations/layer3_attention_weight_min": -94.79193878173828,
      "activations/layer4_attention_weight_max": 98.59425354003906,
      "activations/layer4_attention_weight_min": -99.36981964111328,
      "activations/layer5_attention_weight_max": 47.75495529174805,
      "activations/layer5_attention_weight_min": -62.826141357421875,
      "activations/layer6_attention_weight_max": 41.41593551635742,
      "activations/layer6_attention_weight_min": -45.40373229980469,
      "activations/layer7_attention_weight_max": 90.99919891357422,
      "activations/layer7_attention_weight_min": -92.93347930908203,
      "activations/layer8_attention_weight_max": 42.094886779785156,
      "activations/layer8_attention_weight_min": -42.5017204284668,
      "activations/layer9_attention_weight_max": 33.29030227661133,
      "activations/layer9_attention_weight_min": -34.02423095703125,
      "epoch": 19.93,
      "learning_rate": 2.171325757575757e-05,
      "loss": 2.7182,
      "step": 342950
    },
    {
      "activations/layer0_attention_weight_max": 16.561054229736328,
      "activations/layer0_attention_weight_min": -12.723592758178711,
      "activations/layer10_attention_weight_max": 33.765419006347656,
      "activations/layer10_attention_weight_min": -34.39604187011719,
      "activations/layer11_attention_weight_max": 34.012062072753906,
      "activations/layer11_attention_weight_min": -33.3563232421875,
      "activations/layer12_attention_weight_max": 29.14213752746582,
      "activations/layer12_attention_weight_min": -27.202743530273438,
      "activations/layer13_attention_weight_max": 41.08342742919922,
      "activations/layer13_attention_weight_min": -35.061065673828125,
      "activations/layer14_attention_weight_max": 45.72727584838867,
      "activations/layer14_attention_weight_min": -36.611061096191406,
      "activations/layer15_attention_weight_max": 43.86651611328125,
      "activations/layer15_attention_weight_min": -36.201351165771484,
      "activations/layer16_attention_weight_max": 28.299997329711914,
      "activations/layer16_attention_weight_min": -29.709741592407227,
      "activations/layer17_attention_weight_max": 30.261592864990234,
      "activations/layer17_attention_weight_min": -29.469970703125,
      "activations/layer18_attention_weight_max": 32.532081604003906,
      "activations/layer18_attention_weight_min": -25.2537899017334,
      "activations/layer19_attention_weight_max": 31.288114547729492,
      "activations/layer19_attention_weight_min": -31.994272232055664,
      "activations/layer1_attention_weight_max": 15.680949211120605,
      "activations/layer1_attention_weight_min": -14.72225284576416,
      "activations/layer20_attention_weight_max": 26.1092586517334,
      "activations/layer20_attention_weight_min": -26.019086837768555,
      "activations/layer21_attention_weight_max": 26.546489715576172,
      "activations/layer21_attention_weight_min": -24.955625534057617,
      "activations/layer22_attention_weight_max": 41.401458740234375,
      "activations/layer22_attention_weight_min": -32.48466873168945,
      "activations/layer23_attention_weight_max": 29.60967254638672,
      "activations/layer23_attention_weight_min": -26.13253402709961,
      "activations/layer2_attention_weight_max": 32.18925476074219,
      "activations/layer2_attention_weight_min": -30.11501693725586,
      "activations/layer3_attention_weight_max": 100.68250274658203,
      "activations/layer3_attention_weight_min": -100.4981689453125,
      "activations/layer4_attention_weight_max": 107.17662811279297,
      "activations/layer4_attention_weight_min": -107.0557861328125,
      "activations/layer5_attention_weight_max": 50.22845458984375,
      "activations/layer5_attention_weight_min": -59.46258544921875,
      "activations/layer6_attention_weight_max": 44.0047721862793,
      "activations/layer6_attention_weight_min": -46.43735122680664,
      "activations/layer7_attention_weight_max": 91.4458999633789,
      "activations/layer7_attention_weight_min": -93.29885864257812,
      "activations/layer8_attention_weight_max": 39.52278518676758,
      "activations/layer8_attention_weight_min": -43.655921936035156,
      "activations/layer9_attention_weight_max": 32.47407150268555,
      "activations/layer9_attention_weight_min": -32.91575622558594,
      "epoch": 19.93,
      "learning_rate": 2.169431818181818e-05,
      "loss": 2.7332,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.4968,
      "eval_samples_per_second": 505.367,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.4968,
      "eval_openwebtext_samples_per_second": 505.367,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9681,
      "eval_wikitext_samples_per_second": 231.698,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_lambada_loss": 2.330078125,
      "eval_lambada_ppl": 10.278744528592322,
      "eval_lambada_runtime": 9.5858,
      "eval_lambada_samples_per_second": 507.941,
      "step": 343000
    },
    {
      "activations/layer0_attention_weight_max": 16.097339630126953,
      "activations/layer0_attention_weight_min": -12.548524856567383,
      "activations/layer10_attention_weight_max": 35.25298309326172,
      "activations/layer10_attention_weight_min": -35.17360305786133,
      "activations/layer11_attention_weight_max": 34.88884735107422,
      "activations/layer11_attention_weight_min": -36.33525085449219,
      "activations/layer12_attention_weight_max": 28.01988983154297,
      "activations/layer12_attention_weight_min": -29.488378524780273,
      "activations/layer13_attention_weight_max": 43.14271926879883,
      "activations/layer13_attention_weight_min": -38.32567596435547,
      "activations/layer14_attention_weight_max": 44.52385330200195,
      "activations/layer14_attention_weight_min": -41.44668960571289,
      "activations/layer15_attention_weight_max": 43.12853240966797,
      "activations/layer15_attention_weight_min": -39.23921203613281,
      "activations/layer16_attention_weight_max": 31.565433502197266,
      "activations/layer16_attention_weight_min": -28.436038970947266,
      "activations/layer17_attention_weight_max": 31.318674087524414,
      "activations/layer17_attention_weight_min": -28.206941604614258,
      "activations/layer18_attention_weight_max": 31.805767059326172,
      "activations/layer18_attention_weight_min": -26.986690521240234,
      "activations/layer19_attention_weight_max": 36.29232406616211,
      "activations/layer19_attention_weight_min": -33.52993392944336,
      "activations/layer1_attention_weight_max": 15.835837364196777,
      "activations/layer1_attention_weight_min": -14.948118209838867,
      "activations/layer20_attention_weight_max": 31.65526580810547,
      "activations/layer20_attention_weight_min": -26.916149139404297,
      "activations/layer21_attention_weight_max": 31.98068618774414,
      "activations/layer21_attention_weight_min": -25.076860427856445,
      "activations/layer22_attention_weight_max": 45.61555862426758,
      "activations/layer22_attention_weight_min": -33.53702163696289,
      "activations/layer23_attention_weight_max": 35.52631378173828,
      "activations/layer23_attention_weight_min": -26.655385971069336,
      "activations/layer2_attention_weight_max": 30.922359466552734,
      "activations/layer2_attention_weight_min": -31.52442169189453,
      "activations/layer3_attention_weight_max": 100.58300018310547,
      "activations/layer3_attention_weight_min": -99.55081176757812,
      "activations/layer4_attention_weight_max": 109.34156799316406,
      "activations/layer4_attention_weight_min": -108.65534973144531,
      "activations/layer5_attention_weight_max": 52.22413635253906,
      "activations/layer5_attention_weight_min": -59.526092529296875,
      "activations/layer6_attention_weight_max": 45.52433395385742,
      "activations/layer6_attention_weight_min": -47.853939056396484,
      "activations/layer7_attention_weight_max": 91.77103424072266,
      "activations/layer7_attention_weight_min": -102.89246368408203,
      "activations/layer8_attention_weight_max": 43.67540740966797,
      "activations/layer8_attention_weight_min": -45.768653869628906,
      "activations/layer9_attention_weight_max": 34.152767181396484,
      "activations/layer9_attention_weight_min": -34.276466369628906,
      "epoch": 19.93,
      "learning_rate": 2.1675378787878784e-05,
      "loss": 2.7241,
      "step": 343050
    },
    {
      "activations/layer0_attention_weight_max": 16.005844116210938,
      "activations/layer0_attention_weight_min": -12.974944114685059,
      "activations/layer10_attention_weight_max": 32.0731201171875,
      "activations/layer10_attention_weight_min": -33.5751953125,
      "activations/layer11_attention_weight_max": 32.473724365234375,
      "activations/layer11_attention_weight_min": -34.05682373046875,
      "activations/layer12_attention_weight_max": 25.80817222595215,
      "activations/layer12_attention_weight_min": -26.751893997192383,
      "activations/layer13_attention_weight_max": 38.95494079589844,
      "activations/layer13_attention_weight_min": -36.82855224609375,
      "activations/layer14_attention_weight_max": 42.76139831542969,
      "activations/layer14_attention_weight_min": -39.928165435791016,
      "activations/layer15_attention_weight_max": 38.89710235595703,
      "activations/layer15_attention_weight_min": -36.92477798461914,
      "activations/layer16_attention_weight_max": 28.3691349029541,
      "activations/layer16_attention_weight_min": -28.313196182250977,
      "activations/layer17_attention_weight_max": 29.293176651000977,
      "activations/layer17_attention_weight_min": -25.460704803466797,
      "activations/layer18_attention_weight_max": 33.57693862915039,
      "activations/layer18_attention_weight_min": -25.1272029876709,
      "activations/layer19_attention_weight_max": 32.52836990356445,
      "activations/layer19_attention_weight_min": -30.69266128540039,
      "activations/layer1_attention_weight_max": 15.468659400939941,
      "activations/layer1_attention_weight_min": -13.985394477844238,
      "activations/layer20_attention_weight_max": 27.534032821655273,
      "activations/layer20_attention_weight_min": -25.735698699951172,
      "activations/layer21_attention_weight_max": 29.174148559570312,
      "activations/layer21_attention_weight_min": -25.271474838256836,
      "activations/layer22_attention_weight_max": 41.04775619506836,
      "activations/layer22_attention_weight_min": -31.572731018066406,
      "activations/layer23_attention_weight_max": 30.083576202392578,
      "activations/layer23_attention_weight_min": -27.739652633666992,
      "activations/layer2_attention_weight_max": 32.08972930908203,
      "activations/layer2_attention_weight_min": -30.976158142089844,
      "activations/layer3_attention_weight_max": 98.36587524414062,
      "activations/layer3_attention_weight_min": -100.05952453613281,
      "activations/layer4_attention_weight_max": 104.6680908203125,
      "activations/layer4_attention_weight_min": -113.17646789550781,
      "activations/layer5_attention_weight_max": 50.88519287109375,
      "activations/layer5_attention_weight_min": -60.104469299316406,
      "activations/layer6_attention_weight_max": 44.53246307373047,
      "activations/layer6_attention_weight_min": -48.06748962402344,
      "activations/layer7_attention_weight_max": 89.765625,
      "activations/layer7_attention_weight_min": -96.27571868896484,
      "activations/layer8_attention_weight_max": 39.51404571533203,
      "activations/layer8_attention_weight_min": -44.76194763183594,
      "activations/layer9_attention_weight_max": 29.97659683227539,
      "activations/layer9_attention_weight_min": -35.68085861206055,
      "epoch": 19.94,
      "learning_rate": 2.1656439393939393e-05,
      "loss": 2.7265,
      "step": 343100
    },
    {
      "activations/layer0_attention_weight_max": 15.64155387878418,
      "activations/layer0_attention_weight_min": -13.393231391906738,
      "activations/layer10_attention_weight_max": 32.69232940673828,
      "activations/layer10_attention_weight_min": -34.850425720214844,
      "activations/layer11_attention_weight_max": 32.80352783203125,
      "activations/layer11_attention_weight_min": -34.64756393432617,
      "activations/layer12_attention_weight_max": 28.108497619628906,
      "activations/layer12_attention_weight_min": -27.298377990722656,
      "activations/layer13_attention_weight_max": 43.91170120239258,
      "activations/layer13_attention_weight_min": -37.54612731933594,
      "activations/layer14_attention_weight_max": 46.328834533691406,
      "activations/layer14_attention_weight_min": -41.38735580444336,
      "activations/layer15_attention_weight_max": 44.446693420410156,
      "activations/layer15_attention_weight_min": -36.062931060791016,
      "activations/layer16_attention_weight_max": 32.077606201171875,
      "activations/layer16_attention_weight_min": -28.67237091064453,
      "activations/layer17_attention_weight_max": 31.02237892150879,
      "activations/layer17_attention_weight_min": -27.166881561279297,
      "activations/layer18_attention_weight_max": 36.11185073852539,
      "activations/layer18_attention_weight_min": -25.431560516357422,
      "activations/layer19_attention_weight_max": 39.13538360595703,
      "activations/layer19_attention_weight_min": -31.31145668029785,
      "activations/layer1_attention_weight_max": 15.511563301086426,
      "activations/layer1_attention_weight_min": -14.453963279724121,
      "activations/layer20_attention_weight_max": 32.69886016845703,
      "activations/layer20_attention_weight_min": -25.00893783569336,
      "activations/layer21_attention_weight_max": 33.220794677734375,
      "activations/layer21_attention_weight_min": -24.353593826293945,
      "activations/layer22_attention_weight_max": 51.4970817565918,
      "activations/layer22_attention_weight_min": -32.50419235229492,
      "activations/layer23_attention_weight_max": 39.537269592285156,
      "activations/layer23_attention_weight_min": -26.954113006591797,
      "activations/layer2_attention_weight_max": 31.029014587402344,
      "activations/layer2_attention_weight_min": -30.437923431396484,
      "activations/layer3_attention_weight_max": 99.69761657714844,
      "activations/layer3_attention_weight_min": -103.03741455078125,
      "activations/layer4_attention_weight_max": 103.10517883300781,
      "activations/layer4_attention_weight_min": -111.58707427978516,
      "activations/layer5_attention_weight_max": 52.035335540771484,
      "activations/layer5_attention_weight_min": -65.81330871582031,
      "activations/layer6_attention_weight_max": 44.7904167175293,
      "activations/layer6_attention_weight_min": -49.84751510620117,
      "activations/layer7_attention_weight_max": 96.72911834716797,
      "activations/layer7_attention_weight_min": -102.94268798828125,
      "activations/layer8_attention_weight_max": 41.10211181640625,
      "activations/layer8_attention_weight_min": -45.44724655151367,
      "activations/layer9_attention_weight_max": 30.743228912353516,
      "activations/layer9_attention_weight_min": -35.2130012512207,
      "epoch": 19.94,
      "learning_rate": 2.1637499999999997e-05,
      "loss": 2.724,
      "step": 343150
    },
    {
      "activations/layer0_attention_weight_max": 15.177824020385742,
      "activations/layer0_attention_weight_min": -12.14245891571045,
      "activations/layer10_attention_weight_max": 38.74969482421875,
      "activations/layer10_attention_weight_min": -36.95480728149414,
      "activations/layer11_attention_weight_max": 38.41893768310547,
      "activations/layer11_attention_weight_min": -37.70188903808594,
      "activations/layer12_attention_weight_max": 28.04722785949707,
      "activations/layer12_attention_weight_min": -27.925981521606445,
      "activations/layer13_attention_weight_max": 43.216697692871094,
      "activations/layer13_attention_weight_min": -41.7077751159668,
      "activations/layer14_attention_weight_max": 48.87594985961914,
      "activations/layer14_attention_weight_min": -44.406272888183594,
      "activations/layer15_attention_weight_max": 43.5636100769043,
      "activations/layer15_attention_weight_min": -39.796653747558594,
      "activations/layer16_attention_weight_max": 35.5080680847168,
      "activations/layer16_attention_weight_min": -28.774608612060547,
      "activations/layer17_attention_weight_max": 35.93388366699219,
      "activations/layer17_attention_weight_min": -31.19277572631836,
      "activations/layer18_attention_weight_max": 36.9455451965332,
      "activations/layer18_attention_weight_min": -28.63469886779785,
      "activations/layer19_attention_weight_max": 39.044986724853516,
      "activations/layer19_attention_weight_min": -31.593347549438477,
      "activations/layer1_attention_weight_max": 14.951858520507812,
      "activations/layer1_attention_weight_min": -13.621664047241211,
      "activations/layer20_attention_weight_max": 29.765684127807617,
      "activations/layer20_attention_weight_min": -26.09910774230957,
      "activations/layer21_attention_weight_max": 32.17563247680664,
      "activations/layer21_attention_weight_min": -26.429332733154297,
      "activations/layer22_attention_weight_max": 44.85853576660156,
      "activations/layer22_attention_weight_min": -32.8456916809082,
      "activations/layer23_attention_weight_max": 32.91913986206055,
      "activations/layer23_attention_weight_min": -29.230331420898438,
      "activations/layer2_attention_weight_max": 32.56922912597656,
      "activations/layer2_attention_weight_min": -31.7962646484375,
      "activations/layer3_attention_weight_max": 102.4839859008789,
      "activations/layer3_attention_weight_min": -105.46389770507812,
      "activations/layer4_attention_weight_max": 109.20023345947266,
      "activations/layer4_attention_weight_min": -112.85228729248047,
      "activations/layer5_attention_weight_max": 51.284297943115234,
      "activations/layer5_attention_weight_min": -68.00738525390625,
      "activations/layer6_attention_weight_max": 45.31125259399414,
      "activations/layer6_attention_weight_min": -48.793949127197266,
      "activations/layer7_attention_weight_max": 100.538818359375,
      "activations/layer7_attention_weight_min": -99.09719848632812,
      "activations/layer8_attention_weight_max": 45.206459045410156,
      "activations/layer8_attention_weight_min": -45.13472366333008,
      "activations/layer9_attention_weight_max": 33.910186767578125,
      "activations/layer9_attention_weight_min": -36.7806510925293,
      "epoch": 19.94,
      "learning_rate": 2.1618560606060606e-05,
      "loss": 2.7072,
      "step": 343200
    },
    {
      "activations/layer0_attention_weight_max": 15.894001960754395,
      "activations/layer0_attention_weight_min": -13.12454891204834,
      "activations/layer10_attention_weight_max": 33.66069030761719,
      "activations/layer10_attention_weight_min": -35.402748107910156,
      "activations/layer11_attention_weight_max": 33.15351104736328,
      "activations/layer11_attention_weight_min": -35.25436782836914,
      "activations/layer12_attention_weight_max": 25.501867294311523,
      "activations/layer12_attention_weight_min": -27.08663558959961,
      "activations/layer13_attention_weight_max": 38.44369125366211,
      "activations/layer13_attention_weight_min": -34.902801513671875,
      "activations/layer14_attention_weight_max": 40.6143913269043,
      "activations/layer14_attention_weight_min": -39.42814636230469,
      "activations/layer15_attention_weight_max": 36.82837677001953,
      "activations/layer15_attention_weight_min": -33.84709548950195,
      "activations/layer16_attention_weight_max": 28.7839412689209,
      "activations/layer16_attention_weight_min": -28.419206619262695,
      "activations/layer17_attention_weight_max": 30.890336990356445,
      "activations/layer17_attention_weight_min": -24.01041030883789,
      "activations/layer18_attention_weight_max": 32.316558837890625,
      "activations/layer18_attention_weight_min": -24.703182220458984,
      "activations/layer19_attention_weight_max": 33.458797454833984,
      "activations/layer19_attention_weight_min": -29.894201278686523,
      "activations/layer1_attention_weight_max": 16.76335334777832,
      "activations/layer1_attention_weight_min": -14.487004280090332,
      "activations/layer20_attention_weight_max": 26.206464767456055,
      "activations/layer20_attention_weight_min": -24.47686767578125,
      "activations/layer21_attention_weight_max": 28.610870361328125,
      "activations/layer21_attention_weight_min": -25.239700317382812,
      "activations/layer22_attention_weight_max": 37.87089157104492,
      "activations/layer22_attention_weight_min": -30.389633178710938,
      "activations/layer23_attention_weight_max": 29.949604034423828,
      "activations/layer23_attention_weight_min": -25.49047088623047,
      "activations/layer2_attention_weight_max": 31.941116333007812,
      "activations/layer2_attention_weight_min": -32.27228546142578,
      "activations/layer3_attention_weight_max": 103.85459899902344,
      "activations/layer3_attention_weight_min": -106.3923110961914,
      "activations/layer4_attention_weight_max": 113.94329071044922,
      "activations/layer4_attention_weight_min": -113.92567443847656,
      "activations/layer5_attention_weight_max": 51.331825256347656,
      "activations/layer5_attention_weight_min": -62.840087890625,
      "activations/layer6_attention_weight_max": 47.28706741333008,
      "activations/layer6_attention_weight_min": -47.68050003051758,
      "activations/layer7_attention_weight_max": 95.57906341552734,
      "activations/layer7_attention_weight_min": -102.35716247558594,
      "activations/layer8_attention_weight_max": 43.3872184753418,
      "activations/layer8_attention_weight_min": -47.027549743652344,
      "activations/layer9_attention_weight_max": 32.26478576660156,
      "activations/layer9_attention_weight_min": -35.374595642089844,
      "epoch": 19.94,
      "learning_rate": 2.1599621212121207e-05,
      "loss": 2.7108,
      "step": 343250
    },
    {
      "activations/layer0_attention_weight_max": 15.904974937438965,
      "activations/layer0_attention_weight_min": -12.545787811279297,
      "activations/layer10_attention_weight_max": 37.026771545410156,
      "activations/layer10_attention_weight_min": -36.39766311645508,
      "activations/layer11_attention_weight_max": 37.60335159301758,
      "activations/layer11_attention_weight_min": -37.7144660949707,
      "activations/layer12_attention_weight_max": 29.591642379760742,
      "activations/layer12_attention_weight_min": -26.586767196655273,
      "activations/layer13_attention_weight_max": 45.10799789428711,
      "activations/layer13_attention_weight_min": -35.831764221191406,
      "activations/layer14_attention_weight_max": 43.84821701049805,
      "activations/layer14_attention_weight_min": -37.06406021118164,
      "activations/layer15_attention_weight_max": 40.52092361450195,
      "activations/layer15_attention_weight_min": -36.589534759521484,
      "activations/layer16_attention_weight_max": 31.15595245361328,
      "activations/layer16_attention_weight_min": -27.83665657043457,
      "activations/layer17_attention_weight_max": 29.39211654663086,
      "activations/layer17_attention_weight_min": -26.432050704956055,
      "activations/layer18_attention_weight_max": 32.45199203491211,
      "activations/layer18_attention_weight_min": -27.504962921142578,
      "activations/layer19_attention_weight_max": 32.9030876159668,
      "activations/layer19_attention_weight_min": -33.243797302246094,
      "activations/layer1_attention_weight_max": 16.103899002075195,
      "activations/layer1_attention_weight_min": -14.319131851196289,
      "activations/layer20_attention_weight_max": 29.70501136779785,
      "activations/layer20_attention_weight_min": -24.885475158691406,
      "activations/layer21_attention_weight_max": 28.564800262451172,
      "activations/layer21_attention_weight_min": -24.958574295043945,
      "activations/layer22_attention_weight_max": 39.299224853515625,
      "activations/layer22_attention_weight_min": -33.98380661010742,
      "activations/layer23_attention_weight_max": 31.79615020751953,
      "activations/layer23_attention_weight_min": -27.127159118652344,
      "activations/layer2_attention_weight_max": 32.484134674072266,
      "activations/layer2_attention_weight_min": -32.5326042175293,
      "activations/layer3_attention_weight_max": 104.6466293334961,
      "activations/layer3_attention_weight_min": -104.702880859375,
      "activations/layer4_attention_weight_max": 109.61726379394531,
      "activations/layer4_attention_weight_min": -111.70088958740234,
      "activations/layer5_attention_weight_max": 52.246944427490234,
      "activations/layer5_attention_weight_min": -60.21931076049805,
      "activations/layer6_attention_weight_max": 46.50123596191406,
      "activations/layer6_attention_weight_min": -52.318599700927734,
      "activations/layer7_attention_weight_max": 100.96430969238281,
      "activations/layer7_attention_weight_min": -101.78011322021484,
      "activations/layer8_attention_weight_max": 43.667842864990234,
      "activations/layer8_attention_weight_min": -45.46795654296875,
      "activations/layer9_attention_weight_max": 34.8718147277832,
      "activations/layer9_attention_weight_min": -36.45045852661133,
      "epoch": 19.95,
      "learning_rate": 2.1580681818181815e-05,
      "loss": 2.7194,
      "step": 343300
    },
    {
      "activations/layer0_attention_weight_max": 15.51268196105957,
      "activations/layer0_attention_weight_min": -12.523419380187988,
      "activations/layer10_attention_weight_max": 35.379981994628906,
      "activations/layer10_attention_weight_min": -37.75701141357422,
      "activations/layer11_attention_weight_max": 36.44294738769531,
      "activations/layer11_attention_weight_min": -38.25151062011719,
      "activations/layer12_attention_weight_max": 29.219959259033203,
      "activations/layer12_attention_weight_min": -26.69525718688965,
      "activations/layer13_attention_weight_max": 42.43777084350586,
      "activations/layer13_attention_weight_min": -36.745460510253906,
      "activations/layer14_attention_weight_max": 50.06486129760742,
      "activations/layer14_attention_weight_min": -38.925594329833984,
      "activations/layer15_attention_weight_max": 43.917293548583984,
      "activations/layer15_attention_weight_min": -37.39870834350586,
      "activations/layer16_attention_weight_max": 33.96916198730469,
      "activations/layer16_attention_weight_min": -31.140148162841797,
      "activations/layer17_attention_weight_max": 32.7170295715332,
      "activations/layer17_attention_weight_min": -27.294666290283203,
      "activations/layer18_attention_weight_max": 34.94731521606445,
      "activations/layer18_attention_weight_min": -27.596651077270508,
      "activations/layer19_attention_weight_max": 40.89175033569336,
      "activations/layer19_attention_weight_min": -33.86606979370117,
      "activations/layer1_attention_weight_max": 16.996732711791992,
      "activations/layer1_attention_weight_min": -14.521698951721191,
      "activations/layer20_attention_weight_max": 32.423343658447266,
      "activations/layer20_attention_weight_min": -26.908395767211914,
      "activations/layer21_attention_weight_max": 31.055837631225586,
      "activations/layer21_attention_weight_min": -25.06505012512207,
      "activations/layer22_attention_weight_max": 46.8485107421875,
      "activations/layer22_attention_weight_min": -33.54720687866211,
      "activations/layer23_attention_weight_max": 40.287689208984375,
      "activations/layer23_attention_weight_min": -26.949077606201172,
      "activations/layer2_attention_weight_max": 32.11982345581055,
      "activations/layer2_attention_weight_min": -31.448551177978516,
      "activations/layer3_attention_weight_max": 106.83700561523438,
      "activations/layer3_attention_weight_min": -103.6556396484375,
      "activations/layer4_attention_weight_max": 112.49642181396484,
      "activations/layer4_attention_weight_min": -115.6272964477539,
      "activations/layer5_attention_weight_max": 52.61619567871094,
      "activations/layer5_attention_weight_min": -63.45090866088867,
      "activations/layer6_attention_weight_max": 46.91028594970703,
      "activations/layer6_attention_weight_min": -51.326663970947266,
      "activations/layer7_attention_weight_max": 100.67959594726562,
      "activations/layer7_attention_weight_min": -102.4483871459961,
      "activations/layer8_attention_weight_max": 45.4073486328125,
      "activations/layer8_attention_weight_min": -50.03605651855469,
      "activations/layer9_attention_weight_max": 33.82041549682617,
      "activations/layer9_attention_weight_min": -36.51606750488281,
      "epoch": 19.95,
      "learning_rate": 2.1561742424242424e-05,
      "loss": 2.7374,
      "step": 343350
    },
    {
      "activations/layer0_attention_weight_max": 16.143917083740234,
      "activations/layer0_attention_weight_min": -12.652655601501465,
      "activations/layer10_attention_weight_max": 41.056880950927734,
      "activations/layer10_attention_weight_min": -38.145957946777344,
      "activations/layer11_attention_weight_max": 42.35169982910156,
      "activations/layer11_attention_weight_min": -39.001895904541016,
      "activations/layer12_attention_weight_max": 31.03464126586914,
      "activations/layer12_attention_weight_min": -27.832406997680664,
      "activations/layer13_attention_weight_max": 44.70090103149414,
      "activations/layer13_attention_weight_min": -36.366233825683594,
      "activations/layer14_attention_weight_max": 52.10797882080078,
      "activations/layer14_attention_weight_min": -39.76615905761719,
      "activations/layer15_attention_weight_max": 52.342010498046875,
      "activations/layer15_attention_weight_min": -43.048484802246094,
      "activations/layer16_attention_weight_max": 37.999942779541016,
      "activations/layer16_attention_weight_min": -28.565399169921875,
      "activations/layer17_attention_weight_max": 31.4952392578125,
      "activations/layer17_attention_weight_min": -27.88850212097168,
      "activations/layer18_attention_weight_max": 30.193857192993164,
      "activations/layer18_attention_weight_min": -26.989234924316406,
      "activations/layer19_attention_weight_max": 42.1381950378418,
      "activations/layer19_attention_weight_min": -33.715660095214844,
      "activations/layer1_attention_weight_max": 16.20501708984375,
      "activations/layer1_attention_weight_min": -13.903524398803711,
      "activations/layer20_attention_weight_max": 33.63458251953125,
      "activations/layer20_attention_weight_min": -26.262645721435547,
      "activations/layer21_attention_weight_max": 29.694339752197266,
      "activations/layer21_attention_weight_min": -26.218523025512695,
      "activations/layer22_attention_weight_max": 39.59857177734375,
      "activations/layer22_attention_weight_min": -32.74821090698242,
      "activations/layer23_attention_weight_max": 33.903602600097656,
      "activations/layer23_attention_weight_min": -25.9292049407959,
      "activations/layer2_attention_weight_max": 34.92041778564453,
      "activations/layer2_attention_weight_min": -33.61953353881836,
      "activations/layer3_attention_weight_max": 102.4144515991211,
      "activations/layer3_attention_weight_min": -104.45088958740234,
      "activations/layer4_attention_weight_max": 107.4773941040039,
      "activations/layer4_attention_weight_min": -108.85259246826172,
      "activations/layer5_attention_weight_max": 52.38840866088867,
      "activations/layer5_attention_weight_min": -59.962459564208984,
      "activations/layer6_attention_weight_max": 43.77021789550781,
      "activations/layer6_attention_weight_min": -46.283653259277344,
      "activations/layer7_attention_weight_max": 103.35652160644531,
      "activations/layer7_attention_weight_min": -97.70619201660156,
      "activations/layer8_attention_weight_max": 47.39950942993164,
      "activations/layer8_attention_weight_min": -46.506019592285156,
      "activations/layer9_attention_weight_max": 36.536216735839844,
      "activations/layer9_attention_weight_min": -36.821102142333984,
      "epoch": 19.95,
      "learning_rate": 2.154280303030303e-05,
      "loss": 2.7379,
      "step": 343400
    },
    {
      "activations/layer0_attention_weight_max": 15.833776473999023,
      "activations/layer0_attention_weight_min": -14.146233558654785,
      "activations/layer10_attention_weight_max": 31.84117317199707,
      "activations/layer10_attention_weight_min": -34.26876449584961,
      "activations/layer11_attention_weight_max": 33.06898498535156,
      "activations/layer11_attention_weight_min": -33.60795593261719,
      "activations/layer12_attention_weight_max": 30.104204177856445,
      "activations/layer12_attention_weight_min": -27.868568420410156,
      "activations/layer13_attention_weight_max": 45.124507904052734,
      "activations/layer13_attention_weight_min": -38.67393112182617,
      "activations/layer14_attention_weight_max": 45.403350830078125,
      "activations/layer14_attention_weight_min": -38.57893753051758,
      "activations/layer15_attention_weight_max": 41.879852294921875,
      "activations/layer15_attention_weight_min": -36.293582916259766,
      "activations/layer16_attention_weight_max": 30.43709373474121,
      "activations/layer16_attention_weight_min": -29.260019302368164,
      "activations/layer17_attention_weight_max": 32.28687286376953,
      "activations/layer17_attention_weight_min": -26.395601272583008,
      "activations/layer18_attention_weight_max": 35.000701904296875,
      "activations/layer18_attention_weight_min": -25.21380615234375,
      "activations/layer19_attention_weight_max": 34.79877471923828,
      "activations/layer19_attention_weight_min": -32.168766021728516,
      "activations/layer1_attention_weight_max": 15.750280380249023,
      "activations/layer1_attention_weight_min": -14.311150550842285,
      "activations/layer20_attention_weight_max": 30.40194320678711,
      "activations/layer20_attention_weight_min": -26.002214431762695,
      "activations/layer21_attention_weight_max": 31.83331298828125,
      "activations/layer21_attention_weight_min": -25.297239303588867,
      "activations/layer22_attention_weight_max": 43.60736846923828,
      "activations/layer22_attention_weight_min": -32.217796325683594,
      "activations/layer23_attention_weight_max": 30.864559173583984,
      "activations/layer23_attention_weight_min": -25.39629364013672,
      "activations/layer2_attention_weight_max": 32.98432922363281,
      "activations/layer2_attention_weight_min": -32.14739990234375,
      "activations/layer3_attention_weight_max": 99.71891784667969,
      "activations/layer3_attention_weight_min": -100.04332733154297,
      "activations/layer4_attention_weight_max": 106.37959289550781,
      "activations/layer4_attention_weight_min": -105.9541244506836,
      "activations/layer5_attention_weight_max": 50.335716247558594,
      "activations/layer5_attention_weight_min": -61.916908264160156,
      "activations/layer6_attention_weight_max": 43.67197036743164,
      "activations/layer6_attention_weight_min": -47.20706558227539,
      "activations/layer7_attention_weight_max": 91.48136138916016,
      "activations/layer7_attention_weight_min": -98.09806823730469,
      "activations/layer8_attention_weight_max": 41.26337814331055,
      "activations/layer8_attention_weight_min": -45.23755645751953,
      "activations/layer9_attention_weight_max": 30.90439796447754,
      "activations/layer9_attention_weight_min": -34.04764175415039,
      "epoch": 19.96,
      "learning_rate": 2.1523863636363637e-05,
      "loss": 2.7056,
      "step": 343450
    },
    {
      "activations/layer0_attention_weight_max": 16.0127010345459,
      "activations/layer0_attention_weight_min": -12.572197914123535,
      "activations/layer10_attention_weight_max": 37.33567428588867,
      "activations/layer10_attention_weight_min": -36.54840850830078,
      "activations/layer11_attention_weight_max": 37.64876174926758,
      "activations/layer11_attention_weight_min": -35.6373176574707,
      "activations/layer12_attention_weight_max": 27.49631690979004,
      "activations/layer12_attention_weight_min": -28.162067413330078,
      "activations/layer13_attention_weight_max": 40.5124397277832,
      "activations/layer13_attention_weight_min": -37.60504150390625,
      "activations/layer14_attention_weight_max": 41.533058166503906,
      "activations/layer14_attention_weight_min": -39.462894439697266,
      "activations/layer15_attention_weight_max": 41.59833908081055,
      "activations/layer15_attention_weight_min": -36.861106872558594,
      "activations/layer16_attention_weight_max": 32.137027740478516,
      "activations/layer16_attention_weight_min": -28.644792556762695,
      "activations/layer17_attention_weight_max": 33.55517578125,
      "activations/layer17_attention_weight_min": -25.59372901916504,
      "activations/layer18_attention_weight_max": 34.60014724731445,
      "activations/layer18_attention_weight_min": -26.055904388427734,
      "activations/layer19_attention_weight_max": 36.0486946105957,
      "activations/layer19_attention_weight_min": -31.2634334564209,
      "activations/layer1_attention_weight_max": 15.762076377868652,
      "activations/layer1_attention_weight_min": -15.389555931091309,
      "activations/layer20_attention_weight_max": 31.249298095703125,
      "activations/layer20_attention_weight_min": -26.74467658996582,
      "activations/layer21_attention_weight_max": 30.117961883544922,
      "activations/layer21_attention_weight_min": -25.946256637573242,
      "activations/layer22_attention_weight_max": 44.49680709838867,
      "activations/layer22_attention_weight_min": -31.042221069335938,
      "activations/layer23_attention_weight_max": 32.41529846191406,
      "activations/layer23_attention_weight_min": -27.28600311279297,
      "activations/layer2_attention_weight_max": 32.126426696777344,
      "activations/layer2_attention_weight_min": -30.64947509765625,
      "activations/layer3_attention_weight_max": 100.43339538574219,
      "activations/layer3_attention_weight_min": -98.94364929199219,
      "activations/layer4_attention_weight_max": 102.70814514160156,
      "activations/layer4_attention_weight_min": -106.7650375366211,
      "activations/layer5_attention_weight_max": 52.62398910522461,
      "activations/layer5_attention_weight_min": -58.609100341796875,
      "activations/layer6_attention_weight_max": 45.37688446044922,
      "activations/layer6_attention_weight_min": -47.92485809326172,
      "activations/layer7_attention_weight_max": 97.81391906738281,
      "activations/layer7_attention_weight_min": -104.85342407226562,
      "activations/layer8_attention_weight_max": 43.73828125,
      "activations/layer8_attention_weight_min": -46.28689193725586,
      "activations/layer9_attention_weight_max": 33.682594299316406,
      "activations/layer9_attention_weight_min": -35.9626350402832,
      "epoch": 19.96,
      "learning_rate": 2.1504924242424242e-05,
      "loss": 2.7261,
      "step": 343500
    },
    {
      "activations/layer0_attention_weight_max": 17.248350143432617,
      "activations/layer0_attention_weight_min": -13.776586532592773,
      "activations/layer10_attention_weight_max": 33.809749603271484,
      "activations/layer10_attention_weight_min": -34.17962646484375,
      "activations/layer11_attention_weight_max": 32.805904388427734,
      "activations/layer11_attention_weight_min": -34.82775115966797,
      "activations/layer12_attention_weight_max": 27.775453567504883,
      "activations/layer12_attention_weight_min": -26.981382369995117,
      "activations/layer13_attention_weight_max": 41.378509521484375,
      "activations/layer13_attention_weight_min": -35.973506927490234,
      "activations/layer14_attention_weight_max": 43.947898864746094,
      "activations/layer14_attention_weight_min": -41.05745315551758,
      "activations/layer15_attention_weight_max": 40.64456558227539,
      "activations/layer15_attention_weight_min": -36.95885467529297,
      "activations/layer16_attention_weight_max": 31.43829345703125,
      "activations/layer16_attention_weight_min": -30.27873992919922,
      "activations/layer17_attention_weight_max": 33.22815704345703,
      "activations/layer17_attention_weight_min": -27.23969841003418,
      "activations/layer18_attention_weight_max": 34.415252685546875,
      "activations/layer18_attention_weight_min": -26.426950454711914,
      "activations/layer19_attention_weight_max": 37.78092575073242,
      "activations/layer19_attention_weight_min": -31.788089752197266,
      "activations/layer1_attention_weight_max": 15.738292694091797,
      "activations/layer1_attention_weight_min": -14.497050285339355,
      "activations/layer20_attention_weight_max": 31.95878791809082,
      "activations/layer20_attention_weight_min": -27.246408462524414,
      "activations/layer21_attention_weight_max": 31.373931884765625,
      "activations/layer21_attention_weight_min": -25.270959854125977,
      "activations/layer22_attention_weight_max": 41.41367721557617,
      "activations/layer22_attention_weight_min": -30.493196487426758,
      "activations/layer23_attention_weight_max": 31.983226776123047,
      "activations/layer23_attention_weight_min": -24.093441009521484,
      "activations/layer2_attention_weight_max": 30.533958435058594,
      "activations/layer2_attention_weight_min": -29.50653839111328,
      "activations/layer3_attention_weight_max": 100.1305923461914,
      "activations/layer3_attention_weight_min": -97.39095306396484,
      "activations/layer4_attention_weight_max": 109.34081268310547,
      "activations/layer4_attention_weight_min": -110.44819641113281,
      "activations/layer5_attention_weight_max": 48.52305603027344,
      "activations/layer5_attention_weight_min": -61.678462982177734,
      "activations/layer6_attention_weight_max": 42.93990707397461,
      "activations/layer6_attention_weight_min": -46.763729095458984,
      "activations/layer7_attention_weight_max": 89.16141510009766,
      "activations/layer7_attention_weight_min": -94.8920669555664,
      "activations/layer8_attention_weight_max": 40.80541229248047,
      "activations/layer8_attention_weight_min": -43.0307731628418,
      "activations/layer9_attention_weight_max": 31.683622360229492,
      "activations/layer9_attention_weight_min": -32.234474182128906,
      "epoch": 19.96,
      "learning_rate": 2.1485984848484847e-05,
      "loss": 2.7221,
      "step": 343550
    },
    {
      "activations/layer0_attention_weight_max": 15.956785202026367,
      "activations/layer0_attention_weight_min": -12.498775482177734,
      "activations/layer10_attention_weight_max": 31.69918441772461,
      "activations/layer10_attention_weight_min": -34.48198318481445,
      "activations/layer11_attention_weight_max": 31.242183685302734,
      "activations/layer11_attention_weight_min": -33.73752975463867,
      "activations/layer12_attention_weight_max": 26.62760353088379,
      "activations/layer12_attention_weight_min": -25.93253517150879,
      "activations/layer13_attention_weight_max": 38.742431640625,
      "activations/layer13_attention_weight_min": -34.74282455444336,
      "activations/layer14_attention_weight_max": 40.93132781982422,
      "activations/layer14_attention_weight_min": -37.22684097290039,
      "activations/layer15_attention_weight_max": 38.200721740722656,
      "activations/layer15_attention_weight_min": -35.438446044921875,
      "activations/layer16_attention_weight_max": 28.427705764770508,
      "activations/layer16_attention_weight_min": -28.27332305908203,
      "activations/layer17_attention_weight_max": 28.700769424438477,
      "activations/layer17_attention_weight_min": -24.95325469970703,
      "activations/layer18_attention_weight_max": 32.59366989135742,
      "activations/layer18_attention_weight_min": -25.648963928222656,
      "activations/layer19_attention_weight_max": 32.680721282958984,
      "activations/layer19_attention_weight_min": -32.5440559387207,
      "activations/layer1_attention_weight_max": 15.26749324798584,
      "activations/layer1_attention_weight_min": -13.42193603515625,
      "activations/layer20_attention_weight_max": 29.78191566467285,
      "activations/layer20_attention_weight_min": -26.782005310058594,
      "activations/layer21_attention_weight_max": 30.89620018005371,
      "activations/layer21_attention_weight_min": -25.34442710876465,
      "activations/layer22_attention_weight_max": 45.62221908569336,
      "activations/layer22_attention_weight_min": -32.00318145751953,
      "activations/layer23_attention_weight_max": 30.482845306396484,
      "activations/layer23_attention_weight_min": -25.74445343017578,
      "activations/layer2_attention_weight_max": 30.322105407714844,
      "activations/layer2_attention_weight_min": -30.343223571777344,
      "activations/layer3_attention_weight_max": 95.6976318359375,
      "activations/layer3_attention_weight_min": -95.44454193115234,
      "activations/layer4_attention_weight_max": 101.05940246582031,
      "activations/layer4_attention_weight_min": -103.28871154785156,
      "activations/layer5_attention_weight_max": 52.508697509765625,
      "activations/layer5_attention_weight_min": -68.80236053466797,
      "activations/layer6_attention_weight_max": 43.56296920776367,
      "activations/layer6_attention_weight_min": -46.73068618774414,
      "activations/layer7_attention_weight_max": 89.80915832519531,
      "activations/layer7_attention_weight_min": -94.71394348144531,
      "activations/layer8_attention_weight_max": 40.70549774169922,
      "activations/layer8_attention_weight_min": -42.77287673950195,
      "activations/layer9_attention_weight_max": 31.142230987548828,
      "activations/layer9_attention_weight_min": -33.18452835083008,
      "epoch": 19.97,
      "learning_rate": 2.146704545454545e-05,
      "loss": 2.7166,
      "step": 343600
    },
    {
      "activations/layer0_attention_weight_max": 16.074724197387695,
      "activations/layer0_attention_weight_min": -14.954652786254883,
      "activations/layer10_attention_weight_max": 33.694976806640625,
      "activations/layer10_attention_weight_min": -34.65027618408203,
      "activations/layer11_attention_weight_max": 31.51313591003418,
      "activations/layer11_attention_weight_min": -33.51225662231445,
      "activations/layer12_attention_weight_max": 24.25457191467285,
      "activations/layer12_attention_weight_min": -25.613582611083984,
      "activations/layer13_attention_weight_max": 36.124267578125,
      "activations/layer13_attention_weight_min": -35.04316329956055,
      "activations/layer14_attention_weight_max": 39.383583068847656,
      "activations/layer14_attention_weight_min": -35.0866584777832,
      "activations/layer15_attention_weight_max": 37.588523864746094,
      "activations/layer15_attention_weight_min": -34.77907180786133,
      "activations/layer16_attention_weight_max": 29.046924591064453,
      "activations/layer16_attention_weight_min": -28.125415802001953,
      "activations/layer17_attention_weight_max": 28.7918701171875,
      "activations/layer17_attention_weight_min": -25.55973243713379,
      "activations/layer18_attention_weight_max": 31.210979461669922,
      "activations/layer18_attention_weight_min": -27.094356536865234,
      "activations/layer19_attention_weight_max": 33.8008918762207,
      "activations/layer19_attention_weight_min": -29.279071807861328,
      "activations/layer1_attention_weight_max": 16.233362197875977,
      "activations/layer1_attention_weight_min": -14.460333824157715,
      "activations/layer20_attention_weight_max": 29.381633758544922,
      "activations/layer20_attention_weight_min": -24.6002140045166,
      "activations/layer21_attention_weight_max": 27.512081146240234,
      "activations/layer21_attention_weight_min": -23.699016571044922,
      "activations/layer22_attention_weight_max": 38.30317687988281,
      "activations/layer22_attention_weight_min": -29.630334854125977,
      "activations/layer23_attention_weight_max": 30.754531860351562,
      "activations/layer23_attention_weight_min": -26.767898559570312,
      "activations/layer2_attention_weight_max": 31.622081756591797,
      "activations/layer2_attention_weight_min": -31.148181915283203,
      "activations/layer3_attention_weight_max": 100.03459930419922,
      "activations/layer3_attention_weight_min": -99.12185668945312,
      "activations/layer4_attention_weight_max": 104.99983978271484,
      "activations/layer4_attention_weight_min": -106.79772186279297,
      "activations/layer5_attention_weight_max": 48.212013244628906,
      "activations/layer5_attention_weight_min": -59.85309600830078,
      "activations/layer6_attention_weight_max": 43.38752365112305,
      "activations/layer6_attention_weight_min": -45.754337310791016,
      "activations/layer7_attention_weight_max": 94.61971282958984,
      "activations/layer7_attention_weight_min": -93.43312072753906,
      "activations/layer8_attention_weight_max": 43.35664749145508,
      "activations/layer8_attention_weight_min": -47.940406799316406,
      "activations/layer9_attention_weight_max": 32.329044342041016,
      "activations/layer9_attention_weight_min": -37.25349807739258,
      "epoch": 19.97,
      "learning_rate": 2.144810606060606e-05,
      "loss": 2.7346,
      "step": 343650
    },
    {
      "activations/layer0_attention_weight_max": 15.930492401123047,
      "activations/layer0_attention_weight_min": -13.119415283203125,
      "activations/layer10_attention_weight_max": 33.90618896484375,
      "activations/layer10_attention_weight_min": -34.241668701171875,
      "activations/layer11_attention_weight_max": 35.10844802856445,
      "activations/layer11_attention_weight_min": -34.86297607421875,
      "activations/layer12_attention_weight_max": 25.238248825073242,
      "activations/layer12_attention_weight_min": -29.890233993530273,
      "activations/layer13_attention_weight_max": 39.223609924316406,
      "activations/layer13_attention_weight_min": -34.962646484375,
      "activations/layer14_attention_weight_max": 40.40979766845703,
      "activations/layer14_attention_weight_min": -37.93470764160156,
      "activations/layer15_attention_weight_max": 39.439517974853516,
      "activations/layer15_attention_weight_min": -34.92779541015625,
      "activations/layer16_attention_weight_max": 29.024044036865234,
      "activations/layer16_attention_weight_min": -28.127212524414062,
      "activations/layer17_attention_weight_max": 31.16908073425293,
      "activations/layer17_attention_weight_min": -27.498703002929688,
      "activations/layer18_attention_weight_max": 32.766292572021484,
      "activations/layer18_attention_weight_min": -26.197107315063477,
      "activations/layer19_attention_weight_max": 34.453182220458984,
      "activations/layer19_attention_weight_min": -36.38147735595703,
      "activations/layer1_attention_weight_max": 15.167459487915039,
      "activations/layer1_attention_weight_min": -13.992715835571289,
      "activations/layer20_attention_weight_max": 30.349803924560547,
      "activations/layer20_attention_weight_min": -27.998411178588867,
      "activations/layer21_attention_weight_max": 31.44378662109375,
      "activations/layer21_attention_weight_min": -25.5831298828125,
      "activations/layer22_attention_weight_max": 42.30751419067383,
      "activations/layer22_attention_weight_min": -32.75644302368164,
      "activations/layer23_attention_weight_max": 31.817474365234375,
      "activations/layer23_attention_weight_min": -26.98188591003418,
      "activations/layer2_attention_weight_max": 32.253936767578125,
      "activations/layer2_attention_weight_min": -32.06078338623047,
      "activations/layer3_attention_weight_max": 101.85826110839844,
      "activations/layer3_attention_weight_min": -101.03816223144531,
      "activations/layer4_attention_weight_max": 108.49226379394531,
      "activations/layer4_attention_weight_min": -107.49287414550781,
      "activations/layer5_attention_weight_max": 51.130367279052734,
      "activations/layer5_attention_weight_min": -59.485206604003906,
      "activations/layer6_attention_weight_max": 43.46361541748047,
      "activations/layer6_attention_weight_min": -46.803321838378906,
      "activations/layer7_attention_weight_max": 99.61942291259766,
      "activations/layer7_attention_weight_min": -95.60121154785156,
      "activations/layer8_attention_weight_max": 41.33795928955078,
      "activations/layer8_attention_weight_min": -43.10365676879883,
      "activations/layer9_attention_weight_max": 30.674375534057617,
      "activations/layer9_attention_weight_min": -36.52403259277344,
      "epoch": 19.97,
      "learning_rate": 2.1429166666666665e-05,
      "loss": 2.735,
      "step": 343700
    },
    {
      "activations/layer0_attention_weight_max": 15.628296852111816,
      "activations/layer0_attention_weight_min": -12.843796730041504,
      "activations/layer10_attention_weight_max": 35.34085464477539,
      "activations/layer10_attention_weight_min": -33.18742752075195,
      "activations/layer11_attention_weight_max": 35.05804443359375,
      "activations/layer11_attention_weight_min": -33.94557571411133,
      "activations/layer12_attention_weight_max": 27.183570861816406,
      "activations/layer12_attention_weight_min": -26.063068389892578,
      "activations/layer13_attention_weight_max": 41.20659255981445,
      "activations/layer13_attention_weight_min": -34.93726348876953,
      "activations/layer14_attention_weight_max": 42.15335464477539,
      "activations/layer14_attention_weight_min": -38.7733039855957,
      "activations/layer15_attention_weight_max": 42.14225769042969,
      "activations/layer15_attention_weight_min": -37.270931243896484,
      "activations/layer16_attention_weight_max": 31.3131103515625,
      "activations/layer16_attention_weight_min": -32.48978042602539,
      "activations/layer17_attention_weight_max": 32.33769607543945,
      "activations/layer17_attention_weight_min": -29.481510162353516,
      "activations/layer18_attention_weight_max": 34.165504455566406,
      "activations/layer18_attention_weight_min": -28.03175163269043,
      "activations/layer19_attention_weight_max": 40.06222152709961,
      "activations/layer19_attention_weight_min": -33.81476593017578,
      "activations/layer1_attention_weight_max": 17.95793914794922,
      "activations/layer1_attention_weight_min": -16.19655418395996,
      "activations/layer20_attention_weight_max": 37.135738372802734,
      "activations/layer20_attention_weight_min": -26.849552154541016,
      "activations/layer21_attention_weight_max": 33.83906173706055,
      "activations/layer21_attention_weight_min": -26.99249839782715,
      "activations/layer22_attention_weight_max": 48.936729431152344,
      "activations/layer22_attention_weight_min": -36.14049530029297,
      "activations/layer23_attention_weight_max": 39.696044921875,
      "activations/layer23_attention_weight_min": -26.483745574951172,
      "activations/layer2_attention_weight_max": 33.87333679199219,
      "activations/layer2_attention_weight_min": -31.89789581298828,
      "activations/layer3_attention_weight_max": 104.73284912109375,
      "activations/layer3_attention_weight_min": -102.70914459228516,
      "activations/layer4_attention_weight_max": 113.36103820800781,
      "activations/layer4_attention_weight_min": -113.4812240600586,
      "activations/layer5_attention_weight_max": 52.474205017089844,
      "activations/layer5_attention_weight_min": -63.236995697021484,
      "activations/layer6_attention_weight_max": 43.421669006347656,
      "activations/layer6_attention_weight_min": -46.59103775024414,
      "activations/layer7_attention_weight_max": 95.01986694335938,
      "activations/layer7_attention_weight_min": -94.9592056274414,
      "activations/layer8_attention_weight_max": 42.35905075073242,
      "activations/layer8_attention_weight_min": -43.96915054321289,
      "activations/layer9_attention_weight_max": 32.15154266357422,
      "activations/layer9_attention_weight_min": -33.96055603027344,
      "epoch": 19.97,
      "learning_rate": 2.1410227272727273e-05,
      "loss": 2.7222,
      "step": 343750
    },
    {
      "activations/layer0_attention_weight_max": 18.13847541809082,
      "activations/layer0_attention_weight_min": -12.367390632629395,
      "activations/layer10_attention_weight_max": 35.74977493286133,
      "activations/layer10_attention_weight_min": -35.717533111572266,
      "activations/layer11_attention_weight_max": 35.554718017578125,
      "activations/layer11_attention_weight_min": -36.073631286621094,
      "activations/layer12_attention_weight_max": 25.774818420410156,
      "activations/layer12_attention_weight_min": -27.08953094482422,
      "activations/layer13_attention_weight_max": 37.75993728637695,
      "activations/layer13_attention_weight_min": -37.95891189575195,
      "activations/layer14_attention_weight_max": 44.01005172729492,
      "activations/layer14_attention_weight_min": -39.377445220947266,
      "activations/layer15_attention_weight_max": 38.2150764465332,
      "activations/layer15_attention_weight_min": -38.19221496582031,
      "activations/layer16_attention_weight_max": 28.06261444091797,
      "activations/layer16_attention_weight_min": -28.51432228088379,
      "activations/layer17_attention_weight_max": 27.71274757385254,
      "activations/layer17_attention_weight_min": -26.409574508666992,
      "activations/layer18_attention_weight_max": 28.48678207397461,
      "activations/layer18_attention_weight_min": -27.120586395263672,
      "activations/layer19_attention_weight_max": 31.346097946166992,
      "activations/layer19_attention_weight_min": -34.0778923034668,
      "activations/layer1_attention_weight_max": 16.08478355407715,
      "activations/layer1_attention_weight_min": -14.592049598693848,
      "activations/layer20_attention_weight_max": 27.195045471191406,
      "activations/layer20_attention_weight_min": -26.686864852905273,
      "activations/layer21_attention_weight_max": 28.725814819335938,
      "activations/layer21_attention_weight_min": -27.180442810058594,
      "activations/layer22_attention_weight_max": 39.08652114868164,
      "activations/layer22_attention_weight_min": -32.90046310424805,
      "activations/layer23_attention_weight_max": 33.102333068847656,
      "activations/layer23_attention_weight_min": -26.045082092285156,
      "activations/layer2_attention_weight_max": 33.50990676879883,
      "activations/layer2_attention_weight_min": -31.837230682373047,
      "activations/layer3_attention_weight_max": 101.04737091064453,
      "activations/layer3_attention_weight_min": -100.24243927001953,
      "activations/layer4_attention_weight_max": 105.6572494506836,
      "activations/layer4_attention_weight_min": -108.0718994140625,
      "activations/layer5_attention_weight_max": 50.05107879638672,
      "activations/layer5_attention_weight_min": -64.72291564941406,
      "activations/layer6_attention_weight_max": 46.65359115600586,
      "activations/layer6_attention_weight_min": -46.841251373291016,
      "activations/layer7_attention_weight_max": 95.76111602783203,
      "activations/layer7_attention_weight_min": -102.31566619873047,
      "activations/layer8_attention_weight_max": 44.076026916503906,
      "activations/layer8_attention_weight_min": -46.387874603271484,
      "activations/layer9_attention_weight_max": 33.42926025390625,
      "activations/layer9_attention_weight_min": -34.81504440307617,
      "epoch": 19.98,
      "learning_rate": 2.1391287878787874e-05,
      "loss": 2.7168,
      "step": 343800
    },
    {
      "activations/layer0_attention_weight_max": 15.547993659973145,
      "activations/layer0_attention_weight_min": -12.813451766967773,
      "activations/layer10_attention_weight_max": 33.01051712036133,
      "activations/layer10_attention_weight_min": -33.19866180419922,
      "activations/layer11_attention_weight_max": 33.27873992919922,
      "activations/layer11_attention_weight_min": -33.722049713134766,
      "activations/layer12_attention_weight_max": 25.102497100830078,
      "activations/layer12_attention_weight_min": -28.720748901367188,
      "activations/layer13_attention_weight_max": 37.99732971191406,
      "activations/layer13_attention_weight_min": -37.6152229309082,
      "activations/layer14_attention_weight_max": 39.705284118652344,
      "activations/layer14_attention_weight_min": -39.817081451416016,
      "activations/layer15_attention_weight_max": 37.0775260925293,
      "activations/layer15_attention_weight_min": -39.58595275878906,
      "activations/layer16_attention_weight_max": 28.929624557495117,
      "activations/layer16_attention_weight_min": -28.09624671936035,
      "activations/layer17_attention_weight_max": 28.95142364501953,
      "activations/layer17_attention_weight_min": -27.063247680664062,
      "activations/layer18_attention_weight_max": 32.37450408935547,
      "activations/layer18_attention_weight_min": -30.224008560180664,
      "activations/layer19_attention_weight_max": 33.23838806152344,
      "activations/layer19_attention_weight_min": -32.50461196899414,
      "activations/layer1_attention_weight_max": 16.675382614135742,
      "activations/layer1_attention_weight_min": -14.974715232849121,
      "activations/layer20_attention_weight_max": 29.43682289123535,
      "activations/layer20_attention_weight_min": -25.300247192382812,
      "activations/layer21_attention_weight_max": 28.251989364624023,
      "activations/layer21_attention_weight_min": -25.950151443481445,
      "activations/layer22_attention_weight_max": 44.886837005615234,
      "activations/layer22_attention_weight_min": -32.645835876464844,
      "activations/layer23_attention_weight_max": 33.54789733886719,
      "activations/layer23_attention_weight_min": -27.439285278320312,
      "activations/layer2_attention_weight_max": 32.85256576538086,
      "activations/layer2_attention_weight_min": -30.719280242919922,
      "activations/layer3_attention_weight_max": 98.66725158691406,
      "activations/layer3_attention_weight_min": -101.31483459472656,
      "activations/layer4_attention_weight_max": 104.508544921875,
      "activations/layer4_attention_weight_min": -108.86888885498047,
      "activations/layer5_attention_weight_max": 49.6523551940918,
      "activations/layer5_attention_weight_min": -61.504390716552734,
      "activations/layer6_attention_weight_max": 44.228939056396484,
      "activations/layer6_attention_weight_min": -46.013309478759766,
      "activations/layer7_attention_weight_max": 87.54707336425781,
      "activations/layer7_attention_weight_min": -94.61917877197266,
      "activations/layer8_attention_weight_max": 39.11965560913086,
      "activations/layer8_attention_weight_min": -43.44719696044922,
      "activations/layer9_attention_weight_max": 30.861188888549805,
      "activations/layer9_attention_weight_min": -35.113441467285156,
      "epoch": 19.98,
      "learning_rate": 2.1372348484848483e-05,
      "loss": 2.7407,
      "step": 343850
    },
    {
      "activations/layer0_attention_weight_max": 16.156291961669922,
      "activations/layer0_attention_weight_min": -13.696836471557617,
      "activations/layer10_attention_weight_max": 34.02901077270508,
      "activations/layer10_attention_weight_min": -37.334495544433594,
      "activations/layer11_attention_weight_max": 33.41326141357422,
      "activations/layer11_attention_weight_min": -36.323238372802734,
      "activations/layer12_attention_weight_max": 27.987642288208008,
      "activations/layer12_attention_weight_min": -27.433021545410156,
      "activations/layer13_attention_weight_max": 40.40867233276367,
      "activations/layer13_attention_weight_min": -37.4926643371582,
      "activations/layer14_attention_weight_max": 44.81948471069336,
      "activations/layer14_attention_weight_min": -40.092628479003906,
      "activations/layer15_attention_weight_max": 40.33435821533203,
      "activations/layer15_attention_weight_min": -35.9683837890625,
      "activations/layer16_attention_weight_max": 30.288793563842773,
      "activations/layer16_attention_weight_min": -29.611879348754883,
      "activations/layer17_attention_weight_max": 31.16286277770996,
      "activations/layer17_attention_weight_min": -25.60336685180664,
      "activations/layer18_attention_weight_max": 30.781291961669922,
      "activations/layer18_attention_weight_min": -25.993064880371094,
      "activations/layer19_attention_weight_max": 33.242767333984375,
      "activations/layer19_attention_weight_min": -32.1034049987793,
      "activations/layer1_attention_weight_max": 16.535110473632812,
      "activations/layer1_attention_weight_min": -14.86142635345459,
      "activations/layer20_attention_weight_max": 29.285308837890625,
      "activations/layer20_attention_weight_min": -24.870525360107422,
      "activations/layer21_attention_weight_max": 28.194549560546875,
      "activations/layer21_attention_weight_min": -25.457473754882812,
      "activations/layer22_attention_weight_max": 42.057857513427734,
      "activations/layer22_attention_weight_min": -32.3091926574707,
      "activations/layer23_attention_weight_max": 30.520347595214844,
      "activations/layer23_attention_weight_min": -25.559368133544922,
      "activations/layer2_attention_weight_max": 31.713417053222656,
      "activations/layer2_attention_weight_min": -31.138275146484375,
      "activations/layer3_attention_weight_max": 99.50804901123047,
      "activations/layer3_attention_weight_min": -108.0674057006836,
      "activations/layer4_attention_weight_max": 105.46138000488281,
      "activations/layer4_attention_weight_min": -109.11866760253906,
      "activations/layer5_attention_weight_max": 50.328216552734375,
      "activations/layer5_attention_weight_min": -60.59355163574219,
      "activations/layer6_attention_weight_max": 44.86652374267578,
      "activations/layer6_attention_weight_min": -46.265689849853516,
      "activations/layer7_attention_weight_max": 94.55677795410156,
      "activations/layer7_attention_weight_min": -99.15699768066406,
      "activations/layer8_attention_weight_max": 44.595115661621094,
      "activations/layer8_attention_weight_min": -45.96940994262695,
      "activations/layer9_attention_weight_max": 32.42095947265625,
      "activations/layer9_attention_weight_min": -35.59828567504883,
      "epoch": 19.98,
      "learning_rate": 2.1353409090909088e-05,
      "loss": 2.7102,
      "step": 343900
    },
    {
      "activations/layer0_attention_weight_max": 16.150312423706055,
      "activations/layer0_attention_weight_min": -12.902892112731934,
      "activations/layer10_attention_weight_max": 37.09723663330078,
      "activations/layer10_attention_weight_min": -38.97336959838867,
      "activations/layer11_attention_weight_max": 36.36524963378906,
      "activations/layer11_attention_weight_min": -36.412132263183594,
      "activations/layer12_attention_weight_max": 31.27106285095215,
      "activations/layer12_attention_weight_min": -29.216398239135742,
      "activations/layer13_attention_weight_max": 48.267086029052734,
      "activations/layer13_attention_weight_min": -38.26416015625,
      "activations/layer14_attention_weight_max": 44.96113204956055,
      "activations/layer14_attention_weight_min": -40.30652618408203,
      "activations/layer15_attention_weight_max": 43.64670944213867,
      "activations/layer15_attention_weight_min": -38.71176528930664,
      "activations/layer16_attention_weight_max": 31.118253707885742,
      "activations/layer16_attention_weight_min": -29.748754501342773,
      "activations/layer17_attention_weight_max": 32.72407913208008,
      "activations/layer17_attention_weight_min": -27.836938858032227,
      "activations/layer18_attention_weight_max": 32.684242248535156,
      "activations/layer18_attention_weight_min": -27.71563720703125,
      "activations/layer19_attention_weight_max": 39.44514465332031,
      "activations/layer19_attention_weight_min": -32.91593933105469,
      "activations/layer1_attention_weight_max": 15.661314010620117,
      "activations/layer1_attention_weight_min": -14.351096153259277,
      "activations/layer20_attention_weight_max": 31.401660919189453,
      "activations/layer20_attention_weight_min": -27.15236473083496,
      "activations/layer21_attention_weight_max": 29.50707244873047,
      "activations/layer21_attention_weight_min": -25.82006072998047,
      "activations/layer22_attention_weight_max": 43.360836029052734,
      "activations/layer22_attention_weight_min": -32.84208297729492,
      "activations/layer23_attention_weight_max": 33.64081573486328,
      "activations/layer23_attention_weight_min": -27.363174438476562,
      "activations/layer2_attention_weight_max": 31.028623580932617,
      "activations/layer2_attention_weight_min": -31.016918182373047,
      "activations/layer3_attention_weight_max": 104.83354949951172,
      "activations/layer3_attention_weight_min": -104.06056213378906,
      "activations/layer4_attention_weight_max": 115.409423828125,
      "activations/layer4_attention_weight_min": -114.85289764404297,
      "activations/layer5_attention_weight_max": 50.55768966674805,
      "activations/layer5_attention_weight_min": -66.04110717773438,
      "activations/layer6_attention_weight_max": 46.021385192871094,
      "activations/layer6_attention_weight_min": -50.365394592285156,
      "activations/layer7_attention_weight_max": 101.62325286865234,
      "activations/layer7_attention_weight_min": -106.85025787353516,
      "activations/layer8_attention_weight_max": 45.316917419433594,
      "activations/layer8_attention_weight_min": -50.868934631347656,
      "activations/layer9_attention_weight_max": 34.94115447998047,
      "activations/layer9_attention_weight_min": -38.564693450927734,
      "epoch": 19.99,
      "learning_rate": 2.1334469696969696e-05,
      "loss": 2.7209,
      "step": 343950
    },
    {
      "activations/layer0_attention_weight_max": 16.40973472595215,
      "activations/layer0_attention_weight_min": -13.135875701904297,
      "activations/layer10_attention_weight_max": 36.204551696777344,
      "activations/layer10_attention_weight_min": -34.9561767578125,
      "activations/layer11_attention_weight_max": 35.57014083862305,
      "activations/layer11_attention_weight_min": -37.32257843017578,
      "activations/layer12_attention_weight_max": 26.552780151367188,
      "activations/layer12_attention_weight_min": -27.536624908447266,
      "activations/layer13_attention_weight_max": 38.110198974609375,
      "activations/layer13_attention_weight_min": -36.251014709472656,
      "activations/layer14_attention_weight_max": 41.058597564697266,
      "activations/layer14_attention_weight_min": -38.927608489990234,
      "activations/layer15_attention_weight_max": 41.268619537353516,
      "activations/layer15_attention_weight_min": -37.74098587036133,
      "activations/layer16_attention_weight_max": 29.11130714416504,
      "activations/layer16_attention_weight_min": -27.685354232788086,
      "activations/layer17_attention_weight_max": 29.301368713378906,
      "activations/layer17_attention_weight_min": -26.733118057250977,
      "activations/layer18_attention_weight_max": 33.46864700317383,
      "activations/layer18_attention_weight_min": -24.98893165588379,
      "activations/layer19_attention_weight_max": 32.35250473022461,
      "activations/layer19_attention_weight_min": -29.65195655822754,
      "activations/layer1_attention_weight_max": 15.06805419921875,
      "activations/layer1_attention_weight_min": -15.356062889099121,
      "activations/layer20_attention_weight_max": 28.05093002319336,
      "activations/layer20_attention_weight_min": -24.067224502563477,
      "activations/layer21_attention_weight_max": 28.243791580200195,
      "activations/layer21_attention_weight_min": -25.72116470336914,
      "activations/layer22_attention_weight_max": 41.3947639465332,
      "activations/layer22_attention_weight_min": -30.909626007080078,
      "activations/layer23_attention_weight_max": 32.565345764160156,
      "activations/layer23_attention_weight_min": -25.143543243408203,
      "activations/layer2_attention_weight_max": 31.582740783691406,
      "activations/layer2_attention_weight_min": -31.534526824951172,
      "activations/layer3_attention_weight_max": 100.89798736572266,
      "activations/layer3_attention_weight_min": -99.14409637451172,
      "activations/layer4_attention_weight_max": 109.46282958984375,
      "activations/layer4_attention_weight_min": -111.04923248291016,
      "activations/layer5_attention_weight_max": 53.09113311767578,
      "activations/layer5_attention_weight_min": -63.09303283691406,
      "activations/layer6_attention_weight_max": 45.33393096923828,
      "activations/layer6_attention_weight_min": -47.542137145996094,
      "activations/layer7_attention_weight_max": 102.60826873779297,
      "activations/layer7_attention_weight_min": -96.33051300048828,
      "activations/layer8_attention_weight_max": 43.201900482177734,
      "activations/layer8_attention_weight_min": -46.97309494018555,
      "activations/layer9_attention_weight_max": 32.977901458740234,
      "activations/layer9_attention_weight_min": -36.698490142822266,
      "epoch": 19.99,
      "learning_rate": 2.13155303030303e-05,
      "loss": 2.7223,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.4739,
      "eval_samples_per_second": 506.735,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.4739,
      "eval_openwebtext_samples_per_second": 506.735,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9635,
      "eval_wikitext_samples_per_second": 232.242,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_lambada_loss": 2.318359375,
      "eval_lambada_ppl": 10.158993527607587,
      "eval_lambada_runtime": 9.52,
      "eval_lambada_samples_per_second": 511.45,
      "step": 344000
    },
    {
      "activations/layer0_attention_weight_max": 16.539838790893555,
      "activations/layer0_attention_weight_min": -12.506590843200684,
      "activations/layer10_attention_weight_max": 41.85527801513672,
      "activations/layer10_attention_weight_min": -37.80654525756836,
      "activations/layer11_attention_weight_max": 42.60158920288086,
      "activations/layer11_attention_weight_min": -39.57859420776367,
      "activations/layer12_attention_weight_max": 27.246076583862305,
      "activations/layer12_attention_weight_min": -28.149389266967773,
      "activations/layer13_attention_weight_max": 40.375205993652344,
      "activations/layer13_attention_weight_min": -37.06685256958008,
      "activations/layer14_attention_weight_max": 48.43898010253906,
      "activations/layer14_attention_weight_min": -40.099185943603516,
      "activations/layer15_attention_weight_max": 46.93606185913086,
      "activations/layer15_attention_weight_min": -42.24833297729492,
      "activations/layer16_attention_weight_max": 31.278682708740234,
      "activations/layer16_attention_weight_min": -28.29562759399414,
      "activations/layer17_attention_weight_max": 28.61198616027832,
      "activations/layer17_attention_weight_min": -26.639860153198242,
      "activations/layer18_attention_weight_max": 29.85344123840332,
      "activations/layer18_attention_weight_min": -25.453842163085938,
      "activations/layer19_attention_weight_max": 31.745304107666016,
      "activations/layer19_attention_weight_min": -31.359638214111328,
      "activations/layer1_attention_weight_max": 15.48197078704834,
      "activations/layer1_attention_weight_min": -14.043107032775879,
      "activations/layer20_attention_weight_max": 26.019567489624023,
      "activations/layer20_attention_weight_min": -24.530284881591797,
      "activations/layer21_attention_weight_max": 24.502721786499023,
      "activations/layer21_attention_weight_min": -24.332565307617188,
      "activations/layer22_attention_weight_max": 40.60650634765625,
      "activations/layer22_attention_weight_min": -32.948631286621094,
      "activations/layer23_attention_weight_max": 29.21136474609375,
      "activations/layer23_attention_weight_min": -25.36236572265625,
      "activations/layer2_attention_weight_max": 32.303314208984375,
      "activations/layer2_attention_weight_min": -31.061262130737305,
      "activations/layer3_attention_weight_max": 101.46102905273438,
      "activations/layer3_attention_weight_min": -102.90583801269531,
      "activations/layer4_attention_weight_max": 105.0954360961914,
      "activations/layer4_attention_weight_min": -108.00756072998047,
      "activations/layer5_attention_weight_max": 51.92672348022461,
      "activations/layer5_attention_weight_min": -60.61824035644531,
      "activations/layer6_attention_weight_max": 42.53374481201172,
      "activations/layer6_attention_weight_min": -44.698368072509766,
      "activations/layer7_attention_weight_max": 95.74443817138672,
      "activations/layer7_attention_weight_min": -93.83019256591797,
      "activations/layer8_attention_weight_max": 44.30618667602539,
      "activations/layer8_attention_weight_min": -43.48903274536133,
      "activations/layer9_attention_weight_max": 34.86470031738281,
      "activations/layer9_attention_weight_min": -34.427059173583984,
      "epoch": 19.99,
      "learning_rate": 2.129659090909091e-05,
      "loss": 2.7427,
      "step": 344050
    },
    {
      "activations/layer0_attention_weight_max": 15.684234619140625,
      "activations/layer0_attention_weight_min": -13.542351722717285,
      "activations/layer10_attention_weight_max": 34.290504455566406,
      "activations/layer10_attention_weight_min": -34.91371536254883,
      "activations/layer11_attention_weight_max": 34.355010986328125,
      "activations/layer11_attention_weight_min": -35.39672088623047,
      "activations/layer12_attention_weight_max": 28.03250503540039,
      "activations/layer12_attention_weight_min": -26.420637130737305,
      "activations/layer13_attention_weight_max": 41.942047119140625,
      "activations/layer13_attention_weight_min": -37.3690185546875,
      "activations/layer14_attention_weight_max": 42.7946891784668,
      "activations/layer14_attention_weight_min": -39.23493576049805,
      "activations/layer15_attention_weight_max": 38.20790100097656,
      "activations/layer15_attention_weight_min": -36.170310974121094,
      "activations/layer16_attention_weight_max": 28.52981948852539,
      "activations/layer16_attention_weight_min": -27.42120933532715,
      "activations/layer17_attention_weight_max": 31.27674674987793,
      "activations/layer17_attention_weight_min": -26.888669967651367,
      "activations/layer18_attention_weight_max": 31.585115432739258,
      "activations/layer18_attention_weight_min": -26.905168533325195,
      "activations/layer19_attention_weight_max": 31.892175674438477,
      "activations/layer19_attention_weight_min": -32.4144172668457,
      "activations/layer1_attention_weight_max": 15.569108963012695,
      "activations/layer1_attention_weight_min": -14.772756576538086,
      "activations/layer20_attention_weight_max": 26.2127742767334,
      "activations/layer20_attention_weight_min": -27.802621841430664,
      "activations/layer21_attention_weight_max": 30.151994705200195,
      "activations/layer21_attention_weight_min": -27.291807174682617,
      "activations/layer22_attention_weight_max": 42.20455551147461,
      "activations/layer22_attention_weight_min": -31.911865234375,
      "activations/layer23_attention_weight_max": 30.55360984802246,
      "activations/layer23_attention_weight_min": -26.04296112060547,
      "activations/layer2_attention_weight_max": 31.694866180419922,
      "activations/layer2_attention_weight_min": -31.622055053710938,
      "activations/layer3_attention_weight_max": 100.98800659179688,
      "activations/layer3_attention_weight_min": -106.07366180419922,
      "activations/layer4_attention_weight_max": 106.72283935546875,
      "activations/layer4_attention_weight_min": -113.12064361572266,
      "activations/layer5_attention_weight_max": 53.780670166015625,
      "activations/layer5_attention_weight_min": -61.78998565673828,
      "activations/layer6_attention_weight_max": 48.820552825927734,
      "activations/layer6_attention_weight_min": -48.14617156982422,
      "activations/layer7_attention_weight_max": 97.93782043457031,
      "activations/layer7_attention_weight_min": -102.00657653808594,
      "activations/layer8_attention_weight_max": 42.324153900146484,
      "activations/layer8_attention_weight_min": -43.91941452026367,
      "activations/layer9_attention_weight_max": 32.38233184814453,
      "activations/layer9_attention_weight_min": -34.60367965698242,
      "epoch": 19.99,
      "learning_rate": 2.127765151515151e-05,
      "loss": 2.7288,
      "step": 344100
    },
    {
      "activations/layer0_attention_weight_max": 16.08757781982422,
      "activations/layer0_attention_weight_min": -12.609283447265625,
      "activations/layer10_attention_weight_max": 33.79995346069336,
      "activations/layer10_attention_weight_min": -34.58607864379883,
      "activations/layer11_attention_weight_max": 33.985389709472656,
      "activations/layer11_attention_weight_min": -34.84754180908203,
      "activations/layer12_attention_weight_max": 25.091243743896484,
      "activations/layer12_attention_weight_min": -25.97258949279785,
      "activations/layer13_attention_weight_max": 36.91070556640625,
      "activations/layer13_attention_weight_min": -34.58476257324219,
      "activations/layer14_attention_weight_max": 39.45975875854492,
      "activations/layer14_attention_weight_min": -38.56122970581055,
      "activations/layer15_attention_weight_max": 37.4327507019043,
      "activations/layer15_attention_weight_min": -35.46390151977539,
      "activations/layer16_attention_weight_max": 28.86942481994629,
      "activations/layer16_attention_weight_min": -28.514968872070312,
      "activations/layer17_attention_weight_max": 28.66014289855957,
      "activations/layer17_attention_weight_min": -27.307228088378906,
      "activations/layer18_attention_weight_max": 34.806304931640625,
      "activations/layer18_attention_weight_min": -27.13636589050293,
      "activations/layer19_attention_weight_max": 33.23674774169922,
      "activations/layer19_attention_weight_min": -33.13997268676758,
      "activations/layer1_attention_weight_max": 15.732582092285156,
      "activations/layer1_attention_weight_min": -15.470060348510742,
      "activations/layer20_attention_weight_max": 30.150850296020508,
      "activations/layer20_attention_weight_min": -27.42906379699707,
      "activations/layer21_attention_weight_max": 30.45871925354004,
      "activations/layer21_attention_weight_min": -26.127294540405273,
      "activations/layer22_attention_weight_max": 40.9549560546875,
      "activations/layer22_attention_weight_min": -33.738304138183594,
      "activations/layer23_attention_weight_max": 33.3560676574707,
      "activations/layer23_attention_weight_min": -26.987422943115234,
      "activations/layer2_attention_weight_max": 31.897472381591797,
      "activations/layer2_attention_weight_min": -33.94951248168945,
      "activations/layer3_attention_weight_max": 101.77071380615234,
      "activations/layer3_attention_weight_min": -110.21295166015625,
      "activations/layer4_attention_weight_max": 109.1516342163086,
      "activations/layer4_attention_weight_min": -113.3853759765625,
      "activations/layer5_attention_weight_max": 51.549983978271484,
      "activations/layer5_attention_weight_min": -60.352622985839844,
      "activations/layer6_attention_weight_max": 42.7963981628418,
      "activations/layer6_attention_weight_min": -45.751407623291016,
      "activations/layer7_attention_weight_max": 90.13877868652344,
      "activations/layer7_attention_weight_min": -92.1417007446289,
      "activations/layer8_attention_weight_max": 41.677268981933594,
      "activations/layer8_attention_weight_min": -43.017330169677734,
      "activations/layer9_attention_weight_max": 33.37187957763672,
      "activations/layer9_attention_weight_min": -34.85865783691406,
      "epoch": 20.0,
      "learning_rate": 2.125871212121212e-05,
      "loss": 2.7247,
      "step": 344150
    },
    {
      "activations/layer0_attention_weight_max": 16.031360626220703,
      "activations/layer0_attention_weight_min": -12.594789505004883,
      "activations/layer10_attention_weight_max": 35.84291076660156,
      "activations/layer10_attention_weight_min": -37.31575393676758,
      "activations/layer11_attention_weight_max": 35.639312744140625,
      "activations/layer11_attention_weight_min": -39.414581298828125,
      "activations/layer12_attention_weight_max": 31.807828903198242,
      "activations/layer12_attention_weight_min": -28.800704956054688,
      "activations/layer13_attention_weight_max": 48.14122772216797,
      "activations/layer13_attention_weight_min": -40.449729919433594,
      "activations/layer14_attention_weight_max": 52.15868377685547,
      "activations/layer14_attention_weight_min": -42.63759994506836,
      "activations/layer15_attention_weight_max": 44.59828186035156,
      "activations/layer15_attention_weight_min": -39.60832977294922,
      "activations/layer16_attention_weight_max": 33.072509765625,
      "activations/layer16_attention_weight_min": -30.27733039855957,
      "activations/layer17_attention_weight_max": 33.6981315612793,
      "activations/layer17_attention_weight_min": -28.640628814697266,
      "activations/layer18_attention_weight_max": 34.36298751831055,
      "activations/layer18_attention_weight_min": -28.12384605407715,
      "activations/layer19_attention_weight_max": 37.35999298095703,
      "activations/layer19_attention_weight_min": -34.68067169189453,
      "activations/layer1_attention_weight_max": 15.288239479064941,
      "activations/layer1_attention_weight_min": -15.161727905273438,
      "activations/layer20_attention_weight_max": 31.092966079711914,
      "activations/layer20_attention_weight_min": -27.864341735839844,
      "activations/layer21_attention_weight_max": 31.241130828857422,
      "activations/layer21_attention_weight_min": -26.671789169311523,
      "activations/layer22_attention_weight_max": 48.25604248046875,
      "activations/layer22_attention_weight_min": -37.7101936340332,
      "activations/layer23_attention_weight_max": 35.93359375,
      "activations/layer23_attention_weight_min": -28.718740463256836,
      "activations/layer2_attention_weight_max": 32.27859115600586,
      "activations/layer2_attention_weight_min": -32.17986297607422,
      "activations/layer3_attention_weight_max": 101.5401382446289,
      "activations/layer3_attention_weight_min": -104.17166900634766,
      "activations/layer4_attention_weight_max": 108.41839599609375,
      "activations/layer4_attention_weight_min": -113.28475189208984,
      "activations/layer5_attention_weight_max": 53.263519287109375,
      "activations/layer5_attention_weight_min": -62.61668014526367,
      "activations/layer6_attention_weight_max": 46.36820602416992,
      "activations/layer6_attention_weight_min": -49.584712982177734,
      "activations/layer7_attention_weight_max": 95.83744049072266,
      "activations/layer7_attention_weight_min": -104.92452239990234,
      "activations/layer8_attention_weight_max": 45.06247329711914,
      "activations/layer8_attention_weight_min": -48.06377029418945,
      "activations/layer9_attention_weight_max": 34.0023078918457,
      "activations/layer9_attention_weight_min": -37.55100631713867,
      "epoch": 20.0,
      "learning_rate": 2.1239772727272724e-05,
      "loss": 2.7109,
      "step": 344200
    },
    {
      "activations/layer0_attention_weight_max": 16.201730728149414,
      "activations/layer0_attention_weight_min": -12.986146926879883,
      "activations/layer10_attention_weight_max": 38.43107986450195,
      "activations/layer10_attention_weight_min": -40.08977508544922,
      "activations/layer11_attention_weight_max": 37.385032653808594,
      "activations/layer11_attention_weight_min": -39.971282958984375,
      "activations/layer12_attention_weight_max": 28.78679084777832,
      "activations/layer12_attention_weight_min": -26.475725173950195,
      "activations/layer13_attention_weight_max": 43.79077911376953,
      "activations/layer13_attention_weight_min": -36.50568389892578,
      "activations/layer14_attention_weight_max": 49.0985221862793,
      "activations/layer14_attention_weight_min": -37.38713455200195,
      "activations/layer15_attention_weight_max": 41.069862365722656,
      "activations/layer15_attention_weight_min": -34.683929443359375,
      "activations/layer16_attention_weight_max": 30.21907615661621,
      "activations/layer16_attention_weight_min": -27.912416458129883,
      "activations/layer17_attention_weight_max": 31.009334564208984,
      "activations/layer17_attention_weight_min": -25.270797729492188,
      "activations/layer18_attention_weight_max": 33.485626220703125,
      "activations/layer18_attention_weight_min": -26.00775718688965,
      "activations/layer19_attention_weight_max": 37.467132568359375,
      "activations/layer19_attention_weight_min": -32.312744140625,
      "activations/layer1_attention_weight_max": 15.42349624633789,
      "activations/layer1_attention_weight_min": -14.546160697937012,
      "activations/layer20_attention_weight_max": 29.657573699951172,
      "activations/layer20_attention_weight_min": -26.137306213378906,
      "activations/layer21_attention_weight_max": 30.530467987060547,
      "activations/layer21_attention_weight_min": -26.731779098510742,
      "activations/layer22_attention_weight_max": 40.57382583618164,
      "activations/layer22_attention_weight_min": -35.01436996459961,
      "activations/layer23_attention_weight_max": 32.952613830566406,
      "activations/layer23_attention_weight_min": -29.775962829589844,
      "activations/layer2_attention_weight_max": 33.04132080078125,
      "activations/layer2_attention_weight_min": -32.698787689208984,
      "activations/layer3_attention_weight_max": 106.52657318115234,
      "activations/layer3_attention_weight_min": -105.66301727294922,
      "activations/layer4_attention_weight_max": 109.55317687988281,
      "activations/layer4_attention_weight_min": -113.0356216430664,
      "activations/layer5_attention_weight_max": 51.84434127807617,
      "activations/layer5_attention_weight_min": -66.23331451416016,
      "activations/layer6_attention_weight_max": 47.28013610839844,
      "activations/layer6_attention_weight_min": -50.53889465332031,
      "activations/layer7_attention_weight_max": 99.09588623046875,
      "activations/layer7_attention_weight_min": -107.89542388916016,
      "activations/layer8_attention_weight_max": 46.19172286987305,
      "activations/layer8_attention_weight_min": -50.35308837890625,
      "activations/layer9_attention_weight_max": 34.35324478149414,
      "activations/layer9_attention_weight_min": -37.95989227294922,
      "epoch": 20.0,
      "learning_rate": 2.1220454545454543e-05,
      "loss": 2.7713,
      "step": 344250
    },
    {
      "activations/layer0_attention_weight_max": 16.887393951416016,
      "activations/layer0_attention_weight_min": -12.640296936035156,
      "activations/layer10_attention_weight_max": 34.67041015625,
      "activations/layer10_attention_weight_min": -36.62272644042969,
      "activations/layer11_attention_weight_max": 34.19602966308594,
      "activations/layer11_attention_weight_min": -35.57002258300781,
      "activations/layer12_attention_weight_max": 27.341768264770508,
      "activations/layer12_attention_weight_min": -27.93745231628418,
      "activations/layer13_attention_weight_max": 40.08469772338867,
      "activations/layer13_attention_weight_min": -36.514373779296875,
      "activations/layer14_attention_weight_max": 42.53715896606445,
      "activations/layer14_attention_weight_min": -39.950748443603516,
      "activations/layer15_attention_weight_max": 39.41869354248047,
      "activations/layer15_attention_weight_min": -38.63287353515625,
      "activations/layer16_attention_weight_max": 30.482633590698242,
      "activations/layer16_attention_weight_min": -28.04538345336914,
      "activations/layer17_attention_weight_max": 30.132089614868164,
      "activations/layer17_attention_weight_min": -27.435277938842773,
      "activations/layer18_attention_weight_max": 34.86982345581055,
      "activations/layer18_attention_weight_min": -26.684118270874023,
      "activations/layer19_attention_weight_max": 39.47414779663086,
      "activations/layer19_attention_weight_min": -32.676334381103516,
      "activations/layer1_attention_weight_max": 15.949056625366211,
      "activations/layer1_attention_weight_min": -14.754258155822754,
      "activations/layer20_attention_weight_max": 30.723777770996094,
      "activations/layer20_attention_weight_min": -25.836563110351562,
      "activations/layer21_attention_weight_max": 29.278152465820312,
      "activations/layer21_attention_weight_min": -25.35892105102539,
      "activations/layer22_attention_weight_max": 42.367713928222656,
      "activations/layer22_attention_weight_min": -33.83023452758789,
      "activations/layer23_attention_weight_max": 35.57487487792969,
      "activations/layer23_attention_weight_min": -28.460004806518555,
      "activations/layer2_attention_weight_max": 31.43941879272461,
      "activations/layer2_attention_weight_min": -31.730731964111328,
      "activations/layer3_attention_weight_max": 99.19093322753906,
      "activations/layer3_attention_weight_min": -102.78175354003906,
      "activations/layer4_attention_weight_max": 108.6407241821289,
      "activations/layer4_attention_weight_min": -113.41707611083984,
      "activations/layer5_attention_weight_max": 52.68351364135742,
      "activations/layer5_attention_weight_min": -63.07389831542969,
      "activations/layer6_attention_weight_max": 44.487693786621094,
      "activations/layer6_attention_weight_min": -50.48407745361328,
      "activations/layer7_attention_weight_max": 93.7873306274414,
      "activations/layer7_attention_weight_min": -102.0455322265625,
      "activations/layer8_attention_weight_max": 44.64154815673828,
      "activations/layer8_attention_weight_min": -46.244964599609375,
      "activations/layer9_attention_weight_max": 33.38976287841797,
      "activations/layer9_attention_weight_min": -35.80952072143555,
      "epoch": 20.01,
      "learning_rate": 2.120151515151515e-05,
      "loss": 2.7085,
      "step": 344300
    },
    {
      "activations/layer0_attention_weight_max": 15.826395034790039,
      "activations/layer0_attention_weight_min": -12.252373695373535,
      "activations/layer10_attention_weight_max": 34.32875442504883,
      "activations/layer10_attention_weight_min": -35.660003662109375,
      "activations/layer11_attention_weight_max": 35.1024169921875,
      "activations/layer11_attention_weight_min": -39.397117614746094,
      "activations/layer12_attention_weight_max": 32.07964324951172,
      "activations/layer12_attention_weight_min": -26.311111450195312,
      "activations/layer13_attention_weight_max": 43.08909606933594,
      "activations/layer13_attention_weight_min": -37.26674270629883,
      "activations/layer14_attention_weight_max": 47.25551223754883,
      "activations/layer14_attention_weight_min": -41.13596725463867,
      "activations/layer15_attention_weight_max": 42.40121078491211,
      "activations/layer15_attention_weight_min": -37.86851501464844,
      "activations/layer16_attention_weight_max": 37.75861740112305,
      "activations/layer16_attention_weight_min": -31.01506233215332,
      "activations/layer17_attention_weight_max": 33.66353988647461,
      "activations/layer17_attention_weight_min": -27.348766326904297,
      "activations/layer18_attention_weight_max": 38.01681900024414,
      "activations/layer18_attention_weight_min": -28.914608001708984,
      "activations/layer19_attention_weight_max": 36.1412239074707,
      "activations/layer19_attention_weight_min": -33.279483795166016,
      "activations/layer1_attention_weight_max": 16.13619613647461,
      "activations/layer1_attention_weight_min": -15.084637641906738,
      "activations/layer20_attention_weight_max": 33.134220123291016,
      "activations/layer20_attention_weight_min": -26.710737228393555,
      "activations/layer21_attention_weight_max": 32.437644958496094,
      "activations/layer21_attention_weight_min": -26.20651626586914,
      "activations/layer22_attention_weight_max": 42.769840240478516,
      "activations/layer22_attention_weight_min": -35.50566482543945,
      "activations/layer23_attention_weight_max": 31.48965072631836,
      "activations/layer23_attention_weight_min": -28.6637020111084,
      "activations/layer2_attention_weight_max": 31.678457260131836,
      "activations/layer2_attention_weight_min": -31.238149642944336,
      "activations/layer3_attention_weight_max": 96.7426528930664,
      "activations/layer3_attention_weight_min": -101.25928497314453,
      "activations/layer4_attention_weight_max": 102.60668182373047,
      "activations/layer4_attention_weight_min": -111.72039794921875,
      "activations/layer5_attention_weight_max": 51.154537200927734,
      "activations/layer5_attention_weight_min": -62.19334030151367,
      "activations/layer6_attention_weight_max": 44.38174057006836,
      "activations/layer6_attention_weight_min": -48.59599304199219,
      "activations/layer7_attention_weight_max": 93.0638198852539,
      "activations/layer7_attention_weight_min": -101.03973388671875,
      "activations/layer8_attention_weight_max": 41.8185920715332,
      "activations/layer8_attention_weight_min": -44.67090606689453,
      "activations/layer9_attention_weight_max": 32.26917266845703,
      "activations/layer9_attention_weight_min": -35.939083099365234,
      "epoch": 20.01,
      "learning_rate": 2.1182575757575756e-05,
      "loss": 2.7011,
      "step": 344350
    },
    {
      "activations/layer0_attention_weight_max": 15.31344985961914,
      "activations/layer0_attention_weight_min": -12.408700942993164,
      "activations/layer10_attention_weight_max": 35.75728225708008,
      "activations/layer10_attention_weight_min": -36.274105072021484,
      "activations/layer11_attention_weight_max": 34.49517822265625,
      "activations/layer11_attention_weight_min": -35.3884391784668,
      "activations/layer12_attention_weight_max": 27.56926727294922,
      "activations/layer12_attention_weight_min": -27.84746551513672,
      "activations/layer13_attention_weight_max": 38.588134765625,
      "activations/layer13_attention_weight_min": -35.85123825073242,
      "activations/layer14_attention_weight_max": 42.37775421142578,
      "activations/layer14_attention_weight_min": -39.36088180541992,
      "activations/layer15_attention_weight_max": 41.711097717285156,
      "activations/layer15_attention_weight_min": -36.39369583129883,
      "activations/layer16_attention_weight_max": 31.186555862426758,
      "activations/layer16_attention_weight_min": -27.72886085510254,
      "activations/layer17_attention_weight_max": 29.949066162109375,
      "activations/layer17_attention_weight_min": -27.69889259338379,
      "activations/layer18_attention_weight_max": 32.09696960449219,
      "activations/layer18_attention_weight_min": -27.2540340423584,
      "activations/layer19_attention_weight_max": 35.39652633666992,
      "activations/layer19_attention_weight_min": -32.77317428588867,
      "activations/layer1_attention_weight_max": 16.151588439941406,
      "activations/layer1_attention_weight_min": -15.317946434020996,
      "activations/layer20_attention_weight_max": 28.825822830200195,
      "activations/layer20_attention_weight_min": -26.248390197753906,
      "activations/layer21_attention_weight_max": 28.01335906982422,
      "activations/layer21_attention_weight_min": -26.570648193359375,
      "activations/layer22_attention_weight_max": 44.974327087402344,
      "activations/layer22_attention_weight_min": -35.49128723144531,
      "activations/layer23_attention_weight_max": 32.53310775756836,
      "activations/layer23_attention_weight_min": -26.271800994873047,
      "activations/layer2_attention_weight_max": 31.37145233154297,
      "activations/layer2_attention_weight_min": -32.898921966552734,
      "activations/layer3_attention_weight_max": 98.41038513183594,
      "activations/layer3_attention_weight_min": -99.92660522460938,
      "activations/layer4_attention_weight_max": 102.6243667602539,
      "activations/layer4_attention_weight_min": -108.02501678466797,
      "activations/layer5_attention_weight_max": 51.19605255126953,
      "activations/layer5_attention_weight_min": -62.45603942871094,
      "activations/layer6_attention_weight_max": 44.12025833129883,
      "activations/layer6_attention_weight_min": -47.36530303955078,
      "activations/layer7_attention_weight_max": 94.01858520507812,
      "activations/layer7_attention_weight_min": -103.13908386230469,
      "activations/layer8_attention_weight_max": 42.34830093383789,
      "activations/layer8_attention_weight_min": -45.587738037109375,
      "activations/layer9_attention_weight_max": 32.5024299621582,
      "activations/layer9_attention_weight_min": -35.73629379272461,
      "epoch": 20.01,
      "learning_rate": 2.1163636363636364e-05,
      "loss": 2.7171,
      "step": 344400
    },
    {
      "activations/layer0_attention_weight_max": 15.531538963317871,
      "activations/layer0_attention_weight_min": -14.010430335998535,
      "activations/layer10_attention_weight_max": 33.96461486816406,
      "activations/layer10_attention_weight_min": -33.0125617980957,
      "activations/layer11_attention_weight_max": 33.21894073486328,
      "activations/layer11_attention_weight_min": -33.95515441894531,
      "activations/layer12_attention_weight_max": 29.19558334350586,
      "activations/layer12_attention_weight_min": -28.10099983215332,
      "activations/layer13_attention_weight_max": 43.1122932434082,
      "activations/layer13_attention_weight_min": -39.136138916015625,
      "activations/layer14_attention_weight_max": 43.04835891723633,
      "activations/layer14_attention_weight_min": -38.046932220458984,
      "activations/layer15_attention_weight_max": 41.3624267578125,
      "activations/layer15_attention_weight_min": -35.25927734375,
      "activations/layer16_attention_weight_max": 30.922908782958984,
      "activations/layer16_attention_weight_min": -28.466218948364258,
      "activations/layer17_attention_weight_max": 30.86284828186035,
      "activations/layer17_attention_weight_min": -27.15656089782715,
      "activations/layer18_attention_weight_max": 34.46806716918945,
      "activations/layer18_attention_weight_min": -26.508981704711914,
      "activations/layer19_attention_weight_max": 40.505714416503906,
      "activations/layer19_attention_weight_min": -35.808990478515625,
      "activations/layer1_attention_weight_max": 16.13222312927246,
      "activations/layer1_attention_weight_min": -14.549988746643066,
      "activations/layer20_attention_weight_max": 32.37148666381836,
      "activations/layer20_attention_weight_min": -28.0289306640625,
      "activations/layer21_attention_weight_max": 32.6413688659668,
      "activations/layer21_attention_weight_min": -28.16744041442871,
      "activations/layer22_attention_weight_max": 46.55791473388672,
      "activations/layer22_attention_weight_min": -34.87373352050781,
      "activations/layer23_attention_weight_max": 37.57680130004883,
      "activations/layer23_attention_weight_min": -26.735862731933594,
      "activations/layer2_attention_weight_max": 32.92549133300781,
      "activations/layer2_attention_weight_min": -32.111427307128906,
      "activations/layer3_attention_weight_max": 100.23957824707031,
      "activations/layer3_attention_weight_min": -104.66873931884766,
      "activations/layer4_attention_weight_max": 108.45507049560547,
      "activations/layer4_attention_weight_min": -111.6054458618164,
      "activations/layer5_attention_weight_max": 49.85482406616211,
      "activations/layer5_attention_weight_min": -61.84870910644531,
      "activations/layer6_attention_weight_max": 45.28250503540039,
      "activations/layer6_attention_weight_min": -46.9482536315918,
      "activations/layer7_attention_weight_max": 95.49205017089844,
      "activations/layer7_attention_weight_min": -96.80776977539062,
      "activations/layer8_attention_weight_max": 41.27226638793945,
      "activations/layer8_attention_weight_min": -44.26599884033203,
      "activations/layer9_attention_weight_max": 32.76553726196289,
      "activations/layer9_attention_weight_min": -33.5378532409668,
      "epoch": 20.01,
      "learning_rate": 2.1144696969696965e-05,
      "loss": 2.7107,
      "step": 344450
    },
    {
      "activations/layer0_attention_weight_max": 16.091705322265625,
      "activations/layer0_attention_weight_min": -13.57943058013916,
      "activations/layer10_attention_weight_max": 35.332733154296875,
      "activations/layer10_attention_weight_min": -36.54772186279297,
      "activations/layer11_attention_weight_max": 34.81132507324219,
      "activations/layer11_attention_weight_min": -35.7630615234375,
      "activations/layer12_attention_weight_max": 27.325258255004883,
      "activations/layer12_attention_weight_min": -26.937196731567383,
      "activations/layer13_attention_weight_max": 39.4788703918457,
      "activations/layer13_attention_weight_min": -36.71698760986328,
      "activations/layer14_attention_weight_max": 43.258872985839844,
      "activations/layer14_attention_weight_min": -38.05074691772461,
      "activations/layer15_attention_weight_max": 39.401100158691406,
      "activations/layer15_attention_weight_min": -35.64240646362305,
      "activations/layer16_attention_weight_max": 30.582609176635742,
      "activations/layer16_attention_weight_min": -27.8406982421875,
      "activations/layer17_attention_weight_max": 29.64133071899414,
      "activations/layer17_attention_weight_min": -24.94310760498047,
      "activations/layer18_attention_weight_max": 32.49675750732422,
      "activations/layer18_attention_weight_min": -25.38795280456543,
      "activations/layer19_attention_weight_max": 35.52361297607422,
      "activations/layer19_attention_weight_min": -31.48244285583496,
      "activations/layer1_attention_weight_max": 16.192523956298828,
      "activations/layer1_attention_weight_min": -15.253426551818848,
      "activations/layer20_attention_weight_max": 33.456729888916016,
      "activations/layer20_attention_weight_min": -26.2099552154541,
      "activations/layer21_attention_weight_max": 32.53662109375,
      "activations/layer21_attention_weight_min": -24.61431884765625,
      "activations/layer22_attention_weight_max": 44.35810852050781,
      "activations/layer22_attention_weight_min": -33.83980941772461,
      "activations/layer23_attention_weight_max": 34.48784637451172,
      "activations/layer23_attention_weight_min": -26.71864128112793,
      "activations/layer2_attention_weight_max": 32.74193572998047,
      "activations/layer2_attention_weight_min": -33.15624237060547,
      "activations/layer3_attention_weight_max": 98.6624526977539,
      "activations/layer3_attention_weight_min": -98.46316528320312,
      "activations/layer4_attention_weight_max": 105.73866271972656,
      "activations/layer4_attention_weight_min": -106.6536636352539,
      "activations/layer5_attention_weight_max": 50.3438835144043,
      "activations/layer5_attention_weight_min": -60.746665954589844,
      "activations/layer6_attention_weight_max": 43.43967056274414,
      "activations/layer6_attention_weight_min": -46.84935760498047,
      "activations/layer7_attention_weight_max": 95.58627319335938,
      "activations/layer7_attention_weight_min": -100.42214965820312,
      "activations/layer8_attention_weight_max": 44.18593978881836,
      "activations/layer8_attention_weight_min": -47.6109504699707,
      "activations/layer9_attention_weight_max": 32.997859954833984,
      "activations/layer9_attention_weight_min": -37.759952545166016,
      "epoch": 20.02,
      "learning_rate": 2.1125757575757574e-05,
      "loss": 2.7067,
      "step": 344500
    },
    {
      "activations/layer0_attention_weight_max": 15.714547157287598,
      "activations/layer0_attention_weight_min": -13.740007400512695,
      "activations/layer10_attention_weight_max": 33.800331115722656,
      "activations/layer10_attention_weight_min": -34.28336715698242,
      "activations/layer11_attention_weight_max": 32.50354766845703,
      "activations/layer11_attention_weight_min": -33.047340393066406,
      "activations/layer12_attention_weight_max": 26.502710342407227,
      "activations/layer12_attention_weight_min": -25.916610717773438,
      "activations/layer13_attention_weight_max": 40.26778793334961,
      "activations/layer13_attention_weight_min": -34.420101165771484,
      "activations/layer14_attention_weight_max": 42.88831329345703,
      "activations/layer14_attention_weight_min": -39.15049743652344,
      "activations/layer15_attention_weight_max": 40.0660514831543,
      "activations/layer15_attention_weight_min": -34.68215560913086,
      "activations/layer16_attention_weight_max": 32.889774322509766,
      "activations/layer16_attention_weight_min": -27.486419677734375,
      "activations/layer17_attention_weight_max": 32.332855224609375,
      "activations/layer17_attention_weight_min": -25.87759017944336,
      "activations/layer18_attention_weight_max": 36.68923568725586,
      "activations/layer18_attention_weight_min": -24.058826446533203,
      "activations/layer19_attention_weight_max": 39.5842399597168,
      "activations/layer19_attention_weight_min": -31.151296615600586,
      "activations/layer1_attention_weight_max": 15.920742988586426,
      "activations/layer1_attention_weight_min": -13.824213027954102,
      "activations/layer20_attention_weight_max": 32.59158706665039,
      "activations/layer20_attention_weight_min": -24.53093147277832,
      "activations/layer21_attention_weight_max": 30.802734375,
      "activations/layer21_attention_weight_min": -24.35736083984375,
      "activations/layer22_attention_weight_max": 45.74195861816406,
      "activations/layer22_attention_weight_min": -29.761465072631836,
      "activations/layer23_attention_weight_max": 37.25389099121094,
      "activations/layer23_attention_weight_min": -25.198469161987305,
      "activations/layer2_attention_weight_max": 31.898109436035156,
      "activations/layer2_attention_weight_min": -31.67745018005371,
      "activations/layer3_attention_weight_max": 96.21171569824219,
      "activations/layer3_attention_weight_min": -100.10538482666016,
      "activations/layer4_attention_weight_max": 102.5119857788086,
      "activations/layer4_attention_weight_min": -103.9330825805664,
      "activations/layer5_attention_weight_max": 49.613800048828125,
      "activations/layer5_attention_weight_min": -62.83427047729492,
      "activations/layer6_attention_weight_max": 42.99689865112305,
      "activations/layer6_attention_weight_min": -45.51457977294922,
      "activations/layer7_attention_weight_max": 96.0027084350586,
      "activations/layer7_attention_weight_min": -93.75035095214844,
      "activations/layer8_attention_weight_max": 40.63916015625,
      "activations/layer8_attention_weight_min": -42.578155517578125,
      "activations/layer9_attention_weight_max": 32.07599639892578,
      "activations/layer9_attention_weight_min": -33.52769470214844,
      "epoch": 20.02,
      "learning_rate": 2.110681818181818e-05,
      "loss": 2.7238,
      "step": 344550
    },
    {
      "activations/layer0_attention_weight_max": 15.566878318786621,
      "activations/layer0_attention_weight_min": -15.16943073272705,
      "activations/layer10_attention_weight_max": 35.28397750854492,
      "activations/layer10_attention_weight_min": -35.19984817504883,
      "activations/layer11_attention_weight_max": 37.03814697265625,
      "activations/layer11_attention_weight_min": -34.75743103027344,
      "activations/layer12_attention_weight_max": 27.224021911621094,
      "activations/layer12_attention_weight_min": -27.61911964416504,
      "activations/layer13_attention_weight_max": 39.11333084106445,
      "activations/layer13_attention_weight_min": -36.24449920654297,
      "activations/layer14_attention_weight_max": 46.91752624511719,
      "activations/layer14_attention_weight_min": -39.13371658325195,
      "activations/layer15_attention_weight_max": 43.304039001464844,
      "activations/layer15_attention_weight_min": -39.155067443847656,
      "activations/layer16_attention_weight_max": 30.91414451599121,
      "activations/layer16_attention_weight_min": -29.766845703125,
      "activations/layer17_attention_weight_max": 32.31744384765625,
      "activations/layer17_attention_weight_min": -27.300540924072266,
      "activations/layer18_attention_weight_max": 32.877601623535156,
      "activations/layer18_attention_weight_min": -27.742023468017578,
      "activations/layer19_attention_weight_max": 37.6163444519043,
      "activations/layer19_attention_weight_min": -31.916976928710938,
      "activations/layer1_attention_weight_max": 15.045546531677246,
      "activations/layer1_attention_weight_min": -15.78832721710205,
      "activations/layer20_attention_weight_max": 32.64871597290039,
      "activations/layer20_attention_weight_min": -25.34914207458496,
      "activations/layer21_attention_weight_max": 33.16643142700195,
      "activations/layer21_attention_weight_min": -24.594640731811523,
      "activations/layer22_attention_weight_max": 50.834495544433594,
      "activations/layer22_attention_weight_min": -31.333295822143555,
      "activations/layer23_attention_weight_max": 33.711605072021484,
      "activations/layer23_attention_weight_min": -25.262290954589844,
      "activations/layer2_attention_weight_max": 32.50577926635742,
      "activations/layer2_attention_weight_min": -30.709747314453125,
      "activations/layer3_attention_weight_max": 101.19015502929688,
      "activations/layer3_attention_weight_min": -97.6977310180664,
      "activations/layer4_attention_weight_max": 108.28697967529297,
      "activations/layer4_attention_weight_min": -107.67803192138672,
      "activations/layer5_attention_weight_max": 51.19021224975586,
      "activations/layer5_attention_weight_min": -59.69389343261719,
      "activations/layer6_attention_weight_max": 44.23997116088867,
      "activations/layer6_attention_weight_min": -46.14653778076172,
      "activations/layer7_attention_weight_max": 95.23126220703125,
      "activations/layer7_attention_weight_min": -96.7811050415039,
      "activations/layer8_attention_weight_max": 42.49724197387695,
      "activations/layer8_attention_weight_min": -44.827247619628906,
      "activations/layer9_attention_weight_max": 35.32332992553711,
      "activations/layer9_attention_weight_min": -34.747615814208984,
      "epoch": 20.02,
      "learning_rate": 2.1087878787878787e-05,
      "loss": 2.7119,
      "step": 344600
    },
    {
      "activations/layer0_attention_weight_max": 16.191476821899414,
      "activations/layer0_attention_weight_min": -14.406120300292969,
      "activations/layer10_attention_weight_max": 33.7974739074707,
      "activations/layer10_attention_weight_min": -34.18157196044922,
      "activations/layer11_attention_weight_max": 32.16231918334961,
      "activations/layer11_attention_weight_min": -33.19792175292969,
      "activations/layer12_attention_weight_max": 28.901594161987305,
      "activations/layer12_attention_weight_min": -26.72771453857422,
      "activations/layer13_attention_weight_max": 39.638084411621094,
      "activations/layer13_attention_weight_min": -36.3166618347168,
      "activations/layer14_attention_weight_max": 40.931365966796875,
      "activations/layer14_attention_weight_min": -38.30485153198242,
      "activations/layer15_attention_weight_max": 37.16331481933594,
      "activations/layer15_attention_weight_min": -35.469032287597656,
      "activations/layer16_attention_weight_max": 27.183881759643555,
      "activations/layer16_attention_weight_min": -27.910993576049805,
      "activations/layer17_attention_weight_max": 29.050628662109375,
      "activations/layer17_attention_weight_min": -25.8881893157959,
      "activations/layer18_attention_weight_max": 33.22368621826172,
      "activations/layer18_attention_weight_min": -27.937862396240234,
      "activations/layer19_attention_weight_max": 31.616703033447266,
      "activations/layer19_attention_weight_min": -34.30982208251953,
      "activations/layer1_attention_weight_max": 16.00043296813965,
      "activations/layer1_attention_weight_min": -16.35684585571289,
      "activations/layer20_attention_weight_max": 29.016511917114258,
      "activations/layer20_attention_weight_min": -28.369794845581055,
      "activations/layer21_attention_weight_max": 27.859214782714844,
      "activations/layer21_attention_weight_min": -24.93655776977539,
      "activations/layer22_attention_weight_max": 44.41790771484375,
      "activations/layer22_attention_weight_min": -34.797386169433594,
      "activations/layer23_attention_weight_max": 32.26252746582031,
      "activations/layer23_attention_weight_min": -25.971405029296875,
      "activations/layer2_attention_weight_max": 32.50101089477539,
      "activations/layer2_attention_weight_min": -32.568443298339844,
      "activations/layer3_attention_weight_max": 99.76634216308594,
      "activations/layer3_attention_weight_min": -100.31290435791016,
      "activations/layer4_attention_weight_max": 106.06642150878906,
      "activations/layer4_attention_weight_min": -110.18074798583984,
      "activations/layer5_attention_weight_max": 51.79735565185547,
      "activations/layer5_attention_weight_min": -59.11509323120117,
      "activations/layer6_attention_weight_max": 44.84988784790039,
      "activations/layer6_attention_weight_min": -47.591835021972656,
      "activations/layer7_attention_weight_max": 90.23777770996094,
      "activations/layer7_attention_weight_min": -101.00338745117188,
      "activations/layer8_attention_weight_max": 42.36206817626953,
      "activations/layer8_attention_weight_min": -48.814048767089844,
      "activations/layer9_attention_weight_max": 31.962890625,
      "activations/layer9_attention_weight_min": -35.30512237548828,
      "epoch": 20.03,
      "learning_rate": 2.1068939393939392e-05,
      "loss": 2.7249,
      "step": 344650
    },
    {
      "activations/layer0_attention_weight_max": 15.956398010253906,
      "activations/layer0_attention_weight_min": -14.050969123840332,
      "activations/layer10_attention_weight_max": 33.25410079956055,
      "activations/layer10_attention_weight_min": -35.57888412475586,
      "activations/layer11_attention_weight_max": 32.92924499511719,
      "activations/layer11_attention_weight_min": -35.51839828491211,
      "activations/layer12_attention_weight_max": 25.270246505737305,
      "activations/layer12_attention_weight_min": -27.078916549682617,
      "activations/layer13_attention_weight_max": 40.06692123413086,
      "activations/layer13_attention_weight_min": -38.70648193359375,
      "activations/layer14_attention_weight_max": 41.960025787353516,
      "activations/layer14_attention_weight_min": -37.93830490112305,
      "activations/layer15_attention_weight_max": 39.42815017700195,
      "activations/layer15_attention_weight_min": -35.34722137451172,
      "activations/layer16_attention_weight_max": 30.808956146240234,
      "activations/layer16_attention_weight_min": -27.918624877929688,
      "activations/layer17_attention_weight_max": 28.370309829711914,
      "activations/layer17_attention_weight_min": -25.099441528320312,
      "activations/layer18_attention_weight_max": 32.8304557800293,
      "activations/layer18_attention_weight_min": -27.100603103637695,
      "activations/layer19_attention_weight_max": 31.40342140197754,
      "activations/layer19_attention_weight_min": -31.891706466674805,
      "activations/layer1_attention_weight_max": 15.536825180053711,
      "activations/layer1_attention_weight_min": -14.236390113830566,
      "activations/layer20_attention_weight_max": 30.99986457824707,
      "activations/layer20_attention_weight_min": -25.493602752685547,
      "activations/layer21_attention_weight_max": 28.754606246948242,
      "activations/layer21_attention_weight_min": -24.28282928466797,
      "activations/layer22_attention_weight_max": 37.464290618896484,
      "activations/layer22_attention_weight_min": -31.009511947631836,
      "activations/layer23_attention_weight_max": 30.736135482788086,
      "activations/layer23_attention_weight_min": -25.680755615234375,
      "activations/layer2_attention_weight_max": 31.534481048583984,
      "activations/layer2_attention_weight_min": -32.083274841308594,
      "activations/layer3_attention_weight_max": 98.99015808105469,
      "activations/layer3_attention_weight_min": -101.04768371582031,
      "activations/layer4_attention_weight_max": 103.5628433227539,
      "activations/layer4_attention_weight_min": -107.53204345703125,
      "activations/layer5_attention_weight_max": 49.79215621948242,
      "activations/layer5_attention_weight_min": -60.15654754638672,
      "activations/layer6_attention_weight_max": 44.11362838745117,
      "activations/layer6_attention_weight_min": -49.44223403930664,
      "activations/layer7_attention_weight_max": 94.0741195678711,
      "activations/layer7_attention_weight_min": -97.2706527709961,
      "activations/layer8_attention_weight_max": 41.17768096923828,
      "activations/layer8_attention_weight_min": -44.5797119140625,
      "activations/layer9_attention_weight_max": 31.48695182800293,
      "activations/layer9_attention_weight_min": -36.07009506225586,
      "epoch": 20.03,
      "learning_rate": 2.105e-05,
      "loss": 2.7139,
      "step": 344700
    },
    {
      "activations/layer0_attention_weight_max": 16.47159194946289,
      "activations/layer0_attention_weight_min": -12.346611976623535,
      "activations/layer10_attention_weight_max": 35.78850173950195,
      "activations/layer10_attention_weight_min": -38.139549255371094,
      "activations/layer11_attention_weight_max": 36.09284973144531,
      "activations/layer11_attention_weight_min": -36.79551696777344,
      "activations/layer12_attention_weight_max": 29.067712783813477,
      "activations/layer12_attention_weight_min": -29.125078201293945,
      "activations/layer13_attention_weight_max": 43.021663665771484,
      "activations/layer13_attention_weight_min": -38.17155075073242,
      "activations/layer14_attention_weight_max": 46.83524703979492,
      "activations/layer14_attention_weight_min": -41.5634651184082,
      "activations/layer15_attention_weight_max": 48.42601013183594,
      "activations/layer15_attention_weight_min": -37.766536712646484,
      "activations/layer16_attention_weight_max": 33.85541915893555,
      "activations/layer16_attention_weight_min": -29.902334213256836,
      "activations/layer17_attention_weight_max": 30.527095794677734,
      "activations/layer17_attention_weight_min": -27.14750862121582,
      "activations/layer18_attention_weight_max": 33.13984298706055,
      "activations/layer18_attention_weight_min": -29.7421817779541,
      "activations/layer19_attention_weight_max": 35.94163131713867,
      "activations/layer19_attention_weight_min": -38.39595031738281,
      "activations/layer1_attention_weight_max": 17.234838485717773,
      "activations/layer1_attention_weight_min": -17.517507553100586,
      "activations/layer20_attention_weight_max": 30.572725296020508,
      "activations/layer20_attention_weight_min": -28.876108169555664,
      "activations/layer21_attention_weight_max": 30.19232177734375,
      "activations/layer21_attention_weight_min": -27.917943954467773,
      "activations/layer22_attention_weight_max": 44.59853744506836,
      "activations/layer22_attention_weight_min": -34.849361419677734,
      "activations/layer23_attention_weight_max": 33.595794677734375,
      "activations/layer23_attention_weight_min": -27.233482360839844,
      "activations/layer2_attention_weight_max": 32.97377014160156,
      "activations/layer2_attention_weight_min": -33.15528869628906,
      "activations/layer3_attention_weight_max": 99.62342071533203,
      "activations/layer3_attention_weight_min": -102.8205795288086,
      "activations/layer4_attention_weight_max": 107.27107238769531,
      "activations/layer4_attention_weight_min": -110.09453582763672,
      "activations/layer5_attention_weight_max": 53.34223556518555,
      "activations/layer5_attention_weight_min": -61.052589416503906,
      "activations/layer6_attention_weight_max": 49.29664611816406,
      "activations/layer6_attention_weight_min": -50.02914810180664,
      "activations/layer7_attention_weight_max": 100.14514923095703,
      "activations/layer7_attention_weight_min": -109.12340545654297,
      "activations/layer8_attention_weight_max": 44.443939208984375,
      "activations/layer8_attention_weight_min": -49.41299057006836,
      "activations/layer9_attention_weight_max": 34.10055160522461,
      "activations/layer9_attention_weight_min": -37.0630989074707,
      "epoch": 20.03,
      "learning_rate": 2.10310606060606e-05,
      "loss": 2.7182,
      "step": 344750
    },
    {
      "activations/layer0_attention_weight_max": 16.919212341308594,
      "activations/layer0_attention_weight_min": -15.494778633117676,
      "activations/layer10_attention_weight_max": 32.84051513671875,
      "activations/layer10_attention_weight_min": -36.11647415161133,
      "activations/layer11_attention_weight_max": 34.3531494140625,
      "activations/layer11_attention_weight_min": -34.51449966430664,
      "activations/layer12_attention_weight_max": 27.996469497680664,
      "activations/layer12_attention_weight_min": -27.563325881958008,
      "activations/layer13_attention_weight_max": 44.36538314819336,
      "activations/layer13_attention_weight_min": -39.073116302490234,
      "activations/layer14_attention_weight_max": 44.784244537353516,
      "activations/layer14_attention_weight_min": -42.72047805786133,
      "activations/layer15_attention_weight_max": 40.03053665161133,
      "activations/layer15_attention_weight_min": -38.05284881591797,
      "activations/layer16_attention_weight_max": 33.27442169189453,
      "activations/layer16_attention_weight_min": -28.004289627075195,
      "activations/layer17_attention_weight_max": 30.426382064819336,
      "activations/layer17_attention_weight_min": -26.064651489257812,
      "activations/layer18_attention_weight_max": 32.59096908569336,
      "activations/layer18_attention_weight_min": -26.964323043823242,
      "activations/layer19_attention_weight_max": 35.01801300048828,
      "activations/layer19_attention_weight_min": -34.909950256347656,
      "activations/layer1_attention_weight_max": 16.120018005371094,
      "activations/layer1_attention_weight_min": -14.525076866149902,
      "activations/layer20_attention_weight_max": 30.505615234375,
      "activations/layer20_attention_weight_min": -25.428903579711914,
      "activations/layer21_attention_weight_max": 29.65387535095215,
      "activations/layer21_attention_weight_min": -24.768863677978516,
      "activations/layer22_attention_weight_max": 40.366729736328125,
      "activations/layer22_attention_weight_min": -33.94770050048828,
      "activations/layer23_attention_weight_max": 31.46396255493164,
      "activations/layer23_attention_weight_min": -26.14773178100586,
      "activations/layer2_attention_weight_max": 31.544002532958984,
      "activations/layer2_attention_weight_min": -32.346771240234375,
      "activations/layer3_attention_weight_max": 100.49583435058594,
      "activations/layer3_attention_weight_min": -103.66149139404297,
      "activations/layer4_attention_weight_max": 104.14576721191406,
      "activations/layer4_attention_weight_min": -108.3911361694336,
      "activations/layer5_attention_weight_max": 51.513816833496094,
      "activations/layer5_attention_weight_min": -61.259559631347656,
      "activations/layer6_attention_weight_max": 44.786312103271484,
      "activations/layer6_attention_weight_min": -49.44593811035156,
      "activations/layer7_attention_weight_max": 98.27533721923828,
      "activations/layer7_attention_weight_min": -105.07933044433594,
      "activations/layer8_attention_weight_max": 42.117000579833984,
      "activations/layer8_attention_weight_min": -43.68721389770508,
      "activations/layer9_attention_weight_max": 31.996973037719727,
      "activations/layer9_attention_weight_min": -36.36912536621094,
      "epoch": 20.03,
      "learning_rate": 2.101212121212121e-05,
      "loss": 2.7084,
      "step": 344800
    },
    {
      "activations/layer0_attention_weight_max": 16.42219352722168,
      "activations/layer0_attention_weight_min": -12.544466018676758,
      "activations/layer10_attention_weight_max": 33.64387130737305,
      "activations/layer10_attention_weight_min": -35.67102813720703,
      "activations/layer11_attention_weight_max": 33.43346405029297,
      "activations/layer11_attention_weight_min": -35.800697326660156,
      "activations/layer12_attention_weight_max": 25.76645851135254,
      "activations/layer12_attention_weight_min": -26.92365837097168,
      "activations/layer13_attention_weight_max": 38.12696838378906,
      "activations/layer13_attention_weight_min": -35.156715393066406,
      "activations/layer14_attention_weight_max": 41.062774658203125,
      "activations/layer14_attention_weight_min": -37.65119552612305,
      "activations/layer15_attention_weight_max": 37.49768829345703,
      "activations/layer15_attention_weight_min": -35.5016975402832,
      "activations/layer16_attention_weight_max": 28.038774490356445,
      "activations/layer16_attention_weight_min": -29.50954246520996,
      "activations/layer17_attention_weight_max": 28.99440574645996,
      "activations/layer17_attention_weight_min": -25.42274284362793,
      "activations/layer18_attention_weight_max": 29.917583465576172,
      "activations/layer18_attention_weight_min": -26.976778030395508,
      "activations/layer19_attention_weight_max": 32.44615936279297,
      "activations/layer19_attention_weight_min": -31.77278709411621,
      "activations/layer1_attention_weight_max": 16.673919677734375,
      "activations/layer1_attention_weight_min": -14.697467803955078,
      "activations/layer20_attention_weight_max": 27.97713279724121,
      "activations/layer20_attention_weight_min": -24.995155334472656,
      "activations/layer21_attention_weight_max": 28.06387710571289,
      "activations/layer21_attention_weight_min": -24.913671493530273,
      "activations/layer22_attention_weight_max": 41.69708251953125,
      "activations/layer22_attention_weight_min": -31.929582595825195,
      "activations/layer23_attention_weight_max": 33.15174865722656,
      "activations/layer23_attention_weight_min": -25.32813835144043,
      "activations/layer2_attention_weight_max": 32.290679931640625,
      "activations/layer2_attention_weight_min": -31.271892547607422,
      "activations/layer3_attention_weight_max": 102.13578796386719,
      "activations/layer3_attention_weight_min": -100.24580383300781,
      "activations/layer4_attention_weight_max": 106.96272277832031,
      "activations/layer4_attention_weight_min": -107.5509262084961,
      "activations/layer5_attention_weight_max": 49.78772735595703,
      "activations/layer5_attention_weight_min": -60.72502136230469,
      "activations/layer6_attention_weight_max": 44.305259704589844,
      "activations/layer6_attention_weight_min": -47.79698944091797,
      "activations/layer7_attention_weight_max": 95.98869323730469,
      "activations/layer7_attention_weight_min": -94.7531509399414,
      "activations/layer8_attention_weight_max": 40.75740432739258,
      "activations/layer8_attention_weight_min": -44.27077102661133,
      "activations/layer9_attention_weight_max": 30.991811752319336,
      "activations/layer9_attention_weight_min": -34.23313522338867,
      "epoch": 20.04,
      "learning_rate": 2.0993181818181815e-05,
      "loss": 2.7142,
      "step": 344850
    },
    {
      "activations/layer0_attention_weight_max": 15.836393356323242,
      "activations/layer0_attention_weight_min": -13.933536529541016,
      "activations/layer10_attention_weight_max": 36.5960693359375,
      "activations/layer10_attention_weight_min": -34.506431579589844,
      "activations/layer11_attention_weight_max": 34.41843032836914,
      "activations/layer11_attention_weight_min": -34.14219665527344,
      "activations/layer12_attention_weight_max": 27.1761417388916,
      "activations/layer12_attention_weight_min": -26.638259887695312,
      "activations/layer13_attention_weight_max": 41.73252487182617,
      "activations/layer13_attention_weight_min": -36.38286590576172,
      "activations/layer14_attention_weight_max": 43.23579025268555,
      "activations/layer14_attention_weight_min": -40.95349884033203,
      "activations/layer15_attention_weight_max": 42.74018859863281,
      "activations/layer15_attention_weight_min": -38.979698181152344,
      "activations/layer16_attention_weight_max": 30.856542587280273,
      "activations/layer16_attention_weight_min": -30.161148071289062,
      "activations/layer17_attention_weight_max": 30.96108627319336,
      "activations/layer17_attention_weight_min": -26.232566833496094,
      "activations/layer18_attention_weight_max": 32.29905319213867,
      "activations/layer18_attention_weight_min": -27.686559677124023,
      "activations/layer19_attention_weight_max": 33.110233306884766,
      "activations/layer19_attention_weight_min": -32.943565368652344,
      "activations/layer1_attention_weight_max": 15.766646385192871,
      "activations/layer1_attention_weight_min": -15.49096393585205,
      "activations/layer20_attention_weight_max": 29.402538299560547,
      "activations/layer20_attention_weight_min": -27.496740341186523,
      "activations/layer21_attention_weight_max": 28.619796752929688,
      "activations/layer21_attention_weight_min": -27.287303924560547,
      "activations/layer22_attention_weight_max": 43.20043182373047,
      "activations/layer22_attention_weight_min": -34.08351135253906,
      "activations/layer23_attention_weight_max": 35.55875015258789,
      "activations/layer23_attention_weight_min": -26.872419357299805,
      "activations/layer2_attention_weight_max": 29.987096786499023,
      "activations/layer2_attention_weight_min": -30.329931259155273,
      "activations/layer3_attention_weight_max": 95.04729461669922,
      "activations/layer3_attention_weight_min": -99.13980865478516,
      "activations/layer4_attention_weight_max": 103.40327453613281,
      "activations/layer4_attention_weight_min": -109.96064758300781,
      "activations/layer5_attention_weight_max": 47.175384521484375,
      "activations/layer5_attention_weight_min": -63.3044548034668,
      "activations/layer6_attention_weight_max": 44.005653381347656,
      "activations/layer6_attention_weight_min": -48.67009735107422,
      "activations/layer7_attention_weight_max": 92.4650650024414,
      "activations/layer7_attention_weight_min": -96.26591491699219,
      "activations/layer8_attention_weight_max": 41.889198303222656,
      "activations/layer8_attention_weight_min": -43.886375427246094,
      "activations/layer9_attention_weight_max": 35.0816764831543,
      "activations/layer9_attention_weight_min": -33.82316970825195,
      "epoch": 20.04,
      "learning_rate": 2.0974242424242423e-05,
      "loss": 2.7258,
      "step": 344900
    },
    {
      "activations/layer0_attention_weight_max": 15.656482696533203,
      "activations/layer0_attention_weight_min": -13.402737617492676,
      "activations/layer10_attention_weight_max": 37.37257766723633,
      "activations/layer10_attention_weight_min": -36.716712951660156,
      "activations/layer11_attention_weight_max": 37.68202209472656,
      "activations/layer11_attention_weight_min": -37.04458999633789,
      "activations/layer12_attention_weight_max": 26.090307235717773,
      "activations/layer12_attention_weight_min": -28.548845291137695,
      "activations/layer13_attention_weight_max": 41.01280975341797,
      "activations/layer13_attention_weight_min": -38.346946716308594,
      "activations/layer14_attention_weight_max": 43.17559051513672,
      "activations/layer14_attention_weight_min": -40.43147277832031,
      "activations/layer15_attention_weight_max": 41.4825325012207,
      "activations/layer15_attention_weight_min": -37.85178756713867,
      "activations/layer16_attention_weight_max": 28.46942138671875,
      "activations/layer16_attention_weight_min": -27.94365119934082,
      "activations/layer17_attention_weight_max": 31.321807861328125,
      "activations/layer17_attention_weight_min": -26.366878509521484,
      "activations/layer18_attention_weight_max": 31.88720703125,
      "activations/layer18_attention_weight_min": -27.0805721282959,
      "activations/layer19_attention_weight_max": 32.27236557006836,
      "activations/layer19_attention_weight_min": -33.37672424316406,
      "activations/layer1_attention_weight_max": 15.826757431030273,
      "activations/layer1_attention_weight_min": -15.718910217285156,
      "activations/layer20_attention_weight_max": 27.426414489746094,
      "activations/layer20_attention_weight_min": -24.852575302124023,
      "activations/layer21_attention_weight_max": 28.326303482055664,
      "activations/layer21_attention_weight_min": -25.732385635375977,
      "activations/layer22_attention_weight_max": 40.66190719604492,
      "activations/layer22_attention_weight_min": -30.71771240234375,
      "activations/layer23_attention_weight_max": 30.53362274169922,
      "activations/layer23_attention_weight_min": -24.532201766967773,
      "activations/layer2_attention_weight_max": 32.12554168701172,
      "activations/layer2_attention_weight_min": -31.000137329101562,
      "activations/layer3_attention_weight_max": 103.62158203125,
      "activations/layer3_attention_weight_min": -106.54753875732422,
      "activations/layer4_attention_weight_max": 113.01261138916016,
      "activations/layer4_attention_weight_min": -113.09932708740234,
      "activations/layer5_attention_weight_max": 53.76813888549805,
      "activations/layer5_attention_weight_min": -63.137054443359375,
      "activations/layer6_attention_weight_max": 48.549381256103516,
      "activations/layer6_attention_weight_min": -51.166080474853516,
      "activations/layer7_attention_weight_max": 101.78028106689453,
      "activations/layer7_attention_weight_min": -104.90447235107422,
      "activations/layer8_attention_weight_max": 47.85148620605469,
      "activations/layer8_attention_weight_min": -48.710113525390625,
      "activations/layer9_attention_weight_max": 36.013484954833984,
      "activations/layer9_attention_weight_min": -37.603370666503906,
      "epoch": 20.04,
      "learning_rate": 2.0955303030303028e-05,
      "loss": 2.715,
      "step": 344950
    },
    {
      "activations/layer0_attention_weight_max": 17.473806381225586,
      "activations/layer0_attention_weight_min": -13.947065353393555,
      "activations/layer10_attention_weight_max": 33.09266662597656,
      "activations/layer10_attention_weight_min": -35.29765701293945,
      "activations/layer11_attention_weight_max": 32.59771728515625,
      "activations/layer11_attention_weight_min": -33.945213317871094,
      "activations/layer12_attention_weight_max": 28.22976303100586,
      "activations/layer12_attention_weight_min": -27.386045455932617,
      "activations/layer13_attention_weight_max": 41.71018600463867,
      "activations/layer13_attention_weight_min": -38.59938430786133,
      "activations/layer14_attention_weight_max": 44.37434005737305,
      "activations/layer14_attention_weight_min": -39.48752975463867,
      "activations/layer15_attention_weight_max": 40.012245178222656,
      "activations/layer15_attention_weight_min": -36.268497467041016,
      "activations/layer16_attention_weight_max": 29.057117462158203,
      "activations/layer16_attention_weight_min": -29.33030891418457,
      "activations/layer17_attention_weight_max": 28.543251037597656,
      "activations/layer17_attention_weight_min": -25.364734649658203,
      "activations/layer18_attention_weight_max": 33.790496826171875,
      "activations/layer18_attention_weight_min": -24.928220748901367,
      "activations/layer19_attention_weight_max": 37.92348098754883,
      "activations/layer19_attention_weight_min": -32.44985580444336,
      "activations/layer1_attention_weight_max": 16.32863426208496,
      "activations/layer1_attention_weight_min": -15.443093299865723,
      "activations/layer20_attention_weight_max": 32.98793029785156,
      "activations/layer20_attention_weight_min": -26.83928871154785,
      "activations/layer21_attention_weight_max": 32.41514587402344,
      "activations/layer21_attention_weight_min": -25.192527770996094,
      "activations/layer22_attention_weight_max": 48.40304946899414,
      "activations/layer22_attention_weight_min": -32.737464904785156,
      "activations/layer23_attention_weight_max": 34.29814147949219,
      "activations/layer23_attention_weight_min": -27.32373046875,
      "activations/layer2_attention_weight_max": 30.725059509277344,
      "activations/layer2_attention_weight_min": -30.664573669433594,
      "activations/layer3_attention_weight_max": 97.26160430908203,
      "activations/layer3_attention_weight_min": -98.11897277832031,
      "activations/layer4_attention_weight_max": 104.16871643066406,
      "activations/layer4_attention_weight_min": -109.22074127197266,
      "activations/layer5_attention_weight_max": 48.82255172729492,
      "activations/layer5_attention_weight_min": -62.32891845703125,
      "activations/layer6_attention_weight_max": 43.96794509887695,
      "activations/layer6_attention_weight_min": -49.06998062133789,
      "activations/layer7_attention_weight_max": 88.6033935546875,
      "activations/layer7_attention_weight_min": -95.57533264160156,
      "activations/layer8_attention_weight_max": 39.92554473876953,
      "activations/layer8_attention_weight_min": -43.898773193359375,
      "activations/layer9_attention_weight_max": 30.89752197265625,
      "activations/layer9_attention_weight_min": -35.7607307434082,
      "epoch": 20.05,
      "learning_rate": 2.0936363636363636e-05,
      "loss": 2.7014,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.5119,
      "eval_samples_per_second": 504.471,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.5119,
      "eval_openwebtext_samples_per_second": 504.471,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 2.0055,
      "eval_wikitext_samples_per_second": 227.377,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5683,
      "eval_lambada_samples_per_second": 508.87,
      "step": 345000
    },
    {
      "activations/layer0_attention_weight_max": 15.402427673339844,
      "activations/layer0_attention_weight_min": -14.467150688171387,
      "activations/layer10_attention_weight_max": 32.94827651977539,
      "activations/layer10_attention_weight_min": -33.47947311401367,
      "activations/layer11_attention_weight_max": 33.02256774902344,
      "activations/layer11_attention_weight_min": -32.609375,
      "activations/layer12_attention_weight_max": 27.11962127685547,
      "activations/layer12_attention_weight_min": -30.64579963684082,
      "activations/layer13_attention_weight_max": 43.90712356567383,
      "activations/layer13_attention_weight_min": -37.29425048828125,
      "activations/layer14_attention_weight_max": 45.596343994140625,
      "activations/layer14_attention_weight_min": -40.79019546508789,
      "activations/layer15_attention_weight_max": 45.58713912963867,
      "activations/layer15_attention_weight_min": -39.874813079833984,
      "activations/layer16_attention_weight_max": 33.92587661743164,
      "activations/layer16_attention_weight_min": -30.06993865966797,
      "activations/layer17_attention_weight_max": 33.65846252441406,
      "activations/layer17_attention_weight_min": -26.844928741455078,
      "activations/layer18_attention_weight_max": 33.257991790771484,
      "activations/layer18_attention_weight_min": -24.83727264404297,
      "activations/layer19_attention_weight_max": 39.75503158569336,
      "activations/layer19_attention_weight_min": -30.637800216674805,
      "activations/layer1_attention_weight_max": 15.789112091064453,
      "activations/layer1_attention_weight_min": -15.607207298278809,
      "activations/layer20_attention_weight_max": 33.89279556274414,
      "activations/layer20_attention_weight_min": -26.176000595092773,
      "activations/layer21_attention_weight_max": 32.41840744018555,
      "activations/layer21_attention_weight_min": -26.257389068603516,
      "activations/layer22_attention_weight_max": 47.04853820800781,
      "activations/layer22_attention_weight_min": -32.81996154785156,
      "activations/layer23_attention_weight_max": 37.03752899169922,
      "activations/layer23_attention_weight_min": -26.909818649291992,
      "activations/layer2_attention_weight_max": 31.321048736572266,
      "activations/layer2_attention_weight_min": -31.35138511657715,
      "activations/layer3_attention_weight_max": 101.05065155029297,
      "activations/layer3_attention_weight_min": -100.6025161743164,
      "activations/layer4_attention_weight_max": 105.23165130615234,
      "activations/layer4_attention_weight_min": -107.3604507446289,
      "activations/layer5_attention_weight_max": 48.32551193237305,
      "activations/layer5_attention_weight_min": -61.998680114746094,
      "activations/layer6_attention_weight_max": 45.60080337524414,
      "activations/layer6_attention_weight_min": -46.891597747802734,
      "activations/layer7_attention_weight_max": 97.00323486328125,
      "activations/layer7_attention_weight_min": -99.36141204833984,
      "activations/layer8_attention_weight_max": 43.77434158325195,
      "activations/layer8_attention_weight_min": -45.189048767089844,
      "activations/layer9_attention_weight_max": 30.458105087280273,
      "activations/layer9_attention_weight_min": -34.11899185180664,
      "epoch": 20.05,
      "learning_rate": 2.091742424242424e-05,
      "loss": 2.7011,
      "step": 345050
    },
    {
      "activations/layer0_attention_weight_max": 16.281139373779297,
      "activations/layer0_attention_weight_min": -14.119424819946289,
      "activations/layer10_attention_weight_max": 37.498680114746094,
      "activations/layer10_attention_weight_min": -38.809539794921875,
      "activations/layer11_attention_weight_max": 38.88990783691406,
      "activations/layer11_attention_weight_min": -36.522178649902344,
      "activations/layer12_attention_weight_max": 29.974245071411133,
      "activations/layer12_attention_weight_min": -28.188034057617188,
      "activations/layer13_attention_weight_max": 45.80866622924805,
      "activations/layer13_attention_weight_min": -39.75516891479492,
      "activations/layer14_attention_weight_max": 52.51613998413086,
      "activations/layer14_attention_weight_min": -41.240509033203125,
      "activations/layer15_attention_weight_max": 49.33820343017578,
      "activations/layer15_attention_weight_min": -41.54123306274414,
      "activations/layer16_attention_weight_max": 37.19192123413086,
      "activations/layer16_attention_weight_min": -29.695457458496094,
      "activations/layer17_attention_weight_max": 32.61397933959961,
      "activations/layer17_attention_weight_min": -28.79052734375,
      "activations/layer18_attention_weight_max": 38.62788009643555,
      "activations/layer18_attention_weight_min": -30.105619430541992,
      "activations/layer19_attention_weight_max": 38.14470672607422,
      "activations/layer19_attention_weight_min": -35.034542083740234,
      "activations/layer1_attention_weight_max": 15.829989433288574,
      "activations/layer1_attention_weight_min": -14.81926155090332,
      "activations/layer20_attention_weight_max": 31.943456649780273,
      "activations/layer20_attention_weight_min": -25.95671272277832,
      "activations/layer21_attention_weight_max": 35.29468536376953,
      "activations/layer21_attention_weight_min": -25.109905242919922,
      "activations/layer22_attention_weight_max": 47.863914489746094,
      "activations/layer22_attention_weight_min": -32.58818817138672,
      "activations/layer23_attention_weight_max": 35.41846466064453,
      "activations/layer23_attention_weight_min": -26.805158615112305,
      "activations/layer2_attention_weight_max": 31.298994064331055,
      "activations/layer2_attention_weight_min": -33.33893966674805,
      "activations/layer3_attention_weight_max": 103.02059173583984,
      "activations/layer3_attention_weight_min": -108.21739959716797,
      "activations/layer4_attention_weight_max": 109.5871353149414,
      "activations/layer4_attention_weight_min": -116.77190399169922,
      "activations/layer5_attention_weight_max": 53.24156951904297,
      "activations/layer5_attention_weight_min": -62.7457275390625,
      "activations/layer6_attention_weight_max": 45.288082122802734,
      "activations/layer6_attention_weight_min": -51.66972732543945,
      "activations/layer7_attention_weight_max": 101.16434478759766,
      "activations/layer7_attention_weight_min": -106.41069793701172,
      "activations/layer8_attention_weight_max": 44.94103240966797,
      "activations/layer8_attention_weight_min": -49.15599060058594,
      "activations/layer9_attention_weight_max": 34.11488342285156,
      "activations/layer9_attention_weight_min": -37.17861557006836,
      "epoch": 20.05,
      "learning_rate": 2.0898484848484846e-05,
      "loss": 2.7166,
      "step": 345100
    },
    {
      "activations/layer0_attention_weight_max": 17.314956665039062,
      "activations/layer0_attention_weight_min": -13.016237258911133,
      "activations/layer10_attention_weight_max": 35.78096389770508,
      "activations/layer10_attention_weight_min": -33.969078063964844,
      "activations/layer11_attention_weight_max": 39.770263671875,
      "activations/layer11_attention_weight_min": -35.46589660644531,
      "activations/layer12_attention_weight_max": 27.324508666992188,
      "activations/layer12_attention_weight_min": -27.39400863647461,
      "activations/layer13_attention_weight_max": 41.95928955078125,
      "activations/layer13_attention_weight_min": -39.05860900878906,
      "activations/layer14_attention_weight_max": 53.213504791259766,
      "activations/layer14_attention_weight_min": -42.20536422729492,
      "activations/layer15_attention_weight_max": 44.94687271118164,
      "activations/layer15_attention_weight_min": -41.87676239013672,
      "activations/layer16_attention_weight_max": 33.514286041259766,
      "activations/layer16_attention_weight_min": -29.73377227783203,
      "activations/layer17_attention_weight_max": 30.726343154907227,
      "activations/layer17_attention_weight_min": -28.698564529418945,
      "activations/layer18_attention_weight_max": 31.906574249267578,
      "activations/layer18_attention_weight_min": -26.61223030090332,
      "activations/layer19_attention_weight_max": 35.080650329589844,
      "activations/layer19_attention_weight_min": -32.61415481567383,
      "activations/layer1_attention_weight_max": 16.327449798583984,
      "activations/layer1_attention_weight_min": -14.227672576904297,
      "activations/layer20_attention_weight_max": 29.069839477539062,
      "activations/layer20_attention_weight_min": -23.71311378479004,
      "activations/layer21_attention_weight_max": 28.401657104492188,
      "activations/layer21_attention_weight_min": -23.548351287841797,
      "activations/layer22_attention_weight_max": 45.09431457519531,
      "activations/layer22_attention_weight_min": -31.64209747314453,
      "activations/layer23_attention_weight_max": 34.421875,
      "activations/layer23_attention_weight_min": -24.795013427734375,
      "activations/layer2_attention_weight_max": 31.505390167236328,
      "activations/layer2_attention_weight_min": -30.38742446899414,
      "activations/layer3_attention_weight_max": 98.6762466430664,
      "activations/layer3_attention_weight_min": -99.1817398071289,
      "activations/layer4_attention_weight_max": 106.5252914428711,
      "activations/layer4_attention_weight_min": -105.22957611083984,
      "activations/layer5_attention_weight_max": 48.203033447265625,
      "activations/layer5_attention_weight_min": -60.759429931640625,
      "activations/layer6_attention_weight_max": 41.84275817871094,
      "activations/layer6_attention_weight_min": -44.838863372802734,
      "activations/layer7_attention_weight_max": 98.01453399658203,
      "activations/layer7_attention_weight_min": -95.8735580444336,
      "activations/layer8_attention_weight_max": 41.72023391723633,
      "activations/layer8_attention_weight_min": -43.636348724365234,
      "activations/layer9_attention_weight_max": 31.218503952026367,
      "activations/layer9_attention_weight_min": -32.3155632019043,
      "epoch": 20.06,
      "learning_rate": 2.0879545454545454e-05,
      "loss": 2.7142,
      "step": 345150
    },
    {
      "activations/layer0_attention_weight_max": 16.007305145263672,
      "activations/layer0_attention_weight_min": -13.046435356140137,
      "activations/layer10_attention_weight_max": 33.01142501831055,
      "activations/layer10_attention_weight_min": -35.50649642944336,
      "activations/layer11_attention_weight_max": 33.155582427978516,
      "activations/layer11_attention_weight_min": -33.84355163574219,
      "activations/layer12_attention_weight_max": 24.21961212158203,
      "activations/layer12_attention_weight_min": -26.759366989135742,
      "activations/layer13_attention_weight_max": 33.48109436035156,
      "activations/layer13_attention_weight_min": -35.02670669555664,
      "activations/layer14_attention_weight_max": 35.986724853515625,
      "activations/layer14_attention_weight_min": -36.01508331298828,
      "activations/layer15_attention_weight_max": 34.206119537353516,
      "activations/layer15_attention_weight_min": -39.101158142089844,
      "activations/layer16_attention_weight_max": 27.13150405883789,
      "activations/layer16_attention_weight_min": -28.85820770263672,
      "activations/layer17_attention_weight_max": 26.17926788330078,
      "activations/layer17_attention_weight_min": -24.885839462280273,
      "activations/layer18_attention_weight_max": 31.14708137512207,
      "activations/layer18_attention_weight_min": -26.835861206054688,
      "activations/layer19_attention_weight_max": 29.996997833251953,
      "activations/layer19_attention_weight_min": -34.466468811035156,
      "activations/layer1_attention_weight_max": 15.737768173217773,
      "activations/layer1_attention_weight_min": -15.456924438476562,
      "activations/layer20_attention_weight_max": 28.669797897338867,
      "activations/layer20_attention_weight_min": -26.404102325439453,
      "activations/layer21_attention_weight_max": 28.934463500976562,
      "activations/layer21_attention_weight_min": -25.983806610107422,
      "activations/layer22_attention_weight_max": 38.50260925292969,
      "activations/layer22_attention_weight_min": -33.885643005371094,
      "activations/layer23_attention_weight_max": 28.76065444946289,
      "activations/layer23_attention_weight_min": -31.21033477783203,
      "activations/layer2_attention_weight_max": 32.757869720458984,
      "activations/layer2_attention_weight_min": -33.199668884277344,
      "activations/layer3_attention_weight_max": 101.08831024169922,
      "activations/layer3_attention_weight_min": -106.4339828491211,
      "activations/layer4_attention_weight_max": 105.42203521728516,
      "activations/layer4_attention_weight_min": -109.46868896484375,
      "activations/layer5_attention_weight_max": 52.528404235839844,
      "activations/layer5_attention_weight_min": -60.79269027709961,
      "activations/layer6_attention_weight_max": 43.5567741394043,
      "activations/layer6_attention_weight_min": -47.70221710205078,
      "activations/layer7_attention_weight_max": 89.35123443603516,
      "activations/layer7_attention_weight_min": -97.9818115234375,
      "activations/layer8_attention_weight_max": 39.097625732421875,
      "activations/layer8_attention_weight_min": -42.9970817565918,
      "activations/layer9_attention_weight_max": 30.5394344329834,
      "activations/layer9_attention_weight_min": -33.892032623291016,
      "epoch": 20.06,
      "learning_rate": 2.086060606060606e-05,
      "loss": 2.6984,
      "step": 345200
    },
    {
      "activations/layer0_attention_weight_max": 15.748547554016113,
      "activations/layer0_attention_weight_min": -12.398148536682129,
      "activations/layer10_attention_weight_max": 35.799522399902344,
      "activations/layer10_attention_weight_min": -36.548927307128906,
      "activations/layer11_attention_weight_max": 36.002464294433594,
      "activations/layer11_attention_weight_min": -35.674720764160156,
      "activations/layer12_attention_weight_max": 26.965288162231445,
      "activations/layer12_attention_weight_min": -27.903194427490234,
      "activations/layer13_attention_weight_max": 41.5748405456543,
      "activations/layer13_attention_weight_min": -40.69401931762695,
      "activations/layer14_attention_weight_max": 45.40754318237305,
      "activations/layer14_attention_weight_min": -39.38756561279297,
      "activations/layer15_attention_weight_max": 42.32016372680664,
      "activations/layer15_attention_weight_min": -40.746341705322266,
      "activations/layer16_attention_weight_max": 30.363676071166992,
      "activations/layer16_attention_weight_min": -28.064184188842773,
      "activations/layer17_attention_weight_max": 29.316598892211914,
      "activations/layer17_attention_weight_min": -26.411388397216797,
      "activations/layer18_attention_weight_max": 34.26346206665039,
      "activations/layer18_attention_weight_min": -25.8999080657959,
      "activations/layer19_attention_weight_max": 34.5891227722168,
      "activations/layer19_attention_weight_min": -31.012399673461914,
      "activations/layer1_attention_weight_max": 16.266067504882812,
      "activations/layer1_attention_weight_min": -14.906717300415039,
      "activations/layer20_attention_weight_max": 29.083574295043945,
      "activations/layer20_attention_weight_min": -24.44110107421875,
      "activations/layer21_attention_weight_max": 28.38644790649414,
      "activations/layer21_attention_weight_min": -24.103424072265625,
      "activations/layer22_attention_weight_max": 41.44223403930664,
      "activations/layer22_attention_weight_min": -30.237403869628906,
      "activations/layer23_attention_weight_max": 34.46941375732422,
      "activations/layer23_attention_weight_min": -24.163196563720703,
      "activations/layer2_attention_weight_max": 32.7851448059082,
      "activations/layer2_attention_weight_min": -32.71099853515625,
      "activations/layer3_attention_weight_max": 98.99071502685547,
      "activations/layer3_attention_weight_min": -105.15487670898438,
      "activations/layer4_attention_weight_max": 106.9379653930664,
      "activations/layer4_attention_weight_min": -107.9367904663086,
      "activations/layer5_attention_weight_max": 49.905006408691406,
      "activations/layer5_attention_weight_min": -62.48411178588867,
      "activations/layer6_attention_weight_max": 45.97929763793945,
      "activations/layer6_attention_weight_min": -48.23212814331055,
      "activations/layer7_attention_weight_max": 95.60975646972656,
      "activations/layer7_attention_weight_min": -99.94855499267578,
      "activations/layer8_attention_weight_max": 47.3775749206543,
      "activations/layer8_attention_weight_min": -46.78960037231445,
      "activations/layer9_attention_weight_max": 34.402767181396484,
      "activations/layer9_attention_weight_min": -35.52518844604492,
      "epoch": 20.06,
      "learning_rate": 2.0841666666666667e-05,
      "loss": 2.7053,
      "step": 345250
    },
    {
      "activations/layer0_attention_weight_max": 15.589193344116211,
      "activations/layer0_attention_weight_min": -14.149588584899902,
      "activations/layer10_attention_weight_max": 34.60060501098633,
      "activations/layer10_attention_weight_min": -36.61424255371094,
      "activations/layer11_attention_weight_max": 33.955989837646484,
      "activations/layer11_attention_weight_min": -35.75715255737305,
      "activations/layer12_attention_weight_max": 24.8367919921875,
      "activations/layer12_attention_weight_min": -26.11368751525879,
      "activations/layer13_attention_weight_max": 37.805137634277344,
      "activations/layer13_attention_weight_min": -33.453121185302734,
      "activations/layer14_attention_weight_max": 39.528621673583984,
      "activations/layer14_attention_weight_min": -38.73367691040039,
      "activations/layer15_attention_weight_max": 37.96505355834961,
      "activations/layer15_attention_weight_min": -34.720523834228516,
      "activations/layer16_attention_weight_max": 27.63614845275879,
      "activations/layer16_attention_weight_min": -27.06075668334961,
      "activations/layer17_attention_weight_max": 27.343088150024414,
      "activations/layer17_attention_weight_min": -24.325387954711914,
      "activations/layer18_attention_weight_max": 31.020736694335938,
      "activations/layer18_attention_weight_min": -25.261714935302734,
      "activations/layer19_attention_weight_max": 33.94382858276367,
      "activations/layer19_attention_weight_min": -33.094722747802734,
      "activations/layer1_attention_weight_max": 16.35366439819336,
      "activations/layer1_attention_weight_min": -15.484848976135254,
      "activations/layer20_attention_weight_max": 29.546913146972656,
      "activations/layer20_attention_weight_min": -25.780982971191406,
      "activations/layer21_attention_weight_max": 28.01280403137207,
      "activations/layer21_attention_weight_min": -24.77972984313965,
      "activations/layer22_attention_weight_max": 39.13747024536133,
      "activations/layer22_attention_weight_min": -31.450031280517578,
      "activations/layer23_attention_weight_max": 32.734161376953125,
      "activations/layer23_attention_weight_min": -27.899391174316406,
      "activations/layer2_attention_weight_max": 31.362302780151367,
      "activations/layer2_attention_weight_min": -31.855558395385742,
      "activations/layer3_attention_weight_max": 99.21472930908203,
      "activations/layer3_attention_weight_min": -103.83612060546875,
      "activations/layer4_attention_weight_max": 104.91947174072266,
      "activations/layer4_attention_weight_min": -113.266357421875,
      "activations/layer5_attention_weight_max": 49.699790954589844,
      "activations/layer5_attention_weight_min": -63.85830307006836,
      "activations/layer6_attention_weight_max": 44.64855194091797,
      "activations/layer6_attention_weight_min": -47.8494758605957,
      "activations/layer7_attention_weight_max": 93.53334045410156,
      "activations/layer7_attention_weight_min": -103.99044799804688,
      "activations/layer8_attention_weight_max": 42.01522445678711,
      "activations/layer8_attention_weight_min": -45.109230041503906,
      "activations/layer9_attention_weight_max": 33.011478424072266,
      "activations/layer9_attention_weight_min": -35.67488479614258,
      "epoch": 20.06,
      "learning_rate": 2.082272727272727e-05,
      "loss": 2.7016,
      "step": 345300
    },
    {
      "activations/layer0_attention_weight_max": 15.85599422454834,
      "activations/layer0_attention_weight_min": -13.055776596069336,
      "activations/layer10_attention_weight_max": 35.973663330078125,
      "activations/layer10_attention_weight_min": -35.45383071899414,
      "activations/layer11_attention_weight_max": 36.371551513671875,
      "activations/layer11_attention_weight_min": -37.25706481933594,
      "activations/layer12_attention_weight_max": 29.222837448120117,
      "activations/layer12_attention_weight_min": -27.70297622680664,
      "activations/layer13_attention_weight_max": 43.21193313598633,
      "activations/layer13_attention_weight_min": -37.58039093017578,
      "activations/layer14_attention_weight_max": 44.90044021606445,
      "activations/layer14_attention_weight_min": -40.408504486083984,
      "activations/layer15_attention_weight_max": 42.17723846435547,
      "activations/layer15_attention_weight_min": -36.64333724975586,
      "activations/layer16_attention_weight_max": 33.05944061279297,
      "activations/layer16_attention_weight_min": -30.311323165893555,
      "activations/layer17_attention_weight_max": 33.765892028808594,
      "activations/layer17_attention_weight_min": -27.713634490966797,
      "activations/layer18_attention_weight_max": 31.33172035217285,
      "activations/layer18_attention_weight_min": -26.74872398376465,
      "activations/layer19_attention_weight_max": 32.33640670776367,
      "activations/layer19_attention_weight_min": -31.14735221862793,
      "activations/layer1_attention_weight_max": 16.77546501159668,
      "activations/layer1_attention_weight_min": -15.245810508728027,
      "activations/layer20_attention_weight_max": 28.870065689086914,
      "activations/layer20_attention_weight_min": -25.592849731445312,
      "activations/layer21_attention_weight_max": 28.673168182373047,
      "activations/layer21_attention_weight_min": -26.02220916748047,
      "activations/layer22_attention_weight_max": 41.845458984375,
      "activations/layer22_attention_weight_min": -31.79017448425293,
      "activations/layer23_attention_weight_max": 32.12522888183594,
      "activations/layer23_attention_weight_min": -26.92090606689453,
      "activations/layer2_attention_weight_max": 31.031843185424805,
      "activations/layer2_attention_weight_min": -31.522167205810547,
      "activations/layer3_attention_weight_max": 102.29728698730469,
      "activations/layer3_attention_weight_min": -105.95589447021484,
      "activations/layer4_attention_weight_max": 108.28984832763672,
      "activations/layer4_attention_weight_min": -111.66728973388672,
      "activations/layer5_attention_weight_max": 52.35316467285156,
      "activations/layer5_attention_weight_min": -63.070186614990234,
      "activations/layer6_attention_weight_max": 45.041080474853516,
      "activations/layer6_attention_weight_min": -47.994197845458984,
      "activations/layer7_attention_weight_max": 97.94355773925781,
      "activations/layer7_attention_weight_min": -103.13846588134766,
      "activations/layer8_attention_weight_max": 43.59656524658203,
      "activations/layer8_attention_weight_min": -48.07323455810547,
      "activations/layer9_attention_weight_max": 33.3694953918457,
      "activations/layer9_attention_weight_min": -34.83073425292969,
      "epoch": 20.07,
      "learning_rate": 2.0803787878787877e-05,
      "loss": 2.7202,
      "step": 345350
    },
    {
      "activations/layer0_attention_weight_max": 16.474903106689453,
      "activations/layer0_attention_weight_min": -12.372734069824219,
      "activations/layer10_attention_weight_max": 32.77753448486328,
      "activations/layer10_attention_weight_min": -34.03425216674805,
      "activations/layer11_attention_weight_max": 35.14364242553711,
      "activations/layer11_attention_weight_min": -34.15626525878906,
      "activations/layer12_attention_weight_max": 28.214963912963867,
      "activations/layer12_attention_weight_min": -29.209339141845703,
      "activations/layer13_attention_weight_max": 39.66243362426758,
      "activations/layer13_attention_weight_min": -34.62064743041992,
      "activations/layer14_attention_weight_max": 41.6992301940918,
      "activations/layer14_attention_weight_min": -35.9675178527832,
      "activations/layer15_attention_weight_max": 36.676509857177734,
      "activations/layer15_attention_weight_min": -35.8725700378418,
      "activations/layer16_attention_weight_max": 28.140926361083984,
      "activations/layer16_attention_weight_min": -28.37224006652832,
      "activations/layer17_attention_weight_max": 28.795501708984375,
      "activations/layer17_attention_weight_min": -26.01299285888672,
      "activations/layer18_attention_weight_max": 28.82171058654785,
      "activations/layer18_attention_weight_min": -25.983922958374023,
      "activations/layer19_attention_weight_max": 31.687240600585938,
      "activations/layer19_attention_weight_min": -32.73828887939453,
      "activations/layer1_attention_weight_max": 15.613337516784668,
      "activations/layer1_attention_weight_min": -14.834999084472656,
      "activations/layer20_attention_weight_max": 27.296201705932617,
      "activations/layer20_attention_weight_min": -25.998106002807617,
      "activations/layer21_attention_weight_max": 27.28306007385254,
      "activations/layer21_attention_weight_min": -26.727828979492188,
      "activations/layer22_attention_weight_max": 37.50178909301758,
      "activations/layer22_attention_weight_min": -33.70960998535156,
      "activations/layer23_attention_weight_max": 29.673423767089844,
      "activations/layer23_attention_weight_min": -26.869915008544922,
      "activations/layer2_attention_weight_max": 31.04367446899414,
      "activations/layer2_attention_weight_min": -31.890640258789062,
      "activations/layer3_attention_weight_max": 106.18415069580078,
      "activations/layer3_attention_weight_min": -104.34431457519531,
      "activations/layer4_attention_weight_max": 108.21404266357422,
      "activations/layer4_attention_weight_min": -108.54351043701172,
      "activations/layer5_attention_weight_max": 51.17367172241211,
      "activations/layer5_attention_weight_min": -59.05195617675781,
      "activations/layer6_attention_weight_max": 43.402339935302734,
      "activations/layer6_attention_weight_min": -47.349124908447266,
      "activations/layer7_attention_weight_max": 94.96786499023438,
      "activations/layer7_attention_weight_min": -100.29901123046875,
      "activations/layer8_attention_weight_max": 41.22449493408203,
      "activations/layer8_attention_weight_min": -45.73765182495117,
      "activations/layer9_attention_weight_max": 30.98406982421875,
      "activations/layer9_attention_weight_min": -32.78058624267578,
      "epoch": 20.07,
      "learning_rate": 2.0784848484848482e-05,
      "loss": 2.6959,
      "step": 345400
    },
    {
      "activations/layer0_attention_weight_max": 16.878143310546875,
      "activations/layer0_attention_weight_min": -13.669778823852539,
      "activations/layer10_attention_weight_max": 36.67313003540039,
      "activations/layer10_attention_weight_min": -35.52290344238281,
      "activations/layer11_attention_weight_max": 38.68511962890625,
      "activations/layer11_attention_weight_min": -36.07233810424805,
      "activations/layer12_attention_weight_max": 28.961408615112305,
      "activations/layer12_attention_weight_min": -27.365161895751953,
      "activations/layer13_attention_weight_max": 40.27855682373047,
      "activations/layer13_attention_weight_min": -38.10443878173828,
      "activations/layer14_attention_weight_max": 42.90457534790039,
      "activations/layer14_attention_weight_min": -38.95402526855469,
      "activations/layer15_attention_weight_max": 43.401268005371094,
      "activations/layer15_attention_weight_min": -37.365169525146484,
      "activations/layer16_attention_weight_max": 34.23065185546875,
      "activations/layer16_attention_weight_min": -29.353111267089844,
      "activations/layer17_attention_weight_max": 29.636505126953125,
      "activations/layer17_attention_weight_min": -27.7430362701416,
      "activations/layer18_attention_weight_max": 33.372039794921875,
      "activations/layer18_attention_weight_min": -30.384544372558594,
      "activations/layer19_attention_weight_max": 33.456932067871094,
      "activations/layer19_attention_weight_min": -34.683101654052734,
      "activations/layer1_attention_weight_max": 15.77086353302002,
      "activations/layer1_attention_weight_min": -14.841678619384766,
      "activations/layer20_attention_weight_max": 27.556182861328125,
      "activations/layer20_attention_weight_min": -26.647123336791992,
      "activations/layer21_attention_weight_max": 29.190032958984375,
      "activations/layer21_attention_weight_min": -27.138883590698242,
      "activations/layer22_attention_weight_max": 42.352420806884766,
      "activations/layer22_attention_weight_min": -37.224483489990234,
      "activations/layer23_attention_weight_max": 32.494503021240234,
      "activations/layer23_attention_weight_min": -27.804485321044922,
      "activations/layer2_attention_weight_max": 34.38355255126953,
      "activations/layer2_attention_weight_min": -32.76194763183594,
      "activations/layer3_attention_weight_max": 102.6509017944336,
      "activations/layer3_attention_weight_min": -101.32122039794922,
      "activations/layer4_attention_weight_max": 108.13812255859375,
      "activations/layer4_attention_weight_min": -108.08414459228516,
      "activations/layer5_attention_weight_max": 51.321311950683594,
      "activations/layer5_attention_weight_min": -60.755882263183594,
      "activations/layer6_attention_weight_max": 45.384456634521484,
      "activations/layer6_attention_weight_min": -48.35960006713867,
      "activations/layer7_attention_weight_max": 96.93038177490234,
      "activations/layer7_attention_weight_min": -104.69548034667969,
      "activations/layer8_attention_weight_max": 43.62126159667969,
      "activations/layer8_attention_weight_min": -45.74640655517578,
      "activations/layer9_attention_weight_max": 33.41621398925781,
      "activations/layer9_attention_weight_min": -34.1001091003418,
      "epoch": 20.07,
      "learning_rate": 2.076590909090909e-05,
      "loss": 2.7344,
      "step": 345450
    },
    {
      "activations/layer0_attention_weight_max": 16.639793395996094,
      "activations/layer0_attention_weight_min": -12.687546730041504,
      "activations/layer10_attention_weight_max": 33.99512481689453,
      "activations/layer10_attention_weight_min": -33.744911193847656,
      "activations/layer11_attention_weight_max": 33.39524459838867,
      "activations/layer11_attention_weight_min": -34.34063720703125,
      "activations/layer12_attention_weight_max": 28.709598541259766,
      "activations/layer12_attention_weight_min": -26.469009399414062,
      "activations/layer13_attention_weight_max": 42.66874694824219,
      "activations/layer13_attention_weight_min": -36.400779724121094,
      "activations/layer14_attention_weight_max": 45.93507766723633,
      "activations/layer14_attention_weight_min": -37.87716293334961,
      "activations/layer15_attention_weight_max": 41.898319244384766,
      "activations/layer15_attention_weight_min": -34.256168365478516,
      "activations/layer16_attention_weight_max": 31.15357208251953,
      "activations/layer16_attention_weight_min": -28.605884552001953,
      "activations/layer17_attention_weight_max": 30.775211334228516,
      "activations/layer17_attention_weight_min": -28.04043960571289,
      "activations/layer18_attention_weight_max": 35.13539505004883,
      "activations/layer18_attention_weight_min": -26.586063385009766,
      "activations/layer19_attention_weight_max": 36.59519958496094,
      "activations/layer19_attention_weight_min": -34.59749984741211,
      "activations/layer1_attention_weight_max": 15.201273918151855,
      "activations/layer1_attention_weight_min": -14.69072437286377,
      "activations/layer20_attention_weight_max": 28.973644256591797,
      "activations/layer20_attention_weight_min": -28.223159790039062,
      "activations/layer21_attention_weight_max": 31.164188385009766,
      "activations/layer21_attention_weight_min": -25.978485107421875,
      "activations/layer22_attention_weight_max": 52.624237060546875,
      "activations/layer22_attention_weight_min": -34.15928268432617,
      "activations/layer23_attention_weight_max": 35.93464279174805,
      "activations/layer23_attention_weight_min": -28.070295333862305,
      "activations/layer2_attention_weight_max": 32.02660369873047,
      "activations/layer2_attention_weight_min": -31.959125518798828,
      "activations/layer3_attention_weight_max": 103.24647521972656,
      "activations/layer3_attention_weight_min": -100.45526885986328,
      "activations/layer4_attention_weight_max": 109.37117767333984,
      "activations/layer4_attention_weight_min": -112.4758529663086,
      "activations/layer5_attention_weight_max": 53.67502975463867,
      "activations/layer5_attention_weight_min": -62.50579833984375,
      "activations/layer6_attention_weight_max": 48.41031265258789,
      "activations/layer6_attention_weight_min": -47.942989349365234,
      "activations/layer7_attention_weight_max": 93.78897094726562,
      "activations/layer7_attention_weight_min": -96.64114379882812,
      "activations/layer8_attention_weight_max": 43.95253372192383,
      "activations/layer8_attention_weight_min": -44.40092086791992,
      "activations/layer9_attention_weight_max": 32.76701354980469,
      "activations/layer9_attention_weight_min": -34.751590728759766,
      "epoch": 20.08,
      "learning_rate": 2.0746969696969695e-05,
      "loss": 2.7206,
      "step": 345500
    },
    {
      "activations/layer0_attention_weight_max": 15.43333625793457,
      "activations/layer0_attention_weight_min": -13.24560832977295,
      "activations/layer10_attention_weight_max": 34.07269287109375,
      "activations/layer10_attention_weight_min": -36.05424499511719,
      "activations/layer11_attention_weight_max": 32.000457763671875,
      "activations/layer11_attention_weight_min": -34.297908782958984,
      "activations/layer12_attention_weight_max": 25.302839279174805,
      "activations/layer12_attention_weight_min": -26.830591201782227,
      "activations/layer13_attention_weight_max": 39.98284912109375,
      "activations/layer13_attention_weight_min": -34.188663482666016,
      "activations/layer14_attention_weight_max": 42.77790451049805,
      "activations/layer14_attention_weight_min": -38.4905891418457,
      "activations/layer15_attention_weight_max": 40.54073715209961,
      "activations/layer15_attention_weight_min": -36.67462158203125,
      "activations/layer16_attention_weight_max": 29.357986450195312,
      "activations/layer16_attention_weight_min": -28.405118942260742,
      "activations/layer17_attention_weight_max": 28.070505142211914,
      "activations/layer17_attention_weight_min": -29.59892463684082,
      "activations/layer18_attention_weight_max": 33.469329833984375,
      "activations/layer18_attention_weight_min": -28.439653396606445,
      "activations/layer19_attention_weight_max": 34.99201965332031,
      "activations/layer19_attention_weight_min": -30.8790225982666,
      "activations/layer1_attention_weight_max": 16.1177978515625,
      "activations/layer1_attention_weight_min": -13.772697448730469,
      "activations/layer20_attention_weight_max": 28.17991828918457,
      "activations/layer20_attention_weight_min": -24.69203758239746,
      "activations/layer21_attention_weight_max": 28.66753387451172,
      "activations/layer21_attention_weight_min": -25.85516357421875,
      "activations/layer22_attention_weight_max": 38.39564895629883,
      "activations/layer22_attention_weight_min": -33.76588821411133,
      "activations/layer23_attention_weight_max": 31.747211456298828,
      "activations/layer23_attention_weight_min": -26.943084716796875,
      "activations/layer2_attention_weight_max": 32.648284912109375,
      "activations/layer2_attention_weight_min": -31.36597442626953,
      "activations/layer3_attention_weight_max": 100.93822479248047,
      "activations/layer3_attention_weight_min": -101.9899673461914,
      "activations/layer4_attention_weight_max": 108.29449462890625,
      "activations/layer4_attention_weight_min": -117.11620330810547,
      "activations/layer5_attention_weight_max": 50.35480499267578,
      "activations/layer5_attention_weight_min": -65.80196380615234,
      "activations/layer6_attention_weight_max": 43.72846603393555,
      "activations/layer6_attention_weight_min": -47.314762115478516,
      "activations/layer7_attention_weight_max": 94.168701171875,
      "activations/layer7_attention_weight_min": -95.51318359375,
      "activations/layer8_attention_weight_max": 40.52201461791992,
      "activations/layer8_attention_weight_min": -42.58812713623047,
      "activations/layer9_attention_weight_max": 30.3839168548584,
      "activations/layer9_attention_weight_min": -34.919090270996094,
      "epoch": 20.08,
      "learning_rate": 2.0728030303030303e-05,
      "loss": 2.7189,
      "step": 345550
    },
    {
      "activations/layer0_attention_weight_max": 16.289148330688477,
      "activations/layer0_attention_weight_min": -13.494924545288086,
      "activations/layer10_attention_weight_max": 36.30900192260742,
      "activations/layer10_attention_weight_min": -37.329410552978516,
      "activations/layer11_attention_weight_max": 34.734832763671875,
      "activations/layer11_attention_weight_min": -36.516998291015625,
      "activations/layer12_attention_weight_max": 27.215843200683594,
      "activations/layer12_attention_weight_min": -29.24528694152832,
      "activations/layer13_attention_weight_max": 39.357704162597656,
      "activations/layer13_attention_weight_min": -38.74055099487305,
      "activations/layer14_attention_weight_max": 40.935123443603516,
      "activations/layer14_attention_weight_min": -39.38021469116211,
      "activations/layer15_attention_weight_max": 38.994564056396484,
      "activations/layer15_attention_weight_min": -36.37309646606445,
      "activations/layer16_attention_weight_max": 29.049875259399414,
      "activations/layer16_attention_weight_min": -27.142391204833984,
      "activations/layer17_attention_weight_max": 28.66390609741211,
      "activations/layer17_attention_weight_min": -26.799753189086914,
      "activations/layer18_attention_weight_max": 32.550628662109375,
      "activations/layer18_attention_weight_min": -25.1795597076416,
      "activations/layer19_attention_weight_max": 33.72325134277344,
      "activations/layer19_attention_weight_min": -30.309457778930664,
      "activations/layer1_attention_weight_max": 15.227148056030273,
      "activations/layer1_attention_weight_min": -14.240184783935547,
      "activations/layer20_attention_weight_max": 30.828014373779297,
      "activations/layer20_attention_weight_min": -24.739042282104492,
      "activations/layer21_attention_weight_max": 29.1617488861084,
      "activations/layer21_attention_weight_min": -23.869853973388672,
      "activations/layer22_attention_weight_max": 45.40623474121094,
      "activations/layer22_attention_weight_min": -30.199615478515625,
      "activations/layer23_attention_weight_max": 32.2996826171875,
      "activations/layer23_attention_weight_min": -24.304630279541016,
      "activations/layer2_attention_weight_max": 31.461063385009766,
      "activations/layer2_attention_weight_min": -33.164207458496094,
      "activations/layer3_attention_weight_max": 100.77478790283203,
      "activations/layer3_attention_weight_min": -105.718994140625,
      "activations/layer4_attention_weight_max": 107.59919738769531,
      "activations/layer4_attention_weight_min": -114.3443832397461,
      "activations/layer5_attention_weight_max": 52.0009765625,
      "activations/layer5_attention_weight_min": -62.72367477416992,
      "activations/layer6_attention_weight_max": 46.11653518676758,
      "activations/layer6_attention_weight_min": -50.64460754394531,
      "activations/layer7_attention_weight_max": 96.00952911376953,
      "activations/layer7_attention_weight_min": -107.10195922851562,
      "activations/layer8_attention_weight_max": 45.4426383972168,
      "activations/layer8_attention_weight_min": -47.22373580932617,
      "activations/layer9_attention_weight_max": 33.39101791381836,
      "activations/layer9_attention_weight_min": -35.39680099487305,
      "epoch": 20.08,
      "learning_rate": 2.0709469696969694e-05,
      "loss": 2.7104,
      "step": 345600
    },
    {
      "activations/layer0_attention_weight_max": 17.223209381103516,
      "activations/layer0_attention_weight_min": -13.561357498168945,
      "activations/layer10_attention_weight_max": 32.55550765991211,
      "activations/layer10_attention_weight_min": -35.4521369934082,
      "activations/layer11_attention_weight_max": 32.30230712890625,
      "activations/layer11_attention_weight_min": -36.638587951660156,
      "activations/layer12_attention_weight_max": 26.615036010742188,
      "activations/layer12_attention_weight_min": -25.17044448852539,
      "activations/layer13_attention_weight_max": 40.02372741699219,
      "activations/layer13_attention_weight_min": -36.355857849121094,
      "activations/layer14_attention_weight_max": 42.79061508178711,
      "activations/layer14_attention_weight_min": -37.229652404785156,
      "activations/layer15_attention_weight_max": 41.174957275390625,
      "activations/layer15_attention_weight_min": -35.96842575073242,
      "activations/layer16_attention_weight_max": 30.076974868774414,
      "activations/layer16_attention_weight_min": -26.28886604309082,
      "activations/layer17_attention_weight_max": 29.12709617614746,
      "activations/layer17_attention_weight_min": -25.42888832092285,
      "activations/layer18_attention_weight_max": 31.022245407104492,
      "activations/layer18_attention_weight_min": -24.80731773376465,
      "activations/layer19_attention_weight_max": 30.973073959350586,
      "activations/layer19_attention_weight_min": -31.932920455932617,
      "activations/layer1_attention_weight_max": 15.815849304199219,
      "activations/layer1_attention_weight_min": -15.610249519348145,
      "activations/layer20_attention_weight_max": 28.58494758605957,
      "activations/layer20_attention_weight_min": -26.009563446044922,
      "activations/layer21_attention_weight_max": 28.036983489990234,
      "activations/layer21_attention_weight_min": -24.81451988220215,
      "activations/layer22_attention_weight_max": 37.86053466796875,
      "activations/layer22_attention_weight_min": -31.142066955566406,
      "activations/layer23_attention_weight_max": 32.017154693603516,
      "activations/layer23_attention_weight_min": -26.667884826660156,
      "activations/layer2_attention_weight_max": 32.96894073486328,
      "activations/layer2_attention_weight_min": -30.84075927734375,
      "activations/layer3_attention_weight_max": 98.84452056884766,
      "activations/layer3_attention_weight_min": -101.14128112792969,
      "activations/layer4_attention_weight_max": 105.7625732421875,
      "activations/layer4_attention_weight_min": -111.13492584228516,
      "activations/layer5_attention_weight_max": 51.70101547241211,
      "activations/layer5_attention_weight_min": -59.966697692871094,
      "activations/layer6_attention_weight_max": 44.18081283569336,
      "activations/layer6_attention_weight_min": -46.248966217041016,
      "activations/layer7_attention_weight_max": 95.46560668945312,
      "activations/layer7_attention_weight_min": -94.83853149414062,
      "activations/layer8_attention_weight_max": 43.83454895019531,
      "activations/layer8_attention_weight_min": -43.921783447265625,
      "activations/layer9_attention_weight_max": 32.18891906738281,
      "activations/layer9_attention_weight_min": -33.3625602722168,
      "epoch": 20.08,
      "learning_rate": 2.0690530303030302e-05,
      "loss": 2.7113,
      "step": 345650
    },
    {
      "activations/layer0_attention_weight_max": 15.590407371520996,
      "activations/layer0_attention_weight_min": -13.072539329528809,
      "activations/layer10_attention_weight_max": 32.585933685302734,
      "activations/layer10_attention_weight_min": -34.521827697753906,
      "activations/layer11_attention_weight_max": 32.774559020996094,
      "activations/layer11_attention_weight_min": -33.896324157714844,
      "activations/layer12_attention_weight_max": 24.509449005126953,
      "activations/layer12_attention_weight_min": -29.303470611572266,
      "activations/layer13_attention_weight_max": 34.51939010620117,
      "activations/layer13_attention_weight_min": -36.493011474609375,
      "activations/layer14_attention_weight_max": 39.68143844604492,
      "activations/layer14_attention_weight_min": -39.756805419921875,
      "activations/layer15_attention_weight_max": 35.53654098510742,
      "activations/layer15_attention_weight_min": -36.14508819580078,
      "activations/layer16_attention_weight_max": 26.9530086517334,
      "activations/layer16_attention_weight_min": -28.655351638793945,
      "activations/layer17_attention_weight_max": 26.689395904541016,
      "activations/layer17_attention_weight_min": -26.477357864379883,
      "activations/layer18_attention_weight_max": 30.96795654296875,
      "activations/layer18_attention_weight_min": -26.487165451049805,
      "activations/layer19_attention_weight_max": 31.10853385925293,
      "activations/layer19_attention_weight_min": -30.994230270385742,
      "activations/layer1_attention_weight_max": 15.179129600524902,
      "activations/layer1_attention_weight_min": -13.524786949157715,
      "activations/layer20_attention_weight_max": 27.51292610168457,
      "activations/layer20_attention_weight_min": -25.072952270507812,
      "activations/layer21_attention_weight_max": 23.492820739746094,
      "activations/layer21_attention_weight_min": -24.257272720336914,
      "activations/layer22_attention_weight_max": 33.407466888427734,
      "activations/layer22_attention_weight_min": -31.452369689941406,
      "activations/layer23_attention_weight_max": 29.153366088867188,
      "activations/layer23_attention_weight_min": -27.23373794555664,
      "activations/layer2_attention_weight_max": 32.80302047729492,
      "activations/layer2_attention_weight_min": -31.855430603027344,
      "activations/layer3_attention_weight_max": 98.0875473022461,
      "activations/layer3_attention_weight_min": -104.87523651123047,
      "activations/layer4_attention_weight_max": 103.35737609863281,
      "activations/layer4_attention_weight_min": -112.0671157836914,
      "activations/layer5_attention_weight_max": 49.243988037109375,
      "activations/layer5_attention_weight_min": -63.315242767333984,
      "activations/layer6_attention_weight_max": 42.577362060546875,
      "activations/layer6_attention_weight_min": -47.584781646728516,
      "activations/layer7_attention_weight_max": 92.52757263183594,
      "activations/layer7_attention_weight_min": -102.35997772216797,
      "activations/layer8_attention_weight_max": 40.53725051879883,
      "activations/layer8_attention_weight_min": -43.44485092163086,
      "activations/layer9_attention_weight_max": 30.124792098999023,
      "activations/layer9_attention_weight_min": -34.3358154296875,
      "epoch": 20.09,
      "learning_rate": 2.0671590909090907e-05,
      "loss": 2.719,
      "step": 345700
    },
    {
      "activations/layer0_attention_weight_max": 16.526952743530273,
      "activations/layer0_attention_weight_min": -13.503571510314941,
      "activations/layer10_attention_weight_max": 35.7279167175293,
      "activations/layer10_attention_weight_min": -35.83101272583008,
      "activations/layer11_attention_weight_max": 33.59016418457031,
      "activations/layer11_attention_weight_min": -34.7081298828125,
      "activations/layer12_attention_weight_max": 26.56131935119629,
      "activations/layer12_attention_weight_min": -27.268346786499023,
      "activations/layer13_attention_weight_max": 39.066009521484375,
      "activations/layer13_attention_weight_min": -37.79765701293945,
      "activations/layer14_attention_weight_max": 41.47536849975586,
      "activations/layer14_attention_weight_min": -40.15854263305664,
      "activations/layer15_attention_weight_max": 35.81470489501953,
      "activations/layer15_attention_weight_min": -39.118980407714844,
      "activations/layer16_attention_weight_max": 26.621904373168945,
      "activations/layer16_attention_weight_min": -29.5093936920166,
      "activations/layer17_attention_weight_max": 28.713239669799805,
      "activations/layer17_attention_weight_min": -26.870412826538086,
      "activations/layer18_attention_weight_max": 30.572954177856445,
      "activations/layer18_attention_weight_min": -27.976242065429688,
      "activations/layer19_attention_weight_max": 30.61813735961914,
      "activations/layer19_attention_weight_min": -32.69500732421875,
      "activations/layer1_attention_weight_max": 15.94680118560791,
      "activations/layer1_attention_weight_min": -14.409265518188477,
      "activations/layer20_attention_weight_max": 25.93816375732422,
      "activations/layer20_attention_weight_min": -26.728443145751953,
      "activations/layer21_attention_weight_max": 26.566099166870117,
      "activations/layer21_attention_weight_min": -26.37492561340332,
      "activations/layer22_attention_weight_max": 36.34458541870117,
      "activations/layer22_attention_weight_min": -33.82354736328125,
      "activations/layer23_attention_weight_max": 27.90970230102539,
      "activations/layer23_attention_weight_min": -25.707374572753906,
      "activations/layer2_attention_weight_max": 31.44449234008789,
      "activations/layer2_attention_weight_min": -32.04020690917969,
      "activations/layer3_attention_weight_max": 99.43366241455078,
      "activations/layer3_attention_weight_min": -104.07007598876953,
      "activations/layer4_attention_weight_max": 105.6782455444336,
      "activations/layer4_attention_weight_min": -114.55567932128906,
      "activations/layer5_attention_weight_max": 48.666015625,
      "activations/layer5_attention_weight_min": -59.75226593017578,
      "activations/layer6_attention_weight_max": 44.68259811401367,
      "activations/layer6_attention_weight_min": -49.34351348876953,
      "activations/layer7_attention_weight_max": 94.64044189453125,
      "activations/layer7_attention_weight_min": -105.97994995117188,
      "activations/layer8_attention_weight_max": 42.17619323730469,
      "activations/layer8_attention_weight_min": -46.3148193359375,
      "activations/layer9_attention_weight_max": 33.10853576660156,
      "activations/layer9_attention_weight_min": -36.370906829833984,
      "epoch": 20.09,
      "learning_rate": 2.0652651515151512e-05,
      "loss": 2.7129,
      "step": 345750
    },
    {
      "activations/layer0_attention_weight_max": 16.257970809936523,
      "activations/layer0_attention_weight_min": -13.271868705749512,
      "activations/layer10_attention_weight_max": 33.58344268798828,
      "activations/layer10_attention_weight_min": -33.73208236694336,
      "activations/layer11_attention_weight_max": 34.53803253173828,
      "activations/layer11_attention_weight_min": -33.75279235839844,
      "activations/layer12_attention_weight_max": 24.705406188964844,
      "activations/layer12_attention_weight_min": -26.0151424407959,
      "activations/layer13_attention_weight_max": 36.83732604980469,
      "activations/layer13_attention_weight_min": -34.28628921508789,
      "activations/layer14_attention_weight_max": 42.401519775390625,
      "activations/layer14_attention_weight_min": -36.57637023925781,
      "activations/layer15_attention_weight_max": 38.72164535522461,
      "activations/layer15_attention_weight_min": -34.85150909423828,
      "activations/layer16_attention_weight_max": 27.437795639038086,
      "activations/layer16_attention_weight_min": -28.459035873413086,
      "activations/layer17_attention_weight_max": 30.08753204345703,
      "activations/layer17_attention_weight_min": -25.883041381835938,
      "activations/layer18_attention_weight_max": 30.673114776611328,
      "activations/layer18_attention_weight_min": -26.89438819885254,
      "activations/layer19_attention_weight_max": 33.19417953491211,
      "activations/layer19_attention_weight_min": -31.780170440673828,
      "activations/layer1_attention_weight_max": 16.695878982543945,
      "activations/layer1_attention_weight_min": -14.921247482299805,
      "activations/layer20_attention_weight_max": 29.477388381958008,
      "activations/layer20_attention_weight_min": -26.03246307373047,
      "activations/layer21_attention_weight_max": 29.920629501342773,
      "activations/layer21_attention_weight_min": -27.770904541015625,
      "activations/layer22_attention_weight_max": 38.11233901977539,
      "activations/layer22_attention_weight_min": -35.351829528808594,
      "activations/layer23_attention_weight_max": 29.425092697143555,
      "activations/layer23_attention_weight_min": -27.16340446472168,
      "activations/layer2_attention_weight_max": 32.639286041259766,
      "activations/layer2_attention_weight_min": -32.77885055541992,
      "activations/layer3_attention_weight_max": 102.6123275756836,
      "activations/layer3_attention_weight_min": -105.15767669677734,
      "activations/layer4_attention_weight_max": 109.54772186279297,
      "activations/layer4_attention_weight_min": -110.59185791015625,
      "activations/layer5_attention_weight_max": 49.999298095703125,
      "activations/layer5_attention_weight_min": -62.6165885925293,
      "activations/layer6_attention_weight_max": 44.719810485839844,
      "activations/layer6_attention_weight_min": -47.779319763183594,
      "activations/layer7_attention_weight_max": 95.7562484741211,
      "activations/layer7_attention_weight_min": -99.24759674072266,
      "activations/layer8_attention_weight_max": 40.49807357788086,
      "activations/layer8_attention_weight_min": -43.9088134765625,
      "activations/layer9_attention_weight_max": 30.345489501953125,
      "activations/layer9_attention_weight_min": -34.64209747314453,
      "epoch": 20.09,
      "learning_rate": 2.0633712121212117e-05,
      "loss": 2.7014,
      "step": 345800
    },
    {
      "activations/layer0_attention_weight_max": 15.840879440307617,
      "activations/layer0_attention_weight_min": -12.532651901245117,
      "activations/layer10_attention_weight_max": 35.717323303222656,
      "activations/layer10_attention_weight_min": -35.441558837890625,
      "activations/layer11_attention_weight_max": 36.04646301269531,
      "activations/layer11_attention_weight_min": -36.37565612792969,
      "activations/layer12_attention_weight_max": 27.175512313842773,
      "activations/layer12_attention_weight_min": -26.349180221557617,
      "activations/layer13_attention_weight_max": 41.99981689453125,
      "activations/layer13_attention_weight_min": -35.90277099609375,
      "activations/layer14_attention_weight_max": 44.920318603515625,
      "activations/layer14_attention_weight_min": -38.50904083251953,
      "activations/layer15_attention_weight_max": 40.17901611328125,
      "activations/layer15_attention_weight_min": -36.62642288208008,
      "activations/layer16_attention_weight_max": 29.757999420166016,
      "activations/layer16_attention_weight_min": -28.238788604736328,
      "activations/layer17_attention_weight_max": 29.58854866027832,
      "activations/layer17_attention_weight_min": -25.311243057250977,
      "activations/layer18_attention_weight_max": 34.288570404052734,
      "activations/layer18_attention_weight_min": -27.001718521118164,
      "activations/layer19_attention_weight_max": 33.706424713134766,
      "activations/layer19_attention_weight_min": -31.576007843017578,
      "activations/layer1_attention_weight_max": 16.178823471069336,
      "activations/layer1_attention_weight_min": -13.964958190917969,
      "activations/layer20_attention_weight_max": 28.78417205810547,
      "activations/layer20_attention_weight_min": -25.85006332397461,
      "activations/layer21_attention_weight_max": 29.690378189086914,
      "activations/layer21_attention_weight_min": -24.20204734802246,
      "activations/layer22_attention_weight_max": 44.40107727050781,
      "activations/layer22_attention_weight_min": -31.465805053710938,
      "activations/layer23_attention_weight_max": 34.185760498046875,
      "activations/layer23_attention_weight_min": -25.164653778076172,
      "activations/layer2_attention_weight_max": 31.688949584960938,
      "activations/layer2_attention_weight_min": -31.707136154174805,
      "activations/layer3_attention_weight_max": 101.28324890136719,
      "activations/layer3_attention_weight_min": -100.99093627929688,
      "activations/layer4_attention_weight_max": 106.55741119384766,
      "activations/layer4_attention_weight_min": -108.4566421508789,
      "activations/layer5_attention_weight_max": 48.78155517578125,
      "activations/layer5_attention_weight_min": -63.44153594970703,
      "activations/layer6_attention_weight_max": 44.3641242980957,
      "activations/layer6_attention_weight_min": -47.5925178527832,
      "activations/layer7_attention_weight_max": 91.03496551513672,
      "activations/layer7_attention_weight_min": -99.48429107666016,
      "activations/layer8_attention_weight_max": 41.10340881347656,
      "activations/layer8_attention_weight_min": -44.080142974853516,
      "activations/layer9_attention_weight_max": 32.32883834838867,
      "activations/layer9_attention_weight_min": -34.498924255371094,
      "epoch": 20.1,
      "learning_rate": 2.0614772727272725e-05,
      "loss": 2.718,
      "step": 345850
    },
    {
      "activations/layer0_attention_weight_max": 15.682406425476074,
      "activations/layer0_attention_weight_min": -13.191516876220703,
      "activations/layer10_attention_weight_max": 33.231689453125,
      "activations/layer10_attention_weight_min": -34.99839782714844,
      "activations/layer11_attention_weight_max": 35.68138122558594,
      "activations/layer11_attention_weight_min": -34.9395751953125,
      "activations/layer12_attention_weight_max": 29.49414825439453,
      "activations/layer12_attention_weight_min": -28.192081451416016,
      "activations/layer13_attention_weight_max": 45.695899963378906,
      "activations/layer13_attention_weight_min": -37.0564079284668,
      "activations/layer14_attention_weight_max": 45.416961669921875,
      "activations/layer14_attention_weight_min": -39.58002471923828,
      "activations/layer15_attention_weight_max": 42.85340881347656,
      "activations/layer15_attention_weight_min": -36.681461334228516,
      "activations/layer16_attention_weight_max": 30.945268630981445,
      "activations/layer16_attention_weight_min": -29.925050735473633,
      "activations/layer17_attention_weight_max": 33.519447326660156,
      "activations/layer17_attention_weight_min": -26.033933639526367,
      "activations/layer18_attention_weight_max": 34.83629608154297,
      "activations/layer18_attention_weight_min": -27.087175369262695,
      "activations/layer19_attention_weight_max": 35.995399475097656,
      "activations/layer19_attention_weight_min": -32.8068962097168,
      "activations/layer1_attention_weight_max": 14.826173782348633,
      "activations/layer1_attention_weight_min": -14.045225143432617,
      "activations/layer20_attention_weight_max": 32.80217361450195,
      "activations/layer20_attention_weight_min": -27.755088806152344,
      "activations/layer21_attention_weight_max": 32.94021224975586,
      "activations/layer21_attention_weight_min": -25.544448852539062,
      "activations/layer22_attention_weight_max": 45.35484313964844,
      "activations/layer22_attention_weight_min": -32.89871597290039,
      "activations/layer23_attention_weight_max": 34.74382019042969,
      "activations/layer23_attention_weight_min": -24.435943603515625,
      "activations/layer2_attention_weight_max": 31.749523162841797,
      "activations/layer2_attention_weight_min": -30.883136749267578,
      "activations/layer3_attention_weight_max": 100.13716888427734,
      "activations/layer3_attention_weight_min": -100.5783462524414,
      "activations/layer4_attention_weight_max": 105.4835433959961,
      "activations/layer4_attention_weight_min": -107.49584197998047,
      "activations/layer5_attention_weight_max": 52.09416961669922,
      "activations/layer5_attention_weight_min": -59.553558349609375,
      "activations/layer6_attention_weight_max": 45.71908950805664,
      "activations/layer6_attention_weight_min": -48.117210388183594,
      "activations/layer7_attention_weight_max": 95.81621551513672,
      "activations/layer7_attention_weight_min": -96.9945297241211,
      "activations/layer8_attention_weight_max": 43.2702522277832,
      "activations/layer8_attention_weight_min": -45.50236892700195,
      "activations/layer9_attention_weight_max": 33.415889739990234,
      "activations/layer9_attention_weight_min": -33.49769973754883,
      "epoch": 20.1,
      "learning_rate": 2.059583333333333e-05,
      "loss": 2.719,
      "step": 345900
    },
    {
      "activations/layer0_attention_weight_max": 17.11281394958496,
      "activations/layer0_attention_weight_min": -13.104968070983887,
      "activations/layer10_attention_weight_max": 35.391666412353516,
      "activations/layer10_attention_weight_min": -36.31219482421875,
      "activations/layer11_attention_weight_max": 35.68180847167969,
      "activations/layer11_attention_weight_min": -35.66813659667969,
      "activations/layer12_attention_weight_max": 29.687986373901367,
      "activations/layer12_attention_weight_min": -26.978641510009766,
      "activations/layer13_attention_weight_max": 42.440773010253906,
      "activations/layer13_attention_weight_min": -36.891963958740234,
      "activations/layer14_attention_weight_max": 43.0803337097168,
      "activations/layer14_attention_weight_min": -38.82921600341797,
      "activations/layer15_attention_weight_max": 42.724647521972656,
      "activations/layer15_attention_weight_min": -38.07365798950195,
      "activations/layer16_attention_weight_max": 31.8701171875,
      "activations/layer16_attention_weight_min": -29.81830596923828,
      "activations/layer17_attention_weight_max": 32.08332443237305,
      "activations/layer17_attention_weight_min": -26.552837371826172,
      "activations/layer18_attention_weight_max": 36.27963638305664,
      "activations/layer18_attention_weight_min": -26.933414459228516,
      "activations/layer19_attention_weight_max": 36.49259567260742,
      "activations/layer19_attention_weight_min": -31.302335739135742,
      "activations/layer1_attention_weight_max": 16.532575607299805,
      "activations/layer1_attention_weight_min": -16.28117561340332,
      "activations/layer20_attention_weight_max": 30.517253875732422,
      "activations/layer20_attention_weight_min": -25.80443572998047,
      "activations/layer21_attention_weight_max": 29.10466194152832,
      "activations/layer21_attention_weight_min": -26.16025733947754,
      "activations/layer22_attention_weight_max": 46.55522537231445,
      "activations/layer22_attention_weight_min": -31.84390640258789,
      "activations/layer23_attention_weight_max": 35.121368408203125,
      "activations/layer23_attention_weight_min": -25.639122009277344,
      "activations/layer2_attention_weight_max": 32.28630828857422,
      "activations/layer2_attention_weight_min": -32.06645965576172,
      "activations/layer3_attention_weight_max": 102.01642608642578,
      "activations/layer3_attention_weight_min": -101.52098846435547,
      "activations/layer4_attention_weight_max": 111.22545623779297,
      "activations/layer4_attention_weight_min": -116.10079193115234,
      "activations/layer5_attention_weight_max": 52.16126251220703,
      "activations/layer5_attention_weight_min": -67.10309600830078,
      "activations/layer6_attention_weight_max": 45.487403869628906,
      "activations/layer6_attention_weight_min": -47.51618957519531,
      "activations/layer7_attention_weight_max": 98.29045867919922,
      "activations/layer7_attention_weight_min": -102.86965942382812,
      "activations/layer8_attention_weight_max": 42.92274856567383,
      "activations/layer8_attention_weight_min": -45.458526611328125,
      "activations/layer9_attention_weight_max": 33.59283447265625,
      "activations/layer9_attention_weight_min": -36.36083221435547,
      "epoch": 20.1,
      "learning_rate": 2.057689393939394e-05,
      "loss": 2.7155,
      "step": 345950
    },
    {
      "activations/layer0_attention_weight_max": 15.860129356384277,
      "activations/layer0_attention_weight_min": -13.946224212646484,
      "activations/layer10_attention_weight_max": 35.80531311035156,
      "activations/layer10_attention_weight_min": -36.93555450439453,
      "activations/layer11_attention_weight_max": 35.33633041381836,
      "activations/layer11_attention_weight_min": -35.022605895996094,
      "activations/layer12_attention_weight_max": 26.464466094970703,
      "activations/layer12_attention_weight_min": -27.283456802368164,
      "activations/layer13_attention_weight_max": 37.671382904052734,
      "activations/layer13_attention_weight_min": -37.23455810546875,
      "activations/layer14_attention_weight_max": 41.1038932800293,
      "activations/layer14_attention_weight_min": -37.113304138183594,
      "activations/layer15_attention_weight_max": 37.30514907836914,
      "activations/layer15_attention_weight_min": -33.80141830444336,
      "activations/layer16_attention_weight_max": 28.93633460998535,
      "activations/layer16_attention_weight_min": -28.20211410522461,
      "activations/layer17_attention_weight_max": 26.937829971313477,
      "activations/layer17_attention_weight_min": -26.206750869750977,
      "activations/layer18_attention_weight_max": 30.596477508544922,
      "activations/layer18_attention_weight_min": -27.539241790771484,
      "activations/layer19_attention_weight_max": 33.40812683105469,
      "activations/layer19_attention_weight_min": -32.37312698364258,
      "activations/layer1_attention_weight_max": 15.334441184997559,
      "activations/layer1_attention_weight_min": -14.623514175415039,
      "activations/layer20_attention_weight_max": 29.341567993164062,
      "activations/layer20_attention_weight_min": -25.914945602416992,
      "activations/layer21_attention_weight_max": 28.47307586669922,
      "activations/layer21_attention_weight_min": -24.94818115234375,
      "activations/layer22_attention_weight_max": 41.81180953979492,
      "activations/layer22_attention_weight_min": -31.655908584594727,
      "activations/layer23_attention_weight_max": 31.936370849609375,
      "activations/layer23_attention_weight_min": -27.27886199951172,
      "activations/layer2_attention_weight_max": 32.23936462402344,
      "activations/layer2_attention_weight_min": -32.76750946044922,
      "activations/layer3_attention_weight_max": 99.21218872070312,
      "activations/layer3_attention_weight_min": -100.04689025878906,
      "activations/layer4_attention_weight_max": 109.80687713623047,
      "activations/layer4_attention_weight_min": -115.09053039550781,
      "activations/layer5_attention_weight_max": 50.14545440673828,
      "activations/layer5_attention_weight_min": -61.954925537109375,
      "activations/layer6_attention_weight_max": 45.24721145629883,
      "activations/layer6_attention_weight_min": -48.33466720581055,
      "activations/layer7_attention_weight_max": 95.4610595703125,
      "activations/layer7_attention_weight_min": -96.05906677246094,
      "activations/layer8_attention_weight_max": 42.58212661743164,
      "activations/layer8_attention_weight_min": -44.614967346191406,
      "activations/layer9_attention_weight_max": 31.674833297729492,
      "activations/layer9_attention_weight_min": -36.12376403808594,
      "epoch": 20.1,
      "learning_rate": 2.0557954545454547e-05,
      "loss": 2.6927,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.495,
      "eval_samples_per_second": 505.475,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.495,
      "eval_openwebtext_samples_per_second": 505.475,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9527,
      "eval_wikitext_samples_per_second": 233.527,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_lambada_loss": 2.33203125,
      "eval_lambada_ppl": 10.298839819419056,
      "eval_lambada_runtime": 9.5841,
      "eval_lambada_samples_per_second": 508.027,
      "step": 346000
    },
    {
      "activations/layer0_attention_weight_max": 16.07896614074707,
      "activations/layer0_attention_weight_min": -12.6450777053833,
      "activations/layer10_attention_weight_max": 36.07130813598633,
      "activations/layer10_attention_weight_min": -37.4682502746582,
      "activations/layer11_attention_weight_max": 32.60551452636719,
      "activations/layer11_attention_weight_min": -35.9581413269043,
      "activations/layer12_attention_weight_max": 26.476993560791016,
      "activations/layer12_attention_weight_min": -26.179121017456055,
      "activations/layer13_attention_weight_max": 39.5933952331543,
      "activations/layer13_attention_weight_min": -35.337623596191406,
      "activations/layer14_attention_weight_max": 41.54243469238281,
      "activations/layer14_attention_weight_min": -37.721561431884766,
      "activations/layer15_attention_weight_max": 38.68465805053711,
      "activations/layer15_attention_weight_min": -38.591796875,
      "activations/layer16_attention_weight_max": 27.587644577026367,
      "activations/layer16_attention_weight_min": -28.480775833129883,
      "activations/layer17_attention_weight_max": 27.5919189453125,
      "activations/layer17_attention_weight_min": -26.579242706298828,
      "activations/layer18_attention_weight_max": 30.034677505493164,
      "activations/layer18_attention_weight_min": -26.16416358947754,
      "activations/layer19_attention_weight_max": 31.53043556213379,
      "activations/layer19_attention_weight_min": -30.61468505859375,
      "activations/layer1_attention_weight_max": 15.696513175964355,
      "activations/layer1_attention_weight_min": -15.173257827758789,
      "activations/layer20_attention_weight_max": 27.025318145751953,
      "activations/layer20_attention_weight_min": -24.525041580200195,
      "activations/layer21_attention_weight_max": 28.778154373168945,
      "activations/layer21_attention_weight_min": -24.36417007446289,
      "activations/layer22_attention_weight_max": 39.371978759765625,
      "activations/layer22_attention_weight_min": -30.16087532043457,
      "activations/layer23_attention_weight_max": 29.870880126953125,
      "activations/layer23_attention_weight_min": -26.400625228881836,
      "activations/layer2_attention_weight_max": 32.069068908691406,
      "activations/layer2_attention_weight_min": -31.143077850341797,
      "activations/layer3_attention_weight_max": 101.7610092163086,
      "activations/layer3_attention_weight_min": -104.85352325439453,
      "activations/layer4_attention_weight_max": 104.41301727294922,
      "activations/layer4_attention_weight_min": -110.83758544921875,
      "activations/layer5_attention_weight_max": 52.18354034423828,
      "activations/layer5_attention_weight_min": -60.84355926513672,
      "activations/layer6_attention_weight_max": 44.82761001586914,
      "activations/layer6_attention_weight_min": -49.443294525146484,
      "activations/layer7_attention_weight_max": 96.28018188476562,
      "activations/layer7_attention_weight_min": -104.41973114013672,
      "activations/layer8_attention_weight_max": 41.64964294433594,
      "activations/layer8_attention_weight_min": -45.32148361206055,
      "activations/layer9_attention_weight_max": 34.509151458740234,
      "activations/layer9_attention_weight_min": -37.85346603393555,
      "epoch": 20.11,
      "learning_rate": 2.0539015151515148e-05,
      "loss": 2.7203,
      "step": 346050
    },
    {
      "activations/layer0_attention_weight_max": 16.282155990600586,
      "activations/layer0_attention_weight_min": -13.077726364135742,
      "activations/layer10_attention_weight_max": 35.3089485168457,
      "activations/layer10_attention_weight_min": -35.545143127441406,
      "activations/layer11_attention_weight_max": 34.05232620239258,
      "activations/layer11_attention_weight_min": -35.146427154541016,
      "activations/layer12_attention_weight_max": 26.446414947509766,
      "activations/layer12_attention_weight_min": -26.541187286376953,
      "activations/layer13_attention_weight_max": 40.374671936035156,
      "activations/layer13_attention_weight_min": -35.7467155456543,
      "activations/layer14_attention_weight_max": 48.3462028503418,
      "activations/layer14_attention_weight_min": -38.36382293701172,
      "activations/layer15_attention_weight_max": 41.30146789550781,
      "activations/layer15_attention_weight_min": -35.87687301635742,
      "activations/layer16_attention_weight_max": 30.59025001525879,
      "activations/layer16_attention_weight_min": -26.710845947265625,
      "activations/layer17_attention_weight_max": 30.114282608032227,
      "activations/layer17_attention_weight_min": -24.979480743408203,
      "activations/layer18_attention_weight_max": 33.05870056152344,
      "activations/layer18_attention_weight_min": -24.461950302124023,
      "activations/layer19_attention_weight_max": 35.91634750366211,
      "activations/layer19_attention_weight_min": -31.65394401550293,
      "activations/layer1_attention_weight_max": 16.5495548248291,
      "activations/layer1_attention_weight_min": -14.464218139648438,
      "activations/layer20_attention_weight_max": 30.940916061401367,
      "activations/layer20_attention_weight_min": -24.653303146362305,
      "activations/layer21_attention_weight_max": 31.368637084960938,
      "activations/layer21_attention_weight_min": -23.76923179626465,
      "activations/layer22_attention_weight_max": 39.06980514526367,
      "activations/layer22_attention_weight_min": -30.330501556396484,
      "activations/layer23_attention_weight_max": 30.844558715820312,
      "activations/layer23_attention_weight_min": -24.585384368896484,
      "activations/layer2_attention_weight_max": 31.789609909057617,
      "activations/layer2_attention_weight_min": -30.83076286315918,
      "activations/layer3_attention_weight_max": 98.64917755126953,
      "activations/layer3_attention_weight_min": -98.0472183227539,
      "activations/layer4_attention_weight_max": 102.33848571777344,
      "activations/layer4_attention_weight_min": -103.1225357055664,
      "activations/layer5_attention_weight_max": 50.22771453857422,
      "activations/layer5_attention_weight_min": -60.03631591796875,
      "activations/layer6_attention_weight_max": 44.11955642700195,
      "activations/layer6_attention_weight_min": -47.06934356689453,
      "activations/layer7_attention_weight_max": 96.1161880493164,
      "activations/layer7_attention_weight_min": -99.50252532958984,
      "activations/layer8_attention_weight_max": 42.812835693359375,
      "activations/layer8_attention_weight_min": -45.00760269165039,
      "activations/layer9_attention_weight_max": 32.13622283935547,
      "activations/layer9_attention_weight_min": -34.34218978881836,
      "epoch": 20.11,
      "learning_rate": 2.0520075757575757e-05,
      "loss": 2.7147,
      "step": 346100
    },
    {
      "activations/layer0_attention_weight_max": 16.54296875,
      "activations/layer0_attention_weight_min": -13.380931854248047,
      "activations/layer10_attention_weight_max": 33.901458740234375,
      "activations/layer10_attention_weight_min": -34.559600830078125,
      "activations/layer11_attention_weight_max": 33.414085388183594,
      "activations/layer11_attention_weight_min": -36.53404235839844,
      "activations/layer12_attention_weight_max": 26.67169761657715,
      "activations/layer12_attention_weight_min": -26.868410110473633,
      "activations/layer13_attention_weight_max": 37.48789978027344,
      "activations/layer13_attention_weight_min": -36.70424270629883,
      "activations/layer14_attention_weight_max": 41.22639083862305,
      "activations/layer14_attention_weight_min": -38.16728210449219,
      "activations/layer15_attention_weight_max": 37.9127082824707,
      "activations/layer15_attention_weight_min": -35.55289077758789,
      "activations/layer16_attention_weight_max": 29.590442657470703,
      "activations/layer16_attention_weight_min": -29.121042251586914,
      "activations/layer17_attention_weight_max": 29.95208740234375,
      "activations/layer17_attention_weight_min": -25.486961364746094,
      "activations/layer18_attention_weight_max": 33.56109619140625,
      "activations/layer18_attention_weight_min": -27.452241897583008,
      "activations/layer19_attention_weight_max": 36.71744918823242,
      "activations/layer19_attention_weight_min": -33.65256881713867,
      "activations/layer1_attention_weight_max": 15.54465389251709,
      "activations/layer1_attention_weight_min": -15.217998504638672,
      "activations/layer20_attention_weight_max": 32.66697692871094,
      "activations/layer20_attention_weight_min": -26.04701805114746,
      "activations/layer21_attention_weight_max": 32.79474639892578,
      "activations/layer21_attention_weight_min": -26.203338623046875,
      "activations/layer22_attention_weight_max": 40.61804962158203,
      "activations/layer22_attention_weight_min": -33.03516387939453,
      "activations/layer23_attention_weight_max": 34.008644104003906,
      "activations/layer23_attention_weight_min": -26.361812591552734,
      "activations/layer2_attention_weight_max": 32.74307632446289,
      "activations/layer2_attention_weight_min": -30.88671112060547,
      "activations/layer3_attention_weight_max": 105.64873504638672,
      "activations/layer3_attention_weight_min": -101.5921859741211,
      "activations/layer4_attention_weight_max": 109.275146484375,
      "activations/layer4_attention_weight_min": -107.89253234863281,
      "activations/layer5_attention_weight_max": 51.407081604003906,
      "activations/layer5_attention_weight_min": -61.98516082763672,
      "activations/layer6_attention_weight_max": 45.515560150146484,
      "activations/layer6_attention_weight_min": -45.825157165527344,
      "activations/layer7_attention_weight_max": 96.74171447753906,
      "activations/layer7_attention_weight_min": -94.02702331542969,
      "activations/layer8_attention_weight_max": 42.60139465332031,
      "activations/layer8_attention_weight_min": -43.068241119384766,
      "activations/layer9_attention_weight_max": 32.51261520385742,
      "activations/layer9_attention_weight_min": -35.00013732910156,
      "epoch": 20.11,
      "learning_rate": 2.050113636363636e-05,
      "loss": 2.7094,
      "step": 346150
    },
    {
      "activations/layer0_attention_weight_max": 16.58074188232422,
      "activations/layer0_attention_weight_min": -12.333979606628418,
      "activations/layer10_attention_weight_max": 34.39885711669922,
      "activations/layer10_attention_weight_min": -35.07635498046875,
      "activations/layer11_attention_weight_max": 36.39628982543945,
      "activations/layer11_attention_weight_min": -34.892066955566406,
      "activations/layer12_attention_weight_max": 28.3837947845459,
      "activations/layer12_attention_weight_min": -27.29935073852539,
      "activations/layer13_attention_weight_max": 41.566097259521484,
      "activations/layer13_attention_weight_min": -37.06039047241211,
      "activations/layer14_attention_weight_max": 42.04362487792969,
      "activations/layer14_attention_weight_min": -41.001487731933594,
      "activations/layer15_attention_weight_max": 38.926631927490234,
      "activations/layer15_attention_weight_min": -37.156280517578125,
      "activations/layer16_attention_weight_max": 29.042522430419922,
      "activations/layer16_attention_weight_min": -30.109766006469727,
      "activations/layer17_attention_weight_max": 28.836402893066406,
      "activations/layer17_attention_weight_min": -25.84671974182129,
      "activations/layer18_attention_weight_max": 30.469173431396484,
      "activations/layer18_attention_weight_min": -24.53738021850586,
      "activations/layer19_attention_weight_max": 31.80838394165039,
      "activations/layer19_attention_weight_min": -30.226821899414062,
      "activations/layer1_attention_weight_max": 15.557303428649902,
      "activations/layer1_attention_weight_min": -15.702027320861816,
      "activations/layer20_attention_weight_max": 27.889854431152344,
      "activations/layer20_attention_weight_min": -24.816865921020508,
      "activations/layer21_attention_weight_max": 27.013126373291016,
      "activations/layer21_attention_weight_min": -25.94347381591797,
      "activations/layer22_attention_weight_max": 39.025474548339844,
      "activations/layer22_attention_weight_min": -31.019224166870117,
      "activations/layer23_attention_weight_max": 32.83993148803711,
      "activations/layer23_attention_weight_min": -26.688369750976562,
      "activations/layer2_attention_weight_max": 32.835914611816406,
      "activations/layer2_attention_weight_min": -31.79401397705078,
      "activations/layer3_attention_weight_max": 102.86968994140625,
      "activations/layer3_attention_weight_min": -103.72163391113281,
      "activations/layer4_attention_weight_max": 105.54845428466797,
      "activations/layer4_attention_weight_min": -109.44001770019531,
      "activations/layer5_attention_weight_max": 50.02619552612305,
      "activations/layer5_attention_weight_min": -61.97605514526367,
      "activations/layer6_attention_weight_max": 45.866416931152344,
      "activations/layer6_attention_weight_min": -47.9694938659668,
      "activations/layer7_attention_weight_max": 95.0540542602539,
      "activations/layer7_attention_weight_min": -103.86872863769531,
      "activations/layer8_attention_weight_max": 43.929901123046875,
      "activations/layer8_attention_weight_min": -47.53390884399414,
      "activations/layer9_attention_weight_max": 33.45844268798828,
      "activations/layer9_attention_weight_min": -36.77304458618164,
      "epoch": 20.12,
      "learning_rate": 2.048219696969697e-05,
      "loss": 2.7161,
      "step": 346200
    },
    {
      "activations/layer0_attention_weight_max": 16.120494842529297,
      "activations/layer0_attention_weight_min": -12.626983642578125,
      "activations/layer10_attention_weight_max": 34.76670455932617,
      "activations/layer10_attention_weight_min": -36.675743103027344,
      "activations/layer11_attention_weight_max": 32.49209976196289,
      "activations/layer11_attention_weight_min": -34.415489196777344,
      "activations/layer12_attention_weight_max": 24.98845100402832,
      "activations/layer12_attention_weight_min": -26.542346954345703,
      "activations/layer13_attention_weight_max": 36.73759078979492,
      "activations/layer13_attention_weight_min": -36.22660446166992,
      "activations/layer14_attention_weight_max": 37.15830612182617,
      "activations/layer14_attention_weight_min": -37.85651397705078,
      "activations/layer15_attention_weight_max": 34.404518127441406,
      "activations/layer15_attention_weight_min": -36.379417419433594,
      "activations/layer16_attention_weight_max": 28.25489044189453,
      "activations/layer16_attention_weight_min": -27.056936264038086,
      "activations/layer17_attention_weight_max": 27.970129013061523,
      "activations/layer17_attention_weight_min": -25.74763298034668,
      "activations/layer18_attention_weight_max": 31.696931838989258,
      "activations/layer18_attention_weight_min": -28.969711303710938,
      "activations/layer19_attention_weight_max": 31.590763092041016,
      "activations/layer19_attention_weight_min": -30.529956817626953,
      "activations/layer1_attention_weight_max": 15.933008193969727,
      "activations/layer1_attention_weight_min": -14.84615421295166,
      "activations/layer20_attention_weight_max": 25.314924240112305,
      "activations/layer20_attention_weight_min": -25.444499969482422,
      "activations/layer21_attention_weight_max": 27.841581344604492,
      "activations/layer21_attention_weight_min": -25.107694625854492,
      "activations/layer22_attention_weight_max": 38.94940948486328,
      "activations/layer22_attention_weight_min": -33.91710662841797,
      "activations/layer23_attention_weight_max": 30.862821578979492,
      "activations/layer23_attention_weight_min": -27.287534713745117,
      "activations/layer2_attention_weight_max": 32.59720230102539,
      "activations/layer2_attention_weight_min": -33.21681213378906,
      "activations/layer3_attention_weight_max": 101.45169067382812,
      "activations/layer3_attention_weight_min": -102.95358276367188,
      "activations/layer4_attention_weight_max": 104.64688873291016,
      "activations/layer4_attention_weight_min": -109.14839935302734,
      "activations/layer5_attention_weight_max": 49.156517028808594,
      "activations/layer5_attention_weight_min": -59.165061950683594,
      "activations/layer6_attention_weight_max": 43.64626693725586,
      "activations/layer6_attention_weight_min": -47.384193420410156,
      "activations/layer7_attention_weight_max": 91.60736083984375,
      "activations/layer7_attention_weight_min": -96.28829193115234,
      "activations/layer8_attention_weight_max": 41.84571075439453,
      "activations/layer8_attention_weight_min": -46.08189392089844,
      "activations/layer9_attention_weight_max": 32.86007308959961,
      "activations/layer9_attention_weight_min": -35.29172134399414,
      "epoch": 20.12,
      "learning_rate": 2.0463257575757575e-05,
      "loss": 2.7078,
      "step": 346250
    },
    {
      "activations/layer0_attention_weight_max": 16.74605369567871,
      "activations/layer0_attention_weight_min": -12.82420539855957,
      "activations/layer10_attention_weight_max": 38.289859771728516,
      "activations/layer10_attention_weight_min": -37.70479202270508,
      "activations/layer11_attention_weight_max": 37.124061584472656,
      "activations/layer11_attention_weight_min": -36.353485107421875,
      "activations/layer12_attention_weight_max": 26.97283363342285,
      "activations/layer12_attention_weight_min": -26.725013732910156,
      "activations/layer13_attention_weight_max": 38.62836837768555,
      "activations/layer13_attention_weight_min": -37.5858154296875,
      "activations/layer14_attention_weight_max": 41.180870056152344,
      "activations/layer14_attention_weight_min": -38.20064926147461,
      "activations/layer15_attention_weight_max": 39.21091079711914,
      "activations/layer15_attention_weight_min": -37.01130676269531,
      "activations/layer16_attention_weight_max": 29.37202262878418,
      "activations/layer16_attention_weight_min": -29.580219268798828,
      "activations/layer17_attention_weight_max": 30.288116455078125,
      "activations/layer17_attention_weight_min": -25.408349990844727,
      "activations/layer18_attention_weight_max": 30.774169921875,
      "activations/layer18_attention_weight_min": -27.163837432861328,
      "activations/layer19_attention_weight_max": 32.02920150756836,
      "activations/layer19_attention_weight_min": -31.944122314453125,
      "activations/layer1_attention_weight_max": 15.594575881958008,
      "activations/layer1_attention_weight_min": -14.356030464172363,
      "activations/layer20_attention_weight_max": 28.311767578125,
      "activations/layer20_attention_weight_min": -26.843650817871094,
      "activations/layer21_attention_weight_max": 28.863731384277344,
      "activations/layer21_attention_weight_min": -26.83402442932129,
      "activations/layer22_attention_weight_max": 38.8348274230957,
      "activations/layer22_attention_weight_min": -32.31731414794922,
      "activations/layer23_attention_weight_max": 31.951648712158203,
      "activations/layer23_attention_weight_min": -27.746198654174805,
      "activations/layer2_attention_weight_max": 33.126564025878906,
      "activations/layer2_attention_weight_min": -31.273893356323242,
      "activations/layer3_attention_weight_max": 107.32734680175781,
      "activations/layer3_attention_weight_min": -99.92184448242188,
      "activations/layer4_attention_weight_max": 110.2578353881836,
      "activations/layer4_attention_weight_min": -111.18170166015625,
      "activations/layer5_attention_weight_max": 51.80615997314453,
      "activations/layer5_attention_weight_min": -59.16849899291992,
      "activations/layer6_attention_weight_max": 44.64969253540039,
      "activations/layer6_attention_weight_min": -46.82139205932617,
      "activations/layer7_attention_weight_max": 94.54008483886719,
      "activations/layer7_attention_weight_min": -98.09262084960938,
      "activations/layer8_attention_weight_max": 44.15401077270508,
      "activations/layer8_attention_weight_min": -46.186973571777344,
      "activations/layer9_attention_weight_max": 35.90494918823242,
      "activations/layer9_attention_weight_min": -35.8364372253418,
      "epoch": 20.12,
      "learning_rate": 2.044431818181818e-05,
      "loss": 2.7145,
      "step": 346300
    },
    {
      "activations/layer0_attention_weight_max": 16.272153854370117,
      "activations/layer0_attention_weight_min": -14.216938018798828,
      "activations/layer10_attention_weight_max": 33.94681167602539,
      "activations/layer10_attention_weight_min": -35.23776626586914,
      "activations/layer11_attention_weight_max": 35.579097747802734,
      "activations/layer11_attention_weight_min": -37.24705505371094,
      "activations/layer12_attention_weight_max": 27.67998695373535,
      "activations/layer12_attention_weight_min": -27.811756134033203,
      "activations/layer13_attention_weight_max": 39.50550842285156,
      "activations/layer13_attention_weight_min": -37.571868896484375,
      "activations/layer14_attention_weight_max": 43.84021759033203,
      "activations/layer14_attention_weight_min": -38.26090621948242,
      "activations/layer15_attention_weight_max": 40.10834503173828,
      "activations/layer15_attention_weight_min": -37.361019134521484,
      "activations/layer16_attention_weight_max": 29.66148567199707,
      "activations/layer16_attention_weight_min": -27.393022537231445,
      "activations/layer17_attention_weight_max": 28.481061935424805,
      "activations/layer17_attention_weight_min": -26.25571060180664,
      "activations/layer18_attention_weight_max": 30.012081146240234,
      "activations/layer18_attention_weight_min": -25.017797470092773,
      "activations/layer19_attention_weight_max": 36.63390350341797,
      "activations/layer19_attention_weight_min": -31.127643585205078,
      "activations/layer1_attention_weight_max": 16.219825744628906,
      "activations/layer1_attention_weight_min": -15.167400360107422,
      "activations/layer20_attention_weight_max": 29.81499671936035,
      "activations/layer20_attention_weight_min": -25.136550903320312,
      "activations/layer21_attention_weight_max": 26.86356544494629,
      "activations/layer21_attention_weight_min": -23.9450740814209,
      "activations/layer22_attention_weight_max": 43.723140716552734,
      "activations/layer22_attention_weight_min": -30.08795928955078,
      "activations/layer23_attention_weight_max": 32.9825553894043,
      "activations/layer23_attention_weight_min": -26.593048095703125,
      "activations/layer2_attention_weight_max": 33.292823791503906,
      "activations/layer2_attention_weight_min": -32.1972541809082,
      "activations/layer3_attention_weight_max": 101.65933990478516,
      "activations/layer3_attention_weight_min": -102.5379638671875,
      "activations/layer4_attention_weight_max": 107.73503875732422,
      "activations/layer4_attention_weight_min": -113.907958984375,
      "activations/layer5_attention_weight_max": 50.40437316894531,
      "activations/layer5_attention_weight_min": -61.13692855834961,
      "activations/layer6_attention_weight_max": 46.300655364990234,
      "activations/layer6_attention_weight_min": -50.773780822753906,
      "activations/layer7_attention_weight_max": 93.29930877685547,
      "activations/layer7_attention_weight_min": -101.85333251953125,
      "activations/layer8_attention_weight_max": 43.44102096557617,
      "activations/layer8_attention_weight_min": -47.08403396606445,
      "activations/layer9_attention_weight_max": 30.92661476135254,
      "activations/layer9_attention_weight_min": -36.58954620361328,
      "epoch": 20.12,
      "learning_rate": 2.0425378787878784e-05,
      "loss": 2.7161,
      "step": 346350
    },
    {
      "activations/layer0_attention_weight_max": 17.236642837524414,
      "activations/layer0_attention_weight_min": -13.128296852111816,
      "activations/layer10_attention_weight_max": 41.852745056152344,
      "activations/layer10_attention_weight_min": -40.477378845214844,
      "activations/layer11_attention_weight_max": 44.12876892089844,
      "activations/layer11_attention_weight_min": -42.732215881347656,
      "activations/layer12_attention_weight_max": 29.145065307617188,
      "activations/layer12_attention_weight_min": -30.740293502807617,
      "activations/layer13_attention_weight_max": 41.37066650390625,
      "activations/layer13_attention_weight_min": -38.20160675048828,
      "activations/layer14_attention_weight_max": 47.58793258666992,
      "activations/layer14_attention_weight_min": -41.70443344116211,
      "activations/layer15_attention_weight_max": 46.321983337402344,
      "activations/layer15_attention_weight_min": -39.99686813354492,
      "activations/layer16_attention_weight_max": 35.872615814208984,
      "activations/layer16_attention_weight_min": -28.074291229248047,
      "activations/layer17_attention_weight_max": 33.089786529541016,
      "activations/layer17_attention_weight_min": -26.374170303344727,
      "activations/layer18_attention_weight_max": 38.45891189575195,
      "activations/layer18_attention_weight_min": -28.152780532836914,
      "activations/layer19_attention_weight_max": 39.350852966308594,
      "activations/layer19_attention_weight_min": -33.71403884887695,
      "activations/layer1_attention_weight_max": 16.846521377563477,
      "activations/layer1_attention_weight_min": -14.631797790527344,
      "activations/layer20_attention_weight_max": 32.52637481689453,
      "activations/layer20_attention_weight_min": -27.2458553314209,
      "activations/layer21_attention_weight_max": 33.33136749267578,
      "activations/layer21_attention_weight_min": -26.64989471435547,
      "activations/layer22_attention_weight_max": 44.69288635253906,
      "activations/layer22_attention_weight_min": -33.32152557373047,
      "activations/layer23_attention_weight_max": 38.176673889160156,
      "activations/layer23_attention_weight_min": -27.421058654785156,
      "activations/layer2_attention_weight_max": 36.81063461303711,
      "activations/layer2_attention_weight_min": -33.09333038330078,
      "activations/layer3_attention_weight_max": 104.0365219116211,
      "activations/layer3_attention_weight_min": -99.74044036865234,
      "activations/layer4_attention_weight_max": 106.0072250366211,
      "activations/layer4_attention_weight_min": -106.25765228271484,
      "activations/layer5_attention_weight_max": 50.893211364746094,
      "activations/layer5_attention_weight_min": -61.0838623046875,
      "activations/layer6_attention_weight_max": 46.764671325683594,
      "activations/layer6_attention_weight_min": -47.13649368286133,
      "activations/layer7_attention_weight_max": 104.19075775146484,
      "activations/layer7_attention_weight_min": -98.52461242675781,
      "activations/layer8_attention_weight_max": 47.61229705810547,
      "activations/layer8_attention_weight_min": -48.1756591796875,
      "activations/layer9_attention_weight_max": 36.82969284057617,
      "activations/layer9_attention_weight_min": -38.72349166870117,
      "epoch": 20.13,
      "learning_rate": 2.0406439393939393e-05,
      "loss": 2.7311,
      "step": 346400
    },
    {
      "activations/layer0_attention_weight_max": 16.476757049560547,
      "activations/layer0_attention_weight_min": -14.10273265838623,
      "activations/layer10_attention_weight_max": 34.21639633178711,
      "activations/layer10_attention_weight_min": -34.936561584472656,
      "activations/layer11_attention_weight_max": 34.44065856933594,
      "activations/layer11_attention_weight_min": -36.00907897949219,
      "activations/layer12_attention_weight_max": 26.835126876831055,
      "activations/layer12_attention_weight_min": -26.33619499206543,
      "activations/layer13_attention_weight_max": 39.882049560546875,
      "activations/layer13_attention_weight_min": -37.16595458984375,
      "activations/layer14_attention_weight_max": 41.638404846191406,
      "activations/layer14_attention_weight_min": -39.48629379272461,
      "activations/layer15_attention_weight_max": 38.372406005859375,
      "activations/layer15_attention_weight_min": -37.73720932006836,
      "activations/layer16_attention_weight_max": 30.51874542236328,
      "activations/layer16_attention_weight_min": -28.429624557495117,
      "activations/layer17_attention_weight_max": 29.34952735900879,
      "activations/layer17_attention_weight_min": -24.55827522277832,
      "activations/layer18_attention_weight_max": 33.95763397216797,
      "activations/layer18_attention_weight_min": -25.75958251953125,
      "activations/layer19_attention_weight_max": 32.5567741394043,
      "activations/layer19_attention_weight_min": -30.214502334594727,
      "activations/layer1_attention_weight_max": 15.839898109436035,
      "activations/layer1_attention_weight_min": -14.646414756774902,
      "activations/layer20_attention_weight_max": 28.83199691772461,
      "activations/layer20_attention_weight_min": -24.301176071166992,
      "activations/layer21_attention_weight_max": 26.667510986328125,
      "activations/layer21_attention_weight_min": -24.414575576782227,
      "activations/layer22_attention_weight_max": 41.84257125854492,
      "activations/layer22_attention_weight_min": -31.509674072265625,
      "activations/layer23_attention_weight_max": 32.101619720458984,
      "activations/layer23_attention_weight_min": -27.134267807006836,
      "activations/layer2_attention_weight_max": 33.07379150390625,
      "activations/layer2_attention_weight_min": -32.61674880981445,
      "activations/layer3_attention_weight_max": 105.946533203125,
      "activations/layer3_attention_weight_min": -104.49720764160156,
      "activations/layer4_attention_weight_max": 109.40399169921875,
      "activations/layer4_attention_weight_min": -108.1236801147461,
      "activations/layer5_attention_weight_max": 50.327796936035156,
      "activations/layer5_attention_weight_min": -62.09991455078125,
      "activations/layer6_attention_weight_max": 44.93498992919922,
      "activations/layer6_attention_weight_min": -46.90120315551758,
      "activations/layer7_attention_weight_max": 95.52079010009766,
      "activations/layer7_attention_weight_min": -97.84783172607422,
      "activations/layer8_attention_weight_max": 40.346134185791016,
      "activations/layer8_attention_weight_min": -44.0027961730957,
      "activations/layer9_attention_weight_max": 32.919456481933594,
      "activations/layer9_attention_weight_min": -35.419307708740234,
      "epoch": 20.13,
      "learning_rate": 2.0387499999999998e-05,
      "loss": 2.705,
      "step": 346450
    },
    {
      "activations/layer0_attention_weight_max": 16.962383270263672,
      "activations/layer0_attention_weight_min": -13.236604690551758,
      "activations/layer10_attention_weight_max": 33.757041931152344,
      "activations/layer10_attention_weight_min": -35.03583526611328,
      "activations/layer11_attention_weight_max": 33.50090408325195,
      "activations/layer11_attention_weight_min": -35.48747253417969,
      "activations/layer12_attention_weight_max": 28.403413772583008,
      "activations/layer12_attention_weight_min": -27.125131607055664,
      "activations/layer13_attention_weight_max": 40.86333465576172,
      "activations/layer13_attention_weight_min": -37.517250061035156,
      "activations/layer14_attention_weight_max": 44.20881271362305,
      "activations/layer14_attention_weight_min": -39.731815338134766,
      "activations/layer15_attention_weight_max": 38.90461349487305,
      "activations/layer15_attention_weight_min": -35.54948043823242,
      "activations/layer16_attention_weight_max": 28.945077896118164,
      "activations/layer16_attention_weight_min": -27.87649917602539,
      "activations/layer17_attention_weight_max": 28.810712814331055,
      "activations/layer17_attention_weight_min": -25.82981300354004,
      "activations/layer18_attention_weight_max": 32.87670135498047,
      "activations/layer18_attention_weight_min": -25.118818283081055,
      "activations/layer19_attention_weight_max": 33.2687873840332,
      "activations/layer19_attention_weight_min": -30.47336196899414,
      "activations/layer1_attention_weight_max": 16.329668045043945,
      "activations/layer1_attention_weight_min": -15.477927207946777,
      "activations/layer20_attention_weight_max": 28.476091384887695,
      "activations/layer20_attention_weight_min": -24.78095054626465,
      "activations/layer21_attention_weight_max": 28.578506469726562,
      "activations/layer21_attention_weight_min": -24.420713424682617,
      "activations/layer22_attention_weight_max": 42.512237548828125,
      "activations/layer22_attention_weight_min": -30.547101974487305,
      "activations/layer23_attention_weight_max": 32.282649993896484,
      "activations/layer23_attention_weight_min": -26.779258728027344,
      "activations/layer2_attention_weight_max": 31.6522216796875,
      "activations/layer2_attention_weight_min": -31.800460815429688,
      "activations/layer3_attention_weight_max": 102.59467315673828,
      "activations/layer3_attention_weight_min": -103.3820571899414,
      "activations/layer4_attention_weight_max": 104.96467590332031,
      "activations/layer4_attention_weight_min": -111.05322265625,
      "activations/layer5_attention_weight_max": 49.1336669921875,
      "activations/layer5_attention_weight_min": -62.848548889160156,
      "activations/layer6_attention_weight_max": 43.69065856933594,
      "activations/layer6_attention_weight_min": -48.11653137207031,
      "activations/layer7_attention_weight_max": 92.47450256347656,
      "activations/layer7_attention_weight_min": -99.6412353515625,
      "activations/layer8_attention_weight_max": 40.55904769897461,
      "activations/layer8_attention_weight_min": -43.707828521728516,
      "activations/layer9_attention_weight_max": 31.263952255249023,
      "activations/layer9_attention_weight_min": -33.916866302490234,
      "epoch": 20.13,
      "learning_rate": 2.0368560606060606e-05,
      "loss": 2.7212,
      "step": 346500
    },
    {
      "activations/layer0_attention_weight_max": 16.427295684814453,
      "activations/layer0_attention_weight_min": -13.738143920898438,
      "activations/layer10_attention_weight_max": 35.295745849609375,
      "activations/layer10_attention_weight_min": -34.02882385253906,
      "activations/layer11_attention_weight_max": 33.63383483886719,
      "activations/layer11_attention_weight_min": -33.92186737060547,
      "activations/layer12_attention_weight_max": 26.21746063232422,
      "activations/layer12_attention_weight_min": -25.77726173400879,
      "activations/layer13_attention_weight_max": 39.090118408203125,
      "activations/layer13_attention_weight_min": -35.50039291381836,
      "activations/layer14_attention_weight_max": 43.579200744628906,
      "activations/layer14_attention_weight_min": -35.929649353027344,
      "activations/layer15_attention_weight_max": 38.12125778198242,
      "activations/layer15_attention_weight_min": -37.528072357177734,
      "activations/layer16_attention_weight_max": 34.3510856628418,
      "activations/layer16_attention_weight_min": -27.572988510131836,
      "activations/layer17_attention_weight_max": 30.647661209106445,
      "activations/layer17_attention_weight_min": -25.86774253845215,
      "activations/layer18_attention_weight_max": 30.155065536499023,
      "activations/layer18_attention_weight_min": -26.06618881225586,
      "activations/layer19_attention_weight_max": 35.02138137817383,
      "activations/layer19_attention_weight_min": -33.82390213012695,
      "activations/layer1_attention_weight_max": 15.721144676208496,
      "activations/layer1_attention_weight_min": -15.525297164916992,
      "activations/layer20_attention_weight_max": 27.31950569152832,
      "activations/layer20_attention_weight_min": -25.793981552124023,
      "activations/layer21_attention_weight_max": 28.969181060791016,
      "activations/layer21_attention_weight_min": -25.64682960510254,
      "activations/layer22_attention_weight_max": 38.86759567260742,
      "activations/layer22_attention_weight_min": -34.44871139526367,
      "activations/layer23_attention_weight_max": 31.355215072631836,
      "activations/layer23_attention_weight_min": -26.357501983642578,
      "activations/layer2_attention_weight_max": 31.468055725097656,
      "activations/layer2_attention_weight_min": -30.935901641845703,
      "activations/layer3_attention_weight_max": 102.73436737060547,
      "activations/layer3_attention_weight_min": -101.80599975585938,
      "activations/layer4_attention_weight_max": 104.72933197021484,
      "activations/layer4_attention_weight_min": -111.19974517822266,
      "activations/layer5_attention_weight_max": 47.87694549560547,
      "activations/layer5_attention_weight_min": -62.14112091064453,
      "activations/layer6_attention_weight_max": 45.31209945678711,
      "activations/layer6_attention_weight_min": -47.110347747802734,
      "activations/layer7_attention_weight_max": 89.50048828125,
      "activations/layer7_attention_weight_min": -94.1571044921875,
      "activations/layer8_attention_weight_max": 44.02887725830078,
      "activations/layer8_attention_weight_min": -43.72563171386719,
      "activations/layer9_attention_weight_max": 32.79311752319336,
      "activations/layer9_attention_weight_min": -33.62179946899414,
      "epoch": 20.14,
      "learning_rate": 2.0349621212121207e-05,
      "loss": 2.6993,
      "step": 346550
    },
    {
      "activations/layer0_attention_weight_max": 16.38737678527832,
      "activations/layer0_attention_weight_min": -12.580068588256836,
      "activations/layer10_attention_weight_max": 35.33917236328125,
      "activations/layer10_attention_weight_min": -34.312503814697266,
      "activations/layer11_attention_weight_max": 34.14753723144531,
      "activations/layer11_attention_weight_min": -34.994361877441406,
      "activations/layer12_attention_weight_max": 26.977453231811523,
      "activations/layer12_attention_weight_min": -27.65113067626953,
      "activations/layer13_attention_weight_max": 40.32189178466797,
      "activations/layer13_attention_weight_min": -36.687984466552734,
      "activations/layer14_attention_weight_max": 40.53934860229492,
      "activations/layer14_attention_weight_min": -38.77053451538086,
      "activations/layer15_attention_weight_max": 38.33424758911133,
      "activations/layer15_attention_weight_min": -35.49717330932617,
      "activations/layer16_attention_weight_max": 29.79585838317871,
      "activations/layer16_attention_weight_min": -28.126014709472656,
      "activations/layer17_attention_weight_max": 31.618831634521484,
      "activations/layer17_attention_weight_min": -26.320695877075195,
      "activations/layer18_attention_weight_max": 34.14387893676758,
      "activations/layer18_attention_weight_min": -28.592674255371094,
      "activations/layer19_attention_weight_max": 35.23827362060547,
      "activations/layer19_attention_weight_min": -31.85074806213379,
      "activations/layer1_attention_weight_max": 16.824844360351562,
      "activations/layer1_attention_weight_min": -14.530823707580566,
      "activations/layer20_attention_weight_max": 30.29787254333496,
      "activations/layer20_attention_weight_min": -26.56146240234375,
      "activations/layer21_attention_weight_max": 31.342899322509766,
      "activations/layer21_attention_weight_min": -26.85663414001465,
      "activations/layer22_attention_weight_max": 44.23223114013672,
      "activations/layer22_attention_weight_min": -32.109580993652344,
      "activations/layer23_attention_weight_max": 35.601341247558594,
      "activations/layer23_attention_weight_min": -28.573822021484375,
      "activations/layer2_attention_weight_max": 34.49365234375,
      "activations/layer2_attention_weight_min": -33.17085266113281,
      "activations/layer3_attention_weight_max": 101.30143737792969,
      "activations/layer3_attention_weight_min": -104.69510650634766,
      "activations/layer4_attention_weight_max": 106.83829498291016,
      "activations/layer4_attention_weight_min": -112.15941619873047,
      "activations/layer5_attention_weight_max": 53.89768600463867,
      "activations/layer5_attention_weight_min": -62.08950424194336,
      "activations/layer6_attention_weight_max": 46.24327850341797,
      "activations/layer6_attention_weight_min": -48.02851104736328,
      "activations/layer7_attention_weight_max": 102.07860565185547,
      "activations/layer7_attention_weight_min": -96.06402587890625,
      "activations/layer8_attention_weight_max": 45.21467971801758,
      "activations/layer8_attention_weight_min": -43.2606086730957,
      "activations/layer9_attention_weight_max": 31.435754776000977,
      "activations/layer9_attention_weight_min": -35.311038970947266,
      "epoch": 20.14,
      "learning_rate": 2.0330681818181816e-05,
      "loss": 2.7257,
      "step": 346600
    },
    {
      "activations/layer0_attention_weight_max": 16.080190658569336,
      "activations/layer0_attention_weight_min": -12.651928901672363,
      "activations/layer10_attention_weight_max": 34.988868713378906,
      "activations/layer10_attention_weight_min": -36.50117111206055,
      "activations/layer11_attention_weight_max": 37.53418731689453,
      "activations/layer11_attention_weight_min": -35.83026123046875,
      "activations/layer12_attention_weight_max": 28.29377555847168,
      "activations/layer12_attention_weight_min": -27.642166137695312,
      "activations/layer13_attention_weight_max": 41.869014739990234,
      "activations/layer13_attention_weight_min": -39.97602844238281,
      "activations/layer14_attention_weight_max": 46.59523391723633,
      "activations/layer14_attention_weight_min": -38.300697326660156,
      "activations/layer15_attention_weight_max": 43.589454650878906,
      "activations/layer15_attention_weight_min": -36.241294860839844,
      "activations/layer16_attention_weight_max": 30.156774520874023,
      "activations/layer16_attention_weight_min": -28.79168701171875,
      "activations/layer17_attention_weight_max": 30.91106605529785,
      "activations/layer17_attention_weight_min": -27.03343391418457,
      "activations/layer18_attention_weight_max": 32.562255859375,
      "activations/layer18_attention_weight_min": -25.631343841552734,
      "activations/layer19_attention_weight_max": 35.22321701049805,
      "activations/layer19_attention_weight_min": -35.30459213256836,
      "activations/layer1_attention_weight_max": 14.790788650512695,
      "activations/layer1_attention_weight_min": -13.816292762756348,
      "activations/layer20_attention_weight_max": 29.326494216918945,
      "activations/layer20_attention_weight_min": -26.966535568237305,
      "activations/layer21_attention_weight_max": 29.667997360229492,
      "activations/layer21_attention_weight_min": -26.78493881225586,
      "activations/layer22_attention_weight_max": 44.73257064819336,
      "activations/layer22_attention_weight_min": -33.577754974365234,
      "activations/layer23_attention_weight_max": 33.10739517211914,
      "activations/layer23_attention_weight_min": -25.517635345458984,
      "activations/layer2_attention_weight_max": 33.85508346557617,
      "activations/layer2_attention_weight_min": -31.16815757751465,
      "activations/layer3_attention_weight_max": 103.99693298339844,
      "activations/layer3_attention_weight_min": -103.4345703125,
      "activations/layer4_attention_weight_max": 113.3833236694336,
      "activations/layer4_attention_weight_min": -111.9345932006836,
      "activations/layer5_attention_weight_max": 54.44559860229492,
      "activations/layer5_attention_weight_min": -63.09718322753906,
      "activations/layer6_attention_weight_max": 46.00633239746094,
      "activations/layer6_attention_weight_min": -47.4228401184082,
      "activations/layer7_attention_weight_max": 96.61744689941406,
      "activations/layer7_attention_weight_min": -99.45967102050781,
      "activations/layer8_attention_weight_max": 43.90655517578125,
      "activations/layer8_attention_weight_min": -48.59779739379883,
      "activations/layer9_attention_weight_max": 34.60435104370117,
      "activations/layer9_attention_weight_min": -36.598060607910156,
      "epoch": 20.14,
      "learning_rate": 2.031174242424242e-05,
      "loss": 2.7053,
      "step": 346650
    },
    {
      "activations/layer0_attention_weight_max": 16.17994499206543,
      "activations/layer0_attention_weight_min": -12.739877700805664,
      "activations/layer10_attention_weight_max": 35.16133117675781,
      "activations/layer10_attention_weight_min": -36.04859161376953,
      "activations/layer11_attention_weight_max": 35.62731170654297,
      "activations/layer11_attention_weight_min": -38.033626556396484,
      "activations/layer12_attention_weight_max": 27.77861213684082,
      "activations/layer12_attention_weight_min": -27.911460876464844,
      "activations/layer13_attention_weight_max": 42.69019317626953,
      "activations/layer13_attention_weight_min": -37.975563049316406,
      "activations/layer14_attention_weight_max": 46.1768684387207,
      "activations/layer14_attention_weight_min": -41.802337646484375,
      "activations/layer15_attention_weight_max": 43.5126838684082,
      "activations/layer15_attention_weight_min": -38.80747985839844,
      "activations/layer16_attention_weight_max": 31.964147567749023,
      "activations/layer16_attention_weight_min": -30.22754669189453,
      "activations/layer17_attention_weight_max": 32.299217224121094,
      "activations/layer17_attention_weight_min": -28.302104949951172,
      "activations/layer18_attention_weight_max": 36.26891326904297,
      "activations/layer18_attention_weight_min": -27.4318790435791,
      "activations/layer19_attention_weight_max": 39.444976806640625,
      "activations/layer19_attention_weight_min": -35.23717498779297,
      "activations/layer1_attention_weight_max": 16.348840713500977,
      "activations/layer1_attention_weight_min": -15.244820594787598,
      "activations/layer20_attention_weight_max": 32.48649597167969,
      "activations/layer20_attention_weight_min": -26.185386657714844,
      "activations/layer21_attention_weight_max": 33.9805793762207,
      "activations/layer21_attention_weight_min": -25.16483497619629,
      "activations/layer22_attention_weight_max": 46.04899978637695,
      "activations/layer22_attention_weight_min": -34.0161018371582,
      "activations/layer23_attention_weight_max": 35.664024353027344,
      "activations/layer23_attention_weight_min": -25.258251190185547,
      "activations/layer2_attention_weight_max": 31.861385345458984,
      "activations/layer2_attention_weight_min": -30.44211196899414,
      "activations/layer3_attention_weight_max": 98.5519027709961,
      "activations/layer3_attention_weight_min": -104.07593536376953,
      "activations/layer4_attention_weight_max": 106.21533203125,
      "activations/layer4_attention_weight_min": -109.72450256347656,
      "activations/layer5_attention_weight_max": 48.8294677734375,
      "activations/layer5_attention_weight_min": -59.61643600463867,
      "activations/layer6_attention_weight_max": 43.352333068847656,
      "activations/layer6_attention_weight_min": -45.753700256347656,
      "activations/layer7_attention_weight_max": 93.92622375488281,
      "activations/layer7_attention_weight_min": -97.14486694335938,
      "activations/layer8_attention_weight_max": 44.116661071777344,
      "activations/layer8_attention_weight_min": -46.49422073364258,
      "activations/layer9_attention_weight_max": 32.78778839111328,
      "activations/layer9_attention_weight_min": -36.331520080566406,
      "epoch": 20.15,
      "learning_rate": 2.029280303030303e-05,
      "loss": 2.7063,
      "step": 346700
    },
    {
      "activations/layer0_attention_weight_max": 16.99166488647461,
      "activations/layer0_attention_weight_min": -14.18148422241211,
      "activations/layer10_attention_weight_max": 33.192626953125,
      "activations/layer10_attention_weight_min": -33.51702880859375,
      "activations/layer11_attention_weight_max": 33.38194274902344,
      "activations/layer11_attention_weight_min": -35.500282287597656,
      "activations/layer12_attention_weight_max": 25.12458038330078,
      "activations/layer12_attention_weight_min": -25.870725631713867,
      "activations/layer13_attention_weight_max": 37.14314651489258,
      "activations/layer13_attention_weight_min": -35.848167419433594,
      "activations/layer14_attention_weight_max": 38.97571563720703,
      "activations/layer14_attention_weight_min": -37.64546203613281,
      "activations/layer15_attention_weight_max": 36.77753448486328,
      "activations/layer15_attention_weight_min": -34.76128005981445,
      "activations/layer16_attention_weight_max": 28.987890243530273,
      "activations/layer16_attention_weight_min": -29.309463500976562,
      "activations/layer17_attention_weight_max": 30.925439834594727,
      "activations/layer17_attention_weight_min": -25.7572021484375,
      "activations/layer18_attention_weight_max": 32.53573989868164,
      "activations/layer18_attention_weight_min": -27.756168365478516,
      "activations/layer19_attention_weight_max": 32.241634368896484,
      "activations/layer19_attention_weight_min": -30.71259880065918,
      "activations/layer1_attention_weight_max": 15.784088134765625,
      "activations/layer1_attention_weight_min": -15.39359188079834,
      "activations/layer20_attention_weight_max": 28.06940269470215,
      "activations/layer20_attention_weight_min": -27.665008544921875,
      "activations/layer21_attention_weight_max": 29.34922981262207,
      "activations/layer21_attention_weight_min": -25.780839920043945,
      "activations/layer22_attention_weight_max": 38.874088287353516,
      "activations/layer22_attention_weight_min": -32.56307601928711,
      "activations/layer23_attention_weight_max": 30.55586814880371,
      "activations/layer23_attention_weight_min": -24.785367965698242,
      "activations/layer2_attention_weight_max": 32.17574691772461,
      "activations/layer2_attention_weight_min": -31.388883590698242,
      "activations/layer3_attention_weight_max": 101.52423858642578,
      "activations/layer3_attention_weight_min": -100.91532897949219,
      "activations/layer4_attention_weight_max": 106.09374237060547,
      "activations/layer4_attention_weight_min": -111.67012023925781,
      "activations/layer5_attention_weight_max": 51.89490509033203,
      "activations/layer5_attention_weight_min": -64.49945068359375,
      "activations/layer6_attention_weight_max": 46.03413772583008,
      "activations/layer6_attention_weight_min": -45.18024826049805,
      "activations/layer7_attention_weight_max": 93.94033813476562,
      "activations/layer7_attention_weight_min": -94.8127212524414,
      "activations/layer8_attention_weight_max": 42.420650482177734,
      "activations/layer8_attention_weight_min": -43.12052536010742,
      "activations/layer9_attention_weight_max": 30.577192306518555,
      "activations/layer9_attention_weight_min": -35.236915588378906,
      "epoch": 20.15,
      "learning_rate": 2.0273863636363634e-05,
      "loss": 2.7219,
      "step": 346750
    },
    {
      "activations/layer0_attention_weight_max": 16.664905548095703,
      "activations/layer0_attention_weight_min": -12.623906135559082,
      "activations/layer10_attention_weight_max": 34.244361877441406,
      "activations/layer10_attention_weight_min": -34.713157653808594,
      "activations/layer11_attention_weight_max": 35.07341003417969,
      "activations/layer11_attention_weight_min": -35.955078125,
      "activations/layer12_attention_weight_max": 27.498628616333008,
      "activations/layer12_attention_weight_min": -28.597803115844727,
      "activations/layer13_attention_weight_max": 39.66577911376953,
      "activations/layer13_attention_weight_min": -38.57460021972656,
      "activations/layer14_attention_weight_max": 43.69750213623047,
      "activations/layer14_attention_weight_min": -39.14104080200195,
      "activations/layer15_attention_weight_max": 39.03984451293945,
      "activations/layer15_attention_weight_min": -37.91293716430664,
      "activations/layer16_attention_weight_max": 29.84143829345703,
      "activations/layer16_attention_weight_min": -29.581443786621094,
      "activations/layer17_attention_weight_max": 31.297597885131836,
      "activations/layer17_attention_weight_min": -27.663955688476562,
      "activations/layer18_attention_weight_max": 32.32758712768555,
      "activations/layer18_attention_weight_min": -26.08671760559082,
      "activations/layer19_attention_weight_max": 32.67533493041992,
      "activations/layer19_attention_weight_min": -32.184505462646484,
      "activations/layer1_attention_weight_max": 15.962601661682129,
      "activations/layer1_attention_weight_min": -14.38798713684082,
      "activations/layer20_attention_weight_max": 29.592388153076172,
      "activations/layer20_attention_weight_min": -26.729930877685547,
      "activations/layer21_attention_weight_max": 28.00144386291504,
      "activations/layer21_attention_weight_min": -25.34126853942871,
      "activations/layer22_attention_weight_max": 47.9069938659668,
      "activations/layer22_attention_weight_min": -34.28288269042969,
      "activations/layer23_attention_weight_max": 33.56880187988281,
      "activations/layer23_attention_weight_min": -27.504804611206055,
      "activations/layer2_attention_weight_max": 31.14484977722168,
      "activations/layer2_attention_weight_min": -31.649200439453125,
      "activations/layer3_attention_weight_max": 95.78010559082031,
      "activations/layer3_attention_weight_min": -99.70520782470703,
      "activations/layer4_attention_weight_max": 103.04520416259766,
      "activations/layer4_attention_weight_min": -108.61559295654297,
      "activations/layer5_attention_weight_max": 49.05311584472656,
      "activations/layer5_attention_weight_min": -59.32584762573242,
      "activations/layer6_attention_weight_max": 44.01180648803711,
      "activations/layer6_attention_weight_min": -46.1931037902832,
      "activations/layer7_attention_weight_max": 89.8260726928711,
      "activations/layer7_attention_weight_min": -94.80459594726562,
      "activations/layer8_attention_weight_max": 42.84092330932617,
      "activations/layer8_attention_weight_min": -45.14360046386719,
      "activations/layer9_attention_weight_max": 32.700931549072266,
      "activations/layer9_attention_weight_min": -35.348976135253906,
      "epoch": 20.15,
      "learning_rate": 2.0254924242424242e-05,
      "loss": 2.7252,
      "step": 346800
    },
    {
      "activations/layer0_attention_weight_max": 16.604122161865234,
      "activations/layer0_attention_weight_min": -13.380880355834961,
      "activations/layer10_attention_weight_max": 35.984527587890625,
      "activations/layer10_attention_weight_min": -35.51817321777344,
      "activations/layer11_attention_weight_max": 36.774139404296875,
      "activations/layer11_attention_weight_min": -36.16266632080078,
      "activations/layer12_attention_weight_max": 28.20150375366211,
      "activations/layer12_attention_weight_min": -27.565210342407227,
      "activations/layer13_attention_weight_max": 40.689903259277344,
      "activations/layer13_attention_weight_min": -38.29127502441406,
      "activations/layer14_attention_weight_max": 44.82807540893555,
      "activations/layer14_attention_weight_min": -39.85435104370117,
      "activations/layer15_attention_weight_max": 41.10261154174805,
      "activations/layer15_attention_weight_min": -36.893009185791016,
      "activations/layer16_attention_weight_max": 30.786035537719727,
      "activations/layer16_attention_weight_min": -30.277400970458984,
      "activations/layer17_attention_weight_max": 32.20328903198242,
      "activations/layer17_attention_weight_min": -27.450027465820312,
      "activations/layer18_attention_weight_max": 34.75728988647461,
      "activations/layer18_attention_weight_min": -27.8806095123291,
      "activations/layer19_attention_weight_max": 35.73215866088867,
      "activations/layer19_attention_weight_min": -34.08745193481445,
      "activations/layer1_attention_weight_max": 15.918943405151367,
      "activations/layer1_attention_weight_min": -14.429372787475586,
      "activations/layer20_attention_weight_max": 31.39008140563965,
      "activations/layer20_attention_weight_min": -26.647205352783203,
      "activations/layer21_attention_weight_max": 31.86405372619629,
      "activations/layer21_attention_weight_min": -26.375944137573242,
      "activations/layer22_attention_weight_max": 45.63792419433594,
      "activations/layer22_attention_weight_min": -36.06711196899414,
      "activations/layer23_attention_weight_max": 36.37870407104492,
      "activations/layer23_attention_weight_min": -26.1611328125,
      "activations/layer2_attention_weight_max": 30.89807891845703,
      "activations/layer2_attention_weight_min": -30.71460723876953,
      "activations/layer3_attention_weight_max": 98.14453887939453,
      "activations/layer3_attention_weight_min": -100.68858337402344,
      "activations/layer4_attention_weight_max": 103.02361297607422,
      "activations/layer4_attention_weight_min": -107.44972229003906,
      "activations/layer5_attention_weight_max": 49.4520263671875,
      "activations/layer5_attention_weight_min": -63.426448822021484,
      "activations/layer6_attention_weight_max": 46.08116912841797,
      "activations/layer6_attention_weight_min": -46.67768859863281,
      "activations/layer7_attention_weight_max": 92.74430847167969,
      "activations/layer7_attention_weight_min": -96.10197448730469,
      "activations/layer8_attention_weight_max": 41.54830551147461,
      "activations/layer8_attention_weight_min": -43.57221603393555,
      "activations/layer9_attention_weight_max": 32.98312759399414,
      "activations/layer9_attention_weight_min": -35.69172286987305,
      "epoch": 20.15,
      "learning_rate": 2.023598484848485e-05,
      "loss": 2.711,
      "step": 346850
    },
    {
      "activations/layer0_attention_weight_max": 16.032201766967773,
      "activations/layer0_attention_weight_min": -13.923460006713867,
      "activations/layer10_attention_weight_max": 35.590335845947266,
      "activations/layer10_attention_weight_min": -35.663116455078125,
      "activations/layer11_attention_weight_max": 34.87687683105469,
      "activations/layer11_attention_weight_min": -37.253021240234375,
      "activations/layer12_attention_weight_max": 29.22343635559082,
      "activations/layer12_attention_weight_min": -28.206287384033203,
      "activations/layer13_attention_weight_max": 41.74140167236328,
      "activations/layer13_attention_weight_min": -38.03446578979492,
      "activations/layer14_attention_weight_max": 43.7764892578125,
      "activations/layer14_attention_weight_min": -38.77885437011719,
      "activations/layer15_attention_weight_max": 44.74276351928711,
      "activations/layer15_attention_weight_min": -35.98115921020508,
      "activations/layer16_attention_weight_max": 32.33700942993164,
      "activations/layer16_attention_weight_min": -27.02057647705078,
      "activations/layer17_attention_weight_max": 30.185218811035156,
      "activations/layer17_attention_weight_min": -27.368661880493164,
      "activations/layer18_attention_weight_max": 31.61581039428711,
      "activations/layer18_attention_weight_min": -26.683534622192383,
      "activations/layer19_attention_weight_max": 37.80729293823242,
      "activations/layer19_attention_weight_min": -36.527931213378906,
      "activations/layer1_attention_weight_max": 16.040830612182617,
      "activations/layer1_attention_weight_min": -14.88515853881836,
      "activations/layer20_attention_weight_max": 29.865631103515625,
      "activations/layer20_attention_weight_min": -29.397214889526367,
      "activations/layer21_attention_weight_max": 30.40936279296875,
      "activations/layer21_attention_weight_min": -29.279630661010742,
      "activations/layer22_attention_weight_max": 41.97882080078125,
      "activations/layer22_attention_weight_min": -35.062828063964844,
      "activations/layer23_attention_weight_max": 31.643497467041016,
      "activations/layer23_attention_weight_min": -28.244840621948242,
      "activations/layer2_attention_weight_max": 31.24126434326172,
      "activations/layer2_attention_weight_min": -31.78736686706543,
      "activations/layer3_attention_weight_max": 99.15663146972656,
      "activations/layer3_attention_weight_min": -100.32244873046875,
      "activations/layer4_attention_weight_max": 105.69304656982422,
      "activations/layer4_attention_weight_min": -109.71102142333984,
      "activations/layer5_attention_weight_max": 47.96133041381836,
      "activations/layer5_attention_weight_min": -63.80463409423828,
      "activations/layer6_attention_weight_max": 44.52974319458008,
      "activations/layer6_attention_weight_min": -48.385257720947266,
      "activations/layer7_attention_weight_max": 92.98925018310547,
      "activations/layer7_attention_weight_min": -101.49201965332031,
      "activations/layer8_attention_weight_max": 41.92250061035156,
      "activations/layer8_attention_weight_min": -44.02165603637695,
      "activations/layer9_attention_weight_max": 34.21450424194336,
      "activations/layer9_attention_weight_min": -33.851036071777344,
      "epoch": 20.16,
      "learning_rate": 2.021704545454545e-05,
      "loss": 2.7296,
      "step": 346900
    },
    {
      "activations/layer0_attention_weight_max": 15.601556777954102,
      "activations/layer0_attention_weight_min": -12.651392936706543,
      "activations/layer10_attention_weight_max": 38.03914260864258,
      "activations/layer10_attention_weight_min": -37.73002243041992,
      "activations/layer11_attention_weight_max": 40.6382942199707,
      "activations/layer11_attention_weight_min": -38.713348388671875,
      "activations/layer12_attention_weight_max": 27.712852478027344,
      "activations/layer12_attention_weight_min": -27.331026077270508,
      "activations/layer13_attention_weight_max": 39.70249938964844,
      "activations/layer13_attention_weight_min": -37.61692428588867,
      "activations/layer14_attention_weight_max": 44.559669494628906,
      "activations/layer14_attention_weight_min": -38.73763656616211,
      "activations/layer15_attention_weight_max": 40.65030288696289,
      "activations/layer15_attention_weight_min": -36.659324645996094,
      "activations/layer16_attention_weight_max": 30.256053924560547,
      "activations/layer16_attention_weight_min": -29.001243591308594,
      "activations/layer17_attention_weight_max": 29.82777976989746,
      "activations/layer17_attention_weight_min": -27.07811737060547,
      "activations/layer18_attention_weight_max": 33.07992935180664,
      "activations/layer18_attention_weight_min": -27.912626266479492,
      "activations/layer19_attention_weight_max": 36.665443420410156,
      "activations/layer19_attention_weight_min": -33.26815414428711,
      "activations/layer1_attention_weight_max": 16.52678108215332,
      "activations/layer1_attention_weight_min": -16.819406509399414,
      "activations/layer20_attention_weight_max": 33.07475280761719,
      "activations/layer20_attention_weight_min": -26.207008361816406,
      "activations/layer21_attention_weight_max": 33.58119583129883,
      "activations/layer21_attention_weight_min": -25.590713500976562,
      "activations/layer22_attention_weight_max": 47.60089874267578,
      "activations/layer22_attention_weight_min": -35.36284255981445,
      "activations/layer23_attention_weight_max": 35.45933532714844,
      "activations/layer23_attention_weight_min": -27.1270751953125,
      "activations/layer2_attention_weight_max": 33.437198638916016,
      "activations/layer2_attention_weight_min": -34.371307373046875,
      "activations/layer3_attention_weight_max": 106.3394775390625,
      "activations/layer3_attention_weight_min": -110.77113342285156,
      "activations/layer4_attention_weight_max": 114.56904602050781,
      "activations/layer4_attention_weight_min": -113.83715057373047,
      "activations/layer5_attention_weight_max": 52.38396453857422,
      "activations/layer5_attention_weight_min": -64.90170288085938,
      "activations/layer6_attention_weight_max": 46.093685150146484,
      "activations/layer6_attention_weight_min": -49.762821197509766,
      "activations/layer7_attention_weight_max": 102.2313003540039,
      "activations/layer7_attention_weight_min": -103.6481704711914,
      "activations/layer8_attention_weight_max": 45.29414367675781,
      "activations/layer8_attention_weight_min": -48.63888931274414,
      "activations/layer9_attention_weight_max": 34.14250946044922,
      "activations/layer9_attention_weight_min": -36.37887954711914,
      "epoch": 20.16,
      "learning_rate": 2.019810606060606e-05,
      "loss": 2.7181,
      "step": 346950
    },
    {
      "activations/layer0_attention_weight_max": 15.462836265563965,
      "activations/layer0_attention_weight_min": -13.284459114074707,
      "activations/layer10_attention_weight_max": 37.03009033203125,
      "activations/layer10_attention_weight_min": -34.681640625,
      "activations/layer11_attention_weight_max": 34.34419631958008,
      "activations/layer11_attention_weight_min": -34.82193374633789,
      "activations/layer12_attention_weight_max": 27.712329864501953,
      "activations/layer12_attention_weight_min": -27.43018341064453,
      "activations/layer13_attention_weight_max": 41.30036163330078,
      "activations/layer13_attention_weight_min": -37.619354248046875,
      "activations/layer14_attention_weight_max": 42.99123764038086,
      "activations/layer14_attention_weight_min": -39.644256591796875,
      "activations/layer15_attention_weight_max": 39.65757369995117,
      "activations/layer15_attention_weight_min": -39.41288375854492,
      "activations/layer16_attention_weight_max": 30.478389739990234,
      "activations/layer16_attention_weight_min": -30.152233123779297,
      "activations/layer17_attention_weight_max": 30.798030853271484,
      "activations/layer17_attention_weight_min": -26.82131576538086,
      "activations/layer18_attention_weight_max": 32.65569305419922,
      "activations/layer18_attention_weight_min": -26.33928108215332,
      "activations/layer19_attention_weight_max": 41.12811279296875,
      "activations/layer19_attention_weight_min": -29.984149932861328,
      "activations/layer1_attention_weight_max": 15.434036254882812,
      "activations/layer1_attention_weight_min": -15.149209976196289,
      "activations/layer20_attention_weight_max": 30.646923065185547,
      "activations/layer20_attention_weight_min": -24.95467185974121,
      "activations/layer21_attention_weight_max": 29.576200485229492,
      "activations/layer21_attention_weight_min": -26.220884323120117,
      "activations/layer22_attention_weight_max": 42.02517318725586,
      "activations/layer22_attention_weight_min": -33.13063049316406,
      "activations/layer23_attention_weight_max": 35.0173225402832,
      "activations/layer23_attention_weight_min": -25.74547576904297,
      "activations/layer2_attention_weight_max": 32.38081741333008,
      "activations/layer2_attention_weight_min": -32.23304748535156,
      "activations/layer3_attention_weight_max": 104.81941223144531,
      "activations/layer3_attention_weight_min": -107.6406478881836,
      "activations/layer4_attention_weight_max": 114.9302978515625,
      "activations/layer4_attention_weight_min": -111.19246673583984,
      "activations/layer5_attention_weight_max": 52.393463134765625,
      "activations/layer5_attention_weight_min": -62.59214782714844,
      "activations/layer6_attention_weight_max": 47.26945495605469,
      "activations/layer6_attention_weight_min": -49.513038635253906,
      "activations/layer7_attention_weight_max": 96.07958984375,
      "activations/layer7_attention_weight_min": -103.10557556152344,
      "activations/layer8_attention_weight_max": 43.129825592041016,
      "activations/layer8_attention_weight_min": -45.87117004394531,
      "activations/layer9_attention_weight_max": 34.22434997558594,
      "activations/layer9_attention_weight_min": -36.721858978271484,
      "epoch": 20.16,
      "learning_rate": 2.0179166666666665e-05,
      "loss": 2.7319,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.4886,
      "eval_samples_per_second": 505.855,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.4886,
      "eval_openwebtext_samples_per_second": 505.855,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9596,
      "eval_wikitext_samples_per_second": 232.704,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_lambada_loss": 2.337890625,
      "eval_lambada_ppl": 10.359361721090316,
      "eval_lambada_runtime": 9.5251,
      "eval_lambada_samples_per_second": 511.178,
      "step": 347000
    },
    {
      "activations/layer0_attention_weight_max": 17.269140243530273,
      "activations/layer0_attention_weight_min": -12.946950912475586,
      "activations/layer10_attention_weight_max": 37.08312225341797,
      "activations/layer10_attention_weight_min": -38.74979782104492,
      "activations/layer11_attention_weight_max": 36.2127799987793,
      "activations/layer11_attention_weight_min": -38.330833435058594,
      "activations/layer12_attention_weight_max": 34.59229278564453,
      "activations/layer12_attention_weight_min": -29.44569206237793,
      "activations/layer13_attention_weight_max": 44.86005401611328,
      "activations/layer13_attention_weight_min": -40.438926696777344,
      "activations/layer14_attention_weight_max": 48.496421813964844,
      "activations/layer14_attention_weight_min": -42.280426025390625,
      "activations/layer15_attention_weight_max": 45.12563705444336,
      "activations/layer15_attention_weight_min": -39.65744400024414,
      "activations/layer16_attention_weight_max": 31.797374725341797,
      "activations/layer16_attention_weight_min": -29.171140670776367,
      "activations/layer17_attention_weight_max": 32.83893585205078,
      "activations/layer17_attention_weight_min": -28.35137367248535,
      "activations/layer18_attention_weight_max": 32.94892501831055,
      "activations/layer18_attention_weight_min": -25.878650665283203,
      "activations/layer19_attention_weight_max": 33.19060516357422,
      "activations/layer19_attention_weight_min": -30.597990036010742,
      "activations/layer1_attention_weight_max": 15.958974838256836,
      "activations/layer1_attention_weight_min": -14.727567672729492,
      "activations/layer20_attention_weight_max": 31.699600219726562,
      "activations/layer20_attention_weight_min": -24.440101623535156,
      "activations/layer21_attention_weight_max": 28.922710418701172,
      "activations/layer21_attention_weight_min": -23.15028190612793,
      "activations/layer22_attention_weight_max": 40.4501953125,
      "activations/layer22_attention_weight_min": -31.023069381713867,
      "activations/layer23_attention_weight_max": 32.870975494384766,
      "activations/layer23_attention_weight_min": -25.308063507080078,
      "activations/layer2_attention_weight_max": 33.859230041503906,
      "activations/layer2_attention_weight_min": -32.13136291503906,
      "activations/layer3_attention_weight_max": 101.35086059570312,
      "activations/layer3_attention_weight_min": -101.83320617675781,
      "activations/layer4_attention_weight_max": 107.65541076660156,
      "activations/layer4_attention_weight_min": -108.5127944946289,
      "activations/layer5_attention_weight_max": 50.91978454589844,
      "activations/layer5_attention_weight_min": -63.117210388183594,
      "activations/layer6_attention_weight_max": 44.78544616699219,
      "activations/layer6_attention_weight_min": -48.22557067871094,
      "activations/layer7_attention_weight_max": 97.40599822998047,
      "activations/layer7_attention_weight_min": -100.13895416259766,
      "activations/layer8_attention_weight_max": 44.65517044067383,
      "activations/layer8_attention_weight_min": -45.35533905029297,
      "activations/layer9_attention_weight_max": 35.01426315307617,
      "activations/layer9_attention_weight_min": -36.480247497558594,
      "epoch": 20.17,
      "learning_rate": 2.0160227272727273e-05,
      "loss": 2.6938,
      "step": 347050
    },
    {
      "activations/layer0_attention_weight_max": 16.456655502319336,
      "activations/layer0_attention_weight_min": -14.148993492126465,
      "activations/layer10_attention_weight_max": 32.8287239074707,
      "activations/layer10_attention_weight_min": -33.88824462890625,
      "activations/layer11_attention_weight_max": 32.66117858886719,
      "activations/layer11_attention_weight_min": -32.96278381347656,
      "activations/layer12_attention_weight_max": 25.318742752075195,
      "activations/layer12_attention_weight_min": -25.851171493530273,
      "activations/layer13_attention_weight_max": 37.87752151489258,
      "activations/layer13_attention_weight_min": -35.40217208862305,
      "activations/layer14_attention_weight_max": 43.99188995361328,
      "activations/layer14_attention_weight_min": -36.815284729003906,
      "activations/layer15_attention_weight_max": 39.454742431640625,
      "activations/layer15_attention_weight_min": -35.540199279785156,
      "activations/layer16_attention_weight_max": 28.693096160888672,
      "activations/layer16_attention_weight_min": -29.523374557495117,
      "activations/layer17_attention_weight_max": 30.94660186767578,
      "activations/layer17_attention_weight_min": -26.234664916992188,
      "activations/layer18_attention_weight_max": 32.3526496887207,
      "activations/layer18_attention_weight_min": -26.407129287719727,
      "activations/layer19_attention_weight_max": 32.62764358520508,
      "activations/layer19_attention_weight_min": -31.06717872619629,
      "activations/layer1_attention_weight_max": 16.175413131713867,
      "activations/layer1_attention_weight_min": -15.500191688537598,
      "activations/layer20_attention_weight_max": 30.120609283447266,
      "activations/layer20_attention_weight_min": -25.173267364501953,
      "activations/layer21_attention_weight_max": 29.108367919921875,
      "activations/layer21_attention_weight_min": -25.12154197692871,
      "activations/layer22_attention_weight_max": 40.46467590332031,
      "activations/layer22_attention_weight_min": -31.741235733032227,
      "activations/layer23_attention_weight_max": 34.80065155029297,
      "activations/layer23_attention_weight_min": -25.531131744384766,
      "activations/layer2_attention_weight_max": 31.278404235839844,
      "activations/layer2_attention_weight_min": -30.716001510620117,
      "activations/layer3_attention_weight_max": 102.03949737548828,
      "activations/layer3_attention_weight_min": -99.74398040771484,
      "activations/layer4_attention_weight_max": 105.77071380615234,
      "activations/layer4_attention_weight_min": -107.84272766113281,
      "activations/layer5_attention_weight_max": 51.14175033569336,
      "activations/layer5_attention_weight_min": -61.286346435546875,
      "activations/layer6_attention_weight_max": 42.98027038574219,
      "activations/layer6_attention_weight_min": -46.73289489746094,
      "activations/layer7_attention_weight_max": 87.78730010986328,
      "activations/layer7_attention_weight_min": -90.5745849609375,
      "activations/layer8_attention_weight_max": 39.86195755004883,
      "activations/layer8_attention_weight_min": -42.12836456298828,
      "activations/layer9_attention_weight_max": 30.398427963256836,
      "activations/layer9_attention_weight_min": -33.53249740600586,
      "epoch": 20.17,
      "learning_rate": 2.0141287878787875e-05,
      "loss": 2.7105,
      "step": 347100
    },
    {
      "activations/layer0_attention_weight_max": 15.46756362915039,
      "activations/layer0_attention_weight_min": -13.883596420288086,
      "activations/layer10_attention_weight_max": 34.34485626220703,
      "activations/layer10_attention_weight_min": -34.37345504760742,
      "activations/layer11_attention_weight_max": 31.929882049560547,
      "activations/layer11_attention_weight_min": -33.17445373535156,
      "activations/layer12_attention_weight_max": 27.566362380981445,
      "activations/layer12_attention_weight_min": -28.20772933959961,
      "activations/layer13_attention_weight_max": 38.67622756958008,
      "activations/layer13_attention_weight_min": -36.86846160888672,
      "activations/layer14_attention_weight_max": 43.416873931884766,
      "activations/layer14_attention_weight_min": -38.751068115234375,
      "activations/layer15_attention_weight_max": 42.18415069580078,
      "activations/layer15_attention_weight_min": -38.007503509521484,
      "activations/layer16_attention_weight_max": 31.525272369384766,
      "activations/layer16_attention_weight_min": -27.555465698242188,
      "activations/layer17_attention_weight_max": 30.63111114501953,
      "activations/layer17_attention_weight_min": -27.343320846557617,
      "activations/layer18_attention_weight_max": 31.248014450073242,
      "activations/layer18_attention_weight_min": -26.615516662597656,
      "activations/layer19_attention_weight_max": 38.18974304199219,
      "activations/layer19_attention_weight_min": -33.16592788696289,
      "activations/layer1_attention_weight_max": 15.526758193969727,
      "activations/layer1_attention_weight_min": -14.723350524902344,
      "activations/layer20_attention_weight_max": 30.939720153808594,
      "activations/layer20_attention_weight_min": -27.23318099975586,
      "activations/layer21_attention_weight_max": 30.439786911010742,
      "activations/layer21_attention_weight_min": -27.73024559020996,
      "activations/layer22_attention_weight_max": 40.6960563659668,
      "activations/layer22_attention_weight_min": -31.632789611816406,
      "activations/layer23_attention_weight_max": 33.77922058105469,
      "activations/layer23_attention_weight_min": -26.505889892578125,
      "activations/layer2_attention_weight_max": 33.78138732910156,
      "activations/layer2_attention_weight_min": -32.65449905395508,
      "activations/layer3_attention_weight_max": 101.71489715576172,
      "activations/layer3_attention_weight_min": -102.3427505493164,
      "activations/layer4_attention_weight_max": 108.89696502685547,
      "activations/layer4_attention_weight_min": -110.8426513671875,
      "activations/layer5_attention_weight_max": 50.590179443359375,
      "activations/layer5_attention_weight_min": -61.219993591308594,
      "activations/layer6_attention_weight_max": 43.019493103027344,
      "activations/layer6_attention_weight_min": -47.48061752319336,
      "activations/layer7_attention_weight_max": 91.98090362548828,
      "activations/layer7_attention_weight_min": -95.52117919921875,
      "activations/layer8_attention_weight_max": 43.19126510620117,
      "activations/layer8_attention_weight_min": -44.770748138427734,
      "activations/layer9_attention_weight_max": 31.180683135986328,
      "activations/layer9_attention_weight_min": -35.438167572021484,
      "epoch": 20.17,
      "learning_rate": 2.0122348484848483e-05,
      "loss": 2.7103,
      "step": 347150
    },
    {
      "activations/layer0_attention_weight_max": 16.411151885986328,
      "activations/layer0_attention_weight_min": -13.471923828125,
      "activations/layer10_attention_weight_max": 34.646244049072266,
      "activations/layer10_attention_weight_min": -34.2626838684082,
      "activations/layer11_attention_weight_max": 34.145477294921875,
      "activations/layer11_attention_weight_min": -34.071510314941406,
      "activations/layer12_attention_weight_max": 26.216066360473633,
      "activations/layer12_attention_weight_min": -26.468769073486328,
      "activations/layer13_attention_weight_max": 40.252193450927734,
      "activations/layer13_attention_weight_min": -36.63508605957031,
      "activations/layer14_attention_weight_max": 40.79366683959961,
      "activations/layer14_attention_weight_min": -38.19747543334961,
      "activations/layer15_attention_weight_max": 39.221351623535156,
      "activations/layer15_attention_weight_min": -35.38614273071289,
      "activations/layer16_attention_weight_max": 30.005910873413086,
      "activations/layer16_attention_weight_min": -27.927146911621094,
      "activations/layer17_attention_weight_max": 28.03389549255371,
      "activations/layer17_attention_weight_min": -26.156335830688477,
      "activations/layer18_attention_weight_max": 31.545854568481445,
      "activations/layer18_attention_weight_min": -26.976829528808594,
      "activations/layer19_attention_weight_max": 36.3192253112793,
      "activations/layer19_attention_weight_min": -33.500606536865234,
      "activations/layer1_attention_weight_max": 16.24673080444336,
      "activations/layer1_attention_weight_min": -16.396839141845703,
      "activations/layer20_attention_weight_max": 27.33083152770996,
      "activations/layer20_attention_weight_min": -26.68385887145996,
      "activations/layer21_attention_weight_max": 27.472984313964844,
      "activations/layer21_attention_weight_min": -26.263370513916016,
      "activations/layer22_attention_weight_max": 42.15866470336914,
      "activations/layer22_attention_weight_min": -33.31761169433594,
      "activations/layer23_attention_weight_max": 31.52545738220215,
      "activations/layer23_attention_weight_min": -28.54210662841797,
      "activations/layer2_attention_weight_max": 33.09769821166992,
      "activations/layer2_attention_weight_min": -31.311954498291016,
      "activations/layer3_attention_weight_max": 100.88158416748047,
      "activations/layer3_attention_weight_min": -101.21916961669922,
      "activations/layer4_attention_weight_max": 105.3576889038086,
      "activations/layer4_attention_weight_min": -106.9656753540039,
      "activations/layer5_attention_weight_max": 48.43917465209961,
      "activations/layer5_attention_weight_min": -62.33405685424805,
      "activations/layer6_attention_weight_max": 46.36587905883789,
      "activations/layer6_attention_weight_min": -49.64730453491211,
      "activations/layer7_attention_weight_max": 95.88099670410156,
      "activations/layer7_attention_weight_min": -104.48128509521484,
      "activations/layer8_attention_weight_max": 46.12663269042969,
      "activations/layer8_attention_weight_min": -48.0581169128418,
      "activations/layer9_attention_weight_max": 33.15717315673828,
      "activations/layer9_attention_weight_min": -37.167266845703125,
      "epoch": 20.17,
      "learning_rate": 2.0103409090909088e-05,
      "loss": 2.6981,
      "step": 347200
    },
    {
      "activations/layer0_attention_weight_max": 16.380455017089844,
      "activations/layer0_attention_weight_min": -13.636093139648438,
      "activations/layer10_attention_weight_max": 41.3337516784668,
      "activations/layer10_attention_weight_min": -37.60129165649414,
      "activations/layer11_attention_weight_max": 38.83098602294922,
      "activations/layer11_attention_weight_min": -38.93113708496094,
      "activations/layer12_attention_weight_max": 28.42331886291504,
      "activations/layer12_attention_weight_min": -29.122697830200195,
      "activations/layer13_attention_weight_max": 42.20085144042969,
      "activations/layer13_attention_weight_min": -34.927833557128906,
      "activations/layer14_attention_weight_max": 44.38298797607422,
      "activations/layer14_attention_weight_min": -39.6689567565918,
      "activations/layer15_attention_weight_max": 43.33027648925781,
      "activations/layer15_attention_weight_min": -37.76446533203125,
      "activations/layer16_attention_weight_max": 35.36876678466797,
      "activations/layer16_attention_weight_min": -31.850130081176758,
      "activations/layer17_attention_weight_max": 32.248966217041016,
      "activations/layer17_attention_weight_min": -30.512311935424805,
      "activations/layer18_attention_weight_max": 35.25827407836914,
      "activations/layer18_attention_weight_min": -29.049236297607422,
      "activations/layer19_attention_weight_max": 40.48643112182617,
      "activations/layer19_attention_weight_min": -34.11376190185547,
      "activations/layer1_attention_weight_max": 16.892601013183594,
      "activations/layer1_attention_weight_min": -14.560285568237305,
      "activations/layer20_attention_weight_max": 33.94816207885742,
      "activations/layer20_attention_weight_min": -27.2711181640625,
      "activations/layer21_attention_weight_max": 31.54587745666504,
      "activations/layer21_attention_weight_min": -25.53008460998535,
      "activations/layer22_attention_weight_max": 45.54533004760742,
      "activations/layer22_attention_weight_min": -32.192718505859375,
      "activations/layer23_attention_weight_max": 34.90533447265625,
      "activations/layer23_attention_weight_min": -28.72304916381836,
      "activations/layer2_attention_weight_max": 33.609867095947266,
      "activations/layer2_attention_weight_min": -31.62541961669922,
      "activations/layer3_attention_weight_max": 99.80901336669922,
      "activations/layer3_attention_weight_min": -103.49543762207031,
      "activations/layer4_attention_weight_max": 110.44722747802734,
      "activations/layer4_attention_weight_min": -107.50457763671875,
      "activations/layer5_attention_weight_max": 48.93563461303711,
      "activations/layer5_attention_weight_min": -61.07561492919922,
      "activations/layer6_attention_weight_max": 44.02925491333008,
      "activations/layer6_attention_weight_min": -44.686580657958984,
      "activations/layer7_attention_weight_max": 96.377197265625,
      "activations/layer7_attention_weight_min": -94.37236022949219,
      "activations/layer8_attention_weight_max": 45.16120147705078,
      "activations/layer8_attention_weight_min": -42.684112548828125,
      "activations/layer9_attention_weight_max": 34.70432662963867,
      "activations/layer9_attention_weight_min": -36.41099166870117,
      "epoch": 20.18,
      "learning_rate": 2.0084469696969696e-05,
      "loss": 2.7197,
      "step": 347250
    },
    {
      "activations/layer0_attention_weight_max": 16.46799659729004,
      "activations/layer0_attention_weight_min": -13.563678741455078,
      "activations/layer10_attention_weight_max": 34.16853713989258,
      "activations/layer10_attention_weight_min": -34.619022369384766,
      "activations/layer11_attention_weight_max": 33.233821868896484,
      "activations/layer11_attention_weight_min": -35.78707504272461,
      "activations/layer12_attention_weight_max": 26.08635139465332,
      "activations/layer12_attention_weight_min": -25.983963012695312,
      "activations/layer13_attention_weight_max": 39.41108322143555,
      "activations/layer13_attention_weight_min": -36.03397750854492,
      "activations/layer14_attention_weight_max": 40.7086296081543,
      "activations/layer14_attention_weight_min": -37.064369201660156,
      "activations/layer15_attention_weight_max": 37.339317321777344,
      "activations/layer15_attention_weight_min": -35.36490249633789,
      "activations/layer16_attention_weight_max": 28.213336944580078,
      "activations/layer16_attention_weight_min": -28.04600715637207,
      "activations/layer17_attention_weight_max": 28.366071701049805,
      "activations/layer17_attention_weight_min": -26.661113739013672,
      "activations/layer18_attention_weight_max": 30.716440200805664,
      "activations/layer18_attention_weight_min": -28.18504524230957,
      "activations/layer19_attention_weight_max": 33.89781188964844,
      "activations/layer19_attention_weight_min": -33.94168472290039,
      "activations/layer1_attention_weight_max": 15.417231559753418,
      "activations/layer1_attention_weight_min": -15.818334579467773,
      "activations/layer20_attention_weight_max": 28.47899055480957,
      "activations/layer20_attention_weight_min": -28.338871002197266,
      "activations/layer21_attention_weight_max": 27.051616668701172,
      "activations/layer21_attention_weight_min": -27.88277244567871,
      "activations/layer22_attention_weight_max": 37.771785736083984,
      "activations/layer22_attention_weight_min": -33.862823486328125,
      "activations/layer23_attention_weight_max": 29.672399520874023,
      "activations/layer23_attention_weight_min": -27.947507858276367,
      "activations/layer2_attention_weight_max": 34.99390411376953,
      "activations/layer2_attention_weight_min": -31.772245407104492,
      "activations/layer3_attention_weight_max": 100.00003051757812,
      "activations/layer3_attention_weight_min": -104.89078521728516,
      "activations/layer4_attention_weight_max": 104.57515716552734,
      "activations/layer4_attention_weight_min": -105.00394439697266,
      "activations/layer5_attention_weight_max": 49.48765182495117,
      "activations/layer5_attention_weight_min": -60.59296417236328,
      "activations/layer6_attention_weight_max": 43.19524002075195,
      "activations/layer6_attention_weight_min": -46.70295715332031,
      "activations/layer7_attention_weight_max": 93.5998764038086,
      "activations/layer7_attention_weight_min": -90.39498901367188,
      "activations/layer8_attention_weight_max": 40.32174301147461,
      "activations/layer8_attention_weight_min": -41.3702278137207,
      "activations/layer9_attention_weight_max": 33.315975189208984,
      "activations/layer9_attention_weight_min": -34.04395294189453,
      "epoch": 20.18,
      "learning_rate": 2.00655303030303e-05,
      "loss": 2.6986,
      "step": 347300
    },
    {
      "activations/layer0_attention_weight_max": 15.656394958496094,
      "activations/layer0_attention_weight_min": -14.346506118774414,
      "activations/layer10_attention_weight_max": 35.6901741027832,
      "activations/layer10_attention_weight_min": -36.2215461730957,
      "activations/layer11_attention_weight_max": 35.806434631347656,
      "activations/layer11_attention_weight_min": -36.20697784423828,
      "activations/layer12_attention_weight_max": 29.00043487548828,
      "activations/layer12_attention_weight_min": -26.855985641479492,
      "activations/layer13_attention_weight_max": 41.255393981933594,
      "activations/layer13_attention_weight_min": -37.31884002685547,
      "activations/layer14_attention_weight_max": 44.18543243408203,
      "activations/layer14_attention_weight_min": -38.97505569458008,
      "activations/layer15_attention_weight_max": 39.91529083251953,
      "activations/layer15_attention_weight_min": -39.3985481262207,
      "activations/layer16_attention_weight_max": 28.834684371948242,
      "activations/layer16_attention_weight_min": -28.305788040161133,
      "activations/layer17_attention_weight_max": 29.57931900024414,
      "activations/layer17_attention_weight_min": -27.212623596191406,
      "activations/layer18_attention_weight_max": 31.599567413330078,
      "activations/layer18_attention_weight_min": -26.20151138305664,
      "activations/layer19_attention_weight_max": 34.89736557006836,
      "activations/layer19_attention_weight_min": -33.49651336669922,
      "activations/layer1_attention_weight_max": 15.491436004638672,
      "activations/layer1_attention_weight_min": -14.878701210021973,
      "activations/layer20_attention_weight_max": 29.7120361328125,
      "activations/layer20_attention_weight_min": -27.240575790405273,
      "activations/layer21_attention_weight_max": 33.45636749267578,
      "activations/layer21_attention_weight_min": -25.561309814453125,
      "activations/layer22_attention_weight_max": 41.94449996948242,
      "activations/layer22_attention_weight_min": -32.54891586303711,
      "activations/layer23_attention_weight_max": 33.23323440551758,
      "activations/layer23_attention_weight_min": -27.147815704345703,
      "activations/layer2_attention_weight_max": 32.767452239990234,
      "activations/layer2_attention_weight_min": -32.26844024658203,
      "activations/layer3_attention_weight_max": 103.75743103027344,
      "activations/layer3_attention_weight_min": -102.23454284667969,
      "activations/layer4_attention_weight_max": 107.4137954711914,
      "activations/layer4_attention_weight_min": -109.46875,
      "activations/layer5_attention_weight_max": 52.863685607910156,
      "activations/layer5_attention_weight_min": -65.08226013183594,
      "activations/layer6_attention_weight_max": 48.821109771728516,
      "activations/layer6_attention_weight_min": -46.03983688354492,
      "activations/layer7_attention_weight_max": 106.836669921875,
      "activations/layer7_attention_weight_min": -100.48170471191406,
      "activations/layer8_attention_weight_max": 46.99650192260742,
      "activations/layer8_attention_weight_min": -45.986717224121094,
      "activations/layer9_attention_weight_max": 35.08437728881836,
      "activations/layer9_attention_weight_min": -36.42195510864258,
      "epoch": 20.18,
      "learning_rate": 2.004659090909091e-05,
      "loss": 2.7041,
      "step": 347350
    },
    {
      "activations/layer0_attention_weight_max": 15.823359489440918,
      "activations/layer0_attention_weight_min": -12.54226303100586,
      "activations/layer10_attention_weight_max": 38.0631103515625,
      "activations/layer10_attention_weight_min": -35.18000030517578,
      "activations/layer11_attention_weight_max": 39.1038818359375,
      "activations/layer11_attention_weight_min": -36.18677520751953,
      "activations/layer12_attention_weight_max": 28.154396057128906,
      "activations/layer12_attention_weight_min": -26.876317977905273,
      "activations/layer13_attention_weight_max": 41.61464309692383,
      "activations/layer13_attention_weight_min": -36.637245178222656,
      "activations/layer14_attention_weight_max": 42.84231185913086,
      "activations/layer14_attention_weight_min": -37.600303649902344,
      "activations/layer15_attention_weight_max": 39.921607971191406,
      "activations/layer15_attention_weight_min": -38.65443420410156,
      "activations/layer16_attention_weight_max": 29.73435401916504,
      "activations/layer16_attention_weight_min": -28.570945739746094,
      "activations/layer17_attention_weight_max": 30.77048683166504,
      "activations/layer17_attention_weight_min": -27.83563804626465,
      "activations/layer18_attention_weight_max": 31.47526741027832,
      "activations/layer18_attention_weight_min": -26.462621688842773,
      "activations/layer19_attention_weight_max": 34.218135833740234,
      "activations/layer19_attention_weight_min": -31.194400787353516,
      "activations/layer1_attention_weight_max": 16.535978317260742,
      "activations/layer1_attention_weight_min": -16.125547409057617,
      "activations/layer20_attention_weight_max": 28.757633209228516,
      "activations/layer20_attention_weight_min": -25.1896915435791,
      "activations/layer21_attention_weight_max": 29.308025360107422,
      "activations/layer21_attention_weight_min": -26.91935157775879,
      "activations/layer22_attention_weight_max": 43.02688217163086,
      "activations/layer22_attention_weight_min": -33.87461471557617,
      "activations/layer23_attention_weight_max": 34.36874771118164,
      "activations/layer23_attention_weight_min": -26.644140243530273,
      "activations/layer2_attention_weight_max": 33.44422149658203,
      "activations/layer2_attention_weight_min": -31.936378479003906,
      "activations/layer3_attention_weight_max": 103.30096435546875,
      "activations/layer3_attention_weight_min": -101.68312072753906,
      "activations/layer4_attention_weight_max": 105.57134246826172,
      "activations/layer4_attention_weight_min": -107.01472473144531,
      "activations/layer5_attention_weight_max": 50.539703369140625,
      "activations/layer5_attention_weight_min": -60.07324981689453,
      "activations/layer6_attention_weight_max": 43.79758834838867,
      "activations/layer6_attention_weight_min": -45.12135314941406,
      "activations/layer7_attention_weight_max": 92.7144546508789,
      "activations/layer7_attention_weight_min": -94.59638977050781,
      "activations/layer8_attention_weight_max": 42.260337829589844,
      "activations/layer8_attention_weight_min": -43.143218994140625,
      "activations/layer9_attention_weight_max": 31.663511276245117,
      "activations/layer9_attention_weight_min": -34.17770004272461,
      "epoch": 20.19,
      "learning_rate": 2.002765151515151e-05,
      "loss": 2.7092,
      "step": 347400
    },
    {
      "activations/layer0_attention_weight_max": 16.099220275878906,
      "activations/layer0_attention_weight_min": -13.204748153686523,
      "activations/layer10_attention_weight_max": 35.97522735595703,
      "activations/layer10_attention_weight_min": -37.081321716308594,
      "activations/layer11_attention_weight_max": 35.959774017333984,
      "activations/layer11_attention_weight_min": -34.617713928222656,
      "activations/layer12_attention_weight_max": 29.280839920043945,
      "activations/layer12_attention_weight_min": -27.592390060424805,
      "activations/layer13_attention_weight_max": 44.04977798461914,
      "activations/layer13_attention_weight_min": -35.693241119384766,
      "activations/layer14_attention_weight_max": 44.218631744384766,
      "activations/layer14_attention_weight_min": -37.38801193237305,
      "activations/layer15_attention_weight_max": 42.939456939697266,
      "activations/layer15_attention_weight_min": -36.12790298461914,
      "activations/layer16_attention_weight_max": 28.738061904907227,
      "activations/layer16_attention_weight_min": -29.788314819335938,
      "activations/layer17_attention_weight_max": 31.640567779541016,
      "activations/layer17_attention_weight_min": -27.023765563964844,
      "activations/layer18_attention_weight_max": 32.645267486572266,
      "activations/layer18_attention_weight_min": -25.14716911315918,
      "activations/layer19_attention_weight_max": 32.692108154296875,
      "activations/layer19_attention_weight_min": -33.02136993408203,
      "activations/layer1_attention_weight_max": 14.942336082458496,
      "activations/layer1_attention_weight_min": -14.331724166870117,
      "activations/layer20_attention_weight_max": 29.342023849487305,
      "activations/layer20_attention_weight_min": -26.387861251831055,
      "activations/layer21_attention_weight_max": 29.192535400390625,
      "activations/layer21_attention_weight_min": -25.73250389099121,
      "activations/layer22_attention_weight_max": 41.67448806762695,
      "activations/layer22_attention_weight_min": -32.141536712646484,
      "activations/layer23_attention_weight_max": 34.29728698730469,
      "activations/layer23_attention_weight_min": -25.999052047729492,
      "activations/layer2_attention_weight_max": 33.75080871582031,
      "activations/layer2_attention_weight_min": -33.68507766723633,
      "activations/layer3_attention_weight_max": 104.77046966552734,
      "activations/layer3_attention_weight_min": -103.7747802734375,
      "activations/layer4_attention_weight_max": 111.52812957763672,
      "activations/layer4_attention_weight_min": -112.4287109375,
      "activations/layer5_attention_weight_max": 54.2253532409668,
      "activations/layer5_attention_weight_min": -61.1074333190918,
      "activations/layer6_attention_weight_max": 47.70625305175781,
      "activations/layer6_attention_weight_min": -49.00669860839844,
      "activations/layer7_attention_weight_max": 99.65637969970703,
      "activations/layer7_attention_weight_min": -104.54244232177734,
      "activations/layer8_attention_weight_max": 44.22549819946289,
      "activations/layer8_attention_weight_min": -44.82539749145508,
      "activations/layer9_attention_weight_max": 35.70816421508789,
      "activations/layer9_attention_weight_min": -34.70164108276367,
      "epoch": 20.19,
      "learning_rate": 2.000871212121212e-05,
      "loss": 2.7194,
      "step": 347450
    },
    {
      "activations/layer0_attention_weight_max": 15.980095863342285,
      "activations/layer0_attention_weight_min": -13.113142013549805,
      "activations/layer10_attention_weight_max": 37.00513458251953,
      "activations/layer10_attention_weight_min": -38.04348373413086,
      "activations/layer11_attention_weight_max": 33.8404426574707,
      "activations/layer11_attention_weight_min": -37.011661529541016,
      "activations/layer12_attention_weight_max": 27.3919620513916,
      "activations/layer12_attention_weight_min": -28.099733352661133,
      "activations/layer13_attention_weight_max": 39.1309700012207,
      "activations/layer13_attention_weight_min": -38.97634506225586,
      "activations/layer14_attention_weight_max": 42.1077766418457,
      "activations/layer14_attention_weight_min": -41.218292236328125,
      "activations/layer15_attention_weight_max": 35.732330322265625,
      "activations/layer15_attention_weight_min": -38.34379196166992,
      "activations/layer16_attention_weight_max": 27.544435501098633,
      "activations/layer16_attention_weight_min": -28.032180786132812,
      "activations/layer17_attention_weight_max": 28.95744514465332,
      "activations/layer17_attention_weight_min": -26.43407440185547,
      "activations/layer18_attention_weight_max": 31.02532958984375,
      "activations/layer18_attention_weight_min": -27.684423446655273,
      "activations/layer19_attention_weight_max": 31.486440658569336,
      "activations/layer19_attention_weight_min": -33.701515197753906,
      "activations/layer1_attention_weight_max": 15.572216987609863,
      "activations/layer1_attention_weight_min": -14.059670448303223,
      "activations/layer20_attention_weight_max": 26.244060516357422,
      "activations/layer20_attention_weight_min": -25.0957088470459,
      "activations/layer21_attention_weight_max": 28.65721321105957,
      "activations/layer21_attention_weight_min": -26.670408248901367,
      "activations/layer22_attention_weight_max": 37.68821716308594,
      "activations/layer22_attention_weight_min": -33.95484161376953,
      "activations/layer23_attention_weight_max": 29.24685287475586,
      "activations/layer23_attention_weight_min": -28.147850036621094,
      "activations/layer2_attention_weight_max": 32.283164978027344,
      "activations/layer2_attention_weight_min": -31.671131134033203,
      "activations/layer3_attention_weight_max": 102.66815948486328,
      "activations/layer3_attention_weight_min": -108.88257598876953,
      "activations/layer4_attention_weight_max": 110.510986328125,
      "activations/layer4_attention_weight_min": -113.46070861816406,
      "activations/layer5_attention_weight_max": 54.08031463623047,
      "activations/layer5_attention_weight_min": -63.223655700683594,
      "activations/layer6_attention_weight_max": 44.69807815551758,
      "activations/layer6_attention_weight_min": -47.79814910888672,
      "activations/layer7_attention_weight_max": 99.86162567138672,
      "activations/layer7_attention_weight_min": -104.33663940429688,
      "activations/layer8_attention_weight_max": 44.77164077758789,
      "activations/layer8_attention_weight_min": -46.82146453857422,
      "activations/layer9_attention_weight_max": 33.78862380981445,
      "activations/layer9_attention_weight_min": -37.82490158081055,
      "epoch": 20.19,
      "learning_rate": 1.9989772727272724e-05,
      "loss": 2.7203,
      "step": 347500
    },
    {
      "activations/layer0_attention_weight_max": 16.293750762939453,
      "activations/layer0_attention_weight_min": -12.815831184387207,
      "activations/layer10_attention_weight_max": 34.523719787597656,
      "activations/layer10_attention_weight_min": -34.33827590942383,
      "activations/layer11_attention_weight_max": 35.84101104736328,
      "activations/layer11_attention_weight_min": -34.588653564453125,
      "activations/layer12_attention_weight_max": 31.39556312561035,
      "activations/layer12_attention_weight_min": -28.38886070251465,
      "activations/layer13_attention_weight_max": 48.201255798339844,
      "activations/layer13_attention_weight_min": -37.830474853515625,
      "activations/layer14_attention_weight_max": 47.35319519042969,
      "activations/layer14_attention_weight_min": -38.47877502441406,
      "activations/layer15_attention_weight_max": 45.026580810546875,
      "activations/layer15_attention_weight_min": -37.96327590942383,
      "activations/layer16_attention_weight_max": 32.77798843383789,
      "activations/layer16_attention_weight_min": -28.59989356994629,
      "activations/layer17_attention_weight_max": 31.497053146362305,
      "activations/layer17_attention_weight_min": -30.469900131225586,
      "activations/layer18_attention_weight_max": 32.56406784057617,
      "activations/layer18_attention_weight_min": -28.338163375854492,
      "activations/layer19_attention_weight_max": 34.94125747680664,
      "activations/layer19_attention_weight_min": -32.163875579833984,
      "activations/layer1_attention_weight_max": 15.162363052368164,
      "activations/layer1_attention_weight_min": -15.271370887756348,
      "activations/layer20_attention_weight_max": 32.74365997314453,
      "activations/layer20_attention_weight_min": -27.040891647338867,
      "activations/layer21_attention_weight_max": 28.41020393371582,
      "activations/layer21_attention_weight_min": -26.26523780822754,
      "activations/layer22_attention_weight_max": 44.4769172668457,
      "activations/layer22_attention_weight_min": -34.301658630371094,
      "activations/layer23_attention_weight_max": 33.651065826416016,
      "activations/layer23_attention_weight_min": -28.9996337890625,
      "activations/layer2_attention_weight_max": 31.590526580810547,
      "activations/layer2_attention_weight_min": -31.20276641845703,
      "activations/layer3_attention_weight_max": 100.9824447631836,
      "activations/layer3_attention_weight_min": -100.55065155029297,
      "activations/layer4_attention_weight_max": 109.63233947753906,
      "activations/layer4_attention_weight_min": -111.25008392333984,
      "activations/layer5_attention_weight_max": 50.13550567626953,
      "activations/layer5_attention_weight_min": -62.57640075683594,
      "activations/layer6_attention_weight_max": 45.11539840698242,
      "activations/layer6_attention_weight_min": -47.9119758605957,
      "activations/layer7_attention_weight_max": 94.38866424560547,
      "activations/layer7_attention_weight_min": -97.11641693115234,
      "activations/layer8_attention_weight_max": 41.97944259643555,
      "activations/layer8_attention_weight_min": -43.283470153808594,
      "activations/layer9_attention_weight_max": 31.608495712280273,
      "activations/layer9_attention_weight_min": -33.675662994384766,
      "epoch": 20.19,
      "learning_rate": 1.9970833333333332e-05,
      "loss": 2.7135,
      "step": 347550
    },
    {
      "activations/layer0_attention_weight_max": 16.707225799560547,
      "activations/layer0_attention_weight_min": -12.718437194824219,
      "activations/layer10_attention_weight_max": 34.17771911621094,
      "activations/layer10_attention_weight_min": -34.17081832885742,
      "activations/layer11_attention_weight_max": 35.7080078125,
      "activations/layer11_attention_weight_min": -33.111446380615234,
      "activations/layer12_attention_weight_max": 29.201120376586914,
      "activations/layer12_attention_weight_min": -27.87522315979004,
      "activations/layer13_attention_weight_max": 40.46782302856445,
      "activations/layer13_attention_weight_min": -38.38225555419922,
      "activations/layer14_attention_weight_max": 45.92118835449219,
      "activations/layer14_attention_weight_min": -41.054447174072266,
      "activations/layer15_attention_weight_max": 41.51926040649414,
      "activations/layer15_attention_weight_min": -37.848995208740234,
      "activations/layer16_attention_weight_max": 32.61139678955078,
      "activations/layer16_attention_weight_min": -29.490921020507812,
      "activations/layer17_attention_weight_max": 34.53276062011719,
      "activations/layer17_attention_weight_min": -26.673948287963867,
      "activations/layer18_attention_weight_max": 37.75910568237305,
      "activations/layer18_attention_weight_min": -27.305763244628906,
      "activations/layer19_attention_weight_max": 40.57027816772461,
      "activations/layer19_attention_weight_min": -37.756072998046875,
      "activations/layer1_attention_weight_max": 15.02000904083252,
      "activations/layer1_attention_weight_min": -13.979615211486816,
      "activations/layer20_attention_weight_max": 33.5427360534668,
      "activations/layer20_attention_weight_min": -28.832427978515625,
      "activations/layer21_attention_weight_max": 31.889055252075195,
      "activations/layer21_attention_weight_min": -27.647476196289062,
      "activations/layer22_attention_weight_max": 50.38438415527344,
      "activations/layer22_attention_weight_min": -37.57606887817383,
      "activations/layer23_attention_weight_max": 36.83271789550781,
      "activations/layer23_attention_weight_min": -28.18707275390625,
      "activations/layer2_attention_weight_max": 32.80712890625,
      "activations/layer2_attention_weight_min": -32.97914123535156,
      "activations/layer3_attention_weight_max": 103.14776611328125,
      "activations/layer3_attention_weight_min": -103.77684020996094,
      "activations/layer4_attention_weight_max": 105.36698150634766,
      "activations/layer4_attention_weight_min": -108.53729248046875,
      "activations/layer5_attention_weight_max": 50.02553939819336,
      "activations/layer5_attention_weight_min": -61.58271789550781,
      "activations/layer6_attention_weight_max": 44.551631927490234,
      "activations/layer6_attention_weight_min": -46.80406951904297,
      "activations/layer7_attention_weight_max": 94.53466033935547,
      "activations/layer7_attention_weight_min": -98.55549621582031,
      "activations/layer8_attention_weight_max": 41.09598159790039,
      "activations/layer8_attention_weight_min": -43.75934982299805,
      "activations/layer9_attention_weight_max": 31.817615509033203,
      "activations/layer9_attention_weight_min": -33.549522399902344,
      "epoch": 20.2,
      "learning_rate": 1.9951893939393937e-05,
      "loss": 2.7111,
      "step": 347600
    },
    {
      "activations/layer0_attention_weight_max": 15.915462493896484,
      "activations/layer0_attention_weight_min": -13.224356651306152,
      "activations/layer10_attention_weight_max": 40.4826774597168,
      "activations/layer10_attention_weight_min": -37.47872543334961,
      "activations/layer11_attention_weight_max": 40.27975845336914,
      "activations/layer11_attention_weight_min": -38.901268005371094,
      "activations/layer12_attention_weight_max": 30.162128448486328,
      "activations/layer12_attention_weight_min": -27.38457489013672,
      "activations/layer13_attention_weight_max": 48.29564666748047,
      "activations/layer13_attention_weight_min": -39.79829406738281,
      "activations/layer14_attention_weight_max": 46.976470947265625,
      "activations/layer14_attention_weight_min": -38.86431121826172,
      "activations/layer15_attention_weight_max": 45.85213851928711,
      "activations/layer15_attention_weight_min": -36.78329849243164,
      "activations/layer16_attention_weight_max": 30.00095558166504,
      "activations/layer16_attention_weight_min": -27.501726150512695,
      "activations/layer17_attention_weight_max": 33.07149124145508,
      "activations/layer17_attention_weight_min": -25.82012367248535,
      "activations/layer18_attention_weight_max": 34.62578201293945,
      "activations/layer18_attention_weight_min": -26.242692947387695,
      "activations/layer19_attention_weight_max": 37.39083480834961,
      "activations/layer19_attention_weight_min": -32.016151428222656,
      "activations/layer1_attention_weight_max": 15.063797950744629,
      "activations/layer1_attention_weight_min": -14.63149356842041,
      "activations/layer20_attention_weight_max": 31.604766845703125,
      "activations/layer20_attention_weight_min": -25.441160202026367,
      "activations/layer21_attention_weight_max": 29.45809555053711,
      "activations/layer21_attention_weight_min": -24.96076011657715,
      "activations/layer22_attention_weight_max": 42.33357238769531,
      "activations/layer22_attention_weight_min": -31.926652908325195,
      "activations/layer23_attention_weight_max": 36.466712951660156,
      "activations/layer23_attention_weight_min": -25.180553436279297,
      "activations/layer2_attention_weight_max": 32.58221435546875,
      "activations/layer2_attention_weight_min": -31.90261459350586,
      "activations/layer3_attention_weight_max": 103.2867431640625,
      "activations/layer3_attention_weight_min": -100.88887023925781,
      "activations/layer4_attention_weight_max": 113.20184326171875,
      "activations/layer4_attention_weight_min": -106.46443176269531,
      "activations/layer5_attention_weight_max": 53.67643356323242,
      "activations/layer5_attention_weight_min": -61.33901596069336,
      "activations/layer6_attention_weight_max": 46.64246368408203,
      "activations/layer6_attention_weight_min": -47.615657806396484,
      "activations/layer7_attention_weight_max": 111.77798461914062,
      "activations/layer7_attention_weight_min": -97.61145782470703,
      "activations/layer8_attention_weight_max": 50.15641403198242,
      "activations/layer8_attention_weight_min": -46.61231994628906,
      "activations/layer9_attention_weight_max": 38.4812126159668,
      "activations/layer9_attention_weight_min": -37.06044006347656,
      "epoch": 20.2,
      "learning_rate": 1.9932954545454545e-05,
      "loss": 2.7122,
      "step": 347650
    },
    {
      "activations/layer0_attention_weight_max": 16.55370330810547,
      "activations/layer0_attention_weight_min": -12.473897933959961,
      "activations/layer10_attention_weight_max": 37.397438049316406,
      "activations/layer10_attention_weight_min": -36.188148498535156,
      "activations/layer11_attention_weight_max": 37.120460510253906,
      "activations/layer11_attention_weight_min": -35.81059265136719,
      "activations/layer12_attention_weight_max": 29.37952995300293,
      "activations/layer12_attention_weight_min": -28.095619201660156,
      "activations/layer13_attention_weight_max": 41.71775817871094,
      "activations/layer13_attention_weight_min": -38.73682403564453,
      "activations/layer14_attention_weight_max": 47.71712875366211,
      "activations/layer14_attention_weight_min": -40.79664611816406,
      "activations/layer15_attention_weight_max": 43.305233001708984,
      "activations/layer15_attention_weight_min": -38.144771575927734,
      "activations/layer16_attention_weight_max": 31.51637840270996,
      "activations/layer16_attention_weight_min": -29.726238250732422,
      "activations/layer17_attention_weight_max": 34.276100158691406,
      "activations/layer17_attention_weight_min": -27.777978897094727,
      "activations/layer18_attention_weight_max": 33.792724609375,
      "activations/layer18_attention_weight_min": -28.4859561920166,
      "activations/layer19_attention_weight_max": 33.49033737182617,
      "activations/layer19_attention_weight_min": -33.33457565307617,
      "activations/layer1_attention_weight_max": 15.052927017211914,
      "activations/layer1_attention_weight_min": -14.894669532775879,
      "activations/layer20_attention_weight_max": 27.859466552734375,
      "activations/layer20_attention_weight_min": -26.080238342285156,
      "activations/layer21_attention_weight_max": 28.74017906188965,
      "activations/layer21_attention_weight_min": -24.59765625,
      "activations/layer22_attention_weight_max": 40.57658767700195,
      "activations/layer22_attention_weight_min": -34.19996643066406,
      "activations/layer23_attention_weight_max": 31.35688018798828,
      "activations/layer23_attention_weight_min": -26.684524536132812,
      "activations/layer2_attention_weight_max": 32.96424865722656,
      "activations/layer2_attention_weight_min": -32.395362854003906,
      "activations/layer3_attention_weight_max": 105.4870376586914,
      "activations/layer3_attention_weight_min": -103.77806854248047,
      "activations/layer4_attention_weight_max": 111.22346496582031,
      "activations/layer4_attention_weight_min": -109.4795913696289,
      "activations/layer5_attention_weight_max": 50.24264144897461,
      "activations/layer5_attention_weight_min": -60.443572998046875,
      "activations/layer6_attention_weight_max": 46.68877410888672,
      "activations/layer6_attention_weight_min": -49.22114181518555,
      "activations/layer7_attention_weight_max": 100.94493103027344,
      "activations/layer7_attention_weight_min": -104.215576171875,
      "activations/layer8_attention_weight_max": 45.605445861816406,
      "activations/layer8_attention_weight_min": -47.7121696472168,
      "activations/layer9_attention_weight_max": 34.51182174682617,
      "activations/layer9_attention_weight_min": -36.060367584228516,
      "epoch": 20.2,
      "learning_rate": 1.991401515151515e-05,
      "loss": 2.7118,
      "step": 347700
    },
    {
      "activations/layer0_attention_weight_max": 17.339725494384766,
      "activations/layer0_attention_weight_min": -13.57567024230957,
      "activations/layer10_attention_weight_max": 45.348411560058594,
      "activations/layer10_attention_weight_min": -38.721595764160156,
      "activations/layer11_attention_weight_max": 43.77009582519531,
      "activations/layer11_attention_weight_min": -40.984580993652344,
      "activations/layer12_attention_weight_max": 27.232515335083008,
      "activations/layer12_attention_weight_min": -29.10074234008789,
      "activations/layer13_attention_weight_max": 43.99639129638672,
      "activations/layer13_attention_weight_min": -39.87322235107422,
      "activations/layer14_attention_weight_max": 49.432762145996094,
      "activations/layer14_attention_weight_min": -46.437294006347656,
      "activations/layer15_attention_weight_max": 42.920654296875,
      "activations/layer15_attention_weight_min": -42.152217864990234,
      "activations/layer16_attention_weight_max": 33.51597213745117,
      "activations/layer16_attention_weight_min": -29.225475311279297,
      "activations/layer17_attention_weight_max": 32.257537841796875,
      "activations/layer17_attention_weight_min": -26.79669189453125,
      "activations/layer18_attention_weight_max": 33.84614181518555,
      "activations/layer18_attention_weight_min": -30.189613342285156,
      "activations/layer19_attention_weight_max": 39.82102584838867,
      "activations/layer19_attention_weight_min": -35.315670013427734,
      "activations/layer1_attention_weight_max": 16.20473861694336,
      "activations/layer1_attention_weight_min": -14.611885070800781,
      "activations/layer20_attention_weight_max": 31.716325759887695,
      "activations/layer20_attention_weight_min": -26.00018310546875,
      "activations/layer21_attention_weight_max": 32.0489501953125,
      "activations/layer21_attention_weight_min": -26.684696197509766,
      "activations/layer22_attention_weight_max": 45.97328186035156,
      "activations/layer22_attention_weight_min": -36.30220031738281,
      "activations/layer23_attention_weight_max": 35.01144027709961,
      "activations/layer23_attention_weight_min": -27.178714752197266,
      "activations/layer2_attention_weight_max": 32.73530578613281,
      "activations/layer2_attention_weight_min": -30.959300994873047,
      "activations/layer3_attention_weight_max": 101.73495483398438,
      "activations/layer3_attention_weight_min": -99.96337127685547,
      "activations/layer4_attention_weight_max": 108.97814178466797,
      "activations/layer4_attention_weight_min": -109.62349700927734,
      "activations/layer5_attention_weight_max": 48.59233856201172,
      "activations/layer5_attention_weight_min": -62.13377380371094,
      "activations/layer6_attention_weight_max": 44.86622619628906,
      "activations/layer6_attention_weight_min": -49.01087188720703,
      "activations/layer7_attention_weight_max": 99.92493438720703,
      "activations/layer7_attention_weight_min": -100.15392303466797,
      "activations/layer8_attention_weight_max": 48.35898971557617,
      "activations/layer8_attention_weight_min": -46.07861328125,
      "activations/layer9_attention_weight_max": 36.089969635009766,
      "activations/layer9_attention_weight_min": -37.186946868896484,
      "epoch": 20.21,
      "learning_rate": 1.9895075757575755e-05,
      "loss": 2.6981,
      "step": 347750
    },
    {
      "activations/layer0_attention_weight_max": 16.641204833984375,
      "activations/layer0_attention_weight_min": -13.319378852844238,
      "activations/layer10_attention_weight_max": 46.31991195678711,
      "activations/layer10_attention_weight_min": -45.47856521606445,
      "activations/layer11_attention_weight_max": 48.177772521972656,
      "activations/layer11_attention_weight_min": -47.045284271240234,
      "activations/layer12_attention_weight_max": 29.027250289916992,
      "activations/layer12_attention_weight_min": -30.584671020507812,
      "activations/layer13_attention_weight_max": 41.52674102783203,
      "activations/layer13_attention_weight_min": -40.03044509887695,
      "activations/layer14_attention_weight_max": 47.96282958984375,
      "activations/layer14_attention_weight_min": -44.84957504272461,
      "activations/layer15_attention_weight_max": 42.890071868896484,
      "activations/layer15_attention_weight_min": -41.71998596191406,
      "activations/layer16_attention_weight_max": 31.03141975402832,
      "activations/layer16_attention_weight_min": -28.57024574279785,
      "activations/layer17_attention_weight_max": 32.853004455566406,
      "activations/layer17_attention_weight_min": -26.078866958618164,
      "activations/layer18_attention_weight_max": 32.60139083862305,
      "activations/layer18_attention_weight_min": -27.48430824279785,
      "activations/layer19_attention_weight_max": 36.96672058105469,
      "activations/layer19_attention_weight_min": -32.52078628540039,
      "activations/layer1_attention_weight_max": 16.896244049072266,
      "activations/layer1_attention_weight_min": -15.82024097442627,
      "activations/layer20_attention_weight_max": 32.07286071777344,
      "activations/layer20_attention_weight_min": -24.437543869018555,
      "activations/layer21_attention_weight_max": 31.683584213256836,
      "activations/layer21_attention_weight_min": -24.227920532226562,
      "activations/layer22_attention_weight_max": 44.718936920166016,
      "activations/layer22_attention_weight_min": -32.048675537109375,
      "activations/layer23_attention_weight_max": 36.53616714477539,
      "activations/layer23_attention_weight_min": -24.736438751220703,
      "activations/layer2_attention_weight_max": 32.157257080078125,
      "activations/layer2_attention_weight_min": -30.733951568603516,
      "activations/layer3_attention_weight_max": 97.9593276977539,
      "activations/layer3_attention_weight_min": -97.27141571044922,
      "activations/layer4_attention_weight_max": 104.7342758178711,
      "activations/layer4_attention_weight_min": -104.15815734863281,
      "activations/layer5_attention_weight_max": 51.83103942871094,
      "activations/layer5_attention_weight_min": -58.5852165222168,
      "activations/layer6_attention_weight_max": 42.015052795410156,
      "activations/layer6_attention_weight_min": -44.50636672973633,
      "activations/layer7_attention_weight_max": 112.57100677490234,
      "activations/layer7_attention_weight_min": -90.4892578125,
      "activations/layer8_attention_weight_max": 50.93087387084961,
      "activations/layer8_attention_weight_min": -47.092132568359375,
      "activations/layer9_attention_weight_max": 37.426910400390625,
      "activations/layer9_attention_weight_min": -40.87653732299805,
      "epoch": 20.21,
      "learning_rate": 1.9876136363636363e-05,
      "loss": 2.7078,
      "step": 347800
    },
    {
      "activations/layer0_attention_weight_max": 16.099071502685547,
      "activations/layer0_attention_weight_min": -13.387456893920898,
      "activations/layer10_attention_weight_max": 32.56145095825195,
      "activations/layer10_attention_weight_min": -33.52511978149414,
      "activations/layer11_attention_weight_max": 34.81991958618164,
      "activations/layer11_attention_weight_min": -32.31792449951172,
      "activations/layer12_attention_weight_max": 25.384138107299805,
      "activations/layer12_attention_weight_min": -26.18553924560547,
      "activations/layer13_attention_weight_max": 37.46432113647461,
      "activations/layer13_attention_weight_min": -36.50957489013672,
      "activations/layer14_attention_weight_max": 41.301326751708984,
      "activations/layer14_attention_weight_min": -38.98942947387695,
      "activations/layer15_attention_weight_max": 36.88810348510742,
      "activations/layer15_attention_weight_min": -34.63499450683594,
      "activations/layer16_attention_weight_max": 28.56471824645996,
      "activations/layer16_attention_weight_min": -27.643068313598633,
      "activations/layer17_attention_weight_max": 29.41452980041504,
      "activations/layer17_attention_weight_min": -26.26621437072754,
      "activations/layer18_attention_weight_max": 31.64801025390625,
      "activations/layer18_attention_weight_min": -26.966064453125,
      "activations/layer19_attention_weight_max": 34.04887771606445,
      "activations/layer19_attention_weight_min": -34.43437194824219,
      "activations/layer1_attention_weight_max": 15.671732902526855,
      "activations/layer1_attention_weight_min": -14.974691390991211,
      "activations/layer20_attention_weight_max": 30.13849449157715,
      "activations/layer20_attention_weight_min": -27.0819034576416,
      "activations/layer21_attention_weight_max": 31.01210594177246,
      "activations/layer21_attention_weight_min": -25.852500915527344,
      "activations/layer22_attention_weight_max": 42.87831115722656,
      "activations/layer22_attention_weight_min": -34.398414611816406,
      "activations/layer23_attention_weight_max": 31.045989990234375,
      "activations/layer23_attention_weight_min": -26.455352783203125,
      "activations/layer2_attention_weight_max": 30.61572265625,
      "activations/layer2_attention_weight_min": -32.28975296020508,
      "activations/layer3_attention_weight_max": 97.94940185546875,
      "activations/layer3_attention_weight_min": -103.14728546142578,
      "activations/layer4_attention_weight_max": 102.6052474975586,
      "activations/layer4_attention_weight_min": -104.7713394165039,
      "activations/layer5_attention_weight_max": 50.036190032958984,
      "activations/layer5_attention_weight_min": -62.75800323486328,
      "activations/layer6_attention_weight_max": 41.32407760620117,
      "activations/layer6_attention_weight_min": -45.53294372558594,
      "activations/layer7_attention_weight_max": 94.03093719482422,
      "activations/layer7_attention_weight_min": -96.86561584472656,
      "activations/layer8_attention_weight_max": 39.42136001586914,
      "activations/layer8_attention_weight_min": -43.4127197265625,
      "activations/layer9_attention_weight_max": 31.778945922851562,
      "activations/layer9_attention_weight_min": -34.2527961730957,
      "epoch": 20.21,
      "learning_rate": 1.9857196969696968e-05,
      "loss": 2.7216,
      "step": 347850
    },
    {
      "activations/layer0_attention_weight_max": 15.962814331054688,
      "activations/layer0_attention_weight_min": -13.773228645324707,
      "activations/layer10_attention_weight_max": 36.427650451660156,
      "activations/layer10_attention_weight_min": -37.393035888671875,
      "activations/layer11_attention_weight_max": 36.42640686035156,
      "activations/layer11_attention_weight_min": -34.71473693847656,
      "activations/layer12_attention_weight_max": 28.167009353637695,
      "activations/layer12_attention_weight_min": -28.113712310791016,
      "activations/layer13_attention_weight_max": 40.914398193359375,
      "activations/layer13_attention_weight_min": -38.74654006958008,
      "activations/layer14_attention_weight_max": 41.55462646484375,
      "activations/layer14_attention_weight_min": -40.07401657104492,
      "activations/layer15_attention_weight_max": 41.17789840698242,
      "activations/layer15_attention_weight_min": -38.03789138793945,
      "activations/layer16_attention_weight_max": 29.58802032470703,
      "activations/layer16_attention_weight_min": -28.92502212524414,
      "activations/layer17_attention_weight_max": 31.310171127319336,
      "activations/layer17_attention_weight_min": -26.829435348510742,
      "activations/layer18_attention_weight_max": 34.891841888427734,
      "activations/layer18_attention_weight_min": -27.485876083374023,
      "activations/layer19_attention_weight_max": 34.23075485229492,
      "activations/layer19_attention_weight_min": -33.51403045654297,
      "activations/layer1_attention_weight_max": 15.694746017456055,
      "activations/layer1_attention_weight_min": -15.395999908447266,
      "activations/layer20_attention_weight_max": 28.745880126953125,
      "activations/layer20_attention_weight_min": -26.590909957885742,
      "activations/layer21_attention_weight_max": 28.969676971435547,
      "activations/layer21_attention_weight_min": -26.282882690429688,
      "activations/layer22_attention_weight_max": 39.89065933227539,
      "activations/layer22_attention_weight_min": -35.33098220825195,
      "activations/layer23_attention_weight_max": 35.23942565917969,
      "activations/layer23_attention_weight_min": -26.23839569091797,
      "activations/layer2_attention_weight_max": 32.43634033203125,
      "activations/layer2_attention_weight_min": -32.17681884765625,
      "activations/layer3_attention_weight_max": 101.1133804321289,
      "activations/layer3_attention_weight_min": -101.44702911376953,
      "activations/layer4_attention_weight_max": 103.8708724975586,
      "activations/layer4_attention_weight_min": -113.61235809326172,
      "activations/layer5_attention_weight_max": 50.75852584838867,
      "activations/layer5_attention_weight_min": -63.411781311035156,
      "activations/layer6_attention_weight_max": 46.460025787353516,
      "activations/layer6_attention_weight_min": -50.29523468017578,
      "activations/layer7_attention_weight_max": 98.3788070678711,
      "activations/layer7_attention_weight_min": -106.74308013916016,
      "activations/layer8_attention_weight_max": 42.95689010620117,
      "activations/layer8_attention_weight_min": -46.53790283203125,
      "activations/layer9_attention_weight_max": 33.62238693237305,
      "activations/layer9_attention_weight_min": -39.0135498046875,
      "epoch": 20.21,
      "learning_rate": 1.9838257575757576e-05,
      "loss": 2.7059,
      "step": 347900
    },
    {
      "activations/layer0_attention_weight_max": 16.460525512695312,
      "activations/layer0_attention_weight_min": -13.266562461853027,
      "activations/layer10_attention_weight_max": 34.9832763671875,
      "activations/layer10_attention_weight_min": -35.198890686035156,
      "activations/layer11_attention_weight_max": 35.5527229309082,
      "activations/layer11_attention_weight_min": -35.802345275878906,
      "activations/layer12_attention_weight_max": 28.4757137298584,
      "activations/layer12_attention_weight_min": -28.817672729492188,
      "activations/layer13_attention_weight_max": 40.08364486694336,
      "activations/layer13_attention_weight_min": -37.532981872558594,
      "activations/layer14_attention_weight_max": 42.06302261352539,
      "activations/layer14_attention_weight_min": -37.86100769042969,
      "activations/layer15_attention_weight_max": 43.21982192993164,
      "activations/layer15_attention_weight_min": -37.57018280029297,
      "activations/layer16_attention_weight_max": 32.82289505004883,
      "activations/layer16_attention_weight_min": -29.35552978515625,
      "activations/layer17_attention_weight_max": 32.70595169067383,
      "activations/layer17_attention_weight_min": -27.550199508666992,
      "activations/layer18_attention_weight_max": 33.21414566040039,
      "activations/layer18_attention_weight_min": -25.297321319580078,
      "activations/layer19_attention_weight_max": 37.98280715942383,
      "activations/layer19_attention_weight_min": -30.430063247680664,
      "activations/layer1_attention_weight_max": 15.742496490478516,
      "activations/layer1_attention_weight_min": -15.753440856933594,
      "activations/layer20_attention_weight_max": 29.606300354003906,
      "activations/layer20_attention_weight_min": -24.35051727294922,
      "activations/layer21_attention_weight_max": 29.268239974975586,
      "activations/layer21_attention_weight_min": -23.924694061279297,
      "activations/layer22_attention_weight_max": 41.36279296875,
      "activations/layer22_attention_weight_min": -30.540342330932617,
      "activations/layer23_attention_weight_max": 32.20664596557617,
      "activations/layer23_attention_weight_min": -25.71950912475586,
      "activations/layer2_attention_weight_max": 33.089359283447266,
      "activations/layer2_attention_weight_min": -31.789342880249023,
      "activations/layer3_attention_weight_max": 101.17354583740234,
      "activations/layer3_attention_weight_min": -102.00960540771484,
      "activations/layer4_attention_weight_max": 107.14796447753906,
      "activations/layer4_attention_weight_min": -109.69770812988281,
      "activations/layer5_attention_weight_max": 49.88360595703125,
      "activations/layer5_attention_weight_min": -62.95644760131836,
      "activations/layer6_attention_weight_max": 46.09450912475586,
      "activations/layer6_attention_weight_min": -47.56222152709961,
      "activations/layer7_attention_weight_max": 90.57776641845703,
      "activations/layer7_attention_weight_min": -106.83597564697266,
      "activations/layer8_attention_weight_max": 43.84153366088867,
      "activations/layer8_attention_weight_min": -44.83623123168945,
      "activations/layer9_attention_weight_max": 33.381954193115234,
      "activations/layer9_attention_weight_min": -35.59328842163086,
      "epoch": 20.22,
      "learning_rate": 1.9819318181818178e-05,
      "loss": 2.7185,
      "step": 347950
    },
    {
      "activations/layer0_attention_weight_max": 15.658556938171387,
      "activations/layer0_attention_weight_min": -13.901857376098633,
      "activations/layer10_attention_weight_max": 39.98286819458008,
      "activations/layer10_attention_weight_min": -37.304927825927734,
      "activations/layer11_attention_weight_max": 38.23168182373047,
      "activations/layer11_attention_weight_min": -37.99610137939453,
      "activations/layer12_attention_weight_max": 28.752412796020508,
      "activations/layer12_attention_weight_min": -27.117324829101562,
      "activations/layer13_attention_weight_max": 42.04115295410156,
      "activations/layer13_attention_weight_min": -40.12405014038086,
      "activations/layer14_attention_weight_max": 45.66444396972656,
      "activations/layer14_attention_weight_min": -41.43132019042969,
      "activations/layer15_attention_weight_max": 41.81704330444336,
      "activations/layer15_attention_weight_min": -36.85868835449219,
      "activations/layer16_attention_weight_max": 32.58618927001953,
      "activations/layer16_attention_weight_min": -28.52452278137207,
      "activations/layer17_attention_weight_max": 30.07235336303711,
      "activations/layer17_attention_weight_min": -25.534902572631836,
      "activations/layer18_attention_weight_max": 35.206573486328125,
      "activations/layer18_attention_weight_min": -23.88715934753418,
      "activations/layer19_attention_weight_max": 36.07491683959961,
      "activations/layer19_attention_weight_min": -30.186437606811523,
      "activations/layer1_attention_weight_max": 15.186866760253906,
      "activations/layer1_attention_weight_min": -14.816994667053223,
      "activations/layer20_attention_weight_max": 28.981969833374023,
      "activations/layer20_attention_weight_min": -24.378538131713867,
      "activations/layer21_attention_weight_max": 29.18745994567871,
      "activations/layer21_attention_weight_min": -22.83418846130371,
      "activations/layer22_attention_weight_max": 41.378604888916016,
      "activations/layer22_attention_weight_min": -30.60048484802246,
      "activations/layer23_attention_weight_max": 31.762128829956055,
      "activations/layer23_attention_weight_min": -24.2896728515625,
      "activations/layer2_attention_weight_max": 33.25136947631836,
      "activations/layer2_attention_weight_min": -33.85559844970703,
      "activations/layer3_attention_weight_max": 101.95413208007812,
      "activations/layer3_attention_weight_min": -105.03707122802734,
      "activations/layer4_attention_weight_max": 104.66796875,
      "activations/layer4_attention_weight_min": -108.63961029052734,
      "activations/layer5_attention_weight_max": 50.209693908691406,
      "activations/layer5_attention_weight_min": -60.07358932495117,
      "activations/layer6_attention_weight_max": 43.335384368896484,
      "activations/layer6_attention_weight_min": -46.47792434692383,
      "activations/layer7_attention_weight_max": 93.39032745361328,
      "activations/layer7_attention_weight_min": -103.42117309570312,
      "activations/layer8_attention_weight_max": 44.093814849853516,
      "activations/layer8_attention_weight_min": -46.345703125,
      "activations/layer9_attention_weight_max": 32.5520133972168,
      "activations/layer9_attention_weight_min": -38.04006576538086,
      "epoch": 20.22,
      "learning_rate": 1.9800378787878786e-05,
      "loss": 2.7113,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.5255,
      "eval_samples_per_second": 503.667,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.5255,
      "eval_openwebtext_samples_per_second": 503.667,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9553,
      "eval_wikitext_samples_per_second": 233.208,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_lambada_loss": 2.359375,
      "eval_lambada_ppl": 10.584334175890335,
      "eval_lambada_runtime": 9.5449,
      "eval_lambada_samples_per_second": 510.113,
      "step": 348000
    },
    {
      "activations/layer0_attention_weight_max": 16.165740966796875,
      "activations/layer0_attention_weight_min": -16.591686248779297,
      "activations/layer10_attention_weight_max": 35.94649124145508,
      "activations/layer10_attention_weight_min": -34.51955795288086,
      "activations/layer11_attention_weight_max": 35.09447479248047,
      "activations/layer11_attention_weight_min": -35.85274124145508,
      "activations/layer12_attention_weight_max": 26.23407554626465,
      "activations/layer12_attention_weight_min": -27.577713012695312,
      "activations/layer13_attention_weight_max": 38.91654586791992,
      "activations/layer13_attention_weight_min": -34.994117736816406,
      "activations/layer14_attention_weight_max": 41.56184005737305,
      "activations/layer14_attention_weight_min": -38.81912612915039,
      "activations/layer15_attention_weight_max": 39.98231887817383,
      "activations/layer15_attention_weight_min": -37.039268493652344,
      "activations/layer16_attention_weight_max": 30.60561180114746,
      "activations/layer16_attention_weight_min": -29.124692916870117,
      "activations/layer17_attention_weight_max": 29.52885627746582,
      "activations/layer17_attention_weight_min": -26.68844985961914,
      "activations/layer18_attention_weight_max": 34.41156005859375,
      "activations/layer18_attention_weight_min": -27.78533935546875,
      "activations/layer19_attention_weight_max": 35.125606536865234,
      "activations/layer19_attention_weight_min": -36.01359558105469,
      "activations/layer1_attention_weight_max": 16.082727432250977,
      "activations/layer1_attention_weight_min": -14.982110023498535,
      "activations/layer20_attention_weight_max": 27.181655883789062,
      "activations/layer20_attention_weight_min": -26.433088302612305,
      "activations/layer21_attention_weight_max": 29.067136764526367,
      "activations/layer21_attention_weight_min": -25.67706298828125,
      "activations/layer22_attention_weight_max": 44.05351257324219,
      "activations/layer22_attention_weight_min": -34.96186828613281,
      "activations/layer23_attention_weight_max": 34.59577178955078,
      "activations/layer23_attention_weight_min": -25.86376190185547,
      "activations/layer2_attention_weight_max": 32.19154739379883,
      "activations/layer2_attention_weight_min": -32.917823791503906,
      "activations/layer3_attention_weight_max": 102.56890106201172,
      "activations/layer3_attention_weight_min": -106.01822662353516,
      "activations/layer4_attention_weight_max": 107.0741195678711,
      "activations/layer4_attention_weight_min": -112.4111557006836,
      "activations/layer5_attention_weight_max": 50.25657653808594,
      "activations/layer5_attention_weight_min": -61.824127197265625,
      "activations/layer6_attention_weight_max": 44.06771469116211,
      "activations/layer6_attention_weight_min": -48.640743255615234,
      "activations/layer7_attention_weight_max": 94.28880310058594,
      "activations/layer7_attention_weight_min": -101.89336395263672,
      "activations/layer8_attention_weight_max": 43.11206817626953,
      "activations/layer8_attention_weight_min": -45.06562805175781,
      "activations/layer9_attention_weight_max": 34.78754425048828,
      "activations/layer9_attention_weight_min": -33.920753479003906,
      "epoch": 20.22,
      "learning_rate": 1.978143939393939e-05,
      "loss": 2.7258,
      "step": 348050
    },
    {
      "activations/layer0_attention_weight_max": 16.529752731323242,
      "activations/layer0_attention_weight_min": -12.662728309631348,
      "activations/layer10_attention_weight_max": 35.38102340698242,
      "activations/layer10_attention_weight_min": -37.464900970458984,
      "activations/layer11_attention_weight_max": 36.49937057495117,
      "activations/layer11_attention_weight_min": -36.61078643798828,
      "activations/layer12_attention_weight_max": 27.85276222229004,
      "activations/layer12_attention_weight_min": -27.841135025024414,
      "activations/layer13_attention_weight_max": 44.715457916259766,
      "activations/layer13_attention_weight_min": -39.984954833984375,
      "activations/layer14_attention_weight_max": 50.3243408203125,
      "activations/layer14_attention_weight_min": -41.147132873535156,
      "activations/layer15_attention_weight_max": 46.29342269897461,
      "activations/layer15_attention_weight_min": -37.98954391479492,
      "activations/layer16_attention_weight_max": 35.69310760498047,
      "activations/layer16_attention_weight_min": -28.79660987854004,
      "activations/layer17_attention_weight_max": 34.235755920410156,
      "activations/layer17_attention_weight_min": -27.259416580200195,
      "activations/layer18_attention_weight_max": 35.62615203857422,
      "activations/layer18_attention_weight_min": -27.183748245239258,
      "activations/layer19_attention_weight_max": 40.04762268066406,
      "activations/layer19_attention_weight_min": -36.57087707519531,
      "activations/layer1_attention_weight_max": 15.082714080810547,
      "activations/layer1_attention_weight_min": -14.687291145324707,
      "activations/layer20_attention_weight_max": 34.62782287597656,
      "activations/layer20_attention_weight_min": -26.92955780029297,
      "activations/layer21_attention_weight_max": 33.13263702392578,
      "activations/layer21_attention_weight_min": -25.994609832763672,
      "activations/layer22_attention_weight_max": 45.27251052856445,
      "activations/layer22_attention_weight_min": -35.46851348876953,
      "activations/layer23_attention_weight_max": 35.53687286376953,
      "activations/layer23_attention_weight_min": -28.38092613220215,
      "activations/layer2_attention_weight_max": 33.74998474121094,
      "activations/layer2_attention_weight_min": -31.856151580810547,
      "activations/layer3_attention_weight_max": 104.02862548828125,
      "activations/layer3_attention_weight_min": -107.23016357421875,
      "activations/layer4_attention_weight_max": 108.83000946044922,
      "activations/layer4_attention_weight_min": -114.28375244140625,
      "activations/layer5_attention_weight_max": 55.00757598876953,
      "activations/layer5_attention_weight_min": -64.27073669433594,
      "activations/layer6_attention_weight_max": 47.26447677612305,
      "activations/layer6_attention_weight_min": -48.67207336425781,
      "activations/layer7_attention_weight_max": 100.90840911865234,
      "activations/layer7_attention_weight_min": -106.22164154052734,
      "activations/layer8_attention_weight_max": 47.78010559082031,
      "activations/layer8_attention_weight_min": -47.146461486816406,
      "activations/layer9_attention_weight_max": 34.038143157958984,
      "activations/layer9_attention_weight_min": -36.05973815917969,
      "epoch": 20.23,
      "learning_rate": 1.97625e-05,
      "loss": 2.7218,
      "step": 348100
    },
    {
      "activations/layer0_attention_weight_max": 16.893993377685547,
      "activations/layer0_attention_weight_min": -13.829874038696289,
      "activations/layer10_attention_weight_max": 33.12211227416992,
      "activations/layer10_attention_weight_min": -35.41796875,
      "activations/layer11_attention_weight_max": 33.69173812866211,
      "activations/layer11_attention_weight_min": -34.31864547729492,
      "activations/layer12_attention_weight_max": 27.71440887451172,
      "activations/layer12_attention_weight_min": -27.574132919311523,
      "activations/layer13_attention_weight_max": 40.23610305786133,
      "activations/layer13_attention_weight_min": -36.194580078125,
      "activations/layer14_attention_weight_max": 42.36486053466797,
      "activations/layer14_attention_weight_min": -37.739959716796875,
      "activations/layer15_attention_weight_max": 39.74728775024414,
      "activations/layer15_attention_weight_min": -37.23678970336914,
      "activations/layer16_attention_weight_max": 34.76871871948242,
      "activations/layer16_attention_weight_min": -30.549604415893555,
      "activations/layer17_attention_weight_max": 28.779497146606445,
      "activations/layer17_attention_weight_min": -24.97896957397461,
      "activations/layer18_attention_weight_max": 33.773040771484375,
      "activations/layer18_attention_weight_min": -26.212482452392578,
      "activations/layer19_attention_weight_max": 43.789649963378906,
      "activations/layer19_attention_weight_min": -29.83381462097168,
      "activations/layer1_attention_weight_max": 16.017175674438477,
      "activations/layer1_attention_weight_min": -15.233012199401855,
      "activations/layer20_attention_weight_max": 34.14950180053711,
      "activations/layer20_attention_weight_min": -25.635108947753906,
      "activations/layer21_attention_weight_max": 29.846044540405273,
      "activations/layer21_attention_weight_min": -25.69707679748535,
      "activations/layer22_attention_weight_max": 40.69393539428711,
      "activations/layer22_attention_weight_min": -31.911632537841797,
      "activations/layer23_attention_weight_max": 32.431182861328125,
      "activations/layer23_attention_weight_min": -23.839805603027344,
      "activations/layer2_attention_weight_max": 32.56869888305664,
      "activations/layer2_attention_weight_min": -32.711021423339844,
      "activations/layer3_attention_weight_max": 99.9195785522461,
      "activations/layer3_attention_weight_min": -99.82372283935547,
      "activations/layer4_attention_weight_max": 106.98114013671875,
      "activations/layer4_attention_weight_min": -104.3653564453125,
      "activations/layer5_attention_weight_max": 49.38039779663086,
      "activations/layer5_attention_weight_min": -60.83434295654297,
      "activations/layer6_attention_weight_max": 44.40947341918945,
      "activations/layer6_attention_weight_min": -45.583744049072266,
      "activations/layer7_attention_weight_max": 103.50553894042969,
      "activations/layer7_attention_weight_min": -92.80602264404297,
      "activations/layer8_attention_weight_max": 41.122283935546875,
      "activations/layer8_attention_weight_min": -42.91176223754883,
      "activations/layer9_attention_weight_max": 32.029205322265625,
      "activations/layer9_attention_weight_min": -35.312095642089844,
      "epoch": 20.23,
      "learning_rate": 1.9743560606060604e-05,
      "loss": 2.7303,
      "step": 348150
    },
    {
      "activations/layer0_attention_weight_max": 16.11248779296875,
      "activations/layer0_attention_weight_min": -14.469611167907715,
      "activations/layer10_attention_weight_max": 32.61740493774414,
      "activations/layer10_attention_weight_min": -33.231658935546875,
      "activations/layer11_attention_weight_max": 34.759464263916016,
      "activations/layer11_attention_weight_min": -34.31204605102539,
      "activations/layer12_attention_weight_max": 29.012649536132812,
      "activations/layer12_attention_weight_min": -27.15650177001953,
      "activations/layer13_attention_weight_max": 42.79690933227539,
      "activations/layer13_attention_weight_min": -37.33087158203125,
      "activations/layer14_attention_weight_max": 55.758827209472656,
      "activations/layer14_attention_weight_min": -42.58058547973633,
      "activations/layer15_attention_weight_max": 45.59865951538086,
      "activations/layer15_attention_weight_min": -40.40983581542969,
      "activations/layer16_attention_weight_max": 36.3727912902832,
      "activations/layer16_attention_weight_min": -32.6804313659668,
      "activations/layer17_attention_weight_max": 33.934261322021484,
      "activations/layer17_attention_weight_min": -30.273393630981445,
      "activations/layer18_attention_weight_max": 33.588470458984375,
      "activations/layer18_attention_weight_min": -28.940874099731445,
      "activations/layer19_attention_weight_max": 36.36787796020508,
      "activations/layer19_attention_weight_min": -32.1381721496582,
      "activations/layer1_attention_weight_max": 15.311919212341309,
      "activations/layer1_attention_weight_min": -14.559414863586426,
      "activations/layer20_attention_weight_max": 31.012035369873047,
      "activations/layer20_attention_weight_min": -27.274675369262695,
      "activations/layer21_attention_weight_max": 30.98025131225586,
      "activations/layer21_attention_weight_min": -26.20186996459961,
      "activations/layer22_attention_weight_max": 50.41753005981445,
      "activations/layer22_attention_weight_min": -33.360687255859375,
      "activations/layer23_attention_weight_max": 35.388160705566406,
      "activations/layer23_attention_weight_min": -26.610286712646484,
      "activations/layer2_attention_weight_max": 33.22987747192383,
      "activations/layer2_attention_weight_min": -32.49863815307617,
      "activations/layer3_attention_weight_max": 99.52091979980469,
      "activations/layer3_attention_weight_min": -104.07209777832031,
      "activations/layer4_attention_weight_max": 102.964599609375,
      "activations/layer4_attention_weight_min": -105.40665435791016,
      "activations/layer5_attention_weight_max": 50.26470947265625,
      "activations/layer5_attention_weight_min": -60.5640869140625,
      "activations/layer6_attention_weight_max": 43.59325408935547,
      "activations/layer6_attention_weight_min": -46.5558967590332,
      "activations/layer7_attention_weight_max": 91.84394073486328,
      "activations/layer7_attention_weight_min": -94.30670928955078,
      "activations/layer8_attention_weight_max": 42.21828079223633,
      "activations/layer8_attention_weight_min": -41.5784797668457,
      "activations/layer9_attention_weight_max": 32.50680923461914,
      "activations/layer9_attention_weight_min": -33.5738639831543,
      "epoch": 20.23,
      "learning_rate": 1.9724621212121212e-05,
      "loss": 2.7273,
      "step": 348200
    },
    {
      "activations/layer0_attention_weight_max": 16.498579025268555,
      "activations/layer0_attention_weight_min": -14.501043319702148,
      "activations/layer10_attention_weight_max": 34.25996780395508,
      "activations/layer10_attention_weight_min": -36.505714416503906,
      "activations/layer11_attention_weight_max": 34.67449951171875,
      "activations/layer11_attention_weight_min": -34.74666213989258,
      "activations/layer12_attention_weight_max": 28.25766372680664,
      "activations/layer12_attention_weight_min": -27.712326049804688,
      "activations/layer13_attention_weight_max": 42.90016555786133,
      "activations/layer13_attention_weight_min": -37.00917053222656,
      "activations/layer14_attention_weight_max": 46.63947677612305,
      "activations/layer14_attention_weight_min": -40.664493560791016,
      "activations/layer15_attention_weight_max": 42.458797454833984,
      "activations/layer15_attention_weight_min": -36.71067428588867,
      "activations/layer16_attention_weight_max": 31.813491821289062,
      "activations/layer16_attention_weight_min": -30.41490364074707,
      "activations/layer17_attention_weight_max": 33.80201721191406,
      "activations/layer17_attention_weight_min": -26.855918884277344,
      "activations/layer18_attention_weight_max": 34.83150863647461,
      "activations/layer18_attention_weight_min": -25.52568244934082,
      "activations/layer19_attention_weight_max": 41.44202423095703,
      "activations/layer19_attention_weight_min": -36.31787872314453,
      "activations/layer1_attention_weight_max": 15.059589385986328,
      "activations/layer1_attention_weight_min": -14.484113693237305,
      "activations/layer20_attention_weight_max": 35.17608642578125,
      "activations/layer20_attention_weight_min": -27.847515106201172,
      "activations/layer21_attention_weight_max": 34.57105255126953,
      "activations/layer21_attention_weight_min": -27.065019607543945,
      "activations/layer22_attention_weight_max": 48.95121383666992,
      "activations/layer22_attention_weight_min": -34.9402961730957,
      "activations/layer23_attention_weight_max": 36.54936981201172,
      "activations/layer23_attention_weight_min": -28.439878463745117,
      "activations/layer2_attention_weight_max": 34.23788833618164,
      "activations/layer2_attention_weight_min": -31.86587905883789,
      "activations/layer3_attention_weight_max": 103.83785247802734,
      "activations/layer3_attention_weight_min": -103.45448303222656,
      "activations/layer4_attention_weight_max": 111.63069152832031,
      "activations/layer4_attention_weight_min": -115.27400207519531,
      "activations/layer5_attention_weight_max": 52.83366394042969,
      "activations/layer5_attention_weight_min": -63.0055046081543,
      "activations/layer6_attention_weight_max": 46.83344268798828,
      "activations/layer6_attention_weight_min": -48.89662170410156,
      "activations/layer7_attention_weight_max": 101.6908187866211,
      "activations/layer7_attention_weight_min": -100.60099792480469,
      "activations/layer8_attention_weight_max": 44.2098503112793,
      "activations/layer8_attention_weight_min": -44.0019416809082,
      "activations/layer9_attention_weight_max": 32.718360900878906,
      "activations/layer9_attention_weight_min": -35.79068374633789,
      "epoch": 20.24,
      "learning_rate": 1.9705681818181814e-05,
      "loss": 2.7278,
      "step": 348250
    },
    {
      "activations/layer0_attention_weight_max": 16.028085708618164,
      "activations/layer0_attention_weight_min": -12.642196655273438,
      "activations/layer10_attention_weight_max": 37.843143463134766,
      "activations/layer10_attention_weight_min": -37.747230529785156,
      "activations/layer11_attention_weight_max": 37.707374572753906,
      "activations/layer11_attention_weight_min": -38.374786376953125,
      "activations/layer12_attention_weight_max": 27.043718338012695,
      "activations/layer12_attention_weight_min": -27.604501724243164,
      "activations/layer13_attention_weight_max": 39.00926971435547,
      "activations/layer13_attention_weight_min": -38.284297943115234,
      "activations/layer14_attention_weight_max": 52.07276153564453,
      "activations/layer14_attention_weight_min": -41.70411682128906,
      "activations/layer15_attention_weight_max": 42.609375,
      "activations/layer15_attention_weight_min": -42.203460693359375,
      "activations/layer16_attention_weight_max": 34.44464874267578,
      "activations/layer16_attention_weight_min": -32.50059127807617,
      "activations/layer17_attention_weight_max": 41.46968460083008,
      "activations/layer17_attention_weight_min": -30.65304183959961,
      "activations/layer18_attention_weight_max": 33.03512954711914,
      "activations/layer18_attention_weight_min": -31.862445831298828,
      "activations/layer19_attention_weight_max": 32.76895523071289,
      "activations/layer19_attention_weight_min": -34.15200424194336,
      "activations/layer1_attention_weight_max": 15.62177562713623,
      "activations/layer1_attention_weight_min": -14.30103874206543,
      "activations/layer20_attention_weight_max": 29.8745174407959,
      "activations/layer20_attention_weight_min": -25.529150009155273,
      "activations/layer21_attention_weight_max": 32.09487533569336,
      "activations/layer21_attention_weight_min": -25.065189361572266,
      "activations/layer22_attention_weight_max": 41.93061065673828,
      "activations/layer22_attention_weight_min": -32.529178619384766,
      "activations/layer23_attention_weight_max": 31.523948669433594,
      "activations/layer23_attention_weight_min": -27.260046005249023,
      "activations/layer2_attention_weight_max": 34.5598258972168,
      "activations/layer2_attention_weight_min": -32.821754455566406,
      "activations/layer3_attention_weight_max": 103.99861907958984,
      "activations/layer3_attention_weight_min": -101.0202407836914,
      "activations/layer4_attention_weight_max": 106.09452056884766,
      "activations/layer4_attention_weight_min": -111.13866424560547,
      "activations/layer5_attention_weight_max": 50.099998474121094,
      "activations/layer5_attention_weight_min": -62.15888595581055,
      "activations/layer6_attention_weight_max": 45.02804946899414,
      "activations/layer6_attention_weight_min": -48.13759994506836,
      "activations/layer7_attention_weight_max": 92.68732452392578,
      "activations/layer7_attention_weight_min": -100.35757446289062,
      "activations/layer8_attention_weight_max": 44.27238845825195,
      "activations/layer8_attention_weight_min": -44.491085052490234,
      "activations/layer9_attention_weight_max": 34.68618392944336,
      "activations/layer9_attention_weight_min": -36.09029006958008,
      "epoch": 20.24,
      "learning_rate": 1.9686742424242422e-05,
      "loss": 2.7056,
      "step": 348300
    },
    {
      "activations/layer0_attention_weight_max": 16.027860641479492,
      "activations/layer0_attention_weight_min": -13.94870376586914,
      "activations/layer10_attention_weight_max": 35.973602294921875,
      "activations/layer10_attention_weight_min": -37.55866241455078,
      "activations/layer11_attention_weight_max": 35.1409797668457,
      "activations/layer11_attention_weight_min": -36.8791618347168,
      "activations/layer12_attention_weight_max": 30.393327713012695,
      "activations/layer12_attention_weight_min": -28.1503849029541,
      "activations/layer13_attention_weight_max": 47.3551139831543,
      "activations/layer13_attention_weight_min": -38.712745666503906,
      "activations/layer14_attention_weight_max": 54.27141571044922,
      "activations/layer14_attention_weight_min": -40.87983703613281,
      "activations/layer15_attention_weight_max": 50.4103889465332,
      "activations/layer15_attention_weight_min": -44.455108642578125,
      "activations/layer16_attention_weight_max": 33.011539459228516,
      "activations/layer16_attention_weight_min": -29.434141159057617,
      "activations/layer17_attention_weight_max": 34.700592041015625,
      "activations/layer17_attention_weight_min": -26.394006729125977,
      "activations/layer18_attention_weight_max": 36.12746810913086,
      "activations/layer18_attention_weight_min": -25.028059005737305,
      "activations/layer19_attention_weight_max": 36.73258590698242,
      "activations/layer19_attention_weight_min": -31.527591705322266,
      "activations/layer1_attention_weight_max": 16.830142974853516,
      "activations/layer1_attention_weight_min": -14.940299987792969,
      "activations/layer20_attention_weight_max": 32.65153503417969,
      "activations/layer20_attention_weight_min": -27.9189395904541,
      "activations/layer21_attention_weight_max": 33.587371826171875,
      "activations/layer21_attention_weight_min": -25.1762752532959,
      "activations/layer22_attention_weight_max": 49.96684646606445,
      "activations/layer22_attention_weight_min": -32.36821365356445,
      "activations/layer23_attention_weight_max": 41.455780029296875,
      "activations/layer23_attention_weight_min": -26.82330894470215,
      "activations/layer2_attention_weight_max": 33.975582122802734,
      "activations/layer2_attention_weight_min": -33.804439544677734,
      "activations/layer3_attention_weight_max": 103.41670989990234,
      "activations/layer3_attention_weight_min": -105.90901947021484,
      "activations/layer4_attention_weight_max": 107.73902893066406,
      "activations/layer4_attention_weight_min": -122.10201263427734,
      "activations/layer5_attention_weight_max": 51.81243896484375,
      "activations/layer5_attention_weight_min": -66.10333251953125,
      "activations/layer6_attention_weight_max": 45.37742614746094,
      "activations/layer6_attention_weight_min": -49.90595626831055,
      "activations/layer7_attention_weight_max": 98.88282775878906,
      "activations/layer7_attention_weight_min": -99.27019500732422,
      "activations/layer8_attention_weight_max": 41.750038146972656,
      "activations/layer8_attention_weight_min": -47.003013610839844,
      "activations/layer9_attention_weight_max": 33.66006088256836,
      "activations/layer9_attention_weight_min": -35.577728271484375,
      "epoch": 20.24,
      "learning_rate": 1.9667803030303027e-05,
      "loss": 2.702,
      "step": 348350
    },
    {
      "activations/layer0_attention_weight_max": 15.98046875,
      "activations/layer0_attention_weight_min": -13.066374778747559,
      "activations/layer10_attention_weight_max": 35.77936553955078,
      "activations/layer10_attention_weight_min": -35.22361755371094,
      "activations/layer11_attention_weight_max": 35.03849792480469,
      "activations/layer11_attention_weight_min": -34.92182540893555,
      "activations/layer12_attention_weight_max": 27.203842163085938,
      "activations/layer12_attention_weight_min": -28.34940528869629,
      "activations/layer13_attention_weight_max": 40.76860809326172,
      "activations/layer13_attention_weight_min": -38.1067008972168,
      "activations/layer14_attention_weight_max": 42.714141845703125,
      "activations/layer14_attention_weight_min": -40.958839416503906,
      "activations/layer15_attention_weight_max": 38.6613655090332,
      "activations/layer15_attention_weight_min": -39.18566131591797,
      "activations/layer16_attention_weight_max": 29.421613693237305,
      "activations/layer16_attention_weight_min": -30.213924407958984,
      "activations/layer17_attention_weight_max": 31.252145767211914,
      "activations/layer17_attention_weight_min": -28.085824966430664,
      "activations/layer18_attention_weight_max": 35.70789337158203,
      "activations/layer18_attention_weight_min": -28.501869201660156,
      "activations/layer19_attention_weight_max": 35.44696807861328,
      "activations/layer19_attention_weight_min": -34.256954193115234,
      "activations/layer1_attention_weight_max": 15.348701477050781,
      "activations/layer1_attention_weight_min": -13.751181602478027,
      "activations/layer20_attention_weight_max": 29.93056297302246,
      "activations/layer20_attention_weight_min": -28.3197021484375,
      "activations/layer21_attention_weight_max": 29.377458572387695,
      "activations/layer21_attention_weight_min": -30.840085983276367,
      "activations/layer22_attention_weight_max": 48.5037727355957,
      "activations/layer22_attention_weight_min": -35.29902648925781,
      "activations/layer23_attention_weight_max": 34.35055160522461,
      "activations/layer23_attention_weight_min": -28.398983001708984,
      "activations/layer2_attention_weight_max": 33.485145568847656,
      "activations/layer2_attention_weight_min": -31.940048217773438,
      "activations/layer3_attention_weight_max": 105.51699829101562,
      "activations/layer3_attention_weight_min": -105.70751953125,
      "activations/layer4_attention_weight_max": 111.33885955810547,
      "activations/layer4_attention_weight_min": -112.99342346191406,
      "activations/layer5_attention_weight_max": 51.939308166503906,
      "activations/layer5_attention_weight_min": -60.57860565185547,
      "activations/layer6_attention_weight_max": 46.21919631958008,
      "activations/layer6_attention_weight_min": -49.12419891357422,
      "activations/layer7_attention_weight_max": 96.07254791259766,
      "activations/layer7_attention_weight_min": -102.94347381591797,
      "activations/layer8_attention_weight_max": 45.490596771240234,
      "activations/layer8_attention_weight_min": -46.448997497558594,
      "activations/layer9_attention_weight_max": 33.996238708496094,
      "activations/layer9_attention_weight_min": -34.788997650146484,
      "epoch": 20.24,
      "learning_rate": 1.964924242424242e-05,
      "loss": 2.7233,
      "step": 348400
    },
    {
      "activations/layer0_attention_weight_max": 15.90500259399414,
      "activations/layer0_attention_weight_min": -13.126335144042969,
      "activations/layer10_attention_weight_max": 38.91781234741211,
      "activations/layer10_attention_weight_min": -37.43666458129883,
      "activations/layer11_attention_weight_max": 37.36231994628906,
      "activations/layer11_attention_weight_min": -38.04631805419922,
      "activations/layer12_attention_weight_max": 31.809045791625977,
      "activations/layer12_attention_weight_min": -30.292484283447266,
      "activations/layer13_attention_weight_max": 47.72777557373047,
      "activations/layer13_attention_weight_min": -39.13484191894531,
      "activations/layer14_attention_weight_max": 51.93550109863281,
      "activations/layer14_attention_weight_min": -42.621986389160156,
      "activations/layer15_attention_weight_max": 44.695396423339844,
      "activations/layer15_attention_weight_min": -42.587764739990234,
      "activations/layer16_attention_weight_max": 36.25459289550781,
      "activations/layer16_attention_weight_min": -32.43327331542969,
      "activations/layer17_attention_weight_max": 34.68852233886719,
      "activations/layer17_attention_weight_min": -33.48426818847656,
      "activations/layer18_attention_weight_max": 39.12086486816406,
      "activations/layer18_attention_weight_min": -35.00761413574219,
      "activations/layer19_attention_weight_max": 42.132747650146484,
      "activations/layer19_attention_weight_min": -39.089969635009766,
      "activations/layer1_attention_weight_max": 15.67407512664795,
      "activations/layer1_attention_weight_min": -14.604889869689941,
      "activations/layer20_attention_weight_max": 33.387935638427734,
      "activations/layer20_attention_weight_min": -30.509370803833008,
      "activations/layer21_attention_weight_max": 33.75996780395508,
      "activations/layer21_attention_weight_min": -32.057594299316406,
      "activations/layer22_attention_weight_max": 52.59772491455078,
      "activations/layer22_attention_weight_min": -44.10420608520508,
      "activations/layer23_attention_weight_max": 38.50829315185547,
      "activations/layer23_attention_weight_min": -33.86380386352539,
      "activations/layer2_attention_weight_max": 34.39642333984375,
      "activations/layer2_attention_weight_min": -34.089744567871094,
      "activations/layer3_attention_weight_max": 107.43470001220703,
      "activations/layer3_attention_weight_min": -105.3721694946289,
      "activations/layer4_attention_weight_max": 114.1798095703125,
      "activations/layer4_attention_weight_min": -115.36920166015625,
      "activations/layer5_attention_weight_max": 52.28302764892578,
      "activations/layer5_attention_weight_min": -62.261680603027344,
      "activations/layer6_attention_weight_max": 48.782310485839844,
      "activations/layer6_attention_weight_min": -48.60027313232422,
      "activations/layer7_attention_weight_max": 111.79439544677734,
      "activations/layer7_attention_weight_min": -99.25659942626953,
      "activations/layer8_attention_weight_max": 48.00151062011719,
      "activations/layer8_attention_weight_min": -48.6251106262207,
      "activations/layer9_attention_weight_max": 35.85441970825195,
      "activations/layer9_attention_weight_min": -36.97087478637695,
      "epoch": 20.25,
      "learning_rate": 1.9630303030303026e-05,
      "loss": 2.7155,
      "step": 348450
    },
    {
      "activations/layer0_attention_weight_max": 16.21762466430664,
      "activations/layer0_attention_weight_min": -12.150507926940918,
      "activations/layer10_attention_weight_max": 34.920143127441406,
      "activations/layer10_attention_weight_min": -35.195533752441406,
      "activations/layer11_attention_weight_max": 36.18699645996094,
      "activations/layer11_attention_weight_min": -34.85283660888672,
      "activations/layer12_attention_weight_max": 26.788822174072266,
      "activations/layer12_attention_weight_min": -26.34079360961914,
      "activations/layer13_attention_weight_max": 39.563323974609375,
      "activations/layer13_attention_weight_min": -35.78269577026367,
      "activations/layer14_attention_weight_max": 44.617923736572266,
      "activations/layer14_attention_weight_min": -39.108760833740234,
      "activations/layer15_attention_weight_max": 39.0843620300293,
      "activations/layer15_attention_weight_min": -38.0222282409668,
      "activations/layer16_attention_weight_max": 30.71089744567871,
      "activations/layer16_attention_weight_min": -30.060691833496094,
      "activations/layer17_attention_weight_max": 30.76918601989746,
      "activations/layer17_attention_weight_min": -27.739511489868164,
      "activations/layer18_attention_weight_max": 32.92079544067383,
      "activations/layer18_attention_weight_min": -26.513389587402344,
      "activations/layer19_attention_weight_max": 37.42223358154297,
      "activations/layer19_attention_weight_min": -32.69355392456055,
      "activations/layer1_attention_weight_max": 16.141748428344727,
      "activations/layer1_attention_weight_min": -15.691093444824219,
      "activations/layer20_attention_weight_max": 31.649328231811523,
      "activations/layer20_attention_weight_min": -26.584672927856445,
      "activations/layer21_attention_weight_max": 29.61357307434082,
      "activations/layer21_attention_weight_min": -25.171537399291992,
      "activations/layer22_attention_weight_max": 39.35013198852539,
      "activations/layer22_attention_weight_min": -33.38019943237305,
      "activations/layer23_attention_weight_max": 31.68025016784668,
      "activations/layer23_attention_weight_min": -29.186614990234375,
      "activations/layer2_attention_weight_max": 32.92594909667969,
      "activations/layer2_attention_weight_min": -31.857831954956055,
      "activations/layer3_attention_weight_max": 101.46271514892578,
      "activations/layer3_attention_weight_min": -106.09147644042969,
      "activations/layer4_attention_weight_max": 105.6417007446289,
      "activations/layer4_attention_weight_min": -112.74079895019531,
      "activations/layer5_attention_weight_max": 50.163902282714844,
      "activations/layer5_attention_weight_min": -62.842132568359375,
      "activations/layer6_attention_weight_max": 45.46205520629883,
      "activations/layer6_attention_weight_min": -49.158023834228516,
      "activations/layer7_attention_weight_max": 95.15869903564453,
      "activations/layer7_attention_weight_min": -102.11654663085938,
      "activations/layer8_attention_weight_max": 43.86577224731445,
      "activations/layer8_attention_weight_min": -45.22948455810547,
      "activations/layer9_attention_weight_max": 34.465972900390625,
      "activations/layer9_attention_weight_min": -35.27499008178711,
      "epoch": 20.25,
      "learning_rate": 1.9611363636363634e-05,
      "loss": 2.7389,
      "step": 348500
    },
    {
      "activations/layer0_attention_weight_max": 16.24403953552246,
      "activations/layer0_attention_weight_min": -13.618691444396973,
      "activations/layer10_attention_weight_max": 35.43330764770508,
      "activations/layer10_attention_weight_min": -36.832210540771484,
      "activations/layer11_attention_weight_max": 36.80342102050781,
      "activations/layer11_attention_weight_min": -36.57911682128906,
      "activations/layer12_attention_weight_max": 28.319473266601562,
      "activations/layer12_attention_weight_min": -28.463823318481445,
      "activations/layer13_attention_weight_max": 39.119895935058594,
      "activations/layer13_attention_weight_min": -37.90812301635742,
      "activations/layer14_attention_weight_max": 40.65967559814453,
      "activations/layer14_attention_weight_min": -39.02372360229492,
      "activations/layer15_attention_weight_max": 37.217689514160156,
      "activations/layer15_attention_weight_min": -38.16349411010742,
      "activations/layer16_attention_weight_max": 27.06493377685547,
      "activations/layer16_attention_weight_min": -27.60364532470703,
      "activations/layer17_attention_weight_max": 29.11760902404785,
      "activations/layer17_attention_weight_min": -27.09649658203125,
      "activations/layer18_attention_weight_max": 35.48968505859375,
      "activations/layer18_attention_weight_min": -27.686246871948242,
      "activations/layer19_attention_weight_max": 32.839820861816406,
      "activations/layer19_attention_weight_min": -33.53083419799805,
      "activations/layer1_attention_weight_max": 15.774216651916504,
      "activations/layer1_attention_weight_min": -13.840544700622559,
      "activations/layer20_attention_weight_max": 27.111425399780273,
      "activations/layer20_attention_weight_min": -26.014379501342773,
      "activations/layer21_attention_weight_max": 26.963268280029297,
      "activations/layer21_attention_weight_min": -25.32451057434082,
      "activations/layer22_attention_weight_max": 41.647247314453125,
      "activations/layer22_attention_weight_min": -33.75143051147461,
      "activations/layer23_attention_weight_max": 31.34071922302246,
      "activations/layer23_attention_weight_min": -27.829666137695312,
      "activations/layer2_attention_weight_max": 32.891571044921875,
      "activations/layer2_attention_weight_min": -33.707401275634766,
      "activations/layer3_attention_weight_max": 102.92250061035156,
      "activations/layer3_attention_weight_min": -105.77947998046875,
      "activations/layer4_attention_weight_max": 109.26175689697266,
      "activations/layer4_attention_weight_min": -114.1727066040039,
      "activations/layer5_attention_weight_max": 50.138816833496094,
      "activations/layer5_attention_weight_min": -62.71849060058594,
      "activations/layer6_attention_weight_max": 46.15510177612305,
      "activations/layer6_attention_weight_min": -48.57957077026367,
      "activations/layer7_attention_weight_max": 95.23744201660156,
      "activations/layer7_attention_weight_min": -101.42896270751953,
      "activations/layer8_attention_weight_max": 43.41029739379883,
      "activations/layer8_attention_weight_min": -47.464324951171875,
      "activations/layer9_attention_weight_max": 32.94515609741211,
      "activations/layer9_attention_weight_min": -36.26384353637695,
      "epoch": 20.25,
      "learning_rate": 1.959242424242424e-05,
      "loss": 2.7238,
      "step": 348550
    },
    {
      "activations/layer0_attention_weight_max": 15.95085620880127,
      "activations/layer0_attention_weight_min": -12.6614351272583,
      "activations/layer10_attention_weight_max": 36.04628372192383,
      "activations/layer10_attention_weight_min": -34.89202880859375,
      "activations/layer11_attention_weight_max": 35.749393463134766,
      "activations/layer11_attention_weight_min": -35.666481018066406,
      "activations/layer12_attention_weight_max": 27.39370346069336,
      "activations/layer12_attention_weight_min": -26.521547317504883,
      "activations/layer13_attention_weight_max": 41.19212341308594,
      "activations/layer13_attention_weight_min": -38.04039764404297,
      "activations/layer14_attention_weight_max": 43.206581115722656,
      "activations/layer14_attention_weight_min": -37.76871109008789,
      "activations/layer15_attention_weight_max": 37.70491027832031,
      "activations/layer15_attention_weight_min": -37.467716217041016,
      "activations/layer16_attention_weight_max": 29.62042999267578,
      "activations/layer16_attention_weight_min": -27.889671325683594,
      "activations/layer17_attention_weight_max": 30.043142318725586,
      "activations/layer17_attention_weight_min": -26.421049118041992,
      "activations/layer18_attention_weight_max": 32.44108581542969,
      "activations/layer18_attention_weight_min": -26.345781326293945,
      "activations/layer19_attention_weight_max": 37.40624237060547,
      "activations/layer19_attention_weight_min": -29.988906860351562,
      "activations/layer1_attention_weight_max": 15.85783576965332,
      "activations/layer1_attention_weight_min": -14.399991989135742,
      "activations/layer20_attention_weight_max": 34.27167510986328,
      "activations/layer20_attention_weight_min": -23.44281578063965,
      "activations/layer21_attention_weight_max": 30.40518569946289,
      "activations/layer21_attention_weight_min": -25.784032821655273,
      "activations/layer22_attention_weight_max": 40.777488708496094,
      "activations/layer22_attention_weight_min": -32.2978515625,
      "activations/layer23_attention_weight_max": 31.7406005859375,
      "activations/layer23_attention_weight_min": -24.150333404541016,
      "activations/layer2_attention_weight_max": 33.32489013671875,
      "activations/layer2_attention_weight_min": -34.086761474609375,
      "activations/layer3_attention_weight_max": 106.42391204833984,
      "activations/layer3_attention_weight_min": -107.85942077636719,
      "activations/layer4_attention_weight_max": 107.54390716552734,
      "activations/layer4_attention_weight_min": -106.2349624633789,
      "activations/layer5_attention_weight_max": 50.18766403198242,
      "activations/layer5_attention_weight_min": -62.61225128173828,
      "activations/layer6_attention_weight_max": 43.98908233642578,
      "activations/layer6_attention_weight_min": -45.56422424316406,
      "activations/layer7_attention_weight_max": 93.68073272705078,
      "activations/layer7_attention_weight_min": -95.61433410644531,
      "activations/layer8_attention_weight_max": 42.87914276123047,
      "activations/layer8_attention_weight_min": -44.61035919189453,
      "activations/layer9_attention_weight_max": 31.502914428710938,
      "activations/layer9_attention_weight_min": -33.15508270263672,
      "epoch": 20.26,
      "learning_rate": 1.9573484848484848e-05,
      "loss": 2.7313,
      "step": 348600
    },
    {
      "activations/layer0_attention_weight_max": 18.046934127807617,
      "activations/layer0_attention_weight_min": -12.849567413330078,
      "activations/layer10_attention_weight_max": 33.1673698425293,
      "activations/layer10_attention_weight_min": -33.498756408691406,
      "activations/layer11_attention_weight_max": 31.769153594970703,
      "activations/layer11_attention_weight_min": -34.016624450683594,
      "activations/layer12_attention_weight_max": 27.908823013305664,
      "activations/layer12_attention_weight_min": -25.65900993347168,
      "activations/layer13_attention_weight_max": 35.24394989013672,
      "activations/layer13_attention_weight_min": -33.27096176147461,
      "activations/layer14_attention_weight_max": 37.513179779052734,
      "activations/layer14_attention_weight_min": -35.09033966064453,
      "activations/layer15_attention_weight_max": 35.461585998535156,
      "activations/layer15_attention_weight_min": -35.17644119262695,
      "activations/layer16_attention_weight_max": 33.032752990722656,
      "activations/layer16_attention_weight_min": -27.720510482788086,
      "activations/layer17_attention_weight_max": 26.547164916992188,
      "activations/layer17_attention_weight_min": -26.067564010620117,
      "activations/layer18_attention_weight_max": 30.368831634521484,
      "activations/layer18_attention_weight_min": -24.794370651245117,
      "activations/layer19_attention_weight_max": 34.06291198730469,
      "activations/layer19_attention_weight_min": -31.127649307250977,
      "activations/layer1_attention_weight_max": 15.996344566345215,
      "activations/layer1_attention_weight_min": -14.612975120544434,
      "activations/layer20_attention_weight_max": 28.758928298950195,
      "activations/layer20_attention_weight_min": -24.82452392578125,
      "activations/layer21_attention_weight_max": 29.586660385131836,
      "activations/layer21_attention_weight_min": -24.0853271484375,
      "activations/layer22_attention_weight_max": 44.46354675292969,
      "activations/layer22_attention_weight_min": -31.9895076751709,
      "activations/layer23_attention_weight_max": 32.10508346557617,
      "activations/layer23_attention_weight_min": -24.693267822265625,
      "activations/layer2_attention_weight_max": 32.437191009521484,
      "activations/layer2_attention_weight_min": -32.07709503173828,
      "activations/layer3_attention_weight_max": 100.2879409790039,
      "activations/layer3_attention_weight_min": -105.53619384765625,
      "activations/layer4_attention_weight_max": 105.86397552490234,
      "activations/layer4_attention_weight_min": -109.1613540649414,
      "activations/layer5_attention_weight_max": 51.163997650146484,
      "activations/layer5_attention_weight_min": -60.654296875,
      "activations/layer6_attention_weight_max": 43.82648468017578,
      "activations/layer6_attention_weight_min": -47.194068908691406,
      "activations/layer7_attention_weight_max": 92.43472290039062,
      "activations/layer7_attention_weight_min": -94.92497253417969,
      "activations/layer8_attention_weight_max": 42.12271499633789,
      "activations/layer8_attention_weight_min": -44.573219299316406,
      "activations/layer9_attention_weight_max": 32.14789962768555,
      "activations/layer9_attention_weight_min": -33.13597106933594,
      "epoch": 20.26,
      "learning_rate": 1.9554545454545456e-05,
      "loss": 2.713,
      "step": 348650
    },
    {
      "activations/layer0_attention_weight_max": 15.49330997467041,
      "activations/layer0_attention_weight_min": -13.42441463470459,
      "activations/layer10_attention_weight_max": 32.473690032958984,
      "activations/layer10_attention_weight_min": -33.52690124511719,
      "activations/layer11_attention_weight_max": 31.987369537353516,
      "activations/layer11_attention_weight_min": -33.07121276855469,
      "activations/layer12_attention_weight_max": 26.41716766357422,
      "activations/layer12_attention_weight_min": -26.44390296936035,
      "activations/layer13_attention_weight_max": 38.31690979003906,
      "activations/layer13_attention_weight_min": -35.65678024291992,
      "activations/layer14_attention_weight_max": 40.93482971191406,
      "activations/layer14_attention_weight_min": -37.46165084838867,
      "activations/layer15_attention_weight_max": 36.49675369262695,
      "activations/layer15_attention_weight_min": -37.685367584228516,
      "activations/layer16_attention_weight_max": 29.240108489990234,
      "activations/layer16_attention_weight_min": -27.516054153442383,
      "activations/layer17_attention_weight_max": 31.32711410522461,
      "activations/layer17_attention_weight_min": -28.012805938720703,
      "activations/layer18_attention_weight_max": 30.585350036621094,
      "activations/layer18_attention_weight_min": -27.403583526611328,
      "activations/layer19_attention_weight_max": 32.877315521240234,
      "activations/layer19_attention_weight_min": -32.798397064208984,
      "activations/layer1_attention_weight_max": 16.251190185546875,
      "activations/layer1_attention_weight_min": -14.662331581115723,
      "activations/layer20_attention_weight_max": 29.136449813842773,
      "activations/layer20_attention_weight_min": -25.775157928466797,
      "activations/layer21_attention_weight_max": 28.227373123168945,
      "activations/layer21_attention_weight_min": -25.0933895111084,
      "activations/layer22_attention_weight_max": 40.80974197387695,
      "activations/layer22_attention_weight_min": -30.5040283203125,
      "activations/layer23_attention_weight_max": 30.937442779541016,
      "activations/layer23_attention_weight_min": -26.836959838867188,
      "activations/layer2_attention_weight_max": 32.51692199707031,
      "activations/layer2_attention_weight_min": -32.389488220214844,
      "activations/layer3_attention_weight_max": 97.06578063964844,
      "activations/layer3_attention_weight_min": -100.0486831665039,
      "activations/layer4_attention_weight_max": 102.56044006347656,
      "activations/layer4_attention_weight_min": -105.7685546875,
      "activations/layer5_attention_weight_max": 47.35315704345703,
      "activations/layer5_attention_weight_min": -57.03984832763672,
      "activations/layer6_attention_weight_max": 43.865989685058594,
      "activations/layer6_attention_weight_min": -44.567081451416016,
      "activations/layer7_attention_weight_max": 97.34202575683594,
      "activations/layer7_attention_weight_min": -94.9736328125,
      "activations/layer8_attention_weight_max": 40.23869323730469,
      "activations/layer8_attention_weight_min": -44.70173645019531,
      "activations/layer9_attention_weight_max": 31.380924224853516,
      "activations/layer9_attention_weight_min": -34.804683685302734,
      "epoch": 20.26,
      "learning_rate": 1.9535606060606057e-05,
      "loss": 2.7299,
      "step": 348700
    },
    {
      "activations/layer0_attention_weight_max": 17.221797943115234,
      "activations/layer0_attention_weight_min": -14.030256271362305,
      "activations/layer10_attention_weight_max": 39.535621643066406,
      "activations/layer10_attention_weight_min": -39.1392707824707,
      "activations/layer11_attention_weight_max": 39.55537033081055,
      "activations/layer11_attention_weight_min": -40.68483352661133,
      "activations/layer12_attention_weight_max": 29.362720489501953,
      "activations/layer12_attention_weight_min": -28.696020126342773,
      "activations/layer13_attention_weight_max": 43.883201599121094,
      "activations/layer13_attention_weight_min": -39.776363372802734,
      "activations/layer14_attention_weight_max": 52.268898010253906,
      "activations/layer14_attention_weight_min": -40.518707275390625,
      "activations/layer15_attention_weight_max": 45.73540115356445,
      "activations/layer15_attention_weight_min": -41.36952209472656,
      "activations/layer16_attention_weight_max": 37.96923065185547,
      "activations/layer16_attention_weight_min": -32.21207046508789,
      "activations/layer17_attention_weight_max": 36.82093811035156,
      "activations/layer17_attention_weight_min": -30.71599006652832,
      "activations/layer18_attention_weight_max": 35.719478607177734,
      "activations/layer18_attention_weight_min": -32.650455474853516,
      "activations/layer19_attention_weight_max": 36.8851432800293,
      "activations/layer19_attention_weight_min": -33.793052673339844,
      "activations/layer1_attention_weight_max": 16.22707176208496,
      "activations/layer1_attention_weight_min": -16.189603805541992,
      "activations/layer20_attention_weight_max": 31.287546157836914,
      "activations/layer20_attention_weight_min": -26.64622688293457,
      "activations/layer21_attention_weight_max": 30.46366310119629,
      "activations/layer21_attention_weight_min": -27.09798812866211,
      "activations/layer22_attention_weight_max": 44.7641487121582,
      "activations/layer22_attention_weight_min": -35.31325149536133,
      "activations/layer23_attention_weight_max": 34.686317443847656,
      "activations/layer23_attention_weight_min": -28.40511703491211,
      "activations/layer2_attention_weight_max": 36.67795944213867,
      "activations/layer2_attention_weight_min": -33.21626281738281,
      "activations/layer3_attention_weight_max": 111.86297607421875,
      "activations/layer3_attention_weight_min": -109.00811767578125,
      "activations/layer4_attention_weight_max": 119.910400390625,
      "activations/layer4_attention_weight_min": -116.8659896850586,
      "activations/layer5_attention_weight_max": 57.64897155761719,
      "activations/layer5_attention_weight_min": -67.24506378173828,
      "activations/layer6_attention_weight_max": 49.644527435302734,
      "activations/layer6_attention_weight_min": -49.37385940551758,
      "activations/layer7_attention_weight_max": 103.44037628173828,
      "activations/layer7_attention_weight_min": -106.96978759765625,
      "activations/layer8_attention_weight_max": 50.55449295043945,
      "activations/layer8_attention_weight_min": -50.23414611816406,
      "activations/layer9_attention_weight_max": 38.74951171875,
      "activations/layer9_attention_weight_min": -39.44062423706055,
      "epoch": 20.26,
      "learning_rate": 1.9516666666666666e-05,
      "loss": 2.7303,
      "step": 348750
    },
    {
      "activations/layer0_attention_weight_max": 16.675371170043945,
      "activations/layer0_attention_weight_min": -14.704855918884277,
      "activations/layer10_attention_weight_max": 33.351478576660156,
      "activations/layer10_attention_weight_min": -36.29042053222656,
      "activations/layer11_attention_weight_max": 33.64228057861328,
      "activations/layer11_attention_weight_min": -36.073787689208984,
      "activations/layer12_attention_weight_max": 27.34146499633789,
      "activations/layer12_attention_weight_min": -27.148500442504883,
      "activations/layer13_attention_weight_max": 42.220863342285156,
      "activations/layer13_attention_weight_min": -35.8427848815918,
      "activations/layer14_attention_weight_max": 42.79926681518555,
      "activations/layer14_attention_weight_min": -41.61516189575195,
      "activations/layer15_attention_weight_max": 40.62018966674805,
      "activations/layer15_attention_weight_min": -38.64285659790039,
      "activations/layer16_attention_weight_max": 32.53485107421875,
      "activations/layer16_attention_weight_min": -28.792890548706055,
      "activations/layer17_attention_weight_max": 30.58842658996582,
      "activations/layer17_attention_weight_min": -26.5670108795166,
      "activations/layer18_attention_weight_max": 30.949016571044922,
      "activations/layer18_attention_weight_min": -25.714160919189453,
      "activations/layer19_attention_weight_max": 36.23469543457031,
      "activations/layer19_attention_weight_min": -32.04069900512695,
      "activations/layer1_attention_weight_max": 16.028188705444336,
      "activations/layer1_attention_weight_min": -14.210511207580566,
      "activations/layer20_attention_weight_max": 29.440767288208008,
      "activations/layer20_attention_weight_min": -24.506351470947266,
      "activations/layer21_attention_weight_max": 29.55731773376465,
      "activations/layer21_attention_weight_min": -25.6517276763916,
      "activations/layer22_attention_weight_max": 40.98011779785156,
      "activations/layer22_attention_weight_min": -30.474201202392578,
      "activations/layer23_attention_weight_max": 32.405269622802734,
      "activations/layer23_attention_weight_min": -25.17491912841797,
      "activations/layer2_attention_weight_max": 34.37320327758789,
      "activations/layer2_attention_weight_min": -33.12889862060547,
      "activations/layer3_attention_weight_max": 107.76001739501953,
      "activations/layer3_attention_weight_min": -111.5503921508789,
      "activations/layer4_attention_weight_max": 109.76368713378906,
      "activations/layer4_attention_weight_min": -116.34394836425781,
      "activations/layer5_attention_weight_max": 51.05195617675781,
      "activations/layer5_attention_weight_min": -61.26947784423828,
      "activations/layer6_attention_weight_max": 45.271060943603516,
      "activations/layer6_attention_weight_min": -50.13846969604492,
      "activations/layer7_attention_weight_max": 96.8275146484375,
      "activations/layer7_attention_weight_min": -103.5278549194336,
      "activations/layer8_attention_weight_max": 42.811988830566406,
      "activations/layer8_attention_weight_min": -45.29876708984375,
      "activations/layer9_attention_weight_max": 31.8544979095459,
      "activations/layer9_attention_weight_min": -34.811885833740234,
      "epoch": 20.27,
      "learning_rate": 1.949772727272727e-05,
      "loss": 2.7181,
      "step": 348800
    },
    {
      "activations/layer0_attention_weight_max": 16.416183471679688,
      "activations/layer0_attention_weight_min": -13.303413391113281,
      "activations/layer10_attention_weight_max": 34.26082992553711,
      "activations/layer10_attention_weight_min": -35.990360260009766,
      "activations/layer11_attention_weight_max": 35.42290496826172,
      "activations/layer11_attention_weight_min": -37.80522155761719,
      "activations/layer12_attention_weight_max": 27.8752384185791,
      "activations/layer12_attention_weight_min": -26.762962341308594,
      "activations/layer13_attention_weight_max": 40.48297882080078,
      "activations/layer13_attention_weight_min": -36.49641036987305,
      "activations/layer14_attention_weight_max": 43.2163200378418,
      "activations/layer14_attention_weight_min": -38.56870651245117,
      "activations/layer15_attention_weight_max": 40.66201400756836,
      "activations/layer15_attention_weight_min": -36.80789566040039,
      "activations/layer16_attention_weight_max": 36.5997314453125,
      "activations/layer16_attention_weight_min": -29.57093620300293,
      "activations/layer17_attention_weight_max": 32.183292388916016,
      "activations/layer17_attention_weight_min": -29.461984634399414,
      "activations/layer18_attention_weight_max": 36.006935119628906,
      "activations/layer18_attention_weight_min": -28.031343460083008,
      "activations/layer19_attention_weight_max": 37.40415954589844,
      "activations/layer19_attention_weight_min": -34.644161224365234,
      "activations/layer1_attention_weight_max": 15.967240333557129,
      "activations/layer1_attention_weight_min": -13.349263191223145,
      "activations/layer20_attention_weight_max": 30.940322875976562,
      "activations/layer20_attention_weight_min": -27.45607566833496,
      "activations/layer21_attention_weight_max": 29.306425094604492,
      "activations/layer21_attention_weight_min": -27.63832664489746,
      "activations/layer22_attention_weight_max": 39.717681884765625,
      "activations/layer22_attention_weight_min": -33.730743408203125,
      "activations/layer23_attention_weight_max": 33.25366973876953,
      "activations/layer23_attention_weight_min": -27.308006286621094,
      "activations/layer2_attention_weight_max": 32.05630111694336,
      "activations/layer2_attention_weight_min": -31.936107635498047,
      "activations/layer3_attention_weight_max": 97.72010803222656,
      "activations/layer3_attention_weight_min": -104.40098571777344,
      "activations/layer4_attention_weight_max": 104.43293762207031,
      "activations/layer4_attention_weight_min": -111.9814453125,
      "activations/layer5_attention_weight_max": 49.73287582397461,
      "activations/layer5_attention_weight_min": -59.751564025878906,
      "activations/layer6_attention_weight_max": 44.06196975708008,
      "activations/layer6_attention_weight_min": -50.1639518737793,
      "activations/layer7_attention_weight_max": 94.15585327148438,
      "activations/layer7_attention_weight_min": -97.5115966796875,
      "activations/layer8_attention_weight_max": 40.99930191040039,
      "activations/layer8_attention_weight_min": -43.56621551513672,
      "activations/layer9_attention_weight_max": 31.151107788085938,
      "activations/layer9_attention_weight_min": -34.389591217041016,
      "epoch": 20.27,
      "learning_rate": 1.9479166666666668e-05,
      "loss": 2.7155,
      "step": 348850
    },
    {
      "activations/layer0_attention_weight_max": 16.19612693786621,
      "activations/layer0_attention_weight_min": -12.33605670928955,
      "activations/layer10_attention_weight_max": 38.09254455566406,
      "activations/layer10_attention_weight_min": -36.94344711303711,
      "activations/layer11_attention_weight_max": 35.12816619873047,
      "activations/layer11_attention_weight_min": -37.062416076660156,
      "activations/layer12_attention_weight_max": 28.329591751098633,
      "activations/layer12_attention_weight_min": -27.82526969909668,
      "activations/layer13_attention_weight_max": 42.227439880371094,
      "activations/layer13_attention_weight_min": -36.86388397216797,
      "activations/layer14_attention_weight_max": 46.778236389160156,
      "activations/layer14_attention_weight_min": -39.981021881103516,
      "activations/layer15_attention_weight_max": 42.899131774902344,
      "activations/layer15_attention_weight_min": -37.1925163269043,
      "activations/layer16_attention_weight_max": 35.2395133972168,
      "activations/layer16_attention_weight_min": -29.281940460205078,
      "activations/layer17_attention_weight_max": 32.06932067871094,
      "activations/layer17_attention_weight_min": -27.03144073486328,
      "activations/layer18_attention_weight_max": 31.410785675048828,
      "activations/layer18_attention_weight_min": -27.243722915649414,
      "activations/layer19_attention_weight_max": 43.170753479003906,
      "activations/layer19_attention_weight_min": -31.336820602416992,
      "activations/layer1_attention_weight_max": 15.074378967285156,
      "activations/layer1_attention_weight_min": -15.18159294128418,
      "activations/layer20_attention_weight_max": 36.95708084106445,
      "activations/layer20_attention_weight_min": -24.775999069213867,
      "activations/layer21_attention_weight_max": 33.03889846801758,
      "activations/layer21_attention_weight_min": -24.52463722229004,
      "activations/layer22_attention_weight_max": 45.30620193481445,
      "activations/layer22_attention_weight_min": -31.523340225219727,
      "activations/layer23_attention_weight_max": 33.633663177490234,
      "activations/layer23_attention_weight_min": -26.015113830566406,
      "activations/layer2_attention_weight_max": 32.68819046020508,
      "activations/layer2_attention_weight_min": -32.47789001464844,
      "activations/layer3_attention_weight_max": 104.02617645263672,
      "activations/layer3_attention_weight_min": -107.0134506225586,
      "activations/layer4_attention_weight_max": 108.37076568603516,
      "activations/layer4_attention_weight_min": -113.23014068603516,
      "activations/layer5_attention_weight_max": 51.892539978027344,
      "activations/layer5_attention_weight_min": -63.16499328613281,
      "activations/layer6_attention_weight_max": 45.9190559387207,
      "activations/layer6_attention_weight_min": -48.29829406738281,
      "activations/layer7_attention_weight_max": 102.67850494384766,
      "activations/layer7_attention_weight_min": -105.03441619873047,
      "activations/layer8_attention_weight_max": 46.236480712890625,
      "activations/layer8_attention_weight_min": -48.64806365966797,
      "activations/layer9_attention_weight_max": 35.960594177246094,
      "activations/layer9_attention_weight_min": -37.318599700927734,
      "epoch": 20.27,
      "learning_rate": 1.946022727272727e-05,
      "loss": 2.7216,
      "step": 348900
    },
    {
      "activations/layer0_attention_weight_max": 16.14771270751953,
      "activations/layer0_attention_weight_min": -12.736364364624023,
      "activations/layer10_attention_weight_max": 35.741764068603516,
      "activations/layer10_attention_weight_min": -35.14054870605469,
      "activations/layer11_attention_weight_max": 38.0457649230957,
      "activations/layer11_attention_weight_min": -35.960506439208984,
      "activations/layer12_attention_weight_max": 28.077560424804688,
      "activations/layer12_attention_weight_min": -28.493440628051758,
      "activations/layer13_attention_weight_max": 43.31947708129883,
      "activations/layer13_attention_weight_min": -37.606895446777344,
      "activations/layer14_attention_weight_max": 47.92655944824219,
      "activations/layer14_attention_weight_min": -40.66575622558594,
      "activations/layer15_attention_weight_max": 48.96696853637695,
      "activations/layer15_attention_weight_min": -38.63557434082031,
      "activations/layer16_attention_weight_max": 34.558902740478516,
      "activations/layer16_attention_weight_min": -29.552305221557617,
      "activations/layer17_attention_weight_max": 36.768287658691406,
      "activations/layer17_attention_weight_min": -27.920225143432617,
      "activations/layer18_attention_weight_max": 36.75056076049805,
      "activations/layer18_attention_weight_min": -26.177949905395508,
      "activations/layer19_attention_weight_max": 39.57896041870117,
      "activations/layer19_attention_weight_min": -32.88309097290039,
      "activations/layer1_attention_weight_max": 15.305415153503418,
      "activations/layer1_attention_weight_min": -14.00243091583252,
      "activations/layer20_attention_weight_max": 32.719852447509766,
      "activations/layer20_attention_weight_min": -28.00600242614746,
      "activations/layer21_attention_weight_max": 31.729217529296875,
      "activations/layer21_attention_weight_min": -25.393918991088867,
      "activations/layer22_attention_weight_max": 44.67732238769531,
      "activations/layer22_attention_weight_min": -33.248531341552734,
      "activations/layer23_attention_weight_max": 36.411163330078125,
      "activations/layer23_attention_weight_min": -28.415136337280273,
      "activations/layer2_attention_weight_max": 30.732206344604492,
      "activations/layer2_attention_weight_min": -31.031429290771484,
      "activations/layer3_attention_weight_max": 99.50418090820312,
      "activations/layer3_attention_weight_min": -98.55789947509766,
      "activations/layer4_attention_weight_max": 107.8230972290039,
      "activations/layer4_attention_weight_min": -105.6039047241211,
      "activations/layer5_attention_weight_max": 49.28742218017578,
      "activations/layer5_attention_weight_min": -61.07154846191406,
      "activations/layer6_attention_weight_max": 43.241878509521484,
      "activations/layer6_attention_weight_min": -45.062538146972656,
      "activations/layer7_attention_weight_max": 95.08119201660156,
      "activations/layer7_attention_weight_min": -96.55148315429688,
      "activations/layer8_attention_weight_max": 42.92526626586914,
      "activations/layer8_attention_weight_min": -42.184017181396484,
      "activations/layer9_attention_weight_max": 32.584815979003906,
      "activations/layer9_attention_weight_min": -33.656978607177734,
      "epoch": 20.28,
      "learning_rate": 1.9441287878787878e-05,
      "loss": 2.7273,
      "step": 348950
    },
    {
      "activations/layer0_attention_weight_max": 17.279945373535156,
      "activations/layer0_attention_weight_min": -13.317441940307617,
      "activations/layer10_attention_weight_max": 36.26499557495117,
      "activations/layer10_attention_weight_min": -36.103546142578125,
      "activations/layer11_attention_weight_max": 36.256866455078125,
      "activations/layer11_attention_weight_min": -37.61250305175781,
      "activations/layer12_attention_weight_max": 28.031858444213867,
      "activations/layer12_attention_weight_min": -27.470722198486328,
      "activations/layer13_attention_weight_max": 41.02989959716797,
      "activations/layer13_attention_weight_min": -37.079769134521484,
      "activations/layer14_attention_weight_max": 41.54084396362305,
      "activations/layer14_attention_weight_min": -40.7662467956543,
      "activations/layer15_attention_weight_max": 39.96449661254883,
      "activations/layer15_attention_weight_min": -36.31117630004883,
      "activations/layer16_attention_weight_max": 28.925046920776367,
      "activations/layer16_attention_weight_min": -28.588077545166016,
      "activations/layer17_attention_weight_max": 29.467859268188477,
      "activations/layer17_attention_weight_min": -27.520687103271484,
      "activations/layer18_attention_weight_max": 32.919647216796875,
      "activations/layer18_attention_weight_min": -26.417898178100586,
      "activations/layer19_attention_weight_max": 36.01953887939453,
      "activations/layer19_attention_weight_min": -34.991783142089844,
      "activations/layer1_attention_weight_max": 16.166431427001953,
      "activations/layer1_attention_weight_min": -14.89760971069336,
      "activations/layer20_attention_weight_max": 30.2202091217041,
      "activations/layer20_attention_weight_min": -27.004167556762695,
      "activations/layer21_attention_weight_max": 31.816240310668945,
      "activations/layer21_attention_weight_min": -28.147369384765625,
      "activations/layer22_attention_weight_max": 45.82984924316406,
      "activations/layer22_attention_weight_min": -36.47663497924805,
      "activations/layer23_attention_weight_max": 35.66987609863281,
      "activations/layer23_attention_weight_min": -30.109220504760742,
      "activations/layer2_attention_weight_max": 32.361793518066406,
      "activations/layer2_attention_weight_min": -33.94068908691406,
      "activations/layer3_attention_weight_max": 100.90522766113281,
      "activations/layer3_attention_weight_min": -101.01721954345703,
      "activations/layer4_attention_weight_max": 106.73677825927734,
      "activations/layer4_attention_weight_min": -107.9139633178711,
      "activations/layer5_attention_weight_max": 51.384761810302734,
      "activations/layer5_attention_weight_min": -65.46878051757812,
      "activations/layer6_attention_weight_max": 43.90520477294922,
      "activations/layer6_attention_weight_min": -48.13823318481445,
      "activations/layer7_attention_weight_max": 101.47520446777344,
      "activations/layer7_attention_weight_min": -98.22933197021484,
      "activations/layer8_attention_weight_max": 44.44463348388672,
      "activations/layer8_attention_weight_min": -44.58039855957031,
      "activations/layer9_attention_weight_max": 33.10023880004883,
      "activations/layer9_attention_weight_min": -35.92652893066406,
      "epoch": 20.28,
      "learning_rate": 1.9422348484848483e-05,
      "loss": 2.7063,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.4591,
      "eval_samples_per_second": 507.619,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.4591,
      "eval_openwebtext_samples_per_second": 507.619,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.962,
      "eval_wikitext_samples_per_second": 232.417,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_lambada_loss": 2.39453125,
      "eval_lambada_ppl": 10.963057921054194,
      "eval_lambada_runtime": 9.5393,
      "eval_lambada_samples_per_second": 510.413,
      "step": 349000
    },
    {
      "activations/layer0_attention_weight_max": 16.531856536865234,
      "activations/layer0_attention_weight_min": -12.57642936706543,
      "activations/layer10_attention_weight_max": 36.895206451416016,
      "activations/layer10_attention_weight_min": -35.558387756347656,
      "activations/layer11_attention_weight_max": 37.85654067993164,
      "activations/layer11_attention_weight_min": -36.5477180480957,
      "activations/layer12_attention_weight_max": 28.129255294799805,
      "activations/layer12_attention_weight_min": -26.764196395874023,
      "activations/layer13_attention_weight_max": 42.090877532958984,
      "activations/layer13_attention_weight_min": -38.03180694580078,
      "activations/layer14_attention_weight_max": 49.33737564086914,
      "activations/layer14_attention_weight_min": -42.196617126464844,
      "activations/layer15_attention_weight_max": 47.795841217041016,
      "activations/layer15_attention_weight_min": -40.82789993286133,
      "activations/layer16_attention_weight_max": 35.210880279541016,
      "activations/layer16_attention_weight_min": -28.280351638793945,
      "activations/layer17_attention_weight_max": 33.13446044921875,
      "activations/layer17_attention_weight_min": -28.91615867614746,
      "activations/layer18_attention_weight_max": 36.175472259521484,
      "activations/layer18_attention_weight_min": -27.93117904663086,
      "activations/layer19_attention_weight_max": 44.42340850830078,
      "activations/layer19_attention_weight_min": -33.20576477050781,
      "activations/layer1_attention_weight_max": 15.458170890808105,
      "activations/layer1_attention_weight_min": -14.651044845581055,
      "activations/layer20_attention_weight_max": 34.95152282714844,
      "activations/layer20_attention_weight_min": -25.07893943786621,
      "activations/layer21_attention_weight_max": 33.89715576171875,
      "activations/layer21_attention_weight_min": -24.708898544311523,
      "activations/layer22_attention_weight_max": 50.4594841003418,
      "activations/layer22_attention_weight_min": -35.29350280761719,
      "activations/layer23_attention_weight_max": 37.02095413208008,
      "activations/layer23_attention_weight_min": -26.215877532958984,
      "activations/layer2_attention_weight_max": 35.35852813720703,
      "activations/layer2_attention_weight_min": -32.657344818115234,
      "activations/layer3_attention_weight_max": 101.43424987792969,
      "activations/layer3_attention_weight_min": -102.3471908569336,
      "activations/layer4_attention_weight_max": 105.00847625732422,
      "activations/layer4_attention_weight_min": -105.50141906738281,
      "activations/layer5_attention_weight_max": 52.689964294433594,
      "activations/layer5_attention_weight_min": -59.075294494628906,
      "activations/layer6_attention_weight_max": 44.88515853881836,
      "activations/layer6_attention_weight_min": -43.8971061706543,
      "activations/layer7_attention_weight_max": 96.76033020019531,
      "activations/layer7_attention_weight_min": -97.4302749633789,
      "activations/layer8_attention_weight_max": 43.40835952758789,
      "activations/layer8_attention_weight_min": -42.855010986328125,
      "activations/layer9_attention_weight_max": 34.357601165771484,
      "activations/layer9_attention_weight_min": -34.8394660949707,
      "epoch": 20.28,
      "learning_rate": 1.940340909090909e-05,
      "loss": 2.7172,
      "step": 349050
    },
    {
      "activations/layer0_attention_weight_max": 16.026607513427734,
      "activations/layer0_attention_weight_min": -13.4915771484375,
      "activations/layer10_attention_weight_max": 34.905059814453125,
      "activations/layer10_attention_weight_min": -35.83129119873047,
      "activations/layer11_attention_weight_max": 36.49945831298828,
      "activations/layer11_attention_weight_min": -35.180999755859375,
      "activations/layer12_attention_weight_max": 28.42009162902832,
      "activations/layer12_attention_weight_min": -27.115840911865234,
      "activations/layer13_attention_weight_max": 40.951534271240234,
      "activations/layer13_attention_weight_min": -36.762229919433594,
      "activations/layer14_attention_weight_max": 46.5009880065918,
      "activations/layer14_attention_weight_min": -40.898406982421875,
      "activations/layer15_attention_weight_max": 42.372779846191406,
      "activations/layer15_attention_weight_min": -36.797000885009766,
      "activations/layer16_attention_weight_max": 32.131526947021484,
      "activations/layer16_attention_weight_min": -28.74504852294922,
      "activations/layer17_attention_weight_max": 31.90845489501953,
      "activations/layer17_attention_weight_min": -26.219934463500977,
      "activations/layer18_attention_weight_max": 33.74171829223633,
      "activations/layer18_attention_weight_min": -27.198848724365234,
      "activations/layer19_attention_weight_max": 36.039852142333984,
      "activations/layer19_attention_weight_min": -32.36381912231445,
      "activations/layer1_attention_weight_max": 15.248034477233887,
      "activations/layer1_attention_weight_min": -14.396448135375977,
      "activations/layer20_attention_weight_max": 29.458311080932617,
      "activations/layer20_attention_weight_min": -26.912845611572266,
      "activations/layer21_attention_weight_max": 29.321121215820312,
      "activations/layer21_attention_weight_min": -26.0123348236084,
      "activations/layer22_attention_weight_max": 43.2276611328125,
      "activations/layer22_attention_weight_min": -31.095062255859375,
      "activations/layer23_attention_weight_max": 33.38860321044922,
      "activations/layer23_attention_weight_min": -30.938453674316406,
      "activations/layer2_attention_weight_max": 33.08049392700195,
      "activations/layer2_attention_weight_min": -32.30460739135742,
      "activations/layer3_attention_weight_max": 98.7746353149414,
      "activations/layer3_attention_weight_min": -101.60202026367188,
      "activations/layer4_attention_weight_max": 104.86614227294922,
      "activations/layer4_attention_weight_min": -106.69066619873047,
      "activations/layer5_attention_weight_max": 49.369529724121094,
      "activations/layer5_attention_weight_min": -62.38872528076172,
      "activations/layer6_attention_weight_max": 44.4412727355957,
      "activations/layer6_attention_weight_min": -47.94944381713867,
      "activations/layer7_attention_weight_max": 94.39286041259766,
      "activations/layer7_attention_weight_min": -96.32749938964844,
      "activations/layer8_attention_weight_max": 41.84193801879883,
      "activations/layer8_attention_weight_min": -46.46043395996094,
      "activations/layer9_attention_weight_max": 33.69874954223633,
      "activations/layer9_attention_weight_min": -34.123931884765625,
      "epoch": 20.28,
      "learning_rate": 1.9384469696969692e-05,
      "loss": 2.7103,
      "step": 349100
    },
    {
      "activations/layer0_attention_weight_max": 15.691609382629395,
      "activations/layer0_attention_weight_min": -12.906970977783203,
      "activations/layer10_attention_weight_max": 36.9177131652832,
      "activations/layer10_attention_weight_min": -39.2193717956543,
      "activations/layer11_attention_weight_max": 38.80064392089844,
      "activations/layer11_attention_weight_min": -37.2325553894043,
      "activations/layer12_attention_weight_max": 29.300310134887695,
      "activations/layer12_attention_weight_min": -29.738859176635742,
      "activations/layer13_attention_weight_max": 39.92303466796875,
      "activations/layer13_attention_weight_min": -38.15428924560547,
      "activations/layer14_attention_weight_max": 46.2696533203125,
      "activations/layer14_attention_weight_min": -43.83895492553711,
      "activations/layer15_attention_weight_max": 41.87773132324219,
      "activations/layer15_attention_weight_min": -41.383705139160156,
      "activations/layer16_attention_weight_max": 31.93076515197754,
      "activations/layer16_attention_weight_min": -31.292146682739258,
      "activations/layer17_attention_weight_max": 30.572566986083984,
      "activations/layer17_attention_weight_min": -30.005571365356445,
      "activations/layer18_attention_weight_max": 34.49614715576172,
      "activations/layer18_attention_weight_min": -28.663259506225586,
      "activations/layer19_attention_weight_max": 35.6956901550293,
      "activations/layer19_attention_weight_min": -35.188114166259766,
      "activations/layer1_attention_weight_max": 16.136869430541992,
      "activations/layer1_attention_weight_min": -14.393122673034668,
      "activations/layer20_attention_weight_max": 30.280410766601562,
      "activations/layer20_attention_weight_min": -27.581377029418945,
      "activations/layer21_attention_weight_max": 31.604764938354492,
      "activations/layer21_attention_weight_min": -26.436357498168945,
      "activations/layer22_attention_weight_max": 49.142234802246094,
      "activations/layer22_attention_weight_min": -36.10893249511719,
      "activations/layer23_attention_weight_max": 34.134132385253906,
      "activations/layer23_attention_weight_min": -28.948001861572266,
      "activations/layer2_attention_weight_max": 33.467987060546875,
      "activations/layer2_attention_weight_min": -32.78131103515625,
      "activations/layer3_attention_weight_max": 104.52899169921875,
      "activations/layer3_attention_weight_min": -105.77718353271484,
      "activations/layer4_attention_weight_max": 106.03917694091797,
      "activations/layer4_attention_weight_min": -116.46404266357422,
      "activations/layer5_attention_weight_max": 49.637603759765625,
      "activations/layer5_attention_weight_min": -63.616661071777344,
      "activations/layer6_attention_weight_max": 44.57088851928711,
      "activations/layer6_attention_weight_min": -49.56642532348633,
      "activations/layer7_attention_weight_max": 94.4767074584961,
      "activations/layer7_attention_weight_min": -103.66365051269531,
      "activations/layer8_attention_weight_max": 42.141300201416016,
      "activations/layer8_attention_weight_min": -48.57709503173828,
      "activations/layer9_attention_weight_max": 33.70151138305664,
      "activations/layer9_attention_weight_min": -35.081321716308594,
      "epoch": 20.29,
      "learning_rate": 1.93655303030303e-05,
      "loss": 2.7225,
      "step": 349150
    },
    {
      "activations/layer0_attention_weight_max": 15.690276145935059,
      "activations/layer0_attention_weight_min": -13.413049697875977,
      "activations/layer10_attention_weight_max": 42.0179443359375,
      "activations/layer10_attention_weight_min": -40.148468017578125,
      "activations/layer11_attention_weight_max": 40.51394271850586,
      "activations/layer11_attention_weight_min": -40.347686767578125,
      "activations/layer12_attention_weight_max": 28.092655181884766,
      "activations/layer12_attention_weight_min": -28.876522064208984,
      "activations/layer13_attention_weight_max": 40.09894943237305,
      "activations/layer13_attention_weight_min": -37.56127166748047,
      "activations/layer14_attention_weight_max": 46.32741165161133,
      "activations/layer14_attention_weight_min": -41.304039001464844,
      "activations/layer15_attention_weight_max": 41.74971008300781,
      "activations/layer15_attention_weight_min": -43.72089767456055,
      "activations/layer16_attention_weight_max": 33.41580581665039,
      "activations/layer16_attention_weight_min": -29.87561798095703,
      "activations/layer17_attention_weight_max": 31.509904861450195,
      "activations/layer17_attention_weight_min": -27.2480411529541,
      "activations/layer18_attention_weight_max": 34.44960403442383,
      "activations/layer18_attention_weight_min": -29.09835433959961,
      "activations/layer19_attention_weight_max": 36.05249786376953,
      "activations/layer19_attention_weight_min": -32.13011169433594,
      "activations/layer1_attention_weight_max": 17.110946655273438,
      "activations/layer1_attention_weight_min": -15.523214340209961,
      "activations/layer20_attention_weight_max": 30.55492401123047,
      "activations/layer20_attention_weight_min": -27.104270935058594,
      "activations/layer21_attention_weight_max": 30.039464950561523,
      "activations/layer21_attention_weight_min": -26.911638259887695,
      "activations/layer22_attention_weight_max": 53.47927474975586,
      "activations/layer22_attention_weight_min": -33.28729248046875,
      "activations/layer23_attention_weight_max": 34.64839553833008,
      "activations/layer23_attention_weight_min": -26.5659236907959,
      "activations/layer2_attention_weight_max": 31.905048370361328,
      "activations/layer2_attention_weight_min": -33.65978240966797,
      "activations/layer3_attention_weight_max": 99.4954833984375,
      "activations/layer3_attention_weight_min": -103.65052032470703,
      "activations/layer4_attention_weight_max": 100.69134521484375,
      "activations/layer4_attention_weight_min": -111.5262451171875,
      "activations/layer5_attention_weight_max": 47.003173828125,
      "activations/layer5_attention_weight_min": -61.157798767089844,
      "activations/layer6_attention_weight_max": 44.368751525878906,
      "activations/layer6_attention_weight_min": -48.630393981933594,
      "activations/layer7_attention_weight_max": 98.72834014892578,
      "activations/layer7_attention_weight_min": -104.83749389648438,
      "activations/layer8_attention_weight_max": 45.2881965637207,
      "activations/layer8_attention_weight_min": -51.477046966552734,
      "activations/layer9_attention_weight_max": 36.61697006225586,
      "activations/layer9_attention_weight_min": -38.71219253540039,
      "epoch": 20.29,
      "learning_rate": 1.9346590909090906e-05,
      "loss": 2.7163,
      "step": 349200
    },
    {
      "activations/layer0_attention_weight_max": 16.31662940979004,
      "activations/layer0_attention_weight_min": -13.667439460754395,
      "activations/layer10_attention_weight_max": 33.93439483642578,
      "activations/layer10_attention_weight_min": -35.34653091430664,
      "activations/layer11_attention_weight_max": 33.8042106628418,
      "activations/layer11_attention_weight_min": -35.26670837402344,
      "activations/layer12_attention_weight_max": 26.792509078979492,
      "activations/layer12_attention_weight_min": -26.287309646606445,
      "activations/layer13_attention_weight_max": 41.920528411865234,
      "activations/layer13_attention_weight_min": -34.5833625793457,
      "activations/layer14_attention_weight_max": 44.55902862548828,
      "activations/layer14_attention_weight_min": -37.40546417236328,
      "activations/layer15_attention_weight_max": 38.95490264892578,
      "activations/layer15_attention_weight_min": -36.1812858581543,
      "activations/layer16_attention_weight_max": 31.04179573059082,
      "activations/layer16_attention_weight_min": -29.126707077026367,
      "activations/layer17_attention_weight_max": 30.838783264160156,
      "activations/layer17_attention_weight_min": -27.395084381103516,
      "activations/layer18_attention_weight_max": 30.62824821472168,
      "activations/layer18_attention_weight_min": -26.35616683959961,
      "activations/layer19_attention_weight_max": 33.98299789428711,
      "activations/layer19_attention_weight_min": -30.28781509399414,
      "activations/layer1_attention_weight_max": 15.582429885864258,
      "activations/layer1_attention_weight_min": -14.623266220092773,
      "activations/layer20_attention_weight_max": 27.288299560546875,
      "activations/layer20_attention_weight_min": -26.220457077026367,
      "activations/layer21_attention_weight_max": 27.550405502319336,
      "activations/layer21_attention_weight_min": -24.745473861694336,
      "activations/layer22_attention_weight_max": 43.77109146118164,
      "activations/layer22_attention_weight_min": -29.748809814453125,
      "activations/layer23_attention_weight_max": 31.077781677246094,
      "activations/layer23_attention_weight_min": -26.094205856323242,
      "activations/layer2_attention_weight_max": 31.989505767822266,
      "activations/layer2_attention_weight_min": -33.01331329345703,
      "activations/layer3_attention_weight_max": 100.47765350341797,
      "activations/layer3_attention_weight_min": -105.00772857666016,
      "activations/layer4_attention_weight_max": 108.05313873291016,
      "activations/layer4_attention_weight_min": -105.4408187866211,
      "activations/layer5_attention_weight_max": 48.4107780456543,
      "activations/layer5_attention_weight_min": -59.00431823730469,
      "activations/layer6_attention_weight_max": 42.76724624633789,
      "activations/layer6_attention_weight_min": -48.380836486816406,
      "activations/layer7_attention_weight_max": 89.60027313232422,
      "activations/layer7_attention_weight_min": -101.16007232666016,
      "activations/layer8_attention_weight_max": 41.841983795166016,
      "activations/layer8_attention_weight_min": -46.77070617675781,
      "activations/layer9_attention_weight_max": 30.80501365661621,
      "activations/layer9_attention_weight_min": -32.96279525756836,
      "epoch": 20.29,
      "learning_rate": 1.9327651515151514e-05,
      "loss": 2.7147,
      "step": 349250
    },
    {
      "activations/layer0_attention_weight_max": 16.125993728637695,
      "activations/layer0_attention_weight_min": -13.114054679870605,
      "activations/layer10_attention_weight_max": 34.04931640625,
      "activations/layer10_attention_weight_min": -35.46965789794922,
      "activations/layer11_attention_weight_max": 34.2164192199707,
      "activations/layer11_attention_weight_min": -33.93351364135742,
      "activations/layer12_attention_weight_max": 25.882455825805664,
      "activations/layer12_attention_weight_min": -26.475671768188477,
      "activations/layer13_attention_weight_max": 39.35527801513672,
      "activations/layer13_attention_weight_min": -36.19636154174805,
      "activations/layer14_attention_weight_max": 40.22886657714844,
      "activations/layer14_attention_weight_min": -38.04349136352539,
      "activations/layer15_attention_weight_max": 38.8916015625,
      "activations/layer15_attention_weight_min": -37.22595977783203,
      "activations/layer16_attention_weight_max": 26.973039627075195,
      "activations/layer16_attention_weight_min": -28.32326889038086,
      "activations/layer17_attention_weight_max": 30.51266098022461,
      "activations/layer17_attention_weight_min": -25.971166610717773,
      "activations/layer18_attention_weight_max": 32.645755767822266,
      "activations/layer18_attention_weight_min": -26.230093002319336,
      "activations/layer19_attention_weight_max": 31.721540451049805,
      "activations/layer19_attention_weight_min": -31.044315338134766,
      "activations/layer1_attention_weight_max": 16.168344497680664,
      "activations/layer1_attention_weight_min": -15.114473342895508,
      "activations/layer20_attention_weight_max": 26.9637393951416,
      "activations/layer20_attention_weight_min": -25.948001861572266,
      "activations/layer21_attention_weight_max": 27.133520126342773,
      "activations/layer21_attention_weight_min": -25.412612915039062,
      "activations/layer22_attention_weight_max": 39.010799407958984,
      "activations/layer22_attention_weight_min": -33.46894073486328,
      "activations/layer23_attention_weight_max": 32.33855438232422,
      "activations/layer23_attention_weight_min": -27.11774444580078,
      "activations/layer2_attention_weight_max": 34.41472625732422,
      "activations/layer2_attention_weight_min": -32.19892883300781,
      "activations/layer3_attention_weight_max": 103.12334442138672,
      "activations/layer3_attention_weight_min": -103.05645751953125,
      "activations/layer4_attention_weight_max": 109.59864807128906,
      "activations/layer4_attention_weight_min": -110.00032043457031,
      "activations/layer5_attention_weight_max": 50.78253936767578,
      "activations/layer5_attention_weight_min": -59.10356140136719,
      "activations/layer6_attention_weight_max": 43.98777389526367,
      "activations/layer6_attention_weight_min": -45.268104553222656,
      "activations/layer7_attention_weight_max": 98.05574798583984,
      "activations/layer7_attention_weight_min": -99.36239624023438,
      "activations/layer8_attention_weight_max": 42.54658508300781,
      "activations/layer8_attention_weight_min": -45.00927734375,
      "activations/layer9_attention_weight_max": 32.38364028930664,
      "activations/layer9_attention_weight_min": -35.14836883544922,
      "epoch": 20.3,
      "learning_rate": 1.930871212121212e-05,
      "loss": 2.6997,
      "step": 349300
    },
    {
      "activations/layer0_attention_weight_max": 16.13724708557129,
      "activations/layer0_attention_weight_min": -13.248766899108887,
      "activations/layer10_attention_weight_max": 40.9710578918457,
      "activations/layer10_attention_weight_min": -35.7833137512207,
      "activations/layer11_attention_weight_max": 41.67597961425781,
      "activations/layer11_attention_weight_min": -36.53156280517578,
      "activations/layer12_attention_weight_max": 27.864185333251953,
      "activations/layer12_attention_weight_min": -26.90049171447754,
      "activations/layer13_attention_weight_max": 41.70421600341797,
      "activations/layer13_attention_weight_min": -37.4197883605957,
      "activations/layer14_attention_weight_max": 42.82229232788086,
      "activations/layer14_attention_weight_min": -38.86735534667969,
      "activations/layer15_attention_weight_max": 44.031341552734375,
      "activations/layer15_attention_weight_min": -37.29636764526367,
      "activations/layer16_attention_weight_max": 31.27420997619629,
      "activations/layer16_attention_weight_min": -28.280235290527344,
      "activations/layer17_attention_weight_max": 32.76300811767578,
      "activations/layer17_attention_weight_min": -26.891260147094727,
      "activations/layer18_attention_weight_max": 31.25263786315918,
      "activations/layer18_attention_weight_min": -26.384502410888672,
      "activations/layer19_attention_weight_max": 33.06122970581055,
      "activations/layer19_attention_weight_min": -30.555923461914062,
      "activations/layer1_attention_weight_max": 16.270225524902344,
      "activations/layer1_attention_weight_min": -14.900598526000977,
      "activations/layer20_attention_weight_max": 28.488679885864258,
      "activations/layer20_attention_weight_min": -25.898143768310547,
      "activations/layer21_attention_weight_max": 29.339340209960938,
      "activations/layer21_attention_weight_min": -25.338603973388672,
      "activations/layer22_attention_weight_max": 39.63420486450195,
      "activations/layer22_attention_weight_min": -31.92835235595703,
      "activations/layer23_attention_weight_max": 30.495161056518555,
      "activations/layer23_attention_weight_min": -27.059629440307617,
      "activations/layer2_attention_weight_max": 33.037933349609375,
      "activations/layer2_attention_weight_min": -33.572540283203125,
      "activations/layer3_attention_weight_max": 102.92063903808594,
      "activations/layer3_attention_weight_min": -103.22315216064453,
      "activations/layer4_attention_weight_max": 110.54647064208984,
      "activations/layer4_attention_weight_min": -110.3123550415039,
      "activations/layer5_attention_weight_max": 54.38109588623047,
      "activations/layer5_attention_weight_min": -62.19346618652344,
      "activations/layer6_attention_weight_max": 46.73908615112305,
      "activations/layer6_attention_weight_min": -49.63603973388672,
      "activations/layer7_attention_weight_max": 98.4594955444336,
      "activations/layer7_attention_weight_min": -101.77163696289062,
      "activations/layer8_attention_weight_max": 43.74855422973633,
      "activations/layer8_attention_weight_min": -46.33781051635742,
      "activations/layer9_attention_weight_max": 34.5327262878418,
      "activations/layer9_attention_weight_min": -35.43215560913086,
      "epoch": 20.3,
      "learning_rate": 1.9289772727272727e-05,
      "loss": 2.7146,
      "step": 349350
    },
    {
      "activations/layer0_attention_weight_max": 16.93967056274414,
      "activations/layer0_attention_weight_min": -13.966413497924805,
      "activations/layer10_attention_weight_max": 39.36394119262695,
      "activations/layer10_attention_weight_min": -37.98739242553711,
      "activations/layer11_attention_weight_max": 36.806617736816406,
      "activations/layer11_attention_weight_min": -36.64220428466797,
      "activations/layer12_attention_weight_max": 28.319252014160156,
      "activations/layer12_attention_weight_min": -30.743436813354492,
      "activations/layer13_attention_weight_max": 40.50271224975586,
      "activations/layer13_attention_weight_min": -38.68736267089844,
      "activations/layer14_attention_weight_max": 43.187747955322266,
      "activations/layer14_attention_weight_min": -42.098270416259766,
      "activations/layer15_attention_weight_max": 40.201080322265625,
      "activations/layer15_attention_weight_min": -39.188446044921875,
      "activations/layer16_attention_weight_max": 31.117525100708008,
      "activations/layer16_attention_weight_min": -28.89201545715332,
      "activations/layer17_attention_weight_max": 28.999311447143555,
      "activations/layer17_attention_weight_min": -25.099899291992188,
      "activations/layer18_attention_weight_max": 33.120059967041016,
      "activations/layer18_attention_weight_min": -26.174413681030273,
      "activations/layer19_attention_weight_max": 32.248374938964844,
      "activations/layer19_attention_weight_min": -33.341636657714844,
      "activations/layer1_attention_weight_max": 16.434328079223633,
      "activations/layer1_attention_weight_min": -15.4579439163208,
      "activations/layer20_attention_weight_max": 29.050758361816406,
      "activations/layer20_attention_weight_min": -26.7723445892334,
      "activations/layer21_attention_weight_max": 28.026567459106445,
      "activations/layer21_attention_weight_min": -28.416757583618164,
      "activations/layer22_attention_weight_max": 39.46269989013672,
      "activations/layer22_attention_weight_min": -32.655582427978516,
      "activations/layer23_attention_weight_max": 29.39558982849121,
      "activations/layer23_attention_weight_min": -27.049985885620117,
      "activations/layer2_attention_weight_max": 33.84889602661133,
      "activations/layer2_attention_weight_min": -32.069122314453125,
      "activations/layer3_attention_weight_max": 104.6224594116211,
      "activations/layer3_attention_weight_min": -102.333251953125,
      "activations/layer4_attention_weight_max": 110.65567779541016,
      "activations/layer4_attention_weight_min": -109.8641357421875,
      "activations/layer5_attention_weight_max": 51.9621696472168,
      "activations/layer5_attention_weight_min": -62.26481628417969,
      "activations/layer6_attention_weight_max": 48.22025680541992,
      "activations/layer6_attention_weight_min": -48.937355041503906,
      "activations/layer7_attention_weight_max": 102.95222473144531,
      "activations/layer7_attention_weight_min": -101.43207550048828,
      "activations/layer8_attention_weight_max": 46.76985168457031,
      "activations/layer8_attention_weight_min": -49.03858184814453,
      "activations/layer9_attention_weight_max": 36.82145309448242,
      "activations/layer9_attention_weight_min": -36.22779846191406,
      "epoch": 20.3,
      "learning_rate": 1.927083333333333e-05,
      "loss": 2.7245,
      "step": 349400
    },
    {
      "activations/layer0_attention_weight_max": 16.689008712768555,
      "activations/layer0_attention_weight_min": -13.175146102905273,
      "activations/layer10_attention_weight_max": 34.23967361450195,
      "activations/layer10_attention_weight_min": -35.860992431640625,
      "activations/layer11_attention_weight_max": 34.77165222167969,
      "activations/layer11_attention_weight_min": -35.90253448486328,
      "activations/layer12_attention_weight_max": 25.549015045166016,
      "activations/layer12_attention_weight_min": -27.780031204223633,
      "activations/layer13_attention_weight_max": 36.78522872924805,
      "activations/layer13_attention_weight_min": -35.90239334106445,
      "activations/layer14_attention_weight_max": 42.634342193603516,
      "activations/layer14_attention_weight_min": -37.66384506225586,
      "activations/layer15_attention_weight_max": 36.3752326965332,
      "activations/layer15_attention_weight_min": -36.108821868896484,
      "activations/layer16_attention_weight_max": 30.32948875427246,
      "activations/layer16_attention_weight_min": -28.559091567993164,
      "activations/layer17_attention_weight_max": 30.161664962768555,
      "activations/layer17_attention_weight_min": -27.293298721313477,
      "activations/layer18_attention_weight_max": 32.44506072998047,
      "activations/layer18_attention_weight_min": -26.694068908691406,
      "activations/layer19_attention_weight_max": 34.06757736206055,
      "activations/layer19_attention_weight_min": -32.90523910522461,
      "activations/layer1_attention_weight_max": 15.810590744018555,
      "activations/layer1_attention_weight_min": -14.381261825561523,
      "activations/layer20_attention_weight_max": 29.118711471557617,
      "activations/layer20_attention_weight_min": -25.284574508666992,
      "activations/layer21_attention_weight_max": 30.608684539794922,
      "activations/layer21_attention_weight_min": -25.57415199279785,
      "activations/layer22_attention_weight_max": 40.774436950683594,
      "activations/layer22_attention_weight_min": -32.36777877807617,
      "activations/layer23_attention_weight_max": 31.642528533935547,
      "activations/layer23_attention_weight_min": -26.97716522216797,
      "activations/layer2_attention_weight_max": 32.20170974731445,
      "activations/layer2_attention_weight_min": -31.903709411621094,
      "activations/layer3_attention_weight_max": 98.19296264648438,
      "activations/layer3_attention_weight_min": -101.55985260009766,
      "activations/layer4_attention_weight_max": 103.39322662353516,
      "activations/layer4_attention_weight_min": -110.40084075927734,
      "activations/layer5_attention_weight_max": 51.53138732910156,
      "activations/layer5_attention_weight_min": -61.56447982788086,
      "activations/layer6_attention_weight_max": 44.74428939819336,
      "activations/layer6_attention_weight_min": -46.68301773071289,
      "activations/layer7_attention_weight_max": 91.63225555419922,
      "activations/layer7_attention_weight_min": -97.22737121582031,
      "activations/layer8_attention_weight_max": 42.794490814208984,
      "activations/layer8_attention_weight_min": -45.30348205566406,
      "activations/layer9_attention_weight_max": 32.281158447265625,
      "activations/layer9_attention_weight_min": -34.738399505615234,
      "epoch": 20.31,
      "learning_rate": 1.9251893939393937e-05,
      "loss": 2.7009,
      "step": 349450
    },
    {
      "activations/layer0_attention_weight_max": 16.690799713134766,
      "activations/layer0_attention_weight_min": -13.11194896697998,
      "activations/layer10_attention_weight_max": 36.257667541503906,
      "activations/layer10_attention_weight_min": -36.084320068359375,
      "activations/layer11_attention_weight_max": 38.12458038330078,
      "activations/layer11_attention_weight_min": -38.49861145019531,
      "activations/layer12_attention_weight_max": 26.919858932495117,
      "activations/layer12_attention_weight_min": -26.720035552978516,
      "activations/layer13_attention_weight_max": 37.74554443359375,
      "activations/layer13_attention_weight_min": -34.965267181396484,
      "activations/layer14_attention_weight_max": 41.69388198852539,
      "activations/layer14_attention_weight_min": -36.69252395629883,
      "activations/layer15_attention_weight_max": 36.933387756347656,
      "activations/layer15_attention_weight_min": -35.922210693359375,
      "activations/layer16_attention_weight_max": 27.511837005615234,
      "activations/layer16_attention_weight_min": -28.910322189331055,
      "activations/layer17_attention_weight_max": 28.141515731811523,
      "activations/layer17_attention_weight_min": -26.07779884338379,
      "activations/layer18_attention_weight_max": 32.36863708496094,
      "activations/layer18_attention_weight_min": -27.183837890625,
      "activations/layer19_attention_weight_max": 33.7873649597168,
      "activations/layer19_attention_weight_min": -31.77435302734375,
      "activations/layer1_attention_weight_max": 16.390396118164062,
      "activations/layer1_attention_weight_min": -15.834150314331055,
      "activations/layer20_attention_weight_max": 27.616527557373047,
      "activations/layer20_attention_weight_min": -27.159496307373047,
      "activations/layer21_attention_weight_max": 27.91032600402832,
      "activations/layer21_attention_weight_min": -25.328933715820312,
      "activations/layer22_attention_weight_max": 46.87864303588867,
      "activations/layer22_attention_weight_min": -34.26881790161133,
      "activations/layer23_attention_weight_max": 29.523639678955078,
      "activations/layer23_attention_weight_min": -26.294998168945312,
      "activations/layer2_attention_weight_max": 33.98524475097656,
      "activations/layer2_attention_weight_min": -32.73811340332031,
      "activations/layer3_attention_weight_max": 102.84268951416016,
      "activations/layer3_attention_weight_min": -104.48054504394531,
      "activations/layer4_attention_weight_max": 105.6507568359375,
      "activations/layer4_attention_weight_min": -109.25969696044922,
      "activations/layer5_attention_weight_max": 49.52427291870117,
      "activations/layer5_attention_weight_min": -62.105525970458984,
      "activations/layer6_attention_weight_max": 45.71727752685547,
      "activations/layer6_attention_weight_min": -47.630226135253906,
      "activations/layer7_attention_weight_max": 92.86964416503906,
      "activations/layer7_attention_weight_min": -99.80074310302734,
      "activations/layer8_attention_weight_max": 41.91646194458008,
      "activations/layer8_attention_weight_min": -44.27617263793945,
      "activations/layer9_attention_weight_max": 33.889183044433594,
      "activations/layer9_attention_weight_min": -35.91559982299805,
      "epoch": 20.31,
      "learning_rate": 1.9232954545454545e-05,
      "loss": 2.7041,
      "step": 349500
    },
    {
      "activations/layer0_attention_weight_max": 16.400394439697266,
      "activations/layer0_attention_weight_min": -14.01591968536377,
      "activations/layer10_attention_weight_max": 33.46778106689453,
      "activations/layer10_attention_weight_min": -33.34344482421875,
      "activations/layer11_attention_weight_max": 34.448097229003906,
      "activations/layer11_attention_weight_min": -34.779380798339844,
      "activations/layer12_attention_weight_max": 26.146696090698242,
      "activations/layer12_attention_weight_min": -25.763277053833008,
      "activations/layer13_attention_weight_max": 40.01234436035156,
      "activations/layer13_attention_weight_min": -36.14598846435547,
      "activations/layer14_attention_weight_max": 42.9728889465332,
      "activations/layer14_attention_weight_min": -37.79533767700195,
      "activations/layer15_attention_weight_max": 40.43687438964844,
      "activations/layer15_attention_weight_min": -35.46489334106445,
      "activations/layer16_attention_weight_max": 28.897586822509766,
      "activations/layer16_attention_weight_min": -29.177610397338867,
      "activations/layer17_attention_weight_max": 30.51580238342285,
      "activations/layer17_attention_weight_min": -25.865209579467773,
      "activations/layer18_attention_weight_max": 37.02458953857422,
      "activations/layer18_attention_weight_min": -26.173080444335938,
      "activations/layer19_attention_weight_max": 39.55252456665039,
      "activations/layer19_attention_weight_min": -32.6395149230957,
      "activations/layer1_attention_weight_max": 15.712355613708496,
      "activations/layer1_attention_weight_min": -14.38608455657959,
      "activations/layer20_attention_weight_max": 31.822052001953125,
      "activations/layer20_attention_weight_min": -25.38759422302246,
      "activations/layer21_attention_weight_max": 29.756969451904297,
      "activations/layer21_attention_weight_min": -25.336631774902344,
      "activations/layer22_attention_weight_max": 49.0161247253418,
      "activations/layer22_attention_weight_min": -33.33536911010742,
      "activations/layer23_attention_weight_max": 37.23773956298828,
      "activations/layer23_attention_weight_min": -26.14723014831543,
      "activations/layer2_attention_weight_max": 33.102447509765625,
      "activations/layer2_attention_weight_min": -31.469261169433594,
      "activations/layer3_attention_weight_max": 98.64776611328125,
      "activations/layer3_attention_weight_min": -100.45686340332031,
      "activations/layer4_attention_weight_max": 106.65813446044922,
      "activations/layer4_attention_weight_min": -108.70245361328125,
      "activations/layer5_attention_weight_max": 47.470458984375,
      "activations/layer5_attention_weight_min": -59.210052490234375,
      "activations/layer6_attention_weight_max": 42.65830993652344,
      "activations/layer6_attention_weight_min": -46.41636657714844,
      "activations/layer7_attention_weight_max": 92.85982513427734,
      "activations/layer7_attention_weight_min": -93.02527618408203,
      "activations/layer8_attention_weight_max": 40.85547637939453,
      "activations/layer8_attention_weight_min": -43.37103271484375,
      "activations/layer9_attention_weight_max": 31.020679473876953,
      "activations/layer9_attention_weight_min": -33.83651351928711,
      "epoch": 20.31,
      "learning_rate": 1.921401515151515e-05,
      "loss": 2.715,
      "step": 349550
    },
    {
      "activations/layer0_attention_weight_max": 15.700045585632324,
      "activations/layer0_attention_weight_min": -13.852282524108887,
      "activations/layer10_attention_weight_max": 32.55202865600586,
      "activations/layer10_attention_weight_min": -34.81060791015625,
      "activations/layer11_attention_weight_max": 34.43964385986328,
      "activations/layer11_attention_weight_min": -34.060508728027344,
      "activations/layer12_attention_weight_max": 25.506567001342773,
      "activations/layer12_attention_weight_min": -27.368051528930664,
      "activations/layer13_attention_weight_max": 39.204769134521484,
      "activations/layer13_attention_weight_min": -37.61760330200195,
      "activations/layer14_attention_weight_max": 40.06840896606445,
      "activations/layer14_attention_weight_min": -38.37704849243164,
      "activations/layer15_attention_weight_max": 36.87373733520508,
      "activations/layer15_attention_weight_min": -36.17449188232422,
      "activations/layer16_attention_weight_max": 26.42243003845215,
      "activations/layer16_attention_weight_min": -26.888259887695312,
      "activations/layer17_attention_weight_max": 27.163116455078125,
      "activations/layer17_attention_weight_min": -24.724950790405273,
      "activations/layer18_attention_weight_max": 29.488271713256836,
      "activations/layer18_attention_weight_min": -25.424028396606445,
      "activations/layer19_attention_weight_max": 32.87173080444336,
      "activations/layer19_attention_weight_min": -30.3343505859375,
      "activations/layer1_attention_weight_max": 15.532833099365234,
      "activations/layer1_attention_weight_min": -14.016624450683594,
      "activations/layer20_attention_weight_max": 26.43447494506836,
      "activations/layer20_attention_weight_min": -24.638017654418945,
      "activations/layer21_attention_weight_max": 30.713096618652344,
      "activations/layer21_attention_weight_min": -24.881059646606445,
      "activations/layer22_attention_weight_max": 39.29979705810547,
      "activations/layer22_attention_weight_min": -32.739540100097656,
      "activations/layer23_attention_weight_max": 31.230022430419922,
      "activations/layer23_attention_weight_min": -24.572872161865234,
      "activations/layer2_attention_weight_max": 32.19077682495117,
      "activations/layer2_attention_weight_min": -31.499080657958984,
      "activations/layer3_attention_weight_max": 102.08171844482422,
      "activations/layer3_attention_weight_min": -103.87760162353516,
      "activations/layer4_attention_weight_max": 105.8641357421875,
      "activations/layer4_attention_weight_min": -112.1656723022461,
      "activations/layer5_attention_weight_max": 50.28078842163086,
      "activations/layer5_attention_weight_min": -59.933372497558594,
      "activations/layer6_attention_weight_max": 44.57965850830078,
      "activations/layer6_attention_weight_min": -49.17145919799805,
      "activations/layer7_attention_weight_max": 95.74852752685547,
      "activations/layer7_attention_weight_min": -97.20714569091797,
      "activations/layer8_attention_weight_max": 40.58814239501953,
      "activations/layer8_attention_weight_min": -44.23065185546875,
      "activations/layer9_attention_weight_max": 30.697050094604492,
      "activations/layer9_attention_weight_min": -33.37171936035156,
      "epoch": 20.31,
      "learning_rate": 1.9195075757575758e-05,
      "loss": 2.7294,
      "step": 349600
    },
    {
      "activations/layer0_attention_weight_max": 16.288591384887695,
      "activations/layer0_attention_weight_min": -12.881830215454102,
      "activations/layer10_attention_weight_max": 33.965152740478516,
      "activations/layer10_attention_weight_min": -35.1015625,
      "activations/layer11_attention_weight_max": 33.73368835449219,
      "activations/layer11_attention_weight_min": -35.11491394042969,
      "activations/layer12_attention_weight_max": 28.2844181060791,
      "activations/layer12_attention_weight_min": -26.815332412719727,
      "activations/layer13_attention_weight_max": 41.99114990234375,
      "activations/layer13_attention_weight_min": -35.64432907104492,
      "activations/layer14_attention_weight_max": 43.27560806274414,
      "activations/layer14_attention_weight_min": -37.75121307373047,
      "activations/layer15_attention_weight_max": 40.188472747802734,
      "activations/layer15_attention_weight_min": -35.52772521972656,
      "activations/layer16_attention_weight_max": 29.050025939941406,
      "activations/layer16_attention_weight_min": -29.23158836364746,
      "activations/layer17_attention_weight_max": 30.1348876953125,
      "activations/layer17_attention_weight_min": -25.584861755371094,
      "activations/layer18_attention_weight_max": 35.4793701171875,
      "activations/layer18_attention_weight_min": -24.695453643798828,
      "activations/layer19_attention_weight_max": 32.59901809692383,
      "activations/layer19_attention_weight_min": -30.049057006835938,
      "activations/layer1_attention_weight_max": 15.262964248657227,
      "activations/layer1_attention_weight_min": -14.244881629943848,
      "activations/layer20_attention_weight_max": 30.642793655395508,
      "activations/layer20_attention_weight_min": -24.227815628051758,
      "activations/layer21_attention_weight_max": 28.278362274169922,
      "activations/layer21_attention_weight_min": -23.904935836791992,
      "activations/layer22_attention_weight_max": 40.27786636352539,
      "activations/layer22_attention_weight_min": -30.594745635986328,
      "activations/layer23_attention_weight_max": 31.7618465423584,
      "activations/layer23_attention_weight_min": -25.324352264404297,
      "activations/layer2_attention_weight_max": 32.081111907958984,
      "activations/layer2_attention_weight_min": -32.505157470703125,
      "activations/layer3_attention_weight_max": 101.81354522705078,
      "activations/layer3_attention_weight_min": -100.66690063476562,
      "activations/layer4_attention_weight_max": 107.27342987060547,
      "activations/layer4_attention_weight_min": -108.50594329833984,
      "activations/layer5_attention_weight_max": 48.96179962158203,
      "activations/layer5_attention_weight_min": -63.59827423095703,
      "activations/layer6_attention_weight_max": 45.430301666259766,
      "activations/layer6_attention_weight_min": -49.31841278076172,
      "activations/layer7_attention_weight_max": 100.7431411743164,
      "activations/layer7_attention_weight_min": -102.42157745361328,
      "activations/layer8_attention_weight_max": 43.25689697265625,
      "activations/layer8_attention_weight_min": -46.401737213134766,
      "activations/layer9_attention_weight_max": 31.87116813659668,
      "activations/layer9_attention_weight_min": -34.7724609375,
      "epoch": 20.32,
      "learning_rate": 1.9176136363636363e-05,
      "loss": 2.7121,
      "step": 349650
    },
    {
      "activations/layer0_attention_weight_max": 16.125213623046875,
      "activations/layer0_attention_weight_min": -13.229897499084473,
      "activations/layer10_attention_weight_max": 32.99335479736328,
      "activations/layer10_attention_weight_min": -33.897300720214844,
      "activations/layer11_attention_weight_max": 33.41326904296875,
      "activations/layer11_attention_weight_min": -35.47986602783203,
      "activations/layer12_attention_weight_max": 26.315753936767578,
      "activations/layer12_attention_weight_min": -25.750307083129883,
      "activations/layer13_attention_weight_max": 39.0247917175293,
      "activations/layer13_attention_weight_min": -34.6854362487793,
      "activations/layer14_attention_weight_max": 42.70683670043945,
      "activations/layer14_attention_weight_min": -38.10169982910156,
      "activations/layer15_attention_weight_max": 37.92942428588867,
      "activations/layer15_attention_weight_min": -34.5551872253418,
      "activations/layer16_attention_weight_max": 29.0460147857666,
      "activations/layer16_attention_weight_min": -28.218801498413086,
      "activations/layer17_attention_weight_max": 31.36351203918457,
      "activations/layer17_attention_weight_min": -27.553768157958984,
      "activations/layer18_attention_weight_max": 35.03518295288086,
      "activations/layer18_attention_weight_min": -27.106651306152344,
      "activations/layer19_attention_weight_max": 33.29104995727539,
      "activations/layer19_attention_weight_min": -31.051544189453125,
      "activations/layer1_attention_weight_max": 16.72260093688965,
      "activations/layer1_attention_weight_min": -14.677590370178223,
      "activations/layer20_attention_weight_max": 28.83617401123047,
      "activations/layer20_attention_weight_min": -25.863964080810547,
      "activations/layer21_attention_weight_max": 30.358659744262695,
      "activations/layer21_attention_weight_min": -25.33536148071289,
      "activations/layer22_attention_weight_max": 42.20294189453125,
      "activations/layer22_attention_weight_min": -30.6605281829834,
      "activations/layer23_attention_weight_max": 32.1722412109375,
      "activations/layer23_attention_weight_min": -25.820798873901367,
      "activations/layer2_attention_weight_max": 32.65936279296875,
      "activations/layer2_attention_weight_min": -32.3281135559082,
      "activations/layer3_attention_weight_max": 98.49699401855469,
      "activations/layer3_attention_weight_min": -99.1151351928711,
      "activations/layer4_attention_weight_max": 101.3339614868164,
      "activations/layer4_attention_weight_min": -110.2968978881836,
      "activations/layer5_attention_weight_max": 48.45857238769531,
      "activations/layer5_attention_weight_min": -63.813377380371094,
      "activations/layer6_attention_weight_max": 41.7472038269043,
      "activations/layer6_attention_weight_min": -45.5527229309082,
      "activations/layer7_attention_weight_max": 91.63823699951172,
      "activations/layer7_attention_weight_min": -94.64921569824219,
      "activations/layer8_attention_weight_max": 39.93161392211914,
      "activations/layer8_attention_weight_min": -43.058258056640625,
      "activations/layer9_attention_weight_max": 30.304187774658203,
      "activations/layer9_attention_weight_min": -33.17536163330078,
      "epoch": 20.32,
      "learning_rate": 1.9157196969696968e-05,
      "loss": 2.7196,
      "step": 349700
    },
    {
      "activations/layer0_attention_weight_max": 15.924699783325195,
      "activations/layer0_attention_weight_min": -13.39226245880127,
      "activations/layer10_attention_weight_max": 36.12728500366211,
      "activations/layer10_attention_weight_min": -37.6055908203125,
      "activations/layer11_attention_weight_max": 36.06916046142578,
      "activations/layer11_attention_weight_min": -37.60773849487305,
      "activations/layer12_attention_weight_max": 27.918725967407227,
      "activations/layer12_attention_weight_min": -26.220008850097656,
      "activations/layer13_attention_weight_max": 40.40778732299805,
      "activations/layer13_attention_weight_min": -34.073734283447266,
      "activations/layer14_attention_weight_max": 42.36751174926758,
      "activations/layer14_attention_weight_min": -37.80691909790039,
      "activations/layer15_attention_weight_max": 46.01528549194336,
      "activations/layer15_attention_weight_min": -35.99890899658203,
      "activations/layer16_attention_weight_max": 30.903703689575195,
      "activations/layer16_attention_weight_min": -28.2466983795166,
      "activations/layer17_attention_weight_max": 30.56410026550293,
      "activations/layer17_attention_weight_min": -26.31806182861328,
      "activations/layer18_attention_weight_max": 33.510459899902344,
      "activations/layer18_attention_weight_min": -25.385133743286133,
      "activations/layer19_attention_weight_max": 32.8763542175293,
      "activations/layer19_attention_weight_min": -32.52640151977539,
      "activations/layer1_attention_weight_max": 15.294246673583984,
      "activations/layer1_attention_weight_min": -14.745179176330566,
      "activations/layer20_attention_weight_max": 29.595191955566406,
      "activations/layer20_attention_weight_min": -26.439523696899414,
      "activations/layer21_attention_weight_max": 27.370431900024414,
      "activations/layer21_attention_weight_min": -26.09762954711914,
      "activations/layer22_attention_weight_max": 43.56001281738281,
      "activations/layer22_attention_weight_min": -31.86316680908203,
      "activations/layer23_attention_weight_max": 33.61981201171875,
      "activations/layer23_attention_weight_min": -25.711673736572266,
      "activations/layer2_attention_weight_max": 33.003238677978516,
      "activations/layer2_attention_weight_min": -31.017536163330078,
      "activations/layer3_attention_weight_max": 101.25996398925781,
      "activations/layer3_attention_weight_min": -102.5670166015625,
      "activations/layer4_attention_weight_max": 108.69620513916016,
      "activations/layer4_attention_weight_min": -112.646484375,
      "activations/layer5_attention_weight_max": 48.77668380737305,
      "activations/layer5_attention_weight_min": -68.29603576660156,
      "activations/layer6_attention_weight_max": 44.26509094238281,
      "activations/layer6_attention_weight_min": -48.88633346557617,
      "activations/layer7_attention_weight_max": 98.71288299560547,
      "activations/layer7_attention_weight_min": -101.345458984375,
      "activations/layer8_attention_weight_max": 47.58924102783203,
      "activations/layer8_attention_weight_min": -44.441036224365234,
      "activations/layer9_attention_weight_max": 34.401737213134766,
      "activations/layer9_attention_weight_min": -35.16966247558594,
      "epoch": 20.32,
      "learning_rate": 1.9138257575757573e-05,
      "loss": 2.7074,
      "step": 349750
    },
    {
      "activations/layer0_attention_weight_max": 16.594600677490234,
      "activations/layer0_attention_weight_min": -12.842813491821289,
      "activations/layer10_attention_weight_max": 34.43288040161133,
      "activations/layer10_attention_weight_min": -36.6303596496582,
      "activations/layer11_attention_weight_max": 33.8256950378418,
      "activations/layer11_attention_weight_min": -36.66773986816406,
      "activations/layer12_attention_weight_max": 25.869667053222656,
      "activations/layer12_attention_weight_min": -26.591554641723633,
      "activations/layer13_attention_weight_max": 37.56134033203125,
      "activations/layer13_attention_weight_min": -36.61152648925781,
      "activations/layer14_attention_weight_max": 40.70323944091797,
      "activations/layer14_attention_weight_min": -36.97126388549805,
      "activations/layer15_attention_weight_max": 38.02617645263672,
      "activations/layer15_attention_weight_min": -36.79193878173828,
      "activations/layer16_attention_weight_max": 28.749391555786133,
      "activations/layer16_attention_weight_min": -29.799739837646484,
      "activations/layer17_attention_weight_max": 29.67544937133789,
      "activations/layer17_attention_weight_min": -24.52454376220703,
      "activations/layer18_attention_weight_max": 31.869552612304688,
      "activations/layer18_attention_weight_min": -26.420085906982422,
      "activations/layer19_attention_weight_max": 39.943519592285156,
      "activations/layer19_attention_weight_min": -29.87922477722168,
      "activations/layer1_attention_weight_max": 15.696904182434082,
      "activations/layer1_attention_weight_min": -15.261088371276855,
      "activations/layer20_attention_weight_max": 31.081003189086914,
      "activations/layer20_attention_weight_min": -25.315580368041992,
      "activations/layer21_attention_weight_max": 29.961326599121094,
      "activations/layer21_attention_weight_min": -26.099685668945312,
      "activations/layer22_attention_weight_max": 43.21952438354492,
      "activations/layer22_attention_weight_min": -30.587915420532227,
      "activations/layer23_attention_weight_max": 36.5830078125,
      "activations/layer23_attention_weight_min": -25.513219833374023,
      "activations/layer2_attention_weight_max": 33.00236129760742,
      "activations/layer2_attention_weight_min": -31.773283004760742,
      "activations/layer3_attention_weight_max": 102.02832794189453,
      "activations/layer3_attention_weight_min": -103.09526824951172,
      "activations/layer4_attention_weight_max": 109.50872039794922,
      "activations/layer4_attention_weight_min": -108.6640625,
      "activations/layer5_attention_weight_max": 48.206600189208984,
      "activations/layer5_attention_weight_min": -61.30721664428711,
      "activations/layer6_attention_weight_max": 43.64899444580078,
      "activations/layer6_attention_weight_min": -47.72364807128906,
      "activations/layer7_attention_weight_max": 88.62054443359375,
      "activations/layer7_attention_weight_min": -96.6641616821289,
      "activations/layer8_attention_weight_max": 40.61957931518555,
      "activations/layer8_attention_weight_min": -45.972686767578125,
      "activations/layer9_attention_weight_max": 32.458641052246094,
      "activations/layer9_attention_weight_min": -37.521602630615234,
      "epoch": 20.33,
      "learning_rate": 1.911931818181818e-05,
      "loss": 2.699,
      "step": 349800
    },
    {
      "activations/layer0_attention_weight_max": 16.269088745117188,
      "activations/layer0_attention_weight_min": -14.681863784790039,
      "activations/layer10_attention_weight_max": 32.28073501586914,
      "activations/layer10_attention_weight_min": -33.77093505859375,
      "activations/layer11_attention_weight_max": 31.089496612548828,
      "activations/layer11_attention_weight_min": -33.186702728271484,
      "activations/layer12_attention_weight_max": 26.570938110351562,
      "activations/layer12_attention_weight_min": -25.432159423828125,
      "activations/layer13_attention_weight_max": 39.49352264404297,
      "activations/layer13_attention_weight_min": -34.3520622253418,
      "activations/layer14_attention_weight_max": 42.0815544128418,
      "activations/layer14_attention_weight_min": -36.74862289428711,
      "activations/layer15_attention_weight_max": 38.36355972290039,
      "activations/layer15_attention_weight_min": -34.74222183227539,
      "activations/layer16_attention_weight_max": 27.50236701965332,
      "activations/layer16_attention_weight_min": -27.377084732055664,
      "activations/layer17_attention_weight_max": 30.135562896728516,
      "activations/layer17_attention_weight_min": -27.225086212158203,
      "activations/layer18_attention_weight_max": 32.167938232421875,
      "activations/layer18_attention_weight_min": -28.27716064453125,
      "activations/layer19_attention_weight_max": 32.60105895996094,
      "activations/layer19_attention_weight_min": -31.203062057495117,
      "activations/layer1_attention_weight_max": 15.363554954528809,
      "activations/layer1_attention_weight_min": -15.069918632507324,
      "activations/layer20_attention_weight_max": 29.45775032043457,
      "activations/layer20_attention_weight_min": -26.40890121459961,
      "activations/layer21_attention_weight_max": 29.96476173400879,
      "activations/layer21_attention_weight_min": -25.09762954711914,
      "activations/layer22_attention_weight_max": 40.0235710144043,
      "activations/layer22_attention_weight_min": -31.862825393676758,
      "activations/layer23_attention_weight_max": 29.99091339111328,
      "activations/layer23_attention_weight_min": -27.262218475341797,
      "activations/layer2_attention_weight_max": 32.25011444091797,
      "activations/layer2_attention_weight_min": -32.80571365356445,
      "activations/layer3_attention_weight_max": 101.546142578125,
      "activations/layer3_attention_weight_min": -103.61335754394531,
      "activations/layer4_attention_weight_max": 105.610595703125,
      "activations/layer4_attention_weight_min": -107.2670669555664,
      "activations/layer5_attention_weight_max": 49.27033996582031,
      "activations/layer5_attention_weight_min": -62.10795593261719,
      "activations/layer6_attention_weight_max": 42.310340881347656,
      "activations/layer6_attention_weight_min": -47.11220932006836,
      "activations/layer7_attention_weight_max": 92.08932495117188,
      "activations/layer7_attention_weight_min": -94.13565826416016,
      "activations/layer8_attention_weight_max": 38.75909423828125,
      "activations/layer8_attention_weight_min": -41.523502349853516,
      "activations/layer9_attention_weight_max": 29.927255630493164,
      "activations/layer9_attention_weight_min": -33.459415435791016,
      "epoch": 20.33,
      "learning_rate": 1.9100378787878786e-05,
      "loss": 2.7042,
      "step": 349850
    },
    {
      "activations/layer0_attention_weight_max": 15.503153800964355,
      "activations/layer0_attention_weight_min": -13.334281921386719,
      "activations/layer10_attention_weight_max": 39.76736068725586,
      "activations/layer10_attention_weight_min": -35.486083984375,
      "activations/layer11_attention_weight_max": 36.875038146972656,
      "activations/layer11_attention_weight_min": -37.36469650268555,
      "activations/layer12_attention_weight_max": 29.65237045288086,
      "activations/layer12_attention_weight_min": -30.326576232910156,
      "activations/layer13_attention_weight_max": 43.42231369018555,
      "activations/layer13_attention_weight_min": -40.87809371948242,
      "activations/layer14_attention_weight_max": 45.25299835205078,
      "activations/layer14_attention_weight_min": -42.73712158203125,
      "activations/layer15_attention_weight_max": 45.29928970336914,
      "activations/layer15_attention_weight_min": -39.13699722290039,
      "activations/layer16_attention_weight_max": 30.666759490966797,
      "activations/layer16_attention_weight_min": -28.5195255279541,
      "activations/layer17_attention_weight_max": 31.433570861816406,
      "activations/layer17_attention_weight_min": -27.637557983398438,
      "activations/layer18_attention_weight_max": 33.218624114990234,
      "activations/layer18_attention_weight_min": -27.456865310668945,
      "activations/layer19_attention_weight_max": 35.12052536010742,
      "activations/layer19_attention_weight_min": -34.89785385131836,
      "activations/layer1_attention_weight_max": 15.123632431030273,
      "activations/layer1_attention_weight_min": -14.856267929077148,
      "activations/layer20_attention_weight_max": 28.94142723083496,
      "activations/layer20_attention_weight_min": -25.997730255126953,
      "activations/layer21_attention_weight_max": 29.055448532104492,
      "activations/layer21_attention_weight_min": -24.982412338256836,
      "activations/layer22_attention_weight_max": 42.38652420043945,
      "activations/layer22_attention_weight_min": -33.125343322753906,
      "activations/layer23_attention_weight_max": 31.20844841003418,
      "activations/layer23_attention_weight_min": -28.140262603759766,
      "activations/layer2_attention_weight_max": 33.439971923828125,
      "activations/layer2_attention_weight_min": -32.473445892333984,
      "activations/layer3_attention_weight_max": 101.78203582763672,
      "activations/layer3_attention_weight_min": -103.10134887695312,
      "activations/layer4_attention_weight_max": 108.12953186035156,
      "activations/layer4_attention_weight_min": -108.05675506591797,
      "activations/layer5_attention_weight_max": 52.09955978393555,
      "activations/layer5_attention_weight_min": -59.872249603271484,
      "activations/layer6_attention_weight_max": 46.339576721191406,
      "activations/layer6_attention_weight_min": -46.382076263427734,
      "activations/layer7_attention_weight_max": 97.43364715576172,
      "activations/layer7_attention_weight_min": -98.75984191894531,
      "activations/layer8_attention_weight_max": 43.44391632080078,
      "activations/layer8_attention_weight_min": -45.409515380859375,
      "activations/layer9_attention_weight_max": 35.33583068847656,
      "activations/layer9_attention_weight_min": -36.48414993286133,
      "epoch": 20.33,
      "learning_rate": 1.9081439393939394e-05,
      "loss": 2.7083,
      "step": 349900
    },
    {
      "activations/layer0_attention_weight_max": 16.431848526000977,
      "activations/layer0_attention_weight_min": -14.679655075073242,
      "activations/layer10_attention_weight_max": 35.43178176879883,
      "activations/layer10_attention_weight_min": -35.63370895385742,
      "activations/layer11_attention_weight_max": 35.694766998291016,
      "activations/layer11_attention_weight_min": -36.77947998046875,
      "activations/layer12_attention_weight_max": 27.05801010131836,
      "activations/layer12_attention_weight_min": -26.363508224487305,
      "activations/layer13_attention_weight_max": 40.512184143066406,
      "activations/layer13_attention_weight_min": -36.87961959838867,
      "activations/layer14_attention_weight_max": 47.25730895996094,
      "activations/layer14_attention_weight_min": -37.312137603759766,
      "activations/layer15_attention_weight_max": 44.65509796142578,
      "activations/layer15_attention_weight_min": -36.59390640258789,
      "activations/layer16_attention_weight_max": 31.087202072143555,
      "activations/layer16_attention_weight_min": -28.90617561340332,
      "activations/layer17_attention_weight_max": 32.491783142089844,
      "activations/layer17_attention_weight_min": -26.088090896606445,
      "activations/layer18_attention_weight_max": 35.70370101928711,
      "activations/layer18_attention_weight_min": -25.78836441040039,
      "activations/layer19_attention_weight_max": 37.09164047241211,
      "activations/layer19_attention_weight_min": -33.45707702636719,
      "activations/layer1_attention_weight_max": 15.807280540466309,
      "activations/layer1_attention_weight_min": -15.217450141906738,
      "activations/layer20_attention_weight_max": 30.809104919433594,
      "activations/layer20_attention_weight_min": -25.379301071166992,
      "activations/layer21_attention_weight_max": 32.13658142089844,
      "activations/layer21_attention_weight_min": -26.24441909790039,
      "activations/layer22_attention_weight_max": 47.03005599975586,
      "activations/layer22_attention_weight_min": -31.747177124023438,
      "activations/layer23_attention_weight_max": 31.981666564941406,
      "activations/layer23_attention_weight_min": -26.236774444580078,
      "activations/layer2_attention_weight_max": 32.30986785888672,
      "activations/layer2_attention_weight_min": -31.327823638916016,
      "activations/layer3_attention_weight_max": 100.94084930419922,
      "activations/layer3_attention_weight_min": -99.6854019165039,
      "activations/layer4_attention_weight_max": 103.39775848388672,
      "activations/layer4_attention_weight_min": -113.08209228515625,
      "activations/layer5_attention_weight_max": 52.2414665222168,
      "activations/layer5_attention_weight_min": -65.0346908569336,
      "activations/layer6_attention_weight_max": 43.20582580566406,
      "activations/layer6_attention_weight_min": -46.359439849853516,
      "activations/layer7_attention_weight_max": 91.24409484863281,
      "activations/layer7_attention_weight_min": -99.47891998291016,
      "activations/layer8_attention_weight_max": 39.6603889465332,
      "activations/layer8_attention_weight_min": -44.63639831542969,
      "activations/layer9_attention_weight_max": 31.539264678955078,
      "activations/layer9_attention_weight_min": -34.52863693237305,
      "epoch": 20.33,
      "learning_rate": 1.9062499999999996e-05,
      "loss": 2.7075,
      "step": 349950
    },
    {
      "activations/layer0_attention_weight_max": 16.829011917114258,
      "activations/layer0_attention_weight_min": -16.584516525268555,
      "activations/layer10_attention_weight_max": 35.571533203125,
      "activations/layer10_attention_weight_min": -37.265438079833984,
      "activations/layer11_attention_weight_max": 32.3937873840332,
      "activations/layer11_attention_weight_min": -35.5528564453125,
      "activations/layer12_attention_weight_max": 26.957321166992188,
      "activations/layer12_attention_weight_min": -28.26782989501953,
      "activations/layer13_attention_weight_max": 38.889381408691406,
      "activations/layer13_attention_weight_min": -37.219661712646484,
      "activations/layer14_attention_weight_max": 42.619476318359375,
      "activations/layer14_attention_weight_min": -38.4057731628418,
      "activations/layer15_attention_weight_max": 40.40839767456055,
      "activations/layer15_attention_weight_min": -39.08175277709961,
      "activations/layer16_attention_weight_max": 29.14360809326172,
      "activations/layer16_attention_weight_min": -28.319061279296875,
      "activations/layer17_attention_weight_max": 31.05023956298828,
      "activations/layer17_attention_weight_min": -28.10405921936035,
      "activations/layer18_attention_weight_max": 30.587467193603516,
      "activations/layer18_attention_weight_min": -26.379270553588867,
      "activations/layer19_attention_weight_max": 32.76469802856445,
      "activations/layer19_attention_weight_min": -34.12782669067383,
      "activations/layer1_attention_weight_max": 15.309921264648438,
      "activations/layer1_attention_weight_min": -15.284823417663574,
      "activations/layer20_attention_weight_max": 29.860815048217773,
      "activations/layer20_attention_weight_min": -27.127851486206055,
      "activations/layer21_attention_weight_max": 29.388628005981445,
      "activations/layer21_attention_weight_min": -25.00275230407715,
      "activations/layer22_attention_weight_max": 40.89236068725586,
      "activations/layer22_attention_weight_min": -34.55781173706055,
      "activations/layer23_attention_weight_max": 30.915809631347656,
      "activations/layer23_attention_weight_min": -27.255035400390625,
      "activations/layer2_attention_weight_max": 33.35835266113281,
      "activations/layer2_attention_weight_min": -31.451398849487305,
      "activations/layer3_attention_weight_max": 105.98448944091797,
      "activations/layer3_attention_weight_min": -104.04927062988281,
      "activations/layer4_attention_weight_max": 106.67071533203125,
      "activations/layer4_attention_weight_min": -108.05887603759766,
      "activations/layer5_attention_weight_max": 49.09326171875,
      "activations/layer5_attention_weight_min": -62.99897766113281,
      "activations/layer6_attention_weight_max": 43.773746490478516,
      "activations/layer6_attention_weight_min": -47.764888763427734,
      "activations/layer7_attention_weight_max": 90.9402084350586,
      "activations/layer7_attention_weight_min": -97.83431243896484,
      "activations/layer8_attention_weight_max": 42.692935943603516,
      "activations/layer8_attention_weight_min": -44.95473098754883,
      "activations/layer9_attention_weight_max": 31.55732536315918,
      "activations/layer9_attention_weight_min": -36.10488510131836,
      "epoch": 20.34,
      "learning_rate": 1.9043560606060604e-05,
      "loss": 2.7225,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.4761,
      "eval_samples_per_second": 506.603,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.4761,
      "eval_openwebtext_samples_per_second": 506.603,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9495,
      "eval_wikitext_samples_per_second": 233.903,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_lambada_loss": 2.34765625,
      "eval_lambada_ppl": 10.461022947676224,
      "eval_lambada_runtime": 9.5351,
      "eval_lambada_samples_per_second": 510.642,
      "step": 350000
    },
    {
      "activations/layer0_attention_weight_max": 16.510555267333984,
      "activations/layer0_attention_weight_min": -14.150542259216309,
      "activations/layer10_attention_weight_max": 34.26879119873047,
      "activations/layer10_attention_weight_min": -36.45064163208008,
      "activations/layer11_attention_weight_max": 33.38335418701172,
      "activations/layer11_attention_weight_min": -35.66753005981445,
      "activations/layer12_attention_weight_max": 26.837827682495117,
      "activations/layer12_attention_weight_min": -25.91208839416504,
      "activations/layer13_attention_weight_max": 39.09551239013672,
      "activations/layer13_attention_weight_min": -34.576480865478516,
      "activations/layer14_attention_weight_max": 40.56121063232422,
      "activations/layer14_attention_weight_min": -36.04996109008789,
      "activations/layer15_attention_weight_max": 39.99205017089844,
      "activations/layer15_attention_weight_min": -35.003387451171875,
      "activations/layer16_attention_weight_max": 28.885705947875977,
      "activations/layer16_attention_weight_min": -27.912931442260742,
      "activations/layer17_attention_weight_max": 28.786869049072266,
      "activations/layer17_attention_weight_min": -26.639806747436523,
      "activations/layer18_attention_weight_max": 31.624431610107422,
      "activations/layer18_attention_weight_min": -28.413883209228516,
      "activations/layer19_attention_weight_max": 32.842716217041016,
      "activations/layer19_attention_weight_min": -33.229183197021484,
      "activations/layer1_attention_weight_max": 14.907323837280273,
      "activations/layer1_attention_weight_min": -13.916238784790039,
      "activations/layer20_attention_weight_max": 30.841259002685547,
      "activations/layer20_attention_weight_min": -27.159793853759766,
      "activations/layer21_attention_weight_max": 27.836559295654297,
      "activations/layer21_attention_weight_min": -25.22880744934082,
      "activations/layer22_attention_weight_max": 43.29386520385742,
      "activations/layer22_attention_weight_min": -32.79642868041992,
      "activations/layer23_attention_weight_max": 31.193748474121094,
      "activations/layer23_attention_weight_min": -26.06814193725586,
      "activations/layer2_attention_weight_max": 32.33283233642578,
      "activations/layer2_attention_weight_min": -31.406902313232422,
      "activations/layer3_attention_weight_max": 101.94396209716797,
      "activations/layer3_attention_weight_min": -100.9778060913086,
      "activations/layer4_attention_weight_max": 109.51805877685547,
      "activations/layer4_attention_weight_min": -112.4959716796875,
      "activations/layer5_attention_weight_max": 51.34565353393555,
      "activations/layer5_attention_weight_min": -64.83596801757812,
      "activations/layer6_attention_weight_max": 44.94562911987305,
      "activations/layer6_attention_weight_min": -47.39278793334961,
      "activations/layer7_attention_weight_max": 97.47091674804688,
      "activations/layer7_attention_weight_min": -98.34600067138672,
      "activations/layer8_attention_weight_max": 42.50065231323242,
      "activations/layer8_attention_weight_min": -44.06416702270508,
      "activations/layer9_attention_weight_max": 32.71354293823242,
      "activations/layer9_attention_weight_min": -34.793880462646484,
      "epoch": 20.34,
      "learning_rate": 1.902462121212121e-05,
      "loss": 2.7055,
      "step": 350050
    },
    {
      "activations/layer0_attention_weight_max": 16.0150089263916,
      "activations/layer0_attention_weight_min": -13.425918579101562,
      "activations/layer10_attention_weight_max": 33.30682373046875,
      "activations/layer10_attention_weight_min": -33.8231201171875,
      "activations/layer11_attention_weight_max": 33.743289947509766,
      "activations/layer11_attention_weight_min": -33.98039245605469,
      "activations/layer12_attention_weight_max": 26.09038543701172,
      "activations/layer12_attention_weight_min": -26.50113868713379,
      "activations/layer13_attention_weight_max": 37.11357879638672,
      "activations/layer13_attention_weight_min": -35.00872802734375,
      "activations/layer14_attention_weight_max": 44.10525894165039,
      "activations/layer14_attention_weight_min": -38.899356842041016,
      "activations/layer15_attention_weight_max": 39.17784881591797,
      "activations/layer15_attention_weight_min": -36.312225341796875,
      "activations/layer16_attention_weight_max": 30.518766403198242,
      "activations/layer16_attention_weight_min": -29.786741256713867,
      "activations/layer17_attention_weight_max": 30.674428939819336,
      "activations/layer17_attention_weight_min": -27.945552825927734,
      "activations/layer18_attention_weight_max": 32.351078033447266,
      "activations/layer18_attention_weight_min": -26.166973114013672,
      "activations/layer19_attention_weight_max": 34.38499069213867,
      "activations/layer19_attention_weight_min": -32.71635818481445,
      "activations/layer1_attention_weight_max": 15.260885238647461,
      "activations/layer1_attention_weight_min": -13.99789047241211,
      "activations/layer20_attention_weight_max": 30.448455810546875,
      "activations/layer20_attention_weight_min": -26.038427352905273,
      "activations/layer21_attention_weight_max": 30.881168365478516,
      "activations/layer21_attention_weight_min": -26.340421676635742,
      "activations/layer22_attention_weight_max": 42.36157989501953,
      "activations/layer22_attention_weight_min": -31.69567108154297,
      "activations/layer23_attention_weight_max": 33.09458923339844,
      "activations/layer23_attention_weight_min": -26.200702667236328,
      "activations/layer2_attention_weight_max": 31.583660125732422,
      "activations/layer2_attention_weight_min": -30.47694969177246,
      "activations/layer3_attention_weight_max": 95.48793029785156,
      "activations/layer3_attention_weight_min": -97.5796127319336,
      "activations/layer4_attention_weight_max": 103.12650299072266,
      "activations/layer4_attention_weight_min": -107.1910629272461,
      "activations/layer5_attention_weight_max": 48.06171417236328,
      "activations/layer5_attention_weight_min": -59.078094482421875,
      "activations/layer6_attention_weight_max": 43.09996032714844,
      "activations/layer6_attention_weight_min": -46.701473236083984,
      "activations/layer7_attention_weight_max": 93.0567398071289,
      "activations/layer7_attention_weight_min": -94.4378662109375,
      "activations/layer8_attention_weight_max": 41.083988189697266,
      "activations/layer8_attention_weight_min": -44.35489273071289,
      "activations/layer9_attention_weight_max": 32.64424133300781,
      "activations/layer9_attention_weight_min": -33.57057189941406,
      "epoch": 20.34,
      "learning_rate": 1.9005681818181817e-05,
      "loss": 2.7113,
      "step": 350100
    },
    {
      "activations/layer0_attention_weight_max": 15.757529258728027,
      "activations/layer0_attention_weight_min": -13.527565956115723,
      "activations/layer10_attention_weight_max": 35.69181442260742,
      "activations/layer10_attention_weight_min": -35.21904373168945,
      "activations/layer11_attention_weight_max": 35.47645568847656,
      "activations/layer11_attention_weight_min": -39.70549011230469,
      "activations/layer12_attention_weight_max": 27.104843139648438,
      "activations/layer12_attention_weight_min": -28.88048553466797,
      "activations/layer13_attention_weight_max": 40.46055603027344,
      "activations/layer13_attention_weight_min": -36.174808502197266,
      "activations/layer14_attention_weight_max": 46.644012451171875,
      "activations/layer14_attention_weight_min": -38.51333236694336,
      "activations/layer15_attention_weight_max": 40.282005310058594,
      "activations/layer15_attention_weight_min": -37.998497009277344,
      "activations/layer16_attention_weight_max": 29.000572204589844,
      "activations/layer16_attention_weight_min": -27.904064178466797,
      "activations/layer17_attention_weight_max": 32.3071174621582,
      "activations/layer17_attention_weight_min": -27.082109451293945,
      "activations/layer18_attention_weight_max": 36.13825607299805,
      "activations/layer18_attention_weight_min": -25.67064094543457,
      "activations/layer19_attention_weight_max": 41.68804168701172,
      "activations/layer19_attention_weight_min": -31.578781127929688,
      "activations/layer1_attention_weight_max": 15.969432830810547,
      "activations/layer1_attention_weight_min": -15.382010459899902,
      "activations/layer20_attention_weight_max": 34.59093475341797,
      "activations/layer20_attention_weight_min": -26.349637985229492,
      "activations/layer21_attention_weight_max": 33.59340286254883,
      "activations/layer21_attention_weight_min": -25.76579475402832,
      "activations/layer22_attention_weight_max": 45.69284439086914,
      "activations/layer22_attention_weight_min": -32.37977981567383,
      "activations/layer23_attention_weight_max": 35.67803192138672,
      "activations/layer23_attention_weight_min": -26.778106689453125,
      "activations/layer2_attention_weight_max": 34.71371841430664,
      "activations/layer2_attention_weight_min": -32.520206451416016,
      "activations/layer3_attention_weight_max": 107.43209838867188,
      "activations/layer3_attention_weight_min": -106.22576141357422,
      "activations/layer4_attention_weight_max": 108.2311782836914,
      "activations/layer4_attention_weight_min": -111.99529266357422,
      "activations/layer5_attention_weight_max": 53.40421676635742,
      "activations/layer5_attention_weight_min": -62.549049377441406,
      "activations/layer6_attention_weight_max": 45.979515075683594,
      "activations/layer6_attention_weight_min": -47.51740264892578,
      "activations/layer7_attention_weight_max": 97.83448791503906,
      "activations/layer7_attention_weight_min": -98.40658569335938,
      "activations/layer8_attention_weight_max": 46.101531982421875,
      "activations/layer8_attention_weight_min": -46.6054573059082,
      "activations/layer9_attention_weight_max": 33.998104095458984,
      "activations/layer9_attention_weight_min": -36.54096603393555,
      "epoch": 20.35,
      "learning_rate": 1.8986742424242422e-05,
      "loss": 2.7194,
      "step": 350150
    },
    {
      "activations/layer0_attention_weight_max": 16.728261947631836,
      "activations/layer0_attention_weight_min": -13.768445014953613,
      "activations/layer10_attention_weight_max": 36.52677917480469,
      "activations/layer10_attention_weight_min": -36.13229751586914,
      "activations/layer11_attention_weight_max": 38.49718475341797,
      "activations/layer11_attention_weight_min": -38.75691223144531,
      "activations/layer12_attention_weight_max": 32.03617858886719,
      "activations/layer12_attention_weight_min": -29.30320167541504,
      "activations/layer13_attention_weight_max": 50.464073181152344,
      "activations/layer13_attention_weight_min": -40.81943893432617,
      "activations/layer14_attention_weight_max": 57.26473617553711,
      "activations/layer14_attention_weight_min": -45.233280181884766,
      "activations/layer15_attention_weight_max": 52.45623016357422,
      "activations/layer15_attention_weight_min": -41.899776458740234,
      "activations/layer16_attention_weight_max": 34.73057556152344,
      "activations/layer16_attention_weight_min": -30.07175636291504,
      "activations/layer17_attention_weight_max": 34.939659118652344,
      "activations/layer17_attention_weight_min": -30.089656829833984,
      "activations/layer18_attention_weight_max": 37.07685852050781,
      "activations/layer18_attention_weight_min": -28.348237991333008,
      "activations/layer19_attention_weight_max": 39.20033264160156,
      "activations/layer19_attention_weight_min": -35.64406967163086,
      "activations/layer1_attention_weight_max": 15.837458610534668,
      "activations/layer1_attention_weight_min": -16.12739372253418,
      "activations/layer20_attention_weight_max": 32.81963348388672,
      "activations/layer20_attention_weight_min": -26.256975173950195,
      "activations/layer21_attention_weight_max": 29.392967224121094,
      "activations/layer21_attention_weight_min": -24.370880126953125,
      "activations/layer22_attention_weight_max": 47.45576858520508,
      "activations/layer22_attention_weight_min": -32.593929290771484,
      "activations/layer23_attention_weight_max": 32.48313903808594,
      "activations/layer23_attention_weight_min": -27.046451568603516,
      "activations/layer2_attention_weight_max": 33.66828155517578,
      "activations/layer2_attention_weight_min": -33.68537139892578,
      "activations/layer3_attention_weight_max": 105.03016662597656,
      "activations/layer3_attention_weight_min": -108.27933502197266,
      "activations/layer4_attention_weight_max": 109.4966812133789,
      "activations/layer4_attention_weight_min": -110.45951080322266,
      "activations/layer5_attention_weight_max": 50.279388427734375,
      "activations/layer5_attention_weight_min": -64.55570983886719,
      "activations/layer6_attention_weight_max": 45.540672302246094,
      "activations/layer6_attention_weight_min": -47.014957427978516,
      "activations/layer7_attention_weight_max": 91.6949234008789,
      "activations/layer7_attention_weight_min": -101.07383728027344,
      "activations/layer8_attention_weight_max": 42.6396369934082,
      "activations/layer8_attention_weight_min": -47.0482292175293,
      "activations/layer9_attention_weight_max": 33.05406188964844,
      "activations/layer9_attention_weight_min": -36.23353576660156,
      "epoch": 20.35,
      "learning_rate": 1.896780303030303e-05,
      "loss": 2.7272,
      "step": 350200
    },
    {
      "activations/layer0_attention_weight_max": 15.873973846435547,
      "activations/layer0_attention_weight_min": -13.545751571655273,
      "activations/layer10_attention_weight_max": 36.668670654296875,
      "activations/layer10_attention_weight_min": -35.577728271484375,
      "activations/layer11_attention_weight_max": 36.130126953125,
      "activations/layer11_attention_weight_min": -36.116676330566406,
      "activations/layer12_attention_weight_max": 28.355806350708008,
      "activations/layer12_attention_weight_min": -28.59889030456543,
      "activations/layer13_attention_weight_max": 41.761390686035156,
      "activations/layer13_attention_weight_min": -37.2666015625,
      "activations/layer14_attention_weight_max": 44.61558151245117,
      "activations/layer14_attention_weight_min": -39.59099197387695,
      "activations/layer15_attention_weight_max": 40.72112274169922,
      "activations/layer15_attention_weight_min": -36.846717834472656,
      "activations/layer16_attention_weight_max": 30.704666137695312,
      "activations/layer16_attention_weight_min": -28.520902633666992,
      "activations/layer17_attention_weight_max": 34.047054290771484,
      "activations/layer17_attention_weight_min": -28.16426658630371,
      "activations/layer18_attention_weight_max": 34.80781173706055,
      "activations/layer18_attention_weight_min": -30.549039840698242,
      "activations/layer19_attention_weight_max": 39.46044921875,
      "activations/layer19_attention_weight_min": -36.160465240478516,
      "activations/layer1_attention_weight_max": 16.466264724731445,
      "activations/layer1_attention_weight_min": -14.856908798217773,
      "activations/layer20_attention_weight_max": 33.18413543701172,
      "activations/layer20_attention_weight_min": -30.218231201171875,
      "activations/layer21_attention_weight_max": 32.988487243652344,
      "activations/layer21_attention_weight_min": -29.035734176635742,
      "activations/layer22_attention_weight_max": 48.1025390625,
      "activations/layer22_attention_weight_min": -34.69013214111328,
      "activations/layer23_attention_weight_max": 34.29913330078125,
      "activations/layer23_attention_weight_min": -26.780235290527344,
      "activations/layer2_attention_weight_max": 33.51613998413086,
      "activations/layer2_attention_weight_min": -32.027442932128906,
      "activations/layer3_attention_weight_max": 107.12278747558594,
      "activations/layer3_attention_weight_min": -99.6526870727539,
      "activations/layer4_attention_weight_max": 114.10796356201172,
      "activations/layer4_attention_weight_min": -109.415771484375,
      "activations/layer5_attention_weight_max": 53.524349212646484,
      "activations/layer5_attention_weight_min": -60.7585563659668,
      "activations/layer6_attention_weight_max": 49.08155822753906,
      "activations/layer6_attention_weight_min": -48.73686218261719,
      "activations/layer7_attention_weight_max": 105.23487854003906,
      "activations/layer7_attention_weight_min": -103.5068359375,
      "activations/layer8_attention_weight_max": 46.73920822143555,
      "activations/layer8_attention_weight_min": -46.470638275146484,
      "activations/layer9_attention_weight_max": 34.55923843383789,
      "activations/layer9_attention_weight_min": -35.48049545288086,
      "epoch": 20.35,
      "learning_rate": 1.8948863636363632e-05,
      "loss": 2.6974,
      "step": 350250
    },
    {
      "activations/layer0_attention_weight_max": 15.711831092834473,
      "activations/layer0_attention_weight_min": -14.940616607666016,
      "activations/layer10_attention_weight_max": 33.29729080200195,
      "activations/layer10_attention_weight_min": -36.58184814453125,
      "activations/layer11_attention_weight_max": 34.862953186035156,
      "activations/layer11_attention_weight_min": -35.1358642578125,
      "activations/layer12_attention_weight_max": 27.317684173583984,
      "activations/layer12_attention_weight_min": -28.28117561340332,
      "activations/layer13_attention_weight_max": 39.35350799560547,
      "activations/layer13_attention_weight_min": -37.39686965942383,
      "activations/layer14_attention_weight_max": 44.47359085083008,
      "activations/layer14_attention_weight_min": -38.68815994262695,
      "activations/layer15_attention_weight_max": 40.717445373535156,
      "activations/layer15_attention_weight_min": -37.08418655395508,
      "activations/layer16_attention_weight_max": 31.85169219970703,
      "activations/layer16_attention_weight_min": -27.508586883544922,
      "activations/layer17_attention_weight_max": 30.67457389831543,
      "activations/layer17_attention_weight_min": -26.253808975219727,
      "activations/layer18_attention_weight_max": 34.186431884765625,
      "activations/layer18_attention_weight_min": -26.26093101501465,
      "activations/layer19_attention_weight_max": 35.842002868652344,
      "activations/layer19_attention_weight_min": -31.459203720092773,
      "activations/layer1_attention_weight_max": 15.256173133850098,
      "activations/layer1_attention_weight_min": -14.874418258666992,
      "activations/layer20_attention_weight_max": 31.63271141052246,
      "activations/layer20_attention_weight_min": -25.773765563964844,
      "activations/layer21_attention_weight_max": 30.70294761657715,
      "activations/layer21_attention_weight_min": -26.20700454711914,
      "activations/layer22_attention_weight_max": 43.010921478271484,
      "activations/layer22_attention_weight_min": -32.45089340209961,
      "activations/layer23_attention_weight_max": 30.15863609313965,
      "activations/layer23_attention_weight_min": -27.124950408935547,
      "activations/layer2_attention_weight_max": 33.60748291015625,
      "activations/layer2_attention_weight_min": -33.15950012207031,
      "activations/layer3_attention_weight_max": 101.88733673095703,
      "activations/layer3_attention_weight_min": -105.62464904785156,
      "activations/layer4_attention_weight_max": 104.09687805175781,
      "activations/layer4_attention_weight_min": -110.7737045288086,
      "activations/layer5_attention_weight_max": 48.5057373046875,
      "activations/layer5_attention_weight_min": -62.74204635620117,
      "activations/layer6_attention_weight_max": 44.15107345581055,
      "activations/layer6_attention_weight_min": -47.547386169433594,
      "activations/layer7_attention_weight_max": 93.356201171875,
      "activations/layer7_attention_weight_min": -103.11863708496094,
      "activations/layer8_attention_weight_max": 41.243526458740234,
      "activations/layer8_attention_weight_min": -48.360443115234375,
      "activations/layer9_attention_weight_max": 33.275108337402344,
      "activations/layer9_attention_weight_min": -38.049407958984375,
      "epoch": 20.35,
      "learning_rate": 1.892992424242424e-05,
      "loss": 2.7178,
      "step": 350300
    },
    {
      "activations/layer0_attention_weight_max": 15.927648544311523,
      "activations/layer0_attention_weight_min": -12.376501083374023,
      "activations/layer10_attention_weight_max": 35.834510803222656,
      "activations/layer10_attention_weight_min": -35.299503326416016,
      "activations/layer11_attention_weight_max": 35.83804702758789,
      "activations/layer11_attention_weight_min": -35.97771453857422,
      "activations/layer12_attention_weight_max": 26.119369506835938,
      "activations/layer12_attention_weight_min": -28.01573371887207,
      "activations/layer13_attention_weight_max": 37.45570373535156,
      "activations/layer13_attention_weight_min": -36.83332061767578,
      "activations/layer14_attention_weight_max": 43.00084686279297,
      "activations/layer14_attention_weight_min": -38.880104064941406,
      "activations/layer15_attention_weight_max": 41.68330764770508,
      "activations/layer15_attention_weight_min": -39.34323501586914,
      "activations/layer16_attention_weight_max": 32.6116943359375,
      "activations/layer16_attention_weight_min": -28.89869499206543,
      "activations/layer17_attention_weight_max": 30.98805046081543,
      "activations/layer17_attention_weight_min": -27.881193161010742,
      "activations/layer18_attention_weight_max": 32.02818298339844,
      "activations/layer18_attention_weight_min": -25.37421226501465,
      "activations/layer19_attention_weight_max": 35.071903228759766,
      "activations/layer19_attention_weight_min": -32.114070892333984,
      "activations/layer1_attention_weight_max": 15.888798713684082,
      "activations/layer1_attention_weight_min": -14.211112022399902,
      "activations/layer20_attention_weight_max": 30.55939292907715,
      "activations/layer20_attention_weight_min": -27.021554946899414,
      "activations/layer21_attention_weight_max": 29.17521095275879,
      "activations/layer21_attention_weight_min": -24.92533302307129,
      "activations/layer22_attention_weight_max": 42.64918518066406,
      "activations/layer22_attention_weight_min": -31.693525314331055,
      "activations/layer23_attention_weight_max": 31.015827178955078,
      "activations/layer23_attention_weight_min": -25.122303009033203,
      "activations/layer2_attention_weight_max": 32.74082946777344,
      "activations/layer2_attention_weight_min": -31.974308013916016,
      "activations/layer3_attention_weight_max": 100.23275756835938,
      "activations/layer3_attention_weight_min": -101.8056869506836,
      "activations/layer4_attention_weight_max": 104.57203674316406,
      "activations/layer4_attention_weight_min": -104.52677154541016,
      "activations/layer5_attention_weight_max": 49.87504196166992,
      "activations/layer5_attention_weight_min": -60.11543273925781,
      "activations/layer6_attention_weight_max": 43.91596221923828,
      "activations/layer6_attention_weight_min": -45.261802673339844,
      "activations/layer7_attention_weight_max": 93.44156646728516,
      "activations/layer7_attention_weight_min": -98.33576965332031,
      "activations/layer8_attention_weight_max": 41.632015228271484,
      "activations/layer8_attention_weight_min": -44.32273864746094,
      "activations/layer9_attention_weight_max": 31.30140495300293,
      "activations/layer9_attention_weight_min": -34.00091552734375,
      "epoch": 20.36,
      "learning_rate": 1.891098484848485e-05,
      "loss": 2.7241,
      "step": 350350
    },
    {
      "activations/layer0_attention_weight_max": 15.735685348510742,
      "activations/layer0_attention_weight_min": -13.142709732055664,
      "activations/layer10_attention_weight_max": 37.91574478149414,
      "activations/layer10_attention_weight_min": -37.60219192504883,
      "activations/layer11_attention_weight_max": 38.52388381958008,
      "activations/layer11_attention_weight_min": -37.722755432128906,
      "activations/layer12_attention_weight_max": 30.928287506103516,
      "activations/layer12_attention_weight_min": -29.7221622467041,
      "activations/layer13_attention_weight_max": 44.024349212646484,
      "activations/layer13_attention_weight_min": -38.479225158691406,
      "activations/layer14_attention_weight_max": 43.09498596191406,
      "activations/layer14_attention_weight_min": -40.58333969116211,
      "activations/layer15_attention_weight_max": 43.9284553527832,
      "activations/layer15_attention_weight_min": -37.736595153808594,
      "activations/layer16_attention_weight_max": 32.18745803833008,
      "activations/layer16_attention_weight_min": -27.34925651550293,
      "activations/layer17_attention_weight_max": 29.774240493774414,
      "activations/layer17_attention_weight_min": -25.802236557006836,
      "activations/layer18_attention_weight_max": 32.70827102661133,
      "activations/layer18_attention_weight_min": -24.978004455566406,
      "activations/layer19_attention_weight_max": 35.23088455200195,
      "activations/layer19_attention_weight_min": -32.07033920288086,
      "activations/layer1_attention_weight_max": 16.17816925048828,
      "activations/layer1_attention_weight_min": -14.46036434173584,
      "activations/layer20_attention_weight_max": 30.7799015045166,
      "activations/layer20_attention_weight_min": -24.205745697021484,
      "activations/layer21_attention_weight_max": 31.573617935180664,
      "activations/layer21_attention_weight_min": -24.606809616088867,
      "activations/layer22_attention_weight_max": 41.166717529296875,
      "activations/layer22_attention_weight_min": -31.028039932250977,
      "activations/layer23_attention_weight_max": 34.009971618652344,
      "activations/layer23_attention_weight_min": -25.98858642578125,
      "activations/layer2_attention_weight_max": 33.14531707763672,
      "activations/layer2_attention_weight_min": -33.08220291137695,
      "activations/layer3_attention_weight_max": 106.0456771850586,
      "activations/layer3_attention_weight_min": -105.42717742919922,
      "activations/layer4_attention_weight_max": 112.03924560546875,
      "activations/layer4_attention_weight_min": -112.28431701660156,
      "activations/layer5_attention_weight_max": 51.920982360839844,
      "activations/layer5_attention_weight_min": -61.01150131225586,
      "activations/layer6_attention_weight_max": 46.45363235473633,
      "activations/layer6_attention_weight_min": -50.749427795410156,
      "activations/layer7_attention_weight_max": 105.72010040283203,
      "activations/layer7_attention_weight_min": -111.00894927978516,
      "activations/layer8_attention_weight_max": 45.27418899536133,
      "activations/layer8_attention_weight_min": -49.8901481628418,
      "activations/layer9_attention_weight_max": 34.73698043823242,
      "activations/layer9_attention_weight_min": -36.01618576049805,
      "epoch": 20.36,
      "learning_rate": 1.8892045454545453e-05,
      "loss": 2.7145,
      "step": 350400
    },
    {
      "activations/layer0_attention_weight_max": 16.394878387451172,
      "activations/layer0_attention_weight_min": -14.195415496826172,
      "activations/layer10_attention_weight_max": 35.54047393798828,
      "activations/layer10_attention_weight_min": -36.57514572143555,
      "activations/layer11_attention_weight_max": 35.516815185546875,
      "activations/layer11_attention_weight_min": -37.202415466308594,
      "activations/layer12_attention_weight_max": 29.430946350097656,
      "activations/layer12_attention_weight_min": -27.705345153808594,
      "activations/layer13_attention_weight_max": 43.48902130126953,
      "activations/layer13_attention_weight_min": -38.433345794677734,
      "activations/layer14_attention_weight_max": 43.86777877807617,
      "activations/layer14_attention_weight_min": -38.79664611816406,
      "activations/layer15_attention_weight_max": 41.32569122314453,
      "activations/layer15_attention_weight_min": -36.220611572265625,
      "activations/layer16_attention_weight_max": 31.143726348876953,
      "activations/layer16_attention_weight_min": -29.664411544799805,
      "activations/layer17_attention_weight_max": 29.14125633239746,
      "activations/layer17_attention_weight_min": -27.63730812072754,
      "activations/layer18_attention_weight_max": 35.68893814086914,
      "activations/layer18_attention_weight_min": -28.260366439819336,
      "activations/layer19_attention_weight_max": 34.94437789916992,
      "activations/layer19_attention_weight_min": -33.97734451293945,
      "activations/layer1_attention_weight_max": 16.741586685180664,
      "activations/layer1_attention_weight_min": -14.36587905883789,
      "activations/layer20_attention_weight_max": 29.806499481201172,
      "activations/layer20_attention_weight_min": -27.513195037841797,
      "activations/layer21_attention_weight_max": 31.8181209564209,
      "activations/layer21_attention_weight_min": -26.9425048828125,
      "activations/layer22_attention_weight_max": 44.29668426513672,
      "activations/layer22_attention_weight_min": -35.05847930908203,
      "activations/layer23_attention_weight_max": 31.89179229736328,
      "activations/layer23_attention_weight_min": -27.041067123413086,
      "activations/layer2_attention_weight_max": 33.90913391113281,
      "activations/layer2_attention_weight_min": -30.59193229675293,
      "activations/layer3_attention_weight_max": 100.1627197265625,
      "activations/layer3_attention_weight_min": -99.43653106689453,
      "activations/layer4_attention_weight_max": 105.18575286865234,
      "activations/layer4_attention_weight_min": -105.97456359863281,
      "activations/layer5_attention_weight_max": 52.363677978515625,
      "activations/layer5_attention_weight_min": -61.780059814453125,
      "activations/layer6_attention_weight_max": 47.19693374633789,
      "activations/layer6_attention_weight_min": -49.10444641113281,
      "activations/layer7_attention_weight_max": 95.3922119140625,
      "activations/layer7_attention_weight_min": -98.89240264892578,
      "activations/layer8_attention_weight_max": 44.15925216674805,
      "activations/layer8_attention_weight_min": -46.69733810424805,
      "activations/layer9_attention_weight_max": 32.85716247558594,
      "activations/layer9_attention_weight_min": -36.57238006591797,
      "epoch": 20.36,
      "learning_rate": 1.887310606060606e-05,
      "loss": 2.7163,
      "step": 350450
    },
    {
      "activations/layer0_attention_weight_max": 15.9447660446167,
      "activations/layer0_attention_weight_min": -12.84072208404541,
      "activations/layer10_attention_weight_max": 37.44440460205078,
      "activations/layer10_attention_weight_min": -37.20061111450195,
      "activations/layer11_attention_weight_max": 36.554100036621094,
      "activations/layer11_attention_weight_min": -36.06200408935547,
      "activations/layer12_attention_weight_max": 28.197450637817383,
      "activations/layer12_attention_weight_min": -26.40062141418457,
      "activations/layer13_attention_weight_max": 38.7197151184082,
      "activations/layer13_attention_weight_min": -33.36233139038086,
      "activations/layer14_attention_weight_max": 44.00096893310547,
      "activations/layer14_attention_weight_min": -37.554473876953125,
      "activations/layer15_attention_weight_max": 39.33622360229492,
      "activations/layer15_attention_weight_min": -37.37542724609375,
      "activations/layer16_attention_weight_max": 26.262310028076172,
      "activations/layer16_attention_weight_min": -27.23002815246582,
      "activations/layer17_attention_weight_max": 28.340808868408203,
      "activations/layer17_attention_weight_min": -24.60334587097168,
      "activations/layer18_attention_weight_max": 31.041950225830078,
      "activations/layer18_attention_weight_min": -25.663755416870117,
      "activations/layer19_attention_weight_max": 30.802997589111328,
      "activations/layer19_attention_weight_min": -31.642187118530273,
      "activations/layer1_attention_weight_max": 15.638940811157227,
      "activations/layer1_attention_weight_min": -15.141886711120605,
      "activations/layer20_attention_weight_max": 26.05718994140625,
      "activations/layer20_attention_weight_min": -24.13844871520996,
      "activations/layer21_attention_weight_max": 26.5382137298584,
      "activations/layer21_attention_weight_min": -23.878971099853516,
      "activations/layer22_attention_weight_max": 47.38838577270508,
      "activations/layer22_attention_weight_min": -33.3769416809082,
      "activations/layer23_attention_weight_max": 30.392532348632812,
      "activations/layer23_attention_weight_min": -25.06153106689453,
      "activations/layer2_attention_weight_max": 35.32685089111328,
      "activations/layer2_attention_weight_min": -33.53013610839844,
      "activations/layer3_attention_weight_max": 102.85999298095703,
      "activations/layer3_attention_weight_min": -102.16549682617188,
      "activations/layer4_attention_weight_max": 105.6748275756836,
      "activations/layer4_attention_weight_min": -106.32889556884766,
      "activations/layer5_attention_weight_max": 49.74457550048828,
      "activations/layer5_attention_weight_min": -62.382110595703125,
      "activations/layer6_attention_weight_max": 44.001258850097656,
      "activations/layer6_attention_weight_min": -46.81416702270508,
      "activations/layer7_attention_weight_max": 93.0479736328125,
      "activations/layer7_attention_weight_min": -96.33135223388672,
      "activations/layer8_attention_weight_max": 46.16708755493164,
      "activations/layer8_attention_weight_min": -46.81401443481445,
      "activations/layer9_attention_weight_max": 33.497276306152344,
      "activations/layer9_attention_weight_min": -36.63328170776367,
      "epoch": 20.37,
      "learning_rate": 1.8854166666666663e-05,
      "loss": 2.718,
      "step": 350500
    },
    {
      "activations/layer0_attention_weight_max": 16.130144119262695,
      "activations/layer0_attention_weight_min": -14.3109130859375,
      "activations/layer10_attention_weight_max": 38.507850646972656,
      "activations/layer10_attention_weight_min": -36.9000244140625,
      "activations/layer11_attention_weight_max": 38.087249755859375,
      "activations/layer11_attention_weight_min": -37.463199615478516,
      "activations/layer12_attention_weight_max": 28.99113655090332,
      "activations/layer12_attention_weight_min": -25.880775451660156,
      "activations/layer13_attention_weight_max": 42.81929016113281,
      "activations/layer13_attention_weight_min": -34.42026901245117,
      "activations/layer14_attention_weight_max": 45.41101837158203,
      "activations/layer14_attention_weight_min": -37.619598388671875,
      "activations/layer15_attention_weight_max": 40.52584457397461,
      "activations/layer15_attention_weight_min": -34.08320236206055,
      "activations/layer16_attention_weight_max": 28.846717834472656,
      "activations/layer16_attention_weight_min": -27.77586555480957,
      "activations/layer17_attention_weight_max": 30.839115142822266,
      "activations/layer17_attention_weight_min": -26.956525802612305,
      "activations/layer18_attention_weight_max": 34.38848876953125,
      "activations/layer18_attention_weight_min": -26.154598236083984,
      "activations/layer19_attention_weight_max": 34.363128662109375,
      "activations/layer19_attention_weight_min": -31.5660343170166,
      "activations/layer1_attention_weight_max": 15.315801620483398,
      "activations/layer1_attention_weight_min": -14.593728065490723,
      "activations/layer20_attention_weight_max": 31.7777042388916,
      "activations/layer20_attention_weight_min": -24.156091690063477,
      "activations/layer21_attention_weight_max": 30.402969360351562,
      "activations/layer21_attention_weight_min": -24.122417449951172,
      "activations/layer22_attention_weight_max": 43.84161376953125,
      "activations/layer22_attention_weight_min": -29.981842041015625,
      "activations/layer23_attention_weight_max": 32.70579528808594,
      "activations/layer23_attention_weight_min": -25.39189910888672,
      "activations/layer2_attention_weight_max": 32.39479064941406,
      "activations/layer2_attention_weight_min": -32.94761657714844,
      "activations/layer3_attention_weight_max": 101.79378509521484,
      "activations/layer3_attention_weight_min": -103.05967712402344,
      "activations/layer4_attention_weight_max": 104.32356262207031,
      "activations/layer4_attention_weight_min": -110.63519287109375,
      "activations/layer5_attention_weight_max": 49.85828399658203,
      "activations/layer5_attention_weight_min": -61.09994125366211,
      "activations/layer6_attention_weight_max": 44.589500427246094,
      "activations/layer6_attention_weight_min": -48.66440963745117,
      "activations/layer7_attention_weight_max": 96.6759262084961,
      "activations/layer7_attention_weight_min": -100.5405502319336,
      "activations/layer8_attention_weight_max": 43.07965850830078,
      "activations/layer8_attention_weight_min": -46.81081008911133,
      "activations/layer9_attention_weight_max": 33.89658737182617,
      "activations/layer9_attention_weight_min": -35.5717887878418,
      "epoch": 20.37,
      "learning_rate": 1.883522727272727e-05,
      "loss": 2.7154,
      "step": 350550
    },
    {
      "activations/layer0_attention_weight_max": 16.206485748291016,
      "activations/layer0_attention_weight_min": -13.410726547241211,
      "activations/layer10_attention_weight_max": 36.253753662109375,
      "activations/layer10_attention_weight_min": -36.8663215637207,
      "activations/layer11_attention_weight_max": 35.004608154296875,
      "activations/layer11_attention_weight_min": -36.23846435546875,
      "activations/layer12_attention_weight_max": 30.65289878845215,
      "activations/layer12_attention_weight_min": -29.603435516357422,
      "activations/layer13_attention_weight_max": 45.14881134033203,
      "activations/layer13_attention_weight_min": -39.22304153442383,
      "activations/layer14_attention_weight_max": 47.524776458740234,
      "activations/layer14_attention_weight_min": -40.19316482543945,
      "activations/layer15_attention_weight_max": 42.64386749267578,
      "activations/layer15_attention_weight_min": -37.17855453491211,
      "activations/layer16_attention_weight_max": 32.00718688964844,
      "activations/layer16_attention_weight_min": -29.019304275512695,
      "activations/layer17_attention_weight_max": 34.81553649902344,
      "activations/layer17_attention_weight_min": -25.78619384765625,
      "activations/layer18_attention_weight_max": 33.323665618896484,
      "activations/layer18_attention_weight_min": -26.51954460144043,
      "activations/layer19_attention_weight_max": 38.42970657348633,
      "activations/layer19_attention_weight_min": -32.7824592590332,
      "activations/layer1_attention_weight_max": 16.198854446411133,
      "activations/layer1_attention_weight_min": -14.513066291809082,
      "activations/layer20_attention_weight_max": 31.63898277282715,
      "activations/layer20_attention_weight_min": -26.61977767944336,
      "activations/layer21_attention_weight_max": 29.946109771728516,
      "activations/layer21_attention_weight_min": -25.82930564880371,
      "activations/layer22_attention_weight_max": 43.99747085571289,
      "activations/layer22_attention_weight_min": -36.607540130615234,
      "activations/layer23_attention_weight_max": 34.21570587158203,
      "activations/layer23_attention_weight_min": -26.653226852416992,
      "activations/layer2_attention_weight_max": 32.37429428100586,
      "activations/layer2_attention_weight_min": -30.511272430419922,
      "activations/layer3_attention_weight_max": 100.52498626708984,
      "activations/layer3_attention_weight_min": -99.92500305175781,
      "activations/layer4_attention_weight_max": 105.75341796875,
      "activations/layer4_attention_weight_min": -110.75225067138672,
      "activations/layer5_attention_weight_max": 53.103790283203125,
      "activations/layer5_attention_weight_min": -62.320159912109375,
      "activations/layer6_attention_weight_max": 44.4515266418457,
      "activations/layer6_attention_weight_min": -48.74726486206055,
      "activations/layer7_attention_weight_max": 98.00540924072266,
      "activations/layer7_attention_weight_min": -104.62948608398438,
      "activations/layer8_attention_weight_max": 43.53300094604492,
      "activations/layer8_attention_weight_min": -45.51011657714844,
      "activations/layer9_attention_weight_max": 33.78647994995117,
      "activations/layer9_attention_weight_min": -37.78754806518555,
      "epoch": 20.37,
      "learning_rate": 1.8816287878787876e-05,
      "loss": 2.7144,
      "step": 350600
    },
    {
      "activations/layer0_attention_weight_max": 17.084232330322266,
      "activations/layer0_attention_weight_min": -16.07056427001953,
      "activations/layer10_attention_weight_max": 36.019325256347656,
      "activations/layer10_attention_weight_min": -37.125885009765625,
      "activations/layer11_attention_weight_max": 35.992103576660156,
      "activations/layer11_attention_weight_min": -39.355472564697266,
      "activations/layer12_attention_weight_max": 28.46589469909668,
      "activations/layer12_attention_weight_min": -27.51451301574707,
      "activations/layer13_attention_weight_max": 40.697532653808594,
      "activations/layer13_attention_weight_min": -39.74772644042969,
      "activations/layer14_attention_weight_max": 44.8595085144043,
      "activations/layer14_attention_weight_min": -44.528953552246094,
      "activations/layer15_attention_weight_max": 40.49339294433594,
      "activations/layer15_attention_weight_min": -42.18765640258789,
      "activations/layer16_attention_weight_max": 28.489925384521484,
      "activations/layer16_attention_weight_min": -30.155614852905273,
      "activations/layer17_attention_weight_max": 29.21615219116211,
      "activations/layer17_attention_weight_min": -29.555952072143555,
      "activations/layer18_attention_weight_max": 31.504131317138672,
      "activations/layer18_attention_weight_min": -26.91067123413086,
      "activations/layer19_attention_weight_max": 36.619258880615234,
      "activations/layer19_attention_weight_min": -32.89395523071289,
      "activations/layer1_attention_weight_max": 15.433051109313965,
      "activations/layer1_attention_weight_min": -14.612579345703125,
      "activations/layer20_attention_weight_max": 31.314186096191406,
      "activations/layer20_attention_weight_min": -25.791879653930664,
      "activations/layer21_attention_weight_max": 31.319854736328125,
      "activations/layer21_attention_weight_min": -26.51119041442871,
      "activations/layer22_attention_weight_max": 44.540931701660156,
      "activations/layer22_attention_weight_min": -33.287452697753906,
      "activations/layer23_attention_weight_max": 32.931739807128906,
      "activations/layer23_attention_weight_min": -25.986204147338867,
      "activations/layer2_attention_weight_max": 33.066505432128906,
      "activations/layer2_attention_weight_min": -31.585844039916992,
      "activations/layer3_attention_weight_max": 99.73185729980469,
      "activations/layer3_attention_weight_min": -98.78944396972656,
      "activations/layer4_attention_weight_max": 106.95356750488281,
      "activations/layer4_attention_weight_min": -108.45198822021484,
      "activations/layer5_attention_weight_max": 48.61274337768555,
      "activations/layer5_attention_weight_min": -61.848445892333984,
      "activations/layer6_attention_weight_max": 44.07495880126953,
      "activations/layer6_attention_weight_min": -46.80105972290039,
      "activations/layer7_attention_weight_max": 93.2203598022461,
      "activations/layer7_attention_weight_min": -95.64054870605469,
      "activations/layer8_attention_weight_max": 43.647438049316406,
      "activations/layer8_attention_weight_min": -45.36909866333008,
      "activations/layer9_attention_weight_max": 33.50908279418945,
      "activations/layer9_attention_weight_min": -35.63413619995117,
      "epoch": 20.37,
      "learning_rate": 1.8797348484848484e-05,
      "loss": 2.7246,
      "step": 350650
    },
    {
      "activations/layer0_attention_weight_max": 16.534496307373047,
      "activations/layer0_attention_weight_min": -14.13744068145752,
      "activations/layer10_attention_weight_max": 32.73844528198242,
      "activations/layer10_attention_weight_min": -34.35017395019531,
      "activations/layer11_attention_weight_max": 32.27285385131836,
      "activations/layer11_attention_weight_min": -35.366127014160156,
      "activations/layer12_attention_weight_max": 28.323348999023438,
      "activations/layer12_attention_weight_min": -28.609317779541016,
      "activations/layer13_attention_weight_max": 40.53175354003906,
      "activations/layer13_attention_weight_min": -36.08563995361328,
      "activations/layer14_attention_weight_max": 45.34071350097656,
      "activations/layer14_attention_weight_min": -39.40583038330078,
      "activations/layer15_attention_weight_max": 37.765052795410156,
      "activations/layer15_attention_weight_min": -35.03956985473633,
      "activations/layer16_attention_weight_max": 28.787870407104492,
      "activations/layer16_attention_weight_min": -26.31597328186035,
      "activations/layer17_attention_weight_max": 30.68427848815918,
      "activations/layer17_attention_weight_min": -25.73467254638672,
      "activations/layer18_attention_weight_max": 32.71123123168945,
      "activations/layer18_attention_weight_min": -27.38521957397461,
      "activations/layer19_attention_weight_max": 32.94977569580078,
      "activations/layer19_attention_weight_min": -31.98331069946289,
      "activations/layer1_attention_weight_max": 16.47875213623047,
      "activations/layer1_attention_weight_min": -14.061137199401855,
      "activations/layer20_attention_weight_max": 29.741416931152344,
      "activations/layer20_attention_weight_min": -26.103832244873047,
      "activations/layer21_attention_weight_max": 28.25359344482422,
      "activations/layer21_attention_weight_min": -24.71508026123047,
      "activations/layer22_attention_weight_max": 38.56379699707031,
      "activations/layer22_attention_weight_min": -32.01807403564453,
      "activations/layer23_attention_weight_max": 31.541494369506836,
      "activations/layer23_attention_weight_min": -25.139968872070312,
      "activations/layer2_attention_weight_max": 32.55007553100586,
      "activations/layer2_attention_weight_min": -31.424835205078125,
      "activations/layer3_attention_weight_max": 97.60879516601562,
      "activations/layer3_attention_weight_min": -100.0903549194336,
      "activations/layer4_attention_weight_max": 104.0145034790039,
      "activations/layer4_attention_weight_min": -110.32274627685547,
      "activations/layer5_attention_weight_max": 49.25699996948242,
      "activations/layer5_attention_weight_min": -64.70295715332031,
      "activations/layer6_attention_weight_max": 44.39468765258789,
      "activations/layer6_attention_weight_min": -47.869449615478516,
      "activations/layer7_attention_weight_max": 91.28617858886719,
      "activations/layer7_attention_weight_min": -101.43729400634766,
      "activations/layer8_attention_weight_max": 39.67936706542969,
      "activations/layer8_attention_weight_min": -42.38486099243164,
      "activations/layer9_attention_weight_max": 30.280282974243164,
      "activations/layer9_attention_weight_min": -33.18370819091797,
      "epoch": 20.38,
      "learning_rate": 1.877840909090909e-05,
      "loss": 2.7106,
      "step": 350700
    },
    {
      "activations/layer0_attention_weight_max": 16.252662658691406,
      "activations/layer0_attention_weight_min": -13.911865234375,
      "activations/layer10_attention_weight_max": 41.80076599121094,
      "activations/layer10_attention_weight_min": -42.41545867919922,
      "activations/layer11_attention_weight_max": 40.79076385498047,
      "activations/layer11_attention_weight_min": -40.738040924072266,
      "activations/layer12_attention_weight_max": 30.027441024780273,
      "activations/layer12_attention_weight_min": -29.572141647338867,
      "activations/layer13_attention_weight_max": 45.26914978027344,
      "activations/layer13_attention_weight_min": -39.29288864135742,
      "activations/layer14_attention_weight_max": 48.892242431640625,
      "activations/layer14_attention_weight_min": -42.75496292114258,
      "activations/layer15_attention_weight_max": 46.0462532043457,
      "activations/layer15_attention_weight_min": -39.87235641479492,
      "activations/layer16_attention_weight_max": 33.084041595458984,
      "activations/layer16_attention_weight_min": -28.674116134643555,
      "activations/layer17_attention_weight_max": 32.24205017089844,
      "activations/layer17_attention_weight_min": -26.104352951049805,
      "activations/layer18_attention_weight_max": 32.93122482299805,
      "activations/layer18_attention_weight_min": -26.828723907470703,
      "activations/layer19_attention_weight_max": 34.36174011230469,
      "activations/layer19_attention_weight_min": -32.473384857177734,
      "activations/layer1_attention_weight_max": 15.287216186523438,
      "activations/layer1_attention_weight_min": -14.901620864868164,
      "activations/layer20_attention_weight_max": 28.223787307739258,
      "activations/layer20_attention_weight_min": -27.93507957458496,
      "activations/layer21_attention_weight_max": 30.519065856933594,
      "activations/layer21_attention_weight_min": -27.253183364868164,
      "activations/layer22_attention_weight_max": 45.59490966796875,
      "activations/layer22_attention_weight_min": -34.588008880615234,
      "activations/layer23_attention_weight_max": 32.27325439453125,
      "activations/layer23_attention_weight_min": -26.23486328125,
      "activations/layer2_attention_weight_max": 34.62742614746094,
      "activations/layer2_attention_weight_min": -33.4250602722168,
      "activations/layer3_attention_weight_max": 105.92279815673828,
      "activations/layer3_attention_weight_min": -109.17273712158203,
      "activations/layer4_attention_weight_max": 110.50080871582031,
      "activations/layer4_attention_weight_min": -116.73866271972656,
      "activations/layer5_attention_weight_max": 54.3057975769043,
      "activations/layer5_attention_weight_min": -63.543479919433594,
      "activations/layer6_attention_weight_max": 47.13381576538086,
      "activations/layer6_attention_weight_min": -50.71520233154297,
      "activations/layer7_attention_weight_max": 102.65918731689453,
      "activations/layer7_attention_weight_min": -104.70584869384766,
      "activations/layer8_attention_weight_max": 47.69286346435547,
      "activations/layer8_attention_weight_min": -47.93801498413086,
      "activations/layer9_attention_weight_max": 37.50211715698242,
      "activations/layer9_attention_weight_min": -39.095367431640625,
      "epoch": 20.38,
      "learning_rate": 1.8759469696969698e-05,
      "loss": 2.7192,
      "step": 350750
    },
    {
      "activations/layer0_attention_weight_max": 17.20890235900879,
      "activations/layer0_attention_weight_min": -12.892620086669922,
      "activations/layer10_attention_weight_max": 41.408199310302734,
      "activations/layer10_attention_weight_min": -41.12861251831055,
      "activations/layer11_attention_weight_max": 40.762306213378906,
      "activations/layer11_attention_weight_min": -41.181175231933594,
      "activations/layer12_attention_weight_max": 29.863527297973633,
      "activations/layer12_attention_weight_min": -28.579383850097656,
      "activations/layer13_attention_weight_max": 45.17683410644531,
      "activations/layer13_attention_weight_min": -35.84901809692383,
      "activations/layer14_attention_weight_max": 49.852195739746094,
      "activations/layer14_attention_weight_min": -40.647666931152344,
      "activations/layer15_attention_weight_max": 44.65421676635742,
      "activations/layer15_attention_weight_min": -39.82008743286133,
      "activations/layer16_attention_weight_max": 32.03397750854492,
      "activations/layer16_attention_weight_min": -27.433910369873047,
      "activations/layer17_attention_weight_max": 32.672264099121094,
      "activations/layer17_attention_weight_min": -26.66912269592285,
      "activations/layer18_attention_weight_max": 35.16387939453125,
      "activations/layer18_attention_weight_min": -28.109886169433594,
      "activations/layer19_attention_weight_max": 35.897193908691406,
      "activations/layer19_attention_weight_min": -31.347320556640625,
      "activations/layer1_attention_weight_max": 15.924287796020508,
      "activations/layer1_attention_weight_min": -14.538846969604492,
      "activations/layer20_attention_weight_max": 32.90735626220703,
      "activations/layer20_attention_weight_min": -27.539228439331055,
      "activations/layer21_attention_weight_max": 34.27922439575195,
      "activations/layer21_attention_weight_min": -25.82370948791504,
      "activations/layer22_attention_weight_max": 45.0267333984375,
      "activations/layer22_attention_weight_min": -35.23694610595703,
      "activations/layer23_attention_weight_max": 35.36553955078125,
      "activations/layer23_attention_weight_min": -25.052242279052734,
      "activations/layer2_attention_weight_max": 33.325992584228516,
      "activations/layer2_attention_weight_min": -34.575889587402344,
      "activations/layer3_attention_weight_max": 104.57705688476562,
      "activations/layer3_attention_weight_min": -109.26126861572266,
      "activations/layer4_attention_weight_max": 106.43025970458984,
      "activations/layer4_attention_weight_min": -108.94597625732422,
      "activations/layer5_attention_weight_max": 50.900184631347656,
      "activations/layer5_attention_weight_min": -64.4188003540039,
      "activations/layer6_attention_weight_max": 46.61495590209961,
      "activations/layer6_attention_weight_min": -49.358741760253906,
      "activations/layer7_attention_weight_max": 96.7349624633789,
      "activations/layer7_attention_weight_min": -101.77225494384766,
      "activations/layer8_attention_weight_max": 46.79998016357422,
      "activations/layer8_attention_weight_min": -49.05093765258789,
      "activations/layer9_attention_weight_max": 35.267948150634766,
      "activations/layer9_attention_weight_min": -40.47669219970703,
      "epoch": 20.38,
      "learning_rate": 1.8740530303030302e-05,
      "loss": 2.7189,
      "step": 350800
    },
    {
      "activations/layer0_attention_weight_max": 16.80097007751465,
      "activations/layer0_attention_weight_min": -12.561208724975586,
      "activations/layer10_attention_weight_max": 37.361629486083984,
      "activations/layer10_attention_weight_min": -37.102020263671875,
      "activations/layer11_attention_weight_max": 37.16451644897461,
      "activations/layer11_attention_weight_min": -36.49884033203125,
      "activations/layer12_attention_weight_max": 29.96406364440918,
      "activations/layer12_attention_weight_min": -28.01218605041504,
      "activations/layer13_attention_weight_max": 41.84614562988281,
      "activations/layer13_attention_weight_min": -39.22517395019531,
      "activations/layer14_attention_weight_max": 44.474185943603516,
      "activations/layer14_attention_weight_min": -43.20062255859375,
      "activations/layer15_attention_weight_max": 40.158973693847656,
      "activations/layer15_attention_weight_min": -39.27081298828125,
      "activations/layer16_attention_weight_max": 27.58970832824707,
      "activations/layer16_attention_weight_min": -29.855356216430664,
      "activations/layer17_attention_weight_max": 29.326494216918945,
      "activations/layer17_attention_weight_min": -26.881786346435547,
      "activations/layer18_attention_weight_max": 29.104318618774414,
      "activations/layer18_attention_weight_min": -27.184974670410156,
      "activations/layer19_attention_weight_max": 30.576553344726562,
      "activations/layer19_attention_weight_min": -31.22212791442871,
      "activations/layer1_attention_weight_max": 16.568735122680664,
      "activations/layer1_attention_weight_min": -13.202571868896484,
      "activations/layer20_attention_weight_max": 25.38043212890625,
      "activations/layer20_attention_weight_min": -24.638225555419922,
      "activations/layer21_attention_weight_max": 24.128398895263672,
      "activations/layer21_attention_weight_min": -23.714689254760742,
      "activations/layer22_attention_weight_max": 39.330711364746094,
      "activations/layer22_attention_weight_min": -31.350847244262695,
      "activations/layer23_attention_weight_max": 30.195907592773438,
      "activations/layer23_attention_weight_min": -25.631420135498047,
      "activations/layer2_attention_weight_max": 32.69257736206055,
      "activations/layer2_attention_weight_min": -35.84590530395508,
      "activations/layer3_attention_weight_max": 102.17053985595703,
      "activations/layer3_attention_weight_min": -104.95905303955078,
      "activations/layer4_attention_weight_max": 106.83733367919922,
      "activations/layer4_attention_weight_min": -112.21435546875,
      "activations/layer5_attention_weight_max": 52.34446716308594,
      "activations/layer5_attention_weight_min": -58.79824447631836,
      "activations/layer6_attention_weight_max": 46.0850715637207,
      "activations/layer6_attention_weight_min": -47.47578430175781,
      "activations/layer7_attention_weight_max": 100.14527893066406,
      "activations/layer7_attention_weight_min": -100.0328140258789,
      "activations/layer8_attention_weight_max": 43.80708694458008,
      "activations/layer8_attention_weight_min": -46.37529373168945,
      "activations/layer9_attention_weight_max": 35.171348571777344,
      "activations/layer9_attention_weight_min": -37.32166290283203,
      "epoch": 20.39,
      "learning_rate": 1.8721590909090907e-05,
      "loss": 2.7206,
      "step": 350850
    },
    {
      "activations/layer0_attention_weight_max": 15.439371109008789,
      "activations/layer0_attention_weight_min": -12.621305465698242,
      "activations/layer10_attention_weight_max": 37.34347152709961,
      "activations/layer10_attention_weight_min": -38.11690902709961,
      "activations/layer11_attention_weight_max": 38.8784065246582,
      "activations/layer11_attention_weight_min": -39.08902359008789,
      "activations/layer12_attention_weight_max": 27.33987045288086,
      "activations/layer12_attention_weight_min": -27.913190841674805,
      "activations/layer13_attention_weight_max": 42.193756103515625,
      "activations/layer13_attention_weight_min": -38.26587677001953,
      "activations/layer14_attention_weight_max": 44.691593170166016,
      "activations/layer14_attention_weight_min": -42.61970520019531,
      "activations/layer15_attention_weight_max": 42.33742141723633,
      "activations/layer15_attention_weight_min": -40.933170318603516,
      "activations/layer16_attention_weight_max": 32.12425994873047,
      "activations/layer16_attention_weight_min": -30.417863845825195,
      "activations/layer17_attention_weight_max": 31.720855712890625,
      "activations/layer17_attention_weight_min": -28.733652114868164,
      "activations/layer18_attention_weight_max": 32.870086669921875,
      "activations/layer18_attention_weight_min": -27.65593147277832,
      "activations/layer19_attention_weight_max": 34.3786506652832,
      "activations/layer19_attention_weight_min": -32.59809494018555,
      "activations/layer1_attention_weight_max": 15.80735969543457,
      "activations/layer1_attention_weight_min": -14.517623901367188,
      "activations/layer20_attention_weight_max": 29.3497314453125,
      "activations/layer20_attention_weight_min": -25.47212791442871,
      "activations/layer21_attention_weight_max": 29.57724952697754,
      "activations/layer21_attention_weight_min": -24.410964965820312,
      "activations/layer22_attention_weight_max": 46.55591583251953,
      "activations/layer22_attention_weight_min": -33.67483901977539,
      "activations/layer23_attention_weight_max": 33.880859375,
      "activations/layer23_attention_weight_min": -24.165943145751953,
      "activations/layer2_attention_weight_max": 34.040733337402344,
      "activations/layer2_attention_weight_min": -32.473365783691406,
      "activations/layer3_attention_weight_max": 104.45767974853516,
      "activations/layer3_attention_weight_min": -105.837158203125,
      "activations/layer4_attention_weight_max": 109.4313735961914,
      "activations/layer4_attention_weight_min": -112.83275604248047,
      "activations/layer5_attention_weight_max": 52.77495574951172,
      "activations/layer5_attention_weight_min": -63.11200714111328,
      "activations/layer6_attention_weight_max": 48.48714065551758,
      "activations/layer6_attention_weight_min": -51.002952575683594,
      "activations/layer7_attention_weight_max": 98.8984146118164,
      "activations/layer7_attention_weight_min": -105.44795227050781,
      "activations/layer8_attention_weight_max": 46.85908508300781,
      "activations/layer8_attention_weight_min": -51.5422248840332,
      "activations/layer9_attention_weight_max": 34.621673583984375,
      "activations/layer9_attention_weight_min": -39.45896911621094,
      "epoch": 20.39,
      "learning_rate": 1.8702651515151516e-05,
      "loss": 2.7212,
      "step": 350900
    },
    {
      "activations/layer0_attention_weight_max": 16.461933135986328,
      "activations/layer0_attention_weight_min": -14.73438549041748,
      "activations/layer10_attention_weight_max": 34.64252853393555,
      "activations/layer10_attention_weight_min": -35.087562561035156,
      "activations/layer11_attention_weight_max": 34.343746185302734,
      "activations/layer11_attention_weight_min": -34.471702575683594,
      "activations/layer12_attention_weight_max": 27.518888473510742,
      "activations/layer12_attention_weight_min": -27.120756149291992,
      "activations/layer13_attention_weight_max": 38.22822952270508,
      "activations/layer13_attention_weight_min": -37.00092697143555,
      "activations/layer14_attention_weight_max": 41.176788330078125,
      "activations/layer14_attention_weight_min": -38.15480422973633,
      "activations/layer15_attention_weight_max": 39.16004180908203,
      "activations/layer15_attention_weight_min": -39.84513854980469,
      "activations/layer16_attention_weight_max": 29.329797744750977,
      "activations/layer16_attention_weight_min": -27.11273956298828,
      "activations/layer17_attention_weight_max": 29.67873191833496,
      "activations/layer17_attention_weight_min": -25.95941734313965,
      "activations/layer18_attention_weight_max": 30.0640926361084,
      "activations/layer18_attention_weight_min": -23.7415771484375,
      "activations/layer19_attention_weight_max": 33.06883239746094,
      "activations/layer19_attention_weight_min": -30.926549911499023,
      "activations/layer1_attention_weight_max": 14.7808198928833,
      "activations/layer1_attention_weight_min": -14.470304489135742,
      "activations/layer20_attention_weight_max": 27.934953689575195,
      "activations/layer20_attention_weight_min": -24.141054153442383,
      "activations/layer21_attention_weight_max": 27.812376022338867,
      "activations/layer21_attention_weight_min": -24.111303329467773,
      "activations/layer22_attention_weight_max": 42.4764289855957,
      "activations/layer22_attention_weight_min": -30.657360076904297,
      "activations/layer23_attention_weight_max": 32.88351821899414,
      "activations/layer23_attention_weight_min": -25.130939483642578,
      "activations/layer2_attention_weight_max": 31.31275177001953,
      "activations/layer2_attention_weight_min": -33.51288986206055,
      "activations/layer3_attention_weight_max": 98.86796569824219,
      "activations/layer3_attention_weight_min": -102.5963134765625,
      "activations/layer4_attention_weight_max": 106.59246063232422,
      "activations/layer4_attention_weight_min": -113.83839416503906,
      "activations/layer5_attention_weight_max": 48.06571578979492,
      "activations/layer5_attention_weight_min": -61.194618225097656,
      "activations/layer6_attention_weight_max": 46.22580337524414,
      "activations/layer6_attention_weight_min": -48.68607711791992,
      "activations/layer7_attention_weight_max": 95.44110870361328,
      "activations/layer7_attention_weight_min": -101.04997253417969,
      "activations/layer8_attention_weight_max": 42.21236038208008,
      "activations/layer8_attention_weight_min": -45.17723846435547,
      "activations/layer9_attention_weight_max": 32.95331573486328,
      "activations/layer9_attention_weight_min": -36.40591812133789,
      "epoch": 20.39,
      "learning_rate": 1.868371212121212e-05,
      "loss": 2.7236,
      "step": 350950
    },
    {
      "activations/layer0_attention_weight_max": 16.04727554321289,
      "activations/layer0_attention_weight_min": -13.500650405883789,
      "activations/layer10_attention_weight_max": 36.615692138671875,
      "activations/layer10_attention_weight_min": -34.206153869628906,
      "activations/layer11_attention_weight_max": 35.05738067626953,
      "activations/layer11_attention_weight_min": -36.02281951904297,
      "activations/layer12_attention_weight_max": 27.304969787597656,
      "activations/layer12_attention_weight_min": -25.889179229736328,
      "activations/layer13_attention_weight_max": 38.204586029052734,
      "activations/layer13_attention_weight_min": -35.80799102783203,
      "activations/layer14_attention_weight_max": 42.32758331298828,
      "activations/layer14_attention_weight_min": -36.93613052368164,
      "activations/layer15_attention_weight_max": 37.534088134765625,
      "activations/layer15_attention_weight_min": -35.7256965637207,
      "activations/layer16_attention_weight_max": 31.277559280395508,
      "activations/layer16_attention_weight_min": -27.099016189575195,
      "activations/layer17_attention_weight_max": 28.914899826049805,
      "activations/layer17_attention_weight_min": -26.12074089050293,
      "activations/layer18_attention_weight_max": 30.704259872436523,
      "activations/layer18_attention_weight_min": -24.95604133605957,
      "activations/layer19_attention_weight_max": 33.69553756713867,
      "activations/layer19_attention_weight_min": -30.48447036743164,
      "activations/layer1_attention_weight_max": 16.05316734313965,
      "activations/layer1_attention_weight_min": -13.936020851135254,
      "activations/layer20_attention_weight_max": 30.089298248291016,
      "activations/layer20_attention_weight_min": -25.73461151123047,
      "activations/layer21_attention_weight_max": 29.44890022277832,
      "activations/layer21_attention_weight_min": -25.3859806060791,
      "activations/layer22_attention_weight_max": 44.938079833984375,
      "activations/layer22_attention_weight_min": -31.591182708740234,
      "activations/layer23_attention_weight_max": 35.08447265625,
      "activations/layer23_attention_weight_min": -26.94091796875,
      "activations/layer2_attention_weight_max": 32.01201629638672,
      "activations/layer2_attention_weight_min": -31.576330184936523,
      "activations/layer3_attention_weight_max": 98.80828857421875,
      "activations/layer3_attention_weight_min": -101.36856079101562,
      "activations/layer4_attention_weight_max": 103.86063385009766,
      "activations/layer4_attention_weight_min": -110.272705078125,
      "activations/layer5_attention_weight_max": 49.60676574707031,
      "activations/layer5_attention_weight_min": -62.883201599121094,
      "activations/layer6_attention_weight_max": 46.79130935668945,
      "activations/layer6_attention_weight_min": -49.771644592285156,
      "activations/layer7_attention_weight_max": 95.40859985351562,
      "activations/layer7_attention_weight_min": -98.82588195800781,
      "activations/layer8_attention_weight_max": 45.520347595214844,
      "activations/layer8_attention_weight_min": -46.248565673828125,
      "activations/layer9_attention_weight_max": 32.22738265991211,
      "activations/layer9_attention_weight_min": -35.391258239746094,
      "epoch": 20.4,
      "learning_rate": 1.8664772727272725e-05,
      "loss": 2.7123,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.4624,
      "eval_samples_per_second": 507.419,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.4624,
      "eval_openwebtext_samples_per_second": 507.419,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9496,
      "eval_wikitext_samples_per_second": 233.889,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_lambada_loss": 2.333984375,
      "eval_lambada_ppl": 10.318974397214378,
      "eval_lambada_runtime": 9.5797,
      "eval_lambada_samples_per_second": 508.263,
      "step": 351000
    },
    {
      "activations/layer0_attention_weight_max": 15.979142189025879,
      "activations/layer0_attention_weight_min": -13.366313934326172,
      "activations/layer10_attention_weight_max": 35.03196334838867,
      "activations/layer10_attention_weight_min": -34.866146087646484,
      "activations/layer11_attention_weight_max": 35.31904602050781,
      "activations/layer11_attention_weight_min": -35.218631744384766,
      "activations/layer12_attention_weight_max": 30.064064025878906,
      "activations/layer12_attention_weight_min": -28.15818214416504,
      "activations/layer13_attention_weight_max": 41.887168884277344,
      "activations/layer13_attention_weight_min": -36.27820587158203,
      "activations/layer14_attention_weight_max": 44.68272399902344,
      "activations/layer14_attention_weight_min": -38.609561920166016,
      "activations/layer15_attention_weight_max": 40.82540512084961,
      "activations/layer15_attention_weight_min": -37.13075256347656,
      "activations/layer16_attention_weight_max": 28.089466094970703,
      "activations/layer16_attention_weight_min": -28.230314254760742,
      "activations/layer17_attention_weight_max": 31.08236312866211,
      "activations/layer17_attention_weight_min": -25.866390228271484,
      "activations/layer18_attention_weight_max": 33.408939361572266,
      "activations/layer18_attention_weight_min": -27.214801788330078,
      "activations/layer19_attention_weight_max": 35.478214263916016,
      "activations/layer19_attention_weight_min": -32.626625061035156,
      "activations/layer1_attention_weight_max": 15.728561401367188,
      "activations/layer1_attention_weight_min": -15.42690658569336,
      "activations/layer20_attention_weight_max": 28.669233322143555,
      "activations/layer20_attention_weight_min": -25.725595474243164,
      "activations/layer21_attention_weight_max": 30.720626831054688,
      "activations/layer21_attention_weight_min": -25.073204040527344,
      "activations/layer22_attention_weight_max": 42.188804626464844,
      "activations/layer22_attention_weight_min": -30.328792572021484,
      "activations/layer23_attention_weight_max": 32.48157501220703,
      "activations/layer23_attention_weight_min": -27.132047653198242,
      "activations/layer2_attention_weight_max": 32.37196731567383,
      "activations/layer2_attention_weight_min": -32.888511657714844,
      "activations/layer3_attention_weight_max": 103.86762237548828,
      "activations/layer3_attention_weight_min": -105.65650177001953,
      "activations/layer4_attention_weight_max": 109.7083740234375,
      "activations/layer4_attention_weight_min": -112.77680969238281,
      "activations/layer5_attention_weight_max": 51.70388412475586,
      "activations/layer5_attention_weight_min": -62.36469650268555,
      "activations/layer6_attention_weight_max": 44.47514724731445,
      "activations/layer6_attention_weight_min": -50.85367202758789,
      "activations/layer7_attention_weight_max": 104.98480987548828,
      "activations/layer7_attention_weight_min": -102.924072265625,
      "activations/layer8_attention_weight_max": 45.08558654785156,
      "activations/layer8_attention_weight_min": -48.171600341796875,
      "activations/layer9_attention_weight_max": 33.59850311279297,
      "activations/layer9_attention_weight_min": -36.60469436645508,
      "epoch": 20.4,
      "learning_rate": 1.8645833333333334e-05,
      "loss": 2.7359,
      "step": 351050
    },
    {
      "activations/layer0_attention_weight_max": 16.226070404052734,
      "activations/layer0_attention_weight_min": -14.086433410644531,
      "activations/layer10_attention_weight_max": 34.95580291748047,
      "activations/layer10_attention_weight_min": -35.10251998901367,
      "activations/layer11_attention_weight_max": 32.991092681884766,
      "activations/layer11_attention_weight_min": -33.97018051147461,
      "activations/layer12_attention_weight_max": 26.330060958862305,
      "activations/layer12_attention_weight_min": -26.82500648498535,
      "activations/layer13_attention_weight_max": 37.66704559326172,
      "activations/layer13_attention_weight_min": -35.79218673706055,
      "activations/layer14_attention_weight_max": 41.6319580078125,
      "activations/layer14_attention_weight_min": -38.79100036621094,
      "activations/layer15_attention_weight_max": 36.537689208984375,
      "activations/layer15_attention_weight_min": -36.11685562133789,
      "activations/layer16_attention_weight_max": 29.283151626586914,
      "activations/layer16_attention_weight_min": -30.086225509643555,
      "activations/layer17_attention_weight_max": 29.442983627319336,
      "activations/layer17_attention_weight_min": -26.409433364868164,
      "activations/layer18_attention_weight_max": 31.623022079467773,
      "activations/layer18_attention_weight_min": -25.85427474975586,
      "activations/layer19_attention_weight_max": 32.81940841674805,
      "activations/layer19_attention_weight_min": -32.44392395019531,
      "activations/layer1_attention_weight_max": 15.739643096923828,
      "activations/layer1_attention_weight_min": -14.260750770568848,
      "activations/layer20_attention_weight_max": 27.458112716674805,
      "activations/layer20_attention_weight_min": -26.4493350982666,
      "activations/layer21_attention_weight_max": 27.40811538696289,
      "activations/layer21_attention_weight_min": -25.271793365478516,
      "activations/layer22_attention_weight_max": 39.83860397338867,
      "activations/layer22_attention_weight_min": -31.47915267944336,
      "activations/layer23_attention_weight_max": 32.36132049560547,
      "activations/layer23_attention_weight_min": -25.915966033935547,
      "activations/layer2_attention_weight_max": 31.578893661499023,
      "activations/layer2_attention_weight_min": -33.79613494873047,
      "activations/layer3_attention_weight_max": 101.85738372802734,
      "activations/layer3_attention_weight_min": -104.73628234863281,
      "activations/layer4_attention_weight_max": 107.99483489990234,
      "activations/layer4_attention_weight_min": -113.00142669677734,
      "activations/layer5_attention_weight_max": 54.398681640625,
      "activations/layer5_attention_weight_min": -65.19308471679688,
      "activations/layer6_attention_weight_max": 46.73360061645508,
      "activations/layer6_attention_weight_min": -50.01157760620117,
      "activations/layer7_attention_weight_max": 99.37345123291016,
      "activations/layer7_attention_weight_min": -98.93649291992188,
      "activations/layer8_attention_weight_max": 44.5151481628418,
      "activations/layer8_attention_weight_min": -45.95317459106445,
      "activations/layer9_attention_weight_max": 32.336055755615234,
      "activations/layer9_attention_weight_min": -34.49156188964844,
      "epoch": 20.4,
      "learning_rate": 1.862689393939394e-05,
      "loss": 2.7147,
      "step": 351100
    },
    {
      "activations/layer0_attention_weight_max": 15.824838638305664,
      "activations/layer0_attention_weight_min": -14.466714859008789,
      "activations/layer10_attention_weight_max": 36.97660827636719,
      "activations/layer10_attention_weight_min": -38.825233459472656,
      "activations/layer11_attention_weight_max": 36.087890625,
      "activations/layer11_attention_weight_min": -40.201988220214844,
      "activations/layer12_attention_weight_max": 29.392351150512695,
      "activations/layer12_attention_weight_min": -29.269020080566406,
      "activations/layer13_attention_weight_max": 44.395694732666016,
      "activations/layer13_attention_weight_min": -38.43042755126953,
      "activations/layer14_attention_weight_max": 43.12703323364258,
      "activations/layer14_attention_weight_min": -42.41523361206055,
      "activations/layer15_attention_weight_max": 40.28378677368164,
      "activations/layer15_attention_weight_min": -37.82514953613281,
      "activations/layer16_attention_weight_max": 29.851041793823242,
      "activations/layer16_attention_weight_min": -27.566131591796875,
      "activations/layer17_attention_weight_max": 28.065366744995117,
      "activations/layer17_attention_weight_min": -25.10814094543457,
      "activations/layer18_attention_weight_max": 32.36445617675781,
      "activations/layer18_attention_weight_min": -25.586166381835938,
      "activations/layer19_attention_weight_max": 32.881526947021484,
      "activations/layer19_attention_weight_min": -31.112613677978516,
      "activations/layer1_attention_weight_max": 15.528421401977539,
      "activations/layer1_attention_weight_min": -14.857024192810059,
      "activations/layer20_attention_weight_max": 28.927621841430664,
      "activations/layer20_attention_weight_min": -24.932762145996094,
      "activations/layer21_attention_weight_max": 29.485727310180664,
      "activations/layer21_attention_weight_min": -22.952306747436523,
      "activations/layer22_attention_weight_max": 39.676361083984375,
      "activations/layer22_attention_weight_min": -32.50015640258789,
      "activations/layer23_attention_weight_max": 34.08711242675781,
      "activations/layer23_attention_weight_min": -27.740835189819336,
      "activations/layer2_attention_weight_max": 35.55353546142578,
      "activations/layer2_attention_weight_min": -34.79120635986328,
      "activations/layer3_attention_weight_max": 107.23955535888672,
      "activations/layer3_attention_weight_min": -110.11951446533203,
      "activations/layer4_attention_weight_max": 112.0371322631836,
      "activations/layer4_attention_weight_min": -118.26507568359375,
      "activations/layer5_attention_weight_max": 52.59187316894531,
      "activations/layer5_attention_weight_min": -63.937477111816406,
      "activations/layer6_attention_weight_max": 46.75397491455078,
      "activations/layer6_attention_weight_min": -50.5405158996582,
      "activations/layer7_attention_weight_max": 101.53144073486328,
      "activations/layer7_attention_weight_min": -108.0732650756836,
      "activations/layer8_attention_weight_max": 45.7184944152832,
      "activations/layer8_attention_weight_min": -50.23593521118164,
      "activations/layer9_attention_weight_max": 34.92970275878906,
      "activations/layer9_attention_weight_min": -38.89680099487305,
      "epoch": 20.4,
      "learning_rate": 1.8607954545454543e-05,
      "loss": 2.7265,
      "step": 351150
    },
    {
      "activations/layer0_attention_weight_max": 16.045482635498047,
      "activations/layer0_attention_weight_min": -15.014144897460938,
      "activations/layer10_attention_weight_max": 41.47796630859375,
      "activations/layer10_attention_weight_min": -38.718421936035156,
      "activations/layer11_attention_weight_max": 43.65277099609375,
      "activations/layer11_attention_weight_min": -42.73716354370117,
      "activations/layer12_attention_weight_max": 26.836442947387695,
      "activations/layer12_attention_weight_min": -27.28350257873535,
      "activations/layer13_attention_weight_max": 37.4361686706543,
      "activations/layer13_attention_weight_min": -35.21530532836914,
      "activations/layer14_attention_weight_max": 42.3314208984375,
      "activations/layer14_attention_weight_min": -37.463348388671875,
      "activations/layer15_attention_weight_max": 40.10337448120117,
      "activations/layer15_attention_weight_min": -38.26386642456055,
      "activations/layer16_attention_weight_max": 27.975788116455078,
      "activations/layer16_attention_weight_min": -26.774370193481445,
      "activations/layer17_attention_weight_max": 27.391870498657227,
      "activations/layer17_attention_weight_min": -27.445676803588867,
      "activations/layer18_attention_weight_max": 31.776094436645508,
      "activations/layer18_attention_weight_min": -25.178466796875,
      "activations/layer19_attention_weight_max": 32.46503448486328,
      "activations/layer19_attention_weight_min": -29.950193405151367,
      "activations/layer1_attention_weight_max": 16.37240982055664,
      "activations/layer1_attention_weight_min": -14.501558303833008,
      "activations/layer20_attention_weight_max": 27.9556884765625,
      "activations/layer20_attention_weight_min": -24.30387306213379,
      "activations/layer21_attention_weight_max": 27.046342849731445,
      "activations/layer21_attention_weight_min": -24.088207244873047,
      "activations/layer22_attention_weight_max": 39.6043815612793,
      "activations/layer22_attention_weight_min": -32.5299186706543,
      "activations/layer23_attention_weight_max": 30.515304565429688,
      "activations/layer23_attention_weight_min": -25.022991180419922,
      "activations/layer2_attention_weight_max": 33.04890441894531,
      "activations/layer2_attention_weight_min": -33.34959411621094,
      "activations/layer3_attention_weight_max": 105.6855697631836,
      "activations/layer3_attention_weight_min": -104.22662353515625,
      "activations/layer4_attention_weight_max": 107.20660400390625,
      "activations/layer4_attention_weight_min": -107.42919158935547,
      "activations/layer5_attention_weight_max": 53.58544158935547,
      "activations/layer5_attention_weight_min": -61.865318298339844,
      "activations/layer6_attention_weight_max": 44.24126052856445,
      "activations/layer6_attention_weight_min": -46.62913513183594,
      "activations/layer7_attention_weight_max": 96.03089904785156,
      "activations/layer7_attention_weight_min": -100.62779235839844,
      "activations/layer8_attention_weight_max": 46.1584358215332,
      "activations/layer8_attention_weight_min": -47.08262634277344,
      "activations/layer9_attention_weight_max": 35.53947067260742,
      "activations/layer9_attention_weight_min": -37.02758026123047,
      "epoch": 20.41,
      "learning_rate": 1.858901515151515e-05,
      "loss": 2.7116,
      "step": 351200
    },
    {
      "activations/layer0_attention_weight_max": 16.800521850585938,
      "activations/layer0_attention_weight_min": -13.071459770202637,
      "activations/layer10_attention_weight_max": 35.00350570678711,
      "activations/layer10_attention_weight_min": -35.28569793701172,
      "activations/layer11_attention_weight_max": 34.360557556152344,
      "activations/layer11_attention_weight_min": -35.974918365478516,
      "activations/layer12_attention_weight_max": 26.81549835205078,
      "activations/layer12_attention_weight_min": -26.478717803955078,
      "activations/layer13_attention_weight_max": 38.89809799194336,
      "activations/layer13_attention_weight_min": -37.888160705566406,
      "activations/layer14_attention_weight_max": 43.26653289794922,
      "activations/layer14_attention_weight_min": -39.075294494628906,
      "activations/layer15_attention_weight_max": 41.432315826416016,
      "activations/layer15_attention_weight_min": -35.70751953125,
      "activations/layer16_attention_weight_max": 31.06114387512207,
      "activations/layer16_attention_weight_min": -28.561717987060547,
      "activations/layer17_attention_weight_max": 27.96544075012207,
      "activations/layer17_attention_weight_min": -25.35801124572754,
      "activations/layer18_attention_weight_max": 31.77509880065918,
      "activations/layer18_attention_weight_min": -28.544353485107422,
      "activations/layer19_attention_weight_max": 30.59263038635254,
      "activations/layer19_attention_weight_min": -33.04286575317383,
      "activations/layer1_attention_weight_max": 15.54821491241455,
      "activations/layer1_attention_weight_min": -15.460740089416504,
      "activations/layer20_attention_weight_max": 27.363718032836914,
      "activations/layer20_attention_weight_min": -27.398895263671875,
      "activations/layer21_attention_weight_max": 27.73777198791504,
      "activations/layer21_attention_weight_min": -26.10057830810547,
      "activations/layer22_attention_weight_max": 38.35831069946289,
      "activations/layer22_attention_weight_min": -31.2294979095459,
      "activations/layer23_attention_weight_max": 29.039846420288086,
      "activations/layer23_attention_weight_min": -26.228309631347656,
      "activations/layer2_attention_weight_max": 32.84291076660156,
      "activations/layer2_attention_weight_min": -31.993560791015625,
      "activations/layer3_attention_weight_max": 101.90168762207031,
      "activations/layer3_attention_weight_min": -101.79141998291016,
      "activations/layer4_attention_weight_max": 107.70696258544922,
      "activations/layer4_attention_weight_min": -109.40990447998047,
      "activations/layer5_attention_weight_max": 49.379676818847656,
      "activations/layer5_attention_weight_min": -62.76691436767578,
      "activations/layer6_attention_weight_max": 45.25423812866211,
      "activations/layer6_attention_weight_min": -48.03923416137695,
      "activations/layer7_attention_weight_max": 93.38909912109375,
      "activations/layer7_attention_weight_min": -99.33336639404297,
      "activations/layer8_attention_weight_max": 43.45894241333008,
      "activations/layer8_attention_weight_min": -44.49785232543945,
      "activations/layer9_attention_weight_max": 33.60203552246094,
      "activations/layer9_attention_weight_min": -35.217071533203125,
      "epoch": 20.41,
      "learning_rate": 1.8570075757575757e-05,
      "loss": 2.7134,
      "step": 351250
    },
    {
      "activations/layer0_attention_weight_max": 16.88922119140625,
      "activations/layer0_attention_weight_min": -13.271160125732422,
      "activations/layer10_attention_weight_max": 37.02747344970703,
      "activations/layer10_attention_weight_min": -35.42396545410156,
      "activations/layer11_attention_weight_max": 36.529991149902344,
      "activations/layer11_attention_weight_min": -36.435211181640625,
      "activations/layer12_attention_weight_max": 27.362201690673828,
      "activations/layer12_attention_weight_min": -25.691503524780273,
      "activations/layer13_attention_weight_max": 40.28133010864258,
      "activations/layer13_attention_weight_min": -34.99849319458008,
      "activations/layer14_attention_weight_max": 45.45180130004883,
      "activations/layer14_attention_weight_min": -39.859432220458984,
      "activations/layer15_attention_weight_max": 44.201297760009766,
      "activations/layer15_attention_weight_min": -39.31026840209961,
      "activations/layer16_attention_weight_max": 31.14954376220703,
      "activations/layer16_attention_weight_min": -29.499170303344727,
      "activations/layer17_attention_weight_max": 29.303043365478516,
      "activations/layer17_attention_weight_min": -25.295116424560547,
      "activations/layer18_attention_weight_max": 30.15190315246582,
      "activations/layer18_attention_weight_min": -25.237211227416992,
      "activations/layer19_attention_weight_max": 34.15312194824219,
      "activations/layer19_attention_weight_min": -31.919387817382812,
      "activations/layer1_attention_weight_max": 15.99100112915039,
      "activations/layer1_attention_weight_min": -15.088629722595215,
      "activations/layer20_attention_weight_max": 28.381107330322266,
      "activations/layer20_attention_weight_min": -26.5496768951416,
      "activations/layer21_attention_weight_max": 26.934770584106445,
      "activations/layer21_attention_weight_min": -24.68878173828125,
      "activations/layer22_attention_weight_max": 40.07112503051758,
      "activations/layer22_attention_weight_min": -30.314729690551758,
      "activations/layer23_attention_weight_max": 32.962181091308594,
      "activations/layer23_attention_weight_min": -26.47340965270996,
      "activations/layer2_attention_weight_max": 34.25062561035156,
      "activations/layer2_attention_weight_min": -33.40522766113281,
      "activations/layer3_attention_weight_max": 103.04147338867188,
      "activations/layer3_attention_weight_min": -104.98099517822266,
      "activations/layer4_attention_weight_max": 110.29786682128906,
      "activations/layer4_attention_weight_min": -114.46768951416016,
      "activations/layer5_attention_weight_max": 51.17842102050781,
      "activations/layer5_attention_weight_min": -60.613014221191406,
      "activations/layer6_attention_weight_max": 43.944149017333984,
      "activations/layer6_attention_weight_min": -46.32534408569336,
      "activations/layer7_attention_weight_max": 100.98570251464844,
      "activations/layer7_attention_weight_min": -102.07047271728516,
      "activations/layer8_attention_weight_max": 43.438720703125,
      "activations/layer8_attention_weight_min": -48.43720626831055,
      "activations/layer9_attention_weight_max": 32.13407897949219,
      "activations/layer9_attention_weight_min": -35.37696075439453,
      "epoch": 20.41,
      "learning_rate": 1.855113636363636e-05,
      "loss": 2.7032,
      "step": 351300
    },
    {
      "activations/layer0_attention_weight_max": 15.875920295715332,
      "activations/layer0_attention_weight_min": -12.961896896362305,
      "activations/layer10_attention_weight_max": 37.670684814453125,
      "activations/layer10_attention_weight_min": -38.1543083190918,
      "activations/layer11_attention_weight_max": 36.477256774902344,
      "activations/layer11_attention_weight_min": -38.08714294433594,
      "activations/layer12_attention_weight_max": 30.223758697509766,
      "activations/layer12_attention_weight_min": -28.40921401977539,
      "activations/layer13_attention_weight_max": 46.3570442199707,
      "activations/layer13_attention_weight_min": -39.26617431640625,
      "activations/layer14_attention_weight_max": 46.29202651977539,
      "activations/layer14_attention_weight_min": -41.28942108154297,
      "activations/layer15_attention_weight_max": 47.26016616821289,
      "activations/layer15_attention_weight_min": -39.0879020690918,
      "activations/layer16_attention_weight_max": 35.08994674682617,
      "activations/layer16_attention_weight_min": -29.787763595581055,
      "activations/layer17_attention_weight_max": 32.50543212890625,
      "activations/layer17_attention_weight_min": -26.170001983642578,
      "activations/layer18_attention_weight_max": 34.63630294799805,
      "activations/layer18_attention_weight_min": -26.72174835205078,
      "activations/layer19_attention_weight_max": 35.97629165649414,
      "activations/layer19_attention_weight_min": -34.10281753540039,
      "activations/layer1_attention_weight_max": 16.1217098236084,
      "activations/layer1_attention_weight_min": -14.096857070922852,
      "activations/layer20_attention_weight_max": 32.17664337158203,
      "activations/layer20_attention_weight_min": -26.368745803833008,
      "activations/layer21_attention_weight_max": 32.87621307373047,
      "activations/layer21_attention_weight_min": -25.591129302978516,
      "activations/layer22_attention_weight_max": 43.321693420410156,
      "activations/layer22_attention_weight_min": -33.11101150512695,
      "activations/layer23_attention_weight_max": 31.66206932067871,
      "activations/layer23_attention_weight_min": -27.18505859375,
      "activations/layer2_attention_weight_max": 31.76160430908203,
      "activations/layer2_attention_weight_min": -31.557174682617188,
      "activations/layer3_attention_weight_max": 103.84687805175781,
      "activations/layer3_attention_weight_min": -105.08419036865234,
      "activations/layer4_attention_weight_max": 107.03697967529297,
      "activations/layer4_attention_weight_min": -111.9180908203125,
      "activations/layer5_attention_weight_max": 50.68099594116211,
      "activations/layer5_attention_weight_min": -60.8912353515625,
      "activations/layer6_attention_weight_max": 47.216487884521484,
      "activations/layer6_attention_weight_min": -47.428016662597656,
      "activations/layer7_attention_weight_max": 98.87715911865234,
      "activations/layer7_attention_weight_min": -99.02594757080078,
      "activations/layer8_attention_weight_max": 45.518653869628906,
      "activations/layer8_attention_weight_min": -47.109230041503906,
      "activations/layer9_attention_weight_max": 36.36628723144531,
      "activations/layer9_attention_weight_min": -37.320186614990234,
      "epoch": 20.42,
      "learning_rate": 1.8532196969696966e-05,
      "loss": 2.7109,
      "step": 351350
    },
    {
      "activations/layer0_attention_weight_max": 16.43233299255371,
      "activations/layer0_attention_weight_min": -14.080977439880371,
      "activations/layer10_attention_weight_max": 37.200313568115234,
      "activations/layer10_attention_weight_min": -36.77880859375,
      "activations/layer11_attention_weight_max": 36.378746032714844,
      "activations/layer11_attention_weight_min": -35.506126403808594,
      "activations/layer12_attention_weight_max": 28.146522521972656,
      "activations/layer12_attention_weight_min": -29.972898483276367,
      "activations/layer13_attention_weight_max": 41.37977981567383,
      "activations/layer13_attention_weight_min": -37.6735725402832,
      "activations/layer14_attention_weight_max": 43.316673278808594,
      "activations/layer14_attention_weight_min": -39.73982620239258,
      "activations/layer15_attention_weight_max": 41.04961395263672,
      "activations/layer15_attention_weight_min": -37.8660774230957,
      "activations/layer16_attention_weight_max": 30.457616806030273,
      "activations/layer16_attention_weight_min": -30.589599609375,
      "activations/layer17_attention_weight_max": 31.551753997802734,
      "activations/layer17_attention_weight_min": -28.375207901000977,
      "activations/layer18_attention_weight_max": 32.268028259277344,
      "activations/layer18_attention_weight_min": -26.560102462768555,
      "activations/layer19_attention_weight_max": 35.286903381347656,
      "activations/layer19_attention_weight_min": -30.81589126586914,
      "activations/layer1_attention_weight_max": 16.336002349853516,
      "activations/layer1_attention_weight_min": -14.35367202758789,
      "activations/layer20_attention_weight_max": 29.71478843688965,
      "activations/layer20_attention_weight_min": -27.223604202270508,
      "activations/layer21_attention_weight_max": 31.886425018310547,
      "activations/layer21_attention_weight_min": -25.466115951538086,
      "activations/layer22_attention_weight_max": 45.12625503540039,
      "activations/layer22_attention_weight_min": -32.07354736328125,
      "activations/layer23_attention_weight_max": 32.212608337402344,
      "activations/layer23_attention_weight_min": -28.252944946289062,
      "activations/layer2_attention_weight_max": 32.74489212036133,
      "activations/layer2_attention_weight_min": -31.738677978515625,
      "activations/layer3_attention_weight_max": 100.59999084472656,
      "activations/layer3_attention_weight_min": -103.20459747314453,
      "activations/layer4_attention_weight_max": 107.1236801147461,
      "activations/layer4_attention_weight_min": -113.57855224609375,
      "activations/layer5_attention_weight_max": 51.670074462890625,
      "activations/layer5_attention_weight_min": -64.69708251953125,
      "activations/layer6_attention_weight_max": 45.15580368041992,
      "activations/layer6_attention_weight_min": -49.782920837402344,
      "activations/layer7_attention_weight_max": 100.09230041503906,
      "activations/layer7_attention_weight_min": -102.7593002319336,
      "activations/layer8_attention_weight_max": 43.55116271972656,
      "activations/layer8_attention_weight_min": -46.00914001464844,
      "activations/layer9_attention_weight_max": 34.8197135925293,
      "activations/layer9_attention_weight_min": -34.69880294799805,
      "epoch": 20.42,
      "learning_rate": 1.851363636363636e-05,
      "loss": 2.7078,
      "step": 351400
    },
    {
      "activations/layer0_attention_weight_max": 16.007251739501953,
      "activations/layer0_attention_weight_min": -13.455826759338379,
      "activations/layer10_attention_weight_max": 32.675071716308594,
      "activations/layer10_attention_weight_min": -33.08660125732422,
      "activations/layer11_attention_weight_max": 34.39889907836914,
      "activations/layer11_attention_weight_min": -34.54404067993164,
      "activations/layer12_attention_weight_max": 27.4239559173584,
      "activations/layer12_attention_weight_min": -26.663074493408203,
      "activations/layer13_attention_weight_max": 39.24874496459961,
      "activations/layer13_attention_weight_min": -36.753395080566406,
      "activations/layer14_attention_weight_max": 43.3665771484375,
      "activations/layer14_attention_weight_min": -37.194725036621094,
      "activations/layer15_attention_weight_max": 42.255340576171875,
      "activations/layer15_attention_weight_min": -38.88105392456055,
      "activations/layer16_attention_weight_max": 32.25822067260742,
      "activations/layer16_attention_weight_min": -29.26363182067871,
      "activations/layer17_attention_weight_max": 30.434967041015625,
      "activations/layer17_attention_weight_min": -27.529525756835938,
      "activations/layer18_attention_weight_max": 31.72930335998535,
      "activations/layer18_attention_weight_min": -26.388185501098633,
      "activations/layer19_attention_weight_max": 36.18445587158203,
      "activations/layer19_attention_weight_min": -32.39364242553711,
      "activations/layer1_attention_weight_max": 16.140228271484375,
      "activations/layer1_attention_weight_min": -15.3699951171875,
      "activations/layer20_attention_weight_max": 29.897422790527344,
      "activations/layer20_attention_weight_min": -27.822065353393555,
      "activations/layer21_attention_weight_max": 28.902883529663086,
      "activations/layer21_attention_weight_min": -26.50215721130371,
      "activations/layer22_attention_weight_max": 40.0267448425293,
      "activations/layer22_attention_weight_min": -30.916658401489258,
      "activations/layer23_attention_weight_max": 33.32368469238281,
      "activations/layer23_attention_weight_min": -27.209129333496094,
      "activations/layer2_attention_weight_max": 33.440792083740234,
      "activations/layer2_attention_weight_min": -32.74079513549805,
      "activations/layer3_attention_weight_max": 108.2469253540039,
      "activations/layer3_attention_weight_min": -103.40545654296875,
      "activations/layer4_attention_weight_max": 109.8995361328125,
      "activations/layer4_attention_weight_min": -109.21012115478516,
      "activations/layer5_attention_weight_max": 52.367767333984375,
      "activations/layer5_attention_weight_min": -61.99188232421875,
      "activations/layer6_attention_weight_max": 45.33030700683594,
      "activations/layer6_attention_weight_min": -46.90605926513672,
      "activations/layer7_attention_weight_max": 93.50894165039062,
      "activations/layer7_attention_weight_min": -98.19144439697266,
      "activations/layer8_attention_weight_max": 42.15635299682617,
      "activations/layer8_attention_weight_min": -43.21217727661133,
      "activations/layer9_attention_weight_max": 31.865697860717773,
      "activations/layer9_attention_weight_min": -33.21747970581055,
      "epoch": 20.42,
      "learning_rate": 1.849469696969697e-05,
      "loss": 2.7122,
      "step": 351450
    },
    {
      "activations/layer0_attention_weight_max": 16.600486755371094,
      "activations/layer0_attention_weight_min": -12.77490520477295,
      "activations/layer10_attention_weight_max": 41.954288482666016,
      "activations/layer10_attention_weight_min": -41.38711166381836,
      "activations/layer11_attention_weight_max": 42.29936218261719,
      "activations/layer11_attention_weight_min": -39.355648040771484,
      "activations/layer12_attention_weight_max": 28.55164909362793,
      "activations/layer12_attention_weight_min": -27.11475372314453,
      "activations/layer13_attention_weight_max": 41.60469055175781,
      "activations/layer13_attention_weight_min": -36.70283126831055,
      "activations/layer14_attention_weight_max": 46.53451919555664,
      "activations/layer14_attention_weight_min": -41.01408004760742,
      "activations/layer15_attention_weight_max": 42.04025650024414,
      "activations/layer15_attention_weight_min": -39.05719757080078,
      "activations/layer16_attention_weight_max": 29.64983367919922,
      "activations/layer16_attention_weight_min": -29.97021484375,
      "activations/layer17_attention_weight_max": 30.016693115234375,
      "activations/layer17_attention_weight_min": -26.310516357421875,
      "activations/layer18_attention_weight_max": 30.92855453491211,
      "activations/layer18_attention_weight_min": -24.959611892700195,
      "activations/layer19_attention_weight_max": 34.35319900512695,
      "activations/layer19_attention_weight_min": -31.091718673706055,
      "activations/layer1_attention_weight_max": 16.06602668762207,
      "activations/layer1_attention_weight_min": -15.104262351989746,
      "activations/layer20_attention_weight_max": 30.320587158203125,
      "activations/layer20_attention_weight_min": -25.916730880737305,
      "activations/layer21_attention_weight_max": 30.85832977294922,
      "activations/layer21_attention_weight_min": -24.841630935668945,
      "activations/layer22_attention_weight_max": 42.90946578979492,
      "activations/layer22_attention_weight_min": -33.15095901489258,
      "activations/layer23_attention_weight_max": 31.853775024414062,
      "activations/layer23_attention_weight_min": -26.253910064697266,
      "activations/layer2_attention_weight_max": 33.51321792602539,
      "activations/layer2_attention_weight_min": -33.7708740234375,
      "activations/layer3_attention_weight_max": 108.7226791381836,
      "activations/layer3_attention_weight_min": -113.99183654785156,
      "activations/layer4_attention_weight_max": 110.44185638427734,
      "activations/layer4_attention_weight_min": -113.57330322265625,
      "activations/layer5_attention_weight_max": 53.07784652709961,
      "activations/layer5_attention_weight_min": -64.1240005493164,
      "activations/layer6_attention_weight_max": 50.45066452026367,
      "activations/layer6_attention_weight_min": -49.00763702392578,
      "activations/layer7_attention_weight_max": 100.7455825805664,
      "activations/layer7_attention_weight_min": -103.80159759521484,
      "activations/layer8_attention_weight_max": 45.308837890625,
      "activations/layer8_attention_weight_min": -47.63186264038086,
      "activations/layer9_attention_weight_max": 36.44148635864258,
      "activations/layer9_attention_weight_min": -40.24152374267578,
      "epoch": 20.42,
      "learning_rate": 1.8475757575757574e-05,
      "loss": 2.7206,
      "step": 351500
    },
    {
      "activations/layer0_attention_weight_max": 16.8255672454834,
      "activations/layer0_attention_weight_min": -13.688141822814941,
      "activations/layer10_attention_weight_max": 37.79993438720703,
      "activations/layer10_attention_weight_min": -37.01704025268555,
      "activations/layer11_attention_weight_max": 37.832000732421875,
      "activations/layer11_attention_weight_min": -38.43869400024414,
      "activations/layer12_attention_weight_max": 28.015037536621094,
      "activations/layer12_attention_weight_min": -26.16769027709961,
      "activations/layer13_attention_weight_max": 40.99163055419922,
      "activations/layer13_attention_weight_min": -37.34807205200195,
      "activations/layer14_attention_weight_max": 42.45416259765625,
      "activations/layer14_attention_weight_min": -38.32999801635742,
      "activations/layer15_attention_weight_max": 39.03317642211914,
      "activations/layer15_attention_weight_min": -37.74854278564453,
      "activations/layer16_attention_weight_max": 28.958890914916992,
      "activations/layer16_attention_weight_min": -29.910747528076172,
      "activations/layer17_attention_weight_max": 29.1563663482666,
      "activations/layer17_attention_weight_min": -26.204166412353516,
      "activations/layer18_attention_weight_max": 32.14574432373047,
      "activations/layer18_attention_weight_min": -27.46868896484375,
      "activations/layer19_attention_weight_max": 32.47404098510742,
      "activations/layer19_attention_weight_min": -33.104705810546875,
      "activations/layer1_attention_weight_max": 16.245874404907227,
      "activations/layer1_attention_weight_min": -14.753279685974121,
      "activations/layer20_attention_weight_max": 28.486671447753906,
      "activations/layer20_attention_weight_min": -25.8779354095459,
      "activations/layer21_attention_weight_max": 27.97359848022461,
      "activations/layer21_attention_weight_min": -25.394216537475586,
      "activations/layer22_attention_weight_max": 43.30884552001953,
      "activations/layer22_attention_weight_min": -32.89866638183594,
      "activations/layer23_attention_weight_max": 30.66651725769043,
      "activations/layer23_attention_weight_min": -28.14405632019043,
      "activations/layer2_attention_weight_max": 36.294952392578125,
      "activations/layer2_attention_weight_min": -33.01020812988281,
      "activations/layer3_attention_weight_max": 103.75074005126953,
      "activations/layer3_attention_weight_min": -105.40110778808594,
      "activations/layer4_attention_weight_max": 108.2986831665039,
      "activations/layer4_attention_weight_min": -112.32645416259766,
      "activations/layer5_attention_weight_max": 51.09994125366211,
      "activations/layer5_attention_weight_min": -62.95404815673828,
      "activations/layer6_attention_weight_max": 46.77674102783203,
      "activations/layer6_attention_weight_min": -48.47371292114258,
      "activations/layer7_attention_weight_max": 98.27246856689453,
      "activations/layer7_attention_weight_min": -104.83273315429688,
      "activations/layer8_attention_weight_max": 45.765628814697266,
      "activations/layer8_attention_weight_min": -47.667510986328125,
      "activations/layer9_attention_weight_max": 35.42657470703125,
      "activations/layer9_attention_weight_min": -35.787105560302734,
      "epoch": 20.43,
      "learning_rate": 1.845681818181818e-05,
      "loss": 2.7168,
      "step": 351550
    },
    {
      "activations/layer0_attention_weight_max": 16.124011993408203,
      "activations/layer0_attention_weight_min": -15.201617240905762,
      "activations/layer10_attention_weight_max": 34.14460372924805,
      "activations/layer10_attention_weight_min": -33.35127639770508,
      "activations/layer11_attention_weight_max": 33.817466735839844,
      "activations/layer11_attention_weight_min": -34.74604034423828,
      "activations/layer12_attention_weight_max": 27.538265228271484,
      "activations/layer12_attention_weight_min": -25.99842643737793,
      "activations/layer13_attention_weight_max": 37.91848373413086,
      "activations/layer13_attention_weight_min": -35.31515121459961,
      "activations/layer14_attention_weight_max": 40.5454216003418,
      "activations/layer14_attention_weight_min": -35.137691497802734,
      "activations/layer15_attention_weight_max": 38.385433197021484,
      "activations/layer15_attention_weight_min": -34.729984283447266,
      "activations/layer16_attention_weight_max": 28.151988983154297,
      "activations/layer16_attention_weight_min": -28.55118179321289,
      "activations/layer17_attention_weight_max": 28.41118812561035,
      "activations/layer17_attention_weight_min": -26.58909034729004,
      "activations/layer18_attention_weight_max": 32.248897552490234,
      "activations/layer18_attention_weight_min": -28.180124282836914,
      "activations/layer19_attention_weight_max": 33.70062255859375,
      "activations/layer19_attention_weight_min": -31.974884033203125,
      "activations/layer1_attention_weight_max": 15.692999839782715,
      "activations/layer1_attention_weight_min": -15.460220336914062,
      "activations/layer20_attention_weight_max": 30.39479637145996,
      "activations/layer20_attention_weight_min": -24.84174156188965,
      "activations/layer21_attention_weight_max": 29.492137908935547,
      "activations/layer21_attention_weight_min": -25.397409439086914,
      "activations/layer22_attention_weight_max": 41.027320861816406,
      "activations/layer22_attention_weight_min": -31.447540283203125,
      "activations/layer23_attention_weight_max": 31.42966079711914,
      "activations/layer23_attention_weight_min": -27.425357818603516,
      "activations/layer2_attention_weight_max": 32.252708435058594,
      "activations/layer2_attention_weight_min": -31.426013946533203,
      "activations/layer3_attention_weight_max": 99.58170318603516,
      "activations/layer3_attention_weight_min": -100.17957305908203,
      "activations/layer4_attention_weight_max": 104.7422866821289,
      "activations/layer4_attention_weight_min": -106.57225799560547,
      "activations/layer5_attention_weight_max": 52.519527435302734,
      "activations/layer5_attention_weight_min": -58.88263702392578,
      "activations/layer6_attention_weight_max": 43.226802825927734,
      "activations/layer6_attention_weight_min": -46.07826232910156,
      "activations/layer7_attention_weight_max": 100.244873046875,
      "activations/layer7_attention_weight_min": -91.28717041015625,
      "activations/layer8_attention_weight_max": 40.67912673950195,
      "activations/layer8_attention_weight_min": -43.68132400512695,
      "activations/layer9_attention_weight_max": 31.465917587280273,
      "activations/layer9_attention_weight_min": -33.2491340637207,
      "epoch": 20.43,
      "learning_rate": 1.8438257575757573e-05,
      "loss": 2.7192,
      "step": 351600
    },
    {
      "activations/layer0_attention_weight_max": 15.6541748046875,
      "activations/layer0_attention_weight_min": -14.826807022094727,
      "activations/layer10_attention_weight_max": 39.00456619262695,
      "activations/layer10_attention_weight_min": -36.059654235839844,
      "activations/layer11_attention_weight_max": 37.885704040527344,
      "activations/layer11_attention_weight_min": -37.757476806640625,
      "activations/layer12_attention_weight_max": 27.43886947631836,
      "activations/layer12_attention_weight_min": -28.464860916137695,
      "activations/layer13_attention_weight_max": 40.30723571777344,
      "activations/layer13_attention_weight_min": -38.924312591552734,
      "activations/layer14_attention_weight_max": 50.53730010986328,
      "activations/layer14_attention_weight_min": -41.96185302734375,
      "activations/layer15_attention_weight_max": 43.271522521972656,
      "activations/layer15_attention_weight_min": -43.0024528503418,
      "activations/layer16_attention_weight_max": 32.18144226074219,
      "activations/layer16_attention_weight_min": -29.5703182220459,
      "activations/layer17_attention_weight_max": 31.97616958618164,
      "activations/layer17_attention_weight_min": -26.496580123901367,
      "activations/layer18_attention_weight_max": 31.42681884765625,
      "activations/layer18_attention_weight_min": -25.98601722717285,
      "activations/layer19_attention_weight_max": 44.335323333740234,
      "activations/layer19_attention_weight_min": -33.08671951293945,
      "activations/layer1_attention_weight_max": 16.017499923706055,
      "activations/layer1_attention_weight_min": -15.639966011047363,
      "activations/layer20_attention_weight_max": 30.339805603027344,
      "activations/layer20_attention_weight_min": -24.690507888793945,
      "activations/layer21_attention_weight_max": 28.007291793823242,
      "activations/layer21_attention_weight_min": -24.428752899169922,
      "activations/layer22_attention_weight_max": 45.8768196105957,
      "activations/layer22_attention_weight_min": -31.073362350463867,
      "activations/layer23_attention_weight_max": 34.30757141113281,
      "activations/layer23_attention_weight_min": -25.669933319091797,
      "activations/layer2_attention_weight_max": 34.582725524902344,
      "activations/layer2_attention_weight_min": -33.68567657470703,
      "activations/layer3_attention_weight_max": 101.38675689697266,
      "activations/layer3_attention_weight_min": -105.41085815429688,
      "activations/layer4_attention_weight_max": 105.29176330566406,
      "activations/layer4_attention_weight_min": -111.90047454833984,
      "activations/layer5_attention_weight_max": 54.00121307373047,
      "activations/layer5_attention_weight_min": -70.20378112792969,
      "activations/layer6_attention_weight_max": 43.5276985168457,
      "activations/layer6_attention_weight_min": -47.81062698364258,
      "activations/layer7_attention_weight_max": 98.54479217529297,
      "activations/layer7_attention_weight_min": -96.44889068603516,
      "activations/layer8_attention_weight_max": 45.8068962097168,
      "activations/layer8_attention_weight_min": -44.65151596069336,
      "activations/layer9_attention_weight_max": 36.55671691894531,
      "activations/layer9_attention_weight_min": -36.92667770385742,
      "epoch": 20.43,
      "learning_rate": 1.841931818181818e-05,
      "loss": 2.7024,
      "step": 351650
    },
    {
      "activations/layer0_attention_weight_max": 15.966094017028809,
      "activations/layer0_attention_weight_min": -14.36467170715332,
      "activations/layer10_attention_weight_max": 43.43929672241211,
      "activations/layer10_attention_weight_min": -39.4913330078125,
      "activations/layer11_attention_weight_max": 44.318965911865234,
      "activations/layer11_attention_weight_min": -41.57210922241211,
      "activations/layer12_attention_weight_max": 29.803817749023438,
      "activations/layer12_attention_weight_min": -27.54897689819336,
      "activations/layer13_attention_weight_max": 42.365596771240234,
      "activations/layer13_attention_weight_min": -36.80836868286133,
      "activations/layer14_attention_weight_max": 54.12198257446289,
      "activations/layer14_attention_weight_min": -45.87998962402344,
      "activations/layer15_attention_weight_max": 47.31882858276367,
      "activations/layer15_attention_weight_min": -43.28502655029297,
      "activations/layer16_attention_weight_max": 34.84492492675781,
      "activations/layer16_attention_weight_min": -31.36121368408203,
      "activations/layer17_attention_weight_max": 29.172468185424805,
      "activations/layer17_attention_weight_min": -26.447969436645508,
      "activations/layer18_attention_weight_max": 34.00155258178711,
      "activations/layer18_attention_weight_min": -27.542217254638672,
      "activations/layer19_attention_weight_max": 40.93876266479492,
      "activations/layer19_attention_weight_min": -34.132938385009766,
      "activations/layer1_attention_weight_max": 17.2540225982666,
      "activations/layer1_attention_weight_min": -15.689173698425293,
      "activations/layer20_attention_weight_max": 32.72483444213867,
      "activations/layer20_attention_weight_min": -25.90303611755371,
      "activations/layer21_attention_weight_max": 29.99315643310547,
      "activations/layer21_attention_weight_min": -26.23273277282715,
      "activations/layer22_attention_weight_max": 50.84556579589844,
      "activations/layer22_attention_weight_min": -36.4805793762207,
      "activations/layer23_attention_weight_max": 35.24044418334961,
      "activations/layer23_attention_weight_min": -27.199371337890625,
      "activations/layer2_attention_weight_max": 33.2373161315918,
      "activations/layer2_attention_weight_min": -32.77553939819336,
      "activations/layer3_attention_weight_max": 103.67537689208984,
      "activations/layer3_attention_weight_min": -106.69169616699219,
      "activations/layer4_attention_weight_max": 108.25917053222656,
      "activations/layer4_attention_weight_min": -109.9361343383789,
      "activations/layer5_attention_weight_max": 49.36025619506836,
      "activations/layer5_attention_weight_min": -61.83238220214844,
      "activations/layer6_attention_weight_max": 46.23200988769531,
      "activations/layer6_attention_weight_min": -49.416019439697266,
      "activations/layer7_attention_weight_max": 103.7322769165039,
      "activations/layer7_attention_weight_min": -105.00289154052734,
      "activations/layer8_attention_weight_max": 47.44513702392578,
      "activations/layer8_attention_weight_min": -49.29373550415039,
      "activations/layer9_attention_weight_max": 36.55552291870117,
      "activations/layer9_attention_weight_min": -37.09391403198242,
      "epoch": 20.44,
      "learning_rate": 1.8400378787878786e-05,
      "loss": 2.7253,
      "step": 351700
    },
    {
      "activations/layer0_attention_weight_max": 15.603923797607422,
      "activations/layer0_attention_weight_min": -14.009602546691895,
      "activations/layer10_attention_weight_max": 36.7347412109375,
      "activations/layer10_attention_weight_min": -36.81251907348633,
      "activations/layer11_attention_weight_max": 35.80327606201172,
      "activations/layer11_attention_weight_min": -37.92757797241211,
      "activations/layer12_attention_weight_max": 26.2094783782959,
      "activations/layer12_attention_weight_min": -27.11629867553711,
      "activations/layer13_attention_weight_max": 37.43209457397461,
      "activations/layer13_attention_weight_min": -34.900997161865234,
      "activations/layer14_attention_weight_max": 40.44163131713867,
      "activations/layer14_attention_weight_min": -36.03038787841797,
      "activations/layer15_attention_weight_max": 35.699703216552734,
      "activations/layer15_attention_weight_min": -33.855712890625,
      "activations/layer16_attention_weight_max": 27.96462059020996,
      "activations/layer16_attention_weight_min": -26.538394927978516,
      "activations/layer17_attention_weight_max": 29.64295768737793,
      "activations/layer17_attention_weight_min": -25.336626052856445,
      "activations/layer18_attention_weight_max": 30.964252471923828,
      "activations/layer18_attention_weight_min": -25.09243392944336,
      "activations/layer19_attention_weight_max": 33.455596923828125,
      "activations/layer19_attention_weight_min": -30.540515899658203,
      "activations/layer1_attention_weight_max": 15.261893272399902,
      "activations/layer1_attention_weight_min": -15.087631225585938,
      "activations/layer20_attention_weight_max": 29.106122970581055,
      "activations/layer20_attention_weight_min": -24.37078857421875,
      "activations/layer21_attention_weight_max": 28.314165115356445,
      "activations/layer21_attention_weight_min": -24.558849334716797,
      "activations/layer22_attention_weight_max": 42.93339920043945,
      "activations/layer22_attention_weight_min": -32.036415100097656,
      "activations/layer23_attention_weight_max": 31.27254867553711,
      "activations/layer23_attention_weight_min": -24.682043075561523,
      "activations/layer2_attention_weight_max": 33.143409729003906,
      "activations/layer2_attention_weight_min": -32.16407775878906,
      "activations/layer3_attention_weight_max": 102.78411865234375,
      "activations/layer3_attention_weight_min": -103.6325454711914,
      "activations/layer4_attention_weight_max": 109.7266616821289,
      "activations/layer4_attention_weight_min": -110.45806884765625,
      "activations/layer5_attention_weight_max": 52.825523376464844,
      "activations/layer5_attention_weight_min": -59.918724060058594,
      "activations/layer6_attention_weight_max": 46.66593551635742,
      "activations/layer6_attention_weight_min": -49.020164489746094,
      "activations/layer7_attention_weight_max": 98.36811828613281,
      "activations/layer7_attention_weight_min": -101.99275207519531,
      "activations/layer8_attention_weight_max": 45.68534469604492,
      "activations/layer8_attention_weight_min": -45.94773864746094,
      "activations/layer9_attention_weight_max": 34.52731704711914,
      "activations/layer9_attention_weight_min": -37.018821716308594,
      "epoch": 20.44,
      "learning_rate": 1.838143939393939e-05,
      "loss": 2.7097,
      "step": 351750
    },
    {
      "activations/layer0_attention_weight_max": 16.537546157836914,
      "activations/layer0_attention_weight_min": -15.442021369934082,
      "activations/layer10_attention_weight_max": 32.923763275146484,
      "activations/layer10_attention_weight_min": -34.12793731689453,
      "activations/layer11_attention_weight_max": 33.99561309814453,
      "activations/layer11_attention_weight_min": -33.66328430175781,
      "activations/layer12_attention_weight_max": 27.309537887573242,
      "activations/layer12_attention_weight_min": -26.470882415771484,
      "activations/layer13_attention_weight_max": 38.93486785888672,
      "activations/layer13_attention_weight_min": -35.36870193481445,
      "activations/layer14_attention_weight_max": 42.03173065185547,
      "activations/layer14_attention_weight_min": -36.30260467529297,
      "activations/layer15_attention_weight_max": 38.22909164428711,
      "activations/layer15_attention_weight_min": -35.584232330322266,
      "activations/layer16_attention_weight_max": 29.927865982055664,
      "activations/layer16_attention_weight_min": -28.542402267456055,
      "activations/layer17_attention_weight_max": 32.7315788269043,
      "activations/layer17_attention_weight_min": -27.81278419494629,
      "activations/layer18_attention_weight_max": 36.12031936645508,
      "activations/layer18_attention_weight_min": -27.515544891357422,
      "activations/layer19_attention_weight_max": 39.555870056152344,
      "activations/layer19_attention_weight_min": -34.4410400390625,
      "activations/layer1_attention_weight_max": 15.912764549255371,
      "activations/layer1_attention_weight_min": -14.948920249938965,
      "activations/layer20_attention_weight_max": 34.023067474365234,
      "activations/layer20_attention_weight_min": -28.4409122467041,
      "activations/layer21_attention_weight_max": 32.55607604980469,
      "activations/layer21_attention_weight_min": -28.38006591796875,
      "activations/layer22_attention_weight_max": 49.92866897583008,
      "activations/layer22_attention_weight_min": -34.82972717285156,
      "activations/layer23_attention_weight_max": 38.09819030761719,
      "activations/layer23_attention_weight_min": -27.083999633789062,
      "activations/layer2_attention_weight_max": 33.58380889892578,
      "activations/layer2_attention_weight_min": -33.46269607543945,
      "activations/layer3_attention_weight_max": 103.32393646240234,
      "activations/layer3_attention_weight_min": -104.58502960205078,
      "activations/layer4_attention_weight_max": 106.96429443359375,
      "activations/layer4_attention_weight_min": -108.75939178466797,
      "activations/layer5_attention_weight_max": 48.32261657714844,
      "activations/layer5_attention_weight_min": -61.338050842285156,
      "activations/layer6_attention_weight_max": 42.98635482788086,
      "activations/layer6_attention_weight_min": -46.97703552246094,
      "activations/layer7_attention_weight_max": 91.90418243408203,
      "activations/layer7_attention_weight_min": -96.9128646850586,
      "activations/layer8_attention_weight_max": 43.006927490234375,
      "activations/layer8_attention_weight_min": -45.621089935302734,
      "activations/layer9_attention_weight_max": 31.661508560180664,
      "activations/layer9_attention_weight_min": -34.27374267578125,
      "epoch": 20.44,
      "learning_rate": 1.8362499999999996e-05,
      "loss": 2.7149,
      "step": 351800
    },
    {
      "activations/layer0_attention_weight_max": 16.442672729492188,
      "activations/layer0_attention_weight_min": -13.120349884033203,
      "activations/layer10_attention_weight_max": 37.59755325317383,
      "activations/layer10_attention_weight_min": -37.16987228393555,
      "activations/layer11_attention_weight_max": 36.07173156738281,
      "activations/layer11_attention_weight_min": -32.65813064575195,
      "activations/layer12_attention_weight_max": 26.020572662353516,
      "activations/layer12_attention_weight_min": -29.85523223876953,
      "activations/layer13_attention_weight_max": 39.12792205810547,
      "activations/layer13_attention_weight_min": -35.175865173339844,
      "activations/layer14_attention_weight_max": 39.73627853393555,
      "activations/layer14_attention_weight_min": -36.941165924072266,
      "activations/layer15_attention_weight_max": 35.6226806640625,
      "activations/layer15_attention_weight_min": -38.682437896728516,
      "activations/layer16_attention_weight_max": 30.883508682250977,
      "activations/layer16_attention_weight_min": -26.569595336914062,
      "activations/layer17_attention_weight_max": 29.431745529174805,
      "activations/layer17_attention_weight_min": -25.77486801147461,
      "activations/layer18_attention_weight_max": 33.56779479980469,
      "activations/layer18_attention_weight_min": -26.790390014648438,
      "activations/layer19_attention_weight_max": 34.46717071533203,
      "activations/layer19_attention_weight_min": -32.850765228271484,
      "activations/layer1_attention_weight_max": 15.359612464904785,
      "activations/layer1_attention_weight_min": -15.383724212646484,
      "activations/layer20_attention_weight_max": 32.19351577758789,
      "activations/layer20_attention_weight_min": -25.610301971435547,
      "activations/layer21_attention_weight_max": 29.919322967529297,
      "activations/layer21_attention_weight_min": -24.93523406982422,
      "activations/layer22_attention_weight_max": 42.33465576171875,
      "activations/layer22_attention_weight_min": -32.62268829345703,
      "activations/layer23_attention_weight_max": 32.95541000366211,
      "activations/layer23_attention_weight_min": -24.718746185302734,
      "activations/layer2_attention_weight_max": 35.628135681152344,
      "activations/layer2_attention_weight_min": -32.56932067871094,
      "activations/layer3_attention_weight_max": 100.91321563720703,
      "activations/layer3_attention_weight_min": -104.24440002441406,
      "activations/layer4_attention_weight_max": 110.81124114990234,
      "activations/layer4_attention_weight_min": -112.7562484741211,
      "activations/layer5_attention_weight_max": 51.197021484375,
      "activations/layer5_attention_weight_min": -59.39858627319336,
      "activations/layer6_attention_weight_max": 44.01265335083008,
      "activations/layer6_attention_weight_min": -46.0025749206543,
      "activations/layer7_attention_weight_max": 95.28426361083984,
      "activations/layer7_attention_weight_min": -96.3877944946289,
      "activations/layer8_attention_weight_max": 45.50257110595703,
      "activations/layer8_attention_weight_min": -45.9786262512207,
      "activations/layer9_attention_weight_max": 33.6538200378418,
      "activations/layer9_attention_weight_min": -37.65190505981445,
      "epoch": 20.44,
      "learning_rate": 1.8343560606060604e-05,
      "loss": 2.7084,
      "step": 351850
    },
    {
      "activations/layer0_attention_weight_max": 16.152374267578125,
      "activations/layer0_attention_weight_min": -13.792885780334473,
      "activations/layer10_attention_weight_max": 35.552757263183594,
      "activations/layer10_attention_weight_min": -36.836341857910156,
      "activations/layer11_attention_weight_max": 36.160980224609375,
      "activations/layer11_attention_weight_min": -36.22686004638672,
      "activations/layer12_attention_weight_max": 28.3848934173584,
      "activations/layer12_attention_weight_min": -28.000608444213867,
      "activations/layer13_attention_weight_max": 42.29669952392578,
      "activations/layer13_attention_weight_min": -36.213199615478516,
      "activations/layer14_attention_weight_max": 48.68791198730469,
      "activations/layer14_attention_weight_min": -38.048038482666016,
      "activations/layer15_attention_weight_max": 43.566001892089844,
      "activations/layer15_attention_weight_min": -37.25807571411133,
      "activations/layer16_attention_weight_max": 33.623291015625,
      "activations/layer16_attention_weight_min": -28.608200073242188,
      "activations/layer17_attention_weight_max": 31.6207218170166,
      "activations/layer17_attention_weight_min": -26.80753517150879,
      "activations/layer18_attention_weight_max": 36.86688232421875,
      "activations/layer18_attention_weight_min": -26.467632293701172,
      "activations/layer19_attention_weight_max": 43.410091400146484,
      "activations/layer19_attention_weight_min": -31.280715942382812,
      "activations/layer1_attention_weight_max": 16.824588775634766,
      "activations/layer1_attention_weight_min": -15.078536987304688,
      "activations/layer20_attention_weight_max": 34.33122253417969,
      "activations/layer20_attention_weight_min": -25.661075592041016,
      "activations/layer21_attention_weight_max": 34.334266662597656,
      "activations/layer21_attention_weight_min": -24.01041603088379,
      "activations/layer22_attention_weight_max": 45.0699348449707,
      "activations/layer22_attention_weight_min": -31.648250579833984,
      "activations/layer23_attention_weight_max": 38.45579528808594,
      "activations/layer23_attention_weight_min": -25.41767120361328,
      "activations/layer2_attention_weight_max": 32.082176208496094,
      "activations/layer2_attention_weight_min": -31.128246307373047,
      "activations/layer3_attention_weight_max": 104.64493560791016,
      "activations/layer3_attention_weight_min": -105.0418701171875,
      "activations/layer4_attention_weight_max": 106.73397064208984,
      "activations/layer4_attention_weight_min": -108.60713195800781,
      "activations/layer5_attention_weight_max": 51.922760009765625,
      "activations/layer5_attention_weight_min": -59.34590148925781,
      "activations/layer6_attention_weight_max": 45.994022369384766,
      "activations/layer6_attention_weight_min": -47.647823333740234,
      "activations/layer7_attention_weight_max": 99.20575714111328,
      "activations/layer7_attention_weight_min": -100.79387664794922,
      "activations/layer8_attention_weight_max": 42.35251998901367,
      "activations/layer8_attention_weight_min": -47.302860260009766,
      "activations/layer9_attention_weight_max": 33.455177307128906,
      "activations/layer9_attention_weight_min": -36.3020133972168,
      "epoch": 20.45,
      "learning_rate": 1.8324621212121212e-05,
      "loss": 2.7112,
      "step": 351900
    },
    {
      "activations/layer0_attention_weight_max": 17.173954010009766,
      "activations/layer0_attention_weight_min": -13.94914436340332,
      "activations/layer10_attention_weight_max": 37.74671173095703,
      "activations/layer10_attention_weight_min": -37.640594482421875,
      "activations/layer11_attention_weight_max": 36.22924041748047,
      "activations/layer11_attention_weight_min": -36.82524490356445,
      "activations/layer12_attention_weight_max": 28.36754035949707,
      "activations/layer12_attention_weight_min": -27.911928176879883,
      "activations/layer13_attention_weight_max": 42.68550109863281,
      "activations/layer13_attention_weight_min": -38.05065155029297,
      "activations/layer14_attention_weight_max": 46.334163665771484,
      "activations/layer14_attention_weight_min": -39.49928665161133,
      "activations/layer15_attention_weight_max": 40.986900329589844,
      "activations/layer15_attention_weight_min": -39.49909210205078,
      "activations/layer16_attention_weight_max": 30.574169158935547,
      "activations/layer16_attention_weight_min": -29.83876609802246,
      "activations/layer17_attention_weight_max": 32.27846908569336,
      "activations/layer17_attention_weight_min": -27.679100036621094,
      "activations/layer18_attention_weight_max": 33.81897735595703,
      "activations/layer18_attention_weight_min": -26.75822639465332,
      "activations/layer19_attention_weight_max": 35.81575393676758,
      "activations/layer19_attention_weight_min": -32.80241012573242,
      "activations/layer1_attention_weight_max": 16.37163734436035,
      "activations/layer1_attention_weight_min": -14.417684555053711,
      "activations/layer20_attention_weight_max": 31.3962459564209,
      "activations/layer20_attention_weight_min": -26.64203453063965,
      "activations/layer21_attention_weight_max": 32.29443359375,
      "activations/layer21_attention_weight_min": -26.96499252319336,
      "activations/layer22_attention_weight_max": 40.51284408569336,
      "activations/layer22_attention_weight_min": -31.83186149597168,
      "activations/layer23_attention_weight_max": 32.870811462402344,
      "activations/layer23_attention_weight_min": -25.828720092773438,
      "activations/layer2_attention_weight_max": 32.91948318481445,
      "activations/layer2_attention_weight_min": -32.257850646972656,
      "activations/layer3_attention_weight_max": 102.88020324707031,
      "activations/layer3_attention_weight_min": -104.76502227783203,
      "activations/layer4_attention_weight_max": 108.46563720703125,
      "activations/layer4_attention_weight_min": -111.15653228759766,
      "activations/layer5_attention_weight_max": 51.413631439208984,
      "activations/layer5_attention_weight_min": -61.2047233581543,
      "activations/layer6_attention_weight_max": 45.747379302978516,
      "activations/layer6_attention_weight_min": -48.671852111816406,
      "activations/layer7_attention_weight_max": 97.14204406738281,
      "activations/layer7_attention_weight_min": -104.73435974121094,
      "activations/layer8_attention_weight_max": 43.89015579223633,
      "activations/layer8_attention_weight_min": -46.07924270629883,
      "activations/layer9_attention_weight_max": 34.62014389038086,
      "activations/layer9_attention_weight_min": -35.96140670776367,
      "epoch": 20.45,
      "learning_rate": 1.8305681818181817e-05,
      "loss": 2.7261,
      "step": 351950
    },
    {
      "activations/layer0_attention_weight_max": 16.31195831298828,
      "activations/layer0_attention_weight_min": -15.66991901397705,
      "activations/layer10_attention_weight_max": 38.47553253173828,
      "activations/layer10_attention_weight_min": -37.05976104736328,
      "activations/layer11_attention_weight_max": 38.12782287597656,
      "activations/layer11_attention_weight_min": -37.48408889770508,
      "activations/layer12_attention_weight_max": 26.76614761352539,
      "activations/layer12_attention_weight_min": -31.612693786621094,
      "activations/layer13_attention_weight_max": 39.34566879272461,
      "activations/layer13_attention_weight_min": -35.043670654296875,
      "activations/layer14_attention_weight_max": 42.98830795288086,
      "activations/layer14_attention_weight_min": -37.58076095581055,
      "activations/layer15_attention_weight_max": 37.775089263916016,
      "activations/layer15_attention_weight_min": -34.519554138183594,
      "activations/layer16_attention_weight_max": 31.327348709106445,
      "activations/layer16_attention_weight_min": -28.766305923461914,
      "activations/layer17_attention_weight_max": 29.413286209106445,
      "activations/layer17_attention_weight_min": -27.03043556213379,
      "activations/layer18_attention_weight_max": 33.33263397216797,
      "activations/layer18_attention_weight_min": -24.86459732055664,
      "activations/layer19_attention_weight_max": 31.987747192382812,
      "activations/layer19_attention_weight_min": -31.010831832885742,
      "activations/layer1_attention_weight_max": 16.148792266845703,
      "activations/layer1_attention_weight_min": -14.633492469787598,
      "activations/layer20_attention_weight_max": 29.00055503845215,
      "activations/layer20_attention_weight_min": -25.710973739624023,
      "activations/layer21_attention_weight_max": 26.75798225402832,
      "activations/layer21_attention_weight_min": -24.634050369262695,
      "activations/layer22_attention_weight_max": 42.37211227416992,
      "activations/layer22_attention_weight_min": -31.922435760498047,
      "activations/layer23_attention_weight_max": 31.962263107299805,
      "activations/layer23_attention_weight_min": -27.330814361572266,
      "activations/layer2_attention_weight_max": 34.37790298461914,
      "activations/layer2_attention_weight_min": -32.13298034667969,
      "activations/layer3_attention_weight_max": 101.18258666992188,
      "activations/layer3_attention_weight_min": -101.84030151367188,
      "activations/layer4_attention_weight_max": 106.27507781982422,
      "activations/layer4_attention_weight_min": -105.52569580078125,
      "activations/layer5_attention_weight_max": 49.792686462402344,
      "activations/layer5_attention_weight_min": -58.73290252685547,
      "activations/layer6_attention_weight_max": 43.93283462524414,
      "activations/layer6_attention_weight_min": -47.09799575805664,
      "activations/layer7_attention_weight_max": 97.66278076171875,
      "activations/layer7_attention_weight_min": -98.48530578613281,
      "activations/layer8_attention_weight_max": 43.19612121582031,
      "activations/layer8_attention_weight_min": -44.71985626220703,
      "activations/layer9_attention_weight_max": 32.909725189208984,
      "activations/layer9_attention_weight_min": -35.660579681396484,
      "epoch": 20.45,
      "learning_rate": 1.8286742424242422e-05,
      "loss": 2.71,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4656,
      "eval_samples_per_second": 507.227,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4656,
      "eval_openwebtext_samples_per_second": 507.227,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.6313,
      "eval_wikitext_samples_per_second": 279.532,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_lambada_loss": 2.3046875,
      "eval_lambada_ppl": 10.021046186131915,
      "eval_lambada_runtime": 9.5215,
      "eval_lambada_samples_per_second": 511.367,
      "step": 352000
    },
    {
      "activations/layer0_attention_weight_max": 16.301013946533203,
      "activations/layer0_attention_weight_min": -13.591774940490723,
      "activations/layer10_attention_weight_max": 37.82158660888672,
      "activations/layer10_attention_weight_min": -34.52931594848633,
      "activations/layer11_attention_weight_max": 35.64671325683594,
      "activations/layer11_attention_weight_min": -36.206966400146484,
      "activations/layer12_attention_weight_max": 27.08527946472168,
      "activations/layer12_attention_weight_min": -28.108041763305664,
      "activations/layer13_attention_weight_max": 38.76005172729492,
      "activations/layer13_attention_weight_min": -38.4977912902832,
      "activations/layer14_attention_weight_max": 43.79560852050781,
      "activations/layer14_attention_weight_min": -39.12186813354492,
      "activations/layer15_attention_weight_max": 39.3773307800293,
      "activations/layer15_attention_weight_min": -36.8343620300293,
      "activations/layer16_attention_weight_max": 29.49423599243164,
      "activations/layer16_attention_weight_min": -29.848003387451172,
      "activations/layer17_attention_weight_max": 30.6976318359375,
      "activations/layer17_attention_weight_min": -27.919578552246094,
      "activations/layer18_attention_weight_max": 31.436622619628906,
      "activations/layer18_attention_weight_min": -27.35113525390625,
      "activations/layer19_attention_weight_max": 33.695858001708984,
      "activations/layer19_attention_weight_min": -34.935630798339844,
      "activations/layer1_attention_weight_max": 17.378183364868164,
      "activations/layer1_attention_weight_min": -15.49057674407959,
      "activations/layer20_attention_weight_max": 26.824838638305664,
      "activations/layer20_attention_weight_min": -27.794450759887695,
      "activations/layer21_attention_weight_max": 28.06174087524414,
      "activations/layer21_attention_weight_min": -27.644207000732422,
      "activations/layer22_attention_weight_max": 42.864280700683594,
      "activations/layer22_attention_weight_min": -40.21302032470703,
      "activations/layer23_attention_weight_max": 31.617366790771484,
      "activations/layer23_attention_weight_min": -29.22230339050293,
      "activations/layer2_attention_weight_max": 32.11170959472656,
      "activations/layer2_attention_weight_min": -32.21331024169922,
      "activations/layer3_attention_weight_max": 105.97547912597656,
      "activations/layer3_attention_weight_min": -105.032958984375,
      "activations/layer4_attention_weight_max": 110.96925354003906,
      "activations/layer4_attention_weight_min": -111.00086975097656,
      "activations/layer5_attention_weight_max": 52.957618713378906,
      "activations/layer5_attention_weight_min": -61.1329345703125,
      "activations/layer6_attention_weight_max": 47.65925216674805,
      "activations/layer6_attention_weight_min": -50.975059509277344,
      "activations/layer7_attention_weight_max": 101.58148193359375,
      "activations/layer7_attention_weight_min": -106.22183227539062,
      "activations/layer8_attention_weight_max": 44.79255294799805,
      "activations/layer8_attention_weight_min": -46.607688903808594,
      "activations/layer9_attention_weight_max": 34.147056579589844,
      "activations/layer9_attention_weight_min": -36.023231506347656,
      "epoch": 20.46,
      "learning_rate": 1.826780303030303e-05,
      "loss": 2.7244,
      "step": 352050
    },
    {
      "activations/layer0_attention_weight_max": 17.068159103393555,
      "activations/layer0_attention_weight_min": -14.500802040100098,
      "activations/layer10_attention_weight_max": 33.225440979003906,
      "activations/layer10_attention_weight_min": -33.818275451660156,
      "activations/layer11_attention_weight_max": 33.62677001953125,
      "activations/layer11_attention_weight_min": -35.50491714477539,
      "activations/layer12_attention_weight_max": 28.413925170898438,
      "activations/layer12_attention_weight_min": -27.532270431518555,
      "activations/layer13_attention_weight_max": 40.06795883178711,
      "activations/layer13_attention_weight_min": -37.437801361083984,
      "activations/layer14_attention_weight_max": 45.25791549682617,
      "activations/layer14_attention_weight_min": -38.62018585205078,
      "activations/layer15_attention_weight_max": 41.11334991455078,
      "activations/layer15_attention_weight_min": -41.90179443359375,
      "activations/layer16_attention_weight_max": 33.73902130126953,
      "activations/layer16_attention_weight_min": -31.00852394104004,
      "activations/layer17_attention_weight_max": 31.046682357788086,
      "activations/layer17_attention_weight_min": -27.73914909362793,
      "activations/layer18_attention_weight_max": 35.2274284362793,
      "activations/layer18_attention_weight_min": -26.503225326538086,
      "activations/layer19_attention_weight_max": 37.85078048706055,
      "activations/layer19_attention_weight_min": -32.49668502807617,
      "activations/layer1_attention_weight_max": 17.060375213623047,
      "activations/layer1_attention_weight_min": -14.86829948425293,
      "activations/layer20_attention_weight_max": 31.789615631103516,
      "activations/layer20_attention_weight_min": -27.03639793395996,
      "activations/layer21_attention_weight_max": 32.25274658203125,
      "activations/layer21_attention_weight_min": -27.14934539794922,
      "activations/layer22_attention_weight_max": 42.94313049316406,
      "activations/layer22_attention_weight_min": -34.17290115356445,
      "activations/layer23_attention_weight_max": 34.823543548583984,
      "activations/layer23_attention_weight_min": -26.397125244140625,
      "activations/layer2_attention_weight_max": 32.795249938964844,
      "activations/layer2_attention_weight_min": -32.33590316772461,
      "activations/layer3_attention_weight_max": 105.2923355102539,
      "activations/layer3_attention_weight_min": -102.02415466308594,
      "activations/layer4_attention_weight_max": 109.3450927734375,
      "activations/layer4_attention_weight_min": -107.6073989868164,
      "activations/layer5_attention_weight_max": 51.98761749267578,
      "activations/layer5_attention_weight_min": -58.57517623901367,
      "activations/layer6_attention_weight_max": 46.05431365966797,
      "activations/layer6_attention_weight_min": -46.957435607910156,
      "activations/layer7_attention_weight_max": 96.68791198730469,
      "activations/layer7_attention_weight_min": -102.39302062988281,
      "activations/layer8_attention_weight_max": 46.0193977355957,
      "activations/layer8_attention_weight_min": -48.35141372680664,
      "activations/layer9_attention_weight_max": 32.38780212402344,
      "activations/layer9_attention_weight_min": -36.601661682128906,
      "epoch": 20.46,
      "learning_rate": 1.8248863636363635e-05,
      "loss": 2.7092,
      "step": 352100
    },
    {
      "activations/layer0_attention_weight_max": 16.490997314453125,
      "activations/layer0_attention_weight_min": -14.519582748413086,
      "activations/layer10_attention_weight_max": 36.81156539916992,
      "activations/layer10_attention_weight_min": -36.74167251586914,
      "activations/layer11_attention_weight_max": 35.81322479248047,
      "activations/layer11_attention_weight_min": -35.0228157043457,
      "activations/layer12_attention_weight_max": 26.733049392700195,
      "activations/layer12_attention_weight_min": -27.06690788269043,
      "activations/layer13_attention_weight_max": 41.07261657714844,
      "activations/layer13_attention_weight_min": -39.18052291870117,
      "activations/layer14_attention_weight_max": 42.12480545043945,
      "activations/layer14_attention_weight_min": -38.07780456542969,
      "activations/layer15_attention_weight_max": 42.84490966796875,
      "activations/layer15_attention_weight_min": -36.68743133544922,
      "activations/layer16_attention_weight_max": 31.085412979125977,
      "activations/layer16_attention_weight_min": -31.714168548583984,
      "activations/layer17_attention_weight_max": 30.498220443725586,
      "activations/layer17_attention_weight_min": -26.808984756469727,
      "activations/layer18_attention_weight_max": 32.46800994873047,
      "activations/layer18_attention_weight_min": -27.133724212646484,
      "activations/layer19_attention_weight_max": 37.18814468383789,
      "activations/layer19_attention_weight_min": -32.95496368408203,
      "activations/layer1_attention_weight_max": 15.649728775024414,
      "activations/layer1_attention_weight_min": -15.082499504089355,
      "activations/layer20_attention_weight_max": 28.962446212768555,
      "activations/layer20_attention_weight_min": -26.016727447509766,
      "activations/layer21_attention_weight_max": 28.040014266967773,
      "activations/layer21_attention_weight_min": -25.111364364624023,
      "activations/layer22_attention_weight_max": 44.97568893432617,
      "activations/layer22_attention_weight_min": -33.49555206298828,
      "activations/layer23_attention_weight_max": 36.4150276184082,
      "activations/layer23_attention_weight_min": -27.67658042907715,
      "activations/layer2_attention_weight_max": 35.49262237548828,
      "activations/layer2_attention_weight_min": -34.0238151550293,
      "activations/layer3_attention_weight_max": 105.76687622070312,
      "activations/layer3_attention_weight_min": -104.11111450195312,
      "activations/layer4_attention_weight_max": 109.11278533935547,
      "activations/layer4_attention_weight_min": -110.5073013305664,
      "activations/layer5_attention_weight_max": 50.506324768066406,
      "activations/layer5_attention_weight_min": -60.51865768432617,
      "activations/layer6_attention_weight_max": 45.379676818847656,
      "activations/layer6_attention_weight_min": -49.163089752197266,
      "activations/layer7_attention_weight_max": 94.22891235351562,
      "activations/layer7_attention_weight_min": -96.19405364990234,
      "activations/layer8_attention_weight_max": 43.66059875488281,
      "activations/layer8_attention_weight_min": -45.53535079956055,
      "activations/layer9_attention_weight_max": 33.43696975708008,
      "activations/layer9_attention_weight_min": -36.32927703857422,
      "epoch": 20.46,
      "learning_rate": 1.822992424242424e-05,
      "loss": 2.6965,
      "step": 352150
    },
    {
      "activations/layer0_attention_weight_max": 17.13055992126465,
      "activations/layer0_attention_weight_min": -14.313846588134766,
      "activations/layer10_attention_weight_max": 39.81414794921875,
      "activations/layer10_attention_weight_min": -38.128517150878906,
      "activations/layer11_attention_weight_max": 38.78671646118164,
      "activations/layer11_attention_weight_min": -36.2962760925293,
      "activations/layer12_attention_weight_max": 27.709152221679688,
      "activations/layer12_attention_weight_min": -28.076011657714844,
      "activations/layer13_attention_weight_max": 40.82575225830078,
      "activations/layer13_attention_weight_min": -36.89037322998047,
      "activations/layer14_attention_weight_max": 45.93857955932617,
      "activations/layer14_attention_weight_min": -38.84169006347656,
      "activations/layer15_attention_weight_max": 40.56826400756836,
      "activations/layer15_attention_weight_min": -35.27920150756836,
      "activations/layer16_attention_weight_max": 30.297571182250977,
      "activations/layer16_attention_weight_min": -29.799516677856445,
      "activations/layer17_attention_weight_max": 31.633317947387695,
      "activations/layer17_attention_weight_min": -29.63804817199707,
      "activations/layer18_attention_weight_max": 35.00639343261719,
      "activations/layer18_attention_weight_min": -26.931396484375,
      "activations/layer19_attention_weight_max": 35.03654098510742,
      "activations/layer19_attention_weight_min": -31.654403686523438,
      "activations/layer1_attention_weight_max": 15.812820434570312,
      "activations/layer1_attention_weight_min": -14.301712989807129,
      "activations/layer20_attention_weight_max": 29.428449630737305,
      "activations/layer20_attention_weight_min": -25.94875717163086,
      "activations/layer21_attention_weight_max": 29.511608123779297,
      "activations/layer21_attention_weight_min": -24.687297821044922,
      "activations/layer22_attention_weight_max": 46.30856704711914,
      "activations/layer22_attention_weight_min": -34.17576217651367,
      "activations/layer23_attention_weight_max": 35.419593811035156,
      "activations/layer23_attention_weight_min": -26.472522735595703,
      "activations/layer2_attention_weight_max": 30.29793930053711,
      "activations/layer2_attention_weight_min": -31.932621002197266,
      "activations/layer3_attention_weight_max": 101.2843017578125,
      "activations/layer3_attention_weight_min": -102.46481323242188,
      "activations/layer4_attention_weight_max": 108.9799575805664,
      "activations/layer4_attention_weight_min": -114.02828216552734,
      "activations/layer5_attention_weight_max": 53.125511169433594,
      "activations/layer5_attention_weight_min": -59.815181732177734,
      "activations/layer6_attention_weight_max": 47.19136047363281,
      "activations/layer6_attention_weight_min": -51.08378219604492,
      "activations/layer7_attention_weight_max": 100.49276733398438,
      "activations/layer7_attention_weight_min": -105.843017578125,
      "activations/layer8_attention_weight_max": 45.96240234375,
      "activations/layer8_attention_weight_min": -47.672611236572266,
      "activations/layer9_attention_weight_max": 34.929508209228516,
      "activations/layer9_attention_weight_min": -37.50669860839844,
      "epoch": 20.46,
      "learning_rate": 1.821098484848485e-05,
      "loss": 2.7117,
      "step": 352200
    },
    {
      "activations/layer0_attention_weight_max": 16.91698455810547,
      "activations/layer0_attention_weight_min": -14.15339183807373,
      "activations/layer10_attention_weight_max": 30.97141456604004,
      "activations/layer10_attention_weight_min": -34.501686096191406,
      "activations/layer11_attention_weight_max": 34.345455169677734,
      "activations/layer11_attention_weight_min": -33.79397201538086,
      "activations/layer12_attention_weight_max": 26.550931930541992,
      "activations/layer12_attention_weight_min": -28.464426040649414,
      "activations/layer13_attention_weight_max": 39.12251281738281,
      "activations/layer13_attention_weight_min": -36.79010009765625,
      "activations/layer14_attention_weight_max": 40.79965591430664,
      "activations/layer14_attention_weight_min": -39.723812103271484,
      "activations/layer15_attention_weight_max": 36.599342346191406,
      "activations/layer15_attention_weight_min": -38.741397857666016,
      "activations/layer16_attention_weight_max": 27.88585090637207,
      "activations/layer16_attention_weight_min": -29.4918270111084,
      "activations/layer17_attention_weight_max": 30.601037979125977,
      "activations/layer17_attention_weight_min": -31.420637130737305,
      "activations/layer18_attention_weight_max": 31.88796615600586,
      "activations/layer18_attention_weight_min": -29.1934871673584,
      "activations/layer19_attention_weight_max": 35.57802963256836,
      "activations/layer19_attention_weight_min": -33.18317794799805,
      "activations/layer1_attention_weight_max": 16.461782455444336,
      "activations/layer1_attention_weight_min": -14.355595588684082,
      "activations/layer20_attention_weight_max": 31.354217529296875,
      "activations/layer20_attention_weight_min": -28.424909591674805,
      "activations/layer21_attention_weight_max": 31.318571090698242,
      "activations/layer21_attention_weight_min": -25.940006256103516,
      "activations/layer22_attention_weight_max": 39.04229736328125,
      "activations/layer22_attention_weight_min": -32.54990768432617,
      "activations/layer23_attention_weight_max": 32.84946060180664,
      "activations/layer23_attention_weight_min": -26.849193572998047,
      "activations/layer2_attention_weight_max": 33.58039093017578,
      "activations/layer2_attention_weight_min": -31.884471893310547,
      "activations/layer3_attention_weight_max": 106.7772216796875,
      "activations/layer3_attention_weight_min": -103.41356658935547,
      "activations/layer4_attention_weight_max": 112.25821685791016,
      "activations/layer4_attention_weight_min": -114.81542205810547,
      "activations/layer5_attention_weight_max": 52.8912353515625,
      "activations/layer5_attention_weight_min": -64.47955322265625,
      "activations/layer6_attention_weight_max": 45.19663619995117,
      "activations/layer6_attention_weight_min": -47.296810150146484,
      "activations/layer7_attention_weight_max": 93.06056213378906,
      "activations/layer7_attention_weight_min": -96.16500091552734,
      "activations/layer8_attention_weight_max": 42.3032112121582,
      "activations/layer8_attention_weight_min": -42.34591293334961,
      "activations/layer9_attention_weight_max": 31.312366485595703,
      "activations/layer9_attention_weight_min": -34.8149528503418,
      "epoch": 20.47,
      "learning_rate": 1.8192045454545453e-05,
      "loss": 2.7345,
      "step": 352250
    },
    {
      "activations/layer0_attention_weight_max": 16.31037712097168,
      "activations/layer0_attention_weight_min": -14.057924270629883,
      "activations/layer10_attention_weight_max": 31.98782730102539,
      "activations/layer10_attention_weight_min": -34.573551177978516,
      "activations/layer11_attention_weight_max": 34.31554412841797,
      "activations/layer11_attention_weight_min": -35.6658935546875,
      "activations/layer12_attention_weight_max": 25.816835403442383,
      "activations/layer12_attention_weight_min": -27.636539459228516,
      "activations/layer13_attention_weight_max": 41.33918380737305,
      "activations/layer13_attention_weight_min": -36.403011322021484,
      "activations/layer14_attention_weight_max": 44.35226058959961,
      "activations/layer14_attention_weight_min": -38.94132995605469,
      "activations/layer15_attention_weight_max": 42.90419006347656,
      "activations/layer15_attention_weight_min": -37.351444244384766,
      "activations/layer16_attention_weight_max": 33.290626525878906,
      "activations/layer16_attention_weight_min": -28.134092330932617,
      "activations/layer17_attention_weight_max": 31.878889083862305,
      "activations/layer17_attention_weight_min": -30.109785079956055,
      "activations/layer18_attention_weight_max": 33.35880661010742,
      "activations/layer18_attention_weight_min": -29.15467643737793,
      "activations/layer19_attention_weight_max": 33.385860443115234,
      "activations/layer19_attention_weight_min": -34.75156021118164,
      "activations/layer1_attention_weight_max": 15.872361183166504,
      "activations/layer1_attention_weight_min": -15.532814025878906,
      "activations/layer20_attention_weight_max": 29.735210418701172,
      "activations/layer20_attention_weight_min": -26.8563175201416,
      "activations/layer21_attention_weight_max": 27.311344146728516,
      "activations/layer21_attention_weight_min": -26.24166488647461,
      "activations/layer22_attention_weight_max": 40.861167907714844,
      "activations/layer22_attention_weight_min": -32.22513198852539,
      "activations/layer23_attention_weight_max": 30.34664535522461,
      "activations/layer23_attention_weight_min": -26.251379013061523,
      "activations/layer2_attention_weight_max": 33.13751983642578,
      "activations/layer2_attention_weight_min": -31.554187774658203,
      "activations/layer3_attention_weight_max": 102.7978286743164,
      "activations/layer3_attention_weight_min": -103.1471939086914,
      "activations/layer4_attention_weight_max": 106.70283508300781,
      "activations/layer4_attention_weight_min": -107.06678771972656,
      "activations/layer5_attention_weight_max": 52.87034606933594,
      "activations/layer5_attention_weight_min": -60.91698455810547,
      "activations/layer6_attention_weight_max": 43.6365852355957,
      "activations/layer6_attention_weight_min": -46.62234878540039,
      "activations/layer7_attention_weight_max": 96.15496826171875,
      "activations/layer7_attention_weight_min": -96.5350112915039,
      "activations/layer8_attention_weight_max": 42.610198974609375,
      "activations/layer8_attention_weight_min": -43.480770111083984,
      "activations/layer9_attention_weight_max": 32.74198913574219,
      "activations/layer9_attention_weight_min": -34.816917419433594,
      "epoch": 20.47,
      "learning_rate": 1.8173106060606058e-05,
      "loss": 2.7158,
      "step": 352300
    },
    {
      "activations/layer0_attention_weight_max": 15.816983222961426,
      "activations/layer0_attention_weight_min": -13.63764476776123,
      "activations/layer10_attention_weight_max": 35.079742431640625,
      "activations/layer10_attention_weight_min": -34.99872970581055,
      "activations/layer11_attention_weight_max": 35.42660903930664,
      "activations/layer11_attention_weight_min": -34.69451904296875,
      "activations/layer12_attention_weight_max": 28.2547550201416,
      "activations/layer12_attention_weight_min": -28.41227912902832,
      "activations/layer13_attention_weight_max": 42.12960433959961,
      "activations/layer13_attention_weight_min": -36.23386764526367,
      "activations/layer14_attention_weight_max": 47.10720443725586,
      "activations/layer14_attention_weight_min": -38.376922607421875,
      "activations/layer15_attention_weight_max": 40.64381790161133,
      "activations/layer15_attention_weight_min": -38.139522552490234,
      "activations/layer16_attention_weight_max": 28.648204803466797,
      "activations/layer16_attention_weight_min": -27.72311782836914,
      "activations/layer17_attention_weight_max": 29.349687576293945,
      "activations/layer17_attention_weight_min": -27.92250633239746,
      "activations/layer18_attention_weight_max": 31.6877498626709,
      "activations/layer18_attention_weight_min": -29.252525329589844,
      "activations/layer19_attention_weight_max": 32.87330627441406,
      "activations/layer19_attention_weight_min": -33.304046630859375,
      "activations/layer1_attention_weight_max": 15.53802490234375,
      "activations/layer1_attention_weight_min": -14.94166374206543,
      "activations/layer20_attention_weight_max": 29.199474334716797,
      "activations/layer20_attention_weight_min": -25.98212242126465,
      "activations/layer21_attention_weight_max": 29.391006469726562,
      "activations/layer21_attention_weight_min": -26.99405860900879,
      "activations/layer22_attention_weight_max": 39.54633331298828,
      "activations/layer22_attention_weight_min": -35.547515869140625,
      "activations/layer23_attention_weight_max": 32.033199310302734,
      "activations/layer23_attention_weight_min": -27.604305267333984,
      "activations/layer2_attention_weight_max": 33.135032653808594,
      "activations/layer2_attention_weight_min": -29.97462272644043,
      "activations/layer3_attention_weight_max": 100.3709945678711,
      "activations/layer3_attention_weight_min": -102.40218353271484,
      "activations/layer4_attention_weight_max": 105.49458312988281,
      "activations/layer4_attention_weight_min": -107.18403625488281,
      "activations/layer5_attention_weight_max": 52.136810302734375,
      "activations/layer5_attention_weight_min": -59.89509201049805,
      "activations/layer6_attention_weight_max": 43.04593276977539,
      "activations/layer6_attention_weight_min": -45.937686920166016,
      "activations/layer7_attention_weight_max": 93.16812133789062,
      "activations/layer7_attention_weight_min": -96.24162292480469,
      "activations/layer8_attention_weight_max": 40.1178092956543,
      "activations/layer8_attention_weight_min": -42.76057815551758,
      "activations/layer9_attention_weight_max": 32.37174987792969,
      "activations/layer9_attention_weight_min": -34.193016052246094,
      "epoch": 20.47,
      "learning_rate": 1.8154166666666666e-05,
      "loss": 2.724,
      "step": 352350
    },
    {
      "activations/layer0_attention_weight_max": 16.281482696533203,
      "activations/layer0_attention_weight_min": -13.278950691223145,
      "activations/layer10_attention_weight_max": 37.31077575683594,
      "activations/layer10_attention_weight_min": -38.37470245361328,
      "activations/layer11_attention_weight_max": 37.664546966552734,
      "activations/layer11_attention_weight_min": -39.916229248046875,
      "activations/layer12_attention_weight_max": 30.98638343811035,
      "activations/layer12_attention_weight_min": -28.651403427124023,
      "activations/layer13_attention_weight_max": 45.43162536621094,
      "activations/layer13_attention_weight_min": -39.36997985839844,
      "activations/layer14_attention_weight_max": 47.481773376464844,
      "activations/layer14_attention_weight_min": -41.24422073364258,
      "activations/layer15_attention_weight_max": 43.84912872314453,
      "activations/layer15_attention_weight_min": -38.176116943359375,
      "activations/layer16_attention_weight_max": 32.377593994140625,
      "activations/layer16_attention_weight_min": -29.184839248657227,
      "activations/layer17_attention_weight_max": 30.653411865234375,
      "activations/layer17_attention_weight_min": -26.06053352355957,
      "activations/layer18_attention_weight_max": 33.36109924316406,
      "activations/layer18_attention_weight_min": -27.482229232788086,
      "activations/layer19_attention_weight_max": 40.57114791870117,
      "activations/layer19_attention_weight_min": -32.6917839050293,
      "activations/layer1_attention_weight_max": 15.740821838378906,
      "activations/layer1_attention_weight_min": -15.625738143920898,
      "activations/layer20_attention_weight_max": 30.389514923095703,
      "activations/layer20_attention_weight_min": -25.084095001220703,
      "activations/layer21_attention_weight_max": 29.729400634765625,
      "activations/layer21_attention_weight_min": -23.864532470703125,
      "activations/layer22_attention_weight_max": 42.250850677490234,
      "activations/layer22_attention_weight_min": -33.08500289916992,
      "activations/layer23_attention_weight_max": 32.87360382080078,
      "activations/layer23_attention_weight_min": -25.16733169555664,
      "activations/layer2_attention_weight_max": 32.40364074707031,
      "activations/layer2_attention_weight_min": -32.04631423950195,
      "activations/layer3_attention_weight_max": 105.15819549560547,
      "activations/layer3_attention_weight_min": -107.51793670654297,
      "activations/layer4_attention_weight_max": 111.1986083984375,
      "activations/layer4_attention_weight_min": -112.63553619384766,
      "activations/layer5_attention_weight_max": 52.51765060424805,
      "activations/layer5_attention_weight_min": -63.876197814941406,
      "activations/layer6_attention_weight_max": 47.15943145751953,
      "activations/layer6_attention_weight_min": -50.263832092285156,
      "activations/layer7_attention_weight_max": 99.59175109863281,
      "activations/layer7_attention_weight_min": -105.79603576660156,
      "activations/layer8_attention_weight_max": 45.19536209106445,
      "activations/layer8_attention_weight_min": -50.260982513427734,
      "activations/layer9_attention_weight_max": 35.55598449707031,
      "activations/layer9_attention_weight_min": -38.23747634887695,
      "epoch": 20.48,
      "learning_rate": 1.813522727272727e-05,
      "loss": 2.7117,
      "step": 352400
    },
    {
      "activations/layer0_attention_weight_max": 16.101062774658203,
      "activations/layer0_attention_weight_min": -14.088478088378906,
      "activations/layer10_attention_weight_max": 42.20977783203125,
      "activations/layer10_attention_weight_min": -43.6425895690918,
      "activations/layer11_attention_weight_max": 41.83355712890625,
      "activations/layer11_attention_weight_min": -43.388465881347656,
      "activations/layer12_attention_weight_max": 31.500167846679688,
      "activations/layer12_attention_weight_min": -30.895767211914062,
      "activations/layer13_attention_weight_max": 46.90999984741211,
      "activations/layer13_attention_weight_min": -42.226131439208984,
      "activations/layer14_attention_weight_max": 55.43015670776367,
      "activations/layer14_attention_weight_min": -46.51496124267578,
      "activations/layer15_attention_weight_max": 51.59578323364258,
      "activations/layer15_attention_weight_min": -44.03475570678711,
      "activations/layer16_attention_weight_max": 36.529300689697266,
      "activations/layer16_attention_weight_min": -31.45124053955078,
      "activations/layer17_attention_weight_max": 35.728878021240234,
      "activations/layer17_attention_weight_min": -28.99508285522461,
      "activations/layer18_attention_weight_max": 40.45956802368164,
      "activations/layer18_attention_weight_min": -29.977083206176758,
      "activations/layer19_attention_weight_max": 43.498374938964844,
      "activations/layer19_attention_weight_min": -33.893653869628906,
      "activations/layer1_attention_weight_max": 15.690593719482422,
      "activations/layer1_attention_weight_min": -14.795279502868652,
      "activations/layer20_attention_weight_max": 34.350467681884766,
      "activations/layer20_attention_weight_min": -26.29320526123047,
      "activations/layer21_attention_weight_max": 34.21974563598633,
      "activations/layer21_attention_weight_min": -25.98253059387207,
      "activations/layer22_attention_weight_max": 56.5741081237793,
      "activations/layer22_attention_weight_min": -35.20240783691406,
      "activations/layer23_attention_weight_max": 41.8410530090332,
      "activations/layer23_attention_weight_min": -27.71436309814453,
      "activations/layer2_attention_weight_max": 34.479698181152344,
      "activations/layer2_attention_weight_min": -31.875354766845703,
      "activations/layer3_attention_weight_max": 103.2799301147461,
      "activations/layer3_attention_weight_min": -105.09229278564453,
      "activations/layer4_attention_weight_max": 108.24967193603516,
      "activations/layer4_attention_weight_min": -113.1182861328125,
      "activations/layer5_attention_weight_max": 52.51247787475586,
      "activations/layer5_attention_weight_min": -63.187095642089844,
      "activations/layer6_attention_weight_max": 46.4521369934082,
      "activations/layer6_attention_weight_min": -47.97547149658203,
      "activations/layer7_attention_weight_max": 101.13263702392578,
      "activations/layer7_attention_weight_min": -109.16728210449219,
      "activations/layer8_attention_weight_max": 46.838897705078125,
      "activations/layer8_attention_weight_min": -49.61787796020508,
      "activations/layer9_attention_weight_max": 38.38710403442383,
      "activations/layer9_attention_weight_min": -42.268028259277344,
      "epoch": 20.48,
      "learning_rate": 1.8116287878787876e-05,
      "loss": 2.6913,
      "step": 352450
    },
    {
      "activations/layer0_attention_weight_max": 15.985356330871582,
      "activations/layer0_attention_weight_min": -14.274055480957031,
      "activations/layer10_attention_weight_max": 34.49980545043945,
      "activations/layer10_attention_weight_min": -37.46080780029297,
      "activations/layer11_attention_weight_max": 35.85157775878906,
      "activations/layer11_attention_weight_min": -35.8361930847168,
      "activations/layer12_attention_weight_max": 26.882478713989258,
      "activations/layer12_attention_weight_min": -27.729022979736328,
      "activations/layer13_attention_weight_max": 40.24675750732422,
      "activations/layer13_attention_weight_min": -38.10200500488281,
      "activations/layer14_attention_weight_max": 42.85822677612305,
      "activations/layer14_attention_weight_min": -37.510841369628906,
      "activations/layer15_attention_weight_max": 40.248287200927734,
      "activations/layer15_attention_weight_min": -36.813663482666016,
      "activations/layer16_attention_weight_max": 28.039875030517578,
      "activations/layer16_attention_weight_min": -30.355070114135742,
      "activations/layer17_attention_weight_max": 28.9542236328125,
      "activations/layer17_attention_weight_min": -28.578378677368164,
      "activations/layer18_attention_weight_max": 32.39113998413086,
      "activations/layer18_attention_weight_min": -30.083452224731445,
      "activations/layer19_attention_weight_max": 35.277015686035156,
      "activations/layer19_attention_weight_min": -32.21921920776367,
      "activations/layer1_attention_weight_max": 15.774306297302246,
      "activations/layer1_attention_weight_min": -14.331974983215332,
      "activations/layer20_attention_weight_max": 27.293554306030273,
      "activations/layer20_attention_weight_min": -27.32890510559082,
      "activations/layer21_attention_weight_max": 29.07072639465332,
      "activations/layer21_attention_weight_min": -25.92962074279785,
      "activations/layer22_attention_weight_max": 42.76050567626953,
      "activations/layer22_attention_weight_min": -31.27469825744629,
      "activations/layer23_attention_weight_max": 34.8626823425293,
      "activations/layer23_attention_weight_min": -27.2015380859375,
      "activations/layer2_attention_weight_max": 32.81589889526367,
      "activations/layer2_attention_weight_min": -32.64095687866211,
      "activations/layer3_attention_weight_max": 105.62207794189453,
      "activations/layer3_attention_weight_min": -106.27456665039062,
      "activations/layer4_attention_weight_max": 109.77330780029297,
      "activations/layer4_attention_weight_min": -111.87847900390625,
      "activations/layer5_attention_weight_max": 51.168922424316406,
      "activations/layer5_attention_weight_min": -65.56883239746094,
      "activations/layer6_attention_weight_max": 45.39082717895508,
      "activations/layer6_attention_weight_min": -48.140785217285156,
      "activations/layer7_attention_weight_max": 97.26012420654297,
      "activations/layer7_attention_weight_min": -101.04359436035156,
      "activations/layer8_attention_weight_max": 42.68571090698242,
      "activations/layer8_attention_weight_min": -45.930946350097656,
      "activations/layer9_attention_weight_max": 33.257198333740234,
      "activations/layer9_attention_weight_min": -35.84479904174805,
      "epoch": 20.48,
      "learning_rate": 1.809734848484848e-05,
      "loss": 2.737,
      "step": 352500
    },
    {
      "activations/layer0_attention_weight_max": 16.211668014526367,
      "activations/layer0_attention_weight_min": -15.329060554504395,
      "activations/layer10_attention_weight_max": 37.96438980102539,
      "activations/layer10_attention_weight_min": -38.9216423034668,
      "activations/layer11_attention_weight_max": 36.38285446166992,
      "activations/layer11_attention_weight_min": -36.36595916748047,
      "activations/layer12_attention_weight_max": 27.9904727935791,
      "activations/layer12_attention_weight_min": -29.20350456237793,
      "activations/layer13_attention_weight_max": 42.0861701965332,
      "activations/layer13_attention_weight_min": -38.822872161865234,
      "activations/layer14_attention_weight_max": 45.10375213623047,
      "activations/layer14_attention_weight_min": -44.73187255859375,
      "activations/layer15_attention_weight_max": 40.345279693603516,
      "activations/layer15_attention_weight_min": -38.538002014160156,
      "activations/layer16_attention_weight_max": 31.51311492919922,
      "activations/layer16_attention_weight_min": -28.534711837768555,
      "activations/layer17_attention_weight_max": 30.420021057128906,
      "activations/layer17_attention_weight_min": -27.3959903717041,
      "activations/layer18_attention_weight_max": 33.63634490966797,
      "activations/layer18_attention_weight_min": -27.434127807617188,
      "activations/layer19_attention_weight_max": 36.07780075073242,
      "activations/layer19_attention_weight_min": -31.00617027282715,
      "activations/layer1_attention_weight_max": 15.467093467712402,
      "activations/layer1_attention_weight_min": -14.76411247253418,
      "activations/layer20_attention_weight_max": 30.556365966796875,
      "activations/layer20_attention_weight_min": -26.422517776489258,
      "activations/layer21_attention_weight_max": 29.66857147216797,
      "activations/layer21_attention_weight_min": -26.040847778320312,
      "activations/layer22_attention_weight_max": 46.70880889892578,
      "activations/layer22_attention_weight_min": -32.9550895690918,
      "activations/layer23_attention_weight_max": 31.602523803710938,
      "activations/layer23_attention_weight_min": -25.689083099365234,
      "activations/layer2_attention_weight_max": 33.45415496826172,
      "activations/layer2_attention_weight_min": -33.12139129638672,
      "activations/layer3_attention_weight_max": 102.86731719970703,
      "activations/layer3_attention_weight_min": -107.75617218017578,
      "activations/layer4_attention_weight_max": 112.86893463134766,
      "activations/layer4_attention_weight_min": -112.66670989990234,
      "activations/layer5_attention_weight_max": 51.82993698120117,
      "activations/layer5_attention_weight_min": -61.95498275756836,
      "activations/layer6_attention_weight_max": 45.60243606567383,
      "activations/layer6_attention_weight_min": -49.034976959228516,
      "activations/layer7_attention_weight_max": 98.94712829589844,
      "activations/layer7_attention_weight_min": -104.80775451660156,
      "activations/layer8_attention_weight_max": 44.75447082519531,
      "activations/layer8_attention_weight_min": -46.206085205078125,
      "activations/layer9_attention_weight_max": 34.63901901245117,
      "activations/layer9_attention_weight_min": -36.08614730834961,
      "epoch": 20.49,
      "learning_rate": 1.807840909090909e-05,
      "loss": 2.717,
      "step": 352550
    },
    {
      "activations/layer0_attention_weight_max": 17.898691177368164,
      "activations/layer0_attention_weight_min": -14.63240909576416,
      "activations/layer10_attention_weight_max": 35.2015266418457,
      "activations/layer10_attention_weight_min": -35.6248893737793,
      "activations/layer11_attention_weight_max": 34.73283386230469,
      "activations/layer11_attention_weight_min": -37.12789535522461,
      "activations/layer12_attention_weight_max": 26.870744705200195,
      "activations/layer12_attention_weight_min": -33.055782318115234,
      "activations/layer13_attention_weight_max": 40.49962615966797,
      "activations/layer13_attention_weight_min": -37.9140625,
      "activations/layer14_attention_weight_max": 41.88386154174805,
      "activations/layer14_attention_weight_min": -40.20762252807617,
      "activations/layer15_attention_weight_max": 40.58804702758789,
      "activations/layer15_attention_weight_min": -36.1602897644043,
      "activations/layer16_attention_weight_max": 28.76569938659668,
      "activations/layer16_attention_weight_min": -26.956501007080078,
      "activations/layer17_attention_weight_max": 29.22660255432129,
      "activations/layer17_attention_weight_min": -25.933040618896484,
      "activations/layer18_attention_weight_max": 33.571556091308594,
      "activations/layer18_attention_weight_min": -26.583309173583984,
      "activations/layer19_attention_weight_max": 32.52777099609375,
      "activations/layer19_attention_weight_min": -32.96553039550781,
      "activations/layer1_attention_weight_max": 15.183135986328125,
      "activations/layer1_attention_weight_min": -14.061820030212402,
      "activations/layer20_attention_weight_max": 28.34859275817871,
      "activations/layer20_attention_weight_min": -26.670963287353516,
      "activations/layer21_attention_weight_max": 28.480627059936523,
      "activations/layer21_attention_weight_min": -24.807329177856445,
      "activations/layer22_attention_weight_max": 39.968631744384766,
      "activations/layer22_attention_weight_min": -30.6380615234375,
      "activations/layer23_attention_weight_max": 31.591405868530273,
      "activations/layer23_attention_weight_min": -24.611675262451172,
      "activations/layer2_attention_weight_max": 32.09812927246094,
      "activations/layer2_attention_weight_min": -31.87175750732422,
      "activations/layer3_attention_weight_max": 98.79045104980469,
      "activations/layer3_attention_weight_min": -100.7485122680664,
      "activations/layer4_attention_weight_max": 102.30697631835938,
      "activations/layer4_attention_weight_min": -105.82906341552734,
      "activations/layer5_attention_weight_max": 48.57900619506836,
      "activations/layer5_attention_weight_min": -62.67632293701172,
      "activations/layer6_attention_weight_max": 43.64611053466797,
      "activations/layer6_attention_weight_min": -48.79716873168945,
      "activations/layer7_attention_weight_max": 94.99169921875,
      "activations/layer7_attention_weight_min": -103.57756042480469,
      "activations/layer8_attention_weight_max": 44.04884719848633,
      "activations/layer8_attention_weight_min": -47.73197555541992,
      "activations/layer9_attention_weight_max": 33.58367919921875,
      "activations/layer9_attention_weight_min": -36.188682556152344,
      "epoch": 20.49,
      "learning_rate": 1.8059469696969694e-05,
      "loss": 2.7041,
      "step": 352600
    },
    {
      "activations/layer0_attention_weight_max": 16.80268096923828,
      "activations/layer0_attention_weight_min": -14.221565246582031,
      "activations/layer10_attention_weight_max": 38.592491149902344,
      "activations/layer10_attention_weight_min": -35.94127655029297,
      "activations/layer11_attention_weight_max": 38.63151550292969,
      "activations/layer11_attention_weight_min": -36.60430145263672,
      "activations/layer12_attention_weight_max": 26.19846534729004,
      "activations/layer12_attention_weight_min": -27.853620529174805,
      "activations/layer13_attention_weight_max": 39.46208190917969,
      "activations/layer13_attention_weight_min": -37.03849411010742,
      "activations/layer14_attention_weight_max": 41.14916229248047,
      "activations/layer14_attention_weight_min": -39.85293197631836,
      "activations/layer15_attention_weight_max": 42.62876892089844,
      "activations/layer15_attention_weight_min": -39.63043212890625,
      "activations/layer16_attention_weight_max": 32.60619354248047,
      "activations/layer16_attention_weight_min": -29.13549041748047,
      "activations/layer17_attention_weight_max": 31.590646743774414,
      "activations/layer17_attention_weight_min": -30.62139320373535,
      "activations/layer18_attention_weight_max": 35.21772003173828,
      "activations/layer18_attention_weight_min": -26.394241333007812,
      "activations/layer19_attention_weight_max": 41.795433044433594,
      "activations/layer19_attention_weight_min": -31.169443130493164,
      "activations/layer1_attention_weight_max": 16.11305809020996,
      "activations/layer1_attention_weight_min": -13.980753898620605,
      "activations/layer20_attention_weight_max": 34.01908874511719,
      "activations/layer20_attention_weight_min": -25.064598083496094,
      "activations/layer21_attention_weight_max": 31.74419403076172,
      "activations/layer21_attention_weight_min": -24.692167282104492,
      "activations/layer22_attention_weight_max": 45.94963455200195,
      "activations/layer22_attention_weight_min": -29.708391189575195,
      "activations/layer23_attention_weight_max": 37.891990661621094,
      "activations/layer23_attention_weight_min": -25.127370834350586,
      "activations/layer2_attention_weight_max": 32.77080535888672,
      "activations/layer2_attention_weight_min": -32.36690902709961,
      "activations/layer3_attention_weight_max": 102.68976593017578,
      "activations/layer3_attention_weight_min": -102.74869537353516,
      "activations/layer4_attention_weight_max": 105.80223846435547,
      "activations/layer4_attention_weight_min": -110.9006118774414,
      "activations/layer5_attention_weight_max": 50.2169189453125,
      "activations/layer5_attention_weight_min": -63.443634033203125,
      "activations/layer6_attention_weight_max": 44.761253356933594,
      "activations/layer6_attention_weight_min": -48.621315002441406,
      "activations/layer7_attention_weight_max": 98.8232192993164,
      "activations/layer7_attention_weight_min": -99.86823272705078,
      "activations/layer8_attention_weight_max": 44.35445022583008,
      "activations/layer8_attention_weight_min": -45.66783905029297,
      "activations/layer9_attention_weight_max": 33.8516960144043,
      "activations/layer9_attention_weight_min": -34.873565673828125,
      "epoch": 20.49,
      "learning_rate": 1.80405303030303e-05,
      "loss": 2.7319,
      "step": 352650
    },
    {
      "activations/layer0_attention_weight_max": 15.69865894317627,
      "activations/layer0_attention_weight_min": -14.983491897583008,
      "activations/layer10_attention_weight_max": 33.536865234375,
      "activations/layer10_attention_weight_min": -33.838722229003906,
      "activations/layer11_attention_weight_max": 35.61695098876953,
      "activations/layer11_attention_weight_min": -34.987213134765625,
      "activations/layer12_attention_weight_max": 26.996150970458984,
      "activations/layer12_attention_weight_min": -27.671993255615234,
      "activations/layer13_attention_weight_max": 41.03074264526367,
      "activations/layer13_attention_weight_min": -36.23586654663086,
      "activations/layer14_attention_weight_max": 44.846900939941406,
      "activations/layer14_attention_weight_min": -40.63815689086914,
      "activations/layer15_attention_weight_max": 42.89239501953125,
      "activations/layer15_attention_weight_min": -37.83409881591797,
      "activations/layer16_attention_weight_max": 29.441701889038086,
      "activations/layer16_attention_weight_min": -28.570188522338867,
      "activations/layer17_attention_weight_max": 30.16349983215332,
      "activations/layer17_attention_weight_min": -25.6403865814209,
      "activations/layer18_attention_weight_max": 30.48554039001465,
      "activations/layer18_attention_weight_min": -26.055330276489258,
      "activations/layer19_attention_weight_max": 36.77853775024414,
      "activations/layer19_attention_weight_min": -31.167966842651367,
      "activations/layer1_attention_weight_max": 15.1054105758667,
      "activations/layer1_attention_weight_min": -13.898295402526855,
      "activations/layer20_attention_weight_max": 32.36710739135742,
      "activations/layer20_attention_weight_min": -25.70388412475586,
      "activations/layer21_attention_weight_max": 30.60020637512207,
      "activations/layer21_attention_weight_min": -24.77876853942871,
      "activations/layer22_attention_weight_max": 39.74903106689453,
      "activations/layer22_attention_weight_min": -33.33159255981445,
      "activations/layer23_attention_weight_max": 34.057804107666016,
      "activations/layer23_attention_weight_min": -25.28302764892578,
      "activations/layer2_attention_weight_max": 32.57015609741211,
      "activations/layer2_attention_weight_min": -31.42506980895996,
      "activations/layer3_attention_weight_max": 107.61144256591797,
      "activations/layer3_attention_weight_min": -103.98058319091797,
      "activations/layer4_attention_weight_max": 112.84252166748047,
      "activations/layer4_attention_weight_min": -109.75272369384766,
      "activations/layer5_attention_weight_max": 52.65283966064453,
      "activations/layer5_attention_weight_min": -63.967323303222656,
      "activations/layer6_attention_weight_max": 46.39326477050781,
      "activations/layer6_attention_weight_min": -48.035404205322266,
      "activations/layer7_attention_weight_max": 97.81652069091797,
      "activations/layer7_attention_weight_min": -101.17232513427734,
      "activations/layer8_attention_weight_max": 42.29341506958008,
      "activations/layer8_attention_weight_min": -47.045013427734375,
      "activations/layer9_attention_weight_max": 31.511886596679688,
      "activations/layer9_attention_weight_min": -34.89064407348633,
      "epoch": 20.49,
      "learning_rate": 1.8021590909090907e-05,
      "loss": 2.7067,
      "step": 352700
    },
    {
      "activations/layer0_attention_weight_max": 16.45426368713379,
      "activations/layer0_attention_weight_min": -14.905895233154297,
      "activations/layer10_attention_weight_max": 35.933868408203125,
      "activations/layer10_attention_weight_min": -34.63923645019531,
      "activations/layer11_attention_weight_max": 35.82788848876953,
      "activations/layer11_attention_weight_min": -35.93070983886719,
      "activations/layer12_attention_weight_max": 24.321250915527344,
      "activations/layer12_attention_weight_min": -26.577381134033203,
      "activations/layer13_attention_weight_max": 38.307777404785156,
      "activations/layer13_attention_weight_min": -35.29353332519531,
      "activations/layer14_attention_weight_max": 40.279197692871094,
      "activations/layer14_attention_weight_min": -37.902156829833984,
      "activations/layer15_attention_weight_max": 38.960609436035156,
      "activations/layer15_attention_weight_min": -35.016334533691406,
      "activations/layer16_attention_weight_max": 28.430585861206055,
      "activations/layer16_attention_weight_min": -28.196693420410156,
      "activations/layer17_attention_weight_max": 29.51311683654785,
      "activations/layer17_attention_weight_min": -27.16864585876465,
      "activations/layer18_attention_weight_max": 29.5064640045166,
      "activations/layer18_attention_weight_min": -25.030603408813477,
      "activations/layer19_attention_weight_max": 34.93779754638672,
      "activations/layer19_attention_weight_min": -32.70793914794922,
      "activations/layer1_attention_weight_max": 15.895105361938477,
      "activations/layer1_attention_weight_min": -13.707488059997559,
      "activations/layer20_attention_weight_max": 29.745763778686523,
      "activations/layer20_attention_weight_min": -25.988941192626953,
      "activations/layer21_attention_weight_max": 29.108726501464844,
      "activations/layer21_attention_weight_min": -26.424644470214844,
      "activations/layer22_attention_weight_max": 37.51105499267578,
      "activations/layer22_attention_weight_min": -30.269962310791016,
      "activations/layer23_attention_weight_max": 29.07919692993164,
      "activations/layer23_attention_weight_min": -24.850553512573242,
      "activations/layer2_attention_weight_max": 33.25712966918945,
      "activations/layer2_attention_weight_min": -30.780372619628906,
      "activations/layer3_attention_weight_max": 103.32588195800781,
      "activations/layer3_attention_weight_min": -102.98738861083984,
      "activations/layer4_attention_weight_max": 109.54713439941406,
      "activations/layer4_attention_weight_min": -107.15986633300781,
      "activations/layer5_attention_weight_max": 54.594146728515625,
      "activations/layer5_attention_weight_min": -59.74059295654297,
      "activations/layer6_attention_weight_max": 45.119651794433594,
      "activations/layer6_attention_weight_min": -45.997982025146484,
      "activations/layer7_attention_weight_max": 99.96630096435547,
      "activations/layer7_attention_weight_min": -94.92566680908203,
      "activations/layer8_attention_weight_max": 44.65203094482422,
      "activations/layer8_attention_weight_min": -43.654415130615234,
      "activations/layer9_attention_weight_max": 32.98685073852539,
      "activations/layer9_attention_weight_min": -37.675106048583984,
      "epoch": 20.5,
      "learning_rate": 1.8002651515151516e-05,
      "loss": 2.7217,
      "step": 352750
    },
    {
      "activations/layer0_attention_weight_max": 16.52151107788086,
      "activations/layer0_attention_weight_min": -14.105881690979004,
      "activations/layer10_attention_weight_max": 33.044403076171875,
      "activations/layer10_attention_weight_min": -35.057186126708984,
      "activations/layer11_attention_weight_max": 32.115325927734375,
      "activations/layer11_attention_weight_min": -34.44407653808594,
      "activations/layer12_attention_weight_max": 25.914064407348633,
      "activations/layer12_attention_weight_min": -26.22223663330078,
      "activations/layer13_attention_weight_max": 38.84483337402344,
      "activations/layer13_attention_weight_min": -34.68228530883789,
      "activations/layer14_attention_weight_max": 43.17974853515625,
      "activations/layer14_attention_weight_min": -37.90745544433594,
      "activations/layer15_attention_weight_max": 42.37471389770508,
      "activations/layer15_attention_weight_min": -37.134124755859375,
      "activations/layer16_attention_weight_max": 29.583782196044922,
      "activations/layer16_attention_weight_min": -28.274843215942383,
      "activations/layer17_attention_weight_max": 28.629127502441406,
      "activations/layer17_attention_weight_min": -27.63837242126465,
      "activations/layer18_attention_weight_max": 31.856176376342773,
      "activations/layer18_attention_weight_min": -28.050024032592773,
      "activations/layer19_attention_weight_max": 34.022342681884766,
      "activations/layer19_attention_weight_min": -33.37412643432617,
      "activations/layer1_attention_weight_max": 15.620292663574219,
      "activations/layer1_attention_weight_min": -14.869080543518066,
      "activations/layer20_attention_weight_max": 27.748395919799805,
      "activations/layer20_attention_weight_min": -26.73052215576172,
      "activations/layer21_attention_weight_max": 29.361106872558594,
      "activations/layer21_attention_weight_min": -26.070556640625,
      "activations/layer22_attention_weight_max": 48.75691223144531,
      "activations/layer22_attention_weight_min": -32.77232360839844,
      "activations/layer23_attention_weight_max": 36.06001281738281,
      "activations/layer23_attention_weight_min": -25.62777328491211,
      "activations/layer2_attention_weight_max": 30.724403381347656,
      "activations/layer2_attention_weight_min": -31.89548110961914,
      "activations/layer3_attention_weight_max": 97.7697525024414,
      "activations/layer3_attention_weight_min": -100.47201538085938,
      "activations/layer4_attention_weight_max": 99.9021224975586,
      "activations/layer4_attention_weight_min": -106.41828918457031,
      "activations/layer5_attention_weight_max": 48.69349670410156,
      "activations/layer5_attention_weight_min": -61.408294677734375,
      "activations/layer6_attention_weight_max": 41.787715911865234,
      "activations/layer6_attention_weight_min": -47.05144500732422,
      "activations/layer7_attention_weight_max": 91.30184173583984,
      "activations/layer7_attention_weight_min": -98.61776733398438,
      "activations/layer8_attention_weight_max": 41.39802932739258,
      "activations/layer8_attention_weight_min": -43.994178771972656,
      "activations/layer9_attention_weight_max": 30.289182662963867,
      "activations/layer9_attention_weight_min": -33.02238464355469,
      "epoch": 20.5,
      "learning_rate": 1.798371212121212e-05,
      "loss": 2.7253,
      "step": 352800
    },
    {
      "activations/layer0_attention_weight_max": 16.423622131347656,
      "activations/layer0_attention_weight_min": -13.213829040527344,
      "activations/layer10_attention_weight_max": 35.57814407348633,
      "activations/layer10_attention_weight_min": -36.08742141723633,
      "activations/layer11_attention_weight_max": 35.95709991455078,
      "activations/layer11_attention_weight_min": -36.49745559692383,
      "activations/layer12_attention_weight_max": 28.094316482543945,
      "activations/layer12_attention_weight_min": -26.500593185424805,
      "activations/layer13_attention_weight_max": 37.84511184692383,
      "activations/layer13_attention_weight_min": -34.935340881347656,
      "activations/layer14_attention_weight_max": 42.33013153076172,
      "activations/layer14_attention_weight_min": -38.30982971191406,
      "activations/layer15_attention_weight_max": 40.00163269042969,
      "activations/layer15_attention_weight_min": -36.70120620727539,
      "activations/layer16_attention_weight_max": 27.527555465698242,
      "activations/layer16_attention_weight_min": -28.05116081237793,
      "activations/layer17_attention_weight_max": 27.798751831054688,
      "activations/layer17_attention_weight_min": -25.231191635131836,
      "activations/layer18_attention_weight_max": 29.676414489746094,
      "activations/layer18_attention_weight_min": -24.04936408996582,
      "activations/layer19_attention_weight_max": 34.097198486328125,
      "activations/layer19_attention_weight_min": -29.45716667175293,
      "activations/layer1_attention_weight_max": 16.12241554260254,
      "activations/layer1_attention_weight_min": -14.545202255249023,
      "activations/layer20_attention_weight_max": 28.42986297607422,
      "activations/layer20_attention_weight_min": -23.420291900634766,
      "activations/layer21_attention_weight_max": 28.71493148803711,
      "activations/layer21_attention_weight_min": -23.57941246032715,
      "activations/layer22_attention_weight_max": 40.1572265625,
      "activations/layer22_attention_weight_min": -29.85118865966797,
      "activations/layer23_attention_weight_max": 31.97821807861328,
      "activations/layer23_attention_weight_min": -23.807201385498047,
      "activations/layer2_attention_weight_max": 32.369903564453125,
      "activations/layer2_attention_weight_min": -31.68398666381836,
      "activations/layer3_attention_weight_max": 102.37336730957031,
      "activations/layer3_attention_weight_min": -105.423828125,
      "activations/layer4_attention_weight_max": 105.61856842041016,
      "activations/layer4_attention_weight_min": -108.97566986083984,
      "activations/layer5_attention_weight_max": 50.284759521484375,
      "activations/layer5_attention_weight_min": -64.38095092773438,
      "activations/layer6_attention_weight_max": 45.661861419677734,
      "activations/layer6_attention_weight_min": -48.62718963623047,
      "activations/layer7_attention_weight_max": 94.36151885986328,
      "activations/layer7_attention_weight_min": -97.38614654541016,
      "activations/layer8_attention_weight_max": 44.868438720703125,
      "activations/layer8_attention_weight_min": -45.8741455078125,
      "activations/layer9_attention_weight_max": 33.09212112426758,
      "activations/layer9_attention_weight_min": -36.08582305908203,
      "epoch": 20.5,
      "learning_rate": 1.7964772727272725e-05,
      "loss": 2.7359,
      "step": 352850
    },
    {
      "activations/layer0_attention_weight_max": 16.17251968383789,
      "activations/layer0_attention_weight_min": -15.92125129699707,
      "activations/layer10_attention_weight_max": 34.644073486328125,
      "activations/layer10_attention_weight_min": -34.06549835205078,
      "activations/layer11_attention_weight_max": 34.342037200927734,
      "activations/layer11_attention_weight_min": -34.9384765625,
      "activations/layer12_attention_weight_max": 28.520198822021484,
      "activations/layer12_attention_weight_min": -26.332481384277344,
      "activations/layer13_attention_weight_max": 40.7640495300293,
      "activations/layer13_attention_weight_min": -34.83835983276367,
      "activations/layer14_attention_weight_max": 52.606163024902344,
      "activations/layer14_attention_weight_min": -38.639915466308594,
      "activations/layer15_attention_weight_max": 41.63413619995117,
      "activations/layer15_attention_weight_min": -38.58291244506836,
      "activations/layer16_attention_weight_max": 35.6507453918457,
      "activations/layer16_attention_weight_min": -29.15380859375,
      "activations/layer17_attention_weight_max": 29.352657318115234,
      "activations/layer17_attention_weight_min": -26.338546752929688,
      "activations/layer18_attention_weight_max": 32.72560501098633,
      "activations/layer18_attention_weight_min": -25.622180938720703,
      "activations/layer19_attention_weight_max": 36.267189025878906,
      "activations/layer19_attention_weight_min": -32.9681396484375,
      "activations/layer1_attention_weight_max": 16.04323959350586,
      "activations/layer1_attention_weight_min": -16.4136962890625,
      "activations/layer20_attention_weight_max": 29.632877349853516,
      "activations/layer20_attention_weight_min": -26.471105575561523,
      "activations/layer21_attention_weight_max": 27.316848754882812,
      "activations/layer21_attention_weight_min": -25.02311134338379,
      "activations/layer22_attention_weight_max": 46.179019927978516,
      "activations/layer22_attention_weight_min": -32.663516998291016,
      "activations/layer23_attention_weight_max": 35.07096862792969,
      "activations/layer23_attention_weight_min": -27.276762008666992,
      "activations/layer2_attention_weight_max": 33.75428771972656,
      "activations/layer2_attention_weight_min": -32.01734924316406,
      "activations/layer3_attention_weight_max": 103.31249237060547,
      "activations/layer3_attention_weight_min": -104.89376831054688,
      "activations/layer4_attention_weight_max": 104.04271697998047,
      "activations/layer4_attention_weight_min": -111.3389663696289,
      "activations/layer5_attention_weight_max": 50.5578727722168,
      "activations/layer5_attention_weight_min": -61.11444854736328,
      "activations/layer6_attention_weight_max": 45.302120208740234,
      "activations/layer6_attention_weight_min": -46.54210662841797,
      "activations/layer7_attention_weight_max": 93.12653350830078,
      "activations/layer7_attention_weight_min": -97.44012451171875,
      "activations/layer8_attention_weight_max": 45.352378845214844,
      "activations/layer8_attention_weight_min": -45.153568267822266,
      "activations/layer9_attention_weight_max": 32.08061218261719,
      "activations/layer9_attention_weight_min": -33.88942337036133,
      "epoch": 20.51,
      "learning_rate": 1.7945833333333334e-05,
      "loss": 2.717,
      "step": 352900
    },
    {
      "activations/layer0_attention_weight_max": 17.710693359375,
      "activations/layer0_attention_weight_min": -13.317557334899902,
      "activations/layer10_attention_weight_max": 34.20534133911133,
      "activations/layer10_attention_weight_min": -35.810245513916016,
      "activations/layer11_attention_weight_max": 32.664581298828125,
      "activations/layer11_attention_weight_min": -35.61621856689453,
      "activations/layer12_attention_weight_max": 26.76879119873047,
      "activations/layer12_attention_weight_min": -31.936702728271484,
      "activations/layer13_attention_weight_max": 37.700035095214844,
      "activations/layer13_attention_weight_min": -36.150997161865234,
      "activations/layer14_attention_weight_max": 39.33997344970703,
      "activations/layer14_attention_weight_min": -38.67884826660156,
      "activations/layer15_attention_weight_max": 36.154850006103516,
      "activations/layer15_attention_weight_min": -39.503662109375,
      "activations/layer16_attention_weight_max": 28.94282341003418,
      "activations/layer16_attention_weight_min": -28.2099666595459,
      "activations/layer17_attention_weight_max": 30.79525375366211,
      "activations/layer17_attention_weight_min": -24.900484085083008,
      "activations/layer18_attention_weight_max": 35.4296989440918,
      "activations/layer18_attention_weight_min": -25.572757720947266,
      "activations/layer19_attention_weight_max": 37.56484603881836,
      "activations/layer19_attention_weight_min": -31.81150245666504,
      "activations/layer1_attention_weight_max": 15.838079452514648,
      "activations/layer1_attention_weight_min": -15.769739151000977,
      "activations/layer20_attention_weight_max": 29.156936645507812,
      "activations/layer20_attention_weight_min": -25.696699142456055,
      "activations/layer21_attention_weight_max": 29.71630859375,
      "activations/layer21_attention_weight_min": -25.338573455810547,
      "activations/layer22_attention_weight_max": 50.51996612548828,
      "activations/layer22_attention_weight_min": -33.3447151184082,
      "activations/layer23_attention_weight_max": 39.10832977294922,
      "activations/layer23_attention_weight_min": -27.17174530029297,
      "activations/layer2_attention_weight_max": 33.369178771972656,
      "activations/layer2_attention_weight_min": -33.641639709472656,
      "activations/layer3_attention_weight_max": 104.11898803710938,
      "activations/layer3_attention_weight_min": -109.27408599853516,
      "activations/layer4_attention_weight_max": 105.58546447753906,
      "activations/layer4_attention_weight_min": -108.9601821899414,
      "activations/layer5_attention_weight_max": 49.5312614440918,
      "activations/layer5_attention_weight_min": -60.051307678222656,
      "activations/layer6_attention_weight_max": 42.94217300415039,
      "activations/layer6_attention_weight_min": -47.264869689941406,
      "activations/layer7_attention_weight_max": 101.0592269897461,
      "activations/layer7_attention_weight_min": -96.99197387695312,
      "activations/layer8_attention_weight_max": 42.39683151245117,
      "activations/layer8_attention_weight_min": -43.50159454345703,
      "activations/layer9_attention_weight_max": 30.394607543945312,
      "activations/layer9_attention_weight_min": -34.730533599853516,
      "epoch": 20.51,
      "learning_rate": 1.792689393939394e-05,
      "loss": 2.7155,
      "step": 352950
    },
    {
      "activations/layer0_attention_weight_max": 16.22201156616211,
      "activations/layer0_attention_weight_min": -14.555251121520996,
      "activations/layer10_attention_weight_max": 31.42868995666504,
      "activations/layer10_attention_weight_min": -33.50961685180664,
      "activations/layer11_attention_weight_max": 33.9737548828125,
      "activations/layer11_attention_weight_min": -33.20084762573242,
      "activations/layer12_attention_weight_max": 26.030799865722656,
      "activations/layer12_attention_weight_min": -25.527835845947266,
      "activations/layer13_attention_weight_max": 37.45559310913086,
      "activations/layer13_attention_weight_min": -34.36428451538086,
      "activations/layer14_attention_weight_max": 40.77180480957031,
      "activations/layer14_attention_weight_min": -36.98172378540039,
      "activations/layer15_attention_weight_max": 36.19282531738281,
      "activations/layer15_attention_weight_min": -36.96100997924805,
      "activations/layer16_attention_weight_max": 30.667863845825195,
      "activations/layer16_attention_weight_min": -30.120738983154297,
      "activations/layer17_attention_weight_max": 29.92759895324707,
      "activations/layer17_attention_weight_min": -26.35538101196289,
      "activations/layer18_attention_weight_max": 35.324195861816406,
      "activations/layer18_attention_weight_min": -26.794816970825195,
      "activations/layer19_attention_weight_max": 34.13599395751953,
      "activations/layer19_attention_weight_min": -31.573984146118164,
      "activations/layer1_attention_weight_max": 15.37158489227295,
      "activations/layer1_attention_weight_min": -14.490422248840332,
      "activations/layer20_attention_weight_max": 28.740581512451172,
      "activations/layer20_attention_weight_min": -24.147842407226562,
      "activations/layer21_attention_weight_max": 26.973257064819336,
      "activations/layer21_attention_weight_min": -23.768394470214844,
      "activations/layer22_attention_weight_max": 43.35987854003906,
      "activations/layer22_attention_weight_min": -31.455936431884766,
      "activations/layer23_attention_weight_max": 34.85691833496094,
      "activations/layer23_attention_weight_min": -25.862043380737305,
      "activations/layer2_attention_weight_max": 32.09059524536133,
      "activations/layer2_attention_weight_min": -30.472293853759766,
      "activations/layer3_attention_weight_max": 97.94788360595703,
      "activations/layer3_attention_weight_min": -100.05613708496094,
      "activations/layer4_attention_weight_max": 98.13599395751953,
      "activations/layer4_attention_weight_min": -104.7532730102539,
      "activations/layer5_attention_weight_max": 49.48493957519531,
      "activations/layer5_attention_weight_min": -64.17554473876953,
      "activations/layer6_attention_weight_max": 41.72020721435547,
      "activations/layer6_attention_weight_min": -46.54690933227539,
      "activations/layer7_attention_weight_max": 92.64454650878906,
      "activations/layer7_attention_weight_min": -103.801513671875,
      "activations/layer8_attention_weight_max": 40.36997985839844,
      "activations/layer8_attention_weight_min": -45.570125579833984,
      "activations/layer9_attention_weight_max": 30.351810455322266,
      "activations/layer9_attention_weight_min": -32.618892669677734,
      "epoch": 20.51,
      "learning_rate": 1.7907954545454543e-05,
      "loss": 2.7085,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.5184,
      "eval_samples_per_second": 504.085,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.5184,
      "eval_openwebtext_samples_per_second": 504.085,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9517,
      "eval_wikitext_samples_per_second": 233.637,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5806,
      "eval_lambada_samples_per_second": 508.214,
      "step": 353000
    },
    {
      "activations/layer0_attention_weight_max": 16.870548248291016,
      "activations/layer0_attention_weight_min": -12.773736000061035,
      "activations/layer10_attention_weight_max": 42.242855072021484,
      "activations/layer10_attention_weight_min": -41.436153411865234,
      "activations/layer11_attention_weight_max": 40.02637481689453,
      "activations/layer11_attention_weight_min": -41.117244720458984,
      "activations/layer12_attention_weight_max": 29.87874984741211,
      "activations/layer12_attention_weight_min": -27.524961471557617,
      "activations/layer13_attention_weight_max": 43.5993537902832,
      "activations/layer13_attention_weight_min": -37.6036376953125,
      "activations/layer14_attention_weight_max": 46.70916748046875,
      "activations/layer14_attention_weight_min": -39.9903678894043,
      "activations/layer15_attention_weight_max": 47.38003158569336,
      "activations/layer15_attention_weight_min": -42.672122955322266,
      "activations/layer16_attention_weight_max": 34.604305267333984,
      "activations/layer16_attention_weight_min": -29.078811645507812,
      "activations/layer17_attention_weight_max": 32.04808044433594,
      "activations/layer17_attention_weight_min": -26.530269622802734,
      "activations/layer18_attention_weight_max": 35.119171142578125,
      "activations/layer18_attention_weight_min": -28.030208587646484,
      "activations/layer19_attention_weight_max": 43.566410064697266,
      "activations/layer19_attention_weight_min": -33.76028060913086,
      "activations/layer1_attention_weight_max": 15.349976539611816,
      "activations/layer1_attention_weight_min": -14.11478328704834,
      "activations/layer20_attention_weight_max": 33.86026382446289,
      "activations/layer20_attention_weight_min": -26.437450408935547,
      "activations/layer21_attention_weight_max": 33.02362823486328,
      "activations/layer21_attention_weight_min": -26.199148178100586,
      "activations/layer22_attention_weight_max": 50.93765640258789,
      "activations/layer22_attention_weight_min": -34.5633544921875,
      "activations/layer23_attention_weight_max": 36.777976989746094,
      "activations/layer23_attention_weight_min": -31.14729881286621,
      "activations/layer2_attention_weight_max": 33.97726058959961,
      "activations/layer2_attention_weight_min": -32.648685455322266,
      "activations/layer3_attention_weight_max": 106.47569274902344,
      "activations/layer3_attention_weight_min": -105.85697174072266,
      "activations/layer4_attention_weight_max": 113.59066009521484,
      "activations/layer4_attention_weight_min": -110.08953094482422,
      "activations/layer5_attention_weight_max": 54.44233703613281,
      "activations/layer5_attention_weight_min": -61.2880859375,
      "activations/layer6_attention_weight_max": 50.726009368896484,
      "activations/layer6_attention_weight_min": -47.2747917175293,
      "activations/layer7_attention_weight_max": 107.73458862304688,
      "activations/layer7_attention_weight_min": -101.42922973632812,
      "activations/layer8_attention_weight_max": 48.72525405883789,
      "activations/layer8_attention_weight_min": -49.43223190307617,
      "activations/layer9_attention_weight_max": 36.372886657714844,
      "activations/layer9_attention_weight_min": -41.049320220947266,
      "epoch": 20.51,
      "learning_rate": 1.788901515151515e-05,
      "loss": 2.7074,
      "step": 353050
    },
    {
      "activations/layer0_attention_weight_max": 16.45967674255371,
      "activations/layer0_attention_weight_min": -13.86927318572998,
      "activations/layer10_attention_weight_max": 34.26204299926758,
      "activations/layer10_attention_weight_min": -34.94474411010742,
      "activations/layer11_attention_weight_max": 33.94982147216797,
      "activations/layer11_attention_weight_min": -36.85735321044922,
      "activations/layer12_attention_weight_max": 25.4417781829834,
      "activations/layer12_attention_weight_min": -26.121557235717773,
      "activations/layer13_attention_weight_max": 36.274932861328125,
      "activations/layer13_attention_weight_min": -34.43231964111328,
      "activations/layer14_attention_weight_max": 39.79619216918945,
      "activations/layer14_attention_weight_min": -36.93022918701172,
      "activations/layer15_attention_weight_max": 35.13847732543945,
      "activations/layer15_attention_weight_min": -34.75250244140625,
      "activations/layer16_attention_weight_max": 29.077762603759766,
      "activations/layer16_attention_weight_min": -27.800338745117188,
      "activations/layer17_attention_weight_max": 29.160669326782227,
      "activations/layer17_attention_weight_min": -25.629472732543945,
      "activations/layer18_attention_weight_max": 31.278427124023438,
      "activations/layer18_attention_weight_min": -26.28770637512207,
      "activations/layer19_attention_weight_max": 32.137908935546875,
      "activations/layer19_attention_weight_min": -32.382896423339844,
      "activations/layer1_attention_weight_max": 16.266328811645508,
      "activations/layer1_attention_weight_min": -15.170412063598633,
      "activations/layer20_attention_weight_max": 26.720584869384766,
      "activations/layer20_attention_weight_min": -25.910367965698242,
      "activations/layer21_attention_weight_max": 28.904211044311523,
      "activations/layer21_attention_weight_min": -26.248172760009766,
      "activations/layer22_attention_weight_max": 37.287757873535156,
      "activations/layer22_attention_weight_min": -33.055782318115234,
      "activations/layer23_attention_weight_max": 29.244754791259766,
      "activations/layer23_attention_weight_min": -27.870365142822266,
      "activations/layer2_attention_weight_max": 33.692752838134766,
      "activations/layer2_attention_weight_min": -31.71402359008789,
      "activations/layer3_attention_weight_max": 104.75865173339844,
      "activations/layer3_attention_weight_min": -106.06665802001953,
      "activations/layer4_attention_weight_max": 108.1043701171875,
      "activations/layer4_attention_weight_min": -114.40288543701172,
      "activations/layer5_attention_weight_max": 50.54075622558594,
      "activations/layer5_attention_weight_min": -59.246742248535156,
      "activations/layer6_attention_weight_max": 44.56952667236328,
      "activations/layer6_attention_weight_min": -47.887779235839844,
      "activations/layer7_attention_weight_max": 92.55278778076172,
      "activations/layer7_attention_weight_min": -97.53082275390625,
      "activations/layer8_attention_weight_max": 41.93294143676758,
      "activations/layer8_attention_weight_min": -45.32762908935547,
      "activations/layer9_attention_weight_max": 30.694278717041016,
      "activations/layer9_attention_weight_min": -34.161720275878906,
      "epoch": 20.52,
      "learning_rate": 1.7870075757575757e-05,
      "loss": 2.7041,
      "step": 353100
    },
    {
      "activations/layer0_attention_weight_max": 17.2536563873291,
      "activations/layer0_attention_weight_min": -13.76425552368164,
      "activations/layer10_attention_weight_max": 32.61818313598633,
      "activations/layer10_attention_weight_min": -34.179229736328125,
      "activations/layer11_attention_weight_max": 32.69135665893555,
      "activations/layer11_attention_weight_min": -33.16796875,
      "activations/layer12_attention_weight_max": 26.14022445678711,
      "activations/layer12_attention_weight_min": -26.169652938842773,
      "activations/layer13_attention_weight_max": 39.210079193115234,
      "activations/layer13_attention_weight_min": -38.226905822753906,
      "activations/layer14_attention_weight_max": 41.20009231567383,
      "activations/layer14_attention_weight_min": -39.67048645019531,
      "activations/layer15_attention_weight_max": 37.740684509277344,
      "activations/layer15_attention_weight_min": -36.765838623046875,
      "activations/layer16_attention_weight_max": 27.52474594116211,
      "activations/layer16_attention_weight_min": -27.81599998474121,
      "activations/layer17_attention_weight_max": 30.132150650024414,
      "activations/layer17_attention_weight_min": -26.76790428161621,
      "activations/layer18_attention_weight_max": 32.536319732666016,
      "activations/layer18_attention_weight_min": -26.843271255493164,
      "activations/layer19_attention_weight_max": 33.09843444824219,
      "activations/layer19_attention_weight_min": -32.47385787963867,
      "activations/layer1_attention_weight_max": 16.187503814697266,
      "activations/layer1_attention_weight_min": -14.647490501403809,
      "activations/layer20_attention_weight_max": 27.644134521484375,
      "activations/layer20_attention_weight_min": -24.468154907226562,
      "activations/layer21_attention_weight_max": 28.192399978637695,
      "activations/layer21_attention_weight_min": -23.64798355102539,
      "activations/layer22_attention_weight_max": 40.20412063598633,
      "activations/layer22_attention_weight_min": -29.786680221557617,
      "activations/layer23_attention_weight_max": 30.554861068725586,
      "activations/layer23_attention_weight_min": -24.44727325439453,
      "activations/layer2_attention_weight_max": 32.742034912109375,
      "activations/layer2_attention_weight_min": -31.901878356933594,
      "activations/layer3_attention_weight_max": 102.66270446777344,
      "activations/layer3_attention_weight_min": -103.96978759765625,
      "activations/layer4_attention_weight_max": 104.5818099975586,
      "activations/layer4_attention_weight_min": -109.03265380859375,
      "activations/layer5_attention_weight_max": 47.551422119140625,
      "activations/layer5_attention_weight_min": -61.65980529785156,
      "activations/layer6_attention_weight_max": 44.27254104614258,
      "activations/layer6_attention_weight_min": -48.21297073364258,
      "activations/layer7_attention_weight_max": 95.84854125976562,
      "activations/layer7_attention_weight_min": -96.79110717773438,
      "activations/layer8_attention_weight_max": 43.505924224853516,
      "activations/layer8_attention_weight_min": -45.97455596923828,
      "activations/layer9_attention_weight_max": 31.820539474487305,
      "activations/layer9_attention_weight_min": -32.967227935791016,
      "epoch": 20.52,
      "learning_rate": 1.785113636363636e-05,
      "loss": 2.7249,
      "step": 353150
    },
    {
      "activations/layer0_attention_weight_max": 17.90265464782715,
      "activations/layer0_attention_weight_min": -13.101696968078613,
      "activations/layer10_attention_weight_max": 34.09871292114258,
      "activations/layer10_attention_weight_min": -36.32147216796875,
      "activations/layer11_attention_weight_max": 35.76629638671875,
      "activations/layer11_attention_weight_min": -36.63789367675781,
      "activations/layer12_attention_weight_max": 26.489322662353516,
      "activations/layer12_attention_weight_min": -26.762269973754883,
      "activations/layer13_attention_weight_max": 39.68819808959961,
      "activations/layer13_attention_weight_min": -37.60833740234375,
      "activations/layer14_attention_weight_max": 40.80033493041992,
      "activations/layer14_attention_weight_min": -38.577571868896484,
      "activations/layer15_attention_weight_max": 39.35309600830078,
      "activations/layer15_attention_weight_min": -36.955509185791016,
      "activations/layer16_attention_weight_max": 29.824071884155273,
      "activations/layer16_attention_weight_min": -28.08643341064453,
      "activations/layer17_attention_weight_max": 27.849206924438477,
      "activations/layer17_attention_weight_min": -27.50075340270996,
      "activations/layer18_attention_weight_max": 33.597660064697266,
      "activations/layer18_attention_weight_min": -26.478328704833984,
      "activations/layer19_attention_weight_max": 31.097335815429688,
      "activations/layer19_attention_weight_min": -30.562824249267578,
      "activations/layer1_attention_weight_max": 15.693258285522461,
      "activations/layer1_attention_weight_min": -15.329627990722656,
      "activations/layer20_attention_weight_max": 27.046142578125,
      "activations/layer20_attention_weight_min": -24.261486053466797,
      "activations/layer21_attention_weight_max": 28.22714614868164,
      "activations/layer21_attention_weight_min": -23.438125610351562,
      "activations/layer22_attention_weight_max": 47.50572204589844,
      "activations/layer22_attention_weight_min": -31.103975296020508,
      "activations/layer23_attention_weight_max": 32.60344314575195,
      "activations/layer23_attention_weight_min": -26.454404830932617,
      "activations/layer2_attention_weight_max": 34.53620910644531,
      "activations/layer2_attention_weight_min": -32.99687194824219,
      "activations/layer3_attention_weight_max": 102.20230865478516,
      "activations/layer3_attention_weight_min": -103.9164810180664,
      "activations/layer4_attention_weight_max": 106.31463623046875,
      "activations/layer4_attention_weight_min": -111.79449462890625,
      "activations/layer5_attention_weight_max": 49.70072555541992,
      "activations/layer5_attention_weight_min": -64.89823913574219,
      "activations/layer6_attention_weight_max": 45.49268341064453,
      "activations/layer6_attention_weight_min": -48.6445426940918,
      "activations/layer7_attention_weight_max": 93.488525390625,
      "activations/layer7_attention_weight_min": -95.11483001708984,
      "activations/layer8_attention_weight_max": 40.89564514160156,
      "activations/layer8_attention_weight_min": -42.844940185546875,
      "activations/layer9_attention_weight_max": 32.44245910644531,
      "activations/layer9_attention_weight_min": -36.1866569519043,
      "epoch": 20.52,
      "learning_rate": 1.7832196969696966e-05,
      "loss": 2.699,
      "step": 353200
    },
    {
      "activations/layer0_attention_weight_max": 16.039016723632812,
      "activations/layer0_attention_weight_min": -12.56344223022461,
      "activations/layer10_attention_weight_max": 32.42759323120117,
      "activations/layer10_attention_weight_min": -33.01994323730469,
      "activations/layer11_attention_weight_max": 34.58043670654297,
      "activations/layer11_attention_weight_min": -34.29411697387695,
      "activations/layer12_attention_weight_max": 26.621009826660156,
      "activations/layer12_attention_weight_min": -25.542232513427734,
      "activations/layer13_attention_weight_max": 40.6057014465332,
      "activations/layer13_attention_weight_min": -34.687217712402344,
      "activations/layer14_attention_weight_max": 44.18056869506836,
      "activations/layer14_attention_weight_min": -38.71489334106445,
      "activations/layer15_attention_weight_max": 41.6328239440918,
      "activations/layer15_attention_weight_min": -36.385009765625,
      "activations/layer16_attention_weight_max": 30.541040420532227,
      "activations/layer16_attention_weight_min": -27.072734832763672,
      "activations/layer17_attention_weight_max": 31.65448760986328,
      "activations/layer17_attention_weight_min": -26.90572166442871,
      "activations/layer18_attention_weight_max": 31.906108856201172,
      "activations/layer18_attention_weight_min": -27.073312759399414,
      "activations/layer19_attention_weight_max": 36.52486801147461,
      "activations/layer19_attention_weight_min": -30.956253051757812,
      "activations/layer1_attention_weight_max": 15.635076522827148,
      "activations/layer1_attention_weight_min": -15.033270835876465,
      "activations/layer20_attention_weight_max": 29.42434310913086,
      "activations/layer20_attention_weight_min": -25.854595184326172,
      "activations/layer21_attention_weight_max": 29.820920944213867,
      "activations/layer21_attention_weight_min": -26.916545867919922,
      "activations/layer22_attention_weight_max": 41.21427917480469,
      "activations/layer22_attention_weight_min": -31.77345085144043,
      "activations/layer23_attention_weight_max": 32.25543212890625,
      "activations/layer23_attention_weight_min": -27.05010986328125,
      "activations/layer2_attention_weight_max": 31.765470504760742,
      "activations/layer2_attention_weight_min": -31.445743560791016,
      "activations/layer3_attention_weight_max": 98.74067687988281,
      "activations/layer3_attention_weight_min": -101.59315490722656,
      "activations/layer4_attention_weight_max": 102.75487518310547,
      "activations/layer4_attention_weight_min": -109.20112609863281,
      "activations/layer5_attention_weight_max": 48.66300582885742,
      "activations/layer5_attention_weight_min": -61.24955368041992,
      "activations/layer6_attention_weight_max": 42.275230407714844,
      "activations/layer6_attention_weight_min": -47.178306579589844,
      "activations/layer7_attention_weight_max": 94.92584991455078,
      "activations/layer7_attention_weight_min": -96.31513214111328,
      "activations/layer8_attention_weight_max": 41.21231460571289,
      "activations/layer8_attention_weight_min": -41.96281814575195,
      "activations/layer9_attention_weight_max": 30.281970977783203,
      "activations/layer9_attention_weight_min": -32.56032943725586,
      "epoch": 20.53,
      "learning_rate": 1.7813257575757575e-05,
      "loss": 2.7085,
      "step": 353250
    },
    {
      "activations/layer0_attention_weight_max": 17.1208438873291,
      "activations/layer0_attention_weight_min": -14.412169456481934,
      "activations/layer10_attention_weight_max": 36.858070373535156,
      "activations/layer10_attention_weight_min": -39.01856994628906,
      "activations/layer11_attention_weight_max": 36.05641174316406,
      "activations/layer11_attention_weight_min": -36.51793670654297,
      "activations/layer12_attention_weight_max": 28.508798599243164,
      "activations/layer12_attention_weight_min": -27.379369735717773,
      "activations/layer13_attention_weight_max": 43.443885803222656,
      "activations/layer13_attention_weight_min": -37.1641845703125,
      "activations/layer14_attention_weight_max": 46.88542556762695,
      "activations/layer14_attention_weight_min": -43.02324295043945,
      "activations/layer15_attention_weight_max": 41.83839797973633,
      "activations/layer15_attention_weight_min": -38.313663482666016,
      "activations/layer16_attention_weight_max": 36.950050354003906,
      "activations/layer16_attention_weight_min": -29.498458862304688,
      "activations/layer17_attention_weight_max": 31.504615783691406,
      "activations/layer17_attention_weight_min": -28.475385665893555,
      "activations/layer18_attention_weight_max": 34.03368377685547,
      "activations/layer18_attention_weight_min": -29.873491287231445,
      "activations/layer19_attention_weight_max": 37.084651947021484,
      "activations/layer19_attention_weight_min": -32.297569274902344,
      "activations/layer1_attention_weight_max": 15.930549621582031,
      "activations/layer1_attention_weight_min": -14.76618480682373,
      "activations/layer20_attention_weight_max": 30.615921020507812,
      "activations/layer20_attention_weight_min": -26.193021774291992,
      "activations/layer21_attention_weight_max": 30.119585037231445,
      "activations/layer21_attention_weight_min": -25.726165771484375,
      "activations/layer22_attention_weight_max": 45.18582534790039,
      "activations/layer22_attention_weight_min": -33.25619888305664,
      "activations/layer23_attention_weight_max": 33.2227783203125,
      "activations/layer23_attention_weight_min": -25.763179779052734,
      "activations/layer2_attention_weight_max": 34.022193908691406,
      "activations/layer2_attention_weight_min": -31.445850372314453,
      "activations/layer3_attention_weight_max": 104.53871154785156,
      "activations/layer3_attention_weight_min": -103.69628143310547,
      "activations/layer4_attention_weight_max": 106.83109283447266,
      "activations/layer4_attention_weight_min": -108.2774887084961,
      "activations/layer5_attention_weight_max": 51.96171188354492,
      "activations/layer5_attention_weight_min": -65.04560852050781,
      "activations/layer6_attention_weight_max": 46.45200729370117,
      "activations/layer6_attention_weight_min": -47.77096176147461,
      "activations/layer7_attention_weight_max": 99.3368911743164,
      "activations/layer7_attention_weight_min": -109.46627044677734,
      "activations/layer8_attention_weight_max": 44.84526443481445,
      "activations/layer8_attention_weight_min": -48.63386535644531,
      "activations/layer9_attention_weight_max": 34.848052978515625,
      "activations/layer9_attention_weight_min": -37.643062591552734,
      "epoch": 20.53,
      "learning_rate": 1.779431818181818e-05,
      "loss": 2.699,
      "step": 353300
    },
    {
      "activations/layer0_attention_weight_max": 16.325668334960938,
      "activations/layer0_attention_weight_min": -13.35588550567627,
      "activations/layer10_attention_weight_max": 35.15583038330078,
      "activations/layer10_attention_weight_min": -35.187232971191406,
      "activations/layer11_attention_weight_max": 35.99449157714844,
      "activations/layer11_attention_weight_min": -38.12366485595703,
      "activations/layer12_attention_weight_max": 29.154565811157227,
      "activations/layer12_attention_weight_min": -28.629159927368164,
      "activations/layer13_attention_weight_max": 45.47126388549805,
      "activations/layer13_attention_weight_min": -36.07084655761719,
      "activations/layer14_attention_weight_max": 48.38749313354492,
      "activations/layer14_attention_weight_min": -39.2652702331543,
      "activations/layer15_attention_weight_max": 48.682472229003906,
      "activations/layer15_attention_weight_min": -37.764122009277344,
      "activations/layer16_attention_weight_max": 35.34101867675781,
      "activations/layer16_attention_weight_min": -27.59764862060547,
      "activations/layer17_attention_weight_max": 34.24810028076172,
      "activations/layer17_attention_weight_min": -25.48442268371582,
      "activations/layer18_attention_weight_max": 33.99463653564453,
      "activations/layer18_attention_weight_min": -24.989728927612305,
      "activations/layer19_attention_weight_max": 36.915748596191406,
      "activations/layer19_attention_weight_min": -32.5296745300293,
      "activations/layer1_attention_weight_max": 15.713062286376953,
      "activations/layer1_attention_weight_min": -14.593612670898438,
      "activations/layer20_attention_weight_max": 30.910552978515625,
      "activations/layer20_attention_weight_min": -24.42790412902832,
      "activations/layer21_attention_weight_max": 29.44948387145996,
      "activations/layer21_attention_weight_min": -24.26785659790039,
      "activations/layer22_attention_weight_max": 42.224449157714844,
      "activations/layer22_attention_weight_min": -31.52458381652832,
      "activations/layer23_attention_weight_max": 33.24054718017578,
      "activations/layer23_attention_weight_min": -26.295856475830078,
      "activations/layer2_attention_weight_max": 33.78355407714844,
      "activations/layer2_attention_weight_min": -34.435035705566406,
      "activations/layer3_attention_weight_max": 107.30378723144531,
      "activations/layer3_attention_weight_min": -110.14520263671875,
      "activations/layer4_attention_weight_max": 110.7640609741211,
      "activations/layer4_attention_weight_min": -112.82582092285156,
      "activations/layer5_attention_weight_max": 52.52518844604492,
      "activations/layer5_attention_weight_min": -61.23181915283203,
      "activations/layer6_attention_weight_max": 47.412437438964844,
      "activations/layer6_attention_weight_min": -50.13905715942383,
      "activations/layer7_attention_weight_max": 98.66902923583984,
      "activations/layer7_attention_weight_min": -108.82730865478516,
      "activations/layer8_attention_weight_max": 44.86876678466797,
      "activations/layer8_attention_weight_min": -50.00442886352539,
      "activations/layer9_attention_weight_max": 34.11170196533203,
      "activations/layer9_attention_weight_min": -39.45011520385742,
      "epoch": 20.53,
      "learning_rate": 1.7775378787878784e-05,
      "loss": 2.7155,
      "step": 353350
    },
    {
      "activations/layer0_attention_weight_max": 16.523618698120117,
      "activations/layer0_attention_weight_min": -13.428720474243164,
      "activations/layer10_attention_weight_max": 34.03610610961914,
      "activations/layer10_attention_weight_min": -36.83570098876953,
      "activations/layer11_attention_weight_max": 33.261837005615234,
      "activations/layer11_attention_weight_min": -34.75969696044922,
      "activations/layer12_attention_weight_max": 26.823701858520508,
      "activations/layer12_attention_weight_min": -24.96318817138672,
      "activations/layer13_attention_weight_max": 39.11745071411133,
      "activations/layer13_attention_weight_min": -34.63200378417969,
      "activations/layer14_attention_weight_max": 39.98261642456055,
      "activations/layer14_attention_weight_min": -36.48588943481445,
      "activations/layer15_attention_weight_max": 38.0389289855957,
      "activations/layer15_attention_weight_min": -35.727840423583984,
      "activations/layer16_attention_weight_max": 29.048065185546875,
      "activations/layer16_attention_weight_min": -26.83100700378418,
      "activations/layer17_attention_weight_max": 29.244157791137695,
      "activations/layer17_attention_weight_min": -26.178050994873047,
      "activations/layer18_attention_weight_max": 31.020273208618164,
      "activations/layer18_attention_weight_min": -28.806072235107422,
      "activations/layer19_attention_weight_max": 31.247892379760742,
      "activations/layer19_attention_weight_min": -29.585901260375977,
      "activations/layer1_attention_weight_max": 15.362441062927246,
      "activations/layer1_attention_weight_min": -13.901880264282227,
      "activations/layer20_attention_weight_max": 26.084545135498047,
      "activations/layer20_attention_weight_min": -27.05229949951172,
      "activations/layer21_attention_weight_max": 27.596874237060547,
      "activations/layer21_attention_weight_min": -26.377811431884766,
      "activations/layer22_attention_weight_max": 37.634586334228516,
      "activations/layer22_attention_weight_min": -32.39801025390625,
      "activations/layer23_attention_weight_max": 29.690465927124023,
      "activations/layer23_attention_weight_min": -24.730182647705078,
      "activations/layer2_attention_weight_max": 33.06488037109375,
      "activations/layer2_attention_weight_min": -31.982465744018555,
      "activations/layer3_attention_weight_max": 103.27007293701172,
      "activations/layer3_attention_weight_min": -104.94922637939453,
      "activations/layer4_attention_weight_max": 107.5096435546875,
      "activations/layer4_attention_weight_min": -106.86494445800781,
      "activations/layer5_attention_weight_max": 50.81134033203125,
      "activations/layer5_attention_weight_min": -59.759666442871094,
      "activations/layer6_attention_weight_max": 45.342044830322266,
      "activations/layer6_attention_weight_min": -48.85053634643555,
      "activations/layer7_attention_weight_max": 93.87518310546875,
      "activations/layer7_attention_weight_min": -100.83506774902344,
      "activations/layer8_attention_weight_max": 43.09543991088867,
      "activations/layer8_attention_weight_min": -45.59086990356445,
      "activations/layer9_attention_weight_max": 31.831064224243164,
      "activations/layer9_attention_weight_min": -36.45481491088867,
      "epoch": 20.53,
      "learning_rate": 1.7756439393939393e-05,
      "loss": 2.7186,
      "step": 353400
    },
    {
      "activations/layer0_attention_weight_max": 16.726428985595703,
      "activations/layer0_attention_weight_min": -14.568629264831543,
      "activations/layer10_attention_weight_max": 38.71400833129883,
      "activations/layer10_attention_weight_min": -39.5560188293457,
      "activations/layer11_attention_weight_max": 36.51042175292969,
      "activations/layer11_attention_weight_min": -39.15556716918945,
      "activations/layer12_attention_weight_max": 29.569347381591797,
      "activations/layer12_attention_weight_min": -28.40534210205078,
      "activations/layer13_attention_weight_max": 43.12511444091797,
      "activations/layer13_attention_weight_min": -37.97431564331055,
      "activations/layer14_attention_weight_max": 49.540557861328125,
      "activations/layer14_attention_weight_min": -42.51791763305664,
      "activations/layer15_attention_weight_max": 42.28702163696289,
      "activations/layer15_attention_weight_min": -39.62273025512695,
      "activations/layer16_attention_weight_max": 31.287870407104492,
      "activations/layer16_attention_weight_min": -30.80461883544922,
      "activations/layer17_attention_weight_max": 31.206594467163086,
      "activations/layer17_attention_weight_min": -28.675188064575195,
      "activations/layer18_attention_weight_max": 35.13814926147461,
      "activations/layer18_attention_weight_min": -30.06716537475586,
      "activations/layer19_attention_weight_max": 35.38528060913086,
      "activations/layer19_attention_weight_min": -36.203285217285156,
      "activations/layer1_attention_weight_max": 16.01555824279785,
      "activations/layer1_attention_weight_min": -15.059418678283691,
      "activations/layer20_attention_weight_max": 31.44490623474121,
      "activations/layer20_attention_weight_min": -27.94124412536621,
      "activations/layer21_attention_weight_max": 30.497753143310547,
      "activations/layer21_attention_weight_min": -26.22344207763672,
      "activations/layer22_attention_weight_max": 48.48126983642578,
      "activations/layer22_attention_weight_min": -37.081077575683594,
      "activations/layer23_attention_weight_max": 33.33447265625,
      "activations/layer23_attention_weight_min": -28.591115951538086,
      "activations/layer2_attention_weight_max": 34.073184967041016,
      "activations/layer2_attention_weight_min": -33.93400573730469,
      "activations/layer3_attention_weight_max": 107.09573364257812,
      "activations/layer3_attention_weight_min": -108.2153549194336,
      "activations/layer4_attention_weight_max": 110.3875961303711,
      "activations/layer4_attention_weight_min": -115.90099334716797,
      "activations/layer5_attention_weight_max": 54.01024627685547,
      "activations/layer5_attention_weight_min": -59.82937240600586,
      "activations/layer6_attention_weight_max": 46.73770523071289,
      "activations/layer6_attention_weight_min": -50.13728713989258,
      "activations/layer7_attention_weight_max": 105.88776397705078,
      "activations/layer7_attention_weight_min": -107.22042846679688,
      "activations/layer8_attention_weight_max": 49.64666748046875,
      "activations/layer8_attention_weight_min": -51.80376434326172,
      "activations/layer9_attention_weight_max": 37.1847038269043,
      "activations/layer9_attention_weight_min": -39.48588180541992,
      "epoch": 20.54,
      "learning_rate": 1.7737499999999997e-05,
      "loss": 2.7076,
      "step": 353450
    },
    {
      "activations/layer0_attention_weight_max": 16.44462013244629,
      "activations/layer0_attention_weight_min": -13.531267166137695,
      "activations/layer10_attention_weight_max": 38.68616485595703,
      "activations/layer10_attention_weight_min": -36.71891784667969,
      "activations/layer11_attention_weight_max": 38.36964797973633,
      "activations/layer11_attention_weight_min": -37.19672393798828,
      "activations/layer12_attention_weight_max": 28.91910171508789,
      "activations/layer12_attention_weight_min": -27.72816276550293,
      "activations/layer13_attention_weight_max": 41.779762268066406,
      "activations/layer13_attention_weight_min": -37.83220672607422,
      "activations/layer14_attention_weight_max": 42.07937240600586,
      "activations/layer14_attention_weight_min": -41.49164581298828,
      "activations/layer15_attention_weight_max": 38.725746154785156,
      "activations/layer15_attention_weight_min": -37.19206619262695,
      "activations/layer16_attention_weight_max": 31.417158126831055,
      "activations/layer16_attention_weight_min": -30.29905891418457,
      "activations/layer17_attention_weight_max": 32.48579788208008,
      "activations/layer17_attention_weight_min": -27.182981491088867,
      "activations/layer18_attention_weight_max": 35.441444396972656,
      "activations/layer18_attention_weight_min": -28.427860260009766,
      "activations/layer19_attention_weight_max": 37.9452018737793,
      "activations/layer19_attention_weight_min": -35.32072067260742,
      "activations/layer1_attention_weight_max": 15.619959831237793,
      "activations/layer1_attention_weight_min": -14.843329429626465,
      "activations/layer20_attention_weight_max": 30.910676956176758,
      "activations/layer20_attention_weight_min": -28.72276496887207,
      "activations/layer21_attention_weight_max": 30.901840209960938,
      "activations/layer21_attention_weight_min": -28.291898727416992,
      "activations/layer22_attention_weight_max": 42.29673385620117,
      "activations/layer22_attention_weight_min": -34.16544723510742,
      "activations/layer23_attention_weight_max": 34.21031188964844,
      "activations/layer23_attention_weight_min": -28.113773345947266,
      "activations/layer2_attention_weight_max": 33.664459228515625,
      "activations/layer2_attention_weight_min": -32.24494171142578,
      "activations/layer3_attention_weight_max": 106.67322540283203,
      "activations/layer3_attention_weight_min": -107.94991302490234,
      "activations/layer4_attention_weight_max": 107.80049133300781,
      "activations/layer4_attention_weight_min": -113.863037109375,
      "activations/layer5_attention_weight_max": 53.87187194824219,
      "activations/layer5_attention_weight_min": -64.09624481201172,
      "activations/layer6_attention_weight_max": 45.37855911254883,
      "activations/layer6_attention_weight_min": -49.923316955566406,
      "activations/layer7_attention_weight_max": 92.8166732788086,
      "activations/layer7_attention_weight_min": -98.53614807128906,
      "activations/layer8_attention_weight_max": 44.9479866027832,
      "activations/layer8_attention_weight_min": -46.09016418457031,
      "activations/layer9_attention_weight_max": 35.516326904296875,
      "activations/layer9_attention_weight_min": -36.71958541870117,
      "epoch": 20.54,
      "learning_rate": 1.7718560606060602e-05,
      "loss": 2.7315,
      "step": 353500
    },
    {
      "activations/layer0_attention_weight_max": 16.292858123779297,
      "activations/layer0_attention_weight_min": -13.051745414733887,
      "activations/layer10_attention_weight_max": 36.2299919128418,
      "activations/layer10_attention_weight_min": -35.3974494934082,
      "activations/layer11_attention_weight_max": 34.891754150390625,
      "activations/layer11_attention_weight_min": -34.26233673095703,
      "activations/layer12_attention_weight_max": 28.38249969482422,
      "activations/layer12_attention_weight_min": -27.060932159423828,
      "activations/layer13_attention_weight_max": 41.544830322265625,
      "activations/layer13_attention_weight_min": -36.164161682128906,
      "activations/layer14_attention_weight_max": 44.786277770996094,
      "activations/layer14_attention_weight_min": -41.98130798339844,
      "activations/layer15_attention_weight_max": 43.2712287902832,
      "activations/layer15_attention_weight_min": -43.316593170166016,
      "activations/layer16_attention_weight_max": 32.54158401489258,
      "activations/layer16_attention_weight_min": -30.583906173706055,
      "activations/layer17_attention_weight_max": 32.27544021606445,
      "activations/layer17_attention_weight_min": -27.01569938659668,
      "activations/layer18_attention_weight_max": 33.58002853393555,
      "activations/layer18_attention_weight_min": -28.210063934326172,
      "activations/layer19_attention_weight_max": 36.54836654663086,
      "activations/layer19_attention_weight_min": -33.28639221191406,
      "activations/layer1_attention_weight_max": 15.213488578796387,
      "activations/layer1_attention_weight_min": -14.509987831115723,
      "activations/layer20_attention_weight_max": 32.19049835205078,
      "activations/layer20_attention_weight_min": -27.144811630249023,
      "activations/layer21_attention_weight_max": 31.898229598999023,
      "activations/layer21_attention_weight_min": -25.774276733398438,
      "activations/layer22_attention_weight_max": 44.86748123168945,
      "activations/layer22_attention_weight_min": -35.307228088378906,
      "activations/layer23_attention_weight_max": 35.520755767822266,
      "activations/layer23_attention_weight_min": -27.22353744506836,
      "activations/layer2_attention_weight_max": 32.281761169433594,
      "activations/layer2_attention_weight_min": -31.445358276367188,
      "activations/layer3_attention_weight_max": 102.9339599609375,
      "activations/layer3_attention_weight_min": -104.85993957519531,
      "activations/layer4_attention_weight_max": 106.65498352050781,
      "activations/layer4_attention_weight_min": -110.8318099975586,
      "activations/layer5_attention_weight_max": 50.32176971435547,
      "activations/layer5_attention_weight_min": -62.99979782104492,
      "activations/layer6_attention_weight_max": 45.31819534301758,
      "activations/layer6_attention_weight_min": -46.71113204956055,
      "activations/layer7_attention_weight_max": 100.06317901611328,
      "activations/layer7_attention_weight_min": -99.321533203125,
      "activations/layer8_attention_weight_max": 44.164894104003906,
      "activations/layer8_attention_weight_min": -46.012874603271484,
      "activations/layer9_attention_weight_max": 33.620521545410156,
      "activations/layer9_attention_weight_min": -33.2202262878418,
      "epoch": 20.54,
      "learning_rate": 1.769962121212121e-05,
      "loss": 2.7018,
      "step": 353550
    },
    {
      "activations/layer0_attention_weight_max": 16.030378341674805,
      "activations/layer0_attention_weight_min": -13.84292221069336,
      "activations/layer10_attention_weight_max": 36.06555938720703,
      "activations/layer10_attention_weight_min": -36.02318572998047,
      "activations/layer11_attention_weight_max": 36.62640380859375,
      "activations/layer11_attention_weight_min": -34.57113265991211,
      "activations/layer12_attention_weight_max": 27.813154220581055,
      "activations/layer12_attention_weight_min": -27.304628372192383,
      "activations/layer13_attention_weight_max": 42.55431365966797,
      "activations/layer13_attention_weight_min": -36.514129638671875,
      "activations/layer14_attention_weight_max": 48.109832763671875,
      "activations/layer14_attention_weight_min": -37.96432876586914,
      "activations/layer15_attention_weight_max": 41.6497802734375,
      "activations/layer15_attention_weight_min": -37.175453186035156,
      "activations/layer16_attention_weight_max": 33.92392349243164,
      "activations/layer16_attention_weight_min": -29.3707332611084,
      "activations/layer17_attention_weight_max": 32.70933151245117,
      "activations/layer17_attention_weight_min": -28.218355178833008,
      "activations/layer18_attention_weight_max": 31.831993103027344,
      "activations/layer18_attention_weight_min": -27.37553596496582,
      "activations/layer19_attention_weight_max": 39.31119918823242,
      "activations/layer19_attention_weight_min": -32.309669494628906,
      "activations/layer1_attention_weight_max": 15.644184112548828,
      "activations/layer1_attention_weight_min": -15.076354026794434,
      "activations/layer20_attention_weight_max": 32.3371467590332,
      "activations/layer20_attention_weight_min": -25.90139389038086,
      "activations/layer21_attention_weight_max": 30.373960494995117,
      "activations/layer21_attention_weight_min": -26.1826229095459,
      "activations/layer22_attention_weight_max": 47.24225616455078,
      "activations/layer22_attention_weight_min": -34.36284637451172,
      "activations/layer23_attention_weight_max": 32.29370880126953,
      "activations/layer23_attention_weight_min": -27.85321044921875,
      "activations/layer2_attention_weight_max": 34.20773696899414,
      "activations/layer2_attention_weight_min": -32.66889953613281,
      "activations/layer3_attention_weight_max": 108.53109741210938,
      "activations/layer3_attention_weight_min": -107.78421020507812,
      "activations/layer4_attention_weight_max": 111.0427017211914,
      "activations/layer4_attention_weight_min": -112.16588592529297,
      "activations/layer5_attention_weight_max": 50.95246887207031,
      "activations/layer5_attention_weight_min": -61.89949035644531,
      "activations/layer6_attention_weight_max": 48.04424285888672,
      "activations/layer6_attention_weight_min": -50.24913787841797,
      "activations/layer7_attention_weight_max": 97.12213134765625,
      "activations/layer7_attention_weight_min": -101.69139862060547,
      "activations/layer8_attention_weight_max": 43.46703338623047,
      "activations/layer8_attention_weight_min": -48.79985046386719,
      "activations/layer9_attention_weight_max": 33.252052307128906,
      "activations/layer9_attention_weight_min": -35.83632278442383,
      "epoch": 20.55,
      "learning_rate": 1.768068181818182e-05,
      "loss": 2.7159,
      "step": 353600
    },
    {
      "activations/layer0_attention_weight_max": 16.088274002075195,
      "activations/layer0_attention_weight_min": -15.525607109069824,
      "activations/layer10_attention_weight_max": 35.810394287109375,
      "activations/layer10_attention_weight_min": -36.19378662109375,
      "activations/layer11_attention_weight_max": 34.79694747924805,
      "activations/layer11_attention_weight_min": -36.900508880615234,
      "activations/layer12_attention_weight_max": 28.125032424926758,
      "activations/layer12_attention_weight_min": -26.929975509643555,
      "activations/layer13_attention_weight_max": 38.54174041748047,
      "activations/layer13_attention_weight_min": -37.12168502807617,
      "activations/layer14_attention_weight_max": 42.158817291259766,
      "activations/layer14_attention_weight_min": -40.2580451965332,
      "activations/layer15_attention_weight_max": 37.74463653564453,
      "activations/layer15_attention_weight_min": -37.27700424194336,
      "activations/layer16_attention_weight_max": 30.210693359375,
      "activations/layer16_attention_weight_min": -29.1901798248291,
      "activations/layer17_attention_weight_max": 28.944849014282227,
      "activations/layer17_attention_weight_min": -26.773372650146484,
      "activations/layer18_attention_weight_max": 32.264949798583984,
      "activations/layer18_attention_weight_min": -24.004491806030273,
      "activations/layer19_attention_weight_max": 36.96586227416992,
      "activations/layer19_attention_weight_min": -30.191640853881836,
      "activations/layer1_attention_weight_max": 14.975740432739258,
      "activations/layer1_attention_weight_min": -14.60538387298584,
      "activations/layer20_attention_weight_max": 28.517105102539062,
      "activations/layer20_attention_weight_min": -24.443748474121094,
      "activations/layer21_attention_weight_max": 27.812053680419922,
      "activations/layer21_attention_weight_min": -23.871477127075195,
      "activations/layer22_attention_weight_max": 37.48188018798828,
      "activations/layer22_attention_weight_min": -29.63331413269043,
      "activations/layer23_attention_weight_max": 29.410612106323242,
      "activations/layer23_attention_weight_min": -25.0701961517334,
      "activations/layer2_attention_weight_max": 33.606163024902344,
      "activations/layer2_attention_weight_min": -32.214271545410156,
      "activations/layer3_attention_weight_max": 105.0062026977539,
      "activations/layer3_attention_weight_min": -105.30371856689453,
      "activations/layer4_attention_weight_max": 111.62251281738281,
      "activations/layer4_attention_weight_min": -107.73470306396484,
      "activations/layer5_attention_weight_max": 49.654232025146484,
      "activations/layer5_attention_weight_min": -61.405784606933594,
      "activations/layer6_attention_weight_max": 45.44142532348633,
      "activations/layer6_attention_weight_min": -49.17496871948242,
      "activations/layer7_attention_weight_max": 100.04425048828125,
      "activations/layer7_attention_weight_min": -105.36951446533203,
      "activations/layer8_attention_weight_max": 44.253143310546875,
      "activations/layer8_attention_weight_min": -45.55621337890625,
      "activations/layer9_attention_weight_max": 33.240325927734375,
      "activations/layer9_attention_weight_min": -34.54719161987305,
      "epoch": 20.55,
      "learning_rate": 1.7661742424242424e-05,
      "loss": 2.7199,
      "step": 353650
    },
    {
      "activations/layer0_attention_weight_max": 15.896590232849121,
      "activations/layer0_attention_weight_min": -13.876677513122559,
      "activations/layer10_attention_weight_max": 35.703407287597656,
      "activations/layer10_attention_weight_min": -35.52854537963867,
      "activations/layer11_attention_weight_max": 37.34015655517578,
      "activations/layer11_attention_weight_min": -34.99491500854492,
      "activations/layer12_attention_weight_max": 27.65513801574707,
      "activations/layer12_attention_weight_min": -26.361042022705078,
      "activations/layer13_attention_weight_max": 39.52159118652344,
      "activations/layer13_attention_weight_min": -35.827667236328125,
      "activations/layer14_attention_weight_max": 43.550804138183594,
      "activations/layer14_attention_weight_min": -37.69242477416992,
      "activations/layer15_attention_weight_max": 42.45534896850586,
      "activations/layer15_attention_weight_min": -37.0791015625,
      "activations/layer16_attention_weight_max": 32.991302490234375,
      "activations/layer16_attention_weight_min": -29.30924415588379,
      "activations/layer17_attention_weight_max": 28.976837158203125,
      "activations/layer17_attention_weight_min": -26.59848976135254,
      "activations/layer18_attention_weight_max": 34.885894775390625,
      "activations/layer18_attention_weight_min": -29.552343368530273,
      "activations/layer19_attention_weight_max": 33.835567474365234,
      "activations/layer19_attention_weight_min": -33.18142318725586,
      "activations/layer1_attention_weight_max": 15.46284008026123,
      "activations/layer1_attention_weight_min": -14.205981254577637,
      "activations/layer20_attention_weight_max": 30.53887367248535,
      "activations/layer20_attention_weight_min": -26.35021209716797,
      "activations/layer21_attention_weight_max": 27.00393295288086,
      "activations/layer21_attention_weight_min": -25.95475196838379,
      "activations/layer22_attention_weight_max": 37.93003463745117,
      "activations/layer22_attention_weight_min": -32.993370056152344,
      "activations/layer23_attention_weight_max": 31.28185272216797,
      "activations/layer23_attention_weight_min": -27.05891990661621,
      "activations/layer2_attention_weight_max": 33.2049446105957,
      "activations/layer2_attention_weight_min": -32.7474479675293,
      "activations/layer3_attention_weight_max": 101.96635437011719,
      "activations/layer3_attention_weight_min": -103.46324157714844,
      "activations/layer4_attention_weight_max": 108.85285949707031,
      "activations/layer4_attention_weight_min": -108.89745330810547,
      "activations/layer5_attention_weight_max": 50.97856903076172,
      "activations/layer5_attention_weight_min": -61.81794357299805,
      "activations/layer6_attention_weight_max": 44.616188049316406,
      "activations/layer6_attention_weight_min": -49.115150451660156,
      "activations/layer7_attention_weight_max": 98.70166015625,
      "activations/layer7_attention_weight_min": -104.94065856933594,
      "activations/layer8_attention_weight_max": 43.40277862548828,
      "activations/layer8_attention_weight_min": -45.16952896118164,
      "activations/layer9_attention_weight_max": 35.20525360107422,
      "activations/layer9_attention_weight_min": -34.81165313720703,
      "epoch": 20.55,
      "learning_rate": 1.764280303030303e-05,
      "loss": 2.7332,
      "step": 353700
    },
    {
      "activations/layer0_attention_weight_max": 16.29488754272461,
      "activations/layer0_attention_weight_min": -12.243334770202637,
      "activations/layer10_attention_weight_max": 35.56999588012695,
      "activations/layer10_attention_weight_min": -35.905975341796875,
      "activations/layer11_attention_weight_max": 35.84701156616211,
      "activations/layer11_attention_weight_min": -37.143436431884766,
      "activations/layer12_attention_weight_max": 30.703641891479492,
      "activations/layer12_attention_weight_min": -28.48361587524414,
      "activations/layer13_attention_weight_max": 43.230323791503906,
      "activations/layer13_attention_weight_min": -39.09710693359375,
      "activations/layer14_attention_weight_max": 45.3007926940918,
      "activations/layer14_attention_weight_min": -42.46031188964844,
      "activations/layer15_attention_weight_max": 42.737937927246094,
      "activations/layer15_attention_weight_min": -41.477378845214844,
      "activations/layer16_attention_weight_max": 35.99757385253906,
      "activations/layer16_attention_weight_min": -32.24306106567383,
      "activations/layer17_attention_weight_max": 34.117122650146484,
      "activations/layer17_attention_weight_min": -27.757125854492188,
      "activations/layer18_attention_weight_max": 33.7890625,
      "activations/layer18_attention_weight_min": -26.781875610351562,
      "activations/layer19_attention_weight_max": 36.28209686279297,
      "activations/layer19_attention_weight_min": -33.83692932128906,
      "activations/layer1_attention_weight_max": 16.342866897583008,
      "activations/layer1_attention_weight_min": -14.700502395629883,
      "activations/layer20_attention_weight_max": 33.24574661254883,
      "activations/layer20_attention_weight_min": -26.305356979370117,
      "activations/layer21_attention_weight_max": 33.48903274536133,
      "activations/layer21_attention_weight_min": -25.971166610717773,
      "activations/layer22_attention_weight_max": 44.27437973022461,
      "activations/layer22_attention_weight_min": -34.465354919433594,
      "activations/layer23_attention_weight_max": 33.036468505859375,
      "activations/layer23_attention_weight_min": -28.285245895385742,
      "activations/layer2_attention_weight_max": 32.64289474487305,
      "activations/layer2_attention_weight_min": -33.364662170410156,
      "activations/layer3_attention_weight_max": 101.69627380371094,
      "activations/layer3_attention_weight_min": -104.23294067382812,
      "activations/layer4_attention_weight_max": 106.22997283935547,
      "activations/layer4_attention_weight_min": -109.25316619873047,
      "activations/layer5_attention_weight_max": 51.47353744506836,
      "activations/layer5_attention_weight_min": -62.26050567626953,
      "activations/layer6_attention_weight_max": 46.21704864501953,
      "activations/layer6_attention_weight_min": -48.577857971191406,
      "activations/layer7_attention_weight_max": 98.70025634765625,
      "activations/layer7_attention_weight_min": -100.76454162597656,
      "activations/layer8_attention_weight_max": 43.406639099121094,
      "activations/layer8_attention_weight_min": -45.50288009643555,
      "activations/layer9_attention_weight_max": 34.3513298034668,
      "activations/layer9_attention_weight_min": -34.7734489440918,
      "epoch": 20.55,
      "learning_rate": 1.7623863636363637e-05,
      "loss": 2.7069,
      "step": 353750
    },
    {
      "activations/layer0_attention_weight_max": 16.492843627929688,
      "activations/layer0_attention_weight_min": -15.166579246520996,
      "activations/layer10_attention_weight_max": 35.648643493652344,
      "activations/layer10_attention_weight_min": -34.641929626464844,
      "activations/layer11_attention_weight_max": 33.669921875,
      "activations/layer11_attention_weight_min": -34.92522430419922,
      "activations/layer12_attention_weight_max": 27.963895797729492,
      "activations/layer12_attention_weight_min": -27.20118522644043,
      "activations/layer13_attention_weight_max": 42.736690521240234,
      "activations/layer13_attention_weight_min": -36.8122444152832,
      "activations/layer14_attention_weight_max": 44.431968688964844,
      "activations/layer14_attention_weight_min": -38.532630920410156,
      "activations/layer15_attention_weight_max": 41.58018112182617,
      "activations/layer15_attention_weight_min": -39.55217361450195,
      "activations/layer16_attention_weight_max": 30.92075538635254,
      "activations/layer16_attention_weight_min": -29.150907516479492,
      "activations/layer17_attention_weight_max": 32.196475982666016,
      "activations/layer17_attention_weight_min": -25.678836822509766,
      "activations/layer18_attention_weight_max": 35.09230041503906,
      "activations/layer18_attention_weight_min": -27.355905532836914,
      "activations/layer19_attention_weight_max": 35.603206634521484,
      "activations/layer19_attention_weight_min": -31.884326934814453,
      "activations/layer1_attention_weight_max": 16.67295265197754,
      "activations/layer1_attention_weight_min": -15.449149131774902,
      "activations/layer20_attention_weight_max": 31.895362854003906,
      "activations/layer20_attention_weight_min": -25.712900161743164,
      "activations/layer21_attention_weight_max": 31.657222747802734,
      "activations/layer21_attention_weight_min": -25.575668334960938,
      "activations/layer22_attention_weight_max": 41.21654510498047,
      "activations/layer22_attention_weight_min": -34.715232849121094,
      "activations/layer23_attention_weight_max": 31.917339324951172,
      "activations/layer23_attention_weight_min": -26.6444091796875,
      "activations/layer2_attention_weight_max": 33.480472564697266,
      "activations/layer2_attention_weight_min": -33.18518829345703,
      "activations/layer3_attention_weight_max": 104.47295379638672,
      "activations/layer3_attention_weight_min": -103.64342498779297,
      "activations/layer4_attention_weight_max": 105.05706787109375,
      "activations/layer4_attention_weight_min": -108.67098236083984,
      "activations/layer5_attention_weight_max": 50.33216857910156,
      "activations/layer5_attention_weight_min": -63.57863998413086,
      "activations/layer6_attention_weight_max": 44.7121467590332,
      "activations/layer6_attention_weight_min": -46.76668167114258,
      "activations/layer7_attention_weight_max": 94.1160888671875,
      "activations/layer7_attention_weight_min": -101.27472686767578,
      "activations/layer8_attention_weight_max": 41.84121322631836,
      "activations/layer8_attention_weight_min": -43.95328140258789,
      "activations/layer9_attention_weight_max": 32.12320327758789,
      "activations/layer9_attention_weight_min": -34.272891998291016,
      "epoch": 20.56,
      "learning_rate": 1.7604924242424242e-05,
      "loss": 2.7168,
      "step": 353800
    },
    {
      "activations/layer0_attention_weight_max": 17.83261489868164,
      "activations/layer0_attention_weight_min": -15.076787948608398,
      "activations/layer10_attention_weight_max": 33.703948974609375,
      "activations/layer10_attention_weight_min": -33.40317916870117,
      "activations/layer11_attention_weight_max": 34.07738494873047,
      "activations/layer11_attention_weight_min": -35.960296630859375,
      "activations/layer12_attention_weight_max": 27.86273765563965,
      "activations/layer12_attention_weight_min": -26.405000686645508,
      "activations/layer13_attention_weight_max": 38.93384552001953,
      "activations/layer13_attention_weight_min": -34.37197494506836,
      "activations/layer14_attention_weight_max": 38.78886032104492,
      "activations/layer14_attention_weight_min": -39.89336013793945,
      "activations/layer15_attention_weight_max": 36.06425476074219,
      "activations/layer15_attention_weight_min": -37.14068603515625,
      "activations/layer16_attention_weight_max": 26.69654655456543,
      "activations/layer16_attention_weight_min": -28.75226593017578,
      "activations/layer17_attention_weight_max": 27.701833724975586,
      "activations/layer17_attention_weight_min": -26.547271728515625,
      "activations/layer18_attention_weight_max": 30.65363883972168,
      "activations/layer18_attention_weight_min": -26.544734954833984,
      "activations/layer19_attention_weight_max": 33.441078186035156,
      "activations/layer19_attention_weight_min": -33.71235275268555,
      "activations/layer1_attention_weight_max": 15.286481857299805,
      "activations/layer1_attention_weight_min": -15.188416481018066,
      "activations/layer20_attention_weight_max": 27.08209800720215,
      "activations/layer20_attention_weight_min": -28.347450256347656,
      "activations/layer21_attention_weight_max": 26.21183967590332,
      "activations/layer21_attention_weight_min": -26.781822204589844,
      "activations/layer22_attention_weight_max": 36.28641891479492,
      "activations/layer22_attention_weight_min": -33.08911895751953,
      "activations/layer23_attention_weight_max": 32.77679443359375,
      "activations/layer23_attention_weight_min": -28.134063720703125,
      "activations/layer2_attention_weight_max": 32.71685028076172,
      "activations/layer2_attention_weight_min": -32.5809326171875,
      "activations/layer3_attention_weight_max": 106.31218719482422,
      "activations/layer3_attention_weight_min": -103.37181091308594,
      "activations/layer4_attention_weight_max": 107.6074447631836,
      "activations/layer4_attention_weight_min": -107.9256820678711,
      "activations/layer5_attention_weight_max": 51.12473678588867,
      "activations/layer5_attention_weight_min": -63.297828674316406,
      "activations/layer6_attention_weight_max": 43.682716369628906,
      "activations/layer6_attention_weight_min": -47.570011138916016,
      "activations/layer7_attention_weight_max": 96.35162353515625,
      "activations/layer7_attention_weight_min": -97.83907318115234,
      "activations/layer8_attention_weight_max": 41.20744323730469,
      "activations/layer8_attention_weight_min": -43.00123596191406,
      "activations/layer9_attention_weight_max": 32.13032150268555,
      "activations/layer9_attention_weight_min": -34.16071701049805,
      "epoch": 20.56,
      "learning_rate": 1.7585984848484847e-05,
      "loss": 2.6986,
      "step": 353850
    },
    {
      "activations/layer0_attention_weight_max": 15.835848808288574,
      "activations/layer0_attention_weight_min": -14.150227546691895,
      "activations/layer10_attention_weight_max": 38.340484619140625,
      "activations/layer10_attention_weight_min": -35.90504837036133,
      "activations/layer11_attention_weight_max": 34.217987060546875,
      "activations/layer11_attention_weight_min": -34.54118728637695,
      "activations/layer12_attention_weight_max": 26.25047492980957,
      "activations/layer12_attention_weight_min": -27.49036407470703,
      "activations/layer13_attention_weight_max": 40.692806243896484,
      "activations/layer13_attention_weight_min": -36.52421951293945,
      "activations/layer14_attention_weight_max": 42.451934814453125,
      "activations/layer14_attention_weight_min": -38.9788932800293,
      "activations/layer15_attention_weight_max": 39.33245086669922,
      "activations/layer15_attention_weight_min": -36.705299377441406,
      "activations/layer16_attention_weight_max": 29.414682388305664,
      "activations/layer16_attention_weight_min": -31.737987518310547,
      "activations/layer17_attention_weight_max": 28.68075180053711,
      "activations/layer17_attention_weight_min": -30.0643310546875,
      "activations/layer18_attention_weight_max": 33.077362060546875,
      "activations/layer18_attention_weight_min": -28.11250114440918,
      "activations/layer19_attention_weight_max": 34.674930572509766,
      "activations/layer19_attention_weight_min": -32.39027404785156,
      "activations/layer1_attention_weight_max": 15.5443754196167,
      "activations/layer1_attention_weight_min": -17.42862319946289,
      "activations/layer20_attention_weight_max": 31.10504722595215,
      "activations/layer20_attention_weight_min": -25.462369918823242,
      "activations/layer21_attention_weight_max": 30.57036590576172,
      "activations/layer21_attention_weight_min": -25.91153335571289,
      "activations/layer22_attention_weight_max": 40.771602630615234,
      "activations/layer22_attention_weight_min": -34.22320556640625,
      "activations/layer23_attention_weight_max": 32.230690002441406,
      "activations/layer23_attention_weight_min": -26.993099212646484,
      "activations/layer2_attention_weight_max": 34.64506149291992,
      "activations/layer2_attention_weight_min": -33.32929229736328,
      "activations/layer3_attention_weight_max": 106.4117202758789,
      "activations/layer3_attention_weight_min": -109.96223449707031,
      "activations/layer4_attention_weight_max": 111.10064697265625,
      "activations/layer4_attention_weight_min": -112.38856506347656,
      "activations/layer5_attention_weight_max": 51.47764587402344,
      "activations/layer5_attention_weight_min": -61.130699157714844,
      "activations/layer6_attention_weight_max": 45.261417388916016,
      "activations/layer6_attention_weight_min": -47.85625457763672,
      "activations/layer7_attention_weight_max": 93.35528564453125,
      "activations/layer7_attention_weight_min": -100.93294525146484,
      "activations/layer8_attention_weight_max": 42.64360427856445,
      "activations/layer8_attention_weight_min": -47.27280807495117,
      "activations/layer9_attention_weight_max": 33.03941345214844,
      "activations/layer9_attention_weight_min": -34.828521728515625,
      "epoch": 20.56,
      "learning_rate": 1.756704545454545e-05,
      "loss": 2.7217,
      "step": 353900
    },
    {
      "activations/layer0_attention_weight_max": 16.628801345825195,
      "activations/layer0_attention_weight_min": -13.485729217529297,
      "activations/layer10_attention_weight_max": 37.73750686645508,
      "activations/layer10_attention_weight_min": -38.11494827270508,
      "activations/layer11_attention_weight_max": 37.20964050292969,
      "activations/layer11_attention_weight_min": -36.58623123168945,
      "activations/layer12_attention_weight_max": 29.944992065429688,
      "activations/layer12_attention_weight_min": -27.878841400146484,
      "activations/layer13_attention_weight_max": 46.601234436035156,
      "activations/layer13_attention_weight_min": -36.646766662597656,
      "activations/layer14_attention_weight_max": 47.891483306884766,
      "activations/layer14_attention_weight_min": -38.96616744995117,
      "activations/layer15_attention_weight_max": 43.87149429321289,
      "activations/layer15_attention_weight_min": -37.80009841918945,
      "activations/layer16_attention_weight_max": 32.712215423583984,
      "activations/layer16_attention_weight_min": -27.536195755004883,
      "activations/layer17_attention_weight_max": 32.62572479248047,
      "activations/layer17_attention_weight_min": -25.648117065429688,
      "activations/layer18_attention_weight_max": 33.830074310302734,
      "activations/layer18_attention_weight_min": -26.654464721679688,
      "activations/layer19_attention_weight_max": 36.3501091003418,
      "activations/layer19_attention_weight_min": -30.898923873901367,
      "activations/layer1_attention_weight_max": 15.746583938598633,
      "activations/layer1_attention_weight_min": -13.946544647216797,
      "activations/layer20_attention_weight_max": 30.802438735961914,
      "activations/layer20_attention_weight_min": -26.399179458618164,
      "activations/layer21_attention_weight_max": 30.4420223236084,
      "activations/layer21_attention_weight_min": -25.565935134887695,
      "activations/layer22_attention_weight_max": 49.08226013183594,
      "activations/layer22_attention_weight_min": -32.37002182006836,
      "activations/layer23_attention_weight_max": 35.61737060546875,
      "activations/layer23_attention_weight_min": -25.604047775268555,
      "activations/layer2_attention_weight_max": 34.088478088378906,
      "activations/layer2_attention_weight_min": -33.50800704956055,
      "activations/layer3_attention_weight_max": 101.76345825195312,
      "activations/layer3_attention_weight_min": -110.63723754882812,
      "activations/layer4_attention_weight_max": 107.01350402832031,
      "activations/layer4_attention_weight_min": -108.59654998779297,
      "activations/layer5_attention_weight_max": 48.60272979736328,
      "activations/layer5_attention_weight_min": -60.525535583496094,
      "activations/layer6_attention_weight_max": 47.38303756713867,
      "activations/layer6_attention_weight_min": -49.0744743347168,
      "activations/layer7_attention_weight_max": 105.72724151611328,
      "activations/layer7_attention_weight_min": -104.41622924804688,
      "activations/layer8_attention_weight_max": 46.704307556152344,
      "activations/layer8_attention_weight_min": -48.069488525390625,
      "activations/layer9_attention_weight_max": 36.21609115600586,
      "activations/layer9_attention_weight_min": -37.599456787109375,
      "epoch": 20.57,
      "learning_rate": 1.754810606060606e-05,
      "loss": 2.6916,
      "step": 353950
    },
    {
      "activations/layer0_attention_weight_max": 16.782209396362305,
      "activations/layer0_attention_weight_min": -15.031302452087402,
      "activations/layer10_attention_weight_max": 41.41179656982422,
      "activations/layer10_attention_weight_min": -39.8483772277832,
      "activations/layer11_attention_weight_max": 37.55107116699219,
      "activations/layer11_attention_weight_min": -38.946250915527344,
      "activations/layer12_attention_weight_max": 30.060998916625977,
      "activations/layer12_attention_weight_min": -29.161333084106445,
      "activations/layer13_attention_weight_max": 39.5580940246582,
      "activations/layer13_attention_weight_min": -36.899208068847656,
      "activations/layer14_attention_weight_max": 41.99620819091797,
      "activations/layer14_attention_weight_min": -40.07991409301758,
      "activations/layer15_attention_weight_max": 41.331233978271484,
      "activations/layer15_attention_weight_min": -39.456459045410156,
      "activations/layer16_attention_weight_max": 27.14196014404297,
      "activations/layer16_attention_weight_min": -28.476329803466797,
      "activations/layer17_attention_weight_max": 28.508161544799805,
      "activations/layer17_attention_weight_min": -27.537822723388672,
      "activations/layer18_attention_weight_max": 29.014652252197266,
      "activations/layer18_attention_weight_min": -25.747650146484375,
      "activations/layer19_attention_weight_max": 30.641508102416992,
      "activations/layer19_attention_weight_min": -30.664337158203125,
      "activations/layer1_attention_weight_max": 15.386810302734375,
      "activations/layer1_attention_weight_min": -15.030172348022461,
      "activations/layer20_attention_weight_max": 26.21139144897461,
      "activations/layer20_attention_weight_min": -25.14794158935547,
      "activations/layer21_attention_weight_max": 27.28971290588379,
      "activations/layer21_attention_weight_min": -23.58165740966797,
      "activations/layer22_attention_weight_max": 38.03145217895508,
      "activations/layer22_attention_weight_min": -30.869842529296875,
      "activations/layer23_attention_weight_max": 28.162885665893555,
      "activations/layer23_attention_weight_min": -26.086233139038086,
      "activations/layer2_attention_weight_max": 34.285640716552734,
      "activations/layer2_attention_weight_min": -33.54822540283203,
      "activations/layer3_attention_weight_max": 106.45006561279297,
      "activations/layer3_attention_weight_min": -109.02117156982422,
      "activations/layer4_attention_weight_max": 109.05769348144531,
      "activations/layer4_attention_weight_min": -112.95753479003906,
      "activations/layer5_attention_weight_max": 50.99055480957031,
      "activations/layer5_attention_weight_min": -62.640869140625,
      "activations/layer6_attention_weight_max": 45.90387725830078,
      "activations/layer6_attention_weight_min": -49.71742630004883,
      "activations/layer7_attention_weight_max": 106.43634033203125,
      "activations/layer7_attention_weight_min": -110.82783508300781,
      "activations/layer8_attention_weight_max": 47.50002670288086,
      "activations/layer8_attention_weight_min": -49.19403076171875,
      "activations/layer9_attention_weight_max": 36.76897048950195,
      "activations/layer9_attention_weight_min": -40.262855529785156,
      "epoch": 20.57,
      "learning_rate": 1.7529166666666665e-05,
      "loss": 2.7216,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4449,
      "eval_samples_per_second": 508.473,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4449,
      "eval_openwebtext_samples_per_second": 508.473,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9548,
      "eval_wikitext_samples_per_second": 233.277,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_lambada_loss": 2.3203125,
      "eval_lambada_ppl": 10.17885470120484,
      "eval_lambada_runtime": 9.5574,
      "eval_lambada_samples_per_second": 509.446,
      "step": 354000
    },
    {
      "activations/layer0_attention_weight_max": 16.395658493041992,
      "activations/layer0_attention_weight_min": -14.636857032775879,
      "activations/layer10_attention_weight_max": 36.89658737182617,
      "activations/layer10_attention_weight_min": -35.185791015625,
      "activations/layer11_attention_weight_max": 37.16435241699219,
      "activations/layer11_attention_weight_min": -35.45458984375,
      "activations/layer12_attention_weight_max": 29.101118087768555,
      "activations/layer12_attention_weight_min": -31.604650497436523,
      "activations/layer13_attention_weight_max": 42.795467376708984,
      "activations/layer13_attention_weight_min": -39.2484130859375,
      "activations/layer14_attention_weight_max": 49.036346435546875,
      "activations/layer14_attention_weight_min": -39.97903060913086,
      "activations/layer15_attention_weight_max": 40.7050895690918,
      "activations/layer15_attention_weight_min": -38.94668197631836,
      "activations/layer16_attention_weight_max": 35.95733642578125,
      "activations/layer16_attention_weight_min": -30.04930305480957,
      "activations/layer17_attention_weight_max": 32.02093505859375,
      "activations/layer17_attention_weight_min": -26.95954132080078,
      "activations/layer18_attention_weight_max": 32.641876220703125,
      "activations/layer18_attention_weight_min": -26.079374313354492,
      "activations/layer19_attention_weight_max": 35.19907760620117,
      "activations/layer19_attention_weight_min": -31.97071647644043,
      "activations/layer1_attention_weight_max": 15.753633499145508,
      "activations/layer1_attention_weight_min": -14.490853309631348,
      "activations/layer20_attention_weight_max": 30.855207443237305,
      "activations/layer20_attention_weight_min": -25.2576961517334,
      "activations/layer21_attention_weight_max": 30.18222427368164,
      "activations/layer21_attention_weight_min": -24.9390926361084,
      "activations/layer22_attention_weight_max": 43.97843933105469,
      "activations/layer22_attention_weight_min": -35.32562255859375,
      "activations/layer23_attention_weight_max": 30.77015495300293,
      "activations/layer23_attention_weight_min": -24.932064056396484,
      "activations/layer2_attention_weight_max": 34.20058822631836,
      "activations/layer2_attention_weight_min": -32.50637435913086,
      "activations/layer3_attention_weight_max": 103.43527221679688,
      "activations/layer3_attention_weight_min": -107.7646255493164,
      "activations/layer4_attention_weight_max": 104.90807342529297,
      "activations/layer4_attention_weight_min": -110.65886688232422,
      "activations/layer5_attention_weight_max": 50.6562614440918,
      "activations/layer5_attention_weight_min": -65.29037475585938,
      "activations/layer6_attention_weight_max": 45.262149810791016,
      "activations/layer6_attention_weight_min": -47.26467514038086,
      "activations/layer7_attention_weight_max": 94.44473266601562,
      "activations/layer7_attention_weight_min": -98.01622009277344,
      "activations/layer8_attention_weight_max": 43.4691047668457,
      "activations/layer8_attention_weight_min": -46.00432205200195,
      "activations/layer9_attention_weight_max": 32.866241455078125,
      "activations/layer9_attention_weight_min": -34.667755126953125,
      "epoch": 20.57,
      "learning_rate": 1.751022727272727e-05,
      "loss": 2.7091,
      "step": 354050
    },
    {
      "activations/layer0_attention_weight_max": 16.616724014282227,
      "activations/layer0_attention_weight_min": -13.064833641052246,
      "activations/layer10_attention_weight_max": 35.66054916381836,
      "activations/layer10_attention_weight_min": -36.595497131347656,
      "activations/layer11_attention_weight_max": 35.25516128540039,
      "activations/layer11_attention_weight_min": -35.33245849609375,
      "activations/layer12_attention_weight_max": 27.79403305053711,
      "activations/layer12_attention_weight_min": -26.805198669433594,
      "activations/layer13_attention_weight_max": 41.176307678222656,
      "activations/layer13_attention_weight_min": -35.900917053222656,
      "activations/layer14_attention_weight_max": 44.03133773803711,
      "activations/layer14_attention_weight_min": -39.794593811035156,
      "activations/layer15_attention_weight_max": 43.27350616455078,
      "activations/layer15_attention_weight_min": -37.03419494628906,
      "activations/layer16_attention_weight_max": 29.944786071777344,
      "activations/layer16_attention_weight_min": -30.713720321655273,
      "activations/layer17_attention_weight_max": 29.63489532470703,
      "activations/layer17_attention_weight_min": -28.129430770874023,
      "activations/layer18_attention_weight_max": 31.801815032958984,
      "activations/layer18_attention_weight_min": -26.45292091369629,
      "activations/layer19_attention_weight_max": 34.72584915161133,
      "activations/layer19_attention_weight_min": -32.13717269897461,
      "activations/layer1_attention_weight_max": 14.937438011169434,
      "activations/layer1_attention_weight_min": -14.978673934936523,
      "activations/layer20_attention_weight_max": 28.432899475097656,
      "activations/layer20_attention_weight_min": -25.78725242614746,
      "activations/layer21_attention_weight_max": 29.930908203125,
      "activations/layer21_attention_weight_min": -27.34183692932129,
      "activations/layer22_attention_weight_max": 37.69019317626953,
      "activations/layer22_attention_weight_min": -31.128835678100586,
      "activations/layer23_attention_weight_max": 32.48448181152344,
      "activations/layer23_attention_weight_min": -25.741613388061523,
      "activations/layer2_attention_weight_max": 33.102386474609375,
      "activations/layer2_attention_weight_min": -31.77822494506836,
      "activations/layer3_attention_weight_max": 101.78240966796875,
      "activations/layer3_attention_weight_min": -102.56283569335938,
      "activations/layer4_attention_weight_max": 107.5987548828125,
      "activations/layer4_attention_weight_min": -109.51744842529297,
      "activations/layer5_attention_weight_max": 52.56203842163086,
      "activations/layer5_attention_weight_min": -61.669212341308594,
      "activations/layer6_attention_weight_max": 44.1729850769043,
      "activations/layer6_attention_weight_min": -49.037052154541016,
      "activations/layer7_attention_weight_max": 101.94892120361328,
      "activations/layer7_attention_weight_min": -102.4355239868164,
      "activations/layer8_attention_weight_max": 42.455535888671875,
      "activations/layer8_attention_weight_min": -46.04206085205078,
      "activations/layer9_attention_weight_max": 32.49898910522461,
      "activations/layer9_attention_weight_min": -35.86016082763672,
      "epoch": 20.58,
      "learning_rate": 1.7491287878787878e-05,
      "loss": 2.7351,
      "step": 354100
    },
    {
      "activations/layer0_attention_weight_max": 16.39266586303711,
      "activations/layer0_attention_weight_min": -12.61049747467041,
      "activations/layer10_attention_weight_max": 36.87006378173828,
      "activations/layer10_attention_weight_min": -37.36429977416992,
      "activations/layer11_attention_weight_max": 37.05964660644531,
      "activations/layer11_attention_weight_min": -37.14140319824219,
      "activations/layer12_attention_weight_max": 27.825685501098633,
      "activations/layer12_attention_weight_min": -29.716304779052734,
      "activations/layer13_attention_weight_max": 41.489524841308594,
      "activations/layer13_attention_weight_min": -40.13511276245117,
      "activations/layer14_attention_weight_max": 43.62502670288086,
      "activations/layer14_attention_weight_min": -39.810523986816406,
      "activations/layer15_attention_weight_max": 37.79505157470703,
      "activations/layer15_attention_weight_min": -36.60479736328125,
      "activations/layer16_attention_weight_max": 30.022951126098633,
      "activations/layer16_attention_weight_min": -28.514253616333008,
      "activations/layer17_attention_weight_max": 30.242977142333984,
      "activations/layer17_attention_weight_min": -28.637855529785156,
      "activations/layer18_attention_weight_max": 33.83933639526367,
      "activations/layer18_attention_weight_min": -27.957223892211914,
      "activations/layer19_attention_weight_max": 35.97618103027344,
      "activations/layer19_attention_weight_min": -33.67976379394531,
      "activations/layer1_attention_weight_max": 15.223782539367676,
      "activations/layer1_attention_weight_min": -14.25610065460205,
      "activations/layer20_attention_weight_max": 29.790048599243164,
      "activations/layer20_attention_weight_min": -28.225461959838867,
      "activations/layer21_attention_weight_max": 29.08930015563965,
      "activations/layer21_attention_weight_min": -26.88820457458496,
      "activations/layer22_attention_weight_max": 44.57490921020508,
      "activations/layer22_attention_weight_min": -35.07611083984375,
      "activations/layer23_attention_weight_max": 34.27332305908203,
      "activations/layer23_attention_weight_min": -27.73138999938965,
      "activations/layer2_attention_weight_max": 34.45588302612305,
      "activations/layer2_attention_weight_min": -33.27263641357422,
      "activations/layer3_attention_weight_max": 108.13812255859375,
      "activations/layer3_attention_weight_min": -107.48542785644531,
      "activations/layer4_attention_weight_max": 108.87457275390625,
      "activations/layer4_attention_weight_min": -110.62544250488281,
      "activations/layer5_attention_weight_max": 52.01625061035156,
      "activations/layer5_attention_weight_min": -61.783424377441406,
      "activations/layer6_attention_weight_max": 46.81296157836914,
      "activations/layer6_attention_weight_min": -49.78530502319336,
      "activations/layer7_attention_weight_max": 101.61534118652344,
      "activations/layer7_attention_weight_min": -106.33601379394531,
      "activations/layer8_attention_weight_max": 42.655540466308594,
      "activations/layer8_attention_weight_min": -45.1422004699707,
      "activations/layer9_attention_weight_max": 33.99882507324219,
      "activations/layer9_attention_weight_min": -35.582332611083984,
      "epoch": 20.58,
      "learning_rate": 1.7472727272727272e-05,
      "loss": 2.722,
      "step": 354150
    },
    {
      "activations/layer0_attention_weight_max": 16.45941162109375,
      "activations/layer0_attention_weight_min": -13.741029739379883,
      "activations/layer10_attention_weight_max": 37.770042419433594,
      "activations/layer10_attention_weight_min": -37.748687744140625,
      "activations/layer11_attention_weight_max": 36.72492218017578,
      "activations/layer11_attention_weight_min": -37.904964447021484,
      "activations/layer12_attention_weight_max": 28.62299156188965,
      "activations/layer12_attention_weight_min": -27.628822326660156,
      "activations/layer13_attention_weight_max": 40.71904373168945,
      "activations/layer13_attention_weight_min": -37.27117919921875,
      "activations/layer14_attention_weight_max": 43.72005844116211,
      "activations/layer14_attention_weight_min": -38.00841522216797,
      "activations/layer15_attention_weight_max": 40.556243896484375,
      "activations/layer15_attention_weight_min": -35.992977142333984,
      "activations/layer16_attention_weight_max": 30.379411697387695,
      "activations/layer16_attention_weight_min": -30.374040603637695,
      "activations/layer17_attention_weight_max": 31.69127082824707,
      "activations/layer17_attention_weight_min": -27.433561325073242,
      "activations/layer18_attention_weight_max": 33.05997085571289,
      "activations/layer18_attention_weight_min": -28.7811222076416,
      "activations/layer19_attention_weight_max": 34.416473388671875,
      "activations/layer19_attention_weight_min": -34.364219665527344,
      "activations/layer1_attention_weight_max": 16.22391128540039,
      "activations/layer1_attention_weight_min": -15.352655410766602,
      "activations/layer20_attention_weight_max": 29.276262283325195,
      "activations/layer20_attention_weight_min": -27.808931350708008,
      "activations/layer21_attention_weight_max": 31.538223266601562,
      "activations/layer21_attention_weight_min": -27.632200241088867,
      "activations/layer22_attention_weight_max": 42.20542907714844,
      "activations/layer22_attention_weight_min": -36.931278228759766,
      "activations/layer23_attention_weight_max": 32.47692108154297,
      "activations/layer23_attention_weight_min": -29.555280685424805,
      "activations/layer2_attention_weight_max": 34.05956268310547,
      "activations/layer2_attention_weight_min": -32.76202392578125,
      "activations/layer3_attention_weight_max": 104.78832244873047,
      "activations/layer3_attention_weight_min": -107.74681854248047,
      "activations/layer4_attention_weight_max": 107.34410858154297,
      "activations/layer4_attention_weight_min": -111.92827606201172,
      "activations/layer5_attention_weight_max": 51.29055404663086,
      "activations/layer5_attention_weight_min": -63.90568542480469,
      "activations/layer6_attention_weight_max": 46.81604766845703,
      "activations/layer6_attention_weight_min": -50.57360076904297,
      "activations/layer7_attention_weight_max": 96.46549224853516,
      "activations/layer7_attention_weight_min": -104.17276000976562,
      "activations/layer8_attention_weight_max": 44.865997314453125,
      "activations/layer8_attention_weight_min": -46.92210388183594,
      "activations/layer9_attention_weight_max": 34.21865463256836,
      "activations/layer9_attention_weight_min": -36.438873291015625,
      "epoch": 20.58,
      "learning_rate": 1.7453787878787877e-05,
      "loss": 2.7189,
      "step": 354200
    },
    {
      "activations/layer0_attention_weight_max": 16.23790168762207,
      "activations/layer0_attention_weight_min": -14.104581832885742,
      "activations/layer10_attention_weight_max": 37.00961685180664,
      "activations/layer10_attention_weight_min": -37.368690490722656,
      "activations/layer11_attention_weight_max": 35.855133056640625,
      "activations/layer11_attention_weight_min": -36.09605407714844,
      "activations/layer12_attention_weight_max": 28.269752502441406,
      "activations/layer12_attention_weight_min": -28.289674758911133,
      "activations/layer13_attention_weight_max": 42.158809661865234,
      "activations/layer13_attention_weight_min": -37.79931640625,
      "activations/layer14_attention_weight_max": 42.510807037353516,
      "activations/layer14_attention_weight_min": -40.97956466674805,
      "activations/layer15_attention_weight_max": 43.6364860534668,
      "activations/layer15_attention_weight_min": -36.23113250732422,
      "activations/layer16_attention_weight_max": 33.77787780761719,
      "activations/layer16_attention_weight_min": -28.15567970275879,
      "activations/layer17_attention_weight_max": 32.56667709350586,
      "activations/layer17_attention_weight_min": -26.647680282592773,
      "activations/layer18_attention_weight_max": 33.77859878540039,
      "activations/layer18_attention_weight_min": -26.296415328979492,
      "activations/layer19_attention_weight_max": 37.87177658081055,
      "activations/layer19_attention_weight_min": -32.572364807128906,
      "activations/layer1_attention_weight_max": 16.001771926879883,
      "activations/layer1_attention_weight_min": -15.175411224365234,
      "activations/layer20_attention_weight_max": 30.307016372680664,
      "activations/layer20_attention_weight_min": -25.92892074584961,
      "activations/layer21_attention_weight_max": 29.48891258239746,
      "activations/layer21_attention_weight_min": -23.734264373779297,
      "activations/layer22_attention_weight_max": 41.95153045654297,
      "activations/layer22_attention_weight_min": -31.546995162963867,
      "activations/layer23_attention_weight_max": 35.00538635253906,
      "activations/layer23_attention_weight_min": -24.716991424560547,
      "activations/layer2_attention_weight_max": 34.42519760131836,
      "activations/layer2_attention_weight_min": -32.918704986572266,
      "activations/layer3_attention_weight_max": 107.2379150390625,
      "activations/layer3_attention_weight_min": -106.66085815429688,
      "activations/layer4_attention_weight_max": 109.69896697998047,
      "activations/layer4_attention_weight_min": -121.33739471435547,
      "activations/layer5_attention_weight_max": 52.059444427490234,
      "activations/layer5_attention_weight_min": -60.68440246582031,
      "activations/layer6_attention_weight_max": 46.345680236816406,
      "activations/layer6_attention_weight_min": -47.4447135925293,
      "activations/layer7_attention_weight_max": 97.92494201660156,
      "activations/layer7_attention_weight_min": -104.02381134033203,
      "activations/layer8_attention_weight_max": 44.5335578918457,
      "activations/layer8_attention_weight_min": -47.48367691040039,
      "activations/layer9_attention_weight_max": 34.57468795776367,
      "activations/layer9_attention_weight_min": -36.488101959228516,
      "epoch": 20.58,
      "learning_rate": 1.7434848484848482e-05,
      "loss": 2.6989,
      "step": 354250
    },
    {
      "activations/layer0_attention_weight_max": 16.22012710571289,
      "activations/layer0_attention_weight_min": -13.837191581726074,
      "activations/layer10_attention_weight_max": 47.65134811401367,
      "activations/layer10_attention_weight_min": -45.294673919677734,
      "activations/layer11_attention_weight_max": 45.67750549316406,
      "activations/layer11_attention_weight_min": -43.91018295288086,
      "activations/layer12_attention_weight_max": 35.713443756103516,
      "activations/layer12_attention_weight_min": -33.030784606933594,
      "activations/layer13_attention_weight_max": 48.45553207397461,
      "activations/layer13_attention_weight_min": -42.85563278198242,
      "activations/layer14_attention_weight_max": 54.919429779052734,
      "activations/layer14_attention_weight_min": -50.46928787231445,
      "activations/layer15_attention_weight_max": 51.50281524658203,
      "activations/layer15_attention_weight_min": -47.32561111450195,
      "activations/layer16_attention_weight_max": 36.96649932861328,
      "activations/layer16_attention_weight_min": -33.33635330200195,
      "activations/layer17_attention_weight_max": 36.66948318481445,
      "activations/layer17_attention_weight_min": -31.465383529663086,
      "activations/layer18_attention_weight_max": 36.22610855102539,
      "activations/layer18_attention_weight_min": -33.17356872558594,
      "activations/layer19_attention_weight_max": 41.17588424682617,
      "activations/layer19_attention_weight_min": -39.0593147277832,
      "activations/layer1_attention_weight_max": 16.146522521972656,
      "activations/layer1_attention_weight_min": -15.035725593566895,
      "activations/layer20_attention_weight_max": 35.39743423461914,
      "activations/layer20_attention_weight_min": -28.86968421936035,
      "activations/layer21_attention_weight_max": 31.094131469726562,
      "activations/layer21_attention_weight_min": -25.351547241210938,
      "activations/layer22_attention_weight_max": 48.922019958496094,
      "activations/layer22_attention_weight_min": -34.49545669555664,
      "activations/layer23_attention_weight_max": 35.35010528564453,
      "activations/layer23_attention_weight_min": -25.466527938842773,
      "activations/layer2_attention_weight_max": 33.92897033691406,
      "activations/layer2_attention_weight_min": -33.24778747558594,
      "activations/layer3_attention_weight_max": 106.34834289550781,
      "activations/layer3_attention_weight_min": -108.29756927490234,
      "activations/layer4_attention_weight_max": 113.37196350097656,
      "activations/layer4_attention_weight_min": -112.57415771484375,
      "activations/layer5_attention_weight_max": 57.6174430847168,
      "activations/layer5_attention_weight_min": -64.41825103759766,
      "activations/layer6_attention_weight_max": 50.94063949584961,
      "activations/layer6_attention_weight_min": -50.07455062866211,
      "activations/layer7_attention_weight_max": 112.98716735839844,
      "activations/layer7_attention_weight_min": -111.44141387939453,
      "activations/layer8_attention_weight_max": 54.667179107666016,
      "activations/layer8_attention_weight_min": -52.325374603271484,
      "activations/layer9_attention_weight_max": 40.86354446411133,
      "activations/layer9_attention_weight_min": -42.2728271484375,
      "epoch": 20.59,
      "learning_rate": 1.741590909090909e-05,
      "loss": 2.7194,
      "step": 354300
    },
    {
      "activations/layer0_attention_weight_max": 16.270729064941406,
      "activations/layer0_attention_weight_min": -13.138201713562012,
      "activations/layer10_attention_weight_max": 35.35768127441406,
      "activations/layer10_attention_weight_min": -35.0640754699707,
      "activations/layer11_attention_weight_max": 34.887447357177734,
      "activations/layer11_attention_weight_min": -35.836246490478516,
      "activations/layer12_attention_weight_max": 27.427167892456055,
      "activations/layer12_attention_weight_min": -26.356637954711914,
      "activations/layer13_attention_weight_max": 40.648860931396484,
      "activations/layer13_attention_weight_min": -35.150054931640625,
      "activations/layer14_attention_weight_max": 46.868186950683594,
      "activations/layer14_attention_weight_min": -39.96013641357422,
      "activations/layer15_attention_weight_max": 40.37504577636719,
      "activations/layer15_attention_weight_min": -40.24906539916992,
      "activations/layer16_attention_weight_max": 30.50766944885254,
      "activations/layer16_attention_weight_min": -28.67816925048828,
      "activations/layer17_attention_weight_max": 29.58736228942871,
      "activations/layer17_attention_weight_min": -25.027883529663086,
      "activations/layer18_attention_weight_max": 31.296274185180664,
      "activations/layer18_attention_weight_min": -26.720603942871094,
      "activations/layer19_attention_weight_max": 35.163551330566406,
      "activations/layer19_attention_weight_min": -31.041061401367188,
      "activations/layer1_attention_weight_max": 16.158634185791016,
      "activations/layer1_attention_weight_min": -15.322077751159668,
      "activations/layer20_attention_weight_max": 31.682376861572266,
      "activations/layer20_attention_weight_min": -23.29749870300293,
      "activations/layer21_attention_weight_max": 31.39569091796875,
      "activations/layer21_attention_weight_min": -23.954105377197266,
      "activations/layer22_attention_weight_max": 40.95042037963867,
      "activations/layer22_attention_weight_min": -34.29785919189453,
      "activations/layer23_attention_weight_max": 30.367813110351562,
      "activations/layer23_attention_weight_min": -26.170902252197266,
      "activations/layer2_attention_weight_max": 33.04389190673828,
      "activations/layer2_attention_weight_min": -31.75448989868164,
      "activations/layer3_attention_weight_max": 106.74103546142578,
      "activations/layer3_attention_weight_min": -105.3796615600586,
      "activations/layer4_attention_weight_max": 112.54207611083984,
      "activations/layer4_attention_weight_min": -108.3311538696289,
      "activations/layer5_attention_weight_max": 49.489402770996094,
      "activations/layer5_attention_weight_min": -62.69184494018555,
      "activations/layer6_attention_weight_max": 43.747764587402344,
      "activations/layer6_attention_weight_min": -46.782466888427734,
      "activations/layer7_attention_weight_max": 95.19080352783203,
      "activations/layer7_attention_weight_min": -97.47984313964844,
      "activations/layer8_attention_weight_max": 44.55647277832031,
      "activations/layer8_attention_weight_min": -44.99684143066406,
      "activations/layer9_attention_weight_max": 31.470050811767578,
      "activations/layer9_attention_weight_min": -34.17642593383789,
      "epoch": 20.59,
      "learning_rate": 1.7396969696969695e-05,
      "loss": 2.7132,
      "step": 354350
    },
    {
      "activations/layer0_attention_weight_max": 17.16710090637207,
      "activations/layer0_attention_weight_min": -12.66922664642334,
      "activations/layer10_attention_weight_max": 34.581607818603516,
      "activations/layer10_attention_weight_min": -34.83698272705078,
      "activations/layer11_attention_weight_max": 34.118019104003906,
      "activations/layer11_attention_weight_min": -36.312740325927734,
      "activations/layer12_attention_weight_max": 26.58899688720703,
      "activations/layer12_attention_weight_min": -26.892709732055664,
      "activations/layer13_attention_weight_max": 39.77577590942383,
      "activations/layer13_attention_weight_min": -36.0063362121582,
      "activations/layer14_attention_weight_max": 41.330135345458984,
      "activations/layer14_attention_weight_min": -38.07756805419922,
      "activations/layer15_attention_weight_max": 38.328189849853516,
      "activations/layer15_attention_weight_min": -36.89118576049805,
      "activations/layer16_attention_weight_max": 30.086652755737305,
      "activations/layer16_attention_weight_min": -28.52690315246582,
      "activations/layer17_attention_weight_max": 29.518531799316406,
      "activations/layer17_attention_weight_min": -28.14812660217285,
      "activations/layer18_attention_weight_max": 31.21192741394043,
      "activations/layer18_attention_weight_min": -26.375659942626953,
      "activations/layer19_attention_weight_max": 31.94467544555664,
      "activations/layer19_attention_weight_min": -31.470651626586914,
      "activations/layer1_attention_weight_max": 16.177104949951172,
      "activations/layer1_attention_weight_min": -14.626448631286621,
      "activations/layer20_attention_weight_max": 28.28466033935547,
      "activations/layer20_attention_weight_min": -26.804786682128906,
      "activations/layer21_attention_weight_max": 26.813732147216797,
      "activations/layer21_attention_weight_min": -25.2637882232666,
      "activations/layer22_attention_weight_max": 39.13751220703125,
      "activations/layer22_attention_weight_min": -31.632295608520508,
      "activations/layer23_attention_weight_max": 28.949382781982422,
      "activations/layer23_attention_weight_min": -27.342273712158203,
      "activations/layer2_attention_weight_max": 31.7337703704834,
      "activations/layer2_attention_weight_min": -30.819982528686523,
      "activations/layer3_attention_weight_max": 98.04254150390625,
      "activations/layer3_attention_weight_min": -98.56072235107422,
      "activations/layer4_attention_weight_max": 105.62235260009766,
      "activations/layer4_attention_weight_min": -108.09162902832031,
      "activations/layer5_attention_weight_max": 48.27030944824219,
      "activations/layer5_attention_weight_min": -61.63202667236328,
      "activations/layer6_attention_weight_max": 44.19917297363281,
      "activations/layer6_attention_weight_min": -46.77534866333008,
      "activations/layer7_attention_weight_max": 97.39488220214844,
      "activations/layer7_attention_weight_min": -97.64823913574219,
      "activations/layer8_attention_weight_max": 40.23180389404297,
      "activations/layer8_attention_weight_min": -44.50514602661133,
      "activations/layer9_attention_weight_max": 30.674560546875,
      "activations/layer9_attention_weight_min": -33.9246940612793,
      "epoch": 20.59,
      "learning_rate": 1.73780303030303e-05,
      "loss": 2.7217,
      "step": 354400
    },
    {
      "activations/layer0_attention_weight_max": 16.438541412353516,
      "activations/layer0_attention_weight_min": -12.847314834594727,
      "activations/layer10_attention_weight_max": 35.18452453613281,
      "activations/layer10_attention_weight_min": -36.511024475097656,
      "activations/layer11_attention_weight_max": 33.6356315612793,
      "activations/layer11_attention_weight_min": -35.54046630859375,
      "activations/layer12_attention_weight_max": 26.44757843017578,
      "activations/layer12_attention_weight_min": -27.192609786987305,
      "activations/layer13_attention_weight_max": 39.93912887573242,
      "activations/layer13_attention_weight_min": -37.06459426879883,
      "activations/layer14_attention_weight_max": 41.970340728759766,
      "activations/layer14_attention_weight_min": -39.45671844482422,
      "activations/layer15_attention_weight_max": 39.634220123291016,
      "activations/layer15_attention_weight_min": -38.244625091552734,
      "activations/layer16_attention_weight_max": 29.68739891052246,
      "activations/layer16_attention_weight_min": -28.51119041442871,
      "activations/layer17_attention_weight_max": 27.18934440612793,
      "activations/layer17_attention_weight_min": -28.462005615234375,
      "activations/layer18_attention_weight_max": 29.62063217163086,
      "activations/layer18_attention_weight_min": -26.454484939575195,
      "activations/layer19_attention_weight_max": 31.486745834350586,
      "activations/layer19_attention_weight_min": -32.300716400146484,
      "activations/layer1_attention_weight_max": 15.615378379821777,
      "activations/layer1_attention_weight_min": -14.07150650024414,
      "activations/layer20_attention_weight_max": 27.39751434326172,
      "activations/layer20_attention_weight_min": -25.953386306762695,
      "activations/layer21_attention_weight_max": 26.22617530822754,
      "activations/layer21_attention_weight_min": -24.657699584960938,
      "activations/layer22_attention_weight_max": 36.54423522949219,
      "activations/layer22_attention_weight_min": -31.410234451293945,
      "activations/layer23_attention_weight_max": 29.32056427001953,
      "activations/layer23_attention_weight_min": -27.283409118652344,
      "activations/layer2_attention_weight_max": 32.525634765625,
      "activations/layer2_attention_weight_min": -31.873260498046875,
      "activations/layer3_attention_weight_max": 102.14463806152344,
      "activations/layer3_attention_weight_min": -100.20349884033203,
      "activations/layer4_attention_weight_max": 101.5943832397461,
      "activations/layer4_attention_weight_min": -106.39696502685547,
      "activations/layer5_attention_weight_max": 48.521728515625,
      "activations/layer5_attention_weight_min": -60.430599212646484,
      "activations/layer6_attention_weight_max": 45.510860443115234,
      "activations/layer6_attention_weight_min": -47.71422576904297,
      "activations/layer7_attention_weight_max": 91.9466552734375,
      "activations/layer7_attention_weight_min": -96.52430725097656,
      "activations/layer8_attention_weight_max": 41.7880973815918,
      "activations/layer8_attention_weight_min": -44.58747100830078,
      "activations/layer9_attention_weight_max": 33.25218963623047,
      "activations/layer9_attention_weight_min": -36.03075408935547,
      "epoch": 20.6,
      "learning_rate": 1.7359090909090908e-05,
      "loss": 2.7361,
      "step": 354450
    },
    {
      "activations/layer0_attention_weight_max": 16.467761993408203,
      "activations/layer0_attention_weight_min": -13.215250015258789,
      "activations/layer10_attention_weight_max": 43.35565948486328,
      "activations/layer10_attention_weight_min": -40.94048309326172,
      "activations/layer11_attention_weight_max": 41.76116180419922,
      "activations/layer11_attention_weight_min": -39.129676818847656,
      "activations/layer12_attention_weight_max": 30.265958786010742,
      "activations/layer12_attention_weight_min": -27.71869659423828,
      "activations/layer13_attention_weight_max": 41.602264404296875,
      "activations/layer13_attention_weight_min": -37.91245651245117,
      "activations/layer14_attention_weight_max": 47.75172805786133,
      "activations/layer14_attention_weight_min": -42.76201248168945,
      "activations/layer15_attention_weight_max": 44.17961883544922,
      "activations/layer15_attention_weight_min": -41.08154296875,
      "activations/layer16_attention_weight_max": 31.91645050048828,
      "activations/layer16_attention_weight_min": -27.75164031982422,
      "activations/layer17_attention_weight_max": 28.23723793029785,
      "activations/layer17_attention_weight_min": -27.276227951049805,
      "activations/layer18_attention_weight_max": 29.921049118041992,
      "activations/layer18_attention_weight_min": -25.714982986450195,
      "activations/layer19_attention_weight_max": 32.155269622802734,
      "activations/layer19_attention_weight_min": -31.59235191345215,
      "activations/layer1_attention_weight_max": 17.492570877075195,
      "activations/layer1_attention_weight_min": -14.62060546875,
      "activations/layer20_attention_weight_max": 27.744041442871094,
      "activations/layer20_attention_weight_min": -25.65249252319336,
      "activations/layer21_attention_weight_max": 26.691972732543945,
      "activations/layer21_attention_weight_min": -26.852201461791992,
      "activations/layer22_attention_weight_max": 37.31357955932617,
      "activations/layer22_attention_weight_min": -31.895444869995117,
      "activations/layer23_attention_weight_max": 30.532079696655273,
      "activations/layer23_attention_weight_min": -25.420095443725586,
      "activations/layer2_attention_weight_max": 35.71766662597656,
      "activations/layer2_attention_weight_min": -34.531333923339844,
      "activations/layer3_attention_weight_max": 106.09903717041016,
      "activations/layer3_attention_weight_min": -111.64212036132812,
      "activations/layer4_attention_weight_max": 106.88590240478516,
      "activations/layer4_attention_weight_min": -109.86842346191406,
      "activations/layer5_attention_weight_max": 50.11537170410156,
      "activations/layer5_attention_weight_min": -63.5648307800293,
      "activations/layer6_attention_weight_max": 44.618873596191406,
      "activations/layer6_attention_weight_min": -49.409400939941406,
      "activations/layer7_attention_weight_max": 112.4525146484375,
      "activations/layer7_attention_weight_min": -103.4381332397461,
      "activations/layer8_attention_weight_max": 50.13475799560547,
      "activations/layer8_attention_weight_min": -48.824119567871094,
      "activations/layer9_attention_weight_max": 38.309471130371094,
      "activations/layer9_attention_weight_min": -39.65677261352539,
      "epoch": 20.6,
      "learning_rate": 1.7340151515151513e-05,
      "loss": 2.7047,
      "step": 354500
    },
    {
      "activations/layer0_attention_weight_max": 15.970765113830566,
      "activations/layer0_attention_weight_min": -13.296643257141113,
      "activations/layer10_attention_weight_max": 34.311580657958984,
      "activations/layer10_attention_weight_min": -34.73797607421875,
      "activations/layer11_attention_weight_max": 35.62163543701172,
      "activations/layer11_attention_weight_min": -34.824012756347656,
      "activations/layer12_attention_weight_max": 26.5805606842041,
      "activations/layer12_attention_weight_min": -25.58768653869629,
      "activations/layer13_attention_weight_max": 38.57442855834961,
      "activations/layer13_attention_weight_min": -37.54039001464844,
      "activations/layer14_attention_weight_max": 41.57355499267578,
      "activations/layer14_attention_weight_min": -38.07423400878906,
      "activations/layer15_attention_weight_max": 38.11811828613281,
      "activations/layer15_attention_weight_min": -35.414390563964844,
      "activations/layer16_attention_weight_max": 28.090110778808594,
      "activations/layer16_attention_weight_min": -28.601285934448242,
      "activations/layer17_attention_weight_max": 29.88673973083496,
      "activations/layer17_attention_weight_min": -25.24127960205078,
      "activations/layer18_attention_weight_max": 34.56178283691406,
      "activations/layer18_attention_weight_min": -25.322641372680664,
      "activations/layer19_attention_weight_max": 35.53782272338867,
      "activations/layer19_attention_weight_min": -30.424392700195312,
      "activations/layer1_attention_weight_max": 15.352629661560059,
      "activations/layer1_attention_weight_min": -14.89243221282959,
      "activations/layer20_attention_weight_max": 32.31067657470703,
      "activations/layer20_attention_weight_min": -24.944217681884766,
      "activations/layer21_attention_weight_max": 34.414581298828125,
      "activations/layer21_attention_weight_min": -24.84343719482422,
      "activations/layer22_attention_weight_max": 43.54196548461914,
      "activations/layer22_attention_weight_min": -30.564146041870117,
      "activations/layer23_attention_weight_max": 31.41145896911621,
      "activations/layer23_attention_weight_min": -26.22978973388672,
      "activations/layer2_attention_weight_max": 33.46754837036133,
      "activations/layer2_attention_weight_min": -31.94027328491211,
      "activations/layer3_attention_weight_max": 106.07029724121094,
      "activations/layer3_attention_weight_min": -103.237060546875,
      "activations/layer4_attention_weight_max": 104.9312515258789,
      "activations/layer4_attention_weight_min": -106.9181900024414,
      "activations/layer5_attention_weight_max": 49.113685607910156,
      "activations/layer5_attention_weight_min": -61.92664337158203,
      "activations/layer6_attention_weight_max": 44.1791877746582,
      "activations/layer6_attention_weight_min": -46.97981643676758,
      "activations/layer7_attention_weight_max": 94.12593078613281,
      "activations/layer7_attention_weight_min": -100.39521789550781,
      "activations/layer8_attention_weight_max": 42.9791145324707,
      "activations/layer8_attention_weight_min": -43.947425842285156,
      "activations/layer9_attention_weight_max": 33.32857894897461,
      "activations/layer9_attention_weight_min": -33.954986572265625,
      "epoch": 20.6,
      "learning_rate": 1.732121212121212e-05,
      "loss": 2.7271,
      "step": 354550
    },
    {
      "activations/layer0_attention_weight_max": 16.738616943359375,
      "activations/layer0_attention_weight_min": -13.368786811828613,
      "activations/layer10_attention_weight_max": 34.47035217285156,
      "activations/layer10_attention_weight_min": -36.1907958984375,
      "activations/layer11_attention_weight_max": 34.80039978027344,
      "activations/layer11_attention_weight_min": -36.62352752685547,
      "activations/layer12_attention_weight_max": 28.783584594726562,
      "activations/layer12_attention_weight_min": -29.59265899658203,
      "activations/layer13_attention_weight_max": 41.09528350830078,
      "activations/layer13_attention_weight_min": -39.567138671875,
      "activations/layer14_attention_weight_max": 44.4196891784668,
      "activations/layer14_attention_weight_min": -40.84703063964844,
      "activations/layer15_attention_weight_max": 42.00933074951172,
      "activations/layer15_attention_weight_min": -38.28548049926758,
      "activations/layer16_attention_weight_max": 30.70662498474121,
      "activations/layer16_attention_weight_min": -29.689414978027344,
      "activations/layer17_attention_weight_max": 30.565555572509766,
      "activations/layer17_attention_weight_min": -26.659778594970703,
      "activations/layer18_attention_weight_max": 32.39958953857422,
      "activations/layer18_attention_weight_min": -26.990068435668945,
      "activations/layer19_attention_weight_max": 40.37125015258789,
      "activations/layer19_attention_weight_min": -33.44700622558594,
      "activations/layer1_attention_weight_max": 15.023552894592285,
      "activations/layer1_attention_weight_min": -14.853689193725586,
      "activations/layer20_attention_weight_max": 33.13173294067383,
      "activations/layer20_attention_weight_min": -25.439085006713867,
      "activations/layer21_attention_weight_max": 32.07594299316406,
      "activations/layer21_attention_weight_min": -25.79781150817871,
      "activations/layer22_attention_weight_max": 45.250450134277344,
      "activations/layer22_attention_weight_min": -33.38539505004883,
      "activations/layer23_attention_weight_max": 34.379058837890625,
      "activations/layer23_attention_weight_min": -26.32369041442871,
      "activations/layer2_attention_weight_max": 32.85530471801758,
      "activations/layer2_attention_weight_min": -32.390968322753906,
      "activations/layer3_attention_weight_max": 107.33443450927734,
      "activations/layer3_attention_weight_min": -108.55046844482422,
      "activations/layer4_attention_weight_max": 108.29188537597656,
      "activations/layer4_attention_weight_min": -110.66533660888672,
      "activations/layer5_attention_weight_max": 51.62236022949219,
      "activations/layer5_attention_weight_min": -61.125545501708984,
      "activations/layer6_attention_weight_max": 44.75137710571289,
      "activations/layer6_attention_weight_min": -48.03759002685547,
      "activations/layer7_attention_weight_max": 97.3864974975586,
      "activations/layer7_attention_weight_min": -96.09184265136719,
      "activations/layer8_attention_weight_max": 46.59423828125,
      "activations/layer8_attention_weight_min": -45.9698600769043,
      "activations/layer9_attention_weight_max": 31.633914947509766,
      "activations/layer9_attention_weight_min": -36.78054428100586,
      "epoch": 20.6,
      "learning_rate": 1.7302272727272726e-05,
      "loss": 2.7098,
      "step": 354600
    },
    {
      "activations/layer0_attention_weight_max": 16.12795639038086,
      "activations/layer0_attention_weight_min": -12.692824363708496,
      "activations/layer10_attention_weight_max": 36.32564926147461,
      "activations/layer10_attention_weight_min": -37.8071174621582,
      "activations/layer11_attention_weight_max": 36.36957550048828,
      "activations/layer11_attention_weight_min": -36.7342529296875,
      "activations/layer12_attention_weight_max": 30.508296966552734,
      "activations/layer12_attention_weight_min": -28.40294647216797,
      "activations/layer13_attention_weight_max": 45.21031188964844,
      "activations/layer13_attention_weight_min": -37.886104583740234,
      "activations/layer14_attention_weight_max": 49.5983772277832,
      "activations/layer14_attention_weight_min": -41.624610900878906,
      "activations/layer15_attention_weight_max": 45.26335144042969,
      "activations/layer15_attention_weight_min": -40.450714111328125,
      "activations/layer16_attention_weight_max": 34.88603591918945,
      "activations/layer16_attention_weight_min": -28.413578033447266,
      "activations/layer17_attention_weight_max": 31.794416427612305,
      "activations/layer17_attention_weight_min": -26.39192771911621,
      "activations/layer18_attention_weight_max": 33.725379943847656,
      "activations/layer18_attention_weight_min": -30.463842391967773,
      "activations/layer19_attention_weight_max": 44.158992767333984,
      "activations/layer19_attention_weight_min": -39.65477752685547,
      "activations/layer1_attention_weight_max": 15.904899597167969,
      "activations/layer1_attention_weight_min": -16.066038131713867,
      "activations/layer20_attention_weight_max": 33.158809661865234,
      "activations/layer20_attention_weight_min": -30.445859909057617,
      "activations/layer21_attention_weight_max": 31.95542335510254,
      "activations/layer21_attention_weight_min": -29.96116828918457,
      "activations/layer22_attention_weight_max": 43.1286506652832,
      "activations/layer22_attention_weight_min": -40.17382049560547,
      "activations/layer23_attention_weight_max": 33.206382751464844,
      "activations/layer23_attention_weight_min": -30.48640251159668,
      "activations/layer2_attention_weight_max": 34.19325256347656,
      "activations/layer2_attention_weight_min": -33.533447265625,
      "activations/layer3_attention_weight_max": 104.94001007080078,
      "activations/layer3_attention_weight_min": -109.10182189941406,
      "activations/layer4_attention_weight_max": 109.04552459716797,
      "activations/layer4_attention_weight_min": -113.57134246826172,
      "activations/layer5_attention_weight_max": 54.719932556152344,
      "activations/layer5_attention_weight_min": -64.51127624511719,
      "activations/layer6_attention_weight_max": 47.62495040893555,
      "activations/layer6_attention_weight_min": -50.31185531616211,
      "activations/layer7_attention_weight_max": 106.11556243896484,
      "activations/layer7_attention_weight_min": -105.37688446044922,
      "activations/layer8_attention_weight_max": 46.787410736083984,
      "activations/layer8_attention_weight_min": -46.05200958251953,
      "activations/layer9_attention_weight_max": 33.18526840209961,
      "activations/layer9_attention_weight_min": -35.286251068115234,
      "epoch": 20.61,
      "learning_rate": 1.728333333333333e-05,
      "loss": 2.7077,
      "step": 354650
    },
    {
      "activations/layer0_attention_weight_max": 17.447763442993164,
      "activations/layer0_attention_weight_min": -14.06910514831543,
      "activations/layer10_attention_weight_max": 33.20457458496094,
      "activations/layer10_attention_weight_min": -35.412933349609375,
      "activations/layer11_attention_weight_max": 34.505332946777344,
      "activations/layer11_attention_weight_min": -36.68928146362305,
      "activations/layer12_attention_weight_max": 26.060373306274414,
      "activations/layer12_attention_weight_min": -28.559009552001953,
      "activations/layer13_attention_weight_max": 38.89867401123047,
      "activations/layer13_attention_weight_min": -41.67390441894531,
      "activations/layer14_attention_weight_max": 41.73003387451172,
      "activations/layer14_attention_weight_min": -40.28095245361328,
      "activations/layer15_attention_weight_max": 37.45178985595703,
      "activations/layer15_attention_weight_min": -37.403560638427734,
      "activations/layer16_attention_weight_max": 29.505706787109375,
      "activations/layer16_attention_weight_min": -28.888479232788086,
      "activations/layer17_attention_weight_max": 28.00380516052246,
      "activations/layer17_attention_weight_min": -26.202985763549805,
      "activations/layer18_attention_weight_max": 30.43858528137207,
      "activations/layer18_attention_weight_min": -27.52312660217285,
      "activations/layer19_attention_weight_max": 37.31167221069336,
      "activations/layer19_attention_weight_min": -32.67769241333008,
      "activations/layer1_attention_weight_max": 15.631173133850098,
      "activations/layer1_attention_weight_min": -15.043251991271973,
      "activations/layer20_attention_weight_max": 30.170541763305664,
      "activations/layer20_attention_weight_min": -25.5462589263916,
      "activations/layer21_attention_weight_max": 28.308603286743164,
      "activations/layer21_attention_weight_min": -25.387176513671875,
      "activations/layer22_attention_weight_max": 38.713802337646484,
      "activations/layer22_attention_weight_min": -35.33306884765625,
      "activations/layer23_attention_weight_max": 33.07999038696289,
      "activations/layer23_attention_weight_min": -27.409564971923828,
      "activations/layer2_attention_weight_max": 34.62296676635742,
      "activations/layer2_attention_weight_min": -32.2154541015625,
      "activations/layer3_attention_weight_max": 101.08528137207031,
      "activations/layer3_attention_weight_min": -104.494384765625,
      "activations/layer4_attention_weight_max": 106.08577728271484,
      "activations/layer4_attention_weight_min": -109.63805389404297,
      "activations/layer5_attention_weight_max": 51.264766693115234,
      "activations/layer5_attention_weight_min": -61.19753646850586,
      "activations/layer6_attention_weight_max": 44.739383697509766,
      "activations/layer6_attention_weight_min": -47.8768196105957,
      "activations/layer7_attention_weight_max": 93.25316619873047,
      "activations/layer7_attention_weight_min": -100.48916625976562,
      "activations/layer8_attention_weight_max": 40.88764572143555,
      "activations/layer8_attention_weight_min": -47.34227752685547,
      "activations/layer9_attention_weight_max": 32.19724655151367,
      "activations/layer9_attention_weight_min": -35.31589889526367,
      "epoch": 20.61,
      "learning_rate": 1.7264772727272725e-05,
      "loss": 2.7122,
      "step": 354700
    },
    {
      "activations/layer0_attention_weight_max": 16.665746688842773,
      "activations/layer0_attention_weight_min": -12.882579803466797,
      "activations/layer10_attention_weight_max": 35.93260192871094,
      "activations/layer10_attention_weight_min": -35.9815673828125,
      "activations/layer11_attention_weight_max": 37.20595169067383,
      "activations/layer11_attention_weight_min": -36.45650100708008,
      "activations/layer12_attention_weight_max": 29.170700073242188,
      "activations/layer12_attention_weight_min": -28.32417106628418,
      "activations/layer13_attention_weight_max": 42.2448616027832,
      "activations/layer13_attention_weight_min": -38.37718200683594,
      "activations/layer14_attention_weight_max": 44.82553482055664,
      "activations/layer14_attention_weight_min": -41.3378791809082,
      "activations/layer15_attention_weight_max": 40.89488220214844,
      "activations/layer15_attention_weight_min": -37.07236862182617,
      "activations/layer16_attention_weight_max": 31.081209182739258,
      "activations/layer16_attention_weight_min": -30.581478118896484,
      "activations/layer17_attention_weight_max": 30.289886474609375,
      "activations/layer17_attention_weight_min": -28.302631378173828,
      "activations/layer18_attention_weight_max": 32.97483444213867,
      "activations/layer18_attention_weight_min": -29.54153060913086,
      "activations/layer19_attention_weight_max": 35.127567291259766,
      "activations/layer19_attention_weight_min": -31.836734771728516,
      "activations/layer1_attention_weight_max": 16.274944305419922,
      "activations/layer1_attention_weight_min": -17.307138442993164,
      "activations/layer20_attention_weight_max": 29.37419319152832,
      "activations/layer20_attention_weight_min": -25.91902732849121,
      "activations/layer21_attention_weight_max": 29.21672821044922,
      "activations/layer21_attention_weight_min": -25.987585067749023,
      "activations/layer22_attention_weight_max": 49.03786849975586,
      "activations/layer22_attention_weight_min": -32.69049835205078,
      "activations/layer23_attention_weight_max": 38.09214782714844,
      "activations/layer23_attention_weight_min": -27.511573791503906,
      "activations/layer2_attention_weight_max": 34.49895477294922,
      "activations/layer2_attention_weight_min": -33.15561294555664,
      "activations/layer3_attention_weight_max": 103.06166076660156,
      "activations/layer3_attention_weight_min": -104.76144409179688,
      "activations/layer4_attention_weight_max": 110.0559310913086,
      "activations/layer4_attention_weight_min": -116.64320373535156,
      "activations/layer5_attention_weight_max": 51.66909408569336,
      "activations/layer5_attention_weight_min": -63.456878662109375,
      "activations/layer6_attention_weight_max": 46.145076751708984,
      "activations/layer6_attention_weight_min": -50.12421798706055,
      "activations/layer7_attention_weight_max": 101.72350311279297,
      "activations/layer7_attention_weight_min": -104.4981460571289,
      "activations/layer8_attention_weight_max": 45.277713775634766,
      "activations/layer8_attention_weight_min": -48.879791259765625,
      "activations/layer9_attention_weight_max": 33.26542663574219,
      "activations/layer9_attention_weight_min": -35.64402770996094,
      "epoch": 20.61,
      "learning_rate": 1.7245833333333334e-05,
      "loss": 2.7189,
      "step": 354750
    },
    {
      "activations/layer0_attention_weight_max": 15.41818904876709,
      "activations/layer0_attention_weight_min": -12.773250579833984,
      "activations/layer10_attention_weight_max": 35.23286437988281,
      "activations/layer10_attention_weight_min": -35.31104278564453,
      "activations/layer11_attention_weight_max": 34.482696533203125,
      "activations/layer11_attention_weight_min": -34.38053894042969,
      "activations/layer12_attention_weight_max": 26.809696197509766,
      "activations/layer12_attention_weight_min": -26.792512893676758,
      "activations/layer13_attention_weight_max": 39.7612190246582,
      "activations/layer13_attention_weight_min": -37.410125732421875,
      "activations/layer14_attention_weight_max": 45.10980987548828,
      "activations/layer14_attention_weight_min": -39.841819763183594,
      "activations/layer15_attention_weight_max": 39.34196853637695,
      "activations/layer15_attention_weight_min": -35.671363830566406,
      "activations/layer16_attention_weight_max": 29.349523544311523,
      "activations/layer16_attention_weight_min": -26.548274993896484,
      "activations/layer17_attention_weight_max": 30.22243309020996,
      "activations/layer17_attention_weight_min": -26.38517189025879,
      "activations/layer18_attention_weight_max": 33.755332946777344,
      "activations/layer18_attention_weight_min": -28.251615524291992,
      "activations/layer19_attention_weight_max": 35.8770751953125,
      "activations/layer19_attention_weight_min": -33.34397506713867,
      "activations/layer1_attention_weight_max": 15.327240943908691,
      "activations/layer1_attention_weight_min": -14.330122947692871,
      "activations/layer20_attention_weight_max": 32.30470657348633,
      "activations/layer20_attention_weight_min": -26.655492782592773,
      "activations/layer21_attention_weight_max": 32.11397171020508,
      "activations/layer21_attention_weight_min": -26.69490623474121,
      "activations/layer22_attention_weight_max": 46.511444091796875,
      "activations/layer22_attention_weight_min": -34.35471725463867,
      "activations/layer23_attention_weight_max": 34.45557403564453,
      "activations/layer23_attention_weight_min": -27.293148040771484,
      "activations/layer2_attention_weight_max": 34.426979064941406,
      "activations/layer2_attention_weight_min": -32.02085494995117,
      "activations/layer3_attention_weight_max": 105.55860900878906,
      "activations/layer3_attention_weight_min": -104.34362030029297,
      "activations/layer4_attention_weight_max": 108.12158203125,
      "activations/layer4_attention_weight_min": -110.96102142333984,
      "activations/layer5_attention_weight_max": 49.61788558959961,
      "activations/layer5_attention_weight_min": -61.153594970703125,
      "activations/layer6_attention_weight_max": 44.04049301147461,
      "activations/layer6_attention_weight_min": -48.04719543457031,
      "activations/layer7_attention_weight_max": 96.984130859375,
      "activations/layer7_attention_weight_min": -102.29907989501953,
      "activations/layer8_attention_weight_max": 41.82393264770508,
      "activations/layer8_attention_weight_min": -44.77965545654297,
      "activations/layer9_attention_weight_max": 32.5645866394043,
      "activations/layer9_attention_weight_min": -36.409523010253906,
      "epoch": 20.62,
      "learning_rate": 1.722689393939394e-05,
      "loss": 2.7092,
      "step": 354800
    },
    {
      "activations/layer0_attention_weight_max": 16.258708953857422,
      "activations/layer0_attention_weight_min": -13.638097763061523,
      "activations/layer10_attention_weight_max": 35.539947509765625,
      "activations/layer10_attention_weight_min": -35.620426177978516,
      "activations/layer11_attention_weight_max": 35.359039306640625,
      "activations/layer11_attention_weight_min": -34.44266128540039,
      "activations/layer12_attention_weight_max": 27.574644088745117,
      "activations/layer12_attention_weight_min": -27.442934036254883,
      "activations/layer13_attention_weight_max": 43.106956481933594,
      "activations/layer13_attention_weight_min": -39.90633773803711,
      "activations/layer14_attention_weight_max": 43.255008697509766,
      "activations/layer14_attention_weight_min": -43.96147537231445,
      "activations/layer15_attention_weight_max": 42.392765045166016,
      "activations/layer15_attention_weight_min": -38.53511428833008,
      "activations/layer16_attention_weight_max": 30.536054611206055,
      "activations/layer16_attention_weight_min": -29.73954963684082,
      "activations/layer17_attention_weight_max": 31.080318450927734,
      "activations/layer17_attention_weight_min": -27.76620101928711,
      "activations/layer18_attention_weight_max": 33.21833419799805,
      "activations/layer18_attention_weight_min": -28.852365493774414,
      "activations/layer19_attention_weight_max": 34.85155487060547,
      "activations/layer19_attention_weight_min": -33.837162017822266,
      "activations/layer1_attention_weight_max": 15.512749671936035,
      "activations/layer1_attention_weight_min": -14.532511711120605,
      "activations/layer20_attention_weight_max": 29.290210723876953,
      "activations/layer20_attention_weight_min": -27.666519165039062,
      "activations/layer21_attention_weight_max": 29.801143646240234,
      "activations/layer21_attention_weight_min": -26.226444244384766,
      "activations/layer22_attention_weight_max": 42.31303787231445,
      "activations/layer22_attention_weight_min": -34.67875289916992,
      "activations/layer23_attention_weight_max": 30.856029510498047,
      "activations/layer23_attention_weight_min": -27.806503295898438,
      "activations/layer2_attention_weight_max": 33.47013854980469,
      "activations/layer2_attention_weight_min": -31.60109519958496,
      "activations/layer3_attention_weight_max": 100.01695251464844,
      "activations/layer3_attention_weight_min": -101.70442199707031,
      "activations/layer4_attention_weight_max": 106.02210998535156,
      "activations/layer4_attention_weight_min": -107.3732681274414,
      "activations/layer5_attention_weight_max": 53.67289352416992,
      "activations/layer5_attention_weight_min": -59.68779754638672,
      "activations/layer6_attention_weight_max": 44.096595764160156,
      "activations/layer6_attention_weight_min": -46.96458053588867,
      "activations/layer7_attention_weight_max": 92.1767807006836,
      "activations/layer7_attention_weight_min": -97.85774230957031,
      "activations/layer8_attention_weight_max": 42.0289192199707,
      "activations/layer8_attention_weight_min": -45.25459289550781,
      "activations/layer9_attention_weight_max": 32.83388900756836,
      "activations/layer9_attention_weight_min": -34.099037170410156,
      "epoch": 20.62,
      "learning_rate": 1.7207954545454543e-05,
      "loss": 2.7137,
      "step": 354850
    },
    {
      "activations/layer0_attention_weight_max": 16.255088806152344,
      "activations/layer0_attention_weight_min": -12.543184280395508,
      "activations/layer10_attention_weight_max": 36.16291046142578,
      "activations/layer10_attention_weight_min": -36.60926055908203,
      "activations/layer11_attention_weight_max": 35.788482666015625,
      "activations/layer11_attention_weight_min": -36.355751037597656,
      "activations/layer12_attention_weight_max": 27.526830673217773,
      "activations/layer12_attention_weight_min": -27.467914581298828,
      "activations/layer13_attention_weight_max": 42.828102111816406,
      "activations/layer13_attention_weight_min": -40.500450134277344,
      "activations/layer14_attention_weight_max": 48.46503448486328,
      "activations/layer14_attention_weight_min": -42.138343811035156,
      "activations/layer15_attention_weight_max": 41.959007263183594,
      "activations/layer15_attention_weight_min": -37.693824768066406,
      "activations/layer16_attention_weight_max": 33.55780029296875,
      "activations/layer16_attention_weight_min": -30.828670501708984,
      "activations/layer17_attention_weight_max": 36.16769027709961,
      "activations/layer17_attention_weight_min": -28.974342346191406,
      "activations/layer18_attention_weight_max": 46.79658508300781,
      "activations/layer18_attention_weight_min": -30.765316009521484,
      "activations/layer19_attention_weight_max": 45.648345947265625,
      "activations/layer19_attention_weight_min": -34.770896911621094,
      "activations/layer1_attention_weight_max": 16.043704986572266,
      "activations/layer1_attention_weight_min": -15.728490829467773,
      "activations/layer20_attention_weight_max": 36.465476989746094,
      "activations/layer20_attention_weight_min": -27.38310432434082,
      "activations/layer21_attention_weight_max": 38.17868423461914,
      "activations/layer21_attention_weight_min": -27.94492530822754,
      "activations/layer22_attention_weight_max": 56.1895751953125,
      "activations/layer22_attention_weight_min": -37.68708801269531,
      "activations/layer23_attention_weight_max": 41.52947998046875,
      "activations/layer23_attention_weight_min": -26.448360443115234,
      "activations/layer2_attention_weight_max": 32.83785629272461,
      "activations/layer2_attention_weight_min": -33.24235916137695,
      "activations/layer3_attention_weight_max": 106.91792297363281,
      "activations/layer3_attention_weight_min": -109.50442504882812,
      "activations/layer4_attention_weight_max": 108.52300262451172,
      "activations/layer4_attention_weight_min": -111.98480224609375,
      "activations/layer5_attention_weight_max": 50.25092315673828,
      "activations/layer5_attention_weight_min": -61.58280563354492,
      "activations/layer6_attention_weight_max": 47.047462463378906,
      "activations/layer6_attention_weight_min": -47.838687896728516,
      "activations/layer7_attention_weight_max": 99.55203247070312,
      "activations/layer7_attention_weight_min": -103.90049743652344,
      "activations/layer8_attention_weight_max": 43.05658721923828,
      "activations/layer8_attention_weight_min": -47.899139404296875,
      "activations/layer9_attention_weight_max": 33.994441986083984,
      "activations/layer9_attention_weight_min": -36.56044006347656,
      "epoch": 20.62,
      "learning_rate": 1.718901515151515e-05,
      "loss": 2.7118,
      "step": 354900
    },
    {
      "activations/layer0_attention_weight_max": 15.974899291992188,
      "activations/layer0_attention_weight_min": -14.618485450744629,
      "activations/layer10_attention_weight_max": 38.76655960083008,
      "activations/layer10_attention_weight_min": -36.2099609375,
      "activations/layer11_attention_weight_max": 37.222652435302734,
      "activations/layer11_attention_weight_min": -35.71465301513672,
      "activations/layer12_attention_weight_max": 26.163204193115234,
      "activations/layer12_attention_weight_min": -27.546205520629883,
      "activations/layer13_attention_weight_max": 39.12251663208008,
      "activations/layer13_attention_weight_min": -39.530208587646484,
      "activations/layer14_attention_weight_max": 45.87452697753906,
      "activations/layer14_attention_weight_min": -42.99407196044922,
      "activations/layer15_attention_weight_max": 39.14333724975586,
      "activations/layer15_attention_weight_min": -39.50309753417969,
      "activations/layer16_attention_weight_max": 29.505704879760742,
      "activations/layer16_attention_weight_min": -30.602807998657227,
      "activations/layer17_attention_weight_max": 29.429508209228516,
      "activations/layer17_attention_weight_min": -25.47739028930664,
      "activations/layer18_attention_weight_max": 31.811176300048828,
      "activations/layer18_attention_weight_min": -24.685657501220703,
      "activations/layer19_attention_weight_max": 33.67282485961914,
      "activations/layer19_attention_weight_min": -31.636611938476562,
      "activations/layer1_attention_weight_max": 15.594070434570312,
      "activations/layer1_attention_weight_min": -14.79947566986084,
      "activations/layer20_attention_weight_max": 28.621313095092773,
      "activations/layer20_attention_weight_min": -26.369230270385742,
      "activations/layer21_attention_weight_max": 25.593467712402344,
      "activations/layer21_attention_weight_min": -25.59141731262207,
      "activations/layer22_attention_weight_max": 36.98535919189453,
      "activations/layer22_attention_weight_min": -31.129426956176758,
      "activations/layer23_attention_weight_max": 28.783002853393555,
      "activations/layer23_attention_weight_min": -25.91938018798828,
      "activations/layer2_attention_weight_max": 34.26006317138672,
      "activations/layer2_attention_weight_min": -32.73634338378906,
      "activations/layer3_attention_weight_max": 103.08114624023438,
      "activations/layer3_attention_weight_min": -102.9795913696289,
      "activations/layer4_attention_weight_max": 106.93009185791016,
      "activations/layer4_attention_weight_min": -106.2918472290039,
      "activations/layer5_attention_weight_max": 48.33358383178711,
      "activations/layer5_attention_weight_min": -61.39482116699219,
      "activations/layer6_attention_weight_max": 45.042396545410156,
      "activations/layer6_attention_weight_min": -46.553916931152344,
      "activations/layer7_attention_weight_max": 105.9698486328125,
      "activations/layer7_attention_weight_min": -101.35514068603516,
      "activations/layer8_attention_weight_max": 45.733123779296875,
      "activations/layer8_attention_weight_min": -46.29286193847656,
      "activations/layer9_attention_weight_max": 33.992305755615234,
      "activations/layer9_attention_weight_min": -36.54460525512695,
      "epoch": 20.62,
      "learning_rate": 1.7170075757575756e-05,
      "loss": 2.7093,
      "step": 354950
    },
    {
      "activations/layer0_attention_weight_max": 16.25541114807129,
      "activations/layer0_attention_weight_min": -13.296796798706055,
      "activations/layer10_attention_weight_max": 39.072357177734375,
      "activations/layer10_attention_weight_min": -38.04376220703125,
      "activations/layer11_attention_weight_max": 37.82307434082031,
      "activations/layer11_attention_weight_min": -37.554954528808594,
      "activations/layer12_attention_weight_max": 28.281782150268555,
      "activations/layer12_attention_weight_min": -29.792518615722656,
      "activations/layer13_attention_weight_max": 44.25017166137695,
      "activations/layer13_attention_weight_min": -41.06227493286133,
      "activations/layer14_attention_weight_max": 48.718780517578125,
      "activations/layer14_attention_weight_min": -44.91069793701172,
      "activations/layer15_attention_weight_max": 44.60869598388672,
      "activations/layer15_attention_weight_min": -42.747276306152344,
      "activations/layer16_attention_weight_max": 33.90095901489258,
      "activations/layer16_attention_weight_min": -30.548770904541016,
      "activations/layer17_attention_weight_max": 33.35919189453125,
      "activations/layer17_attention_weight_min": -29.58268165588379,
      "activations/layer18_attention_weight_max": 35.01573944091797,
      "activations/layer18_attention_weight_min": -30.614599227905273,
      "activations/layer19_attention_weight_max": 35.2003288269043,
      "activations/layer19_attention_weight_min": -33.63749694824219,
      "activations/layer1_attention_weight_max": 15.890588760375977,
      "activations/layer1_attention_weight_min": -15.007983207702637,
      "activations/layer20_attention_weight_max": 27.75874900817871,
      "activations/layer20_attention_weight_min": -26.433576583862305,
      "activations/layer21_attention_weight_max": 26.270442962646484,
      "activations/layer21_attention_weight_min": -26.333492279052734,
      "activations/layer22_attention_weight_max": 42.13875198364258,
      "activations/layer22_attention_weight_min": -32.31220245361328,
      "activations/layer23_attention_weight_max": 31.242935180664062,
      "activations/layer23_attention_weight_min": -27.186172485351562,
      "activations/layer2_attention_weight_max": 33.13661193847656,
      "activations/layer2_attention_weight_min": -33.99040603637695,
      "activations/layer3_attention_weight_max": 101.90269470214844,
      "activations/layer3_attention_weight_min": -103.61172485351562,
      "activations/layer4_attention_weight_max": 108.88581848144531,
      "activations/layer4_attention_weight_min": -113.17333221435547,
      "activations/layer5_attention_weight_max": 51.6025505065918,
      "activations/layer5_attention_weight_min": -62.5900993347168,
      "activations/layer6_attention_weight_max": 45.598751068115234,
      "activations/layer6_attention_weight_min": -51.0896110534668,
      "activations/layer7_attention_weight_max": 96.77529907226562,
      "activations/layer7_attention_weight_min": -102.94229125976562,
      "activations/layer8_attention_weight_max": 43.67444610595703,
      "activations/layer8_attention_weight_min": -47.590736389160156,
      "activations/layer9_attention_weight_max": 35.816925048828125,
      "activations/layer9_attention_weight_min": -41.010433197021484,
      "epoch": 20.63,
      "learning_rate": 1.715113636363636e-05,
      "loss": 2.7031,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4619,
      "eval_samples_per_second": 507.452,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4619,
      "eval_openwebtext_samples_per_second": 507.452,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9696,
      "eval_wikitext_samples_per_second": 231.523,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_lambada_loss": 2.35546875,
      "eval_lambada_ppl": 10.54306976753373,
      "eval_lambada_runtime": 9.5356,
      "eval_lambada_samples_per_second": 510.614,
      "step": 355000
    },
    {
      "activations/layer0_attention_weight_max": 15.570261001586914,
      "activations/layer0_attention_weight_min": -14.865615844726562,
      "activations/layer10_attention_weight_max": 33.89021301269531,
      "activations/layer10_attention_weight_min": -33.80216979980469,
      "activations/layer11_attention_weight_max": 34.50211715698242,
      "activations/layer11_attention_weight_min": -34.1720085144043,
      "activations/layer12_attention_weight_max": 27.018884658813477,
      "activations/layer12_attention_weight_min": -26.932540893554688,
      "activations/layer13_attention_weight_max": 41.29658889770508,
      "activations/layer13_attention_weight_min": -35.88336181640625,
      "activations/layer14_attention_weight_max": 47.02996826171875,
      "activations/layer14_attention_weight_min": -37.19804763793945,
      "activations/layer15_attention_weight_max": 41.41950988769531,
      "activations/layer15_attention_weight_min": -35.54671859741211,
      "activations/layer16_attention_weight_max": 29.27876853942871,
      "activations/layer16_attention_weight_min": -28.69340705871582,
      "activations/layer17_attention_weight_max": 29.75376319885254,
      "activations/layer17_attention_weight_min": -27.749935150146484,
      "activations/layer18_attention_weight_max": 30.257545471191406,
      "activations/layer18_attention_weight_min": -25.06330108642578,
      "activations/layer19_attention_weight_max": 34.191646575927734,
      "activations/layer19_attention_weight_min": -33.71843719482422,
      "activations/layer1_attention_weight_max": 15.997230529785156,
      "activations/layer1_attention_weight_min": -14.885725021362305,
      "activations/layer20_attention_weight_max": 28.8158016204834,
      "activations/layer20_attention_weight_min": -26.790693283081055,
      "activations/layer21_attention_weight_max": 26.833160400390625,
      "activations/layer21_attention_weight_min": -26.153005599975586,
      "activations/layer22_attention_weight_max": 38.876670837402344,
      "activations/layer22_attention_weight_min": -33.68647766113281,
      "activations/layer23_attention_weight_max": 30.940732955932617,
      "activations/layer23_attention_weight_min": -27.00606918334961,
      "activations/layer2_attention_weight_max": 32.326568603515625,
      "activations/layer2_attention_weight_min": -32.03626251220703,
      "activations/layer3_attention_weight_max": 103.37677764892578,
      "activations/layer3_attention_weight_min": -104.49830627441406,
      "activations/layer4_attention_weight_max": 104.52235412597656,
      "activations/layer4_attention_weight_min": -107.1687240600586,
      "activations/layer5_attention_weight_max": 51.146629333496094,
      "activations/layer5_attention_weight_min": -61.14738082885742,
      "activations/layer6_attention_weight_max": 44.619266510009766,
      "activations/layer6_attention_weight_min": -45.2638053894043,
      "activations/layer7_attention_weight_max": 93.4434814453125,
      "activations/layer7_attention_weight_min": -96.80133056640625,
      "activations/layer8_attention_weight_max": 43.908958435058594,
      "activations/layer8_attention_weight_min": -44.15886688232422,
      "activations/layer9_attention_weight_max": 33.40659713745117,
      "activations/layer9_attention_weight_min": -34.870872497558594,
      "epoch": 20.63,
      "learning_rate": 1.713219696969697e-05,
      "loss": 2.7034,
      "step": 355050
    },
    {
      "activations/layer0_attention_weight_max": 16.34272003173828,
      "activations/layer0_attention_weight_min": -13.27467155456543,
      "activations/layer10_attention_weight_max": 34.23257827758789,
      "activations/layer10_attention_weight_min": -34.77214050292969,
      "activations/layer11_attention_weight_max": 33.40683364868164,
      "activations/layer11_attention_weight_min": -35.198848724365234,
      "activations/layer12_attention_weight_max": 27.338088989257812,
      "activations/layer12_attention_weight_min": -26.203229904174805,
      "activations/layer13_attention_weight_max": 40.454498291015625,
      "activations/layer13_attention_weight_min": -36.29910659790039,
      "activations/layer14_attention_weight_max": 42.7198600769043,
      "activations/layer14_attention_weight_min": -36.59931945800781,
      "activations/layer15_attention_weight_max": 39.913047790527344,
      "activations/layer15_attention_weight_min": -36.37324142456055,
      "activations/layer16_attention_weight_max": 30.333614349365234,
      "activations/layer16_attention_weight_min": -28.579648971557617,
      "activations/layer17_attention_weight_max": 31.170467376708984,
      "activations/layer17_attention_weight_min": -26.279708862304688,
      "activations/layer18_attention_weight_max": 33.65850067138672,
      "activations/layer18_attention_weight_min": -26.47700309753418,
      "activations/layer19_attention_weight_max": 34.55394744873047,
      "activations/layer19_attention_weight_min": -33.829524993896484,
      "activations/layer1_attention_weight_max": 15.311964988708496,
      "activations/layer1_attention_weight_min": -14.040604591369629,
      "activations/layer20_attention_weight_max": 30.375436782836914,
      "activations/layer20_attention_weight_min": -24.01462745666504,
      "activations/layer21_attention_weight_max": 32.24871826171875,
      "activations/layer21_attention_weight_min": -27.10324478149414,
      "activations/layer22_attention_weight_max": 42.69709014892578,
      "activations/layer22_attention_weight_min": -33.20791244506836,
      "activations/layer23_attention_weight_max": 34.35662841796875,
      "activations/layer23_attention_weight_min": -27.272159576416016,
      "activations/layer2_attention_weight_max": 32.791988372802734,
      "activations/layer2_attention_weight_min": -32.78916931152344,
      "activations/layer3_attention_weight_max": 107.26457214355469,
      "activations/layer3_attention_weight_min": -102.39437103271484,
      "activations/layer4_attention_weight_max": 116.3008804321289,
      "activations/layer4_attention_weight_min": -107.40565490722656,
      "activations/layer5_attention_weight_max": 48.67768859863281,
      "activations/layer5_attention_weight_min": -61.93171310424805,
      "activations/layer6_attention_weight_max": 44.44906997680664,
      "activations/layer6_attention_weight_min": -46.932777404785156,
      "activations/layer7_attention_weight_max": 93.03631591796875,
      "activations/layer7_attention_weight_min": -94.7175064086914,
      "activations/layer8_attention_weight_max": 40.0710563659668,
      "activations/layer8_attention_weight_min": -43.544158935546875,
      "activations/layer9_attention_weight_max": 32.220455169677734,
      "activations/layer9_attention_weight_min": -33.999732971191406,
      "epoch": 20.63,
      "learning_rate": 1.7113257575757574e-05,
      "loss": 2.7076,
      "step": 355100
    },
    {
      "activations/layer0_attention_weight_max": 16.51186752319336,
      "activations/layer0_attention_weight_min": -13.807393074035645,
      "activations/layer10_attention_weight_max": 36.627410888671875,
      "activations/layer10_attention_weight_min": -36.496490478515625,
      "activations/layer11_attention_weight_max": 37.05507278442383,
      "activations/layer11_attention_weight_min": -38.356964111328125,
      "activations/layer12_attention_weight_max": 27.677658081054688,
      "activations/layer12_attention_weight_min": -27.262733459472656,
      "activations/layer13_attention_weight_max": 39.097618103027344,
      "activations/layer13_attention_weight_min": -37.10805130004883,
      "activations/layer14_attention_weight_max": 45.04801940917969,
      "activations/layer14_attention_weight_min": -40.157344818115234,
      "activations/layer15_attention_weight_max": 38.7252311706543,
      "activations/layer15_attention_weight_min": -36.142276763916016,
      "activations/layer16_attention_weight_max": 29.037126541137695,
      "activations/layer16_attention_weight_min": -27.91490936279297,
      "activations/layer17_attention_weight_max": 29.33051872253418,
      "activations/layer17_attention_weight_min": -26.865673065185547,
      "activations/layer18_attention_weight_max": 31.95442008972168,
      "activations/layer18_attention_weight_min": -25.66791343688965,
      "activations/layer19_attention_weight_max": 34.435237884521484,
      "activations/layer19_attention_weight_min": -30.581546783447266,
      "activations/layer1_attention_weight_max": 15.826506614685059,
      "activations/layer1_attention_weight_min": -14.327690124511719,
      "activations/layer20_attention_weight_max": 27.050537109375,
      "activations/layer20_attention_weight_min": -25.98101043701172,
      "activations/layer21_attention_weight_max": 26.641393661499023,
      "activations/layer21_attention_weight_min": -24.73468017578125,
      "activations/layer22_attention_weight_max": 42.556034088134766,
      "activations/layer22_attention_weight_min": -31.74761962890625,
      "activations/layer23_attention_weight_max": 33.34027099609375,
      "activations/layer23_attention_weight_min": -25.81613540649414,
      "activations/layer2_attention_weight_max": 32.53483200073242,
      "activations/layer2_attention_weight_min": -32.16299819946289,
      "activations/layer3_attention_weight_max": 98.59249114990234,
      "activations/layer3_attention_weight_min": -102.79033660888672,
      "activations/layer4_attention_weight_max": 103.2594985961914,
      "activations/layer4_attention_weight_min": -110.3171157836914,
      "activations/layer5_attention_weight_max": 49.094451904296875,
      "activations/layer5_attention_weight_min": -58.46372604370117,
      "activations/layer6_attention_weight_max": 42.351192474365234,
      "activations/layer6_attention_weight_min": -47.68085861206055,
      "activations/layer7_attention_weight_max": 95.01641845703125,
      "activations/layer7_attention_weight_min": -99.05853271484375,
      "activations/layer8_attention_weight_max": 42.306732177734375,
      "activations/layer8_attention_weight_min": -45.797298431396484,
      "activations/layer9_attention_weight_max": 32.0845947265625,
      "activations/layer9_attention_weight_min": -36.80873489379883,
      "epoch": 20.64,
      "learning_rate": 1.709431818181818e-05,
      "loss": 2.7117,
      "step": 355150
    },
    {
      "activations/layer0_attention_weight_max": 17.240690231323242,
      "activations/layer0_attention_weight_min": -13.546433448791504,
      "activations/layer10_attention_weight_max": 36.022979736328125,
      "activations/layer10_attention_weight_min": -35.50574493408203,
      "activations/layer11_attention_weight_max": 34.917335510253906,
      "activations/layer11_attention_weight_min": -35.924625396728516,
      "activations/layer12_attention_weight_max": 28.11453628540039,
      "activations/layer12_attention_weight_min": -27.277103424072266,
      "activations/layer13_attention_weight_max": 43.65622329711914,
      "activations/layer13_attention_weight_min": -36.20496368408203,
      "activations/layer14_attention_weight_max": 44.959720611572266,
      "activations/layer14_attention_weight_min": -38.928558349609375,
      "activations/layer15_attention_weight_max": 42.45751190185547,
      "activations/layer15_attention_weight_min": -37.05044174194336,
      "activations/layer16_attention_weight_max": 29.860870361328125,
      "activations/layer16_attention_weight_min": -27.200122833251953,
      "activations/layer17_attention_weight_max": 30.394535064697266,
      "activations/layer17_attention_weight_min": -26.0595645904541,
      "activations/layer18_attention_weight_max": 34.6229133605957,
      "activations/layer18_attention_weight_min": -25.69205665588379,
      "activations/layer19_attention_weight_max": 32.12198257446289,
      "activations/layer19_attention_weight_min": -30.286283493041992,
      "activations/layer1_attention_weight_max": 16.072097778320312,
      "activations/layer1_attention_weight_min": -15.476844787597656,
      "activations/layer20_attention_weight_max": 28.829389572143555,
      "activations/layer20_attention_weight_min": -24.376893997192383,
      "activations/layer21_attention_weight_max": 29.284801483154297,
      "activations/layer21_attention_weight_min": -25.022703170776367,
      "activations/layer22_attention_weight_max": 40.29249572753906,
      "activations/layer22_attention_weight_min": -32.41193389892578,
      "activations/layer23_attention_weight_max": 32.37122344970703,
      "activations/layer23_attention_weight_min": -26.311397552490234,
      "activations/layer2_attention_weight_max": 33.29370880126953,
      "activations/layer2_attention_weight_min": -32.64712142944336,
      "activations/layer3_attention_weight_max": 104.2885513305664,
      "activations/layer3_attention_weight_min": -104.92827606201172,
      "activations/layer4_attention_weight_max": 108.16619873046875,
      "activations/layer4_attention_weight_min": -111.26704406738281,
      "activations/layer5_attention_weight_max": 50.054412841796875,
      "activations/layer5_attention_weight_min": -60.52004623413086,
      "activations/layer6_attention_weight_max": 46.077030181884766,
      "activations/layer6_attention_weight_min": -49.30518341064453,
      "activations/layer7_attention_weight_max": 97.78666687011719,
      "activations/layer7_attention_weight_min": -105.0425033569336,
      "activations/layer8_attention_weight_max": 45.08015441894531,
      "activations/layer8_attention_weight_min": -49.9417839050293,
      "activations/layer9_attention_weight_max": 33.29851531982422,
      "activations/layer9_attention_weight_min": -36.30952453613281,
      "epoch": 20.64,
      "learning_rate": 1.7075378787878784e-05,
      "loss": 2.7227,
      "step": 355200
    },
    {
      "activations/layer0_attention_weight_max": 16.320463180541992,
      "activations/layer0_attention_weight_min": -13.721087455749512,
      "activations/layer10_attention_weight_max": 35.78153991699219,
      "activations/layer10_attention_weight_min": -34.120948791503906,
      "activations/layer11_attention_weight_max": 35.17410659790039,
      "activations/layer11_attention_weight_min": -36.37184143066406,
      "activations/layer12_attention_weight_max": 29.55693244934082,
      "activations/layer12_attention_weight_min": -27.844675064086914,
      "activations/layer13_attention_weight_max": 40.53786087036133,
      "activations/layer13_attention_weight_min": -37.467987060546875,
      "activations/layer14_attention_weight_max": 47.765472412109375,
      "activations/layer14_attention_weight_min": -41.67024230957031,
      "activations/layer15_attention_weight_max": 42.98359680175781,
      "activations/layer15_attention_weight_min": -38.58784484863281,
      "activations/layer16_attention_weight_max": 30.91442108154297,
      "activations/layer16_attention_weight_min": -32.32706832885742,
      "activations/layer17_attention_weight_max": 31.923662185668945,
      "activations/layer17_attention_weight_min": -26.934789657592773,
      "activations/layer18_attention_weight_max": 36.50738525390625,
      "activations/layer18_attention_weight_min": -27.378604888916016,
      "activations/layer19_attention_weight_max": 37.33303451538086,
      "activations/layer19_attention_weight_min": -33.08544158935547,
      "activations/layer1_attention_weight_max": 16.04293441772461,
      "activations/layer1_attention_weight_min": -14.258228302001953,
      "activations/layer20_attention_weight_max": 31.13970947265625,
      "activations/layer20_attention_weight_min": -28.085948944091797,
      "activations/layer21_attention_weight_max": 33.38397979736328,
      "activations/layer21_attention_weight_min": -28.899051666259766,
      "activations/layer22_attention_weight_max": 44.12782669067383,
      "activations/layer22_attention_weight_min": -33.450523376464844,
      "activations/layer23_attention_weight_max": 34.193641662597656,
      "activations/layer23_attention_weight_min": -25.931730270385742,
      "activations/layer2_attention_weight_max": 33.791038513183594,
      "activations/layer2_attention_weight_min": -32.30781936645508,
      "activations/layer3_attention_weight_max": 103.47400665283203,
      "activations/layer3_attention_weight_min": -102.800537109375,
      "activations/layer4_attention_weight_max": 106.8841781616211,
      "activations/layer4_attention_weight_min": -107.57745361328125,
      "activations/layer5_attention_weight_max": 47.298912048339844,
      "activations/layer5_attention_weight_min": -62.960567474365234,
      "activations/layer6_attention_weight_max": 43.22866439819336,
      "activations/layer6_attention_weight_min": -45.88337326049805,
      "activations/layer7_attention_weight_max": 90.13835906982422,
      "activations/layer7_attention_weight_min": -98.48320007324219,
      "activations/layer8_attention_weight_max": 42.56887435913086,
      "activations/layer8_attention_weight_min": -43.75827407836914,
      "activations/layer9_attention_weight_max": 32.81391525268555,
      "activations/layer9_attention_weight_min": -35.67793655395508,
      "epoch": 20.64,
      "learning_rate": 1.7056439393939393e-05,
      "loss": 2.7126,
      "step": 355250
    },
    {
      "activations/layer0_attention_weight_max": 15.921260833740234,
      "activations/layer0_attention_weight_min": -15.13981819152832,
      "activations/layer10_attention_weight_max": 40.551597595214844,
      "activations/layer10_attention_weight_min": -41.40126419067383,
      "activations/layer11_attention_weight_max": 41.573429107666016,
      "activations/layer11_attention_weight_min": -43.45052719116211,
      "activations/layer12_attention_weight_max": 27.868061065673828,
      "activations/layer12_attention_weight_min": -29.706363677978516,
      "activations/layer13_attention_weight_max": 38.446372985839844,
      "activations/layer13_attention_weight_min": -39.21073532104492,
      "activations/layer14_attention_weight_max": 44.79827880859375,
      "activations/layer14_attention_weight_min": -47.7618408203125,
      "activations/layer15_attention_weight_max": 46.83376693725586,
      "activations/layer15_attention_weight_min": -46.85037612915039,
      "activations/layer16_attention_weight_max": 31.917688369750977,
      "activations/layer16_attention_weight_min": -28.478307723999023,
      "activations/layer17_attention_weight_max": 30.55879020690918,
      "activations/layer17_attention_weight_min": -26.939468383789062,
      "activations/layer18_attention_weight_max": 33.16918182373047,
      "activations/layer18_attention_weight_min": -25.590951919555664,
      "activations/layer19_attention_weight_max": 38.07381820678711,
      "activations/layer19_attention_weight_min": -30.134647369384766,
      "activations/layer1_attention_weight_max": 15.99204158782959,
      "activations/layer1_attention_weight_min": -15.460153579711914,
      "activations/layer20_attention_weight_max": 30.0741024017334,
      "activations/layer20_attention_weight_min": -26.14015769958496,
      "activations/layer21_attention_weight_max": 29.864532470703125,
      "activations/layer21_attention_weight_min": -24.372766494750977,
      "activations/layer22_attention_weight_max": 45.805118560791016,
      "activations/layer22_attention_weight_min": -32.35249328613281,
      "activations/layer23_attention_weight_max": 32.81717300415039,
      "activations/layer23_attention_weight_min": -24.95022964477539,
      "activations/layer2_attention_weight_max": 34.516502380371094,
      "activations/layer2_attention_weight_min": -32.523834228515625,
      "activations/layer3_attention_weight_max": 103.46240997314453,
      "activations/layer3_attention_weight_min": -102.21729278564453,
      "activations/layer4_attention_weight_max": 104.10755920410156,
      "activations/layer4_attention_weight_min": -107.07649230957031,
      "activations/layer5_attention_weight_max": 49.04755783081055,
      "activations/layer5_attention_weight_min": -61.291473388671875,
      "activations/layer6_attention_weight_max": 43.132728576660156,
      "activations/layer6_attention_weight_min": -46.060184478759766,
      "activations/layer7_attention_weight_max": 99.85322570800781,
      "activations/layer7_attention_weight_min": -100.40966033935547,
      "activations/layer8_attention_weight_max": 47.15779495239258,
      "activations/layer8_attention_weight_min": -49.13633346557617,
      "activations/layer9_attention_weight_max": 36.19596481323242,
      "activations/layer9_attention_weight_min": -38.93832015991211,
      "epoch": 20.64,
      "learning_rate": 1.7037499999999997e-05,
      "loss": 2.728,
      "step": 355300
    },
    {
      "activations/layer0_attention_weight_max": 16.60719108581543,
      "activations/layer0_attention_weight_min": -13.50298023223877,
      "activations/layer10_attention_weight_max": 49.0004997253418,
      "activations/layer10_attention_weight_min": -45.45638656616211,
      "activations/layer11_attention_weight_max": 50.026756286621094,
      "activations/layer11_attention_weight_min": -47.913597106933594,
      "activations/layer12_attention_weight_max": 39.00104522705078,
      "activations/layer12_attention_weight_min": -30.00179672241211,
      "activations/layer13_attention_weight_max": 52.98445129394531,
      "activations/layer13_attention_weight_min": -41.3970832824707,
      "activations/layer14_attention_weight_max": 48.93955993652344,
      "activations/layer14_attention_weight_min": -46.16431427001953,
      "activations/layer15_attention_weight_max": 49.734375,
      "activations/layer15_attention_weight_min": -48.8488655090332,
      "activations/layer16_attention_weight_max": 34.383419036865234,
      "activations/layer16_attention_weight_min": -29.319000244140625,
      "activations/layer17_attention_weight_max": 31.71820068359375,
      "activations/layer17_attention_weight_min": -29.670413970947266,
      "activations/layer18_attention_weight_max": 35.17973327636719,
      "activations/layer18_attention_weight_min": -30.004352569580078,
      "activations/layer19_attention_weight_max": 37.316585540771484,
      "activations/layer19_attention_weight_min": -33.90538787841797,
      "activations/layer1_attention_weight_max": 16.354610443115234,
      "activations/layer1_attention_weight_min": -14.248457908630371,
      "activations/layer20_attention_weight_max": 33.248775482177734,
      "activations/layer20_attention_weight_min": -28.673038482666016,
      "activations/layer21_attention_weight_max": 33.60273742675781,
      "activations/layer21_attention_weight_min": -26.55403709411621,
      "activations/layer22_attention_weight_max": 46.16339874267578,
      "activations/layer22_attention_weight_min": -33.90835952758789,
      "activations/layer23_attention_weight_max": 38.362937927246094,
      "activations/layer23_attention_weight_min": -27.399028778076172,
      "activations/layer2_attention_weight_max": 34.78643035888672,
      "activations/layer2_attention_weight_min": -35.02503967285156,
      "activations/layer3_attention_weight_max": 103.71713256835938,
      "activations/layer3_attention_weight_min": -106.19239044189453,
      "activations/layer4_attention_weight_max": 110.24385833740234,
      "activations/layer4_attention_weight_min": -114.52819061279297,
      "activations/layer5_attention_weight_max": 53.47650909423828,
      "activations/layer5_attention_weight_min": -62.52810287475586,
      "activations/layer6_attention_weight_max": 48.272377014160156,
      "activations/layer6_attention_weight_min": -47.897396087646484,
      "activations/layer7_attention_weight_max": 123.02339935302734,
      "activations/layer7_attention_weight_min": -110.63935089111328,
      "activations/layer8_attention_weight_max": 57.23147201538086,
      "activations/layer8_attention_weight_min": -52.8216438293457,
      "activations/layer9_attention_weight_max": 43.12252426147461,
      "activations/layer9_attention_weight_min": -44.0458869934082,
      "epoch": 20.65,
      "learning_rate": 1.7018560606060602e-05,
      "loss": 2.7174,
      "step": 355350
    },
    {
      "activations/layer0_attention_weight_max": 16.538158416748047,
      "activations/layer0_attention_weight_min": -12.951356887817383,
      "activations/layer10_attention_weight_max": 34.18461608886719,
      "activations/layer10_attention_weight_min": -34.66259002685547,
      "activations/layer11_attention_weight_max": 35.30707931518555,
      "activations/layer11_attention_weight_min": -35.22992706298828,
      "activations/layer12_attention_weight_max": 29.352201461791992,
      "activations/layer12_attention_weight_min": -27.224210739135742,
      "activations/layer13_attention_weight_max": 43.47356414794922,
      "activations/layer13_attention_weight_min": -39.487884521484375,
      "activations/layer14_attention_weight_max": 48.349891662597656,
      "activations/layer14_attention_weight_min": -41.249908447265625,
      "activations/layer15_attention_weight_max": 45.04027557373047,
      "activations/layer15_attention_weight_min": -39.71030044555664,
      "activations/layer16_attention_weight_max": 32.43698501586914,
      "activations/layer16_attention_weight_min": -30.501611709594727,
      "activations/layer17_attention_weight_max": 32.85472106933594,
      "activations/layer17_attention_weight_min": -26.08192253112793,
      "activations/layer18_attention_weight_max": 34.091697692871094,
      "activations/layer18_attention_weight_min": -25.697376251220703,
      "activations/layer19_attention_weight_max": 37.78843307495117,
      "activations/layer19_attention_weight_min": -31.804336547851562,
      "activations/layer1_attention_weight_max": 15.916487693786621,
      "activations/layer1_attention_weight_min": -13.671414375305176,
      "activations/layer20_attention_weight_max": 36.16398620605469,
      "activations/layer20_attention_weight_min": -27.15230369567871,
      "activations/layer21_attention_weight_max": 35.340450286865234,
      "activations/layer21_attention_weight_min": -27.808086395263672,
      "activations/layer22_attention_weight_max": 57.247314453125,
      "activations/layer22_attention_weight_min": -33.74811935424805,
      "activations/layer23_attention_weight_max": 35.43458557128906,
      "activations/layer23_attention_weight_min": -24.871747970581055,
      "activations/layer2_attention_weight_max": 33.25448226928711,
      "activations/layer2_attention_weight_min": -32.431217193603516,
      "activations/layer3_attention_weight_max": 101.47705078125,
      "activations/layer3_attention_weight_min": -105.35797882080078,
      "activations/layer4_attention_weight_max": 105.21392059326172,
      "activations/layer4_attention_weight_min": -110.07484436035156,
      "activations/layer5_attention_weight_max": 50.54393768310547,
      "activations/layer5_attention_weight_min": -64.54582214355469,
      "activations/layer6_attention_weight_max": 44.47722625732422,
      "activations/layer6_attention_weight_min": -46.48640060424805,
      "activations/layer7_attention_weight_max": 95.34378814697266,
      "activations/layer7_attention_weight_min": -98.13020324707031,
      "activations/layer8_attention_weight_max": 44.248924255371094,
      "activations/layer8_attention_weight_min": -43.65871810913086,
      "activations/layer9_attention_weight_max": 32.757022857666016,
      "activations/layer9_attention_weight_min": -34.501678466796875,
      "epoch": 20.65,
      "learning_rate": 1.699962121212121e-05,
      "loss": 2.7227,
      "step": 355400
    },
    {
      "activations/layer0_attention_weight_max": 16.69215202331543,
      "activations/layer0_attention_weight_min": -13.869342803955078,
      "activations/layer10_attention_weight_max": 36.9310188293457,
      "activations/layer10_attention_weight_min": -35.97473907470703,
      "activations/layer11_attention_weight_max": 36.686431884765625,
      "activations/layer11_attention_weight_min": -35.59636688232422,
      "activations/layer12_attention_weight_max": 27.19483757019043,
      "activations/layer12_attention_weight_min": -27.25757598876953,
      "activations/layer13_attention_weight_max": 42.15425491333008,
      "activations/layer13_attention_weight_min": -37.961238861083984,
      "activations/layer14_attention_weight_max": 50.40695571899414,
      "activations/layer14_attention_weight_min": -41.12416458129883,
      "activations/layer15_attention_weight_max": 43.788055419921875,
      "activations/layer15_attention_weight_min": -37.16630554199219,
      "activations/layer16_attention_weight_max": 32.25571060180664,
      "activations/layer16_attention_weight_min": -28.555213928222656,
      "activations/layer17_attention_weight_max": 31.20961570739746,
      "activations/layer17_attention_weight_min": -26.580339431762695,
      "activations/layer18_attention_weight_max": 32.530033111572266,
      "activations/layer18_attention_weight_min": -28.0648136138916,
      "activations/layer19_attention_weight_max": 38.63100814819336,
      "activations/layer19_attention_weight_min": -32.48366928100586,
      "activations/layer1_attention_weight_max": 16.019750595092773,
      "activations/layer1_attention_weight_min": -14.812442779541016,
      "activations/layer20_attention_weight_max": 32.83549118041992,
      "activations/layer20_attention_weight_min": -28.322420120239258,
      "activations/layer21_attention_weight_max": 32.58796310424805,
      "activations/layer21_attention_weight_min": -26.555124282836914,
      "activations/layer22_attention_weight_max": 50.849090576171875,
      "activations/layer22_attention_weight_min": -36.10935592651367,
      "activations/layer23_attention_weight_max": 35.97108840942383,
      "activations/layer23_attention_weight_min": -26.97833824157715,
      "activations/layer2_attention_weight_max": 34.066932678222656,
      "activations/layer2_attention_weight_min": -34.01057815551758,
      "activations/layer3_attention_weight_max": 102.72380828857422,
      "activations/layer3_attention_weight_min": -107.30683898925781,
      "activations/layer4_attention_weight_max": 106.86803436279297,
      "activations/layer4_attention_weight_min": -111.4211654663086,
      "activations/layer5_attention_weight_max": 52.6490592956543,
      "activations/layer5_attention_weight_min": -61.13669967651367,
      "activations/layer6_attention_weight_max": 46.42811584472656,
      "activations/layer6_attention_weight_min": -47.9410285949707,
      "activations/layer7_attention_weight_max": 98.2229232788086,
      "activations/layer7_attention_weight_min": -105.13211822509766,
      "activations/layer8_attention_weight_max": 42.530330657958984,
      "activations/layer8_attention_weight_min": -45.063907623291016,
      "activations/layer9_attention_weight_max": 33.01652908325195,
      "activations/layer9_attention_weight_min": -35.730587005615234,
      "epoch": 20.65,
      "learning_rate": 1.698068181818182e-05,
      "loss": 2.7087,
      "step": 355450
    },
    {
      "activations/layer0_attention_weight_max": 16.35647201538086,
      "activations/layer0_attention_weight_min": -13.98464584350586,
      "activations/layer10_attention_weight_max": 36.719329833984375,
      "activations/layer10_attention_weight_min": -36.171913146972656,
      "activations/layer11_attention_weight_max": 35.340843200683594,
      "activations/layer11_attention_weight_min": -35.57585525512695,
      "activations/layer12_attention_weight_max": 27.78446388244629,
      "activations/layer12_attention_weight_min": -26.959957122802734,
      "activations/layer13_attention_weight_max": 40.0086555480957,
      "activations/layer13_attention_weight_min": -39.950111389160156,
      "activations/layer14_attention_weight_max": 41.876258850097656,
      "activations/layer14_attention_weight_min": -39.5539665222168,
      "activations/layer15_attention_weight_max": 38.763885498046875,
      "activations/layer15_attention_weight_min": -38.24814987182617,
      "activations/layer16_attention_weight_max": 30.36570930480957,
      "activations/layer16_attention_weight_min": -29.17301368713379,
      "activations/layer17_attention_weight_max": 30.613075256347656,
      "activations/layer17_attention_weight_min": -27.407102584838867,
      "activations/layer18_attention_weight_max": 33.51620101928711,
      "activations/layer18_attention_weight_min": -28.56883430480957,
      "activations/layer19_attention_weight_max": 34.0122184753418,
      "activations/layer19_attention_weight_min": -31.121496200561523,
      "activations/layer1_attention_weight_max": 17.357662200927734,
      "activations/layer1_attention_weight_min": -14.681376457214355,
      "activations/layer20_attention_weight_max": 30.849273681640625,
      "activations/layer20_attention_weight_min": -26.55092430114746,
      "activations/layer21_attention_weight_max": 31.27065658569336,
      "activations/layer21_attention_weight_min": -24.804494857788086,
      "activations/layer22_attention_weight_max": 38.863582611083984,
      "activations/layer22_attention_weight_min": -30.924015045166016,
      "activations/layer23_attention_weight_max": 32.43409729003906,
      "activations/layer23_attention_weight_min": -24.85871124267578,
      "activations/layer2_attention_weight_max": 33.56147003173828,
      "activations/layer2_attention_weight_min": -33.10810470581055,
      "activations/layer3_attention_weight_max": 102.84941101074219,
      "activations/layer3_attention_weight_min": -106.54248809814453,
      "activations/layer4_attention_weight_max": 107.13289642333984,
      "activations/layer4_attention_weight_min": -109.2091293334961,
      "activations/layer5_attention_weight_max": 52.467464447021484,
      "activations/layer5_attention_weight_min": -62.86824035644531,
      "activations/layer6_attention_weight_max": 46.05712127685547,
      "activations/layer6_attention_weight_min": -48.13973617553711,
      "activations/layer7_attention_weight_max": 94.2228012084961,
      "activations/layer7_attention_weight_min": -103.11495971679688,
      "activations/layer8_attention_weight_max": 44.61189651489258,
      "activations/layer8_attention_weight_min": -46.93881607055664,
      "activations/layer9_attention_weight_max": 32.08735656738281,
      "activations/layer9_attention_weight_min": -36.247005462646484,
      "epoch": 20.66,
      "learning_rate": 1.6961742424242424e-05,
      "loss": 2.7118,
      "step": 355500
    },
    {
      "activations/layer0_attention_weight_max": 16.72889518737793,
      "activations/layer0_attention_weight_min": -13.37546443939209,
      "activations/layer10_attention_weight_max": 35.61127471923828,
      "activations/layer10_attention_weight_min": -35.438873291015625,
      "activations/layer11_attention_weight_max": 34.849281311035156,
      "activations/layer11_attention_weight_min": -34.79549789428711,
      "activations/layer12_attention_weight_max": 27.542905807495117,
      "activations/layer12_attention_weight_min": -28.661514282226562,
      "activations/layer13_attention_weight_max": 40.021392822265625,
      "activations/layer13_attention_weight_min": -38.50090408325195,
      "activations/layer14_attention_weight_max": 41.87211608886719,
      "activations/layer14_attention_weight_min": -43.23439407348633,
      "activations/layer15_attention_weight_max": 37.121341705322266,
      "activations/layer15_attention_weight_min": -41.579490661621094,
      "activations/layer16_attention_weight_max": 28.642505645751953,
      "activations/layer16_attention_weight_min": -28.132694244384766,
      "activations/layer17_attention_weight_max": 28.47027587890625,
      "activations/layer17_attention_weight_min": -26.88154411315918,
      "activations/layer18_attention_weight_max": 34.399200439453125,
      "activations/layer18_attention_weight_min": -27.751623153686523,
      "activations/layer19_attention_weight_max": 33.88039779663086,
      "activations/layer19_attention_weight_min": -31.72237205505371,
      "activations/layer1_attention_weight_max": 16.12264633178711,
      "activations/layer1_attention_weight_min": -15.305331230163574,
      "activations/layer20_attention_weight_max": 30.60289192199707,
      "activations/layer20_attention_weight_min": -24.408788681030273,
      "activations/layer21_attention_weight_max": 28.634258270263672,
      "activations/layer21_attention_weight_min": -25.04277992248535,
      "activations/layer22_attention_weight_max": 40.750911712646484,
      "activations/layer22_attention_weight_min": -34.847774505615234,
      "activations/layer23_attention_weight_max": 31.68182945251465,
      "activations/layer23_attention_weight_min": -26.802326202392578,
      "activations/layer2_attention_weight_max": 37.18848419189453,
      "activations/layer2_attention_weight_min": -33.17539596557617,
      "activations/layer3_attention_weight_max": 106.7510757446289,
      "activations/layer3_attention_weight_min": -108.54013061523438,
      "activations/layer4_attention_weight_max": 108.0282974243164,
      "activations/layer4_attention_weight_min": -114.25090789794922,
      "activations/layer5_attention_weight_max": 48.45068359375,
      "activations/layer5_attention_weight_min": -61.24366760253906,
      "activations/layer6_attention_weight_max": 44.19485855102539,
      "activations/layer6_attention_weight_min": -47.1079216003418,
      "activations/layer7_attention_weight_max": 97.33412170410156,
      "activations/layer7_attention_weight_min": -96.5976791381836,
      "activations/layer8_attention_weight_max": 43.49907302856445,
      "activations/layer8_attention_weight_min": -45.571659088134766,
      "activations/layer9_attention_weight_max": 34.44062042236328,
      "activations/layer9_attention_weight_min": -35.56180953979492,
      "epoch": 20.66,
      "learning_rate": 1.694280303030303e-05,
      "loss": 2.7124,
      "step": 355550
    },
    {
      "activations/layer0_attention_weight_max": 16.051982879638672,
      "activations/layer0_attention_weight_min": -12.89903736114502,
      "activations/layer10_attention_weight_max": 34.803958892822266,
      "activations/layer10_attention_weight_min": -33.375709533691406,
      "activations/layer11_attention_weight_max": 33.456417083740234,
      "activations/layer11_attention_weight_min": -33.37274932861328,
      "activations/layer12_attention_weight_max": 26.208694458007812,
      "activations/layer12_attention_weight_min": -29.39668083190918,
      "activations/layer13_attention_weight_max": 39.01621627807617,
      "activations/layer13_attention_weight_min": -36.59642028808594,
      "activations/layer14_attention_weight_max": 42.75711441040039,
      "activations/layer14_attention_weight_min": -38.11185836791992,
      "activations/layer15_attention_weight_max": 38.376258850097656,
      "activations/layer15_attention_weight_min": -36.44314193725586,
      "activations/layer16_attention_weight_max": 30.25462532043457,
      "activations/layer16_attention_weight_min": -28.47072982788086,
      "activations/layer17_attention_weight_max": 29.83576774597168,
      "activations/layer17_attention_weight_min": -27.694089889526367,
      "activations/layer18_attention_weight_max": 33.200096130371094,
      "activations/layer18_attention_weight_min": -27.52764320373535,
      "activations/layer19_attention_weight_max": 39.452911376953125,
      "activations/layer19_attention_weight_min": -31.896697998046875,
      "activations/layer1_attention_weight_max": 15.67753791809082,
      "activations/layer1_attention_weight_min": -14.892300605773926,
      "activations/layer20_attention_weight_max": 36.137203216552734,
      "activations/layer20_attention_weight_min": -25.92000389099121,
      "activations/layer21_attention_weight_max": 32.74089431762695,
      "activations/layer21_attention_weight_min": -26.046188354492188,
      "activations/layer22_attention_weight_max": 43.32508850097656,
      "activations/layer22_attention_weight_min": -30.72539520263672,
      "activations/layer23_attention_weight_max": 34.9083251953125,
      "activations/layer23_attention_weight_min": -26.2360897064209,
      "activations/layer2_attention_weight_max": 33.56665802001953,
      "activations/layer2_attention_weight_min": -31.463611602783203,
      "activations/layer3_attention_weight_max": 99.48924255371094,
      "activations/layer3_attention_weight_min": -103.50028991699219,
      "activations/layer4_attention_weight_max": 103.94658660888672,
      "activations/layer4_attention_weight_min": -106.74720764160156,
      "activations/layer5_attention_weight_max": 49.29513168334961,
      "activations/layer5_attention_weight_min": -61.952430725097656,
      "activations/layer6_attention_weight_max": 44.32706832885742,
      "activations/layer6_attention_weight_min": -46.17123794555664,
      "activations/layer7_attention_weight_max": 95.70654296875,
      "activations/layer7_attention_weight_min": -95.76016998291016,
      "activations/layer8_attention_weight_max": 41.87981414794922,
      "activations/layer8_attention_weight_min": -44.86655807495117,
      "activations/layer9_attention_weight_max": 30.163436889648438,
      "activations/layer9_attention_weight_min": -33.59360885620117,
      "epoch": 20.66,
      "learning_rate": 1.6923863636363637e-05,
      "loss": 2.7031,
      "step": 355600
    },
    {
      "activations/layer0_attention_weight_max": 17.557907104492188,
      "activations/layer0_attention_weight_min": -13.67137336730957,
      "activations/layer10_attention_weight_max": 34.02355194091797,
      "activations/layer10_attention_weight_min": -34.666229248046875,
      "activations/layer11_attention_weight_max": 33.722328186035156,
      "activations/layer11_attention_weight_min": -35.13069534301758,
      "activations/layer12_attention_weight_max": 25.484827041625977,
      "activations/layer12_attention_weight_min": -26.386844635009766,
      "activations/layer13_attention_weight_max": 38.162200927734375,
      "activations/layer13_attention_weight_min": -35.26067352294922,
      "activations/layer14_attention_weight_max": 41.36416244506836,
      "activations/layer14_attention_weight_min": -37.82050323486328,
      "activations/layer15_attention_weight_max": 37.29202651977539,
      "activations/layer15_attention_weight_min": -36.25226593017578,
      "activations/layer16_attention_weight_max": 28.585975646972656,
      "activations/layer16_attention_weight_min": -27.912681579589844,
      "activations/layer17_attention_weight_max": 31.671194076538086,
      "activations/layer17_attention_weight_min": -26.860427856445312,
      "activations/layer18_attention_weight_max": 34.639488220214844,
      "activations/layer18_attention_weight_min": -26.050771713256836,
      "activations/layer19_attention_weight_max": 35.40400695800781,
      "activations/layer19_attention_weight_min": -32.21981430053711,
      "activations/layer1_attention_weight_max": 15.471220970153809,
      "activations/layer1_attention_weight_min": -13.133821487426758,
      "activations/layer20_attention_weight_max": 28.778106689453125,
      "activations/layer20_attention_weight_min": -25.992979049682617,
      "activations/layer21_attention_weight_max": 29.03898048400879,
      "activations/layer21_attention_weight_min": -24.783763885498047,
      "activations/layer22_attention_weight_max": 41.70503616333008,
      "activations/layer22_attention_weight_min": -32.372467041015625,
      "activations/layer23_attention_weight_max": 33.35983657836914,
      "activations/layer23_attention_weight_min": -27.055835723876953,
      "activations/layer2_attention_weight_max": 34.74385070800781,
      "activations/layer2_attention_weight_min": -30.968395233154297,
      "activations/layer3_attention_weight_max": 99.03605651855469,
      "activations/layer3_attention_weight_min": -98.91584777832031,
      "activations/layer4_attention_weight_max": 106.17750549316406,
      "activations/layer4_attention_weight_min": -105.8501205444336,
      "activations/layer5_attention_weight_max": 50.93918991088867,
      "activations/layer5_attention_weight_min": -61.398162841796875,
      "activations/layer6_attention_weight_max": 47.84491729736328,
      "activations/layer6_attention_weight_min": -47.55555725097656,
      "activations/layer7_attention_weight_max": 96.30900573730469,
      "activations/layer7_attention_weight_min": -98.13644409179688,
      "activations/layer8_attention_weight_max": 43.01838302612305,
      "activations/layer8_attention_weight_min": -44.117584228515625,
      "activations/layer9_attention_weight_max": 33.11688232421875,
      "activations/layer9_attention_weight_min": -34.50553512573242,
      "epoch": 20.67,
      "learning_rate": 1.6904924242424242e-05,
      "loss": 2.7151,
      "step": 355650
    },
    {
      "activations/layer0_attention_weight_max": 16.321430206298828,
      "activations/layer0_attention_weight_min": -13.079257011413574,
      "activations/layer10_attention_weight_max": 37.38597106933594,
      "activations/layer10_attention_weight_min": -36.69694137573242,
      "activations/layer11_attention_weight_max": 35.986602783203125,
      "activations/layer11_attention_weight_min": -39.2131462097168,
      "activations/layer12_attention_weight_max": 30.012248992919922,
      "activations/layer12_attention_weight_min": -29.748123168945312,
      "activations/layer13_attention_weight_max": 42.222450256347656,
      "activations/layer13_attention_weight_min": -36.65314865112305,
      "activations/layer14_attention_weight_max": 42.96243667602539,
      "activations/layer14_attention_weight_min": -38.248313903808594,
      "activations/layer15_attention_weight_max": 40.855133056640625,
      "activations/layer15_attention_weight_min": -37.66391372680664,
      "activations/layer16_attention_weight_max": 31.381046295166016,
      "activations/layer16_attention_weight_min": -29.616195678710938,
      "activations/layer17_attention_weight_max": 30.230913162231445,
      "activations/layer17_attention_weight_min": -26.197372436523438,
      "activations/layer18_attention_weight_max": 31.991310119628906,
      "activations/layer18_attention_weight_min": -25.69822883605957,
      "activations/layer19_attention_weight_max": 38.359230041503906,
      "activations/layer19_attention_weight_min": -31.602710723876953,
      "activations/layer1_attention_weight_max": 15.591753005981445,
      "activations/layer1_attention_weight_min": -13.937573432922363,
      "activations/layer20_attention_weight_max": 29.07091522216797,
      "activations/layer20_attention_weight_min": -25.44734764099121,
      "activations/layer21_attention_weight_max": 31.88681983947754,
      "activations/layer21_attention_weight_min": -24.777074813842773,
      "activations/layer22_attention_weight_max": 44.7496337890625,
      "activations/layer22_attention_weight_min": -32.33343505859375,
      "activations/layer23_attention_weight_max": 34.243446350097656,
      "activations/layer23_attention_weight_min": -26.173442840576172,
      "activations/layer2_attention_weight_max": 33.13902282714844,
      "activations/layer2_attention_weight_min": -33.23630142211914,
      "activations/layer3_attention_weight_max": 100.01586151123047,
      "activations/layer3_attention_weight_min": -105.904052734375,
      "activations/layer4_attention_weight_max": 104.8098373413086,
      "activations/layer4_attention_weight_min": -111.61551666259766,
      "activations/layer5_attention_weight_max": 49.22241973876953,
      "activations/layer5_attention_weight_min": -62.077178955078125,
      "activations/layer6_attention_weight_max": 46.114295959472656,
      "activations/layer6_attention_weight_min": -49.16728591918945,
      "activations/layer7_attention_weight_max": 99.48624420166016,
      "activations/layer7_attention_weight_min": -102.1323471069336,
      "activations/layer8_attention_weight_max": 42.61537170410156,
      "activations/layer8_attention_weight_min": -47.164276123046875,
      "activations/layer9_attention_weight_max": 32.55934143066406,
      "activations/layer9_attention_weight_min": -37.6134147644043,
      "epoch": 20.67,
      "learning_rate": 1.6885984848484847e-05,
      "loss": 2.6964,
      "step": 355700
    },
    {
      "activations/layer0_attention_weight_max": 16.06387710571289,
      "activations/layer0_attention_weight_min": -14.001776695251465,
      "activations/layer10_attention_weight_max": 42.3896484375,
      "activations/layer10_attention_weight_min": -35.97724151611328,
      "activations/layer11_attention_weight_max": 40.36138153076172,
      "activations/layer11_attention_weight_min": -36.46607208251953,
      "activations/layer12_attention_weight_max": 27.289505004882812,
      "activations/layer12_attention_weight_min": -26.59453010559082,
      "activations/layer13_attention_weight_max": 40.3859748840332,
      "activations/layer13_attention_weight_min": -35.58698272705078,
      "activations/layer14_attention_weight_max": 41.20368194580078,
      "activations/layer14_attention_weight_min": -38.264244079589844,
      "activations/layer15_attention_weight_max": 40.281864166259766,
      "activations/layer15_attention_weight_min": -36.61391830444336,
      "activations/layer16_attention_weight_max": 28.915695190429688,
      "activations/layer16_attention_weight_min": -28.54253387451172,
      "activations/layer17_attention_weight_max": 30.19013214111328,
      "activations/layer17_attention_weight_min": -26.279172897338867,
      "activations/layer18_attention_weight_max": 30.7637882232666,
      "activations/layer18_attention_weight_min": -25.12246322631836,
      "activations/layer19_attention_weight_max": 29.775583267211914,
      "activations/layer19_attention_weight_min": -32.62149429321289,
      "activations/layer1_attention_weight_max": 15.35711669921875,
      "activations/layer1_attention_weight_min": -14.0994291305542,
      "activations/layer20_attention_weight_max": 26.0579833984375,
      "activations/layer20_attention_weight_min": -25.184085845947266,
      "activations/layer21_attention_weight_max": 26.57729721069336,
      "activations/layer21_attention_weight_min": -23.793930053710938,
      "activations/layer22_attention_weight_max": 40.68934631347656,
      "activations/layer22_attention_weight_min": -31.15397834777832,
      "activations/layer23_attention_weight_max": 28.31203269958496,
      "activations/layer23_attention_weight_min": -25.55909538269043,
      "activations/layer2_attention_weight_max": 31.716833114624023,
      "activations/layer2_attention_weight_min": -31.7601318359375,
      "activations/layer3_attention_weight_max": 105.01719665527344,
      "activations/layer3_attention_weight_min": -103.0239028930664,
      "activations/layer4_attention_weight_max": 110.58199310302734,
      "activations/layer4_attention_weight_min": -109.3734359741211,
      "activations/layer5_attention_weight_max": 52.70915985107422,
      "activations/layer5_attention_weight_min": -61.9129753112793,
      "activations/layer6_attention_weight_max": 45.55268478393555,
      "activations/layer6_attention_weight_min": -48.58557891845703,
      "activations/layer7_attention_weight_max": 109.56849670410156,
      "activations/layer7_attention_weight_min": -105.3711929321289,
      "activations/layer8_attention_weight_max": 47.866878509521484,
      "activations/layer8_attention_weight_min": -47.2039680480957,
      "activations/layer9_attention_weight_max": 36.968284606933594,
      "activations/layer9_attention_weight_min": -38.47535705566406,
      "epoch": 20.67,
      "learning_rate": 1.6867045454545455e-05,
      "loss": 2.7151,
      "step": 355750
    },
    {
      "activations/layer0_attention_weight_max": 15.854681968688965,
      "activations/layer0_attention_weight_min": -12.989141464233398,
      "activations/layer10_attention_weight_max": 34.55568313598633,
      "activations/layer10_attention_weight_min": -34.87287902832031,
      "activations/layer11_attention_weight_max": 35.064369201660156,
      "activations/layer11_attention_weight_min": -37.807289123535156,
      "activations/layer12_attention_weight_max": 28.805139541625977,
      "activations/layer12_attention_weight_min": -27.404329299926758,
      "activations/layer13_attention_weight_max": 41.53300094604492,
      "activations/layer13_attention_weight_min": -40.33818435668945,
      "activations/layer14_attention_weight_max": 45.1202507019043,
      "activations/layer14_attention_weight_min": -38.846351623535156,
      "activations/layer15_attention_weight_max": 40.79802322387695,
      "activations/layer15_attention_weight_min": -36.754539489746094,
      "activations/layer16_attention_weight_max": 31.49479866027832,
      "activations/layer16_attention_weight_min": -27.673458099365234,
      "activations/layer17_attention_weight_max": 32.72578811645508,
      "activations/layer17_attention_weight_min": -25.474443435668945,
      "activations/layer18_attention_weight_max": 31.668039321899414,
      "activations/layer18_attention_weight_min": -25.615819931030273,
      "activations/layer19_attention_weight_max": 35.71241760253906,
      "activations/layer19_attention_weight_min": -33.93416976928711,
      "activations/layer1_attention_weight_max": 15.125264167785645,
      "activations/layer1_attention_weight_min": -14.441681861877441,
      "activations/layer20_attention_weight_max": 29.118562698364258,
      "activations/layer20_attention_weight_min": -25.284061431884766,
      "activations/layer21_attention_weight_max": 31.979690551757812,
      "activations/layer21_attention_weight_min": -25.36344337463379,
      "activations/layer22_attention_weight_max": 45.64569854736328,
      "activations/layer22_attention_weight_min": -32.11219787597656,
      "activations/layer23_attention_weight_max": 33.58616638183594,
      "activations/layer23_attention_weight_min": -27.29889488220215,
      "activations/layer2_attention_weight_max": 34.608131408691406,
      "activations/layer2_attention_weight_min": -32.70030212402344,
      "activations/layer3_attention_weight_max": 107.87458801269531,
      "activations/layer3_attention_weight_min": -105.83576965332031,
      "activations/layer4_attention_weight_max": 117.64253234863281,
      "activations/layer4_attention_weight_min": -114.29769134521484,
      "activations/layer5_attention_weight_max": 56.59287643432617,
      "activations/layer5_attention_weight_min": -64.27748107910156,
      "activations/layer6_attention_weight_max": 46.909420013427734,
      "activations/layer6_attention_weight_min": -49.61018753051758,
      "activations/layer7_attention_weight_max": 101.22538757324219,
      "activations/layer7_attention_weight_min": -99.95027160644531,
      "activations/layer8_attention_weight_max": 44.41264343261719,
      "activations/layer8_attention_weight_min": -45.73286056518555,
      "activations/layer9_attention_weight_max": 31.933578491210938,
      "activations/layer9_attention_weight_min": -34.77368927001953,
      "epoch": 20.67,
      "learning_rate": 1.684810606060606e-05,
      "loss": 2.7166,
      "step": 355800
    },
    {
      "activations/layer0_attention_weight_max": 16.287128448486328,
      "activations/layer0_attention_weight_min": -13.062736511230469,
      "activations/layer10_attention_weight_max": 32.81977462768555,
      "activations/layer10_attention_weight_min": -33.906646728515625,
      "activations/layer11_attention_weight_max": 33.692352294921875,
      "activations/layer11_attention_weight_min": -33.012001037597656,
      "activations/layer12_attention_weight_max": 26.692005157470703,
      "activations/layer12_attention_weight_min": -27.31216049194336,
      "activations/layer13_attention_weight_max": 41.11747741699219,
      "activations/layer13_attention_weight_min": -35.79194259643555,
      "activations/layer14_attention_weight_max": 46.70298385620117,
      "activations/layer14_attention_weight_min": -39.3006477355957,
      "activations/layer15_attention_weight_max": 44.92902755737305,
      "activations/layer15_attention_weight_min": -39.18867874145508,
      "activations/layer16_attention_weight_max": 28.87982749938965,
      "activations/layer16_attention_weight_min": -29.24118995666504,
      "activations/layer17_attention_weight_max": 29.749197006225586,
      "activations/layer17_attention_weight_min": -27.47151756286621,
      "activations/layer18_attention_weight_max": 33.94590759277344,
      "activations/layer18_attention_weight_min": -26.390560150146484,
      "activations/layer19_attention_weight_max": 34.29456329345703,
      "activations/layer19_attention_weight_min": -31.678466796875,
      "activations/layer1_attention_weight_max": 15.590524673461914,
      "activations/layer1_attention_weight_min": -14.33932113647461,
      "activations/layer20_attention_weight_max": 28.537330627441406,
      "activations/layer20_attention_weight_min": -26.272661209106445,
      "activations/layer21_attention_weight_max": 30.585208892822266,
      "activations/layer21_attention_weight_min": -25.405019760131836,
      "activations/layer22_attention_weight_max": 42.27908706665039,
      "activations/layer22_attention_weight_min": -32.519493103027344,
      "activations/layer23_attention_weight_max": 33.85443115234375,
      "activations/layer23_attention_weight_min": -26.486305236816406,
      "activations/layer2_attention_weight_max": 32.05266571044922,
      "activations/layer2_attention_weight_min": -31.196849822998047,
      "activations/layer3_attention_weight_max": 103.52412414550781,
      "activations/layer3_attention_weight_min": -104.17680358886719,
      "activations/layer4_attention_weight_max": 105.8627700805664,
      "activations/layer4_attention_weight_min": -108.36903381347656,
      "activations/layer5_attention_weight_max": 50.388702392578125,
      "activations/layer5_attention_weight_min": -63.555599212646484,
      "activations/layer6_attention_weight_max": 42.56727600097656,
      "activations/layer6_attention_weight_min": -45.94150161743164,
      "activations/layer7_attention_weight_max": 94.85984802246094,
      "activations/layer7_attention_weight_min": -98.55844116210938,
      "activations/layer8_attention_weight_max": 42.49067687988281,
      "activations/layer8_attention_weight_min": -44.88364028930664,
      "activations/layer9_attention_weight_max": 31.688125610351562,
      "activations/layer9_attention_weight_min": -34.428653717041016,
      "epoch": 20.68,
      "learning_rate": 1.6829166666666665e-05,
      "loss": 2.7294,
      "step": 355850
    },
    {
      "activations/layer0_attention_weight_max": 16.69907569885254,
      "activations/layer0_attention_weight_min": -13.054084777832031,
      "activations/layer10_attention_weight_max": 38.116973876953125,
      "activations/layer10_attention_weight_min": -37.01323318481445,
      "activations/layer11_attention_weight_max": 35.434242248535156,
      "activations/layer11_attention_weight_min": -35.49361801147461,
      "activations/layer12_attention_weight_max": 28.902219772338867,
      "activations/layer12_attention_weight_min": -27.63740348815918,
      "activations/layer13_attention_weight_max": 41.59771728515625,
      "activations/layer13_attention_weight_min": -36.83375930786133,
      "activations/layer14_attention_weight_max": 53.7685546875,
      "activations/layer14_attention_weight_min": -42.103302001953125,
      "activations/layer15_attention_weight_max": 46.53518295288086,
      "activations/layer15_attention_weight_min": -38.31083297729492,
      "activations/layer16_attention_weight_max": 32.8405647277832,
      "activations/layer16_attention_weight_min": -28.931732177734375,
      "activations/layer17_attention_weight_max": 31.14795684814453,
      "activations/layer17_attention_weight_min": -26.82674789428711,
      "activations/layer18_attention_weight_max": 33.8133659362793,
      "activations/layer18_attention_weight_min": -26.571863174438477,
      "activations/layer19_attention_weight_max": 40.58027267456055,
      "activations/layer19_attention_weight_min": -32.09912109375,
      "activations/layer1_attention_weight_max": 15.970577239990234,
      "activations/layer1_attention_weight_min": -15.833456993103027,
      "activations/layer20_attention_weight_max": 35.42372512817383,
      "activations/layer20_attention_weight_min": -25.317270278930664,
      "activations/layer21_attention_weight_max": 34.00074768066406,
      "activations/layer21_attention_weight_min": -25.79620361328125,
      "activations/layer22_attention_weight_max": 56.55836486816406,
      "activations/layer22_attention_weight_min": -34.01804733276367,
      "activations/layer23_attention_weight_max": 39.69017791748047,
      "activations/layer23_attention_weight_min": -27.644445419311523,
      "activations/layer2_attention_weight_max": 33.782493591308594,
      "activations/layer2_attention_weight_min": -32.09756088256836,
      "activations/layer3_attention_weight_max": 101.81690216064453,
      "activations/layer3_attention_weight_min": -103.68740844726562,
      "activations/layer4_attention_weight_max": 106.680908203125,
      "activations/layer4_attention_weight_min": -110.72286224365234,
      "activations/layer5_attention_weight_max": 52.098384857177734,
      "activations/layer5_attention_weight_min": -63.96858215332031,
      "activations/layer6_attention_weight_max": 45.314552307128906,
      "activations/layer6_attention_weight_min": -48.43892288208008,
      "activations/layer7_attention_weight_max": 95.32535552978516,
      "activations/layer7_attention_weight_min": -98.72666931152344,
      "activations/layer8_attention_weight_max": 45.40528869628906,
      "activations/layer8_attention_weight_min": -46.73678207397461,
      "activations/layer9_attention_weight_max": 33.17047119140625,
      "activations/layer9_attention_weight_min": -35.82262420654297,
      "epoch": 20.68,
      "learning_rate": 1.681022727272727e-05,
      "loss": 2.7198,
      "step": 355900
    },
    {
      "activations/layer0_attention_weight_max": 16.901782989501953,
      "activations/layer0_attention_weight_min": -13.960491180419922,
      "activations/layer10_attention_weight_max": 35.550540924072266,
      "activations/layer10_attention_weight_min": -35.6243896484375,
      "activations/layer11_attention_weight_max": 33.575687408447266,
      "activations/layer11_attention_weight_min": -33.530372619628906,
      "activations/layer12_attention_weight_max": 27.64582633972168,
      "activations/layer12_attention_weight_min": -25.757699966430664,
      "activations/layer13_attention_weight_max": 39.97087478637695,
      "activations/layer13_attention_weight_min": -36.507572174072266,
      "activations/layer14_attention_weight_max": 41.75107192993164,
      "activations/layer14_attention_weight_min": -36.83406066894531,
      "activations/layer15_attention_weight_max": 38.82219696044922,
      "activations/layer15_attention_weight_min": -34.74066925048828,
      "activations/layer16_attention_weight_max": 27.558069229125977,
      "activations/layer16_attention_weight_min": -28.74730110168457,
      "activations/layer17_attention_weight_max": 29.1630802154541,
      "activations/layer17_attention_weight_min": -26.419429779052734,
      "activations/layer18_attention_weight_max": 30.21259880065918,
      "activations/layer18_attention_weight_min": -25.675188064575195,
      "activations/layer19_attention_weight_max": 30.641088485717773,
      "activations/layer19_attention_weight_min": -32.17008972167969,
      "activations/layer1_attention_weight_max": 15.863773345947266,
      "activations/layer1_attention_weight_min": -16.8568172454834,
      "activations/layer20_attention_weight_max": 27.6931209564209,
      "activations/layer20_attention_weight_min": -25.874221801757812,
      "activations/layer21_attention_weight_max": 26.397628784179688,
      "activations/layer21_attention_weight_min": -24.528440475463867,
      "activations/layer22_attention_weight_max": 38.34590530395508,
      "activations/layer22_attention_weight_min": -30.204010009765625,
      "activations/layer23_attention_weight_max": 27.74336051940918,
      "activations/layer23_attention_weight_min": -25.564884185791016,
      "activations/layer2_attention_weight_max": 33.889801025390625,
      "activations/layer2_attention_weight_min": -33.851226806640625,
      "activations/layer3_attention_weight_max": 102.739501953125,
      "activations/layer3_attention_weight_min": -106.74609375,
      "activations/layer4_attention_weight_max": 106.7313003540039,
      "activations/layer4_attention_weight_min": -108.50167083740234,
      "activations/layer5_attention_weight_max": 53.582122802734375,
      "activations/layer5_attention_weight_min": -59.762420654296875,
      "activations/layer6_attention_weight_max": 43.93151092529297,
      "activations/layer6_attention_weight_min": -47.31581115722656,
      "activations/layer7_attention_weight_max": 98.14588928222656,
      "activations/layer7_attention_weight_min": -100.62057495117188,
      "activations/layer8_attention_weight_max": 43.31757736206055,
      "activations/layer8_attention_weight_min": -46.39204025268555,
      "activations/layer9_attention_weight_max": 32.073184967041016,
      "activations/layer9_attention_weight_min": -34.6049919128418,
      "epoch": 20.68,
      "learning_rate": 1.6791287878787878e-05,
      "loss": 2.7185,
      "step": 355950
    },
    {
      "activations/layer0_attention_weight_max": 17.65535545349121,
      "activations/layer0_attention_weight_min": -13.503747940063477,
      "activations/layer10_attention_weight_max": 38.66838073730469,
      "activations/layer10_attention_weight_min": -37.03767013549805,
      "activations/layer11_attention_weight_max": 36.31101989746094,
      "activations/layer11_attention_weight_min": -35.997413635253906,
      "activations/layer12_attention_weight_max": 28.205520629882812,
      "activations/layer12_attention_weight_min": -28.030860900878906,
      "activations/layer13_attention_weight_max": 42.680843353271484,
      "activations/layer13_attention_weight_min": -36.21864318847656,
      "activations/layer14_attention_weight_max": 42.613250732421875,
      "activations/layer14_attention_weight_min": -37.39295959472656,
      "activations/layer15_attention_weight_max": 41.40560531616211,
      "activations/layer15_attention_weight_min": -35.01231384277344,
      "activations/layer16_attention_weight_max": 30.106422424316406,
      "activations/layer16_attention_weight_min": -27.283227920532227,
      "activations/layer17_attention_weight_max": 29.322614669799805,
      "activations/layer17_attention_weight_min": -26.018449783325195,
      "activations/layer18_attention_weight_max": 34.7730827331543,
      "activations/layer18_attention_weight_min": -28.005104064941406,
      "activations/layer19_attention_weight_max": 36.03767776489258,
      "activations/layer19_attention_weight_min": -34.020877838134766,
      "activations/layer1_attention_weight_max": 15.717988014221191,
      "activations/layer1_attention_weight_min": -14.317130088806152,
      "activations/layer20_attention_weight_max": 30.233501434326172,
      "activations/layer20_attention_weight_min": -26.561338424682617,
      "activations/layer21_attention_weight_max": 29.0802059173584,
      "activations/layer21_attention_weight_min": -25.356985092163086,
      "activations/layer22_attention_weight_max": 40.832672119140625,
      "activations/layer22_attention_weight_min": -33.71631622314453,
      "activations/layer23_attention_weight_max": 35.214420318603516,
      "activations/layer23_attention_weight_min": -26.92531967163086,
      "activations/layer2_attention_weight_max": 33.13551330566406,
      "activations/layer2_attention_weight_min": -34.668067932128906,
      "activations/layer3_attention_weight_max": 105.10286712646484,
      "activations/layer3_attention_weight_min": -108.27660369873047,
      "activations/layer4_attention_weight_max": 109.38134002685547,
      "activations/layer4_attention_weight_min": -114.76810455322266,
      "activations/layer5_attention_weight_max": 51.2036247253418,
      "activations/layer5_attention_weight_min": -63.97175216674805,
      "activations/layer6_attention_weight_max": 45.862728118896484,
      "activations/layer6_attention_weight_min": -52.60702133178711,
      "activations/layer7_attention_weight_max": 101.18433380126953,
      "activations/layer7_attention_weight_min": -107.95829010009766,
      "activations/layer8_attention_weight_max": 44.44676971435547,
      "activations/layer8_attention_weight_min": -48.07868194580078,
      "activations/layer9_attention_weight_max": 36.42173385620117,
      "activations/layer9_attention_weight_min": -35.88239669799805,
      "epoch": 20.69,
      "learning_rate": 1.6772348484848483e-05,
      "loss": 2.7056,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4749,
      "eval_samples_per_second": 506.672,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4749,
      "eval_openwebtext_samples_per_second": 506.672,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9684,
      "eval_wikitext_samples_per_second": 231.661,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_lambada_loss": 2.30859375,
      "eval_lambada_ppl": 10.060267451958905,
      "eval_lambada_runtime": 9.5419,
      "eval_lambada_samples_per_second": 510.277,
      "step": 356000
    },
    {
      "activations/layer0_attention_weight_max": 17.66718864440918,
      "activations/layer0_attention_weight_min": -13.3035306930542,
      "activations/layer10_attention_weight_max": 34.09782791137695,
      "activations/layer10_attention_weight_min": -35.17749786376953,
      "activations/layer11_attention_weight_max": 33.178001403808594,
      "activations/layer11_attention_weight_min": -35.03772735595703,
      "activations/layer12_attention_weight_max": 26.999025344848633,
      "activations/layer12_attention_weight_min": -27.047805786132812,
      "activations/layer13_attention_weight_max": 41.76799774169922,
      "activations/layer13_attention_weight_min": -35.923728942871094,
      "activations/layer14_attention_weight_max": 47.130523681640625,
      "activations/layer14_attention_weight_min": -40.232444763183594,
      "activations/layer15_attention_weight_max": 45.94428253173828,
      "activations/layer15_attention_weight_min": -38.42376708984375,
      "activations/layer16_attention_weight_max": 34.643333435058594,
      "activations/layer16_attention_weight_min": -27.735103607177734,
      "activations/layer17_attention_weight_max": 31.610389709472656,
      "activations/layer17_attention_weight_min": -26.173160552978516,
      "activations/layer18_attention_weight_max": 32.108394622802734,
      "activations/layer18_attention_weight_min": -26.536474227905273,
      "activations/layer19_attention_weight_max": 43.11931610107422,
      "activations/layer19_attention_weight_min": -33.05804443359375,
      "activations/layer1_attention_weight_max": 15.313883781433105,
      "activations/layer1_attention_weight_min": -15.595874786376953,
      "activations/layer20_attention_weight_max": 32.686256408691406,
      "activations/layer20_attention_weight_min": -25.935911178588867,
      "activations/layer21_attention_weight_max": 30.937149047851562,
      "activations/layer21_attention_weight_min": -25.75113868713379,
      "activations/layer22_attention_weight_max": 46.69632339477539,
      "activations/layer22_attention_weight_min": -35.71247863769531,
      "activations/layer23_attention_weight_max": 36.63224792480469,
      "activations/layer23_attention_weight_min": -26.069669723510742,
      "activations/layer2_attention_weight_max": 34.20849609375,
      "activations/layer2_attention_weight_min": -33.38908386230469,
      "activations/layer3_attention_weight_max": 97.77190399169922,
      "activations/layer3_attention_weight_min": -104.26362609863281,
      "activations/layer4_attention_weight_max": 103.08550262451172,
      "activations/layer4_attention_weight_min": -107.4419937133789,
      "activations/layer5_attention_weight_max": 48.85868453979492,
      "activations/layer5_attention_weight_min": -62.362640380859375,
      "activations/layer6_attention_weight_max": 43.41844177246094,
      "activations/layer6_attention_weight_min": -45.848304748535156,
      "activations/layer7_attention_weight_max": 93.56987762451172,
      "activations/layer7_attention_weight_min": -97.35955810546875,
      "activations/layer8_attention_weight_max": 42.5648307800293,
      "activations/layer8_attention_weight_min": -44.10494613647461,
      "activations/layer9_attention_weight_max": 31.735891342163086,
      "activations/layer9_attention_weight_min": -34.644832611083984,
      "epoch": 20.69,
      "learning_rate": 1.6753409090909088e-05,
      "loss": 2.7139,
      "step": 356050
    },
    {
      "activations/layer0_attention_weight_max": 15.792634963989258,
      "activations/layer0_attention_weight_min": -14.48510456085205,
      "activations/layer10_attention_weight_max": 36.56969451904297,
      "activations/layer10_attention_weight_min": -37.07482147216797,
      "activations/layer11_attention_weight_max": 37.17521286010742,
      "activations/layer11_attention_weight_min": -36.79969787597656,
      "activations/layer12_attention_weight_max": 27.72892951965332,
      "activations/layer12_attention_weight_min": -25.75572395324707,
      "activations/layer13_attention_weight_max": 42.670021057128906,
      "activations/layer13_attention_weight_min": -34.775569915771484,
      "activations/layer14_attention_weight_max": 41.56182861328125,
      "activations/layer14_attention_weight_min": -35.9182014465332,
      "activations/layer15_attention_weight_max": 41.147159576416016,
      "activations/layer15_attention_weight_min": -35.278377532958984,
      "activations/layer16_attention_weight_max": 29.95236587524414,
      "activations/layer16_attention_weight_min": -28.22966957092285,
      "activations/layer17_attention_weight_max": 30.755502700805664,
      "activations/layer17_attention_weight_min": -26.064844131469727,
      "activations/layer18_attention_weight_max": 33.500450134277344,
      "activations/layer18_attention_weight_min": -26.370128631591797,
      "activations/layer19_attention_weight_max": 35.52109909057617,
      "activations/layer19_attention_weight_min": -30.075397491455078,
      "activations/layer1_attention_weight_max": 15.609110832214355,
      "activations/layer1_attention_weight_min": -14.838963508605957,
      "activations/layer20_attention_weight_max": 29.944580078125,
      "activations/layer20_attention_weight_min": -24.53987693786621,
      "activations/layer21_attention_weight_max": 29.56671142578125,
      "activations/layer21_attention_weight_min": -25.03196144104004,
      "activations/layer22_attention_weight_max": 43.843833923339844,
      "activations/layer22_attention_weight_min": -31.40730094909668,
      "activations/layer23_attention_weight_max": 34.207923889160156,
      "activations/layer23_attention_weight_min": -25.800886154174805,
      "activations/layer2_attention_weight_max": 33.07032775878906,
      "activations/layer2_attention_weight_min": -32.564002990722656,
      "activations/layer3_attention_weight_max": 105.23663330078125,
      "activations/layer3_attention_weight_min": -101.98271179199219,
      "activations/layer4_attention_weight_max": 109.46662902832031,
      "activations/layer4_attention_weight_min": -110.30670166015625,
      "activations/layer5_attention_weight_max": 50.53434371948242,
      "activations/layer5_attention_weight_min": -61.839317321777344,
      "activations/layer6_attention_weight_max": 45.70412826538086,
      "activations/layer6_attention_weight_min": -49.11880111694336,
      "activations/layer7_attention_weight_max": 96.26742553710938,
      "activations/layer7_attention_weight_min": -101.43297576904297,
      "activations/layer8_attention_weight_max": 45.5777702331543,
      "activations/layer8_attention_weight_min": -44.680450439453125,
      "activations/layer9_attention_weight_max": 32.878360748291016,
      "activations/layer9_attention_weight_min": -35.304718017578125,
      "epoch": 20.69,
      "learning_rate": 1.6734469696969696e-05,
      "loss": 2.7013,
      "step": 356100
    },
    {
      "activations/layer0_attention_weight_max": 18.031644821166992,
      "activations/layer0_attention_weight_min": -14.807158470153809,
      "activations/layer10_attention_weight_max": 33.14909362792969,
      "activations/layer10_attention_weight_min": -35.53252029418945,
      "activations/layer11_attention_weight_max": 35.67719268798828,
      "activations/layer11_attention_weight_min": -34.537986755371094,
      "activations/layer12_attention_weight_max": 28.5513973236084,
      "activations/layer12_attention_weight_min": -27.735353469848633,
      "activations/layer13_attention_weight_max": 42.130821228027344,
      "activations/layer13_attention_weight_min": -36.593074798583984,
      "activations/layer14_attention_weight_max": 43.75341033935547,
      "activations/layer14_attention_weight_min": -39.268497467041016,
      "activations/layer15_attention_weight_max": 41.56507873535156,
      "activations/layer15_attention_weight_min": -37.13309097290039,
      "activations/layer16_attention_weight_max": 31.75279998779297,
      "activations/layer16_attention_weight_min": -28.903074264526367,
      "activations/layer17_attention_weight_max": 31.721065521240234,
      "activations/layer17_attention_weight_min": -27.935800552368164,
      "activations/layer18_attention_weight_max": 37.87548065185547,
      "activations/layer18_attention_weight_min": -27.574604034423828,
      "activations/layer19_attention_weight_max": 42.3587532043457,
      "activations/layer19_attention_weight_min": -34.26266860961914,
      "activations/layer1_attention_weight_max": 15.92999267578125,
      "activations/layer1_attention_weight_min": -15.380241394042969,
      "activations/layer20_attention_weight_max": 32.560951232910156,
      "activations/layer20_attention_weight_min": -28.5682315826416,
      "activations/layer21_attention_weight_max": 33.21393966674805,
      "activations/layer21_attention_weight_min": -29.73158836364746,
      "activations/layer22_attention_weight_max": 48.63467788696289,
      "activations/layer22_attention_weight_min": -37.9088249206543,
      "activations/layer23_attention_weight_max": 39.272987365722656,
      "activations/layer23_attention_weight_min": -26.47610092163086,
      "activations/layer2_attention_weight_max": 34.13308334350586,
      "activations/layer2_attention_weight_min": -32.293853759765625,
      "activations/layer3_attention_weight_max": 105.62088775634766,
      "activations/layer3_attention_weight_min": -105.03980255126953,
      "activations/layer4_attention_weight_max": 110.4408187866211,
      "activations/layer4_attention_weight_min": -115.666259765625,
      "activations/layer5_attention_weight_max": 51.34153747558594,
      "activations/layer5_attention_weight_min": -66.44049072265625,
      "activations/layer6_attention_weight_max": 44.90513229370117,
      "activations/layer6_attention_weight_min": -47.98015213012695,
      "activations/layer7_attention_weight_max": 96.0660171508789,
      "activations/layer7_attention_weight_min": -99.91619873046875,
      "activations/layer8_attention_weight_max": 43.351318359375,
      "activations/layer8_attention_weight_min": -44.92807388305664,
      "activations/layer9_attention_weight_max": 33.004005432128906,
      "activations/layer9_attention_weight_min": -36.34496307373047,
      "epoch": 20.69,
      "learning_rate": 1.67155303030303e-05,
      "loss": 2.7157,
      "step": 356150
    },
    {
      "activations/layer0_attention_weight_max": 16.77708625793457,
      "activations/layer0_attention_weight_min": -13.143027305603027,
      "activations/layer10_attention_weight_max": 39.99997329711914,
      "activations/layer10_attention_weight_min": -35.75664138793945,
      "activations/layer11_attention_weight_max": 39.437644958496094,
      "activations/layer11_attention_weight_min": -40.09773254394531,
      "activations/layer12_attention_weight_max": 25.845685958862305,
      "activations/layer12_attention_weight_min": -28.16280746459961,
      "activations/layer13_attention_weight_max": 40.41075134277344,
      "activations/layer13_attention_weight_min": -39.83527374267578,
      "activations/layer14_attention_weight_max": 43.668643951416016,
      "activations/layer14_attention_weight_min": -43.231163024902344,
      "activations/layer15_attention_weight_max": 37.05126190185547,
      "activations/layer15_attention_weight_min": -38.00510025024414,
      "activations/layer16_attention_weight_max": 30.079429626464844,
      "activations/layer16_attention_weight_min": -29.910844802856445,
      "activations/layer17_attention_weight_max": 27.94865608215332,
      "activations/layer17_attention_weight_min": -27.481124877929688,
      "activations/layer18_attention_weight_max": 32.56855773925781,
      "activations/layer18_attention_weight_min": -25.181865692138672,
      "activations/layer19_attention_weight_max": 33.14814376831055,
      "activations/layer19_attention_weight_min": -32.89719772338867,
      "activations/layer1_attention_weight_max": 15.96318244934082,
      "activations/layer1_attention_weight_min": -16.079795837402344,
      "activations/layer20_attention_weight_max": 28.631696701049805,
      "activations/layer20_attention_weight_min": -25.297945022583008,
      "activations/layer21_attention_weight_max": 27.589908599853516,
      "activations/layer21_attention_weight_min": -23.912742614746094,
      "activations/layer22_attention_weight_max": 42.52233123779297,
      "activations/layer22_attention_weight_min": -31.24917221069336,
      "activations/layer23_attention_weight_max": 31.471797943115234,
      "activations/layer23_attention_weight_min": -26.095134735107422,
      "activations/layer2_attention_weight_max": 35.30036926269531,
      "activations/layer2_attention_weight_min": -35.486351013183594,
      "activations/layer3_attention_weight_max": 104.1895980834961,
      "activations/layer3_attention_weight_min": -104.9016342163086,
      "activations/layer4_attention_weight_max": 105.17736053466797,
      "activations/layer4_attention_weight_min": -109.6153564453125,
      "activations/layer5_attention_weight_max": 49.94840621948242,
      "activations/layer5_attention_weight_min": -59.999183654785156,
      "activations/layer6_attention_weight_max": 44.31387710571289,
      "activations/layer6_attention_weight_min": -47.48899459838867,
      "activations/layer7_attention_weight_max": 103.66045379638672,
      "activations/layer7_attention_weight_min": -97.60823822021484,
      "activations/layer8_attention_weight_max": 46.43992233276367,
      "activations/layer8_attention_weight_min": -44.02019500732422,
      "activations/layer9_attention_weight_max": 31.52805519104004,
      "activations/layer9_attention_weight_min": -34.95254135131836,
      "epoch": 20.7,
      "learning_rate": 1.6696590909090906e-05,
      "loss": 2.7081,
      "step": 356200
    },
    {
      "activations/layer0_attention_weight_max": 16.414871215820312,
      "activations/layer0_attention_weight_min": -14.320256233215332,
      "activations/layer10_attention_weight_max": 35.04680633544922,
      "activations/layer10_attention_weight_min": -38.406898498535156,
      "activations/layer11_attention_weight_max": 34.78731155395508,
      "activations/layer11_attention_weight_min": -36.99816131591797,
      "activations/layer12_attention_weight_max": 27.309152603149414,
      "activations/layer12_attention_weight_min": -26.864089965820312,
      "activations/layer13_attention_weight_max": 40.7586669921875,
      "activations/layer13_attention_weight_min": -36.327980041503906,
      "activations/layer14_attention_weight_max": 41.61545181274414,
      "activations/layer14_attention_weight_min": -37.693138122558594,
      "activations/layer15_attention_weight_max": 39.923240661621094,
      "activations/layer15_attention_weight_min": -38.00228500366211,
      "activations/layer16_attention_weight_max": 29.97726058959961,
      "activations/layer16_attention_weight_min": -29.457176208496094,
      "activations/layer17_attention_weight_max": 29.039133071899414,
      "activations/layer17_attention_weight_min": -27.419700622558594,
      "activations/layer18_attention_weight_max": 34.09620666503906,
      "activations/layer18_attention_weight_min": -27.278465270996094,
      "activations/layer19_attention_weight_max": 33.84673309326172,
      "activations/layer19_attention_weight_min": -32.922725677490234,
      "activations/layer1_attention_weight_max": 17.420530319213867,
      "activations/layer1_attention_weight_min": -16.880456924438477,
      "activations/layer20_attention_weight_max": 28.609350204467773,
      "activations/layer20_attention_weight_min": -27.472639083862305,
      "activations/layer21_attention_weight_max": 28.61913299560547,
      "activations/layer21_attention_weight_min": -26.2921199798584,
      "activations/layer22_attention_weight_max": 37.995750427246094,
      "activations/layer22_attention_weight_min": -32.21858215332031,
      "activations/layer23_attention_weight_max": 32.66109848022461,
      "activations/layer23_attention_weight_min": -26.20132827758789,
      "activations/layer2_attention_weight_max": 32.65428924560547,
      "activations/layer2_attention_weight_min": -32.19303894042969,
      "activations/layer3_attention_weight_max": 106.55538940429688,
      "activations/layer3_attention_weight_min": -105.40845489501953,
      "activations/layer4_attention_weight_max": 112.80968475341797,
      "activations/layer4_attention_weight_min": -112.45482635498047,
      "activations/layer5_attention_weight_max": 51.737213134765625,
      "activations/layer5_attention_weight_min": -66.0008544921875,
      "activations/layer6_attention_weight_max": 46.2534065246582,
      "activations/layer6_attention_weight_min": -49.555416107177734,
      "activations/layer7_attention_weight_max": 98.20584106445312,
      "activations/layer7_attention_weight_min": -104.3964614868164,
      "activations/layer8_attention_weight_max": 43.7334098815918,
      "activations/layer8_attention_weight_min": -49.712032318115234,
      "activations/layer9_attention_weight_max": 34.456764221191406,
      "activations/layer9_attention_weight_min": -38.3381233215332,
      "epoch": 20.7,
      "learning_rate": 1.6677651515151514e-05,
      "loss": 2.7153,
      "step": 356250
    },
    {
      "activations/layer0_attention_weight_max": 17.159799575805664,
      "activations/layer0_attention_weight_min": -13.197088241577148,
      "activations/layer10_attention_weight_max": 34.634918212890625,
      "activations/layer10_attention_weight_min": -36.54401397705078,
      "activations/layer11_attention_weight_max": 34.6619873046875,
      "activations/layer11_attention_weight_min": -35.49393081665039,
      "activations/layer12_attention_weight_max": 28.386201858520508,
      "activations/layer12_attention_weight_min": -26.538986206054688,
      "activations/layer13_attention_weight_max": 40.8746337890625,
      "activations/layer13_attention_weight_min": -36.741722106933594,
      "activations/layer14_attention_weight_max": 43.936161041259766,
      "activations/layer14_attention_weight_min": -38.94070053100586,
      "activations/layer15_attention_weight_max": 40.33450698852539,
      "activations/layer15_attention_weight_min": -37.91749954223633,
      "activations/layer16_attention_weight_max": 33.54781723022461,
      "activations/layer16_attention_weight_min": -27.556713104248047,
      "activations/layer17_attention_weight_max": 31.7408390045166,
      "activations/layer17_attention_weight_min": -27.702009201049805,
      "activations/layer18_attention_weight_max": 32.45079803466797,
      "activations/layer18_attention_weight_min": -29.075021743774414,
      "activations/layer19_attention_weight_max": 38.00765609741211,
      "activations/layer19_attention_weight_min": -37.75385665893555,
      "activations/layer1_attention_weight_max": 15.933271408081055,
      "activations/layer1_attention_weight_min": -15.227912902832031,
      "activations/layer20_attention_weight_max": 31.477519989013672,
      "activations/layer20_attention_weight_min": -28.805986404418945,
      "activations/layer21_attention_weight_max": 32.576839447021484,
      "activations/layer21_attention_weight_min": -27.717254638671875,
      "activations/layer22_attention_weight_max": 46.13389205932617,
      "activations/layer22_attention_weight_min": -34.676353454589844,
      "activations/layer23_attention_weight_max": 32.736915588378906,
      "activations/layer23_attention_weight_min": -27.13196563720703,
      "activations/layer2_attention_weight_max": 33.525299072265625,
      "activations/layer2_attention_weight_min": -32.64020919799805,
      "activations/layer3_attention_weight_max": 101.96623992919922,
      "activations/layer3_attention_weight_min": -106.27172088623047,
      "activations/layer4_attention_weight_max": 106.78175354003906,
      "activations/layer4_attention_weight_min": -112.5297622680664,
      "activations/layer5_attention_weight_max": 49.05584716796875,
      "activations/layer5_attention_weight_min": -61.44212341308594,
      "activations/layer6_attention_weight_max": 43.88626480102539,
      "activations/layer6_attention_weight_min": -48.179500579833984,
      "activations/layer7_attention_weight_max": 95.62042236328125,
      "activations/layer7_attention_weight_min": -93.44905090332031,
      "activations/layer8_attention_weight_max": 42.087764739990234,
      "activations/layer8_attention_weight_min": -45.27495193481445,
      "activations/layer9_attention_weight_max": 32.39107131958008,
      "activations/layer9_attention_weight_min": -35.4782829284668,
      "epoch": 20.7,
      "learning_rate": 1.6658712121212122e-05,
      "loss": 2.7105,
      "step": 356300
    },
    {
      "activations/layer0_attention_weight_max": 16.817218780517578,
      "activations/layer0_attention_weight_min": -14.692662239074707,
      "activations/layer10_attention_weight_max": 34.349063873291016,
      "activations/layer10_attention_weight_min": -34.13068389892578,
      "activations/layer11_attention_weight_max": 35.093238830566406,
      "activations/layer11_attention_weight_min": -34.92795181274414,
      "activations/layer12_attention_weight_max": 26.63924789428711,
      "activations/layer12_attention_weight_min": -26.513647079467773,
      "activations/layer13_attention_weight_max": 40.511932373046875,
      "activations/layer13_attention_weight_min": -36.20240783691406,
      "activations/layer14_attention_weight_max": 43.05097961425781,
      "activations/layer14_attention_weight_min": -37.85081100463867,
      "activations/layer15_attention_weight_max": 41.834678649902344,
      "activations/layer15_attention_weight_min": -35.361328125,
      "activations/layer16_attention_weight_max": 30.82067108154297,
      "activations/layer16_attention_weight_min": -29.22740364074707,
      "activations/layer17_attention_weight_max": 30.056238174438477,
      "activations/layer17_attention_weight_min": -26.14304542541504,
      "activations/layer18_attention_weight_max": 30.034931182861328,
      "activations/layer18_attention_weight_min": -23.5085391998291,
      "activations/layer19_attention_weight_max": 36.367122650146484,
      "activations/layer19_attention_weight_min": -32.102806091308594,
      "activations/layer1_attention_weight_max": 15.671332359313965,
      "activations/layer1_attention_weight_min": -14.775529861450195,
      "activations/layer20_attention_weight_max": 31.461997985839844,
      "activations/layer20_attention_weight_min": -24.751781463623047,
      "activations/layer21_attention_weight_max": 31.645858764648438,
      "activations/layer21_attention_weight_min": -24.44615936279297,
      "activations/layer22_attention_weight_max": 38.535057067871094,
      "activations/layer22_attention_weight_min": -32.06391525268555,
      "activations/layer23_attention_weight_max": 31.302127838134766,
      "activations/layer23_attention_weight_min": -25.124553680419922,
      "activations/layer2_attention_weight_max": 33.283355712890625,
      "activations/layer2_attention_weight_min": -32.814613342285156,
      "activations/layer3_attention_weight_max": 105.02239227294922,
      "activations/layer3_attention_weight_min": -104.77576446533203,
      "activations/layer4_attention_weight_max": 107.83373260498047,
      "activations/layer4_attention_weight_min": -111.29131317138672,
      "activations/layer5_attention_weight_max": 52.0816650390625,
      "activations/layer5_attention_weight_min": -60.733856201171875,
      "activations/layer6_attention_weight_max": 44.152828216552734,
      "activations/layer6_attention_weight_min": -45.72663879394531,
      "activations/layer7_attention_weight_max": 95.17691040039062,
      "activations/layer7_attention_weight_min": -96.44161224365234,
      "activations/layer8_attention_weight_max": 43.12004089355469,
      "activations/layer8_attention_weight_min": -43.289432525634766,
      "activations/layer9_attention_weight_max": 32.45149230957031,
      "activations/layer9_attention_weight_min": -34.11229705810547,
      "epoch": 20.71,
      "learning_rate": 1.6639772727272727e-05,
      "loss": 2.7122,
      "step": 356350
    },
    {
      "activations/layer0_attention_weight_max": 17.024707794189453,
      "activations/layer0_attention_weight_min": -14.049424171447754,
      "activations/layer10_attention_weight_max": 35.69710159301758,
      "activations/layer10_attention_weight_min": -36.78871154785156,
      "activations/layer11_attention_weight_max": 36.297271728515625,
      "activations/layer11_attention_weight_min": -34.32598114013672,
      "activations/layer12_attention_weight_max": 27.823518753051758,
      "activations/layer12_attention_weight_min": -28.350324630737305,
      "activations/layer13_attention_weight_max": 40.292938232421875,
      "activations/layer13_attention_weight_min": -38.01533126831055,
      "activations/layer14_attention_weight_max": 43.5289421081543,
      "activations/layer14_attention_weight_min": -40.35726547241211,
      "activations/layer15_attention_weight_max": 39.309513092041016,
      "activations/layer15_attention_weight_min": -39.13801956176758,
      "activations/layer16_attention_weight_max": 29.700170516967773,
      "activations/layer16_attention_weight_min": -29.398969650268555,
      "activations/layer17_attention_weight_max": 31.068275451660156,
      "activations/layer17_attention_weight_min": -28.033098220825195,
      "activations/layer18_attention_weight_max": 32.07653045654297,
      "activations/layer18_attention_weight_min": -28.68581771850586,
      "activations/layer19_attention_weight_max": 38.619293212890625,
      "activations/layer19_attention_weight_min": -31.67453384399414,
      "activations/layer1_attention_weight_max": 16.76563262939453,
      "activations/layer1_attention_weight_min": -15.592986106872559,
      "activations/layer20_attention_weight_max": 32.23432159423828,
      "activations/layer20_attention_weight_min": -26.214061737060547,
      "activations/layer21_attention_weight_max": 31.782394409179688,
      "activations/layer21_attention_weight_min": -26.4735164642334,
      "activations/layer22_attention_weight_max": 38.91372299194336,
      "activations/layer22_attention_weight_min": -32.63246154785156,
      "activations/layer23_attention_weight_max": 32.60980987548828,
      "activations/layer23_attention_weight_min": -27.413402557373047,
      "activations/layer2_attention_weight_max": 34.24016189575195,
      "activations/layer2_attention_weight_min": -33.29099655151367,
      "activations/layer3_attention_weight_max": 108.8368911743164,
      "activations/layer3_attention_weight_min": -106.37239074707031,
      "activations/layer4_attention_weight_max": 115.4014892578125,
      "activations/layer4_attention_weight_min": -108.16786193847656,
      "activations/layer5_attention_weight_max": 50.06884002685547,
      "activations/layer5_attention_weight_min": -63.70893096923828,
      "activations/layer6_attention_weight_max": 45.80686950683594,
      "activations/layer6_attention_weight_min": -45.26219177246094,
      "activations/layer7_attention_weight_max": 95.91861724853516,
      "activations/layer7_attention_weight_min": -96.09772491455078,
      "activations/layer8_attention_weight_max": 44.647308349609375,
      "activations/layer8_attention_weight_min": -44.29317855834961,
      "activations/layer9_attention_weight_max": 33.206295013427734,
      "activations/layer9_attention_weight_min": -35.947486877441406,
      "epoch": 20.71,
      "learning_rate": 1.6620833333333332e-05,
      "loss": 2.7228,
      "step": 356400
    },
    {
      "activations/layer0_attention_weight_max": 16.476455688476562,
      "activations/layer0_attention_weight_min": -13.41518783569336,
      "activations/layer10_attention_weight_max": 43.25712966918945,
      "activations/layer10_attention_weight_min": -35.82164001464844,
      "activations/layer11_attention_weight_max": 40.793251037597656,
      "activations/layer11_attention_weight_min": -36.386871337890625,
      "activations/layer12_attention_weight_max": 26.024320602416992,
      "activations/layer12_attention_weight_min": -25.499893188476562,
      "activations/layer13_attention_weight_max": 38.86732482910156,
      "activations/layer13_attention_weight_min": -35.88547134399414,
      "activations/layer14_attention_weight_max": 40.284141540527344,
      "activations/layer14_attention_weight_min": -36.41294479370117,
      "activations/layer15_attention_weight_max": 38.29589080810547,
      "activations/layer15_attention_weight_min": -36.28496551513672,
      "activations/layer16_attention_weight_max": 27.191574096679688,
      "activations/layer16_attention_weight_min": -32.07551193237305,
      "activations/layer17_attention_weight_max": 27.788326263427734,
      "activations/layer17_attention_weight_min": -27.29328727722168,
      "activations/layer18_attention_weight_max": 29.649019241333008,
      "activations/layer18_attention_weight_min": -24.624547958374023,
      "activations/layer19_attention_weight_max": 31.745710372924805,
      "activations/layer19_attention_weight_min": -30.90826988220215,
      "activations/layer1_attention_weight_max": 15.843259811401367,
      "activations/layer1_attention_weight_min": -16.36236572265625,
      "activations/layer20_attention_weight_max": 26.055574417114258,
      "activations/layer20_attention_weight_min": -24.277873992919922,
      "activations/layer21_attention_weight_max": 25.997478485107422,
      "activations/layer21_attention_weight_min": -24.46098518371582,
      "activations/layer22_attention_weight_max": 37.666465759277344,
      "activations/layer22_attention_weight_min": -32.43324279785156,
      "activations/layer23_attention_weight_max": 28.858726501464844,
      "activations/layer23_attention_weight_min": -24.01820945739746,
      "activations/layer2_attention_weight_max": 33.49059295654297,
      "activations/layer2_attention_weight_min": -31.183300018310547,
      "activations/layer3_attention_weight_max": 100.75862121582031,
      "activations/layer3_attention_weight_min": -102.43693542480469,
      "activations/layer4_attention_weight_max": 108.1928482055664,
      "activations/layer4_attention_weight_min": -111.2552719116211,
      "activations/layer5_attention_weight_max": 49.795433044433594,
      "activations/layer5_attention_weight_min": -61.59593200683594,
      "activations/layer6_attention_weight_max": 43.454402923583984,
      "activations/layer6_attention_weight_min": -49.11513900756836,
      "activations/layer7_attention_weight_max": 106.08995056152344,
      "activations/layer7_attention_weight_min": -102.7142105102539,
      "activations/layer8_attention_weight_max": 48.9797248840332,
      "activations/layer8_attention_weight_min": -46.55461883544922,
      "activations/layer9_attention_weight_max": 36.23368835449219,
      "activations/layer9_attention_weight_min": -37.72425842285156,
      "epoch": 20.71,
      "learning_rate": 1.660189393939394e-05,
      "loss": 2.7109,
      "step": 356450
    },
    {
      "activations/layer0_attention_weight_max": 15.82557487487793,
      "activations/layer0_attention_weight_min": -13.012529373168945,
      "activations/layer10_attention_weight_max": 36.674049377441406,
      "activations/layer10_attention_weight_min": -38.7261962890625,
      "activations/layer11_attention_weight_max": 37.680267333984375,
      "activations/layer11_attention_weight_min": -37.92821502685547,
      "activations/layer12_attention_weight_max": 30.23383903503418,
      "activations/layer12_attention_weight_min": -28.166553497314453,
      "activations/layer13_attention_weight_max": 41.35112762451172,
      "activations/layer13_attention_weight_min": -37.22800827026367,
      "activations/layer14_attention_weight_max": 46.18115234375,
      "activations/layer14_attention_weight_min": -39.62018966674805,
      "activations/layer15_attention_weight_max": 42.657291412353516,
      "activations/layer15_attention_weight_min": -38.69198226928711,
      "activations/layer16_attention_weight_max": 30.36945343017578,
      "activations/layer16_attention_weight_min": -29.05940055847168,
      "activations/layer17_attention_weight_max": 29.656938552856445,
      "activations/layer17_attention_weight_min": -25.670434951782227,
      "activations/layer18_attention_weight_max": 33.15336608886719,
      "activations/layer18_attention_weight_min": -24.217998504638672,
      "activations/layer19_attention_weight_max": 36.18684387207031,
      "activations/layer19_attention_weight_min": -31.58758544921875,
      "activations/layer1_attention_weight_max": 15.783296585083008,
      "activations/layer1_attention_weight_min": -14.698420524597168,
      "activations/layer20_attention_weight_max": 29.970346450805664,
      "activations/layer20_attention_weight_min": -26.405590057373047,
      "activations/layer21_attention_weight_max": 32.448883056640625,
      "activations/layer21_attention_weight_min": -24.809581756591797,
      "activations/layer22_attention_weight_max": 40.89239501953125,
      "activations/layer22_attention_weight_min": -31.441858291625977,
      "activations/layer23_attention_weight_max": 34.249732971191406,
      "activations/layer23_attention_weight_min": -25.697383880615234,
      "activations/layer2_attention_weight_max": 32.75865936279297,
      "activations/layer2_attention_weight_min": -34.30244064331055,
      "activations/layer3_attention_weight_max": 105.84798431396484,
      "activations/layer3_attention_weight_min": -107.74385070800781,
      "activations/layer4_attention_weight_max": 104.8942642211914,
      "activations/layer4_attention_weight_min": -107.63777160644531,
      "activations/layer5_attention_weight_max": 51.319183349609375,
      "activations/layer5_attention_weight_min": -60.057247161865234,
      "activations/layer6_attention_weight_max": 44.46955108642578,
      "activations/layer6_attention_weight_min": -45.386558532714844,
      "activations/layer7_attention_weight_max": 93.33811950683594,
      "activations/layer7_attention_weight_min": -105.11903381347656,
      "activations/layer8_attention_weight_max": 42.83778381347656,
      "activations/layer8_attention_weight_min": -45.37935256958008,
      "activations/layer9_attention_weight_max": 32.84857940673828,
      "activations/layer9_attention_weight_min": -35.5903434753418,
      "epoch": 20.71,
      "learning_rate": 1.6582954545454545e-05,
      "loss": 2.7232,
      "step": 356500
    },
    {
      "activations/layer0_attention_weight_max": 15.882854461669922,
      "activations/layer0_attention_weight_min": -12.890215873718262,
      "activations/layer10_attention_weight_max": 34.9903564453125,
      "activations/layer10_attention_weight_min": -34.837310791015625,
      "activations/layer11_attention_weight_max": 34.0865364074707,
      "activations/layer11_attention_weight_min": -36.746192932128906,
      "activations/layer12_attention_weight_max": 26.65618133544922,
      "activations/layer12_attention_weight_min": -28.681798934936523,
      "activations/layer13_attention_weight_max": 40.241905212402344,
      "activations/layer13_attention_weight_min": -37.89128494262695,
      "activations/layer14_attention_weight_max": 41.59684371948242,
      "activations/layer14_attention_weight_min": -39.326210021972656,
      "activations/layer15_attention_weight_max": 41.21434783935547,
      "activations/layer15_attention_weight_min": -40.021339416503906,
      "activations/layer16_attention_weight_max": 32.353878021240234,
      "activations/layer16_attention_weight_min": -31.5294189453125,
      "activations/layer17_attention_weight_max": 31.15907096862793,
      "activations/layer17_attention_weight_min": -28.08249855041504,
      "activations/layer18_attention_weight_max": 31.908817291259766,
      "activations/layer18_attention_weight_min": -26.92461395263672,
      "activations/layer19_attention_weight_max": 36.68459701538086,
      "activations/layer19_attention_weight_min": -34.57892608642578,
      "activations/layer1_attention_weight_max": 15.7141752243042,
      "activations/layer1_attention_weight_min": -16.135984420776367,
      "activations/layer20_attention_weight_max": 29.40111541748047,
      "activations/layer20_attention_weight_min": -27.164348602294922,
      "activations/layer21_attention_weight_max": 30.829898834228516,
      "activations/layer21_attention_weight_min": -27.8441219329834,
      "activations/layer22_attention_weight_max": 40.95845413208008,
      "activations/layer22_attention_weight_min": -31.689315795898438,
      "activations/layer23_attention_weight_max": 32.66960906982422,
      "activations/layer23_attention_weight_min": -27.056446075439453,
      "activations/layer2_attention_weight_max": 33.65061950683594,
      "activations/layer2_attention_weight_min": -33.96126174926758,
      "activations/layer3_attention_weight_max": 103.42532348632812,
      "activations/layer3_attention_weight_min": -106.03807067871094,
      "activations/layer4_attention_weight_max": 111.70882415771484,
      "activations/layer4_attention_weight_min": -114.12056732177734,
      "activations/layer5_attention_weight_max": 49.61341857910156,
      "activations/layer5_attention_weight_min": -65.04763793945312,
      "activations/layer6_attention_weight_max": 45.3070182800293,
      "activations/layer6_attention_weight_min": -48.96607971191406,
      "activations/layer7_attention_weight_max": 95.91830444335938,
      "activations/layer7_attention_weight_min": -106.3796157836914,
      "activations/layer8_attention_weight_max": 42.46113967895508,
      "activations/layer8_attention_weight_min": -45.08073043823242,
      "activations/layer9_attention_weight_max": 32.51659393310547,
      "activations/layer9_attention_weight_min": -36.297969818115234,
      "epoch": 20.72,
      "learning_rate": 1.656401515151515e-05,
      "loss": 2.6998,
      "step": 356550
    },
    {
      "activations/layer0_attention_weight_max": 16.909709930419922,
      "activations/layer0_attention_weight_min": -13.272186279296875,
      "activations/layer10_attention_weight_max": 31.918554306030273,
      "activations/layer10_attention_weight_min": -34.26090621948242,
      "activations/layer11_attention_weight_max": 31.432607650756836,
      "activations/layer11_attention_weight_min": -35.4093017578125,
      "activations/layer12_attention_weight_max": 27.2788143157959,
      "activations/layer12_attention_weight_min": -25.900522232055664,
      "activations/layer13_attention_weight_max": 39.71303939819336,
      "activations/layer13_attention_weight_min": -34.262359619140625,
      "activations/layer14_attention_weight_max": 42.14128494262695,
      "activations/layer14_attention_weight_min": -37.1394157409668,
      "activations/layer15_attention_weight_max": 38.2769889831543,
      "activations/layer15_attention_weight_min": -36.25418472290039,
      "activations/layer16_attention_weight_max": 29.742647171020508,
      "activations/layer16_attention_weight_min": -26.984127044677734,
      "activations/layer17_attention_weight_max": 29.743032455444336,
      "activations/layer17_attention_weight_min": -25.576393127441406,
      "activations/layer18_attention_weight_max": 33.1003532409668,
      "activations/layer18_attention_weight_min": -25.685258865356445,
      "activations/layer19_attention_weight_max": 32.97418212890625,
      "activations/layer19_attention_weight_min": -33.03263473510742,
      "activations/layer1_attention_weight_max": 15.71981430053711,
      "activations/layer1_attention_weight_min": -14.467172622680664,
      "activations/layer20_attention_weight_max": 28.62969970703125,
      "activations/layer20_attention_weight_min": -26.2207088470459,
      "activations/layer21_attention_weight_max": 27.903642654418945,
      "activations/layer21_attention_weight_min": -24.829832077026367,
      "activations/layer22_attention_weight_max": 41.186668395996094,
      "activations/layer22_attention_weight_min": -33.649436950683594,
      "activations/layer23_attention_weight_max": 29.95025634765625,
      "activations/layer23_attention_weight_min": -26.182941436767578,
      "activations/layer2_attention_weight_max": 35.139835357666016,
      "activations/layer2_attention_weight_min": -32.728759765625,
      "activations/layer3_attention_weight_max": 101.46299743652344,
      "activations/layer3_attention_weight_min": -103.66263580322266,
      "activations/layer4_attention_weight_max": 106.68885803222656,
      "activations/layer4_attention_weight_min": -107.90692138671875,
      "activations/layer5_attention_weight_max": 50.512962341308594,
      "activations/layer5_attention_weight_min": -60.039817810058594,
      "activations/layer6_attention_weight_max": 44.89342498779297,
      "activations/layer6_attention_weight_min": -47.53837585449219,
      "activations/layer7_attention_weight_max": 93.26849365234375,
      "activations/layer7_attention_weight_min": -96.61724853515625,
      "activations/layer8_attention_weight_max": 40.97236251831055,
      "activations/layer8_attention_weight_min": -44.012489318847656,
      "activations/layer9_attention_weight_max": 31.768957138061523,
      "activations/layer9_attention_weight_min": -33.50469970703125,
      "epoch": 20.72,
      "learning_rate": 1.6545075757575755e-05,
      "loss": 2.7332,
      "step": 356600
    },
    {
      "activations/layer0_attention_weight_max": 17.46177101135254,
      "activations/layer0_attention_weight_min": -13.341526985168457,
      "activations/layer10_attention_weight_max": 45.412654876708984,
      "activations/layer10_attention_weight_min": -42.95674133300781,
      "activations/layer11_attention_weight_max": 41.88816452026367,
      "activations/layer11_attention_weight_min": -42.12364959716797,
      "activations/layer12_attention_weight_max": 28.303544998168945,
      "activations/layer12_attention_weight_min": -28.44082260131836,
      "activations/layer13_attention_weight_max": 39.22767639160156,
      "activations/layer13_attention_weight_min": -36.98417663574219,
      "activations/layer14_attention_weight_max": 42.58422088623047,
      "activations/layer14_attention_weight_min": -39.295188903808594,
      "activations/layer15_attention_weight_max": 40.92422103881836,
      "activations/layer15_attention_weight_min": -39.981998443603516,
      "activations/layer16_attention_weight_max": 31.490251541137695,
      "activations/layer16_attention_weight_min": -28.41415786743164,
      "activations/layer17_attention_weight_max": 28.92501449584961,
      "activations/layer17_attention_weight_min": -28.365394592285156,
      "activations/layer18_attention_weight_max": 30.408353805541992,
      "activations/layer18_attention_weight_min": -26.465545654296875,
      "activations/layer19_attention_weight_max": 37.847206115722656,
      "activations/layer19_attention_weight_min": -32.50613021850586,
      "activations/layer1_attention_weight_max": 15.853156089782715,
      "activations/layer1_attention_weight_min": -14.634923934936523,
      "activations/layer20_attention_weight_max": 30.57988739013672,
      "activations/layer20_attention_weight_min": -25.688064575195312,
      "activations/layer21_attention_weight_max": 27.163358688354492,
      "activations/layer21_attention_weight_min": -25.714481353759766,
      "activations/layer22_attention_weight_max": 41.61875534057617,
      "activations/layer22_attention_weight_min": -31.995840072631836,
      "activations/layer23_attention_weight_max": 28.721885681152344,
      "activations/layer23_attention_weight_min": -27.220869064331055,
      "activations/layer2_attention_weight_max": 33.37916564941406,
      "activations/layer2_attention_weight_min": -33.823482513427734,
      "activations/layer3_attention_weight_max": 102.53519439697266,
      "activations/layer3_attention_weight_min": -105.98542022705078,
      "activations/layer4_attention_weight_max": 105.6882095336914,
      "activations/layer4_attention_weight_min": -109.20415496826172,
      "activations/layer5_attention_weight_max": 50.14094543457031,
      "activations/layer5_attention_weight_min": -60.99296188354492,
      "activations/layer6_attention_weight_max": 50.5654182434082,
      "activations/layer6_attention_weight_min": -48.67025375366211,
      "activations/layer7_attention_weight_max": 112.21187591552734,
      "activations/layer7_attention_weight_min": -110.17427062988281,
      "activations/layer8_attention_weight_max": 50.63338851928711,
      "activations/layer8_attention_weight_min": -53.37233352661133,
      "activations/layer9_attention_weight_max": 39.02638244628906,
      "activations/layer9_attention_weight_min": -41.14362716674805,
      "epoch": 20.72,
      "learning_rate": 1.6526136363636363e-05,
      "loss": 2.7019,
      "step": 356650
    },
    {
      "activations/layer0_attention_weight_max": 16.30670738220215,
      "activations/layer0_attention_weight_min": -14.890501976013184,
      "activations/layer10_attention_weight_max": 34.453521728515625,
      "activations/layer10_attention_weight_min": -35.647918701171875,
      "activations/layer11_attention_weight_max": 36.023101806640625,
      "activations/layer11_attention_weight_min": -34.884864807128906,
      "activations/layer12_attention_weight_max": 26.489669799804688,
      "activations/layer12_attention_weight_min": -27.016782760620117,
      "activations/layer13_attention_weight_max": 40.364410400390625,
      "activations/layer13_attention_weight_min": -35.784759521484375,
      "activations/layer14_attention_weight_max": 41.69164276123047,
      "activations/layer14_attention_weight_min": -37.68262481689453,
      "activations/layer15_attention_weight_max": 40.59748077392578,
      "activations/layer15_attention_weight_min": -36.61870574951172,
      "activations/layer16_attention_weight_max": 28.340557098388672,
      "activations/layer16_attention_weight_min": -30.118993759155273,
      "activations/layer17_attention_weight_max": 30.90190315246582,
      "activations/layer17_attention_weight_min": -26.67950439453125,
      "activations/layer18_attention_weight_max": 35.1992301940918,
      "activations/layer18_attention_weight_min": -27.770015716552734,
      "activations/layer19_attention_weight_max": 34.29669952392578,
      "activations/layer19_attention_weight_min": -32.47270584106445,
      "activations/layer1_attention_weight_max": 15.674327850341797,
      "activations/layer1_attention_weight_min": -15.273937225341797,
      "activations/layer20_attention_weight_max": 30.547821044921875,
      "activations/layer20_attention_weight_min": -28.018808364868164,
      "activations/layer21_attention_weight_max": 31.62804412841797,
      "activations/layer21_attention_weight_min": -26.114282608032227,
      "activations/layer22_attention_weight_max": 43.3887939453125,
      "activations/layer22_attention_weight_min": -33.21000671386719,
      "activations/layer23_attention_weight_max": 33.19410705566406,
      "activations/layer23_attention_weight_min": -27.073843002319336,
      "activations/layer2_attention_weight_max": 34.22300720214844,
      "activations/layer2_attention_weight_min": -33.4896354675293,
      "activations/layer3_attention_weight_max": 106.85985565185547,
      "activations/layer3_attention_weight_min": -106.90872192382812,
      "activations/layer4_attention_weight_max": 108.38519287109375,
      "activations/layer4_attention_weight_min": -111.3996810913086,
      "activations/layer5_attention_weight_max": 50.43159484863281,
      "activations/layer5_attention_weight_min": -61.647247314453125,
      "activations/layer6_attention_weight_max": 46.11067581176758,
      "activations/layer6_attention_weight_min": -50.08018112182617,
      "activations/layer7_attention_weight_max": 97.92774200439453,
      "activations/layer7_attention_weight_min": -100.7125473022461,
      "activations/layer8_attention_weight_max": 42.338050842285156,
      "activations/layer8_attention_weight_min": -44.430763244628906,
      "activations/layer9_attention_weight_max": 33.079490661621094,
      "activations/layer9_attention_weight_min": -35.48751449584961,
      "epoch": 20.73,
      "learning_rate": 1.6507196969696968e-05,
      "loss": 2.7187,
      "step": 356700
    },
    {
      "activations/layer0_attention_weight_max": 16.97863006591797,
      "activations/layer0_attention_weight_min": -14.402406692504883,
      "activations/layer10_attention_weight_max": 37.455078125,
      "activations/layer10_attention_weight_min": -37.67435073852539,
      "activations/layer11_attention_weight_max": 36.84388732910156,
      "activations/layer11_attention_weight_min": -38.607688903808594,
      "activations/layer12_attention_weight_max": 29.61185073852539,
      "activations/layer12_attention_weight_min": -27.860258102416992,
      "activations/layer13_attention_weight_max": 42.06388854980469,
      "activations/layer13_attention_weight_min": -35.929378509521484,
      "activations/layer14_attention_weight_max": 44.982215881347656,
      "activations/layer14_attention_weight_min": -39.05057144165039,
      "activations/layer15_attention_weight_max": 40.756492614746094,
      "activations/layer15_attention_weight_min": -38.21425247192383,
      "activations/layer16_attention_weight_max": 32.76531219482422,
      "activations/layer16_attention_weight_min": -28.853458404541016,
      "activations/layer17_attention_weight_max": 31.883867263793945,
      "activations/layer17_attention_weight_min": -28.86661148071289,
      "activations/layer18_attention_weight_max": 34.07599639892578,
      "activations/layer18_attention_weight_min": -30.311729431152344,
      "activations/layer19_attention_weight_max": 37.37671661376953,
      "activations/layer19_attention_weight_min": -32.51558303833008,
      "activations/layer1_attention_weight_max": 15.429000854492188,
      "activations/layer1_attention_weight_min": -15.008820533752441,
      "activations/layer20_attention_weight_max": 31.694957733154297,
      "activations/layer20_attention_weight_min": -26.469982147216797,
      "activations/layer21_attention_weight_max": 31.624107360839844,
      "activations/layer21_attention_weight_min": -27.076711654663086,
      "activations/layer22_attention_weight_max": 43.77056884765625,
      "activations/layer22_attention_weight_min": -34.89033508300781,
      "activations/layer23_attention_weight_max": 38.81828308105469,
      "activations/layer23_attention_weight_min": -28.28179168701172,
      "activations/layer2_attention_weight_max": 34.926483154296875,
      "activations/layer2_attention_weight_min": -34.15890121459961,
      "activations/layer3_attention_weight_max": 107.66905975341797,
      "activations/layer3_attention_weight_min": -105.93412017822266,
      "activations/layer4_attention_weight_max": 109.74161529541016,
      "activations/layer4_attention_weight_min": -112.288818359375,
      "activations/layer5_attention_weight_max": 52.54343795776367,
      "activations/layer5_attention_weight_min": -62.50984191894531,
      "activations/layer6_attention_weight_max": 48.23204040527344,
      "activations/layer6_attention_weight_min": -50.71434783935547,
      "activations/layer7_attention_weight_max": 99.73546600341797,
      "activations/layer7_attention_weight_min": -101.69804382324219,
      "activations/layer8_attention_weight_max": 46.09593963623047,
      "activations/layer8_attention_weight_min": -49.23907470703125,
      "activations/layer9_attention_weight_max": 35.80873489379883,
      "activations/layer9_attention_weight_min": -38.5361213684082,
      "epoch": 20.73,
      "learning_rate": 1.6488257575757573e-05,
      "loss": 2.7141,
      "step": 356750
    },
    {
      "activations/layer0_attention_weight_max": 16.54665756225586,
      "activations/layer0_attention_weight_min": -13.155508995056152,
      "activations/layer10_attention_weight_max": 31.963516235351562,
      "activations/layer10_attention_weight_min": -33.77526092529297,
      "activations/layer11_attention_weight_max": 33.17619323730469,
      "activations/layer11_attention_weight_min": -32.748748779296875,
      "activations/layer12_attention_weight_max": 27.853958129882812,
      "activations/layer12_attention_weight_min": -26.870990753173828,
      "activations/layer13_attention_weight_max": 37.78782653808594,
      "activations/layer13_attention_weight_min": -34.96760940551758,
      "activations/layer14_attention_weight_max": 40.22295379638672,
      "activations/layer14_attention_weight_min": -37.71025085449219,
      "activations/layer15_attention_weight_max": 40.09114074707031,
      "activations/layer15_attention_weight_min": -34.858760833740234,
      "activations/layer16_attention_weight_max": 29.949420928955078,
      "activations/layer16_attention_weight_min": -29.558597564697266,
      "activations/layer17_attention_weight_max": 29.63591194152832,
      "activations/layer17_attention_weight_min": -26.489267349243164,
      "activations/layer18_attention_weight_max": 31.96587562561035,
      "activations/layer18_attention_weight_min": -26.814706802368164,
      "activations/layer19_attention_weight_max": 38.1595458984375,
      "activations/layer19_attention_weight_min": -31.851383209228516,
      "activations/layer1_attention_weight_max": 15.984698295593262,
      "activations/layer1_attention_weight_min": -15.92333984375,
      "activations/layer20_attention_weight_max": 33.32149124145508,
      "activations/layer20_attention_weight_min": -25.665225982666016,
      "activations/layer21_attention_weight_max": 30.6765193939209,
      "activations/layer21_attention_weight_min": -23.93206024169922,
      "activations/layer22_attention_weight_max": 45.57569885253906,
      "activations/layer22_attention_weight_min": -31.522594451904297,
      "activations/layer23_attention_weight_max": 33.87516784667969,
      "activations/layer23_attention_weight_min": -24.98493194580078,
      "activations/layer2_attention_weight_max": 34.013824462890625,
      "activations/layer2_attention_weight_min": -32.737266540527344,
      "activations/layer3_attention_weight_max": 100.99848937988281,
      "activations/layer3_attention_weight_min": -101.13995361328125,
      "activations/layer4_attention_weight_max": 106.31771087646484,
      "activations/layer4_attention_weight_min": -111.41849517822266,
      "activations/layer5_attention_weight_max": 48.970619201660156,
      "activations/layer5_attention_weight_min": -65.98331451416016,
      "activations/layer6_attention_weight_max": 44.21610641479492,
      "activations/layer6_attention_weight_min": -45.62912368774414,
      "activations/layer7_attention_weight_max": 88.90003967285156,
      "activations/layer7_attention_weight_min": -98.95402526855469,
      "activations/layer8_attention_weight_max": 43.4692268371582,
      "activations/layer8_attention_weight_min": -41.56700134277344,
      "activations/layer9_attention_weight_max": 33.07505416870117,
      "activations/layer9_attention_weight_min": -33.51034164428711,
      "epoch": 20.73,
      "learning_rate": 1.646931818181818e-05,
      "loss": 2.7173,
      "step": 356800
    },
    {
      "activations/layer0_attention_weight_max": 17.07539176940918,
      "activations/layer0_attention_weight_min": -12.77128791809082,
      "activations/layer10_attention_weight_max": 36.0329475402832,
      "activations/layer10_attention_weight_min": -34.36129379272461,
      "activations/layer11_attention_weight_max": 35.235836029052734,
      "activations/layer11_attention_weight_min": -34.89588928222656,
      "activations/layer12_attention_weight_max": 29.954479217529297,
      "activations/layer12_attention_weight_min": -27.862979888916016,
      "activations/layer13_attention_weight_max": 44.25135803222656,
      "activations/layer13_attention_weight_min": -37.38212203979492,
      "activations/layer14_attention_weight_max": 47.34431838989258,
      "activations/layer14_attention_weight_min": -39.45274353027344,
      "activations/layer15_attention_weight_max": 41.03667449951172,
      "activations/layer15_attention_weight_min": -36.71455383300781,
      "activations/layer16_attention_weight_max": 32.991127014160156,
      "activations/layer16_attention_weight_min": -27.347471237182617,
      "activations/layer17_attention_weight_max": 30.75325584411621,
      "activations/layer17_attention_weight_min": -26.46560287475586,
      "activations/layer18_attention_weight_max": 32.202274322509766,
      "activations/layer18_attention_weight_min": -23.63553237915039,
      "activations/layer19_attention_weight_max": 36.888301849365234,
      "activations/layer19_attention_weight_min": -31.2788143157959,
      "activations/layer1_attention_weight_max": 16.24647331237793,
      "activations/layer1_attention_weight_min": -15.512923240661621,
      "activations/layer20_attention_weight_max": 32.22907638549805,
      "activations/layer20_attention_weight_min": -25.20494842529297,
      "activations/layer21_attention_weight_max": 29.069231033325195,
      "activations/layer21_attention_weight_min": -24.722002029418945,
      "activations/layer22_attention_weight_max": 47.51676559448242,
      "activations/layer22_attention_weight_min": -30.82212257385254,
      "activations/layer23_attention_weight_max": 34.7962760925293,
      "activations/layer23_attention_weight_min": -26.360815048217773,
      "activations/layer2_attention_weight_max": 34.22614288330078,
      "activations/layer2_attention_weight_min": -32.449012756347656,
      "activations/layer3_attention_weight_max": 105.11373901367188,
      "activations/layer3_attention_weight_min": -108.6505355834961,
      "activations/layer4_attention_weight_max": 109.28004455566406,
      "activations/layer4_attention_weight_min": -111.4091796875,
      "activations/layer5_attention_weight_max": 52.19310760498047,
      "activations/layer5_attention_weight_min": -63.418800354003906,
      "activations/layer6_attention_weight_max": 47.02406311035156,
      "activations/layer6_attention_weight_min": -47.824764251708984,
      "activations/layer7_attention_weight_max": 95.53533935546875,
      "activations/layer7_attention_weight_min": -102.31755828857422,
      "activations/layer8_attention_weight_max": 46.08456039428711,
      "activations/layer8_attention_weight_min": -46.19361877441406,
      "activations/layer9_attention_weight_max": 34.04528045654297,
      "activations/layer9_attention_weight_min": -35.555606842041016,
      "epoch": 20.74,
      "learning_rate": 1.6450378787878786e-05,
      "loss": 2.7051,
      "step": 356850
    },
    {
      "activations/layer0_attention_weight_max": 16.171890258789062,
      "activations/layer0_attention_weight_min": -13.785504341125488,
      "activations/layer10_attention_weight_max": 34.724281311035156,
      "activations/layer10_attention_weight_min": -35.68592071533203,
      "activations/layer11_attention_weight_max": 34.92234802246094,
      "activations/layer11_attention_weight_min": -36.27136993408203,
      "activations/layer12_attention_weight_max": 26.93999481201172,
      "activations/layer12_attention_weight_min": -27.817277908325195,
      "activations/layer13_attention_weight_max": 39.9580078125,
      "activations/layer13_attention_weight_min": -36.79203414916992,
      "activations/layer14_attention_weight_max": 43.69459915161133,
      "activations/layer14_attention_weight_min": -39.3089485168457,
      "activations/layer15_attention_weight_max": 40.0765380859375,
      "activations/layer15_attention_weight_min": -36.24728012084961,
      "activations/layer16_attention_weight_max": 30.329105377197266,
      "activations/layer16_attention_weight_min": -29.08641815185547,
      "activations/layer17_attention_weight_max": 30.811452865600586,
      "activations/layer17_attention_weight_min": -28.004667282104492,
      "activations/layer18_attention_weight_max": 33.113372802734375,
      "activations/layer18_attention_weight_min": -27.149208068847656,
      "activations/layer19_attention_weight_max": 35.61138916015625,
      "activations/layer19_attention_weight_min": -32.737640380859375,
      "activations/layer1_attention_weight_max": 15.680094718933105,
      "activations/layer1_attention_weight_min": -14.859532356262207,
      "activations/layer20_attention_weight_max": 29.536596298217773,
      "activations/layer20_attention_weight_min": -26.13886070251465,
      "activations/layer21_attention_weight_max": 29.286752700805664,
      "activations/layer21_attention_weight_min": -26.518558502197266,
      "activations/layer22_attention_weight_max": 40.05358123779297,
      "activations/layer22_attention_weight_min": -33.06863021850586,
      "activations/layer23_attention_weight_max": 33.199371337890625,
      "activations/layer23_attention_weight_min": -25.662708282470703,
      "activations/layer2_attention_weight_max": 33.79133605957031,
      "activations/layer2_attention_weight_min": -33.20026397705078,
      "activations/layer3_attention_weight_max": 105.6329574584961,
      "activations/layer3_attention_weight_min": -106.75457000732422,
      "activations/layer4_attention_weight_max": 108.57555389404297,
      "activations/layer4_attention_weight_min": -108.3241195678711,
      "activations/layer5_attention_weight_max": 50.256561279296875,
      "activations/layer5_attention_weight_min": -58.76395797729492,
      "activations/layer6_attention_weight_max": 44.92228698730469,
      "activations/layer6_attention_weight_min": -48.35301208496094,
      "activations/layer7_attention_weight_max": 92.09750366210938,
      "activations/layer7_attention_weight_min": -99.23059844970703,
      "activations/layer8_attention_weight_max": 44.281654357910156,
      "activations/layer8_attention_weight_min": -46.56377029418945,
      "activations/layer9_attention_weight_max": 33.36716842651367,
      "activations/layer9_attention_weight_min": -34.79694747924805,
      "epoch": 20.74,
      "learning_rate": 1.643143939393939e-05,
      "loss": 2.6915,
      "step": 356900
    },
    {
      "activations/layer0_attention_weight_max": 16.295658111572266,
      "activations/layer0_attention_weight_min": -15.646831512451172,
      "activations/layer10_attention_weight_max": 35.824981689453125,
      "activations/layer10_attention_weight_min": -38.17622375488281,
      "activations/layer11_attention_weight_max": 36.30936050415039,
      "activations/layer11_attention_weight_min": -34.8386344909668,
      "activations/layer12_attention_weight_max": 26.683900833129883,
      "activations/layer12_attention_weight_min": -26.751693725585938,
      "activations/layer13_attention_weight_max": 39.416412353515625,
      "activations/layer13_attention_weight_min": -35.54883575439453,
      "activations/layer14_attention_weight_max": 40.74115753173828,
      "activations/layer14_attention_weight_min": -38.687469482421875,
      "activations/layer15_attention_weight_max": 35.7712287902832,
      "activations/layer15_attention_weight_min": -35.58509063720703,
      "activations/layer16_attention_weight_max": 27.679359436035156,
      "activations/layer16_attention_weight_min": -28.465044021606445,
      "activations/layer17_attention_weight_max": 27.7591609954834,
      "activations/layer17_attention_weight_min": -24.89197540283203,
      "activations/layer18_attention_weight_max": 33.378536224365234,
      "activations/layer18_attention_weight_min": -26.965166091918945,
      "activations/layer19_attention_weight_max": 33.37058639526367,
      "activations/layer19_attention_weight_min": -32.96219253540039,
      "activations/layer1_attention_weight_max": 15.878569602966309,
      "activations/layer1_attention_weight_min": -14.820208549499512,
      "activations/layer20_attention_weight_max": 29.282718658447266,
      "activations/layer20_attention_weight_min": -26.753984451293945,
      "activations/layer21_attention_weight_max": 28.01675033569336,
      "activations/layer21_attention_weight_min": -25.990922927856445,
      "activations/layer22_attention_weight_max": 42.14988327026367,
      "activations/layer22_attention_weight_min": -34.09071350097656,
      "activations/layer23_attention_weight_max": 31.383045196533203,
      "activations/layer23_attention_weight_min": -27.402496337890625,
      "activations/layer2_attention_weight_max": 33.62346649169922,
      "activations/layer2_attention_weight_min": -33.69612503051758,
      "activations/layer3_attention_weight_max": 105.02610778808594,
      "activations/layer3_attention_weight_min": -107.06463623046875,
      "activations/layer4_attention_weight_max": 108.59297943115234,
      "activations/layer4_attention_weight_min": -110.94412994384766,
      "activations/layer5_attention_weight_max": 53.20656967163086,
      "activations/layer5_attention_weight_min": -62.05683898925781,
      "activations/layer6_attention_weight_max": 49.47286605834961,
      "activations/layer6_attention_weight_min": -48.69095230102539,
      "activations/layer7_attention_weight_max": 103.04033660888672,
      "activations/layer7_attention_weight_min": -104.68635559082031,
      "activations/layer8_attention_weight_max": 44.11195755004883,
      "activations/layer8_attention_weight_min": -50.36689758300781,
      "activations/layer9_attention_weight_max": 35.2645378112793,
      "activations/layer9_attention_weight_min": -38.15019607543945,
      "epoch": 20.74,
      "learning_rate": 1.64125e-05,
      "loss": 2.7091,
      "step": 356950
    },
    {
      "activations/layer0_attention_weight_max": 16.024255752563477,
      "activations/layer0_attention_weight_min": -14.665739059448242,
      "activations/layer10_attention_weight_max": 39.309242248535156,
      "activations/layer10_attention_weight_min": -39.18898010253906,
      "activations/layer11_attention_weight_max": 35.867584228515625,
      "activations/layer11_attention_weight_min": -36.86688995361328,
      "activations/layer12_attention_weight_max": 27.12430763244629,
      "activations/layer12_attention_weight_min": -28.256845474243164,
      "activations/layer13_attention_weight_max": 41.0799446105957,
      "activations/layer13_attention_weight_min": -37.93764114379883,
      "activations/layer14_attention_weight_max": 42.79248809814453,
      "activations/layer14_attention_weight_min": -43.89043045043945,
      "activations/layer15_attention_weight_max": 37.65232467651367,
      "activations/layer15_attention_weight_min": -36.1212158203125,
      "activations/layer16_attention_weight_max": 29.720245361328125,
      "activations/layer16_attention_weight_min": -28.762161254882812,
      "activations/layer17_attention_weight_max": 27.787120819091797,
      "activations/layer17_attention_weight_min": -27.710336685180664,
      "activations/layer18_attention_weight_max": 31.430709838867188,
      "activations/layer18_attention_weight_min": -27.2133731842041,
      "activations/layer19_attention_weight_max": 38.68535614013672,
      "activations/layer19_attention_weight_min": -33.28589630126953,
      "activations/layer1_attention_weight_max": 15.649807929992676,
      "activations/layer1_attention_weight_min": -15.004525184631348,
      "activations/layer20_attention_weight_max": 30.519792556762695,
      "activations/layer20_attention_weight_min": -28.305131912231445,
      "activations/layer21_attention_weight_max": 28.880319595336914,
      "activations/layer21_attention_weight_min": -25.353288650512695,
      "activations/layer22_attention_weight_max": 44.696250915527344,
      "activations/layer22_attention_weight_min": -34.88788986206055,
      "activations/layer23_attention_weight_max": 31.131528854370117,
      "activations/layer23_attention_weight_min": -25.258243560791016,
      "activations/layer2_attention_weight_max": 35.8662109375,
      "activations/layer2_attention_weight_min": -33.36872100830078,
      "activations/layer3_attention_weight_max": 105.36369323730469,
      "activations/layer3_attention_weight_min": -106.24486541748047,
      "activations/layer4_attention_weight_max": 109.20375061035156,
      "activations/layer4_attention_weight_min": -112.89730072021484,
      "activations/layer5_attention_weight_max": 50.499595642089844,
      "activations/layer5_attention_weight_min": -66.86471557617188,
      "activations/layer6_attention_weight_max": 46.65244674682617,
      "activations/layer6_attention_weight_min": -50.16651153564453,
      "activations/layer7_attention_weight_max": 102.01722717285156,
      "activations/layer7_attention_weight_min": -107.11793518066406,
      "activations/layer8_attention_weight_max": 46.77347183227539,
      "activations/layer8_attention_weight_min": -47.509429931640625,
      "activations/layer9_attention_weight_max": 34.73975372314453,
      "activations/layer9_attention_weight_min": -39.1814079284668,
      "epoch": 20.74,
      "learning_rate": 1.6393560606060604e-05,
      "loss": 2.7135,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4878,
      "eval_samples_per_second": 505.901,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4878,
      "eval_openwebtext_samples_per_second": 505.901,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9775,
      "eval_wikitext_samples_per_second": 230.598,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_lambada_loss": 2.359375,
      "eval_lambada_ppl": 10.584334175890335,
      "eval_lambada_runtime": 9.5327,
      "eval_lambada_samples_per_second": 510.766,
      "step": 357000
    },
    {
      "activations/layer0_attention_weight_max": 16.955921173095703,
      "activations/layer0_attention_weight_min": -13.464886665344238,
      "activations/layer10_attention_weight_max": 34.270652770996094,
      "activations/layer10_attention_weight_min": -33.85613250732422,
      "activations/layer11_attention_weight_max": 38.29472351074219,
      "activations/layer11_attention_weight_min": -35.61283874511719,
      "activations/layer12_attention_weight_max": 27.751070022583008,
      "activations/layer12_attention_weight_min": -28.32685661315918,
      "activations/layer13_attention_weight_max": 41.65574645996094,
      "activations/layer13_attention_weight_min": -37.06568908691406,
      "activations/layer14_attention_weight_max": 45.872962951660156,
      "activations/layer14_attention_weight_min": -39.10383987426758,
      "activations/layer15_attention_weight_max": 41.1358528137207,
      "activations/layer15_attention_weight_min": -37.483760833740234,
      "activations/layer16_attention_weight_max": 29.86969757080078,
      "activations/layer16_attention_weight_min": -28.924943923950195,
      "activations/layer17_attention_weight_max": 32.02526092529297,
      "activations/layer17_attention_weight_min": -28.040096282958984,
      "activations/layer18_attention_weight_max": 36.74570846557617,
      "activations/layer18_attention_weight_min": -30.387971878051758,
      "activations/layer19_attention_weight_max": 35.98859405517578,
      "activations/layer19_attention_weight_min": -34.71192169189453,
      "activations/layer1_attention_weight_max": 15.756414413452148,
      "activations/layer1_attention_weight_min": -14.874165534973145,
      "activations/layer20_attention_weight_max": 31.329715728759766,
      "activations/layer20_attention_weight_min": -28.069429397583008,
      "activations/layer21_attention_weight_max": 32.82017517089844,
      "activations/layer21_attention_weight_min": -28.02840805053711,
      "activations/layer22_attention_weight_max": 42.72581100463867,
      "activations/layer22_attention_weight_min": -32.92074966430664,
      "activations/layer23_attention_weight_max": 34.11997985839844,
      "activations/layer23_attention_weight_min": -25.784435272216797,
      "activations/layer2_attention_weight_max": 33.31385040283203,
      "activations/layer2_attention_weight_min": -32.35729217529297,
      "activations/layer3_attention_weight_max": 103.49214172363281,
      "activations/layer3_attention_weight_min": -105.82524871826172,
      "activations/layer4_attention_weight_max": 105.8985366821289,
      "activations/layer4_attention_weight_min": -112.11641693115234,
      "activations/layer5_attention_weight_max": 51.813819885253906,
      "activations/layer5_attention_weight_min": -64.01765441894531,
      "activations/layer6_attention_weight_max": 44.88605880737305,
      "activations/layer6_attention_weight_min": -49.540523529052734,
      "activations/layer7_attention_weight_max": 94.61949920654297,
      "activations/layer7_attention_weight_min": -104.5981216430664,
      "activations/layer8_attention_weight_max": 42.23616409301758,
      "activations/layer8_attention_weight_min": -46.59510803222656,
      "activations/layer9_attention_weight_max": 33.11730194091797,
      "activations/layer9_attention_weight_min": -35.505958557128906,
      "epoch": 20.75,
      "learning_rate": 1.637462121212121e-05,
      "loss": 2.7249,
      "step": 357050
    },
    {
      "activations/layer0_attention_weight_max": 16.119977951049805,
      "activations/layer0_attention_weight_min": -13.263598442077637,
      "activations/layer10_attention_weight_max": 34.79994583129883,
      "activations/layer10_attention_weight_min": -35.2529182434082,
      "activations/layer11_attention_weight_max": 33.79649353027344,
      "activations/layer11_attention_weight_min": -37.31892776489258,
      "activations/layer12_attention_weight_max": 27.23106575012207,
      "activations/layer12_attention_weight_min": -27.83272933959961,
      "activations/layer13_attention_weight_max": 37.99493408203125,
      "activations/layer13_attention_weight_min": -37.90122985839844,
      "activations/layer14_attention_weight_max": 44.41521453857422,
      "activations/layer14_attention_weight_min": -39.35430145263672,
      "activations/layer15_attention_weight_max": 41.41340255737305,
      "activations/layer15_attention_weight_min": -39.2154426574707,
      "activations/layer16_attention_weight_max": 29.484352111816406,
      "activations/layer16_attention_weight_min": -29.536638259887695,
      "activations/layer17_attention_weight_max": 29.13511085510254,
      "activations/layer17_attention_weight_min": -26.13298988342285,
      "activations/layer18_attention_weight_max": 33.296607971191406,
      "activations/layer18_attention_weight_min": -27.332857131958008,
      "activations/layer19_attention_weight_max": 34.54802322387695,
      "activations/layer19_attention_weight_min": -30.97368812561035,
      "activations/layer1_attention_weight_max": 16.209646224975586,
      "activations/layer1_attention_weight_min": -16.353925704956055,
      "activations/layer20_attention_weight_max": 28.340728759765625,
      "activations/layer20_attention_weight_min": -24.841190338134766,
      "activations/layer21_attention_weight_max": 28.594419479370117,
      "activations/layer21_attention_weight_min": -24.51045799255371,
      "activations/layer22_attention_weight_max": 40.66191482543945,
      "activations/layer22_attention_weight_min": -32.195213317871094,
      "activations/layer23_attention_weight_max": 31.95827293395996,
      "activations/layer23_attention_weight_min": -26.587684631347656,
      "activations/layer2_attention_weight_max": 34.55533981323242,
      "activations/layer2_attention_weight_min": -32.59202194213867,
      "activations/layer3_attention_weight_max": 104.73346710205078,
      "activations/layer3_attention_weight_min": -106.485595703125,
      "activations/layer4_attention_weight_max": 107.8863525390625,
      "activations/layer4_attention_weight_min": -119.20784759521484,
      "activations/layer5_attention_weight_max": 49.151580810546875,
      "activations/layer5_attention_weight_min": -67.09124755859375,
      "activations/layer6_attention_weight_max": 42.887733459472656,
      "activations/layer6_attention_weight_min": -46.1297721862793,
      "activations/layer7_attention_weight_max": 93.86764526367188,
      "activations/layer7_attention_weight_min": -99.55315399169922,
      "activations/layer8_attention_weight_max": 40.92110061645508,
      "activations/layer8_attention_weight_min": -45.36206817626953,
      "activations/layer9_attention_weight_max": 31.635412216186523,
      "activations/layer9_attention_weight_min": -35.992454528808594,
      "epoch": 20.75,
      "learning_rate": 1.6355681818181817e-05,
      "loss": 2.7226,
      "step": 357100
    },
    {
      "activations/layer0_attention_weight_max": 18.023332595825195,
      "activations/layer0_attention_weight_min": -13.693279266357422,
      "activations/layer10_attention_weight_max": 34.598358154296875,
      "activations/layer10_attention_weight_min": -36.9460563659668,
      "activations/layer11_attention_weight_max": 32.04708480834961,
      "activations/layer11_attention_weight_min": -35.10820770263672,
      "activations/layer12_attention_weight_max": 26.460935592651367,
      "activations/layer12_attention_weight_min": -31.99712371826172,
      "activations/layer13_attention_weight_max": 37.609947204589844,
      "activations/layer13_attention_weight_min": -38.361175537109375,
      "activations/layer14_attention_weight_max": 41.1335563659668,
      "activations/layer14_attention_weight_min": -40.7422981262207,
      "activations/layer15_attention_weight_max": 36.0053596496582,
      "activations/layer15_attention_weight_min": -37.92844009399414,
      "activations/layer16_attention_weight_max": 28.2597599029541,
      "activations/layer16_attention_weight_min": -27.781322479248047,
      "activations/layer17_attention_weight_max": 27.506935119628906,
      "activations/layer17_attention_weight_min": -27.342008590698242,
      "activations/layer18_attention_weight_max": 30.907245635986328,
      "activations/layer18_attention_weight_min": -26.841632843017578,
      "activations/layer19_attention_weight_max": 32.31447982788086,
      "activations/layer19_attention_weight_min": -30.840978622436523,
      "activations/layer1_attention_weight_max": 15.7412748336792,
      "activations/layer1_attention_weight_min": -15.879334449768066,
      "activations/layer20_attention_weight_max": 28.236825942993164,
      "activations/layer20_attention_weight_min": -25.07250213623047,
      "activations/layer21_attention_weight_max": 28.19582748413086,
      "activations/layer21_attention_weight_min": -24.52569007873535,
      "activations/layer22_attention_weight_max": 39.976741790771484,
      "activations/layer22_attention_weight_min": -31.469770431518555,
      "activations/layer23_attention_weight_max": 31.398536682128906,
      "activations/layer23_attention_weight_min": -25.87270164489746,
      "activations/layer2_attention_weight_max": 34.033721923828125,
      "activations/layer2_attention_weight_min": -32.393699645996094,
      "activations/layer3_attention_weight_max": 105.02703857421875,
      "activations/layer3_attention_weight_min": -103.19493103027344,
      "activations/layer4_attention_weight_max": 106.869140625,
      "activations/layer4_attention_weight_min": -112.43712615966797,
      "activations/layer5_attention_weight_max": 50.436668395996094,
      "activations/layer5_attention_weight_min": -62.18822479248047,
      "activations/layer6_attention_weight_max": 44.20399475097656,
      "activations/layer6_attention_weight_min": -48.494483947753906,
      "activations/layer7_attention_weight_max": 97.96993255615234,
      "activations/layer7_attention_weight_min": -99.75271606445312,
      "activations/layer8_attention_weight_max": 42.50802993774414,
      "activations/layer8_attention_weight_min": -43.961402893066406,
      "activations/layer9_attention_weight_max": 30.764530181884766,
      "activations/layer9_attention_weight_min": -35.168792724609375,
      "epoch": 20.75,
      "learning_rate": 1.6336742424242422e-05,
      "loss": 2.7058,
      "step": 357150
    },
    {
      "activations/layer0_attention_weight_max": 16.341554641723633,
      "activations/layer0_attention_weight_min": -14.432991981506348,
      "activations/layer10_attention_weight_max": 35.88750076293945,
      "activations/layer10_attention_weight_min": -35.443702697753906,
      "activations/layer11_attention_weight_max": 34.817543029785156,
      "activations/layer11_attention_weight_min": -36.397377014160156,
      "activations/layer12_attention_weight_max": 26.829872131347656,
      "activations/layer12_attention_weight_min": -27.89966583251953,
      "activations/layer13_attention_weight_max": 42.091583251953125,
      "activations/layer13_attention_weight_min": -37.14257049560547,
      "activations/layer14_attention_weight_max": 51.37847137451172,
      "activations/layer14_attention_weight_min": -40.97060012817383,
      "activations/layer15_attention_weight_max": 50.38650131225586,
      "activations/layer15_attention_weight_min": -39.14887619018555,
      "activations/layer16_attention_weight_max": 41.0526008605957,
      "activations/layer16_attention_weight_min": -30.535566329956055,
      "activations/layer17_attention_weight_max": 35.3394660949707,
      "activations/layer17_attention_weight_min": -27.642696380615234,
      "activations/layer18_attention_weight_max": 40.91366195678711,
      "activations/layer18_attention_weight_min": -27.54844856262207,
      "activations/layer19_attention_weight_max": 46.533260345458984,
      "activations/layer19_attention_weight_min": -32.94233703613281,
      "activations/layer1_attention_weight_max": 15.740079879760742,
      "activations/layer1_attention_weight_min": -15.181365013122559,
      "activations/layer20_attention_weight_max": 38.11648941040039,
      "activations/layer20_attention_weight_min": -24.77104949951172,
      "activations/layer21_attention_weight_max": 38.536766052246094,
      "activations/layer21_attention_weight_min": -24.284379959106445,
      "activations/layer22_attention_weight_max": 61.43864822387695,
      "activations/layer22_attention_weight_min": -35.08091735839844,
      "activations/layer23_attention_weight_max": 42.240821838378906,
      "activations/layer23_attention_weight_min": -25.92573356628418,
      "activations/layer2_attention_weight_max": 34.71074676513672,
      "activations/layer2_attention_weight_min": -33.63613510131836,
      "activations/layer3_attention_weight_max": 102.36430358886719,
      "activations/layer3_attention_weight_min": -106.90917205810547,
      "activations/layer4_attention_weight_max": 109.48644256591797,
      "activations/layer4_attention_weight_min": -113.1787338256836,
      "activations/layer5_attention_weight_max": 51.4948844909668,
      "activations/layer5_attention_weight_min": -64.96968841552734,
      "activations/layer6_attention_weight_max": 46.82999801635742,
      "activations/layer6_attention_weight_min": -49.34789276123047,
      "activations/layer7_attention_weight_max": 101.76679992675781,
      "activations/layer7_attention_weight_min": -102.1718521118164,
      "activations/layer8_attention_weight_max": 43.722835540771484,
      "activations/layer8_attention_weight_min": -43.58507537841797,
      "activations/layer9_attention_weight_max": 33.17170715332031,
      "activations/layer9_attention_weight_min": -34.49702072143555,
      "epoch": 20.76,
      "learning_rate": 1.631780303030303e-05,
      "loss": 2.6907,
      "step": 357200
    },
    {
      "activations/layer0_attention_weight_max": 17.045413970947266,
      "activations/layer0_attention_weight_min": -13.372702598571777,
      "activations/layer10_attention_weight_max": 33.653411865234375,
      "activations/layer10_attention_weight_min": -34.189361572265625,
      "activations/layer11_attention_weight_max": 38.34038543701172,
      "activations/layer11_attention_weight_min": -35.35582733154297,
      "activations/layer12_attention_weight_max": 25.543350219726562,
      "activations/layer12_attention_weight_min": -26.859561920166016,
      "activations/layer13_attention_weight_max": 38.4804573059082,
      "activations/layer13_attention_weight_min": -35.50925827026367,
      "activations/layer14_attention_weight_max": 45.232627868652344,
      "activations/layer14_attention_weight_min": -40.18161392211914,
      "activations/layer15_attention_weight_max": 45.80484390258789,
      "activations/layer15_attention_weight_min": -40.741703033447266,
      "activations/layer16_attention_weight_max": 35.17422866821289,
      "activations/layer16_attention_weight_min": -31.05816650390625,
      "activations/layer17_attention_weight_max": 29.966943740844727,
      "activations/layer17_attention_weight_min": -28.98623275756836,
      "activations/layer18_attention_weight_max": 32.59678268432617,
      "activations/layer18_attention_weight_min": -27.359464645385742,
      "activations/layer19_attention_weight_max": 32.75083541870117,
      "activations/layer19_attention_weight_min": -31.661497116088867,
      "activations/layer1_attention_weight_max": 16.113637924194336,
      "activations/layer1_attention_weight_min": -14.57663345336914,
      "activations/layer20_attention_weight_max": 27.69881820678711,
      "activations/layer20_attention_weight_min": -25.182565689086914,
      "activations/layer21_attention_weight_max": 27.487424850463867,
      "activations/layer21_attention_weight_min": -26.114089965820312,
      "activations/layer22_attention_weight_max": 40.56897735595703,
      "activations/layer22_attention_weight_min": -35.441036224365234,
      "activations/layer23_attention_weight_max": 31.247638702392578,
      "activations/layer23_attention_weight_min": -27.610822677612305,
      "activations/layer2_attention_weight_max": 34.89067077636719,
      "activations/layer2_attention_weight_min": -33.46765899658203,
      "activations/layer3_attention_weight_max": 103.27301788330078,
      "activations/layer3_attention_weight_min": -104.09577941894531,
      "activations/layer4_attention_weight_max": 105.496337890625,
      "activations/layer4_attention_weight_min": -110.42485809326172,
      "activations/layer5_attention_weight_max": 50.15422821044922,
      "activations/layer5_attention_weight_min": -61.949127197265625,
      "activations/layer6_attention_weight_max": 41.94416046142578,
      "activations/layer6_attention_weight_min": -45.606101989746094,
      "activations/layer7_attention_weight_max": 90.90033721923828,
      "activations/layer7_attention_weight_min": -95.6439208984375,
      "activations/layer8_attention_weight_max": 40.67892837524414,
      "activations/layer8_attention_weight_min": -44.287105560302734,
      "activations/layer9_attention_weight_max": 30.45612144470215,
      "activations/layer9_attention_weight_min": -34.934505462646484,
      "epoch": 20.76,
      "learning_rate": 1.6298863636363635e-05,
      "loss": 2.7007,
      "step": 357250
    },
    {
      "activations/layer0_attention_weight_max": 16.129865646362305,
      "activations/layer0_attention_weight_min": -13.625593185424805,
      "activations/layer10_attention_weight_max": 35.68088150024414,
      "activations/layer10_attention_weight_min": -37.08247756958008,
      "activations/layer11_attention_weight_max": 35.6857795715332,
      "activations/layer11_attention_weight_min": -37.9495964050293,
      "activations/layer12_attention_weight_max": 27.266443252563477,
      "activations/layer12_attention_weight_min": -30.047988891601562,
      "activations/layer13_attention_weight_max": 42.49308395385742,
      "activations/layer13_attention_weight_min": -39.401424407958984,
      "activations/layer14_attention_weight_max": 42.419212341308594,
      "activations/layer14_attention_weight_min": -42.59259796142578,
      "activations/layer15_attention_weight_max": 41.22860336303711,
      "activations/layer15_attention_weight_min": -39.668209075927734,
      "activations/layer16_attention_weight_max": 30.75556182861328,
      "activations/layer16_attention_weight_min": -30.420669555664062,
      "activations/layer17_attention_weight_max": 30.112722396850586,
      "activations/layer17_attention_weight_min": -26.27567481994629,
      "activations/layer18_attention_weight_max": 32.834651947021484,
      "activations/layer18_attention_weight_min": -26.397504806518555,
      "activations/layer19_attention_weight_max": 35.1405143737793,
      "activations/layer19_attention_weight_min": -29.456253051757812,
      "activations/layer1_attention_weight_max": 14.865763664245605,
      "activations/layer1_attention_weight_min": -14.53481674194336,
      "activations/layer20_attention_weight_max": 29.485811233520508,
      "activations/layer20_attention_weight_min": -25.5694522857666,
      "activations/layer21_attention_weight_max": 29.25383758544922,
      "activations/layer21_attention_weight_min": -26.203433990478516,
      "activations/layer22_attention_weight_max": 38.62943649291992,
      "activations/layer22_attention_weight_min": -31.06000328063965,
      "activations/layer23_attention_weight_max": 30.508758544921875,
      "activations/layer23_attention_weight_min": -26.395343780517578,
      "activations/layer2_attention_weight_max": 34.0998420715332,
      "activations/layer2_attention_weight_min": -32.834999084472656,
      "activations/layer3_attention_weight_max": 104.15263366699219,
      "activations/layer3_attention_weight_min": -105.4719467163086,
      "activations/layer4_attention_weight_max": 108.68975067138672,
      "activations/layer4_attention_weight_min": -114.5746078491211,
      "activations/layer5_attention_weight_max": 51.256996154785156,
      "activations/layer5_attention_weight_min": -62.480987548828125,
      "activations/layer6_attention_weight_max": 48.321529388427734,
      "activations/layer6_attention_weight_min": -49.983924865722656,
      "activations/layer7_attention_weight_max": 99.86125946044922,
      "activations/layer7_attention_weight_min": -106.5108642578125,
      "activations/layer8_attention_weight_max": 43.723148345947266,
      "activations/layer8_attention_weight_min": -47.94661331176758,
      "activations/layer9_attention_weight_max": 34.325130462646484,
      "activations/layer9_attention_weight_min": -38.60293197631836,
      "epoch": 20.76,
      "learning_rate": 1.627992424242424e-05,
      "loss": 2.6955,
      "step": 357300
    },
    {
      "activations/layer0_attention_weight_max": 16.629344940185547,
      "activations/layer0_attention_weight_min": -14.273343086242676,
      "activations/layer10_attention_weight_max": 34.484703063964844,
      "activations/layer10_attention_weight_min": -34.96118927001953,
      "activations/layer11_attention_weight_max": 34.596435546875,
      "activations/layer11_attention_weight_min": -35.604408264160156,
      "activations/layer12_attention_weight_max": 26.907934188842773,
      "activations/layer12_attention_weight_min": -28.822891235351562,
      "activations/layer13_attention_weight_max": 38.61772918701172,
      "activations/layer13_attention_weight_min": -35.196834564208984,
      "activations/layer14_attention_weight_max": 45.447418212890625,
      "activations/layer14_attention_weight_min": -36.621524810791016,
      "activations/layer15_attention_weight_max": 40.48492431640625,
      "activations/layer15_attention_weight_min": -35.66910171508789,
      "activations/layer16_attention_weight_max": 30.05022430419922,
      "activations/layer16_attention_weight_min": -29.819536209106445,
      "activations/layer17_attention_weight_max": 28.61605453491211,
      "activations/layer17_attention_weight_min": -24.597352981567383,
      "activations/layer18_attention_weight_max": 31.998056411743164,
      "activations/layer18_attention_weight_min": -24.956539154052734,
      "activations/layer19_attention_weight_max": 31.230762481689453,
      "activations/layer19_attention_weight_min": -32.00246047973633,
      "activations/layer1_attention_weight_max": 14.999361991882324,
      "activations/layer1_attention_weight_min": -14.998108863830566,
      "activations/layer20_attention_weight_max": 27.05575180053711,
      "activations/layer20_attention_weight_min": -25.071426391601562,
      "activations/layer21_attention_weight_max": 25.974437713623047,
      "activations/layer21_attention_weight_min": -23.239322662353516,
      "activations/layer22_attention_weight_max": 36.48458480834961,
      "activations/layer22_attention_weight_min": -29.050317764282227,
      "activations/layer23_attention_weight_max": 31.623046875,
      "activations/layer23_attention_weight_min": -22.87245750427246,
      "activations/layer2_attention_weight_max": 33.48971939086914,
      "activations/layer2_attention_weight_min": -32.90459060668945,
      "activations/layer3_attention_weight_max": 107.65892028808594,
      "activations/layer3_attention_weight_min": -109.57151794433594,
      "activations/layer4_attention_weight_max": 108.85897064208984,
      "activations/layer4_attention_weight_min": -108.96435546875,
      "activations/layer5_attention_weight_max": 49.513671875,
      "activations/layer5_attention_weight_min": -62.95439147949219,
      "activations/layer6_attention_weight_max": 45.04526138305664,
      "activations/layer6_attention_weight_min": -46.61392593383789,
      "activations/layer7_attention_weight_max": 99.84630584716797,
      "activations/layer7_attention_weight_min": -95.75403594970703,
      "activations/layer8_attention_weight_max": 40.87392044067383,
      "activations/layer8_attention_weight_min": -43.71250534057617,
      "activations/layer9_attention_weight_max": 32.198429107666016,
      "activations/layer9_attention_weight_min": -33.63092041015625,
      "epoch": 20.76,
      "learning_rate": 1.626098484848485e-05,
      "loss": 2.7237,
      "step": 357350
    },
    {
      "activations/layer0_attention_weight_max": 16.321821212768555,
      "activations/layer0_attention_weight_min": -14.284308433532715,
      "activations/layer10_attention_weight_max": 35.17058181762695,
      "activations/layer10_attention_weight_min": -36.07131576538086,
      "activations/layer11_attention_weight_max": 34.062835693359375,
      "activations/layer11_attention_weight_min": -35.11317443847656,
      "activations/layer12_attention_weight_max": 27.77484703063965,
      "activations/layer12_attention_weight_min": -27.276460647583008,
      "activations/layer13_attention_weight_max": 40.60514450073242,
      "activations/layer13_attention_weight_min": -36.720664978027344,
      "activations/layer14_attention_weight_max": 44.25812911987305,
      "activations/layer14_attention_weight_min": -36.88408279418945,
      "activations/layer15_attention_weight_max": 39.00771713256836,
      "activations/layer15_attention_weight_min": -35.34403610229492,
      "activations/layer16_attention_weight_max": 30.967945098876953,
      "activations/layer16_attention_weight_min": -29.643095016479492,
      "activations/layer17_attention_weight_max": 29.17057228088379,
      "activations/layer17_attention_weight_min": -25.790071487426758,
      "activations/layer18_attention_weight_max": 30.822538375854492,
      "activations/layer18_attention_weight_min": -25.24840545654297,
      "activations/layer19_attention_weight_max": 33.36591720581055,
      "activations/layer19_attention_weight_min": -30.96512794494629,
      "activations/layer1_attention_weight_max": 15.772314071655273,
      "activations/layer1_attention_weight_min": -15.924818992614746,
      "activations/layer20_attention_weight_max": 29.9625186920166,
      "activations/layer20_attention_weight_min": -25.792978286743164,
      "activations/layer21_attention_weight_max": 30.19516944885254,
      "activations/layer21_attention_weight_min": -25.77298355102539,
      "activations/layer22_attention_weight_max": 43.19122314453125,
      "activations/layer22_attention_weight_min": -32.989585876464844,
      "activations/layer23_attention_weight_max": 31.369428634643555,
      "activations/layer23_attention_weight_min": -27.105571746826172,
      "activations/layer2_attention_weight_max": 32.860023498535156,
      "activations/layer2_attention_weight_min": -32.97412872314453,
      "activations/layer3_attention_weight_max": 104.5099105834961,
      "activations/layer3_attention_weight_min": -105.66699981689453,
      "activations/layer4_attention_weight_max": 108.96651458740234,
      "activations/layer4_attention_weight_min": -109.5068130493164,
      "activations/layer5_attention_weight_max": 50.06794738769531,
      "activations/layer5_attention_weight_min": -64.08492279052734,
      "activations/layer6_attention_weight_max": 45.39720153808594,
      "activations/layer6_attention_weight_min": -49.873512268066406,
      "activations/layer7_attention_weight_max": 97.64822387695312,
      "activations/layer7_attention_weight_min": -102.63931274414062,
      "activations/layer8_attention_weight_max": 43.93906784057617,
      "activations/layer8_attention_weight_min": -45.48794937133789,
      "activations/layer9_attention_weight_max": 33.54844665527344,
      "activations/layer9_attention_weight_min": -36.55875778198242,
      "epoch": 20.77,
      "learning_rate": 1.6242045454545453e-05,
      "loss": 2.6912,
      "step": 357400
    },
    {
      "activations/layer0_attention_weight_max": 16.702260971069336,
      "activations/layer0_attention_weight_min": -13.716195106506348,
      "activations/layer10_attention_weight_max": 38.91608428955078,
      "activations/layer10_attention_weight_min": -37.26214599609375,
      "activations/layer11_attention_weight_max": 36.75519561767578,
      "activations/layer11_attention_weight_min": -37.323246002197266,
      "activations/layer12_attention_weight_max": 32.8930549621582,
      "activations/layer12_attention_weight_min": -29.931079864501953,
      "activations/layer13_attention_weight_max": 47.09000778198242,
      "activations/layer13_attention_weight_min": -40.48112869262695,
      "activations/layer14_attention_weight_max": 44.26803207397461,
      "activations/layer14_attention_weight_min": -42.95729064941406,
      "activations/layer15_attention_weight_max": 43.064918518066406,
      "activations/layer15_attention_weight_min": -41.17348098754883,
      "activations/layer16_attention_weight_max": 30.706226348876953,
      "activations/layer16_attention_weight_min": -29.33247184753418,
      "activations/layer17_attention_weight_max": 29.860971450805664,
      "activations/layer17_attention_weight_min": -26.989669799804688,
      "activations/layer18_attention_weight_max": 33.712135314941406,
      "activations/layer18_attention_weight_min": -27.153493881225586,
      "activations/layer19_attention_weight_max": 33.256343841552734,
      "activations/layer19_attention_weight_min": -32.44915771484375,
      "activations/layer1_attention_weight_max": 15.194724082946777,
      "activations/layer1_attention_weight_min": -15.73409366607666,
      "activations/layer20_attention_weight_max": 30.5534725189209,
      "activations/layer20_attention_weight_min": -25.493200302124023,
      "activations/layer21_attention_weight_max": 29.94431495666504,
      "activations/layer21_attention_weight_min": -26.19403076171875,
      "activations/layer22_attention_weight_max": 37.95977783203125,
      "activations/layer22_attention_weight_min": -31.41157341003418,
      "activations/layer23_attention_weight_max": 35.17351531982422,
      "activations/layer23_attention_weight_min": -27.47119140625,
      "activations/layer2_attention_weight_max": 33.360748291015625,
      "activations/layer2_attention_weight_min": -34.277286529541016,
      "activations/layer3_attention_weight_max": 101.78336334228516,
      "activations/layer3_attention_weight_min": -105.64189147949219,
      "activations/layer4_attention_weight_max": 104.802490234375,
      "activations/layer4_attention_weight_min": -108.7325668334961,
      "activations/layer5_attention_weight_max": 49.887996673583984,
      "activations/layer5_attention_weight_min": -67.63796997070312,
      "activations/layer6_attention_weight_max": 45.838768005371094,
      "activations/layer6_attention_weight_min": -47.26811218261719,
      "activations/layer7_attention_weight_max": 104.50411987304688,
      "activations/layer7_attention_weight_min": -101.37371063232422,
      "activations/layer8_attention_weight_max": 42.85972595214844,
      "activations/layer8_attention_weight_min": -42.99528121948242,
      "activations/layer9_attention_weight_max": 36.01789474487305,
      "activations/layer9_attention_weight_min": -35.8056755065918,
      "epoch": 20.77,
      "learning_rate": 1.6223106060606058e-05,
      "loss": 2.724,
      "step": 357450
    },
    {
      "activations/layer0_attention_weight_max": 16.342769622802734,
      "activations/layer0_attention_weight_min": -13.55805778503418,
      "activations/layer10_attention_weight_max": 32.75218200683594,
      "activations/layer10_attention_weight_min": -34.073760986328125,
      "activations/layer11_attention_weight_max": 30.995285034179688,
      "activations/layer11_attention_weight_min": -34.48809814453125,
      "activations/layer12_attention_weight_max": 27.565671920776367,
      "activations/layer12_attention_weight_min": -26.14870834350586,
      "activations/layer13_attention_weight_max": 38.923004150390625,
      "activations/layer13_attention_weight_min": -34.627960205078125,
      "activations/layer14_attention_weight_max": 40.341209411621094,
      "activations/layer14_attention_weight_min": -36.91074752807617,
      "activations/layer15_attention_weight_max": 37.98391342163086,
      "activations/layer15_attention_weight_min": -37.64075469970703,
      "activations/layer16_attention_weight_max": 29.386688232421875,
      "activations/layer16_attention_weight_min": -27.261247634887695,
      "activations/layer17_attention_weight_max": 29.587249755859375,
      "activations/layer17_attention_weight_min": -25.624711990356445,
      "activations/layer18_attention_weight_max": 31.923194885253906,
      "activations/layer18_attention_weight_min": -25.930810928344727,
      "activations/layer19_attention_weight_max": 35.10844039916992,
      "activations/layer19_attention_weight_min": -31.283538818359375,
      "activations/layer1_attention_weight_max": 14.958525657653809,
      "activations/layer1_attention_weight_min": -15.39763069152832,
      "activations/layer20_attention_weight_max": 29.489959716796875,
      "activations/layer20_attention_weight_min": -27.860069274902344,
      "activations/layer21_attention_weight_max": 29.412710189819336,
      "activations/layer21_attention_weight_min": -25.887290954589844,
      "activations/layer22_attention_weight_max": 39.722373962402344,
      "activations/layer22_attention_weight_min": -32.733028411865234,
      "activations/layer23_attention_weight_max": 33.80696487426758,
      "activations/layer23_attention_weight_min": -26.67483901977539,
      "activations/layer2_attention_weight_max": 33.170372009277344,
      "activations/layer2_attention_weight_min": -32.265464782714844,
      "activations/layer3_attention_weight_max": 101.37962341308594,
      "activations/layer3_attention_weight_min": -105.17726135253906,
      "activations/layer4_attention_weight_max": 104.5958251953125,
      "activations/layer4_attention_weight_min": -115.02008056640625,
      "activations/layer5_attention_weight_max": 49.559478759765625,
      "activations/layer5_attention_weight_min": -62.59259796142578,
      "activations/layer6_attention_weight_max": 42.950191497802734,
      "activations/layer6_attention_weight_min": -46.551124572753906,
      "activations/layer7_attention_weight_max": 90.8648681640625,
      "activations/layer7_attention_weight_min": -97.58085632324219,
      "activations/layer8_attention_weight_max": 41.03997039794922,
      "activations/layer8_attention_weight_min": -44.92411422729492,
      "activations/layer9_attention_weight_max": 31.4360294342041,
      "activations/layer9_attention_weight_min": -36.07988357543945,
      "epoch": 20.77,
      "learning_rate": 1.6204166666666666e-05,
      "loss": 2.7028,
      "step": 357500
    },
    {
      "activations/layer0_attention_weight_max": 16.047321319580078,
      "activations/layer0_attention_weight_min": -14.268967628479004,
      "activations/layer10_attention_weight_max": 33.404457092285156,
      "activations/layer10_attention_weight_min": -36.1251220703125,
      "activations/layer11_attention_weight_max": 32.54988098144531,
      "activations/layer11_attention_weight_min": -35.201454162597656,
      "activations/layer12_attention_weight_max": 25.99888038635254,
      "activations/layer12_attention_weight_min": -27.18385124206543,
      "activations/layer13_attention_weight_max": 41.634307861328125,
      "activations/layer13_attention_weight_min": -35.30803298950195,
      "activations/layer14_attention_weight_max": 43.056358337402344,
      "activations/layer14_attention_weight_min": -37.914676666259766,
      "activations/layer15_attention_weight_max": 45.04904556274414,
      "activations/layer15_attention_weight_min": -37.37446594238281,
      "activations/layer16_attention_weight_max": 31.83391571044922,
      "activations/layer16_attention_weight_min": -28.877891540527344,
      "activations/layer17_attention_weight_max": 31.17417335510254,
      "activations/layer17_attention_weight_min": -28.43640899658203,
      "activations/layer18_attention_weight_max": 30.950496673583984,
      "activations/layer18_attention_weight_min": -26.17388343811035,
      "activations/layer19_attention_weight_max": 37.13146209716797,
      "activations/layer19_attention_weight_min": -30.912866592407227,
      "activations/layer1_attention_weight_max": 15.965747833251953,
      "activations/layer1_attention_weight_min": -14.861205101013184,
      "activations/layer20_attention_weight_max": 34.204532623291016,
      "activations/layer20_attention_weight_min": -24.440994262695312,
      "activations/layer21_attention_weight_max": 30.714269638061523,
      "activations/layer21_attention_weight_min": -23.085424423217773,
      "activations/layer22_attention_weight_max": 41.36225128173828,
      "activations/layer22_attention_weight_min": -31.06200408935547,
      "activations/layer23_attention_weight_max": 34.461273193359375,
      "activations/layer23_attention_weight_min": -25.415489196777344,
      "activations/layer2_attention_weight_max": 31.584949493408203,
      "activations/layer2_attention_weight_min": -31.156354904174805,
      "activations/layer3_attention_weight_max": 102.29390716552734,
      "activations/layer3_attention_weight_min": -103.194580078125,
      "activations/layer4_attention_weight_max": 107.24346160888672,
      "activations/layer4_attention_weight_min": -107.4827651977539,
      "activations/layer5_attention_weight_max": 49.17216110229492,
      "activations/layer5_attention_weight_min": -64.16744995117188,
      "activations/layer6_attention_weight_max": 43.56420135498047,
      "activations/layer6_attention_weight_min": -47.48992156982422,
      "activations/layer7_attention_weight_max": 92.50847625732422,
      "activations/layer7_attention_weight_min": -96.4388427734375,
      "activations/layer8_attention_weight_max": 43.2855224609375,
      "activations/layer8_attention_weight_min": -45.656036376953125,
      "activations/layer9_attention_weight_max": 33.59957504272461,
      "activations/layer9_attention_weight_min": -35.636993408203125,
      "epoch": 20.78,
      "learning_rate": 1.618560606060606e-05,
      "loss": 2.7133,
      "step": 357550
    },
    {
      "activations/layer0_attention_weight_max": 16.46397590637207,
      "activations/layer0_attention_weight_min": -14.480405807495117,
      "activations/layer10_attention_weight_max": 37.06658935546875,
      "activations/layer10_attention_weight_min": -35.61994552612305,
      "activations/layer11_attention_weight_max": 35.03152847290039,
      "activations/layer11_attention_weight_min": -37.09181213378906,
      "activations/layer12_attention_weight_max": 28.841171264648438,
      "activations/layer12_attention_weight_min": -29.38326644897461,
      "activations/layer13_attention_weight_max": 43.525821685791016,
      "activations/layer13_attention_weight_min": -39.03522491455078,
      "activations/layer14_attention_weight_max": 45.702484130859375,
      "activations/layer14_attention_weight_min": -42.484619140625,
      "activations/layer15_attention_weight_max": 41.46361541748047,
      "activations/layer15_attention_weight_min": -36.58757019042969,
      "activations/layer16_attention_weight_max": 33.239112854003906,
      "activations/layer16_attention_weight_min": -29.011659622192383,
      "activations/layer17_attention_weight_max": 34.058738708496094,
      "activations/layer17_attention_weight_min": -27.064014434814453,
      "activations/layer18_attention_weight_max": 36.145751953125,
      "activations/layer18_attention_weight_min": -30.051307678222656,
      "activations/layer19_attention_weight_max": 35.356624603271484,
      "activations/layer19_attention_weight_min": -33.79356384277344,
      "activations/layer1_attention_weight_max": 16.114852905273438,
      "activations/layer1_attention_weight_min": -15.99819278717041,
      "activations/layer20_attention_weight_max": 31.124582290649414,
      "activations/layer20_attention_weight_min": -26.69280242919922,
      "activations/layer21_attention_weight_max": 30.050167083740234,
      "activations/layer21_attention_weight_min": -25.637731552124023,
      "activations/layer22_attention_weight_max": 45.3458137512207,
      "activations/layer22_attention_weight_min": -33.2120361328125,
      "activations/layer23_attention_weight_max": 34.351463317871094,
      "activations/layer23_attention_weight_min": -25.4705810546875,
      "activations/layer2_attention_weight_max": 35.17329406738281,
      "activations/layer2_attention_weight_min": -34.79465866088867,
      "activations/layer3_attention_weight_max": 105.83027648925781,
      "activations/layer3_attention_weight_min": -107.41621398925781,
      "activations/layer4_attention_weight_max": 111.8420181274414,
      "activations/layer4_attention_weight_min": -112.3846664428711,
      "activations/layer5_attention_weight_max": 50.10895919799805,
      "activations/layer5_attention_weight_min": -65.46306610107422,
      "activations/layer6_attention_weight_max": 46.64240646362305,
      "activations/layer6_attention_weight_min": -46.27714920043945,
      "activations/layer7_attention_weight_max": 100.39201354980469,
      "activations/layer7_attention_weight_min": -98.85812377929688,
      "activations/layer8_attention_weight_max": 46.0538215637207,
      "activations/layer8_attention_weight_min": -45.025455474853516,
      "activations/layer9_attention_weight_max": 34.97411346435547,
      "activations/layer9_attention_weight_min": -35.47877502441406,
      "epoch": 20.78,
      "learning_rate": 1.6166666666666665e-05,
      "loss": 2.7134,
      "step": 357600
    },
    {
      "activations/layer0_attention_weight_max": 16.59336280822754,
      "activations/layer0_attention_weight_min": -12.85353946685791,
      "activations/layer10_attention_weight_max": 37.070491790771484,
      "activations/layer10_attention_weight_min": -34.91189956665039,
      "activations/layer11_attention_weight_max": 36.56616973876953,
      "activations/layer11_attention_weight_min": -34.95895004272461,
      "activations/layer12_attention_weight_max": 27.748266220092773,
      "activations/layer12_attention_weight_min": -27.721500396728516,
      "activations/layer13_attention_weight_max": 39.84741973876953,
      "activations/layer13_attention_weight_min": -35.906986236572266,
      "activations/layer14_attention_weight_max": 42.85388946533203,
      "activations/layer14_attention_weight_min": -37.631248474121094,
      "activations/layer15_attention_weight_max": 41.38550567626953,
      "activations/layer15_attention_weight_min": -36.32109069824219,
      "activations/layer16_attention_weight_max": 31.008081436157227,
      "activations/layer16_attention_weight_min": -27.551897048950195,
      "activations/layer17_attention_weight_max": 28.619470596313477,
      "activations/layer17_attention_weight_min": -24.656457901000977,
      "activations/layer18_attention_weight_max": 30.950748443603516,
      "activations/layer18_attention_weight_min": -25.15465545654297,
      "activations/layer19_attention_weight_max": 31.153833389282227,
      "activations/layer19_attention_weight_min": -30.03421401977539,
      "activations/layer1_attention_weight_max": 16.21780776977539,
      "activations/layer1_attention_weight_min": -13.67479133605957,
      "activations/layer20_attention_weight_max": 25.981660842895508,
      "activations/layer20_attention_weight_min": -24.550395965576172,
      "activations/layer21_attention_weight_max": 24.417112350463867,
      "activations/layer21_attention_weight_min": -23.393980026245117,
      "activations/layer22_attention_weight_max": 37.34817886352539,
      "activations/layer22_attention_weight_min": -30.73069953918457,
      "activations/layer23_attention_weight_max": 28.99386978149414,
      "activations/layer23_attention_weight_min": -23.737173080444336,
      "activations/layer2_attention_weight_max": 32.44477081298828,
      "activations/layer2_attention_weight_min": -31.976665496826172,
      "activations/layer3_attention_weight_max": 102.28565979003906,
      "activations/layer3_attention_weight_min": -103.88219451904297,
      "activations/layer4_attention_weight_max": 107.7169418334961,
      "activations/layer4_attention_weight_min": -109.0518798828125,
      "activations/layer5_attention_weight_max": 49.40592956542969,
      "activations/layer5_attention_weight_min": -60.13125228881836,
      "activations/layer6_attention_weight_max": 43.32786560058594,
      "activations/layer6_attention_weight_min": -47.8487434387207,
      "activations/layer7_attention_weight_max": 96.07466888427734,
      "activations/layer7_attention_weight_min": -102.1377944946289,
      "activations/layer8_attention_weight_max": 42.78107833862305,
      "activations/layer8_attention_weight_min": -47.4722785949707,
      "activations/layer9_attention_weight_max": 33.38778305053711,
      "activations/layer9_attention_weight_min": -34.478336334228516,
      "epoch": 20.78,
      "learning_rate": 1.614772727272727e-05,
      "loss": 2.716,
      "step": 357650
    },
    {
      "activations/layer0_attention_weight_max": 16.199359893798828,
      "activations/layer0_attention_weight_min": -13.479857444763184,
      "activations/layer10_attention_weight_max": 34.281925201416016,
      "activations/layer10_attention_weight_min": -33.973243713378906,
      "activations/layer11_attention_weight_max": 35.2164306640625,
      "activations/layer11_attention_weight_min": -34.704200744628906,
      "activations/layer12_attention_weight_max": 28.874940872192383,
      "activations/layer12_attention_weight_min": -26.909015655517578,
      "activations/layer13_attention_weight_max": 43.998207092285156,
      "activations/layer13_attention_weight_min": -35.53057098388672,
      "activations/layer14_attention_weight_max": 47.00031280517578,
      "activations/layer14_attention_weight_min": -39.1665153503418,
      "activations/layer15_attention_weight_max": 43.209808349609375,
      "activations/layer15_attention_weight_min": -36.19511795043945,
      "activations/layer16_attention_weight_max": 32.90599822998047,
      "activations/layer16_attention_weight_min": -28.989173889160156,
      "activations/layer17_attention_weight_max": 30.93079376220703,
      "activations/layer17_attention_weight_min": -26.181705474853516,
      "activations/layer18_attention_weight_max": 34.12981033325195,
      "activations/layer18_attention_weight_min": -27.034257888793945,
      "activations/layer19_attention_weight_max": 37.793636322021484,
      "activations/layer19_attention_weight_min": -34.00346755981445,
      "activations/layer1_attention_weight_max": 15.714755058288574,
      "activations/layer1_attention_weight_min": -15.6704740524292,
      "activations/layer20_attention_weight_max": 30.184215545654297,
      "activations/layer20_attention_weight_min": -26.659818649291992,
      "activations/layer21_attention_weight_max": 31.36894416809082,
      "activations/layer21_attention_weight_min": -25.68636131286621,
      "activations/layer22_attention_weight_max": 45.67094802856445,
      "activations/layer22_attention_weight_min": -33.01693344116211,
      "activations/layer23_attention_weight_max": 35.46123504638672,
      "activations/layer23_attention_weight_min": -24.664457321166992,
      "activations/layer2_attention_weight_max": 34.04470443725586,
      "activations/layer2_attention_weight_min": -32.71540069580078,
      "activations/layer3_attention_weight_max": 104.88931274414062,
      "activations/layer3_attention_weight_min": -102.59156799316406,
      "activations/layer4_attention_weight_max": 106.45330810546875,
      "activations/layer4_attention_weight_min": -110.68804168701172,
      "activations/layer5_attention_weight_max": 51.992393493652344,
      "activations/layer5_attention_weight_min": -60.44807434082031,
      "activations/layer6_attention_weight_max": 45.470394134521484,
      "activations/layer6_attention_weight_min": -49.396183013916016,
      "activations/layer7_attention_weight_max": 97.39215850830078,
      "activations/layer7_attention_weight_min": -104.96624755859375,
      "activations/layer8_attention_weight_max": 43.47327423095703,
      "activations/layer8_attention_weight_min": -46.98078918457031,
      "activations/layer9_attention_weight_max": 31.710519790649414,
      "activations/layer9_attention_weight_min": -35.38985061645508,
      "epoch": 20.78,
      "learning_rate": 1.612878787878788e-05,
      "loss": 2.719,
      "step": 357700
    },
    {
      "activations/layer0_attention_weight_max": 16.298513412475586,
      "activations/layer0_attention_weight_min": -13.656344413757324,
      "activations/layer10_attention_weight_max": 33.220123291015625,
      "activations/layer10_attention_weight_min": -34.66779327392578,
      "activations/layer11_attention_weight_max": 32.8817138671875,
      "activations/layer11_attention_weight_min": -34.23369598388672,
      "activations/layer12_attention_weight_max": 24.912363052368164,
      "activations/layer12_attention_weight_min": -27.03211212158203,
      "activations/layer13_attention_weight_max": 36.33958053588867,
      "activations/layer13_attention_weight_min": -36.18107604980469,
      "activations/layer14_attention_weight_max": 38.94599914550781,
      "activations/layer14_attention_weight_min": -40.05812072753906,
      "activations/layer15_attention_weight_max": 39.734195709228516,
      "activations/layer15_attention_weight_min": -38.62232971191406,
      "activations/layer16_attention_weight_max": 29.323745727539062,
      "activations/layer16_attention_weight_min": -26.300453186035156,
      "activations/layer17_attention_weight_max": 29.21137809753418,
      "activations/layer17_attention_weight_min": -25.619909286499023,
      "activations/layer18_attention_weight_max": 31.908620834350586,
      "activations/layer18_attention_weight_min": -25.6158504486084,
      "activations/layer19_attention_weight_max": 34.204437255859375,
      "activations/layer19_attention_weight_min": -32.389190673828125,
      "activations/layer1_attention_weight_max": 15.662620544433594,
      "activations/layer1_attention_weight_min": -15.076334953308105,
      "activations/layer20_attention_weight_max": 28.321983337402344,
      "activations/layer20_attention_weight_min": -25.449596405029297,
      "activations/layer21_attention_weight_max": 27.907987594604492,
      "activations/layer21_attention_weight_min": -25.6087589263916,
      "activations/layer22_attention_weight_max": 42.689598083496094,
      "activations/layer22_attention_weight_min": -33.270050048828125,
      "activations/layer23_attention_weight_max": 30.540861129760742,
      "activations/layer23_attention_weight_min": -26.442459106445312,
      "activations/layer2_attention_weight_max": 33.91838455200195,
      "activations/layer2_attention_weight_min": -33.408424377441406,
      "activations/layer3_attention_weight_max": 101.74050903320312,
      "activations/layer3_attention_weight_min": -107.65290069580078,
      "activations/layer4_attention_weight_max": 106.48990631103516,
      "activations/layer4_attention_weight_min": -108.78704071044922,
      "activations/layer5_attention_weight_max": 51.07891082763672,
      "activations/layer5_attention_weight_min": -61.84828186035156,
      "activations/layer6_attention_weight_max": 43.62712097167969,
      "activations/layer6_attention_weight_min": -47.762184143066406,
      "activations/layer7_attention_weight_max": 95.56932067871094,
      "activations/layer7_attention_weight_min": -96.89884185791016,
      "activations/layer8_attention_weight_max": 42.09685134887695,
      "activations/layer8_attention_weight_min": -43.3757438659668,
      "activations/layer9_attention_weight_max": 31.325231552124023,
      "activations/layer9_attention_weight_min": -33.76484298706055,
      "epoch": 20.79,
      "learning_rate": 1.6109848484848484e-05,
      "loss": 2.7273,
      "step": 357750
    },
    {
      "activations/layer0_attention_weight_max": 15.591354370117188,
      "activations/layer0_attention_weight_min": -12.782367706298828,
      "activations/layer10_attention_weight_max": 35.910762786865234,
      "activations/layer10_attention_weight_min": -36.505470275878906,
      "activations/layer11_attention_weight_max": 35.38068771362305,
      "activations/layer11_attention_weight_min": -36.89055252075195,
      "activations/layer12_attention_weight_max": 25.874988555908203,
      "activations/layer12_attention_weight_min": -27.23602867126465,
      "activations/layer13_attention_weight_max": 39.27663803100586,
      "activations/layer13_attention_weight_min": -36.6838264465332,
      "activations/layer14_attention_weight_max": 42.058258056640625,
      "activations/layer14_attention_weight_min": -39.61026382446289,
      "activations/layer15_attention_weight_max": 40.84263610839844,
      "activations/layer15_attention_weight_min": -35.40910339355469,
      "activations/layer16_attention_weight_max": 30.808425903320312,
      "activations/layer16_attention_weight_min": -29.494592666625977,
      "activations/layer17_attention_weight_max": 31.270633697509766,
      "activations/layer17_attention_weight_min": -28.096216201782227,
      "activations/layer18_attention_weight_max": 33.57383346557617,
      "activations/layer18_attention_weight_min": -30.9570369720459,
      "activations/layer19_attention_weight_max": 37.05817794799805,
      "activations/layer19_attention_weight_min": -35.009613037109375,
      "activations/layer1_attention_weight_max": 16.218809127807617,
      "activations/layer1_attention_weight_min": -14.232601165771484,
      "activations/layer20_attention_weight_max": 32.52260208129883,
      "activations/layer20_attention_weight_min": -26.660022735595703,
      "activations/layer21_attention_weight_max": 32.92063522338867,
      "activations/layer21_attention_weight_min": -26.713958740234375,
      "activations/layer22_attention_weight_max": 45.370697021484375,
      "activations/layer22_attention_weight_min": -35.61384582519531,
      "activations/layer23_attention_weight_max": 36.75062561035156,
      "activations/layer23_attention_weight_min": -26.997467041015625,
      "activations/layer2_attention_weight_max": 34.8142204284668,
      "activations/layer2_attention_weight_min": -31.762256622314453,
      "activations/layer3_attention_weight_max": 105.79743194580078,
      "activations/layer3_attention_weight_min": -106.34445190429688,
      "activations/layer4_attention_weight_max": 114.19243621826172,
      "activations/layer4_attention_weight_min": -111.41747283935547,
      "activations/layer5_attention_weight_max": 53.13252258300781,
      "activations/layer5_attention_weight_min": -62.87358093261719,
      "activations/layer6_attention_weight_max": 44.89485168457031,
      "activations/layer6_attention_weight_min": -46.24223709106445,
      "activations/layer7_attention_weight_max": 96.2490234375,
      "activations/layer7_attention_weight_min": -99.44605255126953,
      "activations/layer8_attention_weight_max": 45.085044860839844,
      "activations/layer8_attention_weight_min": -46.773658752441406,
      "activations/layer9_attention_weight_max": 32.60451889038086,
      "activations/layer9_attention_weight_min": -35.70969772338867,
      "epoch": 20.79,
      "learning_rate": 1.609090909090909e-05,
      "loss": 2.7089,
      "step": 357800
    },
    {
      "activations/layer0_attention_weight_max": 16.256717681884766,
      "activations/layer0_attention_weight_min": -13.011948585510254,
      "activations/layer10_attention_weight_max": 48.42514419555664,
      "activations/layer10_attention_weight_min": -47.37491989135742,
      "activations/layer11_attention_weight_max": 49.32276916503906,
      "activations/layer11_attention_weight_min": -51.910926818847656,
      "activations/layer12_attention_weight_max": 38.17229461669922,
      "activations/layer12_attention_weight_min": -33.49134063720703,
      "activations/layer13_attention_weight_max": 56.06342697143555,
      "activations/layer13_attention_weight_min": -41.049110412597656,
      "activations/layer14_attention_weight_max": 54.73786544799805,
      "activations/layer14_attention_weight_min": -48.28460693359375,
      "activations/layer15_attention_weight_max": 51.76091003417969,
      "activations/layer15_attention_weight_min": -49.58419418334961,
      "activations/layer16_attention_weight_max": 34.21243667602539,
      "activations/layer16_attention_weight_min": -31.058292388916016,
      "activations/layer17_attention_weight_max": 30.775604248046875,
      "activations/layer17_attention_weight_min": -32.35738754272461,
      "activations/layer18_attention_weight_max": 33.49116897583008,
      "activations/layer18_attention_weight_min": -30.20221519470215,
      "activations/layer19_attention_weight_max": 32.168087005615234,
      "activations/layer19_attention_weight_min": -32.25983810424805,
      "activations/layer1_attention_weight_max": 16.274616241455078,
      "activations/layer1_attention_weight_min": -14.551639556884766,
      "activations/layer20_attention_weight_max": 28.195695877075195,
      "activations/layer20_attention_weight_min": -25.151704788208008,
      "activations/layer21_attention_weight_max": 27.888118743896484,
      "activations/layer21_attention_weight_min": -25.465221405029297,
      "activations/layer22_attention_weight_max": 39.09628677368164,
      "activations/layer22_attention_weight_min": -32.683433532714844,
      "activations/layer23_attention_weight_max": 29.103879928588867,
      "activations/layer23_attention_weight_min": -27.214933395385742,
      "activations/layer2_attention_weight_max": 32.43234634399414,
      "activations/layer2_attention_weight_min": -33.88154220581055,
      "activations/layer3_attention_weight_max": 102.90898895263672,
      "activations/layer3_attention_weight_min": -104.93653869628906,
      "activations/layer4_attention_weight_max": 105.73355865478516,
      "activations/layer4_attention_weight_min": -112.30924224853516,
      "activations/layer5_attention_weight_max": 53.9522819519043,
      "activations/layer5_attention_weight_min": -61.037776947021484,
      "activations/layer6_attention_weight_max": 45.444602966308594,
      "activations/layer6_attention_weight_min": -47.40877151489258,
      "activations/layer7_attention_weight_max": 118.72708892822266,
      "activations/layer7_attention_weight_min": -99.57466888427734,
      "activations/layer8_attention_weight_max": 52.70901107788086,
      "activations/layer8_attention_weight_min": -51.548160552978516,
      "activations/layer9_attention_weight_max": 39.859554290771484,
      "activations/layer9_attention_weight_min": -43.843326568603516,
      "epoch": 20.79,
      "learning_rate": 1.6071969696969697e-05,
      "loss": 2.7165,
      "step": 357850
    },
    {
      "activations/layer0_attention_weight_max": 18.177753448486328,
      "activations/layer0_attention_weight_min": -14.046767234802246,
      "activations/layer10_attention_weight_max": 36.40537643432617,
      "activations/layer10_attention_weight_min": -33.82484817504883,
      "activations/layer11_attention_weight_max": 37.27779006958008,
      "activations/layer11_attention_weight_min": -34.21412658691406,
      "activations/layer12_attention_weight_max": 26.230493545532227,
      "activations/layer12_attention_weight_min": -26.433395385742188,
      "activations/layer13_attention_weight_max": 36.55177307128906,
      "activations/layer13_attention_weight_min": -34.40766906738281,
      "activations/layer14_attention_weight_max": 40.804107666015625,
      "activations/layer14_attention_weight_min": -35.87311935424805,
      "activations/layer15_attention_weight_max": 36.38193130493164,
      "activations/layer15_attention_weight_min": -35.966407775878906,
      "activations/layer16_attention_weight_max": 29.645156860351562,
      "activations/layer16_attention_weight_min": -26.804170608520508,
      "activations/layer17_attention_weight_max": 27.36427116394043,
      "activations/layer17_attention_weight_min": -25.69083595275879,
      "activations/layer18_attention_weight_max": 29.041072845458984,
      "activations/layer18_attention_weight_min": -25.36234474182129,
      "activations/layer19_attention_weight_max": 31.2247371673584,
      "activations/layer19_attention_weight_min": -30.390844345092773,
      "activations/layer1_attention_weight_max": 15.938312530517578,
      "activations/layer1_attention_weight_min": -14.1873197555542,
      "activations/layer20_attention_weight_max": 28.545055389404297,
      "activations/layer20_attention_weight_min": -25.449766159057617,
      "activations/layer21_attention_weight_max": 26.78786849975586,
      "activations/layer21_attention_weight_min": -24.340038299560547,
      "activations/layer22_attention_weight_max": 37.35451126098633,
      "activations/layer22_attention_weight_min": -30.43285369873047,
      "activations/layer23_attention_weight_max": 28.65212631225586,
      "activations/layer23_attention_weight_min": -26.36389923095703,
      "activations/layer2_attention_weight_max": 33.7374267578125,
      "activations/layer2_attention_weight_min": -31.64527130126953,
      "activations/layer3_attention_weight_max": 98.36448669433594,
      "activations/layer3_attention_weight_min": -101.15159606933594,
      "activations/layer4_attention_weight_max": 101.03662109375,
      "activations/layer4_attention_weight_min": -108.44012451171875,
      "activations/layer5_attention_weight_max": 48.79102325439453,
      "activations/layer5_attention_weight_min": -66.27261352539062,
      "activations/layer6_attention_weight_max": 42.34355545043945,
      "activations/layer6_attention_weight_min": -45.79079818725586,
      "activations/layer7_attention_weight_max": 101.04788208007812,
      "activations/layer7_attention_weight_min": -93.11116027832031,
      "activations/layer8_attention_weight_max": 45.959930419921875,
      "activations/layer8_attention_weight_min": -41.67402648925781,
      "activations/layer9_attention_weight_max": 29.483264923095703,
      "activations/layer9_attention_weight_min": -34.22303771972656,
      "epoch": 20.8,
      "learning_rate": 1.60530303030303e-05,
      "loss": 2.7201,
      "step": 357900
    },
    {
      "activations/layer0_attention_weight_max": 17.13825798034668,
      "activations/layer0_attention_weight_min": -13.703925132751465,
      "activations/layer10_attention_weight_max": 33.9332389831543,
      "activations/layer10_attention_weight_min": -33.940643310546875,
      "activations/layer11_attention_weight_max": 36.56557846069336,
      "activations/layer11_attention_weight_min": -34.17485046386719,
      "activations/layer12_attention_weight_max": 24.72488021850586,
      "activations/layer12_attention_weight_min": -25.8507137298584,
      "activations/layer13_attention_weight_max": 36.41521072387695,
      "activations/layer13_attention_weight_min": -36.3018684387207,
      "activations/layer14_attention_weight_max": 38.58376693725586,
      "activations/layer14_attention_weight_min": -37.53794479370117,
      "activations/layer15_attention_weight_max": 37.94752883911133,
      "activations/layer15_attention_weight_min": -35.71428680419922,
      "activations/layer16_attention_weight_max": 28.835134506225586,
      "activations/layer16_attention_weight_min": -30.41860008239746,
      "activations/layer17_attention_weight_max": 29.15652847290039,
      "activations/layer17_attention_weight_min": -24.979328155517578,
      "activations/layer18_attention_weight_max": 31.284563064575195,
      "activations/layer18_attention_weight_min": -27.175325393676758,
      "activations/layer19_attention_weight_max": 32.35702133178711,
      "activations/layer19_attention_weight_min": -31.985219955444336,
      "activations/layer1_attention_weight_max": 16.3862247467041,
      "activations/layer1_attention_weight_min": -14.833930969238281,
      "activations/layer20_attention_weight_max": 30.631553649902344,
      "activations/layer20_attention_weight_min": -24.459087371826172,
      "activations/layer21_attention_weight_max": 27.45880126953125,
      "activations/layer21_attention_weight_min": -23.84971809387207,
      "activations/layer22_attention_weight_max": 38.759952545166016,
      "activations/layer22_attention_weight_min": -30.39781951904297,
      "activations/layer23_attention_weight_max": 35.34556579589844,
      "activations/layer23_attention_weight_min": -25.96670913696289,
      "activations/layer2_attention_weight_max": 33.385841369628906,
      "activations/layer2_attention_weight_min": -32.56449890136719,
      "activations/layer3_attention_weight_max": 104.70536041259766,
      "activations/layer3_attention_weight_min": -102.7818603515625,
      "activations/layer4_attention_weight_max": 104.64070892333984,
      "activations/layer4_attention_weight_min": -108.09566497802734,
      "activations/layer5_attention_weight_max": 50.41746520996094,
      "activations/layer5_attention_weight_min": -64.30242156982422,
      "activations/layer6_attention_weight_max": 42.03996276855469,
      "activations/layer6_attention_weight_min": -46.04221725463867,
      "activations/layer7_attention_weight_max": 92.01277923583984,
      "activations/layer7_attention_weight_min": -96.05089569091797,
      "activations/layer8_attention_weight_max": 41.95409393310547,
      "activations/layer8_attention_weight_min": -43.32209014892578,
      "activations/layer9_attention_weight_max": 32.776729583740234,
      "activations/layer9_attention_weight_min": -33.797550201416016,
      "epoch": 20.8,
      "learning_rate": 1.6034090909090906e-05,
      "loss": 2.71,
      "step": 357950
    },
    {
      "activations/layer0_attention_weight_max": 16.6397762298584,
      "activations/layer0_attention_weight_min": -13.416398048400879,
      "activations/layer10_attention_weight_max": 33.55193328857422,
      "activations/layer10_attention_weight_min": -35.437828063964844,
      "activations/layer11_attention_weight_max": 32.268287658691406,
      "activations/layer11_attention_weight_min": -36.26130676269531,
      "activations/layer12_attention_weight_max": 28.380773544311523,
      "activations/layer12_attention_weight_min": -27.991641998291016,
      "activations/layer13_attention_weight_max": 42.63243103027344,
      "activations/layer13_attention_weight_min": -37.611839294433594,
      "activations/layer14_attention_weight_max": 42.68317794799805,
      "activations/layer14_attention_weight_min": -39.1595344543457,
      "activations/layer15_attention_weight_max": 38.693267822265625,
      "activations/layer15_attention_weight_min": -37.35575866699219,
      "activations/layer16_attention_weight_max": 29.360891342163086,
      "activations/layer16_attention_weight_min": -29.19253158569336,
      "activations/layer17_attention_weight_max": 30.94342803955078,
      "activations/layer17_attention_weight_min": -26.558460235595703,
      "activations/layer18_attention_weight_max": 33.497833251953125,
      "activations/layer18_attention_weight_min": -27.229341506958008,
      "activations/layer19_attention_weight_max": 32.387264251708984,
      "activations/layer19_attention_weight_min": -32.794979095458984,
      "activations/layer1_attention_weight_max": 15.676586151123047,
      "activations/layer1_attention_weight_min": -14.501750946044922,
      "activations/layer20_attention_weight_max": 30.469465255737305,
      "activations/layer20_attention_weight_min": -26.887401580810547,
      "activations/layer21_attention_weight_max": 29.922237396240234,
      "activations/layer21_attention_weight_min": -26.24284553527832,
      "activations/layer22_attention_weight_max": 40.376708984375,
      "activations/layer22_attention_weight_min": -33.325584411621094,
      "activations/layer23_attention_weight_max": 32.11683654785156,
      "activations/layer23_attention_weight_min": -30.138198852539062,
      "activations/layer2_attention_weight_max": 33.2131233215332,
      "activations/layer2_attention_weight_min": -33.474693298339844,
      "activations/layer3_attention_weight_max": 103.71360778808594,
      "activations/layer3_attention_weight_min": -106.61138153076172,
      "activations/layer4_attention_weight_max": 106.84004974365234,
      "activations/layer4_attention_weight_min": -110.7243881225586,
      "activations/layer5_attention_weight_max": 53.14729309082031,
      "activations/layer5_attention_weight_min": -63.26801300048828,
      "activations/layer6_attention_weight_max": 44.622779846191406,
      "activations/layer6_attention_weight_min": -48.068748474121094,
      "activations/layer7_attention_weight_max": 93.44684600830078,
      "activations/layer7_attention_weight_min": -97.44182586669922,
      "activations/layer8_attention_weight_max": 40.56199264526367,
      "activations/layer8_attention_weight_min": -44.4833869934082,
      "activations/layer9_attention_weight_max": 33.089508056640625,
      "activations/layer9_attention_weight_min": -36.264060974121094,
      "epoch": 20.8,
      "learning_rate": 1.601515151515151e-05,
      "loss": 2.718,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4813,
      "eval_samples_per_second": 506.29,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4813,
      "eval_openwebtext_samples_per_second": 506.29,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9445,
      "eval_wikitext_samples_per_second": 234.508,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_lambada_loss": 2.337890625,
      "eval_lambada_ppl": 10.359361721090316,
      "eval_lambada_runtime": 9.5204,
      "eval_lambada_samples_per_second": 511.43,
      "step": 358000
    },
    {
      "activations/layer0_attention_weight_max": 16.50529670715332,
      "activations/layer0_attention_weight_min": -13.537217140197754,
      "activations/layer10_attention_weight_max": 35.567535400390625,
      "activations/layer10_attention_weight_min": -34.93794631958008,
      "activations/layer11_attention_weight_max": 34.11248779296875,
      "activations/layer11_attention_weight_min": -35.816864013671875,
      "activations/layer12_attention_weight_max": 29.481422424316406,
      "activations/layer12_attention_weight_min": -26.846616744995117,
      "activations/layer13_attention_weight_max": 46.10932540893555,
      "activations/layer13_attention_weight_min": -36.156856536865234,
      "activations/layer14_attention_weight_max": 46.440391540527344,
      "activations/layer14_attention_weight_min": -40.22340774536133,
      "activations/layer15_attention_weight_max": 41.40272521972656,
      "activations/layer15_attention_weight_min": -36.942413330078125,
      "activations/layer16_attention_weight_max": 30.71923065185547,
      "activations/layer16_attention_weight_min": -28.949140548706055,
      "activations/layer17_attention_weight_max": 34.6778564453125,
      "activations/layer17_attention_weight_min": -27.251323699951172,
      "activations/layer18_attention_weight_max": 38.6336784362793,
      "activations/layer18_attention_weight_min": -27.472623825073242,
      "activations/layer19_attention_weight_max": 42.57551193237305,
      "activations/layer19_attention_weight_min": -30.857358932495117,
      "activations/layer1_attention_weight_max": 15.435420036315918,
      "activations/layer1_attention_weight_min": -13.89428424835205,
      "activations/layer20_attention_weight_max": 33.574031829833984,
      "activations/layer20_attention_weight_min": -26.49739646911621,
      "activations/layer21_attention_weight_max": 31.779911041259766,
      "activations/layer21_attention_weight_min": -24.770694732666016,
      "activations/layer22_attention_weight_max": 45.07916259765625,
      "activations/layer22_attention_weight_min": -31.43260955810547,
      "activations/layer23_attention_weight_max": 37.59978103637695,
      "activations/layer23_attention_weight_min": -28.3464412689209,
      "activations/layer2_attention_weight_max": 32.65629577636719,
      "activations/layer2_attention_weight_min": -31.49886703491211,
      "activations/layer3_attention_weight_max": 103.96480560302734,
      "activations/layer3_attention_weight_min": -104.04948425292969,
      "activations/layer4_attention_weight_max": 108.96954345703125,
      "activations/layer4_attention_weight_min": -113.55370330810547,
      "activations/layer5_attention_weight_max": 49.983642578125,
      "activations/layer5_attention_weight_min": -61.376155853271484,
      "activations/layer6_attention_weight_max": 46.580284118652344,
      "activations/layer6_attention_weight_min": -48.28251266479492,
      "activations/layer7_attention_weight_max": 96.28987121582031,
      "activations/layer7_attention_weight_min": -101.34445190429688,
      "activations/layer8_attention_weight_max": 43.771385192871094,
      "activations/layer8_attention_weight_min": -47.004066467285156,
      "activations/layer9_attention_weight_max": 34.3490104675293,
      "activations/layer9_attention_weight_min": -35.44343185424805,
      "epoch": 20.8,
      "learning_rate": 1.599621212121212e-05,
      "loss": 2.7064,
      "step": 358050
    },
    {
      "activations/layer0_attention_weight_max": 17.04680633544922,
      "activations/layer0_attention_weight_min": -13.484724998474121,
      "activations/layer10_attention_weight_max": 46.42818069458008,
      "activations/layer10_attention_weight_min": -43.79359817504883,
      "activations/layer11_attention_weight_max": 46.89889144897461,
      "activations/layer11_attention_weight_min": -46.473060607910156,
      "activations/layer12_attention_weight_max": 28.62868881225586,
      "activations/layer12_attention_weight_min": -31.1545352935791,
      "activations/layer13_attention_weight_max": 41.189720153808594,
      "activations/layer13_attention_weight_min": -35.28474044799805,
      "activations/layer14_attention_weight_max": 46.34183883666992,
      "activations/layer14_attention_weight_min": -39.42084503173828,
      "activations/layer15_attention_weight_max": 44.1729850769043,
      "activations/layer15_attention_weight_min": -38.842708587646484,
      "activations/layer16_attention_weight_max": 31.158817291259766,
      "activations/layer16_attention_weight_min": -27.026514053344727,
      "activations/layer17_attention_weight_max": 25.861351013183594,
      "activations/layer17_attention_weight_min": -25.338571548461914,
      "activations/layer18_attention_weight_max": 30.082857131958008,
      "activations/layer18_attention_weight_min": -25.68381118774414,
      "activations/layer19_attention_weight_max": 30.21613121032715,
      "activations/layer19_attention_weight_min": -30.821292877197266,
      "activations/layer1_attention_weight_max": 16.346099853515625,
      "activations/layer1_attention_weight_min": -14.474024772644043,
      "activations/layer20_attention_weight_max": 25.099502563476562,
      "activations/layer20_attention_weight_min": -23.555234909057617,
      "activations/layer21_attention_weight_max": 23.374948501586914,
      "activations/layer21_attention_weight_min": -23.942733764648438,
      "activations/layer22_attention_weight_max": 33.95168685913086,
      "activations/layer22_attention_weight_min": -29.026262283325195,
      "activations/layer23_attention_weight_max": 26.122352600097656,
      "activations/layer23_attention_weight_min": -23.909894943237305,
      "activations/layer2_attention_weight_max": 35.39436340332031,
      "activations/layer2_attention_weight_min": -34.271392822265625,
      "activations/layer3_attention_weight_max": 112.23975372314453,
      "activations/layer3_attention_weight_min": -110.50325775146484,
      "activations/layer4_attention_weight_max": 109.74796295166016,
      "activations/layer4_attention_weight_min": -107.59806060791016,
      "activations/layer5_attention_weight_max": 54.231712341308594,
      "activations/layer5_attention_weight_min": -61.734580993652344,
      "activations/layer6_attention_weight_max": 44.18422317504883,
      "activations/layer6_attention_weight_min": -47.027339935302734,
      "activations/layer7_attention_weight_max": 116.64994049072266,
      "activations/layer7_attention_weight_min": -104.29214477539062,
      "activations/layer8_attention_weight_max": 53.10736083984375,
      "activations/layer8_attention_weight_min": -49.329654693603516,
      "activations/layer9_attention_weight_max": 38.309226989746094,
      "activations/layer9_attention_weight_min": -41.056556701660156,
      "epoch": 20.81,
      "learning_rate": 1.5977272727272728e-05,
      "loss": 2.728,
      "step": 358100
    },
    {
      "activations/layer0_attention_weight_max": 16.31024169921875,
      "activations/layer0_attention_weight_min": -14.301228523254395,
      "activations/layer10_attention_weight_max": 38.77052688598633,
      "activations/layer10_attention_weight_min": -38.90267562866211,
      "activations/layer11_attention_weight_max": 38.402835845947266,
      "activations/layer11_attention_weight_min": -40.54390335083008,
      "activations/layer12_attention_weight_max": 28.308015823364258,
      "activations/layer12_attention_weight_min": -27.889026641845703,
      "activations/layer13_attention_weight_max": 40.582855224609375,
      "activations/layer13_attention_weight_min": -36.55252456665039,
      "activations/layer14_attention_weight_max": 45.41012954711914,
      "activations/layer14_attention_weight_min": -39.36283493041992,
      "activations/layer15_attention_weight_max": 38.71540069580078,
      "activations/layer15_attention_weight_min": -37.729000091552734,
      "activations/layer16_attention_weight_max": 29.30864715576172,
      "activations/layer16_attention_weight_min": -30.06346321105957,
      "activations/layer17_attention_weight_max": 28.957183837890625,
      "activations/layer17_attention_weight_min": -26.40574073791504,
      "activations/layer18_attention_weight_max": 32.87517166137695,
      "activations/layer18_attention_weight_min": -28.245033264160156,
      "activations/layer19_attention_weight_max": 35.78084945678711,
      "activations/layer19_attention_weight_min": -31.52896499633789,
      "activations/layer1_attention_weight_max": 15.608135223388672,
      "activations/layer1_attention_weight_min": -14.484968185424805,
      "activations/layer20_attention_weight_max": 31.771867752075195,
      "activations/layer20_attention_weight_min": -25.129621505737305,
      "activations/layer21_attention_weight_max": 27.48983383178711,
      "activations/layer21_attention_weight_min": -25.039751052856445,
      "activations/layer22_attention_weight_max": 40.36104202270508,
      "activations/layer22_attention_weight_min": -32.25532531738281,
      "activations/layer23_attention_weight_max": 34.860103607177734,
      "activations/layer23_attention_weight_min": -27.179975509643555,
      "activations/layer2_attention_weight_max": 33.79783630371094,
      "activations/layer2_attention_weight_min": -33.019962310791016,
      "activations/layer3_attention_weight_max": 105.75921630859375,
      "activations/layer3_attention_weight_min": -108.86881256103516,
      "activations/layer4_attention_weight_max": 107.93128967285156,
      "activations/layer4_attention_weight_min": -109.88423156738281,
      "activations/layer5_attention_weight_max": 51.28742599487305,
      "activations/layer5_attention_weight_min": -59.989540100097656,
      "activations/layer6_attention_weight_max": 46.83130645751953,
      "activations/layer6_attention_weight_min": -46.710575103759766,
      "activations/layer7_attention_weight_max": 97.99805450439453,
      "activations/layer7_attention_weight_min": -101.80048370361328,
      "activations/layer8_attention_weight_max": 46.640316009521484,
      "activations/layer8_attention_weight_min": -47.6658821105957,
      "activations/layer9_attention_weight_max": 35.888092041015625,
      "activations/layer9_attention_weight_min": -39.2503662109375,
      "epoch": 20.81,
      "learning_rate": 1.5958333333333333e-05,
      "loss": 2.7272,
      "step": 358150
    },
    {
      "activations/layer0_attention_weight_max": 16.126686096191406,
      "activations/layer0_attention_weight_min": -13.233903884887695,
      "activations/layer10_attention_weight_max": 36.30513381958008,
      "activations/layer10_attention_weight_min": -37.880775451660156,
      "activations/layer11_attention_weight_max": 38.018829345703125,
      "activations/layer11_attention_weight_min": -39.7606201171875,
      "activations/layer12_attention_weight_max": 29.517797470092773,
      "activations/layer12_attention_weight_min": -28.738361358642578,
      "activations/layer13_attention_weight_max": 41.762001037597656,
      "activations/layer13_attention_weight_min": -39.3420295715332,
      "activations/layer14_attention_weight_max": 44.953521728515625,
      "activations/layer14_attention_weight_min": -45.278411865234375,
      "activations/layer15_attention_weight_max": 44.878143310546875,
      "activations/layer15_attention_weight_min": -40.529579162597656,
      "activations/layer16_attention_weight_max": 34.262611389160156,
      "activations/layer16_attention_weight_min": -27.536205291748047,
      "activations/layer17_attention_weight_max": 30.635427474975586,
      "activations/layer17_attention_weight_min": -25.72186851501465,
      "activations/layer18_attention_weight_max": 33.439186096191406,
      "activations/layer18_attention_weight_min": -27.722383499145508,
      "activations/layer19_attention_weight_max": 36.40373229980469,
      "activations/layer19_attention_weight_min": -32.71931076049805,
      "activations/layer1_attention_weight_max": 15.709895133972168,
      "activations/layer1_attention_weight_min": -14.512638092041016,
      "activations/layer20_attention_weight_max": 29.510581970214844,
      "activations/layer20_attention_weight_min": -24.664304733276367,
      "activations/layer21_attention_weight_max": 28.616710662841797,
      "activations/layer21_attention_weight_min": -24.66506004333496,
      "activations/layer22_attention_weight_max": 43.94638442993164,
      "activations/layer22_attention_weight_min": -32.36348342895508,
      "activations/layer23_attention_weight_max": 34.99451446533203,
      "activations/layer23_attention_weight_min": -25.49567985534668,
      "activations/layer2_attention_weight_max": 35.56144332885742,
      "activations/layer2_attention_weight_min": -32.93830490112305,
      "activations/layer3_attention_weight_max": 106.08380889892578,
      "activations/layer3_attention_weight_min": -106.26798248291016,
      "activations/layer4_attention_weight_max": 108.55596160888672,
      "activations/layer4_attention_weight_min": -119.18849182128906,
      "activations/layer5_attention_weight_max": 49.63546371459961,
      "activations/layer5_attention_weight_min": -65.0499038696289,
      "activations/layer6_attention_weight_max": 45.21400833129883,
      "activations/layer6_attention_weight_min": -48.11943817138672,
      "activations/layer7_attention_weight_max": 96.03614044189453,
      "activations/layer7_attention_weight_min": -100.50190734863281,
      "activations/layer8_attention_weight_max": 44.233097076416016,
      "activations/layer8_attention_weight_min": -44.495487213134766,
      "activations/layer9_attention_weight_max": 33.01785659790039,
      "activations/layer9_attention_weight_min": -35.895694732666016,
      "epoch": 20.81,
      "learning_rate": 1.5939393939393938e-05,
      "loss": 2.7122,
      "step": 358200
    },
    {
      "activations/layer0_attention_weight_max": 16.007640838623047,
      "activations/layer0_attention_weight_min": -13.179197311401367,
      "activations/layer10_attention_weight_max": 35.28307342529297,
      "activations/layer10_attention_weight_min": -36.296966552734375,
      "activations/layer11_attention_weight_max": 36.16279983520508,
      "activations/layer11_attention_weight_min": -35.54804611206055,
      "activations/layer12_attention_weight_max": 26.535301208496094,
      "activations/layer12_attention_weight_min": -28.362443923950195,
      "activations/layer13_attention_weight_max": 36.4813232421875,
      "activations/layer13_attention_weight_min": -35.17479705810547,
      "activations/layer14_attention_weight_max": 43.00144577026367,
      "activations/layer14_attention_weight_min": -38.94109344482422,
      "activations/layer15_attention_weight_max": 36.549713134765625,
      "activations/layer15_attention_weight_min": -36.09618377685547,
      "activations/layer16_attention_weight_max": 28.575042724609375,
      "activations/layer16_attention_weight_min": -29.086639404296875,
      "activations/layer17_attention_weight_max": 28.012136459350586,
      "activations/layer17_attention_weight_min": -25.365205764770508,
      "activations/layer18_attention_weight_max": 28.814472198486328,
      "activations/layer18_attention_weight_min": -27.8071346282959,
      "activations/layer19_attention_weight_max": 33.81129837036133,
      "activations/layer19_attention_weight_min": -30.798948287963867,
      "activations/layer1_attention_weight_max": 16.07903480529785,
      "activations/layer1_attention_weight_min": -14.368345260620117,
      "activations/layer20_attention_weight_max": 28.88921356201172,
      "activations/layer20_attention_weight_min": -24.17452621459961,
      "activations/layer21_attention_weight_max": 27.912139892578125,
      "activations/layer21_attention_weight_min": -24.264598846435547,
      "activations/layer22_attention_weight_max": 39.914493560791016,
      "activations/layer22_attention_weight_min": -31.02397346496582,
      "activations/layer23_attention_weight_max": 30.385772705078125,
      "activations/layer23_attention_weight_min": -25.185537338256836,
      "activations/layer2_attention_weight_max": 35.970523834228516,
      "activations/layer2_attention_weight_min": -33.61676788330078,
      "activations/layer3_attention_weight_max": 106.6148681640625,
      "activations/layer3_attention_weight_min": -108.17741394042969,
      "activations/layer4_attention_weight_max": 106.2804946899414,
      "activations/layer4_attention_weight_min": -106.03181457519531,
      "activations/layer5_attention_weight_max": 48.3514404296875,
      "activations/layer5_attention_weight_min": -62.412254333496094,
      "activations/layer6_attention_weight_max": 42.90168762207031,
      "activations/layer6_attention_weight_min": -45.86997985839844,
      "activations/layer7_attention_weight_max": 92.33624267578125,
      "activations/layer7_attention_weight_min": -96.7744140625,
      "activations/layer8_attention_weight_max": 43.05903625488281,
      "activations/layer8_attention_weight_min": -43.6009407043457,
      "activations/layer9_attention_weight_max": 34.5324821472168,
      "activations/layer9_attention_weight_min": -33.29304885864258,
      "epoch": 20.82,
      "learning_rate": 1.5920454545454546e-05,
      "loss": 2.7215,
      "step": 358250
    },
    {
      "activations/layer0_attention_weight_max": 17.186199188232422,
      "activations/layer0_attention_weight_min": -13.894349098205566,
      "activations/layer10_attention_weight_max": 34.176116943359375,
      "activations/layer10_attention_weight_min": -39.045005798339844,
      "activations/layer11_attention_weight_max": 35.829925537109375,
      "activations/layer11_attention_weight_min": -34.0665283203125,
      "activations/layer12_attention_weight_max": 27.262048721313477,
      "activations/layer12_attention_weight_min": -26.13810157775879,
      "activations/layer13_attention_weight_max": 40.45854187011719,
      "activations/layer13_attention_weight_min": -36.17637634277344,
      "activations/layer14_attention_weight_max": 40.49046325683594,
      "activations/layer14_attention_weight_min": -38.02542495727539,
      "activations/layer15_attention_weight_max": 39.772918701171875,
      "activations/layer15_attention_weight_min": -35.57376480102539,
      "activations/layer16_attention_weight_max": 29.142929077148438,
      "activations/layer16_attention_weight_min": -32.44594192504883,
      "activations/layer17_attention_weight_max": 27.729448318481445,
      "activations/layer17_attention_weight_min": -26.97223472595215,
      "activations/layer18_attention_weight_max": 31.325294494628906,
      "activations/layer18_attention_weight_min": -26.995439529418945,
      "activations/layer19_attention_weight_max": 35.699737548828125,
      "activations/layer19_attention_weight_min": -31.814334869384766,
      "activations/layer1_attention_weight_max": 15.549039840698242,
      "activations/layer1_attention_weight_min": -15.08859634399414,
      "activations/layer20_attention_weight_max": 27.329883575439453,
      "activations/layer20_attention_weight_min": -24.9940242767334,
      "activations/layer21_attention_weight_max": 27.123098373413086,
      "activations/layer21_attention_weight_min": -23.50674819946289,
      "activations/layer22_attention_weight_max": 39.321533203125,
      "activations/layer22_attention_weight_min": -33.202693939208984,
      "activations/layer23_attention_weight_max": 30.435039520263672,
      "activations/layer23_attention_weight_min": -24.884601593017578,
      "activations/layer2_attention_weight_max": 34.40177917480469,
      "activations/layer2_attention_weight_min": -32.429527282714844,
      "activations/layer3_attention_weight_max": 107.27210235595703,
      "activations/layer3_attention_weight_min": -103.6311264038086,
      "activations/layer4_attention_weight_max": 107.67730712890625,
      "activations/layer4_attention_weight_min": -107.96947479248047,
      "activations/layer5_attention_weight_max": 49.52196502685547,
      "activations/layer5_attention_weight_min": -62.13594055175781,
      "activations/layer6_attention_weight_max": 45.48709487915039,
      "activations/layer6_attention_weight_min": -46.95081329345703,
      "activations/layer7_attention_weight_max": 97.01114654541016,
      "activations/layer7_attention_weight_min": -98.09314727783203,
      "activations/layer8_attention_weight_max": 42.35576629638672,
      "activations/layer8_attention_weight_min": -48.82350540161133,
      "activations/layer9_attention_weight_max": 31.060888290405273,
      "activations/layer9_attention_weight_min": -37.66468811035156,
      "epoch": 20.82,
      "learning_rate": 1.590151515151515e-05,
      "loss": 2.7048,
      "step": 358300
    },
    {
      "activations/layer0_attention_weight_max": 15.639949798583984,
      "activations/layer0_attention_weight_min": -13.053958892822266,
      "activations/layer10_attention_weight_max": 42.930389404296875,
      "activations/layer10_attention_weight_min": -38.62703323364258,
      "activations/layer11_attention_weight_max": 41.432125091552734,
      "activations/layer11_attention_weight_min": -40.27650833129883,
      "activations/layer12_attention_weight_max": 33.02109146118164,
      "activations/layer12_attention_weight_min": -31.838441848754883,
      "activations/layer13_attention_weight_max": 50.3122444152832,
      "activations/layer13_attention_weight_min": -39.35437774658203,
      "activations/layer14_attention_weight_max": 53.29345703125,
      "activations/layer14_attention_weight_min": -45.319793701171875,
      "activations/layer15_attention_weight_max": 54.112060546875,
      "activations/layer15_attention_weight_min": -42.474613189697266,
      "activations/layer16_attention_weight_max": 37.778472900390625,
      "activations/layer16_attention_weight_min": -30.440616607666016,
      "activations/layer17_attention_weight_max": 35.82408142089844,
      "activations/layer17_attention_weight_min": -29.51936149597168,
      "activations/layer18_attention_weight_max": 35.410369873046875,
      "activations/layer18_attention_weight_min": -26.51470947265625,
      "activations/layer19_attention_weight_max": 42.07340621948242,
      "activations/layer19_attention_weight_min": -34.451560974121094,
      "activations/layer1_attention_weight_max": 15.638742446899414,
      "activations/layer1_attention_weight_min": -14.697942733764648,
      "activations/layer20_attention_weight_max": 35.273902893066406,
      "activations/layer20_attention_weight_min": -26.616043090820312,
      "activations/layer21_attention_weight_max": 33.04717254638672,
      "activations/layer21_attention_weight_min": -24.44651985168457,
      "activations/layer22_attention_weight_max": 41.15854263305664,
      "activations/layer22_attention_weight_min": -32.23341751098633,
      "activations/layer23_attention_weight_max": 31.619428634643555,
      "activations/layer23_attention_weight_min": -25.195051193237305,
      "activations/layer2_attention_weight_max": 33.35985565185547,
      "activations/layer2_attention_weight_min": -31.922142028808594,
      "activations/layer3_attention_weight_max": 104.76373291015625,
      "activations/layer3_attention_weight_min": -105.49636840820312,
      "activations/layer4_attention_weight_max": 109.77263641357422,
      "activations/layer4_attention_weight_min": -112.25823974609375,
      "activations/layer5_attention_weight_max": 51.889339447021484,
      "activations/layer5_attention_weight_min": -62.858436584472656,
      "activations/layer6_attention_weight_max": 47.07649612426758,
      "activations/layer6_attention_weight_min": -49.739444732666016,
      "activations/layer7_attention_weight_max": 106.41966247558594,
      "activations/layer7_attention_weight_min": -106.19245147705078,
      "activations/layer8_attention_weight_max": 48.54954528808594,
      "activations/layer8_attention_weight_min": -50.86030197143555,
      "activations/layer9_attention_weight_max": 39.39011001586914,
      "activations/layer9_attention_weight_min": -40.06509017944336,
      "epoch": 20.82,
      "learning_rate": 1.5882575757575756e-05,
      "loss": 2.7043,
      "step": 358350
    },
    {
      "activations/layer0_attention_weight_max": 17.198904037475586,
      "activations/layer0_attention_weight_min": -14.964993476867676,
      "activations/layer10_attention_weight_max": 36.56343460083008,
      "activations/layer10_attention_weight_min": -36.37359619140625,
      "activations/layer11_attention_weight_max": 35.24713897705078,
      "activations/layer11_attention_weight_min": -34.61862564086914,
      "activations/layer12_attention_weight_max": 26.99581527709961,
      "activations/layer12_attention_weight_min": -27.19390106201172,
      "activations/layer13_attention_weight_max": 39.61172103881836,
      "activations/layer13_attention_weight_min": -35.318687438964844,
      "activations/layer14_attention_weight_max": 41.54404830932617,
      "activations/layer14_attention_weight_min": -37.5589485168457,
      "activations/layer15_attention_weight_max": 36.541378021240234,
      "activations/layer15_attention_weight_min": -34.140750885009766,
      "activations/layer16_attention_weight_max": 27.57635498046875,
      "activations/layer16_attention_weight_min": -28.283939361572266,
      "activations/layer17_attention_weight_max": 27.66103744506836,
      "activations/layer17_attention_weight_min": -26.5989990234375,
      "activations/layer18_attention_weight_max": 30.31183433532715,
      "activations/layer18_attention_weight_min": -25.510093688964844,
      "activations/layer19_attention_weight_max": 34.092838287353516,
      "activations/layer19_attention_weight_min": -31.624780654907227,
      "activations/layer1_attention_weight_max": 15.917952537536621,
      "activations/layer1_attention_weight_min": -16.11090087890625,
      "activations/layer20_attention_weight_max": 26.92308235168457,
      "activations/layer20_attention_weight_min": -25.16502571105957,
      "activations/layer21_attention_weight_max": 25.800376892089844,
      "activations/layer21_attention_weight_min": -27.658029556274414,
      "activations/layer22_attention_weight_max": 39.5676155090332,
      "activations/layer22_attention_weight_min": -33.572425842285156,
      "activations/layer23_attention_weight_max": 32.164024353027344,
      "activations/layer23_attention_weight_min": -27.722333908081055,
      "activations/layer2_attention_weight_max": 34.09324264526367,
      "activations/layer2_attention_weight_min": -33.02577590942383,
      "activations/layer3_attention_weight_max": 101.67475891113281,
      "activations/layer3_attention_weight_min": -101.91569519042969,
      "activations/layer4_attention_weight_max": 105.52520751953125,
      "activations/layer4_attention_weight_min": -109.66339111328125,
      "activations/layer5_attention_weight_max": 51.91242218017578,
      "activations/layer5_attention_weight_min": -62.76122283935547,
      "activations/layer6_attention_weight_max": 42.10024642944336,
      "activations/layer6_attention_weight_min": -46.00939178466797,
      "activations/layer7_attention_weight_max": 91.1852798461914,
      "activations/layer7_attention_weight_min": -95.38312530517578,
      "activations/layer8_attention_weight_max": 41.0766487121582,
      "activations/layer8_attention_weight_min": -42.90239715576172,
      "activations/layer9_attention_weight_max": 32.95026779174805,
      "activations/layer9_attention_weight_min": -35.7091178894043,
      "epoch": 20.83,
      "learning_rate": 1.5863636363636364e-05,
      "loss": 2.725,
      "step": 358400
    },
    {
      "activations/layer0_attention_weight_max": 16.998371124267578,
      "activations/layer0_attention_weight_min": -14.60159969329834,
      "activations/layer10_attention_weight_max": 34.760284423828125,
      "activations/layer10_attention_weight_min": -35.44390869140625,
      "activations/layer11_attention_weight_max": 34.28001403808594,
      "activations/layer11_attention_weight_min": -36.57734680175781,
      "activations/layer12_attention_weight_max": 32.36261749267578,
      "activations/layer12_attention_weight_min": -27.930051803588867,
      "activations/layer13_attention_weight_max": 44.38936996459961,
      "activations/layer13_attention_weight_min": -36.619415283203125,
      "activations/layer14_attention_weight_max": 45.555118560791016,
      "activations/layer14_attention_weight_min": -38.27039337158203,
      "activations/layer15_attention_weight_max": 44.21244812011719,
      "activations/layer15_attention_weight_min": -38.80573654174805,
      "activations/layer16_attention_weight_max": 31.90038299560547,
      "activations/layer16_attention_weight_min": -28.76081657409668,
      "activations/layer17_attention_weight_max": 28.405067443847656,
      "activations/layer17_attention_weight_min": -26.647266387939453,
      "activations/layer18_attention_weight_max": 30.649423599243164,
      "activations/layer18_attention_weight_min": -25.023727416992188,
      "activations/layer19_attention_weight_max": 32.34894943237305,
      "activations/layer19_attention_weight_min": -31.867963790893555,
      "activations/layer1_attention_weight_max": 15.360600471496582,
      "activations/layer1_attention_weight_min": -14.548124313354492,
      "activations/layer20_attention_weight_max": 26.492534637451172,
      "activations/layer20_attention_weight_min": -27.006072998046875,
      "activations/layer21_attention_weight_max": 27.047489166259766,
      "activations/layer21_attention_weight_min": -25.4473934173584,
      "activations/layer22_attention_weight_max": 41.96013641357422,
      "activations/layer22_attention_weight_min": -30.941234588623047,
      "activations/layer23_attention_weight_max": 29.73564910888672,
      "activations/layer23_attention_weight_min": -26.46297264099121,
      "activations/layer2_attention_weight_max": 33.35859680175781,
      "activations/layer2_attention_weight_min": -32.42434310913086,
      "activations/layer3_attention_weight_max": 102.04389190673828,
      "activations/layer3_attention_weight_min": -103.58770751953125,
      "activations/layer4_attention_weight_max": 105.0486831665039,
      "activations/layer4_attention_weight_min": -110.1554946899414,
      "activations/layer5_attention_weight_max": 52.765621185302734,
      "activations/layer5_attention_weight_min": -59.131099700927734,
      "activations/layer6_attention_weight_max": 42.857704162597656,
      "activations/layer6_attention_weight_min": -45.966976165771484,
      "activations/layer7_attention_weight_max": 95.48158264160156,
      "activations/layer7_attention_weight_min": -94.49961853027344,
      "activations/layer8_attention_weight_max": 42.12757110595703,
      "activations/layer8_attention_weight_min": -44.36503982543945,
      "activations/layer9_attention_weight_max": 32.635467529296875,
      "activations/layer9_attention_weight_min": -34.71871566772461,
      "epoch": 20.83,
      "learning_rate": 1.584469696969697e-05,
      "loss": 2.7102,
      "step": 358450
    },
    {
      "activations/layer0_attention_weight_max": 15.927762985229492,
      "activations/layer0_attention_weight_min": -13.867141723632812,
      "activations/layer10_attention_weight_max": 36.754886627197266,
      "activations/layer10_attention_weight_min": -35.024070739746094,
      "activations/layer11_attention_weight_max": 34.637229919433594,
      "activations/layer11_attention_weight_min": -36.24878692626953,
      "activations/layer12_attention_weight_max": 26.65134048461914,
      "activations/layer12_attention_weight_min": -26.355884552001953,
      "activations/layer13_attention_weight_max": 39.81427764892578,
      "activations/layer13_attention_weight_min": -36.474830627441406,
      "activations/layer14_attention_weight_max": 45.29404067993164,
      "activations/layer14_attention_weight_min": -38.078269958496094,
      "activations/layer15_attention_weight_max": 39.842567443847656,
      "activations/layer15_attention_weight_min": -36.381717681884766,
      "activations/layer16_attention_weight_max": 30.90167236328125,
      "activations/layer16_attention_weight_min": -26.248981475830078,
      "activations/layer17_attention_weight_max": 30.4027099609375,
      "activations/layer17_attention_weight_min": -24.814889907836914,
      "activations/layer18_attention_weight_max": 32.889427185058594,
      "activations/layer18_attention_weight_min": -27.36359214782715,
      "activations/layer19_attention_weight_max": 32.358699798583984,
      "activations/layer19_attention_weight_min": -32.116661071777344,
      "activations/layer1_attention_weight_max": 16.43572998046875,
      "activations/layer1_attention_weight_min": -15.857041358947754,
      "activations/layer20_attention_weight_max": 29.149206161499023,
      "activations/layer20_attention_weight_min": -26.522327423095703,
      "activations/layer21_attention_weight_max": 28.648618698120117,
      "activations/layer21_attention_weight_min": -24.1600341796875,
      "activations/layer22_attention_weight_max": 45.97466278076172,
      "activations/layer22_attention_weight_min": -29.632213592529297,
      "activations/layer23_attention_weight_max": 31.41735076904297,
      "activations/layer23_attention_weight_min": -25.072813034057617,
      "activations/layer2_attention_weight_max": 34.432106018066406,
      "activations/layer2_attention_weight_min": -33.60179901123047,
      "activations/layer3_attention_weight_max": 112.75072479248047,
      "activations/layer3_attention_weight_min": -108.4910888671875,
      "activations/layer4_attention_weight_max": 115.80262756347656,
      "activations/layer4_attention_weight_min": -116.1029052734375,
      "activations/layer5_attention_weight_max": 50.48557662963867,
      "activations/layer5_attention_weight_min": -62.54380798339844,
      "activations/layer6_attention_weight_max": 45.00658416748047,
      "activations/layer6_attention_weight_min": -48.15778350830078,
      "activations/layer7_attention_weight_max": 94.96452331542969,
      "activations/layer7_attention_weight_min": -96.21212005615234,
      "activations/layer8_attention_weight_max": 42.57711410522461,
      "activations/layer8_attention_weight_min": -46.929771423339844,
      "activations/layer9_attention_weight_max": 31.89453125,
      "activations/layer9_attention_weight_min": -36.584224700927734,
      "epoch": 20.83,
      "learning_rate": 1.5825757575757574e-05,
      "loss": 2.7074,
      "step": 358500
    },
    {
      "activations/layer0_attention_weight_max": 17.446914672851562,
      "activations/layer0_attention_weight_min": -13.162101745605469,
      "activations/layer10_attention_weight_max": 35.048553466796875,
      "activations/layer10_attention_weight_min": -34.91379165649414,
      "activations/layer11_attention_weight_max": 34.420501708984375,
      "activations/layer11_attention_weight_min": -34.0137939453125,
      "activations/layer12_attention_weight_max": 25.672183990478516,
      "activations/layer12_attention_weight_min": -26.495681762695312,
      "activations/layer13_attention_weight_max": 40.51274871826172,
      "activations/layer13_attention_weight_min": -37.19352340698242,
      "activations/layer14_attention_weight_max": 41.982276916503906,
      "activations/layer14_attention_weight_min": -39.280662536621094,
      "activations/layer15_attention_weight_max": 38.81082534790039,
      "activations/layer15_attention_weight_min": -35.961570739746094,
      "activations/layer16_attention_weight_max": 27.53496551513672,
      "activations/layer16_attention_weight_min": -28.997859954833984,
      "activations/layer17_attention_weight_max": 29.88470458984375,
      "activations/layer17_attention_weight_min": -27.70302963256836,
      "activations/layer18_attention_weight_max": 34.81559753417969,
      "activations/layer18_attention_weight_min": -26.452917098999023,
      "activations/layer19_attention_weight_max": 30.805753707885742,
      "activations/layer19_attention_weight_min": -31.4669189453125,
      "activations/layer1_attention_weight_max": 15.866164207458496,
      "activations/layer1_attention_weight_min": -14.362471580505371,
      "activations/layer20_attention_weight_max": 27.100683212280273,
      "activations/layer20_attention_weight_min": -25.712181091308594,
      "activations/layer21_attention_weight_max": 28.4353084564209,
      "activations/layer21_attention_weight_min": -25.935569763183594,
      "activations/layer22_attention_weight_max": 37.82327651977539,
      "activations/layer22_attention_weight_min": -31.325435638427734,
      "activations/layer23_attention_weight_max": 30.89337730407715,
      "activations/layer23_attention_weight_min": -28.169050216674805,
      "activations/layer2_attention_weight_max": 32.74750518798828,
      "activations/layer2_attention_weight_min": -33.370460510253906,
      "activations/layer3_attention_weight_max": 103.59857940673828,
      "activations/layer3_attention_weight_min": -105.24082946777344,
      "activations/layer4_attention_weight_max": 107.41808319091797,
      "activations/layer4_attention_weight_min": -112.31822967529297,
      "activations/layer5_attention_weight_max": 53.12459945678711,
      "activations/layer5_attention_weight_min": -60.43136215209961,
      "activations/layer6_attention_weight_max": 47.44469451904297,
      "activations/layer6_attention_weight_min": -48.131553649902344,
      "activations/layer7_attention_weight_max": 92.35089874267578,
      "activations/layer7_attention_weight_min": -101.17806243896484,
      "activations/layer8_attention_weight_max": 42.31769943237305,
      "activations/layer8_attention_weight_min": -43.871891021728516,
      "activations/layer9_attention_weight_max": 32.96357727050781,
      "activations/layer9_attention_weight_min": -34.24152755737305,
      "epoch": 20.83,
      "learning_rate": 1.580681818181818e-05,
      "loss": 2.7145,
      "step": 358550
    },
    {
      "activations/layer0_attention_weight_max": 16.438716888427734,
      "activations/layer0_attention_weight_min": -14.467744827270508,
      "activations/layer10_attention_weight_max": 36.436500549316406,
      "activations/layer10_attention_weight_min": -37.03944396972656,
      "activations/layer11_attention_weight_max": 35.665283203125,
      "activations/layer11_attention_weight_min": -36.51788330078125,
      "activations/layer12_attention_weight_max": 29.896095275878906,
      "activations/layer12_attention_weight_min": -27.92959976196289,
      "activations/layer13_attention_weight_max": 43.34340286254883,
      "activations/layer13_attention_weight_min": -37.900489807128906,
      "activations/layer14_attention_weight_max": 44.7696647644043,
      "activations/layer14_attention_weight_min": -38.13328552246094,
      "activations/layer15_attention_weight_max": 42.97926712036133,
      "activations/layer15_attention_weight_min": -37.03033447265625,
      "activations/layer16_attention_weight_max": 33.86676788330078,
      "activations/layer16_attention_weight_min": -28.3194637298584,
      "activations/layer17_attention_weight_max": 30.21501350402832,
      "activations/layer17_attention_weight_min": -27.247751235961914,
      "activations/layer18_attention_weight_max": 32.190765380859375,
      "activations/layer18_attention_weight_min": -26.802101135253906,
      "activations/layer19_attention_weight_max": 34.44770431518555,
      "activations/layer19_attention_weight_min": -31.793903350830078,
      "activations/layer1_attention_weight_max": 15.920256614685059,
      "activations/layer1_attention_weight_min": -14.186415672302246,
      "activations/layer20_attention_weight_max": 26.95524787902832,
      "activations/layer20_attention_weight_min": -25.733320236206055,
      "activations/layer21_attention_weight_max": 27.856552124023438,
      "activations/layer21_attention_weight_min": -26.022628784179688,
      "activations/layer22_attention_weight_max": 38.7269401550293,
      "activations/layer22_attention_weight_min": -33.001441955566406,
      "activations/layer23_attention_weight_max": 31.99429702758789,
      "activations/layer23_attention_weight_min": -26.732763290405273,
      "activations/layer2_attention_weight_max": 33.3469123840332,
      "activations/layer2_attention_weight_min": -32.47025680541992,
      "activations/layer3_attention_weight_max": 107.56751251220703,
      "activations/layer3_attention_weight_min": -103.9952392578125,
      "activations/layer4_attention_weight_max": 112.29766082763672,
      "activations/layer4_attention_weight_min": -109.99481201171875,
      "activations/layer5_attention_weight_max": 51.50886535644531,
      "activations/layer5_attention_weight_min": -64.23989868164062,
      "activations/layer6_attention_weight_max": 46.448673248291016,
      "activations/layer6_attention_weight_min": -48.71934127807617,
      "activations/layer7_attention_weight_max": 102.29685974121094,
      "activations/layer7_attention_weight_min": -107.5174331665039,
      "activations/layer8_attention_weight_max": 45.17680740356445,
      "activations/layer8_attention_weight_min": -46.10280227661133,
      "activations/layer9_attention_weight_max": 34.64468002319336,
      "activations/layer9_attention_weight_min": -36.43967819213867,
      "epoch": 20.84,
      "learning_rate": 1.5787878787878787e-05,
      "loss": 2.7161,
      "step": 358600
    },
    {
      "activations/layer0_attention_weight_max": 16.01780128479004,
      "activations/layer0_attention_weight_min": -13.907333374023438,
      "activations/layer10_attention_weight_max": 35.6673583984375,
      "activations/layer10_attention_weight_min": -35.876556396484375,
      "activations/layer11_attention_weight_max": 35.1615104675293,
      "activations/layer11_attention_weight_min": -36.56344985961914,
      "activations/layer12_attention_weight_max": 29.496435165405273,
      "activations/layer12_attention_weight_min": -27.739139556884766,
      "activations/layer13_attention_weight_max": 45.1232795715332,
      "activations/layer13_attention_weight_min": -38.6826171875,
      "activations/layer14_attention_weight_max": 46.49848556518555,
      "activations/layer14_attention_weight_min": -39.46876525878906,
      "activations/layer15_attention_weight_max": 45.50682067871094,
      "activations/layer15_attention_weight_min": -38.10093688964844,
      "activations/layer16_attention_weight_max": 31.136137008666992,
      "activations/layer16_attention_weight_min": -27.522052764892578,
      "activations/layer17_attention_weight_max": 32.762428283691406,
      "activations/layer17_attention_weight_min": -24.900205612182617,
      "activations/layer18_attention_weight_max": 34.050819396972656,
      "activations/layer18_attention_weight_min": -25.77091407775879,
      "activations/layer19_attention_weight_max": 34.060203552246094,
      "activations/layer19_attention_weight_min": -30.86172866821289,
      "activations/layer1_attention_weight_max": 16.000267028808594,
      "activations/layer1_attention_weight_min": -15.05263614654541,
      "activations/layer20_attention_weight_max": 31.75615119934082,
      "activations/layer20_attention_weight_min": -24.465099334716797,
      "activations/layer21_attention_weight_max": 31.481908798217773,
      "activations/layer21_attention_weight_min": -24.992101669311523,
      "activations/layer22_attention_weight_max": 43.206398010253906,
      "activations/layer22_attention_weight_min": -31.486705780029297,
      "activations/layer23_attention_weight_max": 32.28281784057617,
      "activations/layer23_attention_weight_min": -24.912982940673828,
      "activations/layer2_attention_weight_max": 33.264793395996094,
      "activations/layer2_attention_weight_min": -32.86372375488281,
      "activations/layer3_attention_weight_max": 103.74565124511719,
      "activations/layer3_attention_weight_min": -104.67018127441406,
      "activations/layer4_attention_weight_max": 109.4251480102539,
      "activations/layer4_attention_weight_min": -112.24071502685547,
      "activations/layer5_attention_weight_max": 50.01038360595703,
      "activations/layer5_attention_weight_min": -63.1195068359375,
      "activations/layer6_attention_weight_max": 46.00337219238281,
      "activations/layer6_attention_weight_min": -47.1662712097168,
      "activations/layer7_attention_weight_max": 101.53936004638672,
      "activations/layer7_attention_weight_min": -99.54032897949219,
      "activations/layer8_attention_weight_max": 43.43831253051758,
      "activations/layer8_attention_weight_min": -45.790470123291016,
      "activations/layer9_attention_weight_max": 32.58817672729492,
      "activations/layer9_attention_weight_min": -36.24455642700195,
      "epoch": 20.84,
      "learning_rate": 1.5768939393939392e-05,
      "loss": 2.7058,
      "step": 358650
    },
    {
      "activations/layer0_attention_weight_max": 17.092453002929688,
      "activations/layer0_attention_weight_min": -13.893844604492188,
      "activations/layer10_attention_weight_max": 36.96855926513672,
      "activations/layer10_attention_weight_min": -37.89942169189453,
      "activations/layer11_attention_weight_max": 35.6342658996582,
      "activations/layer11_attention_weight_min": -36.67824935913086,
      "activations/layer12_attention_weight_max": 28.758132934570312,
      "activations/layer12_attention_weight_min": -30.76375961303711,
      "activations/layer13_attention_weight_max": 41.18970489501953,
      "activations/layer13_attention_weight_min": -41.20951461791992,
      "activations/layer14_attention_weight_max": 44.23704147338867,
      "activations/layer14_attention_weight_min": -43.55250549316406,
      "activations/layer15_attention_weight_max": 40.23505401611328,
      "activations/layer15_attention_weight_min": -39.094242095947266,
      "activations/layer16_attention_weight_max": 30.253894805908203,
      "activations/layer16_attention_weight_min": -30.478654861450195,
      "activations/layer17_attention_weight_max": 31.111751556396484,
      "activations/layer17_attention_weight_min": -26.833547592163086,
      "activations/layer18_attention_weight_max": 34.81948471069336,
      "activations/layer18_attention_weight_min": -26.586875915527344,
      "activations/layer19_attention_weight_max": 35.351829528808594,
      "activations/layer19_attention_weight_min": -33.96685791015625,
      "activations/layer1_attention_weight_max": 16.114213943481445,
      "activations/layer1_attention_weight_min": -14.690030097961426,
      "activations/layer20_attention_weight_max": 29.594526290893555,
      "activations/layer20_attention_weight_min": -29.04800033569336,
      "activations/layer21_attention_weight_max": 32.37894058227539,
      "activations/layer21_attention_weight_min": -26.657421112060547,
      "activations/layer22_attention_weight_max": 46.23308563232422,
      "activations/layer22_attention_weight_min": -32.787986755371094,
      "activations/layer23_attention_weight_max": 37.92136001586914,
      "activations/layer23_attention_weight_min": -27.80354118347168,
      "activations/layer2_attention_weight_max": 33.903228759765625,
      "activations/layer2_attention_weight_min": -34.401641845703125,
      "activations/layer3_attention_weight_max": 102.690673828125,
      "activations/layer3_attention_weight_min": -106.80572509765625,
      "activations/layer4_attention_weight_max": 108.38130950927734,
      "activations/layer4_attention_weight_min": -110.97430419921875,
      "activations/layer5_attention_weight_max": 52.216243743896484,
      "activations/layer5_attention_weight_min": -61.62226486206055,
      "activations/layer6_attention_weight_max": 45.6446647644043,
      "activations/layer6_attention_weight_min": -48.269832611083984,
      "activations/layer7_attention_weight_max": 102.72013092041016,
      "activations/layer7_attention_weight_min": -102.04627227783203,
      "activations/layer8_attention_weight_max": 43.52028274536133,
      "activations/layer8_attention_weight_min": -47.66221237182617,
      "activations/layer9_attention_weight_max": 34.05154037475586,
      "activations/layer9_attention_weight_min": -37.572322845458984,
      "epoch": 20.84,
      "learning_rate": 1.5749999999999997e-05,
      "loss": 2.7203,
      "step": 358700
    },
    {
      "activations/layer0_attention_weight_max": 16.047433853149414,
      "activations/layer0_attention_weight_min": -15.608141899108887,
      "activations/layer10_attention_weight_max": 36.38882064819336,
      "activations/layer10_attention_weight_min": -34.9644660949707,
      "activations/layer11_attention_weight_max": 34.69913864135742,
      "activations/layer11_attention_weight_min": -34.956111907958984,
      "activations/layer12_attention_weight_max": 27.26961898803711,
      "activations/layer12_attention_weight_min": -26.498897552490234,
      "activations/layer13_attention_weight_max": 41.38459396362305,
      "activations/layer13_attention_weight_min": -34.852439880371094,
      "activations/layer14_attention_weight_max": 41.4869270324707,
      "activations/layer14_attention_weight_min": -37.92569351196289,
      "activations/layer15_attention_weight_max": 40.68037033081055,
      "activations/layer15_attention_weight_min": -35.38698959350586,
      "activations/layer16_attention_weight_max": 28.67159080505371,
      "activations/layer16_attention_weight_min": -28.574033737182617,
      "activations/layer17_attention_weight_max": 30.61812400817871,
      "activations/layer17_attention_weight_min": -26.221105575561523,
      "activations/layer18_attention_weight_max": 33.78008270263672,
      "activations/layer18_attention_weight_min": -28.65141487121582,
      "activations/layer19_attention_weight_max": 36.432098388671875,
      "activations/layer19_attention_weight_min": -32.3831901550293,
      "activations/layer1_attention_weight_max": 15.081964492797852,
      "activations/layer1_attention_weight_min": -14.669072151184082,
      "activations/layer20_attention_weight_max": 31.088228225708008,
      "activations/layer20_attention_weight_min": -25.809532165527344,
      "activations/layer21_attention_weight_max": 30.17144012451172,
      "activations/layer21_attention_weight_min": -26.925048828125,
      "activations/layer22_attention_weight_max": 41.42881393432617,
      "activations/layer22_attention_weight_min": -32.36805725097656,
      "activations/layer23_attention_weight_max": 30.25261688232422,
      "activations/layer23_attention_weight_min": -27.403228759765625,
      "activations/layer2_attention_weight_max": 33.41799545288086,
      "activations/layer2_attention_weight_min": -32.91263198852539,
      "activations/layer3_attention_weight_max": 102.18878936767578,
      "activations/layer3_attention_weight_min": -102.40927124023438,
      "activations/layer4_attention_weight_max": 107.41873931884766,
      "activations/layer4_attention_weight_min": -107.4266586303711,
      "activations/layer5_attention_weight_max": 52.01744842529297,
      "activations/layer5_attention_weight_min": -67.3912353515625,
      "activations/layer6_attention_weight_max": 43.697227478027344,
      "activations/layer6_attention_weight_min": -46.53414535522461,
      "activations/layer7_attention_weight_max": 92.17547607421875,
      "activations/layer7_attention_weight_min": -95.67161560058594,
      "activations/layer8_attention_weight_max": 41.03107452392578,
      "activations/layer8_attention_weight_min": -45.27580261230469,
      "activations/layer9_attention_weight_max": 32.14307403564453,
      "activations/layer9_attention_weight_min": -34.455909729003906,
      "epoch": 20.85,
      "learning_rate": 1.5731060606060605e-05,
      "loss": 2.7167,
      "step": 358750
    },
    {
      "activations/layer0_attention_weight_max": 15.75955581665039,
      "activations/layer0_attention_weight_min": -14.650627136230469,
      "activations/layer10_attention_weight_max": 32.47205352783203,
      "activations/layer10_attention_weight_min": -35.2166748046875,
      "activations/layer11_attention_weight_max": 33.172855377197266,
      "activations/layer11_attention_weight_min": -34.82155990600586,
      "activations/layer12_attention_weight_max": 25.37004280090332,
      "activations/layer12_attention_weight_min": -29.039817810058594,
      "activations/layer13_attention_weight_max": 37.85203170776367,
      "activations/layer13_attention_weight_min": -37.654685974121094,
      "activations/layer14_attention_weight_max": 41.338661193847656,
      "activations/layer14_attention_weight_min": -41.91481399536133,
      "activations/layer15_attention_weight_max": 37.75840759277344,
      "activations/layer15_attention_weight_min": -37.43418884277344,
      "activations/layer16_attention_weight_max": 29.621912002563477,
      "activations/layer16_attention_weight_min": -32.10132598876953,
      "activations/layer17_attention_weight_max": 28.9338321685791,
      "activations/layer17_attention_weight_min": -28.270097732543945,
      "activations/layer18_attention_weight_max": 30.622005462646484,
      "activations/layer18_attention_weight_min": -29.548906326293945,
      "activations/layer19_attention_weight_max": 34.26755142211914,
      "activations/layer19_attention_weight_min": -33.184181213378906,
      "activations/layer1_attention_weight_max": 16.13347053527832,
      "activations/layer1_attention_weight_min": -14.81722640991211,
      "activations/layer20_attention_weight_max": 28.087095260620117,
      "activations/layer20_attention_weight_min": -28.44194984436035,
      "activations/layer21_attention_weight_max": 28.65597152709961,
      "activations/layer21_attention_weight_min": -28.727447509765625,
      "activations/layer22_attention_weight_max": 38.43379211425781,
      "activations/layer22_attention_weight_min": -34.94705581665039,
      "activations/layer23_attention_weight_max": 29.581321716308594,
      "activations/layer23_attention_weight_min": -26.41867446899414,
      "activations/layer2_attention_weight_max": 35.23356628417969,
      "activations/layer2_attention_weight_min": -33.359962463378906,
      "activations/layer3_attention_weight_max": 104.89960479736328,
      "activations/layer3_attention_weight_min": -104.68402099609375,
      "activations/layer4_attention_weight_max": 109.867431640625,
      "activations/layer4_attention_weight_min": -108.83853912353516,
      "activations/layer5_attention_weight_max": 50.83695602416992,
      "activations/layer5_attention_weight_min": -59.76597213745117,
      "activations/layer6_attention_weight_max": 45.32380676269531,
      "activations/layer6_attention_weight_min": -47.88869857788086,
      "activations/layer7_attention_weight_max": 94.32503509521484,
      "activations/layer7_attention_weight_min": -99.96601104736328,
      "activations/layer8_attention_weight_max": 41.5584602355957,
      "activations/layer8_attention_weight_min": -46.03962707519531,
      "activations/layer9_attention_weight_max": 30.12442398071289,
      "activations/layer9_attention_weight_min": -35.125244140625,
      "epoch": 20.85,
      "learning_rate": 1.57125e-05,
      "loss": 2.718,
      "step": 358800
    },
    {
      "activations/layer0_attention_weight_max": 18.091312408447266,
      "activations/layer0_attention_weight_min": -15.616670608520508,
      "activations/layer10_attention_weight_max": 37.88594055175781,
      "activations/layer10_attention_weight_min": -40.52644348144531,
      "activations/layer11_attention_weight_max": 37.2784538269043,
      "activations/layer11_attention_weight_min": -39.1622314453125,
      "activations/layer12_attention_weight_max": 26.4288387298584,
      "activations/layer12_attention_weight_min": -29.718769073486328,
      "activations/layer13_attention_weight_max": 40.540626525878906,
      "activations/layer13_attention_weight_min": -41.61737823486328,
      "activations/layer14_attention_weight_max": 45.309471130371094,
      "activations/layer14_attention_weight_min": -43.03002166748047,
      "activations/layer15_attention_weight_max": 41.51866149902344,
      "activations/layer15_attention_weight_min": -42.33333969116211,
      "activations/layer16_attention_weight_max": 30.265897750854492,
      "activations/layer16_attention_weight_min": -29.150039672851562,
      "activations/layer17_attention_weight_max": 29.444881439208984,
      "activations/layer17_attention_weight_min": -26.58433723449707,
      "activations/layer18_attention_weight_max": 31.34730339050293,
      "activations/layer18_attention_weight_min": -29.892745971679688,
      "activations/layer19_attention_weight_max": 34.13398361206055,
      "activations/layer19_attention_weight_min": -33.25264358520508,
      "activations/layer1_attention_weight_max": 16.455873489379883,
      "activations/layer1_attention_weight_min": -14.930378913879395,
      "activations/layer20_attention_weight_max": 27.620922088623047,
      "activations/layer20_attention_weight_min": -25.71380043029785,
      "activations/layer21_attention_weight_max": 26.749168395996094,
      "activations/layer21_attention_weight_min": -25.882272720336914,
      "activations/layer22_attention_weight_max": 46.57542419433594,
      "activations/layer22_attention_weight_min": -34.70266342163086,
      "activations/layer23_attention_weight_max": 30.978750228881836,
      "activations/layer23_attention_weight_min": -27.810293197631836,
      "activations/layer2_attention_weight_max": 33.8125,
      "activations/layer2_attention_weight_min": -32.9183464050293,
      "activations/layer3_attention_weight_max": 106.92269134521484,
      "activations/layer3_attention_weight_min": -105.00228118896484,
      "activations/layer4_attention_weight_max": 111.8699722290039,
      "activations/layer4_attention_weight_min": -107.48567962646484,
      "activations/layer5_attention_weight_max": 51.641502380371094,
      "activations/layer5_attention_weight_min": -61.312679290771484,
      "activations/layer6_attention_weight_max": 46.26791763305664,
      "activations/layer6_attention_weight_min": -48.291324615478516,
      "activations/layer7_attention_weight_max": 99.24664306640625,
      "activations/layer7_attention_weight_min": -97.89920806884766,
      "activations/layer8_attention_weight_max": 46.62467956542969,
      "activations/layer8_attention_weight_min": -51.27827835083008,
      "activations/layer9_attention_weight_max": 36.053184509277344,
      "activations/layer9_attention_weight_min": -40.56138229370117,
      "epoch": 20.85,
      "learning_rate": 1.5693560606060604e-05,
      "loss": 2.7081,
      "step": 358850
    },
    {
      "activations/layer0_attention_weight_max": 17.915407180786133,
      "activations/layer0_attention_weight_min": -13.728775024414062,
      "activations/layer10_attention_weight_max": 33.03077697753906,
      "activations/layer10_attention_weight_min": -34.721153259277344,
      "activations/layer11_attention_weight_max": 33.14406967163086,
      "activations/layer11_attention_weight_min": -33.598480224609375,
      "activations/layer12_attention_weight_max": 27.75411605834961,
      "activations/layer12_attention_weight_min": -27.525712966918945,
      "activations/layer13_attention_weight_max": 37.189208984375,
      "activations/layer13_attention_weight_min": -36.9891471862793,
      "activations/layer14_attention_weight_max": 40.48313522338867,
      "activations/layer14_attention_weight_min": -38.907569885253906,
      "activations/layer15_attention_weight_max": 36.437076568603516,
      "activations/layer15_attention_weight_min": -35.30788040161133,
      "activations/layer16_attention_weight_max": 29.615896224975586,
      "activations/layer16_attention_weight_min": -27.55295181274414,
      "activations/layer17_attention_weight_max": 28.661386489868164,
      "activations/layer17_attention_weight_min": -26.667482376098633,
      "activations/layer18_attention_weight_max": 29.211856842041016,
      "activations/layer18_attention_weight_min": -25.64069366455078,
      "activations/layer19_attention_weight_max": 32.31655502319336,
      "activations/layer19_attention_weight_min": -30.794498443603516,
      "activations/layer1_attention_weight_max": 15.965964317321777,
      "activations/layer1_attention_weight_min": -14.091541290283203,
      "activations/layer20_attention_weight_max": 29.611988067626953,
      "activations/layer20_attention_weight_min": -25.4665584564209,
      "activations/layer21_attention_weight_max": 28.77813720703125,
      "activations/layer21_attention_weight_min": -24.584487915039062,
      "activations/layer22_attention_weight_max": 38.97718811035156,
      "activations/layer22_attention_weight_min": -31.33080291748047,
      "activations/layer23_attention_weight_max": 28.6990966796875,
      "activations/layer23_attention_weight_min": -26.822200775146484,
      "activations/layer2_attention_weight_max": 33.11358642578125,
      "activations/layer2_attention_weight_min": -32.13591766357422,
      "activations/layer3_attention_weight_max": 105.37062072753906,
      "activations/layer3_attention_weight_min": -107.69561004638672,
      "activations/layer4_attention_weight_max": 107.58595275878906,
      "activations/layer4_attention_weight_min": -111.54833984375,
      "activations/layer5_attention_weight_max": 49.587669372558594,
      "activations/layer5_attention_weight_min": -65.53299713134766,
      "activations/layer6_attention_weight_max": 45.36436080932617,
      "activations/layer6_attention_weight_min": -47.365909576416016,
      "activations/layer7_attention_weight_max": 93.78627014160156,
      "activations/layer7_attention_weight_min": -97.29164123535156,
      "activations/layer8_attention_weight_max": 41.332000732421875,
      "activations/layer8_attention_weight_min": -43.6160774230957,
      "activations/layer9_attention_weight_max": 31.006277084350586,
      "activations/layer9_attention_weight_min": -34.23188400268555,
      "epoch": 20.85,
      "learning_rate": 1.567462121212121e-05,
      "loss": 2.7197,
      "step": 358900
    },
    {
      "activations/layer0_attention_weight_max": 16.771081924438477,
      "activations/layer0_attention_weight_min": -14.732939720153809,
      "activations/layer10_attention_weight_max": 35.05448532104492,
      "activations/layer10_attention_weight_min": -36.89168167114258,
      "activations/layer11_attention_weight_max": 36.14091873168945,
      "activations/layer11_attention_weight_min": -35.460105895996094,
      "activations/layer12_attention_weight_max": 28.598201751708984,
      "activations/layer12_attention_weight_min": -27.59947395324707,
      "activations/layer13_attention_weight_max": 41.62516403198242,
      "activations/layer13_attention_weight_min": -35.70675277709961,
      "activations/layer14_attention_weight_max": 45.60482406616211,
      "activations/layer14_attention_weight_min": -38.24063491821289,
      "activations/layer15_attention_weight_max": 42.60627365112305,
      "activations/layer15_attention_weight_min": -37.19089889526367,
      "activations/layer16_attention_weight_max": 31.670528411865234,
      "activations/layer16_attention_weight_min": -29.876543045043945,
      "activations/layer17_attention_weight_max": 33.42628860473633,
      "activations/layer17_attention_weight_min": -29.446807861328125,
      "activations/layer18_attention_weight_max": 33.94171905517578,
      "activations/layer18_attention_weight_min": -31.331151962280273,
      "activations/layer19_attention_weight_max": 35.7742805480957,
      "activations/layer19_attention_weight_min": -35.207984924316406,
      "activations/layer1_attention_weight_max": 15.770299911499023,
      "activations/layer1_attention_weight_min": -13.950342178344727,
      "activations/layer20_attention_weight_max": 30.897300720214844,
      "activations/layer20_attention_weight_min": -28.79606819152832,
      "activations/layer21_attention_weight_max": 29.8170166015625,
      "activations/layer21_attention_weight_min": -30.722061157226562,
      "activations/layer22_attention_weight_max": 43.44427490234375,
      "activations/layer22_attention_weight_min": -41.69527816772461,
      "activations/layer23_attention_weight_max": 33.61952209472656,
      "activations/layer23_attention_weight_min": -30.95977020263672,
      "activations/layer2_attention_weight_max": 33.67681884765625,
      "activations/layer2_attention_weight_min": -33.35553741455078,
      "activations/layer3_attention_weight_max": 108.15750885009766,
      "activations/layer3_attention_weight_min": -105.03960418701172,
      "activations/layer4_attention_weight_max": 112.7910385131836,
      "activations/layer4_attention_weight_min": -112.72896575927734,
      "activations/layer5_attention_weight_max": 50.24094772338867,
      "activations/layer5_attention_weight_min": -63.79895782470703,
      "activations/layer6_attention_weight_max": 45.25178146362305,
      "activations/layer6_attention_weight_min": -49.430912017822266,
      "activations/layer7_attention_weight_max": 97.24700927734375,
      "activations/layer7_attention_weight_min": -105.89756774902344,
      "activations/layer8_attention_weight_max": 41.78884506225586,
      "activations/layer8_attention_weight_min": -46.445125579833984,
      "activations/layer9_attention_weight_max": 32.78599548339844,
      "activations/layer9_attention_weight_min": -35.553680419921875,
      "epoch": 20.86,
      "learning_rate": 1.5655681818181817e-05,
      "loss": 2.7207,
      "step": 358950
    },
    {
      "activations/layer0_attention_weight_max": 16.397607803344727,
      "activations/layer0_attention_weight_min": -13.79177474975586,
      "activations/layer10_attention_weight_max": 38.05219268798828,
      "activations/layer10_attention_weight_min": -37.349727630615234,
      "activations/layer11_attention_weight_max": 38.201751708984375,
      "activations/layer11_attention_weight_min": -36.71619415283203,
      "activations/layer12_attention_weight_max": 28.138216018676758,
      "activations/layer12_attention_weight_min": -28.114580154418945,
      "activations/layer13_attention_weight_max": 44.47171401977539,
      "activations/layer13_attention_weight_min": -37.27619171142578,
      "activations/layer14_attention_weight_max": 43.670127868652344,
      "activations/layer14_attention_weight_min": -39.18998718261719,
      "activations/layer15_attention_weight_max": 42.53582763671875,
      "activations/layer15_attention_weight_min": -38.495155334472656,
      "activations/layer16_attention_weight_max": 31.8436279296875,
      "activations/layer16_attention_weight_min": -27.65920639038086,
      "activations/layer17_attention_weight_max": 32.92304229736328,
      "activations/layer17_attention_weight_min": -26.525554656982422,
      "activations/layer18_attention_weight_max": 31.323415756225586,
      "activations/layer18_attention_weight_min": -25.70425796508789,
      "activations/layer19_attention_weight_max": 32.44695281982422,
      "activations/layer19_attention_weight_min": -34.579402923583984,
      "activations/layer1_attention_weight_max": 15.921724319458008,
      "activations/layer1_attention_weight_min": -13.98754596710205,
      "activations/layer20_attention_weight_max": 29.009326934814453,
      "activations/layer20_attention_weight_min": -27.5007266998291,
      "activations/layer21_attention_weight_max": 28.620506286621094,
      "activations/layer21_attention_weight_min": -26.536888122558594,
      "activations/layer22_attention_weight_max": 39.910606384277344,
      "activations/layer22_attention_weight_min": -36.09825897216797,
      "activations/layer23_attention_weight_max": 31.8264217376709,
      "activations/layer23_attention_weight_min": -28.13563346862793,
      "activations/layer2_attention_weight_max": 33.67079162597656,
      "activations/layer2_attention_weight_min": -32.50035095214844,
      "activations/layer3_attention_weight_max": 104.10379791259766,
      "activations/layer3_attention_weight_min": -105.04080963134766,
      "activations/layer4_attention_weight_max": 109.145263671875,
      "activations/layer4_attention_weight_min": -108.74364471435547,
      "activations/layer5_attention_weight_max": 48.96233367919922,
      "activations/layer5_attention_weight_min": -62.44892501831055,
      "activations/layer6_attention_weight_max": 44.29326629638672,
      "activations/layer6_attention_weight_min": -47.612281799316406,
      "activations/layer7_attention_weight_max": 97.33238220214844,
      "activations/layer7_attention_weight_min": -99.18411254882812,
      "activations/layer8_attention_weight_max": 42.855873107910156,
      "activations/layer8_attention_weight_min": -47.52735137939453,
      "activations/layer9_attention_weight_max": 36.09921646118164,
      "activations/layer9_attention_weight_min": -34.760555267333984,
      "epoch": 20.86,
      "learning_rate": 1.5636742424242425e-05,
      "loss": 2.7033,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4587,
      "eval_samples_per_second": 507.645,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4587,
      "eval_openwebtext_samples_per_second": 507.645,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9913,
      "eval_wikitext_samples_per_second": 229.0,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_lambada_loss": 2.333984375,
      "eval_lambada_ppl": 10.318974397214378,
      "eval_lambada_runtime": 9.5432,
      "eval_lambada_samples_per_second": 510.206,
      "step": 359000
    },
    {
      "activations/layer0_attention_weight_max": 16.59125518798828,
      "activations/layer0_attention_weight_min": -13.816507339477539,
      "activations/layer10_attention_weight_max": 39.6822395324707,
      "activations/layer10_attention_weight_min": -39.136898040771484,
      "activations/layer11_attention_weight_max": 38.34856414794922,
      "activations/layer11_attention_weight_min": -36.205047607421875,
      "activations/layer12_attention_weight_max": 33.06806945800781,
      "activations/layer12_attention_weight_min": -29.006446838378906,
      "activations/layer13_attention_weight_max": 46.79732131958008,
      "activations/layer13_attention_weight_min": -37.83237838745117,
      "activations/layer14_attention_weight_max": 49.723331451416016,
      "activations/layer14_attention_weight_min": -40.11842346191406,
      "activations/layer15_attention_weight_max": 46.50531768798828,
      "activations/layer15_attention_weight_min": -37.612030029296875,
      "activations/layer16_attention_weight_max": 34.31633377075195,
      "activations/layer16_attention_weight_min": -29.44023323059082,
      "activations/layer17_attention_weight_max": 32.93771743774414,
      "activations/layer17_attention_weight_min": -27.032915115356445,
      "activations/layer18_attention_weight_max": 33.02745819091797,
      "activations/layer18_attention_weight_min": -26.68784523010254,
      "activations/layer19_attention_weight_max": 39.15591049194336,
      "activations/layer19_attention_weight_min": -34.19052505493164,
      "activations/layer1_attention_weight_max": 15.649794578552246,
      "activations/layer1_attention_weight_min": -16.18522071838379,
      "activations/layer20_attention_weight_max": 30.67458152770996,
      "activations/layer20_attention_weight_min": -27.644662857055664,
      "activations/layer21_attention_weight_max": 29.49175262451172,
      "activations/layer21_attention_weight_min": -25.23024559020996,
      "activations/layer22_attention_weight_max": 47.28055953979492,
      "activations/layer22_attention_weight_min": -31.803464889526367,
      "activations/layer23_attention_weight_max": 37.006080627441406,
      "activations/layer23_attention_weight_min": -28.439239501953125,
      "activations/layer2_attention_weight_max": 35.40188980102539,
      "activations/layer2_attention_weight_min": -34.41059494018555,
      "activations/layer3_attention_weight_max": 108.42216491699219,
      "activations/layer3_attention_weight_min": -107.8456802368164,
      "activations/layer4_attention_weight_max": 109.4894027709961,
      "activations/layer4_attention_weight_min": -107.79598236083984,
      "activations/layer5_attention_weight_max": 53.144676208496094,
      "activations/layer5_attention_weight_min": -62.21588134765625,
      "activations/layer6_attention_weight_max": 48.69808578491211,
      "activations/layer6_attention_weight_min": -50.23424530029297,
      "activations/layer7_attention_weight_max": 101.75826263427734,
      "activations/layer7_attention_weight_min": -105.63299560546875,
      "activations/layer8_attention_weight_max": 49.626338958740234,
      "activations/layer8_attention_weight_min": -49.998538970947266,
      "activations/layer9_attention_weight_max": 36.73127365112305,
      "activations/layer9_attention_weight_min": -39.8508415222168,
      "epoch": 20.86,
      "learning_rate": 1.561780303030303e-05,
      "loss": 2.7432,
      "step": 359050
    },
    {
      "activations/layer0_attention_weight_max": 17.320690155029297,
      "activations/layer0_attention_weight_min": -13.488985061645508,
      "activations/layer10_attention_weight_max": 38.73746871948242,
      "activations/layer10_attention_weight_min": -38.38655090332031,
      "activations/layer11_attention_weight_max": 37.780479431152344,
      "activations/layer11_attention_weight_min": -37.01414489746094,
      "activations/layer12_attention_weight_max": 28.160293579101562,
      "activations/layer12_attention_weight_min": -27.90107536315918,
      "activations/layer13_attention_weight_max": 40.94704055786133,
      "activations/layer13_attention_weight_min": -36.00181198120117,
      "activations/layer14_attention_weight_max": 45.598121643066406,
      "activations/layer14_attention_weight_min": -38.16078186035156,
      "activations/layer15_attention_weight_max": 40.8581657409668,
      "activations/layer15_attention_weight_min": -37.102352142333984,
      "activations/layer16_attention_weight_max": 31.396926879882812,
      "activations/layer16_attention_weight_min": -29.766355514526367,
      "activations/layer17_attention_weight_max": 29.336374282836914,
      "activations/layer17_attention_weight_min": -25.577083587646484,
      "activations/layer18_attention_weight_max": 32.29526901245117,
      "activations/layer18_attention_weight_min": -24.099397659301758,
      "activations/layer19_attention_weight_max": 33.14122009277344,
      "activations/layer19_attention_weight_min": -29.738691329956055,
      "activations/layer1_attention_weight_max": 15.987154006958008,
      "activations/layer1_attention_weight_min": -16.251920700073242,
      "activations/layer20_attention_weight_max": 29.255380630493164,
      "activations/layer20_attention_weight_min": -24.258960723876953,
      "activations/layer21_attention_weight_max": 29.41884994506836,
      "activations/layer21_attention_weight_min": -24.796642303466797,
      "activations/layer22_attention_weight_max": 41.38440704345703,
      "activations/layer22_attention_weight_min": -30.69257164001465,
      "activations/layer23_attention_weight_max": 31.369314193725586,
      "activations/layer23_attention_weight_min": -25.38094711303711,
      "activations/layer2_attention_weight_max": 34.03154373168945,
      "activations/layer2_attention_weight_min": -33.279293060302734,
      "activations/layer3_attention_weight_max": 104.00353240966797,
      "activations/layer3_attention_weight_min": -112.57421875,
      "activations/layer4_attention_weight_max": 109.88215637207031,
      "activations/layer4_attention_weight_min": -114.6812515258789,
      "activations/layer5_attention_weight_max": 50.25434494018555,
      "activations/layer5_attention_weight_min": -63.1541748046875,
      "activations/layer6_attention_weight_max": 46.48395919799805,
      "activations/layer6_attention_weight_min": -50.95476531982422,
      "activations/layer7_attention_weight_max": 98.82865905761719,
      "activations/layer7_attention_weight_min": -111.13761901855469,
      "activations/layer8_attention_weight_max": 44.709228515625,
      "activations/layer8_attention_weight_min": -48.4927864074707,
      "activations/layer9_attention_weight_max": 36.91362762451172,
      "activations/layer9_attention_weight_min": -37.74289321899414,
      "epoch": 20.87,
      "learning_rate": 1.5598863636363635e-05,
      "loss": 2.7114,
      "step": 359100
    },
    {
      "activations/layer0_attention_weight_max": 17.732580184936523,
      "activations/layer0_attention_weight_min": -14.871329307556152,
      "activations/layer10_attention_weight_max": 38.04745101928711,
      "activations/layer10_attention_weight_min": -35.7387580871582,
      "activations/layer11_attention_weight_max": 35.01245880126953,
      "activations/layer11_attention_weight_min": -34.58673858642578,
      "activations/layer12_attention_weight_max": 25.856164932250977,
      "activations/layer12_attention_weight_min": -26.386106491088867,
      "activations/layer13_attention_weight_max": 38.94966125488281,
      "activations/layer13_attention_weight_min": -36.26704025268555,
      "activations/layer14_attention_weight_max": 43.52473831176758,
      "activations/layer14_attention_weight_min": -36.53508758544922,
      "activations/layer15_attention_weight_max": 40.470890045166016,
      "activations/layer15_attention_weight_min": -37.8345832824707,
      "activations/layer16_attention_weight_max": 29.567180633544922,
      "activations/layer16_attention_weight_min": -28.183069229125977,
      "activations/layer17_attention_weight_max": 33.47910690307617,
      "activations/layer17_attention_weight_min": -25.82028579711914,
      "activations/layer18_attention_weight_max": 34.76530456542969,
      "activations/layer18_attention_weight_min": -26.729734420776367,
      "activations/layer19_attention_weight_max": 34.8934440612793,
      "activations/layer19_attention_weight_min": -30.64103126525879,
      "activations/layer1_attention_weight_max": 16.353919982910156,
      "activations/layer1_attention_weight_min": -14.818297386169434,
      "activations/layer20_attention_weight_max": 30.727739334106445,
      "activations/layer20_attention_weight_min": -26.004283905029297,
      "activations/layer21_attention_weight_max": 29.951396942138672,
      "activations/layer21_attention_weight_min": -25.067296981811523,
      "activations/layer22_attention_weight_max": 40.45256805419922,
      "activations/layer22_attention_weight_min": -31.67049789428711,
      "activations/layer23_attention_weight_max": 31.248111724853516,
      "activations/layer23_attention_weight_min": -25.175025939941406,
      "activations/layer2_attention_weight_max": 33.512901306152344,
      "activations/layer2_attention_weight_min": -32.39832305908203,
      "activations/layer3_attention_weight_max": 105.31748962402344,
      "activations/layer3_attention_weight_min": -106.07746124267578,
      "activations/layer4_attention_weight_max": 109.33995819091797,
      "activations/layer4_attention_weight_min": -112.4045639038086,
      "activations/layer5_attention_weight_max": 49.98353576660156,
      "activations/layer5_attention_weight_min": -61.22848129272461,
      "activations/layer6_attention_weight_max": 44.44071960449219,
      "activations/layer6_attention_weight_min": -49.07535934448242,
      "activations/layer7_attention_weight_max": 98.26856231689453,
      "activations/layer7_attention_weight_min": -97.32868957519531,
      "activations/layer8_attention_weight_max": 43.2668342590332,
      "activations/layer8_attention_weight_min": -43.33819580078125,
      "activations/layer9_attention_weight_max": 32.59373092651367,
      "activations/layer9_attention_weight_min": -33.750518798828125,
      "epoch": 20.87,
      "learning_rate": 1.557992424242424e-05,
      "loss": 2.7105,
      "step": 359150
    },
    {
      "activations/layer0_attention_weight_max": 16.466489791870117,
      "activations/layer0_attention_weight_min": -13.771998405456543,
      "activations/layer10_attention_weight_max": 45.3807373046875,
      "activations/layer10_attention_weight_min": -40.69952392578125,
      "activations/layer11_attention_weight_max": 46.51591491699219,
      "activations/layer11_attention_weight_min": -43.56382369995117,
      "activations/layer12_attention_weight_max": 30.08716583251953,
      "activations/layer12_attention_weight_min": -29.02787208557129,
      "activations/layer13_attention_weight_max": 44.22407150268555,
      "activations/layer13_attention_weight_min": -37.610687255859375,
      "activations/layer14_attention_weight_max": 48.00934982299805,
      "activations/layer14_attention_weight_min": -42.40348815917969,
      "activations/layer15_attention_weight_max": 45.60067367553711,
      "activations/layer15_attention_weight_min": -41.563507080078125,
      "activations/layer16_attention_weight_max": 32.24750900268555,
      "activations/layer16_attention_weight_min": -28.206621170043945,
      "activations/layer17_attention_weight_max": 31.24074363708496,
      "activations/layer17_attention_weight_min": -26.982254028320312,
      "activations/layer18_attention_weight_max": 32.50572204589844,
      "activations/layer18_attention_weight_min": -26.077144622802734,
      "activations/layer19_attention_weight_max": 36.391014099121094,
      "activations/layer19_attention_weight_min": -31.41701316833496,
      "activations/layer1_attention_weight_max": 17.24394416809082,
      "activations/layer1_attention_weight_min": -15.646651268005371,
      "activations/layer20_attention_weight_max": 27.947513580322266,
      "activations/layer20_attention_weight_min": -25.597423553466797,
      "activations/layer21_attention_weight_max": 27.12489891052246,
      "activations/layer21_attention_weight_min": -25.714488983154297,
      "activations/layer22_attention_weight_max": 38.48847961425781,
      "activations/layer22_attention_weight_min": -31.433313369750977,
      "activations/layer23_attention_weight_max": 32.12085723876953,
      "activations/layer23_attention_weight_min": -25.689023971557617,
      "activations/layer2_attention_weight_max": 37.3073616027832,
      "activations/layer2_attention_weight_min": -35.69380187988281,
      "activations/layer3_attention_weight_max": 109.67335510253906,
      "activations/layer3_attention_weight_min": -110.67713165283203,
      "activations/layer4_attention_weight_max": 107.89013671875,
      "activations/layer4_attention_weight_min": -108.1194076538086,
      "activations/layer5_attention_weight_max": 50.824703216552734,
      "activations/layer5_attention_weight_min": -58.86676025390625,
      "activations/layer6_attention_weight_max": 45.68019104003906,
      "activations/layer6_attention_weight_min": -47.261722564697266,
      "activations/layer7_attention_weight_max": 106.56346893310547,
      "activations/layer7_attention_weight_min": -102.70327758789062,
      "activations/layer8_attention_weight_max": 49.8167610168457,
      "activations/layer8_attention_weight_min": -51.3275260925293,
      "activations/layer9_attention_weight_max": 38.22089767456055,
      "activations/layer9_attention_weight_min": -37.66325759887695,
      "epoch": 20.87,
      "learning_rate": 1.5560984848484848e-05,
      "loss": 2.702,
      "step": 359200
    },
    {
      "activations/layer0_attention_weight_max": 16.35338592529297,
      "activations/layer0_attention_weight_min": -16.252357482910156,
      "activations/layer10_attention_weight_max": 44.84636688232422,
      "activations/layer10_attention_weight_min": -42.23463439941406,
      "activations/layer11_attention_weight_max": 46.01720428466797,
      "activations/layer11_attention_weight_min": -42.52700424194336,
      "activations/layer12_attention_weight_max": 25.334753036499023,
      "activations/layer12_attention_weight_min": -29.220355987548828,
      "activations/layer13_attention_weight_max": 38.2277717590332,
      "activations/layer13_attention_weight_min": -34.357723236083984,
      "activations/layer14_attention_weight_max": 39.484371185302734,
      "activations/layer14_attention_weight_min": -35.37334060668945,
      "activations/layer15_attention_weight_max": 37.1986198425293,
      "activations/layer15_attention_weight_min": -34.97284698486328,
      "activations/layer16_attention_weight_max": 28.294992446899414,
      "activations/layer16_attention_weight_min": -26.788087844848633,
      "activations/layer17_attention_weight_max": 29.054216384887695,
      "activations/layer17_attention_weight_min": -23.996675491333008,
      "activations/layer18_attention_weight_max": 31.058874130249023,
      "activations/layer18_attention_weight_min": -23.821468353271484,
      "activations/layer19_attention_weight_max": 31.268888473510742,
      "activations/layer19_attention_weight_min": -30.212299346923828,
      "activations/layer1_attention_weight_max": 16.1933650970459,
      "activations/layer1_attention_weight_min": -14.88607406616211,
      "activations/layer20_attention_weight_max": 26.163257598876953,
      "activations/layer20_attention_weight_min": -22.792743682861328,
      "activations/layer21_attention_weight_max": 26.977828979492188,
      "activations/layer21_attention_weight_min": -22.37921142578125,
      "activations/layer22_attention_weight_max": 34.657535552978516,
      "activations/layer22_attention_weight_min": -28.31895637512207,
      "activations/layer23_attention_weight_max": 33.9710807800293,
      "activations/layer23_attention_weight_min": -24.939754486083984,
      "activations/layer2_attention_weight_max": 32.82456588745117,
      "activations/layer2_attention_weight_min": -32.856483459472656,
      "activations/layer3_attention_weight_max": 99.91221618652344,
      "activations/layer3_attention_weight_min": -102.95787048339844,
      "activations/layer4_attention_weight_max": 103.5934829711914,
      "activations/layer4_attention_weight_min": -109.43206024169922,
      "activations/layer5_attention_weight_max": 50.941925048828125,
      "activations/layer5_attention_weight_min": -64.67974853515625,
      "activations/layer6_attention_weight_max": 43.91596603393555,
      "activations/layer6_attention_weight_min": -45.86979675292969,
      "activations/layer7_attention_weight_max": 112.36563110351562,
      "activations/layer7_attention_weight_min": -97.98458862304688,
      "activations/layer8_attention_weight_max": 51.48505783081055,
      "activations/layer8_attention_weight_min": -46.415706634521484,
      "activations/layer9_attention_weight_max": 36.080841064453125,
      "activations/layer9_attention_weight_min": -38.62869644165039,
      "epoch": 20.87,
      "learning_rate": 1.5542045454545453e-05,
      "loss": 2.7074,
      "step": 359250
    },
    {
      "activations/layer0_attention_weight_max": 16.209983825683594,
      "activations/layer0_attention_weight_min": -13.847070693969727,
      "activations/layer10_attention_weight_max": 36.09253692626953,
      "activations/layer10_attention_weight_min": -36.00121307373047,
      "activations/layer11_attention_weight_max": 35.93062210083008,
      "activations/layer11_attention_weight_min": -37.235069274902344,
      "activations/layer12_attention_weight_max": 30.070615768432617,
      "activations/layer12_attention_weight_min": -29.421701431274414,
      "activations/layer13_attention_weight_max": 43.12538528442383,
      "activations/layer13_attention_weight_min": -40.162086486816406,
      "activations/layer14_attention_weight_max": 49.130733489990234,
      "activations/layer14_attention_weight_min": -39.543575286865234,
      "activations/layer15_attention_weight_max": 41.4725456237793,
      "activations/layer15_attention_weight_min": -38.05674362182617,
      "activations/layer16_attention_weight_max": 30.999950408935547,
      "activations/layer16_attention_weight_min": -31.08482551574707,
      "activations/layer17_attention_weight_max": 30.942529678344727,
      "activations/layer17_attention_weight_min": -27.286563873291016,
      "activations/layer18_attention_weight_max": 32.590911865234375,
      "activations/layer18_attention_weight_min": -27.2382869720459,
      "activations/layer19_attention_weight_max": 34.88593292236328,
      "activations/layer19_attention_weight_min": -32.908660888671875,
      "activations/layer1_attention_weight_max": 15.984870910644531,
      "activations/layer1_attention_weight_min": -14.449562072753906,
      "activations/layer20_attention_weight_max": 29.180644989013672,
      "activations/layer20_attention_weight_min": -27.963939666748047,
      "activations/layer21_attention_weight_max": 30.7354793548584,
      "activations/layer21_attention_weight_min": -26.239986419677734,
      "activations/layer22_attention_weight_max": 43.04344177246094,
      "activations/layer22_attention_weight_min": -31.68912124633789,
      "activations/layer23_attention_weight_max": 37.10676193237305,
      "activations/layer23_attention_weight_min": -26.4867000579834,
      "activations/layer2_attention_weight_max": 31.671588897705078,
      "activations/layer2_attention_weight_min": -31.053701400756836,
      "activations/layer3_attention_weight_max": 103.32457733154297,
      "activations/layer3_attention_weight_min": -104.52339935302734,
      "activations/layer4_attention_weight_max": 108.75927734375,
      "activations/layer4_attention_weight_min": -114.16094207763672,
      "activations/layer5_attention_weight_max": 51.57530212402344,
      "activations/layer5_attention_weight_min": -64.847900390625,
      "activations/layer6_attention_weight_max": 48.526466369628906,
      "activations/layer6_attention_weight_min": -51.33448791503906,
      "activations/layer7_attention_weight_max": 102.6482925415039,
      "activations/layer7_attention_weight_min": -108.45894622802734,
      "activations/layer8_attention_weight_max": 44.43465042114258,
      "activations/layer8_attention_weight_min": -47.64766311645508,
      "activations/layer9_attention_weight_max": 34.29135513305664,
      "activations/layer9_attention_weight_min": -35.326377868652344,
      "epoch": 20.88,
      "learning_rate": 1.5523106060606058e-05,
      "loss": 2.7029,
      "step": 359300
    },
    {
      "activations/layer0_attention_weight_max": 16.90071678161621,
      "activations/layer0_attention_weight_min": -14.003005981445312,
      "activations/layer10_attention_weight_max": 32.702091217041016,
      "activations/layer10_attention_weight_min": -35.61485290527344,
      "activations/layer11_attention_weight_max": 33.62156295776367,
      "activations/layer11_attention_weight_min": -35.7968635559082,
      "activations/layer12_attention_weight_max": 26.013341903686523,
      "activations/layer12_attention_weight_min": -27.234891891479492,
      "activations/layer13_attention_weight_max": 37.62589645385742,
      "activations/layer13_attention_weight_min": -36.319480895996094,
      "activations/layer14_attention_weight_max": 41.02827453613281,
      "activations/layer14_attention_weight_min": -39.338016510009766,
      "activations/layer15_attention_weight_max": 36.316871643066406,
      "activations/layer15_attention_weight_min": -37.581878662109375,
      "activations/layer16_attention_weight_max": 28.875377655029297,
      "activations/layer16_attention_weight_min": -28.466991424560547,
      "activations/layer17_attention_weight_max": 28.432649612426758,
      "activations/layer17_attention_weight_min": -26.22153663635254,
      "activations/layer18_attention_weight_max": 30.481515884399414,
      "activations/layer18_attention_weight_min": -26.441604614257812,
      "activations/layer19_attention_weight_max": 31.93250846862793,
      "activations/layer19_attention_weight_min": -31.722843170166016,
      "activations/layer1_attention_weight_max": 18.30572509765625,
      "activations/layer1_attention_weight_min": -14.644231796264648,
      "activations/layer20_attention_weight_max": 25.929676055908203,
      "activations/layer20_attention_weight_min": -23.519853591918945,
      "activations/layer21_attention_weight_max": 28.850839614868164,
      "activations/layer21_attention_weight_min": -23.702354431152344,
      "activations/layer22_attention_weight_max": 41.383758544921875,
      "activations/layer22_attention_weight_min": -32.25691223144531,
      "activations/layer23_attention_weight_max": 29.822052001953125,
      "activations/layer23_attention_weight_min": -26.90648651123047,
      "activations/layer2_attention_weight_max": 34.12749099731445,
      "activations/layer2_attention_weight_min": -34.23637771606445,
      "activations/layer3_attention_weight_max": 107.00216674804688,
      "activations/layer3_attention_weight_min": -108.0416488647461,
      "activations/layer4_attention_weight_max": 110.507080078125,
      "activations/layer4_attention_weight_min": -116.33556365966797,
      "activations/layer5_attention_weight_max": 54.15448760986328,
      "activations/layer5_attention_weight_min": -61.40692901611328,
      "activations/layer6_attention_weight_max": 47.12664031982422,
      "activations/layer6_attention_weight_min": -49.734291076660156,
      "activations/layer7_attention_weight_max": 95.9567642211914,
      "activations/layer7_attention_weight_min": -102.28429412841797,
      "activations/layer8_attention_weight_max": 45.30241775512695,
      "activations/layer8_attention_weight_min": -46.84420394897461,
      "activations/layer9_attention_weight_max": 32.16563415527344,
      "activations/layer9_attention_weight_min": -35.289634704589844,
      "epoch": 20.88,
      "learning_rate": 1.5504166666666666e-05,
      "loss": 2.7202,
      "step": 359350
    },
    {
      "activations/layer0_attention_weight_max": 17.243070602416992,
      "activations/layer0_attention_weight_min": -13.682242393493652,
      "activations/layer10_attention_weight_max": 36.8513069152832,
      "activations/layer10_attention_weight_min": -34.6591796875,
      "activations/layer11_attention_weight_max": 34.76838684082031,
      "activations/layer11_attention_weight_min": -35.858360290527344,
      "activations/layer12_attention_weight_max": 26.344100952148438,
      "activations/layer12_attention_weight_min": -26.937461853027344,
      "activations/layer13_attention_weight_max": 41.6043815612793,
      "activations/layer13_attention_weight_min": -37.6805419921875,
      "activations/layer14_attention_weight_max": 43.711097717285156,
      "activations/layer14_attention_weight_min": -39.00858688354492,
      "activations/layer15_attention_weight_max": 41.76953125,
      "activations/layer15_attention_weight_min": -36.04715347290039,
      "activations/layer16_attention_weight_max": 28.545856475830078,
      "activations/layer16_attention_weight_min": -28.27964973449707,
      "activations/layer17_attention_weight_max": 32.858219146728516,
      "activations/layer17_attention_weight_min": -26.381071090698242,
      "activations/layer18_attention_weight_max": 31.707304000854492,
      "activations/layer18_attention_weight_min": -25.457868576049805,
      "activations/layer19_attention_weight_max": 35.407955169677734,
      "activations/layer19_attention_weight_min": -31.452468872070312,
      "activations/layer1_attention_weight_max": 16.17296600341797,
      "activations/layer1_attention_weight_min": -14.23069953918457,
      "activations/layer20_attention_weight_max": 28.308671951293945,
      "activations/layer20_attention_weight_min": -26.198762893676758,
      "activations/layer21_attention_weight_max": 28.239444732666016,
      "activations/layer21_attention_weight_min": -25.31459617614746,
      "activations/layer22_attention_weight_max": 39.95114517211914,
      "activations/layer22_attention_weight_min": -33.2260627746582,
      "activations/layer23_attention_weight_max": 32.60297393798828,
      "activations/layer23_attention_weight_min": -25.136577606201172,
      "activations/layer2_attention_weight_max": 34.699424743652344,
      "activations/layer2_attention_weight_min": -33.21446990966797,
      "activations/layer3_attention_weight_max": 107.62027740478516,
      "activations/layer3_attention_weight_min": -108.38133239746094,
      "activations/layer4_attention_weight_max": 111.02306365966797,
      "activations/layer4_attention_weight_min": -115.3196792602539,
      "activations/layer5_attention_weight_max": 52.39911651611328,
      "activations/layer5_attention_weight_min": -64.21663665771484,
      "activations/layer6_attention_weight_max": 45.15668869018555,
      "activations/layer6_attention_weight_min": -47.06088638305664,
      "activations/layer7_attention_weight_max": 93.10363006591797,
      "activations/layer7_attention_weight_min": -95.9575424194336,
      "activations/layer8_attention_weight_max": 44.43313980102539,
      "activations/layer8_attention_weight_min": -44.11875534057617,
      "activations/layer9_attention_weight_max": 32.93195343017578,
      "activations/layer9_attention_weight_min": -35.02284240722656,
      "epoch": 20.88,
      "learning_rate": 1.548522727272727e-05,
      "loss": 2.7157,
      "step": 359400
    },
    {
      "activations/layer0_attention_weight_max": 16.571073532104492,
      "activations/layer0_attention_weight_min": -14.249070167541504,
      "activations/layer10_attention_weight_max": 36.528038024902344,
      "activations/layer10_attention_weight_min": -36.72819900512695,
      "activations/layer11_attention_weight_max": 35.641849517822266,
      "activations/layer11_attention_weight_min": -35.58747482299805,
      "activations/layer12_attention_weight_max": 28.800188064575195,
      "activations/layer12_attention_weight_min": -27.91667938232422,
      "activations/layer13_attention_weight_max": 43.21921157836914,
      "activations/layer13_attention_weight_min": -36.494319915771484,
      "activations/layer14_attention_weight_max": 43.04014205932617,
      "activations/layer14_attention_weight_min": -37.432254791259766,
      "activations/layer15_attention_weight_max": 40.65387725830078,
      "activations/layer15_attention_weight_min": -36.340362548828125,
      "activations/layer16_attention_weight_max": 29.276165008544922,
      "activations/layer16_attention_weight_min": -30.111459732055664,
      "activations/layer17_attention_weight_max": 32.10225296020508,
      "activations/layer17_attention_weight_min": -26.818639755249023,
      "activations/layer18_attention_weight_max": 33.2059326171875,
      "activations/layer18_attention_weight_min": -27.78814697265625,
      "activations/layer19_attention_weight_max": 33.23165512084961,
      "activations/layer19_attention_weight_min": -32.46970748901367,
      "activations/layer1_attention_weight_max": 17.44480323791504,
      "activations/layer1_attention_weight_min": -14.933690071105957,
      "activations/layer20_attention_weight_max": 29.604286193847656,
      "activations/layer20_attention_weight_min": -25.077924728393555,
      "activations/layer21_attention_weight_max": 28.986064910888672,
      "activations/layer21_attention_weight_min": -25.20978355407715,
      "activations/layer22_attention_weight_max": 40.83380126953125,
      "activations/layer22_attention_weight_min": -32.43590545654297,
      "activations/layer23_attention_weight_max": 32.16505813598633,
      "activations/layer23_attention_weight_min": -25.90964126586914,
      "activations/layer2_attention_weight_max": 32.212459564208984,
      "activations/layer2_attention_weight_min": -32.727943420410156,
      "activations/layer3_attention_weight_max": 105.14845275878906,
      "activations/layer3_attention_weight_min": -104.31298828125,
      "activations/layer4_attention_weight_max": 111.567138671875,
      "activations/layer4_attention_weight_min": -114.9906234741211,
      "activations/layer5_attention_weight_max": 49.89772033691406,
      "activations/layer5_attention_weight_min": -64.52911376953125,
      "activations/layer6_attention_weight_max": 46.32064437866211,
      "activations/layer6_attention_weight_min": -48.60764694213867,
      "activations/layer7_attention_weight_max": 98.47280883789062,
      "activations/layer7_attention_weight_min": -100.40189361572266,
      "activations/layer8_attention_weight_max": 44.12561798095703,
      "activations/layer8_attention_weight_min": -46.011600494384766,
      "activations/layer9_attention_weight_max": 34.49827575683594,
      "activations/layer9_attention_weight_min": -36.2829704284668,
      "epoch": 20.89,
      "learning_rate": 1.5466287878787876e-05,
      "loss": 2.7015,
      "step": 359450
    },
    {
      "activations/layer0_attention_weight_max": 16.15329360961914,
      "activations/layer0_attention_weight_min": -14.904317855834961,
      "activations/layer10_attention_weight_max": 35.43180465698242,
      "activations/layer10_attention_weight_min": -36.30091094970703,
      "activations/layer11_attention_weight_max": 34.11737060546875,
      "activations/layer11_attention_weight_min": -35.034339904785156,
      "activations/layer12_attention_weight_max": 27.01934242248535,
      "activations/layer12_attention_weight_min": -29.32558822631836,
      "activations/layer13_attention_weight_max": 38.96065139770508,
      "activations/layer13_attention_weight_min": -37.46059036254883,
      "activations/layer14_attention_weight_max": 42.76639175415039,
      "activations/layer14_attention_weight_min": -39.55880355834961,
      "activations/layer15_attention_weight_max": 38.6783332824707,
      "activations/layer15_attention_weight_min": -38.130775451660156,
      "activations/layer16_attention_weight_max": 30.499828338623047,
      "activations/layer16_attention_weight_min": -29.226715087890625,
      "activations/layer17_attention_weight_max": 29.506391525268555,
      "activations/layer17_attention_weight_min": -30.522403717041016,
      "activations/layer18_attention_weight_max": 32.39922332763672,
      "activations/layer18_attention_weight_min": -27.92912483215332,
      "activations/layer19_attention_weight_max": 34.94355773925781,
      "activations/layer19_attention_weight_min": -32.39078903198242,
      "activations/layer1_attention_weight_max": 18.86222267150879,
      "activations/layer1_attention_weight_min": -15.395848274230957,
      "activations/layer20_attention_weight_max": 28.56255531311035,
      "activations/layer20_attention_weight_min": -26.801834106445312,
      "activations/layer21_attention_weight_max": 28.673404693603516,
      "activations/layer21_attention_weight_min": -27.14505386352539,
      "activations/layer22_attention_weight_max": 40.717124938964844,
      "activations/layer22_attention_weight_min": -34.60609436035156,
      "activations/layer23_attention_weight_max": 30.218639373779297,
      "activations/layer23_attention_weight_min": -25.794452667236328,
      "activations/layer2_attention_weight_max": 34.549686431884766,
      "activations/layer2_attention_weight_min": -33.17558288574219,
      "activations/layer3_attention_weight_max": 108.69945526123047,
      "activations/layer3_attention_weight_min": -105.4109878540039,
      "activations/layer4_attention_weight_max": 110.72338104248047,
      "activations/layer4_attention_weight_min": -113.9145736694336,
      "activations/layer5_attention_weight_max": 52.370849609375,
      "activations/layer5_attention_weight_min": -61.85282897949219,
      "activations/layer6_attention_weight_max": 45.43738555908203,
      "activations/layer6_attention_weight_min": -48.825706481933594,
      "activations/layer7_attention_weight_max": 98.60363006591797,
      "activations/layer7_attention_weight_min": -98.08863067626953,
      "activations/layer8_attention_weight_max": 43.11774826049805,
      "activations/layer8_attention_weight_min": -46.392696380615234,
      "activations/layer9_attention_weight_max": 31.860570907592773,
      "activations/layer9_attention_weight_min": -35.421321868896484,
      "epoch": 20.89,
      "learning_rate": 1.5447348484848484e-05,
      "loss": 2.6954,
      "step": 359500
    },
    {
      "activations/layer0_attention_weight_max": 15.908623695373535,
      "activations/layer0_attention_weight_min": -13.77739429473877,
      "activations/layer10_attention_weight_max": 34.63039779663086,
      "activations/layer10_attention_weight_min": -38.135040283203125,
      "activations/layer11_attention_weight_max": 35.86604690551758,
      "activations/layer11_attention_weight_min": -37.62440490722656,
      "activations/layer12_attention_weight_max": 27.861711502075195,
      "activations/layer12_attention_weight_min": -28.635658264160156,
      "activations/layer13_attention_weight_max": 40.233856201171875,
      "activations/layer13_attention_weight_min": -39.255001068115234,
      "activations/layer14_attention_weight_max": 44.70116424560547,
      "activations/layer14_attention_weight_min": -42.216129302978516,
      "activations/layer15_attention_weight_max": 42.12229537963867,
      "activations/layer15_attention_weight_min": -40.50663375854492,
      "activations/layer16_attention_weight_max": 31.437389373779297,
      "activations/layer16_attention_weight_min": -28.219924926757812,
      "activations/layer17_attention_weight_max": 30.970062255859375,
      "activations/layer17_attention_weight_min": -26.13346290588379,
      "activations/layer18_attention_weight_max": 32.666019439697266,
      "activations/layer18_attention_weight_min": -26.490764617919922,
      "activations/layer19_attention_weight_max": 34.01125717163086,
      "activations/layer19_attention_weight_min": -32.36797332763672,
      "activations/layer1_attention_weight_max": 16.173477172851562,
      "activations/layer1_attention_weight_min": -16.384017944335938,
      "activations/layer20_attention_weight_max": 29.694358825683594,
      "activations/layer20_attention_weight_min": -24.699228286743164,
      "activations/layer21_attention_weight_max": 30.62746238708496,
      "activations/layer21_attention_weight_min": -24.005008697509766,
      "activations/layer22_attention_weight_max": 43.24576187133789,
      "activations/layer22_attention_weight_min": -30.948299407958984,
      "activations/layer23_attention_weight_max": 34.73984909057617,
      "activations/layer23_attention_weight_min": -24.96591567993164,
      "activations/layer2_attention_weight_max": 32.54676818847656,
      "activations/layer2_attention_weight_min": -32.20260238647461,
      "activations/layer3_attention_weight_max": 109.03147888183594,
      "activations/layer3_attention_weight_min": -109.95389556884766,
      "activations/layer4_attention_weight_max": 112.10277557373047,
      "activations/layer4_attention_weight_min": -111.22777557373047,
      "activations/layer5_attention_weight_max": 50.81867218017578,
      "activations/layer5_attention_weight_min": -64.05574035644531,
      "activations/layer6_attention_weight_max": 45.89366912841797,
      "activations/layer6_attention_weight_min": -48.500160217285156,
      "activations/layer7_attention_weight_max": 98.77084350585938,
      "activations/layer7_attention_weight_min": -104.9796371459961,
      "activations/layer8_attention_weight_max": 44.930320739746094,
      "activations/layer8_attention_weight_min": -49.95200729370117,
      "activations/layer9_attention_weight_max": 32.922035217285156,
      "activations/layer9_attention_weight_min": -36.83871078491211,
      "epoch": 20.89,
      "learning_rate": 1.542840909090909e-05,
      "loss": 2.7096,
      "step": 359550
    },
    {
      "activations/layer0_attention_weight_max": 16.985177993774414,
      "activations/layer0_attention_weight_min": -14.710561752319336,
      "activations/layer10_attention_weight_max": 36.60447692871094,
      "activations/layer10_attention_weight_min": -36.339263916015625,
      "activations/layer11_attention_weight_max": 35.288185119628906,
      "activations/layer11_attention_weight_min": -36.18069076538086,
      "activations/layer12_attention_weight_max": 31.176591873168945,
      "activations/layer12_attention_weight_min": -27.926233291625977,
      "activations/layer13_attention_weight_max": 43.31621551513672,
      "activations/layer13_attention_weight_min": -37.81047058105469,
      "activations/layer14_attention_weight_max": 48.42387771606445,
      "activations/layer14_attention_weight_min": -38.415809631347656,
      "activations/layer15_attention_weight_max": 44.90045166015625,
      "activations/layer15_attention_weight_min": -37.14906692504883,
      "activations/layer16_attention_weight_max": 31.10589599609375,
      "activations/layer16_attention_weight_min": -28.56949234008789,
      "activations/layer17_attention_weight_max": 29.355607986450195,
      "activations/layer17_attention_weight_min": -28.458133697509766,
      "activations/layer18_attention_weight_max": 32.217708587646484,
      "activations/layer18_attention_weight_min": -26.90030288696289,
      "activations/layer19_attention_weight_max": 33.212284088134766,
      "activations/layer19_attention_weight_min": -32.8625373840332,
      "activations/layer1_attention_weight_max": 16.632810592651367,
      "activations/layer1_attention_weight_min": -15.662468910217285,
      "activations/layer20_attention_weight_max": 27.464454650878906,
      "activations/layer20_attention_weight_min": -24.317846298217773,
      "activations/layer21_attention_weight_max": 26.237096786499023,
      "activations/layer21_attention_weight_min": -23.899429321289062,
      "activations/layer22_attention_weight_max": 43.24860763549805,
      "activations/layer22_attention_weight_min": -30.44597625732422,
      "activations/layer23_attention_weight_max": 31.485916137695312,
      "activations/layer23_attention_weight_min": -25.86819839477539,
      "activations/layer2_attention_weight_max": 33.836151123046875,
      "activations/layer2_attention_weight_min": -34.19474411010742,
      "activations/layer3_attention_weight_max": 108.899169921875,
      "activations/layer3_attention_weight_min": -106.1052474975586,
      "activations/layer4_attention_weight_max": 111.3245620727539,
      "activations/layer4_attention_weight_min": -112.75577545166016,
      "activations/layer5_attention_weight_max": 51.231868743896484,
      "activations/layer5_attention_weight_min": -64.99385070800781,
      "activations/layer6_attention_weight_max": 45.24446105957031,
      "activations/layer6_attention_weight_min": -46.93648910522461,
      "activations/layer7_attention_weight_max": 93.93500518798828,
      "activations/layer7_attention_weight_min": -96.80491638183594,
      "activations/layer8_attention_weight_max": 43.4892578125,
      "activations/layer8_attention_weight_min": -44.72385787963867,
      "activations/layer9_attention_weight_max": 34.90007781982422,
      "activations/layer9_attention_weight_min": -35.30757522583008,
      "epoch": 20.89,
      "learning_rate": 1.5409469696969694e-05,
      "loss": 2.7243,
      "step": 359600
    },
    {
      "activations/layer0_attention_weight_max": 16.51985740661621,
      "activations/layer0_attention_weight_min": -14.251296043395996,
      "activations/layer10_attention_weight_max": 35.13555145263672,
      "activations/layer10_attention_weight_min": -36.7518196105957,
      "activations/layer11_attention_weight_max": 36.800453186035156,
      "activations/layer11_attention_weight_min": -36.830867767333984,
      "activations/layer12_attention_weight_max": 28.714820861816406,
      "activations/layer12_attention_weight_min": -28.00165557861328,
      "activations/layer13_attention_weight_max": 42.71921920776367,
      "activations/layer13_attention_weight_min": -36.64805221557617,
      "activations/layer14_attention_weight_max": 43.26244354248047,
      "activations/layer14_attention_weight_min": -38.25630569458008,
      "activations/layer15_attention_weight_max": 39.44225311279297,
      "activations/layer15_attention_weight_min": -36.49967956542969,
      "activations/layer16_attention_weight_max": 30.128034591674805,
      "activations/layer16_attention_weight_min": -30.38889503479004,
      "activations/layer17_attention_weight_max": 32.82907485961914,
      "activations/layer17_attention_weight_min": -26.952178955078125,
      "activations/layer18_attention_weight_max": 35.22390365600586,
      "activations/layer18_attention_weight_min": -27.768281936645508,
      "activations/layer19_attention_weight_max": 35.74296951293945,
      "activations/layer19_attention_weight_min": -31.424535751342773,
      "activations/layer1_attention_weight_max": 15.514626502990723,
      "activations/layer1_attention_weight_min": -14.928980827331543,
      "activations/layer20_attention_weight_max": 30.40913963317871,
      "activations/layer20_attention_weight_min": -25.0924015045166,
      "activations/layer21_attention_weight_max": 33.0030517578125,
      "activations/layer21_attention_weight_min": -27.534677505493164,
      "activations/layer22_attention_weight_max": 47.61273193359375,
      "activations/layer22_attention_weight_min": -33.085426330566406,
      "activations/layer23_attention_weight_max": 34.35077667236328,
      "activations/layer23_attention_weight_min": -24.80517578125,
      "activations/layer2_attention_weight_max": 35.397315979003906,
      "activations/layer2_attention_weight_min": -35.24996566772461,
      "activations/layer3_attention_weight_max": 107.20441436767578,
      "activations/layer3_attention_weight_min": -110.13716888427734,
      "activations/layer4_attention_weight_max": 110.2159423828125,
      "activations/layer4_attention_weight_min": -114.48664855957031,
      "activations/layer5_attention_weight_max": 50.685264587402344,
      "activations/layer5_attention_weight_min": -62.575618743896484,
      "activations/layer6_attention_weight_max": 45.50957107543945,
      "activations/layer6_attention_weight_min": -48.29624938964844,
      "activations/layer7_attention_weight_max": 95.94986724853516,
      "activations/layer7_attention_weight_min": -97.24724578857422,
      "activations/layer8_attention_weight_max": 42.87206268310547,
      "activations/layer8_attention_weight_min": -44.53315734863281,
      "activations/layer9_attention_weight_max": 34.72242736816406,
      "activations/layer9_attention_weight_min": -37.1765251159668,
      "epoch": 20.9,
      "learning_rate": 1.5390530303030302e-05,
      "loss": 2.7241,
      "step": 359650
    },
    {
      "activations/layer0_attention_weight_max": 16.214237213134766,
      "activations/layer0_attention_weight_min": -16.14061164855957,
      "activations/layer10_attention_weight_max": 35.36134719848633,
      "activations/layer10_attention_weight_min": -36.241939544677734,
      "activations/layer11_attention_weight_max": 32.671573638916016,
      "activations/layer11_attention_weight_min": -34.81577682495117,
      "activations/layer12_attention_weight_max": 26.662050247192383,
      "activations/layer12_attention_weight_min": -26.692401885986328,
      "activations/layer13_attention_weight_max": 39.54253387451172,
      "activations/layer13_attention_weight_min": -37.262168884277344,
      "activations/layer14_attention_weight_max": 41.58646774291992,
      "activations/layer14_attention_weight_min": -37.209877014160156,
      "activations/layer15_attention_weight_max": 41.59873962402344,
      "activations/layer15_attention_weight_min": -38.2607307434082,
      "activations/layer16_attention_weight_max": 29.11604118347168,
      "activations/layer16_attention_weight_min": -28.594158172607422,
      "activations/layer17_attention_weight_max": 28.87490463256836,
      "activations/layer17_attention_weight_min": -27.994169235229492,
      "activations/layer18_attention_weight_max": 31.46784782409668,
      "activations/layer18_attention_weight_min": -28.349750518798828,
      "activations/layer19_attention_weight_max": 32.01139450073242,
      "activations/layer19_attention_weight_min": -31.29877281188965,
      "activations/layer1_attention_weight_max": 15.383326530456543,
      "activations/layer1_attention_weight_min": -15.865827560424805,
      "activations/layer20_attention_weight_max": 25.815195083618164,
      "activations/layer20_attention_weight_min": -24.953584671020508,
      "activations/layer21_attention_weight_max": 26.839391708374023,
      "activations/layer21_attention_weight_min": -25.497037887573242,
      "activations/layer22_attention_weight_max": 40.66737365722656,
      "activations/layer22_attention_weight_min": -32.78245162963867,
      "activations/layer23_attention_weight_max": 30.662771224975586,
      "activations/layer23_attention_weight_min": -26.842084884643555,
      "activations/layer2_attention_weight_max": 34.7459716796875,
      "activations/layer2_attention_weight_min": -33.01639938354492,
      "activations/layer3_attention_weight_max": 102.85345458984375,
      "activations/layer3_attention_weight_min": -108.06938934326172,
      "activations/layer4_attention_weight_max": 110.73685455322266,
      "activations/layer4_attention_weight_min": -111.72472381591797,
      "activations/layer5_attention_weight_max": 49.164093017578125,
      "activations/layer5_attention_weight_min": -60.18659973144531,
      "activations/layer6_attention_weight_max": 43.22338104248047,
      "activations/layer6_attention_weight_min": -47.19164276123047,
      "activations/layer7_attention_weight_max": 94.07209014892578,
      "activations/layer7_attention_weight_min": -95.79913330078125,
      "activations/layer8_attention_weight_max": 41.96774673461914,
      "activations/layer8_attention_weight_min": -44.03556823730469,
      "activations/layer9_attention_weight_max": 32.0905876159668,
      "activations/layer9_attention_weight_min": -35.466243743896484,
      "epoch": 20.9,
      "learning_rate": 1.5371590909090907e-05,
      "loss": 2.701,
      "step": 359700
    },
    {
      "activations/layer0_attention_weight_max": 16.76056480407715,
      "activations/layer0_attention_weight_min": -13.33939266204834,
      "activations/layer10_attention_weight_max": 34.79517364501953,
      "activations/layer10_attention_weight_min": -36.63077926635742,
      "activations/layer11_attention_weight_max": 35.023704528808594,
      "activations/layer11_attention_weight_min": -37.0788688659668,
      "activations/layer12_attention_weight_max": 26.542760848999023,
      "activations/layer12_attention_weight_min": -26.96967124938965,
      "activations/layer13_attention_weight_max": 42.57914733886719,
      "activations/layer13_attention_weight_min": -35.530555725097656,
      "activations/layer14_attention_weight_max": 43.5328483581543,
      "activations/layer14_attention_weight_min": -37.68699645996094,
      "activations/layer15_attention_weight_max": 44.46772766113281,
      "activations/layer15_attention_weight_min": -35.363101959228516,
      "activations/layer16_attention_weight_max": 30.555395126342773,
      "activations/layer16_attention_weight_min": -28.74070930480957,
      "activations/layer17_attention_weight_max": 31.227203369140625,
      "activations/layer17_attention_weight_min": -28.07438087463379,
      "activations/layer18_attention_weight_max": 33.8956298828125,
      "activations/layer18_attention_weight_min": -27.47270393371582,
      "activations/layer19_attention_weight_max": 37.36862564086914,
      "activations/layer19_attention_weight_min": -32.77666473388672,
      "activations/layer1_attention_weight_max": 16.12574577331543,
      "activations/layer1_attention_weight_min": -15.079512596130371,
      "activations/layer20_attention_weight_max": 32.90544509887695,
      "activations/layer20_attention_weight_min": -26.982643127441406,
      "activations/layer21_attention_weight_max": 32.007720947265625,
      "activations/layer21_attention_weight_min": -26.62429428100586,
      "activations/layer22_attention_weight_max": 42.83050537109375,
      "activations/layer22_attention_weight_min": -34.73204803466797,
      "activations/layer23_attention_weight_max": 33.643516540527344,
      "activations/layer23_attention_weight_min": -29.597227096557617,
      "activations/layer2_attention_weight_max": 34.81412887573242,
      "activations/layer2_attention_weight_min": -34.34461212158203,
      "activations/layer3_attention_weight_max": 103.81629943847656,
      "activations/layer3_attention_weight_min": -105.77850341796875,
      "activations/layer4_attention_weight_max": 110.59734344482422,
      "activations/layer4_attention_weight_min": -111.1136245727539,
      "activations/layer5_attention_weight_max": 54.53112030029297,
      "activations/layer5_attention_weight_min": -67.09242248535156,
      "activations/layer6_attention_weight_max": 45.47545623779297,
      "activations/layer6_attention_weight_min": -49.50272750854492,
      "activations/layer7_attention_weight_max": 96.18927001953125,
      "activations/layer7_attention_weight_min": -102.35884094238281,
      "activations/layer8_attention_weight_max": 44.9063835144043,
      "activations/layer8_attention_weight_min": -46.56078338623047,
      "activations/layer9_attention_weight_max": 33.77646255493164,
      "activations/layer9_attention_weight_min": -34.7332878112793,
      "epoch": 20.9,
      "learning_rate": 1.5352651515151512e-05,
      "loss": 2.7059,
      "step": 359750
    },
    {
      "activations/layer0_attention_weight_max": 16.829103469848633,
      "activations/layer0_attention_weight_min": -14.443063735961914,
      "activations/layer10_attention_weight_max": 32.44450759887695,
      "activations/layer10_attention_weight_min": -32.565460205078125,
      "activations/layer11_attention_weight_max": 32.4496955871582,
      "activations/layer11_attention_weight_min": -32.88813018798828,
      "activations/layer12_attention_weight_max": 26.18387222290039,
      "activations/layer12_attention_weight_min": -28.107141494750977,
      "activations/layer13_attention_weight_max": 38.78413772583008,
      "activations/layer13_attention_weight_min": -34.036128997802734,
      "activations/layer14_attention_weight_max": 41.998085021972656,
      "activations/layer14_attention_weight_min": -37.777408599853516,
      "activations/layer15_attention_weight_max": 37.63833236694336,
      "activations/layer15_attention_weight_min": -35.72697067260742,
      "activations/layer16_attention_weight_max": 28.231367111206055,
      "activations/layer16_attention_weight_min": -29.43317985534668,
      "activations/layer17_attention_weight_max": 30.18337631225586,
      "activations/layer17_attention_weight_min": -27.778642654418945,
      "activations/layer18_attention_weight_max": 33.62773895263672,
      "activations/layer18_attention_weight_min": -26.179140090942383,
      "activations/layer19_attention_weight_max": 33.6392707824707,
      "activations/layer19_attention_weight_min": -34.48426055908203,
      "activations/layer1_attention_weight_max": 16.29018783569336,
      "activations/layer1_attention_weight_min": -15.1190824508667,
      "activations/layer20_attention_weight_max": 32.20949935913086,
      "activations/layer20_attention_weight_min": -26.583507537841797,
      "activations/layer21_attention_weight_max": 32.479095458984375,
      "activations/layer21_attention_weight_min": -24.849502563476562,
      "activations/layer22_attention_weight_max": 40.73519515991211,
      "activations/layer22_attention_weight_min": -30.548112869262695,
      "activations/layer23_attention_weight_max": 32.131526947021484,
      "activations/layer23_attention_weight_min": -24.994354248046875,
      "activations/layer2_attention_weight_max": 34.100494384765625,
      "activations/layer2_attention_weight_min": -32.10900115966797,
      "activations/layer3_attention_weight_max": 103.60041809082031,
      "activations/layer3_attention_weight_min": -101.00670623779297,
      "activations/layer4_attention_weight_max": 102.0876693725586,
      "activations/layer4_attention_weight_min": -109.11144256591797,
      "activations/layer5_attention_weight_max": 49.57959747314453,
      "activations/layer5_attention_weight_min": -59.22438049316406,
      "activations/layer6_attention_weight_max": 42.11617660522461,
      "activations/layer6_attention_weight_min": -46.043819427490234,
      "activations/layer7_attention_weight_max": 90.00750732421875,
      "activations/layer7_attention_weight_min": -93.2198486328125,
      "activations/layer8_attention_weight_max": 39.42681121826172,
      "activations/layer8_attention_weight_min": -42.50777816772461,
      "activations/layer9_attention_weight_max": 28.988752365112305,
      "activations/layer9_attention_weight_min": -32.537044525146484,
      "epoch": 20.91,
      "learning_rate": 1.533371212121212e-05,
      "loss": 2.7136,
      "step": 359800
    },
    {
      "activations/layer0_attention_weight_max": 16.2726993560791,
      "activations/layer0_attention_weight_min": -14.853074073791504,
      "activations/layer10_attention_weight_max": 33.318016052246094,
      "activations/layer10_attention_weight_min": -35.76237487792969,
      "activations/layer11_attention_weight_max": 34.02155303955078,
      "activations/layer11_attention_weight_min": -34.21107482910156,
      "activations/layer12_attention_weight_max": 26.814634323120117,
      "activations/layer12_attention_weight_min": -26.863996505737305,
      "activations/layer13_attention_weight_max": 39.092628479003906,
      "activations/layer13_attention_weight_min": -35.32350158691406,
      "activations/layer14_attention_weight_max": 39.84083938598633,
      "activations/layer14_attention_weight_min": -38.97214889526367,
      "activations/layer15_attention_weight_max": 36.65752029418945,
      "activations/layer15_attention_weight_min": -37.42871856689453,
      "activations/layer16_attention_weight_max": 29.972919464111328,
      "activations/layer16_attention_weight_min": -30.268360137939453,
      "activations/layer17_attention_weight_max": 31.019630432128906,
      "activations/layer17_attention_weight_min": -26.234012603759766,
      "activations/layer18_attention_weight_max": 32.406700134277344,
      "activations/layer18_attention_weight_min": -27.601980209350586,
      "activations/layer19_attention_weight_max": 35.80173873901367,
      "activations/layer19_attention_weight_min": -32.70347595214844,
      "activations/layer1_attention_weight_max": 15.542174339294434,
      "activations/layer1_attention_weight_min": -14.331513404846191,
      "activations/layer20_attention_weight_max": 29.761550903320312,
      "activations/layer20_attention_weight_min": -25.245361328125,
      "activations/layer21_attention_weight_max": 28.2724609375,
      "activations/layer21_attention_weight_min": -25.026853561401367,
      "activations/layer22_attention_weight_max": 38.92813491821289,
      "activations/layer22_attention_weight_min": -33.69870376586914,
      "activations/layer23_attention_weight_max": 32.04911422729492,
      "activations/layer23_attention_weight_min": -27.349746704101562,
      "activations/layer2_attention_weight_max": 35.2435302734375,
      "activations/layer2_attention_weight_min": -33.34632873535156,
      "activations/layer3_attention_weight_max": 101.87684631347656,
      "activations/layer3_attention_weight_min": -105.65512084960938,
      "activations/layer4_attention_weight_max": 103.65289306640625,
      "activations/layer4_attention_weight_min": -106.83528900146484,
      "activations/layer5_attention_weight_max": 51.74071502685547,
      "activations/layer5_attention_weight_min": -63.20487976074219,
      "activations/layer6_attention_weight_max": 42.58482360839844,
      "activations/layer6_attention_weight_min": -47.25840377807617,
      "activations/layer7_attention_weight_max": 92.20964050292969,
      "activations/layer7_attention_weight_min": -97.57665252685547,
      "activations/layer8_attention_weight_max": 42.770660400390625,
      "activations/layer8_attention_weight_min": -44.126766204833984,
      "activations/layer9_attention_weight_max": 31.60101890563965,
      "activations/layer9_attention_weight_min": -33.63727951049805,
      "epoch": 20.91,
      "learning_rate": 1.5314772727272725e-05,
      "loss": 2.7109,
      "step": 359850
    },
    {
      "activations/layer0_attention_weight_max": 16.0196590423584,
      "activations/layer0_attention_weight_min": -13.945625305175781,
      "activations/layer10_attention_weight_max": 35.34449005126953,
      "activations/layer10_attention_weight_min": -36.26642990112305,
      "activations/layer11_attention_weight_max": 35.169525146484375,
      "activations/layer11_attention_weight_min": -35.801780700683594,
      "activations/layer12_attention_weight_max": 26.759111404418945,
      "activations/layer12_attention_weight_min": -28.450904846191406,
      "activations/layer13_attention_weight_max": 37.27521896362305,
      "activations/layer13_attention_weight_min": -37.50756072998047,
      "activations/layer14_attention_weight_max": 40.94682312011719,
      "activations/layer14_attention_weight_min": -40.92885971069336,
      "activations/layer15_attention_weight_max": 37.823368072509766,
      "activations/layer15_attention_weight_min": -42.33249282836914,
      "activations/layer16_attention_weight_max": 27.87864875793457,
      "activations/layer16_attention_weight_min": -31.147066116333008,
      "activations/layer17_attention_weight_max": 28.72989845275879,
      "activations/layer17_attention_weight_min": -28.464712142944336,
      "activations/layer18_attention_weight_max": 29.465213775634766,
      "activations/layer18_attention_weight_min": -27.592130661010742,
      "activations/layer19_attention_weight_max": 30.32232093811035,
      "activations/layer19_attention_weight_min": -33.917232513427734,
      "activations/layer1_attention_weight_max": 16.305274963378906,
      "activations/layer1_attention_weight_min": -14.527754783630371,
      "activations/layer20_attention_weight_max": 29.630775451660156,
      "activations/layer20_attention_weight_min": -27.2983341217041,
      "activations/layer21_attention_weight_max": 29.053075790405273,
      "activations/layer21_attention_weight_min": -26.188400268554688,
      "activations/layer22_attention_weight_max": 37.304256439208984,
      "activations/layer22_attention_weight_min": -35.83345413208008,
      "activations/layer23_attention_weight_max": 30.299739837646484,
      "activations/layer23_attention_weight_min": -28.572006225585938,
      "activations/layer2_attention_weight_max": 32.24211120605469,
      "activations/layer2_attention_weight_min": -32.050506591796875,
      "activations/layer3_attention_weight_max": 104.598388671875,
      "activations/layer3_attention_weight_min": -102.38648986816406,
      "activations/layer4_attention_weight_max": 108.07938385009766,
      "activations/layer4_attention_weight_min": -110.86600494384766,
      "activations/layer5_attention_weight_max": 49.6419792175293,
      "activations/layer5_attention_weight_min": -64.98638153076172,
      "activations/layer6_attention_weight_max": 45.50912857055664,
      "activations/layer6_attention_weight_min": -46.95993423461914,
      "activations/layer7_attention_weight_max": 96.35588073730469,
      "activations/layer7_attention_weight_min": -101.22490692138672,
      "activations/layer8_attention_weight_max": 42.47065734863281,
      "activations/layer8_attention_weight_min": -44.93234634399414,
      "activations/layer9_attention_weight_max": 35.0516242980957,
      "activations/layer9_attention_weight_min": -35.28688430786133,
      "epoch": 20.91,
      "learning_rate": 1.5295833333333334e-05,
      "loss": 2.7091,
      "step": 359900
    },
    {
      "activations/layer0_attention_weight_max": 16.91086196899414,
      "activations/layer0_attention_weight_min": -13.756889343261719,
      "activations/layer10_attention_weight_max": 35.32489013671875,
      "activations/layer10_attention_weight_min": -35.6387825012207,
      "activations/layer11_attention_weight_max": 36.727386474609375,
      "activations/layer11_attention_weight_min": -35.046363830566406,
      "activations/layer12_attention_weight_max": 26.436704635620117,
      "activations/layer12_attention_weight_min": -27.059755325317383,
      "activations/layer13_attention_weight_max": 37.62282943725586,
      "activations/layer13_attention_weight_min": -38.43239212036133,
      "activations/layer14_attention_weight_max": 41.93235397338867,
      "activations/layer14_attention_weight_min": -39.68558120727539,
      "activations/layer15_attention_weight_max": 37.020660400390625,
      "activations/layer15_attention_weight_min": -35.934783935546875,
      "activations/layer16_attention_weight_max": 27.628416061401367,
      "activations/layer16_attention_weight_min": -27.166532516479492,
      "activations/layer17_attention_weight_max": 29.04673194885254,
      "activations/layer17_attention_weight_min": -24.85394859313965,
      "activations/layer18_attention_weight_max": 30.13133430480957,
      "activations/layer18_attention_weight_min": -26.079254150390625,
      "activations/layer19_attention_weight_max": 34.257694244384766,
      "activations/layer19_attention_weight_min": -32.509010314941406,
      "activations/layer1_attention_weight_max": 15.406990051269531,
      "activations/layer1_attention_weight_min": -14.89861011505127,
      "activations/layer20_attention_weight_max": 27.946269989013672,
      "activations/layer20_attention_weight_min": -24.498027801513672,
      "activations/layer21_attention_weight_max": 27.427804946899414,
      "activations/layer21_attention_weight_min": -24.17653465270996,
      "activations/layer22_attention_weight_max": 37.378570556640625,
      "activations/layer22_attention_weight_min": -32.22574996948242,
      "activations/layer23_attention_weight_max": 34.074462890625,
      "activations/layer23_attention_weight_min": -25.22997283935547,
      "activations/layer2_attention_weight_max": 32.74973678588867,
      "activations/layer2_attention_weight_min": -32.29145812988281,
      "activations/layer3_attention_weight_max": 102.04662322998047,
      "activations/layer3_attention_weight_min": -100.57234191894531,
      "activations/layer4_attention_weight_max": 106.37169647216797,
      "activations/layer4_attention_weight_min": -107.69310760498047,
      "activations/layer5_attention_weight_max": 48.01495361328125,
      "activations/layer5_attention_weight_min": -65.21273040771484,
      "activations/layer6_attention_weight_max": 46.33169937133789,
      "activations/layer6_attention_weight_min": -48.611148834228516,
      "activations/layer7_attention_weight_max": 91.25057220458984,
      "activations/layer7_attention_weight_min": -100.48809814453125,
      "activations/layer8_attention_weight_max": 42.190032958984375,
      "activations/layer8_attention_weight_min": -44.84267044067383,
      "activations/layer9_attention_weight_max": 33.086883544921875,
      "activations/layer9_attention_weight_min": -34.821231842041016,
      "epoch": 20.92,
      "learning_rate": 1.527689393939394e-05,
      "loss": 2.7057,
      "step": 359950
    },
    {
      "activations/layer0_attention_weight_max": 16.522735595703125,
      "activations/layer0_attention_weight_min": -13.830397605895996,
      "activations/layer10_attention_weight_max": 34.13704299926758,
      "activations/layer10_attention_weight_min": -35.7026252746582,
      "activations/layer11_attention_weight_max": 32.81866455078125,
      "activations/layer11_attention_weight_min": -34.47947692871094,
      "activations/layer12_attention_weight_max": 24.971940994262695,
      "activations/layer12_attention_weight_min": -26.6624813079834,
      "activations/layer13_attention_weight_max": 37.538307189941406,
      "activations/layer13_attention_weight_min": -34.76286697387695,
      "activations/layer14_attention_weight_max": 40.030826568603516,
      "activations/layer14_attention_weight_min": -36.38164520263672,
      "activations/layer15_attention_weight_max": 36.6685905456543,
      "activations/layer15_attention_weight_min": -37.33030700683594,
      "activations/layer16_attention_weight_max": 28.371959686279297,
      "activations/layer16_attention_weight_min": -30.852062225341797,
      "activations/layer17_attention_weight_max": 27.561195373535156,
      "activations/layer17_attention_weight_min": -30.2587833404541,
      "activations/layer18_attention_weight_max": 31.213815689086914,
      "activations/layer18_attention_weight_min": -27.444095611572266,
      "activations/layer19_attention_weight_max": 33.12419128417969,
      "activations/layer19_attention_weight_min": -32.875267028808594,
      "activations/layer1_attention_weight_max": 16.8583927154541,
      "activations/layer1_attention_weight_min": -16.45160675048828,
      "activations/layer20_attention_weight_max": 30.356712341308594,
      "activations/layer20_attention_weight_min": -25.04338264465332,
      "activations/layer21_attention_weight_max": 29.449687957763672,
      "activations/layer21_attention_weight_min": -25.40763282775879,
      "activations/layer22_attention_weight_max": 40.312496185302734,
      "activations/layer22_attention_weight_min": -32.084407806396484,
      "activations/layer23_attention_weight_max": 33.15743637084961,
      "activations/layer23_attention_weight_min": -26.661792755126953,
      "activations/layer2_attention_weight_max": 34.25972366333008,
      "activations/layer2_attention_weight_min": -32.461578369140625,
      "activations/layer3_attention_weight_max": 100.28408813476562,
      "activations/layer3_attention_weight_min": -100.80424499511719,
      "activations/layer4_attention_weight_max": 104.60448455810547,
      "activations/layer4_attention_weight_min": -106.71868133544922,
      "activations/layer5_attention_weight_max": 50.74436950683594,
      "activations/layer5_attention_weight_min": -60.33248519897461,
      "activations/layer6_attention_weight_max": 42.24628829956055,
      "activations/layer6_attention_weight_min": -46.8253288269043,
      "activations/layer7_attention_weight_max": 90.79418182373047,
      "activations/layer7_attention_weight_min": -96.82379913330078,
      "activations/layer8_attention_weight_max": 39.80829620361328,
      "activations/layer8_attention_weight_min": -42.3743782043457,
      "activations/layer9_attention_weight_max": 30.712360382080078,
      "activations/layer9_attention_weight_min": -34.84983825683594,
      "epoch": 20.92,
      "learning_rate": 1.5257954545454545e-05,
      "loss": 2.7171,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4573,
      "eval_samples_per_second": 507.725,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4573,
      "eval_openwebtext_samples_per_second": 507.725,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9387,
      "eval_wikitext_samples_per_second": 235.207,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_lambada_loss": 2.380859375,
      "eval_lambada_ppl": 10.814192318320675,
      "eval_lambada_runtime": 9.518,
      "eval_lambada_samples_per_second": 511.558,
      "step": 360000
    },
    {
      "activations/layer0_attention_weight_max": 17.529247283935547,
      "activations/layer0_attention_weight_min": -14.133975982666016,
      "activations/layer10_attention_weight_max": 40.67977523803711,
      "activations/layer10_attention_weight_min": -40.72196960449219,
      "activations/layer11_attention_weight_max": 42.88795471191406,
      "activations/layer11_attention_weight_min": -39.805259704589844,
      "activations/layer12_attention_weight_max": 32.35550308227539,
      "activations/layer12_attention_weight_min": -29.72015953063965,
      "activations/layer13_attention_weight_max": 47.098182678222656,
      "activations/layer13_attention_weight_min": -38.73755645751953,
      "activations/layer14_attention_weight_max": 52.281959533691406,
      "activations/layer14_attention_weight_min": -44.52712631225586,
      "activations/layer15_attention_weight_max": 45.93720626831055,
      "activations/layer15_attention_weight_min": -42.53997802734375,
      "activations/layer16_attention_weight_max": 37.31845474243164,
      "activations/layer16_attention_weight_min": -29.86406135559082,
      "activations/layer17_attention_weight_max": 41.85415267944336,
      "activations/layer17_attention_weight_min": -30.956024169921875,
      "activations/layer18_attention_weight_max": 37.30288314819336,
      "activations/layer18_attention_weight_min": -30.898578643798828,
      "activations/layer19_attention_weight_max": 41.644439697265625,
      "activations/layer19_attention_weight_min": -33.66532516479492,
      "activations/layer1_attention_weight_max": 17.3453369140625,
      "activations/layer1_attention_weight_min": -14.475224494934082,
      "activations/layer20_attention_weight_max": 33.13228988647461,
      "activations/layer20_attention_weight_min": -25.449129104614258,
      "activations/layer21_attention_weight_max": 32.23765182495117,
      "activations/layer21_attention_weight_min": -26.054000854492188,
      "activations/layer22_attention_weight_max": 46.559242248535156,
      "activations/layer22_attention_weight_min": -35.577919006347656,
      "activations/layer23_attention_weight_max": 34.68914794921875,
      "activations/layer23_attention_weight_min": -27.95442008972168,
      "activations/layer2_attention_weight_max": 35.85282897949219,
      "activations/layer2_attention_weight_min": -35.14094924926758,
      "activations/layer3_attention_weight_max": 107.74260711669922,
      "activations/layer3_attention_weight_min": -107.96582794189453,
      "activations/layer4_attention_weight_max": 111.1899642944336,
      "activations/layer4_attention_weight_min": -115.99871826171875,
      "activations/layer5_attention_weight_max": 50.73862838745117,
      "activations/layer5_attention_weight_min": -62.131805419921875,
      "activations/layer6_attention_weight_max": 48.61360168457031,
      "activations/layer6_attention_weight_min": -49.33428192138672,
      "activations/layer7_attention_weight_max": 104.56982421875,
      "activations/layer7_attention_weight_min": -106.00753784179688,
      "activations/layer8_attention_weight_max": 50.14038848876953,
      "activations/layer8_attention_weight_min": -49.91242218017578,
      "activations/layer9_attention_weight_max": 36.65506362915039,
      "activations/layer9_attention_weight_min": -38.56686782836914,
      "epoch": 20.92,
      "learning_rate": 1.523901515151515e-05,
      "loss": 2.709,
      "step": 360050
    },
    {
      "activations/layer0_attention_weight_max": 16.221643447875977,
      "activations/layer0_attention_weight_min": -13.105412483215332,
      "activations/layer10_attention_weight_max": 36.651214599609375,
      "activations/layer10_attention_weight_min": -35.80451583862305,
      "activations/layer11_attention_weight_max": 35.385169982910156,
      "activations/layer11_attention_weight_min": -34.213897705078125,
      "activations/layer12_attention_weight_max": 26.259305953979492,
      "activations/layer12_attention_weight_min": -28.072607040405273,
      "activations/layer13_attention_weight_max": 41.993804931640625,
      "activations/layer13_attention_weight_min": -38.78281784057617,
      "activations/layer14_attention_weight_max": 46.462425231933594,
      "activations/layer14_attention_weight_min": -42.328651428222656,
      "activations/layer15_attention_weight_max": 40.02610397338867,
      "activations/layer15_attention_weight_min": -38.24177551269531,
      "activations/layer16_attention_weight_max": 31.0401611328125,
      "activations/layer16_attention_weight_min": -28.64622688293457,
      "activations/layer17_attention_weight_max": 30.385528564453125,
      "activations/layer17_attention_weight_min": -26.68515968322754,
      "activations/layer18_attention_weight_max": 34.26723861694336,
      "activations/layer18_attention_weight_min": -28.42778205871582,
      "activations/layer19_attention_weight_max": 37.87957763671875,
      "activations/layer19_attention_weight_min": -33.03654098510742,
      "activations/layer1_attention_weight_max": 16.14787483215332,
      "activations/layer1_attention_weight_min": -15.550287246704102,
      "activations/layer20_attention_weight_max": 32.502593994140625,
      "activations/layer20_attention_weight_min": -26.374584197998047,
      "activations/layer21_attention_weight_max": 30.4079647064209,
      "activations/layer21_attention_weight_min": -24.857177734375,
      "activations/layer22_attention_weight_max": 51.73039627075195,
      "activations/layer22_attention_weight_min": -35.064144134521484,
      "activations/layer23_attention_weight_max": 36.30965042114258,
      "activations/layer23_attention_weight_min": -26.153974533081055,
      "activations/layer2_attention_weight_max": 33.85566711425781,
      "activations/layer2_attention_weight_min": -32.56671142578125,
      "activations/layer3_attention_weight_max": 108.9005126953125,
      "activations/layer3_attention_weight_min": -107.01274108886719,
      "activations/layer4_attention_weight_max": 113.24413299560547,
      "activations/layer4_attention_weight_min": -114.9348373413086,
      "activations/layer5_attention_weight_max": 51.93925476074219,
      "activations/layer5_attention_weight_min": -62.87289047241211,
      "activations/layer6_attention_weight_max": 45.38860321044922,
      "activations/layer6_attention_weight_min": -47.7118034362793,
      "activations/layer7_attention_weight_max": 93.26776123046875,
      "activations/layer7_attention_weight_min": -99.9246826171875,
      "activations/layer8_attention_weight_max": 41.179840087890625,
      "activations/layer8_attention_weight_min": -45.067291259765625,
      "activations/layer9_attention_weight_max": 31.653295516967773,
      "activations/layer9_attention_weight_min": -36.55696487426758,
      "epoch": 20.92,
      "learning_rate": 1.5220075757575756e-05,
      "loss": 2.7194,
      "step": 360100
    },
    {
      "activations/layer0_attention_weight_max": 15.891705513000488,
      "activations/layer0_attention_weight_min": -14.50144100189209,
      "activations/layer10_attention_weight_max": 32.91634750366211,
      "activations/layer10_attention_weight_min": -35.063133239746094,
      "activations/layer11_attention_weight_max": 33.90167236328125,
      "activations/layer11_attention_weight_min": -35.25150680541992,
      "activations/layer12_attention_weight_max": 26.803041458129883,
      "activations/layer12_attention_weight_min": -26.873886108398438,
      "activations/layer13_attention_weight_max": 37.65221405029297,
      "activations/layer13_attention_weight_min": -35.105567932128906,
      "activations/layer14_attention_weight_max": 43.79342269897461,
      "activations/layer14_attention_weight_min": -38.14292907714844,
      "activations/layer15_attention_weight_max": 39.39125442504883,
      "activations/layer15_attention_weight_min": -35.53841018676758,
      "activations/layer16_attention_weight_max": 30.301403045654297,
      "activations/layer16_attention_weight_min": -29.265827178955078,
      "activations/layer17_attention_weight_max": 30.160932540893555,
      "activations/layer17_attention_weight_min": -25.832136154174805,
      "activations/layer18_attention_weight_max": 31.335886001586914,
      "activations/layer18_attention_weight_min": -26.84178352355957,
      "activations/layer19_attention_weight_max": 34.02091598510742,
      "activations/layer19_attention_weight_min": -30.7905330657959,
      "activations/layer1_attention_weight_max": 16.268287658691406,
      "activations/layer1_attention_weight_min": -15.119561195373535,
      "activations/layer20_attention_weight_max": 27.996353149414062,
      "activations/layer20_attention_weight_min": -23.561325073242188,
      "activations/layer21_attention_weight_max": 26.767545700073242,
      "activations/layer21_attention_weight_min": -24.209697723388672,
      "activations/layer22_attention_weight_max": 43.554840087890625,
      "activations/layer22_attention_weight_min": -30.318256378173828,
      "activations/layer23_attention_weight_max": 31.736316680908203,
      "activations/layer23_attention_weight_min": -25.021177291870117,
      "activations/layer2_attention_weight_max": 34.375919342041016,
      "activations/layer2_attention_weight_min": -33.07459259033203,
      "activations/layer3_attention_weight_max": 100.96263885498047,
      "activations/layer3_attention_weight_min": -102.48356628417969,
      "activations/layer4_attention_weight_max": 106.43031311035156,
      "activations/layer4_attention_weight_min": -108.67597961425781,
      "activations/layer5_attention_weight_max": 50.507965087890625,
      "activations/layer5_attention_weight_min": -60.82407760620117,
      "activations/layer6_attention_weight_max": 44.00716781616211,
      "activations/layer6_attention_weight_min": -47.03135299682617,
      "activations/layer7_attention_weight_max": 95.30785369873047,
      "activations/layer7_attention_weight_min": -97.42926025390625,
      "activations/layer8_attention_weight_max": 42.205955505371094,
      "activations/layer8_attention_weight_min": -44.77519607543945,
      "activations/layer9_attention_weight_max": 31.714981079101562,
      "activations/layer9_attention_weight_min": -33.418914794921875,
      "epoch": 20.93,
      "learning_rate": 1.5201136363636363e-05,
      "loss": 2.7197,
      "step": 360150
    },
    {
      "activations/layer0_attention_weight_max": 17.0430850982666,
      "activations/layer0_attention_weight_min": -13.539240837097168,
      "activations/layer10_attention_weight_max": 35.59336471557617,
      "activations/layer10_attention_weight_min": -36.221561431884766,
      "activations/layer11_attention_weight_max": 36.55653762817383,
      "activations/layer11_attention_weight_min": -34.811607360839844,
      "activations/layer12_attention_weight_max": 27.47214698791504,
      "activations/layer12_attention_weight_min": -26.78449249267578,
      "activations/layer13_attention_weight_max": 40.45534133911133,
      "activations/layer13_attention_weight_min": -35.960723876953125,
      "activations/layer14_attention_weight_max": 41.14488983154297,
      "activations/layer14_attention_weight_min": -38.4006233215332,
      "activations/layer15_attention_weight_max": 37.546600341796875,
      "activations/layer15_attention_weight_min": -36.520904541015625,
      "activations/layer16_attention_weight_max": 29.79967498779297,
      "activations/layer16_attention_weight_min": -27.182743072509766,
      "activations/layer17_attention_weight_max": 28.7362003326416,
      "activations/layer17_attention_weight_min": -25.83306312561035,
      "activations/layer18_attention_weight_max": 32.06254196166992,
      "activations/layer18_attention_weight_min": -25.927391052246094,
      "activations/layer19_attention_weight_max": 33.5482292175293,
      "activations/layer19_attention_weight_min": -30.700727462768555,
      "activations/layer1_attention_weight_max": 15.20390510559082,
      "activations/layer1_attention_weight_min": -14.653718948364258,
      "activations/layer20_attention_weight_max": 28.284210205078125,
      "activations/layer20_attention_weight_min": -25.603992462158203,
      "activations/layer21_attention_weight_max": 32.93284225463867,
      "activations/layer21_attention_weight_min": -24.634836196899414,
      "activations/layer22_attention_weight_max": 41.96647262573242,
      "activations/layer22_attention_weight_min": -32.964290618896484,
      "activations/layer23_attention_weight_max": 31.554006576538086,
      "activations/layer23_attention_weight_min": -26.081192016601562,
      "activations/layer2_attention_weight_max": 33.09627151489258,
      "activations/layer2_attention_weight_min": -33.2376708984375,
      "activations/layer3_attention_weight_max": 104.54019165039062,
      "activations/layer3_attention_weight_min": -107.6372299194336,
      "activations/layer4_attention_weight_max": 108.05779266357422,
      "activations/layer4_attention_weight_min": -112.60072326660156,
      "activations/layer5_attention_weight_max": 51.62649154663086,
      "activations/layer5_attention_weight_min": -63.705955505371094,
      "activations/layer6_attention_weight_max": 43.40721893310547,
      "activations/layer6_attention_weight_min": -46.635032653808594,
      "activations/layer7_attention_weight_max": 96.4719009399414,
      "activations/layer7_attention_weight_min": -96.8662338256836,
      "activations/layer8_attention_weight_max": 44.39339065551758,
      "activations/layer8_attention_weight_min": -46.32841873168945,
      "activations/layer9_attention_weight_max": 34.367088317871094,
      "activations/layer9_attention_weight_min": -36.46947479248047,
      "epoch": 20.93,
      "learning_rate": 1.5182575757575757e-05,
      "loss": 2.692,
      "step": 360200
    },
    {
      "activations/layer0_attention_weight_max": 17.21298599243164,
      "activations/layer0_attention_weight_min": -13.40981674194336,
      "activations/layer10_attention_weight_max": 39.40206527709961,
      "activations/layer10_attention_weight_min": -37.72787857055664,
      "activations/layer11_attention_weight_max": 37.90552520751953,
      "activations/layer11_attention_weight_min": -39.3421630859375,
      "activations/layer12_attention_weight_max": 31.565523147583008,
      "activations/layer12_attention_weight_min": -30.83246612548828,
      "activations/layer13_attention_weight_max": 48.46399688720703,
      "activations/layer13_attention_weight_min": -40.38407897949219,
      "activations/layer14_attention_weight_max": 52.9100456237793,
      "activations/layer14_attention_weight_min": -44.857452392578125,
      "activations/layer15_attention_weight_max": 50.959136962890625,
      "activations/layer15_attention_weight_min": -40.58965301513672,
      "activations/layer16_attention_weight_max": 38.35957336425781,
      "activations/layer16_attention_weight_min": -33.637847900390625,
      "activations/layer17_attention_weight_max": 38.12704849243164,
      "activations/layer17_attention_weight_min": -31.216400146484375,
      "activations/layer18_attention_weight_max": 38.34297561645508,
      "activations/layer18_attention_weight_min": -30.939266204833984,
      "activations/layer19_attention_weight_max": 47.00200271606445,
      "activations/layer19_attention_weight_min": -36.57795333862305,
      "activations/layer1_attention_weight_max": 15.375862121582031,
      "activations/layer1_attention_weight_min": -13.9899320602417,
      "activations/layer20_attention_weight_max": 36.93914794921875,
      "activations/layer20_attention_weight_min": -26.484647750854492,
      "activations/layer21_attention_weight_max": 34.68305206298828,
      "activations/layer21_attention_weight_min": -25.933378219604492,
      "activations/layer22_attention_weight_max": 49.8225212097168,
      "activations/layer22_attention_weight_min": -36.151145935058594,
      "activations/layer23_attention_weight_max": 38.768653869628906,
      "activations/layer23_attention_weight_min": -27.38851547241211,
      "activations/layer2_attention_weight_max": 35.606285095214844,
      "activations/layer2_attention_weight_min": -35.760459899902344,
      "activations/layer3_attention_weight_max": 105.6391830444336,
      "activations/layer3_attention_weight_min": -107.89302062988281,
      "activations/layer4_attention_weight_max": 107.46905517578125,
      "activations/layer4_attention_weight_min": -111.13297271728516,
      "activations/layer5_attention_weight_max": 52.67186737060547,
      "activations/layer5_attention_weight_min": -61.653804779052734,
      "activations/layer6_attention_weight_max": 46.00463104248047,
      "activations/layer6_attention_weight_min": -48.740806579589844,
      "activations/layer7_attention_weight_max": 107.64276885986328,
      "activations/layer7_attention_weight_min": -103.08404541015625,
      "activations/layer8_attention_weight_max": 45.746437072753906,
      "activations/layer8_attention_weight_min": -47.20771026611328,
      "activations/layer9_attention_weight_max": 33.15542221069336,
      "activations/layer9_attention_weight_min": -37.94044876098633,
      "epoch": 20.93,
      "learning_rate": 1.5163636363636362e-05,
      "loss": 2.7008,
      "step": 360250
    },
    {
      "activations/layer0_attention_weight_max": 17.380361557006836,
      "activations/layer0_attention_weight_min": -16.22646141052246,
      "activations/layer10_attention_weight_max": 35.33723068237305,
      "activations/layer10_attention_weight_min": -34.733306884765625,
      "activations/layer11_attention_weight_max": 33.23118591308594,
      "activations/layer11_attention_weight_min": -34.6057014465332,
      "activations/layer12_attention_weight_max": 27.469972610473633,
      "activations/layer12_attention_weight_min": -26.009912490844727,
      "activations/layer13_attention_weight_max": 40.68373107910156,
      "activations/layer13_attention_weight_min": -34.28933334350586,
      "activations/layer14_attention_weight_max": 42.59955978393555,
      "activations/layer14_attention_weight_min": -37.45921325683594,
      "activations/layer15_attention_weight_max": 36.61894607543945,
      "activations/layer15_attention_weight_min": -35.07467269897461,
      "activations/layer16_attention_weight_max": 28.576082229614258,
      "activations/layer16_attention_weight_min": -31.037734985351562,
      "activations/layer17_attention_weight_max": 28.712656021118164,
      "activations/layer17_attention_weight_min": -26.526559829711914,
      "activations/layer18_attention_weight_max": 30.544527053833008,
      "activations/layer18_attention_weight_min": -26.401653289794922,
      "activations/layer19_attention_weight_max": 31.43122673034668,
      "activations/layer19_attention_weight_min": -31.820297241210938,
      "activations/layer1_attention_weight_max": 15.637524604797363,
      "activations/layer1_attention_weight_min": -14.35753345489502,
      "activations/layer20_attention_weight_max": 26.56926727294922,
      "activations/layer20_attention_weight_min": -26.02065086364746,
      "activations/layer21_attention_weight_max": 27.43016815185547,
      "activations/layer21_attention_weight_min": -25.2586612701416,
      "activations/layer22_attention_weight_max": 39.394248962402344,
      "activations/layer22_attention_weight_min": -31.90386962890625,
      "activations/layer23_attention_weight_max": 31.927764892578125,
      "activations/layer23_attention_weight_min": -25.814571380615234,
      "activations/layer2_attention_weight_max": 34.64661407470703,
      "activations/layer2_attention_weight_min": -34.42544174194336,
      "activations/layer3_attention_weight_max": 106.9187240600586,
      "activations/layer3_attention_weight_min": -107.00257873535156,
      "activations/layer4_attention_weight_max": 105.97562408447266,
      "activations/layer4_attention_weight_min": -111.9769058227539,
      "activations/layer5_attention_weight_max": 49.2458610534668,
      "activations/layer5_attention_weight_min": -61.525787353515625,
      "activations/layer6_attention_weight_max": 45.182743072509766,
      "activations/layer6_attention_weight_min": -47.644935607910156,
      "activations/layer7_attention_weight_max": 96.7226333618164,
      "activations/layer7_attention_weight_min": -98.94294738769531,
      "activations/layer8_attention_weight_max": 42.901912689208984,
      "activations/layer8_attention_weight_min": -44.99508285522461,
      "activations/layer9_attention_weight_max": 31.63680076599121,
      "activations/layer9_attention_weight_min": -33.421470642089844,
      "epoch": 20.94,
      "learning_rate": 1.5144696969696969e-05,
      "loss": 2.7118,
      "step": 360300
    },
    {
      "activations/layer0_attention_weight_max": 17.197629928588867,
      "activations/layer0_attention_weight_min": -13.324592590332031,
      "activations/layer10_attention_weight_max": 42.97829055786133,
      "activations/layer10_attention_weight_min": -40.33285903930664,
      "activations/layer11_attention_weight_max": 42.206459045410156,
      "activations/layer11_attention_weight_min": -43.25310516357422,
      "activations/layer12_attention_weight_max": 27.94621467590332,
      "activations/layer12_attention_weight_min": -27.00828742980957,
      "activations/layer13_attention_weight_max": 42.19868087768555,
      "activations/layer13_attention_weight_min": -36.44422912597656,
      "activations/layer14_attention_weight_max": 41.60321807861328,
      "activations/layer14_attention_weight_min": -37.598114013671875,
      "activations/layer15_attention_weight_max": 39.89700698852539,
      "activations/layer15_attention_weight_min": -39.48368453979492,
      "activations/layer16_attention_weight_max": 30.19230842590332,
      "activations/layer16_attention_weight_min": -28.1806583404541,
      "activations/layer17_attention_weight_max": 30.743072509765625,
      "activations/layer17_attention_weight_min": -28.29348373413086,
      "activations/layer18_attention_weight_max": 32.745426177978516,
      "activations/layer18_attention_weight_min": -28.71429443359375,
      "activations/layer19_attention_weight_max": 34.42397689819336,
      "activations/layer19_attention_weight_min": -34.63859558105469,
      "activations/layer1_attention_weight_max": 16.607406616210938,
      "activations/layer1_attention_weight_min": -14.402751922607422,
      "activations/layer20_attention_weight_max": 31.9300537109375,
      "activations/layer20_attention_weight_min": -27.081947326660156,
      "activations/layer21_attention_weight_max": 31.13541030883789,
      "activations/layer21_attention_weight_min": -26.312469482421875,
      "activations/layer22_attention_weight_max": 40.93468475341797,
      "activations/layer22_attention_weight_min": -34.877288818359375,
      "activations/layer23_attention_weight_max": 33.20947265625,
      "activations/layer23_attention_weight_min": -29.953161239624023,
      "activations/layer2_attention_weight_max": 35.727203369140625,
      "activations/layer2_attention_weight_min": -32.38037872314453,
      "activations/layer3_attention_weight_max": 105.89334106445312,
      "activations/layer3_attention_weight_min": -105.9406967163086,
      "activations/layer4_attention_weight_max": 110.9837646484375,
      "activations/layer4_attention_weight_min": -110.2277603149414,
      "activations/layer5_attention_weight_max": 54.64579391479492,
      "activations/layer5_attention_weight_min": -63.44654083251953,
      "activations/layer6_attention_weight_max": 43.68684005737305,
      "activations/layer6_attention_weight_min": -51.51714324951172,
      "activations/layer7_attention_weight_max": 110.70101165771484,
      "activations/layer7_attention_weight_min": -100.94225311279297,
      "activations/layer8_attention_weight_max": 49.8140869140625,
      "activations/layer8_attention_weight_min": -46.92018508911133,
      "activations/layer9_attention_weight_max": 35.9915657043457,
      "activations/layer9_attention_weight_min": -37.821903228759766,
      "epoch": 20.94,
      "learning_rate": 1.5125757575757574e-05,
      "loss": 2.7274,
      "step": 360350
    },
    {
      "activations/layer0_attention_weight_max": 16.600759506225586,
      "activations/layer0_attention_weight_min": -12.770601272583008,
      "activations/layer10_attention_weight_max": 31.51328468322754,
      "activations/layer10_attention_weight_min": -34.44624328613281,
      "activations/layer11_attention_weight_max": 31.775360107421875,
      "activations/layer11_attention_weight_min": -33.5315055847168,
      "activations/layer12_attention_weight_max": 27.327543258666992,
      "activations/layer12_attention_weight_min": -27.93316078186035,
      "activations/layer13_attention_weight_max": 41.2651481628418,
      "activations/layer13_attention_weight_min": -40.170928955078125,
      "activations/layer14_attention_weight_max": 43.83742904663086,
      "activations/layer14_attention_weight_min": -40.09026336669922,
      "activations/layer15_attention_weight_max": 39.20055389404297,
      "activations/layer15_attention_weight_min": -38.77993392944336,
      "activations/layer16_attention_weight_max": 29.04205894470215,
      "activations/layer16_attention_weight_min": -30.617467880249023,
      "activations/layer17_attention_weight_max": 30.112892150878906,
      "activations/layer17_attention_weight_min": -27.021997451782227,
      "activations/layer18_attention_weight_max": 32.457061767578125,
      "activations/layer18_attention_weight_min": -29.115943908691406,
      "activations/layer19_attention_weight_max": 31.727869033813477,
      "activations/layer19_attention_weight_min": -36.90723419189453,
      "activations/layer1_attention_weight_max": 15.94011402130127,
      "activations/layer1_attention_weight_min": -14.917608261108398,
      "activations/layer20_attention_weight_max": 27.217803955078125,
      "activations/layer20_attention_weight_min": -26.564830780029297,
      "activations/layer21_attention_weight_max": 27.852426528930664,
      "activations/layer21_attention_weight_min": -26.367687225341797,
      "activations/layer22_attention_weight_max": 42.36328125,
      "activations/layer22_attention_weight_min": -33.7938117980957,
      "activations/layer23_attention_weight_max": 31.023719787597656,
      "activations/layer23_attention_weight_min": -26.885005950927734,
      "activations/layer2_attention_weight_max": 32.70191192626953,
      "activations/layer2_attention_weight_min": -31.11249351501465,
      "activations/layer3_attention_weight_max": 101.4732437133789,
      "activations/layer3_attention_weight_min": -102.4106674194336,
      "activations/layer4_attention_weight_max": 103.216064453125,
      "activations/layer4_attention_weight_min": -108.35518646240234,
      "activations/layer5_attention_weight_max": 49.32914352416992,
      "activations/layer5_attention_weight_min": -64.0068359375,
      "activations/layer6_attention_weight_max": 43.355953216552734,
      "activations/layer6_attention_weight_min": -49.3690071105957,
      "activations/layer7_attention_weight_max": 92.97685241699219,
      "activations/layer7_attention_weight_min": -101.3132553100586,
      "activations/layer8_attention_weight_max": 39.85075759887695,
      "activations/layer8_attention_weight_min": -45.66373825073242,
      "activations/layer9_attention_weight_max": 29.962888717651367,
      "activations/layer9_attention_weight_min": -33.1621208190918,
      "epoch": 20.94,
      "learning_rate": 1.510681818181818e-05,
      "loss": 2.7246,
      "step": 360400
    },
    {
      "activations/layer0_attention_weight_max": 17.08090591430664,
      "activations/layer0_attention_weight_min": -13.648038864135742,
      "activations/layer10_attention_weight_max": 32.40534591674805,
      "activations/layer10_attention_weight_min": -35.966407775878906,
      "activations/layer11_attention_weight_max": 34.34101867675781,
      "activations/layer11_attention_weight_min": -34.26921463012695,
      "activations/layer12_attention_weight_max": 26.08783721923828,
      "activations/layer12_attention_weight_min": -26.945093154907227,
      "activations/layer13_attention_weight_max": 36.492244720458984,
      "activations/layer13_attention_weight_min": -35.10674285888672,
      "activations/layer14_attention_weight_max": 43.54753112792969,
      "activations/layer14_attention_weight_min": -37.4223747253418,
      "activations/layer15_attention_weight_max": 38.06318664550781,
      "activations/layer15_attention_weight_min": -35.44097137451172,
      "activations/layer16_attention_weight_max": 30.565961837768555,
      "activations/layer16_attention_weight_min": -28.419511795043945,
      "activations/layer17_attention_weight_max": 31.468286514282227,
      "activations/layer17_attention_weight_min": -25.318822860717773,
      "activations/layer18_attention_weight_max": 32.377681732177734,
      "activations/layer18_attention_weight_min": -25.45890998840332,
      "activations/layer19_attention_weight_max": 36.62623596191406,
      "activations/layer19_attention_weight_min": -31.322315216064453,
      "activations/layer1_attention_weight_max": 15.783644676208496,
      "activations/layer1_attention_weight_min": -15.241840362548828,
      "activations/layer20_attention_weight_max": 30.840879440307617,
      "activations/layer20_attention_weight_min": -24.83254623413086,
      "activations/layer21_attention_weight_max": 31.656505584716797,
      "activations/layer21_attention_weight_min": -25.150083541870117,
      "activations/layer22_attention_weight_max": 44.73933410644531,
      "activations/layer22_attention_weight_min": -32.53361892700195,
      "activations/layer23_attention_weight_max": 31.124286651611328,
      "activations/layer23_attention_weight_min": -25.640071868896484,
      "activations/layer2_attention_weight_max": 33.30937576293945,
      "activations/layer2_attention_weight_min": -31.43762969970703,
      "activations/layer3_attention_weight_max": 106.12692260742188,
      "activations/layer3_attention_weight_min": -106.20166015625,
      "activations/layer4_attention_weight_max": 111.83326721191406,
      "activations/layer4_attention_weight_min": -112.7649154663086,
      "activations/layer5_attention_weight_max": 49.79118347167969,
      "activations/layer5_attention_weight_min": -65.45133972167969,
      "activations/layer6_attention_weight_max": 45.04804992675781,
      "activations/layer6_attention_weight_min": -48.49217987060547,
      "activations/layer7_attention_weight_max": 95.30113220214844,
      "activations/layer7_attention_weight_min": -97.85152435302734,
      "activations/layer8_attention_weight_max": 40.746883392333984,
      "activations/layer8_attention_weight_min": -44.23341369628906,
      "activations/layer9_attention_weight_max": 31.247777938842773,
      "activations/layer9_attention_weight_min": -33.787322998046875,
      "epoch": 20.94,
      "learning_rate": 1.5087878787878787e-05,
      "loss": 2.7123,
      "step": 360450
    },
    {
      "activations/layer0_attention_weight_max": 16.50369644165039,
      "activations/layer0_attention_weight_min": -13.578804969787598,
      "activations/layer10_attention_weight_max": 36.83447265625,
      "activations/layer10_attention_weight_min": -37.22774887084961,
      "activations/layer11_attention_weight_max": 38.88151931762695,
      "activations/layer11_attention_weight_min": -36.06111526489258,
      "activations/layer12_attention_weight_max": 28.617094039916992,
      "activations/layer12_attention_weight_min": -26.851316452026367,
      "activations/layer13_attention_weight_max": 38.568084716796875,
      "activations/layer13_attention_weight_min": -37.29225158691406,
      "activations/layer14_attention_weight_max": 42.80314254760742,
      "activations/layer14_attention_weight_min": -40.038028717041016,
      "activations/layer15_attention_weight_max": 41.54426956176758,
      "activations/layer15_attention_weight_min": -36.15523147583008,
      "activations/layer16_attention_weight_max": 30.7515869140625,
      "activations/layer16_attention_weight_min": -29.48514747619629,
      "activations/layer17_attention_weight_max": 28.852813720703125,
      "activations/layer17_attention_weight_min": -26.254201889038086,
      "activations/layer18_attention_weight_max": 32.88998794555664,
      "activations/layer18_attention_weight_min": -26.107454299926758,
      "activations/layer19_attention_weight_max": 36.1173210144043,
      "activations/layer19_attention_weight_min": -33.36465072631836,
      "activations/layer1_attention_weight_max": 15.793902397155762,
      "activations/layer1_attention_weight_min": -15.006258964538574,
      "activations/layer20_attention_weight_max": 31.27570152282715,
      "activations/layer20_attention_weight_min": -26.446292877197266,
      "activations/layer21_attention_weight_max": 29.430156707763672,
      "activations/layer21_attention_weight_min": -25.217477798461914,
      "activations/layer22_attention_weight_max": 43.5386962890625,
      "activations/layer22_attention_weight_min": -31.67745590209961,
      "activations/layer23_attention_weight_max": 33.93023681640625,
      "activations/layer23_attention_weight_min": -27.741065979003906,
      "activations/layer2_attention_weight_max": 35.95670700073242,
      "activations/layer2_attention_weight_min": -33.43520736694336,
      "activations/layer3_attention_weight_max": 108.05901336669922,
      "activations/layer3_attention_weight_min": -106.4127426147461,
      "activations/layer4_attention_weight_max": 112.67694091796875,
      "activations/layer4_attention_weight_min": -109.90864562988281,
      "activations/layer5_attention_weight_max": 52.74587631225586,
      "activations/layer5_attention_weight_min": -61.48633575439453,
      "activations/layer6_attention_weight_max": 42.89406204223633,
      "activations/layer6_attention_weight_min": -48.9917106628418,
      "activations/layer7_attention_weight_max": 96.076904296875,
      "activations/layer7_attention_weight_min": -95.58916473388672,
      "activations/layer8_attention_weight_max": 43.76457214355469,
      "activations/layer8_attention_weight_min": -45.304931640625,
      "activations/layer9_attention_weight_max": 30.413427352905273,
      "activations/layer9_attention_weight_min": -34.72209167480469,
      "epoch": 20.95,
      "learning_rate": 1.5068939393939392e-05,
      "loss": 2.7125,
      "step": 360500
    },
    {
      "activations/layer0_attention_weight_max": 16.64437484741211,
      "activations/layer0_attention_weight_min": -14.239960670471191,
      "activations/layer10_attention_weight_max": 38.0391960144043,
      "activations/layer10_attention_weight_min": -38.619361877441406,
      "activations/layer11_attention_weight_max": 36.05852508544922,
      "activations/layer11_attention_weight_min": -36.83293914794922,
      "activations/layer12_attention_weight_max": 31.051042556762695,
      "activations/layer12_attention_weight_min": -28.326627731323242,
      "activations/layer13_attention_weight_max": 47.57836151123047,
      "activations/layer13_attention_weight_min": -40.776878356933594,
      "activations/layer14_attention_weight_max": 49.891788482666016,
      "activations/layer14_attention_weight_min": -43.135475158691406,
      "activations/layer15_attention_weight_max": 46.74802017211914,
      "activations/layer15_attention_weight_min": -39.60344696044922,
      "activations/layer16_attention_weight_max": 35.25875473022461,
      "activations/layer16_attention_weight_min": -29.157089233398438,
      "activations/layer17_attention_weight_max": 35.24093246459961,
      "activations/layer17_attention_weight_min": -28.155506134033203,
      "activations/layer18_attention_weight_max": 38.46518325805664,
      "activations/layer18_attention_weight_min": -27.948974609375,
      "activations/layer19_attention_weight_max": 41.12386703491211,
      "activations/layer19_attention_weight_min": -34.90099334716797,
      "activations/layer1_attention_weight_max": 15.495140075683594,
      "activations/layer1_attention_weight_min": -15.586071014404297,
      "activations/layer20_attention_weight_max": 35.87744903564453,
      "activations/layer20_attention_weight_min": -27.082857131958008,
      "activations/layer21_attention_weight_max": 34.31289291381836,
      "activations/layer21_attention_weight_min": -27.36151123046875,
      "activations/layer22_attention_weight_max": 54.67319107055664,
      "activations/layer22_attention_weight_min": -34.911827087402344,
      "activations/layer23_attention_weight_max": 39.85850524902344,
      "activations/layer23_attention_weight_min": -26.953269958496094,
      "activations/layer2_attention_weight_max": 35.2771110534668,
      "activations/layer2_attention_weight_min": -32.40480041503906,
      "activations/layer3_attention_weight_max": 103.78926086425781,
      "activations/layer3_attention_weight_min": -106.71944427490234,
      "activations/layer4_attention_weight_max": 110.84547424316406,
      "activations/layer4_attention_weight_min": -109.88655090332031,
      "activations/layer5_attention_weight_max": 52.95719528198242,
      "activations/layer5_attention_weight_min": -64.98072814941406,
      "activations/layer6_attention_weight_max": 46.887996673583984,
      "activations/layer6_attention_weight_min": -54.83822250366211,
      "activations/layer7_attention_weight_max": 103.69705963134766,
      "activations/layer7_attention_weight_min": -118.46994018554688,
      "activations/layer8_attention_weight_max": 47.82586669921875,
      "activations/layer8_attention_weight_min": -52.80839920043945,
      "activations/layer9_attention_weight_max": 35.17079162597656,
      "activations/layer9_attention_weight_min": -40.003013610839844,
      "epoch": 20.95,
      "learning_rate": 1.5049999999999998e-05,
      "loss": 2.7114,
      "step": 360550
    },
    {
      "activations/layer0_attention_weight_max": 16.231382369995117,
      "activations/layer0_attention_weight_min": -13.411781311035156,
      "activations/layer10_attention_weight_max": 37.611446380615234,
      "activations/layer10_attention_weight_min": -37.7485237121582,
      "activations/layer11_attention_weight_max": 37.42664337158203,
      "activations/layer11_attention_weight_min": -38.30872344970703,
      "activations/layer12_attention_weight_max": 29.711872100830078,
      "activations/layer12_attention_weight_min": -28.445344924926758,
      "activations/layer13_attention_weight_max": 43.29106903076172,
      "activations/layer13_attention_weight_min": -38.46617889404297,
      "activations/layer14_attention_weight_max": 43.537620544433594,
      "activations/layer14_attention_weight_min": -39.595027923583984,
      "activations/layer15_attention_weight_max": 42.48623275756836,
      "activations/layer15_attention_weight_min": -37.283447265625,
      "activations/layer16_attention_weight_max": 30.794139862060547,
      "activations/layer16_attention_weight_min": -29.485361099243164,
      "activations/layer17_attention_weight_max": 30.844690322875977,
      "activations/layer17_attention_weight_min": -28.243417739868164,
      "activations/layer18_attention_weight_max": 36.096282958984375,
      "activations/layer18_attention_weight_min": -27.92034339904785,
      "activations/layer19_attention_weight_max": 35.82619857788086,
      "activations/layer19_attention_weight_min": -33.83488082885742,
      "activations/layer1_attention_weight_max": 16.3502140045166,
      "activations/layer1_attention_weight_min": -16.29744529724121,
      "activations/layer20_attention_weight_max": 28.589611053466797,
      "activations/layer20_attention_weight_min": -24.801002502441406,
      "activations/layer21_attention_weight_max": 28.92740821838379,
      "activations/layer21_attention_weight_min": -26.244993209838867,
      "activations/layer22_attention_weight_max": 44.60438537597656,
      "activations/layer22_attention_weight_min": -32.45970916748047,
      "activations/layer23_attention_weight_max": 33.251556396484375,
      "activations/layer23_attention_weight_min": -27.26187515258789,
      "activations/layer2_attention_weight_max": 33.16313552856445,
      "activations/layer2_attention_weight_min": -34.277217864990234,
      "activations/layer3_attention_weight_max": 107.10995483398438,
      "activations/layer3_attention_weight_min": -105.54478454589844,
      "activations/layer4_attention_weight_max": 109.2597427368164,
      "activations/layer4_attention_weight_min": -112.7693862915039,
      "activations/layer5_attention_weight_max": 52.29621124267578,
      "activations/layer5_attention_weight_min": -63.141258239746094,
      "activations/layer6_attention_weight_max": 46.92123031616211,
      "activations/layer6_attention_weight_min": -51.82227325439453,
      "activations/layer7_attention_weight_max": 96.49031066894531,
      "activations/layer7_attention_weight_min": -105.37549591064453,
      "activations/layer8_attention_weight_max": 46.09121322631836,
      "activations/layer8_attention_weight_min": -47.905670166015625,
      "activations/layer9_attention_weight_max": 36.562591552734375,
      "activations/layer9_attention_weight_min": -37.3629150390625,
      "epoch": 20.95,
      "learning_rate": 1.5031060606060605e-05,
      "loss": 2.7148,
      "step": 360600
    },
    {
      "activations/layer0_attention_weight_max": 16.67146873474121,
      "activations/layer0_attention_weight_min": -13.468836784362793,
      "activations/layer10_attention_weight_max": 37.7661247253418,
      "activations/layer10_attention_weight_min": -39.147804260253906,
      "activations/layer11_attention_weight_max": 37.91782760620117,
      "activations/layer11_attention_weight_min": -42.39917755126953,
      "activations/layer12_attention_weight_max": 28.968162536621094,
      "activations/layer12_attention_weight_min": -32.059226989746094,
      "activations/layer13_attention_weight_max": 40.159515380859375,
      "activations/layer13_attention_weight_min": -39.55918884277344,
      "activations/layer14_attention_weight_max": 46.253013610839844,
      "activations/layer14_attention_weight_min": -45.18550109863281,
      "activations/layer15_attention_weight_max": 41.707584381103516,
      "activations/layer15_attention_weight_min": -44.3235969543457,
      "activations/layer16_attention_weight_max": 30.547204971313477,
      "activations/layer16_attention_weight_min": -30.392309188842773,
      "activations/layer17_attention_weight_max": 32.35347366333008,
      "activations/layer17_attention_weight_min": -30.6938419342041,
      "activations/layer18_attention_weight_max": 31.291851043701172,
      "activations/layer18_attention_weight_min": -28.793575286865234,
      "activations/layer19_attention_weight_max": 33.665306091308594,
      "activations/layer19_attention_weight_min": -33.6291618347168,
      "activations/layer1_attention_weight_max": 16.481767654418945,
      "activations/layer1_attention_weight_min": -15.325348854064941,
      "activations/layer20_attention_weight_max": 28.229373931884766,
      "activations/layer20_attention_weight_min": -27.413896560668945,
      "activations/layer21_attention_weight_max": 28.38186264038086,
      "activations/layer21_attention_weight_min": -26.252300262451172,
      "activations/layer22_attention_weight_max": 42.100181579589844,
      "activations/layer22_attention_weight_min": -34.88129425048828,
      "activations/layer23_attention_weight_max": 31.39922332763672,
      "activations/layer23_attention_weight_min": -26.44244384765625,
      "activations/layer2_attention_weight_max": 33.423484802246094,
      "activations/layer2_attention_weight_min": -33.849388122558594,
      "activations/layer3_attention_weight_max": 102.1317138671875,
      "activations/layer3_attention_weight_min": -105.85530853271484,
      "activations/layer4_attention_weight_max": 102.64947509765625,
      "activations/layer4_attention_weight_min": -109.20135498046875,
      "activations/layer5_attention_weight_max": 48.170040130615234,
      "activations/layer5_attention_weight_min": -62.40685272216797,
      "activations/layer6_attention_weight_max": 43.051673889160156,
      "activations/layer6_attention_weight_min": -46.96134567260742,
      "activations/layer7_attention_weight_max": 98.41181945800781,
      "activations/layer7_attention_weight_min": -101.18985748291016,
      "activations/layer8_attention_weight_max": 43.864585876464844,
      "activations/layer8_attention_weight_min": -47.98468017578125,
      "activations/layer9_attention_weight_max": 34.08353805541992,
      "activations/layer9_attention_weight_min": -37.20882797241211,
      "epoch": 20.96,
      "learning_rate": 1.501212121212121e-05,
      "loss": 2.7072,
      "step": 360650
    },
    {
      "activations/layer0_attention_weight_max": 16.53872299194336,
      "activations/layer0_attention_weight_min": -14.456978797912598,
      "activations/layer10_attention_weight_max": 36.46268844604492,
      "activations/layer10_attention_weight_min": -36.320960998535156,
      "activations/layer11_attention_weight_max": 34.52413558959961,
      "activations/layer11_attention_weight_min": -35.643959045410156,
      "activations/layer12_attention_weight_max": 28.638078689575195,
      "activations/layer12_attention_weight_min": -27.33465003967285,
      "activations/layer13_attention_weight_max": 39.4173583984375,
      "activations/layer13_attention_weight_min": -36.191349029541016,
      "activations/layer14_attention_weight_max": 42.49324417114258,
      "activations/layer14_attention_weight_min": -39.05365753173828,
      "activations/layer15_attention_weight_max": 37.73341369628906,
      "activations/layer15_attention_weight_min": -35.52576446533203,
      "activations/layer16_attention_weight_max": 28.50615882873535,
      "activations/layer16_attention_weight_min": -29.960285186767578,
      "activations/layer17_attention_weight_max": 28.844438552856445,
      "activations/layer17_attention_weight_min": -26.77226448059082,
      "activations/layer18_attention_weight_max": 32.48081588745117,
      "activations/layer18_attention_weight_min": -25.34467124938965,
      "activations/layer19_attention_weight_max": 32.5362663269043,
      "activations/layer19_attention_weight_min": -32.334346771240234,
      "activations/layer1_attention_weight_max": 15.715875625610352,
      "activations/layer1_attention_weight_min": -15.720528602600098,
      "activations/layer20_attention_weight_max": 29.258033752441406,
      "activations/layer20_attention_weight_min": -24.804807662963867,
      "activations/layer21_attention_weight_max": 28.704238891601562,
      "activations/layer21_attention_weight_min": -25.45119285583496,
      "activations/layer22_attention_weight_max": 39.952152252197266,
      "activations/layer22_attention_weight_min": -31.554306030273438,
      "activations/layer23_attention_weight_max": 29.911766052246094,
      "activations/layer23_attention_weight_min": -25.63300323486328,
      "activations/layer2_attention_weight_max": 33.99543762207031,
      "activations/layer2_attention_weight_min": -32.41810989379883,
      "activations/layer3_attention_weight_max": 110.51499938964844,
      "activations/layer3_attention_weight_min": -107.92803192138672,
      "activations/layer4_attention_weight_max": 112.35774993896484,
      "activations/layer4_attention_weight_min": -111.69661712646484,
      "activations/layer5_attention_weight_max": 51.52619171142578,
      "activations/layer5_attention_weight_min": -61.70268249511719,
      "activations/layer6_attention_weight_max": 45.73971176147461,
      "activations/layer6_attention_weight_min": -48.31864547729492,
      "activations/layer7_attention_weight_max": 97.46965789794922,
      "activations/layer7_attention_weight_min": -104.84040069580078,
      "activations/layer8_attention_weight_max": 43.136070251464844,
      "activations/layer8_attention_weight_min": -46.78147888183594,
      "activations/layer9_attention_weight_max": 31.940946578979492,
      "activations/layer9_attention_weight_min": -35.17208480834961,
      "epoch": 20.96,
      "learning_rate": 1.4993181818181816e-05,
      "loss": 2.7233,
      "step": 360700
    },
    {
      "activations/layer0_attention_weight_max": 16.690601348876953,
      "activations/layer0_attention_weight_min": -13.861138343811035,
      "activations/layer10_attention_weight_max": 33.0811767578125,
      "activations/layer10_attention_weight_min": -34.53826904296875,
      "activations/layer11_attention_weight_max": 33.85634231567383,
      "activations/layer11_attention_weight_min": -35.67401885986328,
      "activations/layer12_attention_weight_max": 33.37652587890625,
      "activations/layer12_attention_weight_min": -27.318296432495117,
      "activations/layer13_attention_weight_max": 43.93735122680664,
      "activations/layer13_attention_weight_min": -37.34007263183594,
      "activations/layer14_attention_weight_max": 39.55766296386719,
      "activations/layer14_attention_weight_min": -38.52109146118164,
      "activations/layer15_attention_weight_max": 39.740447998046875,
      "activations/layer15_attention_weight_min": -37.921852111816406,
      "activations/layer16_attention_weight_max": 29.76663589477539,
      "activations/layer16_attention_weight_min": -27.935861587524414,
      "activations/layer17_attention_weight_max": 27.311023712158203,
      "activations/layer17_attention_weight_min": -26.30153465270996,
      "activations/layer18_attention_weight_max": 32.46765899658203,
      "activations/layer18_attention_weight_min": -27.497148513793945,
      "activations/layer19_attention_weight_max": 29.927785873413086,
      "activations/layer19_attention_weight_min": -31.673538208007812,
      "activations/layer1_attention_weight_max": 15.667556762695312,
      "activations/layer1_attention_weight_min": -14.43532657623291,
      "activations/layer20_attention_weight_max": 25.499116897583008,
      "activations/layer20_attention_weight_min": -25.566415786743164,
      "activations/layer21_attention_weight_max": 27.377685546875,
      "activations/layer21_attention_weight_min": -24.22675132751465,
      "activations/layer22_attention_weight_max": 37.76081085205078,
      "activations/layer22_attention_weight_min": -32.355125427246094,
      "activations/layer23_attention_weight_max": 31.1159725189209,
      "activations/layer23_attention_weight_min": -25.930034637451172,
      "activations/layer2_attention_weight_max": 34.17042541503906,
      "activations/layer2_attention_weight_min": -32.42536926269531,
      "activations/layer3_attention_weight_max": 101.704345703125,
      "activations/layer3_attention_weight_min": -102.91897583007812,
      "activations/layer4_attention_weight_max": 104.90416717529297,
      "activations/layer4_attention_weight_min": -107.46107482910156,
      "activations/layer5_attention_weight_max": 48.26903533935547,
      "activations/layer5_attention_weight_min": -61.965850830078125,
      "activations/layer6_attention_weight_max": 44.18069076538086,
      "activations/layer6_attention_weight_min": -46.682254791259766,
      "activations/layer7_attention_weight_max": 93.08995056152344,
      "activations/layer7_attention_weight_min": -94.97669982910156,
      "activations/layer8_attention_weight_max": 43.007930755615234,
      "activations/layer8_attention_weight_min": -43.229209899902344,
      "activations/layer9_attention_weight_max": 31.525671005249023,
      "activations/layer9_attention_weight_min": -34.90475845336914,
      "epoch": 20.96,
      "learning_rate": 1.4974242424242421e-05,
      "loss": 2.7127,
      "step": 360750
    },
    {
      "activations/layer0_attention_weight_max": 17.012073516845703,
      "activations/layer0_attention_weight_min": -15.529853820800781,
      "activations/layer10_attention_weight_max": 37.88738250732422,
      "activations/layer10_attention_weight_min": -35.727237701416016,
      "activations/layer11_attention_weight_max": 40.12818145751953,
      "activations/layer11_attention_weight_min": -38.41785430908203,
      "activations/layer12_attention_weight_max": 27.748737335205078,
      "activations/layer12_attention_weight_min": -27.23249626159668,
      "activations/layer13_attention_weight_max": 42.95780563354492,
      "activations/layer13_attention_weight_min": -36.33766555786133,
      "activations/layer14_attention_weight_max": 47.99211502075195,
      "activations/layer14_attention_weight_min": -39.727699279785156,
      "activations/layer15_attention_weight_max": 43.9312629699707,
      "activations/layer15_attention_weight_min": -36.59994125366211,
      "activations/layer16_attention_weight_max": 31.486507415771484,
      "activations/layer16_attention_weight_min": -28.80906867980957,
      "activations/layer17_attention_weight_max": 37.03062057495117,
      "activations/layer17_attention_weight_min": -26.70220375061035,
      "activations/layer18_attention_weight_max": 31.959796905517578,
      "activations/layer18_attention_weight_min": -24.99526023864746,
      "activations/layer19_attention_weight_max": 34.1952018737793,
      "activations/layer19_attention_weight_min": -31.13982582092285,
      "activations/layer1_attention_weight_max": 16.85606575012207,
      "activations/layer1_attention_weight_min": -15.533417701721191,
      "activations/layer20_attention_weight_max": 29.42168426513672,
      "activations/layer20_attention_weight_min": -25.465404510498047,
      "activations/layer21_attention_weight_max": 28.56377410888672,
      "activations/layer21_attention_weight_min": -25.416656494140625,
      "activations/layer22_attention_weight_max": 39.1722526550293,
      "activations/layer22_attention_weight_min": -29.635374069213867,
      "activations/layer23_attention_weight_max": 32.558631896972656,
      "activations/layer23_attention_weight_min": -27.097333908081055,
      "activations/layer2_attention_weight_max": 35.550025939941406,
      "activations/layer2_attention_weight_min": -34.09035873413086,
      "activations/layer3_attention_weight_max": 105.2144775390625,
      "activations/layer3_attention_weight_min": -107.587158203125,
      "activations/layer4_attention_weight_max": 109.43309783935547,
      "activations/layer4_attention_weight_min": -112.23150634765625,
      "activations/layer5_attention_weight_max": 50.73326110839844,
      "activations/layer5_attention_weight_min": -63.58551025390625,
      "activations/layer6_attention_weight_max": 46.00989532470703,
      "activations/layer6_attention_weight_min": -49.54566192626953,
      "activations/layer7_attention_weight_max": 99.22578430175781,
      "activations/layer7_attention_weight_min": -99.50511169433594,
      "activations/layer8_attention_weight_max": 43.82781219482422,
      "activations/layer8_attention_weight_min": -47.28401565551758,
      "activations/layer9_attention_weight_max": 34.257080078125,
      "activations/layer9_attention_weight_min": -36.321163177490234,
      "epoch": 20.96,
      "learning_rate": 1.495530303030303e-05,
      "loss": 2.709,
      "step": 360800
    },
    {
      "activations/layer0_attention_weight_max": 16.703283309936523,
      "activations/layer0_attention_weight_min": -13.40365219116211,
      "activations/layer10_attention_weight_max": 47.77919387817383,
      "activations/layer10_attention_weight_min": -43.44300842285156,
      "activations/layer11_attention_weight_max": 49.87238693237305,
      "activations/layer11_attention_weight_min": -46.15704345703125,
      "activations/layer12_attention_weight_max": 33.43669509887695,
      "activations/layer12_attention_weight_min": -32.14631652832031,
      "activations/layer13_attention_weight_max": 45.550411224365234,
      "activations/layer13_attention_weight_min": -37.50775909423828,
      "activations/layer14_attention_weight_max": 49.13948059082031,
      "activations/layer14_attention_weight_min": -41.63589859008789,
      "activations/layer15_attention_weight_max": 48.38946533203125,
      "activations/layer15_attention_weight_min": -42.27995681762695,
      "activations/layer16_attention_weight_max": 35.03253173828125,
      "activations/layer16_attention_weight_min": -29.526981353759766,
      "activations/layer17_attention_weight_max": 32.09514236450195,
      "activations/layer17_attention_weight_min": -28.36791229248047,
      "activations/layer18_attention_weight_max": 35.119991302490234,
      "activations/layer18_attention_weight_min": -28.09056282043457,
      "activations/layer19_attention_weight_max": 37.14639663696289,
      "activations/layer19_attention_weight_min": -33.2128791809082,
      "activations/layer1_attention_weight_max": 15.44744873046875,
      "activations/layer1_attention_weight_min": -15.340478897094727,
      "activations/layer20_attention_weight_max": 27.90312957763672,
      "activations/layer20_attention_weight_min": -25.55361557006836,
      "activations/layer21_attention_weight_max": 28.38918113708496,
      "activations/layer21_attention_weight_min": -24.45317268371582,
      "activations/layer22_attention_weight_max": 40.86613082885742,
      "activations/layer22_attention_weight_min": -32.56268310546875,
      "activations/layer23_attention_weight_max": 35.375885009765625,
      "activations/layer23_attention_weight_min": -26.94635009765625,
      "activations/layer2_attention_weight_max": 37.16645431518555,
      "activations/layer2_attention_weight_min": -34.3383674621582,
      "activations/layer3_attention_weight_max": 109.33447265625,
      "activations/layer3_attention_weight_min": -109.44286346435547,
      "activations/layer4_attention_weight_max": 116.9261703491211,
      "activations/layer4_attention_weight_min": -111.09832763671875,
      "activations/layer5_attention_weight_max": 50.4058952331543,
      "activations/layer5_attention_weight_min": -61.54301071166992,
      "activations/layer6_attention_weight_max": 46.85655975341797,
      "activations/layer6_attention_weight_min": -50.490665435791016,
      "activations/layer7_attention_weight_max": 108.45030975341797,
      "activations/layer7_attention_weight_min": -105.97994232177734,
      "activations/layer8_attention_weight_max": 51.41124725341797,
      "activations/layer8_attention_weight_min": -52.347740173339844,
      "activations/layer9_attention_weight_max": 38.59496307373047,
      "activations/layer9_attention_weight_min": -41.10763168334961,
      "epoch": 20.97,
      "learning_rate": 1.4936363636363636e-05,
      "loss": 2.7071,
      "step": 360850
    },
    {
      "activations/layer0_attention_weight_max": 16.42941665649414,
      "activations/layer0_attention_weight_min": -13.35992431640625,
      "activations/layer10_attention_weight_max": 40.510963439941406,
      "activations/layer10_attention_weight_min": -38.4658088684082,
      "activations/layer11_attention_weight_max": 41.77172088623047,
      "activations/layer11_attention_weight_min": -38.669822692871094,
      "activations/layer12_attention_weight_max": 32.470218658447266,
      "activations/layer12_attention_weight_min": -30.80316734313965,
      "activations/layer13_attention_weight_max": 48.90627670288086,
      "activations/layer13_attention_weight_min": -40.09244918823242,
      "activations/layer14_attention_weight_max": 55.03413772583008,
      "activations/layer14_attention_weight_min": -44.43518829345703,
      "activations/layer15_attention_weight_max": 51.20757293701172,
      "activations/layer15_attention_weight_min": -42.09931945800781,
      "activations/layer16_attention_weight_max": 37.650535583496094,
      "activations/layer16_attention_weight_min": -30.34520149230957,
      "activations/layer17_attention_weight_max": 34.971405029296875,
      "activations/layer17_attention_weight_min": -28.88787269592285,
      "activations/layer18_attention_weight_max": 37.154632568359375,
      "activations/layer18_attention_weight_min": -27.509639739990234,
      "activations/layer19_attention_weight_max": 42.14014434814453,
      "activations/layer19_attention_weight_min": -32.38496398925781,
      "activations/layer1_attention_weight_max": 15.457064628601074,
      "activations/layer1_attention_weight_min": -14.296991348266602,
      "activations/layer20_attention_weight_max": 35.78478240966797,
      "activations/layer20_attention_weight_min": -24.771289825439453,
      "activations/layer21_attention_weight_max": 36.995609283447266,
      "activations/layer21_attention_weight_min": -24.368572235107422,
      "activations/layer22_attention_weight_max": 53.58645248413086,
      "activations/layer22_attention_weight_min": -35.223201751708984,
      "activations/layer23_attention_weight_max": 39.28435134887695,
      "activations/layer23_attention_weight_min": -24.2802791595459,
      "activations/layer2_attention_weight_max": 31.391498565673828,
      "activations/layer2_attention_weight_min": -30.062437057495117,
      "activations/layer3_attention_weight_max": 99.97888946533203,
      "activations/layer3_attention_weight_min": -96.11705780029297,
      "activations/layer4_attention_weight_max": 104.88885498046875,
      "activations/layer4_attention_weight_min": -104.17972564697266,
      "activations/layer5_attention_weight_max": 48.793006896972656,
      "activations/layer5_attention_weight_min": -57.86253356933594,
      "activations/layer6_attention_weight_max": 45.09230422973633,
      "activations/layer6_attention_weight_min": -47.07258224487305,
      "activations/layer7_attention_weight_max": 96.52154541015625,
      "activations/layer7_attention_weight_min": -104.48820495605469,
      "activations/layer8_attention_weight_max": 45.44990158081055,
      "activations/layer8_attention_weight_min": -47.1453742980957,
      "activations/layer9_attention_weight_max": 35.91023254394531,
      "activations/layer9_attention_weight_min": -36.973751068115234,
      "epoch": 20.97,
      "learning_rate": 1.4917424242424243e-05,
      "loss": 2.7207,
      "step": 360900
    },
    {
      "activations/layer0_attention_weight_max": 16.034669876098633,
      "activations/layer0_attention_weight_min": -14.888150215148926,
      "activations/layer10_attention_weight_max": 41.59791946411133,
      "activations/layer10_attention_weight_min": -42.14213562011719,
      "activations/layer11_attention_weight_max": 40.27680587768555,
      "activations/layer11_attention_weight_min": -42.27778625488281,
      "activations/layer12_attention_weight_max": 33.0677375793457,
      "activations/layer12_attention_weight_min": -30.11414337158203,
      "activations/layer13_attention_weight_max": 48.4283561706543,
      "activations/layer13_attention_weight_min": -39.947181701660156,
      "activations/layer14_attention_weight_max": 49.12252426147461,
      "activations/layer14_attention_weight_min": -42.764835357666016,
      "activations/layer15_attention_weight_max": 46.174678802490234,
      "activations/layer15_attention_weight_min": -39.08778381347656,
      "activations/layer16_attention_weight_max": 33.76605987548828,
      "activations/layer16_attention_weight_min": -29.5161190032959,
      "activations/layer17_attention_weight_max": 35.050010681152344,
      "activations/layer17_attention_weight_min": -27.62057876586914,
      "activations/layer18_attention_weight_max": 34.85746765136719,
      "activations/layer18_attention_weight_min": -25.365459442138672,
      "activations/layer19_attention_weight_max": 42.209049224853516,
      "activations/layer19_attention_weight_min": -33.61821746826172,
      "activations/layer1_attention_weight_max": 15.366141319274902,
      "activations/layer1_attention_weight_min": -14.02994155883789,
      "activations/layer20_attention_weight_max": 35.84770202636719,
      "activations/layer20_attention_weight_min": -23.827674865722656,
      "activations/layer21_attention_weight_max": 33.955848693847656,
      "activations/layer21_attention_weight_min": -23.490934371948242,
      "activations/layer22_attention_weight_max": 45.79031753540039,
      "activations/layer22_attention_weight_min": -30.690935134887695,
      "activations/layer23_attention_weight_max": 37.591304779052734,
      "activations/layer23_attention_weight_min": -25.593435287475586,
      "activations/layer2_attention_weight_max": 32.93161392211914,
      "activations/layer2_attention_weight_min": -33.373146057128906,
      "activations/layer3_attention_weight_max": 105.4034652709961,
      "activations/layer3_attention_weight_min": -107.1935806274414,
      "activations/layer4_attention_weight_max": 111.98253631591797,
      "activations/layer4_attention_weight_min": -113.44677734375,
      "activations/layer5_attention_weight_max": 53.737518310546875,
      "activations/layer5_attention_weight_min": -63.38642120361328,
      "activations/layer6_attention_weight_max": 50.32428741455078,
      "activations/layer6_attention_weight_min": -53.62975311279297,
      "activations/layer7_attention_weight_max": 101.46324157714844,
      "activations/layer7_attention_weight_min": -113.32746124267578,
      "activations/layer8_attention_weight_max": 50.93120193481445,
      "activations/layer8_attention_weight_min": -50.27370071411133,
      "activations/layer9_attention_weight_max": 37.60311508178711,
      "activations/layer9_attention_weight_min": -41.58620071411133,
      "epoch": 20.97,
      "learning_rate": 1.4898484848484847e-05,
      "loss": 2.7132,
      "step": 360950
    },
    {
      "activations/layer0_attention_weight_max": 16.885406494140625,
      "activations/layer0_attention_weight_min": -13.625661849975586,
      "activations/layer10_attention_weight_max": 34.75086212158203,
      "activations/layer10_attention_weight_min": -36.462440490722656,
      "activations/layer11_attention_weight_max": 34.1033935546875,
      "activations/layer11_attention_weight_min": -34.50680923461914,
      "activations/layer12_attention_weight_max": 27.402328491210938,
      "activations/layer12_attention_weight_min": -28.104345321655273,
      "activations/layer13_attention_weight_max": 41.539833068847656,
      "activations/layer13_attention_weight_min": -38.39122772216797,
      "activations/layer14_attention_weight_max": 44.77542495727539,
      "activations/layer14_attention_weight_min": -40.55668640136719,
      "activations/layer15_attention_weight_max": 42.14369583129883,
      "activations/layer15_attention_weight_min": -39.022830963134766,
      "activations/layer16_attention_weight_max": 30.5426025390625,
      "activations/layer16_attention_weight_min": -29.703975677490234,
      "activations/layer17_attention_weight_max": 28.70669174194336,
      "activations/layer17_attention_weight_min": -26.511072158813477,
      "activations/layer18_attention_weight_max": 30.1462459564209,
      "activations/layer18_attention_weight_min": -27.171056747436523,
      "activations/layer19_attention_weight_max": 32.78568649291992,
      "activations/layer19_attention_weight_min": -31.979904174804688,
      "activations/layer1_attention_weight_max": 16.120769500732422,
      "activations/layer1_attention_weight_min": -15.085687637329102,
      "activations/layer20_attention_weight_max": 25.683155059814453,
      "activations/layer20_attention_weight_min": -25.344518661499023,
      "activations/layer21_attention_weight_max": 26.992202758789062,
      "activations/layer21_attention_weight_min": -25.718795776367188,
      "activations/layer22_attention_weight_max": 39.59178161621094,
      "activations/layer22_attention_weight_min": -34.215721130371094,
      "activations/layer23_attention_weight_max": 29.963088989257812,
      "activations/layer23_attention_weight_min": -27.5463924407959,
      "activations/layer2_attention_weight_max": 35.724143981933594,
      "activations/layer2_attention_weight_min": -33.97903060913086,
      "activations/layer3_attention_weight_max": 105.0375747680664,
      "activations/layer3_attention_weight_min": -103.64098358154297,
      "activations/layer4_attention_weight_max": 110.42656707763672,
      "activations/layer4_attention_weight_min": -110.4749755859375,
      "activations/layer5_attention_weight_max": 55.997764587402344,
      "activations/layer5_attention_weight_min": -62.849464416503906,
      "activations/layer6_attention_weight_max": 48.17075729370117,
      "activations/layer6_attention_weight_min": -47.7089729309082,
      "activations/layer7_attention_weight_max": 95.3421630859375,
      "activations/layer7_attention_weight_min": -100.07872009277344,
      "activations/layer8_attention_weight_max": 43.08782196044922,
      "activations/layer8_attention_weight_min": -44.03820037841797,
      "activations/layer9_attention_weight_max": 33.21854782104492,
      "activations/layer9_attention_weight_min": -35.71451950073242,
      "epoch": 20.98,
      "learning_rate": 1.4879545454545454e-05,
      "loss": 2.7024,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4694,
      "eval_samples_per_second": 507.001,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4694,
      "eval_openwebtext_samples_per_second": 507.001,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9689,
      "eval_wikitext_samples_per_second": 231.597,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_lambada_loss": 2.337890625,
      "eval_lambada_ppl": 10.359361721090316,
      "eval_lambada_runtime": 9.5557,
      "eval_lambada_samples_per_second": 509.537,
      "step": 361000
    },
    {
      "activations/layer0_attention_weight_max": 15.877050399780273,
      "activations/layer0_attention_weight_min": -14.807193756103516,
      "activations/layer10_attention_weight_max": 36.451786041259766,
      "activations/layer10_attention_weight_min": -37.45147705078125,
      "activations/layer11_attention_weight_max": 34.23957443237305,
      "activations/layer11_attention_weight_min": -37.5850944519043,
      "activations/layer12_attention_weight_max": 27.359827041625977,
      "activations/layer12_attention_weight_min": -26.142038345336914,
      "activations/layer13_attention_weight_max": 40.0792350769043,
      "activations/layer13_attention_weight_min": -34.88465881347656,
      "activations/layer14_attention_weight_max": 43.265045166015625,
      "activations/layer14_attention_weight_min": -37.75800704956055,
      "activations/layer15_attention_weight_max": 39.02790832519531,
      "activations/layer15_attention_weight_min": -34.105430603027344,
      "activations/layer16_attention_weight_max": 28.599170684814453,
      "activations/layer16_attention_weight_min": -30.34099006652832,
      "activations/layer17_attention_weight_max": 29.209028244018555,
      "activations/layer17_attention_weight_min": -27.31893539428711,
      "activations/layer18_attention_weight_max": 31.25532341003418,
      "activations/layer18_attention_weight_min": -26.52488899230957,
      "activations/layer19_attention_weight_max": 35.52009201049805,
      "activations/layer19_attention_weight_min": -30.14577293395996,
      "activations/layer1_attention_weight_max": 16.152233123779297,
      "activations/layer1_attention_weight_min": -14.61274528503418,
      "activations/layer20_attention_weight_max": 29.768260955810547,
      "activations/layer20_attention_weight_min": -26.529569625854492,
      "activations/layer21_attention_weight_max": 28.92742919921875,
      "activations/layer21_attention_weight_min": -27.4566650390625,
      "activations/layer22_attention_weight_max": 44.32815170288086,
      "activations/layer22_attention_weight_min": -31.59245491027832,
      "activations/layer23_attention_weight_max": 32.09135055541992,
      "activations/layer23_attention_weight_min": -24.625377655029297,
      "activations/layer2_attention_weight_max": 33.54036331176758,
      "activations/layer2_attention_weight_min": -33.87540817260742,
      "activations/layer3_attention_weight_max": 104.89175415039062,
      "activations/layer3_attention_weight_min": -104.84318542480469,
      "activations/layer4_attention_weight_max": 109.5105209350586,
      "activations/layer4_attention_weight_min": -108.95983123779297,
      "activations/layer5_attention_weight_max": 51.23961639404297,
      "activations/layer5_attention_weight_min": -62.72528076171875,
      "activations/layer6_attention_weight_max": 45.3718376159668,
      "activations/layer6_attention_weight_min": -47.285335540771484,
      "activations/layer7_attention_weight_max": 95.71448516845703,
      "activations/layer7_attention_weight_min": -93.06974792480469,
      "activations/layer8_attention_weight_max": 43.64778518676758,
      "activations/layer8_attention_weight_min": -45.39665985107422,
      "activations/layer9_attention_weight_max": 33.43547439575195,
      "activations/layer9_attention_weight_min": -36.55070114135742,
      "epoch": 20.98,
      "learning_rate": 1.4860606060606059e-05,
      "loss": 2.7205,
      "step": 361050
    },
    {
      "activations/layer0_attention_weight_max": 16.62014389038086,
      "activations/layer0_attention_weight_min": -13.572113990783691,
      "activations/layer10_attention_weight_max": 33.769718170166016,
      "activations/layer10_attention_weight_min": -37.231414794921875,
      "activations/layer11_attention_weight_max": 34.522735595703125,
      "activations/layer11_attention_weight_min": -37.21052551269531,
      "activations/layer12_attention_weight_max": 26.21708106994629,
      "activations/layer12_attention_weight_min": -27.186786651611328,
      "activations/layer13_attention_weight_max": 39.78642654418945,
      "activations/layer13_attention_weight_min": -34.87539291381836,
      "activations/layer14_attention_weight_max": 42.133766174316406,
      "activations/layer14_attention_weight_min": -38.130096435546875,
      "activations/layer15_attention_weight_max": 38.51018142700195,
      "activations/layer15_attention_weight_min": -36.472137451171875,
      "activations/layer16_attention_weight_max": 31.35508155822754,
      "activations/layer16_attention_weight_min": -27.690608978271484,
      "activations/layer17_attention_weight_max": 27.687389373779297,
      "activations/layer17_attention_weight_min": -24.689966201782227,
      "activations/layer18_attention_weight_max": 33.25090789794922,
      "activations/layer18_attention_weight_min": -25.412137985229492,
      "activations/layer19_attention_weight_max": 34.204444885253906,
      "activations/layer19_attention_weight_min": -31.70133399963379,
      "activations/layer1_attention_weight_max": 16.094383239746094,
      "activations/layer1_attention_weight_min": -14.863545417785645,
      "activations/layer20_attention_weight_max": 27.066246032714844,
      "activations/layer20_attention_weight_min": -24.8800106048584,
      "activations/layer21_attention_weight_max": 30.77890968322754,
      "activations/layer21_attention_weight_min": -25.128129959106445,
      "activations/layer22_attention_weight_max": 43.20914840698242,
      "activations/layer22_attention_weight_min": -34.72261047363281,
      "activations/layer23_attention_weight_max": 34.331478118896484,
      "activations/layer23_attention_weight_min": -28.624492645263672,
      "activations/layer2_attention_weight_max": 32.60021209716797,
      "activations/layer2_attention_weight_min": -32.0413932800293,
      "activations/layer3_attention_weight_max": 101.02372741699219,
      "activations/layer3_attention_weight_min": -104.1574478149414,
      "activations/layer4_attention_weight_max": 104.90074920654297,
      "activations/layer4_attention_weight_min": -109.1111831665039,
      "activations/layer5_attention_weight_max": 49.79725646972656,
      "activations/layer5_attention_weight_min": -61.034698486328125,
      "activations/layer6_attention_weight_max": 44.00190734863281,
      "activations/layer6_attention_weight_min": -48.02021789550781,
      "activations/layer7_attention_weight_max": 92.5319595336914,
      "activations/layer7_attention_weight_min": -101.41519165039062,
      "activations/layer8_attention_weight_max": 41.47167205810547,
      "activations/layer8_attention_weight_min": -44.959651947021484,
      "activations/layer9_attention_weight_max": 32.634613037109375,
      "activations/layer9_attention_weight_min": -35.41689682006836,
      "epoch": 20.98,
      "learning_rate": 1.4841666666666665e-05,
      "loss": 2.7093,
      "step": 361100
    },
    {
      "activations/layer0_attention_weight_max": 16.54555320739746,
      "activations/layer0_attention_weight_min": -13.905646324157715,
      "activations/layer10_attention_weight_max": 37.366519927978516,
      "activations/layer10_attention_weight_min": -35.52390670776367,
      "activations/layer11_attention_weight_max": 36.861515045166016,
      "activations/layer11_attention_weight_min": -37.44175720214844,
      "activations/layer12_attention_weight_max": 28.074670791625977,
      "activations/layer12_attention_weight_min": -26.725826263427734,
      "activations/layer13_attention_weight_max": 39.50521469116211,
      "activations/layer13_attention_weight_min": -35.07059097290039,
      "activations/layer14_attention_weight_max": 42.30366134643555,
      "activations/layer14_attention_weight_min": -37.14026641845703,
      "activations/layer15_attention_weight_max": 41.21165084838867,
      "activations/layer15_attention_weight_min": -35.10711669921875,
      "activations/layer16_attention_weight_max": 29.838254928588867,
      "activations/layer16_attention_weight_min": -27.28261375427246,
      "activations/layer17_attention_weight_max": 29.194204330444336,
      "activations/layer17_attention_weight_min": -26.455751419067383,
      "activations/layer18_attention_weight_max": 30.193931579589844,
      "activations/layer18_attention_weight_min": -25.893386840820312,
      "activations/layer19_attention_weight_max": 35.811180114746094,
      "activations/layer19_attention_weight_min": -32.20466232299805,
      "activations/layer1_attention_weight_max": 15.708887100219727,
      "activations/layer1_attention_weight_min": -14.757107734680176,
      "activations/layer20_attention_weight_max": 30.329544067382812,
      "activations/layer20_attention_weight_min": -26.247512817382812,
      "activations/layer21_attention_weight_max": 28.941020965576172,
      "activations/layer21_attention_weight_min": -25.159027099609375,
      "activations/layer22_attention_weight_max": 44.1304817199707,
      "activations/layer22_attention_weight_min": -35.43537139892578,
      "activations/layer23_attention_weight_max": 35.39643859863281,
      "activations/layer23_attention_weight_min": -26.17446517944336,
      "activations/layer2_attention_weight_max": 34.33562469482422,
      "activations/layer2_attention_weight_min": -31.764217376708984,
      "activations/layer3_attention_weight_max": 105.37702178955078,
      "activations/layer3_attention_weight_min": -105.82550811767578,
      "activations/layer4_attention_weight_max": 109.1748275756836,
      "activations/layer4_attention_weight_min": -108.7906723022461,
      "activations/layer5_attention_weight_max": 50.557857513427734,
      "activations/layer5_attention_weight_min": -65.34991455078125,
      "activations/layer6_attention_weight_max": 47.42422103881836,
      "activations/layer6_attention_weight_min": -50.21949005126953,
      "activations/layer7_attention_weight_max": 97.90556335449219,
      "activations/layer7_attention_weight_min": -99.52023315429688,
      "activations/layer8_attention_weight_max": 43.840660095214844,
      "activations/layer8_attention_weight_min": -45.34794998168945,
      "activations/layer9_attention_weight_max": 33.75162887573242,
      "activations/layer9_attention_weight_min": -36.12763595581055,
      "epoch": 20.98,
      "learning_rate": 1.4822727272727272e-05,
      "loss": 2.7043,
      "step": 361150
    },
    {
      "activations/layer0_attention_weight_max": 17.22245216369629,
      "activations/layer0_attention_weight_min": -13.359745979309082,
      "activations/layer10_attention_weight_max": 38.298194885253906,
      "activations/layer10_attention_weight_min": -37.4522705078125,
      "activations/layer11_attention_weight_max": 36.68074035644531,
      "activations/layer11_attention_weight_min": -38.70258331298828,
      "activations/layer12_attention_weight_max": 30.611330032348633,
      "activations/layer12_attention_weight_min": -27.826683044433594,
      "activations/layer13_attention_weight_max": 43.268836975097656,
      "activations/layer13_attention_weight_min": -37.1600227355957,
      "activations/layer14_attention_weight_max": 44.35694122314453,
      "activations/layer14_attention_weight_min": -38.409889221191406,
      "activations/layer15_attention_weight_max": 46.51185607910156,
      "activations/layer15_attention_weight_min": -36.121307373046875,
      "activations/layer16_attention_weight_max": 29.41483497619629,
      "activations/layer16_attention_weight_min": -28.016422271728516,
      "activations/layer17_attention_weight_max": 31.45785140991211,
      "activations/layer17_attention_weight_min": -24.825681686401367,
      "activations/layer18_attention_weight_max": 31.4978084564209,
      "activations/layer18_attention_weight_min": -24.116085052490234,
      "activations/layer19_attention_weight_max": 32.49242401123047,
      "activations/layer19_attention_weight_min": -30.700307846069336,
      "activations/layer1_attention_weight_max": 16.135347366333008,
      "activations/layer1_attention_weight_min": -15.33320140838623,
      "activations/layer20_attention_weight_max": 29.276044845581055,
      "activations/layer20_attention_weight_min": -24.83146858215332,
      "activations/layer21_attention_weight_max": 28.791946411132812,
      "activations/layer21_attention_weight_min": -24.31816291809082,
      "activations/layer22_attention_weight_max": 43.841819763183594,
      "activations/layer22_attention_weight_min": -29.76970100402832,
      "activations/layer23_attention_weight_max": 31.519643783569336,
      "activations/layer23_attention_weight_min": -24.984020233154297,
      "activations/layer2_attention_weight_max": 34.094383239746094,
      "activations/layer2_attention_weight_min": -31.573997497558594,
      "activations/layer3_attention_weight_max": 102.72169494628906,
      "activations/layer3_attention_weight_min": -101.69379425048828,
      "activations/layer4_attention_weight_max": 106.55413055419922,
      "activations/layer4_attention_weight_min": -111.93128967285156,
      "activations/layer5_attention_weight_max": 51.86114501953125,
      "activations/layer5_attention_weight_min": -62.398109436035156,
      "activations/layer6_attention_weight_max": 46.06688690185547,
      "activations/layer6_attention_weight_min": -51.617393493652344,
      "activations/layer7_attention_weight_max": 96.4747085571289,
      "activations/layer7_attention_weight_min": -105.48621368408203,
      "activations/layer8_attention_weight_max": 44.93484878540039,
      "activations/layer8_attention_weight_min": -46.82756805419922,
      "activations/layer9_attention_weight_max": 33.862606048583984,
      "activations/layer9_attention_weight_min": -37.679046630859375,
      "epoch": 20.99,
      "learning_rate": 1.4803787878787877e-05,
      "loss": 2.7115,
      "step": 361200
    },
    {
      "activations/layer0_attention_weight_max": 18.191713333129883,
      "activations/layer0_attention_weight_min": -14.888970375061035,
      "activations/layer10_attention_weight_max": 33.286109924316406,
      "activations/layer10_attention_weight_min": -33.51047134399414,
      "activations/layer11_attention_weight_max": 34.446533203125,
      "activations/layer11_attention_weight_min": -35.49298095703125,
      "activations/layer12_attention_weight_max": 27.23039436340332,
      "activations/layer12_attention_weight_min": -26.582416534423828,
      "activations/layer13_attention_weight_max": 40.784446716308594,
      "activations/layer13_attention_weight_min": -36.56850814819336,
      "activations/layer14_attention_weight_max": 44.74823760986328,
      "activations/layer14_attention_weight_min": -37.62004852294922,
      "activations/layer15_attention_weight_max": 39.257755279541016,
      "activations/layer15_attention_weight_min": -37.3199348449707,
      "activations/layer16_attention_weight_max": 30.974727630615234,
      "activations/layer16_attention_weight_min": -27.787885665893555,
      "activations/layer17_attention_weight_max": 31.21895408630371,
      "activations/layer17_attention_weight_min": -27.078516006469727,
      "activations/layer18_attention_weight_max": 32.9289665222168,
      "activations/layer18_attention_weight_min": -25.96910858154297,
      "activations/layer19_attention_weight_max": 34.395320892333984,
      "activations/layer19_attention_weight_min": -32.20070266723633,
      "activations/layer1_attention_weight_max": 15.21438980102539,
      "activations/layer1_attention_weight_min": -14.553182601928711,
      "activations/layer20_attention_weight_max": 32.059722900390625,
      "activations/layer20_attention_weight_min": -27.102916717529297,
      "activations/layer21_attention_weight_max": 34.483795166015625,
      "activations/layer21_attention_weight_min": -26.8438720703125,
      "activations/layer22_attention_weight_max": 41.2207145690918,
      "activations/layer22_attention_weight_min": -33.93870162963867,
      "activations/layer23_attention_weight_max": 33.59698486328125,
      "activations/layer23_attention_weight_min": -28.386625289916992,
      "activations/layer2_attention_weight_max": 33.20082092285156,
      "activations/layer2_attention_weight_min": -32.666542053222656,
      "activations/layer3_attention_weight_max": 102.12421417236328,
      "activations/layer3_attention_weight_min": -102.40400695800781,
      "activations/layer4_attention_weight_max": 106.8510513305664,
      "activations/layer4_attention_weight_min": -107.97769927978516,
      "activations/layer5_attention_weight_max": 52.78754806518555,
      "activations/layer5_attention_weight_min": -59.91506576538086,
      "activations/layer6_attention_weight_max": 44.163204193115234,
      "activations/layer6_attention_weight_min": -45.76591110229492,
      "activations/layer7_attention_weight_max": 94.20467376708984,
      "activations/layer7_attention_weight_min": -94.20039367675781,
      "activations/layer8_attention_weight_max": 42.66927719116211,
      "activations/layer8_attention_weight_min": -44.416481018066406,
      "activations/layer9_attention_weight_max": 33.29085159301758,
      "activations/layer9_attention_weight_min": -33.591617584228516,
      "epoch": 20.99,
      "learning_rate": 1.4784848484848483e-05,
      "loss": 2.7185,
      "step": 361250
    },
    {
      "activations/layer0_attention_weight_max": 17.107852935791016,
      "activations/layer0_attention_weight_min": -14.794361114501953,
      "activations/layer10_attention_weight_max": 37.09532928466797,
      "activations/layer10_attention_weight_min": -39.00141143798828,
      "activations/layer11_attention_weight_max": 35.79294204711914,
      "activations/layer11_attention_weight_min": -36.76692199707031,
      "activations/layer12_attention_weight_max": 29.60264778137207,
      "activations/layer12_attention_weight_min": -27.58474349975586,
      "activations/layer13_attention_weight_max": 42.08952331542969,
      "activations/layer13_attention_weight_min": -39.022804260253906,
      "activations/layer14_attention_weight_max": 45.44786834716797,
      "activations/layer14_attention_weight_min": -42.83683395385742,
      "activations/layer15_attention_weight_max": 44.48564147949219,
      "activations/layer15_attention_weight_min": -37.893863677978516,
      "activations/layer16_attention_weight_max": 31.513538360595703,
      "activations/layer16_attention_weight_min": -28.80772590637207,
      "activations/layer17_attention_weight_max": 32.85392761230469,
      "activations/layer17_attention_weight_min": -25.68358612060547,
      "activations/layer18_attention_weight_max": 32.0556755065918,
      "activations/layer18_attention_weight_min": -26.062625885009766,
      "activations/layer19_attention_weight_max": 36.67249298095703,
      "activations/layer19_attention_weight_min": -33.19286346435547,
      "activations/layer1_attention_weight_max": 15.458810806274414,
      "activations/layer1_attention_weight_min": -14.33862590789795,
      "activations/layer20_attention_weight_max": 30.64590072631836,
      "activations/layer20_attention_weight_min": -24.264291763305664,
      "activations/layer21_attention_weight_max": 29.969284057617188,
      "activations/layer21_attention_weight_min": -23.732519149780273,
      "activations/layer22_attention_weight_max": 44.10517883300781,
      "activations/layer22_attention_weight_min": -32.135738372802734,
      "activations/layer23_attention_weight_max": 35.077796936035156,
      "activations/layer23_attention_weight_min": -26.183937072753906,
      "activations/layer2_attention_weight_max": 32.89346694946289,
      "activations/layer2_attention_weight_min": -32.865867614746094,
      "activations/layer3_attention_weight_max": 99.01898193359375,
      "activations/layer3_attention_weight_min": -101.4771728515625,
      "activations/layer4_attention_weight_max": 102.31029510498047,
      "activations/layer4_attention_weight_min": -110.48738098144531,
      "activations/layer5_attention_weight_max": 48.14842987060547,
      "activations/layer5_attention_weight_min": -60.81989669799805,
      "activations/layer6_attention_weight_max": 43.19462966918945,
      "activations/layer6_attention_weight_min": -47.8210563659668,
      "activations/layer7_attention_weight_max": 94.6646728515625,
      "activations/layer7_attention_weight_min": -100.84767150878906,
      "activations/layer8_attention_weight_max": 41.68441390991211,
      "activations/layer8_attention_weight_min": -49.283233642578125,
      "activations/layer9_attention_weight_max": 32.78112030029297,
      "activations/layer9_attention_weight_min": -37.419151306152344,
      "epoch": 20.99,
      "learning_rate": 1.476590909090909e-05,
      "loss": 2.7244,
      "step": 361300
    },
    {
      "activations/layer0_attention_weight_max": 16.196407318115234,
      "activations/layer0_attention_weight_min": -12.786431312561035,
      "activations/layer10_attention_weight_max": 34.90449905395508,
      "activations/layer10_attention_weight_min": -36.81752395629883,
      "activations/layer11_attention_weight_max": 35.871788024902344,
      "activations/layer11_attention_weight_min": -36.03335189819336,
      "activations/layer12_attention_weight_max": 28.426069259643555,
      "activations/layer12_attention_weight_min": -27.447288513183594,
      "activations/layer13_attention_weight_max": 40.33911895751953,
      "activations/layer13_attention_weight_min": -36.03141403198242,
      "activations/layer14_attention_weight_max": 42.63542556762695,
      "activations/layer14_attention_weight_min": -38.92411804199219,
      "activations/layer15_attention_weight_max": 42.24111557006836,
      "activations/layer15_attention_weight_min": -36.523284912109375,
      "activations/layer16_attention_weight_max": 30.647069931030273,
      "activations/layer16_attention_weight_min": -29.272790908813477,
      "activations/layer17_attention_weight_max": 29.34200096130371,
      "activations/layer17_attention_weight_min": -27.517044067382812,
      "activations/layer18_attention_weight_max": 31.219032287597656,
      "activations/layer18_attention_weight_min": -26.639469146728516,
      "activations/layer19_attention_weight_max": 36.90987777709961,
      "activations/layer19_attention_weight_min": -31.25898551940918,
      "activations/layer1_attention_weight_max": 15.514986038208008,
      "activations/layer1_attention_weight_min": -15.661238670349121,
      "activations/layer20_attention_weight_max": 32.019527435302734,
      "activations/layer20_attention_weight_min": -25.637346267700195,
      "activations/layer21_attention_weight_max": 31.24803352355957,
      "activations/layer21_attention_weight_min": -25.347129821777344,
      "activations/layer22_attention_weight_max": 46.400997161865234,
      "activations/layer22_attention_weight_min": -30.94696807861328,
      "activations/layer23_attention_weight_max": 36.048851013183594,
      "activations/layer23_attention_weight_min": -26.872867584228516,
      "activations/layer2_attention_weight_max": 32.816253662109375,
      "activations/layer2_attention_weight_min": -33.68312072753906,
      "activations/layer3_attention_weight_max": 101.95372009277344,
      "activations/layer3_attention_weight_min": -107.02066040039062,
      "activations/layer4_attention_weight_max": 109.2190933227539,
      "activations/layer4_attention_weight_min": -110.7981948852539,
      "activations/layer5_attention_weight_max": 51.01827621459961,
      "activations/layer5_attention_weight_min": -60.77045440673828,
      "activations/layer6_attention_weight_max": 46.00806427001953,
      "activations/layer6_attention_weight_min": -49.39704895019531,
      "activations/layer7_attention_weight_max": 96.42322540283203,
      "activations/layer7_attention_weight_min": -100.97383117675781,
      "activations/layer8_attention_weight_max": 43.00718307495117,
      "activations/layer8_attention_weight_min": -45.65206527709961,
      "activations/layer9_attention_weight_max": 33.61186981201172,
      "activations/layer9_attention_weight_min": -36.20473861694336,
      "epoch": 21.0,
      "learning_rate": 1.4746969696969695e-05,
      "loss": 2.7094,
      "step": 361350
    },
    {
      "activations/layer0_attention_weight_max": 16.602256774902344,
      "activations/layer0_attention_weight_min": -15.037079811096191,
      "activations/layer10_attention_weight_max": 31.906606674194336,
      "activations/layer10_attention_weight_min": -33.50800704956055,
      "activations/layer11_attention_weight_max": 32.17365264892578,
      "activations/layer11_attention_weight_min": -34.27155303955078,
      "activations/layer12_attention_weight_max": 24.29450798034668,
      "activations/layer12_attention_weight_min": -28.04105567932129,
      "activations/layer13_attention_weight_max": 36.14445114135742,
      "activations/layer13_attention_weight_min": -34.97749710083008,
      "activations/layer14_attention_weight_max": 37.51289749145508,
      "activations/layer14_attention_weight_min": -38.47452926635742,
      "activations/layer15_attention_weight_max": 33.29290771484375,
      "activations/layer15_attention_weight_min": -37.84221649169922,
      "activations/layer16_attention_weight_max": 26.448617935180664,
      "activations/layer16_attention_weight_min": -29.060487747192383,
      "activations/layer17_attention_weight_max": 26.224557876586914,
      "activations/layer17_attention_weight_min": -27.33710289001465,
      "activations/layer18_attention_weight_max": 29.914714813232422,
      "activations/layer18_attention_weight_min": -26.35264778137207,
      "activations/layer19_attention_weight_max": 30.269744873046875,
      "activations/layer19_attention_weight_min": -32.35307693481445,
      "activations/layer1_attention_weight_max": 16.864913940429688,
      "activations/layer1_attention_weight_min": -15.64059066772461,
      "activations/layer20_attention_weight_max": 27.87279510498047,
      "activations/layer20_attention_weight_min": -25.34065818786621,
      "activations/layer21_attention_weight_max": 28.907073974609375,
      "activations/layer21_attention_weight_min": -23.674448013305664,
      "activations/layer22_attention_weight_max": 38.59589385986328,
      "activations/layer22_attention_weight_min": -31.01078224182129,
      "activations/layer23_attention_weight_max": 29.60772705078125,
      "activations/layer23_attention_weight_min": -27.23248863220215,
      "activations/layer2_attention_weight_max": 34.50931930541992,
      "activations/layer2_attention_weight_min": -33.198394775390625,
      "activations/layer3_attention_weight_max": 102.34957885742188,
      "activations/layer3_attention_weight_min": -104.25581359863281,
      "activations/layer4_attention_weight_max": 106.2970962524414,
      "activations/layer4_attention_weight_min": -109.1371841430664,
      "activations/layer5_attention_weight_max": 48.4122314453125,
      "activations/layer5_attention_weight_min": -62.98591232299805,
      "activations/layer6_attention_weight_max": 43.40031051635742,
      "activations/layer6_attention_weight_min": -46.53889083862305,
      "activations/layer7_attention_weight_max": 92.0593032836914,
      "activations/layer7_attention_weight_min": -95.91741943359375,
      "activations/layer8_attention_weight_max": 41.5152473449707,
      "activations/layer8_attention_weight_min": -44.016571044921875,
      "activations/layer9_attention_weight_max": 29.931604385375977,
      "activations/layer9_attention_weight_min": -34.204673767089844,
      "epoch": 21.0,
      "learning_rate": 1.4728030303030302e-05,
      "loss": 2.7084,
      "step": 361400
    },
    {
      "activations/layer0_attention_weight_max": 16.58982276916504,
      "activations/layer0_attention_weight_min": -14.763227462768555,
      "activations/layer10_attention_weight_max": 35.31745910644531,
      "activations/layer10_attention_weight_min": -32.77375793457031,
      "activations/layer11_attention_weight_max": 36.446224212646484,
      "activations/layer11_attention_weight_min": -34.777488708496094,
      "activations/layer12_attention_weight_max": 28.5975284576416,
      "activations/layer12_attention_weight_min": -29.87403106689453,
      "activations/layer13_attention_weight_max": 43.21997833251953,
      "activations/layer13_attention_weight_min": -36.268856048583984,
      "activations/layer14_attention_weight_max": 45.633609771728516,
      "activations/layer14_attention_weight_min": -37.01581573486328,
      "activations/layer15_attention_weight_max": 41.31892013549805,
      "activations/layer15_attention_weight_min": -37.04743194580078,
      "activations/layer16_attention_weight_max": 34.529541015625,
      "activations/layer16_attention_weight_min": -29.141807556152344,
      "activations/layer17_attention_weight_max": 28.92945098876953,
      "activations/layer17_attention_weight_min": -26.98893165588379,
      "activations/layer18_attention_weight_max": 36.184295654296875,
      "activations/layer18_attention_weight_min": -27.36754035949707,
      "activations/layer19_attention_weight_max": 38.56856155395508,
      "activations/layer19_attention_weight_min": -36.002845764160156,
      "activations/layer1_attention_weight_max": 15.496710777282715,
      "activations/layer1_attention_weight_min": -14.452862739562988,
      "activations/layer20_attention_weight_max": 32.97687530517578,
      "activations/layer20_attention_weight_min": -28.73488426208496,
      "activations/layer21_attention_weight_max": 37.19013977050781,
      "activations/layer21_attention_weight_min": -29.110410690307617,
      "activations/layer22_attention_weight_max": 50.29124450683594,
      "activations/layer22_attention_weight_min": -36.73017883300781,
      "activations/layer23_attention_weight_max": 36.47080993652344,
      "activations/layer23_attention_weight_min": -27.49337387084961,
      "activations/layer2_attention_weight_max": 34.51798629760742,
      "activations/layer2_attention_weight_min": -32.92255401611328,
      "activations/layer3_attention_weight_max": 101.13362884521484,
      "activations/layer3_attention_weight_min": -106.26412200927734,
      "activations/layer4_attention_weight_max": 105.98323059082031,
      "activations/layer4_attention_weight_min": -106.4513931274414,
      "activations/layer5_attention_weight_max": 47.12788391113281,
      "activations/layer5_attention_weight_min": -63.71677780151367,
      "activations/layer6_attention_weight_max": 43.04154968261719,
      "activations/layer6_attention_weight_min": -45.25800704956055,
      "activations/layer7_attention_weight_max": 91.32160186767578,
      "activations/layer7_attention_weight_min": -98.49244689941406,
      "activations/layer8_attention_weight_max": 40.632389068603516,
      "activations/layer8_attention_weight_min": -41.48391342163086,
      "activations/layer9_attention_weight_max": 31.1903133392334,
      "activations/layer9_attention_weight_min": -32.668235778808594,
      "epoch": 21.0,
      "learning_rate": 1.4709469696969696e-05,
      "loss": 2.7387,
      "step": 361450
    },
    {
      "activations/layer0_attention_weight_max": 16.386632919311523,
      "activations/layer0_attention_weight_min": -13.446953773498535,
      "activations/layer10_attention_weight_max": 36.123416900634766,
      "activations/layer10_attention_weight_min": -34.39741134643555,
      "activations/layer11_attention_weight_max": 34.85261917114258,
      "activations/layer11_attention_weight_min": -36.72612762451172,
      "activations/layer12_attention_weight_max": 26.718538284301758,
      "activations/layer12_attention_weight_min": -27.892200469970703,
      "activations/layer13_attention_weight_max": 38.00140380859375,
      "activations/layer13_attention_weight_min": -36.72721481323242,
      "activations/layer14_attention_weight_max": 40.77989196777344,
      "activations/layer14_attention_weight_min": -38.935543060302734,
      "activations/layer15_attention_weight_max": 37.01310729980469,
      "activations/layer15_attention_weight_min": -37.24000930786133,
      "activations/layer16_attention_weight_max": 29.713647842407227,
      "activations/layer16_attention_weight_min": -28.31821632385254,
      "activations/layer17_attention_weight_max": 32.46316146850586,
      "activations/layer17_attention_weight_min": -26.54951286315918,
      "activations/layer18_attention_weight_max": 34.508758544921875,
      "activations/layer18_attention_weight_min": -27.651323318481445,
      "activations/layer19_attention_weight_max": 33.57964324951172,
      "activations/layer19_attention_weight_min": -32.483097076416016,
      "activations/layer1_attention_weight_max": 15.285470962524414,
      "activations/layer1_attention_weight_min": -15.253413200378418,
      "activations/layer20_attention_weight_max": 30.206087112426758,
      "activations/layer20_attention_weight_min": -25.76350975036621,
      "activations/layer21_attention_weight_max": 31.307918548583984,
      "activations/layer21_attention_weight_min": -27.2002010345459,
      "activations/layer22_attention_weight_max": 41.84229278564453,
      "activations/layer22_attention_weight_min": -34.532676696777344,
      "activations/layer23_attention_weight_max": 34.04573059082031,
      "activations/layer23_attention_weight_min": -27.42318344116211,
      "activations/layer2_attention_weight_max": 34.512664794921875,
      "activations/layer2_attention_weight_min": -34.13301467895508,
      "activations/layer3_attention_weight_max": 102.22298431396484,
      "activations/layer3_attention_weight_min": -107.72660827636719,
      "activations/layer4_attention_weight_max": 106.12042236328125,
      "activations/layer4_attention_weight_min": -108.3390121459961,
      "activations/layer5_attention_weight_max": 51.66927719116211,
      "activations/layer5_attention_weight_min": -64.44142150878906,
      "activations/layer6_attention_weight_max": 43.5207405090332,
      "activations/layer6_attention_weight_min": -47.88801193237305,
      "activations/layer7_attention_weight_max": 97.51644897460938,
      "activations/layer7_attention_weight_min": -101.4836654663086,
      "activations/layer8_attention_weight_max": 42.5887336730957,
      "activations/layer8_attention_weight_min": -43.90781784057617,
      "activations/layer9_attention_weight_max": 34.10213851928711,
      "activations/layer9_attention_weight_min": -34.81428527832031,
      "epoch": 21.01,
      "learning_rate": 1.46905303030303e-05,
      "loss": 2.7007,
      "step": 361500
    },
    {
      "activations/layer0_attention_weight_max": 16.926349639892578,
      "activations/layer0_attention_weight_min": -14.594832420349121,
      "activations/layer10_attention_weight_max": 31.775146484375,
      "activations/layer10_attention_weight_min": -34.49610900878906,
      "activations/layer11_attention_weight_max": 32.514434814453125,
      "activations/layer11_attention_weight_min": -35.29345703125,
      "activations/layer12_attention_weight_max": 26.57190704345703,
      "activations/layer12_attention_weight_min": -26.218677520751953,
      "activations/layer13_attention_weight_max": 39.346527099609375,
      "activations/layer13_attention_weight_min": -37.83527374267578,
      "activations/layer14_attention_weight_max": 43.2474250793457,
      "activations/layer14_attention_weight_min": -40.38857650756836,
      "activations/layer15_attention_weight_max": 41.65792465209961,
      "activations/layer15_attention_weight_min": -40.13032531738281,
      "activations/layer16_attention_weight_max": 31.426164627075195,
      "activations/layer16_attention_weight_min": -30.41063117980957,
      "activations/layer17_attention_weight_max": 29.286787033081055,
      "activations/layer17_attention_weight_min": -27.35491943359375,
      "activations/layer18_attention_weight_max": 33.766387939453125,
      "activations/layer18_attention_weight_min": -27.794462203979492,
      "activations/layer19_attention_weight_max": 38.94788360595703,
      "activations/layer19_attention_weight_min": -34.500511169433594,
      "activations/layer1_attention_weight_max": 15.512619972229004,
      "activations/layer1_attention_weight_min": -15.491962432861328,
      "activations/layer20_attention_weight_max": 32.999446868896484,
      "activations/layer20_attention_weight_min": -26.327442169189453,
      "activations/layer21_attention_weight_max": 31.418718338012695,
      "activations/layer21_attention_weight_min": -24.435514450073242,
      "activations/layer22_attention_weight_max": 49.169837951660156,
      "activations/layer22_attention_weight_min": -32.81546401977539,
      "activations/layer23_attention_weight_max": 35.504913330078125,
      "activations/layer23_attention_weight_min": -26.850431442260742,
      "activations/layer2_attention_weight_max": 34.428741455078125,
      "activations/layer2_attention_weight_min": -33.24747848510742,
      "activations/layer3_attention_weight_max": 105.7308349609375,
      "activations/layer3_attention_weight_min": -106.83478546142578,
      "activations/layer4_attention_weight_max": 106.59517669677734,
      "activations/layer4_attention_weight_min": -112.6511459350586,
      "activations/layer5_attention_weight_max": 52.01820373535156,
      "activations/layer5_attention_weight_min": -62.07159423828125,
      "activations/layer6_attention_weight_max": 44.36848449707031,
      "activations/layer6_attention_weight_min": -47.74894332885742,
      "activations/layer7_attention_weight_max": 91.99279022216797,
      "activations/layer7_attention_weight_min": -100.14932250976562,
      "activations/layer8_attention_weight_max": 40.6488151550293,
      "activations/layer8_attention_weight_min": -45.753662109375,
      "activations/layer9_attention_weight_max": 30.395843505859375,
      "activations/layer9_attention_weight_min": -33.30009078979492,
      "epoch": 21.01,
      "learning_rate": 1.4671590909090907e-05,
      "loss": 2.7021,
      "step": 361550
    },
    {
      "activations/layer0_attention_weight_max": 16.875364303588867,
      "activations/layer0_attention_weight_min": -14.918213844299316,
      "activations/layer10_attention_weight_max": 32.270477294921875,
      "activations/layer10_attention_weight_min": -35.44245529174805,
      "activations/layer11_attention_weight_max": 33.63701629638672,
      "activations/layer11_attention_weight_min": -34.127769470214844,
      "activations/layer12_attention_weight_max": 26.401573181152344,
      "activations/layer12_attention_weight_min": -26.208881378173828,
      "activations/layer13_attention_weight_max": 39.06114959716797,
      "activations/layer13_attention_weight_min": -36.853816986083984,
      "activations/layer14_attention_weight_max": 43.30240249633789,
      "activations/layer14_attention_weight_min": -38.104713439941406,
      "activations/layer15_attention_weight_max": 38.99806213378906,
      "activations/layer15_attention_weight_min": -35.196773529052734,
      "activations/layer16_attention_weight_max": 34.45615768432617,
      "activations/layer16_attention_weight_min": -27.704730987548828,
      "activations/layer17_attention_weight_max": 30.20276641845703,
      "activations/layer17_attention_weight_min": -26.383703231811523,
      "activations/layer18_attention_weight_max": 36.17376708984375,
      "activations/layer18_attention_weight_min": -28.0980224609375,
      "activations/layer19_attention_weight_max": 38.292659759521484,
      "activations/layer19_attention_weight_min": -37.83083724975586,
      "activations/layer1_attention_weight_max": 15.5336275100708,
      "activations/layer1_attention_weight_min": -13.400883674621582,
      "activations/layer20_attention_weight_max": 32.99155044555664,
      "activations/layer20_attention_weight_min": -29.420482635498047,
      "activations/layer21_attention_weight_max": 31.654699325561523,
      "activations/layer21_attention_weight_min": -26.04245948791504,
      "activations/layer22_attention_weight_max": 46.89316177368164,
      "activations/layer22_attention_weight_min": -33.05207061767578,
      "activations/layer23_attention_weight_max": 37.957969665527344,
      "activations/layer23_attention_weight_min": -27.612442016601562,
      "activations/layer2_attention_weight_max": 33.8929328918457,
      "activations/layer2_attention_weight_min": -33.26749801635742,
      "activations/layer3_attention_weight_max": 105.373046875,
      "activations/layer3_attention_weight_min": -109.61022186279297,
      "activations/layer4_attention_weight_max": 108.50550842285156,
      "activations/layer4_attention_weight_min": -112.94856262207031,
      "activations/layer5_attention_weight_max": 50.35770797729492,
      "activations/layer5_attention_weight_min": -63.205589294433594,
      "activations/layer6_attention_weight_max": 44.77735900878906,
      "activations/layer6_attention_weight_min": -48.129249572753906,
      "activations/layer7_attention_weight_max": 88.74842834472656,
      "activations/layer7_attention_weight_min": -100.4561996459961,
      "activations/layer8_attention_weight_max": 41.60063171386719,
      "activations/layer8_attention_weight_min": -46.92478942871094,
      "activations/layer9_attention_weight_max": 29.289478302001953,
      "activations/layer9_attention_weight_min": -33.424861907958984,
      "epoch": 21.01,
      "learning_rate": 1.4652651515151514e-05,
      "loss": 2.687,
      "step": 361600
    },
    {
      "activations/layer0_attention_weight_max": 16.30362319946289,
      "activations/layer0_attention_weight_min": -15.222737312316895,
      "activations/layer10_attention_weight_max": 33.19736099243164,
      "activations/layer10_attention_weight_min": -35.02888870239258,
      "activations/layer11_attention_weight_max": 33.616451263427734,
      "activations/layer11_attention_weight_min": -36.00260543823242,
      "activations/layer12_attention_weight_max": 26.61408805847168,
      "activations/layer12_attention_weight_min": -26.421430587768555,
      "activations/layer13_attention_weight_max": 39.94913864135742,
      "activations/layer13_attention_weight_min": -37.01802444458008,
      "activations/layer14_attention_weight_max": 42.39645767211914,
      "activations/layer14_attention_weight_min": -38.31962966918945,
      "activations/layer15_attention_weight_max": 41.475833892822266,
      "activations/layer15_attention_weight_min": -39.317527770996094,
      "activations/layer16_attention_weight_max": 32.278297424316406,
      "activations/layer16_attention_weight_min": -28.40846824645996,
      "activations/layer17_attention_weight_max": 30.682321548461914,
      "activations/layer17_attention_weight_min": -27.479509353637695,
      "activations/layer18_attention_weight_max": 30.5523624420166,
      "activations/layer18_attention_weight_min": -29.736103057861328,
      "activations/layer19_attention_weight_max": 36.77195739746094,
      "activations/layer19_attention_weight_min": -31.617116928100586,
      "activations/layer1_attention_weight_max": 15.64123249053955,
      "activations/layer1_attention_weight_min": -16.00771713256836,
      "activations/layer20_attention_weight_max": 31.62272834777832,
      "activations/layer20_attention_weight_min": -26.5575008392334,
      "activations/layer21_attention_weight_max": 26.56254768371582,
      "activations/layer21_attention_weight_min": -26.994359970092773,
      "activations/layer22_attention_weight_max": 40.45099639892578,
      "activations/layer22_attention_weight_min": -34.320438385009766,
      "activations/layer23_attention_weight_max": 30.885971069335938,
      "activations/layer23_attention_weight_min": -27.546689987182617,
      "activations/layer2_attention_weight_max": 32.979671478271484,
      "activations/layer2_attention_weight_min": -32.36000061035156,
      "activations/layer3_attention_weight_max": 102.12877655029297,
      "activations/layer3_attention_weight_min": -104.99089050292969,
      "activations/layer4_attention_weight_max": 106.7039566040039,
      "activations/layer4_attention_weight_min": -110.92620849609375,
      "activations/layer5_attention_weight_max": 50.208858489990234,
      "activations/layer5_attention_weight_min": -61.178951263427734,
      "activations/layer6_attention_weight_max": 44.72914505004883,
      "activations/layer6_attention_weight_min": -49.7726936340332,
      "activations/layer7_attention_weight_max": 91.29175567626953,
      "activations/layer7_attention_weight_min": -103.44718170166016,
      "activations/layer8_attention_weight_max": 40.94060134887695,
      "activations/layer8_attention_weight_min": -46.96556091308594,
      "activations/layer9_attention_weight_max": 31.412755966186523,
      "activations/layer9_attention_weight_min": -34.34627914428711,
      "epoch": 21.01,
      "learning_rate": 1.4633712121212119e-05,
      "loss": 2.6956,
      "step": 361650
    },
    {
      "activations/layer0_attention_weight_max": 16.46623992919922,
      "activations/layer0_attention_weight_min": -12.850837707519531,
      "activations/layer10_attention_weight_max": 47.25569534301758,
      "activations/layer10_attention_weight_min": -44.96967697143555,
      "activations/layer11_attention_weight_max": 49.773040771484375,
      "activations/layer11_attention_weight_min": -45.549129486083984,
      "activations/layer12_attention_weight_max": 33.36444091796875,
      "activations/layer12_attention_weight_min": -31.403852462768555,
      "activations/layer13_attention_weight_max": 50.11634063720703,
      "activations/layer13_attention_weight_min": -39.329307556152344,
      "activations/layer14_attention_weight_max": 57.71272659301758,
      "activations/layer14_attention_weight_min": -43.08254623413086,
      "activations/layer15_attention_weight_max": 57.09532928466797,
      "activations/layer15_attention_weight_min": -45.686771392822266,
      "activations/layer16_attention_weight_max": 37.29251480102539,
      "activations/layer16_attention_weight_min": -31.083105087280273,
      "activations/layer17_attention_weight_max": 33.725955963134766,
      "activations/layer17_attention_weight_min": -29.00316047668457,
      "activations/layer18_attention_weight_max": 41.70291519165039,
      "activations/layer18_attention_weight_min": -32.67507553100586,
      "activations/layer19_attention_weight_max": 50.58827590942383,
      "activations/layer19_attention_weight_min": -35.89430618286133,
      "activations/layer1_attention_weight_max": 15.582356452941895,
      "activations/layer1_attention_weight_min": -14.927597045898438,
      "activations/layer20_attention_weight_max": 39.937744140625,
      "activations/layer20_attention_weight_min": -29.097991943359375,
      "activations/layer21_attention_weight_max": 37.2248649597168,
      "activations/layer21_attention_weight_min": -28.69073486328125,
      "activations/layer22_attention_weight_max": 58.34021759033203,
      "activations/layer22_attention_weight_min": -37.47296905517578,
      "activations/layer23_attention_weight_max": 42.28211975097656,
      "activations/layer23_attention_weight_min": -26.966999053955078,
      "activations/layer2_attention_weight_max": 36.171451568603516,
      "activations/layer2_attention_weight_min": -33.000858306884766,
      "activations/layer3_attention_weight_max": 105.54137420654297,
      "activations/layer3_attention_weight_min": -102.16161346435547,
      "activations/layer4_attention_weight_max": 115.86901092529297,
      "activations/layer4_attention_weight_min": -115.64192962646484,
      "activations/layer5_attention_weight_max": 51.90775680541992,
      "activations/layer5_attention_weight_min": -60.71385955810547,
      "activations/layer6_attention_weight_max": 44.99055862426758,
      "activations/layer6_attention_weight_min": -45.739864349365234,
      "activations/layer7_attention_weight_max": 105.3697509765625,
      "activations/layer7_attention_weight_min": -96.69562530517578,
      "activations/layer8_attention_weight_max": 51.484378814697266,
      "activations/layer8_attention_weight_min": -49.21597671508789,
      "activations/layer9_attention_weight_max": 39.146724700927734,
      "activations/layer9_attention_weight_min": -42.13875961303711,
      "epoch": 21.02,
      "learning_rate": 1.4614772727272725e-05,
      "loss": 2.7062,
      "step": 361700
    },
    {
      "activations/layer0_attention_weight_max": 17.499618530273438,
      "activations/layer0_attention_weight_min": -14.26158618927002,
      "activations/layer10_attention_weight_max": 37.81800842285156,
      "activations/layer10_attention_weight_min": -38.03868103027344,
      "activations/layer11_attention_weight_max": 38.57230758666992,
      "activations/layer11_attention_weight_min": -37.65740203857422,
      "activations/layer12_attention_weight_max": 29.098712921142578,
      "activations/layer12_attention_weight_min": -28.331497192382812,
      "activations/layer13_attention_weight_max": 43.2413444519043,
      "activations/layer13_attention_weight_min": -39.318626403808594,
      "activations/layer14_attention_weight_max": 51.35878372192383,
      "activations/layer14_attention_weight_min": -40.82365417480469,
      "activations/layer15_attention_weight_max": 44.4990348815918,
      "activations/layer15_attention_weight_min": -38.2192497253418,
      "activations/layer16_attention_weight_max": 34.20999526977539,
      "activations/layer16_attention_weight_min": -29.44513511657715,
      "activations/layer17_attention_weight_max": 34.0230598449707,
      "activations/layer17_attention_weight_min": -26.419126510620117,
      "activations/layer18_attention_weight_max": 34.151771545410156,
      "activations/layer18_attention_weight_min": -26.33185386657715,
      "activations/layer19_attention_weight_max": 37.05293655395508,
      "activations/layer19_attention_weight_min": -34.72145080566406,
      "activations/layer1_attention_weight_max": 15.968132972717285,
      "activations/layer1_attention_weight_min": -14.650760650634766,
      "activations/layer20_attention_weight_max": 35.908851623535156,
      "activations/layer20_attention_weight_min": -26.671775817871094,
      "activations/layer21_attention_weight_max": 32.92699432373047,
      "activations/layer21_attention_weight_min": -25.486722946166992,
      "activations/layer22_attention_weight_max": 45.81743240356445,
      "activations/layer22_attention_weight_min": -33.74797058105469,
      "activations/layer23_attention_weight_max": 34.346038818359375,
      "activations/layer23_attention_weight_min": -26.904218673706055,
      "activations/layer2_attention_weight_max": 32.632476806640625,
      "activations/layer2_attention_weight_min": -32.081787109375,
      "activations/layer3_attention_weight_max": 104.79061126708984,
      "activations/layer3_attention_weight_min": -110.15121459960938,
      "activations/layer4_attention_weight_max": 110.93830871582031,
      "activations/layer4_attention_weight_min": -114.69911193847656,
      "activations/layer5_attention_weight_max": 51.72114181518555,
      "activations/layer5_attention_weight_min": -65.98025512695312,
      "activations/layer6_attention_weight_max": 46.7667121887207,
      "activations/layer6_attention_weight_min": -50.120460510253906,
      "activations/layer7_attention_weight_max": 100.02925109863281,
      "activations/layer7_attention_weight_min": -103.97257995605469,
      "activations/layer8_attention_weight_max": 47.314815521240234,
      "activations/layer8_attention_weight_min": -49.6132926940918,
      "activations/layer9_attention_weight_max": 37.508724212646484,
      "activations/layer9_attention_weight_min": -37.08192443847656,
      "epoch": 21.02,
      "learning_rate": 1.4595833333333333e-05,
      "loss": 2.7271,
      "step": 361750
    },
    {
      "activations/layer0_attention_weight_max": 16.7310733795166,
      "activations/layer0_attention_weight_min": -13.566405296325684,
      "activations/layer10_attention_weight_max": 36.04122543334961,
      "activations/layer10_attention_weight_min": -36.548858642578125,
      "activations/layer11_attention_weight_max": 35.11882781982422,
      "activations/layer11_attention_weight_min": -38.10686492919922,
      "activations/layer12_attention_weight_max": 26.696590423583984,
      "activations/layer12_attention_weight_min": -26.85356903076172,
      "activations/layer13_attention_weight_max": 38.0771484375,
      "activations/layer13_attention_weight_min": -36.91755294799805,
      "activations/layer14_attention_weight_max": 40.54580307006836,
      "activations/layer14_attention_weight_min": -40.161434173583984,
      "activations/layer15_attention_weight_max": 37.30296325683594,
      "activations/layer15_attention_weight_min": -36.97218322753906,
      "activations/layer16_attention_weight_max": 27.158754348754883,
      "activations/layer16_attention_weight_min": -29.112707138061523,
      "activations/layer17_attention_weight_max": 30.2170352935791,
      "activations/layer17_attention_weight_min": -27.425077438354492,
      "activations/layer18_attention_weight_max": 31.06474494934082,
      "activations/layer18_attention_weight_min": -28.434711456298828,
      "activations/layer19_attention_weight_max": 32.26641082763672,
      "activations/layer19_attention_weight_min": -32.220947265625,
      "activations/layer1_attention_weight_max": 15.829172134399414,
      "activations/layer1_attention_weight_min": -15.196992874145508,
      "activations/layer20_attention_weight_max": 27.18191146850586,
      "activations/layer20_attention_weight_min": -25.184314727783203,
      "activations/layer21_attention_weight_max": 31.49458885192871,
      "activations/layer21_attention_weight_min": -25.726022720336914,
      "activations/layer22_attention_weight_max": 41.531585693359375,
      "activations/layer22_attention_weight_min": -32.25069808959961,
      "activations/layer23_attention_weight_max": 32.077354431152344,
      "activations/layer23_attention_weight_min": -26.800382614135742,
      "activations/layer2_attention_weight_max": 33.93269348144531,
      "activations/layer2_attention_weight_min": -33.38687515258789,
      "activations/layer3_attention_weight_max": 105.40380096435547,
      "activations/layer3_attention_weight_min": -108.499267578125,
      "activations/layer4_attention_weight_max": 109.41981506347656,
      "activations/layer4_attention_weight_min": -117.56539154052734,
      "activations/layer5_attention_weight_max": 49.677181243896484,
      "activations/layer5_attention_weight_min": -61.94702911376953,
      "activations/layer6_attention_weight_max": 44.76942443847656,
      "activations/layer6_attention_weight_min": -47.27956008911133,
      "activations/layer7_attention_weight_max": 93.804931640625,
      "activations/layer7_attention_weight_min": -99.86920166015625,
      "activations/layer8_attention_weight_max": 42.49030303955078,
      "activations/layer8_attention_weight_min": -46.754493713378906,
      "activations/layer9_attention_weight_max": 32.31731414794922,
      "activations/layer9_attention_weight_min": -36.176605224609375,
      "epoch": 21.02,
      "learning_rate": 1.4576893939393938e-05,
      "loss": 2.6952,
      "step": 361800
    },
    {
      "activations/layer0_attention_weight_max": 15.949438095092773,
      "activations/layer0_attention_weight_min": -14.773961067199707,
      "activations/layer10_attention_weight_max": 34.03007507324219,
      "activations/layer10_attention_weight_min": -34.7210578918457,
      "activations/layer11_attention_weight_max": 34.38296890258789,
      "activations/layer11_attention_weight_min": -35.057289123535156,
      "activations/layer12_attention_weight_max": 27.489225387573242,
      "activations/layer12_attention_weight_min": -27.735679626464844,
      "activations/layer13_attention_weight_max": 41.927146911621094,
      "activations/layer13_attention_weight_min": -36.129608154296875,
      "activations/layer14_attention_weight_max": 41.70454788208008,
      "activations/layer14_attention_weight_min": -39.76888656616211,
      "activations/layer15_attention_weight_max": 38.92694091796875,
      "activations/layer15_attention_weight_min": -36.86684799194336,
      "activations/layer16_attention_weight_max": 28.2239933013916,
      "activations/layer16_attention_weight_min": -27.83598518371582,
      "activations/layer17_attention_weight_max": 31.52672004699707,
      "activations/layer17_attention_weight_min": -26.398311614990234,
      "activations/layer18_attention_weight_max": 32.74253845214844,
      "activations/layer18_attention_weight_min": -24.83968734741211,
      "activations/layer19_attention_weight_max": 39.40810012817383,
      "activations/layer19_attention_weight_min": -29.371816635131836,
      "activations/layer1_attention_weight_max": 15.594122886657715,
      "activations/layer1_attention_weight_min": -15.320345878601074,
      "activations/layer20_attention_weight_max": 30.17719078063965,
      "activations/layer20_attention_weight_min": -25.132089614868164,
      "activations/layer21_attention_weight_max": 32.81868362426758,
      "activations/layer21_attention_weight_min": -26.12200927734375,
      "activations/layer22_attention_weight_max": 45.28627014160156,
      "activations/layer22_attention_weight_min": -30.452260971069336,
      "activations/layer23_attention_weight_max": 34.252357482910156,
      "activations/layer23_attention_weight_min": -29.804340362548828,
      "activations/layer2_attention_weight_max": 32.87533950805664,
      "activations/layer2_attention_weight_min": -32.40227508544922,
      "activations/layer3_attention_weight_max": 106.84698486328125,
      "activations/layer3_attention_weight_min": -108.0043716430664,
      "activations/layer4_attention_weight_max": 109.93144989013672,
      "activations/layer4_attention_weight_min": -115.8805923461914,
      "activations/layer5_attention_weight_max": 48.85505676269531,
      "activations/layer5_attention_weight_min": -61.452205657958984,
      "activations/layer6_attention_weight_max": 46.78236770629883,
      "activations/layer6_attention_weight_min": -49.00236892700195,
      "activations/layer7_attention_weight_max": 96.8158950805664,
      "activations/layer7_attention_weight_min": -103.80046844482422,
      "activations/layer8_attention_weight_max": 42.43791580200195,
      "activations/layer8_attention_weight_min": -47.389862060546875,
      "activations/layer9_attention_weight_max": 32.043514251708984,
      "activations/layer9_attention_weight_min": -35.0832405090332,
      "epoch": 21.03,
      "learning_rate": 1.4557954545454545e-05,
      "loss": 2.7157,
      "step": 361850
    },
    {
      "activations/layer0_attention_weight_max": 17.6307315826416,
      "activations/layer0_attention_weight_min": -13.776361465454102,
      "activations/layer10_attention_weight_max": 36.13529586791992,
      "activations/layer10_attention_weight_min": -35.97605895996094,
      "activations/layer11_attention_weight_max": 35.842498779296875,
      "activations/layer11_attention_weight_min": -35.34149932861328,
      "activations/layer12_attention_weight_max": 27.42702865600586,
      "activations/layer12_attention_weight_min": -28.006240844726562,
      "activations/layer13_attention_weight_max": 42.08572006225586,
      "activations/layer13_attention_weight_min": -36.43833923339844,
      "activations/layer14_attention_weight_max": 43.69349670410156,
      "activations/layer14_attention_weight_min": -40.07286071777344,
      "activations/layer15_attention_weight_max": 42.33142852783203,
      "activations/layer15_attention_weight_min": -38.89185333251953,
      "activations/layer16_attention_weight_max": 32.04866027832031,
      "activations/layer16_attention_weight_min": -29.259490966796875,
      "activations/layer17_attention_weight_max": 29.95297622680664,
      "activations/layer17_attention_weight_min": -26.23776626586914,
      "activations/layer18_attention_weight_max": 31.32192611694336,
      "activations/layer18_attention_weight_min": -27.35801887512207,
      "activations/layer19_attention_weight_max": 39.33992385864258,
      "activations/layer19_attention_weight_min": -34.77169418334961,
      "activations/layer1_attention_weight_max": 16.4897403717041,
      "activations/layer1_attention_weight_min": -16.635103225708008,
      "activations/layer20_attention_weight_max": 32.30772018432617,
      "activations/layer20_attention_weight_min": -27.86476707458496,
      "activations/layer21_attention_weight_max": 31.120235443115234,
      "activations/layer21_attention_weight_min": -27.44191551208496,
      "activations/layer22_attention_weight_max": 41.24018478393555,
      "activations/layer22_attention_weight_min": -33.551414489746094,
      "activations/layer23_attention_weight_max": 33.932037353515625,
      "activations/layer23_attention_weight_min": -28.72739028930664,
      "activations/layer2_attention_weight_max": 34.8500862121582,
      "activations/layer2_attention_weight_min": -33.24140930175781,
      "activations/layer3_attention_weight_max": 103.5479507446289,
      "activations/layer3_attention_weight_min": -105.18521881103516,
      "activations/layer4_attention_weight_max": 114.79143524169922,
      "activations/layer4_attention_weight_min": -113.47456359863281,
      "activations/layer5_attention_weight_max": 58.294288635253906,
      "activations/layer5_attention_weight_min": -64.34125518798828,
      "activations/layer6_attention_weight_max": 46.58156204223633,
      "activations/layer6_attention_weight_min": -49.83391571044922,
      "activations/layer7_attention_weight_max": 100.46858978271484,
      "activations/layer7_attention_weight_min": -104.27040100097656,
      "activations/layer8_attention_weight_max": 43.599273681640625,
      "activations/layer8_attention_weight_min": -46.34601593017578,
      "activations/layer9_attention_weight_max": 31.686813354492188,
      "activations/layer9_attention_weight_min": -33.54136276245117,
      "epoch": 21.03,
      "learning_rate": 1.4539015151515152e-05,
      "loss": 2.7058,
      "step": 361900
    },
    {
      "activations/layer0_attention_weight_max": 16.378673553466797,
      "activations/layer0_attention_weight_min": -13.237009048461914,
      "activations/layer10_attention_weight_max": 36.02762985229492,
      "activations/layer10_attention_weight_min": -36.2667350769043,
      "activations/layer11_attention_weight_max": 36.94567108154297,
      "activations/layer11_attention_weight_min": -36.623146057128906,
      "activations/layer12_attention_weight_max": 27.090930938720703,
      "activations/layer12_attention_weight_min": -27.95549201965332,
      "activations/layer13_attention_weight_max": 40.582977294921875,
      "activations/layer13_attention_weight_min": -39.65442657470703,
      "activations/layer14_attention_weight_max": 45.89176940917969,
      "activations/layer14_attention_weight_min": -38.91376876831055,
      "activations/layer15_attention_weight_max": 46.487388610839844,
      "activations/layer15_attention_weight_min": -38.52330017089844,
      "activations/layer16_attention_weight_max": 35.222412109375,
      "activations/layer16_attention_weight_min": -29.087345123291016,
      "activations/layer17_attention_weight_max": 35.396568298339844,
      "activations/layer17_attention_weight_min": -24.21776008605957,
      "activations/layer18_attention_weight_max": 37.48149490356445,
      "activations/layer18_attention_weight_min": -24.68785858154297,
      "activations/layer19_attention_weight_max": 38.54451370239258,
      "activations/layer19_attention_weight_min": -32.533931732177734,
      "activations/layer1_attention_weight_max": 16.197965621948242,
      "activations/layer1_attention_weight_min": -15.912466049194336,
      "activations/layer20_attention_weight_max": 33.13322067260742,
      "activations/layer20_attention_weight_min": -26.22402572631836,
      "activations/layer21_attention_weight_max": 29.12279510498047,
      "activations/layer21_attention_weight_min": -25.119659423828125,
      "activations/layer22_attention_weight_max": 40.22216033935547,
      "activations/layer22_attention_weight_min": -31.41956329345703,
      "activations/layer23_attention_weight_max": 33.35014724731445,
      "activations/layer23_attention_weight_min": -24.76584243774414,
      "activations/layer2_attention_weight_max": 33.933509826660156,
      "activations/layer2_attention_weight_min": -32.741455078125,
      "activations/layer3_attention_weight_max": 102.91845703125,
      "activations/layer3_attention_weight_min": -104.77078247070312,
      "activations/layer4_attention_weight_max": 109.8908462524414,
      "activations/layer4_attention_weight_min": -112.493408203125,
      "activations/layer5_attention_weight_max": 50.96245574951172,
      "activations/layer5_attention_weight_min": -64.36126708984375,
      "activations/layer6_attention_weight_max": 44.03616714477539,
      "activations/layer6_attention_weight_min": -46.59772491455078,
      "activations/layer7_attention_weight_max": 96.54532623291016,
      "activations/layer7_attention_weight_min": -96.41355895996094,
      "activations/layer8_attention_weight_max": 42.8420295715332,
      "activations/layer8_attention_weight_min": -42.82960891723633,
      "activations/layer9_attention_weight_max": 31.557727813720703,
      "activations/layer9_attention_weight_min": -33.941654205322266,
      "epoch": 21.03,
      "learning_rate": 1.4520075757575756e-05,
      "loss": 2.7035,
      "step": 361950
    },
    {
      "activations/layer0_attention_weight_max": 15.61888599395752,
      "activations/layer0_attention_weight_min": -14.868304252624512,
      "activations/layer10_attention_weight_max": 34.84647750854492,
      "activations/layer10_attention_weight_min": -33.31208419799805,
      "activations/layer11_attention_weight_max": 34.50965881347656,
      "activations/layer11_attention_weight_min": -32.849891662597656,
      "activations/layer12_attention_weight_max": 26.94498062133789,
      "activations/layer12_attention_weight_min": -26.15043830871582,
      "activations/layer13_attention_weight_max": 43.00006866455078,
      "activations/layer13_attention_weight_min": -35.93251037597656,
      "activations/layer14_attention_weight_max": 42.595802307128906,
      "activations/layer14_attention_weight_min": -39.85211944580078,
      "activations/layer15_attention_weight_max": 40.25065612792969,
      "activations/layer15_attention_weight_min": -37.141387939453125,
      "activations/layer16_attention_weight_max": 30.69566535949707,
      "activations/layer16_attention_weight_min": -28.302465438842773,
      "activations/layer17_attention_weight_max": 31.022077560424805,
      "activations/layer17_attention_weight_min": -25.964204788208008,
      "activations/layer18_attention_weight_max": 30.852985382080078,
      "activations/layer18_attention_weight_min": -26.089046478271484,
      "activations/layer19_attention_weight_max": 31.281530380249023,
      "activations/layer19_attention_weight_min": -32.041351318359375,
      "activations/layer1_attention_weight_max": 15.578582763671875,
      "activations/layer1_attention_weight_min": -14.920645713806152,
      "activations/layer20_attention_weight_max": 27.180858612060547,
      "activations/layer20_attention_weight_min": -26.298545837402344,
      "activations/layer21_attention_weight_max": 28.99053955078125,
      "activations/layer21_attention_weight_min": -26.161863327026367,
      "activations/layer22_attention_weight_max": 38.984405517578125,
      "activations/layer22_attention_weight_min": -29.990753173828125,
      "activations/layer23_attention_weight_max": 31.15711784362793,
      "activations/layer23_attention_weight_min": -25.564655303955078,
      "activations/layer2_attention_weight_max": 31.700515747070312,
      "activations/layer2_attention_weight_min": -32.036109924316406,
      "activations/layer3_attention_weight_max": 100.67902374267578,
      "activations/layer3_attention_weight_min": -99.28700256347656,
      "activations/layer4_attention_weight_max": 108.38581848144531,
      "activations/layer4_attention_weight_min": -109.64073944091797,
      "activations/layer5_attention_weight_max": 49.43024826049805,
      "activations/layer5_attention_weight_min": -62.97848129272461,
      "activations/layer6_attention_weight_max": 42.826622009277344,
      "activations/layer6_attention_weight_min": -46.053531646728516,
      "activations/layer7_attention_weight_max": 90.4275894165039,
      "activations/layer7_attention_weight_min": -99.77389526367188,
      "activations/layer8_attention_weight_max": 40.12500762939453,
      "activations/layer8_attention_weight_min": -43.388999938964844,
      "activations/layer9_attention_weight_max": 31.593374252319336,
      "activations/layer9_attention_weight_min": -34.268367767333984,
      "epoch": 21.03,
      "learning_rate": 1.4501136363636363e-05,
      "loss": 2.6999,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4937,
      "eval_samples_per_second": 505.553,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4937,
      "eval_openwebtext_samples_per_second": 505.553,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9718,
      "eval_wikitext_samples_per_second": 231.261,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_lambada_loss": 2.33984375,
      "eval_lambada_ppl": 10.379614621236398,
      "eval_lambada_runtime": 9.5361,
      "eval_lambada_samples_per_second": 510.585,
      "step": 362000
    },
    {
      "activations/layer0_attention_weight_max": 15.84825325012207,
      "activations/layer0_attention_weight_min": -14.387831687927246,
      "activations/layer10_attention_weight_max": 37.02886962890625,
      "activations/layer10_attention_weight_min": -37.10109329223633,
      "activations/layer11_attention_weight_max": 38.06682205200195,
      "activations/layer11_attention_weight_min": -37.88740921020508,
      "activations/layer12_attention_weight_max": 32.08159637451172,
      "activations/layer12_attention_weight_min": -28.63125991821289,
      "activations/layer13_attention_weight_max": 45.39865493774414,
      "activations/layer13_attention_weight_min": -37.920860290527344,
      "activations/layer14_attention_weight_max": 48.410987854003906,
      "activations/layer14_attention_weight_min": -39.99398422241211,
      "activations/layer15_attention_weight_max": 48.041099548339844,
      "activations/layer15_attention_weight_min": -39.75006866455078,
      "activations/layer16_attention_weight_max": 36.04250717163086,
      "activations/layer16_attention_weight_min": -29.646503448486328,
      "activations/layer17_attention_weight_max": 35.23471450805664,
      "activations/layer17_attention_weight_min": -27.414417266845703,
      "activations/layer18_attention_weight_max": 32.87389373779297,
      "activations/layer18_attention_weight_min": -27.889915466308594,
      "activations/layer19_attention_weight_max": 40.47194290161133,
      "activations/layer19_attention_weight_min": -31.963546752929688,
      "activations/layer1_attention_weight_max": 15.076766967773438,
      "activations/layer1_attention_weight_min": -14.693846702575684,
      "activations/layer20_attention_weight_max": 34.997196197509766,
      "activations/layer20_attention_weight_min": -26.024700164794922,
      "activations/layer21_attention_weight_max": 30.53453826904297,
      "activations/layer21_attention_weight_min": -25.088476181030273,
      "activations/layer22_attention_weight_max": 42.75669479370117,
      "activations/layer22_attention_weight_min": -31.916805267333984,
      "activations/layer23_attention_weight_max": 34.78833770751953,
      "activations/layer23_attention_weight_min": -25.5485897064209,
      "activations/layer2_attention_weight_max": 33.52314758300781,
      "activations/layer2_attention_weight_min": -32.004371643066406,
      "activations/layer3_attention_weight_max": 105.55895233154297,
      "activations/layer3_attention_weight_min": -104.88130187988281,
      "activations/layer4_attention_weight_max": 109.4605712890625,
      "activations/layer4_attention_weight_min": -112.74698638916016,
      "activations/layer5_attention_weight_max": 51.78205490112305,
      "activations/layer5_attention_weight_min": -64.05428314208984,
      "activations/layer6_attention_weight_max": 46.0037727355957,
      "activations/layer6_attention_weight_min": -48.2653694152832,
      "activations/layer7_attention_weight_max": 98.67288208007812,
      "activations/layer7_attention_weight_min": -101.56591033935547,
      "activations/layer8_attention_weight_max": 43.4346809387207,
      "activations/layer8_attention_weight_min": -46.97270202636719,
      "activations/layer9_attention_weight_max": 35.64552688598633,
      "activations/layer9_attention_weight_min": -40.05146408081055,
      "epoch": 21.04,
      "learning_rate": 1.4482196969696968e-05,
      "loss": 2.6956,
      "step": 362050
    },
    {
      "activations/layer0_attention_weight_max": 16.670616149902344,
      "activations/layer0_attention_weight_min": -14.812853813171387,
      "activations/layer10_attention_weight_max": 36.7852783203125,
      "activations/layer10_attention_weight_min": -38.09592819213867,
      "activations/layer11_attention_weight_max": 35.773948669433594,
      "activations/layer11_attention_weight_min": -36.52870178222656,
      "activations/layer12_attention_weight_max": 29.206886291503906,
      "activations/layer12_attention_weight_min": -28.464675903320312,
      "activations/layer13_attention_weight_max": 41.285701751708984,
      "activations/layer13_attention_weight_min": -38.17924499511719,
      "activations/layer14_attention_weight_max": 41.49928283691406,
      "activations/layer14_attention_weight_min": -38.50221633911133,
      "activations/layer15_attention_weight_max": 41.09206008911133,
      "activations/layer15_attention_weight_min": -37.716426849365234,
      "activations/layer16_attention_weight_max": 29.783353805541992,
      "activations/layer16_attention_weight_min": -28.277896881103516,
      "activations/layer17_attention_weight_max": 29.027551651000977,
      "activations/layer17_attention_weight_min": -25.399635314941406,
      "activations/layer18_attention_weight_max": 32.016578674316406,
      "activations/layer18_attention_weight_min": -25.07023811340332,
      "activations/layer19_attention_weight_max": 37.41010284423828,
      "activations/layer19_attention_weight_min": -31.6175479888916,
      "activations/layer1_attention_weight_max": 15.95559310913086,
      "activations/layer1_attention_weight_min": -15.597204208374023,
      "activations/layer20_attention_weight_max": 29.277740478515625,
      "activations/layer20_attention_weight_min": -25.383146286010742,
      "activations/layer21_attention_weight_max": 30.920339584350586,
      "activations/layer21_attention_weight_min": -25.160905838012695,
      "activations/layer22_attention_weight_max": 43.27646255493164,
      "activations/layer22_attention_weight_min": -30.99163055419922,
      "activations/layer23_attention_weight_max": 34.857826232910156,
      "activations/layer23_attention_weight_min": -27.00843620300293,
      "activations/layer2_attention_weight_max": 35.71917724609375,
      "activations/layer2_attention_weight_min": -33.34668731689453,
      "activations/layer3_attention_weight_max": 110.89839935302734,
      "activations/layer3_attention_weight_min": -110.92610168457031,
      "activations/layer4_attention_weight_max": 116.15267181396484,
      "activations/layer4_attention_weight_min": -115.52381896972656,
      "activations/layer5_attention_weight_max": 52.46534729003906,
      "activations/layer5_attention_weight_min": -61.798675537109375,
      "activations/layer6_attention_weight_max": 47.49019241333008,
      "activations/layer6_attention_weight_min": -50.33405685424805,
      "activations/layer7_attention_weight_max": 101.59529876708984,
      "activations/layer7_attention_weight_min": -101.77887725830078,
      "activations/layer8_attention_weight_max": 46.28337860107422,
      "activations/layer8_attention_weight_min": -46.13843536376953,
      "activations/layer9_attention_weight_max": 34.64695739746094,
      "activations/layer9_attention_weight_min": -36.34921646118164,
      "epoch": 21.04,
      "learning_rate": 1.4463257575757574e-05,
      "loss": 2.7028,
      "step": 362100
    },
    {
      "activations/layer0_attention_weight_max": 16.648134231567383,
      "activations/layer0_attention_weight_min": -13.233634948730469,
      "activations/layer10_attention_weight_max": 35.55819320678711,
      "activations/layer10_attention_weight_min": -36.53487014770508,
      "activations/layer11_attention_weight_max": 37.00981903076172,
      "activations/layer11_attention_weight_min": -36.066524505615234,
      "activations/layer12_attention_weight_max": 28.65609359741211,
      "activations/layer12_attention_weight_min": -28.22311782836914,
      "activations/layer13_attention_weight_max": 41.69828796386719,
      "activations/layer13_attention_weight_min": -36.685523986816406,
      "activations/layer14_attention_weight_max": 41.041465759277344,
      "activations/layer14_attention_weight_min": -39.07905578613281,
      "activations/layer15_attention_weight_max": 40.08307647705078,
      "activations/layer15_attention_weight_min": -38.27092742919922,
      "activations/layer16_attention_weight_max": 31.091445922851562,
      "activations/layer16_attention_weight_min": -29.11349105834961,
      "activations/layer17_attention_weight_max": 29.969934463500977,
      "activations/layer17_attention_weight_min": -26.84077262878418,
      "activations/layer18_attention_weight_max": 30.455522537231445,
      "activations/layer18_attention_weight_min": -29.030508041381836,
      "activations/layer19_attention_weight_max": 35.69685745239258,
      "activations/layer19_attention_weight_min": -35.20680618286133,
      "activations/layer1_attention_weight_max": 15.668750762939453,
      "activations/layer1_attention_weight_min": -14.889836311340332,
      "activations/layer20_attention_weight_max": 31.275611877441406,
      "activations/layer20_attention_weight_min": -27.94207191467285,
      "activations/layer21_attention_weight_max": 30.696006774902344,
      "activations/layer21_attention_weight_min": -27.713083267211914,
      "activations/layer22_attention_weight_max": 42.42634201049805,
      "activations/layer22_attention_weight_min": -36.52146530151367,
      "activations/layer23_attention_weight_max": 32.06161880493164,
      "activations/layer23_attention_weight_min": -27.872114181518555,
      "activations/layer2_attention_weight_max": 34.08573913574219,
      "activations/layer2_attention_weight_min": -32.69562530517578,
      "activations/layer3_attention_weight_max": 105.35652160644531,
      "activations/layer3_attention_weight_min": -105.58984375,
      "activations/layer4_attention_weight_max": 111.9865493774414,
      "activations/layer4_attention_weight_min": -110.36871337890625,
      "activations/layer5_attention_weight_max": 50.97406005859375,
      "activations/layer5_attention_weight_min": -63.37586975097656,
      "activations/layer6_attention_weight_max": 46.137210845947266,
      "activations/layer6_attention_weight_min": -49.49844741821289,
      "activations/layer7_attention_weight_max": 94.26212310791016,
      "activations/layer7_attention_weight_min": -97.00885772705078,
      "activations/layer8_attention_weight_max": 41.660308837890625,
      "activations/layer8_attention_weight_min": -45.08222961425781,
      "activations/layer9_attention_weight_max": 32.80171585083008,
      "activations/layer9_attention_weight_min": -37.39354705810547,
      "epoch": 21.04,
      "learning_rate": 1.4444318181818181e-05,
      "loss": 2.7052,
      "step": 362150
    },
    {
      "activations/layer0_attention_weight_max": 16.55820655822754,
      "activations/layer0_attention_weight_min": -13.895485877990723,
      "activations/layer10_attention_weight_max": 48.45004653930664,
      "activations/layer10_attention_weight_min": -45.71470260620117,
      "activations/layer11_attention_weight_max": 48.482452392578125,
      "activations/layer11_attention_weight_min": -45.994407653808594,
      "activations/layer12_attention_weight_max": 31.066816329956055,
      "activations/layer12_attention_weight_min": -31.28826141357422,
      "activations/layer13_attention_weight_max": 41.209957122802734,
      "activations/layer13_attention_weight_min": -37.76710510253906,
      "activations/layer14_attention_weight_max": 46.99713134765625,
      "activations/layer14_attention_weight_min": -46.06483459472656,
      "activations/layer15_attention_weight_max": 46.114532470703125,
      "activations/layer15_attention_weight_min": -44.27105712890625,
      "activations/layer16_attention_weight_max": 32.39533996582031,
      "activations/layer16_attention_weight_min": -30.553869247436523,
      "activations/layer17_attention_weight_max": 28.631460189819336,
      "activations/layer17_attention_weight_min": -27.88274574279785,
      "activations/layer18_attention_weight_max": 27.716167449951172,
      "activations/layer18_attention_weight_min": -28.433734893798828,
      "activations/layer19_attention_weight_max": 34.55353927612305,
      "activations/layer19_attention_weight_min": -32.35588455200195,
      "activations/layer1_attention_weight_max": 15.574251174926758,
      "activations/layer1_attention_weight_min": -15.30915355682373,
      "activations/layer20_attention_weight_max": 28.54875373840332,
      "activations/layer20_attention_weight_min": -25.464677810668945,
      "activations/layer21_attention_weight_max": 27.185911178588867,
      "activations/layer21_attention_weight_min": -25.064800262451172,
      "activations/layer22_attention_weight_max": 44.4927978515625,
      "activations/layer22_attention_weight_min": -32.82059860229492,
      "activations/layer23_attention_weight_max": 31.420743942260742,
      "activations/layer23_attention_weight_min": -27.691791534423828,
      "activations/layer2_attention_weight_max": 38.271305084228516,
      "activations/layer2_attention_weight_min": -36.686988830566406,
      "activations/layer3_attention_weight_max": 117.67899322509766,
      "activations/layer3_attention_weight_min": -117.87193298339844,
      "activations/layer4_attention_weight_max": 120.78954315185547,
      "activations/layer4_attention_weight_min": -113.93489837646484,
      "activations/layer5_attention_weight_max": 49.993160247802734,
      "activations/layer5_attention_weight_min": -62.69417953491211,
      "activations/layer6_attention_weight_max": 47.1854133605957,
      "activations/layer6_attention_weight_min": -49.87482452392578,
      "activations/layer7_attention_weight_max": 120.10150146484375,
      "activations/layer7_attention_weight_min": -111.1996078491211,
      "activations/layer8_attention_weight_max": 53.494606018066406,
      "activations/layer8_attention_weight_min": -53.6309814453125,
      "activations/layer9_attention_weight_max": 39.64373779296875,
      "activations/layer9_attention_weight_min": -40.74152755737305,
      "epoch": 21.05,
      "learning_rate": 1.4425378787878786e-05,
      "loss": 2.7124,
      "step": 362200
    },
    {
      "activations/layer0_attention_weight_max": 16.15361785888672,
      "activations/layer0_attention_weight_min": -15.227008819580078,
      "activations/layer10_attention_weight_max": 36.53064727783203,
      "activations/layer10_attention_weight_min": -38.0556640625,
      "activations/layer11_attention_weight_max": 34.877525329589844,
      "activations/layer11_attention_weight_min": -37.156070709228516,
      "activations/layer12_attention_weight_max": 27.56504249572754,
      "activations/layer12_attention_weight_min": -26.6658992767334,
      "activations/layer13_attention_weight_max": 40.56764221191406,
      "activations/layer13_attention_weight_min": -35.114078521728516,
      "activations/layer14_attention_weight_max": 45.15556716918945,
      "activations/layer14_attention_weight_min": -38.93253707885742,
      "activations/layer15_attention_weight_max": 42.277652740478516,
      "activations/layer15_attention_weight_min": -36.99753952026367,
      "activations/layer16_attention_weight_max": 27.597129821777344,
      "activations/layer16_attention_weight_min": -29.286266326904297,
      "activations/layer17_attention_weight_max": 29.219919204711914,
      "activations/layer17_attention_weight_min": -26.233999252319336,
      "activations/layer18_attention_weight_max": 32.657867431640625,
      "activations/layer18_attention_weight_min": -29.69137191772461,
      "activations/layer19_attention_weight_max": 34.40925598144531,
      "activations/layer19_attention_weight_min": -32.12800216674805,
      "activations/layer1_attention_weight_max": 15.949254989624023,
      "activations/layer1_attention_weight_min": -16.109149932861328,
      "activations/layer20_attention_weight_max": 29.704723358154297,
      "activations/layer20_attention_weight_min": -25.278457641601562,
      "activations/layer21_attention_weight_max": 32.699222564697266,
      "activations/layer21_attention_weight_min": -25.254064559936523,
      "activations/layer22_attention_weight_max": 42.71274185180664,
      "activations/layer22_attention_weight_min": -31.80743980407715,
      "activations/layer23_attention_weight_max": 31.224151611328125,
      "activations/layer23_attention_weight_min": -26.836219787597656,
      "activations/layer2_attention_weight_max": 33.555152893066406,
      "activations/layer2_attention_weight_min": -34.24342727661133,
      "activations/layer3_attention_weight_max": 105.52481842041016,
      "activations/layer3_attention_weight_min": -106.6545181274414,
      "activations/layer4_attention_weight_max": 106.52229309082031,
      "activations/layer4_attention_weight_min": -113.04610443115234,
      "activations/layer5_attention_weight_max": 50.0924072265625,
      "activations/layer5_attention_weight_min": -66.12515258789062,
      "activations/layer6_attention_weight_max": 43.52253723144531,
      "activations/layer6_attention_weight_min": -47.6751594543457,
      "activations/layer7_attention_weight_max": 95.19175720214844,
      "activations/layer7_attention_weight_min": -102.36003875732422,
      "activations/layer8_attention_weight_max": 43.134864807128906,
      "activations/layer8_attention_weight_min": -46.29206085205078,
      "activations/layer9_attention_weight_max": 33.47764205932617,
      "activations/layer9_attention_weight_min": -37.657344818115234,
      "epoch": 21.05,
      "learning_rate": 1.4406439393939392e-05,
      "loss": 2.694,
      "step": 362250
    },
    {
      "activations/layer0_attention_weight_max": 16.804485321044922,
      "activations/layer0_attention_weight_min": -14.709877967834473,
      "activations/layer10_attention_weight_max": 35.963157653808594,
      "activations/layer10_attention_weight_min": -36.11873245239258,
      "activations/layer11_attention_weight_max": 35.11627960205078,
      "activations/layer11_attention_weight_min": -36.330894470214844,
      "activations/layer12_attention_weight_max": 27.008127212524414,
      "activations/layer12_attention_weight_min": -27.182357788085938,
      "activations/layer13_attention_weight_max": 38.13957595825195,
      "activations/layer13_attention_weight_min": -34.38875961303711,
      "activations/layer14_attention_weight_max": 40.50685119628906,
      "activations/layer14_attention_weight_min": -37.548316955566406,
      "activations/layer15_attention_weight_max": 37.261878967285156,
      "activations/layer15_attention_weight_min": -36.310325622558594,
      "activations/layer16_attention_weight_max": 28.510696411132812,
      "activations/layer16_attention_weight_min": -28.958890914916992,
      "activations/layer17_attention_weight_max": 28.12571144104004,
      "activations/layer17_attention_weight_min": -26.565332412719727,
      "activations/layer18_attention_weight_max": 34.55695343017578,
      "activations/layer18_attention_weight_min": -26.933616638183594,
      "activations/layer19_attention_weight_max": 37.65917205810547,
      "activations/layer19_attention_weight_min": -35.276123046875,
      "activations/layer1_attention_weight_max": 15.714759826660156,
      "activations/layer1_attention_weight_min": -15.226810455322266,
      "activations/layer20_attention_weight_max": 29.965110778808594,
      "activations/layer20_attention_weight_min": -26.040508270263672,
      "activations/layer21_attention_weight_max": 31.49456787109375,
      "activations/layer21_attention_weight_min": -26.01727867126465,
      "activations/layer22_attention_weight_max": 44.051090240478516,
      "activations/layer22_attention_weight_min": -33.7288818359375,
      "activations/layer23_attention_weight_max": 35.351715087890625,
      "activations/layer23_attention_weight_min": -26.21849822998047,
      "activations/layer2_attention_weight_max": 33.75373840332031,
      "activations/layer2_attention_weight_min": -33.89931106567383,
      "activations/layer3_attention_weight_max": 104.86709594726562,
      "activations/layer3_attention_weight_min": -104.49957275390625,
      "activations/layer4_attention_weight_max": 111.7466049194336,
      "activations/layer4_attention_weight_min": -113.27079010009766,
      "activations/layer5_attention_weight_max": 52.14934539794922,
      "activations/layer5_attention_weight_min": -64.40469360351562,
      "activations/layer6_attention_weight_max": 45.8301887512207,
      "activations/layer6_attention_weight_min": -47.674625396728516,
      "activations/layer7_attention_weight_max": 101.48824310302734,
      "activations/layer7_attention_weight_min": -101.47614288330078,
      "activations/layer8_attention_weight_max": 44.00179672241211,
      "activations/layer8_attention_weight_min": -45.50252151489258,
      "activations/layer9_attention_weight_max": 33.520328521728516,
      "activations/layer9_attention_weight_min": -34.801631927490234,
      "epoch": 21.05,
      "learning_rate": 1.4387499999999999e-05,
      "loss": 2.7041,
      "step": 362300
    },
    {
      "activations/layer0_attention_weight_max": 16.378328323364258,
      "activations/layer0_attention_weight_min": -13.962284088134766,
      "activations/layer10_attention_weight_max": 33.2148323059082,
      "activations/layer10_attention_weight_min": -34.8379020690918,
      "activations/layer11_attention_weight_max": 34.22332763671875,
      "activations/layer11_attention_weight_min": -37.1182861328125,
      "activations/layer12_attention_weight_max": 29.841426849365234,
      "activations/layer12_attention_weight_min": -27.035213470458984,
      "activations/layer13_attention_weight_max": 39.629940032958984,
      "activations/layer13_attention_weight_min": -36.706207275390625,
      "activations/layer14_attention_weight_max": 42.513160705566406,
      "activations/layer14_attention_weight_min": -38.035457611083984,
      "activations/layer15_attention_weight_max": 39.28024673461914,
      "activations/layer15_attention_weight_min": -37.668479919433594,
      "activations/layer16_attention_weight_max": 29.111221313476562,
      "activations/layer16_attention_weight_min": -29.095659255981445,
      "activations/layer17_attention_weight_max": 29.816415786743164,
      "activations/layer17_attention_weight_min": -26.3146915435791,
      "activations/layer18_attention_weight_max": 29.686485290527344,
      "activations/layer18_attention_weight_min": -27.540958404541016,
      "activations/layer19_attention_weight_max": 38.678157806396484,
      "activations/layer19_attention_weight_min": -31.91315269470215,
      "activations/layer1_attention_weight_max": 16.063613891601562,
      "activations/layer1_attention_weight_min": -17.55043601989746,
      "activations/layer20_attention_weight_max": 30.1043758392334,
      "activations/layer20_attention_weight_min": -26.861183166503906,
      "activations/layer21_attention_weight_max": 28.437681198120117,
      "activations/layer21_attention_weight_min": -26.779653549194336,
      "activations/layer22_attention_weight_max": 42.18947219848633,
      "activations/layer22_attention_weight_min": -32.916481018066406,
      "activations/layer23_attention_weight_max": 30.83413314819336,
      "activations/layer23_attention_weight_min": -24.895498275756836,
      "activations/layer2_attention_weight_max": 33.153099060058594,
      "activations/layer2_attention_weight_min": -34.032997131347656,
      "activations/layer3_attention_weight_max": 103.343505859375,
      "activations/layer3_attention_weight_min": -108.4528579711914,
      "activations/layer4_attention_weight_max": 111.7934799194336,
      "activations/layer4_attention_weight_min": -116.87236785888672,
      "activations/layer5_attention_weight_max": 50.630409240722656,
      "activations/layer5_attention_weight_min": -63.00440979003906,
      "activations/layer6_attention_weight_max": 47.2823600769043,
      "activations/layer6_attention_weight_min": -50.3106803894043,
      "activations/layer7_attention_weight_max": 94.7817611694336,
      "activations/layer7_attention_weight_min": -103.4681167602539,
      "activations/layer8_attention_weight_max": 44.64592742919922,
      "activations/layer8_attention_weight_min": -46.07533645629883,
      "activations/layer9_attention_weight_max": 31.169919967651367,
      "activations/layer9_attention_weight_min": -34.30705642700195,
      "epoch": 21.05,
      "learning_rate": 1.4368560606060604e-05,
      "loss": 2.6982,
      "step": 362350
    },
    {
      "activations/layer0_attention_weight_max": 16.379058837890625,
      "activations/layer0_attention_weight_min": -14.638455390930176,
      "activations/layer10_attention_weight_max": 36.30696487426758,
      "activations/layer10_attention_weight_min": -35.81177520751953,
      "activations/layer11_attention_weight_max": 37.08745193481445,
      "activations/layer11_attention_weight_min": -34.420982360839844,
      "activations/layer12_attention_weight_max": 28.909439086914062,
      "activations/layer12_attention_weight_min": -27.186317443847656,
      "activations/layer13_attention_weight_max": 42.09133529663086,
      "activations/layer13_attention_weight_min": -36.68477249145508,
      "activations/layer14_attention_weight_max": 45.964691162109375,
      "activations/layer14_attention_weight_min": -39.119754791259766,
      "activations/layer15_attention_weight_max": 42.204647064208984,
      "activations/layer15_attention_weight_min": -35.90214157104492,
      "activations/layer16_attention_weight_max": 28.907575607299805,
      "activations/layer16_attention_weight_min": -28.678447723388672,
      "activations/layer17_attention_weight_max": 32.219425201416016,
      "activations/layer17_attention_weight_min": -25.9541072845459,
      "activations/layer18_attention_weight_max": 31.695064544677734,
      "activations/layer18_attention_weight_min": -26.077491760253906,
      "activations/layer19_attention_weight_max": 33.350101470947266,
      "activations/layer19_attention_weight_min": -31.365966796875,
      "activations/layer1_attention_weight_max": 16.080265045166016,
      "activations/layer1_attention_weight_min": -14.451330184936523,
      "activations/layer20_attention_weight_max": 26.977609634399414,
      "activations/layer20_attention_weight_min": -25.576555252075195,
      "activations/layer21_attention_weight_max": 28.675579071044922,
      "activations/layer21_attention_weight_min": -24.382999420166016,
      "activations/layer22_attention_weight_max": 41.63873291015625,
      "activations/layer22_attention_weight_min": -32.09946823120117,
      "activations/layer23_attention_weight_max": 33.14344024658203,
      "activations/layer23_attention_weight_min": -26.12318229675293,
      "activations/layer2_attention_weight_max": 32.3477783203125,
      "activations/layer2_attention_weight_min": -32.246612548828125,
      "activations/layer3_attention_weight_max": 103.1961441040039,
      "activations/layer3_attention_weight_min": -102.75958251953125,
      "activations/layer4_attention_weight_max": 107.12483978271484,
      "activations/layer4_attention_weight_min": -110.85125732421875,
      "activations/layer5_attention_weight_max": 49.383522033691406,
      "activations/layer5_attention_weight_min": -60.89391326904297,
      "activations/layer6_attention_weight_max": 43.04572677612305,
      "activations/layer6_attention_weight_min": -46.595829010009766,
      "activations/layer7_attention_weight_max": 94.16632843017578,
      "activations/layer7_attention_weight_min": -99.47891998291016,
      "activations/layer8_attention_weight_max": 44.67024612426758,
      "activations/layer8_attention_weight_min": -44.51696014404297,
      "activations/layer9_attention_weight_max": 34.88502502441406,
      "activations/layer9_attention_weight_min": -34.58700180053711,
      "epoch": 21.06,
      "learning_rate": 1.434962121212121e-05,
      "loss": 2.7173,
      "step": 362400
    },
    {
      "activations/layer0_attention_weight_max": 17.14837074279785,
      "activations/layer0_attention_weight_min": -15.554940223693848,
      "activations/layer10_attention_weight_max": 33.4833984375,
      "activations/layer10_attention_weight_min": -34.346351623535156,
      "activations/layer11_attention_weight_max": 35.8724365234375,
      "activations/layer11_attention_weight_min": -35.83905029296875,
      "activations/layer12_attention_weight_max": 26.989933013916016,
      "activations/layer12_attention_weight_min": -26.088346481323242,
      "activations/layer13_attention_weight_max": 40.36127853393555,
      "activations/layer13_attention_weight_min": -35.39989471435547,
      "activations/layer14_attention_weight_max": 44.11919403076172,
      "activations/layer14_attention_weight_min": -38.027198791503906,
      "activations/layer15_attention_weight_max": 42.75465774536133,
      "activations/layer15_attention_weight_min": -38.687477111816406,
      "activations/layer16_attention_weight_max": 32.86417007446289,
      "activations/layer16_attention_weight_min": -29.308820724487305,
      "activations/layer17_attention_weight_max": 28.694698333740234,
      "activations/layer17_attention_weight_min": -26.123167037963867,
      "activations/layer18_attention_weight_max": 34.751766204833984,
      "activations/layer18_attention_weight_min": -29.358688354492188,
      "activations/layer19_attention_weight_max": 32.087947845458984,
      "activations/layer19_attention_weight_min": -32.21402359008789,
      "activations/layer1_attention_weight_max": 16.264251708984375,
      "activations/layer1_attention_weight_min": -15.937664985656738,
      "activations/layer20_attention_weight_max": 27.734018325805664,
      "activations/layer20_attention_weight_min": -25.759836196899414,
      "activations/layer21_attention_weight_max": 29.667081832885742,
      "activations/layer21_attention_weight_min": -25.551090240478516,
      "activations/layer22_attention_weight_max": 39.2602424621582,
      "activations/layer22_attention_weight_min": -33.77217102050781,
      "activations/layer23_attention_weight_max": 30.527629852294922,
      "activations/layer23_attention_weight_min": -27.495521545410156,
      "activations/layer2_attention_weight_max": 34.31884002685547,
      "activations/layer2_attention_weight_min": -32.98334503173828,
      "activations/layer3_attention_weight_max": 102.75224304199219,
      "activations/layer3_attention_weight_min": -101.57022094726562,
      "activations/layer4_attention_weight_max": 106.60149383544922,
      "activations/layer4_attention_weight_min": -108.55816650390625,
      "activations/layer5_attention_weight_max": 49.107460021972656,
      "activations/layer5_attention_weight_min": -62.333457946777344,
      "activations/layer6_attention_weight_max": 43.60697937011719,
      "activations/layer6_attention_weight_min": -46.93142318725586,
      "activations/layer7_attention_weight_max": 95.27041625976562,
      "activations/layer7_attention_weight_min": -96.76642608642578,
      "activations/layer8_attention_weight_max": 42.4017333984375,
      "activations/layer8_attention_weight_min": -44.41614532470703,
      "activations/layer9_attention_weight_max": 32.05510330200195,
      "activations/layer9_attention_weight_min": -34.53687286376953,
      "epoch": 21.06,
      "learning_rate": 1.4330681818181815e-05,
      "loss": 2.6917,
      "step": 362450
    },
    {
      "activations/layer0_attention_weight_max": 16.706714630126953,
      "activations/layer0_attention_weight_min": -13.390389442443848,
      "activations/layer10_attention_weight_max": 37.751766204833984,
      "activations/layer10_attention_weight_min": -37.20701599121094,
      "activations/layer11_attention_weight_max": 35.705589294433594,
      "activations/layer11_attention_weight_min": -39.3642463684082,
      "activations/layer12_attention_weight_max": 29.195335388183594,
      "activations/layer12_attention_weight_min": -29.98356819152832,
      "activations/layer13_attention_weight_max": 44.45444869995117,
      "activations/layer13_attention_weight_min": -38.13058090209961,
      "activations/layer14_attention_weight_max": 42.94362258911133,
      "activations/layer14_attention_weight_min": -38.978302001953125,
      "activations/layer15_attention_weight_max": 43.023887634277344,
      "activations/layer15_attention_weight_min": -39.16958999633789,
      "activations/layer16_attention_weight_max": 29.194799423217773,
      "activations/layer16_attention_weight_min": -29.02544403076172,
      "activations/layer17_attention_weight_max": 31.388044357299805,
      "activations/layer17_attention_weight_min": -28.952566146850586,
      "activations/layer18_attention_weight_max": 33.68445587158203,
      "activations/layer18_attention_weight_min": -30.46576499938965,
      "activations/layer19_attention_weight_max": 34.02627182006836,
      "activations/layer19_attention_weight_min": -34.03843307495117,
      "activations/layer1_attention_weight_max": 15.585719108581543,
      "activations/layer1_attention_weight_min": -14.550126075744629,
      "activations/layer20_attention_weight_max": 32.23710632324219,
      "activations/layer20_attention_weight_min": -27.119171142578125,
      "activations/layer21_attention_weight_max": 30.589569091796875,
      "activations/layer21_attention_weight_min": -26.986228942871094,
      "activations/layer22_attention_weight_max": 40.334312438964844,
      "activations/layer22_attention_weight_min": -34.648128509521484,
      "activations/layer23_attention_weight_max": 31.320356369018555,
      "activations/layer23_attention_weight_min": -25.87145233154297,
      "activations/layer2_attention_weight_max": 32.749229431152344,
      "activations/layer2_attention_weight_min": -31.869792938232422,
      "activations/layer3_attention_weight_max": 102.61884307861328,
      "activations/layer3_attention_weight_min": -103.77326965332031,
      "activations/layer4_attention_weight_max": 111.04347229003906,
      "activations/layer4_attention_weight_min": -111.50306701660156,
      "activations/layer5_attention_weight_max": 51.05599594116211,
      "activations/layer5_attention_weight_min": -63.75797653198242,
      "activations/layer6_attention_weight_max": 45.41433334350586,
      "activations/layer6_attention_weight_min": -49.681095123291016,
      "activations/layer7_attention_weight_max": 105.2314224243164,
      "activations/layer7_attention_weight_min": -102.56956481933594,
      "activations/layer8_attention_weight_max": 45.9479866027832,
      "activations/layer8_attention_weight_min": -48.43194580078125,
      "activations/layer9_attention_weight_max": 35.9290885925293,
      "activations/layer9_attention_weight_min": -37.26919174194336,
      "epoch": 21.06,
      "learning_rate": 1.4311742424242422e-05,
      "loss": 2.7213,
      "step": 362500
    },
    {
      "activations/layer0_attention_weight_max": 17.105756759643555,
      "activations/layer0_attention_weight_min": -14.396795272827148,
      "activations/layer10_attention_weight_max": 33.971866607666016,
      "activations/layer10_attention_weight_min": -34.788108825683594,
      "activations/layer11_attention_weight_max": 33.30712127685547,
      "activations/layer11_attention_weight_min": -35.107383728027344,
      "activations/layer12_attention_weight_max": 25.7742977142334,
      "activations/layer12_attention_weight_min": -25.696565628051758,
      "activations/layer13_attention_weight_max": 38.1075325012207,
      "activations/layer13_attention_weight_min": -34.92131042480469,
      "activations/layer14_attention_weight_max": 42.35620880126953,
      "activations/layer14_attention_weight_min": -38.234371185302734,
      "activations/layer15_attention_weight_max": 40.007320404052734,
      "activations/layer15_attention_weight_min": -36.061458587646484,
      "activations/layer16_attention_weight_max": 31.01305389404297,
      "activations/layer16_attention_weight_min": -28.096542358398438,
      "activations/layer17_attention_weight_max": 29.205820083618164,
      "activations/layer17_attention_weight_min": -26.739770889282227,
      "activations/layer18_attention_weight_max": 31.10845947265625,
      "activations/layer18_attention_weight_min": -25.989887237548828,
      "activations/layer19_attention_weight_max": 33.00349044799805,
      "activations/layer19_attention_weight_min": -32.00166320800781,
      "activations/layer1_attention_weight_max": 15.555984497070312,
      "activations/layer1_attention_weight_min": -15.608147621154785,
      "activations/layer20_attention_weight_max": 28.131879806518555,
      "activations/layer20_attention_weight_min": -25.86383628845215,
      "activations/layer21_attention_weight_max": 29.597875595092773,
      "activations/layer21_attention_weight_min": -24.614280700683594,
      "activations/layer22_attention_weight_max": 41.80378723144531,
      "activations/layer22_attention_weight_min": -32.089900970458984,
      "activations/layer23_attention_weight_max": 33.85051727294922,
      "activations/layer23_attention_weight_min": -28.631954193115234,
      "activations/layer2_attention_weight_max": 31.285749435424805,
      "activations/layer2_attention_weight_min": -31.633201599121094,
      "activations/layer3_attention_weight_max": 102.6856460571289,
      "activations/layer3_attention_weight_min": -102.40404510498047,
      "activations/layer4_attention_weight_max": 105.4721450805664,
      "activations/layer4_attention_weight_min": -108.89688873291016,
      "activations/layer5_attention_weight_max": 48.14786911010742,
      "activations/layer5_attention_weight_min": -65.63933563232422,
      "activations/layer6_attention_weight_max": 43.18205261230469,
      "activations/layer6_attention_weight_min": -47.12662887573242,
      "activations/layer7_attention_weight_max": 89.87019348144531,
      "activations/layer7_attention_weight_min": -98.05109405517578,
      "activations/layer8_attention_weight_max": 42.84514236450195,
      "activations/layer8_attention_weight_min": -44.50999069213867,
      "activations/layer9_attention_weight_max": 31.979461669921875,
      "activations/layer9_attention_weight_min": -33.50216293334961,
      "epoch": 21.07,
      "learning_rate": 1.4292803030303029e-05,
      "loss": 2.7118,
      "step": 362550
    },
    {
      "activations/layer0_attention_weight_max": 16.548845291137695,
      "activations/layer0_attention_weight_min": -13.341838836669922,
      "activations/layer10_attention_weight_max": 36.28017044067383,
      "activations/layer10_attention_weight_min": -34.6212043762207,
      "activations/layer11_attention_weight_max": 34.015445709228516,
      "activations/layer11_attention_weight_min": -34.8282356262207,
      "activations/layer12_attention_weight_max": 26.107948303222656,
      "activations/layer12_attention_weight_min": -27.71834945678711,
      "activations/layer13_attention_weight_max": 39.9759521484375,
      "activations/layer13_attention_weight_min": -37.1967887878418,
      "activations/layer14_attention_weight_max": 42.805641174316406,
      "activations/layer14_attention_weight_min": -38.633419036865234,
      "activations/layer15_attention_weight_max": 41.41157913208008,
      "activations/layer15_attention_weight_min": -39.467140197753906,
      "activations/layer16_attention_weight_max": 29.501712799072266,
      "activations/layer16_attention_weight_min": -29.302690505981445,
      "activations/layer17_attention_weight_max": 31.54689598083496,
      "activations/layer17_attention_weight_min": -27.747943878173828,
      "activations/layer18_attention_weight_max": 37.11494064331055,
      "activations/layer18_attention_weight_min": -25.07526969909668,
      "activations/layer19_attention_weight_max": 34.719486236572266,
      "activations/layer19_attention_weight_min": -32.475616455078125,
      "activations/layer1_attention_weight_max": 15.151491165161133,
      "activations/layer1_attention_weight_min": -14.58576488494873,
      "activations/layer20_attention_weight_max": 30.854713439941406,
      "activations/layer20_attention_weight_min": -25.722455978393555,
      "activations/layer21_attention_weight_max": 29.286733627319336,
      "activations/layer21_attention_weight_min": -25.455835342407227,
      "activations/layer22_attention_weight_max": 41.377586364746094,
      "activations/layer22_attention_weight_min": -30.71712875366211,
      "activations/layer23_attention_weight_max": 30.277610778808594,
      "activations/layer23_attention_weight_min": -26.358970642089844,
      "activations/layer2_attention_weight_max": 32.51312255859375,
      "activations/layer2_attention_weight_min": -32.88818359375,
      "activations/layer3_attention_weight_max": 106.00861358642578,
      "activations/layer3_attention_weight_min": -104.38623046875,
      "activations/layer4_attention_weight_max": 106.471923828125,
      "activations/layer4_attention_weight_min": -110.3585433959961,
      "activations/layer5_attention_weight_max": 50.206634521484375,
      "activations/layer5_attention_weight_min": -59.937042236328125,
      "activations/layer6_attention_weight_max": 46.82356643676758,
      "activations/layer6_attention_weight_min": -48.920021057128906,
      "activations/layer7_attention_weight_max": 95.00431823730469,
      "activations/layer7_attention_weight_min": -102.1136245727539,
      "activations/layer8_attention_weight_max": 44.503082275390625,
      "activations/layer8_attention_weight_min": -45.99652862548828,
      "activations/layer9_attention_weight_max": 34.38018798828125,
      "activations/layer9_attention_weight_min": -35.7221794128418,
      "epoch": 21.07,
      "learning_rate": 1.4273863636363637e-05,
      "loss": 2.7042,
      "step": 362600
    },
    {
      "activations/layer0_attention_weight_max": 16.373767852783203,
      "activations/layer0_attention_weight_min": -13.488844871520996,
      "activations/layer10_attention_weight_max": 40.49315643310547,
      "activations/layer10_attention_weight_min": -40.72289276123047,
      "activations/layer11_attention_weight_max": 41.21758270263672,
      "activations/layer11_attention_weight_min": -43.776607513427734,
      "activations/layer12_attention_weight_max": 28.456954956054688,
      "activations/layer12_attention_weight_min": -29.47034454345703,
      "activations/layer13_attention_weight_max": 41.552818298339844,
      "activations/layer13_attention_weight_min": -39.3907356262207,
      "activations/layer14_attention_weight_max": 44.78103256225586,
      "activations/layer14_attention_weight_min": -39.76923370361328,
      "activations/layer15_attention_weight_max": 42.9864387512207,
      "activations/layer15_attention_weight_min": -43.04179382324219,
      "activations/layer16_attention_weight_max": 31.78684425354004,
      "activations/layer16_attention_weight_min": -31.168277740478516,
      "activations/layer17_attention_weight_max": 32.838619232177734,
      "activations/layer17_attention_weight_min": -29.17417335510254,
      "activations/layer18_attention_weight_max": 34.974456787109375,
      "activations/layer18_attention_weight_min": -30.68377113342285,
      "activations/layer19_attention_weight_max": 34.450008392333984,
      "activations/layer19_attention_weight_min": -32.30946350097656,
      "activations/layer1_attention_weight_max": 15.86579418182373,
      "activations/layer1_attention_weight_min": -14.992778778076172,
      "activations/layer20_attention_weight_max": 29.41539764404297,
      "activations/layer20_attention_weight_min": -24.409093856811523,
      "activations/layer21_attention_weight_max": 28.45844078063965,
      "activations/layer21_attention_weight_min": -24.596391677856445,
      "activations/layer22_attention_weight_max": 42.891639709472656,
      "activations/layer22_attention_weight_min": -31.596500396728516,
      "activations/layer23_attention_weight_max": 34.45729064941406,
      "activations/layer23_attention_weight_min": -25.018657684326172,
      "activations/layer2_attention_weight_max": 34.137908935546875,
      "activations/layer2_attention_weight_min": -34.51228332519531,
      "activations/layer3_attention_weight_max": 104.90519714355469,
      "activations/layer3_attention_weight_min": -110.75244903564453,
      "activations/layer4_attention_weight_max": 113.9728775024414,
      "activations/layer4_attention_weight_min": -114.29393768310547,
      "activations/layer5_attention_weight_max": 53.5526237487793,
      "activations/layer5_attention_weight_min": -60.099239349365234,
      "activations/layer6_attention_weight_max": 48.09565353393555,
      "activations/layer6_attention_weight_min": -49.248695373535156,
      "activations/layer7_attention_weight_max": 109.72742462158203,
      "activations/layer7_attention_weight_min": -104.17674255371094,
      "activations/layer8_attention_weight_max": 47.11806869506836,
      "activations/layer8_attention_weight_min": -50.80267333984375,
      "activations/layer9_attention_weight_max": 34.77724838256836,
      "activations/layer9_attention_weight_min": -40.774356842041016,
      "epoch": 21.07,
      "learning_rate": 1.4254924242424242e-05,
      "loss": 2.712,
      "step": 362650
    },
    {
      "activations/layer0_attention_weight_max": 16.425220489501953,
      "activations/layer0_attention_weight_min": -13.177292823791504,
      "activations/layer10_attention_weight_max": 35.17908477783203,
      "activations/layer10_attention_weight_min": -36.663665771484375,
      "activations/layer11_attention_weight_max": 34.47254943847656,
      "activations/layer11_attention_weight_min": -36.502342224121094,
      "activations/layer12_attention_weight_max": 25.752857208251953,
      "activations/layer12_attention_weight_min": -26.188108444213867,
      "activations/layer13_attention_weight_max": 38.49739456176758,
      "activations/layer13_attention_weight_min": -35.75236892700195,
      "activations/layer14_attention_weight_max": 43.47013854980469,
      "activations/layer14_attention_weight_min": -37.17621994018555,
      "activations/layer15_attention_weight_max": 37.89306640625,
      "activations/layer15_attention_weight_min": -36.4034423828125,
      "activations/layer16_attention_weight_max": 27.614768981933594,
      "activations/layer16_attention_weight_min": -27.69339942932129,
      "activations/layer17_attention_weight_max": 28.368562698364258,
      "activations/layer17_attention_weight_min": -24.625789642333984,
      "activations/layer18_attention_weight_max": 32.908695220947266,
      "activations/layer18_attention_weight_min": -24.645435333251953,
      "activations/layer19_attention_weight_max": 37.40936279296875,
      "activations/layer19_attention_weight_min": -30.644851684570312,
      "activations/layer1_attention_weight_max": 15.82114028930664,
      "activations/layer1_attention_weight_min": -15.19121265411377,
      "activations/layer20_attention_weight_max": 29.670812606811523,
      "activations/layer20_attention_weight_min": -26.076841354370117,
      "activations/layer21_attention_weight_max": 30.754440307617188,
      "activations/layer21_attention_weight_min": -23.79695701599121,
      "activations/layer22_attention_weight_max": 42.57645034790039,
      "activations/layer22_attention_weight_min": -30.137422561645508,
      "activations/layer23_attention_weight_max": 33.432273864746094,
      "activations/layer23_attention_weight_min": -27.01342010498047,
      "activations/layer2_attention_weight_max": 32.86398696899414,
      "activations/layer2_attention_weight_min": -31.891263961791992,
      "activations/layer3_attention_weight_max": 101.17134857177734,
      "activations/layer3_attention_weight_min": -103.07638549804688,
      "activations/layer4_attention_weight_max": 103.3550033569336,
      "activations/layer4_attention_weight_min": -108.29840087890625,
      "activations/layer5_attention_weight_max": 49.09120178222656,
      "activations/layer5_attention_weight_min": -63.3270263671875,
      "activations/layer6_attention_weight_max": 42.90464782714844,
      "activations/layer6_attention_weight_min": -48.591880798339844,
      "activations/layer7_attention_weight_max": 93.07969665527344,
      "activations/layer7_attention_weight_min": -97.98320007324219,
      "activations/layer8_attention_weight_max": 41.96913528442383,
      "activations/layer8_attention_weight_min": -44.54903793334961,
      "activations/layer9_attention_weight_max": 32.45161819458008,
      "activations/layer9_attention_weight_min": -36.94654083251953,
      "epoch": 21.07,
      "learning_rate": 1.4235984848484848e-05,
      "loss": 2.7318,
      "step": 362700
    },
    {
      "activations/layer0_attention_weight_max": 17.157371520996094,
      "activations/layer0_attention_weight_min": -13.908491134643555,
      "activations/layer10_attention_weight_max": 35.01555633544922,
      "activations/layer10_attention_weight_min": -36.22069549560547,
      "activations/layer11_attention_weight_max": 33.37273406982422,
      "activations/layer11_attention_weight_min": -35.22340774536133,
      "activations/layer12_attention_weight_max": 27.64307403564453,
      "activations/layer12_attention_weight_min": -26.96040153503418,
      "activations/layer13_attention_weight_max": 38.60628128051758,
      "activations/layer13_attention_weight_min": -38.63328552246094,
      "activations/layer14_attention_weight_max": 40.087066650390625,
      "activations/layer14_attention_weight_min": -37.6072883605957,
      "activations/layer15_attention_weight_max": 36.63700485229492,
      "activations/layer15_attention_weight_min": -36.68055725097656,
      "activations/layer16_attention_weight_max": 29.205894470214844,
      "activations/layer16_attention_weight_min": -28.478878021240234,
      "activations/layer17_attention_weight_max": 28.77121353149414,
      "activations/layer17_attention_weight_min": -28.60477066040039,
      "activations/layer18_attention_weight_max": 31.83953285217285,
      "activations/layer18_attention_weight_min": -26.874675750732422,
      "activations/layer19_attention_weight_max": 35.765350341796875,
      "activations/layer19_attention_weight_min": -34.421836853027344,
      "activations/layer1_attention_weight_max": 15.862364768981934,
      "activations/layer1_attention_weight_min": -14.707344055175781,
      "activations/layer20_attention_weight_max": 28.499927520751953,
      "activations/layer20_attention_weight_min": -27.047910690307617,
      "activations/layer21_attention_weight_max": 28.11904525756836,
      "activations/layer21_attention_weight_min": -27.02809715270996,
      "activations/layer22_attention_weight_max": 39.943363189697266,
      "activations/layer22_attention_weight_min": -33.315452575683594,
      "activations/layer23_attention_weight_max": 29.972087860107422,
      "activations/layer23_attention_weight_min": -27.034523010253906,
      "activations/layer2_attention_weight_max": 31.99714469909668,
      "activations/layer2_attention_weight_min": -32.313297271728516,
      "activations/layer3_attention_weight_max": 103.22064971923828,
      "activations/layer3_attention_weight_min": -101.21724700927734,
      "activations/layer4_attention_weight_max": 105.72175598144531,
      "activations/layer4_attention_weight_min": -112.063232421875,
      "activations/layer5_attention_weight_max": 51.95623016357422,
      "activations/layer5_attention_weight_min": -62.27781295776367,
      "activations/layer6_attention_weight_max": 45.23830032348633,
      "activations/layer6_attention_weight_min": -49.150733947753906,
      "activations/layer7_attention_weight_max": 92.41378784179688,
      "activations/layer7_attention_weight_min": -97.4862289428711,
      "activations/layer8_attention_weight_max": 41.1177978515625,
      "activations/layer8_attention_weight_min": -45.4981575012207,
      "activations/layer9_attention_weight_max": 31.925764083862305,
      "activations/layer9_attention_weight_min": -35.69321823120117,
      "epoch": 21.08,
      "learning_rate": 1.4217424242424242e-05,
      "loss": 2.7081,
      "step": 362750
    },
    {
      "activations/layer0_attention_weight_max": 15.821880340576172,
      "activations/layer0_attention_weight_min": -14.585801124572754,
      "activations/layer10_attention_weight_max": 34.39303970336914,
      "activations/layer10_attention_weight_min": -33.981964111328125,
      "activations/layer11_attention_weight_max": 33.71662521362305,
      "activations/layer11_attention_weight_min": -35.075172424316406,
      "activations/layer12_attention_weight_max": 26.186063766479492,
      "activations/layer12_attention_weight_min": -27.764657974243164,
      "activations/layer13_attention_weight_max": 38.50887680053711,
      "activations/layer13_attention_weight_min": -37.130043029785156,
      "activations/layer14_attention_weight_max": 44.412841796875,
      "activations/layer14_attention_weight_min": -41.597084045410156,
      "activations/layer15_attention_weight_max": 42.02434539794922,
      "activations/layer15_attention_weight_min": -38.53996658325195,
      "activations/layer16_attention_weight_max": 29.21634864807129,
      "activations/layer16_attention_weight_min": -28.526901245117188,
      "activations/layer17_attention_weight_max": 31.616024017333984,
      "activations/layer17_attention_weight_min": -26.996240615844727,
      "activations/layer18_attention_weight_max": 32.875335693359375,
      "activations/layer18_attention_weight_min": -25.837215423583984,
      "activations/layer19_attention_weight_max": 35.64535140991211,
      "activations/layer19_attention_weight_min": -32.36414337158203,
      "activations/layer1_attention_weight_max": 15.39797592163086,
      "activations/layer1_attention_weight_min": -14.541749954223633,
      "activations/layer20_attention_weight_max": 30.21807861328125,
      "activations/layer20_attention_weight_min": -25.351118087768555,
      "activations/layer21_attention_weight_max": 29.944042205810547,
      "activations/layer21_attention_weight_min": -25.026592254638672,
      "activations/layer22_attention_weight_max": 44.71831512451172,
      "activations/layer22_attention_weight_min": -31.290199279785156,
      "activations/layer23_attention_weight_max": 35.22795104980469,
      "activations/layer23_attention_weight_min": -26.13400650024414,
      "activations/layer2_attention_weight_max": 32.339210510253906,
      "activations/layer2_attention_weight_min": -31.548736572265625,
      "activations/layer3_attention_weight_max": 99.75788879394531,
      "activations/layer3_attention_weight_min": -102.1866683959961,
      "activations/layer4_attention_weight_max": 104.2440185546875,
      "activations/layer4_attention_weight_min": -107.99288177490234,
      "activations/layer5_attention_weight_max": 49.544898986816406,
      "activations/layer5_attention_weight_min": -63.1087760925293,
      "activations/layer6_attention_weight_max": 44.20138931274414,
      "activations/layer6_attention_weight_min": -46.94002151489258,
      "activations/layer7_attention_weight_max": 92.4027099609375,
      "activations/layer7_attention_weight_min": -97.03173065185547,
      "activations/layer8_attention_weight_max": 40.294673919677734,
      "activations/layer8_attention_weight_min": -43.861141204833984,
      "activations/layer9_attention_weight_max": 31.996631622314453,
      "activations/layer9_attention_weight_min": -33.241539001464844,
      "epoch": 21.08,
      "learning_rate": 1.4198484848484847e-05,
      "loss": 2.6942,
      "step": 362800
    },
    {
      "activations/layer0_attention_weight_max": 16.32257080078125,
      "activations/layer0_attention_weight_min": -13.309749603271484,
      "activations/layer10_attention_weight_max": 35.00258255004883,
      "activations/layer10_attention_weight_min": -37.39261245727539,
      "activations/layer11_attention_weight_max": 35.103424072265625,
      "activations/layer11_attention_weight_min": -36.123207092285156,
      "activations/layer12_attention_weight_max": 27.045454025268555,
      "activations/layer12_attention_weight_min": -28.131393432617188,
      "activations/layer13_attention_weight_max": 41.58733367919922,
      "activations/layer13_attention_weight_min": -37.266998291015625,
      "activations/layer14_attention_weight_max": 44.101165771484375,
      "activations/layer14_attention_weight_min": -39.92924118041992,
      "activations/layer15_attention_weight_max": 41.03925704956055,
      "activations/layer15_attention_weight_min": -38.00981140136719,
      "activations/layer16_attention_weight_max": 29.005735397338867,
      "activations/layer16_attention_weight_min": -29.62181854248047,
      "activations/layer17_attention_weight_max": 30.49403953552246,
      "activations/layer17_attention_weight_min": -26.170747756958008,
      "activations/layer18_attention_weight_max": 32.67510223388672,
      "activations/layer18_attention_weight_min": -27.703041076660156,
      "activations/layer19_attention_weight_max": 37.53362274169922,
      "activations/layer19_attention_weight_min": -31.966251373291016,
      "activations/layer1_attention_weight_max": 15.693346977233887,
      "activations/layer1_attention_weight_min": -15.553377151489258,
      "activations/layer20_attention_weight_max": 29.03215789794922,
      "activations/layer20_attention_weight_min": -25.527416229248047,
      "activations/layer21_attention_weight_max": 28.105119705200195,
      "activations/layer21_attention_weight_min": -24.703611373901367,
      "activations/layer22_attention_weight_max": 42.12373352050781,
      "activations/layer22_attention_weight_min": -31.05364227294922,
      "activations/layer23_attention_weight_max": 31.833885192871094,
      "activations/layer23_attention_weight_min": -27.469440460205078,
      "activations/layer2_attention_weight_max": 35.57383346557617,
      "activations/layer2_attention_weight_min": -34.05064010620117,
      "activations/layer3_attention_weight_max": 104.29015350341797,
      "activations/layer3_attention_weight_min": -104.35383605957031,
      "activations/layer4_attention_weight_max": 106.44962310791016,
      "activations/layer4_attention_weight_min": -110.67591857910156,
      "activations/layer5_attention_weight_max": 51.280086517333984,
      "activations/layer5_attention_weight_min": -61.963623046875,
      "activations/layer6_attention_weight_max": 45.49148178100586,
      "activations/layer6_attention_weight_min": -49.88167190551758,
      "activations/layer7_attention_weight_max": 98.89131927490234,
      "activations/layer7_attention_weight_min": -102.64948272705078,
      "activations/layer8_attention_weight_max": 43.55264663696289,
      "activations/layer8_attention_weight_min": -48.7144775390625,
      "activations/layer9_attention_weight_max": 34.57169723510742,
      "activations/layer9_attention_weight_min": -40.09699630737305,
      "epoch": 21.08,
      "learning_rate": 1.4179545454545454e-05,
      "loss": 2.6936,
      "step": 362850
    },
    {
      "activations/layer0_attention_weight_max": 16.046419143676758,
      "activations/layer0_attention_weight_min": -14.024608612060547,
      "activations/layer10_attention_weight_max": 37.03325653076172,
      "activations/layer10_attention_weight_min": -37.86117935180664,
      "activations/layer11_attention_weight_max": 35.97181701660156,
      "activations/layer11_attention_weight_min": -36.95750045776367,
      "activations/layer12_attention_weight_max": 28.04193687438965,
      "activations/layer12_attention_weight_min": -27.209997177124023,
      "activations/layer13_attention_weight_max": 40.98716354370117,
      "activations/layer13_attention_weight_min": -36.40970993041992,
      "activations/layer14_attention_weight_max": 42.67219924926758,
      "activations/layer14_attention_weight_min": -41.10041046142578,
      "activations/layer15_attention_weight_max": 44.53956604003906,
      "activations/layer15_attention_weight_min": -35.875938415527344,
      "activations/layer16_attention_weight_max": 32.69814682006836,
      "activations/layer16_attention_weight_min": -28.909931182861328,
      "activations/layer17_attention_weight_max": 30.18708038330078,
      "activations/layer17_attention_weight_min": -27.688556671142578,
      "activations/layer18_attention_weight_max": 32.498111724853516,
      "activations/layer18_attention_weight_min": -25.44985008239746,
      "activations/layer19_attention_weight_max": 35.38085174560547,
      "activations/layer19_attention_weight_min": -31.956235885620117,
      "activations/layer1_attention_weight_max": 15.079290390014648,
      "activations/layer1_attention_weight_min": -14.831356048583984,
      "activations/layer20_attention_weight_max": 32.092124938964844,
      "activations/layer20_attention_weight_min": -26.11065673828125,
      "activations/layer21_attention_weight_max": 29.990964889526367,
      "activations/layer21_attention_weight_min": -25.122262954711914,
      "activations/layer22_attention_weight_max": 43.99404525756836,
      "activations/layer22_attention_weight_min": -30.250885009765625,
      "activations/layer23_attention_weight_max": 31.348922729492188,
      "activations/layer23_attention_weight_min": -24.787174224853516,
      "activations/layer2_attention_weight_max": 33.827171325683594,
      "activations/layer2_attention_weight_min": -33.08952331542969,
      "activations/layer3_attention_weight_max": 103.11371612548828,
      "activations/layer3_attention_weight_min": -106.21356201171875,
      "activations/layer4_attention_weight_max": 106.61225891113281,
      "activations/layer4_attention_weight_min": -113.95635986328125,
      "activations/layer5_attention_weight_max": 50.72509765625,
      "activations/layer5_attention_weight_min": -64.53987884521484,
      "activations/layer6_attention_weight_max": 46.3084602355957,
      "activations/layer6_attention_weight_min": -48.486019134521484,
      "activations/layer7_attention_weight_max": 96.6700668334961,
      "activations/layer7_attention_weight_min": -104.44390869140625,
      "activations/layer8_attention_weight_max": 45.709102630615234,
      "activations/layer8_attention_weight_min": -49.28938674926758,
      "activations/layer9_attention_weight_max": 33.28154754638672,
      "activations/layer9_attention_weight_min": -36.21010971069336,
      "epoch": 21.09,
      "learning_rate": 1.416060606060606e-05,
      "loss": 2.7163,
      "step": 362900
    },
    {
      "activations/layer0_attention_weight_max": 16.905975341796875,
      "activations/layer0_attention_weight_min": -15.306328773498535,
      "activations/layer10_attention_weight_max": 31.681875228881836,
      "activations/layer10_attention_weight_min": -33.220550537109375,
      "activations/layer11_attention_weight_max": 33.534950256347656,
      "activations/layer11_attention_weight_min": -32.81433868408203,
      "activations/layer12_attention_weight_max": 28.393753051757812,
      "activations/layer12_attention_weight_min": -26.913599014282227,
      "activations/layer13_attention_weight_max": 39.183467864990234,
      "activations/layer13_attention_weight_min": -35.391357421875,
      "activations/layer14_attention_weight_max": 43.32080078125,
      "activations/layer14_attention_weight_min": -37.892250061035156,
      "activations/layer15_attention_weight_max": 39.50507354736328,
      "activations/layer15_attention_weight_min": -35.291908264160156,
      "activations/layer16_attention_weight_max": 26.387454986572266,
      "activations/layer16_attention_weight_min": -26.643157958984375,
      "activations/layer17_attention_weight_max": 29.86386489868164,
      "activations/layer17_attention_weight_min": -24.69736099243164,
      "activations/layer18_attention_weight_max": 32.93510818481445,
      "activations/layer18_attention_weight_min": -26.567745208740234,
      "activations/layer19_attention_weight_max": 31.376550674438477,
      "activations/layer19_attention_weight_min": -31.479581832885742,
      "activations/layer1_attention_weight_max": 15.71781063079834,
      "activations/layer1_attention_weight_min": -14.111847877502441,
      "activations/layer20_attention_weight_max": 27.102432250976562,
      "activations/layer20_attention_weight_min": -24.734567642211914,
      "activations/layer21_attention_weight_max": 27.32836151123047,
      "activations/layer21_attention_weight_min": -25.321081161499023,
      "activations/layer22_attention_weight_max": 41.31268310546875,
      "activations/layer22_attention_weight_min": -29.383785247802734,
      "activations/layer23_attention_weight_max": 31.034042358398438,
      "activations/layer23_attention_weight_min": -25.042221069335938,
      "activations/layer2_attention_weight_max": 33.454254150390625,
      "activations/layer2_attention_weight_min": -31.999847412109375,
      "activations/layer3_attention_weight_max": 99.61787414550781,
      "activations/layer3_attention_weight_min": -100.1552734375,
      "activations/layer4_attention_weight_max": 106.05818939208984,
      "activations/layer4_attention_weight_min": -109.47977447509766,
      "activations/layer5_attention_weight_max": 50.95227813720703,
      "activations/layer5_attention_weight_min": -63.944740295410156,
      "activations/layer6_attention_weight_max": 43.486576080322266,
      "activations/layer6_attention_weight_min": -49.55392837524414,
      "activations/layer7_attention_weight_max": 95.68135833740234,
      "activations/layer7_attention_weight_min": -96.50366973876953,
      "activations/layer8_attention_weight_max": 40.533512115478516,
      "activations/layer8_attention_weight_min": -42.799137115478516,
      "activations/layer9_attention_weight_max": 31.5452823638916,
      "activations/layer9_attention_weight_min": -33.835609436035156,
      "epoch": 21.09,
      "learning_rate": 1.4141666666666665e-05,
      "loss": 2.7083,
      "step": 362950
    },
    {
      "activations/layer0_attention_weight_max": 16.822141647338867,
      "activations/layer0_attention_weight_min": -13.140937805175781,
      "activations/layer10_attention_weight_max": 37.172733306884766,
      "activations/layer10_attention_weight_min": -37.67021560668945,
      "activations/layer11_attention_weight_max": 36.636783599853516,
      "activations/layer11_attention_weight_min": -38.387691497802734,
      "activations/layer12_attention_weight_max": 30.268823623657227,
      "activations/layer12_attention_weight_min": -28.052160263061523,
      "activations/layer13_attention_weight_max": 45.716739654541016,
      "activations/layer13_attention_weight_min": -36.74439239501953,
      "activations/layer14_attention_weight_max": 45.68238067626953,
      "activations/layer14_attention_weight_min": -39.525447845458984,
      "activations/layer15_attention_weight_max": 46.213348388671875,
      "activations/layer15_attention_weight_min": -36.015342712402344,
      "activations/layer16_attention_weight_max": 34.761714935302734,
      "activations/layer16_attention_weight_min": -27.19088363647461,
      "activations/layer17_attention_weight_max": 30.297983169555664,
      "activations/layer17_attention_weight_min": -26.048215866088867,
      "activations/layer18_attention_weight_max": 34.68537521362305,
      "activations/layer18_attention_weight_min": -24.40350341796875,
      "activations/layer19_attention_weight_max": 40.807716369628906,
      "activations/layer19_attention_weight_min": -30.477827072143555,
      "activations/layer1_attention_weight_max": 15.648422241210938,
      "activations/layer1_attention_weight_min": -15.172054290771484,
      "activations/layer20_attention_weight_max": 33.07408905029297,
      "activations/layer20_attention_weight_min": -24.03774070739746,
      "activations/layer21_attention_weight_max": 30.423391342163086,
      "activations/layer21_attention_weight_min": -25.664138793945312,
      "activations/layer22_attention_weight_max": 45.87457275390625,
      "activations/layer22_attention_weight_min": -31.222484588623047,
      "activations/layer23_attention_weight_max": 39.71235656738281,
      "activations/layer23_attention_weight_min": -25.862516403198242,
      "activations/layer2_attention_weight_max": 34.131622314453125,
      "activations/layer2_attention_weight_min": -34.599735260009766,
      "activations/layer3_attention_weight_max": 110.20611572265625,
      "activations/layer3_attention_weight_min": -109.08700561523438,
      "activations/layer4_attention_weight_max": 114.40999603271484,
      "activations/layer4_attention_weight_min": -114.11860656738281,
      "activations/layer5_attention_weight_max": 55.157135009765625,
      "activations/layer5_attention_weight_min": -64.68759155273438,
      "activations/layer6_attention_weight_max": 47.96320724487305,
      "activations/layer6_attention_weight_min": -50.369998931884766,
      "activations/layer7_attention_weight_max": 104.16935729980469,
      "activations/layer7_attention_weight_min": -102.38013458251953,
      "activations/layer8_attention_weight_max": 42.395328521728516,
      "activations/layer8_attention_weight_min": -47.91460037231445,
      "activations/layer9_attention_weight_max": 34.34903335571289,
      "activations/layer9_attention_weight_min": -36.71251678466797,
      "epoch": 21.09,
      "learning_rate": 1.4122727272727272e-05,
      "loss": 2.7122,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4615,
      "eval_samples_per_second": 507.475,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4615,
      "eval_openwebtext_samples_per_second": 507.475,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9649,
      "eval_wikitext_samples_per_second": 232.072,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_lambada_loss": 2.3359375,
      "eval_lambada_ppl": 10.33914833878563,
      "eval_lambada_runtime": 9.5686,
      "eval_lambada_samples_per_second": 508.853,
      "step": 363000
    },
    {
      "activations/layer0_attention_weight_max": 16.5767879486084,
      "activations/layer0_attention_weight_min": -12.862021446228027,
      "activations/layer10_attention_weight_max": 43.16438674926758,
      "activations/layer10_attention_weight_min": -37.83400344848633,
      "activations/layer11_attention_weight_max": 41.93992614746094,
      "activations/layer11_attention_weight_min": -35.887664794921875,
      "activations/layer12_attention_weight_max": 25.97662925720215,
      "activations/layer12_attention_weight_min": -27.911624908447266,
      "activations/layer13_attention_weight_max": 37.60148239135742,
      "activations/layer13_attention_weight_min": -36.38285827636719,
      "activations/layer14_attention_weight_max": 43.82387924194336,
      "activations/layer14_attention_weight_min": -38.095123291015625,
      "activations/layer15_attention_weight_max": 42.313358306884766,
      "activations/layer15_attention_weight_min": -37.8017578125,
      "activations/layer16_attention_weight_max": 32.78609085083008,
      "activations/layer16_attention_weight_min": -29.375314712524414,
      "activations/layer17_attention_weight_max": 41.33820343017578,
      "activations/layer17_attention_weight_min": -27.800079345703125,
      "activations/layer18_attention_weight_max": 34.8789176940918,
      "activations/layer18_attention_weight_min": -27.864181518554688,
      "activations/layer19_attention_weight_max": 40.08296585083008,
      "activations/layer19_attention_weight_min": -31.642934799194336,
      "activations/layer1_attention_weight_max": 16.168418884277344,
      "activations/layer1_attention_weight_min": -14.583211898803711,
      "activations/layer20_attention_weight_max": 33.096370697021484,
      "activations/layer20_attention_weight_min": -25.099708557128906,
      "activations/layer21_attention_weight_max": 31.789037704467773,
      "activations/layer21_attention_weight_min": -23.66259765625,
      "activations/layer22_attention_weight_max": 48.48738098144531,
      "activations/layer22_attention_weight_min": -34.84933853149414,
      "activations/layer23_attention_weight_max": 34.75621795654297,
      "activations/layer23_attention_weight_min": -27.136754989624023,
      "activations/layer2_attention_weight_max": 34.62586212158203,
      "activations/layer2_attention_weight_min": -33.51482391357422,
      "activations/layer3_attention_weight_max": 105.67428588867188,
      "activations/layer3_attention_weight_min": -105.939208984375,
      "activations/layer4_attention_weight_max": 107.45938873291016,
      "activations/layer4_attention_weight_min": -110.74828338623047,
      "activations/layer5_attention_weight_max": 50.71568298339844,
      "activations/layer5_attention_weight_min": -62.8972282409668,
      "activations/layer6_attention_weight_max": 46.00413131713867,
      "activations/layer6_attention_weight_min": -50.39688491821289,
      "activations/layer7_attention_weight_max": 108.62909698486328,
      "activations/layer7_attention_weight_min": -107.35266876220703,
      "activations/layer8_attention_weight_max": 49.286834716796875,
      "activations/layer8_attention_weight_min": -48.545467376708984,
      "activations/layer9_attention_weight_max": 36.6723747253418,
      "activations/layer9_attention_weight_min": -36.63759994506836,
      "epoch": 21.1,
      "learning_rate": 1.4103787878787877e-05,
      "loss": 2.7112,
      "step": 363050
    },
    {
      "activations/layer0_attention_weight_max": 16.59159278869629,
      "activations/layer0_attention_weight_min": -14.219408988952637,
      "activations/layer10_attention_weight_max": 35.490604400634766,
      "activations/layer10_attention_weight_min": -40.18220901489258,
      "activations/layer11_attention_weight_max": 35.76819610595703,
      "activations/layer11_attention_weight_min": -36.67379379272461,
      "activations/layer12_attention_weight_max": 30.148042678833008,
      "activations/layer12_attention_weight_min": -30.055103302001953,
      "activations/layer13_attention_weight_max": 44.28705978393555,
      "activations/layer13_attention_weight_min": -42.59675216674805,
      "activations/layer14_attention_weight_max": 45.930870056152344,
      "activations/layer14_attention_weight_min": -46.01431655883789,
      "activations/layer15_attention_weight_max": 41.87828826904297,
      "activations/layer15_attention_weight_min": -38.321739196777344,
      "activations/layer16_attention_weight_max": 30.976274490356445,
      "activations/layer16_attention_weight_min": -28.85658073425293,
      "activations/layer17_attention_weight_max": 30.503984451293945,
      "activations/layer17_attention_weight_min": -25.578046798706055,
      "activations/layer18_attention_weight_max": 36.22243881225586,
      "activations/layer18_attention_weight_min": -25.334632873535156,
      "activations/layer19_attention_weight_max": 36.29497528076172,
      "activations/layer19_attention_weight_min": -33.068634033203125,
      "activations/layer1_attention_weight_max": 15.789725303649902,
      "activations/layer1_attention_weight_min": -15.06149959564209,
      "activations/layer20_attention_weight_max": 30.72429847717285,
      "activations/layer20_attention_weight_min": -26.215726852416992,
      "activations/layer21_attention_weight_max": 28.588157653808594,
      "activations/layer21_attention_weight_min": -24.904338836669922,
      "activations/layer22_attention_weight_max": 44.10612106323242,
      "activations/layer22_attention_weight_min": -34.33369445800781,
      "activations/layer23_attention_weight_max": 33.91974639892578,
      "activations/layer23_attention_weight_min": -27.79232406616211,
      "activations/layer2_attention_weight_max": 34.6588020324707,
      "activations/layer2_attention_weight_min": -32.94090270996094,
      "activations/layer3_attention_weight_max": 101.88369750976562,
      "activations/layer3_attention_weight_min": -105.29523468017578,
      "activations/layer4_attention_weight_max": 105.41377258300781,
      "activations/layer4_attention_weight_min": -113.3331298828125,
      "activations/layer5_attention_weight_max": 50.81487274169922,
      "activations/layer5_attention_weight_min": -61.672080993652344,
      "activations/layer6_attention_weight_max": 46.40923309326172,
      "activations/layer6_attention_weight_min": -50.36280822753906,
      "activations/layer7_attention_weight_max": 94.99300384521484,
      "activations/layer7_attention_weight_min": -102.65771484375,
      "activations/layer8_attention_weight_max": 42.579254150390625,
      "activations/layer8_attention_weight_min": -49.845489501953125,
      "activations/layer9_attention_weight_max": 32.04487228393555,
      "activations/layer9_attention_weight_min": -36.64155197143555,
      "epoch": 21.1,
      "learning_rate": 1.4084848484848483e-05,
      "loss": 2.6941,
      "step": 363100
    },
    {
      "activations/layer0_attention_weight_max": 15.966901779174805,
      "activations/layer0_attention_weight_min": -12.803531646728516,
      "activations/layer10_attention_weight_max": 33.88724136352539,
      "activations/layer10_attention_weight_min": -34.27348327636719,
      "activations/layer11_attention_weight_max": 33.54710388183594,
      "activations/layer11_attention_weight_min": -34.69294738769531,
      "activations/layer12_attention_weight_max": 26.875186920166016,
      "activations/layer12_attention_weight_min": -26.602476119995117,
      "activations/layer13_attention_weight_max": 40.360225677490234,
      "activations/layer13_attention_weight_min": -36.17527770996094,
      "activations/layer14_attention_weight_max": 42.764076232910156,
      "activations/layer14_attention_weight_min": -38.11845397949219,
      "activations/layer15_attention_weight_max": 41.965057373046875,
      "activations/layer15_attention_weight_min": -36.38060760498047,
      "activations/layer16_attention_weight_max": 30.071325302124023,
      "activations/layer16_attention_weight_min": -29.85038948059082,
      "activations/layer17_attention_weight_max": 30.32754898071289,
      "activations/layer17_attention_weight_min": -26.582653045654297,
      "activations/layer18_attention_weight_max": 33.264923095703125,
      "activations/layer18_attention_weight_min": -25.231369018554688,
      "activations/layer19_attention_weight_max": 35.50326156616211,
      "activations/layer19_attention_weight_min": -31.88547706604004,
      "activations/layer1_attention_weight_max": 16.13054656982422,
      "activations/layer1_attention_weight_min": -15.418481826782227,
      "activations/layer20_attention_weight_max": 30.074207305908203,
      "activations/layer20_attention_weight_min": -25.8938045501709,
      "activations/layer21_attention_weight_max": 33.01185607910156,
      "activations/layer21_attention_weight_min": -25.2978572845459,
      "activations/layer22_attention_weight_max": 40.56663513183594,
      "activations/layer22_attention_weight_min": -31.117475509643555,
      "activations/layer23_attention_weight_max": 33.244110107421875,
      "activations/layer23_attention_weight_min": -25.768386840820312,
      "activations/layer2_attention_weight_max": 33.94889450073242,
      "activations/layer2_attention_weight_min": -31.109588623046875,
      "activations/layer3_attention_weight_max": 98.82463073730469,
      "activations/layer3_attention_weight_min": -97.47306823730469,
      "activations/layer4_attention_weight_max": 106.91595458984375,
      "activations/layer4_attention_weight_min": -106.99102020263672,
      "activations/layer5_attention_weight_max": 49.284271240234375,
      "activations/layer5_attention_weight_min": -60.967647552490234,
      "activations/layer6_attention_weight_max": 44.90091323852539,
      "activations/layer6_attention_weight_min": -50.976924896240234,
      "activations/layer7_attention_weight_max": 97.61201477050781,
      "activations/layer7_attention_weight_min": -101.66309356689453,
      "activations/layer8_attention_weight_max": 41.458160400390625,
      "activations/layer8_attention_weight_min": -43.78837203979492,
      "activations/layer9_attention_weight_max": 31.58806800842285,
      "activations/layer9_attention_weight_min": -34.01638412475586,
      "epoch": 21.1,
      "learning_rate": 1.406590909090909e-05,
      "loss": 2.6971,
      "step": 363150
    },
    {
      "activations/layer0_attention_weight_max": 17.988662719726562,
      "activations/layer0_attention_weight_min": -13.045254707336426,
      "activations/layer10_attention_weight_max": 35.71953201293945,
      "activations/layer10_attention_weight_min": -37.56892776489258,
      "activations/layer11_attention_weight_max": 37.26319885253906,
      "activations/layer11_attention_weight_min": -36.32879638671875,
      "activations/layer12_attention_weight_max": 27.391361236572266,
      "activations/layer12_attention_weight_min": -27.967222213745117,
      "activations/layer13_attention_weight_max": 40.34081268310547,
      "activations/layer13_attention_weight_min": -38.02894973754883,
      "activations/layer14_attention_weight_max": 43.54381561279297,
      "activations/layer14_attention_weight_min": -40.8277473449707,
      "activations/layer15_attention_weight_max": 39.55352020263672,
      "activations/layer15_attention_weight_min": -38.66231155395508,
      "activations/layer16_attention_weight_max": 31.814054489135742,
      "activations/layer16_attention_weight_min": -28.647628784179688,
      "activations/layer17_attention_weight_max": 29.66288948059082,
      "activations/layer17_attention_weight_min": -26.07842254638672,
      "activations/layer18_attention_weight_max": 31.92293930053711,
      "activations/layer18_attention_weight_min": -27.54505729675293,
      "activations/layer19_attention_weight_max": 34.80056381225586,
      "activations/layer19_attention_weight_min": -32.11457061767578,
      "activations/layer1_attention_weight_max": 15.65859317779541,
      "activations/layer1_attention_weight_min": -14.573623657226562,
      "activations/layer20_attention_weight_max": 28.078655242919922,
      "activations/layer20_attention_weight_min": -24.64342498779297,
      "activations/layer21_attention_weight_max": 26.317916870117188,
      "activations/layer21_attention_weight_min": -25.09830665588379,
      "activations/layer22_attention_weight_max": 41.2021598815918,
      "activations/layer22_attention_weight_min": -31.970272064208984,
      "activations/layer23_attention_weight_max": 34.968849182128906,
      "activations/layer23_attention_weight_min": -26.140140533447266,
      "activations/layer2_attention_weight_max": 33.07660675048828,
      "activations/layer2_attention_weight_min": -32.668983459472656,
      "activations/layer3_attention_weight_max": 102.10057067871094,
      "activations/layer3_attention_weight_min": -103.63945007324219,
      "activations/layer4_attention_weight_max": 107.46114349365234,
      "activations/layer4_attention_weight_min": -108.62506103515625,
      "activations/layer5_attention_weight_max": 52.470619201660156,
      "activations/layer5_attention_weight_min": -63.573158264160156,
      "activations/layer6_attention_weight_max": 47.033687591552734,
      "activations/layer6_attention_weight_min": -49.18782424926758,
      "activations/layer7_attention_weight_max": 97.73743438720703,
      "activations/layer7_attention_weight_min": -108.45242309570312,
      "activations/layer8_attention_weight_max": 44.09515380859375,
      "activations/layer8_attention_weight_min": -48.14958572387695,
      "activations/layer9_attention_weight_max": 33.53420639038086,
      "activations/layer9_attention_weight_min": -36.75590896606445,
      "epoch": 21.1,
      "learning_rate": 1.4047348484848484e-05,
      "loss": 2.7142,
      "step": 363200
    },
    {
      "activations/layer0_attention_weight_max": 17.28742027282715,
      "activations/layer0_attention_weight_min": -15.556527137756348,
      "activations/layer10_attention_weight_max": 32.67946243286133,
      "activations/layer10_attention_weight_min": -35.36357498168945,
      "activations/layer11_attention_weight_max": 31.929738998413086,
      "activations/layer11_attention_weight_min": -34.96803283691406,
      "activations/layer12_attention_weight_max": 26.24883460998535,
      "activations/layer12_attention_weight_min": -27.46672248840332,
      "activations/layer13_attention_weight_max": 38.87599563598633,
      "activations/layer13_attention_weight_min": -35.702877044677734,
      "activations/layer14_attention_weight_max": 42.714263916015625,
      "activations/layer14_attention_weight_min": -41.061092376708984,
      "activations/layer15_attention_weight_max": 37.13894271850586,
      "activations/layer15_attention_weight_min": -39.59154510498047,
      "activations/layer16_attention_weight_max": 28.208236694335938,
      "activations/layer16_attention_weight_min": -28.514631271362305,
      "activations/layer17_attention_weight_max": 29.71401023864746,
      "activations/layer17_attention_weight_min": -26.74055290222168,
      "activations/layer18_attention_weight_max": 33.624183654785156,
      "activations/layer18_attention_weight_min": -28.477638244628906,
      "activations/layer19_attention_weight_max": 34.45585632324219,
      "activations/layer19_attention_weight_min": -31.920705795288086,
      "activations/layer1_attention_weight_max": 16.437116622924805,
      "activations/layer1_attention_weight_min": -15.025328636169434,
      "activations/layer20_attention_weight_max": 28.285560607910156,
      "activations/layer20_attention_weight_min": -25.931333541870117,
      "activations/layer21_attention_weight_max": 28.118003845214844,
      "activations/layer21_attention_weight_min": -26.19114875793457,
      "activations/layer22_attention_weight_max": 41.320194244384766,
      "activations/layer22_attention_weight_min": -33.87360763549805,
      "activations/layer23_attention_weight_max": 33.789878845214844,
      "activations/layer23_attention_weight_min": -27.335731506347656,
      "activations/layer2_attention_weight_max": 33.358642578125,
      "activations/layer2_attention_weight_min": -33.38990783691406,
      "activations/layer3_attention_weight_max": 103.78306579589844,
      "activations/layer3_attention_weight_min": -103.34730529785156,
      "activations/layer4_attention_weight_max": 108.17391204833984,
      "activations/layer4_attention_weight_min": -112.8258285522461,
      "activations/layer5_attention_weight_max": 48.905723571777344,
      "activations/layer5_attention_weight_min": -63.02912139892578,
      "activations/layer6_attention_weight_max": 43.662296295166016,
      "activations/layer6_attention_weight_min": -49.37498092651367,
      "activations/layer7_attention_weight_max": 89.52991485595703,
      "activations/layer7_attention_weight_min": -95.38777160644531,
      "activations/layer8_attention_weight_max": 39.73875045776367,
      "activations/layer8_attention_weight_min": -44.83314895629883,
      "activations/layer9_attention_weight_max": 30.81711196899414,
      "activations/layer9_attention_weight_min": -34.31826400756836,
      "epoch": 21.11,
      "learning_rate": 1.4028409090909089e-05,
      "loss": 2.7154,
      "step": 363250
    },
    {
      "activations/layer0_attention_weight_max": 16.470027923583984,
      "activations/layer0_attention_weight_min": -13.93812084197998,
      "activations/layer10_attention_weight_max": 39.98040771484375,
      "activations/layer10_attention_weight_min": -40.86307907104492,
      "activations/layer11_attention_weight_max": 40.936363220214844,
      "activations/layer11_attention_weight_min": -38.656219482421875,
      "activations/layer12_attention_weight_max": 29.634483337402344,
      "activations/layer12_attention_weight_min": -29.342378616333008,
      "activations/layer13_attention_weight_max": 43.20889663696289,
      "activations/layer13_attention_weight_min": -39.64162826538086,
      "activations/layer14_attention_weight_max": 48.307498931884766,
      "activations/layer14_attention_weight_min": -42.3709716796875,
      "activations/layer15_attention_weight_max": 45.03480529785156,
      "activations/layer15_attention_weight_min": -39.513336181640625,
      "activations/layer16_attention_weight_max": 35.78293991088867,
      "activations/layer16_attention_weight_min": -30.209455490112305,
      "activations/layer17_attention_weight_max": 33.53544616699219,
      "activations/layer17_attention_weight_min": -27.521305084228516,
      "activations/layer18_attention_weight_max": 39.73928451538086,
      "activations/layer18_attention_weight_min": -27.95644760131836,
      "activations/layer19_attention_weight_max": 40.90856170654297,
      "activations/layer19_attention_weight_min": -34.31605529785156,
      "activations/layer1_attention_weight_max": 15.56527328491211,
      "activations/layer1_attention_weight_min": -15.611176490783691,
      "activations/layer20_attention_weight_max": 36.84874725341797,
      "activations/layer20_attention_weight_min": -27.557769775390625,
      "activations/layer21_attention_weight_max": 36.14680862426758,
      "activations/layer21_attention_weight_min": -26.421648025512695,
      "activations/layer22_attention_weight_max": 48.71613311767578,
      "activations/layer22_attention_weight_min": -33.660362243652344,
      "activations/layer23_attention_weight_max": 41.89433288574219,
      "activations/layer23_attention_weight_min": -25.5545711517334,
      "activations/layer2_attention_weight_max": 34.58013153076172,
      "activations/layer2_attention_weight_min": -33.86623001098633,
      "activations/layer3_attention_weight_max": 105.98706817626953,
      "activations/layer3_attention_weight_min": -111.56681823730469,
      "activations/layer4_attention_weight_max": 113.24669647216797,
      "activations/layer4_attention_weight_min": -112.17183685302734,
      "activations/layer5_attention_weight_max": 51.777488708496094,
      "activations/layer5_attention_weight_min": -61.63227462768555,
      "activations/layer6_attention_weight_max": 49.088138580322266,
      "activations/layer6_attention_weight_min": -51.38288879394531,
      "activations/layer7_attention_weight_max": 104.99238586425781,
      "activations/layer7_attention_weight_min": -111.57178497314453,
      "activations/layer8_attention_weight_max": 48.77845001220703,
      "activations/layer8_attention_weight_min": -53.68152618408203,
      "activations/layer9_attention_weight_max": 37.71183395385742,
      "activations/layer9_attention_weight_min": -39.9964485168457,
      "epoch": 21.11,
      "learning_rate": 1.4009469696969696e-05,
      "loss": 2.7192,
      "step": 363300
    },
    {
      "activations/layer0_attention_weight_max": 16.421735763549805,
      "activations/layer0_attention_weight_min": -14.146066665649414,
      "activations/layer10_attention_weight_max": 34.24892044067383,
      "activations/layer10_attention_weight_min": -35.25519561767578,
      "activations/layer11_attention_weight_max": 33.530677795410156,
      "activations/layer11_attention_weight_min": -34.74565124511719,
      "activations/layer12_attention_weight_max": 27.403642654418945,
      "activations/layer12_attention_weight_min": -29.27198028564453,
      "activations/layer13_attention_weight_max": 40.30177307128906,
      "activations/layer13_attention_weight_min": -39.57845687866211,
      "activations/layer14_attention_weight_max": 46.28888702392578,
      "activations/layer14_attention_weight_min": -42.59160614013672,
      "activations/layer15_attention_weight_max": 43.38917922973633,
      "activations/layer15_attention_weight_min": -38.42342758178711,
      "activations/layer16_attention_weight_max": 33.693878173828125,
      "activations/layer16_attention_weight_min": -30.818496704101562,
      "activations/layer17_attention_weight_max": 32.96546173095703,
      "activations/layer17_attention_weight_min": -27.443078994750977,
      "activations/layer18_attention_weight_max": 34.551753997802734,
      "activations/layer18_attention_weight_min": -28.356525421142578,
      "activations/layer19_attention_weight_max": 34.90037155151367,
      "activations/layer19_attention_weight_min": -33.48904800415039,
      "activations/layer1_attention_weight_max": 16.168071746826172,
      "activations/layer1_attention_weight_min": -15.487935066223145,
      "activations/layer20_attention_weight_max": 30.70644760131836,
      "activations/layer20_attention_weight_min": -29.089332580566406,
      "activations/layer21_attention_weight_max": 31.773691177368164,
      "activations/layer21_attention_weight_min": -25.287567138671875,
      "activations/layer22_attention_weight_max": 46.83265686035156,
      "activations/layer22_attention_weight_min": -32.40047073364258,
      "activations/layer23_attention_weight_max": 33.73715591430664,
      "activations/layer23_attention_weight_min": -28.38258934020996,
      "activations/layer2_attention_weight_max": 33.850181579589844,
      "activations/layer2_attention_weight_min": -33.710777282714844,
      "activations/layer3_attention_weight_max": 104.5283203125,
      "activations/layer3_attention_weight_min": -105.23272705078125,
      "activations/layer4_attention_weight_max": 112.4803237915039,
      "activations/layer4_attention_weight_min": -111.54569244384766,
      "activations/layer5_attention_weight_max": 49.8117790222168,
      "activations/layer5_attention_weight_min": -62.63127136230469,
      "activations/layer6_attention_weight_max": 47.9388313293457,
      "activations/layer6_attention_weight_min": -47.8364143371582,
      "activations/layer7_attention_weight_max": 98.29435729980469,
      "activations/layer7_attention_weight_min": -100.36866760253906,
      "activations/layer8_attention_weight_max": 44.347938537597656,
      "activations/layer8_attention_weight_min": -44.26560974121094,
      "activations/layer9_attention_weight_max": 33.980712890625,
      "activations/layer9_attention_weight_min": -33.40209197998047,
      "epoch": 21.11,
      "learning_rate": 1.39905303030303e-05,
      "loss": 2.7014,
      "step": 363350
    },
    {
      "activations/layer0_attention_weight_max": 17.64508819580078,
      "activations/layer0_attention_weight_min": -13.595791816711426,
      "activations/layer10_attention_weight_max": 34.96141815185547,
      "activations/layer10_attention_weight_min": -35.97169876098633,
      "activations/layer11_attention_weight_max": 37.05547332763672,
      "activations/layer11_attention_weight_min": -37.62092971801758,
      "activations/layer12_attention_weight_max": 31.849315643310547,
      "activations/layer12_attention_weight_min": -28.637575149536133,
      "activations/layer13_attention_weight_max": 48.828102111816406,
      "activations/layer13_attention_weight_min": -39.55406188964844,
      "activations/layer14_attention_weight_max": 50.746028900146484,
      "activations/layer14_attention_weight_min": -40.77617645263672,
      "activations/layer15_attention_weight_max": 51.58698654174805,
      "activations/layer15_attention_weight_min": -45.89105224609375,
      "activations/layer16_attention_weight_max": 36.49811935424805,
      "activations/layer16_attention_weight_min": -31.48940086364746,
      "activations/layer17_attention_weight_max": 38.026668548583984,
      "activations/layer17_attention_weight_min": -28.70472526550293,
      "activations/layer18_attention_weight_max": 39.87342834472656,
      "activations/layer18_attention_weight_min": -27.025615692138672,
      "activations/layer19_attention_weight_max": 42.65639877319336,
      "activations/layer19_attention_weight_min": -33.93209457397461,
      "activations/layer1_attention_weight_max": 17.475996017456055,
      "activations/layer1_attention_weight_min": -15.516213417053223,
      "activations/layer20_attention_weight_max": 36.214229583740234,
      "activations/layer20_attention_weight_min": -24.074581146240234,
      "activations/layer21_attention_weight_max": 36.70934295654297,
      "activations/layer21_attention_weight_min": -23.50090980529785,
      "activations/layer22_attention_weight_max": 50.7436637878418,
      "activations/layer22_attention_weight_min": -32.22111129760742,
      "activations/layer23_attention_weight_max": 40.24052429199219,
      "activations/layer23_attention_weight_min": -25.035184860229492,
      "activations/layer2_attention_weight_max": 34.58347702026367,
      "activations/layer2_attention_weight_min": -34.089813232421875,
      "activations/layer3_attention_weight_max": 106.83455657958984,
      "activations/layer3_attention_weight_min": -107.46983337402344,
      "activations/layer4_attention_weight_max": 109.9051284790039,
      "activations/layer4_attention_weight_min": -115.53617095947266,
      "activations/layer5_attention_weight_max": 51.02680969238281,
      "activations/layer5_attention_weight_min": -63.33726501464844,
      "activations/layer6_attention_weight_max": 45.10773849487305,
      "activations/layer6_attention_weight_min": -47.73183822631836,
      "activations/layer7_attention_weight_max": 99.53666687011719,
      "activations/layer7_attention_weight_min": -99.22112274169922,
      "activations/layer8_attention_weight_max": 43.0687370300293,
      "activations/layer8_attention_weight_min": -47.99393081665039,
      "activations/layer9_attention_weight_max": 33.8325309753418,
      "activations/layer9_attention_weight_min": -34.87502670288086,
      "epoch": 21.12,
      "learning_rate": 1.3971590909090907e-05,
      "loss": 2.7093,
      "step": 363400
    },
    {
      "activations/layer0_attention_weight_max": 16.05714225769043,
      "activations/layer0_attention_weight_min": -14.588234901428223,
      "activations/layer10_attention_weight_max": 37.267215728759766,
      "activations/layer10_attention_weight_min": -34.19513702392578,
      "activations/layer11_attention_weight_max": 34.66508865356445,
      "activations/layer11_attention_weight_min": -35.243507385253906,
      "activations/layer12_attention_weight_max": 25.12405776977539,
      "activations/layer12_attention_weight_min": -29.902080535888672,
      "activations/layer13_attention_weight_max": 36.913055419921875,
      "activations/layer13_attention_weight_min": -34.54091262817383,
      "activations/layer14_attention_weight_max": 39.56803512573242,
      "activations/layer14_attention_weight_min": -35.296714782714844,
      "activations/layer15_attention_weight_max": 37.352333068847656,
      "activations/layer15_attention_weight_min": -33.09726333618164,
      "activations/layer16_attention_weight_max": 29.239957809448242,
      "activations/layer16_attention_weight_min": -28.593894958496094,
      "activations/layer17_attention_weight_max": 31.40800666809082,
      "activations/layer17_attention_weight_min": -24.313798904418945,
      "activations/layer18_attention_weight_max": 33.0146598815918,
      "activations/layer18_attention_weight_min": -25.880258560180664,
      "activations/layer19_attention_weight_max": 34.93024826049805,
      "activations/layer19_attention_weight_min": -30.47537612915039,
      "activations/layer1_attention_weight_max": 15.907098770141602,
      "activations/layer1_attention_weight_min": -14.443286895751953,
      "activations/layer20_attention_weight_max": 28.321170806884766,
      "activations/layer20_attention_weight_min": -25.064163208007812,
      "activations/layer21_attention_weight_max": 28.259862899780273,
      "activations/layer21_attention_weight_min": -24.791717529296875,
      "activations/layer22_attention_weight_max": 43.41952133178711,
      "activations/layer22_attention_weight_min": -32.792442321777344,
      "activations/layer23_attention_weight_max": 32.61144256591797,
      "activations/layer23_attention_weight_min": -26.46541404724121,
      "activations/layer2_attention_weight_max": 34.10892868041992,
      "activations/layer2_attention_weight_min": -33.077178955078125,
      "activations/layer3_attention_weight_max": 106.97525787353516,
      "activations/layer3_attention_weight_min": -106.54603576660156,
      "activations/layer4_attention_weight_max": 109.27274322509766,
      "activations/layer4_attention_weight_min": -109.7936019897461,
      "activations/layer5_attention_weight_max": 54.01521301269531,
      "activations/layer5_attention_weight_min": -63.0250244140625,
      "activations/layer6_attention_weight_max": 44.779544830322266,
      "activations/layer6_attention_weight_min": -47.81125259399414,
      "activations/layer7_attention_weight_max": 103.69603729248047,
      "activations/layer7_attention_weight_min": -97.366943359375,
      "activations/layer8_attention_weight_max": 45.249359130859375,
      "activations/layer8_attention_weight_min": -45.856178283691406,
      "activations/layer9_attention_weight_max": 31.774999618530273,
      "activations/layer9_attention_weight_min": -32.69572067260742,
      "epoch": 21.12,
      "learning_rate": 1.3952651515151514e-05,
      "loss": 2.7083,
      "step": 363450
    },
    {
      "activations/layer0_attention_weight_max": 18.31715965270996,
      "activations/layer0_attention_weight_min": -14.902843475341797,
      "activations/layer10_attention_weight_max": 33.16582107543945,
      "activations/layer10_attention_weight_min": -36.48329544067383,
      "activations/layer11_attention_weight_max": 32.454078674316406,
      "activations/layer11_attention_weight_min": -33.2667121887207,
      "activations/layer12_attention_weight_max": 24.816736221313477,
      "activations/layer12_attention_weight_min": -26.322586059570312,
      "activations/layer13_attention_weight_max": 37.757713317871094,
      "activations/layer13_attention_weight_min": -35.756874084472656,
      "activations/layer14_attention_weight_max": 39.45449447631836,
      "activations/layer14_attention_weight_min": -36.75327682495117,
      "activations/layer15_attention_weight_max": 38.15677261352539,
      "activations/layer15_attention_weight_min": -35.20684051513672,
      "activations/layer16_attention_weight_max": 27.789623260498047,
      "activations/layer16_attention_weight_min": -27.813390731811523,
      "activations/layer17_attention_weight_max": 29.86518096923828,
      "activations/layer17_attention_weight_min": -26.153526306152344,
      "activations/layer18_attention_weight_max": 29.504562377929688,
      "activations/layer18_attention_weight_min": -26.25006866455078,
      "activations/layer19_attention_weight_max": 32.07720947265625,
      "activations/layer19_attention_weight_min": -34.10789108276367,
      "activations/layer1_attention_weight_max": 15.940363883972168,
      "activations/layer1_attention_weight_min": -15.283287048339844,
      "activations/layer20_attention_weight_max": 26.560834884643555,
      "activations/layer20_attention_weight_min": -26.89501953125,
      "activations/layer21_attention_weight_max": 28.222259521484375,
      "activations/layer21_attention_weight_min": -27.036705017089844,
      "activations/layer22_attention_weight_max": 37.724266052246094,
      "activations/layer22_attention_weight_min": -32.601806640625,
      "activations/layer23_attention_weight_max": 31.763830184936523,
      "activations/layer23_attention_weight_min": -27.311275482177734,
      "activations/layer2_attention_weight_max": 33.053321838378906,
      "activations/layer2_attention_weight_min": -33.602783203125,
      "activations/layer3_attention_weight_max": 105.3635025024414,
      "activations/layer3_attention_weight_min": -103.47234344482422,
      "activations/layer4_attention_weight_max": 106.78192138671875,
      "activations/layer4_attention_weight_min": -111.00482177734375,
      "activations/layer5_attention_weight_max": 48.95915985107422,
      "activations/layer5_attention_weight_min": -61.80289840698242,
      "activations/layer6_attention_weight_max": 43.29389190673828,
      "activations/layer6_attention_weight_min": -47.94752883911133,
      "activations/layer7_attention_weight_max": 89.29792022705078,
      "activations/layer7_attention_weight_min": -101.97021484375,
      "activations/layer8_attention_weight_max": 43.51361083984375,
      "activations/layer8_attention_weight_min": -49.57101058959961,
      "activations/layer9_attention_weight_max": 33.92281723022461,
      "activations/layer9_attention_weight_min": -39.15657043457031,
      "epoch": 21.12,
      "learning_rate": 1.3933712121212119e-05,
      "loss": 2.6985,
      "step": 363500
    },
    {
      "activations/layer0_attention_weight_max": 16.293041229248047,
      "activations/layer0_attention_weight_min": -13.73196792602539,
      "activations/layer10_attention_weight_max": 34.88988494873047,
      "activations/layer10_attention_weight_min": -37.0392951965332,
      "activations/layer11_attention_weight_max": 36.902915954589844,
      "activations/layer11_attention_weight_min": -37.69082260131836,
      "activations/layer12_attention_weight_max": 27.074703216552734,
      "activations/layer12_attention_weight_min": -26.839021682739258,
      "activations/layer13_attention_weight_max": 41.313209533691406,
      "activations/layer13_attention_weight_min": -36.98005676269531,
      "activations/layer14_attention_weight_max": 41.84511947631836,
      "activations/layer14_attention_weight_min": -38.24232482910156,
      "activations/layer15_attention_weight_max": 40.52830505371094,
      "activations/layer15_attention_weight_min": -38.0434455871582,
      "activations/layer16_attention_weight_max": 31.623449325561523,
      "activations/layer16_attention_weight_min": -30.026771545410156,
      "activations/layer17_attention_weight_max": 31.677249908447266,
      "activations/layer17_attention_weight_min": -26.769533157348633,
      "activations/layer18_attention_weight_max": 31.604267120361328,
      "activations/layer18_attention_weight_min": -25.334810256958008,
      "activations/layer19_attention_weight_max": 33.07722091674805,
      "activations/layer19_attention_weight_min": -30.714345932006836,
      "activations/layer1_attention_weight_max": 15.875876426696777,
      "activations/layer1_attention_weight_min": -14.630146026611328,
      "activations/layer20_attention_weight_max": 29.018829345703125,
      "activations/layer20_attention_weight_min": -24.278871536254883,
      "activations/layer21_attention_weight_max": 28.034133911132812,
      "activations/layer21_attention_weight_min": -23.58592987060547,
      "activations/layer22_attention_weight_max": 47.93081283569336,
      "activations/layer22_attention_weight_min": -29.688936233520508,
      "activations/layer23_attention_weight_max": 34.14459228515625,
      "activations/layer23_attention_weight_min": -25.02515983581543,
      "activations/layer2_attention_weight_max": 34.19224548339844,
      "activations/layer2_attention_weight_min": -32.207942962646484,
      "activations/layer3_attention_weight_max": 105.0777816772461,
      "activations/layer3_attention_weight_min": -102.7637710571289,
      "activations/layer4_attention_weight_max": 107.11444091796875,
      "activations/layer4_attention_weight_min": -109.58308410644531,
      "activations/layer5_attention_weight_max": 53.2099609375,
      "activations/layer5_attention_weight_min": -60.426849365234375,
      "activations/layer6_attention_weight_max": 48.22030258178711,
      "activations/layer6_attention_weight_min": -47.79175567626953,
      "activations/layer7_attention_weight_max": 95.54452514648438,
      "activations/layer7_attention_weight_min": -101.99203491210938,
      "activations/layer8_attention_weight_max": 46.89336013793945,
      "activations/layer8_attention_weight_min": -49.31316375732422,
      "activations/layer9_attention_weight_max": 33.97675704956055,
      "activations/layer9_attention_weight_min": -38.44622802734375,
      "epoch": 21.12,
      "learning_rate": 1.3914772727272725e-05,
      "loss": 2.7165,
      "step": 363550
    },
    {
      "activations/layer0_attention_weight_max": 16.863855361938477,
      "activations/layer0_attention_weight_min": -14.572724342346191,
      "activations/layer10_attention_weight_max": 34.22715759277344,
      "activations/layer10_attention_weight_min": -35.83527374267578,
      "activations/layer11_attention_weight_max": 36.30904769897461,
      "activations/layer11_attention_weight_min": -34.969627380371094,
      "activations/layer12_attention_weight_max": 27.539485931396484,
      "activations/layer12_attention_weight_min": -27.81600570678711,
      "activations/layer13_attention_weight_max": 39.03029251098633,
      "activations/layer13_attention_weight_min": -37.98259735107422,
      "activations/layer14_attention_weight_max": 44.86867141723633,
      "activations/layer14_attention_weight_min": -38.60183334350586,
      "activations/layer15_attention_weight_max": 39.36573791503906,
      "activations/layer15_attention_weight_min": -37.958168029785156,
      "activations/layer16_attention_weight_max": 29.01000213623047,
      "activations/layer16_attention_weight_min": -30.06829833984375,
      "activations/layer17_attention_weight_max": 31.11110496520996,
      "activations/layer17_attention_weight_min": -28.840036392211914,
      "activations/layer18_attention_weight_max": 32.121559143066406,
      "activations/layer18_attention_weight_min": -27.811479568481445,
      "activations/layer19_attention_weight_max": 35.088436126708984,
      "activations/layer19_attention_weight_min": -32.877201080322266,
      "activations/layer1_attention_weight_max": 15.352605819702148,
      "activations/layer1_attention_weight_min": -15.661979675292969,
      "activations/layer20_attention_weight_max": 28.98568344116211,
      "activations/layer20_attention_weight_min": -26.053932189941406,
      "activations/layer21_attention_weight_max": 30.24992561340332,
      "activations/layer21_attention_weight_min": -28.155675888061523,
      "activations/layer22_attention_weight_max": 44.771183013916016,
      "activations/layer22_attention_weight_min": -30.932968139648438,
      "activations/layer23_attention_weight_max": 33.00198745727539,
      "activations/layer23_attention_weight_min": -26.277782440185547,
      "activations/layer2_attention_weight_max": 33.08587646484375,
      "activations/layer2_attention_weight_min": -32.769439697265625,
      "activations/layer3_attention_weight_max": 100.81024932861328,
      "activations/layer3_attention_weight_min": -102.76848602294922,
      "activations/layer4_attention_weight_max": 106.95668029785156,
      "activations/layer4_attention_weight_min": -110.5037841796875,
      "activations/layer5_attention_weight_max": 55.08384323120117,
      "activations/layer5_attention_weight_min": -64.64895629882812,
      "activations/layer6_attention_weight_max": 44.1940803527832,
      "activations/layer6_attention_weight_min": -50.7672004699707,
      "activations/layer7_attention_weight_max": 96.54239654541016,
      "activations/layer7_attention_weight_min": -99.49052429199219,
      "activations/layer8_attention_weight_max": 42.81352615356445,
      "activations/layer8_attention_weight_min": -47.27292251586914,
      "activations/layer9_attention_weight_max": 31.156064987182617,
      "activations/layer9_attention_weight_min": -35.62574005126953,
      "epoch": 21.13,
      "learning_rate": 1.3895833333333333e-05,
      "loss": 2.7025,
      "step": 363600
    },
    {
      "activations/layer0_attention_weight_max": 15.870850563049316,
      "activations/layer0_attention_weight_min": -13.518668174743652,
      "activations/layer10_attention_weight_max": 33.520320892333984,
      "activations/layer10_attention_weight_min": -34.58964920043945,
      "activations/layer11_attention_weight_max": 33.437896728515625,
      "activations/layer11_attention_weight_min": -33.09222412109375,
      "activations/layer12_attention_weight_max": 28.888710021972656,
      "activations/layer12_attention_weight_min": -26.027551651000977,
      "activations/layer13_attention_weight_max": 41.34063720703125,
      "activations/layer13_attention_weight_min": -34.23146057128906,
      "activations/layer14_attention_weight_max": 43.52768325805664,
      "activations/layer14_attention_weight_min": -36.83312225341797,
      "activations/layer15_attention_weight_max": 39.49066162109375,
      "activations/layer15_attention_weight_min": -33.74367141723633,
      "activations/layer16_attention_weight_max": 29.97037124633789,
      "activations/layer16_attention_weight_min": -28.064884185791016,
      "activations/layer17_attention_weight_max": 32.52173614501953,
      "activations/layer17_attention_weight_min": -28.072078704833984,
      "activations/layer18_attention_weight_max": 33.51346206665039,
      "activations/layer18_attention_weight_min": -24.702537536621094,
      "activations/layer19_attention_weight_max": 34.12929153442383,
      "activations/layer19_attention_weight_min": -34.526187896728516,
      "activations/layer1_attention_weight_max": 17.135711669921875,
      "activations/layer1_attention_weight_min": -16.339733123779297,
      "activations/layer20_attention_weight_max": 31.385507583618164,
      "activations/layer20_attention_weight_min": -26.83416748046875,
      "activations/layer21_attention_weight_max": 30.8049373626709,
      "activations/layer21_attention_weight_min": -25.830810546875,
      "activations/layer22_attention_weight_max": 45.31681442260742,
      "activations/layer22_attention_weight_min": -31.50513458251953,
      "activations/layer23_attention_weight_max": 35.561248779296875,
      "activations/layer23_attention_weight_min": -26.80908203125,
      "activations/layer2_attention_weight_max": 33.53257751464844,
      "activations/layer2_attention_weight_min": -33.309329986572266,
      "activations/layer3_attention_weight_max": 104.85859680175781,
      "activations/layer3_attention_weight_min": -105.70870971679688,
      "activations/layer4_attention_weight_max": 105.04192352294922,
      "activations/layer4_attention_weight_min": -111.12701416015625,
      "activations/layer5_attention_weight_max": 50.953304290771484,
      "activations/layer5_attention_weight_min": -60.78318405151367,
      "activations/layer6_attention_weight_max": 46.615787506103516,
      "activations/layer6_attention_weight_min": -48.79500198364258,
      "activations/layer7_attention_weight_max": 98.99036407470703,
      "activations/layer7_attention_weight_min": -101.11924743652344,
      "activations/layer8_attention_weight_max": 47.56813430786133,
      "activations/layer8_attention_weight_min": -49.102073669433594,
      "activations/layer9_attention_weight_max": 36.15089797973633,
      "activations/layer9_attention_weight_min": -35.816593170166016,
      "epoch": 21.13,
      "learning_rate": 1.3876893939393938e-05,
      "loss": 2.7021,
      "step": 363650
    },
    {
      "activations/layer0_attention_weight_max": 16.93813133239746,
      "activations/layer0_attention_weight_min": -15.101813316345215,
      "activations/layer10_attention_weight_max": 35.236671447753906,
      "activations/layer10_attention_weight_min": -35.58477783203125,
      "activations/layer11_attention_weight_max": 34.049461364746094,
      "activations/layer11_attention_weight_min": -36.44294357299805,
      "activations/layer12_attention_weight_max": 27.728015899658203,
      "activations/layer12_attention_weight_min": -28.15581703186035,
      "activations/layer13_attention_weight_max": 40.81493377685547,
      "activations/layer13_attention_weight_min": -35.966060638427734,
      "activations/layer14_attention_weight_max": 44.33428955078125,
      "activations/layer14_attention_weight_min": -38.028648376464844,
      "activations/layer15_attention_weight_max": 39.707176208496094,
      "activations/layer15_attention_weight_min": -36.770362854003906,
      "activations/layer16_attention_weight_max": 30.011337280273438,
      "activations/layer16_attention_weight_min": -27.67002296447754,
      "activations/layer17_attention_weight_max": 31.898345947265625,
      "activations/layer17_attention_weight_min": -26.155200958251953,
      "activations/layer18_attention_weight_max": 35.84733200073242,
      "activations/layer18_attention_weight_min": -26.441335678100586,
      "activations/layer19_attention_weight_max": 30.906524658203125,
      "activations/layer19_attention_weight_min": -31.215290069580078,
      "activations/layer1_attention_weight_max": 15.727865219116211,
      "activations/layer1_attention_weight_min": -15.703096389770508,
      "activations/layer20_attention_weight_max": 28.440807342529297,
      "activations/layer20_attention_weight_min": -24.84735107421875,
      "activations/layer21_attention_weight_max": 27.8344783782959,
      "activations/layer21_attention_weight_min": -24.31253433227539,
      "activations/layer22_attention_weight_max": 41.84992218017578,
      "activations/layer22_attention_weight_min": -30.08294677734375,
      "activations/layer23_attention_weight_max": 31.901334762573242,
      "activations/layer23_attention_weight_min": -26.01801300048828,
      "activations/layer2_attention_weight_max": 32.045631408691406,
      "activations/layer2_attention_weight_min": -31.999347686767578,
      "activations/layer3_attention_weight_max": 100.71833801269531,
      "activations/layer3_attention_weight_min": -105.36890411376953,
      "activations/layer4_attention_weight_max": 106.9287109375,
      "activations/layer4_attention_weight_min": -111.12569427490234,
      "activations/layer5_attention_weight_max": 48.8657341003418,
      "activations/layer5_attention_weight_min": -62.65830993652344,
      "activations/layer6_attention_weight_max": 45.69452667236328,
      "activations/layer6_attention_weight_min": -48.77175521850586,
      "activations/layer7_attention_weight_max": 95.66856384277344,
      "activations/layer7_attention_weight_min": -104.34738159179688,
      "activations/layer8_attention_weight_max": 41.25444412231445,
      "activations/layer8_attention_weight_min": -47.08905792236328,
      "activations/layer9_attention_weight_max": 33.19874954223633,
      "activations/layer9_attention_weight_min": -34.596920013427734,
      "epoch": 21.13,
      "learning_rate": 1.3857954545454545e-05,
      "loss": 2.7154,
      "step": 363700
    },
    {
      "activations/layer0_attention_weight_max": 16.577198028564453,
      "activations/layer0_attention_weight_min": -15.005775451660156,
      "activations/layer10_attention_weight_max": 35.32451248168945,
      "activations/layer10_attention_weight_min": -37.203704833984375,
      "activations/layer11_attention_weight_max": 37.112144470214844,
      "activations/layer11_attention_weight_min": -36.61997604370117,
      "activations/layer12_attention_weight_max": 26.545610427856445,
      "activations/layer12_attention_weight_min": -26.073719024658203,
      "activations/layer13_attention_weight_max": 39.02708053588867,
      "activations/layer13_attention_weight_min": -34.938053131103516,
      "activations/layer14_attention_weight_max": 42.494300842285156,
      "activations/layer14_attention_weight_min": -36.82473373413086,
      "activations/layer15_attention_weight_max": 38.36991882324219,
      "activations/layer15_attention_weight_min": -35.17227554321289,
      "activations/layer16_attention_weight_max": 30.505712509155273,
      "activations/layer16_attention_weight_min": -29.554996490478516,
      "activations/layer17_attention_weight_max": 31.20615577697754,
      "activations/layer17_attention_weight_min": -24.84303855895996,
      "activations/layer18_attention_weight_max": 28.843730926513672,
      "activations/layer18_attention_weight_min": -24.92186164855957,
      "activations/layer19_attention_weight_max": 31.748937606811523,
      "activations/layer19_attention_weight_min": -31.538040161132812,
      "activations/layer1_attention_weight_max": 16.4412841796875,
      "activations/layer1_attention_weight_min": -14.747267723083496,
      "activations/layer20_attention_weight_max": 26.209928512573242,
      "activations/layer20_attention_weight_min": -25.5399227142334,
      "activations/layer21_attention_weight_max": 27.71682357788086,
      "activations/layer21_attention_weight_min": -25.24821662902832,
      "activations/layer22_attention_weight_max": 38.215171813964844,
      "activations/layer22_attention_weight_min": -31.239986419677734,
      "activations/layer23_attention_weight_max": 27.985334396362305,
      "activations/layer23_attention_weight_min": -25.283432006835938,
      "activations/layer2_attention_weight_max": 34.053955078125,
      "activations/layer2_attention_weight_min": -34.500579833984375,
      "activations/layer3_attention_weight_max": 105.39151763916016,
      "activations/layer3_attention_weight_min": -106.41812896728516,
      "activations/layer4_attention_weight_max": 107.56982421875,
      "activations/layer4_attention_weight_min": -112.19390106201172,
      "activations/layer5_attention_weight_max": 52.4322624206543,
      "activations/layer5_attention_weight_min": -63.903282165527344,
      "activations/layer6_attention_weight_max": 47.58836364746094,
      "activations/layer6_attention_weight_min": -50.90553665161133,
      "activations/layer7_attention_weight_max": 96.8472900390625,
      "activations/layer7_attention_weight_min": -104.44673919677734,
      "activations/layer8_attention_weight_max": 43.453399658203125,
      "activations/layer8_attention_weight_min": -46.79536819458008,
      "activations/layer9_attention_weight_max": 32.46132278442383,
      "activations/layer9_attention_weight_min": -35.62813186645508,
      "epoch": 21.14,
      "learning_rate": 1.3839015151515151e-05,
      "loss": 2.713,
      "step": 363750
    },
    {
      "activations/layer0_attention_weight_max": 16.4132022857666,
      "activations/layer0_attention_weight_min": -14.258614540100098,
      "activations/layer10_attention_weight_max": 32.70903396606445,
      "activations/layer10_attention_weight_min": -35.616455078125,
      "activations/layer11_attention_weight_max": 32.03875732421875,
      "activations/layer11_attention_weight_min": -34.65546417236328,
      "activations/layer12_attention_weight_max": 24.91977882385254,
      "activations/layer12_attention_weight_min": -26.879514694213867,
      "activations/layer13_attention_weight_max": 38.82396697998047,
      "activations/layer13_attention_weight_min": -35.705413818359375,
      "activations/layer14_attention_weight_max": 39.02199172973633,
      "activations/layer14_attention_weight_min": -38.00156021118164,
      "activations/layer15_attention_weight_max": 37.455936431884766,
      "activations/layer15_attention_weight_min": -36.50006866455078,
      "activations/layer16_attention_weight_max": 27.727001190185547,
      "activations/layer16_attention_weight_min": -27.306386947631836,
      "activations/layer17_attention_weight_max": 26.809329986572266,
      "activations/layer17_attention_weight_min": -25.91069793701172,
      "activations/layer18_attention_weight_max": 31.057540893554688,
      "activations/layer18_attention_weight_min": -26.475522994995117,
      "activations/layer19_attention_weight_max": 34.65788650512695,
      "activations/layer19_attention_weight_min": -32.05860137939453,
      "activations/layer1_attention_weight_max": 16.81127166748047,
      "activations/layer1_attention_weight_min": -13.840970993041992,
      "activations/layer20_attention_weight_max": 29.250288009643555,
      "activations/layer20_attention_weight_min": -25.289640426635742,
      "activations/layer21_attention_weight_max": 29.690635681152344,
      "activations/layer21_attention_weight_min": -25.472457885742188,
      "activations/layer22_attention_weight_max": 42.89630126953125,
      "activations/layer22_attention_weight_min": -32.63560104370117,
      "activations/layer23_attention_weight_max": 30.41815757751465,
      "activations/layer23_attention_weight_min": -27.063146591186523,
      "activations/layer2_attention_weight_max": 34.37104034423828,
      "activations/layer2_attention_weight_min": -32.92948913574219,
      "activations/layer3_attention_weight_max": 101.95597076416016,
      "activations/layer3_attention_weight_min": -104.18376922607422,
      "activations/layer4_attention_weight_max": 107.52474212646484,
      "activations/layer4_attention_weight_min": -113.06050109863281,
      "activations/layer5_attention_weight_max": 47.916664123535156,
      "activations/layer5_attention_weight_min": -63.793792724609375,
      "activations/layer6_attention_weight_max": 43.986595153808594,
      "activations/layer6_attention_weight_min": -48.78337478637695,
      "activations/layer7_attention_weight_max": 97.39639282226562,
      "activations/layer7_attention_weight_min": -98.27510833740234,
      "activations/layer8_attention_weight_max": 40.509456634521484,
      "activations/layer8_attention_weight_min": -46.816368103027344,
      "activations/layer9_attention_weight_max": 32.96234130859375,
      "activations/layer9_attention_weight_min": -34.45105743408203,
      "epoch": 21.14,
      "learning_rate": 1.3820075757575756e-05,
      "loss": 2.7048,
      "step": 363800
    },
    {
      "activations/layer0_attention_weight_max": 16.499752044677734,
      "activations/layer0_attention_weight_min": -13.201116561889648,
      "activations/layer10_attention_weight_max": 37.31242370605469,
      "activations/layer10_attention_weight_min": -38.18860626220703,
      "activations/layer11_attention_weight_max": 36.58860397338867,
      "activations/layer11_attention_weight_min": -36.98849868774414,
      "activations/layer12_attention_weight_max": 30.046688079833984,
      "activations/layer12_attention_weight_min": -28.04962921142578,
      "activations/layer13_attention_weight_max": 43.93796920776367,
      "activations/layer13_attention_weight_min": -39.033016204833984,
      "activations/layer14_attention_weight_max": 45.569969177246094,
      "activations/layer14_attention_weight_min": -40.56068420410156,
      "activations/layer15_attention_weight_max": 42.07064437866211,
      "activations/layer15_attention_weight_min": -37.57013702392578,
      "activations/layer16_attention_weight_max": 29.812583923339844,
      "activations/layer16_attention_weight_min": -28.16733169555664,
      "activations/layer17_attention_weight_max": 28.85068130493164,
      "activations/layer17_attention_weight_min": -27.721879959106445,
      "activations/layer18_attention_weight_max": 32.01189041137695,
      "activations/layer18_attention_weight_min": -27.519193649291992,
      "activations/layer19_attention_weight_max": 33.538787841796875,
      "activations/layer19_attention_weight_min": -31.64503288269043,
      "activations/layer1_attention_weight_max": 17.12603759765625,
      "activations/layer1_attention_weight_min": -15.410259246826172,
      "activations/layer20_attention_weight_max": 28.87047576904297,
      "activations/layer20_attention_weight_min": -25.73773765563965,
      "activations/layer21_attention_weight_max": 27.024776458740234,
      "activations/layer21_attention_weight_min": -25.09799575805664,
      "activations/layer22_attention_weight_max": 40.1488037109375,
      "activations/layer22_attention_weight_min": -34.0172119140625,
      "activations/layer23_attention_weight_max": 32.540218353271484,
      "activations/layer23_attention_weight_min": -27.378684997558594,
      "activations/layer2_attention_weight_max": 32.52351760864258,
      "activations/layer2_attention_weight_min": -33.374351501464844,
      "activations/layer3_attention_weight_max": 104.60967254638672,
      "activations/layer3_attention_weight_min": -104.85157775878906,
      "activations/layer4_attention_weight_max": 111.442626953125,
      "activations/layer4_attention_weight_min": -113.079833984375,
      "activations/layer5_attention_weight_max": 52.74756622314453,
      "activations/layer5_attention_weight_min": -64.5550765991211,
      "activations/layer6_attention_weight_max": 46.30009078979492,
      "activations/layer6_attention_weight_min": -48.418235778808594,
      "activations/layer7_attention_weight_max": 100.05301666259766,
      "activations/layer7_attention_weight_min": -99.54264831542969,
      "activations/layer8_attention_weight_max": 44.09952926635742,
      "activations/layer8_attention_weight_min": -47.55214309692383,
      "activations/layer9_attention_weight_max": 33.64876937866211,
      "activations/layer9_attention_weight_min": -37.6549072265625,
      "epoch": 21.14,
      "learning_rate": 1.3801136363636363e-05,
      "loss": 2.707,
      "step": 363850
    },
    {
      "activations/layer0_attention_weight_max": 15.846835136413574,
      "activations/layer0_attention_weight_min": -14.530217170715332,
      "activations/layer10_attention_weight_max": 36.13320541381836,
      "activations/layer10_attention_weight_min": -34.977073669433594,
      "activations/layer11_attention_weight_max": 35.7697639465332,
      "activations/layer11_attention_weight_min": -36.054786682128906,
      "activations/layer12_attention_weight_max": 27.84885025024414,
      "activations/layer12_attention_weight_min": -27.567916870117188,
      "activations/layer13_attention_weight_max": 39.827571868896484,
      "activations/layer13_attention_weight_min": -36.53892135620117,
      "activations/layer14_attention_weight_max": 43.11627960205078,
      "activations/layer14_attention_weight_min": -39.767852783203125,
      "activations/layer15_attention_weight_max": 41.156272888183594,
      "activations/layer15_attention_weight_min": -40.19276809692383,
      "activations/layer16_attention_weight_max": 30.569034576416016,
      "activations/layer16_attention_weight_min": -29.02700424194336,
      "activations/layer17_attention_weight_max": 30.468807220458984,
      "activations/layer17_attention_weight_min": -28.01136589050293,
      "activations/layer18_attention_weight_max": 36.4492073059082,
      "activations/layer18_attention_weight_min": -28.541603088378906,
      "activations/layer19_attention_weight_max": 35.816715240478516,
      "activations/layer19_attention_weight_min": -32.8907356262207,
      "activations/layer1_attention_weight_max": 15.905604362487793,
      "activations/layer1_attention_weight_min": -14.657279968261719,
      "activations/layer20_attention_weight_max": 33.463436126708984,
      "activations/layer20_attention_weight_min": -27.245054244995117,
      "activations/layer21_attention_weight_max": 32.584957122802734,
      "activations/layer21_attention_weight_min": -25.976699829101562,
      "activations/layer22_attention_weight_max": 43.69491958618164,
      "activations/layer22_attention_weight_min": -34.52574920654297,
      "activations/layer23_attention_weight_max": 32.189510345458984,
      "activations/layer23_attention_weight_min": -26.778671264648438,
      "activations/layer2_attention_weight_max": 33.43046569824219,
      "activations/layer2_attention_weight_min": -33.28096389770508,
      "activations/layer3_attention_weight_max": 108.42403411865234,
      "activations/layer3_attention_weight_min": -107.8467025756836,
      "activations/layer4_attention_weight_max": 113.38301849365234,
      "activations/layer4_attention_weight_min": -110.5005111694336,
      "activations/layer5_attention_weight_max": 56.014007568359375,
      "activations/layer5_attention_weight_min": -63.310707092285156,
      "activations/layer6_attention_weight_max": 45.793155670166016,
      "activations/layer6_attention_weight_min": -48.77583694458008,
      "activations/layer7_attention_weight_max": 97.3864974975586,
      "activations/layer7_attention_weight_min": -108.00287628173828,
      "activations/layer8_attention_weight_max": 45.4328498840332,
      "activations/layer8_attention_weight_min": -44.31875991821289,
      "activations/layer9_attention_weight_max": 33.34215545654297,
      "activations/layer9_attention_weight_min": -35.304203033447266,
      "epoch": 21.14,
      "learning_rate": 1.378219696969697e-05,
      "loss": 2.7157,
      "step": 363900
    },
    {
      "activations/layer0_attention_weight_max": 16.340871810913086,
      "activations/layer0_attention_weight_min": -16.14134979248047,
      "activations/layer10_attention_weight_max": 33.093238830566406,
      "activations/layer10_attention_weight_min": -35.67046356201172,
      "activations/layer11_attention_weight_max": 33.123653411865234,
      "activations/layer11_attention_weight_min": -35.60044860839844,
      "activations/layer12_attention_weight_max": 26.495725631713867,
      "activations/layer12_attention_weight_min": -27.197832107543945,
      "activations/layer13_attention_weight_max": 40.086753845214844,
      "activations/layer13_attention_weight_min": -36.316429138183594,
      "activations/layer14_attention_weight_max": 43.18259048461914,
      "activations/layer14_attention_weight_min": -39.64833068847656,
      "activations/layer15_attention_weight_max": 41.205692291259766,
      "activations/layer15_attention_weight_min": -36.45494079589844,
      "activations/layer16_attention_weight_max": 28.284345626831055,
      "activations/layer16_attention_weight_min": -32.44465637207031,
      "activations/layer17_attention_weight_max": 28.883962631225586,
      "activations/layer17_attention_weight_min": -26.29633331298828,
      "activations/layer18_attention_weight_max": 31.400487899780273,
      "activations/layer18_attention_weight_min": -25.85262680053711,
      "activations/layer19_attention_weight_max": 37.74158477783203,
      "activations/layer19_attention_weight_min": -30.506208419799805,
      "activations/layer1_attention_weight_max": 15.6677885055542,
      "activations/layer1_attention_weight_min": -14.152565956115723,
      "activations/layer20_attention_weight_max": 32.30072784423828,
      "activations/layer20_attention_weight_min": -25.463294982910156,
      "activations/layer21_attention_weight_max": 30.741085052490234,
      "activations/layer21_attention_weight_min": -24.13229751586914,
      "activations/layer22_attention_weight_max": 44.93643569946289,
      "activations/layer22_attention_weight_min": -32.944026947021484,
      "activations/layer23_attention_weight_max": 36.088775634765625,
      "activations/layer23_attention_weight_min": -25.871824264526367,
      "activations/layer2_attention_weight_max": 32.404056549072266,
      "activations/layer2_attention_weight_min": -31.789962768554688,
      "activations/layer3_attention_weight_max": 103.47898864746094,
      "activations/layer3_attention_weight_min": -101.85835266113281,
      "activations/layer4_attention_weight_max": 108.21955871582031,
      "activations/layer4_attention_weight_min": -107.46943664550781,
      "activations/layer5_attention_weight_max": 50.91484069824219,
      "activations/layer5_attention_weight_min": -62.7132568359375,
      "activations/layer6_attention_weight_max": 45.5693473815918,
      "activations/layer6_attention_weight_min": -46.057254791259766,
      "activations/layer7_attention_weight_max": 91.5328598022461,
      "activations/layer7_attention_weight_min": -95.8936538696289,
      "activations/layer8_attention_weight_max": 40.8388786315918,
      "activations/layer8_attention_weight_min": -44.902992248535156,
      "activations/layer9_attention_weight_max": 31.306425094604492,
      "activations/layer9_attention_weight_min": -34.165771484375,
      "epoch": 21.15,
      "learning_rate": 1.3763257575757574e-05,
      "loss": 2.708,
      "step": 363950
    },
    {
      "activations/layer0_attention_weight_max": 16.910215377807617,
      "activations/layer0_attention_weight_min": -13.032798767089844,
      "activations/layer10_attention_weight_max": 38.00180435180664,
      "activations/layer10_attention_weight_min": -40.395904541015625,
      "activations/layer11_attention_weight_max": 37.345130920410156,
      "activations/layer11_attention_weight_min": -40.33293914794922,
      "activations/layer12_attention_weight_max": 27.61397933959961,
      "activations/layer12_attention_weight_min": -26.69257354736328,
      "activations/layer13_attention_weight_max": 40.4791374206543,
      "activations/layer13_attention_weight_min": -37.69489669799805,
      "activations/layer14_attention_weight_max": 43.48994445800781,
      "activations/layer14_attention_weight_min": -39.62308883666992,
      "activations/layer15_attention_weight_max": 41.93814468383789,
      "activations/layer15_attention_weight_min": -37.21108627319336,
      "activations/layer16_attention_weight_max": 30.371492385864258,
      "activations/layer16_attention_weight_min": -28.846174240112305,
      "activations/layer17_attention_weight_max": 30.31191062927246,
      "activations/layer17_attention_weight_min": -27.002092361450195,
      "activations/layer18_attention_weight_max": 34.21979904174805,
      "activations/layer18_attention_weight_min": -24.83245086669922,
      "activations/layer19_attention_weight_max": 40.95497512817383,
      "activations/layer19_attention_weight_min": -32.214752197265625,
      "activations/layer1_attention_weight_max": 15.66298770904541,
      "activations/layer1_attention_weight_min": -14.662492752075195,
      "activations/layer20_attention_weight_max": 32.61404800415039,
      "activations/layer20_attention_weight_min": -26.455883026123047,
      "activations/layer21_attention_weight_max": 32.16373825073242,
      "activations/layer21_attention_weight_min": -24.782548904418945,
      "activations/layer22_attention_weight_max": 44.9952392578125,
      "activations/layer22_attention_weight_min": -29.66007423400879,
      "activations/layer23_attention_weight_max": 34.396541595458984,
      "activations/layer23_attention_weight_min": -24.4392147064209,
      "activations/layer2_attention_weight_max": 34.215614318847656,
      "activations/layer2_attention_weight_min": -32.86250305175781,
      "activations/layer3_attention_weight_max": 106.95548248291016,
      "activations/layer3_attention_weight_min": -106.777587890625,
      "activations/layer4_attention_weight_max": 110.91304016113281,
      "activations/layer4_attention_weight_min": -112.71895599365234,
      "activations/layer5_attention_weight_max": 50.38441467285156,
      "activations/layer5_attention_weight_min": -64.11050415039062,
      "activations/layer6_attention_weight_max": 45.97315216064453,
      "activations/layer6_attention_weight_min": -49.03697204589844,
      "activations/layer7_attention_weight_max": 97.29933166503906,
      "activations/layer7_attention_weight_min": -99.14545440673828,
      "activations/layer8_attention_weight_max": 45.343971252441406,
      "activations/layer8_attention_weight_min": -46.37876510620117,
      "activations/layer9_attention_weight_max": 36.654605865478516,
      "activations/layer9_attention_weight_min": -39.277767181396484,
      "epoch": 21.15,
      "learning_rate": 1.3744318181818181e-05,
      "loss": 2.7095,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.3945,
      "eval_samples_per_second": 511.525,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.3945,
      "eval_openwebtext_samples_per_second": 511.525,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9658,
      "eval_wikitext_samples_per_second": 231.97,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_lambada_loss": 2.310546875,
      "eval_lambada_ppl": 10.079935612761957,
      "eval_lambada_runtime": 9.5531,
      "eval_lambada_samples_per_second": 509.676,
      "step": 364000
    },
    {
      "activations/layer0_attention_weight_max": 17.698135375976562,
      "activations/layer0_attention_weight_min": -13.551713943481445,
      "activations/layer10_attention_weight_max": 36.311866760253906,
      "activations/layer10_attention_weight_min": -35.86881637573242,
      "activations/layer11_attention_weight_max": 35.24839782714844,
      "activations/layer11_attention_weight_min": -35.84404373168945,
      "activations/layer12_attention_weight_max": 26.16441535949707,
      "activations/layer12_attention_weight_min": -30.063386917114258,
      "activations/layer13_attention_weight_max": 40.222137451171875,
      "activations/layer13_attention_weight_min": -35.3923225402832,
      "activations/layer14_attention_weight_max": 40.34068298339844,
      "activations/layer14_attention_weight_min": -37.43205642700195,
      "activations/layer15_attention_weight_max": 39.69404602050781,
      "activations/layer15_attention_weight_min": -36.470542907714844,
      "activations/layer16_attention_weight_max": 28.27849769592285,
      "activations/layer16_attention_weight_min": -29.275222778320312,
      "activations/layer17_attention_weight_max": 30.54288673400879,
      "activations/layer17_attention_weight_min": -26.43303108215332,
      "activations/layer18_attention_weight_max": 35.338199615478516,
      "activations/layer18_attention_weight_min": -27.841238021850586,
      "activations/layer19_attention_weight_max": 32.541221618652344,
      "activations/layer19_attention_weight_min": -31.25579833984375,
      "activations/layer1_attention_weight_max": 16.223283767700195,
      "activations/layer1_attention_weight_min": -15.683435440063477,
      "activations/layer20_attention_weight_max": 28.6683349609375,
      "activations/layer20_attention_weight_min": -25.730222702026367,
      "activations/layer21_attention_weight_max": 27.485454559326172,
      "activations/layer21_attention_weight_min": -24.452922821044922,
      "activations/layer22_attention_weight_max": 37.173282623291016,
      "activations/layer22_attention_weight_min": -32.04471969604492,
      "activations/layer23_attention_weight_max": 29.03462791442871,
      "activations/layer23_attention_weight_min": -26.642621994018555,
      "activations/layer2_attention_weight_max": 32.78193283081055,
      "activations/layer2_attention_weight_min": -32.967140197753906,
      "activations/layer3_attention_weight_max": 102.29630279541016,
      "activations/layer3_attention_weight_min": -107.98593139648438,
      "activations/layer4_attention_weight_max": 109.43269348144531,
      "activations/layer4_attention_weight_min": -107.62788391113281,
      "activations/layer5_attention_weight_max": 49.80137252807617,
      "activations/layer5_attention_weight_min": -59.48324203491211,
      "activations/layer6_attention_weight_max": 43.713260650634766,
      "activations/layer6_attention_weight_min": -47.76740646362305,
      "activations/layer7_attention_weight_max": 93.47996520996094,
      "activations/layer7_attention_weight_min": -105.1598892211914,
      "activations/layer8_attention_weight_max": 42.514854431152344,
      "activations/layer8_attention_weight_min": -46.55721664428711,
      "activations/layer9_attention_weight_max": 33.0219841003418,
      "activations/layer9_attention_weight_min": -36.282012939453125,
      "epoch": 21.15,
      "learning_rate": 1.3725378787878786e-05,
      "loss": 2.7005,
      "step": 364050
    },
    {
      "activations/layer0_attention_weight_max": 16.658002853393555,
      "activations/layer0_attention_weight_min": -13.652664184570312,
      "activations/layer10_attention_weight_max": 34.28551483154297,
      "activations/layer10_attention_weight_min": -35.65419387817383,
      "activations/layer11_attention_weight_max": 36.41946029663086,
      "activations/layer11_attention_weight_min": -35.711647033691406,
      "activations/layer12_attention_weight_max": 27.874834060668945,
      "activations/layer12_attention_weight_min": -27.645814895629883,
      "activations/layer13_attention_weight_max": 39.8011474609375,
      "activations/layer13_attention_weight_min": -36.53418731689453,
      "activations/layer14_attention_weight_max": 45.76990509033203,
      "activations/layer14_attention_weight_min": -39.66812515258789,
      "activations/layer15_attention_weight_max": 44.150108337402344,
      "activations/layer15_attention_weight_min": -35.84708786010742,
      "activations/layer16_attention_weight_max": 31.592012405395508,
      "activations/layer16_attention_weight_min": -28.834075927734375,
      "activations/layer17_attention_weight_max": 33.16432571411133,
      "activations/layer17_attention_weight_min": -26.23941993713379,
      "activations/layer18_attention_weight_max": 35.32746124267578,
      "activations/layer18_attention_weight_min": -26.224084854125977,
      "activations/layer19_attention_weight_max": 38.57651901245117,
      "activations/layer19_attention_weight_min": -31.089847564697266,
      "activations/layer1_attention_weight_max": 16.363319396972656,
      "activations/layer1_attention_weight_min": -14.839367866516113,
      "activations/layer20_attention_weight_max": 30.342327117919922,
      "activations/layer20_attention_weight_min": -27.045581817626953,
      "activations/layer21_attention_weight_max": 29.66665267944336,
      "activations/layer21_attention_weight_min": -24.406694412231445,
      "activations/layer22_attention_weight_max": 41.698123931884766,
      "activations/layer22_attention_weight_min": -32.500572204589844,
      "activations/layer23_attention_weight_max": 33.25737380981445,
      "activations/layer23_attention_weight_min": -26.333393096923828,
      "activations/layer2_attention_weight_max": 32.283504486083984,
      "activations/layer2_attention_weight_min": -32.517295837402344,
      "activations/layer3_attention_weight_max": 102.09007263183594,
      "activations/layer3_attention_weight_min": -103.02473449707031,
      "activations/layer4_attention_weight_max": 106.26753234863281,
      "activations/layer4_attention_weight_min": -107.90319061279297,
      "activations/layer5_attention_weight_max": 52.78251266479492,
      "activations/layer5_attention_weight_min": -61.53370666503906,
      "activations/layer6_attention_weight_max": 44.35618591308594,
      "activations/layer6_attention_weight_min": -47.76132583618164,
      "activations/layer7_attention_weight_max": 103.58243560791016,
      "activations/layer7_attention_weight_min": -95.31777954101562,
      "activations/layer8_attention_weight_max": 43.66960144042969,
      "activations/layer8_attention_weight_min": -45.90687561035156,
      "activations/layer9_attention_weight_max": 34.2758903503418,
      "activations/layer9_attention_weight_min": -37.316768646240234,
      "epoch": 21.16,
      "learning_rate": 1.3706439393939392e-05,
      "loss": 2.7088,
      "step": 364100
    },
    {
      "activations/layer0_attention_weight_max": 16.72174072265625,
      "activations/layer0_attention_weight_min": -14.62555980682373,
      "activations/layer10_attention_weight_max": 33.431617736816406,
      "activations/layer10_attention_weight_min": -34.624061584472656,
      "activations/layer11_attention_weight_max": 32.6089973449707,
      "activations/layer11_attention_weight_min": -35.98097229003906,
      "activations/layer12_attention_weight_max": 25.682783126831055,
      "activations/layer12_attention_weight_min": -27.07830238342285,
      "activations/layer13_attention_weight_max": 36.82499694824219,
      "activations/layer13_attention_weight_min": -35.31418991088867,
      "activations/layer14_attention_weight_max": 40.31514358520508,
      "activations/layer14_attention_weight_min": -38.546043395996094,
      "activations/layer15_attention_weight_max": 37.1573600769043,
      "activations/layer15_attention_weight_min": -37.14216995239258,
      "activations/layer16_attention_weight_max": 27.266435623168945,
      "activations/layer16_attention_weight_min": -27.422454833984375,
      "activations/layer17_attention_weight_max": 29.456098556518555,
      "activations/layer17_attention_weight_min": -25.550031661987305,
      "activations/layer18_attention_weight_max": 31.655370712280273,
      "activations/layer18_attention_weight_min": -26.551082611083984,
      "activations/layer19_attention_weight_max": 33.128746032714844,
      "activations/layer19_attention_weight_min": -33.21408462524414,
      "activations/layer1_attention_weight_max": 15.846604347229004,
      "activations/layer1_attention_weight_min": -15.089433670043945,
      "activations/layer20_attention_weight_max": 27.957447052001953,
      "activations/layer20_attention_weight_min": -24.8809757232666,
      "activations/layer21_attention_weight_max": 27.50394058227539,
      "activations/layer21_attention_weight_min": -24.605199813842773,
      "activations/layer22_attention_weight_max": 40.0574836730957,
      "activations/layer22_attention_weight_min": -31.800106048583984,
      "activations/layer23_attention_weight_max": 28.52052879333496,
      "activations/layer23_attention_weight_min": -26.122852325439453,
      "activations/layer2_attention_weight_max": 32.47681427001953,
      "activations/layer2_attention_weight_min": -32.421260833740234,
      "activations/layer3_attention_weight_max": 102.61677551269531,
      "activations/layer3_attention_weight_min": -102.1945571899414,
      "activations/layer4_attention_weight_max": 104.71978759765625,
      "activations/layer4_attention_weight_min": -109.7081298828125,
      "activations/layer5_attention_weight_max": 49.07319641113281,
      "activations/layer5_attention_weight_min": -58.96196746826172,
      "activations/layer6_attention_weight_max": 44.14472198486328,
      "activations/layer6_attention_weight_min": -45.80367660522461,
      "activations/layer7_attention_weight_max": 91.74066925048828,
      "activations/layer7_attention_weight_min": -92.80071258544922,
      "activations/layer8_attention_weight_max": 42.0912971496582,
      "activations/layer8_attention_weight_min": -44.04788589477539,
      "activations/layer9_attention_weight_max": 31.872385025024414,
      "activations/layer9_attention_weight_min": -33.44913101196289,
      "epoch": 21.16,
      "learning_rate": 1.3687499999999999e-05,
      "loss": 2.7178,
      "step": 364150
    },
    {
      "activations/layer0_attention_weight_max": 17.386777877807617,
      "activations/layer0_attention_weight_min": -14.342480659484863,
      "activations/layer10_attention_weight_max": 32.292510986328125,
      "activations/layer10_attention_weight_min": -33.52126693725586,
      "activations/layer11_attention_weight_max": 35.7132682800293,
      "activations/layer11_attention_weight_min": -34.608863830566406,
      "activations/layer12_attention_weight_max": 28.21665382385254,
      "activations/layer12_attention_weight_min": -26.792844772338867,
      "activations/layer13_attention_weight_max": 39.11896514892578,
      "activations/layer13_attention_weight_min": -34.60866928100586,
      "activations/layer14_attention_weight_max": 42.652530670166016,
      "activations/layer14_attention_weight_min": -39.51820755004883,
      "activations/layer15_attention_weight_max": 39.933189392089844,
      "activations/layer15_attention_weight_min": -36.93839645385742,
      "activations/layer16_attention_weight_max": 29.42837142944336,
      "activations/layer16_attention_weight_min": -29.534543991088867,
      "activations/layer17_attention_weight_max": 31.105241775512695,
      "activations/layer17_attention_weight_min": -28.017322540283203,
      "activations/layer18_attention_weight_max": 32.376983642578125,
      "activations/layer18_attention_weight_min": -29.560401916503906,
      "activations/layer19_attention_weight_max": 31.624357223510742,
      "activations/layer19_attention_weight_min": -32.796722412109375,
      "activations/layer1_attention_weight_max": 15.757132530212402,
      "activations/layer1_attention_weight_min": -16.03935432434082,
      "activations/layer20_attention_weight_max": 26.809301376342773,
      "activations/layer20_attention_weight_min": -28.686012268066406,
      "activations/layer21_attention_weight_max": 25.116968154907227,
      "activations/layer21_attention_weight_min": -25.652923583984375,
      "activations/layer22_attention_weight_max": 37.01847457885742,
      "activations/layer22_attention_weight_min": -33.39107894897461,
      "activations/layer23_attention_weight_max": 29.82192611694336,
      "activations/layer23_attention_weight_min": -27.753948211669922,
      "activations/layer2_attention_weight_max": 33.180030822753906,
      "activations/layer2_attention_weight_min": -31.70808982849121,
      "activations/layer3_attention_weight_max": 103.78205108642578,
      "activations/layer3_attention_weight_min": -104.44173431396484,
      "activations/layer4_attention_weight_max": 107.5672607421875,
      "activations/layer4_attention_weight_min": -110.24279022216797,
      "activations/layer5_attention_weight_max": 48.53255844116211,
      "activations/layer5_attention_weight_min": -63.97608184814453,
      "activations/layer6_attention_weight_max": 45.14395523071289,
      "activations/layer6_attention_weight_min": -48.85063934326172,
      "activations/layer7_attention_weight_max": 95.072265625,
      "activations/layer7_attention_weight_min": -96.81233978271484,
      "activations/layer8_attention_weight_max": 42.22252655029297,
      "activations/layer8_attention_weight_min": -44.17033767700195,
      "activations/layer9_attention_weight_max": 30.684720993041992,
      "activations/layer9_attention_weight_min": -33.97378158569336,
      "epoch": 21.16,
      "learning_rate": 1.3668560606060604e-05,
      "loss": 2.7046,
      "step": 364200
    },
    {
      "activations/layer0_attention_weight_max": 16.696374893188477,
      "activations/layer0_attention_weight_min": -13.314224243164062,
      "activations/layer10_attention_weight_max": 34.03093719482422,
      "activations/layer10_attention_weight_min": -35.72047805786133,
      "activations/layer11_attention_weight_max": 34.154014587402344,
      "activations/layer11_attention_weight_min": -36.22804641723633,
      "activations/layer12_attention_weight_max": 27.215408325195312,
      "activations/layer12_attention_weight_min": -28.168106079101562,
      "activations/layer13_attention_weight_max": 41.666988372802734,
      "activations/layer13_attention_weight_min": -37.353694915771484,
      "activations/layer14_attention_weight_max": 42.4448356628418,
      "activations/layer14_attention_weight_min": -38.966331481933594,
      "activations/layer15_attention_weight_max": 39.23311996459961,
      "activations/layer15_attention_weight_min": -37.97168731689453,
      "activations/layer16_attention_weight_max": 30.253490447998047,
      "activations/layer16_attention_weight_min": -29.510759353637695,
      "activations/layer17_attention_weight_max": 32.55813217163086,
      "activations/layer17_attention_weight_min": -28.14603614807129,
      "activations/layer18_attention_weight_max": 34.081729888916016,
      "activations/layer18_attention_weight_min": -26.749021530151367,
      "activations/layer19_attention_weight_max": 36.94485092163086,
      "activations/layer19_attention_weight_min": -30.5874080657959,
      "activations/layer1_attention_weight_max": 15.102766990661621,
      "activations/layer1_attention_weight_min": -14.664576530456543,
      "activations/layer20_attention_weight_max": 31.023475646972656,
      "activations/layer20_attention_weight_min": -24.93235969543457,
      "activations/layer21_attention_weight_max": 30.01788330078125,
      "activations/layer21_attention_weight_min": -24.195499420166016,
      "activations/layer22_attention_weight_max": 44.92005920410156,
      "activations/layer22_attention_weight_min": -33.18225860595703,
      "activations/layer23_attention_weight_max": 32.6900634765625,
      "activations/layer23_attention_weight_min": -27.089336395263672,
      "activations/layer2_attention_weight_max": 33.88441467285156,
      "activations/layer2_attention_weight_min": -33.49009704589844,
      "activations/layer3_attention_weight_max": 103.03296661376953,
      "activations/layer3_attention_weight_min": -107.90221405029297,
      "activations/layer4_attention_weight_max": 107.87874603271484,
      "activations/layer4_attention_weight_min": -113.93120574951172,
      "activations/layer5_attention_weight_max": 49.39832305908203,
      "activations/layer5_attention_weight_min": -61.92177963256836,
      "activations/layer6_attention_weight_max": 45.469791412353516,
      "activations/layer6_attention_weight_min": -49.62586212158203,
      "activations/layer7_attention_weight_max": 98.37761688232422,
      "activations/layer7_attention_weight_min": -102.97660827636719,
      "activations/layer8_attention_weight_max": 41.330326080322266,
      "activations/layer8_attention_weight_min": -44.27426528930664,
      "activations/layer9_attention_weight_max": 32.56544494628906,
      "activations/layer9_attention_weight_min": -34.849056243896484,
      "epoch": 21.17,
      "learning_rate": 1.364962121212121e-05,
      "loss": 2.7235,
      "step": 364250
    },
    {
      "activations/layer0_attention_weight_max": 16.373790740966797,
      "activations/layer0_attention_weight_min": -13.49936580657959,
      "activations/layer10_attention_weight_max": 35.68449020385742,
      "activations/layer10_attention_weight_min": -41.65300750732422,
      "activations/layer11_attention_weight_max": 35.4830436706543,
      "activations/layer11_attention_weight_min": -39.74775314331055,
      "activations/layer12_attention_weight_max": 28.29023551940918,
      "activations/layer12_attention_weight_min": -28.873947143554688,
      "activations/layer13_attention_weight_max": 38.09645080566406,
      "activations/layer13_attention_weight_min": -36.205787658691406,
      "activations/layer14_attention_weight_max": 41.012718200683594,
      "activations/layer14_attention_weight_min": -39.59276580810547,
      "activations/layer15_attention_weight_max": 37.904518127441406,
      "activations/layer15_attention_weight_min": -37.34125518798828,
      "activations/layer16_attention_weight_max": 28.19878578186035,
      "activations/layer16_attention_weight_min": -29.19638442993164,
      "activations/layer17_attention_weight_max": 27.71877670288086,
      "activations/layer17_attention_weight_min": -26.515039443969727,
      "activations/layer18_attention_weight_max": 30.57805061340332,
      "activations/layer18_attention_weight_min": -26.55634117126465,
      "activations/layer19_attention_weight_max": 31.03262710571289,
      "activations/layer19_attention_weight_min": -31.032657623291016,
      "activations/layer1_attention_weight_max": 15.806448936462402,
      "activations/layer1_attention_weight_min": -15.30036735534668,
      "activations/layer20_attention_weight_max": 25.40727996826172,
      "activations/layer20_attention_weight_min": -25.33333969116211,
      "activations/layer21_attention_weight_max": 25.224979400634766,
      "activations/layer21_attention_weight_min": -27.830913543701172,
      "activations/layer22_attention_weight_max": 35.80266571044922,
      "activations/layer22_attention_weight_min": -33.93687057495117,
      "activations/layer23_attention_weight_max": 30.569156646728516,
      "activations/layer23_attention_weight_min": -27.503604888916016,
      "activations/layer2_attention_weight_max": 32.84869384765625,
      "activations/layer2_attention_weight_min": -33.75458526611328,
      "activations/layer3_attention_weight_max": 103.2166748046875,
      "activations/layer3_attention_weight_min": -105.20157623291016,
      "activations/layer4_attention_weight_max": 109.00292205810547,
      "activations/layer4_attention_weight_min": -111.66218566894531,
      "activations/layer5_attention_weight_max": 49.846717834472656,
      "activations/layer5_attention_weight_min": -60.53284454345703,
      "activations/layer6_attention_weight_max": 45.81131362915039,
      "activations/layer6_attention_weight_min": -48.63660430908203,
      "activations/layer7_attention_weight_max": 99.54402160644531,
      "activations/layer7_attention_weight_min": -113.74382019042969,
      "activations/layer8_attention_weight_max": 45.11083221435547,
      "activations/layer8_attention_weight_min": -53.591339111328125,
      "activations/layer9_attention_weight_max": 33.31686782836914,
      "activations/layer9_attention_weight_min": -39.536128997802734,
      "epoch": 21.17,
      "learning_rate": 1.3630681818181817e-05,
      "loss": 2.6941,
      "step": 364300
    },
    {
      "activations/layer0_attention_weight_max": 17.030324935913086,
      "activations/layer0_attention_weight_min": -13.743721961975098,
      "activations/layer10_attention_weight_max": 34.7381477355957,
      "activations/layer10_attention_weight_min": -36.356910705566406,
      "activations/layer11_attention_weight_max": 35.41398239135742,
      "activations/layer11_attention_weight_min": -35.53004455566406,
      "activations/layer12_attention_weight_max": 29.548397064208984,
      "activations/layer12_attention_weight_min": -28.33628273010254,
      "activations/layer13_attention_weight_max": 40.55810546875,
      "activations/layer13_attention_weight_min": -37.227317810058594,
      "activations/layer14_attention_weight_max": 40.48036193847656,
      "activations/layer14_attention_weight_min": -39.23183822631836,
      "activations/layer15_attention_weight_max": 39.383907318115234,
      "activations/layer15_attention_weight_min": -35.357566833496094,
      "activations/layer16_attention_weight_max": 28.110624313354492,
      "activations/layer16_attention_weight_min": -27.692371368408203,
      "activations/layer17_attention_weight_max": 28.653093338012695,
      "activations/layer17_attention_weight_min": -26.978628158569336,
      "activations/layer18_attention_weight_max": 29.74700355529785,
      "activations/layer18_attention_weight_min": -28.986549377441406,
      "activations/layer19_attention_weight_max": 32.14658737182617,
      "activations/layer19_attention_weight_min": -34.21903991699219,
      "activations/layer1_attention_weight_max": 15.145661354064941,
      "activations/layer1_attention_weight_min": -14.632285118103027,
      "activations/layer20_attention_weight_max": 27.966230392456055,
      "activations/layer20_attention_weight_min": -26.500797271728516,
      "activations/layer21_attention_weight_max": 27.924619674682617,
      "activations/layer21_attention_weight_min": -25.327133178710938,
      "activations/layer22_attention_weight_max": 38.77143096923828,
      "activations/layer22_attention_weight_min": -30.029470443725586,
      "activations/layer23_attention_weight_max": 30.9494686126709,
      "activations/layer23_attention_weight_min": -27.494430541992188,
      "activations/layer2_attention_weight_max": 32.35950469970703,
      "activations/layer2_attention_weight_min": -32.24622344970703,
      "activations/layer3_attention_weight_max": 101.32852935791016,
      "activations/layer3_attention_weight_min": -103.78398895263672,
      "activations/layer4_attention_weight_max": 107.50878143310547,
      "activations/layer4_attention_weight_min": -110.06168365478516,
      "activations/layer5_attention_weight_max": 49.38043975830078,
      "activations/layer5_attention_weight_min": -60.9052848815918,
      "activations/layer6_attention_weight_max": 44.73481750488281,
      "activations/layer6_attention_weight_min": -48.915653228759766,
      "activations/layer7_attention_weight_max": 95.41459655761719,
      "activations/layer7_attention_weight_min": -101.21345520019531,
      "activations/layer8_attention_weight_max": 41.7682991027832,
      "activations/layer8_attention_weight_min": -45.12290573120117,
      "activations/layer9_attention_weight_max": 32.45253372192383,
      "activations/layer9_attention_weight_min": -35.49397659301758,
      "epoch": 21.17,
      "learning_rate": 1.3611742424242422e-05,
      "loss": 2.7108,
      "step": 364350
    },
    {
      "activations/layer0_attention_weight_max": 16.313251495361328,
      "activations/layer0_attention_weight_min": -15.368820190429688,
      "activations/layer10_attention_weight_max": 32.5579719543457,
      "activations/layer10_attention_weight_min": -34.64789581298828,
      "activations/layer11_attention_weight_max": 33.3646125793457,
      "activations/layer11_attention_weight_min": -34.76612091064453,
      "activations/layer12_attention_weight_max": 27.175561904907227,
      "activations/layer12_attention_weight_min": -28.468666076660156,
      "activations/layer13_attention_weight_max": 38.491756439208984,
      "activations/layer13_attention_weight_min": -36.892723083496094,
      "activations/layer14_attention_weight_max": 40.3571891784668,
      "activations/layer14_attention_weight_min": -37.51206588745117,
      "activations/layer15_attention_weight_max": 37.46245193481445,
      "activations/layer15_attention_weight_min": -35.1585807800293,
      "activations/layer16_attention_weight_max": 34.51681137084961,
      "activations/layer16_attention_weight_min": -29.287185668945312,
      "activations/layer17_attention_weight_max": 27.96773338317871,
      "activations/layer17_attention_weight_min": -27.40815544128418,
      "activations/layer18_attention_weight_max": 30.983646392822266,
      "activations/layer18_attention_weight_min": -26.722732543945312,
      "activations/layer19_attention_weight_max": 40.0609130859375,
      "activations/layer19_attention_weight_min": -31.376768112182617,
      "activations/layer1_attention_weight_max": 16.282365798950195,
      "activations/layer1_attention_weight_min": -15.851428985595703,
      "activations/layer20_attention_weight_max": 30.544267654418945,
      "activations/layer20_attention_weight_min": -25.762365341186523,
      "activations/layer21_attention_weight_max": 28.87510108947754,
      "activations/layer21_attention_weight_min": -25.482666015625,
      "activations/layer22_attention_weight_max": 39.8699951171875,
      "activations/layer22_attention_weight_min": -32.12840270996094,
      "activations/layer23_attention_weight_max": 32.15668487548828,
      "activations/layer23_attention_weight_min": -25.342849731445312,
      "activations/layer2_attention_weight_max": 34.98212432861328,
      "activations/layer2_attention_weight_min": -32.351383209228516,
      "activations/layer3_attention_weight_max": 101.43692016601562,
      "activations/layer3_attention_weight_min": -102.38189697265625,
      "activations/layer4_attention_weight_max": 104.7811508178711,
      "activations/layer4_attention_weight_min": -106.52962493896484,
      "activations/layer5_attention_weight_max": 49.187530517578125,
      "activations/layer5_attention_weight_min": -61.65827941894531,
      "activations/layer6_attention_weight_max": 43.02786636352539,
      "activations/layer6_attention_weight_min": -47.300724029541016,
      "activations/layer7_attention_weight_max": 92.29899597167969,
      "activations/layer7_attention_weight_min": -94.40327453613281,
      "activations/layer8_attention_weight_max": 41.26258850097656,
      "activations/layer8_attention_weight_min": -45.63880920410156,
      "activations/layer9_attention_weight_max": 31.895536422729492,
      "activations/layer9_attention_weight_min": -33.24359130859375,
      "epoch": 21.17,
      "learning_rate": 1.3592803030303028e-05,
      "loss": 2.7102,
      "step": 364400
    },
    {
      "activations/layer0_attention_weight_max": 15.904044151306152,
      "activations/layer0_attention_weight_min": -13.833939552307129,
      "activations/layer10_attention_weight_max": 36.1442756652832,
      "activations/layer10_attention_weight_min": -37.962867736816406,
      "activations/layer11_attention_weight_max": 36.47751235961914,
      "activations/layer11_attention_weight_min": -35.622962951660156,
      "activations/layer12_attention_weight_max": 28.0604190826416,
      "activations/layer12_attention_weight_min": -26.15335464477539,
      "activations/layer13_attention_weight_max": 39.27248764038086,
      "activations/layer13_attention_weight_min": -35.35627365112305,
      "activations/layer14_attention_weight_max": 43.020416259765625,
      "activations/layer14_attention_weight_min": -38.66053009033203,
      "activations/layer15_attention_weight_max": 38.399349212646484,
      "activations/layer15_attention_weight_min": -36.00981903076172,
      "activations/layer16_attention_weight_max": 31.326915740966797,
      "activations/layer16_attention_weight_min": -28.691936492919922,
      "activations/layer17_attention_weight_max": 29.07399559020996,
      "activations/layer17_attention_weight_min": -25.373350143432617,
      "activations/layer18_attention_weight_max": 31.590415954589844,
      "activations/layer18_attention_weight_min": -26.045303344726562,
      "activations/layer19_attention_weight_max": 35.55434036254883,
      "activations/layer19_attention_weight_min": -31.943695068359375,
      "activations/layer1_attention_weight_max": 15.22479248046875,
      "activations/layer1_attention_weight_min": -14.373167991638184,
      "activations/layer20_attention_weight_max": 29.15471839904785,
      "activations/layer20_attention_weight_min": -25.9852237701416,
      "activations/layer21_attention_weight_max": 29.180400848388672,
      "activations/layer21_attention_weight_min": -24.65557098388672,
      "activations/layer22_attention_weight_max": 40.55525207519531,
      "activations/layer22_attention_weight_min": -30.15855598449707,
      "activations/layer23_attention_weight_max": 30.178451538085938,
      "activations/layer23_attention_weight_min": -25.19754409790039,
      "activations/layer2_attention_weight_max": 34.354515075683594,
      "activations/layer2_attention_weight_min": -32.83356475830078,
      "activations/layer3_attention_weight_max": 107.42240905761719,
      "activations/layer3_attention_weight_min": -106.86620330810547,
      "activations/layer4_attention_weight_max": 107.41629791259766,
      "activations/layer4_attention_weight_min": -112.59615325927734,
      "activations/layer5_attention_weight_max": 50.7266845703125,
      "activations/layer5_attention_weight_min": -63.26093673706055,
      "activations/layer6_attention_weight_max": 44.29428482055664,
      "activations/layer6_attention_weight_min": -48.30899429321289,
      "activations/layer7_attention_weight_max": 96.16337585449219,
      "activations/layer7_attention_weight_min": -102.87469482421875,
      "activations/layer8_attention_weight_max": 45.156681060791016,
      "activations/layer8_attention_weight_min": -47.865753173828125,
      "activations/layer9_attention_weight_max": 32.28402328491211,
      "activations/layer9_attention_weight_min": -34.573829650878906,
      "epoch": 21.18,
      "learning_rate": 1.3573863636363637e-05,
      "loss": 2.6971,
      "step": 364450
    },
    {
      "activations/layer0_attention_weight_max": 16.815824508666992,
      "activations/layer0_attention_weight_min": -13.971990585327148,
      "activations/layer10_attention_weight_max": 36.25872802734375,
      "activations/layer10_attention_weight_min": -36.98855972290039,
      "activations/layer11_attention_weight_max": 33.52736282348633,
      "activations/layer11_attention_weight_min": -36.26817321777344,
      "activations/layer12_attention_weight_max": 27.226648330688477,
      "activations/layer12_attention_weight_min": -28.243228912353516,
      "activations/layer13_attention_weight_max": 41.026180267333984,
      "activations/layer13_attention_weight_min": -38.12162780761719,
      "activations/layer14_attention_weight_max": 40.90427017211914,
      "activations/layer14_attention_weight_min": -41.04790115356445,
      "activations/layer15_attention_weight_max": 38.12547302246094,
      "activations/layer15_attention_weight_min": -36.47504806518555,
      "activations/layer16_attention_weight_max": 29.05183982849121,
      "activations/layer16_attention_weight_min": -28.23175048828125,
      "activations/layer17_attention_weight_max": 28.76193618774414,
      "activations/layer17_attention_weight_min": -26.203399658203125,
      "activations/layer18_attention_weight_max": 33.8255729675293,
      "activations/layer18_attention_weight_min": -25.239057540893555,
      "activations/layer19_attention_weight_max": 34.26142501831055,
      "activations/layer19_attention_weight_min": -30.672754287719727,
      "activations/layer1_attention_weight_max": 15.413384437561035,
      "activations/layer1_attention_weight_min": -15.355354309082031,
      "activations/layer20_attention_weight_max": 28.402963638305664,
      "activations/layer20_attention_weight_min": -25.461362838745117,
      "activations/layer21_attention_weight_max": 29.665983200073242,
      "activations/layer21_attention_weight_min": -26.06883430480957,
      "activations/layer22_attention_weight_max": 40.242549896240234,
      "activations/layer22_attention_weight_min": -30.46822738647461,
      "activations/layer23_attention_weight_max": 35.49456024169922,
      "activations/layer23_attention_weight_min": -25.595853805541992,
      "activations/layer2_attention_weight_max": 33.851802825927734,
      "activations/layer2_attention_weight_min": -33.14499282836914,
      "activations/layer3_attention_weight_max": 104.65913391113281,
      "activations/layer3_attention_weight_min": -105.22810363769531,
      "activations/layer4_attention_weight_max": 111.65535736083984,
      "activations/layer4_attention_weight_min": -108.12198638916016,
      "activations/layer5_attention_weight_max": 51.4641227722168,
      "activations/layer5_attention_weight_min": -59.891143798828125,
      "activations/layer6_attention_weight_max": 45.79988479614258,
      "activations/layer6_attention_weight_min": -50.84313201904297,
      "activations/layer7_attention_weight_max": 97.4415054321289,
      "activations/layer7_attention_weight_min": -103.69857788085938,
      "activations/layer8_attention_weight_max": 44.71377182006836,
      "activations/layer8_attention_weight_min": -46.64466857910156,
      "activations/layer9_attention_weight_max": 33.43498992919922,
      "activations/layer9_attention_weight_min": -36.575462341308594,
      "epoch": 21.18,
      "learning_rate": 1.3554924242424242e-05,
      "loss": 2.702,
      "step": 364500
    },
    {
      "activations/layer0_attention_weight_max": 17.516307830810547,
      "activations/layer0_attention_weight_min": -15.177960395812988,
      "activations/layer10_attention_weight_max": 40.43499755859375,
      "activations/layer10_attention_weight_min": -36.08831024169922,
      "activations/layer11_attention_weight_max": 40.55367660522461,
      "activations/layer11_attention_weight_min": -38.32487487792969,
      "activations/layer12_attention_weight_max": 35.21942138671875,
      "activations/layer12_attention_weight_min": -28.338647842407227,
      "activations/layer13_attention_weight_max": 47.66789245605469,
      "activations/layer13_attention_weight_min": -36.65129089355469,
      "activations/layer14_attention_weight_max": 44.51535415649414,
      "activations/layer14_attention_weight_min": -39.68216323852539,
      "activations/layer15_attention_weight_max": 41.9985466003418,
      "activations/layer15_attention_weight_min": -39.17340087890625,
      "activations/layer16_attention_weight_max": 30.157032012939453,
      "activations/layer16_attention_weight_min": -29.193988800048828,
      "activations/layer17_attention_weight_max": 30.99393081665039,
      "activations/layer17_attention_weight_min": -25.824201583862305,
      "activations/layer18_attention_weight_max": 33.490806579589844,
      "activations/layer18_attention_weight_min": -26.513423919677734,
      "activations/layer19_attention_weight_max": 35.69585037231445,
      "activations/layer19_attention_weight_min": -32.47492599487305,
      "activations/layer1_attention_weight_max": 15.108238220214844,
      "activations/layer1_attention_weight_min": -15.188201904296875,
      "activations/layer20_attention_weight_max": 31.024036407470703,
      "activations/layer20_attention_weight_min": -25.114538192749023,
      "activations/layer21_attention_weight_max": 30.820096969604492,
      "activations/layer21_attention_weight_min": -24.20647621154785,
      "activations/layer22_attention_weight_max": 40.37411880493164,
      "activations/layer22_attention_weight_min": -32.16423416137695,
      "activations/layer23_attention_weight_max": 34.948055267333984,
      "activations/layer23_attention_weight_min": -26.179229736328125,
      "activations/layer2_attention_weight_max": 34.93561553955078,
      "activations/layer2_attention_weight_min": -35.39131164550781,
      "activations/layer3_attention_weight_max": 106.15726470947266,
      "activations/layer3_attention_weight_min": -113.89419555664062,
      "activations/layer4_attention_weight_max": 111.111328125,
      "activations/layer4_attention_weight_min": -110.1381607055664,
      "activations/layer5_attention_weight_max": 52.275081634521484,
      "activations/layer5_attention_weight_min": -63.50576400756836,
      "activations/layer6_attention_weight_max": 45.0300407409668,
      "activations/layer6_attention_weight_min": -47.94688415527344,
      "activations/layer7_attention_weight_max": 110.0420150756836,
      "activations/layer7_attention_weight_min": -97.63179016113281,
      "activations/layer8_attention_weight_max": 48.06964111328125,
      "activations/layer8_attention_weight_min": -46.626914978027344,
      "activations/layer9_attention_weight_max": 35.13551330566406,
      "activations/layer9_attention_weight_min": -36.011226654052734,
      "epoch": 21.18,
      "learning_rate": 1.3535984848484848e-05,
      "loss": 2.7111,
      "step": 364550
    },
    {
      "activations/layer0_attention_weight_max": 16.060516357421875,
      "activations/layer0_attention_weight_min": -13.448060035705566,
      "activations/layer10_attention_weight_max": 40.63531494140625,
      "activations/layer10_attention_weight_min": -35.7572135925293,
      "activations/layer11_attention_weight_max": 38.444305419921875,
      "activations/layer11_attention_weight_min": -36.18609619140625,
      "activations/layer12_attention_weight_max": 26.007850646972656,
      "activations/layer12_attention_weight_min": -27.186464309692383,
      "activations/layer13_attention_weight_max": 37.95753479003906,
      "activations/layer13_attention_weight_min": -37.064754486083984,
      "activations/layer14_attention_weight_max": 43.34904861450195,
      "activations/layer14_attention_weight_min": -40.44746398925781,
      "activations/layer15_attention_weight_max": 40.75160217285156,
      "activations/layer15_attention_weight_min": -37.217323303222656,
      "activations/layer16_attention_weight_max": 28.87249183654785,
      "activations/layer16_attention_weight_min": -28.438825607299805,
      "activations/layer17_attention_weight_max": 30.20090103149414,
      "activations/layer17_attention_weight_min": -27.213563919067383,
      "activations/layer18_attention_weight_max": 34.42937088012695,
      "activations/layer18_attention_weight_min": -27.048439025878906,
      "activations/layer19_attention_weight_max": 38.2088623046875,
      "activations/layer19_attention_weight_min": -30.533512115478516,
      "activations/layer1_attention_weight_max": 15.776078224182129,
      "activations/layer1_attention_weight_min": -15.187809944152832,
      "activations/layer20_attention_weight_max": 32.239498138427734,
      "activations/layer20_attention_weight_min": -25.09749412536621,
      "activations/layer21_attention_weight_max": 32.81165313720703,
      "activations/layer21_attention_weight_min": -26.046106338500977,
      "activations/layer22_attention_weight_max": 41.768577575683594,
      "activations/layer22_attention_weight_min": -31.797143936157227,
      "activations/layer23_attention_weight_max": 33.44765090942383,
      "activations/layer23_attention_weight_min": -26.28748321533203,
      "activations/layer2_attention_weight_max": 35.07305145263672,
      "activations/layer2_attention_weight_min": -34.14479446411133,
      "activations/layer3_attention_weight_max": 106.174072265625,
      "activations/layer3_attention_weight_min": -108.34886932373047,
      "activations/layer4_attention_weight_max": 106.21630859375,
      "activations/layer4_attention_weight_min": -112.99291229248047,
      "activations/layer5_attention_weight_max": 49.93906784057617,
      "activations/layer5_attention_weight_min": -60.771263122558594,
      "activations/layer6_attention_weight_max": 45.16471862792969,
      "activations/layer6_attention_weight_min": -49.370948791503906,
      "activations/layer7_attention_weight_max": 96.52652740478516,
      "activations/layer7_attention_weight_min": -102.91992950439453,
      "activations/layer8_attention_weight_max": 44.697174072265625,
      "activations/layer8_attention_weight_min": -45.18879318237305,
      "activations/layer9_attention_weight_max": 32.683868408203125,
      "activations/layer9_attention_weight_min": -34.09880447387695,
      "epoch": 21.19,
      "learning_rate": 1.3517045454545455e-05,
      "loss": 2.7205,
      "step": 364600
    },
    {
      "activations/layer0_attention_weight_max": 17.63911247253418,
      "activations/layer0_attention_weight_min": -12.557170867919922,
      "activations/layer10_attention_weight_max": 40.455726623535156,
      "activations/layer10_attention_weight_min": -37.66750717163086,
      "activations/layer11_attention_weight_max": 37.03163146972656,
      "activations/layer11_attention_weight_min": -38.71306610107422,
      "activations/layer12_attention_weight_max": 32.13081359863281,
      "activations/layer12_attention_weight_min": -30.636539459228516,
      "activations/layer13_attention_weight_max": 43.23696517944336,
      "activations/layer13_attention_weight_min": -38.149288177490234,
      "activations/layer14_attention_weight_max": 46.25938034057617,
      "activations/layer14_attention_weight_min": -43.735626220703125,
      "activations/layer15_attention_weight_max": 45.373077392578125,
      "activations/layer15_attention_weight_min": -42.125667572021484,
      "activations/layer16_attention_weight_max": 33.60807418823242,
      "activations/layer16_attention_weight_min": -30.94197654724121,
      "activations/layer17_attention_weight_max": 35.364078521728516,
      "activations/layer17_attention_weight_min": -28.61590576171875,
      "activations/layer18_attention_weight_max": 35.78425598144531,
      "activations/layer18_attention_weight_min": -28.28045082092285,
      "activations/layer19_attention_weight_max": 39.775943756103516,
      "activations/layer19_attention_weight_min": -33.89406967163086,
      "activations/layer1_attention_weight_max": 15.659358024597168,
      "activations/layer1_attention_weight_min": -14.625819206237793,
      "activations/layer20_attention_weight_max": 33.32774353027344,
      "activations/layer20_attention_weight_min": -28.41640281677246,
      "activations/layer21_attention_weight_max": 35.50475311279297,
      "activations/layer21_attention_weight_min": -28.293582916259766,
      "activations/layer22_attention_weight_max": 42.05545425415039,
      "activations/layer22_attention_weight_min": -35.356197357177734,
      "activations/layer23_attention_weight_max": 33.16650390625,
      "activations/layer23_attention_weight_min": -29.736408233642578,
      "activations/layer2_attention_weight_max": 34.69575500488281,
      "activations/layer2_attention_weight_min": -33.95220947265625,
      "activations/layer3_attention_weight_max": 105.6230697631836,
      "activations/layer3_attention_weight_min": -106.4122085571289,
      "activations/layer4_attention_weight_max": 108.19329071044922,
      "activations/layer4_attention_weight_min": -113.44508361816406,
      "activations/layer5_attention_weight_max": 56.31865692138672,
      "activations/layer5_attention_weight_min": -64.715087890625,
      "activations/layer6_attention_weight_max": 46.17197036743164,
      "activations/layer6_attention_weight_min": -47.96171188354492,
      "activations/layer7_attention_weight_max": 103.60572814941406,
      "activations/layer7_attention_weight_min": -99.88197326660156,
      "activations/layer8_attention_weight_max": 47.71419143676758,
      "activations/layer8_attention_weight_min": -49.851131439208984,
      "activations/layer9_attention_weight_max": 33.866451263427734,
      "activations/layer9_attention_weight_min": -38.057498931884766,
      "epoch": 21.19,
      "learning_rate": 1.349810606060606e-05,
      "loss": 2.6961,
      "step": 364650
    },
    {
      "activations/layer0_attention_weight_max": 16.28485870361328,
      "activations/layer0_attention_weight_min": -14.099333763122559,
      "activations/layer10_attention_weight_max": 34.21619415283203,
      "activations/layer10_attention_weight_min": -35.6574821472168,
      "activations/layer11_attention_weight_max": 35.4622802734375,
      "activations/layer11_attention_weight_min": -35.67845916748047,
      "activations/layer12_attention_weight_max": 27.864900588989258,
      "activations/layer12_attention_weight_min": -27.13739776611328,
      "activations/layer13_attention_weight_max": 39.19381332397461,
      "activations/layer13_attention_weight_min": -36.07843017578125,
      "activations/layer14_attention_weight_max": 42.37493133544922,
      "activations/layer14_attention_weight_min": -39.820899963378906,
      "activations/layer15_attention_weight_max": 39.88069534301758,
      "activations/layer15_attention_weight_min": -37.653907775878906,
      "activations/layer16_attention_weight_max": 27.884700775146484,
      "activations/layer16_attention_weight_min": -29.516616821289062,
      "activations/layer17_attention_weight_max": 31.176347732543945,
      "activations/layer17_attention_weight_min": -27.306201934814453,
      "activations/layer18_attention_weight_max": 31.44228744506836,
      "activations/layer18_attention_weight_min": -26.3496150970459,
      "activations/layer19_attention_weight_max": 32.83588409423828,
      "activations/layer19_attention_weight_min": -32.72658157348633,
      "activations/layer1_attention_weight_max": 17.097578048706055,
      "activations/layer1_attention_weight_min": -15.498915672302246,
      "activations/layer20_attention_weight_max": 29.1114444732666,
      "activations/layer20_attention_weight_min": -25.882291793823242,
      "activations/layer21_attention_weight_max": 27.95702362060547,
      "activations/layer21_attention_weight_min": -24.93674087524414,
      "activations/layer22_attention_weight_max": 37.4954719543457,
      "activations/layer22_attention_weight_min": -31.079524993896484,
      "activations/layer23_attention_weight_max": 30.263357162475586,
      "activations/layer23_attention_weight_min": -26.670703887939453,
      "activations/layer2_attention_weight_max": 34.105506896972656,
      "activations/layer2_attention_weight_min": -34.218544006347656,
      "activations/layer3_attention_weight_max": 105.86824798583984,
      "activations/layer3_attention_weight_min": -107.04874420166016,
      "activations/layer4_attention_weight_max": 113.636474609375,
      "activations/layer4_attention_weight_min": -114.09545135498047,
      "activations/layer5_attention_weight_max": 52.51856231689453,
      "activations/layer5_attention_weight_min": -62.903289794921875,
      "activations/layer6_attention_weight_max": 44.87580490112305,
      "activations/layer6_attention_weight_min": -51.69135284423828,
      "activations/layer7_attention_weight_max": 98.17820739746094,
      "activations/layer7_attention_weight_min": -101.77464294433594,
      "activations/layer8_attention_weight_max": 42.581722259521484,
      "activations/layer8_attention_weight_min": -45.00065612792969,
      "activations/layer9_attention_weight_max": 31.67474937438965,
      "activations/layer9_attention_weight_min": -34.84351348876953,
      "epoch": 21.19,
      "learning_rate": 1.3479166666666666e-05,
      "loss": 2.701,
      "step": 364700
    },
    {
      "activations/layer0_attention_weight_max": 16.845149993896484,
      "activations/layer0_attention_weight_min": -15.913053512573242,
      "activations/layer10_attention_weight_max": 35.20248031616211,
      "activations/layer10_attention_weight_min": -36.49184036254883,
      "activations/layer11_attention_weight_max": 34.69169235229492,
      "activations/layer11_attention_weight_min": -34.291595458984375,
      "activations/layer12_attention_weight_max": 27.252784729003906,
      "activations/layer12_attention_weight_min": -27.370285034179688,
      "activations/layer13_attention_weight_max": 39.5577278137207,
      "activations/layer13_attention_weight_min": -36.90515899658203,
      "activations/layer14_attention_weight_max": 44.29132843017578,
      "activations/layer14_attention_weight_min": -39.6297492980957,
      "activations/layer15_attention_weight_max": 39.93445587158203,
      "activations/layer15_attention_weight_min": -36.93473434448242,
      "activations/layer16_attention_weight_max": 28.598098754882812,
      "activations/layer16_attention_weight_min": -30.34065818786621,
      "activations/layer17_attention_weight_max": 30.218149185180664,
      "activations/layer17_attention_weight_min": -27.622846603393555,
      "activations/layer18_attention_weight_max": 34.832069396972656,
      "activations/layer18_attention_weight_min": -25.624446868896484,
      "activations/layer19_attention_weight_max": 36.87924575805664,
      "activations/layer19_attention_weight_min": -31.0150146484375,
      "activations/layer1_attention_weight_max": 16.048683166503906,
      "activations/layer1_attention_weight_min": -14.317227363586426,
      "activations/layer20_attention_weight_max": 28.60138511657715,
      "activations/layer20_attention_weight_min": -24.958799362182617,
      "activations/layer21_attention_weight_max": 27.60607147216797,
      "activations/layer21_attention_weight_min": -24.302886962890625,
      "activations/layer22_attention_weight_max": 45.122947692871094,
      "activations/layer22_attention_weight_min": -31.910717010498047,
      "activations/layer23_attention_weight_max": 34.638267517089844,
      "activations/layer23_attention_weight_min": -28.315176010131836,
      "activations/layer2_attention_weight_max": 34.94792938232422,
      "activations/layer2_attention_weight_min": -34.25880432128906,
      "activations/layer3_attention_weight_max": 103.95813751220703,
      "activations/layer3_attention_weight_min": -107.99396514892578,
      "activations/layer4_attention_weight_max": 108.23128509521484,
      "activations/layer4_attention_weight_min": -114.53498840332031,
      "activations/layer5_attention_weight_max": 51.573890686035156,
      "activations/layer5_attention_weight_min": -63.276119232177734,
      "activations/layer6_attention_weight_max": 46.328800201416016,
      "activations/layer6_attention_weight_min": -50.78724670410156,
      "activations/layer7_attention_weight_max": 99.6985092163086,
      "activations/layer7_attention_weight_min": -109.26209259033203,
      "activations/layer8_attention_weight_max": 44.1246452331543,
      "activations/layer8_attention_weight_min": -49.11703872680664,
      "activations/layer9_attention_weight_max": 32.26557540893555,
      "activations/layer9_attention_weight_min": -35.725833892822266,
      "epoch": 21.19,
      "learning_rate": 1.3460227272727271e-05,
      "loss": 2.7048,
      "step": 364750
    },
    {
      "activations/layer0_attention_weight_max": 16.52117919921875,
      "activations/layer0_attention_weight_min": -13.093618392944336,
      "activations/layer10_attention_weight_max": 41.27393341064453,
      "activations/layer10_attention_weight_min": -36.06758499145508,
      "activations/layer11_attention_weight_max": 41.820045471191406,
      "activations/layer11_attention_weight_min": -36.65483093261719,
      "activations/layer12_attention_weight_max": 25.35211181640625,
      "activations/layer12_attention_weight_min": -27.390085220336914,
      "activations/layer13_attention_weight_max": 39.40696334838867,
      "activations/layer13_attention_weight_min": -36.29301071166992,
      "activations/layer14_attention_weight_max": 46.167789459228516,
      "activations/layer14_attention_weight_min": -39.40886688232422,
      "activations/layer15_attention_weight_max": 44.63225173950195,
      "activations/layer15_attention_weight_min": -42.182228088378906,
      "activations/layer16_attention_weight_max": 38.42572021484375,
      "activations/layer16_attention_weight_min": -31.195531845092773,
      "activations/layer17_attention_weight_max": 37.221920013427734,
      "activations/layer17_attention_weight_min": -30.860212326049805,
      "activations/layer18_attention_weight_max": 39.92901611328125,
      "activations/layer18_attention_weight_min": -31.84343147277832,
      "activations/layer19_attention_weight_max": 41.06332778930664,
      "activations/layer19_attention_weight_min": -35.12175369262695,
      "activations/layer1_attention_weight_max": 15.765812873840332,
      "activations/layer1_attention_weight_min": -14.52791976928711,
      "activations/layer20_attention_weight_max": 32.5598258972168,
      "activations/layer20_attention_weight_min": -26.453983306884766,
      "activations/layer21_attention_weight_max": 31.837852478027344,
      "activations/layer21_attention_weight_min": -25.459495544433594,
      "activations/layer22_attention_weight_max": 49.4251708984375,
      "activations/layer22_attention_weight_min": -34.35004425048828,
      "activations/layer23_attention_weight_max": 38.42485809326172,
      "activations/layer23_attention_weight_min": -28.753171920776367,
      "activations/layer2_attention_weight_max": 35.30352020263672,
      "activations/layer2_attention_weight_min": -34.99225616455078,
      "activations/layer3_attention_weight_max": 110.01902770996094,
      "activations/layer3_attention_weight_min": -110.41449737548828,
      "activations/layer4_attention_weight_max": 116.67559051513672,
      "activations/layer4_attention_weight_min": -110.57039642333984,
      "activations/layer5_attention_weight_max": 53.41428756713867,
      "activations/layer5_attention_weight_min": -63.25346374511719,
      "activations/layer6_attention_weight_max": 49.44990921020508,
      "activations/layer6_attention_weight_min": -48.4129753112793,
      "activations/layer7_attention_weight_max": 112.62163543701172,
      "activations/layer7_attention_weight_min": -107.34444427490234,
      "activations/layer8_attention_weight_max": 50.361019134521484,
      "activations/layer8_attention_weight_min": -49.56089782714844,
      "activations/layer9_attention_weight_max": 36.80603790283203,
      "activations/layer9_attention_weight_min": -38.130001068115234,
      "epoch": 21.2,
      "learning_rate": 1.3441287878787878e-05,
      "loss": 2.6925,
      "step": 364800
    },
    {
      "activations/layer0_attention_weight_max": 16.079315185546875,
      "activations/layer0_attention_weight_min": -13.961804389953613,
      "activations/layer10_attention_weight_max": 40.82954406738281,
      "activations/layer10_attention_weight_min": -37.875850677490234,
      "activations/layer11_attention_weight_max": 39.986778259277344,
      "activations/layer11_attention_weight_min": -39.29310607910156,
      "activations/layer12_attention_weight_max": 33.43780517578125,
      "activations/layer12_attention_weight_min": -32.882076263427734,
      "activations/layer13_attention_weight_max": 50.40129852294922,
      "activations/layer13_attention_weight_min": -44.19723892211914,
      "activations/layer14_attention_weight_max": 56.930641174316406,
      "activations/layer14_attention_weight_min": -48.0660400390625,
      "activations/layer15_attention_weight_max": 51.258480072021484,
      "activations/layer15_attention_weight_min": -44.314674377441406,
      "activations/layer16_attention_weight_max": 38.53712844848633,
      "activations/layer16_attention_weight_min": -29.487815856933594,
      "activations/layer17_attention_weight_max": 36.18205642700195,
      "activations/layer17_attention_weight_min": -28.342220306396484,
      "activations/layer18_attention_weight_max": 43.01057052612305,
      "activations/layer18_attention_weight_min": -30.184476852416992,
      "activations/layer19_attention_weight_max": 43.7667350769043,
      "activations/layer19_attention_weight_min": -32.706947326660156,
      "activations/layer1_attention_weight_max": 16.313507080078125,
      "activations/layer1_attention_weight_min": -14.610405921936035,
      "activations/layer20_attention_weight_max": 35.67924499511719,
      "activations/layer20_attention_weight_min": -24.985977172851562,
      "activations/layer21_attention_weight_max": 36.55188751220703,
      "activations/layer21_attention_weight_min": -23.90413475036621,
      "activations/layer22_attention_weight_max": 51.680049896240234,
      "activations/layer22_attention_weight_min": -34.87857437133789,
      "activations/layer23_attention_weight_max": 39.242454528808594,
      "activations/layer23_attention_weight_min": -25.977399826049805,
      "activations/layer2_attention_weight_max": 35.87207794189453,
      "activations/layer2_attention_weight_min": -33.75667190551758,
      "activations/layer3_attention_weight_max": 108.92652893066406,
      "activations/layer3_attention_weight_min": -111.712646484375,
      "activations/layer4_attention_weight_max": 115.36177062988281,
      "activations/layer4_attention_weight_min": -118.35169982910156,
      "activations/layer5_attention_weight_max": 53.03800582885742,
      "activations/layer5_attention_weight_min": -64.10906982421875,
      "activations/layer6_attention_weight_max": 49.677391052246094,
      "activations/layer6_attention_weight_min": -52.55862045288086,
      "activations/layer7_attention_weight_max": 102.29263305664062,
      "activations/layer7_attention_weight_min": -108.0152359008789,
      "activations/layer8_attention_weight_max": 47.594913482666016,
      "activations/layer8_attention_weight_min": -49.752017974853516,
      "activations/layer9_attention_weight_max": 36.84233093261719,
      "activations/layer9_attention_weight_min": -40.58403396606445,
      "epoch": 21.2,
      "learning_rate": 1.3422348484848484e-05,
      "loss": 2.712,
      "step": 364850
    },
    {
      "activations/layer0_attention_weight_max": 16.526060104370117,
      "activations/layer0_attention_weight_min": -14.063383102416992,
      "activations/layer10_attention_weight_max": 48.15199661254883,
      "activations/layer10_attention_weight_min": -46.75667190551758,
      "activations/layer11_attention_weight_max": 49.56049346923828,
      "activations/layer11_attention_weight_min": -46.19285583496094,
      "activations/layer12_attention_weight_max": 34.08802795410156,
      "activations/layer12_attention_weight_min": -29.773237228393555,
      "activations/layer13_attention_weight_max": 47.10100555419922,
      "activations/layer13_attention_weight_min": -42.215240478515625,
      "activations/layer14_attention_weight_max": 56.57530975341797,
      "activations/layer14_attention_weight_min": -45.908355712890625,
      "activations/layer15_attention_weight_max": 49.14217758178711,
      "activations/layer15_attention_weight_min": -44.49045944213867,
      "activations/layer16_attention_weight_max": 37.842201232910156,
      "activations/layer16_attention_weight_min": -27.629060745239258,
      "activations/layer17_attention_weight_max": 32.54338073730469,
      "activations/layer17_attention_weight_min": -28.298561096191406,
      "activations/layer18_attention_weight_max": 35.44728088378906,
      "activations/layer18_attention_weight_min": -26.734943389892578,
      "activations/layer19_attention_weight_max": 39.782291412353516,
      "activations/layer19_attention_weight_min": -31.814788818359375,
      "activations/layer1_attention_weight_max": 16.419513702392578,
      "activations/layer1_attention_weight_min": -14.049609184265137,
      "activations/layer20_attention_weight_max": 33.83677673339844,
      "activations/layer20_attention_weight_min": -26.933977127075195,
      "activations/layer21_attention_weight_max": 33.59440231323242,
      "activations/layer21_attention_weight_min": -25.94256591796875,
      "activations/layer22_attention_weight_max": 44.36914825439453,
      "activations/layer22_attention_weight_min": -33.14619064331055,
      "activations/layer23_attention_weight_max": 32.87085723876953,
      "activations/layer23_attention_weight_min": -26.51755142211914,
      "activations/layer2_attention_weight_max": 37.08677673339844,
      "activations/layer2_attention_weight_min": -34.89134216308594,
      "activations/layer3_attention_weight_max": 110.00390625,
      "activations/layer3_attention_weight_min": -110.07266235351562,
      "activations/layer4_attention_weight_max": 112.2763671875,
      "activations/layer4_attention_weight_min": -109.30179595947266,
      "activations/layer5_attention_weight_max": 51.96192169189453,
      "activations/layer5_attention_weight_min": -61.79425811767578,
      "activations/layer6_attention_weight_max": 48.93545150756836,
      "activations/layer6_attention_weight_min": -50.57880401611328,
      "activations/layer7_attention_weight_max": 117.30738830566406,
      "activations/layer7_attention_weight_min": -105.11908721923828,
      "activations/layer8_attention_weight_max": 52.63651657104492,
      "activations/layer8_attention_weight_min": -51.08549118041992,
      "activations/layer9_attention_weight_max": 41.33366394042969,
      "activations/layer9_attention_weight_min": -42.9932975769043,
      "epoch": 21.2,
      "learning_rate": 1.3403409090909089e-05,
      "loss": 2.7188,
      "step": 364900
    },
    {
      "activations/layer0_attention_weight_max": 17.58083724975586,
      "activations/layer0_attention_weight_min": -15.985550880432129,
      "activations/layer10_attention_weight_max": 36.43669128417969,
      "activations/layer10_attention_weight_min": -35.19150924682617,
      "activations/layer11_attention_weight_max": 35.8798942565918,
      "activations/layer11_attention_weight_min": -33.844295501708984,
      "activations/layer12_attention_weight_max": 26.62710189819336,
      "activations/layer12_attention_weight_min": -27.594987869262695,
      "activations/layer13_attention_weight_max": 38.18017578125,
      "activations/layer13_attention_weight_min": -35.35365295410156,
      "activations/layer14_attention_weight_max": 38.74620819091797,
      "activations/layer14_attention_weight_min": -39.251319885253906,
      "activations/layer15_attention_weight_max": 36.909584045410156,
      "activations/layer15_attention_weight_min": -35.900875091552734,
      "activations/layer16_attention_weight_max": 27.421777725219727,
      "activations/layer16_attention_weight_min": -29.087636947631836,
      "activations/layer17_attention_weight_max": 29.755977630615234,
      "activations/layer17_attention_weight_min": -26.160959243774414,
      "activations/layer18_attention_weight_max": 30.039592742919922,
      "activations/layer18_attention_weight_min": -27.183443069458008,
      "activations/layer19_attention_weight_max": 31.902318954467773,
      "activations/layer19_attention_weight_min": -32.56595993041992,
      "activations/layer1_attention_weight_max": 15.459434509277344,
      "activations/layer1_attention_weight_min": -15.750774383544922,
      "activations/layer20_attention_weight_max": 29.306310653686523,
      "activations/layer20_attention_weight_min": -25.959531784057617,
      "activations/layer21_attention_weight_max": 26.22035789489746,
      "activations/layer21_attention_weight_min": -25.53855323791504,
      "activations/layer22_attention_weight_max": 40.79082107543945,
      "activations/layer22_attention_weight_min": -32.23435592651367,
      "activations/layer23_attention_weight_max": 30.014251708984375,
      "activations/layer23_attention_weight_min": -26.271991729736328,
      "activations/layer2_attention_weight_max": 33.31339645385742,
      "activations/layer2_attention_weight_min": -32.79265594482422,
      "activations/layer3_attention_weight_max": 109.51896667480469,
      "activations/layer3_attention_weight_min": -105.79737091064453,
      "activations/layer4_attention_weight_max": 112.10383605957031,
      "activations/layer4_attention_weight_min": -110.37281799316406,
      "activations/layer5_attention_weight_max": 52.712379455566406,
      "activations/layer5_attention_weight_min": -59.29804611206055,
      "activations/layer6_attention_weight_max": 46.92784118652344,
      "activations/layer6_attention_weight_min": -48.45198440551758,
      "activations/layer7_attention_weight_max": 95.23373413085938,
      "activations/layer7_attention_weight_min": -101.22320556640625,
      "activations/layer8_attention_weight_max": 45.30128479003906,
      "activations/layer8_attention_weight_min": -46.37690353393555,
      "activations/layer9_attention_weight_max": 35.00678253173828,
      "activations/layer9_attention_weight_min": -35.319210052490234,
      "epoch": 21.21,
      "learning_rate": 1.3384469696969696e-05,
      "loss": 2.713,
      "step": 364950
    },
    {
      "activations/layer0_attention_weight_max": 15.784506797790527,
      "activations/layer0_attention_weight_min": -14.737369537353516,
      "activations/layer10_attention_weight_max": 39.01653289794922,
      "activations/layer10_attention_weight_min": -35.92634201049805,
      "activations/layer11_attention_weight_max": 38.51199722290039,
      "activations/layer11_attention_weight_min": -36.3828125,
      "activations/layer12_attention_weight_max": 27.61186981201172,
      "activations/layer12_attention_weight_min": -28.149450302124023,
      "activations/layer13_attention_weight_max": 41.9785270690918,
      "activations/layer13_attention_weight_min": -37.615440368652344,
      "activations/layer14_attention_weight_max": 49.51358413696289,
      "activations/layer14_attention_weight_min": -40.58461380004883,
      "activations/layer15_attention_weight_max": 45.76312255859375,
      "activations/layer15_attention_weight_min": -39.92873764038086,
      "activations/layer16_attention_weight_max": 35.04720687866211,
      "activations/layer16_attention_weight_min": -33.76679992675781,
      "activations/layer17_attention_weight_max": 34.57448196411133,
      "activations/layer17_attention_weight_min": -28.99040412902832,
      "activations/layer18_attention_weight_max": 35.69050216674805,
      "activations/layer18_attention_weight_min": -28.41639518737793,
      "activations/layer19_attention_weight_max": 37.81168746948242,
      "activations/layer19_attention_weight_min": -37.87945556640625,
      "activations/layer1_attention_weight_max": 15.505102157592773,
      "activations/layer1_attention_weight_min": -14.708597183227539,
      "activations/layer20_attention_weight_max": 31.356887817382812,
      "activations/layer20_attention_weight_min": -29.754777908325195,
      "activations/layer21_attention_weight_max": 30.811307907104492,
      "activations/layer21_attention_weight_min": -29.115097045898438,
      "activations/layer22_attention_weight_max": 50.719425201416016,
      "activations/layer22_attention_weight_min": -34.476375579833984,
      "activations/layer23_attention_weight_max": 35.803749084472656,
      "activations/layer23_attention_weight_min": -27.48590850830078,
      "activations/layer2_attention_weight_max": 35.05485916137695,
      "activations/layer2_attention_weight_min": -33.38948059082031,
      "activations/layer3_attention_weight_max": 106.60033416748047,
      "activations/layer3_attention_weight_min": -107.12303924560547,
      "activations/layer4_attention_weight_max": 110.94469451904297,
      "activations/layer4_attention_weight_min": -121.9281005859375,
      "activations/layer5_attention_weight_max": 52.019927978515625,
      "activations/layer5_attention_weight_min": -65.07408142089844,
      "activations/layer6_attention_weight_max": 47.45488357543945,
      "activations/layer6_attention_weight_min": -51.71449661254883,
      "activations/layer7_attention_weight_max": 105.6121826171875,
      "activations/layer7_attention_weight_min": -104.12544250488281,
      "activations/layer8_attention_weight_max": 46.75901794433594,
      "activations/layer8_attention_weight_min": -46.5876350402832,
      "activations/layer9_attention_weight_max": 32.83453369140625,
      "activations/layer9_attention_weight_min": -35.96575164794922,
      "epoch": 21.21,
      "learning_rate": 1.3365530303030302e-05,
      "loss": 2.7159,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4699,
      "eval_samples_per_second": 506.971,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4699,
      "eval_openwebtext_samples_per_second": 506.971,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9544,
      "eval_wikitext_samples_per_second": 233.325,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_lambada_loss": 2.291015625,
      "eval_lambada_ppl": 9.884972009450424,
      "eval_lambada_runtime": 9.5726,
      "eval_lambada_samples_per_second": 508.639,
      "step": 365000
    },
    {
      "activations/layer0_attention_weight_max": 17.379390716552734,
      "activations/layer0_attention_weight_min": -15.804166793823242,
      "activations/layer10_attention_weight_max": 34.3599739074707,
      "activations/layer10_attention_weight_min": -35.19099426269531,
      "activations/layer11_attention_weight_max": 36.46480941772461,
      "activations/layer11_attention_weight_min": -35.731414794921875,
      "activations/layer12_attention_weight_max": 30.60065460205078,
      "activations/layer12_attention_weight_min": -28.44829559326172,
      "activations/layer13_attention_weight_max": 43.52925109863281,
      "activations/layer13_attention_weight_min": -38.71864318847656,
      "activations/layer14_attention_weight_max": 48.78494644165039,
      "activations/layer14_attention_weight_min": -42.39045715332031,
      "activations/layer15_attention_weight_max": 49.28916549682617,
      "activations/layer15_attention_weight_min": -43.065006256103516,
      "activations/layer16_attention_weight_max": 34.13136291503906,
      "activations/layer16_attention_weight_min": -29.953771591186523,
      "activations/layer17_attention_weight_max": 35.5864372253418,
      "activations/layer17_attention_weight_min": -28.615991592407227,
      "activations/layer18_attention_weight_max": 36.09790802001953,
      "activations/layer18_attention_weight_min": -28.042123794555664,
      "activations/layer19_attention_weight_max": 37.3955192565918,
      "activations/layer19_attention_weight_min": -35.75078582763672,
      "activations/layer1_attention_weight_max": 15.651432037353516,
      "activations/layer1_attention_weight_min": -14.184025764465332,
      "activations/layer20_attention_weight_max": 31.505393981933594,
      "activations/layer20_attention_weight_min": -28.549842834472656,
      "activations/layer21_attention_weight_max": 32.57801055908203,
      "activations/layer21_attention_weight_min": -27.60751724243164,
      "activations/layer22_attention_weight_max": 53.6756706237793,
      "activations/layer22_attention_weight_min": -37.00661849975586,
      "activations/layer23_attention_weight_max": 34.78327941894531,
      "activations/layer23_attention_weight_min": -31.61353874206543,
      "activations/layer2_attention_weight_max": 34.36773681640625,
      "activations/layer2_attention_weight_min": -33.712379455566406,
      "activations/layer3_attention_weight_max": 108.33662414550781,
      "activations/layer3_attention_weight_min": -108.1319351196289,
      "activations/layer4_attention_weight_max": 108.42768859863281,
      "activations/layer4_attention_weight_min": -110.79676818847656,
      "activations/layer5_attention_weight_max": 50.860206604003906,
      "activations/layer5_attention_weight_min": -60.29085922241211,
      "activations/layer6_attention_weight_max": 43.77055358886719,
      "activations/layer6_attention_weight_min": -46.26314163208008,
      "activations/layer7_attention_weight_max": 98.03667449951172,
      "activations/layer7_attention_weight_min": -96.93738555908203,
      "activations/layer8_attention_weight_max": 43.94981002807617,
      "activations/layer8_attention_weight_min": -45.14271545410156,
      "activations/layer9_attention_weight_max": 32.74423599243164,
      "activations/layer9_attention_weight_min": -34.84164047241211,
      "epoch": 21.21,
      "learning_rate": 1.3346590909090907e-05,
      "loss": 2.7097,
      "step": 365050
    },
    {
      "activations/layer0_attention_weight_max": 17.19538688659668,
      "activations/layer0_attention_weight_min": -13.868107795715332,
      "activations/layer10_attention_weight_max": 46.186161041259766,
      "activations/layer10_attention_weight_min": -46.87648391723633,
      "activations/layer11_attention_weight_max": 48.72222900390625,
      "activations/layer11_attention_weight_min": -49.13080978393555,
      "activations/layer12_attention_weight_max": 38.75989532470703,
      "activations/layer12_attention_weight_min": -31.387096405029297,
      "activations/layer13_attention_weight_max": 56.78886795043945,
      "activations/layer13_attention_weight_min": -43.73875045776367,
      "activations/layer14_attention_weight_max": 56.83379364013672,
      "activations/layer14_attention_weight_min": -47.209754943847656,
      "activations/layer15_attention_weight_max": 60.869754791259766,
      "activations/layer15_attention_weight_min": -50.71985626220703,
      "activations/layer16_attention_weight_max": 39.11188507080078,
      "activations/layer16_attention_weight_min": -31.94898223876953,
      "activations/layer17_attention_weight_max": 34.639034271240234,
      "activations/layer17_attention_weight_min": -32.38747024536133,
      "activations/layer18_attention_weight_max": 34.315128326416016,
      "activations/layer18_attention_weight_min": -31.532602310180664,
      "activations/layer19_attention_weight_max": 40.7276725769043,
      "activations/layer19_attention_weight_min": -32.90470886230469,
      "activations/layer1_attention_weight_max": 16.17752456665039,
      "activations/layer1_attention_weight_min": -14.840458869934082,
      "activations/layer20_attention_weight_max": 35.16144943237305,
      "activations/layer20_attention_weight_min": -24.126523971557617,
      "activations/layer21_attention_weight_max": 31.036685943603516,
      "activations/layer21_attention_weight_min": -24.0772705078125,
      "activations/layer22_attention_weight_max": 49.97030258178711,
      "activations/layer22_attention_weight_min": -31.76321029663086,
      "activations/layer23_attention_weight_max": 32.541236877441406,
      "activations/layer23_attention_weight_min": -24.990859985351562,
      "activations/layer2_attention_weight_max": 36.64596939086914,
      "activations/layer2_attention_weight_min": -34.51148986816406,
      "activations/layer3_attention_weight_max": 110.67770385742188,
      "activations/layer3_attention_weight_min": -109.66755676269531,
      "activations/layer4_attention_weight_max": 111.89666748046875,
      "activations/layer4_attention_weight_min": -121.13768768310547,
      "activations/layer5_attention_weight_max": 51.248287200927734,
      "activations/layer5_attention_weight_min": -68.55241394042969,
      "activations/layer6_attention_weight_max": 49.37915802001953,
      "activations/layer6_attention_weight_min": -49.53657913208008,
      "activations/layer7_attention_weight_max": 108.20917510986328,
      "activations/layer7_attention_weight_min": -107.47791290283203,
      "activations/layer8_attention_weight_max": 50.23421859741211,
      "activations/layer8_attention_weight_min": -51.175533294677734,
      "activations/layer9_attention_weight_max": 41.01896667480469,
      "activations/layer9_attention_weight_min": -42.1475830078125,
      "epoch": 21.21,
      "learning_rate": 1.3327651515151514e-05,
      "loss": 2.7113,
      "step": 365100
    },
    {
      "activations/layer0_attention_weight_max": 16.089073181152344,
      "activations/layer0_attention_weight_min": -13.612907409667969,
      "activations/layer10_attention_weight_max": 36.482421875,
      "activations/layer10_attention_weight_min": -36.42758560180664,
      "activations/layer11_attention_weight_max": 36.59649658203125,
      "activations/layer11_attention_weight_min": -38.261112213134766,
      "activations/layer12_attention_weight_max": 28.364830017089844,
      "activations/layer12_attention_weight_min": -28.88524627685547,
      "activations/layer13_attention_weight_max": 42.447235107421875,
      "activations/layer13_attention_weight_min": -37.968528747558594,
      "activations/layer14_attention_weight_max": 43.90214538574219,
      "activations/layer14_attention_weight_min": -41.26186752319336,
      "activations/layer15_attention_weight_max": 38.96992111206055,
      "activations/layer15_attention_weight_min": -37.92069625854492,
      "activations/layer16_attention_weight_max": 31.087739944458008,
      "activations/layer16_attention_weight_min": -29.573415756225586,
      "activations/layer17_attention_weight_max": 32.4986457824707,
      "activations/layer17_attention_weight_min": -29.87864112854004,
      "activations/layer18_attention_weight_max": 33.25904846191406,
      "activations/layer18_attention_weight_min": -30.231462478637695,
      "activations/layer19_attention_weight_max": 35.96403121948242,
      "activations/layer19_attention_weight_min": -37.85665512084961,
      "activations/layer1_attention_weight_max": 15.826193809509277,
      "activations/layer1_attention_weight_min": -14.628052711486816,
      "activations/layer20_attention_weight_max": 31.336740493774414,
      "activations/layer20_attention_weight_min": -29.371335983276367,
      "activations/layer21_attention_weight_max": 29.542753219604492,
      "activations/layer21_attention_weight_min": -27.467985153198242,
      "activations/layer22_attention_weight_max": 42.65711975097656,
      "activations/layer22_attention_weight_min": -35.77400207519531,
      "activations/layer23_attention_weight_max": 35.279605865478516,
      "activations/layer23_attention_weight_min": -29.458545684814453,
      "activations/layer2_attention_weight_max": 32.26013946533203,
      "activations/layer2_attention_weight_min": -32.64936447143555,
      "activations/layer3_attention_weight_max": 106.3897933959961,
      "activations/layer3_attention_weight_min": -106.80398559570312,
      "activations/layer4_attention_weight_max": 110.7152328491211,
      "activations/layer4_attention_weight_min": -113.97332763671875,
      "activations/layer5_attention_weight_max": 54.037742614746094,
      "activations/layer5_attention_weight_min": -63.943477630615234,
      "activations/layer6_attention_weight_max": 47.671058654785156,
      "activations/layer6_attention_weight_min": -51.57816696166992,
      "activations/layer7_attention_weight_max": 102.63668823242188,
      "activations/layer7_attention_weight_min": -110.9654541015625,
      "activations/layer8_attention_weight_max": 44.89503479003906,
      "activations/layer8_attention_weight_min": -48.133785247802734,
      "activations/layer9_attention_weight_max": 35.61689758300781,
      "activations/layer9_attention_weight_min": -35.84136199951172,
      "epoch": 21.22,
      "learning_rate": 1.3308712121212119e-05,
      "loss": 2.6975,
      "step": 365150
    },
    {
      "activations/layer0_attention_weight_max": 17.15323257446289,
      "activations/layer0_attention_weight_min": -13.41660213470459,
      "activations/layer10_attention_weight_max": 46.75993347167969,
      "activations/layer10_attention_weight_min": -45.49542999267578,
      "activations/layer11_attention_weight_max": 47.37571334838867,
      "activations/layer11_attention_weight_min": -48.04610824584961,
      "activations/layer12_attention_weight_max": 29.322572708129883,
      "activations/layer12_attention_weight_min": -31.36034393310547,
      "activations/layer13_attention_weight_max": 42.620269775390625,
      "activations/layer13_attention_weight_min": -38.723506927490234,
      "activations/layer14_attention_weight_max": 43.76158142089844,
      "activations/layer14_attention_weight_min": -42.05400848388672,
      "activations/layer15_attention_weight_max": 43.9773063659668,
      "activations/layer15_attention_weight_min": -39.34605026245117,
      "activations/layer16_attention_weight_max": 32.43159866333008,
      "activations/layer16_attention_weight_min": -28.288375854492188,
      "activations/layer17_attention_weight_max": 30.76046371459961,
      "activations/layer17_attention_weight_min": -27.469438552856445,
      "activations/layer18_attention_weight_max": 31.930986404418945,
      "activations/layer18_attention_weight_min": -26.270614624023438,
      "activations/layer19_attention_weight_max": 33.311553955078125,
      "activations/layer19_attention_weight_min": -33.11861801147461,
      "activations/layer1_attention_weight_max": 15.660788536071777,
      "activations/layer1_attention_weight_min": -14.968205451965332,
      "activations/layer20_attention_weight_max": 28.92970848083496,
      "activations/layer20_attention_weight_min": -25.9058895111084,
      "activations/layer21_attention_weight_max": 27.818416595458984,
      "activations/layer21_attention_weight_min": -24.856216430664062,
      "activations/layer22_attention_weight_max": 38.36583709716797,
      "activations/layer22_attention_weight_min": -31.12908935546875,
      "activations/layer23_attention_weight_max": 29.243412017822266,
      "activations/layer23_attention_weight_min": -26.6377010345459,
      "activations/layer2_attention_weight_max": 32.79681396484375,
      "activations/layer2_attention_weight_min": -32.46611022949219,
      "activations/layer3_attention_weight_max": 100.38248443603516,
      "activations/layer3_attention_weight_min": -107.60427856445312,
      "activations/layer4_attention_weight_max": 112.0123291015625,
      "activations/layer4_attention_weight_min": -107.67242431640625,
      "activations/layer5_attention_weight_max": 56.55510711669922,
      "activations/layer5_attention_weight_min": -61.07218551635742,
      "activations/layer6_attention_weight_max": 48.89756393432617,
      "activations/layer6_attention_weight_min": -48.1313362121582,
      "activations/layer7_attention_weight_max": 116.22319793701172,
      "activations/layer7_attention_weight_min": -112.76447296142578,
      "activations/layer8_attention_weight_max": 56.204559326171875,
      "activations/layer8_attention_weight_min": -52.85193634033203,
      "activations/layer9_attention_weight_max": 38.07499694824219,
      "activations/layer9_attention_weight_min": -42.29862976074219,
      "epoch": 21.22,
      "learning_rate": 1.3289772727272725e-05,
      "loss": 2.7071,
      "step": 365200
    },
    {
      "activations/layer0_attention_weight_max": 16.06163787841797,
      "activations/layer0_attention_weight_min": -15.185161590576172,
      "activations/layer10_attention_weight_max": 35.222381591796875,
      "activations/layer10_attention_weight_min": -34.4129524230957,
      "activations/layer11_attention_weight_max": 35.38414764404297,
      "activations/layer11_attention_weight_min": -33.34992218017578,
      "activations/layer12_attention_weight_max": 27.43470001220703,
      "activations/layer12_attention_weight_min": -26.957778930664062,
      "activations/layer13_attention_weight_max": 42.46782302856445,
      "activations/layer13_attention_weight_min": -36.246456146240234,
      "activations/layer14_attention_weight_max": 47.08060836791992,
      "activations/layer14_attention_weight_min": -38.10471725463867,
      "activations/layer15_attention_weight_max": 41.76834487915039,
      "activations/layer15_attention_weight_min": -39.021114349365234,
      "activations/layer16_attention_weight_max": 32.00918960571289,
      "activations/layer16_attention_weight_min": -26.90845489501953,
      "activations/layer17_attention_weight_max": 29.017133712768555,
      "activations/layer17_attention_weight_min": -26.71552276611328,
      "activations/layer18_attention_weight_max": 31.178897857666016,
      "activations/layer18_attention_weight_min": -26.596805572509766,
      "activations/layer19_attention_weight_max": 32.333377838134766,
      "activations/layer19_attention_weight_min": -30.665761947631836,
      "activations/layer1_attention_weight_max": 16.443256378173828,
      "activations/layer1_attention_weight_min": -15.934531211853027,
      "activations/layer20_attention_weight_max": 27.959299087524414,
      "activations/layer20_attention_weight_min": -24.92210578918457,
      "activations/layer21_attention_weight_max": 26.866939544677734,
      "activations/layer21_attention_weight_min": -23.785045623779297,
      "activations/layer22_attention_weight_max": 41.48444366455078,
      "activations/layer22_attention_weight_min": -32.260520935058594,
      "activations/layer23_attention_weight_max": 30.0128231048584,
      "activations/layer23_attention_weight_min": -25.15222930908203,
      "activations/layer2_attention_weight_max": 33.23206329345703,
      "activations/layer2_attention_weight_min": -32.56842803955078,
      "activations/layer3_attention_weight_max": 100.80718994140625,
      "activations/layer3_attention_weight_min": -102.7790756225586,
      "activations/layer4_attention_weight_max": 105.61798095703125,
      "activations/layer4_attention_weight_min": -109.67176818847656,
      "activations/layer5_attention_weight_max": 51.75077819824219,
      "activations/layer5_attention_weight_min": -63.62936019897461,
      "activations/layer6_attention_weight_max": 44.033199310302734,
      "activations/layer6_attention_weight_min": -49.56138229370117,
      "activations/layer7_attention_weight_max": 98.33880615234375,
      "activations/layer7_attention_weight_min": -99.87871551513672,
      "activations/layer8_attention_weight_max": 41.94661331176758,
      "activations/layer8_attention_weight_min": -45.13176345825195,
      "activations/layer9_attention_weight_max": 30.650110244750977,
      "activations/layer9_attention_weight_min": -34.41655731201172,
      "epoch": 21.22,
      "learning_rate": 1.3270833333333332e-05,
      "loss": 2.7207,
      "step": 365250
    },
    {
      "activations/layer0_attention_weight_max": 15.457523345947266,
      "activations/layer0_attention_weight_min": -13.683074951171875,
      "activations/layer10_attention_weight_max": 36.4759521484375,
      "activations/layer10_attention_weight_min": -35.78850173950195,
      "activations/layer11_attention_weight_max": 35.55480194091797,
      "activations/layer11_attention_weight_min": -36.630592346191406,
      "activations/layer12_attention_weight_max": 29.226051330566406,
      "activations/layer12_attention_weight_min": -28.528715133666992,
      "activations/layer13_attention_weight_max": 43.70656967163086,
      "activations/layer13_attention_weight_min": -37.00758743286133,
      "activations/layer14_attention_weight_max": 51.659671783447266,
      "activations/layer14_attention_weight_min": -40.283592224121094,
      "activations/layer15_attention_weight_max": 49.46365737915039,
      "activations/layer15_attention_weight_min": -38.18169403076172,
      "activations/layer16_attention_weight_max": 31.77467918395996,
      "activations/layer16_attention_weight_min": -28.720375061035156,
      "activations/layer17_attention_weight_max": 31.90817642211914,
      "activations/layer17_attention_weight_min": -27.28179931640625,
      "activations/layer18_attention_weight_max": 30.79615592956543,
      "activations/layer18_attention_weight_min": -26.25965690612793,
      "activations/layer19_attention_weight_max": 34.341712951660156,
      "activations/layer19_attention_weight_min": -30.675418853759766,
      "activations/layer1_attention_weight_max": 15.537348747253418,
      "activations/layer1_attention_weight_min": -14.983909606933594,
      "activations/layer20_attention_weight_max": 30.393930435180664,
      "activations/layer20_attention_weight_min": -26.081037521362305,
      "activations/layer21_attention_weight_max": 30.55696678161621,
      "activations/layer21_attention_weight_min": -25.593719482421875,
      "activations/layer22_attention_weight_max": 40.48289489746094,
      "activations/layer22_attention_weight_min": -33.20388412475586,
      "activations/layer23_attention_weight_max": 30.04671287536621,
      "activations/layer23_attention_weight_min": -26.803424835205078,
      "activations/layer2_attention_weight_max": 34.432594299316406,
      "activations/layer2_attention_weight_min": -32.49171829223633,
      "activations/layer3_attention_weight_max": 104.7627944946289,
      "activations/layer3_attention_weight_min": -107.66409301757812,
      "activations/layer4_attention_weight_max": 110.46678924560547,
      "activations/layer4_attention_weight_min": -110.17899322509766,
      "activations/layer5_attention_weight_max": 50.935089111328125,
      "activations/layer5_attention_weight_min": -62.092308044433594,
      "activations/layer6_attention_weight_max": 45.236793518066406,
      "activations/layer6_attention_weight_min": -48.749122619628906,
      "activations/layer7_attention_weight_max": 98.44805908203125,
      "activations/layer7_attention_weight_min": -97.73556518554688,
      "activations/layer8_attention_weight_max": 42.43178176879883,
      "activations/layer8_attention_weight_min": -46.35007095336914,
      "activations/layer9_attention_weight_max": 32.02509689331055,
      "activations/layer9_attention_weight_min": -35.017120361328125,
      "epoch": 21.23,
      "learning_rate": 1.325189393939394e-05,
      "loss": 2.6996,
      "step": 365300
    },
    {
      "activations/layer0_attention_weight_max": 16.132322311401367,
      "activations/layer0_attention_weight_min": -14.525778770446777,
      "activations/layer10_attention_weight_max": 45.32945251464844,
      "activations/layer10_attention_weight_min": -40.55260467529297,
      "activations/layer11_attention_weight_max": 46.54722213745117,
      "activations/layer11_attention_weight_min": -41.8754997253418,
      "activations/layer12_attention_weight_max": 33.42258071899414,
      "activations/layer12_attention_weight_min": -29.744997024536133,
      "activations/layer13_attention_weight_max": 46.93290710449219,
      "activations/layer13_attention_weight_min": -38.23041534423828,
      "activations/layer14_attention_weight_max": 54.547325134277344,
      "activations/layer14_attention_weight_min": -42.688602447509766,
      "activations/layer15_attention_weight_max": 50.66263961791992,
      "activations/layer15_attention_weight_min": -41.7647819519043,
      "activations/layer16_attention_weight_max": 36.53718948364258,
      "activations/layer16_attention_weight_min": -28.191104888916016,
      "activations/layer17_attention_weight_max": 32.81339645385742,
      "activations/layer17_attention_weight_min": -27.88520622253418,
      "activations/layer18_attention_weight_max": 36.53928756713867,
      "activations/layer18_attention_weight_min": -28.140647888183594,
      "activations/layer19_attention_weight_max": 37.01655578613281,
      "activations/layer19_attention_weight_min": -32.65023422241211,
      "activations/layer1_attention_weight_max": 15.872840881347656,
      "activations/layer1_attention_weight_min": -15.388331413269043,
      "activations/layer20_attention_weight_max": 32.06685256958008,
      "activations/layer20_attention_weight_min": -25.436052322387695,
      "activations/layer21_attention_weight_max": 32.8304557800293,
      "activations/layer21_attention_weight_min": -24.652999877929688,
      "activations/layer22_attention_weight_max": 45.575504302978516,
      "activations/layer22_attention_weight_min": -37.116607666015625,
      "activations/layer23_attention_weight_max": 34.479312896728516,
      "activations/layer23_attention_weight_min": -30.273433685302734,
      "activations/layer2_attention_weight_max": 33.352508544921875,
      "activations/layer2_attention_weight_min": -32.406036376953125,
      "activations/layer3_attention_weight_max": 100.83744049072266,
      "activations/layer3_attention_weight_min": -102.64826202392578,
      "activations/layer4_attention_weight_max": 105.92738342285156,
      "activations/layer4_attention_weight_min": -109.83992004394531,
      "activations/layer5_attention_weight_max": 51.99144744873047,
      "activations/layer5_attention_weight_min": -62.413841247558594,
      "activations/layer6_attention_weight_max": 44.85564041137695,
      "activations/layer6_attention_weight_min": -48.38009262084961,
      "activations/layer7_attention_weight_max": 111.76985168457031,
      "activations/layer7_attention_weight_min": -103.47752380371094,
      "activations/layer8_attention_weight_max": 51.80724334716797,
      "activations/layer8_attention_weight_min": -46.75318908691406,
      "activations/layer9_attention_weight_max": 38.55119323730469,
      "activations/layer9_attention_weight_min": -40.189815521240234,
      "epoch": 21.23,
      "learning_rate": 1.3232954545454545e-05,
      "loss": 2.7041,
      "step": 365350
    },
    {
      "activations/layer0_attention_weight_max": 16.2448787689209,
      "activations/layer0_attention_weight_min": -13.804357528686523,
      "activations/layer10_attention_weight_max": 35.19399642944336,
      "activations/layer10_attention_weight_min": -35.412593841552734,
      "activations/layer11_attention_weight_max": 34.487789154052734,
      "activations/layer11_attention_weight_min": -34.672691345214844,
      "activations/layer12_attention_weight_max": 27.684322357177734,
      "activations/layer12_attention_weight_min": -27.505550384521484,
      "activations/layer13_attention_weight_max": 41.00741195678711,
      "activations/layer13_attention_weight_min": -37.0860710144043,
      "activations/layer14_attention_weight_max": 45.30317306518555,
      "activations/layer14_attention_weight_min": -38.032264709472656,
      "activations/layer15_attention_weight_max": 41.00825500488281,
      "activations/layer15_attention_weight_min": -35.77250671386719,
      "activations/layer16_attention_weight_max": 30.297134399414062,
      "activations/layer16_attention_weight_min": -31.148454666137695,
      "activations/layer17_attention_weight_max": 29.52704620361328,
      "activations/layer17_attention_weight_min": -28.3472957611084,
      "activations/layer18_attention_weight_max": 30.681034088134766,
      "activations/layer18_attention_weight_min": -27.847143173217773,
      "activations/layer19_attention_weight_max": 34.15434646606445,
      "activations/layer19_attention_weight_min": -32.68040466308594,
      "activations/layer1_attention_weight_max": 16.123130798339844,
      "activations/layer1_attention_weight_min": -15.327065467834473,
      "activations/layer20_attention_weight_max": 30.023193359375,
      "activations/layer20_attention_weight_min": -29.39595603942871,
      "activations/layer21_attention_weight_max": 28.63479232788086,
      "activations/layer21_attention_weight_min": -27.762754440307617,
      "activations/layer22_attention_weight_max": 44.21538543701172,
      "activations/layer22_attention_weight_min": -33.28567886352539,
      "activations/layer23_attention_weight_max": 34.80598449707031,
      "activations/layer23_attention_weight_min": -26.96501922607422,
      "activations/layer2_attention_weight_max": 33.9879264831543,
      "activations/layer2_attention_weight_min": -35.593505859375,
      "activations/layer3_attention_weight_max": 107.96631622314453,
      "activations/layer3_attention_weight_min": -106.74465942382812,
      "activations/layer4_attention_weight_max": 111.09797668457031,
      "activations/layer4_attention_weight_min": -113.2910385131836,
      "activations/layer5_attention_weight_max": 50.451629638671875,
      "activations/layer5_attention_weight_min": -63.373313903808594,
      "activations/layer6_attention_weight_max": 46.644771575927734,
      "activations/layer6_attention_weight_min": -47.60861587524414,
      "activations/layer7_attention_weight_max": 99.20948028564453,
      "activations/layer7_attention_weight_min": -100.30299377441406,
      "activations/layer8_attention_weight_max": 42.93656921386719,
      "activations/layer8_attention_weight_min": -46.08946990966797,
      "activations/layer9_attention_weight_max": 32.3503532409668,
      "activations/layer9_attention_weight_min": -35.291015625,
      "epoch": 21.23,
      "learning_rate": 1.3214015151515152e-05,
      "loss": 2.7158,
      "step": 365400
    },
    {
      "activations/layer0_attention_weight_max": 16.34208106994629,
      "activations/layer0_attention_weight_min": -14.21624755859375,
      "activations/layer10_attention_weight_max": 36.5572509765625,
      "activations/layer10_attention_weight_min": -38.08448028564453,
      "activations/layer11_attention_weight_max": 36.805179595947266,
      "activations/layer11_attention_weight_min": -38.36299133300781,
      "activations/layer12_attention_weight_max": 29.16796112060547,
      "activations/layer12_attention_weight_min": -28.041866302490234,
      "activations/layer13_attention_weight_max": 41.972068786621094,
      "activations/layer13_attention_weight_min": -38.07243347167969,
      "activations/layer14_attention_weight_max": 45.19117736816406,
      "activations/layer14_attention_weight_min": -39.52507019042969,
      "activations/layer15_attention_weight_max": 41.441932678222656,
      "activations/layer15_attention_weight_min": -38.588600158691406,
      "activations/layer16_attention_weight_max": 29.466081619262695,
      "activations/layer16_attention_weight_min": -29.015918731689453,
      "activations/layer17_attention_weight_max": 29.99456787109375,
      "activations/layer17_attention_weight_min": -28.53700065612793,
      "activations/layer18_attention_weight_max": 32.6994743347168,
      "activations/layer18_attention_weight_min": -27.594467163085938,
      "activations/layer19_attention_weight_max": 34.54665756225586,
      "activations/layer19_attention_weight_min": -33.48268127441406,
      "activations/layer1_attention_weight_max": 15.277502059936523,
      "activations/layer1_attention_weight_min": -15.344069480895996,
      "activations/layer20_attention_weight_max": 28.593538284301758,
      "activations/layer20_attention_weight_min": -29.015789031982422,
      "activations/layer21_attention_weight_max": 28.35097885131836,
      "activations/layer21_attention_weight_min": -27.806583404541016,
      "activations/layer22_attention_weight_max": 41.84222412109375,
      "activations/layer22_attention_weight_min": -35.110740661621094,
      "activations/layer23_attention_weight_max": 33.45173263549805,
      "activations/layer23_attention_weight_min": -28.989849090576172,
      "activations/layer2_attention_weight_max": 32.734962463378906,
      "activations/layer2_attention_weight_min": -33.57487869262695,
      "activations/layer3_attention_weight_max": 105.54383087158203,
      "activations/layer3_attention_weight_min": -110.07904815673828,
      "activations/layer4_attention_weight_max": 109.43082427978516,
      "activations/layer4_attention_weight_min": -112.1241455078125,
      "activations/layer5_attention_weight_max": 48.47889709472656,
      "activations/layer5_attention_weight_min": -64.26435852050781,
      "activations/layer6_attention_weight_max": 46.80266571044922,
      "activations/layer6_attention_weight_min": -48.87061309814453,
      "activations/layer7_attention_weight_max": 100.3042221069336,
      "activations/layer7_attention_weight_min": -102.30553436279297,
      "activations/layer8_attention_weight_max": 46.25955581665039,
      "activations/layer8_attention_weight_min": -49.630470275878906,
      "activations/layer9_attention_weight_max": 34.5739631652832,
      "activations/layer9_attention_weight_min": -36.373538970947266,
      "epoch": 21.23,
      "learning_rate": 1.3195075757575756e-05,
      "loss": 2.7069,
      "step": 365450
    },
    {
      "activations/layer0_attention_weight_max": 16.0497989654541,
      "activations/layer0_attention_weight_min": -15.244171142578125,
      "activations/layer10_attention_weight_max": 37.30704879760742,
      "activations/layer10_attention_weight_min": -35.28481674194336,
      "activations/layer11_attention_weight_max": 35.75651550292969,
      "activations/layer11_attention_weight_min": -35.924652099609375,
      "activations/layer12_attention_weight_max": 30.465051651000977,
      "activations/layer12_attention_weight_min": -27.72233772277832,
      "activations/layer13_attention_weight_max": 44.556053161621094,
      "activations/layer13_attention_weight_min": -37.67527770996094,
      "activations/layer14_attention_weight_max": 48.548683166503906,
      "activations/layer14_attention_weight_min": -39.2645149230957,
      "activations/layer15_attention_weight_max": 46.02645492553711,
      "activations/layer15_attention_weight_min": -38.76234436035156,
      "activations/layer16_attention_weight_max": 32.31412887573242,
      "activations/layer16_attention_weight_min": -28.15178108215332,
      "activations/layer17_attention_weight_max": 31.77937889099121,
      "activations/layer17_attention_weight_min": -26.54606056213379,
      "activations/layer18_attention_weight_max": 33.23387145996094,
      "activations/layer18_attention_weight_min": -25.369075775146484,
      "activations/layer19_attention_weight_max": 34.515419006347656,
      "activations/layer19_attention_weight_min": -31.256744384765625,
      "activations/layer1_attention_weight_max": 16.377111434936523,
      "activations/layer1_attention_weight_min": -14.418478012084961,
      "activations/layer20_attention_weight_max": 30.14248275756836,
      "activations/layer20_attention_weight_min": -27.173192977905273,
      "activations/layer21_attention_weight_max": 31.13193130493164,
      "activations/layer21_attention_weight_min": -25.082828521728516,
      "activations/layer22_attention_weight_max": 45.8007926940918,
      "activations/layer22_attention_weight_min": -33.96482467651367,
      "activations/layer23_attention_weight_max": 33.6922607421875,
      "activations/layer23_attention_weight_min": -29.584348678588867,
      "activations/layer2_attention_weight_max": 32.066322326660156,
      "activations/layer2_attention_weight_min": -32.31372833251953,
      "activations/layer3_attention_weight_max": 103.20521545410156,
      "activations/layer3_attention_weight_min": -105.66954803466797,
      "activations/layer4_attention_weight_max": 104.31103515625,
      "activations/layer4_attention_weight_min": -109.3512954711914,
      "activations/layer5_attention_weight_max": 49.440406799316406,
      "activations/layer5_attention_weight_min": -59.52523422241211,
      "activations/layer6_attention_weight_max": 44.68826675415039,
      "activations/layer6_attention_weight_min": -47.45518112182617,
      "activations/layer7_attention_weight_max": 100.7197036743164,
      "activations/layer7_attention_weight_min": -96.94953155517578,
      "activations/layer8_attention_weight_max": 43.20256805419922,
      "activations/layer8_attention_weight_min": -45.811283111572266,
      "activations/layer9_attention_weight_max": 34.21480178833008,
      "activations/layer9_attention_weight_min": -33.683006286621094,
      "epoch": 21.24,
      "learning_rate": 1.3176136363636363e-05,
      "loss": 2.7,
      "step": 365500
    },
    {
      "activations/layer0_attention_weight_max": 16.92026710510254,
      "activations/layer0_attention_weight_min": -14.344788551330566,
      "activations/layer10_attention_weight_max": 38.379981994628906,
      "activations/layer10_attention_weight_min": -36.11652374267578,
      "activations/layer11_attention_weight_max": 35.77654266357422,
      "activations/layer11_attention_weight_min": -35.3079833984375,
      "activations/layer12_attention_weight_max": 26.90150260925293,
      "activations/layer12_attention_weight_min": -27.59110450744629,
      "activations/layer13_attention_weight_max": 39.17369842529297,
      "activations/layer13_attention_weight_min": -36.54281997680664,
      "activations/layer14_attention_weight_max": 41.51443099975586,
      "activations/layer14_attention_weight_min": -42.853271484375,
      "activations/layer15_attention_weight_max": 37.66048049926758,
      "activations/layer15_attention_weight_min": -42.64560317993164,
      "activations/layer16_attention_weight_max": 30.069744110107422,
      "activations/layer16_attention_weight_min": -29.378421783447266,
      "activations/layer17_attention_weight_max": 30.99243927001953,
      "activations/layer17_attention_weight_min": -26.036476135253906,
      "activations/layer18_attention_weight_max": 32.929046630859375,
      "activations/layer18_attention_weight_min": -26.439868927001953,
      "activations/layer19_attention_weight_max": 35.83553695678711,
      "activations/layer19_attention_weight_min": -32.73141860961914,
      "activations/layer1_attention_weight_max": 15.556364059448242,
      "activations/layer1_attention_weight_min": -14.667973518371582,
      "activations/layer20_attention_weight_max": 29.03078842163086,
      "activations/layer20_attention_weight_min": -27.294483184814453,
      "activations/layer21_attention_weight_max": 29.273527145385742,
      "activations/layer21_attention_weight_min": -26.013717651367188,
      "activations/layer22_attention_weight_max": 41.15732955932617,
      "activations/layer22_attention_weight_min": -33.92543029785156,
      "activations/layer23_attention_weight_max": 33.78506088256836,
      "activations/layer23_attention_weight_min": -26.808719635009766,
      "activations/layer2_attention_weight_max": 34.66767120361328,
      "activations/layer2_attention_weight_min": -32.671409606933594,
      "activations/layer3_attention_weight_max": 102.06716918945312,
      "activations/layer3_attention_weight_min": -107.48416900634766,
      "activations/layer4_attention_weight_max": 105.87471771240234,
      "activations/layer4_attention_weight_min": -110.3939437866211,
      "activations/layer5_attention_weight_max": 51.91657638549805,
      "activations/layer5_attention_weight_min": -63.37611389160156,
      "activations/layer6_attention_weight_max": 45.44060134887695,
      "activations/layer6_attention_weight_min": -48.77426528930664,
      "activations/layer7_attention_weight_max": 93.30016326904297,
      "activations/layer7_attention_weight_min": -102.54222106933594,
      "activations/layer8_attention_weight_max": 43.81779479980469,
      "activations/layer8_attention_weight_min": -45.34726333618164,
      "activations/layer9_attention_weight_max": 35.386573791503906,
      "activations/layer9_attention_weight_min": -34.8272819519043,
      "epoch": 21.24,
      "learning_rate": 1.315719696969697e-05,
      "loss": 2.719,
      "step": 365550
    },
    {
      "activations/layer0_attention_weight_max": 16.629287719726562,
      "activations/layer0_attention_weight_min": -12.893453598022461,
      "activations/layer10_attention_weight_max": 34.40062713623047,
      "activations/layer10_attention_weight_min": -33.64334487915039,
      "activations/layer11_attention_weight_max": 33.20509719848633,
      "activations/layer11_attention_weight_min": -34.91712951660156,
      "activations/layer12_attention_weight_max": 27.296724319458008,
      "activations/layer12_attention_weight_min": -27.25297737121582,
      "activations/layer13_attention_weight_max": 40.9487419128418,
      "activations/layer13_attention_weight_min": -35.70930480957031,
      "activations/layer14_attention_weight_max": 42.448551177978516,
      "activations/layer14_attention_weight_min": -37.62666320800781,
      "activations/layer15_attention_weight_max": 37.64198684692383,
      "activations/layer15_attention_weight_min": -37.143089294433594,
      "activations/layer16_attention_weight_max": 28.599760055541992,
      "activations/layer16_attention_weight_min": -27.08285903930664,
      "activations/layer17_attention_weight_max": 28.427228927612305,
      "activations/layer17_attention_weight_min": -25.292858123779297,
      "activations/layer18_attention_weight_max": 33.239864349365234,
      "activations/layer18_attention_weight_min": -25.792490005493164,
      "activations/layer19_attention_weight_max": 33.2998046875,
      "activations/layer19_attention_weight_min": -30.73539161682129,
      "activations/layer1_attention_weight_max": 16.288829803466797,
      "activations/layer1_attention_weight_min": -15.1586275100708,
      "activations/layer20_attention_weight_max": 28.38898277282715,
      "activations/layer20_attention_weight_min": -24.48076629638672,
      "activations/layer21_attention_weight_max": 29.42897605895996,
      "activations/layer21_attention_weight_min": -24.169353485107422,
      "activations/layer22_attention_weight_max": 42.4676513671875,
      "activations/layer22_attention_weight_min": -31.749454498291016,
      "activations/layer23_attention_weight_max": 30.785003662109375,
      "activations/layer23_attention_weight_min": -25.61764144897461,
      "activations/layer2_attention_weight_max": 32.51742172241211,
      "activations/layer2_attention_weight_min": -33.47721862792969,
      "activations/layer3_attention_weight_max": 102.3062744140625,
      "activations/layer3_attention_weight_min": -108.52099609375,
      "activations/layer4_attention_weight_max": 109.8870620727539,
      "activations/layer4_attention_weight_min": -108.5339584350586,
      "activations/layer5_attention_weight_max": 50.151607513427734,
      "activations/layer5_attention_weight_min": -60.25912094116211,
      "activations/layer6_attention_weight_max": 44.466453552246094,
      "activations/layer6_attention_weight_min": -47.55789566040039,
      "activations/layer7_attention_weight_max": 97.16570281982422,
      "activations/layer7_attention_weight_min": -100.18042755126953,
      "activations/layer8_attention_weight_max": 39.932037353515625,
      "activations/layer8_attention_weight_min": -42.54188919067383,
      "activations/layer9_attention_weight_max": 30.90390396118164,
      "activations/layer9_attention_weight_min": -33.87529373168945,
      "epoch": 21.24,
      "learning_rate": 1.3138257575757574e-05,
      "loss": 2.704,
      "step": 365600
    },
    {
      "activations/layer0_attention_weight_max": 17.42042350769043,
      "activations/layer0_attention_weight_min": -13.290331840515137,
      "activations/layer10_attention_weight_max": 35.2728271484375,
      "activations/layer10_attention_weight_min": -37.75602340698242,
      "activations/layer11_attention_weight_max": 36.25455093383789,
      "activations/layer11_attention_weight_min": -36.01976776123047,
      "activations/layer12_attention_weight_max": 29.85504913330078,
      "activations/layer12_attention_weight_min": -28.446744918823242,
      "activations/layer13_attention_weight_max": 41.00901412963867,
      "activations/layer13_attention_weight_min": -37.370887756347656,
      "activations/layer14_attention_weight_max": 45.32288360595703,
      "activations/layer14_attention_weight_min": -40.99588394165039,
      "activations/layer15_attention_weight_max": 42.125083923339844,
      "activations/layer15_attention_weight_min": -37.99106979370117,
      "activations/layer16_attention_weight_max": 31.166030883789062,
      "activations/layer16_attention_weight_min": -30.327774047851562,
      "activations/layer17_attention_weight_max": 31.36431312561035,
      "activations/layer17_attention_weight_min": -27.54854965209961,
      "activations/layer18_attention_weight_max": 34.16939926147461,
      "activations/layer18_attention_weight_min": -25.051454544067383,
      "activations/layer19_attention_weight_max": 36.00996780395508,
      "activations/layer19_attention_weight_min": -32.636512756347656,
      "activations/layer1_attention_weight_max": 16.053146362304688,
      "activations/layer1_attention_weight_min": -15.533164024353027,
      "activations/layer20_attention_weight_max": 30.194894790649414,
      "activations/layer20_attention_weight_min": -26.49267578125,
      "activations/layer21_attention_weight_max": 30.326000213623047,
      "activations/layer21_attention_weight_min": -26.859630584716797,
      "activations/layer22_attention_weight_max": 40.47706604003906,
      "activations/layer22_attention_weight_min": -33.160152435302734,
      "activations/layer23_attention_weight_max": 31.548444747924805,
      "activations/layer23_attention_weight_min": -27.71628189086914,
      "activations/layer2_attention_weight_max": 33.8863639831543,
      "activations/layer2_attention_weight_min": -32.905242919921875,
      "activations/layer3_attention_weight_max": 105.29811096191406,
      "activations/layer3_attention_weight_min": -104.29113006591797,
      "activations/layer4_attention_weight_max": 107.49044036865234,
      "activations/layer4_attention_weight_min": -110.28620910644531,
      "activations/layer5_attention_weight_max": 49.53559875488281,
      "activations/layer5_attention_weight_min": -64.665283203125,
      "activations/layer6_attention_weight_max": 46.61933135986328,
      "activations/layer6_attention_weight_min": -47.121952056884766,
      "activations/layer7_attention_weight_max": 93.34078216552734,
      "activations/layer7_attention_weight_min": -97.02391815185547,
      "activations/layer8_attention_weight_max": 42.054039001464844,
      "activations/layer8_attention_weight_min": -43.757904052734375,
      "activations/layer9_attention_weight_max": 32.771183013916016,
      "activations/layer9_attention_weight_min": -37.06968688964844,
      "epoch": 21.25,
      "learning_rate": 1.3119318181818181e-05,
      "loss": 2.7157,
      "step": 365650
    },
    {
      "activations/layer0_attention_weight_max": 17.161161422729492,
      "activations/layer0_attention_weight_min": -14.17301082611084,
      "activations/layer10_attention_weight_max": 33.47100067138672,
      "activations/layer10_attention_weight_min": -34.076438903808594,
      "activations/layer11_attention_weight_max": 33.56543731689453,
      "activations/layer11_attention_weight_min": -34.631309509277344,
      "activations/layer12_attention_weight_max": 25.19219970703125,
      "activations/layer12_attention_weight_min": -26.03104591369629,
      "activations/layer13_attention_weight_max": 38.164833068847656,
      "activations/layer13_attention_weight_min": -37.1627311706543,
      "activations/layer14_attention_weight_max": 42.87660598754883,
      "activations/layer14_attention_weight_min": -38.00843048095703,
      "activations/layer15_attention_weight_max": 39.958133697509766,
      "activations/layer15_attention_weight_min": -36.05854034423828,
      "activations/layer16_attention_weight_max": 31.559717178344727,
      "activations/layer16_attention_weight_min": -31.359081268310547,
      "activations/layer17_attention_weight_max": 29.9908390045166,
      "activations/layer17_attention_weight_min": -29.276697158813477,
      "activations/layer18_attention_weight_max": 30.562536239624023,
      "activations/layer18_attention_weight_min": -28.385892868041992,
      "activations/layer19_attention_weight_max": 40.812252044677734,
      "activations/layer19_attention_weight_min": -32.02676010131836,
      "activations/layer1_attention_weight_max": 15.41430377960205,
      "activations/layer1_attention_weight_min": -16.683399200439453,
      "activations/layer20_attention_weight_max": 33.289241790771484,
      "activations/layer20_attention_weight_min": -25.882362365722656,
      "activations/layer21_attention_weight_max": 31.790218353271484,
      "activations/layer21_attention_weight_min": -24.858346939086914,
      "activations/layer22_attention_weight_max": 42.85385513305664,
      "activations/layer22_attention_weight_min": -34.271018981933594,
      "activations/layer23_attention_weight_max": 34.3303337097168,
      "activations/layer23_attention_weight_min": -25.822134017944336,
      "activations/layer2_attention_weight_max": 33.18193817138672,
      "activations/layer2_attention_weight_min": -31.632551193237305,
      "activations/layer3_attention_weight_max": 104.4410400390625,
      "activations/layer3_attention_weight_min": -101.97405242919922,
      "activations/layer4_attention_weight_max": 107.02422332763672,
      "activations/layer4_attention_weight_min": -107.59561920166016,
      "activations/layer5_attention_weight_max": 51.27378845214844,
      "activations/layer5_attention_weight_min": -59.99971008300781,
      "activations/layer6_attention_weight_max": 44.19649887084961,
      "activations/layer6_attention_weight_min": -47.44646453857422,
      "activations/layer7_attention_weight_max": 91.49272918701172,
      "activations/layer7_attention_weight_min": -94.20716094970703,
      "activations/layer8_attention_weight_max": 40.54521942138672,
      "activations/layer8_attention_weight_min": -43.36362075805664,
      "activations/layer9_attention_weight_max": 34.51395034790039,
      "activations/layer9_attention_weight_min": -34.69782638549805,
      "epoch": 21.25,
      "learning_rate": 1.3100378787878788e-05,
      "loss": 2.7063,
      "step": 365700
    },
    {
      "activations/layer0_attention_weight_max": 16.373538970947266,
      "activations/layer0_attention_weight_min": -12.946603775024414,
      "activations/layer10_attention_weight_max": 35.555294036865234,
      "activations/layer10_attention_weight_min": -35.112098693847656,
      "activations/layer11_attention_weight_max": 35.935150146484375,
      "activations/layer11_attention_weight_min": -34.6046142578125,
      "activations/layer12_attention_weight_max": 27.74424934387207,
      "activations/layer12_attention_weight_min": -26.495588302612305,
      "activations/layer13_attention_weight_max": 37.09896469116211,
      "activations/layer13_attention_weight_min": -35.789005279541016,
      "activations/layer14_attention_weight_max": 42.14103698730469,
      "activations/layer14_attention_weight_min": -37.76100158691406,
      "activations/layer15_attention_weight_max": 41.12432861328125,
      "activations/layer15_attention_weight_min": -36.97126007080078,
      "activations/layer16_attention_weight_max": 29.55229949951172,
      "activations/layer16_attention_weight_min": -26.85845184326172,
      "activations/layer17_attention_weight_max": 28.726430892944336,
      "activations/layer17_attention_weight_min": -25.81988525390625,
      "activations/layer18_attention_weight_max": 32.38368606567383,
      "activations/layer18_attention_weight_min": -26.01198387145996,
      "activations/layer19_attention_weight_max": 31.064563751220703,
      "activations/layer19_attention_weight_min": -31.857751846313477,
      "activations/layer1_attention_weight_max": 15.946868896484375,
      "activations/layer1_attention_weight_min": -14.776721000671387,
      "activations/layer20_attention_weight_max": 25.29323959350586,
      "activations/layer20_attention_weight_min": -24.526084899902344,
      "activations/layer21_attention_weight_max": 26.81349754333496,
      "activations/layer21_attention_weight_min": -26.239187240600586,
      "activations/layer22_attention_weight_max": 38.63544464111328,
      "activations/layer22_attention_weight_min": -30.71778106689453,
      "activations/layer23_attention_weight_max": 29.80389404296875,
      "activations/layer23_attention_weight_min": -23.711002349853516,
      "activations/layer2_attention_weight_max": 33.8143196105957,
      "activations/layer2_attention_weight_min": -33.160003662109375,
      "activations/layer3_attention_weight_max": 101.86255645751953,
      "activations/layer3_attention_weight_min": -108.7066421508789,
      "activations/layer4_attention_weight_max": 107.01556396484375,
      "activations/layer4_attention_weight_min": -117.04786682128906,
      "activations/layer5_attention_weight_max": 50.667903900146484,
      "activations/layer5_attention_weight_min": -58.6499137878418,
      "activations/layer6_attention_weight_max": 43.92721176147461,
      "activations/layer6_attention_weight_min": -47.17560577392578,
      "activations/layer7_attention_weight_max": 97.06328582763672,
      "activations/layer7_attention_weight_min": -98.68986511230469,
      "activations/layer8_attention_weight_max": 42.72433090209961,
      "activations/layer8_attention_weight_min": -45.14525604248047,
      "activations/layer9_attention_weight_max": 34.96845626831055,
      "activations/layer9_attention_weight_min": -36.06977844238281,
      "epoch": 21.25,
      "learning_rate": 1.3081439393939392e-05,
      "loss": 2.7146,
      "step": 365750
    },
    {
      "activations/layer0_attention_weight_max": 16.28860092163086,
      "activations/layer0_attention_weight_min": -12.67456340789795,
      "activations/layer10_attention_weight_max": 36.45052719116211,
      "activations/layer10_attention_weight_min": -37.61526870727539,
      "activations/layer11_attention_weight_max": 37.74099349975586,
      "activations/layer11_attention_weight_min": -38.73094940185547,
      "activations/layer12_attention_weight_max": 28.962419509887695,
      "activations/layer12_attention_weight_min": -29.169675827026367,
      "activations/layer13_attention_weight_max": 42.84409713745117,
      "activations/layer13_attention_weight_min": -38.99830627441406,
      "activations/layer14_attention_weight_max": 42.758689880371094,
      "activations/layer14_attention_weight_min": -40.66409683227539,
      "activations/layer15_attention_weight_max": 39.129398345947266,
      "activations/layer15_attention_weight_min": -37.64690399169922,
      "activations/layer16_attention_weight_max": 29.07663917541504,
      "activations/layer16_attention_weight_min": -27.324132919311523,
      "activations/layer17_attention_weight_max": 32.5350227355957,
      "activations/layer17_attention_weight_min": -26.351991653442383,
      "activations/layer18_attention_weight_max": 32.62751007080078,
      "activations/layer18_attention_weight_min": -27.127946853637695,
      "activations/layer19_attention_weight_max": 36.319679260253906,
      "activations/layer19_attention_weight_min": -32.2807731628418,
      "activations/layer1_attention_weight_max": 15.561354637145996,
      "activations/layer1_attention_weight_min": -14.620001792907715,
      "activations/layer20_attention_weight_max": 32.65766906738281,
      "activations/layer20_attention_weight_min": -25.232805252075195,
      "activations/layer21_attention_weight_max": 33.36217498779297,
      "activations/layer21_attention_weight_min": -25.30629539489746,
      "activations/layer22_attention_weight_max": 46.496726989746094,
      "activations/layer22_attention_weight_min": -34.00554275512695,
      "activations/layer23_attention_weight_max": 32.57448196411133,
      "activations/layer23_attention_weight_min": -27.149433135986328,
      "activations/layer2_attention_weight_max": 34.00182342529297,
      "activations/layer2_attention_weight_min": -34.25300979614258,
      "activations/layer3_attention_weight_max": 105.4870834350586,
      "activations/layer3_attention_weight_min": -114.11072540283203,
      "activations/layer4_attention_weight_max": 108.94010162353516,
      "activations/layer4_attention_weight_min": -112.6251449584961,
      "activations/layer5_attention_weight_max": 50.3842887878418,
      "activations/layer5_attention_weight_min": -61.24770736694336,
      "activations/layer6_attention_weight_max": 46.1121711730957,
      "activations/layer6_attention_weight_min": -49.46867370605469,
      "activations/layer7_attention_weight_max": 100.46054077148438,
      "activations/layer7_attention_weight_min": -103.33860778808594,
      "activations/layer8_attention_weight_max": 43.07783889770508,
      "activations/layer8_attention_weight_min": -45.299434661865234,
      "activations/layer9_attention_weight_max": 35.24321746826172,
      "activations/layer9_attention_weight_min": -39.86479187011719,
      "epoch": 21.26,
      "learning_rate": 1.3062499999999999e-05,
      "loss": 2.7124,
      "step": 365800
    },
    {
      "activations/layer0_attention_weight_max": 16.2318172454834,
      "activations/layer0_attention_weight_min": -14.774385452270508,
      "activations/layer10_attention_weight_max": 34.03519058227539,
      "activations/layer10_attention_weight_min": -34.79043197631836,
      "activations/layer11_attention_weight_max": 34.33087158203125,
      "activations/layer11_attention_weight_min": -34.50066375732422,
      "activations/layer12_attention_weight_max": 26.143171310424805,
      "activations/layer12_attention_weight_min": -29.737873077392578,
      "activations/layer13_attention_weight_max": 39.85053634643555,
      "activations/layer13_attention_weight_min": -37.18135070800781,
      "activations/layer14_attention_weight_max": 41.421024322509766,
      "activations/layer14_attention_weight_min": -39.40367126464844,
      "activations/layer15_attention_weight_max": 37.8992805480957,
      "activations/layer15_attention_weight_min": -37.7295036315918,
      "activations/layer16_attention_weight_max": 28.073835372924805,
      "activations/layer16_attention_weight_min": -29.759233474731445,
      "activations/layer17_attention_weight_max": 30.366153717041016,
      "activations/layer17_attention_weight_min": -27.09531593322754,
      "activations/layer18_attention_weight_max": 35.33823013305664,
      "activations/layer18_attention_weight_min": -26.363969802856445,
      "activations/layer19_attention_weight_max": 34.6074104309082,
      "activations/layer19_attention_weight_min": -33.777584075927734,
      "activations/layer1_attention_weight_max": 16.01151466369629,
      "activations/layer1_attention_weight_min": -15.484832763671875,
      "activations/layer20_attention_weight_max": 30.049072265625,
      "activations/layer20_attention_weight_min": -26.91329574584961,
      "activations/layer21_attention_weight_max": 28.21108055114746,
      "activations/layer21_attention_weight_min": -27.225692749023438,
      "activations/layer22_attention_weight_max": 42.628440856933594,
      "activations/layer22_attention_weight_min": -35.086997985839844,
      "activations/layer23_attention_weight_max": 34.88334655761719,
      "activations/layer23_attention_weight_min": -29.83148765563965,
      "activations/layer2_attention_weight_max": 36.00779342651367,
      "activations/layer2_attention_weight_min": -33.57148742675781,
      "activations/layer3_attention_weight_max": 105.44083404541016,
      "activations/layer3_attention_weight_min": -108.5232925415039,
      "activations/layer4_attention_weight_max": 112.57435607910156,
      "activations/layer4_attention_weight_min": -114.8456802368164,
      "activations/layer5_attention_weight_max": 52.45222473144531,
      "activations/layer5_attention_weight_min": -67.454833984375,
      "activations/layer6_attention_weight_max": 46.84691619873047,
      "activations/layer6_attention_weight_min": -49.48176574707031,
      "activations/layer7_attention_weight_max": 97.90614318847656,
      "activations/layer7_attention_weight_min": -107.27587890625,
      "activations/layer8_attention_weight_max": 43.325172424316406,
      "activations/layer8_attention_weight_min": -46.85828399658203,
      "activations/layer9_attention_weight_max": 34.006290435791016,
      "activations/layer9_attention_weight_min": -37.25339889526367,
      "epoch": 21.26,
      "learning_rate": 1.3043560606060604e-05,
      "loss": 2.7042,
      "step": 365850
    },
    {
      "activations/layer0_attention_weight_max": 16.861356735229492,
      "activations/layer0_attention_weight_min": -14.23387336730957,
      "activations/layer10_attention_weight_max": 34.10877227783203,
      "activations/layer10_attention_weight_min": -36.532989501953125,
      "activations/layer11_attention_weight_max": 34.144012451171875,
      "activations/layer11_attention_weight_min": -35.63761901855469,
      "activations/layer12_attention_weight_max": 28.28734016418457,
      "activations/layer12_attention_weight_min": -26.61775016784668,
      "activations/layer13_attention_weight_max": 41.331661224365234,
      "activations/layer13_attention_weight_min": -36.281036376953125,
      "activations/layer14_attention_weight_max": 43.41819763183594,
      "activations/layer14_attention_weight_min": -39.4394645690918,
      "activations/layer15_attention_weight_max": 39.350276947021484,
      "activations/layer15_attention_weight_min": -36.556732177734375,
      "activations/layer16_attention_weight_max": 29.324737548828125,
      "activations/layer16_attention_weight_min": -28.43419075012207,
      "activations/layer17_attention_weight_max": 30.26274299621582,
      "activations/layer17_attention_weight_min": -29.92254638671875,
      "activations/layer18_attention_weight_max": 31.12159538269043,
      "activations/layer18_attention_weight_min": -26.231218338012695,
      "activations/layer19_attention_weight_max": 33.3486213684082,
      "activations/layer19_attention_weight_min": -31.356016159057617,
      "activations/layer1_attention_weight_max": 17.24018096923828,
      "activations/layer1_attention_weight_min": -16.11086654663086,
      "activations/layer20_attention_weight_max": 28.80061912536621,
      "activations/layer20_attention_weight_min": -24.891857147216797,
      "activations/layer21_attention_weight_max": 29.129379272460938,
      "activations/layer21_attention_weight_min": -24.75006675720215,
      "activations/layer22_attention_weight_max": 40.42597198486328,
      "activations/layer22_attention_weight_min": -32.49949264526367,
      "activations/layer23_attention_weight_max": 30.768089294433594,
      "activations/layer23_attention_weight_min": -26.312780380249023,
      "activations/layer2_attention_weight_max": 35.36841583251953,
      "activations/layer2_attention_weight_min": -33.20981979370117,
      "activations/layer3_attention_weight_max": 101.3270034790039,
      "activations/layer3_attention_weight_min": -104.22618865966797,
      "activations/layer4_attention_weight_max": 107.6168441772461,
      "activations/layer4_attention_weight_min": -110.61402893066406,
      "activations/layer5_attention_weight_max": 50.08769226074219,
      "activations/layer5_attention_weight_min": -62.60322189331055,
      "activations/layer6_attention_weight_max": 43.90657043457031,
      "activations/layer6_attention_weight_min": -47.80582046508789,
      "activations/layer7_attention_weight_max": 98.53526306152344,
      "activations/layer7_attention_weight_min": -95.90394592285156,
      "activations/layer8_attention_weight_max": 46.00044631958008,
      "activations/layer8_attention_weight_min": -45.98416519165039,
      "activations/layer9_attention_weight_max": 32.38905715942383,
      "activations/layer9_attention_weight_min": -34.68760681152344,
      "epoch": 21.26,
      "learning_rate": 1.302462121212121e-05,
      "loss": 2.691,
      "step": 365900
    },
    {
      "activations/layer0_attention_weight_max": 16.17636489868164,
      "activations/layer0_attention_weight_min": -14.11205005645752,
      "activations/layer10_attention_weight_max": 32.50736618041992,
      "activations/layer10_attention_weight_min": -33.91261672973633,
      "activations/layer11_attention_weight_max": 32.676307678222656,
      "activations/layer11_attention_weight_min": -34.41377639770508,
      "activations/layer12_attention_weight_max": 25.937883377075195,
      "activations/layer12_attention_weight_min": -26.115205764770508,
      "activations/layer13_attention_weight_max": 39.1700439453125,
      "activations/layer13_attention_weight_min": -34.83830642700195,
      "activations/layer14_attention_weight_max": 43.67411804199219,
      "activations/layer14_attention_weight_min": -37.029396057128906,
      "activations/layer15_attention_weight_max": 39.0391845703125,
      "activations/layer15_attention_weight_min": -35.13941955566406,
      "activations/layer16_attention_weight_max": 30.618059158325195,
      "activations/layer16_attention_weight_min": -28.589900970458984,
      "activations/layer17_attention_weight_max": 29.67400550842285,
      "activations/layer17_attention_weight_min": -26.64975929260254,
      "activations/layer18_attention_weight_max": 34.10434341430664,
      "activations/layer18_attention_weight_min": -26.274486541748047,
      "activations/layer19_attention_weight_max": 34.24061584472656,
      "activations/layer19_attention_weight_min": -31.458398818969727,
      "activations/layer1_attention_weight_max": 15.172882080078125,
      "activations/layer1_attention_weight_min": -14.781852722167969,
      "activations/layer20_attention_weight_max": 29.322019577026367,
      "activations/layer20_attention_weight_min": -25.645702362060547,
      "activations/layer21_attention_weight_max": 31.819616317749023,
      "activations/layer21_attention_weight_min": -25.712955474853516,
      "activations/layer22_attention_weight_max": 41.02854537963867,
      "activations/layer22_attention_weight_min": -30.063310623168945,
      "activations/layer23_attention_weight_max": 33.37560272216797,
      "activations/layer23_attention_weight_min": -25.559307098388672,
      "activations/layer2_attention_weight_max": 33.665706634521484,
      "activations/layer2_attention_weight_min": -31.70379638671875,
      "activations/layer3_attention_weight_max": 102.7127685546875,
      "activations/layer3_attention_weight_min": -104.0348129272461,
      "activations/layer4_attention_weight_max": 109.45340728759766,
      "activations/layer4_attention_weight_min": -109.6736068725586,
      "activations/layer5_attention_weight_max": 49.42418670654297,
      "activations/layer5_attention_weight_min": -61.87887191772461,
      "activations/layer6_attention_weight_max": 43.519405364990234,
      "activations/layer6_attention_weight_min": -46.5525016784668,
      "activations/layer7_attention_weight_max": 93.40869140625,
      "activations/layer7_attention_weight_min": -97.61321258544922,
      "activations/layer8_attention_weight_max": 40.449832916259766,
      "activations/layer8_attention_weight_min": -47.699764251708984,
      "activations/layer9_attention_weight_max": 31.915267944335938,
      "activations/layer9_attention_weight_min": -33.7614860534668,
      "epoch": 21.26,
      "learning_rate": 1.3005681818181817e-05,
      "loss": 2.7179,
      "step": 365950
    },
    {
      "activations/layer0_attention_weight_max": 16.655397415161133,
      "activations/layer0_attention_weight_min": -14.615265846252441,
      "activations/layer10_attention_weight_max": 43.0176887512207,
      "activations/layer10_attention_weight_min": -38.11704635620117,
      "activations/layer11_attention_weight_max": 39.46971130371094,
      "activations/layer11_attention_weight_min": -38.618446350097656,
      "activations/layer12_attention_weight_max": 27.490352630615234,
      "activations/layer12_attention_weight_min": -27.80790901184082,
      "activations/layer13_attention_weight_max": 42.23590087890625,
      "activations/layer13_attention_weight_min": -38.70075607299805,
      "activations/layer14_attention_weight_max": 44.2089958190918,
      "activations/layer14_attention_weight_min": -40.15994644165039,
      "activations/layer15_attention_weight_max": 41.971824645996094,
      "activations/layer15_attention_weight_min": -40.41737365722656,
      "activations/layer16_attention_weight_max": 30.570356369018555,
      "activations/layer16_attention_weight_min": -26.946800231933594,
      "activations/layer17_attention_weight_max": 32.03608322143555,
      "activations/layer17_attention_weight_min": -26.257780075073242,
      "activations/layer18_attention_weight_max": 32.116703033447266,
      "activations/layer18_attention_weight_min": -28.485511779785156,
      "activations/layer19_attention_weight_max": 35.04478073120117,
      "activations/layer19_attention_weight_min": -32.4393424987793,
      "activations/layer1_attention_weight_max": 16.400272369384766,
      "activations/layer1_attention_weight_min": -14.544758796691895,
      "activations/layer20_attention_weight_max": 29.31636619567871,
      "activations/layer20_attention_weight_min": -25.128520965576172,
      "activations/layer21_attention_weight_max": 28.838285446166992,
      "activations/layer21_attention_weight_min": -25.45220184326172,
      "activations/layer22_attention_weight_max": 42.78458786010742,
      "activations/layer22_attention_weight_min": -33.18488693237305,
      "activations/layer23_attention_weight_max": 34.59660720825195,
      "activations/layer23_attention_weight_min": -27.389904022216797,
      "activations/layer2_attention_weight_max": 32.3321647644043,
      "activations/layer2_attention_weight_min": -32.92647171020508,
      "activations/layer3_attention_weight_max": 96.58674621582031,
      "activations/layer3_attention_weight_min": -100.91358184814453,
      "activations/layer4_attention_weight_max": 102.74440002441406,
      "activations/layer4_attention_weight_min": -107.83745574951172,
      "activations/layer5_attention_weight_max": 47.31425094604492,
      "activations/layer5_attention_weight_min": -59.576995849609375,
      "activations/layer6_attention_weight_max": 42.86550521850586,
      "activations/layer6_attention_weight_min": -44.53767395019531,
      "activations/layer7_attention_weight_max": 104.95178985595703,
      "activations/layer7_attention_weight_min": -96.3786849975586,
      "activations/layer8_attention_weight_max": 45.73353958129883,
      "activations/layer8_attention_weight_min": -46.8511848449707,
      "activations/layer9_attention_weight_max": 33.15074157714844,
      "activations/layer9_attention_weight_min": -36.74293518066406,
      "epoch": 21.27,
      "learning_rate": 1.2986742424242422e-05,
      "loss": 2.7155,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4943,
      "eval_samples_per_second": 505.513,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4943,
      "eval_openwebtext_samples_per_second": 505.513,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9787,
      "eval_wikitext_samples_per_second": 230.457,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_lambada_loss": 2.318359375,
      "eval_lambada_ppl": 10.158993527607587,
      "eval_lambada_runtime": 9.5531,
      "eval_lambada_samples_per_second": 509.678,
      "step": 366000
    },
    {
      "activations/layer0_attention_weight_max": 16.67876625061035,
      "activations/layer0_attention_weight_min": -13.765138626098633,
      "activations/layer10_attention_weight_max": 32.7525749206543,
      "activations/layer10_attention_weight_min": -34.77503204345703,
      "activations/layer11_attention_weight_max": 33.47356414794922,
      "activations/layer11_attention_weight_min": -35.36420440673828,
      "activations/layer12_attention_weight_max": 27.24738883972168,
      "activations/layer12_attention_weight_min": -26.960451126098633,
      "activations/layer13_attention_weight_max": 40.956886291503906,
      "activations/layer13_attention_weight_min": -38.11153793334961,
      "activations/layer14_attention_weight_max": 43.389434814453125,
      "activations/layer14_attention_weight_min": -41.484130859375,
      "activations/layer15_attention_weight_max": 40.166175842285156,
      "activations/layer15_attention_weight_min": -37.74890899658203,
      "activations/layer16_attention_weight_max": 29.328811645507812,
      "activations/layer16_attention_weight_min": -28.198556900024414,
      "activations/layer17_attention_weight_max": 28.812089920043945,
      "activations/layer17_attention_weight_min": -25.197959899902344,
      "activations/layer18_attention_weight_max": 33.04888153076172,
      "activations/layer18_attention_weight_min": -25.283164978027344,
      "activations/layer19_attention_weight_max": 33.86447525024414,
      "activations/layer19_attention_weight_min": -32.229942321777344,
      "activations/layer1_attention_weight_max": 15.217916488647461,
      "activations/layer1_attention_weight_min": -15.1110258102417,
      "activations/layer20_attention_weight_max": 29.189281463623047,
      "activations/layer20_attention_weight_min": -26.701753616333008,
      "activations/layer21_attention_weight_max": 28.389677047729492,
      "activations/layer21_attention_weight_min": -27.234586715698242,
      "activations/layer22_attention_weight_max": 42.849205017089844,
      "activations/layer22_attention_weight_min": -31.54669761657715,
      "activations/layer23_attention_weight_max": 33.84458923339844,
      "activations/layer23_attention_weight_min": -27.129966735839844,
      "activations/layer2_attention_weight_max": 33.126441955566406,
      "activations/layer2_attention_weight_min": -32.43635940551758,
      "activations/layer3_attention_weight_max": 103.02461242675781,
      "activations/layer3_attention_weight_min": -106.09515380859375,
      "activations/layer4_attention_weight_max": 107.39620208740234,
      "activations/layer4_attention_weight_min": -110.95001220703125,
      "activations/layer5_attention_weight_max": 53.49721908569336,
      "activations/layer5_attention_weight_min": -63.47433090209961,
      "activations/layer6_attention_weight_max": 45.353755950927734,
      "activations/layer6_attention_weight_min": -50.987709045410156,
      "activations/layer7_attention_weight_max": 95.96748352050781,
      "activations/layer7_attention_weight_min": -99.33208465576172,
      "activations/layer8_attention_weight_max": 42.71310043334961,
      "activations/layer8_attention_weight_min": -43.679595947265625,
      "activations/layer9_attention_weight_max": 30.68246841430664,
      "activations/layer9_attention_weight_min": -33.715084075927734,
      "epoch": 21.27,
      "learning_rate": 1.2967803030303029e-05,
      "loss": 2.7191,
      "step": 366050
    },
    {
      "activations/layer0_attention_weight_max": 16.165822982788086,
      "activations/layer0_attention_weight_min": -13.821471214294434,
      "activations/layer10_attention_weight_max": 34.61049270629883,
      "activations/layer10_attention_weight_min": -37.162845611572266,
      "activations/layer11_attention_weight_max": 35.041404724121094,
      "activations/layer11_attention_weight_min": -35.79091262817383,
      "activations/layer12_attention_weight_max": 29.567726135253906,
      "activations/layer12_attention_weight_min": -28.988744735717773,
      "activations/layer13_attention_weight_max": 45.492835998535156,
      "activations/layer13_attention_weight_min": -40.871700286865234,
      "activations/layer14_attention_weight_max": 48.254878997802734,
      "activations/layer14_attention_weight_min": -42.67188262939453,
      "activations/layer15_attention_weight_max": 41.78377151489258,
      "activations/layer15_attention_weight_min": -38.38505172729492,
      "activations/layer16_attention_weight_max": 31.905717849731445,
      "activations/layer16_attention_weight_min": -28.494335174560547,
      "activations/layer17_attention_weight_max": 31.589563369750977,
      "activations/layer17_attention_weight_min": -27.164573669433594,
      "activations/layer18_attention_weight_max": 33.85976791381836,
      "activations/layer18_attention_weight_min": -26.523235321044922,
      "activations/layer19_attention_weight_max": 37.12662887573242,
      "activations/layer19_attention_weight_min": -31.722766876220703,
      "activations/layer1_attention_weight_max": 16.010814666748047,
      "activations/layer1_attention_weight_min": -15.57997989654541,
      "activations/layer20_attention_weight_max": 31.763349533081055,
      "activations/layer20_attention_weight_min": -27.098176956176758,
      "activations/layer21_attention_weight_max": 31.584928512573242,
      "activations/layer21_attention_weight_min": -24.913442611694336,
      "activations/layer22_attention_weight_max": 45.35417556762695,
      "activations/layer22_attention_weight_min": -32.499725341796875,
      "activations/layer23_attention_weight_max": 31.132484436035156,
      "activations/layer23_attention_weight_min": -26.96278190612793,
      "activations/layer2_attention_weight_max": 31.488187789916992,
      "activations/layer2_attention_weight_min": -31.777097702026367,
      "activations/layer3_attention_weight_max": 102.16084289550781,
      "activations/layer3_attention_weight_min": -105.30136108398438,
      "activations/layer4_attention_weight_max": 108.91942596435547,
      "activations/layer4_attention_weight_min": -107.4582290649414,
      "activations/layer5_attention_weight_max": 51.29173278808594,
      "activations/layer5_attention_weight_min": -60.38334655761719,
      "activations/layer6_attention_weight_max": 45.474674224853516,
      "activations/layer6_attention_weight_min": -45.36671829223633,
      "activations/layer7_attention_weight_max": 97.58415222167969,
      "activations/layer7_attention_weight_min": -100.00016021728516,
      "activations/layer8_attention_weight_max": 41.86830520629883,
      "activations/layer8_attention_weight_min": -44.9608268737793,
      "activations/layer9_attention_weight_max": 33.43291473388672,
      "activations/layer9_attention_weight_min": -36.07759475708008,
      "epoch": 21.27,
      "learning_rate": 1.2949242424242423e-05,
      "loss": 2.7216,
      "step": 366100
    },
    {
      "activations/layer0_attention_weight_max": 16.498065948486328,
      "activations/layer0_attention_weight_min": -14.154253959655762,
      "activations/layer10_attention_weight_max": 37.277549743652344,
      "activations/layer10_attention_weight_min": -35.795265197753906,
      "activations/layer11_attention_weight_max": 37.168575286865234,
      "activations/layer11_attention_weight_min": -37.53905487060547,
      "activations/layer12_attention_weight_max": 28.463224411010742,
      "activations/layer12_attention_weight_min": -27.829072952270508,
      "activations/layer13_attention_weight_max": 44.434391021728516,
      "activations/layer13_attention_weight_min": -37.996429443359375,
      "activations/layer14_attention_weight_max": 47.198001861572266,
      "activations/layer14_attention_weight_min": -41.222938537597656,
      "activations/layer15_attention_weight_max": 44.673622131347656,
      "activations/layer15_attention_weight_min": -38.07075881958008,
      "activations/layer16_attention_weight_max": 33.39993667602539,
      "activations/layer16_attention_weight_min": -30.316320419311523,
      "activations/layer17_attention_weight_max": 33.33577346801758,
      "activations/layer17_attention_weight_min": -28.11250877380371,
      "activations/layer18_attention_weight_max": 34.421112060546875,
      "activations/layer18_attention_weight_min": -26.957048416137695,
      "activations/layer19_attention_weight_max": 33.867374420166016,
      "activations/layer19_attention_weight_min": -31.4137020111084,
      "activations/layer1_attention_weight_max": 16.44002342224121,
      "activations/layer1_attention_weight_min": -14.98055362701416,
      "activations/layer20_attention_weight_max": 31.030099868774414,
      "activations/layer20_attention_weight_min": -26.800125122070312,
      "activations/layer21_attention_weight_max": 31.548702239990234,
      "activations/layer21_attention_weight_min": -26.108144760131836,
      "activations/layer22_attention_weight_max": 45.406192779541016,
      "activations/layer22_attention_weight_min": -31.79323387145996,
      "activations/layer23_attention_weight_max": 33.53187942504883,
      "activations/layer23_attention_weight_min": -26.73256492614746,
      "activations/layer2_attention_weight_max": 32.05168533325195,
      "activations/layer2_attention_weight_min": -32.54946517944336,
      "activations/layer3_attention_weight_max": 103.26118469238281,
      "activations/layer3_attention_weight_min": -103.06634521484375,
      "activations/layer4_attention_weight_max": 106.20011138916016,
      "activations/layer4_attention_weight_min": -111.80818939208984,
      "activations/layer5_attention_weight_max": 49.438533782958984,
      "activations/layer5_attention_weight_min": -58.693031311035156,
      "activations/layer6_attention_weight_max": 45.0422248840332,
      "activations/layer6_attention_weight_min": -48.356788635253906,
      "activations/layer7_attention_weight_max": 97.68930053710938,
      "activations/layer7_attention_weight_min": -102.12226104736328,
      "activations/layer8_attention_weight_max": 45.39015579223633,
      "activations/layer8_attention_weight_min": -44.42009353637695,
      "activations/layer9_attention_weight_max": 35.89297103881836,
      "activations/layer9_attention_weight_min": -36.89469528198242,
      "epoch": 21.28,
      "learning_rate": 1.2930303030303028e-05,
      "loss": 2.7039,
      "step": 366150
    },
    {
      "activations/layer0_attention_weight_max": 16.950809478759766,
      "activations/layer0_attention_weight_min": -13.165396690368652,
      "activations/layer10_attention_weight_max": 36.0638542175293,
      "activations/layer10_attention_weight_min": -36.47233200073242,
      "activations/layer11_attention_weight_max": 36.74815368652344,
      "activations/layer11_attention_weight_min": -35.1004638671875,
      "activations/layer12_attention_weight_max": 31.91533851623535,
      "activations/layer12_attention_weight_min": -29.584497451782227,
      "activations/layer13_attention_weight_max": 42.98341369628906,
      "activations/layer13_attention_weight_min": -38.678932189941406,
      "activations/layer14_attention_weight_max": 48.856719970703125,
      "activations/layer14_attention_weight_min": -41.156803131103516,
      "activations/layer15_attention_weight_max": 49.299407958984375,
      "activations/layer15_attention_weight_min": -39.00770568847656,
      "activations/layer16_attention_weight_max": 36.72803497314453,
      "activations/layer16_attention_weight_min": -28.22640609741211,
      "activations/layer17_attention_weight_max": 35.26764678955078,
      "activations/layer17_attention_weight_min": -28.77206039428711,
      "activations/layer18_attention_weight_max": 37.1012077331543,
      "activations/layer18_attention_weight_min": -28.199254989624023,
      "activations/layer19_attention_weight_max": 47.69830322265625,
      "activations/layer19_attention_weight_min": -36.38984298706055,
      "activations/layer1_attention_weight_max": 15.85783576965332,
      "activations/layer1_attention_weight_min": -15.354972839355469,
      "activations/layer20_attention_weight_max": 37.312400817871094,
      "activations/layer20_attention_weight_min": -26.349552154541016,
      "activations/layer21_attention_weight_max": 37.39948272705078,
      "activations/layer21_attention_weight_min": -26.50752830505371,
      "activations/layer22_attention_weight_max": 53.94558334350586,
      "activations/layer22_attention_weight_min": -34.60721206665039,
      "activations/layer23_attention_weight_max": 43.30219268798828,
      "activations/layer23_attention_weight_min": -25.431386947631836,
      "activations/layer2_attention_weight_max": 34.41089630126953,
      "activations/layer2_attention_weight_min": -32.83188247680664,
      "activations/layer3_attention_weight_max": 108.53679656982422,
      "activations/layer3_attention_weight_min": -105.13932037353516,
      "activations/layer4_attention_weight_max": 109.70458221435547,
      "activations/layer4_attention_weight_min": -113.30599212646484,
      "activations/layer5_attention_weight_max": 51.834938049316406,
      "activations/layer5_attention_weight_min": -64.29570770263672,
      "activations/layer6_attention_weight_max": 48.099849700927734,
      "activations/layer6_attention_weight_min": -49.986000061035156,
      "activations/layer7_attention_weight_max": 97.91243743896484,
      "activations/layer7_attention_weight_min": -107.75565338134766,
      "activations/layer8_attention_weight_max": 46.496498107910156,
      "activations/layer8_attention_weight_min": -47.972347259521484,
      "activations/layer9_attention_weight_max": 35.92521286010742,
      "activations/layer9_attention_weight_min": -37.03476333618164,
      "epoch": 21.28,
      "learning_rate": 1.2911363636363634e-05,
      "loss": 2.7054,
      "step": 366200
    },
    {
      "activations/layer0_attention_weight_max": 16.355926513671875,
      "activations/layer0_attention_weight_min": -14.422503471374512,
      "activations/layer10_attention_weight_max": 35.042808532714844,
      "activations/layer10_attention_weight_min": -37.244503021240234,
      "activations/layer11_attention_weight_max": 35.71495819091797,
      "activations/layer11_attention_weight_min": -37.62950134277344,
      "activations/layer12_attention_weight_max": 28.21261978149414,
      "activations/layer12_attention_weight_min": -28.07431411743164,
      "activations/layer13_attention_weight_max": 41.80048370361328,
      "activations/layer13_attention_weight_min": -37.27824020385742,
      "activations/layer14_attention_weight_max": 44.74673080444336,
      "activations/layer14_attention_weight_min": -39.07315444946289,
      "activations/layer15_attention_weight_max": 41.41813659667969,
      "activations/layer15_attention_weight_min": -37.82707977294922,
      "activations/layer16_attention_weight_max": 30.568567276000977,
      "activations/layer16_attention_weight_min": -27.631155014038086,
      "activations/layer17_attention_weight_max": 30.97445297241211,
      "activations/layer17_attention_weight_min": -28.598506927490234,
      "activations/layer18_attention_weight_max": 33.21303939819336,
      "activations/layer18_attention_weight_min": -28.207895278930664,
      "activations/layer19_attention_weight_max": 33.90475082397461,
      "activations/layer19_attention_weight_min": -31.148351669311523,
      "activations/layer1_attention_weight_max": 15.88779354095459,
      "activations/layer1_attention_weight_min": -15.48224925994873,
      "activations/layer20_attention_weight_max": 28.975406646728516,
      "activations/layer20_attention_weight_min": -25.70734214782715,
      "activations/layer21_attention_weight_max": 29.141910552978516,
      "activations/layer21_attention_weight_min": -26.070419311523438,
      "activations/layer22_attention_weight_max": 42.9976921081543,
      "activations/layer22_attention_weight_min": -33.521846771240234,
      "activations/layer23_attention_weight_max": 32.92207717895508,
      "activations/layer23_attention_weight_min": -27.207725524902344,
      "activations/layer2_attention_weight_max": 33.97987747192383,
      "activations/layer2_attention_weight_min": -32.45854949951172,
      "activations/layer3_attention_weight_max": 106.4621810913086,
      "activations/layer3_attention_weight_min": -104.56876373291016,
      "activations/layer4_attention_weight_max": 106.33451080322266,
      "activations/layer4_attention_weight_min": -111.26593017578125,
      "activations/layer5_attention_weight_max": 49.78579330444336,
      "activations/layer5_attention_weight_min": -61.221736907958984,
      "activations/layer6_attention_weight_max": 44.405250549316406,
      "activations/layer6_attention_weight_min": -47.61148452758789,
      "activations/layer7_attention_weight_max": 93.78728485107422,
      "activations/layer7_attention_weight_min": -100.9068832397461,
      "activations/layer8_attention_weight_max": 41.900360107421875,
      "activations/layer8_attention_weight_min": -46.47953796386719,
      "activations/layer9_attention_weight_max": 33.62012481689453,
      "activations/layer9_attention_weight_min": -36.08035659790039,
      "epoch": 21.28,
      "learning_rate": 1.2892424242424242e-05,
      "loss": 2.7015,
      "step": 366250
    },
    {
      "activations/layer0_attention_weight_max": 16.20444107055664,
      "activations/layer0_attention_weight_min": -14.361252784729004,
      "activations/layer10_attention_weight_max": 36.00287628173828,
      "activations/layer10_attention_weight_min": -36.51226806640625,
      "activations/layer11_attention_weight_max": 35.113677978515625,
      "activations/layer11_attention_weight_min": -36.29877853393555,
      "activations/layer12_attention_weight_max": 28.57337760925293,
      "activations/layer12_attention_weight_min": -27.052759170532227,
      "activations/layer13_attention_weight_max": 38.97780227661133,
      "activations/layer13_attention_weight_min": -36.133705139160156,
      "activations/layer14_attention_weight_max": 41.25449752807617,
      "activations/layer14_attention_weight_min": -38.99726867675781,
      "activations/layer15_attention_weight_max": 39.64026641845703,
      "activations/layer15_attention_weight_min": -36.38515853881836,
      "activations/layer16_attention_weight_max": 29.291553497314453,
      "activations/layer16_attention_weight_min": -29.684314727783203,
      "activations/layer17_attention_weight_max": 30.21689796447754,
      "activations/layer17_attention_weight_min": -27.702425003051758,
      "activations/layer18_attention_weight_max": 32.9207878112793,
      "activations/layer18_attention_weight_min": -27.363500595092773,
      "activations/layer19_attention_weight_max": 34.220821380615234,
      "activations/layer19_attention_weight_min": -33.52867889404297,
      "activations/layer1_attention_weight_max": 15.8611421585083,
      "activations/layer1_attention_weight_min": -15.176511764526367,
      "activations/layer20_attention_weight_max": 31.284770965576172,
      "activations/layer20_attention_weight_min": -28.78643798828125,
      "activations/layer21_attention_weight_max": 31.174358367919922,
      "activations/layer21_attention_weight_min": -25.6028995513916,
      "activations/layer22_attention_weight_max": 44.775726318359375,
      "activations/layer22_attention_weight_min": -35.37813949584961,
      "activations/layer23_attention_weight_max": 33.187530517578125,
      "activations/layer23_attention_weight_min": -28.453311920166016,
      "activations/layer2_attention_weight_max": 35.71405029296875,
      "activations/layer2_attention_weight_min": -32.718570709228516,
      "activations/layer3_attention_weight_max": 109.95442962646484,
      "activations/layer3_attention_weight_min": -105.67498016357422,
      "activations/layer4_attention_weight_max": 117.94844818115234,
      "activations/layer4_attention_weight_min": -114.1016845703125,
      "activations/layer5_attention_weight_max": 53.389808654785156,
      "activations/layer5_attention_weight_min": -63.621986389160156,
      "activations/layer6_attention_weight_max": 45.82616424560547,
      "activations/layer6_attention_weight_min": -48.10738754272461,
      "activations/layer7_attention_weight_max": 95.1934585571289,
      "activations/layer7_attention_weight_min": -98.53231811523438,
      "activations/layer8_attention_weight_max": 42.89265441894531,
      "activations/layer8_attention_weight_min": -45.36060333251953,
      "activations/layer9_attention_weight_max": 32.0512809753418,
      "activations/layer9_attention_weight_min": -35.6820182800293,
      "epoch": 21.28,
      "learning_rate": 1.2873484848484849e-05,
      "loss": 2.6993,
      "step": 366300
    },
    {
      "activations/layer0_attention_weight_max": 16.305654525756836,
      "activations/layer0_attention_weight_min": -14.99527645111084,
      "activations/layer10_attention_weight_max": 34.25090408325195,
      "activations/layer10_attention_weight_min": -35.342716217041016,
      "activations/layer11_attention_weight_max": 33.73584747314453,
      "activations/layer11_attention_weight_min": -34.13396453857422,
      "activations/layer12_attention_weight_max": 27.799091339111328,
      "activations/layer12_attention_weight_min": -27.401941299438477,
      "activations/layer13_attention_weight_max": 39.58330154418945,
      "activations/layer13_attention_weight_min": -36.5445671081543,
      "activations/layer14_attention_weight_max": 40.666786193847656,
      "activations/layer14_attention_weight_min": -37.88001251220703,
      "activations/layer15_attention_weight_max": 37.667293548583984,
      "activations/layer15_attention_weight_min": -36.32181930541992,
      "activations/layer16_attention_weight_max": 31.239437103271484,
      "activations/layer16_attention_weight_min": -27.868885040283203,
      "activations/layer17_attention_weight_max": 28.743080139160156,
      "activations/layer17_attention_weight_min": -27.422334671020508,
      "activations/layer18_attention_weight_max": 34.33063888549805,
      "activations/layer18_attention_weight_min": -28.336034774780273,
      "activations/layer19_attention_weight_max": 35.06559371948242,
      "activations/layer19_attention_weight_min": -32.32436752319336,
      "activations/layer1_attention_weight_max": 16.12824821472168,
      "activations/layer1_attention_weight_min": -15.570796966552734,
      "activations/layer20_attention_weight_max": 30.535383224487305,
      "activations/layer20_attention_weight_min": -25.806493759155273,
      "activations/layer21_attention_weight_max": 30.648513793945312,
      "activations/layer21_attention_weight_min": -24.763967514038086,
      "activations/layer22_attention_weight_max": 41.6585693359375,
      "activations/layer22_attention_weight_min": -32.64661407470703,
      "activations/layer23_attention_weight_max": 31.210464477539062,
      "activations/layer23_attention_weight_min": -28.13423728942871,
      "activations/layer2_attention_weight_max": 35.10130310058594,
      "activations/layer2_attention_weight_min": -34.072052001953125,
      "activations/layer3_attention_weight_max": 106.1136703491211,
      "activations/layer3_attention_weight_min": -108.74198913574219,
      "activations/layer4_attention_weight_max": 114.76616668701172,
      "activations/layer4_attention_weight_min": -117.07289123535156,
      "activations/layer5_attention_weight_max": 52.2751579284668,
      "activations/layer5_attention_weight_min": -62.82481002807617,
      "activations/layer6_attention_weight_max": 46.75752639770508,
      "activations/layer6_attention_weight_min": -51.031715393066406,
      "activations/layer7_attention_weight_max": 95.34742736816406,
      "activations/layer7_attention_weight_min": -102.46810913085938,
      "activations/layer8_attention_weight_max": 44.025489807128906,
      "activations/layer8_attention_weight_min": -46.21561813354492,
      "activations/layer9_attention_weight_max": 33.16895294189453,
      "activations/layer9_attention_weight_min": -36.19731140136719,
      "epoch": 21.29,
      "learning_rate": 1.2854545454545454e-05,
      "loss": 2.7132,
      "step": 366350
    },
    {
      "activations/layer0_attention_weight_max": 16.24060821533203,
      "activations/layer0_attention_weight_min": -14.505730628967285,
      "activations/layer10_attention_weight_max": 34.01585388183594,
      "activations/layer10_attention_weight_min": -35.876953125,
      "activations/layer11_attention_weight_max": 37.09716033935547,
      "activations/layer11_attention_weight_min": -33.89519500732422,
      "activations/layer12_attention_weight_max": 26.422407150268555,
      "activations/layer12_attention_weight_min": -27.99264144897461,
      "activations/layer13_attention_weight_max": 39.65675354003906,
      "activations/layer13_attention_weight_min": -37.002281188964844,
      "activations/layer14_attention_weight_max": 42.276580810546875,
      "activations/layer14_attention_weight_min": -38.794437408447266,
      "activations/layer15_attention_weight_max": 37.85524368286133,
      "activations/layer15_attention_weight_min": -35.40056610107422,
      "activations/layer16_attention_weight_max": 31.64006233215332,
      "activations/layer16_attention_weight_min": -30.389253616333008,
      "activations/layer17_attention_weight_max": 29.74200439453125,
      "activations/layer17_attention_weight_min": -28.47486686706543,
      "activations/layer18_attention_weight_max": 31.71357536315918,
      "activations/layer18_attention_weight_min": -24.713647842407227,
      "activations/layer19_attention_weight_max": 36.51702880859375,
      "activations/layer19_attention_weight_min": -32.58595275878906,
      "activations/layer1_attention_weight_max": 15.998220443725586,
      "activations/layer1_attention_weight_min": -15.475939750671387,
      "activations/layer20_attention_weight_max": 29.694822311401367,
      "activations/layer20_attention_weight_min": -24.871253967285156,
      "activations/layer21_attention_weight_max": 30.49905014038086,
      "activations/layer21_attention_weight_min": -24.659109115600586,
      "activations/layer22_attention_weight_max": 46.47629928588867,
      "activations/layer22_attention_weight_min": -31.945039749145508,
      "activations/layer23_attention_weight_max": 35.61363220214844,
      "activations/layer23_attention_weight_min": -25.44188690185547,
      "activations/layer2_attention_weight_max": 33.08985900878906,
      "activations/layer2_attention_weight_min": -31.904996871948242,
      "activations/layer3_attention_weight_max": 102.29625701904297,
      "activations/layer3_attention_weight_min": -100.19613647460938,
      "activations/layer4_attention_weight_max": 109.7714614868164,
      "activations/layer4_attention_weight_min": -109.53474426269531,
      "activations/layer5_attention_weight_max": 52.5470085144043,
      "activations/layer5_attention_weight_min": -61.48566436767578,
      "activations/layer6_attention_weight_max": 46.46163558959961,
      "activations/layer6_attention_weight_min": -49.42757034301758,
      "activations/layer7_attention_weight_max": 95.25445556640625,
      "activations/layer7_attention_weight_min": -99.02348327636719,
      "activations/layer8_attention_weight_max": 42.97190856933594,
      "activations/layer8_attention_weight_min": -45.27992248535156,
      "activations/layer9_attention_weight_max": 31.5784854888916,
      "activations/layer9_attention_weight_min": -36.31306076049805,
      "epoch": 21.29,
      "learning_rate": 1.283560606060606e-05,
      "loss": 2.7288,
      "step": 366400
    },
    {
      "activations/layer0_attention_weight_max": 16.75132179260254,
      "activations/layer0_attention_weight_min": -14.10348892211914,
      "activations/layer10_attention_weight_max": 33.075592041015625,
      "activations/layer10_attention_weight_min": -35.90219497680664,
      "activations/layer11_attention_weight_max": 35.02851486206055,
      "activations/layer11_attention_weight_min": -37.443443298339844,
      "activations/layer12_attention_weight_max": 28.6003475189209,
      "activations/layer12_attention_weight_min": -29.46223258972168,
      "activations/layer13_attention_weight_max": 41.81699752807617,
      "activations/layer13_attention_weight_min": -40.111305236816406,
      "activations/layer14_attention_weight_max": 48.82495880126953,
      "activations/layer14_attention_weight_min": -44.43010330200195,
      "activations/layer15_attention_weight_max": 42.07164764404297,
      "activations/layer15_attention_weight_min": -42.406192779541016,
      "activations/layer16_attention_weight_max": 35.348243713378906,
      "activations/layer16_attention_weight_min": -31.148706436157227,
      "activations/layer17_attention_weight_max": 33.17121505737305,
      "activations/layer17_attention_weight_min": -28.488672256469727,
      "activations/layer18_attention_weight_max": 35.28007125854492,
      "activations/layer18_attention_weight_min": -28.323036193847656,
      "activations/layer19_attention_weight_max": 41.25022506713867,
      "activations/layer19_attention_weight_min": -36.469390869140625,
      "activations/layer1_attention_weight_max": 16.486299514770508,
      "activations/layer1_attention_weight_min": -14.938604354858398,
      "activations/layer20_attention_weight_max": 34.081783294677734,
      "activations/layer20_attention_weight_min": -28.354352951049805,
      "activations/layer21_attention_weight_max": 34.53162384033203,
      "activations/layer21_attention_weight_min": -29.098966598510742,
      "activations/layer22_attention_weight_max": 46.70668029785156,
      "activations/layer22_attention_weight_min": -36.194068908691406,
      "activations/layer23_attention_weight_max": 38.64514923095703,
      "activations/layer23_attention_weight_min": -29.38809585571289,
      "activations/layer2_attention_weight_max": 33.15936279296875,
      "activations/layer2_attention_weight_min": -32.93205642700195,
      "activations/layer3_attention_weight_max": 104.8973388671875,
      "activations/layer3_attention_weight_min": -106.23135375976562,
      "activations/layer4_attention_weight_max": 106.98707580566406,
      "activations/layer4_attention_weight_min": -111.94548797607422,
      "activations/layer5_attention_weight_max": 51.78148651123047,
      "activations/layer5_attention_weight_min": -66.54605102539062,
      "activations/layer6_attention_weight_max": 47.225711822509766,
      "activations/layer6_attention_weight_min": -49.79066467285156,
      "activations/layer7_attention_weight_max": 95.61788940429688,
      "activations/layer7_attention_weight_min": -102.90819549560547,
      "activations/layer8_attention_weight_max": 45.89052200317383,
      "activations/layer8_attention_weight_min": -48.53163528442383,
      "activations/layer9_attention_weight_max": 36.39985275268555,
      "activations/layer9_attention_weight_min": -40.72109603881836,
      "epoch": 21.29,
      "learning_rate": 1.2816666666666665e-05,
      "loss": 2.7099,
      "step": 366450
    },
    {
      "activations/layer0_attention_weight_max": 16.827760696411133,
      "activations/layer0_attention_weight_min": -13.349498748779297,
      "activations/layer10_attention_weight_max": 38.37611770629883,
      "activations/layer10_attention_weight_min": -37.85215759277344,
      "activations/layer11_attention_weight_max": 36.26179504394531,
      "activations/layer11_attention_weight_min": -40.19715118408203,
      "activations/layer12_attention_weight_max": 28.053730010986328,
      "activations/layer12_attention_weight_min": -27.49796485900879,
      "activations/layer13_attention_weight_max": 41.76887512207031,
      "activations/layer13_attention_weight_min": -36.56266784667969,
      "activations/layer14_attention_weight_max": 44.825199127197266,
      "activations/layer14_attention_weight_min": -39.7807731628418,
      "activations/layer15_attention_weight_max": 43.634456634521484,
      "activations/layer15_attention_weight_min": -36.825252532958984,
      "activations/layer16_attention_weight_max": 30.482173919677734,
      "activations/layer16_attention_weight_min": -28.11898422241211,
      "activations/layer17_attention_weight_max": 32.14558410644531,
      "activations/layer17_attention_weight_min": -27.163677215576172,
      "activations/layer18_attention_weight_max": 34.65777587890625,
      "activations/layer18_attention_weight_min": -25.491500854492188,
      "activations/layer19_attention_weight_max": 33.947715759277344,
      "activations/layer19_attention_weight_min": -32.541988372802734,
      "activations/layer1_attention_weight_max": 16.830720901489258,
      "activations/layer1_attention_weight_min": -15.679579734802246,
      "activations/layer20_attention_weight_max": 29.740928649902344,
      "activations/layer20_attention_weight_min": -25.795358657836914,
      "activations/layer21_attention_weight_max": 29.348323822021484,
      "activations/layer21_attention_weight_min": -27.21477508544922,
      "activations/layer22_attention_weight_max": 40.295101165771484,
      "activations/layer22_attention_weight_min": -31.89790916442871,
      "activations/layer23_attention_weight_max": 31.510826110839844,
      "activations/layer23_attention_weight_min": -27.026592254638672,
      "activations/layer2_attention_weight_max": 33.57536315917969,
      "activations/layer2_attention_weight_min": -34.62370300292969,
      "activations/layer3_attention_weight_max": 106.2457504272461,
      "activations/layer3_attention_weight_min": -109.6223373413086,
      "activations/layer4_attention_weight_max": 110.32576751708984,
      "activations/layer4_attention_weight_min": -113.2303466796875,
      "activations/layer5_attention_weight_max": 51.450279235839844,
      "activations/layer5_attention_weight_min": -60.647579193115234,
      "activations/layer6_attention_weight_max": 47.711124420166016,
      "activations/layer6_attention_weight_min": -48.11021423339844,
      "activations/layer7_attention_weight_max": 100.94369506835938,
      "activations/layer7_attention_weight_min": -102.4793472290039,
      "activations/layer8_attention_weight_max": 47.51506423950195,
      "activations/layer8_attention_weight_min": -48.64082717895508,
      "activations/layer9_attention_weight_max": 37.144832611083984,
      "activations/layer9_attention_weight_min": -36.69915008544922,
      "epoch": 21.3,
      "learning_rate": 1.2797727272727272e-05,
      "loss": 2.7326,
      "step": 366500
    },
    {
      "activations/layer0_attention_weight_max": 18.048154830932617,
      "activations/layer0_attention_weight_min": -13.197025299072266,
      "activations/layer10_attention_weight_max": 37.95901107788086,
      "activations/layer10_attention_weight_min": -38.34318923950195,
      "activations/layer11_attention_weight_max": 35.475364685058594,
      "activations/layer11_attention_weight_min": -35.761436462402344,
      "activations/layer12_attention_weight_max": 26.15576934814453,
      "activations/layer12_attention_weight_min": -26.02227210998535,
      "activations/layer13_attention_weight_max": 38.191497802734375,
      "activations/layer13_attention_weight_min": -36.76934051513672,
      "activations/layer14_attention_weight_max": 43.876895904541016,
      "activations/layer14_attention_weight_min": -39.686954498291016,
      "activations/layer15_attention_weight_max": 42.43307876586914,
      "activations/layer15_attention_weight_min": -38.632530212402344,
      "activations/layer16_attention_weight_max": 29.37974739074707,
      "activations/layer16_attention_weight_min": -29.589275360107422,
      "activations/layer17_attention_weight_max": 29.497610092163086,
      "activations/layer17_attention_weight_min": -27.32415008544922,
      "activations/layer18_attention_weight_max": 30.93780517578125,
      "activations/layer18_attention_weight_min": -27.250844955444336,
      "activations/layer19_attention_weight_max": 40.77108383178711,
      "activations/layer19_attention_weight_min": -31.48050308227539,
      "activations/layer1_attention_weight_max": 16.50050163269043,
      "activations/layer1_attention_weight_min": -15.545707702636719,
      "activations/layer20_attention_weight_max": 32.84456253051758,
      "activations/layer20_attention_weight_min": -25.443944931030273,
      "activations/layer21_attention_weight_max": 30.021024703979492,
      "activations/layer21_attention_weight_min": -24.095232009887695,
      "activations/layer22_attention_weight_max": 40.6047248840332,
      "activations/layer22_attention_weight_min": -32.16124725341797,
      "activations/layer23_attention_weight_max": 32.109432220458984,
      "activations/layer23_attention_weight_min": -27.87689208984375,
      "activations/layer2_attention_weight_max": 33.58597183227539,
      "activations/layer2_attention_weight_min": -33.164695739746094,
      "activations/layer3_attention_weight_max": 106.92265319824219,
      "activations/layer3_attention_weight_min": -109.12126922607422,
      "activations/layer4_attention_weight_max": 111.8642807006836,
      "activations/layer4_attention_weight_min": -113.2946548461914,
      "activations/layer5_attention_weight_max": 49.94155502319336,
      "activations/layer5_attention_weight_min": -62.369651794433594,
      "activations/layer6_attention_weight_max": 44.10588455200195,
      "activations/layer6_attention_weight_min": -47.2516975402832,
      "activations/layer7_attention_weight_max": 97.25802612304688,
      "activations/layer7_attention_weight_min": -105.63390350341797,
      "activations/layer8_attention_weight_max": 44.30123519897461,
      "activations/layer8_attention_weight_min": -46.15358352661133,
      "activations/layer9_attention_weight_max": 33.407283782958984,
      "activations/layer9_attention_weight_min": -36.80731964111328,
      "epoch": 21.3,
      "learning_rate": 1.2778787878787879e-05,
      "loss": 2.7027,
      "step": 366550
    },
    {
      "activations/layer0_attention_weight_max": 17.226478576660156,
      "activations/layer0_attention_weight_min": -14.017696380615234,
      "activations/layer10_attention_weight_max": 38.3134651184082,
      "activations/layer10_attention_weight_min": -37.18766403198242,
      "activations/layer11_attention_weight_max": 40.174190521240234,
      "activations/layer11_attention_weight_min": -37.15486526489258,
      "activations/layer12_attention_weight_max": 31.18518829345703,
      "activations/layer12_attention_weight_min": -29.994930267333984,
      "activations/layer13_attention_weight_max": 50.40835189819336,
      "activations/layer13_attention_weight_min": -41.965850830078125,
      "activations/layer14_attention_weight_max": 50.54419708251953,
      "activations/layer14_attention_weight_min": -42.7413215637207,
      "activations/layer15_attention_weight_max": 46.87546920776367,
      "activations/layer15_attention_weight_min": -42.71131134033203,
      "activations/layer16_attention_weight_max": 33.28067398071289,
      "activations/layer16_attention_weight_min": -29.909442901611328,
      "activations/layer17_attention_weight_max": 31.562042236328125,
      "activations/layer17_attention_weight_min": -28.141279220581055,
      "activations/layer18_attention_weight_max": 34.85649108886719,
      "activations/layer18_attention_weight_min": -30.99785804748535,
      "activations/layer19_attention_weight_max": 37.6817626953125,
      "activations/layer19_attention_weight_min": -33.95713806152344,
      "activations/layer1_attention_weight_max": 15.38300895690918,
      "activations/layer1_attention_weight_min": -14.026137351989746,
      "activations/layer20_attention_weight_max": 31.238962173461914,
      "activations/layer20_attention_weight_min": -26.78472137451172,
      "activations/layer21_attention_weight_max": 30.50699234008789,
      "activations/layer21_attention_weight_min": -25.652328491210938,
      "activations/layer22_attention_weight_max": 43.186920166015625,
      "activations/layer22_attention_weight_min": -33.89405059814453,
      "activations/layer23_attention_weight_max": 36.243141174316406,
      "activations/layer23_attention_weight_min": -26.82343864440918,
      "activations/layer2_attention_weight_max": 33.57468032836914,
      "activations/layer2_attention_weight_min": -31.927364349365234,
      "activations/layer3_attention_weight_max": 106.67089080810547,
      "activations/layer3_attention_weight_min": -108.0096435546875,
      "activations/layer4_attention_weight_max": 110.10880279541016,
      "activations/layer4_attention_weight_min": -115.00758361816406,
      "activations/layer5_attention_weight_max": 48.95604705810547,
      "activations/layer5_attention_weight_min": -63.03612518310547,
      "activations/layer6_attention_weight_max": 45.65695571899414,
      "activations/layer6_attention_weight_min": -49.530540466308594,
      "activations/layer7_attention_weight_max": 96.76188659667969,
      "activations/layer7_attention_weight_min": -103.0307846069336,
      "activations/layer8_attention_weight_max": 47.71535110473633,
      "activations/layer8_attention_weight_min": -45.6483268737793,
      "activations/layer9_attention_weight_max": 35.71895980834961,
      "activations/layer9_attention_weight_min": -36.98826217651367,
      "epoch": 21.3,
      "learning_rate": 1.2759848484848483e-05,
      "loss": 2.7033,
      "step": 366600
    },
    {
      "activations/layer0_attention_weight_max": 16.51495933532715,
      "activations/layer0_attention_weight_min": -13.513107299804688,
      "activations/layer10_attention_weight_max": 38.68864059448242,
      "activations/layer10_attention_weight_min": -37.14790725708008,
      "activations/layer11_attention_weight_max": 36.59447479248047,
      "activations/layer11_attention_weight_min": -35.98072052001953,
      "activations/layer12_attention_weight_max": 28.72498893737793,
      "activations/layer12_attention_weight_min": -30.007524490356445,
      "activations/layer13_attention_weight_max": 43.3565788269043,
      "activations/layer13_attention_weight_min": -39.43278121948242,
      "activations/layer14_attention_weight_max": 44.00542449951172,
      "activations/layer14_attention_weight_min": -42.11503219604492,
      "activations/layer15_attention_weight_max": 37.98910140991211,
      "activations/layer15_attention_weight_min": -36.73655700683594,
      "activations/layer16_attention_weight_max": 32.933082580566406,
      "activations/layer16_attention_weight_min": -30.243717193603516,
      "activations/layer17_attention_weight_max": 29.88991355895996,
      "activations/layer17_attention_weight_min": -25.966506958007812,
      "activations/layer18_attention_weight_max": 30.300434112548828,
      "activations/layer18_attention_weight_min": -23.20570945739746,
      "activations/layer19_attention_weight_max": 31.240468978881836,
      "activations/layer19_attention_weight_min": -31.393613815307617,
      "activations/layer1_attention_weight_max": 16.1641788482666,
      "activations/layer1_attention_weight_min": -15.441600799560547,
      "activations/layer20_attention_weight_max": 26.062698364257812,
      "activations/layer20_attention_weight_min": -23.701595306396484,
      "activations/layer21_attention_weight_max": 27.981565475463867,
      "activations/layer21_attention_weight_min": -23.981069564819336,
      "activations/layer22_attention_weight_max": 37.30280685424805,
      "activations/layer22_attention_weight_min": -28.973894119262695,
      "activations/layer23_attention_weight_max": 30.12154197692871,
      "activations/layer23_attention_weight_min": -26.929485321044922,
      "activations/layer2_attention_weight_max": 33.85907745361328,
      "activations/layer2_attention_weight_min": -35.336891174316406,
      "activations/layer3_attention_weight_max": 106.65665435791016,
      "activations/layer3_attention_weight_min": -110.49557495117188,
      "activations/layer4_attention_weight_max": 113.49867248535156,
      "activations/layer4_attention_weight_min": -118.62142181396484,
      "activations/layer5_attention_weight_max": 52.560218811035156,
      "activations/layer5_attention_weight_min": -66.5646743774414,
      "activations/layer6_attention_weight_max": 46.977684020996094,
      "activations/layer6_attention_weight_min": -50.871028900146484,
      "activations/layer7_attention_weight_max": 99.58639526367188,
      "activations/layer7_attention_weight_min": -105.75638580322266,
      "activations/layer8_attention_weight_max": 46.18061065673828,
      "activations/layer8_attention_weight_min": -47.26214599609375,
      "activations/layer9_attention_weight_max": 34.104087829589844,
      "activations/layer9_attention_weight_min": -36.78748321533203,
      "epoch": 21.3,
      "learning_rate": 1.274090909090909e-05,
      "loss": 2.7097,
      "step": 366650
    },
    {
      "activations/layer0_attention_weight_max": 16.341026306152344,
      "activations/layer0_attention_weight_min": -14.204593658447266,
      "activations/layer10_attention_weight_max": 37.09616470336914,
      "activations/layer10_attention_weight_min": -37.65781021118164,
      "activations/layer11_attention_weight_max": 34.35478210449219,
      "activations/layer11_attention_weight_min": -36.798500061035156,
      "activations/layer12_attention_weight_max": 27.36536979675293,
      "activations/layer12_attention_weight_min": -27.114839553833008,
      "activations/layer13_attention_weight_max": 45.50259017944336,
      "activations/layer13_attention_weight_min": -36.69367980957031,
      "activations/layer14_attention_weight_max": 49.38947677612305,
      "activations/layer14_attention_weight_min": -41.310665130615234,
      "activations/layer15_attention_weight_max": 50.300411224365234,
      "activations/layer15_attention_weight_min": -43.29966735839844,
      "activations/layer16_attention_weight_max": 35.7617073059082,
      "activations/layer16_attention_weight_min": -30.45735740661621,
      "activations/layer17_attention_weight_max": 29.906301498413086,
      "activations/layer17_attention_weight_min": -28.364152908325195,
      "activations/layer18_attention_weight_max": 31.967111587524414,
      "activations/layer18_attention_weight_min": -26.737075805664062,
      "activations/layer19_attention_weight_max": 31.0377140045166,
      "activations/layer19_attention_weight_min": -30.546783447265625,
      "activations/layer1_attention_weight_max": 15.945409774780273,
      "activations/layer1_attention_weight_min": -15.73009967803955,
      "activations/layer20_attention_weight_max": 25.801368713378906,
      "activations/layer20_attention_weight_min": -24.39226531982422,
      "activations/layer21_attention_weight_max": 27.519683837890625,
      "activations/layer21_attention_weight_min": -23.994096755981445,
      "activations/layer22_attention_weight_max": 37.206912994384766,
      "activations/layer22_attention_weight_min": -31.211063385009766,
      "activations/layer23_attention_weight_max": 27.771373748779297,
      "activations/layer23_attention_weight_min": -24.943660736083984,
      "activations/layer2_attention_weight_max": 34.573341369628906,
      "activations/layer2_attention_weight_min": -34.34130096435547,
      "activations/layer3_attention_weight_max": 107.12198638916016,
      "activations/layer3_attention_weight_min": -108.2259292602539,
      "activations/layer4_attention_weight_max": 109.13575744628906,
      "activations/layer4_attention_weight_min": -113.0508804321289,
      "activations/layer5_attention_weight_max": 50.10264205932617,
      "activations/layer5_attention_weight_min": -65.12895202636719,
      "activations/layer6_attention_weight_max": 48.46049499511719,
      "activations/layer6_attention_weight_min": -51.362037658691406,
      "activations/layer7_attention_weight_max": 101.67560577392578,
      "activations/layer7_attention_weight_min": -110.53360748291016,
      "activations/layer8_attention_weight_max": 44.02846908569336,
      "activations/layer8_attention_weight_min": -50.49775695800781,
      "activations/layer9_attention_weight_max": 36.08832931518555,
      "activations/layer9_attention_weight_min": -37.20566177368164,
      "epoch": 21.31,
      "learning_rate": 1.2721969696969697e-05,
      "loss": 2.6861,
      "step": 366700
    },
    {
      "activations/layer0_attention_weight_max": 16.342458724975586,
      "activations/layer0_attention_weight_min": -14.641916275024414,
      "activations/layer10_attention_weight_max": 34.783912658691406,
      "activations/layer10_attention_weight_min": -35.472042083740234,
      "activations/layer11_attention_weight_max": 33.599151611328125,
      "activations/layer11_attention_weight_min": -35.44874572753906,
      "activations/layer12_attention_weight_max": 27.022804260253906,
      "activations/layer12_attention_weight_min": -27.130582809448242,
      "activations/layer13_attention_weight_max": 39.71468734741211,
      "activations/layer13_attention_weight_min": -36.87004470825195,
      "activations/layer14_attention_weight_max": 43.521888732910156,
      "activations/layer14_attention_weight_min": -41.704307556152344,
      "activations/layer15_attention_weight_max": 39.48377990722656,
      "activations/layer15_attention_weight_min": -38.137245178222656,
      "activations/layer16_attention_weight_max": 29.81951904296875,
      "activations/layer16_attention_weight_min": -31.823827743530273,
      "activations/layer17_attention_weight_max": 29.967615127563477,
      "activations/layer17_attention_weight_min": -29.40414810180664,
      "activations/layer18_attention_weight_max": 30.586402893066406,
      "activations/layer18_attention_weight_min": -27.25808334350586,
      "activations/layer19_attention_weight_max": 34.61880874633789,
      "activations/layer19_attention_weight_min": -33.48143005371094,
      "activations/layer1_attention_weight_max": 16.043012619018555,
      "activations/layer1_attention_weight_min": -14.779809951782227,
      "activations/layer20_attention_weight_max": 29.615459442138672,
      "activations/layer20_attention_weight_min": -27.798831939697266,
      "activations/layer21_attention_weight_max": 28.77621078491211,
      "activations/layer21_attention_weight_min": -26.0362548828125,
      "activations/layer22_attention_weight_max": 49.06387710571289,
      "activations/layer22_attention_weight_min": -32.075233459472656,
      "activations/layer23_attention_weight_max": 30.81041717529297,
      "activations/layer23_attention_weight_min": -25.50795555114746,
      "activations/layer2_attention_weight_max": 34.249813079833984,
      "activations/layer2_attention_weight_min": -32.39582824707031,
      "activations/layer3_attention_weight_max": 102.08910369873047,
      "activations/layer3_attention_weight_min": -103.21461486816406,
      "activations/layer4_attention_weight_max": 110.5646743774414,
      "activations/layer4_attention_weight_min": -111.36810302734375,
      "activations/layer5_attention_weight_max": 53.42420959472656,
      "activations/layer5_attention_weight_min": -61.88517761230469,
      "activations/layer6_attention_weight_max": 46.41399383544922,
      "activations/layer6_attention_weight_min": -48.13253402709961,
      "activations/layer7_attention_weight_max": 93.19456481933594,
      "activations/layer7_attention_weight_min": -106.9057846069336,
      "activations/layer8_attention_weight_max": 41.930213928222656,
      "activations/layer8_attention_weight_min": -46.081172943115234,
      "activations/layer9_attention_weight_max": 32.30064010620117,
      "activations/layer9_attention_weight_min": -35.731834411621094,
      "epoch": 21.31,
      "learning_rate": 1.2703030303030301e-05,
      "loss": 2.6953,
      "step": 366750
    },
    {
      "activations/layer0_attention_weight_max": 16.597002029418945,
      "activations/layer0_attention_weight_min": -14.304217338562012,
      "activations/layer10_attention_weight_max": 37.28984451293945,
      "activations/layer10_attention_weight_min": -36.57890701293945,
      "activations/layer11_attention_weight_max": 36.6768913269043,
      "activations/layer11_attention_weight_min": -36.399566650390625,
      "activations/layer12_attention_weight_max": 28.246915817260742,
      "activations/layer12_attention_weight_min": -27.851646423339844,
      "activations/layer13_attention_weight_max": 43.47397232055664,
      "activations/layer13_attention_weight_min": -36.8385009765625,
      "activations/layer14_attention_weight_max": 44.07426071166992,
      "activations/layer14_attention_weight_min": -37.810646057128906,
      "activations/layer15_attention_weight_max": 42.671722412109375,
      "activations/layer15_attention_weight_min": -37.81627655029297,
      "activations/layer16_attention_weight_max": 31.33534812927246,
      "activations/layer16_attention_weight_min": -30.453174591064453,
      "activations/layer17_attention_weight_max": 30.430315017700195,
      "activations/layer17_attention_weight_min": -26.232786178588867,
      "activations/layer18_attention_weight_max": 32.070133209228516,
      "activations/layer18_attention_weight_min": -26.05345344543457,
      "activations/layer19_attention_weight_max": 32.6192741394043,
      "activations/layer19_attention_weight_min": -31.29721450805664,
      "activations/layer1_attention_weight_max": 16.68567657470703,
      "activations/layer1_attention_weight_min": -14.080581665039062,
      "activations/layer20_attention_weight_max": 26.732481002807617,
      "activations/layer20_attention_weight_min": -25.961042404174805,
      "activations/layer21_attention_weight_max": 26.6706600189209,
      "activations/layer21_attention_weight_min": -26.014562606811523,
      "activations/layer22_attention_weight_max": 39.553428649902344,
      "activations/layer22_attention_weight_min": -31.99880599975586,
      "activations/layer23_attention_weight_max": 33.521141052246094,
      "activations/layer23_attention_weight_min": -27.559593200683594,
      "activations/layer2_attention_weight_max": 32.974891662597656,
      "activations/layer2_attention_weight_min": -33.838111877441406,
      "activations/layer3_attention_weight_max": 105.95622253417969,
      "activations/layer3_attention_weight_min": -112.26598358154297,
      "activations/layer4_attention_weight_max": 111.83941650390625,
      "activations/layer4_attention_weight_min": -115.390869140625,
      "activations/layer5_attention_weight_max": 51.43099594116211,
      "activations/layer5_attention_weight_min": -67.09809875488281,
      "activations/layer6_attention_weight_max": 45.82258605957031,
      "activations/layer6_attention_weight_min": -50.91520309448242,
      "activations/layer7_attention_weight_max": 96.08924102783203,
      "activations/layer7_attention_weight_min": -105.67378997802734,
      "activations/layer8_attention_weight_max": 46.421653747558594,
      "activations/layer8_attention_weight_min": -51.813724517822266,
      "activations/layer9_attention_weight_max": 33.41215133666992,
      "activations/layer9_attention_weight_min": -37.1564826965332,
      "epoch": 21.31,
      "learning_rate": 1.2684090909090908e-05,
      "loss": 2.7318,
      "step": 366800
    },
    {
      "activations/layer0_attention_weight_max": 17.498939514160156,
      "activations/layer0_attention_weight_min": -13.288779258728027,
      "activations/layer10_attention_weight_max": 35.93088150024414,
      "activations/layer10_attention_weight_min": -36.62760543823242,
      "activations/layer11_attention_weight_max": 34.84400939941406,
      "activations/layer11_attention_weight_min": -37.362674713134766,
      "activations/layer12_attention_weight_max": 25.478355407714844,
      "activations/layer12_attention_weight_min": -26.063440322875977,
      "activations/layer13_attention_weight_max": 39.24339294433594,
      "activations/layer13_attention_weight_min": -35.06842803955078,
      "activations/layer14_attention_weight_max": 42.10962677001953,
      "activations/layer14_attention_weight_min": -36.99787902832031,
      "activations/layer15_attention_weight_max": 40.745059967041016,
      "activations/layer15_attention_weight_min": -36.861576080322266,
      "activations/layer16_attention_weight_max": 27.95980453491211,
      "activations/layer16_attention_weight_min": -32.302669525146484,
      "activations/layer17_attention_weight_max": 28.43024253845215,
      "activations/layer17_attention_weight_min": -25.78714942932129,
      "activations/layer18_attention_weight_max": 29.0391788482666,
      "activations/layer18_attention_weight_min": -25.11905288696289,
      "activations/layer19_attention_weight_max": 29.664321899414062,
      "activations/layer19_attention_weight_min": -31.86346435546875,
      "activations/layer1_attention_weight_max": 15.387560844421387,
      "activations/layer1_attention_weight_min": -14.75114917755127,
      "activations/layer20_attention_weight_max": 25.268468856811523,
      "activations/layer20_attention_weight_min": -27.105772018432617,
      "activations/layer21_attention_weight_max": 24.98935890197754,
      "activations/layer21_attention_weight_min": -26.266841888427734,
      "activations/layer22_attention_weight_max": 39.78427505493164,
      "activations/layer22_attention_weight_min": -30.647281646728516,
      "activations/layer23_attention_weight_max": 32.12065124511719,
      "activations/layer23_attention_weight_min": -26.111133575439453,
      "activations/layer2_attention_weight_max": 33.936859130859375,
      "activations/layer2_attention_weight_min": -32.72362518310547,
      "activations/layer3_attention_weight_max": 106.85028839111328,
      "activations/layer3_attention_weight_min": -108.47049713134766,
      "activations/layer4_attention_weight_max": 113.7238540649414,
      "activations/layer4_attention_weight_min": -113.27379608154297,
      "activations/layer5_attention_weight_max": 52.754981994628906,
      "activations/layer5_attention_weight_min": -62.08547592163086,
      "activations/layer6_attention_weight_max": 45.196449279785156,
      "activations/layer6_attention_weight_min": -49.8349723815918,
      "activations/layer7_attention_weight_max": 95.91748046875,
      "activations/layer7_attention_weight_min": -100.78086853027344,
      "activations/layer8_attention_weight_max": 42.833351135253906,
      "activations/layer8_attention_weight_min": -46.34330749511719,
      "activations/layer9_attention_weight_max": 31.967885971069336,
      "activations/layer9_attention_weight_min": -35.40656280517578,
      "epoch": 21.32,
      "learning_rate": 1.2665151515151513e-05,
      "loss": 2.7012,
      "step": 366850
    },
    {
      "activations/layer0_attention_weight_max": 16.71446418762207,
      "activations/layer0_attention_weight_min": -14.836389541625977,
      "activations/layer10_attention_weight_max": 33.14491271972656,
      "activations/layer10_attention_weight_min": -35.3913459777832,
      "activations/layer11_attention_weight_max": 33.921234130859375,
      "activations/layer11_attention_weight_min": -34.829200744628906,
      "activations/layer12_attention_weight_max": 26.112573623657227,
      "activations/layer12_attention_weight_min": -27.769271850585938,
      "activations/layer13_attention_weight_max": 38.88130187988281,
      "activations/layer13_attention_weight_min": -35.40986251831055,
      "activations/layer14_attention_weight_max": 41.383689880371094,
      "activations/layer14_attention_weight_min": -39.105369567871094,
      "activations/layer15_attention_weight_max": 36.67329406738281,
      "activations/layer15_attention_weight_min": -36.47909927368164,
      "activations/layer16_attention_weight_max": 27.6921443939209,
      "activations/layer16_attention_weight_min": -27.33649444580078,
      "activations/layer17_attention_weight_max": 29.266361236572266,
      "activations/layer17_attention_weight_min": -25.85670280456543,
      "activations/layer18_attention_weight_max": 32.37334442138672,
      "activations/layer18_attention_weight_min": -27.39509391784668,
      "activations/layer19_attention_weight_max": 34.545196533203125,
      "activations/layer19_attention_weight_min": -32.33872985839844,
      "activations/layer1_attention_weight_max": 16.04831314086914,
      "activations/layer1_attention_weight_min": -14.335267066955566,
      "activations/layer20_attention_weight_max": 28.571636199951172,
      "activations/layer20_attention_weight_min": -26.056684494018555,
      "activations/layer21_attention_weight_max": 27.174907684326172,
      "activations/layer21_attention_weight_min": -24.976451873779297,
      "activations/layer22_attention_weight_max": 41.91900634765625,
      "activations/layer22_attention_weight_min": -30.485490798950195,
      "activations/layer23_attention_weight_max": 34.47698974609375,
      "activations/layer23_attention_weight_min": -27.210742950439453,
      "activations/layer2_attention_weight_max": 34.18257522583008,
      "activations/layer2_attention_weight_min": -32.7025146484375,
      "activations/layer3_attention_weight_max": 103.87745666503906,
      "activations/layer3_attention_weight_min": -107.62215423583984,
      "activations/layer4_attention_weight_max": 106.50965881347656,
      "activations/layer4_attention_weight_min": -107.73641204833984,
      "activations/layer5_attention_weight_max": 52.173194885253906,
      "activations/layer5_attention_weight_min": -63.25298309326172,
      "activations/layer6_attention_weight_max": 43.02558135986328,
      "activations/layer6_attention_weight_min": -47.08127212524414,
      "activations/layer7_attention_weight_max": 91.20195007324219,
      "activations/layer7_attention_weight_min": -94.05294036865234,
      "activations/layer8_attention_weight_max": 40.68006896972656,
      "activations/layer8_attention_weight_min": -42.92530059814453,
      "activations/layer9_attention_weight_max": 31.2684383392334,
      "activations/layer9_attention_weight_min": -33.81744384765625,
      "epoch": 21.32,
      "learning_rate": 1.264621212121212e-05,
      "loss": 2.6876,
      "step": 366900
    },
    {
      "activations/layer0_attention_weight_max": 16.77223014831543,
      "activations/layer0_attention_weight_min": -13.646500587463379,
      "activations/layer10_attention_weight_max": 36.9931640625,
      "activations/layer10_attention_weight_min": -36.083534240722656,
      "activations/layer11_attention_weight_max": 40.186248779296875,
      "activations/layer11_attention_weight_min": -36.84736251831055,
      "activations/layer12_attention_weight_max": 26.364145278930664,
      "activations/layer12_attention_weight_min": -31.18826675415039,
      "activations/layer13_attention_weight_max": 34.96306228637695,
      "activations/layer13_attention_weight_min": -35.79319381713867,
      "activations/layer14_attention_weight_max": 41.669620513916016,
      "activations/layer14_attention_weight_min": -39.09981155395508,
      "activations/layer15_attention_weight_max": 39.553619384765625,
      "activations/layer15_attention_weight_min": -37.46518325805664,
      "activations/layer16_attention_weight_max": 29.393239974975586,
      "activations/layer16_attention_weight_min": -27.488388061523438,
      "activations/layer17_attention_weight_max": 28.72463607788086,
      "activations/layer17_attention_weight_min": -27.65115737915039,
      "activations/layer18_attention_weight_max": 33.21953201293945,
      "activations/layer18_attention_weight_min": -26.71908187866211,
      "activations/layer19_attention_weight_max": 32.18144226074219,
      "activations/layer19_attention_weight_min": -31.339435577392578,
      "activations/layer1_attention_weight_max": 15.498350143432617,
      "activations/layer1_attention_weight_min": -14.804167747497559,
      "activations/layer20_attention_weight_max": 29.000137329101562,
      "activations/layer20_attention_weight_min": -25.55712127685547,
      "activations/layer21_attention_weight_max": 26.324010848999023,
      "activations/layer21_attention_weight_min": -26.276592254638672,
      "activations/layer22_attention_weight_max": 38.786415100097656,
      "activations/layer22_attention_weight_min": -29.529918670654297,
      "activations/layer23_attention_weight_max": 29.202983856201172,
      "activations/layer23_attention_weight_min": -27.364789962768555,
      "activations/layer2_attention_weight_max": 34.095149993896484,
      "activations/layer2_attention_weight_min": -33.32597732543945,
      "activations/layer3_attention_weight_max": 106.52954864501953,
      "activations/layer3_attention_weight_min": -105.03536224365234,
      "activations/layer4_attention_weight_max": 109.8167495727539,
      "activations/layer4_attention_weight_min": -108.1651840209961,
      "activations/layer5_attention_weight_max": 50.33891296386719,
      "activations/layer5_attention_weight_min": -60.295711517333984,
      "activations/layer6_attention_weight_max": 44.058876037597656,
      "activations/layer6_attention_weight_min": -49.095890045166016,
      "activations/layer7_attention_weight_max": 102.09900665283203,
      "activations/layer7_attention_weight_min": -102.66378021240234,
      "activations/layer8_attention_weight_max": 42.660404205322266,
      "activations/layer8_attention_weight_min": -46.63961410522461,
      "activations/layer9_attention_weight_max": 32.91484069824219,
      "activations/layer9_attention_weight_min": -35.38309097290039,
      "epoch": 21.32,
      "learning_rate": 1.2627272727272726e-05,
      "loss": 2.7304,
      "step": 366950
    },
    {
      "activations/layer0_attention_weight_max": 16.902578353881836,
      "activations/layer0_attention_weight_min": -14.439105033874512,
      "activations/layer10_attention_weight_max": 39.430633544921875,
      "activations/layer10_attention_weight_min": -37.12441635131836,
      "activations/layer11_attention_weight_max": 39.121063232421875,
      "activations/layer11_attention_weight_min": -38.448883056640625,
      "activations/layer12_attention_weight_max": 32.08939743041992,
      "activations/layer12_attention_weight_min": -28.009483337402344,
      "activations/layer13_attention_weight_max": 47.709964752197266,
      "activations/layer13_attention_weight_min": -38.17943572998047,
      "activations/layer14_attention_weight_max": 47.175052642822266,
      "activations/layer14_attention_weight_min": -40.951236724853516,
      "activations/layer15_attention_weight_max": 46.53377151489258,
      "activations/layer15_attention_weight_min": -38.85927200317383,
      "activations/layer16_attention_weight_max": 33.19399642944336,
      "activations/layer16_attention_weight_min": -30.49399757385254,
      "activations/layer17_attention_weight_max": 30.80863380432129,
      "activations/layer17_attention_weight_min": -27.14664649963379,
      "activations/layer18_attention_weight_max": 33.11361312866211,
      "activations/layer18_attention_weight_min": -27.052982330322266,
      "activations/layer19_attention_weight_max": 34.84603500366211,
      "activations/layer19_attention_weight_min": -31.289459228515625,
      "activations/layer1_attention_weight_max": 14.901391983032227,
      "activations/layer1_attention_weight_min": -14.207368850708008,
      "activations/layer20_attention_weight_max": 29.976659774780273,
      "activations/layer20_attention_weight_min": -27.810535430908203,
      "activations/layer21_attention_weight_max": 28.306028366088867,
      "activations/layer21_attention_weight_min": -24.418413162231445,
      "activations/layer22_attention_weight_max": 42.00962448120117,
      "activations/layer22_attention_weight_min": -36.2116813659668,
      "activations/layer23_attention_weight_max": 34.285072326660156,
      "activations/layer23_attention_weight_min": -26.375831604003906,
      "activations/layer2_attention_weight_max": 33.85688018798828,
      "activations/layer2_attention_weight_min": -33.79297637939453,
      "activations/layer3_attention_weight_max": 108.94027709960938,
      "activations/layer3_attention_weight_min": -104.4418716430664,
      "activations/layer4_attention_weight_max": 115.7612533569336,
      "activations/layer4_attention_weight_min": -110.07038879394531,
      "activations/layer5_attention_weight_max": 51.20484161376953,
      "activations/layer5_attention_weight_min": -66.0758056640625,
      "activations/layer6_attention_weight_max": 47.584800720214844,
      "activations/layer6_attention_weight_min": -47.4946174621582,
      "activations/layer7_attention_weight_max": 104.86326599121094,
      "activations/layer7_attention_weight_min": -105.43506622314453,
      "activations/layer8_attention_weight_max": 48.29487991333008,
      "activations/layer8_attention_weight_min": -48.84714126586914,
      "activations/layer9_attention_weight_max": 34.87937545776367,
      "activations/layer9_attention_weight_min": -36.335872650146484,
      "epoch": 21.32,
      "learning_rate": 1.2608333333333331e-05,
      "loss": 2.707,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.5035,
      "eval_samples_per_second": 504.967,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.5035,
      "eval_openwebtext_samples_per_second": 504.967,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9734,
      "eval_wikitext_samples_per_second": 231.077,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_lambada_loss": 2.376953125,
      "eval_lambada_ppl": 10.77203177799245,
      "eval_lambada_runtime": 9.5784,
      "eval_lambada_samples_per_second": 508.33,
      "step": 367000
    },
    {
      "activations/layer0_attention_weight_max": 15.867474555969238,
      "activations/layer0_attention_weight_min": -14.001986503601074,
      "activations/layer10_attention_weight_max": 37.37628173828125,
      "activations/layer10_attention_weight_min": -36.83230209350586,
      "activations/layer11_attention_weight_max": 36.82097625732422,
      "activations/layer11_attention_weight_min": -35.004234313964844,
      "activations/layer12_attention_weight_max": 27.34308624267578,
      "activations/layer12_attention_weight_min": -27.115922927856445,
      "activations/layer13_attention_weight_max": 40.45034408569336,
      "activations/layer13_attention_weight_min": -37.02278518676758,
      "activations/layer14_attention_weight_max": 43.54369354248047,
      "activations/layer14_attention_weight_min": -38.229408264160156,
      "activations/layer15_attention_weight_max": 41.772518157958984,
      "activations/layer15_attention_weight_min": -36.99760818481445,
      "activations/layer16_attention_weight_max": 29.33001708984375,
      "activations/layer16_attention_weight_min": -30.47020721435547,
      "activations/layer17_attention_weight_max": 30.591312408447266,
      "activations/layer17_attention_weight_min": -27.450342178344727,
      "activations/layer18_attention_weight_max": 32.21120071411133,
      "activations/layer18_attention_weight_min": -28.869661331176758,
      "activations/layer19_attention_weight_max": 36.7774543762207,
      "activations/layer19_attention_weight_min": -34.894405364990234,
      "activations/layer1_attention_weight_max": 15.360920906066895,
      "activations/layer1_attention_weight_min": -14.850846290588379,
      "activations/layer20_attention_weight_max": 30.042423248291016,
      "activations/layer20_attention_weight_min": -27.325204849243164,
      "activations/layer21_attention_weight_max": 27.88412857055664,
      "activations/layer21_attention_weight_min": -27.01272964477539,
      "activations/layer22_attention_weight_max": 43.45512008666992,
      "activations/layer22_attention_weight_min": -35.4577751159668,
      "activations/layer23_attention_weight_max": 33.74065017700195,
      "activations/layer23_attention_weight_min": -29.776174545288086,
      "activations/layer2_attention_weight_max": 37.030296325683594,
      "activations/layer2_attention_weight_min": -34.36846923828125,
      "activations/layer3_attention_weight_max": 107.26708221435547,
      "activations/layer3_attention_weight_min": -107.85916900634766,
      "activations/layer4_attention_weight_max": 111.26090240478516,
      "activations/layer4_attention_weight_min": -116.63288879394531,
      "activations/layer5_attention_weight_max": 53.42081832885742,
      "activations/layer5_attention_weight_min": -64.03659057617188,
      "activations/layer6_attention_weight_max": 47.08015441894531,
      "activations/layer6_attention_weight_min": -49.43981170654297,
      "activations/layer7_attention_weight_max": 101.83592987060547,
      "activations/layer7_attention_weight_min": -103.41947937011719,
      "activations/layer8_attention_weight_max": 43.63102340698242,
      "activations/layer8_attention_weight_min": -49.41005325317383,
      "activations/layer9_attention_weight_max": 32.3007926940918,
      "activations/layer9_attention_weight_min": -37.46571731567383,
      "epoch": 21.33,
      "learning_rate": 1.2589393939393937e-05,
      "loss": 2.6991,
      "step": 367050
    },
    {
      "activations/layer0_attention_weight_max": 16.827274322509766,
      "activations/layer0_attention_weight_min": -13.57040023803711,
      "activations/layer10_attention_weight_max": 35.734710693359375,
      "activations/layer10_attention_weight_min": -36.863037109375,
      "activations/layer11_attention_weight_max": 34.093421936035156,
      "activations/layer11_attention_weight_min": -39.073448181152344,
      "activations/layer12_attention_weight_max": 29.28098487854004,
      "activations/layer12_attention_weight_min": -29.057889938354492,
      "activations/layer13_attention_weight_max": 41.06563186645508,
      "activations/layer13_attention_weight_min": -40.53506088256836,
      "activations/layer14_attention_weight_max": 43.88169860839844,
      "activations/layer14_attention_weight_min": -40.73359298706055,
      "activations/layer15_attention_weight_max": 39.13434600830078,
      "activations/layer15_attention_weight_min": -38.72584915161133,
      "activations/layer16_attention_weight_max": 28.017578125,
      "activations/layer16_attention_weight_min": -28.0283260345459,
      "activations/layer17_attention_weight_max": 31.98501968383789,
      "activations/layer17_attention_weight_min": -26.021560668945312,
      "activations/layer18_attention_weight_max": 32.95487594604492,
      "activations/layer18_attention_weight_min": -27.52362632751465,
      "activations/layer19_attention_weight_max": 34.50181579589844,
      "activations/layer19_attention_weight_min": -34.885047912597656,
      "activations/layer1_attention_weight_max": 15.944404602050781,
      "activations/layer1_attention_weight_min": -14.869036674499512,
      "activations/layer20_attention_weight_max": 28.777753829956055,
      "activations/layer20_attention_weight_min": -26.760465621948242,
      "activations/layer21_attention_weight_max": 30.6573486328125,
      "activations/layer21_attention_weight_min": -25.374004364013672,
      "activations/layer22_attention_weight_max": 45.59553527832031,
      "activations/layer22_attention_weight_min": -34.089385986328125,
      "activations/layer23_attention_weight_max": 33.37801742553711,
      "activations/layer23_attention_weight_min": -27.51030731201172,
      "activations/layer2_attention_weight_max": 35.56047058105469,
      "activations/layer2_attention_weight_min": -33.93745422363281,
      "activations/layer3_attention_weight_max": 111.96714782714844,
      "activations/layer3_attention_weight_min": -107.5634994506836,
      "activations/layer4_attention_weight_max": 115.27706146240234,
      "activations/layer4_attention_weight_min": -111.67308807373047,
      "activations/layer5_attention_weight_max": 54.33280563354492,
      "activations/layer5_attention_weight_min": -61.18836975097656,
      "activations/layer6_attention_weight_max": 47.00240707397461,
      "activations/layer6_attention_weight_min": -46.750450134277344,
      "activations/layer7_attention_weight_max": 96.0362319946289,
      "activations/layer7_attention_weight_min": -100.5477523803711,
      "activations/layer8_attention_weight_max": 43.364505767822266,
      "activations/layer8_attention_weight_min": -48.43388366699219,
      "activations/layer9_attention_weight_max": 34.12485122680664,
      "activations/layer9_attention_weight_min": -35.54534912109375,
      "epoch": 21.33,
      "learning_rate": 1.2570454545454546e-05,
      "loss": 2.7141,
      "step": 367100
    },
    {
      "activations/layer0_attention_weight_max": 16.438064575195312,
      "activations/layer0_attention_weight_min": -12.87294864654541,
      "activations/layer10_attention_weight_max": 36.38179397583008,
      "activations/layer10_attention_weight_min": -38.495121002197266,
      "activations/layer11_attention_weight_max": 41.082618713378906,
      "activations/layer11_attention_weight_min": -40.32901382446289,
      "activations/layer12_attention_weight_max": 28.08905601501465,
      "activations/layer12_attention_weight_min": -28.035341262817383,
      "activations/layer13_attention_weight_max": 40.92864227294922,
      "activations/layer13_attention_weight_min": -37.62563705444336,
      "activations/layer14_attention_weight_max": 46.32503890991211,
      "activations/layer14_attention_weight_min": -42.201194763183594,
      "activations/layer15_attention_weight_max": 46.586570739746094,
      "activations/layer15_attention_weight_min": -39.38043212890625,
      "activations/layer16_attention_weight_max": 29.566415786743164,
      "activations/layer16_attention_weight_min": -27.888301849365234,
      "activations/layer17_attention_weight_max": 30.832977294921875,
      "activations/layer17_attention_weight_min": -26.1895694732666,
      "activations/layer18_attention_weight_max": 35.016239166259766,
      "activations/layer18_attention_weight_min": -28.168476104736328,
      "activations/layer19_attention_weight_max": 35.9897575378418,
      "activations/layer19_attention_weight_min": -30.53804588317871,
      "activations/layer1_attention_weight_max": 16.217025756835938,
      "activations/layer1_attention_weight_min": -14.748478889465332,
      "activations/layer20_attention_weight_max": 31.850122451782227,
      "activations/layer20_attention_weight_min": -25.574125289916992,
      "activations/layer21_attention_weight_max": 29.799522399902344,
      "activations/layer21_attention_weight_min": -24.69542694091797,
      "activations/layer22_attention_weight_max": 44.598670959472656,
      "activations/layer22_attention_weight_min": -31.23680877685547,
      "activations/layer23_attention_weight_max": 33.61025619506836,
      "activations/layer23_attention_weight_min": -24.85030746459961,
      "activations/layer2_attention_weight_max": 33.46022415161133,
      "activations/layer2_attention_weight_min": -33.40752410888672,
      "activations/layer3_attention_weight_max": 105.46751403808594,
      "activations/layer3_attention_weight_min": -106.02094268798828,
      "activations/layer4_attention_weight_max": 110.96158599853516,
      "activations/layer4_attention_weight_min": -112.3710708618164,
      "activations/layer5_attention_weight_max": 50.836753845214844,
      "activations/layer5_attention_weight_min": -61.91758346557617,
      "activations/layer6_attention_weight_max": 47.028465270996094,
      "activations/layer6_attention_weight_min": -49.47880172729492,
      "activations/layer7_attention_weight_max": 99.80390930175781,
      "activations/layer7_attention_weight_min": -105.57379150390625,
      "activations/layer8_attention_weight_max": 43.70286178588867,
      "activations/layer8_attention_weight_min": -47.88451385498047,
      "activations/layer9_attention_weight_max": 33.400447845458984,
      "activations/layer9_attention_weight_min": -38.18314743041992,
      "epoch": 21.33,
      "learning_rate": 1.2551893939393937e-05,
      "loss": 2.7079,
      "step": 367150
    },
    {
      "activations/layer0_attention_weight_max": 15.80577564239502,
      "activations/layer0_attention_weight_min": -14.661211013793945,
      "activations/layer10_attention_weight_max": 38.37116241455078,
      "activations/layer10_attention_weight_min": -37.29716110229492,
      "activations/layer11_attention_weight_max": 36.06968688964844,
      "activations/layer11_attention_weight_min": -38.603702545166016,
      "activations/layer12_attention_weight_max": 28.39609718322754,
      "activations/layer12_attention_weight_min": -27.704206466674805,
      "activations/layer13_attention_weight_max": 41.36194610595703,
      "activations/layer13_attention_weight_min": -37.22459411621094,
      "activations/layer14_attention_weight_max": 44.90757369995117,
      "activations/layer14_attention_weight_min": -39.01240921020508,
      "activations/layer15_attention_weight_max": 44.95506286621094,
      "activations/layer15_attention_weight_min": -37.630672454833984,
      "activations/layer16_attention_weight_max": 34.69133377075195,
      "activations/layer16_attention_weight_min": -29.87734603881836,
      "activations/layer17_attention_weight_max": 32.3244514465332,
      "activations/layer17_attention_weight_min": -28.415485382080078,
      "activations/layer18_attention_weight_max": 35.367679595947266,
      "activations/layer18_attention_weight_min": -29.04118537902832,
      "activations/layer19_attention_weight_max": 34.851924896240234,
      "activations/layer19_attention_weight_min": -31.90375328063965,
      "activations/layer1_attention_weight_max": 16.994800567626953,
      "activations/layer1_attention_weight_min": -13.877716064453125,
      "activations/layer20_attention_weight_max": 29.41832160949707,
      "activations/layer20_attention_weight_min": -25.169965744018555,
      "activations/layer21_attention_weight_max": 29.298397064208984,
      "activations/layer21_attention_weight_min": -24.348331451416016,
      "activations/layer22_attention_weight_max": 39.9019889831543,
      "activations/layer22_attention_weight_min": -34.47265625,
      "activations/layer23_attention_weight_max": 32.33720397949219,
      "activations/layer23_attention_weight_min": -25.467041015625,
      "activations/layer2_attention_weight_max": 34.09492111206055,
      "activations/layer2_attention_weight_min": -32.80034637451172,
      "activations/layer3_attention_weight_max": 108.38270568847656,
      "activations/layer3_attention_weight_min": -105.52360534667969,
      "activations/layer4_attention_weight_max": 107.61714935302734,
      "activations/layer4_attention_weight_min": -110.5962905883789,
      "activations/layer5_attention_weight_max": 52.124534606933594,
      "activations/layer5_attention_weight_min": -61.86692428588867,
      "activations/layer6_attention_weight_max": 45.85026550292969,
      "activations/layer6_attention_weight_min": -48.934749603271484,
      "activations/layer7_attention_weight_max": 103.77143859863281,
      "activations/layer7_attention_weight_min": -100.69467163085938,
      "activations/layer8_attention_weight_max": 44.736915588378906,
      "activations/layer8_attention_weight_min": -47.54875183105469,
      "activations/layer9_attention_weight_max": 34.14608383178711,
      "activations/layer9_attention_weight_min": -37.88125228881836,
      "epoch": 21.34,
      "learning_rate": 1.2532954545454545e-05,
      "loss": 2.7034,
      "step": 367200
    },
    {
      "activations/layer0_attention_weight_max": 16.111289978027344,
      "activations/layer0_attention_weight_min": -13.249358177185059,
      "activations/layer10_attention_weight_max": 33.700157165527344,
      "activations/layer10_attention_weight_min": -35.200233459472656,
      "activations/layer11_attention_weight_max": 33.35923385620117,
      "activations/layer11_attention_weight_min": -35.8812141418457,
      "activations/layer12_attention_weight_max": 28.801301956176758,
      "activations/layer12_attention_weight_min": -28.738004684448242,
      "activations/layer13_attention_weight_max": 44.582664489746094,
      "activations/layer13_attention_weight_min": -37.88420486450195,
      "activations/layer14_attention_weight_max": 47.56873321533203,
      "activations/layer14_attention_weight_min": -39.42949676513672,
      "activations/layer15_attention_weight_max": 43.912254333496094,
      "activations/layer15_attention_weight_min": -37.33474349975586,
      "activations/layer16_attention_weight_max": 30.35423469543457,
      "activations/layer16_attention_weight_min": -29.356603622436523,
      "activations/layer17_attention_weight_max": 30.7325439453125,
      "activations/layer17_attention_weight_min": -26.182804107666016,
      "activations/layer18_attention_weight_max": 32.75374984741211,
      "activations/layer18_attention_weight_min": -27.285593032836914,
      "activations/layer19_attention_weight_max": 35.8580436706543,
      "activations/layer19_attention_weight_min": -30.332082748413086,
      "activations/layer1_attention_weight_max": 15.37261962890625,
      "activations/layer1_attention_weight_min": -14.373373031616211,
      "activations/layer20_attention_weight_max": 30.242015838623047,
      "activations/layer20_attention_weight_min": -24.832700729370117,
      "activations/layer21_attention_weight_max": 31.587100982666016,
      "activations/layer21_attention_weight_min": -25.0897274017334,
      "activations/layer22_attention_weight_max": 42.168975830078125,
      "activations/layer22_attention_weight_min": -30.847469329833984,
      "activations/layer23_attention_weight_max": 32.63060760498047,
      "activations/layer23_attention_weight_min": -26.958858489990234,
      "activations/layer2_attention_weight_max": 33.847984313964844,
      "activations/layer2_attention_weight_min": -33.52303695678711,
      "activations/layer3_attention_weight_max": 105.35895538330078,
      "activations/layer3_attention_weight_min": -104.66746520996094,
      "activations/layer4_attention_weight_max": 107.6925277709961,
      "activations/layer4_attention_weight_min": -114.61669158935547,
      "activations/layer5_attention_weight_max": 49.96192169189453,
      "activations/layer5_attention_weight_min": -69.70525360107422,
      "activations/layer6_attention_weight_max": 49.23723602294922,
      "activations/layer6_attention_weight_min": -50.12971496582031,
      "activations/layer7_attention_weight_max": 100.85712432861328,
      "activations/layer7_attention_weight_min": -103.96316528320312,
      "activations/layer8_attention_weight_max": 45.78459548950195,
      "activations/layer8_attention_weight_min": -46.4632682800293,
      "activations/layer9_attention_weight_max": 33.900596618652344,
      "activations/layer9_attention_weight_min": -35.04069137573242,
      "epoch": 21.34,
      "learning_rate": 1.2514015151515151e-05,
      "loss": 2.7083,
      "step": 367250
    },
    {
      "activations/layer0_attention_weight_max": 16.030685424804688,
      "activations/layer0_attention_weight_min": -14.251692771911621,
      "activations/layer10_attention_weight_max": 33.919254302978516,
      "activations/layer10_attention_weight_min": -36.258968353271484,
      "activations/layer11_attention_weight_max": 36.632381439208984,
      "activations/layer11_attention_weight_min": -37.12738800048828,
      "activations/layer12_attention_weight_max": 27.523954391479492,
      "activations/layer12_attention_weight_min": -26.467321395874023,
      "activations/layer13_attention_weight_max": 41.907135009765625,
      "activations/layer13_attention_weight_min": -36.61404037475586,
      "activations/layer14_attention_weight_max": 44.660430908203125,
      "activations/layer14_attention_weight_min": -38.07661819458008,
      "activations/layer15_attention_weight_max": 40.81724166870117,
      "activations/layer15_attention_weight_min": -34.50594711303711,
      "activations/layer16_attention_weight_max": 31.52974510192871,
      "activations/layer16_attention_weight_min": -27.655834197998047,
      "activations/layer17_attention_weight_max": 30.78632926940918,
      "activations/layer17_attention_weight_min": -24.471054077148438,
      "activations/layer18_attention_weight_max": 33.66444396972656,
      "activations/layer18_attention_weight_min": -25.83307647705078,
      "activations/layer19_attention_weight_max": 34.05358123779297,
      "activations/layer19_attention_weight_min": -29.75724220275879,
      "activations/layer1_attention_weight_max": 15.055005073547363,
      "activations/layer1_attention_weight_min": -15.188536643981934,
      "activations/layer20_attention_weight_max": 30.589832305908203,
      "activations/layer20_attention_weight_min": -24.14293670654297,
      "activations/layer21_attention_weight_max": 30.267669677734375,
      "activations/layer21_attention_weight_min": -22.72473907470703,
      "activations/layer22_attention_weight_max": 44.01723098754883,
      "activations/layer22_attention_weight_min": -30.4494686126709,
      "activations/layer23_attention_weight_max": 34.66217041015625,
      "activations/layer23_attention_weight_min": -24.955684661865234,
      "activations/layer2_attention_weight_max": 33.344669342041016,
      "activations/layer2_attention_weight_min": -33.5633659362793,
      "activations/layer3_attention_weight_max": 103.36565399169922,
      "activations/layer3_attention_weight_min": -110.22298431396484,
      "activations/layer4_attention_weight_max": 109.77813720703125,
      "activations/layer4_attention_weight_min": -111.8797378540039,
      "activations/layer5_attention_weight_max": 52.40230941772461,
      "activations/layer5_attention_weight_min": -63.344383239746094,
      "activations/layer6_attention_weight_max": 44.61636734008789,
      "activations/layer6_attention_weight_min": -47.78675079345703,
      "activations/layer7_attention_weight_max": 97.07270050048828,
      "activations/layer7_attention_weight_min": -101.12899017333984,
      "activations/layer8_attention_weight_max": 43.15334701538086,
      "activations/layer8_attention_weight_min": -47.14288330078125,
      "activations/layer9_attention_weight_max": 32.699642181396484,
      "activations/layer9_attention_weight_min": -37.06911087036133,
      "epoch": 21.34,
      "learning_rate": 1.2495075757575758e-05,
      "loss": 2.722,
      "step": 367300
    },
    {
      "activations/layer0_attention_weight_max": 16.292476654052734,
      "activations/layer0_attention_weight_min": -14.368083953857422,
      "activations/layer10_attention_weight_max": 36.68490982055664,
      "activations/layer10_attention_weight_min": -35.81665802001953,
      "activations/layer11_attention_weight_max": 34.97617721557617,
      "activations/layer11_attention_weight_min": -34.68769073486328,
      "activations/layer12_attention_weight_max": 25.083633422851562,
      "activations/layer12_attention_weight_min": -26.944124221801758,
      "activations/layer13_attention_weight_max": 37.52349853515625,
      "activations/layer13_attention_weight_min": -36.95954513549805,
      "activations/layer14_attention_weight_max": 40.93608856201172,
      "activations/layer14_attention_weight_min": -41.93049240112305,
      "activations/layer15_attention_weight_max": 38.34782409667969,
      "activations/layer15_attention_weight_min": -38.80722427368164,
      "activations/layer16_attention_weight_max": 28.74090576171875,
      "activations/layer16_attention_weight_min": -27.56010627746582,
      "activations/layer17_attention_weight_max": 29.840044021606445,
      "activations/layer17_attention_weight_min": -25.828977584838867,
      "activations/layer18_attention_weight_max": 34.24436569213867,
      "activations/layer18_attention_weight_min": -26.213048934936523,
      "activations/layer19_attention_weight_max": 30.26944351196289,
      "activations/layer19_attention_weight_min": -31.451358795166016,
      "activations/layer1_attention_weight_max": 16.340307235717773,
      "activations/layer1_attention_weight_min": -15.327240943908691,
      "activations/layer20_attention_weight_max": 28.08233642578125,
      "activations/layer20_attention_weight_min": -24.319364547729492,
      "activations/layer21_attention_weight_max": 29.57610321044922,
      "activations/layer21_attention_weight_min": -24.286174774169922,
      "activations/layer22_attention_weight_max": 43.106101989746094,
      "activations/layer22_attention_weight_min": -30.147560119628906,
      "activations/layer23_attention_weight_max": 29.668434143066406,
      "activations/layer23_attention_weight_min": -26.620777130126953,
      "activations/layer2_attention_weight_max": 32.36348342895508,
      "activations/layer2_attention_weight_min": -32.065528869628906,
      "activations/layer3_attention_weight_max": 102.99811553955078,
      "activations/layer3_attention_weight_min": -105.46983337402344,
      "activations/layer4_attention_weight_max": 107.3717041015625,
      "activations/layer4_attention_weight_min": -108.63459777832031,
      "activations/layer5_attention_weight_max": 50.16520309448242,
      "activations/layer5_attention_weight_min": -64.99269104003906,
      "activations/layer6_attention_weight_max": 45.37596893310547,
      "activations/layer6_attention_weight_min": -47.669960021972656,
      "activations/layer7_attention_weight_max": 101.2197036743164,
      "activations/layer7_attention_weight_min": -101.68934631347656,
      "activations/layer8_attention_weight_max": 45.06001663208008,
      "activations/layer8_attention_weight_min": -47.90120315551758,
      "activations/layer9_attention_weight_max": 33.85191345214844,
      "activations/layer9_attention_weight_min": -35.252750396728516,
      "epoch": 21.35,
      "learning_rate": 1.2476136363636363e-05,
      "loss": 2.7092,
      "step": 367350
    },
    {
      "activations/layer0_attention_weight_max": 16.276334762573242,
      "activations/layer0_attention_weight_min": -12.356878280639648,
      "activations/layer10_attention_weight_max": 35.87529754638672,
      "activations/layer10_attention_weight_min": -35.38494110107422,
      "activations/layer11_attention_weight_max": 35.27311706542969,
      "activations/layer11_attention_weight_min": -34.70542907714844,
      "activations/layer12_attention_weight_max": 34.25765609741211,
      "activations/layer12_attention_weight_min": -29.35586929321289,
      "activations/layer13_attention_weight_max": 47.43848419189453,
      "activations/layer13_attention_weight_min": -36.76215744018555,
      "activations/layer14_attention_weight_max": 42.780921936035156,
      "activations/layer14_attention_weight_min": -38.84160232543945,
      "activations/layer15_attention_weight_max": 44.190887451171875,
      "activations/layer15_attention_weight_min": -37.18606948852539,
      "activations/layer16_attention_weight_max": 32.02250671386719,
      "activations/layer16_attention_weight_min": -29.27678108215332,
      "activations/layer17_attention_weight_max": 30.31670570373535,
      "activations/layer17_attention_weight_min": -26.499954223632812,
      "activations/layer18_attention_weight_max": 30.102108001708984,
      "activations/layer18_attention_weight_min": -25.28553581237793,
      "activations/layer19_attention_weight_max": 36.407840728759766,
      "activations/layer19_attention_weight_min": -31.112680435180664,
      "activations/layer1_attention_weight_max": 15.9854097366333,
      "activations/layer1_attention_weight_min": -14.874371528625488,
      "activations/layer20_attention_weight_max": 31.973108291625977,
      "activations/layer20_attention_weight_min": -26.9771671295166,
      "activations/layer21_attention_weight_max": 32.08107376098633,
      "activations/layer21_attention_weight_min": -25.776748657226562,
      "activations/layer22_attention_weight_max": 38.992919921875,
      "activations/layer22_attention_weight_min": -29.97895050048828,
      "activations/layer23_attention_weight_max": 31.057781219482422,
      "activations/layer23_attention_weight_min": -23.925146102905273,
      "activations/layer2_attention_weight_max": 34.824947357177734,
      "activations/layer2_attention_weight_min": -33.17084884643555,
      "activations/layer3_attention_weight_max": 107.09980010986328,
      "activations/layer3_attention_weight_min": -105.58413696289062,
      "activations/layer4_attention_weight_max": 106.32958984375,
      "activations/layer4_attention_weight_min": -116.08576965332031,
      "activations/layer5_attention_weight_max": 50.39295196533203,
      "activations/layer5_attention_weight_min": -59.00545883178711,
      "activations/layer6_attention_weight_max": 44.43031311035156,
      "activations/layer6_attention_weight_min": -47.478721618652344,
      "activations/layer7_attention_weight_max": 94.4415283203125,
      "activations/layer7_attention_weight_min": -100.97635650634766,
      "activations/layer8_attention_weight_max": 43.397579193115234,
      "activations/layer8_attention_weight_min": -46.05024337768555,
      "activations/layer9_attention_weight_max": 32.828250885009766,
      "activations/layer9_attention_weight_min": -34.5384635925293,
      "epoch": 21.35,
      "learning_rate": 1.245719696969697e-05,
      "loss": 2.7094,
      "step": 367400
    },
    {
      "activations/layer0_attention_weight_max": 16.55223274230957,
      "activations/layer0_attention_weight_min": -15.257865905761719,
      "activations/layer10_attention_weight_max": 33.70226287841797,
      "activations/layer10_attention_weight_min": -34.150699615478516,
      "activations/layer11_attention_weight_max": 32.78536605834961,
      "activations/layer11_attention_weight_min": -33.48442077636719,
      "activations/layer12_attention_weight_max": 26.108137130737305,
      "activations/layer12_attention_weight_min": -27.797744750976562,
      "activations/layer13_attention_weight_max": 38.407073974609375,
      "activations/layer13_attention_weight_min": -34.57548141479492,
      "activations/layer14_attention_weight_max": 41.2253303527832,
      "activations/layer14_attention_weight_min": -37.65734100341797,
      "activations/layer15_attention_weight_max": 36.58510971069336,
      "activations/layer15_attention_weight_min": -34.71847152709961,
      "activations/layer16_attention_weight_max": 29.3970947265625,
      "activations/layer16_attention_weight_min": -29.64320945739746,
      "activations/layer17_attention_weight_max": 29.255292892456055,
      "activations/layer17_attention_weight_min": -26.410133361816406,
      "activations/layer18_attention_weight_max": 32.21552658081055,
      "activations/layer18_attention_weight_min": -25.78042984008789,
      "activations/layer19_attention_weight_max": 33.12095260620117,
      "activations/layer19_attention_weight_min": -33.353759765625,
      "activations/layer1_attention_weight_max": 15.397991180419922,
      "activations/layer1_attention_weight_min": -14.834179878234863,
      "activations/layer20_attention_weight_max": 27.79095458984375,
      "activations/layer20_attention_weight_min": -26.476346969604492,
      "activations/layer21_attention_weight_max": 29.517690658569336,
      "activations/layer21_attention_weight_min": -25.755870819091797,
      "activations/layer22_attention_weight_max": 39.58908462524414,
      "activations/layer22_attention_weight_min": -33.91141891479492,
      "activations/layer23_attention_weight_max": 32.74011993408203,
      "activations/layer23_attention_weight_min": -26.932106018066406,
      "activations/layer2_attention_weight_max": 37.40776062011719,
      "activations/layer2_attention_weight_min": -35.75702667236328,
      "activations/layer3_attention_weight_max": 107.7549819946289,
      "activations/layer3_attention_weight_min": -110.92681884765625,
      "activations/layer4_attention_weight_max": 112.2520523071289,
      "activations/layer4_attention_weight_min": -110.89996337890625,
      "activations/layer5_attention_weight_max": 50.02680969238281,
      "activations/layer5_attention_weight_min": -59.435821533203125,
      "activations/layer6_attention_weight_max": 44.93598556518555,
      "activations/layer6_attention_weight_min": -47.7545280456543,
      "activations/layer7_attention_weight_max": 93.1041259765625,
      "activations/layer7_attention_weight_min": -95.95054626464844,
      "activations/layer8_attention_weight_max": 41.30307388305664,
      "activations/layer8_attention_weight_min": -45.46135330200195,
      "activations/layer9_attention_weight_max": 31.32381248474121,
      "activations/layer9_attention_weight_min": -33.88996887207031,
      "epoch": 21.35,
      "learning_rate": 1.2438257575757574e-05,
      "loss": 2.7032,
      "step": 367450
    },
    {
      "activations/layer0_attention_weight_max": 16.097759246826172,
      "activations/layer0_attention_weight_min": -15.203448295593262,
      "activations/layer10_attention_weight_max": 33.1345100402832,
      "activations/layer10_attention_weight_min": -34.361228942871094,
      "activations/layer11_attention_weight_max": 32.807159423828125,
      "activations/layer11_attention_weight_min": -34.384765625,
      "activations/layer12_attention_weight_max": 28.148902893066406,
      "activations/layer12_attention_weight_min": -26.963394165039062,
      "activations/layer13_attention_weight_max": 37.831809997558594,
      "activations/layer13_attention_weight_min": -35.956295013427734,
      "activations/layer14_attention_weight_max": 41.97894287109375,
      "activations/layer14_attention_weight_min": -38.10972595214844,
      "activations/layer15_attention_weight_max": 40.02498245239258,
      "activations/layer15_attention_weight_min": -35.86384963989258,
      "activations/layer16_attention_weight_max": 30.544010162353516,
      "activations/layer16_attention_weight_min": -27.845949172973633,
      "activations/layer17_attention_weight_max": 28.956180572509766,
      "activations/layer17_attention_weight_min": -26.11370277404785,
      "activations/layer18_attention_weight_max": 36.44609832763672,
      "activations/layer18_attention_weight_min": -26.201217651367188,
      "activations/layer19_attention_weight_max": 42.950584411621094,
      "activations/layer19_attention_weight_min": -33.82210922241211,
      "activations/layer1_attention_weight_max": 15.336249351501465,
      "activations/layer1_attention_weight_min": -14.593215942382812,
      "activations/layer20_attention_weight_max": 32.65620422363281,
      "activations/layer20_attention_weight_min": -25.6832275390625,
      "activations/layer21_attention_weight_max": 33.25955581665039,
      "activations/layer21_attention_weight_min": -25.280263900756836,
      "activations/layer22_attention_weight_max": 46.897525787353516,
      "activations/layer22_attention_weight_min": -33.15275955200195,
      "activations/layer23_attention_weight_max": 39.21360778808594,
      "activations/layer23_attention_weight_min": -27.088150024414062,
      "activations/layer2_attention_weight_max": 32.91145706176758,
      "activations/layer2_attention_weight_min": -32.723838806152344,
      "activations/layer3_attention_weight_max": 99.50856018066406,
      "activations/layer3_attention_weight_min": -106.11492156982422,
      "activations/layer4_attention_weight_max": 105.06771087646484,
      "activations/layer4_attention_weight_min": -109.24710845947266,
      "activations/layer5_attention_weight_max": 48.93562316894531,
      "activations/layer5_attention_weight_min": -61.533782958984375,
      "activations/layer6_attention_weight_max": 44.182655334472656,
      "activations/layer6_attention_weight_min": -47.29802322387695,
      "activations/layer7_attention_weight_max": 93.48790740966797,
      "activations/layer7_attention_weight_min": -102.64682006835938,
      "activations/layer8_attention_weight_max": 43.697444915771484,
      "activations/layer8_attention_weight_min": -46.499717712402344,
      "activations/layer9_attention_weight_max": 32.89189529418945,
      "activations/layer9_attention_weight_min": -34.29183578491211,
      "epoch": 21.35,
      "learning_rate": 1.2419318181818181e-05,
      "loss": 2.7128,
      "step": 367500
    },
    {
      "activations/layer0_attention_weight_max": 16.82845115661621,
      "activations/layer0_attention_weight_min": -13.996966361999512,
      "activations/layer10_attention_weight_max": 37.9105224609375,
      "activations/layer10_attention_weight_min": -35.91606140136719,
      "activations/layer11_attention_weight_max": 37.56343078613281,
      "activations/layer11_attention_weight_min": -34.80317687988281,
      "activations/layer12_attention_weight_max": 28.25572395324707,
      "activations/layer12_attention_weight_min": -26.844783782958984,
      "activations/layer13_attention_weight_max": 41.5720100402832,
      "activations/layer13_attention_weight_min": -37.9021110534668,
      "activations/layer14_attention_weight_max": 43.70547866821289,
      "activations/layer14_attention_weight_min": -40.29762649536133,
      "activations/layer15_attention_weight_max": 40.23627853393555,
      "activations/layer15_attention_weight_min": -35.72174072265625,
      "activations/layer16_attention_weight_max": 30.176069259643555,
      "activations/layer16_attention_weight_min": -29.59224510192871,
      "activations/layer17_attention_weight_max": 30.582284927368164,
      "activations/layer17_attention_weight_min": -24.431365966796875,
      "activations/layer18_attention_weight_max": 30.323217391967773,
      "activations/layer18_attention_weight_min": -23.757545471191406,
      "activations/layer19_attention_weight_max": 34.44771194458008,
      "activations/layer19_attention_weight_min": -30.107458114624023,
      "activations/layer1_attention_weight_max": 16.485610961914062,
      "activations/layer1_attention_weight_min": -15.895689010620117,
      "activations/layer20_attention_weight_max": 30.714519500732422,
      "activations/layer20_attention_weight_min": -22.70860481262207,
      "activations/layer21_attention_weight_max": 27.68255615234375,
      "activations/layer21_attention_weight_min": -22.988815307617188,
      "activations/layer22_attention_weight_max": 40.800926208496094,
      "activations/layer22_attention_weight_min": -29.224748611450195,
      "activations/layer23_attention_weight_max": 32.566383361816406,
      "activations/layer23_attention_weight_min": -24.327266693115234,
      "activations/layer2_attention_weight_max": 36.92832946777344,
      "activations/layer2_attention_weight_min": -32.49752426147461,
      "activations/layer3_attention_weight_max": 109.22550201416016,
      "activations/layer3_attention_weight_min": -106.5658187866211,
      "activations/layer4_attention_weight_max": 114.2584457397461,
      "activations/layer4_attention_weight_min": -112.43880462646484,
      "activations/layer5_attention_weight_max": 56.35471725463867,
      "activations/layer5_attention_weight_min": -60.87618637084961,
      "activations/layer6_attention_weight_max": 49.89952850341797,
      "activations/layer6_attention_weight_min": -47.66316223144531,
      "activations/layer7_attention_weight_max": 109.46485900878906,
      "activations/layer7_attention_weight_min": -105.2643051147461,
      "activations/layer8_attention_weight_max": 46.45665740966797,
      "activations/layer8_attention_weight_min": -47.092926025390625,
      "activations/layer9_attention_weight_max": 36.62162399291992,
      "activations/layer9_attention_weight_min": -36.67299270629883,
      "epoch": 21.36,
      "learning_rate": 1.2400378787878787e-05,
      "loss": 2.711,
      "step": 367550
    },
    {
      "activations/layer0_attention_weight_max": 18.45623779296875,
      "activations/layer0_attention_weight_min": -13.258502960205078,
      "activations/layer10_attention_weight_max": 36.300601959228516,
      "activations/layer10_attention_weight_min": -37.49824905395508,
      "activations/layer11_attention_weight_max": 37.352516174316406,
      "activations/layer11_attention_weight_min": -37.901371002197266,
      "activations/layer12_attention_weight_max": 27.673486709594727,
      "activations/layer12_attention_weight_min": -28.110225677490234,
      "activations/layer13_attention_weight_max": 40.0638313293457,
      "activations/layer13_attention_weight_min": -37.52552795410156,
      "activations/layer14_attention_weight_max": 46.11252212524414,
      "activations/layer14_attention_weight_min": -40.072227478027344,
      "activations/layer15_attention_weight_max": 41.576236724853516,
      "activations/layer15_attention_weight_min": -40.01377868652344,
      "activations/layer16_attention_weight_max": 36.23847198486328,
      "activations/layer16_attention_weight_min": -28.744441986083984,
      "activations/layer17_attention_weight_max": 34.57970428466797,
      "activations/layer17_attention_weight_min": -28.33173942565918,
      "activations/layer18_attention_weight_max": 34.654449462890625,
      "activations/layer18_attention_weight_min": -28.590862274169922,
      "activations/layer19_attention_weight_max": 40.810638427734375,
      "activations/layer19_attention_weight_min": -31.930356979370117,
      "activations/layer1_attention_weight_max": 15.913297653198242,
      "activations/layer1_attention_weight_min": -15.701030731201172,
      "activations/layer20_attention_weight_max": 31.119422912597656,
      "activations/layer20_attention_weight_min": -25.139680862426758,
      "activations/layer21_attention_weight_max": 27.98048973083496,
      "activations/layer21_attention_weight_min": -23.87859535217285,
      "activations/layer22_attention_weight_max": 44.72800827026367,
      "activations/layer22_attention_weight_min": -32.35526657104492,
      "activations/layer23_attention_weight_max": 34.65846252441406,
      "activations/layer23_attention_weight_min": -27.467578887939453,
      "activations/layer2_attention_weight_max": 34.481346130371094,
      "activations/layer2_attention_weight_min": -33.091957092285156,
      "activations/layer3_attention_weight_max": 104.21200561523438,
      "activations/layer3_attention_weight_min": -104.36891174316406,
      "activations/layer4_attention_weight_max": 109.365966796875,
      "activations/layer4_attention_weight_min": -109.5191421508789,
      "activations/layer5_attention_weight_max": 51.85869598388672,
      "activations/layer5_attention_weight_min": -60.548187255859375,
      "activations/layer6_attention_weight_max": 45.659278869628906,
      "activations/layer6_attention_weight_min": -46.812992095947266,
      "activations/layer7_attention_weight_max": 97.86481475830078,
      "activations/layer7_attention_weight_min": -99.79147338867188,
      "activations/layer8_attention_weight_max": 45.04106140136719,
      "activations/layer8_attention_weight_min": -46.91756820678711,
      "activations/layer9_attention_weight_max": 33.109275817871094,
      "activations/layer9_attention_weight_min": -35.600162506103516,
      "epoch": 21.36,
      "learning_rate": 1.2381439393939392e-05,
      "loss": 2.7177,
      "step": 367600
    },
    {
      "activations/layer0_attention_weight_max": 16.19260597229004,
      "activations/layer0_attention_weight_min": -14.533648490905762,
      "activations/layer10_attention_weight_max": 36.21891403198242,
      "activations/layer10_attention_weight_min": -37.41815948486328,
      "activations/layer11_attention_weight_max": 34.14485168457031,
      "activations/layer11_attention_weight_min": -37.52427291870117,
      "activations/layer12_attention_weight_max": 27.617223739624023,
      "activations/layer12_attention_weight_min": -28.043907165527344,
      "activations/layer13_attention_weight_max": 37.963157653808594,
      "activations/layer13_attention_weight_min": -37.56559753417969,
      "activations/layer14_attention_weight_max": 42.75531768798828,
      "activations/layer14_attention_weight_min": -42.29071807861328,
      "activations/layer15_attention_weight_max": 38.49498748779297,
      "activations/layer15_attention_weight_min": -40.99898910522461,
      "activations/layer16_attention_weight_max": 29.6275634765625,
      "activations/layer16_attention_weight_min": -30.316768646240234,
      "activations/layer17_attention_weight_max": 28.415790557861328,
      "activations/layer17_attention_weight_min": -30.80611228942871,
      "activations/layer18_attention_weight_max": 32.06382751464844,
      "activations/layer18_attention_weight_min": -28.940458297729492,
      "activations/layer19_attention_weight_max": 34.57783508300781,
      "activations/layer19_attention_weight_min": -31.26788330078125,
      "activations/layer1_attention_weight_max": 15.381135940551758,
      "activations/layer1_attention_weight_min": -13.976723670959473,
      "activations/layer20_attention_weight_max": 27.68495750427246,
      "activations/layer20_attention_weight_min": -25.165258407592773,
      "activations/layer21_attention_weight_max": 28.568565368652344,
      "activations/layer21_attention_weight_min": -25.82297134399414,
      "activations/layer22_attention_weight_max": 38.74009323120117,
      "activations/layer22_attention_weight_min": -31.795772552490234,
      "activations/layer23_attention_weight_max": 30.33489227294922,
      "activations/layer23_attention_weight_min": -26.703962326049805,
      "activations/layer2_attention_weight_max": 32.16001892089844,
      "activations/layer2_attention_weight_min": -32.66629409790039,
      "activations/layer3_attention_weight_max": 104.27589416503906,
      "activations/layer3_attention_weight_min": -102.12237548828125,
      "activations/layer4_attention_weight_max": 109.43387603759766,
      "activations/layer4_attention_weight_min": -112.79957580566406,
      "activations/layer5_attention_weight_max": 50.5653190612793,
      "activations/layer5_attention_weight_min": -63.74846649169922,
      "activations/layer6_attention_weight_max": 45.42011642456055,
      "activations/layer6_attention_weight_min": -47.95326614379883,
      "activations/layer7_attention_weight_max": 93.99032592773438,
      "activations/layer7_attention_weight_min": -97.81858825683594,
      "activations/layer8_attention_weight_max": 43.8990364074707,
      "activations/layer8_attention_weight_min": -46.90437316894531,
      "activations/layer9_attention_weight_max": 34.6225700378418,
      "activations/layer9_attention_weight_min": -38.00810241699219,
      "epoch": 21.36,
      "learning_rate": 1.2362878787878787e-05,
      "loss": 2.7026,
      "step": 367650
    },
    {
      "activations/layer0_attention_weight_max": 16.937036514282227,
      "activations/layer0_attention_weight_min": -14.295498847961426,
      "activations/layer10_attention_weight_max": 39.35295867919922,
      "activations/layer10_attention_weight_min": -34.86763381958008,
      "activations/layer11_attention_weight_max": 35.980064392089844,
      "activations/layer11_attention_weight_min": -36.21363830566406,
      "activations/layer12_attention_weight_max": 28.283361434936523,
      "activations/layer12_attention_weight_min": -27.89439582824707,
      "activations/layer13_attention_weight_max": 40.15517807006836,
      "activations/layer13_attention_weight_min": -36.31733703613281,
      "activations/layer14_attention_weight_max": 46.66108322143555,
      "activations/layer14_attention_weight_min": -40.784645080566406,
      "activations/layer15_attention_weight_max": 41.984619140625,
      "activations/layer15_attention_weight_min": -38.60622024536133,
      "activations/layer16_attention_weight_max": 31.50102424621582,
      "activations/layer16_attention_weight_min": -30.191389083862305,
      "activations/layer17_attention_weight_max": 31.610002517700195,
      "activations/layer17_attention_weight_min": -29.46714973449707,
      "activations/layer18_attention_weight_max": 30.875761032104492,
      "activations/layer18_attention_weight_min": -25.936996459960938,
      "activations/layer19_attention_weight_max": 35.7902946472168,
      "activations/layer19_attention_weight_min": -34.2880744934082,
      "activations/layer1_attention_weight_max": 15.663240432739258,
      "activations/layer1_attention_weight_min": -14.71522331237793,
      "activations/layer20_attention_weight_max": 29.902542114257812,
      "activations/layer20_attention_weight_min": -26.221153259277344,
      "activations/layer21_attention_weight_max": 30.408071517944336,
      "activations/layer21_attention_weight_min": -25.017929077148438,
      "activations/layer22_attention_weight_max": 42.626502990722656,
      "activations/layer22_attention_weight_min": -30.571746826171875,
      "activations/layer23_attention_weight_max": 30.12916374206543,
      "activations/layer23_attention_weight_min": -24.984729766845703,
      "activations/layer2_attention_weight_max": 33.61212921142578,
      "activations/layer2_attention_weight_min": -32.65852737426758,
      "activations/layer3_attention_weight_max": 101.58238983154297,
      "activations/layer3_attention_weight_min": -105.14965057373047,
      "activations/layer4_attention_weight_max": 108.38916015625,
      "activations/layer4_attention_weight_min": -113.7646484375,
      "activations/layer5_attention_weight_max": 49.028221130371094,
      "activations/layer5_attention_weight_min": -62.341461181640625,
      "activations/layer6_attention_weight_max": 45.53801345825195,
      "activations/layer6_attention_weight_min": -47.564449310302734,
      "activations/layer7_attention_weight_max": 94.19140625,
      "activations/layer7_attention_weight_min": -100.63084411621094,
      "activations/layer8_attention_weight_max": 44.088966369628906,
      "activations/layer8_attention_weight_min": -46.43593215942383,
      "activations/layer9_attention_weight_max": 30.65542984008789,
      "activations/layer9_attention_weight_min": -33.864776611328125,
      "epoch": 21.37,
      "learning_rate": 1.2343939393939393e-05,
      "loss": 2.6972,
      "step": 367700
    },
    {
      "activations/layer0_attention_weight_max": 16.550111770629883,
      "activations/layer0_attention_weight_min": -13.565389633178711,
      "activations/layer10_attention_weight_max": 36.26761245727539,
      "activations/layer10_attention_weight_min": -34.20108413696289,
      "activations/layer11_attention_weight_max": 38.75725555419922,
      "activations/layer11_attention_weight_min": -36.090660095214844,
      "activations/layer12_attention_weight_max": 27.8351993560791,
      "activations/layer12_attention_weight_min": -32.83158874511719,
      "activations/layer13_attention_weight_max": 41.29293441772461,
      "activations/layer13_attention_weight_min": -36.04042053222656,
      "activations/layer14_attention_weight_max": 40.12142562866211,
      "activations/layer14_attention_weight_min": -35.46415328979492,
      "activations/layer15_attention_weight_max": 37.292076110839844,
      "activations/layer15_attention_weight_min": -34.55867004394531,
      "activations/layer16_attention_weight_max": 30.368391036987305,
      "activations/layer16_attention_weight_min": -28.026517868041992,
      "activations/layer17_attention_weight_max": 32.7279167175293,
      "activations/layer17_attention_weight_min": -27.48806381225586,
      "activations/layer18_attention_weight_max": 32.689693450927734,
      "activations/layer18_attention_weight_min": -25.242162704467773,
      "activations/layer19_attention_weight_max": 31.0162410736084,
      "activations/layer19_attention_weight_min": -30.308401107788086,
      "activations/layer1_attention_weight_max": 15.72801399230957,
      "activations/layer1_attention_weight_min": -14.861283302307129,
      "activations/layer20_attention_weight_max": 27.026615142822266,
      "activations/layer20_attention_weight_min": -27.921539306640625,
      "activations/layer21_attention_weight_max": 26.595481872558594,
      "activations/layer21_attention_weight_min": -24.017986297607422,
      "activations/layer22_attention_weight_max": 40.66765213012695,
      "activations/layer22_attention_weight_min": -33.23004913330078,
      "activations/layer23_attention_weight_max": 30.18157958984375,
      "activations/layer23_attention_weight_min": -26.484052658081055,
      "activations/layer2_attention_weight_max": 33.37982177734375,
      "activations/layer2_attention_weight_min": -33.176177978515625,
      "activations/layer3_attention_weight_max": 105.0082015991211,
      "activations/layer3_attention_weight_min": -106.49504089355469,
      "activations/layer4_attention_weight_max": 109.37242889404297,
      "activations/layer4_attention_weight_min": -115.4211196899414,
      "activations/layer5_attention_weight_max": 51.3560905456543,
      "activations/layer5_attention_weight_min": -62.906044006347656,
      "activations/layer6_attention_weight_max": 45.59622573852539,
      "activations/layer6_attention_weight_min": -49.53285217285156,
      "activations/layer7_attention_weight_max": 95.95038604736328,
      "activations/layer7_attention_weight_min": -99.09126281738281,
      "activations/layer8_attention_weight_max": 44.61119842529297,
      "activations/layer8_attention_weight_min": -46.121070861816406,
      "activations/layer9_attention_weight_max": 33.03219223022461,
      "activations/layer9_attention_weight_min": -35.47150802612305,
      "epoch": 21.37,
      "learning_rate": 1.2324999999999998e-05,
      "loss": 2.7108,
      "step": 367750
    },
    {
      "activations/layer0_attention_weight_max": 17.109432220458984,
      "activations/layer0_attention_weight_min": -14.979551315307617,
      "activations/layer10_attention_weight_max": 35.57466506958008,
      "activations/layer10_attention_weight_min": -35.453712463378906,
      "activations/layer11_attention_weight_max": 33.38640213012695,
      "activations/layer11_attention_weight_min": -35.574554443359375,
      "activations/layer12_attention_weight_max": 27.68767547607422,
      "activations/layer12_attention_weight_min": -26.523080825805664,
      "activations/layer13_attention_weight_max": 41.619728088378906,
      "activations/layer13_attention_weight_min": -36.272010803222656,
      "activations/layer14_attention_weight_max": 42.34228515625,
      "activations/layer14_attention_weight_min": -41.17551803588867,
      "activations/layer15_attention_weight_max": 40.461280822753906,
      "activations/layer15_attention_weight_min": -36.81351852416992,
      "activations/layer16_attention_weight_max": 28.199813842773438,
      "activations/layer16_attention_weight_min": -30.146760940551758,
      "activations/layer17_attention_weight_max": 28.95814323425293,
      "activations/layer17_attention_weight_min": -26.62196159362793,
      "activations/layer18_attention_weight_max": 31.6528263092041,
      "activations/layer18_attention_weight_min": -26.08409881591797,
      "activations/layer19_attention_weight_max": 34.25484848022461,
      "activations/layer19_attention_weight_min": -31.99892234802246,
      "activations/layer1_attention_weight_max": 15.962911605834961,
      "activations/layer1_attention_weight_min": -14.98244857788086,
      "activations/layer20_attention_weight_max": 29.467510223388672,
      "activations/layer20_attention_weight_min": -25.1236572265625,
      "activations/layer21_attention_weight_max": 30.738475799560547,
      "activations/layer21_attention_weight_min": -24.871261596679688,
      "activations/layer22_attention_weight_max": 42.62612533569336,
      "activations/layer22_attention_weight_min": -32.096214294433594,
      "activations/layer23_attention_weight_max": 32.236793518066406,
      "activations/layer23_attention_weight_min": -25.224645614624023,
      "activations/layer2_attention_weight_max": 33.47096633911133,
      "activations/layer2_attention_weight_min": -33.63383865356445,
      "activations/layer3_attention_weight_max": 105.1235580444336,
      "activations/layer3_attention_weight_min": -108.37467956542969,
      "activations/layer4_attention_weight_max": 108.02813720703125,
      "activations/layer4_attention_weight_min": -115.0354995727539,
      "activations/layer5_attention_weight_max": 54.69623947143555,
      "activations/layer5_attention_weight_min": -65.5224609375,
      "activations/layer6_attention_weight_max": 45.14072799682617,
      "activations/layer6_attention_weight_min": -48.08241653442383,
      "activations/layer7_attention_weight_max": 92.7944107055664,
      "activations/layer7_attention_weight_min": -100.40308380126953,
      "activations/layer8_attention_weight_max": 43.819210052490234,
      "activations/layer8_attention_weight_min": -44.987735748291016,
      "activations/layer9_attention_weight_max": 32.6286735534668,
      "activations/layer9_attention_weight_min": -35.329864501953125,
      "epoch": 21.37,
      "learning_rate": 1.2306060606060605e-05,
      "loss": 2.6913,
      "step": 367800
    },
    {
      "activations/layer0_attention_weight_max": 16.13007354736328,
      "activations/layer0_attention_weight_min": -14.50890827178955,
      "activations/layer10_attention_weight_max": 36.418312072753906,
      "activations/layer10_attention_weight_min": -38.733131408691406,
      "activations/layer11_attention_weight_max": 38.16498947143555,
      "activations/layer11_attention_weight_min": -39.82185363769531,
      "activations/layer12_attention_weight_max": 27.973859786987305,
      "activations/layer12_attention_weight_min": -29.23089027404785,
      "activations/layer13_attention_weight_max": 41.47149658203125,
      "activations/layer13_attention_weight_min": -38.25874328613281,
      "activations/layer14_attention_weight_max": 48.65293884277344,
      "activations/layer14_attention_weight_min": -38.745330810546875,
      "activations/layer15_attention_weight_max": 47.945960998535156,
      "activations/layer15_attention_weight_min": -39.398681640625,
      "activations/layer16_attention_weight_max": 35.8195686340332,
      "activations/layer16_attention_weight_min": -30.621305465698242,
      "activations/layer17_attention_weight_max": 33.22508239746094,
      "activations/layer17_attention_weight_min": -28.23464584350586,
      "activations/layer18_attention_weight_max": 35.470855712890625,
      "activations/layer18_attention_weight_min": -29.213485717773438,
      "activations/layer19_attention_weight_max": 37.51960372924805,
      "activations/layer19_attention_weight_min": -34.80902862548828,
      "activations/layer1_attention_weight_max": 15.184547424316406,
      "activations/layer1_attention_weight_min": -15.452458381652832,
      "activations/layer20_attention_weight_max": 31.8649959564209,
      "activations/layer20_attention_weight_min": -25.010496139526367,
      "activations/layer21_attention_weight_max": 30.55940818786621,
      "activations/layer21_attention_weight_min": -25.09954833984375,
      "activations/layer22_attention_weight_max": 43.95378875732422,
      "activations/layer22_attention_weight_min": -34.557525634765625,
      "activations/layer23_attention_weight_max": 32.498390197753906,
      "activations/layer23_attention_weight_min": -26.43893814086914,
      "activations/layer2_attention_weight_max": 33.958656311035156,
      "activations/layer2_attention_weight_min": -33.70124053955078,
      "activations/layer3_attention_weight_max": 106.74492645263672,
      "activations/layer3_attention_weight_min": -111.93782043457031,
      "activations/layer4_attention_weight_max": 110.75323486328125,
      "activations/layer4_attention_weight_min": -114.32159423828125,
      "activations/layer5_attention_weight_max": 54.23163604736328,
      "activations/layer5_attention_weight_min": -60.80466842651367,
      "activations/layer6_attention_weight_max": 50.23812484741211,
      "activations/layer6_attention_weight_min": -51.15871047973633,
      "activations/layer7_attention_weight_max": 103.76506042480469,
      "activations/layer7_attention_weight_min": -106.93795013427734,
      "activations/layer8_attention_weight_max": 44.06254959106445,
      "activations/layer8_attention_weight_min": -48.60633850097656,
      "activations/layer9_attention_weight_max": 33.72105026245117,
      "activations/layer9_attention_weight_min": -37.13143539428711,
      "epoch": 21.37,
      "learning_rate": 1.2287121212121211e-05,
      "loss": 2.7104,
      "step": 367850
    },
    {
      "activations/layer0_attention_weight_max": 18.396198272705078,
      "activations/layer0_attention_weight_min": -14.048823356628418,
      "activations/layer10_attention_weight_max": 33.05331039428711,
      "activations/layer10_attention_weight_min": -35.091468811035156,
      "activations/layer11_attention_weight_max": 35.50190353393555,
      "activations/layer11_attention_weight_min": -33.64455795288086,
      "activations/layer12_attention_weight_max": 25.771915435791016,
      "activations/layer12_attention_weight_min": -26.489526748657227,
      "activations/layer13_attention_weight_max": 40.29359436035156,
      "activations/layer13_attention_weight_min": -36.85863494873047,
      "activations/layer14_attention_weight_max": 42.5390510559082,
      "activations/layer14_attention_weight_min": -40.50935363769531,
      "activations/layer15_attention_weight_max": 38.369659423828125,
      "activations/layer15_attention_weight_min": -36.38851547241211,
      "activations/layer16_attention_weight_max": 32.35911560058594,
      "activations/layer16_attention_weight_min": -26.507328033447266,
      "activations/layer17_attention_weight_max": 29.70151710510254,
      "activations/layer17_attention_weight_min": -26.237991333007812,
      "activations/layer18_attention_weight_max": 30.210426330566406,
      "activations/layer18_attention_weight_min": -26.934539794921875,
      "activations/layer19_attention_weight_max": 34.87960433959961,
      "activations/layer19_attention_weight_min": -32.431121826171875,
      "activations/layer1_attention_weight_max": 15.223609924316406,
      "activations/layer1_attention_weight_min": -14.662469863891602,
      "activations/layer20_attention_weight_max": 31.133867263793945,
      "activations/layer20_attention_weight_min": -27.53314208984375,
      "activations/layer21_attention_weight_max": 30.379995346069336,
      "activations/layer21_attention_weight_min": -26.528778076171875,
      "activations/layer22_attention_weight_max": 40.27106857299805,
      "activations/layer22_attention_weight_min": -32.49100875854492,
      "activations/layer23_attention_weight_max": 31.549041748046875,
      "activations/layer23_attention_weight_min": -26.76849365234375,
      "activations/layer2_attention_weight_max": 34.583839416503906,
      "activations/layer2_attention_weight_min": -33.580284118652344,
      "activations/layer3_attention_weight_max": 103.87272644042969,
      "activations/layer3_attention_weight_min": -103.55303192138672,
      "activations/layer4_attention_weight_max": 107.96263885498047,
      "activations/layer4_attention_weight_min": -115.99629974365234,
      "activations/layer5_attention_weight_max": 48.78829574584961,
      "activations/layer5_attention_weight_min": -60.92503356933594,
      "activations/layer6_attention_weight_max": 44.20034408569336,
      "activations/layer6_attention_weight_min": -49.71422576904297,
      "activations/layer7_attention_weight_max": 96.05328369140625,
      "activations/layer7_attention_weight_min": -105.19971466064453,
      "activations/layer8_attention_weight_max": 43.83354568481445,
      "activations/layer8_attention_weight_min": -44.1496467590332,
      "activations/layer9_attention_weight_max": 30.666519165039062,
      "activations/layer9_attention_weight_min": -34.50761795043945,
      "epoch": 21.38,
      "learning_rate": 1.2268181818181816e-05,
      "loss": 2.7098,
      "step": 367900
    },
    {
      "activations/layer0_attention_weight_max": 16.90077781677246,
      "activations/layer0_attention_weight_min": -14.586481094360352,
      "activations/layer10_attention_weight_max": 36.900238037109375,
      "activations/layer10_attention_weight_min": -37.22040557861328,
      "activations/layer11_attention_weight_max": 36.41043472290039,
      "activations/layer11_attention_weight_min": -35.77893829345703,
      "activations/layer12_attention_weight_max": 29.006296157836914,
      "activations/layer12_attention_weight_min": -27.225727081298828,
      "activations/layer13_attention_weight_max": 42.22283935546875,
      "activations/layer13_attention_weight_min": -37.565608978271484,
      "activations/layer14_attention_weight_max": 44.900169372558594,
      "activations/layer14_attention_weight_min": -40.327430725097656,
      "activations/layer15_attention_weight_max": 40.06171798706055,
      "activations/layer15_attention_weight_min": -37.03373718261719,
      "activations/layer16_attention_weight_max": 29.96228790283203,
      "activations/layer16_attention_weight_min": -28.647985458374023,
      "activations/layer17_attention_weight_max": 30.707759857177734,
      "activations/layer17_attention_weight_min": -26.674440383911133,
      "activations/layer18_attention_weight_max": 37.11961364746094,
      "activations/layer18_attention_weight_min": -29.052608489990234,
      "activations/layer19_attention_weight_max": 39.70603561401367,
      "activations/layer19_attention_weight_min": -34.33326721191406,
      "activations/layer1_attention_weight_max": 15.515604972839355,
      "activations/layer1_attention_weight_min": -15.35295581817627,
      "activations/layer20_attention_weight_max": 29.93050765991211,
      "activations/layer20_attention_weight_min": -25.773557662963867,
      "activations/layer21_attention_weight_max": 29.338939666748047,
      "activations/layer21_attention_weight_min": -25.01578712463379,
      "activations/layer22_attention_weight_max": 44.86894989013672,
      "activations/layer22_attention_weight_min": -33.80998229980469,
      "activations/layer23_attention_weight_max": 31.915786743164062,
      "activations/layer23_attention_weight_min": -27.05139923095703,
      "activations/layer2_attention_weight_max": 35.213218688964844,
      "activations/layer2_attention_weight_min": -33.13452911376953,
      "activations/layer3_attention_weight_max": 109.58775329589844,
      "activations/layer3_attention_weight_min": -107.72943878173828,
      "activations/layer4_attention_weight_max": 113.64810943603516,
      "activations/layer4_attention_weight_min": -111.61510467529297,
      "activations/layer5_attention_weight_max": 52.433616638183594,
      "activations/layer5_attention_weight_min": -63.300594329833984,
      "activations/layer6_attention_weight_max": 47.79370880126953,
      "activations/layer6_attention_weight_min": -49.286109924316406,
      "activations/layer7_attention_weight_max": 99.55903625488281,
      "activations/layer7_attention_weight_min": -103.52728271484375,
      "activations/layer8_attention_weight_max": 45.0246467590332,
      "activations/layer8_attention_weight_min": -47.020145416259766,
      "activations/layer9_attention_weight_max": 35.221195220947266,
      "activations/layer9_attention_weight_min": -36.66691589355469,
      "epoch": 21.38,
      "learning_rate": 1.2249242424242423e-05,
      "loss": 2.6994,
      "step": 367950
    },
    {
      "activations/layer0_attention_weight_max": 15.965593338012695,
      "activations/layer0_attention_weight_min": -15.108588218688965,
      "activations/layer10_attention_weight_max": 34.079925537109375,
      "activations/layer10_attention_weight_min": -33.95191192626953,
      "activations/layer11_attention_weight_max": 34.10795593261719,
      "activations/layer11_attention_weight_min": -34.48622131347656,
      "activations/layer12_attention_weight_max": 26.956647872924805,
      "activations/layer12_attention_weight_min": -27.884767532348633,
      "activations/layer13_attention_weight_max": 40.86511993408203,
      "activations/layer13_attention_weight_min": -36.33855438232422,
      "activations/layer14_attention_weight_max": 45.14531326293945,
      "activations/layer14_attention_weight_min": -37.172401428222656,
      "activations/layer15_attention_weight_max": 40.18751907348633,
      "activations/layer15_attention_weight_min": -37.933265686035156,
      "activations/layer16_attention_weight_max": 34.057708740234375,
      "activations/layer16_attention_weight_min": -27.746187210083008,
      "activations/layer17_attention_weight_max": 30.90218734741211,
      "activations/layer17_attention_weight_min": -26.654865264892578,
      "activations/layer18_attention_weight_max": 36.60749053955078,
      "activations/layer18_attention_weight_min": -24.833742141723633,
      "activations/layer19_attention_weight_max": 37.15308380126953,
      "activations/layer19_attention_weight_min": -32.546607971191406,
      "activations/layer1_attention_weight_max": 15.282588958740234,
      "activations/layer1_attention_weight_min": -14.646565437316895,
      "activations/layer20_attention_weight_max": 32.402496337890625,
      "activations/layer20_attention_weight_min": -25.095008850097656,
      "activations/layer21_attention_weight_max": 31.636627197265625,
      "activations/layer21_attention_weight_min": -24.6948184967041,
      "activations/layer22_attention_weight_max": 49.35563278198242,
      "activations/layer22_attention_weight_min": -31.711444854736328,
      "activations/layer23_attention_weight_max": 32.697349548339844,
      "activations/layer23_attention_weight_min": -25.808500289916992,
      "activations/layer2_attention_weight_max": 34.48797607421875,
      "activations/layer2_attention_weight_min": -32.410911560058594,
      "activations/layer3_attention_weight_max": 102.96959686279297,
      "activations/layer3_attention_weight_min": -104.83894348144531,
      "activations/layer4_attention_weight_max": 104.18535614013672,
      "activations/layer4_attention_weight_min": -109.3537368774414,
      "activations/layer5_attention_weight_max": 50.555274963378906,
      "activations/layer5_attention_weight_min": -65.56358337402344,
      "activations/layer6_attention_weight_max": 44.57112121582031,
      "activations/layer6_attention_weight_min": -48.54945755004883,
      "activations/layer7_attention_weight_max": 97.20128631591797,
      "activations/layer7_attention_weight_min": -97.1678466796875,
      "activations/layer8_attention_weight_max": 40.99329376220703,
      "activations/layer8_attention_weight_min": -43.48363494873047,
      "activations/layer9_attention_weight_max": 33.7063102722168,
      "activations/layer9_attention_weight_min": -34.2403564453125,
      "epoch": 21.38,
      "learning_rate": 1.223030303030303e-05,
      "loss": 2.7281,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.4536,
      "eval_samples_per_second": 507.948,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.4536,
      "eval_openwebtext_samples_per_second": 507.948,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 2.0008,
      "eval_wikitext_samples_per_second": 227.913,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_lambada_loss": 2.294921875,
      "eval_lambada_ppl": 9.923660696008076,
      "eval_lambada_runtime": 9.5688,
      "eval_lambada_samples_per_second": 508.843,
      "step": 368000
    },
    {
      "activations/layer0_attention_weight_max": 16.28469467163086,
      "activations/layer0_attention_weight_min": -13.815471649169922,
      "activations/layer10_attention_weight_max": 33.341957092285156,
      "activations/layer10_attention_weight_min": -35.43376159667969,
      "activations/layer11_attention_weight_max": 34.800899505615234,
      "activations/layer11_attention_weight_min": -34.651615142822266,
      "activations/layer12_attention_weight_max": 25.654069900512695,
      "activations/layer12_attention_weight_min": -27.463964462280273,
      "activations/layer13_attention_weight_max": 36.63346862792969,
      "activations/layer13_attention_weight_min": -36.38408660888672,
      "activations/layer14_attention_weight_max": 39.48737716674805,
      "activations/layer14_attention_weight_min": -35.6954231262207,
      "activations/layer15_attention_weight_max": 36.47582244873047,
      "activations/layer15_attention_weight_min": -33.46829605102539,
      "activations/layer16_attention_weight_max": 27.55936050415039,
      "activations/layer16_attention_weight_min": -29.508750915527344,
      "activations/layer17_attention_weight_max": 27.665645599365234,
      "activations/layer17_attention_weight_min": -25.21012306213379,
      "activations/layer18_attention_weight_max": 28.477636337280273,
      "activations/layer18_attention_weight_min": -24.514034271240234,
      "activations/layer19_attention_weight_max": 32.215328216552734,
      "activations/layer19_attention_weight_min": -32.28649139404297,
      "activations/layer1_attention_weight_max": 15.015600204467773,
      "activations/layer1_attention_weight_min": -14.408843994140625,
      "activations/layer20_attention_weight_max": 25.798484802246094,
      "activations/layer20_attention_weight_min": -24.90985679626465,
      "activations/layer21_attention_weight_max": 27.792375564575195,
      "activations/layer21_attention_weight_min": -24.015832901000977,
      "activations/layer22_attention_weight_max": 40.70698547363281,
      "activations/layer22_attention_weight_min": -31.165945053100586,
      "activations/layer23_attention_weight_max": 33.36464309692383,
      "activations/layer23_attention_weight_min": -24.136905670166016,
      "activations/layer2_attention_weight_max": 33.72917175292969,
      "activations/layer2_attention_weight_min": -33.80384063720703,
      "activations/layer3_attention_weight_max": 103.47464752197266,
      "activations/layer3_attention_weight_min": -104.29910278320312,
      "activations/layer4_attention_weight_max": 110.6205062866211,
      "activations/layer4_attention_weight_min": -117.41064453125,
      "activations/layer5_attention_weight_max": 50.578914642333984,
      "activations/layer5_attention_weight_min": -60.849849700927734,
      "activations/layer6_attention_weight_max": 44.43498611450195,
      "activations/layer6_attention_weight_min": -48.463722229003906,
      "activations/layer7_attention_weight_max": 99.22952270507812,
      "activations/layer7_attention_weight_min": -99.00164794921875,
      "activations/layer8_attention_weight_max": 42.054386138916016,
      "activations/layer8_attention_weight_min": -44.3755989074707,
      "activations/layer9_attention_weight_max": 32.0300407409668,
      "activations/layer9_attention_weight_min": -34.91011428833008,
      "epoch": 21.39,
      "learning_rate": 1.2211363636363634e-05,
      "loss": 2.7131,
      "step": 368050
    },
    {
      "activations/layer0_attention_weight_max": 16.857816696166992,
      "activations/layer0_attention_weight_min": -15.713507652282715,
      "activations/layer10_attention_weight_max": 38.807559967041016,
      "activations/layer10_attention_weight_min": -38.66547393798828,
      "activations/layer11_attention_weight_max": 37.13700485229492,
      "activations/layer11_attention_weight_min": -38.21812438964844,
      "activations/layer12_attention_weight_max": 29.18539810180664,
      "activations/layer12_attention_weight_min": -27.830245971679688,
      "activations/layer13_attention_weight_max": 41.07402420043945,
      "activations/layer13_attention_weight_min": -38.373802185058594,
      "activations/layer14_attention_weight_max": 47.703678131103516,
      "activations/layer14_attention_weight_min": -41.582435607910156,
      "activations/layer15_attention_weight_max": 45.718929290771484,
      "activations/layer15_attention_weight_min": -40.19087219238281,
      "activations/layer16_attention_weight_max": 32.123050689697266,
      "activations/layer16_attention_weight_min": -29.515586853027344,
      "activations/layer17_attention_weight_max": 33.45076370239258,
      "activations/layer17_attention_weight_min": -27.543848037719727,
      "activations/layer18_attention_weight_max": 37.49568176269531,
      "activations/layer18_attention_weight_min": -27.64189338684082,
      "activations/layer19_attention_weight_max": 35.79392623901367,
      "activations/layer19_attention_weight_min": -33.57362365722656,
      "activations/layer1_attention_weight_max": 15.828529357910156,
      "activations/layer1_attention_weight_min": -14.207175254821777,
      "activations/layer20_attention_weight_max": 29.14242172241211,
      "activations/layer20_attention_weight_min": -27.68099594116211,
      "activations/layer21_attention_weight_max": 28.52450180053711,
      "activations/layer21_attention_weight_min": -27.011550903320312,
      "activations/layer22_attention_weight_max": 41.38114547729492,
      "activations/layer22_attention_weight_min": -33.35173416137695,
      "activations/layer23_attention_weight_max": 33.08101272583008,
      "activations/layer23_attention_weight_min": -26.553485870361328,
      "activations/layer2_attention_weight_max": 34.695411682128906,
      "activations/layer2_attention_weight_min": -32.88227462768555,
      "activations/layer3_attention_weight_max": 103.87165069580078,
      "activations/layer3_attention_weight_min": -102.53402709960938,
      "activations/layer4_attention_weight_max": 102.77423095703125,
      "activations/layer4_attention_weight_min": -109.13701629638672,
      "activations/layer5_attention_weight_max": 48.803462982177734,
      "activations/layer5_attention_weight_min": -62.08799362182617,
      "activations/layer6_attention_weight_max": 45.95397186279297,
      "activations/layer6_attention_weight_min": -47.69241714477539,
      "activations/layer7_attention_weight_max": 97.90876770019531,
      "activations/layer7_attention_weight_min": -102.5895004272461,
      "activations/layer8_attention_weight_max": 45.85320281982422,
      "activations/layer8_attention_weight_min": -50.377479553222656,
      "activations/layer9_attention_weight_max": 37.53788375854492,
      "activations/layer9_attention_weight_min": -37.26033020019531,
      "epoch": 21.39,
      "learning_rate": 1.2192803030303028e-05,
      "loss": 2.7128,
      "step": 368100
    },
    {
      "activations/layer0_attention_weight_max": 16.34099769592285,
      "activations/layer0_attention_weight_min": -14.218854904174805,
      "activations/layer10_attention_weight_max": 35.14012145996094,
      "activations/layer10_attention_weight_min": -35.79827880859375,
      "activations/layer11_attention_weight_max": 37.51055908203125,
      "activations/layer11_attention_weight_min": -38.400333404541016,
      "activations/layer12_attention_weight_max": 29.34171485900879,
      "activations/layer12_attention_weight_min": -28.01117706298828,
      "activations/layer13_attention_weight_max": 42.01918411254883,
      "activations/layer13_attention_weight_min": -36.074710845947266,
      "activations/layer14_attention_weight_max": 43.73747253417969,
      "activations/layer14_attention_weight_min": -37.77785873413086,
      "activations/layer15_attention_weight_max": 42.903076171875,
      "activations/layer15_attention_weight_min": -37.44408416748047,
      "activations/layer16_attention_weight_max": 34.5666389465332,
      "activations/layer16_attention_weight_min": -29.23871421813965,
      "activations/layer17_attention_weight_max": 32.15971755981445,
      "activations/layer17_attention_weight_min": -26.634700775146484,
      "activations/layer18_attention_weight_max": 39.70249557495117,
      "activations/layer18_attention_weight_min": -28.21648597717285,
      "activations/layer19_attention_weight_max": 43.74474334716797,
      "activations/layer19_attention_weight_min": -33.04323959350586,
      "activations/layer1_attention_weight_max": 15.012443542480469,
      "activations/layer1_attention_weight_min": -14.548188209533691,
      "activations/layer20_attention_weight_max": 32.15651321411133,
      "activations/layer20_attention_weight_min": -26.4658145904541,
      "activations/layer21_attention_weight_max": 31.57546043395996,
      "activations/layer21_attention_weight_min": -25.680747985839844,
      "activations/layer22_attention_weight_max": 44.6335563659668,
      "activations/layer22_attention_weight_min": -32.59781265258789,
      "activations/layer23_attention_weight_max": 35.11735916137695,
      "activations/layer23_attention_weight_min": -26.501739501953125,
      "activations/layer2_attention_weight_max": 33.77369689941406,
      "activations/layer2_attention_weight_min": -33.98118591308594,
      "activations/layer3_attention_weight_max": 106.06267547607422,
      "activations/layer3_attention_weight_min": -110.91967010498047,
      "activations/layer4_attention_weight_max": 110.1361083984375,
      "activations/layer4_attention_weight_min": -118.1424331665039,
      "activations/layer5_attention_weight_max": 50.20505905151367,
      "activations/layer5_attention_weight_min": -64.216064453125,
      "activations/layer6_attention_weight_max": 46.55474090576172,
      "activations/layer6_attention_weight_min": -51.92494201660156,
      "activations/layer7_attention_weight_max": 98.325439453125,
      "activations/layer7_attention_weight_min": -100.4850845336914,
      "activations/layer8_attention_weight_max": 43.32763671875,
      "activations/layer8_attention_weight_min": -44.62582778930664,
      "activations/layer9_attention_weight_max": 33.528663635253906,
      "activations/layer9_attention_weight_min": -34.53741455078125,
      "epoch": 21.39,
      "learning_rate": 1.2173863636363635e-05,
      "loss": 2.707,
      "step": 368150
    },
    {
      "activations/layer0_attention_weight_max": 16.964216232299805,
      "activations/layer0_attention_weight_min": -13.726218223571777,
      "activations/layer10_attention_weight_max": 34.331024169921875,
      "activations/layer10_attention_weight_min": -35.07654571533203,
      "activations/layer11_attention_weight_max": 34.95780944824219,
      "activations/layer11_attention_weight_min": -34.895626068115234,
      "activations/layer12_attention_weight_max": 27.095680236816406,
      "activations/layer12_attention_weight_min": -27.036745071411133,
      "activations/layer13_attention_weight_max": 40.70125198364258,
      "activations/layer13_attention_weight_min": -39.23600387573242,
      "activations/layer14_attention_weight_max": 43.22028350830078,
      "activations/layer14_attention_weight_min": -39.72059631347656,
      "activations/layer15_attention_weight_max": 39.550048828125,
      "activations/layer15_attention_weight_min": -38.64613723754883,
      "activations/layer16_attention_weight_max": 29.02937126159668,
      "activations/layer16_attention_weight_min": -29.6423397064209,
      "activations/layer17_attention_weight_max": 29.336355209350586,
      "activations/layer17_attention_weight_min": -25.007503509521484,
      "activations/layer18_attention_weight_max": 32.015933990478516,
      "activations/layer18_attention_weight_min": -25.62314796447754,
      "activations/layer19_attention_weight_max": 36.09752655029297,
      "activations/layer19_attention_weight_min": -31.492996215820312,
      "activations/layer1_attention_weight_max": 15.915565490722656,
      "activations/layer1_attention_weight_min": -14.658226013183594,
      "activations/layer20_attention_weight_max": 28.804574966430664,
      "activations/layer20_attention_weight_min": -24.114910125732422,
      "activations/layer21_attention_weight_max": 29.482309341430664,
      "activations/layer21_attention_weight_min": -23.779001235961914,
      "activations/layer22_attention_weight_max": 38.65253448486328,
      "activations/layer22_attention_weight_min": -31.05237579345703,
      "activations/layer23_attention_weight_max": 32.60003662109375,
      "activations/layer23_attention_weight_min": -26.36737823486328,
      "activations/layer2_attention_weight_max": 33.412940979003906,
      "activations/layer2_attention_weight_min": -33.23468780517578,
      "activations/layer3_attention_weight_max": 104.72901916503906,
      "activations/layer3_attention_weight_min": -104.98755645751953,
      "activations/layer4_attention_weight_max": 112.33906555175781,
      "activations/layer4_attention_weight_min": -111.7167739868164,
      "activations/layer5_attention_weight_max": 54.992034912109375,
      "activations/layer5_attention_weight_min": -60.21926498413086,
      "activations/layer6_attention_weight_max": 45.64842987060547,
      "activations/layer6_attention_weight_min": -49.01041793823242,
      "activations/layer7_attention_weight_max": 100.39328002929688,
      "activations/layer7_attention_weight_min": -100.95834350585938,
      "activations/layer8_attention_weight_max": 43.31361389160156,
      "activations/layer8_attention_weight_min": -44.825138092041016,
      "activations/layer9_attention_weight_max": 32.66904067993164,
      "activations/layer9_attention_weight_min": -34.50511932373047,
      "epoch": 21.39,
      "learning_rate": 1.2154924242424243e-05,
      "loss": 2.731,
      "step": 368200
    },
    {
      "activations/layer0_attention_weight_max": 16.92837905883789,
      "activations/layer0_attention_weight_min": -13.896479606628418,
      "activations/layer10_attention_weight_max": 33.406097412109375,
      "activations/layer10_attention_weight_min": -35.255104064941406,
      "activations/layer11_attention_weight_max": 33.54869079589844,
      "activations/layer11_attention_weight_min": -36.308189392089844,
      "activations/layer12_attention_weight_max": 28.089664459228516,
      "activations/layer12_attention_weight_min": -27.510066986083984,
      "activations/layer13_attention_weight_max": 39.695289611816406,
      "activations/layer13_attention_weight_min": -36.44087600708008,
      "activations/layer14_attention_weight_max": 43.39005661010742,
      "activations/layer14_attention_weight_min": -38.27400207519531,
      "activations/layer15_attention_weight_max": 40.79441833496094,
      "activations/layer15_attention_weight_min": -35.75092697143555,
      "activations/layer16_attention_weight_max": 29.572834014892578,
      "activations/layer16_attention_weight_min": -28.78765869140625,
      "activations/layer17_attention_weight_max": 29.72088623046875,
      "activations/layer17_attention_weight_min": -25.356653213500977,
      "activations/layer18_attention_weight_max": 33.245216369628906,
      "activations/layer18_attention_weight_min": -26.430679321289062,
      "activations/layer19_attention_weight_max": 31.81660270690918,
      "activations/layer19_attention_weight_min": -32.13739776611328,
      "activations/layer1_attention_weight_max": 15.72946548461914,
      "activations/layer1_attention_weight_min": -14.926077842712402,
      "activations/layer20_attention_weight_max": 28.097028732299805,
      "activations/layer20_attention_weight_min": -27.309242248535156,
      "activations/layer21_attention_weight_max": 26.775360107421875,
      "activations/layer21_attention_weight_min": -28.289213180541992,
      "activations/layer22_attention_weight_max": 39.8271484375,
      "activations/layer22_attention_weight_min": -33.06764221191406,
      "activations/layer23_attention_weight_max": 31.064678192138672,
      "activations/layer23_attention_weight_min": -25.940242767333984,
      "activations/layer2_attention_weight_max": 33.37396240234375,
      "activations/layer2_attention_weight_min": -33.72200393676758,
      "activations/layer3_attention_weight_max": 102.01776885986328,
      "activations/layer3_attention_weight_min": -105.01097869873047,
      "activations/layer4_attention_weight_max": 107.41082763671875,
      "activations/layer4_attention_weight_min": -115.1156997680664,
      "activations/layer5_attention_weight_max": 52.19292449951172,
      "activations/layer5_attention_weight_min": -61.75301742553711,
      "activations/layer6_attention_weight_max": 46.20145797729492,
      "activations/layer6_attention_weight_min": -50.686546325683594,
      "activations/layer7_attention_weight_max": 96.70992279052734,
      "activations/layer7_attention_weight_min": -103.53811645507812,
      "activations/layer8_attention_weight_max": 43.59926223754883,
      "activations/layer8_attention_weight_min": -45.21555709838867,
      "activations/layer9_attention_weight_max": 33.09518051147461,
      "activations/layer9_attention_weight_min": -35.12461853027344,
      "epoch": 21.4,
      "learning_rate": 1.2135984848484848e-05,
      "loss": 2.7228,
      "step": 368250
    },
    {
      "activations/layer0_attention_weight_max": 17.024600982666016,
      "activations/layer0_attention_weight_min": -15.025857925415039,
      "activations/layer10_attention_weight_max": 42.91268539428711,
      "activations/layer10_attention_weight_min": -39.41769027709961,
      "activations/layer11_attention_weight_max": 37.032188415527344,
      "activations/layer11_attention_weight_min": -38.31118392944336,
      "activations/layer12_attention_weight_max": 27.24254608154297,
      "activations/layer12_attention_weight_min": -28.196725845336914,
      "activations/layer13_attention_weight_max": 40.88050842285156,
      "activations/layer13_attention_weight_min": -37.460426330566406,
      "activations/layer14_attention_weight_max": 43.42426681518555,
      "activations/layer14_attention_weight_min": -38.14002990722656,
      "activations/layer15_attention_weight_max": 40.547698974609375,
      "activations/layer15_attention_weight_min": -36.61861038208008,
      "activations/layer16_attention_weight_max": 34.49243927001953,
      "activations/layer16_attention_weight_min": -30.478275299072266,
      "activations/layer17_attention_weight_max": 34.8008918762207,
      "activations/layer17_attention_weight_min": -27.86274528503418,
      "activations/layer18_attention_weight_max": 39.243709564208984,
      "activations/layer18_attention_weight_min": -29.494853973388672,
      "activations/layer19_attention_weight_max": 39.842220306396484,
      "activations/layer19_attention_weight_min": -31.404584884643555,
      "activations/layer1_attention_weight_max": 15.617548942565918,
      "activations/layer1_attention_weight_min": -15.03857707977295,
      "activations/layer20_attention_weight_max": 32.53487014770508,
      "activations/layer20_attention_weight_min": -24.888303756713867,
      "activations/layer21_attention_weight_max": 32.023658752441406,
      "activations/layer21_attention_weight_min": -24.957298278808594,
      "activations/layer22_attention_weight_max": 45.07437515258789,
      "activations/layer22_attention_weight_min": -32.80646896362305,
      "activations/layer23_attention_weight_max": 37.55354309082031,
      "activations/layer23_attention_weight_min": -26.65046501159668,
      "activations/layer2_attention_weight_max": 35.82390594482422,
      "activations/layer2_attention_weight_min": -35.73643112182617,
      "activations/layer3_attention_weight_max": 107.13394165039062,
      "activations/layer3_attention_weight_min": -110.67086029052734,
      "activations/layer4_attention_weight_max": 114.18685150146484,
      "activations/layer4_attention_weight_min": -115.2846908569336,
      "activations/layer5_attention_weight_max": 52.87713623046875,
      "activations/layer5_attention_weight_min": -64.57791900634766,
      "activations/layer6_attention_weight_max": 50.304935455322266,
      "activations/layer6_attention_weight_min": -53.13024139404297,
      "activations/layer7_attention_weight_max": 111.30572509765625,
      "activations/layer7_attention_weight_min": -111.96330261230469,
      "activations/layer8_attention_weight_max": 47.59177017211914,
      "activations/layer8_attention_weight_min": -49.51752853393555,
      "activations/layer9_attention_weight_max": 36.835845947265625,
      "activations/layer9_attention_weight_min": -38.94352722167969,
      "epoch": 21.4,
      "learning_rate": 1.2117045454545455e-05,
      "loss": 2.6884,
      "step": 368300
    },
    {
      "activations/layer0_attention_weight_max": 16.83211898803711,
      "activations/layer0_attention_weight_min": -14.411775588989258,
      "activations/layer10_attention_weight_max": 37.45427322387695,
      "activations/layer10_attention_weight_min": -38.29522705078125,
      "activations/layer11_attention_weight_max": 38.419189453125,
      "activations/layer11_attention_weight_min": -40.89564895629883,
      "activations/layer12_attention_weight_max": 30.068931579589844,
      "activations/layer12_attention_weight_min": -28.364870071411133,
      "activations/layer13_attention_weight_max": 45.48451614379883,
      "activations/layer13_attention_weight_min": -39.5550537109375,
      "activations/layer14_attention_weight_max": 48.875518798828125,
      "activations/layer14_attention_weight_min": -42.13542175292969,
      "activations/layer15_attention_weight_max": 47.05137252807617,
      "activations/layer15_attention_weight_min": -41.93128204345703,
      "activations/layer16_attention_weight_max": 35.53211212158203,
      "activations/layer16_attention_weight_min": -31.81485939025879,
      "activations/layer17_attention_weight_max": 35.163204193115234,
      "activations/layer17_attention_weight_min": -31.048898696899414,
      "activations/layer18_attention_weight_max": 36.14603805541992,
      "activations/layer18_attention_weight_min": -29.680545806884766,
      "activations/layer19_attention_weight_max": 42.27027893066406,
      "activations/layer19_attention_weight_min": -35.09825897216797,
      "activations/layer1_attention_weight_max": 15.588170051574707,
      "activations/layer1_attention_weight_min": -15.464762687683105,
      "activations/layer20_attention_weight_max": 34.96503829956055,
      "activations/layer20_attention_weight_min": -26.754350662231445,
      "activations/layer21_attention_weight_max": 32.63869857788086,
      "activations/layer21_attention_weight_min": -25.771923065185547,
      "activations/layer22_attention_weight_max": 43.020179748535156,
      "activations/layer22_attention_weight_min": -33.28886795043945,
      "activations/layer23_attention_weight_max": 35.31929016113281,
      "activations/layer23_attention_weight_min": -28.47301483154297,
      "activations/layer2_attention_weight_max": 33.463382720947266,
      "activations/layer2_attention_weight_min": -31.967254638671875,
      "activations/layer3_attention_weight_max": 106.0300064086914,
      "activations/layer3_attention_weight_min": -106.8587875366211,
      "activations/layer4_attention_weight_max": 111.89830780029297,
      "activations/layer4_attention_weight_min": -114.67015838623047,
      "activations/layer5_attention_weight_max": 48.67876434326172,
      "activations/layer5_attention_weight_min": -68.27326965332031,
      "activations/layer6_attention_weight_max": 44.04492950439453,
      "activations/layer6_attention_weight_min": -45.63522720336914,
      "activations/layer7_attention_weight_max": 101.88103485107422,
      "activations/layer7_attention_weight_min": -100.90397644042969,
      "activations/layer8_attention_weight_max": 44.7501106262207,
      "activations/layer8_attention_weight_min": -44.89809036254883,
      "activations/layer9_attention_weight_max": 31.763046264648438,
      "activations/layer9_attention_weight_min": -36.417335510253906,
      "epoch": 21.4,
      "learning_rate": 1.209810606060606e-05,
      "loss": 2.7137,
      "step": 368350
    },
    {
      "activations/layer0_attention_weight_max": 16.658721923828125,
      "activations/layer0_attention_weight_min": -14.582115173339844,
      "activations/layer10_attention_weight_max": 34.28025817871094,
      "activations/layer10_attention_weight_min": -35.311553955078125,
      "activations/layer11_attention_weight_max": 32.45124816894531,
      "activations/layer11_attention_weight_min": -33.285552978515625,
      "activations/layer12_attention_weight_max": 26.26905632019043,
      "activations/layer12_attention_weight_min": -26.83118438720703,
      "activations/layer13_attention_weight_max": 40.91896438598633,
      "activations/layer13_attention_weight_min": -36.98735046386719,
      "activations/layer14_attention_weight_max": 39.7381477355957,
      "activations/layer14_attention_weight_min": -39.7613639831543,
      "activations/layer15_attention_weight_max": 38.256080627441406,
      "activations/layer15_attention_weight_min": -35.65899658203125,
      "activations/layer16_attention_weight_max": 28.096765518188477,
      "activations/layer16_attention_weight_min": -29.075542449951172,
      "activations/layer17_attention_weight_max": 26.207284927368164,
      "activations/layer17_attention_weight_min": -27.293493270874023,
      "activations/layer18_attention_weight_max": 31.303321838378906,
      "activations/layer18_attention_weight_min": -25.93647003173828,
      "activations/layer19_attention_weight_max": 31.0734920501709,
      "activations/layer19_attention_weight_min": -32.006954193115234,
      "activations/layer1_attention_weight_max": 15.108392715454102,
      "activations/layer1_attention_weight_min": -14.793146133422852,
      "activations/layer20_attention_weight_max": 28.241865158081055,
      "activations/layer20_attention_weight_min": -24.380294799804688,
      "activations/layer21_attention_weight_max": 27.54042625427246,
      "activations/layer21_attention_weight_min": -23.710683822631836,
      "activations/layer22_attention_weight_max": 35.6538200378418,
      "activations/layer22_attention_weight_min": -30.918075561523438,
      "activations/layer23_attention_weight_max": 28.83167266845703,
      "activations/layer23_attention_weight_min": -25.89754867553711,
      "activations/layer2_attention_weight_max": 33.72410583496094,
      "activations/layer2_attention_weight_min": -32.989585876464844,
      "activations/layer3_attention_weight_max": 108.2711181640625,
      "activations/layer3_attention_weight_min": -106.37222290039062,
      "activations/layer4_attention_weight_max": 108.4150161743164,
      "activations/layer4_attention_weight_min": -114.282470703125,
      "activations/layer5_attention_weight_max": 53.578399658203125,
      "activations/layer5_attention_weight_min": -65.3951416015625,
      "activations/layer6_attention_weight_max": 45.06488800048828,
      "activations/layer6_attention_weight_min": -51.235111236572266,
      "activations/layer7_attention_weight_max": 96.27759552001953,
      "activations/layer7_attention_weight_min": -104.86090087890625,
      "activations/layer8_attention_weight_max": 44.03288650512695,
      "activations/layer8_attention_weight_min": -46.19043731689453,
      "activations/layer9_attention_weight_max": 31.923288345336914,
      "activations/layer9_attention_weight_min": -34.87181854248047,
      "epoch": 21.41,
      "learning_rate": 1.2079166666666666e-05,
      "loss": 2.7178,
      "step": 368400
    },
    {
      "activations/layer0_attention_weight_max": 16.729230880737305,
      "activations/layer0_attention_weight_min": -13.014191627502441,
      "activations/layer10_attention_weight_max": 39.582183837890625,
      "activations/layer10_attention_weight_min": -39.25758743286133,
      "activations/layer11_attention_weight_max": 38.361663818359375,
      "activations/layer11_attention_weight_min": -36.15751647949219,
      "activations/layer12_attention_weight_max": 29.66695785522461,
      "activations/layer12_attention_weight_min": -28.214120864868164,
      "activations/layer13_attention_weight_max": 42.869667053222656,
      "activations/layer13_attention_weight_min": -37.09271240234375,
      "activations/layer14_attention_weight_max": 50.612022399902344,
      "activations/layer14_attention_weight_min": -41.73210906982422,
      "activations/layer15_attention_weight_max": 43.67109298706055,
      "activations/layer15_attention_weight_min": -41.90748596191406,
      "activations/layer16_attention_weight_max": 32.30879211425781,
      "activations/layer16_attention_weight_min": -31.94377326965332,
      "activations/layer17_attention_weight_max": 31.20229148864746,
      "activations/layer17_attention_weight_min": -30.431888580322266,
      "activations/layer18_attention_weight_max": 32.1368408203125,
      "activations/layer18_attention_weight_min": -29.96446418762207,
      "activations/layer19_attention_weight_max": 33.85578155517578,
      "activations/layer19_attention_weight_min": -33.5849723815918,
      "activations/layer1_attention_weight_max": 15.470558166503906,
      "activations/layer1_attention_weight_min": -15.164342880249023,
      "activations/layer20_attention_weight_max": 29.395061492919922,
      "activations/layer20_attention_weight_min": -27.13558578491211,
      "activations/layer21_attention_weight_max": 31.47640037536621,
      "activations/layer21_attention_weight_min": -25.629112243652344,
      "activations/layer22_attention_weight_max": 45.69089889526367,
      "activations/layer22_attention_weight_min": -35.3657112121582,
      "activations/layer23_attention_weight_max": 35.600196838378906,
      "activations/layer23_attention_weight_min": -28.31601333618164,
      "activations/layer2_attention_weight_max": 34.54566192626953,
      "activations/layer2_attention_weight_min": -33.39808654785156,
      "activations/layer3_attention_weight_max": 109.61221313476562,
      "activations/layer3_attention_weight_min": -111.53892517089844,
      "activations/layer4_attention_weight_max": 112.00679779052734,
      "activations/layer4_attention_weight_min": -114.2639389038086,
      "activations/layer5_attention_weight_max": 52.23432540893555,
      "activations/layer5_attention_weight_min": -62.841304779052734,
      "activations/layer6_attention_weight_max": 47.782657623291016,
      "activations/layer6_attention_weight_min": -51.71516799926758,
      "activations/layer7_attention_weight_max": 100.67586517333984,
      "activations/layer7_attention_weight_min": -110.11543273925781,
      "activations/layer8_attention_weight_max": 48.897552490234375,
      "activations/layer8_attention_weight_min": -49.30044937133789,
      "activations/layer9_attention_weight_max": 38.71046829223633,
      "activations/layer9_attention_weight_min": -38.56212615966797,
      "epoch": 21.41,
      "learning_rate": 1.2060227272727273e-05,
      "loss": 2.6918,
      "step": 368450
    },
    {
      "activations/layer0_attention_weight_max": 15.878962516784668,
      "activations/layer0_attention_weight_min": -14.154182434082031,
      "activations/layer10_attention_weight_max": 41.04990005493164,
      "activations/layer10_attention_weight_min": -40.196895599365234,
      "activations/layer11_attention_weight_max": 40.47998046875,
      "activations/layer11_attention_weight_min": -41.23220443725586,
      "activations/layer12_attention_weight_max": 29.30775260925293,
      "activations/layer12_attention_weight_min": -28.92670440673828,
      "activations/layer13_attention_weight_max": 43.8763313293457,
      "activations/layer13_attention_weight_min": -39.584407806396484,
      "activations/layer14_attention_weight_max": 44.980472564697266,
      "activations/layer14_attention_weight_min": -42.89543533325195,
      "activations/layer15_attention_weight_max": 44.98307800292969,
      "activations/layer15_attention_weight_min": -38.74989700317383,
      "activations/layer16_attention_weight_max": 29.76698875427246,
      "activations/layer16_attention_weight_min": -29.97879409790039,
      "activations/layer17_attention_weight_max": 30.492151260375977,
      "activations/layer17_attention_weight_min": -28.984514236450195,
      "activations/layer18_attention_weight_max": 34.42881393432617,
      "activations/layer18_attention_weight_min": -27.8614559173584,
      "activations/layer19_attention_weight_max": 34.528968811035156,
      "activations/layer19_attention_weight_min": -32.07892990112305,
      "activations/layer1_attention_weight_max": 16.692224502563477,
      "activations/layer1_attention_weight_min": -15.809369087219238,
      "activations/layer20_attention_weight_max": 30.847387313842773,
      "activations/layer20_attention_weight_min": -27.355512619018555,
      "activations/layer21_attention_weight_max": 35.28049087524414,
      "activations/layer21_attention_weight_min": -26.603715896606445,
      "activations/layer22_attention_weight_max": 47.98616027832031,
      "activations/layer22_attention_weight_min": -34.2896728515625,
      "activations/layer23_attention_weight_max": 33.46570587158203,
      "activations/layer23_attention_weight_min": -26.473203659057617,
      "activations/layer2_attention_weight_max": 35.669952392578125,
      "activations/layer2_attention_weight_min": -33.42436218261719,
      "activations/layer3_attention_weight_max": 113.2045669555664,
      "activations/layer3_attention_weight_min": -109.45431518554688,
      "activations/layer4_attention_weight_max": 116.65132141113281,
      "activations/layer4_attention_weight_min": -111.1261215209961,
      "activations/layer5_attention_weight_max": 54.92286682128906,
      "activations/layer5_attention_weight_min": -60.6507682800293,
      "activations/layer6_attention_weight_max": 48.560665130615234,
      "activations/layer6_attention_weight_min": -48.968021392822266,
      "activations/layer7_attention_weight_max": 106.50479888916016,
      "activations/layer7_attention_weight_min": -101.92557525634766,
      "activations/layer8_attention_weight_max": 47.59001541137695,
      "activations/layer8_attention_weight_min": -50.35740280151367,
      "activations/layer9_attention_weight_max": 36.249141693115234,
      "activations/layer9_attention_weight_min": -38.12518310546875,
      "epoch": 21.41,
      "learning_rate": 1.2041287878787878e-05,
      "loss": 2.6874,
      "step": 368500
    },
    {
      "activations/layer0_attention_weight_max": 16.67694854736328,
      "activations/layer0_attention_weight_min": -13.373610496520996,
      "activations/layer10_attention_weight_max": 36.23887252807617,
      "activations/layer10_attention_weight_min": -36.076663970947266,
      "activations/layer11_attention_weight_max": 35.50560760498047,
      "activations/layer11_attention_weight_min": -35.819305419921875,
      "activations/layer12_attention_weight_max": 25.33197593688965,
      "activations/layer12_attention_weight_min": -27.359575271606445,
      "activations/layer13_attention_weight_max": 37.45417404174805,
      "activations/layer13_attention_weight_min": -35.824859619140625,
      "activations/layer14_attention_weight_max": 41.58778381347656,
      "activations/layer14_attention_weight_min": -38.70313262939453,
      "activations/layer15_attention_weight_max": 35.79281997680664,
      "activations/layer15_attention_weight_min": -35.23278045654297,
      "activations/layer16_attention_weight_max": 29.01719856262207,
      "activations/layer16_attention_weight_min": -27.580337524414062,
      "activations/layer17_attention_weight_max": 28.155099868774414,
      "activations/layer17_attention_weight_min": -25.8823299407959,
      "activations/layer18_attention_weight_max": 33.15056610107422,
      "activations/layer18_attention_weight_min": -25.01764488220215,
      "activations/layer19_attention_weight_max": 31.034971237182617,
      "activations/layer19_attention_weight_min": -31.150650024414062,
      "activations/layer1_attention_weight_max": 16.013248443603516,
      "activations/layer1_attention_weight_min": -16.319995880126953,
      "activations/layer20_attention_weight_max": 27.136211395263672,
      "activations/layer20_attention_weight_min": -24.821189880371094,
      "activations/layer21_attention_weight_max": 26.04512596130371,
      "activations/layer21_attention_weight_min": -24.28722381591797,
      "activations/layer22_attention_weight_max": 43.38420486450195,
      "activations/layer22_attention_weight_min": -30.306028366088867,
      "activations/layer23_attention_weight_max": 30.146015167236328,
      "activations/layer23_attention_weight_min": -23.46048355102539,
      "activations/layer2_attention_weight_max": 34.14592742919922,
      "activations/layer2_attention_weight_min": -35.18631362915039,
      "activations/layer3_attention_weight_max": 105.23723602294922,
      "activations/layer3_attention_weight_min": -109.17264556884766,
      "activations/layer4_attention_weight_max": 107.2191390991211,
      "activations/layer4_attention_weight_min": -115.14493560791016,
      "activations/layer5_attention_weight_max": 50.54252243041992,
      "activations/layer5_attention_weight_min": -60.92870330810547,
      "activations/layer6_attention_weight_max": 46.96343231201172,
      "activations/layer6_attention_weight_min": -49.39281463623047,
      "activations/layer7_attention_weight_max": 101.26612854003906,
      "activations/layer7_attention_weight_min": -107.40144348144531,
      "activations/layer8_attention_weight_max": 42.89784240722656,
      "activations/layer8_attention_weight_min": -45.408206939697266,
      "activations/layer9_attention_weight_max": 33.16721725463867,
      "activations/layer9_attention_weight_min": -36.789154052734375,
      "epoch": 21.41,
      "learning_rate": 1.2022348484848484e-05,
      "loss": 2.7106,
      "step": 368550
    },
    {
      "activations/layer0_attention_weight_max": 16.92034912109375,
      "activations/layer0_attention_weight_min": -13.76581859588623,
      "activations/layer10_attention_weight_max": 52.025203704833984,
      "activations/layer10_attention_weight_min": -48.81767272949219,
      "activations/layer11_attention_weight_max": 49.146541595458984,
      "activations/layer11_attention_weight_min": -46.71323776245117,
      "activations/layer12_attention_weight_max": 37.51953887939453,
      "activations/layer12_attention_weight_min": -32.829689025878906,
      "activations/layer13_attention_weight_max": 48.264705657958984,
      "activations/layer13_attention_weight_min": -42.34078598022461,
      "activations/layer14_attention_weight_max": 55.53266143798828,
      "activations/layer14_attention_weight_min": -45.97966766357422,
      "activations/layer15_attention_weight_max": 60.81734848022461,
      "activations/layer15_attention_weight_min": -44.14133834838867,
      "activations/layer16_attention_weight_max": 41.237144470214844,
      "activations/layer16_attention_weight_min": -28.035320281982422,
      "activations/layer17_attention_weight_max": 34.09880447387695,
      "activations/layer17_attention_weight_min": -27.98374366760254,
      "activations/layer18_attention_weight_max": 36.8308219909668,
      "activations/layer18_attention_weight_min": -27.7945499420166,
      "activations/layer19_attention_weight_max": 38.90682601928711,
      "activations/layer19_attention_weight_min": -31.365924835205078,
      "activations/layer1_attention_weight_max": 16.293840408325195,
      "activations/layer1_attention_weight_min": -14.935321807861328,
      "activations/layer20_attention_weight_max": 30.248497009277344,
      "activations/layer20_attention_weight_min": -24.465036392211914,
      "activations/layer21_attention_weight_max": 29.607078552246094,
      "activations/layer21_attention_weight_min": -24.926158905029297,
      "activations/layer22_attention_weight_max": 46.268768310546875,
      "activations/layer22_attention_weight_min": -31.787059783935547,
      "activations/layer23_attention_weight_max": 35.090858459472656,
      "activations/layer23_attention_weight_min": -26.522855758666992,
      "activations/layer2_attention_weight_max": 35.797428131103516,
      "activations/layer2_attention_weight_min": -34.00275802612305,
      "activations/layer3_attention_weight_max": 105.77115631103516,
      "activations/layer3_attention_weight_min": -107.31333923339844,
      "activations/layer4_attention_weight_max": 109.51464080810547,
      "activations/layer4_attention_weight_min": -116.33290100097656,
      "activations/layer5_attention_weight_max": 53.88984298706055,
      "activations/layer5_attention_weight_min": -63.41769027709961,
      "activations/layer6_attention_weight_max": 48.099571228027344,
      "activations/layer6_attention_weight_min": -49.00413131713867,
      "activations/layer7_attention_weight_max": 105.50989532470703,
      "activations/layer7_attention_weight_min": -110.13399505615234,
      "activations/layer8_attention_weight_max": 50.41650390625,
      "activations/layer8_attention_weight_min": -50.70808410644531,
      "activations/layer9_attention_weight_max": 41.97402572631836,
      "activations/layer9_attention_weight_min": -42.9227180480957,
      "epoch": 21.42,
      "learning_rate": 1.200340909090909e-05,
      "loss": 2.7096,
      "step": 368600
    },
    {
      "activations/layer0_attention_weight_max": 17.070024490356445,
      "activations/layer0_attention_weight_min": -13.88320255279541,
      "activations/layer10_attention_weight_max": 34.897865295410156,
      "activations/layer10_attention_weight_min": -36.77191925048828,
      "activations/layer11_attention_weight_max": 34.94093704223633,
      "activations/layer11_attention_weight_min": -38.454811096191406,
      "activations/layer12_attention_weight_max": 26.407678604125977,
      "activations/layer12_attention_weight_min": -28.262598037719727,
      "activations/layer13_attention_weight_max": 40.071144104003906,
      "activations/layer13_attention_weight_min": -37.44624328613281,
      "activations/layer14_attention_weight_max": 41.532867431640625,
      "activations/layer14_attention_weight_min": -40.76091384887695,
      "activations/layer15_attention_weight_max": 38.18172836303711,
      "activations/layer15_attention_weight_min": -40.11056137084961,
      "activations/layer16_attention_weight_max": 29.15421485900879,
      "activations/layer16_attention_weight_min": -28.068279266357422,
      "activations/layer17_attention_weight_max": 28.730323791503906,
      "activations/layer17_attention_weight_min": -26.813995361328125,
      "activations/layer18_attention_weight_max": 31.335834503173828,
      "activations/layer18_attention_weight_min": -27.91506576538086,
      "activations/layer19_attention_weight_max": 32.8004264831543,
      "activations/layer19_attention_weight_min": -31.5632381439209,
      "activations/layer1_attention_weight_max": 15.781316757202148,
      "activations/layer1_attention_weight_min": -15.108718872070312,
      "activations/layer20_attention_weight_max": 27.171581268310547,
      "activations/layer20_attention_weight_min": -25.18181800842285,
      "activations/layer21_attention_weight_max": 27.65084457397461,
      "activations/layer21_attention_weight_min": -24.472768783569336,
      "activations/layer22_attention_weight_max": 42.46918487548828,
      "activations/layer22_attention_weight_min": -33.37446594238281,
      "activations/layer23_attention_weight_max": 30.934797286987305,
      "activations/layer23_attention_weight_min": -25.993019104003906,
      "activations/layer2_attention_weight_max": 33.412818908691406,
      "activations/layer2_attention_weight_min": -34.907615661621094,
      "activations/layer3_attention_weight_max": 102.65673065185547,
      "activations/layer3_attention_weight_min": -105.28730773925781,
      "activations/layer4_attention_weight_max": 111.58831787109375,
      "activations/layer4_attention_weight_min": -110.1727523803711,
      "activations/layer5_attention_weight_max": 50.83159637451172,
      "activations/layer5_attention_weight_min": -64.41075134277344,
      "activations/layer6_attention_weight_max": 44.3418083190918,
      "activations/layer6_attention_weight_min": -50.478145599365234,
      "activations/layer7_attention_weight_max": 93.31302642822266,
      "activations/layer7_attention_weight_min": -98.95506286621094,
      "activations/layer8_attention_weight_max": 43.20577621459961,
      "activations/layer8_attention_weight_min": -44.31960678100586,
      "activations/layer9_attention_weight_max": 33.50227737426758,
      "activations/layer9_attention_weight_min": -35.08765411376953,
      "epoch": 21.42,
      "learning_rate": 1.1984469696969696e-05,
      "loss": 2.7107,
      "step": 368650
    },
    {
      "activations/layer0_attention_weight_max": 16.320592880249023,
      "activations/layer0_attention_weight_min": -13.80207633972168,
      "activations/layer10_attention_weight_max": 37.69614028930664,
      "activations/layer10_attention_weight_min": -37.30564880371094,
      "activations/layer11_attention_weight_max": 35.975013732910156,
      "activations/layer11_attention_weight_min": -37.22599792480469,
      "activations/layer12_attention_weight_max": 28.4152774810791,
      "activations/layer12_attention_weight_min": -29.749561309814453,
      "activations/layer13_attention_weight_max": 42.61288070678711,
      "activations/layer13_attention_weight_min": -40.57685852050781,
      "activations/layer14_attention_weight_max": 45.32426452636719,
      "activations/layer14_attention_weight_min": -42.10676193237305,
      "activations/layer15_attention_weight_max": 41.454891204833984,
      "activations/layer15_attention_weight_min": -38.93260192871094,
      "activations/layer16_attention_weight_max": 30.330156326293945,
      "activations/layer16_attention_weight_min": -31.694913864135742,
      "activations/layer17_attention_weight_max": 34.2271728515625,
      "activations/layer17_attention_weight_min": -28.072099685668945,
      "activations/layer18_attention_weight_max": 33.63972473144531,
      "activations/layer18_attention_weight_min": -27.30654525756836,
      "activations/layer19_attention_weight_max": 33.44427490234375,
      "activations/layer19_attention_weight_min": -33.080379486083984,
      "activations/layer1_attention_weight_max": 16.28248405456543,
      "activations/layer1_attention_weight_min": -14.980240821838379,
      "activations/layer20_attention_weight_max": 29.747934341430664,
      "activations/layer20_attention_weight_min": -26.969276428222656,
      "activations/layer21_attention_weight_max": 29.22861671447754,
      "activations/layer21_attention_weight_min": -26.96381187438965,
      "activations/layer22_attention_weight_max": 50.64347457885742,
      "activations/layer22_attention_weight_min": -32.199195861816406,
      "activations/layer23_attention_weight_max": 32.44070053100586,
      "activations/layer23_attention_weight_min": -26.3641357421875,
      "activations/layer2_attention_weight_max": 34.568668365478516,
      "activations/layer2_attention_weight_min": -33.859161376953125,
      "activations/layer3_attention_weight_max": 111.73270416259766,
      "activations/layer3_attention_weight_min": -110.18252563476562,
      "activations/layer4_attention_weight_max": 116.58477783203125,
      "activations/layer4_attention_weight_min": -122.97071838378906,
      "activations/layer5_attention_weight_max": 53.02700424194336,
      "activations/layer5_attention_weight_min": -63.71722412109375,
      "activations/layer6_attention_weight_max": 50.33922576904297,
      "activations/layer6_attention_weight_min": -54.360355377197266,
      "activations/layer7_attention_weight_max": 108.35430145263672,
      "activations/layer7_attention_weight_min": -112.45103454589844,
      "activations/layer8_attention_weight_max": 47.31752014160156,
      "activations/layer8_attention_weight_min": -52.6926383972168,
      "activations/layer9_attention_weight_max": 34.64454650878906,
      "activations/layer9_attention_weight_min": -38.867488861083984,
      "epoch": 21.42,
      "learning_rate": 1.1965530303030302e-05,
      "loss": 2.7115,
      "step": 368700
    },
    {
      "activations/layer0_attention_weight_max": 16.00935935974121,
      "activations/layer0_attention_weight_min": -15.118062019348145,
      "activations/layer10_attention_weight_max": 36.66584014892578,
      "activations/layer10_attention_weight_min": -37.65824890136719,
      "activations/layer11_attention_weight_max": 34.786834716796875,
      "activations/layer11_attention_weight_min": -36.38116455078125,
      "activations/layer12_attention_weight_max": 28.550540924072266,
      "activations/layer12_attention_weight_min": -26.473819732666016,
      "activations/layer13_attention_weight_max": 41.377960205078125,
      "activations/layer13_attention_weight_min": -37.32720184326172,
      "activations/layer14_attention_weight_max": 45.289859771728516,
      "activations/layer14_attention_weight_min": -38.728755950927734,
      "activations/layer15_attention_weight_max": 42.51104736328125,
      "activations/layer15_attention_weight_min": -38.15540313720703,
      "activations/layer16_attention_weight_max": 29.64879035949707,
      "activations/layer16_attention_weight_min": -28.63213348388672,
      "activations/layer17_attention_weight_max": 30.09625244140625,
      "activations/layer17_attention_weight_min": -28.08333396911621,
      "activations/layer18_attention_weight_max": 35.22031021118164,
      "activations/layer18_attention_weight_min": -28.095590591430664,
      "activations/layer19_attention_weight_max": 40.47825241088867,
      "activations/layer19_attention_weight_min": -33.273590087890625,
      "activations/layer1_attention_weight_max": 15.964411735534668,
      "activations/layer1_attention_weight_min": -15.645829200744629,
      "activations/layer20_attention_weight_max": 30.989395141601562,
      "activations/layer20_attention_weight_min": -27.32501792907715,
      "activations/layer21_attention_weight_max": 31.733238220214844,
      "activations/layer21_attention_weight_min": -26.212749481201172,
      "activations/layer22_attention_weight_max": 53.35861587524414,
      "activations/layer22_attention_weight_min": -35.82803726196289,
      "activations/layer23_attention_weight_max": 33.92727279663086,
      "activations/layer23_attention_weight_min": -27.53437614440918,
      "activations/layer2_attention_weight_max": 34.101036071777344,
      "activations/layer2_attention_weight_min": -33.492427825927734,
      "activations/layer3_attention_weight_max": 103.31133270263672,
      "activations/layer3_attention_weight_min": -107.40709686279297,
      "activations/layer4_attention_weight_max": 110.0258560180664,
      "activations/layer4_attention_weight_min": -114.31917572021484,
      "activations/layer5_attention_weight_max": 50.79166793823242,
      "activations/layer5_attention_weight_min": -60.39632797241211,
      "activations/layer6_attention_weight_max": 45.373836517333984,
      "activations/layer6_attention_weight_min": -50.54220199584961,
      "activations/layer7_attention_weight_max": 101.12764739990234,
      "activations/layer7_attention_weight_min": -104.04926300048828,
      "activations/layer8_attention_weight_max": 44.13920974731445,
      "activations/layer8_attention_weight_min": -46.693397521972656,
      "activations/layer9_attention_weight_max": 32.76392364501953,
      "activations/layer9_attention_weight_min": -36.778289794921875,
      "epoch": 21.43,
      "learning_rate": 1.1946590909090907e-05,
      "loss": 2.7276,
      "step": 368750
    },
    {
      "activations/layer0_attention_weight_max": 15.607474327087402,
      "activations/layer0_attention_weight_min": -14.945075988769531,
      "activations/layer10_attention_weight_max": 38.666934967041016,
      "activations/layer10_attention_weight_min": -36.5691032409668,
      "activations/layer11_attention_weight_max": 38.201377868652344,
      "activations/layer11_attention_weight_min": -37.918983459472656,
      "activations/layer12_attention_weight_max": 30.436800003051758,
      "activations/layer12_attention_weight_min": -29.60670280456543,
      "activations/layer13_attention_weight_max": 46.0089225769043,
      "activations/layer13_attention_weight_min": -39.93595886230469,
      "activations/layer14_attention_weight_max": 49.56206130981445,
      "activations/layer14_attention_weight_min": -43.8114013671875,
      "activations/layer15_attention_weight_max": 52.182456970214844,
      "activations/layer15_attention_weight_min": -39.553157806396484,
      "activations/layer16_attention_weight_max": 34.55839920043945,
      "activations/layer16_attention_weight_min": -28.25676155090332,
      "activations/layer17_attention_weight_max": 31.22264289855957,
      "activations/layer17_attention_weight_min": -27.43355369567871,
      "activations/layer18_attention_weight_max": 31.676715850830078,
      "activations/layer18_attention_weight_min": -25.758758544921875,
      "activations/layer19_attention_weight_max": 37.48470687866211,
      "activations/layer19_attention_weight_min": -31.6295166015625,
      "activations/layer1_attention_weight_max": 16.755117416381836,
      "activations/layer1_attention_weight_min": -14.270532608032227,
      "activations/layer20_attention_weight_max": 31.343482971191406,
      "activations/layer20_attention_weight_min": -26.900007247924805,
      "activations/layer21_attention_weight_max": 31.987417221069336,
      "activations/layer21_attention_weight_min": -25.413904190063477,
      "activations/layer22_attention_weight_max": 44.55860137939453,
      "activations/layer22_attention_weight_min": -34.779640197753906,
      "activations/layer23_attention_weight_max": 34.92769241333008,
      "activations/layer23_attention_weight_min": -27.318721771240234,
      "activations/layer2_attention_weight_max": 34.73465347290039,
      "activations/layer2_attention_weight_min": -33.83640670776367,
      "activations/layer3_attention_weight_max": 106.81673431396484,
      "activations/layer3_attention_weight_min": -105.17951202392578,
      "activations/layer4_attention_weight_max": 112.95609283447266,
      "activations/layer4_attention_weight_min": -108.7612075805664,
      "activations/layer5_attention_weight_max": 53.34336853027344,
      "activations/layer5_attention_weight_min": -63.04303741455078,
      "activations/layer6_attention_weight_max": 49.50878143310547,
      "activations/layer6_attention_weight_min": -52.24393844604492,
      "activations/layer7_attention_weight_max": 102.340087890625,
      "activations/layer7_attention_weight_min": -107.78560638427734,
      "activations/layer8_attention_weight_max": 50.94942855834961,
      "activations/layer8_attention_weight_min": -50.45746994018555,
      "activations/layer9_attention_weight_max": 36.42172622680664,
      "activations/layer9_attention_weight_min": -37.69624328613281,
      "epoch": 21.43,
      "learning_rate": 1.1927651515151514e-05,
      "loss": 2.6969,
      "step": 368800
    },
    {
      "activations/layer0_attention_weight_max": 16.43181037902832,
      "activations/layer0_attention_weight_min": -13.444928169250488,
      "activations/layer10_attention_weight_max": 35.97803497314453,
      "activations/layer10_attention_weight_min": -36.1291618347168,
      "activations/layer11_attention_weight_max": 33.7368049621582,
      "activations/layer11_attention_weight_min": -36.668819427490234,
      "activations/layer12_attention_weight_max": 26.595829010009766,
      "activations/layer12_attention_weight_min": -29.379077911376953,
      "activations/layer13_attention_weight_max": 39.8724250793457,
      "activations/layer13_attention_weight_min": -38.78264236450195,
      "activations/layer14_attention_weight_max": 42.35451889038086,
      "activations/layer14_attention_weight_min": -41.49712371826172,
      "activations/layer15_attention_weight_max": 38.930259704589844,
      "activations/layer15_attention_weight_min": -37.456382751464844,
      "activations/layer16_attention_weight_max": 31.026138305664062,
      "activations/layer16_attention_weight_min": -27.341503143310547,
      "activations/layer17_attention_weight_max": 29.205163955688477,
      "activations/layer17_attention_weight_min": -27.271699905395508,
      "activations/layer18_attention_weight_max": 34.2602653503418,
      "activations/layer18_attention_weight_min": -27.579368591308594,
      "activations/layer19_attention_weight_max": 34.39199447631836,
      "activations/layer19_attention_weight_min": -35.03390884399414,
      "activations/layer1_attention_weight_max": 17.278484344482422,
      "activations/layer1_attention_weight_min": -16.903356552124023,
      "activations/layer20_attention_weight_max": 29.803112030029297,
      "activations/layer20_attention_weight_min": -28.152379989624023,
      "activations/layer21_attention_weight_max": 28.80451011657715,
      "activations/layer21_attention_weight_min": -28.51064109802246,
      "activations/layer22_attention_weight_max": 41.28641128540039,
      "activations/layer22_attention_weight_min": -34.145538330078125,
      "activations/layer23_attention_weight_max": 31.100936889648438,
      "activations/layer23_attention_weight_min": -27.221309661865234,
      "activations/layer2_attention_weight_max": 34.6124267578125,
      "activations/layer2_attention_weight_min": -33.72117614746094,
      "activations/layer3_attention_weight_max": 106.10376739501953,
      "activations/layer3_attention_weight_min": -105.76985168457031,
      "activations/layer4_attention_weight_max": 111.732421875,
      "activations/layer4_attention_weight_min": -114.18242645263672,
      "activations/layer5_attention_weight_max": 51.36119079589844,
      "activations/layer5_attention_weight_min": -63.92634582519531,
      "activations/layer6_attention_weight_max": 45.80802917480469,
      "activations/layer6_attention_weight_min": -51.0216178894043,
      "activations/layer7_attention_weight_max": 100.96910858154297,
      "activations/layer7_attention_weight_min": -107.63095092773438,
      "activations/layer8_attention_weight_max": 43.622779846191406,
      "activations/layer8_attention_weight_min": -47.54936218261719,
      "activations/layer9_attention_weight_max": 32.34485626220703,
      "activations/layer9_attention_weight_min": -35.9029426574707,
      "epoch": 21.43,
      "learning_rate": 1.190871212121212e-05,
      "loss": 2.7151,
      "step": 368850
    },
    {
      "activations/layer0_attention_weight_max": 16.386260986328125,
      "activations/layer0_attention_weight_min": -15.227579116821289,
      "activations/layer10_attention_weight_max": 35.36347198486328,
      "activations/layer10_attention_weight_min": -35.21926498413086,
      "activations/layer11_attention_weight_max": 35.45252990722656,
      "activations/layer11_attention_weight_min": -34.81700897216797,
      "activations/layer12_attention_weight_max": 29.6378231048584,
      "activations/layer12_attention_weight_min": -28.12863540649414,
      "activations/layer13_attention_weight_max": 41.389854431152344,
      "activations/layer13_attention_weight_min": -36.72136688232422,
      "activations/layer14_attention_weight_max": 44.33433532714844,
      "activations/layer14_attention_weight_min": -38.32023239135742,
      "activations/layer15_attention_weight_max": 41.48926544189453,
      "activations/layer15_attention_weight_min": -36.0814094543457,
      "activations/layer16_attention_weight_max": 32.290653228759766,
      "activations/layer16_attention_weight_min": -28.34944725036621,
      "activations/layer17_attention_weight_max": 30.678823471069336,
      "activations/layer17_attention_weight_min": -26.012779235839844,
      "activations/layer18_attention_weight_max": 33.20771026611328,
      "activations/layer18_attention_weight_min": -26.868017196655273,
      "activations/layer19_attention_weight_max": 37.011741638183594,
      "activations/layer19_attention_weight_min": -32.80015563964844,
      "activations/layer1_attention_weight_max": 15.867790222167969,
      "activations/layer1_attention_weight_min": -15.481362342834473,
      "activations/layer20_attention_weight_max": 30.35527801513672,
      "activations/layer20_attention_weight_min": -26.842533111572266,
      "activations/layer21_attention_weight_max": 32.06486892700195,
      "activations/layer21_attention_weight_min": -26.377450942993164,
      "activations/layer22_attention_weight_max": 42.8505744934082,
      "activations/layer22_attention_weight_min": -34.79529571533203,
      "activations/layer23_attention_weight_max": 33.53944778442383,
      "activations/layer23_attention_weight_min": -27.807823181152344,
      "activations/layer2_attention_weight_max": 34.2819938659668,
      "activations/layer2_attention_weight_min": -32.43401336669922,
      "activations/layer3_attention_weight_max": 108.96793365478516,
      "activations/layer3_attention_weight_min": -103.92147064208984,
      "activations/layer4_attention_weight_max": 113.95633697509766,
      "activations/layer4_attention_weight_min": -116.30074310302734,
      "activations/layer5_attention_weight_max": 53.675071716308594,
      "activations/layer5_attention_weight_min": -65.82017517089844,
      "activations/layer6_attention_weight_max": 49.08838653564453,
      "activations/layer6_attention_weight_min": -51.63326644897461,
      "activations/layer7_attention_weight_max": 100.57818603515625,
      "activations/layer7_attention_weight_min": -110.03968811035156,
      "activations/layer8_attention_weight_max": 45.75905227661133,
      "activations/layer8_attention_weight_min": -47.470245361328125,
      "activations/layer9_attention_weight_max": 31.965381622314453,
      "activations/layer9_attention_weight_min": -34.32294464111328,
      "epoch": 21.44,
      "learning_rate": 1.1889772727272725e-05,
      "loss": 2.7028,
      "step": 368900
    },
    {
      "activations/layer0_attention_weight_max": 16.70556640625,
      "activations/layer0_attention_weight_min": -15.226944923400879,
      "activations/layer10_attention_weight_max": 36.22966766357422,
      "activations/layer10_attention_weight_min": -35.53121566772461,
      "activations/layer11_attention_weight_max": 35.364463806152344,
      "activations/layer11_attention_weight_min": -35.876304626464844,
      "activations/layer12_attention_weight_max": 30.026697158813477,
      "activations/layer12_attention_weight_min": -27.926132202148438,
      "activations/layer13_attention_weight_max": 42.11616897583008,
      "activations/layer13_attention_weight_min": -37.45833969116211,
      "activations/layer14_attention_weight_max": 45.503273010253906,
      "activations/layer14_attention_weight_min": -38.536476135253906,
      "activations/layer15_attention_weight_max": 40.67831039428711,
      "activations/layer15_attention_weight_min": -36.21722412109375,
      "activations/layer16_attention_weight_max": 29.752870559692383,
      "activations/layer16_attention_weight_min": -28.413372039794922,
      "activations/layer17_attention_weight_max": 32.02814483642578,
      "activations/layer17_attention_weight_min": -28.141197204589844,
      "activations/layer18_attention_weight_max": 36.77318572998047,
      "activations/layer18_attention_weight_min": -27.297016143798828,
      "activations/layer19_attention_weight_max": 35.27470397949219,
      "activations/layer19_attention_weight_min": -33.80350875854492,
      "activations/layer1_attention_weight_max": 16.568145751953125,
      "activations/layer1_attention_weight_min": -15.471899032592773,
      "activations/layer20_attention_weight_max": 29.465002059936523,
      "activations/layer20_attention_weight_min": -25.91203498840332,
      "activations/layer21_attention_weight_max": 32.3480339050293,
      "activations/layer21_attention_weight_min": -26.83799934387207,
      "activations/layer22_attention_weight_max": 46.9529914855957,
      "activations/layer22_attention_weight_min": -34.07579803466797,
      "activations/layer23_attention_weight_max": 35.62519836425781,
      "activations/layer23_attention_weight_min": -27.48062515258789,
      "activations/layer2_attention_weight_max": 33.54596710205078,
      "activations/layer2_attention_weight_min": -33.520442962646484,
      "activations/layer3_attention_weight_max": 107.47039031982422,
      "activations/layer3_attention_weight_min": -108.84571838378906,
      "activations/layer4_attention_weight_max": 111.64910888671875,
      "activations/layer4_attention_weight_min": -112.97186279296875,
      "activations/layer5_attention_weight_max": 50.853172302246094,
      "activations/layer5_attention_weight_min": -61.79130172729492,
      "activations/layer6_attention_weight_max": 45.71204376220703,
      "activations/layer6_attention_weight_min": -48.11784744262695,
      "activations/layer7_attention_weight_max": 93.65612030029297,
      "activations/layer7_attention_weight_min": -100.23370361328125,
      "activations/layer8_attention_weight_max": 44.40705490112305,
      "activations/layer8_attention_weight_min": -45.68635940551758,
      "activations/layer9_attention_weight_max": 33.21950912475586,
      "activations/layer9_attention_weight_min": -34.51515197753906,
      "epoch": 21.44,
      "learning_rate": 1.1870833333333332e-05,
      "loss": 2.689,
      "step": 368950
    },
    {
      "activations/layer0_attention_weight_max": 16.381017684936523,
      "activations/layer0_attention_weight_min": -14.211493492126465,
      "activations/layer10_attention_weight_max": 39.489501953125,
      "activations/layer10_attention_weight_min": -40.32096862792969,
      "activations/layer11_attention_weight_max": 38.57784652709961,
      "activations/layer11_attention_weight_min": -38.530799865722656,
      "activations/layer12_attention_weight_max": 29.34492301940918,
      "activations/layer12_attention_weight_min": -28.226577758789062,
      "activations/layer13_attention_weight_max": 41.94502258300781,
      "activations/layer13_attention_weight_min": -38.05393600463867,
      "activations/layer14_attention_weight_max": 45.611419677734375,
      "activations/layer14_attention_weight_min": -39.78346633911133,
      "activations/layer15_attention_weight_max": 41.927879333496094,
      "activations/layer15_attention_weight_min": -37.95639419555664,
      "activations/layer16_attention_weight_max": 29.251432418823242,
      "activations/layer16_attention_weight_min": -28.638505935668945,
      "activations/layer17_attention_weight_max": 29.991939544677734,
      "activations/layer17_attention_weight_min": -26.528568267822266,
      "activations/layer18_attention_weight_max": 32.594505310058594,
      "activations/layer18_attention_weight_min": -24.271480560302734,
      "activations/layer19_attention_weight_max": 35.543121337890625,
      "activations/layer19_attention_weight_min": -31.23296546936035,
      "activations/layer1_attention_weight_max": 15.189326286315918,
      "activations/layer1_attention_weight_min": -15.125105857849121,
      "activations/layer20_attention_weight_max": 30.526948928833008,
      "activations/layer20_attention_weight_min": -25.792818069458008,
      "activations/layer21_attention_weight_max": 28.477447509765625,
      "activations/layer21_attention_weight_min": -25.40016746520996,
      "activations/layer22_attention_weight_max": 41.38482666015625,
      "activations/layer22_attention_weight_min": -32.60784912109375,
      "activations/layer23_attention_weight_max": 31.9608154296875,
      "activations/layer23_attention_weight_min": -26.01373863220215,
      "activations/layer2_attention_weight_max": 34.897499084472656,
      "activations/layer2_attention_weight_min": -34.131858825683594,
      "activations/layer3_attention_weight_max": 108.48194885253906,
      "activations/layer3_attention_weight_min": -109.75009155273438,
      "activations/layer4_attention_weight_max": 114.3995590209961,
      "activations/layer4_attention_weight_min": -114.4647445678711,
      "activations/layer5_attention_weight_max": 52.95927429199219,
      "activations/layer5_attention_weight_min": -62.153404235839844,
      "activations/layer6_attention_weight_max": 46.669620513916016,
      "activations/layer6_attention_weight_min": -49.87381362915039,
      "activations/layer7_attention_weight_max": 102.2662124633789,
      "activations/layer7_attention_weight_min": -106.05851745605469,
      "activations/layer8_attention_weight_max": 45.42092514038086,
      "activations/layer8_attention_weight_min": -49.641963958740234,
      "activations/layer9_attention_weight_max": 35.92831039428711,
      "activations/layer9_attention_weight_min": -38.78042221069336,
      "epoch": 21.44,
      "learning_rate": 1.1851893939393938e-05,
      "loss": 2.6917,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.1921,
      "eval_samples_per_second": 524.165,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.1921,
      "eval_openwebtext_samples_per_second": 524.165,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9714,
      "eval_wikitext_samples_per_second": 231.312,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_lambada_loss": 2.337890625,
      "eval_lambada_ppl": 10.359361721090316,
      "eval_lambada_runtime": 9.2911,
      "eval_lambada_samples_per_second": 524.049,
      "step": 369000
    },
    {
      "activations/layer0_attention_weight_max": 17.471309661865234,
      "activations/layer0_attention_weight_min": -14.055012702941895,
      "activations/layer10_attention_weight_max": 33.76089096069336,
      "activations/layer10_attention_weight_min": -35.62416458129883,
      "activations/layer11_attention_weight_max": 34.26590347290039,
      "activations/layer11_attention_weight_min": -34.50714874267578,
      "activations/layer12_attention_weight_max": 26.976306915283203,
      "activations/layer12_attention_weight_min": -26.565208435058594,
      "activations/layer13_attention_weight_max": 39.7160530090332,
      "activations/layer13_attention_weight_min": -35.31028366088867,
      "activations/layer14_attention_weight_max": 42.51490020751953,
      "activations/layer14_attention_weight_min": -40.25130844116211,
      "activations/layer15_attention_weight_max": 40.06585693359375,
      "activations/layer15_attention_weight_min": -39.7943115234375,
      "activations/layer16_attention_weight_max": 28.073060989379883,
      "activations/layer16_attention_weight_min": -28.143712997436523,
      "activations/layer17_attention_weight_max": 29.417354583740234,
      "activations/layer17_attention_weight_min": -27.22051429748535,
      "activations/layer18_attention_weight_max": 31.39373207092285,
      "activations/layer18_attention_weight_min": -27.192569732666016,
      "activations/layer19_attention_weight_max": 36.15073013305664,
      "activations/layer19_attention_weight_min": -30.495620727539062,
      "activations/layer1_attention_weight_max": 15.184657096862793,
      "activations/layer1_attention_weight_min": -14.749281883239746,
      "activations/layer20_attention_weight_max": 27.680580139160156,
      "activations/layer20_attention_weight_min": -23.98031997680664,
      "activations/layer21_attention_weight_max": 28.25446128845215,
      "activations/layer21_attention_weight_min": -24.140953063964844,
      "activations/layer22_attention_weight_max": 38.798606872558594,
      "activations/layer22_attention_weight_min": -29.32677459716797,
      "activations/layer23_attention_weight_max": 33.55858612060547,
      "activations/layer23_attention_weight_min": -24.452190399169922,
      "activations/layer2_attention_weight_max": 34.72297668457031,
      "activations/layer2_attention_weight_min": -34.10240173339844,
      "activations/layer3_attention_weight_max": 104.30940246582031,
      "activations/layer3_attention_weight_min": -106.72449493408203,
      "activations/layer4_attention_weight_max": 107.61095428466797,
      "activations/layer4_attention_weight_min": -108.43450927734375,
      "activations/layer5_attention_weight_max": 48.19231414794922,
      "activations/layer5_attention_weight_min": -64.35275268554688,
      "activations/layer6_attention_weight_max": 44.184566497802734,
      "activations/layer6_attention_weight_min": -48.52754592895508,
      "activations/layer7_attention_weight_max": 92.70878601074219,
      "activations/layer7_attention_weight_min": -101.5646743774414,
      "activations/layer8_attention_weight_max": 41.32818603515625,
      "activations/layer8_attention_weight_min": -45.564414978027344,
      "activations/layer9_attention_weight_max": 32.36336135864258,
      "activations/layer9_attention_weight_min": -35.66030502319336,
      "epoch": 21.44,
      "learning_rate": 1.1832954545454543e-05,
      "loss": 2.7151,
      "step": 369050
    },
    {
      "activations/layer0_attention_weight_max": 16.649078369140625,
      "activations/layer0_attention_weight_min": -15.605216979980469,
      "activations/layer10_attention_weight_max": 38.23225784301758,
      "activations/layer10_attention_weight_min": -37.114776611328125,
      "activations/layer11_attention_weight_max": 37.8334846496582,
      "activations/layer11_attention_weight_min": -37.29096221923828,
      "activations/layer12_attention_weight_max": 27.879932403564453,
      "activations/layer12_attention_weight_min": -28.674192428588867,
      "activations/layer13_attention_weight_max": 43.19174575805664,
      "activations/layer13_attention_weight_min": -38.65217971801758,
      "activations/layer14_attention_weight_max": 47.169700622558594,
      "activations/layer14_attention_weight_min": -40.98170471191406,
      "activations/layer15_attention_weight_max": 42.04638671875,
      "activations/layer15_attention_weight_min": -38.55054473876953,
      "activations/layer16_attention_weight_max": 31.535669326782227,
      "activations/layer16_attention_weight_min": -29.16552734375,
      "activations/layer17_attention_weight_max": 31.530744552612305,
      "activations/layer17_attention_weight_min": -25.2147159576416,
      "activations/layer18_attention_weight_max": 30.748470306396484,
      "activations/layer18_attention_weight_min": -26.035661697387695,
      "activations/layer19_attention_weight_max": 36.03681182861328,
      "activations/layer19_attention_weight_min": -30.072996139526367,
      "activations/layer1_attention_weight_max": 14.758477210998535,
      "activations/layer1_attention_weight_min": -15.266148567199707,
      "activations/layer20_attention_weight_max": 31.918773651123047,
      "activations/layer20_attention_weight_min": -26.546546936035156,
      "activations/layer21_attention_weight_max": 31.43404197692871,
      "activations/layer21_attention_weight_min": -25.274986267089844,
      "activations/layer22_attention_weight_max": 44.26393127441406,
      "activations/layer22_attention_weight_min": -34.654510498046875,
      "activations/layer23_attention_weight_max": 35.271514892578125,
      "activations/layer23_attention_weight_min": -24.275955200195312,
      "activations/layer2_attention_weight_max": 35.238399505615234,
      "activations/layer2_attention_weight_min": -33.335811614990234,
      "activations/layer3_attention_weight_max": 104.95071411132812,
      "activations/layer3_attention_weight_min": -105.43409729003906,
      "activations/layer4_attention_weight_max": 112.60787200927734,
      "activations/layer4_attention_weight_min": -112.9109878540039,
      "activations/layer5_attention_weight_max": 52.65736389160156,
      "activations/layer5_attention_weight_min": -63.12095260620117,
      "activations/layer6_attention_weight_max": 46.749027252197266,
      "activations/layer6_attention_weight_min": -50.15031433105469,
      "activations/layer7_attention_weight_max": 103.71720886230469,
      "activations/layer7_attention_weight_min": -103.192626953125,
      "activations/layer8_attention_weight_max": 47.20437240600586,
      "activations/layer8_attention_weight_min": -47.46829605102539,
      "activations/layer9_attention_weight_max": 35.10543441772461,
      "activations/layer9_attention_weight_min": -36.49293518066406,
      "epoch": 21.45,
      "learning_rate": 1.1814015151515151e-05,
      "loss": 2.7176,
      "step": 369100
    },
    {
      "activations/layer0_attention_weight_max": 16.33180809020996,
      "activations/layer0_attention_weight_min": -16.11721420288086,
      "activations/layer10_attention_weight_max": 45.565147399902344,
      "activations/layer10_attention_weight_min": -43.38801956176758,
      "activations/layer11_attention_weight_max": 43.76171875,
      "activations/layer11_attention_weight_min": -45.87371826171875,
      "activations/layer12_attention_weight_max": 27.79986000061035,
      "activations/layer12_attention_weight_min": -29.70130157470703,
      "activations/layer13_attention_weight_max": 40.821197509765625,
      "activations/layer13_attention_weight_min": -36.801021575927734,
      "activations/layer14_attention_weight_max": 44.37785339355469,
      "activations/layer14_attention_weight_min": -39.64713668823242,
      "activations/layer15_attention_weight_max": 40.22368240356445,
      "activations/layer15_attention_weight_min": -43.41579818725586,
      "activations/layer16_attention_weight_max": 29.811992645263672,
      "activations/layer16_attention_weight_min": -29.09908103942871,
      "activations/layer17_attention_weight_max": 28.831552505493164,
      "activations/layer17_attention_weight_min": -29.888994216918945,
      "activations/layer18_attention_weight_max": 31.460819244384766,
      "activations/layer18_attention_weight_min": -26.954208374023438,
      "activations/layer19_attention_weight_max": 32.96488571166992,
      "activations/layer19_attention_weight_min": -33.635799407958984,
      "activations/layer1_attention_weight_max": 16.279550552368164,
      "activations/layer1_attention_weight_min": -15.002984046936035,
      "activations/layer20_attention_weight_max": 28.790653228759766,
      "activations/layer20_attention_weight_min": -25.193151473999023,
      "activations/layer21_attention_weight_max": 28.559232711791992,
      "activations/layer21_attention_weight_min": -25.37906837463379,
      "activations/layer22_attention_weight_max": 40.90497589111328,
      "activations/layer22_attention_weight_min": -33.232017517089844,
      "activations/layer23_attention_weight_max": 30.890165328979492,
      "activations/layer23_attention_weight_min": -24.655309677124023,
      "activations/layer2_attention_weight_max": 34.82137680053711,
      "activations/layer2_attention_weight_min": -33.46421432495117,
      "activations/layer3_attention_weight_max": 110.53834533691406,
      "activations/layer3_attention_weight_min": -107.48853302001953,
      "activations/layer4_attention_weight_max": 110.81719207763672,
      "activations/layer4_attention_weight_min": -115.1791000366211,
      "activations/layer5_attention_weight_max": 53.87458801269531,
      "activations/layer5_attention_weight_min": -60.21334457397461,
      "activations/layer6_attention_weight_max": 45.38723373413086,
      "activations/layer6_attention_weight_min": -49.606143951416016,
      "activations/layer7_attention_weight_max": 113.53529357910156,
      "activations/layer7_attention_weight_min": -108.32891845703125,
      "activations/layer8_attention_weight_max": 50.826297760009766,
      "activations/layer8_attention_weight_min": -50.79922866821289,
      "activations/layer9_attention_weight_max": 35.64986801147461,
      "activations/layer9_attention_weight_min": -39.1362419128418,
      "epoch": 21.45,
      "learning_rate": 1.1795075757575758e-05,
      "loss": 2.7033,
      "step": 369150
    },
    {
      "activations/layer0_attention_weight_max": 16.44788360595703,
      "activations/layer0_attention_weight_min": -14.223403930664062,
      "activations/layer10_attention_weight_max": 35.9028434753418,
      "activations/layer10_attention_weight_min": -35.7349967956543,
      "activations/layer11_attention_weight_max": 37.300537109375,
      "activations/layer11_attention_weight_min": -35.69729232788086,
      "activations/layer12_attention_weight_max": 27.979276657104492,
      "activations/layer12_attention_weight_min": -27.117158889770508,
      "activations/layer13_attention_weight_max": 40.24372482299805,
      "activations/layer13_attention_weight_min": -36.94297790527344,
      "activations/layer14_attention_weight_max": 41.79029083251953,
      "activations/layer14_attention_weight_min": -39.13101577758789,
      "activations/layer15_attention_weight_max": 39.0988883972168,
      "activations/layer15_attention_weight_min": -40.39081573486328,
      "activations/layer16_attention_weight_max": 30.161514282226562,
      "activations/layer16_attention_weight_min": -29.156206130981445,
      "activations/layer17_attention_weight_max": 34.3758544921875,
      "activations/layer17_attention_weight_min": -28.66605567932129,
      "activations/layer18_attention_weight_max": 30.849056243896484,
      "activations/layer18_attention_weight_min": -27.859285354614258,
      "activations/layer19_attention_weight_max": 36.57387161254883,
      "activations/layer19_attention_weight_min": -34.82639694213867,
      "activations/layer1_attention_weight_max": 15.911981582641602,
      "activations/layer1_attention_weight_min": -14.504191398620605,
      "activations/layer20_attention_weight_max": 32.71977996826172,
      "activations/layer20_attention_weight_min": -24.994516372680664,
      "activations/layer21_attention_weight_max": 30.060503005981445,
      "activations/layer21_attention_weight_min": -24.43503761291504,
      "activations/layer22_attention_weight_max": 42.031211853027344,
      "activations/layer22_attention_weight_min": -32.83832550048828,
      "activations/layer23_attention_weight_max": 32.193267822265625,
      "activations/layer23_attention_weight_min": -25.984312057495117,
      "activations/layer2_attention_weight_max": 33.10050964355469,
      "activations/layer2_attention_weight_min": -31.889537811279297,
      "activations/layer3_attention_weight_max": 105.90036010742188,
      "activations/layer3_attention_weight_min": -105.74137878417969,
      "activations/layer4_attention_weight_max": 113.76165008544922,
      "activations/layer4_attention_weight_min": -111.44850158691406,
      "activations/layer5_attention_weight_max": 51.612762451171875,
      "activations/layer5_attention_weight_min": -64.62272644042969,
      "activations/layer6_attention_weight_max": 46.06605529785156,
      "activations/layer6_attention_weight_min": -48.18778610229492,
      "activations/layer7_attention_weight_max": 95.82984924316406,
      "activations/layer7_attention_weight_min": -102.93603515625,
      "activations/layer8_attention_weight_max": 42.49998092651367,
      "activations/layer8_attention_weight_min": -44.70039367675781,
      "activations/layer9_attention_weight_max": 32.39612579345703,
      "activations/layer9_attention_weight_min": -34.5854606628418,
      "epoch": 21.45,
      "learning_rate": 1.1776136363636363e-05,
      "loss": 2.7133,
      "step": 369200
    },
    {
      "activations/layer0_attention_weight_max": 15.535676956176758,
      "activations/layer0_attention_weight_min": -14.279698371887207,
      "activations/layer10_attention_weight_max": 34.71250534057617,
      "activations/layer10_attention_weight_min": -34.63648223876953,
      "activations/layer11_attention_weight_max": 31.60330581665039,
      "activations/layer11_attention_weight_min": -34.895896911621094,
      "activations/layer12_attention_weight_max": 26.561389923095703,
      "activations/layer12_attention_weight_min": -25.676782608032227,
      "activations/layer13_attention_weight_max": 39.23692321777344,
      "activations/layer13_attention_weight_min": -34.53902816772461,
      "activations/layer14_attention_weight_max": 43.192176818847656,
      "activations/layer14_attention_weight_min": -37.5796012878418,
      "activations/layer15_attention_weight_max": 40.258663177490234,
      "activations/layer15_attention_weight_min": -37.04017639160156,
      "activations/layer16_attention_weight_max": 29.363191604614258,
      "activations/layer16_attention_weight_min": -27.771055221557617,
      "activations/layer17_attention_weight_max": 29.770233154296875,
      "activations/layer17_attention_weight_min": -25.772672653198242,
      "activations/layer18_attention_weight_max": 31.817960739135742,
      "activations/layer18_attention_weight_min": -26.25643539428711,
      "activations/layer19_attention_weight_max": 35.585636138916016,
      "activations/layer19_attention_weight_min": -32.81144332885742,
      "activations/layer1_attention_weight_max": 15.619421005249023,
      "activations/layer1_attention_weight_min": -15.351325035095215,
      "activations/layer20_attention_weight_max": 27.356843948364258,
      "activations/layer20_attention_weight_min": -24.442975997924805,
      "activations/layer21_attention_weight_max": 27.44857406616211,
      "activations/layer21_attention_weight_min": -24.236047744750977,
      "activations/layer22_attention_weight_max": 41.38388442993164,
      "activations/layer22_attention_weight_min": -30.597227096557617,
      "activations/layer23_attention_weight_max": 31.407989501953125,
      "activations/layer23_attention_weight_min": -24.803768157958984,
      "activations/layer2_attention_weight_max": 33.14761734008789,
      "activations/layer2_attention_weight_min": -32.59098815917969,
      "activations/layer3_attention_weight_max": 100.2482681274414,
      "activations/layer3_attention_weight_min": -101.19017791748047,
      "activations/layer4_attention_weight_max": 103.42894744873047,
      "activations/layer4_attention_weight_min": -110.94266510009766,
      "activations/layer5_attention_weight_max": 49.37535858154297,
      "activations/layer5_attention_weight_min": -58.34096908569336,
      "activations/layer6_attention_weight_max": 43.854881286621094,
      "activations/layer6_attention_weight_min": -46.69576644897461,
      "activations/layer7_attention_weight_max": 92.51654815673828,
      "activations/layer7_attention_weight_min": -93.78787994384766,
      "activations/layer8_attention_weight_max": 40.93085861206055,
      "activations/layer8_attention_weight_min": -43.97744369506836,
      "activations/layer9_attention_weight_max": 33.01357650756836,
      "activations/layer9_attention_weight_min": -33.211875915527344,
      "epoch": 21.46,
      "learning_rate": 1.175719696969697e-05,
      "loss": 2.7148,
      "step": 369250
    },
    {
      "activations/layer0_attention_weight_max": 16.764686584472656,
      "activations/layer0_attention_weight_min": -13.078676223754883,
      "activations/layer10_attention_weight_max": 35.104469299316406,
      "activations/layer10_attention_weight_min": -35.366546630859375,
      "activations/layer11_attention_weight_max": 35.78050994873047,
      "activations/layer11_attention_weight_min": -33.81916046142578,
      "activations/layer12_attention_weight_max": 28.958179473876953,
      "activations/layer12_attention_weight_min": -27.097856521606445,
      "activations/layer13_attention_weight_max": 38.890254974365234,
      "activations/layer13_attention_weight_min": -36.18241500854492,
      "activations/layer14_attention_weight_max": 43.17310333251953,
      "activations/layer14_attention_weight_min": -39.30992126464844,
      "activations/layer15_attention_weight_max": 38.954254150390625,
      "activations/layer15_attention_weight_min": -36.44253158569336,
      "activations/layer16_attention_weight_max": 30.959192276000977,
      "activations/layer16_attention_weight_min": -27.250659942626953,
      "activations/layer17_attention_weight_max": 29.491167068481445,
      "activations/layer17_attention_weight_min": -27.592737197875977,
      "activations/layer18_attention_weight_max": 32.040679931640625,
      "activations/layer18_attention_weight_min": -25.642396926879883,
      "activations/layer19_attention_weight_max": 35.89482116699219,
      "activations/layer19_attention_weight_min": -30.96866798400879,
      "activations/layer1_attention_weight_max": 15.620467185974121,
      "activations/layer1_attention_weight_min": -14.06375789642334,
      "activations/layer20_attention_weight_max": 29.94918441772461,
      "activations/layer20_attention_weight_min": -24.863431930541992,
      "activations/layer21_attention_weight_max": 27.61638069152832,
      "activations/layer21_attention_weight_min": -24.124479293823242,
      "activations/layer22_attention_weight_max": 43.940040588378906,
      "activations/layer22_attention_weight_min": -32.00168228149414,
      "activations/layer23_attention_weight_max": 33.11362838745117,
      "activations/layer23_attention_weight_min": -26.491567611694336,
      "activations/layer2_attention_weight_max": 34.64112854003906,
      "activations/layer2_attention_weight_min": -32.17986297607422,
      "activations/layer3_attention_weight_max": 100.48773956298828,
      "activations/layer3_attention_weight_min": -104.65666198730469,
      "activations/layer4_attention_weight_max": 105.16263580322266,
      "activations/layer4_attention_weight_min": -111.39656829833984,
      "activations/layer5_attention_weight_max": 50.00323486328125,
      "activations/layer5_attention_weight_min": -59.81209945678711,
      "activations/layer6_attention_weight_max": 45.264122009277344,
      "activations/layer6_attention_weight_min": -46.913246154785156,
      "activations/layer7_attention_weight_max": 94.47723388671875,
      "activations/layer7_attention_weight_min": -100.38629150390625,
      "activations/layer8_attention_weight_max": 46.36279296875,
      "activations/layer8_attention_weight_min": -44.3255615234375,
      "activations/layer9_attention_weight_max": 33.4176139831543,
      "activations/layer9_attention_weight_min": -34.95541000366211,
      "epoch": 21.46,
      "learning_rate": 1.1738257575757576e-05,
      "loss": 2.7039,
      "step": 369300
    },
    {
      "activations/layer0_attention_weight_max": 16.673982620239258,
      "activations/layer0_attention_weight_min": -14.229266166687012,
      "activations/layer10_attention_weight_max": 35.16556930541992,
      "activations/layer10_attention_weight_min": -36.67304611206055,
      "activations/layer11_attention_weight_max": 34.03572082519531,
      "activations/layer11_attention_weight_min": -37.12327194213867,
      "activations/layer12_attention_weight_max": 25.176193237304688,
      "activations/layer12_attention_weight_min": -26.6423397064209,
      "activations/layer13_attention_weight_max": 36.6382942199707,
      "activations/layer13_attention_weight_min": -35.45131301879883,
      "activations/layer14_attention_weight_max": 42.44987106323242,
      "activations/layer14_attention_weight_min": -38.375179290771484,
      "activations/layer15_attention_weight_max": 38.302181243896484,
      "activations/layer15_attention_weight_min": -41.693603515625,
      "activations/layer16_attention_weight_max": 27.684650421142578,
      "activations/layer16_attention_weight_min": -28.14752769470215,
      "activations/layer17_attention_weight_max": 29.001197814941406,
      "activations/layer17_attention_weight_min": -25.34075927734375,
      "activations/layer18_attention_weight_max": 30.029146194458008,
      "activations/layer18_attention_weight_min": -26.521406173706055,
      "activations/layer19_attention_weight_max": 30.84541893005371,
      "activations/layer19_attention_weight_min": -29.697418212890625,
      "activations/layer1_attention_weight_max": 17.054733276367188,
      "activations/layer1_attention_weight_min": -15.058967590332031,
      "activations/layer20_attention_weight_max": 27.2330379486084,
      "activations/layer20_attention_weight_min": -25.462419509887695,
      "activations/layer21_attention_weight_max": 27.115402221679688,
      "activations/layer21_attention_weight_min": -24.462493896484375,
      "activations/layer22_attention_weight_max": 37.206932067871094,
      "activations/layer22_attention_weight_min": -31.534685134887695,
      "activations/layer23_attention_weight_max": 32.330482482910156,
      "activations/layer23_attention_weight_min": -28.004117965698242,
      "activations/layer2_attention_weight_max": 33.36409378051758,
      "activations/layer2_attention_weight_min": -33.4283447265625,
      "activations/layer3_attention_weight_max": 104.80582427978516,
      "activations/layer3_attention_weight_min": -106.62733459472656,
      "activations/layer4_attention_weight_max": 106.18704986572266,
      "activations/layer4_attention_weight_min": -108.43944549560547,
      "activations/layer5_attention_weight_max": 49.113685607910156,
      "activations/layer5_attention_weight_min": -62.165771484375,
      "activations/layer6_attention_weight_max": 46.556434631347656,
      "activations/layer6_attention_weight_min": -47.437076568603516,
      "activations/layer7_attention_weight_max": 92.94282531738281,
      "activations/layer7_attention_weight_min": -95.11773681640625,
      "activations/layer8_attention_weight_max": 42.10240173339844,
      "activations/layer8_attention_weight_min": -43.422752380371094,
      "activations/layer9_attention_weight_max": 34.18520736694336,
      "activations/layer9_attention_weight_min": -35.054134368896484,
      "epoch": 21.46,
      "learning_rate": 1.171931818181818e-05,
      "loss": 2.6962,
      "step": 369350
    },
    {
      "activations/layer0_attention_weight_max": 17.00490951538086,
      "activations/layer0_attention_weight_min": -13.885957717895508,
      "activations/layer10_attention_weight_max": 33.20962905883789,
      "activations/layer10_attention_weight_min": -35.762733459472656,
      "activations/layer11_attention_weight_max": 34.08430480957031,
      "activations/layer11_attention_weight_min": -36.38752746582031,
      "activations/layer12_attention_weight_max": 26.018224716186523,
      "activations/layer12_attention_weight_min": -27.091047286987305,
      "activations/layer13_attention_weight_max": 39.96525573730469,
      "activations/layer13_attention_weight_min": -37.93136215209961,
      "activations/layer14_attention_weight_max": 41.9224739074707,
      "activations/layer14_attention_weight_min": -39.23017120361328,
      "activations/layer15_attention_weight_max": 38.191471099853516,
      "activations/layer15_attention_weight_min": -40.229366302490234,
      "activations/layer16_attention_weight_max": 27.643205642700195,
      "activations/layer16_attention_weight_min": -27.206838607788086,
      "activations/layer17_attention_weight_max": 28.36338233947754,
      "activations/layer17_attention_weight_min": -24.572309494018555,
      "activations/layer18_attention_weight_max": 31.809612274169922,
      "activations/layer18_attention_weight_min": -27.503707885742188,
      "activations/layer19_attention_weight_max": 33.30354309082031,
      "activations/layer19_attention_weight_min": -31.614294052124023,
      "activations/layer1_attention_weight_max": 16.509992599487305,
      "activations/layer1_attention_weight_min": -15.085700988769531,
      "activations/layer20_attention_weight_max": 27.316267013549805,
      "activations/layer20_attention_weight_min": -25.720678329467773,
      "activations/layer21_attention_weight_max": 27.908830642700195,
      "activations/layer21_attention_weight_min": -25.51715087890625,
      "activations/layer22_attention_weight_max": 41.65098190307617,
      "activations/layer22_attention_weight_min": -30.86943817138672,
      "activations/layer23_attention_weight_max": 32.23481750488281,
      "activations/layer23_attention_weight_min": -27.090377807617188,
      "activations/layer2_attention_weight_max": 33.96363067626953,
      "activations/layer2_attention_weight_min": -32.48769760131836,
      "activations/layer3_attention_weight_max": 100.67137908935547,
      "activations/layer3_attention_weight_min": -102.33827209472656,
      "activations/layer4_attention_weight_max": 106.3741455078125,
      "activations/layer4_attention_weight_min": -113.6076889038086,
      "activations/layer5_attention_weight_max": 50.34139633178711,
      "activations/layer5_attention_weight_min": -60.04917907714844,
      "activations/layer6_attention_weight_max": 44.21154022216797,
      "activations/layer6_attention_weight_min": -46.402381896972656,
      "activations/layer7_attention_weight_max": 92.13011169433594,
      "activations/layer7_attention_weight_min": -96.7425308227539,
      "activations/layer8_attention_weight_max": 41.019081115722656,
      "activations/layer8_attention_weight_min": -43.518394470214844,
      "activations/layer9_attention_weight_max": 31.467884063720703,
      "activations/layer9_attention_weight_min": -35.944950103759766,
      "epoch": 21.46,
      "learning_rate": 1.1700378787878787e-05,
      "loss": 2.7157,
      "step": 369400
    },
    {
      "activations/layer0_attention_weight_max": 16.743074417114258,
      "activations/layer0_attention_weight_min": -14.030289649963379,
      "activations/layer10_attention_weight_max": 35.28499221801758,
      "activations/layer10_attention_weight_min": -35.6954345703125,
      "activations/layer11_attention_weight_max": 33.95915603637695,
      "activations/layer11_attention_weight_min": -35.55107116699219,
      "activations/layer12_attention_weight_max": 25.297605514526367,
      "activations/layer12_attention_weight_min": -27.67584800720215,
      "activations/layer13_attention_weight_max": 36.040924072265625,
      "activations/layer13_attention_weight_min": -36.36487579345703,
      "activations/layer14_attention_weight_max": 38.51879119873047,
      "activations/layer14_attention_weight_min": -36.786251068115234,
      "activations/layer15_attention_weight_max": 35.86509323120117,
      "activations/layer15_attention_weight_min": -34.791507720947266,
      "activations/layer16_attention_weight_max": 28.8038330078125,
      "activations/layer16_attention_weight_min": -29.445735931396484,
      "activations/layer17_attention_weight_max": 28.312267303466797,
      "activations/layer17_attention_weight_min": -25.151039123535156,
      "activations/layer18_attention_weight_max": 32.29759979248047,
      "activations/layer18_attention_weight_min": -25.217647552490234,
      "activations/layer19_attention_weight_max": 33.60227966308594,
      "activations/layer19_attention_weight_min": -30.762802124023438,
      "activations/layer1_attention_weight_max": 15.6681489944458,
      "activations/layer1_attention_weight_min": -14.375646591186523,
      "activations/layer20_attention_weight_max": 29.173477172851562,
      "activations/layer20_attention_weight_min": -25.644880294799805,
      "activations/layer21_attention_weight_max": 27.747440338134766,
      "activations/layer21_attention_weight_min": -25.123682022094727,
      "activations/layer22_attention_weight_max": 39.91545486450195,
      "activations/layer22_attention_weight_min": -28.96738052368164,
      "activations/layer23_attention_weight_max": 32.24980163574219,
      "activations/layer23_attention_weight_min": -25.19300651550293,
      "activations/layer2_attention_weight_max": 31.829700469970703,
      "activations/layer2_attention_weight_min": -32.78383255004883,
      "activations/layer3_attention_weight_max": 102.06375122070312,
      "activations/layer3_attention_weight_min": -105.7245864868164,
      "activations/layer4_attention_weight_max": 107.9117202758789,
      "activations/layer4_attention_weight_min": -110.75504302978516,
      "activations/layer5_attention_weight_max": 53.171287536621094,
      "activations/layer5_attention_weight_min": -62.74929428100586,
      "activations/layer6_attention_weight_max": 45.191917419433594,
      "activations/layer6_attention_weight_min": -47.978267669677734,
      "activations/layer7_attention_weight_max": 95.58002471923828,
      "activations/layer7_attention_weight_min": -101.75465393066406,
      "activations/layer8_attention_weight_max": 46.96907424926758,
      "activations/layer8_attention_weight_min": -47.13927459716797,
      "activations/layer9_attention_weight_max": 34.887264251708984,
      "activations/layer9_attention_weight_min": -36.350765228271484,
      "epoch": 21.47,
      "learning_rate": 1.1681439393939392e-05,
      "loss": 2.7,
      "step": 369450
    },
    {
      "activations/layer0_attention_weight_max": 16.830211639404297,
      "activations/layer0_attention_weight_min": -13.388711929321289,
      "activations/layer10_attention_weight_max": 37.59625244140625,
      "activations/layer10_attention_weight_min": -35.67417526245117,
      "activations/layer11_attention_weight_max": 34.76871109008789,
      "activations/layer11_attention_weight_min": -36.566749572753906,
      "activations/layer12_attention_weight_max": 25.458715438842773,
      "activations/layer12_attention_weight_min": -28.883201599121094,
      "activations/layer13_attention_weight_max": 39.24201202392578,
      "activations/layer13_attention_weight_min": -37.8602409362793,
      "activations/layer14_attention_weight_max": 40.00454330444336,
      "activations/layer14_attention_weight_min": -39.64590835571289,
      "activations/layer15_attention_weight_max": 37.6768913269043,
      "activations/layer15_attention_weight_min": -37.1876335144043,
      "activations/layer16_attention_weight_max": 30.2255859375,
      "activations/layer16_attention_weight_min": -28.521257400512695,
      "activations/layer17_attention_weight_max": 31.66110610961914,
      "activations/layer17_attention_weight_min": -28.609880447387695,
      "activations/layer18_attention_weight_max": 32.758426666259766,
      "activations/layer18_attention_weight_min": -28.918550491333008,
      "activations/layer19_attention_weight_max": 37.01711654663086,
      "activations/layer19_attention_weight_min": -33.538482666015625,
      "activations/layer1_attention_weight_max": 15.167914390563965,
      "activations/layer1_attention_weight_min": -14.597797393798828,
      "activations/layer20_attention_weight_max": 31.04167366027832,
      "activations/layer20_attention_weight_min": -26.315656661987305,
      "activations/layer21_attention_weight_max": 33.09624099731445,
      "activations/layer21_attention_weight_min": -27.883024215698242,
      "activations/layer22_attention_weight_max": 46.94239044189453,
      "activations/layer22_attention_weight_min": -38.5280647277832,
      "activations/layer23_attention_weight_max": 35.19681167602539,
      "activations/layer23_attention_weight_min": -29.10580062866211,
      "activations/layer2_attention_weight_max": 33.89508819580078,
      "activations/layer2_attention_weight_min": -33.16218948364258,
      "activations/layer3_attention_weight_max": 103.69847869873047,
      "activations/layer3_attention_weight_min": -107.9437484741211,
      "activations/layer4_attention_weight_max": 108.74751281738281,
      "activations/layer4_attention_weight_min": -114.7732162475586,
      "activations/layer5_attention_weight_max": 52.00201416015625,
      "activations/layer5_attention_weight_min": -64.81564331054688,
      "activations/layer6_attention_weight_max": 46.207725524902344,
      "activations/layer6_attention_weight_min": -49.73683166503906,
      "activations/layer7_attention_weight_max": 99.52074432373047,
      "activations/layer7_attention_weight_min": -103.78678894042969,
      "activations/layer8_attention_weight_max": 44.254974365234375,
      "activations/layer8_attention_weight_min": -46.75885009765625,
      "activations/layer9_attention_weight_max": 35.421443939208984,
      "activations/layer9_attention_weight_min": -37.460723876953125,
      "epoch": 21.47,
      "learning_rate": 1.1662499999999999e-05,
      "loss": 2.7055,
      "step": 369500
    },
    {
      "activations/layer0_attention_weight_max": 15.959720611572266,
      "activations/layer0_attention_weight_min": -13.027029991149902,
      "activations/layer10_attention_weight_max": 37.40907669067383,
      "activations/layer10_attention_weight_min": -34.984535217285156,
      "activations/layer11_attention_weight_max": 36.117164611816406,
      "activations/layer11_attention_weight_min": -34.884403228759766,
      "activations/layer12_attention_weight_max": 27.969377517700195,
      "activations/layer12_attention_weight_min": -26.781450271606445,
      "activations/layer13_attention_weight_max": 39.16044616699219,
      "activations/layer13_attention_weight_min": -36.76246643066406,
      "activations/layer14_attention_weight_max": 42.110450744628906,
      "activations/layer14_attention_weight_min": -37.885398864746094,
      "activations/layer15_attention_weight_max": 39.77294158935547,
      "activations/layer15_attention_weight_min": -36.054569244384766,
      "activations/layer16_attention_weight_max": 28.610750198364258,
      "activations/layer16_attention_weight_min": -28.54351806640625,
      "activations/layer17_attention_weight_max": 29.667030334472656,
      "activations/layer17_attention_weight_min": -27.44186782836914,
      "activations/layer18_attention_weight_max": 29.53158187866211,
      "activations/layer18_attention_weight_min": -24.47368049621582,
      "activations/layer19_attention_weight_max": 33.167537689208984,
      "activations/layer19_attention_weight_min": -30.694171905517578,
      "activations/layer1_attention_weight_max": 15.724325180053711,
      "activations/layer1_attention_weight_min": -15.230369567871094,
      "activations/layer20_attention_weight_max": 28.578514099121094,
      "activations/layer20_attention_weight_min": -24.318010330200195,
      "activations/layer21_attention_weight_max": 27.9719295501709,
      "activations/layer21_attention_weight_min": -24.915414810180664,
      "activations/layer22_attention_weight_max": 42.65275955200195,
      "activations/layer22_attention_weight_min": -31.61693572998047,
      "activations/layer23_attention_weight_max": 31.199378967285156,
      "activations/layer23_attention_weight_min": -26.34526824951172,
      "activations/layer2_attention_weight_max": 33.961856842041016,
      "activations/layer2_attention_weight_min": -32.35029220581055,
      "activations/layer3_attention_weight_max": 104.50556945800781,
      "activations/layer3_attention_weight_min": -107.18355560302734,
      "activations/layer4_attention_weight_max": 110.19081115722656,
      "activations/layer4_attention_weight_min": -112.07344055175781,
      "activations/layer5_attention_weight_max": 49.66813659667969,
      "activations/layer5_attention_weight_min": -61.91565704345703,
      "activations/layer6_attention_weight_max": 45.49469757080078,
      "activations/layer6_attention_weight_min": -49.012184143066406,
      "activations/layer7_attention_weight_max": 97.1823501586914,
      "activations/layer7_attention_weight_min": -101.09122467041016,
      "activations/layer8_attention_weight_max": 43.633445739746094,
      "activations/layer8_attention_weight_min": -46.97456741333008,
      "activations/layer9_attention_weight_max": 34.11494445800781,
      "activations/layer9_attention_weight_min": -35.748477935791016,
      "epoch": 21.47,
      "learning_rate": 1.1643560606060605e-05,
      "loss": 2.7235,
      "step": 369550
    },
    {
      "activations/layer0_attention_weight_max": 17.27136993408203,
      "activations/layer0_attention_weight_min": -14.347513198852539,
      "activations/layer10_attention_weight_max": 34.253257751464844,
      "activations/layer10_attention_weight_min": -36.373966217041016,
      "activations/layer11_attention_weight_max": 33.986732482910156,
      "activations/layer11_attention_weight_min": -34.923526763916016,
      "activations/layer12_attention_weight_max": 29.4334659576416,
      "activations/layer12_attention_weight_min": -27.35476303100586,
      "activations/layer13_attention_weight_max": 43.129310607910156,
      "activations/layer13_attention_weight_min": -36.07273483276367,
      "activations/layer14_attention_weight_max": 43.98230743408203,
      "activations/layer14_attention_weight_min": -38.319156646728516,
      "activations/layer15_attention_weight_max": 40.810646057128906,
      "activations/layer15_attention_weight_min": -35.321014404296875,
      "activations/layer16_attention_weight_max": 30.42053985595703,
      "activations/layer16_attention_weight_min": -30.215213775634766,
      "activations/layer17_attention_weight_max": 29.523752212524414,
      "activations/layer17_attention_weight_min": -28.24329948425293,
      "activations/layer18_attention_weight_max": 34.79528045654297,
      "activations/layer18_attention_weight_min": -26.3978214263916,
      "activations/layer19_attention_weight_max": 31.1872501373291,
      "activations/layer19_attention_weight_min": -32.49094772338867,
      "activations/layer1_attention_weight_max": 15.756182670593262,
      "activations/layer1_attention_weight_min": -14.196085929870605,
      "activations/layer20_attention_weight_max": 27.733524322509766,
      "activations/layer20_attention_weight_min": -25.228797912597656,
      "activations/layer21_attention_weight_max": 26.15140151977539,
      "activations/layer21_attention_weight_min": -24.71941566467285,
      "activations/layer22_attention_weight_max": 46.22727584838867,
      "activations/layer22_attention_weight_min": -34.47958755493164,
      "activations/layer23_attention_weight_max": 32.67346954345703,
      "activations/layer23_attention_weight_min": -28.167648315429688,
      "activations/layer2_attention_weight_max": 32.535186767578125,
      "activations/layer2_attention_weight_min": -32.681190490722656,
      "activations/layer3_attention_weight_max": 106.0413589477539,
      "activations/layer3_attention_weight_min": -104.44995880126953,
      "activations/layer4_attention_weight_max": 113.03987121582031,
      "activations/layer4_attention_weight_min": -113.02892303466797,
      "activations/layer5_attention_weight_max": 55.604087829589844,
      "activations/layer5_attention_weight_min": -61.865474700927734,
      "activations/layer6_attention_weight_max": 45.385677337646484,
      "activations/layer6_attention_weight_min": -48.659481048583984,
      "activations/layer7_attention_weight_max": 99.97379302978516,
      "activations/layer7_attention_weight_min": -103.27212524414062,
      "activations/layer8_attention_weight_max": 44.07453536987305,
      "activations/layer8_attention_weight_min": -47.27676773071289,
      "activations/layer9_attention_weight_max": 35.25046157836914,
      "activations/layer9_attention_weight_min": -35.68351364135742,
      "epoch": 21.48,
      "learning_rate": 1.162462121212121e-05,
      "loss": 2.7317,
      "step": 369600
    },
    {
      "activations/layer0_attention_weight_max": 17.307235717773438,
      "activations/layer0_attention_weight_min": -14.333084106445312,
      "activations/layer10_attention_weight_max": 50.67351150512695,
      "activations/layer10_attention_weight_min": -44.26129150390625,
      "activations/layer11_attention_weight_max": 47.0415153503418,
      "activations/layer11_attention_weight_min": -44.65252685546875,
      "activations/layer12_attention_weight_max": 30.529285430908203,
      "activations/layer12_attention_weight_min": -29.265316009521484,
      "activations/layer13_attention_weight_max": 38.62504577636719,
      "activations/layer13_attention_weight_min": -38.62596893310547,
      "activations/layer14_attention_weight_max": 45.268516540527344,
      "activations/layer14_attention_weight_min": -42.99052810668945,
      "activations/layer15_attention_weight_max": 42.49946975708008,
      "activations/layer15_attention_weight_min": -38.383270263671875,
      "activations/layer16_attention_weight_max": 29.895658493041992,
      "activations/layer16_attention_weight_min": -30.589344024658203,
      "activations/layer17_attention_weight_max": 29.66514778137207,
      "activations/layer17_attention_weight_min": -26.668304443359375,
      "activations/layer18_attention_weight_max": 32.75039291381836,
      "activations/layer18_attention_weight_min": -27.097135543823242,
      "activations/layer19_attention_weight_max": 33.044742584228516,
      "activations/layer19_attention_weight_min": -32.07447814941406,
      "activations/layer1_attention_weight_max": 15.155850410461426,
      "activations/layer1_attention_weight_min": -15.094500541687012,
      "activations/layer20_attention_weight_max": 27.625518798828125,
      "activations/layer20_attention_weight_min": -25.210609436035156,
      "activations/layer21_attention_weight_max": 30.960613250732422,
      "activations/layer21_attention_weight_min": -25.974533081054688,
      "activations/layer22_attention_weight_max": 44.78443908691406,
      "activations/layer22_attention_weight_min": -33.214717864990234,
      "activations/layer23_attention_weight_max": 32.84354782104492,
      "activations/layer23_attention_weight_min": -26.0992488861084,
      "activations/layer2_attention_weight_max": 35.571022033691406,
      "activations/layer2_attention_weight_min": -34.362281799316406,
      "activations/layer3_attention_weight_max": 105.6192398071289,
      "activations/layer3_attention_weight_min": -108.83070373535156,
      "activations/layer4_attention_weight_max": 112.6494140625,
      "activations/layer4_attention_weight_min": -114.15727996826172,
      "activations/layer5_attention_weight_max": 55.63862228393555,
      "activations/layer5_attention_weight_min": -60.67561340332031,
      "activations/layer6_attention_weight_max": 48.74110412597656,
      "activations/layer6_attention_weight_min": -48.53487014770508,
      "activations/layer7_attention_weight_max": 115.93661499023438,
      "activations/layer7_attention_weight_min": -106.5169677734375,
      "activations/layer8_attention_weight_max": 55.9179573059082,
      "activations/layer8_attention_weight_min": -51.825565338134766,
      "activations/layer9_attention_weight_max": 42.44390869140625,
      "activations/layer9_attention_weight_min": -44.26676559448242,
      "epoch": 21.48,
      "learning_rate": 1.1605681818181817e-05,
      "loss": 2.7106,
      "step": 369650
    },
    {
      "activations/layer0_attention_weight_max": 16.79474449157715,
      "activations/layer0_attention_weight_min": -14.311591148376465,
      "activations/layer10_attention_weight_max": 37.55033874511719,
      "activations/layer10_attention_weight_min": -35.80940628051758,
      "activations/layer11_attention_weight_max": 36.27836227416992,
      "activations/layer11_attention_weight_min": -36.10675048828125,
      "activations/layer12_attention_weight_max": 28.922685623168945,
      "activations/layer12_attention_weight_min": -27.9819278717041,
      "activations/layer13_attention_weight_max": 39.984222412109375,
      "activations/layer13_attention_weight_min": -37.859676361083984,
      "activations/layer14_attention_weight_max": 42.98772048950195,
      "activations/layer14_attention_weight_min": -39.10264587402344,
      "activations/layer15_attention_weight_max": 41.17911911010742,
      "activations/layer15_attention_weight_min": -36.2516975402832,
      "activations/layer16_attention_weight_max": 31.06462287902832,
      "activations/layer16_attention_weight_min": -29.947572708129883,
      "activations/layer17_attention_weight_max": 33.10448455810547,
      "activations/layer17_attention_weight_min": -28.608558654785156,
      "activations/layer18_attention_weight_max": 37.768917083740234,
      "activations/layer18_attention_weight_min": -27.439552307128906,
      "activations/layer19_attention_weight_max": 39.81010055541992,
      "activations/layer19_attention_weight_min": -32.42885971069336,
      "activations/layer1_attention_weight_max": 15.53209400177002,
      "activations/layer1_attention_weight_min": -14.130268096923828,
      "activations/layer20_attention_weight_max": 34.73606872558594,
      "activations/layer20_attention_weight_min": -26.884044647216797,
      "activations/layer21_attention_weight_max": 37.28753662109375,
      "activations/layer21_attention_weight_min": -25.008346557617188,
      "activations/layer22_attention_weight_max": 49.826072692871094,
      "activations/layer22_attention_weight_min": -32.44749450683594,
      "activations/layer23_attention_weight_max": 40.1220817565918,
      "activations/layer23_attention_weight_min": -26.663143157958984,
      "activations/layer2_attention_weight_max": 33.57333755493164,
      "activations/layer2_attention_weight_min": -33.48472213745117,
      "activations/layer3_attention_weight_max": 101.51923370361328,
      "activations/layer3_attention_weight_min": -106.00726318359375,
      "activations/layer4_attention_weight_max": 108.6679916381836,
      "activations/layer4_attention_weight_min": -111.4239501953125,
      "activations/layer5_attention_weight_max": 48.29692840576172,
      "activations/layer5_attention_weight_min": -65.25186157226562,
      "activations/layer6_attention_weight_max": 46.89362716674805,
      "activations/layer6_attention_weight_min": -50.218502044677734,
      "activations/layer7_attention_weight_max": 99.62425231933594,
      "activations/layer7_attention_weight_min": -106.24957275390625,
      "activations/layer8_attention_weight_max": 43.56606674194336,
      "activations/layer8_attention_weight_min": -48.77192306518555,
      "activations/layer9_attention_weight_max": 33.68428039550781,
      "activations/layer9_attention_weight_min": -35.7903938293457,
      "epoch": 21.48,
      "learning_rate": 1.1586742424242423e-05,
      "loss": 2.7128,
      "step": 369700
    },
    {
      "activations/layer0_attention_weight_max": 15.662410736083984,
      "activations/layer0_attention_weight_min": -14.642314910888672,
      "activations/layer10_attention_weight_max": 34.89807891845703,
      "activations/layer10_attention_weight_min": -35.4516487121582,
      "activations/layer11_attention_weight_max": 36.31342697143555,
      "activations/layer11_attention_weight_min": -35.04095458984375,
      "activations/layer12_attention_weight_max": 26.634334564208984,
      "activations/layer12_attention_weight_min": -27.66801643371582,
      "activations/layer13_attention_weight_max": 40.143272399902344,
      "activations/layer13_attention_weight_min": -37.41228485107422,
      "activations/layer14_attention_weight_max": 41.578311920166016,
      "activations/layer14_attention_weight_min": -39.28287124633789,
      "activations/layer15_attention_weight_max": 43.5455207824707,
      "activations/layer15_attention_weight_min": -38.00645446777344,
      "activations/layer16_attention_weight_max": 30.99033546447754,
      "activations/layer16_attention_weight_min": -28.390623092651367,
      "activations/layer17_attention_weight_max": 30.162261962890625,
      "activations/layer17_attention_weight_min": -27.970462799072266,
      "activations/layer18_attention_weight_max": 32.36323165893555,
      "activations/layer18_attention_weight_min": -26.142244338989258,
      "activations/layer19_attention_weight_max": 35.40833282470703,
      "activations/layer19_attention_weight_min": -32.16631317138672,
      "activations/layer1_attention_weight_max": 15.750423431396484,
      "activations/layer1_attention_weight_min": -14.568485260009766,
      "activations/layer20_attention_weight_max": 31.423721313476562,
      "activations/layer20_attention_weight_min": -25.846721649169922,
      "activations/layer21_attention_weight_max": 29.424400329589844,
      "activations/layer21_attention_weight_min": -26.965984344482422,
      "activations/layer22_attention_weight_max": 40.1727294921875,
      "activations/layer22_attention_weight_min": -32.42323684692383,
      "activations/layer23_attention_weight_max": 34.97226333618164,
      "activations/layer23_attention_weight_min": -28.022502899169922,
      "activations/layer2_attention_weight_max": 33.222412109375,
      "activations/layer2_attention_weight_min": -33.033329010009766,
      "activations/layer3_attention_weight_max": 102.29167175292969,
      "activations/layer3_attention_weight_min": -104.41739654541016,
      "activations/layer4_attention_weight_max": 111.06617736816406,
      "activations/layer4_attention_weight_min": -108.89796447753906,
      "activations/layer5_attention_weight_max": 51.24089813232422,
      "activations/layer5_attention_weight_min": -60.363739013671875,
      "activations/layer6_attention_weight_max": 45.6784782409668,
      "activations/layer6_attention_weight_min": -47.31391143798828,
      "activations/layer7_attention_weight_max": 97.00238037109375,
      "activations/layer7_attention_weight_min": -100.4425048828125,
      "activations/layer8_attention_weight_max": 43.284629821777344,
      "activations/layer8_attention_weight_min": -44.236934661865234,
      "activations/layer9_attention_weight_max": 33.803707122802734,
      "activations/layer9_attention_weight_min": -34.34223556518555,
      "epoch": 21.48,
      "learning_rate": 1.1567803030303028e-05,
      "loss": 2.7147,
      "step": 369750
    },
    {
      "activations/layer0_attention_weight_max": 15.915568351745605,
      "activations/layer0_attention_weight_min": -13.565059661865234,
      "activations/layer10_attention_weight_max": 34.73014450073242,
      "activations/layer10_attention_weight_min": -37.83964920043945,
      "activations/layer11_attention_weight_max": 34.92400360107422,
      "activations/layer11_attention_weight_min": -37.15058135986328,
      "activations/layer12_attention_weight_max": 27.63779067993164,
      "activations/layer12_attention_weight_min": -29.368616104125977,
      "activations/layer13_attention_weight_max": 42.3530387878418,
      "activations/layer13_attention_weight_min": -40.795379638671875,
      "activations/layer14_attention_weight_max": 44.14310073852539,
      "activations/layer14_attention_weight_min": -44.36114501953125,
      "activations/layer15_attention_weight_max": 39.373653411865234,
      "activations/layer15_attention_weight_min": -37.54031753540039,
      "activations/layer16_attention_weight_max": 28.378801345825195,
      "activations/layer16_attention_weight_min": -28.99287223815918,
      "activations/layer17_attention_weight_max": 30.350990295410156,
      "activations/layer17_attention_weight_min": -26.849267959594727,
      "activations/layer18_attention_weight_max": 33.172027587890625,
      "activations/layer18_attention_weight_min": -26.26726531982422,
      "activations/layer19_attention_weight_max": 36.89158248901367,
      "activations/layer19_attention_weight_min": -31.6693058013916,
      "activations/layer1_attention_weight_max": 15.834053039550781,
      "activations/layer1_attention_weight_min": -14.02961254119873,
      "activations/layer20_attention_weight_max": 29.342498779296875,
      "activations/layer20_attention_weight_min": -25.5975341796875,
      "activations/layer21_attention_weight_max": 32.95953369140625,
      "activations/layer21_attention_weight_min": -26.366634368896484,
      "activations/layer22_attention_weight_max": 42.964439392089844,
      "activations/layer22_attention_weight_min": -31.70248794555664,
      "activations/layer23_attention_weight_max": 35.92817687988281,
      "activations/layer23_attention_weight_min": -27.766803741455078,
      "activations/layer2_attention_weight_max": 34.59303665161133,
      "activations/layer2_attention_weight_min": -34.855106353759766,
      "activations/layer3_attention_weight_max": 106.06944274902344,
      "activations/layer3_attention_weight_min": -109.73223876953125,
      "activations/layer4_attention_weight_max": 112.99759674072266,
      "activations/layer4_attention_weight_min": -114.3912582397461,
      "activations/layer5_attention_weight_max": 52.58700180053711,
      "activations/layer5_attention_weight_min": -65.26164245605469,
      "activations/layer6_attention_weight_max": 46.793800354003906,
      "activations/layer6_attention_weight_min": -50.63349533081055,
      "activations/layer7_attention_weight_max": 102.38337707519531,
      "activations/layer7_attention_weight_min": -108.82587432861328,
      "activations/layer8_attention_weight_max": 43.8997917175293,
      "activations/layer8_attention_weight_min": -48.770957946777344,
      "activations/layer9_attention_weight_max": 31.309843063354492,
      "activations/layer9_attention_weight_min": -38.460994720458984,
      "epoch": 21.49,
      "learning_rate": 1.1548863636363635e-05,
      "loss": 2.7164,
      "step": 369800
    },
    {
      "activations/layer0_attention_weight_max": 16.208202362060547,
      "activations/layer0_attention_weight_min": -13.223790168762207,
      "activations/layer10_attention_weight_max": 34.468353271484375,
      "activations/layer10_attention_weight_min": -36.564697265625,
      "activations/layer11_attention_weight_max": 35.16429138183594,
      "activations/layer11_attention_weight_min": -35.64629364013672,
      "activations/layer12_attention_weight_max": 27.66098403930664,
      "activations/layer12_attention_weight_min": -27.88846778869629,
      "activations/layer13_attention_weight_max": 40.300262451171875,
      "activations/layer13_attention_weight_min": -35.9132194519043,
      "activations/layer14_attention_weight_max": 43.91282653808594,
      "activations/layer14_attention_weight_min": -38.05574035644531,
      "activations/layer15_attention_weight_max": 41.15004348754883,
      "activations/layer15_attention_weight_min": -39.24656295776367,
      "activations/layer16_attention_weight_max": 31.04698371887207,
      "activations/layer16_attention_weight_min": -30.616060256958008,
      "activations/layer17_attention_weight_max": 32.00374984741211,
      "activations/layer17_attention_weight_min": -27.957035064697266,
      "activations/layer18_attention_weight_max": 32.208038330078125,
      "activations/layer18_attention_weight_min": -26.0394287109375,
      "activations/layer19_attention_weight_max": 37.160648345947266,
      "activations/layer19_attention_weight_min": -32.09396743774414,
      "activations/layer1_attention_weight_max": 16.090585708618164,
      "activations/layer1_attention_weight_min": -15.116556167602539,
      "activations/layer20_attention_weight_max": 31.229724884033203,
      "activations/layer20_attention_weight_min": -25.095436096191406,
      "activations/layer21_attention_weight_max": 30.571012496948242,
      "activations/layer21_attention_weight_min": -26.893754959106445,
      "activations/layer22_attention_weight_max": 40.81705856323242,
      "activations/layer22_attention_weight_min": -31.54768180847168,
      "activations/layer23_attention_weight_max": 31.292598724365234,
      "activations/layer23_attention_weight_min": -24.55567741394043,
      "activations/layer2_attention_weight_max": 32.80648422241211,
      "activations/layer2_attention_weight_min": -32.52555465698242,
      "activations/layer3_attention_weight_max": 102.55958557128906,
      "activations/layer3_attention_weight_min": -103.7984619140625,
      "activations/layer4_attention_weight_max": 109.64501953125,
      "activations/layer4_attention_weight_min": -112.2496566772461,
      "activations/layer5_attention_weight_max": 49.75120544433594,
      "activations/layer5_attention_weight_min": -63.54620361328125,
      "activations/layer6_attention_weight_max": 45.51667404174805,
      "activations/layer6_attention_weight_min": -49.950355529785156,
      "activations/layer7_attention_weight_max": 93.0728988647461,
      "activations/layer7_attention_weight_min": -100.93151092529297,
      "activations/layer8_attention_weight_max": 41.32744598388672,
      "activations/layer8_attention_weight_min": -46.47734069824219,
      "activations/layer9_attention_weight_max": 32.46125030517578,
      "activations/layer9_attention_weight_min": -35.96599197387695,
      "epoch": 21.49,
      "learning_rate": 1.152992424242424e-05,
      "loss": 2.7148,
      "step": 369850
    },
    {
      "activations/layer0_attention_weight_max": 16.181320190429688,
      "activations/layer0_attention_weight_min": -14.516395568847656,
      "activations/layer10_attention_weight_max": 35.82802963256836,
      "activations/layer10_attention_weight_min": -35.76065444946289,
      "activations/layer11_attention_weight_max": 34.34597396850586,
      "activations/layer11_attention_weight_min": -37.459659576416016,
      "activations/layer12_attention_weight_max": 27.356103897094727,
      "activations/layer12_attention_weight_min": -28.26923370361328,
      "activations/layer13_attention_weight_max": 42.188995361328125,
      "activations/layer13_attention_weight_min": -34.620277404785156,
      "activations/layer14_attention_weight_max": 43.35935974121094,
      "activations/layer14_attention_weight_min": -37.89488983154297,
      "activations/layer15_attention_weight_max": 39.430782318115234,
      "activations/layer15_attention_weight_min": -37.33052444458008,
      "activations/layer16_attention_weight_max": 28.5246524810791,
      "activations/layer16_attention_weight_min": -27.943450927734375,
      "activations/layer17_attention_weight_max": 29.99298858642578,
      "activations/layer17_attention_weight_min": -25.72191047668457,
      "activations/layer18_attention_weight_max": 32.761592864990234,
      "activations/layer18_attention_weight_min": -26.300580978393555,
      "activations/layer19_attention_weight_max": 34.9274787902832,
      "activations/layer19_attention_weight_min": -32.12100601196289,
      "activations/layer1_attention_weight_max": 15.100861549377441,
      "activations/layer1_attention_weight_min": -14.891142845153809,
      "activations/layer20_attention_weight_max": 29.056121826171875,
      "activations/layer20_attention_weight_min": -26.436660766601562,
      "activations/layer21_attention_weight_max": 27.670028686523438,
      "activations/layer21_attention_weight_min": -26.078384399414062,
      "activations/layer22_attention_weight_max": 43.373390197753906,
      "activations/layer22_attention_weight_min": -31.584701538085938,
      "activations/layer23_attention_weight_max": 34.35765838623047,
      "activations/layer23_attention_weight_min": -26.05588150024414,
      "activations/layer2_attention_weight_max": 33.181129455566406,
      "activations/layer2_attention_weight_min": -31.829517364501953,
      "activations/layer3_attention_weight_max": 105.51895141601562,
      "activations/layer3_attention_weight_min": -108.3444595336914,
      "activations/layer4_attention_weight_max": 107.03279876708984,
      "activations/layer4_attention_weight_min": -111.03688049316406,
      "activations/layer5_attention_weight_max": 48.773643493652344,
      "activations/layer5_attention_weight_min": -62.8634147644043,
      "activations/layer6_attention_weight_max": 44.377681732177734,
      "activations/layer6_attention_weight_min": -48.71459197998047,
      "activations/layer7_attention_weight_max": 95.67899322509766,
      "activations/layer7_attention_weight_min": -98.74418640136719,
      "activations/layer8_attention_weight_max": 43.31465530395508,
      "activations/layer8_attention_weight_min": -47.59163284301758,
      "activations/layer9_attention_weight_max": 31.746896743774414,
      "activations/layer9_attention_weight_min": -33.64278030395508,
      "epoch": 21.49,
      "learning_rate": 1.1510984848484846e-05,
      "loss": 2.7133,
      "step": 369900
    },
    {
      "activations/layer0_attention_weight_max": 16.428186416625977,
      "activations/layer0_attention_weight_min": -14.757302284240723,
      "activations/layer10_attention_weight_max": 36.06977081298828,
      "activations/layer10_attention_weight_min": -37.196128845214844,
      "activations/layer11_attention_weight_max": 37.090782165527344,
      "activations/layer11_attention_weight_min": -34.840579986572266,
      "activations/layer12_attention_weight_max": 28.26894760131836,
      "activations/layer12_attention_weight_min": -26.088966369628906,
      "activations/layer13_attention_weight_max": 41.689842224121094,
      "activations/layer13_attention_weight_min": -35.26604461669922,
      "activations/layer14_attention_weight_max": 44.09111022949219,
      "activations/layer14_attention_weight_min": -36.650726318359375,
      "activations/layer15_attention_weight_max": 42.260765075683594,
      "activations/layer15_attention_weight_min": -35.128990173339844,
      "activations/layer16_attention_weight_max": 31.081314086914062,
      "activations/layer16_attention_weight_min": -28.296478271484375,
      "activations/layer17_attention_weight_max": 33.28641128540039,
      "activations/layer17_attention_weight_min": -27.452922821044922,
      "activations/layer18_attention_weight_max": 32.730003356933594,
      "activations/layer18_attention_weight_min": -26.3521785736084,
      "activations/layer19_attention_weight_max": 32.46354293823242,
      "activations/layer19_attention_weight_min": -31.828311920166016,
      "activations/layer1_attention_weight_max": 15.642741203308105,
      "activations/layer1_attention_weight_min": -14.66508960723877,
      "activations/layer20_attention_weight_max": 29.633464813232422,
      "activations/layer20_attention_weight_min": -26.634618759155273,
      "activations/layer21_attention_weight_max": 30.71979331970215,
      "activations/layer21_attention_weight_min": -24.3952579498291,
      "activations/layer22_attention_weight_max": 43.61442565917969,
      "activations/layer22_attention_weight_min": -31.34689712524414,
      "activations/layer23_attention_weight_max": 30.274944305419922,
      "activations/layer23_attention_weight_min": -25.506324768066406,
      "activations/layer2_attention_weight_max": 35.24581527709961,
      "activations/layer2_attention_weight_min": -33.0572395324707,
      "activations/layer3_attention_weight_max": 106.13206481933594,
      "activations/layer3_attention_weight_min": -107.55490112304688,
      "activations/layer4_attention_weight_max": 109.40483856201172,
      "activations/layer4_attention_weight_min": -115.46981048583984,
      "activations/layer5_attention_weight_max": 53.964744567871094,
      "activations/layer5_attention_weight_min": -62.576446533203125,
      "activations/layer6_attention_weight_max": 46.3863410949707,
      "activations/layer6_attention_weight_min": -48.3173942565918,
      "activations/layer7_attention_weight_max": 98.36424255371094,
      "activations/layer7_attention_weight_min": -103.04935455322266,
      "activations/layer8_attention_weight_max": 47.17036819458008,
      "activations/layer8_attention_weight_min": -44.65715408325195,
      "activations/layer9_attention_weight_max": 33.337074279785156,
      "activations/layer9_attention_weight_min": -34.01041793823242,
      "epoch": 21.5,
      "learning_rate": 1.1492045454545455e-05,
      "loss": 2.6988,
      "step": 369950
    },
    {
      "activations/layer0_attention_weight_max": 17.226932525634766,
      "activations/layer0_attention_weight_min": -15.005054473876953,
      "activations/layer10_attention_weight_max": 35.36908721923828,
      "activations/layer10_attention_weight_min": -38.4094352722168,
      "activations/layer11_attention_weight_max": 34.17177963256836,
      "activations/layer11_attention_weight_min": -37.212398529052734,
      "activations/layer12_attention_weight_max": 27.915016174316406,
      "activations/layer12_attention_weight_min": -27.981767654418945,
      "activations/layer13_attention_weight_max": 42.308895111083984,
      "activations/layer13_attention_weight_min": -38.58443069458008,
      "activations/layer14_attention_weight_max": 45.54572677612305,
      "activations/layer14_attention_weight_min": -41.72352981567383,
      "activations/layer15_attention_weight_max": 42.49888229370117,
      "activations/layer15_attention_weight_min": -40.35664749145508,
      "activations/layer16_attention_weight_max": 33.57160568237305,
      "activations/layer16_attention_weight_min": -28.19362449645996,
      "activations/layer17_attention_weight_max": 31.819278717041016,
      "activations/layer17_attention_weight_min": -29.767805099487305,
      "activations/layer18_attention_weight_max": 34.45637512207031,
      "activations/layer18_attention_weight_min": -31.116018295288086,
      "activations/layer19_attention_weight_max": 35.24213409423828,
      "activations/layer19_attention_weight_min": -33.2999382019043,
      "activations/layer1_attention_weight_max": 15.483026504516602,
      "activations/layer1_attention_weight_min": -15.378678321838379,
      "activations/layer20_attention_weight_max": 32.77549743652344,
      "activations/layer20_attention_weight_min": -27.109012603759766,
      "activations/layer21_attention_weight_max": 29.5828857421875,
      "activations/layer21_attention_weight_min": -27.35105323791504,
      "activations/layer22_attention_weight_max": 44.046566009521484,
      "activations/layer22_attention_weight_min": -36.63027572631836,
      "activations/layer23_attention_weight_max": 35.380714416503906,
      "activations/layer23_attention_weight_min": -29.786399841308594,
      "activations/layer2_attention_weight_max": 33.977073669433594,
      "activations/layer2_attention_weight_min": -33.708595275878906,
      "activations/layer3_attention_weight_max": 103.09686279296875,
      "activations/layer3_attention_weight_min": -104.60470581054688,
      "activations/layer4_attention_weight_max": 108.0497817993164,
      "activations/layer4_attention_weight_min": -110.750244140625,
      "activations/layer5_attention_weight_max": 50.3801383972168,
      "activations/layer5_attention_weight_min": -64.6950454711914,
      "activations/layer6_attention_weight_max": 44.42985534667969,
      "activations/layer6_attention_weight_min": -49.764442443847656,
      "activations/layer7_attention_weight_max": 96.43543243408203,
      "activations/layer7_attention_weight_min": -108.80211639404297,
      "activations/layer8_attention_weight_max": 43.667720794677734,
      "activations/layer8_attention_weight_min": -47.80727005004883,
      "activations/layer9_attention_weight_max": 32.905067443847656,
      "activations/layer9_attention_weight_min": -37.03697204589844,
      "epoch": 21.5,
      "learning_rate": 1.1473106060606061e-05,
      "loss": 2.7117,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.4054,
      "eval_samples_per_second": 510.86,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.4054,
      "eval_openwebtext_samples_per_second": 510.86,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 2.0127,
      "eval_wikitext_samples_per_second": 226.563,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_lambada_loss": 2.294921875,
      "eval_lambada_ppl": 9.923660696008076,
      "eval_lambada_runtime": 9.5991,
      "eval_lambada_samples_per_second": 507.233,
      "step": 370000
    },
    {
      "activations/layer0_attention_weight_max": 16.046218872070312,
      "activations/layer0_attention_weight_min": -16.684314727783203,
      "activations/layer10_attention_weight_max": 34.152427673339844,
      "activations/layer10_attention_weight_min": -33.95402526855469,
      "activations/layer11_attention_weight_max": 32.087493896484375,
      "activations/layer11_attention_weight_min": -36.741798400878906,
      "activations/layer12_attention_weight_max": 25.637514114379883,
      "activations/layer12_attention_weight_min": -25.6971378326416,
      "activations/layer13_attention_weight_max": 37.85002136230469,
      "activations/layer13_attention_weight_min": -34.558292388916016,
      "activations/layer14_attention_weight_max": 40.334693908691406,
      "activations/layer14_attention_weight_min": -37.93952178955078,
      "activations/layer15_attention_weight_max": 41.25334930419922,
      "activations/layer15_attention_weight_min": -37.56508255004883,
      "activations/layer16_attention_weight_max": 31.445405960083008,
      "activations/layer16_attention_weight_min": -28.507156372070312,
      "activations/layer17_attention_weight_max": 29.825342178344727,
      "activations/layer17_attention_weight_min": -25.789648056030273,
      "activations/layer18_attention_weight_max": 33.409236907958984,
      "activations/layer18_attention_weight_min": -26.512693405151367,
      "activations/layer19_attention_weight_max": 35.5963134765625,
      "activations/layer19_attention_weight_min": -30.496997833251953,
      "activations/layer1_attention_weight_max": 15.23044490814209,
      "activations/layer1_attention_weight_min": -14.593544006347656,
      "activations/layer20_attention_weight_max": 30.607275009155273,
      "activations/layer20_attention_weight_min": -26.061962127685547,
      "activations/layer21_attention_weight_max": 31.761680603027344,
      "activations/layer21_attention_weight_min": -25.039249420166016,
      "activations/layer22_attention_weight_max": 38.631614685058594,
      "activations/layer22_attention_weight_min": -31.660526275634766,
      "activations/layer23_attention_weight_max": 32.35597229003906,
      "activations/layer23_attention_weight_min": -25.630399703979492,
      "activations/layer2_attention_weight_max": 32.95270538330078,
      "activations/layer2_attention_weight_min": -33.16986083984375,
      "activations/layer3_attention_weight_max": 102.58634185791016,
      "activations/layer3_attention_weight_min": -102.59447479248047,
      "activations/layer4_attention_weight_max": 104.8326187133789,
      "activations/layer4_attention_weight_min": -111.0977783203125,
      "activations/layer5_attention_weight_max": 49.76585006713867,
      "activations/layer5_attention_weight_min": -61.95350646972656,
      "activations/layer6_attention_weight_max": 43.06356430053711,
      "activations/layer6_attention_weight_min": -46.759071350097656,
      "activations/layer7_attention_weight_max": 93.27618408203125,
      "activations/layer7_attention_weight_min": -96.486083984375,
      "activations/layer8_attention_weight_max": 41.24167251586914,
      "activations/layer8_attention_weight_min": -42.92110061645508,
      "activations/layer9_attention_weight_max": 32.06333923339844,
      "activations/layer9_attention_weight_min": -34.6697883605957,
      "epoch": 21.5,
      "learning_rate": 1.1454166666666666e-05,
      "loss": 2.707,
      "step": 370050
    },
    {
      "activations/layer0_attention_weight_max": 16.501033782958984,
      "activations/layer0_attention_weight_min": -14.177452087402344,
      "activations/layer10_attention_weight_max": 41.77260971069336,
      "activations/layer10_attention_weight_min": -38.02239990234375,
      "activations/layer11_attention_weight_max": 38.26715087890625,
      "activations/layer11_attention_weight_min": -37.057525634765625,
      "activations/layer12_attention_weight_max": 30.252296447753906,
      "activations/layer12_attention_weight_min": -28.43926239013672,
      "activations/layer13_attention_weight_max": 44.74348449707031,
      "activations/layer13_attention_weight_min": -36.73234558105469,
      "activations/layer14_attention_weight_max": 43.3355827331543,
      "activations/layer14_attention_weight_min": -38.97526931762695,
      "activations/layer15_attention_weight_max": 39.02071762084961,
      "activations/layer15_attention_weight_min": -36.6077766418457,
      "activations/layer16_attention_weight_max": 28.590307235717773,
      "activations/layer16_attention_weight_min": -27.785120010375977,
      "activations/layer17_attention_weight_max": 31.36165428161621,
      "activations/layer17_attention_weight_min": -25.055950164794922,
      "activations/layer18_attention_weight_max": 33.356964111328125,
      "activations/layer18_attention_weight_min": -25.217554092407227,
      "activations/layer19_attention_weight_max": 31.94806480407715,
      "activations/layer19_attention_weight_min": -31.137678146362305,
      "activations/layer1_attention_weight_max": 16.39370346069336,
      "activations/layer1_attention_weight_min": -17.05667495727539,
      "activations/layer20_attention_weight_max": 28.39183235168457,
      "activations/layer20_attention_weight_min": -25.012901306152344,
      "activations/layer21_attention_weight_max": 27.14604949951172,
      "activations/layer21_attention_weight_min": -23.735021591186523,
      "activations/layer22_attention_weight_max": 41.425926208496094,
      "activations/layer22_attention_weight_min": -31.38246726989746,
      "activations/layer23_attention_weight_max": 31.55257797241211,
      "activations/layer23_attention_weight_min": -26.857410430908203,
      "activations/layer2_attention_weight_max": 35.13466262817383,
      "activations/layer2_attention_weight_min": -34.24494934082031,
      "activations/layer3_attention_weight_max": 108.26657104492188,
      "activations/layer3_attention_weight_min": -110.47289276123047,
      "activations/layer4_attention_weight_max": 111.74824523925781,
      "activations/layer4_attention_weight_min": -115.04890441894531,
      "activations/layer5_attention_weight_max": 51.7622184753418,
      "activations/layer5_attention_weight_min": -64.70361328125,
      "activations/layer6_attention_weight_max": 45.90599822998047,
      "activations/layer6_attention_weight_min": -49.608009338378906,
      "activations/layer7_attention_weight_max": 106.00196075439453,
      "activations/layer7_attention_weight_min": -105.08256530761719,
      "activations/layer8_attention_weight_max": 49.913604736328125,
      "activations/layer8_attention_weight_min": -47.8377799987793,
      "activations/layer9_attention_weight_max": 35.077239990234375,
      "activations/layer9_attention_weight_min": -37.794490814208984,
      "epoch": 21.5,
      "learning_rate": 1.1435227272727273e-05,
      "loss": 2.7113,
      "step": 370100
    },
    {
      "activations/layer0_attention_weight_max": 16.63155174255371,
      "activations/layer0_attention_weight_min": -15.073765754699707,
      "activations/layer10_attention_weight_max": 36.22564697265625,
      "activations/layer10_attention_weight_min": -34.0565071105957,
      "activations/layer11_attention_weight_max": 38.951358795166016,
      "activations/layer11_attention_weight_min": -35.186737060546875,
      "activations/layer12_attention_weight_max": 26.27373695373535,
      "activations/layer12_attention_weight_min": -26.078691482543945,
      "activations/layer13_attention_weight_max": 40.24491500854492,
      "activations/layer13_attention_weight_min": -34.57046127319336,
      "activations/layer14_attention_weight_max": 41.88607406616211,
      "activations/layer14_attention_weight_min": -36.78072738647461,
      "activations/layer15_attention_weight_max": 38.40950012207031,
      "activations/layer15_attention_weight_min": -36.2962760925293,
      "activations/layer16_attention_weight_max": 29.076457977294922,
      "activations/layer16_attention_weight_min": -26.540740966796875,
      "activations/layer17_attention_weight_max": 30.466623306274414,
      "activations/layer17_attention_weight_min": -25.45147132873535,
      "activations/layer18_attention_weight_max": 34.38056945800781,
      "activations/layer18_attention_weight_min": -26.963415145874023,
      "activations/layer19_attention_weight_max": 34.11716842651367,
      "activations/layer19_attention_weight_min": -30.86771583557129,
      "activations/layer1_attention_weight_max": 16.33001136779785,
      "activations/layer1_attention_weight_min": -15.2606201171875,
      "activations/layer20_attention_weight_max": 29.003755569458008,
      "activations/layer20_attention_weight_min": -24.95219612121582,
      "activations/layer21_attention_weight_max": 29.388219833374023,
      "activations/layer21_attention_weight_min": -23.758926391601562,
      "activations/layer22_attention_weight_max": 41.0027961730957,
      "activations/layer22_attention_weight_min": -31.23201560974121,
      "activations/layer23_attention_weight_max": 31.311294555664062,
      "activations/layer23_attention_weight_min": -24.22547721862793,
      "activations/layer2_attention_weight_max": 33.695682525634766,
      "activations/layer2_attention_weight_min": -32.95820236206055,
      "activations/layer3_attention_weight_max": 105.43180847167969,
      "activations/layer3_attention_weight_min": -105.9859619140625,
      "activations/layer4_attention_weight_max": 108.30130767822266,
      "activations/layer4_attention_weight_min": -112.50016021728516,
      "activations/layer5_attention_weight_max": 53.455284118652344,
      "activations/layer5_attention_weight_min": -62.87293243408203,
      "activations/layer6_attention_weight_max": 46.60758590698242,
      "activations/layer6_attention_weight_min": -49.83017349243164,
      "activations/layer7_attention_weight_max": 96.09649658203125,
      "activations/layer7_attention_weight_min": -97.75468444824219,
      "activations/layer8_attention_weight_max": 44.265098571777344,
      "activations/layer8_attention_weight_min": -45.79310607910156,
      "activations/layer9_attention_weight_max": 31.961196899414062,
      "activations/layer9_attention_weight_min": -33.5190315246582,
      "epoch": 21.51,
      "learning_rate": 1.1416287878787878e-05,
      "loss": 2.6985,
      "step": 370150
    },
    {
      "activations/layer0_attention_weight_max": 15.740110397338867,
      "activations/layer0_attention_weight_min": -14.148602485656738,
      "activations/layer10_attention_weight_max": 37.90446853637695,
      "activations/layer10_attention_weight_min": -38.73368453979492,
      "activations/layer11_attention_weight_max": 38.825767517089844,
      "activations/layer11_attention_weight_min": -39.54823303222656,
      "activations/layer12_attention_weight_max": 30.21234703063965,
      "activations/layer12_attention_weight_min": -30.202836990356445,
      "activations/layer13_attention_weight_max": 45.95927429199219,
      "activations/layer13_attention_weight_min": -39.63262939453125,
      "activations/layer14_attention_weight_max": 48.6357421875,
      "activations/layer14_attention_weight_min": -43.66858673095703,
      "activations/layer15_attention_weight_max": 49.64251708984375,
      "activations/layer15_attention_weight_min": -43.819053649902344,
      "activations/layer16_attention_weight_max": 36.5518913269043,
      "activations/layer16_attention_weight_min": -31.553068161010742,
      "activations/layer17_attention_weight_max": 38.01468276977539,
      "activations/layer17_attention_weight_min": -28.394065856933594,
      "activations/layer18_attention_weight_max": 36.49563980102539,
      "activations/layer18_attention_weight_min": -27.852853775024414,
      "activations/layer19_attention_weight_max": 43.322330474853516,
      "activations/layer19_attention_weight_min": -35.99971008300781,
      "activations/layer1_attention_weight_max": 15.749674797058105,
      "activations/layer1_attention_weight_min": -14.616608619689941,
      "activations/layer20_attention_weight_max": 36.736141204833984,
      "activations/layer20_attention_weight_min": -27.776859283447266,
      "activations/layer21_attention_weight_max": 34.029544830322266,
      "activations/layer21_attention_weight_min": -26.060461044311523,
      "activations/layer22_attention_weight_max": 49.76380920410156,
      "activations/layer22_attention_weight_min": -33.752681732177734,
      "activations/layer23_attention_weight_max": 39.48528289794922,
      "activations/layer23_attention_weight_min": -26.752185821533203,
      "activations/layer2_attention_weight_max": 32.43215560913086,
      "activations/layer2_attention_weight_min": -32.716209411621094,
      "activations/layer3_attention_weight_max": 105.03214263916016,
      "activations/layer3_attention_weight_min": -105.6659927368164,
      "activations/layer4_attention_weight_max": 107.57787322998047,
      "activations/layer4_attention_weight_min": -113.81131744384766,
      "activations/layer5_attention_weight_max": 52.732513427734375,
      "activations/layer5_attention_weight_min": -62.25605773925781,
      "activations/layer6_attention_weight_max": 45.88962936401367,
      "activations/layer6_attention_weight_min": -48.2263298034668,
      "activations/layer7_attention_weight_max": 97.73090362548828,
      "activations/layer7_attention_weight_min": -98.12496948242188,
      "activations/layer8_attention_weight_max": 43.783626556396484,
      "activations/layer8_attention_weight_min": -47.47875213623047,
      "activations/layer9_attention_weight_max": 35.81822967529297,
      "activations/layer9_attention_weight_min": -37.9837760925293,
      "epoch": 21.51,
      "learning_rate": 1.1397348484848484e-05,
      "loss": 2.7033,
      "step": 370200
    },
    {
      "activations/layer0_attention_weight_max": 17.935102462768555,
      "activations/layer0_attention_weight_min": -14.782186508178711,
      "activations/layer10_attention_weight_max": 36.51490020751953,
      "activations/layer10_attention_weight_min": -35.2999153137207,
      "activations/layer11_attention_weight_max": 34.16865539550781,
      "activations/layer11_attention_weight_min": -36.543800354003906,
      "activations/layer12_attention_weight_max": 27.968814849853516,
      "activations/layer12_attention_weight_min": -27.16106605529785,
      "activations/layer13_attention_weight_max": 41.5483512878418,
      "activations/layer13_attention_weight_min": -35.919166564941406,
      "activations/layer14_attention_weight_max": 41.51750564575195,
      "activations/layer14_attention_weight_min": -41.42021560668945,
      "activations/layer15_attention_weight_max": 41.871856689453125,
      "activations/layer15_attention_weight_min": -36.636070251464844,
      "activations/layer16_attention_weight_max": 30.51734161376953,
      "activations/layer16_attention_weight_min": -27.531267166137695,
      "activations/layer17_attention_weight_max": 33.29415512084961,
      "activations/layer17_attention_weight_min": -25.695228576660156,
      "activations/layer18_attention_weight_max": 31.045053482055664,
      "activations/layer18_attention_weight_min": -26.711809158325195,
      "activations/layer19_attention_weight_max": 37.9672737121582,
      "activations/layer19_attention_weight_min": -32.0909309387207,
      "activations/layer1_attention_weight_max": 15.786489486694336,
      "activations/layer1_attention_weight_min": -14.78881549835205,
      "activations/layer20_attention_weight_max": 29.045419692993164,
      "activations/layer20_attention_weight_min": -25.9268798828125,
      "activations/layer21_attention_weight_max": 28.338104248046875,
      "activations/layer21_attention_weight_min": -23.885986328125,
      "activations/layer22_attention_weight_max": 37.456642150878906,
      "activations/layer22_attention_weight_min": -31.22072982788086,
      "activations/layer23_attention_weight_max": 31.75436019897461,
      "activations/layer23_attention_weight_min": -25.95929718017578,
      "activations/layer2_attention_weight_max": 32.23835754394531,
      "activations/layer2_attention_weight_min": -33.75169372558594,
      "activations/layer3_attention_weight_max": 103.15950012207031,
      "activations/layer3_attention_weight_min": -103.4293212890625,
      "activations/layer4_attention_weight_max": 107.17742919921875,
      "activations/layer4_attention_weight_min": -110.42736053466797,
      "activations/layer5_attention_weight_max": 49.63935852050781,
      "activations/layer5_attention_weight_min": -60.845191955566406,
      "activations/layer6_attention_weight_max": 45.11222457885742,
      "activations/layer6_attention_weight_min": -47.72697067260742,
      "activations/layer7_attention_weight_max": 94.53327178955078,
      "activations/layer7_attention_weight_min": -97.44035339355469,
      "activations/layer8_attention_weight_max": 43.24635696411133,
      "activations/layer8_attention_weight_min": -44.30098342895508,
      "activations/layer9_attention_weight_max": 32.268218994140625,
      "activations/layer9_attention_weight_min": -34.46540832519531,
      "epoch": 21.51,
      "learning_rate": 1.137840909090909e-05,
      "loss": 2.705,
      "step": 370250
    },
    {
      "activations/layer0_attention_weight_max": 15.811737060546875,
      "activations/layer0_attention_weight_min": -14.77888011932373,
      "activations/layer10_attention_weight_max": 33.969303131103516,
      "activations/layer10_attention_weight_min": -34.48372268676758,
      "activations/layer11_attention_weight_max": 33.53052520751953,
      "activations/layer11_attention_weight_min": -35.51203918457031,
      "activations/layer12_attention_weight_max": 28.409059524536133,
      "activations/layer12_attention_weight_min": -27.103666305541992,
      "activations/layer13_attention_weight_max": 43.20768356323242,
      "activations/layer13_attention_weight_min": -37.21440887451172,
      "activations/layer14_attention_weight_max": 46.74665451049805,
      "activations/layer14_attention_weight_min": -40.762027740478516,
      "activations/layer15_attention_weight_max": 46.620155334472656,
      "activations/layer15_attention_weight_min": -39.04576873779297,
      "activations/layer16_attention_weight_max": 31.521638870239258,
      "activations/layer16_attention_weight_min": -26.898685455322266,
      "activations/layer17_attention_weight_max": 29.24260902404785,
      "activations/layer17_attention_weight_min": -26.133901596069336,
      "activations/layer18_attention_weight_max": 32.088138580322266,
      "activations/layer18_attention_weight_min": -25.758333206176758,
      "activations/layer19_attention_weight_max": 32.001041412353516,
      "activations/layer19_attention_weight_min": -30.833959579467773,
      "activations/layer1_attention_weight_max": 16.738811492919922,
      "activations/layer1_attention_weight_min": -15.484219551086426,
      "activations/layer20_attention_weight_max": 28.36623191833496,
      "activations/layer20_attention_weight_min": -24.82133674621582,
      "activations/layer21_attention_weight_max": 27.82674789428711,
      "activations/layer21_attention_weight_min": -24.393503189086914,
      "activations/layer22_attention_weight_max": 42.18743133544922,
      "activations/layer22_attention_weight_min": -30.500091552734375,
      "activations/layer23_attention_weight_max": 34.849735260009766,
      "activations/layer23_attention_weight_min": -26.126766204833984,
      "activations/layer2_attention_weight_max": 34.01696014404297,
      "activations/layer2_attention_weight_min": -31.956817626953125,
      "activations/layer3_attention_weight_max": 108.75228118896484,
      "activations/layer3_attention_weight_min": -106.12975311279297,
      "activations/layer4_attention_weight_max": 112.10710906982422,
      "activations/layer4_attention_weight_min": -117.17586517333984,
      "activations/layer5_attention_weight_max": 51.99469757080078,
      "activations/layer5_attention_weight_min": -62.020015716552734,
      "activations/layer6_attention_weight_max": 47.5123176574707,
      "activations/layer6_attention_weight_min": -47.91997528076172,
      "activations/layer7_attention_weight_max": 97.5733642578125,
      "activations/layer7_attention_weight_min": -94.93838500976562,
      "activations/layer8_attention_weight_max": 42.883480072021484,
      "activations/layer8_attention_weight_min": -43.60966110229492,
      "activations/layer9_attention_weight_max": 32.93357849121094,
      "activations/layer9_attention_weight_min": -34.291099548339844,
      "epoch": 21.52,
      "learning_rate": 1.1359469696969696e-05,
      "loss": 2.7111,
      "step": 370300
    },
    {
      "activations/layer0_attention_weight_max": 16.307231903076172,
      "activations/layer0_attention_weight_min": -16.077198028564453,
      "activations/layer10_attention_weight_max": 34.59669876098633,
      "activations/layer10_attention_weight_min": -35.75433349609375,
      "activations/layer11_attention_weight_max": 33.648963928222656,
      "activations/layer11_attention_weight_min": -36.418190002441406,
      "activations/layer12_attention_weight_max": 27.113975524902344,
      "activations/layer12_attention_weight_min": -28.61676788330078,
      "activations/layer13_attention_weight_max": 39.45777130126953,
      "activations/layer13_attention_weight_min": -37.06954574584961,
      "activations/layer14_attention_weight_max": 40.18232345581055,
      "activations/layer14_attention_weight_min": -40.802669525146484,
      "activations/layer15_attention_weight_max": 42.33433151245117,
      "activations/layer15_attention_weight_min": -38.46199035644531,
      "activations/layer16_attention_weight_max": 27.483522415161133,
      "activations/layer16_attention_weight_min": -27.935747146606445,
      "activations/layer17_attention_weight_max": 29.06897735595703,
      "activations/layer17_attention_weight_min": -26.760217666625977,
      "activations/layer18_attention_weight_max": 32.33332061767578,
      "activations/layer18_attention_weight_min": -26.56005859375,
      "activations/layer19_attention_weight_max": 31.737934112548828,
      "activations/layer19_attention_weight_min": -31.211698532104492,
      "activations/layer1_attention_weight_max": 15.377984046936035,
      "activations/layer1_attention_weight_min": -15.010993003845215,
      "activations/layer20_attention_weight_max": 28.990001678466797,
      "activations/layer20_attention_weight_min": -25.766956329345703,
      "activations/layer21_attention_weight_max": 26.48274803161621,
      "activations/layer21_attention_weight_min": -25.790019989013672,
      "activations/layer22_attention_weight_max": 38.81364059448242,
      "activations/layer22_attention_weight_min": -33.632591247558594,
      "activations/layer23_attention_weight_max": 31.57709503173828,
      "activations/layer23_attention_weight_min": -25.024478912353516,
      "activations/layer2_attention_weight_max": 33.460182189941406,
      "activations/layer2_attention_weight_min": -32.45359802246094,
      "activations/layer3_attention_weight_max": 104.38336944580078,
      "activations/layer3_attention_weight_min": -104.6287841796875,
      "activations/layer4_attention_weight_max": 108.48613739013672,
      "activations/layer4_attention_weight_min": -112.86830139160156,
      "activations/layer5_attention_weight_max": 52.576637268066406,
      "activations/layer5_attention_weight_min": -61.57756042480469,
      "activations/layer6_attention_weight_max": 43.85491180419922,
      "activations/layer6_attention_weight_min": -47.54690170288086,
      "activations/layer7_attention_weight_max": 92.33419799804688,
      "activations/layer7_attention_weight_min": -98.69239044189453,
      "activations/layer8_attention_weight_max": 41.185672760009766,
      "activations/layer8_attention_weight_min": -44.47597122192383,
      "activations/layer9_attention_weight_max": 32.10081481933594,
      "activations/layer9_attention_weight_min": -35.72401428222656,
      "epoch": 21.52,
      "learning_rate": 1.1340530303030302e-05,
      "loss": 2.6902,
      "step": 370350
    },
    {
      "activations/layer0_attention_weight_max": 16.027029037475586,
      "activations/layer0_attention_weight_min": -14.424312591552734,
      "activations/layer10_attention_weight_max": 36.6013298034668,
      "activations/layer10_attention_weight_min": -35.19688415527344,
      "activations/layer11_attention_weight_max": 36.713539123535156,
      "activations/layer11_attention_weight_min": -34.51082992553711,
      "activations/layer12_attention_weight_max": 27.772769927978516,
      "activations/layer12_attention_weight_min": -26.55052947998047,
      "activations/layer13_attention_weight_max": 41.0230712890625,
      "activations/layer13_attention_weight_min": -35.38874435424805,
      "activations/layer14_attention_weight_max": 42.88713455200195,
      "activations/layer14_attention_weight_min": -38.56943893432617,
      "activations/layer15_attention_weight_max": 40.836952209472656,
      "activations/layer15_attention_weight_min": -37.13473892211914,
      "activations/layer16_attention_weight_max": 29.171255111694336,
      "activations/layer16_attention_weight_min": -29.73594093322754,
      "activations/layer17_attention_weight_max": 29.31101417541504,
      "activations/layer17_attention_weight_min": -26.311067581176758,
      "activations/layer18_attention_weight_max": 34.21829605102539,
      "activations/layer18_attention_weight_min": -28.120685577392578,
      "activations/layer19_attention_weight_max": 36.923160552978516,
      "activations/layer19_attention_weight_min": -34.25278854370117,
      "activations/layer1_attention_weight_max": 15.882060050964355,
      "activations/layer1_attention_weight_min": -15.71356201171875,
      "activations/layer20_attention_weight_max": 28.619783401489258,
      "activations/layer20_attention_weight_min": -25.557390213012695,
      "activations/layer21_attention_weight_max": 28.095359802246094,
      "activations/layer21_attention_weight_min": -25.93840980529785,
      "activations/layer22_attention_weight_max": 47.32481384277344,
      "activations/layer22_attention_weight_min": -32.883358001708984,
      "activations/layer23_attention_weight_max": 35.531951904296875,
      "activations/layer23_attention_weight_min": -26.898000717163086,
      "activations/layer2_attention_weight_max": 34.41860580444336,
      "activations/layer2_attention_weight_min": -32.498634338378906,
      "activations/layer3_attention_weight_max": 105.67211151123047,
      "activations/layer3_attention_weight_min": -106.02774810791016,
      "activations/layer4_attention_weight_max": 109.5843276977539,
      "activations/layer4_attention_weight_min": -111.39790344238281,
      "activations/layer5_attention_weight_max": 51.710601806640625,
      "activations/layer5_attention_weight_min": -60.146541595458984,
      "activations/layer6_attention_weight_max": 45.73679733276367,
      "activations/layer6_attention_weight_min": -49.427310943603516,
      "activations/layer7_attention_weight_max": 97.0118637084961,
      "activations/layer7_attention_weight_min": -108.88697814941406,
      "activations/layer8_attention_weight_max": 45.103118896484375,
      "activations/layer8_attention_weight_min": -45.556095123291016,
      "activations/layer9_attention_weight_max": 34.644859313964844,
      "activations/layer9_attention_weight_min": -35.383201599121094,
      "epoch": 21.52,
      "learning_rate": 1.1321590909090909e-05,
      "loss": 2.7011,
      "step": 370400
    },
    {
      "activations/layer0_attention_weight_max": 16.185091018676758,
      "activations/layer0_attention_weight_min": -13.714165687561035,
      "activations/layer10_attention_weight_max": 37.8301887512207,
      "activations/layer10_attention_weight_min": -33.447052001953125,
      "activations/layer11_attention_weight_max": 39.26519012451172,
      "activations/layer11_attention_weight_min": -36.55488204956055,
      "activations/layer12_attention_weight_max": 29.311250686645508,
      "activations/layer12_attention_weight_min": -26.12609100341797,
      "activations/layer13_attention_weight_max": 45.12395477294922,
      "activations/layer13_attention_weight_min": -35.17067337036133,
      "activations/layer14_attention_weight_max": 41.617637634277344,
      "activations/layer14_attention_weight_min": -36.91989517211914,
      "activations/layer15_attention_weight_max": 46.13406753540039,
      "activations/layer15_attention_weight_min": -39.71872329711914,
      "activations/layer16_attention_weight_max": 30.367040634155273,
      "activations/layer16_attention_weight_min": -28.742692947387695,
      "activations/layer17_attention_weight_max": 28.059749603271484,
      "activations/layer17_attention_weight_min": -26.52935218811035,
      "activations/layer18_attention_weight_max": 33.09923553466797,
      "activations/layer18_attention_weight_min": -25.63447380065918,
      "activations/layer19_attention_weight_max": 34.694454193115234,
      "activations/layer19_attention_weight_min": -34.14814376831055,
      "activations/layer1_attention_weight_max": 15.626148223876953,
      "activations/layer1_attention_weight_min": -16.026081085205078,
      "activations/layer20_attention_weight_max": 29.8238468170166,
      "activations/layer20_attention_weight_min": -25.151138305664062,
      "activations/layer21_attention_weight_max": 30.70127296447754,
      "activations/layer21_attention_weight_min": -24.562095642089844,
      "activations/layer22_attention_weight_max": 41.04109191894531,
      "activations/layer22_attention_weight_min": -32.06875228881836,
      "activations/layer23_attention_weight_max": 34.026466369628906,
      "activations/layer23_attention_weight_min": -25.63361930847168,
      "activations/layer2_attention_weight_max": 33.09716796875,
      "activations/layer2_attention_weight_min": -33.68346405029297,
      "activations/layer3_attention_weight_max": 107.41210174560547,
      "activations/layer3_attention_weight_min": -108.48773193359375,
      "activations/layer4_attention_weight_max": 115.5876235961914,
      "activations/layer4_attention_weight_min": -118.73319244384766,
      "activations/layer5_attention_weight_max": 50.8719482421875,
      "activations/layer5_attention_weight_min": -62.18395233154297,
      "activations/layer6_attention_weight_max": 44.10491180419922,
      "activations/layer6_attention_weight_min": -46.50969314575195,
      "activations/layer7_attention_weight_max": 101.31803894042969,
      "activations/layer7_attention_weight_min": -99.99459075927734,
      "activations/layer8_attention_weight_max": 43.682212829589844,
      "activations/layer8_attention_weight_min": -43.93168640136719,
      "activations/layer9_attention_weight_max": 31.93167495727539,
      "activations/layer9_attention_weight_min": -34.25867462158203,
      "epoch": 21.53,
      "learning_rate": 1.1302651515151514e-05,
      "loss": 2.7054,
      "step": 370450
    },
    {
      "activations/layer0_attention_weight_max": 16.661727905273438,
      "activations/layer0_attention_weight_min": -14.537139892578125,
      "activations/layer10_attention_weight_max": 34.99995803833008,
      "activations/layer10_attention_weight_min": -34.81265640258789,
      "activations/layer11_attention_weight_max": 34.88355255126953,
      "activations/layer11_attention_weight_min": -35.51394271850586,
      "activations/layer12_attention_weight_max": 27.708322525024414,
      "activations/layer12_attention_weight_min": -28.386337280273438,
      "activations/layer13_attention_weight_max": 41.13729476928711,
      "activations/layer13_attention_weight_min": -36.81415557861328,
      "activations/layer14_attention_weight_max": 41.401668548583984,
      "activations/layer14_attention_weight_min": -38.429832458496094,
      "activations/layer15_attention_weight_max": 40.336387634277344,
      "activations/layer15_attention_weight_min": -39.77658462524414,
      "activations/layer16_attention_weight_max": 33.64995574951172,
      "activations/layer16_attention_weight_min": -30.44371795654297,
      "activations/layer17_attention_weight_max": 30.05999755859375,
      "activations/layer17_attention_weight_min": -28.215160369873047,
      "activations/layer18_attention_weight_max": 33.87417221069336,
      "activations/layer18_attention_weight_min": -27.841957092285156,
      "activations/layer19_attention_weight_max": 38.02824783325195,
      "activations/layer19_attention_weight_min": -34.37076187133789,
      "activations/layer1_attention_weight_max": 15.729161262512207,
      "activations/layer1_attention_weight_min": -15.240949630737305,
      "activations/layer20_attention_weight_max": 28.867307662963867,
      "activations/layer20_attention_weight_min": -25.58189582824707,
      "activations/layer21_attention_weight_max": 28.859600067138672,
      "activations/layer21_attention_weight_min": -26.2088680267334,
      "activations/layer22_attention_weight_max": 40.29374694824219,
      "activations/layer22_attention_weight_min": -33.5552864074707,
      "activations/layer23_attention_weight_max": 34.635520935058594,
      "activations/layer23_attention_weight_min": -26.419002532958984,
      "activations/layer2_attention_weight_max": 34.6784782409668,
      "activations/layer2_attention_weight_min": -34.789939880371094,
      "activations/layer3_attention_weight_max": 105.32793426513672,
      "activations/layer3_attention_weight_min": -108.59774017333984,
      "activations/layer4_attention_weight_max": 107.85433197021484,
      "activations/layer4_attention_weight_min": -115.2909927368164,
      "activations/layer5_attention_weight_max": 51.76832580566406,
      "activations/layer5_attention_weight_min": -62.717613220214844,
      "activations/layer6_attention_weight_max": 43.864036560058594,
      "activations/layer6_attention_weight_min": -49.46184539794922,
      "activations/layer7_attention_weight_max": 94.42208862304688,
      "activations/layer7_attention_weight_min": -102.61422729492188,
      "activations/layer8_attention_weight_max": 41.01842498779297,
      "activations/layer8_attention_weight_min": -44.2196159362793,
      "activations/layer9_attention_weight_max": 31.450946807861328,
      "activations/layer9_attention_weight_min": -34.43761444091797,
      "epoch": 21.53,
      "learning_rate": 1.128371212121212e-05,
      "loss": 2.7025,
      "step": 370500
    },
    {
      "activations/layer0_attention_weight_max": 17.389469146728516,
      "activations/layer0_attention_weight_min": -14.235246658325195,
      "activations/layer10_attention_weight_max": 33.455078125,
      "activations/layer10_attention_weight_min": -33.86305618286133,
      "activations/layer11_attention_weight_max": 34.540924072265625,
      "activations/layer11_attention_weight_min": -34.55284118652344,
      "activations/layer12_attention_weight_max": 28.53095817565918,
      "activations/layer12_attention_weight_min": -25.911951065063477,
      "activations/layer13_attention_weight_max": 43.015785217285156,
      "activations/layer13_attention_weight_min": -34.63570022583008,
      "activations/layer14_attention_weight_max": 43.0984992980957,
      "activations/layer14_attention_weight_min": -38.0743293762207,
      "activations/layer15_attention_weight_max": 40.463714599609375,
      "activations/layer15_attention_weight_min": -35.74571990966797,
      "activations/layer16_attention_weight_max": 27.74101448059082,
      "activations/layer16_attention_weight_min": -27.226625442504883,
      "activations/layer17_attention_weight_max": 30.87887954711914,
      "activations/layer17_attention_weight_min": -25.759387969970703,
      "activations/layer18_attention_weight_max": 32.57514572143555,
      "activations/layer18_attention_weight_min": -27.048885345458984,
      "activations/layer19_attention_weight_max": 32.951263427734375,
      "activations/layer19_attention_weight_min": -31.938434600830078,
      "activations/layer1_attention_weight_max": 15.684316635131836,
      "activations/layer1_attention_weight_min": -15.673836708068848,
      "activations/layer20_attention_weight_max": 29.506053924560547,
      "activations/layer20_attention_weight_min": -26.217313766479492,
      "activations/layer21_attention_weight_max": 32.979835510253906,
      "activations/layer21_attention_weight_min": -25.97200584411621,
      "activations/layer22_attention_weight_max": 42.07123565673828,
      "activations/layer22_attention_weight_min": -31.22024917602539,
      "activations/layer23_attention_weight_max": 32.55631637573242,
      "activations/layer23_attention_weight_min": -26.022846221923828,
      "activations/layer2_attention_weight_max": 32.73101806640625,
      "activations/layer2_attention_weight_min": -33.306575775146484,
      "activations/layer3_attention_weight_max": 106.37371826171875,
      "activations/layer3_attention_weight_min": -106.0581283569336,
      "activations/layer4_attention_weight_max": 111.78246307373047,
      "activations/layer4_attention_weight_min": -113.29278564453125,
      "activations/layer5_attention_weight_max": 51.83856201171875,
      "activations/layer5_attention_weight_min": -63.089500427246094,
      "activations/layer6_attention_weight_max": 44.89772415161133,
      "activations/layer6_attention_weight_min": -47.919898986816406,
      "activations/layer7_attention_weight_max": 93.9516372680664,
      "activations/layer7_attention_weight_min": -103.40253448486328,
      "activations/layer8_attention_weight_max": 42.56622314453125,
      "activations/layer8_attention_weight_min": -44.380306243896484,
      "activations/layer9_attention_weight_max": 32.44047164916992,
      "activations/layer9_attention_weight_min": -34.63584899902344,
      "epoch": 21.53,
      "learning_rate": 1.1264772727272725e-05,
      "loss": 2.7083,
      "step": 370550
    },
    {
      "activations/layer0_attention_weight_max": 16.28934097290039,
      "activations/layer0_attention_weight_min": -12.998936653137207,
      "activations/layer10_attention_weight_max": 37.42685317993164,
      "activations/layer10_attention_weight_min": -36.374847412109375,
      "activations/layer11_attention_weight_max": 37.557613372802734,
      "activations/layer11_attention_weight_min": -37.97335433959961,
      "activations/layer12_attention_weight_max": 29.29508399963379,
      "activations/layer12_attention_weight_min": -26.80835723876953,
      "activations/layer13_attention_weight_max": 40.38749694824219,
      "activations/layer13_attention_weight_min": -35.62346267700195,
      "activations/layer14_attention_weight_max": 47.417808532714844,
      "activations/layer14_attention_weight_min": -37.482486724853516,
      "activations/layer15_attention_weight_max": 44.795753479003906,
      "activations/layer15_attention_weight_min": -37.051639556884766,
      "activations/layer16_attention_weight_max": 32.07361602783203,
      "activations/layer16_attention_weight_min": -28.85411834716797,
      "activations/layer17_attention_weight_max": 31.102136611938477,
      "activations/layer17_attention_weight_min": -26.7064266204834,
      "activations/layer18_attention_weight_max": 32.226531982421875,
      "activations/layer18_attention_weight_min": -25.601675033569336,
      "activations/layer19_attention_weight_max": 35.134952545166016,
      "activations/layer19_attention_weight_min": -32.39448165893555,
      "activations/layer1_attention_weight_max": 17.233261108398438,
      "activations/layer1_attention_weight_min": -16.331045150756836,
      "activations/layer20_attention_weight_max": 30.227087020874023,
      "activations/layer20_attention_weight_min": -25.426860809326172,
      "activations/layer21_attention_weight_max": 31.683494567871094,
      "activations/layer21_attention_weight_min": -24.33384895324707,
      "activations/layer22_attention_weight_max": 42.70745849609375,
      "activations/layer22_attention_weight_min": -30.719039916992188,
      "activations/layer23_attention_weight_max": 31.042232513427734,
      "activations/layer23_attention_weight_min": -26.49349594116211,
      "activations/layer2_attention_weight_max": 34.095848083496094,
      "activations/layer2_attention_weight_min": -34.04319763183594,
      "activations/layer3_attention_weight_max": 104.84477996826172,
      "activations/layer3_attention_weight_min": -101.7058334350586,
      "activations/layer4_attention_weight_max": 106.52484130859375,
      "activations/layer4_attention_weight_min": -106.25543212890625,
      "activations/layer5_attention_weight_max": 50.37406539916992,
      "activations/layer5_attention_weight_min": -61.123291015625,
      "activations/layer6_attention_weight_max": 45.98056411743164,
      "activations/layer6_attention_weight_min": -46.63497543334961,
      "activations/layer7_attention_weight_max": 102.77879333496094,
      "activations/layer7_attention_weight_min": -96.01713562011719,
      "activations/layer8_attention_weight_max": 44.574249267578125,
      "activations/layer8_attention_weight_min": -46.03019714355469,
      "activations/layer9_attention_weight_max": 33.31086349487305,
      "activations/layer9_attention_weight_min": -36.527793884277344,
      "epoch": 21.53,
      "learning_rate": 1.1245833333333332e-05,
      "loss": 2.7193,
      "step": 370600
    },
    {
      "activations/layer0_attention_weight_max": 15.523736953735352,
      "activations/layer0_attention_weight_min": -13.93325424194336,
      "activations/layer10_attention_weight_max": 42.60970687866211,
      "activations/layer10_attention_weight_min": -38.35109329223633,
      "activations/layer11_attention_weight_max": 42.455833435058594,
      "activations/layer11_attention_weight_min": -40.0848388671875,
      "activations/layer12_attention_weight_max": 28.66445541381836,
      "activations/layer12_attention_weight_min": -26.87711524963379,
      "activations/layer13_attention_weight_max": 40.74441146850586,
      "activations/layer13_attention_weight_min": -36.94499206542969,
      "activations/layer14_attention_weight_max": 51.13493347167969,
      "activations/layer14_attention_weight_min": -38.89937210083008,
      "activations/layer15_attention_weight_max": 43.30649185180664,
      "activations/layer15_attention_weight_min": -37.42128372192383,
      "activations/layer16_attention_weight_max": 30.785722732543945,
      "activations/layer16_attention_weight_min": -28.045244216918945,
      "activations/layer17_attention_weight_max": 29.91628646850586,
      "activations/layer17_attention_weight_min": -25.870615005493164,
      "activations/layer18_attention_weight_max": 33.911895751953125,
      "activations/layer18_attention_weight_min": -25.618114471435547,
      "activations/layer19_attention_weight_max": 35.903289794921875,
      "activations/layer19_attention_weight_min": -32.40920639038086,
      "activations/layer1_attention_weight_max": 15.906888008117676,
      "activations/layer1_attention_weight_min": -16.283157348632812,
      "activations/layer20_attention_weight_max": 29.079694747924805,
      "activations/layer20_attention_weight_min": -24.78363037109375,
      "activations/layer21_attention_weight_max": 28.762569427490234,
      "activations/layer21_attention_weight_min": -24.07974624633789,
      "activations/layer22_attention_weight_max": 41.769775390625,
      "activations/layer22_attention_weight_min": -32.616512298583984,
      "activations/layer23_attention_weight_max": 33.81824493408203,
      "activations/layer23_attention_weight_min": -24.72058868408203,
      "activations/layer2_attention_weight_max": 35.35459899902344,
      "activations/layer2_attention_weight_min": -34.975345611572266,
      "activations/layer3_attention_weight_max": 110.68890380859375,
      "activations/layer3_attention_weight_min": -109.46709442138672,
      "activations/layer4_attention_weight_max": 113.15406799316406,
      "activations/layer4_attention_weight_min": -116.2684326171875,
      "activations/layer5_attention_weight_max": 53.645782470703125,
      "activations/layer5_attention_weight_min": -60.72100067138672,
      "activations/layer6_attention_weight_max": 46.82815170288086,
      "activations/layer6_attention_weight_min": -53.712074279785156,
      "activations/layer7_attention_weight_max": 112.4485855102539,
      "activations/layer7_attention_weight_min": -105.35313415527344,
      "activations/layer8_attention_weight_max": 48.746192932128906,
      "activations/layer8_attention_weight_min": -50.55133819580078,
      "activations/layer9_attention_weight_max": 35.992103576660156,
      "activations/layer9_attention_weight_min": -38.37661361694336,
      "epoch": 21.54,
      "learning_rate": 1.1226893939393938e-05,
      "loss": 2.7024,
      "step": 370650
    },
    {
      "activations/layer0_attention_weight_max": 16.586153030395508,
      "activations/layer0_attention_weight_min": -12.929640769958496,
      "activations/layer10_attention_weight_max": 37.53757095336914,
      "activations/layer10_attention_weight_min": -36.164833068847656,
      "activations/layer11_attention_weight_max": 38.39677810668945,
      "activations/layer11_attention_weight_min": -38.31510925292969,
      "activations/layer12_attention_weight_max": 29.291059494018555,
      "activations/layer12_attention_weight_min": -27.745155334472656,
      "activations/layer13_attention_weight_max": 44.468326568603516,
      "activations/layer13_attention_weight_min": -38.45567321777344,
      "activations/layer14_attention_weight_max": 48.542991638183594,
      "activations/layer14_attention_weight_min": -42.44953918457031,
      "activations/layer15_attention_weight_max": 45.09019088745117,
      "activations/layer15_attention_weight_min": -39.577693939208984,
      "activations/layer16_attention_weight_max": 31.162609100341797,
      "activations/layer16_attention_weight_min": -29.448410034179688,
      "activations/layer17_attention_weight_max": 30.444772720336914,
      "activations/layer17_attention_weight_min": -26.322635650634766,
      "activations/layer18_attention_weight_max": 35.840301513671875,
      "activations/layer18_attention_weight_min": -26.692794799804688,
      "activations/layer19_attention_weight_max": 38.97002410888672,
      "activations/layer19_attention_weight_min": -32.222206115722656,
      "activations/layer1_attention_weight_max": 16.832569122314453,
      "activations/layer1_attention_weight_min": -16.255229949951172,
      "activations/layer20_attention_weight_max": 34.36814498901367,
      "activations/layer20_attention_weight_min": -25.315584182739258,
      "activations/layer21_attention_weight_max": 31.7581844329834,
      "activations/layer21_attention_weight_min": -26.686025619506836,
      "activations/layer22_attention_weight_max": 47.41465759277344,
      "activations/layer22_attention_weight_min": -35.57014465332031,
      "activations/layer23_attention_weight_max": 38.88825988769531,
      "activations/layer23_attention_weight_min": -26.421432495117188,
      "activations/layer2_attention_weight_max": 33.99545669555664,
      "activations/layer2_attention_weight_min": -32.9088134765625,
      "activations/layer3_attention_weight_max": 108.28356170654297,
      "activations/layer3_attention_weight_min": -105.98918151855469,
      "activations/layer4_attention_weight_max": 111.19935607910156,
      "activations/layer4_attention_weight_min": -112.07097625732422,
      "activations/layer5_attention_weight_max": 51.30855941772461,
      "activations/layer5_attention_weight_min": -64.60941314697266,
      "activations/layer6_attention_weight_max": 45.270626068115234,
      "activations/layer6_attention_weight_min": -49.04719543457031,
      "activations/layer7_attention_weight_max": 94.111328125,
      "activations/layer7_attention_weight_min": -102.53619384765625,
      "activations/layer8_attention_weight_max": 45.074466705322266,
      "activations/layer8_attention_weight_min": -46.979000091552734,
      "activations/layer9_attention_weight_max": 32.785457611083984,
      "activations/layer9_attention_weight_min": -37.043312072753906,
      "epoch": 21.54,
      "learning_rate": 1.1207954545454543e-05,
      "loss": 2.7046,
      "step": 370700
    },
    {
      "activations/layer0_attention_weight_max": 18.523046493530273,
      "activations/layer0_attention_weight_min": -13.449679374694824,
      "activations/layer10_attention_weight_max": 34.89164352416992,
      "activations/layer10_attention_weight_min": -34.356727600097656,
      "activations/layer11_attention_weight_max": 35.49720764160156,
      "activations/layer11_attention_weight_min": -36.20903015136719,
      "activations/layer12_attention_weight_max": 26.471027374267578,
      "activations/layer12_attention_weight_min": -26.591901779174805,
      "activations/layer13_attention_weight_max": 38.75643539428711,
      "activations/layer13_attention_weight_min": -36.70859909057617,
      "activations/layer14_attention_weight_max": 41.723854064941406,
      "activations/layer14_attention_weight_min": -39.38682556152344,
      "activations/layer15_attention_weight_max": 43.75060272216797,
      "activations/layer15_attention_weight_min": -36.64125061035156,
      "activations/layer16_attention_weight_max": 31.355045318603516,
      "activations/layer16_attention_weight_min": -30.03043556213379,
      "activations/layer17_attention_weight_max": 31.1263370513916,
      "activations/layer17_attention_weight_min": -24.865032196044922,
      "activations/layer18_attention_weight_max": 34.55583572387695,
      "activations/layer18_attention_weight_min": -26.288991928100586,
      "activations/layer19_attention_weight_max": 39.08705520629883,
      "activations/layer19_attention_weight_min": -34.10103988647461,
      "activations/layer1_attention_weight_max": 16.042070388793945,
      "activations/layer1_attention_weight_min": -14.939790725708008,
      "activations/layer20_attention_weight_max": 35.0313835144043,
      "activations/layer20_attention_weight_min": -23.79071044921875,
      "activations/layer21_attention_weight_max": 32.92836380004883,
      "activations/layer21_attention_weight_min": -24.178007125854492,
      "activations/layer22_attention_weight_max": 42.84728240966797,
      "activations/layer22_attention_weight_min": -31.187129974365234,
      "activations/layer23_attention_weight_max": 33.247623443603516,
      "activations/layer23_attention_weight_min": -25.292484283447266,
      "activations/layer2_attention_weight_max": 36.877586364746094,
      "activations/layer2_attention_weight_min": -32.739898681640625,
      "activations/layer3_attention_weight_max": 105.11540222167969,
      "activations/layer3_attention_weight_min": -107.13220977783203,
      "activations/layer4_attention_weight_max": 108.08082580566406,
      "activations/layer4_attention_weight_min": -111.845703125,
      "activations/layer5_attention_weight_max": 51.12314987182617,
      "activations/layer5_attention_weight_min": -61.75395202636719,
      "activations/layer6_attention_weight_max": 46.44163513183594,
      "activations/layer6_attention_weight_min": -49.42280197143555,
      "activations/layer7_attention_weight_max": 98.39704895019531,
      "activations/layer7_attention_weight_min": -97.90668487548828,
      "activations/layer8_attention_weight_max": 41.96071243286133,
      "activations/layer8_attention_weight_min": -45.062347412109375,
      "activations/layer9_attention_weight_max": 30.942941665649414,
      "activations/layer9_attention_weight_min": -34.24924850463867,
      "epoch": 21.54,
      "learning_rate": 1.118901515151515e-05,
      "loss": 2.7054,
      "step": 370750
    },
    {
      "activations/layer0_attention_weight_max": 17.137483596801758,
      "activations/layer0_attention_weight_min": -15.54143238067627,
      "activations/layer10_attention_weight_max": 43.4051399230957,
      "activations/layer10_attention_weight_min": -40.52705764770508,
      "activations/layer11_attention_weight_max": 42.29550552368164,
      "activations/layer11_attention_weight_min": -39.771156311035156,
      "activations/layer12_attention_weight_max": 27.616451263427734,
      "activations/layer12_attention_weight_min": -27.65060043334961,
      "activations/layer13_attention_weight_max": 39.19707107543945,
      "activations/layer13_attention_weight_min": -36.781429290771484,
      "activations/layer14_attention_weight_max": 45.87167739868164,
      "activations/layer14_attention_weight_min": -38.60102462768555,
      "activations/layer15_attention_weight_max": 40.944400787353516,
      "activations/layer15_attention_weight_min": -37.9215087890625,
      "activations/layer16_attention_weight_max": 31.021987915039062,
      "activations/layer16_attention_weight_min": -27.609804153442383,
      "activations/layer17_attention_weight_max": 28.728830337524414,
      "activations/layer17_attention_weight_min": -27.12822723388672,
      "activations/layer18_attention_weight_max": 32.5211181640625,
      "activations/layer18_attention_weight_min": -25.43273162841797,
      "activations/layer19_attention_weight_max": 34.582183837890625,
      "activations/layer19_attention_weight_min": -32.145816802978516,
      "activations/layer1_attention_weight_max": 16.547245025634766,
      "activations/layer1_attention_weight_min": -15.746745109558105,
      "activations/layer20_attention_weight_max": 30.27684783935547,
      "activations/layer20_attention_weight_min": -24.135229110717773,
      "activations/layer21_attention_weight_max": 29.898666381835938,
      "activations/layer21_attention_weight_min": -23.607290267944336,
      "activations/layer22_attention_weight_max": 45.518402099609375,
      "activations/layer22_attention_weight_min": -31.01622772216797,
      "activations/layer23_attention_weight_max": 33.780616760253906,
      "activations/layer23_attention_weight_min": -26.70587730407715,
      "activations/layer2_attention_weight_max": 35.602783203125,
      "activations/layer2_attention_weight_min": -34.30792236328125,
      "activations/layer3_attention_weight_max": 108.14714813232422,
      "activations/layer3_attention_weight_min": -105.81334686279297,
      "activations/layer4_attention_weight_max": 117.04058837890625,
      "activations/layer4_attention_weight_min": -112.162353515625,
      "activations/layer5_attention_weight_max": 51.78807830810547,
      "activations/layer5_attention_weight_min": -66.15818786621094,
      "activations/layer6_attention_weight_max": 46.68552780151367,
      "activations/layer6_attention_weight_min": -49.99617004394531,
      "activations/layer7_attention_weight_max": 107.85955810546875,
      "activations/layer7_attention_weight_min": -106.01204681396484,
      "activations/layer8_attention_weight_max": 47.9415397644043,
      "activations/layer8_attention_weight_min": -49.89650344848633,
      "activations/layer9_attention_weight_max": 35.468040466308594,
      "activations/layer9_attention_weight_min": -38.48015594482422,
      "epoch": 21.55,
      "learning_rate": 1.1170075757575758e-05,
      "loss": 2.7276,
      "step": 370800
    },
    {
      "activations/layer0_attention_weight_max": 16.33791732788086,
      "activations/layer0_attention_weight_min": -14.22515869140625,
      "activations/layer10_attention_weight_max": 35.86286926269531,
      "activations/layer10_attention_weight_min": -38.81786346435547,
      "activations/layer11_attention_weight_max": 36.4854850769043,
      "activations/layer11_attention_weight_min": -37.04701232910156,
      "activations/layer12_attention_weight_max": 31.1700496673584,
      "activations/layer12_attention_weight_min": -29.489986419677734,
      "activations/layer13_attention_weight_max": 46.618507385253906,
      "activations/layer13_attention_weight_min": -38.12342834472656,
      "activations/layer14_attention_weight_max": 50.04395294189453,
      "activations/layer14_attention_weight_min": -42.79283142089844,
      "activations/layer15_attention_weight_max": 47.56224822998047,
      "activations/layer15_attention_weight_min": -42.51343536376953,
      "activations/layer16_attention_weight_max": 37.09794616699219,
      "activations/layer16_attention_weight_min": -30.81866455078125,
      "activations/layer17_attention_weight_max": 34.79792404174805,
      "activations/layer17_attention_weight_min": -27.48961067199707,
      "activations/layer18_attention_weight_max": 40.019264221191406,
      "activations/layer18_attention_weight_min": -26.961307525634766,
      "activations/layer19_attention_weight_max": 45.21855163574219,
      "activations/layer19_attention_weight_min": -33.14098358154297,
      "activations/layer1_attention_weight_max": 17.246435165405273,
      "activations/layer1_attention_weight_min": -16.1690673828125,
      "activations/layer20_attention_weight_max": 38.294761657714844,
      "activations/layer20_attention_weight_min": -26.095853805541992,
      "activations/layer21_attention_weight_max": 38.115116119384766,
      "activations/layer21_attention_weight_min": -24.628158569335938,
      "activations/layer22_attention_weight_max": 61.051658630371094,
      "activations/layer22_attention_weight_min": -32.94812774658203,
      "activations/layer23_attention_weight_max": 41.613319396972656,
      "activations/layer23_attention_weight_min": -27.456684112548828,
      "activations/layer2_attention_weight_max": 34.27657699584961,
      "activations/layer2_attention_weight_min": -33.641754150390625,
      "activations/layer3_attention_weight_max": 107.34017181396484,
      "activations/layer3_attention_weight_min": -107.57450103759766,
      "activations/layer4_attention_weight_max": 112.6583023071289,
      "activations/layer4_attention_weight_min": -113.95771789550781,
      "activations/layer5_attention_weight_max": 52.59178161621094,
      "activations/layer5_attention_weight_min": -64.33655548095703,
      "activations/layer6_attention_weight_max": 47.793190002441406,
      "activations/layer6_attention_weight_min": -52.256160736083984,
      "activations/layer7_attention_weight_max": 105.28583526611328,
      "activations/layer7_attention_weight_min": -110.77976989746094,
      "activations/layer8_attention_weight_max": 47.43446350097656,
      "activations/layer8_attention_weight_min": -49.65774917602539,
      "activations/layer9_attention_weight_max": 35.88278579711914,
      "activations/layer9_attention_weight_min": -37.05973815917969,
      "epoch": 21.55,
      "learning_rate": 1.1151136363636363e-05,
      "loss": 2.6982,
      "step": 370850
    },
    {
      "activations/layer0_attention_weight_max": 16.22950553894043,
      "activations/layer0_attention_weight_min": -13.635513305664062,
      "activations/layer10_attention_weight_max": 37.88367462158203,
      "activations/layer10_attention_weight_min": -37.28436279296875,
      "activations/layer11_attention_weight_max": 37.31443786621094,
      "activations/layer11_attention_weight_min": -38.037109375,
      "activations/layer12_attention_weight_max": 29.06186294555664,
      "activations/layer12_attention_weight_min": -27.520212173461914,
      "activations/layer13_attention_weight_max": 43.71691131591797,
      "activations/layer13_attention_weight_min": -38.14634704589844,
      "activations/layer14_attention_weight_max": 41.95298385620117,
      "activations/layer14_attention_weight_min": -39.414527893066406,
      "activations/layer15_attention_weight_max": 40.8007698059082,
      "activations/layer15_attention_weight_min": -38.847171783447266,
      "activations/layer16_attention_weight_max": 31.741853713989258,
      "activations/layer16_attention_weight_min": -31.812646865844727,
      "activations/layer17_attention_weight_max": 31.728750228881836,
      "activations/layer17_attention_weight_min": -30.64657974243164,
      "activations/layer18_attention_weight_max": 34.615482330322266,
      "activations/layer18_attention_weight_min": -28.80828285217285,
      "activations/layer19_attention_weight_max": 38.87793731689453,
      "activations/layer19_attention_weight_min": -33.8828239440918,
      "activations/layer1_attention_weight_max": 15.86705493927002,
      "activations/layer1_attention_weight_min": -15.444786071777344,
      "activations/layer20_attention_weight_max": 34.61592483520508,
      "activations/layer20_attention_weight_min": -28.519622802734375,
      "activations/layer21_attention_weight_max": 35.290470123291016,
      "activations/layer21_attention_weight_min": -27.70206069946289,
      "activations/layer22_attention_weight_max": 45.29916763305664,
      "activations/layer22_attention_weight_min": -32.83011245727539,
      "activations/layer23_attention_weight_max": 35.43841552734375,
      "activations/layer23_attention_weight_min": -27.160879135131836,
      "activations/layer2_attention_weight_max": 33.415687561035156,
      "activations/layer2_attention_weight_min": -35.48866271972656,
      "activations/layer3_attention_weight_max": 107.7812271118164,
      "activations/layer3_attention_weight_min": -115.94935607910156,
      "activations/layer4_attention_weight_max": 112.45035552978516,
      "activations/layer4_attention_weight_min": -115.32454681396484,
      "activations/layer5_attention_weight_max": 53.98511505126953,
      "activations/layer5_attention_weight_min": -64.29145812988281,
      "activations/layer6_attention_weight_max": 46.687007904052734,
      "activations/layer6_attention_weight_min": -49.39702606201172,
      "activations/layer7_attention_weight_max": 97.35865020751953,
      "activations/layer7_attention_weight_min": -105.23760223388672,
      "activations/layer8_attention_weight_max": 44.9146842956543,
      "activations/layer8_attention_weight_min": -47.01386642456055,
      "activations/layer9_attention_weight_max": 33.97829818725586,
      "activations/layer9_attention_weight_min": -36.568580627441406,
      "epoch": 21.55,
      "learning_rate": 1.113219696969697e-05,
      "loss": 2.6927,
      "step": 370900
    },
    {
      "activations/layer0_attention_weight_max": 16.204164505004883,
      "activations/layer0_attention_weight_min": -14.268442153930664,
      "activations/layer10_attention_weight_max": 33.686893463134766,
      "activations/layer10_attention_weight_min": -35.69542694091797,
      "activations/layer11_attention_weight_max": 32.99840545654297,
      "activations/layer11_attention_weight_min": -35.22672653198242,
      "activations/layer12_attention_weight_max": 26.835290908813477,
      "activations/layer12_attention_weight_min": -27.18650245666504,
      "activations/layer13_attention_weight_max": 39.04269027709961,
      "activations/layer13_attention_weight_min": -35.22380828857422,
      "activations/layer14_attention_weight_max": 42.17023849487305,
      "activations/layer14_attention_weight_min": -37.62268829345703,
      "activations/layer15_attention_weight_max": 41.52545166015625,
      "activations/layer15_attention_weight_min": -36.908721923828125,
      "activations/layer16_attention_weight_max": 29.701295852661133,
      "activations/layer16_attention_weight_min": -28.671045303344727,
      "activations/layer17_attention_weight_max": 29.31698226928711,
      "activations/layer17_attention_weight_min": -26.628644943237305,
      "activations/layer18_attention_weight_max": 38.34033203125,
      "activations/layer18_attention_weight_min": -27.3507080078125,
      "activations/layer19_attention_weight_max": 38.677696228027344,
      "activations/layer19_attention_weight_min": -34.39058303833008,
      "activations/layer1_attention_weight_max": 15.54968547821045,
      "activations/layer1_attention_weight_min": -15.759950637817383,
      "activations/layer20_attention_weight_max": 31.762231826782227,
      "activations/layer20_attention_weight_min": -25.23244857788086,
      "activations/layer21_attention_weight_max": 31.65978240966797,
      "activations/layer21_attention_weight_min": -26.568859100341797,
      "activations/layer22_attention_weight_max": 43.766075134277344,
      "activations/layer22_attention_weight_min": -34.87998962402344,
      "activations/layer23_attention_weight_max": 33.05824279785156,
      "activations/layer23_attention_weight_min": -28.584569931030273,
      "activations/layer2_attention_weight_max": 33.5167121887207,
      "activations/layer2_attention_weight_min": -33.29541015625,
      "activations/layer3_attention_weight_max": 104.95208740234375,
      "activations/layer3_attention_weight_min": -107.86726379394531,
      "activations/layer4_attention_weight_max": 112.29964447021484,
      "activations/layer4_attention_weight_min": -118.6147689819336,
      "activations/layer5_attention_weight_max": 59.29157257080078,
      "activations/layer5_attention_weight_min": -62.667442321777344,
      "activations/layer6_attention_weight_max": 45.63966369628906,
      "activations/layer6_attention_weight_min": -49.855857849121094,
      "activations/layer7_attention_weight_max": 97.1287612915039,
      "activations/layer7_attention_weight_min": -100.42001342773438,
      "activations/layer8_attention_weight_max": 41.111385345458984,
      "activations/layer8_attention_weight_min": -46.199798583984375,
      "activations/layer9_attention_weight_max": 31.00838279724121,
      "activations/layer9_attention_weight_min": -35.47042465209961,
      "epoch": 21.55,
      "learning_rate": 1.1113257575757576e-05,
      "loss": 2.7262,
      "step": 370950
    },
    {
      "activations/layer0_attention_weight_max": 16.782638549804688,
      "activations/layer0_attention_weight_min": -15.361347198486328,
      "activations/layer10_attention_weight_max": 37.5088996887207,
      "activations/layer10_attention_weight_min": -36.073707580566406,
      "activations/layer11_attention_weight_max": 37.010643005371094,
      "activations/layer11_attention_weight_min": -37.574005126953125,
      "activations/layer12_attention_weight_max": 28.296512603759766,
      "activations/layer12_attention_weight_min": -27.84229850769043,
      "activations/layer13_attention_weight_max": 41.65554428100586,
      "activations/layer13_attention_weight_min": -36.49622344970703,
      "activations/layer14_attention_weight_max": 45.71644973754883,
      "activations/layer14_attention_weight_min": -37.068485260009766,
      "activations/layer15_attention_weight_max": 39.58978271484375,
      "activations/layer15_attention_weight_min": -34.84032440185547,
      "activations/layer16_attention_weight_max": 31.639501571655273,
      "activations/layer16_attention_weight_min": -28.126638412475586,
      "activations/layer17_attention_weight_max": 32.78065872192383,
      "activations/layer17_attention_weight_min": -25.584394454956055,
      "activations/layer18_attention_weight_max": 34.39305877685547,
      "activations/layer18_attention_weight_min": -25.030780792236328,
      "activations/layer19_attention_weight_max": 37.31826400756836,
      "activations/layer19_attention_weight_min": -34.18369674682617,
      "activations/layer1_attention_weight_max": 15.80967903137207,
      "activations/layer1_attention_weight_min": -14.503670692443848,
      "activations/layer20_attention_weight_max": 30.954992294311523,
      "activations/layer20_attention_weight_min": -25.154706954956055,
      "activations/layer21_attention_weight_max": 33.283058166503906,
      "activations/layer21_attention_weight_min": -25.689138412475586,
      "activations/layer22_attention_weight_max": 45.13815689086914,
      "activations/layer22_attention_weight_min": -31.563995361328125,
      "activations/layer23_attention_weight_max": 34.01559829711914,
      "activations/layer23_attention_weight_min": -25.01608657836914,
      "activations/layer2_attention_weight_max": 33.92894744873047,
      "activations/layer2_attention_weight_min": -34.33184051513672,
      "activations/layer3_attention_weight_max": 106.03042602539062,
      "activations/layer3_attention_weight_min": -107.66342163085938,
      "activations/layer4_attention_weight_max": 110.19327545166016,
      "activations/layer4_attention_weight_min": -114.4562759399414,
      "activations/layer5_attention_weight_max": 49.54639434814453,
      "activations/layer5_attention_weight_min": -63.60771942138672,
      "activations/layer6_attention_weight_max": 46.44849395751953,
      "activations/layer6_attention_weight_min": -50.115779876708984,
      "activations/layer7_attention_weight_max": 99.75969696044922,
      "activations/layer7_attention_weight_min": -102.6266860961914,
      "activations/layer8_attention_weight_max": 43.127784729003906,
      "activations/layer8_attention_weight_min": -47.68214797973633,
      "activations/layer9_attention_weight_max": 32.37043380737305,
      "activations/layer9_attention_weight_min": -35.56285095214844,
      "epoch": 21.56,
      "learning_rate": 1.1094318181818181e-05,
      "loss": 2.7175,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.4939,
      "eval_samples_per_second": 505.541,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.4939,
      "eval_openwebtext_samples_per_second": 505.541,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9877,
      "eval_wikitext_samples_per_second": 229.414,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_lambada_loss": 2.296875,
      "eval_lambada_ppl": 9.943061786014486,
      "eval_lambada_runtime": 9.591,
      "eval_lambada_samples_per_second": 507.665,
      "step": 371000
    },
    {
      "activations/layer0_attention_weight_max": 17.08300018310547,
      "activations/layer0_attention_weight_min": -14.895060539245605,
      "activations/layer10_attention_weight_max": 36.26528549194336,
      "activations/layer10_attention_weight_min": -37.513980865478516,
      "activations/layer11_attention_weight_max": 36.632259368896484,
      "activations/layer11_attention_weight_min": -38.7406120300293,
      "activations/layer12_attention_weight_max": 31.736478805541992,
      "activations/layer12_attention_weight_min": -27.983945846557617,
      "activations/layer13_attention_weight_max": 40.163414001464844,
      "activations/layer13_attention_weight_min": -36.957794189453125,
      "activations/layer14_attention_weight_max": 43.07756423950195,
      "activations/layer14_attention_weight_min": -39.22799301147461,
      "activations/layer15_attention_weight_max": 39.73550033569336,
      "activations/layer15_attention_weight_min": -36.836421966552734,
      "activations/layer16_attention_weight_max": 29.320514678955078,
      "activations/layer16_attention_weight_min": -28.865697860717773,
      "activations/layer17_attention_weight_max": 29.35103416442871,
      "activations/layer17_attention_weight_min": -25.622753143310547,
      "activations/layer18_attention_weight_max": 33.35432434082031,
      "activations/layer18_attention_weight_min": -25.14003562927246,
      "activations/layer19_attention_weight_max": 36.24305725097656,
      "activations/layer19_attention_weight_min": -32.71456527709961,
      "activations/layer1_attention_weight_max": 15.497345924377441,
      "activations/layer1_attention_weight_min": -14.453919410705566,
      "activations/layer20_attention_weight_max": 30.708999633789062,
      "activations/layer20_attention_weight_min": -24.831165313720703,
      "activations/layer21_attention_weight_max": 30.65812110900879,
      "activations/layer21_attention_weight_min": -24.069326400756836,
      "activations/layer22_attention_weight_max": 41.360931396484375,
      "activations/layer22_attention_weight_min": -32.916866302490234,
      "activations/layer23_attention_weight_max": 33.16320037841797,
      "activations/layer23_attention_weight_min": -25.380971908569336,
      "activations/layer2_attention_weight_max": 35.40199279785156,
      "activations/layer2_attention_weight_min": -33.513587951660156,
      "activations/layer3_attention_weight_max": 107.49190521240234,
      "activations/layer3_attention_weight_min": -106.71417236328125,
      "activations/layer4_attention_weight_max": 112.21404266357422,
      "activations/layer4_attention_weight_min": -112.28399658203125,
      "activations/layer5_attention_weight_max": 54.618675231933594,
      "activations/layer5_attention_weight_min": -67.06804656982422,
      "activations/layer6_attention_weight_max": 45.79439163208008,
      "activations/layer6_attention_weight_min": -50.12379455566406,
      "activations/layer7_attention_weight_max": 99.73970794677734,
      "activations/layer7_attention_weight_min": -105.26207733154297,
      "activations/layer8_attention_weight_max": 43.80025863647461,
      "activations/layer8_attention_weight_min": -47.63644790649414,
      "activations/layer9_attention_weight_max": 33.6198844909668,
      "activations/layer9_attention_weight_min": -38.57552719116211,
      "epoch": 21.56,
      "learning_rate": 1.1075378787878787e-05,
      "loss": 2.7038,
      "step": 371050
    },
    {
      "activations/layer0_attention_weight_max": 16.694869995117188,
      "activations/layer0_attention_weight_min": -14.653478622436523,
      "activations/layer10_attention_weight_max": 39.426273345947266,
      "activations/layer10_attention_weight_min": -35.99525833129883,
      "activations/layer11_attention_weight_max": 37.561363220214844,
      "activations/layer11_attention_weight_min": -35.734657287597656,
      "activations/layer12_attention_weight_max": 29.553728103637695,
      "activations/layer12_attention_weight_min": -28.496522903442383,
      "activations/layer13_attention_weight_max": 43.19520568847656,
      "activations/layer13_attention_weight_min": -37.949398040771484,
      "activations/layer14_attention_weight_max": 45.63875198364258,
      "activations/layer14_attention_weight_min": -45.564552307128906,
      "activations/layer15_attention_weight_max": 42.462127685546875,
      "activations/layer15_attention_weight_min": -37.59834671020508,
      "activations/layer16_attention_weight_max": 31.970857620239258,
      "activations/layer16_attention_weight_min": -29.321937561035156,
      "activations/layer17_attention_weight_max": 32.22533416748047,
      "activations/layer17_attention_weight_min": -29.15781021118164,
      "activations/layer18_attention_weight_max": 35.978397369384766,
      "activations/layer18_attention_weight_min": -26.759496688842773,
      "activations/layer19_attention_weight_max": 36.69874954223633,
      "activations/layer19_attention_weight_min": -34.32453536987305,
      "activations/layer1_attention_weight_max": 15.409146308898926,
      "activations/layer1_attention_weight_min": -15.332246780395508,
      "activations/layer20_attention_weight_max": 31.61919403076172,
      "activations/layer20_attention_weight_min": -27.214866638183594,
      "activations/layer21_attention_weight_max": 30.744503021240234,
      "activations/layer21_attention_weight_min": -26.706218719482422,
      "activations/layer22_attention_weight_max": 49.09347152709961,
      "activations/layer22_attention_weight_min": -33.688140869140625,
      "activations/layer23_attention_weight_max": 35.149139404296875,
      "activations/layer23_attention_weight_min": -25.449941635131836,
      "activations/layer2_attention_weight_max": 34.75962829589844,
      "activations/layer2_attention_weight_min": -33.192813873291016,
      "activations/layer3_attention_weight_max": 112.08395385742188,
      "activations/layer3_attention_weight_min": -107.58917236328125,
      "activations/layer4_attention_weight_max": 118.29912567138672,
      "activations/layer4_attention_weight_min": -115.74444580078125,
      "activations/layer5_attention_weight_max": 51.62040328979492,
      "activations/layer5_attention_weight_min": -63.686256408691406,
      "activations/layer6_attention_weight_max": 46.39434051513672,
      "activations/layer6_attention_weight_min": -48.8615608215332,
      "activations/layer7_attention_weight_max": 99.53678131103516,
      "activations/layer7_attention_weight_min": -99.46565246582031,
      "activations/layer8_attention_weight_max": 43.233184814453125,
      "activations/layer8_attention_weight_min": -44.45856475830078,
      "activations/layer9_attention_weight_max": 34.63656234741211,
      "activations/layer9_attention_weight_min": -35.7564697265625,
      "epoch": 21.56,
      "learning_rate": 1.1056439393939392e-05,
      "loss": 2.6973,
      "step": 371100
    },
    {
      "activations/layer0_attention_weight_max": 16.367151260375977,
      "activations/layer0_attention_weight_min": -15.243209838867188,
      "activations/layer10_attention_weight_max": 35.290565490722656,
      "activations/layer10_attention_weight_min": -36.32708740234375,
      "activations/layer11_attention_weight_max": 35.10581970214844,
      "activations/layer11_attention_weight_min": -35.913787841796875,
      "activations/layer12_attention_weight_max": 26.477073669433594,
      "activations/layer12_attention_weight_min": -26.22054672241211,
      "activations/layer13_attention_weight_max": 38.454524993896484,
      "activations/layer13_attention_weight_min": -34.082855224609375,
      "activations/layer14_attention_weight_max": 39.418949127197266,
      "activations/layer14_attention_weight_min": -36.874420166015625,
      "activations/layer15_attention_weight_max": 37.51725387573242,
      "activations/layer15_attention_weight_min": -34.06607437133789,
      "activations/layer16_attention_weight_max": 29.97699546813965,
      "activations/layer16_attention_weight_min": -27.516626358032227,
      "activations/layer17_attention_weight_max": 28.399436950683594,
      "activations/layer17_attention_weight_min": -26.486574172973633,
      "activations/layer18_attention_weight_max": 32.0865592956543,
      "activations/layer18_attention_weight_min": -27.113887786865234,
      "activations/layer19_attention_weight_max": 33.083961486816406,
      "activations/layer19_attention_weight_min": -34.91621017456055,
      "activations/layer1_attention_weight_max": 16.012662887573242,
      "activations/layer1_attention_weight_min": -15.119975090026855,
      "activations/layer20_attention_weight_max": 28.065113067626953,
      "activations/layer20_attention_weight_min": -25.403980255126953,
      "activations/layer21_attention_weight_max": 28.394651412963867,
      "activations/layer21_attention_weight_min": -27.61067771911621,
      "activations/layer22_attention_weight_max": 47.21137237548828,
      "activations/layer22_attention_weight_min": -30.79505157470703,
      "activations/layer23_attention_weight_max": 30.840503692626953,
      "activations/layer23_attention_weight_min": -27.055204391479492,
      "activations/layer2_attention_weight_max": 33.53476333618164,
      "activations/layer2_attention_weight_min": -32.75752258300781,
      "activations/layer3_attention_weight_max": 105.52806091308594,
      "activations/layer3_attention_weight_min": -106.85481262207031,
      "activations/layer4_attention_weight_max": 107.80720520019531,
      "activations/layer4_attention_weight_min": -115.15704345703125,
      "activations/layer5_attention_weight_max": 52.21395492553711,
      "activations/layer5_attention_weight_min": -66.59619140625,
      "activations/layer6_attention_weight_max": 45.705665588378906,
      "activations/layer6_attention_weight_min": -47.84660339355469,
      "activations/layer7_attention_weight_max": 94.94220733642578,
      "activations/layer7_attention_weight_min": -104.05625915527344,
      "activations/layer8_attention_weight_max": 41.69991683959961,
      "activations/layer8_attention_weight_min": -45.84971237182617,
      "activations/layer9_attention_weight_max": 32.993316650390625,
      "activations/layer9_attention_weight_min": -35.83494186401367,
      "epoch": 21.57,
      "learning_rate": 1.1037499999999999e-05,
      "loss": 2.7018,
      "step": 371150
    },
    {
      "activations/layer0_attention_weight_max": 16.397109985351562,
      "activations/layer0_attention_weight_min": -14.442914009094238,
      "activations/layer10_attention_weight_max": 44.99834060668945,
      "activations/layer10_attention_weight_min": -41.188751220703125,
      "activations/layer11_attention_weight_max": 44.57115173339844,
      "activations/layer11_attention_weight_min": -44.79737091064453,
      "activations/layer12_attention_weight_max": 34.89242935180664,
      "activations/layer12_attention_weight_min": -31.28429412841797,
      "activations/layer13_attention_weight_max": 57.064056396484375,
      "activations/layer13_attention_weight_min": -42.6099853515625,
      "activations/layer14_attention_weight_max": 61.949440002441406,
      "activations/layer14_attention_weight_min": -46.377899169921875,
      "activations/layer15_attention_weight_max": 59.33388900756836,
      "activations/layer15_attention_weight_min": -48.9820671081543,
      "activations/layer16_attention_weight_max": 43.159175872802734,
      "activations/layer16_attention_weight_min": -30.200077056884766,
      "activations/layer17_attention_weight_max": 49.410587310791016,
      "activations/layer17_attention_weight_min": -29.983978271484375,
      "activations/layer18_attention_weight_max": 39.315696716308594,
      "activations/layer18_attention_weight_min": -28.895971298217773,
      "activations/layer19_attention_weight_max": 41.87949752807617,
      "activations/layer19_attention_weight_min": -31.964115142822266,
      "activations/layer1_attention_weight_max": 16.336509704589844,
      "activations/layer1_attention_weight_min": -15.08363151550293,
      "activations/layer20_attention_weight_max": 35.56800079345703,
      "activations/layer20_attention_weight_min": -25.512256622314453,
      "activations/layer21_attention_weight_max": 33.45575714111328,
      "activations/layer21_attention_weight_min": -25.661144256591797,
      "activations/layer22_attention_weight_max": 57.982757568359375,
      "activations/layer22_attention_weight_min": -35.22240447998047,
      "activations/layer23_attention_weight_max": 37.93924331665039,
      "activations/layer23_attention_weight_min": -25.646345138549805,
      "activations/layer2_attention_weight_max": 35.65325164794922,
      "activations/layer2_attention_weight_min": -35.577789306640625,
      "activations/layer3_attention_weight_max": 109.03987121582031,
      "activations/layer3_attention_weight_min": -112.01087188720703,
      "activations/layer4_attention_weight_max": 106.71268463134766,
      "activations/layer4_attention_weight_min": -113.97411346435547,
      "activations/layer5_attention_weight_max": 49.97707748413086,
      "activations/layer5_attention_weight_min": -65.39385986328125,
      "activations/layer6_attention_weight_max": 47.38750457763672,
      "activations/layer6_attention_weight_min": -49.02700424194336,
      "activations/layer7_attention_weight_max": 107.58259582519531,
      "activations/layer7_attention_weight_min": -104.0824203491211,
      "activations/layer8_attention_weight_max": 50.42918395996094,
      "activations/layer8_attention_weight_min": -50.06364440917969,
      "activations/layer9_attention_weight_max": 38.474822998046875,
      "activations/layer9_attention_weight_min": -41.56184005737305,
      "epoch": 21.57,
      "learning_rate": 1.1018560606060606e-05,
      "loss": 2.6908,
      "step": 371200
    },
    {
      "activations/layer0_attention_weight_max": 16.718486785888672,
      "activations/layer0_attention_weight_min": -16.148828506469727,
      "activations/layer10_attention_weight_max": 36.84474182128906,
      "activations/layer10_attention_weight_min": -38.64103317260742,
      "activations/layer11_attention_weight_max": 34.67807388305664,
      "activations/layer11_attention_weight_min": -39.02336120605469,
      "activations/layer12_attention_weight_max": 27.888809204101562,
      "activations/layer12_attention_weight_min": -28.533998489379883,
      "activations/layer13_attention_weight_max": 41.28426742553711,
      "activations/layer13_attention_weight_min": -37.396114349365234,
      "activations/layer14_attention_weight_max": 44.99774169921875,
      "activations/layer14_attention_weight_min": -38.86408996582031,
      "activations/layer15_attention_weight_max": 41.16315841674805,
      "activations/layer15_attention_weight_min": -37.48160934448242,
      "activations/layer16_attention_weight_max": 28.836618423461914,
      "activations/layer16_attention_weight_min": -27.44330596923828,
      "activations/layer17_attention_weight_max": 31.384321212768555,
      "activations/layer17_attention_weight_min": -26.427528381347656,
      "activations/layer18_attention_weight_max": 33.04043197631836,
      "activations/layer18_attention_weight_min": -27.6352481842041,
      "activations/layer19_attention_weight_max": 33.60345458984375,
      "activations/layer19_attention_weight_min": -30.972991943359375,
      "activations/layer1_attention_weight_max": 16.04696273803711,
      "activations/layer1_attention_weight_min": -13.964723587036133,
      "activations/layer20_attention_weight_max": 28.865034103393555,
      "activations/layer20_attention_weight_min": -25.688579559326172,
      "activations/layer21_attention_weight_max": 28.185081481933594,
      "activations/layer21_attention_weight_min": -24.64311408996582,
      "activations/layer22_attention_weight_max": 42.17594528198242,
      "activations/layer22_attention_weight_min": -32.392757415771484,
      "activations/layer23_attention_weight_max": 30.654560089111328,
      "activations/layer23_attention_weight_min": -24.94831085205078,
      "activations/layer2_attention_weight_max": 34.35298156738281,
      "activations/layer2_attention_weight_min": -34.610084533691406,
      "activations/layer3_attention_weight_max": 106.89385223388672,
      "activations/layer3_attention_weight_min": -110.65203094482422,
      "activations/layer4_attention_weight_max": 114.62564849853516,
      "activations/layer4_attention_weight_min": -116.8467788696289,
      "activations/layer5_attention_weight_max": 52.59255599975586,
      "activations/layer5_attention_weight_min": -61.48674774169922,
      "activations/layer6_attention_weight_max": 45.540401458740234,
      "activations/layer6_attention_weight_min": -49.12541580200195,
      "activations/layer7_attention_weight_max": 94.1030044555664,
      "activations/layer7_attention_weight_min": -104.5723876953125,
      "activations/layer8_attention_weight_max": 44.045440673828125,
      "activations/layer8_attention_weight_min": -47.39632034301758,
      "activations/layer9_attention_weight_max": 33.98042297363281,
      "activations/layer9_attention_weight_min": -36.66862106323242,
      "epoch": 21.57,
      "learning_rate": 1.099962121212121e-05,
      "loss": 2.7045,
      "step": 371250
    },
    {
      "activations/layer0_attention_weight_max": 17.03013801574707,
      "activations/layer0_attention_weight_min": -13.463752746582031,
      "activations/layer10_attention_weight_max": 35.70127487182617,
      "activations/layer10_attention_weight_min": -36.23661422729492,
      "activations/layer11_attention_weight_max": 34.75583267211914,
      "activations/layer11_attention_weight_min": -35.22672653198242,
      "activations/layer12_attention_weight_max": 29.991121292114258,
      "activations/layer12_attention_weight_min": -27.35603141784668,
      "activations/layer13_attention_weight_max": 46.0839958190918,
      "activations/layer13_attention_weight_min": -37.31215286254883,
      "activations/layer14_attention_weight_max": 50.751708984375,
      "activations/layer14_attention_weight_min": -42.271705627441406,
      "activations/layer15_attention_weight_max": 47.183319091796875,
      "activations/layer15_attention_weight_min": -39.17129898071289,
      "activations/layer16_attention_weight_max": 34.494937896728516,
      "activations/layer16_attention_weight_min": -28.769506454467773,
      "activations/layer17_attention_weight_max": 32.1243896484375,
      "activations/layer17_attention_weight_min": -26.94759750366211,
      "activations/layer18_attention_weight_max": 39.388675689697266,
      "activations/layer18_attention_weight_min": -26.607410430908203,
      "activations/layer19_attention_weight_max": 39.17996597290039,
      "activations/layer19_attention_weight_min": -31.88058853149414,
      "activations/layer1_attention_weight_max": 15.514122009277344,
      "activations/layer1_attention_weight_min": -14.796008110046387,
      "activations/layer20_attention_weight_max": 36.5229377746582,
      "activations/layer20_attention_weight_min": -24.796079635620117,
      "activations/layer21_attention_weight_max": 35.9781379699707,
      "activations/layer21_attention_weight_min": -24.548662185668945,
      "activations/layer22_attention_weight_max": 47.56971740722656,
      "activations/layer22_attention_weight_min": -33.32423400878906,
      "activations/layer23_attention_weight_max": 38.12387466430664,
      "activations/layer23_attention_weight_min": -27.032352447509766,
      "activations/layer2_attention_weight_max": 33.40342712402344,
      "activations/layer2_attention_weight_min": -32.96782684326172,
      "activations/layer3_attention_weight_max": 104.6685562133789,
      "activations/layer3_attention_weight_min": -108.0687026977539,
      "activations/layer4_attention_weight_max": 106.84844970703125,
      "activations/layer4_attention_weight_min": -115.80580139160156,
      "activations/layer5_attention_weight_max": 52.17259979248047,
      "activations/layer5_attention_weight_min": -61.555110931396484,
      "activations/layer6_attention_weight_max": 45.097984313964844,
      "activations/layer6_attention_weight_min": -48.706199645996094,
      "activations/layer7_attention_weight_max": 96.29728698730469,
      "activations/layer7_attention_weight_min": -109.39197540283203,
      "activations/layer8_attention_weight_max": 43.13017272949219,
      "activations/layer8_attention_weight_min": -48.089874267578125,
      "activations/layer9_attention_weight_max": 33.750946044921875,
      "activations/layer9_attention_weight_min": -36.479610443115234,
      "epoch": 21.57,
      "learning_rate": 1.0980681818181817e-05,
      "loss": 2.7162,
      "step": 371300
    },
    {
      "activations/layer0_attention_weight_max": 16.843034744262695,
      "activations/layer0_attention_weight_min": -13.577433586120605,
      "activations/layer10_attention_weight_max": 35.484073638916016,
      "activations/layer10_attention_weight_min": -36.56221389770508,
      "activations/layer11_attention_weight_max": 34.7472038269043,
      "activations/layer11_attention_weight_min": -37.61244201660156,
      "activations/layer12_attention_weight_max": 27.310834884643555,
      "activations/layer12_attention_weight_min": -28.21988868713379,
      "activations/layer13_attention_weight_max": 41.43962097167969,
      "activations/layer13_attention_weight_min": -40.07161331176758,
      "activations/layer14_attention_weight_max": 45.921417236328125,
      "activations/layer14_attention_weight_min": -40.917606353759766,
      "activations/layer15_attention_weight_max": 41.20077133178711,
      "activations/layer15_attention_weight_min": -41.420249938964844,
      "activations/layer16_attention_weight_max": 30.743343353271484,
      "activations/layer16_attention_weight_min": -28.923627853393555,
      "activations/layer17_attention_weight_max": 30.33069610595703,
      "activations/layer17_attention_weight_min": -28.756046295166016,
      "activations/layer18_attention_weight_max": 32.87449264526367,
      "activations/layer18_attention_weight_min": -27.942302703857422,
      "activations/layer19_attention_weight_max": 33.63962936401367,
      "activations/layer19_attention_weight_min": -30.9141788482666,
      "activations/layer1_attention_weight_max": 14.947896957397461,
      "activations/layer1_attention_weight_min": -15.156746864318848,
      "activations/layer20_attention_weight_max": 27.124866485595703,
      "activations/layer20_attention_weight_min": -24.714675903320312,
      "activations/layer21_attention_weight_max": 29.04384422302246,
      "activations/layer21_attention_weight_min": -24.360692977905273,
      "activations/layer22_attention_weight_max": 42.25684356689453,
      "activations/layer22_attention_weight_min": -31.6258544921875,
      "activations/layer23_attention_weight_max": 32.53580093383789,
      "activations/layer23_attention_weight_min": -28.167158126831055,
      "activations/layer2_attention_weight_max": 34.58990478515625,
      "activations/layer2_attention_weight_min": -34.47035217285156,
      "activations/layer3_attention_weight_max": 108.97013854980469,
      "activations/layer3_attention_weight_min": -113.76570892333984,
      "activations/layer4_attention_weight_max": 111.5708236694336,
      "activations/layer4_attention_weight_min": -113.7880859375,
      "activations/layer5_attention_weight_max": 51.13243865966797,
      "activations/layer5_attention_weight_min": -62.73710632324219,
      "activations/layer6_attention_weight_max": 47.899173736572266,
      "activations/layer6_attention_weight_min": -51.45191192626953,
      "activations/layer7_attention_weight_max": 99.05254364013672,
      "activations/layer7_attention_weight_min": -103.7039566040039,
      "activations/layer8_attention_weight_max": 44.381317138671875,
      "activations/layer8_attention_weight_min": -45.82628631591797,
      "activations/layer9_attention_weight_max": 32.765357971191406,
      "activations/layer9_attention_weight_min": -36.647064208984375,
      "epoch": 21.58,
      "learning_rate": 1.0961742424242424e-05,
      "loss": 2.6995,
      "step": 371350
    },
    {
      "activations/layer0_attention_weight_max": 16.206043243408203,
      "activations/layer0_attention_weight_min": -14.05382251739502,
      "activations/layer10_attention_weight_max": 37.015323638916016,
      "activations/layer10_attention_weight_min": -36.3847541809082,
      "activations/layer11_attention_weight_max": 35.77873229980469,
      "activations/layer11_attention_weight_min": -36.24326705932617,
      "activations/layer12_attention_weight_max": 29.683141708374023,
      "activations/layer12_attention_weight_min": -26.554903030395508,
      "activations/layer13_attention_weight_max": 40.72939682006836,
      "activations/layer13_attention_weight_min": -35.80750274658203,
      "activations/layer14_attention_weight_max": 43.32634353637695,
      "activations/layer14_attention_weight_min": -36.772926330566406,
      "activations/layer15_attention_weight_max": 38.9004020690918,
      "activations/layer15_attention_weight_min": -36.1690673828125,
      "activations/layer16_attention_weight_max": 29.113971710205078,
      "activations/layer16_attention_weight_min": -29.302574157714844,
      "activations/layer17_attention_weight_max": 28.11590003967285,
      "activations/layer17_attention_weight_min": -27.30792999267578,
      "activations/layer18_attention_weight_max": 31.431419372558594,
      "activations/layer18_attention_weight_min": -28.885047912597656,
      "activations/layer19_attention_weight_max": 33.66400909423828,
      "activations/layer19_attention_weight_min": -32.80937194824219,
      "activations/layer1_attention_weight_max": 15.635607719421387,
      "activations/layer1_attention_weight_min": -15.405571937561035,
      "activations/layer20_attention_weight_max": 27.9925537109375,
      "activations/layer20_attention_weight_min": -25.71593475341797,
      "activations/layer21_attention_weight_max": 28.526113510131836,
      "activations/layer21_attention_weight_min": -26.992124557495117,
      "activations/layer22_attention_weight_max": 46.15237808227539,
      "activations/layer22_attention_weight_min": -33.19314956665039,
      "activations/layer23_attention_weight_max": 32.76897430419922,
      "activations/layer23_attention_weight_min": -26.599454879760742,
      "activations/layer2_attention_weight_max": 33.23372268676758,
      "activations/layer2_attention_weight_min": -33.18444061279297,
      "activations/layer3_attention_weight_max": 107.02210998535156,
      "activations/layer3_attention_weight_min": -104.8099136352539,
      "activations/layer4_attention_weight_max": 107.71351623535156,
      "activations/layer4_attention_weight_min": -116.01253509521484,
      "activations/layer5_attention_weight_max": 50.4810905456543,
      "activations/layer5_attention_weight_min": -62.56071472167969,
      "activations/layer6_attention_weight_max": 46.770450592041016,
      "activations/layer6_attention_weight_min": -49.62129592895508,
      "activations/layer7_attention_weight_max": 99.20038604736328,
      "activations/layer7_attention_weight_min": -101.7716293334961,
      "activations/layer8_attention_weight_max": 44.98687744140625,
      "activations/layer8_attention_weight_min": -46.54570770263672,
      "activations/layer9_attention_weight_max": 35.02970504760742,
      "activations/layer9_attention_weight_min": -36.346309661865234,
      "epoch": 21.58,
      "learning_rate": 1.0942803030303028e-05,
      "loss": 2.7106,
      "step": 371400
    },
    {
      "activations/layer0_attention_weight_max": 16.75600814819336,
      "activations/layer0_attention_weight_min": -14.776867866516113,
      "activations/layer10_attention_weight_max": 34.3222770690918,
      "activations/layer10_attention_weight_min": -35.00232696533203,
      "activations/layer11_attention_weight_max": 35.54947280883789,
      "activations/layer11_attention_weight_min": -35.92393493652344,
      "activations/layer12_attention_weight_max": 25.7242488861084,
      "activations/layer12_attention_weight_min": -27.710771560668945,
      "activations/layer13_attention_weight_max": 40.06022644042969,
      "activations/layer13_attention_weight_min": -36.17280197143555,
      "activations/layer14_attention_weight_max": 41.14092254638672,
      "activations/layer14_attention_weight_min": -38.55070114135742,
      "activations/layer15_attention_weight_max": 38.95742416381836,
      "activations/layer15_attention_weight_min": -36.9697265625,
      "activations/layer16_attention_weight_max": 29.774187088012695,
      "activations/layer16_attention_weight_min": -29.98521614074707,
      "activations/layer17_attention_weight_max": 30.363502502441406,
      "activations/layer17_attention_weight_min": -27.931488037109375,
      "activations/layer18_attention_weight_max": 32.59528732299805,
      "activations/layer18_attention_weight_min": -27.24537467956543,
      "activations/layer19_attention_weight_max": 32.41812515258789,
      "activations/layer19_attention_weight_min": -31.600095748901367,
      "activations/layer1_attention_weight_max": 15.523419380187988,
      "activations/layer1_attention_weight_min": -14.723642349243164,
      "activations/layer20_attention_weight_max": 26.069307327270508,
      "activations/layer20_attention_weight_min": -25.328100204467773,
      "activations/layer21_attention_weight_max": 28.66771697998047,
      "activations/layer21_attention_weight_min": -25.333053588867188,
      "activations/layer22_attention_weight_max": 40.04129409790039,
      "activations/layer22_attention_weight_min": -30.928735733032227,
      "activations/layer23_attention_weight_max": 30.920318603515625,
      "activations/layer23_attention_weight_min": -26.069616317749023,
      "activations/layer2_attention_weight_max": 33.22710418701172,
      "activations/layer2_attention_weight_min": -33.37199783325195,
      "activations/layer3_attention_weight_max": 108.6545181274414,
      "activations/layer3_attention_weight_min": -110.53507232666016,
      "activations/layer4_attention_weight_max": 110.13740539550781,
      "activations/layer4_attention_weight_min": -109.51997375488281,
      "activations/layer5_attention_weight_max": 48.85590362548828,
      "activations/layer5_attention_weight_min": -60.12820053100586,
      "activations/layer6_attention_weight_max": 44.24741744995117,
      "activations/layer6_attention_weight_min": -46.644344329833984,
      "activations/layer7_attention_weight_max": 93.2967529296875,
      "activations/layer7_attention_weight_min": -94.83951568603516,
      "activations/layer8_attention_weight_max": 43.388671875,
      "activations/layer8_attention_weight_min": -44.769248962402344,
      "activations/layer9_attention_weight_max": 32.8018913269043,
      "activations/layer9_attention_weight_min": -34.978031158447266,
      "epoch": 21.58,
      "learning_rate": 1.0923863636363635e-05,
      "loss": 2.7175,
      "step": 371450
    },
    {
      "activations/layer0_attention_weight_max": 16.237295150756836,
      "activations/layer0_attention_weight_min": -13.406352996826172,
      "activations/layer10_attention_weight_max": 38.559783935546875,
      "activations/layer10_attention_weight_min": -39.85240936279297,
      "activations/layer11_attention_weight_max": 39.95314025878906,
      "activations/layer11_attention_weight_min": -41.160919189453125,
      "activations/layer12_attention_weight_max": 29.739835739135742,
      "activations/layer12_attention_weight_min": -29.21409797668457,
      "activations/layer13_attention_weight_max": 44.54079055786133,
      "activations/layer13_attention_weight_min": -39.07829666137695,
      "activations/layer14_attention_weight_max": 49.1068229675293,
      "activations/layer14_attention_weight_min": -43.28553771972656,
      "activations/layer15_attention_weight_max": 47.98844528198242,
      "activations/layer15_attention_weight_min": -39.359596252441406,
      "activations/layer16_attention_weight_max": 32.17576599121094,
      "activations/layer16_attention_weight_min": -29.80484390258789,
      "activations/layer17_attention_weight_max": 33.270423889160156,
      "activations/layer17_attention_weight_min": -26.490564346313477,
      "activations/layer18_attention_weight_max": 33.234703063964844,
      "activations/layer18_attention_weight_min": -24.37557029724121,
      "activations/layer19_attention_weight_max": 41.57828903198242,
      "activations/layer19_attention_weight_min": -32.16309356689453,
      "activations/layer1_attention_weight_max": 15.248016357421875,
      "activations/layer1_attention_weight_min": -15.475829124450684,
      "activations/layer20_attention_weight_max": 36.0076904296875,
      "activations/layer20_attention_weight_min": -24.35299301147461,
      "activations/layer21_attention_weight_max": 35.07453155517578,
      "activations/layer21_attention_weight_min": -24.20078468322754,
      "activations/layer22_attention_weight_max": 47.76590347290039,
      "activations/layer22_attention_weight_min": -30.07468605041504,
      "activations/layer23_attention_weight_max": 35.626163482666016,
      "activations/layer23_attention_weight_min": -25.576505661010742,
      "activations/layer2_attention_weight_max": 34.214324951171875,
      "activations/layer2_attention_weight_min": -33.31172561645508,
      "activations/layer3_attention_weight_max": 106.24201965332031,
      "activations/layer3_attention_weight_min": -106.7640151977539,
      "activations/layer4_attention_weight_max": 111.25545501708984,
      "activations/layer4_attention_weight_min": -113.58647918701172,
      "activations/layer5_attention_weight_max": 55.89714813232422,
      "activations/layer5_attention_weight_min": -65.68692779541016,
      "activations/layer6_attention_weight_max": 47.22206497192383,
      "activations/layer6_attention_weight_min": -49.21162033081055,
      "activations/layer7_attention_weight_max": 102.67647552490234,
      "activations/layer7_attention_weight_min": -101.9662094116211,
      "activations/layer8_attention_weight_max": 47.0885009765625,
      "activations/layer8_attention_weight_min": -48.16549301147461,
      "activations/layer9_attention_weight_max": 38.368858337402344,
      "activations/layer9_attention_weight_min": -40.60710525512695,
      "epoch": 21.59,
      "learning_rate": 1.090492424242424e-05,
      "loss": 2.7089,
      "step": 371500
    },
    {
      "activations/layer0_attention_weight_max": 16.70313262939453,
      "activations/layer0_attention_weight_min": -14.673795700073242,
      "activations/layer10_attention_weight_max": 36.12271499633789,
      "activations/layer10_attention_weight_min": -37.655723571777344,
      "activations/layer11_attention_weight_max": 35.79104995727539,
      "activations/layer11_attention_weight_min": -37.69843673706055,
      "activations/layer12_attention_weight_max": 26.52425765991211,
      "activations/layer12_attention_weight_min": -29.18386459350586,
      "activations/layer13_attention_weight_max": 41.22920608520508,
      "activations/layer13_attention_weight_min": -37.872840881347656,
      "activations/layer14_attention_weight_max": 41.49462890625,
      "activations/layer14_attention_weight_min": -37.665462493896484,
      "activations/layer15_attention_weight_max": 38.77332305908203,
      "activations/layer15_attention_weight_min": -35.99563217163086,
      "activations/layer16_attention_weight_max": 30.55546760559082,
      "activations/layer16_attention_weight_min": -27.04304313659668,
      "activations/layer17_attention_weight_max": 32.74716567993164,
      "activations/layer17_attention_weight_min": -25.54976463317871,
      "activations/layer18_attention_weight_max": 34.10105514526367,
      "activations/layer18_attention_weight_min": -25.252155303955078,
      "activations/layer19_attention_weight_max": 35.45088195800781,
      "activations/layer19_attention_weight_min": -31.24106788635254,
      "activations/layer1_attention_weight_max": 15.074856758117676,
      "activations/layer1_attention_weight_min": -14.284615516662598,
      "activations/layer20_attention_weight_max": 30.581846237182617,
      "activations/layer20_attention_weight_min": -23.709613800048828,
      "activations/layer21_attention_weight_max": 28.997066497802734,
      "activations/layer21_attention_weight_min": -23.917078018188477,
      "activations/layer22_attention_weight_max": 41.56843185424805,
      "activations/layer22_attention_weight_min": -30.25634765625,
      "activations/layer23_attention_weight_max": 32.35129165649414,
      "activations/layer23_attention_weight_min": -23.825851440429688,
      "activations/layer2_attention_weight_max": 34.992576599121094,
      "activations/layer2_attention_weight_min": -33.83916473388672,
      "activations/layer3_attention_weight_max": 109.08696746826172,
      "activations/layer3_attention_weight_min": -106.16072845458984,
      "activations/layer4_attention_weight_max": 113.5445556640625,
      "activations/layer4_attention_weight_min": -112.84425354003906,
      "activations/layer5_attention_weight_max": 50.53707504272461,
      "activations/layer5_attention_weight_min": -61.25634002685547,
      "activations/layer6_attention_weight_max": 46.70595169067383,
      "activations/layer6_attention_weight_min": -48.605445861816406,
      "activations/layer7_attention_weight_max": 100.10812377929688,
      "activations/layer7_attention_weight_min": -100.70166778564453,
      "activations/layer8_attention_weight_max": 44.644683837890625,
      "activations/layer8_attention_weight_min": -48.27681350708008,
      "activations/layer9_attention_weight_max": 35.750736236572266,
      "activations/layer9_attention_weight_min": -39.501407623291016,
      "epoch": 21.59,
      "learning_rate": 1.0885984848484846e-05,
      "loss": 2.7013,
      "step": 371550
    },
    {
      "activations/layer0_attention_weight_max": 16.544132232666016,
      "activations/layer0_attention_weight_min": -14.410056114196777,
      "activations/layer10_attention_weight_max": 33.15369415283203,
      "activations/layer10_attention_weight_min": -33.530029296875,
      "activations/layer11_attention_weight_max": 33.86985778808594,
      "activations/layer11_attention_weight_min": -34.364864349365234,
      "activations/layer12_attention_weight_max": 26.20673942565918,
      "activations/layer12_attention_weight_min": -25.732803344726562,
      "activations/layer13_attention_weight_max": 38.31990051269531,
      "activations/layer13_attention_weight_min": -35.12004089355469,
      "activations/layer14_attention_weight_max": 41.392154693603516,
      "activations/layer14_attention_weight_min": -38.438961029052734,
      "activations/layer15_attention_weight_max": 38.573726654052734,
      "activations/layer15_attention_weight_min": -39.13829803466797,
      "activations/layer16_attention_weight_max": 30.25592041015625,
      "activations/layer16_attention_weight_min": -28.990793228149414,
      "activations/layer17_attention_weight_max": 29.243520736694336,
      "activations/layer17_attention_weight_min": -24.878681182861328,
      "activations/layer18_attention_weight_max": 31.957345962524414,
      "activations/layer18_attention_weight_min": -25.065311431884766,
      "activations/layer19_attention_weight_max": 32.40348434448242,
      "activations/layer19_attention_weight_min": -30.489704132080078,
      "activations/layer1_attention_weight_max": 15.82833480834961,
      "activations/layer1_attention_weight_min": -14.892768859863281,
      "activations/layer20_attention_weight_max": 28.586824417114258,
      "activations/layer20_attention_weight_min": -24.381507873535156,
      "activations/layer21_attention_weight_max": 29.236900329589844,
      "activations/layer21_attention_weight_min": -24.280736923217773,
      "activations/layer22_attention_weight_max": 39.148746490478516,
      "activations/layer22_attention_weight_min": -32.38833236694336,
      "activations/layer23_attention_weight_max": 31.358875274658203,
      "activations/layer23_attention_weight_min": -26.28394889831543,
      "activations/layer2_attention_weight_max": 31.993358612060547,
      "activations/layer2_attention_weight_min": -31.764760971069336,
      "activations/layer3_attention_weight_max": 102.82987213134766,
      "activations/layer3_attention_weight_min": -101.75273132324219,
      "activations/layer4_attention_weight_max": 103.84065246582031,
      "activations/layer4_attention_weight_min": -109.99867248535156,
      "activations/layer5_attention_weight_max": 49.24351501464844,
      "activations/layer5_attention_weight_min": -62.56450653076172,
      "activations/layer6_attention_weight_max": 43.29366683959961,
      "activations/layer6_attention_weight_min": -45.43179702758789,
      "activations/layer7_attention_weight_max": 93.66726684570312,
      "activations/layer7_attention_weight_min": -96.32538604736328,
      "activations/layer8_attention_weight_max": 41.78905487060547,
      "activations/layer8_attention_weight_min": -43.43098831176758,
      "activations/layer9_attention_weight_max": 31.8184814453125,
      "activations/layer9_attention_weight_min": -34.1167106628418,
      "epoch": 21.59,
      "learning_rate": 1.0867045454545453e-05,
      "loss": 2.7237,
      "step": 371600
    },
    {
      "activations/layer0_attention_weight_max": 16.137725830078125,
      "activations/layer0_attention_weight_min": -13.222930908203125,
      "activations/layer10_attention_weight_max": 35.162147521972656,
      "activations/layer10_attention_weight_min": -37.05787658691406,
      "activations/layer11_attention_weight_max": 36.56777572631836,
      "activations/layer11_attention_weight_min": -36.072513580322266,
      "activations/layer12_attention_weight_max": 26.71035385131836,
      "activations/layer12_attention_weight_min": -27.37467384338379,
      "activations/layer13_attention_weight_max": 39.97721862792969,
      "activations/layer13_attention_weight_min": -36.89952087402344,
      "activations/layer14_attention_weight_max": 44.01184844970703,
      "activations/layer14_attention_weight_min": -37.16731262207031,
      "activations/layer15_attention_weight_max": 37.880714416503906,
      "activations/layer15_attention_weight_min": -36.26015090942383,
      "activations/layer16_attention_weight_max": 29.81428337097168,
      "activations/layer16_attention_weight_min": -28.433513641357422,
      "activations/layer17_attention_weight_max": 28.851444244384766,
      "activations/layer17_attention_weight_min": -27.495464324951172,
      "activations/layer18_attention_weight_max": 31.296504974365234,
      "activations/layer18_attention_weight_min": -27.886249542236328,
      "activations/layer19_attention_weight_max": 30.925323486328125,
      "activations/layer19_attention_weight_min": -35.236473083496094,
      "activations/layer1_attention_weight_max": 17.675077438354492,
      "activations/layer1_attention_weight_min": -17.06731605529785,
      "activations/layer20_attention_weight_max": 28.903120040893555,
      "activations/layer20_attention_weight_min": -25.231050491333008,
      "activations/layer21_attention_weight_max": 28.613750457763672,
      "activations/layer21_attention_weight_min": -24.389854431152344,
      "activations/layer22_attention_weight_max": 40.727088928222656,
      "activations/layer22_attention_weight_min": -32.37440490722656,
      "activations/layer23_attention_weight_max": 31.254125595092773,
      "activations/layer23_attention_weight_min": -27.39295768737793,
      "activations/layer2_attention_weight_max": 33.838375091552734,
      "activations/layer2_attention_weight_min": -33.74637985229492,
      "activations/layer3_attention_weight_max": 106.36463928222656,
      "activations/layer3_attention_weight_min": -106.30813598632812,
      "activations/layer4_attention_weight_max": 110.4209213256836,
      "activations/layer4_attention_weight_min": -114.49006652832031,
      "activations/layer5_attention_weight_max": 51.851654052734375,
      "activations/layer5_attention_weight_min": -61.51752853393555,
      "activations/layer6_attention_weight_max": 46.38365173339844,
      "activations/layer6_attention_weight_min": -48.3643913269043,
      "activations/layer7_attention_weight_max": 99.8537368774414,
      "activations/layer7_attention_weight_min": -101.77633666992188,
      "activations/layer8_attention_weight_max": 42.56033706665039,
      "activations/layer8_attention_weight_min": -46.263126373291016,
      "activations/layer9_attention_weight_max": 32.055320739746094,
      "activations/layer9_attention_weight_min": -36.00619125366211,
      "epoch": 21.59,
      "learning_rate": 1.0848106060606061e-05,
      "loss": 2.7001,
      "step": 371650
    },
    {
      "activations/layer0_attention_weight_max": 16.999507904052734,
      "activations/layer0_attention_weight_min": -13.811866760253906,
      "activations/layer10_attention_weight_max": 39.563812255859375,
      "activations/layer10_attention_weight_min": -37.738277435302734,
      "activations/layer11_attention_weight_max": 38.127952575683594,
      "activations/layer11_attention_weight_min": -36.78768539428711,
      "activations/layer12_attention_weight_max": 29.17889976501465,
      "activations/layer12_attention_weight_min": -29.628381729125977,
      "activations/layer13_attention_weight_max": 44.58614730834961,
      "activations/layer13_attention_weight_min": -37.51373291015625,
      "activations/layer14_attention_weight_max": 45.595863342285156,
      "activations/layer14_attention_weight_min": -40.40188980102539,
      "activations/layer15_attention_weight_max": 43.097171783447266,
      "activations/layer15_attention_weight_min": -38.017539978027344,
      "activations/layer16_attention_weight_max": 30.473567962646484,
      "activations/layer16_attention_weight_min": -28.893980026245117,
      "activations/layer17_attention_weight_max": 31.41579818725586,
      "activations/layer17_attention_weight_min": -27.229902267456055,
      "activations/layer18_attention_weight_max": 34.590843200683594,
      "activations/layer18_attention_weight_min": -28.192787170410156,
      "activations/layer19_attention_weight_max": 35.18960189819336,
      "activations/layer19_attention_weight_min": -35.30178451538086,
      "activations/layer1_attention_weight_max": 15.859848976135254,
      "activations/layer1_attention_weight_min": -13.521069526672363,
      "activations/layer20_attention_weight_max": 30.148418426513672,
      "activations/layer20_attention_weight_min": -28.121326446533203,
      "activations/layer21_attention_weight_max": 28.53094482421875,
      "activations/layer21_attention_weight_min": -25.57329750061035,
      "activations/layer22_attention_weight_max": 43.783145904541016,
      "activations/layer22_attention_weight_min": -36.21877670288086,
      "activations/layer23_attention_weight_max": 34.765289306640625,
      "activations/layer23_attention_weight_min": -28.31575584411621,
      "activations/layer2_attention_weight_max": 32.36252212524414,
      "activations/layer2_attention_weight_min": -32.038265228271484,
      "activations/layer3_attention_weight_max": 101.10490417480469,
      "activations/layer3_attention_weight_min": -101.92189025878906,
      "activations/layer4_attention_weight_max": 108.73189544677734,
      "activations/layer4_attention_weight_min": -113.88973236083984,
      "activations/layer5_attention_weight_max": 51.63165283203125,
      "activations/layer5_attention_weight_min": -60.802703857421875,
      "activations/layer6_attention_weight_max": 48.11334991455078,
      "activations/layer6_attention_weight_min": -49.328670501708984,
      "activations/layer7_attention_weight_max": 106.01038360595703,
      "activations/layer7_attention_weight_min": -105.60393524169922,
      "activations/layer8_attention_weight_max": 44.35017395019531,
      "activations/layer8_attention_weight_min": -46.7093505859375,
      "activations/layer9_attention_weight_max": 34.44343185424805,
      "activations/layer9_attention_weight_min": -37.884117126464844,
      "epoch": 21.6,
      "learning_rate": 1.0829166666666666e-05,
      "loss": 2.7153,
      "step": 371700
    },
    {
      "activations/layer0_attention_weight_max": 16.63214111328125,
      "activations/layer0_attention_weight_min": -13.274140357971191,
      "activations/layer10_attention_weight_max": 37.252540588378906,
      "activations/layer10_attention_weight_min": -37.07362747192383,
      "activations/layer11_attention_weight_max": 39.82139205932617,
      "activations/layer11_attention_weight_min": -35.72910690307617,
      "activations/layer12_attention_weight_max": 28.155536651611328,
      "activations/layer12_attention_weight_min": -27.473722457885742,
      "activations/layer13_attention_weight_max": 42.65642547607422,
      "activations/layer13_attention_weight_min": -37.33999252319336,
      "activations/layer14_attention_weight_max": 44.3094482421875,
      "activations/layer14_attention_weight_min": -39.89088439941406,
      "activations/layer15_attention_weight_max": 42.45509719848633,
      "activations/layer15_attention_weight_min": -38.112892150878906,
      "activations/layer16_attention_weight_max": 33.34150695800781,
      "activations/layer16_attention_weight_min": -28.586849212646484,
      "activations/layer17_attention_weight_max": 31.01384162902832,
      "activations/layer17_attention_weight_min": -27.35355567932129,
      "activations/layer18_attention_weight_max": 33.7539176940918,
      "activations/layer18_attention_weight_min": -27.614957809448242,
      "activations/layer19_attention_weight_max": 34.2129020690918,
      "activations/layer19_attention_weight_min": -31.471113204956055,
      "activations/layer1_attention_weight_max": 15.646402359008789,
      "activations/layer1_attention_weight_min": -15.038095474243164,
      "activations/layer20_attention_weight_max": 31.134443283081055,
      "activations/layer20_attention_weight_min": -26.790620803833008,
      "activations/layer21_attention_weight_max": 29.218889236450195,
      "activations/layer21_attention_weight_min": -26.336462020874023,
      "activations/layer22_attention_weight_max": 44.275047302246094,
      "activations/layer22_attention_weight_min": -32.85147476196289,
      "activations/layer23_attention_weight_max": 32.883750915527344,
      "activations/layer23_attention_weight_min": -26.57730484008789,
      "activations/layer2_attention_weight_max": 33.16513442993164,
      "activations/layer2_attention_weight_min": -35.11808776855469,
      "activations/layer3_attention_weight_max": 101.91490936279297,
      "activations/layer3_attention_weight_min": -110.97960662841797,
      "activations/layer4_attention_weight_max": 106.25396728515625,
      "activations/layer4_attention_weight_min": -108.79264068603516,
      "activations/layer5_attention_weight_max": 52.650020599365234,
      "activations/layer5_attention_weight_min": -61.98406219482422,
      "activations/layer6_attention_weight_max": 47.825870513916016,
      "activations/layer6_attention_weight_min": -50.66667556762695,
      "activations/layer7_attention_weight_max": 99.2449951171875,
      "activations/layer7_attention_weight_min": -105.2938232421875,
      "activations/layer8_attention_weight_max": 47.20538330078125,
      "activations/layer8_attention_weight_min": -47.88690948486328,
      "activations/layer9_attention_weight_max": 34.292423248291016,
      "activations/layer9_attention_weight_min": -36.195823669433594,
      "epoch": 21.6,
      "learning_rate": 1.081060606060606e-05,
      "loss": 2.7116,
      "step": 371750
    },
    {
      "activations/layer0_attention_weight_max": 16.255151748657227,
      "activations/layer0_attention_weight_min": -13.9871826171875,
      "activations/layer10_attention_weight_max": 37.99180603027344,
      "activations/layer10_attention_weight_min": -36.3270378112793,
      "activations/layer11_attention_weight_max": 36.91779708862305,
      "activations/layer11_attention_weight_min": -36.70147705078125,
      "activations/layer12_attention_weight_max": 31.19867515563965,
      "activations/layer12_attention_weight_min": -29.232627868652344,
      "activations/layer13_attention_weight_max": 45.78266525268555,
      "activations/layer13_attention_weight_min": -38.26040267944336,
      "activations/layer14_attention_weight_max": 51.06247329711914,
      "activations/layer14_attention_weight_min": -42.2539176940918,
      "activations/layer15_attention_weight_max": 47.560630798339844,
      "activations/layer15_attention_weight_min": -43.58658981323242,
      "activations/layer16_attention_weight_max": 34.62238693237305,
      "activations/layer16_attention_weight_min": -30.039573669433594,
      "activations/layer17_attention_weight_max": 32.25386047363281,
      "activations/layer17_attention_weight_min": -28.205276489257812,
      "activations/layer18_attention_weight_max": 35.294166564941406,
      "activations/layer18_attention_weight_min": -27.831071853637695,
      "activations/layer19_attention_weight_max": 36.59647750854492,
      "activations/layer19_attention_weight_min": -33.29616928100586,
      "activations/layer1_attention_weight_max": 16.247053146362305,
      "activations/layer1_attention_weight_min": -15.486948013305664,
      "activations/layer20_attention_weight_max": 32.02543640136719,
      "activations/layer20_attention_weight_min": -25.929845809936523,
      "activations/layer21_attention_weight_max": 33.809814453125,
      "activations/layer21_attention_weight_min": -27.73776626586914,
      "activations/layer22_attention_weight_max": 54.23699188232422,
      "activations/layer22_attention_weight_min": -39.543479919433594,
      "activations/layer23_attention_weight_max": 36.54737091064453,
      "activations/layer23_attention_weight_min": -28.730731964111328,
      "activations/layer2_attention_weight_max": 34.546329498291016,
      "activations/layer2_attention_weight_min": -33.931495666503906,
      "activations/layer3_attention_weight_max": 108.91024017333984,
      "activations/layer3_attention_weight_min": -110.72836303710938,
      "activations/layer4_attention_weight_max": 112.3800277709961,
      "activations/layer4_attention_weight_min": -114.48011779785156,
      "activations/layer5_attention_weight_max": 52.91986083984375,
      "activations/layer5_attention_weight_min": -62.987098693847656,
      "activations/layer6_attention_weight_max": 47.15195083618164,
      "activations/layer6_attention_weight_min": -49.760562896728516,
      "activations/layer7_attention_weight_max": 97.59251403808594,
      "activations/layer7_attention_weight_min": -99.89893341064453,
      "activations/layer8_attention_weight_max": 45.346282958984375,
      "activations/layer8_attention_weight_min": -45.82579803466797,
      "activations/layer9_attention_weight_max": 34.213775634765625,
      "activations/layer9_attention_weight_min": -36.50422286987305,
      "epoch": 21.6,
      "learning_rate": 1.0791666666666667e-05,
      "loss": 2.7094,
      "step": 371800
    },
    {
      "activations/layer0_attention_weight_max": 17.48255157470703,
      "activations/layer0_attention_weight_min": -15.167817115783691,
      "activations/layer10_attention_weight_max": 39.80887222290039,
      "activations/layer10_attention_weight_min": -36.42372512817383,
      "activations/layer11_attention_weight_max": 36.9725227355957,
      "activations/layer11_attention_weight_min": -36.68390655517578,
      "activations/layer12_attention_weight_max": 28.162817001342773,
      "activations/layer12_attention_weight_min": -28.0166072845459,
      "activations/layer13_attention_weight_max": 42.303688049316406,
      "activations/layer13_attention_weight_min": -38.552955627441406,
      "activations/layer14_attention_weight_max": 44.786041259765625,
      "activations/layer14_attention_weight_min": -41.08906936645508,
      "activations/layer15_attention_weight_max": 40.813968658447266,
      "activations/layer15_attention_weight_min": -38.39839172363281,
      "activations/layer16_attention_weight_max": 29.448598861694336,
      "activations/layer16_attention_weight_min": -29.677776336669922,
      "activations/layer17_attention_weight_max": 29.061548233032227,
      "activations/layer17_attention_weight_min": -27.464311599731445,
      "activations/layer18_attention_weight_max": 31.008663177490234,
      "activations/layer18_attention_weight_min": -27.928462982177734,
      "activations/layer19_attention_weight_max": 30.649396896362305,
      "activations/layer19_attention_weight_min": -31.994937896728516,
      "activations/layer1_attention_weight_max": 15.947705268859863,
      "activations/layer1_attention_weight_min": -14.927806854248047,
      "activations/layer20_attention_weight_max": 26.864635467529297,
      "activations/layer20_attention_weight_min": -25.484819412231445,
      "activations/layer21_attention_weight_max": 25.79998016357422,
      "activations/layer21_attention_weight_min": -24.93815040588379,
      "activations/layer22_attention_weight_max": 39.47447967529297,
      "activations/layer22_attention_weight_min": -32.942108154296875,
      "activations/layer23_attention_weight_max": 29.1888427734375,
      "activations/layer23_attention_weight_min": -29.352771759033203,
      "activations/layer2_attention_weight_max": 34.47929763793945,
      "activations/layer2_attention_weight_min": -34.38346862792969,
      "activations/layer3_attention_weight_max": 106.90786743164062,
      "activations/layer3_attention_weight_min": -108.60273742675781,
      "activations/layer4_attention_weight_max": 108.59630584716797,
      "activations/layer4_attention_weight_min": -112.11266326904297,
      "activations/layer5_attention_weight_max": 51.52854919433594,
      "activations/layer5_attention_weight_min": -60.92052459716797,
      "activations/layer6_attention_weight_max": 47.642032623291016,
      "activations/layer6_attention_weight_min": -48.103267669677734,
      "activations/layer7_attention_weight_max": 102.7175064086914,
      "activations/layer7_attention_weight_min": -102.69049835205078,
      "activations/layer8_attention_weight_max": 46.02315902709961,
      "activations/layer8_attention_weight_min": -47.73139572143555,
      "activations/layer9_attention_weight_max": 34.369163513183594,
      "activations/layer9_attention_weight_min": -36.816612243652344,
      "epoch": 21.61,
      "learning_rate": 1.0772727272727272e-05,
      "loss": 2.7155,
      "step": 371850
    },
    {
      "activations/layer0_attention_weight_max": 17.15174102783203,
      "activations/layer0_attention_weight_min": -14.056510925292969,
      "activations/layer10_attention_weight_max": 36.460975646972656,
      "activations/layer10_attention_weight_min": -35.887569427490234,
      "activations/layer11_attention_weight_max": 36.40359878540039,
      "activations/layer11_attention_weight_min": -34.60115051269531,
      "activations/layer12_attention_weight_max": 27.058008193969727,
      "activations/layer12_attention_weight_min": -27.10905647277832,
      "activations/layer13_attention_weight_max": 39.2009391784668,
      "activations/layer13_attention_weight_min": -38.072235107421875,
      "activations/layer14_attention_weight_max": 44.29692077636719,
      "activations/layer14_attention_weight_min": -40.18372344970703,
      "activations/layer15_attention_weight_max": 42.4650764465332,
      "activations/layer15_attention_weight_min": -36.26689147949219,
      "activations/layer16_attention_weight_max": 31.42680549621582,
      "activations/layer16_attention_weight_min": -29.98211097717285,
      "activations/layer17_attention_weight_max": 30.508188247680664,
      "activations/layer17_attention_weight_min": -26.77747344970703,
      "activations/layer18_attention_weight_max": 29.80400276184082,
      "activations/layer18_attention_weight_min": -26.980937957763672,
      "activations/layer19_attention_weight_max": 32.41407012939453,
      "activations/layer19_attention_weight_min": -31.26230239868164,
      "activations/layer1_attention_weight_max": 15.90149974822998,
      "activations/layer1_attention_weight_min": -14.387157440185547,
      "activations/layer20_attention_weight_max": 27.482559204101562,
      "activations/layer20_attention_weight_min": -25.880691528320312,
      "activations/layer21_attention_weight_max": 27.657405853271484,
      "activations/layer21_attention_weight_min": -25.439542770385742,
      "activations/layer22_attention_weight_max": 35.2559700012207,
      "activations/layer22_attention_weight_min": -32.258541107177734,
      "activations/layer23_attention_weight_max": 31.290706634521484,
      "activations/layer23_attention_weight_min": -26.047361373901367,
      "activations/layer2_attention_weight_max": 34.592369079589844,
      "activations/layer2_attention_weight_min": -34.01995086669922,
      "activations/layer3_attention_weight_max": 107.54235076904297,
      "activations/layer3_attention_weight_min": -107.24021911621094,
      "activations/layer4_attention_weight_max": 111.60562896728516,
      "activations/layer4_attention_weight_min": -112.1554946899414,
      "activations/layer5_attention_weight_max": 53.41594696044922,
      "activations/layer5_attention_weight_min": -64.46819305419922,
      "activations/layer6_attention_weight_max": 46.7018928527832,
      "activations/layer6_attention_weight_min": -47.339908599853516,
      "activations/layer7_attention_weight_max": 100.80105590820312,
      "activations/layer7_attention_weight_min": -102.40606689453125,
      "activations/layer8_attention_weight_max": 44.427154541015625,
      "activations/layer8_attention_weight_min": -46.18933868408203,
      "activations/layer9_attention_weight_max": 33.38221740722656,
      "activations/layer9_attention_weight_min": -35.78395080566406,
      "epoch": 21.61,
      "learning_rate": 1.0753787878787878e-05,
      "loss": 2.7096,
      "step": 371900
    },
    {
      "activations/layer0_attention_weight_max": 15.977503776550293,
      "activations/layer0_attention_weight_min": -15.42535400390625,
      "activations/layer10_attention_weight_max": 37.171485900878906,
      "activations/layer10_attention_weight_min": -36.67082214355469,
      "activations/layer11_attention_weight_max": 38.99062728881836,
      "activations/layer11_attention_weight_min": -37.71160888671875,
      "activations/layer12_attention_weight_max": 27.99205207824707,
      "activations/layer12_attention_weight_min": -27.97779655456543,
      "activations/layer13_attention_weight_max": 40.30678939819336,
      "activations/layer13_attention_weight_min": -37.195186614990234,
      "activations/layer14_attention_weight_max": 41.987548828125,
      "activations/layer14_attention_weight_min": -39.26199722290039,
      "activations/layer15_attention_weight_max": 40.00132751464844,
      "activations/layer15_attention_weight_min": -36.586429595947266,
      "activations/layer16_attention_weight_max": 29.674978256225586,
      "activations/layer16_attention_weight_min": -29.67829132080078,
      "activations/layer17_attention_weight_max": 28.712026596069336,
      "activations/layer17_attention_weight_min": -26.33001708984375,
      "activations/layer18_attention_weight_max": 29.9913272857666,
      "activations/layer18_attention_weight_min": -27.806493759155273,
      "activations/layer19_attention_weight_max": 37.01775360107422,
      "activations/layer19_attention_weight_min": -31.073291778564453,
      "activations/layer1_attention_weight_max": 15.714542388916016,
      "activations/layer1_attention_weight_min": -14.581907272338867,
      "activations/layer20_attention_weight_max": 28.476438522338867,
      "activations/layer20_attention_weight_min": -24.97261619567871,
      "activations/layer21_attention_weight_max": 28.119028091430664,
      "activations/layer21_attention_weight_min": -23.68486976623535,
      "activations/layer22_attention_weight_max": 42.09362030029297,
      "activations/layer22_attention_weight_min": -29.76515007019043,
      "activations/layer23_attention_weight_max": 30.865100860595703,
      "activations/layer23_attention_weight_min": -25.512815475463867,
      "activations/layer2_attention_weight_max": 35.364234924316406,
      "activations/layer2_attention_weight_min": -35.14960479736328,
      "activations/layer3_attention_weight_max": 107.80333709716797,
      "activations/layer3_attention_weight_min": -110.03903198242188,
      "activations/layer4_attention_weight_max": 111.2645034790039,
      "activations/layer4_attention_weight_min": -115.6619873046875,
      "activations/layer5_attention_weight_max": 50.869606018066406,
      "activations/layer5_attention_weight_min": -64.94795227050781,
      "activations/layer6_attention_weight_max": 46.64048767089844,
      "activations/layer6_attention_weight_min": -49.48808288574219,
      "activations/layer7_attention_weight_max": 100.00723266601562,
      "activations/layer7_attention_weight_min": -108.27301788330078,
      "activations/layer8_attention_weight_max": 45.20405960083008,
      "activations/layer8_attention_weight_min": -47.752891540527344,
      "activations/layer9_attention_weight_max": 34.63521194458008,
      "activations/layer9_attention_weight_min": -35.860023498535156,
      "epoch": 21.61,
      "learning_rate": 1.0734848484848485e-05,
      "loss": 2.7075,
      "step": 371950
    },
    {
      "activations/layer0_attention_weight_max": 16.66548728942871,
      "activations/layer0_attention_weight_min": -13.34157657623291,
      "activations/layer10_attention_weight_max": 35.66879653930664,
      "activations/layer10_attention_weight_min": -36.39445114135742,
      "activations/layer11_attention_weight_max": 35.107215881347656,
      "activations/layer11_attention_weight_min": -39.7764892578125,
      "activations/layer12_attention_weight_max": 27.927717208862305,
      "activations/layer12_attention_weight_min": -28.2694034576416,
      "activations/layer13_attention_weight_max": 42.396053314208984,
      "activations/layer13_attention_weight_min": -37.94557571411133,
      "activations/layer14_attention_weight_max": 45.58161544799805,
      "activations/layer14_attention_weight_min": -39.87194061279297,
      "activations/layer15_attention_weight_max": 40.0566291809082,
      "activations/layer15_attention_weight_min": -38.6525993347168,
      "activations/layer16_attention_weight_max": 33.067569732666016,
      "activations/layer16_attention_weight_min": -27.71405792236328,
      "activations/layer17_attention_weight_max": 30.756534576416016,
      "activations/layer17_attention_weight_min": -27.04636573791504,
      "activations/layer18_attention_weight_max": 30.204879760742188,
      "activations/layer18_attention_weight_min": -25.960161209106445,
      "activations/layer19_attention_weight_max": 33.65980529785156,
      "activations/layer19_attention_weight_min": -32.11631393432617,
      "activations/layer1_attention_weight_max": 15.683075904846191,
      "activations/layer1_attention_weight_min": -14.030661582946777,
      "activations/layer20_attention_weight_max": 28.101716995239258,
      "activations/layer20_attention_weight_min": -26.075115203857422,
      "activations/layer21_attention_weight_max": 29.060874938964844,
      "activations/layer21_attention_weight_min": -25.18277931213379,
      "activations/layer22_attention_weight_max": 39.815391540527344,
      "activations/layer22_attention_weight_min": -29.77230453491211,
      "activations/layer23_attention_weight_max": 30.817462921142578,
      "activations/layer23_attention_weight_min": -26.333654403686523,
      "activations/layer2_attention_weight_max": 33.37969970703125,
      "activations/layer2_attention_weight_min": -33.15150451660156,
      "activations/layer3_attention_weight_max": 104.99180603027344,
      "activations/layer3_attention_weight_min": -106.46243286132812,
      "activations/layer4_attention_weight_max": 116.29052734375,
      "activations/layer4_attention_weight_min": -114.84577941894531,
      "activations/layer5_attention_weight_max": 50.76752471923828,
      "activations/layer5_attention_weight_min": -60.383949279785156,
      "activations/layer6_attention_weight_max": 45.315452575683594,
      "activations/layer6_attention_weight_min": -50.74760437011719,
      "activations/layer7_attention_weight_max": 95.76020050048828,
      "activations/layer7_attention_weight_min": -104.954345703125,
      "activations/layer8_attention_weight_max": 45.399635314941406,
      "activations/layer8_attention_weight_min": -49.088623046875,
      "activations/layer9_attention_weight_max": 34.592735290527344,
      "activations/layer9_attention_weight_min": -36.414649963378906,
      "epoch": 21.62,
      "learning_rate": 1.071590909090909e-05,
      "loss": 2.711,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.483,
      "eval_samples_per_second": 506.187,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.483,
      "eval_openwebtext_samples_per_second": 506.187,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9795,
      "eval_wikitext_samples_per_second": 230.357,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_lambada_loss": 2.333984375,
      "eval_lambada_ppl": 10.318974397214378,
      "eval_lambada_runtime": 9.5525,
      "eval_lambada_samples_per_second": 509.711,
      "step": 372000
    },
    {
      "activations/layer0_attention_weight_max": 16.58909797668457,
      "activations/layer0_attention_weight_min": -14.349231719970703,
      "activations/layer10_attention_weight_max": 48.13663101196289,
      "activations/layer10_attention_weight_min": -47.91264343261719,
      "activations/layer11_attention_weight_max": 46.178306579589844,
      "activations/layer11_attention_weight_min": -47.73777389526367,
      "activations/layer12_attention_weight_max": 29.92930793762207,
      "activations/layer12_attention_weight_min": -30.225357055664062,
      "activations/layer13_attention_weight_max": 43.306800842285156,
      "activations/layer13_attention_weight_min": -42.017940521240234,
      "activations/layer14_attention_weight_max": 47.76858901977539,
      "activations/layer14_attention_weight_min": -45.137664794921875,
      "activations/layer15_attention_weight_max": 43.439720153808594,
      "activations/layer15_attention_weight_min": -44.04462432861328,
      "activations/layer16_attention_weight_max": 32.36858367919922,
      "activations/layer16_attention_weight_min": -30.55987548828125,
      "activations/layer17_attention_weight_max": 30.76204490661621,
      "activations/layer17_attention_weight_min": -28.461416244506836,
      "activations/layer18_attention_weight_max": 31.169767379760742,
      "activations/layer18_attention_weight_min": -26.837955474853516,
      "activations/layer19_attention_weight_max": 32.19306182861328,
      "activations/layer19_attention_weight_min": -30.899343490600586,
      "activations/layer1_attention_weight_max": 16.405776977539062,
      "activations/layer1_attention_weight_min": -14.659024238586426,
      "activations/layer20_attention_weight_max": 28.5374698638916,
      "activations/layer20_attention_weight_min": -25.668394088745117,
      "activations/layer21_attention_weight_max": 25.046709060668945,
      "activations/layer21_attention_weight_min": -24.57219123840332,
      "activations/layer22_attention_weight_max": 36.96152877807617,
      "activations/layer22_attention_weight_min": -30.47247314453125,
      "activations/layer23_attention_weight_max": 28.987287521362305,
      "activations/layer23_attention_weight_min": -24.36506462097168,
      "activations/layer2_attention_weight_max": 36.85028076171875,
      "activations/layer2_attention_weight_min": -35.535003662109375,
      "activations/layer3_attention_weight_max": 109.62606811523438,
      "activations/layer3_attention_weight_min": -112.35469818115234,
      "activations/layer4_attention_weight_max": 116.3116226196289,
      "activations/layer4_attention_weight_min": -115.33772277832031,
      "activations/layer5_attention_weight_max": 53.07613754272461,
      "activations/layer5_attention_weight_min": -62.130245208740234,
      "activations/layer6_attention_weight_max": 50.845394134521484,
      "activations/layer6_attention_weight_min": -52.55113220214844,
      "activations/layer7_attention_weight_max": 114.46112060546875,
      "activations/layer7_attention_weight_min": -118.31233978271484,
      "activations/layer8_attention_weight_max": 55.13983154296875,
      "activations/layer8_attention_weight_min": -56.754600524902344,
      "activations/layer9_attention_weight_max": 43.600284576416016,
      "activations/layer9_attention_weight_min": -46.93410873413086,
      "epoch": 21.62,
      "learning_rate": 1.0696969696969696e-05,
      "loss": 2.7113,
      "step": 372050
    },
    {
      "activations/layer0_attention_weight_max": 16.733457565307617,
      "activations/layer0_attention_weight_min": -14.003520965576172,
      "activations/layer10_attention_weight_max": 33.52839279174805,
      "activations/layer10_attention_weight_min": -36.676605224609375,
      "activations/layer11_attention_weight_max": 33.68967819213867,
      "activations/layer11_attention_weight_min": -36.547481536865234,
      "activations/layer12_attention_weight_max": 29.026016235351562,
      "activations/layer12_attention_weight_min": -28.89826202392578,
      "activations/layer13_attention_weight_max": 42.71293258666992,
      "activations/layer13_attention_weight_min": -39.19541549682617,
      "activations/layer14_attention_weight_max": 46.33036422729492,
      "activations/layer14_attention_weight_min": -42.66956329345703,
      "activations/layer15_attention_weight_max": 45.704769134521484,
      "activations/layer15_attention_weight_min": -41.97614288330078,
      "activations/layer16_attention_weight_max": 31.907670974731445,
      "activations/layer16_attention_weight_min": -29.65420150756836,
      "activations/layer17_attention_weight_max": 33.24160385131836,
      "activations/layer17_attention_weight_min": -30.604536056518555,
      "activations/layer18_attention_weight_max": 34.23466873168945,
      "activations/layer18_attention_weight_min": -28.290206909179688,
      "activations/layer19_attention_weight_max": 36.62031936645508,
      "activations/layer19_attention_weight_min": -33.217857360839844,
      "activations/layer1_attention_weight_max": 16.973840713500977,
      "activations/layer1_attention_weight_min": -14.936304092407227,
      "activations/layer20_attention_weight_max": 31.1047306060791,
      "activations/layer20_attention_weight_min": -27.015708923339844,
      "activations/layer21_attention_weight_max": 33.192962646484375,
      "activations/layer21_attention_weight_min": -26.812437057495117,
      "activations/layer22_attention_weight_max": 42.83229446411133,
      "activations/layer22_attention_weight_min": -36.53292465209961,
      "activations/layer23_attention_weight_max": 31.505783081054688,
      "activations/layer23_attention_weight_min": -30.066055297851562,
      "activations/layer2_attention_weight_max": 33.992679595947266,
      "activations/layer2_attention_weight_min": -33.95722961425781,
      "activations/layer3_attention_weight_max": 104.4781494140625,
      "activations/layer3_attention_weight_min": -107.79395294189453,
      "activations/layer4_attention_weight_max": 110.06072998046875,
      "activations/layer4_attention_weight_min": -114.12611389160156,
      "activations/layer5_attention_weight_max": 54.87507247924805,
      "activations/layer5_attention_weight_min": -64.62315368652344,
      "activations/layer6_attention_weight_max": 46.369022369384766,
      "activations/layer6_attention_weight_min": -50.8455696105957,
      "activations/layer7_attention_weight_max": 97.63836669921875,
      "activations/layer7_attention_weight_min": -102.00276947021484,
      "activations/layer8_attention_weight_max": 42.64402389526367,
      "activations/layer8_attention_weight_min": -47.73289108276367,
      "activations/layer9_attention_weight_max": 33.18749237060547,
      "activations/layer9_attention_weight_min": -36.17672348022461,
      "epoch": 21.62,
      "learning_rate": 1.0678030303030301e-05,
      "loss": 2.6907,
      "step": 372100
    },
    {
      "activations/layer0_attention_weight_max": 16.745166778564453,
      "activations/layer0_attention_weight_min": -14.190642356872559,
      "activations/layer10_attention_weight_max": 35.00750732421875,
      "activations/layer10_attention_weight_min": -34.54767608642578,
      "activations/layer11_attention_weight_max": 34.457603454589844,
      "activations/layer11_attention_weight_min": -35.52790069580078,
      "activations/layer12_attention_weight_max": 29.80390167236328,
      "activations/layer12_attention_weight_min": -28.326520919799805,
      "activations/layer13_attention_weight_max": 43.61656188964844,
      "activations/layer13_attention_weight_min": -38.15427780151367,
      "activations/layer14_attention_weight_max": 47.43404006958008,
      "activations/layer14_attention_weight_min": -40.1957893371582,
      "activations/layer15_attention_weight_max": 44.45416259765625,
      "activations/layer15_attention_weight_min": -37.039066314697266,
      "activations/layer16_attention_weight_max": 35.267112731933594,
      "activations/layer16_attention_weight_min": -28.89837074279785,
      "activations/layer17_attention_weight_max": 33.86322784423828,
      "activations/layer17_attention_weight_min": -25.90067481994629,
      "activations/layer18_attention_weight_max": 35.81971740722656,
      "activations/layer18_attention_weight_min": -27.012895584106445,
      "activations/layer19_attention_weight_max": 39.75984573364258,
      "activations/layer19_attention_weight_min": -31.60333824157715,
      "activations/layer1_attention_weight_max": 15.798498153686523,
      "activations/layer1_attention_weight_min": -16.064138412475586,
      "activations/layer20_attention_weight_max": 32.12178039550781,
      "activations/layer20_attention_weight_min": -27.84831428527832,
      "activations/layer21_attention_weight_max": 28.591672897338867,
      "activations/layer21_attention_weight_min": -24.276151657104492,
      "activations/layer22_attention_weight_max": 42.208351135253906,
      "activations/layer22_attention_weight_min": -31.44406509399414,
      "activations/layer23_attention_weight_max": 37.52846145629883,
      "activations/layer23_attention_weight_min": -26.60858726501465,
      "activations/layer2_attention_weight_max": 34.0938720703125,
      "activations/layer2_attention_weight_min": -34.18914031982422,
      "activations/layer3_attention_weight_max": 108.4615478515625,
      "activations/layer3_attention_weight_min": -108.0624008178711,
      "activations/layer4_attention_weight_max": 111.94390869140625,
      "activations/layer4_attention_weight_min": -117.05323791503906,
      "activations/layer5_attention_weight_max": 54.92749786376953,
      "activations/layer5_attention_weight_min": -65.58245086669922,
      "activations/layer6_attention_weight_max": 45.77875518798828,
      "activations/layer6_attention_weight_min": -49.0643196105957,
      "activations/layer7_attention_weight_max": 95.00843811035156,
      "activations/layer7_attention_weight_min": -98.82685852050781,
      "activations/layer8_attention_weight_max": 42.71923828125,
      "activations/layer8_attention_weight_min": -46.509307861328125,
      "activations/layer9_attention_weight_max": 32.59929275512695,
      "activations/layer9_attention_weight_min": -34.411922454833984,
      "epoch": 21.62,
      "learning_rate": 1.0659090909090908e-05,
      "loss": 2.7019,
      "step": 372150
    },
    {
      "activations/layer0_attention_weight_max": 16.051851272583008,
      "activations/layer0_attention_weight_min": -14.205615043640137,
      "activations/layer10_attention_weight_max": 35.591468811035156,
      "activations/layer10_attention_weight_min": -38.184425354003906,
      "activations/layer11_attention_weight_max": 36.793434143066406,
      "activations/layer11_attention_weight_min": -36.642234802246094,
      "activations/layer12_attention_weight_max": 27.146671295166016,
      "activations/layer12_attention_weight_min": -28.039369583129883,
      "activations/layer13_attention_weight_max": 39.09341049194336,
      "activations/layer13_attention_weight_min": -37.925533294677734,
      "activations/layer14_attention_weight_max": 41.86919403076172,
      "activations/layer14_attention_weight_min": -40.000572204589844,
      "activations/layer15_attention_weight_max": 37.514190673828125,
      "activations/layer15_attention_weight_min": -36.76620864868164,
      "activations/layer16_attention_weight_max": 27.79984474182129,
      "activations/layer16_attention_weight_min": -29.641916275024414,
      "activations/layer17_attention_weight_max": 29.93642234802246,
      "activations/layer17_attention_weight_min": -26.30701446533203,
      "activations/layer18_attention_weight_max": 32.0821533203125,
      "activations/layer18_attention_weight_min": -26.74034309387207,
      "activations/layer19_attention_weight_max": 31.495115280151367,
      "activations/layer19_attention_weight_min": -32.000240325927734,
      "activations/layer1_attention_weight_max": 15.206223487854004,
      "activations/layer1_attention_weight_min": -13.82609748840332,
      "activations/layer20_attention_weight_max": 28.747215270996094,
      "activations/layer20_attention_weight_min": -25.147016525268555,
      "activations/layer21_attention_weight_max": 28.772789001464844,
      "activations/layer21_attention_weight_min": -24.472087860107422,
      "activations/layer22_attention_weight_max": 41.1041374206543,
      "activations/layer22_attention_weight_min": -32.04914855957031,
      "activations/layer23_attention_weight_max": 31.497230529785156,
      "activations/layer23_attention_weight_min": -26.346500396728516,
      "activations/layer2_attention_weight_max": 32.77943420410156,
      "activations/layer2_attention_weight_min": -33.14698028564453,
      "activations/layer3_attention_weight_max": 103.37918090820312,
      "activations/layer3_attention_weight_min": -107.41602325439453,
      "activations/layer4_attention_weight_max": 107.58155822753906,
      "activations/layer4_attention_weight_min": -113.14802551269531,
      "activations/layer5_attention_weight_max": 50.11640930175781,
      "activations/layer5_attention_weight_min": -61.78792953491211,
      "activations/layer6_attention_weight_max": 46.03978729248047,
      "activations/layer6_attention_weight_min": -49.9006462097168,
      "activations/layer7_attention_weight_max": 96.93678283691406,
      "activations/layer7_attention_weight_min": -104.16893005371094,
      "activations/layer8_attention_weight_max": 43.85142135620117,
      "activations/layer8_attention_weight_min": -47.27864456176758,
      "activations/layer9_attention_weight_max": 32.826454162597656,
      "activations/layer9_attention_weight_min": -37.29450607299805,
      "epoch": 21.63,
      "learning_rate": 1.0640151515151514e-05,
      "loss": 2.6894,
      "step": 372200
    },
    {
      "activations/layer0_attention_weight_max": 15.975273132324219,
      "activations/layer0_attention_weight_min": -15.033564567565918,
      "activations/layer10_attention_weight_max": 34.67443084716797,
      "activations/layer10_attention_weight_min": -38.02232360839844,
      "activations/layer11_attention_weight_max": 33.35453796386719,
      "activations/layer11_attention_weight_min": -35.76713562011719,
      "activations/layer12_attention_weight_max": 24.28717613220215,
      "activations/layer12_attention_weight_min": -27.559436798095703,
      "activations/layer13_attention_weight_max": 36.898521423339844,
      "activations/layer13_attention_weight_min": -33.883079528808594,
      "activations/layer14_attention_weight_max": 43.4405632019043,
      "activations/layer14_attention_weight_min": -38.61139678955078,
      "activations/layer15_attention_weight_max": 41.15516662597656,
      "activations/layer15_attention_weight_min": -36.71787643432617,
      "activations/layer16_attention_weight_max": 32.47800827026367,
      "activations/layer16_attention_weight_min": -29.69379997253418,
      "activations/layer17_attention_weight_max": 29.746938705444336,
      "activations/layer17_attention_weight_min": -26.344627380371094,
      "activations/layer18_attention_weight_max": 32.72081756591797,
      "activations/layer18_attention_weight_min": -27.372098922729492,
      "activations/layer19_attention_weight_max": 36.57517623901367,
      "activations/layer19_attention_weight_min": -33.607643127441406,
      "activations/layer1_attention_weight_max": 15.455316543579102,
      "activations/layer1_attention_weight_min": -14.430885314941406,
      "activations/layer20_attention_weight_max": 32.996273040771484,
      "activations/layer20_attention_weight_min": -26.251218795776367,
      "activations/layer21_attention_weight_max": 33.33903503417969,
      "activations/layer21_attention_weight_min": -27.663450241088867,
      "activations/layer22_attention_weight_max": 43.76498031616211,
      "activations/layer22_attention_weight_min": -36.50547409057617,
      "activations/layer23_attention_weight_max": 34.20194625854492,
      "activations/layer23_attention_weight_min": -29.131973266601562,
      "activations/layer2_attention_weight_max": 35.6413688659668,
      "activations/layer2_attention_weight_min": -33.56723403930664,
      "activations/layer3_attention_weight_max": 103.91514587402344,
      "activations/layer3_attention_weight_min": -104.3200454711914,
      "activations/layer4_attention_weight_max": 105.2166976928711,
      "activations/layer4_attention_weight_min": -112.76158905029297,
      "activations/layer5_attention_weight_max": 50.26475143432617,
      "activations/layer5_attention_weight_min": -61.89741134643555,
      "activations/layer6_attention_weight_max": 42.57361602783203,
      "activations/layer6_attention_weight_min": -47.20924758911133,
      "activations/layer7_attention_weight_max": 100.78623962402344,
      "activations/layer7_attention_weight_min": -100.94807434082031,
      "activations/layer8_attention_weight_max": 43.35371017456055,
      "activations/layer8_attention_weight_min": -42.83082580566406,
      "activations/layer9_attention_weight_max": 31.1899356842041,
      "activations/layer9_attention_weight_min": -34.99476623535156,
      "epoch": 21.63,
      "learning_rate": 1.062121212121212e-05,
      "loss": 2.7189,
      "step": 372250
    },
    {
      "activations/layer0_attention_weight_max": 17.222318649291992,
      "activations/layer0_attention_weight_min": -13.833258628845215,
      "activations/layer10_attention_weight_max": 36.309844970703125,
      "activations/layer10_attention_weight_min": -36.69503402709961,
      "activations/layer11_attention_weight_max": 37.366363525390625,
      "activations/layer11_attention_weight_min": -35.159278869628906,
      "activations/layer12_attention_weight_max": 27.117399215698242,
      "activations/layer12_attention_weight_min": -27.746183395385742,
      "activations/layer13_attention_weight_max": 41.678070068359375,
      "activations/layer13_attention_weight_min": -38.95487976074219,
      "activations/layer14_attention_weight_max": 41.887062072753906,
      "activations/layer14_attention_weight_min": -41.09746170043945,
      "activations/layer15_attention_weight_max": 37.873653411865234,
      "activations/layer15_attention_weight_min": -38.08230209350586,
      "activations/layer16_attention_weight_max": 31.559741973876953,
      "activations/layer16_attention_weight_min": -28.944725036621094,
      "activations/layer17_attention_weight_max": 28.587114334106445,
      "activations/layer17_attention_weight_min": -26.80879020690918,
      "activations/layer18_attention_weight_max": 28.776586532592773,
      "activations/layer18_attention_weight_min": -28.5755615234375,
      "activations/layer19_attention_weight_max": 37.96356201171875,
      "activations/layer19_attention_weight_min": -33.17202377319336,
      "activations/layer1_attention_weight_max": 15.775834083557129,
      "activations/layer1_attention_weight_min": -14.06325912475586,
      "activations/layer20_attention_weight_max": 31.196393966674805,
      "activations/layer20_attention_weight_min": -26.760007858276367,
      "activations/layer21_attention_weight_max": 30.00486946105957,
      "activations/layer21_attention_weight_min": -26.302377700805664,
      "activations/layer22_attention_weight_max": 40.420047760009766,
      "activations/layer22_attention_weight_min": -34.37187957763672,
      "activations/layer23_attention_weight_max": 34.89863586425781,
      "activations/layer23_attention_weight_min": -27.97598648071289,
      "activations/layer2_attention_weight_max": 34.38190460205078,
      "activations/layer2_attention_weight_min": -33.9985466003418,
      "activations/layer3_attention_weight_max": 106.17436981201172,
      "activations/layer3_attention_weight_min": -107.06339263916016,
      "activations/layer4_attention_weight_max": 110.25243377685547,
      "activations/layer4_attention_weight_min": -113.8995590209961,
      "activations/layer5_attention_weight_max": 49.341712951660156,
      "activations/layer5_attention_weight_min": -60.22875213623047,
      "activations/layer6_attention_weight_max": 45.26736831665039,
      "activations/layer6_attention_weight_min": -48.646732330322266,
      "activations/layer7_attention_weight_max": 96.7440414428711,
      "activations/layer7_attention_weight_min": -98.39634704589844,
      "activations/layer8_attention_weight_max": 46.07453918457031,
      "activations/layer8_attention_weight_min": -47.0025520324707,
      "activations/layer9_attention_weight_max": 34.44883346557617,
      "activations/layer9_attention_weight_min": -35.46652603149414,
      "epoch": 21.63,
      "learning_rate": 1.0602272727272726e-05,
      "loss": 2.6995,
      "step": 372300
    },
    {
      "activations/layer0_attention_weight_max": 16.340166091918945,
      "activations/layer0_attention_weight_min": -13.730297088623047,
      "activations/layer10_attention_weight_max": 33.689453125,
      "activations/layer10_attention_weight_min": -34.65567398071289,
      "activations/layer11_attention_weight_max": 33.472816467285156,
      "activations/layer11_attention_weight_min": -36.7895393371582,
      "activations/layer12_attention_weight_max": 27.55555534362793,
      "activations/layer12_attention_weight_min": -28.375795364379883,
      "activations/layer13_attention_weight_max": 39.337066650390625,
      "activations/layer13_attention_weight_min": -36.37113571166992,
      "activations/layer14_attention_weight_max": 43.25089645385742,
      "activations/layer14_attention_weight_min": -40.87861633300781,
      "activations/layer15_attention_weight_max": 39.211708068847656,
      "activations/layer15_attention_weight_min": -35.54582595825195,
      "activations/layer16_attention_weight_max": 28.02694320678711,
      "activations/layer16_attention_weight_min": -28.64409637451172,
      "activations/layer17_attention_weight_max": 29.750879287719727,
      "activations/layer17_attention_weight_min": -24.627769470214844,
      "activations/layer18_attention_weight_max": 32.534976959228516,
      "activations/layer18_attention_weight_min": -25.234262466430664,
      "activations/layer19_attention_weight_max": 35.848690032958984,
      "activations/layer19_attention_weight_min": -31.540359497070312,
      "activations/layer1_attention_weight_max": 15.360297203063965,
      "activations/layer1_attention_weight_min": -14.226052284240723,
      "activations/layer20_attention_weight_max": 29.717039108276367,
      "activations/layer20_attention_weight_min": -26.230838775634766,
      "activations/layer21_attention_weight_max": 30.080278396606445,
      "activations/layer21_attention_weight_min": -26.837339401245117,
      "activations/layer22_attention_weight_max": 39.36632537841797,
      "activations/layer22_attention_weight_min": -32.84003448486328,
      "activations/layer23_attention_weight_max": 28.68172264099121,
      "activations/layer23_attention_weight_min": -26.063108444213867,
      "activations/layer2_attention_weight_max": 33.53010940551758,
      "activations/layer2_attention_weight_min": -33.460975646972656,
      "activations/layer3_attention_weight_max": 102.84916687011719,
      "activations/layer3_attention_weight_min": -106.60276794433594,
      "activations/layer4_attention_weight_max": 105.81585693359375,
      "activations/layer4_attention_weight_min": -109.50292205810547,
      "activations/layer5_attention_weight_max": 50.00285339355469,
      "activations/layer5_attention_weight_min": -60.6978874206543,
      "activations/layer6_attention_weight_max": 44.55454635620117,
      "activations/layer6_attention_weight_min": -47.244991302490234,
      "activations/layer7_attention_weight_max": 93.75065612792969,
      "activations/layer7_attention_weight_min": -102.00240325927734,
      "activations/layer8_attention_weight_max": 42.70500564575195,
      "activations/layer8_attention_weight_min": -44.880855560302734,
      "activations/layer9_attention_weight_max": 31.833425521850586,
      "activations/layer9_attention_weight_min": -34.67177963256836,
      "epoch": 21.64,
      "learning_rate": 1.0583333333333333e-05,
      "loss": 2.7126,
      "step": 372350
    },
    {
      "activations/layer0_attention_weight_max": 16.461402893066406,
      "activations/layer0_attention_weight_min": -13.364689826965332,
      "activations/layer10_attention_weight_max": 35.1270866394043,
      "activations/layer10_attention_weight_min": -35.40974807739258,
      "activations/layer11_attention_weight_max": 34.95594024658203,
      "activations/layer11_attention_weight_min": -35.00949478149414,
      "activations/layer12_attention_weight_max": 28.282358169555664,
      "activations/layer12_attention_weight_min": -30.06969451904297,
      "activations/layer13_attention_weight_max": 38.9506721496582,
      "activations/layer13_attention_weight_min": -38.43684005737305,
      "activations/layer14_attention_weight_max": 43.84954071044922,
      "activations/layer14_attention_weight_min": -42.703311920166016,
      "activations/layer15_attention_weight_max": 39.98126220703125,
      "activations/layer15_attention_weight_min": -37.518802642822266,
      "activations/layer16_attention_weight_max": 31.058679580688477,
      "activations/layer16_attention_weight_min": -30.254682540893555,
      "activations/layer17_attention_weight_max": 30.877065658569336,
      "activations/layer17_attention_weight_min": -28.407081604003906,
      "activations/layer18_attention_weight_max": 33.818817138671875,
      "activations/layer18_attention_weight_min": -27.042259216308594,
      "activations/layer19_attention_weight_max": 34.43724822998047,
      "activations/layer19_attention_weight_min": -32.91093063354492,
      "activations/layer1_attention_weight_max": 16.838529586791992,
      "activations/layer1_attention_weight_min": -15.022468566894531,
      "activations/layer20_attention_weight_max": 28.821521759033203,
      "activations/layer20_attention_weight_min": -26.2672119140625,
      "activations/layer21_attention_weight_max": 28.331588745117188,
      "activations/layer21_attention_weight_min": -26.221261978149414,
      "activations/layer22_attention_weight_max": 42.74886703491211,
      "activations/layer22_attention_weight_min": -34.57718276977539,
      "activations/layer23_attention_weight_max": 32.141780853271484,
      "activations/layer23_attention_weight_min": -27.64809799194336,
      "activations/layer2_attention_weight_max": 33.666175842285156,
      "activations/layer2_attention_weight_min": -33.489013671875,
      "activations/layer3_attention_weight_max": 105.51486206054688,
      "activations/layer3_attention_weight_min": -106.48037719726562,
      "activations/layer4_attention_weight_max": 109.2223892211914,
      "activations/layer4_attention_weight_min": -109.62548065185547,
      "activations/layer5_attention_weight_max": 53.707801818847656,
      "activations/layer5_attention_weight_min": -63.9091796875,
      "activations/layer6_attention_weight_max": 46.63066101074219,
      "activations/layer6_attention_weight_min": -50.059810638427734,
      "activations/layer7_attention_weight_max": 97.46819305419922,
      "activations/layer7_attention_weight_min": -102.38655090332031,
      "activations/layer8_attention_weight_max": 42.47986602783203,
      "activations/layer8_attention_weight_min": -45.308685302734375,
      "activations/layer9_attention_weight_max": 33.225059509277344,
      "activations/layer9_attention_weight_min": -35.068077087402344,
      "epoch": 21.64,
      "learning_rate": 1.0564393939393937e-05,
      "loss": 2.7083,
      "step": 372400
    },
    {
      "activations/layer0_attention_weight_max": 15.966586112976074,
      "activations/layer0_attention_weight_min": -13.913480758666992,
      "activations/layer10_attention_weight_max": 49.4264030456543,
      "activations/layer10_attention_weight_min": -45.08931350708008,
      "activations/layer11_attention_weight_max": 47.8995361328125,
      "activations/layer11_attention_weight_min": -46.271602630615234,
      "activations/layer12_attention_weight_max": 31.08782196044922,
      "activations/layer12_attention_weight_min": -28.033449172973633,
      "activations/layer13_attention_weight_max": 42.56110382080078,
      "activations/layer13_attention_weight_min": -39.8461799621582,
      "activations/layer14_attention_weight_max": 54.60831832885742,
      "activations/layer14_attention_weight_min": -41.54016876220703,
      "activations/layer15_attention_weight_max": 48.82081604003906,
      "activations/layer15_attention_weight_min": -43.618438720703125,
      "activations/layer16_attention_weight_max": 34.759681701660156,
      "activations/layer16_attention_weight_min": -30.129817962646484,
      "activations/layer17_attention_weight_max": 30.321123123168945,
      "activations/layer17_attention_weight_min": -27.602319717407227,
      "activations/layer18_attention_weight_max": 33.53024673461914,
      "activations/layer18_attention_weight_min": -29.228900909423828,
      "activations/layer19_attention_weight_max": 34.325714111328125,
      "activations/layer19_attention_weight_min": -33.313289642333984,
      "activations/layer1_attention_weight_max": 15.911499977111816,
      "activations/layer1_attention_weight_min": -15.18746566772461,
      "activations/layer20_attention_weight_max": 28.9374942779541,
      "activations/layer20_attention_weight_min": -27.436410903930664,
      "activations/layer21_attention_weight_max": 29.91353988647461,
      "activations/layer21_attention_weight_min": -27.176483154296875,
      "activations/layer22_attention_weight_max": 44.30894088745117,
      "activations/layer22_attention_weight_min": -35.68381881713867,
      "activations/layer23_attention_weight_max": 34.81182861328125,
      "activations/layer23_attention_weight_min": -29.477588653564453,
      "activations/layer2_attention_weight_max": 37.856773376464844,
      "activations/layer2_attention_weight_min": -34.733333587646484,
      "activations/layer3_attention_weight_max": 110.00260162353516,
      "activations/layer3_attention_weight_min": -108.6952896118164,
      "activations/layer4_attention_weight_max": 115.15889739990234,
      "activations/layer4_attention_weight_min": -114.91937255859375,
      "activations/layer5_attention_weight_max": 54.47304916381836,
      "activations/layer5_attention_weight_min": -65.7205581665039,
      "activations/layer6_attention_weight_max": 51.68470001220703,
      "activations/layer6_attention_weight_min": -49.56290054321289,
      "activations/layer7_attention_weight_max": 127.08216857910156,
      "activations/layer7_attention_weight_min": -110.95093536376953,
      "activations/layer8_attention_weight_max": 56.482421875,
      "activations/layer8_attention_weight_min": -53.44532012939453,
      "activations/layer9_attention_weight_max": 43.377540588378906,
      "activations/layer9_attention_weight_min": -42.8947868347168,
      "epoch": 21.64,
      "learning_rate": 1.0545454545454544e-05,
      "loss": 2.7218,
      "step": 372450
    },
    {
      "activations/layer0_attention_weight_max": 16.904760360717773,
      "activations/layer0_attention_weight_min": -14.21285629272461,
      "activations/layer10_attention_weight_max": 35.007965087890625,
      "activations/layer10_attention_weight_min": -35.67726516723633,
      "activations/layer11_attention_weight_max": 32.907188415527344,
      "activations/layer11_attention_weight_min": -35.562801361083984,
      "activations/layer12_attention_weight_max": 26.763704299926758,
      "activations/layer12_attention_weight_min": -29.375080108642578,
      "activations/layer13_attention_weight_max": 39.8511962890625,
      "activations/layer13_attention_weight_min": -41.1717414855957,
      "activations/layer14_attention_weight_max": 43.89711380004883,
      "activations/layer14_attention_weight_min": -40.920040130615234,
      "activations/layer15_attention_weight_max": 41.89124298095703,
      "activations/layer15_attention_weight_min": -38.853370666503906,
      "activations/layer16_attention_weight_max": 29.85137939453125,
      "activations/layer16_attention_weight_min": -30.349620819091797,
      "activations/layer17_attention_weight_max": 27.492578506469727,
      "activations/layer17_attention_weight_min": -26.130599975585938,
      "activations/layer18_attention_weight_max": 30.999534606933594,
      "activations/layer18_attention_weight_min": -26.79793930053711,
      "activations/layer19_attention_weight_max": 31.44111442565918,
      "activations/layer19_attention_weight_min": -33.655452728271484,
      "activations/layer1_attention_weight_max": 16.974504470825195,
      "activations/layer1_attention_weight_min": -15.419203758239746,
      "activations/layer20_attention_weight_max": 28.385662078857422,
      "activations/layer20_attention_weight_min": -25.577634811401367,
      "activations/layer21_attention_weight_max": 27.73399543762207,
      "activations/layer21_attention_weight_min": -24.98166847229004,
      "activations/layer22_attention_weight_max": 38.67685317993164,
      "activations/layer22_attention_weight_min": -32.46293258666992,
      "activations/layer23_attention_weight_max": 29.767566680908203,
      "activations/layer23_attention_weight_min": -26.693593978881836,
      "activations/layer2_attention_weight_max": 34.44068908691406,
      "activations/layer2_attention_weight_min": -32.71244430541992,
      "activations/layer3_attention_weight_max": 106.40290069580078,
      "activations/layer3_attention_weight_min": -105.31806182861328,
      "activations/layer4_attention_weight_max": 108.96251678466797,
      "activations/layer4_attention_weight_min": -110.8749008178711,
      "activations/layer5_attention_weight_max": 48.779396057128906,
      "activations/layer5_attention_weight_min": -60.37933349609375,
      "activations/layer6_attention_weight_max": 42.30068588256836,
      "activations/layer6_attention_weight_min": -46.75078582763672,
      "activations/layer7_attention_weight_max": 90.46424102783203,
      "activations/layer7_attention_weight_min": -97.14187622070312,
      "activations/layer8_attention_weight_max": 40.6524543762207,
      "activations/layer8_attention_weight_min": -46.26848220825195,
      "activations/layer9_attention_weight_max": 30.35077476501465,
      "activations/layer9_attention_weight_min": -34.99775314331055,
      "epoch": 21.64,
      "learning_rate": 1.0526515151515149e-05,
      "loss": 2.7192,
      "step": 372500
    },
    {
      "activations/layer0_attention_weight_max": 16.483842849731445,
      "activations/layer0_attention_weight_min": -14.571791648864746,
      "activations/layer10_attention_weight_max": 32.866004943847656,
      "activations/layer10_attention_weight_min": -34.457908630371094,
      "activations/layer11_attention_weight_max": 32.874935150146484,
      "activations/layer11_attention_weight_min": -34.74740219116211,
      "activations/layer12_attention_weight_max": 27.89090347290039,
      "activations/layer12_attention_weight_min": -26.816482543945312,
      "activations/layer13_attention_weight_max": 39.60028076171875,
      "activations/layer13_attention_weight_min": -35.321048736572266,
      "activations/layer14_attention_weight_max": 40.659786224365234,
      "activations/layer14_attention_weight_min": -36.188629150390625,
      "activations/layer15_attention_weight_max": 38.06614303588867,
      "activations/layer15_attention_weight_min": -36.15708923339844,
      "activations/layer16_attention_weight_max": 28.18543243408203,
      "activations/layer16_attention_weight_min": -29.20215606689453,
      "activations/layer17_attention_weight_max": 28.153051376342773,
      "activations/layer17_attention_weight_min": -26.904645919799805,
      "activations/layer18_attention_weight_max": 30.21668815612793,
      "activations/layer18_attention_weight_min": -27.064790725708008,
      "activations/layer19_attention_weight_max": 29.97170066833496,
      "activations/layer19_attention_weight_min": -31.94339370727539,
      "activations/layer1_attention_weight_max": 16.47636604309082,
      "activations/layer1_attention_weight_min": -14.763206481933594,
      "activations/layer20_attention_weight_max": 27.070789337158203,
      "activations/layer20_attention_weight_min": -24.406221389770508,
      "activations/layer21_attention_weight_max": 28.17559051513672,
      "activations/layer21_attention_weight_min": -24.528594970703125,
      "activations/layer22_attention_weight_max": 40.71976089477539,
      "activations/layer22_attention_weight_min": -31.47846794128418,
      "activations/layer23_attention_weight_max": 32.01857376098633,
      "activations/layer23_attention_weight_min": -25.77564811706543,
      "activations/layer2_attention_weight_max": 33.359046936035156,
      "activations/layer2_attention_weight_min": -34.2549934387207,
      "activations/layer3_attention_weight_max": 102.71337890625,
      "activations/layer3_attention_weight_min": -102.72157287597656,
      "activations/layer4_attention_weight_max": 102.56246185302734,
      "activations/layer4_attention_weight_min": -109.57427978515625,
      "activations/layer5_attention_weight_max": 47.02224349975586,
      "activations/layer5_attention_weight_min": -58.69102478027344,
      "activations/layer6_attention_weight_max": 44.72095489501953,
      "activations/layer6_attention_weight_min": -45.43907928466797,
      "activations/layer7_attention_weight_max": 91.55350494384766,
      "activations/layer7_attention_weight_min": -95.04070281982422,
      "activations/layer8_attention_weight_max": 39.597259521484375,
      "activations/layer8_attention_weight_min": -41.64394760131836,
      "activations/layer9_attention_weight_max": 29.839628219604492,
      "activations/layer9_attention_weight_min": -32.43671798706055,
      "epoch": 21.65,
      "learning_rate": 1.0507575757575755e-05,
      "loss": 2.711,
      "step": 372550
    },
    {
      "activations/layer0_attention_weight_max": 16.204954147338867,
      "activations/layer0_attention_weight_min": -13.542862892150879,
      "activations/layer10_attention_weight_max": 38.390480041503906,
      "activations/layer10_attention_weight_min": -37.65935516357422,
      "activations/layer11_attention_weight_max": 37.081275939941406,
      "activations/layer11_attention_weight_min": -36.24469757080078,
      "activations/layer12_attention_weight_max": 28.229646682739258,
      "activations/layer12_attention_weight_min": -29.708375930786133,
      "activations/layer13_attention_weight_max": 39.891693115234375,
      "activations/layer13_attention_weight_min": -39.451805114746094,
      "activations/layer14_attention_weight_max": 42.28519821166992,
      "activations/layer14_attention_weight_min": -40.21943664550781,
      "activations/layer15_attention_weight_max": 41.19382095336914,
      "activations/layer15_attention_weight_min": -38.06509780883789,
      "activations/layer16_attention_weight_max": 31.22953224182129,
      "activations/layer16_attention_weight_min": -28.792404174804688,
      "activations/layer17_attention_weight_max": 29.4642333984375,
      "activations/layer17_attention_weight_min": -28.045753479003906,
      "activations/layer18_attention_weight_max": 30.930429458618164,
      "activations/layer18_attention_weight_min": -25.99976348876953,
      "activations/layer19_attention_weight_max": 35.3320426940918,
      "activations/layer19_attention_weight_min": -33.82006072998047,
      "activations/layer1_attention_weight_max": 15.746837615966797,
      "activations/layer1_attention_weight_min": -14.579794883728027,
      "activations/layer20_attention_weight_max": 29.7573184967041,
      "activations/layer20_attention_weight_min": -25.692758560180664,
      "activations/layer21_attention_weight_max": 27.10759162902832,
      "activations/layer21_attention_weight_min": -24.73997688293457,
      "activations/layer22_attention_weight_max": 41.820335388183594,
      "activations/layer22_attention_weight_min": -31.974504470825195,
      "activations/layer23_attention_weight_max": 31.259891510009766,
      "activations/layer23_attention_weight_min": -25.559879302978516,
      "activations/layer2_attention_weight_max": 36.261962890625,
      "activations/layer2_attention_weight_min": -33.83539581298828,
      "activations/layer3_attention_weight_max": 104.60104370117188,
      "activations/layer3_attention_weight_min": -105.22264099121094,
      "activations/layer4_attention_weight_max": 107.76629638671875,
      "activations/layer4_attention_weight_min": -116.44988250732422,
      "activations/layer5_attention_weight_max": 50.70221710205078,
      "activations/layer5_attention_weight_min": -65.76206970214844,
      "activations/layer6_attention_weight_max": 44.622501373291016,
      "activations/layer6_attention_weight_min": -48.466548919677734,
      "activations/layer7_attention_weight_max": 94.79731750488281,
      "activations/layer7_attention_weight_min": -105.88243865966797,
      "activations/layer8_attention_weight_max": 45.522953033447266,
      "activations/layer8_attention_weight_min": -48.09762191772461,
      "activations/layer9_attention_weight_max": 35.584415435791016,
      "activations/layer9_attention_weight_min": -36.46950149536133,
      "epoch": 21.65,
      "learning_rate": 1.0488636363636364e-05,
      "loss": 2.6856,
      "step": 372600
    },
    {
      "activations/layer0_attention_weight_max": 15.983052253723145,
      "activations/layer0_attention_weight_min": -13.691530227661133,
      "activations/layer10_attention_weight_max": 37.66652297973633,
      "activations/layer10_attention_weight_min": -36.688026428222656,
      "activations/layer11_attention_weight_max": 38.11772918701172,
      "activations/layer11_attention_weight_min": -36.68694305419922,
      "activations/layer12_attention_weight_max": 28.290109634399414,
      "activations/layer12_attention_weight_min": -27.19931983947754,
      "activations/layer13_attention_weight_max": 43.06691360473633,
      "activations/layer13_attention_weight_min": -37.5364990234375,
      "activations/layer14_attention_weight_max": 45.46134567260742,
      "activations/layer14_attention_weight_min": -38.77985382080078,
      "activations/layer15_attention_weight_max": 43.36690902709961,
      "activations/layer15_attention_weight_min": -35.683692932128906,
      "activations/layer16_attention_weight_max": 31.977811813354492,
      "activations/layer16_attention_weight_min": -28.251462936401367,
      "activations/layer17_attention_weight_max": 30.943315505981445,
      "activations/layer17_attention_weight_min": -27.21895408630371,
      "activations/layer18_attention_weight_max": 37.10173416137695,
      "activations/layer18_attention_weight_min": -26.789918899536133,
      "activations/layer19_attention_weight_max": 36.385826110839844,
      "activations/layer19_attention_weight_min": -32.34928894042969,
      "activations/layer1_attention_weight_max": 16.2564754486084,
      "activations/layer1_attention_weight_min": -15.821817398071289,
      "activations/layer20_attention_weight_max": 29.837871551513672,
      "activations/layer20_attention_weight_min": -25.426746368408203,
      "activations/layer21_attention_weight_max": 30.219589233398438,
      "activations/layer21_attention_weight_min": -25.510210037231445,
      "activations/layer22_attention_weight_max": 41.857059478759766,
      "activations/layer22_attention_weight_min": -31.738971710205078,
      "activations/layer23_attention_weight_max": 32.98213195800781,
      "activations/layer23_attention_weight_min": -28.122785568237305,
      "activations/layer2_attention_weight_max": 35.32380676269531,
      "activations/layer2_attention_weight_min": -33.74077224731445,
      "activations/layer3_attention_weight_max": 105.21503448486328,
      "activations/layer3_attention_weight_min": -106.44551086425781,
      "activations/layer4_attention_weight_max": 110.2020492553711,
      "activations/layer4_attention_weight_min": -113.28313446044922,
      "activations/layer5_attention_weight_max": 50.75918197631836,
      "activations/layer5_attention_weight_min": -63.521575927734375,
      "activations/layer6_attention_weight_max": 46.10233688354492,
      "activations/layer6_attention_weight_min": -48.40420913696289,
      "activations/layer7_attention_weight_max": 102.67984008789062,
      "activations/layer7_attention_weight_min": -100.91533660888672,
      "activations/layer8_attention_weight_max": 45.22499084472656,
      "activations/layer8_attention_weight_min": -46.33156204223633,
      "activations/layer9_attention_weight_max": 34.1119499206543,
      "activations/layer9_attention_weight_min": -35.812862396240234,
      "epoch": 21.65,
      "learning_rate": 1.046969696969697e-05,
      "loss": 2.7319,
      "step": 372650
    },
    {
      "activations/layer0_attention_weight_max": 16.913532257080078,
      "activations/layer0_attention_weight_min": -14.214944839477539,
      "activations/layer10_attention_weight_max": 34.244205474853516,
      "activations/layer10_attention_weight_min": -34.971920013427734,
      "activations/layer11_attention_weight_max": 34.623573303222656,
      "activations/layer11_attention_weight_min": -35.42540740966797,
      "activations/layer12_attention_weight_max": 27.38435173034668,
      "activations/layer12_attention_weight_min": -26.933448791503906,
      "activations/layer13_attention_weight_max": 41.82633590698242,
      "activations/layer13_attention_weight_min": -36.49445724487305,
      "activations/layer14_attention_weight_max": 45.62198257446289,
      "activations/layer14_attention_weight_min": -37.681800842285156,
      "activations/layer15_attention_weight_max": 40.339969635009766,
      "activations/layer15_attention_weight_min": -35.94223403930664,
      "activations/layer16_attention_weight_max": 29.670207977294922,
      "activations/layer16_attention_weight_min": -29.072710037231445,
      "activations/layer17_attention_weight_max": 29.10349464416504,
      "activations/layer17_attention_weight_min": -26.762157440185547,
      "activations/layer18_attention_weight_max": 30.430110931396484,
      "activations/layer18_attention_weight_min": -24.665109634399414,
      "activations/layer19_attention_weight_max": 31.627151489257812,
      "activations/layer19_attention_weight_min": -32.05692672729492,
      "activations/layer1_attention_weight_max": 15.645868301391602,
      "activations/layer1_attention_weight_min": -14.813285827636719,
      "activations/layer20_attention_weight_max": 27.408674240112305,
      "activations/layer20_attention_weight_min": -25.913175582885742,
      "activations/layer21_attention_weight_max": 30.61104965209961,
      "activations/layer21_attention_weight_min": -25.6746883392334,
      "activations/layer22_attention_weight_max": 41.41885757446289,
      "activations/layer22_attention_weight_min": -31.21185874938965,
      "activations/layer23_attention_weight_max": 33.61454772949219,
      "activations/layer23_attention_weight_min": -27.146581649780273,
      "activations/layer2_attention_weight_max": 33.42509841918945,
      "activations/layer2_attention_weight_min": -33.58296203613281,
      "activations/layer3_attention_weight_max": 103.72806549072266,
      "activations/layer3_attention_weight_min": -106.8250732421875,
      "activations/layer4_attention_weight_max": 110.9042739868164,
      "activations/layer4_attention_weight_min": -113.46881103515625,
      "activations/layer5_attention_weight_max": 50.035518646240234,
      "activations/layer5_attention_weight_min": -64.44865417480469,
      "activations/layer6_attention_weight_max": 45.8682975769043,
      "activations/layer6_attention_weight_min": -48.731937408447266,
      "activations/layer7_attention_weight_max": 93.20972442626953,
      "activations/layer7_attention_weight_min": -100.93600463867188,
      "activations/layer8_attention_weight_max": 41.959320068359375,
      "activations/layer8_attention_weight_min": -45.14971923828125,
      "activations/layer9_attention_weight_max": 32.595428466796875,
      "activations/layer9_attention_weight_min": -34.154483795166016,
      "epoch": 21.66,
      "learning_rate": 1.0450757575757575e-05,
      "loss": 2.6905,
      "step": 372700
    },
    {
      "activations/layer0_attention_weight_max": 17.071935653686523,
      "activations/layer0_attention_weight_min": -14.024024963378906,
      "activations/layer10_attention_weight_max": 34.70061111450195,
      "activations/layer10_attention_weight_min": -36.16713333129883,
      "activations/layer11_attention_weight_max": 33.14883804321289,
      "activations/layer11_attention_weight_min": -36.306182861328125,
      "activations/layer12_attention_weight_max": 26.597549438476562,
      "activations/layer12_attention_weight_min": -27.187564849853516,
      "activations/layer13_attention_weight_max": 39.9900016784668,
      "activations/layer13_attention_weight_min": -35.9907341003418,
      "activations/layer14_attention_weight_max": 42.65618896484375,
      "activations/layer14_attention_weight_min": -36.31095504760742,
      "activations/layer15_attention_weight_max": 38.69225311279297,
      "activations/layer15_attention_weight_min": -36.21171951293945,
      "activations/layer16_attention_weight_max": 28.007387161254883,
      "activations/layer16_attention_weight_min": -27.60563087463379,
      "activations/layer17_attention_weight_max": 26.815473556518555,
      "activations/layer17_attention_weight_min": -26.295780181884766,
      "activations/layer18_attention_weight_max": 32.024559020996094,
      "activations/layer18_attention_weight_min": -25.462833404541016,
      "activations/layer19_attention_weight_max": 35.61970138549805,
      "activations/layer19_attention_weight_min": -31.137720108032227,
      "activations/layer1_attention_weight_max": 15.579634666442871,
      "activations/layer1_attention_weight_min": -14.816356658935547,
      "activations/layer20_attention_weight_max": 31.603734970092773,
      "activations/layer20_attention_weight_min": -24.991661071777344,
      "activations/layer21_attention_weight_max": 32.17586898803711,
      "activations/layer21_attention_weight_min": -24.70476722717285,
      "activations/layer22_attention_weight_max": 42.4703369140625,
      "activations/layer22_attention_weight_min": -31.502683639526367,
      "activations/layer23_attention_weight_max": 34.73336410522461,
      "activations/layer23_attention_weight_min": -25.1119384765625,
      "activations/layer2_attention_weight_max": 33.9684944152832,
      "activations/layer2_attention_weight_min": -32.904083251953125,
      "activations/layer3_attention_weight_max": 102.77803802490234,
      "activations/layer3_attention_weight_min": -102.05596160888672,
      "activations/layer4_attention_weight_max": 105.97712707519531,
      "activations/layer4_attention_weight_min": -109.91378021240234,
      "activations/layer5_attention_weight_max": 49.12864685058594,
      "activations/layer5_attention_weight_min": -61.98411178588867,
      "activations/layer6_attention_weight_max": 44.65349197387695,
      "activations/layer6_attention_weight_min": -45.414466857910156,
      "activations/layer7_attention_weight_max": 90.17240905761719,
      "activations/layer7_attention_weight_min": -93.49992370605469,
      "activations/layer8_attention_weight_max": 41.27564239501953,
      "activations/layer8_attention_weight_min": -44.25310134887695,
      "activations/layer9_attention_weight_max": 32.01184844970703,
      "activations/layer9_attention_weight_min": -35.04530334472656,
      "epoch": 21.66,
      "learning_rate": 1.0431818181818182e-05,
      "loss": 2.7061,
      "step": 372750
    },
    {
      "activations/layer0_attention_weight_max": 15.768049240112305,
      "activations/layer0_attention_weight_min": -14.225645065307617,
      "activations/layer10_attention_weight_max": 35.004703521728516,
      "activations/layer10_attention_weight_min": -37.82012176513672,
      "activations/layer11_attention_weight_max": 33.97791290283203,
      "activations/layer11_attention_weight_min": -36.62580490112305,
      "activations/layer12_attention_weight_max": 28.90137481689453,
      "activations/layer12_attention_weight_min": -27.96455192565918,
      "activations/layer13_attention_weight_max": 39.6265754699707,
      "activations/layer13_attention_weight_min": -38.45567321777344,
      "activations/layer14_attention_weight_max": 46.0401725769043,
      "activations/layer14_attention_weight_min": -43.12921142578125,
      "activations/layer15_attention_weight_max": 40.588985443115234,
      "activations/layer15_attention_weight_min": -37.603302001953125,
      "activations/layer16_attention_weight_max": 33.83586120605469,
      "activations/layer16_attention_weight_min": -28.9395694732666,
      "activations/layer17_attention_weight_max": 28.35037612915039,
      "activations/layer17_attention_weight_min": -25.348886489868164,
      "activations/layer18_attention_weight_max": 30.24850082397461,
      "activations/layer18_attention_weight_min": -24.165653228759766,
      "activations/layer19_attention_weight_max": 32.624267578125,
      "activations/layer19_attention_weight_min": -30.627700805664062,
      "activations/layer1_attention_weight_max": 15.782268524169922,
      "activations/layer1_attention_weight_min": -14.550960540771484,
      "activations/layer20_attention_weight_max": 27.846996307373047,
      "activations/layer20_attention_weight_min": -23.249853134155273,
      "activations/layer21_attention_weight_max": 27.546964645385742,
      "activations/layer21_attention_weight_min": -22.55719566345215,
      "activations/layer22_attention_weight_max": 39.049713134765625,
      "activations/layer22_attention_weight_min": -29.43400764465332,
      "activations/layer23_attention_weight_max": 30.105125427246094,
      "activations/layer23_attention_weight_min": -22.444984436035156,
      "activations/layer2_attention_weight_max": 34.216453552246094,
      "activations/layer2_attention_weight_min": -33.72769546508789,
      "activations/layer3_attention_weight_max": 109.75819396972656,
      "activations/layer3_attention_weight_min": -112.54212951660156,
      "activations/layer4_attention_weight_max": 116.35880279541016,
      "activations/layer4_attention_weight_min": -111.49391174316406,
      "activations/layer5_attention_weight_max": 51.37470626831055,
      "activations/layer5_attention_weight_min": -61.61588668823242,
      "activations/layer6_attention_weight_max": 46.41256332397461,
      "activations/layer6_attention_weight_min": -48.483680725097656,
      "activations/layer7_attention_weight_max": 98.01322937011719,
      "activations/layer7_attention_weight_min": -103.1596450805664,
      "activations/layer8_attention_weight_max": 45.401451110839844,
      "activations/layer8_attention_weight_min": -48.5614013671875,
      "activations/layer9_attention_weight_max": 33.703433990478516,
      "activations/layer9_attention_weight_min": -37.904850006103516,
      "epoch": 21.66,
      "learning_rate": 1.0412878787878787e-05,
      "loss": 2.6949,
      "step": 372800
    },
    {
      "activations/layer0_attention_weight_max": 16.95030403137207,
      "activations/layer0_attention_weight_min": -16.088876724243164,
      "activations/layer10_attention_weight_max": 32.249755859375,
      "activations/layer10_attention_weight_min": -35.538177490234375,
      "activations/layer11_attention_weight_max": 33.39723205566406,
      "activations/layer11_attention_weight_min": -36.56895446777344,
      "activations/layer12_attention_weight_max": 26.42753791809082,
      "activations/layer12_attention_weight_min": -26.683883666992188,
      "activations/layer13_attention_weight_max": 37.50615692138672,
      "activations/layer13_attention_weight_min": -35.84270477294922,
      "activations/layer14_attention_weight_max": 42.3890266418457,
      "activations/layer14_attention_weight_min": -36.11129379272461,
      "activations/layer15_attention_weight_max": 36.90288543701172,
      "activations/layer15_attention_weight_min": -34.82807540893555,
      "activations/layer16_attention_weight_max": 28.739641189575195,
      "activations/layer16_attention_weight_min": -28.78358268737793,
      "activations/layer17_attention_weight_max": 32.309329986572266,
      "activations/layer17_attention_weight_min": -26.289155960083008,
      "activations/layer18_attention_weight_max": 34.47133255004883,
      "activations/layer18_attention_weight_min": -27.82522201538086,
      "activations/layer19_attention_weight_max": 34.61861801147461,
      "activations/layer19_attention_weight_min": -33.367679595947266,
      "activations/layer1_attention_weight_max": 16.287687301635742,
      "activations/layer1_attention_weight_min": -14.324844360351562,
      "activations/layer20_attention_weight_max": 30.739910125732422,
      "activations/layer20_attention_weight_min": -27.742473602294922,
      "activations/layer21_attention_weight_max": 30.369779586791992,
      "activations/layer21_attention_weight_min": -25.631980895996094,
      "activations/layer22_attention_weight_max": 39.776241302490234,
      "activations/layer22_attention_weight_min": -31.759628295898438,
      "activations/layer23_attention_weight_max": 33.05384063720703,
      "activations/layer23_attention_weight_min": -28.89984130859375,
      "activations/layer2_attention_weight_max": 33.802860260009766,
      "activations/layer2_attention_weight_min": -32.91535949707031,
      "activations/layer3_attention_weight_max": 102.23457336425781,
      "activations/layer3_attention_weight_min": -105.51884460449219,
      "activations/layer4_attention_weight_max": 103.62210845947266,
      "activations/layer4_attention_weight_min": -109.7157974243164,
      "activations/layer5_attention_weight_max": 49.34038543701172,
      "activations/layer5_attention_weight_min": -64.19709777832031,
      "activations/layer6_attention_weight_max": 44.14885711669922,
      "activations/layer6_attention_weight_min": -48.22045135498047,
      "activations/layer7_attention_weight_max": 93.88756561279297,
      "activations/layer7_attention_weight_min": -97.16878509521484,
      "activations/layer8_attention_weight_max": 41.48171615600586,
      "activations/layer8_attention_weight_min": -44.3424072265625,
      "activations/layer9_attention_weight_max": 31.985071182250977,
      "activations/layer9_attention_weight_min": -35.09415054321289,
      "epoch": 21.66,
      "learning_rate": 1.0393939393939393e-05,
      "loss": 2.7051,
      "step": 372850
    },
    {
      "activations/layer0_attention_weight_max": 16.371034622192383,
      "activations/layer0_attention_weight_min": -13.33698844909668,
      "activations/layer10_attention_weight_max": 37.05935287475586,
      "activations/layer10_attention_weight_min": -38.313743591308594,
      "activations/layer11_attention_weight_max": 33.015045166015625,
      "activations/layer11_attention_weight_min": -35.54404067993164,
      "activations/layer12_attention_weight_max": 25.51508140563965,
      "activations/layer12_attention_weight_min": -27.09575653076172,
      "activations/layer13_attention_weight_max": 40.88883972167969,
      "activations/layer13_attention_weight_min": -36.29267883300781,
      "activations/layer14_attention_weight_max": 43.268497467041016,
      "activations/layer14_attention_weight_min": -37.56725311279297,
      "activations/layer15_attention_weight_max": 37.78822326660156,
      "activations/layer15_attention_weight_min": -35.17216110229492,
      "activations/layer16_attention_weight_max": 28.90664291381836,
      "activations/layer16_attention_weight_min": -30.273889541625977,
      "activations/layer17_attention_weight_max": 28.286745071411133,
      "activations/layer17_attention_weight_min": -26.698352813720703,
      "activations/layer18_attention_weight_max": 30.680458068847656,
      "activations/layer18_attention_weight_min": -25.667261123657227,
      "activations/layer19_attention_weight_max": 37.143699645996094,
      "activations/layer19_attention_weight_min": -32.9444694519043,
      "activations/layer1_attention_weight_max": 15.116985321044922,
      "activations/layer1_attention_weight_min": -15.161721229553223,
      "activations/layer20_attention_weight_max": 29.85406494140625,
      "activations/layer20_attention_weight_min": -25.47882652282715,
      "activations/layer21_attention_weight_max": 28.992807388305664,
      "activations/layer21_attention_weight_min": -26.112045288085938,
      "activations/layer22_attention_weight_max": 43.31318283081055,
      "activations/layer22_attention_weight_min": -33.31473922729492,
      "activations/layer23_attention_weight_max": 32.7344856262207,
      "activations/layer23_attention_weight_min": -28.35266876220703,
      "activations/layer2_attention_weight_max": 33.36146545410156,
      "activations/layer2_attention_weight_min": -33.333984375,
      "activations/layer3_attention_weight_max": 106.8915023803711,
      "activations/layer3_attention_weight_min": -108.72653198242188,
      "activations/layer4_attention_weight_max": 109.92974853515625,
      "activations/layer4_attention_weight_min": -114.46907806396484,
      "activations/layer5_attention_weight_max": 51.23273849487305,
      "activations/layer5_attention_weight_min": -63.19735336303711,
      "activations/layer6_attention_weight_max": 45.88008499145508,
      "activations/layer6_attention_weight_min": -51.314029693603516,
      "activations/layer7_attention_weight_max": 105.00843048095703,
      "activations/layer7_attention_weight_min": -110.51092529296875,
      "activations/layer8_attention_weight_max": 46.22682571411133,
      "activations/layer8_attention_weight_min": -47.513999938964844,
      "activations/layer9_attention_weight_max": 33.700042724609375,
      "activations/layer9_attention_weight_min": -37.0821418762207,
      "epoch": 21.67,
      "learning_rate": 1.0375e-05,
      "loss": 2.7089,
      "step": 372900
    },
    {
      "activations/layer0_attention_weight_max": 16.84178924560547,
      "activations/layer0_attention_weight_min": -15.48073959350586,
      "activations/layer10_attention_weight_max": 35.003997802734375,
      "activations/layer10_attention_weight_min": -37.13835144042969,
      "activations/layer11_attention_weight_max": 34.78569412231445,
      "activations/layer11_attention_weight_min": -37.77744674682617,
      "activations/layer12_attention_weight_max": 29.02263832092285,
      "activations/layer12_attention_weight_min": -28.02893829345703,
      "activations/layer13_attention_weight_max": 41.41149139404297,
      "activations/layer13_attention_weight_min": -37.48378372192383,
      "activations/layer14_attention_weight_max": 47.90764236450195,
      "activations/layer14_attention_weight_min": -42.54973602294922,
      "activations/layer15_attention_weight_max": 45.68905258178711,
      "activations/layer15_attention_weight_min": -38.88664245605469,
      "activations/layer16_attention_weight_max": 32.23277282714844,
      "activations/layer16_attention_weight_min": -28.55592155456543,
      "activations/layer17_attention_weight_max": 33.66362380981445,
      "activations/layer17_attention_weight_min": -27.276132583618164,
      "activations/layer18_attention_weight_max": 36.65913009643555,
      "activations/layer18_attention_weight_min": -29.782516479492188,
      "activations/layer19_attention_weight_max": 40.15620040893555,
      "activations/layer19_attention_weight_min": -37.639312744140625,
      "activations/layer1_attention_weight_max": 17.99972915649414,
      "activations/layer1_attention_weight_min": -15.578375816345215,
      "activations/layer20_attention_weight_max": 31.354528427124023,
      "activations/layer20_attention_weight_min": -27.438304901123047,
      "activations/layer21_attention_weight_max": 31.019390106201172,
      "activations/layer21_attention_weight_min": -27.343528747558594,
      "activations/layer22_attention_weight_max": 44.17298126220703,
      "activations/layer22_attention_weight_min": -33.37416076660156,
      "activations/layer23_attention_weight_max": 35.716514587402344,
      "activations/layer23_attention_weight_min": -28.88400650024414,
      "activations/layer2_attention_weight_max": 35.88483428955078,
      "activations/layer2_attention_weight_min": -33.031856536865234,
      "activations/layer3_attention_weight_max": 109.62358856201172,
      "activations/layer3_attention_weight_min": -106.0594482421875,
      "activations/layer4_attention_weight_max": 112.67484283447266,
      "activations/layer4_attention_weight_min": -115.45719146728516,
      "activations/layer5_attention_weight_max": 51.50777816772461,
      "activations/layer5_attention_weight_min": -60.236305236816406,
      "activations/layer6_attention_weight_max": 46.30786895751953,
      "activations/layer6_attention_weight_min": -48.43845748901367,
      "activations/layer7_attention_weight_max": 99.03076171875,
      "activations/layer7_attention_weight_min": -99.83683776855469,
      "activations/layer8_attention_weight_max": 41.55533218383789,
      "activations/layer8_attention_weight_min": -45.95833206176758,
      "activations/layer9_attention_weight_max": 32.80076217651367,
      "activations/layer9_attention_weight_min": -36.8819465637207,
      "epoch": 21.67,
      "learning_rate": 1.0356060606060605e-05,
      "loss": 2.7193,
      "step": 372950
    },
    {
      "activations/layer0_attention_weight_max": 17.268205642700195,
      "activations/layer0_attention_weight_min": -13.782312393188477,
      "activations/layer10_attention_weight_max": 36.17522048950195,
      "activations/layer10_attention_weight_min": -35.66707229614258,
      "activations/layer11_attention_weight_max": 36.43327331542969,
      "activations/layer11_attention_weight_min": -35.4941520690918,
      "activations/layer12_attention_weight_max": 29.156526565551758,
      "activations/layer12_attention_weight_min": -28.5922794342041,
      "activations/layer13_attention_weight_max": 42.410884857177734,
      "activations/layer13_attention_weight_min": -39.83375549316406,
      "activations/layer14_attention_weight_max": 43.15121841430664,
      "activations/layer14_attention_weight_min": -40.379005432128906,
      "activations/layer15_attention_weight_max": 43.08646774291992,
      "activations/layer15_attention_weight_min": -37.77423858642578,
      "activations/layer16_attention_weight_max": 29.55290412902832,
      "activations/layer16_attention_weight_min": -28.469579696655273,
      "activations/layer17_attention_weight_max": 29.879343032836914,
      "activations/layer17_attention_weight_min": -27.550800323486328,
      "activations/layer18_attention_weight_max": 34.40098190307617,
      "activations/layer18_attention_weight_min": -27.867416381835938,
      "activations/layer19_attention_weight_max": 35.32709884643555,
      "activations/layer19_attention_weight_min": -33.56612777709961,
      "activations/layer1_attention_weight_max": 15.588395118713379,
      "activations/layer1_attention_weight_min": -14.035290718078613,
      "activations/layer20_attention_weight_max": 34.17649841308594,
      "activations/layer20_attention_weight_min": -27.14630126953125,
      "activations/layer21_attention_weight_max": 33.71541213989258,
      "activations/layer21_attention_weight_min": -27.29974937438965,
      "activations/layer22_attention_weight_max": 45.46333312988281,
      "activations/layer22_attention_weight_min": -34.47314453125,
      "activations/layer23_attention_weight_max": 32.23920440673828,
      "activations/layer23_attention_weight_min": -25.664730072021484,
      "activations/layer2_attention_weight_max": 33.055206298828125,
      "activations/layer2_attention_weight_min": -34.24647903442383,
      "activations/layer3_attention_weight_max": 109.35379028320312,
      "activations/layer3_attention_weight_min": -109.5640640258789,
      "activations/layer4_attention_weight_max": 112.16180419921875,
      "activations/layer4_attention_weight_min": -114.1484146118164,
      "activations/layer5_attention_weight_max": 51.647491455078125,
      "activations/layer5_attention_weight_min": -62.100128173828125,
      "activations/layer6_attention_weight_max": 46.03200149536133,
      "activations/layer6_attention_weight_min": -50.86148452758789,
      "activations/layer7_attention_weight_max": 105.41513061523438,
      "activations/layer7_attention_weight_min": -106.02163696289062,
      "activations/layer8_attention_weight_max": 45.61306381225586,
      "activations/layer8_attention_weight_min": -48.7725830078125,
      "activations/layer9_attention_weight_max": 33.77119064331055,
      "activations/layer9_attention_weight_min": -39.256446838378906,
      "epoch": 21.67,
      "learning_rate": 1.0337121212121211e-05,
      "loss": 2.7052,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4937,
      "eval_samples_per_second": 505.551,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4937,
      "eval_openwebtext_samples_per_second": 505.551,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9836,
      "eval_wikitext_samples_per_second": 229.884,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_lambada_loss": 2.330078125,
      "eval_lambada_ppl": 10.278744528592322,
      "eval_lambada_runtime": 9.5444,
      "eval_lambada_samples_per_second": 510.144,
      "step": 373000
    },
    {
      "activations/layer0_attention_weight_max": 17.49945640563965,
      "activations/layer0_attention_weight_min": -15.417129516601562,
      "activations/layer10_attention_weight_max": 39.21490478515625,
      "activations/layer10_attention_weight_min": -36.1945686340332,
      "activations/layer11_attention_weight_max": 34.83332824707031,
      "activations/layer11_attention_weight_min": -36.453712463378906,
      "activations/layer12_attention_weight_max": 25.06834602355957,
      "activations/layer12_attention_weight_min": -25.31410026550293,
      "activations/layer13_attention_weight_max": 36.41891860961914,
      "activations/layer13_attention_weight_min": -35.29624557495117,
      "activations/layer14_attention_weight_max": 40.33562469482422,
      "activations/layer14_attention_weight_min": -36.42913818359375,
      "activations/layer15_attention_weight_max": 38.39622497558594,
      "activations/layer15_attention_weight_min": -38.221405029296875,
      "activations/layer16_attention_weight_max": 28.95353126525879,
      "activations/layer16_attention_weight_min": -26.537464141845703,
      "activations/layer17_attention_weight_max": 29.467023849487305,
      "activations/layer17_attention_weight_min": -26.085037231445312,
      "activations/layer18_attention_weight_max": 32.96913146972656,
      "activations/layer18_attention_weight_min": -26.843097686767578,
      "activations/layer19_attention_weight_max": 33.685428619384766,
      "activations/layer19_attention_weight_min": -31.91543960571289,
      "activations/layer1_attention_weight_max": 15.83344554901123,
      "activations/layer1_attention_weight_min": -15.116668701171875,
      "activations/layer20_attention_weight_max": 28.255918502807617,
      "activations/layer20_attention_weight_min": -24.801240921020508,
      "activations/layer21_attention_weight_max": 27.813859939575195,
      "activations/layer21_attention_weight_min": -24.11705207824707,
      "activations/layer22_attention_weight_max": 42.3195686340332,
      "activations/layer22_attention_weight_min": -32.600067138671875,
      "activations/layer23_attention_weight_max": 30.046743392944336,
      "activations/layer23_attention_weight_min": -25.74323272705078,
      "activations/layer2_attention_weight_max": 33.430503845214844,
      "activations/layer2_attention_weight_min": -33.72083282470703,
      "activations/layer3_attention_weight_max": 105.97749328613281,
      "activations/layer3_attention_weight_min": -109.03218078613281,
      "activations/layer4_attention_weight_max": 111.09830474853516,
      "activations/layer4_attention_weight_min": -110.5169448852539,
      "activations/layer5_attention_weight_max": 49.7396354675293,
      "activations/layer5_attention_weight_min": -63.200279235839844,
      "activations/layer6_attention_weight_max": 45.091068267822266,
      "activations/layer6_attention_weight_min": -46.60685348510742,
      "activations/layer7_attention_weight_max": 108.24856567382812,
      "activations/layer7_attention_weight_min": -99.6241226196289,
      "activations/layer8_attention_weight_max": 47.95064163208008,
      "activations/layer8_attention_weight_min": -47.138240814208984,
      "activations/layer9_attention_weight_max": 33.87187194824219,
      "activations/layer9_attention_weight_min": -40.70735549926758,
      "epoch": 21.68,
      "learning_rate": 1.0318181818181818e-05,
      "loss": 2.7096,
      "step": 373050
    },
    {
      "activations/layer0_attention_weight_max": 16.12781524658203,
      "activations/layer0_attention_weight_min": -14.350184440612793,
      "activations/layer10_attention_weight_max": 32.57101058959961,
      "activations/layer10_attention_weight_min": -34.98299026489258,
      "activations/layer11_attention_weight_max": 32.22001647949219,
      "activations/layer11_attention_weight_min": -35.682518005371094,
      "activations/layer12_attention_weight_max": 27.3294734954834,
      "activations/layer12_attention_weight_min": -26.829111099243164,
      "activations/layer13_attention_weight_max": 40.8647575378418,
      "activations/layer13_attention_weight_min": -34.31941604614258,
      "activations/layer14_attention_weight_max": 40.73872375488281,
      "activations/layer14_attention_weight_min": -38.27625274658203,
      "activations/layer15_attention_weight_max": 43.29401397705078,
      "activations/layer15_attention_weight_min": -35.523399353027344,
      "activations/layer16_attention_weight_max": 30.715478897094727,
      "activations/layer16_attention_weight_min": -27.792295455932617,
      "activations/layer17_attention_weight_max": 28.747222900390625,
      "activations/layer17_attention_weight_min": -25.900775909423828,
      "activations/layer18_attention_weight_max": 31.319313049316406,
      "activations/layer18_attention_weight_min": -28.130510330200195,
      "activations/layer19_attention_weight_max": 31.120946884155273,
      "activations/layer19_attention_weight_min": -31.961645126342773,
      "activations/layer1_attention_weight_max": 15.860512733459473,
      "activations/layer1_attention_weight_min": -15.4337739944458,
      "activations/layer20_attention_weight_max": 28.043134689331055,
      "activations/layer20_attention_weight_min": -25.356678009033203,
      "activations/layer21_attention_weight_max": 27.81494140625,
      "activations/layer21_attention_weight_min": -24.67252540588379,
      "activations/layer22_attention_weight_max": 36.58662796020508,
      "activations/layer22_attention_weight_min": -29.47456169128418,
      "activations/layer23_attention_weight_max": 31.891128540039062,
      "activations/layer23_attention_weight_min": -26.208831787109375,
      "activations/layer2_attention_weight_max": 33.73724365234375,
      "activations/layer2_attention_weight_min": -32.91717529296875,
      "activations/layer3_attention_weight_max": 106.88745880126953,
      "activations/layer3_attention_weight_min": -106.58160400390625,
      "activations/layer4_attention_weight_max": 107.961181640625,
      "activations/layer4_attention_weight_min": -115.35626220703125,
      "activations/layer5_attention_weight_max": 49.32741928100586,
      "activations/layer5_attention_weight_min": -64.57717895507812,
      "activations/layer6_attention_weight_max": 45.322532653808594,
      "activations/layer6_attention_weight_min": -48.42528533935547,
      "activations/layer7_attention_weight_max": 96.78520965576172,
      "activations/layer7_attention_weight_min": -98.79668426513672,
      "activations/layer8_attention_weight_max": 44.45530700683594,
      "activations/layer8_attention_weight_min": -44.86626052856445,
      "activations/layer9_attention_weight_max": 31.668441772460938,
      "activations/layer9_attention_weight_min": -34.43513107299805,
      "epoch": 21.68,
      "learning_rate": 1.0299242424242423e-05,
      "loss": 2.6987,
      "step": 373100
    },
    {
      "activations/layer0_attention_weight_max": 16.270591735839844,
      "activations/layer0_attention_weight_min": -15.06700325012207,
      "activations/layer10_attention_weight_max": 39.4207763671875,
      "activations/layer10_attention_weight_min": -40.4407844543457,
      "activations/layer11_attention_weight_max": 40.94538116455078,
      "activations/layer11_attention_weight_min": -39.76521301269531,
      "activations/layer12_attention_weight_max": 29.05199432373047,
      "activations/layer12_attention_weight_min": -29.936134338378906,
      "activations/layer13_attention_weight_max": 42.7041130065918,
      "activations/layer13_attention_weight_min": -40.83146286010742,
      "activations/layer14_attention_weight_max": 45.47566604614258,
      "activations/layer14_attention_weight_min": -43.04600143432617,
      "activations/layer15_attention_weight_max": 42.883460998535156,
      "activations/layer15_attention_weight_min": -39.670654296875,
      "activations/layer16_attention_weight_max": 29.651294708251953,
      "activations/layer16_attention_weight_min": -30.363338470458984,
      "activations/layer17_attention_weight_max": 29.109506607055664,
      "activations/layer17_attention_weight_min": -29.062164306640625,
      "activations/layer18_attention_weight_max": 34.0923957824707,
      "activations/layer18_attention_weight_min": -26.38920021057129,
      "activations/layer19_attention_weight_max": 35.5560188293457,
      "activations/layer19_attention_weight_min": -35.342002868652344,
      "activations/layer1_attention_weight_max": 16.83851432800293,
      "activations/layer1_attention_weight_min": -16.35480499267578,
      "activations/layer20_attention_weight_max": 31.498735427856445,
      "activations/layer20_attention_weight_min": -27.223087310791016,
      "activations/layer21_attention_weight_max": 29.485626220703125,
      "activations/layer21_attention_weight_min": -27.6591796875,
      "activations/layer22_attention_weight_max": 42.5245361328125,
      "activations/layer22_attention_weight_min": -35.362850189208984,
      "activations/layer23_attention_weight_max": 35.12808609008789,
      "activations/layer23_attention_weight_min": -27.436683654785156,
      "activations/layer2_attention_weight_max": 35.32344436645508,
      "activations/layer2_attention_weight_min": -33.58446502685547,
      "activations/layer3_attention_weight_max": 110.8288803100586,
      "activations/layer3_attention_weight_min": -104.76976776123047,
      "activations/layer4_attention_weight_max": 118.82173919677734,
      "activations/layer4_attention_weight_min": -117.11354064941406,
      "activations/layer5_attention_weight_max": 56.29835510253906,
      "activations/layer5_attention_weight_min": -61.78074264526367,
      "activations/layer6_attention_weight_max": 51.94386291503906,
      "activations/layer6_attention_weight_min": -51.684349060058594,
      "activations/layer7_attention_weight_max": 109.96366119384766,
      "activations/layer7_attention_weight_min": -115.78700256347656,
      "activations/layer8_attention_weight_max": 51.7984504699707,
      "activations/layer8_attention_weight_min": -55.60152816772461,
      "activations/layer9_attention_weight_max": 39.9208984375,
      "activations/layer9_attention_weight_min": -42.07640075683594,
      "epoch": 21.68,
      "learning_rate": 1.028030303030303e-05,
      "loss": 2.7038,
      "step": 373150
    },
    {
      "activations/layer0_attention_weight_max": 16.47152328491211,
      "activations/layer0_attention_weight_min": -15.469313621520996,
      "activations/layer10_attention_weight_max": 38.06935119628906,
      "activations/layer10_attention_weight_min": -36.5,
      "activations/layer11_attention_weight_max": 38.43000411987305,
      "activations/layer11_attention_weight_min": -38.21562957763672,
      "activations/layer12_attention_weight_max": 25.930143356323242,
      "activations/layer12_attention_weight_min": -26.587156295776367,
      "activations/layer13_attention_weight_max": 37.452850341796875,
      "activations/layer13_attention_weight_min": -35.15365982055664,
      "activations/layer14_attention_weight_max": 40.291900634765625,
      "activations/layer14_attention_weight_min": -35.742305755615234,
      "activations/layer15_attention_weight_max": 36.27446746826172,
      "activations/layer15_attention_weight_min": -35.7331657409668,
      "activations/layer16_attention_weight_max": 28.052932739257812,
      "activations/layer16_attention_weight_min": -29.363840103149414,
      "activations/layer17_attention_weight_max": 30.265947341918945,
      "activations/layer17_attention_weight_min": -26.41377830505371,
      "activations/layer18_attention_weight_max": 29.786489486694336,
      "activations/layer18_attention_weight_min": -28.262813568115234,
      "activations/layer19_attention_weight_max": 28.821069717407227,
      "activations/layer19_attention_weight_min": -32.677555084228516,
      "activations/layer1_attention_weight_max": 16.361427307128906,
      "activations/layer1_attention_weight_min": -15.52796459197998,
      "activations/layer20_attention_weight_max": 27.011384963989258,
      "activations/layer20_attention_weight_min": -25.322166442871094,
      "activations/layer21_attention_weight_max": 25.71988296508789,
      "activations/layer21_attention_weight_min": -24.84174156188965,
      "activations/layer22_attention_weight_max": 35.79231262207031,
      "activations/layer22_attention_weight_min": -32.663787841796875,
      "activations/layer23_attention_weight_max": 28.849689483642578,
      "activations/layer23_attention_weight_min": -26.255382537841797,
      "activations/layer2_attention_weight_max": 34.02532196044922,
      "activations/layer2_attention_weight_min": -33.0520133972168,
      "activations/layer3_attention_weight_max": 107.7346420288086,
      "activations/layer3_attention_weight_min": -110.99578857421875,
      "activations/layer4_attention_weight_max": 113.8610610961914,
      "activations/layer4_attention_weight_min": -113.62049865722656,
      "activations/layer5_attention_weight_max": 50.709468841552734,
      "activations/layer5_attention_weight_min": -62.699668884277344,
      "activations/layer6_attention_weight_max": 46.01383972167969,
      "activations/layer6_attention_weight_min": -51.26416778564453,
      "activations/layer7_attention_weight_max": 98.98099517822266,
      "activations/layer7_attention_weight_min": -105.9792251586914,
      "activations/layer8_attention_weight_max": 43.99700927734375,
      "activations/layer8_attention_weight_min": -48.94319534301758,
      "activations/layer9_attention_weight_max": 34.53449630737305,
      "activations/layer9_attention_weight_min": -36.097900390625,
      "epoch": 21.69,
      "learning_rate": 1.0261363636363634e-05,
      "loss": 2.706,
      "step": 373200
    },
    {
      "activations/layer0_attention_weight_max": 16.483686447143555,
      "activations/layer0_attention_weight_min": -13.924285888671875,
      "activations/layer10_attention_weight_max": 34.614654541015625,
      "activations/layer10_attention_weight_min": -33.85771560668945,
      "activations/layer11_attention_weight_max": 34.45917510986328,
      "activations/layer11_attention_weight_min": -32.939964294433594,
      "activations/layer12_attention_weight_max": 26.207035064697266,
      "activations/layer12_attention_weight_min": -27.348289489746094,
      "activations/layer13_attention_weight_max": 37.18038558959961,
      "activations/layer13_attention_weight_min": -36.42072296142578,
      "activations/layer14_attention_weight_max": 40.49733352661133,
      "activations/layer14_attention_weight_min": -38.52263259887695,
      "activations/layer15_attention_weight_max": 36.27529525756836,
      "activations/layer15_attention_weight_min": -36.525413513183594,
      "activations/layer16_attention_weight_max": 30.456863403320312,
      "activations/layer16_attention_weight_min": -27.456499099731445,
      "activations/layer17_attention_weight_max": 30.379756927490234,
      "activations/layer17_attention_weight_min": -26.060611724853516,
      "activations/layer18_attention_weight_max": 33.97847366333008,
      "activations/layer18_attention_weight_min": -27.62385368347168,
      "activations/layer19_attention_weight_max": 34.80339813232422,
      "activations/layer19_attention_weight_min": -32.156272888183594,
      "activations/layer1_attention_weight_max": 16.556833267211914,
      "activations/layer1_attention_weight_min": -16.986726760864258,
      "activations/layer20_attention_weight_max": 27.590131759643555,
      "activations/layer20_attention_weight_min": -26.085525512695312,
      "activations/layer21_attention_weight_max": 27.616924285888672,
      "activations/layer21_attention_weight_min": -26.367874145507812,
      "activations/layer22_attention_weight_max": 41.60003662109375,
      "activations/layer22_attention_weight_min": -32.360111236572266,
      "activations/layer23_attention_weight_max": 31.690078735351562,
      "activations/layer23_attention_weight_min": -28.284421920776367,
      "activations/layer2_attention_weight_max": 34.20208740234375,
      "activations/layer2_attention_weight_min": -32.31622314453125,
      "activations/layer3_attention_weight_max": 110.56978607177734,
      "activations/layer3_attention_weight_min": -107.38838195800781,
      "activations/layer4_attention_weight_max": 113.9354476928711,
      "activations/layer4_attention_weight_min": -114.40599822998047,
      "activations/layer5_attention_weight_max": 50.26471710205078,
      "activations/layer5_attention_weight_min": -64.06245422363281,
      "activations/layer6_attention_weight_max": 45.02415466308594,
      "activations/layer6_attention_weight_min": -48.23574447631836,
      "activations/layer7_attention_weight_max": 99.8573989868164,
      "activations/layer7_attention_weight_min": -101.00389099121094,
      "activations/layer8_attention_weight_max": 44.00393295288086,
      "activations/layer8_attention_weight_min": -46.54710388183594,
      "activations/layer9_attention_weight_max": 32.691097259521484,
      "activations/layer9_attention_weight_min": -34.44559860229492,
      "epoch": 21.69,
      "learning_rate": 1.024242424242424e-05,
      "loss": 2.7046,
      "step": 373250
    },
    {
      "activations/layer0_attention_weight_max": 16.689998626708984,
      "activations/layer0_attention_weight_min": -14.258307456970215,
      "activations/layer10_attention_weight_max": 35.54773712158203,
      "activations/layer10_attention_weight_min": -35.091827392578125,
      "activations/layer11_attention_weight_max": 35.655826568603516,
      "activations/layer11_attention_weight_min": -35.940086364746094,
      "activations/layer12_attention_weight_max": 28.825754165649414,
      "activations/layer12_attention_weight_min": -27.789905548095703,
      "activations/layer13_attention_weight_max": 41.04253387451172,
      "activations/layer13_attention_weight_min": -35.790069580078125,
      "activations/layer14_attention_weight_max": 45.94755935668945,
      "activations/layer14_attention_weight_min": -37.796119689941406,
      "activations/layer15_attention_weight_max": 41.01258850097656,
      "activations/layer15_attention_weight_min": -36.70836639404297,
      "activations/layer16_attention_weight_max": 31.813358306884766,
      "activations/layer16_attention_weight_min": -30.252323150634766,
      "activations/layer17_attention_weight_max": 30.43060302734375,
      "activations/layer17_attention_weight_min": -27.33774757385254,
      "activations/layer18_attention_weight_max": 33.08796310424805,
      "activations/layer18_attention_weight_min": -25.93442153930664,
      "activations/layer19_attention_weight_max": 34.92880630493164,
      "activations/layer19_attention_weight_min": -32.77901840209961,
      "activations/layer1_attention_weight_max": 17.831491470336914,
      "activations/layer1_attention_weight_min": -14.949094772338867,
      "activations/layer20_attention_weight_max": 29.55891990661621,
      "activations/layer20_attention_weight_min": -24.612436294555664,
      "activations/layer21_attention_weight_max": 29.554033279418945,
      "activations/layer21_attention_weight_min": -27.515453338623047,
      "activations/layer22_attention_weight_max": 41.0396614074707,
      "activations/layer22_attention_weight_min": -34.474998474121094,
      "activations/layer23_attention_weight_max": 36.06529235839844,
      "activations/layer23_attention_weight_min": -27.166074752807617,
      "activations/layer2_attention_weight_max": 33.43260955810547,
      "activations/layer2_attention_weight_min": -33.07569122314453,
      "activations/layer3_attention_weight_max": 102.70773315429688,
      "activations/layer3_attention_weight_min": -105.77204895019531,
      "activations/layer4_attention_weight_max": 109.15875244140625,
      "activations/layer4_attention_weight_min": -112.05712127685547,
      "activations/layer5_attention_weight_max": 53.117210388183594,
      "activations/layer5_attention_weight_min": -60.772361755371094,
      "activations/layer6_attention_weight_max": 44.086082458496094,
      "activations/layer6_attention_weight_min": -48.346885681152344,
      "activations/layer7_attention_weight_max": 93.73047637939453,
      "activations/layer7_attention_weight_min": -100.8950424194336,
      "activations/layer8_attention_weight_max": 45.78293991088867,
      "activations/layer8_attention_weight_min": -44.97153091430664,
      "activations/layer9_attention_weight_max": 32.93056869506836,
      "activations/layer9_attention_weight_min": -34.8980598449707,
      "epoch": 21.69,
      "learning_rate": 1.0223863636363635e-05,
      "loss": 2.7068,
      "step": 373300
    },
    {
      "activations/layer0_attention_weight_max": 16.81917381286621,
      "activations/layer0_attention_weight_min": -14.557268142700195,
      "activations/layer10_attention_weight_max": 37.03264617919922,
      "activations/layer10_attention_weight_min": -37.28108215332031,
      "activations/layer11_attention_weight_max": 33.83451461791992,
      "activations/layer11_attention_weight_min": -37.00947952270508,
      "activations/layer12_attention_weight_max": 27.79963493347168,
      "activations/layer12_attention_weight_min": -28.74758529663086,
      "activations/layer13_attention_weight_max": 39.83931350708008,
      "activations/layer13_attention_weight_min": -35.29425811767578,
      "activations/layer14_attention_weight_max": 44.96360778808594,
      "activations/layer14_attention_weight_min": -40.1020393371582,
      "activations/layer15_attention_weight_max": 38.71658706665039,
      "activations/layer15_attention_weight_min": -36.77189254760742,
      "activations/layer16_attention_weight_max": 27.782276153564453,
      "activations/layer16_attention_weight_min": -28.61896514892578,
      "activations/layer17_attention_weight_max": 27.78668212890625,
      "activations/layer17_attention_weight_min": -26.282936096191406,
      "activations/layer18_attention_weight_max": 34.35354995727539,
      "activations/layer18_attention_weight_min": -24.67995834350586,
      "activations/layer19_attention_weight_max": 35.45936965942383,
      "activations/layer19_attention_weight_min": -30.67609977722168,
      "activations/layer1_attention_weight_max": 15.628621101379395,
      "activations/layer1_attention_weight_min": -15.520931243896484,
      "activations/layer20_attention_weight_max": 26.8590087890625,
      "activations/layer20_attention_weight_min": -24.800395965576172,
      "activations/layer21_attention_weight_max": 28.263132095336914,
      "activations/layer21_attention_weight_min": -24.957775115966797,
      "activations/layer22_attention_weight_max": 42.46169662475586,
      "activations/layer22_attention_weight_min": -31.193668365478516,
      "activations/layer23_attention_weight_max": 31.37893295288086,
      "activations/layer23_attention_weight_min": -24.554462432861328,
      "activations/layer2_attention_weight_max": 33.05805587768555,
      "activations/layer2_attention_weight_min": -33.27155303955078,
      "activations/layer3_attention_weight_max": 105.0676498413086,
      "activations/layer3_attention_weight_min": -105.87361145019531,
      "activations/layer4_attention_weight_max": 108.00836181640625,
      "activations/layer4_attention_weight_min": -116.94935607910156,
      "activations/layer5_attention_weight_max": 51.223140716552734,
      "activations/layer5_attention_weight_min": -62.003700256347656,
      "activations/layer6_attention_weight_max": 45.71757888793945,
      "activations/layer6_attention_weight_min": -49.83472442626953,
      "activations/layer7_attention_weight_max": 99.76740264892578,
      "activations/layer7_attention_weight_min": -109.84587860107422,
      "activations/layer8_attention_weight_max": 45.93277359008789,
      "activations/layer8_attention_weight_min": -52.79329299926758,
      "activations/layer9_attention_weight_max": 33.97706604003906,
      "activations/layer9_attention_weight_min": -40.9189338684082,
      "epoch": 21.69,
      "learning_rate": 1.0204924242424241e-05,
      "loss": 2.7209,
      "step": 373350
    },
    {
      "activations/layer0_attention_weight_max": 16.309947967529297,
      "activations/layer0_attention_weight_min": -14.336535453796387,
      "activations/layer10_attention_weight_max": 37.208412170410156,
      "activations/layer10_attention_weight_min": -37.862876892089844,
      "activations/layer11_attention_weight_max": 39.731109619140625,
      "activations/layer11_attention_weight_min": -36.705177307128906,
      "activations/layer12_attention_weight_max": 30.885190963745117,
      "activations/layer12_attention_weight_min": -27.516807556152344,
      "activations/layer13_attention_weight_max": 45.16915512084961,
      "activations/layer13_attention_weight_min": -36.9087028503418,
      "activations/layer14_attention_weight_max": 52.970603942871094,
      "activations/layer14_attention_weight_min": -40.630191802978516,
      "activations/layer15_attention_weight_max": 52.24900817871094,
      "activations/layer15_attention_weight_min": -40.6185188293457,
      "activations/layer16_attention_weight_max": 37.78203201293945,
      "activations/layer16_attention_weight_min": -29.091230392456055,
      "activations/layer17_attention_weight_max": 33.9272575378418,
      "activations/layer17_attention_weight_min": -27.77898597717285,
      "activations/layer18_attention_weight_max": 34.17951583862305,
      "activations/layer18_attention_weight_min": -29.410728454589844,
      "activations/layer19_attention_weight_max": 40.389827728271484,
      "activations/layer19_attention_weight_min": -37.78133010864258,
      "activations/layer1_attention_weight_max": 15.879154205322266,
      "activations/layer1_attention_weight_min": -14.744343757629395,
      "activations/layer20_attention_weight_max": 33.38144302368164,
      "activations/layer20_attention_weight_min": -29.445695877075195,
      "activations/layer21_attention_weight_max": 31.22666358947754,
      "activations/layer21_attention_weight_min": -29.082822799682617,
      "activations/layer22_attention_weight_max": 57.57413101196289,
      "activations/layer22_attention_weight_min": -43.13516616821289,
      "activations/layer23_attention_weight_max": 37.86016082763672,
      "activations/layer23_attention_weight_min": -31.96036148071289,
      "activations/layer2_attention_weight_max": 35.893836975097656,
      "activations/layer2_attention_weight_min": -32.87910079956055,
      "activations/layer3_attention_weight_max": 107.77479553222656,
      "activations/layer3_attention_weight_min": -113.28547668457031,
      "activations/layer4_attention_weight_max": 107.2583236694336,
      "activations/layer4_attention_weight_min": -115.34480285644531,
      "activations/layer5_attention_weight_max": 50.120094299316406,
      "activations/layer5_attention_weight_min": -66.51187133789062,
      "activations/layer6_attention_weight_max": 46.540138244628906,
      "activations/layer6_attention_weight_min": -51.657920837402344,
      "activations/layer7_attention_weight_max": 94.39981079101562,
      "activations/layer7_attention_weight_min": -100.40282440185547,
      "activations/layer8_attention_weight_max": 41.76120376586914,
      "activations/layer8_attention_weight_min": -44.87621307373047,
      "activations/layer9_attention_weight_max": 32.801509857177734,
      "activations/layer9_attention_weight_min": -37.11484909057617,
      "epoch": 21.7,
      "learning_rate": 1.0185984848484846e-05,
      "loss": 2.7,
      "step": 373400
    },
    {
      "activations/layer0_attention_weight_max": 16.968931198120117,
      "activations/layer0_attention_weight_min": -13.523834228515625,
      "activations/layer10_attention_weight_max": 35.176456451416016,
      "activations/layer10_attention_weight_min": -34.36821746826172,
      "activations/layer11_attention_weight_max": 33.872703552246094,
      "activations/layer11_attention_weight_min": -34.524871826171875,
      "activations/layer12_attention_weight_max": 26.378742218017578,
      "activations/layer12_attention_weight_min": -26.954809188842773,
      "activations/layer13_attention_weight_max": 39.44759750366211,
      "activations/layer13_attention_weight_min": -36.16217803955078,
      "activations/layer14_attention_weight_max": 40.49697494506836,
      "activations/layer14_attention_weight_min": -37.685455322265625,
      "activations/layer15_attention_weight_max": 35.928855895996094,
      "activations/layer15_attention_weight_min": -35.47378921508789,
      "activations/layer16_attention_weight_max": 26.60411262512207,
      "activations/layer16_attention_weight_min": -27.621726989746094,
      "activations/layer17_attention_weight_max": 29.97918128967285,
      "activations/layer17_attention_weight_min": -27.133691787719727,
      "activations/layer18_attention_weight_max": 31.597856521606445,
      "activations/layer18_attention_weight_min": -26.311601638793945,
      "activations/layer19_attention_weight_max": 34.80830383300781,
      "activations/layer19_attention_weight_min": -33.945762634277344,
      "activations/layer1_attention_weight_max": 15.864715576171875,
      "activations/layer1_attention_weight_min": -14.719817161560059,
      "activations/layer20_attention_weight_max": 27.44335174560547,
      "activations/layer20_attention_weight_min": -27.0587100982666,
      "activations/layer21_attention_weight_max": 27.1998348236084,
      "activations/layer21_attention_weight_min": -27.70928192138672,
      "activations/layer22_attention_weight_max": 38.9487190246582,
      "activations/layer22_attention_weight_min": -32.78276824951172,
      "activations/layer23_attention_weight_max": 33.402069091796875,
      "activations/layer23_attention_weight_min": -24.97547721862793,
      "activations/layer2_attention_weight_max": 34.051143646240234,
      "activations/layer2_attention_weight_min": -34.27863311767578,
      "activations/layer3_attention_weight_max": 103.63207244873047,
      "activations/layer3_attention_weight_min": -110.68960571289062,
      "activations/layer4_attention_weight_max": 109.4577407836914,
      "activations/layer4_attention_weight_min": -111.05013275146484,
      "activations/layer5_attention_weight_max": 53.9176025390625,
      "activations/layer5_attention_weight_min": -61.9664421081543,
      "activations/layer6_attention_weight_max": 44.44218444824219,
      "activations/layer6_attention_weight_min": -47.196327209472656,
      "activations/layer7_attention_weight_max": 92.0779037475586,
      "activations/layer7_attention_weight_min": -98.14049530029297,
      "activations/layer8_attention_weight_max": 40.21665954589844,
      "activations/layer8_attention_weight_min": -43.851844787597656,
      "activations/layer9_attention_weight_max": 32.17890548706055,
      "activations/layer9_attention_weight_min": -34.60417175292969,
      "epoch": 21.7,
      "learning_rate": 1.0167045454545453e-05,
      "loss": 2.7083,
      "step": 373450
    },
    {
      "activations/layer0_attention_weight_max": 16.81562614440918,
      "activations/layer0_attention_weight_min": -14.91146469116211,
      "activations/layer10_attention_weight_max": 33.73820495605469,
      "activations/layer10_attention_weight_min": -35.539371490478516,
      "activations/layer11_attention_weight_max": 33.378108978271484,
      "activations/layer11_attention_weight_min": -34.75510025024414,
      "activations/layer12_attention_weight_max": 27.10073471069336,
      "activations/layer12_attention_weight_min": -28.093032836914062,
      "activations/layer13_attention_weight_max": 39.62855529785156,
      "activations/layer13_attention_weight_min": -35.9658317565918,
      "activations/layer14_attention_weight_max": 42.96002197265625,
      "activations/layer14_attention_weight_min": -38.07514190673828,
      "activations/layer15_attention_weight_max": 41.45695877075195,
      "activations/layer15_attention_weight_min": -38.162620544433594,
      "activations/layer16_attention_weight_max": 31.26253318786621,
      "activations/layer16_attention_weight_min": -30.382999420166016,
      "activations/layer17_attention_weight_max": 31.152042388916016,
      "activations/layer17_attention_weight_min": -28.589406967163086,
      "activations/layer18_attention_weight_max": 33.17256546020508,
      "activations/layer18_attention_weight_min": -26.227951049804688,
      "activations/layer19_attention_weight_max": 34.4660758972168,
      "activations/layer19_attention_weight_min": -34.53843307495117,
      "activations/layer1_attention_weight_max": 16.229915618896484,
      "activations/layer1_attention_weight_min": -16.389148712158203,
      "activations/layer20_attention_weight_max": 27.563995361328125,
      "activations/layer20_attention_weight_min": -28.6350154876709,
      "activations/layer21_attention_weight_max": 30.13858413696289,
      "activations/layer21_attention_weight_min": -27.442758560180664,
      "activations/layer22_attention_weight_max": 43.34473419189453,
      "activations/layer22_attention_weight_min": -33.217586517333984,
      "activations/layer23_attention_weight_max": 33.48567581176758,
      "activations/layer23_attention_weight_min": -26.539337158203125,
      "activations/layer2_attention_weight_max": 34.904605865478516,
      "activations/layer2_attention_weight_min": -34.530948638916016,
      "activations/layer3_attention_weight_max": 109.81509399414062,
      "activations/layer3_attention_weight_min": -110.9875259399414,
      "activations/layer4_attention_weight_max": 112.81670379638672,
      "activations/layer4_attention_weight_min": -112.08363342285156,
      "activations/layer5_attention_weight_max": 52.7432861328125,
      "activations/layer5_attention_weight_min": -64.68704986572266,
      "activations/layer6_attention_weight_max": 47.247581481933594,
      "activations/layer6_attention_weight_min": -46.837833404541016,
      "activations/layer7_attention_weight_max": 97.37637329101562,
      "activations/layer7_attention_weight_min": -104.41083526611328,
      "activations/layer8_attention_weight_max": 43.36252975463867,
      "activations/layer8_attention_weight_min": -45.86345291137695,
      "activations/layer9_attention_weight_max": 32.32637023925781,
      "activations/layer9_attention_weight_min": -34.202571868896484,
      "epoch": 21.7,
      "learning_rate": 1.0148106060606061e-05,
      "loss": 2.6937,
      "step": 373500
    },
    {
      "activations/layer0_attention_weight_max": 17.24497413635254,
      "activations/layer0_attention_weight_min": -14.046136856079102,
      "activations/layer10_attention_weight_max": 37.549041748046875,
      "activations/layer10_attention_weight_min": -36.26225662231445,
      "activations/layer11_attention_weight_max": 37.846927642822266,
      "activations/layer11_attention_weight_min": -38.34693145751953,
      "activations/layer12_attention_weight_max": 25.04412078857422,
      "activations/layer12_attention_weight_min": -28.171876907348633,
      "activations/layer13_attention_weight_max": 37.28338623046875,
      "activations/layer13_attention_weight_min": -37.551551818847656,
      "activations/layer14_attention_weight_max": 40.57235336303711,
      "activations/layer14_attention_weight_min": -37.004180908203125,
      "activations/layer15_attention_weight_max": 39.077964782714844,
      "activations/layer15_attention_weight_min": -37.21303176879883,
      "activations/layer16_attention_weight_max": 27.434799194335938,
      "activations/layer16_attention_weight_min": -27.974716186523438,
      "activations/layer17_attention_weight_max": 27.34714126586914,
      "activations/layer17_attention_weight_min": -25.930055618286133,
      "activations/layer18_attention_weight_max": 30.706340789794922,
      "activations/layer18_attention_weight_min": -24.938756942749023,
      "activations/layer19_attention_weight_max": 30.518789291381836,
      "activations/layer19_attention_weight_min": -30.1201171875,
      "activations/layer1_attention_weight_max": 15.908360481262207,
      "activations/layer1_attention_weight_min": -15.133442878723145,
      "activations/layer20_attention_weight_max": 28.365398406982422,
      "activations/layer20_attention_weight_min": -26.990598678588867,
      "activations/layer21_attention_weight_max": 26.604887008666992,
      "activations/layer21_attention_weight_min": -23.326261520385742,
      "activations/layer22_attention_weight_max": 39.85715866088867,
      "activations/layer22_attention_weight_min": -30.06968116760254,
      "activations/layer23_attention_weight_max": 27.876148223876953,
      "activations/layer23_attention_weight_min": -26.492061614990234,
      "activations/layer2_attention_weight_max": 34.96220016479492,
      "activations/layer2_attention_weight_min": -34.16613006591797,
      "activations/layer3_attention_weight_max": 109.3029556274414,
      "activations/layer3_attention_weight_min": -110.04256439208984,
      "activations/layer4_attention_weight_max": 114.10717010498047,
      "activations/layer4_attention_weight_min": -109.781005859375,
      "activations/layer5_attention_weight_max": 51.26432418823242,
      "activations/layer5_attention_weight_min": -64.53330993652344,
      "activations/layer6_attention_weight_max": 44.86111831665039,
      "activations/layer6_attention_weight_min": -47.661991119384766,
      "activations/layer7_attention_weight_max": 99.41932678222656,
      "activations/layer7_attention_weight_min": -101.96858978271484,
      "activations/layer8_attention_weight_max": 44.78868103027344,
      "activations/layer8_attention_weight_min": -47.269622802734375,
      "activations/layer9_attention_weight_max": 31.812009811401367,
      "activations/layer9_attention_weight_min": -35.91947555541992,
      "epoch": 21.71,
      "learning_rate": 1.0129166666666666e-05,
      "loss": 2.7143,
      "step": 373550
    },
    {
      "activations/layer0_attention_weight_max": 16.61973762512207,
      "activations/layer0_attention_weight_min": -13.999801635742188,
      "activations/layer10_attention_weight_max": 33.320655822753906,
      "activations/layer10_attention_weight_min": -36.59687042236328,
      "activations/layer11_attention_weight_max": 34.17726135253906,
      "activations/layer11_attention_weight_min": -35.71411895751953,
      "activations/layer12_attention_weight_max": 26.619266510009766,
      "activations/layer12_attention_weight_min": -26.26592254638672,
      "activations/layer13_attention_weight_max": 39.91390609741211,
      "activations/layer13_attention_weight_min": -36.2791862487793,
      "activations/layer14_attention_weight_max": 40.30585861206055,
      "activations/layer14_attention_weight_min": -36.00503921508789,
      "activations/layer15_attention_weight_max": 38.71226119995117,
      "activations/layer15_attention_weight_min": -34.48072052001953,
      "activations/layer16_attention_weight_max": 30.466047286987305,
      "activations/layer16_attention_weight_min": -28.93169593811035,
      "activations/layer17_attention_weight_max": 31.38088035583496,
      "activations/layer17_attention_weight_min": -25.03307342529297,
      "activations/layer18_attention_weight_max": 32.55860137939453,
      "activations/layer18_attention_weight_min": -25.0852108001709,
      "activations/layer19_attention_weight_max": 35.128944396972656,
      "activations/layer19_attention_weight_min": -31.068693161010742,
      "activations/layer1_attention_weight_max": 15.563136100769043,
      "activations/layer1_attention_weight_min": -15.275386810302734,
      "activations/layer20_attention_weight_max": 29.208316802978516,
      "activations/layer20_attention_weight_min": -24.8680362701416,
      "activations/layer21_attention_weight_max": 27.59954833984375,
      "activations/layer21_attention_weight_min": -24.946319580078125,
      "activations/layer22_attention_weight_max": 39.80284118652344,
      "activations/layer22_attention_weight_min": -32.388362884521484,
      "activations/layer23_attention_weight_max": 31.813060760498047,
      "activations/layer23_attention_weight_min": -28.904333114624023,
      "activations/layer2_attention_weight_max": 32.34716796875,
      "activations/layer2_attention_weight_min": -34.012821197509766,
      "activations/layer3_attention_weight_max": 105.01437377929688,
      "activations/layer3_attention_weight_min": -107.96910095214844,
      "activations/layer4_attention_weight_max": 109.00322723388672,
      "activations/layer4_attention_weight_min": -114.08534240722656,
      "activations/layer5_attention_weight_max": 48.70888900756836,
      "activations/layer5_attention_weight_min": -61.97553634643555,
      "activations/layer6_attention_weight_max": 44.34426498413086,
      "activations/layer6_attention_weight_min": -49.84213638305664,
      "activations/layer7_attention_weight_max": 94.79403686523438,
      "activations/layer7_attention_weight_min": -99.13129425048828,
      "activations/layer8_attention_weight_max": 43.78053665161133,
      "activations/layer8_attention_weight_min": -44.721561431884766,
      "activations/layer9_attention_weight_max": 32.201576232910156,
      "activations/layer9_attention_weight_min": -33.59609603881836,
      "epoch": 21.71,
      "learning_rate": 1.0110227272727273e-05,
      "loss": 2.7092,
      "step": 373600
    },
    {
      "activations/layer0_attention_weight_max": 17.025684356689453,
      "activations/layer0_attention_weight_min": -14.286733627319336,
      "activations/layer10_attention_weight_max": 37.35223388671875,
      "activations/layer10_attention_weight_min": -38.897979736328125,
      "activations/layer11_attention_weight_max": 38.8427734375,
      "activations/layer11_attention_weight_min": -40.083404541015625,
      "activations/layer12_attention_weight_max": 32.956886291503906,
      "activations/layer12_attention_weight_min": -29.248538970947266,
      "activations/layer13_attention_weight_max": 46.15215301513672,
      "activations/layer13_attention_weight_min": -36.604942321777344,
      "activations/layer14_attention_weight_max": 45.05045700073242,
      "activations/layer14_attention_weight_min": -39.245548248291016,
      "activations/layer15_attention_weight_max": 42.98488235473633,
      "activations/layer15_attention_weight_min": -39.05803680419922,
      "activations/layer16_attention_weight_max": 31.789342880249023,
      "activations/layer16_attention_weight_min": -29.867177963256836,
      "activations/layer17_attention_weight_max": 32.72648620605469,
      "activations/layer17_attention_weight_min": -26.453590393066406,
      "activations/layer18_attention_weight_max": 34.98485565185547,
      "activations/layer18_attention_weight_min": -26.577421188354492,
      "activations/layer19_attention_weight_max": 38.863868713378906,
      "activations/layer19_attention_weight_min": -33.08146286010742,
      "activations/layer1_attention_weight_max": 15.528553009033203,
      "activations/layer1_attention_weight_min": -14.462723731994629,
      "activations/layer20_attention_weight_max": 32.56389236450195,
      "activations/layer20_attention_weight_min": -27.0506591796875,
      "activations/layer21_attention_weight_max": 29.81900978088379,
      "activations/layer21_attention_weight_min": -25.30037498474121,
      "activations/layer22_attention_weight_max": 40.24128723144531,
      "activations/layer22_attention_weight_min": -30.296485900878906,
      "activations/layer23_attention_weight_max": 32.5943489074707,
      "activations/layer23_attention_weight_min": -24.15645980834961,
      "activations/layer2_attention_weight_max": 33.57631301879883,
      "activations/layer2_attention_weight_min": -34.047607421875,
      "activations/layer3_attention_weight_max": 109.2970199584961,
      "activations/layer3_attention_weight_min": -108.19416046142578,
      "activations/layer4_attention_weight_max": 117.581298828125,
      "activations/layer4_attention_weight_min": -114.03216552734375,
      "activations/layer5_attention_weight_max": 53.327632904052734,
      "activations/layer5_attention_weight_min": -61.408416748046875,
      "activations/layer6_attention_weight_max": 50.128196716308594,
      "activations/layer6_attention_weight_min": -49.754356384277344,
      "activations/layer7_attention_weight_max": 104.3955078125,
      "activations/layer7_attention_weight_min": -110.35051727294922,
      "activations/layer8_attention_weight_max": 50.84071350097656,
      "activations/layer8_attention_weight_min": -50.427799224853516,
      "activations/layer9_attention_weight_max": 36.591148376464844,
      "activations/layer9_attention_weight_min": -38.062374114990234,
      "epoch": 21.71,
      "learning_rate": 1.009128787878788e-05,
      "loss": 2.7274,
      "step": 373650
    },
    {
      "activations/layer0_attention_weight_max": 16.138959884643555,
      "activations/layer0_attention_weight_min": -13.912455558776855,
      "activations/layer10_attention_weight_max": 36.70122528076172,
      "activations/layer10_attention_weight_min": -36.99201583862305,
      "activations/layer11_attention_weight_max": 38.70033264160156,
      "activations/layer11_attention_weight_min": -36.47541046142578,
      "activations/layer12_attention_weight_max": 30.56816291809082,
      "activations/layer12_attention_weight_min": -28.98009490966797,
      "activations/layer13_attention_weight_max": 42.81179428100586,
      "activations/layer13_attention_weight_min": -38.84674835205078,
      "activations/layer14_attention_weight_max": 46.041603088378906,
      "activations/layer14_attention_weight_min": -40.523162841796875,
      "activations/layer15_attention_weight_max": 41.47710418701172,
      "activations/layer15_attention_weight_min": -39.41548156738281,
      "activations/layer16_attention_weight_max": 29.50413703918457,
      "activations/layer16_attention_weight_min": -29.044300079345703,
      "activations/layer17_attention_weight_max": 32.11104202270508,
      "activations/layer17_attention_weight_min": -30.1956729888916,
      "activations/layer18_attention_weight_max": 32.722267150878906,
      "activations/layer18_attention_weight_min": -29.756141662597656,
      "activations/layer19_attention_weight_max": 34.26743698120117,
      "activations/layer19_attention_weight_min": -35.02189636230469,
      "activations/layer1_attention_weight_max": 15.498330116271973,
      "activations/layer1_attention_weight_min": -15.299654960632324,
      "activations/layer20_attention_weight_max": 31.373910903930664,
      "activations/layer20_attention_weight_min": -28.323884963989258,
      "activations/layer21_attention_weight_max": 30.520408630371094,
      "activations/layer21_attention_weight_min": -27.68575096130371,
      "activations/layer22_attention_weight_max": 49.75210189819336,
      "activations/layer22_attention_weight_min": -35.41603088378906,
      "activations/layer23_attention_weight_max": 30.729671478271484,
      "activations/layer23_attention_weight_min": -26.645862579345703,
      "activations/layer2_attention_weight_max": 34.232364654541016,
      "activations/layer2_attention_weight_min": -35.134727478027344,
      "activations/layer3_attention_weight_max": 106.70854187011719,
      "activations/layer3_attention_weight_min": -110.16378021240234,
      "activations/layer4_attention_weight_max": 109.9244384765625,
      "activations/layer4_attention_weight_min": -119.89265441894531,
      "activations/layer5_attention_weight_max": 53.40357971191406,
      "activations/layer5_attention_weight_min": -71.75857543945312,
      "activations/layer6_attention_weight_max": 47.2277946472168,
      "activations/layer6_attention_weight_min": -51.09912872314453,
      "activations/layer7_attention_weight_max": 97.56825256347656,
      "activations/layer7_attention_weight_min": -106.81400299072266,
      "activations/layer8_attention_weight_max": 43.548824310302734,
      "activations/layer8_attention_weight_min": -47.42924118041992,
      "activations/layer9_attention_weight_max": 33.0806999206543,
      "activations/layer9_attention_weight_min": -36.04279708862305,
      "epoch": 21.71,
      "learning_rate": 1.0072348484848484e-05,
      "loss": 2.7113,
      "step": 373700
    },
    {
      "activations/layer0_attention_weight_max": 16.303571701049805,
      "activations/layer0_attention_weight_min": -14.652862548828125,
      "activations/layer10_attention_weight_max": 36.91488265991211,
      "activations/layer10_attention_weight_min": -37.053009033203125,
      "activations/layer11_attention_weight_max": 38.91891860961914,
      "activations/layer11_attention_weight_min": -36.32781982421875,
      "activations/layer12_attention_weight_max": 32.569820404052734,
      "activations/layer12_attention_weight_min": -28.984962463378906,
      "activations/layer13_attention_weight_max": 42.64253616333008,
      "activations/layer13_attention_weight_min": -38.13024139404297,
      "activations/layer14_attention_weight_max": 44.912757873535156,
      "activations/layer14_attention_weight_min": -42.308006286621094,
      "activations/layer15_attention_weight_max": 43.447303771972656,
      "activations/layer15_attention_weight_min": -39.8586540222168,
      "activations/layer16_attention_weight_max": 31.455732345581055,
      "activations/layer16_attention_weight_min": -28.98490333557129,
      "activations/layer17_attention_weight_max": 32.42298889160156,
      "activations/layer17_attention_weight_min": -26.629907608032227,
      "activations/layer18_attention_weight_max": 32.84099578857422,
      "activations/layer18_attention_weight_min": -26.103010177612305,
      "activations/layer19_attention_weight_max": 37.549468994140625,
      "activations/layer19_attention_weight_min": -31.542922973632812,
      "activations/layer1_attention_weight_max": 16.35210609436035,
      "activations/layer1_attention_weight_min": -14.541090965270996,
      "activations/layer20_attention_weight_max": 31.873424530029297,
      "activations/layer20_attention_weight_min": -25.95574951171875,
      "activations/layer21_attention_weight_max": 34.022499084472656,
      "activations/layer21_attention_weight_min": -26.430753707885742,
      "activations/layer22_attention_weight_max": 44.6239128112793,
      "activations/layer22_attention_weight_min": -31.77644157409668,
      "activations/layer23_attention_weight_max": 33.45409393310547,
      "activations/layer23_attention_weight_min": -26.166181564331055,
      "activations/layer2_attention_weight_max": 33.03596496582031,
      "activations/layer2_attention_weight_min": -33.53988265991211,
      "activations/layer3_attention_weight_max": 107.25788879394531,
      "activations/layer3_attention_weight_min": -106.34452819824219,
      "activations/layer4_attention_weight_max": 112.4508056640625,
      "activations/layer4_attention_weight_min": -109.68544006347656,
      "activations/layer5_attention_weight_max": 51.535308837890625,
      "activations/layer5_attention_weight_min": -61.93882751464844,
      "activations/layer6_attention_weight_max": 47.0080451965332,
      "activations/layer6_attention_weight_min": -47.732059478759766,
      "activations/layer7_attention_weight_max": 99.7255630493164,
      "activations/layer7_attention_weight_min": -103.70535278320312,
      "activations/layer8_attention_weight_max": 44.24562454223633,
      "activations/layer8_attention_weight_min": -46.798458099365234,
      "activations/layer9_attention_weight_max": 34.722110748291016,
      "activations/layer9_attention_weight_min": -35.50599670410156,
      "epoch": 21.72,
      "learning_rate": 1.005340909090909e-05,
      "loss": 2.6972,
      "step": 373750
    },
    {
      "activations/layer0_attention_weight_max": 16.849224090576172,
      "activations/layer0_attention_weight_min": -14.517818450927734,
      "activations/layer10_attention_weight_max": 33.7836799621582,
      "activations/layer10_attention_weight_min": -34.58694076538086,
      "activations/layer11_attention_weight_max": 35.2392463684082,
      "activations/layer11_attention_weight_min": -34.9038200378418,
      "activations/layer12_attention_weight_max": 27.920963287353516,
      "activations/layer12_attention_weight_min": -27.14879608154297,
      "activations/layer13_attention_weight_max": 40.50727844238281,
      "activations/layer13_attention_weight_min": -36.81889343261719,
      "activations/layer14_attention_weight_max": 44.17507553100586,
      "activations/layer14_attention_weight_min": -41.096343994140625,
      "activations/layer15_attention_weight_max": 38.92744445800781,
      "activations/layer15_attention_weight_min": -36.49570846557617,
      "activations/layer16_attention_weight_max": 30.506437301635742,
      "activations/layer16_attention_weight_min": -28.706037521362305,
      "activations/layer17_attention_weight_max": 29.38042640686035,
      "activations/layer17_attention_weight_min": -27.19782257080078,
      "activations/layer18_attention_weight_max": 33.59467697143555,
      "activations/layer18_attention_weight_min": -25.693754196166992,
      "activations/layer19_attention_weight_max": 35.06095504760742,
      "activations/layer19_attention_weight_min": -35.37956619262695,
      "activations/layer1_attention_weight_max": 16.165193557739258,
      "activations/layer1_attention_weight_min": -14.931407928466797,
      "activations/layer20_attention_weight_max": 30.303417205810547,
      "activations/layer20_attention_weight_min": -29.342138290405273,
      "activations/layer21_attention_weight_max": 30.599430084228516,
      "activations/layer21_attention_weight_min": -26.700956344604492,
      "activations/layer22_attention_weight_max": 48.79399871826172,
      "activations/layer22_attention_weight_min": -33.23384094238281,
      "activations/layer23_attention_weight_max": 33.592864990234375,
      "activations/layer23_attention_weight_min": -26.493408203125,
      "activations/layer2_attention_weight_max": 33.67917251586914,
      "activations/layer2_attention_weight_min": -32.81059265136719,
      "activations/layer3_attention_weight_max": 107.1574935913086,
      "activations/layer3_attention_weight_min": -107.11911010742188,
      "activations/layer4_attention_weight_max": 112.91207122802734,
      "activations/layer4_attention_weight_min": -114.2841567993164,
      "activations/layer5_attention_weight_max": 55.124080657958984,
      "activations/layer5_attention_weight_min": -64.50320434570312,
      "activations/layer6_attention_weight_max": 46.17776107788086,
      "activations/layer6_attention_weight_min": -50.26667785644531,
      "activations/layer7_attention_weight_max": 99.87361145019531,
      "activations/layer7_attention_weight_min": -102.15530395507812,
      "activations/layer8_attention_weight_max": 42.408447265625,
      "activations/layer8_attention_weight_min": -46.61643981933594,
      "activations/layer9_attention_weight_max": 32.42820358276367,
      "activations/layer9_attention_weight_min": -33.98983383178711,
      "epoch": 21.72,
      "learning_rate": 1.0034469696969696e-05,
      "loss": 2.725,
      "step": 373800
    },
    {
      "activations/layer0_attention_weight_max": 16.55417251586914,
      "activations/layer0_attention_weight_min": -14.625348091125488,
      "activations/layer10_attention_weight_max": 38.5095100402832,
      "activations/layer10_attention_weight_min": -38.578060150146484,
      "activations/layer11_attention_weight_max": 37.91240692138672,
      "activations/layer11_attention_weight_min": -38.92060089111328,
      "activations/layer12_attention_weight_max": 30.16973304748535,
      "activations/layer12_attention_weight_min": -30.467493057250977,
      "activations/layer13_attention_weight_max": 43.552974700927734,
      "activations/layer13_attention_weight_min": -39.90157699584961,
      "activations/layer14_attention_weight_max": 47.585968017578125,
      "activations/layer14_attention_weight_min": -42.29523468017578,
      "activations/layer15_attention_weight_max": 44.06845474243164,
      "activations/layer15_attention_weight_min": -39.06827926635742,
      "activations/layer16_attention_weight_max": 32.35124206542969,
      "activations/layer16_attention_weight_min": -29.90213394165039,
      "activations/layer17_attention_weight_max": 31.694496154785156,
      "activations/layer17_attention_weight_min": -27.72917366027832,
      "activations/layer18_attention_weight_max": 35.14049530029297,
      "activations/layer18_attention_weight_min": -27.15493392944336,
      "activations/layer19_attention_weight_max": 36.01223373413086,
      "activations/layer19_attention_weight_min": -32.10911178588867,
      "activations/layer1_attention_weight_max": 16.429798126220703,
      "activations/layer1_attention_weight_min": -16.311763763427734,
      "activations/layer20_attention_weight_max": 29.932344436645508,
      "activations/layer20_attention_weight_min": -26.665441513061523,
      "activations/layer21_attention_weight_max": 29.878610610961914,
      "activations/layer21_attention_weight_min": -26.197111129760742,
      "activations/layer22_attention_weight_max": 44.51620101928711,
      "activations/layer22_attention_weight_min": -33.84055709838867,
      "activations/layer23_attention_weight_max": 32.57759094238281,
      "activations/layer23_attention_weight_min": -26.33118438720703,
      "activations/layer2_attention_weight_max": 34.192283630371094,
      "activations/layer2_attention_weight_min": -33.6478271484375,
      "activations/layer3_attention_weight_max": 107.10944366455078,
      "activations/layer3_attention_weight_min": -107.38249206542969,
      "activations/layer4_attention_weight_max": 112.7049331665039,
      "activations/layer4_attention_weight_min": -115.07108306884766,
      "activations/layer5_attention_weight_max": 51.6718864440918,
      "activations/layer5_attention_weight_min": -59.816993713378906,
      "activations/layer6_attention_weight_max": 47.24618148803711,
      "activations/layer6_attention_weight_min": -50.39600372314453,
      "activations/layer7_attention_weight_max": 99.28297424316406,
      "activations/layer7_attention_weight_min": -112.79219055175781,
      "activations/layer8_attention_weight_max": 47.20904541015625,
      "activations/layer8_attention_weight_min": -47.98289108276367,
      "activations/layer9_attention_weight_max": 37.567176818847656,
      "activations/layer9_attention_weight_min": -39.70459747314453,
      "epoch": 21.72,
      "learning_rate": 1.0015530303030302e-05,
      "loss": 2.7185,
      "step": 373850
    },
    {
      "activations/layer0_attention_weight_max": 17.345096588134766,
      "activations/layer0_attention_weight_min": -13.911911964416504,
      "activations/layer10_attention_weight_max": 35.131874084472656,
      "activations/layer10_attention_weight_min": -38.7005500793457,
      "activations/layer11_attention_weight_max": 35.681243896484375,
      "activations/layer11_attention_weight_min": -36.028785705566406,
      "activations/layer12_attention_weight_max": 26.645069122314453,
      "activations/layer12_attention_weight_min": -26.997211456298828,
      "activations/layer13_attention_weight_max": 40.88892364501953,
      "activations/layer13_attention_weight_min": -36.157169342041016,
      "activations/layer14_attention_weight_max": 47.63454055786133,
      "activations/layer14_attention_weight_min": -39.554073333740234,
      "activations/layer15_attention_weight_max": 44.74478530883789,
      "activations/layer15_attention_weight_min": -37.552303314208984,
      "activations/layer16_attention_weight_max": 33.48941421508789,
      "activations/layer16_attention_weight_min": -30.910282135009766,
      "activations/layer17_attention_weight_max": 32.50353240966797,
      "activations/layer17_attention_weight_min": -27.010692596435547,
      "activations/layer18_attention_weight_max": 36.58127975463867,
      "activations/layer18_attention_weight_min": -29.331632614135742,
      "activations/layer19_attention_weight_max": 37.68635177612305,
      "activations/layer19_attention_weight_min": -32.4776611328125,
      "activations/layer1_attention_weight_max": 15.503480911254883,
      "activations/layer1_attention_weight_min": -16.37946891784668,
      "activations/layer20_attention_weight_max": 32.61467361450195,
      "activations/layer20_attention_weight_min": -26.51915168762207,
      "activations/layer21_attention_weight_max": 30.39206314086914,
      "activations/layer21_attention_weight_min": -24.595993041992188,
      "activations/layer22_attention_weight_max": 46.58372497558594,
      "activations/layer22_attention_weight_min": -32.080345153808594,
      "activations/layer23_attention_weight_max": 35.20962905883789,
      "activations/layer23_attention_weight_min": -25.875225067138672,
      "activations/layer2_attention_weight_max": 33.34572982788086,
      "activations/layer2_attention_weight_min": -32.812442779541016,
      "activations/layer3_attention_weight_max": 105.35877990722656,
      "activations/layer3_attention_weight_min": -106.49407196044922,
      "activations/layer4_attention_weight_max": 109.69998931884766,
      "activations/layer4_attention_weight_min": -117.3275146484375,
      "activations/layer5_attention_weight_max": 49.609981536865234,
      "activations/layer5_attention_weight_min": -62.69548034667969,
      "activations/layer6_attention_weight_max": 46.07950210571289,
      "activations/layer6_attention_weight_min": -50.363075256347656,
      "activations/layer7_attention_weight_max": 99.94097137451172,
      "activations/layer7_attention_weight_min": -104.46915435791016,
      "activations/layer8_attention_weight_max": 42.57435989379883,
      "activations/layer8_attention_weight_min": -47.99553680419922,
      "activations/layer9_attention_weight_max": 32.89149856567383,
      "activations/layer9_attention_weight_min": -38.98091125488281,
      "epoch": 21.73,
      "learning_rate": 9.996590909090909e-06,
      "loss": 2.7066,
      "step": 373900
    },
    {
      "activations/layer0_attention_weight_max": 17.238452911376953,
      "activations/layer0_attention_weight_min": -15.8320894241333,
      "activations/layer10_attention_weight_max": 38.150814056396484,
      "activations/layer10_attention_weight_min": -37.79869842529297,
      "activations/layer11_attention_weight_max": 35.502254486083984,
      "activations/layer11_attention_weight_min": -37.655250549316406,
      "activations/layer12_attention_weight_max": 26.463825225830078,
      "activations/layer12_attention_weight_min": -28.072233200073242,
      "activations/layer13_attention_weight_max": 40.764705657958984,
      "activations/layer13_attention_weight_min": -40.15153884887695,
      "activations/layer14_attention_weight_max": 41.36177444458008,
      "activations/layer14_attention_weight_min": -40.46907424926758,
      "activations/layer15_attention_weight_max": 38.65519332885742,
      "activations/layer15_attention_weight_min": -38.350685119628906,
      "activations/layer16_attention_weight_max": 28.436805725097656,
      "activations/layer16_attention_weight_min": -28.92698860168457,
      "activations/layer17_attention_weight_max": 34.54829406738281,
      "activations/layer17_attention_weight_min": -27.224239349365234,
      "activations/layer18_attention_weight_max": 36.89515686035156,
      "activations/layer18_attention_weight_min": -28.48440170288086,
      "activations/layer19_attention_weight_max": 35.951934814453125,
      "activations/layer19_attention_weight_min": -32.390235900878906,
      "activations/layer1_attention_weight_max": 17.709089279174805,
      "activations/layer1_attention_weight_min": -15.371602058410645,
      "activations/layer20_attention_weight_max": 33.82090759277344,
      "activations/layer20_attention_weight_min": -28.195470809936523,
      "activations/layer21_attention_weight_max": 32.4658317565918,
      "activations/layer21_attention_weight_min": -27.38918113708496,
      "activations/layer22_attention_weight_max": 43.00217819213867,
      "activations/layer22_attention_weight_min": -33.507774353027344,
      "activations/layer23_attention_weight_max": 31.443357467651367,
      "activations/layer23_attention_weight_min": -25.526723861694336,
      "activations/layer2_attention_weight_max": 34.42658233642578,
      "activations/layer2_attention_weight_min": -34.02444839477539,
      "activations/layer3_attention_weight_max": 108.0642318725586,
      "activations/layer3_attention_weight_min": -109.00019073486328,
      "activations/layer4_attention_weight_max": 111.41578674316406,
      "activations/layer4_attention_weight_min": -114.75399017333984,
      "activations/layer5_attention_weight_max": 52.39387130737305,
      "activations/layer5_attention_weight_min": -64.08883666992188,
      "activations/layer6_attention_weight_max": 48.38274383544922,
      "activations/layer6_attention_weight_min": -52.232444763183594,
      "activations/layer7_attention_weight_max": 102.30927276611328,
      "activations/layer7_attention_weight_min": -113.04589080810547,
      "activations/layer8_attention_weight_max": 46.14641571044922,
      "activations/layer8_attention_weight_min": -47.75934600830078,
      "activations/layer9_attention_weight_max": 33.95942306518555,
      "activations/layer9_attention_weight_min": -38.404788970947266,
      "epoch": 21.73,
      "learning_rate": 9.977651515151514e-06,
      "loss": 2.7212,
      "step": 373950
    },
    {
      "activations/layer0_attention_weight_max": 16.41750717163086,
      "activations/layer0_attention_weight_min": -15.571764945983887,
      "activations/layer10_attention_weight_max": 35.950904846191406,
      "activations/layer10_attention_weight_min": -35.558963775634766,
      "activations/layer11_attention_weight_max": 32.513084411621094,
      "activations/layer11_attention_weight_min": -36.57229232788086,
      "activations/layer12_attention_weight_max": 33.06357955932617,
      "activations/layer12_attention_weight_min": -27.825122833251953,
      "activations/layer13_attention_weight_max": 42.90983963012695,
      "activations/layer13_attention_weight_min": -35.96422576904297,
      "activations/layer14_attention_weight_max": 45.761924743652344,
      "activations/layer14_attention_weight_min": -39.540061950683594,
      "activations/layer15_attention_weight_max": 41.81615447998047,
      "activations/layer15_attention_weight_min": -36.118831634521484,
      "activations/layer16_attention_weight_max": 31.302261352539062,
      "activations/layer16_attention_weight_min": -27.70549201965332,
      "activations/layer17_attention_weight_max": 33.5235595703125,
      "activations/layer17_attention_weight_min": -26.95368194580078,
      "activations/layer18_attention_weight_max": 38.46278381347656,
      "activations/layer18_attention_weight_min": -27.097261428833008,
      "activations/layer19_attention_weight_max": 37.87428283691406,
      "activations/layer19_attention_weight_min": -32.077022552490234,
      "activations/layer1_attention_weight_max": 16.119525909423828,
      "activations/layer1_attention_weight_min": -14.964129447937012,
      "activations/layer20_attention_weight_max": 31.96840476989746,
      "activations/layer20_attention_weight_min": -25.480022430419922,
      "activations/layer21_attention_weight_max": 35.482479095458984,
      "activations/layer21_attention_weight_min": -26.064043045043945,
      "activations/layer22_attention_weight_max": 45.51844787597656,
      "activations/layer22_attention_weight_min": -33.108211517333984,
      "activations/layer23_attention_weight_max": 35.48137283325195,
      "activations/layer23_attention_weight_min": -25.99262046813965,
      "activations/layer2_attention_weight_max": 33.760826110839844,
      "activations/layer2_attention_weight_min": -33.638572692871094,
      "activations/layer3_attention_weight_max": 104.4467544555664,
      "activations/layer3_attention_weight_min": -104.29753875732422,
      "activations/layer4_attention_weight_max": 110.31319427490234,
      "activations/layer4_attention_weight_min": -113.5414047241211,
      "activations/layer5_attention_weight_max": 50.53412628173828,
      "activations/layer5_attention_weight_min": -60.860252380371094,
      "activations/layer6_attention_weight_max": 46.212947845458984,
      "activations/layer6_attention_weight_min": -47.068580627441406,
      "activations/layer7_attention_weight_max": 101.256103515625,
      "activations/layer7_attention_weight_min": -99.47850036621094,
      "activations/layer8_attention_weight_max": 48.251190185546875,
      "activations/layer8_attention_weight_min": -47.836280822753906,
      "activations/layer9_attention_weight_max": 34.041038513183594,
      "activations/layer9_attention_weight_min": -36.30720138549805,
      "epoch": 21.73,
      "learning_rate": 9.95871212121212e-06,
      "loss": 2.7038,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.5246,
      "eval_samples_per_second": 503.718,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.5246,
      "eval_openwebtext_samples_per_second": 503.718,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9668,
      "eval_wikitext_samples_per_second": 231.847,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_lambada_loss": 2.29296875,
      "eval_lambada_ppl": 9.904297461775021,
      "eval_lambada_runtime": 9.538,
      "eval_lambada_samples_per_second": 510.486,
      "step": 374000
    },
    {
      "activations/layer0_attention_weight_max": 17.709550857543945,
      "activations/layer0_attention_weight_min": -14.54536247253418,
      "activations/layer10_attention_weight_max": 36.1868896484375,
      "activations/layer10_attention_weight_min": -35.070899963378906,
      "activations/layer11_attention_weight_max": 37.81782531738281,
      "activations/layer11_attention_weight_min": -36.79097366333008,
      "activations/layer12_attention_weight_max": 26.691795349121094,
      "activations/layer12_attention_weight_min": -27.97095489501953,
      "activations/layer13_attention_weight_max": 39.994651794433594,
      "activations/layer13_attention_weight_min": -35.16919708251953,
      "activations/layer14_attention_weight_max": 39.034339904785156,
      "activations/layer14_attention_weight_min": -38.998775482177734,
      "activations/layer15_attention_weight_max": 38.696102142333984,
      "activations/layer15_attention_weight_min": -36.01032638549805,
      "activations/layer16_attention_weight_max": 27.54290199279785,
      "activations/layer16_attention_weight_min": -28.53192710876465,
      "activations/layer17_attention_weight_max": 28.99128532409668,
      "activations/layer17_attention_weight_min": -25.018901824951172,
      "activations/layer18_attention_weight_max": 33.12968444824219,
      "activations/layer18_attention_weight_min": -25.37010383605957,
      "activations/layer19_attention_weight_max": 33.835201263427734,
      "activations/layer19_attention_weight_min": -31.535856246948242,
      "activations/layer1_attention_weight_max": 15.44306468963623,
      "activations/layer1_attention_weight_min": -15.005899429321289,
      "activations/layer20_attention_weight_max": 27.647233963012695,
      "activations/layer20_attention_weight_min": -25.717723846435547,
      "activations/layer21_attention_weight_max": 29.99802017211914,
      "activations/layer21_attention_weight_min": -26.39621925354004,
      "activations/layer22_attention_weight_max": 38.775634765625,
      "activations/layer22_attention_weight_min": -33.443172454833984,
      "activations/layer23_attention_weight_max": 32.59107208251953,
      "activations/layer23_attention_weight_min": -27.897619247436523,
      "activations/layer2_attention_weight_max": 34.39015579223633,
      "activations/layer2_attention_weight_min": -33.750118255615234,
      "activations/layer3_attention_weight_max": 108.24078369140625,
      "activations/layer3_attention_weight_min": -107.09947967529297,
      "activations/layer4_attention_weight_max": 114.7219009399414,
      "activations/layer4_attention_weight_min": -112.3756103515625,
      "activations/layer5_attention_weight_max": 52.96918487548828,
      "activations/layer5_attention_weight_min": -61.992950439453125,
      "activations/layer6_attention_weight_max": 45.2498893737793,
      "activations/layer6_attention_weight_min": -47.86617660522461,
      "activations/layer7_attention_weight_max": 100.06517028808594,
      "activations/layer7_attention_weight_min": -102.80242156982422,
      "activations/layer8_attention_weight_max": 43.54220962524414,
      "activations/layer8_attention_weight_min": -44.68904495239258,
      "activations/layer9_attention_weight_max": 31.56369972229004,
      "activations/layer9_attention_weight_min": -33.56892776489258,
      "epoch": 21.73,
      "learning_rate": 9.939772727272727e-06,
      "loss": 2.7098,
      "step": 374050
    },
    {
      "activations/layer0_attention_weight_max": 16.763025283813477,
      "activations/layer0_attention_weight_min": -14.360809326171875,
      "activations/layer10_attention_weight_max": 33.88544464111328,
      "activations/layer10_attention_weight_min": -34.97517395019531,
      "activations/layer11_attention_weight_max": 34.034297943115234,
      "activations/layer11_attention_weight_min": -36.169952392578125,
      "activations/layer12_attention_weight_max": 27.11612319946289,
      "activations/layer12_attention_weight_min": -25.365604400634766,
      "activations/layer13_attention_weight_max": 41.1580924987793,
      "activations/layer13_attention_weight_min": -34.94013214111328,
      "activations/layer14_attention_weight_max": 42.22484588623047,
      "activations/layer14_attention_weight_min": -38.01852035522461,
      "activations/layer15_attention_weight_max": 39.20805740356445,
      "activations/layer15_attention_weight_min": -36.28459167480469,
      "activations/layer16_attention_weight_max": 27.56516456604004,
      "activations/layer16_attention_weight_min": -27.997859954833984,
      "activations/layer17_attention_weight_max": 28.90488052368164,
      "activations/layer17_attention_weight_min": -26.78443717956543,
      "activations/layer18_attention_weight_max": 32.762535095214844,
      "activations/layer18_attention_weight_min": -26.125749588012695,
      "activations/layer19_attention_weight_max": 30.516220092773438,
      "activations/layer19_attention_weight_min": -30.759565353393555,
      "activations/layer1_attention_weight_max": 17.22175407409668,
      "activations/layer1_attention_weight_min": -15.81972599029541,
      "activations/layer20_attention_weight_max": 25.569303512573242,
      "activations/layer20_attention_weight_min": -25.073413848876953,
      "activations/layer21_attention_weight_max": 26.618375778198242,
      "activations/layer21_attention_weight_min": -24.165307998657227,
      "activations/layer22_attention_weight_max": 37.687522888183594,
      "activations/layer22_attention_weight_min": -32.350093841552734,
      "activations/layer23_attention_weight_max": 30.393020629882812,
      "activations/layer23_attention_weight_min": -27.70412826538086,
      "activations/layer2_attention_weight_max": 34.827354431152344,
      "activations/layer2_attention_weight_min": -32.93441390991211,
      "activations/layer3_attention_weight_max": 106.16819763183594,
      "activations/layer3_attention_weight_min": -104.1785888671875,
      "activations/layer4_attention_weight_max": 112.26385498046875,
      "activations/layer4_attention_weight_min": -115.78059387207031,
      "activations/layer5_attention_weight_max": 51.38993453979492,
      "activations/layer5_attention_weight_min": -63.3253173828125,
      "activations/layer6_attention_weight_max": 45.56095504760742,
      "activations/layer6_attention_weight_min": -51.9863395690918,
      "activations/layer7_attention_weight_max": 98.64286804199219,
      "activations/layer7_attention_weight_min": -97.69054412841797,
      "activations/layer8_attention_weight_max": 44.328121185302734,
      "activations/layer8_attention_weight_min": -44.71639633178711,
      "activations/layer9_attention_weight_max": 35.36753463745117,
      "activations/layer9_attention_weight_min": -34.82902145385742,
      "epoch": 21.74,
      "learning_rate": 9.920833333333332e-06,
      "loss": 2.7022,
      "step": 374100
    },
    {
      "activations/layer0_attention_weight_max": 16.9620304107666,
      "activations/layer0_attention_weight_min": -14.296871185302734,
      "activations/layer10_attention_weight_max": 39.739498138427734,
      "activations/layer10_attention_weight_min": -36.646644592285156,
      "activations/layer11_attention_weight_max": 37.501834869384766,
      "activations/layer11_attention_weight_min": -36.16415023803711,
      "activations/layer12_attention_weight_max": 27.085569381713867,
      "activations/layer12_attention_weight_min": -26.890426635742188,
      "activations/layer13_attention_weight_max": 39.51893997192383,
      "activations/layer13_attention_weight_min": -35.716026306152344,
      "activations/layer14_attention_weight_max": 44.54208755493164,
      "activations/layer14_attention_weight_min": -36.77523422241211,
      "activations/layer15_attention_weight_max": 39.445369720458984,
      "activations/layer15_attention_weight_min": -35.01531219482422,
      "activations/layer16_attention_weight_max": 29.333709716796875,
      "activations/layer16_attention_weight_min": -27.64314842224121,
      "activations/layer17_attention_weight_max": 29.487274169921875,
      "activations/layer17_attention_weight_min": -27.056764602661133,
      "activations/layer18_attention_weight_max": 37.024818420410156,
      "activations/layer18_attention_weight_min": -30.448837280273438,
      "activations/layer19_attention_weight_max": 36.188446044921875,
      "activations/layer19_attention_weight_min": -33.720909118652344,
      "activations/layer1_attention_weight_max": 16.876338958740234,
      "activations/layer1_attention_weight_min": -15.13249397277832,
      "activations/layer20_attention_weight_max": 30.984954833984375,
      "activations/layer20_attention_weight_min": -27.097026824951172,
      "activations/layer21_attention_weight_max": 30.47336769104004,
      "activations/layer21_attention_weight_min": -26.735240936279297,
      "activations/layer22_attention_weight_max": 48.695030212402344,
      "activations/layer22_attention_weight_min": -36.1032829284668,
      "activations/layer23_attention_weight_max": 34.88938522338867,
      "activations/layer23_attention_weight_min": -26.948707580566406,
      "activations/layer2_attention_weight_max": 32.94254684448242,
      "activations/layer2_attention_weight_min": -32.999114990234375,
      "activations/layer3_attention_weight_max": 103.78775024414062,
      "activations/layer3_attention_weight_min": -106.25528717041016,
      "activations/layer4_attention_weight_max": 108.654052734375,
      "activations/layer4_attention_weight_min": -117.37060546875,
      "activations/layer5_attention_weight_max": 54.03640365600586,
      "activations/layer5_attention_weight_min": -63.023223876953125,
      "activations/layer6_attention_weight_max": 45.655128479003906,
      "activations/layer6_attention_weight_min": -47.55165481567383,
      "activations/layer7_attention_weight_max": 95.57025146484375,
      "activations/layer7_attention_weight_min": -103.60258483886719,
      "activations/layer8_attention_weight_max": 43.51911926269531,
      "activations/layer8_attention_weight_min": -45.6815185546875,
      "activations/layer9_attention_weight_max": 36.72390365600586,
      "activations/layer9_attention_weight_min": -37.30439376831055,
      "epoch": 21.74,
      "learning_rate": 9.901893939393938e-06,
      "loss": 2.7019,
      "step": 374150
    },
    {
      "activations/layer0_attention_weight_max": 16.691810607910156,
      "activations/layer0_attention_weight_min": -13.963237762451172,
      "activations/layer10_attention_weight_max": 34.127410888671875,
      "activations/layer10_attention_weight_min": -35.67804718017578,
      "activations/layer11_attention_weight_max": 33.710487365722656,
      "activations/layer11_attention_weight_min": -37.034706115722656,
      "activations/layer12_attention_weight_max": 26.2128963470459,
      "activations/layer12_attention_weight_min": -27.79462242126465,
      "activations/layer13_attention_weight_max": 39.55331039428711,
      "activations/layer13_attention_weight_min": -35.916107177734375,
      "activations/layer14_attention_weight_max": 44.41934585571289,
      "activations/layer14_attention_weight_min": -42.24203109741211,
      "activations/layer15_attention_weight_max": 40.170589447021484,
      "activations/layer15_attention_weight_min": -40.09098434448242,
      "activations/layer16_attention_weight_max": 31.033430099487305,
      "activations/layer16_attention_weight_min": -28.97446632385254,
      "activations/layer17_attention_weight_max": 29.31399154663086,
      "activations/layer17_attention_weight_min": -26.945674896240234,
      "activations/layer18_attention_weight_max": 30.851396560668945,
      "activations/layer18_attention_weight_min": -27.438392639160156,
      "activations/layer19_attention_weight_max": 34.71334457397461,
      "activations/layer19_attention_weight_min": -34.505882263183594,
      "activations/layer1_attention_weight_max": 17.36612319946289,
      "activations/layer1_attention_weight_min": -15.685467720031738,
      "activations/layer20_attention_weight_max": 27.212574005126953,
      "activations/layer20_attention_weight_min": -27.347217559814453,
      "activations/layer21_attention_weight_max": 26.831029891967773,
      "activations/layer21_attention_weight_min": -27.95916748046875,
      "activations/layer22_attention_weight_max": 44.73282241821289,
      "activations/layer22_attention_weight_min": -32.22006607055664,
      "activations/layer23_attention_weight_max": 30.875526428222656,
      "activations/layer23_attention_weight_min": -26.66434097290039,
      "activations/layer2_attention_weight_max": 34.241912841796875,
      "activations/layer2_attention_weight_min": -33.04372024536133,
      "activations/layer3_attention_weight_max": 109.72357940673828,
      "activations/layer3_attention_weight_min": -106.08442687988281,
      "activations/layer4_attention_weight_max": 111.58418273925781,
      "activations/layer4_attention_weight_min": -116.28584289550781,
      "activations/layer5_attention_weight_max": 51.89345932006836,
      "activations/layer5_attention_weight_min": -66.72998046875,
      "activations/layer6_attention_weight_max": 47.152339935302734,
      "activations/layer6_attention_weight_min": -49.069942474365234,
      "activations/layer7_attention_weight_max": 99.13201904296875,
      "activations/layer7_attention_weight_min": -106.52281188964844,
      "activations/layer8_attention_weight_max": 42.78285598754883,
      "activations/layer8_attention_weight_min": -49.63947296142578,
      "activations/layer9_attention_weight_max": 32.23551559448242,
      "activations/layer9_attention_weight_min": -37.45166778564453,
      "epoch": 21.74,
      "learning_rate": 9.882954545454543e-06,
      "loss": 2.709,
      "step": 374200
    },
    {
      "activations/layer0_attention_weight_max": 17.449926376342773,
      "activations/layer0_attention_weight_min": -13.315388679504395,
      "activations/layer10_attention_weight_max": 33.832763671875,
      "activations/layer10_attention_weight_min": -35.45984649658203,
      "activations/layer11_attention_weight_max": 34.433353424072266,
      "activations/layer11_attention_weight_min": -35.91102981567383,
      "activations/layer12_attention_weight_max": 26.87456512451172,
      "activations/layer12_attention_weight_min": -26.644502639770508,
      "activations/layer13_attention_weight_max": 40.33707809448242,
      "activations/layer13_attention_weight_min": -37.612083435058594,
      "activations/layer14_attention_weight_max": 43.248268127441406,
      "activations/layer14_attention_weight_min": -40.31072235107422,
      "activations/layer15_attention_weight_max": 41.00876998901367,
      "activations/layer15_attention_weight_min": -37.14191436767578,
      "activations/layer16_attention_weight_max": 33.830814361572266,
      "activations/layer16_attention_weight_min": -28.335742950439453,
      "activations/layer17_attention_weight_max": 31.500024795532227,
      "activations/layer17_attention_weight_min": -25.41893768310547,
      "activations/layer18_attention_weight_max": 32.12396240234375,
      "activations/layer18_attention_weight_min": -27.146638870239258,
      "activations/layer19_attention_weight_max": 36.028446197509766,
      "activations/layer19_attention_weight_min": -31.054107666015625,
      "activations/layer1_attention_weight_max": 15.662104606628418,
      "activations/layer1_attention_weight_min": -15.83902359008789,
      "activations/layer20_attention_weight_max": 30.505964279174805,
      "activations/layer20_attention_weight_min": -24.832639694213867,
      "activations/layer21_attention_weight_max": 32.753055572509766,
      "activations/layer21_attention_weight_min": -23.49749755859375,
      "activations/layer22_attention_weight_max": 41.66960525512695,
      "activations/layer22_attention_weight_min": -30.939056396484375,
      "activations/layer23_attention_weight_max": 31.477989196777344,
      "activations/layer23_attention_weight_min": -27.267505645751953,
      "activations/layer2_attention_weight_max": 33.47556686401367,
      "activations/layer2_attention_weight_min": -34.76499938964844,
      "activations/layer3_attention_weight_max": 107.04581451416016,
      "activations/layer3_attention_weight_min": -105.75333404541016,
      "activations/layer4_attention_weight_max": 112.88197326660156,
      "activations/layer4_attention_weight_min": -112.79615783691406,
      "activations/layer5_attention_weight_max": 53.15024948120117,
      "activations/layer5_attention_weight_min": -62.3885612487793,
      "activations/layer6_attention_weight_max": 47.3115119934082,
      "activations/layer6_attention_weight_min": -50.038841247558594,
      "activations/layer7_attention_weight_max": 103.97169494628906,
      "activations/layer7_attention_weight_min": -98.82823181152344,
      "activations/layer8_attention_weight_max": 42.696781158447266,
      "activations/layer8_attention_weight_min": -43.10944747924805,
      "activations/layer9_attention_weight_max": 32.631431579589844,
      "activations/layer9_attention_weight_min": -35.86841583251953,
      "epoch": 21.75,
      "learning_rate": 9.86401515151515e-06,
      "loss": 2.7154,
      "step": 374250
    },
    {
      "activations/layer0_attention_weight_max": 16.917102813720703,
      "activations/layer0_attention_weight_min": -13.12003231048584,
      "activations/layer10_attention_weight_max": 38.512386322021484,
      "activations/layer10_attention_weight_min": -36.798057556152344,
      "activations/layer11_attention_weight_max": 36.39070129394531,
      "activations/layer11_attention_weight_min": -35.914981842041016,
      "activations/layer12_attention_weight_max": 32.397281646728516,
      "activations/layer12_attention_weight_min": -30.645694732666016,
      "activations/layer13_attention_weight_max": 47.21698760986328,
      "activations/layer13_attention_weight_min": -41.97031784057617,
      "activations/layer14_attention_weight_max": 49.742488861083984,
      "activations/layer14_attention_weight_min": -46.38042068481445,
      "activations/layer15_attention_weight_max": 50.81355285644531,
      "activations/layer15_attention_weight_min": -44.52463150024414,
      "activations/layer16_attention_weight_max": 32.99130630493164,
      "activations/layer16_attention_weight_min": -30.111818313598633,
      "activations/layer17_attention_weight_max": 32.213417053222656,
      "activations/layer17_attention_weight_min": -31.223466873168945,
      "activations/layer18_attention_weight_max": 34.377540588378906,
      "activations/layer18_attention_weight_min": -28.592233657836914,
      "activations/layer19_attention_weight_max": 36.36235809326172,
      "activations/layer19_attention_weight_min": -33.39443588256836,
      "activations/layer1_attention_weight_max": 15.204562187194824,
      "activations/layer1_attention_weight_min": -15.348844528198242,
      "activations/layer20_attention_weight_max": 30.76813316345215,
      "activations/layer20_attention_weight_min": -25.598278045654297,
      "activations/layer21_attention_weight_max": 33.344635009765625,
      "activations/layer21_attention_weight_min": -26.348087310791016,
      "activations/layer22_attention_weight_max": 48.865570068359375,
      "activations/layer22_attention_weight_min": -37.07400894165039,
      "activations/layer23_attention_weight_max": 34.73719024658203,
      "activations/layer23_attention_weight_min": -26.882139205932617,
      "activations/layer2_attention_weight_max": 32.77787399291992,
      "activations/layer2_attention_weight_min": -32.4617919921875,
      "activations/layer3_attention_weight_max": 102.46026611328125,
      "activations/layer3_attention_weight_min": -106.05567169189453,
      "activations/layer4_attention_weight_max": 106.17687225341797,
      "activations/layer4_attention_weight_min": -117.93888092041016,
      "activations/layer5_attention_weight_max": 49.433372497558594,
      "activations/layer5_attention_weight_min": -62.834938049316406,
      "activations/layer6_attention_weight_max": 44.716773986816406,
      "activations/layer6_attention_weight_min": -49.11217498779297,
      "activations/layer7_attention_weight_max": 96.46673583984375,
      "activations/layer7_attention_weight_min": -102.13668823242188,
      "activations/layer8_attention_weight_max": 43.296287536621094,
      "activations/layer8_attention_weight_min": -48.03928756713867,
      "activations/layer9_attention_weight_max": 35.79341125488281,
      "activations/layer9_attention_weight_min": -35.18131637573242,
      "epoch": 21.75,
      "learning_rate": 9.845075757575756e-06,
      "loss": 2.7268,
      "step": 374300
    },
    {
      "activations/layer0_attention_weight_max": 16.798967361450195,
      "activations/layer0_attention_weight_min": -15.695571899414062,
      "activations/layer10_attention_weight_max": 38.65827941894531,
      "activations/layer10_attention_weight_min": -37.54164505004883,
      "activations/layer11_attention_weight_max": 37.44131088256836,
      "activations/layer11_attention_weight_min": -36.37950134277344,
      "activations/layer12_attention_weight_max": 28.292667388916016,
      "activations/layer12_attention_weight_min": -27.986101150512695,
      "activations/layer13_attention_weight_max": 41.89451217651367,
      "activations/layer13_attention_weight_min": -36.700679779052734,
      "activations/layer14_attention_weight_max": 43.0514030456543,
      "activations/layer14_attention_weight_min": -38.43638610839844,
      "activations/layer15_attention_weight_max": 40.396827697753906,
      "activations/layer15_attention_weight_min": -38.09119415283203,
      "activations/layer16_attention_weight_max": 28.29199981689453,
      "activations/layer16_attention_weight_min": -29.127012252807617,
      "activations/layer17_attention_weight_max": 29.72920036315918,
      "activations/layer17_attention_weight_min": -26.78375816345215,
      "activations/layer18_attention_weight_max": 33.45897674560547,
      "activations/layer18_attention_weight_min": -27.039310455322266,
      "activations/layer19_attention_weight_max": 41.842437744140625,
      "activations/layer19_attention_weight_min": -31.35172462463379,
      "activations/layer1_attention_weight_max": 16.63810920715332,
      "activations/layer1_attention_weight_min": -17.400693893432617,
      "activations/layer20_attention_weight_max": 32.89342498779297,
      "activations/layer20_attention_weight_min": -24.946935653686523,
      "activations/layer21_attention_weight_max": 32.60491943359375,
      "activations/layer21_attention_weight_min": -24.580236434936523,
      "activations/layer22_attention_weight_max": 44.53927993774414,
      "activations/layer22_attention_weight_min": -32.288509368896484,
      "activations/layer23_attention_weight_max": 35.67048263549805,
      "activations/layer23_attention_weight_min": -26.756973266601562,
      "activations/layer2_attention_weight_max": 34.76068878173828,
      "activations/layer2_attention_weight_min": -35.2127685546875,
      "activations/layer3_attention_weight_max": 109.92267608642578,
      "activations/layer3_attention_weight_min": -110.70257568359375,
      "activations/layer4_attention_weight_max": 114.1886978149414,
      "activations/layer4_attention_weight_min": -113.17103576660156,
      "activations/layer5_attention_weight_max": 52.741668701171875,
      "activations/layer5_attention_weight_min": -70.21832275390625,
      "activations/layer6_attention_weight_max": 48.12930679321289,
      "activations/layer6_attention_weight_min": -51.01063537597656,
      "activations/layer7_attention_weight_max": 102.28438568115234,
      "activations/layer7_attention_weight_min": -107.6314468383789,
      "activations/layer8_attention_weight_max": 47.45474624633789,
      "activations/layer8_attention_weight_min": -46.85981750488281,
      "activations/layer9_attention_weight_max": 34.09355163574219,
      "activations/layer9_attention_weight_min": -37.48652267456055,
      "epoch": 21.75,
      "learning_rate": 9.826136363636365e-06,
      "loss": 2.6906,
      "step": 374350
    },
    {
      "activations/layer0_attention_weight_max": 16.74835777282715,
      "activations/layer0_attention_weight_min": -13.439888954162598,
      "activations/layer10_attention_weight_max": 38.338809967041016,
      "activations/layer10_attention_weight_min": -37.195186614990234,
      "activations/layer11_attention_weight_max": 37.06867218017578,
      "activations/layer11_attention_weight_min": -37.61634063720703,
      "activations/layer12_attention_weight_max": 27.432893753051758,
      "activations/layer12_attention_weight_min": -27.179946899414062,
      "activations/layer13_attention_weight_max": 38.492347717285156,
      "activations/layer13_attention_weight_min": -37.476619720458984,
      "activations/layer14_attention_weight_max": 39.72373962402344,
      "activations/layer14_attention_weight_min": -40.31775665283203,
      "activations/layer15_attention_weight_max": 39.27573013305664,
      "activations/layer15_attention_weight_min": -37.367244720458984,
      "activations/layer16_attention_weight_max": 27.910600662231445,
      "activations/layer16_attention_weight_min": -28.61367416381836,
      "activations/layer17_attention_weight_max": 28.304767608642578,
      "activations/layer17_attention_weight_min": -25.84100341796875,
      "activations/layer18_attention_weight_max": 32.901763916015625,
      "activations/layer18_attention_weight_min": -25.11209487915039,
      "activations/layer19_attention_weight_max": 32.112483978271484,
      "activations/layer19_attention_weight_min": -30.99446678161621,
      "activations/layer1_attention_weight_max": 15.571606636047363,
      "activations/layer1_attention_weight_min": -14.93359375,
      "activations/layer20_attention_weight_max": 28.79009246826172,
      "activations/layer20_attention_weight_min": -26.262361526489258,
      "activations/layer21_attention_weight_max": 29.97093963623047,
      "activations/layer21_attention_weight_min": -26.54545783996582,
      "activations/layer22_attention_weight_max": 39.85093307495117,
      "activations/layer22_attention_weight_min": -31.069416046142578,
      "activations/layer23_attention_weight_max": 31.18739891052246,
      "activations/layer23_attention_weight_min": -25.481609344482422,
      "activations/layer2_attention_weight_max": 33.945072174072266,
      "activations/layer2_attention_weight_min": -34.75244903564453,
      "activations/layer3_attention_weight_max": 110.67362976074219,
      "activations/layer3_attention_weight_min": -109.7401351928711,
      "activations/layer4_attention_weight_max": 114.7976303100586,
      "activations/layer4_attention_weight_min": -113.3344955444336,
      "activations/layer5_attention_weight_max": 53.966087341308594,
      "activations/layer5_attention_weight_min": -60.91923141479492,
      "activations/layer6_attention_weight_max": 47.99306106567383,
      "activations/layer6_attention_weight_min": -48.9185676574707,
      "activations/layer7_attention_weight_max": 103.9695053100586,
      "activations/layer7_attention_weight_min": -104.38912200927734,
      "activations/layer8_attention_weight_max": 47.22021484375,
      "activations/layer8_attention_weight_min": -48.43224334716797,
      "activations/layer9_attention_weight_max": 38.09626007080078,
      "activations/layer9_attention_weight_min": -40.20609664916992,
      "epoch": 21.75,
      "learning_rate": 9.80719696969697e-06,
      "loss": 2.7215,
      "step": 374400
    },
    {
      "activations/layer0_attention_weight_max": 16.645145416259766,
      "activations/layer0_attention_weight_min": -13.107466697692871,
      "activations/layer10_attention_weight_max": 33.34468460083008,
      "activations/layer10_attention_weight_min": -36.126407623291016,
      "activations/layer11_attention_weight_max": 34.00202560424805,
      "activations/layer11_attention_weight_min": -34.854896545410156,
      "activations/layer12_attention_weight_max": 26.77634620666504,
      "activations/layer12_attention_weight_min": -28.8933048248291,
      "activations/layer13_attention_weight_max": 37.306602478027344,
      "activations/layer13_attention_weight_min": -37.83600997924805,
      "activations/layer14_attention_weight_max": 39.54812240600586,
      "activations/layer14_attention_weight_min": -39.29743194580078,
      "activations/layer15_attention_weight_max": 37.81769561767578,
      "activations/layer15_attention_weight_min": -38.4788818359375,
      "activations/layer16_attention_weight_max": 27.641172409057617,
      "activations/layer16_attention_weight_min": -27.299907684326172,
      "activations/layer17_attention_weight_max": 30.39866828918457,
      "activations/layer17_attention_weight_min": -26.70560646057129,
      "activations/layer18_attention_weight_max": 32.010623931884766,
      "activations/layer18_attention_weight_min": -27.97657585144043,
      "activations/layer19_attention_weight_max": 34.54581069946289,
      "activations/layer19_attention_weight_min": -32.103309631347656,
      "activations/layer1_attention_weight_max": 16.461986541748047,
      "activations/layer1_attention_weight_min": -15.39391040802002,
      "activations/layer20_attention_weight_max": 28.286712646484375,
      "activations/layer20_attention_weight_min": -24.700712203979492,
      "activations/layer21_attention_weight_max": 28.721458435058594,
      "activations/layer21_attention_weight_min": -26.71279525756836,
      "activations/layer22_attention_weight_max": 40.16327667236328,
      "activations/layer22_attention_weight_min": -31.87506675720215,
      "activations/layer23_attention_weight_max": 32.41522979736328,
      "activations/layer23_attention_weight_min": -25.8758602142334,
      "activations/layer2_attention_weight_max": 33.4305305480957,
      "activations/layer2_attention_weight_min": -32.19193649291992,
      "activations/layer3_attention_weight_max": 101.55597686767578,
      "activations/layer3_attention_weight_min": -103.37960052490234,
      "activations/layer4_attention_weight_max": 105.81584930419922,
      "activations/layer4_attention_weight_min": -109.31402587890625,
      "activations/layer5_attention_weight_max": 49.62187957763672,
      "activations/layer5_attention_weight_min": -61.7086296081543,
      "activations/layer6_attention_weight_max": 44.86440658569336,
      "activations/layer6_attention_weight_min": -48.46705627441406,
      "activations/layer7_attention_weight_max": 93.80984497070312,
      "activations/layer7_attention_weight_min": -97.76598358154297,
      "activations/layer8_attention_weight_max": 44.46674346923828,
      "activations/layer8_attention_weight_min": -43.917198181152344,
      "activations/layer9_attention_weight_max": 32.087154388427734,
      "activations/layer9_attention_weight_min": -35.84959411621094,
      "epoch": 21.76,
      "learning_rate": 9.788257575757576e-06,
      "loss": 2.6943,
      "step": 374450
    },
    {
      "activations/layer0_attention_weight_max": 16.953189849853516,
      "activations/layer0_attention_weight_min": -13.925139427185059,
      "activations/layer10_attention_weight_max": 48.05449295043945,
      "activations/layer10_attention_weight_min": -47.29693603515625,
      "activations/layer11_attention_weight_max": 50.671661376953125,
      "activations/layer11_attention_weight_min": -50.822628021240234,
      "activations/layer12_attention_weight_max": 33.295040130615234,
      "activations/layer12_attention_weight_min": -32.13576126098633,
      "activations/layer13_attention_weight_max": 44.77445983886719,
      "activations/layer13_attention_weight_min": -42.009666442871094,
      "activations/layer14_attention_weight_max": 51.751644134521484,
      "activations/layer14_attention_weight_min": -43.40031433105469,
      "activations/layer15_attention_weight_max": 50.95454406738281,
      "activations/layer15_attention_weight_min": -47.68278503417969,
      "activations/layer16_attention_weight_max": 35.32038879394531,
      "activations/layer16_attention_weight_min": -30.563228607177734,
      "activations/layer17_attention_weight_max": 33.23841094970703,
      "activations/layer17_attention_weight_min": -32.00590896606445,
      "activations/layer18_attention_weight_max": 34.874908447265625,
      "activations/layer18_attention_weight_min": -30.342613220214844,
      "activations/layer19_attention_weight_max": 36.80643844604492,
      "activations/layer19_attention_weight_min": -32.770626068115234,
      "activations/layer1_attention_weight_max": 16.097631454467773,
      "activations/layer1_attention_weight_min": -15.471576690673828,
      "activations/layer20_attention_weight_max": 31.206499099731445,
      "activations/layer20_attention_weight_min": -25.783239364624023,
      "activations/layer21_attention_weight_max": 32.0774040222168,
      "activations/layer21_attention_weight_min": -25.57004737854004,
      "activations/layer22_attention_weight_max": 42.125389099121094,
      "activations/layer22_attention_weight_min": -35.06111145019531,
      "activations/layer23_attention_weight_max": 31.950809478759766,
      "activations/layer23_attention_weight_min": -29.378334045410156,
      "activations/layer2_attention_weight_max": 36.10226058959961,
      "activations/layer2_attention_weight_min": -36.301815032958984,
      "activations/layer3_attention_weight_max": 106.19144439697266,
      "activations/layer3_attention_weight_min": -111.8735580444336,
      "activations/layer4_attention_weight_max": 109.36649322509766,
      "activations/layer4_attention_weight_min": -111.0680923461914,
      "activations/layer5_attention_weight_max": 55.39043426513672,
      "activations/layer5_attention_weight_min": -60.975887298583984,
      "activations/layer6_attention_weight_max": 46.25868225097656,
      "activations/layer6_attention_weight_min": -48.05624771118164,
      "activations/layer7_attention_weight_max": 112.51679992675781,
      "activations/layer7_attention_weight_min": -110.35183715820312,
      "activations/layer8_attention_weight_max": 54.30508804321289,
      "activations/layer8_attention_weight_min": -53.71158981323242,
      "activations/layer9_attention_weight_max": 39.90338897705078,
      "activations/layer9_attention_weight_min": -44.12569046020508,
      "epoch": 21.76,
      "learning_rate": 9.769318181818181e-06,
      "loss": 2.7069,
      "step": 374500
    },
    {
      "activations/layer0_attention_weight_max": 15.94920825958252,
      "activations/layer0_attention_weight_min": -14.42061710357666,
      "activations/layer10_attention_weight_max": 34.89326858520508,
      "activations/layer10_attention_weight_min": -34.10447311401367,
      "activations/layer11_attention_weight_max": 36.29597473144531,
      "activations/layer11_attention_weight_min": -33.20005416870117,
      "activations/layer12_attention_weight_max": 30.5975284576416,
      "activations/layer12_attention_weight_min": -28.27117156982422,
      "activations/layer13_attention_weight_max": 43.05719757080078,
      "activations/layer13_attention_weight_min": -36.328529357910156,
      "activations/layer14_attention_weight_max": 45.56537628173828,
      "activations/layer14_attention_weight_min": -40.93839645385742,
      "activations/layer15_attention_weight_max": 44.7876091003418,
      "activations/layer15_attention_weight_min": -36.72515106201172,
      "activations/layer16_attention_weight_max": 30.804981231689453,
      "activations/layer16_attention_weight_min": -29.835819244384766,
      "activations/layer17_attention_weight_max": 32.642826080322266,
      "activations/layer17_attention_weight_min": -26.478679656982422,
      "activations/layer18_attention_weight_max": 35.0169792175293,
      "activations/layer18_attention_weight_min": -27.005943298339844,
      "activations/layer19_attention_weight_max": 35.235939025878906,
      "activations/layer19_attention_weight_min": -31.845401763916016,
      "activations/layer1_attention_weight_max": 16.051904678344727,
      "activations/layer1_attention_weight_min": -15.073222160339355,
      "activations/layer20_attention_weight_max": 31.408958435058594,
      "activations/layer20_attention_weight_min": -25.695573806762695,
      "activations/layer21_attention_weight_max": 31.86368751525879,
      "activations/layer21_attention_weight_min": -23.151247024536133,
      "activations/layer22_attention_weight_max": 40.436893463134766,
      "activations/layer22_attention_weight_min": -31.09902000427246,
      "activations/layer23_attention_weight_max": 31.957611083984375,
      "activations/layer23_attention_weight_min": -23.737852096557617,
      "activations/layer2_attention_weight_max": 33.626792907714844,
      "activations/layer2_attention_weight_min": -33.23081588745117,
      "activations/layer3_attention_weight_max": 102.12129211425781,
      "activations/layer3_attention_weight_min": -103.14786529541016,
      "activations/layer4_attention_weight_max": 106.07427215576172,
      "activations/layer4_attention_weight_min": -110.86946105957031,
      "activations/layer5_attention_weight_max": 49.4644889831543,
      "activations/layer5_attention_weight_min": -60.15684127807617,
      "activations/layer6_attention_weight_max": 44.50743103027344,
      "activations/layer6_attention_weight_min": -46.90053939819336,
      "activations/layer7_attention_weight_max": 95.01425170898438,
      "activations/layer7_attention_weight_min": -96.9593734741211,
      "activations/layer8_attention_weight_max": 42.73330307006836,
      "activations/layer8_attention_weight_min": -42.77913284301758,
      "activations/layer9_attention_weight_max": 33.22119903564453,
      "activations/layer9_attention_weight_min": -34.51995086669922,
      "epoch": 21.76,
      "learning_rate": 9.750378787878787e-06,
      "loss": 2.7137,
      "step": 374550
    },
    {
      "activations/layer0_attention_weight_max": 16.70197105407715,
      "activations/layer0_attention_weight_min": -13.611359596252441,
      "activations/layer10_attention_weight_max": 41.62275314331055,
      "activations/layer10_attention_weight_min": -39.78125762939453,
      "activations/layer11_attention_weight_max": 39.99353790283203,
      "activations/layer11_attention_weight_min": -37.75553512573242,
      "activations/layer12_attention_weight_max": 30.182632446289062,
      "activations/layer12_attention_weight_min": -29.085708618164062,
      "activations/layer13_attention_weight_max": 45.06087112426758,
      "activations/layer13_attention_weight_min": -40.16258239746094,
      "activations/layer14_attention_weight_max": 48.6888427734375,
      "activations/layer14_attention_weight_min": -39.6891975402832,
      "activations/layer15_attention_weight_max": 43.120052337646484,
      "activations/layer15_attention_weight_min": -39.573394775390625,
      "activations/layer16_attention_weight_max": 32.16627502441406,
      "activations/layer16_attention_weight_min": -28.892154693603516,
      "activations/layer17_attention_weight_max": 34.350921630859375,
      "activations/layer17_attention_weight_min": -29.231489181518555,
      "activations/layer18_attention_weight_max": 39.762535095214844,
      "activations/layer18_attention_weight_min": -27.95579719543457,
      "activations/layer19_attention_weight_max": 42.71097946166992,
      "activations/layer19_attention_weight_min": -35.6657829284668,
      "activations/layer1_attention_weight_max": 18.044898986816406,
      "activations/layer1_attention_weight_min": -15.464784622192383,
      "activations/layer20_attention_weight_max": 35.279991149902344,
      "activations/layer20_attention_weight_min": -26.12190818786621,
      "activations/layer21_attention_weight_max": 35.0542106628418,
      "activations/layer21_attention_weight_min": -26.734848022460938,
      "activations/layer22_attention_weight_max": 46.60515594482422,
      "activations/layer22_attention_weight_min": -35.70587158203125,
      "activations/layer23_attention_weight_max": 36.44062042236328,
      "activations/layer23_attention_weight_min": -26.773059844970703,
      "activations/layer2_attention_weight_max": 34.934295654296875,
      "activations/layer2_attention_weight_min": -34.47568893432617,
      "activations/layer3_attention_weight_max": 112.21184539794922,
      "activations/layer3_attention_weight_min": -111.43753051757812,
      "activations/layer4_attention_weight_max": 118.80641174316406,
      "activations/layer4_attention_weight_min": -118.6392593383789,
      "activations/layer5_attention_weight_max": 55.83228302001953,
      "activations/layer5_attention_weight_min": -64.25054931640625,
      "activations/layer6_attention_weight_max": 49.50297927856445,
      "activations/layer6_attention_weight_min": -53.62411880493164,
      "activations/layer7_attention_weight_max": 104.06805419921875,
      "activations/layer7_attention_weight_min": -116.4699478149414,
      "activations/layer8_attention_weight_max": 48.579345703125,
      "activations/layer8_attention_weight_min": -53.47478103637695,
      "activations/layer9_attention_weight_max": 36.53546905517578,
      "activations/layer9_attention_weight_min": -38.4433708190918,
      "epoch": 21.77,
      "learning_rate": 9.731439393939394e-06,
      "loss": 2.6979,
      "step": 374600
    },
    {
      "activations/layer0_attention_weight_max": 16.711313247680664,
      "activations/layer0_attention_weight_min": -14.290290832519531,
      "activations/layer10_attention_weight_max": 37.0159797668457,
      "activations/layer10_attention_weight_min": -35.02311706542969,
      "activations/layer11_attention_weight_max": 39.13457489013672,
      "activations/layer11_attention_weight_min": -35.12562942504883,
      "activations/layer12_attention_weight_max": 27.91132926940918,
      "activations/layer12_attention_weight_min": -28.550535202026367,
      "activations/layer13_attention_weight_max": 42.292415618896484,
      "activations/layer13_attention_weight_min": -37.38970947265625,
      "activations/layer14_attention_weight_max": 43.74238204956055,
      "activations/layer14_attention_weight_min": -40.71931457519531,
      "activations/layer15_attention_weight_max": 41.94078826904297,
      "activations/layer15_attention_weight_min": -36.24939727783203,
      "activations/layer16_attention_weight_max": 30.415937423706055,
      "activations/layer16_attention_weight_min": -28.602205276489258,
      "activations/layer17_attention_weight_max": 30.6776123046875,
      "activations/layer17_attention_weight_min": -27.21506118774414,
      "activations/layer18_attention_weight_max": 30.828989028930664,
      "activations/layer18_attention_weight_min": -26.81814956665039,
      "activations/layer19_attention_weight_max": 33.945430755615234,
      "activations/layer19_attention_weight_min": -32.331787109375,
      "activations/layer1_attention_weight_max": 18.345138549804688,
      "activations/layer1_attention_weight_min": -15.179935455322266,
      "activations/layer20_attention_weight_max": 27.73467254638672,
      "activations/layer20_attention_weight_min": -26.53947639465332,
      "activations/layer21_attention_weight_max": 28.745975494384766,
      "activations/layer21_attention_weight_min": -26.96665382385254,
      "activations/layer22_attention_weight_max": 40.31263732910156,
      "activations/layer22_attention_weight_min": -31.65464973449707,
      "activations/layer23_attention_weight_max": 31.758893966674805,
      "activations/layer23_attention_weight_min": -25.3721981048584,
      "activations/layer2_attention_weight_max": 35.574501037597656,
      "activations/layer2_attention_weight_min": -32.981143951416016,
      "activations/layer3_attention_weight_max": 107.45130920410156,
      "activations/layer3_attention_weight_min": -104.62598419189453,
      "activations/layer4_attention_weight_max": 113.24442291259766,
      "activations/layer4_attention_weight_min": -113.25685119628906,
      "activations/layer5_attention_weight_max": 51.00889205932617,
      "activations/layer5_attention_weight_min": -66.69476318359375,
      "activations/layer6_attention_weight_max": 43.397544860839844,
      "activations/layer6_attention_weight_min": -49.266231536865234,
      "activations/layer7_attention_weight_max": 94.8770980834961,
      "activations/layer7_attention_weight_min": -101.49102783203125,
      "activations/layer8_attention_weight_max": 45.643333435058594,
      "activations/layer8_attention_weight_min": -47.155582427978516,
      "activations/layer9_attention_weight_max": 34.749019622802734,
      "activations/layer9_attention_weight_min": -34.845516204833984,
      "epoch": 21.77,
      "learning_rate": 9.712499999999999e-06,
      "loss": 2.701,
      "step": 374650
    },
    {
      "activations/layer0_attention_weight_max": 17.104528427124023,
      "activations/layer0_attention_weight_min": -14.593399047851562,
      "activations/layer10_attention_weight_max": 34.261661529541016,
      "activations/layer10_attention_weight_min": -36.948883056640625,
      "activations/layer11_attention_weight_max": 35.64513397216797,
      "activations/layer11_attention_weight_min": -36.36811447143555,
      "activations/layer12_attention_weight_max": 26.036882400512695,
      "activations/layer12_attention_weight_min": -26.811506271362305,
      "activations/layer13_attention_weight_max": 38.02372741699219,
      "activations/layer13_attention_weight_min": -37.004451751708984,
      "activations/layer14_attention_weight_max": 42.14528274536133,
      "activations/layer14_attention_weight_min": -40.032493591308594,
      "activations/layer15_attention_weight_max": 38.24000930786133,
      "activations/layer15_attention_weight_min": -38.533077239990234,
      "activations/layer16_attention_weight_max": 29.277812957763672,
      "activations/layer16_attention_weight_min": -27.879426956176758,
      "activations/layer17_attention_weight_max": 33.571449279785156,
      "activations/layer17_attention_weight_min": -27.22696876525879,
      "activations/layer18_attention_weight_max": 33.69404983520508,
      "activations/layer18_attention_weight_min": -26.972190856933594,
      "activations/layer19_attention_weight_max": 39.741615295410156,
      "activations/layer19_attention_weight_min": -31.99744987487793,
      "activations/layer1_attention_weight_max": 15.640778541564941,
      "activations/layer1_attention_weight_min": -14.935026168823242,
      "activations/layer20_attention_weight_max": 29.999237060546875,
      "activations/layer20_attention_weight_min": -25.170995712280273,
      "activations/layer21_attention_weight_max": 30.721330642700195,
      "activations/layer21_attention_weight_min": -24.85489845275879,
      "activations/layer22_attention_weight_max": 45.4768180847168,
      "activations/layer22_attention_weight_min": -30.48789405822754,
      "activations/layer23_attention_weight_max": 32.820030212402344,
      "activations/layer23_attention_weight_min": -25.000049591064453,
      "activations/layer2_attention_weight_max": 33.439815521240234,
      "activations/layer2_attention_weight_min": -33.5087776184082,
      "activations/layer3_attention_weight_max": 107.88870239257812,
      "activations/layer3_attention_weight_min": -108.95246887207031,
      "activations/layer4_attention_weight_max": 110.12910461425781,
      "activations/layer4_attention_weight_min": -110.42813873291016,
      "activations/layer5_attention_weight_max": 51.11744689941406,
      "activations/layer5_attention_weight_min": -63.2531852722168,
      "activations/layer6_attention_weight_max": 49.720375061035156,
      "activations/layer6_attention_weight_min": -48.691505432128906,
      "activations/layer7_attention_weight_max": 100.75128936767578,
      "activations/layer7_attention_weight_min": -103.34759521484375,
      "activations/layer8_attention_weight_max": 45.070045471191406,
      "activations/layer8_attention_weight_min": -48.24796676635742,
      "activations/layer9_attention_weight_max": 33.14484786987305,
      "activations/layer9_attention_weight_min": -37.9539794921875,
      "epoch": 21.77,
      "learning_rate": 9.693560606060605e-06,
      "loss": 2.7107,
      "step": 374700
    },
    {
      "activations/layer0_attention_weight_max": 16.442710876464844,
      "activations/layer0_attention_weight_min": -14.248451232910156,
      "activations/layer10_attention_weight_max": 34.26411437988281,
      "activations/layer10_attention_weight_min": -36.597389221191406,
      "activations/layer11_attention_weight_max": 35.54456329345703,
      "activations/layer11_attention_weight_min": -36.89588928222656,
      "activations/layer12_attention_weight_max": 30.514387130737305,
      "activations/layer12_attention_weight_min": -28.876981735229492,
      "activations/layer13_attention_weight_max": 42.881385803222656,
      "activations/layer13_attention_weight_min": -40.66511535644531,
      "activations/layer14_attention_weight_max": 51.336307525634766,
      "activations/layer14_attention_weight_min": -44.39533615112305,
      "activations/layer15_attention_weight_max": 44.08289337158203,
      "activations/layer15_attention_weight_min": -42.165985107421875,
      "activations/layer16_attention_weight_max": 36.140052795410156,
      "activations/layer16_attention_weight_min": -30.91080665588379,
      "activations/layer17_attention_weight_max": 32.676551818847656,
      "activations/layer17_attention_weight_min": -28.58687973022461,
      "activations/layer18_attention_weight_max": 36.512908935546875,
      "activations/layer18_attention_weight_min": -31.064552307128906,
      "activations/layer19_attention_weight_max": 42.79523849487305,
      "activations/layer19_attention_weight_min": -35.092586517333984,
      "activations/layer1_attention_weight_max": 16.84950828552246,
      "activations/layer1_attention_weight_min": -15.800957679748535,
      "activations/layer20_attention_weight_max": 36.809471130371094,
      "activations/layer20_attention_weight_min": -26.86515998840332,
      "activations/layer21_attention_weight_max": 35.85204315185547,
      "activations/layer21_attention_weight_min": -26.347110748291016,
      "activations/layer22_attention_weight_max": 53.61394119262695,
      "activations/layer22_attention_weight_min": -34.55636978149414,
      "activations/layer23_attention_weight_max": 35.9537467956543,
      "activations/layer23_attention_weight_min": -27.7724609375,
      "activations/layer2_attention_weight_max": 35.065528869628906,
      "activations/layer2_attention_weight_min": -34.418052673339844,
      "activations/layer3_attention_weight_max": 110.26683044433594,
      "activations/layer3_attention_weight_min": -106.88340759277344,
      "activations/layer4_attention_weight_max": 113.54666900634766,
      "activations/layer4_attention_weight_min": -116.13829803466797,
      "activations/layer5_attention_weight_max": 53.616695404052734,
      "activations/layer5_attention_weight_min": -63.42610168457031,
      "activations/layer6_attention_weight_max": 46.747398376464844,
      "activations/layer6_attention_weight_min": -50.507240295410156,
      "activations/layer7_attention_weight_max": 96.9668960571289,
      "activations/layer7_attention_weight_min": -102.98629760742188,
      "activations/layer8_attention_weight_max": 43.29872512817383,
      "activations/layer8_attention_weight_min": -45.497955322265625,
      "activations/layer9_attention_weight_max": 33.11661148071289,
      "activations/layer9_attention_weight_min": -36.62757110595703,
      "epoch": 21.78,
      "learning_rate": 9.675e-06,
      "loss": 2.705,
      "step": 374750
    },
    {
      "activations/layer0_attention_weight_max": 16.895219802856445,
      "activations/layer0_attention_weight_min": -14.859070777893066,
      "activations/layer10_attention_weight_max": 34.332759857177734,
      "activations/layer10_attention_weight_min": -34.41523742675781,
      "activations/layer11_attention_weight_max": 33.72398376464844,
      "activations/layer11_attention_weight_min": -34.22674560546875,
      "activations/layer12_attention_weight_max": 26.642581939697266,
      "activations/layer12_attention_weight_min": -25.970544815063477,
      "activations/layer13_attention_weight_max": 37.866371154785156,
      "activations/layer13_attention_weight_min": -35.0583381652832,
      "activations/layer14_attention_weight_max": 39.387306213378906,
      "activations/layer14_attention_weight_min": -37.72581100463867,
      "activations/layer15_attention_weight_max": 37.64139175415039,
      "activations/layer15_attention_weight_min": -35.575225830078125,
      "activations/layer16_attention_weight_max": 28.052106857299805,
      "activations/layer16_attention_weight_min": -29.98382568359375,
      "activations/layer17_attention_weight_max": 28.25670623779297,
      "activations/layer17_attention_weight_min": -26.316967010498047,
      "activations/layer18_attention_weight_max": 31.186601638793945,
      "activations/layer18_attention_weight_min": -27.983163833618164,
      "activations/layer19_attention_weight_max": 31.582366943359375,
      "activations/layer19_attention_weight_min": -33.43171691894531,
      "activations/layer1_attention_weight_max": 15.667828559875488,
      "activations/layer1_attention_weight_min": -15.71805477142334,
      "activations/layer20_attention_weight_max": 26.414775848388672,
      "activations/layer20_attention_weight_min": -25.64986801147461,
      "activations/layer21_attention_weight_max": 26.762474060058594,
      "activations/layer21_attention_weight_min": -25.104825973510742,
      "activations/layer22_attention_weight_max": 37.88541793823242,
      "activations/layer22_attention_weight_min": -33.30622863769531,
      "activations/layer23_attention_weight_max": 28.187610626220703,
      "activations/layer23_attention_weight_min": -29.090179443359375,
      "activations/layer2_attention_weight_max": 33.633216857910156,
      "activations/layer2_attention_weight_min": -34.257598876953125,
      "activations/layer3_attention_weight_max": 104.91041564941406,
      "activations/layer3_attention_weight_min": -109.59893798828125,
      "activations/layer4_attention_weight_max": 108.09752655029297,
      "activations/layer4_attention_weight_min": -113.08223724365234,
      "activations/layer5_attention_weight_max": 49.36212158203125,
      "activations/layer5_attention_weight_min": -63.74679183959961,
      "activations/layer6_attention_weight_max": 44.55708312988281,
      "activations/layer6_attention_weight_min": -49.166053771972656,
      "activations/layer7_attention_weight_max": 101.81517791748047,
      "activations/layer7_attention_weight_min": -103.86878204345703,
      "activations/layer8_attention_weight_max": 44.502254486083984,
      "activations/layer8_attention_weight_min": -45.24416732788086,
      "activations/layer9_attention_weight_max": 33.678951263427734,
      "activations/layer9_attention_weight_min": -35.13258361816406,
      "epoch": 21.78,
      "learning_rate": 9.656060606060605e-06,
      "loss": 2.6974,
      "step": 374800
    },
    {
      "activations/layer0_attention_weight_max": 16.230815887451172,
      "activations/layer0_attention_weight_min": -14.818150520324707,
      "activations/layer10_attention_weight_max": 35.657875061035156,
      "activations/layer10_attention_weight_min": -35.113441467285156,
      "activations/layer11_attention_weight_max": 33.67311096191406,
      "activations/layer11_attention_weight_min": -34.187171936035156,
      "activations/layer12_attention_weight_max": 26.738792419433594,
      "activations/layer12_attention_weight_min": -28.013795852661133,
      "activations/layer13_attention_weight_max": 40.66374588012695,
      "activations/layer13_attention_weight_min": -37.15508270263672,
      "activations/layer14_attention_weight_max": 43.70307540893555,
      "activations/layer14_attention_weight_min": -40.707332611083984,
      "activations/layer15_attention_weight_max": 40.897159576416016,
      "activations/layer15_attention_weight_min": -37.967288970947266,
      "activations/layer16_attention_weight_max": 28.681238174438477,
      "activations/layer16_attention_weight_min": -27.963062286376953,
      "activations/layer17_attention_weight_max": 31.52651023864746,
      "activations/layer17_attention_weight_min": -28.79899787902832,
      "activations/layer18_attention_weight_max": 33.79676055908203,
      "activations/layer18_attention_weight_min": -26.504127502441406,
      "activations/layer19_attention_weight_max": 36.541324615478516,
      "activations/layer19_attention_weight_min": -31.0189266204834,
      "activations/layer1_attention_weight_max": 16.860252380371094,
      "activations/layer1_attention_weight_min": -16.466506958007812,
      "activations/layer20_attention_weight_max": 31.169517517089844,
      "activations/layer20_attention_weight_min": -24.101390838623047,
      "activations/layer21_attention_weight_max": 28.38727378845215,
      "activations/layer21_attention_weight_min": -23.892911911010742,
      "activations/layer22_attention_weight_max": 42.5755615234375,
      "activations/layer22_attention_weight_min": -29.483808517456055,
      "activations/layer23_attention_weight_max": 30.9110164642334,
      "activations/layer23_attention_weight_min": -24.20155906677246,
      "activations/layer2_attention_weight_max": 33.58134078979492,
      "activations/layer2_attention_weight_min": -32.92790603637695,
      "activations/layer3_attention_weight_max": 103.12773132324219,
      "activations/layer3_attention_weight_min": -111.15879821777344,
      "activations/layer4_attention_weight_max": 106.60908508300781,
      "activations/layer4_attention_weight_min": -112.66051483154297,
      "activations/layer5_attention_weight_max": 50.190147399902344,
      "activations/layer5_attention_weight_min": -61.01154327392578,
      "activations/layer6_attention_weight_max": 43.81526565551758,
      "activations/layer6_attention_weight_min": -46.26070022583008,
      "activations/layer7_attention_weight_max": 92.82015991210938,
      "activations/layer7_attention_weight_min": -98.92034149169922,
      "activations/layer8_attention_weight_max": 43.62285614013672,
      "activations/layer8_attention_weight_min": -45.4022216796875,
      "activations/layer9_attention_weight_max": 33.04856491088867,
      "activations/layer9_attention_weight_min": -35.3848991394043,
      "epoch": 21.78,
      "learning_rate": 9.637121212121211e-06,
      "loss": 2.7037,
      "step": 374850
    },
    {
      "activations/layer0_attention_weight_max": 17.584360122680664,
      "activations/layer0_attention_weight_min": -14.4234037399292,
      "activations/layer10_attention_weight_max": 35.12923812866211,
      "activations/layer10_attention_weight_min": -36.654666900634766,
      "activations/layer11_attention_weight_max": 34.0200309753418,
      "activations/layer11_attention_weight_min": -35.36048889160156,
      "activations/layer12_attention_weight_max": 24.602130889892578,
      "activations/layer12_attention_weight_min": -27.198442459106445,
      "activations/layer13_attention_weight_max": 37.12018585205078,
      "activations/layer13_attention_weight_min": -35.8043327331543,
      "activations/layer14_attention_weight_max": 41.57075881958008,
      "activations/layer14_attention_weight_min": -38.72642517089844,
      "activations/layer15_attention_weight_max": 37.74575424194336,
      "activations/layer15_attention_weight_min": -38.2226676940918,
      "activations/layer16_attention_weight_max": 26.731801986694336,
      "activations/layer16_attention_weight_min": -29.94284439086914,
      "activations/layer17_attention_weight_max": 27.43326759338379,
      "activations/layer17_attention_weight_min": -26.179643630981445,
      "activations/layer18_attention_weight_max": 31.42827796936035,
      "activations/layer18_attention_weight_min": -26.94157600402832,
      "activations/layer19_attention_weight_max": 32.18770980834961,
      "activations/layer19_attention_weight_min": -34.61698532104492,
      "activations/layer1_attention_weight_max": 17.403366088867188,
      "activations/layer1_attention_weight_min": -16.125925064086914,
      "activations/layer20_attention_weight_max": 27.388954162597656,
      "activations/layer20_attention_weight_min": -26.392032623291016,
      "activations/layer21_attention_weight_max": 27.610990524291992,
      "activations/layer21_attention_weight_min": -25.977062225341797,
      "activations/layer22_attention_weight_max": 36.8782844543457,
      "activations/layer22_attention_weight_min": -32.25550079345703,
      "activations/layer23_attention_weight_max": 30.104881286621094,
      "activations/layer23_attention_weight_min": -25.715862274169922,
      "activations/layer2_attention_weight_max": 33.59024429321289,
      "activations/layer2_attention_weight_min": -32.22828674316406,
      "activations/layer3_attention_weight_max": 105.4177474975586,
      "activations/layer3_attention_weight_min": -105.20552825927734,
      "activations/layer4_attention_weight_max": 107.3828125,
      "activations/layer4_attention_weight_min": -108.81200408935547,
      "activations/layer5_attention_weight_max": 49.26445770263672,
      "activations/layer5_attention_weight_min": -62.439483642578125,
      "activations/layer6_attention_weight_max": 44.67502212524414,
      "activations/layer6_attention_weight_min": -48.80824661254883,
      "activations/layer7_attention_weight_max": 95.1964340209961,
      "activations/layer7_attention_weight_min": -102.62291717529297,
      "activations/layer8_attention_weight_max": 42.850128173828125,
      "activations/layer8_attention_weight_min": -44.25577163696289,
      "activations/layer9_attention_weight_max": 33.13382339477539,
      "activations/layer9_attention_weight_min": -36.2629508972168,
      "epoch": 21.78,
      "learning_rate": 9.618181818181818e-06,
      "loss": 2.7148,
      "step": 374900
    },
    {
      "activations/layer0_attention_weight_max": 17.107709884643555,
      "activations/layer0_attention_weight_min": -14.548645973205566,
      "activations/layer10_attention_weight_max": 38.60175323486328,
      "activations/layer10_attention_weight_min": -39.63886260986328,
      "activations/layer11_attention_weight_max": 38.798675537109375,
      "activations/layer11_attention_weight_min": -42.04499816894531,
      "activations/layer12_attention_weight_max": 29.851808547973633,
      "activations/layer12_attention_weight_min": -30.16439437866211,
      "activations/layer13_attention_weight_max": 45.17240905761719,
      "activations/layer13_attention_weight_min": -40.840736389160156,
      "activations/layer14_attention_weight_max": 46.120018005371094,
      "activations/layer14_attention_weight_min": -43.021339416503906,
      "activations/layer15_attention_weight_max": 42.43828582763672,
      "activations/layer15_attention_weight_min": -40.568904876708984,
      "activations/layer16_attention_weight_max": 30.993677139282227,
      "activations/layer16_attention_weight_min": -28.73949432373047,
      "activations/layer17_attention_weight_max": 33.193790435791016,
      "activations/layer17_attention_weight_min": -26.26612663269043,
      "activations/layer18_attention_weight_max": 32.91169357299805,
      "activations/layer18_attention_weight_min": -25.542755126953125,
      "activations/layer19_attention_weight_max": 35.828189849853516,
      "activations/layer19_attention_weight_min": -30.61164665222168,
      "activations/layer1_attention_weight_max": 16.592235565185547,
      "activations/layer1_attention_weight_min": -17.074628829956055,
      "activations/layer20_attention_weight_max": 31.501462936401367,
      "activations/layer20_attention_weight_min": -24.477977752685547,
      "activations/layer21_attention_weight_max": 28.505651473999023,
      "activations/layer21_attention_weight_min": -24.096115112304688,
      "activations/layer22_attention_weight_max": 45.27329635620117,
      "activations/layer22_attention_weight_min": -30.63745880126953,
      "activations/layer23_attention_weight_max": 32.17170333862305,
      "activations/layer23_attention_weight_min": -25.26293182373047,
      "activations/layer2_attention_weight_max": 33.28245544433594,
      "activations/layer2_attention_weight_min": -33.84088134765625,
      "activations/layer3_attention_weight_max": 107.49676513671875,
      "activations/layer3_attention_weight_min": -111.7099838256836,
      "activations/layer4_attention_weight_max": 113.08219146728516,
      "activations/layer4_attention_weight_min": -115.8825454711914,
      "activations/layer5_attention_weight_max": 50.885337829589844,
      "activations/layer5_attention_weight_min": -64.50027465820312,
      "activations/layer6_attention_weight_max": 46.66987228393555,
      "activations/layer6_attention_weight_min": -49.711368560791016,
      "activations/layer7_attention_weight_max": 104.10074615478516,
      "activations/layer7_attention_weight_min": -103.44879150390625,
      "activations/layer8_attention_weight_max": 46.658512115478516,
      "activations/layer8_attention_weight_min": -46.65195846557617,
      "activations/layer9_attention_weight_max": 36.124446868896484,
      "activations/layer9_attention_weight_min": -37.63909149169922,
      "epoch": 21.79,
      "learning_rate": 9.599242424242423e-06,
      "loss": 2.7096,
      "step": 374950
    },
    {
      "activations/layer0_attention_weight_max": 17.45363998413086,
      "activations/layer0_attention_weight_min": -13.34253215789795,
      "activations/layer10_attention_weight_max": 37.39183044433594,
      "activations/layer10_attention_weight_min": -37.589378356933594,
      "activations/layer11_attention_weight_max": 35.56449890136719,
      "activations/layer11_attention_weight_min": -38.93855667114258,
      "activations/layer12_attention_weight_max": 30.60456085205078,
      "activations/layer12_attention_weight_min": -35.037803649902344,
      "activations/layer13_attention_weight_max": 41.92305374145508,
      "activations/layer13_attention_weight_min": -40.2179069519043,
      "activations/layer14_attention_weight_max": 43.12041473388672,
      "activations/layer14_attention_weight_min": -38.81093978881836,
      "activations/layer15_attention_weight_max": 40.862361907958984,
      "activations/layer15_attention_weight_min": -38.34158706665039,
      "activations/layer16_attention_weight_max": 32.30977249145508,
      "activations/layer16_attention_weight_min": -30.880449295043945,
      "activations/layer17_attention_weight_max": 33.22087860107422,
      "activations/layer17_attention_weight_min": -25.260995864868164,
      "activations/layer18_attention_weight_max": 33.48661422729492,
      "activations/layer18_attention_weight_min": -25.8636531829834,
      "activations/layer19_attention_weight_max": 41.77901840209961,
      "activations/layer19_attention_weight_min": -30.56666374206543,
      "activations/layer1_attention_weight_max": 15.823070526123047,
      "activations/layer1_attention_weight_min": -15.367377281188965,
      "activations/layer20_attention_weight_max": 34.350955963134766,
      "activations/layer20_attention_weight_min": -23.959814071655273,
      "activations/layer21_attention_weight_max": 30.659257888793945,
      "activations/layer21_attention_weight_min": -22.86469078063965,
      "activations/layer22_attention_weight_max": 44.058528900146484,
      "activations/layer22_attention_weight_min": -34.022098541259766,
      "activations/layer23_attention_weight_max": 32.91175079345703,
      "activations/layer23_attention_weight_min": -29.173002243041992,
      "activations/layer2_attention_weight_max": 33.3412971496582,
      "activations/layer2_attention_weight_min": -33.54283905029297,
      "activations/layer3_attention_weight_max": 102.53289031982422,
      "activations/layer3_attention_weight_min": -107.36198425292969,
      "activations/layer4_attention_weight_max": 107.9727554321289,
      "activations/layer4_attention_weight_min": -109.3944091796875,
      "activations/layer5_attention_weight_max": 52.49177169799805,
      "activations/layer5_attention_weight_min": -61.78855895996094,
      "activations/layer6_attention_weight_max": 44.8946418762207,
      "activations/layer6_attention_weight_min": -46.47439956665039,
      "activations/layer7_attention_weight_max": 98.66670989990234,
      "activations/layer7_attention_weight_min": -109.63501739501953,
      "activations/layer8_attention_weight_max": 44.28578567504883,
      "activations/layer8_attention_weight_min": -45.05573654174805,
      "activations/layer9_attention_weight_max": 33.45977020263672,
      "activations/layer9_attention_weight_min": -35.3687858581543,
      "epoch": 21.79,
      "learning_rate": 9.58030303030303e-06,
      "loss": 2.7058,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4799,
      "eval_samples_per_second": 506.375,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4799,
      "eval_openwebtext_samples_per_second": 506.375,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9897,
      "eval_wikitext_samples_per_second": 229.18,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_lambada_loss": 2.2890625,
      "eval_lambada_ppl": 9.865684265313508,
      "eval_lambada_runtime": 9.5584,
      "eval_lambada_samples_per_second": 509.397,
      "step": 375000
    },
    {
      "activations/layer0_attention_weight_max": 15.809011459350586,
      "activations/layer0_attention_weight_min": -13.945245742797852,
      "activations/layer10_attention_weight_max": 37.33661651611328,
      "activations/layer10_attention_weight_min": -38.53110885620117,
      "activations/layer11_attention_weight_max": 36.133033752441406,
      "activations/layer11_attention_weight_min": -35.26080322265625,
      "activations/layer12_attention_weight_max": 28.18181800842285,
      "activations/layer12_attention_weight_min": -28.3846378326416,
      "activations/layer13_attention_weight_max": 41.124664306640625,
      "activations/layer13_attention_weight_min": -37.51758575439453,
      "activations/layer14_attention_weight_max": 43.758121490478516,
      "activations/layer14_attention_weight_min": -38.129825592041016,
      "activations/layer15_attention_weight_max": 42.583255767822266,
      "activations/layer15_attention_weight_min": -35.16838455200195,
      "activations/layer16_attention_weight_max": 30.64813995361328,
      "activations/layer16_attention_weight_min": -28.34906578063965,
      "activations/layer17_attention_weight_max": 32.647308349609375,
      "activations/layer17_attention_weight_min": -27.2980899810791,
      "activations/layer18_attention_weight_max": 33.82502365112305,
      "activations/layer18_attention_weight_min": -25.986696243286133,
      "activations/layer19_attention_weight_max": 36.536441802978516,
      "activations/layer19_attention_weight_min": -30.855056762695312,
      "activations/layer1_attention_weight_max": 16.553264617919922,
      "activations/layer1_attention_weight_min": -16.15001678466797,
      "activations/layer20_attention_weight_max": 30.228008270263672,
      "activations/layer20_attention_weight_min": -24.868928909301758,
      "activations/layer21_attention_weight_max": 31.245708465576172,
      "activations/layer21_attention_weight_min": -23.88449478149414,
      "activations/layer22_attention_weight_max": 45.865684509277344,
      "activations/layer22_attention_weight_min": -32.774505615234375,
      "activations/layer23_attention_weight_max": 33.70429229736328,
      "activations/layer23_attention_weight_min": -26.748485565185547,
      "activations/layer2_attention_weight_max": 32.878074645996094,
      "activations/layer2_attention_weight_min": -32.5760498046875,
      "activations/layer3_attention_weight_max": 101.34307861328125,
      "activations/layer3_attention_weight_min": -105.71831512451172,
      "activations/layer4_attention_weight_max": 109.88347625732422,
      "activations/layer4_attention_weight_min": -111.9200210571289,
      "activations/layer5_attention_weight_max": 50.39568328857422,
      "activations/layer5_attention_weight_min": -67.89463806152344,
      "activations/layer6_attention_weight_max": 46.379249572753906,
      "activations/layer6_attention_weight_min": -49.377010345458984,
      "activations/layer7_attention_weight_max": 100.52664947509766,
      "activations/layer7_attention_weight_min": -102.52350616455078,
      "activations/layer8_attention_weight_max": 47.066650390625,
      "activations/layer8_attention_weight_min": -45.90696716308594,
      "activations/layer9_attention_weight_max": 34.15208053588867,
      "activations/layer9_attention_weight_min": -35.283416748046875,
      "epoch": 21.79,
      "learning_rate": 9.561363636363636e-06,
      "loss": 2.7045,
      "step": 375050
    },
    {
      "activations/layer0_attention_weight_max": 16.56433868408203,
      "activations/layer0_attention_weight_min": -13.811898231506348,
      "activations/layer10_attention_weight_max": 36.14142608642578,
      "activations/layer10_attention_weight_min": -36.29216003417969,
      "activations/layer11_attention_weight_max": 36.509307861328125,
      "activations/layer11_attention_weight_min": -35.200950622558594,
      "activations/layer12_attention_weight_max": 28.51163101196289,
      "activations/layer12_attention_weight_min": -27.08238410949707,
      "activations/layer13_attention_weight_max": 39.3642578125,
      "activations/layer13_attention_weight_min": -37.40476989746094,
      "activations/layer14_attention_weight_max": 45.51692581176758,
      "activations/layer14_attention_weight_min": -39.50431442260742,
      "activations/layer15_attention_weight_max": 40.09955596923828,
      "activations/layer15_attention_weight_min": -42.56963348388672,
      "activations/layer16_attention_weight_max": 31.53592872619629,
      "activations/layer16_attention_weight_min": -30.738977432250977,
      "activations/layer17_attention_weight_max": 29.277557373046875,
      "activations/layer17_attention_weight_min": -27.914644241333008,
      "activations/layer18_attention_weight_max": 35.14266586303711,
      "activations/layer18_attention_weight_min": -29.96375846862793,
      "activations/layer19_attention_weight_max": 35.165489196777344,
      "activations/layer19_attention_weight_min": -33.3219108581543,
      "activations/layer1_attention_weight_max": 15.419144630432129,
      "activations/layer1_attention_weight_min": -14.770415306091309,
      "activations/layer20_attention_weight_max": 29.590435028076172,
      "activations/layer20_attention_weight_min": -25.964616775512695,
      "activations/layer21_attention_weight_max": 28.754947662353516,
      "activations/layer21_attention_weight_min": -26.441801071166992,
      "activations/layer22_attention_weight_max": 41.6407470703125,
      "activations/layer22_attention_weight_min": -38.27739715576172,
      "activations/layer23_attention_weight_max": 32.43716812133789,
      "activations/layer23_attention_weight_min": -28.40118980407715,
      "activations/layer2_attention_weight_max": 32.871089935302734,
      "activations/layer2_attention_weight_min": -33.74917984008789,
      "activations/layer3_attention_weight_max": 104.86138916015625,
      "activations/layer3_attention_weight_min": -107.64620971679688,
      "activations/layer4_attention_weight_max": 110.4258804321289,
      "activations/layer4_attention_weight_min": -111.38789367675781,
      "activations/layer5_attention_weight_max": 51.80084991455078,
      "activations/layer5_attention_weight_min": -62.589508056640625,
      "activations/layer6_attention_weight_max": 47.23318099975586,
      "activations/layer6_attention_weight_min": -50.24843978881836,
      "activations/layer7_attention_weight_max": 103.1932373046875,
      "activations/layer7_attention_weight_min": -106.36080932617188,
      "activations/layer8_attention_weight_max": 43.861610412597656,
      "activations/layer8_attention_weight_min": -47.99644088745117,
      "activations/layer9_attention_weight_max": 33.216896057128906,
      "activations/layer9_attention_weight_min": -36.8466682434082,
      "epoch": 21.8,
      "learning_rate": 9.54242424242424e-06,
      "loss": 2.6946,
      "step": 375100
    },
    {
      "activations/layer0_attention_weight_max": 15.754644393920898,
      "activations/layer0_attention_weight_min": -14.411870002746582,
      "activations/layer10_attention_weight_max": 38.8571891784668,
      "activations/layer10_attention_weight_min": -36.95838165283203,
      "activations/layer11_attention_weight_max": 37.343082427978516,
      "activations/layer11_attention_weight_min": -37.388877868652344,
      "activations/layer12_attention_weight_max": 30.423376083374023,
      "activations/layer12_attention_weight_min": -29.384397506713867,
      "activations/layer13_attention_weight_max": 43.35182189941406,
      "activations/layer13_attention_weight_min": -38.55289840698242,
      "activations/layer14_attention_weight_max": 46.61238479614258,
      "activations/layer14_attention_weight_min": -40.910560607910156,
      "activations/layer15_attention_weight_max": 40.3756217956543,
      "activations/layer15_attention_weight_min": -37.27016067504883,
      "activations/layer16_attention_weight_max": 30.84355354309082,
      "activations/layer16_attention_weight_min": -29.144601821899414,
      "activations/layer17_attention_weight_max": 33.64064407348633,
      "activations/layer17_attention_weight_min": -27.574718475341797,
      "activations/layer18_attention_weight_max": 32.97074890136719,
      "activations/layer18_attention_weight_min": -27.247343063354492,
      "activations/layer19_attention_weight_max": 37.15420150756836,
      "activations/layer19_attention_weight_min": -33.687583923339844,
      "activations/layer1_attention_weight_max": 15.653755187988281,
      "activations/layer1_attention_weight_min": -15.456840515136719,
      "activations/layer20_attention_weight_max": 28.82662010192871,
      "activations/layer20_attention_weight_min": -26.4022216796875,
      "activations/layer21_attention_weight_max": 30.682418823242188,
      "activations/layer21_attention_weight_min": -26.85417366027832,
      "activations/layer22_attention_weight_max": 41.86330795288086,
      "activations/layer22_attention_weight_min": -33.39223861694336,
      "activations/layer23_attention_weight_max": 35.28580093383789,
      "activations/layer23_attention_weight_min": -26.889389038085938,
      "activations/layer2_attention_weight_max": 33.86758804321289,
      "activations/layer2_attention_weight_min": -33.288177490234375,
      "activations/layer3_attention_weight_max": 106.45073699951172,
      "activations/layer3_attention_weight_min": -109.54820251464844,
      "activations/layer4_attention_weight_max": 115.47151184082031,
      "activations/layer4_attention_weight_min": -115.99010467529297,
      "activations/layer5_attention_weight_max": 52.23462677001953,
      "activations/layer5_attention_weight_min": -61.49314880371094,
      "activations/layer6_attention_weight_max": 49.22385025024414,
      "activations/layer6_attention_weight_min": -50.36789321899414,
      "activations/layer7_attention_weight_max": 102.42463684082031,
      "activations/layer7_attention_weight_min": -107.60775756835938,
      "activations/layer8_attention_weight_max": 46.30509948730469,
      "activations/layer8_attention_weight_min": -50.032936096191406,
      "activations/layer9_attention_weight_max": 36.87482833862305,
      "activations/layer9_attention_weight_min": -38.11857223510742,
      "epoch": 21.8,
      "learning_rate": 9.523484848484847e-06,
      "loss": 2.6964,
      "step": 375150
    },
    {
      "activations/layer0_attention_weight_max": 17.418787002563477,
      "activations/layer0_attention_weight_min": -14.208681106567383,
      "activations/layer10_attention_weight_max": 37.702964782714844,
      "activations/layer10_attention_weight_min": -36.70344924926758,
      "activations/layer11_attention_weight_max": 37.58164978027344,
      "activations/layer11_attention_weight_min": -37.54093933105469,
      "activations/layer12_attention_weight_max": 28.11483383178711,
      "activations/layer12_attention_weight_min": -28.940874099731445,
      "activations/layer13_attention_weight_max": 42.16302490234375,
      "activations/layer13_attention_weight_min": -38.885009765625,
      "activations/layer14_attention_weight_max": 45.309730529785156,
      "activations/layer14_attention_weight_min": -42.76499557495117,
      "activations/layer15_attention_weight_max": 43.21538543701172,
      "activations/layer15_attention_weight_min": -39.634010314941406,
      "activations/layer16_attention_weight_max": 31.94788360595703,
      "activations/layer16_attention_weight_min": -30.78166389465332,
      "activations/layer17_attention_weight_max": 31.813501358032227,
      "activations/layer17_attention_weight_min": -26.888778686523438,
      "activations/layer18_attention_weight_max": 36.705318450927734,
      "activations/layer18_attention_weight_min": -26.42075538635254,
      "activations/layer19_attention_weight_max": 43.37111282348633,
      "activations/layer19_attention_weight_min": -33.66790008544922,
      "activations/layer1_attention_weight_max": 15.103382110595703,
      "activations/layer1_attention_weight_min": -14.869712829589844,
      "activations/layer20_attention_weight_max": 35.55717086791992,
      "activations/layer20_attention_weight_min": -27.854833602905273,
      "activations/layer21_attention_weight_max": 36.0892219543457,
      "activations/layer21_attention_weight_min": -26.34294891357422,
      "activations/layer22_attention_weight_max": 44.7061767578125,
      "activations/layer22_attention_weight_min": -33.77060317993164,
      "activations/layer23_attention_weight_max": 37.43038558959961,
      "activations/layer23_attention_weight_min": -27.141273498535156,
      "activations/layer2_attention_weight_max": 33.45962142944336,
      "activations/layer2_attention_weight_min": -32.877845764160156,
      "activations/layer3_attention_weight_max": 107.44439697265625,
      "activations/layer3_attention_weight_min": -105.94688415527344,
      "activations/layer4_attention_weight_max": 112.12605285644531,
      "activations/layer4_attention_weight_min": -113.2579116821289,
      "activations/layer5_attention_weight_max": 51.03623962402344,
      "activations/layer5_attention_weight_min": -61.05390548706055,
      "activations/layer6_attention_weight_max": 45.97121810913086,
      "activations/layer6_attention_weight_min": -49.36052322387695,
      "activations/layer7_attention_weight_max": 99.75371551513672,
      "activations/layer7_attention_weight_min": -106.88849639892578,
      "activations/layer8_attention_weight_max": 44.06416702270508,
      "activations/layer8_attention_weight_min": -48.53929901123047,
      "activations/layer9_attention_weight_max": 34.049861907958984,
      "activations/layer9_attention_weight_min": -35.99632263183594,
      "epoch": 21.8,
      "learning_rate": 9.504545454545452e-06,
      "loss": 2.6982,
      "step": 375200
    },
    {
      "activations/layer0_attention_weight_max": 15.959193229675293,
      "activations/layer0_attention_weight_min": -14.29625415802002,
      "activations/layer10_attention_weight_max": 36.00821304321289,
      "activations/layer10_attention_weight_min": -36.02803421020508,
      "activations/layer11_attention_weight_max": 34.978271484375,
      "activations/layer11_attention_weight_min": -35.66181182861328,
      "activations/layer12_attention_weight_max": 27.254165649414062,
      "activations/layer12_attention_weight_min": -27.253267288208008,
      "activations/layer13_attention_weight_max": 39.37239074707031,
      "activations/layer13_attention_weight_min": -36.952552795410156,
      "activations/layer14_attention_weight_max": 43.09872055053711,
      "activations/layer14_attention_weight_min": -39.68550109863281,
      "activations/layer15_attention_weight_max": 38.04832458496094,
      "activations/layer15_attention_weight_min": -37.43889236450195,
      "activations/layer16_attention_weight_max": 29.410322189331055,
      "activations/layer16_attention_weight_min": -27.397262573242188,
      "activations/layer17_attention_weight_max": 30.514095306396484,
      "activations/layer17_attention_weight_min": -28.644325256347656,
      "activations/layer18_attention_weight_max": 30.589946746826172,
      "activations/layer18_attention_weight_min": -27.327430725097656,
      "activations/layer19_attention_weight_max": 35.325645446777344,
      "activations/layer19_attention_weight_min": -36.183284759521484,
      "activations/layer1_attention_weight_max": 15.215130805969238,
      "activations/layer1_attention_weight_min": -14.650686264038086,
      "activations/layer20_attention_weight_max": 31.367137908935547,
      "activations/layer20_attention_weight_min": -30.884252548217773,
      "activations/layer21_attention_weight_max": 31.288183212280273,
      "activations/layer21_attention_weight_min": -28.795011520385742,
      "activations/layer22_attention_weight_max": 40.20404052734375,
      "activations/layer22_attention_weight_min": -36.032222747802734,
      "activations/layer23_attention_weight_max": 35.377708435058594,
      "activations/layer23_attention_weight_min": -27.15363121032715,
      "activations/layer2_attention_weight_max": 33.046424865722656,
      "activations/layer2_attention_weight_min": -32.883544921875,
      "activations/layer3_attention_weight_max": 107.79576873779297,
      "activations/layer3_attention_weight_min": -106.06609344482422,
      "activations/layer4_attention_weight_max": 109.8336410522461,
      "activations/layer4_attention_weight_min": -113.156982421875,
      "activations/layer5_attention_weight_max": 51.40965270996094,
      "activations/layer5_attention_weight_min": -60.7703971862793,
      "activations/layer6_attention_weight_max": 47.13502502441406,
      "activations/layer6_attention_weight_min": -48.939353942871094,
      "activations/layer7_attention_weight_max": 97.0320053100586,
      "activations/layer7_attention_weight_min": -98.95313262939453,
      "activations/layer8_attention_weight_max": 45.62757873535156,
      "activations/layer8_attention_weight_min": -47.28939437866211,
      "activations/layer9_attention_weight_max": 32.88343048095703,
      "activations/layer9_attention_weight_min": -36.13310241699219,
      "epoch": 21.8,
      "learning_rate": 9.485606060606059e-06,
      "loss": 2.7089,
      "step": 375250
    },
    {
      "activations/layer0_attention_weight_max": 17.04469108581543,
      "activations/layer0_attention_weight_min": -15.672384262084961,
      "activations/layer10_attention_weight_max": 36.19417190551758,
      "activations/layer10_attention_weight_min": -34.814247131347656,
      "activations/layer11_attention_weight_max": 35.99211502075195,
      "activations/layer11_attention_weight_min": -34.61903381347656,
      "activations/layer12_attention_weight_max": 27.36450958251953,
      "activations/layer12_attention_weight_min": -25.591384887695312,
      "activations/layer13_attention_weight_max": 40.33758544921875,
      "activations/layer13_attention_weight_min": -34.917884826660156,
      "activations/layer14_attention_weight_max": 42.71876907348633,
      "activations/layer14_attention_weight_min": -36.86606979370117,
      "activations/layer15_attention_weight_max": 37.736480712890625,
      "activations/layer15_attention_weight_min": -35.28193664550781,
      "activations/layer16_attention_weight_max": 27.73011589050293,
      "activations/layer16_attention_weight_min": -27.961732864379883,
      "activations/layer17_attention_weight_max": 30.941650390625,
      "activations/layer17_attention_weight_min": -24.981714248657227,
      "activations/layer18_attention_weight_max": 27.64447021484375,
      "activations/layer18_attention_weight_min": -24.280778884887695,
      "activations/layer19_attention_weight_max": 32.33769226074219,
      "activations/layer19_attention_weight_min": -29.68544578552246,
      "activations/layer1_attention_weight_max": 15.827530860900879,
      "activations/layer1_attention_weight_min": -14.366400718688965,
      "activations/layer20_attention_weight_max": 28.05696678161621,
      "activations/layer20_attention_weight_min": -23.26365852355957,
      "activations/layer21_attention_weight_max": 25.77874183654785,
      "activations/layer21_attention_weight_min": -22.758058547973633,
      "activations/layer22_attention_weight_max": 37.51496887207031,
      "activations/layer22_attention_weight_min": -28.451812744140625,
      "activations/layer23_attention_weight_max": 29.220365524291992,
      "activations/layer23_attention_weight_min": -23.504058837890625,
      "activations/layer2_attention_weight_max": 34.35196304321289,
      "activations/layer2_attention_weight_min": -33.1475944519043,
      "activations/layer3_attention_weight_max": 108.55281066894531,
      "activations/layer3_attention_weight_min": -106.99256896972656,
      "activations/layer4_attention_weight_max": 110.3187026977539,
      "activations/layer4_attention_weight_min": -111.61014556884766,
      "activations/layer5_attention_weight_max": 49.283531188964844,
      "activations/layer5_attention_weight_min": -62.245574951171875,
      "activations/layer6_attention_weight_max": 44.875511169433594,
      "activations/layer6_attention_weight_min": -48.04899597167969,
      "activations/layer7_attention_weight_max": 92.4783935546875,
      "activations/layer7_attention_weight_min": -102.38813781738281,
      "activations/layer8_attention_weight_max": 46.826194763183594,
      "activations/layer8_attention_weight_min": -46.469871520996094,
      "activations/layer9_attention_weight_max": 31.70318603515625,
      "activations/layer9_attention_weight_min": -35.70669174194336,
      "epoch": 21.81,
      "learning_rate": 9.466666666666667e-06,
      "loss": 2.6938,
      "step": 375300
    },
    {
      "activations/layer0_attention_weight_max": 17.510316848754883,
      "activations/layer0_attention_weight_min": -14.701766014099121,
      "activations/layer10_attention_weight_max": 36.23764419555664,
      "activations/layer10_attention_weight_min": -36.7227783203125,
      "activations/layer11_attention_weight_max": 34.81291961669922,
      "activations/layer11_attention_weight_min": -36.91828155517578,
      "activations/layer12_attention_weight_max": 26.7501163482666,
      "activations/layer12_attention_weight_min": -27.859546661376953,
      "activations/layer13_attention_weight_max": 38.16065216064453,
      "activations/layer13_attention_weight_min": -36.87242126464844,
      "activations/layer14_attention_weight_max": 42.59468078613281,
      "activations/layer14_attention_weight_min": -39.55642318725586,
      "activations/layer15_attention_weight_max": 41.144439697265625,
      "activations/layer15_attention_weight_min": -37.373390197753906,
      "activations/layer16_attention_weight_max": 29.3093204498291,
      "activations/layer16_attention_weight_min": -28.3122615814209,
      "activations/layer17_attention_weight_max": 29.238948822021484,
      "activations/layer17_attention_weight_min": -28.846038818359375,
      "activations/layer18_attention_weight_max": 30.510881423950195,
      "activations/layer18_attention_weight_min": -28.480806350708008,
      "activations/layer19_attention_weight_max": 32.51036071777344,
      "activations/layer19_attention_weight_min": -32.08865737915039,
      "activations/layer1_attention_weight_max": 16.09272575378418,
      "activations/layer1_attention_weight_min": -16.977468490600586,
      "activations/layer20_attention_weight_max": 27.806129455566406,
      "activations/layer20_attention_weight_min": -25.301389694213867,
      "activations/layer21_attention_weight_max": 28.189565658569336,
      "activations/layer21_attention_weight_min": -25.127283096313477,
      "activations/layer22_attention_weight_max": 40.16212844848633,
      "activations/layer22_attention_weight_min": -31.79937744140625,
      "activations/layer23_attention_weight_max": 30.712478637695312,
      "activations/layer23_attention_weight_min": -26.484786987304688,
      "activations/layer2_attention_weight_max": 35.34565734863281,
      "activations/layer2_attention_weight_min": -34.90766143798828,
      "activations/layer3_attention_weight_max": 108.89514923095703,
      "activations/layer3_attention_weight_min": -108.07025909423828,
      "activations/layer4_attention_weight_max": 108.88675689697266,
      "activations/layer4_attention_weight_min": -113.44112396240234,
      "activations/layer5_attention_weight_max": 50.44523620605469,
      "activations/layer5_attention_weight_min": -63.52245330810547,
      "activations/layer6_attention_weight_max": 45.15011215209961,
      "activations/layer6_attention_weight_min": -48.557945251464844,
      "activations/layer7_attention_weight_max": 102.0357894897461,
      "activations/layer7_attention_weight_min": -96.9355697631836,
      "activations/layer8_attention_weight_max": 42.37424087524414,
      "activations/layer8_attention_weight_min": -46.58366012573242,
      "activations/layer9_attention_weight_max": 32.015377044677734,
      "activations/layer9_attention_weight_min": -36.62503433227539,
      "epoch": 21.81,
      "learning_rate": 9.447727272727274e-06,
      "loss": 2.7051,
      "step": 375350
    },
    {
      "activations/layer0_attention_weight_max": 16.483835220336914,
      "activations/layer0_attention_weight_min": -16.414762496948242,
      "activations/layer10_attention_weight_max": 34.949092864990234,
      "activations/layer10_attention_weight_min": -36.392784118652344,
      "activations/layer11_attention_weight_max": 35.081085205078125,
      "activations/layer11_attention_weight_min": -35.52473068237305,
      "activations/layer12_attention_weight_max": 32.86540603637695,
      "activations/layer12_attention_weight_min": -29.382278442382812,
      "activations/layer13_attention_weight_max": 42.39432907104492,
      "activations/layer13_attention_weight_min": -37.547080993652344,
      "activations/layer14_attention_weight_max": 44.241119384765625,
      "activations/layer14_attention_weight_min": -38.289188385009766,
      "activations/layer15_attention_weight_max": 39.89625930786133,
      "activations/layer15_attention_weight_min": -39.65501022338867,
      "activations/layer16_attention_weight_max": 30.145421981811523,
      "activations/layer16_attention_weight_min": -28.581825256347656,
      "activations/layer17_attention_weight_max": 30.433929443359375,
      "activations/layer17_attention_weight_min": -27.30423355102539,
      "activations/layer18_attention_weight_max": 31.668869018554688,
      "activations/layer18_attention_weight_min": -25.450056076049805,
      "activations/layer19_attention_weight_max": 32.44303512573242,
      "activations/layer19_attention_weight_min": -33.12886047363281,
      "activations/layer1_attention_weight_max": 16.70248031616211,
      "activations/layer1_attention_weight_min": -15.942418098449707,
      "activations/layer20_attention_weight_max": 28.2000732421875,
      "activations/layer20_attention_weight_min": -24.086708068847656,
      "activations/layer21_attention_weight_max": 28.80224609375,
      "activations/layer21_attention_weight_min": -24.38357925415039,
      "activations/layer22_attention_weight_max": 41.12393569946289,
      "activations/layer22_attention_weight_min": -31.019311904907227,
      "activations/layer23_attention_weight_max": 30.510366439819336,
      "activations/layer23_attention_weight_min": -25.10271453857422,
      "activations/layer2_attention_weight_max": 33.02396774291992,
      "activations/layer2_attention_weight_min": -33.460540771484375,
      "activations/layer3_attention_weight_max": 104.6973648071289,
      "activations/layer3_attention_weight_min": -106.05162811279297,
      "activations/layer4_attention_weight_max": 110.89920806884766,
      "activations/layer4_attention_weight_min": -114.80427551269531,
      "activations/layer5_attention_weight_max": 51.162872314453125,
      "activations/layer5_attention_weight_min": -64.01942443847656,
      "activations/layer6_attention_weight_max": 46.39175033569336,
      "activations/layer6_attention_weight_min": -48.8714485168457,
      "activations/layer7_attention_weight_max": 93.08057403564453,
      "activations/layer7_attention_weight_min": -106.9488525390625,
      "activations/layer8_attention_weight_max": 44.89024353027344,
      "activations/layer8_attention_weight_min": -45.279197692871094,
      "activations/layer9_attention_weight_max": 32.69012451171875,
      "activations/layer9_attention_weight_min": -35.55839538574219,
      "epoch": 21.81,
      "learning_rate": 9.428787878787878e-06,
      "loss": 2.7018,
      "step": 375400
    },
    {
      "activations/layer0_attention_weight_max": 17.074384689331055,
      "activations/layer0_attention_weight_min": -14.040475845336914,
      "activations/layer10_attention_weight_max": 34.07322311401367,
      "activations/layer10_attention_weight_min": -35.13685607910156,
      "activations/layer11_attention_weight_max": 32.91329574584961,
      "activations/layer11_attention_weight_min": -33.72318649291992,
      "activations/layer12_attention_weight_max": 24.92742347717285,
      "activations/layer12_attention_weight_min": -26.253231048583984,
      "activations/layer13_attention_weight_max": 39.630306243896484,
      "activations/layer13_attention_weight_min": -35.93497848510742,
      "activations/layer14_attention_weight_max": 40.11852264404297,
      "activations/layer14_attention_weight_min": -40.10089874267578,
      "activations/layer15_attention_weight_max": 37.19422149658203,
      "activations/layer15_attention_weight_min": -35.157161712646484,
      "activations/layer16_attention_weight_max": 31.29376983642578,
      "activations/layer16_attention_weight_min": -27.933061599731445,
      "activations/layer17_attention_weight_max": 28.92136001586914,
      "activations/layer17_attention_weight_min": -25.918874740600586,
      "activations/layer18_attention_weight_max": 30.455482482910156,
      "activations/layer18_attention_weight_min": -27.653114318847656,
      "activations/layer19_attention_weight_max": 34.09298324584961,
      "activations/layer19_attention_weight_min": -31.221052169799805,
      "activations/layer1_attention_weight_max": 16.954198837280273,
      "activations/layer1_attention_weight_min": -15.053821563720703,
      "activations/layer20_attention_weight_max": 32.07304763793945,
      "activations/layer20_attention_weight_min": -23.19988250732422,
      "activations/layer21_attention_weight_max": 28.428855895996094,
      "activations/layer21_attention_weight_min": -23.9603271484375,
      "activations/layer22_attention_weight_max": 38.50652313232422,
      "activations/layer22_attention_weight_min": -29.933401107788086,
      "activations/layer23_attention_weight_max": 30.71355438232422,
      "activations/layer23_attention_weight_min": -25.3839111328125,
      "activations/layer2_attention_weight_max": 32.926734924316406,
      "activations/layer2_attention_weight_min": -31.461551666259766,
      "activations/layer3_attention_weight_max": 97.80331420898438,
      "activations/layer3_attention_weight_min": -103.0837173461914,
      "activations/layer4_attention_weight_max": 102.49299621582031,
      "activations/layer4_attention_weight_min": -109.0914535522461,
      "activations/layer5_attention_weight_max": 48.61568832397461,
      "activations/layer5_attention_weight_min": -57.41106414794922,
      "activations/layer6_attention_weight_max": 43.67129898071289,
      "activations/layer6_attention_weight_min": -46.686988830566406,
      "activations/layer7_attention_weight_max": 97.75537109375,
      "activations/layer7_attention_weight_min": -105.2592544555664,
      "activations/layer8_attention_weight_max": 45.19941711425781,
      "activations/layer8_attention_weight_min": -45.00825119018555,
      "activations/layer9_attention_weight_max": 32.1461067199707,
      "activations/layer9_attention_weight_min": -33.99937438964844,
      "epoch": 21.82,
      "learning_rate": 9.409848484848485e-06,
      "loss": 2.7066,
      "step": 375450
    },
    {
      "activations/layer0_attention_weight_max": 15.789132118225098,
      "activations/layer0_attention_weight_min": -13.816163063049316,
      "activations/layer10_attention_weight_max": 37.15494155883789,
      "activations/layer10_attention_weight_min": -38.95700454711914,
      "activations/layer11_attention_weight_max": 36.72996139526367,
      "activations/layer11_attention_weight_min": -38.82377624511719,
      "activations/layer12_attention_weight_max": 29.41275405883789,
      "activations/layer12_attention_weight_min": -28.717775344848633,
      "activations/layer13_attention_weight_max": 44.14430236816406,
      "activations/layer13_attention_weight_min": -38.801170349121094,
      "activations/layer14_attention_weight_max": 49.0897331237793,
      "activations/layer14_attention_weight_min": -42.0791015625,
      "activations/layer15_attention_weight_max": 41.725303649902344,
      "activations/layer15_attention_weight_min": -41.02085876464844,
      "activations/layer16_attention_weight_max": 30.18638801574707,
      "activations/layer16_attention_weight_min": -29.160791397094727,
      "activations/layer17_attention_weight_max": 30.200883865356445,
      "activations/layer17_attention_weight_min": -27.263370513916016,
      "activations/layer18_attention_weight_max": 34.793460845947266,
      "activations/layer18_attention_weight_min": -26.363935470581055,
      "activations/layer19_attention_weight_max": 35.6796989440918,
      "activations/layer19_attention_weight_min": -32.88215637207031,
      "activations/layer1_attention_weight_max": 16.427288055419922,
      "activations/layer1_attention_weight_min": -15.216907501220703,
      "activations/layer20_attention_weight_max": 32.49917221069336,
      "activations/layer20_attention_weight_min": -25.54527473449707,
      "activations/layer21_attention_weight_max": 29.592803955078125,
      "activations/layer21_attention_weight_min": -24.29283332824707,
      "activations/layer22_attention_weight_max": 48.26018142700195,
      "activations/layer22_attention_weight_min": -32.22908020019531,
      "activations/layer23_attention_weight_max": 33.88357162475586,
      "activations/layer23_attention_weight_min": -26.539386749267578,
      "activations/layer2_attention_weight_max": 36.13360595703125,
      "activations/layer2_attention_weight_min": -35.4781379699707,
      "activations/layer3_attention_weight_max": 116.68419647216797,
      "activations/layer3_attention_weight_min": -114.98992919921875,
      "activations/layer4_attention_weight_max": 118.09138488769531,
      "activations/layer4_attention_weight_min": -122.7909164428711,
      "activations/layer5_attention_weight_max": 54.37995910644531,
      "activations/layer5_attention_weight_min": -65.36952209472656,
      "activations/layer6_attention_weight_max": 50.04204559326172,
      "activations/layer6_attention_weight_min": -50.56374740600586,
      "activations/layer7_attention_weight_max": 104.81761932373047,
      "activations/layer7_attention_weight_min": -110.10636901855469,
      "activations/layer8_attention_weight_max": 48.15372848510742,
      "activations/layer8_attention_weight_min": -50.25978469848633,
      "activations/layer9_attention_weight_max": 37.11429977416992,
      "activations/layer9_attention_weight_min": -36.5829963684082,
      "epoch": 21.82,
      "learning_rate": 9.39090909090909e-06,
      "loss": 2.7141,
      "step": 375500
    },
    {
      "activations/layer0_attention_weight_max": 17.08708381652832,
      "activations/layer0_attention_weight_min": -12.794387817382812,
      "activations/layer10_attention_weight_max": 36.11745071411133,
      "activations/layer10_attention_weight_min": -35.9135627746582,
      "activations/layer11_attention_weight_max": 37.81977081298828,
      "activations/layer11_attention_weight_min": -36.07310485839844,
      "activations/layer12_attention_weight_max": 28.247339248657227,
      "activations/layer12_attention_weight_min": -27.63132667541504,
      "activations/layer13_attention_weight_max": 41.557247161865234,
      "activations/layer13_attention_weight_min": -36.38094711303711,
      "activations/layer14_attention_weight_max": 42.95714569091797,
      "activations/layer14_attention_weight_min": -39.97089767456055,
      "activations/layer15_attention_weight_max": 38.843143463134766,
      "activations/layer15_attention_weight_min": -36.94207000732422,
      "activations/layer16_attention_weight_max": 28.824480056762695,
      "activations/layer16_attention_weight_min": -27.504484176635742,
      "activations/layer17_attention_weight_max": 33.28206253051758,
      "activations/layer17_attention_weight_min": -26.457386016845703,
      "activations/layer18_attention_weight_max": 31.4977970123291,
      "activations/layer18_attention_weight_min": -25.437129974365234,
      "activations/layer19_attention_weight_max": 37.659820556640625,
      "activations/layer19_attention_weight_min": -31.32741355895996,
      "activations/layer1_attention_weight_max": 17.86960220336914,
      "activations/layer1_attention_weight_min": -14.616607666015625,
      "activations/layer20_attention_weight_max": 33.30750274658203,
      "activations/layer20_attention_weight_min": -26.25792121887207,
      "activations/layer21_attention_weight_max": 31.662017822265625,
      "activations/layer21_attention_weight_min": -25.444595336914062,
      "activations/layer22_attention_weight_max": 44.1933708190918,
      "activations/layer22_attention_weight_min": -31.364492416381836,
      "activations/layer23_attention_weight_max": 36.949745178222656,
      "activations/layer23_attention_weight_min": -24.915199279785156,
      "activations/layer2_attention_weight_max": 33.16023635864258,
      "activations/layer2_attention_weight_min": -32.57100296020508,
      "activations/layer3_attention_weight_max": 106.47117614746094,
      "activations/layer3_attention_weight_min": -108.1451187133789,
      "activations/layer4_attention_weight_max": 108.52352142333984,
      "activations/layer4_attention_weight_min": -111.0027084350586,
      "activations/layer5_attention_weight_max": 51.866966247558594,
      "activations/layer5_attention_weight_min": -61.12199401855469,
      "activations/layer6_attention_weight_max": 45.518924713134766,
      "activations/layer6_attention_weight_min": -49.35563278198242,
      "activations/layer7_attention_weight_max": 94.23766326904297,
      "activations/layer7_attention_weight_min": -101.7796401977539,
      "activations/layer8_attention_weight_max": 42.39894104003906,
      "activations/layer8_attention_weight_min": -45.15316390991211,
      "activations/layer9_attention_weight_max": 33.109981536865234,
      "activations/layer9_attention_weight_min": -36.018592834472656,
      "epoch": 21.82,
      "learning_rate": 9.371969696969696e-06,
      "loss": 2.7137,
      "step": 375550
    },
    {
      "activations/layer0_attention_weight_max": 17.563499450683594,
      "activations/layer0_attention_weight_min": -14.493938446044922,
      "activations/layer10_attention_weight_max": 36.605628967285156,
      "activations/layer10_attention_weight_min": -35.960792541503906,
      "activations/layer11_attention_weight_max": 38.64594650268555,
      "activations/layer11_attention_weight_min": -37.26459884643555,
      "activations/layer12_attention_weight_max": 26.90666961669922,
      "activations/layer12_attention_weight_min": -28.561355590820312,
      "activations/layer13_attention_weight_max": 40.70644760131836,
      "activations/layer13_attention_weight_min": -36.3780517578125,
      "activations/layer14_attention_weight_max": 44.374488830566406,
      "activations/layer14_attention_weight_min": -40.318782806396484,
      "activations/layer15_attention_weight_max": 42.43054962158203,
      "activations/layer15_attention_weight_min": -38.7868537902832,
      "activations/layer16_attention_weight_max": 32.20505905151367,
      "activations/layer16_attention_weight_min": -29.057891845703125,
      "activations/layer17_attention_weight_max": 30.669946670532227,
      "activations/layer17_attention_weight_min": -27.45383071899414,
      "activations/layer18_attention_weight_max": 36.31822967529297,
      "activations/layer18_attention_weight_min": -27.160667419433594,
      "activations/layer19_attention_weight_max": 37.14287567138672,
      "activations/layer19_attention_weight_min": -34.39179611206055,
      "activations/layer1_attention_weight_max": 17.831449508666992,
      "activations/layer1_attention_weight_min": -14.598387718200684,
      "activations/layer20_attention_weight_max": 28.819494247436523,
      "activations/layer20_attention_weight_min": -24.980791091918945,
      "activations/layer21_attention_weight_max": 29.513364791870117,
      "activations/layer21_attention_weight_min": -24.129287719726562,
      "activations/layer22_attention_weight_max": 43.59933090209961,
      "activations/layer22_attention_weight_min": -31.96553611755371,
      "activations/layer23_attention_weight_max": 33.18421173095703,
      "activations/layer23_attention_weight_min": -28.147754669189453,
      "activations/layer2_attention_weight_max": 33.666107177734375,
      "activations/layer2_attention_weight_min": -33.96458435058594,
      "activations/layer3_attention_weight_max": 107.09700775146484,
      "activations/layer3_attention_weight_min": -107.28477478027344,
      "activations/layer4_attention_weight_max": 109.33922576904297,
      "activations/layer4_attention_weight_min": -113.46562957763672,
      "activations/layer5_attention_weight_max": 52.80219268798828,
      "activations/layer5_attention_weight_min": -62.21855163574219,
      "activations/layer6_attention_weight_max": 45.04016876220703,
      "activations/layer6_attention_weight_min": -48.18470764160156,
      "activations/layer7_attention_weight_max": 103.48871612548828,
      "activations/layer7_attention_weight_min": -102.11198425292969,
      "activations/layer8_attention_weight_max": 43.40950012207031,
      "activations/layer8_attention_weight_min": -47.73629379272461,
      "activations/layer9_attention_weight_max": 33.951507568359375,
      "activations/layer9_attention_weight_min": -36.25307846069336,
      "epoch": 21.82,
      "learning_rate": 9.353030303030303e-06,
      "loss": 2.7136,
      "step": 375600
    },
    {
      "activations/layer0_attention_weight_max": 17.392578125,
      "activations/layer0_attention_weight_min": -13.682968139648438,
      "activations/layer10_attention_weight_max": 37.72788619995117,
      "activations/layer10_attention_weight_min": -35.149932861328125,
      "activations/layer11_attention_weight_max": 36.609256744384766,
      "activations/layer11_attention_weight_min": -35.600494384765625,
      "activations/layer12_attention_weight_max": 26.368213653564453,
      "activations/layer12_attention_weight_min": -27.275264739990234,
      "activations/layer13_attention_weight_max": 41.79185485839844,
      "activations/layer13_attention_weight_min": -39.3602294921875,
      "activations/layer14_attention_weight_max": 44.413291931152344,
      "activations/layer14_attention_weight_min": -40.693538665771484,
      "activations/layer15_attention_weight_max": 43.62742233276367,
      "activations/layer15_attention_weight_min": -37.953407287597656,
      "activations/layer16_attention_weight_max": 28.5374755859375,
      "activations/layer16_attention_weight_min": -27.937318801879883,
      "activations/layer17_attention_weight_max": 27.13915252685547,
      "activations/layer17_attention_weight_min": -26.54823112487793,
      "activations/layer18_attention_weight_max": 31.006332397460938,
      "activations/layer18_attention_weight_min": -25.763608932495117,
      "activations/layer19_attention_weight_max": 31.598783493041992,
      "activations/layer19_attention_weight_min": -31.526092529296875,
      "activations/layer1_attention_weight_max": 16.922489166259766,
      "activations/layer1_attention_weight_min": -16.67624282836914,
      "activations/layer20_attention_weight_max": 28.654550552368164,
      "activations/layer20_attention_weight_min": -25.25627326965332,
      "activations/layer21_attention_weight_max": 26.579038619995117,
      "activations/layer21_attention_weight_min": -23.68383026123047,
      "activations/layer22_attention_weight_max": 36.49858474731445,
      "activations/layer22_attention_weight_min": -30.28312110900879,
      "activations/layer23_attention_weight_max": 29.568077087402344,
      "activations/layer23_attention_weight_min": -25.127073287963867,
      "activations/layer2_attention_weight_max": 33.398834228515625,
      "activations/layer2_attention_weight_min": -32.87135696411133,
      "activations/layer3_attention_weight_max": 111.70616912841797,
      "activations/layer3_attention_weight_min": -107.12713623046875,
      "activations/layer4_attention_weight_max": 115.56917572021484,
      "activations/layer4_attention_weight_min": -113.47380828857422,
      "activations/layer5_attention_weight_max": 53.849822998046875,
      "activations/layer5_attention_weight_min": -63.52408981323242,
      "activations/layer6_attention_weight_max": 47.57618713378906,
      "activations/layer6_attention_weight_min": -48.909847259521484,
      "activations/layer7_attention_weight_max": 102.47970581054688,
      "activations/layer7_attention_weight_min": -103.75855255126953,
      "activations/layer8_attention_weight_max": 47.105857849121094,
      "activations/layer8_attention_weight_min": -49.38372802734375,
      "activations/layer9_attention_weight_max": 33.926456451416016,
      "activations/layer9_attention_weight_min": -36.07416534423828,
      "epoch": 21.83,
      "learning_rate": 9.334090909090908e-06,
      "loss": 2.7042,
      "step": 375650
    },
    {
      "activations/layer0_attention_weight_max": 16.755346298217773,
      "activations/layer0_attention_weight_min": -15.575518608093262,
      "activations/layer10_attention_weight_max": 38.4757194519043,
      "activations/layer10_attention_weight_min": -38.95656204223633,
      "activations/layer11_attention_weight_max": 37.623748779296875,
      "activations/layer11_attention_weight_min": -38.454925537109375,
      "activations/layer12_attention_weight_max": 26.367277145385742,
      "activations/layer12_attention_weight_min": -29.352230072021484,
      "activations/layer13_attention_weight_max": 38.53426742553711,
      "activations/layer13_attention_weight_min": -37.14811325073242,
      "activations/layer14_attention_weight_max": 43.99342346191406,
      "activations/layer14_attention_weight_min": -39.84459686279297,
      "activations/layer15_attention_weight_max": 37.295677185058594,
      "activations/layer15_attention_weight_min": -39.36996078491211,
      "activations/layer16_attention_weight_max": 29.23512840270996,
      "activations/layer16_attention_weight_min": -29.9801025390625,
      "activations/layer17_attention_weight_max": 28.232507705688477,
      "activations/layer17_attention_weight_min": -27.725051879882812,
      "activations/layer18_attention_weight_max": 30.079383850097656,
      "activations/layer18_attention_weight_min": -26.58982276916504,
      "activations/layer19_attention_weight_max": 32.21596145629883,
      "activations/layer19_attention_weight_min": -32.6818733215332,
      "activations/layer1_attention_weight_max": 17.32819366455078,
      "activations/layer1_attention_weight_min": -16.62626075744629,
      "activations/layer20_attention_weight_max": 26.80776596069336,
      "activations/layer20_attention_weight_min": -25.887678146362305,
      "activations/layer21_attention_weight_max": 29.678592681884766,
      "activations/layer21_attention_weight_min": -27.27922248840332,
      "activations/layer22_attention_weight_max": 42.58709716796875,
      "activations/layer22_attention_weight_min": -33.956233978271484,
      "activations/layer23_attention_weight_max": 33.066062927246094,
      "activations/layer23_attention_weight_min": -28.932552337646484,
      "activations/layer2_attention_weight_max": 33.37061309814453,
      "activations/layer2_attention_weight_min": -34.001243591308594,
      "activations/layer3_attention_weight_max": 107.47472381591797,
      "activations/layer3_attention_weight_min": -111.75717163085938,
      "activations/layer4_attention_weight_max": 110.5910873413086,
      "activations/layer4_attention_weight_min": -108.984619140625,
      "activations/layer5_attention_weight_max": 52.004737854003906,
      "activations/layer5_attention_weight_min": -65.8427734375,
      "activations/layer6_attention_weight_max": 47.72904586791992,
      "activations/layer6_attention_weight_min": -48.834537506103516,
      "activations/layer7_attention_weight_max": 96.1353759765625,
      "activations/layer7_attention_weight_min": -104.70556640625,
      "activations/layer8_attention_weight_max": 45.36476516723633,
      "activations/layer8_attention_weight_min": -47.64212417602539,
      "activations/layer9_attention_weight_max": 35.58857345581055,
      "activations/layer9_attention_weight_min": -37.7073974609375,
      "epoch": 21.83,
      "learning_rate": 9.315151515151514e-06,
      "loss": 2.7021,
      "step": 375700
    },
    {
      "activations/layer0_attention_weight_max": 16.33876609802246,
      "activations/layer0_attention_weight_min": -14.029760360717773,
      "activations/layer10_attention_weight_max": 37.82785415649414,
      "activations/layer10_attention_weight_min": -36.751861572265625,
      "activations/layer11_attention_weight_max": 37.777381896972656,
      "activations/layer11_attention_weight_min": -36.2777099609375,
      "activations/layer12_attention_weight_max": 29.113346099853516,
      "activations/layer12_attention_weight_min": -28.957660675048828,
      "activations/layer13_attention_weight_max": 43.514102935791016,
      "activations/layer13_attention_weight_min": -38.48219299316406,
      "activations/layer14_attention_weight_max": 42.92634963989258,
      "activations/layer14_attention_weight_min": -37.8472785949707,
      "activations/layer15_attention_weight_max": 39.8015251159668,
      "activations/layer15_attention_weight_min": -36.89615249633789,
      "activations/layer16_attention_weight_max": 29.45088005065918,
      "activations/layer16_attention_weight_min": -27.52910614013672,
      "activations/layer17_attention_weight_max": 33.71978759765625,
      "activations/layer17_attention_weight_min": -25.689682006835938,
      "activations/layer18_attention_weight_max": 31.96368408203125,
      "activations/layer18_attention_weight_min": -25.973892211914062,
      "activations/layer19_attention_weight_max": 35.04774475097656,
      "activations/layer19_attention_weight_min": -33.81923294067383,
      "activations/layer1_attention_weight_max": 16.700929641723633,
      "activations/layer1_attention_weight_min": -16.165855407714844,
      "activations/layer20_attention_weight_max": 30.341405868530273,
      "activations/layer20_attention_weight_min": -25.112327575683594,
      "activations/layer21_attention_weight_max": 29.726211547851562,
      "activations/layer21_attention_weight_min": -24.870922088623047,
      "activations/layer22_attention_weight_max": 43.61520004272461,
      "activations/layer22_attention_weight_min": -34.2084846496582,
      "activations/layer23_attention_weight_max": 35.10405731201172,
      "activations/layer23_attention_weight_min": -28.735233306884766,
      "activations/layer2_attention_weight_max": 32.719627380371094,
      "activations/layer2_attention_weight_min": -33.5413818359375,
      "activations/layer3_attention_weight_max": 103.45862579345703,
      "activations/layer3_attention_weight_min": -108.72225189208984,
      "activations/layer4_attention_weight_max": 107.83039855957031,
      "activations/layer4_attention_weight_min": -112.60123443603516,
      "activations/layer5_attention_weight_max": 50.408935546875,
      "activations/layer5_attention_weight_min": -71.45796203613281,
      "activations/layer6_attention_weight_max": 46.77873611450195,
      "activations/layer6_attention_weight_min": -49.833377838134766,
      "activations/layer7_attention_weight_max": 98.85198211669922,
      "activations/layer7_attention_weight_min": -109.25270080566406,
      "activations/layer8_attention_weight_max": 45.29985046386719,
      "activations/layer8_attention_weight_min": -47.38666534423828,
      "activations/layer9_attention_weight_max": 36.50901794433594,
      "activations/layer9_attention_weight_min": -35.12125015258789,
      "epoch": 21.83,
      "learning_rate": 9.296212121212121e-06,
      "loss": 2.7034,
      "step": 375750
    },
    {
      "activations/layer0_attention_weight_max": 16.389219284057617,
      "activations/layer0_attention_weight_min": -15.0267972946167,
      "activations/layer10_attention_weight_max": 33.47235107421875,
      "activations/layer10_attention_weight_min": -35.694332122802734,
      "activations/layer11_attention_weight_max": 34.65924072265625,
      "activations/layer11_attention_weight_min": -36.51193618774414,
      "activations/layer12_attention_weight_max": 26.029664993286133,
      "activations/layer12_attention_weight_min": -26.980920791625977,
      "activations/layer13_attention_weight_max": 39.201576232910156,
      "activations/layer13_attention_weight_min": -35.26759338378906,
      "activations/layer14_attention_weight_max": 41.615928649902344,
      "activations/layer14_attention_weight_min": -38.777435302734375,
      "activations/layer15_attention_weight_max": 38.1141242980957,
      "activations/layer15_attention_weight_min": -39.05020523071289,
      "activations/layer16_attention_weight_max": 31.247268676757812,
      "activations/layer16_attention_weight_min": -31.10887336730957,
      "activations/layer17_attention_weight_max": 30.049442291259766,
      "activations/layer17_attention_weight_min": -28.35456085205078,
      "activations/layer18_attention_weight_max": 31.120037078857422,
      "activations/layer18_attention_weight_min": -28.287822723388672,
      "activations/layer19_attention_weight_max": 33.35197067260742,
      "activations/layer19_attention_weight_min": -31.23996353149414,
      "activations/layer1_attention_weight_max": 15.99567985534668,
      "activations/layer1_attention_weight_min": -15.038220405578613,
      "activations/layer20_attention_weight_max": 29.29567527770996,
      "activations/layer20_attention_weight_min": -27.15886878967285,
      "activations/layer21_attention_weight_max": 27.91042709350586,
      "activations/layer21_attention_weight_min": -25.380216598510742,
      "activations/layer22_attention_weight_max": 39.297325134277344,
      "activations/layer22_attention_weight_min": -31.62349510192871,
      "activations/layer23_attention_weight_max": 30.3814754486084,
      "activations/layer23_attention_weight_min": -25.974180221557617,
      "activations/layer2_attention_weight_max": 34.62431335449219,
      "activations/layer2_attention_weight_min": -32.701072692871094,
      "activations/layer3_attention_weight_max": 111.25064849853516,
      "activations/layer3_attention_weight_min": -108.13786315917969,
      "activations/layer4_attention_weight_max": 112.22633361816406,
      "activations/layer4_attention_weight_min": -111.60498046875,
      "activations/layer5_attention_weight_max": 55.56786346435547,
      "activations/layer5_attention_weight_min": -61.22991180419922,
      "activations/layer6_attention_weight_max": 45.682411193847656,
      "activations/layer6_attention_weight_min": -47.373085021972656,
      "activations/layer7_attention_weight_max": 95.91072845458984,
      "activations/layer7_attention_weight_min": -101.19644927978516,
      "activations/layer8_attention_weight_max": 43.26925277709961,
      "activations/layer8_attention_weight_min": -45.19582748413086,
      "activations/layer9_attention_weight_max": 33.083335876464844,
      "activations/layer9_attention_weight_min": -34.12187194824219,
      "epoch": 21.84,
      "learning_rate": 9.277272727272726e-06,
      "loss": 2.7013,
      "step": 375800
    },
    {
      "activations/layer0_attention_weight_max": 17.44903564453125,
      "activations/layer0_attention_weight_min": -14.390854835510254,
      "activations/layer10_attention_weight_max": 35.83762741088867,
      "activations/layer10_attention_weight_min": -36.247657775878906,
      "activations/layer11_attention_weight_max": 36.80897521972656,
      "activations/layer11_attention_weight_min": -36.83013916015625,
      "activations/layer12_attention_weight_max": 25.673805236816406,
      "activations/layer12_attention_weight_min": -26.59153175354004,
      "activations/layer13_attention_weight_max": 38.4840202331543,
      "activations/layer13_attention_weight_min": -35.619136810302734,
      "activations/layer14_attention_weight_max": 41.46636199951172,
      "activations/layer14_attention_weight_min": -37.79924392700195,
      "activations/layer15_attention_weight_max": 39.64840316772461,
      "activations/layer15_attention_weight_min": -35.981266021728516,
      "activations/layer16_attention_weight_max": 28.577219009399414,
      "activations/layer16_attention_weight_min": -28.2138729095459,
      "activations/layer17_attention_weight_max": 29.824947357177734,
      "activations/layer17_attention_weight_min": -25.54356575012207,
      "activations/layer18_attention_weight_max": 33.74462890625,
      "activations/layer18_attention_weight_min": -27.31660270690918,
      "activations/layer19_attention_weight_max": 38.24872970581055,
      "activations/layer19_attention_weight_min": -34.20151901245117,
      "activations/layer1_attention_weight_max": 16.217451095581055,
      "activations/layer1_attention_weight_min": -15.168973922729492,
      "activations/layer20_attention_weight_max": 31.30110740661621,
      "activations/layer20_attention_weight_min": -26.3574161529541,
      "activations/layer21_attention_weight_max": 31.30850601196289,
      "activations/layer21_attention_weight_min": -24.933055877685547,
      "activations/layer22_attention_weight_max": 47.7856559753418,
      "activations/layer22_attention_weight_min": -32.35677719116211,
      "activations/layer23_attention_weight_max": 34.542694091796875,
      "activations/layer23_attention_weight_min": -26.372547149658203,
      "activations/layer2_attention_weight_max": 34.555091857910156,
      "activations/layer2_attention_weight_min": -33.031341552734375,
      "activations/layer3_attention_weight_max": 108.27906036376953,
      "activations/layer3_attention_weight_min": -109.9451675415039,
      "activations/layer4_attention_weight_max": 110.50846099853516,
      "activations/layer4_attention_weight_min": -116.76686096191406,
      "activations/layer5_attention_weight_max": 50.795753479003906,
      "activations/layer5_attention_weight_min": -61.90476608276367,
      "activations/layer6_attention_weight_max": 46.88080596923828,
      "activations/layer6_attention_weight_min": -51.195167541503906,
      "activations/layer7_attention_weight_max": 98.45201110839844,
      "activations/layer7_attention_weight_min": -103.32980346679688,
      "activations/layer8_attention_weight_max": 43.05868911743164,
      "activations/layer8_attention_weight_min": -46.52925109863281,
      "activations/layer9_attention_weight_max": 33.44184494018555,
      "activations/layer9_attention_weight_min": -37.150997161865234,
      "epoch": 21.84,
      "learning_rate": 9.258333333333332e-06,
      "loss": 2.6956,
      "step": 375850
    },
    {
      "activations/layer0_attention_weight_max": 16.824687957763672,
      "activations/layer0_attention_weight_min": -13.196523666381836,
      "activations/layer10_attention_weight_max": 35.680572509765625,
      "activations/layer10_attention_weight_min": -36.23713302612305,
      "activations/layer11_attention_weight_max": 35.02272033691406,
      "activations/layer11_attention_weight_min": -36.80531692504883,
      "activations/layer12_attention_weight_max": 27.09031105041504,
      "activations/layer12_attention_weight_min": -28.374526977539062,
      "activations/layer13_attention_weight_max": 40.391944885253906,
      "activations/layer13_attention_weight_min": -39.18862533569336,
      "activations/layer14_attention_weight_max": 45.57880783081055,
      "activations/layer14_attention_weight_min": -42.211063385009766,
      "activations/layer15_attention_weight_max": 41.769718170166016,
      "activations/layer15_attention_weight_min": -38.90928649902344,
      "activations/layer16_attention_weight_max": 30.99488067626953,
      "activations/layer16_attention_weight_min": -27.70469093322754,
      "activations/layer17_attention_weight_max": 30.66614532470703,
      "activations/layer17_attention_weight_min": -26.169240951538086,
      "activations/layer18_attention_weight_max": 34.26959991455078,
      "activations/layer18_attention_weight_min": -26.841073989868164,
      "activations/layer19_attention_weight_max": 37.626338958740234,
      "activations/layer19_attention_weight_min": -33.773521423339844,
      "activations/layer1_attention_weight_max": 16.095731735229492,
      "activations/layer1_attention_weight_min": -15.407378196716309,
      "activations/layer20_attention_weight_max": 30.74467658996582,
      "activations/layer20_attention_weight_min": -27.93181610107422,
      "activations/layer21_attention_weight_max": 28.038278579711914,
      "activations/layer21_attention_weight_min": -26.961523056030273,
      "activations/layer22_attention_weight_max": 42.80820083618164,
      "activations/layer22_attention_weight_min": -32.84153366088867,
      "activations/layer23_attention_weight_max": 32.73733901977539,
      "activations/layer23_attention_weight_min": -27.74404525756836,
      "activations/layer2_attention_weight_max": 33.618896484375,
      "activations/layer2_attention_weight_min": -32.47458267211914,
      "activations/layer3_attention_weight_max": 108.76426696777344,
      "activations/layer3_attention_weight_min": -107.0318832397461,
      "activations/layer4_attention_weight_max": 106.25482177734375,
      "activations/layer4_attention_weight_min": -112.69429016113281,
      "activations/layer5_attention_weight_max": 55.93427276611328,
      "activations/layer5_attention_weight_min": -63.95751953125,
      "activations/layer6_attention_weight_max": 46.916629791259766,
      "activations/layer6_attention_weight_min": -48.151607513427734,
      "activations/layer7_attention_weight_max": 102.22169494628906,
      "activations/layer7_attention_weight_min": -103.81085205078125,
      "activations/layer8_attention_weight_max": 40.81558609008789,
      "activations/layer8_attention_weight_min": -45.12264633178711,
      "activations/layer9_attention_weight_max": 31.990325927734375,
      "activations/layer9_attention_weight_min": -35.005558013916016,
      "epoch": 21.84,
      "learning_rate": 9.239393939393937e-06,
      "loss": 2.7284,
      "step": 375900
    },
    {
      "activations/layer0_attention_weight_max": 15.854894638061523,
      "activations/layer0_attention_weight_min": -14.38811206817627,
      "activations/layer10_attention_weight_max": 37.32520294189453,
      "activations/layer10_attention_weight_min": -36.77569580078125,
      "activations/layer11_attention_weight_max": 36.97908020019531,
      "activations/layer11_attention_weight_min": -38.01924514770508,
      "activations/layer12_attention_weight_max": 28.03838348388672,
      "activations/layer12_attention_weight_min": -29.626583099365234,
      "activations/layer13_attention_weight_max": 43.182621002197266,
      "activations/layer13_attention_weight_min": -41.096092224121094,
      "activations/layer14_attention_weight_max": 45.10132598876953,
      "activations/layer14_attention_weight_min": -43.21297836303711,
      "activations/layer15_attention_weight_max": 50.458648681640625,
      "activations/layer15_attention_weight_min": -41.233802795410156,
      "activations/layer16_attention_weight_max": 31.04111671447754,
      "activations/layer16_attention_weight_min": -27.897062301635742,
      "activations/layer17_attention_weight_max": 30.481658935546875,
      "activations/layer17_attention_weight_min": -25.476795196533203,
      "activations/layer18_attention_weight_max": 30.359365463256836,
      "activations/layer18_attention_weight_min": -25.008865356445312,
      "activations/layer19_attention_weight_max": 36.54927444458008,
      "activations/layer19_attention_weight_min": -30.163522720336914,
      "activations/layer1_attention_weight_max": 16.138774871826172,
      "activations/layer1_attention_weight_min": -15.117521286010742,
      "activations/layer20_attention_weight_max": 28.551231384277344,
      "activations/layer20_attention_weight_min": -24.227170944213867,
      "activations/layer21_attention_weight_max": 27.777462005615234,
      "activations/layer21_attention_weight_min": -26.59457015991211,
      "activations/layer22_attention_weight_max": 44.239131927490234,
      "activations/layer22_attention_weight_min": -31.310470581054688,
      "activations/layer23_attention_weight_max": 30.15675163269043,
      "activations/layer23_attention_weight_min": -25.709739685058594,
      "activations/layer2_attention_weight_max": 33.564903259277344,
      "activations/layer2_attention_weight_min": -33.39033508300781,
      "activations/layer3_attention_weight_max": 104.80564880371094,
      "activations/layer3_attention_weight_min": -109.57215118408203,
      "activations/layer4_attention_weight_max": 109.24773406982422,
      "activations/layer4_attention_weight_min": -114.01899719238281,
      "activations/layer5_attention_weight_max": 51.27783203125,
      "activations/layer5_attention_weight_min": -60.8830680847168,
      "activations/layer6_attention_weight_max": 46.27381896972656,
      "activations/layer6_attention_weight_min": -50.47744369506836,
      "activations/layer7_attention_weight_max": 99.61646270751953,
      "activations/layer7_attention_weight_min": -108.54304504394531,
      "activations/layer8_attention_weight_max": 44.99403381347656,
      "activations/layer8_attention_weight_min": -50.36966323852539,
      "activations/layer9_attention_weight_max": 34.13813781738281,
      "activations/layer9_attention_weight_min": -37.16674041748047,
      "epoch": 21.84,
      "learning_rate": 9.220454545454546e-06,
      "loss": 2.6987,
      "step": 375950
    },
    {
      "activations/layer0_attention_weight_max": 16.285329818725586,
      "activations/layer0_attention_weight_min": -14.250252723693848,
      "activations/layer10_attention_weight_max": 38.10601806640625,
      "activations/layer10_attention_weight_min": -39.24176788330078,
      "activations/layer11_attention_weight_max": 38.785682678222656,
      "activations/layer11_attention_weight_min": -37.980655670166016,
      "activations/layer12_attention_weight_max": 28.006393432617188,
      "activations/layer12_attention_weight_min": -28.3737735748291,
      "activations/layer13_attention_weight_max": 38.96770477294922,
      "activations/layer13_attention_weight_min": -40.97850036621094,
      "activations/layer14_attention_weight_max": 41.80314254760742,
      "activations/layer14_attention_weight_min": -38.38182067871094,
      "activations/layer15_attention_weight_max": 40.24721908569336,
      "activations/layer15_attention_weight_min": -38.630306243896484,
      "activations/layer16_attention_weight_max": 30.48090362548828,
      "activations/layer16_attention_weight_min": -30.01021385192871,
      "activations/layer17_attention_weight_max": 31.368295669555664,
      "activations/layer17_attention_weight_min": -27.27711296081543,
      "activations/layer18_attention_weight_max": 32.407371520996094,
      "activations/layer18_attention_weight_min": -27.558683395385742,
      "activations/layer19_attention_weight_max": 32.84609603881836,
      "activations/layer19_attention_weight_min": -32.851539611816406,
      "activations/layer1_attention_weight_max": 16.575607299804688,
      "activations/layer1_attention_weight_min": -16.37794303894043,
      "activations/layer20_attention_weight_max": 28.0655460357666,
      "activations/layer20_attention_weight_min": -25.920963287353516,
      "activations/layer21_attention_weight_max": 28.276578903198242,
      "activations/layer21_attention_weight_min": -26.126432418823242,
      "activations/layer22_attention_weight_max": 43.32623291015625,
      "activations/layer22_attention_weight_min": -32.78783416748047,
      "activations/layer23_attention_weight_max": 31.88478660583496,
      "activations/layer23_attention_weight_min": -27.482511520385742,
      "activations/layer2_attention_weight_max": 35.03464889526367,
      "activations/layer2_attention_weight_min": -34.3137092590332,
      "activations/layer3_attention_weight_max": 110.55924224853516,
      "activations/layer3_attention_weight_min": -107.54681396484375,
      "activations/layer4_attention_weight_max": 116.05511474609375,
      "activations/layer4_attention_weight_min": -114.61673736572266,
      "activations/layer5_attention_weight_max": 53.576595306396484,
      "activations/layer5_attention_weight_min": -61.97695541381836,
      "activations/layer6_attention_weight_max": 48.24314880371094,
      "activations/layer6_attention_weight_min": -49.89228439331055,
      "activations/layer7_attention_weight_max": 108.19217681884766,
      "activations/layer7_attention_weight_min": -103.34733581542969,
      "activations/layer8_attention_weight_max": 48.88181686401367,
      "activations/layer8_attention_weight_min": -47.32590103149414,
      "activations/layer9_attention_weight_max": 32.976924896240234,
      "activations/layer9_attention_weight_min": -36.875144958496094,
      "epoch": 21.85,
      "learning_rate": 9.20151515151515e-06,
      "loss": 2.7127,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.5233,
      "eval_samples_per_second": 503.796,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.5233,
      "eval_openwebtext_samples_per_second": 503.796,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.967,
      "eval_wikitext_samples_per_second": 231.829,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_lambada_loss": 2.341796875,
      "eval_lambada_ppl": 10.399907116482579,
      "eval_lambada_runtime": 9.5589,
      "eval_lambada_samples_per_second": 509.367,
      "step": 376000
    },
    {
      "activations/layer0_attention_weight_max": 16.192354202270508,
      "activations/layer0_attention_weight_min": -14.663981437683105,
      "activations/layer10_attention_weight_max": 32.41252517700195,
      "activations/layer10_attention_weight_min": -34.451717376708984,
      "activations/layer11_attention_weight_max": 34.20629119873047,
      "activations/layer11_attention_weight_min": -34.97964096069336,
      "activations/layer12_attention_weight_max": 27.120798110961914,
      "activations/layer12_attention_weight_min": -26.50615119934082,
      "activations/layer13_attention_weight_max": 40.81718826293945,
      "activations/layer13_attention_weight_min": -34.72896957397461,
      "activations/layer14_attention_weight_max": 42.4444694519043,
      "activations/layer14_attention_weight_min": -37.331642150878906,
      "activations/layer15_attention_weight_max": 38.54512405395508,
      "activations/layer15_attention_weight_min": -37.51677322387695,
      "activations/layer16_attention_weight_max": 32.02286911010742,
      "activations/layer16_attention_weight_min": -33.88571548461914,
      "activations/layer17_attention_weight_max": 27.865835189819336,
      "activations/layer17_attention_weight_min": -29.77576446533203,
      "activations/layer18_attention_weight_max": 34.12394714355469,
      "activations/layer18_attention_weight_min": -25.40046501159668,
      "activations/layer19_attention_weight_max": 34.44799041748047,
      "activations/layer19_attention_weight_min": -33.47507858276367,
      "activations/layer1_attention_weight_max": 16.544673919677734,
      "activations/layer1_attention_weight_min": -16.106101989746094,
      "activations/layer20_attention_weight_max": 29.715641021728516,
      "activations/layer20_attention_weight_min": -25.297779083251953,
      "activations/layer21_attention_weight_max": 32.9000244140625,
      "activations/layer21_attention_weight_min": -24.827423095703125,
      "activations/layer22_attention_weight_max": 38.967552185058594,
      "activations/layer22_attention_weight_min": -32.629730224609375,
      "activations/layer23_attention_weight_max": 29.60821533203125,
      "activations/layer23_attention_weight_min": -28.065872192382812,
      "activations/layer2_attention_weight_max": 35.061546325683594,
      "activations/layer2_attention_weight_min": -34.5748291015625,
      "activations/layer3_attention_weight_max": 106.4029312133789,
      "activations/layer3_attention_weight_min": -105.38834381103516,
      "activations/layer4_attention_weight_max": 109.74401092529297,
      "activations/layer4_attention_weight_min": -111.21180725097656,
      "activations/layer5_attention_weight_max": 49.863582611083984,
      "activations/layer5_attention_weight_min": -64.72468566894531,
      "activations/layer6_attention_weight_max": 46.59236145019531,
      "activations/layer6_attention_weight_min": -48.015933990478516,
      "activations/layer7_attention_weight_max": 96.7142105102539,
      "activations/layer7_attention_weight_min": -102.48753356933594,
      "activations/layer8_attention_weight_max": 40.82579040527344,
      "activations/layer8_attention_weight_min": -46.26448059082031,
      "activations/layer9_attention_weight_max": 30.451709747314453,
      "activations/layer9_attention_weight_min": -34.782283782958984,
      "epoch": 21.85,
      "learning_rate": 9.182954545454545e-06,
      "loss": 2.7052,
      "step": 376050
    },
    {
      "activations/layer0_attention_weight_max": 16.417396545410156,
      "activations/layer0_attention_weight_min": -13.494710922241211,
      "activations/layer10_attention_weight_max": 40.94858932495117,
      "activations/layer10_attention_weight_min": -37.36042022705078,
      "activations/layer11_attention_weight_max": 39.882625579833984,
      "activations/layer11_attention_weight_min": -37.04768371582031,
      "activations/layer12_attention_weight_max": 32.56660079956055,
      "activations/layer12_attention_weight_min": -29.51372718811035,
      "activations/layer13_attention_weight_max": 45.1080207824707,
      "activations/layer13_attention_weight_min": -37.865386962890625,
      "activations/layer14_attention_weight_max": 49.555511474609375,
      "activations/layer14_attention_weight_min": -43.13006591796875,
      "activations/layer15_attention_weight_max": 53.62088394165039,
      "activations/layer15_attention_weight_min": -41.29380416870117,
      "activations/layer16_attention_weight_max": 37.00801467895508,
      "activations/layer16_attention_weight_min": -29.23632049560547,
      "activations/layer17_attention_weight_max": 33.728240966796875,
      "activations/layer17_attention_weight_min": -26.776399612426758,
      "activations/layer18_attention_weight_max": 33.64137268066406,
      "activations/layer18_attention_weight_min": -26.639516830444336,
      "activations/layer19_attention_weight_max": 33.875396728515625,
      "activations/layer19_attention_weight_min": -32.10503005981445,
      "activations/layer1_attention_weight_max": 16.496501922607422,
      "activations/layer1_attention_weight_min": -15.166400909423828,
      "activations/layer20_attention_weight_max": 30.016355514526367,
      "activations/layer20_attention_weight_min": -25.647964477539062,
      "activations/layer21_attention_weight_max": 28.851943969726562,
      "activations/layer21_attention_weight_min": -24.487756729125977,
      "activations/layer22_attention_weight_max": 41.394073486328125,
      "activations/layer22_attention_weight_min": -31.918643951416016,
      "activations/layer23_attention_weight_max": 32.02627944946289,
      "activations/layer23_attention_weight_min": -25.027996063232422,
      "activations/layer2_attention_weight_max": 34.62892150878906,
      "activations/layer2_attention_weight_min": -33.2501220703125,
      "activations/layer3_attention_weight_max": 108.75525665283203,
      "activations/layer3_attention_weight_min": -106.34603881835938,
      "activations/layer4_attention_weight_max": 116.00288391113281,
      "activations/layer4_attention_weight_min": -111.07308197021484,
      "activations/layer5_attention_weight_max": 50.03898620605469,
      "activations/layer5_attention_weight_min": -62.38745880126953,
      "activations/layer6_attention_weight_max": 46.35860824584961,
      "activations/layer6_attention_weight_min": -48.15281295776367,
      "activations/layer7_attention_weight_max": 101.4818344116211,
      "activations/layer7_attention_weight_min": -105.59870147705078,
      "activations/layer8_attention_weight_max": 44.15245819091797,
      "activations/layer8_attention_weight_min": -47.1830940246582,
      "activations/layer9_attention_weight_max": 35.448699951171875,
      "activations/layer9_attention_weight_min": -37.86457061767578,
      "epoch": 21.85,
      "learning_rate": 9.164015151515151e-06,
      "loss": 2.7197,
      "step": 376100
    },
    {
      "activations/layer0_attention_weight_max": 15.528822898864746,
      "activations/layer0_attention_weight_min": -14.40632438659668,
      "activations/layer10_attention_weight_max": 45.93360137939453,
      "activations/layer10_attention_weight_min": -44.953060150146484,
      "activations/layer11_attention_weight_max": 42.82441329956055,
      "activations/layer11_attention_weight_min": -43.53202819824219,
      "activations/layer12_attention_weight_max": 29.477907180786133,
      "activations/layer12_attention_weight_min": -29.6554012298584,
      "activations/layer13_attention_weight_max": 42.996280670166016,
      "activations/layer13_attention_weight_min": -38.96171569824219,
      "activations/layer14_attention_weight_max": 51.461605072021484,
      "activations/layer14_attention_weight_min": -42.80582046508789,
      "activations/layer15_attention_weight_max": 42.67170333862305,
      "activations/layer15_attention_weight_min": -42.620399475097656,
      "activations/layer16_attention_weight_max": 33.5064582824707,
      "activations/layer16_attention_weight_min": -29.11968994140625,
      "activations/layer17_attention_weight_max": 34.69863510131836,
      "activations/layer17_attention_weight_min": -28.36411476135254,
      "activations/layer18_attention_weight_max": 35.211090087890625,
      "activations/layer18_attention_weight_min": -29.92957878112793,
      "activations/layer19_attention_weight_max": 38.44451904296875,
      "activations/layer19_attention_weight_min": -33.55397033691406,
      "activations/layer1_attention_weight_max": 16.522705078125,
      "activations/layer1_attention_weight_min": -16.447118759155273,
      "activations/layer20_attention_weight_max": 33.62387466430664,
      "activations/layer20_attention_weight_min": -25.95067596435547,
      "activations/layer21_attention_weight_max": 30.96222496032715,
      "activations/layer21_attention_weight_min": -25.01641082763672,
      "activations/layer22_attention_weight_max": 47.698387145996094,
      "activations/layer22_attention_weight_min": -32.49277877807617,
      "activations/layer23_attention_weight_max": 31.977954864501953,
      "activations/layer23_attention_weight_min": -26.1749324798584,
      "activations/layer2_attention_weight_max": 40.518348693847656,
      "activations/layer2_attention_weight_min": -34.736122131347656,
      "activations/layer3_attention_weight_max": 113.12567901611328,
      "activations/layer3_attention_weight_min": -110.1395034790039,
      "activations/layer4_attention_weight_max": 118.6257095336914,
      "activations/layer4_attention_weight_min": -113.5766830444336,
      "activations/layer5_attention_weight_max": 53.02785873413086,
      "activations/layer5_attention_weight_min": -64.92225646972656,
      "activations/layer6_attention_weight_max": 48.944786071777344,
      "activations/layer6_attention_weight_min": -49.4567985534668,
      "activations/layer7_attention_weight_max": 111.22500610351562,
      "activations/layer7_attention_weight_min": -113.212158203125,
      "activations/layer8_attention_weight_max": 52.23029708862305,
      "activations/layer8_attention_weight_min": -54.07957458496094,
      "activations/layer9_attention_weight_max": 39.169769287109375,
      "activations/layer9_attention_weight_min": -41.88457107543945,
      "epoch": 21.86,
      "learning_rate": 9.145075757575756e-06,
      "loss": 2.7176,
      "step": 376150
    },
    {
      "activations/layer0_attention_weight_max": 16.919574737548828,
      "activations/layer0_attention_weight_min": -13.804431915283203,
      "activations/layer10_attention_weight_max": 35.609046936035156,
      "activations/layer10_attention_weight_min": -36.95948791503906,
      "activations/layer11_attention_weight_max": 34.69060516357422,
      "activations/layer11_attention_weight_min": -36.2164421081543,
      "activations/layer12_attention_weight_max": 28.1645565032959,
      "activations/layer12_attention_weight_min": -27.169986724853516,
      "activations/layer13_attention_weight_max": 39.442359924316406,
      "activations/layer13_attention_weight_min": -35.71810531616211,
      "activations/layer14_attention_weight_max": 43.741127014160156,
      "activations/layer14_attention_weight_min": -40.195255279541016,
      "activations/layer15_attention_weight_max": 38.63422393798828,
      "activations/layer15_attention_weight_min": -36.65510940551758,
      "activations/layer16_attention_weight_max": 28.982194900512695,
      "activations/layer16_attention_weight_min": -27.322416305541992,
      "activations/layer17_attention_weight_max": 28.869850158691406,
      "activations/layer17_attention_weight_min": -28.432437896728516,
      "activations/layer18_attention_weight_max": 30.501625061035156,
      "activations/layer18_attention_weight_min": -25.63783073425293,
      "activations/layer19_attention_weight_max": 32.234779357910156,
      "activations/layer19_attention_weight_min": -32.359230041503906,
      "activations/layer1_attention_weight_max": 15.767426490783691,
      "activations/layer1_attention_weight_min": -15.417564392089844,
      "activations/layer20_attention_weight_max": 27.036333084106445,
      "activations/layer20_attention_weight_min": -24.443737030029297,
      "activations/layer21_attention_weight_max": 28.157798767089844,
      "activations/layer21_attention_weight_min": -25.246475219726562,
      "activations/layer22_attention_weight_max": 43.01548385620117,
      "activations/layer22_attention_weight_min": -31.310409545898438,
      "activations/layer23_attention_weight_max": 34.06064987182617,
      "activations/layer23_attention_weight_min": -25.327320098876953,
      "activations/layer2_attention_weight_max": 34.724021911621094,
      "activations/layer2_attention_weight_min": -34.55527114868164,
      "activations/layer3_attention_weight_max": 114.12360382080078,
      "activations/layer3_attention_weight_min": -112.3349380493164,
      "activations/layer4_attention_weight_max": 114.8266372680664,
      "activations/layer4_attention_weight_min": -113.42010498046875,
      "activations/layer5_attention_weight_max": 52.24308395385742,
      "activations/layer5_attention_weight_min": -63.46990966796875,
      "activations/layer6_attention_weight_max": 48.603736877441406,
      "activations/layer6_attention_weight_min": -53.031742095947266,
      "activations/layer7_attention_weight_max": 99.91593170166016,
      "activations/layer7_attention_weight_min": -110.72450256347656,
      "activations/layer8_attention_weight_max": 43.99939727783203,
      "activations/layer8_attention_weight_min": -47.08076858520508,
      "activations/layer9_attention_weight_max": 34.06674575805664,
      "activations/layer9_attention_weight_min": -35.95497131347656,
      "epoch": 21.86,
      "learning_rate": 9.126136363636363e-06,
      "loss": 2.715,
      "step": 376200
    },
    {
      "activations/layer0_attention_weight_max": 16.501113891601562,
      "activations/layer0_attention_weight_min": -13.787649154663086,
      "activations/layer10_attention_weight_max": 39.1342658996582,
      "activations/layer10_attention_weight_min": -40.44154357910156,
      "activations/layer11_attention_weight_max": 38.404544830322266,
      "activations/layer11_attention_weight_min": -40.564666748046875,
      "activations/layer12_attention_weight_max": 28.43642807006836,
      "activations/layer12_attention_weight_min": -30.106306076049805,
      "activations/layer13_attention_weight_max": 41.030975341796875,
      "activations/layer13_attention_weight_min": -39.27470016479492,
      "activations/layer14_attention_weight_max": 45.95197677612305,
      "activations/layer14_attention_weight_min": -41.78935623168945,
      "activations/layer15_attention_weight_max": 44.801822662353516,
      "activations/layer15_attention_weight_min": -44.231101989746094,
      "activations/layer16_attention_weight_max": 32.45927429199219,
      "activations/layer16_attention_weight_min": -29.82837677001953,
      "activations/layer17_attention_weight_max": 29.577205657958984,
      "activations/layer17_attention_weight_min": -30.36995506286621,
      "activations/layer18_attention_weight_max": 32.43003463745117,
      "activations/layer18_attention_weight_min": -29.374910354614258,
      "activations/layer19_attention_weight_max": 39.443443298339844,
      "activations/layer19_attention_weight_min": -33.910118103027344,
      "activations/layer1_attention_weight_max": 16.23213005065918,
      "activations/layer1_attention_weight_min": -15.38741397857666,
      "activations/layer20_attention_weight_max": 27.55498695373535,
      "activations/layer20_attention_weight_min": -24.447160720825195,
      "activations/layer21_attention_weight_max": 27.365936279296875,
      "activations/layer21_attention_weight_min": -23.53063201904297,
      "activations/layer22_attention_weight_max": 41.29955291748047,
      "activations/layer22_attention_weight_min": -31.78901481628418,
      "activations/layer23_attention_weight_max": 32.264312744140625,
      "activations/layer23_attention_weight_min": -25.49219512939453,
      "activations/layer2_attention_weight_max": 34.04363250732422,
      "activations/layer2_attention_weight_min": -34.37584686279297,
      "activations/layer3_attention_weight_max": 105.11339569091797,
      "activations/layer3_attention_weight_min": -106.9157485961914,
      "activations/layer4_attention_weight_max": 105.50617980957031,
      "activations/layer4_attention_weight_min": -112.54389953613281,
      "activations/layer5_attention_weight_max": 52.18757247924805,
      "activations/layer5_attention_weight_min": -60.266353607177734,
      "activations/layer6_attention_weight_max": 46.2450065612793,
      "activations/layer6_attention_weight_min": -47.37244415283203,
      "activations/layer7_attention_weight_max": 100.65278625488281,
      "activations/layer7_attention_weight_min": -107.62130737304688,
      "activations/layer8_attention_weight_max": 45.8775634765625,
      "activations/layer8_attention_weight_min": -49.790626525878906,
      "activations/layer9_attention_weight_max": 35.904396057128906,
      "activations/layer9_attention_weight_min": -40.307395935058594,
      "epoch": 21.86,
      "learning_rate": 9.10719696969697e-06,
      "loss": 2.7002,
      "step": 376250
    },
    {
      "activations/layer0_attention_weight_max": 16.467504501342773,
      "activations/layer0_attention_weight_min": -13.45503044128418,
      "activations/layer10_attention_weight_max": 37.343727111816406,
      "activations/layer10_attention_weight_min": -36.609195709228516,
      "activations/layer11_attention_weight_max": 37.03971481323242,
      "activations/layer11_attention_weight_min": -37.01996994018555,
      "activations/layer12_attention_weight_max": 27.75710678100586,
      "activations/layer12_attention_weight_min": -27.31554412841797,
      "activations/layer13_attention_weight_max": 41.13337326049805,
      "activations/layer13_attention_weight_min": -38.587249755859375,
      "activations/layer14_attention_weight_max": 44.6003532409668,
      "activations/layer14_attention_weight_min": -41.89453887939453,
      "activations/layer15_attention_weight_max": 40.33745574951172,
      "activations/layer15_attention_weight_min": -38.506065368652344,
      "activations/layer16_attention_weight_max": 33.41477966308594,
      "activations/layer16_attention_weight_min": -27.801944732666016,
      "activations/layer17_attention_weight_max": 30.129236221313477,
      "activations/layer17_attention_weight_min": -25.758718490600586,
      "activations/layer18_attention_weight_max": 33.88580322265625,
      "activations/layer18_attention_weight_min": -25.988603591918945,
      "activations/layer19_attention_weight_max": 36.28219223022461,
      "activations/layer19_attention_weight_min": -31.577238082885742,
      "activations/layer1_attention_weight_max": 15.347785949707031,
      "activations/layer1_attention_weight_min": -14.919693946838379,
      "activations/layer20_attention_weight_max": 30.875015258789062,
      "activations/layer20_attention_weight_min": -23.417526245117188,
      "activations/layer21_attention_weight_max": 29.005796432495117,
      "activations/layer21_attention_weight_min": -24.177648544311523,
      "activations/layer22_attention_weight_max": 42.837459564208984,
      "activations/layer22_attention_weight_min": -32.89466094970703,
      "activations/layer23_attention_weight_max": 30.85504722595215,
      "activations/layer23_attention_weight_min": -25.57512664794922,
      "activations/layer2_attention_weight_max": 35.35902404785156,
      "activations/layer2_attention_weight_min": -33.69400405883789,
      "activations/layer3_attention_weight_max": 107.06495666503906,
      "activations/layer3_attention_weight_min": -103.91053771972656,
      "activations/layer4_attention_weight_max": 109.37928771972656,
      "activations/layer4_attention_weight_min": -112.46980285644531,
      "activations/layer5_attention_weight_max": 50.66619110107422,
      "activations/layer5_attention_weight_min": -59.13680648803711,
      "activations/layer6_attention_weight_max": 48.748409271240234,
      "activations/layer6_attention_weight_min": -52.054630279541016,
      "activations/layer7_attention_weight_max": 103.53729248046875,
      "activations/layer7_attention_weight_min": -106.40202331542969,
      "activations/layer8_attention_weight_max": 46.27080154418945,
      "activations/layer8_attention_weight_min": -46.26128387451172,
      "activations/layer9_attention_weight_max": 35.694984436035156,
      "activations/layer9_attention_weight_min": -36.37946701049805,
      "epoch": 21.87,
      "learning_rate": 9.088257575757574e-06,
      "loss": 2.693,
      "step": 376300
    },
    {
      "activations/layer0_attention_weight_max": 16.28045654296875,
      "activations/layer0_attention_weight_min": -14.128296852111816,
      "activations/layer10_attention_weight_max": 33.8454475402832,
      "activations/layer10_attention_weight_min": -35.76036834716797,
      "activations/layer11_attention_weight_max": 34.04190444946289,
      "activations/layer11_attention_weight_min": -34.532859802246094,
      "activations/layer12_attention_weight_max": 28.338579177856445,
      "activations/layer12_attention_weight_min": -26.674837112426758,
      "activations/layer13_attention_weight_max": 40.211326599121094,
      "activations/layer13_attention_weight_min": -36.624267578125,
      "activations/layer14_attention_weight_max": 43.97027587890625,
      "activations/layer14_attention_weight_min": -36.91184616088867,
      "activations/layer15_attention_weight_max": 41.03511047363281,
      "activations/layer15_attention_weight_min": -36.85866928100586,
      "activations/layer16_attention_weight_max": 28.67508888244629,
      "activations/layer16_attention_weight_min": -27.173385620117188,
      "activations/layer17_attention_weight_max": 29.015235900878906,
      "activations/layer17_attention_weight_min": -27.105636596679688,
      "activations/layer18_attention_weight_max": 33.95331954956055,
      "activations/layer18_attention_weight_min": -25.686084747314453,
      "activations/layer19_attention_weight_max": 34.232421875,
      "activations/layer19_attention_weight_min": -33.726871490478516,
      "activations/layer1_attention_weight_max": 15.737218856811523,
      "activations/layer1_attention_weight_min": -15.671297073364258,
      "activations/layer20_attention_weight_max": 27.371341705322266,
      "activations/layer20_attention_weight_min": -24.97414207458496,
      "activations/layer21_attention_weight_max": 34.38533401489258,
      "activations/layer21_attention_weight_min": -24.341501235961914,
      "activations/layer22_attention_weight_max": 39.81309127807617,
      "activations/layer22_attention_weight_min": -32.55329513549805,
      "activations/layer23_attention_weight_max": 31.60327911376953,
      "activations/layer23_attention_weight_min": -25.831153869628906,
      "activations/layer2_attention_weight_max": 33.54533767700195,
      "activations/layer2_attention_weight_min": -32.53883361816406,
      "activations/layer3_attention_weight_max": 103.15640258789062,
      "activations/layer3_attention_weight_min": -106.03736114501953,
      "activations/layer4_attention_weight_max": 111.8065414428711,
      "activations/layer4_attention_weight_min": -109.66712951660156,
      "activations/layer5_attention_weight_max": 48.40576934814453,
      "activations/layer5_attention_weight_min": -61.08999252319336,
      "activations/layer6_attention_weight_max": 44.962867736816406,
      "activations/layer6_attention_weight_min": -48.8096809387207,
      "activations/layer7_attention_weight_max": 93.41077423095703,
      "activations/layer7_attention_weight_min": -102.12818145751953,
      "activations/layer8_attention_weight_max": 43.032386779785156,
      "activations/layer8_attention_weight_min": -50.88016891479492,
      "activations/layer9_attention_weight_max": 32.72153854370117,
      "activations/layer9_attention_weight_min": -35.215274810791016,
      "epoch": 21.87,
      "learning_rate": 9.06931818181818e-06,
      "loss": 2.7231,
      "step": 376350
    },
    {
      "activations/layer0_attention_weight_max": 16.60838508605957,
      "activations/layer0_attention_weight_min": -14.043346405029297,
      "activations/layer10_attention_weight_max": 35.66965866088867,
      "activations/layer10_attention_weight_min": -35.07275390625,
      "activations/layer11_attention_weight_max": 36.19415283203125,
      "activations/layer11_attention_weight_min": -36.087013244628906,
      "activations/layer12_attention_weight_max": 25.905261993408203,
      "activations/layer12_attention_weight_min": -26.81284523010254,
      "activations/layer13_attention_weight_max": 38.66266632080078,
      "activations/layer13_attention_weight_min": -35.79212188720703,
      "activations/layer14_attention_weight_max": 41.45268630981445,
      "activations/layer14_attention_weight_min": -38.80183792114258,
      "activations/layer15_attention_weight_max": 36.78274917602539,
      "activations/layer15_attention_weight_min": -35.394309997558594,
      "activations/layer16_attention_weight_max": 28.893144607543945,
      "activations/layer16_attention_weight_min": -26.97551155090332,
      "activations/layer17_attention_weight_max": 28.455535888671875,
      "activations/layer17_attention_weight_min": -25.900732040405273,
      "activations/layer18_attention_weight_max": 34.98600387573242,
      "activations/layer18_attention_weight_min": -26.639020919799805,
      "activations/layer19_attention_weight_max": 33.87569808959961,
      "activations/layer19_attention_weight_min": -31.45050048828125,
      "activations/layer1_attention_weight_max": 15.980206489562988,
      "activations/layer1_attention_weight_min": -14.210208892822266,
      "activations/layer20_attention_weight_max": 28.274248123168945,
      "activations/layer20_attention_weight_min": -28.804059982299805,
      "activations/layer21_attention_weight_max": 28.143754959106445,
      "activations/layer21_attention_weight_min": -27.706785202026367,
      "activations/layer22_attention_weight_max": 41.26300048828125,
      "activations/layer22_attention_weight_min": -31.882253646850586,
      "activations/layer23_attention_weight_max": 34.78840637207031,
      "activations/layer23_attention_weight_min": -25.45309829711914,
      "activations/layer2_attention_weight_max": 33.833900451660156,
      "activations/layer2_attention_weight_min": -32.84088897705078,
      "activations/layer3_attention_weight_max": 107.04750061035156,
      "activations/layer3_attention_weight_min": -106.8584213256836,
      "activations/layer4_attention_weight_max": 110.36832427978516,
      "activations/layer4_attention_weight_min": -111.57771301269531,
      "activations/layer5_attention_weight_max": 49.83361053466797,
      "activations/layer5_attention_weight_min": -63.74253463745117,
      "activations/layer6_attention_weight_max": 45.46894454956055,
      "activations/layer6_attention_weight_min": -49.689090728759766,
      "activations/layer7_attention_weight_max": 98.66473388671875,
      "activations/layer7_attention_weight_min": -102.82303619384766,
      "activations/layer8_attention_weight_max": 43.453269958496094,
      "activations/layer8_attention_weight_min": -43.52537155151367,
      "activations/layer9_attention_weight_max": 33.73944854736328,
      "activations/layer9_attention_weight_min": -35.13881301879883,
      "epoch": 21.87,
      "learning_rate": 9.050757575757575e-06,
      "loss": 2.6994,
      "step": 376400
    },
    {
      "activations/layer0_attention_weight_max": 16.648529052734375,
      "activations/layer0_attention_weight_min": -14.309708595275879,
      "activations/layer10_attention_weight_max": 37.254310607910156,
      "activations/layer10_attention_weight_min": -37.12641143798828,
      "activations/layer11_attention_weight_max": 36.11258316040039,
      "activations/layer11_attention_weight_min": -37.86407470703125,
      "activations/layer12_attention_weight_max": 27.705541610717773,
      "activations/layer12_attention_weight_min": -27.550392150878906,
      "activations/layer13_attention_weight_max": 39.88043975830078,
      "activations/layer13_attention_weight_min": -37.20476150512695,
      "activations/layer14_attention_weight_max": 43.05219268798828,
      "activations/layer14_attention_weight_min": -40.2027473449707,
      "activations/layer15_attention_weight_max": 41.40078353881836,
      "activations/layer15_attention_weight_min": -38.99502944946289,
      "activations/layer16_attention_weight_max": 30.259353637695312,
      "activations/layer16_attention_weight_min": -28.86224365234375,
      "activations/layer17_attention_weight_max": 32.62208938598633,
      "activations/layer17_attention_weight_min": -28.15877914428711,
      "activations/layer18_attention_weight_max": 34.6886100769043,
      "activations/layer18_attention_weight_min": -29.7177791595459,
      "activations/layer19_attention_weight_max": 37.749324798583984,
      "activations/layer19_attention_weight_min": -32.545379638671875,
      "activations/layer1_attention_weight_max": 15.980575561523438,
      "activations/layer1_attention_weight_min": -15.670221328735352,
      "activations/layer20_attention_weight_max": 29.415803909301758,
      "activations/layer20_attention_weight_min": -24.92345428466797,
      "activations/layer21_attention_weight_max": 27.89481544494629,
      "activations/layer21_attention_weight_min": -24.808380126953125,
      "activations/layer22_attention_weight_max": 43.136558532714844,
      "activations/layer22_attention_weight_min": -32.078433990478516,
      "activations/layer23_attention_weight_max": 32.34386444091797,
      "activations/layer23_attention_weight_min": -24.832382202148438,
      "activations/layer2_attention_weight_max": 31.86713981628418,
      "activations/layer2_attention_weight_min": -32.57289505004883,
      "activations/layer3_attention_weight_max": 103.62939453125,
      "activations/layer3_attention_weight_min": -107.64002227783203,
      "activations/layer4_attention_weight_max": 108.8017349243164,
      "activations/layer4_attention_weight_min": -110.24415588378906,
      "activations/layer5_attention_weight_max": 50.151485443115234,
      "activations/layer5_attention_weight_min": -62.93034362792969,
      "activations/layer6_attention_weight_max": 45.86227798461914,
      "activations/layer6_attention_weight_min": -48.2548713684082,
      "activations/layer7_attention_weight_max": 95.1554946899414,
      "activations/layer7_attention_weight_min": -102.73829650878906,
      "activations/layer8_attention_weight_max": 42.66643142700195,
      "activations/layer8_attention_weight_min": -47.80141830444336,
      "activations/layer9_attention_weight_max": 34.107391357421875,
      "activations/layer9_attention_weight_min": -35.630191802978516,
      "epoch": 21.87,
      "learning_rate": 9.03181818181818e-06,
      "loss": 2.7109,
      "step": 376450
    },
    {
      "activations/layer0_attention_weight_max": 16.753389358520508,
      "activations/layer0_attention_weight_min": -14.457202911376953,
      "activations/layer10_attention_weight_max": 38.86479568481445,
      "activations/layer10_attention_weight_min": -38.01730728149414,
      "activations/layer11_attention_weight_max": 37.608856201171875,
      "activations/layer11_attention_weight_min": -37.76689147949219,
      "activations/layer12_attention_weight_max": 26.984037399291992,
      "activations/layer12_attention_weight_min": -27.07840347290039,
      "activations/layer13_attention_weight_max": 38.188072204589844,
      "activations/layer13_attention_weight_min": -37.55205154418945,
      "activations/layer14_attention_weight_max": 42.94886016845703,
      "activations/layer14_attention_weight_min": -37.642311096191406,
      "activations/layer15_attention_weight_max": 37.579017639160156,
      "activations/layer15_attention_weight_min": -36.3510856628418,
      "activations/layer16_attention_weight_max": 29.823646545410156,
      "activations/layer16_attention_weight_min": -28.515228271484375,
      "activations/layer17_attention_weight_max": 30.57040023803711,
      "activations/layer17_attention_weight_min": -25.512405395507812,
      "activations/layer18_attention_weight_max": 31.687740325927734,
      "activations/layer18_attention_weight_min": -26.720722198486328,
      "activations/layer19_attention_weight_max": 34.99446487426758,
      "activations/layer19_attention_weight_min": -33.03171920776367,
      "activations/layer1_attention_weight_max": 15.318918228149414,
      "activations/layer1_attention_weight_min": -14.442344665527344,
      "activations/layer20_attention_weight_max": 26.987957000732422,
      "activations/layer20_attention_weight_min": -25.071943283081055,
      "activations/layer21_attention_weight_max": 28.375534057617188,
      "activations/layer21_attention_weight_min": -25.569578170776367,
      "activations/layer22_attention_weight_max": 40.861061096191406,
      "activations/layer22_attention_weight_min": -34.00139617919922,
      "activations/layer23_attention_weight_max": 31.916614532470703,
      "activations/layer23_attention_weight_min": -26.237255096435547,
      "activations/layer2_attention_weight_max": 33.73308563232422,
      "activations/layer2_attention_weight_min": -31.41565704345703,
      "activations/layer3_attention_weight_max": 106.14115905761719,
      "activations/layer3_attention_weight_min": -104.64820098876953,
      "activations/layer4_attention_weight_max": 110.15984344482422,
      "activations/layer4_attention_weight_min": -109.063232421875,
      "activations/layer5_attention_weight_max": 49.81498336791992,
      "activations/layer5_attention_weight_min": -62.12199020385742,
      "activations/layer6_attention_weight_max": 46.69440841674805,
      "activations/layer6_attention_weight_min": -49.362606048583984,
      "activations/layer7_attention_weight_max": 98.9361801147461,
      "activations/layer7_attention_weight_min": -103.58245086669922,
      "activations/layer8_attention_weight_max": 47.31473159790039,
      "activations/layer8_attention_weight_min": -47.63872146606445,
      "activations/layer9_attention_weight_max": 36.916988372802734,
      "activations/layer9_attention_weight_min": -36.67087173461914,
      "epoch": 21.88,
      "learning_rate": 9.012878787878786e-06,
      "loss": 2.6975,
      "step": 376500
    },
    {
      "activations/layer0_attention_weight_max": 16.65789031982422,
      "activations/layer0_attention_weight_min": -14.90949821472168,
      "activations/layer10_attention_weight_max": 37.99420928955078,
      "activations/layer10_attention_weight_min": -36.704498291015625,
      "activations/layer11_attention_weight_max": 36.673179626464844,
      "activations/layer11_attention_weight_min": -36.48502731323242,
      "activations/layer12_attention_weight_max": 28.363758087158203,
      "activations/layer12_attention_weight_min": -26.803138732910156,
      "activations/layer13_attention_weight_max": 41.45774459838867,
      "activations/layer13_attention_weight_min": -37.244529724121094,
      "activations/layer14_attention_weight_max": 46.21780776977539,
      "activations/layer14_attention_weight_min": -41.15096664428711,
      "activations/layer15_attention_weight_max": 43.057533264160156,
      "activations/layer15_attention_weight_min": -38.16681671142578,
      "activations/layer16_attention_weight_max": 34.886619567871094,
      "activations/layer16_attention_weight_min": -29.21523094177246,
      "activations/layer17_attention_weight_max": 33.46515655517578,
      "activations/layer17_attention_weight_min": -26.693586349487305,
      "activations/layer18_attention_weight_max": 37.63267135620117,
      "activations/layer18_attention_weight_min": -26.909950256347656,
      "activations/layer19_attention_weight_max": 38.58168411254883,
      "activations/layer19_attention_weight_min": -32.15951919555664,
      "activations/layer1_attention_weight_max": 16.304386138916016,
      "activations/layer1_attention_weight_min": -16.135766983032227,
      "activations/layer20_attention_weight_max": 33.23030471801758,
      "activations/layer20_attention_weight_min": -25.443025588989258,
      "activations/layer21_attention_weight_max": 31.91006088256836,
      "activations/layer21_attention_weight_min": -24.978227615356445,
      "activations/layer22_attention_weight_max": 43.77029800415039,
      "activations/layer22_attention_weight_min": -30.348867416381836,
      "activations/layer23_attention_weight_max": 33.732547760009766,
      "activations/layer23_attention_weight_min": -26.689258575439453,
      "activations/layer2_attention_weight_max": 34.343406677246094,
      "activations/layer2_attention_weight_min": -33.533878326416016,
      "activations/layer3_attention_weight_max": 109.56002044677734,
      "activations/layer3_attention_weight_min": -110.0995101928711,
      "activations/layer4_attention_weight_max": 113.12664031982422,
      "activations/layer4_attention_weight_min": -113.00941467285156,
      "activations/layer5_attention_weight_max": 53.01708984375,
      "activations/layer5_attention_weight_min": -64.51362609863281,
      "activations/layer6_attention_weight_max": 46.1935920715332,
      "activations/layer6_attention_weight_min": -49.33662796020508,
      "activations/layer7_attention_weight_max": 97.87802124023438,
      "activations/layer7_attention_weight_min": -100.04671478271484,
      "activations/layer8_attention_weight_max": 44.01716995239258,
      "activations/layer8_attention_weight_min": -46.62053298950195,
      "activations/layer9_attention_weight_max": 33.127750396728516,
      "activations/layer9_attention_weight_min": -35.15692901611328,
      "epoch": 21.88,
      "learning_rate": 8.993939393939393e-06,
      "loss": 2.678,
      "step": 376550
    },
    {
      "activations/layer0_attention_weight_max": 17.293649673461914,
      "activations/layer0_attention_weight_min": -13.78576946258545,
      "activations/layer10_attention_weight_max": 36.356021881103516,
      "activations/layer10_attention_weight_min": -35.8665657043457,
      "activations/layer11_attention_weight_max": 32.84452819824219,
      "activations/layer11_attention_weight_min": -34.765716552734375,
      "activations/layer12_attention_weight_max": 25.895387649536133,
      "activations/layer12_attention_weight_min": -26.149370193481445,
      "activations/layer13_attention_weight_max": 37.82213592529297,
      "activations/layer13_attention_weight_min": -35.03746032714844,
      "activations/layer14_attention_weight_max": 41.448875427246094,
      "activations/layer14_attention_weight_min": -35.68526840209961,
      "activations/layer15_attention_weight_max": 38.30856704711914,
      "activations/layer15_attention_weight_min": -34.335208892822266,
      "activations/layer16_attention_weight_max": 27.3323974609375,
      "activations/layer16_attention_weight_min": -26.59091567993164,
      "activations/layer17_attention_weight_max": 28.77766990661621,
      "activations/layer17_attention_weight_min": -24.995559692382812,
      "activations/layer18_attention_weight_max": 32.177486419677734,
      "activations/layer18_attention_weight_min": -26.216163635253906,
      "activations/layer19_attention_weight_max": 31.024534225463867,
      "activations/layer19_attention_weight_min": -32.89876937866211,
      "activations/layer1_attention_weight_max": 15.622002601623535,
      "activations/layer1_attention_weight_min": -14.832711219787598,
      "activations/layer20_attention_weight_max": 26.14766502380371,
      "activations/layer20_attention_weight_min": -25.883893966674805,
      "activations/layer21_attention_weight_max": 25.750158309936523,
      "activations/layer21_attention_weight_min": -25.690139770507812,
      "activations/layer22_attention_weight_max": 42.65057373046875,
      "activations/layer22_attention_weight_min": -34.11886978149414,
      "activations/layer23_attention_weight_max": 32.12887954711914,
      "activations/layer23_attention_weight_min": -25.66855239868164,
      "activations/layer2_attention_weight_max": 33.33473587036133,
      "activations/layer2_attention_weight_min": -32.52339172363281,
      "activations/layer3_attention_weight_max": 98.65570068359375,
      "activations/layer3_attention_weight_min": -100.79448699951172,
      "activations/layer4_attention_weight_max": 104.6161880493164,
      "activations/layer4_attention_weight_min": -106.8037109375,
      "activations/layer5_attention_weight_max": 48.279327392578125,
      "activations/layer5_attention_weight_min": -58.038414001464844,
      "activations/layer6_attention_weight_max": 43.01279830932617,
      "activations/layer6_attention_weight_min": -45.518611907958984,
      "activations/layer7_attention_weight_max": 92.47341918945312,
      "activations/layer7_attention_weight_min": -95.94921112060547,
      "activations/layer8_attention_weight_max": 40.44296646118164,
      "activations/layer8_attention_weight_min": -44.369991302490234,
      "activations/layer9_attention_weight_max": 30.478864669799805,
      "activations/layer9_attention_weight_min": -34.59849548339844,
      "epoch": 21.88,
      "learning_rate": 8.975e-06,
      "loss": 2.7064,
      "step": 376600
    },
    {
      "activations/layer0_attention_weight_max": 17.759777069091797,
      "activations/layer0_attention_weight_min": -13.828028678894043,
      "activations/layer10_attention_weight_max": 34.44123077392578,
      "activations/layer10_attention_weight_min": -35.438602447509766,
      "activations/layer11_attention_weight_max": 35.07980728149414,
      "activations/layer11_attention_weight_min": -34.06427764892578,
      "activations/layer12_attention_weight_max": 26.99815559387207,
      "activations/layer12_attention_weight_min": -29.400300979614258,
      "activations/layer13_attention_weight_max": 40.872982025146484,
      "activations/layer13_attention_weight_min": -37.683589935302734,
      "activations/layer14_attention_weight_max": 42.79446792602539,
      "activations/layer14_attention_weight_min": -39.413047790527344,
      "activations/layer15_attention_weight_max": 40.494972229003906,
      "activations/layer15_attention_weight_min": -38.836669921875,
      "activations/layer16_attention_weight_max": 29.15941047668457,
      "activations/layer16_attention_weight_min": -28.161930084228516,
      "activations/layer17_attention_weight_max": 30.1140079498291,
      "activations/layer17_attention_weight_min": -27.421165466308594,
      "activations/layer18_attention_weight_max": 33.45414352416992,
      "activations/layer18_attention_weight_min": -29.20338249206543,
      "activations/layer19_attention_weight_max": 35.62825393676758,
      "activations/layer19_attention_weight_min": -33.34719467163086,
      "activations/layer1_attention_weight_max": 15.995863914489746,
      "activations/layer1_attention_weight_min": -14.225922584533691,
      "activations/layer20_attention_weight_max": 29.194089889526367,
      "activations/layer20_attention_weight_min": -29.16455078125,
      "activations/layer21_attention_weight_max": 30.817161560058594,
      "activations/layer21_attention_weight_min": -28.25970458984375,
      "activations/layer22_attention_weight_max": 46.57378387451172,
      "activations/layer22_attention_weight_min": -36.67806625366211,
      "activations/layer23_attention_weight_max": 31.84619140625,
      "activations/layer23_attention_weight_min": -27.67992401123047,
      "activations/layer2_attention_weight_max": 33.163414001464844,
      "activations/layer2_attention_weight_min": -33.169620513916016,
      "activations/layer3_attention_weight_max": 105.35441589355469,
      "activations/layer3_attention_weight_min": -104.00177764892578,
      "activations/layer4_attention_weight_max": 108.5595932006836,
      "activations/layer4_attention_weight_min": -111.72293853759766,
      "activations/layer5_attention_weight_max": 49.717437744140625,
      "activations/layer5_attention_weight_min": -59.697105407714844,
      "activations/layer6_attention_weight_max": 46.18112564086914,
      "activations/layer6_attention_weight_min": -48.144737243652344,
      "activations/layer7_attention_weight_max": 102.37651824951172,
      "activations/layer7_attention_weight_min": -97.69606018066406,
      "activations/layer8_attention_weight_max": 43.252891540527344,
      "activations/layer8_attention_weight_min": -44.10087203979492,
      "activations/layer9_attention_weight_max": 32.46418762207031,
      "activations/layer9_attention_weight_min": -34.46208572387695,
      "epoch": 21.89,
      "learning_rate": 8.956060606060606e-06,
      "loss": 2.7087,
      "step": 376650
    },
    {
      "activations/layer0_attention_weight_max": 16.479494094848633,
      "activations/layer0_attention_weight_min": -15.663175582885742,
      "activations/layer10_attention_weight_max": 35.660030364990234,
      "activations/layer10_attention_weight_min": -35.36018753051758,
      "activations/layer11_attention_weight_max": 34.18880844116211,
      "activations/layer11_attention_weight_min": -36.08736038208008,
      "activations/layer12_attention_weight_max": 28.10004997253418,
      "activations/layer12_attention_weight_min": -28.49933624267578,
      "activations/layer13_attention_weight_max": 40.75138473510742,
      "activations/layer13_attention_weight_min": -39.46321487426758,
      "activations/layer14_attention_weight_max": 44.38849639892578,
      "activations/layer14_attention_weight_min": -44.6347770690918,
      "activations/layer15_attention_weight_max": 40.888206481933594,
      "activations/layer15_attention_weight_min": -39.91638946533203,
      "activations/layer16_attention_weight_max": 32.313385009765625,
      "activations/layer16_attention_weight_min": -31.32642364501953,
      "activations/layer17_attention_weight_max": 28.585519790649414,
      "activations/layer17_attention_weight_min": -28.835269927978516,
      "activations/layer18_attention_weight_max": 33.711456298828125,
      "activations/layer18_attention_weight_min": -25.674331665039062,
      "activations/layer19_attention_weight_max": 36.19245147705078,
      "activations/layer19_attention_weight_min": -32.84463119506836,
      "activations/layer1_attention_weight_max": 16.309167861938477,
      "activations/layer1_attention_weight_min": -15.544949531555176,
      "activations/layer20_attention_weight_max": 30.269790649414062,
      "activations/layer20_attention_weight_min": -25.664995193481445,
      "activations/layer21_attention_weight_max": 28.054271697998047,
      "activations/layer21_attention_weight_min": -24.1182861328125,
      "activations/layer22_attention_weight_max": 40.88434600830078,
      "activations/layer22_attention_weight_min": -31.3148250579834,
      "activations/layer23_attention_weight_max": 31.43084716796875,
      "activations/layer23_attention_weight_min": -26.541667938232422,
      "activations/layer2_attention_weight_max": 34.75007629394531,
      "activations/layer2_attention_weight_min": -34.49810028076172,
      "activations/layer3_attention_weight_max": 108.80162048339844,
      "activations/layer3_attention_weight_min": -110.63945007324219,
      "activations/layer4_attention_weight_max": 114.40140533447266,
      "activations/layer4_attention_weight_min": -116.75817108154297,
      "activations/layer5_attention_weight_max": 50.227882385253906,
      "activations/layer5_attention_weight_min": -62.431419372558594,
      "activations/layer6_attention_weight_max": 44.58485412597656,
      "activations/layer6_attention_weight_min": -48.44194412231445,
      "activations/layer7_attention_weight_max": 101.20649719238281,
      "activations/layer7_attention_weight_min": -103.06119537353516,
      "activations/layer8_attention_weight_max": 43.265716552734375,
      "activations/layer8_attention_weight_min": -47.018436431884766,
      "activations/layer9_attention_weight_max": 32.56695556640625,
      "activations/layer9_attention_weight_min": -34.88590621948242,
      "epoch": 21.89,
      "learning_rate": 8.937121212121211e-06,
      "loss": 2.7065,
      "step": 376700
    },
    {
      "activations/layer0_attention_weight_max": 15.7554349899292,
      "activations/layer0_attention_weight_min": -13.480035781860352,
      "activations/layer10_attention_weight_max": 37.121337890625,
      "activations/layer10_attention_weight_min": -37.249427795410156,
      "activations/layer11_attention_weight_max": 36.68955993652344,
      "activations/layer11_attention_weight_min": -39.49980926513672,
      "activations/layer12_attention_weight_max": 28.026065826416016,
      "activations/layer12_attention_weight_min": -28.171537399291992,
      "activations/layer13_attention_weight_max": 41.50548553466797,
      "activations/layer13_attention_weight_min": -37.01945495605469,
      "activations/layer14_attention_weight_max": 42.621299743652344,
      "activations/layer14_attention_weight_min": -39.301387786865234,
      "activations/layer15_attention_weight_max": 39.80356216430664,
      "activations/layer15_attention_weight_min": -37.156063079833984,
      "activations/layer16_attention_weight_max": 28.42960548400879,
      "activations/layer16_attention_weight_min": -27.525236129760742,
      "activations/layer17_attention_weight_max": 29.663631439208984,
      "activations/layer17_attention_weight_min": -25.111791610717773,
      "activations/layer18_attention_weight_max": 33.88184356689453,
      "activations/layer18_attention_weight_min": -24.395366668701172,
      "activations/layer19_attention_weight_max": 36.06521224975586,
      "activations/layer19_attention_weight_min": -30.999343872070312,
      "activations/layer1_attention_weight_max": 15.616354942321777,
      "activations/layer1_attention_weight_min": -16.04631996154785,
      "activations/layer20_attention_weight_max": 28.744359970092773,
      "activations/layer20_attention_weight_min": -23.605358123779297,
      "activations/layer21_attention_weight_max": 30.852718353271484,
      "activations/layer21_attention_weight_min": -23.85985565185547,
      "activations/layer22_attention_weight_max": 42.10589599609375,
      "activations/layer22_attention_weight_min": -31.86359405517578,
      "activations/layer23_attention_weight_max": 33.0395393371582,
      "activations/layer23_attention_weight_min": -24.06463623046875,
      "activations/layer2_attention_weight_max": 34.687339782714844,
      "activations/layer2_attention_weight_min": -34.15972900390625,
      "activations/layer3_attention_weight_max": 109.11238098144531,
      "activations/layer3_attention_weight_min": -111.90609741210938,
      "activations/layer4_attention_weight_max": 117.97442626953125,
      "activations/layer4_attention_weight_min": -120.34908294677734,
      "activations/layer5_attention_weight_max": 52.238197326660156,
      "activations/layer5_attention_weight_min": -65.39651489257812,
      "activations/layer6_attention_weight_max": 49.616458892822266,
      "activations/layer6_attention_weight_min": -53.284732818603516,
      "activations/layer7_attention_weight_max": 101.49333953857422,
      "activations/layer7_attention_weight_min": -107.67671966552734,
      "activations/layer8_attention_weight_max": 46.495025634765625,
      "activations/layer8_attention_weight_min": -49.26821517944336,
      "activations/layer9_attention_weight_max": 36.06043243408203,
      "activations/layer9_attention_weight_min": -39.169857025146484,
      "epoch": 21.89,
      "learning_rate": 8.918181818181818e-06,
      "loss": 2.7077,
      "step": 376750
    },
    {
      "activations/layer0_attention_weight_max": 16.411317825317383,
      "activations/layer0_attention_weight_min": -14.433313369750977,
      "activations/layer10_attention_weight_max": 38.38005065917969,
      "activations/layer10_attention_weight_min": -38.8442268371582,
      "activations/layer11_attention_weight_max": 37.59947204589844,
      "activations/layer11_attention_weight_min": -39.89564514160156,
      "activations/layer12_attention_weight_max": 28.621232986450195,
      "activations/layer12_attention_weight_min": -29.688764572143555,
      "activations/layer13_attention_weight_max": 43.44778060913086,
      "activations/layer13_attention_weight_min": -36.90373992919922,
      "activations/layer14_attention_weight_max": 48.95096969604492,
      "activations/layer14_attention_weight_min": -39.56586837768555,
      "activations/layer15_attention_weight_max": 42.66200256347656,
      "activations/layer15_attention_weight_min": -37.46589660644531,
      "activations/layer16_attention_weight_max": 30.000455856323242,
      "activations/layer16_attention_weight_min": -30.717988967895508,
      "activations/layer17_attention_weight_max": 32.76335525512695,
      "activations/layer17_attention_weight_min": -26.37485694885254,
      "activations/layer18_attention_weight_max": 32.277671813964844,
      "activations/layer18_attention_weight_min": -27.34712028503418,
      "activations/layer19_attention_weight_max": 34.94435119628906,
      "activations/layer19_attention_weight_min": -31.709592819213867,
      "activations/layer1_attention_weight_max": 15.619413375854492,
      "activations/layer1_attention_weight_min": -15.120047569274902,
      "activations/layer20_attention_weight_max": 29.41510009765625,
      "activations/layer20_attention_weight_min": -27.7222843170166,
      "activations/layer21_attention_weight_max": 31.22146987915039,
      "activations/layer21_attention_weight_min": -25.89365577697754,
      "activations/layer22_attention_weight_max": 42.688716888427734,
      "activations/layer22_attention_weight_min": -31.640243530273438,
      "activations/layer23_attention_weight_max": 32.77021026611328,
      "activations/layer23_attention_weight_min": -25.05980682373047,
      "activations/layer2_attention_weight_max": 36.00426483154297,
      "activations/layer2_attention_weight_min": -34.12649154663086,
      "activations/layer3_attention_weight_max": 107.79986572265625,
      "activations/layer3_attention_weight_min": -110.66167449951172,
      "activations/layer4_attention_weight_max": 114.4237289428711,
      "activations/layer4_attention_weight_min": -114.93407440185547,
      "activations/layer5_attention_weight_max": 51.943023681640625,
      "activations/layer5_attention_weight_min": -63.7072868347168,
      "activations/layer6_attention_weight_max": 47.382694244384766,
      "activations/layer6_attention_weight_min": -48.29519271850586,
      "activations/layer7_attention_weight_max": 100.7905502319336,
      "activations/layer7_attention_weight_min": -102.10159301757812,
      "activations/layer8_attention_weight_max": 48.32098388671875,
      "activations/layer8_attention_weight_min": -47.56034469604492,
      "activations/layer9_attention_weight_max": 36.02621078491211,
      "activations/layer9_attention_weight_min": -36.75893783569336,
      "epoch": 21.89,
      "learning_rate": 8.899242424242423e-06,
      "loss": 2.7241,
      "step": 376800
    },
    {
      "activations/layer0_attention_weight_max": 17.146432876586914,
      "activations/layer0_attention_weight_min": -15.237783432006836,
      "activations/layer10_attention_weight_max": 35.69552230834961,
      "activations/layer10_attention_weight_min": -35.232208251953125,
      "activations/layer11_attention_weight_max": 34.76996994018555,
      "activations/layer11_attention_weight_min": -34.79001998901367,
      "activations/layer12_attention_weight_max": 27.197589874267578,
      "activations/layer12_attention_weight_min": -32.21207809448242,
      "activations/layer13_attention_weight_max": 39.975154876708984,
      "activations/layer13_attention_weight_min": -36.83033752441406,
      "activations/layer14_attention_weight_max": 44.2570686340332,
      "activations/layer14_attention_weight_min": -39.84281539916992,
      "activations/layer15_attention_weight_max": 39.80838394165039,
      "activations/layer15_attention_weight_min": -37.435184478759766,
      "activations/layer16_attention_weight_max": 30.068456649780273,
      "activations/layer16_attention_weight_min": -30.829898834228516,
      "activations/layer17_attention_weight_max": 29.209184646606445,
      "activations/layer17_attention_weight_min": -25.933317184448242,
      "activations/layer18_attention_weight_max": 34.86228561401367,
      "activations/layer18_attention_weight_min": -26.651653289794922,
      "activations/layer19_attention_weight_max": 34.90464782714844,
      "activations/layer19_attention_weight_min": -32.5318489074707,
      "activations/layer1_attention_weight_max": 15.517563819885254,
      "activations/layer1_attention_weight_min": -14.23788070678711,
      "activations/layer20_attention_weight_max": 29.186296463012695,
      "activations/layer20_attention_weight_min": -24.3958797454834,
      "activations/layer21_attention_weight_max": 28.34246063232422,
      "activations/layer21_attention_weight_min": -24.24802589416504,
      "activations/layer22_attention_weight_max": 43.45899963378906,
      "activations/layer22_attention_weight_min": -30.266571044921875,
      "activations/layer23_attention_weight_max": 31.87885856628418,
      "activations/layer23_attention_weight_min": -25.172788619995117,
      "activations/layer2_attention_weight_max": 34.856319427490234,
      "activations/layer2_attention_weight_min": -33.61665344238281,
      "activations/layer3_attention_weight_max": 106.74691009521484,
      "activations/layer3_attention_weight_min": -108.16487884521484,
      "activations/layer4_attention_weight_max": 107.53173828125,
      "activations/layer4_attention_weight_min": -112.3305892944336,
      "activations/layer5_attention_weight_max": 51.67637252807617,
      "activations/layer5_attention_weight_min": -59.098812103271484,
      "activations/layer6_attention_weight_max": 46.64273452758789,
      "activations/layer6_attention_weight_min": -49.212642669677734,
      "activations/layer7_attention_weight_max": 97.59080505371094,
      "activations/layer7_attention_weight_min": -107.8115234375,
      "activations/layer8_attention_weight_max": 42.79524612426758,
      "activations/layer8_attention_weight_min": -48.59332275390625,
      "activations/layer9_attention_weight_max": 33.404327392578125,
      "activations/layer9_attention_weight_min": -35.61151885986328,
      "epoch": 21.9,
      "learning_rate": 8.880303030303029e-06,
      "loss": 2.6989,
      "step": 376850
    },
    {
      "activations/layer0_attention_weight_max": 16.549734115600586,
      "activations/layer0_attention_weight_min": -14.878788948059082,
      "activations/layer10_attention_weight_max": 35.49851608276367,
      "activations/layer10_attention_weight_min": -37.56501007080078,
      "activations/layer11_attention_weight_max": 36.186546325683594,
      "activations/layer11_attention_weight_min": -36.16294479370117,
      "activations/layer12_attention_weight_max": 29.269489288330078,
      "activations/layer12_attention_weight_min": -27.5278263092041,
      "activations/layer13_attention_weight_max": 41.511871337890625,
      "activations/layer13_attention_weight_min": -36.93606185913086,
      "activations/layer14_attention_weight_max": 45.242103576660156,
      "activations/layer14_attention_weight_min": -37.71363067626953,
      "activations/layer15_attention_weight_max": 46.525672912597656,
      "activations/layer15_attention_weight_min": -37.81398010253906,
      "activations/layer16_attention_weight_max": 29.690231323242188,
      "activations/layer16_attention_weight_min": -31.009599685668945,
      "activations/layer17_attention_weight_max": 31.14387321472168,
      "activations/layer17_attention_weight_min": -25.980255126953125,
      "activations/layer18_attention_weight_max": 36.768890380859375,
      "activations/layer18_attention_weight_min": -27.15766716003418,
      "activations/layer19_attention_weight_max": 37.29998779296875,
      "activations/layer19_attention_weight_min": -30.878082275390625,
      "activations/layer1_attention_weight_max": 16.717613220214844,
      "activations/layer1_attention_weight_min": -14.482002258300781,
      "activations/layer20_attention_weight_max": 31.880016326904297,
      "activations/layer20_attention_weight_min": -24.841297149658203,
      "activations/layer21_attention_weight_max": 31.87271499633789,
      "activations/layer21_attention_weight_min": -25.25911521911621,
      "activations/layer22_attention_weight_max": 44.12284851074219,
      "activations/layer22_attention_weight_min": -31.241817474365234,
      "activations/layer23_attention_weight_max": 34.7923469543457,
      "activations/layer23_attention_weight_min": -25.24622344970703,
      "activations/layer2_attention_weight_max": 33.21405792236328,
      "activations/layer2_attention_weight_min": -32.717071533203125,
      "activations/layer3_attention_weight_max": 105.48303985595703,
      "activations/layer3_attention_weight_min": -107.6624984741211,
      "activations/layer4_attention_weight_max": 110.37590789794922,
      "activations/layer4_attention_weight_min": -118.04689025878906,
      "activations/layer5_attention_weight_max": 50.76068878173828,
      "activations/layer5_attention_weight_min": -63.29491424560547,
      "activations/layer6_attention_weight_max": 46.39063262939453,
      "activations/layer6_attention_weight_min": -50.667457580566406,
      "activations/layer7_attention_weight_max": 97.31849670410156,
      "activations/layer7_attention_weight_min": -106.52822875976562,
      "activations/layer8_attention_weight_max": 43.210594177246094,
      "activations/layer8_attention_weight_min": -51.25670623779297,
      "activations/layer9_attention_weight_max": 33.95756912231445,
      "activations/layer9_attention_weight_min": -37.839630126953125,
      "epoch": 21.9,
      "learning_rate": 8.861363636363636e-06,
      "loss": 2.7078,
      "step": 376900
    },
    {
      "activations/layer0_attention_weight_max": 16.51357650756836,
      "activations/layer0_attention_weight_min": -14.471904754638672,
      "activations/layer10_attention_weight_max": 35.72726821899414,
      "activations/layer10_attention_weight_min": -37.79356384277344,
      "activations/layer11_attention_weight_max": 34.23786926269531,
      "activations/layer11_attention_weight_min": -36.62629699707031,
      "activations/layer12_attention_weight_max": 26.777036666870117,
      "activations/layer12_attention_weight_min": -27.887357711791992,
      "activations/layer13_attention_weight_max": 39.88555145263672,
      "activations/layer13_attention_weight_min": -38.67979431152344,
      "activations/layer14_attention_weight_max": 42.06558609008789,
      "activations/layer14_attention_weight_min": -38.831844329833984,
      "activations/layer15_attention_weight_max": 40.463497161865234,
      "activations/layer15_attention_weight_min": -37.811092376708984,
      "activations/layer16_attention_weight_max": 29.57243537902832,
      "activations/layer16_attention_weight_min": -28.351226806640625,
      "activations/layer17_attention_weight_max": 30.112857818603516,
      "activations/layer17_attention_weight_min": -26.51426124572754,
      "activations/layer18_attention_weight_max": 32.36675262451172,
      "activations/layer18_attention_weight_min": -27.8455753326416,
      "activations/layer19_attention_weight_max": 34.990074157714844,
      "activations/layer19_attention_weight_min": -36.5750846862793,
      "activations/layer1_attention_weight_max": 15.248746871948242,
      "activations/layer1_attention_weight_min": -14.750030517578125,
      "activations/layer20_attention_weight_max": 30.4364013671875,
      "activations/layer20_attention_weight_min": -27.64083480834961,
      "activations/layer21_attention_weight_max": 29.87843894958496,
      "activations/layer21_attention_weight_min": -26.90205955505371,
      "activations/layer22_attention_weight_max": 42.22384262084961,
      "activations/layer22_attention_weight_min": -34.10406494140625,
      "activations/layer23_attention_weight_max": 33.071678161621094,
      "activations/layer23_attention_weight_min": -28.867656707763672,
      "activations/layer2_attention_weight_max": 33.134429931640625,
      "activations/layer2_attention_weight_min": -32.859107971191406,
      "activations/layer3_attention_weight_max": 107.23238372802734,
      "activations/layer3_attention_weight_min": -106.0652084350586,
      "activations/layer4_attention_weight_max": 110.87969970703125,
      "activations/layer4_attention_weight_min": -116.88780975341797,
      "activations/layer5_attention_weight_max": 50.663169860839844,
      "activations/layer5_attention_weight_min": -62.09833526611328,
      "activations/layer6_attention_weight_max": 45.771053314208984,
      "activations/layer6_attention_weight_min": -48.6389045715332,
      "activations/layer7_attention_weight_max": 104.47589874267578,
      "activations/layer7_attention_weight_min": -104.15414428710938,
      "activations/layer8_attention_weight_max": 44.25181198120117,
      "activations/layer8_attention_weight_min": -46.7371711730957,
      "activations/layer9_attention_weight_max": 33.495216369628906,
      "activations/layer9_attention_weight_min": -35.898040771484375,
      "epoch": 21.9,
      "learning_rate": 8.84242424242424e-06,
      "loss": 2.7168,
      "step": 376950
    },
    {
      "activations/layer0_attention_weight_max": 16.37240219116211,
      "activations/layer0_attention_weight_min": -14.384961128234863,
      "activations/layer10_attention_weight_max": 36.89582824707031,
      "activations/layer10_attention_weight_min": -38.33586120605469,
      "activations/layer11_attention_weight_max": 37.34205627441406,
      "activations/layer11_attention_weight_min": -35.8415412902832,
      "activations/layer12_attention_weight_max": 28.482133865356445,
      "activations/layer12_attention_weight_min": -28.903255462646484,
      "activations/layer13_attention_weight_max": 43.17450714111328,
      "activations/layer13_attention_weight_min": -38.07490158081055,
      "activations/layer14_attention_weight_max": 43.848602294921875,
      "activations/layer14_attention_weight_min": -41.03474044799805,
      "activations/layer15_attention_weight_max": 41.05770492553711,
      "activations/layer15_attention_weight_min": -36.86471939086914,
      "activations/layer16_attention_weight_max": 29.444562911987305,
      "activations/layer16_attention_weight_min": -29.102373123168945,
      "activations/layer17_attention_weight_max": 32.44161605834961,
      "activations/layer17_attention_weight_min": -26.436113357543945,
      "activations/layer18_attention_weight_max": 32.39955139160156,
      "activations/layer18_attention_weight_min": -27.8698673248291,
      "activations/layer19_attention_weight_max": 38.51123809814453,
      "activations/layer19_attention_weight_min": -30.507766723632812,
      "activations/layer1_attention_weight_max": 16.1204833984375,
      "activations/layer1_attention_weight_min": -14.567173957824707,
      "activations/layer20_attention_weight_max": 34.103294372558594,
      "activations/layer20_attention_weight_min": -26.08917808532715,
      "activations/layer21_attention_weight_max": 31.406131744384766,
      "activations/layer21_attention_weight_min": -27.33642578125,
      "activations/layer22_attention_weight_max": 44.81475830078125,
      "activations/layer22_attention_weight_min": -32.23031997680664,
      "activations/layer23_attention_weight_max": 37.24574279785156,
      "activations/layer23_attention_weight_min": -25.515066146850586,
      "activations/layer2_attention_weight_max": 35.37794494628906,
      "activations/layer2_attention_weight_min": -34.072635650634766,
      "activations/layer3_attention_weight_max": 109.6839828491211,
      "activations/layer3_attention_weight_min": -108.43000793457031,
      "activations/layer4_attention_weight_max": 113.798828125,
      "activations/layer4_attention_weight_min": -115.68646240234375,
      "activations/layer5_attention_weight_max": 52.5199089050293,
      "activations/layer5_attention_weight_min": -61.30184555053711,
      "activations/layer6_attention_weight_max": 46.800254821777344,
      "activations/layer6_attention_weight_min": -48.89543151855469,
      "activations/layer7_attention_weight_max": 98.56491088867188,
      "activations/layer7_attention_weight_min": -105.2087631225586,
      "activations/layer8_attention_weight_max": 44.58003616333008,
      "activations/layer8_attention_weight_min": -49.222991943359375,
      "activations/layer9_attention_weight_max": 33.2636833190918,
      "activations/layer9_attention_weight_min": -37.7386589050293,
      "epoch": 21.91,
      "learning_rate": 8.823484848484849e-06,
      "loss": 2.6873,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4964,
      "eval_samples_per_second": 505.392,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4964,
      "eval_openwebtext_samples_per_second": 505.392,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9884,
      "eval_wikitext_samples_per_second": 229.331,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_lambada_loss": 2.349609375,
      "eval_lambada_ppl": 10.481474598935181,
      "eval_lambada_runtime": 9.5755,
      "eval_lambada_samples_per_second": 508.483,
      "step": 377000
    },
    {
      "activations/layer0_attention_weight_max": 16.84950828552246,
      "activations/layer0_attention_weight_min": -14.10207462310791,
      "activations/layer10_attention_weight_max": 39.94160079956055,
      "activations/layer10_attention_weight_min": -39.40650177001953,
      "activations/layer11_attention_weight_max": 37.31189727783203,
      "activations/layer11_attention_weight_min": -38.28783416748047,
      "activations/layer12_attention_weight_max": 29.124589920043945,
      "activations/layer12_attention_weight_min": -29.17732048034668,
      "activations/layer13_attention_weight_max": 41.59690475463867,
      "activations/layer13_attention_weight_min": -38.88526153564453,
      "activations/layer14_attention_weight_max": 43.706260681152344,
      "activations/layer14_attention_weight_min": -40.391448974609375,
      "activations/layer15_attention_weight_max": 38.378196716308594,
      "activations/layer15_attention_weight_min": -37.922515869140625,
      "activations/layer16_attention_weight_max": 28.623369216918945,
      "activations/layer16_attention_weight_min": -28.9449405670166,
      "activations/layer17_attention_weight_max": 30.229597091674805,
      "activations/layer17_attention_weight_min": -27.021467208862305,
      "activations/layer18_attention_weight_max": 31.803743362426758,
      "activations/layer18_attention_weight_min": -25.334712982177734,
      "activations/layer19_attention_weight_max": 35.61690902709961,
      "activations/layer19_attention_weight_min": -31.523916244506836,
      "activations/layer1_attention_weight_max": 14.951255798339844,
      "activations/layer1_attention_weight_min": -14.68332576751709,
      "activations/layer20_attention_weight_max": 26.242324829101562,
      "activations/layer20_attention_weight_min": -27.359825134277344,
      "activations/layer21_attention_weight_max": 29.202707290649414,
      "activations/layer21_attention_weight_min": -26.04781150817871,
      "activations/layer22_attention_weight_max": 39.88809585571289,
      "activations/layer22_attention_weight_min": -30.558881759643555,
      "activations/layer23_attention_weight_max": 30.894336700439453,
      "activations/layer23_attention_weight_min": -25.034282684326172,
      "activations/layer2_attention_weight_max": 33.67150115966797,
      "activations/layer2_attention_weight_min": -33.69599151611328,
      "activations/layer3_attention_weight_max": 110.07144165039062,
      "activations/layer3_attention_weight_min": -108.45214080810547,
      "activations/layer4_attention_weight_max": 112.77172088623047,
      "activations/layer4_attention_weight_min": -117.48028564453125,
      "activations/layer5_attention_weight_max": 53.162349700927734,
      "activations/layer5_attention_weight_min": -63.98302459716797,
      "activations/layer6_attention_weight_max": 49.90688705444336,
      "activations/layer6_attention_weight_min": -52.957618713378906,
      "activations/layer7_attention_weight_max": 101.64765167236328,
      "activations/layer7_attention_weight_min": -115.60835266113281,
      "activations/layer8_attention_weight_max": 46.085628509521484,
      "activations/layer8_attention_weight_min": -49.99308776855469,
      "activations/layer9_attention_weight_max": 35.562400817871094,
      "activations/layer9_attention_weight_min": -38.054161071777344,
      "epoch": 21.91,
      "learning_rate": 8.804545454545454e-06,
      "loss": 2.7143,
      "step": 377050
    },
    {
      "activations/layer0_attention_weight_max": 16.141206741333008,
      "activations/layer0_attention_weight_min": -13.758587837219238,
      "activations/layer10_attention_weight_max": 36.64879608154297,
      "activations/layer10_attention_weight_min": -37.14839172363281,
      "activations/layer11_attention_weight_max": 34.834564208984375,
      "activations/layer11_attention_weight_min": -36.50592041015625,
      "activations/layer12_attention_weight_max": 31.84988784790039,
      "activations/layer12_attention_weight_min": -29.49418830871582,
      "activations/layer13_attention_weight_max": 44.48080825805664,
      "activations/layer13_attention_weight_min": -39.06911087036133,
      "activations/layer14_attention_weight_max": 52.181724548339844,
      "activations/layer14_attention_weight_min": -41.03525924682617,
      "activations/layer15_attention_weight_max": 44.12432098388672,
      "activations/layer15_attention_weight_min": -38.97093963623047,
      "activations/layer16_attention_weight_max": 30.672664642333984,
      "activations/layer16_attention_weight_min": -31.704055786132812,
      "activations/layer17_attention_weight_max": 32.1068115234375,
      "activations/layer17_attention_weight_min": -26.640100479125977,
      "activations/layer18_attention_weight_max": 31.90334701538086,
      "activations/layer18_attention_weight_min": -26.019678115844727,
      "activations/layer19_attention_weight_max": 37.549190521240234,
      "activations/layer19_attention_weight_min": -34.82065200805664,
      "activations/layer1_attention_weight_max": 16.174968719482422,
      "activations/layer1_attention_weight_min": -14.85608959197998,
      "activations/layer20_attention_weight_max": 28.55422592163086,
      "activations/layer20_attention_weight_min": -25.64205551147461,
      "activations/layer21_attention_weight_max": 27.597562789916992,
      "activations/layer21_attention_weight_min": -24.86385154724121,
      "activations/layer22_attention_weight_max": 43.980873107910156,
      "activations/layer22_attention_weight_min": -34.17386245727539,
      "activations/layer23_attention_weight_max": 34.66082000732422,
      "activations/layer23_attention_weight_min": -28.955610275268555,
      "activations/layer2_attention_weight_max": 34.95850372314453,
      "activations/layer2_attention_weight_min": -33.25062942504883,
      "activations/layer3_attention_weight_max": 110.16093444824219,
      "activations/layer3_attention_weight_min": -107.2651138305664,
      "activations/layer4_attention_weight_max": 112.04924774169922,
      "activations/layer4_attention_weight_min": -115.11248779296875,
      "activations/layer5_attention_weight_max": 52.6713981628418,
      "activations/layer5_attention_weight_min": -67.98356628417969,
      "activations/layer6_attention_weight_max": 47.26988983154297,
      "activations/layer6_attention_weight_min": -50.06491470336914,
      "activations/layer7_attention_weight_max": 102.15255737304688,
      "activations/layer7_attention_weight_min": -109.28431701660156,
      "activations/layer8_attention_weight_max": 46.234291076660156,
      "activations/layer8_attention_weight_min": -49.94700241088867,
      "activations/layer9_attention_weight_max": 37.38229751586914,
      "activations/layer9_attention_weight_min": -37.332515716552734,
      "epoch": 21.91,
      "learning_rate": 8.78560606060606e-06,
      "loss": 2.7022,
      "step": 377100
    },
    {
      "activations/layer0_attention_weight_max": 17.15253257751465,
      "activations/layer0_attention_weight_min": -14.043901443481445,
      "activations/layer10_attention_weight_max": 36.748252868652344,
      "activations/layer10_attention_weight_min": -36.26028060913086,
      "activations/layer11_attention_weight_max": 36.07392501831055,
      "activations/layer11_attention_weight_min": -34.97422790527344,
      "activations/layer12_attention_weight_max": 26.231739044189453,
      "activations/layer12_attention_weight_min": -29.054948806762695,
      "activations/layer13_attention_weight_max": 38.24391555786133,
      "activations/layer13_attention_weight_min": -38.928443908691406,
      "activations/layer14_attention_weight_max": 39.67937088012695,
      "activations/layer14_attention_weight_min": -42.262290954589844,
      "activations/layer15_attention_weight_max": 38.692626953125,
      "activations/layer15_attention_weight_min": -38.68619155883789,
      "activations/layer16_attention_weight_max": 29.96255111694336,
      "activations/layer16_attention_weight_min": -28.503068923950195,
      "activations/layer17_attention_weight_max": 32.20967102050781,
      "activations/layer17_attention_weight_min": -26.528900146484375,
      "activations/layer18_attention_weight_max": 33.18858337402344,
      "activations/layer18_attention_weight_min": -26.401447296142578,
      "activations/layer19_attention_weight_max": 35.868125915527344,
      "activations/layer19_attention_weight_min": -32.36454391479492,
      "activations/layer1_attention_weight_max": 15.572136878967285,
      "activations/layer1_attention_weight_min": -14.77144718170166,
      "activations/layer20_attention_weight_max": 31.7900447845459,
      "activations/layer20_attention_weight_min": -25.204788208007812,
      "activations/layer21_attention_weight_max": 31.715465545654297,
      "activations/layer21_attention_weight_min": -24.8196964263916,
      "activations/layer22_attention_weight_max": 48.875144958496094,
      "activations/layer22_attention_weight_min": -32.4323616027832,
      "activations/layer23_attention_weight_max": 37.18572998046875,
      "activations/layer23_attention_weight_min": -24.33870506286621,
      "activations/layer2_attention_weight_max": 35.44039535522461,
      "activations/layer2_attention_weight_min": -34.56604766845703,
      "activations/layer3_attention_weight_max": 105.68779754638672,
      "activations/layer3_attention_weight_min": -111.17979431152344,
      "activations/layer4_attention_weight_max": 108.34551239013672,
      "activations/layer4_attention_weight_min": -114.0605697631836,
      "activations/layer5_attention_weight_max": 50.3061637878418,
      "activations/layer5_attention_weight_min": -63.71722412109375,
      "activations/layer6_attention_weight_max": 45.19921112060547,
      "activations/layer6_attention_weight_min": -48.53417205810547,
      "activations/layer7_attention_weight_max": 95.27039337158203,
      "activations/layer7_attention_weight_min": -106.02130889892578,
      "activations/layer8_attention_weight_max": 43.361419677734375,
      "activations/layer8_attention_weight_min": -45.377010345458984,
      "activations/layer9_attention_weight_max": 32.72081756591797,
      "activations/layer9_attention_weight_min": -35.1545295715332,
      "epoch": 21.91,
      "learning_rate": 8.766666666666665e-06,
      "loss": 2.706,
      "step": 377150
    },
    {
      "activations/layer0_attention_weight_max": 17.03227996826172,
      "activations/layer0_attention_weight_min": -15.030522346496582,
      "activations/layer10_attention_weight_max": 39.98310089111328,
      "activations/layer10_attention_weight_min": -40.286502838134766,
      "activations/layer11_attention_weight_max": 39.436805725097656,
      "activations/layer11_attention_weight_min": -39.32691192626953,
      "activations/layer12_attention_weight_max": 26.849943161010742,
      "activations/layer12_attention_weight_min": -27.628328323364258,
      "activations/layer13_attention_weight_max": 39.17369079589844,
      "activations/layer13_attention_weight_min": -35.99367904663086,
      "activations/layer14_attention_weight_max": 42.06192398071289,
      "activations/layer14_attention_weight_min": -39.53872299194336,
      "activations/layer15_attention_weight_max": 36.93366622924805,
      "activations/layer15_attention_weight_min": -38.9589729309082,
      "activations/layer16_attention_weight_max": 29.716638565063477,
      "activations/layer16_attention_weight_min": -28.833433151245117,
      "activations/layer17_attention_weight_max": 29.12795639038086,
      "activations/layer17_attention_weight_min": -27.347118377685547,
      "activations/layer18_attention_weight_max": 31.78634262084961,
      "activations/layer18_attention_weight_min": -25.147674560546875,
      "activations/layer19_attention_weight_max": 36.365779876708984,
      "activations/layer19_attention_weight_min": -33.239585876464844,
      "activations/layer1_attention_weight_max": 16.243473052978516,
      "activations/layer1_attention_weight_min": -15.581278800964355,
      "activations/layer20_attention_weight_max": 29.41341209411621,
      "activations/layer20_attention_weight_min": -25.7060604095459,
      "activations/layer21_attention_weight_max": 28.25971031188965,
      "activations/layer21_attention_weight_min": -25.02484130859375,
      "activations/layer22_attention_weight_max": 40.788856506347656,
      "activations/layer22_attention_weight_min": -30.230676651000977,
      "activations/layer23_attention_weight_max": 34.410377502441406,
      "activations/layer23_attention_weight_min": -25.564624786376953,
      "activations/layer2_attention_weight_max": 32.93817138671875,
      "activations/layer2_attention_weight_min": -33.01463317871094,
      "activations/layer3_attention_weight_max": 103.87188720703125,
      "activations/layer3_attention_weight_min": -104.8231430053711,
      "activations/layer4_attention_weight_max": 108.9951171875,
      "activations/layer4_attention_weight_min": -109.83135986328125,
      "activations/layer5_attention_weight_max": 50.168636322021484,
      "activations/layer5_attention_weight_min": -64.71017456054688,
      "activations/layer6_attention_weight_max": 45.5355339050293,
      "activations/layer6_attention_weight_min": -47.81757354736328,
      "activations/layer7_attention_weight_max": 96.04280090332031,
      "activations/layer7_attention_weight_min": -104.64481353759766,
      "activations/layer8_attention_weight_max": 46.41740417480469,
      "activations/layer8_attention_weight_min": -49.43927764892578,
      "activations/layer9_attention_weight_max": 36.1503791809082,
      "activations/layer9_attention_weight_min": -39.439144134521484,
      "epoch": 21.92,
      "learning_rate": 8.747727272727272e-06,
      "loss": 2.7114,
      "step": 377200
    },
    {
      "activations/layer0_attention_weight_max": 16.53795623779297,
      "activations/layer0_attention_weight_min": -14.08292293548584,
      "activations/layer10_attention_weight_max": 33.174774169921875,
      "activations/layer10_attention_weight_min": -34.924781799316406,
      "activations/layer11_attention_weight_max": 34.85444641113281,
      "activations/layer11_attention_weight_min": -36.48359298706055,
      "activations/layer12_attention_weight_max": 27.77374839782715,
      "activations/layer12_attention_weight_min": -26.767154693603516,
      "activations/layer13_attention_weight_max": 41.10601043701172,
      "activations/layer13_attention_weight_min": -35.81300735473633,
      "activations/layer14_attention_weight_max": 46.935211181640625,
      "activations/layer14_attention_weight_min": -37.17255401611328,
      "activations/layer15_attention_weight_max": 40.76008605957031,
      "activations/layer15_attention_weight_min": -36.39603042602539,
      "activations/layer16_attention_weight_max": 33.932186126708984,
      "activations/layer16_attention_weight_min": -27.761911392211914,
      "activations/layer17_attention_weight_max": 31.68471336364746,
      "activations/layer17_attention_weight_min": -26.961700439453125,
      "activations/layer18_attention_weight_max": 35.297428131103516,
      "activations/layer18_attention_weight_min": -25.238195419311523,
      "activations/layer19_attention_weight_max": 36.53440475463867,
      "activations/layer19_attention_weight_min": -30.421228408813477,
      "activations/layer1_attention_weight_max": 16.42793846130371,
      "activations/layer1_attention_weight_min": -14.655905723571777,
      "activations/layer20_attention_weight_max": 32.96519470214844,
      "activations/layer20_attention_weight_min": -24.820024490356445,
      "activations/layer21_attention_weight_max": 32.93785858154297,
      "activations/layer21_attention_weight_min": -24.585744857788086,
      "activations/layer22_attention_weight_max": 45.85890197753906,
      "activations/layer22_attention_weight_min": -29.82659149169922,
      "activations/layer23_attention_weight_max": 34.65716552734375,
      "activations/layer23_attention_weight_min": -26.31583023071289,
      "activations/layer2_attention_weight_max": 34.491737365722656,
      "activations/layer2_attention_weight_min": -34.38616943359375,
      "activations/layer3_attention_weight_max": 108.054931640625,
      "activations/layer3_attention_weight_min": -109.48857879638672,
      "activations/layer4_attention_weight_max": 115.09870910644531,
      "activations/layer4_attention_weight_min": -111.7703857421875,
      "activations/layer5_attention_weight_max": 51.04432678222656,
      "activations/layer5_attention_weight_min": -63.242340087890625,
      "activations/layer6_attention_weight_max": 45.0149040222168,
      "activations/layer6_attention_weight_min": -48.6729621887207,
      "activations/layer7_attention_weight_max": 102.0014419555664,
      "activations/layer7_attention_weight_min": -101.04530334472656,
      "activations/layer8_attention_weight_max": 42.4533576965332,
      "activations/layer8_attention_weight_min": -43.34649658203125,
      "activations/layer9_attention_weight_max": 32.58391571044922,
      "activations/layer9_attention_weight_min": -34.22251892089844,
      "epoch": 21.92,
      "learning_rate": 8.728787878787878e-06,
      "loss": 2.7303,
      "step": 377250
    },
    {
      "activations/layer0_attention_weight_max": 16.738317489624023,
      "activations/layer0_attention_weight_min": -14.560066223144531,
      "activations/layer10_attention_weight_max": 34.43342590332031,
      "activations/layer10_attention_weight_min": -35.1181640625,
      "activations/layer11_attention_weight_max": 34.936832427978516,
      "activations/layer11_attention_weight_min": -35.21489334106445,
      "activations/layer12_attention_weight_max": 26.756288528442383,
      "activations/layer12_attention_weight_min": -26.897262573242188,
      "activations/layer13_attention_weight_max": 40.848609924316406,
      "activations/layer13_attention_weight_min": -36.4044303894043,
      "activations/layer14_attention_weight_max": 43.980106353759766,
      "activations/layer14_attention_weight_min": -40.55294418334961,
      "activations/layer15_attention_weight_max": 39.53326416015625,
      "activations/layer15_attention_weight_min": -36.21437454223633,
      "activations/layer16_attention_weight_max": 31.02606964111328,
      "activations/layer16_attention_weight_min": -28.797561645507812,
      "activations/layer17_attention_weight_max": 31.046560287475586,
      "activations/layer17_attention_weight_min": -25.134288787841797,
      "activations/layer18_attention_weight_max": 33.58627700805664,
      "activations/layer18_attention_weight_min": -24.753536224365234,
      "activations/layer19_attention_weight_max": 35.94554138183594,
      "activations/layer19_attention_weight_min": -31.532917022705078,
      "activations/layer1_attention_weight_max": 18.400144577026367,
      "activations/layer1_attention_weight_min": -16.960142135620117,
      "activations/layer20_attention_weight_max": 32.908748626708984,
      "activations/layer20_attention_weight_min": -24.04014015197754,
      "activations/layer21_attention_weight_max": 31.011991500854492,
      "activations/layer21_attention_weight_min": -24.226621627807617,
      "activations/layer22_attention_weight_max": 40.96836853027344,
      "activations/layer22_attention_weight_min": -31.379039764404297,
      "activations/layer23_attention_weight_max": 30.55936050415039,
      "activations/layer23_attention_weight_min": -24.599651336669922,
      "activations/layer2_attention_weight_max": 35.81034851074219,
      "activations/layer2_attention_weight_min": -33.73419952392578,
      "activations/layer3_attention_weight_max": 108.68376159667969,
      "activations/layer3_attention_weight_min": -110.57572937011719,
      "activations/layer4_attention_weight_max": 112.51512908935547,
      "activations/layer4_attention_weight_min": -113.15511322021484,
      "activations/layer5_attention_weight_max": 51.016990661621094,
      "activations/layer5_attention_weight_min": -63.109962463378906,
      "activations/layer6_attention_weight_max": 46.35374450683594,
      "activations/layer6_attention_weight_min": -47.62546157836914,
      "activations/layer7_attention_weight_max": 97.01959228515625,
      "activations/layer7_attention_weight_min": -103.1546401977539,
      "activations/layer8_attention_weight_max": 43.01968002319336,
      "activations/layer8_attention_weight_min": -46.887611389160156,
      "activations/layer9_attention_weight_max": 32.90277862548828,
      "activations/layer9_attention_weight_min": -35.170631408691406,
      "epoch": 21.92,
      "learning_rate": 8.709848484848483e-06,
      "loss": 2.7045,
      "step": 377300
    },
    {
      "activations/layer0_attention_weight_max": 16.202957153320312,
      "activations/layer0_attention_weight_min": -13.763365745544434,
      "activations/layer10_attention_weight_max": 37.29024124145508,
      "activations/layer10_attention_weight_min": -36.99406814575195,
      "activations/layer11_attention_weight_max": 35.79005813598633,
      "activations/layer11_attention_weight_min": -37.223114013671875,
      "activations/layer12_attention_weight_max": 28.187925338745117,
      "activations/layer12_attention_weight_min": -29.34417152404785,
      "activations/layer13_attention_weight_max": 40.56733703613281,
      "activations/layer13_attention_weight_min": -39.19392395019531,
      "activations/layer14_attention_weight_max": 45.52040481567383,
      "activations/layer14_attention_weight_min": -40.44381332397461,
      "activations/layer15_attention_weight_max": 40.42443084716797,
      "activations/layer15_attention_weight_min": -39.65633773803711,
      "activations/layer16_attention_weight_max": 30.668970108032227,
      "activations/layer16_attention_weight_min": -29.14183235168457,
      "activations/layer17_attention_weight_max": 29.18892478942871,
      "activations/layer17_attention_weight_min": -26.804895401000977,
      "activations/layer18_attention_weight_max": 36.937320709228516,
      "activations/layer18_attention_weight_min": -28.41847038269043,
      "activations/layer19_attention_weight_max": 39.01415252685547,
      "activations/layer19_attention_weight_min": -32.22942352294922,
      "activations/layer1_attention_weight_max": 16.117509841918945,
      "activations/layer1_attention_weight_min": -15.510047912597656,
      "activations/layer20_attention_weight_max": 30.954172134399414,
      "activations/layer20_attention_weight_min": -26.405332565307617,
      "activations/layer21_attention_weight_max": 30.497148513793945,
      "activations/layer21_attention_weight_min": -25.79522705078125,
      "activations/layer22_attention_weight_max": 43.97718811035156,
      "activations/layer22_attention_weight_min": -33.5020751953125,
      "activations/layer23_attention_weight_max": 35.871337890625,
      "activations/layer23_attention_weight_min": -29.96278190612793,
      "activations/layer2_attention_weight_max": 33.10940170288086,
      "activations/layer2_attention_weight_min": -33.55470275878906,
      "activations/layer3_attention_weight_max": 109.75341033935547,
      "activations/layer3_attention_weight_min": -110.13655090332031,
      "activations/layer4_attention_weight_max": 113.40278625488281,
      "activations/layer4_attention_weight_min": -115.05767059326172,
      "activations/layer5_attention_weight_max": 53.48420715332031,
      "activations/layer5_attention_weight_min": -61.12162780761719,
      "activations/layer6_attention_weight_max": 47.81946563720703,
      "activations/layer6_attention_weight_min": -49.37998962402344,
      "activations/layer7_attention_weight_max": 101.41561889648438,
      "activations/layer7_attention_weight_min": -105.93412017822266,
      "activations/layer8_attention_weight_max": 45.01542663574219,
      "activations/layer8_attention_weight_min": -50.8098030090332,
      "activations/layer9_attention_weight_max": 34.779842376708984,
      "activations/layer9_attention_weight_min": -37.0378532409668,
      "epoch": 21.93,
      "learning_rate": 8.69090909090909e-06,
      "loss": 2.6948,
      "step": 377350
    },
    {
      "activations/layer0_attention_weight_max": 17.415843963623047,
      "activations/layer0_attention_weight_min": -14.572784423828125,
      "activations/layer10_attention_weight_max": 40.94252014160156,
      "activations/layer10_attention_weight_min": -40.26758575439453,
      "activations/layer11_attention_weight_max": 41.288352966308594,
      "activations/layer11_attention_weight_min": -40.14859390258789,
      "activations/layer12_attention_weight_max": 30.413679122924805,
      "activations/layer12_attention_weight_min": -28.419355392456055,
      "activations/layer13_attention_weight_max": 42.14540100097656,
      "activations/layer13_attention_weight_min": -37.49150085449219,
      "activations/layer14_attention_weight_max": 47.50562286376953,
      "activations/layer14_attention_weight_min": -40.239051818847656,
      "activations/layer15_attention_weight_max": 40.35213851928711,
      "activations/layer15_attention_weight_min": -43.502845764160156,
      "activations/layer16_attention_weight_max": 30.75053596496582,
      "activations/layer16_attention_weight_min": -29.309568405151367,
      "activations/layer17_attention_weight_max": 33.21672439575195,
      "activations/layer17_attention_weight_min": -29.453447341918945,
      "activations/layer18_attention_weight_max": 33.737396240234375,
      "activations/layer18_attention_weight_min": -30.214143753051758,
      "activations/layer19_attention_weight_max": 31.31511878967285,
      "activations/layer19_attention_weight_min": -34.82915496826172,
      "activations/layer1_attention_weight_max": 16.510665893554688,
      "activations/layer1_attention_weight_min": -15.653424263000488,
      "activations/layer20_attention_weight_max": 30.09124755859375,
      "activations/layer20_attention_weight_min": -26.8547306060791,
      "activations/layer21_attention_weight_max": 27.957645416259766,
      "activations/layer21_attention_weight_min": -26.759246826171875,
      "activations/layer22_attention_weight_max": 38.828529357910156,
      "activations/layer22_attention_weight_min": -34.78607177734375,
      "activations/layer23_attention_weight_max": 30.232099533081055,
      "activations/layer23_attention_weight_min": -25.383460998535156,
      "activations/layer2_attention_weight_max": 34.873207092285156,
      "activations/layer2_attention_weight_min": -34.92694854736328,
      "activations/layer3_attention_weight_max": 109.62186431884766,
      "activations/layer3_attention_weight_min": -111.64802551269531,
      "activations/layer4_attention_weight_max": 111.0229721069336,
      "activations/layer4_attention_weight_min": -113.66156005859375,
      "activations/layer5_attention_weight_max": 51.85622787475586,
      "activations/layer5_attention_weight_min": -62.79825973510742,
      "activations/layer6_attention_weight_max": 49.12491989135742,
      "activations/layer6_attention_weight_min": -51.21772766113281,
      "activations/layer7_attention_weight_max": 104.92757415771484,
      "activations/layer7_attention_weight_min": -110.55052185058594,
      "activations/layer8_attention_weight_max": 47.27714920043945,
      "activations/layer8_attention_weight_min": -49.319068908691406,
      "activations/layer9_attention_weight_max": 38.8615608215332,
      "activations/layer9_attention_weight_min": -38.64818572998047,
      "epoch": 21.93,
      "learning_rate": 8.671969696969696e-06,
      "loss": 2.7232,
      "step": 377400
    },
    {
      "activations/layer0_attention_weight_max": 16.199871063232422,
      "activations/layer0_attention_weight_min": -13.440305709838867,
      "activations/layer10_attention_weight_max": 35.954307556152344,
      "activations/layer10_attention_weight_min": -36.30180740356445,
      "activations/layer11_attention_weight_max": 35.79182434082031,
      "activations/layer11_attention_weight_min": -36.082237243652344,
      "activations/layer12_attention_weight_max": 28.510517120361328,
      "activations/layer12_attention_weight_min": -30.37269401550293,
      "activations/layer13_attention_weight_max": 45.16333770751953,
      "activations/layer13_attention_weight_min": -39.75651550292969,
      "activations/layer14_attention_weight_max": 47.58138656616211,
      "activations/layer14_attention_weight_min": -40.097015380859375,
      "activations/layer15_attention_weight_max": 40.060184478759766,
      "activations/layer15_attention_weight_min": -37.48350524902344,
      "activations/layer16_attention_weight_max": 28.86172103881836,
      "activations/layer16_attention_weight_min": -27.10364532470703,
      "activations/layer17_attention_weight_max": 33.4299430847168,
      "activations/layer17_attention_weight_min": -25.333242416381836,
      "activations/layer18_attention_weight_max": 33.4160041809082,
      "activations/layer18_attention_weight_min": -26.040306091308594,
      "activations/layer19_attention_weight_max": 33.2309684753418,
      "activations/layer19_attention_weight_min": -31.359704971313477,
      "activations/layer1_attention_weight_max": 18.24207878112793,
      "activations/layer1_attention_weight_min": -16.8737850189209,
      "activations/layer20_attention_weight_max": 32.88690948486328,
      "activations/layer20_attention_weight_min": -25.46674156188965,
      "activations/layer21_attention_weight_max": 30.81413459777832,
      "activations/layer21_attention_weight_min": -25.41676902770996,
      "activations/layer22_attention_weight_max": 41.851924896240234,
      "activations/layer22_attention_weight_min": -31.751115798950195,
      "activations/layer23_attention_weight_max": 31.67430877685547,
      "activations/layer23_attention_weight_min": -26.0601863861084,
      "activations/layer2_attention_weight_max": 34.887760162353516,
      "activations/layer2_attention_weight_min": -32.59290313720703,
      "activations/layer3_attention_weight_max": 106.61199951171875,
      "activations/layer3_attention_weight_min": -106.04517364501953,
      "activations/layer4_attention_weight_max": 112.07994842529297,
      "activations/layer4_attention_weight_min": -117.7577896118164,
      "activations/layer5_attention_weight_max": 53.913116455078125,
      "activations/layer5_attention_weight_min": -64.95648193359375,
      "activations/layer6_attention_weight_max": 46.415061950683594,
      "activations/layer6_attention_weight_min": -48.91033172607422,
      "activations/layer7_attention_weight_max": 101.28451538085938,
      "activations/layer7_attention_weight_min": -108.53425598144531,
      "activations/layer8_attention_weight_max": 44.02821350097656,
      "activations/layer8_attention_weight_min": -48.218387603759766,
      "activations/layer9_attention_weight_max": 33.49552536010742,
      "activations/layer9_attention_weight_min": -36.1737060546875,
      "epoch": 21.93,
      "learning_rate": 8.653030303030303e-06,
      "loss": 2.7159,
      "step": 377450
    },
    {
      "activations/layer0_attention_weight_max": 17.089038848876953,
      "activations/layer0_attention_weight_min": -13.609373092651367,
      "activations/layer10_attention_weight_max": 34.906288146972656,
      "activations/layer10_attention_weight_min": -35.48359680175781,
      "activations/layer11_attention_weight_max": 33.21772003173828,
      "activations/layer11_attention_weight_min": -35.42993927001953,
      "activations/layer12_attention_weight_max": 28.073078155517578,
      "activations/layer12_attention_weight_min": -26.833051681518555,
      "activations/layer13_attention_weight_max": 40.913719177246094,
      "activations/layer13_attention_weight_min": -36.79570007324219,
      "activations/layer14_attention_weight_max": 44.843971252441406,
      "activations/layer14_attention_weight_min": -39.101749420166016,
      "activations/layer15_attention_weight_max": 39.46805953979492,
      "activations/layer15_attention_weight_min": -38.262813568115234,
      "activations/layer16_attention_weight_max": 29.285573959350586,
      "activations/layer16_attention_weight_min": -28.7344970703125,
      "activations/layer17_attention_weight_max": 30.464786529541016,
      "activations/layer17_attention_weight_min": -27.341907501220703,
      "activations/layer18_attention_weight_max": 32.9619255065918,
      "activations/layer18_attention_weight_min": -27.15983772277832,
      "activations/layer19_attention_weight_max": 33.44361877441406,
      "activations/layer19_attention_weight_min": -34.567935943603516,
      "activations/layer1_attention_weight_max": 16.440603256225586,
      "activations/layer1_attention_weight_min": -16.59458351135254,
      "activations/layer20_attention_weight_max": 30.43852996826172,
      "activations/layer20_attention_weight_min": -29.168649673461914,
      "activations/layer21_attention_weight_max": 29.155593872070312,
      "activations/layer21_attention_weight_min": -26.955322265625,
      "activations/layer22_attention_weight_max": 40.304954528808594,
      "activations/layer22_attention_weight_min": -37.211788177490234,
      "activations/layer23_attention_weight_max": 31.510351181030273,
      "activations/layer23_attention_weight_min": -26.970069885253906,
      "activations/layer2_attention_weight_max": 34.99042510986328,
      "activations/layer2_attention_weight_min": -34.803531646728516,
      "activations/layer3_attention_weight_max": 106.40850067138672,
      "activations/layer3_attention_weight_min": -107.81928253173828,
      "activations/layer4_attention_weight_max": 114.15936279296875,
      "activations/layer4_attention_weight_min": -113.6747055053711,
      "activations/layer5_attention_weight_max": 50.17152404785156,
      "activations/layer5_attention_weight_min": -63.025508880615234,
      "activations/layer6_attention_weight_max": 47.00554275512695,
      "activations/layer6_attention_weight_min": -49.865901947021484,
      "activations/layer7_attention_weight_max": 98.79164123535156,
      "activations/layer7_attention_weight_min": -103.27597045898438,
      "activations/layer8_attention_weight_max": 42.5831298828125,
      "activations/layer8_attention_weight_min": -45.810665130615234,
      "activations/layer9_attention_weight_max": 32.48878860473633,
      "activations/layer9_attention_weight_min": -35.14652633666992,
      "epoch": 21.93,
      "learning_rate": 8.634090909090908e-06,
      "loss": 2.7063,
      "step": 377500
    },
    {
      "activations/layer0_attention_weight_max": 17.035568237304688,
      "activations/layer0_attention_weight_min": -13.706513404846191,
      "activations/layer10_attention_weight_max": 34.362091064453125,
      "activations/layer10_attention_weight_min": -36.20791244506836,
      "activations/layer11_attention_weight_max": 32.74482727050781,
      "activations/layer11_attention_weight_min": -37.95954895019531,
      "activations/layer12_attention_weight_max": 25.487964630126953,
      "activations/layer12_attention_weight_min": -30.413183212280273,
      "activations/layer13_attention_weight_max": 37.93626403808594,
      "activations/layer13_attention_weight_min": -35.72280502319336,
      "activations/layer14_attention_weight_max": 40.13679122924805,
      "activations/layer14_attention_weight_min": -38.144561767578125,
      "activations/layer15_attention_weight_max": 38.5616455078125,
      "activations/layer15_attention_weight_min": -35.35994338989258,
      "activations/layer16_attention_weight_max": 31.264644622802734,
      "activations/layer16_attention_weight_min": -28.130258560180664,
      "activations/layer17_attention_weight_max": 28.98931312561035,
      "activations/layer17_attention_weight_min": -25.8795108795166,
      "activations/layer18_attention_weight_max": 32.436279296875,
      "activations/layer18_attention_weight_min": -25.52607536315918,
      "activations/layer19_attention_weight_max": 37.46609115600586,
      "activations/layer19_attention_weight_min": -32.32760238647461,
      "activations/layer1_attention_weight_max": 14.86404800415039,
      "activations/layer1_attention_weight_min": -14.48815631866455,
      "activations/layer20_attention_weight_max": 32.32715606689453,
      "activations/layer20_attention_weight_min": -24.85582733154297,
      "activations/layer21_attention_weight_max": 30.72385025024414,
      "activations/layer21_attention_weight_min": -23.938451766967773,
      "activations/layer22_attention_weight_max": 41.22827911376953,
      "activations/layer22_attention_weight_min": -31.07320213317871,
      "activations/layer23_attention_weight_max": 30.781757354736328,
      "activations/layer23_attention_weight_min": -24.569812774658203,
      "activations/layer2_attention_weight_max": 34.91813659667969,
      "activations/layer2_attention_weight_min": -32.563568115234375,
      "activations/layer3_attention_weight_max": 106.00326538085938,
      "activations/layer3_attention_weight_min": -105.53688049316406,
      "activations/layer4_attention_weight_max": 113.92042541503906,
      "activations/layer4_attention_weight_min": -114.13429260253906,
      "activations/layer5_attention_weight_max": 52.62433624267578,
      "activations/layer5_attention_weight_min": -65.72506713867188,
      "activations/layer6_attention_weight_max": 44.30232620239258,
      "activations/layer6_attention_weight_min": -47.915687561035156,
      "activations/layer7_attention_weight_max": 98.3777084350586,
      "activations/layer7_attention_weight_min": -96.69987487792969,
      "activations/layer8_attention_weight_max": 40.379302978515625,
      "activations/layer8_attention_weight_min": -43.71864318847656,
      "activations/layer9_attention_weight_max": 31.03338050842285,
      "activations/layer9_attention_weight_min": -32.762996673583984,
      "epoch": 21.94,
      "learning_rate": 8.615151515151514e-06,
      "loss": 2.7058,
      "step": 377550
    },
    {
      "activations/layer0_attention_weight_max": 16.276351928710938,
      "activations/layer0_attention_weight_min": -14.644720077514648,
      "activations/layer10_attention_weight_max": 36.96582794189453,
      "activations/layer10_attention_weight_min": -36.66403579711914,
      "activations/layer11_attention_weight_max": 36.18358612060547,
      "activations/layer11_attention_weight_min": -37.63660430908203,
      "activations/layer12_attention_weight_max": 26.886341094970703,
      "activations/layer12_attention_weight_min": -26.62079620361328,
      "activations/layer13_attention_weight_max": 38.850379943847656,
      "activations/layer13_attention_weight_min": -36.05287170410156,
      "activations/layer14_attention_weight_max": 40.56403350830078,
      "activations/layer14_attention_weight_min": -36.94751739501953,
      "activations/layer15_attention_weight_max": 37.90389633178711,
      "activations/layer15_attention_weight_min": -35.469749450683594,
      "activations/layer16_attention_weight_max": 29.259016036987305,
      "activations/layer16_attention_weight_min": -30.04728889465332,
      "activations/layer17_attention_weight_max": 31.29475975036621,
      "activations/layer17_attention_weight_min": -25.751989364624023,
      "activations/layer18_attention_weight_max": 33.932987213134766,
      "activations/layer18_attention_weight_min": -25.9551944732666,
      "activations/layer19_attention_weight_max": 37.53683853149414,
      "activations/layer19_attention_weight_min": -34.62714385986328,
      "activations/layer1_attention_weight_max": 17.031253814697266,
      "activations/layer1_attention_weight_min": -16.72970962524414,
      "activations/layer20_attention_weight_max": 31.818883895874023,
      "activations/layer20_attention_weight_min": -27.014265060424805,
      "activations/layer21_attention_weight_max": 32.61357498168945,
      "activations/layer21_attention_weight_min": -26.74051856994629,
      "activations/layer22_attention_weight_max": 43.72365188598633,
      "activations/layer22_attention_weight_min": -32.622127532958984,
      "activations/layer23_attention_weight_max": 33.343685150146484,
      "activations/layer23_attention_weight_min": -25.99941062927246,
      "activations/layer2_attention_weight_max": 33.56605529785156,
      "activations/layer2_attention_weight_min": -33.48322677612305,
      "activations/layer3_attention_weight_max": 104.37354278564453,
      "activations/layer3_attention_weight_min": -108.91304779052734,
      "activations/layer4_attention_weight_max": 109.06294250488281,
      "activations/layer4_attention_weight_min": -110.92488098144531,
      "activations/layer5_attention_weight_max": 49.71336364746094,
      "activations/layer5_attention_weight_min": -61.39470672607422,
      "activations/layer6_attention_weight_max": 45.759456634521484,
      "activations/layer6_attention_weight_min": -47.75603485107422,
      "activations/layer7_attention_weight_max": 96.8942642211914,
      "activations/layer7_attention_weight_min": -99.31846618652344,
      "activations/layer8_attention_weight_max": 42.82545852661133,
      "activations/layer8_attention_weight_min": -46.50868225097656,
      "activations/layer9_attention_weight_max": 32.78538513183594,
      "activations/layer9_attention_weight_min": -36.75364303588867,
      "epoch": 21.94,
      "learning_rate": 8.596212121212121e-06,
      "loss": 2.7165,
      "step": 377600
    },
    {
      "activations/layer0_attention_weight_max": 16.881610870361328,
      "activations/layer0_attention_weight_min": -14.757586479187012,
      "activations/layer10_attention_weight_max": 34.23671340942383,
      "activations/layer10_attention_weight_min": -36.75361251831055,
      "activations/layer11_attention_weight_max": 33.90327453613281,
      "activations/layer11_attention_weight_min": -35.86522674560547,
      "activations/layer12_attention_weight_max": 25.419052124023438,
      "activations/layer12_attention_weight_min": -28.55304718017578,
      "activations/layer13_attention_weight_max": 36.408782958984375,
      "activations/layer13_attention_weight_min": -37.668434143066406,
      "activations/layer14_attention_weight_max": 37.97615051269531,
      "activations/layer14_attention_weight_min": -37.56270217895508,
      "activations/layer15_attention_weight_max": 36.5263786315918,
      "activations/layer15_attention_weight_min": -38.29901885986328,
      "activations/layer16_attention_weight_max": 28.006811141967773,
      "activations/layer16_attention_weight_min": -27.950326919555664,
      "activations/layer17_attention_weight_max": 28.45551300048828,
      "activations/layer17_attention_weight_min": -28.27240562438965,
      "activations/layer18_attention_weight_max": 33.14552307128906,
      "activations/layer18_attention_weight_min": -31.572433471679688,
      "activations/layer19_attention_weight_max": 35.41816711425781,
      "activations/layer19_attention_weight_min": -35.67916488647461,
      "activations/layer1_attention_weight_max": 17.517311096191406,
      "activations/layer1_attention_weight_min": -17.848918914794922,
      "activations/layer20_attention_weight_max": 29.61394691467285,
      "activations/layer20_attention_weight_min": -28.183269500732422,
      "activations/layer21_attention_weight_max": 31.0875301361084,
      "activations/layer21_attention_weight_min": -27.538211822509766,
      "activations/layer22_attention_weight_max": 43.19544219970703,
      "activations/layer22_attention_weight_min": -36.233036041259766,
      "activations/layer23_attention_weight_max": 33.90752410888672,
      "activations/layer23_attention_weight_min": -28.129989624023438,
      "activations/layer2_attention_weight_max": 33.569175720214844,
      "activations/layer2_attention_weight_min": -34.036354064941406,
      "activations/layer3_attention_weight_max": 105.01045227050781,
      "activations/layer3_attention_weight_min": -110.53543853759766,
      "activations/layer4_attention_weight_max": 107.88726806640625,
      "activations/layer4_attention_weight_min": -113.36180877685547,
      "activations/layer5_attention_weight_max": 50.509666442871094,
      "activations/layer5_attention_weight_min": -64.41866302490234,
      "activations/layer6_attention_weight_max": 45.73088073730469,
      "activations/layer6_attention_weight_min": -50.28862380981445,
      "activations/layer7_attention_weight_max": 99.47864532470703,
      "activations/layer7_attention_weight_min": -106.1973648071289,
      "activations/layer8_attention_weight_max": 42.12163162231445,
      "activations/layer8_attention_weight_min": -49.726253509521484,
      "activations/layer9_attention_weight_max": 31.976749420166016,
      "activations/layer9_attention_weight_min": -35.798274993896484,
      "epoch": 21.94,
      "learning_rate": 8.577272727272726e-06,
      "loss": 2.7071,
      "step": 377650
    },
    {
      "activations/layer0_attention_weight_max": 16.52685546875,
      "activations/layer0_attention_weight_min": -15.117181777954102,
      "activations/layer10_attention_weight_max": 33.07632827758789,
      "activations/layer10_attention_weight_min": -34.37487030029297,
      "activations/layer11_attention_weight_max": 33.50966262817383,
      "activations/layer11_attention_weight_min": -33.53050994873047,
      "activations/layer12_attention_weight_max": 28.345914840698242,
      "activations/layer12_attention_weight_min": -27.747533798217773,
      "activations/layer13_attention_weight_max": 41.43894958496094,
      "activations/layer13_attention_weight_min": -36.31713104248047,
      "activations/layer14_attention_weight_max": 45.657859802246094,
      "activations/layer14_attention_weight_min": -39.82682418823242,
      "activations/layer15_attention_weight_max": 40.54764938354492,
      "activations/layer15_attention_weight_min": -36.06345748901367,
      "activations/layer16_attention_weight_max": 30.98456573486328,
      "activations/layer16_attention_weight_min": -28.56719970703125,
      "activations/layer17_attention_weight_max": 30.295244216918945,
      "activations/layer17_attention_weight_min": -26.802675247192383,
      "activations/layer18_attention_weight_max": 32.63126754760742,
      "activations/layer18_attention_weight_min": -27.592580795288086,
      "activations/layer19_attention_weight_max": 35.66794204711914,
      "activations/layer19_attention_weight_min": -35.70002746582031,
      "activations/layer1_attention_weight_max": 16.104536056518555,
      "activations/layer1_attention_weight_min": -16.247358322143555,
      "activations/layer20_attention_weight_max": 30.356298446655273,
      "activations/layer20_attention_weight_min": -27.3601131439209,
      "activations/layer21_attention_weight_max": 29.09197235107422,
      "activations/layer21_attention_weight_min": -26.308517456054688,
      "activations/layer22_attention_weight_max": 40.85979080200195,
      "activations/layer22_attention_weight_min": -35.16236877441406,
      "activations/layer23_attention_weight_max": 32.96901321411133,
      "activations/layer23_attention_weight_min": -26.103557586669922,
      "activations/layer2_attention_weight_max": 33.714534759521484,
      "activations/layer2_attention_weight_min": -33.6285285949707,
      "activations/layer3_attention_weight_max": 107.73081970214844,
      "activations/layer3_attention_weight_min": -108.51226806640625,
      "activations/layer4_attention_weight_max": 110.13435363769531,
      "activations/layer4_attention_weight_min": -114.64058685302734,
      "activations/layer5_attention_weight_max": 49.85816955566406,
      "activations/layer5_attention_weight_min": -62.14719772338867,
      "activations/layer6_attention_weight_max": 45.072853088378906,
      "activations/layer6_attention_weight_min": -47.313907623291016,
      "activations/layer7_attention_weight_max": 95.2734375,
      "activations/layer7_attention_weight_min": -105.61296081542969,
      "activations/layer8_attention_weight_max": 42.036312103271484,
      "activations/layer8_attention_weight_min": -44.56169509887695,
      "activations/layer9_attention_weight_max": 32.2572135925293,
      "activations/layer9_attention_weight_min": -33.87120056152344,
      "epoch": 21.95,
      "learning_rate": 8.558333333333332e-06,
      "loss": 2.701,
      "step": 377700
    },
    {
      "activations/layer0_attention_weight_max": 16.15385627746582,
      "activations/layer0_attention_weight_min": -12.771538734436035,
      "activations/layer10_attention_weight_max": 39.9434814453125,
      "activations/layer10_attention_weight_min": -38.0267219543457,
      "activations/layer11_attention_weight_max": 38.56592559814453,
      "activations/layer11_attention_weight_min": -38.87797546386719,
      "activations/layer12_attention_weight_max": 27.985795974731445,
      "activations/layer12_attention_weight_min": -28.04661750793457,
      "activations/layer13_attention_weight_max": 44.50733184814453,
      "activations/layer13_attention_weight_min": -38.46623229980469,
      "activations/layer14_attention_weight_max": 44.431121826171875,
      "activations/layer14_attention_weight_min": -41.98138427734375,
      "activations/layer15_attention_weight_max": 40.54231262207031,
      "activations/layer15_attention_weight_min": -35.91624069213867,
      "activations/layer16_attention_weight_max": 29.74759864807129,
      "activations/layer16_attention_weight_min": -29.582571029663086,
      "activations/layer17_attention_weight_max": 32.624698638916016,
      "activations/layer17_attention_weight_min": -26.09978675842285,
      "activations/layer18_attention_weight_max": 35.61833953857422,
      "activations/layer18_attention_weight_min": -26.295028686523438,
      "activations/layer19_attention_weight_max": 34.50030517578125,
      "activations/layer19_attention_weight_min": -31.09113883972168,
      "activations/layer1_attention_weight_max": 16.178436279296875,
      "activations/layer1_attention_weight_min": -15.549936294555664,
      "activations/layer20_attention_weight_max": 30.38279914855957,
      "activations/layer20_attention_weight_min": -22.923521041870117,
      "activations/layer21_attention_weight_max": 28.113767623901367,
      "activations/layer21_attention_weight_min": -22.335779190063477,
      "activations/layer22_attention_weight_max": 42.663841247558594,
      "activations/layer22_attention_weight_min": -30.568830490112305,
      "activations/layer23_attention_weight_max": 31.861684799194336,
      "activations/layer23_attention_weight_min": -24.475616455078125,
      "activations/layer2_attention_weight_max": 33.932674407958984,
      "activations/layer2_attention_weight_min": -34.29934310913086,
      "activations/layer3_attention_weight_max": 108.86603546142578,
      "activations/layer3_attention_weight_min": -109.6607666015625,
      "activations/layer4_attention_weight_max": 112.0509262084961,
      "activations/layer4_attention_weight_min": -116.94400787353516,
      "activations/layer5_attention_weight_max": 50.82807540893555,
      "activations/layer5_attention_weight_min": -60.546913146972656,
      "activations/layer6_attention_weight_max": 46.34012222290039,
      "activations/layer6_attention_weight_min": -51.05097198486328,
      "activations/layer7_attention_weight_max": 97.71543884277344,
      "activations/layer7_attention_weight_min": -103.23544311523438,
      "activations/layer8_attention_weight_max": 45.79800796508789,
      "activations/layer8_attention_weight_min": -48.72856521606445,
      "activations/layer9_attention_weight_max": 35.71552658081055,
      "activations/layer9_attention_weight_min": -38.36196517944336,
      "epoch": 21.95,
      "learning_rate": 8.539393939393939e-06,
      "loss": 2.7115,
      "step": 377750
    },
    {
      "activations/layer0_attention_weight_max": 17.19600486755371,
      "activations/layer0_attention_weight_min": -14.022473335266113,
      "activations/layer10_attention_weight_max": 47.55900955200195,
      "activations/layer10_attention_weight_min": -41.11201858520508,
      "activations/layer11_attention_weight_max": 42.18976593017578,
      "activations/layer11_attention_weight_min": -40.67596435546875,
      "activations/layer12_attention_weight_max": 31.775861740112305,
      "activations/layer12_attention_weight_min": -28.47496795654297,
      "activations/layer13_attention_weight_max": 39.08650588989258,
      "activations/layer13_attention_weight_min": -35.5887336730957,
      "activations/layer14_attention_weight_max": 43.04444885253906,
      "activations/layer14_attention_weight_min": -38.37577438354492,
      "activations/layer15_attention_weight_max": 40.88906478881836,
      "activations/layer15_attention_weight_min": -39.28502655029297,
      "activations/layer16_attention_weight_max": 29.600648880004883,
      "activations/layer16_attention_weight_min": -27.948387145996094,
      "activations/layer17_attention_weight_max": 30.443538665771484,
      "activations/layer17_attention_weight_min": -26.49921226501465,
      "activations/layer18_attention_weight_max": 33.726776123046875,
      "activations/layer18_attention_weight_min": -27.504384994506836,
      "activations/layer19_attention_weight_max": 36.000938415527344,
      "activations/layer19_attention_weight_min": -34.11300277709961,
      "activations/layer1_attention_weight_max": 17.26767349243164,
      "activations/layer1_attention_weight_min": -17.571605682373047,
      "activations/layer20_attention_weight_max": 29.175884246826172,
      "activations/layer20_attention_weight_min": -27.252538681030273,
      "activations/layer21_attention_weight_max": 31.954299926757812,
      "activations/layer21_attention_weight_min": -26.496856689453125,
      "activations/layer22_attention_weight_max": 42.36273956298828,
      "activations/layer22_attention_weight_min": -34.96474838256836,
      "activations/layer23_attention_weight_max": 35.00198745727539,
      "activations/layer23_attention_weight_min": -26.71182632446289,
      "activations/layer2_attention_weight_max": 34.666595458984375,
      "activations/layer2_attention_weight_min": -34.8532600402832,
      "activations/layer3_attention_weight_max": 108.01484680175781,
      "activations/layer3_attention_weight_min": -104.81938171386719,
      "activations/layer4_attention_weight_max": 109.24105072021484,
      "activations/layer4_attention_weight_min": -109.3777084350586,
      "activations/layer5_attention_weight_max": 50.77755355834961,
      "activations/layer5_attention_weight_min": -59.92327880859375,
      "activations/layer6_attention_weight_max": 46.7904167175293,
      "activations/layer6_attention_weight_min": -46.88542938232422,
      "activations/layer7_attention_weight_max": 117.70118713378906,
      "activations/layer7_attention_weight_min": -106.44910430908203,
      "activations/layer8_attention_weight_max": 55.4925422668457,
      "activations/layer8_attention_weight_min": -50.58097839355469,
      "activations/layer9_attention_weight_max": 40.96894073486328,
      "activations/layer9_attention_weight_min": -40.33839416503906,
      "epoch": 21.95,
      "learning_rate": 8.520454545454544e-06,
      "loss": 2.7218,
      "step": 377800
    },
    {
      "activations/layer0_attention_weight_max": 15.610687255859375,
      "activations/layer0_attention_weight_min": -14.715110778808594,
      "activations/layer10_attention_weight_max": 40.05182647705078,
      "activations/layer10_attention_weight_min": -39.047645568847656,
      "activations/layer11_attention_weight_max": 39.03083038330078,
      "activations/layer11_attention_weight_min": -39.81496810913086,
      "activations/layer12_attention_weight_max": 31.162046432495117,
      "activations/layer12_attention_weight_min": -29.19607925415039,
      "activations/layer13_attention_weight_max": 47.88251876831055,
      "activations/layer13_attention_weight_min": -39.5590934753418,
      "activations/layer14_attention_weight_max": 48.97795486450195,
      "activations/layer14_attention_weight_min": -41.05003356933594,
      "activations/layer15_attention_weight_max": 47.8066520690918,
      "activations/layer15_attention_weight_min": -40.1916389465332,
      "activations/layer16_attention_weight_max": 35.665557861328125,
      "activations/layer16_attention_weight_min": -29.175355911254883,
      "activations/layer17_attention_weight_max": 32.99302291870117,
      "activations/layer17_attention_weight_min": -25.785377502441406,
      "activations/layer18_attention_weight_max": 35.9578742980957,
      "activations/layer18_attention_weight_min": -30.720731735229492,
      "activations/layer19_attention_weight_max": 38.10211181640625,
      "activations/layer19_attention_weight_min": -32.19269561767578,
      "activations/layer1_attention_weight_max": 17.875396728515625,
      "activations/layer1_attention_weight_min": -14.932177543640137,
      "activations/layer20_attention_weight_max": 31.00718879699707,
      "activations/layer20_attention_weight_min": -25.76491928100586,
      "activations/layer21_attention_weight_max": 31.33652114868164,
      "activations/layer21_attention_weight_min": -25.618270874023438,
      "activations/layer22_attention_weight_max": 47.44269561767578,
      "activations/layer22_attention_weight_min": -32.891380310058594,
      "activations/layer23_attention_weight_max": 34.89690017700195,
      "activations/layer23_attention_weight_min": -25.435447692871094,
      "activations/layer2_attention_weight_max": 33.66576385498047,
      "activations/layer2_attention_weight_min": -33.28441619873047,
      "activations/layer3_attention_weight_max": 106.9988021850586,
      "activations/layer3_attention_weight_min": -105.71611785888672,
      "activations/layer4_attention_weight_max": 112.69099426269531,
      "activations/layer4_attention_weight_min": -111.18745422363281,
      "activations/layer5_attention_weight_max": 51.794960021972656,
      "activations/layer5_attention_weight_min": -61.10471725463867,
      "activations/layer6_attention_weight_max": 47.349178314208984,
      "activations/layer6_attention_weight_min": -50.027156829833984,
      "activations/layer7_attention_weight_max": 104.60225677490234,
      "activations/layer7_attention_weight_min": -109.56246185302734,
      "activations/layer8_attention_weight_max": 47.581485748291016,
      "activations/layer8_attention_weight_min": -48.932376861572266,
      "activations/layer9_attention_weight_max": 36.3671875,
      "activations/layer9_attention_weight_min": -38.22519302368164,
      "epoch": 21.96,
      "learning_rate": 8.50151515151515e-06,
      "loss": 2.7058,
      "step": 377850
    },
    {
      "activations/layer0_attention_weight_max": 16.78989028930664,
      "activations/layer0_attention_weight_min": -14.799094200134277,
      "activations/layer10_attention_weight_max": 36.33119583129883,
      "activations/layer10_attention_weight_min": -35.81260681152344,
      "activations/layer11_attention_weight_max": 35.37714385986328,
      "activations/layer11_attention_weight_min": -36.07365417480469,
      "activations/layer12_attention_weight_max": 35.28180694580078,
      "activations/layer12_attention_weight_min": -27.885332107543945,
      "activations/layer13_attention_weight_max": 47.22766876220703,
      "activations/layer13_attention_weight_min": -38.5708122253418,
      "activations/layer14_attention_weight_max": 45.272727966308594,
      "activations/layer14_attention_weight_min": -39.33283615112305,
      "activations/layer15_attention_weight_max": 42.12367248535156,
      "activations/layer15_attention_weight_min": -40.153167724609375,
      "activations/layer16_attention_weight_max": 31.02549934387207,
      "activations/layer16_attention_weight_min": -29.596139907836914,
      "activations/layer17_attention_weight_max": 32.78125762939453,
      "activations/layer17_attention_weight_min": -27.06473731994629,
      "activations/layer18_attention_weight_max": 39.55179214477539,
      "activations/layer18_attention_weight_min": -29.8789005279541,
      "activations/layer19_attention_weight_max": 40.84952926635742,
      "activations/layer19_attention_weight_min": -32.7734489440918,
      "activations/layer1_attention_weight_max": 16.4880313873291,
      "activations/layer1_attention_weight_min": -15.053991317749023,
      "activations/layer20_attention_weight_max": 34.875526428222656,
      "activations/layer20_attention_weight_min": -25.950061798095703,
      "activations/layer21_attention_weight_max": 34.50333023071289,
      "activations/layer21_attention_weight_min": -26.969802856445312,
      "activations/layer22_attention_weight_max": 50.1551513671875,
      "activations/layer22_attention_weight_min": -37.194557189941406,
      "activations/layer23_attention_weight_max": 37.45939636230469,
      "activations/layer23_attention_weight_min": -27.282169342041016,
      "activations/layer2_attention_weight_max": 35.4963264465332,
      "activations/layer2_attention_weight_min": -34.578956604003906,
      "activations/layer3_attention_weight_max": 110.26165771484375,
      "activations/layer3_attention_weight_min": -115.01640319824219,
      "activations/layer4_attention_weight_max": 112.57489013671875,
      "activations/layer4_attention_weight_min": -116.70500946044922,
      "activations/layer5_attention_weight_max": 52.359039306640625,
      "activations/layer5_attention_weight_min": -60.140892028808594,
      "activations/layer6_attention_weight_max": 45.945560455322266,
      "activations/layer6_attention_weight_min": -50.01093292236328,
      "activations/layer7_attention_weight_max": 95.56454467773438,
      "activations/layer7_attention_weight_min": -101.54840850830078,
      "activations/layer8_attention_weight_max": 43.13999938964844,
      "activations/layer8_attention_weight_min": -45.074951171875,
      "activations/layer9_attention_weight_max": 33.472896575927734,
      "activations/layer9_attention_weight_min": -36.9674186706543,
      "epoch": 21.96,
      "learning_rate": 8.482575757575757e-06,
      "loss": 2.6947,
      "step": 377900
    },
    {
      "activations/layer0_attention_weight_max": 16.43941879272461,
      "activations/layer0_attention_weight_min": -15.099382400512695,
      "activations/layer10_attention_weight_max": 35.23109817504883,
      "activations/layer10_attention_weight_min": -37.7012939453125,
      "activations/layer11_attention_weight_max": 37.26366424560547,
      "activations/layer11_attention_weight_min": -36.687225341796875,
      "activations/layer12_attention_weight_max": 30.415857315063477,
      "activations/layer12_attention_weight_min": -29.73900032043457,
      "activations/layer13_attention_weight_max": 45.94826889038086,
      "activations/layer13_attention_weight_min": -39.47150802612305,
      "activations/layer14_attention_weight_max": 48.83097457885742,
      "activations/layer14_attention_weight_min": -43.46579360961914,
      "activations/layer15_attention_weight_max": 48.868011474609375,
      "activations/layer15_attention_weight_min": -43.96139144897461,
      "activations/layer16_attention_weight_max": 41.02321243286133,
      "activations/layer16_attention_weight_min": -31.286876678466797,
      "activations/layer17_attention_weight_max": 35.8647575378418,
      "activations/layer17_attention_weight_min": -31.07927131652832,
      "activations/layer18_attention_weight_max": 40.70268630981445,
      "activations/layer18_attention_weight_min": -31.57417106628418,
      "activations/layer19_attention_weight_max": 41.10783386230469,
      "activations/layer19_attention_weight_min": -35.9709358215332,
      "activations/layer1_attention_weight_max": 15.816452980041504,
      "activations/layer1_attention_weight_min": -16.007261276245117,
      "activations/layer20_attention_weight_max": 33.66999053955078,
      "activations/layer20_attention_weight_min": -29.721471786499023,
      "activations/layer21_attention_weight_max": 35.99204635620117,
      "activations/layer21_attention_weight_min": -30.428787231445312,
      "activations/layer22_attention_weight_max": 46.2166748046875,
      "activations/layer22_attention_weight_min": -39.20765686035156,
      "activations/layer23_attention_weight_max": 36.00105667114258,
      "activations/layer23_attention_weight_min": -29.08758544921875,
      "activations/layer2_attention_weight_max": 34.16217803955078,
      "activations/layer2_attention_weight_min": -33.50476837158203,
      "activations/layer3_attention_weight_max": 106.98092651367188,
      "activations/layer3_attention_weight_min": -108.83935546875,
      "activations/layer4_attention_weight_max": 111.6199951171875,
      "activations/layer4_attention_weight_min": -114.02723693847656,
      "activations/layer5_attention_weight_max": 51.706016540527344,
      "activations/layer5_attention_weight_min": -62.672279357910156,
      "activations/layer6_attention_weight_max": 46.60155487060547,
      "activations/layer6_attention_weight_min": -49.9279899597168,
      "activations/layer7_attention_weight_max": 98.26240539550781,
      "activations/layer7_attention_weight_min": -108.18741607666016,
      "activations/layer8_attention_weight_max": 43.489498138427734,
      "activations/layer8_attention_weight_min": -48.281707763671875,
      "activations/layer9_attention_weight_max": 34.0175895690918,
      "activations/layer9_attention_weight_min": -40.576637268066406,
      "epoch": 21.96,
      "learning_rate": 8.463636363636364e-06,
      "loss": 2.7035,
      "step": 377950
    },
    {
      "activations/layer0_attention_weight_max": 16.85737419128418,
      "activations/layer0_attention_weight_min": -14.555066108703613,
      "activations/layer10_attention_weight_max": 36.00244140625,
      "activations/layer10_attention_weight_min": -36.50617218017578,
      "activations/layer11_attention_weight_max": 38.06550598144531,
      "activations/layer11_attention_weight_min": -35.385841369628906,
      "activations/layer12_attention_weight_max": 26.154315948486328,
      "activations/layer12_attention_weight_min": -26.740575790405273,
      "activations/layer13_attention_weight_max": 38.39960479736328,
      "activations/layer13_attention_weight_min": -38.11640167236328,
      "activations/layer14_attention_weight_max": 40.40859603881836,
      "activations/layer14_attention_weight_min": -38.56333923339844,
      "activations/layer15_attention_weight_max": 37.93149185180664,
      "activations/layer15_attention_weight_min": -35.99348449707031,
      "activations/layer16_attention_weight_max": 29.565105438232422,
      "activations/layer16_attention_weight_min": -27.521238327026367,
      "activations/layer17_attention_weight_max": 27.509275436401367,
      "activations/layer17_attention_weight_min": -26.687114715576172,
      "activations/layer18_attention_weight_max": 30.05126953125,
      "activations/layer18_attention_weight_min": -27.133644104003906,
      "activations/layer19_attention_weight_max": 33.76382064819336,
      "activations/layer19_attention_weight_min": -31.261869430541992,
      "activations/layer1_attention_weight_max": 15.782180786132812,
      "activations/layer1_attention_weight_min": -15.225273132324219,
      "activations/layer20_attention_weight_max": 28.70745277404785,
      "activations/layer20_attention_weight_min": -25.68630027770996,
      "activations/layer21_attention_weight_max": 28.313817977905273,
      "activations/layer21_attention_weight_min": -24.8424129486084,
      "activations/layer22_attention_weight_max": 39.61121368408203,
      "activations/layer22_attention_weight_min": -31.996755599975586,
      "activations/layer23_attention_weight_max": 30.40420913696289,
      "activations/layer23_attention_weight_min": -24.627347946166992,
      "activations/layer2_attention_weight_max": 34.60753631591797,
      "activations/layer2_attention_weight_min": -33.355133056640625,
      "activations/layer3_attention_weight_max": 104.62926483154297,
      "activations/layer3_attention_weight_min": -106.93614959716797,
      "activations/layer4_attention_weight_max": 114.26219940185547,
      "activations/layer4_attention_weight_min": -111.61038970947266,
      "activations/layer5_attention_weight_max": 51.1368408203125,
      "activations/layer5_attention_weight_min": -60.93496322631836,
      "activations/layer6_attention_weight_max": 46.098140716552734,
      "activations/layer6_attention_weight_min": -49.51616668701172,
      "activations/layer7_attention_weight_max": 98.19720458984375,
      "activations/layer7_attention_weight_min": -100.88624572753906,
      "activations/layer8_attention_weight_max": 43.591033935546875,
      "activations/layer8_attention_weight_min": -46.79851150512695,
      "activations/layer9_attention_weight_max": 33.05367660522461,
      "activations/layer9_attention_weight_min": -36.621055603027344,
      "epoch": 21.96,
      "learning_rate": 8.444696969696968e-06,
      "loss": 2.6945,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4238,
      "eval_samples_per_second": 509.748,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4238,
      "eval_openwebtext_samples_per_second": 509.748,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.958,
      "eval_wikitext_samples_per_second": 232.889,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_lambada_loss": 2.306640625,
      "eval_lambada_ppl": 10.040637668042795,
      "eval_lambada_runtime": 9.5581,
      "eval_lambada_samples_per_second": 509.41,
      "step": 378000
    },
    {
      "activations/layer0_attention_weight_max": 17.319223403930664,
      "activations/layer0_attention_weight_min": -15.204713821411133,
      "activations/layer10_attention_weight_max": 37.31249237060547,
      "activations/layer10_attention_weight_min": -38.18020248413086,
      "activations/layer11_attention_weight_max": 36.6702880859375,
      "activations/layer11_attention_weight_min": -36.94350051879883,
      "activations/layer12_attention_weight_max": 30.898311614990234,
      "activations/layer12_attention_weight_min": -28.812782287597656,
      "activations/layer13_attention_weight_max": 44.37607192993164,
      "activations/layer13_attention_weight_min": -36.789974212646484,
      "activations/layer14_attention_weight_max": 43.957740783691406,
      "activations/layer14_attention_weight_min": -37.426509857177734,
      "activations/layer15_attention_weight_max": 42.91116714477539,
      "activations/layer15_attention_weight_min": -36.94843673706055,
      "activations/layer16_attention_weight_max": 30.274356842041016,
      "activations/layer16_attention_weight_min": -28.050073623657227,
      "activations/layer17_attention_weight_max": 31.44727325439453,
      "activations/layer17_attention_weight_min": -28.582706451416016,
      "activations/layer18_attention_weight_max": 31.9647274017334,
      "activations/layer18_attention_weight_min": -29.108572006225586,
      "activations/layer19_attention_weight_max": 34.43467330932617,
      "activations/layer19_attention_weight_min": -36.040138244628906,
      "activations/layer1_attention_weight_max": 15.489564895629883,
      "activations/layer1_attention_weight_min": -14.787833213806152,
      "activations/layer20_attention_weight_max": 30.345348358154297,
      "activations/layer20_attention_weight_min": -30.354900360107422,
      "activations/layer21_attention_weight_max": 29.739151000976562,
      "activations/layer21_attention_weight_min": -29.17465591430664,
      "activations/layer22_attention_weight_max": 46.51422882080078,
      "activations/layer22_attention_weight_min": -36.805320739746094,
      "activations/layer23_attention_weight_max": 33.11504364013672,
      "activations/layer23_attention_weight_min": -27.561460494995117,
      "activations/layer2_attention_weight_max": 34.59888458251953,
      "activations/layer2_attention_weight_min": -34.90787124633789,
      "activations/layer3_attention_weight_max": 108.91718292236328,
      "activations/layer3_attention_weight_min": -110.25984954833984,
      "activations/layer4_attention_weight_max": 112.67561340332031,
      "activations/layer4_attention_weight_min": -116.8348617553711,
      "activations/layer5_attention_weight_max": 53.37908172607422,
      "activations/layer5_attention_weight_min": -60.607398986816406,
      "activations/layer6_attention_weight_max": 47.748985290527344,
      "activations/layer6_attention_weight_min": -51.03644943237305,
      "activations/layer7_attention_weight_max": 105.39192962646484,
      "activations/layer7_attention_weight_min": -109.05615997314453,
      "activations/layer8_attention_weight_max": 47.67671203613281,
      "activations/layer8_attention_weight_min": -49.79936981201172,
      "activations/layer9_attention_weight_max": 33.58323669433594,
      "activations/layer9_attention_weight_min": -36.706050872802734,
      "epoch": 21.97,
      "learning_rate": 8.425757575757575e-06,
      "loss": 2.7112,
      "step": 378050
    },
    {
      "activations/layer0_attention_weight_max": 16.603084564208984,
      "activations/layer0_attention_weight_min": -13.933795928955078,
      "activations/layer10_attention_weight_max": 36.43888473510742,
      "activations/layer10_attention_weight_min": -36.020103454589844,
      "activations/layer11_attention_weight_max": 34.43720626831055,
      "activations/layer11_attention_weight_min": -36.93891143798828,
      "activations/layer12_attention_weight_max": 28.264644622802734,
      "activations/layer12_attention_weight_min": -28.310651779174805,
      "activations/layer13_attention_weight_max": 41.62800979614258,
      "activations/layer13_attention_weight_min": -36.32954025268555,
      "activations/layer14_attention_weight_max": 42.621002197265625,
      "activations/layer14_attention_weight_min": -38.54997634887695,
      "activations/layer15_attention_weight_max": 39.409244537353516,
      "activations/layer15_attention_weight_min": -37.18330764770508,
      "activations/layer16_attention_weight_max": 28.06580352783203,
      "activations/layer16_attention_weight_min": -30.460046768188477,
      "activations/layer17_attention_weight_max": 29.38333511352539,
      "activations/layer17_attention_weight_min": -26.578731536865234,
      "activations/layer18_attention_weight_max": 30.768943786621094,
      "activations/layer18_attention_weight_min": -26.37089729309082,
      "activations/layer19_attention_weight_max": 34.07863998413086,
      "activations/layer19_attention_weight_min": -31.27302360534668,
      "activations/layer1_attention_weight_max": 15.909360885620117,
      "activations/layer1_attention_weight_min": -15.386682510375977,
      "activations/layer20_attention_weight_max": 28.9521484375,
      "activations/layer20_attention_weight_min": -26.000524520874023,
      "activations/layer21_attention_weight_max": 30.608489990234375,
      "activations/layer21_attention_weight_min": -26.03609275817871,
      "activations/layer22_attention_weight_max": 39.93700408935547,
      "activations/layer22_attention_weight_min": -32.35944366455078,
      "activations/layer23_attention_weight_max": 31.669322967529297,
      "activations/layer23_attention_weight_min": -26.174192428588867,
      "activations/layer2_attention_weight_max": 34.26765441894531,
      "activations/layer2_attention_weight_min": -33.19493865966797,
      "activations/layer3_attention_weight_max": 108.25907897949219,
      "activations/layer3_attention_weight_min": -107.8642349243164,
      "activations/layer4_attention_weight_max": 111.44962310791016,
      "activations/layer4_attention_weight_min": -120.20704650878906,
      "activations/layer5_attention_weight_max": 52.139732360839844,
      "activations/layer5_attention_weight_min": -65.97357177734375,
      "activations/layer6_attention_weight_max": 45.598594665527344,
      "activations/layer6_attention_weight_min": -51.9387321472168,
      "activations/layer7_attention_weight_max": 99.97927856445312,
      "activations/layer7_attention_weight_min": -99.29481506347656,
      "activations/layer8_attention_weight_max": 43.7005500793457,
      "activations/layer8_attention_weight_min": -47.151885986328125,
      "activations/layer9_attention_weight_max": 33.68914031982422,
      "activations/layer9_attention_weight_min": -37.932796478271484,
      "epoch": 21.97,
      "learning_rate": 8.406818181818182e-06,
      "loss": 2.7197,
      "step": 378100
    },
    {
      "activations/layer0_attention_weight_max": 16.891801834106445,
      "activations/layer0_attention_weight_min": -14.90641975402832,
      "activations/layer10_attention_weight_max": 32.79897689819336,
      "activations/layer10_attention_weight_min": -34.67420196533203,
      "activations/layer11_attention_weight_max": 34.461708068847656,
      "activations/layer11_attention_weight_min": -34.536712646484375,
      "activations/layer12_attention_weight_max": 26.91529655456543,
      "activations/layer12_attention_weight_min": -27.554080963134766,
      "activations/layer13_attention_weight_max": 40.188568115234375,
      "activations/layer13_attention_weight_min": -36.32701110839844,
      "activations/layer14_attention_weight_max": 41.81754684448242,
      "activations/layer14_attention_weight_min": -40.85511779785156,
      "activations/layer15_attention_weight_max": 39.011295318603516,
      "activations/layer15_attention_weight_min": -38.3713264465332,
      "activations/layer16_attention_weight_max": 31.76293182373047,
      "activations/layer16_attention_weight_min": -31.35881233215332,
      "activations/layer17_attention_weight_max": 30.495853424072266,
      "activations/layer17_attention_weight_min": -29.696855545043945,
      "activations/layer18_attention_weight_max": 33.46672058105469,
      "activations/layer18_attention_weight_min": -28.66267204284668,
      "activations/layer19_attention_weight_max": 34.1395378112793,
      "activations/layer19_attention_weight_min": -34.64323806762695,
      "activations/layer1_attention_weight_max": 15.859136581420898,
      "activations/layer1_attention_weight_min": -15.0628080368042,
      "activations/layer20_attention_weight_max": 26.96431541442871,
      "activations/layer20_attention_weight_min": -28.764270782470703,
      "activations/layer21_attention_weight_max": 29.050596237182617,
      "activations/layer21_attention_weight_min": -27.75990867614746,
      "activations/layer22_attention_weight_max": 45.49531555175781,
      "activations/layer22_attention_weight_min": -34.630184173583984,
      "activations/layer23_attention_weight_max": 30.03204917907715,
      "activations/layer23_attention_weight_min": -27.28635025024414,
      "activations/layer2_attention_weight_max": 33.382110595703125,
      "activations/layer2_attention_weight_min": -33.23771286010742,
      "activations/layer3_attention_weight_max": 106.22364807128906,
      "activations/layer3_attention_weight_min": -107.08488464355469,
      "activations/layer4_attention_weight_max": 107.71172332763672,
      "activations/layer4_attention_weight_min": -114.4292221069336,
      "activations/layer5_attention_weight_max": 50.2712516784668,
      "activations/layer5_attention_weight_min": -63.62739181518555,
      "activations/layer6_attention_weight_max": 44.468875885009766,
      "activations/layer6_attention_weight_min": -48.23521041870117,
      "activations/layer7_attention_weight_max": 94.81140899658203,
      "activations/layer7_attention_weight_min": -99.06169891357422,
      "activations/layer8_attention_weight_max": 41.84285354614258,
      "activations/layer8_attention_weight_min": -44.04143524169922,
      "activations/layer9_attention_weight_max": 31.334543228149414,
      "activations/layer9_attention_weight_min": -34.07345199584961,
      "epoch": 21.97,
      "learning_rate": 8.387878787878787e-06,
      "loss": 2.7138,
      "step": 378150
    },
    {
      "activations/layer0_attention_weight_max": 16.939849853515625,
      "activations/layer0_attention_weight_min": -14.483755111694336,
      "activations/layer10_attention_weight_max": 38.28752517700195,
      "activations/layer10_attention_weight_min": -38.76727294921875,
      "activations/layer11_attention_weight_max": 38.21329116821289,
      "activations/layer11_attention_weight_min": -39.60076904296875,
      "activations/layer12_attention_weight_max": 28.886314392089844,
      "activations/layer12_attention_weight_min": -28.534074783325195,
      "activations/layer13_attention_weight_max": 45.826541900634766,
      "activations/layer13_attention_weight_min": -38.62406921386719,
      "activations/layer14_attention_weight_max": 46.14766311645508,
      "activations/layer14_attention_weight_min": -40.88669967651367,
      "activations/layer15_attention_weight_max": 45.517547607421875,
      "activations/layer15_attention_weight_min": -38.3313102722168,
      "activations/layer16_attention_weight_max": 29.784868240356445,
      "activations/layer16_attention_weight_min": -28.768016815185547,
      "activations/layer17_attention_weight_max": 30.37169075012207,
      "activations/layer17_attention_weight_min": -27.35676383972168,
      "activations/layer18_attention_weight_max": 31.146474838256836,
      "activations/layer18_attention_weight_min": -25.873825073242188,
      "activations/layer19_attention_weight_max": 35.4467887878418,
      "activations/layer19_attention_weight_min": -34.577796936035156,
      "activations/layer1_attention_weight_max": 15.860902786254883,
      "activations/layer1_attention_weight_min": -16.449501037597656,
      "activations/layer20_attention_weight_max": 28.45934295654297,
      "activations/layer20_attention_weight_min": -24.63298988342285,
      "activations/layer21_attention_weight_max": 27.998998641967773,
      "activations/layer21_attention_weight_min": -24.5048828125,
      "activations/layer22_attention_weight_max": 39.7286262512207,
      "activations/layer22_attention_weight_min": -30.33672332763672,
      "activations/layer23_attention_weight_max": 29.973609924316406,
      "activations/layer23_attention_weight_min": -24.770442962646484,
      "activations/layer2_attention_weight_max": 35.31782531738281,
      "activations/layer2_attention_weight_min": -33.354496002197266,
      "activations/layer3_attention_weight_max": 110.07196807861328,
      "activations/layer3_attention_weight_min": -111.10427856445312,
      "activations/layer4_attention_weight_max": 113.23828125,
      "activations/layer4_attention_weight_min": -112.5416030883789,
      "activations/layer5_attention_weight_max": 54.53308868408203,
      "activations/layer5_attention_weight_min": -65.41929626464844,
      "activations/layer6_attention_weight_max": 49.80425262451172,
      "activations/layer6_attention_weight_min": -50.38947677612305,
      "activations/layer7_attention_weight_max": 108.87081909179688,
      "activations/layer7_attention_weight_min": -107.40477752685547,
      "activations/layer8_attention_weight_max": 45.8154411315918,
      "activations/layer8_attention_weight_min": -50.38874435424805,
      "activations/layer9_attention_weight_max": 35.96505355834961,
      "activations/layer9_attention_weight_min": -37.69355010986328,
      "epoch": 21.98,
      "learning_rate": 8.368939393939393e-06,
      "loss": 2.6975,
      "step": 378200
    },
    {
      "activations/layer0_attention_weight_max": 16.710376739501953,
      "activations/layer0_attention_weight_min": -14.184020042419434,
      "activations/layer10_attention_weight_max": 32.679176330566406,
      "activations/layer10_attention_weight_min": -34.554412841796875,
      "activations/layer11_attention_weight_max": 34.1158447265625,
      "activations/layer11_attention_weight_min": -35.44729995727539,
      "activations/layer12_attention_weight_max": 26.488378524780273,
      "activations/layer12_attention_weight_min": -27.14844512939453,
      "activations/layer13_attention_weight_max": 39.734100341796875,
      "activations/layer13_attention_weight_min": -36.28483200073242,
      "activations/layer14_attention_weight_max": 44.27328872680664,
      "activations/layer14_attention_weight_min": -39.358768463134766,
      "activations/layer15_attention_weight_max": 38.49913787841797,
      "activations/layer15_attention_weight_min": -35.812103271484375,
      "activations/layer16_attention_weight_max": 31.347063064575195,
      "activations/layer16_attention_weight_min": -28.384681701660156,
      "activations/layer17_attention_weight_max": 31.943410873413086,
      "activations/layer17_attention_weight_min": -26.479084014892578,
      "activations/layer18_attention_weight_max": 35.67292022705078,
      "activations/layer18_attention_weight_min": -27.18040657043457,
      "activations/layer19_attention_weight_max": 38.36643600463867,
      "activations/layer19_attention_weight_min": -33.87131881713867,
      "activations/layer1_attention_weight_max": 15.942375183105469,
      "activations/layer1_attention_weight_min": -14.758695602416992,
      "activations/layer20_attention_weight_max": 31.010759353637695,
      "activations/layer20_attention_weight_min": -29.212554931640625,
      "activations/layer21_attention_weight_max": 28.963010787963867,
      "activations/layer21_attention_weight_min": -27.803817749023438,
      "activations/layer22_attention_weight_max": 44.83348083496094,
      "activations/layer22_attention_weight_min": -33.05974197387695,
      "activations/layer23_attention_weight_max": 35.76477813720703,
      "activations/layer23_attention_weight_min": -26.952327728271484,
      "activations/layer2_attention_weight_max": 34.31364440917969,
      "activations/layer2_attention_weight_min": -33.43201446533203,
      "activations/layer3_attention_weight_max": 109.50078582763672,
      "activations/layer3_attention_weight_min": -107.89244079589844,
      "activations/layer4_attention_weight_max": 115.56429290771484,
      "activations/layer4_attention_weight_min": -116.34796142578125,
      "activations/layer5_attention_weight_max": 49.599544525146484,
      "activations/layer5_attention_weight_min": -62.43933868408203,
      "activations/layer6_attention_weight_max": 45.26272201538086,
      "activations/layer6_attention_weight_min": -47.68059539794922,
      "activations/layer7_attention_weight_max": 93.87139129638672,
      "activations/layer7_attention_weight_min": -100.14491271972656,
      "activations/layer8_attention_weight_max": 42.13382339477539,
      "activations/layer8_attention_weight_min": -44.12199401855469,
      "activations/layer9_attention_weight_max": 31.760934829711914,
      "activations/layer9_attention_weight_min": -33.904476165771484,
      "epoch": 21.98,
      "learning_rate": 8.35e-06,
      "loss": 2.701,
      "step": 378250
    },
    {
      "activations/layer0_attention_weight_max": 18.52298927307129,
      "activations/layer0_attention_weight_min": -13.275513648986816,
      "activations/layer10_attention_weight_max": 36.82512664794922,
      "activations/layer10_attention_weight_min": -38.865081787109375,
      "activations/layer11_attention_weight_max": 36.21574401855469,
      "activations/layer11_attention_weight_min": -37.2778205871582,
      "activations/layer12_attention_weight_max": 31.51668930053711,
      "activations/layer12_attention_weight_min": -29.411279678344727,
      "activations/layer13_attention_weight_max": 44.315101623535156,
      "activations/layer13_attention_weight_min": -38.47660446166992,
      "activations/layer14_attention_weight_max": 46.583740234375,
      "activations/layer14_attention_weight_min": -40.68861389160156,
      "activations/layer15_attention_weight_max": 46.15119934082031,
      "activations/layer15_attention_weight_min": -39.79884719848633,
      "activations/layer16_attention_weight_max": 32.632747650146484,
      "activations/layer16_attention_weight_min": -28.862293243408203,
      "activations/layer17_attention_weight_max": 35.93914794921875,
      "activations/layer17_attention_weight_min": -27.864450454711914,
      "activations/layer18_attention_weight_max": 36.52485275268555,
      "activations/layer18_attention_weight_min": -26.49310874938965,
      "activations/layer19_attention_weight_max": 38.3435173034668,
      "activations/layer19_attention_weight_min": -32.181697845458984,
      "activations/layer1_attention_weight_max": 16.139148712158203,
      "activations/layer1_attention_weight_min": -16.279314041137695,
      "activations/layer20_attention_weight_max": 31.607486724853516,
      "activations/layer20_attention_weight_min": -27.839494705200195,
      "activations/layer21_attention_weight_max": 33.76032638549805,
      "activations/layer21_attention_weight_min": -25.33879852294922,
      "activations/layer22_attention_weight_max": 47.01694869995117,
      "activations/layer22_attention_weight_min": -32.84270477294922,
      "activations/layer23_attention_weight_max": 33.69816589355469,
      "activations/layer23_attention_weight_min": -25.42125701904297,
      "activations/layer2_attention_weight_max": 34.25446319580078,
      "activations/layer2_attention_weight_min": -33.927337646484375,
      "activations/layer3_attention_weight_max": 107.45243835449219,
      "activations/layer3_attention_weight_min": -109.29656982421875,
      "activations/layer4_attention_weight_max": 111.81230926513672,
      "activations/layer4_attention_weight_min": -113.95062255859375,
      "activations/layer5_attention_weight_max": 51.940120697021484,
      "activations/layer5_attention_weight_min": -64.43173217773438,
      "activations/layer6_attention_weight_max": 49.399356842041016,
      "activations/layer6_attention_weight_min": -51.55363464355469,
      "activations/layer7_attention_weight_max": 103.20574188232422,
      "activations/layer7_attention_weight_min": -109.3899154663086,
      "activations/layer8_attention_weight_max": 44.0703010559082,
      "activations/layer8_attention_weight_min": -52.265167236328125,
      "activations/layer9_attention_weight_max": 34.58384323120117,
      "activations/layer9_attention_weight_min": -38.11585235595703,
      "epoch": 21.98,
      "learning_rate": 8.331060606060606e-06,
      "loss": 2.6965,
      "step": 378300
    },
    {
      "activations/layer0_attention_weight_max": 17.758392333984375,
      "activations/layer0_attention_weight_min": -14.397387504577637,
      "activations/layer10_attention_weight_max": 40.8956298828125,
      "activations/layer10_attention_weight_min": -37.6407470703125,
      "activations/layer11_attention_weight_max": 38.73936462402344,
      "activations/layer11_attention_weight_min": -39.5078010559082,
      "activations/layer12_attention_weight_max": 29.91046714782715,
      "activations/layer12_attention_weight_min": -28.893760681152344,
      "activations/layer13_attention_weight_max": 43.43102264404297,
      "activations/layer13_attention_weight_min": -37.76716232299805,
      "activations/layer14_attention_weight_max": 45.67071533203125,
      "activations/layer14_attention_weight_min": -39.27238082885742,
      "activations/layer15_attention_weight_max": 40.97208023071289,
      "activations/layer15_attention_weight_min": -36.6888313293457,
      "activations/layer16_attention_weight_max": 28.05101776123047,
      "activations/layer16_attention_weight_min": -28.559814453125,
      "activations/layer17_attention_weight_max": 31.594003677368164,
      "activations/layer17_attention_weight_min": -26.025089263916016,
      "activations/layer18_attention_weight_max": 34.165767669677734,
      "activations/layer18_attention_weight_min": -25.45052146911621,
      "activations/layer19_attention_weight_max": 31.4599666595459,
      "activations/layer19_attention_weight_min": -32.2894401550293,
      "activations/layer1_attention_weight_max": 16.81769561767578,
      "activations/layer1_attention_weight_min": -15.4849853515625,
      "activations/layer20_attention_weight_max": 27.975679397583008,
      "activations/layer20_attention_weight_min": -27.30727195739746,
      "activations/layer21_attention_weight_max": 28.191680908203125,
      "activations/layer21_attention_weight_min": -26.70831871032715,
      "activations/layer22_attention_weight_max": 37.938636779785156,
      "activations/layer22_attention_weight_min": -32.05607223510742,
      "activations/layer23_attention_weight_max": 29.274765014648438,
      "activations/layer23_attention_weight_min": -25.94751739501953,
      "activations/layer2_attention_weight_max": 34.60436248779297,
      "activations/layer2_attention_weight_min": -34.670162200927734,
      "activations/layer3_attention_weight_max": 111.06756591796875,
      "activations/layer3_attention_weight_min": -114.19538879394531,
      "activations/layer4_attention_weight_max": 114.15361785888672,
      "activations/layer4_attention_weight_min": -114.8219223022461,
      "activations/layer5_attention_weight_max": 50.91764450073242,
      "activations/layer5_attention_weight_min": -65.87391662597656,
      "activations/layer6_attention_weight_max": 48.61722183227539,
      "activations/layer6_attention_weight_min": -52.46487045288086,
      "activations/layer7_attention_weight_max": 101.90978240966797,
      "activations/layer7_attention_weight_min": -113.29670715332031,
      "activations/layer8_attention_weight_max": 46.4871940612793,
      "activations/layer8_attention_weight_min": -49.40150451660156,
      "activations/layer9_attention_weight_max": 35.94028091430664,
      "activations/layer9_attention_weight_min": -37.58381271362305,
      "epoch": 21.98,
      "learning_rate": 8.312121212121211e-06,
      "loss": 2.695,
      "step": 378350
    },
    {
      "activations/layer0_attention_weight_max": 17.16313362121582,
      "activations/layer0_attention_weight_min": -15.51664924621582,
      "activations/layer10_attention_weight_max": 40.55474090576172,
      "activations/layer10_attention_weight_min": -37.46586227416992,
      "activations/layer11_attention_weight_max": 39.48343276977539,
      "activations/layer11_attention_weight_min": -39.5262451171875,
      "activations/layer12_attention_weight_max": 30.851848602294922,
      "activations/layer12_attention_weight_min": -30.271949768066406,
      "activations/layer13_attention_weight_max": 45.50797653198242,
      "activations/layer13_attention_weight_min": -39.4646110534668,
      "activations/layer14_attention_weight_max": 51.13936233520508,
      "activations/layer14_attention_weight_min": -40.72727966308594,
      "activations/layer15_attention_weight_max": 43.22643280029297,
      "activations/layer15_attention_weight_min": -37.66596221923828,
      "activations/layer16_attention_weight_max": 31.357421875,
      "activations/layer16_attention_weight_min": -28.793466567993164,
      "activations/layer17_attention_weight_max": 32.55647277832031,
      "activations/layer17_attention_weight_min": -27.530513763427734,
      "activations/layer18_attention_weight_max": 33.16048049926758,
      "activations/layer18_attention_weight_min": -26.064638137817383,
      "activations/layer19_attention_weight_max": 38.97904586791992,
      "activations/layer19_attention_weight_min": -33.20954513549805,
      "activations/layer1_attention_weight_max": 15.478814125061035,
      "activations/layer1_attention_weight_min": -15.6016845703125,
      "activations/layer20_attention_weight_max": 33.00282287597656,
      "activations/layer20_attention_weight_min": -27.560564041137695,
      "activations/layer21_attention_weight_max": 31.378585815429688,
      "activations/layer21_attention_weight_min": -26.62074851989746,
      "activations/layer22_attention_weight_max": 51.71646499633789,
      "activations/layer22_attention_weight_min": -33.038394927978516,
      "activations/layer23_attention_weight_max": 34.68663787841797,
      "activations/layer23_attention_weight_min": -27.385116577148438,
      "activations/layer2_attention_weight_max": 35.10664367675781,
      "activations/layer2_attention_weight_min": -34.15806579589844,
      "activations/layer3_attention_weight_max": 110.07520294189453,
      "activations/layer3_attention_weight_min": -112.7833480834961,
      "activations/layer4_attention_weight_max": 112.41094970703125,
      "activations/layer4_attention_weight_min": -117.16239166259766,
      "activations/layer5_attention_weight_max": 51.90229415893555,
      "activations/layer5_attention_weight_min": -67.42373657226562,
      "activations/layer6_attention_weight_max": 47.65529251098633,
      "activations/layer6_attention_weight_min": -51.25886535644531,
      "activations/layer7_attention_weight_max": 100.73178100585938,
      "activations/layer7_attention_weight_min": -109.98892211914062,
      "activations/layer8_attention_weight_max": 48.956871032714844,
      "activations/layer8_attention_weight_min": -48.00141906738281,
      "activations/layer9_attention_weight_max": 38.82494354248047,
      "activations/layer9_attention_weight_min": -35.29885482788086,
      "epoch": 21.99,
      "learning_rate": 8.293181818181818e-06,
      "loss": 2.7251,
      "step": 378400
    },
    {
      "activations/layer0_attention_weight_max": 16.145936965942383,
      "activations/layer0_attention_weight_min": -12.936114311218262,
      "activations/layer10_attention_weight_max": 35.88405990600586,
      "activations/layer10_attention_weight_min": -38.234832763671875,
      "activations/layer11_attention_weight_max": 35.03514862060547,
      "activations/layer11_attention_weight_min": -34.66236114501953,
      "activations/layer12_attention_weight_max": 26.618871688842773,
      "activations/layer12_attention_weight_min": -26.804391860961914,
      "activations/layer13_attention_weight_max": 38.23332214355469,
      "activations/layer13_attention_weight_min": -37.933738708496094,
      "activations/layer14_attention_weight_max": 43.25189971923828,
      "activations/layer14_attention_weight_min": -38.786537170410156,
      "activations/layer15_attention_weight_max": 39.22804641723633,
      "activations/layer15_attention_weight_min": -40.00011444091797,
      "activations/layer16_attention_weight_max": 30.67228889465332,
      "activations/layer16_attention_weight_min": -32.40281677246094,
      "activations/layer17_attention_weight_max": 28.338022232055664,
      "activations/layer17_attention_weight_min": -27.83591079711914,
      "activations/layer18_attention_weight_max": 31.4133358001709,
      "activations/layer18_attention_weight_min": -28.75172233581543,
      "activations/layer19_attention_weight_max": 41.6505241394043,
      "activations/layer19_attention_weight_min": -33.27485275268555,
      "activations/layer1_attention_weight_max": 15.835933685302734,
      "activations/layer1_attention_weight_min": -15.13849925994873,
      "activations/layer20_attention_weight_max": 30.259449005126953,
      "activations/layer20_attention_weight_min": -28.15609359741211,
      "activations/layer21_attention_weight_max": 30.22344970703125,
      "activations/layer21_attention_weight_min": -28.957931518554688,
      "activations/layer22_attention_weight_max": 41.6013069152832,
      "activations/layer22_attention_weight_min": -34.64165115356445,
      "activations/layer23_attention_weight_max": 31.187255859375,
      "activations/layer23_attention_weight_min": -26.963674545288086,
      "activations/layer2_attention_weight_max": 34.6676025390625,
      "activations/layer2_attention_weight_min": -33.757774353027344,
      "activations/layer3_attention_weight_max": 107.97794342041016,
      "activations/layer3_attention_weight_min": -107.62457275390625,
      "activations/layer4_attention_weight_max": 113.27330780029297,
      "activations/layer4_attention_weight_min": -112.00011444091797,
      "activations/layer5_attention_weight_max": 51.174827575683594,
      "activations/layer5_attention_weight_min": -64.92595672607422,
      "activations/layer6_attention_weight_max": 44.96516799926758,
      "activations/layer6_attention_weight_min": -48.492679595947266,
      "activations/layer7_attention_weight_max": 95.3493881225586,
      "activations/layer7_attention_weight_min": -109.71046447753906,
      "activations/layer8_attention_weight_max": 41.967140197753906,
      "activations/layer8_attention_weight_min": -47.357887268066406,
      "activations/layer9_attention_weight_max": 36.31808853149414,
      "activations/layer9_attention_weight_min": -37.21285629272461,
      "epoch": 21.99,
      "learning_rate": 8.274242424242424e-06,
      "loss": 2.6983,
      "step": 378450
    },
    {
      "activations/layer0_attention_weight_max": 15.930840492248535,
      "activations/layer0_attention_weight_min": -14.156974792480469,
      "activations/layer10_attention_weight_max": 39.993133544921875,
      "activations/layer10_attention_weight_min": -37.86011505126953,
      "activations/layer11_attention_weight_max": 34.93968200683594,
      "activations/layer11_attention_weight_min": -36.407920837402344,
      "activations/layer12_attention_weight_max": 28.217525482177734,
      "activations/layer12_attention_weight_min": -27.02285385131836,
      "activations/layer13_attention_weight_max": 43.23828887939453,
      "activations/layer13_attention_weight_min": -37.17424774169922,
      "activations/layer14_attention_weight_max": 45.84341812133789,
      "activations/layer14_attention_weight_min": -40.56734085083008,
      "activations/layer15_attention_weight_max": 40.42715072631836,
      "activations/layer15_attention_weight_min": -36.50421142578125,
      "activations/layer16_attention_weight_max": 29.91068458557129,
      "activations/layer16_attention_weight_min": -28.566776275634766,
      "activations/layer17_attention_weight_max": 30.071821212768555,
      "activations/layer17_attention_weight_min": -28.66421127319336,
      "activations/layer18_attention_weight_max": 31.1284236907959,
      "activations/layer18_attention_weight_min": -31.551368713378906,
      "activations/layer19_attention_weight_max": 33.86796569824219,
      "activations/layer19_attention_weight_min": -36.431358337402344,
      "activations/layer1_attention_weight_max": 16.284460067749023,
      "activations/layer1_attention_weight_min": -15.142221450805664,
      "activations/layer20_attention_weight_max": 28.88450813293457,
      "activations/layer20_attention_weight_min": -28.09481430053711,
      "activations/layer21_attention_weight_max": 27.264591217041016,
      "activations/layer21_attention_weight_min": -27.052133560180664,
      "activations/layer22_attention_weight_max": 47.98501205444336,
      "activations/layer22_attention_weight_min": -35.62382507324219,
      "activations/layer23_attention_weight_max": 32.340152740478516,
      "activations/layer23_attention_weight_min": -27.92148208618164,
      "activations/layer2_attention_weight_max": 34.47420883178711,
      "activations/layer2_attention_weight_min": -33.067588806152344,
      "activations/layer3_attention_weight_max": 108.15756225585938,
      "activations/layer3_attention_weight_min": -108.15447998046875,
      "activations/layer4_attention_weight_max": 110.70050048828125,
      "activations/layer4_attention_weight_min": -114.95458221435547,
      "activations/layer5_attention_weight_max": 51.35476303100586,
      "activations/layer5_attention_weight_min": -64.51904296875,
      "activations/layer6_attention_weight_max": 45.723411560058594,
      "activations/layer6_attention_weight_min": -50.35850524902344,
      "activations/layer7_attention_weight_max": 109.02978515625,
      "activations/layer7_attention_weight_min": -107.63792419433594,
      "activations/layer8_attention_weight_max": 47.1859016418457,
      "activations/layer8_attention_weight_min": -49.555843353271484,
      "activations/layer9_attention_weight_max": 35.718379974365234,
      "activations/layer9_attention_weight_min": -37.69585418701172,
      "epoch": 21.99,
      "learning_rate": 8.25530303030303e-06,
      "loss": 2.7161,
      "step": 378500
    },
    {
      "activations/layer0_attention_weight_max": 16.274614334106445,
      "activations/layer0_attention_weight_min": -13.339959144592285,
      "activations/layer10_attention_weight_max": 44.206607818603516,
      "activations/layer10_attention_weight_min": -43.586997985839844,
      "activations/layer11_attention_weight_max": 43.79706573486328,
      "activations/layer11_attention_weight_min": -44.03041076660156,
      "activations/layer12_attention_weight_max": 30.20846939086914,
      "activations/layer12_attention_weight_min": -29.67440414428711,
      "activations/layer13_attention_weight_max": 46.66483688354492,
      "activations/layer13_attention_weight_min": -38.595943450927734,
      "activations/layer14_attention_weight_max": 45.92867660522461,
      "activations/layer14_attention_weight_min": -41.27290344238281,
      "activations/layer15_attention_weight_max": 42.77980041503906,
      "activations/layer15_attention_weight_min": -42.903926849365234,
      "activations/layer16_attention_weight_max": 31.428075790405273,
      "activations/layer16_attention_weight_min": -29.772422790527344,
      "activations/layer17_attention_weight_max": 29.20193862915039,
      "activations/layer17_attention_weight_min": -28.429319381713867,
      "activations/layer18_attention_weight_max": 32.606101989746094,
      "activations/layer18_attention_weight_min": -27.0406436920166,
      "activations/layer19_attention_weight_max": 34.43937683105469,
      "activations/layer19_attention_weight_min": -31.049711227416992,
      "activations/layer1_attention_weight_max": 15.904243469238281,
      "activations/layer1_attention_weight_min": -15.514069557189941,
      "activations/layer20_attention_weight_max": 28.688465118408203,
      "activations/layer20_attention_weight_min": -26.299692153930664,
      "activations/layer21_attention_weight_max": 28.949289321899414,
      "activations/layer21_attention_weight_min": -25.13179588317871,
      "activations/layer22_attention_weight_max": 43.796470642089844,
      "activations/layer22_attention_weight_min": -34.32583999633789,
      "activations/layer23_attention_weight_max": 31.60979461669922,
      "activations/layer23_attention_weight_min": -30.075828552246094,
      "activations/layer2_attention_weight_max": 37.00670623779297,
      "activations/layer2_attention_weight_min": -34.6718864440918,
      "activations/layer3_attention_weight_max": 110.07649993896484,
      "activations/layer3_attention_weight_min": -111.0833740234375,
      "activations/layer4_attention_weight_max": 111.0931625366211,
      "activations/layer4_attention_weight_min": -110.1945571899414,
      "activations/layer5_attention_weight_max": 52.420684814453125,
      "activations/layer5_attention_weight_min": -61.6417236328125,
      "activations/layer6_attention_weight_max": 44.872398376464844,
      "activations/layer6_attention_weight_min": -47.78232192993164,
      "activations/layer7_attention_weight_max": 106.96958923339844,
      "activations/layer7_attention_weight_min": -107.99813842773438,
      "activations/layer8_attention_weight_max": 49.66985321044922,
      "activations/layer8_attention_weight_min": -50.949954986572266,
      "activations/layer9_attention_weight_max": 40.92538070678711,
      "activations/layer9_attention_weight_min": -43.87049102783203,
      "epoch": 22.0,
      "learning_rate": 8.236363636363636e-06,
      "loss": 2.708,
      "step": 378550
    },
    {
      "activations/layer0_attention_weight_max": 16.58125877380371,
      "activations/layer0_attention_weight_min": -13.120326042175293,
      "activations/layer10_attention_weight_max": 39.882720947265625,
      "activations/layer10_attention_weight_min": -38.83952713012695,
      "activations/layer11_attention_weight_max": 41.280967712402344,
      "activations/layer11_attention_weight_min": -41.610137939453125,
      "activations/layer12_attention_weight_max": 30.232223510742188,
      "activations/layer12_attention_weight_min": -29.9263973236084,
      "activations/layer13_attention_weight_max": 41.348724365234375,
      "activations/layer13_attention_weight_min": -39.67046356201172,
      "activations/layer14_attention_weight_max": 45.506019592285156,
      "activations/layer14_attention_weight_min": -43.35429382324219,
      "activations/layer15_attention_weight_max": 44.47016906738281,
      "activations/layer15_attention_weight_min": -39.68251419067383,
      "activations/layer16_attention_weight_max": 33.46568298339844,
      "activations/layer16_attention_weight_min": -27.446765899658203,
      "activations/layer17_attention_weight_max": 28.642181396484375,
      "activations/layer17_attention_weight_min": -25.166654586791992,
      "activations/layer18_attention_weight_max": 30.968936920166016,
      "activations/layer18_attention_weight_min": -25.84584617614746,
      "activations/layer19_attention_weight_max": 34.12308883666992,
      "activations/layer19_attention_weight_min": -30.577392578125,
      "activations/layer1_attention_weight_max": 16.271154403686523,
      "activations/layer1_attention_weight_min": -15.25279712677002,
      "activations/layer20_attention_weight_max": 29.619304656982422,
      "activations/layer20_attention_weight_min": -24.557273864746094,
      "activations/layer21_attention_weight_max": 28.263790130615234,
      "activations/layer21_attention_weight_min": -24.2045955657959,
      "activations/layer22_attention_weight_max": 44.039920806884766,
      "activations/layer22_attention_weight_min": -30.62503433227539,
      "activations/layer23_attention_weight_max": 32.619049072265625,
      "activations/layer23_attention_weight_min": -25.66992950439453,
      "activations/layer2_attention_weight_max": 34.18412780761719,
      "activations/layer2_attention_weight_min": -34.94277572631836,
      "activations/layer3_attention_weight_max": 106.6644287109375,
      "activations/layer3_attention_weight_min": -112.60234832763672,
      "activations/layer4_attention_weight_max": 110.13851165771484,
      "activations/layer4_attention_weight_min": -119.67164611816406,
      "activations/layer5_attention_weight_max": 53.05064010620117,
      "activations/layer5_attention_weight_min": -62.278106689453125,
      "activations/layer6_attention_weight_max": 47.22526931762695,
      "activations/layer6_attention_weight_min": -49.42481231689453,
      "activations/layer7_attention_weight_max": 105.6318130493164,
      "activations/layer7_attention_weight_min": -105.01721954345703,
      "activations/layer8_attention_weight_max": 45.580692291259766,
      "activations/layer8_attention_weight_min": -47.14374542236328,
      "activations/layer9_attention_weight_max": 38.33818435668945,
      "activations/layer9_attention_weight_min": -39.83049392700195,
      "epoch": 22.0,
      "learning_rate": 8.21742424242424e-06,
      "loss": 2.7038,
      "step": 378600
    },
    {
      "activations/layer0_attention_weight_max": 18.418598175048828,
      "activations/layer0_attention_weight_min": -14.616458892822266,
      "activations/layer10_attention_weight_max": 37.78776168823242,
      "activations/layer10_attention_weight_min": -38.662147521972656,
      "activations/layer11_attention_weight_max": 35.25569152832031,
      "activations/layer11_attention_weight_min": -38.55817413330078,
      "activations/layer12_attention_weight_max": 27.212480545043945,
      "activations/layer12_attention_weight_min": -27.360013961791992,
      "activations/layer13_attention_weight_max": 40.56782913208008,
      "activations/layer13_attention_weight_min": -38.97944641113281,
      "activations/layer14_attention_weight_max": 42.88117599487305,
      "activations/layer14_attention_weight_min": -40.10074996948242,
      "activations/layer15_attention_weight_max": 42.31356430053711,
      "activations/layer15_attention_weight_min": -37.25628662109375,
      "activations/layer16_attention_weight_max": 30.45557403564453,
      "activations/layer16_attention_weight_min": -29.169692993164062,
      "activations/layer17_attention_weight_max": 29.79741096496582,
      "activations/layer17_attention_weight_min": -26.49468994140625,
      "activations/layer18_attention_weight_max": 32.29655838012695,
      "activations/layer18_attention_weight_min": -27.643938064575195,
      "activations/layer19_attention_weight_max": 33.4118766784668,
      "activations/layer19_attention_weight_min": -31.609832763671875,
      "activations/layer1_attention_weight_max": 15.249832153320312,
      "activations/layer1_attention_weight_min": -14.455451011657715,
      "activations/layer20_attention_weight_max": 31.67621421813965,
      "activations/layer20_attention_weight_min": -26.136507034301758,
      "activations/layer21_attention_weight_max": 32.894142150878906,
      "activations/layer21_attention_weight_min": -25.07274627685547,
      "activations/layer22_attention_weight_max": 41.29660415649414,
      "activations/layer22_attention_weight_min": -32.13583755493164,
      "activations/layer23_attention_weight_max": 32.0500373840332,
      "activations/layer23_attention_weight_min": -25.762144088745117,
      "activations/layer2_attention_weight_max": 32.56792449951172,
      "activations/layer2_attention_weight_min": -34.364646911621094,
      "activations/layer3_attention_weight_max": 104.15375518798828,
      "activations/layer3_attention_weight_min": -106.28279876708984,
      "activations/layer4_attention_weight_max": 108.7252197265625,
      "activations/layer4_attention_weight_min": -110.40669250488281,
      "activations/layer5_attention_weight_max": 50.0501708984375,
      "activations/layer5_attention_weight_min": -62.06913375854492,
      "activations/layer6_attention_weight_max": 43.90285110473633,
      "activations/layer6_attention_weight_min": -47.22284698486328,
      "activations/layer7_attention_weight_max": 101.15586853027344,
      "activations/layer7_attention_weight_min": -103.77240753173828,
      "activations/layer8_attention_weight_max": 43.62578582763672,
      "activations/layer8_attention_weight_min": -44.906715393066406,
      "activations/layer9_attention_weight_max": 36.205589294433594,
      "activations/layer9_attention_weight_min": -41.29313659667969,
      "epoch": 22.0,
      "learning_rate": 8.19810606060606e-06,
      "loss": 2.7492,
      "step": 378650
    },
    {
      "activations/layer0_attention_weight_max": 16.873619079589844,
      "activations/layer0_attention_weight_min": -15.296318054199219,
      "activations/layer10_attention_weight_max": 36.16472244262695,
      "activations/layer10_attention_weight_min": -34.72517395019531,
      "activations/layer11_attention_weight_max": 38.4261474609375,
      "activations/layer11_attention_weight_min": -35.377689361572266,
      "activations/layer12_attention_weight_max": 25.984577178955078,
      "activations/layer12_attention_weight_min": -26.608102798461914,
      "activations/layer13_attention_weight_max": 38.75600051879883,
      "activations/layer13_attention_weight_min": -37.57960510253906,
      "activations/layer14_attention_weight_max": 42.629966735839844,
      "activations/layer14_attention_weight_min": -39.19029998779297,
      "activations/layer15_attention_weight_max": 38.547950744628906,
      "activations/layer15_attention_weight_min": -38.57902145385742,
      "activations/layer16_attention_weight_max": 27.51120948791504,
      "activations/layer16_attention_weight_min": -27.81633758544922,
      "activations/layer17_attention_weight_max": 28.76413917541504,
      "activations/layer17_attention_weight_min": -25.48278045654297,
      "activations/layer18_attention_weight_max": 31.224882125854492,
      "activations/layer18_attention_weight_min": -25.65338706970215,
      "activations/layer19_attention_weight_max": 32.10386657714844,
      "activations/layer19_attention_weight_min": -33.81624221801758,
      "activations/layer1_attention_weight_max": 16.675640106201172,
      "activations/layer1_attention_weight_min": -14.667824745178223,
      "activations/layer20_attention_weight_max": 27.30499839782715,
      "activations/layer20_attention_weight_min": -25.8443603515625,
      "activations/layer21_attention_weight_max": 25.750164031982422,
      "activations/layer21_attention_weight_min": -25.239316940307617,
      "activations/layer22_attention_weight_max": 38.8400764465332,
      "activations/layer22_attention_weight_min": -32.903228759765625,
      "activations/layer23_attention_weight_max": 28.778766632080078,
      "activations/layer23_attention_weight_min": -25.540523529052734,
      "activations/layer2_attention_weight_max": 32.428470611572266,
      "activations/layer2_attention_weight_min": -32.400840759277344,
      "activations/layer3_attention_weight_max": 105.82833099365234,
      "activations/layer3_attention_weight_min": -107.94744110107422,
      "activations/layer4_attention_weight_max": 114.36396789550781,
      "activations/layer4_attention_weight_min": -115.34619140625,
      "activations/layer5_attention_weight_max": 52.48537826538086,
      "activations/layer5_attention_weight_min": -65.9974594116211,
      "activations/layer6_attention_weight_max": 48.02505874633789,
      "activations/layer6_attention_weight_min": -53.34514236450195,
      "activations/layer7_attention_weight_max": 100.0365219116211,
      "activations/layer7_attention_weight_min": -108.89401245117188,
      "activations/layer8_attention_weight_max": 45.35038757324219,
      "activations/layer8_attention_weight_min": -46.53678512573242,
      "activations/layer9_attention_weight_max": 31.907804489135742,
      "activations/layer9_attention_weight_min": -35.95982360839844,
      "epoch": 22.0,
      "learning_rate": 8.179166666666666e-06,
      "loss": 2.7081,
      "step": 378700
    },
    {
      "activations/layer0_attention_weight_max": 16.34813690185547,
      "activations/layer0_attention_weight_min": -14.467971801757812,
      "activations/layer10_attention_weight_max": 37.311180114746094,
      "activations/layer10_attention_weight_min": -37.57996368408203,
      "activations/layer11_attention_weight_max": 38.14006423950195,
      "activations/layer11_attention_weight_min": -37.12478256225586,
      "activations/layer12_attention_weight_max": 29.427921295166016,
      "activations/layer12_attention_weight_min": -27.634679794311523,
      "activations/layer13_attention_weight_max": 42.84611129760742,
      "activations/layer13_attention_weight_min": -37.76485061645508,
      "activations/layer14_attention_weight_max": 46.46088409423828,
      "activations/layer14_attention_weight_min": -38.70577621459961,
      "activations/layer15_attention_weight_max": 44.55531692504883,
      "activations/layer15_attention_weight_min": -38.583412170410156,
      "activations/layer16_attention_weight_max": 31.150972366333008,
      "activations/layer16_attention_weight_min": -29.099397659301758,
      "activations/layer17_attention_weight_max": 32.51455307006836,
      "activations/layer17_attention_weight_min": -26.469093322753906,
      "activations/layer18_attention_weight_max": 33.81928634643555,
      "activations/layer18_attention_weight_min": -28.592918395996094,
      "activations/layer19_attention_weight_max": 36.34971237182617,
      "activations/layer19_attention_weight_min": -31.14338493347168,
      "activations/layer1_attention_weight_max": 17.20391845703125,
      "activations/layer1_attention_weight_min": -15.847801208496094,
      "activations/layer20_attention_weight_max": 29.75313949584961,
      "activations/layer20_attention_weight_min": -25.921175003051758,
      "activations/layer21_attention_weight_max": 29.29745864868164,
      "activations/layer21_attention_weight_min": -25.45897674560547,
      "activations/layer22_attention_weight_max": 46.9539680480957,
      "activations/layer22_attention_weight_min": -31.64896011352539,
      "activations/layer23_attention_weight_max": 34.39678955078125,
      "activations/layer23_attention_weight_min": -25.68368148803711,
      "activations/layer2_attention_weight_max": 34.907413482666016,
      "activations/layer2_attention_weight_min": -34.71512985229492,
      "activations/layer3_attention_weight_max": 111.50816345214844,
      "activations/layer3_attention_weight_min": -109.61666107177734,
      "activations/layer4_attention_weight_max": 112.67938232421875,
      "activations/layer4_attention_weight_min": -113.319580078125,
      "activations/layer5_attention_weight_max": 56.28618621826172,
      "activations/layer5_attention_weight_min": -61.722862243652344,
      "activations/layer6_attention_weight_max": 46.58456802368164,
      "activations/layer6_attention_weight_min": -52.78514099121094,
      "activations/layer7_attention_weight_max": 101.25885009765625,
      "activations/layer7_attention_weight_min": -108.34367370605469,
      "activations/layer8_attention_weight_max": 46.881568908691406,
      "activations/layer8_attention_weight_min": -49.27272415161133,
      "activations/layer9_attention_weight_max": 36.14960861206055,
      "activations/layer9_attention_weight_min": -38.66300964355469,
      "epoch": 22.01,
      "learning_rate": 8.160227272727273e-06,
      "loss": 2.6906,
      "step": 378750
    },
    {
      "activations/layer0_attention_weight_max": 17.166118621826172,
      "activations/layer0_attention_weight_min": -13.754945755004883,
      "activations/layer10_attention_weight_max": 33.73317337036133,
      "activations/layer10_attention_weight_min": -34.05250930786133,
      "activations/layer11_attention_weight_max": 32.819339752197266,
      "activations/layer11_attention_weight_min": -35.113914489746094,
      "activations/layer12_attention_weight_max": 28.303115844726562,
      "activations/layer12_attention_weight_min": -27.034238815307617,
      "activations/layer13_attention_weight_max": 40.66939163208008,
      "activations/layer13_attention_weight_min": -35.091678619384766,
      "activations/layer14_attention_weight_max": 44.67643737792969,
      "activations/layer14_attention_weight_min": -37.76417922973633,
      "activations/layer15_attention_weight_max": 39.6462516784668,
      "activations/layer15_attention_weight_min": -36.27241516113281,
      "activations/layer16_attention_weight_max": 30.71140480041504,
      "activations/layer16_attention_weight_min": -29.17646026611328,
      "activations/layer17_attention_weight_max": 30.6605224609375,
      "activations/layer17_attention_weight_min": -28.109312057495117,
      "activations/layer18_attention_weight_max": 33.64669418334961,
      "activations/layer18_attention_weight_min": -25.31808090209961,
      "activations/layer19_attention_weight_max": 34.60313034057617,
      "activations/layer19_attention_weight_min": -33.8836555480957,
      "activations/layer1_attention_weight_max": 16.460861206054688,
      "activations/layer1_attention_weight_min": -16.825498580932617,
      "activations/layer20_attention_weight_max": 32.41727066040039,
      "activations/layer20_attention_weight_min": -26.522859573364258,
      "activations/layer21_attention_weight_max": 33.878963470458984,
      "activations/layer21_attention_weight_min": -25.933086395263672,
      "activations/layer22_attention_weight_max": 41.43235778808594,
      "activations/layer22_attention_weight_min": -30.642833709716797,
      "activations/layer23_attention_weight_max": 32.3758659362793,
      "activations/layer23_attention_weight_min": -25.092151641845703,
      "activations/layer2_attention_weight_max": 33.860130310058594,
      "activations/layer2_attention_weight_min": -33.87956237792969,
      "activations/layer3_attention_weight_max": 104.17150115966797,
      "activations/layer3_attention_weight_min": -107.8533706665039,
      "activations/layer4_attention_weight_max": 109.8209228515625,
      "activations/layer4_attention_weight_min": -118.51688385009766,
      "activations/layer5_attention_weight_max": 51.9918212890625,
      "activations/layer5_attention_weight_min": -61.84418487548828,
      "activations/layer6_attention_weight_max": 46.30081558227539,
      "activations/layer6_attention_weight_min": -49.95219421386719,
      "activations/layer7_attention_weight_max": 99.36421966552734,
      "activations/layer7_attention_weight_min": -105.61300659179688,
      "activations/layer8_attention_weight_max": 43.182186126708984,
      "activations/layer8_attention_weight_min": -48.398860931396484,
      "activations/layer9_attention_weight_max": 33.063594818115234,
      "activations/layer9_attention_weight_min": -35.61896896362305,
      "epoch": 22.01,
      "learning_rate": 8.141287878787878e-06,
      "loss": 2.7089,
      "step": 378800
    },
    {
      "activations/layer0_attention_weight_max": 17.525596618652344,
      "activations/layer0_attention_weight_min": -14.035868644714355,
      "activations/layer10_attention_weight_max": 40.33281326293945,
      "activations/layer10_attention_weight_min": -39.33313751220703,
      "activations/layer11_attention_weight_max": 39.24494934082031,
      "activations/layer11_attention_weight_min": -39.164947509765625,
      "activations/layer12_attention_weight_max": 30.621458053588867,
      "activations/layer12_attention_weight_min": -30.225753784179688,
      "activations/layer13_attention_weight_max": 42.68532180786133,
      "activations/layer13_attention_weight_min": -40.69346237182617,
      "activations/layer14_attention_weight_max": 47.66843795776367,
      "activations/layer14_attention_weight_min": -40.85563659667969,
      "activations/layer15_attention_weight_max": 43.41392517089844,
      "activations/layer15_attention_weight_min": -37.406192779541016,
      "activations/layer16_attention_weight_max": 29.66301727294922,
      "activations/layer16_attention_weight_min": -27.703189849853516,
      "activations/layer17_attention_weight_max": 33.874454498291016,
      "activations/layer17_attention_weight_min": -25.820947647094727,
      "activations/layer18_attention_weight_max": 32.057315826416016,
      "activations/layer18_attention_weight_min": -27.027498245239258,
      "activations/layer19_attention_weight_max": 34.03207015991211,
      "activations/layer19_attention_weight_min": -33.11924362182617,
      "activations/layer1_attention_weight_max": 16.776777267456055,
      "activations/layer1_attention_weight_min": -16.37161636352539,
      "activations/layer20_attention_weight_max": 28.539011001586914,
      "activations/layer20_attention_weight_min": -25.337356567382812,
      "activations/layer21_attention_weight_max": 29.193296432495117,
      "activations/layer21_attention_weight_min": -25.77956199645996,
      "activations/layer22_attention_weight_max": 44.74897766113281,
      "activations/layer22_attention_weight_min": -34.17142868041992,
      "activations/layer23_attention_weight_max": 34.33390426635742,
      "activations/layer23_attention_weight_min": -26.368574142456055,
      "activations/layer2_attention_weight_max": 36.78118896484375,
      "activations/layer2_attention_weight_min": -36.5394287109375,
      "activations/layer3_attention_weight_max": 112.6938705444336,
      "activations/layer3_attention_weight_min": -119.0579605102539,
      "activations/layer4_attention_weight_max": 117.0863037109375,
      "activations/layer4_attention_weight_min": -120.6180419921875,
      "activations/layer5_attention_weight_max": 53.203914642333984,
      "activations/layer5_attention_weight_min": -61.37238311767578,
      "activations/layer6_attention_weight_max": 48.60932159423828,
      "activations/layer6_attention_weight_min": -49.85441207885742,
      "activations/layer7_attention_weight_max": 99.55319213867188,
      "activations/layer7_attention_weight_min": -108.03805541992188,
      "activations/layer8_attention_weight_max": 46.99997329711914,
      "activations/layer8_attention_weight_min": -48.06296157836914,
      "activations/layer9_attention_weight_max": 35.753021240234375,
      "activations/layer9_attention_weight_min": -37.76144790649414,
      "epoch": 22.01,
      "learning_rate": 8.122348484848484e-06,
      "loss": 2.7069,
      "step": 378850
    },
    {
      "activations/layer0_attention_weight_max": 17.021665573120117,
      "activations/layer0_attention_weight_min": -15.827293395996094,
      "activations/layer10_attention_weight_max": 37.37613296508789,
      "activations/layer10_attention_weight_min": -40.01151657104492,
      "activations/layer11_attention_weight_max": 37.4947624206543,
      "activations/layer11_attention_weight_min": -38.638187408447266,
      "activations/layer12_attention_weight_max": 29.078731536865234,
      "activations/layer12_attention_weight_min": -27.48932647705078,
      "activations/layer13_attention_weight_max": 40.43301010131836,
      "activations/layer13_attention_weight_min": -34.77580261230469,
      "activations/layer14_attention_weight_max": 43.786991119384766,
      "activations/layer14_attention_weight_min": -38.61687088012695,
      "activations/layer15_attention_weight_max": 39.63496398925781,
      "activations/layer15_attention_weight_min": -36.30769729614258,
      "activations/layer16_attention_weight_max": 28.8443546295166,
      "activations/layer16_attention_weight_min": -28.21959114074707,
      "activations/layer17_attention_weight_max": 30.41912269592285,
      "activations/layer17_attention_weight_min": -25.734386444091797,
      "activations/layer18_attention_weight_max": 30.713623046875,
      "activations/layer18_attention_weight_min": -24.824464797973633,
      "activations/layer19_attention_weight_max": 34.627017974853516,
      "activations/layer19_attention_weight_min": -30.399860382080078,
      "activations/layer1_attention_weight_max": 15.447208404541016,
      "activations/layer1_attention_weight_min": -14.71152400970459,
      "activations/layer20_attention_weight_max": 28.072731018066406,
      "activations/layer20_attention_weight_min": -25.325105667114258,
      "activations/layer21_attention_weight_max": 28.756093978881836,
      "activations/layer21_attention_weight_min": -24.9560489654541,
      "activations/layer22_attention_weight_max": 42.65860366821289,
      "activations/layer22_attention_weight_min": -32.950340270996094,
      "activations/layer23_attention_weight_max": 31.924163818359375,
      "activations/layer23_attention_weight_min": -25.517959594726562,
      "activations/layer2_attention_weight_max": 34.046112060546875,
      "activations/layer2_attention_weight_min": -33.61199951171875,
      "activations/layer3_attention_weight_max": 104.54721069335938,
      "activations/layer3_attention_weight_min": -107.57801818847656,
      "activations/layer4_attention_weight_max": 108.08784484863281,
      "activations/layer4_attention_weight_min": -111.74507904052734,
      "activations/layer5_attention_weight_max": 49.89472579956055,
      "activations/layer5_attention_weight_min": -62.033905029296875,
      "activations/layer6_attention_weight_max": 46.05793380737305,
      "activations/layer6_attention_weight_min": -47.8652229309082,
      "activations/layer7_attention_weight_max": 101.24615478515625,
      "activations/layer7_attention_weight_min": -107.7807846069336,
      "activations/layer8_attention_weight_max": 46.64588165283203,
      "activations/layer8_attention_weight_min": -50.94763946533203,
      "activations/layer9_attention_weight_max": 35.290828704833984,
      "activations/layer9_attention_weight_min": -38.531124114990234,
      "epoch": 22.02,
      "learning_rate": 8.10340909090909e-06,
      "loss": 2.715,
      "step": 378900
    },
    {
      "activations/layer0_attention_weight_max": 17.486974716186523,
      "activations/layer0_attention_weight_min": -15.399093627929688,
      "activations/layer10_attention_weight_max": 34.88893127441406,
      "activations/layer10_attention_weight_min": -34.27388000488281,
      "activations/layer11_attention_weight_max": 33.4014892578125,
      "activations/layer11_attention_weight_min": -34.500755310058594,
      "activations/layer12_attention_weight_max": 28.56792449951172,
      "activations/layer12_attention_weight_min": -27.297550201416016,
      "activations/layer13_attention_weight_max": 40.86131286621094,
      "activations/layer13_attention_weight_min": -35.52168273925781,
      "activations/layer14_attention_weight_max": 42.37233352661133,
      "activations/layer14_attention_weight_min": -38.9570198059082,
      "activations/layer15_attention_weight_max": 38.68363571166992,
      "activations/layer15_attention_weight_min": -37.268680572509766,
      "activations/layer16_attention_weight_max": 29.031797409057617,
      "activations/layer16_attention_weight_min": -28.034135818481445,
      "activations/layer17_attention_weight_max": 29.800256729125977,
      "activations/layer17_attention_weight_min": -26.156038284301758,
      "activations/layer18_attention_weight_max": 34.26122283935547,
      "activations/layer18_attention_weight_min": -26.80890464782715,
      "activations/layer19_attention_weight_max": 36.5215950012207,
      "activations/layer19_attention_weight_min": -31.94854164123535,
      "activations/layer1_attention_weight_max": 15.92402172088623,
      "activations/layer1_attention_weight_min": -15.749009132385254,
      "activations/layer20_attention_weight_max": 32.66305923461914,
      "activations/layer20_attention_weight_min": -27.289018630981445,
      "activations/layer21_attention_weight_max": 33.09363555908203,
      "activations/layer21_attention_weight_min": -26.34263038635254,
      "activations/layer22_attention_weight_max": 48.19477081298828,
      "activations/layer22_attention_weight_min": -31.948123931884766,
      "activations/layer23_attention_weight_max": 34.21247100830078,
      "activations/layer23_attention_weight_min": -24.987272262573242,
      "activations/layer2_attention_weight_max": 33.46735763549805,
      "activations/layer2_attention_weight_min": -34.616172790527344,
      "activations/layer3_attention_weight_max": 109.32527160644531,
      "activations/layer3_attention_weight_min": -110.70793151855469,
      "activations/layer4_attention_weight_max": 114.34520721435547,
      "activations/layer4_attention_weight_min": -114.34679412841797,
      "activations/layer5_attention_weight_max": 52.813514709472656,
      "activations/layer5_attention_weight_min": -63.453826904296875,
      "activations/layer6_attention_weight_max": 49.97340393066406,
      "activations/layer6_attention_weight_min": -52.32038879394531,
      "activations/layer7_attention_weight_max": 98.56768798828125,
      "activations/layer7_attention_weight_min": -110.60166931152344,
      "activations/layer8_attention_weight_max": 44.17249298095703,
      "activations/layer8_attention_weight_min": -46.90625762939453,
      "activations/layer9_attention_weight_max": 32.638797760009766,
      "activations/layer9_attention_weight_min": -34.500545501708984,
      "epoch": 22.02,
      "learning_rate": 8.084469696969696e-06,
      "loss": 2.691,
      "step": 378950
    },
    {
      "activations/layer0_attention_weight_max": 16.488052368164062,
      "activations/layer0_attention_weight_min": -15.174760818481445,
      "activations/layer10_attention_weight_max": 34.33525085449219,
      "activations/layer10_attention_weight_min": -37.257171630859375,
      "activations/layer11_attention_weight_max": 35.59274673461914,
      "activations/layer11_attention_weight_min": -39.22257614135742,
      "activations/layer12_attention_weight_max": 28.21946907043457,
      "activations/layer12_attention_weight_min": -27.391902923583984,
      "activations/layer13_attention_weight_max": 40.63874816894531,
      "activations/layer13_attention_weight_min": -36.88309860229492,
      "activations/layer14_attention_weight_max": 45.214988708496094,
      "activations/layer14_attention_weight_min": -39.37361526489258,
      "activations/layer15_attention_weight_max": 40.084651947021484,
      "activations/layer15_attention_weight_min": -37.17104721069336,
      "activations/layer16_attention_weight_max": 27.51127052307129,
      "activations/layer16_attention_weight_min": -29.5851993560791,
      "activations/layer17_attention_weight_max": 27.51473617553711,
      "activations/layer17_attention_weight_min": -25.999622344970703,
      "activations/layer18_attention_weight_max": 30.229469299316406,
      "activations/layer18_attention_weight_min": -25.579063415527344,
      "activations/layer19_attention_weight_max": 31.859594345092773,
      "activations/layer19_attention_weight_min": -31.070220947265625,
      "activations/layer1_attention_weight_max": 16.207609176635742,
      "activations/layer1_attention_weight_min": -16.731595993041992,
      "activations/layer20_attention_weight_max": 27.10565185546875,
      "activations/layer20_attention_weight_min": -23.934715270996094,
      "activations/layer21_attention_weight_max": 25.511306762695312,
      "activations/layer21_attention_weight_min": -25.387296676635742,
      "activations/layer22_attention_weight_max": 36.805416107177734,
      "activations/layer22_attention_weight_min": -31.06341552734375,
      "activations/layer23_attention_weight_max": 28.137386322021484,
      "activations/layer23_attention_weight_min": -25.17989158630371,
      "activations/layer2_attention_weight_max": 33.215087890625,
      "activations/layer2_attention_weight_min": -34.67842483520508,
      "activations/layer3_attention_weight_max": 104.19304656982422,
      "activations/layer3_attention_weight_min": -109.33328247070312,
      "activations/layer4_attention_weight_max": 110.56417083740234,
      "activations/layer4_attention_weight_min": -112.57020568847656,
      "activations/layer5_attention_weight_max": 51.84992218017578,
      "activations/layer5_attention_weight_min": -61.67306137084961,
      "activations/layer6_attention_weight_max": 45.60405731201172,
      "activations/layer6_attention_weight_min": -49.55392837524414,
      "activations/layer7_attention_weight_max": 100.0693130493164,
      "activations/layer7_attention_weight_min": -107.00807189941406,
      "activations/layer8_attention_weight_max": 45.939064025878906,
      "activations/layer8_attention_weight_min": -47.251338958740234,
      "activations/layer9_attention_weight_max": 34.15538787841797,
      "activations/layer9_attention_weight_min": -36.120521545410156,
      "epoch": 22.02,
      "learning_rate": 8.065530303030302e-06,
      "loss": 2.6984,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.5632,
      "eval_samples_per_second": 501.448,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.5632,
      "eval_openwebtext_samples_per_second": 501.448,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.962,
      "eval_wikitext_samples_per_second": 232.415,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_lambada_loss": 2.375,
      "eval_lambada_ppl": 10.751013186076355,
      "eval_lambada_runtime": 9.5974,
      "eval_lambada_samples_per_second": 507.323,
      "step": 379000
    },
    {
      "activations/layer0_attention_weight_max": 17.670007705688477,
      "activations/layer0_attention_weight_min": -14.011149406433105,
      "activations/layer10_attention_weight_max": 39.0006217956543,
      "activations/layer10_attention_weight_min": -38.59033966064453,
      "activations/layer11_attention_weight_max": 37.518898010253906,
      "activations/layer11_attention_weight_min": -36.98107147216797,
      "activations/layer12_attention_weight_max": 28.66585922241211,
      "activations/layer12_attention_weight_min": -27.59186553955078,
      "activations/layer13_attention_weight_max": 42.391380310058594,
      "activations/layer13_attention_weight_min": -36.86567687988281,
      "activations/layer14_attention_weight_max": 47.44294738769531,
      "activations/layer14_attention_weight_min": -40.402217864990234,
      "activations/layer15_attention_weight_max": 43.2716178894043,
      "activations/layer15_attention_weight_min": -38.94569396972656,
      "activations/layer16_attention_weight_max": 33.590087890625,
      "activations/layer16_attention_weight_min": -29.304187774658203,
      "activations/layer17_attention_weight_max": 31.375810623168945,
      "activations/layer17_attention_weight_min": -26.32546615600586,
      "activations/layer18_attention_weight_max": 38.02324676513672,
      "activations/layer18_attention_weight_min": -28.933805465698242,
      "activations/layer19_attention_weight_max": 36.290096282958984,
      "activations/layer19_attention_weight_min": -31.739042282104492,
      "activations/layer1_attention_weight_max": 16.341655731201172,
      "activations/layer1_attention_weight_min": -15.650542259216309,
      "activations/layer20_attention_weight_max": 30.09377670288086,
      "activations/layer20_attention_weight_min": -24.772361755371094,
      "activations/layer21_attention_weight_max": 30.74005126953125,
      "activations/layer21_attention_weight_min": -24.288597106933594,
      "activations/layer22_attention_weight_max": 45.60206604003906,
      "activations/layer22_attention_weight_min": -35.18962478637695,
      "activations/layer23_attention_weight_max": 37.688751220703125,
      "activations/layer23_attention_weight_min": -27.071121215820312,
      "activations/layer2_attention_weight_max": 34.25605010986328,
      "activations/layer2_attention_weight_min": -32.44233703613281,
      "activations/layer3_attention_weight_max": 109.29338836669922,
      "activations/layer3_attention_weight_min": -106.87821960449219,
      "activations/layer4_attention_weight_max": 108.966552734375,
      "activations/layer4_attention_weight_min": -110.24659729003906,
      "activations/layer5_attention_weight_max": 49.60171127319336,
      "activations/layer5_attention_weight_min": -65.88487243652344,
      "activations/layer6_attention_weight_max": 49.30134201049805,
      "activations/layer6_attention_weight_min": -50.73268508911133,
      "activations/layer7_attention_weight_max": 99.9781494140625,
      "activations/layer7_attention_weight_min": -102.56781768798828,
      "activations/layer8_attention_weight_max": 49.12078857421875,
      "activations/layer8_attention_weight_min": -50.13871765136719,
      "activations/layer9_attention_weight_max": 36.26801681518555,
      "activations/layer9_attention_weight_min": -37.33467483520508,
      "epoch": 22.02,
      "learning_rate": 8.046590909090909e-06,
      "loss": 2.7026,
      "step": 379050
    },
    {
      "activations/layer0_attention_weight_max": 17.032564163208008,
      "activations/layer0_attention_weight_min": -14.3286714553833,
      "activations/layer10_attention_weight_max": 36.84175491333008,
      "activations/layer10_attention_weight_min": -38.52035903930664,
      "activations/layer11_attention_weight_max": 36.92273712158203,
      "activations/layer11_attention_weight_min": -35.924827575683594,
      "activations/layer12_attention_weight_max": 27.73422622680664,
      "activations/layer12_attention_weight_min": -27.05243682861328,
      "activations/layer13_attention_weight_max": 41.061912536621094,
      "activations/layer13_attention_weight_min": -34.6987419128418,
      "activations/layer14_attention_weight_max": 41.42594528198242,
      "activations/layer14_attention_weight_min": -37.36448287963867,
      "activations/layer15_attention_weight_max": 38.929691314697266,
      "activations/layer15_attention_weight_min": -36.1183967590332,
      "activations/layer16_attention_weight_max": 27.912412643432617,
      "activations/layer16_attention_weight_min": -28.22064781188965,
      "activations/layer17_attention_weight_max": 28.940046310424805,
      "activations/layer17_attention_weight_min": -25.917436599731445,
      "activations/layer18_attention_weight_max": 32.71190643310547,
      "activations/layer18_attention_weight_min": -27.336421966552734,
      "activations/layer19_attention_weight_max": 41.0075569152832,
      "activations/layer19_attention_weight_min": -33.412933349609375,
      "activations/layer1_attention_weight_max": 16.227563858032227,
      "activations/layer1_attention_weight_min": -15.257978439331055,
      "activations/layer20_attention_weight_max": 33.04538345336914,
      "activations/layer20_attention_weight_min": -26.17314338684082,
      "activations/layer21_attention_weight_max": 29.177310943603516,
      "activations/layer21_attention_weight_min": -26.5495662689209,
      "activations/layer22_attention_weight_max": 45.734291076660156,
      "activations/layer22_attention_weight_min": -36.01765441894531,
      "activations/layer23_attention_weight_max": 36.25238800048828,
      "activations/layer23_attention_weight_min": -30.45941162109375,
      "activations/layer2_attention_weight_max": 34.58820343017578,
      "activations/layer2_attention_weight_min": -33.950889587402344,
      "activations/layer3_attention_weight_max": 111.7608871459961,
      "activations/layer3_attention_weight_min": -112.94994354248047,
      "activations/layer4_attention_weight_max": 112.18757629394531,
      "activations/layer4_attention_weight_min": -111.69315338134766,
      "activations/layer5_attention_weight_max": 50.15113830566406,
      "activations/layer5_attention_weight_min": -58.229854583740234,
      "activations/layer6_attention_weight_max": 47.682071685791016,
      "activations/layer6_attention_weight_min": -50.875980377197266,
      "activations/layer7_attention_weight_max": 98.51939392089844,
      "activations/layer7_attention_weight_min": -105.58265686035156,
      "activations/layer8_attention_weight_max": 44.68686294555664,
      "activations/layer8_attention_weight_min": -49.71271896362305,
      "activations/layer9_attention_weight_max": 34.14266586303711,
      "activations/layer9_attention_weight_min": -36.78959655761719,
      "epoch": 22.03,
      "learning_rate": 8.027651515151515e-06,
      "loss": 2.6934,
      "step": 379100
    },
    {
      "activations/layer0_attention_weight_max": 17.22026824951172,
      "activations/layer0_attention_weight_min": -15.141153335571289,
      "activations/layer10_attention_weight_max": 39.1340217590332,
      "activations/layer10_attention_weight_min": -36.21724319458008,
      "activations/layer11_attention_weight_max": 37.217830657958984,
      "activations/layer11_attention_weight_min": -35.2473258972168,
      "activations/layer12_attention_weight_max": 29.388208389282227,
      "activations/layer12_attention_weight_min": -27.851411819458008,
      "activations/layer13_attention_weight_max": 40.85908508300781,
      "activations/layer13_attention_weight_min": -38.30841827392578,
      "activations/layer14_attention_weight_max": 42.95734405517578,
      "activations/layer14_attention_weight_min": -42.1266975402832,
      "activations/layer15_attention_weight_max": 38.59419631958008,
      "activations/layer15_attention_weight_min": -41.1853141784668,
      "activations/layer16_attention_weight_max": 27.999374389648438,
      "activations/layer16_attention_weight_min": -31.116682052612305,
      "activations/layer17_attention_weight_max": 29.223562240600586,
      "activations/layer17_attention_weight_min": -26.495132446289062,
      "activations/layer18_attention_weight_max": 36.220069885253906,
      "activations/layer18_attention_weight_min": -27.70966911315918,
      "activations/layer19_attention_weight_max": 35.783390045166016,
      "activations/layer19_attention_weight_min": -32.79230880737305,
      "activations/layer1_attention_weight_max": 16.02310562133789,
      "activations/layer1_attention_weight_min": -14.734567642211914,
      "activations/layer20_attention_weight_max": 29.370615005493164,
      "activations/layer20_attention_weight_min": -24.07634735107422,
      "activations/layer21_attention_weight_max": 29.268138885498047,
      "activations/layer21_attention_weight_min": -23.763174057006836,
      "activations/layer22_attention_weight_max": 41.38982009887695,
      "activations/layer22_attention_weight_min": -32.06019592285156,
      "activations/layer23_attention_weight_max": 34.58991241455078,
      "activations/layer23_attention_weight_min": -26.47242546081543,
      "activations/layer2_attention_weight_max": 33.60663604736328,
      "activations/layer2_attention_weight_min": -33.157127380371094,
      "activations/layer3_attention_weight_max": 102.6631851196289,
      "activations/layer3_attention_weight_min": -110.06517028808594,
      "activations/layer4_attention_weight_max": 108.27286529541016,
      "activations/layer4_attention_weight_min": -114.25279998779297,
      "activations/layer5_attention_weight_max": 48.282081604003906,
      "activations/layer5_attention_weight_min": -63.192474365234375,
      "activations/layer6_attention_weight_max": 44.123165130615234,
      "activations/layer6_attention_weight_min": -49.06453323364258,
      "activations/layer7_attention_weight_max": 92.60909271240234,
      "activations/layer7_attention_weight_min": -100.03763580322266,
      "activations/layer8_attention_weight_max": 46.39813232421875,
      "activations/layer8_attention_weight_min": -44.836124420166016,
      "activations/layer9_attention_weight_max": 33.78004837036133,
      "activations/layer9_attention_weight_min": -36.87482452392578,
      "epoch": 22.03,
      "learning_rate": 8.00871212121212e-06,
      "loss": 2.7184,
      "step": 379150
    },
    {
      "activations/layer0_attention_weight_max": 16.75250244140625,
      "activations/layer0_attention_weight_min": -14.467204093933105,
      "activations/layer10_attention_weight_max": 37.71678924560547,
      "activations/layer10_attention_weight_min": -36.825172424316406,
      "activations/layer11_attention_weight_max": 37.15843963623047,
      "activations/layer11_attention_weight_min": -37.23887634277344,
      "activations/layer12_attention_weight_max": 30.477554321289062,
      "activations/layer12_attention_weight_min": -29.326644897460938,
      "activations/layer13_attention_weight_max": 45.09223175048828,
      "activations/layer13_attention_weight_min": -40.33171081542969,
      "activations/layer14_attention_weight_max": 49.501747131347656,
      "activations/layer14_attention_weight_min": -42.94408416748047,
      "activations/layer15_attention_weight_max": 43.85322189331055,
      "activations/layer15_attention_weight_min": -41.132080078125,
      "activations/layer16_attention_weight_max": 33.72897720336914,
      "activations/layer16_attention_weight_min": -29.970361709594727,
      "activations/layer17_attention_weight_max": 33.235931396484375,
      "activations/layer17_attention_weight_min": -27.90275764465332,
      "activations/layer18_attention_weight_max": 36.31037521362305,
      "activations/layer18_attention_weight_min": -28.23784065246582,
      "activations/layer19_attention_weight_max": 42.846527099609375,
      "activations/layer19_attention_weight_min": -33.45789337158203,
      "activations/layer1_attention_weight_max": 17.002548217773438,
      "activations/layer1_attention_weight_min": -15.573346138000488,
      "activations/layer20_attention_weight_max": 34.9595832824707,
      "activations/layer20_attention_weight_min": -26.524267196655273,
      "activations/layer21_attention_weight_max": 34.671112060546875,
      "activations/layer21_attention_weight_min": -25.606504440307617,
      "activations/layer22_attention_weight_max": 49.49995803833008,
      "activations/layer22_attention_weight_min": -33.90913391113281,
      "activations/layer23_attention_weight_max": 36.7174072265625,
      "activations/layer23_attention_weight_min": -27.489219665527344,
      "activations/layer2_attention_weight_max": 35.410072326660156,
      "activations/layer2_attention_weight_min": -32.78194808959961,
      "activations/layer3_attention_weight_max": 110.28829193115234,
      "activations/layer3_attention_weight_min": -109.2525405883789,
      "activations/layer4_attention_weight_max": 111.06279754638672,
      "activations/layer4_attention_weight_min": -115.14161682128906,
      "activations/layer5_attention_weight_max": 54.407684326171875,
      "activations/layer5_attention_weight_min": -65.94479370117188,
      "activations/layer6_attention_weight_max": 46.95341110229492,
      "activations/layer6_attention_weight_min": -50.9184455871582,
      "activations/layer7_attention_weight_max": 105.8044662475586,
      "activations/layer7_attention_weight_min": -103.38485717773438,
      "activations/layer8_attention_weight_max": 44.87300109863281,
      "activations/layer8_attention_weight_min": -47.075477600097656,
      "activations/layer9_attention_weight_max": 35.3037223815918,
      "activations/layer9_attention_weight_min": -35.587059020996094,
      "epoch": 22.03,
      "learning_rate": 7.989772727272727e-06,
      "loss": 2.6841,
      "step": 379200
    },
    {
      "activations/layer0_attention_weight_max": 17.091371536254883,
      "activations/layer0_attention_weight_min": -15.466667175292969,
      "activations/layer10_attention_weight_max": 35.77239990234375,
      "activations/layer10_attention_weight_min": -36.14405822753906,
      "activations/layer11_attention_weight_max": 36.02526092529297,
      "activations/layer11_attention_weight_min": -36.291526794433594,
      "activations/layer12_attention_weight_max": 26.666725158691406,
      "activations/layer12_attention_weight_min": -27.92445182800293,
      "activations/layer13_attention_weight_max": 38.580406188964844,
      "activations/layer13_attention_weight_min": -36.34022521972656,
      "activations/layer14_attention_weight_max": 42.805870056152344,
      "activations/layer14_attention_weight_min": -37.711753845214844,
      "activations/layer15_attention_weight_max": 39.43552780151367,
      "activations/layer15_attention_weight_min": -36.79285430908203,
      "activations/layer16_attention_weight_max": 28.306447982788086,
      "activations/layer16_attention_weight_min": -30.587528228759766,
      "activations/layer17_attention_weight_max": 27.987253189086914,
      "activations/layer17_attention_weight_min": -28.371795654296875,
      "activations/layer18_attention_weight_max": 32.92605972290039,
      "activations/layer18_attention_weight_min": -27.977556228637695,
      "activations/layer19_attention_weight_max": 34.27360916137695,
      "activations/layer19_attention_weight_min": -33.52902603149414,
      "activations/layer1_attention_weight_max": 16.712644577026367,
      "activations/layer1_attention_weight_min": -17.48379135131836,
      "activations/layer20_attention_weight_max": 26.18657875061035,
      "activations/layer20_attention_weight_min": -24.188566207885742,
      "activations/layer21_attention_weight_max": 27.179405212402344,
      "activations/layer21_attention_weight_min": -26.602914810180664,
      "activations/layer22_attention_weight_max": 45.46591567993164,
      "activations/layer22_attention_weight_min": -34.11540985107422,
      "activations/layer23_attention_weight_max": 29.76814842224121,
      "activations/layer23_attention_weight_min": -27.964719772338867,
      "activations/layer2_attention_weight_max": 34.48630142211914,
      "activations/layer2_attention_weight_min": -36.30366516113281,
      "activations/layer3_attention_weight_max": 108.32003784179688,
      "activations/layer3_attention_weight_min": -112.02294158935547,
      "activations/layer4_attention_weight_max": 113.15856170654297,
      "activations/layer4_attention_weight_min": -113.8671875,
      "activations/layer5_attention_weight_max": 54.89945983886719,
      "activations/layer5_attention_weight_min": -61.07044219970703,
      "activations/layer6_attention_weight_max": 45.26020812988281,
      "activations/layer6_attention_weight_min": -48.98725509643555,
      "activations/layer7_attention_weight_max": 97.20211791992188,
      "activations/layer7_attention_weight_min": -99.1556396484375,
      "activations/layer8_attention_weight_max": 42.78787612915039,
      "activations/layer8_attention_weight_min": -45.365928649902344,
      "activations/layer9_attention_weight_max": 33.05590057373047,
      "activations/layer9_attention_weight_min": -35.22746658325195,
      "epoch": 22.04,
      "learning_rate": 7.970833333333332e-06,
      "loss": 2.7032,
      "step": 379250
    },
    {
      "activations/layer0_attention_weight_max": 16.51066780090332,
      "activations/layer0_attention_weight_min": -13.895552635192871,
      "activations/layer10_attention_weight_max": 35.68610382080078,
      "activations/layer10_attention_weight_min": -37.348140716552734,
      "activations/layer11_attention_weight_max": 34.77271270751953,
      "activations/layer11_attention_weight_min": -37.11555480957031,
      "activations/layer12_attention_weight_max": 26.84408950805664,
      "activations/layer12_attention_weight_min": -27.00870704650879,
      "activations/layer13_attention_weight_max": 40.91156768798828,
      "activations/layer13_attention_weight_min": -36.279930114746094,
      "activations/layer14_attention_weight_max": 48.122676849365234,
      "activations/layer14_attention_weight_min": -41.55045700073242,
      "activations/layer15_attention_weight_max": 39.75929260253906,
      "activations/layer15_attention_weight_min": -36.95832443237305,
      "activations/layer16_attention_weight_max": 34.71713638305664,
      "activations/layer16_attention_weight_min": -28.337446212768555,
      "activations/layer17_attention_weight_max": 32.53160858154297,
      "activations/layer17_attention_weight_min": -28.17030906677246,
      "activations/layer18_attention_weight_max": 34.35009002685547,
      "activations/layer18_attention_weight_min": -27.926105499267578,
      "activations/layer19_attention_weight_max": 40.29241943359375,
      "activations/layer19_attention_weight_min": -37.59693908691406,
      "activations/layer1_attention_weight_max": 16.553184509277344,
      "activations/layer1_attention_weight_min": -15.584447860717773,
      "activations/layer20_attention_weight_max": 34.86175537109375,
      "activations/layer20_attention_weight_min": -26.544837951660156,
      "activations/layer21_attention_weight_max": 34.86273193359375,
      "activations/layer21_attention_weight_min": -24.8125,
      "activations/layer22_attention_weight_max": 51.77178192138672,
      "activations/layer22_attention_weight_min": -33.13595962524414,
      "activations/layer23_attention_weight_max": 37.61959457397461,
      "activations/layer23_attention_weight_min": -26.555089950561523,
      "activations/layer2_attention_weight_max": 35.2443733215332,
      "activations/layer2_attention_weight_min": -34.466888427734375,
      "activations/layer3_attention_weight_max": 108.6863784790039,
      "activations/layer3_attention_weight_min": -108.5220718383789,
      "activations/layer4_attention_weight_max": 116.2327880859375,
      "activations/layer4_attention_weight_min": -113.01432037353516,
      "activations/layer5_attention_weight_max": 52.257015228271484,
      "activations/layer5_attention_weight_min": -64.1128921508789,
      "activations/layer6_attention_weight_max": 46.18251419067383,
      "activations/layer6_attention_weight_min": -49.709529876708984,
      "activations/layer7_attention_weight_max": 102.82865142822266,
      "activations/layer7_attention_weight_min": -102.1551742553711,
      "activations/layer8_attention_weight_max": 44.41179275512695,
      "activations/layer8_attention_weight_min": -48.76996612548828,
      "activations/layer9_attention_weight_max": 31.963178634643555,
      "activations/layer9_attention_weight_min": -35.87989807128906,
      "epoch": 22.04,
      "learning_rate": 7.951893939393938e-06,
      "loss": 2.7168,
      "step": 379300
    },
    {
      "activations/layer0_attention_weight_max": 16.143301010131836,
      "activations/layer0_attention_weight_min": -15.345423698425293,
      "activations/layer10_attention_weight_max": 34.79814529418945,
      "activations/layer10_attention_weight_min": -35.00257110595703,
      "activations/layer11_attention_weight_max": 33.97481918334961,
      "activations/layer11_attention_weight_min": -34.61410140991211,
      "activations/layer12_attention_weight_max": 28.723390579223633,
      "activations/layer12_attention_weight_min": -28.120323181152344,
      "activations/layer13_attention_weight_max": 44.80649948120117,
      "activations/layer13_attention_weight_min": -37.35920715332031,
      "activations/layer14_attention_weight_max": 44.8531379699707,
      "activations/layer14_attention_weight_min": -40.30415725708008,
      "activations/layer15_attention_weight_max": 46.1755256652832,
      "activations/layer15_attention_weight_min": -40.9630012512207,
      "activations/layer16_attention_weight_max": 31.564273834228516,
      "activations/layer16_attention_weight_min": -30.499773025512695,
      "activations/layer17_attention_weight_max": 30.39702033996582,
      "activations/layer17_attention_weight_min": -25.36931800842285,
      "activations/layer18_attention_weight_max": 35.23798751831055,
      "activations/layer18_attention_weight_min": -25.9587459564209,
      "activations/layer19_attention_weight_max": 38.4596061706543,
      "activations/layer19_attention_weight_min": -32.59141159057617,
      "activations/layer1_attention_weight_max": 16.547761917114258,
      "activations/layer1_attention_weight_min": -16.103666305541992,
      "activations/layer20_attention_weight_max": 34.55070114135742,
      "activations/layer20_attention_weight_min": -24.08364486694336,
      "activations/layer21_attention_weight_max": 33.88838577270508,
      "activations/layer21_attention_weight_min": -25.09104347229004,
      "activations/layer22_attention_weight_max": 41.01303482055664,
      "activations/layer22_attention_weight_min": -30.958038330078125,
      "activations/layer23_attention_weight_max": 29.69781494140625,
      "activations/layer23_attention_weight_min": -25.585369110107422,
      "activations/layer2_attention_weight_max": 35.10874557495117,
      "activations/layer2_attention_weight_min": -35.000492095947266,
      "activations/layer3_attention_weight_max": 104.67494201660156,
      "activations/layer3_attention_weight_min": -108.77088928222656,
      "activations/layer4_attention_weight_max": 108.40746307373047,
      "activations/layer4_attention_weight_min": -118.45491027832031,
      "activations/layer5_attention_weight_max": 52.4591178894043,
      "activations/layer5_attention_weight_min": -61.787376403808594,
      "activations/layer6_attention_weight_max": 46.441036224365234,
      "activations/layer6_attention_weight_min": -48.709205627441406,
      "activations/layer7_attention_weight_max": 93.48957061767578,
      "activations/layer7_attention_weight_min": -100.85172271728516,
      "activations/layer8_attention_weight_max": 45.93109893798828,
      "activations/layer8_attention_weight_min": -46.25833511352539,
      "activations/layer9_attention_weight_max": 31.334836959838867,
      "activations/layer9_attention_weight_min": -33.75014877319336,
      "epoch": 22.04,
      "learning_rate": 7.932954545454545e-06,
      "loss": 2.7121,
      "step": 379350
    },
    {
      "activations/layer0_attention_weight_max": 16.884075164794922,
      "activations/layer0_attention_weight_min": -14.002153396606445,
      "activations/layer10_attention_weight_max": 34.78190231323242,
      "activations/layer10_attention_weight_min": -35.48678970336914,
      "activations/layer11_attention_weight_max": 35.112953186035156,
      "activations/layer11_attention_weight_min": -36.43553924560547,
      "activations/layer12_attention_weight_max": 25.54292106628418,
      "activations/layer12_attention_weight_min": -27.259382247924805,
      "activations/layer13_attention_weight_max": 37.82624816894531,
      "activations/layer13_attention_weight_min": -37.368125915527344,
      "activations/layer14_attention_weight_max": 42.32447052001953,
      "activations/layer14_attention_weight_min": -37.9254264831543,
      "activations/layer15_attention_weight_max": 36.59781265258789,
      "activations/layer15_attention_weight_min": -36.250545501708984,
      "activations/layer16_attention_weight_max": 28.209455490112305,
      "activations/layer16_attention_weight_min": -29.991832733154297,
      "activations/layer17_attention_weight_max": 29.5173282623291,
      "activations/layer17_attention_weight_min": -27.18651580810547,
      "activations/layer18_attention_weight_max": 33.86997985839844,
      "activations/layer18_attention_weight_min": -27.376380920410156,
      "activations/layer19_attention_weight_max": 31.051298141479492,
      "activations/layer19_attention_weight_min": -30.88673973083496,
      "activations/layer1_attention_weight_max": 17.9139404296875,
      "activations/layer1_attention_weight_min": -15.236663818359375,
      "activations/layer20_attention_weight_max": 26.369178771972656,
      "activations/layer20_attention_weight_min": -26.434524536132812,
      "activations/layer21_attention_weight_max": 25.726011276245117,
      "activations/layer21_attention_weight_min": -24.934377670288086,
      "activations/layer22_attention_weight_max": 37.32609176635742,
      "activations/layer22_attention_weight_min": -31.938188552856445,
      "activations/layer23_attention_weight_max": 29.544944763183594,
      "activations/layer23_attention_weight_min": -26.652481079101562,
      "activations/layer2_attention_weight_max": 34.295833587646484,
      "activations/layer2_attention_weight_min": -34.941951751708984,
      "activations/layer3_attention_weight_max": 106.58106231689453,
      "activations/layer3_attention_weight_min": -109.27938079833984,
      "activations/layer4_attention_weight_max": 109.48404693603516,
      "activations/layer4_attention_weight_min": -114.47314453125,
      "activations/layer5_attention_weight_max": 49.65616989135742,
      "activations/layer5_attention_weight_min": -62.14739990234375,
      "activations/layer6_attention_weight_max": 46.86460876464844,
      "activations/layer6_attention_weight_min": -49.65717697143555,
      "activations/layer7_attention_weight_max": 98.39315032958984,
      "activations/layer7_attention_weight_min": -108.27680969238281,
      "activations/layer8_attention_weight_max": 47.02638244628906,
      "activations/layer8_attention_weight_min": -49.81261444091797,
      "activations/layer9_attention_weight_max": 32.838497161865234,
      "activations/layer9_attention_weight_min": -37.48371887207031,
      "epoch": 22.05,
      "learning_rate": 7.91401515151515e-06,
      "loss": 2.6993,
      "step": 379400
    },
    {
      "activations/layer0_attention_weight_max": 17.089685440063477,
      "activations/layer0_attention_weight_min": -14.001413345336914,
      "activations/layer10_attention_weight_max": 35.25726318359375,
      "activations/layer10_attention_weight_min": -35.72140884399414,
      "activations/layer11_attention_weight_max": 34.98038864135742,
      "activations/layer11_attention_weight_min": -37.1054573059082,
      "activations/layer12_attention_weight_max": 27.67544174194336,
      "activations/layer12_attention_weight_min": -28.814685821533203,
      "activations/layer13_attention_weight_max": 38.66238021850586,
      "activations/layer13_attention_weight_min": -37.13819885253906,
      "activations/layer14_attention_weight_max": 44.12543487548828,
      "activations/layer14_attention_weight_min": -38.03085708618164,
      "activations/layer15_attention_weight_max": 39.80099868774414,
      "activations/layer15_attention_weight_min": -35.85038375854492,
      "activations/layer16_attention_weight_max": 29.929414749145508,
      "activations/layer16_attention_weight_min": -28.22714614868164,
      "activations/layer17_attention_weight_max": 30.88850212097168,
      "activations/layer17_attention_weight_min": -25.02049446105957,
      "activations/layer18_attention_weight_max": 34.49742889404297,
      "activations/layer18_attention_weight_min": -25.1561336517334,
      "activations/layer19_attention_weight_max": 32.8470458984375,
      "activations/layer19_attention_weight_min": -30.652746200561523,
      "activations/layer1_attention_weight_max": 16.633834838867188,
      "activations/layer1_attention_weight_min": -14.081201553344727,
      "activations/layer20_attention_weight_max": 30.947053909301758,
      "activations/layer20_attention_weight_min": -23.504940032958984,
      "activations/layer21_attention_weight_max": 27.094646453857422,
      "activations/layer21_attention_weight_min": -24.374204635620117,
      "activations/layer22_attention_weight_max": 40.59336471557617,
      "activations/layer22_attention_weight_min": -31.26531982421875,
      "activations/layer23_attention_weight_max": 29.202251434326172,
      "activations/layer23_attention_weight_min": -25.117658615112305,
      "activations/layer2_attention_weight_max": 33.01543426513672,
      "activations/layer2_attention_weight_min": -32.77912521362305,
      "activations/layer3_attention_weight_max": 104.77239227294922,
      "activations/layer3_attention_weight_min": -104.91973876953125,
      "activations/layer4_attention_weight_max": 109.3849868774414,
      "activations/layer4_attention_weight_min": -115.68070220947266,
      "activations/layer5_attention_weight_max": 49.703182220458984,
      "activations/layer5_attention_weight_min": -62.869163513183594,
      "activations/layer6_attention_weight_max": 44.470115661621094,
      "activations/layer6_attention_weight_min": -50.09218978881836,
      "activations/layer7_attention_weight_max": 97.07218170166016,
      "activations/layer7_attention_weight_min": -101.99015808105469,
      "activations/layer8_attention_weight_max": 42.138587951660156,
      "activations/layer8_attention_weight_min": -46.08671951293945,
      "activations/layer9_attention_weight_max": 31.604022979736328,
      "activations/layer9_attention_weight_min": -35.49641036987305,
      "epoch": 22.05,
      "learning_rate": 7.895075757575758e-06,
      "loss": 2.6984,
      "step": 379450
    },
    {
      "activations/layer0_attention_weight_max": 15.665932655334473,
      "activations/layer0_attention_weight_min": -15.387099266052246,
      "activations/layer10_attention_weight_max": 35.400672912597656,
      "activations/layer10_attention_weight_min": -36.803504943847656,
      "activations/layer11_attention_weight_max": 34.455970764160156,
      "activations/layer11_attention_weight_min": -35.67753982543945,
      "activations/layer12_attention_weight_max": 28.006654739379883,
      "activations/layer12_attention_weight_min": -27.718177795410156,
      "activations/layer13_attention_weight_max": 44.039146423339844,
      "activations/layer13_attention_weight_min": -36.422706604003906,
      "activations/layer14_attention_weight_max": 46.09455871582031,
      "activations/layer14_attention_weight_min": -38.984519958496094,
      "activations/layer15_attention_weight_max": 44.946144104003906,
      "activations/layer15_attention_weight_min": -35.47494125366211,
      "activations/layer16_attention_weight_max": 32.99263000488281,
      "activations/layer16_attention_weight_min": -26.80290412902832,
      "activations/layer17_attention_weight_max": 30.404897689819336,
      "activations/layer17_attention_weight_min": -26.46721649169922,
      "activations/layer18_attention_weight_max": 34.138282775878906,
      "activations/layer18_attention_weight_min": -26.307514190673828,
      "activations/layer19_attention_weight_max": 37.42539596557617,
      "activations/layer19_attention_weight_min": -31.34952163696289,
      "activations/layer1_attention_weight_max": 15.133646965026855,
      "activations/layer1_attention_weight_min": -14.956525802612305,
      "activations/layer20_attention_weight_max": 34.14598083496094,
      "activations/layer20_attention_weight_min": -24.517593383789062,
      "activations/layer21_attention_weight_max": 30.221939086914062,
      "activations/layer21_attention_weight_min": -23.495800018310547,
      "activations/layer22_attention_weight_max": 47.79490661621094,
      "activations/layer22_attention_weight_min": -31.047300338745117,
      "activations/layer23_attention_weight_max": 32.613285064697266,
      "activations/layer23_attention_weight_min": -26.74488639831543,
      "activations/layer2_attention_weight_max": 32.83905792236328,
      "activations/layer2_attention_weight_min": -32.8448486328125,
      "activations/layer3_attention_weight_max": 107.74819946289062,
      "activations/layer3_attention_weight_min": -107.84661865234375,
      "activations/layer4_attention_weight_max": 112.96817779541016,
      "activations/layer4_attention_weight_min": -114.90348052978516,
      "activations/layer5_attention_weight_max": 52.94268035888672,
      "activations/layer5_attention_weight_min": -62.63768005371094,
      "activations/layer6_attention_weight_max": 47.152896881103516,
      "activations/layer6_attention_weight_min": -47.35739517211914,
      "activations/layer7_attention_weight_max": 98.28955078125,
      "activations/layer7_attention_weight_min": -100.52759552001953,
      "activations/layer8_attention_weight_max": 42.673282623291016,
      "activations/layer8_attention_weight_min": -47.095829010009766,
      "activations/layer9_attention_weight_max": 34.22222900390625,
      "activations/layer9_attention_weight_min": -35.67962646484375,
      "epoch": 22.05,
      "learning_rate": 7.876136363636363e-06,
      "loss": 2.6923,
      "step": 379500
    },
    {
      "activations/layer0_attention_weight_max": 17.55166244506836,
      "activations/layer0_attention_weight_min": -13.013927459716797,
      "activations/layer10_attention_weight_max": 49.68726348876953,
      "activations/layer10_attention_weight_min": -47.688194274902344,
      "activations/layer11_attention_weight_max": 48.279876708984375,
      "activations/layer11_attention_weight_min": -46.452857971191406,
      "activations/layer12_attention_weight_max": 37.861576080322266,
      "activations/layer12_attention_weight_min": -33.524009704589844,
      "activations/layer13_attention_weight_max": 56.3351936340332,
      "activations/layer13_attention_weight_min": -47.072750091552734,
      "activations/layer14_attention_weight_max": 64.35042572021484,
      "activations/layer14_attention_weight_min": -54.913841247558594,
      "activations/layer15_attention_weight_max": 60.76643753051758,
      "activations/layer15_attention_weight_min": -55.80831527709961,
      "activations/layer16_attention_weight_max": 39.599082946777344,
      "activations/layer16_attention_weight_min": -29.990049362182617,
      "activations/layer17_attention_weight_max": 34.25191116333008,
      "activations/layer17_attention_weight_min": -32.539913177490234,
      "activations/layer18_attention_weight_max": 38.74288558959961,
      "activations/layer18_attention_weight_min": -35.8685417175293,
      "activations/layer19_attention_weight_max": 46.3272819519043,
      "activations/layer19_attention_weight_min": -36.44485855102539,
      "activations/layer1_attention_weight_max": 15.961256980895996,
      "activations/layer1_attention_weight_min": -16.091421127319336,
      "activations/layer20_attention_weight_max": 38.859405517578125,
      "activations/layer20_attention_weight_min": -28.265642166137695,
      "activations/layer21_attention_weight_max": 34.99266052246094,
      "activations/layer21_attention_weight_min": -28.818723678588867,
      "activations/layer22_attention_weight_max": 58.94550323486328,
      "activations/layer22_attention_weight_min": -37.22793960571289,
      "activations/layer23_attention_weight_max": 36.4166374206543,
      "activations/layer23_attention_weight_min": -24.42532730102539,
      "activations/layer2_attention_weight_max": 39.18556213378906,
      "activations/layer2_attention_weight_min": -35.927146911621094,
      "activations/layer3_attention_weight_max": 118.24110412597656,
      "activations/layer3_attention_weight_min": -115.79413604736328,
      "activations/layer4_attention_weight_max": 122.0240707397461,
      "activations/layer4_attention_weight_min": -117.70655822753906,
      "activations/layer5_attention_weight_max": 56.33873748779297,
      "activations/layer5_attention_weight_min": -66.2756576538086,
      "activations/layer6_attention_weight_max": 47.925331115722656,
      "activations/layer6_attention_weight_min": -50.26893997192383,
      "activations/layer7_attention_weight_max": 119.29067993164062,
      "activations/layer7_attention_weight_min": -110.95452117919922,
      "activations/layer8_attention_weight_max": 55.881202697753906,
      "activations/layer8_attention_weight_min": -54.280818939208984,
      "activations/layer9_attention_weight_max": 41.67436981201172,
      "activations/layer9_attention_weight_min": -44.07441711425781,
      "epoch": 22.05,
      "learning_rate": 7.857575757575757e-06,
      "loss": 2.7143,
      "step": 379550
    },
    {
      "activations/layer0_attention_weight_max": 17.427974700927734,
      "activations/layer0_attention_weight_min": -15.192413330078125,
      "activations/layer10_attention_weight_max": 35.08814239501953,
      "activations/layer10_attention_weight_min": -35.93609619140625,
      "activations/layer11_attention_weight_max": 36.661373138427734,
      "activations/layer11_attention_weight_min": -35.21974182128906,
      "activations/layer12_attention_weight_max": 30.291685104370117,
      "activations/layer12_attention_weight_min": -28.21497917175293,
      "activations/layer13_attention_weight_max": 44.31302261352539,
      "activations/layer13_attention_weight_min": -35.548770904541016,
      "activations/layer14_attention_weight_max": 45.16312789916992,
      "activations/layer14_attention_weight_min": -38.662925720214844,
      "activations/layer15_attention_weight_max": 44.722190856933594,
      "activations/layer15_attention_weight_min": -41.18669128417969,
      "activations/layer16_attention_weight_max": 30.981287002563477,
      "activations/layer16_attention_weight_min": -30.31785011291504,
      "activations/layer17_attention_weight_max": 29.985185623168945,
      "activations/layer17_attention_weight_min": -27.655935287475586,
      "activations/layer18_attention_weight_max": 35.88896942138672,
      "activations/layer18_attention_weight_min": -28.512849807739258,
      "activations/layer19_attention_weight_max": 37.135215759277344,
      "activations/layer19_attention_weight_min": -33.76014709472656,
      "activations/layer1_attention_weight_max": 16.400497436523438,
      "activations/layer1_attention_weight_min": -15.835613250732422,
      "activations/layer20_attention_weight_max": 31.891761779785156,
      "activations/layer20_attention_weight_min": -25.865516662597656,
      "activations/layer21_attention_weight_max": 31.15056610107422,
      "activations/layer21_attention_weight_min": -25.62051773071289,
      "activations/layer22_attention_weight_max": 41.770896911621094,
      "activations/layer22_attention_weight_min": -34.40052032470703,
      "activations/layer23_attention_weight_max": 31.66659164428711,
      "activations/layer23_attention_weight_min": -31.016963958740234,
      "activations/layer2_attention_weight_max": 36.229801177978516,
      "activations/layer2_attention_weight_min": -34.885467529296875,
      "activations/layer3_attention_weight_max": 108.89851379394531,
      "activations/layer3_attention_weight_min": -110.56397247314453,
      "activations/layer4_attention_weight_max": 113.29393768310547,
      "activations/layer4_attention_weight_min": -114.00899505615234,
      "activations/layer5_attention_weight_max": 51.68681335449219,
      "activations/layer5_attention_weight_min": -60.36781311035156,
      "activations/layer6_attention_weight_max": 46.31715774536133,
      "activations/layer6_attention_weight_min": -48.10737228393555,
      "activations/layer7_attention_weight_max": 96.31578063964844,
      "activations/layer7_attention_weight_min": -97.73868560791016,
      "activations/layer8_attention_weight_max": 40.907508850097656,
      "activations/layer8_attention_weight_min": -46.173988342285156,
      "activations/layer9_attention_weight_max": 32.69491195678711,
      "activations/layer9_attention_weight_min": -36.480743408203125,
      "epoch": 22.06,
      "learning_rate": 7.838636363636364e-06,
      "loss": 2.6969,
      "step": 379600
    },
    {
      "activations/layer0_attention_weight_max": 16.56845474243164,
      "activations/layer0_attention_weight_min": -14.091938018798828,
      "activations/layer10_attention_weight_max": 35.6741828918457,
      "activations/layer10_attention_weight_min": -35.514190673828125,
      "activations/layer11_attention_weight_max": 35.193206787109375,
      "activations/layer11_attention_weight_min": -35.23998260498047,
      "activations/layer12_attention_weight_max": 29.364084243774414,
      "activations/layer12_attention_weight_min": -28.380640029907227,
      "activations/layer13_attention_weight_max": 42.27397155761719,
      "activations/layer13_attention_weight_min": -37.10508728027344,
      "activations/layer14_attention_weight_max": 45.028160095214844,
      "activations/layer14_attention_weight_min": -40.46682357788086,
      "activations/layer15_attention_weight_max": 42.97292709350586,
      "activations/layer15_attention_weight_min": -38.155338287353516,
      "activations/layer16_attention_weight_max": 33.09170150756836,
      "activations/layer16_attention_weight_min": -28.5173282623291,
      "activations/layer17_attention_weight_max": 32.655216217041016,
      "activations/layer17_attention_weight_min": -28.716909408569336,
      "activations/layer18_attention_weight_max": 36.077999114990234,
      "activations/layer18_attention_weight_min": -27.867904663085938,
      "activations/layer19_attention_weight_max": 37.38987731933594,
      "activations/layer19_attention_weight_min": -32.67585754394531,
      "activations/layer1_attention_weight_max": 15.551189422607422,
      "activations/layer1_attention_weight_min": -15.417295455932617,
      "activations/layer20_attention_weight_max": 31.175731658935547,
      "activations/layer20_attention_weight_min": -26.13555908203125,
      "activations/layer21_attention_weight_max": 34.602813720703125,
      "activations/layer21_attention_weight_min": -24.756284713745117,
      "activations/layer22_attention_weight_max": 44.24596405029297,
      "activations/layer22_attention_weight_min": -32.32914352416992,
      "activations/layer23_attention_weight_max": 34.82243728637695,
      "activations/layer23_attention_weight_min": -26.742813110351562,
      "activations/layer2_attention_weight_max": 33.92927551269531,
      "activations/layer2_attention_weight_min": -33.30903625488281,
      "activations/layer3_attention_weight_max": 108.59495544433594,
      "activations/layer3_attention_weight_min": -107.63341522216797,
      "activations/layer4_attention_weight_max": 112.5858154296875,
      "activations/layer4_attention_weight_min": -114.5451431274414,
      "activations/layer5_attention_weight_max": 51.23806381225586,
      "activations/layer5_attention_weight_min": -63.85322570800781,
      "activations/layer6_attention_weight_max": 44.99210739135742,
      "activations/layer6_attention_weight_min": -48.79188919067383,
      "activations/layer7_attention_weight_max": 99.34416198730469,
      "activations/layer7_attention_weight_min": -102.38780975341797,
      "activations/layer8_attention_weight_max": 43.51490020751953,
      "activations/layer8_attention_weight_min": -46.31162643432617,
      "activations/layer9_attention_weight_max": 32.74763488769531,
      "activations/layer9_attention_weight_min": -36.45783996582031,
      "epoch": 22.06,
      "learning_rate": 7.819696969696969e-06,
      "loss": 2.7067,
      "step": 379650
    },
    {
      "activations/layer0_attention_weight_max": 16.659753799438477,
      "activations/layer0_attention_weight_min": -13.574822425842285,
      "activations/layer10_attention_weight_max": 34.806976318359375,
      "activations/layer10_attention_weight_min": -37.103599548339844,
      "activations/layer11_attention_weight_max": 34.66737365722656,
      "activations/layer11_attention_weight_min": -35.19366455078125,
      "activations/layer12_attention_weight_max": 30.331897735595703,
      "activations/layer12_attention_weight_min": -28.02484893798828,
      "activations/layer13_attention_weight_max": 44.096248626708984,
      "activations/layer13_attention_weight_min": -39.47755432128906,
      "activations/layer14_attention_weight_max": 47.492767333984375,
      "activations/layer14_attention_weight_min": -40.07135772705078,
      "activations/layer15_attention_weight_max": 42.03089904785156,
      "activations/layer15_attention_weight_min": -37.93062210083008,
      "activations/layer16_attention_weight_max": 32.83472442626953,
      "activations/layer16_attention_weight_min": -30.40442657470703,
      "activations/layer17_attention_weight_max": 32.70471954345703,
      "activations/layer17_attention_weight_min": -25.52184295654297,
      "activations/layer18_attention_weight_max": 35.25111389160156,
      "activations/layer18_attention_weight_min": -27.399089813232422,
      "activations/layer19_attention_weight_max": 37.376888275146484,
      "activations/layer19_attention_weight_min": -32.68022537231445,
      "activations/layer1_attention_weight_max": 15.201353073120117,
      "activations/layer1_attention_weight_min": -14.62585735321045,
      "activations/layer20_attention_weight_max": 30.977989196777344,
      "activations/layer20_attention_weight_min": -25.588481903076172,
      "activations/layer21_attention_weight_max": 32.43474197387695,
      "activations/layer21_attention_weight_min": -26.197086334228516,
      "activations/layer22_attention_weight_max": 44.05603790283203,
      "activations/layer22_attention_weight_min": -36.46884536743164,
      "activations/layer23_attention_weight_max": 33.57970428466797,
      "activations/layer23_attention_weight_min": -29.424365997314453,
      "activations/layer2_attention_weight_max": 34.40997314453125,
      "activations/layer2_attention_weight_min": -33.73310089111328,
      "activations/layer3_attention_weight_max": 108.57208251953125,
      "activations/layer3_attention_weight_min": -109.3654556274414,
      "activations/layer4_attention_weight_max": 110.87751770019531,
      "activations/layer4_attention_weight_min": -110.37591552734375,
      "activations/layer5_attention_weight_max": 50.94647216796875,
      "activations/layer5_attention_weight_min": -62.85924530029297,
      "activations/layer6_attention_weight_max": 45.7715950012207,
      "activations/layer6_attention_weight_min": -47.498435974121094,
      "activations/layer7_attention_weight_max": 99.9034423828125,
      "activations/layer7_attention_weight_min": -100.42340850830078,
      "activations/layer8_attention_weight_max": 44.23771286010742,
      "activations/layer8_attention_weight_min": -45.3174934387207,
      "activations/layer9_attention_weight_max": 35.18381881713867,
      "activations/layer9_attention_weight_min": -35.10329055786133,
      "epoch": 22.06,
      "learning_rate": 7.800757575757575e-06,
      "loss": 2.6989,
      "step": 379700
    },
    {
      "activations/layer0_attention_weight_max": 17.11775016784668,
      "activations/layer0_attention_weight_min": -14.344392776489258,
      "activations/layer10_attention_weight_max": 37.147090911865234,
      "activations/layer10_attention_weight_min": -37.81026077270508,
      "activations/layer11_attention_weight_max": 37.350669860839844,
      "activations/layer11_attention_weight_min": -36.574310302734375,
      "activations/layer12_attention_weight_max": 36.896881103515625,
      "activations/layer12_attention_weight_min": -29.131240844726562,
      "activations/layer13_attention_weight_max": 53.38127517700195,
      "activations/layer13_attention_weight_min": -37.12461853027344,
      "activations/layer14_attention_weight_max": 47.87265396118164,
      "activations/layer14_attention_weight_min": -39.34577178955078,
      "activations/layer15_attention_weight_max": 51.870975494384766,
      "activations/layer15_attention_weight_min": -38.69290542602539,
      "activations/layer16_attention_weight_max": 35.73715591430664,
      "activations/layer16_attention_weight_min": -29.93610954284668,
      "activations/layer17_attention_weight_max": 31.271114349365234,
      "activations/layer17_attention_weight_min": -27.131845474243164,
      "activations/layer18_attention_weight_max": 34.407474517822266,
      "activations/layer18_attention_weight_min": -26.47348403930664,
      "activations/layer19_attention_weight_max": 37.32697677612305,
      "activations/layer19_attention_weight_min": -32.94770431518555,
      "activations/layer1_attention_weight_max": 17.30120086669922,
      "activations/layer1_attention_weight_min": -15.925084114074707,
      "activations/layer20_attention_weight_max": 34.80400848388672,
      "activations/layer20_attention_weight_min": -26.133920669555664,
      "activations/layer21_attention_weight_max": 35.11009979248047,
      "activations/layer21_attention_weight_min": -25.324975967407227,
      "activations/layer22_attention_weight_max": 52.19624328613281,
      "activations/layer22_attention_weight_min": -33.171852111816406,
      "activations/layer23_attention_weight_max": 34.1573371887207,
      "activations/layer23_attention_weight_min": -29.331418991088867,
      "activations/layer2_attention_weight_max": 35.11321258544922,
      "activations/layer2_attention_weight_min": -33.95257568359375,
      "activations/layer3_attention_weight_max": 109.0417251586914,
      "activations/layer3_attention_weight_min": -110.71991729736328,
      "activations/layer4_attention_weight_max": 113.07592010498047,
      "activations/layer4_attention_weight_min": -113.79610443115234,
      "activations/layer5_attention_weight_max": 51.048946380615234,
      "activations/layer5_attention_weight_min": -62.810028076171875,
      "activations/layer6_attention_weight_max": 46.91910934448242,
      "activations/layer6_attention_weight_min": -48.28573989868164,
      "activations/layer7_attention_weight_max": 103.78031921386719,
      "activations/layer7_attention_weight_min": -106.416015625,
      "activations/layer8_attention_weight_max": 43.69999694824219,
      "activations/layer8_attention_weight_min": -49.28003692626953,
      "activations/layer9_attention_weight_max": 32.91446304321289,
      "activations/layer9_attention_weight_min": -36.83956527709961,
      "epoch": 22.07,
      "learning_rate": 7.781818181818182e-06,
      "loss": 2.6913,
      "step": 379750
    },
    {
      "activations/layer0_attention_weight_max": 16.82070541381836,
      "activations/layer0_attention_weight_min": -15.685871124267578,
      "activations/layer10_attention_weight_max": 35.65753173828125,
      "activations/layer10_attention_weight_min": -36.5977783203125,
      "activations/layer11_attention_weight_max": 34.983245849609375,
      "activations/layer11_attention_weight_min": -36.245216369628906,
      "activations/layer12_attention_weight_max": 28.827516555786133,
      "activations/layer12_attention_weight_min": -28.61619758605957,
      "activations/layer13_attention_weight_max": 39.705440521240234,
      "activations/layer13_attention_weight_min": -36.18031692504883,
      "activations/layer14_attention_weight_max": 42.03369903564453,
      "activations/layer14_attention_weight_min": -39.323726654052734,
      "activations/layer15_attention_weight_max": 43.24260330200195,
      "activations/layer15_attention_weight_min": -39.51753234863281,
      "activations/layer16_attention_weight_max": 30.97821807861328,
      "activations/layer16_attention_weight_min": -28.812284469604492,
      "activations/layer17_attention_weight_max": 29.657175064086914,
      "activations/layer17_attention_weight_min": -26.757719039916992,
      "activations/layer18_attention_weight_max": 34.63441848754883,
      "activations/layer18_attention_weight_min": -26.926132202148438,
      "activations/layer19_attention_weight_max": 35.66244888305664,
      "activations/layer19_attention_weight_min": -33.03901290893555,
      "activations/layer1_attention_weight_max": 17.08428955078125,
      "activations/layer1_attention_weight_min": -17.05133819580078,
      "activations/layer20_attention_weight_max": 30.19696044921875,
      "activations/layer20_attention_weight_min": -26.02350425720215,
      "activations/layer21_attention_weight_max": 30.238250732421875,
      "activations/layer21_attention_weight_min": -25.57749366760254,
      "activations/layer22_attention_weight_max": 45.70463943481445,
      "activations/layer22_attention_weight_min": -35.07456588745117,
      "activations/layer23_attention_weight_max": 34.83650207519531,
      "activations/layer23_attention_weight_min": -27.514862060546875,
      "activations/layer2_attention_weight_max": 34.79619598388672,
      "activations/layer2_attention_weight_min": -34.86504364013672,
      "activations/layer3_attention_weight_max": 108.98965454101562,
      "activations/layer3_attention_weight_min": -111.48236083984375,
      "activations/layer4_attention_weight_max": 110.53199768066406,
      "activations/layer4_attention_weight_min": -111.12181854248047,
      "activations/layer5_attention_weight_max": 51.02972412109375,
      "activations/layer5_attention_weight_min": -64.12785339355469,
      "activations/layer6_attention_weight_max": 46.57576370239258,
      "activations/layer6_attention_weight_min": -49.13894271850586,
      "activations/layer7_attention_weight_max": 100.6487045288086,
      "activations/layer7_attention_weight_min": -103.33080291748047,
      "activations/layer8_attention_weight_max": 45.62385940551758,
      "activations/layer8_attention_weight_min": -48.10859680175781,
      "activations/layer9_attention_weight_max": 34.83883285522461,
      "activations/layer9_attention_weight_min": -38.06357955932617,
      "epoch": 22.07,
      "learning_rate": 7.763257575757576e-06,
      "loss": 2.706,
      "step": 379800
    },
    {
      "activations/layer0_attention_weight_max": 17.021024703979492,
      "activations/layer0_attention_weight_min": -14.093864440917969,
      "activations/layer10_attention_weight_max": 34.37567138671875,
      "activations/layer10_attention_weight_min": -36.16604232788086,
      "activations/layer11_attention_weight_max": 33.98535919189453,
      "activations/layer11_attention_weight_min": -35.065826416015625,
      "activations/layer12_attention_weight_max": 25.857479095458984,
      "activations/layer12_attention_weight_min": -29.129600524902344,
      "activations/layer13_attention_weight_max": 36.90311813354492,
      "activations/layer13_attention_weight_min": -39.484283447265625,
      "activations/layer14_attention_weight_max": 41.47431945800781,
      "activations/layer14_attention_weight_min": -40.79589080810547,
      "activations/layer15_attention_weight_max": 40.443763732910156,
      "activations/layer15_attention_weight_min": -38.94680404663086,
      "activations/layer16_attention_weight_max": 28.67259979248047,
      "activations/layer16_attention_weight_min": -29.20588493347168,
      "activations/layer17_attention_weight_max": 30.632205963134766,
      "activations/layer17_attention_weight_min": -28.767494201660156,
      "activations/layer18_attention_weight_max": 36.17702865600586,
      "activations/layer18_attention_weight_min": -31.750629425048828,
      "activations/layer19_attention_weight_max": 32.526695251464844,
      "activations/layer19_attention_weight_min": -34.483516693115234,
      "activations/layer1_attention_weight_max": 16.640979766845703,
      "activations/layer1_attention_weight_min": -16.024887084960938,
      "activations/layer20_attention_weight_max": 28.205371856689453,
      "activations/layer20_attention_weight_min": -26.668350219726562,
      "activations/layer21_attention_weight_max": 27.773818969726562,
      "activations/layer21_attention_weight_min": -27.329479217529297,
      "activations/layer22_attention_weight_max": 39.26810073852539,
      "activations/layer22_attention_weight_min": -35.29243850708008,
      "activations/layer23_attention_weight_max": 30.9879150390625,
      "activations/layer23_attention_weight_min": -28.017337799072266,
      "activations/layer2_attention_weight_max": 35.24040985107422,
      "activations/layer2_attention_weight_min": -32.976776123046875,
      "activations/layer3_attention_weight_max": 104.931884765625,
      "activations/layer3_attention_weight_min": -105.99972534179688,
      "activations/layer4_attention_weight_max": 108.2601318359375,
      "activations/layer4_attention_weight_min": -112.57252502441406,
      "activations/layer5_attention_weight_max": 48.8387451171875,
      "activations/layer5_attention_weight_min": -62.42583465576172,
      "activations/layer6_attention_weight_max": 46.395572662353516,
      "activations/layer6_attention_weight_min": -48.16264343261719,
      "activations/layer7_attention_weight_max": 93.40880584716797,
      "activations/layer7_attention_weight_min": -99.19617462158203,
      "activations/layer8_attention_weight_max": 41.156532287597656,
      "activations/layer8_attention_weight_min": -46.35538101196289,
      "activations/layer9_attention_weight_max": 32.71500015258789,
      "activations/layer9_attention_weight_min": -35.20159912109375,
      "epoch": 22.07,
      "learning_rate": 7.74431818181818e-06,
      "loss": 2.7001,
      "step": 379850
    },
    {
      "activations/layer0_attention_weight_max": 16.535255432128906,
      "activations/layer0_attention_weight_min": -13.941697120666504,
      "activations/layer10_attention_weight_max": 33.82630157470703,
      "activations/layer10_attention_weight_min": -35.75720977783203,
      "activations/layer11_attention_weight_max": 34.675567626953125,
      "activations/layer11_attention_weight_min": -37.02290344238281,
      "activations/layer12_attention_weight_max": 26.422359466552734,
      "activations/layer12_attention_weight_min": -27.639217376708984,
      "activations/layer13_attention_weight_max": 41.18524932861328,
      "activations/layer13_attention_weight_min": -36.54426193237305,
      "activations/layer14_attention_weight_max": 43.48019027709961,
      "activations/layer14_attention_weight_min": -40.68939971923828,
      "activations/layer15_attention_weight_max": 41.95731735229492,
      "activations/layer15_attention_weight_min": -36.93147659301758,
      "activations/layer16_attention_weight_max": 31.387060165405273,
      "activations/layer16_attention_weight_min": -28.46471405029297,
      "activations/layer17_attention_weight_max": 33.074745178222656,
      "activations/layer17_attention_weight_min": -26.0482234954834,
      "activations/layer18_attention_weight_max": 34.671112060546875,
      "activations/layer18_attention_weight_min": -26.439624786376953,
      "activations/layer19_attention_weight_max": 34.718441009521484,
      "activations/layer19_attention_weight_min": -35.02095413208008,
      "activations/layer1_attention_weight_max": 16.38373374938965,
      "activations/layer1_attention_weight_min": -15.358691215515137,
      "activations/layer20_attention_weight_max": 28.377479553222656,
      "activations/layer20_attention_weight_min": -25.769670486450195,
      "activations/layer21_attention_weight_max": 29.80606460571289,
      "activations/layer21_attention_weight_min": -25.14336585998535,
      "activations/layer22_attention_weight_max": 43.597389221191406,
      "activations/layer22_attention_weight_min": -30.56682586669922,
      "activations/layer23_attention_weight_max": 31.26390838623047,
      "activations/layer23_attention_weight_min": -25.47981834411621,
      "activations/layer2_attention_weight_max": 32.85625076293945,
      "activations/layer2_attention_weight_min": -34.260807037353516,
      "activations/layer3_attention_weight_max": 108.93988037109375,
      "activations/layer3_attention_weight_min": -111.9697265625,
      "activations/layer4_attention_weight_max": 112.1405258178711,
      "activations/layer4_attention_weight_min": -117.3097152709961,
      "activations/layer5_attention_weight_max": 51.27264404296875,
      "activations/layer5_attention_weight_min": -62.25912094116211,
      "activations/layer6_attention_weight_max": 46.44919204711914,
      "activations/layer6_attention_weight_min": -52.152095794677734,
      "activations/layer7_attention_weight_max": 99.85414123535156,
      "activations/layer7_attention_weight_min": -108.5484390258789,
      "activations/layer8_attention_weight_max": 44.302154541015625,
      "activations/layer8_attention_weight_min": -49.648719787597656,
      "activations/layer9_attention_weight_max": 32.71854782104492,
      "activations/layer9_attention_weight_min": -35.51660919189453,
      "epoch": 22.07,
      "learning_rate": 7.725378787878787e-06,
      "loss": 2.7003,
      "step": 379900
    },
    {
      "activations/layer0_attention_weight_max": 17.65709686279297,
      "activations/layer0_attention_weight_min": -14.540314674377441,
      "activations/layer10_attention_weight_max": 34.24805450439453,
      "activations/layer10_attention_weight_min": -35.49138259887695,
      "activations/layer11_attention_weight_max": 35.480525970458984,
      "activations/layer11_attention_weight_min": -34.861976623535156,
      "activations/layer12_attention_weight_max": 25.706363677978516,
      "activations/layer12_attention_weight_min": -27.277746200561523,
      "activations/layer13_attention_weight_max": 36.94987869262695,
      "activations/layer13_attention_weight_min": -36.083003997802734,
      "activations/layer14_attention_weight_max": 44.0015869140625,
      "activations/layer14_attention_weight_min": -39.088172912597656,
      "activations/layer15_attention_weight_max": 37.846195220947266,
      "activations/layer15_attention_weight_min": -36.71112823486328,
      "activations/layer16_attention_weight_max": 29.5153751373291,
      "activations/layer16_attention_weight_min": -31.50996971130371,
      "activations/layer17_attention_weight_max": 27.285966873168945,
      "activations/layer17_attention_weight_min": -28.252099990844727,
      "activations/layer18_attention_weight_max": 28.929658889770508,
      "activations/layer18_attention_weight_min": -28.199954986572266,
      "activations/layer19_attention_weight_max": 34.612884521484375,
      "activations/layer19_attention_weight_min": -32.83885955810547,
      "activations/layer1_attention_weight_max": 17.01764488220215,
      "activations/layer1_attention_weight_min": -15.02667236328125,
      "activations/layer20_attention_weight_max": 28.6605167388916,
      "activations/layer20_attention_weight_min": -25.04088020324707,
      "activations/layer21_attention_weight_max": 29.176353454589844,
      "activations/layer21_attention_weight_min": -23.21874237060547,
      "activations/layer22_attention_weight_max": 41.647037506103516,
      "activations/layer22_attention_weight_min": -30.151094436645508,
      "activations/layer23_attention_weight_max": 29.081937789916992,
      "activations/layer23_attention_weight_min": -24.05214500427246,
      "activations/layer2_attention_weight_max": 33.881134033203125,
      "activations/layer2_attention_weight_min": -34.673431396484375,
      "activations/layer3_attention_weight_max": 105.92708587646484,
      "activations/layer3_attention_weight_min": -110.80010986328125,
      "activations/layer4_attention_weight_max": 108.5900650024414,
      "activations/layer4_attention_weight_min": -115.08503723144531,
      "activations/layer5_attention_weight_max": 49.133033752441406,
      "activations/layer5_attention_weight_min": -65.04176330566406,
      "activations/layer6_attention_weight_max": 44.948795318603516,
      "activations/layer6_attention_weight_min": -48.43760299682617,
      "activations/layer7_attention_weight_max": 97.35842895507812,
      "activations/layer7_attention_weight_min": -97.30187225341797,
      "activations/layer8_attention_weight_max": 41.445091247558594,
      "activations/layer8_attention_weight_min": -44.864925384521484,
      "activations/layer9_attention_weight_max": 30.95711326599121,
      "activations/layer9_attention_weight_min": -35.679962158203125,
      "epoch": 22.08,
      "learning_rate": 7.706439393939392e-06,
      "loss": 2.718,
      "step": 379950
    },
    {
      "activations/layer0_attention_weight_max": 17.044025421142578,
      "activations/layer0_attention_weight_min": -15.104838371276855,
      "activations/layer10_attention_weight_max": 36.03760528564453,
      "activations/layer10_attention_weight_min": -38.02796173095703,
      "activations/layer11_attention_weight_max": 34.59898376464844,
      "activations/layer11_attention_weight_min": -38.863277435302734,
      "activations/layer12_attention_weight_max": 28.7799129486084,
      "activations/layer12_attention_weight_min": -27.993135452270508,
      "activations/layer13_attention_weight_max": 42.02846145629883,
      "activations/layer13_attention_weight_min": -37.6612434387207,
      "activations/layer14_attention_weight_max": 45.086063385009766,
      "activations/layer14_attention_weight_min": -40.5081672668457,
      "activations/layer15_attention_weight_max": 41.65597915649414,
      "activations/layer15_attention_weight_min": -39.322200775146484,
      "activations/layer16_attention_weight_max": 30.430191040039062,
      "activations/layer16_attention_weight_min": -30.988115310668945,
      "activations/layer17_attention_weight_max": 29.965681076049805,
      "activations/layer17_attention_weight_min": -30.32621955871582,
      "activations/layer18_attention_weight_max": 33.35593032836914,
      "activations/layer18_attention_weight_min": -28.40591049194336,
      "activations/layer19_attention_weight_max": 38.287750244140625,
      "activations/layer19_attention_weight_min": -34.4786491394043,
      "activations/layer1_attention_weight_max": 16.123729705810547,
      "activations/layer1_attention_weight_min": -15.067793846130371,
      "activations/layer20_attention_weight_max": 29.2137451171875,
      "activations/layer20_attention_weight_min": -27.444690704345703,
      "activations/layer21_attention_weight_max": 29.89151954650879,
      "activations/layer21_attention_weight_min": -26.02361488342285,
      "activations/layer22_attention_weight_max": 42.39350128173828,
      "activations/layer22_attention_weight_min": -33.43498611450195,
      "activations/layer23_attention_weight_max": 33.644927978515625,
      "activations/layer23_attention_weight_min": -27.291244506835938,
      "activations/layer2_attention_weight_max": 35.33595657348633,
      "activations/layer2_attention_weight_min": -33.516441345214844,
      "activations/layer3_attention_weight_max": 107.08961486816406,
      "activations/layer3_attention_weight_min": -108.56450653076172,
      "activations/layer4_attention_weight_max": 110.63667297363281,
      "activations/layer4_attention_weight_min": -113.8594741821289,
      "activations/layer5_attention_weight_max": 49.86323165893555,
      "activations/layer5_attention_weight_min": -63.68464279174805,
      "activations/layer6_attention_weight_max": 45.784278869628906,
      "activations/layer6_attention_weight_min": -50.87510299682617,
      "activations/layer7_attention_weight_max": 98.3583755493164,
      "activations/layer7_attention_weight_min": -103.77556610107422,
      "activations/layer8_attention_weight_max": 44.33015060424805,
      "activations/layer8_attention_weight_min": -47.6909294128418,
      "activations/layer9_attention_weight_max": 32.1086540222168,
      "activations/layer9_attention_weight_min": -36.565887451171875,
      "epoch": 22.08,
      "learning_rate": 7.687499999999999e-06,
      "loss": 2.6952,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.5343,
      "eval_samples_per_second": 503.149,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.5343,
      "eval_openwebtext_samples_per_second": 503.149,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9992,
      "eval_wikitext_samples_per_second": 228.086,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_lambada_loss": 2.33203125,
      "eval_lambada_ppl": 10.298839819419056,
      "eval_lambada_runtime": 9.5547,
      "eval_lambada_samples_per_second": 509.594,
      "step": 380000
    },
    {
      "activations/layer0_attention_weight_max": 16.73607063293457,
      "activations/layer0_attention_weight_min": -13.993109703063965,
      "activations/layer10_attention_weight_max": 35.8276252746582,
      "activations/layer10_attention_weight_min": -35.556365966796875,
      "activations/layer11_attention_weight_max": 34.00286102294922,
      "activations/layer11_attention_weight_min": -35.7227783203125,
      "activations/layer12_attention_weight_max": 28.500747680664062,
      "activations/layer12_attention_weight_min": -27.5200252532959,
      "activations/layer13_attention_weight_max": 40.38310241699219,
      "activations/layer13_attention_weight_min": -38.79247283935547,
      "activations/layer14_attention_weight_max": 44.36164093017578,
      "activations/layer14_attention_weight_min": -39.54483413696289,
      "activations/layer15_attention_weight_max": 40.49094009399414,
      "activations/layer15_attention_weight_min": -39.63038635253906,
      "activations/layer16_attention_weight_max": 30.684232711791992,
      "activations/layer16_attention_weight_min": -28.975353240966797,
      "activations/layer17_attention_weight_max": 31.047027587890625,
      "activations/layer17_attention_weight_min": -27.910337448120117,
      "activations/layer18_attention_weight_max": 37.70750427246094,
      "activations/layer18_attention_weight_min": -29.4871883392334,
      "activations/layer19_attention_weight_max": 35.421627044677734,
      "activations/layer19_attention_weight_min": -33.86212921142578,
      "activations/layer1_attention_weight_max": 16.082962036132812,
      "activations/layer1_attention_weight_min": -14.991729736328125,
      "activations/layer20_attention_weight_max": 27.482980728149414,
      "activations/layer20_attention_weight_min": -26.207069396972656,
      "activations/layer21_attention_weight_max": 28.538963317871094,
      "activations/layer21_attention_weight_min": -27.194622039794922,
      "activations/layer22_attention_weight_max": 41.3985710144043,
      "activations/layer22_attention_weight_min": -34.513328552246094,
      "activations/layer23_attention_weight_max": 32.82639694213867,
      "activations/layer23_attention_weight_min": -27.773788452148438,
      "activations/layer2_attention_weight_max": 34.739524841308594,
      "activations/layer2_attention_weight_min": -35.53008270263672,
      "activations/layer3_attention_weight_max": 110.91187286376953,
      "activations/layer3_attention_weight_min": -110.67003631591797,
      "activations/layer4_attention_weight_max": 112.6424331665039,
      "activations/layer4_attention_weight_min": -117.49169158935547,
      "activations/layer5_attention_weight_max": 51.25838851928711,
      "activations/layer5_attention_weight_min": -66.70159912109375,
      "activations/layer6_attention_weight_max": 48.47416687011719,
      "activations/layer6_attention_weight_min": -49.87295913696289,
      "activations/layer7_attention_weight_max": 98.39934539794922,
      "activations/layer7_attention_weight_min": -100.47779083251953,
      "activations/layer8_attention_weight_max": 41.95734405517578,
      "activations/layer8_attention_weight_min": -46.247283935546875,
      "activations/layer9_attention_weight_max": 31.99274253845215,
      "activations/layer9_attention_weight_min": -35.96610641479492,
      "epoch": 22.08,
      "learning_rate": 7.668560606060605e-06,
      "loss": 2.702,
      "step": 380050
    },
    {
      "activations/layer0_attention_weight_max": 17.081636428833008,
      "activations/layer0_attention_weight_min": -13.874154090881348,
      "activations/layer10_attention_weight_max": 38.061866760253906,
      "activations/layer10_attention_weight_min": -34.84038162231445,
      "activations/layer11_attention_weight_max": 36.25751876831055,
      "activations/layer11_attention_weight_min": -34.99674987792969,
      "activations/layer12_attention_weight_max": 28.597614288330078,
      "activations/layer12_attention_weight_min": -28.50688362121582,
      "activations/layer13_attention_weight_max": 43.83448028564453,
      "activations/layer13_attention_weight_min": -40.0965576171875,
      "activations/layer14_attention_weight_max": 47.807064056396484,
      "activations/layer14_attention_weight_min": -40.645469665527344,
      "activations/layer15_attention_weight_max": 44.754600524902344,
      "activations/layer15_attention_weight_min": -39.12403106689453,
      "activations/layer16_attention_weight_max": 34.27279281616211,
      "activations/layer16_attention_weight_min": -29.57698631286621,
      "activations/layer17_attention_weight_max": 32.35218811035156,
      "activations/layer17_attention_weight_min": -28.126550674438477,
      "activations/layer18_attention_weight_max": 37.32771682739258,
      "activations/layer18_attention_weight_min": -27.666410446166992,
      "activations/layer19_attention_weight_max": 38.21256637573242,
      "activations/layer19_attention_weight_min": -33.081642150878906,
      "activations/layer1_attention_weight_max": 16.144695281982422,
      "activations/layer1_attention_weight_min": -14.876575469970703,
      "activations/layer20_attention_weight_max": 33.46061706542969,
      "activations/layer20_attention_weight_min": -25.39306640625,
      "activations/layer21_attention_weight_max": 37.730125427246094,
      "activations/layer21_attention_weight_min": -29.457542419433594,
      "activations/layer22_attention_weight_max": 50.99932861328125,
      "activations/layer22_attention_weight_min": -38.620689392089844,
      "activations/layer23_attention_weight_max": 36.67182540893555,
      "activations/layer23_attention_weight_min": -29.324607849121094,
      "activations/layer2_attention_weight_max": 36.26679229736328,
      "activations/layer2_attention_weight_min": -33.96886444091797,
      "activations/layer3_attention_weight_max": 106.91950225830078,
      "activations/layer3_attention_weight_min": -108.05632781982422,
      "activations/layer4_attention_weight_max": 110.85787200927734,
      "activations/layer4_attention_weight_min": -112.73365783691406,
      "activations/layer5_attention_weight_max": 51.48912048339844,
      "activations/layer5_attention_weight_min": -62.954315185546875,
      "activations/layer6_attention_weight_max": 44.22288131713867,
      "activations/layer6_attention_weight_min": -48.109378814697266,
      "activations/layer7_attention_weight_max": 101.94647979736328,
      "activations/layer7_attention_weight_min": -97.48632049560547,
      "activations/layer8_attention_weight_max": 43.26948928833008,
      "activations/layer8_attention_weight_min": -45.55397415161133,
      "activations/layer9_attention_weight_max": 31.877614974975586,
      "activations/layer9_attention_weight_min": -34.58168411254883,
      "epoch": 22.09,
      "learning_rate": 7.649621212121212e-06,
      "loss": 2.6953,
      "step": 380100
    },
    {
      "activations/layer0_attention_weight_max": 17.15989112854004,
      "activations/layer0_attention_weight_min": -13.220805168151855,
      "activations/layer10_attention_weight_max": 32.197410583496094,
      "activations/layer10_attention_weight_min": -35.04888916015625,
      "activations/layer11_attention_weight_max": 33.73674774169922,
      "activations/layer11_attention_weight_min": -33.56135177612305,
      "activations/layer12_attention_weight_max": 27.092241287231445,
      "activations/layer12_attention_weight_min": -26.334543228149414,
      "activations/layer13_attention_weight_max": 43.29917907714844,
      "activations/layer13_attention_weight_min": -35.00538635253906,
      "activations/layer14_attention_weight_max": 42.912052154541016,
      "activations/layer14_attention_weight_min": -37.92198944091797,
      "activations/layer15_attention_weight_max": 38.469120025634766,
      "activations/layer15_attention_weight_min": -35.56058120727539,
      "activations/layer16_attention_weight_max": 29.016267776489258,
      "activations/layer16_attention_weight_min": -28.576860427856445,
      "activations/layer17_attention_weight_max": 26.822486877441406,
      "activations/layer17_attention_weight_min": -26.976337432861328,
      "activations/layer18_attention_weight_max": 33.47041702270508,
      "activations/layer18_attention_weight_min": -26.864078521728516,
      "activations/layer19_attention_weight_max": 32.27944564819336,
      "activations/layer19_attention_weight_min": -31.48849868774414,
      "activations/layer1_attention_weight_max": 16.9115047454834,
      "activations/layer1_attention_weight_min": -15.245755195617676,
      "activations/layer20_attention_weight_max": 25.75795555114746,
      "activations/layer20_attention_weight_min": -23.89286231994629,
      "activations/layer21_attention_weight_max": 26.827091217041016,
      "activations/layer21_attention_weight_min": -23.397502899169922,
      "activations/layer22_attention_weight_max": 38.14358901977539,
      "activations/layer22_attention_weight_min": -29.556108474731445,
      "activations/layer23_attention_weight_max": 27.876781463623047,
      "activations/layer23_attention_weight_min": -24.964353561401367,
      "activations/layer2_attention_weight_max": 36.225181579589844,
      "activations/layer2_attention_weight_min": -34.797367095947266,
      "activations/layer3_attention_weight_max": 105.50963592529297,
      "activations/layer3_attention_weight_min": -107.85928344726562,
      "activations/layer4_attention_weight_max": 108.35047912597656,
      "activations/layer4_attention_weight_min": -110.45531463623047,
      "activations/layer5_attention_weight_max": 53.342918395996094,
      "activations/layer5_attention_weight_min": -60.48484802246094,
      "activations/layer6_attention_weight_max": 44.54020690917969,
      "activations/layer6_attention_weight_min": -48.485984802246094,
      "activations/layer7_attention_weight_max": 93.27598571777344,
      "activations/layer7_attention_weight_min": -101.90001678466797,
      "activations/layer8_attention_weight_max": 42.96822738647461,
      "activations/layer8_attention_weight_min": -44.98197555541992,
      "activations/layer9_attention_weight_max": 31.30537986755371,
      "activations/layer9_attention_weight_min": -34.900901794433594,
      "epoch": 22.09,
      "learning_rate": 7.630681818181819e-06,
      "loss": 2.6896,
      "step": 380150
    },
    {
      "activations/layer0_attention_weight_max": 16.688701629638672,
      "activations/layer0_attention_weight_min": -13.392928123474121,
      "activations/layer10_attention_weight_max": 37.33357620239258,
      "activations/layer10_attention_weight_min": -37.06978225708008,
      "activations/layer11_attention_weight_max": 38.563636779785156,
      "activations/layer11_attention_weight_min": -38.04090881347656,
      "activations/layer12_attention_weight_max": 29.13104820251465,
      "activations/layer12_attention_weight_min": -28.394609451293945,
      "activations/layer13_attention_weight_max": 42.5852165222168,
      "activations/layer13_attention_weight_min": -38.030052185058594,
      "activations/layer14_attention_weight_max": 44.0574836730957,
      "activations/layer14_attention_weight_min": -38.6051139831543,
      "activations/layer15_attention_weight_max": 43.107364654541016,
      "activations/layer15_attention_weight_min": -37.90431213378906,
      "activations/layer16_attention_weight_max": 32.70481491088867,
      "activations/layer16_attention_weight_min": -28.630788803100586,
      "activations/layer17_attention_weight_max": 31.197723388671875,
      "activations/layer17_attention_weight_min": -26.02549171447754,
      "activations/layer18_attention_weight_max": 33.666542053222656,
      "activations/layer18_attention_weight_min": -27.586238861083984,
      "activations/layer19_attention_weight_max": 34.10442352294922,
      "activations/layer19_attention_weight_min": -31.117542266845703,
      "activations/layer1_attention_weight_max": 15.925418853759766,
      "activations/layer1_attention_weight_min": -14.870148658752441,
      "activations/layer20_attention_weight_max": 29.459814071655273,
      "activations/layer20_attention_weight_min": -24.045339584350586,
      "activations/layer21_attention_weight_max": 27.994524002075195,
      "activations/layer21_attention_weight_min": -23.712339401245117,
      "activations/layer22_attention_weight_max": 43.90678024291992,
      "activations/layer22_attention_weight_min": -32.290138244628906,
      "activations/layer23_attention_weight_max": 35.00349426269531,
      "activations/layer23_attention_weight_min": -24.232648849487305,
      "activations/layer2_attention_weight_max": 33.703067779541016,
      "activations/layer2_attention_weight_min": -34.403900146484375,
      "activations/layer3_attention_weight_max": 111.9261245727539,
      "activations/layer3_attention_weight_min": -110.62364196777344,
      "activations/layer4_attention_weight_max": 113.95143127441406,
      "activations/layer4_attention_weight_min": -115.97505187988281,
      "activations/layer5_attention_weight_max": 49.79649353027344,
      "activations/layer5_attention_weight_min": -68.42913818359375,
      "activations/layer6_attention_weight_max": 47.336769104003906,
      "activations/layer6_attention_weight_min": -51.40773391723633,
      "activations/layer7_attention_weight_max": 100.32479095458984,
      "activations/layer7_attention_weight_min": -106.79145050048828,
      "activations/layer8_attention_weight_max": 45.174072265625,
      "activations/layer8_attention_weight_min": -47.63295364379883,
      "activations/layer9_attention_weight_max": 34.02245330810547,
      "activations/layer9_attention_weight_min": -35.53988265991211,
      "epoch": 22.09,
      "learning_rate": 7.6117424242424234e-06,
      "loss": 2.7027,
      "step": 380200
    },
    {
      "activations/layer0_attention_weight_max": 16.460838317871094,
      "activations/layer0_attention_weight_min": -14.562471389770508,
      "activations/layer10_attention_weight_max": 33.52558135986328,
      "activations/layer10_attention_weight_min": -36.51031494140625,
      "activations/layer11_attention_weight_max": 34.326332092285156,
      "activations/layer11_attention_weight_min": -35.094398498535156,
      "activations/layer12_attention_weight_max": 26.949283599853516,
      "activations/layer12_attention_weight_min": -29.293432235717773,
      "activations/layer13_attention_weight_max": 40.06359100341797,
      "activations/layer13_attention_weight_min": -36.91875076293945,
      "activations/layer14_attention_weight_max": 40.86994171142578,
      "activations/layer14_attention_weight_min": -42.07882308959961,
      "activations/layer15_attention_weight_max": 40.11516571044922,
      "activations/layer15_attention_weight_min": -38.01335525512695,
      "activations/layer16_attention_weight_max": 30.122623443603516,
      "activations/layer16_attention_weight_min": -29.595827102661133,
      "activations/layer17_attention_weight_max": 29.879533767700195,
      "activations/layer17_attention_weight_min": -28.289045333862305,
      "activations/layer18_attention_weight_max": 33.04118347167969,
      "activations/layer18_attention_weight_min": -25.26844024658203,
      "activations/layer19_attention_weight_max": 37.79570388793945,
      "activations/layer19_attention_weight_min": -35.33738327026367,
      "activations/layer1_attention_weight_max": 16.298229217529297,
      "activations/layer1_attention_weight_min": -15.956680297851562,
      "activations/layer20_attention_weight_max": 30.540292739868164,
      "activations/layer20_attention_weight_min": -26.2078800201416,
      "activations/layer21_attention_weight_max": 29.838598251342773,
      "activations/layer21_attention_weight_min": -26.090280532836914,
      "activations/layer22_attention_weight_max": 43.3585090637207,
      "activations/layer22_attention_weight_min": -31.3673095703125,
      "activations/layer23_attention_weight_max": 33.86504364013672,
      "activations/layer23_attention_weight_min": -25.38039779663086,
      "activations/layer2_attention_weight_max": 33.73772430419922,
      "activations/layer2_attention_weight_min": -33.36568069458008,
      "activations/layer3_attention_weight_max": 111.13125610351562,
      "activations/layer3_attention_weight_min": -112.03978729248047,
      "activations/layer4_attention_weight_max": 113.88795471191406,
      "activations/layer4_attention_weight_min": -120.14437103271484,
      "activations/layer5_attention_weight_max": 54.453914642333984,
      "activations/layer5_attention_weight_min": -67.41863250732422,
      "activations/layer6_attention_weight_max": 45.06140899658203,
      "activations/layer6_attention_weight_min": -49.257938385009766,
      "activations/layer7_attention_weight_max": 99.76908111572266,
      "activations/layer7_attention_weight_min": -103.70985412597656,
      "activations/layer8_attention_weight_max": 42.335479736328125,
      "activations/layer8_attention_weight_min": -46.05902099609375,
      "activations/layer9_attention_weight_max": 31.52359390258789,
      "activations/layer9_attention_weight_min": -35.91830062866211,
      "epoch": 22.09,
      "learning_rate": 7.59280303030303e-06,
      "loss": 2.7055,
      "step": 380250
    },
    {
      "activations/layer0_attention_weight_max": 16.910327911376953,
      "activations/layer0_attention_weight_min": -15.003199577331543,
      "activations/layer10_attention_weight_max": 33.833839416503906,
      "activations/layer10_attention_weight_min": -36.769535064697266,
      "activations/layer11_attention_weight_max": 33.956600189208984,
      "activations/layer11_attention_weight_min": -36.28919982910156,
      "activations/layer12_attention_weight_max": 27.629581451416016,
      "activations/layer12_attention_weight_min": -29.14861297607422,
      "activations/layer13_attention_weight_max": 39.99198913574219,
      "activations/layer13_attention_weight_min": -39.101463317871094,
      "activations/layer14_attention_weight_max": 42.525917053222656,
      "activations/layer14_attention_weight_min": -40.830718994140625,
      "activations/layer15_attention_weight_max": 41.37611770629883,
      "activations/layer15_attention_weight_min": -38.79093551635742,
      "activations/layer16_attention_weight_max": 28.792802810668945,
      "activations/layer16_attention_weight_min": -28.100276947021484,
      "activations/layer17_attention_weight_max": 30.121620178222656,
      "activations/layer17_attention_weight_min": -26.609752655029297,
      "activations/layer18_attention_weight_max": 33.07770919799805,
      "activations/layer18_attention_weight_min": -26.330053329467773,
      "activations/layer19_attention_weight_max": 34.76131057739258,
      "activations/layer19_attention_weight_min": -32.80115509033203,
      "activations/layer1_attention_weight_max": 15.539717674255371,
      "activations/layer1_attention_weight_min": -15.37335205078125,
      "activations/layer20_attention_weight_max": 30.572704315185547,
      "activations/layer20_attention_weight_min": -25.6854305267334,
      "activations/layer21_attention_weight_max": 32.49066162109375,
      "activations/layer21_attention_weight_min": -28.51963233947754,
      "activations/layer22_attention_weight_max": 47.63687515258789,
      "activations/layer22_attention_weight_min": -31.559926986694336,
      "activations/layer23_attention_weight_max": 34.27827453613281,
      "activations/layer23_attention_weight_min": -26.245094299316406,
      "activations/layer2_attention_weight_max": 33.85809326171875,
      "activations/layer2_attention_weight_min": -33.97283172607422,
      "activations/layer3_attention_weight_max": 106.90093231201172,
      "activations/layer3_attention_weight_min": -110.27793884277344,
      "activations/layer4_attention_weight_max": 112.4651107788086,
      "activations/layer4_attention_weight_min": -116.27815246582031,
      "activations/layer5_attention_weight_max": 51.81079864501953,
      "activations/layer5_attention_weight_min": -63.63547897338867,
      "activations/layer6_attention_weight_max": 46.31325912475586,
      "activations/layer6_attention_weight_min": -50.00812530517578,
      "activations/layer7_attention_weight_max": 97.68512725830078,
      "activations/layer7_attention_weight_min": -102.6930923461914,
      "activations/layer8_attention_weight_max": 42.55723571777344,
      "activations/layer8_attention_weight_min": -46.6444091796875,
      "activations/layer9_attention_weight_max": 32.9743766784668,
      "activations/layer9_attention_weight_min": -35.16371154785156,
      "epoch": 22.1,
      "learning_rate": 7.573863636363636e-06,
      "loss": 2.6985,
      "step": 380300
    },
    {
      "activations/layer0_attention_weight_max": 17.65872573852539,
      "activations/layer0_attention_weight_min": -13.820882797241211,
      "activations/layer10_attention_weight_max": 33.6733512878418,
      "activations/layer10_attention_weight_min": -34.29201889038086,
      "activations/layer11_attention_weight_max": 32.27659606933594,
      "activations/layer11_attention_weight_min": -33.067840576171875,
      "activations/layer12_attention_weight_max": 29.19487953186035,
      "activations/layer12_attention_weight_min": -28.227304458618164,
      "activations/layer13_attention_weight_max": 40.205108642578125,
      "activations/layer13_attention_weight_min": -35.148189544677734,
      "activations/layer14_attention_weight_max": 42.97879409790039,
      "activations/layer14_attention_weight_min": -35.78213119506836,
      "activations/layer15_attention_weight_max": 38.243629455566406,
      "activations/layer15_attention_weight_min": -34.86365509033203,
      "activations/layer16_attention_weight_max": 29.368730545043945,
      "activations/layer16_attention_weight_min": -26.92953109741211,
      "activations/layer17_attention_weight_max": 31.18941307067871,
      "activations/layer17_attention_weight_min": -26.301952362060547,
      "activations/layer18_attention_weight_max": 34.5396842956543,
      "activations/layer18_attention_weight_min": -28.653278350830078,
      "activations/layer19_attention_weight_max": 33.34923553466797,
      "activations/layer19_attention_weight_min": -31.443204879760742,
      "activations/layer1_attention_weight_max": 16.555768966674805,
      "activations/layer1_attention_weight_min": -16.750049591064453,
      "activations/layer20_attention_weight_max": 28.1077880859375,
      "activations/layer20_attention_weight_min": -26.77227783203125,
      "activations/layer21_attention_weight_max": 29.08526611328125,
      "activations/layer21_attention_weight_min": -27.77945899963379,
      "activations/layer22_attention_weight_max": 43.47507858276367,
      "activations/layer22_attention_weight_min": -35.146270751953125,
      "activations/layer23_attention_weight_max": 32.403804779052734,
      "activations/layer23_attention_weight_min": -26.914485931396484,
      "activations/layer2_attention_weight_max": 33.0020751953125,
      "activations/layer2_attention_weight_min": -33.229576110839844,
      "activations/layer3_attention_weight_max": 103.18360137939453,
      "activations/layer3_attention_weight_min": -106.99253845214844,
      "activations/layer4_attention_weight_max": 109.1339340209961,
      "activations/layer4_attention_weight_min": -111.38475799560547,
      "activations/layer5_attention_weight_max": 49.458518981933594,
      "activations/layer5_attention_weight_min": -61.54937744140625,
      "activations/layer6_attention_weight_max": 44.52731704711914,
      "activations/layer6_attention_weight_min": -47.48181915283203,
      "activations/layer7_attention_weight_max": 91.88390350341797,
      "activations/layer7_attention_weight_min": -99.1674575805664,
      "activations/layer8_attention_weight_max": 42.1096076965332,
      "activations/layer8_attention_weight_min": -44.608116149902344,
      "activations/layer9_attention_weight_max": 32.7288932800293,
      "activations/layer9_attention_weight_min": -34.7724494934082,
      "epoch": 22.1,
      "learning_rate": 7.5549242424242415e-06,
      "loss": 2.696,
      "step": 380350
    },
    {
      "activations/layer0_attention_weight_max": 16.885177612304688,
      "activations/layer0_attention_weight_min": -14.153953552246094,
      "activations/layer10_attention_weight_max": 36.08240509033203,
      "activations/layer10_attention_weight_min": -39.28749465942383,
      "activations/layer11_attention_weight_max": 36.27870178222656,
      "activations/layer11_attention_weight_min": -37.74071502685547,
      "activations/layer12_attention_weight_max": 29.456789016723633,
      "activations/layer12_attention_weight_min": -31.319480895996094,
      "activations/layer13_attention_weight_max": 44.231468200683594,
      "activations/layer13_attention_weight_min": -40.92774200439453,
      "activations/layer14_attention_weight_max": 46.20576095581055,
      "activations/layer14_attention_weight_min": -42.12224578857422,
      "activations/layer15_attention_weight_max": 43.78371810913086,
      "activations/layer15_attention_weight_min": -39.607967376708984,
      "activations/layer16_attention_weight_max": 32.455875396728516,
      "activations/layer16_attention_weight_min": -29.135332107543945,
      "activations/layer17_attention_weight_max": 34.07577133178711,
      "activations/layer17_attention_weight_min": -27.125450134277344,
      "activations/layer18_attention_weight_max": 34.81639099121094,
      "activations/layer18_attention_weight_min": -27.1756649017334,
      "activations/layer19_attention_weight_max": 36.64695358276367,
      "activations/layer19_attention_weight_min": -31.64275550842285,
      "activations/layer1_attention_weight_max": 16.903993606567383,
      "activations/layer1_attention_weight_min": -16.243806838989258,
      "activations/layer20_attention_weight_max": 31.66502571105957,
      "activations/layer20_attention_weight_min": -26.77718162536621,
      "activations/layer21_attention_weight_max": 30.98948860168457,
      "activations/layer21_attention_weight_min": -25.18756103515625,
      "activations/layer22_attention_weight_max": 46.15918731689453,
      "activations/layer22_attention_weight_min": -32.203269958496094,
      "activations/layer23_attention_weight_max": 32.720726013183594,
      "activations/layer23_attention_weight_min": -27.407085418701172,
      "activations/layer2_attention_weight_max": 34.19781494140625,
      "activations/layer2_attention_weight_min": -33.15736770629883,
      "activations/layer3_attention_weight_max": 106.00881958007812,
      "activations/layer3_attention_weight_min": -105.1107177734375,
      "activations/layer4_attention_weight_max": 113.3174819946289,
      "activations/layer4_attention_weight_min": -113.09880828857422,
      "activations/layer5_attention_weight_max": 52.365325927734375,
      "activations/layer5_attention_weight_min": -62.242027282714844,
      "activations/layer6_attention_weight_max": 47.655433654785156,
      "activations/layer6_attention_weight_min": -49.52570343017578,
      "activations/layer7_attention_weight_max": 100.77981567382812,
      "activations/layer7_attention_weight_min": -106.86906433105469,
      "activations/layer8_attention_weight_max": 43.45115661621094,
      "activations/layer8_attention_weight_min": -46.968441009521484,
      "activations/layer9_attention_weight_max": 31.900741577148438,
      "activations/layer9_attention_weight_min": -35.546749114990234,
      "epoch": 22.1,
      "learning_rate": 7.535984848484847e-06,
      "loss": 2.6907,
      "step": 380400
    },
    {
      "activations/layer0_attention_weight_max": 15.891669273376465,
      "activations/layer0_attention_weight_min": -14.042723655700684,
      "activations/layer10_attention_weight_max": 36.13782501220703,
      "activations/layer10_attention_weight_min": -36.476234436035156,
      "activations/layer11_attention_weight_max": 38.68783950805664,
      "activations/layer11_attention_weight_min": -38.22419738769531,
      "activations/layer12_attention_weight_max": 29.895296096801758,
      "activations/layer12_attention_weight_min": -28.40577507019043,
      "activations/layer13_attention_weight_max": 40.46711349487305,
      "activations/layer13_attention_weight_min": -35.995235443115234,
      "activations/layer14_attention_weight_max": 41.10105514526367,
      "activations/layer14_attention_weight_min": -38.948726654052734,
      "activations/layer15_attention_weight_max": 39.177223205566406,
      "activations/layer15_attention_weight_min": -35.317344665527344,
      "activations/layer16_attention_weight_max": 31.43117332458496,
      "activations/layer16_attention_weight_min": -27.444974899291992,
      "activations/layer17_attention_weight_max": 29.554935455322266,
      "activations/layer17_attention_weight_min": -26.39480972290039,
      "activations/layer18_attention_weight_max": 31.778554916381836,
      "activations/layer18_attention_weight_min": -27.033605575561523,
      "activations/layer19_attention_weight_max": 34.501224517822266,
      "activations/layer19_attention_weight_min": -32.66386032104492,
      "activations/layer1_attention_weight_max": 16.423274993896484,
      "activations/layer1_attention_weight_min": -15.95483112335205,
      "activations/layer20_attention_weight_max": 33.36357116699219,
      "activations/layer20_attention_weight_min": -27.28081703186035,
      "activations/layer21_attention_weight_max": 30.830411911010742,
      "activations/layer21_attention_weight_min": -25.988967895507812,
      "activations/layer22_attention_weight_max": 42.70096969604492,
      "activations/layer22_attention_weight_min": -34.58042907714844,
      "activations/layer23_attention_weight_max": 32.56513214111328,
      "activations/layer23_attention_weight_min": -27.0614013671875,
      "activations/layer2_attention_weight_max": 35.80168151855469,
      "activations/layer2_attention_weight_min": -33.922821044921875,
      "activations/layer3_attention_weight_max": 109.87173461914062,
      "activations/layer3_attention_weight_min": -111.82455444335938,
      "activations/layer4_attention_weight_max": 113.13639068603516,
      "activations/layer4_attention_weight_min": -117.05940246582031,
      "activations/layer5_attention_weight_max": 50.939186096191406,
      "activations/layer5_attention_weight_min": -63.91313171386719,
      "activations/layer6_attention_weight_max": 47.05752182006836,
      "activations/layer6_attention_weight_min": -51.42436981201172,
      "activations/layer7_attention_weight_max": 95.70402526855469,
      "activations/layer7_attention_weight_min": -104.39501953125,
      "activations/layer8_attention_weight_max": 42.43915939331055,
      "activations/layer8_attention_weight_min": -48.81796646118164,
      "activations/layer9_attention_weight_max": 34.2439079284668,
      "activations/layer9_attention_weight_min": -37.168819427490234,
      "epoch": 22.11,
      "learning_rate": 7.517045454545454e-06,
      "loss": 2.6976,
      "step": 380450
    },
    {
      "activations/layer0_attention_weight_max": 17.018163681030273,
      "activations/layer0_attention_weight_min": -13.382830619812012,
      "activations/layer10_attention_weight_max": 39.096046447753906,
      "activations/layer10_attention_weight_min": -40.73101806640625,
      "activations/layer11_attention_weight_max": 38.99724578857422,
      "activations/layer11_attention_weight_min": -43.301353454589844,
      "activations/layer12_attention_weight_max": 26.0323429107666,
      "activations/layer12_attention_weight_min": -27.22107696533203,
      "activations/layer13_attention_weight_max": 38.21921920776367,
      "activations/layer13_attention_weight_min": -37.70262908935547,
      "activations/layer14_attention_weight_max": 42.10015106201172,
      "activations/layer14_attention_weight_min": -39.07278823852539,
      "activations/layer15_attention_weight_max": 36.340904235839844,
      "activations/layer15_attention_weight_min": -38.55350112915039,
      "activations/layer16_attention_weight_max": 26.716567993164062,
      "activations/layer16_attention_weight_min": -29.119247436523438,
      "activations/layer17_attention_weight_max": 29.138139724731445,
      "activations/layer17_attention_weight_min": -26.00149917602539,
      "activations/layer18_attention_weight_max": 29.91799545288086,
      "activations/layer18_attention_weight_min": -25.11996078491211,
      "activations/layer19_attention_weight_max": 29.836063385009766,
      "activations/layer19_attention_weight_min": -30.948949813842773,
      "activations/layer1_attention_weight_max": 16.219554901123047,
      "activations/layer1_attention_weight_min": -15.425408363342285,
      "activations/layer20_attention_weight_max": 25.042224884033203,
      "activations/layer20_attention_weight_min": -24.324254989624023,
      "activations/layer21_attention_weight_max": 28.10842514038086,
      "activations/layer21_attention_weight_min": -24.63903045654297,
      "activations/layer22_attention_weight_max": 35.330039978027344,
      "activations/layer22_attention_weight_min": -30.908851623535156,
      "activations/layer23_attention_weight_max": 28.936309814453125,
      "activations/layer23_attention_weight_min": -26.563526153564453,
      "activations/layer2_attention_weight_max": 34.2853889465332,
      "activations/layer2_attention_weight_min": -33.76865768432617,
      "activations/layer3_attention_weight_max": 109.30899810791016,
      "activations/layer3_attention_weight_min": -107.89629364013672,
      "activations/layer4_attention_weight_max": 110.0175552368164,
      "activations/layer4_attention_weight_min": -113.99808502197266,
      "activations/layer5_attention_weight_max": 48.8271484375,
      "activations/layer5_attention_weight_min": -62.756771087646484,
      "activations/layer6_attention_weight_max": 44.948856353759766,
      "activations/layer6_attention_weight_min": -49.182960510253906,
      "activations/layer7_attention_weight_max": 99.29508209228516,
      "activations/layer7_attention_weight_min": -100.74085235595703,
      "activations/layer8_attention_weight_max": 44.21196365356445,
      "activations/layer8_attention_weight_min": -47.14002990722656,
      "activations/layer9_attention_weight_max": 36.86904525756836,
      "activations/layer9_attention_weight_min": -38.0673828125,
      "epoch": 22.11,
      "learning_rate": 7.49810606060606e-06,
      "loss": 2.7038,
      "step": 380500
    },
    {
      "activations/layer0_attention_weight_max": 16.996068954467773,
      "activations/layer0_attention_weight_min": -14.328279495239258,
      "activations/layer10_attention_weight_max": 38.37834548950195,
      "activations/layer10_attention_weight_min": -37.121334075927734,
      "activations/layer11_attention_weight_max": 35.89920425415039,
      "activations/layer11_attention_weight_min": -36.13691329956055,
      "activations/layer12_attention_weight_max": 30.49227523803711,
      "activations/layer12_attention_weight_min": -29.087358474731445,
      "activations/layer13_attention_weight_max": 43.771324157714844,
      "activations/layer13_attention_weight_min": -39.590084075927734,
      "activations/layer14_attention_weight_max": 46.687198638916016,
      "activations/layer14_attention_weight_min": -41.63778305053711,
      "activations/layer15_attention_weight_max": 42.59693908691406,
      "activations/layer15_attention_weight_min": -43.50627136230469,
      "activations/layer16_attention_weight_max": 30.906326293945312,
      "activations/layer16_attention_weight_min": -28.638519287109375,
      "activations/layer17_attention_weight_max": 30.926111221313477,
      "activations/layer17_attention_weight_min": -29.445709228515625,
      "activations/layer18_attention_weight_max": 31.746492385864258,
      "activations/layer18_attention_weight_min": -28.5787296295166,
      "activations/layer19_attention_weight_max": 36.855712890625,
      "activations/layer19_attention_weight_min": -30.245304107666016,
      "activations/layer1_attention_weight_max": 16.788572311401367,
      "activations/layer1_attention_weight_min": -15.802502632141113,
      "activations/layer20_attention_weight_max": 27.784502029418945,
      "activations/layer20_attention_weight_min": -25.87541961669922,
      "activations/layer21_attention_weight_max": 27.450313568115234,
      "activations/layer21_attention_weight_min": -25.290822982788086,
      "activations/layer22_attention_weight_max": 41.19247055053711,
      "activations/layer22_attention_weight_min": -32.77634811401367,
      "activations/layer23_attention_weight_max": 32.80742645263672,
      "activations/layer23_attention_weight_min": -28.34759521484375,
      "activations/layer2_attention_weight_max": 35.37578582763672,
      "activations/layer2_attention_weight_min": -35.34904479980469,
      "activations/layer3_attention_weight_max": 112.21259307861328,
      "activations/layer3_attention_weight_min": -112.36360931396484,
      "activations/layer4_attention_weight_max": 115.7859878540039,
      "activations/layer4_attention_weight_min": -114.30030822753906,
      "activations/layer5_attention_weight_max": 51.42498779296875,
      "activations/layer5_attention_weight_min": -61.796791076660156,
      "activations/layer6_attention_weight_max": 46.608436584472656,
      "activations/layer6_attention_weight_min": -46.71378707885742,
      "activations/layer7_attention_weight_max": 101.84385681152344,
      "activations/layer7_attention_weight_min": -105.24836730957031,
      "activations/layer8_attention_weight_max": 45.47942352294922,
      "activations/layer8_attention_weight_min": -46.5101203918457,
      "activations/layer9_attention_weight_max": 34.85960388183594,
      "activations/layer9_attention_weight_min": -35.939815521240234,
      "epoch": 22.11,
      "learning_rate": 7.479166666666666e-06,
      "loss": 2.7155,
      "step": 380550
    },
    {
      "activations/layer0_attention_weight_max": 16.992401123046875,
      "activations/layer0_attention_weight_min": -14.216424942016602,
      "activations/layer10_attention_weight_max": 34.71217346191406,
      "activations/layer10_attention_weight_min": -34.25761795043945,
      "activations/layer11_attention_weight_max": 35.61294174194336,
      "activations/layer11_attention_weight_min": -34.70401382446289,
      "activations/layer12_attention_weight_max": 25.50425148010254,
      "activations/layer12_attention_weight_min": -29.915674209594727,
      "activations/layer13_attention_weight_max": 37.88038635253906,
      "activations/layer13_attention_weight_min": -36.18540954589844,
      "activations/layer14_attention_weight_max": 41.078392028808594,
      "activations/layer14_attention_weight_min": -40.20724868774414,
      "activations/layer15_attention_weight_max": 39.01959991455078,
      "activations/layer15_attention_weight_min": -36.24961471557617,
      "activations/layer16_attention_weight_max": 31.56184959411621,
      "activations/layer16_attention_weight_min": -31.58452033996582,
      "activations/layer17_attention_weight_max": 29.104204177856445,
      "activations/layer17_attention_weight_min": -26.573486328125,
      "activations/layer18_attention_weight_max": 31.674480438232422,
      "activations/layer18_attention_weight_min": -27.178325653076172,
      "activations/layer19_attention_weight_max": 30.364139556884766,
      "activations/layer19_attention_weight_min": -32.33726119995117,
      "activations/layer1_attention_weight_max": 15.66988468170166,
      "activations/layer1_attention_weight_min": -14.791934967041016,
      "activations/layer20_attention_weight_max": 26.351171493530273,
      "activations/layer20_attention_weight_min": -27.37309455871582,
      "activations/layer21_attention_weight_max": 26.007986068725586,
      "activations/layer21_attention_weight_min": -25.16853904724121,
      "activations/layer22_attention_weight_max": 40.10683059692383,
      "activations/layer22_attention_weight_min": -31.554847717285156,
      "activations/layer23_attention_weight_max": 29.924823760986328,
      "activations/layer23_attention_weight_min": -24.31826400756836,
      "activations/layer2_attention_weight_max": 33.5893669128418,
      "activations/layer2_attention_weight_min": -33.48120880126953,
      "activations/layer3_attention_weight_max": 109.62953186035156,
      "activations/layer3_attention_weight_min": -116.95481872558594,
      "activations/layer4_attention_weight_max": 113.5057601928711,
      "activations/layer4_attention_weight_min": -115.45206451416016,
      "activations/layer5_attention_weight_max": 49.824432373046875,
      "activations/layer5_attention_weight_min": -62.08903503417969,
      "activations/layer6_attention_weight_max": 47.51091384887695,
      "activations/layer6_attention_weight_min": -47.86228942871094,
      "activations/layer7_attention_weight_max": 96.51443481445312,
      "activations/layer7_attention_weight_min": -100.2197036743164,
      "activations/layer8_attention_weight_max": 44.381465911865234,
      "activations/layer8_attention_weight_min": -44.933990478515625,
      "activations/layer9_attention_weight_max": 33.992000579833984,
      "activations/layer9_attention_weight_min": -34.67194747924805,
      "epoch": 22.12,
      "learning_rate": 7.460227272727273e-06,
      "loss": 2.6982,
      "step": 380600
    },
    {
      "activations/layer0_attention_weight_max": 16.11747932434082,
      "activations/layer0_attention_weight_min": -14.628946304321289,
      "activations/layer10_attention_weight_max": 33.853939056396484,
      "activations/layer10_attention_weight_min": -34.8200569152832,
      "activations/layer11_attention_weight_max": 34.53527069091797,
      "activations/layer11_attention_weight_min": -35.45136260986328,
      "activations/layer12_attention_weight_max": 27.463945388793945,
      "activations/layer12_attention_weight_min": -27.08425521850586,
      "activations/layer13_attention_weight_max": 42.43864822387695,
      "activations/layer13_attention_weight_min": -36.130104064941406,
      "activations/layer14_attention_weight_max": 43.661231994628906,
      "activations/layer14_attention_weight_min": -36.46841049194336,
      "activations/layer15_attention_weight_max": 41.796974182128906,
      "activations/layer15_attention_weight_min": -35.652099609375,
      "activations/layer16_attention_weight_max": 31.076868057250977,
      "activations/layer16_attention_weight_min": -27.56032371520996,
      "activations/layer17_attention_weight_max": 28.275962829589844,
      "activations/layer17_attention_weight_min": -25.24287986755371,
      "activations/layer18_attention_weight_max": 30.593603134155273,
      "activations/layer18_attention_weight_min": -26.49138832092285,
      "activations/layer19_attention_weight_max": 31.961013793945312,
      "activations/layer19_attention_weight_min": -33.15766143798828,
      "activations/layer1_attention_weight_max": 15.940180778503418,
      "activations/layer1_attention_weight_min": -14.285940170288086,
      "activations/layer20_attention_weight_max": 27.968353271484375,
      "activations/layer20_attention_weight_min": -27.518470764160156,
      "activations/layer21_attention_weight_max": 29.160783767700195,
      "activations/layer21_attention_weight_min": -25.010080337524414,
      "activations/layer22_attention_weight_max": 41.78876495361328,
      "activations/layer22_attention_weight_min": -32.7124137878418,
      "activations/layer23_attention_weight_max": 31.90167808532715,
      "activations/layer23_attention_weight_min": -26.985370635986328,
      "activations/layer2_attention_weight_max": 34.177154541015625,
      "activations/layer2_attention_weight_min": -32.84577941894531,
      "activations/layer3_attention_weight_max": 105.34548950195312,
      "activations/layer3_attention_weight_min": -107.44881439208984,
      "activations/layer4_attention_weight_max": 112.40673828125,
      "activations/layer4_attention_weight_min": -113.32783508300781,
      "activations/layer5_attention_weight_max": 54.4194450378418,
      "activations/layer5_attention_weight_min": -68.69509887695312,
      "activations/layer6_attention_weight_max": 44.27876663208008,
      "activations/layer6_attention_weight_min": -47.48106384277344,
      "activations/layer7_attention_weight_max": 94.44003295898438,
      "activations/layer7_attention_weight_min": -97.95069122314453,
      "activations/layer8_attention_weight_max": 42.09755325317383,
      "activations/layer8_attention_weight_min": -44.186466217041016,
      "activations/layer9_attention_weight_max": 33.33514404296875,
      "activations/layer9_attention_weight_min": -34.61619567871094,
      "epoch": 22.12,
      "learning_rate": 7.441287878787878e-06,
      "loss": 2.6772,
      "step": 380650
    },
    {
      "activations/layer0_attention_weight_max": 17.210044860839844,
      "activations/layer0_attention_weight_min": -14.062230110168457,
      "activations/layer10_attention_weight_max": 31.32669448852539,
      "activations/layer10_attention_weight_min": -34.41846466064453,
      "activations/layer11_attention_weight_max": 31.786026000976562,
      "activations/layer11_attention_weight_min": -33.12732696533203,
      "activations/layer12_attention_weight_max": 26.898059844970703,
      "activations/layer12_attention_weight_min": -25.42230987548828,
      "activations/layer13_attention_weight_max": 38.49195098876953,
      "activations/layer13_attention_weight_min": -35.38316345214844,
      "activations/layer14_attention_weight_max": 46.61980056762695,
      "activations/layer14_attention_weight_min": -37.6351432800293,
      "activations/layer15_attention_weight_max": 41.134246826171875,
      "activations/layer15_attention_weight_min": -37.655609130859375,
      "activations/layer16_attention_weight_max": 32.51508712768555,
      "activations/layer16_attention_weight_min": -30.554155349731445,
      "activations/layer17_attention_weight_max": 28.730079650878906,
      "activations/layer17_attention_weight_min": -26.34013557434082,
      "activations/layer18_attention_weight_max": 30.73655891418457,
      "activations/layer18_attention_weight_min": -25.154306411743164,
      "activations/layer19_attention_weight_max": 32.588783264160156,
      "activations/layer19_attention_weight_min": -31.33558464050293,
      "activations/layer1_attention_weight_max": 16.50166130065918,
      "activations/layer1_attention_weight_min": -14.102088928222656,
      "activations/layer20_attention_weight_max": 27.286394119262695,
      "activations/layer20_attention_weight_min": -26.052766799926758,
      "activations/layer21_attention_weight_max": 27.07415008544922,
      "activations/layer21_attention_weight_min": -25.96552085876465,
      "activations/layer22_attention_weight_max": 38.97400665283203,
      "activations/layer22_attention_weight_min": -33.07575607299805,
      "activations/layer23_attention_weight_max": 30.65143585205078,
      "activations/layer23_attention_weight_min": -24.9102840423584,
      "activations/layer2_attention_weight_max": 32.86739730834961,
      "activations/layer2_attention_weight_min": -32.87747573852539,
      "activations/layer3_attention_weight_max": 105.48355865478516,
      "activations/layer3_attention_weight_min": -104.2542495727539,
      "activations/layer4_attention_weight_max": 110.1263198852539,
      "activations/layer4_attention_weight_min": -115.03942108154297,
      "activations/layer5_attention_weight_max": 47.89173126220703,
      "activations/layer5_attention_weight_min": -63.765499114990234,
      "activations/layer6_attention_weight_max": 45.94548797607422,
      "activations/layer6_attention_weight_min": -49.08999252319336,
      "activations/layer7_attention_weight_max": 91.75128173828125,
      "activations/layer7_attention_weight_min": -98.66890716552734,
      "activations/layer8_attention_weight_max": 40.72676467895508,
      "activations/layer8_attention_weight_min": -43.465118408203125,
      "activations/layer9_attention_weight_max": 30.20411491394043,
      "activations/layer9_attention_weight_min": -34.51668930053711,
      "epoch": 22.12,
      "learning_rate": 7.422348484848484e-06,
      "loss": 2.7037,
      "step": 380700
    },
    {
      "activations/layer0_attention_weight_max": 16.4359188079834,
      "activations/layer0_attention_weight_min": -13.604057312011719,
      "activations/layer10_attention_weight_max": 35.17578887939453,
      "activations/layer10_attention_weight_min": -36.557830810546875,
      "activations/layer11_attention_weight_max": 35.62249755859375,
      "activations/layer11_attention_weight_min": -37.190887451171875,
      "activations/layer12_attention_weight_max": 27.789457321166992,
      "activations/layer12_attention_weight_min": -28.14983367919922,
      "activations/layer13_attention_weight_max": 42.067413330078125,
      "activations/layer13_attention_weight_min": -36.510536193847656,
      "activations/layer14_attention_weight_max": 43.35719299316406,
      "activations/layer14_attention_weight_min": -37.54134750366211,
      "activations/layer15_attention_weight_max": 39.54890060424805,
      "activations/layer15_attention_weight_min": -36.66033935546875,
      "activations/layer16_attention_weight_max": 34.251739501953125,
      "activations/layer16_attention_weight_min": -27.84811782836914,
      "activations/layer17_attention_weight_max": 33.059364318847656,
      "activations/layer17_attention_weight_min": -26.318336486816406,
      "activations/layer18_attention_weight_max": 38.31122970581055,
      "activations/layer18_attention_weight_min": -27.02067756652832,
      "activations/layer19_attention_weight_max": 40.75852584838867,
      "activations/layer19_attention_weight_min": -32.87760543823242,
      "activations/layer1_attention_weight_max": 15.368090629577637,
      "activations/layer1_attention_weight_min": -15.254857063293457,
      "activations/layer20_attention_weight_max": 31.942127227783203,
      "activations/layer20_attention_weight_min": -25.64357566833496,
      "activations/layer21_attention_weight_max": 31.947233200073242,
      "activations/layer21_attention_weight_min": -25.488201141357422,
      "activations/layer22_attention_weight_max": 55.530517578125,
      "activations/layer22_attention_weight_min": -33.4588623046875,
      "activations/layer23_attention_weight_max": 36.49060821533203,
      "activations/layer23_attention_weight_min": -26.857257843017578,
      "activations/layer2_attention_weight_max": 35.95084762573242,
      "activations/layer2_attention_weight_min": -34.63084411621094,
      "activations/layer3_attention_weight_max": 107.75907135009766,
      "activations/layer3_attention_weight_min": -109.81839752197266,
      "activations/layer4_attention_weight_max": 109.30925750732422,
      "activations/layer4_attention_weight_min": -112.8047866821289,
      "activations/layer5_attention_weight_max": 51.78916931152344,
      "activations/layer5_attention_weight_min": -62.2518310546875,
      "activations/layer6_attention_weight_max": 46.78504180908203,
      "activations/layer6_attention_weight_min": -49.37803649902344,
      "activations/layer7_attention_weight_max": 103.98538208007812,
      "activations/layer7_attention_weight_min": -102.7327651977539,
      "activations/layer8_attention_weight_max": 43.60166931152344,
      "activations/layer8_attention_weight_min": -46.587120056152344,
      "activations/layer9_attention_weight_max": 33.69633102416992,
      "activations/layer9_attention_weight_min": -36.686195373535156,
      "epoch": 22.12,
      "learning_rate": 7.40340909090909e-06,
      "loss": 2.6939,
      "step": 380750
    },
    {
      "activations/layer0_attention_weight_max": 17.09851837158203,
      "activations/layer0_attention_weight_min": -14.083259582519531,
      "activations/layer10_attention_weight_max": 35.32575988769531,
      "activations/layer10_attention_weight_min": -36.66153335571289,
      "activations/layer11_attention_weight_max": 35.681968688964844,
      "activations/layer11_attention_weight_min": -35.02846908569336,
      "activations/layer12_attention_weight_max": 28.155855178833008,
      "activations/layer12_attention_weight_min": -26.996782302856445,
      "activations/layer13_attention_weight_max": 42.1285514831543,
      "activations/layer13_attention_weight_min": -35.15739440917969,
      "activations/layer14_attention_weight_max": 43.042938232421875,
      "activations/layer14_attention_weight_min": -38.16046905517578,
      "activations/layer15_attention_weight_max": 38.22651290893555,
      "activations/layer15_attention_weight_min": -37.3575553894043,
      "activations/layer16_attention_weight_max": 27.697587966918945,
      "activations/layer16_attention_weight_min": -29.244077682495117,
      "activations/layer17_attention_weight_max": 28.81930160522461,
      "activations/layer17_attention_weight_min": -25.323129653930664,
      "activations/layer18_attention_weight_max": 32.49249267578125,
      "activations/layer18_attention_weight_min": -25.77182960510254,
      "activations/layer19_attention_weight_max": 34.57164764404297,
      "activations/layer19_attention_weight_min": -32.340152740478516,
      "activations/layer1_attention_weight_max": 15.86109447479248,
      "activations/layer1_attention_weight_min": -14.920085906982422,
      "activations/layer20_attention_weight_max": 31.589500427246094,
      "activations/layer20_attention_weight_min": -25.841073989868164,
      "activations/layer21_attention_weight_max": 31.493520736694336,
      "activations/layer21_attention_weight_min": -24.988183975219727,
      "activations/layer22_attention_weight_max": 42.91843032836914,
      "activations/layer22_attention_weight_min": -29.867353439331055,
      "activations/layer23_attention_weight_max": 33.949806213378906,
      "activations/layer23_attention_weight_min": -26.668418884277344,
      "activations/layer2_attention_weight_max": 33.40232849121094,
      "activations/layer2_attention_weight_min": -33.36252975463867,
      "activations/layer3_attention_weight_max": 104.8876724243164,
      "activations/layer3_attention_weight_min": -107.19842529296875,
      "activations/layer4_attention_weight_max": 108.9046859741211,
      "activations/layer4_attention_weight_min": -114.32549285888672,
      "activations/layer5_attention_weight_max": 52.758201599121094,
      "activations/layer5_attention_weight_min": -63.532833099365234,
      "activations/layer6_attention_weight_max": 45.0853385925293,
      "activations/layer6_attention_weight_min": -50.458717346191406,
      "activations/layer7_attention_weight_max": 95.86202239990234,
      "activations/layer7_attention_weight_min": -102.31033325195312,
      "activations/layer8_attention_weight_max": 41.42191696166992,
      "activations/layer8_attention_weight_min": -47.4799919128418,
      "activations/layer9_attention_weight_max": 32.24506378173828,
      "activations/layer9_attention_weight_min": -34.567256927490234,
      "epoch": 22.13,
      "learning_rate": 7.384469696969696e-06,
      "loss": 2.6926,
      "step": 380800
    },
    {
      "activations/layer0_attention_weight_max": 16.53307342529297,
      "activations/layer0_attention_weight_min": -13.973344802856445,
      "activations/layer10_attention_weight_max": 35.09467315673828,
      "activations/layer10_attention_weight_min": -34.961116790771484,
      "activations/layer11_attention_weight_max": 33.39271926879883,
      "activations/layer11_attention_weight_min": -34.22673797607422,
      "activations/layer12_attention_weight_max": 27.344043731689453,
      "activations/layer12_attention_weight_min": -27.40468978881836,
      "activations/layer13_attention_weight_max": 40.65432357788086,
      "activations/layer13_attention_weight_min": -35.16468048095703,
      "activations/layer14_attention_weight_max": 42.80485534667969,
      "activations/layer14_attention_weight_min": -38.28483200073242,
      "activations/layer15_attention_weight_max": 41.703277587890625,
      "activations/layer15_attention_weight_min": -35.653079986572266,
      "activations/layer16_attention_weight_max": 28.853891372680664,
      "activations/layer16_attention_weight_min": -30.10404396057129,
      "activations/layer17_attention_weight_max": 30.13319969177246,
      "activations/layer17_attention_weight_min": -26.399168014526367,
      "activations/layer18_attention_weight_max": 32.40910720825195,
      "activations/layer18_attention_weight_min": -27.963401794433594,
      "activations/layer19_attention_weight_max": 36.29814910888672,
      "activations/layer19_attention_weight_min": -32.547611236572266,
      "activations/layer1_attention_weight_max": 16.186145782470703,
      "activations/layer1_attention_weight_min": -14.986228942871094,
      "activations/layer20_attention_weight_max": 28.824079513549805,
      "activations/layer20_attention_weight_min": -26.412670135498047,
      "activations/layer21_attention_weight_max": 30.088945388793945,
      "activations/layer21_attention_weight_min": -25.45067024230957,
      "activations/layer22_attention_weight_max": 40.38778305053711,
      "activations/layer22_attention_weight_min": -34.006248474121094,
      "activations/layer23_attention_weight_max": 29.700393676757812,
      "activations/layer23_attention_weight_min": -26.57379722595215,
      "activations/layer2_attention_weight_max": 33.54496765136719,
      "activations/layer2_attention_weight_min": -34.38799285888672,
      "activations/layer3_attention_weight_max": 104.96993255615234,
      "activations/layer3_attention_weight_min": -113.92546844482422,
      "activations/layer4_attention_weight_max": 111.18756103515625,
      "activations/layer4_attention_weight_min": -114.49149322509766,
      "activations/layer5_attention_weight_max": 49.56876754760742,
      "activations/layer5_attention_weight_min": -66.17430114746094,
      "activations/layer6_attention_weight_max": 45.05225372314453,
      "activations/layer6_attention_weight_min": -48.42867660522461,
      "activations/layer7_attention_weight_max": 95.07463073730469,
      "activations/layer7_attention_weight_min": -101.612060546875,
      "activations/layer8_attention_weight_max": 42.227882385253906,
      "activations/layer8_attention_weight_min": -47.58284378051758,
      "activations/layer9_attention_weight_max": 31.596622467041016,
      "activations/layer9_attention_weight_min": -33.95451354980469,
      "epoch": 22.13,
      "learning_rate": 7.365530303030302e-06,
      "loss": 2.6942,
      "step": 380850
    },
    {
      "activations/layer0_attention_weight_max": 16.96224021911621,
      "activations/layer0_attention_weight_min": -14.195724487304688,
      "activations/layer10_attention_weight_max": 38.681884765625,
      "activations/layer10_attention_weight_min": -39.982757568359375,
      "activations/layer11_attention_weight_max": 39.04362487792969,
      "activations/layer11_attention_weight_min": -41.69385528564453,
      "activations/layer12_attention_weight_max": 29.6033992767334,
      "activations/layer12_attention_weight_min": -28.048646926879883,
      "activations/layer13_attention_weight_max": 42.54954528808594,
      "activations/layer13_attention_weight_min": -37.439292907714844,
      "activations/layer14_attention_weight_max": 42.62555694580078,
      "activations/layer14_attention_weight_min": -38.456626892089844,
      "activations/layer15_attention_weight_max": 40.35930633544922,
      "activations/layer15_attention_weight_min": -36.51559066772461,
      "activations/layer16_attention_weight_max": 32.11792755126953,
      "activations/layer16_attention_weight_min": -29.861358642578125,
      "activations/layer17_attention_weight_max": 29.470035552978516,
      "activations/layer17_attention_weight_min": -27.97233009338379,
      "activations/layer18_attention_weight_max": 33.87895965576172,
      "activations/layer18_attention_weight_min": -25.5718936920166,
      "activations/layer19_attention_weight_max": 31.792165756225586,
      "activations/layer19_attention_weight_min": -31.005380630493164,
      "activations/layer1_attention_weight_max": 16.17860984802246,
      "activations/layer1_attention_weight_min": -14.591416358947754,
      "activations/layer20_attention_weight_max": 27.41012191772461,
      "activations/layer20_attention_weight_min": -23.786579132080078,
      "activations/layer21_attention_weight_max": 27.198917388916016,
      "activations/layer21_attention_weight_min": -23.939054489135742,
      "activations/layer22_attention_weight_max": 41.62112808227539,
      "activations/layer22_attention_weight_min": -30.723236083984375,
      "activations/layer23_attention_weight_max": 31.022911071777344,
      "activations/layer23_attention_weight_min": -26.07706069946289,
      "activations/layer2_attention_weight_max": 35.23151397705078,
      "activations/layer2_attention_weight_min": -35.42698287963867,
      "activations/layer3_attention_weight_max": 109.04635620117188,
      "activations/layer3_attention_weight_min": -112.61965942382812,
      "activations/layer4_attention_weight_max": 112.48779296875,
      "activations/layer4_attention_weight_min": -118.57057189941406,
      "activations/layer5_attention_weight_max": 51.64111328125,
      "activations/layer5_attention_weight_min": -63.4613151550293,
      "activations/layer6_attention_weight_max": 45.937538146972656,
      "activations/layer6_attention_weight_min": -47.61211013793945,
      "activations/layer7_attention_weight_max": 103.5842056274414,
      "activations/layer7_attention_weight_min": -110.99480438232422,
      "activations/layer8_attention_weight_max": 47.41447448730469,
      "activations/layer8_attention_weight_min": -50.416996002197266,
      "activations/layer9_attention_weight_max": 34.42110824584961,
      "activations/layer9_attention_weight_min": -38.3134880065918,
      "epoch": 22.13,
      "learning_rate": 7.346590909090909e-06,
      "loss": 2.6855,
      "step": 380900
    },
    {
      "activations/layer0_attention_weight_max": 15.375089645385742,
      "activations/layer0_attention_weight_min": -13.022356033325195,
      "activations/layer10_attention_weight_max": 41.42310333251953,
      "activations/layer10_attention_weight_min": -42.12918472290039,
      "activations/layer11_attention_weight_max": 40.321327209472656,
      "activations/layer11_attention_weight_min": -41.407005310058594,
      "activations/layer12_attention_weight_max": 28.655317306518555,
      "activations/layer12_attention_weight_min": -29.168272018432617,
      "activations/layer13_attention_weight_max": 42.88044738769531,
      "activations/layer13_attention_weight_min": -38.44839096069336,
      "activations/layer14_attention_weight_max": 46.7624397277832,
      "activations/layer14_attention_weight_min": -39.0761604309082,
      "activations/layer15_attention_weight_max": 42.04473114013672,
      "activations/layer15_attention_weight_min": -36.105953216552734,
      "activations/layer16_attention_weight_max": 31.817094802856445,
      "activations/layer16_attention_weight_min": -28.295991897583008,
      "activations/layer17_attention_weight_max": 29.744266510009766,
      "activations/layer17_attention_weight_min": -26.4737548828125,
      "activations/layer18_attention_weight_max": 33.17301559448242,
      "activations/layer18_attention_weight_min": -28.439579010009766,
      "activations/layer19_attention_weight_max": 39.42182159423828,
      "activations/layer19_attention_weight_min": -31.39417839050293,
      "activations/layer1_attention_weight_max": 16.31920623779297,
      "activations/layer1_attention_weight_min": -15.316634178161621,
      "activations/layer20_attention_weight_max": 29.740161895751953,
      "activations/layer20_attention_weight_min": -25.197853088378906,
      "activations/layer21_attention_weight_max": 29.280710220336914,
      "activations/layer21_attention_weight_min": -22.60890769958496,
      "activations/layer22_attention_weight_max": 42.056922912597656,
      "activations/layer22_attention_weight_min": -33.077354431152344,
      "activations/layer23_attention_weight_max": 34.6220703125,
      "activations/layer23_attention_weight_min": -23.91356658935547,
      "activations/layer2_attention_weight_max": 33.55309295654297,
      "activations/layer2_attention_weight_min": -31.752012252807617,
      "activations/layer3_attention_weight_max": 109.0259780883789,
      "activations/layer3_attention_weight_min": -108.12107849121094,
      "activations/layer4_attention_weight_max": 114.6653823852539,
      "activations/layer4_attention_weight_min": -109.65677642822266,
      "activations/layer5_attention_weight_max": 53.51545715332031,
      "activations/layer5_attention_weight_min": -61.11808776855469,
      "activations/layer6_attention_weight_max": 48.79379653930664,
      "activations/layer6_attention_weight_min": -51.49878692626953,
      "activations/layer7_attention_weight_max": 104.54468536376953,
      "activations/layer7_attention_weight_min": -110.12211608886719,
      "activations/layer8_attention_weight_max": 49.74653625488281,
      "activations/layer8_attention_weight_min": -53.95630645751953,
      "activations/layer9_attention_weight_max": 37.892696380615234,
      "activations/layer9_attention_weight_min": -42.34811782836914,
      "epoch": 22.14,
      "learning_rate": 7.3276515151515144e-06,
      "loss": 2.705,
      "step": 380950
    },
    {
      "activations/layer0_attention_weight_max": 16.88986587524414,
      "activations/layer0_attention_weight_min": -14.9600830078125,
      "activations/layer10_attention_weight_max": 34.98262405395508,
      "activations/layer10_attention_weight_min": -38.58816909790039,
      "activations/layer11_attention_weight_max": 35.13984298706055,
      "activations/layer11_attention_weight_min": -38.02772903442383,
      "activations/layer12_attention_weight_max": 27.727313995361328,
      "activations/layer12_attention_weight_min": -27.959659576416016,
      "activations/layer13_attention_weight_max": 41.24161911010742,
      "activations/layer13_attention_weight_min": -36.02732467651367,
      "activations/layer14_attention_weight_max": 43.14508056640625,
      "activations/layer14_attention_weight_min": -39.907501220703125,
      "activations/layer15_attention_weight_max": 44.25498962402344,
      "activations/layer15_attention_weight_min": -36.87159729003906,
      "activations/layer16_attention_weight_max": 29.202735900878906,
      "activations/layer16_attention_weight_min": -27.33722686767578,
      "activations/layer17_attention_weight_max": 31.64035415649414,
      "activations/layer17_attention_weight_min": -27.315954208374023,
      "activations/layer18_attention_weight_max": 33.04743194580078,
      "activations/layer18_attention_weight_min": -27.95050048828125,
      "activations/layer19_attention_weight_max": 35.69587707519531,
      "activations/layer19_attention_weight_min": -36.87699890136719,
      "activations/layer1_attention_weight_max": 15.987471580505371,
      "activations/layer1_attention_weight_min": -16.585704803466797,
      "activations/layer20_attention_weight_max": 29.029624938964844,
      "activations/layer20_attention_weight_min": -26.7913761138916,
      "activations/layer21_attention_weight_max": 27.315227508544922,
      "activations/layer21_attention_weight_min": -26.491607666015625,
      "activations/layer22_attention_weight_max": 47.218955993652344,
      "activations/layer22_attention_weight_min": -35.11069107055664,
      "activations/layer23_attention_weight_max": 32.76288604736328,
      "activations/layer23_attention_weight_min": -28.496776580810547,
      "activations/layer2_attention_weight_max": 34.67683410644531,
      "activations/layer2_attention_weight_min": -34.070709228515625,
      "activations/layer3_attention_weight_max": 107.72686004638672,
      "activations/layer3_attention_weight_min": -108.7515869140625,
      "activations/layer4_attention_weight_max": 110.44380950927734,
      "activations/layer4_attention_weight_min": -113.78761291503906,
      "activations/layer5_attention_weight_max": 50.85506820678711,
      "activations/layer5_attention_weight_min": -62.58810043334961,
      "activations/layer6_attention_weight_max": 46.397403717041016,
      "activations/layer6_attention_weight_min": -47.79361343383789,
      "activations/layer7_attention_weight_max": 96.32687377929688,
      "activations/layer7_attention_weight_min": -109.25439453125,
      "activations/layer8_attention_weight_max": 41.60593795776367,
      "activations/layer8_attention_weight_min": -47.39101028442383,
      "activations/layer9_attention_weight_max": 31.129514694213867,
      "activations/layer9_attention_weight_min": -35.75581741333008,
      "epoch": 22.14,
      "learning_rate": 7.308712121212121e-06,
      "loss": 2.6995,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4896,
      "eval_samples_per_second": 505.798,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4896,
      "eval_openwebtext_samples_per_second": 505.798,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9507,
      "eval_wikitext_samples_per_second": 233.757,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_lambada_loss": 2.287109375,
      "eval_lambada_ppl": 9.846434155787346,
      "eval_lambada_runtime": 9.5362,
      "eval_lambada_samples_per_second": 510.58,
      "step": 381000
    },
    {
      "activations/layer0_attention_weight_max": 17.00261878967285,
      "activations/layer0_attention_weight_min": -14.202010154724121,
      "activations/layer10_attention_weight_max": 36.49126434326172,
      "activations/layer10_attention_weight_min": -36.93655776977539,
      "activations/layer11_attention_weight_max": 34.9077262878418,
      "activations/layer11_attention_weight_min": -36.85352325439453,
      "activations/layer12_attention_weight_max": 26.019384384155273,
      "activations/layer12_attention_weight_min": -28.94874382019043,
      "activations/layer13_attention_weight_max": 38.07093048095703,
      "activations/layer13_attention_weight_min": -38.89301681518555,
      "activations/layer14_attention_weight_max": 39.31343078613281,
      "activations/layer14_attention_weight_min": -39.02018737792969,
      "activations/layer15_attention_weight_max": 37.589969635009766,
      "activations/layer15_attention_weight_min": -36.044837951660156,
      "activations/layer16_attention_weight_max": 28.007966995239258,
      "activations/layer16_attention_weight_min": -27.72086524963379,
      "activations/layer17_attention_weight_max": 29.562416076660156,
      "activations/layer17_attention_weight_min": -25.72640037536621,
      "activations/layer18_attention_weight_max": 30.288164138793945,
      "activations/layer18_attention_weight_min": -26.019851684570312,
      "activations/layer19_attention_weight_max": 32.893489837646484,
      "activations/layer19_attention_weight_min": -29.992048263549805,
      "activations/layer1_attention_weight_max": 15.386529922485352,
      "activations/layer1_attention_weight_min": -15.536921501159668,
      "activations/layer20_attention_weight_max": 26.52632713317871,
      "activations/layer20_attention_weight_min": -24.8083553314209,
      "activations/layer21_attention_weight_max": 26.336870193481445,
      "activations/layer21_attention_weight_min": -24.300182342529297,
      "activations/layer22_attention_weight_max": 38.829368591308594,
      "activations/layer22_attention_weight_min": -30.983247756958008,
      "activations/layer23_attention_weight_max": 29.30941390991211,
      "activations/layer23_attention_weight_min": -25.75654411315918,
      "activations/layer2_attention_weight_max": 32.886741638183594,
      "activations/layer2_attention_weight_min": -32.06709671020508,
      "activations/layer3_attention_weight_max": 107.07984161376953,
      "activations/layer3_attention_weight_min": -108.86763000488281,
      "activations/layer4_attention_weight_max": 112.7022476196289,
      "activations/layer4_attention_weight_min": -116.95271301269531,
      "activations/layer5_attention_weight_max": 52.69795227050781,
      "activations/layer5_attention_weight_min": -64.51192474365234,
      "activations/layer6_attention_weight_max": 49.65891647338867,
      "activations/layer6_attention_weight_min": -52.64487838745117,
      "activations/layer7_attention_weight_max": 101.07915496826172,
      "activations/layer7_attention_weight_min": -107.79879760742188,
      "activations/layer8_attention_weight_max": 45.958438873291016,
      "activations/layer8_attention_weight_min": -51.5840950012207,
      "activations/layer9_attention_weight_max": 36.70560836791992,
      "activations/layer9_attention_weight_min": -38.603057861328125,
      "epoch": 22.14,
      "learning_rate": 7.289772727272727e-06,
      "loss": 2.7098,
      "step": 381050
    },
    {
      "activations/layer0_attention_weight_max": 15.73263931274414,
      "activations/layer0_attention_weight_min": -13.327540397644043,
      "activations/layer10_attention_weight_max": 35.37873840332031,
      "activations/layer10_attention_weight_min": -37.02460861206055,
      "activations/layer11_attention_weight_max": 37.72929382324219,
      "activations/layer11_attention_weight_min": -38.27040100097656,
      "activations/layer12_attention_weight_max": 27.279525756835938,
      "activations/layer12_attention_weight_min": -26.21746063232422,
      "activations/layer13_attention_weight_max": 41.687129974365234,
      "activations/layer13_attention_weight_min": -36.1839714050293,
      "activations/layer14_attention_weight_max": 46.20866012573242,
      "activations/layer14_attention_weight_min": -41.04896545410156,
      "activations/layer15_attention_weight_max": 42.19919204711914,
      "activations/layer15_attention_weight_min": -39.37916946411133,
      "activations/layer16_attention_weight_max": 32.08013153076172,
      "activations/layer16_attention_weight_min": -31.189884185791016,
      "activations/layer17_attention_weight_max": 30.508026123046875,
      "activations/layer17_attention_weight_min": -28.848588943481445,
      "activations/layer18_attention_weight_max": 31.610206604003906,
      "activations/layer18_attention_weight_min": -26.425838470458984,
      "activations/layer19_attention_weight_max": 35.6340217590332,
      "activations/layer19_attention_weight_min": -33.44844436645508,
      "activations/layer1_attention_weight_max": 16.49090003967285,
      "activations/layer1_attention_weight_min": -15.799846649169922,
      "activations/layer20_attention_weight_max": 30.29976463317871,
      "activations/layer20_attention_weight_min": -26.6271915435791,
      "activations/layer21_attention_weight_max": 29.591365814208984,
      "activations/layer21_attention_weight_min": -26.06505012512207,
      "activations/layer22_attention_weight_max": 41.57099151611328,
      "activations/layer22_attention_weight_min": -32.9237060546875,
      "activations/layer23_attention_weight_max": 33.29932403564453,
      "activations/layer23_attention_weight_min": -26.779743194580078,
      "activations/layer2_attention_weight_max": 34.939918518066406,
      "activations/layer2_attention_weight_min": -34.311004638671875,
      "activations/layer3_attention_weight_max": 112.01033020019531,
      "activations/layer3_attention_weight_min": -112.42682647705078,
      "activations/layer4_attention_weight_max": 115.0924072265625,
      "activations/layer4_attention_weight_min": -116.9917984008789,
      "activations/layer5_attention_weight_max": 53.3553581237793,
      "activations/layer5_attention_weight_min": -63.62455749511719,
      "activations/layer6_attention_weight_max": 50.77875518798828,
      "activations/layer6_attention_weight_min": -51.44129180908203,
      "activations/layer7_attention_weight_max": 108.82836151123047,
      "activations/layer7_attention_weight_min": -111.84575653076172,
      "activations/layer8_attention_weight_max": 44.94383239746094,
      "activations/layer8_attention_weight_min": -48.855926513671875,
      "activations/layer9_attention_weight_max": 32.37275314331055,
      "activations/layer9_attention_weight_min": -36.890377044677734,
      "epoch": 22.14,
      "learning_rate": 7.271212121212121e-06,
      "loss": 2.6973,
      "step": 381100
    },
    {
      "activations/layer0_attention_weight_max": 15.98117446899414,
      "activations/layer0_attention_weight_min": -14.773395538330078,
      "activations/layer10_attention_weight_max": 37.30225372314453,
      "activations/layer10_attention_weight_min": -37.860939025878906,
      "activations/layer11_attention_weight_max": 37.19276809692383,
      "activations/layer11_attention_weight_min": -37.221527099609375,
      "activations/layer12_attention_weight_max": 26.809951782226562,
      "activations/layer12_attention_weight_min": -28.31757354736328,
      "activations/layer13_attention_weight_max": 41.81456756591797,
      "activations/layer13_attention_weight_min": -38.30603790283203,
      "activations/layer14_attention_weight_max": 45.34799575805664,
      "activations/layer14_attention_weight_min": -41.14915084838867,
      "activations/layer15_attention_weight_max": 41.61899185180664,
      "activations/layer15_attention_weight_min": -38.113258361816406,
      "activations/layer16_attention_weight_max": 30.359485626220703,
      "activations/layer16_attention_weight_min": -29.088115692138672,
      "activations/layer17_attention_weight_max": 29.053192138671875,
      "activations/layer17_attention_weight_min": -28.662248611450195,
      "activations/layer18_attention_weight_max": 31.44359016418457,
      "activations/layer18_attention_weight_min": -26.273479461669922,
      "activations/layer19_attention_weight_max": 35.538265228271484,
      "activations/layer19_attention_weight_min": -32.43039321899414,
      "activations/layer1_attention_weight_max": 15.337615013122559,
      "activations/layer1_attention_weight_min": -14.169649124145508,
      "activations/layer20_attention_weight_max": 27.868484497070312,
      "activations/layer20_attention_weight_min": -24.625280380249023,
      "activations/layer21_attention_weight_max": 28.43979263305664,
      "activations/layer21_attention_weight_min": -24.552820205688477,
      "activations/layer22_attention_weight_max": 44.35755157470703,
      "activations/layer22_attention_weight_min": -32.04462432861328,
      "activations/layer23_attention_weight_max": 29.90833854675293,
      "activations/layer23_attention_weight_min": -28.547353744506836,
      "activations/layer2_attention_weight_max": 33.08326721191406,
      "activations/layer2_attention_weight_min": -31.837448120117188,
      "activations/layer3_attention_weight_max": 109.51415252685547,
      "activations/layer3_attention_weight_min": -107.2423324584961,
      "activations/layer4_attention_weight_max": 116.41231536865234,
      "activations/layer4_attention_weight_min": -113.5808334350586,
      "activations/layer5_attention_weight_max": 53.68894577026367,
      "activations/layer5_attention_weight_min": -61.459835052490234,
      "activations/layer6_attention_weight_max": 45.67241668701172,
      "activations/layer6_attention_weight_min": -51.05387496948242,
      "activations/layer7_attention_weight_max": 100.29730224609375,
      "activations/layer7_attention_weight_min": -107.81263732910156,
      "activations/layer8_attention_weight_max": 44.94812774658203,
      "activations/layer8_attention_weight_min": -47.357398986816406,
      "activations/layer9_attention_weight_max": 33.79512023925781,
      "activations/layer9_attention_weight_min": -37.29698944091797,
      "epoch": 22.15,
      "learning_rate": 7.252272727272727e-06,
      "loss": 2.686,
      "step": 381150
    },
    {
      "activations/layer0_attention_weight_max": 16.629592895507812,
      "activations/layer0_attention_weight_min": -16.007692337036133,
      "activations/layer10_attention_weight_max": 38.88648223876953,
      "activations/layer10_attention_weight_min": -39.91907501220703,
      "activations/layer11_attention_weight_max": 36.969390869140625,
      "activations/layer11_attention_weight_min": -40.160545349121094,
      "activations/layer12_attention_weight_max": 29.118640899658203,
      "activations/layer12_attention_weight_min": -27.838184356689453,
      "activations/layer13_attention_weight_max": 39.16876983642578,
      "activations/layer13_attention_weight_min": -35.5723876953125,
      "activations/layer14_attention_weight_max": 44.60416030883789,
      "activations/layer14_attention_weight_min": -39.301334381103516,
      "activations/layer15_attention_weight_max": 44.705135345458984,
      "activations/layer15_attention_weight_min": -42.83586502075195,
      "activations/layer16_attention_weight_max": 27.749677658081055,
      "activations/layer16_attention_weight_min": -28.11707305908203,
      "activations/layer17_attention_weight_max": 28.5685977935791,
      "activations/layer17_attention_weight_min": -28.327688217163086,
      "activations/layer18_attention_weight_max": 33.80067825317383,
      "activations/layer18_attention_weight_min": -29.41575813293457,
      "activations/layer19_attention_weight_max": 37.9303092956543,
      "activations/layer19_attention_weight_min": -32.56571578979492,
      "activations/layer1_attention_weight_max": 15.984728813171387,
      "activations/layer1_attention_weight_min": -14.784283638000488,
      "activations/layer20_attention_weight_max": 30.507450103759766,
      "activations/layer20_attention_weight_min": -25.634788513183594,
      "activations/layer21_attention_weight_max": 28.129941940307617,
      "activations/layer21_attention_weight_min": -24.600561141967773,
      "activations/layer22_attention_weight_max": 41.997127532958984,
      "activations/layer22_attention_weight_min": -32.93450927734375,
      "activations/layer23_attention_weight_max": 34.450172424316406,
      "activations/layer23_attention_weight_min": -28.207653045654297,
      "activations/layer2_attention_weight_max": 34.3161735534668,
      "activations/layer2_attention_weight_min": -34.207759857177734,
      "activations/layer3_attention_weight_max": 106.7012710571289,
      "activations/layer3_attention_weight_min": -111.17235565185547,
      "activations/layer4_attention_weight_max": 112.5289535522461,
      "activations/layer4_attention_weight_min": -113.1276626586914,
      "activations/layer5_attention_weight_max": 52.26205062866211,
      "activations/layer5_attention_weight_min": -64.2800064086914,
      "activations/layer6_attention_weight_max": 46.7697868347168,
      "activations/layer6_attention_weight_min": -49.60325241088867,
      "activations/layer7_attention_weight_max": 104.04907989501953,
      "activations/layer7_attention_weight_min": -108.63731384277344,
      "activations/layer8_attention_weight_max": 46.31370544433594,
      "activations/layer8_attention_weight_min": -49.48023986816406,
      "activations/layer9_attention_weight_max": 35.82038116455078,
      "activations/layer9_attention_weight_min": -38.56239700317383,
      "epoch": 22.15,
      "learning_rate": 7.233333333333332e-06,
      "loss": 2.6951,
      "step": 381200
    },
    {
      "activations/layer0_attention_weight_max": 16.16138458251953,
      "activations/layer0_attention_weight_min": -13.406618118286133,
      "activations/layer10_attention_weight_max": 37.62845230102539,
      "activations/layer10_attention_weight_min": -36.29840850830078,
      "activations/layer11_attention_weight_max": 35.971473693847656,
      "activations/layer11_attention_weight_min": -37.863956451416016,
      "activations/layer12_attention_weight_max": 28.59257698059082,
      "activations/layer12_attention_weight_min": -27.623573303222656,
      "activations/layer13_attention_weight_max": 44.41813278198242,
      "activations/layer13_attention_weight_min": -36.8575325012207,
      "activations/layer14_attention_weight_max": 44.35517883300781,
      "activations/layer14_attention_weight_min": -38.2613639831543,
      "activations/layer15_attention_weight_max": 40.64670181274414,
      "activations/layer15_attention_weight_min": -35.907325744628906,
      "activations/layer16_attention_weight_max": 29.207637786865234,
      "activations/layer16_attention_weight_min": -27.62528419494629,
      "activations/layer17_attention_weight_max": 29.819278717041016,
      "activations/layer17_attention_weight_min": -25.81257438659668,
      "activations/layer18_attention_weight_max": 32.25570297241211,
      "activations/layer18_attention_weight_min": -24.987512588500977,
      "activations/layer19_attention_weight_max": 32.82784652709961,
      "activations/layer19_attention_weight_min": -29.579431533813477,
      "activations/layer1_attention_weight_max": 15.748791694641113,
      "activations/layer1_attention_weight_min": -15.360796928405762,
      "activations/layer20_attention_weight_max": 28.7132625579834,
      "activations/layer20_attention_weight_min": -23.943317413330078,
      "activations/layer21_attention_weight_max": 28.4563045501709,
      "activations/layer21_attention_weight_min": -24.38987922668457,
      "activations/layer22_attention_weight_max": 42.90792465209961,
      "activations/layer22_attention_weight_min": -32.79341125488281,
      "activations/layer23_attention_weight_max": 31.63467788696289,
      "activations/layer23_attention_weight_min": -27.529125213623047,
      "activations/layer2_attention_weight_max": 34.70551300048828,
      "activations/layer2_attention_weight_min": -33.22271728515625,
      "activations/layer3_attention_weight_max": 110.15126037597656,
      "activations/layer3_attention_weight_min": -108.579345703125,
      "activations/layer4_attention_weight_max": 115.8747329711914,
      "activations/layer4_attention_weight_min": -115.27752685546875,
      "activations/layer5_attention_weight_max": 52.31175231933594,
      "activations/layer5_attention_weight_min": -63.538307189941406,
      "activations/layer6_attention_weight_max": 49.25746154785156,
      "activations/layer6_attention_weight_min": -50.01108932495117,
      "activations/layer7_attention_weight_max": 98.96033477783203,
      "activations/layer7_attention_weight_min": -105.7958755493164,
      "activations/layer8_attention_weight_max": 44.29182434082031,
      "activations/layer8_attention_weight_min": -48.78898239135742,
      "activations/layer9_attention_weight_max": 34.39870071411133,
      "activations/layer9_attention_weight_min": -36.847713470458984,
      "epoch": 22.15,
      "learning_rate": 7.214393939393939e-06,
      "loss": 2.6961,
      "step": 381250
    },
    {
      "activations/layer0_attention_weight_max": 16.992996215820312,
      "activations/layer0_attention_weight_min": -15.637442588806152,
      "activations/layer10_attention_weight_max": 35.4910774230957,
      "activations/layer10_attention_weight_min": -35.30617904663086,
      "activations/layer11_attention_weight_max": 35.52372741699219,
      "activations/layer11_attention_weight_min": -34.92166519165039,
      "activations/layer12_attention_weight_max": 27.474170684814453,
      "activations/layer12_attention_weight_min": -28.174697875976562,
      "activations/layer13_attention_weight_max": 40.51241683959961,
      "activations/layer13_attention_weight_min": -37.297725677490234,
      "activations/layer14_attention_weight_max": 41.05943298339844,
      "activations/layer14_attention_weight_min": -38.121395111083984,
      "activations/layer15_attention_weight_max": 42.136695861816406,
      "activations/layer15_attention_weight_min": -36.57909393310547,
      "activations/layer16_attention_weight_max": 28.82951545715332,
      "activations/layer16_attention_weight_min": -28.189088821411133,
      "activations/layer17_attention_weight_max": 33.2436408996582,
      "activations/layer17_attention_weight_min": -27.85948371887207,
      "activations/layer18_attention_weight_max": 32.515655517578125,
      "activations/layer18_attention_weight_min": -25.483402252197266,
      "activations/layer19_attention_weight_max": 34.77399444580078,
      "activations/layer19_attention_weight_min": -31.513681411743164,
      "activations/layer1_attention_weight_max": 16.566659927368164,
      "activations/layer1_attention_weight_min": -14.580094337463379,
      "activations/layer20_attention_weight_max": 27.43986701965332,
      "activations/layer20_attention_weight_min": -25.877878189086914,
      "activations/layer21_attention_weight_max": 27.60910415649414,
      "activations/layer21_attention_weight_min": -24.52020835876465,
      "activations/layer22_attention_weight_max": 37.47864532470703,
      "activations/layer22_attention_weight_min": -30.542524337768555,
      "activations/layer23_attention_weight_max": 30.072486877441406,
      "activations/layer23_attention_weight_min": -24.19122886657715,
      "activations/layer2_attention_weight_max": 36.38165283203125,
      "activations/layer2_attention_weight_min": -32.59241485595703,
      "activations/layer3_attention_weight_max": 113.83428955078125,
      "activations/layer3_attention_weight_min": -106.1297607421875,
      "activations/layer4_attention_weight_max": 114.6162109375,
      "activations/layer4_attention_weight_min": -110.5220947265625,
      "activations/layer5_attention_weight_max": 49.81990051269531,
      "activations/layer5_attention_weight_min": -62.87435531616211,
      "activations/layer6_attention_weight_max": 44.71110534667969,
      "activations/layer6_attention_weight_min": -49.3881950378418,
      "activations/layer7_attention_weight_max": 96.80622863769531,
      "activations/layer7_attention_weight_min": -103.7310791015625,
      "activations/layer8_attention_weight_max": 44.471099853515625,
      "activations/layer8_attention_weight_min": -45.79984664916992,
      "activations/layer9_attention_weight_max": 33.24571228027344,
      "activations/layer9_attention_weight_min": -36.17089080810547,
      "epoch": 22.16,
      "learning_rate": 7.195454545454545e-06,
      "loss": 2.6756,
      "step": 381300
    },
    {
      "activations/layer0_attention_weight_max": 15.806156158447266,
      "activations/layer0_attention_weight_min": -13.562881469726562,
      "activations/layer10_attention_weight_max": 36.31298828125,
      "activations/layer10_attention_weight_min": -34.83298873901367,
      "activations/layer11_attention_weight_max": 36.30964660644531,
      "activations/layer11_attention_weight_min": -37.5057373046875,
      "activations/layer12_attention_weight_max": 30.65389633178711,
      "activations/layer12_attention_weight_min": -28.80687141418457,
      "activations/layer13_attention_weight_max": 41.0106315612793,
      "activations/layer13_attention_weight_min": -36.89036178588867,
      "activations/layer14_attention_weight_max": 46.150150299072266,
      "activations/layer14_attention_weight_min": -39.77351760864258,
      "activations/layer15_attention_weight_max": 41.95222854614258,
      "activations/layer15_attention_weight_min": -37.786746978759766,
      "activations/layer16_attention_weight_max": 29.992815017700195,
      "activations/layer16_attention_weight_min": -28.534820556640625,
      "activations/layer17_attention_weight_max": 30.8994083404541,
      "activations/layer17_attention_weight_min": -26.214923858642578,
      "activations/layer18_attention_weight_max": 33.711822509765625,
      "activations/layer18_attention_weight_min": -25.379165649414062,
      "activations/layer19_attention_weight_max": 33.84031677246094,
      "activations/layer19_attention_weight_min": -33.83797836303711,
      "activations/layer1_attention_weight_max": 16.507274627685547,
      "activations/layer1_attention_weight_min": -15.943769454956055,
      "activations/layer20_attention_weight_max": 28.950777053833008,
      "activations/layer20_attention_weight_min": -26.18795394897461,
      "activations/layer21_attention_weight_max": 29.351879119873047,
      "activations/layer21_attention_weight_min": -24.17653465270996,
      "activations/layer22_attention_weight_max": 41.16348648071289,
      "activations/layer22_attention_weight_min": -31.26633644104004,
      "activations/layer23_attention_weight_max": 32.10550308227539,
      "activations/layer23_attention_weight_min": -25.410621643066406,
      "activations/layer2_attention_weight_max": 33.543678283691406,
      "activations/layer2_attention_weight_min": -33.533355712890625,
      "activations/layer3_attention_weight_max": 108.42298126220703,
      "activations/layer3_attention_weight_min": -109.88011932373047,
      "activations/layer4_attention_weight_max": 112.12493133544922,
      "activations/layer4_attention_weight_min": -117.10919189453125,
      "activations/layer5_attention_weight_max": 55.317928314208984,
      "activations/layer5_attention_weight_min": -64.23339080810547,
      "activations/layer6_attention_weight_max": 48.59133529663086,
      "activations/layer6_attention_weight_min": -52.18007278442383,
      "activations/layer7_attention_weight_max": 102.60655212402344,
      "activations/layer7_attention_weight_min": -106.83964538574219,
      "activations/layer8_attention_weight_max": 44.90156936645508,
      "activations/layer8_attention_weight_min": -46.4409294128418,
      "activations/layer9_attention_weight_max": 34.48406982421875,
      "activations/layer9_attention_weight_min": -36.476715087890625,
      "epoch": 22.16,
      "learning_rate": 7.1765151515151504e-06,
      "loss": 2.6918,
      "step": 381350
    },
    {
      "activations/layer0_attention_weight_max": 16.506671905517578,
      "activations/layer0_attention_weight_min": -14.961507797241211,
      "activations/layer10_attention_weight_max": 41.94255065917969,
      "activations/layer10_attention_weight_min": -43.23981475830078,
      "activations/layer11_attention_weight_max": 41.09538650512695,
      "activations/layer11_attention_weight_min": -42.678279876708984,
      "activations/layer12_attention_weight_max": 30.21452522277832,
      "activations/layer12_attention_weight_min": -29.09701156616211,
      "activations/layer13_attention_weight_max": 41.39967727661133,
      "activations/layer13_attention_weight_min": -36.80586624145508,
      "activations/layer14_attention_weight_max": 43.78554916381836,
      "activations/layer14_attention_weight_min": -39.161922454833984,
      "activations/layer15_attention_weight_max": 40.77578353881836,
      "activations/layer15_attention_weight_min": -36.799659729003906,
      "activations/layer16_attention_weight_max": 27.689043045043945,
      "activations/layer16_attention_weight_min": -31.426403045654297,
      "activations/layer17_attention_weight_max": 29.657175064086914,
      "activations/layer17_attention_weight_min": -27.569795608520508,
      "activations/layer18_attention_weight_max": 32.82223892211914,
      "activations/layer18_attention_weight_min": -28.87575912475586,
      "activations/layer19_attention_weight_max": 34.025970458984375,
      "activations/layer19_attention_weight_min": -31.953800201416016,
      "activations/layer1_attention_weight_max": 16.20557975769043,
      "activations/layer1_attention_weight_min": -15.099864959716797,
      "activations/layer20_attention_weight_max": 30.817975997924805,
      "activations/layer20_attention_weight_min": -24.890262603759766,
      "activations/layer21_attention_weight_max": 29.315303802490234,
      "activations/layer21_attention_weight_min": -24.470388412475586,
      "activations/layer22_attention_weight_max": 47.72718811035156,
      "activations/layer22_attention_weight_min": -30.56805419921875,
      "activations/layer23_attention_weight_max": 33.703250885009766,
      "activations/layer23_attention_weight_min": -25.538070678710938,
      "activations/layer2_attention_weight_max": 34.13970184326172,
      "activations/layer2_attention_weight_min": -32.93024826049805,
      "activations/layer3_attention_weight_max": 112.35860443115234,
      "activations/layer3_attention_weight_min": -111.68531799316406,
      "activations/layer4_attention_weight_max": 114.38195037841797,
      "activations/layer4_attention_weight_min": -115.9569320678711,
      "activations/layer5_attention_weight_max": 53.32820129394531,
      "activations/layer5_attention_weight_min": -59.75758361816406,
      "activations/layer6_attention_weight_max": 48.46784210205078,
      "activations/layer6_attention_weight_min": -52.02627944946289,
      "activations/layer7_attention_weight_max": 106.58915710449219,
      "activations/layer7_attention_weight_min": -108.70580291748047,
      "activations/layer8_attention_weight_max": 49.799041748046875,
      "activations/layer8_attention_weight_min": -49.34999465942383,
      "activations/layer9_attention_weight_max": 38.260223388671875,
      "activations/layer9_attention_weight_min": -41.76357650756836,
      "epoch": 22.16,
      "learning_rate": 7.157575757575758e-06,
      "loss": 2.6738,
      "step": 381400
    },
    {
      "activations/layer0_attention_weight_max": 17.538320541381836,
      "activations/layer0_attention_weight_min": -14.730984687805176,
      "activations/layer10_attention_weight_max": 34.91355895996094,
      "activations/layer10_attention_weight_min": -34.95625686645508,
      "activations/layer11_attention_weight_max": 35.924896240234375,
      "activations/layer11_attention_weight_min": -36.509178161621094,
      "activations/layer12_attention_weight_max": 29.921266555786133,
      "activations/layer12_attention_weight_min": -26.733078002929688,
      "activations/layer13_attention_weight_max": 43.47615051269531,
      "activations/layer13_attention_weight_min": -36.031734466552734,
      "activations/layer14_attention_weight_max": 47.13587188720703,
      "activations/layer14_attention_weight_min": -38.425872802734375,
      "activations/layer15_attention_weight_max": 41.86119079589844,
      "activations/layer15_attention_weight_min": -35.98164367675781,
      "activations/layer16_attention_weight_max": 30.948152542114258,
      "activations/layer16_attention_weight_min": -27.665393829345703,
      "activations/layer17_attention_weight_max": 32.087364196777344,
      "activations/layer17_attention_weight_min": -26.629308700561523,
      "activations/layer18_attention_weight_max": 35.44358444213867,
      "activations/layer18_attention_weight_min": -29.979877471923828,
      "activations/layer19_attention_weight_max": 39.119773864746094,
      "activations/layer19_attention_weight_min": -32.43354797363281,
      "activations/layer1_attention_weight_max": 17.109336853027344,
      "activations/layer1_attention_weight_min": -15.97396469116211,
      "activations/layer20_attention_weight_max": 34.974971771240234,
      "activations/layer20_attention_weight_min": -25.987436294555664,
      "activations/layer21_attention_weight_max": 31.72222137451172,
      "activations/layer21_attention_weight_min": -24.553295135498047,
      "activations/layer22_attention_weight_max": 52.18987274169922,
      "activations/layer22_attention_weight_min": -34.00190734863281,
      "activations/layer23_attention_weight_max": 34.13559341430664,
      "activations/layer23_attention_weight_min": -25.92467498779297,
      "activations/layer2_attention_weight_max": 34.972686767578125,
      "activations/layer2_attention_weight_min": -33.776771545410156,
      "activations/layer3_attention_weight_max": 111.18062591552734,
      "activations/layer3_attention_weight_min": -108.80506134033203,
      "activations/layer4_attention_weight_max": 111.3800048828125,
      "activations/layer4_attention_weight_min": -120.16670227050781,
      "activations/layer5_attention_weight_max": 51.74461364746094,
      "activations/layer5_attention_weight_min": -62.161956787109375,
      "activations/layer6_attention_weight_max": 48.98073196411133,
      "activations/layer6_attention_weight_min": -51.682044982910156,
      "activations/layer7_attention_weight_max": 99.53594207763672,
      "activations/layer7_attention_weight_min": -108.3985366821289,
      "activations/layer8_attention_weight_max": 45.83616638183594,
      "activations/layer8_attention_weight_min": -45.6080322265625,
      "activations/layer9_attention_weight_max": 32.96256637573242,
      "activations/layer9_attention_weight_min": -35.82329559326172,
      "epoch": 22.16,
      "learning_rate": 7.138636363636364e-06,
      "loss": 2.6976,
      "step": 381450
    },
    {
      "activations/layer0_attention_weight_max": 16.46268081665039,
      "activations/layer0_attention_weight_min": -15.110135078430176,
      "activations/layer10_attention_weight_max": 38.50226593017578,
      "activations/layer10_attention_weight_min": -37.37443923950195,
      "activations/layer11_attention_weight_max": 37.14048767089844,
      "activations/layer11_attention_weight_min": -39.83701705932617,
      "activations/layer12_attention_weight_max": 29.094337463378906,
      "activations/layer12_attention_weight_min": -30.548812866210938,
      "activations/layer13_attention_weight_max": 42.387779235839844,
      "activations/layer13_attention_weight_min": -38.6824836730957,
      "activations/layer14_attention_weight_max": 46.02175521850586,
      "activations/layer14_attention_weight_min": -40.20370101928711,
      "activations/layer15_attention_weight_max": 42.76141357421875,
      "activations/layer15_attention_weight_min": -41.34455490112305,
      "activations/layer16_attention_weight_max": 30.345746994018555,
      "activations/layer16_attention_weight_min": -29.055925369262695,
      "activations/layer17_attention_weight_max": 32.33303451538086,
      "activations/layer17_attention_weight_min": -26.502511978149414,
      "activations/layer18_attention_weight_max": 37.22881317138672,
      "activations/layer18_attention_weight_min": -26.94561195373535,
      "activations/layer19_attention_weight_max": 36.83488082885742,
      "activations/layer19_attention_weight_min": -32.80305099487305,
      "activations/layer1_attention_weight_max": 16.382488250732422,
      "activations/layer1_attention_weight_min": -15.008954048156738,
      "activations/layer20_attention_weight_max": 30.755369186401367,
      "activations/layer20_attention_weight_min": -25.284982681274414,
      "activations/layer21_attention_weight_max": 32.06798553466797,
      "activations/layer21_attention_weight_min": -25.526199340820312,
      "activations/layer22_attention_weight_max": 45.822479248046875,
      "activations/layer22_attention_weight_min": -31.41488265991211,
      "activations/layer23_attention_weight_max": 33.98826599121094,
      "activations/layer23_attention_weight_min": -26.42237663269043,
      "activations/layer2_attention_weight_max": 34.10672378540039,
      "activations/layer2_attention_weight_min": -33.9569091796875,
      "activations/layer3_attention_weight_max": 109.35103607177734,
      "activations/layer3_attention_weight_min": -110.67413330078125,
      "activations/layer4_attention_weight_max": 115.13612365722656,
      "activations/layer4_attention_weight_min": -115.4447021484375,
      "activations/layer5_attention_weight_max": 56.315303802490234,
      "activations/layer5_attention_weight_min": -62.89683151245117,
      "activations/layer6_attention_weight_max": 48.49893569946289,
      "activations/layer6_attention_weight_min": -51.57925796508789,
      "activations/layer7_attention_weight_max": 103.91929626464844,
      "activations/layer7_attention_weight_min": -110.48970794677734,
      "activations/layer8_attention_weight_max": 46.768558502197266,
      "activations/layer8_attention_weight_min": -51.99419021606445,
      "activations/layer9_attention_weight_max": 34.29588317871094,
      "activations/layer9_attention_weight_min": -39.08100128173828,
      "epoch": 22.17,
      "learning_rate": 7.119696969696969e-06,
      "loss": 2.7029,
      "step": 381500
    },
    {
      "activations/layer0_attention_weight_max": 16.667612075805664,
      "activations/layer0_attention_weight_min": -12.915217399597168,
      "activations/layer10_attention_weight_max": 38.51228713989258,
      "activations/layer10_attention_weight_min": -38.75122833251953,
      "activations/layer11_attention_weight_max": 36.5191764831543,
      "activations/layer11_attention_weight_min": -37.13135528564453,
      "activations/layer12_attention_weight_max": 27.711881637573242,
      "activations/layer12_attention_weight_min": -27.6116886138916,
      "activations/layer13_attention_weight_max": 40.84300994873047,
      "activations/layer13_attention_weight_min": -39.825836181640625,
      "activations/layer14_attention_weight_max": 45.89814758300781,
      "activations/layer14_attention_weight_min": -41.299842834472656,
      "activations/layer15_attention_weight_max": 45.50938415527344,
      "activations/layer15_attention_weight_min": -40.170440673828125,
      "activations/layer16_attention_weight_max": 32.132537841796875,
      "activations/layer16_attention_weight_min": -29.87875747680664,
      "activations/layer17_attention_weight_max": 32.74195861816406,
      "activations/layer17_attention_weight_min": -30.552724838256836,
      "activations/layer18_attention_weight_max": 33.59642791748047,
      "activations/layer18_attention_weight_min": -27.92686653137207,
      "activations/layer19_attention_weight_max": 37.144386291503906,
      "activations/layer19_attention_weight_min": -32.673309326171875,
      "activations/layer1_attention_weight_max": 15.90274429321289,
      "activations/layer1_attention_weight_min": -15.09753131866455,
      "activations/layer20_attention_weight_max": 32.84013366699219,
      "activations/layer20_attention_weight_min": -25.058568954467773,
      "activations/layer21_attention_weight_max": 33.30056381225586,
      "activations/layer21_attention_weight_min": -25.013591766357422,
      "activations/layer22_attention_weight_max": 48.827659606933594,
      "activations/layer22_attention_weight_min": -31.561941146850586,
      "activations/layer23_attention_weight_max": 36.35942840576172,
      "activations/layer23_attention_weight_min": -26.86469078063965,
      "activations/layer2_attention_weight_max": 34.55341339111328,
      "activations/layer2_attention_weight_min": -33.998191833496094,
      "activations/layer3_attention_weight_max": 109.0318603515625,
      "activations/layer3_attention_weight_min": -113.39672088623047,
      "activations/layer4_attention_weight_max": 112.29296112060547,
      "activations/layer4_attention_weight_min": -117.5283203125,
      "activations/layer5_attention_weight_max": 51.52613830566406,
      "activations/layer5_attention_weight_min": -62.40926742553711,
      "activations/layer6_attention_weight_max": 45.83168029785156,
      "activations/layer6_attention_weight_min": -49.28490447998047,
      "activations/layer7_attention_weight_max": 98.09809875488281,
      "activations/layer7_attention_weight_min": -105.38402557373047,
      "activations/layer8_attention_weight_max": 45.09395980834961,
      "activations/layer8_attention_weight_min": -47.7403678894043,
      "activations/layer9_attention_weight_max": 34.82652282714844,
      "activations/layer9_attention_weight_min": -36.87990951538086,
      "epoch": 22.17,
      "learning_rate": 7.100757575757575e-06,
      "loss": 2.7043,
      "step": 381550
    },
    {
      "activations/layer0_attention_weight_max": 16.430679321289062,
      "activations/layer0_attention_weight_min": -14.300189018249512,
      "activations/layer10_attention_weight_max": 34.10451126098633,
      "activations/layer10_attention_weight_min": -36.16032791137695,
      "activations/layer11_attention_weight_max": 34.224517822265625,
      "activations/layer11_attention_weight_min": -35.364830017089844,
      "activations/layer12_attention_weight_max": 29.832744598388672,
      "activations/layer12_attention_weight_min": -27.793500900268555,
      "activations/layer13_attention_weight_max": 41.965667724609375,
      "activations/layer13_attention_weight_min": -37.312339782714844,
      "activations/layer14_attention_weight_max": 43.013450622558594,
      "activations/layer14_attention_weight_min": -40.1355094909668,
      "activations/layer15_attention_weight_max": 43.64152908325195,
      "activations/layer15_attention_weight_min": -38.711082458496094,
      "activations/layer16_attention_weight_max": 30.09090805053711,
      "activations/layer16_attention_weight_min": -28.806941986083984,
      "activations/layer17_attention_weight_max": 30.995126724243164,
      "activations/layer17_attention_weight_min": -26.26280975341797,
      "activations/layer18_attention_weight_max": 32.586585998535156,
      "activations/layer18_attention_weight_min": -26.448274612426758,
      "activations/layer19_attention_weight_max": 33.87276840209961,
      "activations/layer19_attention_weight_min": -34.22085952758789,
      "activations/layer1_attention_weight_max": 16.230751037597656,
      "activations/layer1_attention_weight_min": -14.751461029052734,
      "activations/layer20_attention_weight_max": 29.914207458496094,
      "activations/layer20_attention_weight_min": -26.793588638305664,
      "activations/layer21_attention_weight_max": 29.31797218322754,
      "activations/layer21_attention_weight_min": -27.11846923828125,
      "activations/layer22_attention_weight_max": 43.17332077026367,
      "activations/layer22_attention_weight_min": -34.42252731323242,
      "activations/layer23_attention_weight_max": 33.37052917480469,
      "activations/layer23_attention_weight_min": -27.832550048828125,
      "activations/layer2_attention_weight_max": 34.27598190307617,
      "activations/layer2_attention_weight_min": -34.49248504638672,
      "activations/layer3_attention_weight_max": 107.08662414550781,
      "activations/layer3_attention_weight_min": -109.5926513671875,
      "activations/layer4_attention_weight_max": 111.3854751586914,
      "activations/layer4_attention_weight_min": -116.2850570678711,
      "activations/layer5_attention_weight_max": 50.253318786621094,
      "activations/layer5_attention_weight_min": -69.50318908691406,
      "activations/layer6_attention_weight_max": 44.69334030151367,
      "activations/layer6_attention_weight_min": -49.862693786621094,
      "activations/layer7_attention_weight_max": 95.21733093261719,
      "activations/layer7_attention_weight_min": -106.31031799316406,
      "activations/layer8_attention_weight_max": 43.9643669128418,
      "activations/layer8_attention_weight_min": -47.34123611450195,
      "activations/layer9_attention_weight_max": 32.50878143310547,
      "activations/layer9_attention_weight_min": -36.55897521972656,
      "epoch": 22.17,
      "learning_rate": 7.081818181818182e-06,
      "loss": 2.7227,
      "step": 381600
    },
    {
      "activations/layer0_attention_weight_max": 17.759376525878906,
      "activations/layer0_attention_weight_min": -14.997748374938965,
      "activations/layer10_attention_weight_max": 41.80267333984375,
      "activations/layer10_attention_weight_min": -40.49216079711914,
      "activations/layer11_attention_weight_max": 40.45105743408203,
      "activations/layer11_attention_weight_min": -41.04820251464844,
      "activations/layer12_attention_weight_max": 34.255924224853516,
      "activations/layer12_attention_weight_min": -31.369169235229492,
      "activations/layer13_attention_weight_max": 50.989295959472656,
      "activations/layer13_attention_weight_min": -43.024837493896484,
      "activations/layer14_attention_weight_max": 50.81633377075195,
      "activations/layer14_attention_weight_min": -41.97526931762695,
      "activations/layer15_attention_weight_max": 48.30714416503906,
      "activations/layer15_attention_weight_min": -41.49549102783203,
      "activations/layer16_attention_weight_max": 31.28508758544922,
      "activations/layer16_attention_weight_min": -27.959999084472656,
      "activations/layer17_attention_weight_max": 33.84523391723633,
      "activations/layer17_attention_weight_min": -25.77225112915039,
      "activations/layer18_attention_weight_max": 34.72426986694336,
      "activations/layer18_attention_weight_min": -25.603071212768555,
      "activations/layer19_attention_weight_max": 34.007896423339844,
      "activations/layer19_attention_weight_min": -30.400375366210938,
      "activations/layer1_attention_weight_max": 16.242511749267578,
      "activations/layer1_attention_weight_min": -15.99824333190918,
      "activations/layer20_attention_weight_max": 28.885900497436523,
      "activations/layer20_attention_weight_min": -24.582714080810547,
      "activations/layer21_attention_weight_max": 29.826810836791992,
      "activations/layer21_attention_weight_min": -22.74558448791504,
      "activations/layer22_attention_weight_max": 43.89084243774414,
      "activations/layer22_attention_weight_min": -29.41490364074707,
      "activations/layer23_attention_weight_max": 34.81852722167969,
      "activations/layer23_attention_weight_min": -24.591657638549805,
      "activations/layer2_attention_weight_max": 34.4925422668457,
      "activations/layer2_attention_weight_min": -34.574562072753906,
      "activations/layer3_attention_weight_max": 108.21197509765625,
      "activations/layer3_attention_weight_min": -110.63130950927734,
      "activations/layer4_attention_weight_max": 116.35713958740234,
      "activations/layer4_attention_weight_min": -120.51210021972656,
      "activations/layer5_attention_weight_max": 54.59548568725586,
      "activations/layer5_attention_weight_min": -66.90180969238281,
      "activations/layer6_attention_weight_max": 47.86266326904297,
      "activations/layer6_attention_weight_min": -51.19520568847656,
      "activations/layer7_attention_weight_max": 104.65037536621094,
      "activations/layer7_attention_weight_min": -109.37154388427734,
      "activations/layer8_attention_weight_max": 46.89946746826172,
      "activations/layer8_attention_weight_min": -51.26372146606445,
      "activations/layer9_attention_weight_max": 37.41204833984375,
      "activations/layer9_attention_weight_min": -39.892669677734375,
      "epoch": 22.18,
      "learning_rate": 7.062878787878787e-06,
      "loss": 2.6905,
      "step": 381650
    },
    {
      "activations/layer0_attention_weight_max": 17.042652130126953,
      "activations/layer0_attention_weight_min": -16.152341842651367,
      "activations/layer10_attention_weight_max": 37.265323638916016,
      "activations/layer10_attention_weight_min": -37.35830307006836,
      "activations/layer11_attention_weight_max": 35.73262405395508,
      "activations/layer11_attention_weight_min": -38.09522247314453,
      "activations/layer12_attention_weight_max": 29.765222549438477,
      "activations/layer12_attention_weight_min": -29.26360511779785,
      "activations/layer13_attention_weight_max": 42.87059783935547,
      "activations/layer13_attention_weight_min": -39.35887908935547,
      "activations/layer14_attention_weight_max": 45.591941833496094,
      "activations/layer14_attention_weight_min": -42.9521369934082,
      "activations/layer15_attention_weight_max": 42.6550407409668,
      "activations/layer15_attention_weight_min": -40.994956970214844,
      "activations/layer16_attention_weight_max": 31.97325897216797,
      "activations/layer16_attention_weight_min": -30.7432861328125,
      "activations/layer17_attention_weight_max": 34.574764251708984,
      "activations/layer17_attention_weight_min": -31.00697898864746,
      "activations/layer18_attention_weight_max": 37.1649284362793,
      "activations/layer18_attention_weight_min": -29.8125057220459,
      "activations/layer19_attention_weight_max": 40.47175979614258,
      "activations/layer19_attention_weight_min": -34.96363830566406,
      "activations/layer1_attention_weight_max": 15.292839050292969,
      "activations/layer1_attention_weight_min": -15.729501724243164,
      "activations/layer20_attention_weight_max": 32.6605339050293,
      "activations/layer20_attention_weight_min": -26.8258113861084,
      "activations/layer21_attention_weight_max": 33.028411865234375,
      "activations/layer21_attention_weight_min": -26.960363388061523,
      "activations/layer22_attention_weight_max": 47.01596450805664,
      "activations/layer22_attention_weight_min": -37.74842834472656,
      "activations/layer23_attention_weight_max": 37.003631591796875,
      "activations/layer23_attention_weight_min": -29.005165100097656,
      "activations/layer2_attention_weight_max": 34.24383544921875,
      "activations/layer2_attention_weight_min": -34.64695358276367,
      "activations/layer3_attention_weight_max": 108.40914154052734,
      "activations/layer3_attention_weight_min": -109.7522964477539,
      "activations/layer4_attention_weight_max": 109.61295318603516,
      "activations/layer4_attention_weight_min": -111.2558364868164,
      "activations/layer5_attention_weight_max": 51.9358024597168,
      "activations/layer5_attention_weight_min": -62.547847747802734,
      "activations/layer6_attention_weight_max": 46.99689865112305,
      "activations/layer6_attention_weight_min": -49.67539596557617,
      "activations/layer7_attention_weight_max": 100.48968505859375,
      "activations/layer7_attention_weight_min": -99.22293090820312,
      "activations/layer8_attention_weight_max": 44.99382781982422,
      "activations/layer8_attention_weight_min": -46.2634391784668,
      "activations/layer9_attention_weight_max": 37.5476188659668,
      "activations/layer9_attention_weight_min": -36.96750259399414,
      "epoch": 22.18,
      "learning_rate": 7.043939393939393e-06,
      "loss": 2.7048,
      "step": 381700
    },
    {
      "activations/layer0_attention_weight_max": 17.014080047607422,
      "activations/layer0_attention_weight_min": -13.610426902770996,
      "activations/layer10_attention_weight_max": 46.89859390258789,
      "activations/layer10_attention_weight_min": -46.234832763671875,
      "activations/layer11_attention_weight_max": 47.787132263183594,
      "activations/layer11_attention_weight_min": -49.857383728027344,
      "activations/layer12_attention_weight_max": 25.57343864440918,
      "activations/layer12_attention_weight_min": -28.239473342895508,
      "activations/layer13_attention_weight_max": 38.40237045288086,
      "activations/layer13_attention_weight_min": -37.267601013183594,
      "activations/layer14_attention_weight_max": 40.60332489013672,
      "activations/layer14_attention_weight_min": -40.09511947631836,
      "activations/layer15_attention_weight_max": 37.600616455078125,
      "activations/layer15_attention_weight_min": -39.56364059448242,
      "activations/layer16_attention_weight_max": 28.136709213256836,
      "activations/layer16_attention_weight_min": -28.390432357788086,
      "activations/layer17_attention_weight_max": 28.93329620361328,
      "activations/layer17_attention_weight_min": -29.399118423461914,
      "activations/layer18_attention_weight_max": 28.865520477294922,
      "activations/layer18_attention_weight_min": -28.631023406982422,
      "activations/layer19_attention_weight_max": 30.791833877563477,
      "activations/layer19_attention_weight_min": -32.753108978271484,
      "activations/layer1_attention_weight_max": 15.50096607208252,
      "activations/layer1_attention_weight_min": -15.883424758911133,
      "activations/layer20_attention_weight_max": 27.28560447692871,
      "activations/layer20_attention_weight_min": -25.52602767944336,
      "activations/layer21_attention_weight_max": 26.732452392578125,
      "activations/layer21_attention_weight_min": -24.302579879760742,
      "activations/layer22_attention_weight_max": 39.89108657836914,
      "activations/layer22_attention_weight_min": -34.72439956665039,
      "activations/layer23_attention_weight_max": 28.32471466064453,
      "activations/layer23_attention_weight_min": -26.722064971923828,
      "activations/layer2_attention_weight_max": 35.35749053955078,
      "activations/layer2_attention_weight_min": -35.79248046875,
      "activations/layer3_attention_weight_max": 107.0475082397461,
      "activations/layer3_attention_weight_min": -113.64166259765625,
      "activations/layer4_attention_weight_max": 107.55278778076172,
      "activations/layer4_attention_weight_min": -115.96710205078125,
      "activations/layer5_attention_weight_max": 51.51402282714844,
      "activations/layer5_attention_weight_min": -64.82801818847656,
      "activations/layer6_attention_weight_max": 46.99229431152344,
      "activations/layer6_attention_weight_min": -48.67304992675781,
      "activations/layer7_attention_weight_max": 110.6858139038086,
      "activations/layer7_attention_weight_min": -111.85980224609375,
      "activations/layer8_attention_weight_max": 53.84355545043945,
      "activations/layer8_attention_weight_min": -50.63822555541992,
      "activations/layer9_attention_weight_max": 40.7721061706543,
      "activations/layer9_attention_weight_min": -44.842750549316406,
      "epoch": 22.18,
      "learning_rate": 7.024999999999999e-06,
      "loss": 2.6936,
      "step": 381750
    },
    {
      "activations/layer0_attention_weight_max": 17.319664001464844,
      "activations/layer0_attention_weight_min": -14.716486930847168,
      "activations/layer10_attention_weight_max": 38.39081573486328,
      "activations/layer10_attention_weight_min": -38.6817626953125,
      "activations/layer11_attention_weight_max": 39.02861785888672,
      "activations/layer11_attention_weight_min": -38.71674728393555,
      "activations/layer12_attention_weight_max": 26.052839279174805,
      "activations/layer12_attention_weight_min": -31.17705535888672,
      "activations/layer13_attention_weight_max": 38.43624496459961,
      "activations/layer13_attention_weight_min": -35.79863739013672,
      "activations/layer14_attention_weight_max": 41.48665237426758,
      "activations/layer14_attention_weight_min": -37.81552505493164,
      "activations/layer15_attention_weight_max": 38.68069839477539,
      "activations/layer15_attention_weight_min": -38.629005432128906,
      "activations/layer16_attention_weight_max": 27.311567306518555,
      "activations/layer16_attention_weight_min": -29.29265022277832,
      "activations/layer17_attention_weight_max": 29.20029067993164,
      "activations/layer17_attention_weight_min": -26.648923873901367,
      "activations/layer18_attention_weight_max": 30.646703720092773,
      "activations/layer18_attention_weight_min": -26.381996154785156,
      "activations/layer19_attention_weight_max": 34.20045852661133,
      "activations/layer19_attention_weight_min": -31.10355567932129,
      "activations/layer1_attention_weight_max": 15.856027603149414,
      "activations/layer1_attention_weight_min": -14.350032806396484,
      "activations/layer20_attention_weight_max": 26.80156707763672,
      "activations/layer20_attention_weight_min": -23.62563133239746,
      "activations/layer21_attention_weight_max": 26.54723358154297,
      "activations/layer21_attention_weight_min": -23.82994842529297,
      "activations/layer22_attention_weight_max": 40.119239807128906,
      "activations/layer22_attention_weight_min": -32.07660675048828,
      "activations/layer23_attention_weight_max": 34.138153076171875,
      "activations/layer23_attention_weight_min": -25.67404556274414,
      "activations/layer2_attention_weight_max": 32.206451416015625,
      "activations/layer2_attention_weight_min": -33.792057037353516,
      "activations/layer3_attention_weight_max": 103.1453628540039,
      "activations/layer3_attention_weight_min": -104.60284423828125,
      "activations/layer4_attention_weight_max": 105.7151870727539,
      "activations/layer4_attention_weight_min": -108.28411865234375,
      "activations/layer5_attention_weight_max": 50.187652587890625,
      "activations/layer5_attention_weight_min": -59.18340301513672,
      "activations/layer6_attention_weight_max": 43.94752883911133,
      "activations/layer6_attention_weight_min": -46.098976135253906,
      "activations/layer7_attention_weight_max": 92.51921844482422,
      "activations/layer7_attention_weight_min": -103.03018188476562,
      "activations/layer8_attention_weight_max": 43.84150695800781,
      "activations/layer8_attention_weight_min": -46.91622543334961,
      "activations/layer9_attention_weight_max": 34.82433319091797,
      "activations/layer9_attention_weight_min": -35.487022399902344,
      "epoch": 22.18,
      "learning_rate": 7.006060606060605e-06,
      "loss": 2.7075,
      "step": 381800
    },
    {
      "activations/layer0_attention_weight_max": 17.39956283569336,
      "activations/layer0_attention_weight_min": -14.862415313720703,
      "activations/layer10_attention_weight_max": 44.0716667175293,
      "activations/layer10_attention_weight_min": -41.43426513671875,
      "activations/layer11_attention_weight_max": 41.920101165771484,
      "activations/layer11_attention_weight_min": -40.852081298828125,
      "activations/layer12_attention_weight_max": 28.767879486083984,
      "activations/layer12_attention_weight_min": -33.377593994140625,
      "activations/layer13_attention_weight_max": 40.559993743896484,
      "activations/layer13_attention_weight_min": -38.161529541015625,
      "activations/layer14_attention_weight_max": 42.98558807373047,
      "activations/layer14_attention_weight_min": -41.764896392822266,
      "activations/layer15_attention_weight_max": 40.7651252746582,
      "activations/layer15_attention_weight_min": -43.89350891113281,
      "activations/layer16_attention_weight_max": 28.97347068786621,
      "activations/layer16_attention_weight_min": -30.80375862121582,
      "activations/layer17_attention_weight_max": 29.510602951049805,
      "activations/layer17_attention_weight_min": -30.613080978393555,
      "activations/layer18_attention_weight_max": 33.933807373046875,
      "activations/layer18_attention_weight_min": -31.568578720092773,
      "activations/layer19_attention_weight_max": 33.99174499511719,
      "activations/layer19_attention_weight_min": -33.99760818481445,
      "activations/layer1_attention_weight_max": 16.929481506347656,
      "activations/layer1_attention_weight_min": -16.268882751464844,
      "activations/layer20_attention_weight_max": 31.00497055053711,
      "activations/layer20_attention_weight_min": -25.51952362060547,
      "activations/layer21_attention_weight_max": 31.876359939575195,
      "activations/layer21_attention_weight_min": -25.400096893310547,
      "activations/layer22_attention_weight_max": 40.02322769165039,
      "activations/layer22_attention_weight_min": -33.12761306762695,
      "activations/layer23_attention_weight_max": 30.79241180419922,
      "activations/layer23_attention_weight_min": -26.253631591796875,
      "activations/layer2_attention_weight_max": 35.958187103271484,
      "activations/layer2_attention_weight_min": -36.377540588378906,
      "activations/layer3_attention_weight_max": 110.3213882446289,
      "activations/layer3_attention_weight_min": -109.5560531616211,
      "activations/layer4_attention_weight_max": 114.48895263671875,
      "activations/layer4_attention_weight_min": -118.07966613769531,
      "activations/layer5_attention_weight_max": 57.14736557006836,
      "activations/layer5_attention_weight_min": -69.03829956054688,
      "activations/layer6_attention_weight_max": 47.55292510986328,
      "activations/layer6_attention_weight_min": -49.493003845214844,
      "activations/layer7_attention_weight_max": 113.89325714111328,
      "activations/layer7_attention_weight_min": -107.15594482421875,
      "activations/layer8_attention_weight_max": 51.39718246459961,
      "activations/layer8_attention_weight_min": -52.20663070678711,
      "activations/layer9_attention_weight_max": 36.22379684448242,
      "activations/layer9_attention_weight_min": -40.48837661743164,
      "epoch": 22.19,
      "learning_rate": 6.987121212121212e-06,
      "loss": 2.7029,
      "step": 381850
    },
    {
      "activations/layer0_attention_weight_max": 17.213865280151367,
      "activations/layer0_attention_weight_min": -14.858386039733887,
      "activations/layer10_attention_weight_max": 34.52994155883789,
      "activations/layer10_attention_weight_min": -34.31195068359375,
      "activations/layer11_attention_weight_max": 33.92170715332031,
      "activations/layer11_attention_weight_min": -34.55153274536133,
      "activations/layer12_attention_weight_max": 25.952503204345703,
      "activations/layer12_attention_weight_min": -26.69803237915039,
      "activations/layer13_attention_weight_max": 38.48296356201172,
      "activations/layer13_attention_weight_min": -35.52720642089844,
      "activations/layer14_attention_weight_max": 39.8719367980957,
      "activations/layer14_attention_weight_min": -39.16404724121094,
      "activations/layer15_attention_weight_max": 36.42904281616211,
      "activations/layer15_attention_weight_min": -36.21847152709961,
      "activations/layer16_attention_weight_max": 30.820301055908203,
      "activations/layer16_attention_weight_min": -27.45282554626465,
      "activations/layer17_attention_weight_max": 32.36406326293945,
      "activations/layer17_attention_weight_min": -26.78061866760254,
      "activations/layer18_attention_weight_max": 30.92695426940918,
      "activations/layer18_attention_weight_min": -26.678972244262695,
      "activations/layer19_attention_weight_max": 37.08142852783203,
      "activations/layer19_attention_weight_min": -32.05902862548828,
      "activations/layer1_attention_weight_max": 15.21945571899414,
      "activations/layer1_attention_weight_min": -14.401948928833008,
      "activations/layer20_attention_weight_max": 33.478389739990234,
      "activations/layer20_attention_weight_min": -26.114078521728516,
      "activations/layer21_attention_weight_max": 31.742448806762695,
      "activations/layer21_attention_weight_min": -25.39805030822754,
      "activations/layer22_attention_weight_max": 45.040687561035156,
      "activations/layer22_attention_weight_min": -33.33051300048828,
      "activations/layer23_attention_weight_max": 31.745466232299805,
      "activations/layer23_attention_weight_min": -26.153181076049805,
      "activations/layer2_attention_weight_max": 33.6031494140625,
      "activations/layer2_attention_weight_min": -33.35398864746094,
      "activations/layer3_attention_weight_max": 106.1783218383789,
      "activations/layer3_attention_weight_min": -111.26752471923828,
      "activations/layer4_attention_weight_max": 110.60538482666016,
      "activations/layer4_attention_weight_min": -114.18067932128906,
      "activations/layer5_attention_weight_max": 49.745384216308594,
      "activations/layer5_attention_weight_min": -62.43400192260742,
      "activations/layer6_attention_weight_max": 46.13325500488281,
      "activations/layer6_attention_weight_min": -47.97283172607422,
      "activations/layer7_attention_weight_max": 98.36074829101562,
      "activations/layer7_attention_weight_min": -107.75267791748047,
      "activations/layer8_attention_weight_max": 44.57251739501953,
      "activations/layer8_attention_weight_min": -47.109130859375,
      "activations/layer9_attention_weight_max": 31.959447860717773,
      "activations/layer9_attention_weight_min": -36.363338470458984,
      "epoch": 22.19,
      "learning_rate": 6.968181818181818e-06,
      "loss": 2.6905,
      "step": 381900
    },
    {
      "activations/layer0_attention_weight_max": 17.05216407775879,
      "activations/layer0_attention_weight_min": -14.368877410888672,
      "activations/layer10_attention_weight_max": 34.52132797241211,
      "activations/layer10_attention_weight_min": -32.46889114379883,
      "activations/layer11_attention_weight_max": 32.4422492980957,
      "activations/layer11_attention_weight_min": -33.225135803222656,
      "activations/layer12_attention_weight_max": 26.48347282409668,
      "activations/layer12_attention_weight_min": -28.122303009033203,
      "activations/layer13_attention_weight_max": 39.46712112426758,
      "activations/layer13_attention_weight_min": -38.96049880981445,
      "activations/layer14_attention_weight_max": 40.73355484008789,
      "activations/layer14_attention_weight_min": -40.45320510864258,
      "activations/layer15_attention_weight_max": 39.73979949951172,
      "activations/layer15_attention_weight_min": -38.142616271972656,
      "activations/layer16_attention_weight_max": 28.02728843688965,
      "activations/layer16_attention_weight_min": -28.53563117980957,
      "activations/layer17_attention_weight_max": 28.093137741088867,
      "activations/layer17_attention_weight_min": -25.96442413330078,
      "activations/layer18_attention_weight_max": 31.20166015625,
      "activations/layer18_attention_weight_min": -25.878000259399414,
      "activations/layer19_attention_weight_max": 33.4321174621582,
      "activations/layer19_attention_weight_min": -31.251325607299805,
      "activations/layer1_attention_weight_max": 16.580408096313477,
      "activations/layer1_attention_weight_min": -13.74407958984375,
      "activations/layer20_attention_weight_max": 27.258331298828125,
      "activations/layer20_attention_weight_min": -26.285377502441406,
      "activations/layer21_attention_weight_max": 27.833938598632812,
      "activations/layer21_attention_weight_min": -25.21910285949707,
      "activations/layer22_attention_weight_max": 42.70298385620117,
      "activations/layer22_attention_weight_min": -31.808645248413086,
      "activations/layer23_attention_weight_max": 29.39559555053711,
      "activations/layer23_attention_weight_min": -25.372600555419922,
      "activations/layer2_attention_weight_max": 33.863521575927734,
      "activations/layer2_attention_weight_min": -32.96924591064453,
      "activations/layer3_attention_weight_max": 105.49156188964844,
      "activations/layer3_attention_weight_min": -106.56224060058594,
      "activations/layer4_attention_weight_max": 108.96768951416016,
      "activations/layer4_attention_weight_min": -115.13983154296875,
      "activations/layer5_attention_weight_max": 54.65807342529297,
      "activations/layer5_attention_weight_min": -68.61607360839844,
      "activations/layer6_attention_weight_max": 44.24577713012695,
      "activations/layer6_attention_weight_min": -48.4012336730957,
      "activations/layer7_attention_weight_max": 103.93633270263672,
      "activations/layer7_attention_weight_min": -102.96854400634766,
      "activations/layer8_attention_weight_max": 41.80342483520508,
      "activations/layer8_attention_weight_min": -47.39133071899414,
      "activations/layer9_attention_weight_max": 30.866836547851562,
      "activations/layer9_attention_weight_min": -32.751564025878906,
      "epoch": 22.19,
      "learning_rate": 6.949242424242423e-06,
      "loss": 2.6982,
      "step": 381950
    },
    {
      "activations/layer0_attention_weight_max": 17.053499221801758,
      "activations/layer0_attention_weight_min": -13.957931518554688,
      "activations/layer10_attention_weight_max": 34.88383865356445,
      "activations/layer10_attention_weight_min": -35.616912841796875,
      "activations/layer11_attention_weight_max": 34.371578216552734,
      "activations/layer11_attention_weight_min": -35.69612503051758,
      "activations/layer12_attention_weight_max": 27.597986221313477,
      "activations/layer12_attention_weight_min": -26.992990493774414,
      "activations/layer13_attention_weight_max": 41.58711242675781,
      "activations/layer13_attention_weight_min": -36.57643127441406,
      "activations/layer14_attention_weight_max": 46.62065505981445,
      "activations/layer14_attention_weight_min": -38.350467681884766,
      "activations/layer15_attention_weight_max": 44.188323974609375,
      "activations/layer15_attention_weight_min": -37.24546432495117,
      "activations/layer16_attention_weight_max": 31.234102249145508,
      "activations/layer16_attention_weight_min": -30.365589141845703,
      "activations/layer17_attention_weight_max": 33.028038024902344,
      "activations/layer17_attention_weight_min": -25.56374740600586,
      "activations/layer18_attention_weight_max": 33.25566864013672,
      "activations/layer18_attention_weight_min": -26.58795166015625,
      "activations/layer19_attention_weight_max": 34.17434310913086,
      "activations/layer19_attention_weight_min": -32.390777587890625,
      "activations/layer1_attention_weight_max": 15.695343017578125,
      "activations/layer1_attention_weight_min": -16.407684326171875,
      "activations/layer20_attention_weight_max": 27.46486473083496,
      "activations/layer20_attention_weight_min": -24.81456756591797,
      "activations/layer21_attention_weight_max": 28.713720321655273,
      "activations/layer21_attention_weight_min": -25.02043914794922,
      "activations/layer22_attention_weight_max": 40.31764221191406,
      "activations/layer22_attention_weight_min": -30.890527725219727,
      "activations/layer23_attention_weight_max": 29.90133285522461,
      "activations/layer23_attention_weight_min": -26.527034759521484,
      "activations/layer2_attention_weight_max": 35.001102447509766,
      "activations/layer2_attention_weight_min": -34.36167907714844,
      "activations/layer3_attention_weight_max": 107.0208969116211,
      "activations/layer3_attention_weight_min": -108.79434967041016,
      "activations/layer4_attention_weight_max": 114.07649230957031,
      "activations/layer4_attention_weight_min": -114.41796875,
      "activations/layer5_attention_weight_max": 51.9879150390625,
      "activations/layer5_attention_weight_min": -64.63408660888672,
      "activations/layer6_attention_weight_max": 47.92713928222656,
      "activations/layer6_attention_weight_min": -48.17351531982422,
      "activations/layer7_attention_weight_max": 99.65349578857422,
      "activations/layer7_attention_weight_min": -99.23471069335938,
      "activations/layer8_attention_weight_max": 44.68869400024414,
      "activations/layer8_attention_weight_min": -45.84659957885742,
      "activations/layer9_attention_weight_max": 32.28823471069336,
      "activations/layer9_attention_weight_min": -34.584861755371094,
      "epoch": 22.2,
      "learning_rate": 6.93030303030303e-06,
      "loss": 2.7103,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4952,
      "eval_samples_per_second": 505.463,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4952,
      "eval_openwebtext_samples_per_second": 505.463,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.9532,
      "eval_wikitext_samples_per_second": 233.46,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_lambada_loss": 2.283203125,
      "eval_lambada_ppl": 9.808046546977115,
      "eval_lambada_runtime": 9.5197,
      "eval_lambada_samples_per_second": 511.466,
      "step": 382000
    },
    {
      "activations/layer0_attention_weight_max": 16.759550094604492,
      "activations/layer0_attention_weight_min": -14.225666999816895,
      "activations/layer10_attention_weight_max": 37.349998474121094,
      "activations/layer10_attention_weight_min": -37.0964241027832,
      "activations/layer11_attention_weight_max": 37.985748291015625,
      "activations/layer11_attention_weight_min": -36.035884857177734,
      "activations/layer12_attention_weight_max": 31.642107009887695,
      "activations/layer12_attention_weight_min": -30.979570388793945,
      "activations/layer13_attention_weight_max": 45.04674530029297,
      "activations/layer13_attention_weight_min": -39.95198059082031,
      "activations/layer14_attention_weight_max": 52.61689376831055,
      "activations/layer14_attention_weight_min": -43.37218475341797,
      "activations/layer15_attention_weight_max": 46.1300048828125,
      "activations/layer15_attention_weight_min": -42.542808532714844,
      "activations/layer16_attention_weight_max": 33.96065902709961,
      "activations/layer16_attention_weight_min": -29.990530014038086,
      "activations/layer17_attention_weight_max": 35.5842399597168,
      "activations/layer17_attention_weight_min": -27.17653465270996,
      "activations/layer18_attention_weight_max": 35.714874267578125,
      "activations/layer18_attention_weight_min": -26.005958557128906,
      "activations/layer19_attention_weight_max": 42.24751663208008,
      "activations/layer19_attention_weight_min": -33.671016693115234,
      "activations/layer1_attention_weight_max": 16.247217178344727,
      "activations/layer1_attention_weight_min": -15.150198936462402,
      "activations/layer20_attention_weight_max": 35.8793830871582,
      "activations/layer20_attention_weight_min": -25.431987762451172,
      "activations/layer21_attention_weight_max": 36.312889099121094,
      "activations/layer21_attention_weight_min": -24.515281677246094,
      "activations/layer22_attention_weight_max": 49.21490478515625,
      "activations/layer22_attention_weight_min": -33.32371139526367,
      "activations/layer23_attention_weight_max": 37.26169967651367,
      "activations/layer23_attention_weight_min": -25.915082931518555,
      "activations/layer2_attention_weight_max": 33.810035705566406,
      "activations/layer2_attention_weight_min": -33.32167053222656,
      "activations/layer3_attention_weight_max": 107.51307678222656,
      "activations/layer3_attention_weight_min": -108.38667297363281,
      "activations/layer4_attention_weight_max": 111.9666976928711,
      "activations/layer4_attention_weight_min": -115.506591796875,
      "activations/layer5_attention_weight_max": 51.117218017578125,
      "activations/layer5_attention_weight_min": -63.40768051147461,
      "activations/layer6_attention_weight_max": 45.18476867675781,
      "activations/layer6_attention_weight_min": -48.9035758972168,
      "activations/layer7_attention_weight_max": 101.78739929199219,
      "activations/layer7_attention_weight_min": -103.1378173828125,
      "activations/layer8_attention_weight_max": 45.133934020996094,
      "activations/layer8_attention_weight_min": -47.862762451171875,
      "activations/layer9_attention_weight_max": 34.955989837646484,
      "activations/layer9_attention_weight_min": -36.2537727355957,
      "epoch": 22.2,
      "learning_rate": 6.911363636363636e-06,
      "loss": 2.7129,
      "step": 382050
    },
    {
      "activations/layer0_attention_weight_max": 16.108840942382812,
      "activations/layer0_attention_weight_min": -14.135647773742676,
      "activations/layer10_attention_weight_max": 34.5059928894043,
      "activations/layer10_attention_weight_min": -35.44711685180664,
      "activations/layer11_attention_weight_max": 35.44073486328125,
      "activations/layer11_attention_weight_min": -36.5686149597168,
      "activations/layer12_attention_weight_max": 26.221595764160156,
      "activations/layer12_attention_weight_min": -26.645475387573242,
      "activations/layer13_attention_weight_max": 40.74605178833008,
      "activations/layer13_attention_weight_min": -36.877174377441406,
      "activations/layer14_attention_weight_max": 44.23927688598633,
      "activations/layer14_attention_weight_min": -40.320255279541016,
      "activations/layer15_attention_weight_max": 41.018760681152344,
      "activations/layer15_attention_weight_min": -38.19026184082031,
      "activations/layer16_attention_weight_max": 28.93916893005371,
      "activations/layer16_attention_weight_min": -28.03866195678711,
      "activations/layer17_attention_weight_max": 31.08083724975586,
      "activations/layer17_attention_weight_min": -25.703208923339844,
      "activations/layer18_attention_weight_max": 35.2427978515625,
      "activations/layer18_attention_weight_min": -24.031673431396484,
      "activations/layer19_attention_weight_max": 38.29882049560547,
      "activations/layer19_attention_weight_min": -32.13799285888672,
      "activations/layer1_attention_weight_max": 14.718270301818848,
      "activations/layer1_attention_weight_min": -14.850791931152344,
      "activations/layer20_attention_weight_max": 31.4379940032959,
      "activations/layer20_attention_weight_min": -24.888744354248047,
      "activations/layer21_attention_weight_max": 32.324729919433594,
      "activations/layer21_attention_weight_min": -23.86420249938965,
      "activations/layer22_attention_weight_max": 43.74832534790039,
      "activations/layer22_attention_weight_min": -30.64805030822754,
      "activations/layer23_attention_weight_max": 32.222076416015625,
      "activations/layer23_attention_weight_min": -24.34748077392578,
      "activations/layer2_attention_weight_max": 33.17210006713867,
      "activations/layer2_attention_weight_min": -33.3822135925293,
      "activations/layer3_attention_weight_max": 108.33740234375,
      "activations/layer3_attention_weight_min": -107.30524444580078,
      "activations/layer4_attention_weight_max": 110.3232650756836,
      "activations/layer4_attention_weight_min": -112.37303924560547,
      "activations/layer5_attention_weight_max": 50.074520111083984,
      "activations/layer5_attention_weight_min": -63.62118148803711,
      "activations/layer6_attention_weight_max": 44.56352996826172,
      "activations/layer6_attention_weight_min": -47.6091423034668,
      "activations/layer7_attention_weight_max": 97.23111724853516,
      "activations/layer7_attention_weight_min": -99.8816146850586,
      "activations/layer8_attention_weight_max": 42.27267837524414,
      "activations/layer8_attention_weight_min": -44.03419876098633,
      "activations/layer9_attention_weight_max": 32.4598274230957,
      "activations/layer9_attention_weight_min": -35.34552764892578,
      "epoch": 22.2,
      "learning_rate": 6.8924242424242414e-06,
      "loss": 2.7012,
      "step": 382100
    },
    {
      "activations/layer0_attention_weight_max": 17.02713966369629,
      "activations/layer0_attention_weight_min": -13.5903959274292,
      "activations/layer10_attention_weight_max": 35.4941291809082,
      "activations/layer10_attention_weight_min": -37.25996780395508,
      "activations/layer11_attention_weight_max": 35.142822265625,
      "activations/layer11_attention_weight_min": -39.66194152832031,
      "activations/layer12_attention_weight_max": 25.956186294555664,
      "activations/layer12_attention_weight_min": -28.913944244384766,
      "activations/layer13_attention_weight_max": 39.57932662963867,
      "activations/layer13_attention_weight_min": -40.42725372314453,
      "activations/layer14_attention_weight_max": 43.098453521728516,
      "activations/layer14_attention_weight_min": -42.406959533691406,
      "activations/layer15_attention_weight_max": 39.158504486083984,
      "activations/layer15_attention_weight_min": -36.59796905517578,
      "activations/layer16_attention_weight_max": 27.552539825439453,
      "activations/layer16_attention_weight_min": -28.40336799621582,
      "activations/layer17_attention_weight_max": 30.890228271484375,
      "activations/layer17_attention_weight_min": -25.885488510131836,
      "activations/layer18_attention_weight_max": 31.195331573486328,
      "activations/layer18_attention_weight_min": -28.515371322631836,
      "activations/layer19_attention_weight_max": 35.391536712646484,
      "activations/layer19_attention_weight_min": -31.035709381103516,
      "activations/layer1_attention_weight_max": 15.511208534240723,
      "activations/layer1_attention_weight_min": -16.284940719604492,
      "activations/layer20_attention_weight_max": 29.18402099609375,
      "activations/layer20_attention_weight_min": -25.45644187927246,
      "activations/layer21_attention_weight_max": 26.95876693725586,
      "activations/layer21_attention_weight_min": -24.13500213623047,
      "activations/layer22_attention_weight_max": 39.28217315673828,
      "activations/layer22_attention_weight_min": -31.39264678955078,
      "activations/layer23_attention_weight_max": 32.25893783569336,
      "activations/layer23_attention_weight_min": -24.702451705932617,
      "activations/layer2_attention_weight_max": 33.89942932128906,
      "activations/layer2_attention_weight_min": -35.02162170410156,
      "activations/layer3_attention_weight_max": 108.09789276123047,
      "activations/layer3_attention_weight_min": -110.35145568847656,
      "activations/layer4_attention_weight_max": 112.10188293457031,
      "activations/layer4_attention_weight_min": -115.675048828125,
      "activations/layer5_attention_weight_max": 51.78335189819336,
      "activations/layer5_attention_weight_min": -64.34272003173828,
      "activations/layer6_attention_weight_max": 46.92414474487305,
      "activations/layer6_attention_weight_min": -50.710777282714844,
      "activations/layer7_attention_weight_max": 102.42012023925781,
      "activations/layer7_attention_weight_min": -108.42155456542969,
      "activations/layer8_attention_weight_max": 44.476253509521484,
      "activations/layer8_attention_weight_min": -45.858970642089844,
      "activations/layer9_attention_weight_max": 34.062740325927734,
      "activations/layer9_attention_weight_min": -35.823177337646484,
      "epoch": 22.21,
      "learning_rate": 6.873484848484847e-06,
      "loss": 2.7066,
      "step": 382150
    },
    {
      "activations/layer0_attention_weight_max": 17.482868194580078,
      "activations/layer0_attention_weight_min": -15.243853569030762,
      "activations/layer10_attention_weight_max": 36.691497802734375,
      "activations/layer10_attention_weight_min": -35.64493179321289,
      "activations/layer11_attention_weight_max": 35.439910888671875,
      "activations/layer11_attention_weight_min": -36.00651550292969,
      "activations/layer12_attention_weight_max": 25.831193923950195,
      "activations/layer12_attention_weight_min": -27.453821182250977,
      "activations/layer13_attention_weight_max": 37.97233200073242,
      "activations/layer13_attention_weight_min": -37.68971633911133,
      "activations/layer14_attention_weight_max": 40.47844696044922,
      "activations/layer14_attention_weight_min": -37.99800109863281,
      "activations/layer15_attention_weight_max": 39.73615264892578,
      "activations/layer15_attention_weight_min": -37.71529769897461,
      "activations/layer16_attention_weight_max": 30.11720848083496,
      "activations/layer16_attention_weight_min": -29.939796447753906,
      "activations/layer17_attention_weight_max": 29.240753173828125,
      "activations/layer17_attention_weight_min": -27.217918395996094,
      "activations/layer18_attention_weight_max": 34.634666442871094,
      "activations/layer18_attention_weight_min": -28.539085388183594,
      "activations/layer19_attention_weight_max": 32.971229553222656,
      "activations/layer19_attention_weight_min": -31.523954391479492,
      "activations/layer1_attention_weight_max": 15.646039009094238,
      "activations/layer1_attention_weight_min": -15.954736709594727,
      "activations/layer20_attention_weight_max": 28.895933151245117,
      "activations/layer20_attention_weight_min": -25.603044509887695,
      "activations/layer21_attention_weight_max": 28.729644775390625,
      "activations/layer21_attention_weight_min": -24.455575942993164,
      "activations/layer22_attention_weight_max": 39.7603645324707,
      "activations/layer22_attention_weight_min": -31.790884017944336,
      "activations/layer23_attention_weight_max": 28.65475082397461,
      "activations/layer23_attention_weight_min": -27.134132385253906,
      "activations/layer2_attention_weight_max": 34.654659271240234,
      "activations/layer2_attention_weight_min": -33.23603057861328,
      "activations/layer3_attention_weight_max": 108.48878479003906,
      "activations/layer3_attention_weight_min": -107.71414184570312,
      "activations/layer4_attention_weight_max": 113.0796127319336,
      "activations/layer4_attention_weight_min": -118.8752670288086,
      "activations/layer5_attention_weight_max": 52.20408630371094,
      "activations/layer5_attention_weight_min": -64.83194732666016,
      "activations/layer6_attention_weight_max": 45.65312957763672,
      "activations/layer6_attention_weight_min": -50.368289947509766,
      "activations/layer7_attention_weight_max": 103.25849914550781,
      "activations/layer7_attention_weight_min": -103.23601531982422,
      "activations/layer8_attention_weight_max": 41.89583206176758,
      "activations/layer8_attention_weight_min": -47.134552001953125,
      "activations/layer9_attention_weight_max": 33.534637451171875,
      "activations/layer9_attention_weight_min": -36.007991790771484,
      "epoch": 22.21,
      "learning_rate": 6.854545454545454e-06,
      "loss": 2.7123,
      "step": 382200
    },
    {
      "activations/layer0_attention_weight_max": 17.288423538208008,
      "activations/layer0_attention_weight_min": -13.495920181274414,
      "activations/layer10_attention_weight_max": 37.52432632446289,
      "activations/layer10_attention_weight_min": -35.381935119628906,
      "activations/layer11_attention_weight_max": 36.654537200927734,
      "activations/layer11_attention_weight_min": -36.70148468017578,
      "activations/layer12_attention_weight_max": 27.076026916503906,
      "activations/layer12_attention_weight_min": -27.121597290039062,
      "activations/layer13_attention_weight_max": 38.57069396972656,
      "activations/layer13_attention_weight_min": -35.13037109375,
      "activations/layer14_attention_weight_max": 41.74379348754883,
      "activations/layer14_attention_weight_min": -38.01273727416992,
      "activations/layer15_attention_weight_max": 38.55280685424805,
      "activations/layer15_attention_weight_min": -35.324241638183594,
      "activations/layer16_attention_weight_max": 30.10298728942871,
      "activations/layer16_attention_weight_min": -28.08443832397461,
      "activations/layer17_attention_weight_max": 27.576093673706055,
      "activations/layer17_attention_weight_min": -25.27556800842285,
      "activations/layer18_attention_weight_max": 34.64980697631836,
      "activations/layer18_attention_weight_min": -26.512292861938477,
      "activations/layer19_attention_weight_max": 35.13794708251953,
      "activations/layer19_attention_weight_min": -29.815872192382812,
      "activations/layer1_attention_weight_max": 16.34296226501465,
      "activations/layer1_attention_weight_min": -15.232794761657715,
      "activations/layer20_attention_weight_max": 29.104997634887695,
      "activations/layer20_attention_weight_min": -22.60808563232422,
      "activations/layer21_attention_weight_max": 28.10702133178711,
      "activations/layer21_attention_weight_min": -24.3153018951416,
      "activations/layer22_attention_weight_max": 41.70658493041992,
      "activations/layer22_attention_weight_min": -32.01749801635742,
      "activations/layer23_attention_weight_max": 31.131481170654297,
      "activations/layer23_attention_weight_min": -26.885770797729492,
      "activations/layer2_attention_weight_max": 34.02943420410156,
      "activations/layer2_attention_weight_min": -33.50874710083008,
      "activations/layer3_attention_weight_max": 109.64337921142578,
      "activations/layer3_attention_weight_min": -107.67853546142578,
      "activations/layer4_attention_weight_max": 112.79701232910156,
      "activations/layer4_attention_weight_min": -116.92848205566406,
      "activations/layer5_attention_weight_max": 51.016448974609375,
      "activations/layer5_attention_weight_min": -64.92697143554688,
      "activations/layer6_attention_weight_max": 47.454612731933594,
      "activations/layer6_attention_weight_min": -49.04378128051758,
      "activations/layer7_attention_weight_max": 102.8284683227539,
      "activations/layer7_attention_weight_min": -98.7227783203125,
      "activations/layer8_attention_weight_max": 43.80874252319336,
      "activations/layer8_attention_weight_min": -44.972572326660156,
      "activations/layer9_attention_weight_max": 32.669979095458984,
      "activations/layer9_attention_weight_min": -36.07508087158203,
      "epoch": 22.21,
      "learning_rate": 6.83560606060606e-06,
      "loss": 2.6935,
      "step": 382250
    },
    {
      "activations/layer0_attention_weight_max": 17.475812911987305,
      "activations/layer0_attention_weight_min": -13.215946197509766,
      "activations/layer10_attention_weight_max": 38.53444290161133,
      "activations/layer10_attention_weight_min": -42.569602966308594,
      "activations/layer11_attention_weight_max": 39.453712463378906,
      "activations/layer11_attention_weight_min": -40.80712127685547,
      "activations/layer12_attention_weight_max": 32.217437744140625,
      "activations/layer12_attention_weight_min": -30.6073055267334,
      "activations/layer13_attention_weight_max": 49.25267791748047,
      "activations/layer13_attention_weight_min": -41.14946365356445,
      "activations/layer14_attention_weight_max": 54.67642593383789,
      "activations/layer14_attention_weight_min": -44.497962951660156,
      "activations/layer15_attention_weight_max": 48.2347297668457,
      "activations/layer15_attention_weight_min": -40.765201568603516,
      "activations/layer16_attention_weight_max": 37.867340087890625,
      "activations/layer16_attention_weight_min": -30.95516014099121,
      "activations/layer17_attention_weight_max": 34.40011215209961,
      "activations/layer17_attention_weight_min": -30.70595932006836,
      "activations/layer18_attention_weight_max": 35.4348258972168,
      "activations/layer18_attention_weight_min": -29.073284149169922,
      "activations/layer19_attention_weight_max": 40.55711364746094,
      "activations/layer19_attention_weight_min": -32.902034759521484,
      "activations/layer1_attention_weight_max": 16.515995025634766,
      "activations/layer1_attention_weight_min": -15.713218688964844,
      "activations/layer20_attention_weight_max": 29.886430740356445,
      "activations/layer20_attention_weight_min": -25.862031936645508,
      "activations/layer21_attention_weight_max": 27.63945198059082,
      "activations/layer21_attention_weight_min": -25.83945655822754,
      "activations/layer22_attention_weight_max": 41.82776641845703,
      "activations/layer22_attention_weight_min": -31.601131439208984,
      "activations/layer23_attention_weight_max": 31.657760620117188,
      "activations/layer23_attention_weight_min": -24.740751266479492,
      "activations/layer2_attention_weight_max": 35.05609130859375,
      "activations/layer2_attention_weight_min": -35.00529861450195,
      "activations/layer3_attention_weight_max": 112.16376495361328,
      "activations/layer3_attention_weight_min": -112.00767517089844,
      "activations/layer4_attention_weight_max": 116.576416015625,
      "activations/layer4_attention_weight_min": -119.9921646118164,
      "activations/layer5_attention_weight_max": 54.630531311035156,
      "activations/layer5_attention_weight_min": -64.28402709960938,
      "activations/layer6_attention_weight_max": 49.01620864868164,
      "activations/layer6_attention_weight_min": -52.110816955566406,
      "activations/layer7_attention_weight_max": 101.23284149169922,
      "activations/layer7_attention_weight_min": -112.38301086425781,
      "activations/layer8_attention_weight_max": 46.463340759277344,
      "activations/layer8_attention_weight_min": -52.9835205078125,
      "activations/layer9_attention_weight_max": 34.899871826171875,
      "activations/layer9_attention_weight_min": -40.38410568237305,
      "epoch": 22.21,
      "learning_rate": 6.817045454545454e-06,
      "loss": 2.6991,
      "step": 382300
    },
    {
      "activations/layer0_attention_weight_max": 17.189102172851562,
      "activations/layer0_attention_weight_min": -15.033750534057617,
      "activations/layer10_attention_weight_max": 38.33147048950195,
      "activations/layer10_attention_weight_min": -36.767578125,
      "activations/layer11_attention_weight_max": 37.058380126953125,
      "activations/layer11_attention_weight_min": -38.46982192993164,
      "activations/layer12_attention_weight_max": 26.021366119384766,
      "activations/layer12_attention_weight_min": -28.043672561645508,
      "activations/layer13_attention_weight_max": 37.45313262939453,
      "activations/layer13_attention_weight_min": -39.468505859375,
      "activations/layer14_attention_weight_max": 40.014095306396484,
      "activations/layer14_attention_weight_min": -40.593624114990234,
      "activations/layer15_attention_weight_max": 38.20684051513672,
      "activations/layer15_attention_weight_min": -37.49418640136719,
      "activations/layer16_attention_weight_max": 28.347713470458984,
      "activations/layer16_attention_weight_min": -29.440919876098633,
      "activations/layer17_attention_weight_max": 29.800079345703125,
      "activations/layer17_attention_weight_min": -26.482473373413086,
      "activations/layer18_attention_weight_max": 34.0223274230957,
      "activations/layer18_attention_weight_min": -29.321762084960938,
      "activations/layer19_attention_weight_max": 31.950658798217773,
      "activations/layer19_attention_weight_min": -33.65712356567383,
      "activations/layer1_attention_weight_max": 15.672784805297852,
      "activations/layer1_attention_weight_min": -15.514336585998535,
      "activations/layer20_attention_weight_max": 27.11101531982422,
      "activations/layer20_attention_weight_min": -27.155384063720703,
      "activations/layer21_attention_weight_max": 28.818265914916992,
      "activations/layer21_attention_weight_min": -25.879064559936523,
      "activations/layer22_attention_weight_max": 44.12481689453125,
      "activations/layer22_attention_weight_min": -33.96220397949219,
      "activations/layer23_attention_weight_max": 34.05861282348633,
      "activations/layer23_attention_weight_min": -25.77141571044922,
      "activations/layer2_attention_weight_max": 34.432945251464844,
      "activations/layer2_attention_weight_min": -33.856773376464844,
      "activations/layer3_attention_weight_max": 110.82228088378906,
      "activations/layer3_attention_weight_min": -109.86643981933594,
      "activations/layer4_attention_weight_max": 116.30956268310547,
      "activations/layer4_attention_weight_min": -116.4292221069336,
      "activations/layer5_attention_weight_max": 52.445701599121094,
      "activations/layer5_attention_weight_min": -66.66067504882812,
      "activations/layer6_attention_weight_max": 48.318702697753906,
      "activations/layer6_attention_weight_min": -50.12238311767578,
      "activations/layer7_attention_weight_max": 105.7852783203125,
      "activations/layer7_attention_weight_min": -105.9446792602539,
      "activations/layer8_attention_weight_max": 49.8483772277832,
      "activations/layer8_attention_weight_min": -47.801612854003906,
      "activations/layer9_attention_weight_max": 36.643062591552734,
      "activations/layer9_attention_weight_min": -38.88570785522461,
      "epoch": 22.22,
      "learning_rate": 6.79810606060606e-06,
      "loss": 2.6873,
      "step": 382350
    },
    {
      "activations/layer0_attention_weight_max": 16.814668655395508,
      "activations/layer0_attention_weight_min": -16.046865463256836,
      "activations/layer10_attention_weight_max": 35.069557189941406,
      "activations/layer10_attention_weight_min": -34.976341247558594,
      "activations/layer11_attention_weight_max": 35.94780731201172,
      "activations/layer11_attention_weight_min": -34.67100524902344,
      "activations/layer12_attention_weight_max": 26.582319259643555,
      "activations/layer12_attention_weight_min": -27.0596866607666,
      "activations/layer13_attention_weight_max": 38.687889099121094,
      "activations/layer13_attention_weight_min": -37.40153503417969,
      "activations/layer14_attention_weight_max": 42.44197463989258,
      "activations/layer14_attention_weight_min": -35.97673416137695,
      "activations/layer15_attention_weight_max": 39.5026741027832,
      "activations/layer15_attention_weight_min": -38.41329574584961,
      "activations/layer16_attention_weight_max": 28.837966918945312,
      "activations/layer16_attention_weight_min": -28.641557693481445,
      "activations/layer17_attention_weight_max": 29.34759521484375,
      "activations/layer17_attention_weight_min": -27.468830108642578,
      "activations/layer18_attention_weight_max": 31.290225982666016,
      "activations/layer18_attention_weight_min": -26.448461532592773,
      "activations/layer19_attention_weight_max": 36.60490798950195,
      "activations/layer19_attention_weight_min": -33.370399475097656,
      "activations/layer1_attention_weight_max": 15.552746772766113,
      "activations/layer1_attention_weight_min": -15.018278121948242,
      "activations/layer20_attention_weight_max": 31.294233322143555,
      "activations/layer20_attention_weight_min": -24.87004280090332,
      "activations/layer21_attention_weight_max": 30.379220962524414,
      "activations/layer21_attention_weight_min": -24.810958862304688,
      "activations/layer22_attention_weight_max": 42.152923583984375,
      "activations/layer22_attention_weight_min": -34.87748336791992,
      "activations/layer23_attention_weight_max": 31.96405792236328,
      "activations/layer23_attention_weight_min": -26.742103576660156,
      "activations/layer2_attention_weight_max": 37.25840377807617,
      "activations/layer2_attention_weight_min": -33.04004669189453,
      "activations/layer3_attention_weight_max": 103.82317352294922,
      "activations/layer3_attention_weight_min": -106.3379135131836,
      "activations/layer4_attention_weight_max": 108.21390533447266,
      "activations/layer4_attention_weight_min": -111.91527557373047,
      "activations/layer5_attention_weight_max": 48.426597595214844,
      "activations/layer5_attention_weight_min": -60.61286163330078,
      "activations/layer6_attention_weight_max": 45.30193328857422,
      "activations/layer6_attention_weight_min": -48.12849807739258,
      "activations/layer7_attention_weight_max": 95.62371063232422,
      "activations/layer7_attention_weight_min": -102.15560913085938,
      "activations/layer8_attention_weight_max": 43.77107238769531,
      "activations/layer8_attention_weight_min": -44.699241638183594,
      "activations/layer9_attention_weight_max": 31.92371940612793,
      "activations/layer9_attention_weight_min": -35.046539306640625,
      "epoch": 22.22,
      "learning_rate": 6.779166666666667e-06,
      "loss": 2.6913,
      "step": 382400
    },
    {
      "activations/layer0_attention_weight_max": 17.125301361083984,
      "activations/layer0_attention_weight_min": -14.84656047821045,
      "activations/layer10_attention_weight_max": 39.203399658203125,
      "activations/layer10_attention_weight_min": -40.03384017944336,
      "activations/layer11_attention_weight_max": 39.15791702270508,
      "activations/layer11_attention_weight_min": -40.06953811645508,
      "activations/layer12_attention_weight_max": 28.91848373413086,
      "activations/layer12_attention_weight_min": -29.502660751342773,
      "activations/layer13_attention_weight_max": 42.132164001464844,
      "activations/layer13_attention_weight_min": -38.59962844848633,
      "activations/layer14_attention_weight_max": 48.873252868652344,
      "activations/layer14_attention_weight_min": -43.31090545654297,
      "activations/layer15_attention_weight_max": 45.19945526123047,
      "activations/layer15_attention_weight_min": -41.5517578125,
      "activations/layer16_attention_weight_max": 34.7873649597168,
      "activations/layer16_attention_weight_min": -30.706640243530273,
      "activations/layer17_attention_weight_max": 32.841617584228516,
      "activations/layer17_attention_weight_min": -27.230154037475586,
      "activations/layer18_attention_weight_max": 34.53498458862305,
      "activations/layer18_attention_weight_min": -28.735595703125,
      "activations/layer19_attention_weight_max": 39.30546188354492,
      "activations/layer19_attention_weight_min": -35.68929672241211,
      "activations/layer1_attention_weight_max": 16.28712272644043,
      "activations/layer1_attention_weight_min": -17.707374572753906,
      "activations/layer20_attention_weight_max": 33.20743179321289,
      "activations/layer20_attention_weight_min": -27.846805572509766,
      "activations/layer21_attention_weight_max": 30.580699920654297,
      "activations/layer21_attention_weight_min": -27.459056854248047,
      "activations/layer22_attention_weight_max": 45.22845458984375,
      "activations/layer22_attention_weight_min": -33.44712448120117,
      "activations/layer23_attention_weight_max": 32.262542724609375,
      "activations/layer23_attention_weight_min": -25.876182556152344,
      "activations/layer2_attention_weight_max": 34.84617614746094,
      "activations/layer2_attention_weight_min": -33.230560302734375,
      "activations/layer3_attention_weight_max": 108.06939697265625,
      "activations/layer3_attention_weight_min": -108.16905212402344,
      "activations/layer4_attention_weight_max": 116.1100845336914,
      "activations/layer4_attention_weight_min": -118.9451675415039,
      "activations/layer5_attention_weight_max": 51.84203338623047,
      "activations/layer5_attention_weight_min": -65.92143249511719,
      "activations/layer6_attention_weight_max": 47.72322082519531,
      "activations/layer6_attention_weight_min": -50.68971252441406,
      "activations/layer7_attention_weight_max": 109.02873229980469,
      "activations/layer7_attention_weight_min": -103.20085144042969,
      "activations/layer8_attention_weight_max": 46.07320022583008,
      "activations/layer8_attention_weight_min": -46.69770812988281,
      "activations/layer9_attention_weight_max": 35.161869049072266,
      "activations/layer9_attention_weight_min": -38.19456100463867,
      "epoch": 22.22,
      "learning_rate": 6.7602272727272726e-06,
      "loss": 2.6946,
      "step": 382450
    },
    {
      "activations/layer0_attention_weight_max": 16.969013214111328,
      "activations/layer0_attention_weight_min": -15.576431274414062,
      "activations/layer10_attention_weight_max": 34.86212158203125,
      "activations/layer10_attention_weight_min": -35.41511154174805,
      "activations/layer11_attention_weight_max": 34.22646713256836,
      "activations/layer11_attention_weight_min": -34.7763671875,
      "activations/layer12_attention_weight_max": 26.825733184814453,
      "activations/layer12_attention_weight_min": -26.735929489135742,
      "activations/layer13_attention_weight_max": 38.24755859375,
      "activations/layer13_attention_weight_min": -35.10688400268555,
      "activations/layer14_attention_weight_max": 40.99754333496094,
      "activations/layer14_attention_weight_min": -38.45952606201172,
      "activations/layer15_attention_weight_max": 37.29863739013672,
      "activations/layer15_attention_weight_min": -39.09457778930664,
      "activations/layer16_attention_weight_max": 29.051834106445312,
      "activations/layer16_attention_weight_min": -29.20111656188965,
      "activations/layer17_attention_weight_max": 28.59050178527832,
      "activations/layer17_attention_weight_min": -26.935924530029297,
      "activations/layer18_attention_weight_max": 31.71999740600586,
      "activations/layer18_attention_weight_min": -26.917308807373047,
      "activations/layer19_attention_weight_max": 35.56283950805664,
      "activations/layer19_attention_weight_min": -31.396621704101562,
      "activations/layer1_attention_weight_max": 16.885190963745117,
      "activations/layer1_attention_weight_min": -15.610424995422363,
      "activations/layer20_attention_weight_max": 28.774734497070312,
      "activations/layer20_attention_weight_min": -24.648290634155273,
      "activations/layer21_attention_weight_max": 28.718685150146484,
      "activations/layer21_attention_weight_min": -25.179040908813477,
      "activations/layer22_attention_weight_max": 44.483497619628906,
      "activations/layer22_attention_weight_min": -31.90324592590332,
      "activations/layer23_attention_weight_max": 33.47660827636719,
      "activations/layer23_attention_weight_min": -26.45702362060547,
      "activations/layer2_attention_weight_max": 33.04185485839844,
      "activations/layer2_attention_weight_min": -32.696468353271484,
      "activations/layer3_attention_weight_max": 105.5348129272461,
      "activations/layer3_attention_weight_min": -110.14353942871094,
      "activations/layer4_attention_weight_max": 111.35627746582031,
      "activations/layer4_attention_weight_min": -113.66092681884766,
      "activations/layer5_attention_weight_max": 52.029048919677734,
      "activations/layer5_attention_weight_min": -58.886810302734375,
      "activations/layer6_attention_weight_max": 43.95307159423828,
      "activations/layer6_attention_weight_min": -46.65255355834961,
      "activations/layer7_attention_weight_max": 95.90469360351562,
      "activations/layer7_attention_weight_min": -99.46961975097656,
      "activations/layer8_attention_weight_max": 41.88945770263672,
      "activations/layer8_attention_weight_min": -46.55768585205078,
      "activations/layer9_attention_weight_max": 31.60805320739746,
      "activations/layer9_attention_weight_min": -35.352874755859375,
      "epoch": 22.23,
      "learning_rate": 6.741287878787878e-06,
      "loss": 2.6996,
      "step": 382500
    },
    {
      "activations/layer0_attention_weight_max": 16.484663009643555,
      "activations/layer0_attention_weight_min": -13.478795051574707,
      "activations/layer10_attention_weight_max": 35.25199508666992,
      "activations/layer10_attention_weight_min": -35.928932189941406,
      "activations/layer11_attention_weight_max": 35.960411071777344,
      "activations/layer11_attention_weight_min": -35.799373626708984,
      "activations/layer12_attention_weight_max": 27.34476089477539,
      "activations/layer12_attention_weight_min": -26.746788024902344,
      "activations/layer13_attention_weight_max": 40.76857376098633,
      "activations/layer13_attention_weight_min": -38.135780334472656,
      "activations/layer14_attention_weight_max": 45.60210037231445,
      "activations/layer14_attention_weight_min": -37.942100524902344,
      "activations/layer15_attention_weight_max": 40.6024055480957,
      "activations/layer15_attention_weight_min": -36.72373580932617,
      "activations/layer16_attention_weight_max": 29.66252326965332,
      "activations/layer16_attention_weight_min": -29.94523048400879,
      "activations/layer17_attention_weight_max": 29.17198371887207,
      "activations/layer17_attention_weight_min": -28.00895118713379,
      "activations/layer18_attention_weight_max": 31.412029266357422,
      "activations/layer18_attention_weight_min": -26.64627456665039,
      "activations/layer19_attention_weight_max": 34.22385787963867,
      "activations/layer19_attention_weight_min": -34.25684356689453,
      "activations/layer1_attention_weight_max": 16.545791625976562,
      "activations/layer1_attention_weight_min": -15.24930191040039,
      "activations/layer20_attention_weight_max": 28.76605224609375,
      "activations/layer20_attention_weight_min": -27.196922302246094,
      "activations/layer21_attention_weight_max": 30.25075340270996,
      "activations/layer21_attention_weight_min": -27.015079498291016,
      "activations/layer22_attention_weight_max": 45.69183349609375,
      "activations/layer22_attention_weight_min": -34.579490661621094,
      "activations/layer23_attention_weight_max": 38.2475700378418,
      "activations/layer23_attention_weight_min": -27.392425537109375,
      "activations/layer2_attention_weight_max": 35.459007263183594,
      "activations/layer2_attention_weight_min": -33.96372985839844,
      "activations/layer3_attention_weight_max": 106.05011749267578,
      "activations/layer3_attention_weight_min": -110.55831909179688,
      "activations/layer4_attention_weight_max": 111.45361328125,
      "activations/layer4_attention_weight_min": -117.3672866821289,
      "activations/layer5_attention_weight_max": 50.87074279785156,
      "activations/layer5_attention_weight_min": -65.39744567871094,
      "activations/layer6_attention_weight_max": 45.969871520996094,
      "activations/layer6_attention_weight_min": -49.017574310302734,
      "activations/layer7_attention_weight_max": 96.88018798828125,
      "activations/layer7_attention_weight_min": -109.47627258300781,
      "activations/layer8_attention_weight_max": 44.04042434692383,
      "activations/layer8_attention_weight_min": -46.09196472167969,
      "activations/layer9_attention_weight_max": 32.67831802368164,
      "activations/layer9_attention_weight_min": -36.1682243347168,
      "epoch": 22.23,
      "learning_rate": 6.722348484848484e-06,
      "loss": 2.6963,
      "step": 382550
    },
    {
      "activations/layer0_attention_weight_max": 18.082366943359375,
      "activations/layer0_attention_weight_min": -14.402900695800781,
      "activations/layer10_attention_weight_max": 34.24834442138672,
      "activations/layer10_attention_weight_min": -34.492286682128906,
      "activations/layer11_attention_weight_max": 32.67139434814453,
      "activations/layer11_attention_weight_min": -34.63047409057617,
      "activations/layer12_attention_weight_max": 25.068496704101562,
      "activations/layer12_attention_weight_min": -30.625185012817383,
      "activations/layer13_attention_weight_max": 38.43999099731445,
      "activations/layer13_attention_weight_min": -37.269309997558594,
      "activations/layer14_attention_weight_max": 40.77030563354492,
      "activations/layer14_attention_weight_min": -37.05378723144531,
      "activations/layer15_attention_weight_max": 37.802406311035156,
      "activations/layer15_attention_weight_min": -34.56890869140625,
      "activations/layer16_attention_weight_max": 30.418169021606445,
      "activations/layer16_attention_weight_min": -28.304452896118164,
      "activations/layer17_attention_weight_max": 28.525787353515625,
      "activations/layer17_attention_weight_min": -26.669897079467773,
      "activations/layer18_attention_weight_max": 31.169479370117188,
      "activations/layer18_attention_weight_min": -26.763607025146484,
      "activations/layer19_attention_weight_max": 32.72015380859375,
      "activations/layer19_attention_weight_min": -31.4754638671875,
      "activations/layer1_attention_weight_max": 16.5689640045166,
      "activations/layer1_attention_weight_min": -15.02927303314209,
      "activations/layer20_attention_weight_max": 27.80281639099121,
      "activations/layer20_attention_weight_min": -24.70036506652832,
      "activations/layer21_attention_weight_max": 26.243709564208984,
      "activations/layer21_attention_weight_min": -26.19036865234375,
      "activations/layer22_attention_weight_max": 40.42298889160156,
      "activations/layer22_attention_weight_min": -34.4176025390625,
      "activations/layer23_attention_weight_max": 32.87253952026367,
      "activations/layer23_attention_weight_min": -27.871105194091797,
      "activations/layer2_attention_weight_max": 33.93037414550781,
      "activations/layer2_attention_weight_min": -33.97946548461914,
      "activations/layer3_attention_weight_max": 105.24898529052734,
      "activations/layer3_attention_weight_min": -106.39961242675781,
      "activations/layer4_attention_weight_max": 110.73583984375,
      "activations/layer4_attention_weight_min": -114.47234344482422,
      "activations/layer5_attention_weight_max": 49.647274017333984,
      "activations/layer5_attention_weight_min": -62.6124267578125,
      "activations/layer6_attention_weight_max": 45.28107833862305,
      "activations/layer6_attention_weight_min": -49.57227325439453,
      "activations/layer7_attention_weight_max": 96.25667572021484,
      "activations/layer7_attention_weight_min": -100.30316925048828,
      "activations/layer8_attention_weight_max": 42.03041076660156,
      "activations/layer8_attention_weight_min": -45.714874267578125,
      "activations/layer9_attention_weight_max": 32.85564422607422,
      "activations/layer9_attention_weight_min": -35.1706428527832,
      "epoch": 22.23,
      "learning_rate": 6.703409090909091e-06,
      "loss": 2.7066,
      "step": 382600
    },
    {
      "activations/layer0_attention_weight_max": 17.047300338745117,
      "activations/layer0_attention_weight_min": -15.916891098022461,
      "activations/layer10_attention_weight_max": 36.784034729003906,
      "activations/layer10_attention_weight_min": -36.64060974121094,
      "activations/layer11_attention_weight_max": 36.0189208984375,
      "activations/layer11_attention_weight_min": -36.13570785522461,
      "activations/layer12_attention_weight_max": 27.410783767700195,
      "activations/layer12_attention_weight_min": -26.860366821289062,
      "activations/layer13_attention_weight_max": 41.246307373046875,
      "activations/layer13_attention_weight_min": -35.854248046875,
      "activations/layer14_attention_weight_max": 40.69076919555664,
      "activations/layer14_attention_weight_min": -42.110557556152344,
      "activations/layer15_attention_weight_max": 40.20375061035156,
      "activations/layer15_attention_weight_min": -40.46515655517578,
      "activations/layer16_attention_weight_max": 28.635026931762695,
      "activations/layer16_attention_weight_min": -29.87924575805664,
      "activations/layer17_attention_weight_max": 29.212215423583984,
      "activations/layer17_attention_weight_min": -26.285253524780273,
      "activations/layer18_attention_weight_max": 33.14773178100586,
      "activations/layer18_attention_weight_min": -27.268503189086914,
      "activations/layer19_attention_weight_max": 30.2652530670166,
      "activations/layer19_attention_weight_min": -31.2696590423584,
      "activations/layer1_attention_weight_max": 15.474605560302734,
      "activations/layer1_attention_weight_min": -14.615439414978027,
      "activations/layer20_attention_weight_max": 24.836591720581055,
      "activations/layer20_attention_weight_min": -25.418960571289062,
      "activations/layer21_attention_weight_max": 25.973508834838867,
      "activations/layer21_attention_weight_min": -24.85686683654785,
      "activations/layer22_attention_weight_max": 39.041954040527344,
      "activations/layer22_attention_weight_min": -30.28399658203125,
      "activations/layer23_attention_weight_max": 28.07619857788086,
      "activations/layer23_attention_weight_min": -27.14386749267578,
      "activations/layer2_attention_weight_max": 33.79090881347656,
      "activations/layer2_attention_weight_min": -33.75525665283203,
      "activations/layer3_attention_weight_max": 106.9197769165039,
      "activations/layer3_attention_weight_min": -108.52911376953125,
      "activations/layer4_attention_weight_max": 113.3521957397461,
      "activations/layer4_attention_weight_min": -112.56835174560547,
      "activations/layer5_attention_weight_max": 50.76020050048828,
      "activations/layer5_attention_weight_min": -62.84642791748047,
      "activations/layer6_attention_weight_max": 47.99757766723633,
      "activations/layer6_attention_weight_min": -49.50685501098633,
      "activations/layer7_attention_weight_max": 102.89057922363281,
      "activations/layer7_attention_weight_min": -107.49822235107422,
      "activations/layer8_attention_weight_max": 47.806373596191406,
      "activations/layer8_attention_weight_min": -47.56290817260742,
      "activations/layer9_attention_weight_max": 36.161231994628906,
      "activations/layer9_attention_weight_min": -37.020023345947266,
      "epoch": 22.23,
      "learning_rate": 6.684469696969696e-06,
      "loss": 2.6956,
      "step": 382650
    },
    {
      "activations/layer0_attention_weight_max": 16.601335525512695,
      "activations/layer0_attention_weight_min": -13.710412979125977,
      "activations/layer10_attention_weight_max": 34.95539855957031,
      "activations/layer10_attention_weight_min": -37.33671188354492,
      "activations/layer11_attention_weight_max": 36.49399185180664,
      "activations/layer11_attention_weight_min": -38.514732360839844,
      "activations/layer12_attention_weight_max": 29.06930160522461,
      "activations/layer12_attention_weight_min": -27.727575302124023,
      "activations/layer13_attention_weight_max": 43.43459701538086,
      "activations/layer13_attention_weight_min": -39.05168533325195,
      "activations/layer14_attention_weight_max": 47.11521530151367,
      "activations/layer14_attention_weight_min": -40.62166976928711,
      "activations/layer15_attention_weight_max": 44.796871185302734,
      "activations/layer15_attention_weight_min": -38.81023406982422,
      "activations/layer16_attention_weight_max": 32.79070281982422,
      "activations/layer16_attention_weight_min": -31.805822372436523,
      "activations/layer17_attention_weight_max": 30.451416015625,
      "activations/layer17_attention_weight_min": -26.78272247314453,
      "activations/layer18_attention_weight_max": 32.157814025878906,
      "activations/layer18_attention_weight_min": -26.480623245239258,
      "activations/layer19_attention_weight_max": 36.75767135620117,
      "activations/layer19_attention_weight_min": -31.66339111328125,
      "activations/layer1_attention_weight_max": 15.542922973632812,
      "activations/layer1_attention_weight_min": -14.901946067810059,
      "activations/layer20_attention_weight_max": 33.1267204284668,
      "activations/layer20_attention_weight_min": -24.414094924926758,
      "activations/layer21_attention_weight_max": 34.0482177734375,
      "activations/layer21_attention_weight_min": -23.989444732666016,
      "activations/layer22_attention_weight_max": 46.21414566040039,
      "activations/layer22_attention_weight_min": -31.99521827697754,
      "activations/layer23_attention_weight_max": 33.68789291381836,
      "activations/layer23_attention_weight_min": -24.851703643798828,
      "activations/layer2_attention_weight_max": 33.67009735107422,
      "activations/layer2_attention_weight_min": -34.88798141479492,
      "activations/layer3_attention_weight_max": 106.60530853271484,
      "activations/layer3_attention_weight_min": -106.2587890625,
      "activations/layer4_attention_weight_max": 109.57442474365234,
      "activations/layer4_attention_weight_min": -116.84716033935547,
      "activations/layer5_attention_weight_max": 53.523860931396484,
      "activations/layer5_attention_weight_min": -62.2358512878418,
      "activations/layer6_attention_weight_max": 47.076297760009766,
      "activations/layer6_attention_weight_min": -48.35751724243164,
      "activations/layer7_attention_weight_max": 102.5364761352539,
      "activations/layer7_attention_weight_min": -108.0367660522461,
      "activations/layer8_attention_weight_max": 45.88340759277344,
      "activations/layer8_attention_weight_min": -48.096981048583984,
      "activations/layer9_attention_weight_max": 34.988407135009766,
      "activations/layer9_attention_weight_min": -37.29856872558594,
      "epoch": 22.24,
      "learning_rate": 6.665530303030302e-06,
      "loss": 2.6944,
      "step": 382700
    },
    {
      "activations/layer0_attention_weight_max": 16.969324111938477,
      "activations/layer0_attention_weight_min": -13.028489112854004,
      "activations/layer10_attention_weight_max": 40.85787582397461,
      "activations/layer10_attention_weight_min": -40.55726623535156,
      "activations/layer11_attention_weight_max": 39.696685791015625,
      "activations/layer11_attention_weight_min": -39.93360137939453,
      "activations/layer12_attention_weight_max": 31.28436851501465,
      "activations/layer12_attention_weight_min": -29.881853103637695,
      "activations/layer13_attention_weight_max": 46.34193420410156,
      "activations/layer13_attention_weight_min": -40.15769958496094,
      "activations/layer14_attention_weight_max": 52.49300003051758,
      "activations/layer14_attention_weight_min": -43.295799255371094,
      "activations/layer15_attention_weight_max": 53.03683090209961,
      "activations/layer15_attention_weight_min": -40.693965911865234,
      "activations/layer16_attention_weight_max": 34.28689193725586,
      "activations/layer16_attention_weight_min": -30.535932540893555,
      "activations/layer17_attention_weight_max": 35.32244110107422,
      "activations/layer17_attention_weight_min": -28.154428482055664,
      "activations/layer18_attention_weight_max": 37.479862213134766,
      "activations/layer18_attention_weight_min": -28.210792541503906,
      "activations/layer19_attention_weight_max": 39.89724349975586,
      "activations/layer19_attention_weight_min": -35.63532257080078,
      "activations/layer1_attention_weight_max": 15.699325561523438,
      "activations/layer1_attention_weight_min": -15.61099910736084,
      "activations/layer20_attention_weight_max": 32.63169860839844,
      "activations/layer20_attention_weight_min": -27.347312927246094,
      "activations/layer21_attention_weight_max": 30.502239227294922,
      "activations/layer21_attention_weight_min": -26.307796478271484,
      "activations/layer22_attention_weight_max": 45.70917510986328,
      "activations/layer22_attention_weight_min": -33.250423431396484,
      "activations/layer23_attention_weight_max": 32.177608489990234,
      "activations/layer23_attention_weight_min": -26.280536651611328,
      "activations/layer2_attention_weight_max": 34.15637969970703,
      "activations/layer2_attention_weight_min": -35.299522399902344,
      "activations/layer3_attention_weight_max": 110.21163940429688,
      "activations/layer3_attention_weight_min": -108.78974151611328,
      "activations/layer4_attention_weight_max": 111.7569351196289,
      "activations/layer4_attention_weight_min": -115.55474090576172,
      "activations/layer5_attention_weight_max": 54.091064453125,
      "activations/layer5_attention_weight_min": -64.13496398925781,
      "activations/layer6_attention_weight_max": 47.4002799987793,
      "activations/layer6_attention_weight_min": -50.32024383544922,
      "activations/layer7_attention_weight_max": 101.65189361572266,
      "activations/layer7_attention_weight_min": -111.25749969482422,
      "activations/layer8_attention_weight_max": 46.16838455200195,
      "activations/layer8_attention_weight_min": -53.09178924560547,
      "activations/layer9_attention_weight_max": 36.44926834106445,
      "activations/layer9_attention_weight_min": -39.302581787109375,
      "epoch": 22.24,
      "learning_rate": 6.646590909090908e-06,
      "loss": 2.6929,
      "step": 382750
    },
    {
      "activations/layer0_attention_weight_max": 16.472999572753906,
      "activations/layer0_attention_weight_min": -13.384672164916992,
      "activations/layer10_attention_weight_max": 38.1221809387207,
      "activations/layer10_attention_weight_min": -39.65701675415039,
      "activations/layer11_attention_weight_max": 37.343223571777344,
      "activations/layer11_attention_weight_min": -37.37925720214844,
      "activations/layer12_attention_weight_max": 27.47121810913086,
      "activations/layer12_attention_weight_min": -27.751758575439453,
      "activations/layer13_attention_weight_max": 42.1037712097168,
      "activations/layer13_attention_weight_min": -37.33211135864258,
      "activations/layer14_attention_weight_max": 44.872406005859375,
      "activations/layer14_attention_weight_min": -39.23318862915039,
      "activations/layer15_attention_weight_max": 44.7048454284668,
      "activations/layer15_attention_weight_min": -35.99462127685547,
      "activations/layer16_attention_weight_max": 29.6956844329834,
      "activations/layer16_attention_weight_min": -29.589651107788086,
      "activations/layer17_attention_weight_max": 32.926429748535156,
      "activations/layer17_attention_weight_min": -26.449567794799805,
      "activations/layer18_attention_weight_max": 36.83387756347656,
      "activations/layer18_attention_weight_min": -26.165632247924805,
      "activations/layer19_attention_weight_max": 35.60578918457031,
      "activations/layer19_attention_weight_min": -32.39192199707031,
      "activations/layer1_attention_weight_max": 15.201143264770508,
      "activations/layer1_attention_weight_min": -14.856070518493652,
      "activations/layer20_attention_weight_max": 29.632061004638672,
      "activations/layer20_attention_weight_min": -26.51738166809082,
      "activations/layer21_attention_weight_max": 28.960716247558594,
      "activations/layer21_attention_weight_min": -24.58975601196289,
      "activations/layer22_attention_weight_max": 44.45242691040039,
      "activations/layer22_attention_weight_min": -31.816314697265625,
      "activations/layer23_attention_weight_max": 31.76999282836914,
      "activations/layer23_attention_weight_min": -26.944580078125,
      "activations/layer2_attention_weight_max": 33.41947937011719,
      "activations/layer2_attention_weight_min": -33.722537994384766,
      "activations/layer3_attention_weight_max": 108.38159942626953,
      "activations/layer3_attention_weight_min": -110.47722625732422,
      "activations/layer4_attention_weight_max": 109.7315673828125,
      "activations/layer4_attention_weight_min": -112.67842102050781,
      "activations/layer5_attention_weight_max": 50.11750793457031,
      "activations/layer5_attention_weight_min": -61.899169921875,
      "activations/layer6_attention_weight_max": 45.24614334106445,
      "activations/layer6_attention_weight_min": -47.825923919677734,
      "activations/layer7_attention_weight_max": 95.28781127929688,
      "activations/layer7_attention_weight_min": -102.26545715332031,
      "activations/layer8_attention_weight_max": 46.88475036621094,
      "activations/layer8_attention_weight_min": -45.553260803222656,
      "activations/layer9_attention_weight_max": 34.50740432739258,
      "activations/layer9_attention_weight_min": -37.119197845458984,
      "epoch": 22.24,
      "learning_rate": 6.627651515151515e-06,
      "loss": 2.6845,
      "step": 382800
    },
    {
      "activations/layer0_attention_weight_max": 17.763671875,
      "activations/layer0_attention_weight_min": -12.836782455444336,
      "activations/layer10_attention_weight_max": 35.85499954223633,
      "activations/layer10_attention_weight_min": -37.72378921508789,
      "activations/layer11_attention_weight_max": 34.28827667236328,
      "activations/layer11_attention_weight_min": -35.89856719970703,
      "activations/layer12_attention_weight_max": 26.976058959960938,
      "activations/layer12_attention_weight_min": -26.215579986572266,
      "activations/layer13_attention_weight_max": 40.74761199951172,
      "activations/layer13_attention_weight_min": -35.336280822753906,
      "activations/layer14_attention_weight_max": 42.89745330810547,
      "activations/layer14_attention_weight_min": -38.034236907958984,
      "activations/layer15_attention_weight_max": 40.52470397949219,
      "activations/layer15_attention_weight_min": -36.56640625,
      "activations/layer16_attention_weight_max": 29.282081604003906,
      "activations/layer16_attention_weight_min": -27.281299591064453,
      "activations/layer17_attention_weight_max": 30.54254913330078,
      "activations/layer17_attention_weight_min": -26.590129852294922,
      "activations/layer18_attention_weight_max": 33.04557418823242,
      "activations/layer18_attention_weight_min": -26.195327758789062,
      "activations/layer19_attention_weight_max": 32.50765609741211,
      "activations/layer19_attention_weight_min": -32.10215377807617,
      "activations/layer1_attention_weight_max": 15.992355346679688,
      "activations/layer1_attention_weight_min": -14.927663803100586,
      "activations/layer20_attention_weight_max": 27.19767951965332,
      "activations/layer20_attention_weight_min": -25.485424041748047,
      "activations/layer21_attention_weight_max": 27.723506927490234,
      "activations/layer21_attention_weight_min": -24.739187240600586,
      "activations/layer22_attention_weight_max": 43.372474670410156,
      "activations/layer22_attention_weight_min": -31.158599853515625,
      "activations/layer23_attention_weight_max": 29.763980865478516,
      "activations/layer23_attention_weight_min": -23.9395751953125,
      "activations/layer2_attention_weight_max": 34.60144805908203,
      "activations/layer2_attention_weight_min": -32.594696044921875,
      "activations/layer3_attention_weight_max": 109.6272201538086,
      "activations/layer3_attention_weight_min": -107.98820495605469,
      "activations/layer4_attention_weight_max": 114.59329986572266,
      "activations/layer4_attention_weight_min": -112.9397201538086,
      "activations/layer5_attention_weight_max": 53.44441223144531,
      "activations/layer5_attention_weight_min": -62.778953552246094,
      "activations/layer6_attention_weight_max": 46.91853332519531,
      "activations/layer6_attention_weight_min": -50.40007400512695,
      "activations/layer7_attention_weight_max": 102.40129089355469,
      "activations/layer7_attention_weight_min": -107.135009765625,
      "activations/layer8_attention_weight_max": 44.536476135253906,
      "activations/layer8_attention_weight_min": -50.11688232421875,
      "activations/layer9_attention_weight_max": 33.096717834472656,
      "activations/layer9_attention_weight_min": -37.15042495727539,
      "epoch": 22.25,
      "learning_rate": 6.608712121212121e-06,
      "loss": 2.7113,
      "step": 382850
    },
    {
      "activations/layer0_attention_weight_max": 17.270687103271484,
      "activations/layer0_attention_weight_min": -13.905638694763184,
      "activations/layer10_attention_weight_max": 37.0300178527832,
      "activations/layer10_attention_weight_min": -37.395904541015625,
      "activations/layer11_attention_weight_max": 37.0688362121582,
      "activations/layer11_attention_weight_min": -38.13545227050781,
      "activations/layer12_attention_weight_max": 28.954242706298828,
      "activations/layer12_attention_weight_min": -28.745386123657227,
      "activations/layer13_attention_weight_max": 41.36954879760742,
      "activations/layer13_attention_weight_min": -39.15745162963867,
      "activations/layer14_attention_weight_max": 44.77336883544922,
      "activations/layer14_attention_weight_min": -39.6693229675293,
      "activations/layer15_attention_weight_max": 42.588523864746094,
      "activations/layer15_attention_weight_min": -39.50556564331055,
      "activations/layer16_attention_weight_max": 32.00514602661133,
      "activations/layer16_attention_weight_min": -29.39480209350586,
      "activations/layer17_attention_weight_max": 30.517446517944336,
      "activations/layer17_attention_weight_min": -27.160640716552734,
      "activations/layer18_attention_weight_max": 32.850093841552734,
      "activations/layer18_attention_weight_min": -28.09559440612793,
      "activations/layer19_attention_weight_max": 35.74665451049805,
      "activations/layer19_attention_weight_min": -33.50252151489258,
      "activations/layer1_attention_weight_max": 17.14093017578125,
      "activations/layer1_attention_weight_min": -16.954330444335938,
      "activations/layer20_attention_weight_max": 29.961164474487305,
      "activations/layer20_attention_weight_min": -26.264738082885742,
      "activations/layer21_attention_weight_max": 29.80043601989746,
      "activations/layer21_attention_weight_min": -26.161209106445312,
      "activations/layer22_attention_weight_max": 41.13679122924805,
      "activations/layer22_attention_weight_min": -34.66636276245117,
      "activations/layer23_attention_weight_max": 30.612552642822266,
      "activations/layer23_attention_weight_min": -29.655723571777344,
      "activations/layer2_attention_weight_max": 35.648048400878906,
      "activations/layer2_attention_weight_min": -33.627716064453125,
      "activations/layer3_attention_weight_max": 105.79216766357422,
      "activations/layer3_attention_weight_min": -108.07709503173828,
      "activations/layer4_attention_weight_max": 115.25001525878906,
      "activations/layer4_attention_weight_min": -114.96473693847656,
      "activations/layer5_attention_weight_max": 52.49253463745117,
      "activations/layer5_attention_weight_min": -62.365562438964844,
      "activations/layer6_attention_weight_max": 48.962825775146484,
      "activations/layer6_attention_weight_min": -52.09748077392578,
      "activations/layer7_attention_weight_max": 99.61502838134766,
      "activations/layer7_attention_weight_min": -108.71063232421875,
      "activations/layer8_attention_weight_max": 46.45711135864258,
      "activations/layer8_attention_weight_min": -47.595516204833984,
      "activations/layer9_attention_weight_max": 34.58915710449219,
      "activations/layer9_attention_weight_min": -36.91819381713867,
      "epoch": 22.25,
      "learning_rate": 6.589772727272727e-06,
      "loss": 2.6918,
      "step": 382900
    },
    {
      "activations/layer0_attention_weight_max": 17.7539119720459,
      "activations/layer0_attention_weight_min": -15.419585227966309,
      "activations/layer10_attention_weight_max": 35.592437744140625,
      "activations/layer10_attention_weight_min": -36.9854850769043,
      "activations/layer11_attention_weight_max": 37.05541229248047,
      "activations/layer11_attention_weight_min": -36.26545715332031,
      "activations/layer12_attention_weight_max": 27.481063842773438,
      "activations/layer12_attention_weight_min": -27.892841339111328,
      "activations/layer13_attention_weight_max": 41.28717041015625,
      "activations/layer13_attention_weight_min": -36.58480453491211,
      "activations/layer14_attention_weight_max": 47.1988410949707,
      "activations/layer14_attention_weight_min": -39.12873077392578,
      "activations/layer15_attention_weight_max": 39.632957458496094,
      "activations/layer15_attention_weight_min": -38.009315490722656,
      "activations/layer16_attention_weight_max": 30.045059204101562,
      "activations/layer16_attention_weight_min": -28.62785530090332,
      "activations/layer17_attention_weight_max": 30.727434158325195,
      "activations/layer17_attention_weight_min": -27.698257446289062,
      "activations/layer18_attention_weight_max": 31.75294303894043,
      "activations/layer18_attention_weight_min": -26.826213836669922,
      "activations/layer19_attention_weight_max": 36.897918701171875,
      "activations/layer19_attention_weight_min": -31.964750289916992,
      "activations/layer1_attention_weight_max": 16.963809967041016,
      "activations/layer1_attention_weight_min": -15.586647033691406,
      "activations/layer20_attention_weight_max": 30.361919403076172,
      "activations/layer20_attention_weight_min": -25.1393985748291,
      "activations/layer21_attention_weight_max": 30.127208709716797,
      "activations/layer21_attention_weight_min": -25.12480926513672,
      "activations/layer22_attention_weight_max": 38.20918655395508,
      "activations/layer22_attention_weight_min": -30.73883056640625,
      "activations/layer23_attention_weight_max": 32.0052604675293,
      "activations/layer23_attention_weight_min": -25.02322769165039,
      "activations/layer2_attention_weight_max": 34.67483139038086,
      "activations/layer2_attention_weight_min": -35.28704833984375,
      "activations/layer3_attention_weight_max": 104.36068725585938,
      "activations/layer3_attention_weight_min": -108.87220001220703,
      "activations/layer4_attention_weight_max": 108.5247573852539,
      "activations/layer4_attention_weight_min": -111.4075927734375,
      "activations/layer5_attention_weight_max": 49.59435272216797,
      "activations/layer5_attention_weight_min": -59.76848602294922,
      "activations/layer6_attention_weight_max": 46.63856887817383,
      "activations/layer6_attention_weight_min": -48.326316833496094,
      "activations/layer7_attention_weight_max": 99.68464660644531,
      "activations/layer7_attention_weight_min": -99.42806243896484,
      "activations/layer8_attention_weight_max": 44.4519157409668,
      "activations/layer8_attention_weight_min": -46.75191879272461,
      "activations/layer9_attention_weight_max": 35.76656723022461,
      "activations/layer9_attention_weight_min": -37.02865219116211,
      "epoch": 22.25,
      "learning_rate": 6.570833333333332e-06,
      "loss": 2.7083,
      "step": 382950
    },
    {
      "activations/layer0_attention_weight_max": 16.533214569091797,
      "activations/layer0_attention_weight_min": -15.491021156311035,
      "activations/layer10_attention_weight_max": 34.933231353759766,
      "activations/layer10_attention_weight_min": -34.5621337890625,
      "activations/layer11_attention_weight_max": 32.950340270996094,
      "activations/layer11_attention_weight_min": -34.046077728271484,
      "activations/layer12_attention_weight_max": 26.827051162719727,
      "activations/layer12_attention_weight_min": -27.900657653808594,
      "activations/layer13_attention_weight_max": 38.80311965942383,
      "activations/layer13_attention_weight_min": -37.71228790283203,
      "activations/layer14_attention_weight_max": 42.2958869934082,
      "activations/layer14_attention_weight_min": -40.476165771484375,
      "activations/layer15_attention_weight_max": 36.43578338623047,
      "activations/layer15_attention_weight_min": -38.25507354736328,
      "activations/layer16_attention_weight_max": 30.18109893798828,
      "activations/layer16_attention_weight_min": -29.79336929321289,
      "activations/layer17_attention_weight_max": 29.69779396057129,
      "activations/layer17_attention_weight_min": -27.458477020263672,
      "activations/layer18_attention_weight_max": 31.621732711791992,
      "activations/layer18_attention_weight_min": -28.997793197631836,
      "activations/layer19_attention_weight_max": 34.35625076293945,
      "activations/layer19_attention_weight_min": -32.92388153076172,
      "activations/layer1_attention_weight_max": 16.666301727294922,
      "activations/layer1_attention_weight_min": -15.006287574768066,
      "activations/layer20_attention_weight_max": 27.086570739746094,
      "activations/layer20_attention_weight_min": -26.534263610839844,
      "activations/layer21_attention_weight_max": 27.11787986755371,
      "activations/layer21_attention_weight_min": -25.620328903198242,
      "activations/layer22_attention_weight_max": 39.598567962646484,
      "activations/layer22_attention_weight_min": -34.57826614379883,
      "activations/layer23_attention_weight_max": 31.24549674987793,
      "activations/layer23_attention_weight_min": -29.093475341796875,
      "activations/layer2_attention_weight_max": 35.972286224365234,
      "activations/layer2_attention_weight_min": -36.819114685058594,
      "activations/layer3_attention_weight_max": 104.6296157836914,
      "activations/layer3_attention_weight_min": -110.95256805419922,
      "activations/layer4_attention_weight_max": 109.59380340576172,
      "activations/layer4_attention_weight_min": -111.72896575927734,
      "activations/layer5_attention_weight_max": 49.92634201049805,
      "activations/layer5_attention_weight_min": -58.95444869995117,
      "activations/layer6_attention_weight_max": 44.23048782348633,
      "activations/layer6_attention_weight_min": -48.24162673950195,
      "activations/layer7_attention_weight_max": 100.1113510131836,
      "activations/layer7_attention_weight_min": -97.3110580444336,
      "activations/layer8_attention_weight_max": 41.785621643066406,
      "activations/layer8_attention_weight_min": -43.90059280395508,
      "activations/layer9_attention_weight_max": 33.13853073120117,
      "activations/layer9_attention_weight_min": -33.8562126159668,
      "epoch": 22.25,
      "learning_rate": 6.551893939393939e-06,
      "loss": 2.7002,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.7139,
      "eval_samples_per_second": 492.777,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.7139,
      "eval_openwebtext_samples_per_second": 492.777,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.9248,
      "eval_wikitext_samples_per_second": 236.903,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_lambada_loss": 2.33984375,
      "eval_lambada_ppl": 10.379614621236398,
      "eval_lambada_runtime": 9.5376,
      "eval_lambada_samples_per_second": 510.507,
      "step": 383000
    },
    {
      "activations/layer0_attention_weight_max": 16.204524993896484,
      "activations/layer0_attention_weight_min": -13.921077728271484,
      "activations/layer10_attention_weight_max": 38.974586486816406,
      "activations/layer10_attention_weight_min": -37.12873077392578,
      "activations/layer11_attention_weight_max": 37.14520263671875,
      "activations/layer11_attention_weight_min": -38.200233459472656,
      "activations/layer12_attention_weight_max": 27.196203231811523,
      "activations/layer12_attention_weight_min": -28.793766021728516,
      "activations/layer13_attention_weight_max": 40.78226852416992,
      "activations/layer13_attention_weight_min": -40.80199432373047,
      "activations/layer14_attention_weight_max": 48.61618423461914,
      "activations/layer14_attention_weight_min": -42.49372863769531,
      "activations/layer15_attention_weight_max": 42.77838134765625,
      "activations/layer15_attention_weight_min": -41.90568542480469,
      "activations/layer16_attention_weight_max": 30.24662971496582,
      "activations/layer16_attention_weight_min": -28.743619918823242,
      "activations/layer17_attention_weight_max": 29.792551040649414,
      "activations/layer17_attention_weight_min": -26.656126022338867,
      "activations/layer18_attention_weight_max": 32.124027252197266,
      "activations/layer18_attention_weight_min": -26.8807315826416,
      "activations/layer19_attention_weight_max": 34.114402770996094,
      "activations/layer19_attention_weight_min": -33.73912048339844,
      "activations/layer1_attention_weight_max": 15.568724632263184,
      "activations/layer1_attention_weight_min": -16.473228454589844,
      "activations/layer20_attention_weight_max": 27.633399963378906,
      "activations/layer20_attention_weight_min": -26.338390350341797,
      "activations/layer21_attention_weight_max": 28.68175506591797,
      "activations/layer21_attention_weight_min": -25.37930679321289,
      "activations/layer22_attention_weight_max": 39.327693939208984,
      "activations/layer22_attention_weight_min": -33.35279846191406,
      "activations/layer23_attention_weight_max": 31.999610900878906,
      "activations/layer23_attention_weight_min": -25.640289306640625,
      "activations/layer2_attention_weight_max": 36.05641174316406,
      "activations/layer2_attention_weight_min": -34.59954833984375,
      "activations/layer3_attention_weight_max": 108.42807006835938,
      "activations/layer3_attention_weight_min": -107.91728210449219,
      "activations/layer4_attention_weight_max": 113.428955078125,
      "activations/layer4_attention_weight_min": -112.34358215332031,
      "activations/layer5_attention_weight_max": 55.52791976928711,
      "activations/layer5_attention_weight_min": -61.02401351928711,
      "activations/layer6_attention_weight_max": 49.2368278503418,
      "activations/layer6_attention_weight_min": -50.87715148925781,
      "activations/layer7_attention_weight_max": 98.89418029785156,
      "activations/layer7_attention_weight_min": -102.97074890136719,
      "activations/layer8_attention_weight_max": 46.87678146362305,
      "activations/layer8_attention_weight_min": -46.715370178222656,
      "activations/layer9_attention_weight_max": 36.060062408447266,
      "activations/layer9_attention_weight_min": -37.81606674194336,
      "epoch": 22.26,
      "learning_rate": 6.532954545454545e-06,
      "loss": 2.7015,
      "step": 383050
    },
    {
      "activations/layer0_attention_weight_max": 17.126041412353516,
      "activations/layer0_attention_weight_min": -14.833864212036133,
      "activations/layer10_attention_weight_max": 36.418701171875,
      "activations/layer10_attention_weight_min": -36.205589294433594,
      "activations/layer11_attention_weight_max": 38.0900993347168,
      "activations/layer11_attention_weight_min": -38.60020065307617,
      "activations/layer12_attention_weight_max": 26.515949249267578,
      "activations/layer12_attention_weight_min": -28.937219619750977,
      "activations/layer13_attention_weight_max": 40.334407806396484,
      "activations/layer13_attention_weight_min": -38.640357971191406,
      "activations/layer14_attention_weight_max": 43.5518913269043,
      "activations/layer14_attention_weight_min": -41.76161575317383,
      "activations/layer15_attention_weight_max": 41.83788299560547,
      "activations/layer15_attention_weight_min": -39.66102600097656,
      "activations/layer16_attention_weight_max": 30.55570411682129,
      "activations/layer16_attention_weight_min": -29.86237144470215,
      "activations/layer17_attention_weight_max": 30.614139556884766,
      "activations/layer17_attention_weight_min": -27.762510299682617,
      "activations/layer18_attention_weight_max": 34.32675552368164,
      "activations/layer18_attention_weight_min": -27.42413902282715,
      "activations/layer19_attention_weight_max": 41.06342697143555,
      "activations/layer19_attention_weight_min": -31.588956832885742,
      "activations/layer1_attention_weight_max": 16.436391830444336,
      "activations/layer1_attention_weight_min": -15.256392478942871,
      "activations/layer20_attention_weight_max": 30.016517639160156,
      "activations/layer20_attention_weight_min": -26.207487106323242,
      "activations/layer21_attention_weight_max": 33.08441925048828,
      "activations/layer21_attention_weight_min": -25.304691314697266,
      "activations/layer22_attention_weight_max": 44.22918701171875,
      "activations/layer22_attention_weight_min": -33.12415313720703,
      "activations/layer23_attention_weight_max": 31.37419891357422,
      "activations/layer23_attention_weight_min": -26.477977752685547,
      "activations/layer2_attention_weight_max": 33.870574951171875,
      "activations/layer2_attention_weight_min": -32.173057556152344,
      "activations/layer3_attention_weight_max": 109.675048828125,
      "activations/layer3_attention_weight_min": -109.53980255126953,
      "activations/layer4_attention_weight_max": 110.33998107910156,
      "activations/layer4_attention_weight_min": -114.3399658203125,
      "activations/layer5_attention_weight_max": 49.986724853515625,
      "activations/layer5_attention_weight_min": -64.04502868652344,
      "activations/layer6_attention_weight_max": 46.08567428588867,
      "activations/layer6_attention_weight_min": -47.90343475341797,
      "activations/layer7_attention_weight_max": 99.23544311523438,
      "activations/layer7_attention_weight_min": -103.36646270751953,
      "activations/layer8_attention_weight_max": 49.6134147644043,
      "activations/layer8_attention_weight_min": -48.36669158935547,
      "activations/layer9_attention_weight_max": 33.88698959350586,
      "activations/layer9_attention_weight_min": -37.17266845703125,
      "epoch": 22.26,
      "learning_rate": 6.5140151515151504e-06,
      "loss": 2.7035,
      "step": 383100
    },
    {
      "activations/layer0_attention_weight_max": 16.92478370666504,
      "activations/layer0_attention_weight_min": -13.567872047424316,
      "activations/layer10_attention_weight_max": 39.23902130126953,
      "activations/layer10_attention_weight_min": -40.084800720214844,
      "activations/layer11_attention_weight_max": 39.5721321105957,
      "activations/layer11_attention_weight_min": -40.70445251464844,
      "activations/layer12_attention_weight_max": 33.97372055053711,
      "activations/layer12_attention_weight_min": -31.477523803710938,
      "activations/layer13_attention_weight_max": 46.49284744262695,
      "activations/layer13_attention_weight_min": -40.25434112548828,
      "activations/layer14_attention_weight_max": 47.94569396972656,
      "activations/layer14_attention_weight_min": -40.73808670043945,
      "activations/layer15_attention_weight_max": 43.34190368652344,
      "activations/layer15_attention_weight_min": -39.37543869018555,
      "activations/layer16_attention_weight_max": 33.91495132446289,
      "activations/layer16_attention_weight_min": -29.145654678344727,
      "activations/layer17_attention_weight_max": 34.2945442199707,
      "activations/layer17_attention_weight_min": -27.78524398803711,
      "activations/layer18_attention_weight_max": 33.034461975097656,
      "activations/layer18_attention_weight_min": -27.095224380493164,
      "activations/layer19_attention_weight_max": 37.0135383605957,
      "activations/layer19_attention_weight_min": -32.869991302490234,
      "activations/layer1_attention_weight_max": 17.238264083862305,
      "activations/layer1_attention_weight_min": -16.386451721191406,
      "activations/layer20_attention_weight_max": 33.46658706665039,
      "activations/layer20_attention_weight_min": -25.935422897338867,
      "activations/layer21_attention_weight_max": 33.08919143676758,
      "activations/layer21_attention_weight_min": -25.780765533447266,
      "activations/layer22_attention_weight_max": 43.9432487487793,
      "activations/layer22_attention_weight_min": -33.46583557128906,
      "activations/layer23_attention_weight_max": 32.71660232543945,
      "activations/layer23_attention_weight_min": -28.971538543701172,
      "activations/layer2_attention_weight_max": 35.871219635009766,
      "activations/layer2_attention_weight_min": -34.23041915893555,
      "activations/layer3_attention_weight_max": 110.99964141845703,
      "activations/layer3_attention_weight_min": -114.57172393798828,
      "activations/layer4_attention_weight_max": 114.03081512451172,
      "activations/layer4_attention_weight_min": -116.4527816772461,
      "activations/layer5_attention_weight_max": 53.771087646484375,
      "activations/layer5_attention_weight_min": -66.9499740600586,
      "activations/layer6_attention_weight_max": 48.207054138183594,
      "activations/layer6_attention_weight_min": -50.03495788574219,
      "activations/layer7_attention_weight_max": 105.12556457519531,
      "activations/layer7_attention_weight_min": -107.19340515136719,
      "activations/layer8_attention_weight_max": 46.4500617980957,
      "activations/layer8_attention_weight_min": -50.18693923950195,
      "activations/layer9_attention_weight_max": 35.86088562011719,
      "activations/layer9_attention_weight_min": -37.64338302612305,
      "epoch": 22.26,
      "learning_rate": 6.495075757575756e-06,
      "loss": 2.7032,
      "step": 383150
    },
    {
      "activations/layer0_attention_weight_max": 17.177305221557617,
      "activations/layer0_attention_weight_min": -14.316328048706055,
      "activations/layer10_attention_weight_max": 33.738563537597656,
      "activations/layer10_attention_weight_min": -36.37513732910156,
      "activations/layer11_attention_weight_max": 33.707462310791016,
      "activations/layer11_attention_weight_min": -36.59666442871094,
      "activations/layer12_attention_weight_max": 27.9753360748291,
      "activations/layer12_attention_weight_min": -28.2199649810791,
      "activations/layer13_attention_weight_max": 38.9572639465332,
      "activations/layer13_attention_weight_min": -37.061683654785156,
      "activations/layer14_attention_weight_max": 43.82492446899414,
      "activations/layer14_attention_weight_min": -40.423065185546875,
      "activations/layer15_attention_weight_max": 37.538368225097656,
      "activations/layer15_attention_weight_min": -36.69596862792969,
      "activations/layer16_attention_weight_max": 27.695009231567383,
      "activations/layer16_attention_weight_min": -28.834657669067383,
      "activations/layer17_attention_weight_max": 28.84336280822754,
      "activations/layer17_attention_weight_min": -26.442033767700195,
      "activations/layer18_attention_weight_max": 31.592439651489258,
      "activations/layer18_attention_weight_min": -26.19490623474121,
      "activations/layer19_attention_weight_max": 34.52378845214844,
      "activations/layer19_attention_weight_min": -31.5419979095459,
      "activations/layer1_attention_weight_max": 15.632513046264648,
      "activations/layer1_attention_weight_min": -15.321574211120605,
      "activations/layer20_attention_weight_max": 26.98094940185547,
      "activations/layer20_attention_weight_min": -25.746885299682617,
      "activations/layer21_attention_weight_max": 26.539443969726562,
      "activations/layer21_attention_weight_min": -23.887773513793945,
      "activations/layer22_attention_weight_max": 38.382843017578125,
      "activations/layer22_attention_weight_min": -33.2058219909668,
      "activations/layer23_attention_weight_max": 31.548925399780273,
      "activations/layer23_attention_weight_min": -25.182933807373047,
      "activations/layer2_attention_weight_max": 34.20542907714844,
      "activations/layer2_attention_weight_min": -33.62983703613281,
      "activations/layer3_attention_weight_max": 108.80035400390625,
      "activations/layer3_attention_weight_min": -109.84162902832031,
      "activations/layer4_attention_weight_max": 110.64778900146484,
      "activations/layer4_attention_weight_min": -108.31591033935547,
      "activations/layer5_attention_weight_max": 51.08884811401367,
      "activations/layer5_attention_weight_min": -60.531856536865234,
      "activations/layer6_attention_weight_max": 44.92445755004883,
      "activations/layer6_attention_weight_min": -48.66151428222656,
      "activations/layer7_attention_weight_max": 94.14279174804688,
      "activations/layer7_attention_weight_min": -100.98081970214844,
      "activations/layer8_attention_weight_max": 41.99652862548828,
      "activations/layer8_attention_weight_min": -45.70833969116211,
      "activations/layer9_attention_weight_max": 32.14551544189453,
      "activations/layer9_attention_weight_min": -36.44832229614258,
      "epoch": 22.27,
      "learning_rate": 6.476136363636364e-06,
      "loss": 2.7042,
      "step": 383200
    },
    {
      "activations/layer0_attention_weight_max": 17.019691467285156,
      "activations/layer0_attention_weight_min": -15.553237915039062,
      "activations/layer10_attention_weight_max": 35.965545654296875,
      "activations/layer10_attention_weight_min": -37.56060028076172,
      "activations/layer11_attention_weight_max": 37.39961242675781,
      "activations/layer11_attention_weight_min": -37.43714141845703,
      "activations/layer12_attention_weight_max": 27.933073043823242,
      "activations/layer12_attention_weight_min": -27.90705108642578,
      "activations/layer13_attention_weight_max": 41.7918701171875,
      "activations/layer13_attention_weight_min": -37.77445983886719,
      "activations/layer14_attention_weight_max": 44.83412551879883,
      "activations/layer14_attention_weight_min": -39.655643463134766,
      "activations/layer15_attention_weight_max": 39.72188186645508,
      "activations/layer15_attention_weight_min": -36.80311584472656,
      "activations/layer16_attention_weight_max": 26.888168334960938,
      "activations/layer16_attention_weight_min": -27.39637565612793,
      "activations/layer17_attention_weight_max": 28.77124786376953,
      "activations/layer17_attention_weight_min": -26.24280548095703,
      "activations/layer18_attention_weight_max": 30.975360870361328,
      "activations/layer18_attention_weight_min": -26.635562896728516,
      "activations/layer19_attention_weight_max": 32.93560028076172,
      "activations/layer19_attention_weight_min": -31.47684669494629,
      "activations/layer1_attention_weight_max": 16.07693862915039,
      "activations/layer1_attention_weight_min": -17.12099266052246,
      "activations/layer20_attention_weight_max": 29.23897933959961,
      "activations/layer20_attention_weight_min": -24.184843063354492,
      "activations/layer21_attention_weight_max": 26.54918098449707,
      "activations/layer21_attention_weight_min": -24.34642219543457,
      "activations/layer22_attention_weight_max": 41.18898391723633,
      "activations/layer22_attention_weight_min": -31.322040557861328,
      "activations/layer23_attention_weight_max": 31.336528778076172,
      "activations/layer23_attention_weight_min": -25.58905601501465,
      "activations/layer2_attention_weight_max": 35.18244171142578,
      "activations/layer2_attention_weight_min": -33.337791442871094,
      "activations/layer3_attention_weight_max": 107.59379577636719,
      "activations/layer3_attention_weight_min": -106.92632293701172,
      "activations/layer4_attention_weight_max": 113.76722717285156,
      "activations/layer4_attention_weight_min": -113.21776580810547,
      "activations/layer5_attention_weight_max": 51.81831359863281,
      "activations/layer5_attention_weight_min": -62.62800598144531,
      "activations/layer6_attention_weight_max": 48.187461853027344,
      "activations/layer6_attention_weight_min": -50.918556213378906,
      "activations/layer7_attention_weight_max": 98.67732238769531,
      "activations/layer7_attention_weight_min": -105.83354187011719,
      "activations/layer8_attention_weight_max": 45.617774963378906,
      "activations/layer8_attention_weight_min": -46.95684814453125,
      "activations/layer9_attention_weight_max": 33.98781204223633,
      "activations/layer9_attention_weight_min": -36.409095764160156,
      "epoch": 22.27,
      "learning_rate": 6.457196969696969e-06,
      "loss": 2.6967,
      "step": 383250
    },
    {
      "activations/layer0_attention_weight_max": 17.664989471435547,
      "activations/layer0_attention_weight_min": -13.485079765319824,
      "activations/layer10_attention_weight_max": 34.05291748046875,
      "activations/layer10_attention_weight_min": -34.97926712036133,
      "activations/layer11_attention_weight_max": 34.41051483154297,
      "activations/layer11_attention_weight_min": -36.07300567626953,
      "activations/layer12_attention_weight_max": 26.706132888793945,
      "activations/layer12_attention_weight_min": -30.02613067626953,
      "activations/layer13_attention_weight_max": 42.57917404174805,
      "activations/layer13_attention_weight_min": -40.11614227294922,
      "activations/layer14_attention_weight_max": 42.763389587402344,
      "activations/layer14_attention_weight_min": -40.71529769897461,
      "activations/layer15_attention_weight_max": 40.60113525390625,
      "activations/layer15_attention_weight_min": -37.743797302246094,
      "activations/layer16_attention_weight_max": 28.812837600708008,
      "activations/layer16_attention_weight_min": -28.544044494628906,
      "activations/layer17_attention_weight_max": 29.66077995300293,
      "activations/layer17_attention_weight_min": -27.741518020629883,
      "activations/layer18_attention_weight_max": 31.419164657592773,
      "activations/layer18_attention_weight_min": -28.41914176940918,
      "activations/layer19_attention_weight_max": 31.42340087890625,
      "activations/layer19_attention_weight_min": -33.01603317260742,
      "activations/layer1_attention_weight_max": 16.86003875732422,
      "activations/layer1_attention_weight_min": -15.562003135681152,
      "activations/layer20_attention_weight_max": 28.691532135009766,
      "activations/layer20_attention_weight_min": -25.38472557067871,
      "activations/layer21_attention_weight_max": 28.48598289489746,
      "activations/layer21_attention_weight_min": -24.689966201782227,
      "activations/layer22_attention_weight_max": 43.5294075012207,
      "activations/layer22_attention_weight_min": -31.866331100463867,
      "activations/layer23_attention_weight_max": 30.342906951904297,
      "activations/layer23_attention_weight_min": -24.633281707763672,
      "activations/layer2_attention_weight_max": 35.12702941894531,
      "activations/layer2_attention_weight_min": -34.58082580566406,
      "activations/layer3_attention_weight_max": 109.70333099365234,
      "activations/layer3_attention_weight_min": -108.80351257324219,
      "activations/layer4_attention_weight_max": 110.4847183227539,
      "activations/layer4_attention_weight_min": -116.01244354248047,
      "activations/layer5_attention_weight_max": 50.53187561035156,
      "activations/layer5_attention_weight_min": -63.025386810302734,
      "activations/layer6_attention_weight_max": 45.613224029541016,
      "activations/layer6_attention_weight_min": -50.85208511352539,
      "activations/layer7_attention_weight_max": 96.02375030517578,
      "activations/layer7_attention_weight_min": -106.73590850830078,
      "activations/layer8_attention_weight_max": 43.83872985839844,
      "activations/layer8_attention_weight_min": -49.1728515625,
      "activations/layer9_attention_weight_max": 33.27041244506836,
      "activations/layer9_attention_weight_min": -35.03388595581055,
      "epoch": 22.27,
      "learning_rate": 6.438257575757575e-06,
      "loss": 2.6995,
      "step": 383300
    },
    {
      "activations/layer0_attention_weight_max": 16.75398063659668,
      "activations/layer0_attention_weight_min": -13.319132804870605,
      "activations/layer10_attention_weight_max": 38.82325744628906,
      "activations/layer10_attention_weight_min": -39.25005340576172,
      "activations/layer11_attention_weight_max": 38.4698486328125,
      "activations/layer11_attention_weight_min": -39.68551254272461,
      "activations/layer12_attention_weight_max": 31.824316024780273,
      "activations/layer12_attention_weight_min": -30.626197814941406,
      "activations/layer13_attention_weight_max": 43.37907409667969,
      "activations/layer13_attention_weight_min": -39.761260986328125,
      "activations/layer14_attention_weight_max": 46.04346466064453,
      "activations/layer14_attention_weight_min": -41.40140151977539,
      "activations/layer15_attention_weight_max": 40.431968688964844,
      "activations/layer15_attention_weight_min": -38.927730560302734,
      "activations/layer16_attention_weight_max": 30.90821075439453,
      "activations/layer16_attention_weight_min": -29.33935546875,
      "activations/layer17_attention_weight_max": 31.95376205444336,
      "activations/layer17_attention_weight_min": -26.53618812561035,
      "activations/layer18_attention_weight_max": 32.764041900634766,
      "activations/layer18_attention_weight_min": -26.327951431274414,
      "activations/layer19_attention_weight_max": 34.93156051635742,
      "activations/layer19_attention_weight_min": -32.61436080932617,
      "activations/layer1_attention_weight_max": 16.48399543762207,
      "activations/layer1_attention_weight_min": -16.410737991333008,
      "activations/layer20_attention_weight_max": 30.360076904296875,
      "activations/layer20_attention_weight_min": -25.83086395263672,
      "activations/layer21_attention_weight_max": 29.802623748779297,
      "activations/layer21_attention_weight_min": -26.566320419311523,
      "activations/layer22_attention_weight_max": 46.14241027832031,
      "activations/layer22_attention_weight_min": -33.278282165527344,
      "activations/layer23_attention_weight_max": 33.358863830566406,
      "activations/layer23_attention_weight_min": -27.253459930419922,
      "activations/layer2_attention_weight_max": 34.159690856933594,
      "activations/layer2_attention_weight_min": -33.43300247192383,
      "activations/layer3_attention_weight_max": 109.56108093261719,
      "activations/layer3_attention_weight_min": -107.58295440673828,
      "activations/layer4_attention_weight_max": 117.66912841796875,
      "activations/layer4_attention_weight_min": -116.22698211669922,
      "activations/layer5_attention_weight_max": 51.48945236206055,
      "activations/layer5_attention_weight_min": -68.10340881347656,
      "activations/layer6_attention_weight_max": 50.346656799316406,
      "activations/layer6_attention_weight_min": -51.88075256347656,
      "activations/layer7_attention_weight_max": 107.43877410888672,
      "activations/layer7_attention_weight_min": -113.31344604492188,
      "activations/layer8_attention_weight_max": 47.444793701171875,
      "activations/layer8_attention_weight_min": -52.64109802246094,
      "activations/layer9_attention_weight_max": 36.485103607177734,
      "activations/layer9_attention_weight_min": -41.66898727416992,
      "epoch": 22.27,
      "learning_rate": 6.419318181818182e-06,
      "loss": 2.6931,
      "step": 383350
    },
    {
      "activations/layer0_attention_weight_max": 17.560321807861328,
      "activations/layer0_attention_weight_min": -14.706450462341309,
      "activations/layer10_attention_weight_max": 40.62199783325195,
      "activations/layer10_attention_weight_min": -38.24345397949219,
      "activations/layer11_attention_weight_max": 41.45299530029297,
      "activations/layer11_attention_weight_min": -39.45054626464844,
      "activations/layer12_attention_weight_max": 27.617746353149414,
      "activations/layer12_attention_weight_min": -27.43360137939453,
      "activations/layer13_attention_weight_max": 40.50791931152344,
      "activations/layer13_attention_weight_min": -36.52616500854492,
      "activations/layer14_attention_weight_max": 42.48508071899414,
      "activations/layer14_attention_weight_min": -39.31517791748047,
      "activations/layer15_attention_weight_max": 38.88795852661133,
      "activations/layer15_attention_weight_min": -35.6800422668457,
      "activations/layer16_attention_weight_max": 28.342647552490234,
      "activations/layer16_attention_weight_min": -29.046051025390625,
      "activations/layer17_attention_weight_max": 30.852020263671875,
      "activations/layer17_attention_weight_min": -26.23236083984375,
      "activations/layer18_attention_weight_max": 34.17351531982422,
      "activations/layer18_attention_weight_min": -26.29300880432129,
      "activations/layer19_attention_weight_max": 34.99234390258789,
      "activations/layer19_attention_weight_min": -30.976308822631836,
      "activations/layer1_attention_weight_max": 16.162351608276367,
      "activations/layer1_attention_weight_min": -15.215789794921875,
      "activations/layer20_attention_weight_max": 27.868017196655273,
      "activations/layer20_attention_weight_min": -24.714134216308594,
      "activations/layer21_attention_weight_max": 27.55912971496582,
      "activations/layer21_attention_weight_min": -24.65111541748047,
      "activations/layer22_attention_weight_max": 41.21533966064453,
      "activations/layer22_attention_weight_min": -31.67790985107422,
      "activations/layer23_attention_weight_max": 32.43268966674805,
      "activations/layer23_attention_weight_min": -26.281309127807617,
      "activations/layer2_attention_weight_max": 34.54114532470703,
      "activations/layer2_attention_weight_min": -34.447898864746094,
      "activations/layer3_attention_weight_max": 109.66576385498047,
      "activations/layer3_attention_weight_min": -114.10076141357422,
      "activations/layer4_attention_weight_max": 113.5647201538086,
      "activations/layer4_attention_weight_min": -117.95452117919922,
      "activations/layer5_attention_weight_max": 52.96458053588867,
      "activations/layer5_attention_weight_min": -64.92411804199219,
      "activations/layer6_attention_weight_max": 48.29752731323242,
      "activations/layer6_attention_weight_min": -51.68623352050781,
      "activations/layer7_attention_weight_max": 100.09475708007812,
      "activations/layer7_attention_weight_min": -111.60253143310547,
      "activations/layer8_attention_weight_max": 48.231201171875,
      "activations/layer8_attention_weight_min": -49.87479782104492,
      "activations/layer9_attention_weight_max": 36.413936614990234,
      "activations/layer9_attention_weight_min": -39.7401008605957,
      "epoch": 22.28,
      "learning_rate": 6.400378787878787e-06,
      "loss": 2.6962,
      "step": 383400
    },
    {
      "activations/layer0_attention_weight_max": 17.29598617553711,
      "activations/layer0_attention_weight_min": -14.988720893859863,
      "activations/layer10_attention_weight_max": 37.276588439941406,
      "activations/layer10_attention_weight_min": -34.96165084838867,
      "activations/layer11_attention_weight_max": 33.65669631958008,
      "activations/layer11_attention_weight_min": -34.73163986206055,
      "activations/layer12_attention_weight_max": 28.769535064697266,
      "activations/layer12_attention_weight_min": -25.917028427124023,
      "activations/layer13_attention_weight_max": 38.884620666503906,
      "activations/layer13_attention_weight_min": -35.41742706298828,
      "activations/layer14_attention_weight_max": 43.46247863769531,
      "activations/layer14_attention_weight_min": -38.76405334472656,
      "activations/layer15_attention_weight_max": 38.666053771972656,
      "activations/layer15_attention_weight_min": -35.77347183227539,
      "activations/layer16_attention_weight_max": 31.51128387451172,
      "activations/layer16_attention_weight_min": -28.923219680786133,
      "activations/layer17_attention_weight_max": 30.537750244140625,
      "activations/layer17_attention_weight_min": -26.1029109954834,
      "activations/layer18_attention_weight_max": 33.590274810791016,
      "activations/layer18_attention_weight_min": -30.149559020996094,
      "activations/layer19_attention_weight_max": 37.8092155456543,
      "activations/layer19_attention_weight_min": -32.636714935302734,
      "activations/layer1_attention_weight_max": 16.067058563232422,
      "activations/layer1_attention_weight_min": -15.921089172363281,
      "activations/layer20_attention_weight_max": 30.218242645263672,
      "activations/layer20_attention_weight_min": -24.528118133544922,
      "activations/layer21_attention_weight_max": 31.883132934570312,
      "activations/layer21_attention_weight_min": -24.45261001586914,
      "activations/layer22_attention_weight_max": 40.10542297363281,
      "activations/layer22_attention_weight_min": -31.97197723388672,
      "activations/layer23_attention_weight_max": 31.806076049804688,
      "activations/layer23_attention_weight_min": -24.78339195251465,
      "activations/layer2_attention_weight_max": 34.211021423339844,
      "activations/layer2_attention_weight_min": -33.657432556152344,
      "activations/layer3_attention_weight_max": 107.64823913574219,
      "activations/layer3_attention_weight_min": -110.1927261352539,
      "activations/layer4_attention_weight_max": 112.62562561035156,
      "activations/layer4_attention_weight_min": -116.3395004272461,
      "activations/layer5_attention_weight_max": 51.55167770385742,
      "activations/layer5_attention_weight_min": -64.02022552490234,
      "activations/layer6_attention_weight_max": 47.383018493652344,
      "activations/layer6_attention_weight_min": -48.17997360229492,
      "activations/layer7_attention_weight_max": 99.72891998291016,
      "activations/layer7_attention_weight_min": -102.85269927978516,
      "activations/layer8_attention_weight_max": 42.69904708862305,
      "activations/layer8_attention_weight_min": -44.43059158325195,
      "activations/layer9_attention_weight_max": 33.49266815185547,
      "activations/layer9_attention_weight_min": -35.032752990722656,
      "epoch": 22.28,
      "learning_rate": 6.381439393939393e-06,
      "loss": 2.688,
      "step": 383450
    },
    {
      "activations/layer0_attention_weight_max": 16.625585556030273,
      "activations/layer0_attention_weight_min": -13.104697227478027,
      "activations/layer10_attention_weight_max": 37.02494812011719,
      "activations/layer10_attention_weight_min": -38.85976791381836,
      "activations/layer11_attention_weight_max": 34.74048614501953,
      "activations/layer11_attention_weight_min": -37.79172897338867,
      "activations/layer12_attention_weight_max": 28.991044998168945,
      "activations/layer12_attention_weight_min": -27.50420379638672,
      "activations/layer13_attention_weight_max": 40.28002166748047,
      "activations/layer13_attention_weight_min": -36.0291748046875,
      "activations/layer14_attention_weight_max": 42.26455307006836,
      "activations/layer14_attention_weight_min": -38.31928634643555,
      "activations/layer15_attention_weight_max": 38.73993682861328,
      "activations/layer15_attention_weight_min": -38.45604705810547,
      "activations/layer16_attention_weight_max": 28.015764236450195,
      "activations/layer16_attention_weight_min": -28.528583526611328,
      "activations/layer17_attention_weight_max": 28.3724308013916,
      "activations/layer17_attention_weight_min": -26.544353485107422,
      "activations/layer18_attention_weight_max": 30.52029037475586,
      "activations/layer18_attention_weight_min": -27.505306243896484,
      "activations/layer19_attention_weight_max": 32.38315963745117,
      "activations/layer19_attention_weight_min": -34.00014877319336,
      "activations/layer1_attention_weight_max": 15.973040580749512,
      "activations/layer1_attention_weight_min": -15.18696117401123,
      "activations/layer20_attention_weight_max": 27.433202743530273,
      "activations/layer20_attention_weight_min": -29.450725555419922,
      "activations/layer21_attention_weight_max": 28.848106384277344,
      "activations/layer21_attention_weight_min": -27.606510162353516,
      "activations/layer22_attention_weight_max": 39.27078628540039,
      "activations/layer22_attention_weight_min": -33.43250274658203,
      "activations/layer23_attention_weight_max": 32.1639404296875,
      "activations/layer23_attention_weight_min": -27.701183319091797,
      "activations/layer2_attention_weight_max": 32.90410232543945,
      "activations/layer2_attention_weight_min": -32.905006408691406,
      "activations/layer3_attention_weight_max": 107.42269134521484,
      "activations/layer3_attention_weight_min": -107.67308044433594,
      "activations/layer4_attention_weight_max": 112.62617492675781,
      "activations/layer4_attention_weight_min": -113.19896697998047,
      "activations/layer5_attention_weight_max": 51.520503997802734,
      "activations/layer5_attention_weight_min": -61.136253356933594,
      "activations/layer6_attention_weight_max": 46.50248718261719,
      "activations/layer6_attention_weight_min": -46.6889762878418,
      "activations/layer7_attention_weight_max": 96.48995208740234,
      "activations/layer7_attention_weight_min": -100.54674530029297,
      "activations/layer8_attention_weight_max": 43.22153854370117,
      "activations/layer8_attention_weight_min": -45.67018127441406,
      "activations/layer9_attention_weight_max": 33.101158142089844,
      "activations/layer9_attention_weight_min": -35.89713668823242,
      "epoch": 22.28,
      "learning_rate": 6.362499999999999e-06,
      "loss": 2.7073,
      "step": 383500
    },
    {
      "activations/layer0_attention_weight_max": 16.992572784423828,
      "activations/layer0_attention_weight_min": -15.65584659576416,
      "activations/layer10_attention_weight_max": 38.40837860107422,
      "activations/layer10_attention_weight_min": -38.0223274230957,
      "activations/layer11_attention_weight_max": 38.27046203613281,
      "activations/layer11_attention_weight_min": -38.054039001464844,
      "activations/layer12_attention_weight_max": 28.519031524658203,
      "activations/layer12_attention_weight_min": -26.684505462646484,
      "activations/layer13_attention_weight_max": 42.55769729614258,
      "activations/layer13_attention_weight_min": -36.68023681640625,
      "activations/layer14_attention_weight_max": 43.465309143066406,
      "activations/layer14_attention_weight_min": -41.15110397338867,
      "activations/layer15_attention_weight_max": 41.645103454589844,
      "activations/layer15_attention_weight_min": -38.01325225830078,
      "activations/layer16_attention_weight_max": 28.264280319213867,
      "activations/layer16_attention_weight_min": -27.817277908325195,
      "activations/layer17_attention_weight_max": 30.653024673461914,
      "activations/layer17_attention_weight_min": -25.519208908081055,
      "activations/layer18_attention_weight_max": 31.51156234741211,
      "activations/layer18_attention_weight_min": -27.107107162475586,
      "activations/layer19_attention_weight_max": 32.64249038696289,
      "activations/layer19_attention_weight_min": -31.88844108581543,
      "activations/layer1_attention_weight_max": 16.259016036987305,
      "activations/layer1_attention_weight_min": -15.612614631652832,
      "activations/layer20_attention_weight_max": 27.339065551757812,
      "activations/layer20_attention_weight_min": -24.31696128845215,
      "activations/layer21_attention_weight_max": 28.340391159057617,
      "activations/layer21_attention_weight_min": -23.665855407714844,
      "activations/layer22_attention_weight_max": 39.384498596191406,
      "activations/layer22_attention_weight_min": -32.04873275756836,
      "activations/layer23_attention_weight_max": 30.77753257751465,
      "activations/layer23_attention_weight_min": -25.777685165405273,
      "activations/layer2_attention_weight_max": 35.9261474609375,
      "activations/layer2_attention_weight_min": -34.36570358276367,
      "activations/layer3_attention_weight_max": 111.51866149902344,
      "activations/layer3_attention_weight_min": -110.79316711425781,
      "activations/layer4_attention_weight_max": 111.321044921875,
      "activations/layer4_attention_weight_min": -113.6016845703125,
      "activations/layer5_attention_weight_max": 48.637229919433594,
      "activations/layer5_attention_weight_min": -60.38533020019531,
      "activations/layer6_attention_weight_max": 46.67732620239258,
      "activations/layer6_attention_weight_min": -49.829994201660156,
      "activations/layer7_attention_weight_max": 99.9227066040039,
      "activations/layer7_attention_weight_min": -101.74810028076172,
      "activations/layer8_attention_weight_max": 45.66900634765625,
      "activations/layer8_attention_weight_min": -48.27263259887695,
      "activations/layer9_attention_weight_max": 33.52967071533203,
      "activations/layer9_attention_weight_min": -37.036197662353516,
      "epoch": 22.29,
      "learning_rate": 6.343939393939393e-06,
      "loss": 2.7078,
      "step": 383550
    },
    {
      "activations/layer0_attention_weight_max": 16.74189567565918,
      "activations/layer0_attention_weight_min": -13.69467830657959,
      "activations/layer10_attention_weight_max": 34.0390739440918,
      "activations/layer10_attention_weight_min": -37.300472259521484,
      "activations/layer11_attention_weight_max": 35.080413818359375,
      "activations/layer11_attention_weight_min": -37.09862518310547,
      "activations/layer12_attention_weight_max": 26.231149673461914,
      "activations/layer12_attention_weight_min": -27.074777603149414,
      "activations/layer13_attention_weight_max": 37.73800277709961,
      "activations/layer13_attention_weight_min": -34.82285690307617,
      "activations/layer14_attention_weight_max": 43.71068572998047,
      "activations/layer14_attention_weight_min": -37.649967193603516,
      "activations/layer15_attention_weight_max": 38.40467834472656,
      "activations/layer15_attention_weight_min": -36.40911102294922,
      "activations/layer16_attention_weight_max": 32.21120071411133,
      "activations/layer16_attention_weight_min": -29.638628005981445,
      "activations/layer17_attention_weight_max": 29.428955078125,
      "activations/layer17_attention_weight_min": -29.17061996459961,
      "activations/layer18_attention_weight_max": 33.532745361328125,
      "activations/layer18_attention_weight_min": -29.084688186645508,
      "activations/layer19_attention_weight_max": 33.89756774902344,
      "activations/layer19_attention_weight_min": -32.083255767822266,
      "activations/layer1_attention_weight_max": 15.313193321228027,
      "activations/layer1_attention_weight_min": -15.687007904052734,
      "activations/layer20_attention_weight_max": 32.54092788696289,
      "activations/layer20_attention_weight_min": -25.36410140991211,
      "activations/layer21_attention_weight_max": 30.54010009765625,
      "activations/layer21_attention_weight_min": -25.38356590270996,
      "activations/layer22_attention_weight_max": 40.97227478027344,
      "activations/layer22_attention_weight_min": -33.493003845214844,
      "activations/layer23_attention_weight_max": 33.95502471923828,
      "activations/layer23_attention_weight_min": -26.990331649780273,
      "activations/layer2_attention_weight_max": 34.45973205566406,
      "activations/layer2_attention_weight_min": -32.66864776611328,
      "activations/layer3_attention_weight_max": 105.85920715332031,
      "activations/layer3_attention_weight_min": -105.76309967041016,
      "activations/layer4_attention_weight_max": 113.28315734863281,
      "activations/layer4_attention_weight_min": -117.32205963134766,
      "activations/layer5_attention_weight_max": 51.97066879272461,
      "activations/layer5_attention_weight_min": -61.52105712890625,
      "activations/layer6_attention_weight_max": 45.87504577636719,
      "activations/layer6_attention_weight_min": -50.736610412597656,
      "activations/layer7_attention_weight_max": 98.96051025390625,
      "activations/layer7_attention_weight_min": -106.92617797851562,
      "activations/layer8_attention_weight_max": 42.71014404296875,
      "activations/layer8_attention_weight_min": -47.093692779541016,
      "activations/layer9_attention_weight_max": 32.33551025390625,
      "activations/layer9_attention_weight_min": -36.5567741394043,
      "epoch": 22.29,
      "learning_rate": 6.325e-06,
      "loss": 2.6835,
      "step": 383600
    },
    {
      "activations/layer0_attention_weight_max": 16.762407302856445,
      "activations/layer0_attention_weight_min": -14.544610977172852,
      "activations/layer10_attention_weight_max": 34.05399703979492,
      "activations/layer10_attention_weight_min": -35.18927001953125,
      "activations/layer11_attention_weight_max": 34.05568313598633,
      "activations/layer11_attention_weight_min": -35.57499694824219,
      "activations/layer12_attention_weight_max": 27.225309371948242,
      "activations/layer12_attention_weight_min": -26.543716430664062,
      "activations/layer13_attention_weight_max": 38.512569427490234,
      "activations/layer13_attention_weight_min": -35.81576156616211,
      "activations/layer14_attention_weight_max": 43.61699676513672,
      "activations/layer14_attention_weight_min": -38.57337188720703,
      "activations/layer15_attention_weight_max": 37.158329010009766,
      "activations/layer15_attention_weight_min": -35.837623596191406,
      "activations/layer16_attention_weight_max": 29.2064266204834,
      "activations/layer16_attention_weight_min": -28.58755111694336,
      "activations/layer17_attention_weight_max": 27.868741989135742,
      "activations/layer17_attention_weight_min": -26.916616439819336,
      "activations/layer18_attention_weight_max": 31.055438995361328,
      "activations/layer18_attention_weight_min": -25.92050552368164,
      "activations/layer19_attention_weight_max": 29.282535552978516,
      "activations/layer19_attention_weight_min": -32.875892639160156,
      "activations/layer1_attention_weight_max": 16.28618621826172,
      "activations/layer1_attention_weight_min": -15.652361869812012,
      "activations/layer20_attention_weight_max": 26.08382225036621,
      "activations/layer20_attention_weight_min": -25.052860260009766,
      "activations/layer21_attention_weight_max": 26.240737915039062,
      "activations/layer21_attention_weight_min": -24.299484252929688,
      "activations/layer22_attention_weight_max": 36.13358688354492,
      "activations/layer22_attention_weight_min": -30.704452514648438,
      "activations/layer23_attention_weight_max": 28.76154327392578,
      "activations/layer23_attention_weight_min": -25.31593894958496,
      "activations/layer2_attention_weight_max": 32.93677520751953,
      "activations/layer2_attention_weight_min": -32.33396911621094,
      "activations/layer3_attention_weight_max": 103.02019500732422,
      "activations/layer3_attention_weight_min": -104.5691909790039,
      "activations/layer4_attention_weight_max": 105.23326873779297,
      "activations/layer4_attention_weight_min": -111.98451232910156,
      "activations/layer5_attention_weight_max": 49.048423767089844,
      "activations/layer5_attention_weight_min": -61.286773681640625,
      "activations/layer6_attention_weight_max": 42.96686553955078,
      "activations/layer6_attention_weight_min": -48.463687896728516,
      "activations/layer7_attention_weight_max": 93.93008422851562,
      "activations/layer7_attention_weight_min": -104.60359954833984,
      "activations/layer8_attention_weight_max": 41.49189376831055,
      "activations/layer8_attention_weight_min": -43.31813430786133,
      "activations/layer9_attention_weight_max": 30.99725914001465,
      "activations/layer9_attention_weight_min": -34.340576171875,
      "epoch": 22.29,
      "learning_rate": 6.306060606060605e-06,
      "loss": 2.7097,
      "step": 383650
    },
    {
      "activations/layer0_attention_weight_max": 17.17483901977539,
      "activations/layer0_attention_weight_min": -15.422431945800781,
      "activations/layer10_attention_weight_max": 40.75914001464844,
      "activations/layer10_attention_weight_min": -37.9650764465332,
      "activations/layer11_attention_weight_max": 40.215415954589844,
      "activations/layer11_attention_weight_min": -39.160743713378906,
      "activations/layer12_attention_weight_max": 30.588865280151367,
      "activations/layer12_attention_weight_min": -27.52544593811035,
      "activations/layer13_attention_weight_max": 43.69060134887695,
      "activations/layer13_attention_weight_min": -37.11332321166992,
      "activations/layer14_attention_weight_max": 43.740543365478516,
      "activations/layer14_attention_weight_min": -40.44636154174805,
      "activations/layer15_attention_weight_max": 40.76089096069336,
      "activations/layer15_attention_weight_min": -37.83757400512695,
      "activations/layer16_attention_weight_max": 30.251712799072266,
      "activations/layer16_attention_weight_min": -29.15675163269043,
      "activations/layer17_attention_weight_max": 31.1094913482666,
      "activations/layer17_attention_weight_min": -28.15222930908203,
      "activations/layer18_attention_weight_max": 32.42978286743164,
      "activations/layer18_attention_weight_min": -27.685380935668945,
      "activations/layer19_attention_weight_max": 33.24242401123047,
      "activations/layer19_attention_weight_min": -32.44794845581055,
      "activations/layer1_attention_weight_max": 16.038293838500977,
      "activations/layer1_attention_weight_min": -15.25118637084961,
      "activations/layer20_attention_weight_max": 28.430381774902344,
      "activations/layer20_attention_weight_min": -26.614883422851562,
      "activations/layer21_attention_weight_max": 25.259746551513672,
      "activations/layer21_attention_weight_min": -25.30914306640625,
      "activations/layer22_attention_weight_max": 38.8145637512207,
      "activations/layer22_attention_weight_min": -31.227569580078125,
      "activations/layer23_attention_weight_max": 27.56073760986328,
      "activations/layer23_attention_weight_min": -26.692115783691406,
      "activations/layer2_attention_weight_max": 35.262550354003906,
      "activations/layer2_attention_weight_min": -32.764190673828125,
      "activations/layer3_attention_weight_max": 107.5645980834961,
      "activations/layer3_attention_weight_min": -105.7923355102539,
      "activations/layer4_attention_weight_max": 106.72332763671875,
      "activations/layer4_attention_weight_min": -110.64691162109375,
      "activations/layer5_attention_weight_max": 50.360679626464844,
      "activations/layer5_attention_weight_min": -63.080230712890625,
      "activations/layer6_attention_weight_max": 46.859779357910156,
      "activations/layer6_attention_weight_min": -50.11720275878906,
      "activations/layer7_attention_weight_max": 98.44985961914062,
      "activations/layer7_attention_weight_min": -103.03372955322266,
      "activations/layer8_attention_weight_max": 44.773799896240234,
      "activations/layer8_attention_weight_min": -48.05226516723633,
      "activations/layer9_attention_weight_max": 35.871192932128906,
      "activations/layer9_attention_weight_min": -37.168067932128906,
      "epoch": 22.3,
      "learning_rate": 6.287121212121212e-06,
      "loss": 2.6929,
      "step": 383700
    },
    {
      "activations/layer0_attention_weight_max": 16.373462677001953,
      "activations/layer0_attention_weight_min": -14.812341690063477,
      "activations/layer10_attention_weight_max": 41.786415100097656,
      "activations/layer10_attention_weight_min": -42.3326530456543,
      "activations/layer11_attention_weight_max": 38.92599868774414,
      "activations/layer11_attention_weight_min": -39.83168411254883,
      "activations/layer12_attention_weight_max": 30.62618637084961,
      "activations/layer12_attention_weight_min": -28.867877960205078,
      "activations/layer13_attention_weight_max": 42.035057067871094,
      "activations/layer13_attention_weight_min": -34.876953125,
      "activations/layer14_attention_weight_max": 44.06591033935547,
      "activations/layer14_attention_weight_min": -40.940975189208984,
      "activations/layer15_attention_weight_max": 39.64509201049805,
      "activations/layer15_attention_weight_min": -37.19879913330078,
      "activations/layer16_attention_weight_max": 28.65034294128418,
      "activations/layer16_attention_weight_min": -30.423131942749023,
      "activations/layer17_attention_weight_max": 29.448591232299805,
      "activations/layer17_attention_weight_min": -25.866668701171875,
      "activations/layer18_attention_weight_max": 33.44767379760742,
      "activations/layer18_attention_weight_min": -26.134130477905273,
      "activations/layer19_attention_weight_max": 34.757545471191406,
      "activations/layer19_attention_weight_min": -30.513763427734375,
      "activations/layer1_attention_weight_max": 16.502288818359375,
      "activations/layer1_attention_weight_min": -15.89479923248291,
      "activations/layer20_attention_weight_max": 31.943090438842773,
      "activations/layer20_attention_weight_min": -23.985248565673828,
      "activations/layer21_attention_weight_max": 32.01202392578125,
      "activations/layer21_attention_weight_min": -25.31890296936035,
      "activations/layer22_attention_weight_max": 40.86112594604492,
      "activations/layer22_attention_weight_min": -31.74414825439453,
      "activations/layer23_attention_weight_max": 33.797630310058594,
      "activations/layer23_attention_weight_min": -25.612625122070312,
      "activations/layer2_attention_weight_max": 35.38551330566406,
      "activations/layer2_attention_weight_min": -33.83535385131836,
      "activations/layer3_attention_weight_max": 111.40985870361328,
      "activations/layer3_attention_weight_min": -107.55521392822266,
      "activations/layer4_attention_weight_max": 107.57035064697266,
      "activations/layer4_attention_weight_min": -114.6707992553711,
      "activations/layer5_attention_weight_max": 50.1560173034668,
      "activations/layer5_attention_weight_min": -61.79447937011719,
      "activations/layer6_attention_weight_max": 46.178672790527344,
      "activations/layer6_attention_weight_min": -48.97087478637695,
      "activations/layer7_attention_weight_max": 102.3722915649414,
      "activations/layer7_attention_weight_min": -102.92139434814453,
      "activations/layer8_attention_weight_max": 48.26374053955078,
      "activations/layer8_attention_weight_min": -48.51620864868164,
      "activations/layer9_attention_weight_max": 35.24153518676758,
      "activations/layer9_attention_weight_min": -38.90373611450195,
      "epoch": 22.3,
      "learning_rate": 6.268181818181818e-06,
      "loss": 2.7014,
      "step": 383750
    },
    {
      "activations/layer0_attention_weight_max": 16.71367073059082,
      "activations/layer0_attention_weight_min": -13.91980266571045,
      "activations/layer10_attention_weight_max": 35.8512077331543,
      "activations/layer10_attention_weight_min": -36.004234313964844,
      "activations/layer11_attention_weight_max": 36.349571228027344,
      "activations/layer11_attention_weight_min": -37.60176467895508,
      "activations/layer12_attention_weight_max": 29.720884323120117,
      "activations/layer12_attention_weight_min": -29.455564498901367,
      "activations/layer13_attention_weight_max": 44.720237731933594,
      "activations/layer13_attention_weight_min": -40.14813995361328,
      "activations/layer14_attention_weight_max": 45.49210739135742,
      "activations/layer14_attention_weight_min": -41.54243087768555,
      "activations/layer15_attention_weight_max": 43.11075210571289,
      "activations/layer15_attention_weight_min": -42.34992980957031,
      "activations/layer16_attention_weight_max": 31.983915328979492,
      "activations/layer16_attention_weight_min": -29.774110794067383,
      "activations/layer17_attention_weight_max": 31.549957275390625,
      "activations/layer17_attention_weight_min": -28.04584503173828,
      "activations/layer18_attention_weight_max": 38.305946350097656,
      "activations/layer18_attention_weight_min": -27.931442260742188,
      "activations/layer19_attention_weight_max": 36.9102897644043,
      "activations/layer19_attention_weight_min": -34.059471130371094,
      "activations/layer1_attention_weight_max": 15.615254402160645,
      "activations/layer1_attention_weight_min": -15.236628532409668,
      "activations/layer20_attention_weight_max": 32.491844177246094,
      "activations/layer20_attention_weight_min": -26.736522674560547,
      "activations/layer21_attention_weight_max": 29.725051879882812,
      "activations/layer21_attention_weight_min": -26.779277801513672,
      "activations/layer22_attention_weight_max": 43.776336669921875,
      "activations/layer22_attention_weight_min": -34.134708404541016,
      "activations/layer23_attention_weight_max": 34.47114562988281,
      "activations/layer23_attention_weight_min": -28.14630889892578,
      "activations/layer2_attention_weight_max": 34.294677734375,
      "activations/layer2_attention_weight_min": -33.19020462036133,
      "activations/layer3_attention_weight_max": 106.13168334960938,
      "activations/layer3_attention_weight_min": -107.53817749023438,
      "activations/layer4_attention_weight_max": 108.83281707763672,
      "activations/layer4_attention_weight_min": -113.31461334228516,
      "activations/layer5_attention_weight_max": 53.10350036621094,
      "activations/layer5_attention_weight_min": -60.901763916015625,
      "activations/layer6_attention_weight_max": 46.68437957763672,
      "activations/layer6_attention_weight_min": -50.90849685668945,
      "activations/layer7_attention_weight_max": 98.74933624267578,
      "activations/layer7_attention_weight_min": -107.87736511230469,
      "activations/layer8_attention_weight_max": 44.20350646972656,
      "activations/layer8_attention_weight_min": -45.77137756347656,
      "activations/layer9_attention_weight_max": 33.50360107421875,
      "activations/layer9_attention_weight_min": -36.643798828125,
      "epoch": 22.3,
      "learning_rate": 6.249242424242424e-06,
      "loss": 2.6933,
      "step": 383800
    },
    {
      "activations/layer0_attention_weight_max": 18.41191291809082,
      "activations/layer0_attention_weight_min": -14.551039695739746,
      "activations/layer10_attention_weight_max": 37.927223205566406,
      "activations/layer10_attention_weight_min": -35.17938232421875,
      "activations/layer11_attention_weight_max": 34.555999755859375,
      "activations/layer11_attention_weight_min": -36.02685546875,
      "activations/layer12_attention_weight_max": 26.612586975097656,
      "activations/layer12_attention_weight_min": -26.592872619628906,
      "activations/layer13_attention_weight_max": 39.47451400756836,
      "activations/layer13_attention_weight_min": -35.37294006347656,
      "activations/layer14_attention_weight_max": 43.358036041259766,
      "activations/layer14_attention_weight_min": -38.527923583984375,
      "activations/layer15_attention_weight_max": 40.411903381347656,
      "activations/layer15_attention_weight_min": -37.176918029785156,
      "activations/layer16_attention_weight_max": 30.08687973022461,
      "activations/layer16_attention_weight_min": -29.43630027770996,
      "activations/layer17_attention_weight_max": 32.993927001953125,
      "activations/layer17_attention_weight_min": -28.84718132019043,
      "activations/layer18_attention_weight_max": 34.957637786865234,
      "activations/layer18_attention_weight_min": -26.667451858520508,
      "activations/layer19_attention_weight_max": 38.405189514160156,
      "activations/layer19_attention_weight_min": -36.03202438354492,
      "activations/layer1_attention_weight_max": 15.16860580444336,
      "activations/layer1_attention_weight_min": -14.6674222946167,
      "activations/layer20_attention_weight_max": 33.18122100830078,
      "activations/layer20_attention_weight_min": -25.934919357299805,
      "activations/layer21_attention_weight_max": 32.76697540283203,
      "activations/layer21_attention_weight_min": -25.290874481201172,
      "activations/layer22_attention_weight_max": 46.0926399230957,
      "activations/layer22_attention_weight_min": -33.49519348144531,
      "activations/layer23_attention_weight_max": 37.5076904296875,
      "activations/layer23_attention_weight_min": -29.073856353759766,
      "activations/layer2_attention_weight_max": 36.649200439453125,
      "activations/layer2_attention_weight_min": -33.388282775878906,
      "activations/layer3_attention_weight_max": 108.50676727294922,
      "activations/layer3_attention_weight_min": -111.08554077148438,
      "activations/layer4_attention_weight_max": 114.01827239990234,
      "activations/layer4_attention_weight_min": -115.40813446044922,
      "activations/layer5_attention_weight_max": 53.24119186401367,
      "activations/layer5_attention_weight_min": -63.4878044128418,
      "activations/layer6_attention_weight_max": 46.97317123413086,
      "activations/layer6_attention_weight_min": -51.23554992675781,
      "activations/layer7_attention_weight_max": 99.79138946533203,
      "activations/layer7_attention_weight_min": -107.0811538696289,
      "activations/layer8_attention_weight_max": 45.526432037353516,
      "activations/layer8_attention_weight_min": -46.00062942504883,
      "activations/layer9_attention_weight_max": 33.3480339050293,
      "activations/layer9_attention_weight_min": -34.926326751708984,
      "epoch": 22.3,
      "learning_rate": 6.23030303030303e-06,
      "loss": 2.6864,
      "step": 383850
    },
    {
      "activations/layer0_attention_weight_max": 17.1584415435791,
      "activations/layer0_attention_weight_min": -14.692850112915039,
      "activations/layer10_attention_weight_max": 40.22038650512695,
      "activations/layer10_attention_weight_min": -37.94865036010742,
      "activations/layer11_attention_weight_max": 39.47876739501953,
      "activations/layer11_attention_weight_min": -38.148277282714844,
      "activations/layer12_attention_weight_max": 26.90863800048828,
      "activations/layer12_attention_weight_min": -28.624765396118164,
      "activations/layer13_attention_weight_max": 40.59617614746094,
      "activations/layer13_attention_weight_min": -39.6471061706543,
      "activations/layer14_attention_weight_max": 46.11997604370117,
      "activations/layer14_attention_weight_min": -41.715576171875,
      "activations/layer15_attention_weight_max": 42.1448974609375,
      "activations/layer15_attention_weight_min": -39.979679107666016,
      "activations/layer16_attention_weight_max": 28.922672271728516,
      "activations/layer16_attention_weight_min": -27.97228240966797,
      "activations/layer17_attention_weight_max": 34.81498718261719,
      "activations/layer17_attention_weight_min": -26.362146377563477,
      "activations/layer18_attention_weight_max": 34.08671188354492,
      "activations/layer18_attention_weight_min": -26.455949783325195,
      "activations/layer19_attention_weight_max": 36.074607849121094,
      "activations/layer19_attention_weight_min": -31.596363067626953,
      "activations/layer1_attention_weight_max": 15.677915573120117,
      "activations/layer1_attention_weight_min": -15.370534896850586,
      "activations/layer20_attention_weight_max": 28.329992294311523,
      "activations/layer20_attention_weight_min": -23.88838768005371,
      "activations/layer21_attention_weight_max": 28.050748825073242,
      "activations/layer21_attention_weight_min": -23.447429656982422,
      "activations/layer22_attention_weight_max": 39.274532318115234,
      "activations/layer22_attention_weight_min": -31.628780364990234,
      "activations/layer23_attention_weight_max": 33.254478454589844,
      "activations/layer23_attention_weight_min": -25.81532096862793,
      "activations/layer2_attention_weight_max": 34.293663024902344,
      "activations/layer2_attention_weight_min": -33.239898681640625,
      "activations/layer3_attention_weight_max": 108.4126968383789,
      "activations/layer3_attention_weight_min": -108.10720825195312,
      "activations/layer4_attention_weight_max": 116.46602630615234,
      "activations/layer4_attention_weight_min": -116.5099105834961,
      "activations/layer5_attention_weight_max": 53.81321334838867,
      "activations/layer5_attention_weight_min": -62.57210922241211,
      "activations/layer6_attention_weight_max": 50.91065979003906,
      "activations/layer6_attention_weight_min": -53.53306579589844,
      "activations/layer7_attention_weight_max": 107.51419067382812,
      "activations/layer7_attention_weight_min": -106.8600082397461,
      "activations/layer8_attention_weight_max": 47.322235107421875,
      "activations/layer8_attention_weight_min": -48.01103591918945,
      "activations/layer9_attention_weight_max": 36.138328552246094,
      "activations/layer9_attention_weight_min": -37.919189453125,
      "epoch": 22.31,
      "learning_rate": 6.211363636363636e-06,
      "loss": 2.7099,
      "step": 383900
    },
    {
      "activations/layer0_attention_weight_max": 18.020193099975586,
      "activations/layer0_attention_weight_min": -14.4633150100708,
      "activations/layer10_attention_weight_max": 36.248939514160156,
      "activations/layer10_attention_weight_min": -35.936256408691406,
      "activations/layer11_attention_weight_max": 35.08317947387695,
      "activations/layer11_attention_weight_min": -36.232666015625,
      "activations/layer12_attention_weight_max": 27.244709014892578,
      "activations/layer12_attention_weight_min": -28.905078887939453,
      "activations/layer13_attention_weight_max": 39.75063705444336,
      "activations/layer13_attention_weight_min": -37.6161003112793,
      "activations/layer14_attention_weight_max": 44.446807861328125,
      "activations/layer14_attention_weight_min": -39.457420349121094,
      "activations/layer15_attention_weight_max": 40.280330657958984,
      "activations/layer15_attention_weight_min": -39.912696838378906,
      "activations/layer16_attention_weight_max": 30.912626266479492,
      "activations/layer16_attention_weight_min": -29.30133819580078,
      "activations/layer17_attention_weight_max": 32.600669860839844,
      "activations/layer17_attention_weight_min": -30.00130271911621,
      "activations/layer18_attention_weight_max": 33.26016616821289,
      "activations/layer18_attention_weight_min": -29.41392707824707,
      "activations/layer19_attention_weight_max": 40.01606750488281,
      "activations/layer19_attention_weight_min": -34.350154876708984,
      "activations/layer1_attention_weight_max": 16.3005428314209,
      "activations/layer1_attention_weight_min": -16.35284996032715,
      "activations/layer20_attention_weight_max": 29.623302459716797,
      "activations/layer20_attention_weight_min": -25.272991180419922,
      "activations/layer21_attention_weight_max": 28.403013229370117,
      "activations/layer21_attention_weight_min": -27.54129409790039,
      "activations/layer22_attention_weight_max": 46.49579620361328,
      "activations/layer22_attention_weight_min": -37.278743743896484,
      "activations/layer23_attention_weight_max": 32.59661865234375,
      "activations/layer23_attention_weight_min": -30.960620880126953,
      "activations/layer2_attention_weight_max": 34.67662048339844,
      "activations/layer2_attention_weight_min": -32.502098083496094,
      "activations/layer3_attention_weight_max": 104.48971557617188,
      "activations/layer3_attention_weight_min": -106.61432647705078,
      "activations/layer4_attention_weight_max": 108.535400390625,
      "activations/layer4_attention_weight_min": -113.0544662475586,
      "activations/layer5_attention_weight_max": 53.20204162597656,
      "activations/layer5_attention_weight_min": -60.329444885253906,
      "activations/layer6_attention_weight_max": 45.84922409057617,
      "activations/layer6_attention_weight_min": -47.51918411254883,
      "activations/layer7_attention_weight_max": 104.42777252197266,
      "activations/layer7_attention_weight_min": -99.1236572265625,
      "activations/layer8_attention_weight_max": 43.68400955200195,
      "activations/layer8_attention_weight_min": -45.14326858520508,
      "activations/layer9_attention_weight_max": 33.9469108581543,
      "activations/layer9_attention_weight_min": -35.97379684448242,
      "epoch": 22.31,
      "learning_rate": 6.192424242424241e-06,
      "loss": 2.6977,
      "step": 383950
    },
    {
      "activations/layer0_attention_weight_max": 16.091968536376953,
      "activations/layer0_attention_weight_min": -15.504654884338379,
      "activations/layer10_attention_weight_max": 37.50197219848633,
      "activations/layer10_attention_weight_min": -35.769622802734375,
      "activations/layer11_attention_weight_max": 36.66242599487305,
      "activations/layer11_attention_weight_min": -36.62144088745117,
      "activations/layer12_attention_weight_max": 26.109495162963867,
      "activations/layer12_attention_weight_min": -26.65776252746582,
      "activations/layer13_attention_weight_max": 37.50212097167969,
      "activations/layer13_attention_weight_min": -34.14590072631836,
      "activations/layer14_attention_weight_max": 42.95586013793945,
      "activations/layer14_attention_weight_min": -37.1551628112793,
      "activations/layer15_attention_weight_max": 38.10834503173828,
      "activations/layer15_attention_weight_min": -34.72661590576172,
      "activations/layer16_attention_weight_max": 28.66914939880371,
      "activations/layer16_attention_weight_min": -29.789506912231445,
      "activations/layer17_attention_weight_max": 27.968576431274414,
      "activations/layer17_attention_weight_min": -26.552677154541016,
      "activations/layer18_attention_weight_max": 31.019014358520508,
      "activations/layer18_attention_weight_min": -26.983232498168945,
      "activations/layer19_attention_weight_max": 32.02647018432617,
      "activations/layer19_attention_weight_min": -33.314212799072266,
      "activations/layer1_attention_weight_max": 16.332351684570312,
      "activations/layer1_attention_weight_min": -15.656885147094727,
      "activations/layer20_attention_weight_max": 26.29457664489746,
      "activations/layer20_attention_weight_min": -24.15046501159668,
      "activations/layer21_attention_weight_max": 26.531963348388672,
      "activations/layer21_attention_weight_min": -24.385644912719727,
      "activations/layer22_attention_weight_max": 41.48382568359375,
      "activations/layer22_attention_weight_min": -30.356470108032227,
      "activations/layer23_attention_weight_max": 31.603315353393555,
      "activations/layer23_attention_weight_min": -24.706771850585938,
      "activations/layer2_attention_weight_max": 37.021263122558594,
      "activations/layer2_attention_weight_min": -34.98415756225586,
      "activations/layer3_attention_weight_max": 117.46501159667969,
      "activations/layer3_attention_weight_min": -112.42253112792969,
      "activations/layer4_attention_weight_max": 118.9964370727539,
      "activations/layer4_attention_weight_min": -116.29972076416016,
      "activations/layer5_attention_weight_max": 53.5324821472168,
      "activations/layer5_attention_weight_min": -65.8331069946289,
      "activations/layer6_attention_weight_max": 46.3269157409668,
      "activations/layer6_attention_weight_min": -49.19867706298828,
      "activations/layer7_attention_weight_max": 103.1248550415039,
      "activations/layer7_attention_weight_min": -107.88536071777344,
      "activations/layer8_attention_weight_max": 46.05534744262695,
      "activations/layer8_attention_weight_min": -47.214378356933594,
      "activations/layer9_attention_weight_max": 34.592525482177734,
      "activations/layer9_attention_weight_min": -37.04743194580078,
      "epoch": 22.31,
      "learning_rate": 6.173484848484848e-06,
      "loss": 2.6901,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4768,
      "eval_samples_per_second": 506.558,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4768,
      "eval_openwebtext_samples_per_second": 506.558,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.9377,
      "eval_wikitext_samples_per_second": 235.334,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_lambada_loss": 2.30078125,
      "eval_lambada_ppl": 9.9819778295293,
      "eval_lambada_runtime": 9.5134,
      "eval_lambada_samples_per_second": 511.805,
      "step": 384000
    },
    {
      "activations/layer0_attention_weight_max": 16.688798904418945,
      "activations/layer0_attention_weight_min": -13.13118839263916,
      "activations/layer10_attention_weight_max": 35.34856414794922,
      "activations/layer10_attention_weight_min": -36.24231719970703,
      "activations/layer11_attention_weight_max": 32.927223205566406,
      "activations/layer11_attention_weight_min": -34.46574401855469,
      "activations/layer12_attention_weight_max": 25.345516204833984,
      "activations/layer12_attention_weight_min": -26.793514251708984,
      "activations/layer13_attention_weight_max": 36.36672592163086,
      "activations/layer13_attention_weight_min": -35.948848724365234,
      "activations/layer14_attention_weight_max": 38.9788932800293,
      "activations/layer14_attention_weight_min": -38.247249603271484,
      "activations/layer15_attention_weight_max": 37.01382064819336,
      "activations/layer15_attention_weight_min": -35.37397384643555,
      "activations/layer16_attention_weight_max": 29.041616439819336,
      "activations/layer16_attention_weight_min": -27.60758399963379,
      "activations/layer17_attention_weight_max": 29.643409729003906,
      "activations/layer17_attention_weight_min": -27.1965274810791,
      "activations/layer18_attention_weight_max": 35.901275634765625,
      "activations/layer18_attention_weight_min": -28.157812118530273,
      "activations/layer19_attention_weight_max": 35.694610595703125,
      "activations/layer19_attention_weight_min": -31.678455352783203,
      "activations/layer1_attention_weight_max": 15.432333946228027,
      "activations/layer1_attention_weight_min": -14.503026962280273,
      "activations/layer20_attention_weight_max": 30.559185028076172,
      "activations/layer20_attention_weight_min": -26.338695526123047,
      "activations/layer21_attention_weight_max": 29.6463680267334,
      "activations/layer21_attention_weight_min": -25.896427154541016,
      "activations/layer22_attention_weight_max": 43.17146301269531,
      "activations/layer22_attention_weight_min": -32.44548797607422,
      "activations/layer23_attention_weight_max": 33.32274627685547,
      "activations/layer23_attention_weight_min": -27.749526977539062,
      "activations/layer2_attention_weight_max": 34.49219512939453,
      "activations/layer2_attention_weight_min": -34.24637985229492,
      "activations/layer3_attention_weight_max": 105.37969207763672,
      "activations/layer3_attention_weight_min": -109.63359069824219,
      "activations/layer4_attention_weight_max": 109.20143127441406,
      "activations/layer4_attention_weight_min": -115.04869842529297,
      "activations/layer5_attention_weight_max": 49.17323303222656,
      "activations/layer5_attention_weight_min": -61.933074951171875,
      "activations/layer6_attention_weight_max": 44.695064544677734,
      "activations/layer6_attention_weight_min": -51.431678771972656,
      "activations/layer7_attention_weight_max": 95.19513702392578,
      "activations/layer7_attention_weight_min": -100.06534576416016,
      "activations/layer8_attention_weight_max": 42.64621353149414,
      "activations/layer8_attention_weight_min": -46.68617630004883,
      "activations/layer9_attention_weight_max": 33.01173400878906,
      "activations/layer9_attention_weight_min": -36.623538970947266,
      "epoch": 22.32,
      "learning_rate": 6.154545454545454e-06,
      "loss": 2.6983,
      "step": 384050
    },
    {
      "activations/layer0_attention_weight_max": 16.359546661376953,
      "activations/layer0_attention_weight_min": -12.50778579711914,
      "activations/layer10_attention_weight_max": 35.973182678222656,
      "activations/layer10_attention_weight_min": -40.36260986328125,
      "activations/layer11_attention_weight_max": 37.309547424316406,
      "activations/layer11_attention_weight_min": -36.59192657470703,
      "activations/layer12_attention_weight_max": 30.611980438232422,
      "activations/layer12_attention_weight_min": -28.909025192260742,
      "activations/layer13_attention_weight_max": 42.69717025756836,
      "activations/layer13_attention_weight_min": -37.915069580078125,
      "activations/layer14_attention_weight_max": 48.013858795166016,
      "activations/layer14_attention_weight_min": -40.89632797241211,
      "activations/layer15_attention_weight_max": 45.42037582397461,
      "activations/layer15_attention_weight_min": -41.23430633544922,
      "activations/layer16_attention_weight_max": 34.81570053100586,
      "activations/layer16_attention_weight_min": -28.037845611572266,
      "activations/layer17_attention_weight_max": 33.06096267700195,
      "activations/layer17_attention_weight_min": -27.027557373046875,
      "activations/layer18_attention_weight_max": 37.897003173828125,
      "activations/layer18_attention_weight_min": -30.31155014038086,
      "activations/layer19_attention_weight_max": 38.911067962646484,
      "activations/layer19_attention_weight_min": -33.672908782958984,
      "activations/layer1_attention_weight_max": 16.66829490661621,
      "activations/layer1_attention_weight_min": -15.782689094543457,
      "activations/layer20_attention_weight_max": 32.45529556274414,
      "activations/layer20_attention_weight_min": -25.32110023498535,
      "activations/layer21_attention_weight_max": 32.15498733520508,
      "activations/layer21_attention_weight_min": -24.54326820373535,
      "activations/layer22_attention_weight_max": 49.34754943847656,
      "activations/layer22_attention_weight_min": -32.13727569580078,
      "activations/layer23_attention_weight_max": 32.770164489746094,
      "activations/layer23_attention_weight_min": -27.29500389099121,
      "activations/layer2_attention_weight_max": 36.404052734375,
      "activations/layer2_attention_weight_min": -35.188316345214844,
      "activations/layer3_attention_weight_max": 107.68794250488281,
      "activations/layer3_attention_weight_min": -109.95901489257812,
      "activations/layer4_attention_weight_max": 108.6720199584961,
      "activations/layer4_attention_weight_min": -111.58802795410156,
      "activations/layer5_attention_weight_max": 52.9334831237793,
      "activations/layer5_attention_weight_min": -63.29496765136719,
      "activations/layer6_attention_weight_max": 44.163291931152344,
      "activations/layer6_attention_weight_min": -47.07914733886719,
      "activations/layer7_attention_weight_max": 94.77489471435547,
      "activations/layer7_attention_weight_min": -100.12918090820312,
      "activations/layer8_attention_weight_max": 42.682918548583984,
      "activations/layer8_attention_weight_min": -49.25029754638672,
      "activations/layer9_attention_weight_max": 32.62246322631836,
      "activations/layer9_attention_weight_min": -39.64313888549805,
      "epoch": 22.32,
      "learning_rate": 6.135606060606059e-06,
      "loss": 2.6957,
      "step": 384100
    },
    {
      "activations/layer0_attention_weight_max": 16.439382553100586,
      "activations/layer0_attention_weight_min": -14.348238945007324,
      "activations/layer10_attention_weight_max": 36.51338577270508,
      "activations/layer10_attention_weight_min": -37.21156311035156,
      "activations/layer11_attention_weight_max": 37.17292022705078,
      "activations/layer11_attention_weight_min": -38.32304763793945,
      "activations/layer12_attention_weight_max": 28.380050659179688,
      "activations/layer12_attention_weight_min": -27.719606399536133,
      "activations/layer13_attention_weight_max": 43.85213851928711,
      "activations/layer13_attention_weight_min": -36.90682601928711,
      "activations/layer14_attention_weight_max": 46.883766174316406,
      "activations/layer14_attention_weight_min": -38.969093322753906,
      "activations/layer15_attention_weight_max": 46.067481994628906,
      "activations/layer15_attention_weight_min": -39.08519744873047,
      "activations/layer16_attention_weight_max": 30.61463165283203,
      "activations/layer16_attention_weight_min": -29.748226165771484,
      "activations/layer17_attention_weight_max": 33.031761169433594,
      "activations/layer17_attention_weight_min": -27.14927864074707,
      "activations/layer18_attention_weight_max": 33.48168182373047,
      "activations/layer18_attention_weight_min": -29.58521842956543,
      "activations/layer19_attention_weight_max": 34.039710998535156,
      "activations/layer19_attention_weight_min": -32.51165008544922,
      "activations/layer1_attention_weight_max": 16.283950805664062,
      "activations/layer1_attention_weight_min": -15.944218635559082,
      "activations/layer20_attention_weight_max": 28.306222915649414,
      "activations/layer20_attention_weight_min": -26.491975784301758,
      "activations/layer21_attention_weight_max": 27.744626998901367,
      "activations/layer21_attention_weight_min": -25.262962341308594,
      "activations/layer22_attention_weight_max": 42.02735900878906,
      "activations/layer22_attention_weight_min": -34.66120910644531,
      "activations/layer23_attention_weight_max": 34.269317626953125,
      "activations/layer23_attention_weight_min": -28.12633514404297,
      "activations/layer2_attention_weight_max": 34.509674072265625,
      "activations/layer2_attention_weight_min": -34.019569396972656,
      "activations/layer3_attention_weight_max": 110.79617309570312,
      "activations/layer3_attention_weight_min": -108.3316650390625,
      "activations/layer4_attention_weight_max": 110.58495330810547,
      "activations/layer4_attention_weight_min": -112.4013900756836,
      "activations/layer5_attention_weight_max": 50.16555404663086,
      "activations/layer5_attention_weight_min": -62.280941009521484,
      "activations/layer6_attention_weight_max": 44.38671112060547,
      "activations/layer6_attention_weight_min": -46.756649017333984,
      "activations/layer7_attention_weight_max": 97.07896423339844,
      "activations/layer7_attention_weight_min": -98.31902313232422,
      "activations/layer8_attention_weight_max": 43.245784759521484,
      "activations/layer8_attention_weight_min": -46.885276794433594,
      "activations/layer9_attention_weight_max": 32.41972732543945,
      "activations/layer9_attention_weight_min": -36.20274353027344,
      "epoch": 22.32,
      "learning_rate": 6.116666666666667e-06,
      "loss": 2.7081,
      "step": 384150
    },
    {
      "activations/layer0_attention_weight_max": 16.174795150756836,
      "activations/layer0_attention_weight_min": -13.347939491271973,
      "activations/layer10_attention_weight_max": 34.73177719116211,
      "activations/layer10_attention_weight_min": -35.9406623840332,
      "activations/layer11_attention_weight_max": 35.66522979736328,
      "activations/layer11_attention_weight_min": -34.69422912597656,
      "activations/layer12_attention_weight_max": 26.4521541595459,
      "activations/layer12_attention_weight_min": -26.829809188842773,
      "activations/layer13_attention_weight_max": 40.5457763671875,
      "activations/layer13_attention_weight_min": -36.421688079833984,
      "activations/layer14_attention_weight_max": 41.70594024658203,
      "activations/layer14_attention_weight_min": -39.09819030761719,
      "activations/layer15_attention_weight_max": 39.79045867919922,
      "activations/layer15_attention_weight_min": -37.5063591003418,
      "activations/layer16_attention_weight_max": 27.828937530517578,
      "activations/layer16_attention_weight_min": -30.396480560302734,
      "activations/layer17_attention_weight_max": 28.655677795410156,
      "activations/layer17_attention_weight_min": -26.8231258392334,
      "activations/layer18_attention_weight_max": 34.08599853515625,
      "activations/layer18_attention_weight_min": -27.190523147583008,
      "activations/layer19_attention_weight_max": 33.095218658447266,
      "activations/layer19_attention_weight_min": -33.59123229980469,
      "activations/layer1_attention_weight_max": 16.108827590942383,
      "activations/layer1_attention_weight_min": -14.833023071289062,
      "activations/layer20_attention_weight_max": 28.110660552978516,
      "activations/layer20_attention_weight_min": -25.436904907226562,
      "activations/layer21_attention_weight_max": 27.791519165039062,
      "activations/layer21_attention_weight_min": -24.17609214782715,
      "activations/layer22_attention_weight_max": 39.86557388305664,
      "activations/layer22_attention_weight_min": -33.520687103271484,
      "activations/layer23_attention_weight_max": 31.088041305541992,
      "activations/layer23_attention_weight_min": -29.356914520263672,
      "activations/layer2_attention_weight_max": 34.80022430419922,
      "activations/layer2_attention_weight_min": -33.02923583984375,
      "activations/layer3_attention_weight_max": 109.84212493896484,
      "activations/layer3_attention_weight_min": -109.92105865478516,
      "activations/layer4_attention_weight_max": 110.02022552490234,
      "activations/layer4_attention_weight_min": -115.38916015625,
      "activations/layer5_attention_weight_max": 51.802284240722656,
      "activations/layer5_attention_weight_min": -60.8626708984375,
      "activations/layer6_attention_weight_max": 45.06293869018555,
      "activations/layer6_attention_weight_min": -49.05690383911133,
      "activations/layer7_attention_weight_max": 101.92607879638672,
      "activations/layer7_attention_weight_min": -100.06243896484375,
      "activations/layer8_attention_weight_max": 42.351558685302734,
      "activations/layer8_attention_weight_min": -47.39617156982422,
      "activations/layer9_attention_weight_max": 32.766387939453125,
      "activations/layer9_attention_weight_min": -35.847084045410156,
      "epoch": 22.32,
      "learning_rate": 6.0977272727272726e-06,
      "loss": 2.706,
      "step": 384200
    },
    {
      "activations/layer0_attention_weight_max": 16.714801788330078,
      "activations/layer0_attention_weight_min": -15.507536888122559,
      "activations/layer10_attention_weight_max": 36.685611724853516,
      "activations/layer10_attention_weight_min": -36.012306213378906,
      "activations/layer11_attention_weight_max": 38.2731819152832,
      "activations/layer11_attention_weight_min": -37.99888610839844,
      "activations/layer12_attention_weight_max": 28.18844223022461,
      "activations/layer12_attention_weight_min": -30.38401985168457,
      "activations/layer13_attention_weight_max": 43.02174377441406,
      "activations/layer13_attention_weight_min": -39.34895706176758,
      "activations/layer14_attention_weight_max": 46.69876480102539,
      "activations/layer14_attention_weight_min": -44.89643859863281,
      "activations/layer15_attention_weight_max": 45.87299728393555,
      "activations/layer15_attention_weight_min": -41.07636260986328,
      "activations/layer16_attention_weight_max": 32.46677017211914,
      "activations/layer16_attention_weight_min": -30.503507614135742,
      "activations/layer17_attention_weight_max": 30.361005783081055,
      "activations/layer17_attention_weight_min": -29.160797119140625,
      "activations/layer18_attention_weight_max": 33.79271697998047,
      "activations/layer18_attention_weight_min": -29.775962829589844,
      "activations/layer19_attention_weight_max": 40.75830078125,
      "activations/layer19_attention_weight_min": -37.589664459228516,
      "activations/layer1_attention_weight_max": 16.13677215576172,
      "activations/layer1_attention_weight_min": -15.413034439086914,
      "activations/layer20_attention_weight_max": 35.77781295776367,
      "activations/layer20_attention_weight_min": -27.86690330505371,
      "activations/layer21_attention_weight_max": 33.08311462402344,
      "activations/layer21_attention_weight_min": -27.83163070678711,
      "activations/layer22_attention_weight_max": 49.68711853027344,
      "activations/layer22_attention_weight_min": -39.19379425048828,
      "activations/layer23_attention_weight_max": 40.162445068359375,
      "activations/layer23_attention_weight_min": -28.15637969970703,
      "activations/layer2_attention_weight_max": 34.02296447753906,
      "activations/layer2_attention_weight_min": -33.24339294433594,
      "activations/layer3_attention_weight_max": 109.35235595703125,
      "activations/layer3_attention_weight_min": -109.6813735961914,
      "activations/layer4_attention_weight_max": 112.45393371582031,
      "activations/layer4_attention_weight_min": -114.89644622802734,
      "activations/layer5_attention_weight_max": 51.55495834350586,
      "activations/layer5_attention_weight_min": -62.69468688964844,
      "activations/layer6_attention_weight_max": 45.6799430847168,
      "activations/layer6_attention_weight_min": -48.28697967529297,
      "activations/layer7_attention_weight_max": 100.0139389038086,
      "activations/layer7_attention_weight_min": -104.49971008300781,
      "activations/layer8_attention_weight_max": 44.53293228149414,
      "activations/layer8_attention_weight_min": -46.78778076171875,
      "activations/layer9_attention_weight_max": 33.30023193359375,
      "activations/layer9_attention_weight_min": -34.86103439331055,
      "epoch": 22.33,
      "learning_rate": 6.078787878787878e-06,
      "loss": 2.6972,
      "step": 384250
    },
    {
      "activations/layer0_attention_weight_max": 16.30038070678711,
      "activations/layer0_attention_weight_min": -13.900067329406738,
      "activations/layer10_attention_weight_max": 36.19982147216797,
      "activations/layer10_attention_weight_min": -37.94645690917969,
      "activations/layer11_attention_weight_max": 37.373958587646484,
      "activations/layer11_attention_weight_min": -38.6562614440918,
      "activations/layer12_attention_weight_max": 27.32768440246582,
      "activations/layer12_attention_weight_min": -27.68236541748047,
      "activations/layer13_attention_weight_max": 39.2357063293457,
      "activations/layer13_attention_weight_min": -37.43183898925781,
      "activations/layer14_attention_weight_max": 42.79853439331055,
      "activations/layer14_attention_weight_min": -39.215843200683594,
      "activations/layer15_attention_weight_max": 39.12210464477539,
      "activations/layer15_attention_weight_min": -35.66566848754883,
      "activations/layer16_attention_weight_max": 27.758176803588867,
      "activations/layer16_attention_weight_min": -30.152603149414062,
      "activations/layer17_attention_weight_max": 29.093997955322266,
      "activations/layer17_attention_weight_min": -28.93281364440918,
      "activations/layer18_attention_weight_max": 33.65575408935547,
      "activations/layer18_attention_weight_min": -29.848241806030273,
      "activations/layer19_attention_weight_max": 34.13844299316406,
      "activations/layer19_attention_weight_min": -33.175743103027344,
      "activations/layer1_attention_weight_max": 15.723244667053223,
      "activations/layer1_attention_weight_min": -15.338640213012695,
      "activations/layer20_attention_weight_max": 28.245849609375,
      "activations/layer20_attention_weight_min": -25.16592025756836,
      "activations/layer21_attention_weight_max": 27.523962020874023,
      "activations/layer21_attention_weight_min": -24.154504776000977,
      "activations/layer22_attention_weight_max": 42.09629821777344,
      "activations/layer22_attention_weight_min": -33.771156311035156,
      "activations/layer23_attention_weight_max": 32.26239013671875,
      "activations/layer23_attention_weight_min": -26.473873138427734,
      "activations/layer2_attention_weight_max": 33.94002151489258,
      "activations/layer2_attention_weight_min": -34.71002960205078,
      "activations/layer3_attention_weight_max": 104.59962463378906,
      "activations/layer3_attention_weight_min": -111.09160614013672,
      "activations/layer4_attention_weight_max": 110.3646469116211,
      "activations/layer4_attention_weight_min": -116.20732879638672,
      "activations/layer5_attention_weight_max": 51.530052185058594,
      "activations/layer5_attention_weight_min": -60.1878776550293,
      "activations/layer6_attention_weight_max": 45.90640640258789,
      "activations/layer6_attention_weight_min": -49.584774017333984,
      "activations/layer7_attention_weight_max": 95.563232421875,
      "activations/layer7_attention_weight_min": -105.51911163330078,
      "activations/layer8_attention_weight_max": 44.20101547241211,
      "activations/layer8_attention_weight_min": -46.75394821166992,
      "activations/layer9_attention_weight_max": 33.2397575378418,
      "activations/layer9_attention_weight_min": -37.20603561401367,
      "epoch": 22.33,
      "learning_rate": 6.059848484848484e-06,
      "loss": 2.6888,
      "step": 384300
    },
    {
      "activations/layer0_attention_weight_max": 16.811450958251953,
      "activations/layer0_attention_weight_min": -13.170425415039062,
      "activations/layer10_attention_weight_max": 34.877593994140625,
      "activations/layer10_attention_weight_min": -35.2172737121582,
      "activations/layer11_attention_weight_max": 34.04887390136719,
      "activations/layer11_attention_weight_min": -36.735015869140625,
      "activations/layer12_attention_weight_max": 26.793933868408203,
      "activations/layer12_attention_weight_min": -27.054771423339844,
      "activations/layer13_attention_weight_max": 40.697540283203125,
      "activations/layer13_attention_weight_min": -37.71379089355469,
      "activations/layer14_attention_weight_max": 43.0043830871582,
      "activations/layer14_attention_weight_min": -40.57960891723633,
      "activations/layer15_attention_weight_max": 40.82425308227539,
      "activations/layer15_attention_weight_min": -39.220985412597656,
      "activations/layer16_attention_weight_max": 29.783464431762695,
      "activations/layer16_attention_weight_min": -30.634288787841797,
      "activations/layer17_attention_weight_max": 29.247575759887695,
      "activations/layer17_attention_weight_min": -26.705446243286133,
      "activations/layer18_attention_weight_max": 33.053096771240234,
      "activations/layer18_attention_weight_min": -25.24546241760254,
      "activations/layer19_attention_weight_max": 32.174800872802734,
      "activations/layer19_attention_weight_min": -32.06521224975586,
      "activations/layer1_attention_weight_max": 16.84697723388672,
      "activations/layer1_attention_weight_min": -15.08709716796875,
      "activations/layer20_attention_weight_max": 29.100250244140625,
      "activations/layer20_attention_weight_min": -23.659677505493164,
      "activations/layer21_attention_weight_max": 27.928844451904297,
      "activations/layer21_attention_weight_min": -24.63633918762207,
      "activations/layer22_attention_weight_max": 38.66295623779297,
      "activations/layer22_attention_weight_min": -31.399166107177734,
      "activations/layer23_attention_weight_max": 29.18293571472168,
      "activations/layer23_attention_weight_min": -24.757461547851562,
      "activations/layer2_attention_weight_max": 33.852779388427734,
      "activations/layer2_attention_weight_min": -31.53901481628418,
      "activations/layer3_attention_weight_max": 105.16069793701172,
      "activations/layer3_attention_weight_min": -103.89852142333984,
      "activations/layer4_attention_weight_max": 109.8218994140625,
      "activations/layer4_attention_weight_min": -111.72502899169922,
      "activations/layer5_attention_weight_max": 55.16090393066406,
      "activations/layer5_attention_weight_min": -62.57697296142578,
      "activations/layer6_attention_weight_max": 43.52811813354492,
      "activations/layer6_attention_weight_min": -48.97956848144531,
      "activations/layer7_attention_weight_max": 97.33963775634766,
      "activations/layer7_attention_weight_min": -101.49980926513672,
      "activations/layer8_attention_weight_max": 43.013065338134766,
      "activations/layer8_attention_weight_min": -44.4954719543457,
      "activations/layer9_attention_weight_max": 30.395963668823242,
      "activations/layer9_attention_weight_min": -34.491729736328125,
      "epoch": 22.33,
      "learning_rate": 6.040909090909091e-06,
      "loss": 2.7012,
      "step": 384350
    },
    {
      "activations/layer0_attention_weight_max": 17.03495216369629,
      "activations/layer0_attention_weight_min": -12.85692310333252,
      "activations/layer10_attention_weight_max": 34.66493225097656,
      "activations/layer10_attention_weight_min": -38.24745559692383,
      "activations/layer11_attention_weight_max": 33.50510025024414,
      "activations/layer11_attention_weight_min": -34.81966781616211,
      "activations/layer12_attention_weight_max": 28.13766860961914,
      "activations/layer12_attention_weight_min": -27.871387481689453,
      "activations/layer13_attention_weight_max": 41.52555465698242,
      "activations/layer13_attention_weight_min": -36.774803161621094,
      "activations/layer14_attention_weight_max": 43.67204666137695,
      "activations/layer14_attention_weight_min": -37.68545150756836,
      "activations/layer15_attention_weight_max": 44.824806213378906,
      "activations/layer15_attention_weight_min": -38.8111572265625,
      "activations/layer16_attention_weight_max": 32.74386215209961,
      "activations/layer16_attention_weight_min": -30.391239166259766,
      "activations/layer17_attention_weight_max": 29.124582290649414,
      "activations/layer17_attention_weight_min": -28.40131378173828,
      "activations/layer18_attention_weight_max": 33.14841079711914,
      "activations/layer18_attention_weight_min": -28.735355377197266,
      "activations/layer19_attention_weight_max": 43.61101531982422,
      "activations/layer19_attention_weight_min": -34.784461975097656,
      "activations/layer1_attention_weight_max": 15.530061721801758,
      "activations/layer1_attention_weight_min": -15.045090675354004,
      "activations/layer20_attention_weight_max": 33.88119888305664,
      "activations/layer20_attention_weight_min": -26.79226303100586,
      "activations/layer21_attention_weight_max": 30.82461929321289,
      "activations/layer21_attention_weight_min": -25.910314559936523,
      "activations/layer22_attention_weight_max": 46.176536560058594,
      "activations/layer22_attention_weight_min": -34.957763671875,
      "activations/layer23_attention_weight_max": 38.44325256347656,
      "activations/layer23_attention_weight_min": -27.181774139404297,
      "activations/layer2_attention_weight_max": 35.660709381103516,
      "activations/layer2_attention_weight_min": -34.470359802246094,
      "activations/layer3_attention_weight_max": 104.11811065673828,
      "activations/layer3_attention_weight_min": -110.73861694335938,
      "activations/layer4_attention_weight_max": 107.9296646118164,
      "activations/layer4_attention_weight_min": -114.47236633300781,
      "activations/layer5_attention_weight_max": 49.877498626708984,
      "activations/layer5_attention_weight_min": -63.96946716308594,
      "activations/layer6_attention_weight_max": 46.01890182495117,
      "activations/layer6_attention_weight_min": -48.887882232666016,
      "activations/layer7_attention_weight_max": 96.91722106933594,
      "activations/layer7_attention_weight_min": -99.2217788696289,
      "activations/layer8_attention_weight_max": 44.15941619873047,
      "activations/layer8_attention_weight_min": -46.219539642333984,
      "activations/layer9_attention_weight_max": 31.703771591186523,
      "activations/layer9_attention_weight_min": -34.815956115722656,
      "epoch": 22.34,
      "learning_rate": 6.021969696969696e-06,
      "loss": 2.6985,
      "step": 384400
    },
    {
      "activations/layer0_attention_weight_max": 17.88199234008789,
      "activations/layer0_attention_weight_min": -13.18142318725586,
      "activations/layer10_attention_weight_max": 35.333473205566406,
      "activations/layer10_attention_weight_min": -35.88874053955078,
      "activations/layer11_attention_weight_max": 38.27955627441406,
      "activations/layer11_attention_weight_min": -36.024986267089844,
      "activations/layer12_attention_weight_max": 27.552555084228516,
      "activations/layer12_attention_weight_min": -27.083871841430664,
      "activations/layer13_attention_weight_max": 41.2166633605957,
      "activations/layer13_attention_weight_min": -38.48503494262695,
      "activations/layer14_attention_weight_max": 43.85343933105469,
      "activations/layer14_attention_weight_min": -39.589752197265625,
      "activations/layer15_attention_weight_max": 40.29356002807617,
      "activations/layer15_attention_weight_min": -36.699188232421875,
      "activations/layer16_attention_weight_max": 28.345016479492188,
      "activations/layer16_attention_weight_min": -28.39012336730957,
      "activations/layer17_attention_weight_max": 30.89714241027832,
      "activations/layer17_attention_weight_min": -26.066349029541016,
      "activations/layer18_attention_weight_max": 34.38315963745117,
      "activations/layer18_attention_weight_min": -26.040729522705078,
      "activations/layer19_attention_weight_max": 35.39182662963867,
      "activations/layer19_attention_weight_min": -31.52652931213379,
      "activations/layer1_attention_weight_max": 17.41126251220703,
      "activations/layer1_attention_weight_min": -14.958227157592773,
      "activations/layer20_attention_weight_max": 29.74959945678711,
      "activations/layer20_attention_weight_min": -26.494394302368164,
      "activations/layer21_attention_weight_max": 29.22758674621582,
      "activations/layer21_attention_weight_min": -25.234933853149414,
      "activations/layer22_attention_weight_max": 45.12909698486328,
      "activations/layer22_attention_weight_min": -32.76726150512695,
      "activations/layer23_attention_weight_max": 31.531042098999023,
      "activations/layer23_attention_weight_min": -25.441450119018555,
      "activations/layer2_attention_weight_max": 34.22916793823242,
      "activations/layer2_attention_weight_min": -33.929412841796875,
      "activations/layer3_attention_weight_max": 105.1133041381836,
      "activations/layer3_attention_weight_min": -108.83119201660156,
      "activations/layer4_attention_weight_max": 112.04835510253906,
      "activations/layer4_attention_weight_min": -115.44483947753906,
      "activations/layer5_attention_weight_max": 52.819313049316406,
      "activations/layer5_attention_weight_min": -63.91692352294922,
      "activations/layer6_attention_weight_max": 45.61875534057617,
      "activations/layer6_attention_weight_min": -47.49808120727539,
      "activations/layer7_attention_weight_max": 96.4791259765625,
      "activations/layer7_attention_weight_min": -99.96148681640625,
      "activations/layer8_attention_weight_max": 42.37267303466797,
      "activations/layer8_attention_weight_min": -46.148250579833984,
      "activations/layer9_attention_weight_max": 32.813926696777344,
      "activations/layer9_attention_weight_min": -34.85957717895508,
      "epoch": 22.34,
      "learning_rate": 6.003030303030302e-06,
      "loss": 2.7028,
      "step": 384450
    },
    {
      "activations/layer0_attention_weight_max": 16.856706619262695,
      "activations/layer0_attention_weight_min": -13.404991149902344,
      "activations/layer10_attention_weight_max": 41.17177200317383,
      "activations/layer10_attention_weight_min": -38.588321685791016,
      "activations/layer11_attention_weight_max": 40.2844123840332,
      "activations/layer11_attention_weight_min": -41.827796936035156,
      "activations/layer12_attention_weight_max": 28.345630645751953,
      "activations/layer12_attention_weight_min": -28.203571319580078,
      "activations/layer13_attention_weight_max": 40.4683952331543,
      "activations/layer13_attention_weight_min": -37.00658416748047,
      "activations/layer14_attention_weight_max": 45.36851501464844,
      "activations/layer14_attention_weight_min": -38.66340255737305,
      "activations/layer15_attention_weight_max": 42.72128677368164,
      "activations/layer15_attention_weight_min": -37.402488708496094,
      "activations/layer16_attention_weight_max": 30.89140510559082,
      "activations/layer16_attention_weight_min": -30.21141242980957,
      "activations/layer17_attention_weight_max": 31.04486083984375,
      "activations/layer17_attention_weight_min": -27.644062042236328,
      "activations/layer18_attention_weight_max": 31.56879234313965,
      "activations/layer18_attention_weight_min": -26.188344955444336,
      "activations/layer19_attention_weight_max": 39.15960693359375,
      "activations/layer19_attention_weight_min": -30.17426872253418,
      "activations/layer1_attention_weight_max": 16.133525848388672,
      "activations/layer1_attention_weight_min": -14.80297565460205,
      "activations/layer20_attention_weight_max": 29.86595344543457,
      "activations/layer20_attention_weight_min": -25.0585880279541,
      "activations/layer21_attention_weight_max": 26.732879638671875,
      "activations/layer21_attention_weight_min": -25.27080535888672,
      "activations/layer22_attention_weight_max": 43.21434783935547,
      "activations/layer22_attention_weight_min": -32.09236145019531,
      "activations/layer23_attention_weight_max": 30.385162353515625,
      "activations/layer23_attention_weight_min": -27.306020736694336,
      "activations/layer2_attention_weight_max": 34.52851104736328,
      "activations/layer2_attention_weight_min": -33.77783966064453,
      "activations/layer3_attention_weight_max": 108.34391021728516,
      "activations/layer3_attention_weight_min": -111.98062133789062,
      "activations/layer4_attention_weight_max": 112.17506408691406,
      "activations/layer4_attention_weight_min": -113.5416030883789,
      "activations/layer5_attention_weight_max": 51.3236198425293,
      "activations/layer5_attention_weight_min": -62.59832000732422,
      "activations/layer6_attention_weight_max": 47.14509201049805,
      "activations/layer6_attention_weight_min": -48.57021713256836,
      "activations/layer7_attention_weight_max": 110.9549789428711,
      "activations/layer7_attention_weight_min": -105.76921081542969,
      "activations/layer8_attention_weight_max": 52.62367630004883,
      "activations/layer8_attention_weight_min": -50.25766372680664,
      "activations/layer9_attention_weight_max": 38.75577926635742,
      "activations/layer9_attention_weight_min": -37.34415054321289,
      "epoch": 22.34,
      "learning_rate": 5.984090909090908e-06,
      "loss": 2.6887,
      "step": 384500
    },
    {
      "activations/layer0_attention_weight_max": 17.804931640625,
      "activations/layer0_attention_weight_min": -13.875152587890625,
      "activations/layer10_attention_weight_max": 39.01076889038086,
      "activations/layer10_attention_weight_min": -37.662567138671875,
      "activations/layer11_attention_weight_max": 37.94491195678711,
      "activations/layer11_attention_weight_min": -37.45008850097656,
      "activations/layer12_attention_weight_max": 27.68189811706543,
      "activations/layer12_attention_weight_min": -29.56833267211914,
      "activations/layer13_attention_weight_max": 43.19724655151367,
      "activations/layer13_attention_weight_min": -40.5128288269043,
      "activations/layer14_attention_weight_max": 46.36753463745117,
      "activations/layer14_attention_weight_min": -40.48250198364258,
      "activations/layer15_attention_weight_max": 42.68685531616211,
      "activations/layer15_attention_weight_min": -38.25197982788086,
      "activations/layer16_attention_weight_max": 31.838932037353516,
      "activations/layer16_attention_weight_min": -28.93450355529785,
      "activations/layer17_attention_weight_max": 32.76685333251953,
      "activations/layer17_attention_weight_min": -26.59447479248047,
      "activations/layer18_attention_weight_max": 31.03902816772461,
      "activations/layer18_attention_weight_min": -25.983844757080078,
      "activations/layer19_attention_weight_max": 42.347835540771484,
      "activations/layer19_attention_weight_min": -31.517789840698242,
      "activations/layer1_attention_weight_max": 16.941808700561523,
      "activations/layer1_attention_weight_min": -15.092936515808105,
      "activations/layer20_attention_weight_max": 29.301403045654297,
      "activations/layer20_attention_weight_min": -24.327669143676758,
      "activations/layer21_attention_weight_max": 27.004940032958984,
      "activations/layer21_attention_weight_min": -24.090965270996094,
      "activations/layer22_attention_weight_max": 37.19190216064453,
      "activations/layer22_attention_weight_min": -30.70370101928711,
      "activations/layer23_attention_weight_max": 29.622968673706055,
      "activations/layer23_attention_weight_min": -24.859739303588867,
      "activations/layer2_attention_weight_max": 34.90167236328125,
      "activations/layer2_attention_weight_min": -34.08546829223633,
      "activations/layer3_attention_weight_max": 107.41002655029297,
      "activations/layer3_attention_weight_min": -110.74998474121094,
      "activations/layer4_attention_weight_max": 111.22523498535156,
      "activations/layer4_attention_weight_min": -115.8643569946289,
      "activations/layer5_attention_weight_max": 52.067909240722656,
      "activations/layer5_attention_weight_min": -62.459468841552734,
      "activations/layer6_attention_weight_max": 47.77882766723633,
      "activations/layer6_attention_weight_min": -51.31000518798828,
      "activations/layer7_attention_weight_max": 100.49549102783203,
      "activations/layer7_attention_weight_min": -111.16818237304688,
      "activations/layer8_attention_weight_max": 48.63175964355469,
      "activations/layer8_attention_weight_min": -52.16380310058594,
      "activations/layer9_attention_weight_max": 34.58717346191406,
      "activations/layer9_attention_weight_min": -38.93220901489258,
      "epoch": 22.34,
      "learning_rate": 5.965151515151514e-06,
      "loss": 2.6902,
      "step": 384550
    },
    {
      "activations/layer0_attention_weight_max": 16.860469818115234,
      "activations/layer0_attention_weight_min": -14.337433815002441,
      "activations/layer10_attention_weight_max": 35.917781829833984,
      "activations/layer10_attention_weight_min": -36.864906311035156,
      "activations/layer11_attention_weight_max": 36.864784240722656,
      "activations/layer11_attention_weight_min": -39.12311553955078,
      "activations/layer12_attention_weight_max": 27.562294006347656,
      "activations/layer12_attention_weight_min": -28.81269073486328,
      "activations/layer13_attention_weight_max": 38.516265869140625,
      "activations/layer13_attention_weight_min": -36.90098190307617,
      "activations/layer14_attention_weight_max": 41.072731018066406,
      "activations/layer14_attention_weight_min": -39.61814880371094,
      "activations/layer15_attention_weight_max": 39.5972900390625,
      "activations/layer15_attention_weight_min": -39.01506423950195,
      "activations/layer16_attention_weight_max": 31.05838966369629,
      "activations/layer16_attention_weight_min": -31.52657127380371,
      "activations/layer17_attention_weight_max": 33.207637786865234,
      "activations/layer17_attention_weight_min": -26.296506881713867,
      "activations/layer18_attention_weight_max": 34.72301483154297,
      "activations/layer18_attention_weight_min": -29.409334182739258,
      "activations/layer19_attention_weight_max": 36.936405181884766,
      "activations/layer19_attention_weight_min": -32.797157287597656,
      "activations/layer1_attention_weight_max": 16.116989135742188,
      "activations/layer1_attention_weight_min": -14.81053352355957,
      "activations/layer20_attention_weight_max": 30.754528045654297,
      "activations/layer20_attention_weight_min": -26.95556640625,
      "activations/layer21_attention_weight_max": 30.891002655029297,
      "activations/layer21_attention_weight_min": -25.28793716430664,
      "activations/layer22_attention_weight_max": 44.20189666748047,
      "activations/layer22_attention_weight_min": -33.95442199707031,
      "activations/layer23_attention_weight_max": 30.282333374023438,
      "activations/layer23_attention_weight_min": -28.380233764648438,
      "activations/layer2_attention_weight_max": 35.88459777832031,
      "activations/layer2_attention_weight_min": -34.907779693603516,
      "activations/layer3_attention_weight_max": 107.4900131225586,
      "activations/layer3_attention_weight_min": -112.21307373046875,
      "activations/layer4_attention_weight_max": 111.24005126953125,
      "activations/layer4_attention_weight_min": -118.643310546875,
      "activations/layer5_attention_weight_max": 52.225830078125,
      "activations/layer5_attention_weight_min": -64.16056060791016,
      "activations/layer6_attention_weight_max": 45.562923431396484,
      "activations/layer6_attention_weight_min": -49.8432731628418,
      "activations/layer7_attention_weight_max": 100.50748443603516,
      "activations/layer7_attention_weight_min": -101.79007720947266,
      "activations/layer8_attention_weight_max": 43.92500686645508,
      "activations/layer8_attention_weight_min": -47.7453498840332,
      "activations/layer9_attention_weight_max": 32.44750213623047,
      "activations/layer9_attention_weight_min": -38.2377815246582,
      "epoch": 22.35,
      "learning_rate": 5.946212121212121e-06,
      "loss": 2.6913,
      "step": 384600
    },
    {
      "activations/layer0_attention_weight_max": 16.544553756713867,
      "activations/layer0_attention_weight_min": -15.196672439575195,
      "activations/layer10_attention_weight_max": 35.342987060546875,
      "activations/layer10_attention_weight_min": -37.771942138671875,
      "activations/layer11_attention_weight_max": 35.26076126098633,
      "activations/layer11_attention_weight_min": -38.473716735839844,
      "activations/layer12_attention_weight_max": 27.132369995117188,
      "activations/layer12_attention_weight_min": -27.722871780395508,
      "activations/layer13_attention_weight_max": 41.198909759521484,
      "activations/layer13_attention_weight_min": -37.646827697753906,
      "activations/layer14_attention_weight_max": 41.873199462890625,
      "activations/layer14_attention_weight_min": -40.66208267211914,
      "activations/layer15_attention_weight_max": 40.296871185302734,
      "activations/layer15_attention_weight_min": -39.58098220825195,
      "activations/layer16_attention_weight_max": 30.815340042114258,
      "activations/layer16_attention_weight_min": -29.53898048400879,
      "activations/layer17_attention_weight_max": 31.607961654663086,
      "activations/layer17_attention_weight_min": -27.90155792236328,
      "activations/layer18_attention_weight_max": 32.67515182495117,
      "activations/layer18_attention_weight_min": -26.28851318359375,
      "activations/layer19_attention_weight_max": 33.96041488647461,
      "activations/layer19_attention_weight_min": -31.329679489135742,
      "activations/layer1_attention_weight_max": 17.475765228271484,
      "activations/layer1_attention_weight_min": -16.327056884765625,
      "activations/layer20_attention_weight_max": 29.049577713012695,
      "activations/layer20_attention_weight_min": -25.02802085876465,
      "activations/layer21_attention_weight_max": 29.25485610961914,
      "activations/layer21_attention_weight_min": -24.283742904663086,
      "activations/layer22_attention_weight_max": 44.34423828125,
      "activations/layer22_attention_weight_min": -31.562559127807617,
      "activations/layer23_attention_weight_max": 30.928564071655273,
      "activations/layer23_attention_weight_min": -25.050674438476562,
      "activations/layer2_attention_weight_max": 34.791324615478516,
      "activations/layer2_attention_weight_min": -34.225440979003906,
      "activations/layer3_attention_weight_max": 106.83500671386719,
      "activations/layer3_attention_weight_min": -110.32146453857422,
      "activations/layer4_attention_weight_max": 115.6245346069336,
      "activations/layer4_attention_weight_min": -114.5875244140625,
      "activations/layer5_attention_weight_max": 51.492210388183594,
      "activations/layer5_attention_weight_min": -60.94755554199219,
      "activations/layer6_attention_weight_max": 45.46625900268555,
      "activations/layer6_attention_weight_min": -49.09254837036133,
      "activations/layer7_attention_weight_max": 100.59075927734375,
      "activations/layer7_attention_weight_min": -101.48460388183594,
      "activations/layer8_attention_weight_max": 44.176177978515625,
      "activations/layer8_attention_weight_min": -48.945369720458984,
      "activations/layer9_attention_weight_max": 32.926055908203125,
      "activations/layer9_attention_weight_min": -38.59159469604492,
      "epoch": 22.35,
      "learning_rate": 5.927272727272727e-06,
      "loss": 2.6892,
      "step": 384650
    },
    {
      "activations/layer0_attention_weight_max": 17.364540100097656,
      "activations/layer0_attention_weight_min": -14.48541259765625,
      "activations/layer10_attention_weight_max": 34.32286834716797,
      "activations/layer10_attention_weight_min": -36.8886833190918,
      "activations/layer11_attention_weight_max": 37.868560791015625,
      "activations/layer11_attention_weight_min": -38.92784118652344,
      "activations/layer12_attention_weight_max": 27.084306716918945,
      "activations/layer12_attention_weight_min": -28.423755645751953,
      "activations/layer13_attention_weight_max": 38.058074951171875,
      "activations/layer13_attention_weight_min": -36.04199981689453,
      "activations/layer14_attention_weight_max": 43.111671447753906,
      "activations/layer14_attention_weight_min": -39.01933670043945,
      "activations/layer15_attention_weight_max": 39.35345458984375,
      "activations/layer15_attention_weight_min": -39.176334381103516,
      "activations/layer16_attention_weight_max": 27.72597885131836,
      "activations/layer16_attention_weight_min": -29.433425903320312,
      "activations/layer17_attention_weight_max": 27.393770217895508,
      "activations/layer17_attention_weight_min": -27.618892669677734,
      "activations/layer18_attention_weight_max": 31.315580368041992,
      "activations/layer18_attention_weight_min": -29.02580451965332,
      "activations/layer19_attention_weight_max": 32.07418441772461,
      "activations/layer19_attention_weight_min": -31.185972213745117,
      "activations/layer1_attention_weight_max": 16.322946548461914,
      "activations/layer1_attention_weight_min": -16.48118019104004,
      "activations/layer20_attention_weight_max": 26.327890396118164,
      "activations/layer20_attention_weight_min": -25.77239227294922,
      "activations/layer21_attention_weight_max": 26.311920166015625,
      "activations/layer21_attention_weight_min": -25.372976303100586,
      "activations/layer22_attention_weight_max": 38.96407699584961,
      "activations/layer22_attention_weight_min": -34.387413024902344,
      "activations/layer23_attention_weight_max": 29.97564697265625,
      "activations/layer23_attention_weight_min": -25.820152282714844,
      "activations/layer2_attention_weight_max": 34.85124206542969,
      "activations/layer2_attention_weight_min": -33.965545654296875,
      "activations/layer3_attention_weight_max": 111.10747528076172,
      "activations/layer3_attention_weight_min": -108.81658172607422,
      "activations/layer4_attention_weight_max": 110.5813217163086,
      "activations/layer4_attention_weight_min": -112.2541732788086,
      "activations/layer5_attention_weight_max": 48.57257080078125,
      "activations/layer5_attention_weight_min": -61.263580322265625,
      "activations/layer6_attention_weight_max": 44.12577819824219,
      "activations/layer6_attention_weight_min": -48.0374641418457,
      "activations/layer7_attention_weight_max": 95.41795349121094,
      "activations/layer7_attention_weight_min": -99.0676040649414,
      "activations/layer8_attention_weight_max": 43.10710525512695,
      "activations/layer8_attention_weight_min": -46.03035354614258,
      "activations/layer9_attention_weight_max": 32.81697082519531,
      "activations/layer9_attention_weight_min": -36.58424758911133,
      "epoch": 22.35,
      "learning_rate": 5.908333333333333e-06,
      "loss": 2.7127,
      "step": 384700
    },
    {
      "activations/layer0_attention_weight_max": 16.952796936035156,
      "activations/layer0_attention_weight_min": -13.018717765808105,
      "activations/layer10_attention_weight_max": 44.659820556640625,
      "activations/layer10_attention_weight_min": -43.23405075073242,
      "activations/layer11_attention_weight_max": 43.86864471435547,
      "activations/layer11_attention_weight_min": -42.62114334106445,
      "activations/layer12_attention_weight_max": 32.331298828125,
      "activations/layer12_attention_weight_min": -32.307395935058594,
      "activations/layer13_attention_weight_max": 47.89913558959961,
      "activations/layer13_attention_weight_min": -43.53608322143555,
      "activations/layer14_attention_weight_max": 51.41065979003906,
      "activations/layer14_attention_weight_min": -45.87567901611328,
      "activations/layer15_attention_weight_max": 53.61503601074219,
      "activations/layer15_attention_weight_min": -42.27144241333008,
      "activations/layer16_attention_weight_max": 34.73252487182617,
      "activations/layer16_attention_weight_min": -31.208621978759766,
      "activations/layer17_attention_weight_max": 37.57626724243164,
      "activations/layer17_attention_weight_min": -30.93400001525879,
      "activations/layer18_attention_weight_max": 38.04216766357422,
      "activations/layer18_attention_weight_min": -29.447465896606445,
      "activations/layer19_attention_weight_max": 37.0097770690918,
      "activations/layer19_attention_weight_min": -32.27351760864258,
      "activations/layer1_attention_weight_max": 17.091127395629883,
      "activations/layer1_attention_weight_min": -16.590028762817383,
      "activations/layer20_attention_weight_max": 28.95879554748535,
      "activations/layer20_attention_weight_min": -27.319475173950195,
      "activations/layer21_attention_weight_max": 27.22616958618164,
      "activations/layer21_attention_weight_min": -27.69781494140625,
      "activations/layer22_attention_weight_max": 45.07844924926758,
      "activations/layer22_attention_weight_min": -32.77309036254883,
      "activations/layer23_attention_weight_max": 35.291954040527344,
      "activations/layer23_attention_weight_min": -25.699974060058594,
      "activations/layer2_attention_weight_max": 35.26669692993164,
      "activations/layer2_attention_weight_min": -35.52312469482422,
      "activations/layer3_attention_weight_max": 109.8399887084961,
      "activations/layer3_attention_weight_min": -118.25459289550781,
      "activations/layer4_attention_weight_max": 115.5472640991211,
      "activations/layer4_attention_weight_min": -115.19820404052734,
      "activations/layer5_attention_weight_max": 50.51603698730469,
      "activations/layer5_attention_weight_min": -59.36402130126953,
      "activations/layer6_attention_weight_max": 46.878963470458984,
      "activations/layer6_attention_weight_min": -49.67012405395508,
      "activations/layer7_attention_weight_max": 109.38256072998047,
      "activations/layer7_attention_weight_min": -107.92674255371094,
      "activations/layer8_attention_weight_max": 48.75514602661133,
      "activations/layer8_attention_weight_min": -49.20188522338867,
      "activations/layer9_attention_weight_max": 39.418487548828125,
      "activations/layer9_attention_weight_min": -41.00762176513672,
      "epoch": 22.36,
      "learning_rate": 5.889393939393939e-06,
      "loss": 2.6924,
      "step": 384750
    },
    {
      "activations/layer0_attention_weight_max": 16.636911392211914,
      "activations/layer0_attention_weight_min": -15.009500503540039,
      "activations/layer10_attention_weight_max": 34.6497802734375,
      "activations/layer10_attention_weight_min": -36.25399398803711,
      "activations/layer11_attention_weight_max": 33.968711853027344,
      "activations/layer11_attention_weight_min": -35.725257873535156,
      "activations/layer12_attention_weight_max": 27.1137752532959,
      "activations/layer12_attention_weight_min": -26.955867767333984,
      "activations/layer13_attention_weight_max": 40.00653839111328,
      "activations/layer13_attention_weight_min": -35.40843963623047,
      "activations/layer14_attention_weight_max": 42.300941467285156,
      "activations/layer14_attention_weight_min": -39.20896911621094,
      "activations/layer15_attention_weight_max": 39.24470138549805,
      "activations/layer15_attention_weight_min": -37.66322708129883,
      "activations/layer16_attention_weight_max": 27.99318504333496,
      "activations/layer16_attention_weight_min": -29.59076499938965,
      "activations/layer17_attention_weight_max": 29.97771644592285,
      "activations/layer17_attention_weight_min": -27.278200149536133,
      "activations/layer18_attention_weight_max": 33.34174728393555,
      "activations/layer18_attention_weight_min": -25.319774627685547,
      "activations/layer19_attention_weight_max": 31.933664321899414,
      "activations/layer19_attention_weight_min": -30.733123779296875,
      "activations/layer1_attention_weight_max": 15.993813514709473,
      "activations/layer1_attention_weight_min": -15.369412422180176,
      "activations/layer20_attention_weight_max": 29.331621170043945,
      "activations/layer20_attention_weight_min": -26.493263244628906,
      "activations/layer21_attention_weight_max": 29.962167739868164,
      "activations/layer21_attention_weight_min": -24.02427101135254,
      "activations/layer22_attention_weight_max": 39.924983978271484,
      "activations/layer22_attention_weight_min": -29.65376091003418,
      "activations/layer23_attention_weight_max": 31.107465744018555,
      "activations/layer23_attention_weight_min": -24.841106414794922,
      "activations/layer2_attention_weight_max": 33.54526901245117,
      "activations/layer2_attention_weight_min": -32.89836883544922,
      "activations/layer3_attention_weight_max": 110.32913208007812,
      "activations/layer3_attention_weight_min": -109.644775390625,
      "activations/layer4_attention_weight_max": 110.2449722290039,
      "activations/layer4_attention_weight_min": -115.31046295166016,
      "activations/layer5_attention_weight_max": 48.88929748535156,
      "activations/layer5_attention_weight_min": -61.73706817626953,
      "activations/layer6_attention_weight_max": 43.46822738647461,
      "activations/layer6_attention_weight_min": -46.78334045410156,
      "activations/layer7_attention_weight_max": 94.4472427368164,
      "activations/layer7_attention_weight_min": -98.97405242919922,
      "activations/layer8_attention_weight_max": 40.943538665771484,
      "activations/layer8_attention_weight_min": -42.86057662963867,
      "activations/layer9_attention_weight_max": 32.898921966552734,
      "activations/layer9_attention_weight_min": -35.562347412109375,
      "epoch": 22.36,
      "learning_rate": 5.870454545454545e-06,
      "loss": 2.7044,
      "step": 384800
    },
    {
      "activations/layer0_attention_weight_max": 17.030359268188477,
      "activations/layer0_attention_weight_min": -13.695152282714844,
      "activations/layer10_attention_weight_max": 37.02467727661133,
      "activations/layer10_attention_weight_min": -37.20200729370117,
      "activations/layer11_attention_weight_max": 38.323585510253906,
      "activations/layer11_attention_weight_min": -39.165287017822266,
      "activations/layer12_attention_weight_max": 30.238622665405273,
      "activations/layer12_attention_weight_min": -27.505029678344727,
      "activations/layer13_attention_weight_max": 43.365745544433594,
      "activations/layer13_attention_weight_min": -38.53174591064453,
      "activations/layer14_attention_weight_max": 45.406166076660156,
      "activations/layer14_attention_weight_min": -39.64051055908203,
      "activations/layer15_attention_weight_max": 45.30411148071289,
      "activations/layer15_attention_weight_min": -39.4303092956543,
      "activations/layer16_attention_weight_max": 36.631568908691406,
      "activations/layer16_attention_weight_min": -30.310949325561523,
      "activations/layer17_attention_weight_max": 35.74612045288086,
      "activations/layer17_attention_weight_min": -30.207529067993164,
      "activations/layer18_attention_weight_max": 37.091121673583984,
      "activations/layer18_attention_weight_min": -27.704391479492188,
      "activations/layer19_attention_weight_max": 42.64830780029297,
      "activations/layer19_attention_weight_min": -33.92756652832031,
      "activations/layer1_attention_weight_max": 16.370559692382812,
      "activations/layer1_attention_weight_min": -15.904321670532227,
      "activations/layer20_attention_weight_max": 33.9461784362793,
      "activations/layer20_attention_weight_min": -26.778091430664062,
      "activations/layer21_attention_weight_max": 31.839797973632812,
      "activations/layer21_attention_weight_min": -24.80025291442871,
      "activations/layer22_attention_weight_max": 49.463157653808594,
      "activations/layer22_attention_weight_min": -33.9210205078125,
      "activations/layer23_attention_weight_max": 35.307106018066406,
      "activations/layer23_attention_weight_min": -27.671140670776367,
      "activations/layer2_attention_weight_max": 33.363914489746094,
      "activations/layer2_attention_weight_min": -34.14408874511719,
      "activations/layer3_attention_weight_max": 106.9076156616211,
      "activations/layer3_attention_weight_min": -106.08723449707031,
      "activations/layer4_attention_weight_max": 109.18914031982422,
      "activations/layer4_attention_weight_min": -112.93641662597656,
      "activations/layer5_attention_weight_max": 52.217308044433594,
      "activations/layer5_attention_weight_min": -62.44196701049805,
      "activations/layer6_attention_weight_max": 45.83799362182617,
      "activations/layer6_attention_weight_min": -48.22995376586914,
      "activations/layer7_attention_weight_max": 99.03121948242188,
      "activations/layer7_attention_weight_min": -103.8324966430664,
      "activations/layer8_attention_weight_max": 43.907325744628906,
      "activations/layer8_attention_weight_min": -47.14099884033203,
      "activations/layer9_attention_weight_max": 32.64186096191406,
      "activations/layer9_attention_weight_min": -34.92537307739258,
      "epoch": 22.36,
      "learning_rate": 5.85151515151515e-06,
      "loss": 2.6844,
      "step": 384850
    },
    {
      "activations/layer0_attention_weight_max": 16.706396102905273,
      "activations/layer0_attention_weight_min": -13.400248527526855,
      "activations/layer10_attention_weight_max": 33.664588928222656,
      "activations/layer10_attention_weight_min": -35.00086212158203,
      "activations/layer11_attention_weight_max": 33.57533264160156,
      "activations/layer11_attention_weight_min": -35.83978271484375,
      "activations/layer12_attention_weight_max": 27.6422176361084,
      "activations/layer12_attention_weight_min": -26.885149002075195,
      "activations/layer13_attention_weight_max": 40.25795364379883,
      "activations/layer13_attention_weight_min": -37.06620788574219,
      "activations/layer14_attention_weight_max": 43.382957458496094,
      "activations/layer14_attention_weight_min": -37.70219421386719,
      "activations/layer15_attention_weight_max": 39.75033187866211,
      "activations/layer15_attention_weight_min": -39.3966178894043,
      "activations/layer16_attention_weight_max": 29.113967895507812,
      "activations/layer16_attention_weight_min": -29.340227127075195,
      "activations/layer17_attention_weight_max": 30.847341537475586,
      "activations/layer17_attention_weight_min": -28.367809295654297,
      "activations/layer18_attention_weight_max": 31.509695053100586,
      "activations/layer18_attention_weight_min": -28.634502410888672,
      "activations/layer19_attention_weight_max": 33.30072784423828,
      "activations/layer19_attention_weight_min": -32.6092643737793,
      "activations/layer1_attention_weight_max": 16.266407012939453,
      "activations/layer1_attention_weight_min": -15.178411483764648,
      "activations/layer20_attention_weight_max": 27.812822341918945,
      "activations/layer20_attention_weight_min": -26.579692840576172,
      "activations/layer21_attention_weight_max": 29.22903823852539,
      "activations/layer21_attention_weight_min": -26.204336166381836,
      "activations/layer22_attention_weight_max": 43.16253662109375,
      "activations/layer22_attention_weight_min": -33.497249603271484,
      "activations/layer23_attention_weight_max": 34.27272033691406,
      "activations/layer23_attention_weight_min": -28.798629760742188,
      "activations/layer2_attention_weight_max": 34.575782775878906,
      "activations/layer2_attention_weight_min": -32.85585021972656,
      "activations/layer3_attention_weight_max": 104.60797882080078,
      "activations/layer3_attention_weight_min": -107.87271881103516,
      "activations/layer4_attention_weight_max": 107.69963836669922,
      "activations/layer4_attention_weight_min": -114.46417236328125,
      "activations/layer5_attention_weight_max": 51.98612976074219,
      "activations/layer5_attention_weight_min": -63.38392639160156,
      "activations/layer6_attention_weight_max": 45.9652214050293,
      "activations/layer6_attention_weight_min": -48.94803237915039,
      "activations/layer7_attention_weight_max": 94.88565826416016,
      "activations/layer7_attention_weight_min": -98.2916259765625,
      "activations/layer8_attention_weight_max": 41.5261344909668,
      "activations/layer8_attention_weight_min": -42.99301528930664,
      "activations/layer9_attention_weight_max": 31.588979721069336,
      "activations/layer9_attention_weight_min": -33.82497787475586,
      "epoch": 22.36,
      "learning_rate": 5.832575757575757e-06,
      "loss": 2.6985,
      "step": 384900
    },
    {
      "activations/layer0_attention_weight_max": 17.297346115112305,
      "activations/layer0_attention_weight_min": -13.52059268951416,
      "activations/layer10_attention_weight_max": 36.890811920166016,
      "activations/layer10_attention_weight_min": -37.52907180786133,
      "activations/layer11_attention_weight_max": 37.85566711425781,
      "activations/layer11_attention_weight_min": -35.1585578918457,
      "activations/layer12_attention_weight_max": 28.439512252807617,
      "activations/layer12_attention_weight_min": -26.420536041259766,
      "activations/layer13_attention_weight_max": 39.58552551269531,
      "activations/layer13_attention_weight_min": -35.39344024658203,
      "activations/layer14_attention_weight_max": 41.396018981933594,
      "activations/layer14_attention_weight_min": -37.556732177734375,
      "activations/layer15_attention_weight_max": 39.12394714355469,
      "activations/layer15_attention_weight_min": -36.60806655883789,
      "activations/layer16_attention_weight_max": 33.25171661376953,
      "activations/layer16_attention_weight_min": -28.353723526000977,
      "activations/layer17_attention_weight_max": 31.062435150146484,
      "activations/layer17_attention_weight_min": -26.882524490356445,
      "activations/layer18_attention_weight_max": 28.7768497467041,
      "activations/layer18_attention_weight_min": -25.65411949157715,
      "activations/layer19_attention_weight_max": 33.34585189819336,
      "activations/layer19_attention_weight_min": -31.235742568969727,
      "activations/layer1_attention_weight_max": 15.74229907989502,
      "activations/layer1_attention_weight_min": -15.970610618591309,
      "activations/layer20_attention_weight_max": 26.883888244628906,
      "activations/layer20_attention_weight_min": -25.637983322143555,
      "activations/layer21_attention_weight_max": 27.8973388671875,
      "activations/layer21_attention_weight_min": -24.485427856445312,
      "activations/layer22_attention_weight_max": 39.409446716308594,
      "activations/layer22_attention_weight_min": -30.19647216796875,
      "activations/layer23_attention_weight_max": 30.919708251953125,
      "activations/layer23_attention_weight_min": -27.45655632019043,
      "activations/layer2_attention_weight_max": 34.82044982910156,
      "activations/layer2_attention_weight_min": -34.15967559814453,
      "activations/layer3_attention_weight_max": 113.97286987304688,
      "activations/layer3_attention_weight_min": -109.97587585449219,
      "activations/layer4_attention_weight_max": 113.67247772216797,
      "activations/layer4_attention_weight_min": -117.24681854248047,
      "activations/layer5_attention_weight_max": 53.77919006347656,
      "activations/layer5_attention_weight_min": -62.7489013671875,
      "activations/layer6_attention_weight_max": 47.6088752746582,
      "activations/layer6_attention_weight_min": -53.00032424926758,
      "activations/layer7_attention_weight_max": 96.77595520019531,
      "activations/layer7_attention_weight_min": -114.35486602783203,
      "activations/layer8_attention_weight_max": 43.14213180541992,
      "activations/layer8_attention_weight_min": -49.02142333984375,
      "activations/layer9_attention_weight_max": 33.93656539916992,
      "activations/layer9_attention_weight_min": -36.2961540222168,
      "epoch": 22.37,
      "learning_rate": 5.813636363636363e-06,
      "loss": 2.6889,
      "step": 384950
    },
    {
      "activations/layer0_attention_weight_max": 16.682615280151367,
      "activations/layer0_attention_weight_min": -14.05173110961914,
      "activations/layer10_attention_weight_max": 34.2764778137207,
      "activations/layer10_attention_weight_min": -34.55892562866211,
      "activations/layer11_attention_weight_max": 34.52680587768555,
      "activations/layer11_attention_weight_min": -34.55143737792969,
      "activations/layer12_attention_weight_max": 28.57615852355957,
      "activations/layer12_attention_weight_min": -27.70291519165039,
      "activations/layer13_attention_weight_max": 39.209434509277344,
      "activations/layer13_attention_weight_min": -35.99079513549805,
      "activations/layer14_attention_weight_max": 43.84209442138672,
      "activations/layer14_attention_weight_min": -38.385799407958984,
      "activations/layer15_attention_weight_max": 41.956703186035156,
      "activations/layer15_attention_weight_min": -37.664241790771484,
      "activations/layer16_attention_weight_max": 31.535877227783203,
      "activations/layer16_attention_weight_min": -28.650981903076172,
      "activations/layer17_attention_weight_max": 31.071537017822266,
      "activations/layer17_attention_weight_min": -27.549158096313477,
      "activations/layer18_attention_weight_max": 37.259464263916016,
      "activations/layer18_attention_weight_min": -27.97736167907715,
      "activations/layer19_attention_weight_max": 38.38957214355469,
      "activations/layer19_attention_weight_min": -34.26688766479492,
      "activations/layer1_attention_weight_max": 15.4639310836792,
      "activations/layer1_attention_weight_min": -16.352983474731445,
      "activations/layer20_attention_weight_max": 30.367036819458008,
      "activations/layer20_attention_weight_min": -27.339170455932617,
      "activations/layer21_attention_weight_max": 31.868648529052734,
      "activations/layer21_attention_weight_min": -28.528554916381836,
      "activations/layer22_attention_weight_max": 43.80051803588867,
      "activations/layer22_attention_weight_min": -35.59867477416992,
      "activations/layer23_attention_weight_max": 35.832984924316406,
      "activations/layer23_attention_weight_min": -27.134679794311523,
      "activations/layer2_attention_weight_max": 34.00336837768555,
      "activations/layer2_attention_weight_min": -33.033111572265625,
      "activations/layer3_attention_weight_max": 108.71895599365234,
      "activations/layer3_attention_weight_min": -107.46293640136719,
      "activations/layer4_attention_weight_max": 112.02589416503906,
      "activations/layer4_attention_weight_min": -113.54705047607422,
      "activations/layer5_attention_weight_max": 49.32473373413086,
      "activations/layer5_attention_weight_min": -62.679718017578125,
      "activations/layer6_attention_weight_max": 45.5532112121582,
      "activations/layer6_attention_weight_min": -47.55124282836914,
      "activations/layer7_attention_weight_max": 96.94229125976562,
      "activations/layer7_attention_weight_min": -96.89556884765625,
      "activations/layer8_attention_weight_max": 42.22323989868164,
      "activations/layer8_attention_weight_min": -45.10533142089844,
      "activations/layer9_attention_weight_max": 32.07265853881836,
      "activations/layer9_attention_weight_min": -34.294307708740234,
      "epoch": 22.37,
      "learning_rate": 5.795075757575757e-06,
      "loss": 2.7099,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4362,
      "eval_samples_per_second": 508.996,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4362,
      "eval_openwebtext_samples_per_second": 508.996,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9241,
      "eval_wikitext_samples_per_second": 236.995,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_lambada_loss": 2.29296875,
      "eval_lambada_ppl": 9.904297461775021,
      "eval_lambada_runtime": 9.5288,
      "eval_lambada_samples_per_second": 510.977,
      "step": 385000
    },
    {
      "activations/layer0_attention_weight_max": 17.036975860595703,
      "activations/layer0_attention_weight_min": -13.733798027038574,
      "activations/layer10_attention_weight_max": 34.97359848022461,
      "activations/layer10_attention_weight_min": -35.335060119628906,
      "activations/layer11_attention_weight_max": 33.8114013671875,
      "activations/layer11_attention_weight_min": -36.92559814453125,
      "activations/layer12_attention_weight_max": 27.01163673400879,
      "activations/layer12_attention_weight_min": -29.118362426757812,
      "activations/layer13_attention_weight_max": 40.98591232299805,
      "activations/layer13_attention_weight_min": -35.56418991088867,
      "activations/layer14_attention_weight_max": 43.67598342895508,
      "activations/layer14_attention_weight_min": -37.604515075683594,
      "activations/layer15_attention_weight_max": 40.21206283569336,
      "activations/layer15_attention_weight_min": -36.808616638183594,
      "activations/layer16_attention_weight_max": 29.954896926879883,
      "activations/layer16_attention_weight_min": -27.510866165161133,
      "activations/layer17_attention_weight_max": 28.989044189453125,
      "activations/layer17_attention_weight_min": -26.78392219543457,
      "activations/layer18_attention_weight_max": 35.31277847290039,
      "activations/layer18_attention_weight_min": -29.5169734954834,
      "activations/layer19_attention_weight_max": 32.117523193359375,
      "activations/layer19_attention_weight_min": -33.67251205444336,
      "activations/layer1_attention_weight_max": 17.562320709228516,
      "activations/layer1_attention_weight_min": -15.129318237304688,
      "activations/layer20_attention_weight_max": 28.3984375,
      "activations/layer20_attention_weight_min": -26.29690170288086,
      "activations/layer21_attention_weight_max": 26.57823371887207,
      "activations/layer21_attention_weight_min": -25.229063034057617,
      "activations/layer22_attention_weight_max": 40.440673828125,
      "activations/layer22_attention_weight_min": -31.468469619750977,
      "activations/layer23_attention_weight_max": 31.603452682495117,
      "activations/layer23_attention_weight_min": -25.014686584472656,
      "activations/layer2_attention_weight_max": 34.39740753173828,
      "activations/layer2_attention_weight_min": -33.274559020996094,
      "activations/layer3_attention_weight_max": 108.27758026123047,
      "activations/layer3_attention_weight_min": -108.5897445678711,
      "activations/layer4_attention_weight_max": 112.5347671508789,
      "activations/layer4_attention_weight_min": -112.63851165771484,
      "activations/layer5_attention_weight_max": 51.66952133178711,
      "activations/layer5_attention_weight_min": -59.51701354980469,
      "activations/layer6_attention_weight_max": 47.40727615356445,
      "activations/layer6_attention_weight_min": -50.84767150878906,
      "activations/layer7_attention_weight_max": 100.97637176513672,
      "activations/layer7_attention_weight_min": -104.26795959472656,
      "activations/layer8_attention_weight_max": 46.731014251708984,
      "activations/layer8_attention_weight_min": -46.39070129394531,
      "activations/layer9_attention_weight_max": 33.097843170166016,
      "activations/layer9_attention_weight_min": -35.737606048583984,
      "epoch": 22.37,
      "learning_rate": 5.776136363636363e-06,
      "loss": 2.7015,
      "step": 385050
    },
    {
      "activations/layer0_attention_weight_max": 16.29700469970703,
      "activations/layer0_attention_weight_min": -14.189810752868652,
      "activations/layer10_attention_weight_max": 36.39800262451172,
      "activations/layer10_attention_weight_min": -36.70967483520508,
      "activations/layer11_attention_weight_max": 34.625099182128906,
      "activations/layer11_attention_weight_min": -35.63761901855469,
      "activations/layer12_attention_weight_max": 27.362470626831055,
      "activations/layer12_attention_weight_min": -27.283313751220703,
      "activations/layer13_attention_weight_max": 38.78193283081055,
      "activations/layer13_attention_weight_min": -36.08083724975586,
      "activations/layer14_attention_weight_max": 42.5088005065918,
      "activations/layer14_attention_weight_min": -38.682945251464844,
      "activations/layer15_attention_weight_max": 38.456233978271484,
      "activations/layer15_attention_weight_min": -37.620750427246094,
      "activations/layer16_attention_weight_max": 31.961605072021484,
      "activations/layer16_attention_weight_min": -28.482845306396484,
      "activations/layer17_attention_weight_max": 28.031328201293945,
      "activations/layer17_attention_weight_min": -25.83400535583496,
      "activations/layer18_attention_weight_max": 32.65031433105469,
      "activations/layer18_attention_weight_min": -28.229267120361328,
      "activations/layer19_attention_weight_max": 37.47396469116211,
      "activations/layer19_attention_weight_min": -32.234642028808594,
      "activations/layer1_attention_weight_max": 16.03659439086914,
      "activations/layer1_attention_weight_min": -15.530936241149902,
      "activations/layer20_attention_weight_max": 30.213594436645508,
      "activations/layer20_attention_weight_min": -26.085081100463867,
      "activations/layer21_attention_weight_max": 31.534154891967773,
      "activations/layer21_attention_weight_min": -24.431926727294922,
      "activations/layer22_attention_weight_max": 44.04911422729492,
      "activations/layer22_attention_weight_min": -31.97160530090332,
      "activations/layer23_attention_weight_max": 31.187437057495117,
      "activations/layer23_attention_weight_min": -25.335325241088867,
      "activations/layer2_attention_weight_max": 33.52397537231445,
      "activations/layer2_attention_weight_min": -32.48255920410156,
      "activations/layer3_attention_weight_max": 107.6781997680664,
      "activations/layer3_attention_weight_min": -108.02975463867188,
      "activations/layer4_attention_weight_max": 111.05367279052734,
      "activations/layer4_attention_weight_min": -114.74714660644531,
      "activations/layer5_attention_weight_max": 50.84908676147461,
      "activations/layer5_attention_weight_min": -63.588314056396484,
      "activations/layer6_attention_weight_max": 46.65690612792969,
      "activations/layer6_attention_weight_min": -48.832191467285156,
      "activations/layer7_attention_weight_max": 103.71501159667969,
      "activations/layer7_attention_weight_min": -102.66046905517578,
      "activations/layer8_attention_weight_max": 43.9339599609375,
      "activations/layer8_attention_weight_min": -45.5059928894043,
      "activations/layer9_attention_weight_max": 32.96274948120117,
      "activations/layer9_attention_weight_min": -36.525081634521484,
      "epoch": 22.38,
      "learning_rate": 5.757196969696969e-06,
      "loss": 2.6965,
      "step": 385100
    },
    {
      "activations/layer0_attention_weight_max": 17.53759002685547,
      "activations/layer0_attention_weight_min": -13.889739036560059,
      "activations/layer10_attention_weight_max": 35.321868896484375,
      "activations/layer10_attention_weight_min": -35.0833740234375,
      "activations/layer11_attention_weight_max": 35.26498031616211,
      "activations/layer11_attention_weight_min": -36.96560287475586,
      "activations/layer12_attention_weight_max": 28.982328414916992,
      "activations/layer12_attention_weight_min": -29.889209747314453,
      "activations/layer13_attention_weight_max": 42.65437316894531,
      "activations/layer13_attention_weight_min": -40.00077438354492,
      "activations/layer14_attention_weight_max": 43.78119659423828,
      "activations/layer14_attention_weight_min": -41.28611755371094,
      "activations/layer15_attention_weight_max": 40.23369598388672,
      "activations/layer15_attention_weight_min": -38.47343826293945,
      "activations/layer16_attention_weight_max": 29.46552276611328,
      "activations/layer16_attention_weight_min": -29.959548950195312,
      "activations/layer17_attention_weight_max": 30.340173721313477,
      "activations/layer17_attention_weight_min": -30.05897331237793,
      "activations/layer18_attention_weight_max": 35.02961349487305,
      "activations/layer18_attention_weight_min": -28.491920471191406,
      "activations/layer19_attention_weight_max": 32.942806243896484,
      "activations/layer19_attention_weight_min": -33.08744812011719,
      "activations/layer1_attention_weight_max": 14.886584281921387,
      "activations/layer1_attention_weight_min": -15.51695442199707,
      "activations/layer20_attention_weight_max": 28.941648483276367,
      "activations/layer20_attention_weight_min": -26.673255920410156,
      "activations/layer21_attention_weight_max": 28.622831344604492,
      "activations/layer21_attention_weight_min": -26.150781631469727,
      "activations/layer22_attention_weight_max": 41.903221130371094,
      "activations/layer22_attention_weight_min": -33.361446380615234,
      "activations/layer23_attention_weight_max": 35.458580017089844,
      "activations/layer23_attention_weight_min": -27.20265007019043,
      "activations/layer2_attention_weight_max": 33.99375534057617,
      "activations/layer2_attention_weight_min": -33.9229736328125,
      "activations/layer3_attention_weight_max": 111.7380142211914,
      "activations/layer3_attention_weight_min": -110.59954071044922,
      "activations/layer4_attention_weight_max": 117.32059478759766,
      "activations/layer4_attention_weight_min": -118.36616516113281,
      "activations/layer5_attention_weight_max": 53.28791427612305,
      "activations/layer5_attention_weight_min": -63.745361328125,
      "activations/layer6_attention_weight_max": 50.4254264831543,
      "activations/layer6_attention_weight_min": -52.84761047363281,
      "activations/layer7_attention_weight_max": 107.10835266113281,
      "activations/layer7_attention_weight_min": -105.52592468261719,
      "activations/layer8_attention_weight_max": 44.90860366821289,
      "activations/layer8_attention_weight_min": -48.39677810668945,
      "activations/layer9_attention_weight_max": 35.04733657836914,
      "activations/layer9_attention_weight_min": -36.2677116394043,
      "epoch": 22.38,
      "learning_rate": 5.738257575757576e-06,
      "loss": 2.7064,
      "step": 385150
    },
    {
      "activations/layer0_attention_weight_max": 17.373973846435547,
      "activations/layer0_attention_weight_min": -13.486923217773438,
      "activations/layer10_attention_weight_max": 38.38383102416992,
      "activations/layer10_attention_weight_min": -38.33028030395508,
      "activations/layer11_attention_weight_max": 37.543678283691406,
      "activations/layer11_attention_weight_min": -40.11235046386719,
      "activations/layer12_attention_weight_max": 28.94442367553711,
      "activations/layer12_attention_weight_min": -28.396156311035156,
      "activations/layer13_attention_weight_max": 41.86601257324219,
      "activations/layer13_attention_weight_min": -38.44426345825195,
      "activations/layer14_attention_weight_max": 45.896488189697266,
      "activations/layer14_attention_weight_min": -39.822967529296875,
      "activations/layer15_attention_weight_max": 41.36268997192383,
      "activations/layer15_attention_weight_min": -36.0283203125,
      "activations/layer16_attention_weight_max": 30.56645965576172,
      "activations/layer16_attention_weight_min": -28.732311248779297,
      "activations/layer17_attention_weight_max": 32.764225006103516,
      "activations/layer17_attention_weight_min": -28.95045280456543,
      "activations/layer18_attention_weight_max": 34.67765426635742,
      "activations/layer18_attention_weight_min": -27.386384963989258,
      "activations/layer19_attention_weight_max": 36.735618591308594,
      "activations/layer19_attention_weight_min": -32.139930725097656,
      "activations/layer1_attention_weight_max": 15.576128959655762,
      "activations/layer1_attention_weight_min": -15.68692684173584,
      "activations/layer20_attention_weight_max": 31.7888126373291,
      "activations/layer20_attention_weight_min": -26.682329177856445,
      "activations/layer21_attention_weight_max": 32.10526657104492,
      "activations/layer21_attention_weight_min": -25.172266006469727,
      "activations/layer22_attention_weight_max": 45.857730865478516,
      "activations/layer22_attention_weight_min": -33.408382415771484,
      "activations/layer23_attention_weight_max": 35.44843292236328,
      "activations/layer23_attention_weight_min": -27.348857879638672,
      "activations/layer2_attention_weight_max": 33.49943923950195,
      "activations/layer2_attention_weight_min": -35.218780517578125,
      "activations/layer3_attention_weight_max": 108.38442993164062,
      "activations/layer3_attention_weight_min": -116.77043914794922,
      "activations/layer4_attention_weight_max": 117.68395233154297,
      "activations/layer4_attention_weight_min": -118.60147857666016,
      "activations/layer5_attention_weight_max": 55.090850830078125,
      "activations/layer5_attention_weight_min": -64.4168701171875,
      "activations/layer6_attention_weight_max": 49.07535934448242,
      "activations/layer6_attention_weight_min": -53.21018600463867,
      "activations/layer7_attention_weight_max": 104.09294128417969,
      "activations/layer7_attention_weight_min": -119.97905731201172,
      "activations/layer8_attention_weight_max": 46.245147705078125,
      "activations/layer8_attention_weight_min": -52.05415344238281,
      "activations/layer9_attention_weight_max": 34.85939025878906,
      "activations/layer9_attention_weight_min": -38.93793869018555,
      "epoch": 22.38,
      "learning_rate": 5.7193181818181815e-06,
      "loss": 2.6944,
      "step": 385200
    },
    {
      "activations/layer0_attention_weight_max": 17.497154235839844,
      "activations/layer0_attention_weight_min": -14.674433708190918,
      "activations/layer10_attention_weight_max": 36.22969055175781,
      "activations/layer10_attention_weight_min": -39.58213806152344,
      "activations/layer11_attention_weight_max": 35.536441802978516,
      "activations/layer11_attention_weight_min": -38.41517639160156,
      "activations/layer12_attention_weight_max": 27.422866821289062,
      "activations/layer12_attention_weight_min": -29.124996185302734,
      "activations/layer13_attention_weight_max": 38.765777587890625,
      "activations/layer13_attention_weight_min": -38.86311721801758,
      "activations/layer14_attention_weight_max": 40.91164016723633,
      "activations/layer14_attention_weight_min": -41.500465393066406,
      "activations/layer15_attention_weight_max": 38.84546661376953,
      "activations/layer15_attention_weight_min": -40.803592681884766,
      "activations/layer16_attention_weight_max": 28.693897247314453,
      "activations/layer16_attention_weight_min": -29.571170806884766,
      "activations/layer17_attention_weight_max": 31.42941665649414,
      "activations/layer17_attention_weight_min": -28.0758056640625,
      "activations/layer18_attention_weight_max": 34.216392517089844,
      "activations/layer18_attention_weight_min": -27.4150333404541,
      "activations/layer19_attention_weight_max": 35.77170181274414,
      "activations/layer19_attention_weight_min": -32.51180648803711,
      "activations/layer1_attention_weight_max": 15.85622501373291,
      "activations/layer1_attention_weight_min": -15.70676326751709,
      "activations/layer20_attention_weight_max": 29.89061164855957,
      "activations/layer20_attention_weight_min": -27.7074031829834,
      "activations/layer21_attention_weight_max": 28.912559509277344,
      "activations/layer21_attention_weight_min": -26.30126190185547,
      "activations/layer22_attention_weight_max": 42.30210494995117,
      "activations/layer22_attention_weight_min": -33.8508186340332,
      "activations/layer23_attention_weight_max": 34.66755676269531,
      "activations/layer23_attention_weight_min": -28.885799407958984,
      "activations/layer2_attention_weight_max": 35.408992767333984,
      "activations/layer2_attention_weight_min": -33.37993621826172,
      "activations/layer3_attention_weight_max": 108.33544921875,
      "activations/layer3_attention_weight_min": -109.22882843017578,
      "activations/layer4_attention_weight_max": 114.7767333984375,
      "activations/layer4_attention_weight_min": -115.05016326904297,
      "activations/layer5_attention_weight_max": 55.17500305175781,
      "activations/layer5_attention_weight_min": -59.996761322021484,
      "activations/layer6_attention_weight_max": 47.950828552246094,
      "activations/layer6_attention_weight_min": -51.60825729370117,
      "activations/layer7_attention_weight_max": 98.01775360107422,
      "activations/layer7_attention_weight_min": -104.99934387207031,
      "activations/layer8_attention_weight_max": 43.21022415161133,
      "activations/layer8_attention_weight_min": -46.883541107177734,
      "activations/layer9_attention_weight_max": 33.216209411621094,
      "activations/layer9_attention_weight_min": -37.076637268066406,
      "epoch": 22.39,
      "learning_rate": 5.700378787878787e-06,
      "loss": 2.7016,
      "step": 385250
    },
    {
      "activations/layer0_attention_weight_max": 16.076885223388672,
      "activations/layer0_attention_weight_min": -14.85421085357666,
      "activations/layer10_attention_weight_max": 35.29299545288086,
      "activations/layer10_attention_weight_min": -34.42927932739258,
      "activations/layer11_attention_weight_max": 34.688385009765625,
      "activations/layer11_attention_weight_min": -35.75330352783203,
      "activations/layer12_attention_weight_max": 27.166261672973633,
      "activations/layer12_attention_weight_min": -26.087228775024414,
      "activations/layer13_attention_weight_max": 38.74599075317383,
      "activations/layer13_attention_weight_min": -35.74367141723633,
      "activations/layer14_attention_weight_max": 42.49882888793945,
      "activations/layer14_attention_weight_min": -38.57843780517578,
      "activations/layer15_attention_weight_max": 40.194820404052734,
      "activations/layer15_attention_weight_min": -35.878421783447266,
      "activations/layer16_attention_weight_max": 28.344011306762695,
      "activations/layer16_attention_weight_min": -28.318878173828125,
      "activations/layer17_attention_weight_max": 29.234872817993164,
      "activations/layer17_attention_weight_min": -24.947723388671875,
      "activations/layer18_attention_weight_max": 32.40883255004883,
      "activations/layer18_attention_weight_min": -25.477190017700195,
      "activations/layer19_attention_weight_max": 34.43408966064453,
      "activations/layer19_attention_weight_min": -33.13344955444336,
      "activations/layer1_attention_weight_max": 15.653346061706543,
      "activations/layer1_attention_weight_min": -14.735223770141602,
      "activations/layer20_attention_weight_max": 26.464637756347656,
      "activations/layer20_attention_weight_min": -24.25461196899414,
      "activations/layer21_attention_weight_max": 28.319631576538086,
      "activations/layer21_attention_weight_min": -24.116491317749023,
      "activations/layer22_attention_weight_max": 39.75343322753906,
      "activations/layer22_attention_weight_min": -32.853275299072266,
      "activations/layer23_attention_weight_max": 33.80549621582031,
      "activations/layer23_attention_weight_min": -28.053449630737305,
      "activations/layer2_attention_weight_max": 32.78145980834961,
      "activations/layer2_attention_weight_min": -33.617408752441406,
      "activations/layer3_attention_weight_max": 106.55390167236328,
      "activations/layer3_attention_weight_min": -108.06320190429688,
      "activations/layer4_attention_weight_max": 110.06963348388672,
      "activations/layer4_attention_weight_min": -110.84073638916016,
      "activations/layer5_attention_weight_max": 49.76282501220703,
      "activations/layer5_attention_weight_min": -61.192134857177734,
      "activations/layer6_attention_weight_max": 45.45932388305664,
      "activations/layer6_attention_weight_min": -47.97493362426758,
      "activations/layer7_attention_weight_max": 97.59245300292969,
      "activations/layer7_attention_weight_min": -101.69479370117188,
      "activations/layer8_attention_weight_max": 43.55337142944336,
      "activations/layer8_attention_weight_min": -46.260223388671875,
      "activations/layer9_attention_weight_max": 32.3674201965332,
      "activations/layer9_attention_weight_min": -35.060855865478516,
      "epoch": 22.39,
      "learning_rate": 5.681439393939393e-06,
      "loss": 2.7064,
      "step": 385300
    },
    {
      "activations/layer0_attention_weight_max": 16.70551872253418,
      "activations/layer0_attention_weight_min": -13.950945854187012,
      "activations/layer10_attention_weight_max": 38.0557975769043,
      "activations/layer10_attention_weight_min": -38.20375442504883,
      "activations/layer11_attention_weight_max": 38.487762451171875,
      "activations/layer11_attention_weight_min": -38.94049835205078,
      "activations/layer12_attention_weight_max": 28.918582916259766,
      "activations/layer12_attention_weight_min": -28.103574752807617,
      "activations/layer13_attention_weight_max": 42.047454833984375,
      "activations/layer13_attention_weight_min": -39.71043014526367,
      "activations/layer14_attention_weight_max": 43.99843215942383,
      "activations/layer14_attention_weight_min": -42.767581939697266,
      "activations/layer15_attention_weight_max": 42.80216979980469,
      "activations/layer15_attention_weight_min": -42.16640090942383,
      "activations/layer16_attention_weight_max": 29.266830444335938,
      "activations/layer16_attention_weight_min": -29.32231903076172,
      "activations/layer17_attention_weight_max": 29.161054611206055,
      "activations/layer17_attention_weight_min": -26.088871002197266,
      "activations/layer18_attention_weight_max": 34.923702239990234,
      "activations/layer18_attention_weight_min": -26.26779556274414,
      "activations/layer19_attention_weight_max": 39.14265823364258,
      "activations/layer19_attention_weight_min": -32.58308029174805,
      "activations/layer1_attention_weight_max": 15.867069244384766,
      "activations/layer1_attention_weight_min": -14.9893217086792,
      "activations/layer20_attention_weight_max": 30.62367820739746,
      "activations/layer20_attention_weight_min": -25.223127365112305,
      "activations/layer21_attention_weight_max": 32.33155059814453,
      "activations/layer21_attention_weight_min": -25.135868072509766,
      "activations/layer22_attention_weight_max": 41.775657653808594,
      "activations/layer22_attention_weight_min": -32.470237731933594,
      "activations/layer23_attention_weight_max": 30.593280792236328,
      "activations/layer23_attention_weight_min": -25.770076751708984,
      "activations/layer2_attention_weight_max": 34.86430358886719,
      "activations/layer2_attention_weight_min": -34.679351806640625,
      "activations/layer3_attention_weight_max": 109.22002410888672,
      "activations/layer3_attention_weight_min": -111.0783920288086,
      "activations/layer4_attention_weight_max": 113.27371978759766,
      "activations/layer4_attention_weight_min": -122.6703109741211,
      "activations/layer5_attention_weight_max": 52.87453842163086,
      "activations/layer5_attention_weight_min": -63.52683639526367,
      "activations/layer6_attention_weight_max": 49.609622955322266,
      "activations/layer6_attention_weight_min": -52.22441482543945,
      "activations/layer7_attention_weight_max": 108.85740661621094,
      "activations/layer7_attention_weight_min": -113.25360107421875,
      "activations/layer8_attention_weight_max": 48.01180648803711,
      "activations/layer8_attention_weight_min": -50.96482467651367,
      "activations/layer9_attention_weight_max": 36.89159393310547,
      "activations/layer9_attention_weight_min": -39.73660659790039,
      "epoch": 22.39,
      "learning_rate": 5.6624999999999996e-06,
      "loss": 2.6983,
      "step": 385350
    },
    {
      "activations/layer0_attention_weight_max": 18.436763763427734,
      "activations/layer0_attention_weight_min": -12.902634620666504,
      "activations/layer10_attention_weight_max": 38.39226531982422,
      "activations/layer10_attention_weight_min": -39.022701263427734,
      "activations/layer11_attention_weight_max": 38.34782028198242,
      "activations/layer11_attention_weight_min": -39.47629928588867,
      "activations/layer12_attention_weight_max": 31.53152084350586,
      "activations/layer12_attention_weight_min": -29.555591583251953,
      "activations/layer13_attention_weight_max": 44.380882263183594,
      "activations/layer13_attention_weight_min": -39.71277618408203,
      "activations/layer14_attention_weight_max": 44.67561721801758,
      "activations/layer14_attention_weight_min": -42.16937255859375,
      "activations/layer15_attention_weight_max": 40.88485336303711,
      "activations/layer15_attention_weight_min": -37.057125091552734,
      "activations/layer16_attention_weight_max": 33.74761962890625,
      "activations/layer16_attention_weight_min": -29.214637756347656,
      "activations/layer17_attention_weight_max": 31.12924575805664,
      "activations/layer17_attention_weight_min": -25.870378494262695,
      "activations/layer18_attention_weight_max": 31.859054565429688,
      "activations/layer18_attention_weight_min": -26.103364944458008,
      "activations/layer19_attention_weight_max": 39.20318603515625,
      "activations/layer19_attention_weight_min": -33.682159423828125,
      "activations/layer1_attention_weight_max": 16.048513412475586,
      "activations/layer1_attention_weight_min": -15.872891426086426,
      "activations/layer20_attention_weight_max": 32.02021789550781,
      "activations/layer20_attention_weight_min": -25.55413246154785,
      "activations/layer21_attention_weight_max": 28.85359764099121,
      "activations/layer21_attention_weight_min": -26.088577270507812,
      "activations/layer22_attention_weight_max": 40.90969467163086,
      "activations/layer22_attention_weight_min": -33.74436569213867,
      "activations/layer23_attention_weight_max": 33.195186614990234,
      "activations/layer23_attention_weight_min": -26.84395980834961,
      "activations/layer2_attention_weight_max": 34.980220794677734,
      "activations/layer2_attention_weight_min": -33.668479919433594,
      "activations/layer3_attention_weight_max": 112.4454345703125,
      "activations/layer3_attention_weight_min": -109.69861602783203,
      "activations/layer4_attention_weight_max": 114.17073822021484,
      "activations/layer4_attention_weight_min": -114.58159637451172,
      "activations/layer5_attention_weight_max": 51.54582214355469,
      "activations/layer5_attention_weight_min": -62.36931610107422,
      "activations/layer6_attention_weight_max": 46.46309280395508,
      "activations/layer6_attention_weight_min": -49.10010528564453,
      "activations/layer7_attention_weight_max": 102.69403839111328,
      "activations/layer7_attention_weight_min": -107.57073211669922,
      "activations/layer8_attention_weight_max": 46.1610107421875,
      "activations/layer8_attention_weight_min": -50.31222915649414,
      "activations/layer9_attention_weight_max": 34.33621597290039,
      "activations/layer9_attention_weight_min": -38.75053024291992,
      "epoch": 22.39,
      "learning_rate": 5.643560606060605e-06,
      "loss": 2.7066,
      "step": 385400
    },
    {
      "activations/layer0_attention_weight_max": 16.708276748657227,
      "activations/layer0_attention_weight_min": -14.289729118347168,
      "activations/layer10_attention_weight_max": 39.98579025268555,
      "activations/layer10_attention_weight_min": -37.11442184448242,
      "activations/layer11_attention_weight_max": 37.9442138671875,
      "activations/layer11_attention_weight_min": -37.46846389770508,
      "activations/layer12_attention_weight_max": 28.37659454345703,
      "activations/layer12_attention_weight_min": -27.92367935180664,
      "activations/layer13_attention_weight_max": 42.59293746948242,
      "activations/layer13_attention_weight_min": -37.97632598876953,
      "activations/layer14_attention_weight_max": 44.40724563598633,
      "activations/layer14_attention_weight_min": -41.826866149902344,
      "activations/layer15_attention_weight_max": 40.25541687011719,
      "activations/layer15_attention_weight_min": -40.11112976074219,
      "activations/layer16_attention_weight_max": 30.19818878173828,
      "activations/layer16_attention_weight_min": -28.440492630004883,
      "activations/layer17_attention_weight_max": 29.154632568359375,
      "activations/layer17_attention_weight_min": -28.35569190979004,
      "activations/layer18_attention_weight_max": 31.582721710205078,
      "activations/layer18_attention_weight_min": -26.93111801147461,
      "activations/layer19_attention_weight_max": 33.78525924682617,
      "activations/layer19_attention_weight_min": -32.62163543701172,
      "activations/layer1_attention_weight_max": 15.046460151672363,
      "activations/layer1_attention_weight_min": -15.409646987915039,
      "activations/layer20_attention_weight_max": 28.15446662902832,
      "activations/layer20_attention_weight_min": -24.404672622680664,
      "activations/layer21_attention_weight_max": 32.66813659667969,
      "activations/layer21_attention_weight_min": -25.29422378540039,
      "activations/layer22_attention_weight_max": 39.72190475463867,
      "activations/layer22_attention_weight_min": -34.39263153076172,
      "activations/layer23_attention_weight_max": 33.11817169189453,
      "activations/layer23_attention_weight_min": -28.612632751464844,
      "activations/layer2_attention_weight_max": 33.232269287109375,
      "activations/layer2_attention_weight_min": -34.16986846923828,
      "activations/layer3_attention_weight_max": 106.21504211425781,
      "activations/layer3_attention_weight_min": -113.16517639160156,
      "activations/layer4_attention_weight_max": 114.0796127319336,
      "activations/layer4_attention_weight_min": -120.91875457763672,
      "activations/layer5_attention_weight_max": 51.586334228515625,
      "activations/layer5_attention_weight_min": -60.603145599365234,
      "activations/layer6_attention_weight_max": 49.29599380493164,
      "activations/layer6_attention_weight_min": -53.10149002075195,
      "activations/layer7_attention_weight_max": 110.01510620117188,
      "activations/layer7_attention_weight_min": -113.06696319580078,
      "activations/layer8_attention_weight_max": 47.28708267211914,
      "activations/layer8_attention_weight_min": -49.08289337158203,
      "activations/layer9_attention_weight_max": 35.29269027709961,
      "activations/layer9_attention_weight_min": -38.65420913696289,
      "epoch": 22.4,
      "learning_rate": 5.624621212121211e-06,
      "loss": 2.6985,
      "step": 385450
    },
    {
      "activations/layer0_attention_weight_max": 17.287885665893555,
      "activations/layer0_attention_weight_min": -13.763077735900879,
      "activations/layer10_attention_weight_max": 38.4477424621582,
      "activations/layer10_attention_weight_min": -38.947792053222656,
      "activations/layer11_attention_weight_max": 35.30979919433594,
      "activations/layer11_attention_weight_min": -39.159996032714844,
      "activations/layer12_attention_weight_max": 27.917652130126953,
      "activations/layer12_attention_weight_min": -27.92595672607422,
      "activations/layer13_attention_weight_max": 41.16871643066406,
      "activations/layer13_attention_weight_min": -38.43500518798828,
      "activations/layer14_attention_weight_max": 46.98638916015625,
      "activations/layer14_attention_weight_min": -42.83408737182617,
      "activations/layer15_attention_weight_max": 43.7566032409668,
      "activations/layer15_attention_weight_min": -40.33941650390625,
      "activations/layer16_attention_weight_max": 32.64371109008789,
      "activations/layer16_attention_weight_min": -31.379133224487305,
      "activations/layer17_attention_weight_max": 33.55641174316406,
      "activations/layer17_attention_weight_min": -27.800931930541992,
      "activations/layer18_attention_weight_max": 37.160606384277344,
      "activations/layer18_attention_weight_min": -26.881669998168945,
      "activations/layer19_attention_weight_max": 42.59731674194336,
      "activations/layer19_attention_weight_min": -34.24184036254883,
      "activations/layer1_attention_weight_max": 15.387392044067383,
      "activations/layer1_attention_weight_min": -14.32323932647705,
      "activations/layer20_attention_weight_max": 34.47574234008789,
      "activations/layer20_attention_weight_min": -27.33732795715332,
      "activations/layer21_attention_weight_max": 35.13693618774414,
      "activations/layer21_attention_weight_min": -27.83877182006836,
      "activations/layer22_attention_weight_max": 56.9738883972168,
      "activations/layer22_attention_weight_min": -37.08620071411133,
      "activations/layer23_attention_weight_max": 44.348812103271484,
      "activations/layer23_attention_weight_min": -29.782546997070312,
      "activations/layer2_attention_weight_max": 35.81813049316406,
      "activations/layer2_attention_weight_min": -33.12716293334961,
      "activations/layer3_attention_weight_max": 107.84886932373047,
      "activations/layer3_attention_weight_min": -109.6477279663086,
      "activations/layer4_attention_weight_max": 113.37165832519531,
      "activations/layer4_attention_weight_min": -114.29741668701172,
      "activations/layer5_attention_weight_max": 53.73058319091797,
      "activations/layer5_attention_weight_min": -61.05659103393555,
      "activations/layer6_attention_weight_max": 46.986732482910156,
      "activations/layer6_attention_weight_min": -49.43959426879883,
      "activations/layer7_attention_weight_max": 102.72611236572266,
      "activations/layer7_attention_weight_min": -106.08464050292969,
      "activations/layer8_attention_weight_max": 45.0071907043457,
      "activations/layer8_attention_weight_min": -49.60015106201172,
      "activations/layer9_attention_weight_max": 33.035953521728516,
      "activations/layer9_attention_weight_min": -36.3338737487793,
      "epoch": 22.4,
      "learning_rate": 5.6056818181818185e-06,
      "loss": 2.6882,
      "step": 385500
    },
    {
      "activations/layer0_attention_weight_max": 17.313167572021484,
      "activations/layer0_attention_weight_min": -14.667740821838379,
      "activations/layer10_attention_weight_max": 34.664833068847656,
      "activations/layer10_attention_weight_min": -35.75059509277344,
      "activations/layer11_attention_weight_max": 33.80022048950195,
      "activations/layer11_attention_weight_min": -35.94225311279297,
      "activations/layer12_attention_weight_max": 27.29938316345215,
      "activations/layer12_attention_weight_min": -27.06315040588379,
      "activations/layer13_attention_weight_max": 39.66132736206055,
      "activations/layer13_attention_weight_min": -36.3219108581543,
      "activations/layer14_attention_weight_max": 44.10061264038086,
      "activations/layer14_attention_weight_min": -37.06726837158203,
      "activations/layer15_attention_weight_max": 38.08129119873047,
      "activations/layer15_attention_weight_min": -37.550106048583984,
      "activations/layer16_attention_weight_max": 28.166275024414062,
      "activations/layer16_attention_weight_min": -27.661394119262695,
      "activations/layer17_attention_weight_max": 31.267648696899414,
      "activations/layer17_attention_weight_min": -26.91230583190918,
      "activations/layer18_attention_weight_max": 34.5087776184082,
      "activations/layer18_attention_weight_min": -25.96426010131836,
      "activations/layer19_attention_weight_max": 32.736080169677734,
      "activations/layer19_attention_weight_min": -31.9268741607666,
      "activations/layer1_attention_weight_max": 15.964282035827637,
      "activations/layer1_attention_weight_min": -15.548456192016602,
      "activations/layer20_attention_weight_max": 29.90413475036621,
      "activations/layer20_attention_weight_min": -24.915678024291992,
      "activations/layer21_attention_weight_max": 30.620986938476562,
      "activations/layer21_attention_weight_min": -26.048263549804688,
      "activations/layer22_attention_weight_max": 40.777347564697266,
      "activations/layer22_attention_weight_min": -31.78144645690918,
      "activations/layer23_attention_weight_max": 29.232280731201172,
      "activations/layer23_attention_weight_min": -24.811603546142578,
      "activations/layer2_attention_weight_max": 34.01954650878906,
      "activations/layer2_attention_weight_min": -33.60700225830078,
      "activations/layer3_attention_weight_max": 109.5166015625,
      "activations/layer3_attention_weight_min": -110.40849304199219,
      "activations/layer4_attention_weight_max": 112.80079650878906,
      "activations/layer4_attention_weight_min": -116.22938537597656,
      "activations/layer5_attention_weight_max": 54.388389587402344,
      "activations/layer5_attention_weight_min": -62.91950607299805,
      "activations/layer6_attention_weight_max": 48.8030891418457,
      "activations/layer6_attention_weight_min": -49.690528869628906,
      "activations/layer7_attention_weight_max": 100.70195770263672,
      "activations/layer7_attention_weight_min": -104.02950286865234,
      "activations/layer8_attention_weight_max": 43.55801010131836,
      "activations/layer8_attention_weight_min": -45.713401794433594,
      "activations/layer9_attention_weight_max": 32.24721908569336,
      "activations/layer9_attention_weight_min": -35.80260467529297,
      "epoch": 22.4,
      "learning_rate": 5.586742424242424e-06,
      "loss": 2.6957,
      "step": 385550
    },
    {
      "activations/layer0_attention_weight_max": 17.197145462036133,
      "activations/layer0_attention_weight_min": -13.8150634765625,
      "activations/layer10_attention_weight_max": 38.531829833984375,
      "activations/layer10_attention_weight_min": -38.61051940917969,
      "activations/layer11_attention_weight_max": 39.38320541381836,
      "activations/layer11_attention_weight_min": -39.149009704589844,
      "activations/layer12_attention_weight_max": 28.688478469848633,
      "activations/layer12_attention_weight_min": -29.262704849243164,
      "activations/layer13_attention_weight_max": 39.95797348022461,
      "activations/layer13_attention_weight_min": -41.30067443847656,
      "activations/layer14_attention_weight_max": 43.5868034362793,
      "activations/layer14_attention_weight_min": -41.6046257019043,
      "activations/layer15_attention_weight_max": 42.08557891845703,
      "activations/layer15_attention_weight_min": -39.812095642089844,
      "activations/layer16_attention_weight_max": 31.380468368530273,
      "activations/layer16_attention_weight_min": -30.98887825012207,
      "activations/layer17_attention_weight_max": 31.12422752380371,
      "activations/layer17_attention_weight_min": -27.653654098510742,
      "activations/layer18_attention_weight_max": 30.330848693847656,
      "activations/layer18_attention_weight_min": -27.234819412231445,
      "activations/layer19_attention_weight_max": 31.36995506286621,
      "activations/layer19_attention_weight_min": -35.85163879394531,
      "activations/layer1_attention_weight_max": 16.045120239257812,
      "activations/layer1_attention_weight_min": -15.423882484436035,
      "activations/layer20_attention_weight_max": 26.836944580078125,
      "activations/layer20_attention_weight_min": -26.54579734802246,
      "activations/layer21_attention_weight_max": 27.066617965698242,
      "activations/layer21_attention_weight_min": -27.409048080444336,
      "activations/layer22_attention_weight_max": 39.03913879394531,
      "activations/layer22_attention_weight_min": -33.5351676940918,
      "activations/layer23_attention_weight_max": 30.443817138671875,
      "activations/layer23_attention_weight_min": -27.227310180664062,
      "activations/layer2_attention_weight_max": 34.450958251953125,
      "activations/layer2_attention_weight_min": -34.86281967163086,
      "activations/layer3_attention_weight_max": 109.7718276977539,
      "activations/layer3_attention_weight_min": -108.91566467285156,
      "activations/layer4_attention_weight_max": 113.26026916503906,
      "activations/layer4_attention_weight_min": -119.04853820800781,
      "activations/layer5_attention_weight_max": 50.87522888183594,
      "activations/layer5_attention_weight_min": -63.00321960449219,
      "activations/layer6_attention_weight_max": 47.356258392333984,
      "activations/layer6_attention_weight_min": -50.47929382324219,
      "activations/layer7_attention_weight_max": 104.4755859375,
      "activations/layer7_attention_weight_min": -111.2818603515625,
      "activations/layer8_attention_weight_max": 46.24997329711914,
      "activations/layer8_attention_weight_min": -51.707298278808594,
      "activations/layer9_attention_weight_max": 34.96712875366211,
      "activations/layer9_attention_weight_min": -38.1561279296875,
      "epoch": 22.41,
      "learning_rate": 5.56780303030303e-06,
      "loss": 2.692,
      "step": 385600
    },
    {
      "activations/layer0_attention_weight_max": 16.778520584106445,
      "activations/layer0_attention_weight_min": -12.984993934631348,
      "activations/layer10_attention_weight_max": 37.91508483886719,
      "activations/layer10_attention_weight_min": -37.722938537597656,
      "activations/layer11_attention_weight_max": 36.57147979736328,
      "activations/layer11_attention_weight_min": -37.23496627807617,
      "activations/layer12_attention_weight_max": 30.53347396850586,
      "activations/layer12_attention_weight_min": -28.248998641967773,
      "activations/layer13_attention_weight_max": 43.095088958740234,
      "activations/layer13_attention_weight_min": -38.85491180419922,
      "activations/layer14_attention_weight_max": 43.450748443603516,
      "activations/layer14_attention_weight_min": -39.7559700012207,
      "activations/layer15_attention_weight_max": 41.65213394165039,
      "activations/layer15_attention_weight_min": -38.73244857788086,
      "activations/layer16_attention_weight_max": 31.129716873168945,
      "activations/layer16_attention_weight_min": -28.207300186157227,
      "activations/layer17_attention_weight_max": 30.09358787536621,
      "activations/layer17_attention_weight_min": -27.267595291137695,
      "activations/layer18_attention_weight_max": 34.22672653198242,
      "activations/layer18_attention_weight_min": -26.54296112060547,
      "activations/layer19_attention_weight_max": 35.92763900756836,
      "activations/layer19_attention_weight_min": -33.96974563598633,
      "activations/layer1_attention_weight_max": 16.42079734802246,
      "activations/layer1_attention_weight_min": -15.432766914367676,
      "activations/layer20_attention_weight_max": 28.8654842376709,
      "activations/layer20_attention_weight_min": -27.5926570892334,
      "activations/layer21_attention_weight_max": 28.18199920654297,
      "activations/layer21_attention_weight_min": -25.950246810913086,
      "activations/layer22_attention_weight_max": 41.25933074951172,
      "activations/layer22_attention_weight_min": -31.982791900634766,
      "activations/layer23_attention_weight_max": 32.41075134277344,
      "activations/layer23_attention_weight_min": -26.83047866821289,
      "activations/layer2_attention_weight_max": 34.63726806640625,
      "activations/layer2_attention_weight_min": -34.66967010498047,
      "activations/layer3_attention_weight_max": 108.03819274902344,
      "activations/layer3_attention_weight_min": -110.54998016357422,
      "activations/layer4_attention_weight_max": 114.0802001953125,
      "activations/layer4_attention_weight_min": -117.0665054321289,
      "activations/layer5_attention_weight_max": 52.758270263671875,
      "activations/layer5_attention_weight_min": -66.17021179199219,
      "activations/layer6_attention_weight_max": 46.074363708496094,
      "activations/layer6_attention_weight_min": -50.71919250488281,
      "activations/layer7_attention_weight_max": 99.69930267333984,
      "activations/layer7_attention_weight_min": -105.21475982666016,
      "activations/layer8_attention_weight_max": 46.545745849609375,
      "activations/layer8_attention_weight_min": -48.32884979248047,
      "activations/layer9_attention_weight_max": 34.70479965209961,
      "activations/layer9_attention_weight_min": -37.39301300048828,
      "epoch": 22.41,
      "learning_rate": 5.548863636363636e-06,
      "loss": 2.7096,
      "step": 385650
    },
    {
      "activations/layer0_attention_weight_max": 16.61259651184082,
      "activations/layer0_attention_weight_min": -13.284090995788574,
      "activations/layer10_attention_weight_max": 35.829105377197266,
      "activations/layer10_attention_weight_min": -37.064842224121094,
      "activations/layer11_attention_weight_max": 37.05877685546875,
      "activations/layer11_attention_weight_min": -35.90044021606445,
      "activations/layer12_attention_weight_max": 27.624916076660156,
      "activations/layer12_attention_weight_min": -26.75007438659668,
      "activations/layer13_attention_weight_max": 40.0302619934082,
      "activations/layer13_attention_weight_min": -34.2164421081543,
      "activations/layer14_attention_weight_max": 41.57356643676758,
      "activations/layer14_attention_weight_min": -38.4163703918457,
      "activations/layer15_attention_weight_max": 41.2026481628418,
      "activations/layer15_attention_weight_min": -38.71805953979492,
      "activations/layer16_attention_weight_max": 31.177568435668945,
      "activations/layer16_attention_weight_min": -26.46094512939453,
      "activations/layer17_attention_weight_max": 30.010379791259766,
      "activations/layer17_attention_weight_min": -25.715843200683594,
      "activations/layer18_attention_weight_max": 32.61840057373047,
      "activations/layer18_attention_weight_min": -27.949052810668945,
      "activations/layer19_attention_weight_max": 33.89486312866211,
      "activations/layer19_attention_weight_min": -31.9636173248291,
      "activations/layer1_attention_weight_max": 15.444245338439941,
      "activations/layer1_attention_weight_min": -14.641980171203613,
      "activations/layer20_attention_weight_max": 29.65216064453125,
      "activations/layer20_attention_weight_min": -25.530872344970703,
      "activations/layer21_attention_weight_max": 29.033172607421875,
      "activations/layer21_attention_weight_min": -26.81325340270996,
      "activations/layer22_attention_weight_max": 43.58955001831055,
      "activations/layer22_attention_weight_min": -36.09547424316406,
      "activations/layer23_attention_weight_max": 33.35446548461914,
      "activations/layer23_attention_weight_min": -28.781253814697266,
      "activations/layer2_attention_weight_max": 35.02668762207031,
      "activations/layer2_attention_weight_min": -33.85235595703125,
      "activations/layer3_attention_weight_max": 111.87006378173828,
      "activations/layer3_attention_weight_min": -108.13449096679688,
      "activations/layer4_attention_weight_max": 114.1625747680664,
      "activations/layer4_attention_weight_min": -114.87261199951172,
      "activations/layer5_attention_weight_max": 51.654205322265625,
      "activations/layer5_attention_weight_min": -66.77766418457031,
      "activations/layer6_attention_weight_max": 46.359561920166016,
      "activations/layer6_attention_weight_min": -48.522178649902344,
      "activations/layer7_attention_weight_max": 95.54580688476562,
      "activations/layer7_attention_weight_min": -95.5161361694336,
      "activations/layer8_attention_weight_max": 40.6465950012207,
      "activations/layer8_attention_weight_min": -44.252750396728516,
      "activations/layer9_attention_weight_max": 32.17321014404297,
      "activations/layer9_attention_weight_min": -35.5435905456543,
      "epoch": 22.41,
      "learning_rate": 5.529924242424242e-06,
      "loss": 2.6977,
      "step": 385700
    },
    {
      "activations/layer0_attention_weight_max": 17.27375030517578,
      "activations/layer0_attention_weight_min": -12.433476448059082,
      "activations/layer10_attention_weight_max": 50.77447509765625,
      "activations/layer10_attention_weight_min": -48.85004425048828,
      "activations/layer11_attention_weight_max": 52.082664489746094,
      "activations/layer11_attention_weight_min": -52.839942932128906,
      "activations/layer12_attention_weight_max": 39.84187698364258,
      "activations/layer12_attention_weight_min": -35.89946746826172,
      "activations/layer13_attention_weight_max": 60.891056060791016,
      "activations/layer13_attention_weight_min": -50.383506774902344,
      "activations/layer14_attention_weight_max": 66.37532043457031,
      "activations/layer14_attention_weight_min": -53.99061584472656,
      "activations/layer15_attention_weight_max": 63.592105865478516,
      "activations/layer15_attention_weight_min": -55.965816497802734,
      "activations/layer16_attention_weight_max": 43.07775115966797,
      "activations/layer16_attention_weight_min": -34.78147506713867,
      "activations/layer17_attention_weight_max": 38.08958053588867,
      "activations/layer17_attention_weight_min": -36.00800323486328,
      "activations/layer18_attention_weight_max": 36.42137908935547,
      "activations/layer18_attention_weight_min": -32.435089111328125,
      "activations/layer19_attention_weight_max": 35.45512008666992,
      "activations/layer19_attention_weight_min": -32.658748626708984,
      "activations/layer1_attention_weight_max": 16.971494674682617,
      "activations/layer1_attention_weight_min": -14.181169509887695,
      "activations/layer20_attention_weight_max": 28.912092208862305,
      "activations/layer20_attention_weight_min": -25.2877140045166,
      "activations/layer21_attention_weight_max": 28.85121726989746,
      "activations/layer21_attention_weight_min": -25.091745376586914,
      "activations/layer22_attention_weight_max": 42.492549896240234,
      "activations/layer22_attention_weight_min": -34.70497131347656,
      "activations/layer23_attention_weight_max": 33.401023864746094,
      "activations/layer23_attention_weight_min": -26.123022079467773,
      "activations/layer2_attention_weight_max": 34.20266342163086,
      "activations/layer2_attention_weight_min": -34.057594299316406,
      "activations/layer3_attention_weight_max": 107.96812438964844,
      "activations/layer3_attention_weight_min": -108.80000305175781,
      "activations/layer4_attention_weight_max": 112.33390045166016,
      "activations/layer4_attention_weight_min": -113.6141128540039,
      "activations/layer5_attention_weight_max": 55.031166076660156,
      "activations/layer5_attention_weight_min": -64.91253662109375,
      "activations/layer6_attention_weight_max": 48.51755142211914,
      "activations/layer6_attention_weight_min": -50.113224029541016,
      "activations/layer7_attention_weight_max": 121.3848648071289,
      "activations/layer7_attention_weight_min": -103.77619171142578,
      "activations/layer8_attention_weight_max": 59.22761535644531,
      "activations/layer8_attention_weight_min": -53.45062255859375,
      "activations/layer9_attention_weight_max": 41.546905517578125,
      "activations/layer9_attention_weight_min": -44.91559600830078,
      "epoch": 22.41,
      "learning_rate": 5.510984848484848e-06,
      "loss": 2.697,
      "step": 385750
    },
    {
      "activations/layer0_attention_weight_max": 16.72770881652832,
      "activations/layer0_attention_weight_min": -13.59976863861084,
      "activations/layer10_attention_weight_max": 37.751033782958984,
      "activations/layer10_attention_weight_min": -37.43888473510742,
      "activations/layer11_attention_weight_max": 36.0040397644043,
      "activations/layer11_attention_weight_min": -38.54471206665039,
      "activations/layer12_attention_weight_max": 28.984113693237305,
      "activations/layer12_attention_weight_min": -28.10234260559082,
      "activations/layer13_attention_weight_max": 42.36682891845703,
      "activations/layer13_attention_weight_min": -37.2504768371582,
      "activations/layer14_attention_weight_max": 45.114013671875,
      "activations/layer14_attention_weight_min": -39.07160949707031,
      "activations/layer15_attention_weight_max": 41.077606201171875,
      "activations/layer15_attention_weight_min": -38.635963439941406,
      "activations/layer16_attention_weight_max": 30.23548126220703,
      "activations/layer16_attention_weight_min": -28.501218795776367,
      "activations/layer17_attention_weight_max": 29.519330978393555,
      "activations/layer17_attention_weight_min": -26.7135066986084,
      "activations/layer18_attention_weight_max": 33.78028106689453,
      "activations/layer18_attention_weight_min": -28.365577697753906,
      "activations/layer19_attention_weight_max": 34.962615966796875,
      "activations/layer19_attention_weight_min": -33.24294662475586,
      "activations/layer1_attention_weight_max": 16.238317489624023,
      "activations/layer1_attention_weight_min": -16.860366821289062,
      "activations/layer20_attention_weight_max": 27.586706161499023,
      "activations/layer20_attention_weight_min": -25.808940887451172,
      "activations/layer21_attention_weight_max": 28.052978515625,
      "activations/layer21_attention_weight_min": -26.02220916748047,
      "activations/layer22_attention_weight_max": 40.79214859008789,
      "activations/layer22_attention_weight_min": -33.3468017578125,
      "activations/layer23_attention_weight_max": 35.09571075439453,
      "activations/layer23_attention_weight_min": -26.155752182006836,
      "activations/layer2_attention_weight_max": 33.68696594238281,
      "activations/layer2_attention_weight_min": -33.91228103637695,
      "activations/layer3_attention_weight_max": 108.1357650756836,
      "activations/layer3_attention_weight_min": -109.57971954345703,
      "activations/layer4_attention_weight_max": 116.3446044921875,
      "activations/layer4_attention_weight_min": -112.27565002441406,
      "activations/layer5_attention_weight_max": 52.64893341064453,
      "activations/layer5_attention_weight_min": -62.38804244995117,
      "activations/layer6_attention_weight_max": 51.232418060302734,
      "activations/layer6_attention_weight_min": -51.34824752807617,
      "activations/layer7_attention_weight_max": 111.16797637939453,
      "activations/layer7_attention_weight_min": -110.77700805664062,
      "activations/layer8_attention_weight_max": 51.54170227050781,
      "activations/layer8_attention_weight_min": -51.083377838134766,
      "activations/layer9_attention_weight_max": 35.62271499633789,
      "activations/layer9_attention_weight_min": -37.926387786865234,
      "epoch": 22.42,
      "learning_rate": 5.492045454545454e-06,
      "loss": 2.7004,
      "step": 385800
    },
    {
      "activations/layer0_attention_weight_max": 18.114139556884766,
      "activations/layer0_attention_weight_min": -14.16148853302002,
      "activations/layer10_attention_weight_max": 37.46695327758789,
      "activations/layer10_attention_weight_min": -38.042911529541016,
      "activations/layer11_attention_weight_max": 37.307960510253906,
      "activations/layer11_attention_weight_min": -38.922813415527344,
      "activations/layer12_attention_weight_max": 29.067750930786133,
      "activations/layer12_attention_weight_min": -31.321870803833008,
      "activations/layer13_attention_weight_max": 43.21726608276367,
      "activations/layer13_attention_weight_min": -38.09056854248047,
      "activations/layer14_attention_weight_max": 45.84394836425781,
      "activations/layer14_attention_weight_min": -41.85260772705078,
      "activations/layer15_attention_weight_max": 44.90202713012695,
      "activations/layer15_attention_weight_min": -40.44868850708008,
      "activations/layer16_attention_weight_max": 31.27705955505371,
      "activations/layer16_attention_weight_min": -28.33298110961914,
      "activations/layer17_attention_weight_max": 32.78313064575195,
      "activations/layer17_attention_weight_min": -27.13435173034668,
      "activations/layer18_attention_weight_max": 36.745391845703125,
      "activations/layer18_attention_weight_min": -26.230430603027344,
      "activations/layer19_attention_weight_max": 39.15304946899414,
      "activations/layer19_attention_weight_min": -31.707202911376953,
      "activations/layer1_attention_weight_max": 16.76268768310547,
      "activations/layer1_attention_weight_min": -15.216246604919434,
      "activations/layer20_attention_weight_max": 31.8260555267334,
      "activations/layer20_attention_weight_min": -25.844091415405273,
      "activations/layer21_attention_weight_max": 30.670774459838867,
      "activations/layer21_attention_weight_min": -26.48229217529297,
      "activations/layer22_attention_weight_max": 47.49232482910156,
      "activations/layer22_attention_weight_min": -30.17112159729004,
      "activations/layer23_attention_weight_max": 34.452754974365234,
      "activations/layer23_attention_weight_min": -27.242809295654297,
      "activations/layer2_attention_weight_max": 36.57159423828125,
      "activations/layer2_attention_weight_min": -36.618507385253906,
      "activations/layer3_attention_weight_max": 110.79685974121094,
      "activations/layer3_attention_weight_min": -116.5891342163086,
      "activations/layer4_attention_weight_max": 118.6980209350586,
      "activations/layer4_attention_weight_min": -117.71129608154297,
      "activations/layer5_attention_weight_max": 54.87908172607422,
      "activations/layer5_attention_weight_min": -64.16514587402344,
      "activations/layer6_attention_weight_max": 49.31185531616211,
      "activations/layer6_attention_weight_min": -53.613304138183594,
      "activations/layer7_attention_weight_max": 103.2830581665039,
      "activations/layer7_attention_weight_min": -114.41875457763672,
      "activations/layer8_attention_weight_max": 45.69519805908203,
      "activations/layer8_attention_weight_min": -50.35664367675781,
      "activations/layer9_attention_weight_max": 36.13177490234375,
      "activations/layer9_attention_weight_min": -39.11920928955078,
      "epoch": 22.42,
      "learning_rate": 5.473106060606059e-06,
      "loss": 2.7038,
      "step": 385850
    },
    {
      "activations/layer0_attention_weight_max": 17.451068878173828,
      "activations/layer0_attention_weight_min": -13.757719993591309,
      "activations/layer10_attention_weight_max": 36.95822525024414,
      "activations/layer10_attention_weight_min": -36.48090362548828,
      "activations/layer11_attention_weight_max": 35.29800796508789,
      "activations/layer11_attention_weight_min": -36.45856475830078,
      "activations/layer12_attention_weight_max": 28.64514923095703,
      "activations/layer12_attention_weight_min": -26.552061080932617,
      "activations/layer13_attention_weight_max": 40.1273193359375,
      "activations/layer13_attention_weight_min": -36.023860931396484,
      "activations/layer14_attention_weight_max": 43.892154693603516,
      "activations/layer14_attention_weight_min": -38.11202621459961,
      "activations/layer15_attention_weight_max": 41.73472595214844,
      "activations/layer15_attention_weight_min": -37.19382858276367,
      "activations/layer16_attention_weight_max": 34.24563980102539,
      "activations/layer16_attention_weight_min": -27.879568099975586,
      "activations/layer17_attention_weight_max": 31.57187843322754,
      "activations/layer17_attention_weight_min": -27.200239181518555,
      "activations/layer18_attention_weight_max": 41.1154670715332,
      "activations/layer18_attention_weight_min": -26.945201873779297,
      "activations/layer19_attention_weight_max": 40.87438201904297,
      "activations/layer19_attention_weight_min": -31.398244857788086,
      "activations/layer1_attention_weight_max": 15.78454303741455,
      "activations/layer1_attention_weight_min": -15.981863021850586,
      "activations/layer20_attention_weight_max": 32.560672760009766,
      "activations/layer20_attention_weight_min": -25.46282958984375,
      "activations/layer21_attention_weight_max": 32.130916595458984,
      "activations/layer21_attention_weight_min": -25.85227394104004,
      "activations/layer22_attention_weight_max": 46.45100784301758,
      "activations/layer22_attention_weight_min": -31.83445167541504,
      "activations/layer23_attention_weight_max": 35.12248229980469,
      "activations/layer23_attention_weight_min": -25.419157028198242,
      "activations/layer2_attention_weight_max": 33.7203483581543,
      "activations/layer2_attention_weight_min": -33.47700500488281,
      "activations/layer3_attention_weight_max": 107.23143005371094,
      "activations/layer3_attention_weight_min": -106.85908508300781,
      "activations/layer4_attention_weight_max": 108.81108856201172,
      "activations/layer4_attention_weight_min": -111.92364501953125,
      "activations/layer5_attention_weight_max": 49.25830078125,
      "activations/layer5_attention_weight_min": -61.45663070678711,
      "activations/layer6_attention_weight_max": 45.37797927856445,
      "activations/layer6_attention_weight_min": -48.12282180786133,
      "activations/layer7_attention_weight_max": 97.33943176269531,
      "activations/layer7_attention_weight_min": -98.73777770996094,
      "activations/layer8_attention_weight_max": 44.14249038696289,
      "activations/layer8_attention_weight_min": -44.59653091430664,
      "activations/layer9_attention_weight_max": 32.86845016479492,
      "activations/layer9_attention_weight_min": -35.63166427612305,
      "epoch": 22.42,
      "learning_rate": 5.454166666666666e-06,
      "loss": 2.6941,
      "step": 385900
    },
    {
      "activations/layer0_attention_weight_max": 17.820890426635742,
      "activations/layer0_attention_weight_min": -15.244831085205078,
      "activations/layer10_attention_weight_max": 35.082130432128906,
      "activations/layer10_attention_weight_min": -38.11084747314453,
      "activations/layer11_attention_weight_max": 35.37154769897461,
      "activations/layer11_attention_weight_min": -37.35917663574219,
      "activations/layer12_attention_weight_max": 28.50802230834961,
      "activations/layer12_attention_weight_min": -28.494535446166992,
      "activations/layer13_attention_weight_max": 41.572078704833984,
      "activations/layer13_attention_weight_min": -37.774375915527344,
      "activations/layer14_attention_weight_max": 43.03997802734375,
      "activations/layer14_attention_weight_min": -39.81865692138672,
      "activations/layer15_attention_weight_max": 42.6317138671875,
      "activations/layer15_attention_weight_min": -37.06857681274414,
      "activations/layer16_attention_weight_max": 30.959243774414062,
      "activations/layer16_attention_weight_min": -27.82847023010254,
      "activations/layer17_attention_weight_max": 33.82105255126953,
      "activations/layer17_attention_weight_min": -28.86513328552246,
      "activations/layer18_attention_weight_max": 36.56733322143555,
      "activations/layer18_attention_weight_min": -27.26786994934082,
      "activations/layer19_attention_weight_max": 35.68319320678711,
      "activations/layer19_attention_weight_min": -31.694137573242188,
      "activations/layer1_attention_weight_max": 16.49614143371582,
      "activations/layer1_attention_weight_min": -15.847589492797852,
      "activations/layer20_attention_weight_max": 29.309524536132812,
      "activations/layer20_attention_weight_min": -25.977556228637695,
      "activations/layer21_attention_weight_max": 28.41271209716797,
      "activations/layer21_attention_weight_min": -25.948816299438477,
      "activations/layer22_attention_weight_max": 43.33488464355469,
      "activations/layer22_attention_weight_min": -32.62596893310547,
      "activations/layer23_attention_weight_max": 35.289451599121094,
      "activations/layer23_attention_weight_min": -27.83338737487793,
      "activations/layer2_attention_weight_max": 34.84388732910156,
      "activations/layer2_attention_weight_min": -34.658409118652344,
      "activations/layer3_attention_weight_max": 107.12123107910156,
      "activations/layer3_attention_weight_min": -107.8324203491211,
      "activations/layer4_attention_weight_max": 110.76372528076172,
      "activations/layer4_attention_weight_min": -116.14981842041016,
      "activations/layer5_attention_weight_max": 51.52136993408203,
      "activations/layer5_attention_weight_min": -65.22840881347656,
      "activations/layer6_attention_weight_max": 46.90245819091797,
      "activations/layer6_attention_weight_min": -49.62446212768555,
      "activations/layer7_attention_weight_max": 94.91483306884766,
      "activations/layer7_attention_weight_min": -110.67628479003906,
      "activations/layer8_attention_weight_max": 42.83624267578125,
      "activations/layer8_attention_weight_min": -46.43355178833008,
      "activations/layer9_attention_weight_max": 33.50304412841797,
      "activations/layer9_attention_weight_min": -37.940895080566406,
      "epoch": 22.43,
      "learning_rate": 5.435606060606059e-06,
      "loss": 2.6963,
      "step": 385950
    },
    {
      "activations/layer0_attention_weight_max": 16.8618221282959,
      "activations/layer0_attention_weight_min": -13.701526641845703,
      "activations/layer10_attention_weight_max": 33.28687286376953,
      "activations/layer10_attention_weight_min": -34.6102180480957,
      "activations/layer11_attention_weight_max": 32.686065673828125,
      "activations/layer11_attention_weight_min": -35.010528564453125,
      "activations/layer12_attention_weight_max": 26.57425308227539,
      "activations/layer12_attention_weight_min": -28.148107528686523,
      "activations/layer13_attention_weight_max": 39.305931091308594,
      "activations/layer13_attention_weight_min": -38.875274658203125,
      "activations/layer14_attention_weight_max": 41.57913589477539,
      "activations/layer14_attention_weight_min": -41.68724822998047,
      "activations/layer15_attention_weight_max": 40.384849548339844,
      "activations/layer15_attention_weight_min": -39.44132995605469,
      "activations/layer16_attention_weight_max": 29.020700454711914,
      "activations/layer16_attention_weight_min": -28.188627243041992,
      "activations/layer17_attention_weight_max": 31.02572250366211,
      "activations/layer17_attention_weight_min": -25.86844825744629,
      "activations/layer18_attention_weight_max": 31.560359954833984,
      "activations/layer18_attention_weight_min": -25.731618881225586,
      "activations/layer19_attention_weight_max": 32.32398986816406,
      "activations/layer19_attention_weight_min": -31.099241256713867,
      "activations/layer1_attention_weight_max": 15.600473403930664,
      "activations/layer1_attention_weight_min": -15.335820198059082,
      "activations/layer20_attention_weight_max": 29.247167587280273,
      "activations/layer20_attention_weight_min": -24.134296417236328,
      "activations/layer21_attention_weight_max": 27.297414779663086,
      "activations/layer21_attention_weight_min": -24.889665603637695,
      "activations/layer22_attention_weight_max": 41.778446197509766,
      "activations/layer22_attention_weight_min": -31.221548080444336,
      "activations/layer23_attention_weight_max": 31.322538375854492,
      "activations/layer23_attention_weight_min": -25.239986419677734,
      "activations/layer2_attention_weight_max": 32.97197723388672,
      "activations/layer2_attention_weight_min": -34.08881378173828,
      "activations/layer3_attention_weight_max": 107.23882293701172,
      "activations/layer3_attention_weight_min": -108.84152221679688,
      "activations/layer4_attention_weight_max": 113.27294921875,
      "activations/layer4_attention_weight_min": -116.85283660888672,
      "activations/layer5_attention_weight_max": 49.22625732421875,
      "activations/layer5_attention_weight_min": -64.98866271972656,
      "activations/layer6_attention_weight_max": 44.1121940612793,
      "activations/layer6_attention_weight_min": -47.26858139038086,
      "activations/layer7_attention_weight_max": 95.21571350097656,
      "activations/layer7_attention_weight_min": -97.83065032958984,
      "activations/layer8_attention_weight_max": 41.79391860961914,
      "activations/layer8_attention_weight_min": -44.32870101928711,
      "activations/layer9_attention_weight_max": 32.02498245239258,
      "activations/layer9_attention_weight_min": -35.52372741699219,
      "epoch": 22.43,
      "learning_rate": 5.416666666666666e-06,
      "loss": 2.7003,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4302,
      "eval_samples_per_second": 509.36,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4302,
      "eval_openwebtext_samples_per_second": 509.36,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9248,
      "eval_wikitext_samples_per_second": 236.903,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_lambada_loss": 2.283203125,
      "eval_lambada_ppl": 9.808046546977115,
      "eval_lambada_runtime": 9.5108,
      "eval_lambada_samples_per_second": 511.945,
      "step": 386000
    },
    {
      "activations/layer0_attention_weight_max": 18.35563087463379,
      "activations/layer0_attention_weight_min": -14.85588264465332,
      "activations/layer10_attention_weight_max": 35.479984283447266,
      "activations/layer10_attention_weight_min": -36.33906936645508,
      "activations/layer11_attention_weight_max": 35.40230178833008,
      "activations/layer11_attention_weight_min": -35.70068359375,
      "activations/layer12_attention_weight_max": 26.784793853759766,
      "activations/layer12_attention_weight_min": -28.479324340820312,
      "activations/layer13_attention_weight_max": 37.53090286254883,
      "activations/layer13_attention_weight_min": -38.01374053955078,
      "activations/layer14_attention_weight_max": 43.83807373046875,
      "activations/layer14_attention_weight_min": -40.79914855957031,
      "activations/layer15_attention_weight_max": 38.20237350463867,
      "activations/layer15_attention_weight_min": -37.50606918334961,
      "activations/layer16_attention_weight_max": 28.685842514038086,
      "activations/layer16_attention_weight_min": -29.70944595336914,
      "activations/layer17_attention_weight_max": 30.93050193786621,
      "activations/layer17_attention_weight_min": -26.72051429748535,
      "activations/layer18_attention_weight_max": 31.911020278930664,
      "activations/layer18_attention_weight_min": -28.348501205444336,
      "activations/layer19_attention_weight_max": 36.8775749206543,
      "activations/layer19_attention_weight_min": -31.640430450439453,
      "activations/layer1_attention_weight_max": 15.512528419494629,
      "activations/layer1_attention_weight_min": -15.712474822998047,
      "activations/layer20_attention_weight_max": 29.724998474121094,
      "activations/layer20_attention_weight_min": -26.660411834716797,
      "activations/layer21_attention_weight_max": 28.444564819335938,
      "activations/layer21_attention_weight_min": -26.602523803710938,
      "activations/layer22_attention_weight_max": 37.85205078125,
      "activations/layer22_attention_weight_min": -32.60211944580078,
      "activations/layer23_attention_weight_max": 31.737592697143555,
      "activations/layer23_attention_weight_min": -27.787128448486328,
      "activations/layer2_attention_weight_max": 34.26354217529297,
      "activations/layer2_attention_weight_min": -33.36312484741211,
      "activations/layer3_attention_weight_max": 111.34950256347656,
      "activations/layer3_attention_weight_min": -108.2439956665039,
      "activations/layer4_attention_weight_max": 113.45796203613281,
      "activations/layer4_attention_weight_min": -116.87029266357422,
      "activations/layer5_attention_weight_max": 53.867427825927734,
      "activations/layer5_attention_weight_min": -64.08970642089844,
      "activations/layer6_attention_weight_max": 48.81221008300781,
      "activations/layer6_attention_weight_min": -49.73002624511719,
      "activations/layer7_attention_weight_max": 104.03478240966797,
      "activations/layer7_attention_weight_min": -102.3936767578125,
      "activations/layer8_attention_weight_max": 45.46177291870117,
      "activations/layer8_attention_weight_min": -47.628936767578125,
      "activations/layer9_attention_weight_max": 33.331878662109375,
      "activations/layer9_attention_weight_min": -36.87687683105469,
      "epoch": 22.43,
      "learning_rate": 5.3977272727272725e-06,
      "loss": 2.7023,
      "step": 386050
    },
    {
      "activations/layer0_attention_weight_max": 16.852603912353516,
      "activations/layer0_attention_weight_min": -14.898716926574707,
      "activations/layer10_attention_weight_max": 36.28459548950195,
      "activations/layer10_attention_weight_min": -35.11420440673828,
      "activations/layer11_attention_weight_max": 34.89942169189453,
      "activations/layer11_attention_weight_min": -36.646968841552734,
      "activations/layer12_attention_weight_max": 26.1229305267334,
      "activations/layer12_attention_weight_min": -25.943565368652344,
      "activations/layer13_attention_weight_max": 37.20716857910156,
      "activations/layer13_attention_weight_min": -34.48387145996094,
      "activations/layer14_attention_weight_max": 40.720428466796875,
      "activations/layer14_attention_weight_min": -37.890045166015625,
      "activations/layer15_attention_weight_max": 37.30524826049805,
      "activations/layer15_attention_weight_min": -36.70412826538086,
      "activations/layer16_attention_weight_max": 27.639909744262695,
      "activations/layer16_attention_weight_min": -31.247249603271484,
      "activations/layer17_attention_weight_max": 27.78977394104004,
      "activations/layer17_attention_weight_min": -26.000762939453125,
      "activations/layer18_attention_weight_max": 29.817367553710938,
      "activations/layer18_attention_weight_min": -24.614490509033203,
      "activations/layer19_attention_weight_max": 32.37916946411133,
      "activations/layer19_attention_weight_min": -31.007596969604492,
      "activations/layer1_attention_weight_max": 15.440281867980957,
      "activations/layer1_attention_weight_min": -14.37998104095459,
      "activations/layer20_attention_weight_max": 26.77813720703125,
      "activations/layer20_attention_weight_min": -23.296968460083008,
      "activations/layer21_attention_weight_max": 28.190176010131836,
      "activations/layer21_attention_weight_min": -24.133886337280273,
      "activations/layer22_attention_weight_max": 37.83658218383789,
      "activations/layer22_attention_weight_min": -30.36125373840332,
      "activations/layer23_attention_weight_max": 31.32369613647461,
      "activations/layer23_attention_weight_min": -26.4721736907959,
      "activations/layer2_attention_weight_max": 34.3972053527832,
      "activations/layer2_attention_weight_min": -33.061824798583984,
      "activations/layer3_attention_weight_max": 104.93832397460938,
      "activations/layer3_attention_weight_min": -105.43309020996094,
      "activations/layer4_attention_weight_max": 107.2914047241211,
      "activations/layer4_attention_weight_min": -112.97589111328125,
      "activations/layer5_attention_weight_max": 50.30842208862305,
      "activations/layer5_attention_weight_min": -64.59524536132812,
      "activations/layer6_attention_weight_max": 43.1235237121582,
      "activations/layer6_attention_weight_min": -47.286903381347656,
      "activations/layer7_attention_weight_max": 96.54007720947266,
      "activations/layer7_attention_weight_min": -101.330078125,
      "activations/layer8_attention_weight_max": 41.75661087036133,
      "activations/layer8_attention_weight_min": -44.79286575317383,
      "activations/layer9_attention_weight_max": 33.32849884033203,
      "activations/layer9_attention_weight_min": -34.53379440307617,
      "epoch": 22.43,
      "learning_rate": 5.378787878787878e-06,
      "loss": 2.6816,
      "step": 386100
    },
    {
      "activations/layer0_attention_weight_max": 17.07346534729004,
      "activations/layer0_attention_weight_min": -13.543396949768066,
      "activations/layer10_attention_weight_max": 36.247833251953125,
      "activations/layer10_attention_weight_min": -36.27203369140625,
      "activations/layer11_attention_weight_max": 36.913185119628906,
      "activations/layer11_attention_weight_min": -36.179649353027344,
      "activations/layer12_attention_weight_max": 28.096261978149414,
      "activations/layer12_attention_weight_min": -27.373130798339844,
      "activations/layer13_attention_weight_max": 42.34337615966797,
      "activations/layer13_attention_weight_min": -35.0931396484375,
      "activations/layer14_attention_weight_max": 43.34097671508789,
      "activations/layer14_attention_weight_min": -37.039485931396484,
      "activations/layer15_attention_weight_max": 41.174217224121094,
      "activations/layer15_attention_weight_min": -35.65022277832031,
      "activations/layer16_attention_weight_max": 29.7564754486084,
      "activations/layer16_attention_weight_min": -29.01055145263672,
      "activations/layer17_attention_weight_max": 32.04777526855469,
      "activations/layer17_attention_weight_min": -27.774253845214844,
      "activations/layer18_attention_weight_max": 35.78086853027344,
      "activations/layer18_attention_weight_min": -28.268037796020508,
      "activations/layer19_attention_weight_max": 39.88825607299805,
      "activations/layer19_attention_weight_min": -33.56769561767578,
      "activations/layer1_attention_weight_max": 15.92618465423584,
      "activations/layer1_attention_weight_min": -14.459371566772461,
      "activations/layer20_attention_weight_max": 30.360782623291016,
      "activations/layer20_attention_weight_min": -24.671995162963867,
      "activations/layer21_attention_weight_max": 30.333852767944336,
      "activations/layer21_attention_weight_min": -25.744827270507812,
      "activations/layer22_attention_weight_max": 43.65399169921875,
      "activations/layer22_attention_weight_min": -32.7977180480957,
      "activations/layer23_attention_weight_max": 33.92081832885742,
      "activations/layer23_attention_weight_min": -26.932239532470703,
      "activations/layer2_attention_weight_max": 34.58104705810547,
      "activations/layer2_attention_weight_min": -33.768768310546875,
      "activations/layer3_attention_weight_max": 108.3896713256836,
      "activations/layer3_attention_weight_min": -107.24346160888672,
      "activations/layer4_attention_weight_max": 111.2647933959961,
      "activations/layer4_attention_weight_min": -115.76991271972656,
      "activations/layer5_attention_weight_max": 51.19327926635742,
      "activations/layer5_attention_weight_min": -64.00090789794922,
      "activations/layer6_attention_weight_max": 46.19435119628906,
      "activations/layer6_attention_weight_min": -47.802852630615234,
      "activations/layer7_attention_weight_max": 99.32200622558594,
      "activations/layer7_attention_weight_min": -98.8701171875,
      "activations/layer8_attention_weight_max": 45.25120544433594,
      "activations/layer8_attention_weight_min": -46.6754035949707,
      "activations/layer9_attention_weight_max": 32.225894927978516,
      "activations/layer9_attention_weight_min": -35.69468307495117,
      "epoch": 22.44,
      "learning_rate": 5.359848484848485e-06,
      "loss": 2.7034,
      "step": 386150
    },
    {
      "activations/layer0_attention_weight_max": 16.996877670288086,
      "activations/layer0_attention_weight_min": -15.162797927856445,
      "activations/layer10_attention_weight_max": 36.922874450683594,
      "activations/layer10_attention_weight_min": -35.510589599609375,
      "activations/layer11_attention_weight_max": 35.22251892089844,
      "activations/layer11_attention_weight_min": -35.724422454833984,
      "activations/layer12_attention_weight_max": 26.38835334777832,
      "activations/layer12_attention_weight_min": -29.129491806030273,
      "activations/layer13_attention_weight_max": 38.777793884277344,
      "activations/layer13_attention_weight_min": -36.45301055908203,
      "activations/layer14_attention_weight_max": 41.77626037597656,
      "activations/layer14_attention_weight_min": -39.49092102050781,
      "activations/layer15_attention_weight_max": 39.815338134765625,
      "activations/layer15_attention_weight_min": -37.548892974853516,
      "activations/layer16_attention_weight_max": 28.473800659179688,
      "activations/layer16_attention_weight_min": -27.081737518310547,
      "activations/layer17_attention_weight_max": 29.448564529418945,
      "activations/layer17_attention_weight_min": -25.94168472290039,
      "activations/layer18_attention_weight_max": 31.95376205444336,
      "activations/layer18_attention_weight_min": -26.534442901611328,
      "activations/layer19_attention_weight_max": 33.57529830932617,
      "activations/layer19_attention_weight_min": -31.710346221923828,
      "activations/layer1_attention_weight_max": 15.95485782623291,
      "activations/layer1_attention_weight_min": -13.930912017822266,
      "activations/layer20_attention_weight_max": 29.311800003051758,
      "activations/layer20_attention_weight_min": -26.160825729370117,
      "activations/layer21_attention_weight_max": 29.14191246032715,
      "activations/layer21_attention_weight_min": -24.845609664916992,
      "activations/layer22_attention_weight_max": 41.6823844909668,
      "activations/layer22_attention_weight_min": -30.657222747802734,
      "activations/layer23_attention_weight_max": 32.736778259277344,
      "activations/layer23_attention_weight_min": -25.96626853942871,
      "activations/layer2_attention_weight_max": 33.89200973510742,
      "activations/layer2_attention_weight_min": -33.83085632324219,
      "activations/layer3_attention_weight_max": 107.9386978149414,
      "activations/layer3_attention_weight_min": -108.95549011230469,
      "activations/layer4_attention_weight_max": 112.99224853515625,
      "activations/layer4_attention_weight_min": -111.2713623046875,
      "activations/layer5_attention_weight_max": 52.71625900268555,
      "activations/layer5_attention_weight_min": -61.58965301513672,
      "activations/layer6_attention_weight_max": 46.38554763793945,
      "activations/layer6_attention_weight_min": -50.33500289916992,
      "activations/layer7_attention_weight_max": 104.14673614501953,
      "activations/layer7_attention_weight_min": -105.55657196044922,
      "activations/layer8_attention_weight_max": 43.21261215209961,
      "activations/layer8_attention_weight_min": -47.42180252075195,
      "activations/layer9_attention_weight_max": 33.40298080444336,
      "activations/layer9_attention_weight_min": -35.39793014526367,
      "epoch": 22.44,
      "learning_rate": 5.3409090909090905e-06,
      "loss": 2.6928,
      "step": 386200
    },
    {
      "activations/layer0_attention_weight_max": 16.74251937866211,
      "activations/layer0_attention_weight_min": -14.075841903686523,
      "activations/layer10_attention_weight_max": 37.41059112548828,
      "activations/layer10_attention_weight_min": -38.886016845703125,
      "activations/layer11_attention_weight_max": 36.137569427490234,
      "activations/layer11_attention_weight_min": -37.76364517211914,
      "activations/layer12_attention_weight_max": 31.25078773498535,
      "activations/layer12_attention_weight_min": -32.47904586791992,
      "activations/layer13_attention_weight_max": 42.45216751098633,
      "activations/layer13_attention_weight_min": -41.1279296875,
      "activations/layer14_attention_weight_max": 49.66415786743164,
      "activations/layer14_attention_weight_min": -46.60881042480469,
      "activations/layer15_attention_weight_max": 43.845096588134766,
      "activations/layer15_attention_weight_min": -41.555057525634766,
      "activations/layer16_attention_weight_max": 32.938079833984375,
      "activations/layer16_attention_weight_min": -30.40244483947754,
      "activations/layer17_attention_weight_max": 31.051870346069336,
      "activations/layer17_attention_weight_min": -29.774091720581055,
      "activations/layer18_attention_weight_max": 33.536678314208984,
      "activations/layer18_attention_weight_min": -27.03314781188965,
      "activations/layer19_attention_weight_max": 41.98468017578125,
      "activations/layer19_attention_weight_min": -33.703243255615234,
      "activations/layer1_attention_weight_max": 15.482189178466797,
      "activations/layer1_attention_weight_min": -14.918681144714355,
      "activations/layer20_attention_weight_max": 34.187232971191406,
      "activations/layer20_attention_weight_min": -26.193504333496094,
      "activations/layer21_attention_weight_max": 35.04559326171875,
      "activations/layer21_attention_weight_min": -24.62013816833496,
      "activations/layer22_attention_weight_max": 46.15646743774414,
      "activations/layer22_attention_weight_min": -34.60289001464844,
      "activations/layer23_attention_weight_max": 32.23025131225586,
      "activations/layer23_attention_weight_min": -27.205711364746094,
      "activations/layer2_attention_weight_max": 35.43935775756836,
      "activations/layer2_attention_weight_min": -35.092708587646484,
      "activations/layer3_attention_weight_max": 110.92523956298828,
      "activations/layer3_attention_weight_min": -111.22582244873047,
      "activations/layer4_attention_weight_max": 110.79082489013672,
      "activations/layer4_attention_weight_min": -113.82814025878906,
      "activations/layer5_attention_weight_max": 53.0791130065918,
      "activations/layer5_attention_weight_min": -62.263587951660156,
      "activations/layer6_attention_weight_max": 48.1372184753418,
      "activations/layer6_attention_weight_min": -51.268062591552734,
      "activations/layer7_attention_weight_max": 101.09404754638672,
      "activations/layer7_attention_weight_min": -109.00723266601562,
      "activations/layer8_attention_weight_max": 45.89588928222656,
      "activations/layer8_attention_weight_min": -50.0890998840332,
      "activations/layer9_attention_weight_max": 35.15360641479492,
      "activations/layer9_attention_weight_min": -38.778934478759766,
      "epoch": 22.44,
      "learning_rate": 5.321969696969696e-06,
      "loss": 2.6994,
      "step": 386250
    },
    {
      "activations/layer0_attention_weight_max": 16.52796745300293,
      "activations/layer0_attention_weight_min": -14.785323143005371,
      "activations/layer10_attention_weight_max": 38.308101654052734,
      "activations/layer10_attention_weight_min": -39.6543083190918,
      "activations/layer11_attention_weight_max": 37.65642166137695,
      "activations/layer11_attention_weight_min": -38.88157272338867,
      "activations/layer12_attention_weight_max": 28.45016098022461,
      "activations/layer12_attention_weight_min": -28.8089542388916,
      "activations/layer13_attention_weight_max": 42.22752380371094,
      "activations/layer13_attention_weight_min": -37.972312927246094,
      "activations/layer14_attention_weight_max": 45.20964431762695,
      "activations/layer14_attention_weight_min": -42.18082809448242,
      "activations/layer15_attention_weight_max": 40.35434341430664,
      "activations/layer15_attention_weight_min": -38.57733917236328,
      "activations/layer16_attention_weight_max": 33.06957244873047,
      "activations/layer16_attention_weight_min": -29.539751052856445,
      "activations/layer17_attention_weight_max": 30.71994400024414,
      "activations/layer17_attention_weight_min": -27.9176025390625,
      "activations/layer18_attention_weight_max": 33.421451568603516,
      "activations/layer18_attention_weight_min": -28.13917350769043,
      "activations/layer19_attention_weight_max": 34.276153564453125,
      "activations/layer19_attention_weight_min": -31.544370651245117,
      "activations/layer1_attention_weight_max": 15.288056373596191,
      "activations/layer1_attention_weight_min": -14.847589492797852,
      "activations/layer20_attention_weight_max": 30.141517639160156,
      "activations/layer20_attention_weight_min": -26.232383728027344,
      "activations/layer21_attention_weight_max": 32.019264221191406,
      "activations/layer21_attention_weight_min": -25.02623176574707,
      "activations/layer22_attention_weight_max": 44.82508850097656,
      "activations/layer22_attention_weight_min": -33.14064025878906,
      "activations/layer23_attention_weight_max": 34.02800750732422,
      "activations/layer23_attention_weight_min": -29.055007934570312,
      "activations/layer2_attention_weight_max": 33.54240417480469,
      "activations/layer2_attention_weight_min": -34.16472625732422,
      "activations/layer3_attention_weight_max": 110.98275756835938,
      "activations/layer3_attention_weight_min": -112.72152709960938,
      "activations/layer4_attention_weight_max": 111.7435302734375,
      "activations/layer4_attention_weight_min": -118.71761322021484,
      "activations/layer5_attention_weight_max": 53.978515625,
      "activations/layer5_attention_weight_min": -64.85543060302734,
      "activations/layer6_attention_weight_max": 50.2441291809082,
      "activations/layer6_attention_weight_min": -52.8681755065918,
      "activations/layer7_attention_weight_max": 100.01551818847656,
      "activations/layer7_attention_weight_min": -112.92823791503906,
      "activations/layer8_attention_weight_max": 46.951202392578125,
      "activations/layer8_attention_weight_min": -49.46106719970703,
      "activations/layer9_attention_weight_max": 34.525142669677734,
      "activations/layer9_attention_weight_min": -38.91780090332031,
      "epoch": 22.45,
      "learning_rate": 5.303030303030302e-06,
      "loss": 2.6963,
      "step": 386300
    },
    {
      "activations/layer0_attention_weight_max": 16.943767547607422,
      "activations/layer0_attention_weight_min": -14.752193450927734,
      "activations/layer10_attention_weight_max": 37.981292724609375,
      "activations/layer10_attention_weight_min": -37.646907806396484,
      "activations/layer11_attention_weight_max": 37.56756591796875,
      "activations/layer11_attention_weight_min": -36.438011169433594,
      "activations/layer12_attention_weight_max": 32.0144157409668,
      "activations/layer12_attention_weight_min": -28.684141159057617,
      "activations/layer13_attention_weight_max": 50.68437957763672,
      "activations/layer13_attention_weight_min": -38.799522399902344,
      "activations/layer14_attention_weight_max": 54.43152618408203,
      "activations/layer14_attention_weight_min": -41.55953598022461,
      "activations/layer15_attention_weight_max": 50.207237243652344,
      "activations/layer15_attention_weight_min": -40.04620361328125,
      "activations/layer16_attention_weight_max": 37.833335876464844,
      "activations/layer16_attention_weight_min": -31.355512619018555,
      "activations/layer17_attention_weight_max": 35.309688568115234,
      "activations/layer17_attention_weight_min": -29.041616439819336,
      "activations/layer18_attention_weight_max": 38.84300231933594,
      "activations/layer18_attention_weight_min": -30.988859176635742,
      "activations/layer19_attention_weight_max": 43.04465103149414,
      "activations/layer19_attention_weight_min": -35.96651077270508,
      "activations/layer1_attention_weight_max": 16.034862518310547,
      "activations/layer1_attention_weight_min": -15.249210357666016,
      "activations/layer20_attention_weight_max": 37.217525482177734,
      "activations/layer20_attention_weight_min": -26.593727111816406,
      "activations/layer21_attention_weight_max": 36.863372802734375,
      "activations/layer21_attention_weight_min": -25.736486434936523,
      "activations/layer22_attention_weight_max": 62.78713607788086,
      "activations/layer22_attention_weight_min": -35.69882583618164,
      "activations/layer23_attention_weight_max": 41.438926696777344,
      "activations/layer23_attention_weight_min": -27.577816009521484,
      "activations/layer2_attention_weight_max": 35.39153289794922,
      "activations/layer2_attention_weight_min": -33.93440246582031,
      "activations/layer3_attention_weight_max": 110.97997283935547,
      "activations/layer3_attention_weight_min": -109.33436584472656,
      "activations/layer4_attention_weight_max": 113.19915771484375,
      "activations/layer4_attention_weight_min": -114.12345123291016,
      "activations/layer5_attention_weight_max": 50.99615478515625,
      "activations/layer5_attention_weight_min": -65.93302917480469,
      "activations/layer6_attention_weight_max": 46.66325378417969,
      "activations/layer6_attention_weight_min": -48.04120635986328,
      "activations/layer7_attention_weight_max": 103.09278869628906,
      "activations/layer7_attention_weight_min": -102.0805435180664,
      "activations/layer8_attention_weight_max": 46.778846740722656,
      "activations/layer8_attention_weight_min": -48.834842681884766,
      "activations/layer9_attention_weight_max": 35.35598373413086,
      "activations/layer9_attention_weight_min": -37.447322845458984,
      "epoch": 22.45,
      "learning_rate": 5.2840909090909086e-06,
      "loss": 2.6969,
      "step": 386350
    },
    {
      "activations/layer0_attention_weight_max": 16.38559341430664,
      "activations/layer0_attention_weight_min": -14.870223999023438,
      "activations/layer10_attention_weight_max": 31.80849266052246,
      "activations/layer10_attention_weight_min": -34.614566802978516,
      "activations/layer11_attention_weight_max": 32.48014831542969,
      "activations/layer11_attention_weight_min": -33.739990234375,
      "activations/layer12_attention_weight_max": 27.037084579467773,
      "activations/layer12_attention_weight_min": -26.70693588256836,
      "activations/layer13_attention_weight_max": 40.580162048339844,
      "activations/layer13_attention_weight_min": -35.32307434082031,
      "activations/layer14_attention_weight_max": 43.805938720703125,
      "activations/layer14_attention_weight_min": -37.63047409057617,
      "activations/layer15_attention_weight_max": 42.2032470703125,
      "activations/layer15_attention_weight_min": -38.07133102416992,
      "activations/layer16_attention_weight_max": 35.22602081298828,
      "activations/layer16_attention_weight_min": -28.80196762084961,
      "activations/layer17_attention_weight_max": 30.863807678222656,
      "activations/layer17_attention_weight_min": -26.885705947875977,
      "activations/layer18_attention_weight_max": 35.18254852294922,
      "activations/layer18_attention_weight_min": -27.299301147460938,
      "activations/layer19_attention_weight_max": 36.66496658325195,
      "activations/layer19_attention_weight_min": -33.961021423339844,
      "activations/layer1_attention_weight_max": 16.63758659362793,
      "activations/layer1_attention_weight_min": -16.074871063232422,
      "activations/layer20_attention_weight_max": 29.694955825805664,
      "activations/layer20_attention_weight_min": -25.42540168762207,
      "activations/layer21_attention_weight_max": 31.60576629638672,
      "activations/layer21_attention_weight_min": -25.261138916015625,
      "activations/layer22_attention_weight_max": 45.01526641845703,
      "activations/layer22_attention_weight_min": -32.462093353271484,
      "activations/layer23_attention_weight_max": 32.87889862060547,
      "activations/layer23_attention_weight_min": -27.060956954956055,
      "activations/layer2_attention_weight_max": 34.32462692260742,
      "activations/layer2_attention_weight_min": -34.28489685058594,
      "activations/layer3_attention_weight_max": 109.84676361083984,
      "activations/layer3_attention_weight_min": -112.30872344970703,
      "activations/layer4_attention_weight_max": 111.25538635253906,
      "activations/layer4_attention_weight_min": -119.17839813232422,
      "activations/layer5_attention_weight_max": 52.41501998901367,
      "activations/layer5_attention_weight_min": -62.79424285888672,
      "activations/layer6_attention_weight_max": 46.41861343383789,
      "activations/layer6_attention_weight_min": -48.790382385253906,
      "activations/layer7_attention_weight_max": 96.14976501464844,
      "activations/layer7_attention_weight_min": -100.94776153564453,
      "activations/layer8_attention_weight_max": 42.35833740234375,
      "activations/layer8_attention_weight_min": -44.431705474853516,
      "activations/layer9_attention_weight_max": 31.2822265625,
      "activations/layer9_attention_weight_min": -35.09526824951172,
      "epoch": 22.45,
      "learning_rate": 5.265151515151514e-06,
      "loss": 2.7093,
      "step": 386400
    },
    {
      "activations/layer0_attention_weight_max": 16.99148941040039,
      "activations/layer0_attention_weight_min": -13.986989974975586,
      "activations/layer10_attention_weight_max": 34.5513801574707,
      "activations/layer10_attention_weight_min": -35.937095642089844,
      "activations/layer11_attention_weight_max": 35.080596923828125,
      "activations/layer11_attention_weight_min": -35.709232330322266,
      "activations/layer12_attention_weight_max": 26.923297882080078,
      "activations/layer12_attention_weight_min": -26.398487091064453,
      "activations/layer13_attention_weight_max": 37.251220703125,
      "activations/layer13_attention_weight_min": -35.245689392089844,
      "activations/layer14_attention_weight_max": 38.991416931152344,
      "activations/layer14_attention_weight_min": -38.841094970703125,
      "activations/layer15_attention_weight_max": 38.46061325073242,
      "activations/layer15_attention_weight_min": -36.621604919433594,
      "activations/layer16_attention_weight_max": 28.007829666137695,
      "activations/layer16_attention_weight_min": -27.412460327148438,
      "activations/layer17_attention_weight_max": 27.611730575561523,
      "activations/layer17_attention_weight_min": -24.814847946166992,
      "activations/layer18_attention_weight_max": 31.185302734375,
      "activations/layer18_attention_weight_min": -25.47280502319336,
      "activations/layer19_attention_weight_max": 36.4815788269043,
      "activations/layer19_attention_weight_min": -30.121068954467773,
      "activations/layer1_attention_weight_max": 16.396312713623047,
      "activations/layer1_attention_weight_min": -14.668811798095703,
      "activations/layer20_attention_weight_max": 28.396759033203125,
      "activations/layer20_attention_weight_min": -24.504226684570312,
      "activations/layer21_attention_weight_max": 27.217397689819336,
      "activations/layer21_attention_weight_min": -25.4226131439209,
      "activations/layer22_attention_weight_max": 37.11166763305664,
      "activations/layer22_attention_weight_min": -29.87190818786621,
      "activations/layer23_attention_weight_max": 29.92340660095215,
      "activations/layer23_attention_weight_min": -25.469924926757812,
      "activations/layer2_attention_weight_max": 34.25891876220703,
      "activations/layer2_attention_weight_min": -34.913700103759766,
      "activations/layer3_attention_weight_max": 108.14344024658203,
      "activations/layer3_attention_weight_min": -113.16044616699219,
      "activations/layer4_attention_weight_max": 110.76725006103516,
      "activations/layer4_attention_weight_min": -113.2959976196289,
      "activations/layer5_attention_weight_max": 50.85118865966797,
      "activations/layer5_attention_weight_min": -60.52455139160156,
      "activations/layer6_attention_weight_max": 45.6835823059082,
      "activations/layer6_attention_weight_min": -46.48720932006836,
      "activations/layer7_attention_weight_max": 99.69200134277344,
      "activations/layer7_attention_weight_min": -101.7221450805664,
      "activations/layer8_attention_weight_max": 41.29487228393555,
      "activations/layer8_attention_weight_min": -46.6242790222168,
      "activations/layer9_attention_weight_max": 32.18845748901367,
      "activations/layer9_attention_weight_min": -36.40816879272461,
      "epoch": 22.45,
      "learning_rate": 5.246212121212121e-06,
      "loss": 2.6888,
      "step": 386450
    },
    {
      "activations/layer0_attention_weight_max": 16.42313575744629,
      "activations/layer0_attention_weight_min": -13.943207740783691,
      "activations/layer10_attention_weight_max": 38.07185363769531,
      "activations/layer10_attention_weight_min": -36.9608154296875,
      "activations/layer11_attention_weight_max": 38.47911071777344,
      "activations/layer11_attention_weight_min": -39.433475494384766,
      "activations/layer12_attention_weight_max": 30.172176361083984,
      "activations/layer12_attention_weight_min": -29.31662368774414,
      "activations/layer13_attention_weight_max": 42.98347091674805,
      "activations/layer13_attention_weight_min": -38.86012268066406,
      "activations/layer14_attention_weight_max": 45.272674560546875,
      "activations/layer14_attention_weight_min": -42.98493576049805,
      "activations/layer15_attention_weight_max": 43.55910873413086,
      "activations/layer15_attention_weight_min": -42.19118118286133,
      "activations/layer16_attention_weight_max": 33.474735260009766,
      "activations/layer16_attention_weight_min": -29.717451095581055,
      "activations/layer17_attention_weight_max": 30.30379867553711,
      "activations/layer17_attention_weight_min": -27.62786293029785,
      "activations/layer18_attention_weight_max": 32.965003967285156,
      "activations/layer18_attention_weight_min": -27.482389450073242,
      "activations/layer19_attention_weight_max": 37.402557373046875,
      "activations/layer19_attention_weight_min": -35.03994369506836,
      "activations/layer1_attention_weight_max": 16.011625289916992,
      "activations/layer1_attention_weight_min": -14.61975383758545,
      "activations/layer20_attention_weight_max": 30.630083084106445,
      "activations/layer20_attention_weight_min": -26.672645568847656,
      "activations/layer21_attention_weight_max": 30.566038131713867,
      "activations/layer21_attention_weight_min": -26.447174072265625,
      "activations/layer22_attention_weight_max": 41.87100601196289,
      "activations/layer22_attention_weight_min": -33.58302688598633,
      "activations/layer23_attention_weight_max": 32.57781219482422,
      "activations/layer23_attention_weight_min": -26.712461471557617,
      "activations/layer2_attention_weight_max": 34.907413482666016,
      "activations/layer2_attention_weight_min": -33.48006057739258,
      "activations/layer3_attention_weight_max": 111.44735717773438,
      "activations/layer3_attention_weight_min": -107.8851089477539,
      "activations/layer4_attention_weight_max": 114.6341552734375,
      "activations/layer4_attention_weight_min": -115.92098236083984,
      "activations/layer5_attention_weight_max": 51.639892578125,
      "activations/layer5_attention_weight_min": -62.42003631591797,
      "activations/layer6_attention_weight_max": 47.93312454223633,
      "activations/layer6_attention_weight_min": -48.61024856567383,
      "activations/layer7_attention_weight_max": 106.79492950439453,
      "activations/layer7_attention_weight_min": -106.93612670898438,
      "activations/layer8_attention_weight_max": 48.775962829589844,
      "activations/layer8_attention_weight_min": -49.99322509765625,
      "activations/layer9_attention_weight_max": 35.35256576538086,
      "activations/layer9_attention_weight_min": -38.58361053466797,
      "epoch": 22.46,
      "learning_rate": 5.2272727272727274e-06,
      "loss": 2.6929,
      "step": 386500
    },
    {
      "activations/layer0_attention_weight_max": 16.89948272705078,
      "activations/layer0_attention_weight_min": -15.606374740600586,
      "activations/layer10_attention_weight_max": 34.842247009277344,
      "activations/layer10_attention_weight_min": -35.26712417602539,
      "activations/layer11_attention_weight_max": 35.46981430053711,
      "activations/layer11_attention_weight_min": -36.568138122558594,
      "activations/layer12_attention_weight_max": 28.403207778930664,
      "activations/layer12_attention_weight_min": -28.696317672729492,
      "activations/layer13_attention_weight_max": 41.35810852050781,
      "activations/layer13_attention_weight_min": -37.59022521972656,
      "activations/layer14_attention_weight_max": 43.11408615112305,
      "activations/layer14_attention_weight_min": -39.70808029174805,
      "activations/layer15_attention_weight_max": 40.261993408203125,
      "activations/layer15_attention_weight_min": -37.57036209106445,
      "activations/layer16_attention_weight_max": 30.305089950561523,
      "activations/layer16_attention_weight_min": -28.78506851196289,
      "activations/layer17_attention_weight_max": 31.576013565063477,
      "activations/layer17_attention_weight_min": -26.038217544555664,
      "activations/layer18_attention_weight_max": 35.70625686645508,
      "activations/layer18_attention_weight_min": -28.645774841308594,
      "activations/layer19_attention_weight_max": 37.862060546875,
      "activations/layer19_attention_weight_min": -33.18964767456055,
      "activations/layer1_attention_weight_max": 15.499563217163086,
      "activations/layer1_attention_weight_min": -15.4367036819458,
      "activations/layer20_attention_weight_max": 32.14634323120117,
      "activations/layer20_attention_weight_min": -25.99835777282715,
      "activations/layer21_attention_weight_max": 32.78137969970703,
      "activations/layer21_attention_weight_min": -26.595903396606445,
      "activations/layer22_attention_weight_max": 45.84912872314453,
      "activations/layer22_attention_weight_min": -33.195526123046875,
      "activations/layer23_attention_weight_max": 35.88499069213867,
      "activations/layer23_attention_weight_min": -27.37747573852539,
      "activations/layer2_attention_weight_max": 34.044273376464844,
      "activations/layer2_attention_weight_min": -34.30369186401367,
      "activations/layer3_attention_weight_max": 106.82777404785156,
      "activations/layer3_attention_weight_min": -109.38511657714844,
      "activations/layer4_attention_weight_max": 114.34671783447266,
      "activations/layer4_attention_weight_min": -113.36824798583984,
      "activations/layer5_attention_weight_max": 50.209388732910156,
      "activations/layer5_attention_weight_min": -62.4654655456543,
      "activations/layer6_attention_weight_max": 47.810585021972656,
      "activations/layer6_attention_weight_min": -49.62495040893555,
      "activations/layer7_attention_weight_max": 96.97713470458984,
      "activations/layer7_attention_weight_min": -105.84576416015625,
      "activations/layer8_attention_weight_max": 43.901424407958984,
      "activations/layer8_attention_weight_min": -47.345394134521484,
      "activations/layer9_attention_weight_max": 31.7904052734375,
      "activations/layer9_attention_weight_min": -36.35383224487305,
      "epoch": 22.46,
      "learning_rate": 5.208333333333333e-06,
      "loss": 2.6913,
      "step": 386550
    },
    {
      "activations/layer0_attention_weight_max": 16.738500595092773,
      "activations/layer0_attention_weight_min": -15.38943862915039,
      "activations/layer10_attention_weight_max": 35.851253509521484,
      "activations/layer10_attention_weight_min": -36.601749420166016,
      "activations/layer11_attention_weight_max": 36.01289749145508,
      "activations/layer11_attention_weight_min": -37.899986267089844,
      "activations/layer12_attention_weight_max": 27.88511848449707,
      "activations/layer12_attention_weight_min": -27.967145919799805,
      "activations/layer13_attention_weight_max": 41.13898468017578,
      "activations/layer13_attention_weight_min": -37.469459533691406,
      "activations/layer14_attention_weight_max": 44.49375915527344,
      "activations/layer14_attention_weight_min": -39.22938919067383,
      "activations/layer15_attention_weight_max": 42.786766052246094,
      "activations/layer15_attention_weight_min": -37.706092834472656,
      "activations/layer16_attention_weight_max": 31.862964630126953,
      "activations/layer16_attention_weight_min": -30.30838966369629,
      "activations/layer17_attention_weight_max": 31.000295639038086,
      "activations/layer17_attention_weight_min": -27.96556854248047,
      "activations/layer18_attention_weight_max": 35.60081100463867,
      "activations/layer18_attention_weight_min": -26.990201950073242,
      "activations/layer19_attention_weight_max": 35.42918014526367,
      "activations/layer19_attention_weight_min": -34.1855583190918,
      "activations/layer1_attention_weight_max": 16.68222999572754,
      "activations/layer1_attention_weight_min": -15.912864685058594,
      "activations/layer20_attention_weight_max": 29.885902404785156,
      "activations/layer20_attention_weight_min": -26.052928924560547,
      "activations/layer21_attention_weight_max": 29.10161590576172,
      "activations/layer21_attention_weight_min": -25.27559471130371,
      "activations/layer22_attention_weight_max": 47.03497314453125,
      "activations/layer22_attention_weight_min": -31.780818939208984,
      "activations/layer23_attention_weight_max": 32.913368225097656,
      "activations/layer23_attention_weight_min": -25.58814811706543,
      "activations/layer2_attention_weight_max": 36.10662078857422,
      "activations/layer2_attention_weight_min": -34.855342864990234,
      "activations/layer3_attention_weight_max": 113.62586212158203,
      "activations/layer3_attention_weight_min": -108.50433349609375,
      "activations/layer4_attention_weight_max": 120.52565002441406,
      "activations/layer4_attention_weight_min": -116.0588607788086,
      "activations/layer5_attention_weight_max": 52.831878662109375,
      "activations/layer5_attention_weight_min": -66.93209838867188,
      "activations/layer6_attention_weight_max": 48.877708435058594,
      "activations/layer6_attention_weight_min": -51.759334564208984,
      "activations/layer7_attention_weight_max": 105.46549224853516,
      "activations/layer7_attention_weight_min": -104.97148132324219,
      "activations/layer8_attention_weight_max": 45.30245590209961,
      "activations/layer8_attention_weight_min": -49.794281005859375,
      "activations/layer9_attention_weight_max": 32.17005157470703,
      "activations/layer9_attention_weight_min": -36.2941780090332,
      "epoch": 22.46,
      "learning_rate": 5.189393939393939e-06,
      "loss": 2.712,
      "step": 386600
    },
    {
      "activations/layer0_attention_weight_max": 17.42641830444336,
      "activations/layer0_attention_weight_min": -15.899163246154785,
      "activations/layer10_attention_weight_max": 37.294471740722656,
      "activations/layer10_attention_weight_min": -37.55479049682617,
      "activations/layer11_attention_weight_max": 36.7173957824707,
      "activations/layer11_attention_weight_min": -37.64742660522461,
      "activations/layer12_attention_weight_max": 29.58064079284668,
      "activations/layer12_attention_weight_min": -28.676652908325195,
      "activations/layer13_attention_weight_max": 43.58454895019531,
      "activations/layer13_attention_weight_min": -38.63295364379883,
      "activations/layer14_attention_weight_max": 46.37543487548828,
      "activations/layer14_attention_weight_min": -40.521209716796875,
      "activations/layer15_attention_weight_max": 42.09927749633789,
      "activations/layer15_attention_weight_min": -38.756736755371094,
      "activations/layer16_attention_weight_max": 30.468793869018555,
      "activations/layer16_attention_weight_min": -27.976659774780273,
      "activations/layer17_attention_weight_max": 28.49644660949707,
      "activations/layer17_attention_weight_min": -24.591459274291992,
      "activations/layer18_attention_weight_max": 33.133819580078125,
      "activations/layer18_attention_weight_min": -24.89229393005371,
      "activations/layer19_attention_weight_max": 34.35782241821289,
      "activations/layer19_attention_weight_min": -31.337438583374023,
      "activations/layer1_attention_weight_max": 15.488581657409668,
      "activations/layer1_attention_weight_min": -15.377284049987793,
      "activations/layer20_attention_weight_max": 28.04466438293457,
      "activations/layer20_attention_weight_min": -24.06513786315918,
      "activations/layer21_attention_weight_max": 27.11603355407715,
      "activations/layer21_attention_weight_min": -22.974403381347656,
      "activations/layer22_attention_weight_max": 39.375282287597656,
      "activations/layer22_attention_weight_min": -30.166053771972656,
      "activations/layer23_attention_weight_max": 30.386783599853516,
      "activations/layer23_attention_weight_min": -24.319692611694336,
      "activations/layer2_attention_weight_max": 34.40813064575195,
      "activations/layer2_attention_weight_min": -33.673770904541016,
      "activations/layer3_attention_weight_max": 109.03245544433594,
      "activations/layer3_attention_weight_min": -110.2799301147461,
      "activations/layer4_attention_weight_max": 118.2926025390625,
      "activations/layer4_attention_weight_min": -114.8013687133789,
      "activations/layer5_attention_weight_max": 50.61118698120117,
      "activations/layer5_attention_weight_min": -60.80108642578125,
      "activations/layer6_attention_weight_max": 47.74565887451172,
      "activations/layer6_attention_weight_min": -49.16240692138672,
      "activations/layer7_attention_weight_max": 98.6158676147461,
      "activations/layer7_attention_weight_min": -103.30084228515625,
      "activations/layer8_attention_weight_max": 43.67244338989258,
      "activations/layer8_attention_weight_min": -47.74158477783203,
      "activations/layer9_attention_weight_max": 35.90904998779297,
      "activations/layer9_attention_weight_min": -36.18669509887695,
      "epoch": 22.47,
      "learning_rate": 5.170454545454545e-06,
      "loss": 2.6864,
      "step": 386650
    },
    {
      "activations/layer0_attention_weight_max": 16.695348739624023,
      "activations/layer0_attention_weight_min": -14.045242309570312,
      "activations/layer10_attention_weight_max": 34.27822494506836,
      "activations/layer10_attention_weight_min": -35.920021057128906,
      "activations/layer11_attention_weight_max": 33.275428771972656,
      "activations/layer11_attention_weight_min": -34.19103240966797,
      "activations/layer12_attention_weight_max": 27.68956756591797,
      "activations/layer12_attention_weight_min": -28.641788482666016,
      "activations/layer13_attention_weight_max": 39.62681198120117,
      "activations/layer13_attention_weight_min": -38.43703079223633,
      "activations/layer14_attention_weight_max": 41.95915603637695,
      "activations/layer14_attention_weight_min": -39.92257308959961,
      "activations/layer15_attention_weight_max": 38.17393112182617,
      "activations/layer15_attention_weight_min": -36.03862762451172,
      "activations/layer16_attention_weight_max": 27.762165069580078,
      "activations/layer16_attention_weight_min": -28.001251220703125,
      "activations/layer17_attention_weight_max": 28.6933536529541,
      "activations/layer17_attention_weight_min": -25.1812801361084,
      "activations/layer18_attention_weight_max": 30.891986846923828,
      "activations/layer18_attention_weight_min": -26.299312591552734,
      "activations/layer19_attention_weight_max": 32.685489654541016,
      "activations/layer19_attention_weight_min": -32.97319030761719,
      "activations/layer1_attention_weight_max": 17.036869049072266,
      "activations/layer1_attention_weight_min": -15.35051155090332,
      "activations/layer20_attention_weight_max": 26.526329040527344,
      "activations/layer20_attention_weight_min": -24.80596351623535,
      "activations/layer21_attention_weight_max": 28.158254623413086,
      "activations/layer21_attention_weight_min": -23.680644989013672,
      "activations/layer22_attention_weight_max": 38.63665771484375,
      "activations/layer22_attention_weight_min": -31.6075496673584,
      "activations/layer23_attention_weight_max": 30.687435150146484,
      "activations/layer23_attention_weight_min": -26.8239803314209,
      "activations/layer2_attention_weight_max": 33.56934356689453,
      "activations/layer2_attention_weight_min": -33.83612823486328,
      "activations/layer3_attention_weight_max": 105.28488159179688,
      "activations/layer3_attention_weight_min": -107.4201889038086,
      "activations/layer4_attention_weight_max": 110.51248931884766,
      "activations/layer4_attention_weight_min": -115.28446197509766,
      "activations/layer5_attention_weight_max": 51.84078598022461,
      "activations/layer5_attention_weight_min": -62.20907211303711,
      "activations/layer6_attention_weight_max": 45.85335922241211,
      "activations/layer6_attention_weight_min": -50.567806243896484,
      "activations/layer7_attention_weight_max": 96.93389892578125,
      "activations/layer7_attention_weight_min": -102.8214340209961,
      "activations/layer8_attention_weight_max": 42.83354949951172,
      "activations/layer8_attention_weight_min": -45.45974349975586,
      "activations/layer9_attention_weight_max": 33.69358444213867,
      "activations/layer9_attention_weight_min": -36.73976135253906,
      "epoch": 22.47,
      "learning_rate": 5.151515151515151e-06,
      "loss": 2.6956,
      "step": 386700
    },
    {
      "activations/layer0_attention_weight_max": 16.95492172241211,
      "activations/layer0_attention_weight_min": -16.047624588012695,
      "activations/layer10_attention_weight_max": 37.92023468017578,
      "activations/layer10_attention_weight_min": -37.53700256347656,
      "activations/layer11_attention_weight_max": 37.73067855834961,
      "activations/layer11_attention_weight_min": -36.144630432128906,
      "activations/layer12_attention_weight_max": 28.34265899658203,
      "activations/layer12_attention_weight_min": -27.783130645751953,
      "activations/layer13_attention_weight_max": 40.36763000488281,
      "activations/layer13_attention_weight_min": -38.18224334716797,
      "activations/layer14_attention_weight_max": 41.60441207885742,
      "activations/layer14_attention_weight_min": -41.29920196533203,
      "activations/layer15_attention_weight_max": 40.61951446533203,
      "activations/layer15_attention_weight_min": -38.266849517822266,
      "activations/layer16_attention_weight_max": 31.030248641967773,
      "activations/layer16_attention_weight_min": -28.893875122070312,
      "activations/layer17_attention_weight_max": 28.90753173828125,
      "activations/layer17_attention_weight_min": -26.140871047973633,
      "activations/layer18_attention_weight_max": 33.42975616455078,
      "activations/layer18_attention_weight_min": -26.887130737304688,
      "activations/layer19_attention_weight_max": 35.56309509277344,
      "activations/layer19_attention_weight_min": -34.319053649902344,
      "activations/layer1_attention_weight_max": 15.081339836120605,
      "activations/layer1_attention_weight_min": -15.723366737365723,
      "activations/layer20_attention_weight_max": 27.54677391052246,
      "activations/layer20_attention_weight_min": -24.804197311401367,
      "activations/layer21_attention_weight_max": 27.288076400756836,
      "activations/layer21_attention_weight_min": -25.744979858398438,
      "activations/layer22_attention_weight_max": 42.47361373901367,
      "activations/layer22_attention_weight_min": -31.665451049804688,
      "activations/layer23_attention_weight_max": 30.081602096557617,
      "activations/layer23_attention_weight_min": -26.008018493652344,
      "activations/layer2_attention_weight_max": 34.09083557128906,
      "activations/layer2_attention_weight_min": -35.2690315246582,
      "activations/layer3_attention_weight_max": 112.91065216064453,
      "activations/layer3_attention_weight_min": -112.91946411132812,
      "activations/layer4_attention_weight_max": 115.98397064208984,
      "activations/layer4_attention_weight_min": -118.70184326171875,
      "activations/layer5_attention_weight_max": 53.36365509033203,
      "activations/layer5_attention_weight_min": -66.16901397705078,
      "activations/layer6_attention_weight_max": 47.24827194213867,
      "activations/layer6_attention_weight_min": -48.798316955566406,
      "activations/layer7_attention_weight_max": 103.45780181884766,
      "activations/layer7_attention_weight_min": -108.44703674316406,
      "activations/layer8_attention_weight_max": 44.5352897644043,
      "activations/layer8_attention_weight_min": -45.709205627441406,
      "activations/layer9_attention_weight_max": 34.06919479370117,
      "activations/layer9_attention_weight_min": -37.545684814453125,
      "epoch": 22.47,
      "learning_rate": 5.132575757575757e-06,
      "loss": 2.7016,
      "step": 386750
    },
    {
      "activations/layer0_attention_weight_max": 16.723947525024414,
      "activations/layer0_attention_weight_min": -13.595094680786133,
      "activations/layer10_attention_weight_max": 35.64177703857422,
      "activations/layer10_attention_weight_min": -35.63440704345703,
      "activations/layer11_attention_weight_max": 33.85045623779297,
      "activations/layer11_attention_weight_min": -34.01659393310547,
      "activations/layer12_attention_weight_max": 27.562482833862305,
      "activations/layer12_attention_weight_min": -27.417261123657227,
      "activations/layer13_attention_weight_max": 40.31159973144531,
      "activations/layer13_attention_weight_min": -36.316287994384766,
      "activations/layer14_attention_weight_max": 42.87636947631836,
      "activations/layer14_attention_weight_min": -40.08201217651367,
      "activations/layer15_attention_weight_max": 38.87117004394531,
      "activations/layer15_attention_weight_min": -36.7791748046875,
      "activations/layer16_attention_weight_max": 28.03553009033203,
      "activations/layer16_attention_weight_min": -26.789657592773438,
      "activations/layer17_attention_weight_max": 31.68141746520996,
      "activations/layer17_attention_weight_min": -25.9595947265625,
      "activations/layer18_attention_weight_max": 31.265230178833008,
      "activations/layer18_attention_weight_min": -25.959890365600586,
      "activations/layer19_attention_weight_max": 31.26458168029785,
      "activations/layer19_attention_weight_min": -32.809932708740234,
      "activations/layer1_attention_weight_max": 16.130849838256836,
      "activations/layer1_attention_weight_min": -14.821026802062988,
      "activations/layer20_attention_weight_max": 26.963523864746094,
      "activations/layer20_attention_weight_min": -26.46331787109375,
      "activations/layer21_attention_weight_max": 26.348630905151367,
      "activations/layer21_attention_weight_min": -24.560415267944336,
      "activations/layer22_attention_weight_max": 42.25876998901367,
      "activations/layer22_attention_weight_min": -31.954017639160156,
      "activations/layer23_attention_weight_max": 30.03851890563965,
      "activations/layer23_attention_weight_min": -27.859281539916992,
      "activations/layer2_attention_weight_max": 35.21304702758789,
      "activations/layer2_attention_weight_min": -33.95716094970703,
      "activations/layer3_attention_weight_max": 106.2051773071289,
      "activations/layer3_attention_weight_min": -107.81397247314453,
      "activations/layer4_attention_weight_max": 111.02385711669922,
      "activations/layer4_attention_weight_min": -115.47210693359375,
      "activations/layer5_attention_weight_max": 49.168701171875,
      "activations/layer5_attention_weight_min": -61.247589111328125,
      "activations/layer6_attention_weight_max": 46.374656677246094,
      "activations/layer6_attention_weight_min": -47.138980865478516,
      "activations/layer7_attention_weight_max": 94.18547821044922,
      "activations/layer7_attention_weight_min": -104.19983673095703,
      "activations/layer8_attention_weight_max": 42.94929885864258,
      "activations/layer8_attention_weight_min": -44.570987701416016,
      "activations/layer9_attention_weight_max": 31.968732833862305,
      "activations/layer9_attention_weight_min": -35.14339065551758,
      "epoch": 22.48,
      "learning_rate": 5.113636363636363e-06,
      "loss": 2.6926,
      "step": 386800
    },
    {
      "activations/layer0_attention_weight_max": 17.2026424407959,
      "activations/layer0_attention_weight_min": -15.042255401611328,
      "activations/layer10_attention_weight_max": 35.366600036621094,
      "activations/layer10_attention_weight_min": -35.76579666137695,
      "activations/layer11_attention_weight_max": 34.40739822387695,
      "activations/layer11_attention_weight_min": -35.749916076660156,
      "activations/layer12_attention_weight_max": 26.950895309448242,
      "activations/layer12_attention_weight_min": -28.840330123901367,
      "activations/layer13_attention_weight_max": 40.16984558105469,
      "activations/layer13_attention_weight_min": -38.01777648925781,
      "activations/layer14_attention_weight_max": 41.165916442871094,
      "activations/layer14_attention_weight_min": -38.91453552246094,
      "activations/layer15_attention_weight_max": 39.17308807373047,
      "activations/layer15_attention_weight_min": -37.101966857910156,
      "activations/layer16_attention_weight_max": 31.338760375976562,
      "activations/layer16_attention_weight_min": -28.386079788208008,
      "activations/layer17_attention_weight_max": 29.4963321685791,
      "activations/layer17_attention_weight_min": -27.280609130859375,
      "activations/layer18_attention_weight_max": 35.32272720336914,
      "activations/layer18_attention_weight_min": -28.299577713012695,
      "activations/layer19_attention_weight_max": 34.34676742553711,
      "activations/layer19_attention_weight_min": -33.006866455078125,
      "activations/layer1_attention_weight_max": 15.698284149169922,
      "activations/layer1_attention_weight_min": -16.24458885192871,
      "activations/layer20_attention_weight_max": 28.480072021484375,
      "activations/layer20_attention_weight_min": -25.391845703125,
      "activations/layer21_attention_weight_max": 28.758140563964844,
      "activations/layer21_attention_weight_min": -25.12991714477539,
      "activations/layer22_attention_weight_max": 43.47990036010742,
      "activations/layer22_attention_weight_min": -33.77084732055664,
      "activations/layer23_attention_weight_max": 31.374671936035156,
      "activations/layer23_attention_weight_min": -27.998126983642578,
      "activations/layer2_attention_weight_max": 34.75492858886719,
      "activations/layer2_attention_weight_min": -33.53131103515625,
      "activations/layer3_attention_weight_max": 110.4952163696289,
      "activations/layer3_attention_weight_min": -110.37512969970703,
      "activations/layer4_attention_weight_max": 115.88325500488281,
      "activations/layer4_attention_weight_min": -119.241943359375,
      "activations/layer5_attention_weight_max": 52.323631286621094,
      "activations/layer5_attention_weight_min": -62.55057907104492,
      "activations/layer6_attention_weight_max": 48.55141830444336,
      "activations/layer6_attention_weight_min": -51.63667678833008,
      "activations/layer7_attention_weight_max": 100.07352447509766,
      "activations/layer7_attention_weight_min": -106.5215072631836,
      "activations/layer8_attention_weight_max": 44.575477600097656,
      "activations/layer8_attention_weight_min": -47.86968994140625,
      "activations/layer9_attention_weight_max": 33.247982025146484,
      "activations/layer9_attention_weight_min": -37.32610321044922,
      "epoch": 22.48,
      "learning_rate": 5.094696969696968e-06,
      "loss": 2.7094,
      "step": 386850
    },
    {
      "activations/layer0_attention_weight_max": 16.858116149902344,
      "activations/layer0_attention_weight_min": -14.17367935180664,
      "activations/layer10_attention_weight_max": 36.35784149169922,
      "activations/layer10_attention_weight_min": -37.74497985839844,
      "activations/layer11_attention_weight_max": 35.023101806640625,
      "activations/layer11_attention_weight_min": -37.072383880615234,
      "activations/layer12_attention_weight_max": 28.763750076293945,
      "activations/layer12_attention_weight_min": -27.927486419677734,
      "activations/layer13_attention_weight_max": 41.820980072021484,
      "activations/layer13_attention_weight_min": -37.434364318847656,
      "activations/layer14_attention_weight_max": 48.24470138549805,
      "activations/layer14_attention_weight_min": -39.61589431762695,
      "activations/layer15_attention_weight_max": 40.040409088134766,
      "activations/layer15_attention_weight_min": -37.3828010559082,
      "activations/layer16_attention_weight_max": 30.171369552612305,
      "activations/layer16_attention_weight_min": -28.807321548461914,
      "activations/layer17_attention_weight_max": 31.31075668334961,
      "activations/layer17_attention_weight_min": -28.390146255493164,
      "activations/layer18_attention_weight_max": 36.41071319580078,
      "activations/layer18_attention_weight_min": -29.996395111083984,
      "activations/layer19_attention_weight_max": 38.00530242919922,
      "activations/layer19_attention_weight_min": -34.43205642700195,
      "activations/layer1_attention_weight_max": 15.004485130310059,
      "activations/layer1_attention_weight_min": -15.987445831298828,
      "activations/layer20_attention_weight_max": 31.17914390563965,
      "activations/layer20_attention_weight_min": -26.99813461303711,
      "activations/layer21_attention_weight_max": 29.327634811401367,
      "activations/layer21_attention_weight_min": -24.996566772460938,
      "activations/layer22_attention_weight_max": 45.24895095825195,
      "activations/layer22_attention_weight_min": -33.93317794799805,
      "activations/layer23_attention_weight_max": 31.732145309448242,
      "activations/layer23_attention_weight_min": -25.620826721191406,
      "activations/layer2_attention_weight_max": 34.69306945800781,
      "activations/layer2_attention_weight_min": -35.320068359375,
      "activations/layer3_attention_weight_max": 108.25653076171875,
      "activations/layer3_attention_weight_min": -111.51239013671875,
      "activations/layer4_attention_weight_max": 111.94705963134766,
      "activations/layer4_attention_weight_min": -115.11576080322266,
      "activations/layer5_attention_weight_max": 51.3838005065918,
      "activations/layer5_attention_weight_min": -59.271095275878906,
      "activations/layer6_attention_weight_max": 48.574134826660156,
      "activations/layer6_attention_weight_min": -52.623409271240234,
      "activations/layer7_attention_weight_max": 102.84770202636719,
      "activations/layer7_attention_weight_min": -112.54460144042969,
      "activations/layer8_attention_weight_max": 47.36897659301758,
      "activations/layer8_attention_weight_min": -50.68608474731445,
      "activations/layer9_attention_weight_max": 33.7838249206543,
      "activations/layer9_attention_weight_min": -37.33638381958008,
      "epoch": 22.48,
      "learning_rate": 5.075757575757576e-06,
      "loss": 2.6945,
      "step": 386900
    },
    {
      "activations/layer0_attention_weight_max": 17.07461929321289,
      "activations/layer0_attention_weight_min": -14.3076810836792,
      "activations/layer10_attention_weight_max": 36.29507064819336,
      "activations/layer10_attention_weight_min": -36.86354446411133,
      "activations/layer11_attention_weight_max": 36.32831573486328,
      "activations/layer11_attention_weight_min": -35.70341873168945,
      "activations/layer12_attention_weight_max": 27.491004943847656,
      "activations/layer12_attention_weight_min": -28.885103225708008,
      "activations/layer13_attention_weight_max": 39.717769622802734,
      "activations/layer13_attention_weight_min": -38.22245788574219,
      "activations/layer14_attention_weight_max": 41.330204010009766,
      "activations/layer14_attention_weight_min": -41.85420227050781,
      "activations/layer15_attention_weight_max": 38.81673812866211,
      "activations/layer15_attention_weight_min": -39.54706573486328,
      "activations/layer16_attention_weight_max": 29.29537582397461,
      "activations/layer16_attention_weight_min": -30.140214920043945,
      "activations/layer17_attention_weight_max": 31.5685977935791,
      "activations/layer17_attention_weight_min": -27.31378746032715,
      "activations/layer18_attention_weight_max": 31.425260543823242,
      "activations/layer18_attention_weight_min": -26.47408103942871,
      "activations/layer19_attention_weight_max": 33.15754318237305,
      "activations/layer19_attention_weight_min": -32.70038986206055,
      "activations/layer1_attention_weight_max": 15.603167533874512,
      "activations/layer1_attention_weight_min": -15.186964988708496,
      "activations/layer20_attention_weight_max": 27.841514587402344,
      "activations/layer20_attention_weight_min": -28.076526641845703,
      "activations/layer21_attention_weight_max": 29.82613754272461,
      "activations/layer21_attention_weight_min": -28.146968841552734,
      "activations/layer22_attention_weight_max": 43.26253890991211,
      "activations/layer22_attention_weight_min": -33.953983306884766,
      "activations/layer23_attention_weight_max": 32.451332092285156,
      "activations/layer23_attention_weight_min": -28.54010009765625,
      "activations/layer2_attention_weight_max": 36.66249084472656,
      "activations/layer2_attention_weight_min": -33.34083557128906,
      "activations/layer3_attention_weight_max": 111.34300231933594,
      "activations/layer3_attention_weight_min": -107.64134979248047,
      "activations/layer4_attention_weight_max": 114.007080078125,
      "activations/layer4_attention_weight_min": -117.56175994873047,
      "activations/layer5_attention_weight_max": 51.623348236083984,
      "activations/layer5_attention_weight_min": -64.06270599365234,
      "activations/layer6_attention_weight_max": 48.57204818725586,
      "activations/layer6_attention_weight_min": -50.52610397338867,
      "activations/layer7_attention_weight_max": 100.64566040039062,
      "activations/layer7_attention_weight_min": -106.77168273925781,
      "activations/layer8_attention_weight_max": 45.76687240600586,
      "activations/layer8_attention_weight_min": -48.1616096496582,
      "activations/layer9_attention_weight_max": 34.16053771972656,
      "activations/layer9_attention_weight_min": -36.57997131347656,
      "epoch": 22.48,
      "learning_rate": 5.0568181818181815e-06,
      "loss": 2.7015,
      "step": 386950
    },
    {
      "activations/layer0_attention_weight_max": 16.318382263183594,
      "activations/layer0_attention_weight_min": -14.50773811340332,
      "activations/layer10_attention_weight_max": 40.88075637817383,
      "activations/layer10_attention_weight_min": -44.56844711303711,
      "activations/layer11_attention_weight_max": 39.08660125732422,
      "activations/layer11_attention_weight_min": -42.891414642333984,
      "activations/layer12_attention_weight_max": 28.463464736938477,
      "activations/layer12_attention_weight_min": -29.617042541503906,
      "activations/layer13_attention_weight_max": 39.93217849731445,
      "activations/layer13_attention_weight_min": -38.85493469238281,
      "activations/layer14_attention_weight_max": 43.87338638305664,
      "activations/layer14_attention_weight_min": -41.577171325683594,
      "activations/layer15_attention_weight_max": 39.11710739135742,
      "activations/layer15_attention_weight_min": -40.92405700683594,
      "activations/layer16_attention_weight_max": 30.6103515625,
      "activations/layer16_attention_weight_min": -28.771806716918945,
      "activations/layer17_attention_weight_max": 31.271432876586914,
      "activations/layer17_attention_weight_min": -27.26271629333496,
      "activations/layer18_attention_weight_max": 32.8951301574707,
      "activations/layer18_attention_weight_min": -29.467058181762695,
      "activations/layer19_attention_weight_max": 36.76215744018555,
      "activations/layer19_attention_weight_min": -31.00387954711914,
      "activations/layer1_attention_weight_max": 16.410598754882812,
      "activations/layer1_attention_weight_min": -15.354702949523926,
      "activations/layer20_attention_weight_max": 29.43705940246582,
      "activations/layer20_attention_weight_min": -24.658525466918945,
      "activations/layer21_attention_weight_max": 31.572555541992188,
      "activations/layer21_attention_weight_min": -23.82713508605957,
      "activations/layer22_attention_weight_max": 45.72937774658203,
      "activations/layer22_attention_weight_min": -31.920307159423828,
      "activations/layer23_attention_weight_max": 34.2282600402832,
      "activations/layer23_attention_weight_min": -26.45806884765625,
      "activations/layer2_attention_weight_max": 35.82561492919922,
      "activations/layer2_attention_weight_min": -37.559566497802734,
      "activations/layer3_attention_weight_max": 109.28605651855469,
      "activations/layer3_attention_weight_min": -114.89631652832031,
      "activations/layer4_attention_weight_max": 113.40898895263672,
      "activations/layer4_attention_weight_min": -113.53857421875,
      "activations/layer5_attention_weight_max": 50.012481689453125,
      "activations/layer5_attention_weight_min": -61.38322448730469,
      "activations/layer6_attention_weight_max": 49.08860397338867,
      "activations/layer6_attention_weight_min": -52.490413665771484,
      "activations/layer7_attention_weight_max": 102.0071792602539,
      "activations/layer7_attention_weight_min": -115.68756866455078,
      "activations/layer8_attention_weight_max": 48.067909240722656,
      "activations/layer8_attention_weight_min": -53.388450622558594,
      "activations/layer9_attention_weight_max": 36.87565612792969,
      "activations/layer9_attention_weight_min": -42.043087005615234,
      "epoch": 22.49,
      "learning_rate": 5.037878787878787e-06,
      "loss": 2.7157,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4445,
      "eval_samples_per_second": 508.499,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4445,
      "eval_openwebtext_samples_per_second": 508.499,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.9277,
      "eval_wikitext_samples_per_second": 236.547,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_lambada_loss": 2.294921875,
      "eval_lambada_ppl": 9.923660696008076,
      "eval_lambada_runtime": 9.5565,
      "eval_lambada_samples_per_second": 509.498,
      "step": 387000
    },
    {
      "activations/layer0_attention_weight_max": 16.452695846557617,
      "activations/layer0_attention_weight_min": -12.84929084777832,
      "activations/layer10_attention_weight_max": 39.9565544128418,
      "activations/layer10_attention_weight_min": -38.00286102294922,
      "activations/layer11_attention_weight_max": 38.369224548339844,
      "activations/layer11_attention_weight_min": -37.39365005493164,
      "activations/layer12_attention_weight_max": 26.762243270874023,
      "activations/layer12_attention_weight_min": -27.21336555480957,
      "activations/layer13_attention_weight_max": 37.494239807128906,
      "activations/layer13_attention_weight_min": -38.21258544921875,
      "activations/layer14_attention_weight_max": 44.9793586730957,
      "activations/layer14_attention_weight_min": -41.56991195678711,
      "activations/layer15_attention_weight_max": 39.30079650878906,
      "activations/layer15_attention_weight_min": -42.268310546875,
      "activations/layer16_attention_weight_max": 30.565401077270508,
      "activations/layer16_attention_weight_min": -33.80744934082031,
      "activations/layer17_attention_weight_max": 31.49994659423828,
      "activations/layer17_attention_weight_min": -32.07002639770508,
      "activations/layer18_attention_weight_max": 29.68840217590332,
      "activations/layer18_attention_weight_min": -28.766685485839844,
      "activations/layer19_attention_weight_max": 35.864097595214844,
      "activations/layer19_attention_weight_min": -35.24015426635742,
      "activations/layer1_attention_weight_max": 16.03232192993164,
      "activations/layer1_attention_weight_min": -14.394454002380371,
      "activations/layer20_attention_weight_max": 29.5684757232666,
      "activations/layer20_attention_weight_min": -28.203062057495117,
      "activations/layer21_attention_weight_max": 29.237234115600586,
      "activations/layer21_attention_weight_min": -26.21578025817871,
      "activations/layer22_attention_weight_max": 40.04433822631836,
      "activations/layer22_attention_weight_min": -34.64951705932617,
      "activations/layer23_attention_weight_max": 33.56922149658203,
      "activations/layer23_attention_weight_min": -25.599990844726562,
      "activations/layer2_attention_weight_max": 36.35114288330078,
      "activations/layer2_attention_weight_min": -34.808998107910156,
      "activations/layer3_attention_weight_max": 111.88738250732422,
      "activations/layer3_attention_weight_min": -111.2111587524414,
      "activations/layer4_attention_weight_max": 112.82942962646484,
      "activations/layer4_attention_weight_min": -116.4891128540039,
      "activations/layer5_attention_weight_max": 50.844078063964844,
      "activations/layer5_attention_weight_min": -62.70708465576172,
      "activations/layer6_attention_weight_max": 47.14506530761719,
      "activations/layer6_attention_weight_min": -50.476444244384766,
      "activations/layer7_attention_weight_max": 104.60863494873047,
      "activations/layer7_attention_weight_min": -108.75350952148438,
      "activations/layer8_attention_weight_max": 47.43631362915039,
      "activations/layer8_attention_weight_min": -48.10334777832031,
      "activations/layer9_attention_weight_max": 35.239288330078125,
      "activations/layer9_attention_weight_min": -40.06635284423828,
      "epoch": 22.49,
      "learning_rate": 5.018939393939394e-06,
      "loss": 2.7247,
      "step": 387050
    },
    {
      "activations/layer0_attention_weight_max": 16.46161460876465,
      "activations/layer0_attention_weight_min": -15.543492317199707,
      "activations/layer10_attention_weight_max": 36.853302001953125,
      "activations/layer10_attention_weight_min": -35.56610870361328,
      "activations/layer11_attention_weight_max": 36.31083297729492,
      "activations/layer11_attention_weight_min": -35.91202163696289,
      "activations/layer12_attention_weight_max": 28.766996383666992,
      "activations/layer12_attention_weight_min": -28.019681930541992,
      "activations/layer13_attention_weight_max": 39.93941116333008,
      "activations/layer13_attention_weight_min": -38.188411712646484,
      "activations/layer14_attention_weight_max": 41.672332763671875,
      "activations/layer14_attention_weight_min": -37.93236541748047,
      "activations/layer15_attention_weight_max": 37.54787826538086,
      "activations/layer15_attention_weight_min": -35.48148727416992,
      "activations/layer16_attention_weight_max": 28.650970458984375,
      "activations/layer16_attention_weight_min": -28.763216018676758,
      "activations/layer17_attention_weight_max": 31.64579963684082,
      "activations/layer17_attention_weight_min": -26.2110652923584,
      "activations/layer18_attention_weight_max": 34.06583023071289,
      "activations/layer18_attention_weight_min": -26.157470703125,
      "activations/layer19_attention_weight_max": 33.40427780151367,
      "activations/layer19_attention_weight_min": -32.24943542480469,
      "activations/layer1_attention_weight_max": 15.401409149169922,
      "activations/layer1_attention_weight_min": -15.712220191955566,
      "activations/layer20_attention_weight_max": 29.5329647064209,
      "activations/layer20_attention_weight_min": -25.32668685913086,
      "activations/layer21_attention_weight_max": 31.92366600036621,
      "activations/layer21_attention_weight_min": -25.014572143554688,
      "activations/layer22_attention_weight_max": 43.75742721557617,
      "activations/layer22_attention_weight_min": -32.76665115356445,
      "activations/layer23_attention_weight_max": 33.818241119384766,
      "activations/layer23_attention_weight_min": -27.254352569580078,
      "activations/layer2_attention_weight_max": 34.311004638671875,
      "activations/layer2_attention_weight_min": -33.93518829345703,
      "activations/layer3_attention_weight_max": 109.07022094726562,
      "activations/layer3_attention_weight_min": -106.77556610107422,
      "activations/layer4_attention_weight_max": 110.97493743896484,
      "activations/layer4_attention_weight_min": -114.26783752441406,
      "activations/layer5_attention_weight_max": 51.928794860839844,
      "activations/layer5_attention_weight_min": -65.10755920410156,
      "activations/layer6_attention_weight_max": 46.17443084716797,
      "activations/layer6_attention_weight_min": -47.29282760620117,
      "activations/layer7_attention_weight_max": 98.5498046875,
      "activations/layer7_attention_weight_min": -98.88864135742188,
      "activations/layer8_attention_weight_max": 43.88176345825195,
      "activations/layer8_attention_weight_min": -45.1741828918457,
      "activations/layer9_attention_weight_max": 34.06944274902344,
      "activations/layer9_attention_weight_min": -36.58783721923828,
      "epoch": 22.49,
      "learning_rate": 4.9999999999999996e-06,
      "loss": 2.7033,
      "step": 387100
    },
    {
      "activations/layer0_attention_weight_max": 16.628189086914062,
      "activations/layer0_attention_weight_min": -14.06531047821045,
      "activations/layer10_attention_weight_max": 37.814735412597656,
      "activations/layer10_attention_weight_min": -37.63344955444336,
      "activations/layer11_attention_weight_max": 39.73179244995117,
      "activations/layer11_attention_weight_min": -36.835662841796875,
      "activations/layer12_attention_weight_max": 27.17414665222168,
      "activations/layer12_attention_weight_min": -27.870384216308594,
      "activations/layer13_attention_weight_max": 39.06477355957031,
      "activations/layer13_attention_weight_min": -37.48408126831055,
      "activations/layer14_attention_weight_max": 43.47890853881836,
      "activations/layer14_attention_weight_min": -39.157806396484375,
      "activations/layer15_attention_weight_max": 43.094486236572266,
      "activations/layer15_attention_weight_min": -36.59136962890625,
      "activations/layer16_attention_weight_max": 32.00949478149414,
      "activations/layer16_attention_weight_min": -29.508869171142578,
      "activations/layer17_attention_weight_max": 28.726131439208984,
      "activations/layer17_attention_weight_min": -26.51458168029785,
      "activations/layer18_attention_weight_max": 33.881195068359375,
      "activations/layer18_attention_weight_min": -25.852689743041992,
      "activations/layer19_attention_weight_max": 36.02287292480469,
      "activations/layer19_attention_weight_min": -36.10175704956055,
      "activations/layer1_attention_weight_max": 16.101253509521484,
      "activations/layer1_attention_weight_min": -15.064074516296387,
      "activations/layer20_attention_weight_max": 27.693023681640625,
      "activations/layer20_attention_weight_min": -25.120494842529297,
      "activations/layer21_attention_weight_max": 26.398637771606445,
      "activations/layer21_attention_weight_min": -24.908201217651367,
      "activations/layer22_attention_weight_max": 40.965145111083984,
      "activations/layer22_attention_weight_min": -32.5544319152832,
      "activations/layer23_attention_weight_max": 29.760154724121094,
      "activations/layer23_attention_weight_min": -25.135862350463867,
      "activations/layer2_attention_weight_max": 36.07649230957031,
      "activations/layer2_attention_weight_min": -36.00507736206055,
      "activations/layer3_attention_weight_max": 110.16333770751953,
      "activations/layer3_attention_weight_min": -108.70558166503906,
      "activations/layer4_attention_weight_max": 111.15748596191406,
      "activations/layer4_attention_weight_min": -114.75001525878906,
      "activations/layer5_attention_weight_max": 53.298484802246094,
      "activations/layer5_attention_weight_min": -61.40825653076172,
      "activations/layer6_attention_weight_max": 46.647178649902344,
      "activations/layer6_attention_weight_min": -49.995304107666016,
      "activations/layer7_attention_weight_max": 103.95718383789062,
      "activations/layer7_attention_weight_min": -102.76924133300781,
      "activations/layer8_attention_weight_max": 44.6895866394043,
      "activations/layer8_attention_weight_min": -45.85820770263672,
      "activations/layer9_attention_weight_max": 38.14524459838867,
      "activations/layer9_attention_weight_min": -39.064510345458984,
      "epoch": 22.5,
      "learning_rate": 4.981060606060605e-06,
      "loss": 2.724,
      "step": 387150
    },
    {
      "activations/layer0_attention_weight_max": 17.444665908813477,
      "activations/layer0_attention_weight_min": -13.38558292388916,
      "activations/layer10_attention_weight_max": 34.9334716796875,
      "activations/layer10_attention_weight_min": -37.836814880371094,
      "activations/layer11_attention_weight_max": 36.73699951171875,
      "activations/layer11_attention_weight_min": -35.909912109375,
      "activations/layer12_attention_weight_max": 28.728727340698242,
      "activations/layer12_attention_weight_min": -26.7236270904541,
      "activations/layer13_attention_weight_max": 43.391807556152344,
      "activations/layer13_attention_weight_min": -36.58454513549805,
      "activations/layer14_attention_weight_max": 43.06992721557617,
      "activations/layer14_attention_weight_min": -36.810523986816406,
      "activations/layer15_attention_weight_max": 38.773983001708984,
      "activations/layer15_attention_weight_min": -35.70136260986328,
      "activations/layer16_attention_weight_max": 28.46222496032715,
      "activations/layer16_attention_weight_min": -27.031997680664062,
      "activations/layer17_attention_weight_max": 29.97914695739746,
      "activations/layer17_attention_weight_min": -24.650379180908203,
      "activations/layer18_attention_weight_max": 33.225624084472656,
      "activations/layer18_attention_weight_min": -24.307579040527344,
      "activations/layer19_attention_weight_max": 37.94126510620117,
      "activations/layer19_attention_weight_min": -30.787893295288086,
      "activations/layer1_attention_weight_max": 15.237555503845215,
      "activations/layer1_attention_weight_min": -14.50886058807373,
      "activations/layer20_attention_weight_max": 30.329666137695312,
      "activations/layer20_attention_weight_min": -24.571916580200195,
      "activations/layer21_attention_weight_max": 32.843170166015625,
      "activations/layer21_attention_weight_min": -25.985458374023438,
      "activations/layer22_attention_weight_max": 43.02457809448242,
      "activations/layer22_attention_weight_min": -30.526813507080078,
      "activations/layer23_attention_weight_max": 31.509122848510742,
      "activations/layer23_attention_weight_min": -23.53375244140625,
      "activations/layer2_attention_weight_max": 34.393470764160156,
      "activations/layer2_attention_weight_min": -33.20002746582031,
      "activations/layer3_attention_weight_max": 107.05105590820312,
      "activations/layer3_attention_weight_min": -107.96549987792969,
      "activations/layer4_attention_weight_max": 110.38448333740234,
      "activations/layer4_attention_weight_min": -110.16094970703125,
      "activations/layer5_attention_weight_max": 52.42150115966797,
      "activations/layer5_attention_weight_min": -65.85807037353516,
      "activations/layer6_attention_weight_max": 48.65538024902344,
      "activations/layer6_attention_weight_min": -51.53488540649414,
      "activations/layer7_attention_weight_max": 99.4247055053711,
      "activations/layer7_attention_weight_min": -110.98686981201172,
      "activations/layer8_attention_weight_max": 46.11832809448242,
      "activations/layer8_attention_weight_min": -49.329586029052734,
      "activations/layer9_attention_weight_max": 33.514163970947266,
      "activations/layer9_attention_weight_min": -36.36544418334961,
      "epoch": 22.5,
      "learning_rate": 4.962121212121211e-06,
      "loss": 2.7043,
      "step": 387200
    },
    {
      "activations/layer0_attention_weight_max": 16.039409637451172,
      "activations/layer0_attention_weight_min": -15.638262748718262,
      "activations/layer10_attention_weight_max": 42.006317138671875,
      "activations/layer10_attention_weight_min": -39.976314544677734,
      "activations/layer11_attention_weight_max": 37.67681121826172,
      "activations/layer11_attention_weight_min": -39.75914001464844,
      "activations/layer12_attention_weight_max": 32.20161819458008,
      "activations/layer12_attention_weight_min": -31.166154861450195,
      "activations/layer13_attention_weight_max": 48.300254821777344,
      "activations/layer13_attention_weight_min": -43.89400100708008,
      "activations/layer14_attention_weight_max": 51.03645324707031,
      "activations/layer14_attention_weight_min": -47.4310417175293,
      "activations/layer15_attention_weight_max": 44.07927322387695,
      "activations/layer15_attention_weight_min": -43.53356170654297,
      "activations/layer16_attention_weight_max": 30.865108489990234,
      "activations/layer16_attention_weight_min": -31.403234481811523,
      "activations/layer17_attention_weight_max": 32.84047317504883,
      "activations/layer17_attention_weight_min": -29.20926856994629,
      "activations/layer18_attention_weight_max": 36.09131622314453,
      "activations/layer18_attention_weight_min": -28.636308670043945,
      "activations/layer19_attention_weight_max": 34.19587707519531,
      "activations/layer19_attention_weight_min": -32.1641845703125,
      "activations/layer1_attention_weight_max": 15.28036880493164,
      "activations/layer1_attention_weight_min": -14.763694763183594,
      "activations/layer20_attention_weight_max": 27.563020706176758,
      "activations/layer20_attention_weight_min": -26.526409149169922,
      "activations/layer21_attention_weight_max": 27.16629409790039,
      "activations/layer21_attention_weight_min": -25.463645935058594,
      "activations/layer22_attention_weight_max": 44.905189514160156,
      "activations/layer22_attention_weight_min": -30.81797218322754,
      "activations/layer23_attention_weight_max": 33.02064895629883,
      "activations/layer23_attention_weight_min": -25.999664306640625,
      "activations/layer2_attention_weight_max": 35.49851608276367,
      "activations/layer2_attention_weight_min": -35.42189025878906,
      "activations/layer3_attention_weight_max": 110.35696411132812,
      "activations/layer3_attention_weight_min": -111.02706146240234,
      "activations/layer4_attention_weight_max": 113.35363006591797,
      "activations/layer4_attention_weight_min": -116.9720458984375,
      "activations/layer5_attention_weight_max": 51.76911544799805,
      "activations/layer5_attention_weight_min": -62.84249496459961,
      "activations/layer6_attention_weight_max": 47.93451690673828,
      "activations/layer6_attention_weight_min": -51.639774322509766,
      "activations/layer7_attention_weight_max": 103.3673324584961,
      "activations/layer7_attention_weight_min": -113.81124114990234,
      "activations/layer8_attention_weight_max": 48.79166793823242,
      "activations/layer8_attention_weight_min": -51.52462387084961,
      "activations/layer9_attention_weight_max": 37.08870315551758,
      "activations/layer9_attention_weight_min": -38.86713409423828,
      "epoch": 22.5,
      "learning_rate": 4.943181818181818e-06,
      "loss": 2.7037,
      "step": 387250
    },
    {
      "activations/layer0_attention_weight_max": 16.8775691986084,
      "activations/layer0_attention_weight_min": -14.337658882141113,
      "activations/layer10_attention_weight_max": 36.71685028076172,
      "activations/layer10_attention_weight_min": -38.4190673828125,
      "activations/layer11_attention_weight_max": 37.0545654296875,
      "activations/layer11_attention_weight_min": -37.04071807861328,
      "activations/layer12_attention_weight_max": 30.212379455566406,
      "activations/layer12_attention_weight_min": -28.48739242553711,
      "activations/layer13_attention_weight_max": 43.586265563964844,
      "activations/layer13_attention_weight_min": -37.52315902709961,
      "activations/layer14_attention_weight_max": 44.9152946472168,
      "activations/layer14_attention_weight_min": -39.58203887939453,
      "activations/layer15_attention_weight_max": 41.862693786621094,
      "activations/layer15_attention_weight_min": -36.59814453125,
      "activations/layer16_attention_weight_max": 30.349315643310547,
      "activations/layer16_attention_weight_min": -30.270612716674805,
      "activations/layer17_attention_weight_max": 32.40900421142578,
      "activations/layer17_attention_weight_min": -28.616296768188477,
      "activations/layer18_attention_weight_max": 32.717708587646484,
      "activations/layer18_attention_weight_min": -26.824323654174805,
      "activations/layer19_attention_weight_max": 34.3364372253418,
      "activations/layer19_attention_weight_min": -32.97163009643555,
      "activations/layer1_attention_weight_max": 15.602760314941406,
      "activations/layer1_attention_weight_min": -14.051913261413574,
      "activations/layer20_attention_weight_max": 32.27912139892578,
      "activations/layer20_attention_weight_min": -26.538040161132812,
      "activations/layer21_attention_weight_max": 29.67841148376465,
      "activations/layer21_attention_weight_min": -25.702341079711914,
      "activations/layer22_attention_weight_max": 44.203269958496094,
      "activations/layer22_attention_weight_min": -34.06474304199219,
      "activations/layer23_attention_weight_max": 32.41496658325195,
      "activations/layer23_attention_weight_min": -29.277240753173828,
      "activations/layer2_attention_weight_max": 33.695255279541016,
      "activations/layer2_attention_weight_min": -32.645931243896484,
      "activations/layer3_attention_weight_max": 108.85905456542969,
      "activations/layer3_attention_weight_min": -108.16285705566406,
      "activations/layer4_attention_weight_max": 112.76165008544922,
      "activations/layer4_attention_weight_min": -113.39483642578125,
      "activations/layer5_attention_weight_max": 52.243064880371094,
      "activations/layer5_attention_weight_min": -62.213157653808594,
      "activations/layer6_attention_weight_max": 50.3354377746582,
      "activations/layer6_attention_weight_min": -52.325279235839844,
      "activations/layer7_attention_weight_max": 105.87139129638672,
      "activations/layer7_attention_weight_min": -106.21531677246094,
      "activations/layer8_attention_weight_max": 45.60453414916992,
      "activations/layer8_attention_weight_min": -48.88875198364258,
      "activations/layer9_attention_weight_max": 35.49334716796875,
      "activations/layer9_attention_weight_min": -39.54640197753906,
      "epoch": 22.5,
      "learning_rate": 4.924242424242424e-06,
      "loss": 2.6857,
      "step": 387300
    },
    {
      "activations/layer0_attention_weight_max": 16.825464248657227,
      "activations/layer0_attention_weight_min": -14.547262191772461,
      "activations/layer10_attention_weight_max": 51.048336029052734,
      "activations/layer10_attention_weight_min": -48.7001838684082,
      "activations/layer11_attention_weight_max": 54.69880294799805,
      "activations/layer11_attention_weight_min": -51.71038818359375,
      "activations/layer12_attention_weight_max": 33.605594635009766,
      "activations/layer12_attention_weight_min": -30.807580947875977,
      "activations/layer13_attention_weight_max": 48.05664825439453,
      "activations/layer13_attention_weight_min": -40.59151077270508,
      "activations/layer14_attention_weight_max": 56.61508560180664,
      "activations/layer14_attention_weight_min": -46.15644454956055,
      "activations/layer15_attention_weight_max": 53.76736068725586,
      "activations/layer15_attention_weight_min": -46.55693054199219,
      "activations/layer16_attention_weight_max": 39.70600128173828,
      "activations/layer16_attention_weight_min": -29.53668785095215,
      "activations/layer17_attention_weight_max": 34.28218078613281,
      "activations/layer17_attention_weight_min": -29.771575927734375,
      "activations/layer18_attention_weight_max": 33.98093032836914,
      "activations/layer18_attention_weight_min": -28.94222640991211,
      "activations/layer19_attention_weight_max": 38.239479064941406,
      "activations/layer19_attention_weight_min": -34.60206985473633,
      "activations/layer1_attention_weight_max": 15.355157852172852,
      "activations/layer1_attention_weight_min": -15.838479042053223,
      "activations/layer20_attention_weight_max": 31.754653930664062,
      "activations/layer20_attention_weight_min": -26.690446853637695,
      "activations/layer21_attention_weight_max": 29.255496978759766,
      "activations/layer21_attention_weight_min": -24.551389694213867,
      "activations/layer22_attention_weight_max": 41.60441207885742,
      "activations/layer22_attention_weight_min": -35.566383361816406,
      "activations/layer23_attention_weight_max": 33.585838317871094,
      "activations/layer23_attention_weight_min": -25.639074325561523,
      "activations/layer2_attention_weight_max": 37.0426025390625,
      "activations/layer2_attention_weight_min": -35.950992584228516,
      "activations/layer3_attention_weight_max": 114.10584259033203,
      "activations/layer3_attention_weight_min": -115.7226791381836,
      "activations/layer4_attention_weight_max": 116.50133514404297,
      "activations/layer4_attention_weight_min": -113.1209716796875,
      "activations/layer5_attention_weight_max": 52.8250732421875,
      "activations/layer5_attention_weight_min": -63.857479095458984,
      "activations/layer6_attention_weight_max": 52.727745056152344,
      "activations/layer6_attention_weight_min": -50.88026809692383,
      "activations/layer7_attention_weight_max": 119.53852844238281,
      "activations/layer7_attention_weight_min": -116.03001403808594,
      "activations/layer8_attention_weight_max": 57.80421447753906,
      "activations/layer8_attention_weight_min": -55.8629150390625,
      "activations/layer9_attention_weight_max": 44.26762008666992,
      "activations/layer9_attention_weight_min": -46.85082244873047,
      "epoch": 22.51,
      "learning_rate": 4.90530303030303e-06,
      "loss": 2.6947,
      "step": 387350
    },
    {
      "activations/layer0_attention_weight_max": 15.793004989624023,
      "activations/layer0_attention_weight_min": -15.457354545593262,
      "activations/layer10_attention_weight_max": 48.286033630371094,
      "activations/layer10_attention_weight_min": -45.54365921020508,
      "activations/layer11_attention_weight_max": 49.32076644897461,
      "activations/layer11_attention_weight_min": -45.01387023925781,
      "activations/layer12_attention_weight_max": 30.428865432739258,
      "activations/layer12_attention_weight_min": -29.369197845458984,
      "activations/layer13_attention_weight_max": 41.459190368652344,
      "activations/layer13_attention_weight_min": -37.1504020690918,
      "activations/layer14_attention_weight_max": 48.81391143798828,
      "activations/layer14_attention_weight_min": -39.21427536010742,
      "activations/layer15_attention_weight_max": 45.97062301635742,
      "activations/layer15_attention_weight_min": -42.53733825683594,
      "activations/layer16_attention_weight_max": 33.64004135131836,
      "activations/layer16_attention_weight_min": -28.611032485961914,
      "activations/layer17_attention_weight_max": 33.493839263916016,
      "activations/layer17_attention_weight_min": -26.99425506591797,
      "activations/layer18_attention_weight_max": 33.46403121948242,
      "activations/layer18_attention_weight_min": -27.78887367248535,
      "activations/layer19_attention_weight_max": 37.61614227294922,
      "activations/layer19_attention_weight_min": -31.959157943725586,
      "activations/layer1_attention_weight_max": 16.259449005126953,
      "activations/layer1_attention_weight_min": -16.38324546813965,
      "activations/layer20_attention_weight_max": 33.149105072021484,
      "activations/layer20_attention_weight_min": -24.744535446166992,
      "activations/layer21_attention_weight_max": 31.296279907226562,
      "activations/layer21_attention_weight_min": -23.882965087890625,
      "activations/layer22_attention_weight_max": 41.89664840698242,
      "activations/layer22_attention_weight_min": -33.44468307495117,
      "activations/layer23_attention_weight_max": 31.592199325561523,
      "activations/layer23_attention_weight_min": -27.324573516845703,
      "activations/layer2_attention_weight_max": 36.86894607543945,
      "activations/layer2_attention_weight_min": -35.57064437866211,
      "activations/layer3_attention_weight_max": 113.49058532714844,
      "activations/layer3_attention_weight_min": -111.43724060058594,
      "activations/layer4_attention_weight_max": 116.41108703613281,
      "activations/layer4_attention_weight_min": -116.72901916503906,
      "activations/layer5_attention_weight_max": 51.96711349487305,
      "activations/layer5_attention_weight_min": -62.809730529785156,
      "activations/layer6_attention_weight_max": 48.7891960144043,
      "activations/layer6_attention_weight_min": -50.3231315612793,
      "activations/layer7_attention_weight_max": 112.34494018554688,
      "activations/layer7_attention_weight_min": -109.40953826904297,
      "activations/layer8_attention_weight_max": 54.592552185058594,
      "activations/layer8_attention_weight_min": -52.11152267456055,
      "activations/layer9_attention_weight_max": 40.37435531616211,
      "activations/layer9_attention_weight_min": -41.11239242553711,
      "epoch": 22.51,
      "learning_rate": 4.886742424242424e-06,
      "loss": 2.6814,
      "step": 387400
    },
    {
      "activations/layer0_attention_weight_max": 17.223852157592773,
      "activations/layer0_attention_weight_min": -13.891915321350098,
      "activations/layer10_attention_weight_max": 35.37264633178711,
      "activations/layer10_attention_weight_min": -33.76205825805664,
      "activations/layer11_attention_weight_max": 34.11962890625,
      "activations/layer11_attention_weight_min": -34.65599822998047,
      "activations/layer12_attention_weight_max": 25.271804809570312,
      "activations/layer12_attention_weight_min": -27.227848052978516,
      "activations/layer13_attention_weight_max": 38.722862243652344,
      "activations/layer13_attention_weight_min": -34.88737106323242,
      "activations/layer14_attention_weight_max": 39.415035247802734,
      "activations/layer14_attention_weight_min": -36.67372131347656,
      "activations/layer15_attention_weight_max": 36.15489196777344,
      "activations/layer15_attention_weight_min": -34.02570724487305,
      "activations/layer16_attention_weight_max": 28.086807250976562,
      "activations/layer16_attention_weight_min": -29.318227767944336,
      "activations/layer17_attention_weight_max": 27.8309268951416,
      "activations/layer17_attention_weight_min": -26.10354232788086,
      "activations/layer18_attention_weight_max": 30.070283889770508,
      "activations/layer18_attention_weight_min": -24.749780654907227,
      "activations/layer19_attention_weight_max": 34.61224365234375,
      "activations/layer19_attention_weight_min": -31.01564598083496,
      "activations/layer1_attention_weight_max": 15.580779075622559,
      "activations/layer1_attention_weight_min": -15.573980331420898,
      "activations/layer20_attention_weight_max": 27.80699920654297,
      "activations/layer20_attention_weight_min": -23.92796516418457,
      "activations/layer21_attention_weight_max": 26.143585205078125,
      "activations/layer21_attention_weight_min": -23.235891342163086,
      "activations/layer22_attention_weight_max": 38.55315017700195,
      "activations/layer22_attention_weight_min": -29.24826431274414,
      "activations/layer23_attention_weight_max": 28.570842742919922,
      "activations/layer23_attention_weight_min": -25.16039276123047,
      "activations/layer2_attention_weight_max": 33.60481262207031,
      "activations/layer2_attention_weight_min": -33.24311065673828,
      "activations/layer3_attention_weight_max": 106.03665161132812,
      "activations/layer3_attention_weight_min": -110.41905212402344,
      "activations/layer4_attention_weight_max": 109.98632049560547,
      "activations/layer4_attention_weight_min": -111.9715347290039,
      "activations/layer5_attention_weight_max": 50.23765182495117,
      "activations/layer5_attention_weight_min": -61.20365905761719,
      "activations/layer6_attention_weight_max": 45.59347915649414,
      "activations/layer6_attention_weight_min": -48.3603630065918,
      "activations/layer7_attention_weight_max": 96.95547485351562,
      "activations/layer7_attention_weight_min": -105.67181396484375,
      "activations/layer8_attention_weight_max": 43.750919342041016,
      "activations/layer8_attention_weight_min": -44.669918060302734,
      "activations/layer9_attention_weight_max": 32.397335052490234,
      "activations/layer9_attention_weight_min": -34.995643615722656,
      "epoch": 22.51,
      "learning_rate": 4.86780303030303e-06,
      "loss": 2.6965,
      "step": 387450
    },
    {
      "activations/layer0_attention_weight_max": 17.008256912231445,
      "activations/layer0_attention_weight_min": -14.38659381866455,
      "activations/layer10_attention_weight_max": 37.37299728393555,
      "activations/layer10_attention_weight_min": -35.959224700927734,
      "activations/layer11_attention_weight_max": 33.922218322753906,
      "activations/layer11_attention_weight_min": -36.473514556884766,
      "activations/layer12_attention_weight_max": 26.75408172607422,
      "activations/layer12_attention_weight_min": -29.212434768676758,
      "activations/layer13_attention_weight_max": 40.25822830200195,
      "activations/layer13_attention_weight_min": -39.09425735473633,
      "activations/layer14_attention_weight_max": 45.84522247314453,
      "activations/layer14_attention_weight_min": -43.0994873046875,
      "activations/layer15_attention_weight_max": 41.34941864013672,
      "activations/layer15_attention_weight_min": -40.87668228149414,
      "activations/layer16_attention_weight_max": 34.23182678222656,
      "activations/layer16_attention_weight_min": -29.73319435119629,
      "activations/layer17_attention_weight_max": 31.005680084228516,
      "activations/layer17_attention_weight_min": -27.20518684387207,
      "activations/layer18_attention_weight_max": 35.356964111328125,
      "activations/layer18_attention_weight_min": -26.50400733947754,
      "activations/layer19_attention_weight_max": 39.5660400390625,
      "activations/layer19_attention_weight_min": -33.72938537597656,
      "activations/layer1_attention_weight_max": 15.201240539550781,
      "activations/layer1_attention_weight_min": -14.862042427062988,
      "activations/layer20_attention_weight_max": 30.378528594970703,
      "activations/layer20_attention_weight_min": -25.470380783081055,
      "activations/layer21_attention_weight_max": 30.708641052246094,
      "activations/layer21_attention_weight_min": -24.285930633544922,
      "activations/layer22_attention_weight_max": 49.89627456665039,
      "activations/layer22_attention_weight_min": -33.24274826049805,
      "activations/layer23_attention_weight_max": 37.04872512817383,
      "activations/layer23_attention_weight_min": -29.154220581054688,
      "activations/layer2_attention_weight_max": 35.50594711303711,
      "activations/layer2_attention_weight_min": -34.15339279174805,
      "activations/layer3_attention_weight_max": 109.38795471191406,
      "activations/layer3_attention_weight_min": -111.50067901611328,
      "activations/layer4_attention_weight_max": 115.3259506225586,
      "activations/layer4_attention_weight_min": -116.739990234375,
      "activations/layer5_attention_weight_max": 52.726539611816406,
      "activations/layer5_attention_weight_min": -64.96406555175781,
      "activations/layer6_attention_weight_max": 47.278968811035156,
      "activations/layer6_attention_weight_min": -50.07373809814453,
      "activations/layer7_attention_weight_max": 100.44953918457031,
      "activations/layer7_attention_weight_min": -105.33047485351562,
      "activations/layer8_attention_weight_max": 44.44203186035156,
      "activations/layer8_attention_weight_min": -48.007110595703125,
      "activations/layer9_attention_weight_max": 33.69224166870117,
      "activations/layer9_attention_weight_min": -37.84440231323242,
      "epoch": 22.52,
      "learning_rate": 4.848863636363636e-06,
      "loss": 2.696,
      "step": 387500
    },
    {
      "activations/layer0_attention_weight_max": 16.29497528076172,
      "activations/layer0_attention_weight_min": -15.021793365478516,
      "activations/layer10_attention_weight_max": 40.33826446533203,
      "activations/layer10_attention_weight_min": -37.8251838684082,
      "activations/layer11_attention_weight_max": 36.694610595703125,
      "activations/layer11_attention_weight_min": -36.33234405517578,
      "activations/layer12_attention_weight_max": 30.915117263793945,
      "activations/layer12_attention_weight_min": -28.1605281829834,
      "activations/layer13_attention_weight_max": 45.29924774169922,
      "activations/layer13_attention_weight_min": -39.388023376464844,
      "activations/layer14_attention_weight_max": 47.6987419128418,
      "activations/layer14_attention_weight_min": -38.07025909423828,
      "activations/layer15_attention_weight_max": 46.179229736328125,
      "activations/layer15_attention_weight_min": -36.58073806762695,
      "activations/layer16_attention_weight_max": 36.578067779541016,
      "activations/layer16_attention_weight_min": -27.881786346435547,
      "activations/layer17_attention_weight_max": 32.50661087036133,
      "activations/layer17_attention_weight_min": -25.144521713256836,
      "activations/layer18_attention_weight_max": 33.45233154296875,
      "activations/layer18_attention_weight_min": -24.83698081970215,
      "activations/layer19_attention_weight_max": 38.064640045166016,
      "activations/layer19_attention_weight_min": -31.42499351501465,
      "activations/layer1_attention_weight_max": 18.453981399536133,
      "activations/layer1_attention_weight_min": -17.350566864013672,
      "activations/layer20_attention_weight_max": 31.252323150634766,
      "activations/layer20_attention_weight_min": -24.55535125732422,
      "activations/layer21_attention_weight_max": 30.962919235229492,
      "activations/layer21_attention_weight_min": -24.303857803344727,
      "activations/layer22_attention_weight_max": 49.66691970825195,
      "activations/layer22_attention_weight_min": -31.28238868713379,
      "activations/layer23_attention_weight_max": 37.251686096191406,
      "activations/layer23_attention_weight_min": -25.476966857910156,
      "activations/layer2_attention_weight_max": 35.14995574951172,
      "activations/layer2_attention_weight_min": -34.945037841796875,
      "activations/layer3_attention_weight_max": 111.06521606445312,
      "activations/layer3_attention_weight_min": -110.47425079345703,
      "activations/layer4_attention_weight_max": 113.642578125,
      "activations/layer4_attention_weight_min": -116.53292083740234,
      "activations/layer5_attention_weight_max": 51.048057556152344,
      "activations/layer5_attention_weight_min": -62.01711654663086,
      "activations/layer6_attention_weight_max": 48.388099670410156,
      "activations/layer6_attention_weight_min": -50.24857711791992,
      "activations/layer7_attention_weight_max": 102.11038970947266,
      "activations/layer7_attention_weight_min": -110.25883483886719,
      "activations/layer8_attention_weight_max": 46.38570785522461,
      "activations/layer8_attention_weight_min": -49.42753982543945,
      "activations/layer9_attention_weight_max": 37.030391693115234,
      "activations/layer9_attention_weight_min": -36.56218338012695,
      "epoch": 22.52,
      "learning_rate": 4.829924242424242e-06,
      "loss": 2.7148,
      "step": 387550
    },
    {
      "activations/layer0_attention_weight_max": 16.246673583984375,
      "activations/layer0_attention_weight_min": -14.192264556884766,
      "activations/layer10_attention_weight_max": 38.9685173034668,
      "activations/layer10_attention_weight_min": -38.080848693847656,
      "activations/layer11_attention_weight_max": 37.098426818847656,
      "activations/layer11_attention_weight_min": -35.861053466796875,
      "activations/layer12_attention_weight_max": 28.522218704223633,
      "activations/layer12_attention_weight_min": -27.953126907348633,
      "activations/layer13_attention_weight_max": 40.35033416748047,
      "activations/layer13_attention_weight_min": -37.877593994140625,
      "activations/layer14_attention_weight_max": 43.389373779296875,
      "activations/layer14_attention_weight_min": -41.53605651855469,
      "activations/layer15_attention_weight_max": 40.39746856689453,
      "activations/layer15_attention_weight_min": -38.927608489990234,
      "activations/layer16_attention_weight_max": 32.63913345336914,
      "activations/layer16_attention_weight_min": -29.319433212280273,
      "activations/layer17_attention_weight_max": 32.247398376464844,
      "activations/layer17_attention_weight_min": -27.748065948486328,
      "activations/layer18_attention_weight_max": 34.33027267456055,
      "activations/layer18_attention_weight_min": -28.039936065673828,
      "activations/layer19_attention_weight_max": 36.957664489746094,
      "activations/layer19_attention_weight_min": -34.24729537963867,
      "activations/layer1_attention_weight_max": 15.80377197265625,
      "activations/layer1_attention_weight_min": -14.802555084228516,
      "activations/layer20_attention_weight_max": 30.461048126220703,
      "activations/layer20_attention_weight_min": -27.44633674621582,
      "activations/layer21_attention_weight_max": 28.54490089416504,
      "activations/layer21_attention_weight_min": -25.72371482849121,
      "activations/layer22_attention_weight_max": 43.04466247558594,
      "activations/layer22_attention_weight_min": -32.08802795410156,
      "activations/layer23_attention_weight_max": 32.93091583251953,
      "activations/layer23_attention_weight_min": -26.181493759155273,
      "activations/layer2_attention_weight_max": 34.80255126953125,
      "activations/layer2_attention_weight_min": -33.50082778930664,
      "activations/layer3_attention_weight_max": 109.73258972167969,
      "activations/layer3_attention_weight_min": -109.9373779296875,
      "activations/layer4_attention_weight_max": 116.19691467285156,
      "activations/layer4_attention_weight_min": -114.90895080566406,
      "activations/layer5_attention_weight_max": 53.94886016845703,
      "activations/layer5_attention_weight_min": -61.980308532714844,
      "activations/layer6_attention_weight_max": 49.72361755371094,
      "activations/layer6_attention_weight_min": -51.05078887939453,
      "activations/layer7_attention_weight_max": 110.65391540527344,
      "activations/layer7_attention_weight_min": -115.0279769897461,
      "activations/layer8_attention_weight_max": 47.46836853027344,
      "activations/layer8_attention_weight_min": -50.58726119995117,
      "activations/layer9_attention_weight_max": 36.073143005371094,
      "activations/layer9_attention_weight_min": -36.68088150024414,
      "epoch": 22.52,
      "learning_rate": 4.810984848484848e-06,
      "loss": 2.7043,
      "step": 387600
    },
    {
      "activations/layer0_attention_weight_max": 16.658348083496094,
      "activations/layer0_attention_weight_min": -14.268935203552246,
      "activations/layer10_attention_weight_max": 33.9043083190918,
      "activations/layer10_attention_weight_min": -35.55141067504883,
      "activations/layer11_attention_weight_max": 33.70626449584961,
      "activations/layer11_attention_weight_min": -35.46870040893555,
      "activations/layer12_attention_weight_max": 25.718244552612305,
      "activations/layer12_attention_weight_min": -27.21375846862793,
      "activations/layer13_attention_weight_max": 41.482933044433594,
      "activations/layer13_attention_weight_min": -37.15681076049805,
      "activations/layer14_attention_weight_max": 42.28192901611328,
      "activations/layer14_attention_weight_min": -38.33039855957031,
      "activations/layer15_attention_weight_max": 38.55901336669922,
      "activations/layer15_attention_weight_min": -36.31337356567383,
      "activations/layer16_attention_weight_max": 31.617053985595703,
      "activations/layer16_attention_weight_min": -28.282289505004883,
      "activations/layer17_attention_weight_max": 30.858564376831055,
      "activations/layer17_attention_weight_min": -25.687721252441406,
      "activations/layer18_attention_weight_max": 33.309364318847656,
      "activations/layer18_attention_weight_min": -26.57016944885254,
      "activations/layer19_attention_weight_max": 37.22753143310547,
      "activations/layer19_attention_weight_min": -31.523557662963867,
      "activations/layer1_attention_weight_max": 16.139568328857422,
      "activations/layer1_attention_weight_min": -14.945960998535156,
      "activations/layer20_attention_weight_max": 32.08582305908203,
      "activations/layer20_attention_weight_min": -27.602989196777344,
      "activations/layer21_attention_weight_max": 31.587722778320312,
      "activations/layer21_attention_weight_min": -25.03512954711914,
      "activations/layer22_attention_weight_max": 41.32487106323242,
      "activations/layer22_attention_weight_min": -31.372325897216797,
      "activations/layer23_attention_weight_max": 33.97467803955078,
      "activations/layer23_attention_weight_min": -25.677825927734375,
      "activations/layer2_attention_weight_max": 34.54926300048828,
      "activations/layer2_attention_weight_min": -33.27268981933594,
      "activations/layer3_attention_weight_max": 109.01262664794922,
      "activations/layer3_attention_weight_min": -108.29696655273438,
      "activations/layer4_attention_weight_max": 109.8630599975586,
      "activations/layer4_attention_weight_min": -113.50347900390625,
      "activations/layer5_attention_weight_max": 53.268310546875,
      "activations/layer5_attention_weight_min": -63.704647064208984,
      "activations/layer6_attention_weight_max": 47.2682991027832,
      "activations/layer6_attention_weight_min": -48.55077362060547,
      "activations/layer7_attention_weight_max": 98.21720123291016,
      "activations/layer7_attention_weight_min": -102.36014556884766,
      "activations/layer8_attention_weight_max": 42.05096435546875,
      "activations/layer8_attention_weight_min": -44.64093017578125,
      "activations/layer9_attention_weight_max": 31.413034439086914,
      "activations/layer9_attention_weight_min": -34.16193771362305,
      "epoch": 22.52,
      "learning_rate": 4.792045454545454e-06,
      "loss": 2.7094,
      "step": 387650
    },
    {
      "activations/layer0_attention_weight_max": 16.93816566467285,
      "activations/layer0_attention_weight_min": -13.690040588378906,
      "activations/layer10_attention_weight_max": 34.49443054199219,
      "activations/layer10_attention_weight_min": -36.04015350341797,
      "activations/layer11_attention_weight_max": 35.1173095703125,
      "activations/layer11_attention_weight_min": -36.82752227783203,
      "activations/layer12_attention_weight_max": 27.284786224365234,
      "activations/layer12_attention_weight_min": -27.229448318481445,
      "activations/layer13_attention_weight_max": 39.29879379272461,
      "activations/layer13_attention_weight_min": -35.69527816772461,
      "activations/layer14_attention_weight_max": 41.2226676940918,
      "activations/layer14_attention_weight_min": -38.306007385253906,
      "activations/layer15_attention_weight_max": 37.56477355957031,
      "activations/layer15_attention_weight_min": -36.33108139038086,
      "activations/layer16_attention_weight_max": 29.28598976135254,
      "activations/layer16_attention_weight_min": -28.083494186401367,
      "activations/layer17_attention_weight_max": 29.21329116821289,
      "activations/layer17_attention_weight_min": -27.915063858032227,
      "activations/layer18_attention_weight_max": 34.16929626464844,
      "activations/layer18_attention_weight_min": -28.959688186645508,
      "activations/layer19_attention_weight_max": 32.8870849609375,
      "activations/layer19_attention_weight_min": -32.44568634033203,
      "activations/layer1_attention_weight_max": 15.750941276550293,
      "activations/layer1_attention_weight_min": -15.91296672821045,
      "activations/layer20_attention_weight_max": 27.454601287841797,
      "activations/layer20_attention_weight_min": -26.615293502807617,
      "activations/layer21_attention_weight_max": 27.102996826171875,
      "activations/layer21_attention_weight_min": -27.046123504638672,
      "activations/layer22_attention_weight_max": 40.35558319091797,
      "activations/layer22_attention_weight_min": -32.989479064941406,
      "activations/layer23_attention_weight_max": 30.992496490478516,
      "activations/layer23_attention_weight_min": -25.72614860534668,
      "activations/layer2_attention_weight_max": 34.31258010864258,
      "activations/layer2_attention_weight_min": -33.6807746887207,
      "activations/layer3_attention_weight_max": 106.39044189453125,
      "activations/layer3_attention_weight_min": -109.02749633789062,
      "activations/layer4_attention_weight_max": 107.85192108154297,
      "activations/layer4_attention_weight_min": -115.2201156616211,
      "activations/layer5_attention_weight_max": 49.448585510253906,
      "activations/layer5_attention_weight_min": -65.11795043945312,
      "activations/layer6_attention_weight_max": 46.3240852355957,
      "activations/layer6_attention_weight_min": -49.521663665771484,
      "activations/layer7_attention_weight_max": 96.98757934570312,
      "activations/layer7_attention_weight_min": -101.03311920166016,
      "activations/layer8_attention_weight_max": 44.0914306640625,
      "activations/layer8_attention_weight_min": -49.911556243896484,
      "activations/layer9_attention_weight_max": 32.79399871826172,
      "activations/layer9_attention_weight_min": -37.25410842895508,
      "epoch": 22.53,
      "learning_rate": 4.77310606060606e-06,
      "loss": 2.6926,
      "step": 387700
    },
    {
      "activations/layer0_attention_weight_max": 17.7607421875,
      "activations/layer0_attention_weight_min": -14.675431251525879,
      "activations/layer10_attention_weight_max": 33.65402603149414,
      "activations/layer10_attention_weight_min": -33.32374954223633,
      "activations/layer11_attention_weight_max": 32.95942687988281,
      "activations/layer11_attention_weight_min": -34.075706481933594,
      "activations/layer12_attention_weight_max": 26.03490447998047,
      "activations/layer12_attention_weight_min": -26.570642471313477,
      "activations/layer13_attention_weight_max": 39.03783416748047,
      "activations/layer13_attention_weight_min": -35.988914489746094,
      "activations/layer14_attention_weight_max": 42.01660919189453,
      "activations/layer14_attention_weight_min": -39.05024719238281,
      "activations/layer15_attention_weight_max": 37.34107208251953,
      "activations/layer15_attention_weight_min": -40.728919982910156,
      "activations/layer16_attention_weight_max": 28.129798889160156,
      "activations/layer16_attention_weight_min": -28.71796226501465,
      "activations/layer17_attention_weight_max": 30.367652893066406,
      "activations/layer17_attention_weight_min": -27.749977111816406,
      "activations/layer18_attention_weight_max": 33.7150764465332,
      "activations/layer18_attention_weight_min": -29.765596389770508,
      "activations/layer19_attention_weight_max": 34.810829162597656,
      "activations/layer19_attention_weight_min": -32.16974639892578,
      "activations/layer1_attention_weight_max": 16.204317092895508,
      "activations/layer1_attention_weight_min": -16.04509162902832,
      "activations/layer20_attention_weight_max": 28.838499069213867,
      "activations/layer20_attention_weight_min": -26.296550750732422,
      "activations/layer21_attention_weight_max": 29.781082153320312,
      "activations/layer21_attention_weight_min": -25.73525047302246,
      "activations/layer22_attention_weight_max": 40.889564514160156,
      "activations/layer22_attention_weight_min": -34.1771125793457,
      "activations/layer23_attention_weight_max": 31.40530776977539,
      "activations/layer23_attention_weight_min": -26.47142791748047,
      "activations/layer2_attention_weight_max": 34.48258590698242,
      "activations/layer2_attention_weight_min": -32.92839431762695,
      "activations/layer3_attention_weight_max": 108.46836853027344,
      "activations/layer3_attention_weight_min": -107.6797103881836,
      "activations/layer4_attention_weight_max": 112.10595703125,
      "activations/layer4_attention_weight_min": -112.1400375366211,
      "activations/layer5_attention_weight_max": 50.46543884277344,
      "activations/layer5_attention_weight_min": -66.51142883300781,
      "activations/layer6_attention_weight_max": 44.99898910522461,
      "activations/layer6_attention_weight_min": -47.45656967163086,
      "activations/layer7_attention_weight_max": 99.9349136352539,
      "activations/layer7_attention_weight_min": -95.92281341552734,
      "activations/layer8_attention_weight_max": 44.919898986816406,
      "activations/layer8_attention_weight_min": -43.51486587524414,
      "activations/layer9_attention_weight_max": 32.32942199707031,
      "activations/layer9_attention_weight_min": -33.146995544433594,
      "epoch": 22.53,
      "learning_rate": 4.754166666666666e-06,
      "loss": 2.6826,
      "step": 387750
    },
    {
      "activations/layer0_attention_weight_max": 17.64027976989746,
      "activations/layer0_attention_weight_min": -14.593311309814453,
      "activations/layer10_attention_weight_max": 32.61417770385742,
      "activations/layer10_attention_weight_min": -36.33201217651367,
      "activations/layer11_attention_weight_max": 32.7126350402832,
      "activations/layer11_attention_weight_min": -36.03595733642578,
      "activations/layer12_attention_weight_max": 26.693897247314453,
      "activations/layer12_attention_weight_min": -26.62897491455078,
      "activations/layer13_attention_weight_max": 37.92814254760742,
      "activations/layer13_attention_weight_min": -36.46523666381836,
      "activations/layer14_attention_weight_max": 38.26591873168945,
      "activations/layer14_attention_weight_min": -37.51996994018555,
      "activations/layer15_attention_weight_max": 37.75241470336914,
      "activations/layer15_attention_weight_min": -37.19355392456055,
      "activations/layer16_attention_weight_max": 29.60362434387207,
      "activations/layer16_attention_weight_min": -28.896482467651367,
      "activations/layer17_attention_weight_max": 30.687856674194336,
      "activations/layer17_attention_weight_min": -26.7569522857666,
      "activations/layer18_attention_weight_max": 28.659826278686523,
      "activations/layer18_attention_weight_min": -26.43365478515625,
      "activations/layer19_attention_weight_max": 32.43313217163086,
      "activations/layer19_attention_weight_min": -31.29664421081543,
      "activations/layer1_attention_weight_max": 15.091517448425293,
      "activations/layer1_attention_weight_min": -14.262055397033691,
      "activations/layer20_attention_weight_max": 27.522682189941406,
      "activations/layer20_attention_weight_min": -24.85869026184082,
      "activations/layer21_attention_weight_max": 29.057918548583984,
      "activations/layer21_attention_weight_min": -25.965496063232422,
      "activations/layer22_attention_weight_max": 41.36250686645508,
      "activations/layer22_attention_weight_min": -34.62287139892578,
      "activations/layer23_attention_weight_max": 33.88019943237305,
      "activations/layer23_attention_weight_min": -26.67658042907715,
      "activations/layer2_attention_weight_max": 33.64651107788086,
      "activations/layer2_attention_weight_min": -33.35626220703125,
      "activations/layer3_attention_weight_max": 105.115478515625,
      "activations/layer3_attention_weight_min": -107.7554702758789,
      "activations/layer4_attention_weight_max": 110.43562316894531,
      "activations/layer4_attention_weight_min": -113.92827606201172,
      "activations/layer5_attention_weight_max": 52.23809814453125,
      "activations/layer5_attention_weight_min": -61.434051513671875,
      "activations/layer6_attention_weight_max": 43.88225555419922,
      "activations/layer6_attention_weight_min": -49.39856719970703,
      "activations/layer7_attention_weight_max": 96.70105743408203,
      "activations/layer7_attention_weight_min": -107.06217956542969,
      "activations/layer8_attention_weight_max": 40.24820327758789,
      "activations/layer8_attention_weight_min": -44.7332649230957,
      "activations/layer9_attention_weight_max": 32.06669616699219,
      "activations/layer9_attention_weight_min": -34.77818298339844,
      "epoch": 22.53,
      "learning_rate": 4.7352272727272725e-06,
      "loss": 2.7017,
      "step": 387800
    },
    {
      "activations/layer0_attention_weight_max": 16.59073257446289,
      "activations/layer0_attention_weight_min": -14.637715339660645,
      "activations/layer10_attention_weight_max": 38.813751220703125,
      "activations/layer10_attention_weight_min": -38.1881217956543,
      "activations/layer11_attention_weight_max": 38.37913513183594,
      "activations/layer11_attention_weight_min": -38.38412857055664,
      "activations/layer12_attention_weight_max": 26.533180236816406,
      "activations/layer12_attention_weight_min": -27.540138244628906,
      "activations/layer13_attention_weight_max": 41.585906982421875,
      "activations/layer13_attention_weight_min": -37.10243606567383,
      "activations/layer14_attention_weight_max": 42.81243896484375,
      "activations/layer14_attention_weight_min": -40.07254409790039,
      "activations/layer15_attention_weight_max": 40.04640197753906,
      "activations/layer15_attention_weight_min": -39.33869171142578,
      "activations/layer16_attention_weight_max": 32.8206672668457,
      "activations/layer16_attention_weight_min": -27.60715103149414,
      "activations/layer17_attention_weight_max": 28.302791595458984,
      "activations/layer17_attention_weight_min": -25.677696228027344,
      "activations/layer18_attention_weight_max": 32.17422103881836,
      "activations/layer18_attention_weight_min": -25.746440887451172,
      "activations/layer19_attention_weight_max": 33.91469955444336,
      "activations/layer19_attention_weight_min": -30.259952545166016,
      "activations/layer1_attention_weight_max": 16.24295997619629,
      "activations/layer1_attention_weight_min": -14.76009464263916,
      "activations/layer20_attention_weight_max": 27.365215301513672,
      "activations/layer20_attention_weight_min": -25.644174575805664,
      "activations/layer21_attention_weight_max": 26.19852066040039,
      "activations/layer21_attention_weight_min": -24.154863357543945,
      "activations/layer22_attention_weight_max": 39.965396881103516,
      "activations/layer22_attention_weight_min": -32.204708099365234,
      "activations/layer23_attention_weight_max": 28.906370162963867,
      "activations/layer23_attention_weight_min": -24.871030807495117,
      "activations/layer2_attention_weight_max": 36.752410888671875,
      "activations/layer2_attention_weight_min": -33.58421325683594,
      "activations/layer3_attention_weight_max": 108.96672821044922,
      "activations/layer3_attention_weight_min": -115.86286926269531,
      "activations/layer4_attention_weight_max": 110.4201889038086,
      "activations/layer4_attention_weight_min": -113.06507873535156,
      "activations/layer5_attention_weight_max": 51.22733688354492,
      "activations/layer5_attention_weight_min": -63.112464904785156,
      "activations/layer6_attention_weight_max": 45.76851272583008,
      "activations/layer6_attention_weight_min": -48.248374938964844,
      "activations/layer7_attention_weight_max": 98.03659057617188,
      "activations/layer7_attention_weight_min": -103.4782485961914,
      "activations/layer8_attention_weight_max": 43.47185516357422,
      "activations/layer8_attention_weight_min": -47.967445373535156,
      "activations/layer9_attention_weight_max": 33.95064163208008,
      "activations/layer9_attention_weight_min": -36.203243255615234,
      "epoch": 22.54,
      "learning_rate": 4.716287878787879e-06,
      "loss": 2.6868,
      "step": 387850
    },
    {
      "activations/layer0_attention_weight_max": 16.697309494018555,
      "activations/layer0_attention_weight_min": -14.390007019042969,
      "activations/layer10_attention_weight_max": 34.45649337768555,
      "activations/layer10_attention_weight_min": -37.41917037963867,
      "activations/layer11_attention_weight_max": 34.199283599853516,
      "activations/layer11_attention_weight_min": -35.31184387207031,
      "activations/layer12_attention_weight_max": 29.51410484313965,
      "activations/layer12_attention_weight_min": -28.242311477661133,
      "activations/layer13_attention_weight_max": 41.805625915527344,
      "activations/layer13_attention_weight_min": -36.3704833984375,
      "activations/layer14_attention_weight_max": 43.53645706176758,
      "activations/layer14_attention_weight_min": -37.437252044677734,
      "activations/layer15_attention_weight_max": 39.31684494018555,
      "activations/layer15_attention_weight_min": -40.99174118041992,
      "activations/layer16_attention_weight_max": 30.279903411865234,
      "activations/layer16_attention_weight_min": -27.231515884399414,
      "activations/layer17_attention_weight_max": 30.372528076171875,
      "activations/layer17_attention_weight_min": -27.44377326965332,
      "activations/layer18_attention_weight_max": 32.467342376708984,
      "activations/layer18_attention_weight_min": -28.368717193603516,
      "activations/layer19_attention_weight_max": 33.28741455078125,
      "activations/layer19_attention_weight_min": -32.13311004638672,
      "activations/layer1_attention_weight_max": 15.790974617004395,
      "activations/layer1_attention_weight_min": -15.085611343383789,
      "activations/layer20_attention_weight_max": 30.62187385559082,
      "activations/layer20_attention_weight_min": -26.063892364501953,
      "activations/layer21_attention_weight_max": 28.964672088623047,
      "activations/layer21_attention_weight_min": -26.055709838867188,
      "activations/layer22_attention_weight_max": 40.9299201965332,
      "activations/layer22_attention_weight_min": -31.848310470581055,
      "activations/layer23_attention_weight_max": 36.36481475830078,
      "activations/layer23_attention_weight_min": -27.909894943237305,
      "activations/layer2_attention_weight_max": 34.29517364501953,
      "activations/layer2_attention_weight_min": -33.23003387451172,
      "activations/layer3_attention_weight_max": 107.9781723022461,
      "activations/layer3_attention_weight_min": -108.79988098144531,
      "activations/layer4_attention_weight_max": 108.81143951416016,
      "activations/layer4_attention_weight_min": -114.87298583984375,
      "activations/layer5_attention_weight_max": 50.49171447753906,
      "activations/layer5_attention_weight_min": -60.19995880126953,
      "activations/layer6_attention_weight_max": 45.16115951538086,
      "activations/layer6_attention_weight_min": -48.78133010864258,
      "activations/layer7_attention_weight_max": 95.3182601928711,
      "activations/layer7_attention_weight_min": -102.20072937011719,
      "activations/layer8_attention_weight_max": 40.750606536865234,
      "activations/layer8_attention_weight_min": -46.184471130371094,
      "activations/layer9_attention_weight_max": 32.64284896850586,
      "activations/layer9_attention_weight_min": -34.51787567138672,
      "epoch": 22.54,
      "learning_rate": 4.697348484848485e-06,
      "loss": 2.6773,
      "step": 387900
    },
    {
      "activations/layer0_attention_weight_max": 17.399517059326172,
      "activations/layer0_attention_weight_min": -15.117310523986816,
      "activations/layer10_attention_weight_max": 35.69231033325195,
      "activations/layer10_attention_weight_min": -37.72505569458008,
      "activations/layer11_attention_weight_max": 35.872947692871094,
      "activations/layer11_attention_weight_min": -39.365943908691406,
      "activations/layer12_attention_weight_max": 27.280160903930664,
      "activations/layer12_attention_weight_min": -26.552438735961914,
      "activations/layer13_attention_weight_max": 40.97013473510742,
      "activations/layer13_attention_weight_min": -36.433956146240234,
      "activations/layer14_attention_weight_max": 44.2866325378418,
      "activations/layer14_attention_weight_min": -38.808311462402344,
      "activations/layer15_attention_weight_max": 41.96696853637695,
      "activations/layer15_attention_weight_min": -37.037574768066406,
      "activations/layer16_attention_weight_max": 30.381994247436523,
      "activations/layer16_attention_weight_min": -29.155813217163086,
      "activations/layer17_attention_weight_max": 31.259145736694336,
      "activations/layer17_attention_weight_min": -26.679515838623047,
      "activations/layer18_attention_weight_max": 31.072294235229492,
      "activations/layer18_attention_weight_min": -29.060466766357422,
      "activations/layer19_attention_weight_max": 36.870635986328125,
      "activations/layer19_attention_weight_min": -31.2744140625,
      "activations/layer1_attention_weight_max": 16.243505477905273,
      "activations/layer1_attention_weight_min": -15.221954345703125,
      "activations/layer20_attention_weight_max": 29.48095703125,
      "activations/layer20_attention_weight_min": -24.28878402709961,
      "activations/layer21_attention_weight_max": 28.94041633605957,
      "activations/layer21_attention_weight_min": -25.001554489135742,
      "activations/layer22_attention_weight_max": 42.94628143310547,
      "activations/layer22_attention_weight_min": -32.353031158447266,
      "activations/layer23_attention_weight_max": 32.372684478759766,
      "activations/layer23_attention_weight_min": -27.381370544433594,
      "activations/layer2_attention_weight_max": 36.38846206665039,
      "activations/layer2_attention_weight_min": -33.247520446777344,
      "activations/layer3_attention_weight_max": 108.49449920654297,
      "activations/layer3_attention_weight_min": -108.04228210449219,
      "activations/layer4_attention_weight_max": 111.46142578125,
      "activations/layer4_attention_weight_min": -111.0719223022461,
      "activations/layer5_attention_weight_max": 51.35657501220703,
      "activations/layer5_attention_weight_min": -61.6292839050293,
      "activations/layer6_attention_weight_max": 46.58112335205078,
      "activations/layer6_attention_weight_min": -48.631858825683594,
      "activations/layer7_attention_weight_max": 99.86734771728516,
      "activations/layer7_attention_weight_min": -103.30387878417969,
      "activations/layer8_attention_weight_max": 42.75537109375,
      "activations/layer8_attention_weight_min": -47.83720779418945,
      "activations/layer9_attention_weight_max": 35.0609130859375,
      "activations/layer9_attention_weight_min": -37.2140998840332,
      "epoch": 22.54,
      "learning_rate": 4.6784090909090905e-06,
      "loss": 2.707,
      "step": 387950
    },
    {
      "activations/layer0_attention_weight_max": 17.224411010742188,
      "activations/layer0_attention_weight_min": -14.596538543701172,
      "activations/layer10_attention_weight_max": 36.56541061401367,
      "activations/layer10_attention_weight_min": -36.39216995239258,
      "activations/layer11_attention_weight_max": 36.503318786621094,
      "activations/layer11_attention_weight_min": -37.18726348876953,
      "activations/layer12_attention_weight_max": 26.899721145629883,
      "activations/layer12_attention_weight_min": -25.503570556640625,
      "activations/layer13_attention_weight_max": 40.3415641784668,
      "activations/layer13_attention_weight_min": -34.02899169921875,
      "activations/layer14_attention_weight_max": 41.391727447509766,
      "activations/layer14_attention_weight_min": -36.46091842651367,
      "activations/layer15_attention_weight_max": 38.81996154785156,
      "activations/layer15_attention_weight_min": -35.95640182495117,
      "activations/layer16_attention_weight_max": 28.700519561767578,
      "activations/layer16_attention_weight_min": -28.35707664489746,
      "activations/layer17_attention_weight_max": 31.001216888427734,
      "activations/layer17_attention_weight_min": -27.872314453125,
      "activations/layer18_attention_weight_max": 32.9422721862793,
      "activations/layer18_attention_weight_min": -25.57894515991211,
      "activations/layer19_attention_weight_max": 33.97604751586914,
      "activations/layer19_attention_weight_min": -30.5998477935791,
      "activations/layer1_attention_weight_max": 16.53784942626953,
      "activations/layer1_attention_weight_min": -14.707926750183105,
      "activations/layer20_attention_weight_max": 29.832162857055664,
      "activations/layer20_attention_weight_min": -25.091096878051758,
      "activations/layer21_attention_weight_max": 27.753726959228516,
      "activations/layer21_attention_weight_min": -26.137449264526367,
      "activations/layer22_attention_weight_max": 43.04785919189453,
      "activations/layer22_attention_weight_min": -35.24605941772461,
      "activations/layer23_attention_weight_max": 33.16651153564453,
      "activations/layer23_attention_weight_min": -26.946945190429688,
      "activations/layer2_attention_weight_max": 33.992549896240234,
      "activations/layer2_attention_weight_min": -34.397918701171875,
      "activations/layer3_attention_weight_max": 110.33990478515625,
      "activations/layer3_attention_weight_min": -110.05352020263672,
      "activations/layer4_attention_weight_max": 116.30238342285156,
      "activations/layer4_attention_weight_min": -122.41816711425781,
      "activations/layer5_attention_weight_max": 53.83424377441406,
      "activations/layer5_attention_weight_min": -61.95354461669922,
      "activations/layer6_attention_weight_max": 46.923343658447266,
      "activations/layer6_attention_weight_min": -51.17332077026367,
      "activations/layer7_attention_weight_max": 102.50144958496094,
      "activations/layer7_attention_weight_min": -102.56333923339844,
      "activations/layer8_attention_weight_max": 42.60708999633789,
      "activations/layer8_attention_weight_min": -45.9897575378418,
      "activations/layer9_attention_weight_max": 33.13439178466797,
      "activations/layer9_attention_weight_min": -35.828765869140625,
      "epoch": 22.55,
      "learning_rate": 4.659469696969696e-06,
      "loss": 2.6933,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4421,
      "eval_samples_per_second": 508.644,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4421,
      "eval_openwebtext_samples_per_second": 508.644,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.923,
      "eval_wikitext_samples_per_second": 237.127,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_lambada_loss": 2.306640625,
      "eval_lambada_ppl": 10.040637668042795,
      "eval_lambada_runtime": 9.5256,
      "eval_lambada_samples_per_second": 511.15,
      "step": 388000
    },
    {
      "activations/layer0_attention_weight_max": 16.379859924316406,
      "activations/layer0_attention_weight_min": -16.306486129760742,
      "activations/layer10_attention_weight_max": 36.308345794677734,
      "activations/layer10_attention_weight_min": -36.27900695800781,
      "activations/layer11_attention_weight_max": 33.79084777832031,
      "activations/layer11_attention_weight_min": -35.66511535644531,
      "activations/layer12_attention_weight_max": 26.727251052856445,
      "activations/layer12_attention_weight_min": -27.352497100830078,
      "activations/layer13_attention_weight_max": 38.4979248046875,
      "activations/layer13_attention_weight_min": -35.70918273925781,
      "activations/layer14_attention_weight_max": 41.949310302734375,
      "activations/layer14_attention_weight_min": -37.33925247192383,
      "activations/layer15_attention_weight_max": 37.694034576416016,
      "activations/layer15_attention_weight_min": -36.34459686279297,
      "activations/layer16_attention_weight_max": 27.318418502807617,
      "activations/layer16_attention_weight_min": -27.574541091918945,
      "activations/layer17_attention_weight_max": 28.25330924987793,
      "activations/layer17_attention_weight_min": -26.053176879882812,
      "activations/layer18_attention_weight_max": 33.13859939575195,
      "activations/layer18_attention_weight_min": -26.278444290161133,
      "activations/layer19_attention_weight_max": 32.30872344970703,
      "activations/layer19_attention_weight_min": -32.07431411743164,
      "activations/layer1_attention_weight_max": 15.381414413452148,
      "activations/layer1_attention_weight_min": -14.826652526855469,
      "activations/layer20_attention_weight_max": 27.476598739624023,
      "activations/layer20_attention_weight_min": -25.821210861206055,
      "activations/layer21_attention_weight_max": 27.399839401245117,
      "activations/layer21_attention_weight_min": -24.51487922668457,
      "activations/layer22_attention_weight_max": 46.95710754394531,
      "activations/layer22_attention_weight_min": -31.666738510131836,
      "activations/layer23_attention_weight_max": 31.94359588623047,
      "activations/layer23_attention_weight_min": -25.62131118774414,
      "activations/layer2_attention_weight_max": 34.183319091796875,
      "activations/layer2_attention_weight_min": -34.57713317871094,
      "activations/layer3_attention_weight_max": 108.74817657470703,
      "activations/layer3_attention_weight_min": -110.87400817871094,
      "activations/layer4_attention_weight_max": 115.16088104248047,
      "activations/layer4_attention_weight_min": -117.3329849243164,
      "activations/layer5_attention_weight_max": 53.0516357421875,
      "activations/layer5_attention_weight_min": -64.2156753540039,
      "activations/layer6_attention_weight_max": 47.10853958129883,
      "activations/layer6_attention_weight_min": -50.379737854003906,
      "activations/layer7_attention_weight_max": 98.52951049804688,
      "activations/layer7_attention_weight_min": -101.91002655029297,
      "activations/layer8_attention_weight_max": 46.52838134765625,
      "activations/layer8_attention_weight_min": -47.23622512817383,
      "activations/layer9_attention_weight_max": 33.83893966674805,
      "activations/layer9_attention_weight_min": -36.46977615356445,
      "epoch": 22.55,
      "learning_rate": 4.640530303030303e-06,
      "loss": 2.6918,
      "step": 388050
    },
    {
      "activations/layer0_attention_weight_max": 17.552539825439453,
      "activations/layer0_attention_weight_min": -16.34458351135254,
      "activations/layer10_attention_weight_max": 35.98628616333008,
      "activations/layer10_attention_weight_min": -35.60469436645508,
      "activations/layer11_attention_weight_max": 36.566768646240234,
      "activations/layer11_attention_weight_min": -35.3031005859375,
      "activations/layer12_attention_weight_max": 29.363136291503906,
      "activations/layer12_attention_weight_min": -27.725305557250977,
      "activations/layer13_attention_weight_max": 43.05936813354492,
      "activations/layer13_attention_weight_min": -36.05945587158203,
      "activations/layer14_attention_weight_max": 46.46906280517578,
      "activations/layer14_attention_weight_min": -38.740779876708984,
      "activations/layer15_attention_weight_max": 43.087646484375,
      "activations/layer15_attention_weight_min": -39.44973373413086,
      "activations/layer16_attention_weight_max": 31.95894432067871,
      "activations/layer16_attention_weight_min": -29.477022171020508,
      "activations/layer17_attention_weight_max": 30.92446517944336,
      "activations/layer17_attention_weight_min": -27.13587760925293,
      "activations/layer18_attention_weight_max": 35.445465087890625,
      "activations/layer18_attention_weight_min": -26.59375,
      "activations/layer19_attention_weight_max": 40.8065185546875,
      "activations/layer19_attention_weight_min": -36.283836364746094,
      "activations/layer1_attention_weight_max": 15.00784969329834,
      "activations/layer1_attention_weight_min": -15.787176132202148,
      "activations/layer20_attention_weight_max": 34.0910758972168,
      "activations/layer20_attention_weight_min": -27.615266799926758,
      "activations/layer21_attention_weight_max": 35.12164306640625,
      "activations/layer21_attention_weight_min": -27.121124267578125,
      "activations/layer22_attention_weight_max": 46.77861404418945,
      "activations/layer22_attention_weight_min": -33.344200134277344,
      "activations/layer23_attention_weight_max": 33.681846618652344,
      "activations/layer23_attention_weight_min": -27.570913314819336,
      "activations/layer2_attention_weight_max": 34.723472595214844,
      "activations/layer2_attention_weight_min": -33.804237365722656,
      "activations/layer3_attention_weight_max": 106.94889831542969,
      "activations/layer3_attention_weight_min": -110.89718627929688,
      "activations/layer4_attention_weight_max": 113.04994201660156,
      "activations/layer4_attention_weight_min": -113.6730728149414,
      "activations/layer5_attention_weight_max": 52.585540771484375,
      "activations/layer5_attention_weight_min": -64.499267578125,
      "activations/layer6_attention_weight_max": 47.0579719543457,
      "activations/layer6_attention_weight_min": -48.92084884643555,
      "activations/layer7_attention_weight_max": 101.3176498413086,
      "activations/layer7_attention_weight_min": -105.77706146240234,
      "activations/layer8_attention_weight_max": 45.0993537902832,
      "activations/layer8_attention_weight_min": -46.955596923828125,
      "activations/layer9_attention_weight_max": 33.59263610839844,
      "activations/layer9_attention_weight_min": -36.07443618774414,
      "epoch": 22.55,
      "learning_rate": 4.6215909090909085e-06,
      "loss": 2.7088,
      "step": 388100
    },
    {
      "activations/layer0_attention_weight_max": 16.826976776123047,
      "activations/layer0_attention_weight_min": -14.245083808898926,
      "activations/layer10_attention_weight_max": 34.25822830200195,
      "activations/layer10_attention_weight_min": -35.1369514465332,
      "activations/layer11_attention_weight_max": 34.0394172668457,
      "activations/layer11_attention_weight_min": -36.6339111328125,
      "activations/layer12_attention_weight_max": 27.347131729125977,
      "activations/layer12_attention_weight_min": -27.031848907470703,
      "activations/layer13_attention_weight_max": 40.68741989135742,
      "activations/layer13_attention_weight_min": -36.08319091796875,
      "activations/layer14_attention_weight_max": 43.98903274536133,
      "activations/layer14_attention_weight_min": -38.66679382324219,
      "activations/layer15_attention_weight_max": 41.1685791015625,
      "activations/layer15_attention_weight_min": -38.955135345458984,
      "activations/layer16_attention_weight_max": 30.83830451965332,
      "activations/layer16_attention_weight_min": -29.463701248168945,
      "activations/layer17_attention_weight_max": 34.66532516479492,
      "activations/layer17_attention_weight_min": -26.88349723815918,
      "activations/layer18_attention_weight_max": 35.10317611694336,
      "activations/layer18_attention_weight_min": -28.924962997436523,
      "activations/layer19_attention_weight_max": 34.250003814697266,
      "activations/layer19_attention_weight_min": -31.616363525390625,
      "activations/layer1_attention_weight_max": 15.478718757629395,
      "activations/layer1_attention_weight_min": -15.6517972946167,
      "activations/layer20_attention_weight_max": 29.722068786621094,
      "activations/layer20_attention_weight_min": -25.78829002380371,
      "activations/layer21_attention_weight_max": 31.200695037841797,
      "activations/layer21_attention_weight_min": -25.112594604492188,
      "activations/layer22_attention_weight_max": 45.260780334472656,
      "activations/layer22_attention_weight_min": -31.827619552612305,
      "activations/layer23_attention_weight_max": 32.38642501831055,
      "activations/layer23_attention_weight_min": -26.213146209716797,
      "activations/layer2_attention_weight_max": 33.8021240234375,
      "activations/layer2_attention_weight_min": -33.3562126159668,
      "activations/layer3_attention_weight_max": 111.78680419921875,
      "activations/layer3_attention_weight_min": -107.8409652709961,
      "activations/layer4_attention_weight_max": 114.55558013916016,
      "activations/layer4_attention_weight_min": -111.82743072509766,
      "activations/layer5_attention_weight_max": 52.54609298706055,
      "activations/layer5_attention_weight_min": -62.9364128112793,
      "activations/layer6_attention_weight_max": 45.42514419555664,
      "activations/layer6_attention_weight_min": -49.48518371582031,
      "activations/layer7_attention_weight_max": 98.64244079589844,
      "activations/layer7_attention_weight_min": -104.35997772216797,
      "activations/layer8_attention_weight_max": 42.45843505859375,
      "activations/layer8_attention_weight_min": -45.1756706237793,
      "activations/layer9_attention_weight_max": 32.33376693725586,
      "activations/layer9_attention_weight_min": -35.68062973022461,
      "epoch": 22.55,
      "learning_rate": 4.602651515151515e-06,
      "loss": 2.7143,
      "step": 388150
    },
    {
      "activations/layer0_attention_weight_max": 15.868117332458496,
      "activations/layer0_attention_weight_min": -13.83761215209961,
      "activations/layer10_attention_weight_max": 35.22597122192383,
      "activations/layer10_attention_weight_min": -36.62175369262695,
      "activations/layer11_attention_weight_max": 36.40172576904297,
      "activations/layer11_attention_weight_min": -39.95330047607422,
      "activations/layer12_attention_weight_max": 26.555673599243164,
      "activations/layer12_attention_weight_min": -27.103633880615234,
      "activations/layer13_attention_weight_max": 39.93889236450195,
      "activations/layer13_attention_weight_min": -36.08265686035156,
      "activations/layer14_attention_weight_max": 43.422813415527344,
      "activations/layer14_attention_weight_min": -40.145484924316406,
      "activations/layer15_attention_weight_max": 40.8058967590332,
      "activations/layer15_attention_weight_min": -37.7878532409668,
      "activations/layer16_attention_weight_max": 31.355623245239258,
      "activations/layer16_attention_weight_min": -29.824289321899414,
      "activations/layer17_attention_weight_max": 29.752092361450195,
      "activations/layer17_attention_weight_min": -29.224063873291016,
      "activations/layer18_attention_weight_max": 34.41194152832031,
      "activations/layer18_attention_weight_min": -30.594743728637695,
      "activations/layer19_attention_weight_max": 38.05522918701172,
      "activations/layer19_attention_weight_min": -34.519466400146484,
      "activations/layer1_attention_weight_max": 15.354936599731445,
      "activations/layer1_attention_weight_min": -15.04358959197998,
      "activations/layer20_attention_weight_max": 30.562339782714844,
      "activations/layer20_attention_weight_min": -26.235502243041992,
      "activations/layer21_attention_weight_max": 32.43973922729492,
      "activations/layer21_attention_weight_min": -25.621261596679688,
      "activations/layer22_attention_weight_max": 43.25517654418945,
      "activations/layer22_attention_weight_min": -35.25334930419922,
      "activations/layer23_attention_weight_max": 34.59430694580078,
      "activations/layer23_attention_weight_min": -25.622276306152344,
      "activations/layer2_attention_weight_max": 35.20962142944336,
      "activations/layer2_attention_weight_min": -33.050270080566406,
      "activations/layer3_attention_weight_max": 108.13976287841797,
      "activations/layer3_attention_weight_min": -107.73341369628906,
      "activations/layer4_attention_weight_max": 111.68804931640625,
      "activations/layer4_attention_weight_min": -108.8404769897461,
      "activations/layer5_attention_weight_max": 53.98269271850586,
      "activations/layer5_attention_weight_min": -62.47288513183594,
      "activations/layer6_attention_weight_max": 46.01133728027344,
      "activations/layer6_attention_weight_min": -46.82148361206055,
      "activations/layer7_attention_weight_max": 96.17139434814453,
      "activations/layer7_attention_weight_min": -97.40911102294922,
      "activations/layer8_attention_weight_max": 43.6203498840332,
      "activations/layer8_attention_weight_min": -46.86181640625,
      "activations/layer9_attention_weight_max": 33.071449279785156,
      "activations/layer9_attention_weight_min": -35.49615478515625,
      "epoch": 22.56,
      "learning_rate": 4.583712121212121e-06,
      "loss": 2.6929,
      "step": 388200
    },
    {
      "activations/layer0_attention_weight_max": 16.17607307434082,
      "activations/layer0_attention_weight_min": -15.67824649810791,
      "activations/layer10_attention_weight_max": 33.25544357299805,
      "activations/layer10_attention_weight_min": -36.6568603515625,
      "activations/layer11_attention_weight_max": 33.331974029541016,
      "activations/layer11_attention_weight_min": -37.49041748046875,
      "activations/layer12_attention_weight_max": 27.653406143188477,
      "activations/layer12_attention_weight_min": -28.167055130004883,
      "activations/layer13_attention_weight_max": 41.309661865234375,
      "activations/layer13_attention_weight_min": -36.06626510620117,
      "activations/layer14_attention_weight_max": 47.197601318359375,
      "activations/layer14_attention_weight_min": -39.841217041015625,
      "activations/layer15_attention_weight_max": 40.078006744384766,
      "activations/layer15_attention_weight_min": -37.16355895996094,
      "activations/layer16_attention_weight_max": 29.50121307373047,
      "activations/layer16_attention_weight_min": -27.799808502197266,
      "activations/layer17_attention_weight_max": 29.75644302368164,
      "activations/layer17_attention_weight_min": -25.684097290039062,
      "activations/layer18_attention_weight_max": 37.17246627807617,
      "activations/layer18_attention_weight_min": -27.910396575927734,
      "activations/layer19_attention_weight_max": 37.151981353759766,
      "activations/layer19_attention_weight_min": -31.80181884765625,
      "activations/layer1_attention_weight_max": 15.726743698120117,
      "activations/layer1_attention_weight_min": -15.115944862365723,
      "activations/layer20_attention_weight_max": 31.663036346435547,
      "activations/layer20_attention_weight_min": -25.019309997558594,
      "activations/layer21_attention_weight_max": 29.62894630432129,
      "activations/layer21_attention_weight_min": -24.704065322875977,
      "activations/layer22_attention_weight_max": 44.95669937133789,
      "activations/layer22_attention_weight_min": -32.28915786743164,
      "activations/layer23_attention_weight_max": 33.81936264038086,
      "activations/layer23_attention_weight_min": -25.493215560913086,
      "activations/layer2_attention_weight_max": 36.09453582763672,
      "activations/layer2_attention_weight_min": -34.17117691040039,
      "activations/layer3_attention_weight_max": 108.07347106933594,
      "activations/layer3_attention_weight_min": -108.13042449951172,
      "activations/layer4_attention_weight_max": 110.59132385253906,
      "activations/layer4_attention_weight_min": -115.9331283569336,
      "activations/layer5_attention_weight_max": 50.57795715332031,
      "activations/layer5_attention_weight_min": -62.5655517578125,
      "activations/layer6_attention_weight_max": 45.75177764892578,
      "activations/layer6_attention_weight_min": -50.22639083862305,
      "activations/layer7_attention_weight_max": 93.59929656982422,
      "activations/layer7_attention_weight_min": -100.90696716308594,
      "activations/layer8_attention_weight_max": 41.63978958129883,
      "activations/layer8_attention_weight_min": -47.116695404052734,
      "activations/layer9_attention_weight_max": 31.94256019592285,
      "activations/layer9_attention_weight_min": -33.86784744262695,
      "epoch": 22.56,
      "learning_rate": 4.5647727272727266e-06,
      "loss": 2.698,
      "step": 388250
    },
    {
      "activations/layer0_attention_weight_max": 17.053911209106445,
      "activations/layer0_attention_weight_min": -14.450830459594727,
      "activations/layer10_attention_weight_max": 33.774837493896484,
      "activations/layer10_attention_weight_min": -37.441165924072266,
      "activations/layer11_attention_weight_max": 34.606910705566406,
      "activations/layer11_attention_weight_min": -36.1374626159668,
      "activations/layer12_attention_weight_max": 27.3054141998291,
      "activations/layer12_attention_weight_min": -28.618864059448242,
      "activations/layer13_attention_weight_max": 41.44773864746094,
      "activations/layer13_attention_weight_min": -36.710243225097656,
      "activations/layer14_attention_weight_max": 43.898902893066406,
      "activations/layer14_attention_weight_min": -39.99843215942383,
      "activations/layer15_attention_weight_max": 40.36547088623047,
      "activations/layer15_attention_weight_min": -38.31657028198242,
      "activations/layer16_attention_weight_max": 31.3135986328125,
      "activations/layer16_attention_weight_min": -30.56416893005371,
      "activations/layer17_attention_weight_max": 32.29954147338867,
      "activations/layer17_attention_weight_min": -28.52058219909668,
      "activations/layer18_attention_weight_max": 34.009708404541016,
      "activations/layer18_attention_weight_min": -29.797218322753906,
      "activations/layer19_attention_weight_max": 40.73528289794922,
      "activations/layer19_attention_weight_min": -31.91459083557129,
      "activations/layer1_attention_weight_max": 15.59621524810791,
      "activations/layer1_attention_weight_min": -14.76789379119873,
      "activations/layer20_attention_weight_max": 30.9464168548584,
      "activations/layer20_attention_weight_min": -25.91390037536621,
      "activations/layer21_attention_weight_max": 29.835691452026367,
      "activations/layer21_attention_weight_min": -27.55874252319336,
      "activations/layer22_attention_weight_max": 41.928001403808594,
      "activations/layer22_attention_weight_min": -32.764644622802734,
      "activations/layer23_attention_weight_max": 33.26847457885742,
      "activations/layer23_attention_weight_min": -26.33009910583496,
      "activations/layer2_attention_weight_max": 35.058231353759766,
      "activations/layer2_attention_weight_min": -34.620853424072266,
      "activations/layer3_attention_weight_max": 110.39857482910156,
      "activations/layer3_attention_weight_min": -112.76824951171875,
      "activations/layer4_attention_weight_max": 110.15111541748047,
      "activations/layer4_attention_weight_min": -114.56584930419922,
      "activations/layer5_attention_weight_max": 52.20878219604492,
      "activations/layer5_attention_weight_min": -62.53905487060547,
      "activations/layer6_attention_weight_max": 47.06161880493164,
      "activations/layer6_attention_weight_min": -49.74291229248047,
      "activations/layer7_attention_weight_max": 96.35918426513672,
      "activations/layer7_attention_weight_min": -104.53865814208984,
      "activations/layer8_attention_weight_max": 42.350223541259766,
      "activations/layer8_attention_weight_min": -47.742515563964844,
      "activations/layer9_attention_weight_max": 32.06111526489258,
      "activations/layer9_attention_weight_min": -37.36476516723633,
      "epoch": 22.56,
      "learning_rate": 4.545833333333333e-06,
      "loss": 2.7086,
      "step": 388300
    },
    {
      "activations/layer0_attention_weight_max": 17.340608596801758,
      "activations/layer0_attention_weight_min": -14.169017791748047,
      "activations/layer10_attention_weight_max": 35.922882080078125,
      "activations/layer10_attention_weight_min": -34.47798538208008,
      "activations/layer11_attention_weight_max": 33.814064025878906,
      "activations/layer11_attention_weight_min": -33.635948181152344,
      "activations/layer12_attention_weight_max": 30.309551239013672,
      "activations/layer12_attention_weight_min": -28.091188430786133,
      "activations/layer13_attention_weight_max": 44.51414489746094,
      "activations/layer13_attention_weight_min": -37.23646545410156,
      "activations/layer14_attention_weight_max": 44.36017608642578,
      "activations/layer14_attention_weight_min": -38.282142639160156,
      "activations/layer15_attention_weight_max": 41.905311584472656,
      "activations/layer15_attention_weight_min": -37.2057991027832,
      "activations/layer16_attention_weight_max": 29.541582107543945,
      "activations/layer16_attention_weight_min": -29.9945068359375,
      "activations/layer17_attention_weight_max": 31.21755027770996,
      "activations/layer17_attention_weight_min": -27.315624237060547,
      "activations/layer18_attention_weight_max": 34.97417449951172,
      "activations/layer18_attention_weight_min": -26.014545440673828,
      "activations/layer19_attention_weight_max": 33.74908447265625,
      "activations/layer19_attention_weight_min": -31.658084869384766,
      "activations/layer1_attention_weight_max": 16.45641326904297,
      "activations/layer1_attention_weight_min": -14.507377624511719,
      "activations/layer20_attention_weight_max": 29.651477813720703,
      "activations/layer20_attention_weight_min": -24.333860397338867,
      "activations/layer21_attention_weight_max": 29.284671783447266,
      "activations/layer21_attention_weight_min": -24.140756607055664,
      "activations/layer22_attention_weight_max": 41.84833908081055,
      "activations/layer22_attention_weight_min": -31.76909065246582,
      "activations/layer23_attention_weight_max": 32.906715393066406,
      "activations/layer23_attention_weight_min": -25.478130340576172,
      "activations/layer2_attention_weight_max": 35.51262664794922,
      "activations/layer2_attention_weight_min": -33.714515686035156,
      "activations/layer3_attention_weight_max": 103.7399673461914,
      "activations/layer3_attention_weight_min": -106.18292236328125,
      "activations/layer4_attention_weight_max": 108.9201889038086,
      "activations/layer4_attention_weight_min": -112.72124481201172,
      "activations/layer5_attention_weight_max": 49.73474884033203,
      "activations/layer5_attention_weight_min": -58.542030334472656,
      "activations/layer6_attention_weight_max": 44.73292541503906,
      "activations/layer6_attention_weight_min": -50.41572570800781,
      "activations/layer7_attention_weight_max": 95.92814636230469,
      "activations/layer7_attention_weight_min": -96.76068115234375,
      "activations/layer8_attention_weight_max": 41.62489700317383,
      "activations/layer8_attention_weight_min": -43.88151931762695,
      "activations/layer9_attention_weight_max": 31.897672653198242,
      "activations/layer9_attention_weight_min": -34.668373107910156,
      "epoch": 22.57,
      "learning_rate": 4.526893939393939e-06,
      "loss": 2.6965,
      "step": 388350
    },
    {
      "activations/layer0_attention_weight_max": 16.58379364013672,
      "activations/layer0_attention_weight_min": -14.21932315826416,
      "activations/layer10_attention_weight_max": 33.51197052001953,
      "activations/layer10_attention_weight_min": -35.02736282348633,
      "activations/layer11_attention_weight_max": 33.1939811706543,
      "activations/layer11_attention_weight_min": -34.312583923339844,
      "activations/layer12_attention_weight_max": 28.557540893554688,
      "activations/layer12_attention_weight_min": -27.405271530151367,
      "activations/layer13_attention_weight_max": 41.126827239990234,
      "activations/layer13_attention_weight_min": -37.27771759033203,
      "activations/layer14_attention_weight_max": 44.68671417236328,
      "activations/layer14_attention_weight_min": -38.119163513183594,
      "activations/layer15_attention_weight_max": 42.92714309692383,
      "activations/layer15_attention_weight_min": -40.13004684448242,
      "activations/layer16_attention_weight_max": 30.448022842407227,
      "activations/layer16_attention_weight_min": -27.64076805114746,
      "activations/layer17_attention_weight_max": 30.292490005493164,
      "activations/layer17_attention_weight_min": -25.935266494750977,
      "activations/layer18_attention_weight_max": 32.43013000488281,
      "activations/layer18_attention_weight_min": -26.209787368774414,
      "activations/layer19_attention_weight_max": 37.02253341674805,
      "activations/layer19_attention_weight_min": -31.669382095336914,
      "activations/layer1_attention_weight_max": 16.840768814086914,
      "activations/layer1_attention_weight_min": -15.879707336425781,
      "activations/layer20_attention_weight_max": 30.925962448120117,
      "activations/layer20_attention_weight_min": -25.624095916748047,
      "activations/layer21_attention_weight_max": 28.901792526245117,
      "activations/layer21_attention_weight_min": -25.989830017089844,
      "activations/layer22_attention_weight_max": 42.731239318847656,
      "activations/layer22_attention_weight_min": -33.70976638793945,
      "activations/layer23_attention_weight_max": 35.3580207824707,
      "activations/layer23_attention_weight_min": -26.73439598083496,
      "activations/layer2_attention_weight_max": 34.69841384887695,
      "activations/layer2_attention_weight_min": -34.935638427734375,
      "activations/layer3_attention_weight_max": 108.41755676269531,
      "activations/layer3_attention_weight_min": -107.15924072265625,
      "activations/layer4_attention_weight_max": 112.0546646118164,
      "activations/layer4_attention_weight_min": -112.909912109375,
      "activations/layer5_attention_weight_max": 49.76367950439453,
      "activations/layer5_attention_weight_min": -63.12403106689453,
      "activations/layer6_attention_weight_max": 45.45421600341797,
      "activations/layer6_attention_weight_min": -51.05806350708008,
      "activations/layer7_attention_weight_max": 96.8704605102539,
      "activations/layer7_attention_weight_min": -103.91844177246094,
      "activations/layer8_attention_weight_max": 43.19450378417969,
      "activations/layer8_attention_weight_min": -47.45111083984375,
      "activations/layer9_attention_weight_max": 33.37798309326172,
      "activations/layer9_attention_weight_min": -34.63750457763672,
      "epoch": 22.57,
      "learning_rate": 4.5079545454545455e-06,
      "loss": 2.7017,
      "step": 388400
    },
    {
      "activations/layer0_attention_weight_max": 16.974700927734375,
      "activations/layer0_attention_weight_min": -15.684806823730469,
      "activations/layer10_attention_weight_max": 37.294593811035156,
      "activations/layer10_attention_weight_min": -37.401329040527344,
      "activations/layer11_attention_weight_max": 37.33023452758789,
      "activations/layer11_attention_weight_min": -37.49333190917969,
      "activations/layer12_attention_weight_max": 28.741186141967773,
      "activations/layer12_attention_weight_min": -27.905038833618164,
      "activations/layer13_attention_weight_max": 39.45176696777344,
      "activations/layer13_attention_weight_min": -36.967750549316406,
      "activations/layer14_attention_weight_max": 42.422000885009766,
      "activations/layer14_attention_weight_min": -39.477237701416016,
      "activations/layer15_attention_weight_max": 37.0988655090332,
      "activations/layer15_attention_weight_min": -37.064144134521484,
      "activations/layer16_attention_weight_max": 29.05651092529297,
      "activations/layer16_attention_weight_min": -28.199504852294922,
      "activations/layer17_attention_weight_max": 28.84712791442871,
      "activations/layer17_attention_weight_min": -25.593599319458008,
      "activations/layer18_attention_weight_max": 31.032258987426758,
      "activations/layer18_attention_weight_min": -24.190370559692383,
      "activations/layer19_attention_weight_max": 35.42850112915039,
      "activations/layer19_attention_weight_min": -30.9794979095459,
      "activations/layer1_attention_weight_max": 15.237741470336914,
      "activations/layer1_attention_weight_min": -14.113409996032715,
      "activations/layer20_attention_weight_max": 28.271984100341797,
      "activations/layer20_attention_weight_min": -24.644140243530273,
      "activations/layer21_attention_weight_max": 28.35371971130371,
      "activations/layer21_attention_weight_min": -24.721195220947266,
      "activations/layer22_attention_weight_max": 38.526329040527344,
      "activations/layer22_attention_weight_min": -30.883678436279297,
      "activations/layer23_attention_weight_max": 29.90752410888672,
      "activations/layer23_attention_weight_min": -24.661678314208984,
      "activations/layer2_attention_weight_max": 34.53456497192383,
      "activations/layer2_attention_weight_min": -35.49078369140625,
      "activations/layer3_attention_weight_max": 109.85172271728516,
      "activations/layer3_attention_weight_min": -114.16744995117188,
      "activations/layer4_attention_weight_max": 112.16328430175781,
      "activations/layer4_attention_weight_min": -113.114990234375,
      "activations/layer5_attention_weight_max": 50.07429504394531,
      "activations/layer5_attention_weight_min": -60.98003387451172,
      "activations/layer6_attention_weight_max": 47.40009689331055,
      "activations/layer6_attention_weight_min": -50.668636322021484,
      "activations/layer7_attention_weight_max": 99.85513305664062,
      "activations/layer7_attention_weight_min": -107.11837768554688,
      "activations/layer8_attention_weight_max": 46.679988861083984,
      "activations/layer8_attention_weight_min": -49.675392150878906,
      "activations/layer9_attention_weight_max": 35.10718536376953,
      "activations/layer9_attention_weight_min": -37.48820877075195,
      "epoch": 22.57,
      "learning_rate": 4.489015151515151e-06,
      "loss": 2.7068,
      "step": 388450
    },
    {
      "activations/layer0_attention_weight_max": 16.62380599975586,
      "activations/layer0_attention_weight_min": -15.001405715942383,
      "activations/layer10_attention_weight_max": 39.88241958618164,
      "activations/layer10_attention_weight_min": -36.566036224365234,
      "activations/layer11_attention_weight_max": 38.91191101074219,
      "activations/layer11_attention_weight_min": -36.522315979003906,
      "activations/layer12_attention_weight_max": 27.037220001220703,
      "activations/layer12_attention_weight_min": -28.062294006347656,
      "activations/layer13_attention_weight_max": 39.16927719116211,
      "activations/layer13_attention_weight_min": -36.43998718261719,
      "activations/layer14_attention_weight_max": 43.95674133300781,
      "activations/layer14_attention_weight_min": -37.770137786865234,
      "activations/layer15_attention_weight_max": 39.58476257324219,
      "activations/layer15_attention_weight_min": -36.6468391418457,
      "activations/layer16_attention_weight_max": 29.315048217773438,
      "activations/layer16_attention_weight_min": -29.855060577392578,
      "activations/layer17_attention_weight_max": 29.956315994262695,
      "activations/layer17_attention_weight_min": -30.551025390625,
      "activations/layer18_attention_weight_max": 34.59830093383789,
      "activations/layer18_attention_weight_min": -27.67626953125,
      "activations/layer19_attention_weight_max": 38.10185623168945,
      "activations/layer19_attention_weight_min": -35.325843811035156,
      "activations/layer1_attention_weight_max": 15.854028701782227,
      "activations/layer1_attention_weight_min": -15.685647964477539,
      "activations/layer20_attention_weight_max": 29.893430709838867,
      "activations/layer20_attention_weight_min": -29.47393226623535,
      "activations/layer21_attention_weight_max": 27.402830123901367,
      "activations/layer21_attention_weight_min": -27.47765350341797,
      "activations/layer22_attention_weight_max": 41.12062454223633,
      "activations/layer22_attention_weight_min": -33.07054138183594,
      "activations/layer23_attention_weight_max": 33.78961181640625,
      "activations/layer23_attention_weight_min": -27.32596206665039,
      "activations/layer2_attention_weight_max": 35.26617431640625,
      "activations/layer2_attention_weight_min": -35.542213439941406,
      "activations/layer3_attention_weight_max": 112.55606842041016,
      "activations/layer3_attention_weight_min": -113.56419372558594,
      "activations/layer4_attention_weight_max": 114.91949462890625,
      "activations/layer4_attention_weight_min": -113.32585906982422,
      "activations/layer5_attention_weight_max": 51.81758117675781,
      "activations/layer5_attention_weight_min": -61.611812591552734,
      "activations/layer6_attention_weight_max": 46.72557067871094,
      "activations/layer6_attention_weight_min": -48.471275329589844,
      "activations/layer7_attention_weight_max": 99.7728042602539,
      "activations/layer7_attention_weight_min": -102.6715316772461,
      "activations/layer8_attention_weight_max": 45.370872497558594,
      "activations/layer8_attention_weight_min": -45.48292922973633,
      "activations/layer9_attention_weight_max": 34.29737091064453,
      "activations/layer9_attention_weight_min": -35.32204818725586,
      "epoch": 22.57,
      "learning_rate": 4.470075757575757e-06,
      "loss": 2.7033,
      "step": 388500
    },
    {
      "activations/layer0_attention_weight_max": 16.760358810424805,
      "activations/layer0_attention_weight_min": -13.283732414245605,
      "activations/layer10_attention_weight_max": 38.33206558227539,
      "activations/layer10_attention_weight_min": -35.540985107421875,
      "activations/layer11_attention_weight_max": 39.07598876953125,
      "activations/layer11_attention_weight_min": -36.18455505371094,
      "activations/layer12_attention_weight_max": 27.868040084838867,
      "activations/layer12_attention_weight_min": -27.413040161132812,
      "activations/layer13_attention_weight_max": 40.70580291748047,
      "activations/layer13_attention_weight_min": -38.34905242919922,
      "activations/layer14_attention_weight_max": 45.47737121582031,
      "activations/layer14_attention_weight_min": -39.613502502441406,
      "activations/layer15_attention_weight_max": 39.8404655456543,
      "activations/layer15_attention_weight_min": -37.94171905517578,
      "activations/layer16_attention_weight_max": 29.89140510559082,
      "activations/layer16_attention_weight_min": -28.908884048461914,
      "activations/layer17_attention_weight_max": 29.41478157043457,
      "activations/layer17_attention_weight_min": -27.398515701293945,
      "activations/layer18_attention_weight_max": 33.20311737060547,
      "activations/layer18_attention_weight_min": -27.720149993896484,
      "activations/layer19_attention_weight_max": 35.35423278808594,
      "activations/layer19_attention_weight_min": -31.24702262878418,
      "activations/layer1_attention_weight_max": 16.553970336914062,
      "activations/layer1_attention_weight_min": -16.611553192138672,
      "activations/layer20_attention_weight_max": 28.873872756958008,
      "activations/layer20_attention_weight_min": -27.680089950561523,
      "activations/layer21_attention_weight_max": 28.969167709350586,
      "activations/layer21_attention_weight_min": -25.260772705078125,
      "activations/layer22_attention_weight_max": 44.30152130126953,
      "activations/layer22_attention_weight_min": -32.908302307128906,
      "activations/layer23_attention_weight_max": 33.44532775878906,
      "activations/layer23_attention_weight_min": -26.634117126464844,
      "activations/layer2_attention_weight_max": 35.28779983520508,
      "activations/layer2_attention_weight_min": -33.88435745239258,
      "activations/layer3_attention_weight_max": 111.62090301513672,
      "activations/layer3_attention_weight_min": -109.6834945678711,
      "activations/layer4_attention_weight_max": 113.1803970336914,
      "activations/layer4_attention_weight_min": -114.83800506591797,
      "activations/layer5_attention_weight_max": 54.56592559814453,
      "activations/layer5_attention_weight_min": -64.52373504638672,
      "activations/layer6_attention_weight_max": 48.33301544189453,
      "activations/layer6_attention_weight_min": -52.30296325683594,
      "activations/layer7_attention_weight_max": 102.40931701660156,
      "activations/layer7_attention_weight_min": -114.31995391845703,
      "activations/layer8_attention_weight_max": 45.26347732543945,
      "activations/layer8_attention_weight_min": -48.14266586303711,
      "activations/layer9_attention_weight_max": 35.36861038208008,
      "activations/layer9_attention_weight_min": -36.151302337646484,
      "epoch": 22.58,
      "learning_rate": 4.4511363636363635e-06,
      "loss": 2.7015,
      "step": 388550
    },
    {
      "activations/layer0_attention_weight_max": 16.95977020263672,
      "activations/layer0_attention_weight_min": -16.66358184814453,
      "activations/layer10_attention_weight_max": 38.83515167236328,
      "activations/layer10_attention_weight_min": -36.168922424316406,
      "activations/layer11_attention_weight_max": 35.88288879394531,
      "activations/layer11_attention_weight_min": -39.11687088012695,
      "activations/layer12_attention_weight_max": 25.82559585571289,
      "activations/layer12_attention_weight_min": -26.649744033813477,
      "activations/layer13_attention_weight_max": 38.80293273925781,
      "activations/layer13_attention_weight_min": -36.78715133666992,
      "activations/layer14_attention_weight_max": 41.140560150146484,
      "activations/layer14_attention_weight_min": -38.79568099975586,
      "activations/layer15_attention_weight_max": 35.98881149291992,
      "activations/layer15_attention_weight_min": -37.90573501586914,
      "activations/layer16_attention_weight_max": 29.76176643371582,
      "activations/layer16_attention_weight_min": -27.264095306396484,
      "activations/layer17_attention_weight_max": 28.359989166259766,
      "activations/layer17_attention_weight_min": -25.98512077331543,
      "activations/layer18_attention_weight_max": 30.465795516967773,
      "activations/layer18_attention_weight_min": -26.30787467956543,
      "activations/layer19_attention_weight_max": 30.451187133789062,
      "activations/layer19_attention_weight_min": -32.53335189819336,
      "activations/layer1_attention_weight_max": 15.783819198608398,
      "activations/layer1_attention_weight_min": -14.665594100952148,
      "activations/layer20_attention_weight_max": 26.019702911376953,
      "activations/layer20_attention_weight_min": -25.28939437866211,
      "activations/layer21_attention_weight_max": 26.18231201171875,
      "activations/layer21_attention_weight_min": -25.3581600189209,
      "activations/layer22_attention_weight_max": 40.4119987487793,
      "activations/layer22_attention_weight_min": -30.118284225463867,
      "activations/layer23_attention_weight_max": 29.790088653564453,
      "activations/layer23_attention_weight_min": -25.844886779785156,
      "activations/layer2_attention_weight_max": 34.59677505493164,
      "activations/layer2_attention_weight_min": -34.189109802246094,
      "activations/layer3_attention_weight_max": 109.40769958496094,
      "activations/layer3_attention_weight_min": -110.18470764160156,
      "activations/layer4_attention_weight_max": 112.20893859863281,
      "activations/layer4_attention_weight_min": -117.56414031982422,
      "activations/layer5_attention_weight_max": 51.84150314331055,
      "activations/layer5_attention_weight_min": -62.165809631347656,
      "activations/layer6_attention_weight_max": 47.392024993896484,
      "activations/layer6_attention_weight_min": -48.431846618652344,
      "activations/layer7_attention_weight_max": 102.2126693725586,
      "activations/layer7_attention_weight_min": -102.43331146240234,
      "activations/layer8_attention_weight_max": 45.554683685302734,
      "activations/layer8_attention_weight_min": -46.561546325683594,
      "activations/layer9_attention_weight_max": 36.89862060546875,
      "activations/layer9_attention_weight_min": -35.808067321777344,
      "epoch": 22.58,
      "learning_rate": 4.432196969696969e-06,
      "loss": 2.6925,
      "step": 388600
    },
    {
      "activations/layer0_attention_weight_max": 16.847305297851562,
      "activations/layer0_attention_weight_min": -15.848233222961426,
      "activations/layer10_attention_weight_max": 37.651268005371094,
      "activations/layer10_attention_weight_min": -36.202247619628906,
      "activations/layer11_attention_weight_max": 37.110748291015625,
      "activations/layer11_attention_weight_min": -37.46018981933594,
      "activations/layer12_attention_weight_max": 31.075639724731445,
      "activations/layer12_attention_weight_min": -29.946022033691406,
      "activations/layer13_attention_weight_max": 43.39933776855469,
      "activations/layer13_attention_weight_min": -39.88468551635742,
      "activations/layer14_attention_weight_max": 49.09733200073242,
      "activations/layer14_attention_weight_min": -43.15873336791992,
      "activations/layer15_attention_weight_max": 45.132633209228516,
      "activations/layer15_attention_weight_min": -40.13636016845703,
      "activations/layer16_attention_weight_max": 34.217079162597656,
      "activations/layer16_attention_weight_min": -29.36115837097168,
      "activations/layer17_attention_weight_max": 32.861297607421875,
      "activations/layer17_attention_weight_min": -26.888757705688477,
      "activations/layer18_attention_weight_max": 42.26043701171875,
      "activations/layer18_attention_weight_min": -30.5291748046875,
      "activations/layer19_attention_weight_max": 41.043609619140625,
      "activations/layer19_attention_weight_min": -31.902318954467773,
      "activations/layer1_attention_weight_max": 16.473068237304688,
      "activations/layer1_attention_weight_min": -15.274194717407227,
      "activations/layer20_attention_weight_max": 32.72917556762695,
      "activations/layer20_attention_weight_min": -24.649051666259766,
      "activations/layer21_attention_weight_max": 33.1164665222168,
      "activations/layer21_attention_weight_min": -24.657238006591797,
      "activations/layer22_attention_weight_max": 49.38352584838867,
      "activations/layer22_attention_weight_min": -32.23688888549805,
      "activations/layer23_attention_weight_max": 37.528839111328125,
      "activations/layer23_attention_weight_min": -25.326412200927734,
      "activations/layer2_attention_weight_max": 34.37620162963867,
      "activations/layer2_attention_weight_min": -34.60816955566406,
      "activations/layer3_attention_weight_max": 110.98961639404297,
      "activations/layer3_attention_weight_min": -114.61076354980469,
      "activations/layer4_attention_weight_max": 118.85111236572266,
      "activations/layer4_attention_weight_min": -121.70023345947266,
      "activations/layer5_attention_weight_max": 53.403621673583984,
      "activations/layer5_attention_weight_min": -66.56767272949219,
      "activations/layer6_attention_weight_max": 48.65007019042969,
      "activations/layer6_attention_weight_min": -53.09639358520508,
      "activations/layer7_attention_weight_max": 103.32891082763672,
      "activations/layer7_attention_weight_min": -111.61536407470703,
      "activations/layer8_attention_weight_max": 45.541656494140625,
      "activations/layer8_attention_weight_min": -48.70436477661133,
      "activations/layer9_attention_weight_max": 33.68898391723633,
      "activations/layer9_attention_weight_min": -37.11676025390625,
      "epoch": 22.58,
      "learning_rate": 4.413257575757576e-06,
      "loss": 2.7021,
      "step": 388650
    },
    {
      "activations/layer0_attention_weight_max": 16.858078002929688,
      "activations/layer0_attention_weight_min": -13.780323028564453,
      "activations/layer10_attention_weight_max": 35.07879638671875,
      "activations/layer10_attention_weight_min": -34.63180160522461,
      "activations/layer11_attention_weight_max": 34.06877517700195,
      "activations/layer11_attention_weight_min": -35.7861328125,
      "activations/layer12_attention_weight_max": 27.715736389160156,
      "activations/layer12_attention_weight_min": -27.775602340698242,
      "activations/layer13_attention_weight_max": 42.3695068359375,
      "activations/layer13_attention_weight_min": -38.11610412597656,
      "activations/layer14_attention_weight_max": 43.23234939575195,
      "activations/layer14_attention_weight_min": -39.78250503540039,
      "activations/layer15_attention_weight_max": 42.98053741455078,
      "activations/layer15_attention_weight_min": -36.41191482543945,
      "activations/layer16_attention_weight_max": 29.673715591430664,
      "activations/layer16_attention_weight_min": -28.419963836669922,
      "activations/layer17_attention_weight_max": 28.782968521118164,
      "activations/layer17_attention_weight_min": -25.65876579284668,
      "activations/layer18_attention_weight_max": 34.073402404785156,
      "activations/layer18_attention_weight_min": -25.505176544189453,
      "activations/layer19_attention_weight_max": 37.88692092895508,
      "activations/layer19_attention_weight_min": -31.53558349609375,
      "activations/layer1_attention_weight_max": 16.21453094482422,
      "activations/layer1_attention_weight_min": -14.343256950378418,
      "activations/layer20_attention_weight_max": 28.92061424255371,
      "activations/layer20_attention_weight_min": -25.070877075195312,
      "activations/layer21_attention_weight_max": 27.582983016967773,
      "activations/layer21_attention_weight_min": -23.79347801208496,
      "activations/layer22_attention_weight_max": 46.804630279541016,
      "activations/layer22_attention_weight_min": -31.18307113647461,
      "activations/layer23_attention_weight_max": 33.6102180480957,
      "activations/layer23_attention_weight_min": -27.837692260742188,
      "activations/layer2_attention_weight_max": 33.81730270385742,
      "activations/layer2_attention_weight_min": -34.081634521484375,
      "activations/layer3_attention_weight_max": 107.38894653320312,
      "activations/layer3_attention_weight_min": -108.68247985839844,
      "activations/layer4_attention_weight_max": 107.70369720458984,
      "activations/layer4_attention_weight_min": -112.58758544921875,
      "activations/layer5_attention_weight_max": 49.95207977294922,
      "activations/layer5_attention_weight_min": -61.30260467529297,
      "activations/layer6_attention_weight_max": 44.981258392333984,
      "activations/layer6_attention_weight_min": -47.82114791870117,
      "activations/layer7_attention_weight_max": 94.3364028930664,
      "activations/layer7_attention_weight_min": -102.20600891113281,
      "activations/layer8_attention_weight_max": 43.276824951171875,
      "activations/layer8_attention_weight_min": -46.594520568847656,
      "activations/layer9_attention_weight_max": 31.79701805114746,
      "activations/layer9_attention_weight_min": -35.619346618652344,
      "epoch": 22.59,
      "learning_rate": 4.3943181818181815e-06,
      "loss": 2.6866,
      "step": 388700
    },
    {
      "activations/layer0_attention_weight_max": 17.007831573486328,
      "activations/layer0_attention_weight_min": -14.073831558227539,
      "activations/layer10_attention_weight_max": 35.6923942565918,
      "activations/layer10_attention_weight_min": -36.560577392578125,
      "activations/layer11_attention_weight_max": 35.404335021972656,
      "activations/layer11_attention_weight_min": -35.36335754394531,
      "activations/layer12_attention_weight_max": 26.99833869934082,
      "activations/layer12_attention_weight_min": -27.5396785736084,
      "activations/layer13_attention_weight_max": 41.063507080078125,
      "activations/layer13_attention_weight_min": -37.470672607421875,
      "activations/layer14_attention_weight_max": 41.903663635253906,
      "activations/layer14_attention_weight_min": -39.79830551147461,
      "activations/layer15_attention_weight_max": 38.588348388671875,
      "activations/layer15_attention_weight_min": -37.74811553955078,
      "activations/layer16_attention_weight_max": 29.581132888793945,
      "activations/layer16_attention_weight_min": -29.292396545410156,
      "activations/layer17_attention_weight_max": 30.03915786743164,
      "activations/layer17_attention_weight_min": -26.834259033203125,
      "activations/layer18_attention_weight_max": 31.755428314208984,
      "activations/layer18_attention_weight_min": -27.454025268554688,
      "activations/layer19_attention_weight_max": 36.84828186035156,
      "activations/layer19_attention_weight_min": -33.45051193237305,
      "activations/layer1_attention_weight_max": 14.966251373291016,
      "activations/layer1_attention_weight_min": -15.911979675292969,
      "activations/layer20_attention_weight_max": 29.650300979614258,
      "activations/layer20_attention_weight_min": -25.845319747924805,
      "activations/layer21_attention_weight_max": 30.426794052124023,
      "activations/layer21_attention_weight_min": -24.380552291870117,
      "activations/layer22_attention_weight_max": 42.44743728637695,
      "activations/layer22_attention_weight_min": -34.026145935058594,
      "activations/layer23_attention_weight_max": 30.600759506225586,
      "activations/layer23_attention_weight_min": -26.041015625,
      "activations/layer2_attention_weight_max": 35.589813232421875,
      "activations/layer2_attention_weight_min": -33.166358947753906,
      "activations/layer3_attention_weight_max": 108.7154541015625,
      "activations/layer3_attention_weight_min": -107.82650756835938,
      "activations/layer4_attention_weight_max": 113.49639129638672,
      "activations/layer4_attention_weight_min": -112.07991790771484,
      "activations/layer5_attention_weight_max": 50.61225509643555,
      "activations/layer5_attention_weight_min": -63.91619110107422,
      "activations/layer6_attention_weight_max": 45.942710876464844,
      "activations/layer6_attention_weight_min": -48.793582916259766,
      "activations/layer7_attention_weight_max": 103.32575225830078,
      "activations/layer7_attention_weight_min": -101.31768035888672,
      "activations/layer8_attention_weight_max": 43.38478469848633,
      "activations/layer8_attention_weight_min": -45.64966583251953,
      "activations/layer9_attention_weight_max": 32.328792572021484,
      "activations/layer9_attention_weight_min": -34.69116973876953,
      "epoch": 22.59,
      "learning_rate": 4.375378787878787e-06,
      "loss": 2.7055,
      "step": 388750
    },
    {
      "activations/layer0_attention_weight_max": 16.88808822631836,
      "activations/layer0_attention_weight_min": -13.785216331481934,
      "activations/layer10_attention_weight_max": 40.0045051574707,
      "activations/layer10_attention_weight_min": -37.81834030151367,
      "activations/layer11_attention_weight_max": 40.365318298339844,
      "activations/layer11_attention_weight_min": -40.5493278503418,
      "activations/layer12_attention_weight_max": 27.327579498291016,
      "activations/layer12_attention_weight_min": -28.21849822998047,
      "activations/layer13_attention_weight_max": 38.97970199584961,
      "activations/layer13_attention_weight_min": -38.624481201171875,
      "activations/layer14_attention_weight_max": 43.9975700378418,
      "activations/layer14_attention_weight_min": -40.494564056396484,
      "activations/layer15_attention_weight_max": 41.58464813232422,
      "activations/layer15_attention_weight_min": -40.9313850402832,
      "activations/layer16_attention_weight_max": 34.22772216796875,
      "activations/layer16_attention_weight_min": -30.337068557739258,
      "activations/layer17_attention_weight_max": 32.137001037597656,
      "activations/layer17_attention_weight_min": -28.14837074279785,
      "activations/layer18_attention_weight_max": 35.808101654052734,
      "activations/layer18_attention_weight_min": -27.280208587646484,
      "activations/layer19_attention_weight_max": 36.51150894165039,
      "activations/layer19_attention_weight_min": -32.03158187866211,
      "activations/layer1_attention_weight_max": 17.109189987182617,
      "activations/layer1_attention_weight_min": -15.686284065246582,
      "activations/layer20_attention_weight_max": 29.66124725341797,
      "activations/layer20_attention_weight_min": -24.718955993652344,
      "activations/layer21_attention_weight_max": 29.135271072387695,
      "activations/layer21_attention_weight_min": -24.30136489868164,
      "activations/layer22_attention_weight_max": 42.06817626953125,
      "activations/layer22_attention_weight_min": -30.64326286315918,
      "activations/layer23_attention_weight_max": 31.140613555908203,
      "activations/layer23_attention_weight_min": -25.743289947509766,
      "activations/layer2_attention_weight_max": 38.59467315673828,
      "activations/layer2_attention_weight_min": -34.53932189941406,
      "activations/layer3_attention_weight_max": 108.66339874267578,
      "activations/layer3_attention_weight_min": -108.44403839111328,
      "activations/layer4_attention_weight_max": 111.67498016357422,
      "activations/layer4_attention_weight_min": -118.84244537353516,
      "activations/layer5_attention_weight_max": 52.958763122558594,
      "activations/layer5_attention_weight_min": -60.9524040222168,
      "activations/layer6_attention_weight_max": 48.58476257324219,
      "activations/layer6_attention_weight_min": -49.84186935424805,
      "activations/layer7_attention_weight_max": 110.46497344970703,
      "activations/layer7_attention_weight_min": -109.58059692382812,
      "activations/layer8_attention_weight_max": 47.16957473754883,
      "activations/layer8_attention_weight_min": -48.08586120605469,
      "activations/layer9_attention_weight_max": 34.17470169067383,
      "activations/layer9_attention_weight_min": -40.613983154296875,
      "epoch": 22.59,
      "learning_rate": 4.356439393939394e-06,
      "loss": 2.6731,
      "step": 388800
    },
    {
      "activations/layer0_attention_weight_max": 17.052217483520508,
      "activations/layer0_attention_weight_min": -14.422597885131836,
      "activations/layer10_attention_weight_max": 35.7614860534668,
      "activations/layer10_attention_weight_min": -35.87569808959961,
      "activations/layer11_attention_weight_max": 35.70956039428711,
      "activations/layer11_attention_weight_min": -39.354034423828125,
      "activations/layer12_attention_weight_max": 26.977516174316406,
      "activations/layer12_attention_weight_min": -26.952836990356445,
      "activations/layer13_attention_weight_max": 38.44523620605469,
      "activations/layer13_attention_weight_min": -36.188697814941406,
      "activations/layer14_attention_weight_max": 42.59518814086914,
      "activations/layer14_attention_weight_min": -36.69779968261719,
      "activations/layer15_attention_weight_max": 39.067596435546875,
      "activations/layer15_attention_weight_min": -37.86409378051758,
      "activations/layer16_attention_weight_max": 29.189449310302734,
      "activations/layer16_attention_weight_min": -28.142709732055664,
      "activations/layer17_attention_weight_max": 31.2893009185791,
      "activations/layer17_attention_weight_min": -27.049556732177734,
      "activations/layer18_attention_weight_max": 33.06911087036133,
      "activations/layer18_attention_weight_min": -26.8123836517334,
      "activations/layer19_attention_weight_max": 30.823423385620117,
      "activations/layer19_attention_weight_min": -30.980356216430664,
      "activations/layer1_attention_weight_max": 16.094831466674805,
      "activations/layer1_attention_weight_min": -15.593966484069824,
      "activations/layer20_attention_weight_max": 26.6053524017334,
      "activations/layer20_attention_weight_min": -24.757606506347656,
      "activations/layer21_attention_weight_max": 26.487730026245117,
      "activations/layer21_attention_weight_min": -24.06399917602539,
      "activations/layer22_attention_weight_max": 35.576927185058594,
      "activations/layer22_attention_weight_min": -29.61147117614746,
      "activations/layer23_attention_weight_max": 29.169601440429688,
      "activations/layer23_attention_weight_min": -24.32196044921875,
      "activations/layer2_attention_weight_max": 34.45824432373047,
      "activations/layer2_attention_weight_min": -34.15987014770508,
      "activations/layer3_attention_weight_max": 107.3814468383789,
      "activations/layer3_attention_weight_min": -106.10400390625,
      "activations/layer4_attention_weight_max": 110.02873992919922,
      "activations/layer4_attention_weight_min": -116.3558120727539,
      "activations/layer5_attention_weight_max": 49.66316223144531,
      "activations/layer5_attention_weight_min": -64.18341064453125,
      "activations/layer6_attention_weight_max": 46.0999641418457,
      "activations/layer6_attention_weight_min": -47.07387924194336,
      "activations/layer7_attention_weight_max": 94.04022216796875,
      "activations/layer7_attention_weight_min": -100.11257934570312,
      "activations/layer8_attention_weight_max": 42.636863708496094,
      "activations/layer8_attention_weight_min": -46.593135833740234,
      "activations/layer9_attention_weight_max": 32.77546310424805,
      "activations/layer9_attention_weight_min": -36.1620979309082,
      "epoch": 22.59,
      "learning_rate": 4.3374999999999996e-06,
      "loss": 2.6882,
      "step": 388850
    },
    {
      "activations/layer0_attention_weight_max": 17.043853759765625,
      "activations/layer0_attention_weight_min": -14.565091133117676,
      "activations/layer10_attention_weight_max": 38.44853210449219,
      "activations/layer10_attention_weight_min": -37.03657913208008,
      "activations/layer11_attention_weight_max": 36.934593200683594,
      "activations/layer11_attention_weight_min": -36.461883544921875,
      "activations/layer12_attention_weight_max": 31.57790756225586,
      "activations/layer12_attention_weight_min": -27.115694046020508,
      "activations/layer13_attention_weight_max": 43.54686737060547,
      "activations/layer13_attention_weight_min": -37.34962844848633,
      "activations/layer14_attention_weight_max": 47.423439025878906,
      "activations/layer14_attention_weight_min": -37.8036003112793,
      "activations/layer15_attention_weight_max": 47.47382736206055,
      "activations/layer15_attention_weight_min": -38.91625213623047,
      "activations/layer16_attention_weight_max": 37.92145538330078,
      "activations/layer16_attention_weight_min": -28.770082473754883,
      "activations/layer17_attention_weight_max": 32.938560485839844,
      "activations/layer17_attention_weight_min": -27.91886329650879,
      "activations/layer18_attention_weight_max": 34.212642669677734,
      "activations/layer18_attention_weight_min": -28.174104690551758,
      "activations/layer19_attention_weight_max": 37.98431396484375,
      "activations/layer19_attention_weight_min": -32.99382400512695,
      "activations/layer1_attention_weight_max": 15.906620979309082,
      "activations/layer1_attention_weight_min": -15.247313499450684,
      "activations/layer20_attention_weight_max": 32.404083251953125,
      "activations/layer20_attention_weight_min": -25.485265731811523,
      "activations/layer21_attention_weight_max": 31.208206176757812,
      "activations/layer21_attention_weight_min": -25.573928833007812,
      "activations/layer22_attention_weight_max": 46.81062316894531,
      "activations/layer22_attention_weight_min": -34.67670440673828,
      "activations/layer23_attention_weight_max": 39.04104995727539,
      "activations/layer23_attention_weight_min": -26.49898910522461,
      "activations/layer2_attention_weight_max": 35.198997497558594,
      "activations/layer2_attention_weight_min": -33.90369415283203,
      "activations/layer3_attention_weight_max": 108.29110717773438,
      "activations/layer3_attention_weight_min": -108.84371948242188,
      "activations/layer4_attention_weight_max": 112.38834381103516,
      "activations/layer4_attention_weight_min": -114.2441177368164,
      "activations/layer5_attention_weight_max": 48.308433532714844,
      "activations/layer5_attention_weight_min": -64.43550872802734,
      "activations/layer6_attention_weight_max": 43.89081954956055,
      "activations/layer6_attention_weight_min": -49.57636642456055,
      "activations/layer7_attention_weight_max": 102.09556579589844,
      "activations/layer7_attention_weight_min": -105.02520751953125,
      "activations/layer8_attention_weight_max": 43.78660202026367,
      "activations/layer8_attention_weight_min": -46.65559768676758,
      "activations/layer9_attention_weight_max": 33.69651412963867,
      "activations/layer9_attention_weight_min": -35.44832992553711,
      "epoch": 22.6,
      "learning_rate": 4.318560606060605e-06,
      "loss": 2.6966,
      "step": 388900
    },
    {
      "activations/layer0_attention_weight_max": 17.03920555114746,
      "activations/layer0_attention_weight_min": -14.183372497558594,
      "activations/layer10_attention_weight_max": 36.36473846435547,
      "activations/layer10_attention_weight_min": -35.335792541503906,
      "activations/layer11_attention_weight_max": 34.864315032958984,
      "activations/layer11_attention_weight_min": -34.722408294677734,
      "activations/layer12_attention_weight_max": 26.514644622802734,
      "activations/layer12_attention_weight_min": -28.760652542114258,
      "activations/layer13_attention_weight_max": 41.51802444458008,
      "activations/layer13_attention_weight_min": -41.183616638183594,
      "activations/layer14_attention_weight_max": 47.95428466796875,
      "activations/layer14_attention_weight_min": -46.49192428588867,
      "activations/layer15_attention_weight_max": 44.2975959777832,
      "activations/layer15_attention_weight_min": -42.249813079833984,
      "activations/layer16_attention_weight_max": 33.39808654785156,
      "activations/layer16_attention_weight_min": -30.120744705200195,
      "activations/layer17_attention_weight_max": 34.14463424682617,
      "activations/layer17_attention_weight_min": -27.78232192993164,
      "activations/layer18_attention_weight_max": 36.19710922241211,
      "activations/layer18_attention_weight_min": -29.901460647583008,
      "activations/layer19_attention_weight_max": 36.69670867919922,
      "activations/layer19_attention_weight_min": -34.34751510620117,
      "activations/layer1_attention_weight_max": 16.626802444458008,
      "activations/layer1_attention_weight_min": -15.77910041809082,
      "activations/layer20_attention_weight_max": 33.43845748901367,
      "activations/layer20_attention_weight_min": -26.84630012512207,
      "activations/layer21_attention_weight_max": 33.565574645996094,
      "activations/layer21_attention_weight_min": -26.55670738220215,
      "activations/layer22_attention_weight_max": 42.63690948486328,
      "activations/layer22_attention_weight_min": -32.78606033325195,
      "activations/layer23_attention_weight_max": 31.919666290283203,
      "activations/layer23_attention_weight_min": -27.590688705444336,
      "activations/layer2_attention_weight_max": 35.560829162597656,
      "activations/layer2_attention_weight_min": -33.99457931518555,
      "activations/layer3_attention_weight_max": 110.33074951171875,
      "activations/layer3_attention_weight_min": -110.34584045410156,
      "activations/layer4_attention_weight_max": 110.5176010131836,
      "activations/layer4_attention_weight_min": -114.74385833740234,
      "activations/layer5_attention_weight_max": 50.99288558959961,
      "activations/layer5_attention_weight_min": -60.700164794921875,
      "activations/layer6_attention_weight_max": 47.40645217895508,
      "activations/layer6_attention_weight_min": -47.78595733642578,
      "activations/layer7_attention_weight_max": 104.67341613769531,
      "activations/layer7_attention_weight_min": -103.5820541381836,
      "activations/layer8_attention_weight_max": 43.966182708740234,
      "activations/layer8_attention_weight_min": -45.500892639160156,
      "activations/layer9_attention_weight_max": 32.12854766845703,
      "activations/layer9_attention_weight_min": -36.47593307495117,
      "epoch": 22.6,
      "learning_rate": 4.299621212121212e-06,
      "loss": 2.709,
      "step": 388950
    },
    {
      "activations/layer0_attention_weight_max": 16.518587112426758,
      "activations/layer0_attention_weight_min": -13.600133895874023,
      "activations/layer10_attention_weight_max": 36.24268341064453,
      "activations/layer10_attention_weight_min": -38.035621643066406,
      "activations/layer11_attention_weight_max": 37.18232727050781,
      "activations/layer11_attention_weight_min": -38.302860260009766,
      "activations/layer12_attention_weight_max": 28.919099807739258,
      "activations/layer12_attention_weight_min": -28.24961280822754,
      "activations/layer13_attention_weight_max": 40.755393981933594,
      "activations/layer13_attention_weight_min": -38.65324783325195,
      "activations/layer14_attention_weight_max": 44.992645263671875,
      "activations/layer14_attention_weight_min": -43.0399169921875,
      "activations/layer15_attention_weight_max": 41.13922882080078,
      "activations/layer15_attention_weight_min": -40.71815490722656,
      "activations/layer16_attention_weight_max": 30.477420806884766,
      "activations/layer16_attention_weight_min": -30.143665313720703,
      "activations/layer17_attention_weight_max": 30.54029083251953,
      "activations/layer17_attention_weight_min": -27.61114501953125,
      "activations/layer18_attention_weight_max": 31.619144439697266,
      "activations/layer18_attention_weight_min": -26.17604637145996,
      "activations/layer19_attention_weight_max": 35.08260726928711,
      "activations/layer19_attention_weight_min": -31.347827911376953,
      "activations/layer1_attention_weight_max": 15.589406967163086,
      "activations/layer1_attention_weight_min": -15.543869018554688,
      "activations/layer20_attention_weight_max": 28.845714569091797,
      "activations/layer20_attention_weight_min": -25.438011169433594,
      "activations/layer21_attention_weight_max": 31.41680908203125,
      "activations/layer21_attention_weight_min": -23.754995346069336,
      "activations/layer22_attention_weight_max": 39.38661193847656,
      "activations/layer22_attention_weight_min": -30.197940826416016,
      "activations/layer23_attention_weight_max": 31.652692794799805,
      "activations/layer23_attention_weight_min": -25.211179733276367,
      "activations/layer2_attention_weight_max": 34.7012939453125,
      "activations/layer2_attention_weight_min": -34.16730499267578,
      "activations/layer3_attention_weight_max": 110.83316802978516,
      "activations/layer3_attention_weight_min": -109.1876220703125,
      "activations/layer4_attention_weight_max": 113.05587005615234,
      "activations/layer4_attention_weight_min": -115.56388092041016,
      "activations/layer5_attention_weight_max": 53.92694091796875,
      "activations/layer5_attention_weight_min": -66.17561340332031,
      "activations/layer6_attention_weight_max": 49.249473571777344,
      "activations/layer6_attention_weight_min": -51.863853454589844,
      "activations/layer7_attention_weight_max": 106.34107208251953,
      "activations/layer7_attention_weight_min": -110.4737777709961,
      "activations/layer8_attention_weight_max": 45.6483154296875,
      "activations/layer8_attention_weight_min": -47.69084548950195,
      "activations/layer9_attention_weight_max": 34.60942459106445,
      "activations/layer9_attention_weight_min": -36.5308952331543,
      "epoch": 22.6,
      "learning_rate": 4.280681818181818e-06,
      "loss": 2.7042,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4441,
      "eval_samples_per_second": 508.519,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4441,
      "eval_openwebtext_samples_per_second": 508.519,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9353,
      "eval_wikitext_samples_per_second": 235.619,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_lambada_loss": 2.31640625,
      "eval_lambada_ppl": 10.139171107507485,
      "eval_lambada_runtime": 9.5194,
      "eval_lambada_samples_per_second": 511.479,
      "step": 389000
    },
    {
      "activations/layer0_attention_weight_max": 17.457744598388672,
      "activations/layer0_attention_weight_min": -15.817292213439941,
      "activations/layer10_attention_weight_max": 40.80916976928711,
      "activations/layer10_attention_weight_min": -42.475830078125,
      "activations/layer11_attention_weight_max": 38.9677619934082,
      "activations/layer11_attention_weight_min": -38.64643859863281,
      "activations/layer12_attention_weight_max": 31.991382598876953,
      "activations/layer12_attention_weight_min": -30.814579010009766,
      "activations/layer13_attention_weight_max": 46.66239547729492,
      "activations/layer13_attention_weight_min": -40.19139862060547,
      "activations/layer14_attention_weight_max": 47.93626022338867,
      "activations/layer14_attention_weight_min": -42.72871017456055,
      "activations/layer15_attention_weight_max": 43.29380416870117,
      "activations/layer15_attention_weight_min": -39.00032043457031,
      "activations/layer16_attention_weight_max": 31.253677368164062,
      "activations/layer16_attention_weight_min": -31.133655548095703,
      "activations/layer17_attention_weight_max": 32.32301712036133,
      "activations/layer17_attention_weight_min": -28.10008430480957,
      "activations/layer18_attention_weight_max": 34.94279098510742,
      "activations/layer18_attention_weight_min": -30.678964614868164,
      "activations/layer19_attention_weight_max": 36.61787033081055,
      "activations/layer19_attention_weight_min": -36.2396125793457,
      "activations/layer1_attention_weight_max": 16.94926643371582,
      "activations/layer1_attention_weight_min": -15.756806373596191,
      "activations/layer20_attention_weight_max": 29.675010681152344,
      "activations/layer20_attention_weight_min": -27.163644790649414,
      "activations/layer21_attention_weight_max": 31.61458396911621,
      "activations/layer21_attention_weight_min": -26.922725677490234,
      "activations/layer22_attention_weight_max": 47.66115951538086,
      "activations/layer22_attention_weight_min": -31.911333084106445,
      "activations/layer23_attention_weight_max": 35.22871017456055,
      "activations/layer23_attention_weight_min": -25.93952751159668,
      "activations/layer2_attention_weight_max": 34.35979461669922,
      "activations/layer2_attention_weight_min": -34.18282699584961,
      "activations/layer3_attention_weight_max": 113.4599838256836,
      "activations/layer3_attention_weight_min": -115.28375244140625,
      "activations/layer4_attention_weight_max": 114.71089172363281,
      "activations/layer4_attention_weight_min": -121.57903289794922,
      "activations/layer5_attention_weight_max": 56.40174865722656,
      "activations/layer5_attention_weight_min": -66.45928955078125,
      "activations/layer6_attention_weight_max": 49.092525482177734,
      "activations/layer6_attention_weight_min": -53.0737419128418,
      "activations/layer7_attention_weight_max": 110.17022705078125,
      "activations/layer7_attention_weight_min": -114.41150665283203,
      "activations/layer8_attention_weight_max": 49.470550537109375,
      "activations/layer8_attention_weight_min": -53.874473571777344,
      "activations/layer9_attention_weight_max": 38.417911529541016,
      "activations/layer9_attention_weight_min": -43.20454025268555,
      "epoch": 22.61,
      "learning_rate": 4.261742424242424e-06,
      "loss": 2.6973,
      "step": 389050
    },
    {
      "activations/layer0_attention_weight_max": 16.874160766601562,
      "activations/layer0_attention_weight_min": -15.137492179870605,
      "activations/layer10_attention_weight_max": 37.51448059082031,
      "activations/layer10_attention_weight_min": -38.25999450683594,
      "activations/layer11_attention_weight_max": 37.71047592163086,
      "activations/layer11_attention_weight_min": -36.59248352050781,
      "activations/layer12_attention_weight_max": 30.978431701660156,
      "activations/layer12_attention_weight_min": -29.704551696777344,
      "activations/layer13_attention_weight_max": 46.916439056396484,
      "activations/layer13_attention_weight_min": -40.5560188293457,
      "activations/layer14_attention_weight_max": 46.817352294921875,
      "activations/layer14_attention_weight_min": -40.87131881713867,
      "activations/layer15_attention_weight_max": 43.263145446777344,
      "activations/layer15_attention_weight_min": -37.040771484375,
      "activations/layer16_attention_weight_max": 29.706846237182617,
      "activations/layer16_attention_weight_min": -28.871511459350586,
      "activations/layer17_attention_weight_max": 32.76858139038086,
      "activations/layer17_attention_weight_min": -27.549413681030273,
      "activations/layer18_attention_weight_max": 34.773780822753906,
      "activations/layer18_attention_weight_min": -27.693885803222656,
      "activations/layer19_attention_weight_max": 36.444454193115234,
      "activations/layer19_attention_weight_min": -34.558528900146484,
      "activations/layer1_attention_weight_max": 14.960845947265625,
      "activations/layer1_attention_weight_min": -15.06375789642334,
      "activations/layer20_attention_weight_max": 30.74709701538086,
      "activations/layer20_attention_weight_min": -25.77114486694336,
      "activations/layer21_attention_weight_max": 32.836830139160156,
      "activations/layer21_attention_weight_min": -24.75006675720215,
      "activations/layer22_attention_weight_max": 45.81890106201172,
      "activations/layer22_attention_weight_min": -32.63022994995117,
      "activations/layer23_attention_weight_max": 32.4044189453125,
      "activations/layer23_attention_weight_min": -24.261730194091797,
      "activations/layer2_attention_weight_max": 36.096641540527344,
      "activations/layer2_attention_weight_min": -32.90317153930664,
      "activations/layer3_attention_weight_max": 108.91541290283203,
      "activations/layer3_attention_weight_min": -109.17750549316406,
      "activations/layer4_attention_weight_max": 109.9391098022461,
      "activations/layer4_attention_weight_min": -114.45003509521484,
      "activations/layer5_attention_weight_max": 54.15454864501953,
      "activations/layer5_attention_weight_min": -63.777496337890625,
      "activations/layer6_attention_weight_max": 47.68214416503906,
      "activations/layer6_attention_weight_min": -52.087974548339844,
      "activations/layer7_attention_weight_max": 102.01844024658203,
      "activations/layer7_attention_weight_min": -103.77812957763672,
      "activations/layer8_attention_weight_max": 45.385215759277344,
      "activations/layer8_attention_weight_min": -48.363529205322266,
      "activations/layer9_attention_weight_max": 34.47380447387695,
      "activations/layer9_attention_weight_min": -36.570247650146484,
      "epoch": 22.61,
      "learning_rate": 4.24280303030303e-06,
      "loss": 2.6961,
      "step": 389100
    },
    {
      "activations/layer0_attention_weight_max": 16.08808708190918,
      "activations/layer0_attention_weight_min": -14.33113956451416,
      "activations/layer10_attention_weight_max": 39.80311965942383,
      "activations/layer10_attention_weight_min": -37.59931182861328,
      "activations/layer11_attention_weight_max": 38.287010192871094,
      "activations/layer11_attention_weight_min": -37.55963897705078,
      "activations/layer12_attention_weight_max": 33.96000289916992,
      "activations/layer12_attention_weight_min": -32.62589645385742,
      "activations/layer13_attention_weight_max": 52.72793960571289,
      "activations/layer13_attention_weight_min": -41.23411178588867,
      "activations/layer14_attention_weight_max": 55.38481140136719,
      "activations/layer14_attention_weight_min": -48.149837493896484,
      "activations/layer15_attention_weight_max": 52.40053939819336,
      "activations/layer15_attention_weight_min": -42.60039138793945,
      "activations/layer16_attention_weight_max": 38.669647216796875,
      "activations/layer16_attention_weight_min": -29.32613754272461,
      "activations/layer17_attention_weight_max": 35.61075210571289,
      "activations/layer17_attention_weight_min": -27.931434631347656,
      "activations/layer18_attention_weight_max": 36.450836181640625,
      "activations/layer18_attention_weight_min": -27.364633560180664,
      "activations/layer19_attention_weight_max": 39.63709259033203,
      "activations/layer19_attention_weight_min": -32.69662094116211,
      "activations/layer1_attention_weight_max": 16.377124786376953,
      "activations/layer1_attention_weight_min": -15.147500991821289,
      "activations/layer20_attention_weight_max": 34.02726364135742,
      "activations/layer20_attention_weight_min": -26.678062438964844,
      "activations/layer21_attention_weight_max": 32.04730224609375,
      "activations/layer21_attention_weight_min": -25.17705726623535,
      "activations/layer22_attention_weight_max": 47.07176208496094,
      "activations/layer22_attention_weight_min": -31.077281951904297,
      "activations/layer23_attention_weight_max": 36.45780563354492,
      "activations/layer23_attention_weight_min": -24.659835815429688,
      "activations/layer2_attention_weight_max": 34.17877197265625,
      "activations/layer2_attention_weight_min": -33.002559661865234,
      "activations/layer3_attention_weight_max": 110.70525360107422,
      "activations/layer3_attention_weight_min": -109.37451934814453,
      "activations/layer4_attention_weight_max": 115.4486312866211,
      "activations/layer4_attention_weight_min": -113.8268051147461,
      "activations/layer5_attention_weight_max": 53.481651306152344,
      "activations/layer5_attention_weight_min": -65.65419006347656,
      "activations/layer6_attention_weight_max": 51.365604400634766,
      "activations/layer6_attention_weight_min": -49.551822662353516,
      "activations/layer7_attention_weight_max": 108.64508819580078,
      "activations/layer7_attention_weight_min": -108.97185516357422,
      "activations/layer8_attention_weight_max": 48.266265869140625,
      "activations/layer8_attention_weight_min": -47.68053436279297,
      "activations/layer9_attention_weight_max": 35.86030197143555,
      "activations/layer9_attention_weight_min": -36.26774978637695,
      "epoch": 22.61,
      "learning_rate": 4.223863636363636e-06,
      "loss": 2.6951,
      "step": 389150
    },
    {
      "activations/layer0_attention_weight_max": 17.235565185546875,
      "activations/layer0_attention_weight_min": -14.071161270141602,
      "activations/layer10_attention_weight_max": 37.985076904296875,
      "activations/layer10_attention_weight_min": -35.850215911865234,
      "activations/layer11_attention_weight_max": 35.933074951171875,
      "activations/layer11_attention_weight_min": -35.05416488647461,
      "activations/layer12_attention_weight_max": 27.015140533447266,
      "activations/layer12_attention_weight_min": -26.70187759399414,
      "activations/layer13_attention_weight_max": 39.725852966308594,
      "activations/layer13_attention_weight_min": -36.245723724365234,
      "activations/layer14_attention_weight_max": 42.66404724121094,
      "activations/layer14_attention_weight_min": -39.27762222290039,
      "activations/layer15_attention_weight_max": 39.869964599609375,
      "activations/layer15_attention_weight_min": -37.449607849121094,
      "activations/layer16_attention_weight_max": 30.08717155456543,
      "activations/layer16_attention_weight_min": -28.758941650390625,
      "activations/layer17_attention_weight_max": 31.172929763793945,
      "activations/layer17_attention_weight_min": -26.289636611938477,
      "activations/layer18_attention_weight_max": 32.5516471862793,
      "activations/layer18_attention_weight_min": -26.062618255615234,
      "activations/layer19_attention_weight_max": 35.35994338989258,
      "activations/layer19_attention_weight_min": -33.0569953918457,
      "activations/layer1_attention_weight_max": 16.086246490478516,
      "activations/layer1_attention_weight_min": -15.057498931884766,
      "activations/layer20_attention_weight_max": 30.34113121032715,
      "activations/layer20_attention_weight_min": -25.853975296020508,
      "activations/layer21_attention_weight_max": 29.004486083984375,
      "activations/layer21_attention_weight_min": -26.102581024169922,
      "activations/layer22_attention_weight_max": 41.12616729736328,
      "activations/layer22_attention_weight_min": -32.78034210205078,
      "activations/layer23_attention_weight_max": 31.250518798828125,
      "activations/layer23_attention_weight_min": -26.79399871826172,
      "activations/layer2_attention_weight_max": 40.33343505859375,
      "activations/layer2_attention_weight_min": -34.39430236816406,
      "activations/layer3_attention_weight_max": 114.40435028076172,
      "activations/layer3_attention_weight_min": -112.52751159667969,
      "activations/layer4_attention_weight_max": 116.32948303222656,
      "activations/layer4_attention_weight_min": -115.5098648071289,
      "activations/layer5_attention_weight_max": 54.001678466796875,
      "activations/layer5_attention_weight_min": -65.86910247802734,
      "activations/layer6_attention_weight_max": 46.776756286621094,
      "activations/layer6_attention_weight_min": -50.12582778930664,
      "activations/layer7_attention_weight_max": 98.38231658935547,
      "activations/layer7_attention_weight_min": -101.63539123535156,
      "activations/layer8_attention_weight_max": 43.25605010986328,
      "activations/layer8_attention_weight_min": -44.93732452392578,
      "activations/layer9_attention_weight_max": 34.35381317138672,
      "activations/layer9_attention_weight_min": -36.03255844116211,
      "epoch": 22.61,
      "learning_rate": 4.204924242424242e-06,
      "loss": 2.6987,
      "step": 389200
    },
    {
      "activations/layer0_attention_weight_max": 16.31886100769043,
      "activations/layer0_attention_weight_min": -14.229426383972168,
      "activations/layer10_attention_weight_max": 35.193721771240234,
      "activations/layer10_attention_weight_min": -37.673194885253906,
      "activations/layer11_attention_weight_max": 34.7222900390625,
      "activations/layer11_attention_weight_min": -36.34191131591797,
      "activations/layer12_attention_weight_max": 29.268003463745117,
      "activations/layer12_attention_weight_min": -30.514347076416016,
      "activations/layer13_attention_weight_max": 42.108909606933594,
      "activations/layer13_attention_weight_min": -40.499671936035156,
      "activations/layer14_attention_weight_max": 45.716697692871094,
      "activations/layer14_attention_weight_min": -39.761993408203125,
      "activations/layer15_attention_weight_max": 42.07952880859375,
      "activations/layer15_attention_weight_min": -38.074798583984375,
      "activations/layer16_attention_weight_max": 31.91313934326172,
      "activations/layer16_attention_weight_min": -31.048128128051758,
      "activations/layer17_attention_weight_max": 33.00563430786133,
      "activations/layer17_attention_weight_min": -28.615713119506836,
      "activations/layer18_attention_weight_max": 36.65019607543945,
      "activations/layer18_attention_weight_min": -29.972848892211914,
      "activations/layer19_attention_weight_max": 34.45041275024414,
      "activations/layer19_attention_weight_min": -35.48688888549805,
      "activations/layer1_attention_weight_max": 15.277705192565918,
      "activations/layer1_attention_weight_min": -13.929725646972656,
      "activations/layer20_attention_weight_max": 29.862186431884766,
      "activations/layer20_attention_weight_min": -25.038358688354492,
      "activations/layer21_attention_weight_max": 29.46785545349121,
      "activations/layer21_attention_weight_min": -25.610368728637695,
      "activations/layer22_attention_weight_max": 42.63054275512695,
      "activations/layer22_attention_weight_min": -33.245750427246094,
      "activations/layer23_attention_weight_max": 31.877714157104492,
      "activations/layer23_attention_weight_min": -28.828908920288086,
      "activations/layer2_attention_weight_max": 35.267765045166016,
      "activations/layer2_attention_weight_min": -34.966182708740234,
      "activations/layer3_attention_weight_max": 107.88088989257812,
      "activations/layer3_attention_weight_min": -108.3816909790039,
      "activations/layer4_attention_weight_max": 112.20299530029297,
      "activations/layer4_attention_weight_min": -117.0987777709961,
      "activations/layer5_attention_weight_max": 51.3703727722168,
      "activations/layer5_attention_weight_min": -62.10124969482422,
      "activations/layer6_attention_weight_max": 45.69034194946289,
      "activations/layer6_attention_weight_min": -48.55203628540039,
      "activations/layer7_attention_weight_max": 99.4697494506836,
      "activations/layer7_attention_weight_min": -96.87124633789062,
      "activations/layer8_attention_weight_max": 43.3937873840332,
      "activations/layer8_attention_weight_min": -44.77214431762695,
      "activations/layer9_attention_weight_max": 32.948814392089844,
      "activations/layer9_attention_weight_min": -35.10561752319336,
      "epoch": 22.62,
      "learning_rate": 4.1863636363636356e-06,
      "loss": 2.702,
      "step": 389250
    },
    {
      "activations/layer0_attention_weight_max": 16.7758731842041,
      "activations/layer0_attention_weight_min": -14.404947280883789,
      "activations/layer10_attention_weight_max": 37.1212158203125,
      "activations/layer10_attention_weight_min": -36.542572021484375,
      "activations/layer11_attention_weight_max": 38.19182586669922,
      "activations/layer11_attention_weight_min": -36.9453125,
      "activations/layer12_attention_weight_max": 28.39604377746582,
      "activations/layer12_attention_weight_min": -27.03449058532715,
      "activations/layer13_attention_weight_max": 40.53087615966797,
      "activations/layer13_attention_weight_min": -37.21998596191406,
      "activations/layer14_attention_weight_max": 45.53040313720703,
      "activations/layer14_attention_weight_min": -38.226043701171875,
      "activations/layer15_attention_weight_max": 42.05278015136719,
      "activations/layer15_attention_weight_min": -38.63890838623047,
      "activations/layer16_attention_weight_max": 30.845386505126953,
      "activations/layer16_attention_weight_min": -28.740558624267578,
      "activations/layer17_attention_weight_max": 34.75935745239258,
      "activations/layer17_attention_weight_min": -28.534406661987305,
      "activations/layer18_attention_weight_max": 34.71276092529297,
      "activations/layer18_attention_weight_min": -28.426006317138672,
      "activations/layer19_attention_weight_max": 41.32768630981445,
      "activations/layer19_attention_weight_min": -33.15836715698242,
      "activations/layer1_attention_weight_max": 15.728158950805664,
      "activations/layer1_attention_weight_min": -14.930779457092285,
      "activations/layer20_attention_weight_max": 31.811494827270508,
      "activations/layer20_attention_weight_min": -28.4188289642334,
      "activations/layer21_attention_weight_max": 34.378082275390625,
      "activations/layer21_attention_weight_min": -26.692686080932617,
      "activations/layer22_attention_weight_max": 48.75605773925781,
      "activations/layer22_attention_weight_min": -33.62346267700195,
      "activations/layer23_attention_weight_max": 37.939964294433594,
      "activations/layer23_attention_weight_min": -26.677051544189453,
      "activations/layer2_attention_weight_max": 33.8984489440918,
      "activations/layer2_attention_weight_min": -34.5034294128418,
      "activations/layer3_attention_weight_max": 107.370849609375,
      "activations/layer3_attention_weight_min": -110.07596588134766,
      "activations/layer4_attention_weight_max": 112.43572998046875,
      "activations/layer4_attention_weight_min": -115.1122817993164,
      "activations/layer5_attention_weight_max": 51.44860076904297,
      "activations/layer5_attention_weight_min": -66.82388305664062,
      "activations/layer6_attention_weight_max": 46.916263580322266,
      "activations/layer6_attention_weight_min": -49.43843078613281,
      "activations/layer7_attention_weight_max": 100.9429931640625,
      "activations/layer7_attention_weight_min": -110.562255859375,
      "activations/layer8_attention_weight_max": 46.32390594482422,
      "activations/layer8_attention_weight_min": -49.28343963623047,
      "activations/layer9_attention_weight_max": 33.479549407958984,
      "activations/layer9_attention_weight_min": -37.74113845825195,
      "epoch": 22.62,
      "learning_rate": 4.167424242424242e-06,
      "loss": 2.6845,
      "step": 389300
    },
    {
      "activations/layer0_attention_weight_max": 17.138912200927734,
      "activations/layer0_attention_weight_min": -14.889211654663086,
      "activations/layer10_attention_weight_max": 33.521514892578125,
      "activations/layer10_attention_weight_min": -34.682064056396484,
      "activations/layer11_attention_weight_max": 32.865447998046875,
      "activations/layer11_attention_weight_min": -35.11030578613281,
      "activations/layer12_attention_weight_max": 24.982526779174805,
      "activations/layer12_attention_weight_min": -28.400354385375977,
      "activations/layer13_attention_weight_max": 37.29461669921875,
      "activations/layer13_attention_weight_min": -36.961151123046875,
      "activations/layer14_attention_weight_max": 40.676788330078125,
      "activations/layer14_attention_weight_min": -39.260398864746094,
      "activations/layer15_attention_weight_max": 37.9382209777832,
      "activations/layer15_attention_weight_min": -40.2476921081543,
      "activations/layer16_attention_weight_max": 31.647205352783203,
      "activations/layer16_attention_weight_min": -29.611927032470703,
      "activations/layer17_attention_weight_max": 30.095876693725586,
      "activations/layer17_attention_weight_min": -28.41544532775879,
      "activations/layer18_attention_weight_max": 35.181739807128906,
      "activations/layer18_attention_weight_min": -26.376911163330078,
      "activations/layer19_attention_weight_max": 39.57780075073242,
      "activations/layer19_attention_weight_min": -30.23115348815918,
      "activations/layer1_attention_weight_max": 16.481279373168945,
      "activations/layer1_attention_weight_min": -14.55683422088623,
      "activations/layer20_attention_weight_max": 33.53788757324219,
      "activations/layer20_attention_weight_min": -25.617570877075195,
      "activations/layer21_attention_weight_max": 34.06243133544922,
      "activations/layer21_attention_weight_min": -23.80826187133789,
      "activations/layer22_attention_weight_max": 42.79611587524414,
      "activations/layer22_attention_weight_min": -32.66423416137695,
      "activations/layer23_attention_weight_max": 31.94232940673828,
      "activations/layer23_attention_weight_min": -24.325002670288086,
      "activations/layer2_attention_weight_max": 34.75987243652344,
      "activations/layer2_attention_weight_min": -32.980716705322266,
      "activations/layer3_attention_weight_max": 105.03899383544922,
      "activations/layer3_attention_weight_min": -110.07478332519531,
      "activations/layer4_attention_weight_max": 108.43827056884766,
      "activations/layer4_attention_weight_min": -120.95275115966797,
      "activations/layer5_attention_weight_max": 52.958251953125,
      "activations/layer5_attention_weight_min": -63.899314880371094,
      "activations/layer6_attention_weight_max": 44.47635269165039,
      "activations/layer6_attention_weight_min": -48.841209411621094,
      "activations/layer7_attention_weight_max": 97.21085357666016,
      "activations/layer7_attention_weight_min": -99.89759826660156,
      "activations/layer8_attention_weight_max": 41.88153076171875,
      "activations/layer8_attention_weight_min": -46.31360626220703,
      "activations/layer9_attention_weight_max": 31.62214469909668,
      "activations/layer9_attention_weight_min": -35.67414474487305,
      "epoch": 22.62,
      "learning_rate": 4.148484848484848e-06,
      "loss": 2.6882,
      "step": 389350
    },
    {
      "activations/layer0_attention_weight_max": 16.389331817626953,
      "activations/layer0_attention_weight_min": -15.732393264770508,
      "activations/layer10_attention_weight_max": 41.32716369628906,
      "activations/layer10_attention_weight_min": -40.420249938964844,
      "activations/layer11_attention_weight_max": 39.111846923828125,
      "activations/layer11_attention_weight_min": -39.18011474609375,
      "activations/layer12_attention_weight_max": 30.880908966064453,
      "activations/layer12_attention_weight_min": -29.466123580932617,
      "activations/layer13_attention_weight_max": 43.430870056152344,
      "activations/layer13_attention_weight_min": -38.32470703125,
      "activations/layer14_attention_weight_max": 45.035743713378906,
      "activations/layer14_attention_weight_min": -40.581485748291016,
      "activations/layer15_attention_weight_max": 40.63201904296875,
      "activations/layer15_attention_weight_min": -38.005226135253906,
      "activations/layer16_attention_weight_max": 30.153690338134766,
      "activations/layer16_attention_weight_min": -29.728464126586914,
      "activations/layer17_attention_weight_max": 29.016408920288086,
      "activations/layer17_attention_weight_min": -26.984071731567383,
      "activations/layer18_attention_weight_max": 33.807769775390625,
      "activations/layer18_attention_weight_min": -27.047077178955078,
      "activations/layer19_attention_weight_max": 37.15205001831055,
      "activations/layer19_attention_weight_min": -31.759769439697266,
      "activations/layer1_attention_weight_max": 15.666646957397461,
      "activations/layer1_attention_weight_min": -15.156399726867676,
      "activations/layer20_attention_weight_max": 33.639556884765625,
      "activations/layer20_attention_weight_min": -24.745887756347656,
      "activations/layer21_attention_weight_max": 28.718795776367188,
      "activations/layer21_attention_weight_min": -24.125967025756836,
      "activations/layer22_attention_weight_max": 44.4100456237793,
      "activations/layer22_attention_weight_min": -33.11705780029297,
      "activations/layer23_attention_weight_max": 36.336570739746094,
      "activations/layer23_attention_weight_min": -26.81963348388672,
      "activations/layer2_attention_weight_max": 35.060062408447266,
      "activations/layer2_attention_weight_min": -33.614601135253906,
      "activations/layer3_attention_weight_max": 110.16854095458984,
      "activations/layer3_attention_weight_min": -113.57029724121094,
      "activations/layer4_attention_weight_max": 113.70915985107422,
      "activations/layer4_attention_weight_min": -117.55889892578125,
      "activations/layer5_attention_weight_max": 54.39113998413086,
      "activations/layer5_attention_weight_min": -65.01927185058594,
      "activations/layer6_attention_weight_max": 48.06913757324219,
      "activations/layer6_attention_weight_min": -54.145687103271484,
      "activations/layer7_attention_weight_max": 108.06563568115234,
      "activations/layer7_attention_weight_min": -115.48333740234375,
      "activations/layer8_attention_weight_max": 47.3916130065918,
      "activations/layer8_attention_weight_min": -50.3728141784668,
      "activations/layer9_attention_weight_max": 36.876399993896484,
      "activations/layer9_attention_weight_min": -38.72282028198242,
      "epoch": 22.63,
      "learning_rate": 4.1295454545454544e-06,
      "loss": 2.7153,
      "step": 389400
    },
    {
      "activations/layer0_attention_weight_max": 17.42043685913086,
      "activations/layer0_attention_weight_min": -13.999109268188477,
      "activations/layer10_attention_weight_max": 34.628089904785156,
      "activations/layer10_attention_weight_min": -36.040992736816406,
      "activations/layer11_attention_weight_max": 34.169010162353516,
      "activations/layer11_attention_weight_min": -39.359535217285156,
      "activations/layer12_attention_weight_max": 27.63011360168457,
      "activations/layer12_attention_weight_min": -27.93760108947754,
      "activations/layer13_attention_weight_max": 41.438899993896484,
      "activations/layer13_attention_weight_min": -38.29857635498047,
      "activations/layer14_attention_weight_max": 39.777748107910156,
      "activations/layer14_attention_weight_min": -40.85884094238281,
      "activations/layer15_attention_weight_max": 37.26838684082031,
      "activations/layer15_attention_weight_min": -38.62363815307617,
      "activations/layer16_attention_weight_max": 29.449657440185547,
      "activations/layer16_attention_weight_min": -28.252582550048828,
      "activations/layer17_attention_weight_max": 30.058820724487305,
      "activations/layer17_attention_weight_min": -27.304718017578125,
      "activations/layer18_attention_weight_max": 31.426761627197266,
      "activations/layer18_attention_weight_min": -28.2182674407959,
      "activations/layer19_attention_weight_max": 32.56010055541992,
      "activations/layer19_attention_weight_min": -31.1708927154541,
      "activations/layer1_attention_weight_max": 15.458964347839355,
      "activations/layer1_attention_weight_min": -15.458419799804688,
      "activations/layer20_attention_weight_max": 29.906261444091797,
      "activations/layer20_attention_weight_min": -24.561721801757812,
      "activations/layer21_attention_weight_max": 29.000362396240234,
      "activations/layer21_attention_weight_min": -24.888578414916992,
      "activations/layer22_attention_weight_max": 41.9991340637207,
      "activations/layer22_attention_weight_min": -31.705280303955078,
      "activations/layer23_attention_weight_max": 30.677581787109375,
      "activations/layer23_attention_weight_min": -25.617294311523438,
      "activations/layer2_attention_weight_max": 35.131919860839844,
      "activations/layer2_attention_weight_min": -33.147216796875,
      "activations/layer3_attention_weight_max": 107.99808502197266,
      "activations/layer3_attention_weight_min": -110.39762878417969,
      "activations/layer4_attention_weight_max": 112.17671966552734,
      "activations/layer4_attention_weight_min": -118.1849136352539,
      "activations/layer5_attention_weight_max": 51.26219177246094,
      "activations/layer5_attention_weight_min": -67.26411437988281,
      "activations/layer6_attention_weight_max": 46.08955001831055,
      "activations/layer6_attention_weight_min": -49.92140197753906,
      "activations/layer7_attention_weight_max": 96.31594848632812,
      "activations/layer7_attention_weight_min": -102.54155731201172,
      "activations/layer8_attention_weight_max": 43.5338020324707,
      "activations/layer8_attention_weight_min": -47.11800003051758,
      "activations/layer9_attention_weight_max": 34.17576599121094,
      "activations/layer9_attention_weight_min": -35.12369155883789,
      "epoch": 22.63,
      "learning_rate": 4.11060606060606e-06,
      "loss": 2.7154,
      "step": 389450
    },
    {
      "activations/layer0_attention_weight_max": 16.56425666809082,
      "activations/layer0_attention_weight_min": -13.920442581176758,
      "activations/layer10_attention_weight_max": 38.812843322753906,
      "activations/layer10_attention_weight_min": -37.27187728881836,
      "activations/layer11_attention_weight_max": 36.568904876708984,
      "activations/layer11_attention_weight_min": -36.54268264770508,
      "activations/layer12_attention_weight_max": 27.0378475189209,
      "activations/layer12_attention_weight_min": -28.250598907470703,
      "activations/layer13_attention_weight_max": 41.98493957519531,
      "activations/layer13_attention_weight_min": -37.11579132080078,
      "activations/layer14_attention_weight_max": 44.32857894897461,
      "activations/layer14_attention_weight_min": -39.35544967651367,
      "activations/layer15_attention_weight_max": 41.060543060302734,
      "activations/layer15_attention_weight_min": -37.33797836303711,
      "activations/layer16_attention_weight_max": 31.13216209411621,
      "activations/layer16_attention_weight_min": -29.99931526184082,
      "activations/layer17_attention_weight_max": 29.713544845581055,
      "activations/layer17_attention_weight_min": -26.11103057861328,
      "activations/layer18_attention_weight_max": 36.89225769042969,
      "activations/layer18_attention_weight_min": -28.33416175842285,
      "activations/layer19_attention_weight_max": 34.5081672668457,
      "activations/layer19_attention_weight_min": -32.142704010009766,
      "activations/layer1_attention_weight_max": 15.381704330444336,
      "activations/layer1_attention_weight_min": -14.998549461364746,
      "activations/layer20_attention_weight_max": 28.170337677001953,
      "activations/layer20_attention_weight_min": -28.098098754882812,
      "activations/layer21_attention_weight_max": 28.596988677978516,
      "activations/layer21_attention_weight_min": -25.765317916870117,
      "activations/layer22_attention_weight_max": 44.06739044189453,
      "activations/layer22_attention_weight_min": -32.67822265625,
      "activations/layer23_attention_weight_max": 30.159568786621094,
      "activations/layer23_attention_weight_min": -27.354598999023438,
      "activations/layer2_attention_weight_max": 34.768863677978516,
      "activations/layer2_attention_weight_min": -34.122100830078125,
      "activations/layer3_attention_weight_max": 111.7197036743164,
      "activations/layer3_attention_weight_min": -106.03242492675781,
      "activations/layer4_attention_weight_max": 116.75946044921875,
      "activations/layer4_attention_weight_min": -115.52718353271484,
      "activations/layer5_attention_weight_max": 52.46751022338867,
      "activations/layer5_attention_weight_min": -61.67026901245117,
      "activations/layer6_attention_weight_max": 48.22581100463867,
      "activations/layer6_attention_weight_min": -51.73243713378906,
      "activations/layer7_attention_weight_max": 101.43634796142578,
      "activations/layer7_attention_weight_min": -104.86861419677734,
      "activations/layer8_attention_weight_max": 44.634342193603516,
      "activations/layer8_attention_weight_min": -48.047271728515625,
      "activations/layer9_attention_weight_max": 34.172115325927734,
      "activations/layer9_attention_weight_min": -38.387001037597656,
      "epoch": 22.63,
      "learning_rate": 4.091666666666667e-06,
      "loss": 2.7122,
      "step": 389500
    },
    {
      "activations/layer0_attention_weight_max": 17.398685455322266,
      "activations/layer0_attention_weight_min": -14.3192138671875,
      "activations/layer10_attention_weight_max": 36.00595474243164,
      "activations/layer10_attention_weight_min": -36.156829833984375,
      "activations/layer11_attention_weight_max": 35.891868591308594,
      "activations/layer11_attention_weight_min": -37.29883575439453,
      "activations/layer12_attention_weight_max": 28.281084060668945,
      "activations/layer12_attention_weight_min": -27.153064727783203,
      "activations/layer13_attention_weight_max": 42.14411926269531,
      "activations/layer13_attention_weight_min": -36.43452453613281,
      "activations/layer14_attention_weight_max": 46.5356330871582,
      "activations/layer14_attention_weight_min": -38.192726135253906,
      "activations/layer15_attention_weight_max": 44.70915603637695,
      "activations/layer15_attention_weight_min": -37.517608642578125,
      "activations/layer16_attention_weight_max": 31.99816131591797,
      "activations/layer16_attention_weight_min": -29.563631057739258,
      "activations/layer17_attention_weight_max": 35.85353469848633,
      "activations/layer17_attention_weight_min": -26.31423568725586,
      "activations/layer18_attention_weight_max": 36.92890930175781,
      "activations/layer18_attention_weight_min": -26.469385147094727,
      "activations/layer19_attention_weight_max": 39.06510925292969,
      "activations/layer19_attention_weight_min": -32.49494171142578,
      "activations/layer1_attention_weight_max": 15.812326431274414,
      "activations/layer1_attention_weight_min": -14.143980026245117,
      "activations/layer20_attention_weight_max": 33.41109085083008,
      "activations/layer20_attention_weight_min": -25.293437957763672,
      "activations/layer21_attention_weight_max": 32.6631965637207,
      "activations/layer21_attention_weight_min": -24.4482364654541,
      "activations/layer22_attention_weight_max": 52.17184066772461,
      "activations/layer22_attention_weight_min": -31.434078216552734,
      "activations/layer23_attention_weight_max": 39.16947555541992,
      "activations/layer23_attention_weight_min": -24.788122177124023,
      "activations/layer2_attention_weight_max": 35.28567123413086,
      "activations/layer2_attention_weight_min": -33.59682083129883,
      "activations/layer3_attention_weight_max": 110.62557220458984,
      "activations/layer3_attention_weight_min": -113.0595932006836,
      "activations/layer4_attention_weight_max": 110.99950408935547,
      "activations/layer4_attention_weight_min": -113.17436981201172,
      "activations/layer5_attention_weight_max": 49.97285079956055,
      "activations/layer5_attention_weight_min": -63.089111328125,
      "activations/layer6_attention_weight_max": 45.85545349121094,
      "activations/layer6_attention_weight_min": -47.82510757446289,
      "activations/layer7_attention_weight_max": 96.09930419921875,
      "activations/layer7_attention_weight_min": -98.6704330444336,
      "activations/layer8_attention_weight_max": 41.672096252441406,
      "activations/layer8_attention_weight_min": -45.226226806640625,
      "activations/layer9_attention_weight_max": 31.562658309936523,
      "activations/layer9_attention_weight_min": -34.44642639160156,
      "epoch": 22.64,
      "learning_rate": 4.0727272727272725e-06,
      "loss": 2.6988,
      "step": 389550
    },
    {
      "activations/layer0_attention_weight_max": 17.114368438720703,
      "activations/layer0_attention_weight_min": -13.775921821594238,
      "activations/layer10_attention_weight_max": 40.340843200683594,
      "activations/layer10_attention_weight_min": -38.053863525390625,
      "activations/layer11_attention_weight_max": 40.206581115722656,
      "activations/layer11_attention_weight_min": -39.56131362915039,
      "activations/layer12_attention_weight_max": 27.06474494934082,
      "activations/layer12_attention_weight_min": -27.216455459594727,
      "activations/layer13_attention_weight_max": 38.467994689941406,
      "activations/layer13_attention_weight_min": -35.85331344604492,
      "activations/layer14_attention_weight_max": 40.61532211303711,
      "activations/layer14_attention_weight_min": -36.12141799926758,
      "activations/layer15_attention_weight_max": 38.936370849609375,
      "activations/layer15_attention_weight_min": -35.46773147583008,
      "activations/layer16_attention_weight_max": 28.42792510986328,
      "activations/layer16_attention_weight_min": -27.42485237121582,
      "activations/layer17_attention_weight_max": 28.875728607177734,
      "activations/layer17_attention_weight_min": -26.611635208129883,
      "activations/layer18_attention_weight_max": 36.177589416503906,
      "activations/layer18_attention_weight_min": -28.24605941772461,
      "activations/layer19_attention_weight_max": 33.35845947265625,
      "activations/layer19_attention_weight_min": -33.12084197998047,
      "activations/layer1_attention_weight_max": 16.104721069335938,
      "activations/layer1_attention_weight_min": -15.153257369995117,
      "activations/layer20_attention_weight_max": 30.81606101989746,
      "activations/layer20_attention_weight_min": -28.418628692626953,
      "activations/layer21_attention_weight_max": 29.348669052124023,
      "activations/layer21_attention_weight_min": -26.646013259887695,
      "activations/layer22_attention_weight_max": 40.430328369140625,
      "activations/layer22_attention_weight_min": -32.456111907958984,
      "activations/layer23_attention_weight_max": 31.818283081054688,
      "activations/layer23_attention_weight_min": -26.359718322753906,
      "activations/layer2_attention_weight_max": 34.685848236083984,
      "activations/layer2_attention_weight_min": -35.02803039550781,
      "activations/layer3_attention_weight_max": 108.14239501953125,
      "activations/layer3_attention_weight_min": -110.75100708007812,
      "activations/layer4_attention_weight_max": 110.7480239868164,
      "activations/layer4_attention_weight_min": -116.7229232788086,
      "activations/layer5_attention_weight_max": 52.65366744995117,
      "activations/layer5_attention_weight_min": -63.493587493896484,
      "activations/layer6_attention_weight_max": 47.60212707519531,
      "activations/layer6_attention_weight_min": -47.8688850402832,
      "activations/layer7_attention_weight_max": 100.77114868164062,
      "activations/layer7_attention_weight_min": -112.0596694946289,
      "activations/layer8_attention_weight_max": 46.24510955810547,
      "activations/layer8_attention_weight_min": -49.43718338012695,
      "activations/layer9_attention_weight_max": 36.978702545166016,
      "activations/layer9_attention_weight_min": -38.83855056762695,
      "epoch": 22.64,
      "learning_rate": 4.053787878787878e-06,
      "loss": 2.7096,
      "step": 389600
    },
    {
      "activations/layer0_attention_weight_max": 16.616004943847656,
      "activations/layer0_attention_weight_min": -15.16158390045166,
      "activations/layer10_attention_weight_max": 40.686309814453125,
      "activations/layer10_attention_weight_min": -41.57592010498047,
      "activations/layer11_attention_weight_max": 40.64813995361328,
      "activations/layer11_attention_weight_min": -38.50623321533203,
      "activations/layer12_attention_weight_max": 28.898582458496094,
      "activations/layer12_attention_weight_min": -29.48493194580078,
      "activations/layer13_attention_weight_max": 40.55809783935547,
      "activations/layer13_attention_weight_min": -38.084251403808594,
      "activations/layer14_attention_weight_max": 44.09382629394531,
      "activations/layer14_attention_weight_min": -41.965538024902344,
      "activations/layer15_attention_weight_max": 41.923439025878906,
      "activations/layer15_attention_weight_min": -42.10661315917969,
      "activations/layer16_attention_weight_max": 31.796110153198242,
      "activations/layer16_attention_weight_min": -28.897069931030273,
      "activations/layer17_attention_weight_max": 31.207345962524414,
      "activations/layer17_attention_weight_min": -27.220495223999023,
      "activations/layer18_attention_weight_max": 32.17581558227539,
      "activations/layer18_attention_weight_min": -26.79301643371582,
      "activations/layer19_attention_weight_max": 34.368526458740234,
      "activations/layer19_attention_weight_min": -32.516807556152344,
      "activations/layer1_attention_weight_max": 16.095855712890625,
      "activations/layer1_attention_weight_min": -15.356014251708984,
      "activations/layer20_attention_weight_max": 28.968162536621094,
      "activations/layer20_attention_weight_min": -24.780000686645508,
      "activations/layer21_attention_weight_max": 27.707855224609375,
      "activations/layer21_attention_weight_min": -25.359222412109375,
      "activations/layer22_attention_weight_max": 42.56144332885742,
      "activations/layer22_attention_weight_min": -32.98108673095703,
      "activations/layer23_attention_weight_max": 31.446157455444336,
      "activations/layer23_attention_weight_min": -27.488218307495117,
      "activations/layer2_attention_weight_max": 35.44496154785156,
      "activations/layer2_attention_weight_min": -36.22614288330078,
      "activations/layer3_attention_weight_max": 108.84809112548828,
      "activations/layer3_attention_weight_min": -112.74286651611328,
      "activations/layer4_attention_weight_max": 113.64253234863281,
      "activations/layer4_attention_weight_min": -114.57563781738281,
      "activations/layer5_attention_weight_max": 52.7110710144043,
      "activations/layer5_attention_weight_min": -61.32160949707031,
      "activations/layer6_attention_weight_max": 47.35041427612305,
      "activations/layer6_attention_weight_min": -49.63032913208008,
      "activations/layer7_attention_weight_max": 112.65620422363281,
      "activations/layer7_attention_weight_min": -106.59688568115234,
      "activations/layer8_attention_weight_max": 50.79288864135742,
      "activations/layer8_attention_weight_min": -52.85722732543945,
      "activations/layer9_attention_weight_max": 37.90444564819336,
      "activations/layer9_attention_weight_min": -40.270416259765625,
      "epoch": 22.64,
      "learning_rate": 4.034848484848485e-06,
      "loss": 2.6943,
      "step": 389650
    },
    {
      "activations/layer0_attention_weight_max": 16.312952041625977,
      "activations/layer0_attention_weight_min": -14.280535697937012,
      "activations/layer10_attention_weight_max": 36.586334228515625,
      "activations/layer10_attention_weight_min": -37.34733963012695,
      "activations/layer11_attention_weight_max": 37.47960662841797,
      "activations/layer11_attention_weight_min": -37.84837341308594,
      "activations/layer12_attention_weight_max": 30.412578582763672,
      "activations/layer12_attention_weight_min": -28.107255935668945,
      "activations/layer13_attention_weight_max": 43.61619186401367,
      "activations/layer13_attention_weight_min": -38.9886360168457,
      "activations/layer14_attention_weight_max": 48.63739776611328,
      "activations/layer14_attention_weight_min": -44.051605224609375,
      "activations/layer15_attention_weight_max": 44.85829162597656,
      "activations/layer15_attention_weight_min": -38.98560333251953,
      "activations/layer16_attention_weight_max": 33.88251495361328,
      "activations/layer16_attention_weight_min": -29.346609115600586,
      "activations/layer17_attention_weight_max": 36.34201431274414,
      "activations/layer17_attention_weight_min": -27.897743225097656,
      "activations/layer18_attention_weight_max": 35.20331954956055,
      "activations/layer18_attention_weight_min": -27.19957160949707,
      "activations/layer19_attention_weight_max": 37.967830657958984,
      "activations/layer19_attention_weight_min": -32.5229377746582,
      "activations/layer1_attention_weight_max": 15.488146781921387,
      "activations/layer1_attention_weight_min": -14.7406005859375,
      "activations/layer20_attention_weight_max": 32.609100341796875,
      "activations/layer20_attention_weight_min": -25.078744888305664,
      "activations/layer21_attention_weight_max": 32.71228790283203,
      "activations/layer21_attention_weight_min": -24.281522750854492,
      "activations/layer22_attention_weight_max": 43.170494079589844,
      "activations/layer22_attention_weight_min": -33.77507400512695,
      "activations/layer23_attention_weight_max": 37.09986114501953,
      "activations/layer23_attention_weight_min": -25.985286712646484,
      "activations/layer2_attention_weight_max": 34.720191955566406,
      "activations/layer2_attention_weight_min": -33.60658645629883,
      "activations/layer3_attention_weight_max": 108.74898529052734,
      "activations/layer3_attention_weight_min": -108.37318420410156,
      "activations/layer4_attention_weight_max": 114.77349090576172,
      "activations/layer4_attention_weight_min": -114.90843963623047,
      "activations/layer5_attention_weight_max": 50.88787841796875,
      "activations/layer5_attention_weight_min": -64.40229797363281,
      "activations/layer6_attention_weight_max": 46.65526580810547,
      "activations/layer6_attention_weight_min": -48.853580474853516,
      "activations/layer7_attention_weight_max": 101.33717346191406,
      "activations/layer7_attention_weight_min": -105.65818786621094,
      "activations/layer8_attention_weight_max": 44.785335540771484,
      "activations/layer8_attention_weight_min": -48.42729949951172,
      "activations/layer9_attention_weight_max": 34.381996154785156,
      "activations/layer9_attention_weight_min": -36.82025909423828,
      "epoch": 22.64,
      "learning_rate": 4.0159090909090905e-06,
      "loss": 2.7008,
      "step": 389700
    },
    {
      "activations/layer0_attention_weight_max": 16.284940719604492,
      "activations/layer0_attention_weight_min": -13.851509094238281,
      "activations/layer10_attention_weight_max": 34.38336181640625,
      "activations/layer10_attention_weight_min": -37.08651351928711,
      "activations/layer11_attention_weight_max": 35.4471549987793,
      "activations/layer11_attention_weight_min": -36.8118896484375,
      "activations/layer12_attention_weight_max": 29.489845275878906,
      "activations/layer12_attention_weight_min": -28.34128189086914,
      "activations/layer13_attention_weight_max": 40.10017395019531,
      "activations/layer13_attention_weight_min": -38.784034729003906,
      "activations/layer14_attention_weight_max": 46.49029541015625,
      "activations/layer14_attention_weight_min": -40.05485534667969,
      "activations/layer15_attention_weight_max": 40.67845916748047,
      "activations/layer15_attention_weight_min": -38.770450592041016,
      "activations/layer16_attention_weight_max": 31.54694366455078,
      "activations/layer16_attention_weight_min": -29.466196060180664,
      "activations/layer17_attention_weight_max": 31.112958908081055,
      "activations/layer17_attention_weight_min": -25.643644332885742,
      "activations/layer18_attention_weight_max": 34.40008544921875,
      "activations/layer18_attention_weight_min": -26.052066802978516,
      "activations/layer19_attention_weight_max": 36.73481369018555,
      "activations/layer19_attention_weight_min": -31.46558952331543,
      "activations/layer1_attention_weight_max": 15.88244342803955,
      "activations/layer1_attention_weight_min": -14.322866439819336,
      "activations/layer20_attention_weight_max": 31.294687271118164,
      "activations/layer20_attention_weight_min": -24.694385528564453,
      "activations/layer21_attention_weight_max": 31.328792572021484,
      "activations/layer21_attention_weight_min": -24.67308235168457,
      "activations/layer22_attention_weight_max": 48.59675598144531,
      "activations/layer22_attention_weight_min": -32.67366409301758,
      "activations/layer23_attention_weight_max": 31.872772216796875,
      "activations/layer23_attention_weight_min": -27.80780029296875,
      "activations/layer2_attention_weight_max": 34.286651611328125,
      "activations/layer2_attention_weight_min": -35.29366683959961,
      "activations/layer3_attention_weight_max": 107.41419982910156,
      "activations/layer3_attention_weight_min": -114.37773895263672,
      "activations/layer4_attention_weight_max": 114.3985366821289,
      "activations/layer4_attention_weight_min": -115.0423355102539,
      "activations/layer5_attention_weight_max": 51.390071868896484,
      "activations/layer5_attention_weight_min": -65.26667785644531,
      "activations/layer6_attention_weight_max": 47.84476852416992,
      "activations/layer6_attention_weight_min": -50.118553161621094,
      "activations/layer7_attention_weight_max": 96.49197387695312,
      "activations/layer7_attention_weight_min": -109.35662841796875,
      "activations/layer8_attention_weight_max": 43.581573486328125,
      "activations/layer8_attention_weight_min": -49.355716705322266,
      "activations/layer9_attention_weight_max": 33.24162673950195,
      "activations/layer9_attention_weight_min": -35.13365936279297,
      "epoch": 22.65,
      "learning_rate": 3.996969696969697e-06,
      "loss": 2.7174,
      "step": 389750
    },
    {
      "activations/layer0_attention_weight_max": 17.164405822753906,
      "activations/layer0_attention_weight_min": -14.377715110778809,
      "activations/layer10_attention_weight_max": 35.44321823120117,
      "activations/layer10_attention_weight_min": -36.4219856262207,
      "activations/layer11_attention_weight_max": 35.30351257324219,
      "activations/layer11_attention_weight_min": -35.31964111328125,
      "activations/layer12_attention_weight_max": 26.35441780090332,
      "activations/layer12_attention_weight_min": -27.37155532836914,
      "activations/layer13_attention_weight_max": 38.9597053527832,
      "activations/layer13_attention_weight_min": -36.705841064453125,
      "activations/layer14_attention_weight_max": 40.60789489746094,
      "activations/layer14_attention_weight_min": -37.82056427001953,
      "activations/layer15_attention_weight_max": 37.36381530761719,
      "activations/layer15_attention_weight_min": -36.1654167175293,
      "activations/layer16_attention_weight_max": 27.84429168701172,
      "activations/layer16_attention_weight_min": -27.827301025390625,
      "activations/layer17_attention_weight_max": 31.09588050842285,
      "activations/layer17_attention_weight_min": -26.231204986572266,
      "activations/layer18_attention_weight_max": 31.01729965209961,
      "activations/layer18_attention_weight_min": -29.323087692260742,
      "activations/layer19_attention_weight_max": 35.50809097290039,
      "activations/layer19_attention_weight_min": -36.78114318847656,
      "activations/layer1_attention_weight_max": 15.977441787719727,
      "activations/layer1_attention_weight_min": -14.611465454101562,
      "activations/layer20_attention_weight_max": 32.294254302978516,
      "activations/layer20_attention_weight_min": -29.757699966430664,
      "activations/layer21_attention_weight_max": 30.56998634338379,
      "activations/layer21_attention_weight_min": -27.70157241821289,
      "activations/layer22_attention_weight_max": 43.51973342895508,
      "activations/layer22_attention_weight_min": -33.867332458496094,
      "activations/layer23_attention_weight_max": 28.69046401977539,
      "activations/layer23_attention_weight_min": -26.440412521362305,
      "activations/layer2_attention_weight_max": 34.74981689453125,
      "activations/layer2_attention_weight_min": -33.80100631713867,
      "activations/layer3_attention_weight_max": 107.60174560546875,
      "activations/layer3_attention_weight_min": -109.8299789428711,
      "activations/layer4_attention_weight_max": 112.27725982666016,
      "activations/layer4_attention_weight_min": -117.9536361694336,
      "activations/layer5_attention_weight_max": 51.74530792236328,
      "activations/layer5_attention_weight_min": -63.312591552734375,
      "activations/layer6_attention_weight_max": 44.701744079589844,
      "activations/layer6_attention_weight_min": -50.63196563720703,
      "activations/layer7_attention_weight_max": 95.31079864501953,
      "activations/layer7_attention_weight_min": -103.96639251708984,
      "activations/layer8_attention_weight_max": 43.470802307128906,
      "activations/layer8_attention_weight_min": -46.679508209228516,
      "activations/layer9_attention_weight_max": 35.4265251159668,
      "activations/layer9_attention_weight_min": -35.617244720458984,
      "epoch": 22.65,
      "learning_rate": 3.978030303030303e-06,
      "loss": 2.695,
      "step": 389800
    },
    {
      "activations/layer0_attention_weight_max": 16.96271514892578,
      "activations/layer0_attention_weight_min": -15.906661987304688,
      "activations/layer10_attention_weight_max": 33.92873001098633,
      "activations/layer10_attention_weight_min": -34.86357498168945,
      "activations/layer11_attention_weight_max": 33.80585861206055,
      "activations/layer11_attention_weight_min": -34.95178985595703,
      "activations/layer12_attention_weight_max": 26.38927459716797,
      "activations/layer12_attention_weight_min": -26.7624454498291,
      "activations/layer13_attention_weight_max": 37.99068069458008,
      "activations/layer13_attention_weight_min": -34.859893798828125,
      "activations/layer14_attention_weight_max": 40.776859283447266,
      "activations/layer14_attention_weight_min": -37.09757614135742,
      "activations/layer15_attention_weight_max": 37.981712341308594,
      "activations/layer15_attention_weight_min": -36.77972412109375,
      "activations/layer16_attention_weight_max": 29.091552734375,
      "activations/layer16_attention_weight_min": -29.1671142578125,
      "activations/layer17_attention_weight_max": 30.957868576049805,
      "activations/layer17_attention_weight_min": -29.471052169799805,
      "activations/layer18_attention_weight_max": 38.088661193847656,
      "activations/layer18_attention_weight_min": -29.053354263305664,
      "activations/layer19_attention_weight_max": 38.06993865966797,
      "activations/layer19_attention_weight_min": -34.57485580444336,
      "activations/layer1_attention_weight_max": 15.457728385925293,
      "activations/layer1_attention_weight_min": -14.786828994750977,
      "activations/layer20_attention_weight_max": 34.353355407714844,
      "activations/layer20_attention_weight_min": -28.724803924560547,
      "activations/layer21_attention_weight_max": 33.5733642578125,
      "activations/layer21_attention_weight_min": -27.536510467529297,
      "activations/layer22_attention_weight_max": 42.44449234008789,
      "activations/layer22_attention_weight_min": -33.74636459350586,
      "activations/layer23_attention_weight_max": 31.656517028808594,
      "activations/layer23_attention_weight_min": -26.218242645263672,
      "activations/layer2_attention_weight_max": 33.91301345825195,
      "activations/layer2_attention_weight_min": -33.98295974731445,
      "activations/layer3_attention_weight_max": 108.94670867919922,
      "activations/layer3_attention_weight_min": -108.16957092285156,
      "activations/layer4_attention_weight_max": 114.65482330322266,
      "activations/layer4_attention_weight_min": -112.23526763916016,
      "activations/layer5_attention_weight_max": 51.41682052612305,
      "activations/layer5_attention_weight_min": -59.38051986694336,
      "activations/layer6_attention_weight_max": 46.03474426269531,
      "activations/layer6_attention_weight_min": -48.22924041748047,
      "activations/layer7_attention_weight_max": 96.04037475585938,
      "activations/layer7_attention_weight_min": -99.46537780761719,
      "activations/layer8_attention_weight_max": 42.666053771972656,
      "activations/layer8_attention_weight_min": -45.293731689453125,
      "activations/layer9_attention_weight_max": 32.68672180175781,
      "activations/layer9_attention_weight_min": -34.78771209716797,
      "epoch": 22.65,
      "learning_rate": 3.9590909090909085e-06,
      "loss": 2.6908,
      "step": 389850
    },
    {
      "activations/layer0_attention_weight_max": 15.932994842529297,
      "activations/layer0_attention_weight_min": -14.862924575805664,
      "activations/layer10_attention_weight_max": 38.416542053222656,
      "activations/layer10_attention_weight_min": -41.19199752807617,
      "activations/layer11_attention_weight_max": 37.002098083496094,
      "activations/layer11_attention_weight_min": -36.57284927368164,
      "activations/layer12_attention_weight_max": 28.61886215209961,
      "activations/layer12_attention_weight_min": -27.347749710083008,
      "activations/layer13_attention_weight_max": 41.96230697631836,
      "activations/layer13_attention_weight_min": -37.10138702392578,
      "activations/layer14_attention_weight_max": 46.425594329833984,
      "activations/layer14_attention_weight_min": -38.201820373535156,
      "activations/layer15_attention_weight_max": 42.528194427490234,
      "activations/layer15_attention_weight_min": -37.57002258300781,
      "activations/layer16_attention_weight_max": 31.830432891845703,
      "activations/layer16_attention_weight_min": -28.918825149536133,
      "activations/layer17_attention_weight_max": 29.43562889099121,
      "activations/layer17_attention_weight_min": -26.83221435546875,
      "activations/layer18_attention_weight_max": 33.476192474365234,
      "activations/layer18_attention_weight_min": -27.689838409423828,
      "activations/layer19_attention_weight_max": 37.49650573730469,
      "activations/layer19_attention_weight_min": -32.806758880615234,
      "activations/layer1_attention_weight_max": 17.039541244506836,
      "activations/layer1_attention_weight_min": -15.808663368225098,
      "activations/layer20_attention_weight_max": 31.385101318359375,
      "activations/layer20_attention_weight_min": -26.11507797241211,
      "activations/layer21_attention_weight_max": 30.978145599365234,
      "activations/layer21_attention_weight_min": -24.701255798339844,
      "activations/layer22_attention_weight_max": 43.20442581176758,
      "activations/layer22_attention_weight_min": -33.69089889526367,
      "activations/layer23_attention_weight_max": 34.22653579711914,
      "activations/layer23_attention_weight_min": -26.707944869995117,
      "activations/layer2_attention_weight_max": 33.902854919433594,
      "activations/layer2_attention_weight_min": -33.381195068359375,
      "activations/layer3_attention_weight_max": 110.90574645996094,
      "activations/layer3_attention_weight_min": -107.6479721069336,
      "activations/layer4_attention_weight_max": 113.95687103271484,
      "activations/layer4_attention_weight_min": -116.79009246826172,
      "activations/layer5_attention_weight_max": 53.70638656616211,
      "activations/layer5_attention_weight_min": -61.03755187988281,
      "activations/layer6_attention_weight_max": 47.812110900878906,
      "activations/layer6_attention_weight_min": -50.621891021728516,
      "activations/layer7_attention_weight_max": 98.19004821777344,
      "activations/layer7_attention_weight_min": -105.1078872680664,
      "activations/layer8_attention_weight_max": 45.74296951293945,
      "activations/layer8_attention_weight_min": -49.12858963012695,
      "activations/layer9_attention_weight_max": 35.062313079833984,
      "activations/layer9_attention_weight_min": -38.805843353271484,
      "epoch": 22.66,
      "learning_rate": 3.940151515151514e-06,
      "loss": 2.6999,
      "step": 389900
    },
    {
      "activations/layer0_attention_weight_max": 17.156513214111328,
      "activations/layer0_attention_weight_min": -13.759502410888672,
      "activations/layer10_attention_weight_max": 38.30445098876953,
      "activations/layer10_attention_weight_min": -37.105934143066406,
      "activations/layer11_attention_weight_max": 37.02668762207031,
      "activations/layer11_attention_weight_min": -36.13259506225586,
      "activations/layer12_attention_weight_max": 25.7949161529541,
      "activations/layer12_attention_weight_min": -27.38928985595703,
      "activations/layer13_attention_weight_max": 37.14268112182617,
      "activations/layer13_attention_weight_min": -37.54452133178711,
      "activations/layer14_attention_weight_max": 40.56871795654297,
      "activations/layer14_attention_weight_min": -39.812686920166016,
      "activations/layer15_attention_weight_max": 37.065120697021484,
      "activations/layer15_attention_weight_min": -36.60004806518555,
      "activations/layer16_attention_weight_max": 28.962413787841797,
      "activations/layer16_attention_weight_min": -29.571794509887695,
      "activations/layer17_attention_weight_max": 28.389497756958008,
      "activations/layer17_attention_weight_min": -25.233057022094727,
      "activations/layer18_attention_weight_max": 29.95747184753418,
      "activations/layer18_attention_weight_min": -24.78873062133789,
      "activations/layer19_attention_weight_max": 33.249820709228516,
      "activations/layer19_attention_weight_min": -34.13059616088867,
      "activations/layer1_attention_weight_max": 14.764163970947266,
      "activations/layer1_attention_weight_min": -15.548280715942383,
      "activations/layer20_attention_weight_max": 25.804664611816406,
      "activations/layer20_attention_weight_min": -24.73869514465332,
      "activations/layer21_attention_weight_max": 26.477645874023438,
      "activations/layer21_attention_weight_min": -25.945165634155273,
      "activations/layer22_attention_weight_max": 39.69513702392578,
      "activations/layer22_attention_weight_min": -31.20256233215332,
      "activations/layer23_attention_weight_max": 28.052106857299805,
      "activations/layer23_attention_weight_min": -24.947586059570312,
      "activations/layer2_attention_weight_max": 34.31075668334961,
      "activations/layer2_attention_weight_min": -35.82020568847656,
      "activations/layer3_attention_weight_max": 110.90923309326172,
      "activations/layer3_attention_weight_min": -115.2380142211914,
      "activations/layer4_attention_weight_max": 114.35628509521484,
      "activations/layer4_attention_weight_min": -116.00927734375,
      "activations/layer5_attention_weight_max": 52.30894470214844,
      "activations/layer5_attention_weight_min": -62.0447998046875,
      "activations/layer6_attention_weight_max": 48.00417709350586,
      "activations/layer6_attention_weight_min": -51.04347610473633,
      "activations/layer7_attention_weight_max": 102.51319885253906,
      "activations/layer7_attention_weight_min": -102.32395935058594,
      "activations/layer8_attention_weight_max": 43.3871955871582,
      "activations/layer8_attention_weight_min": -47.528717041015625,
      "activations/layer9_attention_weight_max": 31.985509872436523,
      "activations/layer9_attention_weight_min": -36.39434051513672,
      "epoch": 22.66,
      "learning_rate": 3.921212121212121e-06,
      "loss": 2.6987,
      "step": 389950
    },
    {
      "activations/layer0_attention_weight_max": 16.668842315673828,
      "activations/layer0_attention_weight_min": -15.698484420776367,
      "activations/layer10_attention_weight_max": 35.56399154663086,
      "activations/layer10_attention_weight_min": -35.712074279785156,
      "activations/layer11_attention_weight_max": 34.29072570800781,
      "activations/layer11_attention_weight_min": -37.49433898925781,
      "activations/layer12_attention_weight_max": 26.999162673950195,
      "activations/layer12_attention_weight_min": -27.47114372253418,
      "activations/layer13_attention_weight_max": 40.239662170410156,
      "activations/layer13_attention_weight_min": -36.76349639892578,
      "activations/layer14_attention_weight_max": 45.05075454711914,
      "activations/layer14_attention_weight_min": -40.91803741455078,
      "activations/layer15_attention_weight_max": 40.44905090332031,
      "activations/layer15_attention_weight_min": -39.03893280029297,
      "activations/layer16_attention_weight_max": 29.610998153686523,
      "activations/layer16_attention_weight_min": -29.265295028686523,
      "activations/layer17_attention_weight_max": 29.218841552734375,
      "activations/layer17_attention_weight_min": -26.15209197998047,
      "activations/layer18_attention_weight_max": 32.2584114074707,
      "activations/layer18_attention_weight_min": -27.8439998626709,
      "activations/layer19_attention_weight_max": 37.04844284057617,
      "activations/layer19_attention_weight_min": -32.61796569824219,
      "activations/layer1_attention_weight_max": 15.145423889160156,
      "activations/layer1_attention_weight_min": -16.181379318237305,
      "activations/layer20_attention_weight_max": 29.867586135864258,
      "activations/layer20_attention_weight_min": -25.547483444213867,
      "activations/layer21_attention_weight_max": 30.64720916748047,
      "activations/layer21_attention_weight_min": -25.87810516357422,
      "activations/layer22_attention_weight_max": 44.2217903137207,
      "activations/layer22_attention_weight_min": -31.157325744628906,
      "activations/layer23_attention_weight_max": 33.95177459716797,
      "activations/layer23_attention_weight_min": -27.214012145996094,
      "activations/layer2_attention_weight_max": 34.015525817871094,
      "activations/layer2_attention_weight_min": -33.72615051269531,
      "activations/layer3_attention_weight_max": 110.37556457519531,
      "activations/layer3_attention_weight_min": -107.55839538574219,
      "activations/layer4_attention_weight_max": 112.89332580566406,
      "activations/layer4_attention_weight_min": -112.80850982666016,
      "activations/layer5_attention_weight_max": 51.20676803588867,
      "activations/layer5_attention_weight_min": -60.329463958740234,
      "activations/layer6_attention_weight_max": 47.75725555419922,
      "activations/layer6_attention_weight_min": -48.63859939575195,
      "activations/layer7_attention_weight_max": 95.85868835449219,
      "activations/layer7_attention_weight_min": -99.89290618896484,
      "activations/layer8_attention_weight_max": 42.85300827026367,
      "activations/layer8_attention_weight_min": -44.77524948120117,
      "activations/layer9_attention_weight_max": 31.877485275268555,
      "activations/layer9_attention_weight_min": -34.724884033203125,
      "epoch": 22.66,
      "learning_rate": 3.9022727272727266e-06,
      "loss": 2.6985,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4859,
      "eval_samples_per_second": 506.016,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4859,
      "eval_openwebtext_samples_per_second": 506.016,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_wikitext_loss": 2.89453125,
      "eval_wikitext_ppl": 18.075026786359576,
      "eval_wikitext_runtime": 1.9255,
      "eval_wikitext_samples_per_second": 236.826,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_lambada_loss": 2.275390625,
      "eval_lambada_ppl": 9.731719723202135,
      "eval_lambada_runtime": 9.5064,
      "eval_lambada_samples_per_second": 512.179,
      "step": 390000
    },
    {
      "activations/layer0_attention_weight_max": 15.808980941772461,
      "activations/layer0_attention_weight_min": -15.450103759765625,
      "activations/layer10_attention_weight_max": 36.21724319458008,
      "activations/layer10_attention_weight_min": -37.59909439086914,
      "activations/layer11_attention_weight_max": 35.0855827331543,
      "activations/layer11_attention_weight_min": -38.2822265625,
      "activations/layer12_attention_weight_max": 29.102035522460938,
      "activations/layer12_attention_weight_min": -30.32585906982422,
      "activations/layer13_attention_weight_max": 42.68756866455078,
      "activations/layer13_attention_weight_min": -41.10310363769531,
      "activations/layer14_attention_weight_max": 45.541656494140625,
      "activations/layer14_attention_weight_min": -41.67966079711914,
      "activations/layer15_attention_weight_max": 40.55598449707031,
      "activations/layer15_attention_weight_min": -37.893043518066406,
      "activations/layer16_attention_weight_max": 32.41514205932617,
      "activations/layer16_attention_weight_min": -29.07851219177246,
      "activations/layer17_attention_weight_max": 34.21730422973633,
      "activations/layer17_attention_weight_min": -27.352636337280273,
      "activations/layer18_attention_weight_max": 33.580726623535156,
      "activations/layer18_attention_weight_min": -26.82967758178711,
      "activations/layer19_attention_weight_max": 35.55105972290039,
      "activations/layer19_attention_weight_min": -33.537532806396484,
      "activations/layer1_attention_weight_max": 15.79600715637207,
      "activations/layer1_attention_weight_min": -15.384220123291016,
      "activations/layer20_attention_weight_max": 27.962188720703125,
      "activations/layer20_attention_weight_min": -23.701684951782227,
      "activations/layer21_attention_weight_max": 28.925230026245117,
      "activations/layer21_attention_weight_min": -23.780820846557617,
      "activations/layer22_attention_weight_max": 38.47064971923828,
      "activations/layer22_attention_weight_min": -30.311161041259766,
      "activations/layer23_attention_weight_max": 30.988189697265625,
      "activations/layer23_attention_weight_min": -25.7501220703125,
      "activations/layer2_attention_weight_max": 33.57957458496094,
      "activations/layer2_attention_weight_min": -34.031944274902344,
      "activations/layer3_attention_weight_max": 107.45970916748047,
      "activations/layer3_attention_weight_min": -110.34187316894531,
      "activations/layer4_attention_weight_max": 111.74940490722656,
      "activations/layer4_attention_weight_min": -115.55388641357422,
      "activations/layer5_attention_weight_max": 52.032928466796875,
      "activations/layer5_attention_weight_min": -63.49198532104492,
      "activations/layer6_attention_weight_max": 47.11248016357422,
      "activations/layer6_attention_weight_min": -50.47938919067383,
      "activations/layer7_attention_weight_max": 100.3447265625,
      "activations/layer7_attention_weight_min": -104.60575866699219,
      "activations/layer8_attention_weight_max": 43.33725357055664,
      "activations/layer8_attention_weight_min": -50.122013092041016,
      "activations/layer9_attention_weight_max": 33.48928451538086,
      "activations/layer9_attention_weight_min": -35.98297882080078,
      "epoch": 22.66,
      "learning_rate": 3.883333333333333e-06,
      "loss": 2.7086,
      "step": 390050
    },
    {
      "activations/layer0_attention_weight_max": 17.050905227661133,
      "activations/layer0_attention_weight_min": -13.543880462646484,
      "activations/layer10_attention_weight_max": 37.751644134521484,
      "activations/layer10_attention_weight_min": -37.697471618652344,
      "activations/layer11_attention_weight_max": 38.19536590576172,
      "activations/layer11_attention_weight_min": -37.77180480957031,
      "activations/layer12_attention_weight_max": 27.667503356933594,
      "activations/layer12_attention_weight_min": -28.939144134521484,
      "activations/layer13_attention_weight_max": 45.364906311035156,
      "activations/layer13_attention_weight_min": -39.79241180419922,
      "activations/layer14_attention_weight_max": 44.47315216064453,
      "activations/layer14_attention_weight_min": -43.19087219238281,
      "activations/layer15_attention_weight_max": 40.3757209777832,
      "activations/layer15_attention_weight_min": -38.45060729980469,
      "activations/layer16_attention_weight_max": 29.37101936340332,
      "activations/layer16_attention_weight_min": -28.042375564575195,
      "activations/layer17_attention_weight_max": 29.00717544555664,
      "activations/layer17_attention_weight_min": -28.820703506469727,
      "activations/layer18_attention_weight_max": 32.61265563964844,
      "activations/layer18_attention_weight_min": -26.519100189208984,
      "activations/layer19_attention_weight_max": 33.772220611572266,
      "activations/layer19_attention_weight_min": -33.41632843017578,
      "activations/layer1_attention_weight_max": 15.941659927368164,
      "activations/layer1_attention_weight_min": -15.520797729492188,
      "activations/layer20_attention_weight_max": 28.279512405395508,
      "activations/layer20_attention_weight_min": -26.636436462402344,
      "activations/layer21_attention_weight_max": 28.386022567749023,
      "activations/layer21_attention_weight_min": -25.728404998779297,
      "activations/layer22_attention_weight_max": 39.401248931884766,
      "activations/layer22_attention_weight_min": -32.02488327026367,
      "activations/layer23_attention_weight_max": 31.344722747802734,
      "activations/layer23_attention_weight_min": -26.96613121032715,
      "activations/layer2_attention_weight_max": 34.59246063232422,
      "activations/layer2_attention_weight_min": -34.190765380859375,
      "activations/layer3_attention_weight_max": 107.96244812011719,
      "activations/layer3_attention_weight_min": -107.46556091308594,
      "activations/layer4_attention_weight_max": 114.75357818603516,
      "activations/layer4_attention_weight_min": -116.64643859863281,
      "activations/layer5_attention_weight_max": 52.98162841796875,
      "activations/layer5_attention_weight_min": -62.42768478393555,
      "activations/layer6_attention_weight_max": 45.60666275024414,
      "activations/layer6_attention_weight_min": -49.07528305053711,
      "activations/layer7_attention_weight_max": 101.58199310302734,
      "activations/layer7_attention_weight_min": -108.0284194946289,
      "activations/layer8_attention_weight_max": 45.09680938720703,
      "activations/layer8_attention_weight_min": -48.21674346923828,
      "activations/layer9_attention_weight_max": 35.70085525512695,
      "activations/layer9_attention_weight_min": -37.797306060791016,
      "epoch": 22.67,
      "learning_rate": 3.864393939393939e-06,
      "loss": 2.7007,
      "step": 390100
    },
    {
      "activations/layer0_attention_weight_max": 17.10514259338379,
      "activations/layer0_attention_weight_min": -14.604755401611328,
      "activations/layer10_attention_weight_max": 37.519203186035156,
      "activations/layer10_attention_weight_min": -37.235469818115234,
      "activations/layer11_attention_weight_max": 36.76097106933594,
      "activations/layer11_attention_weight_min": -37.86503601074219,
      "activations/layer12_attention_weight_max": 26.59393882751465,
      "activations/layer12_attention_weight_min": -28.0290584564209,
      "activations/layer13_attention_weight_max": 38.648597717285156,
      "activations/layer13_attention_weight_min": -36.06633758544922,
      "activations/layer14_attention_weight_max": 41.99528503417969,
      "activations/layer14_attention_weight_min": -38.882476806640625,
      "activations/layer15_attention_weight_max": 38.22382354736328,
      "activations/layer15_attention_weight_min": -37.212440490722656,
      "activations/layer16_attention_weight_max": 29.390153884887695,
      "activations/layer16_attention_weight_min": -28.799705505371094,
      "activations/layer17_attention_weight_max": 32.15536880493164,
      "activations/layer17_attention_weight_min": -27.665477752685547,
      "activations/layer18_attention_weight_max": 33.71656036376953,
      "activations/layer18_attention_weight_min": -28.814434051513672,
      "activations/layer19_attention_weight_max": 37.16697692871094,
      "activations/layer19_attention_weight_min": -33.24679946899414,
      "activations/layer1_attention_weight_max": 15.233468055725098,
      "activations/layer1_attention_weight_min": -15.830921173095703,
      "activations/layer20_attention_weight_max": 27.867752075195312,
      "activations/layer20_attention_weight_min": -25.91903305053711,
      "activations/layer21_attention_weight_max": 28.293380737304688,
      "activations/layer21_attention_weight_min": -25.423049926757812,
      "activations/layer22_attention_weight_max": 40.23991394042969,
      "activations/layer22_attention_weight_min": -34.47455596923828,
      "activations/layer23_attention_weight_max": 33.9942626953125,
      "activations/layer23_attention_weight_min": -28.469478607177734,
      "activations/layer2_attention_weight_max": 35.74842071533203,
      "activations/layer2_attention_weight_min": -33.189964294433594,
      "activations/layer3_attention_weight_max": 114.45094299316406,
      "activations/layer3_attention_weight_min": -111.29531860351562,
      "activations/layer4_attention_weight_max": 116.05777740478516,
      "activations/layer4_attention_weight_min": -114.92547607421875,
      "activations/layer5_attention_weight_max": 52.86188507080078,
      "activations/layer5_attention_weight_min": -67.81724548339844,
      "activations/layer6_attention_weight_max": 46.319828033447266,
      "activations/layer6_attention_weight_min": -49.7962532043457,
      "activations/layer7_attention_weight_max": 98.76760864257812,
      "activations/layer7_attention_weight_min": -104.59920501708984,
      "activations/layer8_attention_weight_max": 44.5479736328125,
      "activations/layer8_attention_weight_min": -46.442813873291016,
      "activations/layer9_attention_weight_max": 35.58588790893555,
      "activations/layer9_attention_weight_min": -35.59470748901367,
      "epoch": 22.67,
      "learning_rate": 3.8454545454545454e-06,
      "loss": 2.6998,
      "step": 390150
    },
    {
      "activations/layer0_attention_weight_max": 17.278425216674805,
      "activations/layer0_attention_weight_min": -14.918909072875977,
      "activations/layer10_attention_weight_max": 34.20561981201172,
      "activations/layer10_attention_weight_min": -33.423091888427734,
      "activations/layer11_attention_weight_max": 35.538047790527344,
      "activations/layer11_attention_weight_min": -34.35688781738281,
      "activations/layer12_attention_weight_max": 25.469566345214844,
      "activations/layer12_attention_weight_min": -26.918310165405273,
      "activations/layer13_attention_weight_max": 37.43095016479492,
      "activations/layer13_attention_weight_min": -35.13956069946289,
      "activations/layer14_attention_weight_max": 40.42552947998047,
      "activations/layer14_attention_weight_min": -36.92573928833008,
      "activations/layer15_attention_weight_max": 38.665855407714844,
      "activations/layer15_attention_weight_min": -36.32809829711914,
      "activations/layer16_attention_weight_max": 28.14607810974121,
      "activations/layer16_attention_weight_min": -28.064937591552734,
      "activations/layer17_attention_weight_max": 27.873823165893555,
      "activations/layer17_attention_weight_min": -26.789186477661133,
      "activations/layer18_attention_weight_max": 34.006378173828125,
      "activations/layer18_attention_weight_min": -27.481351852416992,
      "activations/layer19_attention_weight_max": 34.04566955566406,
      "activations/layer19_attention_weight_min": -32.16098403930664,
      "activations/layer1_attention_weight_max": 15.547821998596191,
      "activations/layer1_attention_weight_min": -14.803228378295898,
      "activations/layer20_attention_weight_max": 28.486282348632812,
      "activations/layer20_attention_weight_min": -25.53811264038086,
      "activations/layer21_attention_weight_max": 28.312978744506836,
      "activations/layer21_attention_weight_min": -24.819883346557617,
      "activations/layer22_attention_weight_max": 40.20930099487305,
      "activations/layer22_attention_weight_min": -32.32124328613281,
      "activations/layer23_attention_weight_max": 32.63099670410156,
      "activations/layer23_attention_weight_min": -26.443082809448242,
      "activations/layer2_attention_weight_max": 34.59479522705078,
      "activations/layer2_attention_weight_min": -33.35269546508789,
      "activations/layer3_attention_weight_max": 107.48695373535156,
      "activations/layer3_attention_weight_min": -108.78413391113281,
      "activations/layer4_attention_weight_max": 108.65486145019531,
      "activations/layer4_attention_weight_min": -110.1822280883789,
      "activations/layer5_attention_weight_max": 51.256202697753906,
      "activations/layer5_attention_weight_min": -63.097412109375,
      "activations/layer6_attention_weight_max": 46.591575622558594,
      "activations/layer6_attention_weight_min": -48.01084518432617,
      "activations/layer7_attention_weight_max": 97.2183837890625,
      "activations/layer7_attention_weight_min": -99.76031494140625,
      "activations/layer8_attention_weight_max": 41.658966064453125,
      "activations/layer8_attention_weight_min": -45.45252227783203,
      "activations/layer9_attention_weight_max": 33.65148162841797,
      "activations/layer9_attention_weight_min": -34.26496505737305,
      "epoch": 22.67,
      "learning_rate": 3.826515151515151e-06,
      "loss": 2.6818,
      "step": 390200
    },
    {
      "activations/layer0_attention_weight_max": 17.04744529724121,
      "activations/layer0_attention_weight_min": -13.80605697631836,
      "activations/layer10_attention_weight_max": 43.25298309326172,
      "activations/layer10_attention_weight_min": -41.17081069946289,
      "activations/layer11_attention_weight_max": 42.78003692626953,
      "activations/layer11_attention_weight_min": -42.398231506347656,
      "activations/layer12_attention_weight_max": 27.85691261291504,
      "activations/layer12_attention_weight_min": -30.54423713684082,
      "activations/layer13_attention_weight_max": 37.92009353637695,
      "activations/layer13_attention_weight_min": -36.91856384277344,
      "activations/layer14_attention_weight_max": 48.12714767456055,
      "activations/layer14_attention_weight_min": -41.9527702331543,
      "activations/layer15_attention_weight_max": 42.41857147216797,
      "activations/layer15_attention_weight_min": -41.871585845947266,
      "activations/layer16_attention_weight_max": 33.13957595825195,
      "activations/layer16_attention_weight_min": -31.516311645507812,
      "activations/layer17_attention_weight_max": 30.849531173706055,
      "activations/layer17_attention_weight_min": -31.413225173950195,
      "activations/layer18_attention_weight_max": 34.87021255493164,
      "activations/layer18_attention_weight_min": -31.755647659301758,
      "activations/layer19_attention_weight_max": 35.76249313354492,
      "activations/layer19_attention_weight_min": -30.454038619995117,
      "activations/layer1_attention_weight_max": 16.401411056518555,
      "activations/layer1_attention_weight_min": -16.029306411743164,
      "activations/layer20_attention_weight_max": 30.531938552856445,
      "activations/layer20_attention_weight_min": -25.560705184936523,
      "activations/layer21_attention_weight_max": 29.846450805664062,
      "activations/layer21_attention_weight_min": -27.09330940246582,
      "activations/layer22_attention_weight_max": 45.363807678222656,
      "activations/layer22_attention_weight_min": -34.50143814086914,
      "activations/layer23_attention_weight_max": 32.17218017578125,
      "activations/layer23_attention_weight_min": -26.2181453704834,
      "activations/layer2_attention_weight_max": 35.339447021484375,
      "activations/layer2_attention_weight_min": -36.61981201171875,
      "activations/layer3_attention_weight_max": 109.65078735351562,
      "activations/layer3_attention_weight_min": -111.68875885009766,
      "activations/layer4_attention_weight_max": 113.82402801513672,
      "activations/layer4_attention_weight_min": -119.3287353515625,
      "activations/layer5_attention_weight_max": 51.752418518066406,
      "activations/layer5_attention_weight_min": -62.63890838623047,
      "activations/layer6_attention_weight_max": 48.462276458740234,
      "activations/layer6_attention_weight_min": -50.421878814697266,
      "activations/layer7_attention_weight_max": 112.96708679199219,
      "activations/layer7_attention_weight_min": -113.63987731933594,
      "activations/layer8_attention_weight_max": 50.29299545288086,
      "activations/layer8_attention_weight_min": -53.246768951416016,
      "activations/layer9_attention_weight_max": 37.34322738647461,
      "activations/layer9_attention_weight_min": -40.563289642333984,
      "epoch": 22.68,
      "learning_rate": 3.8075757575757573e-06,
      "loss": 2.6797,
      "step": 390250
    },
    {
      "activations/layer0_attention_weight_max": 16.05617904663086,
      "activations/layer0_attention_weight_min": -15.014248847961426,
      "activations/layer10_attention_weight_max": 34.96018600463867,
      "activations/layer10_attention_weight_min": -37.891666412353516,
      "activations/layer11_attention_weight_max": 35.00156021118164,
      "activations/layer11_attention_weight_min": -36.245521545410156,
      "activations/layer12_attention_weight_max": 26.128604888916016,
      "activations/layer12_attention_weight_min": -26.644636154174805,
      "activations/layer13_attention_weight_max": 42.4488410949707,
      "activations/layer13_attention_weight_min": -36.76957702636719,
      "activations/layer14_attention_weight_max": 42.228065490722656,
      "activations/layer14_attention_weight_min": -39.20180130004883,
      "activations/layer15_attention_weight_max": 41.12544631958008,
      "activations/layer15_attention_weight_min": -37.27041244506836,
      "activations/layer16_attention_weight_max": 30.06036949157715,
      "activations/layer16_attention_weight_min": -26.988880157470703,
      "activations/layer17_attention_weight_max": 28.597042083740234,
      "activations/layer17_attention_weight_min": -25.73253631591797,
      "activations/layer18_attention_weight_max": 32.6240348815918,
      "activations/layer18_attention_weight_min": -24.608623504638672,
      "activations/layer19_attention_weight_max": 32.179527282714844,
      "activations/layer19_attention_weight_min": -31.736591339111328,
      "activations/layer1_attention_weight_max": 16.0076847076416,
      "activations/layer1_attention_weight_min": -17.2965030670166,
      "activations/layer20_attention_weight_max": 29.909648895263672,
      "activations/layer20_attention_weight_min": -24.81270408630371,
      "activations/layer21_attention_weight_max": 31.03287696838379,
      "activations/layer21_attention_weight_min": -24.20794677734375,
      "activations/layer22_attention_weight_max": 39.7630500793457,
      "activations/layer22_attention_weight_min": -31.025964736938477,
      "activations/layer23_attention_weight_max": 31.093162536621094,
      "activations/layer23_attention_weight_min": -24.491352081298828,
      "activations/layer2_attention_weight_max": 35.225013732910156,
      "activations/layer2_attention_weight_min": -34.88812255859375,
      "activations/layer3_attention_weight_max": 106.65177917480469,
      "activations/layer3_attention_weight_min": -109.10730743408203,
      "activations/layer4_attention_weight_max": 109.29888916015625,
      "activations/layer4_attention_weight_min": -119.86827850341797,
      "activations/layer5_attention_weight_max": 51.966407775878906,
      "activations/layer5_attention_weight_min": -62.12999725341797,
      "activations/layer6_attention_weight_max": 47.17656707763672,
      "activations/layer6_attention_weight_min": -50.1560173034668,
      "activations/layer7_attention_weight_max": 100.75438690185547,
      "activations/layer7_attention_weight_min": -104.8540267944336,
      "activations/layer8_attention_weight_max": 42.20894241333008,
      "activations/layer8_attention_weight_min": -47.9918098449707,
      "activations/layer9_attention_weight_max": 35.257633209228516,
      "activations/layer9_attention_weight_min": -35.83556365966797,
      "epoch": 22.68,
      "learning_rate": 3.788636363636363e-06,
      "loss": 2.6949,
      "step": 390300
    },
    {
      "activations/layer0_attention_weight_max": 16.397315979003906,
      "activations/layer0_attention_weight_min": -14.84068775177002,
      "activations/layer10_attention_weight_max": 40.746761322021484,
      "activations/layer10_attention_weight_min": -39.11347961425781,
      "activations/layer11_attention_weight_max": 40.59033203125,
      "activations/layer11_attention_weight_min": -38.828941345214844,
      "activations/layer12_attention_weight_max": 29.304094314575195,
      "activations/layer12_attention_weight_min": -29.580883026123047,
      "activations/layer13_attention_weight_max": 47.724422454833984,
      "activations/layer13_attention_weight_min": -39.02427673339844,
      "activations/layer14_attention_weight_max": 48.25266647338867,
      "activations/layer14_attention_weight_min": -40.75871276855469,
      "activations/layer15_attention_weight_max": 51.338714599609375,
      "activations/layer15_attention_weight_min": -39.13834762573242,
      "activations/layer16_attention_weight_max": 31.45201873779297,
      "activations/layer16_attention_weight_min": -29.069101333618164,
      "activations/layer17_attention_weight_max": 30.53287124633789,
      "activations/layer17_attention_weight_min": -28.223459243774414,
      "activations/layer18_attention_weight_max": 30.907123565673828,
      "activations/layer18_attention_weight_min": -25.415374755859375,
      "activations/layer19_attention_weight_max": 36.542537689208984,
      "activations/layer19_attention_weight_min": -32.709163665771484,
      "activations/layer1_attention_weight_max": 17.677400588989258,
      "activations/layer1_attention_weight_min": -16.512928009033203,
      "activations/layer20_attention_weight_max": 33.05009460449219,
      "activations/layer20_attention_weight_min": -24.81548309326172,
      "activations/layer21_attention_weight_max": 29.44535255432129,
      "activations/layer21_attention_weight_min": -24.919973373413086,
      "activations/layer22_attention_weight_max": 41.7252197265625,
      "activations/layer22_attention_weight_min": -31.15270233154297,
      "activations/layer23_attention_weight_max": 35.011680603027344,
      "activations/layer23_attention_weight_min": -25.49700164794922,
      "activations/layer2_attention_weight_max": 35.512107849121094,
      "activations/layer2_attention_weight_min": -34.85536193847656,
      "activations/layer3_attention_weight_max": 113.84616088867188,
      "activations/layer3_attention_weight_min": -110.53671264648438,
      "activations/layer4_attention_weight_max": 118.47359466552734,
      "activations/layer4_attention_weight_min": -114.08087158203125,
      "activations/layer5_attention_weight_max": 52.34896469116211,
      "activations/layer5_attention_weight_min": -63.25544357299805,
      "activations/layer6_attention_weight_max": 48.10389709472656,
      "activations/layer6_attention_weight_min": -50.3677864074707,
      "activations/layer7_attention_weight_max": 103.41826629638672,
      "activations/layer7_attention_weight_min": -104.04917907714844,
      "activations/layer8_attention_weight_max": 47.066619873046875,
      "activations/layer8_attention_weight_min": -51.25535583496094,
      "activations/layer9_attention_weight_max": 35.38027572631836,
      "activations/layer9_attention_weight_min": -39.728759765625,
      "epoch": 22.68,
      "learning_rate": 3.7696969696969696e-06,
      "loss": 2.6817,
      "step": 390350
    },
    {
      "activations/layer0_attention_weight_max": 17.477313995361328,
      "activations/layer0_attention_weight_min": -14.237340927124023,
      "activations/layer10_attention_weight_max": 36.90430450439453,
      "activations/layer10_attention_weight_min": -37.52161407470703,
      "activations/layer11_attention_weight_max": 37.19115447998047,
      "activations/layer11_attention_weight_min": -39.222503662109375,
      "activations/layer12_attention_weight_max": 29.314804077148438,
      "activations/layer12_attention_weight_min": -27.688982009887695,
      "activations/layer13_attention_weight_max": 43.5985221862793,
      "activations/layer13_attention_weight_min": -37.44309997558594,
      "activations/layer14_attention_weight_max": 49.685848236083984,
      "activations/layer14_attention_weight_min": -39.15830612182617,
      "activations/layer15_attention_weight_max": 46.31943893432617,
      "activations/layer15_attention_weight_min": -40.80369567871094,
      "activations/layer16_attention_weight_max": 33.2484130859375,
      "activations/layer16_attention_weight_min": -27.50030517578125,
      "activations/layer17_attention_weight_max": 31.893632888793945,
      "activations/layer17_attention_weight_min": -27.737293243408203,
      "activations/layer18_attention_weight_max": 32.38179016113281,
      "activations/layer18_attention_weight_min": -25.90001678466797,
      "activations/layer19_attention_weight_max": 37.39474105834961,
      "activations/layer19_attention_weight_min": -32.51538848876953,
      "activations/layer1_attention_weight_max": 17.61881446838379,
      "activations/layer1_attention_weight_min": -15.951430320739746,
      "activations/layer20_attention_weight_max": 30.337987899780273,
      "activations/layer20_attention_weight_min": -23.81890106201172,
      "activations/layer21_attention_weight_max": 28.145536422729492,
      "activations/layer21_attention_weight_min": -24.846460342407227,
      "activations/layer22_attention_weight_max": 41.23065948486328,
      "activations/layer22_attention_weight_min": -30.814899444580078,
      "activations/layer23_attention_weight_max": 31.795766830444336,
      "activations/layer23_attention_weight_min": -23.738523483276367,
      "activations/layer2_attention_weight_max": 35.8868408203125,
      "activations/layer2_attention_weight_min": -35.16814041137695,
      "activations/layer3_attention_weight_max": 112.67671966552734,
      "activations/layer3_attention_weight_min": -114.37464141845703,
      "activations/layer4_attention_weight_max": 120.44886779785156,
      "activations/layer4_attention_weight_min": -119.9204330444336,
      "activations/layer5_attention_weight_max": 54.8535270690918,
      "activations/layer5_attention_weight_min": -65.72901153564453,
      "activations/layer6_attention_weight_max": 49.86320877075195,
      "activations/layer6_attention_weight_min": -51.638004302978516,
      "activations/layer7_attention_weight_max": 102.07463836669922,
      "activations/layer7_attention_weight_min": -113.05158233642578,
      "activations/layer8_attention_weight_max": 46.6071662902832,
      "activations/layer8_attention_weight_min": -50.729393005371094,
      "activations/layer9_attention_weight_max": 33.69803237915039,
      "activations/layer9_attention_weight_min": -37.67548751831055,
      "epoch": 22.68,
      "learning_rate": 3.7507575757575754e-06,
      "loss": 2.6999,
      "step": 390400
    },
    {
      "activations/layer0_attention_weight_max": 16.82204246520996,
      "activations/layer0_attention_weight_min": -14.10920524597168,
      "activations/layer10_attention_weight_max": 34.418975830078125,
      "activations/layer10_attention_weight_min": -38.358642578125,
      "activations/layer11_attention_weight_max": 35.802547454833984,
      "activations/layer11_attention_weight_min": -38.02964401245117,
      "activations/layer12_attention_weight_max": 25.983566284179688,
      "activations/layer12_attention_weight_min": -29.120609283447266,
      "activations/layer13_attention_weight_max": 40.323036193847656,
      "activations/layer13_attention_weight_min": -40.27464294433594,
      "activations/layer14_attention_weight_max": 42.638736724853516,
      "activations/layer14_attention_weight_min": -41.09955596923828,
      "activations/layer15_attention_weight_max": 38.709625244140625,
      "activations/layer15_attention_weight_min": -40.719093322753906,
      "activations/layer16_attention_weight_max": 32.46104431152344,
      "activations/layer16_attention_weight_min": -28.998083114624023,
      "activations/layer17_attention_weight_max": 34.729557037353516,
      "activations/layer17_attention_weight_min": -29.439571380615234,
      "activations/layer18_attention_weight_max": 36.31880569458008,
      "activations/layer18_attention_weight_min": -29.821247100830078,
      "activations/layer19_attention_weight_max": 39.74923324584961,
      "activations/layer19_attention_weight_min": -33.11594772338867,
      "activations/layer1_attention_weight_max": 16.047164916992188,
      "activations/layer1_attention_weight_min": -15.628990173339844,
      "activations/layer20_attention_weight_max": 35.417320251464844,
      "activations/layer20_attention_weight_min": -25.948949813842773,
      "activations/layer21_attention_weight_max": 34.05347442626953,
      "activations/layer21_attention_weight_min": -25.96783447265625,
      "activations/layer22_attention_weight_max": 45.52033233642578,
      "activations/layer22_attention_weight_min": -34.93425369262695,
      "activations/layer23_attention_weight_max": 37.44346618652344,
      "activations/layer23_attention_weight_min": -27.385478973388672,
      "activations/layer2_attention_weight_max": 35.71413040161133,
      "activations/layer2_attention_weight_min": -33.67176055908203,
      "activations/layer3_attention_weight_max": 109.38710021972656,
      "activations/layer3_attention_weight_min": -107.16769409179688,
      "activations/layer4_attention_weight_max": 108.3208999633789,
      "activations/layer4_attention_weight_min": -114.97563934326172,
      "activations/layer5_attention_weight_max": 49.020503997802734,
      "activations/layer5_attention_weight_min": -61.13262939453125,
      "activations/layer6_attention_weight_max": 46.95011520385742,
      "activations/layer6_attention_weight_min": -49.42463684082031,
      "activations/layer7_attention_weight_max": 99.82025146484375,
      "activations/layer7_attention_weight_min": -105.84268951416016,
      "activations/layer8_attention_weight_max": 45.11180114746094,
      "activations/layer8_attention_weight_min": -46.689613342285156,
      "activations/layer9_attention_weight_max": 32.924442291259766,
      "activations/layer9_attention_weight_min": -36.55636215209961,
      "epoch": 22.69,
      "learning_rate": 3.7318181818181815e-06,
      "loss": 2.7059,
      "step": 390450
    },
    {
      "activations/layer0_attention_weight_max": 16.02379035949707,
      "activations/layer0_attention_weight_min": -13.428126335144043,
      "activations/layer10_attention_weight_max": 39.072181701660156,
      "activations/layer10_attention_weight_min": -36.6236572265625,
      "activations/layer11_attention_weight_max": 37.541908264160156,
      "activations/layer11_attention_weight_min": -36.75434875488281,
      "activations/layer12_attention_weight_max": 27.032350540161133,
      "activations/layer12_attention_weight_min": -28.430988311767578,
      "activations/layer13_attention_weight_max": 42.12788772583008,
      "activations/layer13_attention_weight_min": -36.86445236206055,
      "activations/layer14_attention_weight_max": 41.73536682128906,
      "activations/layer14_attention_weight_min": -39.75518035888672,
      "activations/layer15_attention_weight_max": 39.82889938354492,
      "activations/layer15_attention_weight_min": -37.14415740966797,
      "activations/layer16_attention_weight_max": 28.927743911743164,
      "activations/layer16_attention_weight_min": -29.044830322265625,
      "activations/layer17_attention_weight_max": 28.954492568969727,
      "activations/layer17_attention_weight_min": -27.714672088623047,
      "activations/layer18_attention_weight_max": 32.25772476196289,
      "activations/layer18_attention_weight_min": -27.726303100585938,
      "activations/layer19_attention_weight_max": 32.15401840209961,
      "activations/layer19_attention_weight_min": -30.823129653930664,
      "activations/layer1_attention_weight_max": 15.385926246643066,
      "activations/layer1_attention_weight_min": -14.586990356445312,
      "activations/layer20_attention_weight_max": 25.49227523803711,
      "activations/layer20_attention_weight_min": -25.355491638183594,
      "activations/layer21_attention_weight_max": 28.878538131713867,
      "activations/layer21_attention_weight_min": -24.210107803344727,
      "activations/layer22_attention_weight_max": 39.88994216918945,
      "activations/layer22_attention_weight_min": -32.00065612792969,
      "activations/layer23_attention_weight_max": 30.125137329101562,
      "activations/layer23_attention_weight_min": -23.423725128173828,
      "activations/layer2_attention_weight_max": 35.01953125,
      "activations/layer2_attention_weight_min": -33.415340423583984,
      "activations/layer3_attention_weight_max": 113.93712615966797,
      "activations/layer3_attention_weight_min": -112.17137145996094,
      "activations/layer4_attention_weight_max": 115.26644134521484,
      "activations/layer4_attention_weight_min": -115.70833587646484,
      "activations/layer5_attention_weight_max": 53.419708251953125,
      "activations/layer5_attention_weight_min": -66.76893615722656,
      "activations/layer6_attention_weight_max": 48.342491149902344,
      "activations/layer6_attention_weight_min": -54.046024322509766,
      "activations/layer7_attention_weight_max": 106.53085327148438,
      "activations/layer7_attention_weight_min": -111.3711166381836,
      "activations/layer8_attention_weight_max": 48.0380859375,
      "activations/layer8_attention_weight_min": -50.34429168701172,
      "activations/layer9_attention_weight_max": 34.85265350341797,
      "activations/layer9_attention_weight_min": -37.92168045043945,
      "epoch": 22.69,
      "learning_rate": 3.7128787878787872e-06,
      "loss": 2.6841,
      "step": 390500
    },
    {
      "activations/layer0_attention_weight_max": 17.471710205078125,
      "activations/layer0_attention_weight_min": -14.807656288146973,
      "activations/layer10_attention_weight_max": 37.814754486083984,
      "activations/layer10_attention_weight_min": -37.827537536621094,
      "activations/layer11_attention_weight_max": 36.693031311035156,
      "activations/layer11_attention_weight_min": -38.83498001098633,
      "activations/layer12_attention_weight_max": 29.150720596313477,
      "activations/layer12_attention_weight_min": -29.062501907348633,
      "activations/layer13_attention_weight_max": 40.57740783691406,
      "activations/layer13_attention_weight_min": -38.2933349609375,
      "activations/layer14_attention_weight_max": 44.90984344482422,
      "activations/layer14_attention_weight_min": -38.972808837890625,
      "activations/layer15_attention_weight_max": 41.08314514160156,
      "activations/layer15_attention_weight_min": -36.9271125793457,
      "activations/layer16_attention_weight_max": 28.34231185913086,
      "activations/layer16_attention_weight_min": -29.32582664489746,
      "activations/layer17_attention_weight_max": 28.239826202392578,
      "activations/layer17_attention_weight_min": -26.42814826965332,
      "activations/layer18_attention_weight_max": 35.09174346923828,
      "activations/layer18_attention_weight_min": -27.317110061645508,
      "activations/layer19_attention_weight_max": 35.25291061401367,
      "activations/layer19_attention_weight_min": -31.072416305541992,
      "activations/layer1_attention_weight_max": 15.688799858093262,
      "activations/layer1_attention_weight_min": -16.40752601623535,
      "activations/layer20_attention_weight_max": 28.09734535217285,
      "activations/layer20_attention_weight_min": -25.14125633239746,
      "activations/layer21_attention_weight_max": 28.693283081054688,
      "activations/layer21_attention_weight_min": -24.47611427307129,
      "activations/layer22_attention_weight_max": 41.41621398925781,
      "activations/layer22_attention_weight_min": -32.036006927490234,
      "activations/layer23_attention_weight_max": 34.06486129760742,
      "activations/layer23_attention_weight_min": -25.242847442626953,
      "activations/layer2_attention_weight_max": 34.165679931640625,
      "activations/layer2_attention_weight_min": -34.554229736328125,
      "activations/layer3_attention_weight_max": 111.21098327636719,
      "activations/layer3_attention_weight_min": -110.41987609863281,
      "activations/layer4_attention_weight_max": 115.64776611328125,
      "activations/layer4_attention_weight_min": -116.41153717041016,
      "activations/layer5_attention_weight_max": 54.296653747558594,
      "activations/layer5_attention_weight_min": -61.41236877441406,
      "activations/layer6_attention_weight_max": 48.73976516723633,
      "activations/layer6_attention_weight_min": -51.51913833618164,
      "activations/layer7_attention_weight_max": 104.7204360961914,
      "activations/layer7_attention_weight_min": -104.29156494140625,
      "activations/layer8_attention_weight_max": 47.103355407714844,
      "activations/layer8_attention_weight_min": -48.832977294921875,
      "activations/layer9_attention_weight_max": 36.03815841674805,
      "activations/layer9_attention_weight_min": -39.91339874267578,
      "epoch": 22.69,
      "learning_rate": 3.6939393939393934e-06,
      "loss": 2.7144,
      "step": 390550
    },
    {
      "activations/layer0_attention_weight_max": 17.10624885559082,
      "activations/layer0_attention_weight_min": -15.147933959960938,
      "activations/layer10_attention_weight_max": 43.62836837768555,
      "activations/layer10_attention_weight_min": -40.32155227661133,
      "activations/layer11_attention_weight_max": 44.92058563232422,
      "activations/layer11_attention_weight_min": -43.100563049316406,
      "activations/layer12_attention_weight_max": 29.35245132446289,
      "activations/layer12_attention_weight_min": -27.42713737487793,
      "activations/layer13_attention_weight_max": 40.74296569824219,
      "activations/layer13_attention_weight_min": -39.479976654052734,
      "activations/layer14_attention_weight_max": 53.27192687988281,
      "activations/layer14_attention_weight_min": -42.30471420288086,
      "activations/layer15_attention_weight_max": 48.40822982788086,
      "activations/layer15_attention_weight_min": -44.26582717895508,
      "activations/layer16_attention_weight_max": 37.34272766113281,
      "activations/layer16_attention_weight_min": -31.066471099853516,
      "activations/layer17_attention_weight_max": 37.55540466308594,
      "activations/layer17_attention_weight_min": -29.471120834350586,
      "activations/layer18_attention_weight_max": 34.030033111572266,
      "activations/layer18_attention_weight_min": -28.524150848388672,
      "activations/layer19_attention_weight_max": 35.9436149597168,
      "activations/layer19_attention_weight_min": -32.748775482177734,
      "activations/layer1_attention_weight_max": 15.853631019592285,
      "activations/layer1_attention_weight_min": -15.603180885314941,
      "activations/layer20_attention_weight_max": 28.03563117980957,
      "activations/layer20_attention_weight_min": -24.74394416809082,
      "activations/layer21_attention_weight_max": 26.453826904296875,
      "activations/layer21_attention_weight_min": -24.79655647277832,
      "activations/layer22_attention_weight_max": 40.95783996582031,
      "activations/layer22_attention_weight_min": -32.82732009887695,
      "activations/layer23_attention_weight_max": 31.95081329345703,
      "activations/layer23_attention_weight_min": -26.333881378173828,
      "activations/layer2_attention_weight_max": 35.10424041748047,
      "activations/layer2_attention_weight_min": -35.47621154785156,
      "activations/layer3_attention_weight_max": 107.21791076660156,
      "activations/layer3_attention_weight_min": -110.11573791503906,
      "activations/layer4_attention_weight_max": 110.87650299072266,
      "activations/layer4_attention_weight_min": -112.56493377685547,
      "activations/layer5_attention_weight_max": 53.89960479736328,
      "activations/layer5_attention_weight_min": -60.72268295288086,
      "activations/layer6_attention_weight_max": 49.13083267211914,
      "activations/layer6_attention_weight_min": -50.58129119873047,
      "activations/layer7_attention_weight_max": 108.86827850341797,
      "activations/layer7_attention_weight_min": -101.64155578613281,
      "activations/layer8_attention_weight_max": 49.15463638305664,
      "activations/layer8_attention_weight_min": -49.00614929199219,
      "activations/layer9_attention_weight_max": 34.57008361816406,
      "activations/layer9_attention_weight_min": -37.52705383300781,
      "epoch": 22.7,
      "learning_rate": 3.675e-06,
      "loss": 2.7042,
      "step": 390600
    },
    {
      "activations/layer0_attention_weight_max": 16.630687713623047,
      "activations/layer0_attention_weight_min": -12.975305557250977,
      "activations/layer10_attention_weight_max": 36.69985580444336,
      "activations/layer10_attention_weight_min": -36.968502044677734,
      "activations/layer11_attention_weight_max": 36.84449768066406,
      "activations/layer11_attention_weight_min": -35.7319450378418,
      "activations/layer12_attention_weight_max": 27.851152420043945,
      "activations/layer12_attention_weight_min": -28.782794952392578,
      "activations/layer13_attention_weight_max": 41.8646354675293,
      "activations/layer13_attention_weight_min": -40.644622802734375,
      "activations/layer14_attention_weight_max": 44.40480422973633,
      "activations/layer14_attention_weight_min": -41.61684036254883,
      "activations/layer15_attention_weight_max": 42.49634552001953,
      "activations/layer15_attention_weight_min": -38.9737663269043,
      "activations/layer16_attention_weight_max": 29.961639404296875,
      "activations/layer16_attention_weight_min": -27.622783660888672,
      "activations/layer17_attention_weight_max": 33.91382598876953,
      "activations/layer17_attention_weight_min": -28.24421501159668,
      "activations/layer18_attention_weight_max": 33.46238708496094,
      "activations/layer18_attention_weight_min": -26.68190574645996,
      "activations/layer19_attention_weight_max": 35.04541015625,
      "activations/layer19_attention_weight_min": -32.367305755615234,
      "activations/layer1_attention_weight_max": 16.014177322387695,
      "activations/layer1_attention_weight_min": -15.690227508544922,
      "activations/layer20_attention_weight_max": 31.2784366607666,
      "activations/layer20_attention_weight_min": -26.201921463012695,
      "activations/layer21_attention_weight_max": 29.661056518554688,
      "activations/layer21_attention_weight_min": -23.66177749633789,
      "activations/layer22_attention_weight_max": 43.27509307861328,
      "activations/layer22_attention_weight_min": -29.51266860961914,
      "activations/layer23_attention_weight_max": 35.62271499633789,
      "activations/layer23_attention_weight_min": -23.319812774658203,
      "activations/layer2_attention_weight_max": 35.39937973022461,
      "activations/layer2_attention_weight_min": -34.953338623046875,
      "activations/layer3_attention_weight_max": 112.55896759033203,
      "activations/layer3_attention_weight_min": -110.99353790283203,
      "activations/layer4_attention_weight_max": 112.0396957397461,
      "activations/layer4_attention_weight_min": -113.46410369873047,
      "activations/layer5_attention_weight_max": 50.88146209716797,
      "activations/layer5_attention_weight_min": -62.669517517089844,
      "activations/layer6_attention_weight_max": 47.63303756713867,
      "activations/layer6_attention_weight_min": -48.670127868652344,
      "activations/layer7_attention_weight_max": 100.87311553955078,
      "activations/layer7_attention_weight_min": -108.94334411621094,
      "activations/layer8_attention_weight_max": 44.71380615234375,
      "activations/layer8_attention_weight_min": -50.986534118652344,
      "activations/layer9_attention_weight_max": 33.86149978637695,
      "activations/layer9_attention_weight_min": -36.99746322631836,
      "epoch": 22.7,
      "learning_rate": 3.6560606060606057e-06,
      "loss": 2.693,
      "step": 390650
    },
    {
      "activations/layer0_attention_weight_max": 16.862512588500977,
      "activations/layer0_attention_weight_min": -14.35630989074707,
      "activations/layer10_attention_weight_max": 33.67115020751953,
      "activations/layer10_attention_weight_min": -34.7528076171875,
      "activations/layer11_attention_weight_max": 34.24994659423828,
      "activations/layer11_attention_weight_min": -35.091278076171875,
      "activations/layer12_attention_weight_max": 27.37388801574707,
      "activations/layer12_attention_weight_min": -27.339990615844727,
      "activations/layer13_attention_weight_max": 38.47184753417969,
      "activations/layer13_attention_weight_min": -37.79447555541992,
      "activations/layer14_attention_weight_max": 42.71988296508789,
      "activations/layer14_attention_weight_min": -41.304725646972656,
      "activations/layer15_attention_weight_max": 37.76472473144531,
      "activations/layer15_attention_weight_min": -37.42267608642578,
      "activations/layer16_attention_weight_max": 31.109195709228516,
      "activations/layer16_attention_weight_min": -29.475191116333008,
      "activations/layer17_attention_weight_max": 28.62848663330078,
      "activations/layer17_attention_weight_min": -28.041160583496094,
      "activations/layer18_attention_weight_max": 32.303653717041016,
      "activations/layer18_attention_weight_min": -29.062522888183594,
      "activations/layer19_attention_weight_max": 33.61095428466797,
      "activations/layer19_attention_weight_min": -35.8026008605957,
      "activations/layer1_attention_weight_max": 15.74524211883545,
      "activations/layer1_attention_weight_min": -15.152886390686035,
      "activations/layer20_attention_weight_max": 29.656902313232422,
      "activations/layer20_attention_weight_min": -26.370586395263672,
      "activations/layer21_attention_weight_max": 30.024198532104492,
      "activations/layer21_attention_weight_min": -25.966394424438477,
      "activations/layer22_attention_weight_max": 40.24886703491211,
      "activations/layer22_attention_weight_min": -33.348873138427734,
      "activations/layer23_attention_weight_max": 30.17696189880371,
      "activations/layer23_attention_weight_min": -26.53089714050293,
      "activations/layer2_attention_weight_max": 33.805545806884766,
      "activations/layer2_attention_weight_min": -34.8896369934082,
      "activations/layer3_attention_weight_max": 107.1686019897461,
      "activations/layer3_attention_weight_min": -111.99111938476562,
      "activations/layer4_attention_weight_max": 111.62994384765625,
      "activations/layer4_attention_weight_min": -116.431396484375,
      "activations/layer5_attention_weight_max": 52.43678283691406,
      "activations/layer5_attention_weight_min": -62.87879943847656,
      "activations/layer6_attention_weight_max": 47.475257873535156,
      "activations/layer6_attention_weight_min": -51.54762649536133,
      "activations/layer7_attention_weight_max": 99.93042755126953,
      "activations/layer7_attention_weight_min": -106.71212005615234,
      "activations/layer8_attention_weight_max": 42.77128601074219,
      "activations/layer8_attention_weight_min": -47.59206008911133,
      "activations/layer9_attention_weight_max": 31.907907485961914,
      "activations/layer9_attention_weight_min": -35.85963821411133,
      "epoch": 22.7,
      "learning_rate": 3.637121212121212e-06,
      "loss": 2.6869,
      "step": 390700
    },
    {
      "activations/layer0_attention_weight_max": 19.205333709716797,
      "activations/layer0_attention_weight_min": -15.33212661743164,
      "activations/layer10_attention_weight_max": 39.444488525390625,
      "activations/layer10_attention_weight_min": -37.655181884765625,
      "activations/layer11_attention_weight_max": 38.57096862792969,
      "activations/layer11_attention_weight_min": -35.904563903808594,
      "activations/layer12_attention_weight_max": 29.71031379699707,
      "activations/layer12_attention_weight_min": -29.348791122436523,
      "activations/layer13_attention_weight_max": 44.228843688964844,
      "activations/layer13_attention_weight_min": -42.591339111328125,
      "activations/layer14_attention_weight_max": 45.8011360168457,
      "activations/layer14_attention_weight_min": -41.89043426513672,
      "activations/layer15_attention_weight_max": 42.99235153198242,
      "activations/layer15_attention_weight_min": -38.39535903930664,
      "activations/layer16_attention_weight_max": 32.67240524291992,
      "activations/layer16_attention_weight_min": -28.71214485168457,
      "activations/layer17_attention_weight_max": 33.149169921875,
      "activations/layer17_attention_weight_min": -26.914087295532227,
      "activations/layer18_attention_weight_max": 35.910037994384766,
      "activations/layer18_attention_weight_min": -28.854557037353516,
      "activations/layer19_attention_weight_max": 39.427085876464844,
      "activations/layer19_attention_weight_min": -35.99346160888672,
      "activations/layer1_attention_weight_max": 17.437040328979492,
      "activations/layer1_attention_weight_min": -16.13983726501465,
      "activations/layer20_attention_weight_max": 31.959917068481445,
      "activations/layer20_attention_weight_min": -29.698680877685547,
      "activations/layer21_attention_weight_max": 33.96458435058594,
      "activations/layer21_attention_weight_min": -29.10666275024414,
      "activations/layer22_attention_weight_max": 43.319908142089844,
      "activations/layer22_attention_weight_min": -35.653011322021484,
      "activations/layer23_attention_weight_max": 34.57416534423828,
      "activations/layer23_attention_weight_min": -28.145328521728516,
      "activations/layer2_attention_weight_max": 34.86540603637695,
      "activations/layer2_attention_weight_min": -33.538841247558594,
      "activations/layer3_attention_weight_max": 105.29744720458984,
      "activations/layer3_attention_weight_min": -107.16790771484375,
      "activations/layer4_attention_weight_max": 110.93355560302734,
      "activations/layer4_attention_weight_min": -114.6576156616211,
      "activations/layer5_attention_weight_max": 51.73149871826172,
      "activations/layer5_attention_weight_min": -66.29528045654297,
      "activations/layer6_attention_weight_max": 46.29255676269531,
      "activations/layer6_attention_weight_min": -48.28370666503906,
      "activations/layer7_attention_weight_max": 104.79425048828125,
      "activations/layer7_attention_weight_min": -107.23650360107422,
      "activations/layer8_attention_weight_max": 45.77121353149414,
      "activations/layer8_attention_weight_min": -49.166324615478516,
      "activations/layer9_attention_weight_max": 33.06370162963867,
      "activations/layer9_attention_weight_min": -36.90968322753906,
      "epoch": 22.7,
      "learning_rate": 3.6185606060606056e-06,
      "loss": 2.6992,
      "step": 390750
    },
    {
      "activations/layer0_attention_weight_max": 16.076251983642578,
      "activations/layer0_attention_weight_min": -14.039414405822754,
      "activations/layer10_attention_weight_max": 35.164066314697266,
      "activations/layer10_attention_weight_min": -35.347564697265625,
      "activations/layer11_attention_weight_max": 34.56165313720703,
      "activations/layer11_attention_weight_min": -36.03579330444336,
      "activations/layer12_attention_weight_max": 27.120195388793945,
      "activations/layer12_attention_weight_min": -30.26787757873535,
      "activations/layer13_attention_weight_max": 40.3757438659668,
      "activations/layer13_attention_weight_min": -36.65779113769531,
      "activations/layer14_attention_weight_max": 43.35639190673828,
      "activations/layer14_attention_weight_min": -37.684722900390625,
      "activations/layer15_attention_weight_max": 38.892173767089844,
      "activations/layer15_attention_weight_min": -38.547035217285156,
      "activations/layer16_attention_weight_max": 28.627138137817383,
      "activations/layer16_attention_weight_min": -28.069087982177734,
      "activations/layer17_attention_weight_max": 27.301616668701172,
      "activations/layer17_attention_weight_min": -25.56557273864746,
      "activations/layer18_attention_weight_max": 30.84813117980957,
      "activations/layer18_attention_weight_min": -25.46031951904297,
      "activations/layer19_attention_weight_max": 33.580116271972656,
      "activations/layer19_attention_weight_min": -33.31287384033203,
      "activations/layer1_attention_weight_max": 16.44573402404785,
      "activations/layer1_attention_weight_min": -15.60409927368164,
      "activations/layer20_attention_weight_max": 28.323068618774414,
      "activations/layer20_attention_weight_min": -24.493417739868164,
      "activations/layer21_attention_weight_max": 26.40214729309082,
      "activations/layer21_attention_weight_min": -23.388425827026367,
      "activations/layer22_attention_weight_max": 35.82796859741211,
      "activations/layer22_attention_weight_min": -30.69440269470215,
      "activations/layer23_attention_weight_max": 29.761154174804688,
      "activations/layer23_attention_weight_min": -28.399267196655273,
      "activations/layer2_attention_weight_max": 33.5009651184082,
      "activations/layer2_attention_weight_min": -34.641292572021484,
      "activations/layer3_attention_weight_max": 109.13684844970703,
      "activations/layer3_attention_weight_min": -110.3884048461914,
      "activations/layer4_attention_weight_max": 112.78304290771484,
      "activations/layer4_attention_weight_min": -114.1935043334961,
      "activations/layer5_attention_weight_max": 51.390830993652344,
      "activations/layer5_attention_weight_min": -64.40834045410156,
      "activations/layer6_attention_weight_max": 45.01506042480469,
      "activations/layer6_attention_weight_min": -49.00749206542969,
      "activations/layer7_attention_weight_max": 95.44554138183594,
      "activations/layer7_attention_weight_min": -102.55001831054688,
      "activations/layer8_attention_weight_max": 43.7654914855957,
      "activations/layer8_attention_weight_min": -46.59641647338867,
      "activations/layer9_attention_weight_max": 35.74195098876953,
      "activations/layer9_attention_weight_min": -34.634315490722656,
      "epoch": 22.71,
      "learning_rate": 3.5996212121212118e-06,
      "loss": 2.6971,
      "step": 390800
    },
    {
      "activations/layer0_attention_weight_max": 17.608095169067383,
      "activations/layer0_attention_weight_min": -13.082621574401855,
      "activations/layer10_attention_weight_max": 33.99276351928711,
      "activations/layer10_attention_weight_min": -35.328495025634766,
      "activations/layer11_attention_weight_max": 34.19060134887695,
      "activations/layer11_attention_weight_min": -35.60808563232422,
      "activations/layer12_attention_weight_max": 28.949678421020508,
      "activations/layer12_attention_weight_min": -27.133668899536133,
      "activations/layer13_attention_weight_max": 41.53855895996094,
      "activations/layer13_attention_weight_min": -36.5026741027832,
      "activations/layer14_attention_weight_max": 42.33665084838867,
      "activations/layer14_attention_weight_min": -38.40448760986328,
      "activations/layer15_attention_weight_max": 38.22175598144531,
      "activations/layer15_attention_weight_min": -38.61669921875,
      "activations/layer16_attention_weight_max": 28.072223663330078,
      "activations/layer16_attention_weight_min": -29.414249420166016,
      "activations/layer17_attention_weight_max": 28.76424789428711,
      "activations/layer17_attention_weight_min": -26.184612274169922,
      "activations/layer18_attention_weight_max": 32.01613235473633,
      "activations/layer18_attention_weight_min": -26.803911209106445,
      "activations/layer19_attention_weight_max": 33.623802185058594,
      "activations/layer19_attention_weight_min": -31.17903709411621,
      "activations/layer1_attention_weight_max": 15.58761215209961,
      "activations/layer1_attention_weight_min": -16.627294540405273,
      "activations/layer20_attention_weight_max": 28.302305221557617,
      "activations/layer20_attention_weight_min": -24.765295028686523,
      "activations/layer21_attention_weight_max": 29.50314712524414,
      "activations/layer21_attention_weight_min": -24.76459312438965,
      "activations/layer22_attention_weight_max": 44.995155334472656,
      "activations/layer22_attention_weight_min": -31.673603057861328,
      "activations/layer23_attention_weight_max": 30.73639678955078,
      "activations/layer23_attention_weight_min": -26.111072540283203,
      "activations/layer2_attention_weight_max": 34.2796745300293,
      "activations/layer2_attention_weight_min": -33.46927261352539,
      "activations/layer3_attention_weight_max": 106.65518188476562,
      "activations/layer3_attention_weight_min": -108.61003875732422,
      "activations/layer4_attention_weight_max": 112.20670318603516,
      "activations/layer4_attention_weight_min": -112.7415771484375,
      "activations/layer5_attention_weight_max": 49.96372985839844,
      "activations/layer5_attention_weight_min": -61.34880828857422,
      "activations/layer6_attention_weight_max": 45.978248596191406,
      "activations/layer6_attention_weight_min": -48.039024353027344,
      "activations/layer7_attention_weight_max": 97.4004898071289,
      "activations/layer7_attention_weight_min": -102.55517578125,
      "activations/layer8_attention_weight_max": 42.825111389160156,
      "activations/layer8_attention_weight_min": -49.33936309814453,
      "activations/layer9_attention_weight_max": 32.21905517578125,
      "activations/layer9_attention_weight_min": -34.32952880859375,
      "epoch": 22.71,
      "learning_rate": 3.5806818181818175e-06,
      "loss": 2.6871,
      "step": 390850
    },
    {
      "activations/layer0_attention_weight_max": 17.158727645874023,
      "activations/layer0_attention_weight_min": -15.687347412109375,
      "activations/layer10_attention_weight_max": 36.01800537109375,
      "activations/layer10_attention_weight_min": -35.718536376953125,
      "activations/layer11_attention_weight_max": 35.27910232543945,
      "activations/layer11_attention_weight_min": -35.6672248840332,
      "activations/layer12_attention_weight_max": 27.159032821655273,
      "activations/layer12_attention_weight_min": -29.117576599121094,
      "activations/layer13_attention_weight_max": 39.7592658996582,
      "activations/layer13_attention_weight_min": -39.083229064941406,
      "activations/layer14_attention_weight_max": 44.94743347167969,
      "activations/layer14_attention_weight_min": -42.81666946411133,
      "activations/layer15_attention_weight_max": 39.47658920288086,
      "activations/layer15_attention_weight_min": -38.67474365234375,
      "activations/layer16_attention_weight_max": 29.836040496826172,
      "activations/layer16_attention_weight_min": -29.95464324951172,
      "activations/layer17_attention_weight_max": 30.84130859375,
      "activations/layer17_attention_weight_min": -26.876008987426758,
      "activations/layer18_attention_weight_max": 34.3297233581543,
      "activations/layer18_attention_weight_min": -28.0142822265625,
      "activations/layer19_attention_weight_max": 35.676395416259766,
      "activations/layer19_attention_weight_min": -32.454673767089844,
      "activations/layer1_attention_weight_max": 16.19145965576172,
      "activations/layer1_attention_weight_min": -16.987266540527344,
      "activations/layer20_attention_weight_max": 32.10015106201172,
      "activations/layer20_attention_weight_min": -26.413990020751953,
      "activations/layer21_attention_weight_max": 29.713247299194336,
      "activations/layer21_attention_weight_min": -25.543155670166016,
      "activations/layer22_attention_weight_max": 41.43921661376953,
      "activations/layer22_attention_weight_min": -33.67292022705078,
      "activations/layer23_attention_weight_max": 32.85374069213867,
      "activations/layer23_attention_weight_min": -26.621070861816406,
      "activations/layer2_attention_weight_max": 34.4670524597168,
      "activations/layer2_attention_weight_min": -33.85594177246094,
      "activations/layer3_attention_weight_max": 109.81141662597656,
      "activations/layer3_attention_weight_min": -107.64633178710938,
      "activations/layer4_attention_weight_max": 112.3119888305664,
      "activations/layer4_attention_weight_min": -112.09740447998047,
      "activations/layer5_attention_weight_max": 50.60610580444336,
      "activations/layer5_attention_weight_min": -63.38800811767578,
      "activations/layer6_attention_weight_max": 45.34823989868164,
      "activations/layer6_attention_weight_min": -48.38401412963867,
      "activations/layer7_attention_weight_max": 98.99060821533203,
      "activations/layer7_attention_weight_min": -99.88694763183594,
      "activations/layer8_attention_weight_max": 44.65237808227539,
      "activations/layer8_attention_weight_min": -43.76802444458008,
      "activations/layer9_attention_weight_max": 32.554866790771484,
      "activations/layer9_attention_weight_min": -35.55900573730469,
      "epoch": 22.71,
      "learning_rate": 3.561742424242424e-06,
      "loss": 2.6899,
      "step": 390900
    },
    {
      "activations/layer0_attention_weight_max": 17.721435546875,
      "activations/layer0_attention_weight_min": -13.301881790161133,
      "activations/layer10_attention_weight_max": 38.470767974853516,
      "activations/layer10_attention_weight_min": -38.66849136352539,
      "activations/layer11_attention_weight_max": 37.188045501708984,
      "activations/layer11_attention_weight_min": -38.464534759521484,
      "activations/layer12_attention_weight_max": 28.516199111938477,
      "activations/layer12_attention_weight_min": -31.238550186157227,
      "activations/layer13_attention_weight_max": 43.30204772949219,
      "activations/layer13_attention_weight_min": -41.063392639160156,
      "activations/layer14_attention_weight_max": 48.165069580078125,
      "activations/layer14_attention_weight_min": -45.73086929321289,
      "activations/layer15_attention_weight_max": 42.94813537597656,
      "activations/layer15_attention_weight_min": -39.20088577270508,
      "activations/layer16_attention_weight_max": 31.60681915283203,
      "activations/layer16_attention_weight_min": -28.6853084564209,
      "activations/layer17_attention_weight_max": 31.689680099487305,
      "activations/layer17_attention_weight_min": -29.076494216918945,
      "activations/layer18_attention_weight_max": 36.36345291137695,
      "activations/layer18_attention_weight_min": -29.45399284362793,
      "activations/layer19_attention_weight_max": 39.447723388671875,
      "activations/layer19_attention_weight_min": -33.37285232543945,
      "activations/layer1_attention_weight_max": 16.080368041992188,
      "activations/layer1_attention_weight_min": -15.517104148864746,
      "activations/layer20_attention_weight_max": 32.969112396240234,
      "activations/layer20_attention_weight_min": -25.898197174072266,
      "activations/layer21_attention_weight_max": 33.58306884765625,
      "activations/layer21_attention_weight_min": -26.03660774230957,
      "activations/layer22_attention_weight_max": 56.4252815246582,
      "activations/layer22_attention_weight_min": -36.785587310791016,
      "activations/layer23_attention_weight_max": 40.34569549560547,
      "activations/layer23_attention_weight_min": -25.405147552490234,
      "activations/layer2_attention_weight_max": 34.523651123046875,
      "activations/layer2_attention_weight_min": -33.88372802734375,
      "activations/layer3_attention_weight_max": 111.24569702148438,
      "activations/layer3_attention_weight_min": -113.64883422851562,
      "activations/layer4_attention_weight_max": 114.3194351196289,
      "activations/layer4_attention_weight_min": -118.51262664794922,
      "activations/layer5_attention_weight_max": 54.04267501831055,
      "activations/layer5_attention_weight_min": -67.88103485107422,
      "activations/layer6_attention_weight_max": 52.120426177978516,
      "activations/layer6_attention_weight_min": -54.33857345581055,
      "activations/layer7_attention_weight_max": 108.86944580078125,
      "activations/layer7_attention_weight_min": -119.19190979003906,
      "activations/layer8_attention_weight_max": 50.15028762817383,
      "activations/layer8_attention_weight_min": -52.48963165283203,
      "activations/layer9_attention_weight_max": 36.3336296081543,
      "activations/layer9_attention_weight_min": -39.65835189819336,
      "epoch": 22.72,
      "learning_rate": 3.54280303030303e-06,
      "loss": 2.7025,
      "step": 390950
    },
    {
      "activations/layer0_attention_weight_max": 16.8919620513916,
      "activations/layer0_attention_weight_min": -15.277824401855469,
      "activations/layer10_attention_weight_max": 35.494869232177734,
      "activations/layer10_attention_weight_min": -36.67476272583008,
      "activations/layer11_attention_weight_max": 34.00291442871094,
      "activations/layer11_attention_weight_min": -36.767112731933594,
      "activations/layer12_attention_weight_max": 25.826025009155273,
      "activations/layer12_attention_weight_min": -28.998825073242188,
      "activations/layer13_attention_weight_max": 38.29215621948242,
      "activations/layer13_attention_weight_min": -36.69880676269531,
      "activations/layer14_attention_weight_max": 42.992374420166016,
      "activations/layer14_attention_weight_min": -39.91743850708008,
      "activations/layer15_attention_weight_max": 38.69940948486328,
      "activations/layer15_attention_weight_min": -37.75135803222656,
      "activations/layer16_attention_weight_max": 27.63629913330078,
      "activations/layer16_attention_weight_min": -28.53024673461914,
      "activations/layer17_attention_weight_max": 28.168724060058594,
      "activations/layer17_attention_weight_min": -27.009775161743164,
      "activations/layer18_attention_weight_max": 34.620628356933594,
      "activations/layer18_attention_weight_min": -25.18108367919922,
      "activations/layer19_attention_weight_max": 31.166723251342773,
      "activations/layer19_attention_weight_min": -33.34634780883789,
      "activations/layer1_attention_weight_max": 15.350281715393066,
      "activations/layer1_attention_weight_min": -14.859597206115723,
      "activations/layer20_attention_weight_max": 28.1375789642334,
      "activations/layer20_attention_weight_min": -25.251262664794922,
      "activations/layer21_attention_weight_max": 28.055374145507812,
      "activations/layer21_attention_weight_min": -23.958219528198242,
      "activations/layer22_attention_weight_max": 44.365779876708984,
      "activations/layer22_attention_weight_min": -30.50179672241211,
      "activations/layer23_attention_weight_max": 28.74421501159668,
      "activations/layer23_attention_weight_min": -25.560237884521484,
      "activations/layer2_attention_weight_max": 34.5951042175293,
      "activations/layer2_attention_weight_min": -32.87303924560547,
      "activations/layer3_attention_weight_max": 109.34964752197266,
      "activations/layer3_attention_weight_min": -108.33253479003906,
      "activations/layer4_attention_weight_max": 113.25850677490234,
      "activations/layer4_attention_weight_min": -116.82795715332031,
      "activations/layer5_attention_weight_max": 50.55975341796875,
      "activations/layer5_attention_weight_min": -63.11205291748047,
      "activations/layer6_attention_weight_max": 48.03022003173828,
      "activations/layer6_attention_weight_min": -51.36766815185547,
      "activations/layer7_attention_weight_max": 100.26616668701172,
      "activations/layer7_attention_weight_min": -108.88984680175781,
      "activations/layer8_attention_weight_max": 44.126739501953125,
      "activations/layer8_attention_weight_min": -48.15044021606445,
      "activations/layer9_attention_weight_max": 33.817073822021484,
      "activations/layer9_attention_weight_min": -37.2595100402832,
      "epoch": 22.72,
      "learning_rate": 3.523863636363636e-06,
      "loss": 2.7151,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.2482,
      "eval_samples_per_second": 520.599,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.2482,
      "eval_openwebtext_samples_per_second": 520.599,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9485,
      "eval_wikitext_samples_per_second": 234.029,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_lambada_loss": 2.33203125,
      "eval_lambada_ppl": 10.298839819419056,
      "eval_lambada_runtime": 9.5512,
      "eval_lambada_samples_per_second": 509.781,
      "step": 391000
    },
    {
      "activations/layer0_attention_weight_max": 17.145048141479492,
      "activations/layer0_attention_weight_min": -15.732908248901367,
      "activations/layer10_attention_weight_max": 36.577938079833984,
      "activations/layer10_attention_weight_min": -36.484642028808594,
      "activations/layer11_attention_weight_max": 36.79833221435547,
      "activations/layer11_attention_weight_min": -36.395503997802734,
      "activations/layer12_attention_weight_max": 29.267663955688477,
      "activations/layer12_attention_weight_min": -29.332841873168945,
      "activations/layer13_attention_weight_max": 43.12016677856445,
      "activations/layer13_attention_weight_min": -39.51155471801758,
      "activations/layer14_attention_weight_max": 45.5230598449707,
      "activations/layer14_attention_weight_min": -45.85004425048828,
      "activations/layer15_attention_weight_max": 43.53183364868164,
      "activations/layer15_attention_weight_min": -40.97762680053711,
      "activations/layer16_attention_weight_max": 31.77928924560547,
      "activations/layer16_attention_weight_min": -32.14043045043945,
      "activations/layer17_attention_weight_max": 32.08828353881836,
      "activations/layer17_attention_weight_min": -29.164419174194336,
      "activations/layer18_attention_weight_max": 35.50425720214844,
      "activations/layer18_attention_weight_min": -28.90020751953125,
      "activations/layer19_attention_weight_max": 37.3211784362793,
      "activations/layer19_attention_weight_min": -35.8773193359375,
      "activations/layer1_attention_weight_max": 16.262348175048828,
      "activations/layer1_attention_weight_min": -15.125341415405273,
      "activations/layer20_attention_weight_max": 30.291519165039062,
      "activations/layer20_attention_weight_min": -28.324907302856445,
      "activations/layer21_attention_weight_max": 32.1093635559082,
      "activations/layer21_attention_weight_min": -27.63591194152832,
      "activations/layer22_attention_weight_max": 44.64153289794922,
      "activations/layer22_attention_weight_min": -34.10007095336914,
      "activations/layer23_attention_weight_max": 31.807823181152344,
      "activations/layer23_attention_weight_min": -27.198333740234375,
      "activations/layer2_attention_weight_max": 36.13548278808594,
      "activations/layer2_attention_weight_min": -34.4959716796875,
      "activations/layer3_attention_weight_max": 110.72792053222656,
      "activations/layer3_attention_weight_min": -111.90723419189453,
      "activations/layer4_attention_weight_max": 118.27275848388672,
      "activations/layer4_attention_weight_min": -118.49288177490234,
      "activations/layer5_attention_weight_max": 51.705589294433594,
      "activations/layer5_attention_weight_min": -64.77520751953125,
      "activations/layer6_attention_weight_max": 48.66635513305664,
      "activations/layer6_attention_weight_min": -49.55109405517578,
      "activations/layer7_attention_weight_max": 101.68724060058594,
      "activations/layer7_attention_weight_min": -102.84663391113281,
      "activations/layer8_attention_weight_max": 44.944053649902344,
      "activations/layer8_attention_weight_min": -47.9595832824707,
      "activations/layer9_attention_weight_max": 34.745418548583984,
      "activations/layer9_attention_weight_min": -37.35332489013672,
      "epoch": 22.72,
      "learning_rate": 3.5049242424242417e-06,
      "loss": 2.6965,
      "step": 391050
    },
    {
      "activations/layer0_attention_weight_max": 17.411821365356445,
      "activations/layer0_attention_weight_min": -13.97192668914795,
      "activations/layer10_attention_weight_max": 37.46617126464844,
      "activations/layer10_attention_weight_min": -37.87659454345703,
      "activations/layer11_attention_weight_max": 37.492515563964844,
      "activations/layer11_attention_weight_min": -39.80188751220703,
      "activations/layer12_attention_weight_max": 26.69285774230957,
      "activations/layer12_attention_weight_min": -27.76500129699707,
      "activations/layer13_attention_weight_max": 40.943885803222656,
      "activations/layer13_attention_weight_min": -36.13515853881836,
      "activations/layer14_attention_weight_max": 43.89714431762695,
      "activations/layer14_attention_weight_min": -40.544193267822266,
      "activations/layer15_attention_weight_max": 40.16572952270508,
      "activations/layer15_attention_weight_min": -36.8619384765625,
      "activations/layer16_attention_weight_max": 29.96946144104004,
      "activations/layer16_attention_weight_min": -29.986234664916992,
      "activations/layer17_attention_weight_max": 29.396005630493164,
      "activations/layer17_attention_weight_min": -27.25168800354004,
      "activations/layer18_attention_weight_max": 31.16248893737793,
      "activations/layer18_attention_weight_min": -28.101533889770508,
      "activations/layer19_attention_weight_max": 32.7197380065918,
      "activations/layer19_attention_weight_min": -32.49355697631836,
      "activations/layer1_attention_weight_max": 15.94225788116455,
      "activations/layer1_attention_weight_min": -15.958662986755371,
      "activations/layer20_attention_weight_max": 27.704072952270508,
      "activations/layer20_attention_weight_min": -26.176301956176758,
      "activations/layer21_attention_weight_max": 26.456619262695312,
      "activations/layer21_attention_weight_min": -24.642101287841797,
      "activations/layer22_attention_weight_max": 37.327510833740234,
      "activations/layer22_attention_weight_min": -31.41535758972168,
      "activations/layer23_attention_weight_max": 32.816162109375,
      "activations/layer23_attention_weight_min": -26.767620086669922,
      "activations/layer2_attention_weight_max": 36.17066192626953,
      "activations/layer2_attention_weight_min": -36.18331527709961,
      "activations/layer3_attention_weight_max": 112.40965270996094,
      "activations/layer3_attention_weight_min": -117.16877746582031,
      "activations/layer4_attention_weight_max": 111.47883605957031,
      "activations/layer4_attention_weight_min": -111.05290985107422,
      "activations/layer5_attention_weight_max": 51.63917541503906,
      "activations/layer5_attention_weight_min": -59.33812713623047,
      "activations/layer6_attention_weight_max": 44.67517852783203,
      "activations/layer6_attention_weight_min": -48.08420944213867,
      "activations/layer7_attention_weight_max": 96.63764190673828,
      "activations/layer7_attention_weight_min": -102.56700134277344,
      "activations/layer8_attention_weight_max": 45.10450744628906,
      "activations/layer8_attention_weight_min": -48.09247970581055,
      "activations/layer9_attention_weight_max": 34.93415832519531,
      "activations/layer9_attention_weight_min": -38.447872161865234,
      "epoch": 22.73,
      "learning_rate": 3.4859848484848483e-06,
      "loss": 2.7124,
      "step": 391100
    },
    {
      "activations/layer0_attention_weight_max": 17.194461822509766,
      "activations/layer0_attention_weight_min": -15.025012969970703,
      "activations/layer10_attention_weight_max": 34.72377395629883,
      "activations/layer10_attention_weight_min": -34.466949462890625,
      "activations/layer11_attention_weight_max": 34.31743621826172,
      "activations/layer11_attention_weight_min": -35.10621643066406,
      "activations/layer12_attention_weight_max": 27.446054458618164,
      "activations/layer12_attention_weight_min": -26.49022102355957,
      "activations/layer13_attention_weight_max": 45.753395080566406,
      "activations/layer13_attention_weight_min": -36.78401565551758,
      "activations/layer14_attention_weight_max": 45.803138732910156,
      "activations/layer14_attention_weight_min": -39.583255767822266,
      "activations/layer15_attention_weight_max": 40.892696380615234,
      "activations/layer15_attention_weight_min": -36.30177307128906,
      "activations/layer16_attention_weight_max": 29.427248001098633,
      "activations/layer16_attention_weight_min": -29.95022201538086,
      "activations/layer17_attention_weight_max": 29.27773666381836,
      "activations/layer17_attention_weight_min": -26.081390380859375,
      "activations/layer18_attention_weight_max": 35.21738052368164,
      "activations/layer18_attention_weight_min": -27.86971664428711,
      "activations/layer19_attention_weight_max": 33.81878662109375,
      "activations/layer19_attention_weight_min": -31.66582679748535,
      "activations/layer1_attention_weight_max": 16.679418563842773,
      "activations/layer1_attention_weight_min": -16.370773315429688,
      "activations/layer20_attention_weight_max": 28.284090042114258,
      "activations/layer20_attention_weight_min": -25.34117317199707,
      "activations/layer21_attention_weight_max": 29.982877731323242,
      "activations/layer21_attention_weight_min": -23.00982666015625,
      "activations/layer22_attention_weight_max": 41.6319465637207,
      "activations/layer22_attention_weight_min": -30.5600643157959,
      "activations/layer23_attention_weight_max": 32.96986770629883,
      "activations/layer23_attention_weight_min": -24.967693328857422,
      "activations/layer2_attention_weight_max": 34.97035217285156,
      "activations/layer2_attention_weight_min": -32.94973373413086,
      "activations/layer3_attention_weight_max": 110.30693054199219,
      "activations/layer3_attention_weight_min": -108.11139678955078,
      "activations/layer4_attention_weight_max": 115.26439666748047,
      "activations/layer4_attention_weight_min": -115.59691619873047,
      "activations/layer5_attention_weight_max": 53.63349914550781,
      "activations/layer5_attention_weight_min": -63.469478607177734,
      "activations/layer6_attention_weight_max": 47.06163024902344,
      "activations/layer6_attention_weight_min": -50.12437438964844,
      "activations/layer7_attention_weight_max": 101.78392028808594,
      "activations/layer7_attention_weight_min": -108.04795837402344,
      "activations/layer8_attention_weight_max": 43.53295135498047,
      "activations/layer8_attention_weight_min": -46.24162673950195,
      "activations/layer9_attention_weight_max": 33.20133590698242,
      "activations/layer9_attention_weight_min": -34.6192626953125,
      "epoch": 22.73,
      "learning_rate": 3.4670454545454544e-06,
      "loss": 2.714,
      "step": 391150
    },
    {
      "activations/layer0_attention_weight_max": 17.13644790649414,
      "activations/layer0_attention_weight_min": -14.970307350158691,
      "activations/layer10_attention_weight_max": 48.64702224731445,
      "activations/layer10_attention_weight_min": -39.62250900268555,
      "activations/layer11_attention_weight_max": 43.70928955078125,
      "activations/layer11_attention_weight_min": -39.80788803100586,
      "activations/layer12_attention_weight_max": 28.180442810058594,
      "activations/layer12_attention_weight_min": -27.446643829345703,
      "activations/layer13_attention_weight_max": 44.57027816772461,
      "activations/layer13_attention_weight_min": -37.48241424560547,
      "activations/layer14_attention_weight_max": 45.112586975097656,
      "activations/layer14_attention_weight_min": -41.53118133544922,
      "activations/layer15_attention_weight_max": 49.17134094238281,
      "activations/layer15_attention_weight_min": -41.610015869140625,
      "activations/layer16_attention_weight_max": 34.6705322265625,
      "activations/layer16_attention_weight_min": -29.85852813720703,
      "activations/layer17_attention_weight_max": 29.174373626708984,
      "activations/layer17_attention_weight_min": -25.745393753051758,
      "activations/layer18_attention_weight_max": 29.89739418029785,
      "activations/layer18_attention_weight_min": -25.72675132751465,
      "activations/layer19_attention_weight_max": 30.09208106994629,
      "activations/layer19_attention_weight_min": -32.675865173339844,
      "activations/layer1_attention_weight_max": 15.703559875488281,
      "activations/layer1_attention_weight_min": -15.939197540283203,
      "activations/layer20_attention_weight_max": 27.831214904785156,
      "activations/layer20_attention_weight_min": -24.48895835876465,
      "activations/layer21_attention_weight_max": 26.660268783569336,
      "activations/layer21_attention_weight_min": -24.565505981445312,
      "activations/layer22_attention_weight_max": 36.72724914550781,
      "activations/layer22_attention_weight_min": -30.38214111328125,
      "activations/layer23_attention_weight_max": 30.52444076538086,
      "activations/layer23_attention_weight_min": -25.013431549072266,
      "activations/layer2_attention_weight_max": 35.075775146484375,
      "activations/layer2_attention_weight_min": -34.531097412109375,
      "activations/layer3_attention_weight_max": 112.6829833984375,
      "activations/layer3_attention_weight_min": -114.78822326660156,
      "activations/layer4_attention_weight_max": 116.38838195800781,
      "activations/layer4_attention_weight_min": -112.926025390625,
      "activations/layer5_attention_weight_max": 55.095272064208984,
      "activations/layer5_attention_weight_min": -60.526344299316406,
      "activations/layer6_attention_weight_max": 45.83200454711914,
      "activations/layer6_attention_weight_min": -48.12986755371094,
      "activations/layer7_attention_weight_max": 111.93655395507812,
      "activations/layer7_attention_weight_min": -107.36073303222656,
      "activations/layer8_attention_weight_max": 49.52139663696289,
      "activations/layer8_attention_weight_min": -48.55928039550781,
      "activations/layer9_attention_weight_max": 38.3216667175293,
      "activations/layer9_attention_weight_min": -39.0889892578125,
      "epoch": 22.73,
      "learning_rate": 3.44810606060606e-06,
      "loss": 2.7033,
      "step": 391200
    },
    {
      "activations/layer0_attention_weight_max": 16.939245223999023,
      "activations/layer0_attention_weight_min": -14.73508358001709,
      "activations/layer10_attention_weight_max": 37.2015266418457,
      "activations/layer10_attention_weight_min": -37.80995178222656,
      "activations/layer11_attention_weight_max": 36.04991912841797,
      "activations/layer11_attention_weight_min": -37.75838088989258,
      "activations/layer12_attention_weight_max": 27.69353675842285,
      "activations/layer12_attention_weight_min": -29.02465057373047,
      "activations/layer13_attention_weight_max": 42.261024475097656,
      "activations/layer13_attention_weight_min": -37.52979278564453,
      "activations/layer14_attention_weight_max": 42.93922424316406,
      "activations/layer14_attention_weight_min": -41.22319793701172,
      "activations/layer15_attention_weight_max": 40.39542770385742,
      "activations/layer15_attention_weight_min": -38.24008560180664,
      "activations/layer16_attention_weight_max": 29.29933738708496,
      "activations/layer16_attention_weight_min": -31.740650177001953,
      "activations/layer17_attention_weight_max": 31.14394760131836,
      "activations/layer17_attention_weight_min": -27.41211700439453,
      "activations/layer18_attention_weight_max": 32.8299674987793,
      "activations/layer18_attention_weight_min": -27.370525360107422,
      "activations/layer19_attention_weight_max": 37.50619125366211,
      "activations/layer19_attention_weight_min": -32.726314544677734,
      "activations/layer1_attention_weight_max": 15.932016372680664,
      "activations/layer1_attention_weight_min": -15.031075477600098,
      "activations/layer20_attention_weight_max": 29.28099250793457,
      "activations/layer20_attention_weight_min": -25.52288818359375,
      "activations/layer21_attention_weight_max": 30.901090621948242,
      "activations/layer21_attention_weight_min": -25.435697555541992,
      "activations/layer22_attention_weight_max": 49.364933013916016,
      "activations/layer22_attention_weight_min": -35.36708450317383,
      "activations/layer23_attention_weight_max": 35.41154479980469,
      "activations/layer23_attention_weight_min": -26.16822052001953,
      "activations/layer2_attention_weight_max": 34.65534591674805,
      "activations/layer2_attention_weight_min": -34.09313201904297,
      "activations/layer3_attention_weight_max": 111.41221618652344,
      "activations/layer3_attention_weight_min": -109.88328552246094,
      "activations/layer4_attention_weight_max": 117.71537017822266,
      "activations/layer4_attention_weight_min": -114.41096496582031,
      "activations/layer5_attention_weight_max": 51.31420135498047,
      "activations/layer5_attention_weight_min": -61.77096176147461,
      "activations/layer6_attention_weight_max": 47.42720031738281,
      "activations/layer6_attention_weight_min": -51.699684143066406,
      "activations/layer7_attention_weight_max": 106.31669616699219,
      "activations/layer7_attention_weight_min": -108.25873565673828,
      "activations/layer8_attention_weight_max": 46.03628158569336,
      "activations/layer8_attention_weight_min": -46.40288543701172,
      "activations/layer9_attention_weight_max": 34.70100021362305,
      "activations/layer9_attention_weight_min": -36.26437759399414,
      "epoch": 22.73,
      "learning_rate": 3.4291666666666663e-06,
      "loss": 2.684,
      "step": 391250
    },
    {
      "activations/layer0_attention_weight_max": 16.883180618286133,
      "activations/layer0_attention_weight_min": -16.00234603881836,
      "activations/layer10_attention_weight_max": 36.768775939941406,
      "activations/layer10_attention_weight_min": -36.532379150390625,
      "activations/layer11_attention_weight_max": 36.7856330871582,
      "activations/layer11_attention_weight_min": -35.51321029663086,
      "activations/layer12_attention_weight_max": 31.346717834472656,
      "activations/layer12_attention_weight_min": -27.915571212768555,
      "activations/layer13_attention_weight_max": 44.60759735107422,
      "activations/layer13_attention_weight_min": -37.84829330444336,
      "activations/layer14_attention_weight_max": 44.32754135131836,
      "activations/layer14_attention_weight_min": -40.112937927246094,
      "activations/layer15_attention_weight_max": 42.13965606689453,
      "activations/layer15_attention_weight_min": -40.23430252075195,
      "activations/layer16_attention_weight_max": 29.88649559020996,
      "activations/layer16_attention_weight_min": -29.026004791259766,
      "activations/layer17_attention_weight_max": 28.768991470336914,
      "activations/layer17_attention_weight_min": -26.016876220703125,
      "activations/layer18_attention_weight_max": 35.23995590209961,
      "activations/layer18_attention_weight_min": -26.54302978515625,
      "activations/layer19_attention_weight_max": 32.00728988647461,
      "activations/layer19_attention_weight_min": -30.523778915405273,
      "activations/layer1_attention_weight_max": 17.01122283935547,
      "activations/layer1_attention_weight_min": -15.552691459655762,
      "activations/layer20_attention_weight_max": 26.704294204711914,
      "activations/layer20_attention_weight_min": -25.956552505493164,
      "activations/layer21_attention_weight_max": 27.25381851196289,
      "activations/layer21_attention_weight_min": -24.4941349029541,
      "activations/layer22_attention_weight_max": 41.87394714355469,
      "activations/layer22_attention_weight_min": -31.846866607666016,
      "activations/layer23_attention_weight_max": 32.709693908691406,
      "activations/layer23_attention_weight_min": -25.51142692565918,
      "activations/layer2_attention_weight_max": 33.87408447265625,
      "activations/layer2_attention_weight_min": -35.471256256103516,
      "activations/layer3_attention_weight_max": 106.04593658447266,
      "activations/layer3_attention_weight_min": -109.22461700439453,
      "activations/layer4_attention_weight_max": 112.72161102294922,
      "activations/layer4_attention_weight_min": -115.35904693603516,
      "activations/layer5_attention_weight_max": 51.55516815185547,
      "activations/layer5_attention_weight_min": -61.59077453613281,
      "activations/layer6_attention_weight_max": 46.56948471069336,
      "activations/layer6_attention_weight_min": -50.308658599853516,
      "activations/layer7_attention_weight_max": 99.32707977294922,
      "activations/layer7_attention_weight_min": -106.65281677246094,
      "activations/layer8_attention_weight_max": 44.45722198486328,
      "activations/layer8_attention_weight_min": -46.08798599243164,
      "activations/layer9_attention_weight_max": 33.05141830444336,
      "activations/layer9_attention_weight_min": -35.5505485534668,
      "epoch": 22.74,
      "learning_rate": 3.4102272727272725e-06,
      "loss": 2.7177,
      "step": 391300
    },
    {
      "activations/layer0_attention_weight_max": 17.287961959838867,
      "activations/layer0_attention_weight_min": -13.882079124450684,
      "activations/layer10_attention_weight_max": 35.10698318481445,
      "activations/layer10_attention_weight_min": -37.317474365234375,
      "activations/layer11_attention_weight_max": 34.133544921875,
      "activations/layer11_attention_weight_min": -36.77444839477539,
      "activations/layer12_attention_weight_max": 27.442279815673828,
      "activations/layer12_attention_weight_min": -26.49106788635254,
      "activations/layer13_attention_weight_max": 38.45524597167969,
      "activations/layer13_attention_weight_min": -35.59186935424805,
      "activations/layer14_attention_weight_max": 41.32526397705078,
      "activations/layer14_attention_weight_min": -36.86980438232422,
      "activations/layer15_attention_weight_max": 40.03861618041992,
      "activations/layer15_attention_weight_min": -36.44633102416992,
      "activations/layer16_attention_weight_max": 29.129823684692383,
      "activations/layer16_attention_weight_min": -28.850217819213867,
      "activations/layer17_attention_weight_max": 28.687843322753906,
      "activations/layer17_attention_weight_min": -25.845752716064453,
      "activations/layer18_attention_weight_max": 32.58930206298828,
      "activations/layer18_attention_weight_min": -26.87760353088379,
      "activations/layer19_attention_weight_max": 37.69902038574219,
      "activations/layer19_attention_weight_min": -33.923946380615234,
      "activations/layer1_attention_weight_max": 15.7926025390625,
      "activations/layer1_attention_weight_min": -16.390148162841797,
      "activations/layer20_attention_weight_max": 30.962312698364258,
      "activations/layer20_attention_weight_min": -26.779802322387695,
      "activations/layer21_attention_weight_max": 28.0539608001709,
      "activations/layer21_attention_weight_min": -25.870956420898438,
      "activations/layer22_attention_weight_max": 41.25241470336914,
      "activations/layer22_attention_weight_min": -31.53179931640625,
      "activations/layer23_attention_weight_max": 32.31642150878906,
      "activations/layer23_attention_weight_min": -27.77444839477539,
      "activations/layer2_attention_weight_max": 34.57328796386719,
      "activations/layer2_attention_weight_min": -33.98712921142578,
      "activations/layer3_attention_weight_max": 110.8817367553711,
      "activations/layer3_attention_weight_min": -113.0887451171875,
      "activations/layer4_attention_weight_max": 111.64847564697266,
      "activations/layer4_attention_weight_min": -115.45633697509766,
      "activations/layer5_attention_weight_max": 50.50570297241211,
      "activations/layer5_attention_weight_min": -62.396339416503906,
      "activations/layer6_attention_weight_max": 45.40656280517578,
      "activations/layer6_attention_weight_min": -48.53749465942383,
      "activations/layer7_attention_weight_max": 95.25028228759766,
      "activations/layer7_attention_weight_min": -101.15733337402344,
      "activations/layer8_attention_weight_max": 41.16553497314453,
      "activations/layer8_attention_weight_min": -46.81050491333008,
      "activations/layer9_attention_weight_max": 31.903573989868164,
      "activations/layer9_attention_weight_min": -34.792015075683594,
      "epoch": 22.74,
      "learning_rate": 3.3912878787878786e-06,
      "loss": 2.7062,
      "step": 391350
    },
    {
      "activations/layer0_attention_weight_max": 16.933609008789062,
      "activations/layer0_attention_weight_min": -13.285774230957031,
      "activations/layer10_attention_weight_max": 35.49366760253906,
      "activations/layer10_attention_weight_min": -37.47600555419922,
      "activations/layer11_attention_weight_max": 34.89850997924805,
      "activations/layer11_attention_weight_min": -36.2294921875,
      "activations/layer12_attention_weight_max": 27.105710983276367,
      "activations/layer12_attention_weight_min": -27.540958404541016,
      "activations/layer13_attention_weight_max": 38.77361297607422,
      "activations/layer13_attention_weight_min": -35.983036041259766,
      "activations/layer14_attention_weight_max": 41.89530944824219,
      "activations/layer14_attention_weight_min": -37.328433990478516,
      "activations/layer15_attention_weight_max": 42.37744140625,
      "activations/layer15_attention_weight_min": -35.457359313964844,
      "activations/layer16_attention_weight_max": 30.782291412353516,
      "activations/layer16_attention_weight_min": -28.505908966064453,
      "activations/layer17_attention_weight_max": 29.866533279418945,
      "activations/layer17_attention_weight_min": -25.725473403930664,
      "activations/layer18_attention_weight_max": 33.03083419799805,
      "activations/layer18_attention_weight_min": -27.80146598815918,
      "activations/layer19_attention_weight_max": 35.60847091674805,
      "activations/layer19_attention_weight_min": -32.630699157714844,
      "activations/layer1_attention_weight_max": 16.49392318725586,
      "activations/layer1_attention_weight_min": -15.569907188415527,
      "activations/layer20_attention_weight_max": 30.782703399658203,
      "activations/layer20_attention_weight_min": -28.720043182373047,
      "activations/layer21_attention_weight_max": 28.769725799560547,
      "activations/layer21_attention_weight_min": -26.319244384765625,
      "activations/layer22_attention_weight_max": 42.19085693359375,
      "activations/layer22_attention_weight_min": -34.727630615234375,
      "activations/layer23_attention_weight_max": 31.299665451049805,
      "activations/layer23_attention_weight_min": -27.415786743164062,
      "activations/layer2_attention_weight_max": 34.49126434326172,
      "activations/layer2_attention_weight_min": -34.08830261230469,
      "activations/layer3_attention_weight_max": 110.47747802734375,
      "activations/layer3_attention_weight_min": -109.05198669433594,
      "activations/layer4_attention_weight_max": 115.3629150390625,
      "activations/layer4_attention_weight_min": -116.74981689453125,
      "activations/layer5_attention_weight_max": 53.533050537109375,
      "activations/layer5_attention_weight_min": -63.443382263183594,
      "activations/layer6_attention_weight_max": 48.41048812866211,
      "activations/layer6_attention_weight_min": -50.42536544799805,
      "activations/layer7_attention_weight_max": 100.04381561279297,
      "activations/layer7_attention_weight_min": -104.23320770263672,
      "activations/layer8_attention_weight_max": 44.049713134765625,
      "activations/layer8_attention_weight_min": -47.0945930480957,
      "activations/layer9_attention_weight_max": 33.70806884765625,
      "activations/layer9_attention_weight_min": -36.998634338378906,
      "epoch": 22.74,
      "learning_rate": 3.3723484848484843e-06,
      "loss": 2.6934,
      "step": 391400
    },
    {
      "activations/layer0_attention_weight_max": 17.227022171020508,
      "activations/layer0_attention_weight_min": -14.341443061828613,
      "activations/layer10_attention_weight_max": 36.6192626953125,
      "activations/layer10_attention_weight_min": -36.44453430175781,
      "activations/layer11_attention_weight_max": 35.043487548828125,
      "activations/layer11_attention_weight_min": -36.398738861083984,
      "activations/layer12_attention_weight_max": 27.54987907409668,
      "activations/layer12_attention_weight_min": -27.94219398498535,
      "activations/layer13_attention_weight_max": 38.62174987792969,
      "activations/layer13_attention_weight_min": -37.70801544189453,
      "activations/layer14_attention_weight_max": 43.72809982299805,
      "activations/layer14_attention_weight_min": -39.28929901123047,
      "activations/layer15_attention_weight_max": 40.08693313598633,
      "activations/layer15_attention_weight_min": -38.68398666381836,
      "activations/layer16_attention_weight_max": 31.426061630249023,
      "activations/layer16_attention_weight_min": -27.825960159301758,
      "activations/layer17_attention_weight_max": 31.3057861328125,
      "activations/layer17_attention_weight_min": -26.65414047241211,
      "activations/layer18_attention_weight_max": 30.619754791259766,
      "activations/layer18_attention_weight_min": -25.65900230407715,
      "activations/layer19_attention_weight_max": 33.58036422729492,
      "activations/layer19_attention_weight_min": -32.5718879699707,
      "activations/layer1_attention_weight_max": 15.393301010131836,
      "activations/layer1_attention_weight_min": -15.601953506469727,
      "activations/layer20_attention_weight_max": 28.4646053314209,
      "activations/layer20_attention_weight_min": -25.21476936340332,
      "activations/layer21_attention_weight_max": 27.94802474975586,
      "activations/layer21_attention_weight_min": -25.18303108215332,
      "activations/layer22_attention_weight_max": 38.09587478637695,
      "activations/layer22_attention_weight_min": -31.136520385742188,
      "activations/layer23_attention_weight_max": 31.380931854248047,
      "activations/layer23_attention_weight_min": -25.687156677246094,
      "activations/layer2_attention_weight_max": 34.41015625,
      "activations/layer2_attention_weight_min": -33.2630615234375,
      "activations/layer3_attention_weight_max": 106.23712158203125,
      "activations/layer3_attention_weight_min": -108.38230895996094,
      "activations/layer4_attention_weight_max": 109.7403793334961,
      "activations/layer4_attention_weight_min": -114.69193267822266,
      "activations/layer5_attention_weight_max": 50.68376922607422,
      "activations/layer5_attention_weight_min": -65.63263702392578,
      "activations/layer6_attention_weight_max": 45.74838638305664,
      "activations/layer6_attention_weight_min": -50.15372085571289,
      "activations/layer7_attention_weight_max": 99.82573699951172,
      "activations/layer7_attention_weight_min": -107.83079528808594,
      "activations/layer8_attention_weight_max": 44.383766174316406,
      "activations/layer8_attention_weight_min": -47.69038009643555,
      "activations/layer9_attention_weight_max": 34.210208892822266,
      "activations/layer9_attention_weight_min": -35.60531234741211,
      "epoch": 22.75,
      "learning_rate": 3.3534090909090905e-06,
      "loss": 2.6905,
      "step": 391450
    },
    {
      "activations/layer0_attention_weight_max": 17.213544845581055,
      "activations/layer0_attention_weight_min": -13.446310997009277,
      "activations/layer10_attention_weight_max": 37.31183624267578,
      "activations/layer10_attention_weight_min": -39.078521728515625,
      "activations/layer11_attention_weight_max": 35.316951751708984,
      "activations/layer11_attention_weight_min": -39.599708557128906,
      "activations/layer12_attention_weight_max": 26.448640823364258,
      "activations/layer12_attention_weight_min": -27.94255828857422,
      "activations/layer13_attention_weight_max": 36.66611862182617,
      "activations/layer13_attention_weight_min": -35.877532958984375,
      "activations/layer14_attention_weight_max": 39.30253982543945,
      "activations/layer14_attention_weight_min": -37.97101974487305,
      "activations/layer15_attention_weight_max": 37.382572174072266,
      "activations/layer15_attention_weight_min": -38.385894775390625,
      "activations/layer16_attention_weight_max": 28.64551544189453,
      "activations/layer16_attention_weight_min": -29.71904182434082,
      "activations/layer17_attention_weight_max": 28.78827667236328,
      "activations/layer17_attention_weight_min": -29.729766845703125,
      "activations/layer18_attention_weight_max": 30.096960067749023,
      "activations/layer18_attention_weight_min": -30.224220275878906,
      "activations/layer19_attention_weight_max": 31.123239517211914,
      "activations/layer19_attention_weight_min": -34.3233757019043,
      "activations/layer1_attention_weight_max": 15.614995956420898,
      "activations/layer1_attention_weight_min": -14.986849784851074,
      "activations/layer20_attention_weight_max": 27.6074275970459,
      "activations/layer20_attention_weight_min": -26.059980392456055,
      "activations/layer21_attention_weight_max": 26.141399383544922,
      "activations/layer21_attention_weight_min": -26.088455200195312,
      "activations/layer22_attention_weight_max": 40.18035888671875,
      "activations/layer22_attention_weight_min": -33.83283615112305,
      "activations/layer23_attention_weight_max": 30.425262451171875,
      "activations/layer23_attention_weight_min": -26.65206527709961,
      "activations/layer2_attention_weight_max": 33.7508430480957,
      "activations/layer2_attention_weight_min": -34.22283172607422,
      "activations/layer3_attention_weight_max": 108.72036743164062,
      "activations/layer3_attention_weight_min": -109.505615234375,
      "activations/layer4_attention_weight_max": 113.3232421875,
      "activations/layer4_attention_weight_min": -114.87456512451172,
      "activations/layer5_attention_weight_max": 50.93553161621094,
      "activations/layer5_attention_weight_min": -60.7559814453125,
      "activations/layer6_attention_weight_max": 45.09133529663086,
      "activations/layer6_attention_weight_min": -46.55622100830078,
      "activations/layer7_attention_weight_max": 93.70247650146484,
      "activations/layer7_attention_weight_min": -100.79337310791016,
      "activations/layer8_attention_weight_max": 44.98939895629883,
      "activations/layer8_attention_weight_min": -47.042179107666016,
      "activations/layer9_attention_weight_max": 32.25142288208008,
      "activations/layer9_attention_weight_min": -36.7245979309082,
      "epoch": 22.75,
      "learning_rate": 3.334469696969697e-06,
      "loss": 2.6993,
      "step": 391500
    },
    {
      "activations/layer0_attention_weight_max": 16.98257827758789,
      "activations/layer0_attention_weight_min": -14.45004653930664,
      "activations/layer10_attention_weight_max": 36.01837158203125,
      "activations/layer10_attention_weight_min": -39.489715576171875,
      "activations/layer11_attention_weight_max": 37.36665344238281,
      "activations/layer11_attention_weight_min": -35.73823547363281,
      "activations/layer12_attention_weight_max": 28.41622543334961,
      "activations/layer12_attention_weight_min": -27.017969131469727,
      "activations/layer13_attention_weight_max": 40.00289535522461,
      "activations/layer13_attention_weight_min": -34.881629943847656,
      "activations/layer14_attention_weight_max": 43.1944694519043,
      "activations/layer14_attention_weight_min": -38.173343658447266,
      "activations/layer15_attention_weight_max": 41.70101547241211,
      "activations/layer15_attention_weight_min": -36.64692687988281,
      "activations/layer16_attention_weight_max": 30.826807022094727,
      "activations/layer16_attention_weight_min": -29.291826248168945,
      "activations/layer17_attention_weight_max": 29.877161026000977,
      "activations/layer17_attention_weight_min": -26.765968322753906,
      "activations/layer18_attention_weight_max": 35.43558883666992,
      "activations/layer18_attention_weight_min": -30.25865364074707,
      "activations/layer19_attention_weight_max": 36.6141357421875,
      "activations/layer19_attention_weight_min": -33.481021881103516,
      "activations/layer1_attention_weight_max": 15.835407257080078,
      "activations/layer1_attention_weight_min": -15.419231414794922,
      "activations/layer20_attention_weight_max": 33.395755767822266,
      "activations/layer20_attention_weight_min": -27.5949649810791,
      "activations/layer21_attention_weight_max": 30.337186813354492,
      "activations/layer21_attention_weight_min": -26.00432777404785,
      "activations/layer22_attention_weight_max": 45.49011993408203,
      "activations/layer22_attention_weight_min": -34.860111236572266,
      "activations/layer23_attention_weight_max": 34.94660186767578,
      "activations/layer23_attention_weight_min": -26.16383171081543,
      "activations/layer2_attention_weight_max": 35.09568405151367,
      "activations/layer2_attention_weight_min": -33.816986083984375,
      "activations/layer3_attention_weight_max": 110.12982177734375,
      "activations/layer3_attention_weight_min": -109.85948181152344,
      "activations/layer4_attention_weight_max": 113.3355941772461,
      "activations/layer4_attention_weight_min": -114.68193817138672,
      "activations/layer5_attention_weight_max": 50.155296325683594,
      "activations/layer5_attention_weight_min": -60.761817932128906,
      "activations/layer6_attention_weight_max": 46.50492858886719,
      "activations/layer6_attention_weight_min": -50.17568588256836,
      "activations/layer7_attention_weight_max": 94.95800018310547,
      "activations/layer7_attention_weight_min": -107.2925033569336,
      "activations/layer8_attention_weight_max": 45.77347946166992,
      "activations/layer8_attention_weight_min": -49.10097122192383,
      "activations/layer9_attention_weight_max": 33.966636657714844,
      "activations/layer9_attention_weight_min": -37.35628128051758,
      "epoch": 22.75,
      "learning_rate": 3.315530303030303e-06,
      "loss": 2.7073,
      "step": 391550
    },
    {
      "activations/layer0_attention_weight_max": 17.335336685180664,
      "activations/layer0_attention_weight_min": -14.97641658782959,
      "activations/layer10_attention_weight_max": 41.02995300292969,
      "activations/layer10_attention_weight_min": -40.4547004699707,
      "activations/layer11_attention_weight_max": 38.10317611694336,
      "activations/layer11_attention_weight_min": -38.33940887451172,
      "activations/layer12_attention_weight_max": 28.096561431884766,
      "activations/layer12_attention_weight_min": -28.362939834594727,
      "activations/layer13_attention_weight_max": 36.90941619873047,
      "activations/layer13_attention_weight_min": -35.43963623046875,
      "activations/layer14_attention_weight_max": 39.738277435302734,
      "activations/layer14_attention_weight_min": -37.30147171020508,
      "activations/layer15_attention_weight_max": 38.13624954223633,
      "activations/layer15_attention_weight_min": -38.79301452636719,
      "activations/layer16_attention_weight_max": 29.412059783935547,
      "activations/layer16_attention_weight_min": -29.808399200439453,
      "activations/layer17_attention_weight_max": 28.348051071166992,
      "activations/layer17_attention_weight_min": -28.056324005126953,
      "activations/layer18_attention_weight_max": 32.58525085449219,
      "activations/layer18_attention_weight_min": -28.0152530670166,
      "activations/layer19_attention_weight_max": 34.3741569519043,
      "activations/layer19_attention_weight_min": -32.04051208496094,
      "activations/layer1_attention_weight_max": 16.01866912841797,
      "activations/layer1_attention_weight_min": -14.971355438232422,
      "activations/layer20_attention_weight_max": 25.900243759155273,
      "activations/layer20_attention_weight_min": -24.047502517700195,
      "activations/layer21_attention_weight_max": 25.938779830932617,
      "activations/layer21_attention_weight_min": -23.87318992614746,
      "activations/layer22_attention_weight_max": 39.569210052490234,
      "activations/layer22_attention_weight_min": -32.0097541809082,
      "activations/layer23_attention_weight_max": 33.86390686035156,
      "activations/layer23_attention_weight_min": -24.96432113647461,
      "activations/layer2_attention_weight_max": 36.6414794921875,
      "activations/layer2_attention_weight_min": -34.66606903076172,
      "activations/layer3_attention_weight_max": 108.0572280883789,
      "activations/layer3_attention_weight_min": -114.14730834960938,
      "activations/layer4_attention_weight_max": 115.63427734375,
      "activations/layer4_attention_weight_min": -116.4382553100586,
      "activations/layer5_attention_weight_max": 50.176368713378906,
      "activations/layer5_attention_weight_min": -65.80953979492188,
      "activations/layer6_attention_weight_max": 47.93357467651367,
      "activations/layer6_attention_weight_min": -49.42105484008789,
      "activations/layer7_attention_weight_max": 99.01127624511719,
      "activations/layer7_attention_weight_min": -107.06037902832031,
      "activations/layer8_attention_weight_max": 45.80335998535156,
      "activations/layer8_attention_weight_min": -49.12128829956055,
      "activations/layer9_attention_weight_max": 35.63302230834961,
      "activations/layer9_attention_weight_min": -38.89006805419922,
      "epoch": 22.75,
      "learning_rate": 3.296590909090909e-06,
      "loss": 2.7004,
      "step": 391600
    },
    {
      "activations/layer0_attention_weight_max": 17.565309524536133,
      "activations/layer0_attention_weight_min": -16.279953002929688,
      "activations/layer10_attention_weight_max": 37.60647964477539,
      "activations/layer10_attention_weight_min": -39.2194938659668,
      "activations/layer11_attention_weight_max": 35.89225769042969,
      "activations/layer11_attention_weight_min": -37.21134948730469,
      "activations/layer12_attention_weight_max": 29.426721572875977,
      "activations/layer12_attention_weight_min": -29.437808990478516,
      "activations/layer13_attention_weight_max": 41.90199661254883,
      "activations/layer13_attention_weight_min": -39.12250518798828,
      "activations/layer14_attention_weight_max": 44.389373779296875,
      "activations/layer14_attention_weight_min": -42.47563934326172,
      "activations/layer15_attention_weight_max": 41.62726593017578,
      "activations/layer15_attention_weight_min": -39.84540939331055,
      "activations/layer16_attention_weight_max": 31.613224029541016,
      "activations/layer16_attention_weight_min": -29.351417541503906,
      "activations/layer17_attention_weight_max": 31.175134658813477,
      "activations/layer17_attention_weight_min": -27.650665283203125,
      "activations/layer18_attention_weight_max": 34.51533126831055,
      "activations/layer18_attention_weight_min": -29.540441513061523,
      "activations/layer19_attention_weight_max": 37.66685104370117,
      "activations/layer19_attention_weight_min": -31.88947868347168,
      "activations/layer1_attention_weight_max": 15.86260986328125,
      "activations/layer1_attention_weight_min": -14.804810523986816,
      "activations/layer20_attention_weight_max": 31.550212860107422,
      "activations/layer20_attention_weight_min": -26.414127349853516,
      "activations/layer21_attention_weight_max": 31.508825302124023,
      "activations/layer21_attention_weight_min": -29.84609603881836,
      "activations/layer22_attention_weight_max": 46.12750244140625,
      "activations/layer22_attention_weight_min": -39.07855987548828,
      "activations/layer23_attention_weight_max": 35.55613327026367,
      "activations/layer23_attention_weight_min": -29.94789695739746,
      "activations/layer2_attention_weight_max": 35.909603118896484,
      "activations/layer2_attention_weight_min": -34.25291442871094,
      "activations/layer3_attention_weight_max": 113.28902435302734,
      "activations/layer3_attention_weight_min": -114.18071746826172,
      "activations/layer4_attention_weight_max": 116.4502182006836,
      "activations/layer4_attention_weight_min": -117.5072250366211,
      "activations/layer5_attention_weight_max": 55.54057312011719,
      "activations/layer5_attention_weight_min": -66.45372009277344,
      "activations/layer6_attention_weight_max": 49.12972640991211,
      "activations/layer6_attention_weight_min": -51.532779693603516,
      "activations/layer7_attention_weight_max": 99.13243865966797,
      "activations/layer7_attention_weight_min": -108.64537048339844,
      "activations/layer8_attention_weight_max": 44.631282806396484,
      "activations/layer8_attention_weight_min": -48.6663818359375,
      "activations/layer9_attention_weight_max": 34.51919937133789,
      "activations/layer9_attention_weight_min": -39.75673294067383,
      "epoch": 22.76,
      "learning_rate": 3.2776515151515147e-06,
      "loss": 2.6895,
      "step": 391650
    },
    {
      "activations/layer0_attention_weight_max": 16.852767944335938,
      "activations/layer0_attention_weight_min": -13.038729667663574,
      "activations/layer10_attention_weight_max": 36.59955978393555,
      "activations/layer10_attention_weight_min": -35.41339874267578,
      "activations/layer11_attention_weight_max": 33.91688537597656,
      "activations/layer11_attention_weight_min": -34.88764953613281,
      "activations/layer12_attention_weight_max": 27.53803253173828,
      "activations/layer12_attention_weight_min": -28.955951690673828,
      "activations/layer13_attention_weight_max": 39.859825134277344,
      "activations/layer13_attention_weight_min": -37.294673919677734,
      "activations/layer14_attention_weight_max": 42.692142486572266,
      "activations/layer14_attention_weight_min": -38.08673858642578,
      "activations/layer15_attention_weight_max": 40.038230895996094,
      "activations/layer15_attention_weight_min": -38.427486419677734,
      "activations/layer16_attention_weight_max": 29.566570281982422,
      "activations/layer16_attention_weight_min": -29.327983856201172,
      "activations/layer17_attention_weight_max": 29.347795486450195,
      "activations/layer17_attention_weight_min": -26.39043617248535,
      "activations/layer18_attention_weight_max": 32.29365158081055,
      "activations/layer18_attention_weight_min": -27.152748107910156,
      "activations/layer19_attention_weight_max": 34.806983947753906,
      "activations/layer19_attention_weight_min": -32.30244064331055,
      "activations/layer1_attention_weight_max": 16.24330711364746,
      "activations/layer1_attention_weight_min": -15.210297584533691,
      "activations/layer20_attention_weight_max": 28.354501724243164,
      "activations/layer20_attention_weight_min": -26.114885330200195,
      "activations/layer21_attention_weight_max": 30.519615173339844,
      "activations/layer21_attention_weight_min": -28.038070678710938,
      "activations/layer22_attention_weight_max": 41.08335494995117,
      "activations/layer22_attention_weight_min": -33.95301818847656,
      "activations/layer23_attention_weight_max": 33.43181228637695,
      "activations/layer23_attention_weight_min": -27.068527221679688,
      "activations/layer2_attention_weight_max": 35.222740173339844,
      "activations/layer2_attention_weight_min": -34.55916213989258,
      "activations/layer3_attention_weight_max": 109.65107727050781,
      "activations/layer3_attention_weight_min": -111.44390106201172,
      "activations/layer4_attention_weight_max": 111.55718994140625,
      "activations/layer4_attention_weight_min": -113.40185546875,
      "activations/layer5_attention_weight_max": 50.71997833251953,
      "activations/layer5_attention_weight_min": -63.747215270996094,
      "activations/layer6_attention_weight_max": 47.42217254638672,
      "activations/layer6_attention_weight_min": -50.4472770690918,
      "activations/layer7_attention_weight_max": 99.553466796875,
      "activations/layer7_attention_weight_min": -100.54309844970703,
      "activations/layer8_attention_weight_max": 43.0287971496582,
      "activations/layer8_attention_weight_min": -45.65101623535156,
      "activations/layer9_attention_weight_max": 32.96664810180664,
      "activations/layer9_attention_weight_min": -34.412681579589844,
      "epoch": 22.76,
      "learning_rate": 3.258712121212121e-06,
      "loss": 2.7089,
      "step": 391700
    },
    {
      "activations/layer0_attention_weight_max": 15.927351951599121,
      "activations/layer0_attention_weight_min": -14.616212844848633,
      "activations/layer10_attention_weight_max": 39.06243896484375,
      "activations/layer10_attention_weight_min": -36.76272201538086,
      "activations/layer11_attention_weight_max": 36.33557891845703,
      "activations/layer11_attention_weight_min": -35.67900085449219,
      "activations/layer12_attention_weight_max": 29.33348274230957,
      "activations/layer12_attention_weight_min": -27.29236602783203,
      "activations/layer13_attention_weight_max": 41.41350555419922,
      "activations/layer13_attention_weight_min": -37.23665237426758,
      "activations/layer14_attention_weight_max": 44.00657653808594,
      "activations/layer14_attention_weight_min": -40.072723388671875,
      "activations/layer15_attention_weight_max": 41.534515380859375,
      "activations/layer15_attention_weight_min": -37.15494918823242,
      "activations/layer16_attention_weight_max": 29.90926170349121,
      "activations/layer16_attention_weight_min": -28.040910720825195,
      "activations/layer17_attention_weight_max": 29.787506103515625,
      "activations/layer17_attention_weight_min": -25.85694122314453,
      "activations/layer18_attention_weight_max": 29.339500427246094,
      "activations/layer18_attention_weight_min": -25.79526138305664,
      "activations/layer19_attention_weight_max": 34.12686538696289,
      "activations/layer19_attention_weight_min": -31.904409408569336,
      "activations/layer1_attention_weight_max": 16.093896865844727,
      "activations/layer1_attention_weight_min": -16.5107421875,
      "activations/layer20_attention_weight_max": 27.520763397216797,
      "activations/layer20_attention_weight_min": -26.258054733276367,
      "activations/layer21_attention_weight_max": 26.74239730834961,
      "activations/layer21_attention_weight_min": -25.431224822998047,
      "activations/layer22_attention_weight_max": 40.2470703125,
      "activations/layer22_attention_weight_min": -32.03154754638672,
      "activations/layer23_attention_weight_max": 30.0980224609375,
      "activations/layer23_attention_weight_min": -26.736896514892578,
      "activations/layer2_attention_weight_max": 34.012977600097656,
      "activations/layer2_attention_weight_min": -34.05751037597656,
      "activations/layer3_attention_weight_max": 108.1096420288086,
      "activations/layer3_attention_weight_min": -107.33843231201172,
      "activations/layer4_attention_weight_max": 114.34183502197266,
      "activations/layer4_attention_weight_min": -115.33626556396484,
      "activations/layer5_attention_weight_max": 53.17350387573242,
      "activations/layer5_attention_weight_min": -59.97460174560547,
      "activations/layer6_attention_weight_max": 46.98939514160156,
      "activations/layer6_attention_weight_min": -50.276405334472656,
      "activations/layer7_attention_weight_max": 105.10648345947266,
      "activations/layer7_attention_weight_min": -104.251953125,
      "activations/layer8_attention_weight_max": 45.8155403137207,
      "activations/layer8_attention_weight_min": -45.8252067565918,
      "activations/layer9_attention_weight_max": 33.801937103271484,
      "activations/layer9_attention_weight_min": -35.81464767456055,
      "epoch": 22.76,
      "learning_rate": 3.239772727272727e-06,
      "loss": 2.6946,
      "step": 391750
    },
    {
      "activations/layer0_attention_weight_max": 17.031953811645508,
      "activations/layer0_attention_weight_min": -15.082045555114746,
      "activations/layer10_attention_weight_max": 34.57924270629883,
      "activations/layer10_attention_weight_min": -35.88041687011719,
      "activations/layer11_attention_weight_max": 34.194175720214844,
      "activations/layer11_attention_weight_min": -35.5369873046875,
      "activations/layer12_attention_weight_max": 28.22928237915039,
      "activations/layer12_attention_weight_min": -27.867454528808594,
      "activations/layer13_attention_weight_max": 47.570594787597656,
      "activations/layer13_attention_weight_min": -37.47582244873047,
      "activations/layer14_attention_weight_max": 47.14499282836914,
      "activations/layer14_attention_weight_min": -39.502872467041016,
      "activations/layer15_attention_weight_max": 44.20491409301758,
      "activations/layer15_attention_weight_min": -36.2457389831543,
      "activations/layer16_attention_weight_max": 31.976245880126953,
      "activations/layer16_attention_weight_min": -29.319093704223633,
      "activations/layer17_attention_weight_max": 31.513174057006836,
      "activations/layer17_attention_weight_min": -27.941530227661133,
      "activations/layer18_attention_weight_max": 34.314388275146484,
      "activations/layer18_attention_weight_min": -30.56633949279785,
      "activations/layer19_attention_weight_max": 35.61772537231445,
      "activations/layer19_attention_weight_min": -33.90127182006836,
      "activations/layer1_attention_weight_max": 16.019506454467773,
      "activations/layer1_attention_weight_min": -15.517071723937988,
      "activations/layer20_attention_weight_max": 28.762699127197266,
      "activations/layer20_attention_weight_min": -26.862884521484375,
      "activations/layer21_attention_weight_max": 29.31273078918457,
      "activations/layer21_attention_weight_min": -26.92986297607422,
      "activations/layer22_attention_weight_max": 43.54003143310547,
      "activations/layer22_attention_weight_min": -34.75029373168945,
      "activations/layer23_attention_weight_max": 34.75546646118164,
      "activations/layer23_attention_weight_min": -29.9630184173584,
      "activations/layer2_attention_weight_max": 34.41526412963867,
      "activations/layer2_attention_weight_min": -35.287200927734375,
      "activations/layer3_attention_weight_max": 111.03828430175781,
      "activations/layer3_attention_weight_min": -115.0245361328125,
      "activations/layer4_attention_weight_max": 110.99043273925781,
      "activations/layer4_attention_weight_min": -116.41915130615234,
      "activations/layer5_attention_weight_max": 52.11759567260742,
      "activations/layer5_attention_weight_min": -63.500526428222656,
      "activations/layer6_attention_weight_max": 48.51387405395508,
      "activations/layer6_attention_weight_min": -51.97352600097656,
      "activations/layer7_attention_weight_max": 101.54959869384766,
      "activations/layer7_attention_weight_min": -111.31656646728516,
      "activations/layer8_attention_weight_max": 45.0279426574707,
      "activations/layer8_attention_weight_min": -50.036617279052734,
      "activations/layer9_attention_weight_max": 32.995357513427734,
      "activations/layer9_attention_weight_min": -35.14068603515625,
      "epoch": 22.77,
      "learning_rate": 3.220833333333333e-06,
      "loss": 2.6932,
      "step": 391800
    },
    {
      "activations/layer0_attention_weight_max": 17.453433990478516,
      "activations/layer0_attention_weight_min": -12.898158073425293,
      "activations/layer10_attention_weight_max": 38.06826400756836,
      "activations/layer10_attention_weight_min": -35.594966888427734,
      "activations/layer11_attention_weight_max": 37.007415771484375,
      "activations/layer11_attention_weight_min": -37.76676559448242,
      "activations/layer12_attention_weight_max": 27.844589233398438,
      "activations/layer12_attention_weight_min": -27.195449829101562,
      "activations/layer13_attention_weight_max": 40.87115478515625,
      "activations/layer13_attention_weight_min": -37.64120101928711,
      "activations/layer14_attention_weight_max": 43.72926712036133,
      "activations/layer14_attention_weight_min": -39.20494079589844,
      "activations/layer15_attention_weight_max": 41.74763107299805,
      "activations/layer15_attention_weight_min": -37.417903900146484,
      "activations/layer16_attention_weight_max": 30.145252227783203,
      "activations/layer16_attention_weight_min": -28.217954635620117,
      "activations/layer17_attention_weight_max": 32.00065994262695,
      "activations/layer17_attention_weight_min": -26.6561222076416,
      "activations/layer18_attention_weight_max": 35.344635009765625,
      "activations/layer18_attention_weight_min": -27.45970344543457,
      "activations/layer19_attention_weight_max": 35.1983757019043,
      "activations/layer19_attention_weight_min": -35.14022445678711,
      "activations/layer1_attention_weight_max": 16.536848068237305,
      "activations/layer1_attention_weight_min": -15.234065055847168,
      "activations/layer20_attention_weight_max": 29.290897369384766,
      "activations/layer20_attention_weight_min": -28.421653747558594,
      "activations/layer21_attention_weight_max": 29.062902450561523,
      "activations/layer21_attention_weight_min": -27.054880142211914,
      "activations/layer22_attention_weight_max": 44.5399055480957,
      "activations/layer22_attention_weight_min": -34.26847457885742,
      "activations/layer23_attention_weight_max": 33.04441452026367,
      "activations/layer23_attention_weight_min": -28.49018669128418,
      "activations/layer2_attention_weight_max": 35.058231353759766,
      "activations/layer2_attention_weight_min": -34.25177764892578,
      "activations/layer3_attention_weight_max": 111.74061584472656,
      "activations/layer3_attention_weight_min": -110.00070190429688,
      "activations/layer4_attention_weight_max": 114.5168228149414,
      "activations/layer4_attention_weight_min": -114.54882049560547,
      "activations/layer5_attention_weight_max": 54.74811935424805,
      "activations/layer5_attention_weight_min": -61.41096496582031,
      "activations/layer6_attention_weight_max": 47.595394134521484,
      "activations/layer6_attention_weight_min": -49.30915451049805,
      "activations/layer7_attention_weight_max": 102.43138122558594,
      "activations/layer7_attention_weight_min": -101.89736938476562,
      "activations/layer8_attention_weight_max": 44.41864776611328,
      "activations/layer8_attention_weight_min": -45.34638977050781,
      "activations/layer9_attention_weight_max": 32.18680191040039,
      "activations/layer9_attention_weight_min": -34.32351303100586,
      "epoch": 22.77,
      "learning_rate": 3.201893939393939e-06,
      "loss": 2.691,
      "step": 391850
    },
    {
      "activations/layer0_attention_weight_max": 16.660837173461914,
      "activations/layer0_attention_weight_min": -14.717608451843262,
      "activations/layer10_attention_weight_max": 35.74452590942383,
      "activations/layer10_attention_weight_min": -37.79726791381836,
      "activations/layer11_attention_weight_max": 35.37458801269531,
      "activations/layer11_attention_weight_min": -34.784446716308594,
      "activations/layer12_attention_weight_max": 26.876291275024414,
      "activations/layer12_attention_weight_min": -27.213899612426758,
      "activations/layer13_attention_weight_max": 40.338294982910156,
      "activations/layer13_attention_weight_min": -35.68317413330078,
      "activations/layer14_attention_weight_max": 49.21565628051758,
      "activations/layer14_attention_weight_min": -38.63776779174805,
      "activations/layer15_attention_weight_max": 40.266258239746094,
      "activations/layer15_attention_weight_min": -37.719058990478516,
      "activations/layer16_attention_weight_max": 33.14625549316406,
      "activations/layer16_attention_weight_min": -29.93031883239746,
      "activations/layer17_attention_weight_max": 31.57172966003418,
      "activations/layer17_attention_weight_min": -26.607166290283203,
      "activations/layer18_attention_weight_max": 32.081363677978516,
      "activations/layer18_attention_weight_min": -24.77291488647461,
      "activations/layer19_attention_weight_max": 40.953861236572266,
      "activations/layer19_attention_weight_min": -33.252872467041016,
      "activations/layer1_attention_weight_max": 16.083925247192383,
      "activations/layer1_attention_weight_min": -14.962604522705078,
      "activations/layer20_attention_weight_max": 30.122745513916016,
      "activations/layer20_attention_weight_min": -25.586042404174805,
      "activations/layer21_attention_weight_max": 29.06968116760254,
      "activations/layer21_attention_weight_min": -24.94993019104004,
      "activations/layer22_attention_weight_max": 43.912742614746094,
      "activations/layer22_attention_weight_min": -31.55460548400879,
      "activations/layer23_attention_weight_max": 40.5822868347168,
      "activations/layer23_attention_weight_min": -25.664794921875,
      "activations/layer2_attention_weight_max": 33.61515426635742,
      "activations/layer2_attention_weight_min": -33.07103729248047,
      "activations/layer3_attention_weight_max": 109.2735824584961,
      "activations/layer3_attention_weight_min": -108.98904418945312,
      "activations/layer4_attention_weight_max": 115.55577850341797,
      "activations/layer4_attention_weight_min": -117.98111724853516,
      "activations/layer5_attention_weight_max": 51.2631950378418,
      "activations/layer5_attention_weight_min": -67.45781707763672,
      "activations/layer6_attention_weight_max": 46.907569885253906,
      "activations/layer6_attention_weight_min": -49.63439178466797,
      "activations/layer7_attention_weight_max": 95.47196960449219,
      "activations/layer7_attention_weight_min": -105.97108459472656,
      "activations/layer8_attention_weight_max": 44.78388595581055,
      "activations/layer8_attention_weight_min": -48.41294479370117,
      "activations/layer9_attention_weight_max": 32.46067428588867,
      "activations/layer9_attention_weight_min": -41.68790817260742,
      "epoch": 22.77,
      "learning_rate": 3.182954545454545e-06,
      "loss": 2.7144,
      "step": 391900
    },
    {
      "activations/layer0_attention_weight_max": 16.38813591003418,
      "activations/layer0_attention_weight_min": -14.620000839233398,
      "activations/layer10_attention_weight_max": 34.42943572998047,
      "activations/layer10_attention_weight_min": -35.539642333984375,
      "activations/layer11_attention_weight_max": 33.0634765625,
      "activations/layer11_attention_weight_min": -33.98632049560547,
      "activations/layer12_attention_weight_max": 26.648561477661133,
      "activations/layer12_attention_weight_min": -26.779329299926758,
      "activations/layer13_attention_weight_max": 38.54220199584961,
      "activations/layer13_attention_weight_min": -36.65386962890625,
      "activations/layer14_attention_weight_max": 39.36843490600586,
      "activations/layer14_attention_weight_min": -37.495601654052734,
      "activations/layer15_attention_weight_max": 36.89393997192383,
      "activations/layer15_attention_weight_min": -35.51408004760742,
      "activations/layer16_attention_weight_max": 30.39151954650879,
      "activations/layer16_attention_weight_min": -28.59387969970703,
      "activations/layer17_attention_weight_max": 29.809839248657227,
      "activations/layer17_attention_weight_min": -27.49441146850586,
      "activations/layer18_attention_weight_max": 30.941320419311523,
      "activations/layer18_attention_weight_min": -25.443649291992188,
      "activations/layer19_attention_weight_max": 33.4387321472168,
      "activations/layer19_attention_weight_min": -31.3699893951416,
      "activations/layer1_attention_weight_max": 16.132835388183594,
      "activations/layer1_attention_weight_min": -16.25247573852539,
      "activations/layer20_attention_weight_max": 30.39459800720215,
      "activations/layer20_attention_weight_min": -23.47797966003418,
      "activations/layer21_attention_weight_max": 29.455827713012695,
      "activations/layer21_attention_weight_min": -23.914215087890625,
      "activations/layer22_attention_weight_max": 39.54618453979492,
      "activations/layer22_attention_weight_min": -32.27820587158203,
      "activations/layer23_attention_weight_max": 30.932872772216797,
      "activations/layer23_attention_weight_min": -27.230606079101562,
      "activations/layer2_attention_weight_max": 33.906776428222656,
      "activations/layer2_attention_weight_min": -33.62805938720703,
      "activations/layer3_attention_weight_max": 108.18816375732422,
      "activations/layer3_attention_weight_min": -108.98204040527344,
      "activations/layer4_attention_weight_max": 110.7786865234375,
      "activations/layer4_attention_weight_min": -114.79609680175781,
      "activations/layer5_attention_weight_max": 50.84525680541992,
      "activations/layer5_attention_weight_min": -64.21733856201172,
      "activations/layer6_attention_weight_max": 47.08842849731445,
      "activations/layer6_attention_weight_min": -49.68465042114258,
      "activations/layer7_attention_weight_max": 99.35713195800781,
      "activations/layer7_attention_weight_min": -104.6330337524414,
      "activations/layer8_attention_weight_max": 45.603065490722656,
      "activations/layer8_attention_weight_min": -47.52463150024414,
      "activations/layer9_attention_weight_max": 31.84739112854004,
      "activations/layer9_attention_weight_min": -35.211483001708984,
      "epoch": 22.77,
      "learning_rate": 3.1640151515151516e-06,
      "loss": 2.6928,
      "step": 391950
    },
    {
      "activations/layer0_attention_weight_max": 17.552412033081055,
      "activations/layer0_attention_weight_min": -13.586676597595215,
      "activations/layer10_attention_weight_max": 43.12548828125,
      "activations/layer10_attention_weight_min": -39.52614212036133,
      "activations/layer11_attention_weight_max": 40.895240783691406,
      "activations/layer11_attention_weight_min": -41.0887451171875,
      "activations/layer12_attention_weight_max": 31.25279426574707,
      "activations/layer12_attention_weight_min": -31.021203994750977,
      "activations/layer13_attention_weight_max": 43.777774810791016,
      "activations/layer13_attention_weight_min": -40.37114334106445,
      "activations/layer14_attention_weight_max": 48.91172790527344,
      "activations/layer14_attention_weight_min": -43.7043571472168,
      "activations/layer15_attention_weight_max": 47.64706039428711,
      "activations/layer15_attention_weight_min": -40.92748260498047,
      "activations/layer16_attention_weight_max": 33.219356536865234,
      "activations/layer16_attention_weight_min": -31.893856048583984,
      "activations/layer17_attention_weight_max": 31.97872543334961,
      "activations/layer17_attention_weight_min": -27.14277458190918,
      "activations/layer18_attention_weight_max": 33.203128814697266,
      "activations/layer18_attention_weight_min": -27.354703903198242,
      "activations/layer19_attention_weight_max": 34.837276458740234,
      "activations/layer19_attention_weight_min": -34.5540885925293,
      "activations/layer1_attention_weight_max": 16.705820083618164,
      "activations/layer1_attention_weight_min": -14.892370223999023,
      "activations/layer20_attention_weight_max": 28.88689613342285,
      "activations/layer20_attention_weight_min": -25.775333404541016,
      "activations/layer21_attention_weight_max": 27.088485717773438,
      "activations/layer21_attention_weight_min": -24.93475914001465,
      "activations/layer22_attention_weight_max": 42.106388092041016,
      "activations/layer22_attention_weight_min": -32.81467056274414,
      "activations/layer23_attention_weight_max": 33.765174865722656,
      "activations/layer23_attention_weight_min": -27.19024658203125,
      "activations/layer2_attention_weight_max": 34.64667892456055,
      "activations/layer2_attention_weight_min": -34.51695251464844,
      "activations/layer3_attention_weight_max": 111.28264617919922,
      "activations/layer3_attention_weight_min": -112.33627319335938,
      "activations/layer4_attention_weight_max": 113.53865814208984,
      "activations/layer4_attention_weight_min": -118.2362289428711,
      "activations/layer5_attention_weight_max": 53.31647491455078,
      "activations/layer5_attention_weight_min": -65.32518005371094,
      "activations/layer6_attention_weight_max": 47.04064178466797,
      "activations/layer6_attention_weight_min": -49.68705368041992,
      "activations/layer7_attention_weight_max": 109.814208984375,
      "activations/layer7_attention_weight_min": -106.95578002929688,
      "activations/layer8_attention_weight_max": 48.429664611816406,
      "activations/layer8_attention_weight_min": -49.77227783203125,
      "activations/layer9_attention_weight_max": 37.493186950683594,
      "activations/layer9_attention_weight_min": -40.25319290161133,
      "epoch": 22.78,
      "learning_rate": 3.1450757575757573e-06,
      "loss": 2.7143,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.4973,
      "eval_samples_per_second": 505.339,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.4973,
      "eval_openwebtext_samples_per_second": 505.339,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9205,
      "eval_wikitext_samples_per_second": 237.443,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_lambada_loss": 2.28515625,
      "eval_lambada_ppl": 9.827221607438576,
      "eval_lambada_runtime": 9.5132,
      "eval_lambada_samples_per_second": 511.818,
      "step": 392000
    },
    {
      "activations/layer0_attention_weight_max": 16.285924911499023,
      "activations/layer0_attention_weight_min": -14.455925941467285,
      "activations/layer10_attention_weight_max": 37.166259765625,
      "activations/layer10_attention_weight_min": -37.66908645629883,
      "activations/layer11_attention_weight_max": 35.06073760986328,
      "activations/layer11_attention_weight_min": -36.29459762573242,
      "activations/layer12_attention_weight_max": 26.486957550048828,
      "activations/layer12_attention_weight_min": -28.296289443969727,
      "activations/layer13_attention_weight_max": 40.267189025878906,
      "activations/layer13_attention_weight_min": -38.715110778808594,
      "activations/layer14_attention_weight_max": 44.60560989379883,
      "activations/layer14_attention_weight_min": -42.18527603149414,
      "activations/layer15_attention_weight_max": 41.81010437011719,
      "activations/layer15_attention_weight_min": -39.084171295166016,
      "activations/layer16_attention_weight_max": 33.10224914550781,
      "activations/layer16_attention_weight_min": -30.31093406677246,
      "activations/layer17_attention_weight_max": 29.632972717285156,
      "activations/layer17_attention_weight_min": -26.409448623657227,
      "activations/layer18_attention_weight_max": 29.24302864074707,
      "activations/layer18_attention_weight_min": -26.003095626831055,
      "activations/layer19_attention_weight_max": 36.23202133178711,
      "activations/layer19_attention_weight_min": -30.88108253479004,
      "activations/layer1_attention_weight_max": 17.234859466552734,
      "activations/layer1_attention_weight_min": -14.708120346069336,
      "activations/layer20_attention_weight_max": 28.99370574951172,
      "activations/layer20_attention_weight_min": -25.060190200805664,
      "activations/layer21_attention_weight_max": 27.87407112121582,
      "activations/layer21_attention_weight_min": -24.667503356933594,
      "activations/layer22_attention_weight_max": 40.42909240722656,
      "activations/layer22_attention_weight_min": -31.291378021240234,
      "activations/layer23_attention_weight_max": 30.143177032470703,
      "activations/layer23_attention_weight_min": -25.727964401245117,
      "activations/layer2_attention_weight_max": 33.43779754638672,
      "activations/layer2_attention_weight_min": -33.980926513671875,
      "activations/layer3_attention_weight_max": 108.25991821289062,
      "activations/layer3_attention_weight_min": -109.47811889648438,
      "activations/layer4_attention_weight_max": 110.14525604248047,
      "activations/layer4_attention_weight_min": -111.37471771240234,
      "activations/layer5_attention_weight_max": 51.52094650268555,
      "activations/layer5_attention_weight_min": -64.22879028320312,
      "activations/layer6_attention_weight_max": 45.316932678222656,
      "activations/layer6_attention_weight_min": -47.18956756591797,
      "activations/layer7_attention_weight_max": 101.0594253540039,
      "activations/layer7_attention_weight_min": -103.62871551513672,
      "activations/layer8_attention_weight_max": 46.859466552734375,
      "activations/layer8_attention_weight_min": -49.35578536987305,
      "activations/layer9_attention_weight_max": 34.78409957885742,
      "activations/layer9_attention_weight_min": -36.95060348510742,
      "epoch": 22.78,
      "learning_rate": 3.1261363636363635e-06,
      "loss": 2.6899,
      "step": 392050
    },
    {
      "activations/layer0_attention_weight_max": 16.645458221435547,
      "activations/layer0_attention_weight_min": -16.412012100219727,
      "activations/layer10_attention_weight_max": 37.000709533691406,
      "activations/layer10_attention_weight_min": -37.872169494628906,
      "activations/layer11_attention_weight_max": 37.677860260009766,
      "activations/layer11_attention_weight_min": -37.660804748535156,
      "activations/layer12_attention_weight_max": 29.983705520629883,
      "activations/layer12_attention_weight_min": -27.227094650268555,
      "activations/layer13_attention_weight_max": 43.61934280395508,
      "activations/layer13_attention_weight_min": -36.882362365722656,
      "activations/layer14_attention_weight_max": 46.22517776489258,
      "activations/layer14_attention_weight_min": -39.89180374145508,
      "activations/layer15_attention_weight_max": 43.05528259277344,
      "activations/layer15_attention_weight_min": -38.9871826171875,
      "activations/layer16_attention_weight_max": 31.148595809936523,
      "activations/layer16_attention_weight_min": -28.165645599365234,
      "activations/layer17_attention_weight_max": 32.207542419433594,
      "activations/layer17_attention_weight_min": -26.647525787353516,
      "activations/layer18_attention_weight_max": 32.426918029785156,
      "activations/layer18_attention_weight_min": -26.950008392333984,
      "activations/layer19_attention_weight_max": 33.793670654296875,
      "activations/layer19_attention_weight_min": -30.480987548828125,
      "activations/layer1_attention_weight_max": 15.913350105285645,
      "activations/layer1_attention_weight_min": -15.19129467010498,
      "activations/layer20_attention_weight_max": 28.360721588134766,
      "activations/layer20_attention_weight_min": -23.558820724487305,
      "activations/layer21_attention_weight_max": 29.844858169555664,
      "activations/layer21_attention_weight_min": -23.023210525512695,
      "activations/layer22_attention_weight_max": 47.75215530395508,
      "activations/layer22_attention_weight_min": -30.692041397094727,
      "activations/layer23_attention_weight_max": 35.37192153930664,
      "activations/layer23_attention_weight_min": -26.5676326751709,
      "activations/layer2_attention_weight_max": 34.39928436279297,
      "activations/layer2_attention_weight_min": -34.85184860229492,
      "activations/layer3_attention_weight_max": 110.64830780029297,
      "activations/layer3_attention_weight_min": -114.12025451660156,
      "activations/layer4_attention_weight_max": 113.77607727050781,
      "activations/layer4_attention_weight_min": -120.06805419921875,
      "activations/layer5_attention_weight_max": 53.1511344909668,
      "activations/layer5_attention_weight_min": -64.43193817138672,
      "activations/layer6_attention_weight_max": 49.16062545776367,
      "activations/layer6_attention_weight_min": -50.048458099365234,
      "activations/layer7_attention_weight_max": 101.55388641357422,
      "activations/layer7_attention_weight_min": -104.62835693359375,
      "activations/layer8_attention_weight_max": 48.54999542236328,
      "activations/layer8_attention_weight_min": -48.2835578918457,
      "activations/layer9_attention_weight_max": 35.09695816040039,
      "activations/layer9_attention_weight_min": -38.370277404785156,
      "epoch": 22.78,
      "learning_rate": 3.1075757575757573e-06,
      "loss": 2.6955,
      "step": 392100
    },
    {
      "activations/layer0_attention_weight_max": 16.114940643310547,
      "activations/layer0_attention_weight_min": -13.308096885681152,
      "activations/layer10_attention_weight_max": 36.62943649291992,
      "activations/layer10_attention_weight_min": -40.25139617919922,
      "activations/layer11_attention_weight_max": 36.824493408203125,
      "activations/layer11_attention_weight_min": -40.339725494384766,
      "activations/layer12_attention_weight_max": 29.508010864257812,
      "activations/layer12_attention_weight_min": -30.858165740966797,
      "activations/layer13_attention_weight_max": 46.970924377441406,
      "activations/layer13_attention_weight_min": -43.632530212402344,
      "activations/layer14_attention_weight_max": 46.03191375732422,
      "activations/layer14_attention_weight_min": -48.268524169921875,
      "activations/layer15_attention_weight_max": 42.63423538208008,
      "activations/layer15_attention_weight_min": -43.113929748535156,
      "activations/layer16_attention_weight_max": 32.03568649291992,
      "activations/layer16_attention_weight_min": -30.248010635375977,
      "activations/layer17_attention_weight_max": 31.52604866027832,
      "activations/layer17_attention_weight_min": -26.3608341217041,
      "activations/layer18_attention_weight_max": 32.70154571533203,
      "activations/layer18_attention_weight_min": -27.175870895385742,
      "activations/layer19_attention_weight_max": 34.557857513427734,
      "activations/layer19_attention_weight_min": -33.389495849609375,
      "activations/layer1_attention_weight_max": 15.094627380371094,
      "activations/layer1_attention_weight_min": -14.791708946228027,
      "activations/layer20_attention_weight_max": 27.387908935546875,
      "activations/layer20_attention_weight_min": -25.70696449279785,
      "activations/layer21_attention_weight_max": 27.886104583740234,
      "activations/layer21_attention_weight_min": -26.4066219329834,
      "activations/layer22_attention_weight_max": 43.13946533203125,
      "activations/layer22_attention_weight_min": -33.62317657470703,
      "activations/layer23_attention_weight_max": 32.36164474487305,
      "activations/layer23_attention_weight_min": -25.873668670654297,
      "activations/layer2_attention_weight_max": 34.87144470214844,
      "activations/layer2_attention_weight_min": -33.497798919677734,
      "activations/layer3_attention_weight_max": 109.25370025634766,
      "activations/layer3_attention_weight_min": -109.2426986694336,
      "activations/layer4_attention_weight_max": 113.29622650146484,
      "activations/layer4_attention_weight_min": -115.34453582763672,
      "activations/layer5_attention_weight_max": 50.454097747802734,
      "activations/layer5_attention_weight_min": -63.35679626464844,
      "activations/layer6_attention_weight_max": 48.19105911254883,
      "activations/layer6_attention_weight_min": -49.10447311401367,
      "activations/layer7_attention_weight_max": 98.80392456054688,
      "activations/layer7_attention_weight_min": -105.77527618408203,
      "activations/layer8_attention_weight_max": 47.36205291748047,
      "activations/layer8_attention_weight_min": -50.11418533325195,
      "activations/layer9_attention_weight_max": 34.7877082824707,
      "activations/layer9_attention_weight_min": -38.85810852050781,
      "epoch": 22.79,
      "learning_rate": 3.0886363636363634e-06,
      "loss": 2.703,
      "step": 392150
    },
    {
      "activations/layer0_attention_weight_max": 17.49995994567871,
      "activations/layer0_attention_weight_min": -15.114577293395996,
      "activations/layer10_attention_weight_max": 35.412689208984375,
      "activations/layer10_attention_weight_min": -35.62272644042969,
      "activations/layer11_attention_weight_max": 35.36350631713867,
      "activations/layer11_attention_weight_min": -37.446624755859375,
      "activations/layer12_attention_weight_max": 25.916688919067383,
      "activations/layer12_attention_weight_min": -27.448068618774414,
      "activations/layer13_attention_weight_max": 36.62067794799805,
      "activations/layer13_attention_weight_min": -35.80060958862305,
      "activations/layer14_attention_weight_max": 38.83879089355469,
      "activations/layer14_attention_weight_min": -36.21086502075195,
      "activations/layer15_attention_weight_max": 36.55202865600586,
      "activations/layer15_attention_weight_min": -36.04692077636719,
      "activations/layer16_attention_weight_max": 26.55797576904297,
      "activations/layer16_attention_weight_min": -27.564353942871094,
      "activations/layer17_attention_weight_max": 27.340726852416992,
      "activations/layer17_attention_weight_min": -26.570831298828125,
      "activations/layer18_attention_weight_max": 32.838714599609375,
      "activations/layer18_attention_weight_min": -26.34934425354004,
      "activations/layer19_attention_weight_max": 30.965139389038086,
      "activations/layer19_attention_weight_min": -31.6130428314209,
      "activations/layer1_attention_weight_max": 16.106651306152344,
      "activations/layer1_attention_weight_min": -15.677135467529297,
      "activations/layer20_attention_weight_max": 26.39434051513672,
      "activations/layer20_attention_weight_min": -24.836824417114258,
      "activations/layer21_attention_weight_max": 25.858016967773438,
      "activations/layer21_attention_weight_min": -25.089799880981445,
      "activations/layer22_attention_weight_max": 39.73003005981445,
      "activations/layer22_attention_weight_min": -31.924293518066406,
      "activations/layer23_attention_weight_max": 28.475399017333984,
      "activations/layer23_attention_weight_min": -25.79143524169922,
      "activations/layer2_attention_weight_max": 34.18586349487305,
      "activations/layer2_attention_weight_min": -34.004539489746094,
      "activations/layer3_attention_weight_max": 107.3040542602539,
      "activations/layer3_attention_weight_min": -109.55107116699219,
      "activations/layer4_attention_weight_max": 111.55167388916016,
      "activations/layer4_attention_weight_min": -116.9466781616211,
      "activations/layer5_attention_weight_max": 50.50440979003906,
      "activations/layer5_attention_weight_min": -61.865440368652344,
      "activations/layer6_attention_weight_max": 46.34358215332031,
      "activations/layer6_attention_weight_min": -51.57680892944336,
      "activations/layer7_attention_weight_max": 96.18690490722656,
      "activations/layer7_attention_weight_min": -105.50391387939453,
      "activations/layer8_attention_weight_max": 40.86125946044922,
      "activations/layer8_attention_weight_min": -45.245521545410156,
      "activations/layer9_attention_weight_max": 32.24489974975586,
      "activations/layer9_attention_weight_min": -37.053794860839844,
      "epoch": 22.79,
      "learning_rate": 3.069696969696969e-06,
      "loss": 2.7113,
      "step": 392200
    },
    {
      "activations/layer0_attention_weight_max": 16.824642181396484,
      "activations/layer0_attention_weight_min": -13.794560432434082,
      "activations/layer10_attention_weight_max": 36.019779205322266,
      "activations/layer10_attention_weight_min": -38.52123260498047,
      "activations/layer11_attention_weight_max": 36.86371612548828,
      "activations/layer11_attention_weight_min": -37.95231628417969,
      "activations/layer12_attention_weight_max": 27.97171401977539,
      "activations/layer12_attention_weight_min": -28.00162696838379,
      "activations/layer13_attention_weight_max": 42.18579864501953,
      "activations/layer13_attention_weight_min": -37.014381408691406,
      "activations/layer14_attention_weight_max": 44.25474548339844,
      "activations/layer14_attention_weight_min": -38.90233612060547,
      "activations/layer15_attention_weight_max": 44.01367950439453,
      "activations/layer15_attention_weight_min": -38.23638916015625,
      "activations/layer16_attention_weight_max": 32.51422119140625,
      "activations/layer16_attention_weight_min": -28.364233016967773,
      "activations/layer17_attention_weight_max": 33.29954147338867,
      "activations/layer17_attention_weight_min": -27.38128089904785,
      "activations/layer18_attention_weight_max": 38.41996383666992,
      "activations/layer18_attention_weight_min": -28.107877731323242,
      "activations/layer19_attention_weight_max": 40.83291244506836,
      "activations/layer19_attention_weight_min": -34.114227294921875,
      "activations/layer1_attention_weight_max": 15.288712501525879,
      "activations/layer1_attention_weight_min": -15.132686614990234,
      "activations/layer20_attention_weight_max": 32.75822830200195,
      "activations/layer20_attention_weight_min": -27.6016788482666,
      "activations/layer21_attention_weight_max": 31.753408432006836,
      "activations/layer21_attention_weight_min": -24.930633544921875,
      "activations/layer22_attention_weight_max": 47.62383270263672,
      "activations/layer22_attention_weight_min": -32.7496452331543,
      "activations/layer23_attention_weight_max": 38.976356506347656,
      "activations/layer23_attention_weight_min": -25.343250274658203,
      "activations/layer2_attention_weight_max": 33.98387908935547,
      "activations/layer2_attention_weight_min": -34.64513397216797,
      "activations/layer3_attention_weight_max": 111.67970275878906,
      "activations/layer3_attention_weight_min": -111.68569946289062,
      "activations/layer4_attention_weight_max": 114.67076873779297,
      "activations/layer4_attention_weight_min": -120.96407318115234,
      "activations/layer5_attention_weight_max": 52.66752624511719,
      "activations/layer5_attention_weight_min": -63.828060150146484,
      "activations/layer6_attention_weight_max": 46.528526306152344,
      "activations/layer6_attention_weight_min": -52.67756652832031,
      "activations/layer7_attention_weight_max": 99.3197250366211,
      "activations/layer7_attention_weight_min": -111.9241943359375,
      "activations/layer8_attention_weight_max": 44.55203628540039,
      "activations/layer8_attention_weight_min": -50.115943908691406,
      "activations/layer9_attention_weight_max": 34.94234848022461,
      "activations/layer9_attention_weight_min": -37.11762619018555,
      "epoch": 22.79,
      "learning_rate": 3.0507575757575757e-06,
      "loss": 2.6893,
      "step": 392250
    },
    {
      "activations/layer0_attention_weight_max": 16.917259216308594,
      "activations/layer0_attention_weight_min": -17.22858428955078,
      "activations/layer10_attention_weight_max": 38.70022201538086,
      "activations/layer10_attention_weight_min": -38.85990905761719,
      "activations/layer11_attention_weight_max": 39.72863006591797,
      "activations/layer11_attention_weight_min": -39.18107604980469,
      "activations/layer12_attention_weight_max": 30.439537048339844,
      "activations/layer12_attention_weight_min": -30.61327362060547,
      "activations/layer13_attention_weight_max": 43.18153381347656,
      "activations/layer13_attention_weight_min": -42.21888732910156,
      "activations/layer14_attention_weight_max": 46.96875762939453,
      "activations/layer14_attention_weight_min": -43.232421875,
      "activations/layer15_attention_weight_max": 45.3526496887207,
      "activations/layer15_attention_weight_min": -39.9947509765625,
      "activations/layer16_attention_weight_max": 31.971097946166992,
      "activations/layer16_attention_weight_min": -29.85765838623047,
      "activations/layer17_attention_weight_max": 33.01987838745117,
      "activations/layer17_attention_weight_min": -28.092124938964844,
      "activations/layer18_attention_weight_max": 35.74473571777344,
      "activations/layer18_attention_weight_min": -27.97266960144043,
      "activations/layer19_attention_weight_max": 33.47882080078125,
      "activations/layer19_attention_weight_min": -33.77839660644531,
      "activations/layer1_attention_weight_max": 15.807411193847656,
      "activations/layer1_attention_weight_min": -16.440143585205078,
      "activations/layer20_attention_weight_max": 29.0903263092041,
      "activations/layer20_attention_weight_min": -25.262004852294922,
      "activations/layer21_attention_weight_max": 28.022703170776367,
      "activations/layer21_attention_weight_min": -23.41550636291504,
      "activations/layer22_attention_weight_max": 42.2999153137207,
      "activations/layer22_attention_weight_min": -30.524873733520508,
      "activations/layer23_attention_weight_max": 32.220550537109375,
      "activations/layer23_attention_weight_min": -24.980009078979492,
      "activations/layer2_attention_weight_max": 35.03533172607422,
      "activations/layer2_attention_weight_min": -32.919376373291016,
      "activations/layer3_attention_weight_max": 110.89987182617188,
      "activations/layer3_attention_weight_min": -109.65894317626953,
      "activations/layer4_attention_weight_max": 116.72484588623047,
      "activations/layer4_attention_weight_min": -119.04559326171875,
      "activations/layer5_attention_weight_max": 53.26821517944336,
      "activations/layer5_attention_weight_min": -64.29635620117188,
      "activations/layer6_attention_weight_max": 47.412654876708984,
      "activations/layer6_attention_weight_min": -52.32499694824219,
      "activations/layer7_attention_weight_max": 103.02733612060547,
      "activations/layer7_attention_weight_min": -109.09725952148438,
      "activations/layer8_attention_weight_max": 48.75603103637695,
      "activations/layer8_attention_weight_min": -51.03739547729492,
      "activations/layer9_attention_weight_max": 36.295318603515625,
      "activations/layer9_attention_weight_min": -38.16011047363281,
      "epoch": 22.79,
      "learning_rate": 3.0318181818181814e-06,
      "loss": 2.715,
      "step": 392300
    },
    {
      "activations/layer0_attention_weight_max": 16.058902740478516,
      "activations/layer0_attention_weight_min": -12.768491744995117,
      "activations/layer10_attention_weight_max": 36.18883514404297,
      "activations/layer10_attention_weight_min": -38.438575744628906,
      "activations/layer11_attention_weight_max": 35.621849060058594,
      "activations/layer11_attention_weight_min": -40.047237396240234,
      "activations/layer12_attention_weight_max": 28.70734214782715,
      "activations/layer12_attention_weight_min": -29.31134605407715,
      "activations/layer13_attention_weight_max": 41.82896041870117,
      "activations/layer13_attention_weight_min": -40.621726989746094,
      "activations/layer14_attention_weight_max": 44.25277328491211,
      "activations/layer14_attention_weight_min": -41.30783462524414,
      "activations/layer15_attention_weight_max": 42.73345947265625,
      "activations/layer15_attention_weight_min": -44.43016052246094,
      "activations/layer16_attention_weight_max": 32.33671951293945,
      "activations/layer16_attention_weight_min": -32.10411071777344,
      "activations/layer17_attention_weight_max": 33.31373596191406,
      "activations/layer17_attention_weight_min": -30.36746597290039,
      "activations/layer18_attention_weight_max": 33.92453384399414,
      "activations/layer18_attention_weight_min": -30.64374351501465,
      "activations/layer19_attention_weight_max": 40.69813919067383,
      "activations/layer19_attention_weight_min": -35.04072952270508,
      "activations/layer1_attention_weight_max": 17.442718505859375,
      "activations/layer1_attention_weight_min": -14.798351287841797,
      "activations/layer20_attention_weight_max": 29.4103946685791,
      "activations/layer20_attention_weight_min": -26.349979400634766,
      "activations/layer21_attention_weight_max": 27.752378463745117,
      "activations/layer21_attention_weight_min": -26.760751724243164,
      "activations/layer22_attention_weight_max": 45.69793701171875,
      "activations/layer22_attention_weight_min": -38.005836486816406,
      "activations/layer23_attention_weight_max": 34.72604751586914,
      "activations/layer23_attention_weight_min": -29.338653564453125,
      "activations/layer2_attention_weight_max": 35.99250030517578,
      "activations/layer2_attention_weight_min": -34.64188766479492,
      "activations/layer3_attention_weight_max": 108.76019287109375,
      "activations/layer3_attention_weight_min": -113.31829071044922,
      "activations/layer4_attention_weight_max": 114.87520599365234,
      "activations/layer4_attention_weight_min": -120.5211181640625,
      "activations/layer5_attention_weight_max": 52.105201721191406,
      "activations/layer5_attention_weight_min": -64.9759292602539,
      "activations/layer6_attention_weight_max": 47.66902160644531,
      "activations/layer6_attention_weight_min": -51.76348114013672,
      "activations/layer7_attention_weight_max": 99.80322265625,
      "activations/layer7_attention_weight_min": -105.09233093261719,
      "activations/layer8_attention_weight_max": 44.68013381958008,
      "activations/layer8_attention_weight_min": -49.28711700439453,
      "activations/layer9_attention_weight_max": 34.9853630065918,
      "activations/layer9_attention_weight_min": -38.30675506591797,
      "epoch": 22.8,
      "learning_rate": 3.0128787878787876e-06,
      "loss": 2.6923,
      "step": 392350
    },
    {
      "activations/layer0_attention_weight_max": 17.194656372070312,
      "activations/layer0_attention_weight_min": -15.66932201385498,
      "activations/layer10_attention_weight_max": 39.43218231201172,
      "activations/layer10_attention_weight_min": -39.76776123046875,
      "activations/layer11_attention_weight_max": 40.029144287109375,
      "activations/layer11_attention_weight_min": -40.2091064453125,
      "activations/layer12_attention_weight_max": 32.007423400878906,
      "activations/layer12_attention_weight_min": -30.729597091674805,
      "activations/layer13_attention_weight_max": 43.72220230102539,
      "activations/layer13_attention_weight_min": -42.66443634033203,
      "activations/layer14_attention_weight_max": 49.73847198486328,
      "activations/layer14_attention_weight_min": -45.33127975463867,
      "activations/layer15_attention_weight_max": 46.98371124267578,
      "activations/layer15_attention_weight_min": -42.893394470214844,
      "activations/layer16_attention_weight_max": 30.72196388244629,
      "activations/layer16_attention_weight_min": -30.392866134643555,
      "activations/layer17_attention_weight_max": 34.08053970336914,
      "activations/layer17_attention_weight_min": -29.27410316467285,
      "activations/layer18_attention_weight_max": 32.28983688354492,
      "activations/layer18_attention_weight_min": -27.743547439575195,
      "activations/layer19_attention_weight_max": 35.254276275634766,
      "activations/layer19_attention_weight_min": -32.430213928222656,
      "activations/layer1_attention_weight_max": 17.049516677856445,
      "activations/layer1_attention_weight_min": -15.513912200927734,
      "activations/layer20_attention_weight_max": 30.77522850036621,
      "activations/layer20_attention_weight_min": -26.253087997436523,
      "activations/layer21_attention_weight_max": 32.02927017211914,
      "activations/layer21_attention_weight_min": -27.871000289916992,
      "activations/layer22_attention_weight_max": 43.605648040771484,
      "activations/layer22_attention_weight_min": -34.9327278137207,
      "activations/layer23_attention_weight_max": 36.56819152832031,
      "activations/layer23_attention_weight_min": -28.070343017578125,
      "activations/layer2_attention_weight_max": 35.607154846191406,
      "activations/layer2_attention_weight_min": -34.09550094604492,
      "activations/layer3_attention_weight_max": 109.26703643798828,
      "activations/layer3_attention_weight_min": -110.28826141357422,
      "activations/layer4_attention_weight_max": 112.5870361328125,
      "activations/layer4_attention_weight_min": -115.85758209228516,
      "activations/layer5_attention_weight_max": 52.69129180908203,
      "activations/layer5_attention_weight_min": -63.261695861816406,
      "activations/layer6_attention_weight_max": 46.11112976074219,
      "activations/layer6_attention_weight_min": -48.062862396240234,
      "activations/layer7_attention_weight_max": 101.78036499023438,
      "activations/layer7_attention_weight_min": -108.15533447265625,
      "activations/layer8_attention_weight_max": 45.851688385009766,
      "activations/layer8_attention_weight_min": -49.535667419433594,
      "activations/layer9_attention_weight_max": 33.892677307128906,
      "activations/layer9_attention_weight_min": -37.65282440185547,
      "epoch": 22.8,
      "learning_rate": 2.9939393939393933e-06,
      "loss": 2.701,
      "step": 392400
    },
    {
      "activations/layer0_attention_weight_max": 16.50033950805664,
      "activations/layer0_attention_weight_min": -14.499751091003418,
      "activations/layer10_attention_weight_max": 36.508506774902344,
      "activations/layer10_attention_weight_min": -35.8590087890625,
      "activations/layer11_attention_weight_max": 35.34075927734375,
      "activations/layer11_attention_weight_min": -35.442996978759766,
      "activations/layer12_attention_weight_max": 28.44331169128418,
      "activations/layer12_attention_weight_min": -28.073251724243164,
      "activations/layer13_attention_weight_max": 43.255531311035156,
      "activations/layer13_attention_weight_min": -37.176246643066406,
      "activations/layer14_attention_weight_max": 46.819129943847656,
      "activations/layer14_attention_weight_min": -39.928367614746094,
      "activations/layer15_attention_weight_max": 42.5706901550293,
      "activations/layer15_attention_weight_min": -39.365413665771484,
      "activations/layer16_attention_weight_max": 33.01829147338867,
      "activations/layer16_attention_weight_min": -29.66708755493164,
      "activations/layer17_attention_weight_max": 36.15665054321289,
      "activations/layer17_attention_weight_min": -30.043046951293945,
      "activations/layer18_attention_weight_max": 35.98521041870117,
      "activations/layer18_attention_weight_min": -28.96965217590332,
      "activations/layer19_attention_weight_max": 39.956443786621094,
      "activations/layer19_attention_weight_min": -32.15263748168945,
      "activations/layer1_attention_weight_max": 15.589997291564941,
      "activations/layer1_attention_weight_min": -15.06169319152832,
      "activations/layer20_attention_weight_max": 36.50871276855469,
      "activations/layer20_attention_weight_min": -25.062353134155273,
      "activations/layer21_attention_weight_max": 33.440799713134766,
      "activations/layer21_attention_weight_min": -24.115581512451172,
      "activations/layer22_attention_weight_max": 46.517372131347656,
      "activations/layer22_attention_weight_min": -29.95056915283203,
      "activations/layer23_attention_weight_max": 37.5572624206543,
      "activations/layer23_attention_weight_min": -27.09203338623047,
      "activations/layer2_attention_weight_max": 36.28431701660156,
      "activations/layer2_attention_weight_min": -34.944091796875,
      "activations/layer3_attention_weight_max": 110.01763153076172,
      "activations/layer3_attention_weight_min": -111.30780029296875,
      "activations/layer4_attention_weight_max": 115.36107635498047,
      "activations/layer4_attention_weight_min": -116.81037902832031,
      "activations/layer5_attention_weight_max": 51.9943962097168,
      "activations/layer5_attention_weight_min": -64.0987777709961,
      "activations/layer6_attention_weight_max": 47.92725372314453,
      "activations/layer6_attention_weight_min": -50.77854537963867,
      "activations/layer7_attention_weight_max": 105.80397033691406,
      "activations/layer7_attention_weight_min": -103.07470703125,
      "activations/layer8_attention_weight_max": 44.17893600463867,
      "activations/layer8_attention_weight_min": -48.2239875793457,
      "activations/layer9_attention_weight_max": 36.400672912597656,
      "activations/layer9_attention_weight_min": -35.39552307128906,
      "epoch": 22.8,
      "learning_rate": 2.975e-06,
      "loss": 2.6994,
      "step": 392450
    },
    {
      "activations/layer0_attention_weight_max": 17.514589309692383,
      "activations/layer0_attention_weight_min": -14.120426177978516,
      "activations/layer10_attention_weight_max": 37.05910110473633,
      "activations/layer10_attention_weight_min": -36.74721908569336,
      "activations/layer11_attention_weight_max": 35.98780059814453,
      "activations/layer11_attention_weight_min": -37.477577209472656,
      "activations/layer12_attention_weight_max": 27.763629913330078,
      "activations/layer12_attention_weight_min": -28.749704360961914,
      "activations/layer13_attention_weight_max": 43.10325241088867,
      "activations/layer13_attention_weight_min": -38.68601608276367,
      "activations/layer14_attention_weight_max": 43.879310607910156,
      "activations/layer14_attention_weight_min": -41.579505920410156,
      "activations/layer15_attention_weight_max": 42.93209457397461,
      "activations/layer15_attention_weight_min": -38.58464813232422,
      "activations/layer16_attention_weight_max": 29.871931076049805,
      "activations/layer16_attention_weight_min": -29.289155960083008,
      "activations/layer17_attention_weight_max": 31.90108871459961,
      "activations/layer17_attention_weight_min": -27.864253997802734,
      "activations/layer18_attention_weight_max": 30.833396911621094,
      "activations/layer18_attention_weight_min": -25.042091369628906,
      "activations/layer19_attention_weight_max": 37.8352165222168,
      "activations/layer19_attention_weight_min": -32.832218170166016,
      "activations/layer1_attention_weight_max": 16.253740310668945,
      "activations/layer1_attention_weight_min": -15.213113784790039,
      "activations/layer20_attention_weight_max": 26.886388778686523,
      "activations/layer20_attention_weight_min": -25.895397186279297,
      "activations/layer21_attention_weight_max": 26.824405670166016,
      "activations/layer21_attention_weight_min": -23.825359344482422,
      "activations/layer22_attention_weight_max": 41.992919921875,
      "activations/layer22_attention_weight_min": -29.31006622314453,
      "activations/layer23_attention_weight_max": 31.138769149780273,
      "activations/layer23_attention_weight_min": -24.939178466796875,
      "activations/layer2_attention_weight_max": 34.80735778808594,
      "activations/layer2_attention_weight_min": -34.94207763671875,
      "activations/layer3_attention_weight_max": 112.36621856689453,
      "activations/layer3_attention_weight_min": -110.38188934326172,
      "activations/layer4_attention_weight_max": 116.28057098388672,
      "activations/layer4_attention_weight_min": -117.14025115966797,
      "activations/layer5_attention_weight_max": 52.245784759521484,
      "activations/layer5_attention_weight_min": -64.21629333496094,
      "activations/layer6_attention_weight_max": 49.12834167480469,
      "activations/layer6_attention_weight_min": -50.484432220458984,
      "activations/layer7_attention_weight_max": 102.04993438720703,
      "activations/layer7_attention_weight_min": -109.60808563232422,
      "activations/layer8_attention_weight_max": 45.08363723754883,
      "activations/layer8_attention_weight_min": -47.66798400878906,
      "activations/layer9_attention_weight_max": 34.17133712768555,
      "activations/layer9_attention_weight_min": -36.68057632446289,
      "epoch": 22.81,
      "learning_rate": 2.956060606060606e-06,
      "loss": 2.6968,
      "step": 392500
    },
    {
      "activations/layer0_attention_weight_max": 17.117382049560547,
      "activations/layer0_attention_weight_min": -16.19776153564453,
      "activations/layer10_attention_weight_max": 37.50859451293945,
      "activations/layer10_attention_weight_min": -37.83167266845703,
      "activations/layer11_attention_weight_max": 37.48631286621094,
      "activations/layer11_attention_weight_min": -37.95120620727539,
      "activations/layer12_attention_weight_max": 30.91086196899414,
      "activations/layer12_attention_weight_min": -30.37791633605957,
      "activations/layer13_attention_weight_max": 43.42325973510742,
      "activations/layer13_attention_weight_min": -38.04938888549805,
      "activations/layer14_attention_weight_max": 45.52048110961914,
      "activations/layer14_attention_weight_min": -39.63743591308594,
      "activations/layer15_attention_weight_max": 43.140037536621094,
      "activations/layer15_attention_weight_min": -39.039913177490234,
      "activations/layer16_attention_weight_max": 28.884775161743164,
      "activations/layer16_attention_weight_min": -28.488985061645508,
      "activations/layer17_attention_weight_max": 30.130619049072266,
      "activations/layer17_attention_weight_min": -25.86597442626953,
      "activations/layer18_attention_weight_max": 31.380826950073242,
      "activations/layer18_attention_weight_min": -25.34951400756836,
      "activations/layer19_attention_weight_max": 33.849815368652344,
      "activations/layer19_attention_weight_min": -31.207834243774414,
      "activations/layer1_attention_weight_max": 16.414052963256836,
      "activations/layer1_attention_weight_min": -15.27380084991455,
      "activations/layer20_attention_weight_max": 28.703840255737305,
      "activations/layer20_attention_weight_min": -24.481077194213867,
      "activations/layer21_attention_weight_max": 29.20420265197754,
      "activations/layer21_attention_weight_min": -27.109115600585938,
      "activations/layer22_attention_weight_max": 41.20530319213867,
      "activations/layer22_attention_weight_min": -30.397294998168945,
      "activations/layer23_attention_weight_max": 32.0887451171875,
      "activations/layer23_attention_weight_min": -26.928081512451172,
      "activations/layer2_attention_weight_max": 36.26856231689453,
      "activations/layer2_attention_weight_min": -33.85459899902344,
      "activations/layer3_attention_weight_max": 111.83448791503906,
      "activations/layer3_attention_weight_min": -111.8107681274414,
      "activations/layer4_attention_weight_max": 115.8174819946289,
      "activations/layer4_attention_weight_min": -119.25587463378906,
      "activations/layer5_attention_weight_max": 55.37000274658203,
      "activations/layer5_attention_weight_min": -61.529518127441406,
      "activations/layer6_attention_weight_max": 50.190895080566406,
      "activations/layer6_attention_weight_min": -51.21116638183594,
      "activations/layer7_attention_weight_max": 107.1800537109375,
      "activations/layer7_attention_weight_min": -108.15707397460938,
      "activations/layer8_attention_weight_max": 49.15690231323242,
      "activations/layer8_attention_weight_min": -48.849613189697266,
      "activations/layer9_attention_weight_max": 35.7232780456543,
      "activations/layer9_attention_weight_min": -37.08941650390625,
      "epoch": 22.81,
      "learning_rate": 2.9371212121212118e-06,
      "loss": 2.7059,
      "step": 392550
    },
    {
      "activations/layer0_attention_weight_max": 16.27931022644043,
      "activations/layer0_attention_weight_min": -13.762924194335938,
      "activations/layer10_attention_weight_max": 41.44979476928711,
      "activations/layer10_attention_weight_min": -41.010353088378906,
      "activations/layer11_attention_weight_max": 43.288787841796875,
      "activations/layer11_attention_weight_min": -42.60192108154297,
      "activations/layer12_attention_weight_max": 27.801671981811523,
      "activations/layer12_attention_weight_min": -30.40045738220215,
      "activations/layer13_attention_weight_max": 40.68235397338867,
      "activations/layer13_attention_weight_min": -37.97951126098633,
      "activations/layer14_attention_weight_max": 46.363182067871094,
      "activations/layer14_attention_weight_min": -46.10843276977539,
      "activations/layer15_attention_weight_max": 47.383480072021484,
      "activations/layer15_attention_weight_min": -41.34640121459961,
      "activations/layer16_attention_weight_max": 32.20155715942383,
      "activations/layer16_attention_weight_min": -29.49508285522461,
      "activations/layer17_attention_weight_max": 30.977882385253906,
      "activations/layer17_attention_weight_min": -29.027713775634766,
      "activations/layer18_attention_weight_max": 36.39033126831055,
      "activations/layer18_attention_weight_min": -30.464725494384766,
      "activations/layer19_attention_weight_max": 40.890037536621094,
      "activations/layer19_attention_weight_min": -34.36370086669922,
      "activations/layer1_attention_weight_max": 16.354459762573242,
      "activations/layer1_attention_weight_min": -16.26030731201172,
      "activations/layer20_attention_weight_max": 36.02409744262695,
      "activations/layer20_attention_weight_min": -27.373329162597656,
      "activations/layer21_attention_weight_max": 33.68967819213867,
      "activations/layer21_attention_weight_min": -26.483171463012695,
      "activations/layer22_attention_weight_max": 47.40536880493164,
      "activations/layer22_attention_weight_min": -34.44963836669922,
      "activations/layer23_attention_weight_max": 33.26215362548828,
      "activations/layer23_attention_weight_min": -26.774335861206055,
      "activations/layer2_attention_weight_max": 34.393131256103516,
      "activations/layer2_attention_weight_min": -35.21977233886719,
      "activations/layer3_attention_weight_max": 110.03084564208984,
      "activations/layer3_attention_weight_min": -115.56163024902344,
      "activations/layer4_attention_weight_max": 113.81013488769531,
      "activations/layer4_attention_weight_min": -116.7179183959961,
      "activations/layer5_attention_weight_max": 52.488731384277344,
      "activations/layer5_attention_weight_min": -60.43794250488281,
      "activations/layer6_attention_weight_max": 48.75316619873047,
      "activations/layer6_attention_weight_min": -51.47616958618164,
      "activations/layer7_attention_weight_max": 109.89669036865234,
      "activations/layer7_attention_weight_min": -114.02818298339844,
      "activations/layer8_attention_weight_max": 50.90314483642578,
      "activations/layer8_attention_weight_min": -56.3897590637207,
      "activations/layer9_attention_weight_max": 36.30659103393555,
      "activations/layer9_attention_weight_min": -40.699058532714844,
      "epoch": 22.81,
      "learning_rate": 2.918181818181818e-06,
      "loss": 2.7014,
      "step": 392600
    },
    {
      "activations/layer0_attention_weight_max": 17.35586929321289,
      "activations/layer0_attention_weight_min": -14.48088264465332,
      "activations/layer10_attention_weight_max": 39.404911041259766,
      "activations/layer10_attention_weight_min": -37.00677490234375,
      "activations/layer11_attention_weight_max": 36.104652404785156,
      "activations/layer11_attention_weight_min": -36.31611251831055,
      "activations/layer12_attention_weight_max": 28.835538864135742,
      "activations/layer12_attention_weight_min": -27.992172241210938,
      "activations/layer13_attention_weight_max": 43.32386779785156,
      "activations/layer13_attention_weight_min": -35.53278732299805,
      "activations/layer14_attention_weight_max": 44.98653030395508,
      "activations/layer14_attention_weight_min": -39.34794235229492,
      "activations/layer15_attention_weight_max": 40.31890869140625,
      "activations/layer15_attention_weight_min": -37.74703598022461,
      "activations/layer16_attention_weight_max": 31.019973754882812,
      "activations/layer16_attention_weight_min": -28.826065063476562,
      "activations/layer17_attention_weight_max": 31.50015640258789,
      "activations/layer17_attention_weight_min": -26.470605850219727,
      "activations/layer18_attention_weight_max": 32.339820861816406,
      "activations/layer18_attention_weight_min": -26.745697021484375,
      "activations/layer19_attention_weight_max": 33.707828521728516,
      "activations/layer19_attention_weight_min": -31.704364776611328,
      "activations/layer1_attention_weight_max": 16.19757843017578,
      "activations/layer1_attention_weight_min": -15.783615112304688,
      "activations/layer20_attention_weight_max": 30.491437911987305,
      "activations/layer20_attention_weight_min": -27.259132385253906,
      "activations/layer21_attention_weight_max": 30.84478759765625,
      "activations/layer21_attention_weight_min": -26.498443603515625,
      "activations/layer22_attention_weight_max": 42.19635772705078,
      "activations/layer22_attention_weight_min": -32.75977325439453,
      "activations/layer23_attention_weight_max": 31.88861083984375,
      "activations/layer23_attention_weight_min": -25.713529586791992,
      "activations/layer2_attention_weight_max": 35.67920684814453,
      "activations/layer2_attention_weight_min": -34.14303207397461,
      "activations/layer3_attention_weight_max": 112.39971923828125,
      "activations/layer3_attention_weight_min": -114.80709838867188,
      "activations/layer4_attention_weight_max": 113.3408432006836,
      "activations/layer4_attention_weight_min": -116.333740234375,
      "activations/layer5_attention_weight_max": 52.77757263183594,
      "activations/layer5_attention_weight_min": -62.65068054199219,
      "activations/layer6_attention_weight_max": 45.60530090332031,
      "activations/layer6_attention_weight_min": -50.451576232910156,
      "activations/layer7_attention_weight_max": 97.56343078613281,
      "activations/layer7_attention_weight_min": -106.35198974609375,
      "activations/layer8_attention_weight_max": 45.32301712036133,
      "activations/layer8_attention_weight_min": -47.28024673461914,
      "activations/layer9_attention_weight_max": 35.18009567260742,
      "activations/layer9_attention_weight_min": -35.70687484741211,
      "epoch": 22.82,
      "learning_rate": 2.899242424242424e-06,
      "loss": 2.685,
      "step": 392650
    },
    {
      "activations/layer0_attention_weight_max": 16.71824836730957,
      "activations/layer0_attention_weight_min": -14.693450927734375,
      "activations/layer10_attention_weight_max": 45.47151565551758,
      "activations/layer10_attention_weight_min": -43.01898956298828,
      "activations/layer11_attention_weight_max": 46.072853088378906,
      "activations/layer11_attention_weight_min": -44.96549987792969,
      "activations/layer12_attention_weight_max": 29.503278732299805,
      "activations/layer12_attention_weight_min": -30.23619270324707,
      "activations/layer13_attention_weight_max": 44.02310562133789,
      "activations/layer13_attention_weight_min": -38.671852111816406,
      "activations/layer14_attention_weight_max": 46.17877197265625,
      "activations/layer14_attention_weight_min": -40.1695671081543,
      "activations/layer15_attention_weight_max": 41.82150650024414,
      "activations/layer15_attention_weight_min": -43.33232879638672,
      "activations/layer16_attention_weight_max": 29.55735206604004,
      "activations/layer16_attention_weight_min": -27.42694091796875,
      "activations/layer17_attention_weight_max": 30.214759826660156,
      "activations/layer17_attention_weight_min": -26.902284622192383,
      "activations/layer18_attention_weight_max": 31.268409729003906,
      "activations/layer18_attention_weight_min": -25.3520565032959,
      "activations/layer19_attention_weight_max": 32.20698165893555,
      "activations/layer19_attention_weight_min": -32.48867416381836,
      "activations/layer1_attention_weight_max": 16.557231903076172,
      "activations/layer1_attention_weight_min": -15.0428466796875,
      "activations/layer20_attention_weight_max": 27.009092330932617,
      "activations/layer20_attention_weight_min": -23.885635375976562,
      "activations/layer21_attention_weight_max": 29.23041343688965,
      "activations/layer21_attention_weight_min": -25.575271606445312,
      "activations/layer22_attention_weight_max": 38.99045181274414,
      "activations/layer22_attention_weight_min": -30.05340576171875,
      "activations/layer23_attention_weight_max": 32.584739685058594,
      "activations/layer23_attention_weight_min": -25.153186798095703,
      "activations/layer2_attention_weight_max": 34.70466613769531,
      "activations/layer2_attention_weight_min": -33.19160461425781,
      "activations/layer3_attention_weight_max": 108.3141098022461,
      "activations/layer3_attention_weight_min": -114.28085327148438,
      "activations/layer4_attention_weight_max": 112.99397277832031,
      "activations/layer4_attention_weight_min": -118.14322662353516,
      "activations/layer5_attention_weight_max": 53.500179290771484,
      "activations/layer5_attention_weight_min": -63.759273529052734,
      "activations/layer6_attention_weight_max": 47.801414489746094,
      "activations/layer6_attention_weight_min": -49.659053802490234,
      "activations/layer7_attention_weight_max": 111.21916198730469,
      "activations/layer7_attention_weight_min": -99.259765625,
      "activations/layer8_attention_weight_max": 51.248268127441406,
      "activations/layer8_attention_weight_min": -48.849422454833984,
      "activations/layer9_attention_weight_max": 37.30877685546875,
      "activations/layer9_attention_weight_min": -37.44801330566406,
      "epoch": 22.82,
      "learning_rate": 2.8803030303030302e-06,
      "loss": 2.685,
      "step": 392700
    },
    {
      "activations/layer0_attention_weight_max": 17.722036361694336,
      "activations/layer0_attention_weight_min": -14.389991760253906,
      "activations/layer10_attention_weight_max": 35.55077362060547,
      "activations/layer10_attention_weight_min": -36.015380859375,
      "activations/layer11_attention_weight_max": 37.90104675292969,
      "activations/layer11_attention_weight_min": -36.08547592163086,
      "activations/layer12_attention_weight_max": 29.490705490112305,
      "activations/layer12_attention_weight_min": -27.42430877685547,
      "activations/layer13_attention_weight_max": 44.073978424072266,
      "activations/layer13_attention_weight_min": -37.794288635253906,
      "activations/layer14_attention_weight_max": 45.70351028442383,
      "activations/layer14_attention_weight_min": -39.56571578979492,
      "activations/layer15_attention_weight_max": 42.98112106323242,
      "activations/layer15_attention_weight_min": -39.745208740234375,
      "activations/layer16_attention_weight_max": 30.14449119567871,
      "activations/layer16_attention_weight_min": -30.796960830688477,
      "activations/layer17_attention_weight_max": 30.14362907409668,
      "activations/layer17_attention_weight_min": -28.306657791137695,
      "activations/layer18_attention_weight_max": 34.64995574951172,
      "activations/layer18_attention_weight_min": -28.4796199798584,
      "activations/layer19_attention_weight_max": 37.04880905151367,
      "activations/layer19_attention_weight_min": -34.86880111694336,
      "activations/layer1_attention_weight_max": 15.445416450500488,
      "activations/layer1_attention_weight_min": -15.813761711120605,
      "activations/layer20_attention_weight_max": 29.4710693359375,
      "activations/layer20_attention_weight_min": -28.503210067749023,
      "activations/layer21_attention_weight_max": 28.1337890625,
      "activations/layer21_attention_weight_min": -27.069528579711914,
      "activations/layer22_attention_weight_max": 39.15642166137695,
      "activations/layer22_attention_weight_min": -34.29305648803711,
      "activations/layer23_attention_weight_max": 32.89775848388672,
      "activations/layer23_attention_weight_min": -27.56271743774414,
      "activations/layer2_attention_weight_max": 35.00205993652344,
      "activations/layer2_attention_weight_min": -34.30491256713867,
      "activations/layer3_attention_weight_max": 111.34685516357422,
      "activations/layer3_attention_weight_min": -114.18289184570312,
      "activations/layer4_attention_weight_max": 115.064208984375,
      "activations/layer4_attention_weight_min": -116.52494049072266,
      "activations/layer5_attention_weight_max": 56.501060485839844,
      "activations/layer5_attention_weight_min": -64.41232299804688,
      "activations/layer6_attention_weight_max": 46.67670440673828,
      "activations/layer6_attention_weight_min": -50.98884582519531,
      "activations/layer7_attention_weight_max": 100.10066986083984,
      "activations/layer7_attention_weight_min": -102.53756713867188,
      "activations/layer8_attention_weight_max": 44.08897018432617,
      "activations/layer8_attention_weight_min": -45.92615509033203,
      "activations/layer9_attention_weight_max": 33.608795166015625,
      "activations/layer9_attention_weight_min": -35.76686096191406,
      "epoch": 22.82,
      "learning_rate": 2.861363636363636e-06,
      "loss": 2.6917,
      "step": 392750
    },
    {
      "activations/layer0_attention_weight_max": 16.41712760925293,
      "activations/layer0_attention_weight_min": -13.945013046264648,
      "activations/layer10_attention_weight_max": 37.06306838989258,
      "activations/layer10_attention_weight_min": -39.7804069519043,
      "activations/layer11_attention_weight_max": 35.951011657714844,
      "activations/layer11_attention_weight_min": -37.56211853027344,
      "activations/layer12_attention_weight_max": 29.597713470458984,
      "activations/layer12_attention_weight_min": -29.063648223876953,
      "activations/layer13_attention_weight_max": 43.24834060668945,
      "activations/layer13_attention_weight_min": -40.369014739990234,
      "activations/layer14_attention_weight_max": 47.7224006652832,
      "activations/layer14_attention_weight_min": -43.115943908691406,
      "activations/layer15_attention_weight_max": 44.233158111572266,
      "activations/layer15_attention_weight_min": -41.08266830444336,
      "activations/layer16_attention_weight_max": 30.715002059936523,
      "activations/layer16_attention_weight_min": -29.743797302246094,
      "activations/layer17_attention_weight_max": 34.439666748046875,
      "activations/layer17_attention_weight_min": -28.61283302307129,
      "activations/layer18_attention_weight_max": 35.80540466308594,
      "activations/layer18_attention_weight_min": -28.547603607177734,
      "activations/layer19_attention_weight_max": 35.8538932800293,
      "activations/layer19_attention_weight_min": -34.10102844238281,
      "activations/layer1_attention_weight_max": 16.678388595581055,
      "activations/layer1_attention_weight_min": -16.11652183532715,
      "activations/layer20_attention_weight_max": 30.246421813964844,
      "activations/layer20_attention_weight_min": -27.970735549926758,
      "activations/layer21_attention_weight_max": 36.0239372253418,
      "activations/layer21_attention_weight_min": -28.91264533996582,
      "activations/layer22_attention_weight_max": 48.20729064941406,
      "activations/layer22_attention_weight_min": -37.16948699951172,
      "activations/layer23_attention_weight_max": 36.16398239135742,
      "activations/layer23_attention_weight_min": -27.262313842773438,
      "activations/layer2_attention_weight_max": 34.818328857421875,
      "activations/layer2_attention_weight_min": -34.17477035522461,
      "activations/layer3_attention_weight_max": 110.55164337158203,
      "activations/layer3_attention_weight_min": -113.856201171875,
      "activations/layer4_attention_weight_max": 114.7965316772461,
      "activations/layer4_attention_weight_min": -118.07002258300781,
      "activations/layer5_attention_weight_max": 59.276546478271484,
      "activations/layer5_attention_weight_min": -62.07311248779297,
      "activations/layer6_attention_weight_max": 51.8194465637207,
      "activations/layer6_attention_weight_min": -52.21134948730469,
      "activations/layer7_attention_weight_max": 106.37696838378906,
      "activations/layer7_attention_weight_min": -112.77180480957031,
      "activations/layer8_attention_weight_max": 48.552650451660156,
      "activations/layer8_attention_weight_min": -51.587711334228516,
      "activations/layer9_attention_weight_max": 36.3392448425293,
      "activations/layer9_attention_weight_min": -40.06475067138672,
      "epoch": 22.82,
      "learning_rate": 2.842424242424242e-06,
      "loss": 2.694,
      "step": 392800
    },
    {
      "activations/layer0_attention_weight_max": 17.001108169555664,
      "activations/layer0_attention_weight_min": -14.7088041305542,
      "activations/layer10_attention_weight_max": 34.41858673095703,
      "activations/layer10_attention_weight_min": -35.42108154296875,
      "activations/layer11_attention_weight_max": 35.71228790283203,
      "activations/layer11_attention_weight_min": -36.93126678466797,
      "activations/layer12_attention_weight_max": 28.41493034362793,
      "activations/layer12_attention_weight_min": -26.778478622436523,
      "activations/layer13_attention_weight_max": 44.746639251708984,
      "activations/layer13_attention_weight_min": -38.953269958496094,
      "activations/layer14_attention_weight_max": 45.96929168701172,
      "activations/layer14_attention_weight_min": -39.26667022705078,
      "activations/layer15_attention_weight_max": 42.37983322143555,
      "activations/layer15_attention_weight_min": -38.24972915649414,
      "activations/layer16_attention_weight_max": 30.971202850341797,
      "activations/layer16_attention_weight_min": -28.586381912231445,
      "activations/layer17_attention_weight_max": 30.19257354736328,
      "activations/layer17_attention_weight_min": -27.328840255737305,
      "activations/layer18_attention_weight_max": 31.692758560180664,
      "activations/layer18_attention_weight_min": -26.85545539855957,
      "activations/layer19_attention_weight_max": 33.396995544433594,
      "activations/layer19_attention_weight_min": -31.006317138671875,
      "activations/layer1_attention_weight_max": 18.12715721130371,
      "activations/layer1_attention_weight_min": -15.396346092224121,
      "activations/layer20_attention_weight_max": 29.709144592285156,
      "activations/layer20_attention_weight_min": -25.129858016967773,
      "activations/layer21_attention_weight_max": 31.19738006591797,
      "activations/layer21_attention_weight_min": -23.639629364013672,
      "activations/layer22_attention_weight_max": 42.76324462890625,
      "activations/layer22_attention_weight_min": -31.402408599853516,
      "activations/layer23_attention_weight_max": 28.71487045288086,
      "activations/layer23_attention_weight_min": -24.731489181518555,
      "activations/layer2_attention_weight_max": 34.241302490234375,
      "activations/layer2_attention_weight_min": -33.58850860595703,
      "activations/layer3_attention_weight_max": 107.3423843383789,
      "activations/layer3_attention_weight_min": -107.11665344238281,
      "activations/layer4_attention_weight_max": 111.83756256103516,
      "activations/layer4_attention_weight_min": -110.95466613769531,
      "activations/layer5_attention_weight_max": 50.841651916503906,
      "activations/layer5_attention_weight_min": -64.27399444580078,
      "activations/layer6_attention_weight_max": 46.708335876464844,
      "activations/layer6_attention_weight_min": -48.1439094543457,
      "activations/layer7_attention_weight_max": 98.03738403320312,
      "activations/layer7_attention_weight_min": -100.79796600341797,
      "activations/layer8_attention_weight_max": 43.57775115966797,
      "activations/layer8_attention_weight_min": -46.41365432739258,
      "activations/layer9_attention_weight_max": 32.70674514770508,
      "activations/layer9_attention_weight_min": -35.041629791259766,
      "epoch": 22.83,
      "learning_rate": 2.823484848484848e-06,
      "loss": 2.6994,
      "step": 392850
    },
    {
      "activations/layer0_attention_weight_max": 16.058868408203125,
      "activations/layer0_attention_weight_min": -13.757363319396973,
      "activations/layer10_attention_weight_max": 34.720436096191406,
      "activations/layer10_attention_weight_min": -34.71617889404297,
      "activations/layer11_attention_weight_max": 36.16701889038086,
      "activations/layer11_attention_weight_min": -35.50920486450195,
      "activations/layer12_attention_weight_max": 28.483102798461914,
      "activations/layer12_attention_weight_min": -27.7603816986084,
      "activations/layer13_attention_weight_max": 43.36823272705078,
      "activations/layer13_attention_weight_min": -36.83576202392578,
      "activations/layer14_attention_weight_max": 43.760276794433594,
      "activations/layer14_attention_weight_min": -41.11738967895508,
      "activations/layer15_attention_weight_max": 42.33212661743164,
      "activations/layer15_attention_weight_min": -39.409934997558594,
      "activations/layer16_attention_weight_max": 32.085941314697266,
      "activations/layer16_attention_weight_min": -30.105934143066406,
      "activations/layer17_attention_weight_max": 34.07695007324219,
      "activations/layer17_attention_weight_min": -29.67003059387207,
      "activations/layer18_attention_weight_max": 35.04103088378906,
      "activations/layer18_attention_weight_min": -25.741729736328125,
      "activations/layer19_attention_weight_max": 36.66151809692383,
      "activations/layer19_attention_weight_min": -32.16209411621094,
      "activations/layer1_attention_weight_max": 17.06818389892578,
      "activations/layer1_attention_weight_min": -18.1995792388916,
      "activations/layer20_attention_weight_max": 32.005393981933594,
      "activations/layer20_attention_weight_min": -26.13176918029785,
      "activations/layer21_attention_weight_max": 29.39778709411621,
      "activations/layer21_attention_weight_min": -24.7076416015625,
      "activations/layer22_attention_weight_max": 49.84819793701172,
      "activations/layer22_attention_weight_min": -33.67017364501953,
      "activations/layer23_attention_weight_max": 33.35295867919922,
      "activations/layer23_attention_weight_min": -25.11928367614746,
      "activations/layer2_attention_weight_max": 37.527462005615234,
      "activations/layer2_attention_weight_min": -34.95915222167969,
      "activations/layer3_attention_weight_max": 109.62407684326172,
      "activations/layer3_attention_weight_min": -106.52510070800781,
      "activations/layer4_attention_weight_max": 110.76859283447266,
      "activations/layer4_attention_weight_min": -113.011962890625,
      "activations/layer5_attention_weight_max": 51.70994567871094,
      "activations/layer5_attention_weight_min": -62.00086212158203,
      "activations/layer6_attention_weight_max": 45.1698112487793,
      "activations/layer6_attention_weight_min": -50.237815856933594,
      "activations/layer7_attention_weight_max": 98.24037170410156,
      "activations/layer7_attention_weight_min": -105.31120300292969,
      "activations/layer8_attention_weight_max": 42.407859802246094,
      "activations/layer8_attention_weight_min": -47.549068450927734,
      "activations/layer9_attention_weight_max": 32.87373352050781,
      "activations/layer9_attention_weight_min": -36.07223129272461,
      "epoch": 22.83,
      "learning_rate": 2.8045454545454544e-06,
      "loss": 2.6898,
      "step": 392900
    },
    {
      "activations/layer0_attention_weight_max": 16.551176071166992,
      "activations/layer0_attention_weight_min": -13.292670249938965,
      "activations/layer10_attention_weight_max": 37.059261322021484,
      "activations/layer10_attention_weight_min": -38.8782844543457,
      "activations/layer11_attention_weight_max": 36.00938415527344,
      "activations/layer11_attention_weight_min": -35.91395568847656,
      "activations/layer12_attention_weight_max": 28.481000900268555,
      "activations/layer12_attention_weight_min": -27.113204956054688,
      "activations/layer13_attention_weight_max": 40.455596923828125,
      "activations/layer13_attention_weight_min": -37.53471374511719,
      "activations/layer14_attention_weight_max": 48.66709899902344,
      "activations/layer14_attention_weight_min": -38.673213958740234,
      "activations/layer15_attention_weight_max": 42.146663665771484,
      "activations/layer15_attention_weight_min": -36.89339065551758,
      "activations/layer16_attention_weight_max": 30.291955947875977,
      "activations/layer16_attention_weight_min": -29.754199981689453,
      "activations/layer17_attention_weight_max": 30.62112808227539,
      "activations/layer17_attention_weight_min": -26.99802017211914,
      "activations/layer18_attention_weight_max": 35.00368881225586,
      "activations/layer18_attention_weight_min": -25.984848022460938,
      "activations/layer19_attention_weight_max": 35.110557556152344,
      "activations/layer19_attention_weight_min": -30.580703735351562,
      "activations/layer1_attention_weight_max": 16.651485443115234,
      "activations/layer1_attention_weight_min": -15.635592460632324,
      "activations/layer20_attention_weight_max": 27.598552703857422,
      "activations/layer20_attention_weight_min": -25.753061294555664,
      "activations/layer21_attention_weight_max": 27.176219940185547,
      "activations/layer21_attention_weight_min": -25.108732223510742,
      "activations/layer22_attention_weight_max": 41.6697998046875,
      "activations/layer22_attention_weight_min": -31.697509765625,
      "activations/layer23_attention_weight_max": 31.016796112060547,
      "activations/layer23_attention_weight_min": -27.252700805664062,
      "activations/layer2_attention_weight_max": 36.06327819824219,
      "activations/layer2_attention_weight_min": -34.1701774597168,
      "activations/layer3_attention_weight_max": 113.30740356445312,
      "activations/layer3_attention_weight_min": -111.4601058959961,
      "activations/layer4_attention_weight_max": 116.40103912353516,
      "activations/layer4_attention_weight_min": -116.23359680175781,
      "activations/layer5_attention_weight_max": 54.12933349609375,
      "activations/layer5_attention_weight_min": -67.59246063232422,
      "activations/layer6_attention_weight_max": 49.97114181518555,
      "activations/layer6_attention_weight_min": -50.60610580444336,
      "activations/layer7_attention_weight_max": 99.94950866699219,
      "activations/layer7_attention_weight_min": -111.55906677246094,
      "activations/layer8_attention_weight_max": 45.167091369628906,
      "activations/layer8_attention_weight_min": -50.06669998168945,
      "activations/layer9_attention_weight_max": 33.87490463256836,
      "activations/layer9_attention_weight_min": -37.94993209838867,
      "epoch": 22.83,
      "learning_rate": 2.7856060606060606e-06,
      "loss": 2.6968,
      "step": 392950
    },
    {
      "activations/layer0_attention_weight_max": 17.342039108276367,
      "activations/layer0_attention_weight_min": -15.467809677124023,
      "activations/layer10_attention_weight_max": 33.49946975708008,
      "activations/layer10_attention_weight_min": -37.0865364074707,
      "activations/layer11_attention_weight_max": 36.02925109863281,
      "activations/layer11_attention_weight_min": -36.35667419433594,
      "activations/layer12_attention_weight_max": 29.297998428344727,
      "activations/layer12_attention_weight_min": -28.5479793548584,
      "activations/layer13_attention_weight_max": 41.90120315551758,
      "activations/layer13_attention_weight_min": -39.57558059692383,
      "activations/layer14_attention_weight_max": 43.72904968261719,
      "activations/layer14_attention_weight_min": -40.1153450012207,
      "activations/layer15_attention_weight_max": 41.230857849121094,
      "activations/layer15_attention_weight_min": -39.815574645996094,
      "activations/layer16_attention_weight_max": 30.20920753479004,
      "activations/layer16_attention_weight_min": -31.226516723632812,
      "activations/layer17_attention_weight_max": 31.294519424438477,
      "activations/layer17_attention_weight_min": -27.2263126373291,
      "activations/layer18_attention_weight_max": 35.04127883911133,
      "activations/layer18_attention_weight_min": -27.644060134887695,
      "activations/layer19_attention_weight_max": 36.05615997314453,
      "activations/layer19_attention_weight_min": -32.08683776855469,
      "activations/layer1_attention_weight_max": 16.012731552124023,
      "activations/layer1_attention_weight_min": -14.6885404586792,
      "activations/layer20_attention_weight_max": 29.640300750732422,
      "activations/layer20_attention_weight_min": -25.3928279876709,
      "activations/layer21_attention_weight_max": 31.208402633666992,
      "activations/layer21_attention_weight_min": -26.083419799804688,
      "activations/layer22_attention_weight_max": 40.77602005004883,
      "activations/layer22_attention_weight_min": -33.69198989868164,
      "activations/layer23_attention_weight_max": 35.16774368286133,
      "activations/layer23_attention_weight_min": -26.67622184753418,
      "activations/layer2_attention_weight_max": 34.000877380371094,
      "activations/layer2_attention_weight_min": -33.54463195800781,
      "activations/layer3_attention_weight_max": 108.95124053955078,
      "activations/layer3_attention_weight_min": -108.2046890258789,
      "activations/layer4_attention_weight_max": 113.81218719482422,
      "activations/layer4_attention_weight_min": -118.7269515991211,
      "activations/layer5_attention_weight_max": 53.94425582885742,
      "activations/layer5_attention_weight_min": -64.9895248413086,
      "activations/layer6_attention_weight_max": 46.71421432495117,
      "activations/layer6_attention_weight_min": -49.21236801147461,
      "activations/layer7_attention_weight_max": 99.49842834472656,
      "activations/layer7_attention_weight_min": -106.1038818359375,
      "activations/layer8_attention_weight_max": 42.783416748046875,
      "activations/layer8_attention_weight_min": -44.65371322631836,
      "activations/layer9_attention_weight_max": 32.05657196044922,
      "activations/layer9_attention_weight_min": -34.608192443847656,
      "epoch": 22.84,
      "learning_rate": 2.7666666666666663e-06,
      "loss": 2.6964,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.462,
      "eval_samples_per_second": 507.443,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.462,
      "eval_openwebtext_samples_per_second": 507.443,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9292,
      "eval_wikitext_samples_per_second": 236.362,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_lambada_loss": 2.318359375,
      "eval_lambada_ppl": 10.158993527607587,
      "eval_lambada_runtime": 9.5265,
      "eval_lambada_samples_per_second": 511.099,
      "step": 393000
    },
    {
      "activations/layer0_attention_weight_max": 16.753936767578125,
      "activations/layer0_attention_weight_min": -14.761272430419922,
      "activations/layer10_attention_weight_max": 36.60327911376953,
      "activations/layer10_attention_weight_min": -36.709564208984375,
      "activations/layer11_attention_weight_max": 35.92295837402344,
      "activations/layer11_attention_weight_min": -35.499122619628906,
      "activations/layer12_attention_weight_max": 27.678268432617188,
      "activations/layer12_attention_weight_min": -27.361719131469727,
      "activations/layer13_attention_weight_max": 38.55438232421875,
      "activations/layer13_attention_weight_min": -37.349388122558594,
      "activations/layer14_attention_weight_max": 40.80330276489258,
      "activations/layer14_attention_weight_min": -37.83091354370117,
      "activations/layer15_attention_weight_max": 36.37102127075195,
      "activations/layer15_attention_weight_min": -37.59006118774414,
      "activations/layer16_attention_weight_max": 30.377347946166992,
      "activations/layer16_attention_weight_min": -29.48240852355957,
      "activations/layer17_attention_weight_max": 28.67551040649414,
      "activations/layer17_attention_weight_min": -28.039804458618164,
      "activations/layer18_attention_weight_max": 31.902103424072266,
      "activations/layer18_attention_weight_min": -27.101116180419922,
      "activations/layer19_attention_weight_max": 34.830631256103516,
      "activations/layer19_attention_weight_min": -31.999713897705078,
      "activations/layer1_attention_weight_max": 16.135225296020508,
      "activations/layer1_attention_weight_min": -15.76001262664795,
      "activations/layer20_attention_weight_max": 29.24819564819336,
      "activations/layer20_attention_weight_min": -26.248796463012695,
      "activations/layer21_attention_weight_max": 28.067710876464844,
      "activations/layer21_attention_weight_min": -25.215646743774414,
      "activations/layer22_attention_weight_max": 45.79156494140625,
      "activations/layer22_attention_weight_min": -34.55324172973633,
      "activations/layer23_attention_weight_max": 30.726776123046875,
      "activations/layer23_attention_weight_min": -27.53836441040039,
      "activations/layer2_attention_weight_max": 34.45088195800781,
      "activations/layer2_attention_weight_min": -34.83126449584961,
      "activations/layer3_attention_weight_max": 109.86511993408203,
      "activations/layer3_attention_weight_min": -112.6120376586914,
      "activations/layer4_attention_weight_max": 112.74051666259766,
      "activations/layer4_attention_weight_min": -116.60462951660156,
      "activations/layer5_attention_weight_max": 51.322669982910156,
      "activations/layer5_attention_weight_min": -63.0787353515625,
      "activations/layer6_attention_weight_max": 46.207069396972656,
      "activations/layer6_attention_weight_min": -49.319984436035156,
      "activations/layer7_attention_weight_max": 96.23141479492188,
      "activations/layer7_attention_weight_min": -102.30970001220703,
      "activations/layer8_attention_weight_max": 43.50978088378906,
      "activations/layer8_attention_weight_min": -46.54452896118164,
      "activations/layer9_attention_weight_max": 33.663665771484375,
      "activations/layer9_attention_weight_min": -35.21645736694336,
      "epoch": 22.84,
      "learning_rate": 2.7477272727272724e-06,
      "loss": 2.7084,
      "step": 393050
    },
    {
      "activations/layer0_attention_weight_max": 15.931707382202148,
      "activations/layer0_attention_weight_min": -15.123353958129883,
      "activations/layer10_attention_weight_max": 35.11458969116211,
      "activations/layer10_attention_weight_min": -35.31781005859375,
      "activations/layer11_attention_weight_max": 35.015899658203125,
      "activations/layer11_attention_weight_min": -35.02849578857422,
      "activations/layer12_attention_weight_max": 26.90558433532715,
      "activations/layer12_attention_weight_min": -26.30645179748535,
      "activations/layer13_attention_weight_max": 41.575889587402344,
      "activations/layer13_attention_weight_min": -36.0885009765625,
      "activations/layer14_attention_weight_max": 44.03994369506836,
      "activations/layer14_attention_weight_min": -38.884708404541016,
      "activations/layer15_attention_weight_max": 40.52549362182617,
      "activations/layer15_attention_weight_min": -36.894744873046875,
      "activations/layer16_attention_weight_max": 32.76118087768555,
      "activations/layer16_attention_weight_min": -28.579307556152344,
      "activations/layer17_attention_weight_max": 31.64665412902832,
      "activations/layer17_attention_weight_min": -27.858360290527344,
      "activations/layer18_attention_weight_max": 35.18672180175781,
      "activations/layer18_attention_weight_min": -28.84663200378418,
      "activations/layer19_attention_weight_max": 35.40669631958008,
      "activations/layer19_attention_weight_min": -30.999542236328125,
      "activations/layer1_attention_weight_max": 15.85279655456543,
      "activations/layer1_attention_weight_min": -15.637735366821289,
      "activations/layer20_attention_weight_max": 30.74846076965332,
      "activations/layer20_attention_weight_min": -25.891555786132812,
      "activations/layer21_attention_weight_max": 29.073312759399414,
      "activations/layer21_attention_weight_min": -23.2492618560791,
      "activations/layer22_attention_weight_max": 47.36663055419922,
      "activations/layer22_attention_weight_min": -30.314695358276367,
      "activations/layer23_attention_weight_max": 32.50105285644531,
      "activations/layer23_attention_weight_min": -24.674957275390625,
      "activations/layer2_attention_weight_max": 34.21373748779297,
      "activations/layer2_attention_weight_min": -32.73200225830078,
      "activations/layer3_attention_weight_max": 109.14292907714844,
      "activations/layer3_attention_weight_min": -107.31229400634766,
      "activations/layer4_attention_weight_max": 111.1492919921875,
      "activations/layer4_attention_weight_min": -113.62053680419922,
      "activations/layer5_attention_weight_max": 49.62653350830078,
      "activations/layer5_attention_weight_min": -63.28054428100586,
      "activations/layer6_attention_weight_max": 46.24049758911133,
      "activations/layer6_attention_weight_min": -46.07365417480469,
      "activations/layer7_attention_weight_max": 94.08644104003906,
      "activations/layer7_attention_weight_min": -102.13468933105469,
      "activations/layer8_attention_weight_max": 42.34786605834961,
      "activations/layer8_attention_weight_min": -44.256690979003906,
      "activations/layer9_attention_weight_max": 32.12653732299805,
      "activations/layer9_attention_weight_min": -34.42417526245117,
      "epoch": 22.84,
      "learning_rate": 2.7287878787878786e-06,
      "loss": 2.6843,
      "step": 393100
    },
    {
      "activations/layer0_attention_weight_max": 16.898035049438477,
      "activations/layer0_attention_weight_min": -14.685358047485352,
      "activations/layer10_attention_weight_max": 34.832149505615234,
      "activations/layer10_attention_weight_min": -34.79738235473633,
      "activations/layer11_attention_weight_max": 33.81563186645508,
      "activations/layer11_attention_weight_min": -36.00314712524414,
      "activations/layer12_attention_weight_max": 28.045408248901367,
      "activations/layer12_attention_weight_min": -26.953149795532227,
      "activations/layer13_attention_weight_max": 40.39088439941406,
      "activations/layer13_attention_weight_min": -35.683834075927734,
      "activations/layer14_attention_weight_max": 44.201988220214844,
      "activations/layer14_attention_weight_min": -38.261390686035156,
      "activations/layer15_attention_weight_max": 42.258975982666016,
      "activations/layer15_attention_weight_min": -37.12217712402344,
      "activations/layer16_attention_weight_max": 31.426342010498047,
      "activations/layer16_attention_weight_min": -29.902725219726562,
      "activations/layer17_attention_weight_max": 32.51724624633789,
      "activations/layer17_attention_weight_min": -27.454578399658203,
      "activations/layer18_attention_weight_max": 35.260902404785156,
      "activations/layer18_attention_weight_min": -26.84735679626465,
      "activations/layer19_attention_weight_max": 38.52145004272461,
      "activations/layer19_attention_weight_min": -33.15470504760742,
      "activations/layer1_attention_weight_max": 16.13658332824707,
      "activations/layer1_attention_weight_min": -16.409692764282227,
      "activations/layer20_attention_weight_max": 30.429386138916016,
      "activations/layer20_attention_weight_min": -26.98775291442871,
      "activations/layer21_attention_weight_max": 31.907833099365234,
      "activations/layer21_attention_weight_min": -28.30763816833496,
      "activations/layer22_attention_weight_max": 46.712669372558594,
      "activations/layer22_attention_weight_min": -35.888301849365234,
      "activations/layer23_attention_weight_max": 35.6009635925293,
      "activations/layer23_attention_weight_min": -25.750354766845703,
      "activations/layer2_attention_weight_max": 35.92756271362305,
      "activations/layer2_attention_weight_min": -32.90504837036133,
      "activations/layer3_attention_weight_max": 105.6457290649414,
      "activations/layer3_attention_weight_min": -104.81503295898438,
      "activations/layer4_attention_weight_max": 109.77803039550781,
      "activations/layer4_attention_weight_min": -113.85060119628906,
      "activations/layer5_attention_weight_max": 52.22276306152344,
      "activations/layer5_attention_weight_min": -61.38302230834961,
      "activations/layer6_attention_weight_max": 48.20295715332031,
      "activations/layer6_attention_weight_min": -48.966758728027344,
      "activations/layer7_attention_weight_max": 98.830322265625,
      "activations/layer7_attention_weight_min": -98.78880310058594,
      "activations/layer8_attention_weight_max": 45.10526657104492,
      "activations/layer8_attention_weight_min": -44.639305114746094,
      "activations/layer9_attention_weight_max": 33.3700065612793,
      "activations/layer9_attention_weight_min": -35.091922760009766,
      "epoch": 22.84,
      "learning_rate": 2.7098484848484848e-06,
      "loss": 2.6936,
      "step": 393150
    },
    {
      "activations/layer0_attention_weight_max": 16.740276336669922,
      "activations/layer0_attention_weight_min": -14.80952262878418,
      "activations/layer10_attention_weight_max": 35.460960388183594,
      "activations/layer10_attention_weight_min": -35.63299560546875,
      "activations/layer11_attention_weight_max": 39.32215881347656,
      "activations/layer11_attention_weight_min": -37.01995849609375,
      "activations/layer12_attention_weight_max": 27.313114166259766,
      "activations/layer12_attention_weight_min": -26.97471046447754,
      "activations/layer13_attention_weight_max": 42.51402282714844,
      "activations/layer13_attention_weight_min": -37.61800003051758,
      "activations/layer14_attention_weight_max": 42.72539520263672,
      "activations/layer14_attention_weight_min": -38.41096115112305,
      "activations/layer15_attention_weight_max": 41.72716522216797,
      "activations/layer15_attention_weight_min": -39.87746810913086,
      "activations/layer16_attention_weight_max": 30.874666213989258,
      "activations/layer16_attention_weight_min": -28.589841842651367,
      "activations/layer17_attention_weight_max": 30.140485763549805,
      "activations/layer17_attention_weight_min": -26.337125778198242,
      "activations/layer18_attention_weight_max": 39.00974655151367,
      "activations/layer18_attention_weight_min": -26.975500106811523,
      "activations/layer19_attention_weight_max": 37.85272216796875,
      "activations/layer19_attention_weight_min": -33.48799133300781,
      "activations/layer1_attention_weight_max": 15.733919143676758,
      "activations/layer1_attention_weight_min": -15.355230331420898,
      "activations/layer20_attention_weight_max": 30.184314727783203,
      "activations/layer20_attention_weight_min": -24.13971519470215,
      "activations/layer21_attention_weight_max": 31.086135864257812,
      "activations/layer21_attention_weight_min": -24.298574447631836,
      "activations/layer22_attention_weight_max": 44.55157470703125,
      "activations/layer22_attention_weight_min": -31.41864776611328,
      "activations/layer23_attention_weight_max": 37.3480224609375,
      "activations/layer23_attention_weight_min": -28.00362205505371,
      "activations/layer2_attention_weight_max": 34.37821578979492,
      "activations/layer2_attention_weight_min": -34.32468795776367,
      "activations/layer3_attention_weight_max": 111.48927307128906,
      "activations/layer3_attention_weight_min": -113.7890853881836,
      "activations/layer4_attention_weight_max": 116.02587127685547,
      "activations/layer4_attention_weight_min": -118.6489028930664,
      "activations/layer5_attention_weight_max": 53.99235534667969,
      "activations/layer5_attention_weight_min": -62.33622741699219,
      "activations/layer6_attention_weight_max": 46.9760856628418,
      "activations/layer6_attention_weight_min": -50.29990005493164,
      "activations/layer7_attention_weight_max": 113.14810180664062,
      "activations/layer7_attention_weight_min": -107.68236541748047,
      "activations/layer8_attention_weight_max": 43.58434295654297,
      "activations/layer8_attention_weight_min": -47.53247833251953,
      "activations/layer9_attention_weight_max": 33.7322883605957,
      "activations/layer9_attention_weight_min": -36.29792022705078,
      "epoch": 22.85,
      "learning_rate": 2.6909090909090905e-06,
      "loss": 2.6839,
      "step": 393200
    },
    {
      "activations/layer0_attention_weight_max": 17.22490882873535,
      "activations/layer0_attention_weight_min": -13.652969360351562,
      "activations/layer10_attention_weight_max": 41.82903289794922,
      "activations/layer10_attention_weight_min": -38.53507614135742,
      "activations/layer11_attention_weight_max": 41.6605339050293,
      "activations/layer11_attention_weight_min": -38.704689025878906,
      "activations/layer12_attention_weight_max": 29.11448097229004,
      "activations/layer12_attention_weight_min": -29.23110580444336,
      "activations/layer13_attention_weight_max": 42.138214111328125,
      "activations/layer13_attention_weight_min": -36.090736389160156,
      "activations/layer14_attention_weight_max": 45.26762008666992,
      "activations/layer14_attention_weight_min": -41.12772750854492,
      "activations/layer15_attention_weight_max": 43.082908630371094,
      "activations/layer15_attention_weight_min": -41.18027877807617,
      "activations/layer16_attention_weight_max": 32.108306884765625,
      "activations/layer16_attention_weight_min": -30.46311378479004,
      "activations/layer17_attention_weight_max": 29.844743728637695,
      "activations/layer17_attention_weight_min": -28.801069259643555,
      "activations/layer18_attention_weight_max": 37.03059768676758,
      "activations/layer18_attention_weight_min": -31.190519332885742,
      "activations/layer19_attention_weight_max": 35.751502990722656,
      "activations/layer19_attention_weight_min": -33.93889617919922,
      "activations/layer1_attention_weight_max": 15.95492935180664,
      "activations/layer1_attention_weight_min": -15.186352729797363,
      "activations/layer20_attention_weight_max": 29.840526580810547,
      "activations/layer20_attention_weight_min": -25.10856819152832,
      "activations/layer21_attention_weight_max": 28.874691009521484,
      "activations/layer21_attention_weight_min": -26.213653564453125,
      "activations/layer22_attention_weight_max": 43.629241943359375,
      "activations/layer22_attention_weight_min": -37.02748107910156,
      "activations/layer23_attention_weight_max": 31.184978485107422,
      "activations/layer23_attention_weight_min": -29.977783203125,
      "activations/layer2_attention_weight_max": 33.64737319946289,
      "activations/layer2_attention_weight_min": -35.013790130615234,
      "activations/layer3_attention_weight_max": 110.43112182617188,
      "activations/layer3_attention_weight_min": -112.42955017089844,
      "activations/layer4_attention_weight_max": 109.44921112060547,
      "activations/layer4_attention_weight_min": -117.43009185791016,
      "activations/layer5_attention_weight_max": 56.534454345703125,
      "activations/layer5_attention_weight_min": -60.44680404663086,
      "activations/layer6_attention_weight_max": 44.829505920410156,
      "activations/layer6_attention_weight_min": -52.9434700012207,
      "activations/layer7_attention_weight_max": 111.98906707763672,
      "activations/layer7_attention_weight_min": -106.211181640625,
      "activations/layer8_attention_weight_max": 51.95329666137695,
      "activations/layer8_attention_weight_min": -48.51996612548828,
      "activations/layer9_attention_weight_max": 34.374576568603516,
      "activations/layer9_attention_weight_min": -37.00246810913086,
      "epoch": 22.85,
      "learning_rate": 2.6719696969696966e-06,
      "loss": 2.6949,
      "step": 393250
    },
    {
      "activations/layer0_attention_weight_max": 17.31031608581543,
      "activations/layer0_attention_weight_min": -14.014659881591797,
      "activations/layer10_attention_weight_max": 44.802188873291016,
      "activations/layer10_attention_weight_min": -40.463279724121094,
      "activations/layer11_attention_weight_max": 39.51957702636719,
      "activations/layer11_attention_weight_min": -41.862483978271484,
      "activations/layer12_attention_weight_max": 29.755155563354492,
      "activations/layer12_attention_weight_min": -27.760005950927734,
      "activations/layer13_attention_weight_max": 40.61613845825195,
      "activations/layer13_attention_weight_min": -39.08424377441406,
      "activations/layer14_attention_weight_max": 43.221466064453125,
      "activations/layer14_attention_weight_min": -40.12913513183594,
      "activations/layer15_attention_weight_max": 37.90644073486328,
      "activations/layer15_attention_weight_min": -36.58198928833008,
      "activations/layer16_attention_weight_max": 28.884408950805664,
      "activations/layer16_attention_weight_min": -28.207185745239258,
      "activations/layer17_attention_weight_max": 29.096981048583984,
      "activations/layer17_attention_weight_min": -26.173479080200195,
      "activations/layer18_attention_weight_max": 31.5181884765625,
      "activations/layer18_attention_weight_min": -25.701635360717773,
      "activations/layer19_attention_weight_max": 31.096120834350586,
      "activations/layer19_attention_weight_min": -31.07416343688965,
      "activations/layer1_attention_weight_max": 17.452707290649414,
      "activations/layer1_attention_weight_min": -15.334319114685059,
      "activations/layer20_attention_weight_max": 27.158967971801758,
      "activations/layer20_attention_weight_min": -24.45342445373535,
      "activations/layer21_attention_weight_max": 26.48588752746582,
      "activations/layer21_attention_weight_min": -23.911510467529297,
      "activations/layer22_attention_weight_max": 41.14706802368164,
      "activations/layer22_attention_weight_min": -32.13663864135742,
      "activations/layer23_attention_weight_max": 31.358020782470703,
      "activations/layer23_attention_weight_min": -25.771392822265625,
      "activations/layer2_attention_weight_max": 33.75080871582031,
      "activations/layer2_attention_weight_min": -32.63323211669922,
      "activations/layer3_attention_weight_max": 106.99838256835938,
      "activations/layer3_attention_weight_min": -107.2649917602539,
      "activations/layer4_attention_weight_max": 111.94042205810547,
      "activations/layer4_attention_weight_min": -113.39058685302734,
      "activations/layer5_attention_weight_max": 57.99517059326172,
      "activations/layer5_attention_weight_min": -62.6607780456543,
      "activations/layer6_attention_weight_max": 47.427242279052734,
      "activations/layer6_attention_weight_min": -50.39683151245117,
      "activations/layer7_attention_weight_max": 114.73688507080078,
      "activations/layer7_attention_weight_min": -108.88853454589844,
      "activations/layer8_attention_weight_max": 53.99870681762695,
      "activations/layer8_attention_weight_min": -53.05876541137695,
      "activations/layer9_attention_weight_max": 36.13261795043945,
      "activations/layer9_attention_weight_min": -39.481971740722656,
      "epoch": 22.85,
      "learning_rate": 2.653030303030303e-06,
      "loss": 2.707,
      "step": 393300
    },
    {
      "activations/layer0_attention_weight_max": 17.18226432800293,
      "activations/layer0_attention_weight_min": -13.503373146057129,
      "activations/layer10_attention_weight_max": 37.08244705200195,
      "activations/layer10_attention_weight_min": -36.62178421020508,
      "activations/layer11_attention_weight_max": 35.829322814941406,
      "activations/layer11_attention_weight_min": -35.670230865478516,
      "activations/layer12_attention_weight_max": 27.958768844604492,
      "activations/layer12_attention_weight_min": -26.89305305480957,
      "activations/layer13_attention_weight_max": 39.64248275756836,
      "activations/layer13_attention_weight_min": -37.13927459716797,
      "activations/layer14_attention_weight_max": 44.004302978515625,
      "activations/layer14_attention_weight_min": -39.0619010925293,
      "activations/layer15_attention_weight_max": 39.91007614135742,
      "activations/layer15_attention_weight_min": -38.49985122680664,
      "activations/layer16_attention_weight_max": 31.336849212646484,
      "activations/layer16_attention_weight_min": -29.415420532226562,
      "activations/layer17_attention_weight_max": 29.40158462524414,
      "activations/layer17_attention_weight_min": -27.26913833618164,
      "activations/layer18_attention_weight_max": 33.420562744140625,
      "activations/layer18_attention_weight_min": -25.54605484008789,
      "activations/layer19_attention_weight_max": 34.426395416259766,
      "activations/layer19_attention_weight_min": -31.400964736938477,
      "activations/layer1_attention_weight_max": 16.570436477661133,
      "activations/layer1_attention_weight_min": -15.032193183898926,
      "activations/layer20_attention_weight_max": 28.869138717651367,
      "activations/layer20_attention_weight_min": -23.159208297729492,
      "activations/layer21_attention_weight_max": 29.519554138183594,
      "activations/layer21_attention_weight_min": -22.94715690612793,
      "activations/layer22_attention_weight_max": 44.9419059753418,
      "activations/layer22_attention_weight_min": -31.340003967285156,
      "activations/layer23_attention_weight_max": 33.40177917480469,
      "activations/layer23_attention_weight_min": -25.238384246826172,
      "activations/layer2_attention_weight_max": 34.61029815673828,
      "activations/layer2_attention_weight_min": -33.95398712158203,
      "activations/layer3_attention_weight_max": 109.92418670654297,
      "activations/layer3_attention_weight_min": -109.68206787109375,
      "activations/layer4_attention_weight_max": 116.18770599365234,
      "activations/layer4_attention_weight_min": -116.34040832519531,
      "activations/layer5_attention_weight_max": 53.82575225830078,
      "activations/layer5_attention_weight_min": -65.57012176513672,
      "activations/layer6_attention_weight_max": 48.75191879272461,
      "activations/layer6_attention_weight_min": -52.0324592590332,
      "activations/layer7_attention_weight_max": 111.37320709228516,
      "activations/layer7_attention_weight_min": -111.81356811523438,
      "activations/layer8_attention_weight_max": 45.74612808227539,
      "activations/layer8_attention_weight_min": -49.45184326171875,
      "activations/layer9_attention_weight_max": 34.60725021362305,
      "activations/layer9_attention_weight_min": -37.63737487792969,
      "epoch": 22.86,
      "learning_rate": 2.634090909090909e-06,
      "loss": 2.6978,
      "step": 393350
    },
    {
      "activations/layer0_attention_weight_max": 17.29073143005371,
      "activations/layer0_attention_weight_min": -14.749733924865723,
      "activations/layer10_attention_weight_max": 36.75211715698242,
      "activations/layer10_attention_weight_min": -37.80179214477539,
      "activations/layer11_attention_weight_max": 34.61225891113281,
      "activations/layer11_attention_weight_min": -37.78329086303711,
      "activations/layer12_attention_weight_max": 26.30512046813965,
      "activations/layer12_attention_weight_min": -29.27855682373047,
      "activations/layer13_attention_weight_max": 38.79785919189453,
      "activations/layer13_attention_weight_min": -36.797218322753906,
      "activations/layer14_attention_weight_max": 42.86395263671875,
      "activations/layer14_attention_weight_min": -39.63834762573242,
      "activations/layer15_attention_weight_max": 39.644771575927734,
      "activations/layer15_attention_weight_min": -39.11106872558594,
      "activations/layer16_attention_weight_max": 32.65359878540039,
      "activations/layer16_attention_weight_min": -32.59140396118164,
      "activations/layer17_attention_weight_max": 33.654701232910156,
      "activations/layer17_attention_weight_min": -31.07639503479004,
      "activations/layer18_attention_weight_max": 33.987247467041016,
      "activations/layer18_attention_weight_min": -30.141582489013672,
      "activations/layer19_attention_weight_max": 38.84586715698242,
      "activations/layer19_attention_weight_min": -36.583343505859375,
      "activations/layer1_attention_weight_max": 16.232189178466797,
      "activations/layer1_attention_weight_min": -15.581624984741211,
      "activations/layer20_attention_weight_max": 30.65492820739746,
      "activations/layer20_attention_weight_min": -30.269397735595703,
      "activations/layer21_attention_weight_max": 30.15279197692871,
      "activations/layer21_attention_weight_min": -27.339902877807617,
      "activations/layer22_attention_weight_max": 40.63724136352539,
      "activations/layer22_attention_weight_min": -37.122047424316406,
      "activations/layer23_attention_weight_max": 36.012176513671875,
      "activations/layer23_attention_weight_min": -27.303680419921875,
      "activations/layer2_attention_weight_max": 34.63662338256836,
      "activations/layer2_attention_weight_min": -34.598426818847656,
      "activations/layer3_attention_weight_max": 111.56192779541016,
      "activations/layer3_attention_weight_min": -112.41869354248047,
      "activations/layer4_attention_weight_max": 116.7560043334961,
      "activations/layer4_attention_weight_min": -114.72716522216797,
      "activations/layer5_attention_weight_max": 53.649261474609375,
      "activations/layer5_attention_weight_min": -64.32896423339844,
      "activations/layer6_attention_weight_max": 48.606014251708984,
      "activations/layer6_attention_weight_min": -48.337379455566406,
      "activations/layer7_attention_weight_max": 106.78350830078125,
      "activations/layer7_attention_weight_min": -105.8203125,
      "activations/layer8_attention_weight_max": 42.925994873046875,
      "activations/layer8_attention_weight_min": -47.6197395324707,
      "activations/layer9_attention_weight_max": 32.46142578125,
      "activations/layer9_attention_weight_min": -34.987728118896484,
      "epoch": 22.86,
      "learning_rate": 2.615151515151515e-06,
      "loss": 2.686,
      "step": 393400
    },
    {
      "activations/layer0_attention_weight_max": 17.19721221923828,
      "activations/layer0_attention_weight_min": -15.359488487243652,
      "activations/layer10_attention_weight_max": 35.62345886230469,
      "activations/layer10_attention_weight_min": -38.73497009277344,
      "activations/layer11_attention_weight_max": 34.421504974365234,
      "activations/layer11_attention_weight_min": -37.286067962646484,
      "activations/layer12_attention_weight_max": 27.69803237915039,
      "activations/layer12_attention_weight_min": -27.047571182250977,
      "activations/layer13_attention_weight_max": 41.81672668457031,
      "activations/layer13_attention_weight_min": -37.476318359375,
      "activations/layer14_attention_weight_max": 43.3427734375,
      "activations/layer14_attention_weight_min": -40.71234893798828,
      "activations/layer15_attention_weight_max": 43.0826416015625,
      "activations/layer15_attention_weight_min": -40.88172912597656,
      "activations/layer16_attention_weight_max": 32.066368103027344,
      "activations/layer16_attention_weight_min": -30.04442024230957,
      "activations/layer17_attention_weight_max": 31.864450454711914,
      "activations/layer17_attention_weight_min": -30.32012939453125,
      "activations/layer18_attention_weight_max": 31.45819664001465,
      "activations/layer18_attention_weight_min": -31.12470817565918,
      "activations/layer19_attention_weight_max": 34.74178695678711,
      "activations/layer19_attention_weight_min": -33.7845573425293,
      "activations/layer1_attention_weight_max": 15.941816329956055,
      "activations/layer1_attention_weight_min": -15.56243896484375,
      "activations/layer20_attention_weight_max": 29.01888656616211,
      "activations/layer20_attention_weight_min": -26.865617752075195,
      "activations/layer21_attention_weight_max": 27.72368049621582,
      "activations/layer21_attention_weight_min": -26.81060028076172,
      "activations/layer22_attention_weight_max": 41.97943115234375,
      "activations/layer22_attention_weight_min": -37.80704116821289,
      "activations/layer23_attention_weight_max": 29.701162338256836,
      "activations/layer23_attention_weight_min": -30.790159225463867,
      "activations/layer2_attention_weight_max": 36.5639762878418,
      "activations/layer2_attention_weight_min": -34.790061950683594,
      "activations/layer3_attention_weight_max": 110.57389068603516,
      "activations/layer3_attention_weight_min": -110.78831481933594,
      "activations/layer4_attention_weight_max": 114.40950775146484,
      "activations/layer4_attention_weight_min": -113.40727996826172,
      "activations/layer5_attention_weight_max": 50.56085205078125,
      "activations/layer5_attention_weight_min": -66.06783294677734,
      "activations/layer6_attention_weight_max": 45.863338470458984,
      "activations/layer6_attention_weight_min": -51.232051849365234,
      "activations/layer7_attention_weight_max": 99.66661834716797,
      "activations/layer7_attention_weight_min": -104.7695541381836,
      "activations/layer8_attention_weight_max": 45.409568786621094,
      "activations/layer8_attention_weight_min": -49.59294128417969,
      "activations/layer9_attention_weight_max": 32.433349609375,
      "activations/layer9_attention_weight_min": -36.77245330810547,
      "epoch": 22.86,
      "learning_rate": 2.596212121212121e-06,
      "loss": 2.6837,
      "step": 393450
    },
    {
      "activations/layer0_attention_weight_max": 17.034589767456055,
      "activations/layer0_attention_weight_min": -15.618639945983887,
      "activations/layer10_attention_weight_max": 34.953208923339844,
      "activations/layer10_attention_weight_min": -36.4899787902832,
      "activations/layer11_attention_weight_max": 35.34502410888672,
      "activations/layer11_attention_weight_min": -34.894798278808594,
      "activations/layer12_attention_weight_max": 24.582141876220703,
      "activations/layer12_attention_weight_min": -26.79792594909668,
      "activations/layer13_attention_weight_max": 39.023860931396484,
      "activations/layer13_attention_weight_min": -35.07998275756836,
      "activations/layer14_attention_weight_max": 43.47559356689453,
      "activations/layer14_attention_weight_min": -39.773624420166016,
      "activations/layer15_attention_weight_max": 40.84576416015625,
      "activations/layer15_attention_weight_min": -37.248756408691406,
      "activations/layer16_attention_weight_max": 29.05579948425293,
      "activations/layer16_attention_weight_min": -29.325971603393555,
      "activations/layer17_attention_weight_max": 27.99590492248535,
      "activations/layer17_attention_weight_min": -26.616636276245117,
      "activations/layer18_attention_weight_max": 30.451635360717773,
      "activations/layer18_attention_weight_min": -26.331348419189453,
      "activations/layer19_attention_weight_max": 31.168746948242188,
      "activations/layer19_attention_weight_min": -31.4429874420166,
      "activations/layer1_attention_weight_max": 15.159272193908691,
      "activations/layer1_attention_weight_min": -14.162483215332031,
      "activations/layer20_attention_weight_max": 25.587804794311523,
      "activations/layer20_attention_weight_min": -25.57915496826172,
      "activations/layer21_attention_weight_max": 26.23043441772461,
      "activations/layer21_attention_weight_min": -26.495595932006836,
      "activations/layer22_attention_weight_max": 35.657127380371094,
      "activations/layer22_attention_weight_min": -31.51739501953125,
      "activations/layer23_attention_weight_max": 28.17542839050293,
      "activations/layer23_attention_weight_min": -24.921897888183594,
      "activations/layer2_attention_weight_max": 32.496334075927734,
      "activations/layer2_attention_weight_min": -33.45807647705078,
      "activations/layer3_attention_weight_max": 109.46424865722656,
      "activations/layer3_attention_weight_min": -109.4994888305664,
      "activations/layer4_attention_weight_max": 107.27584075927734,
      "activations/layer4_attention_weight_min": -112.97795867919922,
      "activations/layer5_attention_weight_max": 52.99674987792969,
      "activations/layer5_attention_weight_min": -61.82175827026367,
      "activations/layer6_attention_weight_max": 44.42938995361328,
      "activations/layer6_attention_weight_min": -48.23434066772461,
      "activations/layer7_attention_weight_max": 107.34749603271484,
      "activations/layer7_attention_weight_min": -95.89949035644531,
      "activations/layer8_attention_weight_max": 44.640533447265625,
      "activations/layer8_attention_weight_min": -45.727073669433594,
      "activations/layer9_attention_weight_max": 33.3082160949707,
      "activations/layer9_attention_weight_min": -35.434749603271484,
      "epoch": 22.86,
      "learning_rate": 2.5772727272727274e-06,
      "loss": 2.6902,
      "step": 393500
    },
    {
      "activations/layer0_attention_weight_max": 17.066076278686523,
      "activations/layer0_attention_weight_min": -13.64963150024414,
      "activations/layer10_attention_weight_max": 36.06044387817383,
      "activations/layer10_attention_weight_min": -39.10818862915039,
      "activations/layer11_attention_weight_max": 34.05509948730469,
      "activations/layer11_attention_weight_min": -37.547420501708984,
      "activations/layer12_attention_weight_max": 27.245691299438477,
      "activations/layer12_attention_weight_min": -28.16254997253418,
      "activations/layer13_attention_weight_max": 41.97809982299805,
      "activations/layer13_attention_weight_min": -37.1544075012207,
      "activations/layer14_attention_weight_max": 47.40646743774414,
      "activations/layer14_attention_weight_min": -41.35141372680664,
      "activations/layer15_attention_weight_max": 42.954811096191406,
      "activations/layer15_attention_weight_min": -40.40569305419922,
      "activations/layer16_attention_weight_max": 31.940505981445312,
      "activations/layer16_attention_weight_min": -29.72158432006836,
      "activations/layer17_attention_weight_max": 32.96935272216797,
      "activations/layer17_attention_weight_min": -28.907907485961914,
      "activations/layer18_attention_weight_max": 35.13451385498047,
      "activations/layer18_attention_weight_min": -28.16802978515625,
      "activations/layer19_attention_weight_max": 37.91508102416992,
      "activations/layer19_attention_weight_min": -32.918697357177734,
      "activations/layer1_attention_weight_max": 16.131685256958008,
      "activations/layer1_attention_weight_min": -15.953132629394531,
      "activations/layer20_attention_weight_max": 31.589447021484375,
      "activations/layer20_attention_weight_min": -25.41650390625,
      "activations/layer21_attention_weight_max": 31.976085662841797,
      "activations/layer21_attention_weight_min": -26.295934677124023,
      "activations/layer22_attention_weight_max": 45.67216110229492,
      "activations/layer22_attention_weight_min": -34.709190368652344,
      "activations/layer23_attention_weight_max": 34.35633850097656,
      "activations/layer23_attention_weight_min": -29.112279891967773,
      "activations/layer2_attention_weight_max": 35.284149169921875,
      "activations/layer2_attention_weight_min": -33.84217834472656,
      "activations/layer3_attention_weight_max": 107.35326385498047,
      "activations/layer3_attention_weight_min": -109.67713165283203,
      "activations/layer4_attention_weight_max": 112.76863861083984,
      "activations/layer4_attention_weight_min": -116.84197998046875,
      "activations/layer5_attention_weight_max": 50.84944152832031,
      "activations/layer5_attention_weight_min": -63.8427734375,
      "activations/layer6_attention_weight_max": 46.314273834228516,
      "activations/layer6_attention_weight_min": -50.14204025268555,
      "activations/layer7_attention_weight_max": 102.47356414794922,
      "activations/layer7_attention_weight_min": -99.98080444335938,
      "activations/layer8_attention_weight_max": 42.82487106323242,
      "activations/layer8_attention_weight_min": -47.57404327392578,
      "activations/layer9_attention_weight_max": 32.57279586791992,
      "activations/layer9_attention_weight_min": -37.22085189819336,
      "epoch": 22.87,
      "learning_rate": 2.558333333333333e-06,
      "loss": 2.6981,
      "step": 393550
    },
    {
      "activations/layer0_attention_weight_max": 17.429916381835938,
      "activations/layer0_attention_weight_min": -14.140655517578125,
      "activations/layer10_attention_weight_max": 35.898685455322266,
      "activations/layer10_attention_weight_min": -36.959373474121094,
      "activations/layer11_attention_weight_max": 34.565147399902344,
      "activations/layer11_attention_weight_min": -35.562286376953125,
      "activations/layer12_attention_weight_max": 26.478906631469727,
      "activations/layer12_attention_weight_min": -27.171567916870117,
      "activations/layer13_attention_weight_max": 38.82684326171875,
      "activations/layer13_attention_weight_min": -36.035789489746094,
      "activations/layer14_attention_weight_max": 44.61582565307617,
      "activations/layer14_attention_weight_min": -39.58963394165039,
      "activations/layer15_attention_weight_max": 38.43780517578125,
      "activations/layer15_attention_weight_min": -37.25543975830078,
      "activations/layer16_attention_weight_max": 30.594274520874023,
      "activations/layer16_attention_weight_min": -29.375690460205078,
      "activations/layer17_attention_weight_max": 34.77198791503906,
      "activations/layer17_attention_weight_min": -28.57535743713379,
      "activations/layer18_attention_weight_max": 36.03264617919922,
      "activations/layer18_attention_weight_min": -28.300405502319336,
      "activations/layer19_attention_weight_max": 34.654090881347656,
      "activations/layer19_attention_weight_min": -32.78678512573242,
      "activations/layer1_attention_weight_max": 15.542135238647461,
      "activations/layer1_attention_weight_min": -15.07429027557373,
      "activations/layer20_attention_weight_max": 29.726490020751953,
      "activations/layer20_attention_weight_min": -28.27138328552246,
      "activations/layer21_attention_weight_max": 30.878734588623047,
      "activations/layer21_attention_weight_min": -27.932647705078125,
      "activations/layer22_attention_weight_max": 48.81088638305664,
      "activations/layer22_attention_weight_min": -37.2108039855957,
      "activations/layer23_attention_weight_max": 33.452537536621094,
      "activations/layer23_attention_weight_min": -29.0838623046875,
      "activations/layer2_attention_weight_max": 34.42053985595703,
      "activations/layer2_attention_weight_min": -34.940330505371094,
      "activations/layer3_attention_weight_max": 110.12786865234375,
      "activations/layer3_attention_weight_min": -110.46807098388672,
      "activations/layer4_attention_weight_max": 118.82323455810547,
      "activations/layer4_attention_weight_min": -118.59881591796875,
      "activations/layer5_attention_weight_max": 52.539024353027344,
      "activations/layer5_attention_weight_min": -66.08180236816406,
      "activations/layer6_attention_weight_max": 49.5042839050293,
      "activations/layer6_attention_weight_min": -50.82328796386719,
      "activations/layer7_attention_weight_max": 100.59976196289062,
      "activations/layer7_attention_weight_min": -104.41305541992188,
      "activations/layer8_attention_weight_max": 48.022216796875,
      "activations/layer8_attention_weight_min": -49.54338455200195,
      "activations/layer9_attention_weight_max": 36.220703125,
      "activations/layer9_attention_weight_min": -38.49155044555664,
      "epoch": 22.87,
      "learning_rate": 2.5393939393939393e-06,
      "loss": 2.6898,
      "step": 393600
    },
    {
      "activations/layer0_attention_weight_max": 16.78199005126953,
      "activations/layer0_attention_weight_min": -12.860520362854004,
      "activations/layer10_attention_weight_max": 48.60187911987305,
      "activations/layer10_attention_weight_min": -42.11103439331055,
      "activations/layer11_attention_weight_max": 45.8717155456543,
      "activations/layer11_attention_weight_min": -42.27046585083008,
      "activations/layer12_attention_weight_max": 35.44013214111328,
      "activations/layer12_attention_weight_min": -29.832862854003906,
      "activations/layer13_attention_weight_max": 46.39555740356445,
      "activations/layer13_attention_weight_min": -42.45509338378906,
      "activations/layer14_attention_weight_max": 46.41044998168945,
      "activations/layer14_attention_weight_min": -42.897361755371094,
      "activations/layer15_attention_weight_max": 43.079322814941406,
      "activations/layer15_attention_weight_min": -44.2288818359375,
      "activations/layer16_attention_weight_max": 29.72353172302246,
      "activations/layer16_attention_weight_min": -30.095590591430664,
      "activations/layer17_attention_weight_max": 30.35236358642578,
      "activations/layer17_attention_weight_min": -26.51425552368164,
      "activations/layer18_attention_weight_max": 33.436859130859375,
      "activations/layer18_attention_weight_min": -26.005834579467773,
      "activations/layer19_attention_weight_max": 37.68317794799805,
      "activations/layer19_attention_weight_min": -32.890010833740234,
      "activations/layer1_attention_weight_max": 15.957128524780273,
      "activations/layer1_attention_weight_min": -15.351604461669922,
      "activations/layer20_attention_weight_max": 29.4462833404541,
      "activations/layer20_attention_weight_min": -24.5577335357666,
      "activations/layer21_attention_weight_max": 29.8772029876709,
      "activations/layer21_attention_weight_min": -24.30807876586914,
      "activations/layer22_attention_weight_max": 38.87239456176758,
      "activations/layer22_attention_weight_min": -30.523366928100586,
      "activations/layer23_attention_weight_max": 32.06489562988281,
      "activations/layer23_attention_weight_min": -25.57830047607422,
      "activations/layer2_attention_weight_max": 38.00172424316406,
      "activations/layer2_attention_weight_min": -34.92010498046875,
      "activations/layer3_attention_weight_max": 116.114501953125,
      "activations/layer3_attention_weight_min": -113.8443603515625,
      "activations/layer4_attention_weight_max": 116.9200668334961,
      "activations/layer4_attention_weight_min": -114.36529541015625,
      "activations/layer5_attention_weight_max": 55.977088928222656,
      "activations/layer5_attention_weight_min": -65.28855895996094,
      "activations/layer6_attention_weight_max": 49.11528778076172,
      "activations/layer6_attention_weight_min": -49.1603889465332,
      "activations/layer7_attention_weight_max": 116.2874526977539,
      "activations/layer7_attention_weight_min": -102.20635986328125,
      "activations/layer8_attention_weight_max": 53.51942825317383,
      "activations/layer8_attention_weight_min": -48.28290557861328,
      "activations/layer9_attention_weight_max": 40.59615707397461,
      "activations/layer9_attention_weight_min": -41.8132209777832,
      "epoch": 22.87,
      "learning_rate": 2.520454545454545e-06,
      "loss": 2.6959,
      "step": 393650
    },
    {
      "activations/layer0_attention_weight_max": 17.124250411987305,
      "activations/layer0_attention_weight_min": -14.530920028686523,
      "activations/layer10_attention_weight_max": 40.21772384643555,
      "activations/layer10_attention_weight_min": -39.4299201965332,
      "activations/layer11_attention_weight_max": 42.20429992675781,
      "activations/layer11_attention_weight_min": -37.95166015625,
      "activations/layer12_attention_weight_max": 29.053363800048828,
      "activations/layer12_attention_weight_min": -29.185794830322266,
      "activations/layer13_attention_weight_max": 41.6248664855957,
      "activations/layer13_attention_weight_min": -38.67182922363281,
      "activations/layer14_attention_weight_max": 44.492095947265625,
      "activations/layer14_attention_weight_min": -43.96291732788086,
      "activations/layer15_attention_weight_max": 43.613162994384766,
      "activations/layer15_attention_weight_min": -38.86834716796875,
      "activations/layer16_attention_weight_max": 32.447208404541016,
      "activations/layer16_attention_weight_min": -30.103290557861328,
      "activations/layer17_attention_weight_max": 28.86318588256836,
      "activations/layer17_attention_weight_min": -26.909481048583984,
      "activations/layer18_attention_weight_max": 37.98567581176758,
      "activations/layer18_attention_weight_min": -29.078533172607422,
      "activations/layer19_attention_weight_max": 38.5313835144043,
      "activations/layer19_attention_weight_min": -33.77338790893555,
      "activations/layer1_attention_weight_max": 17.41324234008789,
      "activations/layer1_attention_weight_min": -16.669452667236328,
      "activations/layer20_attention_weight_max": 30.21108055114746,
      "activations/layer20_attention_weight_min": -26.900840759277344,
      "activations/layer21_attention_weight_max": 32.54115676879883,
      "activations/layer21_attention_weight_min": -28.530105590820312,
      "activations/layer22_attention_weight_max": 49.64197540283203,
      "activations/layer22_attention_weight_min": -35.39273452758789,
      "activations/layer23_attention_weight_max": 35.16297912597656,
      "activations/layer23_attention_weight_min": -29.150005340576172,
      "activations/layer2_attention_weight_max": 35.0820198059082,
      "activations/layer2_attention_weight_min": -34.3907470703125,
      "activations/layer3_attention_weight_max": 109.29985046386719,
      "activations/layer3_attention_weight_min": -110.50733947753906,
      "activations/layer4_attention_weight_max": 116.05159759521484,
      "activations/layer4_attention_weight_min": -114.14151763916016,
      "activations/layer5_attention_weight_max": 53.09292221069336,
      "activations/layer5_attention_weight_min": -65.8769302368164,
      "activations/layer6_attention_weight_max": 48.63756561279297,
      "activations/layer6_attention_weight_min": -51.52280807495117,
      "activations/layer7_attention_weight_max": 104.92774200439453,
      "activations/layer7_attention_weight_min": -107.22054290771484,
      "activations/layer8_attention_weight_max": 46.89152145385742,
      "activations/layer8_attention_weight_min": -50.437660217285156,
      "activations/layer9_attention_weight_max": 35.859798431396484,
      "activations/layer9_attention_weight_min": -39.12129592895508,
      "epoch": 22.88,
      "learning_rate": 2.501515151515151e-06,
      "loss": 2.6962,
      "step": 393700
    },
    {
      "activations/layer0_attention_weight_max": 16.844449996948242,
      "activations/layer0_attention_weight_min": -13.754304885864258,
      "activations/layer10_attention_weight_max": 38.00314712524414,
      "activations/layer10_attention_weight_min": -38.15900421142578,
      "activations/layer11_attention_weight_max": 37.119285583496094,
      "activations/layer11_attention_weight_min": -38.794517517089844,
      "activations/layer12_attention_weight_max": 26.90218734741211,
      "activations/layer12_attention_weight_min": -27.48424530029297,
      "activations/layer13_attention_weight_max": 40.69740676879883,
      "activations/layer13_attention_weight_min": -37.09865188598633,
      "activations/layer14_attention_weight_max": 40.38251876831055,
      "activations/layer14_attention_weight_min": -38.997283935546875,
      "activations/layer15_attention_weight_max": 37.121429443359375,
      "activations/layer15_attention_weight_min": -37.52693557739258,
      "activations/layer16_attention_weight_max": 30.480493545532227,
      "activations/layer16_attention_weight_min": -27.548038482666016,
      "activations/layer17_attention_weight_max": 29.7971134185791,
      "activations/layer17_attention_weight_min": -28.503671646118164,
      "activations/layer18_attention_weight_max": 34.31200408935547,
      "activations/layer18_attention_weight_min": -26.330448150634766,
      "activations/layer19_attention_weight_max": 32.728050231933594,
      "activations/layer19_attention_weight_min": -31.23126220703125,
      "activations/layer1_attention_weight_max": 16.147180557250977,
      "activations/layer1_attention_weight_min": -14.864856719970703,
      "activations/layer20_attention_weight_max": 28.1003475189209,
      "activations/layer20_attention_weight_min": -24.687976837158203,
      "activations/layer21_attention_weight_max": 28.220169067382812,
      "activations/layer21_attention_weight_min": -25.00396156311035,
      "activations/layer22_attention_weight_max": 41.32389450073242,
      "activations/layer22_attention_weight_min": -30.29884147644043,
      "activations/layer23_attention_weight_max": 29.96023941040039,
      "activations/layer23_attention_weight_min": -24.325979232788086,
      "activations/layer2_attention_weight_max": 35.109439849853516,
      "activations/layer2_attention_weight_min": -33.886844635009766,
      "activations/layer3_attention_weight_max": 109.3902816772461,
      "activations/layer3_attention_weight_min": -111.09032440185547,
      "activations/layer4_attention_weight_max": 112.99925994873047,
      "activations/layer4_attention_weight_min": -115.84139251708984,
      "activations/layer5_attention_weight_max": 52.55048370361328,
      "activations/layer5_attention_weight_min": -63.66688537597656,
      "activations/layer6_attention_weight_max": 47.475582122802734,
      "activations/layer6_attention_weight_min": -52.2910041809082,
      "activations/layer7_attention_weight_max": 101.67488098144531,
      "activations/layer7_attention_weight_min": -107.19406127929688,
      "activations/layer8_attention_weight_max": 46.89309310913086,
      "activations/layer8_attention_weight_min": -49.08866500854492,
      "activations/layer9_attention_weight_max": 38.42021179199219,
      "activations/layer9_attention_weight_min": -37.57487106323242,
      "epoch": 22.88,
      "learning_rate": 2.4825757575757577e-06,
      "loss": 2.7122,
      "step": 393750
    },
    {
      "activations/layer0_attention_weight_max": 17.574460983276367,
      "activations/layer0_attention_weight_min": -14.37179183959961,
      "activations/layer10_attention_weight_max": 36.66889572143555,
      "activations/layer10_attention_weight_min": -37.85034942626953,
      "activations/layer11_attention_weight_max": 35.38239288330078,
      "activations/layer11_attention_weight_min": -37.78294372558594,
      "activations/layer12_attention_weight_max": 30.13271713256836,
      "activations/layer12_attention_weight_min": -27.97229766845703,
      "activations/layer13_attention_weight_max": 42.2126350402832,
      "activations/layer13_attention_weight_min": -37.024253845214844,
      "activations/layer14_attention_weight_max": 46.056983947753906,
      "activations/layer14_attention_weight_min": -39.447967529296875,
      "activations/layer15_attention_weight_max": 43.52956008911133,
      "activations/layer15_attention_weight_min": -37.120357513427734,
      "activations/layer16_attention_weight_max": 31.08937644958496,
      "activations/layer16_attention_weight_min": -29.851598739624023,
      "activations/layer17_attention_weight_max": 31.01431655883789,
      "activations/layer17_attention_weight_min": -25.97187042236328,
      "activations/layer18_attention_weight_max": 32.141624450683594,
      "activations/layer18_attention_weight_min": -25.218381881713867,
      "activations/layer19_attention_weight_max": 36.23527145385742,
      "activations/layer19_attention_weight_min": -31.669721603393555,
      "activations/layer1_attention_weight_max": 16.860450744628906,
      "activations/layer1_attention_weight_min": -16.098478317260742,
      "activations/layer20_attention_weight_max": 31.017070770263672,
      "activations/layer20_attention_weight_min": -25.444408416748047,
      "activations/layer21_attention_weight_max": 30.90053939819336,
      "activations/layer21_attention_weight_min": -26.11305809020996,
      "activations/layer22_attention_weight_max": 46.4449462890625,
      "activations/layer22_attention_weight_min": -29.92814826965332,
      "activations/layer23_attention_weight_max": 35.440101623535156,
      "activations/layer23_attention_weight_min": -26.034391403198242,
      "activations/layer2_attention_weight_max": 36.938777923583984,
      "activations/layer2_attention_weight_min": -34.86008071899414,
      "activations/layer3_attention_weight_max": 113.1025161743164,
      "activations/layer3_attention_weight_min": -112.9318618774414,
      "activations/layer4_attention_weight_max": 115.68257904052734,
      "activations/layer4_attention_weight_min": -120.13695526123047,
      "activations/layer5_attention_weight_max": 54.23379135131836,
      "activations/layer5_attention_weight_min": -66.8123779296875,
      "activations/layer6_attention_weight_max": 47.157222747802734,
      "activations/layer6_attention_weight_min": -52.4218635559082,
      "activations/layer7_attention_weight_max": 102.93632507324219,
      "activations/layer7_attention_weight_min": -110.37046813964844,
      "activations/layer8_attention_weight_max": 44.01321029663086,
      "activations/layer8_attention_weight_min": -47.793914794921875,
      "activations/layer9_attention_weight_max": 34.08769226074219,
      "activations/layer9_attention_weight_min": -35.98285675048828,
      "epoch": 22.88,
      "learning_rate": 2.4636363636363635e-06,
      "loss": 2.7049,
      "step": 393800
    },
    {
      "activations/layer0_attention_weight_max": 16.794750213623047,
      "activations/layer0_attention_weight_min": -13.529097557067871,
      "activations/layer10_attention_weight_max": 40.861942291259766,
      "activations/layer10_attention_weight_min": -42.35847854614258,
      "activations/layer11_attention_weight_max": 38.8061408996582,
      "activations/layer11_attention_weight_min": -39.54120635986328,
      "activations/layer12_attention_weight_max": 28.40241241455078,
      "activations/layer12_attention_weight_min": -30.42247200012207,
      "activations/layer13_attention_weight_max": 43.289554595947266,
      "activations/layer13_attention_weight_min": -40.9306640625,
      "activations/layer14_attention_weight_max": 45.49624252319336,
      "activations/layer14_attention_weight_min": -45.44818878173828,
      "activations/layer15_attention_weight_max": 42.07075881958008,
      "activations/layer15_attention_weight_min": -42.98020553588867,
      "activations/layer16_attention_weight_max": 32.64238739013672,
      "activations/layer16_attention_weight_min": -30.341577529907227,
      "activations/layer17_attention_weight_max": 33.014156341552734,
      "activations/layer17_attention_weight_min": -31.64837646484375,
      "activations/layer18_attention_weight_max": 35.856380462646484,
      "activations/layer18_attention_weight_min": -28.69829750061035,
      "activations/layer19_attention_weight_max": 38.633113861083984,
      "activations/layer19_attention_weight_min": -33.831932067871094,
      "activations/layer1_attention_weight_max": 17.12109375,
      "activations/layer1_attention_weight_min": -15.57085132598877,
      "activations/layer20_attention_weight_max": 33.75892639160156,
      "activations/layer20_attention_weight_min": -26.17293357849121,
      "activations/layer21_attention_weight_max": 34.01259994506836,
      "activations/layer21_attention_weight_min": -25.554153442382812,
      "activations/layer22_attention_weight_max": 46.17771530151367,
      "activations/layer22_attention_weight_min": -34.22164535522461,
      "activations/layer23_attention_weight_max": 37.81024932861328,
      "activations/layer23_attention_weight_min": -27.530101776123047,
      "activations/layer2_attention_weight_max": 34.833412170410156,
      "activations/layer2_attention_weight_min": -34.899436950683594,
      "activations/layer3_attention_weight_max": 109.36741638183594,
      "activations/layer3_attention_weight_min": -111.77153778076172,
      "activations/layer4_attention_weight_max": 111.06375885009766,
      "activations/layer4_attention_weight_min": -114.00777435302734,
      "activations/layer5_attention_weight_max": 49.967918395996094,
      "activations/layer5_attention_weight_min": -64.17842102050781,
      "activations/layer6_attention_weight_max": 46.815731048583984,
      "activations/layer6_attention_weight_min": -49.65392303466797,
      "activations/layer7_attention_weight_max": 103.80866241455078,
      "activations/layer7_attention_weight_min": -106.0686264038086,
      "activations/layer8_attention_weight_max": 44.79655838012695,
      "activations/layer8_attention_weight_min": -49.57297134399414,
      "activations/layer9_attention_weight_max": 35.07341384887695,
      "activations/layer9_attention_weight_min": -37.93612289428711,
      "epoch": 22.88,
      "learning_rate": 2.4446969696969696e-06,
      "loss": 2.7032,
      "step": 393850
    },
    {
      "activations/layer0_attention_weight_max": 16.019784927368164,
      "activations/layer0_attention_weight_min": -15.120906829833984,
      "activations/layer10_attention_weight_max": 35.248409271240234,
      "activations/layer10_attention_weight_min": -38.15281677246094,
      "activations/layer11_attention_weight_max": 34.548126220703125,
      "activations/layer11_attention_weight_min": -38.83991622924805,
      "activations/layer12_attention_weight_max": 26.44908332824707,
      "activations/layer12_attention_weight_min": -27.28892707824707,
      "activations/layer13_attention_weight_max": 40.955108642578125,
      "activations/layer13_attention_weight_min": -37.32400131225586,
      "activations/layer14_attention_weight_max": 42.80656051635742,
      "activations/layer14_attention_weight_min": -39.5420036315918,
      "activations/layer15_attention_weight_max": 40.350318908691406,
      "activations/layer15_attention_weight_min": -37.57029342651367,
      "activations/layer16_attention_weight_max": 29.737794876098633,
      "activations/layer16_attention_weight_min": -29.67006492614746,
      "activations/layer17_attention_weight_max": 28.816843032836914,
      "activations/layer17_attention_weight_min": -27.286067962646484,
      "activations/layer18_attention_weight_max": 31.224796295166016,
      "activations/layer18_attention_weight_min": -28.49921989440918,
      "activations/layer19_attention_weight_max": 32.2877082824707,
      "activations/layer19_attention_weight_min": -33.27181625366211,
      "activations/layer1_attention_weight_max": 16.660205841064453,
      "activations/layer1_attention_weight_min": -14.507308006286621,
      "activations/layer20_attention_weight_max": 28.30925750732422,
      "activations/layer20_attention_weight_min": -26.138608932495117,
      "activations/layer21_attention_weight_max": 26.677104949951172,
      "activations/layer21_attention_weight_min": -26.123031616210938,
      "activations/layer22_attention_weight_max": 41.36054992675781,
      "activations/layer22_attention_weight_min": -33.82669448852539,
      "activations/layer23_attention_weight_max": 30.269920349121094,
      "activations/layer23_attention_weight_min": -26.09101676940918,
      "activations/layer2_attention_weight_max": 34.04777526855469,
      "activations/layer2_attention_weight_min": -35.26539993286133,
      "activations/layer3_attention_weight_max": 110.79056549072266,
      "activations/layer3_attention_weight_min": -113.51642608642578,
      "activations/layer4_attention_weight_max": 114.33538055419922,
      "activations/layer4_attention_weight_min": -117.38899230957031,
      "activations/layer5_attention_weight_max": 52.34457015991211,
      "activations/layer5_attention_weight_min": -65.31825256347656,
      "activations/layer6_attention_weight_max": 48.64340591430664,
      "activations/layer6_attention_weight_min": -52.27579879760742,
      "activations/layer7_attention_weight_max": 102.44681549072266,
      "activations/layer7_attention_weight_min": -111.03314971923828,
      "activations/layer8_attention_weight_max": 44.13452911376953,
      "activations/layer8_attention_weight_min": -51.721195220947266,
      "activations/layer9_attention_weight_max": 32.70378875732422,
      "activations/layer9_attention_weight_min": -35.89036178588867,
      "epoch": 22.89,
      "learning_rate": 2.4257575757575753e-06,
      "loss": 2.7022,
      "step": 393900
    },
    {
      "activations/layer0_attention_weight_max": 17.15454864501953,
      "activations/layer0_attention_weight_min": -15.08368968963623,
      "activations/layer10_attention_weight_max": 36.92987060546875,
      "activations/layer10_attention_weight_min": -37.05202865600586,
      "activations/layer11_attention_weight_max": 37.61854553222656,
      "activations/layer11_attention_weight_min": -38.26028823852539,
      "activations/layer12_attention_weight_max": 25.840307235717773,
      "activations/layer12_attention_weight_min": -27.788789749145508,
      "activations/layer13_attention_weight_max": 39.44061279296875,
      "activations/layer13_attention_weight_min": -37.71562576293945,
      "activations/layer14_attention_weight_max": 41.61008071899414,
      "activations/layer14_attention_weight_min": -39.41447448730469,
      "activations/layer15_attention_weight_max": 40.076210021972656,
      "activations/layer15_attention_weight_min": -37.50202560424805,
      "activations/layer16_attention_weight_max": 28.648691177368164,
      "activations/layer16_attention_weight_min": -28.38102149963379,
      "activations/layer17_attention_weight_max": 28.819183349609375,
      "activations/layer17_attention_weight_min": -27.000511169433594,
      "activations/layer18_attention_weight_max": 32.78325653076172,
      "activations/layer18_attention_weight_min": -26.485063552856445,
      "activations/layer19_attention_weight_max": 32.930137634277344,
      "activations/layer19_attention_weight_min": -30.143966674804688,
      "activations/layer1_attention_weight_max": 15.645036697387695,
      "activations/layer1_attention_weight_min": -14.449082374572754,
      "activations/layer20_attention_weight_max": 27.824310302734375,
      "activations/layer20_attention_weight_min": -24.52663230895996,
      "activations/layer21_attention_weight_max": 28.76088523864746,
      "activations/layer21_attention_weight_min": -24.277984619140625,
      "activations/layer22_attention_weight_max": 40.65128707885742,
      "activations/layer22_attention_weight_min": -32.01536178588867,
      "activations/layer23_attention_weight_max": 32.22701644897461,
      "activations/layer23_attention_weight_min": -26.139293670654297,
      "activations/layer2_attention_weight_max": 33.74978256225586,
      "activations/layer2_attention_weight_min": -33.43339538574219,
      "activations/layer3_attention_weight_max": 110.63101959228516,
      "activations/layer3_attention_weight_min": -108.02169036865234,
      "activations/layer4_attention_weight_max": 114.40154266357422,
      "activations/layer4_attention_weight_min": -117.86720275878906,
      "activations/layer5_attention_weight_max": 51.2503662109375,
      "activations/layer5_attention_weight_min": -62.96527862548828,
      "activations/layer6_attention_weight_max": 47.421653747558594,
      "activations/layer6_attention_weight_min": -49.07802963256836,
      "activations/layer7_attention_weight_max": 102.7939682006836,
      "activations/layer7_attention_weight_min": -107.31219482421875,
      "activations/layer8_attention_weight_max": 47.60612869262695,
      "activations/layer8_attention_weight_min": -49.34212112426758,
      "activations/layer9_attention_weight_max": 35.86971664428711,
      "activations/layer9_attention_weight_min": -37.284366607666016,
      "epoch": 22.89,
      "learning_rate": 2.406818181818182e-06,
      "loss": 2.7032,
      "step": 393950
    },
    {
      "activations/layer0_attention_weight_max": 17.191970825195312,
      "activations/layer0_attention_weight_min": -13.102599143981934,
      "activations/layer10_attention_weight_max": 37.225284576416016,
      "activations/layer10_attention_weight_min": -39.308502197265625,
      "activations/layer11_attention_weight_max": 37.79338073730469,
      "activations/layer11_attention_weight_min": -38.753108978271484,
      "activations/layer12_attention_weight_max": 31.232059478759766,
      "activations/layer12_attention_weight_min": -29.23127555847168,
      "activations/layer13_attention_weight_max": 42.247833251953125,
      "activations/layer13_attention_weight_min": -40.8381462097168,
      "activations/layer14_attention_weight_max": 47.054447174072266,
      "activations/layer14_attention_weight_min": -43.700931549072266,
      "activations/layer15_attention_weight_max": 44.06682205200195,
      "activations/layer15_attention_weight_min": -40.890899658203125,
      "activations/layer16_attention_weight_max": 33.94096755981445,
      "activations/layer16_attention_weight_min": -29.286521911621094,
      "activations/layer17_attention_weight_max": 30.76816749572754,
      "activations/layer17_attention_weight_min": -27.99700355529785,
      "activations/layer18_attention_weight_max": 31.69537925720215,
      "activations/layer18_attention_weight_min": -28.191410064697266,
      "activations/layer19_attention_weight_max": 34.99520492553711,
      "activations/layer19_attention_weight_min": -33.73735427856445,
      "activations/layer1_attention_weight_max": 15.922469139099121,
      "activations/layer1_attention_weight_min": -15.231534004211426,
      "activations/layer20_attention_weight_max": 30.003700256347656,
      "activations/layer20_attention_weight_min": -25.920616149902344,
      "activations/layer21_attention_weight_max": 30.21328353881836,
      "activations/layer21_attention_weight_min": -25.138845443725586,
      "activations/layer22_attention_weight_max": 42.24118423461914,
      "activations/layer22_attention_weight_min": -33.33180236816406,
      "activations/layer23_attention_weight_max": 35.18344497680664,
      "activations/layer23_attention_weight_min": -24.870742797851562,
      "activations/layer2_attention_weight_max": 34.6616096496582,
      "activations/layer2_attention_weight_min": -35.77056884765625,
      "activations/layer3_attention_weight_max": 107.5786361694336,
      "activations/layer3_attention_weight_min": -110.07852172851562,
      "activations/layer4_attention_weight_max": 112.43840789794922,
      "activations/layer4_attention_weight_min": -115.1708755493164,
      "activations/layer5_attention_weight_max": 58.350074768066406,
      "activations/layer5_attention_weight_min": -65.51018524169922,
      "activations/layer6_attention_weight_max": 47.341739654541016,
      "activations/layer6_attention_weight_min": -50.418800354003906,
      "activations/layer7_attention_weight_max": 105.9454116821289,
      "activations/layer7_attention_weight_min": -106.41876983642578,
      "activations/layer8_attention_weight_max": 46.625667572021484,
      "activations/layer8_attention_weight_min": -48.360294342041016,
      "activations/layer9_attention_weight_max": 36.23002243041992,
      "activations/layer9_attention_weight_min": -38.196876525878906,
      "epoch": 22.89,
      "learning_rate": 2.3878787878787876e-06,
      "loss": 2.7173,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.4822,
      "eval_samples_per_second": 506.235,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.4822,
      "eval_openwebtext_samples_per_second": 506.235,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9407,
      "eval_wikitext_samples_per_second": 234.97,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_lambada_loss": 2.306640625,
      "eval_lambada_ppl": 10.040637668042795,
      "eval_lambada_runtime": 9.5649,
      "eval_lambada_samples_per_second": 509.047,
      "step": 394000
    },
    {
      "activations/layer0_attention_weight_max": 17.442501068115234,
      "activations/layer0_attention_weight_min": -13.177781105041504,
      "activations/layer10_attention_weight_max": 35.66249465942383,
      "activations/layer10_attention_weight_min": -36.94355773925781,
      "activations/layer11_attention_weight_max": 36.26927185058594,
      "activations/layer11_attention_weight_min": -34.460350036621094,
      "activations/layer12_attention_weight_max": 27.83890151977539,
      "activations/layer12_attention_weight_min": -29.031055450439453,
      "activations/layer13_attention_weight_max": 43.02092361450195,
      "activations/layer13_attention_weight_min": -40.923484802246094,
      "activations/layer14_attention_weight_max": 46.1520881652832,
      "activations/layer14_attention_weight_min": -43.29444885253906,
      "activations/layer15_attention_weight_max": 38.78601837158203,
      "activations/layer15_attention_weight_min": -40.64099884033203,
      "activations/layer16_attention_weight_max": 31.466739654541016,
      "activations/layer16_attention_weight_min": -30.360706329345703,
      "activations/layer17_attention_weight_max": 30.32596206665039,
      "activations/layer17_attention_weight_min": -27.72796630859375,
      "activations/layer18_attention_weight_max": 34.83522033691406,
      "activations/layer18_attention_weight_min": -28.8094482421875,
      "activations/layer19_attention_weight_max": 37.74492263793945,
      "activations/layer19_attention_weight_min": -34.33736801147461,
      "activations/layer1_attention_weight_max": 15.715195655822754,
      "activations/layer1_attention_weight_min": -15.64840030670166,
      "activations/layer20_attention_weight_max": 33.49656295776367,
      "activations/layer20_attention_weight_min": -26.060466766357422,
      "activations/layer21_attention_weight_max": 28.659732818603516,
      "activations/layer21_attention_weight_min": -25.466331481933594,
      "activations/layer22_attention_weight_max": 44.367679595947266,
      "activations/layer22_attention_weight_min": -35.54357147216797,
      "activations/layer23_attention_weight_max": 32.686161041259766,
      "activations/layer23_attention_weight_min": -27.049055099487305,
      "activations/layer2_attention_weight_max": 34.55966567993164,
      "activations/layer2_attention_weight_min": -33.25079345703125,
      "activations/layer3_attention_weight_max": 110.1236343383789,
      "activations/layer3_attention_weight_min": -107.45361328125,
      "activations/layer4_attention_weight_max": 108.18241119384766,
      "activations/layer4_attention_weight_min": -116.0816650390625,
      "activations/layer5_attention_weight_max": 51.1171989440918,
      "activations/layer5_attention_weight_min": -63.17301940917969,
      "activations/layer6_attention_weight_max": 46.04873275756836,
      "activations/layer6_attention_weight_min": -48.90852355957031,
      "activations/layer7_attention_weight_max": 96.53701782226562,
      "activations/layer7_attention_weight_min": -101.8968734741211,
      "activations/layer8_attention_weight_max": 45.296260833740234,
      "activations/layer8_attention_weight_min": -48.773414611816406,
      "activations/layer9_attention_weight_max": 33.03294372558594,
      "activations/layer9_attention_weight_min": -36.78742599487305,
      "epoch": 22.9,
      "learning_rate": 2.368939393939394e-06,
      "loss": 2.693,
      "step": 394050
    },
    {
      "activations/layer0_attention_weight_max": 15.943525314331055,
      "activations/layer0_attention_weight_min": -14.663392066955566,
      "activations/layer10_attention_weight_max": 38.109432220458984,
      "activations/layer10_attention_weight_min": -37.834434509277344,
      "activations/layer11_attention_weight_max": 36.72178268432617,
      "activations/layer11_attention_weight_min": -37.27345275878906,
      "activations/layer12_attention_weight_max": 28.41989517211914,
      "activations/layer12_attention_weight_min": -28.177268981933594,
      "activations/layer13_attention_weight_max": 40.91936111450195,
      "activations/layer13_attention_weight_min": -39.48457717895508,
      "activations/layer14_attention_weight_max": 42.23616409301758,
      "activations/layer14_attention_weight_min": -41.5548095703125,
      "activations/layer15_attention_weight_max": 38.49028778076172,
      "activations/layer15_attention_weight_min": -40.17499923706055,
      "activations/layer16_attention_weight_max": 29.118751525878906,
      "activations/layer16_attention_weight_min": -30.89983367919922,
      "activations/layer17_attention_weight_max": 29.62937355041504,
      "activations/layer17_attention_weight_min": -28.1247501373291,
      "activations/layer18_attention_weight_max": 33.088714599609375,
      "activations/layer18_attention_weight_min": -29.22358512878418,
      "activations/layer19_attention_weight_max": 35.45878982543945,
      "activations/layer19_attention_weight_min": -33.209564208984375,
      "activations/layer1_attention_weight_max": 16.370431900024414,
      "activations/layer1_attention_weight_min": -15.132207870483398,
      "activations/layer20_attention_weight_max": 30.409435272216797,
      "activations/layer20_attention_weight_min": -26.869720458984375,
      "activations/layer21_attention_weight_max": 31.44169044494629,
      "activations/layer21_attention_weight_min": -25.77798843383789,
      "activations/layer22_attention_weight_max": 41.901145935058594,
      "activations/layer22_attention_weight_min": -34.20903396606445,
      "activations/layer23_attention_weight_max": 31.57107925415039,
      "activations/layer23_attention_weight_min": -25.375783920288086,
      "activations/layer2_attention_weight_max": 34.26715087890625,
      "activations/layer2_attention_weight_min": -33.38927459716797,
      "activations/layer3_attention_weight_max": 109.31687927246094,
      "activations/layer3_attention_weight_min": -110.01046752929688,
      "activations/layer4_attention_weight_max": 111.71385955810547,
      "activations/layer4_attention_weight_min": -118.87739562988281,
      "activations/layer5_attention_weight_max": 51.70905685424805,
      "activations/layer5_attention_weight_min": -62.19684982299805,
      "activations/layer6_attention_weight_max": 46.327335357666016,
      "activations/layer6_attention_weight_min": -52.018733978271484,
      "activations/layer7_attention_weight_max": 102.87348937988281,
      "activations/layer7_attention_weight_min": -102.71862030029297,
      "activations/layer8_attention_weight_max": 43.80379104614258,
      "activations/layer8_attention_weight_min": -47.9888801574707,
      "activations/layer9_attention_weight_max": 34.668643951416016,
      "activations/layer9_attention_weight_min": -39.178340911865234,
      "epoch": 22.9,
      "learning_rate": 2.3499999999999995e-06,
      "loss": 2.6744,
      "step": 394100
    },
    {
      "activations/layer0_attention_weight_max": 16.881811141967773,
      "activations/layer0_attention_weight_min": -15.047710418701172,
      "activations/layer10_attention_weight_max": 38.26005935668945,
      "activations/layer10_attention_weight_min": -37.761268615722656,
      "activations/layer11_attention_weight_max": 35.29201889038086,
      "activations/layer11_attention_weight_min": -36.24582290649414,
      "activations/layer12_attention_weight_max": 29.389625549316406,
      "activations/layer12_attention_weight_min": -26.93488311767578,
      "activations/layer13_attention_weight_max": 42.25227355957031,
      "activations/layer13_attention_weight_min": -36.752960205078125,
      "activations/layer14_attention_weight_max": 44.162681579589844,
      "activations/layer14_attention_weight_min": -38.31613540649414,
      "activations/layer15_attention_weight_max": 41.45627975463867,
      "activations/layer15_attention_weight_min": -37.608821868896484,
      "activations/layer16_attention_weight_max": 34.34723663330078,
      "activations/layer16_attention_weight_min": -28.03080940246582,
      "activations/layer17_attention_weight_max": 30.943851470947266,
      "activations/layer17_attention_weight_min": -26.16965103149414,
      "activations/layer18_attention_weight_max": 32.98503112792969,
      "activations/layer18_attention_weight_min": -28.865026473999023,
      "activations/layer19_attention_weight_max": 40.17570877075195,
      "activations/layer19_attention_weight_min": -29.761383056640625,
      "activations/layer1_attention_weight_max": 15.290205001831055,
      "activations/layer1_attention_weight_min": -15.550252914428711,
      "activations/layer20_attention_weight_max": 33.7453727722168,
      "activations/layer20_attention_weight_min": -24.707212448120117,
      "activations/layer21_attention_weight_max": 31.008520126342773,
      "activations/layer21_attention_weight_min": -24.939964294433594,
      "activations/layer22_attention_weight_max": 45.657344818115234,
      "activations/layer22_attention_weight_min": -32.20354080200195,
      "activations/layer23_attention_weight_max": 32.95497131347656,
      "activations/layer23_attention_weight_min": -27.840042114257812,
      "activations/layer2_attention_weight_max": 34.05241775512695,
      "activations/layer2_attention_weight_min": -34.598514556884766,
      "activations/layer3_attention_weight_max": 106.75773620605469,
      "activations/layer3_attention_weight_min": -109.35659790039062,
      "activations/layer4_attention_weight_max": 111.3983154296875,
      "activations/layer4_attention_weight_min": -118.23565673828125,
      "activations/layer5_attention_weight_max": 52.102943420410156,
      "activations/layer5_attention_weight_min": -61.399208068847656,
      "activations/layer6_attention_weight_max": 45.72813415527344,
      "activations/layer6_attention_weight_min": -49.133541107177734,
      "activations/layer7_attention_weight_max": 100.68791198730469,
      "activations/layer7_attention_weight_min": -111.26224517822266,
      "activations/layer8_attention_weight_max": 44.448150634765625,
      "activations/layer8_attention_weight_min": -47.92680740356445,
      "activations/layer9_attention_weight_max": 33.087406158447266,
      "activations/layer9_attention_weight_min": -36.391414642333984,
      "epoch": 22.9,
      "learning_rate": 2.3310606060606057e-06,
      "loss": 2.6919,
      "step": 394150
    },
    {
      "activations/layer0_attention_weight_max": 17.018840789794922,
      "activations/layer0_attention_weight_min": -14.295930862426758,
      "activations/layer10_attention_weight_max": 39.58607864379883,
      "activations/layer10_attention_weight_min": -36.9355354309082,
      "activations/layer11_attention_weight_max": 41.720149993896484,
      "activations/layer11_attention_weight_min": -37.51408767700195,
      "activations/layer12_attention_weight_max": 28.97995948791504,
      "activations/layer12_attention_weight_min": -27.126060485839844,
      "activations/layer13_attention_weight_max": 40.74665451049805,
      "activations/layer13_attention_weight_min": -36.21207809448242,
      "activations/layer14_attention_weight_max": 48.07190704345703,
      "activations/layer14_attention_weight_min": -38.01036834716797,
      "activations/layer15_attention_weight_max": 45.276588439941406,
      "activations/layer15_attention_weight_min": -40.427833557128906,
      "activations/layer16_attention_weight_max": 32.95662307739258,
      "activations/layer16_attention_weight_min": -27.76222801208496,
      "activations/layer17_attention_weight_max": 34.66900634765625,
      "activations/layer17_attention_weight_min": -28.889381408691406,
      "activations/layer18_attention_weight_max": 37.998199462890625,
      "activations/layer18_attention_weight_min": -28.06936264038086,
      "activations/layer19_attention_weight_max": 37.421939849853516,
      "activations/layer19_attention_weight_min": -33.527809143066406,
      "activations/layer1_attention_weight_max": 16.635047912597656,
      "activations/layer1_attention_weight_min": -15.00500202178955,
      "activations/layer20_attention_weight_max": 29.422218322753906,
      "activations/layer20_attention_weight_min": -25.709028244018555,
      "activations/layer21_attention_weight_max": 33.71273422241211,
      "activations/layer21_attention_weight_min": -26.93143081665039,
      "activations/layer22_attention_weight_max": 43.41477584838867,
      "activations/layer22_attention_weight_min": -36.369171142578125,
      "activations/layer23_attention_weight_max": 34.46173095703125,
      "activations/layer23_attention_weight_min": -27.70331382751465,
      "activations/layer2_attention_weight_max": 34.88873291015625,
      "activations/layer2_attention_weight_min": -35.703338623046875,
      "activations/layer3_attention_weight_max": 108.71161651611328,
      "activations/layer3_attention_weight_min": -112.5096206665039,
      "activations/layer4_attention_weight_max": 112.30403137207031,
      "activations/layer4_attention_weight_min": -117.42201232910156,
      "activations/layer5_attention_weight_max": 50.719482421875,
      "activations/layer5_attention_weight_min": -63.21072006225586,
      "activations/layer6_attention_weight_max": 47.73206329345703,
      "activations/layer6_attention_weight_min": -49.03482437133789,
      "activations/layer7_attention_weight_max": 100.28423309326172,
      "activations/layer7_attention_weight_min": -104.484130859375,
      "activations/layer8_attention_weight_max": 47.75747299194336,
      "activations/layer8_attention_weight_min": -47.17697525024414,
      "activations/layer9_attention_weight_max": 34.77116394042969,
      "activations/layer9_attention_weight_min": -37.32730484008789,
      "epoch": 22.91,
      "learning_rate": 2.312121212121212e-06,
      "loss": 2.701,
      "step": 394200
    },
    {
      "activations/layer0_attention_weight_max": 16.49408721923828,
      "activations/layer0_attention_weight_min": -14.833321571350098,
      "activations/layer10_attention_weight_max": 39.894981384277344,
      "activations/layer10_attention_weight_min": -43.026161193847656,
      "activations/layer11_attention_weight_max": 41.975032806396484,
      "activations/layer11_attention_weight_min": -45.09760665893555,
      "activations/layer12_attention_weight_max": 27.981409072875977,
      "activations/layer12_attention_weight_min": -30.982877731323242,
      "activations/layer13_attention_weight_max": 41.526241302490234,
      "activations/layer13_attention_weight_min": -38.487815856933594,
      "activations/layer14_attention_weight_max": 46.510154724121094,
      "activations/layer14_attention_weight_min": -40.41698455810547,
      "activations/layer15_attention_weight_max": 43.09251022338867,
      "activations/layer15_attention_weight_min": -40.51362228393555,
      "activations/layer16_attention_weight_max": 30.841882705688477,
      "activations/layer16_attention_weight_min": -29.946990966796875,
      "activations/layer17_attention_weight_max": 31.15884017944336,
      "activations/layer17_attention_weight_min": -26.504648208618164,
      "activations/layer18_attention_weight_max": 32.3371696472168,
      "activations/layer18_attention_weight_min": -27.131254196166992,
      "activations/layer19_attention_weight_max": 33.16069793701172,
      "activations/layer19_attention_weight_min": -30.784692764282227,
      "activations/layer1_attention_weight_max": 15.965929985046387,
      "activations/layer1_attention_weight_min": -15.539318084716797,
      "activations/layer20_attention_weight_max": 27.74782371520996,
      "activations/layer20_attention_weight_min": -23.521013259887695,
      "activations/layer21_attention_weight_max": 25.71685791015625,
      "activations/layer21_attention_weight_min": -22.700143814086914,
      "activations/layer22_attention_weight_max": 37.95322799682617,
      "activations/layer22_attention_weight_min": -29.543272018432617,
      "activations/layer23_attention_weight_max": 28.28900909423828,
      "activations/layer23_attention_weight_min": -25.927608489990234,
      "activations/layer2_attention_weight_max": 33.979095458984375,
      "activations/layer2_attention_weight_min": -35.21946334838867,
      "activations/layer3_attention_weight_max": 113.13472747802734,
      "activations/layer3_attention_weight_min": -114.3630142211914,
      "activations/layer4_attention_weight_max": 117.07401275634766,
      "activations/layer4_attention_weight_min": -117.38162994384766,
      "activations/layer5_attention_weight_max": 51.6256103515625,
      "activations/layer5_attention_weight_min": -61.64044189453125,
      "activations/layer6_attention_weight_max": 47.515342712402344,
      "activations/layer6_attention_weight_min": -48.979652404785156,
      "activations/layer7_attention_weight_max": 106.4103775024414,
      "activations/layer7_attention_weight_min": -110.95215606689453,
      "activations/layer8_attention_weight_max": 49.05772018432617,
      "activations/layer8_attention_weight_min": -52.5464973449707,
      "activations/layer9_attention_weight_max": 36.859920501708984,
      "activations/layer9_attention_weight_min": -41.61614227294922,
      "epoch": 22.91,
      "learning_rate": 2.293181818181818e-06,
      "loss": 2.6954,
      "step": 394250
    },
    {
      "activations/layer0_attention_weight_max": 17.09619903564453,
      "activations/layer0_attention_weight_min": -13.920010566711426,
      "activations/layer10_attention_weight_max": 39.498985290527344,
      "activations/layer10_attention_weight_min": -38.839744567871094,
      "activations/layer11_attention_weight_max": 39.938987731933594,
      "activations/layer11_attention_weight_min": -40.1757926940918,
      "activations/layer12_attention_weight_max": 32.32196807861328,
      "activations/layer12_attention_weight_min": -28.608919143676758,
      "activations/layer13_attention_weight_max": 49.342079162597656,
      "activations/layer13_attention_weight_min": -41.680458068847656,
      "activations/layer14_attention_weight_max": 59.886634826660156,
      "activations/layer14_attention_weight_min": -42.40092849731445,
      "activations/layer15_attention_weight_max": 54.43074035644531,
      "activations/layer15_attention_weight_min": -47.632110595703125,
      "activations/layer16_attention_weight_max": 41.11459732055664,
      "activations/layer16_attention_weight_min": -28.508747100830078,
      "activations/layer17_attention_weight_max": 35.22501754760742,
      "activations/layer17_attention_weight_min": -27.914913177490234,
      "activations/layer18_attention_weight_max": 33.34063720703125,
      "activations/layer18_attention_weight_min": -25.330806732177734,
      "activations/layer19_attention_weight_max": 35.92158508300781,
      "activations/layer19_attention_weight_min": -33.706172943115234,
      "activations/layer1_attention_weight_max": 15.765907287597656,
      "activations/layer1_attention_weight_min": -15.78880786895752,
      "activations/layer20_attention_weight_max": 30.355064392089844,
      "activations/layer20_attention_weight_min": -26.44256591796875,
      "activations/layer21_attention_weight_max": 28.666118621826172,
      "activations/layer21_attention_weight_min": -24.250444412231445,
      "activations/layer22_attention_weight_max": 44.88956069946289,
      "activations/layer22_attention_weight_min": -34.8485221862793,
      "activations/layer23_attention_weight_max": 34.61623001098633,
      "activations/layer23_attention_weight_min": -27.423721313476562,
      "activations/layer2_attention_weight_max": 34.50936508178711,
      "activations/layer2_attention_weight_min": -34.245216369628906,
      "activations/layer3_attention_weight_max": 111.34529876708984,
      "activations/layer3_attention_weight_min": -117.20423889160156,
      "activations/layer4_attention_weight_max": 114.51773834228516,
      "activations/layer4_attention_weight_min": -123.73111724853516,
      "activations/layer5_attention_weight_max": 52.74095916748047,
      "activations/layer5_attention_weight_min": -64.31005859375,
      "activations/layer6_attention_weight_max": 47.938533782958984,
      "activations/layer6_attention_weight_min": -51.33256912231445,
      "activations/layer7_attention_weight_max": 105.19580078125,
      "activations/layer7_attention_weight_min": -109.61849975585938,
      "activations/layer8_attention_weight_max": 46.36918258666992,
      "activations/layer8_attention_weight_min": -48.939998626708984,
      "activations/layer9_attention_weight_max": 36.58901596069336,
      "activations/layer9_attention_weight_min": -39.817440032958984,
      "epoch": 22.91,
      "learning_rate": 2.274242424242424e-06,
      "loss": 2.694,
      "step": 394300
    },
    {
      "activations/layer0_attention_weight_max": 17.042402267456055,
      "activations/layer0_attention_weight_min": -15.208239555358887,
      "activations/layer10_attention_weight_max": 35.880245208740234,
      "activations/layer10_attention_weight_min": -36.90772247314453,
      "activations/layer11_attention_weight_max": 36.26441955566406,
      "activations/layer11_attention_weight_min": -37.87671661376953,
      "activations/layer12_attention_weight_max": 26.684497833251953,
      "activations/layer12_attention_weight_min": -29.372249603271484,
      "activations/layer13_attention_weight_max": 42.19921875,
      "activations/layer13_attention_weight_min": -37.76856994628906,
      "activations/layer14_attention_weight_max": 41.1805305480957,
      "activations/layer14_attention_weight_min": -40.06778335571289,
      "activations/layer15_attention_weight_max": 40.68653869628906,
      "activations/layer15_attention_weight_min": -38.277565002441406,
      "activations/layer16_attention_weight_max": 28.417524337768555,
      "activations/layer16_attention_weight_min": -29.29107666015625,
      "activations/layer17_attention_weight_max": 31.528947830200195,
      "activations/layer17_attention_weight_min": -27.39076042175293,
      "activations/layer18_attention_weight_max": 33.527408599853516,
      "activations/layer18_attention_weight_min": -25.65084457397461,
      "activations/layer19_attention_weight_max": 32.3349609375,
      "activations/layer19_attention_weight_min": -33.35524368286133,
      "activations/layer1_attention_weight_max": 15.56786823272705,
      "activations/layer1_attention_weight_min": -16.655149459838867,
      "activations/layer20_attention_weight_max": 26.69139289855957,
      "activations/layer20_attention_weight_min": -26.097204208374023,
      "activations/layer21_attention_weight_max": 26.895952224731445,
      "activations/layer21_attention_weight_min": -25.508743286132812,
      "activations/layer22_attention_weight_max": 47.35139846801758,
      "activations/layer22_attention_weight_min": -32.34596252441406,
      "activations/layer23_attention_weight_max": 29.562454223632812,
      "activations/layer23_attention_weight_min": -30.29512596130371,
      "activations/layer2_attention_weight_max": 35.52422332763672,
      "activations/layer2_attention_weight_min": -35.906185150146484,
      "activations/layer3_attention_weight_max": 107.06912994384766,
      "activations/layer3_attention_weight_min": -108.14625549316406,
      "activations/layer4_attention_weight_max": 111.60819244384766,
      "activations/layer4_attention_weight_min": -115.39637756347656,
      "activations/layer5_attention_weight_max": 51.5145378112793,
      "activations/layer5_attention_weight_min": -62.692745208740234,
      "activations/layer6_attention_weight_max": 46.4145393371582,
      "activations/layer6_attention_weight_min": -50.13984680175781,
      "activations/layer7_attention_weight_max": 98.31373596191406,
      "activations/layer7_attention_weight_min": -104.52935028076172,
      "activations/layer8_attention_weight_max": 43.828731536865234,
      "activations/layer8_attention_weight_min": -47.07577133178711,
      "activations/layer9_attention_weight_max": 32.99245071411133,
      "activations/layer9_attention_weight_min": -36.14067840576172,
      "epoch": 22.91,
      "learning_rate": 2.2553030303030303e-06,
      "loss": 2.6953,
      "step": 394350
    },
    {
      "activations/layer0_attention_weight_max": 16.918344497680664,
      "activations/layer0_attention_weight_min": -13.962396621704102,
      "activations/layer10_attention_weight_max": 33.567989349365234,
      "activations/layer10_attention_weight_min": -36.53740310668945,
      "activations/layer11_attention_weight_max": 33.929290771484375,
      "activations/layer11_attention_weight_min": -35.14112854003906,
      "activations/layer12_attention_weight_max": 28.268775939941406,
      "activations/layer12_attention_weight_min": -27.861513137817383,
      "activations/layer13_attention_weight_max": 40.93526077270508,
      "activations/layer13_attention_weight_min": -38.209434509277344,
      "activations/layer14_attention_weight_max": 45.760353088378906,
      "activations/layer14_attention_weight_min": -40.452510833740234,
      "activations/layer15_attention_weight_max": 41.13729476928711,
      "activations/layer15_attention_weight_min": -37.796958923339844,
      "activations/layer16_attention_weight_max": 32.88603591918945,
      "activations/layer16_attention_weight_min": -29.454944610595703,
      "activations/layer17_attention_weight_max": 29.501617431640625,
      "activations/layer17_attention_weight_min": -26.654897689819336,
      "activations/layer18_attention_weight_max": 31.92732048034668,
      "activations/layer18_attention_weight_min": -26.983232498168945,
      "activations/layer19_attention_weight_max": 33.936092376708984,
      "activations/layer19_attention_weight_min": -33.19314956665039,
      "activations/layer1_attention_weight_max": 17.131864547729492,
      "activations/layer1_attention_weight_min": -15.19421672821045,
      "activations/layer20_attention_weight_max": 30.457107543945312,
      "activations/layer20_attention_weight_min": -26.66320037841797,
      "activations/layer21_attention_weight_max": 30.93802833557129,
      "activations/layer21_attention_weight_min": -25.26784324645996,
      "activations/layer22_attention_weight_max": 44.95859909057617,
      "activations/layer22_attention_weight_min": -32.19283676147461,
      "activations/layer23_attention_weight_max": 31.5418758392334,
      "activations/layer23_attention_weight_min": -27.756778717041016,
      "activations/layer2_attention_weight_max": 34.982669830322266,
      "activations/layer2_attention_weight_min": -33.7342529296875,
      "activations/layer3_attention_weight_max": 110.2833023071289,
      "activations/layer3_attention_weight_min": -107.58128356933594,
      "activations/layer4_attention_weight_max": 116.84288787841797,
      "activations/layer4_attention_weight_min": -116.1259536743164,
      "activations/layer5_attention_weight_max": 53.156333923339844,
      "activations/layer5_attention_weight_min": -67.56659698486328,
      "activations/layer6_attention_weight_max": 46.70968246459961,
      "activations/layer6_attention_weight_min": -49.89826965332031,
      "activations/layer7_attention_weight_max": 101.28978729248047,
      "activations/layer7_attention_weight_min": -98.29063415527344,
      "activations/layer8_attention_weight_max": 43.73954772949219,
      "activations/layer8_attention_weight_min": -46.17290115356445,
      "activations/layer9_attention_weight_max": 34.49680709838867,
      "activations/layer9_attention_weight_min": -34.8184814453125,
      "epoch": 22.92,
      "learning_rate": 2.236363636363636e-06,
      "loss": 2.6746,
      "step": 394400
    },
    {
      "activations/layer0_attention_weight_max": 16.92182731628418,
      "activations/layer0_attention_weight_min": -14.310142517089844,
      "activations/layer10_attention_weight_max": 36.52436447143555,
      "activations/layer10_attention_weight_min": -38.99393844604492,
      "activations/layer11_attention_weight_max": 37.87553787231445,
      "activations/layer11_attention_weight_min": -37.738853454589844,
      "activations/layer12_attention_weight_max": 28.561933517456055,
      "activations/layer12_attention_weight_min": -28.036930084228516,
      "activations/layer13_attention_weight_max": 41.74087142944336,
      "activations/layer13_attention_weight_min": -36.68555450439453,
      "activations/layer14_attention_weight_max": 45.94408416748047,
      "activations/layer14_attention_weight_min": -42.10291290283203,
      "activations/layer15_attention_weight_max": 44.57761001586914,
      "activations/layer15_attention_weight_min": -40.613441467285156,
      "activations/layer16_attention_weight_max": 31.234956741333008,
      "activations/layer16_attention_weight_min": -30.44898223876953,
      "activations/layer17_attention_weight_max": 32.8847541809082,
      "activations/layer17_attention_weight_min": -26.761301040649414,
      "activations/layer18_attention_weight_max": 37.31264114379883,
      "activations/layer18_attention_weight_min": -27.83004379272461,
      "activations/layer19_attention_weight_max": 39.9585075378418,
      "activations/layer19_attention_weight_min": -34.2703971862793,
      "activations/layer1_attention_weight_max": 16.44706153869629,
      "activations/layer1_attention_weight_min": -15.3314790725708,
      "activations/layer20_attention_weight_max": 33.45796203613281,
      "activations/layer20_attention_weight_min": -25.021127700805664,
      "activations/layer21_attention_weight_max": 32.61512756347656,
      "activations/layer21_attention_weight_min": -24.026535034179688,
      "activations/layer22_attention_weight_max": 46.0904655456543,
      "activations/layer22_attention_weight_min": -31.02586555480957,
      "activations/layer23_attention_weight_max": 34.0572395324707,
      "activations/layer23_attention_weight_min": -24.729713439941406,
      "activations/layer2_attention_weight_max": 34.0260009765625,
      "activations/layer2_attention_weight_min": -34.07502746582031,
      "activations/layer3_attention_weight_max": 108.37010192871094,
      "activations/layer3_attention_weight_min": -108.90885925292969,
      "activations/layer4_attention_weight_max": 115.0140380859375,
      "activations/layer4_attention_weight_min": -115.88885498046875,
      "activations/layer5_attention_weight_max": 52.649715423583984,
      "activations/layer5_attention_weight_min": -64.29092407226562,
      "activations/layer6_attention_weight_max": 49.38722229003906,
      "activations/layer6_attention_weight_min": -50.81293487548828,
      "activations/layer7_attention_weight_max": 105.3741455078125,
      "activations/layer7_attention_weight_min": -110.77833557128906,
      "activations/layer8_attention_weight_max": 44.03897476196289,
      "activations/layer8_attention_weight_min": -48.6497688293457,
      "activations/layer9_attention_weight_max": 33.11652755737305,
      "activations/layer9_attention_weight_min": -37.08121871948242,
      "epoch": 22.92,
      "learning_rate": 2.217424242424242e-06,
      "loss": 2.6875,
      "step": 394450
    },
    {
      "activations/layer0_attention_weight_max": 16.578350067138672,
      "activations/layer0_attention_weight_min": -14.352633476257324,
      "activations/layer10_attention_weight_max": 37.24729919433594,
      "activations/layer10_attention_weight_min": -37.709197998046875,
      "activations/layer11_attention_weight_max": 35.223052978515625,
      "activations/layer11_attention_weight_min": -39.55012512207031,
      "activations/layer12_attention_weight_max": 28.749555587768555,
      "activations/layer12_attention_weight_min": -28.621719360351562,
      "activations/layer13_attention_weight_max": 42.435874938964844,
      "activations/layer13_attention_weight_min": -37.83344650268555,
      "activations/layer14_attention_weight_max": 50.108177185058594,
      "activations/layer14_attention_weight_min": -44.510440826416016,
      "activations/layer15_attention_weight_max": 49.30633544921875,
      "activations/layer15_attention_weight_min": -41.8775749206543,
      "activations/layer16_attention_weight_max": 36.12382888793945,
      "activations/layer16_attention_weight_min": -30.341304779052734,
      "activations/layer17_attention_weight_max": 35.25026321411133,
      "activations/layer17_attention_weight_min": -30.996973037719727,
      "activations/layer18_attention_weight_max": 39.97441864013672,
      "activations/layer18_attention_weight_min": -30.497243881225586,
      "activations/layer19_attention_weight_max": 49.077423095703125,
      "activations/layer19_attention_weight_min": -38.68734359741211,
      "activations/layer1_attention_weight_max": 15.916645050048828,
      "activations/layer1_attention_weight_min": -14.672843933105469,
      "activations/layer20_attention_weight_max": 43.2618522644043,
      "activations/layer20_attention_weight_min": -30.51824188232422,
      "activations/layer21_attention_weight_max": 40.315635681152344,
      "activations/layer21_attention_weight_min": -28.74070930480957,
      "activations/layer22_attention_weight_max": 62.52500534057617,
      "activations/layer22_attention_weight_min": -39.08847427368164,
      "activations/layer23_attention_weight_max": 45.828407287597656,
      "activations/layer23_attention_weight_min": -29.587608337402344,
      "activations/layer2_attention_weight_max": 34.646514892578125,
      "activations/layer2_attention_weight_min": -33.57340621948242,
      "activations/layer3_attention_weight_max": 108.34500122070312,
      "activations/layer3_attention_weight_min": -109.76737976074219,
      "activations/layer4_attention_weight_max": 112.19820404052734,
      "activations/layer4_attention_weight_min": -112.74720764160156,
      "activations/layer5_attention_weight_max": 52.641536712646484,
      "activations/layer5_attention_weight_min": -60.893035888671875,
      "activations/layer6_attention_weight_max": 47.33989334106445,
      "activations/layer6_attention_weight_min": -48.67417907714844,
      "activations/layer7_attention_weight_max": 96.85099792480469,
      "activations/layer7_attention_weight_min": -105.78324127197266,
      "activations/layer8_attention_weight_max": 44.294185638427734,
      "activations/layer8_attention_weight_min": -47.14311599731445,
      "activations/layer9_attention_weight_max": 34.175376892089844,
      "activations/layer9_attention_weight_min": -36.33737564086914,
      "epoch": 22.92,
      "learning_rate": 2.1984848484848483e-06,
      "loss": 2.7117,
      "step": 394500
    },
    {
      "activations/layer0_attention_weight_max": 17.60554313659668,
      "activations/layer0_attention_weight_min": -14.169073104858398,
      "activations/layer10_attention_weight_max": 42.816680908203125,
      "activations/layer10_attention_weight_min": -40.716670989990234,
      "activations/layer11_attention_weight_max": 43.55489730834961,
      "activations/layer11_attention_weight_min": -42.219322204589844,
      "activations/layer12_attention_weight_max": 30.48213005065918,
      "activations/layer12_attention_weight_min": -29.443178176879883,
      "activations/layer13_attention_weight_max": 45.043575286865234,
      "activations/layer13_attention_weight_min": -39.608577728271484,
      "activations/layer14_attention_weight_max": 54.81951141357422,
      "activations/layer14_attention_weight_min": -43.47866439819336,
      "activations/layer15_attention_weight_max": 51.71953201293945,
      "activations/layer15_attention_weight_min": -41.24002456665039,
      "activations/layer16_attention_weight_max": 37.065433502197266,
      "activations/layer16_attention_weight_min": -30.57362937927246,
      "activations/layer17_attention_weight_max": 34.081321716308594,
      "activations/layer17_attention_weight_min": -27.796504974365234,
      "activations/layer18_attention_weight_max": 34.24863052368164,
      "activations/layer18_attention_weight_min": -28.479543685913086,
      "activations/layer19_attention_weight_max": 42.63546371459961,
      "activations/layer19_attention_weight_min": -34.85941696166992,
      "activations/layer1_attention_weight_max": 16.588407516479492,
      "activations/layer1_attention_weight_min": -15.279998779296875,
      "activations/layer20_attention_weight_max": 33.36276626586914,
      "activations/layer20_attention_weight_min": -25.065462112426758,
      "activations/layer21_attention_weight_max": 30.878097534179688,
      "activations/layer21_attention_weight_min": -23.26304054260254,
      "activations/layer22_attention_weight_max": 51.904510498046875,
      "activations/layer22_attention_weight_min": -31.19435691833496,
      "activations/layer23_attention_weight_max": 34.72332763671875,
      "activations/layer23_attention_weight_min": -24.100929260253906,
      "activations/layer2_attention_weight_max": 35.39447784423828,
      "activations/layer2_attention_weight_min": -34.0008544921875,
      "activations/layer3_attention_weight_max": 113.39212799072266,
      "activations/layer3_attention_weight_min": -115.75743865966797,
      "activations/layer4_attention_weight_max": 116.75025939941406,
      "activations/layer4_attention_weight_min": -113.56200408935547,
      "activations/layer5_attention_weight_max": 55.78595733642578,
      "activations/layer5_attention_weight_min": -61.23969268798828,
      "activations/layer6_attention_weight_max": 50.115596771240234,
      "activations/layer6_attention_weight_min": -51.61863327026367,
      "activations/layer7_attention_weight_max": 108.49046325683594,
      "activations/layer7_attention_weight_min": -110.20941925048828,
      "activations/layer8_attention_weight_max": 50.51503372192383,
      "activations/layer8_attention_weight_min": -50.57490921020508,
      "activations/layer9_attention_weight_max": 36.179439544677734,
      "activations/layer9_attention_weight_min": -40.27143478393555,
      "epoch": 22.93,
      "learning_rate": 2.1795454545454545e-06,
      "loss": 2.6984,
      "step": 394550
    },
    {
      "activations/layer0_attention_weight_max": 17.026020050048828,
      "activations/layer0_attention_weight_min": -14.908869743347168,
      "activations/layer10_attention_weight_max": 35.48175048828125,
      "activations/layer10_attention_weight_min": -36.78814697265625,
      "activations/layer11_attention_weight_max": 35.84092712402344,
      "activations/layer11_attention_weight_min": -37.1610107421875,
      "activations/layer12_attention_weight_max": 25.942684173583984,
      "activations/layer12_attention_weight_min": -27.0601863861084,
      "activations/layer13_attention_weight_max": 39.16630172729492,
      "activations/layer13_attention_weight_min": -36.58418273925781,
      "activations/layer14_attention_weight_max": 42.32185363769531,
      "activations/layer14_attention_weight_min": -37.7160530090332,
      "activations/layer15_attention_weight_max": 40.79315948486328,
      "activations/layer15_attention_weight_min": -38.39908218383789,
      "activations/layer16_attention_weight_max": 32.50554656982422,
      "activations/layer16_attention_weight_min": -29.164783477783203,
      "activations/layer17_attention_weight_max": 30.29859161376953,
      "activations/layer17_attention_weight_min": -25.654001235961914,
      "activations/layer18_attention_weight_max": 34.290367126464844,
      "activations/layer18_attention_weight_min": -24.478063583374023,
      "activations/layer19_attention_weight_max": 34.113128662109375,
      "activations/layer19_attention_weight_min": -29.863754272460938,
      "activations/layer1_attention_weight_max": 15.839261054992676,
      "activations/layer1_attention_weight_min": -15.08347225189209,
      "activations/layer20_attention_weight_max": 30.533897399902344,
      "activations/layer20_attention_weight_min": -24.11016082763672,
      "activations/layer21_attention_weight_max": 31.28801727294922,
      "activations/layer21_attention_weight_min": -23.55925178527832,
      "activations/layer22_attention_weight_max": 45.26896286010742,
      "activations/layer22_attention_weight_min": -32.162254333496094,
      "activations/layer23_attention_weight_max": 33.335655212402344,
      "activations/layer23_attention_weight_min": -26.89334487915039,
      "activations/layer2_attention_weight_max": 34.33006286621094,
      "activations/layer2_attention_weight_min": -34.12122344970703,
      "activations/layer3_attention_weight_max": 111.365478515625,
      "activations/layer3_attention_weight_min": -112.87701416015625,
      "activations/layer4_attention_weight_max": 112.53990936279297,
      "activations/layer4_attention_weight_min": -115.12646484375,
      "activations/layer5_attention_weight_max": 56.08735656738281,
      "activations/layer5_attention_weight_min": -65.10910034179688,
      "activations/layer6_attention_weight_max": 49.82111740112305,
      "activations/layer6_attention_weight_min": -49.072410583496094,
      "activations/layer7_attention_weight_max": 101.49007415771484,
      "activations/layer7_attention_weight_min": -104.12145233154297,
      "activations/layer8_attention_weight_max": 45.450050354003906,
      "activations/layer8_attention_weight_min": -47.6754035949707,
      "activations/layer9_attention_weight_max": 33.58787155151367,
      "activations/layer9_attention_weight_min": -36.463863372802734,
      "epoch": 22.93,
      "learning_rate": 2.16060606060606e-06,
      "loss": 2.6818,
      "step": 394600
    },
    {
      "activations/layer0_attention_weight_max": 17.285730361938477,
      "activations/layer0_attention_weight_min": -14.219624519348145,
      "activations/layer10_attention_weight_max": 36.5244026184082,
      "activations/layer10_attention_weight_min": -36.4488639831543,
      "activations/layer11_attention_weight_max": 36.58392333984375,
      "activations/layer11_attention_weight_min": -36.23104476928711,
      "activations/layer12_attention_weight_max": 27.629840850830078,
      "activations/layer12_attention_weight_min": -29.439699172973633,
      "activations/layer13_attention_weight_max": 40.516090393066406,
      "activations/layer13_attention_weight_min": -38.91549301147461,
      "activations/layer14_attention_weight_max": 44.778160095214844,
      "activations/layer14_attention_weight_min": -41.32076644897461,
      "activations/layer15_attention_weight_max": 41.072837829589844,
      "activations/layer15_attention_weight_min": -40.126953125,
      "activations/layer16_attention_weight_max": 31.450040817260742,
      "activations/layer16_attention_weight_min": -28.74452018737793,
      "activations/layer17_attention_weight_max": 31.80502700805664,
      "activations/layer17_attention_weight_min": -29.43186378479004,
      "activations/layer18_attention_weight_max": 33.33040237426758,
      "activations/layer18_attention_weight_min": -30.265541076660156,
      "activations/layer19_attention_weight_max": 37.58011245727539,
      "activations/layer19_attention_weight_min": -37.69124221801758,
      "activations/layer1_attention_weight_max": 16.496307373046875,
      "activations/layer1_attention_weight_min": -16.642004013061523,
      "activations/layer20_attention_weight_max": 29.9141902923584,
      "activations/layer20_attention_weight_min": -27.958507537841797,
      "activations/layer21_attention_weight_max": 31.740360260009766,
      "activations/layer21_attention_weight_min": -29.163253784179688,
      "activations/layer22_attention_weight_max": 44.5417594909668,
      "activations/layer22_attention_weight_min": -35.52136993408203,
      "activations/layer23_attention_weight_max": 32.05961608886719,
      "activations/layer23_attention_weight_min": -27.28154754638672,
      "activations/layer2_attention_weight_max": 34.37506103515625,
      "activations/layer2_attention_weight_min": -33.231502532958984,
      "activations/layer3_attention_weight_max": 110.51862335205078,
      "activations/layer3_attention_weight_min": -110.57621002197266,
      "activations/layer4_attention_weight_max": 111.38243865966797,
      "activations/layer4_attention_weight_min": -118.17557525634766,
      "activations/layer5_attention_weight_max": 53.00978088378906,
      "activations/layer5_attention_weight_min": -64.4307632446289,
      "activations/layer6_attention_weight_max": 48.843955993652344,
      "activations/layer6_attention_weight_min": -49.91554260253906,
      "activations/layer7_attention_weight_max": 106.99845123291016,
      "activations/layer7_attention_weight_min": -108.61322784423828,
      "activations/layer8_attention_weight_max": 45.59261703491211,
      "activations/layer8_attention_weight_min": -47.77218246459961,
      "activations/layer9_attention_weight_max": 33.8205451965332,
      "activations/layer9_attention_weight_min": -37.309974670410156,
      "epoch": 22.93,
      "learning_rate": 2.1416666666666663e-06,
      "loss": 2.6944,
      "step": 394650
    },
    {
      "activations/layer0_attention_weight_max": 16.54690170288086,
      "activations/layer0_attention_weight_min": -14.704221725463867,
      "activations/layer10_attention_weight_max": 37.73564147949219,
      "activations/layer10_attention_weight_min": -40.790401458740234,
      "activations/layer11_attention_weight_max": 38.68506622314453,
      "activations/layer11_attention_weight_min": -40.832298278808594,
      "activations/layer12_attention_weight_max": 28.847305297851562,
      "activations/layer12_attention_weight_min": -27.721927642822266,
      "activations/layer13_attention_weight_max": 40.9256477355957,
      "activations/layer13_attention_weight_min": -36.5098876953125,
      "activations/layer14_attention_weight_max": 45.94059371948242,
      "activations/layer14_attention_weight_min": -39.85469055175781,
      "activations/layer15_attention_weight_max": 42.67756652832031,
      "activations/layer15_attention_weight_min": -39.65912628173828,
      "activations/layer16_attention_weight_max": 31.20008087158203,
      "activations/layer16_attention_weight_min": -29.732210159301758,
      "activations/layer17_attention_weight_max": 30.182857513427734,
      "activations/layer17_attention_weight_min": -28.780609130859375,
      "activations/layer18_attention_weight_max": 31.670673370361328,
      "activations/layer18_attention_weight_min": -27.645126342773438,
      "activations/layer19_attention_weight_max": 36.29157257080078,
      "activations/layer19_attention_weight_min": -33.15024948120117,
      "activations/layer1_attention_weight_max": 15.963068962097168,
      "activations/layer1_attention_weight_min": -16.0521240234375,
      "activations/layer20_attention_weight_max": 27.89004135131836,
      "activations/layer20_attention_weight_min": -27.454172134399414,
      "activations/layer21_attention_weight_max": 27.500619888305664,
      "activations/layer21_attention_weight_min": -25.33881187438965,
      "activations/layer22_attention_weight_max": 38.17831802368164,
      "activations/layer22_attention_weight_min": -31.68107032775879,
      "activations/layer23_attention_weight_max": 30.034711837768555,
      "activations/layer23_attention_weight_min": -27.42510223388672,
      "activations/layer2_attention_weight_max": 34.25654602050781,
      "activations/layer2_attention_weight_min": -35.034645080566406,
      "activations/layer3_attention_weight_max": 110.2899398803711,
      "activations/layer3_attention_weight_min": -111.4560775756836,
      "activations/layer4_attention_weight_max": 114.2274398803711,
      "activations/layer4_attention_weight_min": -114.47026824951172,
      "activations/layer5_attention_weight_max": 51.40895080566406,
      "activations/layer5_attention_weight_min": -64.60345458984375,
      "activations/layer6_attention_weight_max": 47.765323638916016,
      "activations/layer6_attention_weight_min": -50.304203033447266,
      "activations/layer7_attention_weight_max": 103.49246215820312,
      "activations/layer7_attention_weight_min": -116.2270278930664,
      "activations/layer8_attention_weight_max": 46.48692321777344,
      "activations/layer8_attention_weight_min": -52.279117584228516,
      "activations/layer9_attention_weight_max": 35.75336456298828,
      "activations/layer9_attention_weight_min": -40.1121711730957,
      "epoch": 22.93,
      "learning_rate": 2.1227272727272725e-06,
      "loss": 2.6973,
      "step": 394700
    },
    {
      "activations/layer0_attention_weight_max": 17.080245971679688,
      "activations/layer0_attention_weight_min": -14.774590492248535,
      "activations/layer10_attention_weight_max": 35.73322296142578,
      "activations/layer10_attention_weight_min": -35.41801834106445,
      "activations/layer11_attention_weight_max": 35.53302001953125,
      "activations/layer11_attention_weight_min": -34.257083892822266,
      "activations/layer12_attention_weight_max": 28.83591079711914,
      "activations/layer12_attention_weight_min": -28.45047950744629,
      "activations/layer13_attention_weight_max": 43.426456451416016,
      "activations/layer13_attention_weight_min": -39.095741271972656,
      "activations/layer14_attention_weight_max": 46.15251541137695,
      "activations/layer14_attention_weight_min": -40.67534637451172,
      "activations/layer15_attention_weight_max": 44.455810546875,
      "activations/layer15_attention_weight_min": -37.90725326538086,
      "activations/layer16_attention_weight_max": 33.66324234008789,
      "activations/layer16_attention_weight_min": -29.319217681884766,
      "activations/layer17_attention_weight_max": 32.590267181396484,
      "activations/layer17_attention_weight_min": -27.138525009155273,
      "activations/layer18_attention_weight_max": 32.386226654052734,
      "activations/layer18_attention_weight_min": -27.709068298339844,
      "activations/layer19_attention_weight_max": 36.6934928894043,
      "activations/layer19_attention_weight_min": -35.369537353515625,
      "activations/layer1_attention_weight_max": 15.350608825683594,
      "activations/layer1_attention_weight_min": -15.16036319732666,
      "activations/layer20_attention_weight_max": 31.08818817138672,
      "activations/layer20_attention_weight_min": -28.024389266967773,
      "activations/layer21_attention_weight_max": 30.10159683227539,
      "activations/layer21_attention_weight_min": -27.64573097229004,
      "activations/layer22_attention_weight_max": 40.078495025634766,
      "activations/layer22_attention_weight_min": -34.00236129760742,
      "activations/layer23_attention_weight_max": 32.596473693847656,
      "activations/layer23_attention_weight_min": -28.52838706970215,
      "activations/layer2_attention_weight_max": 33.36607360839844,
      "activations/layer2_attention_weight_min": -34.359954833984375,
      "activations/layer3_attention_weight_max": 107.25796508789062,
      "activations/layer3_attention_weight_min": -111.98091125488281,
      "activations/layer4_attention_weight_max": 110.95870208740234,
      "activations/layer4_attention_weight_min": -113.1250228881836,
      "activations/layer5_attention_weight_max": 52.32508087158203,
      "activations/layer5_attention_weight_min": -62.074947357177734,
      "activations/layer6_attention_weight_max": 46.9970817565918,
      "activations/layer6_attention_weight_min": -48.96164321899414,
      "activations/layer7_attention_weight_max": 100.52369689941406,
      "activations/layer7_attention_weight_min": -99.8846435546875,
      "activations/layer8_attention_weight_max": 42.50382614135742,
      "activations/layer8_attention_weight_min": -47.58097457885742,
      "activations/layer9_attention_weight_max": 33.32432556152344,
      "activations/layer9_attention_weight_min": -38.038578033447266,
      "epoch": 22.94,
      "learning_rate": 2.1041666666666667e-06,
      "loss": 2.6839,
      "step": 394750
    },
    {
      "activations/layer0_attention_weight_max": 17.934329986572266,
      "activations/layer0_attention_weight_min": -15.51232624053955,
      "activations/layer10_attention_weight_max": 34.4478874206543,
      "activations/layer10_attention_weight_min": -35.46962356567383,
      "activations/layer11_attention_weight_max": 34.740760803222656,
      "activations/layer11_attention_weight_min": -35.33818817138672,
      "activations/layer12_attention_weight_max": 26.34054183959961,
      "activations/layer12_attention_weight_min": -26.93387794494629,
      "activations/layer13_attention_weight_max": 37.78392791748047,
      "activations/layer13_attention_weight_min": -35.43068313598633,
      "activations/layer14_attention_weight_max": 41.89676284790039,
      "activations/layer14_attention_weight_min": -38.343292236328125,
      "activations/layer15_attention_weight_max": 38.94129943847656,
      "activations/layer15_attention_weight_min": -36.127723693847656,
      "activations/layer16_attention_weight_max": 29.14761734008789,
      "activations/layer16_attention_weight_min": -28.083433151245117,
      "activations/layer17_attention_weight_max": 29.35976791381836,
      "activations/layer17_attention_weight_min": -25.920900344848633,
      "activations/layer18_attention_weight_max": 30.926334381103516,
      "activations/layer18_attention_weight_min": -25.349166870117188,
      "activations/layer19_attention_weight_max": 33.751007080078125,
      "activations/layer19_attention_weight_min": -31.9226016998291,
      "activations/layer1_attention_weight_max": 15.543749809265137,
      "activations/layer1_attention_weight_min": -14.501063346862793,
      "activations/layer20_attention_weight_max": 28.831052780151367,
      "activations/layer20_attention_weight_min": -24.696144104003906,
      "activations/layer21_attention_weight_max": 28.656204223632812,
      "activations/layer21_attention_weight_min": -23.569543838500977,
      "activations/layer22_attention_weight_max": 38.32412338256836,
      "activations/layer22_attention_weight_min": -30.89467430114746,
      "activations/layer23_attention_weight_max": 32.59687423706055,
      "activations/layer23_attention_weight_min": -25.806793212890625,
      "activations/layer2_attention_weight_max": 34.03998565673828,
      "activations/layer2_attention_weight_min": -33.12043380737305,
      "activations/layer3_attention_weight_max": 108.04619598388672,
      "activations/layer3_attention_weight_min": -112.23854064941406,
      "activations/layer4_attention_weight_max": 115.5502700805664,
      "activations/layer4_attention_weight_min": -113.0491714477539,
      "activations/layer5_attention_weight_max": 51.278194427490234,
      "activations/layer5_attention_weight_min": -65.22238159179688,
      "activations/layer6_attention_weight_max": 48.534767150878906,
      "activations/layer6_attention_weight_min": -50.502994537353516,
      "activations/layer7_attention_weight_max": 99.62623596191406,
      "activations/layer7_attention_weight_min": -104.20716094970703,
      "activations/layer8_attention_weight_max": 42.05467987060547,
      "activations/layer8_attention_weight_min": -47.153968811035156,
      "activations/layer9_attention_weight_max": 32.823062896728516,
      "activations/layer9_attention_weight_min": -36.370174407958984,
      "epoch": 22.94,
      "learning_rate": 2.0852272727272724e-06,
      "loss": 2.7011,
      "step": 394800
    },
    {
      "activations/layer0_attention_weight_max": 16.667551040649414,
      "activations/layer0_attention_weight_min": -14.047707557678223,
      "activations/layer10_attention_weight_max": 36.770973205566406,
      "activations/layer10_attention_weight_min": -36.26539611816406,
      "activations/layer11_attention_weight_max": 34.4498176574707,
      "activations/layer11_attention_weight_min": -37.964439392089844,
      "activations/layer12_attention_weight_max": 25.85818862915039,
      "activations/layer12_attention_weight_min": -26.659181594848633,
      "activations/layer13_attention_weight_max": 36.5470085144043,
      "activations/layer13_attention_weight_min": -35.69228744506836,
      "activations/layer14_attention_weight_max": 40.475128173828125,
      "activations/layer14_attention_weight_min": -38.07772445678711,
      "activations/layer15_attention_weight_max": 38.6782341003418,
      "activations/layer15_attention_weight_min": -34.04288101196289,
      "activations/layer16_attention_weight_max": 29.16923713684082,
      "activations/layer16_attention_weight_min": -28.46550941467285,
      "activations/layer17_attention_weight_max": 31.147701263427734,
      "activations/layer17_attention_weight_min": -26.952959060668945,
      "activations/layer18_attention_weight_max": 35.5623664855957,
      "activations/layer18_attention_weight_min": -26.92300033569336,
      "activations/layer19_attention_weight_max": 34.309410095214844,
      "activations/layer19_attention_weight_min": -36.17148971557617,
      "activations/layer1_attention_weight_max": 15.724160194396973,
      "activations/layer1_attention_weight_min": -14.731986045837402,
      "activations/layer20_attention_weight_max": 29.117656707763672,
      "activations/layer20_attention_weight_min": -25.634536743164062,
      "activations/layer21_attention_weight_max": 27.457082748413086,
      "activations/layer21_attention_weight_min": -26.94978904724121,
      "activations/layer22_attention_weight_max": 39.27705001831055,
      "activations/layer22_attention_weight_min": -34.32670593261719,
      "activations/layer23_attention_weight_max": 31.72772789001465,
      "activations/layer23_attention_weight_min": -27.184513092041016,
      "activations/layer2_attention_weight_max": 34.588294982910156,
      "activations/layer2_attention_weight_min": -34.708526611328125,
      "activations/layer3_attention_weight_max": 108.7229995727539,
      "activations/layer3_attention_weight_min": -113.47888946533203,
      "activations/layer4_attention_weight_max": 111.05217742919922,
      "activations/layer4_attention_weight_min": -118.97663879394531,
      "activations/layer5_attention_weight_max": 51.7830924987793,
      "activations/layer5_attention_weight_min": -62.43008041381836,
      "activations/layer6_attention_weight_max": 46.30280303955078,
      "activations/layer6_attention_weight_min": -48.66445541381836,
      "activations/layer7_attention_weight_max": 99.51725006103516,
      "activations/layer7_attention_weight_min": -98.90319061279297,
      "activations/layer8_attention_weight_max": 43.20158386230469,
      "activations/layer8_attention_weight_min": -46.68538284301758,
      "activations/layer9_attention_weight_max": 32.19059753417969,
      "activations/layer9_attention_weight_min": -37.80006790161133,
      "epoch": 22.94,
      "learning_rate": 2.0662878787878786e-06,
      "loss": 2.7258,
      "step": 394850
    },
    {
      "activations/layer0_attention_weight_max": 16.55723762512207,
      "activations/layer0_attention_weight_min": -14.218523025512695,
      "activations/layer10_attention_weight_max": 32.7437744140625,
      "activations/layer10_attention_weight_min": -34.444366455078125,
      "activations/layer11_attention_weight_max": 31.195858001708984,
      "activations/layer11_attention_weight_min": -34.75236129760742,
      "activations/layer12_attention_weight_max": 25.85698699951172,
      "activations/layer12_attention_weight_min": -27.240421295166016,
      "activations/layer13_attention_weight_max": 37.778133392333984,
      "activations/layer13_attention_weight_min": -35.97596740722656,
      "activations/layer14_attention_weight_max": 38.43746566772461,
      "activations/layer14_attention_weight_min": -39.444400787353516,
      "activations/layer15_attention_weight_max": 36.51087188720703,
      "activations/layer15_attention_weight_min": -37.0985221862793,
      "activations/layer16_attention_weight_max": 28.57500648498535,
      "activations/layer16_attention_weight_min": -30.002426147460938,
      "activations/layer17_attention_weight_max": 26.3085994720459,
      "activations/layer17_attention_weight_min": -27.810365676879883,
      "activations/layer18_attention_weight_max": 32.218963623046875,
      "activations/layer18_attention_weight_min": -26.596176147460938,
      "activations/layer19_attention_weight_max": 34.90563201904297,
      "activations/layer19_attention_weight_min": -35.828636169433594,
      "activations/layer1_attention_weight_max": 15.238700866699219,
      "activations/layer1_attention_weight_min": -15.425345420837402,
      "activations/layer20_attention_weight_max": 27.14826011657715,
      "activations/layer20_attention_weight_min": -25.137332916259766,
      "activations/layer21_attention_weight_max": 29.415342330932617,
      "activations/layer21_attention_weight_min": -25.833738327026367,
      "activations/layer22_attention_weight_max": 42.47414016723633,
      "activations/layer22_attention_weight_min": -33.111358642578125,
      "activations/layer23_attention_weight_max": 36.68331527709961,
      "activations/layer23_attention_weight_min": -28.01885223388672,
      "activations/layer2_attention_weight_max": 35.050262451171875,
      "activations/layer2_attention_weight_min": -33.47200012207031,
      "activations/layer3_attention_weight_max": 109.97753143310547,
      "activations/layer3_attention_weight_min": -109.24015045166016,
      "activations/layer4_attention_weight_max": 114.59584045410156,
      "activations/layer4_attention_weight_min": -118.66683197021484,
      "activations/layer5_attention_weight_max": 55.15530014038086,
      "activations/layer5_attention_weight_min": -68.48303985595703,
      "activations/layer6_attention_weight_max": 46.226863861083984,
      "activations/layer6_attention_weight_min": -50.326271057128906,
      "activations/layer7_attention_weight_max": 96.31394958496094,
      "activations/layer7_attention_weight_min": -104.46644592285156,
      "activations/layer8_attention_weight_max": 40.699954986572266,
      "activations/layer8_attention_weight_min": -45.82182693481445,
      "activations/layer9_attention_weight_max": 32.23046875,
      "activations/layer9_attention_weight_min": -33.97380447387695,
      "epoch": 22.95,
      "learning_rate": 2.0473484848484847e-06,
      "loss": 2.6979,
      "step": 394900
    },
    {
      "activations/layer0_attention_weight_max": 17.572172164916992,
      "activations/layer0_attention_weight_min": -15.488340377807617,
      "activations/layer10_attention_weight_max": 38.48550796508789,
      "activations/layer10_attention_weight_min": -37.386497497558594,
      "activations/layer11_attention_weight_max": 42.20730972290039,
      "activations/layer11_attention_weight_min": -41.686851501464844,
      "activations/layer12_attention_weight_max": 26.994203567504883,
      "activations/layer12_attention_weight_min": -27.32756996154785,
      "activations/layer13_attention_weight_max": 36.830238342285156,
      "activations/layer13_attention_weight_min": -35.082550048828125,
      "activations/layer14_attention_weight_max": 42.10674285888672,
      "activations/layer14_attention_weight_min": -41.35319137573242,
      "activations/layer15_attention_weight_max": 41.58029556274414,
      "activations/layer15_attention_weight_min": -40.041526794433594,
      "activations/layer16_attention_weight_max": 30.548213958740234,
      "activations/layer16_attention_weight_min": -30.011154174804688,
      "activations/layer17_attention_weight_max": 28.41745376586914,
      "activations/layer17_attention_weight_min": -24.765113830566406,
      "activations/layer18_attention_weight_max": 30.83523178100586,
      "activations/layer18_attention_weight_min": -25.683969497680664,
      "activations/layer19_attention_weight_max": 35.89927291870117,
      "activations/layer19_attention_weight_min": -31.061359405517578,
      "activations/layer1_attention_weight_max": 17.28241729736328,
      "activations/layer1_attention_weight_min": -15.562577247619629,
      "activations/layer20_attention_weight_max": 28.786785125732422,
      "activations/layer20_attention_weight_min": -23.194612503051758,
      "activations/layer21_attention_weight_max": 26.34316062927246,
      "activations/layer21_attention_weight_min": -23.726613998413086,
      "activations/layer22_attention_weight_max": 36.96779251098633,
      "activations/layer22_attention_weight_min": -31.037229537963867,
      "activations/layer23_attention_weight_max": 29.334333419799805,
      "activations/layer23_attention_weight_min": -23.50146484375,
      "activations/layer2_attention_weight_max": 34.58675003051758,
      "activations/layer2_attention_weight_min": -33.87239456176758,
      "activations/layer3_attention_weight_max": 111.30702209472656,
      "activations/layer3_attention_weight_min": -111.07190704345703,
      "activations/layer4_attention_weight_max": 114.61461639404297,
      "activations/layer4_attention_weight_min": -117.62945556640625,
      "activations/layer5_attention_weight_max": 53.22217559814453,
      "activations/layer5_attention_weight_min": -64.16767120361328,
      "activations/layer6_attention_weight_max": 47.986549377441406,
      "activations/layer6_attention_weight_min": -50.24321746826172,
      "activations/layer7_attention_weight_max": 101.98780822753906,
      "activations/layer7_attention_weight_min": -102.74955749511719,
      "activations/layer8_attention_weight_max": 45.53496551513672,
      "activations/layer8_attention_weight_min": -46.904720306396484,
      "activations/layer9_attention_weight_max": 33.450767517089844,
      "activations/layer9_attention_weight_min": -38.264404296875,
      "epoch": 22.95,
      "learning_rate": 2.028409090909091e-06,
      "loss": 2.6928,
      "step": 394950
    },
    {
      "activations/layer0_attention_weight_max": 17.240697860717773,
      "activations/layer0_attention_weight_min": -14.222028732299805,
      "activations/layer10_attention_weight_max": 35.345008850097656,
      "activations/layer10_attention_weight_min": -37.88907241821289,
      "activations/layer11_attention_weight_max": 35.12458038330078,
      "activations/layer11_attention_weight_min": -36.942771911621094,
      "activations/layer12_attention_weight_max": 27.16756820678711,
      "activations/layer12_attention_weight_min": -27.94145965576172,
      "activations/layer13_attention_weight_max": 38.5606575012207,
      "activations/layer13_attention_weight_min": -36.75917434692383,
      "activations/layer14_attention_weight_max": 41.98210144042969,
      "activations/layer14_attention_weight_min": -40.36747360229492,
      "activations/layer15_attention_weight_max": 39.25485610961914,
      "activations/layer15_attention_weight_min": -37.4017333984375,
      "activations/layer16_attention_weight_max": 30.45835304260254,
      "activations/layer16_attention_weight_min": -29.532381057739258,
      "activations/layer17_attention_weight_max": 29.05117416381836,
      "activations/layer17_attention_weight_min": -28.010530471801758,
      "activations/layer18_attention_weight_max": 31.206172943115234,
      "activations/layer18_attention_weight_min": -27.047826766967773,
      "activations/layer19_attention_weight_max": 34.9815788269043,
      "activations/layer19_attention_weight_min": -32.91754150390625,
      "activations/layer1_attention_weight_max": 16.082578659057617,
      "activations/layer1_attention_weight_min": -15.889951705932617,
      "activations/layer20_attention_weight_max": 28.35796546936035,
      "activations/layer20_attention_weight_min": -25.370065689086914,
      "activations/layer21_attention_weight_max": 27.647642135620117,
      "activations/layer21_attention_weight_min": -23.736597061157227,
      "activations/layer22_attention_weight_max": 38.07617950439453,
      "activations/layer22_attention_weight_min": -32.073333740234375,
      "activations/layer23_attention_weight_max": 30.288860321044922,
      "activations/layer23_attention_weight_min": -28.530174255371094,
      "activations/layer2_attention_weight_max": 34.39642333984375,
      "activations/layer2_attention_weight_min": -35.04541015625,
      "activations/layer3_attention_weight_max": 109.45716094970703,
      "activations/layer3_attention_weight_min": -110.40756225585938,
      "activations/layer4_attention_weight_max": 112.85829162597656,
      "activations/layer4_attention_weight_min": -116.54918670654297,
      "activations/layer5_attention_weight_max": 51.27294921875,
      "activations/layer5_attention_weight_min": -63.71371078491211,
      "activations/layer6_attention_weight_max": 47.60960006713867,
      "activations/layer6_attention_weight_min": -51.25349426269531,
      "activations/layer7_attention_weight_max": 99.54874420166016,
      "activations/layer7_attention_weight_min": -107.9016342163086,
      "activations/layer8_attention_weight_max": 44.71223831176758,
      "activations/layer8_attention_weight_min": -47.73544692993164,
      "activations/layer9_attention_weight_max": 33.62677764892578,
      "activations/layer9_attention_weight_min": -36.55018615722656,
      "epoch": 22.95,
      "learning_rate": 2.0094696969696966e-06,
      "loss": 2.6894,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.4932,
      "eval_samples_per_second": 505.582,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.4932,
      "eval_openwebtext_samples_per_second": 505.582,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9317,
      "eval_wikitext_samples_per_second": 236.056,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_lambada_loss": 2.302734375,
      "eval_lambada_ppl": 10.001492931490674,
      "eval_lambada_runtime": 9.5256,
      "eval_lambada_samples_per_second": 511.15,
      "step": 395000
    },
    {
      "activations/layer0_attention_weight_max": 16.939172744750977,
      "activations/layer0_attention_weight_min": -15.405570983886719,
      "activations/layer10_attention_weight_max": 35.9617805480957,
      "activations/layer10_attention_weight_min": -37.11679458618164,
      "activations/layer11_attention_weight_max": 35.203514099121094,
      "activations/layer11_attention_weight_min": -37.40379333496094,
      "activations/layer12_attention_weight_max": 26.86195182800293,
      "activations/layer12_attention_weight_min": -27.568363189697266,
      "activations/layer13_attention_weight_max": 36.36176681518555,
      "activations/layer13_attention_weight_min": -35.31541061401367,
      "activations/layer14_attention_weight_max": 40.676090240478516,
      "activations/layer14_attention_weight_min": -36.9042854309082,
      "activations/layer15_attention_weight_max": 36.288631439208984,
      "activations/layer15_attention_weight_min": -35.533729553222656,
      "activations/layer16_attention_weight_max": 27.658889770507812,
      "activations/layer16_attention_weight_min": -27.507888793945312,
      "activations/layer17_attention_weight_max": 27.949535369873047,
      "activations/layer17_attention_weight_min": -25.982995986938477,
      "activations/layer18_attention_weight_max": 28.475811004638672,
      "activations/layer18_attention_weight_min": -27.052480697631836,
      "activations/layer19_attention_weight_max": 33.7497673034668,
      "activations/layer19_attention_weight_min": -32.88032913208008,
      "activations/layer1_attention_weight_max": 15.717106819152832,
      "activations/layer1_attention_weight_min": -15.579923629760742,
      "activations/layer20_attention_weight_max": 26.962451934814453,
      "activations/layer20_attention_weight_min": -25.80841064453125,
      "activations/layer21_attention_weight_max": 28.88408660888672,
      "activations/layer21_attention_weight_min": -24.85894775390625,
      "activations/layer22_attention_weight_max": 42.26481246948242,
      "activations/layer22_attention_weight_min": -35.76145935058594,
      "activations/layer23_attention_weight_max": 33.07722854614258,
      "activations/layer23_attention_weight_min": -26.085386276245117,
      "activations/layer2_attention_weight_max": 33.542442321777344,
      "activations/layer2_attention_weight_min": -34.3262825012207,
      "activations/layer3_attention_weight_max": 110.38729095458984,
      "activations/layer3_attention_weight_min": -115.51997375488281,
      "activations/layer4_attention_weight_max": 115.24962615966797,
      "activations/layer4_attention_weight_min": -115.49774169921875,
      "activations/layer5_attention_weight_max": 51.071102142333984,
      "activations/layer5_attention_weight_min": -63.28874969482422,
      "activations/layer6_attention_weight_max": 46.35664749145508,
      "activations/layer6_attention_weight_min": -51.46284103393555,
      "activations/layer7_attention_weight_max": 100.60049438476562,
      "activations/layer7_attention_weight_min": -103.74237060546875,
      "activations/layer8_attention_weight_max": 42.83628845214844,
      "activations/layer8_attention_weight_min": -47.93403625488281,
      "activations/layer9_attention_weight_max": 32.81515121459961,
      "activations/layer9_attention_weight_min": -35.16146469116211,
      "epoch": 22.95,
      "learning_rate": 1.9905303030303028e-06,
      "loss": 2.6912,
      "step": 395050
    },
    {
      "activations/layer0_attention_weight_max": 16.05573844909668,
      "activations/layer0_attention_weight_min": -13.386261940002441,
      "activations/layer10_attention_weight_max": 35.310028076171875,
      "activations/layer10_attention_weight_min": -36.64918899536133,
      "activations/layer11_attention_weight_max": 35.90321731567383,
      "activations/layer11_attention_weight_min": -38.461177825927734,
      "activations/layer12_attention_weight_max": 29.21674156188965,
      "activations/layer12_attention_weight_min": -28.520166397094727,
      "activations/layer13_attention_weight_max": 40.393226623535156,
      "activations/layer13_attention_weight_min": -38.560325622558594,
      "activations/layer14_attention_weight_max": 43.48702621459961,
      "activations/layer14_attention_weight_min": -40.156192779541016,
      "activations/layer15_attention_weight_max": 41.971412658691406,
      "activations/layer15_attention_weight_min": -39.20038986206055,
      "activations/layer16_attention_weight_max": 29.64947509765625,
      "activations/layer16_attention_weight_min": -29.389135360717773,
      "activations/layer17_attention_weight_max": 29.81012725830078,
      "activations/layer17_attention_weight_min": -27.371591567993164,
      "activations/layer18_attention_weight_max": 32.62425231933594,
      "activations/layer18_attention_weight_min": -27.254642486572266,
      "activations/layer19_attention_weight_max": 33.01547622680664,
      "activations/layer19_attention_weight_min": -31.20245933532715,
      "activations/layer1_attention_weight_max": 15.485957145690918,
      "activations/layer1_attention_weight_min": -15.840903282165527,
      "activations/layer20_attention_weight_max": 29.366243362426758,
      "activations/layer20_attention_weight_min": -26.304153442382812,
      "activations/layer21_attention_weight_max": 31.263334274291992,
      "activations/layer21_attention_weight_min": -25.226608276367188,
      "activations/layer22_attention_weight_max": 40.54905700683594,
      "activations/layer22_attention_weight_min": -31.885793685913086,
      "activations/layer23_attention_weight_max": 30.79199981689453,
      "activations/layer23_attention_weight_min": -27.86774444580078,
      "activations/layer2_attention_weight_max": 34.28486633300781,
      "activations/layer2_attention_weight_min": -34.114601135253906,
      "activations/layer3_attention_weight_max": 108.283935546875,
      "activations/layer3_attention_weight_min": -111.07085418701172,
      "activations/layer4_attention_weight_max": 112.7967758178711,
      "activations/layer4_attention_weight_min": -113.55587005615234,
      "activations/layer5_attention_weight_max": 52.06719970703125,
      "activations/layer5_attention_weight_min": -65.97914123535156,
      "activations/layer6_attention_weight_max": 46.69245910644531,
      "activations/layer6_attention_weight_min": -49.69209671020508,
      "activations/layer7_attention_weight_max": 108.74024963378906,
      "activations/layer7_attention_weight_min": -102.8048324584961,
      "activations/layer8_attention_weight_max": 43.08832931518555,
      "activations/layer8_attention_weight_min": -48.195533752441406,
      "activations/layer9_attention_weight_max": 33.204307556152344,
      "activations/layer9_attention_weight_min": -36.7276725769043,
      "epoch": 22.96,
      "learning_rate": 1.971590909090909e-06,
      "loss": 2.6822,
      "step": 395100
    },
    {
      "activations/layer0_attention_weight_max": 16.38287925720215,
      "activations/layer0_attention_weight_min": -14.748348236083984,
      "activations/layer10_attention_weight_max": 35.448402404785156,
      "activations/layer10_attention_weight_min": -36.107017517089844,
      "activations/layer11_attention_weight_max": 33.11028289794922,
      "activations/layer11_attention_weight_min": -35.77619171142578,
      "activations/layer12_attention_weight_max": 27.634187698364258,
      "activations/layer12_attention_weight_min": -27.551712036132812,
      "activations/layer13_attention_weight_max": 39.99669647216797,
      "activations/layer13_attention_weight_min": -36.57283401489258,
      "activations/layer14_attention_weight_max": 43.28936004638672,
      "activations/layer14_attention_weight_min": -37.434906005859375,
      "activations/layer15_attention_weight_max": 38.84605407714844,
      "activations/layer15_attention_weight_min": -37.52375030517578,
      "activations/layer16_attention_weight_max": 30.76487922668457,
      "activations/layer16_attention_weight_min": -27.907379150390625,
      "activations/layer17_attention_weight_max": 31.259159088134766,
      "activations/layer17_attention_weight_min": -28.343942642211914,
      "activations/layer18_attention_weight_max": 31.658061981201172,
      "activations/layer18_attention_weight_min": -27.4532470703125,
      "activations/layer19_attention_weight_max": 35.89546585083008,
      "activations/layer19_attention_weight_min": -34.572776794433594,
      "activations/layer1_attention_weight_max": 16.686195373535156,
      "activations/layer1_attention_weight_min": -14.974120140075684,
      "activations/layer20_attention_weight_max": 31.739717483520508,
      "activations/layer20_attention_weight_min": -27.579811096191406,
      "activations/layer21_attention_weight_max": 31.02587127685547,
      "activations/layer21_attention_weight_min": -27.02994155883789,
      "activations/layer22_attention_weight_max": 40.40997314453125,
      "activations/layer22_attention_weight_min": -35.78947067260742,
      "activations/layer23_attention_weight_max": 30.186595916748047,
      "activations/layer23_attention_weight_min": -29.539348602294922,
      "activations/layer2_attention_weight_max": 34.07361602783203,
      "activations/layer2_attention_weight_min": -33.65582275390625,
      "activations/layer3_attention_weight_max": 106.7527847290039,
      "activations/layer3_attention_weight_min": -109.27052307128906,
      "activations/layer4_attention_weight_max": 109.56336975097656,
      "activations/layer4_attention_weight_min": -110.95086669921875,
      "activations/layer5_attention_weight_max": 49.148345947265625,
      "activations/layer5_attention_weight_min": -65.38307189941406,
      "activations/layer6_attention_weight_max": 44.60222625732422,
      "activations/layer6_attention_weight_min": -48.13768768310547,
      "activations/layer7_attention_weight_max": 94.23906707763672,
      "activations/layer7_attention_weight_min": -99.30335998535156,
      "activations/layer8_attention_weight_max": 42.47700881958008,
      "activations/layer8_attention_weight_min": -44.613407135009766,
      "activations/layer9_attention_weight_max": 32.520423889160156,
      "activations/layer9_attention_weight_min": -34.78664016723633,
      "epoch": 22.96,
      "learning_rate": 1.952651515151515e-06,
      "loss": 2.7008,
      "step": 395150
    },
    {
      "activations/layer0_attention_weight_max": 17.47366714477539,
      "activations/layer0_attention_weight_min": -13.129334449768066,
      "activations/layer10_attention_weight_max": 37.50861358642578,
      "activations/layer10_attention_weight_min": -37.79743576049805,
      "activations/layer11_attention_weight_max": 35.97284698486328,
      "activations/layer11_attention_weight_min": -35.5475959777832,
      "activations/layer12_attention_weight_max": 30.98943519592285,
      "activations/layer12_attention_weight_min": -29.758350372314453,
      "activations/layer13_attention_weight_max": 44.29747772216797,
      "activations/layer13_attention_weight_min": -40.38080596923828,
      "activations/layer14_attention_weight_max": 46.63136291503906,
      "activations/layer14_attention_weight_min": -41.37925720214844,
      "activations/layer15_attention_weight_max": 43.27635192871094,
      "activations/layer15_attention_weight_min": -40.73577117919922,
      "activations/layer16_attention_weight_max": 34.712059020996094,
      "activations/layer16_attention_weight_min": -30.26475715637207,
      "activations/layer17_attention_weight_max": 33.63789749145508,
      "activations/layer17_attention_weight_min": -29.499250411987305,
      "activations/layer18_attention_weight_max": 37.245269775390625,
      "activations/layer18_attention_weight_min": -28.597457885742188,
      "activations/layer19_attention_weight_max": 41.1861457824707,
      "activations/layer19_attention_weight_min": -33.88214874267578,
      "activations/layer1_attention_weight_max": 16.45111083984375,
      "activations/layer1_attention_weight_min": -15.204463958740234,
      "activations/layer20_attention_weight_max": 33.905517578125,
      "activations/layer20_attention_weight_min": -28.794803619384766,
      "activations/layer21_attention_weight_max": 35.48467254638672,
      "activations/layer21_attention_weight_min": -28.296489715576172,
      "activations/layer22_attention_weight_max": 51.60698699951172,
      "activations/layer22_attention_weight_min": -33.39989471435547,
      "activations/layer23_attention_weight_max": 38.13543701171875,
      "activations/layer23_attention_weight_min": -25.78209686279297,
      "activations/layer2_attention_weight_max": 35.50724411010742,
      "activations/layer2_attention_weight_min": -34.21043395996094,
      "activations/layer3_attention_weight_max": 110.8779525756836,
      "activations/layer3_attention_weight_min": -112.03606414794922,
      "activations/layer4_attention_weight_max": 116.94551086425781,
      "activations/layer4_attention_weight_min": -113.57454681396484,
      "activations/layer5_attention_weight_max": 56.14026641845703,
      "activations/layer5_attention_weight_min": -61.835350036621094,
      "activations/layer6_attention_weight_max": 48.46180725097656,
      "activations/layer6_attention_weight_min": -52.135841369628906,
      "activations/layer7_attention_weight_max": 106.03024291992188,
      "activations/layer7_attention_weight_min": -114.30065155029297,
      "activations/layer8_attention_weight_max": 49.55521011352539,
      "activations/layer8_attention_weight_min": -53.258724212646484,
      "activations/layer9_attention_weight_max": 35.67182159423828,
      "activations/layer9_attention_weight_min": -38.1009635925293,
      "epoch": 22.96,
      "learning_rate": 1.933712121212121e-06,
      "loss": 2.7052,
      "step": 395200
    },
    {
      "activations/layer0_attention_weight_max": 16.527193069458008,
      "activations/layer0_attention_weight_min": -16.161014556884766,
      "activations/layer10_attention_weight_max": 34.90180206298828,
      "activations/layer10_attention_weight_min": -35.972564697265625,
      "activations/layer11_attention_weight_max": 34.51624298095703,
      "activations/layer11_attention_weight_min": -38.056270599365234,
      "activations/layer12_attention_weight_max": 28.662128448486328,
      "activations/layer12_attention_weight_min": -25.98228645324707,
      "activations/layer13_attention_weight_max": 41.545528411865234,
      "activations/layer13_attention_weight_min": -36.94371032714844,
      "activations/layer14_attention_weight_max": 42.619815826416016,
      "activations/layer14_attention_weight_min": -37.35123062133789,
      "activations/layer15_attention_weight_max": 40.5227165222168,
      "activations/layer15_attention_weight_min": -35.83594512939453,
      "activations/layer16_attention_weight_max": 31.707351684570312,
      "activations/layer16_attention_weight_min": -28.354080200195312,
      "activations/layer17_attention_weight_max": 30.12754249572754,
      "activations/layer17_attention_weight_min": -25.21994972229004,
      "activations/layer18_attention_weight_max": 36.30119323730469,
      "activations/layer18_attention_weight_min": -27.353351593017578,
      "activations/layer19_attention_weight_max": 39.70694351196289,
      "activations/layer19_attention_weight_min": -32.160789489746094,
      "activations/layer1_attention_weight_max": 17.27692985534668,
      "activations/layer1_attention_weight_min": -17.193984985351562,
      "activations/layer20_attention_weight_max": 29.6766357421875,
      "activations/layer20_attention_weight_min": -25.906492233276367,
      "activations/layer21_attention_weight_max": 29.532047271728516,
      "activations/layer21_attention_weight_min": -23.722314834594727,
      "activations/layer22_attention_weight_max": 45.5572624206543,
      "activations/layer22_attention_weight_min": -28.412446975708008,
      "activations/layer23_attention_weight_max": 38.253570556640625,
      "activations/layer23_attention_weight_min": -24.797534942626953,
      "activations/layer2_attention_weight_max": 35.47856140136719,
      "activations/layer2_attention_weight_min": -33.72185516357422,
      "activations/layer3_attention_weight_max": 114.08860778808594,
      "activations/layer3_attention_weight_min": -111.12538146972656,
      "activations/layer4_attention_weight_max": 117.68473052978516,
      "activations/layer4_attention_weight_min": -114.73355865478516,
      "activations/layer5_attention_weight_max": 53.2255859375,
      "activations/layer5_attention_weight_min": -62.46977233886719,
      "activations/layer6_attention_weight_max": 46.1223030090332,
      "activations/layer6_attention_weight_min": -48.43415069580078,
      "activations/layer7_attention_weight_max": 103.75244140625,
      "activations/layer7_attention_weight_min": -107.7276840209961,
      "activations/layer8_attention_weight_max": 43.51802444458008,
      "activations/layer8_attention_weight_min": -48.29292678833008,
      "activations/layer9_attention_weight_max": 33.57627487182617,
      "activations/layer9_attention_weight_min": -37.142242431640625,
      "epoch": 22.97,
      "learning_rate": 1.9147727272727274e-06,
      "loss": 2.6991,
      "step": 395250
    },
    {
      "activations/layer0_attention_weight_max": 15.99990463256836,
      "activations/layer0_attention_weight_min": -14.095580101013184,
      "activations/layer10_attention_weight_max": 38.24970245361328,
      "activations/layer10_attention_weight_min": -38.93800735473633,
      "activations/layer11_attention_weight_max": 36.79689025878906,
      "activations/layer11_attention_weight_min": -37.92667007446289,
      "activations/layer12_attention_weight_max": 29.685712814331055,
      "activations/layer12_attention_weight_min": -28.830703735351562,
      "activations/layer13_attention_weight_max": 41.620567321777344,
      "activations/layer13_attention_weight_min": -37.63641357421875,
      "activations/layer14_attention_weight_max": 44.86530303955078,
      "activations/layer14_attention_weight_min": -40.71123123168945,
      "activations/layer15_attention_weight_max": 41.42763137817383,
      "activations/layer15_attention_weight_min": -38.038543701171875,
      "activations/layer16_attention_weight_max": 32.02083969116211,
      "activations/layer16_attention_weight_min": -28.807260513305664,
      "activations/layer17_attention_weight_max": 32.472625732421875,
      "activations/layer17_attention_weight_min": -28.186981201171875,
      "activations/layer18_attention_weight_max": 38.201717376708984,
      "activations/layer18_attention_weight_min": -27.831119537353516,
      "activations/layer19_attention_weight_max": 38.854618072509766,
      "activations/layer19_attention_weight_min": -34.57227325439453,
      "activations/layer1_attention_weight_max": 16.144180297851562,
      "activations/layer1_attention_weight_min": -15.224563598632812,
      "activations/layer20_attention_weight_max": 33.135528564453125,
      "activations/layer20_attention_weight_min": -27.28472900390625,
      "activations/layer21_attention_weight_max": 34.9359016418457,
      "activations/layer21_attention_weight_min": -26.736120223999023,
      "activations/layer22_attention_weight_max": 53.7813835144043,
      "activations/layer22_attention_weight_min": -36.2358283996582,
      "activations/layer23_attention_weight_max": 38.30731964111328,
      "activations/layer23_attention_weight_min": -26.989866256713867,
      "activations/layer2_attention_weight_max": 33.842811584472656,
      "activations/layer2_attention_weight_min": -34.51899719238281,
      "activations/layer3_attention_weight_max": 110.01697540283203,
      "activations/layer3_attention_weight_min": -111.1930923461914,
      "activations/layer4_attention_weight_max": 111.08983612060547,
      "activations/layer4_attention_weight_min": -117.24531555175781,
      "activations/layer5_attention_weight_max": 52.92399215698242,
      "activations/layer5_attention_weight_min": -63.24231719970703,
      "activations/layer6_attention_weight_max": 48.1652946472168,
      "activations/layer6_attention_weight_min": -52.225502014160156,
      "activations/layer7_attention_weight_max": 102.37023162841797,
      "activations/layer7_attention_weight_min": -111.10942840576172,
      "activations/layer8_attention_weight_max": 48.07619094848633,
      "activations/layer8_attention_weight_min": -50.338314056396484,
      "activations/layer9_attention_weight_max": 36.194515228271484,
      "activations/layer9_attention_weight_min": -39.614803314208984,
      "epoch": 22.97,
      "learning_rate": 1.8958333333333331e-06,
      "loss": 2.7047,
      "step": 395300
    },
    {
      "activations/layer0_attention_weight_max": 17.96046257019043,
      "activations/layer0_attention_weight_min": -15.197537422180176,
      "activations/layer10_attention_weight_max": 35.582679748535156,
      "activations/layer10_attention_weight_min": -37.02935028076172,
      "activations/layer11_attention_weight_max": 36.327117919921875,
      "activations/layer11_attention_weight_min": -35.56787872314453,
      "activations/layer12_attention_weight_max": 27.420263290405273,
      "activations/layer12_attention_weight_min": -26.51909065246582,
      "activations/layer13_attention_weight_max": 39.25468826293945,
      "activations/layer13_attention_weight_min": -36.12989807128906,
      "activations/layer14_attention_weight_max": 43.55073547363281,
      "activations/layer14_attention_weight_min": -36.93259048461914,
      "activations/layer15_attention_weight_max": 41.039581298828125,
      "activations/layer15_attention_weight_min": -35.660282135009766,
      "activations/layer16_attention_weight_max": 31.200037002563477,
      "activations/layer16_attention_weight_min": -28.706226348876953,
      "activations/layer17_attention_weight_max": 30.268131256103516,
      "activations/layer17_attention_weight_min": -26.488378524780273,
      "activations/layer18_attention_weight_max": 30.644760131835938,
      "activations/layer18_attention_weight_min": -25.320280075073242,
      "activations/layer19_attention_weight_max": 33.45645523071289,
      "activations/layer19_attention_weight_min": -33.043968200683594,
      "activations/layer1_attention_weight_max": 16.087646484375,
      "activations/layer1_attention_weight_min": -14.859745979309082,
      "activations/layer20_attention_weight_max": 28.987215042114258,
      "activations/layer20_attention_weight_min": -25.54568099975586,
      "activations/layer21_attention_weight_max": 28.260446548461914,
      "activations/layer21_attention_weight_min": -25.384262084960938,
      "activations/layer22_attention_weight_max": 40.07144546508789,
      "activations/layer22_attention_weight_min": -30.51620101928711,
      "activations/layer23_attention_weight_max": 31.20785903930664,
      "activations/layer23_attention_weight_min": -26.778867721557617,
      "activations/layer2_attention_weight_max": 34.392333984375,
      "activations/layer2_attention_weight_min": -35.37260818481445,
      "activations/layer3_attention_weight_max": 110.85365295410156,
      "activations/layer3_attention_weight_min": -107.80684661865234,
      "activations/layer4_attention_weight_max": 115.0638427734375,
      "activations/layer4_attention_weight_min": -117.58690643310547,
      "activations/layer5_attention_weight_max": 50.93480682373047,
      "activations/layer5_attention_weight_min": -66.9024887084961,
      "activations/layer6_attention_weight_max": 47.068180084228516,
      "activations/layer6_attention_weight_min": -49.50461196899414,
      "activations/layer7_attention_weight_max": 95.4435043334961,
      "activations/layer7_attention_weight_min": -100.08030700683594,
      "activations/layer8_attention_weight_max": 42.71155548095703,
      "activations/layer8_attention_weight_min": -45.97706985473633,
      "activations/layer9_attention_weight_max": 33.12446212768555,
      "activations/layer9_attention_weight_min": -35.806419372558594,
      "epoch": 22.97,
      "learning_rate": 1.8768939393939393e-06,
      "loss": 2.696,
      "step": 395350
    },
    {
      "activations/layer0_attention_weight_max": 16.075563430786133,
      "activations/layer0_attention_weight_min": -14.897316932678223,
      "activations/layer10_attention_weight_max": 39.851837158203125,
      "activations/layer10_attention_weight_min": -38.23040008544922,
      "activations/layer11_attention_weight_max": 38.055442810058594,
      "activations/layer11_attention_weight_min": -40.322425842285156,
      "activations/layer12_attention_weight_max": 30.198780059814453,
      "activations/layer12_attention_weight_min": -29.874446868896484,
      "activations/layer13_attention_weight_max": 46.28049850463867,
      "activations/layer13_attention_weight_min": -38.932273864746094,
      "activations/layer14_attention_weight_max": 48.39408874511719,
      "activations/layer14_attention_weight_min": -42.24888610839844,
      "activations/layer15_attention_weight_max": 41.512725830078125,
      "activations/layer15_attention_weight_min": -39.143062591552734,
      "activations/layer16_attention_weight_max": 28.73250389099121,
      "activations/layer16_attention_weight_min": -28.568368911743164,
      "activations/layer17_attention_weight_max": 30.50641632080078,
      "activations/layer17_attention_weight_min": -26.222644805908203,
      "activations/layer18_attention_weight_max": 33.967994689941406,
      "activations/layer18_attention_weight_min": -24.890539169311523,
      "activations/layer19_attention_weight_max": 33.56278991699219,
      "activations/layer19_attention_weight_min": -30.66495132446289,
      "activations/layer1_attention_weight_max": 15.63874340057373,
      "activations/layer1_attention_weight_min": -16.013202667236328,
      "activations/layer20_attention_weight_max": 28.339841842651367,
      "activations/layer20_attention_weight_min": -23.56944465637207,
      "activations/layer21_attention_weight_max": 28.3160457611084,
      "activations/layer21_attention_weight_min": -24.649553298950195,
      "activations/layer22_attention_weight_max": 43.94297409057617,
      "activations/layer22_attention_weight_min": -30.42350196838379,
      "activations/layer23_attention_weight_max": 32.201393127441406,
      "activations/layer23_attention_weight_min": -25.164085388183594,
      "activations/layer2_attention_weight_max": 34.95954513549805,
      "activations/layer2_attention_weight_min": -34.127967834472656,
      "activations/layer3_attention_weight_max": 110.89813995361328,
      "activations/layer3_attention_weight_min": -110.07243347167969,
      "activations/layer4_attention_weight_max": 116.4019546508789,
      "activations/layer4_attention_weight_min": -116.7687759399414,
      "activations/layer5_attention_weight_max": 55.942230224609375,
      "activations/layer5_attention_weight_min": -62.02629089355469,
      "activations/layer6_attention_weight_max": 47.29021072387695,
      "activations/layer6_attention_weight_min": -49.58974075317383,
      "activations/layer7_attention_weight_max": 104.86293029785156,
      "activations/layer7_attention_weight_min": -105.10368347167969,
      "activations/layer8_attention_weight_max": 45.42437744140625,
      "activations/layer8_attention_weight_min": -48.327999114990234,
      "activations/layer9_attention_weight_max": 35.87177658081055,
      "activations/layer9_attention_weight_min": -38.31318283081055,
      "epoch": 22.97,
      "learning_rate": 1.8579545454545452e-06,
      "loss": 2.7145,
      "step": 395400
    },
    {
      "activations/layer0_attention_weight_max": 16.191259384155273,
      "activations/layer0_attention_weight_min": -14.813728332519531,
      "activations/layer10_attention_weight_max": 33.91988754272461,
      "activations/layer10_attention_weight_min": -36.66942596435547,
      "activations/layer11_attention_weight_max": 35.073482513427734,
      "activations/layer11_attention_weight_min": -39.237770080566406,
      "activations/layer12_attention_weight_max": 26.67650032043457,
      "activations/layer12_attention_weight_min": -29.58200454711914,
      "activations/layer13_attention_weight_max": 39.44154739379883,
      "activations/layer13_attention_weight_min": -40.1109733581543,
      "activations/layer14_attention_weight_max": 42.94919967651367,
      "activations/layer14_attention_weight_min": -41.38587188720703,
      "activations/layer15_attention_weight_max": 40.43118667602539,
      "activations/layer15_attention_weight_min": -39.49601745605469,
      "activations/layer16_attention_weight_max": 32.78970718383789,
      "activations/layer16_attention_weight_min": -29.44139862060547,
      "activations/layer17_attention_weight_max": 29.65373420715332,
      "activations/layer17_attention_weight_min": -25.721162796020508,
      "activations/layer18_attention_weight_max": 32.95663070678711,
      "activations/layer18_attention_weight_min": -26.106740951538086,
      "activations/layer19_attention_weight_max": 38.12592697143555,
      "activations/layer19_attention_weight_min": -34.13764572143555,
      "activations/layer1_attention_weight_max": 16.24700164794922,
      "activations/layer1_attention_weight_min": -16.558345794677734,
      "activations/layer20_attention_weight_max": 30.930891036987305,
      "activations/layer20_attention_weight_min": -26.086807250976562,
      "activations/layer21_attention_weight_max": 29.250301361083984,
      "activations/layer21_attention_weight_min": -24.58530044555664,
      "activations/layer22_attention_weight_max": 42.488773345947266,
      "activations/layer22_attention_weight_min": -31.74442481994629,
      "activations/layer23_attention_weight_max": 33.33303451538086,
      "activations/layer23_attention_weight_min": -25.651290893554688,
      "activations/layer2_attention_weight_max": 35.21639633178711,
      "activations/layer2_attention_weight_min": -34.092803955078125,
      "activations/layer3_attention_weight_max": 109.56400299072266,
      "activations/layer3_attention_weight_min": -109.47321319580078,
      "activations/layer4_attention_weight_max": 113.72863006591797,
      "activations/layer4_attention_weight_min": -119.07415771484375,
      "activations/layer5_attention_weight_max": 52.407318115234375,
      "activations/layer5_attention_weight_min": -63.084266662597656,
      "activations/layer6_attention_weight_max": 46.59962844848633,
      "activations/layer6_attention_weight_min": -47.74081039428711,
      "activations/layer7_attention_weight_max": 98.21931457519531,
      "activations/layer7_attention_weight_min": -100.96662139892578,
      "activations/layer8_attention_weight_max": 42.80898666381836,
      "activations/layer8_attention_weight_min": -46.1279182434082,
      "activations/layer9_attention_weight_max": 32.10758972167969,
      "activations/layer9_attention_weight_min": -38.03673553466797,
      "epoch": 22.98,
      "learning_rate": 1.8390151515151511e-06,
      "loss": 2.6912,
      "step": 395450
    },
    {
      "activations/layer0_attention_weight_max": 17.300649642944336,
      "activations/layer0_attention_weight_min": -13.80063247680664,
      "activations/layer10_attention_weight_max": 35.9811897277832,
      "activations/layer10_attention_weight_min": -36.67384719848633,
      "activations/layer11_attention_weight_max": 34.93242263793945,
      "activations/layer11_attention_weight_min": -36.925323486328125,
      "activations/layer12_attention_weight_max": 26.610008239746094,
      "activations/layer12_attention_weight_min": -30.19565200805664,
      "activations/layer13_attention_weight_max": 40.30310821533203,
      "activations/layer13_attention_weight_min": -40.62553024291992,
      "activations/layer14_attention_weight_max": 46.77723693847656,
      "activations/layer14_attention_weight_min": -44.582298278808594,
      "activations/layer15_attention_weight_max": 41.701725006103516,
      "activations/layer15_attention_weight_min": -39.1148567199707,
      "activations/layer16_attention_weight_max": 29.650997161865234,
      "activations/layer16_attention_weight_min": -27.713850021362305,
      "activations/layer17_attention_weight_max": 33.00564956665039,
      "activations/layer17_attention_weight_min": -27.439918518066406,
      "activations/layer18_attention_weight_max": 33.1994743347168,
      "activations/layer18_attention_weight_min": -26.96038246154785,
      "activations/layer19_attention_weight_max": 33.66893768310547,
      "activations/layer19_attention_weight_min": -33.34984588623047,
      "activations/layer1_attention_weight_max": 15.458890914916992,
      "activations/layer1_attention_weight_min": -15.679641723632812,
      "activations/layer20_attention_weight_max": 29.08611488342285,
      "activations/layer20_attention_weight_min": -26.8470458984375,
      "activations/layer21_attention_weight_max": 29.078203201293945,
      "activations/layer21_attention_weight_min": -25.832876205444336,
      "activations/layer22_attention_weight_max": 44.19084548950195,
      "activations/layer22_attention_weight_min": -33.60774612426758,
      "activations/layer23_attention_weight_max": 31.047666549682617,
      "activations/layer23_attention_weight_min": -26.49433708190918,
      "activations/layer2_attention_weight_max": 34.78945541381836,
      "activations/layer2_attention_weight_min": -34.009830474853516,
      "activations/layer3_attention_weight_max": 110.19065856933594,
      "activations/layer3_attention_weight_min": -109.81733703613281,
      "activations/layer4_attention_weight_max": 113.969482421875,
      "activations/layer4_attention_weight_min": -113.74024963378906,
      "activations/layer5_attention_weight_max": 52.41559982299805,
      "activations/layer5_attention_weight_min": -63.06188201904297,
      "activations/layer6_attention_weight_max": 43.58167266845703,
      "activations/layer6_attention_weight_min": -47.96446228027344,
      "activations/layer7_attention_weight_max": 96.30237579345703,
      "activations/layer7_attention_weight_min": -98.3648681640625,
      "activations/layer8_attention_weight_max": 41.58590316772461,
      "activations/layer8_attention_weight_min": -46.316673278808594,
      "activations/layer9_attention_weight_max": 32.15327453613281,
      "activations/layer9_attention_weight_min": -35.61161422729492,
      "epoch": 22.98,
      "learning_rate": 1.8200757575757575e-06,
      "loss": 2.7041,
      "step": 395500
    },
    {
      "activations/layer0_attention_weight_max": 17.3347225189209,
      "activations/layer0_attention_weight_min": -14.326077461242676,
      "activations/layer10_attention_weight_max": 36.979209899902344,
      "activations/layer10_attention_weight_min": -36.81003189086914,
      "activations/layer11_attention_weight_max": 35.43156814575195,
      "activations/layer11_attention_weight_min": -36.30050277709961,
      "activations/layer12_attention_weight_max": 27.549726486206055,
      "activations/layer12_attention_weight_min": -27.678316116333008,
      "activations/layer13_attention_weight_max": 40.48146438598633,
      "activations/layer13_attention_weight_min": -37.002044677734375,
      "activations/layer14_attention_weight_max": 40.89020919799805,
      "activations/layer14_attention_weight_min": -39.70378875732422,
      "activations/layer15_attention_weight_max": 38.817359924316406,
      "activations/layer15_attention_weight_min": -39.59526062011719,
      "activations/layer16_attention_weight_max": 29.63610076904297,
      "activations/layer16_attention_weight_min": -32.23563766479492,
      "activations/layer17_attention_weight_max": 31.480066299438477,
      "activations/layer17_attention_weight_min": -26.827234268188477,
      "activations/layer18_attention_weight_max": 36.70832443237305,
      "activations/layer18_attention_weight_min": -27.19354248046875,
      "activations/layer19_attention_weight_max": 39.34061813354492,
      "activations/layer19_attention_weight_min": -36.19073486328125,
      "activations/layer1_attention_weight_max": 15.94438648223877,
      "activations/layer1_attention_weight_min": -15.769515037536621,
      "activations/layer20_attention_weight_max": 31.471355438232422,
      "activations/layer20_attention_weight_min": -24.94951820373535,
      "activations/layer21_attention_weight_max": 31.61412239074707,
      "activations/layer21_attention_weight_min": -25.02834129333496,
      "activations/layer22_attention_weight_max": 44.68559646606445,
      "activations/layer22_attention_weight_min": -34.15937042236328,
      "activations/layer23_attention_weight_max": 36.643741607666016,
      "activations/layer23_attention_weight_min": -26.648208618164062,
      "activations/layer2_attention_weight_max": 36.19799041748047,
      "activations/layer2_attention_weight_min": -34.14360046386719,
      "activations/layer3_attention_weight_max": 110.56291961669922,
      "activations/layer3_attention_weight_min": -113.373046875,
      "activations/layer4_attention_weight_max": 118.25426483154297,
      "activations/layer4_attention_weight_min": -118.92321014404297,
      "activations/layer5_attention_weight_max": 51.35911560058594,
      "activations/layer5_attention_weight_min": -64.05607604980469,
      "activations/layer6_attention_weight_max": 46.95256805419922,
      "activations/layer6_attention_weight_min": -49.76082229614258,
      "activations/layer7_attention_weight_max": 97.36376190185547,
      "activations/layer7_attention_weight_min": -102.82015991210938,
      "activations/layer8_attention_weight_max": 43.882102966308594,
      "activations/layer8_attention_weight_min": -46.54729080200195,
      "activations/layer9_attention_weight_max": 33.700801849365234,
      "activations/layer9_attention_weight_min": -36.157718658447266,
      "epoch": 22.98,
      "learning_rate": 1.8011363636363634e-06,
      "loss": 2.6994,
      "step": 395550
    },
    {
      "activations/layer0_attention_weight_max": 17.284093856811523,
      "activations/layer0_attention_weight_min": -16.760061264038086,
      "activations/layer10_attention_weight_max": 37.48021697998047,
      "activations/layer10_attention_weight_min": -38.971046447753906,
      "activations/layer11_attention_weight_max": 39.29536437988281,
      "activations/layer11_attention_weight_min": -37.131839752197266,
      "activations/layer12_attention_weight_max": 29.258394241333008,
      "activations/layer12_attention_weight_min": -29.939411163330078,
      "activations/layer13_attention_weight_max": 42.31404495239258,
      "activations/layer13_attention_weight_min": -39.84186935424805,
      "activations/layer14_attention_weight_max": 44.16557312011719,
      "activations/layer14_attention_weight_min": -40.642616271972656,
      "activations/layer15_attention_weight_max": 40.28830337524414,
      "activations/layer15_attention_weight_min": -39.438228607177734,
      "activations/layer16_attention_weight_max": 29.20374298095703,
      "activations/layer16_attention_weight_min": -29.754261016845703,
      "activations/layer17_attention_weight_max": 29.497053146362305,
      "activations/layer17_attention_weight_min": -28.626123428344727,
      "activations/layer18_attention_weight_max": 33.25934982299805,
      "activations/layer18_attention_weight_min": -27.2133731842041,
      "activations/layer19_attention_weight_max": 37.81170654296875,
      "activations/layer19_attention_weight_min": -34.89606475830078,
      "activations/layer1_attention_weight_max": 15.667621612548828,
      "activations/layer1_attention_weight_min": -14.905948638916016,
      "activations/layer20_attention_weight_max": 31.579551696777344,
      "activations/layer20_attention_weight_min": -24.469127655029297,
      "activations/layer21_attention_weight_max": 31.008235931396484,
      "activations/layer21_attention_weight_min": -24.9163761138916,
      "activations/layer22_attention_weight_max": 43.4922981262207,
      "activations/layer22_attention_weight_min": -31.74692726135254,
      "activations/layer23_attention_weight_max": 36.86993408203125,
      "activations/layer23_attention_weight_min": -26.361852645874023,
      "activations/layer2_attention_weight_max": 34.106361389160156,
      "activations/layer2_attention_weight_min": -34.044734954833984,
      "activations/layer3_attention_weight_max": 112.4209976196289,
      "activations/layer3_attention_weight_min": -112.92324829101562,
      "activations/layer4_attention_weight_max": 117.31682586669922,
      "activations/layer4_attention_weight_min": -119.7857666015625,
      "activations/layer5_attention_weight_max": 54.61216735839844,
      "activations/layer5_attention_weight_min": -62.87664794921875,
      "activations/layer6_attention_weight_max": 50.58421325683594,
      "activations/layer6_attention_weight_min": -53.56071472167969,
      "activations/layer7_attention_weight_max": 108.80146026611328,
      "activations/layer7_attention_weight_min": -113.00916290283203,
      "activations/layer8_attention_weight_max": 52.172725677490234,
      "activations/layer8_attention_weight_min": -53.931888580322266,
      "activations/layer9_attention_weight_max": 34.97270584106445,
      "activations/layer9_attention_weight_min": -37.609100341796875,
      "epoch": 22.99,
      "learning_rate": 1.7821969696969696e-06,
      "loss": 2.676,
      "step": 395600
    },
    {
      "activations/layer0_attention_weight_max": 17.23213768005371,
      "activations/layer0_attention_weight_min": -16.051759719848633,
      "activations/layer10_attention_weight_max": 39.00276565551758,
      "activations/layer10_attention_weight_min": -39.726985931396484,
      "activations/layer11_attention_weight_max": 38.80677795410156,
      "activations/layer11_attention_weight_min": -40.307403564453125,
      "activations/layer12_attention_weight_max": 28.7874698638916,
      "activations/layer12_attention_weight_min": -28.824378967285156,
      "activations/layer13_attention_weight_max": 42.789859771728516,
      "activations/layer13_attention_weight_min": -38.30427551269531,
      "activations/layer14_attention_weight_max": 43.58157730102539,
      "activations/layer14_attention_weight_min": -40.246219635009766,
      "activations/layer15_attention_weight_max": 44.06786346435547,
      "activations/layer15_attention_weight_min": -37.7283821105957,
      "activations/layer16_attention_weight_max": 31.97850799560547,
      "activations/layer16_attention_weight_min": -29.601892471313477,
      "activations/layer17_attention_weight_max": 30.89936065673828,
      "activations/layer17_attention_weight_min": -28.31486701965332,
      "activations/layer18_attention_weight_max": 33.62295913696289,
      "activations/layer18_attention_weight_min": -26.93889045715332,
      "activations/layer19_attention_weight_max": 34.76314926147461,
      "activations/layer19_attention_weight_min": -32.681922912597656,
      "activations/layer1_attention_weight_max": 16.0618896484375,
      "activations/layer1_attention_weight_min": -16.347644805908203,
      "activations/layer20_attention_weight_max": 29.197288513183594,
      "activations/layer20_attention_weight_min": -25.51997947692871,
      "activations/layer21_attention_weight_max": 29.235748291015625,
      "activations/layer21_attention_weight_min": -25.113834381103516,
      "activations/layer22_attention_weight_max": 43.89204788208008,
      "activations/layer22_attention_weight_min": -32.12555694580078,
      "activations/layer23_attention_weight_max": 33.05263137817383,
      "activations/layer23_attention_weight_min": -27.956619262695312,
      "activations/layer2_attention_weight_max": 35.14265441894531,
      "activations/layer2_attention_weight_min": -34.916324615478516,
      "activations/layer3_attention_weight_max": 112.5426254272461,
      "activations/layer3_attention_weight_min": -112.42974090576172,
      "activations/layer4_attention_weight_max": 115.15445709228516,
      "activations/layer4_attention_weight_min": -116.96820068359375,
      "activations/layer5_attention_weight_max": 53.34432601928711,
      "activations/layer5_attention_weight_min": -64.39907836914062,
      "activations/layer6_attention_weight_max": 48.53044891357422,
      "activations/layer6_attention_weight_min": -51.09682083129883,
      "activations/layer7_attention_weight_max": 106.27588653564453,
      "activations/layer7_attention_weight_min": -110.7697525024414,
      "activations/layer8_attention_weight_max": 47.10325241088867,
      "activations/layer8_attention_weight_min": -50.154109954833984,
      "activations/layer9_attention_weight_max": 37.428409576416016,
      "activations/layer9_attention_weight_min": -38.23600387573242,
      "epoch": 22.99,
      "learning_rate": 1.7632575757575755e-06,
      "loss": 2.7055,
      "step": 395650
    },
    {
      "activations/layer0_attention_weight_max": 17.041826248168945,
      "activations/layer0_attention_weight_min": -15.38919448852539,
      "activations/layer10_attention_weight_max": 38.28997039794922,
      "activations/layer10_attention_weight_min": -40.451114654541016,
      "activations/layer11_attention_weight_max": 36.04179382324219,
      "activations/layer11_attention_weight_min": -38.121253967285156,
      "activations/layer12_attention_weight_max": 28.587854385375977,
      "activations/layer12_attention_weight_min": -28.20408821105957,
      "activations/layer13_attention_weight_max": 41.73518371582031,
      "activations/layer13_attention_weight_min": -36.78205871582031,
      "activations/layer14_attention_weight_max": 45.166351318359375,
      "activations/layer14_attention_weight_min": -38.404109954833984,
      "activations/layer15_attention_weight_max": 37.72607421875,
      "activations/layer15_attention_weight_min": -36.329444885253906,
      "activations/layer16_attention_weight_max": 30.342350006103516,
      "activations/layer16_attention_weight_min": -28.267885208129883,
      "activations/layer17_attention_weight_max": 29.787439346313477,
      "activations/layer17_attention_weight_min": -26.523576736450195,
      "activations/layer18_attention_weight_max": 31.725332260131836,
      "activations/layer18_attention_weight_min": -26.01555824279785,
      "activations/layer19_attention_weight_max": 34.21269607543945,
      "activations/layer19_attention_weight_min": -33.07111740112305,
      "activations/layer1_attention_weight_max": 15.877575874328613,
      "activations/layer1_attention_weight_min": -15.174358367919922,
      "activations/layer20_attention_weight_max": 30.96657371520996,
      "activations/layer20_attention_weight_min": -27.17983627319336,
      "activations/layer21_attention_weight_max": 31.174503326416016,
      "activations/layer21_attention_weight_min": -27.79999351501465,
      "activations/layer22_attention_weight_max": 46.24873733520508,
      "activations/layer22_attention_weight_min": -33.16960144042969,
      "activations/layer23_attention_weight_max": 33.95338439941406,
      "activations/layer23_attention_weight_min": -26.036312103271484,
      "activations/layer2_attention_weight_max": 34.12120819091797,
      "activations/layer2_attention_weight_min": -34.18437194824219,
      "activations/layer3_attention_weight_max": 110.25370788574219,
      "activations/layer3_attention_weight_min": -113.59852600097656,
      "activations/layer4_attention_weight_max": 112.01454162597656,
      "activations/layer4_attention_weight_min": -117.53792572021484,
      "activations/layer5_attention_weight_max": 54.878841400146484,
      "activations/layer5_attention_weight_min": -67.19989013671875,
      "activations/layer6_attention_weight_max": 49.63668441772461,
      "activations/layer6_attention_weight_min": -51.86354064941406,
      "activations/layer7_attention_weight_max": 98.71094512939453,
      "activations/layer7_attention_weight_min": -107.03715515136719,
      "activations/layer8_attention_weight_max": 46.23237228393555,
      "activations/layer8_attention_weight_min": -52.90205764770508,
      "activations/layer9_attention_weight_max": 33.25471878051758,
      "activations/layer9_attention_weight_min": -40.132835388183594,
      "epoch": 22.99,
      "learning_rate": 1.7443181818181817e-06,
      "loss": 2.6912,
      "step": 395700
    },
    {
      "activations/layer0_attention_weight_max": 16.707599639892578,
      "activations/layer0_attention_weight_min": -13.682098388671875,
      "activations/layer10_attention_weight_max": 45.046600341796875,
      "activations/layer10_attention_weight_min": -40.049041748046875,
      "activations/layer11_attention_weight_max": 46.434654235839844,
      "activations/layer11_attention_weight_min": -44.53414535522461,
      "activations/layer12_attention_weight_max": 30.379945755004883,
      "activations/layer12_attention_weight_min": -28.184858322143555,
      "activations/layer13_attention_weight_max": 42.54546356201172,
      "activations/layer13_attention_weight_min": -36.837615966796875,
      "activations/layer14_attention_weight_max": 44.05339813232422,
      "activations/layer14_attention_weight_min": -40.55794143676758,
      "activations/layer15_attention_weight_max": 43.73214340209961,
      "activations/layer15_attention_weight_min": -40.796165466308594,
      "activations/layer16_attention_weight_max": 30.997425079345703,
      "activations/layer16_attention_weight_min": -29.041685104370117,
      "activations/layer17_attention_weight_max": 31.85719871520996,
      "activations/layer17_attention_weight_min": -26.886062622070312,
      "activations/layer18_attention_weight_max": 33.13551330566406,
      "activations/layer18_attention_weight_min": -27.52955436706543,
      "activations/layer19_attention_weight_max": 33.4634895324707,
      "activations/layer19_attention_weight_min": -33.25718307495117,
      "activations/layer1_attention_weight_max": 16.477319717407227,
      "activations/layer1_attention_weight_min": -15.592931747436523,
      "activations/layer20_attention_weight_max": 28.953157424926758,
      "activations/layer20_attention_weight_min": -27.63954734802246,
      "activations/layer21_attention_weight_max": 27.98113250732422,
      "activations/layer21_attention_weight_min": -27.00326156616211,
      "activations/layer22_attention_weight_max": 39.13507080078125,
      "activations/layer22_attention_weight_min": -34.55404281616211,
      "activations/layer23_attention_weight_max": 30.02822494506836,
      "activations/layer23_attention_weight_min": -27.954008102416992,
      "activations/layer2_attention_weight_max": 34.647281646728516,
      "activations/layer2_attention_weight_min": -33.36682891845703,
      "activations/layer3_attention_weight_max": 106.80534362792969,
      "activations/layer3_attention_weight_min": -106.66161346435547,
      "activations/layer4_attention_weight_max": 108.16603088378906,
      "activations/layer4_attention_weight_min": -112.316650390625,
      "activations/layer5_attention_weight_max": 55.205665588378906,
      "activations/layer5_attention_weight_min": -60.920806884765625,
      "activations/layer6_attention_weight_max": 45.81752395629883,
      "activations/layer6_attention_weight_min": -48.572715759277344,
      "activations/layer7_attention_weight_max": 117.70519256591797,
      "activations/layer7_attention_weight_min": -99.48932647705078,
      "activations/layer8_attention_weight_max": 53.350914001464844,
      "activations/layer8_attention_weight_min": -48.660343170166016,
      "activations/layer9_attention_weight_max": 36.808937072753906,
      "activations/layer9_attention_weight_min": -38.85683822631836,
      "epoch": 23.0,
      "learning_rate": 1.7253787878787876e-06,
      "loss": 2.7092,
      "step": 395750
    },
    {
      "activations/layer0_attention_weight_max": 17.19833755493164,
      "activations/layer0_attention_weight_min": -14.481042861938477,
      "activations/layer10_attention_weight_max": 37.361122131347656,
      "activations/layer10_attention_weight_min": -39.39622116088867,
      "activations/layer11_attention_weight_max": 37.137596130371094,
      "activations/layer11_attention_weight_min": -37.877777099609375,
      "activations/layer12_attention_weight_max": 30.469295501708984,
      "activations/layer12_attention_weight_min": -29.202449798583984,
      "activations/layer13_attention_weight_max": 42.78538131713867,
      "activations/layer13_attention_weight_min": -37.72700119018555,
      "activations/layer14_attention_weight_max": 43.87567138671875,
      "activations/layer14_attention_weight_min": -40.543975830078125,
      "activations/layer15_attention_weight_max": 41.0306282043457,
      "activations/layer15_attention_weight_min": -38.299991607666016,
      "activations/layer16_attention_weight_max": 31.39082908630371,
      "activations/layer16_attention_weight_min": -29.655319213867188,
      "activations/layer17_attention_weight_max": 31.52922248840332,
      "activations/layer17_attention_weight_min": -27.284364700317383,
      "activations/layer18_attention_weight_max": 34.3350944519043,
      "activations/layer18_attention_weight_min": -25.308561325073242,
      "activations/layer19_attention_weight_max": 37.05522537231445,
      "activations/layer19_attention_weight_min": -31.725006103515625,
      "activations/layer1_attention_weight_max": 16.213375091552734,
      "activations/layer1_attention_weight_min": -15.682574272155762,
      "activations/layer20_attention_weight_max": 31.408218383789062,
      "activations/layer20_attention_weight_min": -25.350542068481445,
      "activations/layer21_attention_weight_max": 33.19151306152344,
      "activations/layer21_attention_weight_min": -24.509445190429688,
      "activations/layer22_attention_weight_max": 46.74320602416992,
      "activations/layer22_attention_weight_min": -31.292234420776367,
      "activations/layer23_attention_weight_max": 34.38445281982422,
      "activations/layer23_attention_weight_min": -26.607030868530273,
      "activations/layer2_attention_weight_max": 34.438819885253906,
      "activations/layer2_attention_weight_min": -35.436546325683594,
      "activations/layer3_attention_weight_max": 110.96684265136719,
      "activations/layer3_attention_weight_min": -115.84945678710938,
      "activations/layer4_attention_weight_max": 113.38497161865234,
      "activations/layer4_attention_weight_min": -118.333740234375,
      "activations/layer5_attention_weight_max": 52.17417526245117,
      "activations/layer5_attention_weight_min": -62.95215606689453,
      "activations/layer6_attention_weight_max": 49.68408966064453,
      "activations/layer6_attention_weight_min": -50.33654022216797,
      "activations/layer7_attention_weight_max": 100.65078735351562,
      "activations/layer7_attention_weight_min": -110.42076873779297,
      "activations/layer8_attention_weight_max": 43.845542907714844,
      "activations/layer8_attention_weight_min": -49.6185302734375,
      "activations/layer9_attention_weight_max": 35.03705978393555,
      "activations/layer9_attention_weight_min": -36.19808578491211,
      "epoch": 23.0,
      "learning_rate": 1.7064393939393938e-06,
      "loss": 2.7011,
      "step": 395800
    },
    {
      "activations/layer0_attention_weight_max": 17.35223960876465,
      "activations/layer0_attention_weight_min": -15.209362030029297,
      "activations/layer10_attention_weight_max": 36.86835861206055,
      "activations/layer10_attention_weight_min": -36.18986129760742,
      "activations/layer11_attention_weight_max": 36.160892486572266,
      "activations/layer11_attention_weight_min": -34.928466796875,
      "activations/layer12_attention_weight_max": 29.218477249145508,
      "activations/layer12_attention_weight_min": -28.02487564086914,
      "activations/layer13_attention_weight_max": 41.2748908996582,
      "activations/layer13_attention_weight_min": -39.26177215576172,
      "activations/layer14_attention_weight_max": 44.288185119628906,
      "activations/layer14_attention_weight_min": -41.77202224731445,
      "activations/layer15_attention_weight_max": 41.34880447387695,
      "activations/layer15_attention_weight_min": -38.27591323852539,
      "activations/layer16_attention_weight_max": 31.371931076049805,
      "activations/layer16_attention_weight_min": -29.56048011779785,
      "activations/layer17_attention_weight_max": 29.815454483032227,
      "activations/layer17_attention_weight_min": -27.168352127075195,
      "activations/layer18_attention_weight_max": 33.00691223144531,
      "activations/layer18_attention_weight_min": -26.520198822021484,
      "activations/layer19_attention_weight_max": 34.54743957519531,
      "activations/layer19_attention_weight_min": -33.3573112487793,
      "activations/layer1_attention_weight_max": 15.618937492370605,
      "activations/layer1_attention_weight_min": -16.092071533203125,
      "activations/layer20_attention_weight_max": 28.41359519958496,
      "activations/layer20_attention_weight_min": -25.53703498840332,
      "activations/layer21_attention_weight_max": 28.766775131225586,
      "activations/layer21_attention_weight_min": -25.611942291259766,
      "activations/layer22_attention_weight_max": 40.81660079956055,
      "activations/layer22_attention_weight_min": -32.23118209838867,
      "activations/layer23_attention_weight_max": 30.777576446533203,
      "activations/layer23_attention_weight_min": -27.308788299560547,
      "activations/layer2_attention_weight_max": 35.647918701171875,
      "activations/layer2_attention_weight_min": -33.579345703125,
      "activations/layer3_attention_weight_max": 108.79159545898438,
      "activations/layer3_attention_weight_min": -106.2632827758789,
      "activations/layer4_attention_weight_max": 110.76033782958984,
      "activations/layer4_attention_weight_min": -113.2674789428711,
      "activations/layer5_attention_weight_max": 52.80105209350586,
      "activations/layer5_attention_weight_min": -63.000152587890625,
      "activations/layer6_attention_weight_max": 48.20932388305664,
      "activations/layer6_attention_weight_min": -49.4195671081543,
      "activations/layer7_attention_weight_max": 99.5331039428711,
      "activations/layer7_attention_weight_min": -109.24205017089844,
      "activations/layer8_attention_weight_max": 45.09104919433594,
      "activations/layer8_attention_weight_min": -47.605106353759766,
      "activations/layer9_attention_weight_max": 33.7508659362793,
      "activations/layer9_attention_weight_min": -36.262290954589844,
      "epoch": 23.0,
      "learning_rate": 1.6874999999999997e-06,
      "loss": 2.7378,
      "step": 395850
    },
    {
      "activations/layer0_attention_weight_max": 16.596792221069336,
      "activations/layer0_attention_weight_min": -14.78742790222168,
      "activations/layer10_attention_weight_max": 34.358154296875,
      "activations/layer10_attention_weight_min": -36.5865478515625,
      "activations/layer11_attention_weight_max": 33.522865295410156,
      "activations/layer11_attention_weight_min": -35.122955322265625,
      "activations/layer12_attention_weight_max": 25.865802764892578,
      "activations/layer12_attention_weight_min": -27.65569496154785,
      "activations/layer13_attention_weight_max": 39.94160842895508,
      "activations/layer13_attention_weight_min": -36.72976303100586,
      "activations/layer14_attention_weight_max": 40.52613830566406,
      "activations/layer14_attention_weight_min": -38.702388763427734,
      "activations/layer15_attention_weight_max": 41.02046203613281,
      "activations/layer15_attention_weight_min": -36.564239501953125,
      "activations/layer16_attention_weight_max": 30.679838180541992,
      "activations/layer16_attention_weight_min": -28.592327117919922,
      "activations/layer17_attention_weight_max": 31.729827880859375,
      "activations/layer17_attention_weight_min": -27.78655433654785,
      "activations/layer18_attention_weight_max": 33.80717468261719,
      "activations/layer18_attention_weight_min": -26.13072967529297,
      "activations/layer19_attention_weight_max": 37.29518508911133,
      "activations/layer19_attention_weight_min": -31.15595054626465,
      "activations/layer1_attention_weight_max": 15.754165649414062,
      "activations/layer1_attention_weight_min": -14.99557876586914,
      "activations/layer20_attention_weight_max": 30.805980682373047,
      "activations/layer20_attention_weight_min": -24.800642013549805,
      "activations/layer21_attention_weight_max": 29.328617095947266,
      "activations/layer21_attention_weight_min": -24.468626022338867,
      "activations/layer22_attention_weight_max": 39.8207893371582,
      "activations/layer22_attention_weight_min": -32.885948181152344,
      "activations/layer23_attention_weight_max": 30.94626235961914,
      "activations/layer23_attention_weight_min": -27.56612205505371,
      "activations/layer2_attention_weight_max": 34.67695617675781,
      "activations/layer2_attention_weight_min": -33.396339416503906,
      "activations/layer3_attention_weight_max": 109.74512481689453,
      "activations/layer3_attention_weight_min": -106.66844940185547,
      "activations/layer4_attention_weight_max": 112.17638397216797,
      "activations/layer4_attention_weight_min": -116.44637298583984,
      "activations/layer5_attention_weight_max": 51.8443603515625,
      "activations/layer5_attention_weight_min": -64.0528564453125,
      "activations/layer6_attention_weight_max": 47.60883712768555,
      "activations/layer6_attention_weight_min": -48.83802032470703,
      "activations/layer7_attention_weight_max": 99.78872680664062,
      "activations/layer7_attention_weight_min": -103.42521667480469,
      "activations/layer8_attention_weight_max": 44.023963928222656,
      "activations/layer8_attention_weight_min": -47.212223052978516,
      "activations/layer9_attention_weight_max": 32.59650421142578,
      "activations/layer9_attention_weight_min": -35.95139694213867,
      "epoch": 23.0,
      "learning_rate": 1.668560606060606e-06,
      "loss": 2.6898,
      "step": 395900
    },
    {
      "activations/layer0_attention_weight_max": 16.36909294128418,
      "activations/layer0_attention_weight_min": -15.864256858825684,
      "activations/layer10_attention_weight_max": 37.36040115356445,
      "activations/layer10_attention_weight_min": -37.13844680786133,
      "activations/layer11_attention_weight_max": 36.34738540649414,
      "activations/layer11_attention_weight_min": -36.565773010253906,
      "activations/layer12_attention_weight_max": 28.686250686645508,
      "activations/layer12_attention_weight_min": -28.71855354309082,
      "activations/layer13_attention_weight_max": 44.766780853271484,
      "activations/layer13_attention_weight_min": -39.14521408081055,
      "activations/layer14_attention_weight_max": 47.747100830078125,
      "activations/layer14_attention_weight_min": -42.21947479248047,
      "activations/layer15_attention_weight_max": 42.14250564575195,
      "activations/layer15_attention_weight_min": -39.39289093017578,
      "activations/layer16_attention_weight_max": 29.972251892089844,
      "activations/layer16_attention_weight_min": -35.51332473754883,
      "activations/layer17_attention_weight_max": 30.07402229309082,
      "activations/layer17_attention_weight_min": -28.517465591430664,
      "activations/layer18_attention_weight_max": 32.92201614379883,
      "activations/layer18_attention_weight_min": -27.668550491333008,
      "activations/layer19_attention_weight_max": 34.53104782104492,
      "activations/layer19_attention_weight_min": -34.87533187866211,
      "activations/layer1_attention_weight_max": 16.06259536743164,
      "activations/layer1_attention_weight_min": -15.387960433959961,
      "activations/layer20_attention_weight_max": 28.7191219329834,
      "activations/layer20_attention_weight_min": -27.791786193847656,
      "activations/layer21_attention_weight_max": 30.061477661132812,
      "activations/layer21_attention_weight_min": -25.98324966430664,
      "activations/layer22_attention_weight_max": 43.644264221191406,
      "activations/layer22_attention_weight_min": -35.38088607788086,
      "activations/layer23_attention_weight_max": 34.097686767578125,
      "activations/layer23_attention_weight_min": -27.09212303161621,
      "activations/layer2_attention_weight_max": 33.295108795166016,
      "activations/layer2_attention_weight_min": -33.982975006103516,
      "activations/layer3_attention_weight_max": 107.4963150024414,
      "activations/layer3_attention_weight_min": -107.52218627929688,
      "activations/layer4_attention_weight_max": 112.87963104248047,
      "activations/layer4_attention_weight_min": -116.36385345458984,
      "activations/layer5_attention_weight_max": 50.53880310058594,
      "activations/layer5_attention_weight_min": -66.95428466796875,
      "activations/layer6_attention_weight_max": 46.75193786621094,
      "activations/layer6_attention_weight_min": -51.056819915771484,
      "activations/layer7_attention_weight_max": 105.2992935180664,
      "activations/layer7_attention_weight_min": -105.70714569091797,
      "activations/layer8_attention_weight_max": 45.929630279541016,
      "activations/layer8_attention_weight_min": -47.586788177490234,
      "activations/layer9_attention_weight_max": 36.1713981628418,
      "activations/layer9_attention_weight_min": -37.41547775268555,
      "epoch": 23.01,
      "learning_rate": 1.649621212121212e-06,
      "loss": 2.6884,
      "step": 395950
    },
    {
      "activations/layer0_attention_weight_max": 17.385019302368164,
      "activations/layer0_attention_weight_min": -15.049348831176758,
      "activations/layer10_attention_weight_max": 37.3291015625,
      "activations/layer10_attention_weight_min": -36.12226104736328,
      "activations/layer11_attention_weight_max": 36.11363983154297,
      "activations/layer11_attention_weight_min": -35.457759857177734,
      "activations/layer12_attention_weight_max": 32.53275680541992,
      "activations/layer12_attention_weight_min": -28.685068130493164,
      "activations/layer13_attention_weight_max": 46.38285446166992,
      "activations/layer13_attention_weight_min": -40.07060241699219,
      "activations/layer14_attention_weight_max": 47.436676025390625,
      "activations/layer14_attention_weight_min": -40.50361251831055,
      "activations/layer15_attention_weight_max": 45.04823684692383,
      "activations/layer15_attention_weight_min": -37.37575149536133,
      "activations/layer16_attention_weight_max": 31.06174659729004,
      "activations/layer16_attention_weight_min": -29.131046295166016,
      "activations/layer17_attention_weight_max": 30.417707443237305,
      "activations/layer17_attention_weight_min": -27.1760196685791,
      "activations/layer18_attention_weight_max": 34.10130310058594,
      "activations/layer18_attention_weight_min": -25.97772789001465,
      "activations/layer19_attention_weight_max": 34.25051498413086,
      "activations/layer19_attention_weight_min": -31.92119789123535,
      "activations/layer1_attention_weight_max": 16.4086971282959,
      "activations/layer1_attention_weight_min": -14.96263599395752,
      "activations/layer20_attention_weight_max": 31.693269729614258,
      "activations/layer20_attention_weight_min": -26.39726448059082,
      "activations/layer21_attention_weight_max": 29.589872360229492,
      "activations/layer21_attention_weight_min": -24.574125289916992,
      "activations/layer22_attention_weight_max": 42.72882843017578,
      "activations/layer22_attention_weight_min": -31.857635498046875,
      "activations/layer23_attention_weight_max": 33.95782470703125,
      "activations/layer23_attention_weight_min": -26.19445037841797,
      "activations/layer2_attention_weight_max": 34.46208572387695,
      "activations/layer2_attention_weight_min": -33.67512512207031,
      "activations/layer3_attention_weight_max": 107.30036926269531,
      "activations/layer3_attention_weight_min": -109.97454071044922,
      "activations/layer4_attention_weight_max": 111.1039047241211,
      "activations/layer4_attention_weight_min": -116.74098205566406,
      "activations/layer5_attention_weight_max": 50.74730682373047,
      "activations/layer5_attention_weight_min": -60.780208587646484,
      "activations/layer6_attention_weight_max": 46.89636993408203,
      "activations/layer6_attention_weight_min": -49.250343322753906,
      "activations/layer7_attention_weight_max": 99.69502258300781,
      "activations/layer7_attention_weight_min": -103.00242614746094,
      "activations/layer8_attention_weight_max": 44.35597610473633,
      "activations/layer8_attention_weight_min": -48.42023849487305,
      "activations/layer9_attention_weight_max": 33.49001693725586,
      "activations/layer9_attention_weight_min": -35.882625579833984,
      "epoch": 23.01,
      "learning_rate": 1.6306818181818182e-06,
      "loss": 2.6926,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_loss": 2.66015625,
      "eval_runtime": 8.4998,
      "eval_samples_per_second": 505.186,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_openwebtext_loss": 2.66015625,
      "eval_openwebtext_ppl": 14.298523068373607,
      "eval_openwebtext_runtime": 8.4998,
      "eval_openwebtext_samples_per_second": 505.186,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9779,
      "eval_wikitext_samples_per_second": 230.546,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_lambada_loss": 2.296875,
      "eval_lambada_ppl": 9.943061786014486,
      "eval_lambada_runtime": 9.566,
      "eval_lambada_samples_per_second": 508.989,
      "step": 396000
    },
    {
      "activations/layer0_attention_weight_max": 16.83336639404297,
      "activations/layer0_attention_weight_min": -13.483205795288086,
      "activations/layer10_attention_weight_max": 39.425453186035156,
      "activations/layer10_attention_weight_min": -38.22425079345703,
      "activations/layer11_attention_weight_max": 36.842464447021484,
      "activations/layer11_attention_weight_min": -37.75447082519531,
      "activations/layer12_attention_weight_max": 27.996143341064453,
      "activations/layer12_attention_weight_min": -26.882251739501953,
      "activations/layer13_attention_weight_max": 41.318824768066406,
      "activations/layer13_attention_weight_min": -38.0075798034668,
      "activations/layer14_attention_weight_max": 44.7841911315918,
      "activations/layer14_attention_weight_min": -40.00844192504883,
      "activations/layer15_attention_weight_max": 47.59244155883789,
      "activations/layer15_attention_weight_min": -41.176300048828125,
      "activations/layer16_attention_weight_max": 38.732845306396484,
      "activations/layer16_attention_weight_min": -31.16609764099121,
      "activations/layer17_attention_weight_max": 30.01523780822754,
      "activations/layer17_attention_weight_min": -29.7656192779541,
      "activations/layer18_attention_weight_max": 31.502962112426758,
      "activations/layer18_attention_weight_min": -26.972145080566406,
      "activations/layer19_attention_weight_max": 32.98759078979492,
      "activations/layer19_attention_weight_min": -29.693159103393555,
      "activations/layer1_attention_weight_max": 16.00840950012207,
      "activations/layer1_attention_weight_min": -15.262310981750488,
      "activations/layer20_attention_weight_max": 27.790258407592773,
      "activations/layer20_attention_weight_min": -24.883012771606445,
      "activations/layer21_attention_weight_max": 26.613567352294922,
      "activations/layer21_attention_weight_min": -23.339441299438477,
      "activations/layer22_attention_weight_max": 39.3713493347168,
      "activations/layer22_attention_weight_min": -30.93086814880371,
      "activations/layer23_attention_weight_max": 28.99911880493164,
      "activations/layer23_attention_weight_min": -26.02718734741211,
      "activations/layer2_attention_weight_max": 35.69172668457031,
      "activations/layer2_attention_weight_min": -34.949459075927734,
      "activations/layer3_attention_weight_max": 110.27439880371094,
      "activations/layer3_attention_weight_min": -112.20194244384766,
      "activations/layer4_attention_weight_max": 113.956298828125,
      "activations/layer4_attention_weight_min": -116.96337127685547,
      "activations/layer5_attention_weight_max": 51.42871856689453,
      "activations/layer5_attention_weight_min": -62.62519454956055,
      "activations/layer6_attention_weight_max": 46.825889587402344,
      "activations/layer6_attention_weight_min": -49.99900817871094,
      "activations/layer7_attention_weight_max": 102.7678451538086,
      "activations/layer7_attention_weight_min": -105.20130157470703,
      "activations/layer8_attention_weight_max": 44.101600646972656,
      "activations/layer8_attention_weight_min": -48.87348175048828,
      "activations/layer9_attention_weight_max": 35.042999267578125,
      "activations/layer9_attention_weight_min": -35.8894157409668,
      "epoch": 23.01,
      "learning_rate": 1.612121212121212e-06,
      "loss": 2.6922,
      "step": 396050
    },
    {
      "activations/layer0_attention_weight_max": 16.789260864257812,
      "activations/layer0_attention_weight_min": -13.884722709655762,
      "activations/layer10_attention_weight_max": 37.61763381958008,
      "activations/layer10_attention_weight_min": -39.69893264770508,
      "activations/layer11_attention_weight_max": 36.87993240356445,
      "activations/layer11_attention_weight_min": -40.93648910522461,
      "activations/layer12_attention_weight_max": 30.259920120239258,
      "activations/layer12_attention_weight_min": -31.53179359436035,
      "activations/layer13_attention_weight_max": 47.26726531982422,
      "activations/layer13_attention_weight_min": -42.00868606567383,
      "activations/layer14_attention_weight_max": 50.553497314453125,
      "activations/layer14_attention_weight_min": -48.27783203125,
      "activations/layer15_attention_weight_max": 49.66461944580078,
      "activations/layer15_attention_weight_min": -43.39193344116211,
      "activations/layer16_attention_weight_max": 32.735260009765625,
      "activations/layer16_attention_weight_min": -28.924198150634766,
      "activations/layer17_attention_weight_max": 31.09117317199707,
      "activations/layer17_attention_weight_min": -27.354509353637695,
      "activations/layer18_attention_weight_max": 36.606781005859375,
      "activations/layer18_attention_weight_min": -27.267534255981445,
      "activations/layer19_attention_weight_max": 33.38539123535156,
      "activations/layer19_attention_weight_min": -32.922122955322266,
      "activations/layer1_attention_weight_max": 16.606204986572266,
      "activations/layer1_attention_weight_min": -15.708721160888672,
      "activations/layer20_attention_weight_max": 28.87035369873047,
      "activations/layer20_attention_weight_min": -26.844032287597656,
      "activations/layer21_attention_weight_max": 27.414714813232422,
      "activations/layer21_attention_weight_min": -24.469467163085938,
      "activations/layer22_attention_weight_max": 43.302730560302734,
      "activations/layer22_attention_weight_min": -34.503807067871094,
      "activations/layer23_attention_weight_max": 29.697240829467773,
      "activations/layer23_attention_weight_min": -25.50063705444336,
      "activations/layer2_attention_weight_max": 35.70734786987305,
      "activations/layer2_attention_weight_min": -34.41108322143555,
      "activations/layer3_attention_weight_max": 108.85993957519531,
      "activations/layer3_attention_weight_min": -109.2666244506836,
      "activations/layer4_attention_weight_max": 113.85951232910156,
      "activations/layer4_attention_weight_min": -117.8597183227539,
      "activations/layer5_attention_weight_max": 50.368873596191406,
      "activations/layer5_attention_weight_min": -65.4922103881836,
      "activations/layer6_attention_weight_max": 48.20048141479492,
      "activations/layer6_attention_weight_min": -50.79530334472656,
      "activations/layer7_attention_weight_max": 101.90826416015625,
      "activations/layer7_attention_weight_min": -111.96341705322266,
      "activations/layer8_attention_weight_max": 44.42979049682617,
      "activations/layer8_attention_weight_min": -50.62177276611328,
      "activations/layer9_attention_weight_max": 35.41012954711914,
      "activations/layer9_attention_weight_min": -39.94283676147461,
      "epoch": 23.02,
      "learning_rate": 1.5931818181818181e-06,
      "loss": 2.6939,
      "step": 396100
    },
    {
      "activations/layer0_attention_weight_max": 16.91303825378418,
      "activations/layer0_attention_weight_min": -14.56002426147461,
      "activations/layer10_attention_weight_max": 36.512611389160156,
      "activations/layer10_attention_weight_min": -36.881690979003906,
      "activations/layer11_attention_weight_max": 38.75598907470703,
      "activations/layer11_attention_weight_min": -38.037109375,
      "activations/layer12_attention_weight_max": 28.902475357055664,
      "activations/layer12_attention_weight_min": -27.340791702270508,
      "activations/layer13_attention_weight_max": 40.20488739013672,
      "activations/layer13_attention_weight_min": -37.346343994140625,
      "activations/layer14_attention_weight_max": 43.702606201171875,
      "activations/layer14_attention_weight_min": -41.37684631347656,
      "activations/layer15_attention_weight_max": 41.987640380859375,
      "activations/layer15_attention_weight_min": -39.41286849975586,
      "activations/layer16_attention_weight_max": 32.44109344482422,
      "activations/layer16_attention_weight_min": -29.962682723999023,
      "activations/layer17_attention_weight_max": 29.540658950805664,
      "activations/layer17_attention_weight_min": -28.71165657043457,
      "activations/layer18_attention_weight_max": 34.63335418701172,
      "activations/layer18_attention_weight_min": -30.084505081176758,
      "activations/layer19_attention_weight_max": 35.79624557495117,
      "activations/layer19_attention_weight_min": -34.23621368408203,
      "activations/layer1_attention_weight_max": 16.191545486450195,
      "activations/layer1_attention_weight_min": -15.17884349822998,
      "activations/layer20_attention_weight_max": 31.01168441772461,
      "activations/layer20_attention_weight_min": -27.664379119873047,
      "activations/layer21_attention_weight_max": 30.247121810913086,
      "activations/layer21_attention_weight_min": -26.91280746459961,
      "activations/layer22_attention_weight_max": 39.99205780029297,
      "activations/layer22_attention_weight_min": -34.338741302490234,
      "activations/layer23_attention_weight_max": 29.490257263183594,
      "activations/layer23_attention_weight_min": -27.517866134643555,
      "activations/layer2_attention_weight_max": 36.65595626831055,
      "activations/layer2_attention_weight_min": -32.572303771972656,
      "activations/layer3_attention_weight_max": 109.70848083496094,
      "activations/layer3_attention_weight_min": -109.78109741210938,
      "activations/layer4_attention_weight_max": 113.3445816040039,
      "activations/layer4_attention_weight_min": -114.93431854248047,
      "activations/layer5_attention_weight_max": 53.661739349365234,
      "activations/layer5_attention_weight_min": -64.15011596679688,
      "activations/layer6_attention_weight_max": 46.84321212768555,
      "activations/layer6_attention_weight_min": -52.679264068603516,
      "activations/layer7_attention_weight_max": 104.21644592285156,
      "activations/layer7_attention_weight_min": -107.97419738769531,
      "activations/layer8_attention_weight_max": 44.68514633178711,
      "activations/layer8_attention_weight_min": -48.918785095214844,
      "activations/layer9_attention_weight_max": 35.370792388916016,
      "activations/layer9_attention_weight_min": -35.76930618286133,
      "epoch": 23.02,
      "learning_rate": 1.574242424242424e-06,
      "loss": 2.6892,
      "step": 396150
    },
    {
      "activations/layer0_attention_weight_max": 16.07329559326172,
      "activations/layer0_attention_weight_min": -15.048166275024414,
      "activations/layer10_attention_weight_max": 35.71308517456055,
      "activations/layer10_attention_weight_min": -36.32386016845703,
      "activations/layer11_attention_weight_max": 33.94873046875,
      "activations/layer11_attention_weight_min": -36.70740509033203,
      "activations/layer12_attention_weight_max": 27.190656661987305,
      "activations/layer12_attention_weight_min": -26.811933517456055,
      "activations/layer13_attention_weight_max": 39.80921936035156,
      "activations/layer13_attention_weight_min": -37.32563018798828,
      "activations/layer14_attention_weight_max": 44.337833404541016,
      "activations/layer14_attention_weight_min": -39.84153366088867,
      "activations/layer15_attention_weight_max": 42.220218658447266,
      "activations/layer15_attention_weight_min": -37.58205032348633,
      "activations/layer16_attention_weight_max": 32.795989990234375,
      "activations/layer16_attention_weight_min": -29.354591369628906,
      "activations/layer17_attention_weight_max": 32.605018615722656,
      "activations/layer17_attention_weight_min": -25.352577209472656,
      "activations/layer18_attention_weight_max": 35.01087188720703,
      "activations/layer18_attention_weight_min": -25.990665435791016,
      "activations/layer19_attention_weight_max": 37.75072479248047,
      "activations/layer19_attention_weight_min": -31.353042602539062,
      "activations/layer1_attention_weight_max": 16.736299514770508,
      "activations/layer1_attention_weight_min": -15.205301284790039,
      "activations/layer20_attention_weight_max": 34.64125061035156,
      "activations/layer20_attention_weight_min": -24.65199851989746,
      "activations/layer21_attention_weight_max": 32.08156204223633,
      "activations/layer21_attention_weight_min": -24.578227996826172,
      "activations/layer22_attention_weight_max": 43.7132453918457,
      "activations/layer22_attention_weight_min": -31.944522857666016,
      "activations/layer23_attention_weight_max": 33.608070373535156,
      "activations/layer23_attention_weight_min": -24.595821380615234,
      "activations/layer2_attention_weight_max": 34.04981231689453,
      "activations/layer2_attention_weight_min": -33.23487854003906,
      "activations/layer3_attention_weight_max": 109.9350814819336,
      "activations/layer3_attention_weight_min": -111.52423095703125,
      "activations/layer4_attention_weight_max": 112.05587005615234,
      "activations/layer4_attention_weight_min": -116.12892150878906,
      "activations/layer5_attention_weight_max": 52.42490768432617,
      "activations/layer5_attention_weight_min": -61.954673767089844,
      "activations/layer6_attention_weight_max": 47.187435150146484,
      "activations/layer6_attention_weight_min": -49.53239440917969,
      "activations/layer7_attention_weight_max": 97.79661560058594,
      "activations/layer7_attention_weight_min": -107.84733581542969,
      "activations/layer8_attention_weight_max": 43.12929153442383,
      "activations/layer8_attention_weight_min": -47.83415603637695,
      "activations/layer9_attention_weight_max": 32.85129928588867,
      "activations/layer9_attention_weight_min": -35.81972885131836,
      "epoch": 23.02,
      "learning_rate": 1.5553030303030302e-06,
      "loss": 2.7015,
      "step": 396200
    },
    {
      "activations/layer0_attention_weight_max": 16.992258071899414,
      "activations/layer0_attention_weight_min": -15.410494804382324,
      "activations/layer10_attention_weight_max": 35.003875732421875,
      "activations/layer10_attention_weight_min": -35.29807662963867,
      "activations/layer11_attention_weight_max": 33.91572952270508,
      "activations/layer11_attention_weight_min": -34.988555908203125,
      "activations/layer12_attention_weight_max": 26.661165237426758,
      "activations/layer12_attention_weight_min": -28.47340202331543,
      "activations/layer13_attention_weight_max": 38.53760528564453,
      "activations/layer13_attention_weight_min": -36.156856536865234,
      "activations/layer14_attention_weight_max": 39.89338684082031,
      "activations/layer14_attention_weight_min": -37.82830047607422,
      "activations/layer15_attention_weight_max": 39.2120361328125,
      "activations/layer15_attention_weight_min": -37.87224578857422,
      "activations/layer16_attention_weight_max": 29.259042739868164,
      "activations/layer16_attention_weight_min": -27.854625701904297,
      "activations/layer17_attention_weight_max": 28.86244773864746,
      "activations/layer17_attention_weight_min": -25.0189151763916,
      "activations/layer18_attention_weight_max": 32.416893005371094,
      "activations/layer18_attention_weight_min": -25.4705810546875,
      "activations/layer19_attention_weight_max": 35.16495132446289,
      "activations/layer19_attention_weight_min": -31.01595115661621,
      "activations/layer1_attention_weight_max": 16.428016662597656,
      "activations/layer1_attention_weight_min": -16.04431915283203,
      "activations/layer20_attention_weight_max": 28.920291900634766,
      "activations/layer20_attention_weight_min": -24.600496292114258,
      "activations/layer21_attention_weight_max": 28.887290954589844,
      "activations/layer21_attention_weight_min": -23.975637435913086,
      "activations/layer22_attention_weight_max": 39.94219970703125,
      "activations/layer22_attention_weight_min": -29.868453979492188,
      "activations/layer23_attention_weight_max": 31.21222496032715,
      "activations/layer23_attention_weight_min": -27.222150802612305,
      "activations/layer2_attention_weight_max": 33.68184280395508,
      "activations/layer2_attention_weight_min": -32.54102325439453,
      "activations/layer3_attention_weight_max": 107.9576416015625,
      "activations/layer3_attention_weight_min": -107.45062255859375,
      "activations/layer4_attention_weight_max": 112.276123046875,
      "activations/layer4_attention_weight_min": -115.38496398925781,
      "activations/layer5_attention_weight_max": 56.86307907104492,
      "activations/layer5_attention_weight_min": -63.56378936767578,
      "activations/layer6_attention_weight_max": 45.52851867675781,
      "activations/layer6_attention_weight_min": -48.74617385864258,
      "activations/layer7_attention_weight_max": 101.4580307006836,
      "activations/layer7_attention_weight_min": -100.62467956542969,
      "activations/layer8_attention_weight_max": 43.6668586730957,
      "activations/layer8_attention_weight_min": -45.9754524230957,
      "activations/layer9_attention_weight_max": 33.10934066772461,
      "activations/layer9_attention_weight_min": -35.625858306884766,
      "epoch": 23.02,
      "learning_rate": 1.5363636363636361e-06,
      "loss": 2.6886,
      "step": 396250
    },
    {
      "activations/layer0_attention_weight_max": 16.56168556213379,
      "activations/layer0_attention_weight_min": -15.494812965393066,
      "activations/layer10_attention_weight_max": 36.45848846435547,
      "activations/layer10_attention_weight_min": -36.80363082885742,
      "activations/layer11_attention_weight_max": 37.47267532348633,
      "activations/layer11_attention_weight_min": -37.06343078613281,
      "activations/layer12_attention_weight_max": 30.24445152282715,
      "activations/layer12_attention_weight_min": -27.137496948242188,
      "activations/layer13_attention_weight_max": 46.20797348022461,
      "activations/layer13_attention_weight_min": -36.12466812133789,
      "activations/layer14_attention_weight_max": 49.791221618652344,
      "activations/layer14_attention_weight_min": -40.27178192138672,
      "activations/layer15_attention_weight_max": 50.09623336791992,
      "activations/layer15_attention_weight_min": -39.32013702392578,
      "activations/layer16_attention_weight_max": 37.933712005615234,
      "activations/layer16_attention_weight_min": -28.576711654663086,
      "activations/layer17_attention_weight_max": 37.95610046386719,
      "activations/layer17_attention_weight_min": -28.146509170532227,
      "activations/layer18_attention_weight_max": 39.18245315551758,
      "activations/layer18_attention_weight_min": -27.487455368041992,
      "activations/layer19_attention_weight_max": 48.10731506347656,
      "activations/layer19_attention_weight_min": -34.33664321899414,
      "activations/layer1_attention_weight_max": 15.656888961791992,
      "activations/layer1_attention_weight_min": -15.824501037597656,
      "activations/layer20_attention_weight_max": 40.7298469543457,
      "activations/layer20_attention_weight_min": -25.162263870239258,
      "activations/layer21_attention_weight_max": 41.6109504699707,
      "activations/layer21_attention_weight_min": -24.850627899169922,
      "activations/layer22_attention_weight_max": 51.9056282043457,
      "activations/layer22_attention_weight_min": -33.41965103149414,
      "activations/layer23_attention_weight_max": 40.03546905517578,
      "activations/layer23_attention_weight_min": -25.148801803588867,
      "activations/layer2_attention_weight_max": 35.218074798583984,
      "activations/layer2_attention_weight_min": -33.47481155395508,
      "activations/layer3_attention_weight_max": 113.14769744873047,
      "activations/layer3_attention_weight_min": -109.5816879272461,
      "activations/layer4_attention_weight_max": 118.6725082397461,
      "activations/layer4_attention_weight_min": -113.85555267333984,
      "activations/layer5_attention_weight_max": 51.721004486083984,
      "activations/layer5_attention_weight_min": -62.8491325378418,
      "activations/layer6_attention_weight_max": 46.73143005371094,
      "activations/layer6_attention_weight_min": -49.6881217956543,
      "activations/layer7_attention_weight_max": 102.47319793701172,
      "activations/layer7_attention_weight_min": -100.43038940429688,
      "activations/layer8_attention_weight_max": 43.962440490722656,
      "activations/layer8_attention_weight_min": -50.2899169921875,
      "activations/layer9_attention_weight_max": 33.788021087646484,
      "activations/layer9_attention_weight_min": -35.921730041503906,
      "epoch": 23.03,
      "learning_rate": 1.5174242424242423e-06,
      "loss": 2.688,
      "step": 396300
    },
    {
      "activations/layer0_attention_weight_max": 17.41720962524414,
      "activations/layer0_attention_weight_min": -14.95821762084961,
      "activations/layer10_attention_weight_max": 35.09651184082031,
      "activations/layer10_attention_weight_min": -36.55849838256836,
      "activations/layer11_attention_weight_max": 35.17838668823242,
      "activations/layer11_attention_weight_min": -36.804100036621094,
      "activations/layer12_attention_weight_max": 26.50043296813965,
      "activations/layer12_attention_weight_min": -27.883005142211914,
      "activations/layer13_attention_weight_max": 40.85396957397461,
      "activations/layer13_attention_weight_min": -37.067291259765625,
      "activations/layer14_attention_weight_max": 40.863067626953125,
      "activations/layer14_attention_weight_min": -37.22221374511719,
      "activations/layer15_attention_weight_max": 37.58148956298828,
      "activations/layer15_attention_weight_min": -35.30282211303711,
      "activations/layer16_attention_weight_max": 31.655431747436523,
      "activations/layer16_attention_weight_min": -29.766494750976562,
      "activations/layer17_attention_weight_max": 28.361055374145508,
      "activations/layer17_attention_weight_min": -25.870328903198242,
      "activations/layer18_attention_weight_max": 32.27930450439453,
      "activations/layer18_attention_weight_min": -25.379690170288086,
      "activations/layer19_attention_weight_max": 31.794570922851562,
      "activations/layer19_attention_weight_min": -31.382108688354492,
      "activations/layer1_attention_weight_max": 15.871485710144043,
      "activations/layer1_attention_weight_min": -15.406017303466797,
      "activations/layer20_attention_weight_max": 27.392366409301758,
      "activations/layer20_attention_weight_min": -24.55821418762207,
      "activations/layer21_attention_weight_max": 27.880001068115234,
      "activations/layer21_attention_weight_min": -24.3914794921875,
      "activations/layer22_attention_weight_max": 38.5418586730957,
      "activations/layer22_attention_weight_min": -31.63098907470703,
      "activations/layer23_attention_weight_max": 31.81720733642578,
      "activations/layer23_attention_weight_min": -26.509674072265625,
      "activations/layer2_attention_weight_max": 34.81268310546875,
      "activations/layer2_attention_weight_min": -34.95667266845703,
      "activations/layer3_attention_weight_max": 112.7249755859375,
      "activations/layer3_attention_weight_min": -114.38323211669922,
      "activations/layer4_attention_weight_max": 115.4049072265625,
      "activations/layer4_attention_weight_min": -117.32752990722656,
      "activations/layer5_attention_weight_max": 50.19773483276367,
      "activations/layer5_attention_weight_min": -61.377193450927734,
      "activations/layer6_attention_weight_max": 47.6091423034668,
      "activations/layer6_attention_weight_min": -51.51408767700195,
      "activations/layer7_attention_weight_max": 100.33309936523438,
      "activations/layer7_attention_weight_min": -113.88874816894531,
      "activations/layer8_attention_weight_max": 43.9112434387207,
      "activations/layer8_attention_weight_min": -48.23143768310547,
      "activations/layer9_attention_weight_max": 33.141746520996094,
      "activations/layer9_attention_weight_min": -34.96099853515625,
      "epoch": 23.03,
      "learning_rate": 1.4984848484848482e-06,
      "loss": 2.7061,
      "step": 396350
    },
    {
      "activations/layer0_attention_weight_max": 16.741117477416992,
      "activations/layer0_attention_weight_min": -14.75368881225586,
      "activations/layer10_attention_weight_max": 37.566165924072266,
      "activations/layer10_attention_weight_min": -38.31768798828125,
      "activations/layer11_attention_weight_max": 35.757720947265625,
      "activations/layer11_attention_weight_min": -38.759666442871094,
      "activations/layer12_attention_weight_max": 28.57396125793457,
      "activations/layer12_attention_weight_min": -26.97391700744629,
      "activations/layer13_attention_weight_max": 43.07329177856445,
      "activations/layer13_attention_weight_min": -38.35860824584961,
      "activations/layer14_attention_weight_max": 44.92510223388672,
      "activations/layer14_attention_weight_min": -39.76671600341797,
      "activations/layer15_attention_weight_max": 41.020111083984375,
      "activations/layer15_attention_weight_min": -37.34166717529297,
      "activations/layer16_attention_weight_max": 30.075746536254883,
      "activations/layer16_attention_weight_min": -30.065011978149414,
      "activations/layer17_attention_weight_max": 31.268733978271484,
      "activations/layer17_attention_weight_min": -27.9453125,
      "activations/layer18_attention_weight_max": 34.42365646362305,
      "activations/layer18_attention_weight_min": -29.517826080322266,
      "activations/layer19_attention_weight_max": 37.54097366333008,
      "activations/layer19_attention_weight_min": -36.85920715332031,
      "activations/layer1_attention_weight_max": 16.473724365234375,
      "activations/layer1_attention_weight_min": -15.580991744995117,
      "activations/layer20_attention_weight_max": 30.69097328186035,
      "activations/layer20_attention_weight_min": -28.56171226501465,
      "activations/layer21_attention_weight_max": 33.95728302001953,
      "activations/layer21_attention_weight_min": -27.397396087646484,
      "activations/layer22_attention_weight_max": 43.992427825927734,
      "activations/layer22_attention_weight_min": -37.6891975402832,
      "activations/layer23_attention_weight_max": 33.43999099731445,
      "activations/layer23_attention_weight_min": -31.20649528503418,
      "activations/layer2_attention_weight_max": 34.162208557128906,
      "activations/layer2_attention_weight_min": -33.47228240966797,
      "activations/layer3_attention_weight_max": 109.44263458251953,
      "activations/layer3_attention_weight_min": -107.82379913330078,
      "activations/layer4_attention_weight_max": 112.0643310546875,
      "activations/layer4_attention_weight_min": -111.4842300415039,
      "activations/layer5_attention_weight_max": 53.675872802734375,
      "activations/layer5_attention_weight_min": -61.09943389892578,
      "activations/layer6_attention_weight_max": 46.86420440673828,
      "activations/layer6_attention_weight_min": -47.374229431152344,
      "activations/layer7_attention_weight_max": 95.79461669921875,
      "activations/layer7_attention_weight_min": -103.15447998046875,
      "activations/layer8_attention_weight_max": 45.67988967895508,
      "activations/layer8_attention_weight_min": -44.569427490234375,
      "activations/layer9_attention_weight_max": 34.99457931518555,
      "activations/layer9_attention_weight_min": -37.51687240600586,
      "epoch": 23.03,
      "learning_rate": 1.4795454545454546e-06,
      "loss": 2.6859,
      "step": 396400
    },
    {
      "activations/layer0_attention_weight_max": 16.70417594909668,
      "activations/layer0_attention_weight_min": -13.973920822143555,
      "activations/layer10_attention_weight_max": 42.015098571777344,
      "activations/layer10_attention_weight_min": -38.42097854614258,
      "activations/layer11_attention_weight_max": 40.44982147216797,
      "activations/layer11_attention_weight_min": -37.498863220214844,
      "activations/layer12_attention_weight_max": 27.506484985351562,
      "activations/layer12_attention_weight_min": -29.471126556396484,
      "activations/layer13_attention_weight_max": 40.07406234741211,
      "activations/layer13_attention_weight_min": -36.369346618652344,
      "activations/layer14_attention_weight_max": 41.70351028442383,
      "activations/layer14_attention_weight_min": -38.430503845214844,
      "activations/layer15_attention_weight_max": 40.65375518798828,
      "activations/layer15_attention_weight_min": -37.975521087646484,
      "activations/layer16_attention_weight_max": 30.566423416137695,
      "activations/layer16_attention_weight_min": -27.91545295715332,
      "activations/layer17_attention_weight_max": 29.33011245727539,
      "activations/layer17_attention_weight_min": -25.568614959716797,
      "activations/layer18_attention_weight_max": 37.59828567504883,
      "activations/layer18_attention_weight_min": -25.163515090942383,
      "activations/layer19_attention_weight_max": 38.19950485229492,
      "activations/layer19_attention_weight_min": -30.628049850463867,
      "activations/layer1_attention_weight_max": 15.669353485107422,
      "activations/layer1_attention_weight_min": -14.90904712677002,
      "activations/layer20_attention_weight_max": 32.80474090576172,
      "activations/layer20_attention_weight_min": -25.05817222595215,
      "activations/layer21_attention_weight_max": 31.594938278198242,
      "activations/layer21_attention_weight_min": -24.143728256225586,
      "activations/layer22_attention_weight_max": 44.65846252441406,
      "activations/layer22_attention_weight_min": -31.056354522705078,
      "activations/layer23_attention_weight_max": 32.16360855102539,
      "activations/layer23_attention_weight_min": -24.61186981201172,
      "activations/layer2_attention_weight_max": 34.89394760131836,
      "activations/layer2_attention_weight_min": -33.79376983642578,
      "activations/layer3_attention_weight_max": 110.1921615600586,
      "activations/layer3_attention_weight_min": -110.67253875732422,
      "activations/layer4_attention_weight_max": 110.65001678466797,
      "activations/layer4_attention_weight_min": -112.4209976196289,
      "activations/layer5_attention_weight_max": 51.94684600830078,
      "activations/layer5_attention_weight_min": -59.823787689208984,
      "activations/layer6_attention_weight_max": 47.51668167114258,
      "activations/layer6_attention_weight_min": -49.787574768066406,
      "activations/layer7_attention_weight_max": 105.11013793945312,
      "activations/layer7_attention_weight_min": -104.4957046508789,
      "activations/layer8_attention_weight_max": 48.67893981933594,
      "activations/layer8_attention_weight_min": -48.68092727661133,
      "activations/layer9_attention_weight_max": 36.81162643432617,
      "activations/layer9_attention_weight_min": -37.72537612915039,
      "epoch": 23.04,
      "learning_rate": 1.4606060606060605e-06,
      "loss": 2.6959,
      "step": 396450
    },
    {
      "activations/layer0_attention_weight_max": 17.065446853637695,
      "activations/layer0_attention_weight_min": -14.292805671691895,
      "activations/layer10_attention_weight_max": 36.76686096191406,
      "activations/layer10_attention_weight_min": -35.23872756958008,
      "activations/layer11_attention_weight_max": 35.85176086425781,
      "activations/layer11_attention_weight_min": -35.76200866699219,
      "activations/layer12_attention_weight_max": 27.167591094970703,
      "activations/layer12_attention_weight_min": -28.206233978271484,
      "activations/layer13_attention_weight_max": 41.18134307861328,
      "activations/layer13_attention_weight_min": -41.194915771484375,
      "activations/layer14_attention_weight_max": 42.91399002075195,
      "activations/layer14_attention_weight_min": -40.96394348144531,
      "activations/layer15_attention_weight_max": 40.10255432128906,
      "activations/layer15_attention_weight_min": -38.34342956542969,
      "activations/layer16_attention_weight_max": 30.886425018310547,
      "activations/layer16_attention_weight_min": -28.421913146972656,
      "activations/layer17_attention_weight_max": 30.35449981689453,
      "activations/layer17_attention_weight_min": -26.577030181884766,
      "activations/layer18_attention_weight_max": 31.025747299194336,
      "activations/layer18_attention_weight_min": -26.460939407348633,
      "activations/layer19_attention_weight_max": 34.45933532714844,
      "activations/layer19_attention_weight_min": -32.0907096862793,
      "activations/layer1_attention_weight_max": 15.485004425048828,
      "activations/layer1_attention_weight_min": -14.339457511901855,
      "activations/layer20_attention_weight_max": 28.42352867126465,
      "activations/layer20_attention_weight_min": -25.161813735961914,
      "activations/layer21_attention_weight_max": 33.104408264160156,
      "activations/layer21_attention_weight_min": -25.811010360717773,
      "activations/layer22_attention_weight_max": 40.69453430175781,
      "activations/layer22_attention_weight_min": -32.802181243896484,
      "activations/layer23_attention_weight_max": 32.42366409301758,
      "activations/layer23_attention_weight_min": -26.57767105102539,
      "activations/layer2_attention_weight_max": 33.41028594970703,
      "activations/layer2_attention_weight_min": -33.803489685058594,
      "activations/layer3_attention_weight_max": 110.09465026855469,
      "activations/layer3_attention_weight_min": -111.4560775756836,
      "activations/layer4_attention_weight_max": 114.36998748779297,
      "activations/layer4_attention_weight_min": -115.62274169921875,
      "activations/layer5_attention_weight_max": 52.64012145996094,
      "activations/layer5_attention_weight_min": -62.483665466308594,
      "activations/layer6_attention_weight_max": 48.39700698852539,
      "activations/layer6_attention_weight_min": -49.827049255371094,
      "activations/layer7_attention_weight_max": 96.78441619873047,
      "activations/layer7_attention_weight_min": -100.15328216552734,
      "activations/layer8_attention_weight_max": 44.8178825378418,
      "activations/layer8_attention_weight_min": -45.976783752441406,
      "activations/layer9_attention_weight_max": 34.05887222290039,
      "activations/layer9_attention_weight_min": -35.87062454223633,
      "epoch": 23.04,
      "learning_rate": 1.4416666666666667e-06,
      "loss": 2.6962,
      "step": 396500
    },
    {
      "activations/layer0_attention_weight_max": 17.175180435180664,
      "activations/layer0_attention_weight_min": -14.387989044189453,
      "activations/layer10_attention_weight_max": 40.92281723022461,
      "activations/layer10_attention_weight_min": -38.720733642578125,
      "activations/layer11_attention_weight_max": 39.83246612548828,
      "activations/layer11_attention_weight_min": -40.039634704589844,
      "activations/layer12_attention_weight_max": 27.800392150878906,
      "activations/layer12_attention_weight_min": -27.5604305267334,
      "activations/layer13_attention_weight_max": 42.541748046875,
      "activations/layer13_attention_weight_min": -37.907630920410156,
      "activations/layer14_attention_weight_max": 48.66050338745117,
      "activations/layer14_attention_weight_min": -40.797752380371094,
      "activations/layer15_attention_weight_max": 44.66807174682617,
      "activations/layer15_attention_weight_min": -41.92670822143555,
      "activations/layer16_attention_weight_max": 31.918363571166992,
      "activations/layer16_attention_weight_min": -29.24380111694336,
      "activations/layer17_attention_weight_max": 31.817501068115234,
      "activations/layer17_attention_weight_min": -27.54818344116211,
      "activations/layer18_attention_weight_max": 34.468685150146484,
      "activations/layer18_attention_weight_min": -26.394969940185547,
      "activations/layer19_attention_weight_max": 34.84428787231445,
      "activations/layer19_attention_weight_min": -30.606281280517578,
      "activations/layer1_attention_weight_max": 16.876056671142578,
      "activations/layer1_attention_weight_min": -15.12993335723877,
      "activations/layer20_attention_weight_max": 32.00157165527344,
      "activations/layer20_attention_weight_min": -24.20844841003418,
      "activations/layer21_attention_weight_max": 33.2619514465332,
      "activations/layer21_attention_weight_min": -24.49873924255371,
      "activations/layer22_attention_weight_max": 44.33146286010742,
      "activations/layer22_attention_weight_min": -33.33879470825195,
      "activations/layer23_attention_weight_max": 30.840015411376953,
      "activations/layer23_attention_weight_min": -27.274368286132812,
      "activations/layer2_attention_weight_max": 37.174110412597656,
      "activations/layer2_attention_weight_min": -36.2769660949707,
      "activations/layer3_attention_weight_max": 114.29353332519531,
      "activations/layer3_attention_weight_min": -116.03655242919922,
      "activations/layer4_attention_weight_max": 112.0246353149414,
      "activations/layer4_attention_weight_min": -116.07518768310547,
      "activations/layer5_attention_weight_max": 51.40311813354492,
      "activations/layer5_attention_weight_min": -62.57578659057617,
      "activations/layer6_attention_weight_max": 47.2521858215332,
      "activations/layer6_attention_weight_min": -50.4754524230957,
      "activations/layer7_attention_weight_max": 101.99844360351562,
      "activations/layer7_attention_weight_min": -105.72234344482422,
      "activations/layer8_attention_weight_max": 48.47554016113281,
      "activations/layer8_attention_weight_min": -49.81584548950195,
      "activations/layer9_attention_weight_max": 38.87166976928711,
      "activations/layer9_attention_weight_min": -40.26430892944336,
      "epoch": 23.04,
      "learning_rate": 1.4227272727272726e-06,
      "loss": 2.6931,
      "step": 396550
    },
    {
      "activations/layer0_attention_weight_max": 17.30891990661621,
      "activations/layer0_attention_weight_min": -14.9236421585083,
      "activations/layer10_attention_weight_max": 34.375431060791016,
      "activations/layer10_attention_weight_min": -35.25700759887695,
      "activations/layer11_attention_weight_max": 32.91716384887695,
      "activations/layer11_attention_weight_min": -33.796836853027344,
      "activations/layer12_attention_weight_max": 26.687992095947266,
      "activations/layer12_attention_weight_min": -26.21782684326172,
      "activations/layer13_attention_weight_max": 39.1087760925293,
      "activations/layer13_attention_weight_min": -36.205570220947266,
      "activations/layer14_attention_weight_max": 43.740882873535156,
      "activations/layer14_attention_weight_min": -37.62343215942383,
      "activations/layer15_attention_weight_max": 38.90317153930664,
      "activations/layer15_attention_weight_min": -37.640262603759766,
      "activations/layer16_attention_weight_max": 28.897615432739258,
      "activations/layer16_attention_weight_min": -30.36714744567871,
      "activations/layer17_attention_weight_max": 29.40170669555664,
      "activations/layer17_attention_weight_min": -26.57748031616211,
      "activations/layer18_attention_weight_max": 34.238765716552734,
      "activations/layer18_attention_weight_min": -27.125139236450195,
      "activations/layer19_attention_weight_max": 36.04612731933594,
      "activations/layer19_attention_weight_min": -33.125816345214844,
      "activations/layer1_attention_weight_max": 15.436863899230957,
      "activations/layer1_attention_weight_min": -14.996935844421387,
      "activations/layer20_attention_weight_max": 30.486000061035156,
      "activations/layer20_attention_weight_min": -26.462291717529297,
      "activations/layer21_attention_weight_max": 27.15165138244629,
      "activations/layer21_attention_weight_min": -25.791887283325195,
      "activations/layer22_attention_weight_max": 41.45756149291992,
      "activations/layer22_attention_weight_min": -32.13144302368164,
      "activations/layer23_attention_weight_max": 29.478363037109375,
      "activations/layer23_attention_weight_min": -26.045963287353516,
      "activations/layer2_attention_weight_max": 35.32859420776367,
      "activations/layer2_attention_weight_min": -33.65635299682617,
      "activations/layer3_attention_weight_max": 106.21602630615234,
      "activations/layer3_attention_weight_min": -111.37596130371094,
      "activations/layer4_attention_weight_max": 111.71639251708984,
      "activations/layer4_attention_weight_min": -123.23065185546875,
      "activations/layer5_attention_weight_max": 51.36157989501953,
      "activations/layer5_attention_weight_min": -63.51093673706055,
      "activations/layer6_attention_weight_max": 46.73759841918945,
      "activations/layer6_attention_weight_min": -51.55402374267578,
      "activations/layer7_attention_weight_max": 99.60618591308594,
      "activations/layer7_attention_weight_min": -107.88617706298828,
      "activations/layer8_attention_weight_max": 43.789615631103516,
      "activations/layer8_attention_weight_min": -44.246219635009766,
      "activations/layer9_attention_weight_max": 32.15708541870117,
      "activations/layer9_attention_weight_min": -33.65981674194336,
      "epoch": 23.04,
      "learning_rate": 1.4037878787878786e-06,
      "loss": 2.6901,
      "step": 396600
    },
    {
      "activations/layer0_attention_weight_max": 16.46637725830078,
      "activations/layer0_attention_weight_min": -14.657087326049805,
      "activations/layer10_attention_weight_max": 38.08098602294922,
      "activations/layer10_attention_weight_min": -40.493648529052734,
      "activations/layer11_attention_weight_max": 37.37871170043945,
      "activations/layer11_attention_weight_min": -37.52056884765625,
      "activations/layer12_attention_weight_max": 27.16794204711914,
      "activations/layer12_attention_weight_min": -27.32646942138672,
      "activations/layer13_attention_weight_max": 38.8576774597168,
      "activations/layer13_attention_weight_min": -37.24935531616211,
      "activations/layer14_attention_weight_max": 42.10554504394531,
      "activations/layer14_attention_weight_min": -40.170135498046875,
      "activations/layer15_attention_weight_max": 39.065650939941406,
      "activations/layer15_attention_weight_min": -38.40430450439453,
      "activations/layer16_attention_weight_max": 29.54470443725586,
      "activations/layer16_attention_weight_min": -29.304651260375977,
      "activations/layer17_attention_weight_max": 30.613691329956055,
      "activations/layer17_attention_weight_min": -28.265207290649414,
      "activations/layer18_attention_weight_max": 33.506103515625,
      "activations/layer18_attention_weight_min": -26.8577938079834,
      "activations/layer19_attention_weight_max": 36.625118255615234,
      "activations/layer19_attention_weight_min": -33.85592269897461,
      "activations/layer1_attention_weight_max": 17.897010803222656,
      "activations/layer1_attention_weight_min": -17.10776138305664,
      "activations/layer20_attention_weight_max": 30.616676330566406,
      "activations/layer20_attention_weight_min": -26.701419830322266,
      "activations/layer21_attention_weight_max": 29.50090217590332,
      "activations/layer21_attention_weight_min": -26.082592010498047,
      "activations/layer22_attention_weight_max": 44.42119216918945,
      "activations/layer22_attention_weight_min": -33.53910446166992,
      "activations/layer23_attention_weight_max": 34.78490447998047,
      "activations/layer23_attention_weight_min": -27.12343978881836,
      "activations/layer2_attention_weight_max": 34.44348907470703,
      "activations/layer2_attention_weight_min": -33.773929595947266,
      "activations/layer3_attention_weight_max": 107.72626495361328,
      "activations/layer3_attention_weight_min": -110.08806610107422,
      "activations/layer4_attention_weight_max": 111.72135162353516,
      "activations/layer4_attention_weight_min": -113.44233703613281,
      "activations/layer5_attention_weight_max": 53.27544403076172,
      "activations/layer5_attention_weight_min": -61.38397216796875,
      "activations/layer6_attention_weight_max": 46.684532165527344,
      "activations/layer6_attention_weight_min": -49.030113220214844,
      "activations/layer7_attention_weight_max": 100.87088012695312,
      "activations/layer7_attention_weight_min": -101.65017700195312,
      "activations/layer8_attention_weight_max": 46.330692291259766,
      "activations/layer8_attention_weight_min": -45.18776321411133,
      "activations/layer9_attention_weight_max": 35.390018463134766,
      "activations/layer9_attention_weight_min": -39.62594223022461,
      "epoch": 23.05,
      "learning_rate": 1.3848484848484847e-06,
      "loss": 2.6926,
      "step": 396650
    },
    {
      "activations/layer0_attention_weight_max": 16.88497543334961,
      "activations/layer0_attention_weight_min": -14.942731857299805,
      "activations/layer10_attention_weight_max": 38.14644241333008,
      "activations/layer10_attention_weight_min": -38.81663513183594,
      "activations/layer11_attention_weight_max": 36.341957092285156,
      "activations/layer11_attention_weight_min": -38.33872604370117,
      "activations/layer12_attention_weight_max": 25.59256362915039,
      "activations/layer12_attention_weight_min": -30.034957885742188,
      "activations/layer13_attention_weight_max": 38.19453430175781,
      "activations/layer13_attention_weight_min": -38.26959991455078,
      "activations/layer14_attention_weight_max": 41.97699737548828,
      "activations/layer14_attention_weight_min": -37.85896682739258,
      "activations/layer15_attention_weight_max": 39.37314987182617,
      "activations/layer15_attention_weight_min": -38.25925827026367,
      "activations/layer16_attention_weight_max": 32.16593933105469,
      "activations/layer16_attention_weight_min": -29.46107292175293,
      "activations/layer17_attention_weight_max": 31.39021873474121,
      "activations/layer17_attention_weight_min": -27.0864315032959,
      "activations/layer18_attention_weight_max": 32.30942916870117,
      "activations/layer18_attention_weight_min": -27.376136779785156,
      "activations/layer19_attention_weight_max": 36.479976654052734,
      "activations/layer19_attention_weight_min": -34.281776428222656,
      "activations/layer1_attention_weight_max": 15.644779205322266,
      "activations/layer1_attention_weight_min": -15.35346794128418,
      "activations/layer20_attention_weight_max": 31.274017333984375,
      "activations/layer20_attention_weight_min": -27.406505584716797,
      "activations/layer21_attention_weight_max": 28.753429412841797,
      "activations/layer21_attention_weight_min": -26.38065528869629,
      "activations/layer22_attention_weight_max": 44.7607421875,
      "activations/layer22_attention_weight_min": -32.67737579345703,
      "activations/layer23_attention_weight_max": 32.62334060668945,
      "activations/layer23_attention_weight_min": -24.60848617553711,
      "activations/layer2_attention_weight_max": 35.90715026855469,
      "activations/layer2_attention_weight_min": -35.688804626464844,
      "activations/layer3_attention_weight_max": 113.38668823242188,
      "activations/layer3_attention_weight_min": -115.09237670898438,
      "activations/layer4_attention_weight_max": 113.0215835571289,
      "activations/layer4_attention_weight_min": -119.3171615600586,
      "activations/layer5_attention_weight_max": 51.88262176513672,
      "activations/layer5_attention_weight_min": -62.48054122924805,
      "activations/layer6_attention_weight_max": 46.597164154052734,
      "activations/layer6_attention_weight_min": -48.51943588256836,
      "activations/layer7_attention_weight_max": 99.61714935302734,
      "activations/layer7_attention_weight_min": -109.30097198486328,
      "activations/layer8_attention_weight_max": 46.3152961730957,
      "activations/layer8_attention_weight_min": -49.52055740356445,
      "activations/layer9_attention_weight_max": 33.732791900634766,
      "activations/layer9_attention_weight_min": -37.80343246459961,
      "epoch": 23.05,
      "learning_rate": 1.3659090909090907e-06,
      "loss": 2.7037,
      "step": 396700
    },
    {
      "activations/layer0_attention_weight_max": 15.905147552490234,
      "activations/layer0_attention_weight_min": -14.461108207702637,
      "activations/layer10_attention_weight_max": 35.08525085449219,
      "activations/layer10_attention_weight_min": -36.557586669921875,
      "activations/layer11_attention_weight_max": 36.89949035644531,
      "activations/layer11_attention_weight_min": -37.606197357177734,
      "activations/layer12_attention_weight_max": 28.664676666259766,
      "activations/layer12_attention_weight_min": -28.77835464477539,
      "activations/layer13_attention_weight_max": 40.07695770263672,
      "activations/layer13_attention_weight_min": -39.76824188232422,
      "activations/layer14_attention_weight_max": 46.47262191772461,
      "activations/layer14_attention_weight_min": -39.13015365600586,
      "activations/layer15_attention_weight_max": 42.063140869140625,
      "activations/layer15_attention_weight_min": -40.37054443359375,
      "activations/layer16_attention_weight_max": 28.430496215820312,
      "activations/layer16_attention_weight_min": -27.590749740600586,
      "activations/layer17_attention_weight_max": 33.02643585205078,
      "activations/layer17_attention_weight_min": -26.447738647460938,
      "activations/layer18_attention_weight_max": 34.816009521484375,
      "activations/layer18_attention_weight_min": -30.019540786743164,
      "activations/layer19_attention_weight_max": 40.158199310302734,
      "activations/layer19_attention_weight_min": -35.528018951416016,
      "activations/layer1_attention_weight_max": 15.774706840515137,
      "activations/layer1_attention_weight_min": -16.459318161010742,
      "activations/layer20_attention_weight_max": 32.51056671142578,
      "activations/layer20_attention_weight_min": -27.639968872070312,
      "activations/layer21_attention_weight_max": 32.1467170715332,
      "activations/layer21_attention_weight_min": -27.126487731933594,
      "activations/layer22_attention_weight_max": 45.556236267089844,
      "activations/layer22_attention_weight_min": -34.65781784057617,
      "activations/layer23_attention_weight_max": 35.20781707763672,
      "activations/layer23_attention_weight_min": -27.248600006103516,
      "activations/layer2_attention_weight_max": 34.827239990234375,
      "activations/layer2_attention_weight_min": -35.13780212402344,
      "activations/layer3_attention_weight_max": 110.34063720703125,
      "activations/layer3_attention_weight_min": -110.25399017333984,
      "activations/layer4_attention_weight_max": 111.936279296875,
      "activations/layer4_attention_weight_min": -119.5313720703125,
      "activations/layer5_attention_weight_max": 52.5033073425293,
      "activations/layer5_attention_weight_min": -63.26388168334961,
      "activations/layer6_attention_weight_max": 47.250370025634766,
      "activations/layer6_attention_weight_min": -52.21220397949219,
      "activations/layer7_attention_weight_max": 99.47235107421875,
      "activations/layer7_attention_weight_min": -108.63070678710938,
      "activations/layer8_attention_weight_max": 45.2672119140625,
      "activations/layer8_attention_weight_min": -49.108882904052734,
      "activations/layer9_attention_weight_max": 33.5918083190918,
      "activations/layer9_attention_weight_min": -37.443233489990234,
      "epoch": 23.05,
      "learning_rate": 1.3469696969696968e-06,
      "loss": 2.6886,
      "step": 396750
    },
    {
      "activations/layer0_attention_weight_max": 16.796649932861328,
      "activations/layer0_attention_weight_min": -14.268181800842285,
      "activations/layer10_attention_weight_max": 39.38209533691406,
      "activations/layer10_attention_weight_min": -39.32221221923828,
      "activations/layer11_attention_weight_max": 37.58314514160156,
      "activations/layer11_attention_weight_min": -37.522865295410156,
      "activations/layer12_attention_weight_max": 30.186052322387695,
      "activations/layer12_attention_weight_min": -30.192249298095703,
      "activations/layer13_attention_weight_max": 43.30363464355469,
      "activations/layer13_attention_weight_min": -39.46583557128906,
      "activations/layer14_attention_weight_max": 45.96012878417969,
      "activations/layer14_attention_weight_min": -45.666259765625,
      "activations/layer15_attention_weight_max": 41.61902618408203,
      "activations/layer15_attention_weight_min": -40.892032623291016,
      "activations/layer16_attention_weight_max": 31.171932220458984,
      "activations/layer16_attention_weight_min": -30.81538963317871,
      "activations/layer17_attention_weight_max": 31.31467056274414,
      "activations/layer17_attention_weight_min": -27.914371490478516,
      "activations/layer18_attention_weight_max": 32.11370849609375,
      "activations/layer18_attention_weight_min": -27.689298629760742,
      "activations/layer19_attention_weight_max": 34.009124755859375,
      "activations/layer19_attention_weight_min": -33.42157745361328,
      "activations/layer1_attention_weight_max": 14.941852569580078,
      "activations/layer1_attention_weight_min": -15.122444152832031,
      "activations/layer20_attention_weight_max": 27.752634048461914,
      "activations/layer20_attention_weight_min": -25.318004608154297,
      "activations/layer21_attention_weight_max": 28.456016540527344,
      "activations/layer21_attention_weight_min": -24.139537811279297,
      "activations/layer22_attention_weight_max": 41.2412109375,
      "activations/layer22_attention_weight_min": -31.694503784179688,
      "activations/layer23_attention_weight_max": 30.341400146484375,
      "activations/layer23_attention_weight_min": -24.12457275390625,
      "activations/layer2_attention_weight_max": 33.42749786376953,
      "activations/layer2_attention_weight_min": -33.28395462036133,
      "activations/layer3_attention_weight_max": 107.6405029296875,
      "activations/layer3_attention_weight_min": -108.64031219482422,
      "activations/layer4_attention_weight_max": 110.4039077758789,
      "activations/layer4_attention_weight_min": -114.02310943603516,
      "activations/layer5_attention_weight_max": 51.05155944824219,
      "activations/layer5_attention_weight_min": -66.3779067993164,
      "activations/layer6_attention_weight_max": 45.88584518432617,
      "activations/layer6_attention_weight_min": -50.355403900146484,
      "activations/layer7_attention_weight_max": 98.728759765625,
      "activations/layer7_attention_weight_min": -102.4332046508789,
      "activations/layer8_attention_weight_max": 45.675086975097656,
      "activations/layer8_attention_weight_min": -47.04553985595703,
      "activations/layer9_attention_weight_max": 34.31290817260742,
      "activations/layer9_attention_weight_min": -38.68832015991211,
      "epoch": 23.06,
      "learning_rate": 1.3280303030303028e-06,
      "loss": 2.7023,
      "step": 396800
    },
    {
      "activations/layer0_attention_weight_max": 16.700796127319336,
      "activations/layer0_attention_weight_min": -13.852731704711914,
      "activations/layer10_attention_weight_max": 37.014923095703125,
      "activations/layer10_attention_weight_min": -38.141517639160156,
      "activations/layer11_attention_weight_max": 37.70504379272461,
      "activations/layer11_attention_weight_min": -36.86022186279297,
      "activations/layer12_attention_weight_max": 30.775630950927734,
      "activations/layer12_attention_weight_min": -29.265661239624023,
      "activations/layer13_attention_weight_max": 46.544124603271484,
      "activations/layer13_attention_weight_min": -38.718849182128906,
      "activations/layer14_attention_weight_max": 53.37106704711914,
      "activations/layer14_attention_weight_min": -41.548221588134766,
      "activations/layer15_attention_weight_max": 44.35502243041992,
      "activations/layer15_attention_weight_min": -38.04020690917969,
      "activations/layer16_attention_weight_max": 31.872346878051758,
      "activations/layer16_attention_weight_min": -28.918779373168945,
      "activations/layer17_attention_weight_max": 31.02625846862793,
      "activations/layer17_attention_weight_min": -27.071510314941406,
      "activations/layer18_attention_weight_max": 37.08922576904297,
      "activations/layer18_attention_weight_min": -27.2200984954834,
      "activations/layer19_attention_weight_max": 35.91154479980469,
      "activations/layer19_attention_weight_min": -33.33143997192383,
      "activations/layer1_attention_weight_max": 15.76251220703125,
      "activations/layer1_attention_weight_min": -14.97038459777832,
      "activations/layer20_attention_weight_max": 29.493440628051758,
      "activations/layer20_attention_weight_min": -27.38216781616211,
      "activations/layer21_attention_weight_max": 29.614904403686523,
      "activations/layer21_attention_weight_min": -24.992107391357422,
      "activations/layer22_attention_weight_max": 48.54203414916992,
      "activations/layer22_attention_weight_min": -32.05799865722656,
      "activations/layer23_attention_weight_max": 34.8068962097168,
      "activations/layer23_attention_weight_min": -26.87588119506836,
      "activations/layer2_attention_weight_max": 34.97920608520508,
      "activations/layer2_attention_weight_min": -35.57988739013672,
      "activations/layer3_attention_weight_max": 113.8009262084961,
      "activations/layer3_attention_weight_min": -114.56965637207031,
      "activations/layer4_attention_weight_max": 117.21492004394531,
      "activations/layer4_attention_weight_min": -122.05987548828125,
      "activations/layer5_attention_weight_max": 53.49026107788086,
      "activations/layer5_attention_weight_min": -64.4199447631836,
      "activations/layer6_attention_weight_max": 49.71086883544922,
      "activations/layer6_attention_weight_min": -51.20833969116211,
      "activations/layer7_attention_weight_max": 102.88951873779297,
      "activations/layer7_attention_weight_min": -109.69544982910156,
      "activations/layer8_attention_weight_max": 46.62664031982422,
      "activations/layer8_attention_weight_min": -48.47716522216797,
      "activations/layer9_attention_weight_max": 35.513267517089844,
      "activations/layer9_attention_weight_min": -37.37312316894531,
      "epoch": 23.06,
      "learning_rate": 1.3090909090909091e-06,
      "loss": 2.6829,
      "step": 396850
    },
    {
      "activations/layer0_attention_weight_max": 17.055912017822266,
      "activations/layer0_attention_weight_min": -14.100330352783203,
      "activations/layer10_attention_weight_max": 35.03463363647461,
      "activations/layer10_attention_weight_min": -35.25014877319336,
      "activations/layer11_attention_weight_max": 33.93924331665039,
      "activations/layer11_attention_weight_min": -35.28150177001953,
      "activations/layer12_attention_weight_max": 28.11652946472168,
      "activations/layer12_attention_weight_min": -29.145715713500977,
      "activations/layer13_attention_weight_max": 41.499481201171875,
      "activations/layer13_attention_weight_min": -39.56699752807617,
      "activations/layer14_attention_weight_max": 42.706809997558594,
      "activations/layer14_attention_weight_min": -39.64863967895508,
      "activations/layer15_attention_weight_max": 39.55063247680664,
      "activations/layer15_attention_weight_min": -37.90842819213867,
      "activations/layer16_attention_weight_max": 29.895633697509766,
      "activations/layer16_attention_weight_min": -30.945178985595703,
      "activations/layer17_attention_weight_max": 32.899559020996094,
      "activations/layer17_attention_weight_min": -28.29502296447754,
      "activations/layer18_attention_weight_max": 35.69518280029297,
      "activations/layer18_attention_weight_min": -28.516084671020508,
      "activations/layer19_attention_weight_max": 34.162471771240234,
      "activations/layer19_attention_weight_min": -32.741512298583984,
      "activations/layer1_attention_weight_max": 16.085599899291992,
      "activations/layer1_attention_weight_min": -14.635583877563477,
      "activations/layer20_attention_weight_max": 28.562145233154297,
      "activations/layer20_attention_weight_min": -25.606176376342773,
      "activations/layer21_attention_weight_max": 29.381589889526367,
      "activations/layer21_attention_weight_min": -25.46062660217285,
      "activations/layer22_attention_weight_max": 40.91530990600586,
      "activations/layer22_attention_weight_min": -32.024436950683594,
      "activations/layer23_attention_weight_max": 31.481733322143555,
      "activations/layer23_attention_weight_min": -26.061887741088867,
      "activations/layer2_attention_weight_max": 34.39486312866211,
      "activations/layer2_attention_weight_min": -33.574241638183594,
      "activations/layer3_attention_weight_max": 109.09036254882812,
      "activations/layer3_attention_weight_min": -111.7243881225586,
      "activations/layer4_attention_weight_max": 112.5677490234375,
      "activations/layer4_attention_weight_min": -117.94879150390625,
      "activations/layer5_attention_weight_max": 51.27891159057617,
      "activations/layer5_attention_weight_min": -65.63307189941406,
      "activations/layer6_attention_weight_max": 47.09165573120117,
      "activations/layer6_attention_weight_min": -49.900814056396484,
      "activations/layer7_attention_weight_max": 99.56285095214844,
      "activations/layer7_attention_weight_min": -109.88154602050781,
      "activations/layer8_attention_weight_max": 44.1120491027832,
      "activations/layer8_attention_weight_min": -47.68699264526367,
      "activations/layer9_attention_weight_max": 34.46124267578125,
      "activations/layer9_attention_weight_min": -36.21062469482422,
      "epoch": 23.06,
      "learning_rate": 1.290151515151515e-06,
      "loss": 2.6955,
      "step": 396900
    },
    {
      "activations/layer0_attention_weight_max": 17.356475830078125,
      "activations/layer0_attention_weight_min": -15.006637573242188,
      "activations/layer10_attention_weight_max": 37.20501708984375,
      "activations/layer10_attention_weight_min": -37.114627838134766,
      "activations/layer11_attention_weight_max": 37.33075714111328,
      "activations/layer11_attention_weight_min": -37.11955261230469,
      "activations/layer12_attention_weight_max": 30.476560592651367,
      "activations/layer12_attention_weight_min": -27.928058624267578,
      "activations/layer13_attention_weight_max": 43.82264709472656,
      "activations/layer13_attention_weight_min": -36.35238265991211,
      "activations/layer14_attention_weight_max": 48.789005279541016,
      "activations/layer14_attention_weight_min": -39.4757194519043,
      "activations/layer15_attention_weight_max": 43.61314392089844,
      "activations/layer15_attention_weight_min": -39.94797134399414,
      "activations/layer16_attention_weight_max": 33.873477935791016,
      "activations/layer16_attention_weight_min": -29.912853240966797,
      "activations/layer17_attention_weight_max": 32.65007400512695,
      "activations/layer17_attention_weight_min": -29.07700538635254,
      "activations/layer18_attention_weight_max": 38.11225891113281,
      "activations/layer18_attention_weight_min": -26.798444747924805,
      "activations/layer19_attention_weight_max": 40.23798751831055,
      "activations/layer19_attention_weight_min": -33.67904281616211,
      "activations/layer1_attention_weight_max": 15.247156143188477,
      "activations/layer1_attention_weight_min": -15.06503963470459,
      "activations/layer20_attention_weight_max": 34.737361907958984,
      "activations/layer20_attention_weight_min": -26.11395263671875,
      "activations/layer21_attention_weight_max": 34.82696533203125,
      "activations/layer21_attention_weight_min": -25.00139045715332,
      "activations/layer22_attention_weight_max": 49.250404357910156,
      "activations/layer22_attention_weight_min": -34.445396423339844,
      "activations/layer23_attention_weight_max": 35.56122589111328,
      "activations/layer23_attention_weight_min": -27.298908233642578,
      "activations/layer2_attention_weight_max": 33.97505187988281,
      "activations/layer2_attention_weight_min": -34.78895568847656,
      "activations/layer3_attention_weight_max": 109.94209289550781,
      "activations/layer3_attention_weight_min": -110.93373107910156,
      "activations/layer4_attention_weight_max": 111.82686614990234,
      "activations/layer4_attention_weight_min": -114.93341064453125,
      "activations/layer5_attention_weight_max": 52.03286361694336,
      "activations/layer5_attention_weight_min": -64.12467193603516,
      "activations/layer6_attention_weight_max": 45.568458557128906,
      "activations/layer6_attention_weight_min": -51.01834487915039,
      "activations/layer7_attention_weight_max": 96.94418334960938,
      "activations/layer7_attention_weight_min": -111.45907592773438,
      "activations/layer8_attention_weight_max": 43.3052978515625,
      "activations/layer8_attention_weight_min": -50.51054382324219,
      "activations/layer9_attention_weight_max": 33.628936767578125,
      "activations/layer9_attention_weight_min": -37.43149948120117,
      "epoch": 23.07,
      "learning_rate": 1.2712121212121212e-06,
      "loss": 2.7139,
      "step": 396950
    },
    {
      "activations/layer0_attention_weight_max": 16.527503967285156,
      "activations/layer0_attention_weight_min": -16.764543533325195,
      "activations/layer10_attention_weight_max": 43.4063720703125,
      "activations/layer10_attention_weight_min": -43.52281188964844,
      "activations/layer11_attention_weight_max": 44.95933532714844,
      "activations/layer11_attention_weight_min": -44.51087188720703,
      "activations/layer12_attention_weight_max": 29.92485237121582,
      "activations/layer12_attention_weight_min": -28.693275451660156,
      "activations/layer13_attention_weight_max": 42.94325256347656,
      "activations/layer13_attention_weight_min": -38.18548583984375,
      "activations/layer14_attention_weight_max": 45.31155776977539,
      "activations/layer14_attention_weight_min": -40.37091827392578,
      "activations/layer15_attention_weight_max": 45.14090347290039,
      "activations/layer15_attention_weight_min": -39.49856185913086,
      "activations/layer16_attention_weight_max": 30.65788459777832,
      "activations/layer16_attention_weight_min": -29.863008499145508,
      "activations/layer17_attention_weight_max": 30.191640853881836,
      "activations/layer17_attention_weight_min": -28.508291244506836,
      "activations/layer18_attention_weight_max": 36.3082160949707,
      "activations/layer18_attention_weight_min": -29.19857406616211,
      "activations/layer19_attention_weight_max": 35.56980895996094,
      "activations/layer19_attention_weight_min": -32.74508285522461,
      "activations/layer1_attention_weight_max": 15.953889846801758,
      "activations/layer1_attention_weight_min": -15.487639427185059,
      "activations/layer20_attention_weight_max": 29.129587173461914,
      "activations/layer20_attention_weight_min": -26.73440170288086,
      "activations/layer21_attention_weight_max": 29.37939453125,
      "activations/layer21_attention_weight_min": -25.897878646850586,
      "activations/layer22_attention_weight_max": 45.499794006347656,
      "activations/layer22_attention_weight_min": -32.89744186401367,
      "activations/layer23_attention_weight_max": 31.468212127685547,
      "activations/layer23_attention_weight_min": -26.120506286621094,
      "activations/layer2_attention_weight_max": 33.6201057434082,
      "activations/layer2_attention_weight_min": -34.6300163269043,
      "activations/layer3_attention_weight_max": 108.64041900634766,
      "activations/layer3_attention_weight_min": -110.25740051269531,
      "activations/layer4_attention_weight_max": 112.12049865722656,
      "activations/layer4_attention_weight_min": -114.09276580810547,
      "activations/layer5_attention_weight_max": 51.51021957397461,
      "activations/layer5_attention_weight_min": -62.243873596191406,
      "activations/layer6_attention_weight_max": 47.16187286376953,
      "activations/layer6_attention_weight_min": -48.522403717041016,
      "activations/layer7_attention_weight_max": 108.6352767944336,
      "activations/layer7_attention_weight_min": -107.7845458984375,
      "activations/layer8_attention_weight_max": 51.10834503173828,
      "activations/layer8_attention_weight_min": -51.41556167602539,
      "activations/layer9_attention_weight_max": 39.633846282958984,
      "activations/layer9_attention_weight_min": -44.3568229675293,
      "epoch": 23.07,
      "learning_rate": 1.2522727272727272e-06,
      "loss": 2.7024,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_loss": 2.658203125,
      "eval_runtime": 8.5178,
      "eval_samples_per_second": 504.119,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_openwebtext_loss": 2.658203125,
      "eval_openwebtext_ppl": 14.2706235200274,
      "eval_openwebtext_runtime": 8.5178,
      "eval_openwebtext_samples_per_second": 504.119,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_wikitext_loss": 2.892578125,
      "eval_wikitext_ppl": 18.039758452611142,
      "eval_wikitext_runtime": 1.9987,
      "eval_wikitext_samples_per_second": 228.151,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_lambada_loss": 2.2890625,
      "eval_lambada_ppl": 9.865684265313508,
      "eval_lambada_runtime": 9.5809,
      "eval_lambada_samples_per_second": 508.196,
      "step": 397000
    },
    {
      "activations/layer0_attention_weight_max": 18.641382217407227,
      "activations/layer0_attention_weight_min": -13.188155174255371,
      "activations/layer10_attention_weight_max": 36.14746856689453,
      "activations/layer10_attention_weight_min": -37.71022415161133,
      "activations/layer11_attention_weight_max": 36.6215705871582,
      "activations/layer11_attention_weight_min": -34.805030822753906,
      "activations/layer12_attention_weight_max": 29.20345687866211,
      "activations/layer12_attention_weight_min": -28.18803596496582,
      "activations/layer13_attention_weight_max": 42.647640228271484,
      "activations/layer13_attention_weight_min": -38.72197723388672,
      "activations/layer14_attention_weight_max": 43.702720642089844,
      "activations/layer14_attention_weight_min": -39.424068450927734,
      "activations/layer15_attention_weight_max": 43.39447784423828,
      "activations/layer15_attention_weight_min": -39.19028854370117,
      "activations/layer16_attention_weight_max": 32.77233123779297,
      "activations/layer16_attention_weight_min": -28.161033630371094,
      "activations/layer17_attention_weight_max": 31.93393325805664,
      "activations/layer17_attention_weight_min": -25.732086181640625,
      "activations/layer18_attention_weight_max": 33.0806884765625,
      "activations/layer18_attention_weight_min": -26.5312557220459,
      "activations/layer19_attention_weight_max": 34.86256408691406,
      "activations/layer19_attention_weight_min": -31.413175582885742,
      "activations/layer1_attention_weight_max": 16.00322914123535,
      "activations/layer1_attention_weight_min": -15.067643165588379,
      "activations/layer20_attention_weight_max": 28.84432601928711,
      "activations/layer20_attention_weight_min": -24.210674285888672,
      "activations/layer21_attention_weight_max": 28.41281509399414,
      "activations/layer21_attention_weight_min": -23.014516830444336,
      "activations/layer22_attention_weight_max": 44.47954559326172,
      "activations/layer22_attention_weight_min": -29.97103500366211,
      "activations/layer23_attention_weight_max": 32.91236114501953,
      "activations/layer23_attention_weight_min": -24.095626831054688,
      "activations/layer2_attention_weight_max": 34.84852600097656,
      "activations/layer2_attention_weight_min": -34.75239181518555,
      "activations/layer3_attention_weight_max": 110.89228057861328,
      "activations/layer3_attention_weight_min": -114.06936645507812,
      "activations/layer4_attention_weight_max": 113.32794189453125,
      "activations/layer4_attention_weight_min": -114.98690032958984,
      "activations/layer5_attention_weight_max": 50.859039306640625,
      "activations/layer5_attention_weight_min": -64.14173889160156,
      "activations/layer6_attention_weight_max": 47.89425277709961,
      "activations/layer6_attention_weight_min": -50.28726577758789,
      "activations/layer7_attention_weight_max": 99.03748321533203,
      "activations/layer7_attention_weight_min": -103.9696044921875,
      "activations/layer8_attention_weight_max": 43.53345489501953,
      "activations/layer8_attention_weight_min": -48.58623123168945,
      "activations/layer9_attention_weight_max": 34.35573959350586,
      "activations/layer9_attention_weight_min": -35.637386322021484,
      "epoch": 23.07,
      "learning_rate": 1.2333333333333333e-06,
      "loss": 2.6977,
      "step": 397050
    },
    {
      "activations/layer0_attention_weight_max": 17.050331115722656,
      "activations/layer0_attention_weight_min": -15.669601440429688,
      "activations/layer10_attention_weight_max": 36.0723876953125,
      "activations/layer10_attention_weight_min": -37.05949783325195,
      "activations/layer11_attention_weight_max": 36.08291244506836,
      "activations/layer11_attention_weight_min": -37.98649597167969,
      "activations/layer12_attention_weight_max": 27.34271812438965,
      "activations/layer12_attention_weight_min": -28.225282669067383,
      "activations/layer13_attention_weight_max": 38.03852081298828,
      "activations/layer13_attention_weight_min": -38.04795455932617,
      "activations/layer14_attention_weight_max": 40.24359893798828,
      "activations/layer14_attention_weight_min": -39.182884216308594,
      "activations/layer15_attention_weight_max": 39.56071472167969,
      "activations/layer15_attention_weight_min": -37.14403533935547,
      "activations/layer16_attention_weight_max": 30.55063819885254,
      "activations/layer16_attention_weight_min": -28.565906524658203,
      "activations/layer17_attention_weight_max": 29.353132247924805,
      "activations/layer17_attention_weight_min": -26.011720657348633,
      "activations/layer18_attention_weight_max": 32.00040817260742,
      "activations/layer18_attention_weight_min": -27.182828903198242,
      "activations/layer19_attention_weight_max": 35.53857421875,
      "activations/layer19_attention_weight_min": -35.554893493652344,
      "activations/layer1_attention_weight_max": 15.863903999328613,
      "activations/layer1_attention_weight_min": -15.484896659851074,
      "activations/layer20_attention_weight_max": 30.786874771118164,
      "activations/layer20_attention_weight_min": -27.503284454345703,
      "activations/layer21_attention_weight_max": 30.377580642700195,
      "activations/layer21_attention_weight_min": -24.838775634765625,
      "activations/layer22_attention_weight_max": 44.26971435546875,
      "activations/layer22_attention_weight_min": -31.329851150512695,
      "activations/layer23_attention_weight_max": 31.31786346435547,
      "activations/layer23_attention_weight_min": -26.302734375,
      "activations/layer2_attention_weight_max": 33.8308219909668,
      "activations/layer2_attention_weight_min": -33.396514892578125,
      "activations/layer3_attention_weight_max": 109.96174621582031,
      "activations/layer3_attention_weight_min": -110.8400650024414,
      "activations/layer4_attention_weight_max": 110.3201904296875,
      "activations/layer4_attention_weight_min": -112.64981842041016,
      "activations/layer5_attention_weight_max": 53.084197998046875,
      "activations/layer5_attention_weight_min": -60.97361373901367,
      "activations/layer6_attention_weight_max": 47.00831604003906,
      "activations/layer6_attention_weight_min": -49.84416580200195,
      "activations/layer7_attention_weight_max": 101.4854507446289,
      "activations/layer7_attention_weight_min": -102.94062805175781,
      "activations/layer8_attention_weight_max": 46.10060501098633,
      "activations/layer8_attention_weight_min": -48.118282318115234,
      "activations/layer9_attention_weight_max": 35.74422836303711,
      "activations/layer9_attention_weight_min": -36.748043060302734,
      "epoch": 23.07,
      "learning_rate": 1.2143939393939393e-06,
      "loss": 2.6962,
      "step": 397100
    },
    {
      "activations/layer0_attention_weight_max": 16.02501106262207,
      "activations/layer0_attention_weight_min": -14.56159496307373,
      "activations/layer10_attention_weight_max": 34.758148193359375,
      "activations/layer10_attention_weight_min": -35.76338195800781,
      "activations/layer11_attention_weight_max": 35.30180358886719,
      "activations/layer11_attention_weight_min": -36.42024230957031,
      "activations/layer12_attention_weight_max": 27.995161056518555,
      "activations/layer12_attention_weight_min": -26.795761108398438,
      "activations/layer13_attention_weight_max": 40.91569519042969,
      "activations/layer13_attention_weight_min": -37.29568099975586,
      "activations/layer14_attention_weight_max": 44.233978271484375,
      "activations/layer14_attention_weight_min": -40.12254333496094,
      "activations/layer15_attention_weight_max": 39.34019470214844,
      "activations/layer15_attention_weight_min": -37.36591720581055,
      "activations/layer16_attention_weight_max": 28.89243507385254,
      "activations/layer16_attention_weight_min": -28.847166061401367,
      "activations/layer17_attention_weight_max": 31.402099609375,
      "activations/layer17_attention_weight_min": -28.78440284729004,
      "activations/layer18_attention_weight_max": 34.71881103515625,
      "activations/layer18_attention_weight_min": -27.13365364074707,
      "activations/layer19_attention_weight_max": 35.15407943725586,
      "activations/layer19_attention_weight_min": -35.571861267089844,
      "activations/layer1_attention_weight_max": 15.698406219482422,
      "activations/layer1_attention_weight_min": -14.908143997192383,
      "activations/layer20_attention_weight_max": 27.897464752197266,
      "activations/layer20_attention_weight_min": -28.932964324951172,
      "activations/layer21_attention_weight_max": 30.316190719604492,
      "activations/layer21_attention_weight_min": -28.1956844329834,
      "activations/layer22_attention_weight_max": 42.433509826660156,
      "activations/layer22_attention_weight_min": -32.685359954833984,
      "activations/layer23_attention_weight_max": 32.48082733154297,
      "activations/layer23_attention_weight_min": -26.00192642211914,
      "activations/layer2_attention_weight_max": 34.06316375732422,
      "activations/layer2_attention_weight_min": -33.084808349609375,
      "activations/layer3_attention_weight_max": 109.50163269042969,
      "activations/layer3_attention_weight_min": -110.6082992553711,
      "activations/layer4_attention_weight_max": 112.73616790771484,
      "activations/layer4_attention_weight_min": -115.45613098144531,
      "activations/layer5_attention_weight_max": 55.66289520263672,
      "activations/layer5_attention_weight_min": -67.61639404296875,
      "activations/layer6_attention_weight_max": 47.20017623901367,
      "activations/layer6_attention_weight_min": -51.26600646972656,
      "activations/layer7_attention_weight_max": 99.7330093383789,
      "activations/layer7_attention_weight_min": -106.0958251953125,
      "activations/layer8_attention_weight_max": 45.162147521972656,
      "activations/layer8_attention_weight_min": -46.23112106323242,
      "activations/layer9_attention_weight_max": 33.37175369262695,
      "activations/layer9_attention_weight_min": -34.88267135620117,
      "epoch": 23.08,
      "learning_rate": 1.1954545454545454e-06,
      "loss": 2.6897,
      "step": 397150
    },
    {
      "activations/layer0_attention_weight_max": 16.43295669555664,
      "activations/layer0_attention_weight_min": -14.835589408874512,
      "activations/layer10_attention_weight_max": 36.16151428222656,
      "activations/layer10_attention_weight_min": -36.8158073425293,
      "activations/layer11_attention_weight_max": 37.67223358154297,
      "activations/layer11_attention_weight_min": -36.262123107910156,
      "activations/layer12_attention_weight_max": 28.169513702392578,
      "activations/layer12_attention_weight_min": -28.96605110168457,
      "activations/layer13_attention_weight_max": 40.88114547729492,
      "activations/layer13_attention_weight_min": -38.49488830566406,
      "activations/layer14_attention_weight_max": 44.720741271972656,
      "activations/layer14_attention_weight_min": -39.76864242553711,
      "activations/layer15_attention_weight_max": 40.26192855834961,
      "activations/layer15_attention_weight_min": -38.19428634643555,
      "activations/layer16_attention_weight_max": 29.96121597290039,
      "activations/layer16_attention_weight_min": -27.97455596923828,
      "activations/layer17_attention_weight_max": 28.871904373168945,
      "activations/layer17_attention_weight_min": -25.22855567932129,
      "activations/layer18_attention_weight_max": 34.36765670776367,
      "activations/layer18_attention_weight_min": -26.63207244873047,
      "activations/layer19_attention_weight_max": 36.99441146850586,
      "activations/layer19_attention_weight_min": -34.502811431884766,
      "activations/layer1_attention_weight_max": 16.018383026123047,
      "activations/layer1_attention_weight_min": -14.65123462677002,
      "activations/layer20_attention_weight_max": 29.076515197753906,
      "activations/layer20_attention_weight_min": -26.978254318237305,
      "activations/layer21_attention_weight_max": 29.58868980407715,
      "activations/layer21_attention_weight_min": -24.531387329101562,
      "activations/layer22_attention_weight_max": 43.67192077636719,
      "activations/layer22_attention_weight_min": -31.80922508239746,
      "activations/layer23_attention_weight_max": 31.4338436126709,
      "activations/layer23_attention_weight_min": -25.095441818237305,
      "activations/layer2_attention_weight_max": 35.01518249511719,
      "activations/layer2_attention_weight_min": -34.25739669799805,
      "activations/layer3_attention_weight_max": 109.02132415771484,
      "activations/layer3_attention_weight_min": -110.06881713867188,
      "activations/layer4_attention_weight_max": 112.29432678222656,
      "activations/layer4_attention_weight_min": -114.7841796875,
      "activations/layer5_attention_weight_max": 49.56653594970703,
      "activations/layer5_attention_weight_min": -64.37714385986328,
      "activations/layer6_attention_weight_max": 46.11861038208008,
      "activations/layer6_attention_weight_min": -50.409847259521484,
      "activations/layer7_attention_weight_max": 100.6690673828125,
      "activations/layer7_attention_weight_min": -101.2846450805664,
      "activations/layer8_attention_weight_max": 45.50233459472656,
      "activations/layer8_attention_weight_min": -45.703792572021484,
      "activations/layer9_attention_weight_max": 35.15766906738281,
      "activations/layer9_attention_weight_min": -36.43050765991211,
      "epoch": 23.08,
      "learning_rate": 1.1765151515151513e-06,
      "loss": 2.6981,
      "step": 397200
    },
    {
      "activations/layer0_attention_weight_max": 16.865053176879883,
      "activations/layer0_attention_weight_min": -13.739378929138184,
      "activations/layer10_attention_weight_max": 34.60033416748047,
      "activations/layer10_attention_weight_min": -35.61076736450195,
      "activations/layer11_attention_weight_max": 36.26921844482422,
      "activations/layer11_attention_weight_min": -34.960784912109375,
      "activations/layer12_attention_weight_max": 26.798885345458984,
      "activations/layer12_attention_weight_min": -29.132225036621094,
      "activations/layer13_attention_weight_max": 38.29932403564453,
      "activations/layer13_attention_weight_min": -37.03481674194336,
      "activations/layer14_attention_weight_max": 39.48085403442383,
      "activations/layer14_attention_weight_min": -40.22309112548828,
      "activations/layer15_attention_weight_max": 40.81284713745117,
      "activations/layer15_attention_weight_min": -38.80223846435547,
      "activations/layer16_attention_weight_max": 28.965206146240234,
      "activations/layer16_attention_weight_min": -29.37623405456543,
      "activations/layer17_attention_weight_max": 28.92296028137207,
      "activations/layer17_attention_weight_min": -29.13359260559082,
      "activations/layer18_attention_weight_max": 32.07210922241211,
      "activations/layer18_attention_weight_min": -29.1221981048584,
      "activations/layer19_attention_weight_max": 34.21189498901367,
      "activations/layer19_attention_weight_min": -35.78171920776367,
      "activations/layer1_attention_weight_max": 15.262149810791016,
      "activations/layer1_attention_weight_min": -14.407544136047363,
      "activations/layer20_attention_weight_max": 26.6453857421875,
      "activations/layer20_attention_weight_min": -27.748199462890625,
      "activations/layer21_attention_weight_max": 26.786617279052734,
      "activations/layer21_attention_weight_min": -27.22404670715332,
      "activations/layer22_attention_weight_max": 37.743289947509766,
      "activations/layer22_attention_weight_min": -35.9235725402832,
      "activations/layer23_attention_weight_max": 29.49929428100586,
      "activations/layer23_attention_weight_min": -27.375545501708984,
      "activations/layer2_attention_weight_max": 33.98259735107422,
      "activations/layer2_attention_weight_min": -33.598663330078125,
      "activations/layer3_attention_weight_max": 105.0718994140625,
      "activations/layer3_attention_weight_min": -106.99369812011719,
      "activations/layer4_attention_weight_max": 111.49208068847656,
      "activations/layer4_attention_weight_min": -113.32938385009766,
      "activations/layer5_attention_weight_max": 50.236778259277344,
      "activations/layer5_attention_weight_min": -61.46955108642578,
      "activations/layer6_attention_weight_max": 45.1828498840332,
      "activations/layer6_attention_weight_min": -49.29277038574219,
      "activations/layer7_attention_weight_max": 94.95696258544922,
      "activations/layer7_attention_weight_min": -101.57650756835938,
      "activations/layer8_attention_weight_max": 42.837459564208984,
      "activations/layer8_attention_weight_min": -47.249244689941406,
      "activations/layer9_attention_weight_max": 32.60837936401367,
      "activations/layer9_attention_weight_min": -36.240596771240234,
      "epoch": 23.08,
      "learning_rate": 1.1575757575757575e-06,
      "loss": 2.6825,
      "step": 397250
    },
    {
      "activations/layer0_attention_weight_max": 16.86398696899414,
      "activations/layer0_attention_weight_min": -14.768497467041016,
      "activations/layer10_attention_weight_max": 38.9343376159668,
      "activations/layer10_attention_weight_min": -38.52227783203125,
      "activations/layer11_attention_weight_max": 37.835350036621094,
      "activations/layer11_attention_weight_min": -39.24652099609375,
      "activations/layer12_attention_weight_max": 32.8570442199707,
      "activations/layer12_attention_weight_min": -30.844181060791016,
      "activations/layer13_attention_weight_max": 48.01240921020508,
      "activations/layer13_attention_weight_min": -39.848567962646484,
      "activations/layer14_attention_weight_max": 52.33340835571289,
      "activations/layer14_attention_weight_min": -43.75761032104492,
      "activations/layer15_attention_weight_max": 55.740623474121094,
      "activations/layer15_attention_weight_min": -40.30664825439453,
      "activations/layer16_attention_weight_max": 34.904052734375,
      "activations/layer16_attention_weight_min": -29.950891494750977,
      "activations/layer17_attention_weight_max": 34.93008041381836,
      "activations/layer17_attention_weight_min": -30.586286544799805,
      "activations/layer18_attention_weight_max": 37.66573715209961,
      "activations/layer18_attention_weight_min": -30.65367317199707,
      "activations/layer19_attention_weight_max": 39.830745697021484,
      "activations/layer19_attention_weight_min": -35.87409591674805,
      "activations/layer1_attention_weight_max": 15.755546569824219,
      "activations/layer1_attention_weight_min": -16.058391571044922,
      "activations/layer20_attention_weight_max": 33.905433654785156,
      "activations/layer20_attention_weight_min": -27.65892791748047,
      "activations/layer21_attention_weight_max": 34.6240119934082,
      "activations/layer21_attention_weight_min": -26.89617919921875,
      "activations/layer22_attention_weight_max": 50.42660140991211,
      "activations/layer22_attention_weight_min": -33.573341369628906,
      "activations/layer23_attention_weight_max": 37.51968002319336,
      "activations/layer23_attention_weight_min": -28.798965454101562,
      "activations/layer2_attention_weight_max": 34.69550323486328,
      "activations/layer2_attention_weight_min": -33.98950958251953,
      "activations/layer3_attention_weight_max": 111.71505737304688,
      "activations/layer3_attention_weight_min": -113.76188659667969,
      "activations/layer4_attention_weight_max": 116.78743743896484,
      "activations/layer4_attention_weight_min": -120.58868408203125,
      "activations/layer5_attention_weight_max": 55.65958786010742,
      "activations/layer5_attention_weight_min": -66.4103775024414,
      "activations/layer6_attention_weight_max": 48.397708892822266,
      "activations/layer6_attention_weight_min": -49.65493392944336,
      "activations/layer7_attention_weight_max": 103.69866180419922,
      "activations/layer7_attention_weight_min": -106.55953979492188,
      "activations/layer8_attention_weight_max": 49.50221633911133,
      "activations/layer8_attention_weight_min": -48.39753341674805,
      "activations/layer9_attention_weight_max": 36.512107849121094,
      "activations/layer9_attention_weight_min": -37.99977111816406,
      "epoch": 23.09,
      "learning_rate": 1.1386363636363634e-06,
      "loss": 2.6819,
      "step": 397300
    },
    {
      "activations/layer0_attention_weight_max": 16.175193786621094,
      "activations/layer0_attention_weight_min": -14.876442909240723,
      "activations/layer10_attention_weight_max": 39.28304672241211,
      "activations/layer10_attention_weight_min": -38.51148223876953,
      "activations/layer11_attention_weight_max": 35.80694580078125,
      "activations/layer11_attention_weight_min": -36.4270133972168,
      "activations/layer12_attention_weight_max": 28.237483978271484,
      "activations/layer12_attention_weight_min": -28.643571853637695,
      "activations/layer13_attention_weight_max": 43.61380386352539,
      "activations/layer13_attention_weight_min": -36.81793212890625,
      "activations/layer14_attention_weight_max": 44.45890808105469,
      "activations/layer14_attention_weight_min": -39.91997528076172,
      "activations/layer15_attention_weight_max": 41.185482025146484,
      "activations/layer15_attention_weight_min": -38.5763053894043,
      "activations/layer16_attention_weight_max": 30.334245681762695,
      "activations/layer16_attention_weight_min": -28.65281105041504,
      "activations/layer17_attention_weight_max": 33.06081008911133,
      "activations/layer17_attention_weight_min": -27.115453720092773,
      "activations/layer18_attention_weight_max": 35.02706527709961,
      "activations/layer18_attention_weight_min": -28.539236068725586,
      "activations/layer19_attention_weight_max": 36.31037139892578,
      "activations/layer19_attention_weight_min": -33.849422454833984,
      "activations/layer1_attention_weight_max": 16.144914627075195,
      "activations/layer1_attention_weight_min": -15.140547752380371,
      "activations/layer20_attention_weight_max": 29.69548988342285,
      "activations/layer20_attention_weight_min": -27.670846939086914,
      "activations/layer21_attention_weight_max": 29.420482635498047,
      "activations/layer21_attention_weight_min": -27.579729080200195,
      "activations/layer22_attention_weight_max": 44.68052291870117,
      "activations/layer22_attention_weight_min": -34.155521392822266,
      "activations/layer23_attention_weight_max": 35.08991241455078,
      "activations/layer23_attention_weight_min": -27.50152587890625,
      "activations/layer2_attention_weight_max": 35.017112731933594,
      "activations/layer2_attention_weight_min": -34.483482360839844,
      "activations/layer3_attention_weight_max": 110.03324127197266,
      "activations/layer3_attention_weight_min": -108.87167358398438,
      "activations/layer4_attention_weight_max": 113.18080139160156,
      "activations/layer4_attention_weight_min": -113.5525894165039,
      "activations/layer5_attention_weight_max": 52.24345397949219,
      "activations/layer5_attention_weight_min": -61.501182556152344,
      "activations/layer6_attention_weight_max": 47.40830993652344,
      "activations/layer6_attention_weight_min": -49.828712463378906,
      "activations/layer7_attention_weight_max": 98.69108581542969,
      "activations/layer7_attention_weight_min": -105.738525390625,
      "activations/layer8_attention_weight_max": 46.190853118896484,
      "activations/layer8_attention_weight_min": -49.887489318847656,
      "activations/layer9_attention_weight_max": 34.712528228759766,
      "activations/layer9_attention_weight_min": -37.91331481933594,
      "epoch": 23.09,
      "learning_rate": 1.1196969696969696e-06,
      "loss": 2.6895,
      "step": 397350
    },
    {
      "activations/layer0_attention_weight_max": 16.919343948364258,
      "activations/layer0_attention_weight_min": -14.307933807373047,
      "activations/layer10_attention_weight_max": 37.18180847167969,
      "activations/layer10_attention_weight_min": -37.34987258911133,
      "activations/layer11_attention_weight_max": 38.18440628051758,
      "activations/layer11_attention_weight_min": -40.83179473876953,
      "activations/layer12_attention_weight_max": 27.498825073242188,
      "activations/layer12_attention_weight_min": -29.35649299621582,
      "activations/layer13_attention_weight_max": 39.64134216308594,
      "activations/layer13_attention_weight_min": -37.67776107788086,
      "activations/layer14_attention_weight_max": 43.540306091308594,
      "activations/layer14_attention_weight_min": -40.659751892089844,
      "activations/layer15_attention_weight_max": 40.42365264892578,
      "activations/layer15_attention_weight_min": -39.02875518798828,
      "activations/layer16_attention_weight_max": 31.02634620666504,
      "activations/layer16_attention_weight_min": -28.818567276000977,
      "activations/layer17_attention_weight_max": 29.771020889282227,
      "activations/layer17_attention_weight_min": -26.942270278930664,
      "activations/layer18_attention_weight_max": 34.470760345458984,
      "activations/layer18_attention_weight_min": -26.38235855102539,
      "activations/layer19_attention_weight_max": 34.12775421142578,
      "activations/layer19_attention_weight_min": -31.975875854492188,
      "activations/layer1_attention_weight_max": 15.589988708496094,
      "activations/layer1_attention_weight_min": -16.10094451904297,
      "activations/layer20_attention_weight_max": 27.70013999938965,
      "activations/layer20_attention_weight_min": -24.74776268005371,
      "activations/layer21_attention_weight_max": 27.663652420043945,
      "activations/layer21_attention_weight_min": -24.241342544555664,
      "activations/layer22_attention_weight_max": 40.453514099121094,
      "activations/layer22_attention_weight_min": -32.14984130859375,
      "activations/layer23_attention_weight_max": 29.715621948242188,
      "activations/layer23_attention_weight_min": -24.598846435546875,
      "activations/layer2_attention_weight_max": 35.18706512451172,
      "activations/layer2_attention_weight_min": -33.68225860595703,
      "activations/layer3_attention_weight_max": 109.73490905761719,
      "activations/layer3_attention_weight_min": -113.89125061035156,
      "activations/layer4_attention_weight_max": 114.04320526123047,
      "activations/layer4_attention_weight_min": -118.02877044677734,
      "activations/layer5_attention_weight_max": 53.823116302490234,
      "activations/layer5_attention_weight_min": -64.48554992675781,
      "activations/layer6_attention_weight_max": 47.32602310180664,
      "activations/layer6_attention_weight_min": -50.511146545410156,
      "activations/layer7_attention_weight_max": 102.58948516845703,
      "activations/layer7_attention_weight_min": -106.36245727539062,
      "activations/layer8_attention_weight_max": 43.61552810668945,
      "activations/layer8_attention_weight_min": -48.37640380859375,
      "activations/layer9_attention_weight_max": 34.963008880615234,
      "activations/layer9_attention_weight_min": -38.95832443237305,
      "epoch": 23.09,
      "learning_rate": 1.1007575757575757e-06,
      "loss": 2.6899,
      "step": 397400
    },
    {
      "activations/layer0_attention_weight_max": 17.83528709411621,
      "activations/layer0_attention_weight_min": -13.35184097290039,
      "activations/layer10_attention_weight_max": 38.275508880615234,
      "activations/layer10_attention_weight_min": -40.02527618408203,
      "activations/layer11_attention_weight_max": 36.144020080566406,
      "activations/layer11_attention_weight_min": -37.818870544433594,
      "activations/layer12_attention_weight_max": 27.612876892089844,
      "activations/layer12_attention_weight_min": -26.408409118652344,
      "activations/layer13_attention_weight_max": 42.40925216674805,
      "activations/layer13_attention_weight_min": -35.63828659057617,
      "activations/layer14_attention_weight_max": 47.65290069580078,
      "activations/layer14_attention_weight_min": -37.64116287231445,
      "activations/layer15_attention_weight_max": 42.6840934753418,
      "activations/layer15_attention_weight_min": -36.26088333129883,
      "activations/layer16_attention_weight_max": 32.32794189453125,
      "activations/layer16_attention_weight_min": -28.370037078857422,
      "activations/layer17_attention_weight_max": 32.5752067565918,
      "activations/layer17_attention_weight_min": -26.38442611694336,
      "activations/layer18_attention_weight_max": 32.24260330200195,
      "activations/layer18_attention_weight_min": -28.94599723815918,
      "activations/layer19_attention_weight_max": 36.11784744262695,
      "activations/layer19_attention_weight_min": -32.27096176147461,
      "activations/layer1_attention_weight_max": 16.664514541625977,
      "activations/layer1_attention_weight_min": -15.854411125183105,
      "activations/layer20_attention_weight_max": 29.146133422851562,
      "activations/layer20_attention_weight_min": -26.693790435791016,
      "activations/layer21_attention_weight_max": 31.083297729492188,
      "activations/layer21_attention_weight_min": -25.297821044921875,
      "activations/layer22_attention_weight_max": 44.788124084472656,
      "activations/layer22_attention_weight_min": -33.72020721435547,
      "activations/layer23_attention_weight_max": 33.56022262573242,
      "activations/layer23_attention_weight_min": -26.759550094604492,
      "activations/layer2_attention_weight_max": 34.87549591064453,
      "activations/layer2_attention_weight_min": -34.269073486328125,
      "activations/layer3_attention_weight_max": 110.62538146972656,
      "activations/layer3_attention_weight_min": -109.931884765625,
      "activations/layer4_attention_weight_max": 112.70368957519531,
      "activations/layer4_attention_weight_min": -117.31961822509766,
      "activations/layer5_attention_weight_max": 52.27935791015625,
      "activations/layer5_attention_weight_min": -66.64341735839844,
      "activations/layer6_attention_weight_max": 48.66974639892578,
      "activations/layer6_attention_weight_min": -50.9085578918457,
      "activations/layer7_attention_weight_max": 97.43640899658203,
      "activations/layer7_attention_weight_min": -111.30692291259766,
      "activations/layer8_attention_weight_max": 47.63852310180664,
      "activations/layer8_attention_weight_min": -53.03691101074219,
      "activations/layer9_attention_weight_max": 36.548500061035156,
      "activations/layer9_attention_weight_min": -41.45180892944336,
      "epoch": 23.09,
      "learning_rate": 1.0818181818181817e-06,
      "loss": 2.6912,
      "step": 397450
    },
    {
      "activations/layer0_attention_weight_max": 17.392118453979492,
      "activations/layer0_attention_weight_min": -14.470989227294922,
      "activations/layer10_attention_weight_max": 36.1005973815918,
      "activations/layer10_attention_weight_min": -37.16297149658203,
      "activations/layer11_attention_weight_max": 34.90226745605469,
      "activations/layer11_attention_weight_min": -37.74169921875,
      "activations/layer12_attention_weight_max": 27.892501831054688,
      "activations/layer12_attention_weight_min": -27.396869659423828,
      "activations/layer13_attention_weight_max": 41.43560791015625,
      "activations/layer13_attention_weight_min": -37.31815719604492,
      "activations/layer14_attention_weight_max": 43.16168212890625,
      "activations/layer14_attention_weight_min": -38.82609939575195,
      "activations/layer15_attention_weight_max": 42.236263275146484,
      "activations/layer15_attention_weight_min": -41.331764221191406,
      "activations/layer16_attention_weight_max": 30.466548919677734,
      "activations/layer16_attention_weight_min": -30.42179298400879,
      "activations/layer17_attention_weight_max": 31.396663665771484,
      "activations/layer17_attention_weight_min": -28.99858283996582,
      "activations/layer18_attention_weight_max": 31.701522827148438,
      "activations/layer18_attention_weight_min": -30.048105239868164,
      "activations/layer19_attention_weight_max": 34.06748962402344,
      "activations/layer19_attention_weight_min": -34.40095138549805,
      "activations/layer1_attention_weight_max": 15.23055362701416,
      "activations/layer1_attention_weight_min": -15.869587898254395,
      "activations/layer20_attention_weight_max": 30.024444580078125,
      "activations/layer20_attention_weight_min": -26.589448928833008,
      "activations/layer21_attention_weight_max": 31.506210327148438,
      "activations/layer21_attention_weight_min": -25.441970825195312,
      "activations/layer22_attention_weight_max": 40.55522155761719,
      "activations/layer22_attention_weight_min": -32.199607849121094,
      "activations/layer23_attention_weight_max": 31.660621643066406,
      "activations/layer23_attention_weight_min": -24.85963249206543,
      "activations/layer2_attention_weight_max": 35.44050979614258,
      "activations/layer2_attention_weight_min": -34.44464874267578,
      "activations/layer3_attention_weight_max": 109.3900375366211,
      "activations/layer3_attention_weight_min": -110.90074920654297,
      "activations/layer4_attention_weight_max": 108.32305908203125,
      "activations/layer4_attention_weight_min": -119.22396087646484,
      "activations/layer5_attention_weight_max": 51.71953582763672,
      "activations/layer5_attention_weight_min": -63.440223693847656,
      "activations/layer6_attention_weight_max": 46.471309661865234,
      "activations/layer6_attention_weight_min": -48.985774993896484,
      "activations/layer7_attention_weight_max": 98.05534362792969,
      "activations/layer7_attention_weight_min": -102.7400894165039,
      "activations/layer8_attention_weight_max": 44.05751419067383,
      "activations/layer8_attention_weight_min": -49.587623596191406,
      "activations/layer9_attention_weight_max": 33.59637451171875,
      "activations/layer9_attention_weight_min": -37.098846435546875,
      "epoch": 23.1,
      "learning_rate": 1.0628787878787878e-06,
      "loss": 2.6945,
      "step": 397500
    },
    {
      "activations/layer0_attention_weight_max": 16.873125076293945,
      "activations/layer0_attention_weight_min": -14.236895561218262,
      "activations/layer10_attention_weight_max": 42.24519348144531,
      "activations/layer10_attention_weight_min": -39.63606643676758,
      "activations/layer11_attention_weight_max": 39.999324798583984,
      "activations/layer11_attention_weight_min": -39.17359924316406,
      "activations/layer12_attention_weight_max": 31.150720596313477,
      "activations/layer12_attention_weight_min": -29.22883415222168,
      "activations/layer13_attention_weight_max": 44.89241027832031,
      "activations/layer13_attention_weight_min": -38.865257263183594,
      "activations/layer14_attention_weight_max": 48.3842658996582,
      "activations/layer14_attention_weight_min": -40.22072219848633,
      "activations/layer15_attention_weight_max": 41.90083694458008,
      "activations/layer15_attention_weight_min": -37.09859848022461,
      "activations/layer16_attention_weight_max": 31.921735763549805,
      "activations/layer16_attention_weight_min": -29.082475662231445,
      "activations/layer17_attention_weight_max": 32.77460861206055,
      "activations/layer17_attention_weight_min": -26.98276138305664,
      "activations/layer18_attention_weight_max": 34.09893798828125,
      "activations/layer18_attention_weight_min": -25.823810577392578,
      "activations/layer19_attention_weight_max": 34.441200256347656,
      "activations/layer19_attention_weight_min": -32.74069595336914,
      "activations/layer1_attention_weight_max": 15.14452075958252,
      "activations/layer1_attention_weight_min": -14.873783111572266,
      "activations/layer20_attention_weight_max": 30.11345863342285,
      "activations/layer20_attention_weight_min": -25.862709045410156,
      "activations/layer21_attention_weight_max": 31.039182662963867,
      "activations/layer21_attention_weight_min": -25.117876052856445,
      "activations/layer22_attention_weight_max": 43.31294631958008,
      "activations/layer22_attention_weight_min": -36.07693862915039,
      "activations/layer23_attention_weight_max": 31.75043487548828,
      "activations/layer23_attention_weight_min": -25.40167236328125,
      "activations/layer2_attention_weight_max": 35.695621490478516,
      "activations/layer2_attention_weight_min": -34.27154541015625,
      "activations/layer3_attention_weight_max": 110.1866683959961,
      "activations/layer3_attention_weight_min": -113.84007263183594,
      "activations/layer4_attention_weight_max": 113.63065338134766,
      "activations/layer4_attention_weight_min": -119.38946533203125,
      "activations/layer5_attention_weight_max": 55.53486251831055,
      "activations/layer5_attention_weight_min": -63.381561279296875,
      "activations/layer6_attention_weight_max": 48.634700775146484,
      "activations/layer6_attention_weight_min": -51.31319046020508,
      "activations/layer7_attention_weight_max": 111.29531860351562,
      "activations/layer7_attention_weight_min": -109.84286499023438,
      "activations/layer8_attention_weight_max": 50.15934753417969,
      "activations/layer8_attention_weight_min": -48.36955261230469,
      "activations/layer9_attention_weight_max": 38.49710464477539,
      "activations/layer9_attention_weight_min": -38.265869140625,
      "epoch": 23.1,
      "learning_rate": 1.043939393939394e-06,
      "loss": 2.7002,
      "step": 397550
    },
    {
      "activations/layer0_attention_weight_max": 16.24541473388672,
      "activations/layer0_attention_weight_min": -14.607589721679688,
      "activations/layer10_attention_weight_max": 37.749629974365234,
      "activations/layer10_attention_weight_min": -37.813602447509766,
      "activations/layer11_attention_weight_max": 36.32368469238281,
      "activations/layer11_attention_weight_min": -37.57936096191406,
      "activations/layer12_attention_weight_max": 30.77237319946289,
      "activations/layer12_attention_weight_min": -30.798974990844727,
      "activations/layer13_attention_weight_max": 42.706668853759766,
      "activations/layer13_attention_weight_min": -41.71055221557617,
      "activations/layer14_attention_weight_max": 46.662689208984375,
      "activations/layer14_attention_weight_min": -41.53644561767578,
      "activations/layer15_attention_weight_max": 45.91142272949219,
      "activations/layer15_attention_weight_min": -39.20241928100586,
      "activations/layer16_attention_weight_max": 34.81180953979492,
      "activations/layer16_attention_weight_min": -30.17218017578125,
      "activations/layer17_attention_weight_max": 34.0361442565918,
      "activations/layer17_attention_weight_min": -28.16868782043457,
      "activations/layer18_attention_weight_max": 35.55248260498047,
      "activations/layer18_attention_weight_min": -27.477182388305664,
      "activations/layer19_attention_weight_max": 43.422080993652344,
      "activations/layer19_attention_weight_min": -33.035030364990234,
      "activations/layer1_attention_weight_max": 16.087738037109375,
      "activations/layer1_attention_weight_min": -15.472237586975098,
      "activations/layer20_attention_weight_max": 33.50067901611328,
      "activations/layer20_attention_weight_min": -26.76141929626465,
      "activations/layer21_attention_weight_max": 34.53443145751953,
      "activations/layer21_attention_weight_min": -25.558719635009766,
      "activations/layer22_attention_weight_max": 48.81681442260742,
      "activations/layer22_attention_weight_min": -31.129596710205078,
      "activations/layer23_attention_weight_max": 38.249595642089844,
      "activations/layer23_attention_weight_min": -23.25293731689453,
      "activations/layer2_attention_weight_max": 34.4039306640625,
      "activations/layer2_attention_weight_min": -34.29591369628906,
      "activations/layer3_attention_weight_max": 108.30908966064453,
      "activations/layer3_attention_weight_min": -111.89994812011719,
      "activations/layer4_attention_weight_max": 110.74995422363281,
      "activations/layer4_attention_weight_min": -116.78893280029297,
      "activations/layer5_attention_weight_max": 51.949928283691406,
      "activations/layer5_attention_weight_min": -62.833984375,
      "activations/layer6_attention_weight_max": 49.58018493652344,
      "activations/layer6_attention_weight_min": -48.75648498535156,
      "activations/layer7_attention_weight_max": 99.34722137451172,
      "activations/layer7_attention_weight_min": -100.28549194335938,
      "activations/layer8_attention_weight_max": 44.47098922729492,
      "activations/layer8_attention_weight_min": -47.18324279785156,
      "activations/layer9_attention_weight_max": 34.90156936645508,
      "activations/layer9_attention_weight_min": -36.36624526977539,
      "epoch": 23.1,
      "learning_rate": 1.025e-06,
      "loss": 2.7075,
      "step": 397600
    },
    {
      "activations/layer0_attention_weight_max": 17.325815200805664,
      "activations/layer0_attention_weight_min": -15.664468765258789,
      "activations/layer10_attention_weight_max": 39.918033599853516,
      "activations/layer10_attention_weight_min": -37.13505554199219,
      "activations/layer11_attention_weight_max": 38.4774169921875,
      "activations/layer11_attention_weight_min": -36.35905456542969,
      "activations/layer12_attention_weight_max": 29.40107536315918,
      "activations/layer12_attention_weight_min": -29.10533905029297,
      "activations/layer13_attention_weight_max": 41.36408996582031,
      "activations/layer13_attention_weight_min": -38.19013214111328,
      "activations/layer14_attention_weight_max": 46.061038970947266,
      "activations/layer14_attention_weight_min": -42.3580322265625,
      "activations/layer15_attention_weight_max": 43.9494743347168,
      "activations/layer15_attention_weight_min": -39.899959564208984,
      "activations/layer16_attention_weight_max": 30.954023361206055,
      "activations/layer16_attention_weight_min": -29.364328384399414,
      "activations/layer17_attention_weight_max": 29.14694595336914,
      "activations/layer17_attention_weight_min": -27.66089630126953,
      "activations/layer18_attention_weight_max": 33.04922866821289,
      "activations/layer18_attention_weight_min": -27.908634185791016,
      "activations/layer19_attention_weight_max": 37.68909454345703,
      "activations/layer19_attention_weight_min": -36.363975524902344,
      "activations/layer1_attention_weight_max": 16.245555877685547,
      "activations/layer1_attention_weight_min": -15.55302619934082,
      "activations/layer20_attention_weight_max": 30.92205047607422,
      "activations/layer20_attention_weight_min": -28.42482566833496,
      "activations/layer21_attention_weight_max": 30.712900161743164,
      "activations/layer21_attention_weight_min": -25.522642135620117,
      "activations/layer22_attention_weight_max": 45.61653518676758,
      "activations/layer22_attention_weight_min": -34.2495002746582,
      "activations/layer23_attention_weight_max": 34.52439880371094,
      "activations/layer23_attention_weight_min": -28.065170288085938,
      "activations/layer2_attention_weight_max": 34.805946350097656,
      "activations/layer2_attention_weight_min": -33.5465087890625,
      "activations/layer3_attention_weight_max": 107.48710632324219,
      "activations/layer3_attention_weight_min": -110.23294067382812,
      "activations/layer4_attention_weight_max": 109.69487762451172,
      "activations/layer4_attention_weight_min": -117.6632308959961,
      "activations/layer5_attention_weight_max": 52.82006072998047,
      "activations/layer5_attention_weight_min": -62.180442810058594,
      "activations/layer6_attention_weight_max": 45.24338912963867,
      "activations/layer6_attention_weight_min": -49.657718658447266,
      "activations/layer7_attention_weight_max": 98.99874114990234,
      "activations/layer7_attention_weight_min": -102.85238647460938,
      "activations/layer8_attention_weight_max": 44.6095085144043,
      "activations/layer8_attention_weight_min": -48.40650177001953,
      "activations/layer9_attention_weight_max": 34.435874938964844,
      "activations/layer9_attention_weight_min": -38.07449722290039,
      "epoch": 23.11,
      "learning_rate": 1.006060606060606e-06,
      "loss": 2.6768,
      "step": 397650
    },
    {
      "activations/layer0_attention_weight_max": 16.63945198059082,
      "activations/layer0_attention_weight_min": -14.488770484924316,
      "activations/layer10_attention_weight_max": 34.94599533081055,
      "activations/layer10_attention_weight_min": -36.14339065551758,
      "activations/layer11_attention_weight_max": 35.54390335083008,
      "activations/layer11_attention_weight_min": -36.406761169433594,
      "activations/layer12_attention_weight_max": 27.423988342285156,
      "activations/layer12_attention_weight_min": -30.678911209106445,
      "activations/layer13_attention_weight_max": 38.585968017578125,
      "activations/layer13_attention_weight_min": -37.396636962890625,
      "activations/layer14_attention_weight_max": 42.02340316772461,
      "activations/layer14_attention_weight_min": -38.50627517700195,
      "activations/layer15_attention_weight_max": 39.03630447387695,
      "activations/layer15_attention_weight_min": -37.52796936035156,
      "activations/layer16_attention_weight_max": 29.198265075683594,
      "activations/layer16_attention_weight_min": -28.773914337158203,
      "activations/layer17_attention_weight_max": 29.640533447265625,
      "activations/layer17_attention_weight_min": -27.333337783813477,
      "activations/layer18_attention_weight_max": 32.413631439208984,
      "activations/layer18_attention_weight_min": -27.845928192138672,
      "activations/layer19_attention_weight_max": 33.8537483215332,
      "activations/layer19_attention_weight_min": -33.89208221435547,
      "activations/layer1_attention_weight_max": 16.624645233154297,
      "activations/layer1_attention_weight_min": -15.036493301391602,
      "activations/layer20_attention_weight_max": 28.209264755249023,
      "activations/layer20_attention_weight_min": -26.147315979003906,
      "activations/layer21_attention_weight_max": 28.810300827026367,
      "activations/layer21_attention_weight_min": -25.415861129760742,
      "activations/layer22_attention_weight_max": 40.33488082885742,
      "activations/layer22_attention_weight_min": -34.66554260253906,
      "activations/layer23_attention_weight_max": 31.690214157104492,
      "activations/layer23_attention_weight_min": -28.233131408691406,
      "activations/layer2_attention_weight_max": 35.07096481323242,
      "activations/layer2_attention_weight_min": -33.607765197753906,
      "activations/layer3_attention_weight_max": 110.6785888671875,
      "activations/layer3_attention_weight_min": -111.03599548339844,
      "activations/layer4_attention_weight_max": 111.0899429321289,
      "activations/layer4_attention_weight_min": -115.28873443603516,
      "activations/layer5_attention_weight_max": 54.910091400146484,
      "activations/layer5_attention_weight_min": -62.86632537841797,
      "activations/layer6_attention_weight_max": 46.81665802001953,
      "activations/layer6_attention_weight_min": -52.31232833862305,
      "activations/layer7_attention_weight_max": 96.99009704589844,
      "activations/layer7_attention_weight_min": -102.69732666015625,
      "activations/layer8_attention_weight_max": 42.88273620605469,
      "activations/layer8_attention_weight_min": -46.144004821777344,
      "activations/layer9_attention_weight_max": 33.32455062866211,
      "activations/layer9_attention_weight_min": -37.064571380615234,
      "epoch": 23.11,
      "learning_rate": 9.87121212121212e-07,
      "loss": 2.7149,
      "step": 397700
    },
    {
      "activations/layer0_attention_weight_max": 15.60858154296875,
      "activations/layer0_attention_weight_min": -14.691983222961426,
      "activations/layer10_attention_weight_max": 44.202735900878906,
      "activations/layer10_attention_weight_min": -40.82145309448242,
      "activations/layer11_attention_weight_max": 44.16305160522461,
      "activations/layer11_attention_weight_min": -43.009910583496094,
      "activations/layer12_attention_weight_max": 31.69321060180664,
      "activations/layer12_attention_weight_min": -29.746295928955078,
      "activations/layer13_attention_weight_max": 42.161285400390625,
      "activations/layer13_attention_weight_min": -36.84071350097656,
      "activations/layer14_attention_weight_max": 49.22696304321289,
      "activations/layer14_attention_weight_min": -39.479923248291016,
      "activations/layer15_attention_weight_max": 45.39979553222656,
      "activations/layer15_attention_weight_min": -39.83575439453125,
      "activations/layer16_attention_weight_max": 32.1949462890625,
      "activations/layer16_attention_weight_min": -29.814428329467773,
      "activations/layer17_attention_weight_max": 34.051055908203125,
      "activations/layer17_attention_weight_min": -27.16191864013672,
      "activations/layer18_attention_weight_max": 34.802085876464844,
      "activations/layer18_attention_weight_min": -29.413841247558594,
      "activations/layer19_attention_weight_max": 36.30351257324219,
      "activations/layer19_attention_weight_min": -32.42385482788086,
      "activations/layer1_attention_weight_max": 16.538433074951172,
      "activations/layer1_attention_weight_min": -16.036544799804688,
      "activations/layer20_attention_weight_max": 29.263134002685547,
      "activations/layer20_attention_weight_min": -26.774913787841797,
      "activations/layer21_attention_weight_max": 29.78445816040039,
      "activations/layer21_attention_weight_min": -27.13956642150879,
      "activations/layer22_attention_weight_max": 41.4196662902832,
      "activations/layer22_attention_weight_min": -34.92596435546875,
      "activations/layer23_attention_weight_max": 32.015052795410156,
      "activations/layer23_attention_weight_min": -28.50629425048828,
      "activations/layer2_attention_weight_max": 37.29039764404297,
      "activations/layer2_attention_weight_min": -36.48036193847656,
      "activations/layer3_attention_weight_max": 109.63837432861328,
      "activations/layer3_attention_weight_min": -114.51953887939453,
      "activations/layer4_attention_weight_max": 111.2027816772461,
      "activations/layer4_attention_weight_min": -116.31884765625,
      "activations/layer5_attention_weight_max": 52.340660095214844,
      "activations/layer5_attention_weight_min": -63.71440124511719,
      "activations/layer6_attention_weight_max": 45.62052917480469,
      "activations/layer6_attention_weight_min": -49.44895553588867,
      "activations/layer7_attention_weight_max": 113.81291198730469,
      "activations/layer7_attention_weight_min": -111.42809295654297,
      "activations/layer8_attention_weight_max": 49.77264404296875,
      "activations/layer8_attention_weight_min": -51.78821563720703,
      "activations/layer9_attention_weight_max": 38.06336975097656,
      "activations/layer9_attention_weight_min": -38.901100158691406,
      "epoch": 23.11,
      "learning_rate": 9.68560606060606e-07,
      "loss": 2.6994,
      "step": 397750
    },
    {
      "activations/layer0_attention_weight_max": 17.296932220458984,
      "activations/layer0_attention_weight_min": -14.148613929748535,
      "activations/layer10_attention_weight_max": 39.006961822509766,
      "activations/layer10_attention_weight_min": -37.20216751098633,
      "activations/layer11_attention_weight_max": 36.6973991394043,
      "activations/layer11_attention_weight_min": -37.0001335144043,
      "activations/layer12_attention_weight_max": 29.110126495361328,
      "activations/layer12_attention_weight_min": -28.224506378173828,
      "activations/layer13_attention_weight_max": 40.0090217590332,
      "activations/layer13_attention_weight_min": -37.708106994628906,
      "activations/layer14_attention_weight_max": 41.273311614990234,
      "activations/layer14_attention_weight_min": -41.426734924316406,
      "activations/layer15_attention_weight_max": 38.43790054321289,
      "activations/layer15_attention_weight_min": -37.602561950683594,
      "activations/layer16_attention_weight_max": 29.93144989013672,
      "activations/layer16_attention_weight_min": -27.734085083007812,
      "activations/layer17_attention_weight_max": 30.98674964904785,
      "activations/layer17_attention_weight_min": -25.89349365234375,
      "activations/layer18_attention_weight_max": 32.24953079223633,
      "activations/layer18_attention_weight_min": -25.194669723510742,
      "activations/layer19_attention_weight_max": 35.76131820678711,
      "activations/layer19_attention_weight_min": -31.914920806884766,
      "activations/layer1_attention_weight_max": 15.640094757080078,
      "activations/layer1_attention_weight_min": -15.082695960998535,
      "activations/layer20_attention_weight_max": 28.80400276184082,
      "activations/layer20_attention_weight_min": -24.62401008605957,
      "activations/layer21_attention_weight_max": 28.047807693481445,
      "activations/layer21_attention_weight_min": -23.81296157836914,
      "activations/layer22_attention_weight_max": 43.97645568847656,
      "activations/layer22_attention_weight_min": -31.920917510986328,
      "activations/layer23_attention_weight_max": 32.538734436035156,
      "activations/layer23_attention_weight_min": -26.331207275390625,
      "activations/layer2_attention_weight_max": 34.26694107055664,
      "activations/layer2_attention_weight_min": -34.175392150878906,
      "activations/layer3_attention_weight_max": 114.96591186523438,
      "activations/layer3_attention_weight_min": -111.38699340820312,
      "activations/layer4_attention_weight_max": 115.4076156616211,
      "activations/layer4_attention_weight_min": -115.73099517822266,
      "activations/layer5_attention_weight_max": 57.180355072021484,
      "activations/layer5_attention_weight_min": -61.54240036010742,
      "activations/layer6_attention_weight_max": 50.23120880126953,
      "activations/layer6_attention_weight_min": -53.96532440185547,
      "activations/layer7_attention_weight_max": 104.16854858398438,
      "activations/layer7_attention_weight_min": -109.34207916259766,
      "activations/layer8_attention_weight_max": 48.94321060180664,
      "activations/layer8_attention_weight_min": -51.0367317199707,
      "activations/layer9_attention_weight_max": 36.98342514038086,
      "activations/layer9_attention_weight_min": -37.0009880065918,
      "epoch": 23.11,
      "learning_rate": 9.496212121212121e-07,
      "loss": 2.6777,
      "step": 397800
    },
    {
      "activations/layer0_attention_weight_max": 17.675647735595703,
      "activations/layer0_attention_weight_min": -13.798677444458008,
      "activations/layer10_attention_weight_max": 39.41553497314453,
      "activations/layer10_attention_weight_min": -39.788116455078125,
      "activations/layer11_attention_weight_max": 40.69943618774414,
      "activations/layer11_attention_weight_min": -41.90410614013672,
      "activations/layer12_attention_weight_max": 27.617652893066406,
      "activations/layer12_attention_weight_min": -27.633577346801758,
      "activations/layer13_attention_weight_max": 39.5838623046875,
      "activations/layer13_attention_weight_min": -36.65940856933594,
      "activations/layer14_attention_weight_max": 45.574676513671875,
      "activations/layer14_attention_weight_min": -38.701255798339844,
      "activations/layer15_attention_weight_max": 41.94912338256836,
      "activations/layer15_attention_weight_min": -38.8708610534668,
      "activations/layer16_attention_weight_max": 32.27255630493164,
      "activations/layer16_attention_weight_min": -30.192598342895508,
      "activations/layer17_attention_weight_max": 30.274288177490234,
      "activations/layer17_attention_weight_min": -28.681482315063477,
      "activations/layer18_attention_weight_max": 33.27684783935547,
      "activations/layer18_attention_weight_min": -28.12502670288086,
      "activations/layer19_attention_weight_max": 41.12409210205078,
      "activations/layer19_attention_weight_min": -35.02119827270508,
      "activations/layer1_attention_weight_max": 16.293516159057617,
      "activations/layer1_attention_weight_min": -16.204195022583008,
      "activations/layer20_attention_weight_max": 31.4589786529541,
      "activations/layer20_attention_weight_min": -26.52608299255371,
      "activations/layer21_attention_weight_max": 32.1573371887207,
      "activations/layer21_attention_weight_min": -26.355459213256836,
      "activations/layer22_attention_weight_max": 45.233463287353516,
      "activations/layer22_attention_weight_min": -35.026588439941406,
      "activations/layer23_attention_weight_max": 38.95890426635742,
      "activations/layer23_attention_weight_min": -28.826061248779297,
      "activations/layer2_attention_weight_max": 34.8503532409668,
      "activations/layer2_attention_weight_min": -34.24680709838867,
      "activations/layer3_attention_weight_max": 112.05462646484375,
      "activations/layer3_attention_weight_min": -112.2498779296875,
      "activations/layer4_attention_weight_max": 117.551025390625,
      "activations/layer4_attention_weight_min": -119.66887664794922,
      "activations/layer5_attention_weight_max": 53.054664611816406,
      "activations/layer5_attention_weight_min": -62.087066650390625,
      "activations/layer6_attention_weight_max": 46.558311462402344,
      "activations/layer6_attention_weight_min": -50.57322692871094,
      "activations/layer7_attention_weight_max": 101.99408721923828,
      "activations/layer7_attention_weight_min": -103.25902557373047,
      "activations/layer8_attention_weight_max": 45.481842041015625,
      "activations/layer8_attention_weight_min": -47.831600189208984,
      "activations/layer9_attention_weight_max": 34.99763870239258,
      "activations/layer9_attention_weight_min": -38.5910530090332,
      "epoch": 23.12,
      "learning_rate": 9.306818181818181e-07,
      "loss": 2.6754,
      "step": 397850
    },
    {
      "activations/layer0_attention_weight_max": 16.26902198791504,
      "activations/layer0_attention_weight_min": -14.995414733886719,
      "activations/layer10_attention_weight_max": 38.558563232421875,
      "activations/layer10_attention_weight_min": -39.021453857421875,
      "activations/layer11_attention_weight_max": 37.853057861328125,
      "activations/layer11_attention_weight_min": -40.068397521972656,
      "activations/layer12_attention_weight_max": 30.104225158691406,
      "activations/layer12_attention_weight_min": -30.64214324951172,
      "activations/layer13_attention_weight_max": 46.025901794433594,
      "activations/layer13_attention_weight_min": -42.81668472290039,
      "activations/layer14_attention_weight_max": 46.6067008972168,
      "activations/layer14_attention_weight_min": -44.52058792114258,
      "activations/layer15_attention_weight_max": 43.141605377197266,
      "activations/layer15_attention_weight_min": -43.74253845214844,
      "activations/layer16_attention_weight_max": 31.940181732177734,
      "activations/layer16_attention_weight_min": -30.368192672729492,
      "activations/layer17_attention_weight_max": 35.178802490234375,
      "activations/layer17_attention_weight_min": -29.342811584472656,
      "activations/layer18_attention_weight_max": 34.87383270263672,
      "activations/layer18_attention_weight_min": -28.496675491333008,
      "activations/layer19_attention_weight_max": 35.79648208618164,
      "activations/layer19_attention_weight_min": -37.216732025146484,
      "activations/layer1_attention_weight_max": 16.278799057006836,
      "activations/layer1_attention_weight_min": -16.168439865112305,
      "activations/layer20_attention_weight_max": 30.655590057373047,
      "activations/layer20_attention_weight_min": -28.911724090576172,
      "activations/layer21_attention_weight_max": 30.93792724609375,
      "activations/layer21_attention_weight_min": -28.819820404052734,
      "activations/layer22_attention_weight_max": 45.375797271728516,
      "activations/layer22_attention_weight_min": -35.0283317565918,
      "activations/layer23_attention_weight_max": 33.332157135009766,
      "activations/layer23_attention_weight_min": -28.471328735351562,
      "activations/layer2_attention_weight_max": 34.85899353027344,
      "activations/layer2_attention_weight_min": -35.384620666503906,
      "activations/layer3_attention_weight_max": 113.06047058105469,
      "activations/layer3_attention_weight_min": -116.15152740478516,
      "activations/layer4_attention_weight_max": 119.03570556640625,
      "activations/layer4_attention_weight_min": -121.7413101196289,
      "activations/layer5_attention_weight_max": 56.4669303894043,
      "activations/layer5_attention_weight_min": -61.903438568115234,
      "activations/layer6_attention_weight_max": 49.24031448364258,
      "activations/layer6_attention_weight_min": -53.05583190917969,
      "activations/layer7_attention_weight_max": 105.28553009033203,
      "activations/layer7_attention_weight_min": -116.23818969726562,
      "activations/layer8_attention_weight_max": 48.75642776489258,
      "activations/layer8_attention_weight_min": -50.73890686035156,
      "activations/layer9_attention_weight_max": 34.93296432495117,
      "activations/layer9_attention_weight_min": -40.53807067871094,
      "epoch": 23.12,
      "learning_rate": 9.117424242424242e-07,
      "loss": 2.6918,
      "step": 397900
    },
    {
      "activations/layer0_attention_weight_max": 17.167085647583008,
      "activations/layer0_attention_weight_min": -14.160347938537598,
      "activations/layer10_attention_weight_max": 38.81229019165039,
      "activations/layer10_attention_weight_min": -37.32594680786133,
      "activations/layer11_attention_weight_max": 38.757423400878906,
      "activations/layer11_attention_weight_min": -37.78580093383789,
      "activations/layer12_attention_weight_max": 28.384326934814453,
      "activations/layer12_attention_weight_min": -27.625999450683594,
      "activations/layer13_attention_weight_max": 38.54962921142578,
      "activations/layer13_attention_weight_min": -35.899192810058594,
      "activations/layer14_attention_weight_max": 44.763938903808594,
      "activations/layer14_attention_weight_min": -38.739356994628906,
      "activations/layer15_attention_weight_max": 40.91704177856445,
      "activations/layer15_attention_weight_min": -38.694339752197266,
      "activations/layer16_attention_weight_max": 30.064533233642578,
      "activations/layer16_attention_weight_min": -29.94959831237793,
      "activations/layer17_attention_weight_max": 30.076196670532227,
      "activations/layer17_attention_weight_min": -26.48276710510254,
      "activations/layer18_attention_weight_max": 36.13740921020508,
      "activations/layer18_attention_weight_min": -28.360904693603516,
      "activations/layer19_attention_weight_max": 35.490989685058594,
      "activations/layer19_attention_weight_min": -32.61979675292969,
      "activations/layer1_attention_weight_max": 16.436872482299805,
      "activations/layer1_attention_weight_min": -15.814095497131348,
      "activations/layer20_attention_weight_max": 28.9027099609375,
      "activations/layer20_attention_weight_min": -25.942625045776367,
      "activations/layer21_attention_weight_max": 28.244497299194336,
      "activations/layer21_attention_weight_min": -26.77707862854004,
      "activations/layer22_attention_weight_max": 41.836158752441406,
      "activations/layer22_attention_weight_min": -32.840240478515625,
      "activations/layer23_attention_weight_max": 32.393070220947266,
      "activations/layer23_attention_weight_min": -25.738508224487305,
      "activations/layer2_attention_weight_max": 34.99024963378906,
      "activations/layer2_attention_weight_min": -35.1524543762207,
      "activations/layer3_attention_weight_max": 108.28131866455078,
      "activations/layer3_attention_weight_min": -109.8455581665039,
      "activations/layer4_attention_weight_max": 112.32491302490234,
      "activations/layer4_attention_weight_min": -115.4837646484375,
      "activations/layer5_attention_weight_max": 51.826446533203125,
      "activations/layer5_attention_weight_min": -64.85392761230469,
      "activations/layer6_attention_weight_max": 48.0370979309082,
      "activations/layer6_attention_weight_min": -49.968807220458984,
      "activations/layer7_attention_weight_max": 101.8670425415039,
      "activations/layer7_attention_weight_min": -102.95800018310547,
      "activations/layer8_attention_weight_max": 45.11747360229492,
      "activations/layer8_attention_weight_min": -45.16621398925781,
      "activations/layer9_attention_weight_max": 33.76469802856445,
      "activations/layer9_attention_weight_min": -35.963706970214844,
      "epoch": 23.12,
      "learning_rate": 8.928030303030303e-07,
      "loss": 2.6962,
      "step": 397950
    },
    {
      "activations/layer0_attention_weight_max": 16.775636672973633,
      "activations/layer0_attention_weight_min": -13.904900550842285,
      "activations/layer10_attention_weight_max": 36.88859939575195,
      "activations/layer10_attention_weight_min": -38.46639633178711,
      "activations/layer11_attention_weight_max": 35.53472900390625,
      "activations/layer11_attention_weight_min": -38.332454681396484,
      "activations/layer12_attention_weight_max": 30.039623260498047,
      "activations/layer12_attention_weight_min": -28.1623477935791,
      "activations/layer13_attention_weight_max": 42.6783561706543,
      "activations/layer13_attention_weight_min": -37.00701141357422,
      "activations/layer14_attention_weight_max": 49.12263870239258,
      "activations/layer14_attention_weight_min": -40.454219818115234,
      "activations/layer15_attention_weight_max": 47.62408447265625,
      "activations/layer15_attention_weight_min": -40.40361022949219,
      "activations/layer16_attention_weight_max": 33.11586380004883,
      "activations/layer16_attention_weight_min": -27.992963790893555,
      "activations/layer17_attention_weight_max": 31.230573654174805,
      "activations/layer17_attention_weight_min": -26.920639038085938,
      "activations/layer18_attention_weight_max": 32.52204132080078,
      "activations/layer18_attention_weight_min": -27.15324592590332,
      "activations/layer19_attention_weight_max": 38.81708908081055,
      "activations/layer19_attention_weight_min": -34.956634521484375,
      "activations/layer1_attention_weight_max": 15.596456527709961,
      "activations/layer1_attention_weight_min": -15.339158058166504,
      "activations/layer20_attention_weight_max": 32.37968444824219,
      "activations/layer20_attention_weight_min": -27.623722076416016,
      "activations/layer21_attention_weight_max": 31.640544891357422,
      "activations/layer21_attention_weight_min": -27.225351333618164,
      "activations/layer22_attention_weight_max": 44.130733489990234,
      "activations/layer22_attention_weight_min": -33.93223190307617,
      "activations/layer23_attention_weight_max": 34.04563903808594,
      "activations/layer23_attention_weight_min": -28.29444122314453,
      "activations/layer2_attention_weight_max": 35.58747100830078,
      "activations/layer2_attention_weight_min": -34.0667724609375,
      "activations/layer3_attention_weight_max": 111.55281066894531,
      "activations/layer3_attention_weight_min": -111.32987976074219,
      "activations/layer4_attention_weight_max": 113.61183166503906,
      "activations/layer4_attention_weight_min": -115.03475189208984,
      "activations/layer5_attention_weight_max": 52.60633087158203,
      "activations/layer5_attention_weight_min": -64.62248229980469,
      "activations/layer6_attention_weight_max": 48.782981872558594,
      "activations/layer6_attention_weight_min": -51.3464469909668,
      "activations/layer7_attention_weight_max": 104.91091918945312,
      "activations/layer7_attention_weight_min": -111.69100189208984,
      "activations/layer8_attention_weight_max": 46.652008056640625,
      "activations/layer8_attention_weight_min": -49.96502685546875,
      "activations/layer9_attention_weight_max": 35.15415573120117,
      "activations/layer9_attention_weight_min": -36.45063018798828,
      "epoch": 23.13,
      "learning_rate": 8.738636363636364e-07,
      "loss": 2.6868,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_loss": 2.658203125,
      "eval_runtime": 8.5017,
      "eval_samples_per_second": 505.074,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_openwebtext_loss": 2.658203125,
      "eval_openwebtext_ppl": 14.2706235200274,
      "eval_openwebtext_runtime": 8.5017,
      "eval_openwebtext_samples_per_second": 505.074,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_wikitext_loss": 2.890625,
      "eval_wikitext_ppl": 18.00455893510183,
      "eval_wikitext_runtime": 2.0012,
      "eval_wikitext_samples_per_second": 227.861,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_lambada_loss": 2.306640625,
      "eval_lambada_ppl": 10.040637668042795,
      "eval_lambada_runtime": 9.5864,
      "eval_lambada_samples_per_second": 507.906,
      "step": 398000
    },
    {
      "activations/layer0_attention_weight_max": 16.565446853637695,
      "activations/layer0_attention_weight_min": -13.603342056274414,
      "activations/layer10_attention_weight_max": 36.51411437988281,
      "activations/layer10_attention_weight_min": -37.2548942565918,
      "activations/layer11_attention_weight_max": 37.15571975708008,
      "activations/layer11_attention_weight_min": -37.725887298583984,
      "activations/layer12_attention_weight_max": 28.82809066772461,
      "activations/layer12_attention_weight_min": -27.743396759033203,
      "activations/layer13_attention_weight_max": 41.26351547241211,
      "activations/layer13_attention_weight_min": -38.794219970703125,
      "activations/layer14_attention_weight_max": 47.511810302734375,
      "activations/layer14_attention_weight_min": -41.2395133972168,
      "activations/layer15_attention_weight_max": 43.25259017944336,
      "activations/layer15_attention_weight_min": -41.5702018737793,
      "activations/layer16_attention_weight_max": 34.40614700317383,
      "activations/layer16_attention_weight_min": -30.35209846496582,
      "activations/layer17_attention_weight_max": 32.4591178894043,
      "activations/layer17_attention_weight_min": -29.618478775024414,
      "activations/layer18_attention_weight_max": 34.70231246948242,
      "activations/layer18_attention_weight_min": -30.583946228027344,
      "activations/layer19_attention_weight_max": 40.706241607666016,
      "activations/layer19_attention_weight_min": -33.4613037109375,
      "activations/layer1_attention_weight_max": 15.94479751586914,
      "activations/layer1_attention_weight_min": -15.883322715759277,
      "activations/layer20_attention_weight_max": 31.04889488220215,
      "activations/layer20_attention_weight_min": -27.859928131103516,
      "activations/layer21_attention_weight_max": 31.70876121520996,
      "activations/layer21_attention_weight_min": -28.12169647216797,
      "activations/layer22_attention_weight_max": 46.724571228027344,
      "activations/layer22_attention_weight_min": -39.40629959106445,
      "activations/layer23_attention_weight_max": 33.87662887573242,
      "activations/layer23_attention_weight_min": -30.340185165405273,
      "activations/layer2_attention_weight_max": 34.98876953125,
      "activations/layer2_attention_weight_min": -34.481040954589844,
      "activations/layer3_attention_weight_max": 112.92183685302734,
      "activations/layer3_attention_weight_min": -115.61566925048828,
      "activations/layer4_attention_weight_max": 112.65836334228516,
      "activations/layer4_attention_weight_min": -117.2594223022461,
      "activations/layer5_attention_weight_max": 53.65847396850586,
      "activations/layer5_attention_weight_min": -64.93472290039062,
      "activations/layer6_attention_weight_max": 48.083675384521484,
      "activations/layer6_attention_weight_min": -49.24932098388672,
      "activations/layer7_attention_weight_max": 106.45640563964844,
      "activations/layer7_attention_weight_min": -101.43644714355469,
      "activations/layer8_attention_weight_max": 47.00904846191406,
      "activations/layer8_attention_weight_min": -46.28062438964844,
      "activations/layer9_attention_weight_max": 34.495384216308594,
      "activations/layer9_attention_weight_min": -35.35757064819336,
      "epoch": 23.13,
      "learning_rate": 8.549242424242423e-07,
      "loss": 2.6949,
      "step": 398050
    },
    {
      "activations/layer0_attention_weight_max": 17.234548568725586,
      "activations/layer0_attention_weight_min": -15.001726150512695,
      "activations/layer10_attention_weight_max": 34.68452453613281,
      "activations/layer10_attention_weight_min": -38.06045150756836,
      "activations/layer11_attention_weight_max": 34.46725845336914,
      "activations/layer11_attention_weight_min": -38.04895782470703,
      "activations/layer12_attention_weight_max": 27.831684112548828,
      "activations/layer12_attention_weight_min": -28.659456253051758,
      "activations/layer13_attention_weight_max": 42.405731201171875,
      "activations/layer13_attention_weight_min": -37.74601364135742,
      "activations/layer14_attention_weight_max": 45.54139709472656,
      "activations/layer14_attention_weight_min": -40.17265319824219,
      "activations/layer15_attention_weight_max": 41.331356048583984,
      "activations/layer15_attention_weight_min": -39.71086120605469,
      "activations/layer16_attention_weight_max": 31.321155548095703,
      "activations/layer16_attention_weight_min": -27.523658752441406,
      "activations/layer17_attention_weight_max": 34.21950149536133,
      "activations/layer17_attention_weight_min": -27.535200119018555,
      "activations/layer18_attention_weight_max": 40.957984924316406,
      "activations/layer18_attention_weight_min": -30.20069694519043,
      "activations/layer19_attention_weight_max": 39.26601791381836,
      "activations/layer19_attention_weight_min": -33.75953674316406,
      "activations/layer1_attention_weight_max": 17.0518798828125,
      "activations/layer1_attention_weight_min": -15.733856201171875,
      "activations/layer20_attention_weight_max": 32.00901794433594,
      "activations/layer20_attention_weight_min": -25.401643753051758,
      "activations/layer21_attention_weight_max": 34.04808044433594,
      "activations/layer21_attention_weight_min": -26.061025619506836,
      "activations/layer22_attention_weight_max": 44.64146423339844,
      "activations/layer22_attention_weight_min": -32.51313018798828,
      "activations/layer23_attention_weight_max": 41.08472442626953,
      "activations/layer23_attention_weight_min": -26.530065536499023,
      "activations/layer2_attention_weight_max": 34.74015808105469,
      "activations/layer2_attention_weight_min": -33.429847717285156,
      "activations/layer3_attention_weight_max": 112.05451202392578,
      "activations/layer3_attention_weight_min": -110.8045654296875,
      "activations/layer4_attention_weight_max": 119.32342529296875,
      "activations/layer4_attention_weight_min": -118.08465576171875,
      "activations/layer5_attention_weight_max": 53.563194274902344,
      "activations/layer5_attention_weight_min": -64.35563659667969,
      "activations/layer6_attention_weight_max": 47.97398376464844,
      "activations/layer6_attention_weight_min": -50.66968536376953,
      "activations/layer7_attention_weight_max": 98.93744659423828,
      "activations/layer7_attention_weight_min": -105.04540252685547,
      "activations/layer8_attention_weight_max": 46.19550323486328,
      "activations/layer8_attention_weight_min": -48.13930130004883,
      "activations/layer9_attention_weight_max": 32.87289047241211,
      "activations/layer9_attention_weight_min": -37.386451721191406,
      "epoch": 23.13,
      "learning_rate": 8.359848484848483e-07,
      "loss": 2.6861,
      "step": 398100
    },
    {
      "activations/layer0_attention_weight_max": 16.04387664794922,
      "activations/layer0_attention_weight_min": -14.439974784851074,
      "activations/layer10_attention_weight_max": 37.0780143737793,
      "activations/layer10_attention_weight_min": -36.324989318847656,
      "activations/layer11_attention_weight_max": 37.92124557495117,
      "activations/layer11_attention_weight_min": -36.40332794189453,
      "activations/layer12_attention_weight_max": 28.53370475769043,
      "activations/layer12_attention_weight_min": -30.251569747924805,
      "activations/layer13_attention_weight_max": 41.29212188720703,
      "activations/layer13_attention_weight_min": -38.553466796875,
      "activations/layer14_attention_weight_max": 44.47021484375,
      "activations/layer14_attention_weight_min": -40.69739532470703,
      "activations/layer15_attention_weight_max": 39.68534469604492,
      "activations/layer15_attention_weight_min": -39.808021545410156,
      "activations/layer16_attention_weight_max": 28.17462730407715,
      "activations/layer16_attention_weight_min": -28.924434661865234,
      "activations/layer17_attention_weight_max": 30.104312896728516,
      "activations/layer17_attention_weight_min": -29.065887451171875,
      "activations/layer18_attention_weight_max": 32.391815185546875,
      "activations/layer18_attention_weight_min": -26.697288513183594,
      "activations/layer19_attention_weight_max": 33.74047088623047,
      "activations/layer19_attention_weight_min": -33.370384216308594,
      "activations/layer1_attention_weight_max": 15.92952823638916,
      "activations/layer1_attention_weight_min": -15.197011947631836,
      "activations/layer20_attention_weight_max": 29.325544357299805,
      "activations/layer20_attention_weight_min": -25.3482608795166,
      "activations/layer21_attention_weight_max": 28.611570358276367,
      "activations/layer21_attention_weight_min": -24.511980056762695,
      "activations/layer22_attention_weight_max": 45.31952667236328,
      "activations/layer22_attention_weight_min": -32.403621673583984,
      "activations/layer23_attention_weight_max": 32.076080322265625,
      "activations/layer23_attention_weight_min": -25.990619659423828,
      "activations/layer2_attention_weight_max": 35.8245964050293,
      "activations/layer2_attention_weight_min": -33.69971466064453,
      "activations/layer3_attention_weight_max": 114.35931396484375,
      "activations/layer3_attention_weight_min": -111.24431610107422,
      "activations/layer4_attention_weight_max": 118.35326385498047,
      "activations/layer4_attention_weight_min": -115.69738006591797,
      "activations/layer5_attention_weight_max": 55.70734786987305,
      "activations/layer5_attention_weight_min": -63.84668731689453,
      "activations/layer6_attention_weight_max": 48.59334182739258,
      "activations/layer6_attention_weight_min": -51.842952728271484,
      "activations/layer7_attention_weight_max": 102.38490295410156,
      "activations/layer7_attention_weight_min": -106.48060607910156,
      "activations/layer8_attention_weight_max": 47.15143585205078,
      "activations/layer8_attention_weight_min": -49.69890594482422,
      "activations/layer9_attention_weight_max": 35.94572067260742,
      "activations/layer9_attention_weight_min": -36.34586715698242,
      "epoch": 23.13,
      "learning_rate": 8.170454545454544e-07,
      "loss": 2.6876,
      "step": 398150
    },
    {
      "activations/layer0_attention_weight_max": 16.355894088745117,
      "activations/layer0_attention_weight_min": -13.057045936584473,
      "activations/layer10_attention_weight_max": 38.43741989135742,
      "activations/layer10_attention_weight_min": -39.97730255126953,
      "activations/layer11_attention_weight_max": 38.61163330078125,
      "activations/layer11_attention_weight_min": -39.61338806152344,
      "activations/layer12_attention_weight_max": 33.46767807006836,
      "activations/layer12_attention_weight_min": -30.7943172454834,
      "activations/layer13_attention_weight_max": 47.20460891723633,
      "activations/layer13_attention_weight_min": -42.75471115112305,
      "activations/layer14_attention_weight_max": 50.535404205322266,
      "activations/layer14_attention_weight_min": -44.02777862548828,
      "activations/layer15_attention_weight_max": 46.4220085144043,
      "activations/layer15_attention_weight_min": -39.51983642578125,
      "activations/layer16_attention_weight_max": 34.344757080078125,
      "activations/layer16_attention_weight_min": -31.108474731445312,
      "activations/layer17_attention_weight_max": 35.579429626464844,
      "activations/layer17_attention_weight_min": -28.84278678894043,
      "activations/layer18_attention_weight_max": 37.64469909667969,
      "activations/layer18_attention_weight_min": -28.460716247558594,
      "activations/layer19_attention_weight_max": 38.34867858886719,
      "activations/layer19_attention_weight_min": -33.5610237121582,
      "activations/layer1_attention_weight_max": 15.827648162841797,
      "activations/layer1_attention_weight_min": -15.780355453491211,
      "activations/layer20_attention_weight_max": 33.053348541259766,
      "activations/layer20_attention_weight_min": -28.675918579101562,
      "activations/layer21_attention_weight_max": 34.73491668701172,
      "activations/layer21_attention_weight_min": -26.1387996673584,
      "activations/layer22_attention_weight_max": 52.52214431762695,
      "activations/layer22_attention_weight_min": -35.0510139465332,
      "activations/layer23_attention_weight_max": 35.697410583496094,
      "activations/layer23_attention_weight_min": -26.33269500732422,
      "activations/layer2_attention_weight_max": 35.54566955566406,
      "activations/layer2_attention_weight_min": -34.31106948852539,
      "activations/layer3_attention_weight_max": 115.28279876708984,
      "activations/layer3_attention_weight_min": -112.48348999023438,
      "activations/layer4_attention_weight_max": 120.3709487915039,
      "activations/layer4_attention_weight_min": -118.64334869384766,
      "activations/layer5_attention_weight_max": 53.42733383178711,
      "activations/layer5_attention_weight_min": -63.568641662597656,
      "activations/layer6_attention_weight_max": 48.29372024536133,
      "activations/layer6_attention_weight_min": -52.38644027709961,
      "activations/layer7_attention_weight_max": 106.94267272949219,
      "activations/layer7_attention_weight_min": -111.54833221435547,
      "activations/layer8_attention_weight_max": 46.655723571777344,
      "activations/layer8_attention_weight_min": -49.9051628112793,
      "activations/layer9_attention_weight_max": 36.5341682434082,
      "activations/layer9_attention_weight_min": -38.562625885009766,
      "epoch": 23.14,
      "learning_rate": 7.984848484848485e-07,
      "loss": 2.7041,
      "step": 398200
    },
    {
      "activations/layer0_attention_weight_max": 16.419139862060547,
      "activations/layer0_attention_weight_min": -14.868454933166504,
      "activations/layer10_attention_weight_max": 33.66688537597656,
      "activations/layer10_attention_weight_min": -35.718536376953125,
      "activations/layer11_attention_weight_max": 31.395416259765625,
      "activations/layer11_attention_weight_min": -34.61769485473633,
      "activations/layer12_attention_weight_max": 27.702665328979492,
      "activations/layer12_attention_weight_min": -25.247209548950195,
      "activations/layer13_attention_weight_max": 36.87288284301758,
      "activations/layer13_attention_weight_min": -34.460960388183594,
      "activations/layer14_attention_weight_max": 39.02821731567383,
      "activations/layer14_attention_weight_min": -40.38691711425781,
      "activations/layer15_attention_weight_max": 35.63668441772461,
      "activations/layer15_attention_weight_min": -41.93716049194336,
      "activations/layer16_attention_weight_max": 30.210521697998047,
      "activations/layer16_attention_weight_min": -27.572969436645508,
      "activations/layer17_attention_weight_max": 28.45498275756836,
      "activations/layer17_attention_weight_min": -25.674522399902344,
      "activations/layer18_attention_weight_max": 32.63765335083008,
      "activations/layer18_attention_weight_min": -25.206218719482422,
      "activations/layer19_attention_weight_max": 32.643367767333984,
      "activations/layer19_attention_weight_min": -35.1257438659668,
      "activations/layer1_attention_weight_max": 15.50586986541748,
      "activations/layer1_attention_weight_min": -17.39655113220215,
      "activations/layer20_attention_weight_max": 27.213726043701172,
      "activations/layer20_attention_weight_min": -23.317228317260742,
      "activations/layer21_attention_weight_max": 27.019296646118164,
      "activations/layer21_attention_weight_min": -23.195568084716797,
      "activations/layer22_attention_weight_max": 38.63721466064453,
      "activations/layer22_attention_weight_min": -30.013547897338867,
      "activations/layer23_attention_weight_max": 32.27309799194336,
      "activations/layer23_attention_weight_min": -24.791332244873047,
      "activations/layer2_attention_weight_max": 34.43063735961914,
      "activations/layer2_attention_weight_min": -34.0616455078125,
      "activations/layer3_attention_weight_max": 109.23178100585938,
      "activations/layer3_attention_weight_min": -107.37189483642578,
      "activations/layer4_attention_weight_max": 114.73268127441406,
      "activations/layer4_attention_weight_min": -113.48986053466797,
      "activations/layer5_attention_weight_max": 52.506736755371094,
      "activations/layer5_attention_weight_min": -62.24033737182617,
      "activations/layer6_attention_weight_max": 44.16559982299805,
      "activations/layer6_attention_weight_min": -46.66304397583008,
      "activations/layer7_attention_weight_max": 95.26807403564453,
      "activations/layer7_attention_weight_min": -99.63714599609375,
      "activations/layer8_attention_weight_max": 40.39527893066406,
      "activations/layer8_attention_weight_min": -43.7765998840332,
      "activations/layer9_attention_weight_max": 33.3391227722168,
      "activations/layer9_attention_weight_min": -33.56085968017578,
      "epoch": 23.14,
      "learning_rate": 7.795454545454545e-07,
      "loss": 2.6804,
      "step": 398250
    },
    {
      "activations/layer0_attention_weight_max": 17.165124893188477,
      "activations/layer0_attention_weight_min": -14.494755744934082,
      "activations/layer10_attention_weight_max": 35.747589111328125,
      "activations/layer10_attention_weight_min": -36.76311111450195,
      "activations/layer11_attention_weight_max": 36.86958312988281,
      "activations/layer11_attention_weight_min": -37.80533981323242,
      "activations/layer12_attention_weight_max": 29.491596221923828,
      "activations/layer12_attention_weight_min": -29.76848793029785,
      "activations/layer13_attention_weight_max": 42.39943313598633,
      "activations/layer13_attention_weight_min": -42.72663497924805,
      "activations/layer14_attention_weight_max": 48.84273910522461,
      "activations/layer14_attention_weight_min": -45.01067352294922,
      "activations/layer15_attention_weight_max": 44.57600402832031,
      "activations/layer15_attention_weight_min": -41.34539031982422,
      "activations/layer16_attention_weight_max": 33.26982498168945,
      "activations/layer16_attention_weight_min": -29.57279396057129,
      "activations/layer17_attention_weight_max": 32.58873748779297,
      "activations/layer17_attention_weight_min": -29.327966690063477,
      "activations/layer18_attention_weight_max": 38.27119064331055,
      "activations/layer18_attention_weight_min": -30.663469314575195,
      "activations/layer19_attention_weight_max": 40.57517623901367,
      "activations/layer19_attention_weight_min": -36.903076171875,
      "activations/layer1_attention_weight_max": 16.151735305786133,
      "activations/layer1_attention_weight_min": -15.266600608825684,
      "activations/layer20_attention_weight_max": 36.07791519165039,
      "activations/layer20_attention_weight_min": -28.803953170776367,
      "activations/layer21_attention_weight_max": 34.39905548095703,
      "activations/layer21_attention_weight_min": -27.881437301635742,
      "activations/layer22_attention_weight_max": 48.07505798339844,
      "activations/layer22_attention_weight_min": -36.813480377197266,
      "activations/layer23_attention_weight_max": 33.66231155395508,
      "activations/layer23_attention_weight_min": -26.859603881835938,
      "activations/layer2_attention_weight_max": 33.84326171875,
      "activations/layer2_attention_weight_min": -33.050479888916016,
      "activations/layer3_attention_weight_max": 105.08509826660156,
      "activations/layer3_attention_weight_min": -109.41650390625,
      "activations/layer4_attention_weight_max": 111.60963439941406,
      "activations/layer4_attention_weight_min": -114.59679412841797,
      "activations/layer5_attention_weight_max": 51.98423385620117,
      "activations/layer5_attention_weight_min": -62.057647705078125,
      "activations/layer6_attention_weight_max": 46.872276306152344,
      "activations/layer6_attention_weight_min": -48.04383850097656,
      "activations/layer7_attention_weight_max": 100.3954086303711,
      "activations/layer7_attention_weight_min": -105.23416900634766,
      "activations/layer8_attention_weight_max": 44.71894454956055,
      "activations/layer8_attention_weight_min": -47.98834228515625,
      "activations/layer9_attention_weight_max": 34.84611129760742,
      "activations/layer9_attention_weight_min": -35.91830825805664,
      "epoch": 23.14,
      "learning_rate": 7.606060606060606e-07,
      "loss": 2.7121,
      "step": 398300
    },
    {
      "activations/layer0_attention_weight_max": 15.875985145568848,
      "activations/layer0_attention_weight_min": -14.296046257019043,
      "activations/layer10_attention_weight_max": 35.552734375,
      "activations/layer10_attention_weight_min": -36.236488342285156,
      "activations/layer11_attention_weight_max": 35.13459014892578,
      "activations/layer11_attention_weight_min": -36.70397186279297,
      "activations/layer12_attention_weight_max": 29.17660903930664,
      "activations/layer12_attention_weight_min": -28.962112426757812,
      "activations/layer13_attention_weight_max": 41.04084396362305,
      "activations/layer13_attention_weight_min": -37.872283935546875,
      "activations/layer14_attention_weight_max": 45.21355056762695,
      "activations/layer14_attention_weight_min": -40.87608337402344,
      "activations/layer15_attention_weight_max": 44.79473114013672,
      "activations/layer15_attention_weight_min": -39.44284439086914,
      "activations/layer16_attention_weight_max": 30.947843551635742,
      "activations/layer16_attention_weight_min": -30.889522552490234,
      "activations/layer17_attention_weight_max": 33.01258850097656,
      "activations/layer17_attention_weight_min": -27.726858139038086,
      "activations/layer18_attention_weight_max": 32.82554626464844,
      "activations/layer18_attention_weight_min": -26.241531372070312,
      "activations/layer19_attention_weight_max": 35.29475784301758,
      "activations/layer19_attention_weight_min": -34.21733474731445,
      "activations/layer1_attention_weight_max": 15.956089973449707,
      "activations/layer1_attention_weight_min": -14.327425956726074,
      "activations/layer20_attention_weight_max": 28.919477462768555,
      "activations/layer20_attention_weight_min": -27.264081954956055,
      "activations/layer21_attention_weight_max": 29.894954681396484,
      "activations/layer21_attention_weight_min": -25.359600067138672,
      "activations/layer22_attention_weight_max": 40.809906005859375,
      "activations/layer22_attention_weight_min": -30.944496154785156,
      "activations/layer23_attention_weight_max": 30.743526458740234,
      "activations/layer23_attention_weight_min": -24.83186149597168,
      "activations/layer2_attention_weight_max": 34.939788818359375,
      "activations/layer2_attention_weight_min": -34.15828323364258,
      "activations/layer3_attention_weight_max": 114.82109832763672,
      "activations/layer3_attention_weight_min": -112.98975372314453,
      "activations/layer4_attention_weight_max": 118.54046630859375,
      "activations/layer4_attention_weight_min": -116.06644439697266,
      "activations/layer5_attention_weight_max": 52.74564743041992,
      "activations/layer5_attention_weight_min": -61.670936584472656,
      "activations/layer6_attention_weight_max": 48.439388275146484,
      "activations/layer6_attention_weight_min": -51.502750396728516,
      "activations/layer7_attention_weight_max": 103.48270416259766,
      "activations/layer7_attention_weight_min": -113.8486328125,
      "activations/layer8_attention_weight_max": 47.5189323425293,
      "activations/layer8_attention_weight_min": -51.847721099853516,
      "activations/layer9_attention_weight_max": 34.40949630737305,
      "activations/layer9_attention_weight_min": -37.75654983520508,
      "epoch": 23.15,
      "learning_rate": 7.416666666666665e-07,
      "loss": 2.7036,
      "step": 398350
    },
    {
      "activations/layer0_attention_weight_max": 16.91510772705078,
      "activations/layer0_attention_weight_min": -13.523283004760742,
      "activations/layer10_attention_weight_max": 38.145076751708984,
      "activations/layer10_attention_weight_min": -38.702125549316406,
      "activations/layer11_attention_weight_max": 37.42579650878906,
      "activations/layer11_attention_weight_min": -38.21839141845703,
      "activations/layer12_attention_weight_max": 29.133533477783203,
      "activations/layer12_attention_weight_min": -28.039121627807617,
      "activations/layer13_attention_weight_max": 41.814849853515625,
      "activations/layer13_attention_weight_min": -37.4498176574707,
      "activations/layer14_attention_weight_max": 47.36418151855469,
      "activations/layer14_attention_weight_min": -40.57034683227539,
      "activations/layer15_attention_weight_max": 38.655479431152344,
      "activations/layer15_attention_weight_min": -36.5566291809082,
      "activations/layer16_attention_weight_max": 27.80229377746582,
      "activations/layer16_attention_weight_min": -28.598989486694336,
      "activations/layer17_attention_weight_max": 30.910865783691406,
      "activations/layer17_attention_weight_min": -28.274991989135742,
      "activations/layer18_attention_weight_max": 34.01179504394531,
      "activations/layer18_attention_weight_min": -26.942434310913086,
      "activations/layer19_attention_weight_max": 33.85972213745117,
      "activations/layer19_attention_weight_min": -31.181913375854492,
      "activations/layer1_attention_weight_max": 16.426868438720703,
      "activations/layer1_attention_weight_min": -16.493053436279297,
      "activations/layer20_attention_weight_max": 27.97858238220215,
      "activations/layer20_attention_weight_min": -26.684368133544922,
      "activations/layer21_attention_weight_max": 27.87063217163086,
      "activations/layer21_attention_weight_min": -27.1550350189209,
      "activations/layer22_attention_weight_max": 39.77335739135742,
      "activations/layer22_attention_weight_min": -32.510337829589844,
      "activations/layer23_attention_weight_max": 33.02040100097656,
      "activations/layer23_attention_weight_min": -24.78729820251465,
      "activations/layer2_attention_weight_max": 33.8154296875,
      "activations/layer2_attention_weight_min": -33.302555084228516,
      "activations/layer3_attention_weight_max": 106.37825012207031,
      "activations/layer3_attention_weight_min": -108.15216064453125,
      "activations/layer4_attention_weight_max": 109.37617492675781,
      "activations/layer4_attention_weight_min": -113.98445129394531,
      "activations/layer5_attention_weight_max": 53.93263244628906,
      "activations/layer5_attention_weight_min": -62.45821762084961,
      "activations/layer6_attention_weight_max": 46.4968376159668,
      "activations/layer6_attention_weight_min": -49.72875213623047,
      "activations/layer7_attention_weight_max": 100.70616912841797,
      "activations/layer7_attention_weight_min": -110.64248657226562,
      "activations/layer8_attention_weight_max": 45.07464599609375,
      "activations/layer8_attention_weight_min": -48.494171142578125,
      "activations/layer9_attention_weight_max": 35.199729919433594,
      "activations/layer9_attention_weight_min": -36.700347900390625,
      "epoch": 23.15,
      "learning_rate": 7.227272727272726e-07,
      "loss": 2.7034,
      "step": 398400
    },
    {
      "activations/layer0_attention_weight_max": 17.208654403686523,
      "activations/layer0_attention_weight_min": -14.806159019470215,
      "activations/layer10_attention_weight_max": 37.3359260559082,
      "activations/layer10_attention_weight_min": -35.826171875,
      "activations/layer11_attention_weight_max": 36.32794189453125,
      "activations/layer11_attention_weight_min": -35.58202362060547,
      "activations/layer12_attention_weight_max": 30.03521156311035,
      "activations/layer12_attention_weight_min": -27.355091094970703,
      "activations/layer13_attention_weight_max": 43.807315826416016,
      "activations/layer13_attention_weight_min": -38.3247184753418,
      "activations/layer14_attention_weight_max": 44.00928497314453,
      "activations/layer14_attention_weight_min": -42.019508361816406,
      "activations/layer15_attention_weight_max": 43.895198822021484,
      "activations/layer15_attention_weight_min": -39.388221740722656,
      "activations/layer16_attention_weight_max": 32.13951110839844,
      "activations/layer16_attention_weight_min": -28.404645919799805,
      "activations/layer17_attention_weight_max": 31.840576171875,
      "activations/layer17_attention_weight_min": -26.777128219604492,
      "activations/layer18_attention_weight_max": 33.95801544189453,
      "activations/layer18_attention_weight_min": -26.335567474365234,
      "activations/layer19_attention_weight_max": 33.73655319213867,
      "activations/layer19_attention_weight_min": -31.8068904876709,
      "activations/layer1_attention_weight_max": 15.795708656311035,
      "activations/layer1_attention_weight_min": -15.364690780639648,
      "activations/layer20_attention_weight_max": 29.859285354614258,
      "activations/layer20_attention_weight_min": -23.93352508544922,
      "activations/layer21_attention_weight_max": 29.137022018432617,
      "activations/layer21_attention_weight_min": -24.24945640563965,
      "activations/layer22_attention_weight_max": 38.94968795776367,
      "activations/layer22_attention_weight_min": -29.620378494262695,
      "activations/layer23_attention_weight_max": 32.033851623535156,
      "activations/layer23_attention_weight_min": -25.399139404296875,
      "activations/layer2_attention_weight_max": 33.49278259277344,
      "activations/layer2_attention_weight_min": -33.74010467529297,
      "activations/layer3_attention_weight_max": 108.11984252929688,
      "activations/layer3_attention_weight_min": -108.39813995361328,
      "activations/layer4_attention_weight_max": 110.25875091552734,
      "activations/layer4_attention_weight_min": -116.13561248779297,
      "activations/layer5_attention_weight_max": 50.044334411621094,
      "activations/layer5_attention_weight_min": -62.40831756591797,
      "activations/layer6_attention_weight_max": 44.8542594909668,
      "activations/layer6_attention_weight_min": -47.81819152832031,
      "activations/layer7_attention_weight_max": 104.29302978515625,
      "activations/layer7_attention_weight_min": -104.16024017333984,
      "activations/layer8_attention_weight_max": 45.387935638427734,
      "activations/layer8_attention_weight_min": -48.13939666748047,
      "activations/layer9_attention_weight_max": 35.21271896362305,
      "activations/layer9_attention_weight_min": -35.76923370361328,
      "epoch": 23.15,
      "learning_rate": 7.037878787878787e-07,
      "loss": 2.684,
      "step": 398450
    },
    {
      "activations/layer0_attention_weight_max": 17.13564109802246,
      "activations/layer0_attention_weight_min": -14.141839027404785,
      "activations/layer10_attention_weight_max": 34.49028015136719,
      "activations/layer10_attention_weight_min": -36.10203552246094,
      "activations/layer11_attention_weight_max": 35.600826263427734,
      "activations/layer11_attention_weight_min": -37.308128356933594,
      "activations/layer12_attention_weight_max": 25.230587005615234,
      "activations/layer12_attention_weight_min": -26.99172592163086,
      "activations/layer13_attention_weight_max": 37.440059661865234,
      "activations/layer13_attention_weight_min": -34.716976165771484,
      "activations/layer14_attention_weight_max": 41.31071090698242,
      "activations/layer14_attention_weight_min": -37.09170150756836,
      "activations/layer15_attention_weight_max": 40.427467346191406,
      "activations/layer15_attention_weight_min": -35.46244812011719,
      "activations/layer16_attention_weight_max": 28.506996154785156,
      "activations/layer16_attention_weight_min": -26.770200729370117,
      "activations/layer17_attention_weight_max": 29.334632873535156,
      "activations/layer17_attention_weight_min": -26.994949340820312,
      "activations/layer18_attention_weight_max": 31.234270095825195,
      "activations/layer18_attention_weight_min": -27.059946060180664,
      "activations/layer19_attention_weight_max": 32.005733489990234,
      "activations/layer19_attention_weight_min": -33.605037689208984,
      "activations/layer1_attention_weight_max": 15.640145301818848,
      "activations/layer1_attention_weight_min": -16.340848922729492,
      "activations/layer20_attention_weight_max": 26.61648178100586,
      "activations/layer20_attention_weight_min": -26.92255210876465,
      "activations/layer21_attention_weight_max": 27.07222557067871,
      "activations/layer21_attention_weight_min": -26.26300048828125,
      "activations/layer22_attention_weight_max": 38.23947525024414,
      "activations/layer22_attention_weight_min": -32.588035583496094,
      "activations/layer23_attention_weight_max": 28.393966674804688,
      "activations/layer23_attention_weight_min": -26.18677520751953,
      "activations/layer2_attention_weight_max": 35.000614166259766,
      "activations/layer2_attention_weight_min": -33.79631042480469,
      "activations/layer3_attention_weight_max": 112.3434829711914,
      "activations/layer3_attention_weight_min": -108.42310333251953,
      "activations/layer4_attention_weight_max": 112.20731353759766,
      "activations/layer4_attention_weight_min": -112.68645477294922,
      "activations/layer5_attention_weight_max": 52.447418212890625,
      "activations/layer5_attention_weight_min": -64.443115234375,
      "activations/layer6_attention_weight_max": 46.243736267089844,
      "activations/layer6_attention_weight_min": -48.01722717285156,
      "activations/layer7_attention_weight_max": 95.95476531982422,
      "activations/layer7_attention_weight_min": -100.1336669921875,
      "activations/layer8_attention_weight_max": 41.84587860107422,
      "activations/layer8_attention_weight_min": -45.23073196411133,
      "activations/layer9_attention_weight_max": 33.116886138916016,
      "activations/layer9_attention_weight_min": -34.30998229980469,
      "epoch": 23.16,
      "learning_rate": 6.848484848484848e-07,
      "loss": 2.6985,
      "step": 398500
    },
    {
      "activations/layer0_attention_weight_max": 16.23867416381836,
      "activations/layer0_attention_weight_min": -15.91675090789795,
      "activations/layer10_attention_weight_max": 38.866737365722656,
      "activations/layer10_attention_weight_min": -38.292083740234375,
      "activations/layer11_attention_weight_max": 38.2576904296875,
      "activations/layer11_attention_weight_min": -38.87669372558594,
      "activations/layer12_attention_weight_max": 30.1767520904541,
      "activations/layer12_attention_weight_min": -28.995155334472656,
      "activations/layer13_attention_weight_max": 45.82157516479492,
      "activations/layer13_attention_weight_min": -39.35673522949219,
      "activations/layer14_attention_weight_max": 52.566001892089844,
      "activations/layer14_attention_weight_min": -41.17110824584961,
      "activations/layer15_attention_weight_max": 47.548095703125,
      "activations/layer15_attention_weight_min": -39.80365753173828,
      "activations/layer16_attention_weight_max": 33.347599029541016,
      "activations/layer16_attention_weight_min": -30.632152557373047,
      "activations/layer17_attention_weight_max": 31.996965408325195,
      "activations/layer17_attention_weight_min": -27.77796173095703,
      "activations/layer18_attention_weight_max": 32.395347595214844,
      "activations/layer18_attention_weight_min": -27.3115177154541,
      "activations/layer19_attention_weight_max": 38.58515548706055,
      "activations/layer19_attention_weight_min": -32.82470703125,
      "activations/layer1_attention_weight_max": 16.081350326538086,
      "activations/layer1_attention_weight_min": -15.478605270385742,
      "activations/layer20_attention_weight_max": 30.022918701171875,
      "activations/layer20_attention_weight_min": -25.836795806884766,
      "activations/layer21_attention_weight_max": 30.008586883544922,
      "activations/layer21_attention_weight_min": -25.197906494140625,
      "activations/layer22_attention_weight_max": 44.76321792602539,
      "activations/layer22_attention_weight_min": -32.9432487487793,
      "activations/layer23_attention_weight_max": 30.671588897705078,
      "activations/layer23_attention_weight_min": -26.16423797607422,
      "activations/layer2_attention_weight_max": 36.019412994384766,
      "activations/layer2_attention_weight_min": -37.060420989990234,
      "activations/layer3_attention_weight_max": 107.93156433105469,
      "activations/layer3_attention_weight_min": -111.57778930664062,
      "activations/layer4_attention_weight_max": 113.22322845458984,
      "activations/layer4_attention_weight_min": -113.69998931884766,
      "activations/layer5_attention_weight_max": 49.79830551147461,
      "activations/layer5_attention_weight_min": -63.720970153808594,
      "activations/layer6_attention_weight_max": 47.721641540527344,
      "activations/layer6_attention_weight_min": -47.247291564941406,
      "activations/layer7_attention_weight_max": 100.25210571289062,
      "activations/layer7_attention_weight_min": -101.49987030029297,
      "activations/layer8_attention_weight_max": 44.685272216796875,
      "activations/layer8_attention_weight_min": -47.286930084228516,
      "activations/layer9_attention_weight_max": 34.335323333740234,
      "activations/layer9_attention_weight_min": -36.928131103515625,
      "epoch": 23.16,
      "learning_rate": 6.659090909090908e-07,
      "loss": 2.7083,
      "step": 398550
    },
    {
      "activations/layer0_attention_weight_max": 17.10170555114746,
      "activations/layer0_attention_weight_min": -14.57494831085205,
      "activations/layer10_attention_weight_max": 36.54278564453125,
      "activations/layer10_attention_weight_min": -35.67429733276367,
      "activations/layer11_attention_weight_max": 35.806007385253906,
      "activations/layer11_attention_weight_min": -37.18716049194336,
      "activations/layer12_attention_weight_max": 27.89238739013672,
      "activations/layer12_attention_weight_min": -27.646371841430664,
      "activations/layer13_attention_weight_max": 41.35826873779297,
      "activations/layer13_attention_weight_min": -36.14842224121094,
      "activations/layer14_attention_weight_max": 44.646522521972656,
      "activations/layer14_attention_weight_min": -37.78208923339844,
      "activations/layer15_attention_weight_max": 39.6348876953125,
      "activations/layer15_attention_weight_min": -36.07177734375,
      "activations/layer16_attention_weight_max": 29.979490280151367,
      "activations/layer16_attention_weight_min": -28.12688636779785,
      "activations/layer17_attention_weight_max": 32.23103713989258,
      "activations/layer17_attention_weight_min": -25.96298599243164,
      "activations/layer18_attention_weight_max": 34.67284393310547,
      "activations/layer18_attention_weight_min": -27.563072204589844,
      "activations/layer19_attention_weight_max": 35.536659240722656,
      "activations/layer19_attention_weight_min": -31.798664093017578,
      "activations/layer1_attention_weight_max": 15.473328590393066,
      "activations/layer1_attention_weight_min": -14.79207992553711,
      "activations/layer20_attention_weight_max": 30.072492599487305,
      "activations/layer20_attention_weight_min": -27.122852325439453,
      "activations/layer21_attention_weight_max": 30.489131927490234,
      "activations/layer21_attention_weight_min": -28.29244041442871,
      "activations/layer22_attention_weight_max": 43.52919387817383,
      "activations/layer22_attention_weight_min": -33.070465087890625,
      "activations/layer23_attention_weight_max": 33.17827606201172,
      "activations/layer23_attention_weight_min": -26.914268493652344,
      "activations/layer2_attention_weight_max": 34.19480514526367,
      "activations/layer2_attention_weight_min": -34.75471115112305,
      "activations/layer3_attention_weight_max": 110.46907043457031,
      "activations/layer3_attention_weight_min": -112.85368347167969,
      "activations/layer4_attention_weight_max": 111.534912109375,
      "activations/layer4_attention_weight_min": -117.83448791503906,
      "activations/layer5_attention_weight_max": 51.13898849487305,
      "activations/layer5_attention_weight_min": -65.22987365722656,
      "activations/layer6_attention_weight_max": 45.93964767456055,
      "activations/layer6_attention_weight_min": -49.99413299560547,
      "activations/layer7_attention_weight_max": 100.77017974853516,
      "activations/layer7_attention_weight_min": -105.5425033569336,
      "activations/layer8_attention_weight_max": 44.31471252441406,
      "activations/layer8_attention_weight_min": -46.85829544067383,
      "activations/layer9_attention_weight_max": 32.481746673583984,
      "activations/layer9_attention_weight_min": -34.3895149230957,
      "epoch": 23.16,
      "learning_rate": 6.469696969696969e-07,
      "loss": 2.6836,
      "step": 398600
    },
    {
      "activations/layer0_attention_weight_max": 17.089702606201172,
      "activations/layer0_attention_weight_min": -16.340045928955078,
      "activations/layer10_attention_weight_max": 33.19416427612305,
      "activations/layer10_attention_weight_min": -35.78010940551758,
      "activations/layer11_attention_weight_max": 34.207786560058594,
      "activations/layer11_attention_weight_min": -35.750709533691406,
      "activations/layer12_attention_weight_max": 26.108177185058594,
      "activations/layer12_attention_weight_min": -27.929298400878906,
      "activations/layer13_attention_weight_max": 37.38444519042969,
      "activations/layer13_attention_weight_min": -35.78290939331055,
      "activations/layer14_attention_weight_max": 41.37664031982422,
      "activations/layer14_attention_weight_min": -38.718841552734375,
      "activations/layer15_attention_weight_max": 38.422420501708984,
      "activations/layer15_attention_weight_min": -37.05299377441406,
      "activations/layer16_attention_weight_max": 29.212949752807617,
      "activations/layer16_attention_weight_min": -28.679916381835938,
      "activations/layer17_attention_weight_max": 28.898942947387695,
      "activations/layer17_attention_weight_min": -25.14845085144043,
      "activations/layer18_attention_weight_max": 30.566669464111328,
      "activations/layer18_attention_weight_min": -23.250782012939453,
      "activations/layer19_attention_weight_max": 31.618576049804688,
      "activations/layer19_attention_weight_min": -31.36219596862793,
      "activations/layer1_attention_weight_max": 15.859078407287598,
      "activations/layer1_attention_weight_min": -16.441452026367188,
      "activations/layer20_attention_weight_max": 27.566917419433594,
      "activations/layer20_attention_weight_min": -24.498977661132812,
      "activations/layer21_attention_weight_max": 26.604110717773438,
      "activations/layer21_attention_weight_min": -22.5757999420166,
      "activations/layer22_attention_weight_max": 36.21529006958008,
      "activations/layer22_attention_weight_min": -27.926708221435547,
      "activations/layer23_attention_weight_max": 30.889949798583984,
      "activations/layer23_attention_weight_min": -22.882675170898438,
      "activations/layer2_attention_weight_max": 34.110069274902344,
      "activations/layer2_attention_weight_min": -33.96968078613281,
      "activations/layer3_attention_weight_max": 108.8852767944336,
      "activations/layer3_attention_weight_min": -110.58663177490234,
      "activations/layer4_attention_weight_max": 116.2070083618164,
      "activations/layer4_attention_weight_min": -116.9451904296875,
      "activations/layer5_attention_weight_max": 55.7400016784668,
      "activations/layer5_attention_weight_min": -66.25088500976562,
      "activations/layer6_attention_weight_max": 46.48372268676758,
      "activations/layer6_attention_weight_min": -52.05201721191406,
      "activations/layer7_attention_weight_max": 103.3858413696289,
      "activations/layer7_attention_weight_min": -100.62984466552734,
      "activations/layer8_attention_weight_max": 42.92970657348633,
      "activations/layer8_attention_weight_min": -45.11149978637695,
      "activations/layer9_attention_weight_max": 31.869688034057617,
      "activations/layer9_attention_weight_min": -36.13729476928711,
      "epoch": 23.16,
      "learning_rate": 6.28030303030303e-07,
      "loss": 2.7055,
      "step": 398650
    },
    {
      "activations/layer0_attention_weight_max": 16.308639526367188,
      "activations/layer0_attention_weight_min": -13.513663291931152,
      "activations/layer10_attention_weight_max": 39.82021713256836,
      "activations/layer10_attention_weight_min": -36.149253845214844,
      "activations/layer11_attention_weight_max": 38.49419403076172,
      "activations/layer11_attention_weight_min": -36.97028350830078,
      "activations/layer12_attention_weight_max": 27.122398376464844,
      "activations/layer12_attention_weight_min": -28.62519645690918,
      "activations/layer13_attention_weight_max": 42.2158088684082,
      "activations/layer13_attention_weight_min": -37.744659423828125,
      "activations/layer14_attention_weight_max": 44.78107452392578,
      "activations/layer14_attention_weight_min": -40.432762145996094,
      "activations/layer15_attention_weight_max": 46.12606430053711,
      "activations/layer15_attention_weight_min": -37.97012710571289,
      "activations/layer16_attention_weight_max": 29.67166519165039,
      "activations/layer16_attention_weight_min": -28.541025161743164,
      "activations/layer17_attention_weight_max": 30.01264190673828,
      "activations/layer17_attention_weight_min": -26.208284378051758,
      "activations/layer18_attention_weight_max": 31.728055953979492,
      "activations/layer18_attention_weight_min": -25.689470291137695,
      "activations/layer19_attention_weight_max": 35.87006759643555,
      "activations/layer19_attention_weight_min": -33.15442657470703,
      "activations/layer1_attention_weight_max": 16.156455993652344,
      "activations/layer1_attention_weight_min": -14.810164451599121,
      "activations/layer20_attention_weight_max": 28.068859100341797,
      "activations/layer20_attention_weight_min": -26.37636947631836,
      "activations/layer21_attention_weight_max": 28.81511116027832,
      "activations/layer21_attention_weight_min": -24.96373748779297,
      "activations/layer22_attention_weight_max": 44.9370002746582,
      "activations/layer22_attention_weight_min": -31.791231155395508,
      "activations/layer23_attention_weight_max": 35.41405487060547,
      "activations/layer23_attention_weight_min": -24.240615844726562,
      "activations/layer2_attention_weight_max": 34.202178955078125,
      "activations/layer2_attention_weight_min": -33.26885986328125,
      "activations/layer3_attention_weight_max": 113.2961196899414,
      "activations/layer3_attention_weight_min": -108.05966186523438,
      "activations/layer4_attention_weight_max": 114.55168914794922,
      "activations/layer4_attention_weight_min": -118.70977020263672,
      "activations/layer5_attention_weight_max": 54.623409271240234,
      "activations/layer5_attention_weight_min": -64.5526123046875,
      "activations/layer6_attention_weight_max": 47.31816101074219,
      "activations/layer6_attention_weight_min": -48.10985565185547,
      "activations/layer7_attention_weight_max": 97.9217529296875,
      "activations/layer7_attention_weight_min": -98.70115661621094,
      "activations/layer8_attention_weight_max": 45.01485061645508,
      "activations/layer8_attention_weight_min": -45.26256561279297,
      "activations/layer9_attention_weight_max": 35.164615631103516,
      "activations/layer9_attention_weight_min": -35.2619514465332,
      "epoch": 23.17,
      "learning_rate": 6.090909090909091e-07,
      "loss": 2.697,
      "step": 398700
    },
    {
      "activations/layer0_attention_weight_max": 17.1950740814209,
      "activations/layer0_attention_weight_min": -14.815510749816895,
      "activations/layer10_attention_weight_max": 34.7244873046875,
      "activations/layer10_attention_weight_min": -38.25183868408203,
      "activations/layer11_attention_weight_max": 35.953407287597656,
      "activations/layer11_attention_weight_min": -36.1805419921875,
      "activations/layer12_attention_weight_max": 26.658370971679688,
      "activations/layer12_attention_weight_min": -27.934349060058594,
      "activations/layer13_attention_weight_max": 39.873939514160156,
      "activations/layer13_attention_weight_min": -36.586212158203125,
      "activations/layer14_attention_weight_max": 46.93765640258789,
      "activations/layer14_attention_weight_min": -39.576114654541016,
      "activations/layer15_attention_weight_max": 40.815452575683594,
      "activations/layer15_attention_weight_min": -35.975040435791016,
      "activations/layer16_attention_weight_max": 30.91451072692871,
      "activations/layer16_attention_weight_min": -28.786611557006836,
      "activations/layer17_attention_weight_max": 31.90174674987793,
      "activations/layer17_attention_weight_min": -26.356861114501953,
      "activations/layer18_attention_weight_max": 35.767677307128906,
      "activations/layer18_attention_weight_min": -26.197101593017578,
      "activations/layer19_attention_weight_max": 34.875572204589844,
      "activations/layer19_attention_weight_min": -34.230403900146484,
      "activations/layer1_attention_weight_max": 17.319068908691406,
      "activations/layer1_attention_weight_min": -18.833141326904297,
      "activations/layer20_attention_weight_max": 33.97085189819336,
      "activations/layer20_attention_weight_min": -26.005098342895508,
      "activations/layer21_attention_weight_max": 31.5715389251709,
      "activations/layer21_attention_weight_min": -25.9492130279541,
      "activations/layer22_attention_weight_max": 41.390899658203125,
      "activations/layer22_attention_weight_min": -30.11154556274414,
      "activations/layer23_attention_weight_max": 34.370582580566406,
      "activations/layer23_attention_weight_min": -25.730789184570312,
      "activations/layer2_attention_weight_max": 33.85315704345703,
      "activations/layer2_attention_weight_min": -34.52291488647461,
      "activations/layer3_attention_weight_max": 114.29646301269531,
      "activations/layer3_attention_weight_min": -113.33387756347656,
      "activations/layer4_attention_weight_max": 118.34801483154297,
      "activations/layer4_attention_weight_min": -121.7161865234375,
      "activations/layer5_attention_weight_max": 55.8775749206543,
      "activations/layer5_attention_weight_min": -68.87025451660156,
      "activations/layer6_attention_weight_max": 48.13022232055664,
      "activations/layer6_attention_weight_min": -51.628196716308594,
      "activations/layer7_attention_weight_max": 100.25582122802734,
      "activations/layer7_attention_weight_min": -109.97396087646484,
      "activations/layer8_attention_weight_max": 42.88452911376953,
      "activations/layer8_attention_weight_min": -47.84844207763672,
      "activations/layer9_attention_weight_max": 33.30990219116211,
      "activations/layer9_attention_weight_min": -37.95486068725586,
      "epoch": 23.17,
      "learning_rate": 5.901515151515151e-07,
      "loss": 2.6827,
      "step": 398750
    },
    {
      "activations/layer0_attention_weight_max": 16.827287673950195,
      "activations/layer0_attention_weight_min": -14.731621742248535,
      "activations/layer10_attention_weight_max": 36.435829162597656,
      "activations/layer10_attention_weight_min": -37.72661590576172,
      "activations/layer11_attention_weight_max": 37.412445068359375,
      "activations/layer11_attention_weight_min": -35.415775299072266,
      "activations/layer12_attention_weight_max": 28.913105010986328,
      "activations/layer12_attention_weight_min": -28.769906997680664,
      "activations/layer13_attention_weight_max": 40.22506332397461,
      "activations/layer13_attention_weight_min": -39.371055603027344,
      "activations/layer14_attention_weight_max": 48.13337326049805,
      "activations/layer14_attention_weight_min": -41.0521354675293,
      "activations/layer15_attention_weight_max": 42.79029846191406,
      "activations/layer15_attention_weight_min": -38.21493911743164,
      "activations/layer16_attention_weight_max": 31.496137619018555,
      "activations/layer16_attention_weight_min": -30.492048263549805,
      "activations/layer17_attention_weight_max": 29.8062686920166,
      "activations/layer17_attention_weight_min": -27.15980339050293,
      "activations/layer18_attention_weight_max": 33.53139877319336,
      "activations/layer18_attention_weight_min": -28.57632064819336,
      "activations/layer19_attention_weight_max": 34.97172927856445,
      "activations/layer19_attention_weight_min": -33.44462585449219,
      "activations/layer1_attention_weight_max": 15.424301147460938,
      "activations/layer1_attention_weight_min": -14.776836395263672,
      "activations/layer20_attention_weight_max": 30.296079635620117,
      "activations/layer20_attention_weight_min": -26.3659610748291,
      "activations/layer21_attention_weight_max": 28.08571434020996,
      "activations/layer21_attention_weight_min": -25.803503036499023,
      "activations/layer22_attention_weight_max": 39.40006637573242,
      "activations/layer22_attention_weight_min": -35.21665954589844,
      "activations/layer23_attention_weight_max": 31.78524398803711,
      "activations/layer23_attention_weight_min": -25.92584228515625,
      "activations/layer2_attention_weight_max": 34.65766525268555,
      "activations/layer2_attention_weight_min": -34.103981018066406,
      "activations/layer3_attention_weight_max": 110.47516632080078,
      "activations/layer3_attention_weight_min": -110.49707794189453,
      "activations/layer4_attention_weight_max": 116.6856918334961,
      "activations/layer4_attention_weight_min": -117.42436981201172,
      "activations/layer5_attention_weight_max": 53.375179290771484,
      "activations/layer5_attention_weight_min": -63.098182678222656,
      "activations/layer6_attention_weight_max": 47.084842681884766,
      "activations/layer6_attention_weight_min": -51.10544204711914,
      "activations/layer7_attention_weight_max": 101.4289321899414,
      "activations/layer7_attention_weight_min": -105.98731231689453,
      "activations/layer8_attention_weight_max": 44.34526443481445,
      "activations/layer8_attention_weight_min": -47.59440994262695,
      "activations/layer9_attention_weight_max": 33.31840133666992,
      "activations/layer9_attention_weight_min": -36.80894088745117,
      "epoch": 23.17,
      "learning_rate": 5.712121212121211e-07,
      "loss": 2.7009,
      "step": 398800
    },
    {
      "activations/layer0_attention_weight_max": 16.312301635742188,
      "activations/layer0_attention_weight_min": -14.64302921295166,
      "activations/layer10_attention_weight_max": 36.45269775390625,
      "activations/layer10_attention_weight_min": -37.739994049072266,
      "activations/layer11_attention_weight_max": 35.916751861572266,
      "activations/layer11_attention_weight_min": -35.88978576660156,
      "activations/layer12_attention_weight_max": 29.827564239501953,
      "activations/layer12_attention_weight_min": -27.52627182006836,
      "activations/layer13_attention_weight_max": 43.308712005615234,
      "activations/layer13_attention_weight_min": -37.821407318115234,
      "activations/layer14_attention_weight_max": 42.89461898803711,
      "activations/layer14_attention_weight_min": -42.036216735839844,
      "activations/layer15_attention_weight_max": 38.49435806274414,
      "activations/layer15_attention_weight_min": -39.372825622558594,
      "activations/layer16_attention_weight_max": 28.03712272644043,
      "activations/layer16_attention_weight_min": -30.47898292541504,
      "activations/layer17_attention_weight_max": 29.7253360748291,
      "activations/layer17_attention_weight_min": -27.70194435119629,
      "activations/layer18_attention_weight_max": 33.44863510131836,
      "activations/layer18_attention_weight_min": -27.84309196472168,
      "activations/layer19_attention_weight_max": 35.4791145324707,
      "activations/layer19_attention_weight_min": -35.65620803833008,
      "activations/layer1_attention_weight_max": 15.879781723022461,
      "activations/layer1_attention_weight_min": -15.937363624572754,
      "activations/layer20_attention_weight_max": 30.436901092529297,
      "activations/layer20_attention_weight_min": -27.974449157714844,
      "activations/layer21_attention_weight_max": 32.690399169921875,
      "activations/layer21_attention_weight_min": -26.3475399017334,
      "activations/layer22_attention_weight_max": 40.707828521728516,
      "activations/layer22_attention_weight_min": -33.84718322753906,
      "activations/layer23_attention_weight_max": 33.10483932495117,
      "activations/layer23_attention_weight_min": -26.540367126464844,
      "activations/layer2_attention_weight_max": 35.63214874267578,
      "activations/layer2_attention_weight_min": -34.52922058105469,
      "activations/layer3_attention_weight_max": 111.27691650390625,
      "activations/layer3_attention_weight_min": -112.65711975097656,
      "activations/layer4_attention_weight_max": 118.01433563232422,
      "activations/layer4_attention_weight_min": -118.79542541503906,
      "activations/layer5_attention_weight_max": 54.99205017089844,
      "activations/layer5_attention_weight_min": -63.35578918457031,
      "activations/layer6_attention_weight_max": 47.52491760253906,
      "activations/layer6_attention_weight_min": -52.12333679199219,
      "activations/layer7_attention_weight_max": 105.96906280517578,
      "activations/layer7_attention_weight_min": -104.19457244873047,
      "activations/layer8_attention_weight_max": 43.835636138916016,
      "activations/layer8_attention_weight_min": -46.990840911865234,
      "activations/layer9_attention_weight_max": 33.94961929321289,
      "activations/layer9_attention_weight_min": -37.525421142578125,
      "epoch": 23.18,
      "learning_rate": 5.522727272727272e-07,
      "loss": 2.6969,
      "step": 398850
    },
    {
      "activations/layer0_attention_weight_max": 17.592021942138672,
      "activations/layer0_attention_weight_min": -14.430445671081543,
      "activations/layer10_attention_weight_max": 37.52043533325195,
      "activations/layer10_attention_weight_min": -36.154170989990234,
      "activations/layer11_attention_weight_max": 37.83943176269531,
      "activations/layer11_attention_weight_min": -37.460235595703125,
      "activations/layer12_attention_weight_max": 28.076078414916992,
      "activations/layer12_attention_weight_min": -28.892135620117188,
      "activations/layer13_attention_weight_max": 40.86394119262695,
      "activations/layer13_attention_weight_min": -38.715538024902344,
      "activations/layer14_attention_weight_max": 44.20952224731445,
      "activations/layer14_attention_weight_min": -42.28295135498047,
      "activations/layer15_attention_weight_max": 40.71592330932617,
      "activations/layer15_attention_weight_min": -38.52579879760742,
      "activations/layer16_attention_weight_max": 30.950634002685547,
      "activations/layer16_attention_weight_min": -30.53432273864746,
      "activations/layer17_attention_weight_max": 31.854652404785156,
      "activations/layer17_attention_weight_min": -27.61691665649414,
      "activations/layer18_attention_weight_max": 35.1278190612793,
      "activations/layer18_attention_weight_min": -28.22418785095215,
      "activations/layer19_attention_weight_max": 33.19514846801758,
      "activations/layer19_attention_weight_min": -33.71427536010742,
      "activations/layer1_attention_weight_max": 15.678886413574219,
      "activations/layer1_attention_weight_min": -14.895241737365723,
      "activations/layer20_attention_weight_max": 30.232192993164062,
      "activations/layer20_attention_weight_min": -26.389127731323242,
      "activations/layer21_attention_weight_max": 33.121280670166016,
      "activations/layer21_attention_weight_min": -25.32012367248535,
      "activations/layer22_attention_weight_max": 44.78190994262695,
      "activations/layer22_attention_weight_min": -33.75892639160156,
      "activations/layer23_attention_weight_max": 33.1088752746582,
      "activations/layer23_attention_weight_min": -27.04462432861328,
      "activations/layer2_attention_weight_max": 34.77543258666992,
      "activations/layer2_attention_weight_min": -34.368614196777344,
      "activations/layer3_attention_weight_max": 107.51637268066406,
      "activations/layer3_attention_weight_min": -109.36769104003906,
      "activations/layer4_attention_weight_max": 112.37137603759766,
      "activations/layer4_attention_weight_min": -115.6844711303711,
      "activations/layer5_attention_weight_max": 50.38636779785156,
      "activations/layer5_attention_weight_min": -65.49954986572266,
      "activations/layer6_attention_weight_max": 47.470584869384766,
      "activations/layer6_attention_weight_min": -49.56098175048828,
      "activations/layer7_attention_weight_max": 101.53431701660156,
      "activations/layer7_attention_weight_min": -106.08775329589844,
      "activations/layer8_attention_weight_max": 46.24153518676758,
      "activations/layer8_attention_weight_min": -50.3699836730957,
      "activations/layer9_attention_weight_max": 35.15190505981445,
      "activations/layer9_attention_weight_min": -36.259769439697266,
      "epoch": 23.18,
      "learning_rate": 5.333333333333333e-07,
      "loss": 2.6949,
      "step": 398900
    },
    {
      "activations/layer0_attention_weight_max": 16.783098220825195,
      "activations/layer0_attention_weight_min": -13.746851921081543,
      "activations/layer10_attention_weight_max": 36.25736618041992,
      "activations/layer10_attention_weight_min": -37.39822769165039,
      "activations/layer11_attention_weight_max": 35.67486572265625,
      "activations/layer11_attention_weight_min": -36.3610725402832,
      "activations/layer12_attention_weight_max": 25.658727645874023,
      "activations/layer12_attention_weight_min": -27.45673179626465,
      "activations/layer13_attention_weight_max": 37.55504608154297,
      "activations/layer13_attention_weight_min": -36.19367218017578,
      "activations/layer14_attention_weight_max": 43.948707580566406,
      "activations/layer14_attention_weight_min": -38.484920501708984,
      "activations/layer15_attention_weight_max": 40.35789108276367,
      "activations/layer15_attention_weight_min": -37.751747131347656,
      "activations/layer16_attention_weight_max": 29.042797088623047,
      "activations/layer16_attention_weight_min": -30.778987884521484,
      "activations/layer17_attention_weight_max": 28.530115127563477,
      "activations/layer17_attention_weight_min": -26.481124877929688,
      "activations/layer18_attention_weight_max": 34.37864685058594,
      "activations/layer18_attention_weight_min": -25.639263153076172,
      "activations/layer19_attention_weight_max": 33.03036117553711,
      "activations/layer19_attention_weight_min": -31.284408569335938,
      "activations/layer1_attention_weight_max": 15.537965774536133,
      "activations/layer1_attention_weight_min": -14.307806015014648,
      "activations/layer20_attention_weight_max": 29.533140182495117,
      "activations/layer20_attention_weight_min": -24.349645614624023,
      "activations/layer21_attention_weight_max": 26.579851150512695,
      "activations/layer21_attention_weight_min": -24.418487548828125,
      "activations/layer22_attention_weight_max": 39.17844009399414,
      "activations/layer22_attention_weight_min": -31.013465881347656,
      "activations/layer23_attention_weight_max": 32.13512420654297,
      "activations/layer23_attention_weight_min": -25.843887329101562,
      "activations/layer2_attention_weight_max": 34.67534637451172,
      "activations/layer2_attention_weight_min": -33.64795684814453,
      "activations/layer3_attention_weight_max": 110.3644027709961,
      "activations/layer3_attention_weight_min": -107.48509216308594,
      "activations/layer4_attention_weight_max": 112.89295959472656,
      "activations/layer4_attention_weight_min": -114.1505355834961,
      "activations/layer5_attention_weight_max": 56.52172088623047,
      "activations/layer5_attention_weight_min": -66.05738830566406,
      "activations/layer6_attention_weight_max": 46.05295944213867,
      "activations/layer6_attention_weight_min": -49.481441497802734,
      "activations/layer7_attention_weight_max": 102.68270111083984,
      "activations/layer7_attention_weight_min": -101.67552185058594,
      "activations/layer8_attention_weight_max": 42.38322448730469,
      "activations/layer8_attention_weight_min": -46.4136848449707,
      "activations/layer9_attention_weight_max": 32.4949836730957,
      "activations/layer9_attention_weight_min": -35.99570846557617,
      "epoch": 23.18,
      "learning_rate": 5.143939393939393e-07,
      "loss": 2.6905,
      "step": 398950
    },
    {
      "activations/layer0_attention_weight_max": 16.994020462036133,
      "activations/layer0_attention_weight_min": -14.82564640045166,
      "activations/layer10_attention_weight_max": 33.04521560668945,
      "activations/layer10_attention_weight_min": -35.63933181762695,
      "activations/layer11_attention_weight_max": 36.10600280761719,
      "activations/layer11_attention_weight_min": -37.6427001953125,
      "activations/layer12_attention_weight_max": 26.448566436767578,
      "activations/layer12_attention_weight_min": -27.64963150024414,
      "activations/layer13_attention_weight_max": 37.169593811035156,
      "activations/layer13_attention_weight_min": -36.11796188354492,
      "activations/layer14_attention_weight_max": 44.58454513549805,
      "activations/layer14_attention_weight_min": -37.61219787597656,
      "activations/layer15_attention_weight_max": 41.46318054199219,
      "activations/layer15_attention_weight_min": -35.655670166015625,
      "activations/layer16_attention_weight_max": 31.251005172729492,
      "activations/layer16_attention_weight_min": -27.820207595825195,
      "activations/layer17_attention_weight_max": 28.7585391998291,
      "activations/layer17_attention_weight_min": -25.506738662719727,
      "activations/layer18_attention_weight_max": 31.745319366455078,
      "activations/layer18_attention_weight_min": -26.10383415222168,
      "activations/layer19_attention_weight_max": 34.620487213134766,
      "activations/layer19_attention_weight_min": -31.244632720947266,
      "activations/layer1_attention_weight_max": 15.76357364654541,
      "activations/layer1_attention_weight_min": -14.75372314453125,
      "activations/layer20_attention_weight_max": 28.649940490722656,
      "activations/layer20_attention_weight_min": -26.690784454345703,
      "activations/layer21_attention_weight_max": 28.60546875,
      "activations/layer21_attention_weight_min": -26.85177993774414,
      "activations/layer22_attention_weight_max": 39.412933349609375,
      "activations/layer22_attention_weight_min": -32.28782653808594,
      "activations/layer23_attention_weight_max": 30.560928344726562,
      "activations/layer23_attention_weight_min": -25.999828338623047,
      "activations/layer2_attention_weight_max": 32.83715057373047,
      "activations/layer2_attention_weight_min": -32.952598571777344,
      "activations/layer3_attention_weight_max": 108.78125,
      "activations/layer3_attention_weight_min": -109.38019561767578,
      "activations/layer4_attention_weight_max": 110.2666244506836,
      "activations/layer4_attention_weight_min": -114.9599838256836,
      "activations/layer5_attention_weight_max": 50.06943893432617,
      "activations/layer5_attention_weight_min": -63.83934020996094,
      "activations/layer6_attention_weight_max": 44.047725677490234,
      "activations/layer6_attention_weight_min": -48.21150588989258,
      "activations/layer7_attention_weight_max": 96.54026794433594,
      "activations/layer7_attention_weight_min": -101.12983703613281,
      "activations/layer8_attention_weight_max": 40.70724105834961,
      "activations/layer8_attention_weight_min": -46.41912841796875,
      "activations/layer9_attention_weight_max": 31.892911911010742,
      "activations/layer9_attention_weight_min": -35.966609954833984,
      "epoch": 23.18,
      "learning_rate": 4.954545454545454e-07,
      "loss": 2.6997,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_loss": 2.658203125,
      "eval_runtime": 8.4632,
      "eval_samples_per_second": 507.375,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_openwebtext_loss": 2.658203125,
      "eval_openwebtext_ppl": 14.2706235200274,
      "eval_openwebtext_runtime": 8.4632,
      "eval_openwebtext_samples_per_second": 507.375,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_wikitext_loss": 2.890625,
      "eval_wikitext_ppl": 18.00455893510183,
      "eval_wikitext_runtime": 1.9171,
      "eval_wikitext_samples_per_second": 237.86,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_lambada_loss": 2.29296875,
      "eval_lambada_ppl": 9.904297461775021,
      "eval_lambada_runtime": 9.5384,
      "eval_lambada_samples_per_second": 510.462,
      "step": 399000
    },
    {
      "activations/layer0_attention_weight_max": 17.069984436035156,
      "activations/layer0_attention_weight_min": -13.721738815307617,
      "activations/layer10_attention_weight_max": 46.5604362487793,
      "activations/layer10_attention_weight_min": -42.52735137939453,
      "activations/layer11_attention_weight_max": 44.14274597167969,
      "activations/layer11_attention_weight_min": -44.170448303222656,
      "activations/layer12_attention_weight_max": 29.701871871948242,
      "activations/layer12_attention_weight_min": -28.59569549560547,
      "activations/layer13_attention_weight_max": 42.15214538574219,
      "activations/layer13_attention_weight_min": -37.01986312866211,
      "activations/layer14_attention_weight_max": 46.9578971862793,
      "activations/layer14_attention_weight_min": -39.15837860107422,
      "activations/layer15_attention_weight_max": 43.86812210083008,
      "activations/layer15_attention_weight_min": -38.29122543334961,
      "activations/layer16_attention_weight_max": 33.41376876831055,
      "activations/layer16_attention_weight_min": -27.94376564025879,
      "activations/layer17_attention_weight_max": 31.195133209228516,
      "activations/layer17_attention_weight_min": -25.437326431274414,
      "activations/layer18_attention_weight_max": 33.7981071472168,
      "activations/layer18_attention_weight_min": -26.951608657836914,
      "activations/layer19_attention_weight_max": 39.544944763183594,
      "activations/layer19_attention_weight_min": -32.57539749145508,
      "activations/layer1_attention_weight_max": 15.546259880065918,
      "activations/layer1_attention_weight_min": -15.620025634765625,
      "activations/layer20_attention_weight_max": 29.570545196533203,
      "activations/layer20_attention_weight_min": -25.606870651245117,
      "activations/layer21_attention_weight_max": 29.41037368774414,
      "activations/layer21_attention_weight_min": -27.51937484741211,
      "activations/layer22_attention_weight_max": 46.262779235839844,
      "activations/layer22_attention_weight_min": -39.80309295654297,
      "activations/layer23_attention_weight_max": 33.959983825683594,
      "activations/layer23_attention_weight_min": -30.949790954589844,
      "activations/layer2_attention_weight_max": 34.95732498168945,
      "activations/layer2_attention_weight_min": -34.89145278930664,
      "activations/layer3_attention_weight_max": 111.90997314453125,
      "activations/layer3_attention_weight_min": -113.01609802246094,
      "activations/layer4_attention_weight_max": 112.76519775390625,
      "activations/layer4_attention_weight_min": -113.69661712646484,
      "activations/layer5_attention_weight_max": 52.26424789428711,
      "activations/layer5_attention_weight_min": -63.11713790893555,
      "activations/layer6_attention_weight_max": 46.33678436279297,
      "activations/layer6_attention_weight_min": -49.74679946899414,
      "activations/layer7_attention_weight_max": 121.84960174560547,
      "activations/layer7_attention_weight_min": -102.35232543945312,
      "activations/layer8_attention_weight_max": 53.97053909301758,
      "activations/layer8_attention_weight_min": -52.47124481201172,
      "activations/layer9_attention_weight_max": 38.48529052734375,
      "activations/layer9_attention_weight_min": -42.40626907348633,
      "epoch": 23.19,
      "learning_rate": 4.765151515151515e-07,
      "loss": 2.6919,
      "step": 399050
    },
    {
      "activations/layer0_attention_weight_max": 16.861146926879883,
      "activations/layer0_attention_weight_min": -15.16860580444336,
      "activations/layer10_attention_weight_max": 34.407962799072266,
      "activations/layer10_attention_weight_min": -35.64582824707031,
      "activations/layer11_attention_weight_max": 34.3879508972168,
      "activations/layer11_attention_weight_min": -35.458465576171875,
      "activations/layer12_attention_weight_max": 26.452777862548828,
      "activations/layer12_attention_weight_min": -26.991905212402344,
      "activations/layer13_attention_weight_max": 40.745887756347656,
      "activations/layer13_attention_weight_min": -36.36033248901367,
      "activations/layer14_attention_weight_max": 47.146339416503906,
      "activations/layer14_attention_weight_min": -39.29972457885742,
      "activations/layer15_attention_weight_max": 45.91636657714844,
      "activations/layer15_attention_weight_min": -37.15997314453125,
      "activations/layer16_attention_weight_max": 30.760480880737305,
      "activations/layer16_attention_weight_min": -30.42981719970703,
      "activations/layer17_attention_weight_max": 31.14894676208496,
      "activations/layer17_attention_weight_min": -28.69334602355957,
      "activations/layer18_attention_weight_max": 33.68602752685547,
      "activations/layer18_attention_weight_min": -27.88888168334961,
      "activations/layer19_attention_weight_max": 39.681758880615234,
      "activations/layer19_attention_weight_min": -39.1419677734375,
      "activations/layer1_attention_weight_max": 15.61292552947998,
      "activations/layer1_attention_weight_min": -16.21683120727539,
      "activations/layer20_attention_weight_max": 34.737911224365234,
      "activations/layer20_attention_weight_min": -28.236387252807617,
      "activations/layer21_attention_weight_max": 33.73958206176758,
      "activations/layer21_attention_weight_min": -27.469024658203125,
      "activations/layer22_attention_weight_max": 45.13343048095703,
      "activations/layer22_attention_weight_min": -35.406612396240234,
      "activations/layer23_attention_weight_max": 34.10456848144531,
      "activations/layer23_attention_weight_min": -27.678203582763672,
      "activations/layer2_attention_weight_max": 33.843994140625,
      "activations/layer2_attention_weight_min": -33.42657470703125,
      "activations/layer3_attention_weight_max": 108.3652572631836,
      "activations/layer3_attention_weight_min": -108.03702545166016,
      "activations/layer4_attention_weight_max": 112.5392837524414,
      "activations/layer4_attention_weight_min": -114.79821014404297,
      "activations/layer5_attention_weight_max": 51.960540771484375,
      "activations/layer5_attention_weight_min": -63.62468719482422,
      "activations/layer6_attention_weight_max": 44.93745040893555,
      "activations/layer6_attention_weight_min": -47.69497299194336,
      "activations/layer7_attention_weight_max": 100.77435302734375,
      "activations/layer7_attention_weight_min": -100.08740997314453,
      "activations/layer8_attention_weight_max": 42.41756820678711,
      "activations/layer8_attention_weight_min": -45.1907844543457,
      "activations/layer9_attention_weight_max": 33.75786590576172,
      "activations/layer9_attention_weight_min": -36.06907272338867,
      "epoch": 23.19,
      "learning_rate": 4.5757575757575753e-07,
      "loss": 2.6781,
      "step": 399100
    },
    {
      "activations/layer0_attention_weight_max": 16.525924682617188,
      "activations/layer0_attention_weight_min": -13.711779594421387,
      "activations/layer10_attention_weight_max": 39.01392364501953,
      "activations/layer10_attention_weight_min": -40.726539611816406,
      "activations/layer11_attention_weight_max": 37.84104919433594,
      "activations/layer11_attention_weight_min": -41.785972595214844,
      "activations/layer12_attention_weight_max": 33.95690155029297,
      "activations/layer12_attention_weight_min": -32.976844787597656,
      "activations/layer13_attention_weight_max": 43.29524230957031,
      "activations/layer13_attention_weight_min": -39.8773078918457,
      "activations/layer14_attention_weight_max": 48.4622802734375,
      "activations/layer14_attention_weight_min": -43.34563064575195,
      "activations/layer15_attention_weight_max": 47.39480209350586,
      "activations/layer15_attention_weight_min": -45.60601043701172,
      "activations/layer16_attention_weight_max": 32.2271728515625,
      "activations/layer16_attention_weight_min": -29.3884220123291,
      "activations/layer17_attention_weight_max": 30.85707664489746,
      "activations/layer17_attention_weight_min": -28.56946563720703,
      "activations/layer18_attention_weight_max": 31.808401107788086,
      "activations/layer18_attention_weight_min": -27.404062271118164,
      "activations/layer19_attention_weight_max": 34.60609436035156,
      "activations/layer19_attention_weight_min": -31.457096099853516,
      "activations/layer1_attention_weight_max": 16.228775024414062,
      "activations/layer1_attention_weight_min": -14.961337089538574,
      "activations/layer20_attention_weight_max": 29.0062198638916,
      "activations/layer20_attention_weight_min": -25.49612045288086,
      "activations/layer21_attention_weight_max": 27.979393005371094,
      "activations/layer21_attention_weight_min": -23.86754608154297,
      "activations/layer22_attention_weight_max": 43.33522415161133,
      "activations/layer22_attention_weight_min": -33.65865707397461,
      "activations/layer23_attention_weight_max": 31.49203109741211,
      "activations/layer23_attention_weight_min": -25.48166275024414,
      "activations/layer2_attention_weight_max": 34.72627258300781,
      "activations/layer2_attention_weight_min": -34.3722038269043,
      "activations/layer3_attention_weight_max": 111.8626480102539,
      "activations/layer3_attention_weight_min": -111.98155212402344,
      "activations/layer4_attention_weight_max": 115.94805908203125,
      "activations/layer4_attention_weight_min": -120.25469970703125,
      "activations/layer5_attention_weight_max": 52.537864685058594,
      "activations/layer5_attention_weight_min": -63.10978317260742,
      "activations/layer6_attention_weight_max": 47.81429672241211,
      "activations/layer6_attention_weight_min": -48.735050201416016,
      "activations/layer7_attention_weight_max": 104.62380981445312,
      "activations/layer7_attention_weight_min": -117.46210479736328,
      "activations/layer8_attention_weight_max": 48.253604888916016,
      "activations/layer8_attention_weight_min": -52.152889251708984,
      "activations/layer9_attention_weight_max": 36.25961685180664,
      "activations/layer9_attention_weight_min": -40.196189880371094,
      "epoch": 23.19,
      "learning_rate": 4.386363636363636e-07,
      "loss": 2.699,
      "step": 399150
    },
    {
      "activations/layer0_attention_weight_max": 17.00357437133789,
      "activations/layer0_attention_weight_min": -14.552535057067871,
      "activations/layer10_attention_weight_max": 35.381832122802734,
      "activations/layer10_attention_weight_min": -35.17256546020508,
      "activations/layer11_attention_weight_max": 37.00128936767578,
      "activations/layer11_attention_weight_min": -36.404869079589844,
      "activations/layer12_attention_weight_max": 28.622201919555664,
      "activations/layer12_attention_weight_min": -27.241676330566406,
      "activations/layer13_attention_weight_max": 40.5040397644043,
      "activations/layer13_attention_weight_min": -38.53274154663086,
      "activations/layer14_attention_weight_max": 44.53987503051758,
      "activations/layer14_attention_weight_min": -39.341758728027344,
      "activations/layer15_attention_weight_max": 41.85321044921875,
      "activations/layer15_attention_weight_min": -39.49598693847656,
      "activations/layer16_attention_weight_max": 31.65825653076172,
      "activations/layer16_attention_weight_min": -30.052200317382812,
      "activations/layer17_attention_weight_max": 32.33430480957031,
      "activations/layer17_attention_weight_min": -26.852060317993164,
      "activations/layer18_attention_weight_max": 34.55516052246094,
      "activations/layer18_attention_weight_min": -26.273366928100586,
      "activations/layer19_attention_weight_max": 36.89373016357422,
      "activations/layer19_attention_weight_min": -31.784406661987305,
      "activations/layer1_attention_weight_max": 16.122947692871094,
      "activations/layer1_attention_weight_min": -15.652934074401855,
      "activations/layer20_attention_weight_max": 32.414188385009766,
      "activations/layer20_attention_weight_min": -25.963603973388672,
      "activations/layer21_attention_weight_max": 30.616384506225586,
      "activations/layer21_attention_weight_min": -26.235027313232422,
      "activations/layer22_attention_weight_max": 41.24776077270508,
      "activations/layer22_attention_weight_min": -32.20366287231445,
      "activations/layer23_attention_weight_max": 31.934476852416992,
      "activations/layer23_attention_weight_min": -26.563087463378906,
      "activations/layer2_attention_weight_max": 34.521854400634766,
      "activations/layer2_attention_weight_min": -34.19877243041992,
      "activations/layer3_attention_weight_max": 107.94161987304688,
      "activations/layer3_attention_weight_min": -112.42655944824219,
      "activations/layer4_attention_weight_max": 110.91072082519531,
      "activations/layer4_attention_weight_min": -118.91727447509766,
      "activations/layer5_attention_weight_max": 51.75652313232422,
      "activations/layer5_attention_weight_min": -62.76884460449219,
      "activations/layer6_attention_weight_max": 45.8265495300293,
      "activations/layer6_attention_weight_min": -50.457435607910156,
      "activations/layer7_attention_weight_max": 96.32948303222656,
      "activations/layer7_attention_weight_min": -109.23580169677734,
      "activations/layer8_attention_weight_max": 42.53507614135742,
      "activations/layer8_attention_weight_min": -46.817745208740234,
      "activations/layer9_attention_weight_max": 33.13854217529297,
      "activations/layer9_attention_weight_min": -35.05370330810547,
      "epoch": 23.2,
      "learning_rate": 4.196969696969697e-07,
      "loss": 2.6876,
      "step": 399200
    },
    {
      "activations/layer0_attention_weight_max": 18.158037185668945,
      "activations/layer0_attention_weight_min": -13.17428970336914,
      "activations/layer10_attention_weight_max": 39.22368240356445,
      "activations/layer10_attention_weight_min": -39.034576416015625,
      "activations/layer11_attention_weight_max": 39.9201774597168,
      "activations/layer11_attention_weight_min": -40.194610595703125,
      "activations/layer12_attention_weight_max": 30.18149185180664,
      "activations/layer12_attention_weight_min": -28.86701011657715,
      "activations/layer13_attention_weight_max": 43.889163970947266,
      "activations/layer13_attention_weight_min": -40.736873626708984,
      "activations/layer14_attention_weight_max": 44.579498291015625,
      "activations/layer14_attention_weight_min": -39.754302978515625,
      "activations/layer15_attention_weight_max": 44.223487854003906,
      "activations/layer15_attention_weight_min": -40.131099700927734,
      "activations/layer16_attention_weight_max": 31.33363914489746,
      "activations/layer16_attention_weight_min": -29.250253677368164,
      "activations/layer17_attention_weight_max": 33.11561965942383,
      "activations/layer17_attention_weight_min": -25.513559341430664,
      "activations/layer18_attention_weight_max": 33.84490203857422,
      "activations/layer18_attention_weight_min": -26.186647415161133,
      "activations/layer19_attention_weight_max": 34.185428619384766,
      "activations/layer19_attention_weight_min": -31.383493423461914,
      "activations/layer1_attention_weight_max": 17.200271606445312,
      "activations/layer1_attention_weight_min": -15.473557472229004,
      "activations/layer20_attention_weight_max": 29.26654624938965,
      "activations/layer20_attention_weight_min": -24.8872127532959,
      "activations/layer21_attention_weight_max": 30.33696746826172,
      "activations/layer21_attention_weight_min": -23.924673080444336,
      "activations/layer22_attention_weight_max": 42.3428955078125,
      "activations/layer22_attention_weight_min": -30.45013427734375,
      "activations/layer23_attention_weight_max": 31.402175903320312,
      "activations/layer23_attention_weight_min": -26.468395233154297,
      "activations/layer2_attention_weight_max": 35.86728286743164,
      "activations/layer2_attention_weight_min": -34.190574645996094,
      "activations/layer3_attention_weight_max": 111.98870086669922,
      "activations/layer3_attention_weight_min": -114.40206909179688,
      "activations/layer4_attention_weight_max": 114.1308822631836,
      "activations/layer4_attention_weight_min": -117.52324676513672,
      "activations/layer5_attention_weight_max": 51.954559326171875,
      "activations/layer5_attention_weight_min": -63.59785461425781,
      "activations/layer6_attention_weight_max": 46.49933624267578,
      "activations/layer6_attention_weight_min": -52.17518615722656,
      "activations/layer7_attention_weight_max": 101.93521118164062,
      "activations/layer7_attention_weight_min": -110.33488464355469,
      "activations/layer8_attention_weight_max": 45.98550796508789,
      "activations/layer8_attention_weight_min": -49.10014724731445,
      "activations/layer9_attention_weight_max": 36.641971588134766,
      "activations/layer9_attention_weight_min": -39.185184478759766,
      "epoch": 23.2,
      "learning_rate": 4.007575757575757e-07,
      "loss": 2.7071,
      "step": 399250
    },
    {
      "activations/layer0_attention_weight_max": 16.988101959228516,
      "activations/layer0_attention_weight_min": -15.416227340698242,
      "activations/layer10_attention_weight_max": 38.056121826171875,
      "activations/layer10_attention_weight_min": -37.726539611816406,
      "activations/layer11_attention_weight_max": 37.157989501953125,
      "activations/layer11_attention_weight_min": -36.17867660522461,
      "activations/layer12_attention_weight_max": 28.42312240600586,
      "activations/layer12_attention_weight_min": -29.91766929626465,
      "activations/layer13_attention_weight_max": 41.41460037231445,
      "activations/layer13_attention_weight_min": -38.53733444213867,
      "activations/layer14_attention_weight_max": 43.5645637512207,
      "activations/layer14_attention_weight_min": -42.86025619506836,
      "activations/layer15_attention_weight_max": 42.362430572509766,
      "activations/layer15_attention_weight_min": -41.36012268066406,
      "activations/layer16_attention_weight_max": 29.515871047973633,
      "activations/layer16_attention_weight_min": -29.82164764404297,
      "activations/layer17_attention_weight_max": 28.333486557006836,
      "activations/layer17_attention_weight_min": -26.41413688659668,
      "activations/layer18_attention_weight_max": 30.787723541259766,
      "activations/layer18_attention_weight_min": -25.988197326660156,
      "activations/layer19_attention_weight_max": 32.1607551574707,
      "activations/layer19_attention_weight_min": -31.746875762939453,
      "activations/layer1_attention_weight_max": 16.374120712280273,
      "activations/layer1_attention_weight_min": -15.35036563873291,
      "activations/layer20_attention_weight_max": 27.366119384765625,
      "activations/layer20_attention_weight_min": -23.699865341186523,
      "activations/layer21_attention_weight_max": 26.5859432220459,
      "activations/layer21_attention_weight_min": -23.542139053344727,
      "activations/layer22_attention_weight_max": 41.148521423339844,
      "activations/layer22_attention_weight_min": -28.957212448120117,
      "activations/layer23_attention_weight_max": 29.799436569213867,
      "activations/layer23_attention_weight_min": -24.903371810913086,
      "activations/layer2_attention_weight_max": 34.98695373535156,
      "activations/layer2_attention_weight_min": -34.736778259277344,
      "activations/layer3_attention_weight_max": 111.21729278564453,
      "activations/layer3_attention_weight_min": -111.76071166992188,
      "activations/layer4_attention_weight_max": 115.99366760253906,
      "activations/layer4_attention_weight_min": -119.12740325927734,
      "activations/layer5_attention_weight_max": 50.811946868896484,
      "activations/layer5_attention_weight_min": -64.01561737060547,
      "activations/layer6_attention_weight_max": 47.16727828979492,
      "activations/layer6_attention_weight_min": -48.839561462402344,
      "activations/layer7_attention_weight_max": 107.65399169921875,
      "activations/layer7_attention_weight_min": -104.3143310546875,
      "activations/layer8_attention_weight_max": 44.7418098449707,
      "activations/layer8_attention_weight_min": -46.51254653930664,
      "activations/layer9_attention_weight_max": 34.32729721069336,
      "activations/layer9_attention_weight_min": -40.18686294555664,
      "epoch": 23.2,
      "learning_rate": 3.818181818181818e-07,
      "loss": 2.6772,
      "step": 399300
    },
    {
      "activations/layer0_attention_weight_max": 17.076936721801758,
      "activations/layer0_attention_weight_min": -14.192368507385254,
      "activations/layer10_attention_weight_max": 36.01475524902344,
      "activations/layer10_attention_weight_min": -35.61457061767578,
      "activations/layer11_attention_weight_max": 36.23540496826172,
      "activations/layer11_attention_weight_min": -36.3742790222168,
      "activations/layer12_attention_weight_max": 29.884342193603516,
      "activations/layer12_attention_weight_min": -27.54846954345703,
      "activations/layer13_attention_weight_max": 42.759029388427734,
      "activations/layer13_attention_weight_min": -36.565738677978516,
      "activations/layer14_attention_weight_max": 43.42960739135742,
      "activations/layer14_attention_weight_min": -40.678585052490234,
      "activations/layer15_attention_weight_max": 44.568607330322266,
      "activations/layer15_attention_weight_min": -38.09205627441406,
      "activations/layer16_attention_weight_max": 32.501197814941406,
      "activations/layer16_attention_weight_min": -29.04294776916504,
      "activations/layer17_attention_weight_max": 34.27155303955078,
      "activations/layer17_attention_weight_min": -27.639753341674805,
      "activations/layer18_attention_weight_max": 33.8492431640625,
      "activations/layer18_attention_weight_min": -27.72625160217285,
      "activations/layer19_attention_weight_max": 35.97468566894531,
      "activations/layer19_attention_weight_min": -32.87445068359375,
      "activations/layer1_attention_weight_max": 15.359416007995605,
      "activations/layer1_attention_weight_min": -15.577229499816895,
      "activations/layer20_attention_weight_max": 30.660892486572266,
      "activations/layer20_attention_weight_min": -26.684974670410156,
      "activations/layer21_attention_weight_max": 29.295053482055664,
      "activations/layer21_attention_weight_min": -26.160093307495117,
      "activations/layer22_attention_weight_max": 46.6693115234375,
      "activations/layer22_attention_weight_min": -35.675514221191406,
      "activations/layer23_attention_weight_max": 35.669464111328125,
      "activations/layer23_attention_weight_min": -27.169902801513672,
      "activations/layer2_attention_weight_max": 35.10084915161133,
      "activations/layer2_attention_weight_min": -36.17327880859375,
      "activations/layer3_attention_weight_max": 109.40081787109375,
      "activations/layer3_attention_weight_min": -115.34236145019531,
      "activations/layer4_attention_weight_max": 114.43072509765625,
      "activations/layer4_attention_weight_min": -116.5721206665039,
      "activations/layer5_attention_weight_max": 51.48724365234375,
      "activations/layer5_attention_weight_min": -63.655433654785156,
      "activations/layer6_attention_weight_max": 47.561771392822266,
      "activations/layer6_attention_weight_min": -51.65609359741211,
      "activations/layer7_attention_weight_max": 103.76475524902344,
      "activations/layer7_attention_weight_min": -107.16783905029297,
      "activations/layer8_attention_weight_max": 44.58073043823242,
      "activations/layer8_attention_weight_min": -47.831398010253906,
      "activations/layer9_attention_weight_max": 32.67292785644531,
      "activations/layer9_attention_weight_min": -34.82633590698242,
      "epoch": 23.2,
      "learning_rate": 3.628787878787878e-07,
      "loss": 2.6925,
      "step": 399350
    },
    {
      "activations/layer0_attention_weight_max": 17.484956741333008,
      "activations/layer0_attention_weight_min": -14.043010711669922,
      "activations/layer10_attention_weight_max": 32.49956512451172,
      "activations/layer10_attention_weight_min": -36.4871826171875,
      "activations/layer11_attention_weight_max": 34.153079986572266,
      "activations/layer11_attention_weight_min": -34.424522399902344,
      "activations/layer12_attention_weight_max": 25.562789916992188,
      "activations/layer12_attention_weight_min": -26.06217384338379,
      "activations/layer13_attention_weight_max": 39.393917083740234,
      "activations/layer13_attention_weight_min": -34.819496154785156,
      "activations/layer14_attention_weight_max": 39.87135696411133,
      "activations/layer14_attention_weight_min": -36.86442184448242,
      "activations/layer15_attention_weight_max": 38.372283935546875,
      "activations/layer15_attention_weight_min": -36.989593505859375,
      "activations/layer16_attention_weight_max": 28.033000946044922,
      "activations/layer16_attention_weight_min": -30.397062301635742,
      "activations/layer17_attention_weight_max": 28.483591079711914,
      "activations/layer17_attention_weight_min": -26.486352920532227,
      "activations/layer18_attention_weight_max": 34.19498825073242,
      "activations/layer18_attention_weight_min": -26.97652816772461,
      "activations/layer19_attention_weight_max": 36.7908935546875,
      "activations/layer19_attention_weight_min": -31.931486129760742,
      "activations/layer1_attention_weight_max": 15.470154762268066,
      "activations/layer1_attention_weight_min": -15.183006286621094,
      "activations/layer20_attention_weight_max": 31.154218673706055,
      "activations/layer20_attention_weight_min": -26.62297248840332,
      "activations/layer21_attention_weight_max": 32.4801139831543,
      "activations/layer21_attention_weight_min": -23.837066650390625,
      "activations/layer22_attention_weight_max": 39.851173400878906,
      "activations/layer22_attention_weight_min": -30.58449363708496,
      "activations/layer23_attention_weight_max": 38.18256378173828,
      "activations/layer23_attention_weight_min": -26.59752655029297,
      "activations/layer2_attention_weight_max": 34.14426803588867,
      "activations/layer2_attention_weight_min": -33.70075607299805,
      "activations/layer3_attention_weight_max": 108.13240051269531,
      "activations/layer3_attention_weight_min": -108.93148803710938,
      "activations/layer4_attention_weight_max": 109.12992095947266,
      "activations/layer4_attention_weight_min": -117.58683013916016,
      "activations/layer5_attention_weight_max": 51.92119598388672,
      "activations/layer5_attention_weight_min": -62.20270538330078,
      "activations/layer6_attention_weight_max": 44.19520950317383,
      "activations/layer6_attention_weight_min": -47.82377243041992,
      "activations/layer7_attention_weight_max": 94.29552459716797,
      "activations/layer7_attention_weight_min": -100.65245056152344,
      "activations/layer8_attention_weight_max": 43.729248046875,
      "activations/layer8_attention_weight_min": -47.45164489746094,
      "activations/layer9_attention_weight_max": 31.11018943786621,
      "activations/layer9_attention_weight_min": -35.212554931640625,
      "epoch": 23.21,
      "learning_rate": 3.439393939393939e-07,
      "loss": 2.7129,
      "step": 399400
    },
    {
      "activations/layer0_attention_weight_max": 16.948535919189453,
      "activations/layer0_attention_weight_min": -14.518475532531738,
      "activations/layer10_attention_weight_max": 39.67245101928711,
      "activations/layer10_attention_weight_min": -38.68444061279297,
      "activations/layer11_attention_weight_max": 37.20275115966797,
      "activations/layer11_attention_weight_min": -37.758338928222656,
      "activations/layer12_attention_weight_max": 28.877992630004883,
      "activations/layer12_attention_weight_min": -29.904325485229492,
      "activations/layer13_attention_weight_max": 41.99714279174805,
      "activations/layer13_attention_weight_min": -39.78388214111328,
      "activations/layer14_attention_weight_max": 43.44457244873047,
      "activations/layer14_attention_weight_min": -40.39399337768555,
      "activations/layer15_attention_weight_max": 39.63288879394531,
      "activations/layer15_attention_weight_min": -37.014015197753906,
      "activations/layer16_attention_weight_max": 28.456621170043945,
      "activations/layer16_attention_weight_min": -27.64488983154297,
      "activations/layer17_attention_weight_max": 28.97342300415039,
      "activations/layer17_attention_weight_min": -26.090417861938477,
      "activations/layer18_attention_weight_max": 32.923431396484375,
      "activations/layer18_attention_weight_min": -27.01551628112793,
      "activations/layer19_attention_weight_max": 33.49391174316406,
      "activations/layer19_attention_weight_min": -31.260168075561523,
      "activations/layer1_attention_weight_max": 15.694549560546875,
      "activations/layer1_attention_weight_min": -15.935860633850098,
      "activations/layer20_attention_weight_max": 27.8553409576416,
      "activations/layer20_attention_weight_min": -24.474775314331055,
      "activations/layer21_attention_weight_max": 27.973888397216797,
      "activations/layer21_attention_weight_min": -23.216049194335938,
      "activations/layer22_attention_weight_max": 39.1927375793457,
      "activations/layer22_attention_weight_min": -30.996389389038086,
      "activations/layer23_attention_weight_max": 28.99758529663086,
      "activations/layer23_attention_weight_min": -25.793304443359375,
      "activations/layer2_attention_weight_max": 34.658382415771484,
      "activations/layer2_attention_weight_min": -34.5927619934082,
      "activations/layer3_attention_weight_max": 116.61422729492188,
      "activations/layer3_attention_weight_min": -113.05747985839844,
      "activations/layer4_attention_weight_max": 117.91423797607422,
      "activations/layer4_attention_weight_min": -120.94496154785156,
      "activations/layer5_attention_weight_max": 56.915504455566406,
      "activations/layer5_attention_weight_min": -65.57068634033203,
      "activations/layer6_attention_weight_max": 52.42875289916992,
      "activations/layer6_attention_weight_min": -50.915348052978516,
      "activations/layer7_attention_weight_max": 109.5541000366211,
      "activations/layer7_attention_weight_min": -109.62791442871094,
      "activations/layer8_attention_weight_max": 49.153472900390625,
      "activations/layer8_attention_weight_min": -51.53059768676758,
      "activations/layer9_attention_weight_max": 37.373226165771484,
      "activations/layer9_attention_weight_min": -38.21989059448242,
      "epoch": 23.21,
      "learning_rate": 3.2499999999999996e-07,
      "loss": 2.705,
      "step": 399450
    },
    {
      "activations/layer0_attention_weight_max": 16.686189651489258,
      "activations/layer0_attention_weight_min": -13.879584312438965,
      "activations/layer10_attention_weight_max": 40.17749786376953,
      "activations/layer10_attention_weight_min": -40.48626708984375,
      "activations/layer11_attention_weight_max": 39.8837890625,
      "activations/layer11_attention_weight_min": -40.09754943847656,
      "activations/layer12_attention_weight_max": 29.536462783813477,
      "activations/layer12_attention_weight_min": -28.626161575317383,
      "activations/layer13_attention_weight_max": 43.48392105102539,
      "activations/layer13_attention_weight_min": -39.85810470581055,
      "activations/layer14_attention_weight_max": 47.294471740722656,
      "activations/layer14_attention_weight_min": -42.30064010620117,
      "activations/layer15_attention_weight_max": 43.194801330566406,
      "activations/layer15_attention_weight_min": -42.34596252441406,
      "activations/layer16_attention_weight_max": 34.14431381225586,
      "activations/layer16_attention_weight_min": -31.005565643310547,
      "activations/layer17_attention_weight_max": 34.66572570800781,
      "activations/layer17_attention_weight_min": -29.40105628967285,
      "activations/layer18_attention_weight_max": 37.56724548339844,
      "activations/layer18_attention_weight_min": -31.393827438354492,
      "activations/layer19_attention_weight_max": 42.38957214355469,
      "activations/layer19_attention_weight_min": -39.4715690612793,
      "activations/layer1_attention_weight_max": 16.28537940979004,
      "activations/layer1_attention_weight_min": -16.16838264465332,
      "activations/layer20_attention_weight_max": 34.458797454833984,
      "activations/layer20_attention_weight_min": -29.07997703552246,
      "activations/layer21_attention_weight_max": 33.56587219238281,
      "activations/layer21_attention_weight_min": -27.72866439819336,
      "activations/layer22_attention_weight_max": 44.160888671875,
      "activations/layer22_attention_weight_min": -38.69547653198242,
      "activations/layer23_attention_weight_max": 34.12796401977539,
      "activations/layer23_attention_weight_min": -29.919597625732422,
      "activations/layer2_attention_weight_max": 38.78932571411133,
      "activations/layer2_attention_weight_min": -34.82294464111328,
      "activations/layer3_attention_weight_max": 117.55679321289062,
      "activations/layer3_attention_weight_min": -109.83572387695312,
      "activations/layer4_attention_weight_max": 114.69217681884766,
      "activations/layer4_attention_weight_min": -114.2834701538086,
      "activations/layer5_attention_weight_max": 53.426551818847656,
      "activations/layer5_attention_weight_min": -65.89337158203125,
      "activations/layer6_attention_weight_max": 51.052833557128906,
      "activations/layer6_attention_weight_min": -49.576229095458984,
      "activations/layer7_attention_weight_max": 109.04884338378906,
      "activations/layer7_attention_weight_min": -113.36344909667969,
      "activations/layer8_attention_weight_max": 49.4341926574707,
      "activations/layer8_attention_weight_min": -50.6535530090332,
      "activations/layer9_attention_weight_max": 37.779579162597656,
      "activations/layer9_attention_weight_min": -40.67291259765625,
      "epoch": 23.21,
      "learning_rate": 3.06060606060606e-07,
      "loss": 2.6955,
      "step": 399500
    },
    {
      "activations/layer0_attention_weight_max": 16.577896118164062,
      "activations/layer0_attention_weight_min": -14.229347229003906,
      "activations/layer10_attention_weight_max": 36.07450866699219,
      "activations/layer10_attention_weight_min": -37.058815002441406,
      "activations/layer11_attention_weight_max": 35.981178283691406,
      "activations/layer11_attention_weight_min": -37.415496826171875,
      "activations/layer12_attention_weight_max": 28.60429573059082,
      "activations/layer12_attention_weight_min": -29.007583618164062,
      "activations/layer13_attention_weight_max": 41.84425735473633,
      "activations/layer13_attention_weight_min": -38.01775360107422,
      "activations/layer14_attention_weight_max": 44.121849060058594,
      "activations/layer14_attention_weight_min": -42.547996520996094,
      "activations/layer15_attention_weight_max": 42.13814163208008,
      "activations/layer15_attention_weight_min": -39.90775680541992,
      "activations/layer16_attention_weight_max": 32.43613815307617,
      "activations/layer16_attention_weight_min": -31.577238082885742,
      "activations/layer17_attention_weight_max": 32.60375213623047,
      "activations/layer17_attention_weight_min": -27.73927116394043,
      "activations/layer18_attention_weight_max": 34.677547454833984,
      "activations/layer18_attention_weight_min": -27.298864364624023,
      "activations/layer19_attention_weight_max": 37.348995208740234,
      "activations/layer19_attention_weight_min": -33.817626953125,
      "activations/layer1_attention_weight_max": 16.851301193237305,
      "activations/layer1_attention_weight_min": -14.973093032836914,
      "activations/layer20_attention_weight_max": 33.00530242919922,
      "activations/layer20_attention_weight_min": -26.3282527923584,
      "activations/layer21_attention_weight_max": 32.5623779296875,
      "activations/layer21_attention_weight_min": -26.368572235107422,
      "activations/layer22_attention_weight_max": 50.57179260253906,
      "activations/layer22_attention_weight_min": -34.60565948486328,
      "activations/layer23_attention_weight_max": 35.17700958251953,
      "activations/layer23_attention_weight_min": -26.78313446044922,
      "activations/layer2_attention_weight_max": 35.305912017822266,
      "activations/layer2_attention_weight_min": -33.81502914428711,
      "activations/layer3_attention_weight_max": 112.06303405761719,
      "activations/layer3_attention_weight_min": -112.90480041503906,
      "activations/layer4_attention_weight_max": 114.97682189941406,
      "activations/layer4_attention_weight_min": -113.3583984375,
      "activations/layer5_attention_weight_max": 53.6328239440918,
      "activations/layer5_attention_weight_min": -61.960289001464844,
      "activations/layer6_attention_weight_max": 47.65746307373047,
      "activations/layer6_attention_weight_min": -50.241695404052734,
      "activations/layer7_attention_weight_max": 103.92105865478516,
      "activations/layer7_attention_weight_min": -102.80341339111328,
      "activations/layer8_attention_weight_max": 43.36859130859375,
      "activations/layer8_attention_weight_min": -47.684059143066406,
      "activations/layer9_attention_weight_max": 33.75395584106445,
      "activations/layer9_attention_weight_min": -35.58323287963867,
      "epoch": 23.22,
      "learning_rate": 2.871212121212121e-07,
      "loss": 2.6907,
      "step": 399550
    },
    {
      "activations/layer0_attention_weight_max": 16.76215362548828,
      "activations/layer0_attention_weight_min": -14.581239700317383,
      "activations/layer10_attention_weight_max": 39.67721939086914,
      "activations/layer10_attention_weight_min": -39.14228057861328,
      "activations/layer11_attention_weight_max": 37.635154724121094,
      "activations/layer11_attention_weight_min": -39.22599411010742,
      "activations/layer12_attention_weight_max": 30.80817222595215,
      "activations/layer12_attention_weight_min": -29.35446548461914,
      "activations/layer13_attention_weight_max": 42.206905364990234,
      "activations/layer13_attention_weight_min": -38.595027923583984,
      "activations/layer14_attention_weight_max": 45.543460845947266,
      "activations/layer14_attention_weight_min": -40.74258041381836,
      "activations/layer15_attention_weight_max": 42.12262725830078,
      "activations/layer15_attention_weight_min": -37.7352294921875,
      "activations/layer16_attention_weight_max": 29.853158950805664,
      "activations/layer16_attention_weight_min": -29.746681213378906,
      "activations/layer17_attention_weight_max": 30.673858642578125,
      "activations/layer17_attention_weight_min": -27.2739200592041,
      "activations/layer18_attention_weight_max": 32.47312927246094,
      "activations/layer18_attention_weight_min": -26.012653350830078,
      "activations/layer19_attention_weight_max": 34.356849670410156,
      "activations/layer19_attention_weight_min": -30.8026180267334,
      "activations/layer1_attention_weight_max": 15.544509887695312,
      "activations/layer1_attention_weight_min": -14.986302375793457,
      "activations/layer20_attention_weight_max": 27.83409309387207,
      "activations/layer20_attention_weight_min": -26.364990234375,
      "activations/layer21_attention_weight_max": 26.893112182617188,
      "activations/layer21_attention_weight_min": -24.982730865478516,
      "activations/layer22_attention_weight_max": 50.21435546875,
      "activations/layer22_attention_weight_min": -30.806379318237305,
      "activations/layer23_attention_weight_max": 30.871417999267578,
      "activations/layer23_attention_weight_min": -25.92938804626465,
      "activations/layer2_attention_weight_max": 35.77291488647461,
      "activations/layer2_attention_weight_min": -35.021968841552734,
      "activations/layer3_attention_weight_max": 113.37117767333984,
      "activations/layer3_attention_weight_min": -111.99400329589844,
      "activations/layer4_attention_weight_max": 112.4668197631836,
      "activations/layer4_attention_weight_min": -119.68485260009766,
      "activations/layer5_attention_weight_max": 53.56249237060547,
      "activations/layer5_attention_weight_min": -63.15583419799805,
      "activations/layer6_attention_weight_max": 45.916873931884766,
      "activations/layer6_attention_weight_min": -53.99776840209961,
      "activations/layer7_attention_weight_max": 104.19014739990234,
      "activations/layer7_attention_weight_min": -105.9378662109375,
      "activations/layer8_attention_weight_max": 46.25428009033203,
      "activations/layer8_attention_weight_min": -48.42539596557617,
      "activations/layer9_attention_weight_max": 34.018043518066406,
      "activations/layer9_attention_weight_min": -38.887855529785156,
      "epoch": 23.22,
      "learning_rate": 2.6818181818181815e-07,
      "loss": 2.705,
      "step": 399600
    },
    {
      "activations/layer0_attention_weight_max": 16.287994384765625,
      "activations/layer0_attention_weight_min": -13.932454109191895,
      "activations/layer10_attention_weight_max": 36.62184524536133,
      "activations/layer10_attention_weight_min": -36.09952163696289,
      "activations/layer11_attention_weight_max": 37.90148162841797,
      "activations/layer11_attention_weight_min": -36.47864532470703,
      "activations/layer12_attention_weight_max": 29.746841430664062,
      "activations/layer12_attention_weight_min": -27.106075286865234,
      "activations/layer13_attention_weight_max": 44.68503189086914,
      "activations/layer13_attention_weight_min": -37.45869827270508,
      "activations/layer14_attention_weight_max": 48.298954010009766,
      "activations/layer14_attention_weight_min": -40.339988708496094,
      "activations/layer15_attention_weight_max": 49.38140869140625,
      "activations/layer15_attention_weight_min": -40.56953048706055,
      "activations/layer16_attention_weight_max": 34.356441497802734,
      "activations/layer16_attention_weight_min": -31.379297256469727,
      "activations/layer17_attention_weight_max": 31.090524673461914,
      "activations/layer17_attention_weight_min": -26.6774845123291,
      "activations/layer18_attention_weight_max": 37.849029541015625,
      "activations/layer18_attention_weight_min": -25.606082916259766,
      "activations/layer19_attention_weight_max": 40.96741485595703,
      "activations/layer19_attention_weight_min": -33.535030364990234,
      "activations/layer1_attention_weight_max": 15.939170837402344,
      "activations/layer1_attention_weight_min": -15.181118965148926,
      "activations/layer20_attention_weight_max": 33.52323913574219,
      "activations/layer20_attention_weight_min": -27.839086532592773,
      "activations/layer21_attention_weight_max": 31.58768653869629,
      "activations/layer21_attention_weight_min": -26.30290412902832,
      "activations/layer22_attention_weight_max": 49.2021598815918,
      "activations/layer22_attention_weight_min": -33.38494110107422,
      "activations/layer23_attention_weight_max": 38.15909957885742,
      "activations/layer23_attention_weight_min": -26.852527618408203,
      "activations/layer2_attention_weight_max": 34.93693542480469,
      "activations/layer2_attention_weight_min": -33.746524810791016,
      "activations/layer3_attention_weight_max": 112.1671371459961,
      "activations/layer3_attention_weight_min": -110.14130401611328,
      "activations/layer4_attention_weight_max": 115.7542495727539,
      "activations/layer4_attention_weight_min": -115.6973648071289,
      "activations/layer5_attention_weight_max": 54.13290023803711,
      "activations/layer5_attention_weight_min": -65.89341735839844,
      "activations/layer6_attention_weight_max": 46.4914436340332,
      "activations/layer6_attention_weight_min": -48.37198257446289,
      "activations/layer7_attention_weight_max": 103.2296371459961,
      "activations/layer7_attention_weight_min": -104.55352020263672,
      "activations/layer8_attention_weight_max": 44.78389358520508,
      "activations/layer8_attention_weight_min": -46.73821258544922,
      "activations/layer9_attention_weight_max": 35.25491714477539,
      "activations/layer9_attention_weight_min": -36.378448486328125,
      "epoch": 23.22,
      "learning_rate": 2.492424242424242e-07,
      "loss": 2.7086,
      "step": 399650
    },
    {
      "activations/layer0_attention_weight_max": 16.451021194458008,
      "activations/layer0_attention_weight_min": -15.358606338500977,
      "activations/layer10_attention_weight_max": 36.12154006958008,
      "activations/layer10_attention_weight_min": -37.377342224121094,
      "activations/layer11_attention_weight_max": 33.8091926574707,
      "activations/layer11_attention_weight_min": -36.4882698059082,
      "activations/layer12_attention_weight_max": 26.536056518554688,
      "activations/layer12_attention_weight_min": -27.52696418762207,
      "activations/layer13_attention_weight_max": 38.13374710083008,
      "activations/layer13_attention_weight_min": -36.5341911315918,
      "activations/layer14_attention_weight_max": 42.83666229248047,
      "activations/layer14_attention_weight_min": -38.3618049621582,
      "activations/layer15_attention_weight_max": 41.795860290527344,
      "activations/layer15_attention_weight_min": -36.48405838012695,
      "activations/layer16_attention_weight_max": 29.47612762451172,
      "activations/layer16_attention_weight_min": -28.726593017578125,
      "activations/layer17_attention_weight_max": 29.62226104736328,
      "activations/layer17_attention_weight_min": -27.0336856842041,
      "activations/layer18_attention_weight_max": 33.140296936035156,
      "activations/layer18_attention_weight_min": -28.842689514160156,
      "activations/layer19_attention_weight_max": 34.15395736694336,
      "activations/layer19_attention_weight_min": -32.725318908691406,
      "activations/layer1_attention_weight_max": 15.297809600830078,
      "activations/layer1_attention_weight_min": -15.536239624023438,
      "activations/layer20_attention_weight_max": 30.074472427368164,
      "activations/layer20_attention_weight_min": -25.71832275390625,
      "activations/layer21_attention_weight_max": 31.067867279052734,
      "activations/layer21_attention_weight_min": -26.133453369140625,
      "activations/layer22_attention_weight_max": 44.358985900878906,
      "activations/layer22_attention_weight_min": -33.93641662597656,
      "activations/layer23_attention_weight_max": 34.00005340576172,
      "activations/layer23_attention_weight_min": -25.90131187438965,
      "activations/layer2_attention_weight_max": 35.1074104309082,
      "activations/layer2_attention_weight_min": -34.01881790161133,
      "activations/layer3_attention_weight_max": 113.37577819824219,
      "activations/layer3_attention_weight_min": -109.93537139892578,
      "activations/layer4_attention_weight_max": 116.88103485107422,
      "activations/layer4_attention_weight_min": -113.19966888427734,
      "activations/layer5_attention_weight_max": 55.52679443359375,
      "activations/layer5_attention_weight_min": -61.794090270996094,
      "activations/layer6_attention_weight_max": 46.94416809082031,
      "activations/layer6_attention_weight_min": -48.47750473022461,
      "activations/layer7_attention_weight_max": 98.54422760009766,
      "activations/layer7_attention_weight_min": -101.29170227050781,
      "activations/layer8_attention_weight_max": 41.80388259887695,
      "activations/layer8_attention_weight_min": -46.61933135986328,
      "activations/layer9_attention_weight_max": 31.835569381713867,
      "activations/layer9_attention_weight_min": -35.757816314697266,
      "epoch": 23.22,
      "learning_rate": 2.3030303030303027e-07,
      "loss": 2.6979,
      "step": 399700
    },
    {
      "activations/layer0_attention_weight_max": 17.69521713256836,
      "activations/layer0_attention_weight_min": -14.955851554870605,
      "activations/layer10_attention_weight_max": 40.441349029541016,
      "activations/layer10_attention_weight_min": -41.4038200378418,
      "activations/layer11_attention_weight_max": 38.69025802612305,
      "activations/layer11_attention_weight_min": -40.816123962402344,
      "activations/layer12_attention_weight_max": 30.675695419311523,
      "activations/layer12_attention_weight_min": -31.131752014160156,
      "activations/layer13_attention_weight_max": 43.57671356201172,
      "activations/layer13_attention_weight_min": -39.57017517089844,
      "activations/layer14_attention_weight_max": 45.35652542114258,
      "activations/layer14_attention_weight_min": -41.74246597290039,
      "activations/layer15_attention_weight_max": 44.10348892211914,
      "activations/layer15_attention_weight_min": -41.03793716430664,
      "activations/layer16_attention_weight_max": 32.72261047363281,
      "activations/layer16_attention_weight_min": -30.240015029907227,
      "activations/layer17_attention_weight_max": 32.578372955322266,
      "activations/layer17_attention_weight_min": -29.538766860961914,
      "activations/layer18_attention_weight_max": 35.67668914794922,
      "activations/layer18_attention_weight_min": -29.380332946777344,
      "activations/layer19_attention_weight_max": 34.63848876953125,
      "activations/layer19_attention_weight_min": -35.04025650024414,
      "activations/layer1_attention_weight_max": 15.87962818145752,
      "activations/layer1_attention_weight_min": -16.11500358581543,
      "activations/layer20_attention_weight_max": 30.487430572509766,
      "activations/layer20_attention_weight_min": -27.101207733154297,
      "activations/layer21_attention_weight_max": 33.70984649658203,
      "activations/layer21_attention_weight_min": -28.337596893310547,
      "activations/layer22_attention_weight_max": 41.84873580932617,
      "activations/layer22_attention_weight_min": -37.35714340209961,
      "activations/layer23_attention_weight_max": 30.32757568359375,
      "activations/layer23_attention_weight_min": -27.97088623046875,
      "activations/layer2_attention_weight_max": 34.62432861328125,
      "activations/layer2_attention_weight_min": -34.356666564941406,
      "activations/layer3_attention_weight_max": 111.0131607055664,
      "activations/layer3_attention_weight_min": -111.51068115234375,
      "activations/layer4_attention_weight_max": 112.8406982421875,
      "activations/layer4_attention_weight_min": -121.76374816894531,
      "activations/layer5_attention_weight_max": 50.45145797729492,
      "activations/layer5_attention_weight_min": -68.88553619384766,
      "activations/layer6_attention_weight_max": 46.38480758666992,
      "activations/layer6_attention_weight_min": -51.181270599365234,
      "activations/layer7_attention_weight_max": 101.44789123535156,
      "activations/layer7_attention_weight_min": -111.47770690917969,
      "activations/layer8_attention_weight_max": 45.42854690551758,
      "activations/layer8_attention_weight_min": -53.61691665649414,
      "activations/layer9_attention_weight_max": 34.88658905029297,
      "activations/layer9_attention_weight_min": -39.97951889038086,
      "epoch": 23.23,
      "learning_rate": 2.1136363636363634e-07,
      "loss": 2.7045,
      "step": 399750
    },
    {
      "activations/layer0_attention_weight_max": 15.820443153381348,
      "activations/layer0_attention_weight_min": -15.159859657287598,
      "activations/layer10_attention_weight_max": 38.017608642578125,
      "activations/layer10_attention_weight_min": -37.99884796142578,
      "activations/layer11_attention_weight_max": 36.3224983215332,
      "activations/layer11_attention_weight_min": -40.51478576660156,
      "activations/layer12_attention_weight_max": 28.220914840698242,
      "activations/layer12_attention_weight_min": -28.76382064819336,
      "activations/layer13_attention_weight_max": 39.04330062866211,
      "activations/layer13_attention_weight_min": -36.79277801513672,
      "activations/layer14_attention_weight_max": 42.46694564819336,
      "activations/layer14_attention_weight_min": -39.31563949584961,
      "activations/layer15_attention_weight_max": 38.047264099121094,
      "activations/layer15_attention_weight_min": -37.318485260009766,
      "activations/layer16_attention_weight_max": 28.396331787109375,
      "activations/layer16_attention_weight_min": -29.02024269104004,
      "activations/layer17_attention_weight_max": 31.05961799621582,
      "activations/layer17_attention_weight_min": -26.482789993286133,
      "activations/layer18_attention_weight_max": 30.70814323425293,
      "activations/layer18_attention_weight_min": -25.588850021362305,
      "activations/layer19_attention_weight_max": 35.317787170410156,
      "activations/layer19_attention_weight_min": -32.258758544921875,
      "activations/layer1_attention_weight_max": 15.320393562316895,
      "activations/layer1_attention_weight_min": -15.232842445373535,
      "activations/layer20_attention_weight_max": 26.845640182495117,
      "activations/layer20_attention_weight_min": -25.026716232299805,
      "activations/layer21_attention_weight_max": 27.334228515625,
      "activations/layer21_attention_weight_min": -25.602872848510742,
      "activations/layer22_attention_weight_max": 46.093597412109375,
      "activations/layer22_attention_weight_min": -32.12040328979492,
      "activations/layer23_attention_weight_max": 34.70697784423828,
      "activations/layer23_attention_weight_min": -26.4031982421875,
      "activations/layer2_attention_weight_max": 34.44243621826172,
      "activations/layer2_attention_weight_min": -33.957733154296875,
      "activations/layer3_attention_weight_max": 109.69351196289062,
      "activations/layer3_attention_weight_min": -115.23468780517578,
      "activations/layer4_attention_weight_max": 112.40126037597656,
      "activations/layer4_attention_weight_min": -117.84346008300781,
      "activations/layer5_attention_weight_max": 51.448631286621094,
      "activations/layer5_attention_weight_min": -62.11628341674805,
      "activations/layer6_attention_weight_max": 46.30060577392578,
      "activations/layer6_attention_weight_min": -50.25205612182617,
      "activations/layer7_attention_weight_max": 102.70378112792969,
      "activations/layer7_attention_weight_min": -105.33110809326172,
      "activations/layer8_attention_weight_max": 46.988136291503906,
      "activations/layer8_attention_weight_min": -48.38081359863281,
      "activations/layer9_attention_weight_max": 36.08158493041992,
      "activations/layer9_attention_weight_min": -38.742218017578125,
      "epoch": 23.23,
      "learning_rate": 1.9242424242424241e-07,
      "loss": 2.6739,
      "step": 399800
    },
    {
      "activations/layer0_attention_weight_max": 17.096033096313477,
      "activations/layer0_attention_weight_min": -15.207586288452148,
      "activations/layer10_attention_weight_max": 36.259857177734375,
      "activations/layer10_attention_weight_min": -36.42837142944336,
      "activations/layer11_attention_weight_max": 35.54701232910156,
      "activations/layer11_attention_weight_min": -38.518394470214844,
      "activations/layer12_attention_weight_max": 26.840797424316406,
      "activations/layer12_attention_weight_min": -27.190345764160156,
      "activations/layer13_attention_weight_max": 42.16368103027344,
      "activations/layer13_attention_weight_min": -35.43084716796875,
      "activations/layer14_attention_weight_max": 44.49111557006836,
      "activations/layer14_attention_weight_min": -37.847476959228516,
      "activations/layer15_attention_weight_max": 42.6984977722168,
      "activations/layer15_attention_weight_min": -35.54099655151367,
      "activations/layer16_attention_weight_max": 32.57111740112305,
      "activations/layer16_attention_weight_min": -27.528553009033203,
      "activations/layer17_attention_weight_max": 29.213239669799805,
      "activations/layer17_attention_weight_min": -27.295364379882812,
      "activations/layer18_attention_weight_max": 35.29549789428711,
      "activations/layer18_attention_weight_min": -25.708024978637695,
      "activations/layer19_attention_weight_max": 35.14769744873047,
      "activations/layer19_attention_weight_min": -32.1644287109375,
      "activations/layer1_attention_weight_max": 16.598377227783203,
      "activations/layer1_attention_weight_min": -15.283952713012695,
      "activations/layer20_attention_weight_max": 30.403390884399414,
      "activations/layer20_attention_weight_min": -26.107744216918945,
      "activations/layer21_attention_weight_max": 29.32284927368164,
      "activations/layer21_attention_weight_min": -23.6723690032959,
      "activations/layer22_attention_weight_max": 44.39766311645508,
      "activations/layer22_attention_weight_min": -29.183748245239258,
      "activations/layer23_attention_weight_max": 32.402828216552734,
      "activations/layer23_attention_weight_min": -26.664989471435547,
      "activations/layer2_attention_weight_max": 33.90971374511719,
      "activations/layer2_attention_weight_min": -34.34327697753906,
      "activations/layer3_attention_weight_max": 109.92520904541016,
      "activations/layer3_attention_weight_min": -110.2370376586914,
      "activations/layer4_attention_weight_max": 113.2970199584961,
      "activations/layer4_attention_weight_min": -114.85292053222656,
      "activations/layer5_attention_weight_max": 54.59327697753906,
      "activations/layer5_attention_weight_min": -61.057186126708984,
      "activations/layer6_attention_weight_max": 47.40657043457031,
      "activations/layer6_attention_weight_min": -53.11821746826172,
      "activations/layer7_attention_weight_max": 99.77403259277344,
      "activations/layer7_attention_weight_min": -109.99120330810547,
      "activations/layer8_attention_weight_max": 44.82108688354492,
      "activations/layer8_attention_weight_min": -46.93088912963867,
      "activations/layer9_attention_weight_max": 32.691497802734375,
      "activations/layer9_attention_weight_min": -35.75895309448242,
      "epoch": 23.23,
      "learning_rate": 1.7348484848484846e-07,
      "loss": 2.7077,
      "step": 399850
    },
    {
      "activations/layer0_attention_weight_max": 16.53195571899414,
      "activations/layer0_attention_weight_min": -13.384764671325684,
      "activations/layer10_attention_weight_max": 38.050933837890625,
      "activations/layer10_attention_weight_min": -38.78896713256836,
      "activations/layer11_attention_weight_max": 39.76407241821289,
      "activations/layer11_attention_weight_min": -39.15584182739258,
      "activations/layer12_attention_weight_max": 29.942367553710938,
      "activations/layer12_attention_weight_min": -30.3354434967041,
      "activations/layer13_attention_weight_max": 43.96233367919922,
      "activations/layer13_attention_weight_min": -38.52973937988281,
      "activations/layer14_attention_weight_max": 46.82660675048828,
      "activations/layer14_attention_weight_min": -41.46224594116211,
      "activations/layer15_attention_weight_max": 42.26372146606445,
      "activations/layer15_attention_weight_min": -38.99523162841797,
      "activations/layer16_attention_weight_max": 29.981382369995117,
      "activations/layer16_attention_weight_min": -29.317333221435547,
      "activations/layer17_attention_weight_max": 31.63668441772461,
      "activations/layer17_attention_weight_min": -28.10833168029785,
      "activations/layer18_attention_weight_max": 33.84654235839844,
      "activations/layer18_attention_weight_min": -29.00509262084961,
      "activations/layer19_attention_weight_max": 36.26548385620117,
      "activations/layer19_attention_weight_min": -36.59531021118164,
      "activations/layer1_attention_weight_max": 16.84987449645996,
      "activations/layer1_attention_weight_min": -16.43122673034668,
      "activations/layer20_attention_weight_max": 30.68602752685547,
      "activations/layer20_attention_weight_min": -27.60750961303711,
      "activations/layer21_attention_weight_max": 32.52043533325195,
      "activations/layer21_attention_weight_min": -28.06936264038086,
      "activations/layer22_attention_weight_max": 41.806312561035156,
      "activations/layer22_attention_weight_min": -36.11711120605469,
      "activations/layer23_attention_weight_max": 32.10417175292969,
      "activations/layer23_attention_weight_min": -28.276363372802734,
      "activations/layer2_attention_weight_max": 34.56166458129883,
      "activations/layer2_attention_weight_min": -34.11819076538086,
      "activations/layer3_attention_weight_max": 109.33553314208984,
      "activations/layer3_attention_weight_min": -109.58553314208984,
      "activations/layer4_attention_weight_max": 115.9958267211914,
      "activations/layer4_attention_weight_min": -115.20819091796875,
      "activations/layer5_attention_weight_max": 51.580322265625,
      "activations/layer5_attention_weight_min": -64.60173034667969,
      "activations/layer6_attention_weight_max": 48.4533576965332,
      "activations/layer6_attention_weight_min": -51.09812545776367,
      "activations/layer7_attention_weight_max": 102.3671646118164,
      "activations/layer7_attention_weight_min": -102.74852752685547,
      "activations/layer8_attention_weight_max": 44.764137268066406,
      "activations/layer8_attention_weight_min": -47.795658111572266,
      "activations/layer9_attention_weight_max": 34.67924118041992,
      "activations/layer9_attention_weight_min": -38.86526107788086,
      "epoch": 23.24,
      "learning_rate": 1.5454545454545453e-07,
      "loss": 2.7037,
      "step": 399900
    },
    {
      "activations/layer0_attention_weight_max": 17.447240829467773,
      "activations/layer0_attention_weight_min": -13.2091646194458,
      "activations/layer10_attention_weight_max": 37.192718505859375,
      "activations/layer10_attention_weight_min": -35.98097229003906,
      "activations/layer11_attention_weight_max": 35.82313919067383,
      "activations/layer11_attention_weight_min": -37.8387336730957,
      "activations/layer12_attention_weight_max": 28.621639251708984,
      "activations/layer12_attention_weight_min": -27.784738540649414,
      "activations/layer13_attention_weight_max": 42.83967590332031,
      "activations/layer13_attention_weight_min": -36.58789825439453,
      "activations/layer14_attention_weight_max": 44.149513244628906,
      "activations/layer14_attention_weight_min": -39.20804214477539,
      "activations/layer15_attention_weight_max": 42.25602722167969,
      "activations/layer15_attention_weight_min": -38.51581954956055,
      "activations/layer16_attention_weight_max": 32.88565444946289,
      "activations/layer16_attention_weight_min": -32.760746002197266,
      "activations/layer17_attention_weight_max": 31.807016372680664,
      "activations/layer17_attention_weight_min": -27.854469299316406,
      "activations/layer18_attention_weight_max": 34.413028717041016,
      "activations/layer18_attention_weight_min": -29.091609954833984,
      "activations/layer19_attention_weight_max": 39.055908203125,
      "activations/layer19_attention_weight_min": -33.298431396484375,
      "activations/layer1_attention_weight_max": 15.943928718566895,
      "activations/layer1_attention_weight_min": -16.20858383178711,
      "activations/layer20_attention_weight_max": 33.16581726074219,
      "activations/layer20_attention_weight_min": -26.83665657043457,
      "activations/layer21_attention_weight_max": 33.427974700927734,
      "activations/layer21_attention_weight_min": -25.12442970275879,
      "activations/layer22_attention_weight_max": 43.23967742919922,
      "activations/layer22_attention_weight_min": -31.987112045288086,
      "activations/layer23_attention_weight_max": 31.858259201049805,
      "activations/layer23_attention_weight_min": -25.07122039794922,
      "activations/layer2_attention_weight_max": 35.04509353637695,
      "activations/layer2_attention_weight_min": -35.69108963012695,
      "activations/layer3_attention_weight_max": 109.95630645751953,
      "activations/layer3_attention_weight_min": -111.34839630126953,
      "activations/layer4_attention_weight_max": 115.16969299316406,
      "activations/layer4_attention_weight_min": -112.3775863647461,
      "activations/layer5_attention_weight_max": 52.89898681640625,
      "activations/layer5_attention_weight_min": -63.68274688720703,
      "activations/layer6_attention_weight_max": 46.13341522216797,
      "activations/layer6_attention_weight_min": -48.97627639770508,
      "activations/layer7_attention_weight_max": 102.62467193603516,
      "activations/layer7_attention_weight_min": -101.57942962646484,
      "activations/layer8_attention_weight_max": 44.20112991333008,
      "activations/layer8_attention_weight_min": -46.53605651855469,
      "activations/layer9_attention_weight_max": 34.01267623901367,
      "activations/layer9_attention_weight_min": -36.81992721557617,
      "epoch": 23.24,
      "learning_rate": 1.356060606060606e-07,
      "loss": 2.6786,
      "step": 399950
    },
    {
      "activations/layer0_attention_weight_max": 16.64889144897461,
      "activations/layer0_attention_weight_min": -14.306046485900879,
      "activations/layer10_attention_weight_max": 36.95690155029297,
      "activations/layer10_attention_weight_min": -37.730064392089844,
      "activations/layer11_attention_weight_max": 35.681358337402344,
      "activations/layer11_attention_weight_min": -37.34719467163086,
      "activations/layer12_attention_weight_max": 31.331859588623047,
      "activations/layer12_attention_weight_min": -30.327104568481445,
      "activations/layer13_attention_weight_max": 45.16726303100586,
      "activations/layer13_attention_weight_min": -41.181922912597656,
      "activations/layer14_attention_weight_max": 46.60438919067383,
      "activations/layer14_attention_weight_min": -41.2208366394043,
      "activations/layer15_attention_weight_max": 45.31196594238281,
      "activations/layer15_attention_weight_min": -39.74287414550781,
      "activations/layer16_attention_weight_max": 33.36923599243164,
      "activations/layer16_attention_weight_min": -31.555803298950195,
      "activations/layer17_attention_weight_max": 33.81538772583008,
      "activations/layer17_attention_weight_min": -27.363040924072266,
      "activations/layer18_attention_weight_max": 35.51460266113281,
      "activations/layer18_attention_weight_min": -29.846431732177734,
      "activations/layer19_attention_weight_max": 35.41926956176758,
      "activations/layer19_attention_weight_min": -36.23783493041992,
      "activations/layer1_attention_weight_max": 15.626394271850586,
      "activations/layer1_attention_weight_min": -15.116106986999512,
      "activations/layer20_attention_weight_max": 29.815122604370117,
      "activations/layer20_attention_weight_min": -25.926424026489258,
      "activations/layer21_attention_weight_max": 30.847015380859375,
      "activations/layer21_attention_weight_min": -24.40532875061035,
      "activations/layer22_attention_weight_max": 44.40345001220703,
      "activations/layer22_attention_weight_min": -33.49701690673828,
      "activations/layer23_attention_weight_max": 30.977460861206055,
      "activations/layer23_attention_weight_min": -27.6981201171875,
      "activations/layer2_attention_weight_max": 34.18044662475586,
      "activations/layer2_attention_weight_min": -34.28715515136719,
      "activations/layer3_attention_weight_max": 112.63790130615234,
      "activations/layer3_attention_weight_min": -109.57640075683594,
      "activations/layer4_attention_weight_max": 115.59535217285156,
      "activations/layer4_attention_weight_min": -116.99132537841797,
      "activations/layer5_attention_weight_max": 53.81422424316406,
      "activations/layer5_attention_weight_min": -63.51978302001953,
      "activations/layer6_attention_weight_max": 48.33272171020508,
      "activations/layer6_attention_weight_min": -50.912513732910156,
      "activations/layer7_attention_weight_max": 98.4361801147461,
      "activations/layer7_attention_weight_min": -104.53356170654297,
      "activations/layer8_attention_weight_max": 43.6608772277832,
      "activations/layer8_attention_weight_min": -46.76189041137695,
      "activations/layer9_attention_weight_max": 35.15102005004883,
      "activations/layer9_attention_weight_min": -38.07263946533203,
      "epoch": 23.24,
      "learning_rate": 1.1666666666666665e-07,
      "loss": 2.6892,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_loss": 2.658203125,
      "eval_runtime": 8.47,
      "eval_samples_per_second": 506.965,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_openwebtext_loss": 2.658203125,
      "eval_openwebtext_ppl": 14.2706235200274,
      "eval_openwebtext_runtime": 8.47,
      "eval_openwebtext_samples_per_second": 506.965,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_wikitext_loss": 2.890625,
      "eval_wikitext_ppl": 18.00455893510183,
      "eval_wikitext_runtime": 1.9292,
      "eval_wikitext_samples_per_second": 236.365,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_lambada_loss": 2.302734375,
      "eval_lambada_ppl": 10.001492931490674,
      "eval_lambada_runtime": 9.5496,
      "eval_lambada_samples_per_second": 509.867,
      "step": 400000
    }
  ],
  "max_steps": 400000,
  "num_train_epochs": 24,
  "total_flos": 7.665774739218156e+35,
  "trial_name": null,
  "trial_params": null
}